From c47fc6022d76aef014432608b796290528aa6627 Mon Sep 17 00:00:00 2001
From: Gautam <gautamrbharadwaj@gmail.com>
Date: Wed, 28 Sep 2016 15:14:08 +0530
Subject: [PATCH 0001/2345] Update gradiesnt_descent.py

---
 tensorflow/python/training/gradient_descent.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/training/gradient_descent.py b/tensorflow/python/training/gradient_descent.py
index 3e3f573563..e60949ca87 100644
--- a/tensorflow/python/training/gradient_descent.py
+++ b/tensorflow/python/training/gradient_descent.py
@@ -47,7 +47,7 @@ class GradientDescentOptimizer(optimizer.Optimizer):
     return training_ops.apply_gradient_descent(
         var,
         math_ops.cast(self._learning_rate_tensor, var.dtype.base_dtype),
-        grad,
+        grad.indices,
         use_locking=self._use_locking).op
 
   def _apply_sparse(self, grad, var):
-- 
GitLab


From a76066c6a31f2297bfd1ee7373b7ef5072684d5d Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Thu, 28 Jun 2018 13:48:34 +0000
Subject: [PATCH 0002/2345] Add int16 support for Pack on GPU

This fix tries to add int16 support for Pack on GPU,
so that the issue raised in 20370 could be addressed.

This fix is related to 20370.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/kernels/pack_op.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/pack_op.cc b/tensorflow/core/kernels/pack_op.cc
index 5645275cfa..18ed1ea26a 100644
--- a/tensorflow/core/kernels/pack_op.cc
+++ b/tensorflow/core/kernels/pack_op.cc
@@ -158,7 +158,8 @@ REGISTER_PACK(string);
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU);
 TF_CALL_bfloat16(REGISTER_GPU);
 TF_CALL_int64(REGISTER_GPU);
-REGISTER_GPU(bool);
+TF_CALL_int16(REGISTER_GPU);
+TF_CALL_bool(REGISTER_GPU);
 #undef REGISTER_GPU
 
 // A special GPU kernel for int32.
-- 
GitLab


From 81fefe40e1c3ad9a14d9d7d665b25d7e93fb2dfc Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Thu, 28 Jun 2018 13:49:34 +0000
Subject: [PATCH 0003/2345] Add test case for int16 support of tf.stack/Pack on
 gpu

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/kernel_tests/stack_op_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/stack_op_test.py b/tensorflow/python/kernel_tests/stack_op_test.py
index 2f27d1839b..eadbcabfd1 100644
--- a/tensorflow/python/kernel_tests/stack_op_test.py
+++ b/tensorflow/python/kernel_tests/stack_op_test.py
@@ -76,7 +76,7 @@ class StackOpTest(test.TestCase):
     np.random.seed(7)
     with self.test_session(use_gpu=True):
       for shape in (2,), (3,), (2, 3), (3, 2), (4, 3, 2):
-        for dtype in [np.bool, np.float32, np.int32, np.int64]:
+        for dtype in [np.bool, np.float32, np.int16, np.int32, np.int64]:
           data = np.random.randn(*shape).astype(dtype)
           # Stack back into a single tensorflow tensor directly using np array
           c = array_ops.stack(data)
-- 
GitLab


From 970972d242e26d0cdf635d3af646df1a519dc677 Mon Sep 17 00:00:00 2001
From: ThisIsPIRI <34787507+ThisIsPIRI@users.noreply.github.com>
Date: Mon, 6 Aug 2018 14:41:37 +0900
Subject: [PATCH 0004/2345] Clarify what happens when a new value is input to
 some methods

---
 tensorflow/contrib/training/python/training/hparam.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/training/python/training/hparam.py b/tensorflow/contrib/training/python/training/hparam.py
index 3beb7bfe30..9f5059b4b1 100644
--- a/tensorflow/contrib/training/python/training/hparam.py
+++ b/tensorflow/contrib/training/python/training/hparam.py
@@ -494,6 +494,7 @@ class HParams(object):
       value: New value of the hyperparameter.
 
     Raises:
+      KeyError: If the hyperparameter doesn't exist.
       ValueError: If there is a type mismatch.
     """
     param_type, is_list = self._hparam_types[name]
@@ -510,7 +511,7 @@ class HParams(object):
       setattr(self, name, _cast_to_type_if_compatible(name, param_type, value))
 
   def del_hparam(self, name):
-    """Removes the hyperparameter with key 'name'.
+    """Removes the hyperparameter with key 'name'. Does nothing if it isn't present.
 
     Args:
       name: Name of the hyperparameter.
@@ -532,7 +533,7 @@ class HParams(object):
       The `HParams` instance.
 
     Raises:
-      ValueError: If `values` cannot be parsed.
+      ValueError: If `values` cannot be parsed or a hyperparameter in `values` doesn't exist.
     """
     type_map = dict()
     for name, t in self._hparam_types.items():
@@ -543,7 +544,7 @@ class HParams(object):
     return self.override_from_dict(values_map)
 
   def override_from_dict(self, values_dict):
-    """Override hyperparameter values, parsing new values from a dictionary.
+    """Override existing hyperparameter values, parsing new values from a dictionary.
 
     Args:
       values_dict: Dictionary of name:value pairs.
@@ -552,6 +553,7 @@ class HParams(object):
       The `HParams` instance.
 
     Raises:
+      KeyError: If a hyperparameter in `values_dict` doesn't exist.
       ValueError: If `values_dict` cannot be parsed.
     """
     for name, value in values_dict.items():
@@ -591,7 +593,7 @@ class HParams(object):
         sort_keys=sort_keys)
 
   def parse_json(self, values_json):
-    """Override hyperparameter values, parsing new values from a json object.
+    """Override existing hyperparameter values, parsing new values from a json object.
 
     Args:
       values_json: String containing a json object of name:value pairs.
@@ -600,6 +602,7 @@ class HParams(object):
       The `HParams` instance.
 
     Raises:
+      KeyError: If a hyperparameter in `values_json` doesn't exist.
       ValueError: If `values_json` cannot be parsed.
     """
     values_map = json.loads(values_json)
-- 
GitLab


From 26368188c018cdcc1bbb80ce8205fb04305816c2 Mon Sep 17 00:00:00 2001
From: ThisIsPIRI <34787507+ThisIsPIRI@users.noreply.github.com>
Date: Mon, 6 Aug 2018 14:56:38 +0900
Subject: [PATCH 0005/2345] Add 'existing' to parse's docstring

---
 tensorflow/contrib/training/python/training/hparam.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/training/python/training/hparam.py b/tensorflow/contrib/training/python/training/hparam.py
index 9f5059b4b1..372630df81 100644
--- a/tensorflow/contrib/training/python/training/hparam.py
+++ b/tensorflow/contrib/training/python/training/hparam.py
@@ -521,7 +521,7 @@ class HParams(object):
       del self._hparam_types[name]
 
   def parse(self, values):
-    """Override hyperparameter values, parsing new values from a string.
+    """Override existing hyperparameter values, parsing new values from a string.
 
     See parse_values for more detail on the allowed format for values.
 
-- 
GitLab


From 0b12d0d1ef24d667199d097c200ab65042e4e697 Mon Sep 17 00:00:00 2001
From: kouml <key.mtrgtring@gmail.com>
Date: Wed, 15 Aug 2018 00:55:43 +0900
Subject: [PATCH 0006/2345] avoid url error exception

---
 tensorflow/python/keras/utils/data_utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/utils/data_utils.py b/tensorflow/python/keras/utils/data_utils.py
index c1ee34ae46..5c5e22f0bd 100644
--- a/tensorflow/python/keras/utils/data_utils.py
+++ b/tensorflow/python/keras/utils/data_utils.py
@@ -241,10 +241,10 @@ def get_file(fname,
     try:
       try:
         urlretrieve(origin, fpath, dl_progress)
-      except URLError as e:
-        raise Exception(error_msg.format(origin, e.errno, e.reason))
       except HTTPError as e:
         raise Exception(error_msg.format(origin, e.code, e.msg))
+      except URLError as e:
+        raise Exception(error_msg.format(origin, e.errno, e.reason))
     except (Exception, KeyboardInterrupt) as e:
       if os.path.exists(fpath):
         os.remove(fpath)
-- 
GitLab


From 6655013cf6734dbd8dfc685cd2086fcc032dd04e Mon Sep 17 00:00:00 2001
From: cclauss <cclauss@bluewin.ch>
Date: Sun, 19 Aug 2018 13:42:27 +0200
Subject: [PATCH 0007/2345] ci_build: Upgrade the Python 'six' compatibility
 module

---
 tensorflow/tools/ci_build/install/install_pip_packages.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/ci_build/install/install_pip_packages.sh b/tensorflow/tools/ci_build/install/install_pip_packages.sh
index bb316ecfc9..4943cf0ae1 100755
--- a/tensorflow/tools/ci_build/install/install_pip_packages.sh
+++ b/tensorflow/tools/ci_build/install/install_pip_packages.sh
@@ -31,8 +31,8 @@ pip2 install virtualenv
 pip3 install virtualenv
 
 # Install six.
-pip2 install --upgrade six==1.10.0
-pip3 install --upgrade six==1.10.0
+pip2 install --upgrade six==1.11.0
+pip3 install --upgrade six==1.11.0
 
 # Install absl-py.
 pip2 install --upgrade absl-py
-- 
GitLab


From bd6c11f878e1820417d1ceff1b02222178f60c16 Mon Sep 17 00:00:00 2001
From: Guozhong Zhuang <guozhong.zhuang@intel.com>
Date: Fri, 12 Oct 2018 10:10:21 -0700
Subject: [PATCH 0008/2345] Clean out MKL_ML code from batchnorm ops

---
 .../core/kernels/mkl_fused_batch_norm_op.cc   | 658 +-----------------
 1 file changed, 2 insertions(+), 656 deletions(-)

diff --git a/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc b/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
index 2ec6c8fa89..4b8c066902 100644
--- a/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
+++ b/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
@@ -20,671 +20,19 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/util/tensor_format.h"
-
-#ifndef INTEL_MKL_ML_ONLY
+#include "tensorflow/core/util/mkl_util.h"
 #include "mkldnn.hpp"
+
 using mkldnn::batch_normalization_backward;
 using mkldnn::batch_normalization_forward;
 using mkldnn::prop_kind;
 using mkldnn::stream;
 using mkldnn::use_global_stats;
 using mkldnn::use_scale_shift;
-#else
-#include "mkl_dnn.h"
-#include "mkl_dnn_types.h"
-#endif
-
-#include "tensorflow/core/util/mkl_util.h"
-// TODO(inteltf) Address comments from PR 8968.
 
 namespace tensorflow {
 using CPUDevice = Eigen::ThreadPoolDevice;
 
-#ifdef INTEL_MKL_ML_ONLY
-
-template <typename Device, typename T>
-class MklFusedBatchNormOp : public OpKernel {
- public:
-  explicit MklFusedBatchNormOp(OpKernelConstruction* context)
-      : OpKernel(context) {
-    float epsilon;
-    OP_REQUIRES_OK(context, context->GetAttr("epsilon", &epsilon));
-    epsilon_ = T(epsilon);
-    string tensor_format;
-    OP_REQUIRES_OK(context, context->GetAttr("data_format", &tensor_format));
-    OP_REQUIRES(context, FormatFromString(tensor_format, &tensor_format_),
-                errors::InvalidArgument("Invalid data format"));
-    OP_REQUIRES_OK(context, context->GetAttr("is_training", &is_training_));
-  }
-
-  void Compute(OpKernelContext* context) override {
-    MklFusedBatchNormOpContext mkl_context;
-    const Tensor& input = MklGetInput(context, 0);
-    const Tensor& scale = MklGetInput(context, 1);
-    const Tensor& shift = MklGetInput(context, 2);
-    const Tensor& est_mean = MklGetInput(context, 3);
-    const Tensor& est_variance = MklGetInput(context, 4);
-
-    GetMklShape(context, 0, &(mkl_context.mkl_shape_input_shape));
-    bool input_in_mkl_format = mkl_context.mkl_shape_input_shape.IsMklTensor();
-
-    if (!input_in_mkl_format) {
-      OP_REQUIRES(context, input.dims() == 4,
-                  errors::InvalidArgument("input must be 4-dimensional",
-                                          input.shape().DebugString()));
-    }
-    OP_REQUIRES(context, scale.dims() == 1,
-                errors::InvalidArgument("scale must be 1-dimensional",
-                                        scale.shape().DebugString()));
-    OP_REQUIRES(context, shift.dims() == 1,
-                errors::InvalidArgument("offset must be 1-dimensional",
-                                        shift.shape().DebugString()));
-    OP_REQUIRES(context, est_mean.dims() == 1,
-                errors::InvalidArgument("estimated_mean must be 1-dimensional",
-                                        est_mean.shape().DebugString()));
-
-    OP_REQUIRES(
-        context, est_variance.dims() == 1,
-        errors::InvalidArgument("estimated_variance must be 1-dimensional",
-                                est_variance.shape().DebugString()));
-
-    if (is_training_) {
-      OP_REQUIRES(context, est_mean.dim_size(0) == 0,
-                  errors::InvalidArgument("estimated_mean empty for training",
-                                          est_mean.shape().DebugString()));
-      OP_REQUIRES(context, est_variance.dim_size(0) == 0,
-                  errors::InvalidArgument(
-                      "estimated_variance must be empty for training",
-                      est_variance.shape().DebugString()));
-    }
-
-    unsigned int flag_batch_norm =
-        is_training_ ? dnnUseScaleShift
-                     : (dnnUseInputMeanVariance | dnnUseScaleShift);
-
-    mkl_context.MklExtractParams(context, tensor_format_);
-
-    // Create layout only for input data as it is used in Op primitive.
-    mkl_context.MklCreateInputLayout(context);
-
-    // Create Op primitive.
-    CHECK_EQ(dnnBatchNormalizationCreateForward_v2_F32(
-                 &(mkl_context.mkl_prim_batchnorm), nullptr,
-                 mkl_context.mkl_lt_input, static_cast<float>(epsilon_),
-                 flag_batch_norm),
-             E_SUCCESS);
-
-    // Temporary tensors with buffers for the context inputs, if
-    // conversion to MKL-Op specific layouts are required. It is assumed here
-    // that TF's 1D tensors (scale, shift, est_mean, and est_variance) won't
-    // require any conversion.
-    // Since scale-shift is combined in MKL, a buffer is required.
-    Tensor mkl_tmp_input_buf_tensor, mkl_tmp_scale_shift_buf_tensor;
-    mkl_context.MklPrepareContextInputs(context, &mkl_tmp_input_buf_tensor,
-                                        &mkl_tmp_scale_shift_buf_tensor);
-
-    // Output data in MKL layout
-    Tensor* output = nullptr;
-    TensorShape tf_shape_output;
-    MklShape mkl_shape_output;
-    mkl_shape_output.SetMklTensor(true);
-    mkl_shape_output.SetMklLayout(mkl_context.mkl_prim_batchnorm,
-                                  dnnResourceDst);
-    mkl_shape_output.SetTfLayout(mkl_context.mkl_params.in_dim,
-                                 mkl_context.mkl_params.in_sizes,
-                                 mkl_context.mkl_params.in_strides);
-    mkl_shape_output.SetTfDimOrder(mkl_context.mkl_params.in_dim,
-                                   tensor_format_);
-    tf_shape_output.AddDim(dnnLayoutGetMemorySize_F32(static_cast<dnnLayout_t>(
-                               mkl_shape_output.GetMklLayout())) /
-                           sizeof(T));
-    AllocateOutputSetMklShape(context, 0, &output, tf_shape_output,
-                              mkl_shape_output);
-    mkl_context.mkl_res_batchnorm[dnnResourceDst] =
-        static_cast<void*>(output->flat<T>().data());
-
-    // Batch mean in TF layout
-    Tensor* batch_mean = nullptr;
-    MklShape mkl_shape_batch_mean;
-    mkl_shape_batch_mean.SetMklTensor(false);
-    AllocateOutputSetMklShape(context, 1, &batch_mean, scale.shape(),
-                              mkl_shape_batch_mean);
-    // Batch variance in TF layout
-    Tensor* batch_variance = nullptr;
-    MklShape mkl_shape_batch_variance;
-    mkl_shape_batch_variance.SetMklTensor(false);
-    AllocateOutputSetMklShape(context, 2, &batch_variance, scale.shape(),
-                              mkl_shape_batch_variance);
-    // If training mode, set dnnResourceMean and dnnResourceVariance to
-    // output tensors for batch mean and variance.
-    // Otherwise, set dnnResourceMean and dnnResourceVariance to
-    // estimated mean and variance.
-    if (is_training_)
-      mkl_context.MklSetMeanVariance(*batch_mean, *batch_variance);
-    else
-      mkl_context.MklSetMeanVariance(est_mean, est_variance);
-
-    // Now that all resources are set, it is ready for dnnExecute
-    CHECK_EQ(dnnExecute_F32(mkl_context.mkl_prim_batchnorm,
-                            mkl_context.mkl_res_batchnorm),
-             E_SUCCESS);
-
-    // Mean and variance (without Bessel's correction) saved for backward
-    // computation to serve as pre-computed mean and variance.
-    Tensor* saved_mean = nullptr;
-    MklShape mkl_shape_saved_mean;
-    mkl_shape_saved_mean.SetMklTensor(false);
-    AllocateOutputSetMklShape(context, 3, &saved_mean, scale.shape(),
-                              mkl_shape_saved_mean);
-    std::memcpy(
-        reinterpret_cast<char*>(saved_mean->flat<float>().data()),
-        reinterpret_cast<char*>(mkl_context.mkl_res_batchnorm[dnnResourceMean]),
-        scale.NumElements() * sizeof(float));
-    Tensor* saved_variance = nullptr;
-    MklShape mkl_shape_saved_variance;
-    mkl_shape_saved_variance.SetMklTensor(false);
-    AllocateOutputSetMklShape(context, 4, &saved_variance, scale.shape(),
-                              mkl_shape_saved_variance);
-    std::memcpy(reinterpret_cast<char*>(saved_variance->flat<float>().data()),
-                reinterpret_cast<char*>(
-                    mkl_context.mkl_res_batchnorm[dnnResourceVariance]),
-                scale.NumElements() * sizeof(float));
-
-    // Bessel's correction on variance, if training mode is on
-    if (is_training_) {
-      float* p_var = static_cast<float*>(batch_variance->flat<T>().data());
-      auto depth = mkl_context.mkl_params.depth;
-      size_t orig_size = mkl_context.mkl_params.in_sizes[0] *
-                         mkl_context.mkl_params.in_sizes[1] *
-                         mkl_context.mkl_params.in_sizes[3];
-      size_t adjust_size = orig_size - 1;
-      float adjust_factor = (static_cast<float>(orig_size)) / adjust_size;
-      for (int i = 0; i < depth; i++) p_var[i] = adjust_factor * p_var[i];
-    }
-
-    mkl_context.MklCleanup();
-  }
-
- private:
-  T epsilon_;
-  TensorFormat tensor_format_;
-  bool is_training_;
-
-  // Structure containing all info for MklOp
-  typedef struct {
-    // Parameters used for input and output layouts
-    struct MklBatchNormParams {
-      // BatchNormOp src and
-      size_t in_dim;
-      size_t in_sizes[4];
-      size_t in_strides[4];
-      size_t depth;  // Batch normalization is done for per channel.
-    } mkl_params;
-
-    MklShape mkl_shape_input_shape;
-
-    // MKL primitive and resources for BatchNormOp
-    dnnPrimitive_t mkl_prim_batchnorm = nullptr;
-    void* mkl_res_batchnorm[dnnResourceNumber];
-
-    // MKL layouts for inputs in the context
-    dnnLayout_t mkl_lt_input = nullptr;
-
-    void MklCleanup() {
-      bool input_in_mkl_format = mkl_shape_input_shape.IsMklTensor();
-      if (!input_in_mkl_format) dnnLayoutDelete_F32(mkl_lt_input);
-      if (mkl_prim_batchnorm != nullptr) dnnDelete_F32(mkl_prim_batchnorm);
-    }
-
-    void MklExtractParams(OpKernelContext* context,
-                          const TensorFormat& tensor_format) {
-      const Tensor& input = MklGetInput(context, 0);
-      bool input_in_mkl_format = mkl_shape_input_shape.IsMklTensor();
-      mkl_params.in_dim = input_in_mkl_format
-                              ? mkl_shape_input_shape.GetDimension()
-                              : input.dims();
-      mkl_params.in_sizes[0] = static_cast<size_t>(
-          input_in_mkl_format ? mkl_shape_input_shape.GetSizes()[0]
-                              : GetTensorDim(input, tensor_format, 'W'));
-      mkl_params.in_sizes[1] = static_cast<size_t>(
-          input_in_mkl_format ? mkl_shape_input_shape.GetSizes()[1]
-                              : GetTensorDim(input, tensor_format, 'H'));
-      mkl_params.in_sizes[2] = static_cast<size_t>(
-          input_in_mkl_format ? mkl_shape_input_shape.GetSizes()[2]
-                              : GetTensorDim(input, tensor_format, 'C'));
-      mkl_params.in_sizes[3] = static_cast<size_t>(
-          input_in_mkl_format ? mkl_shape_input_shape.GetSizes()[3]
-                              : GetTensorDim(input, tensor_format, 'N'));
-      mkl_params.depth = mkl_params.in_sizes[2];
-      GetStridesFromSizes(tensor_format, mkl_params.in_strides,
-                          mkl_params.in_sizes);
-    }
-
-    void MklCreateInputLayout(OpKernelContext* context) {
-      const Tensor& input = MklGetInput(context, 0);
-      bool input_in_mkl_format = mkl_shape_input_shape.IsMklTensor();
-      if (input_in_mkl_format) {
-        mkl_lt_input =
-            static_cast<dnnLayout_t>(mkl_shape_input_shape.GetCurLayout());
-      } else {
-        CHECK_EQ(
-            dnnLayoutCreate_F32(&mkl_lt_input, mkl_params.in_dim,
-                                mkl_params.in_sizes, mkl_params.in_strides),
-            E_SUCCESS);
-      }
-    }
-    void MklPrepareContextInputs(OpKernelContext* context,
-                                 Tensor* mkl_tmp_input_buf_tensor,
-                                 Tensor* mkl_tmp_scale_shift_buf_tensor) {
-      bool mkl_convert_input;
-      dnnPrimitive_t mkl_prim_convert_input = nullptr;
-      dnnLayout_t mkl_lt_internal_input = nullptr;
-      void* mkl_buf_converted_input = nullptr;
-      // Compare with internal layouts and convert if needed
-      const Tensor& input = MklGetInput(context, 0);
-      void* mkl_buf_input =
-          const_cast<void*>(static_cast<const void*>(input.flat<T>().data()));
-      CHECK_EQ(dnnLayoutCreateFromPrimitive_F32(
-                   &mkl_lt_internal_input, mkl_prim_batchnorm, dnnResourceSrc),
-               E_SUCCESS);
-      mkl_convert_input =
-          !dnnLayoutCompare_F32(mkl_lt_internal_input, mkl_lt_input);
-      if (mkl_convert_input) {
-        CHECK_EQ(dnnConversionCreate_F32(&mkl_prim_convert_input, mkl_lt_input,
-                                         mkl_lt_internal_input),
-                 E_SUCCESS);
-        AllocTmpBuffer(context, mkl_tmp_input_buf_tensor, mkl_lt_internal_input,
-                       &mkl_buf_converted_input);
-        CHECK_EQ(dnnConversionExecute_F32(mkl_prim_convert_input, mkl_buf_input,
-                                          mkl_buf_converted_input),
-                 E_SUCCESS);
-        dnnDelete_F32(mkl_prim_convert_input);
-      }
-      dnnLayoutDelete_F32(mkl_lt_internal_input);
-      mkl_res_batchnorm[dnnResourceSrc] =
-          (mkl_convert_input) ? mkl_buf_converted_input : mkl_buf_input;
-
-      // scale-shift layout is created from primitive. So no conversion
-      // is needed, however, a buffer has to be allocated.
-      dnnLayout_t mkl_lt_scale_shift = nullptr;
-      void* mkl_buf_scale_shift = nullptr;
-      CHECK_EQ(
-          dnnLayoutCreateFromPrimitive_F32(
-              &mkl_lt_scale_shift, mkl_prim_batchnorm, dnnResourceScaleShift),
-          E_SUCCESS);
-      AllocTmpBuffer(context, mkl_tmp_scale_shift_buf_tensor,
-                     mkl_lt_scale_shift, &mkl_buf_scale_shift);
-      // Fill the scale-shift buffer with data, presumably buffer is 2D array
-      const Tensor& scale = MklGetInput(context, 1);
-      const Tensor& shift = MklGetInput(context, 2);
-      float* buf_scale_shift = static_cast<float*>(mkl_buf_scale_shift);
-      float* buf_scale = const_cast<float*>(
-          static_cast<const float*>(scale.flat<float>().data()));
-      float* buf_shift = const_cast<float*>(
-          static_cast<const float*>(shift.flat<float>().data()));
-      auto depth = mkl_params.depth;
-      for (int i = 0; i < depth; i++) {
-        buf_scale_shift[i] = buf_scale[i];
-        buf_scale_shift[i + depth] = buf_shift[i];
-      }
-      mkl_res_batchnorm[dnnResourceScaleShift] = mkl_buf_scale_shift;
-    }
-
-    inline void MklSetMeanVariance(const Tensor& mean, const Tensor& variance) {
-      mkl_res_batchnorm[dnnResourceMean] = const_cast<void*>(
-          static_cast<const void*>(mean.flat<float>().data()));
-      mkl_res_batchnorm[dnnResourceVariance] = const_cast<void*>(
-          static_cast<const void*>(variance.flat<float>().data()));
-    }
-  } MklFusedBatchNormOpContext;
-};
-
-template <typename Device, typename T>
-class MklFusedBatchNormGradOp : public OpKernel {
- public:
-  explicit MklFusedBatchNormGradOp(OpKernelConstruction* context)
-      : OpKernel(context) {
-    float epsilon;
-    OP_REQUIRES_OK(context, context->GetAttr("epsilon", &epsilon));
-    epsilon_ = T(epsilon);
-    string tensor_format;
-    OP_REQUIRES_OK(context, context->GetAttr("data_format", &tensor_format));
-    OP_REQUIRES(context, FormatFromString(tensor_format, &tensor_format_),
-                errors::InvalidArgument("Invalid data format"));
-  }
-
-  void Compute(OpKernelContext* context) override {
-    MklFusedBatchNormGradOpContext mkl_context;
-
-    const Tensor& out_backprop = MklGetInput(context, 0);
-    const Tensor& input = MklGetInput(context, 1);
-    const Tensor& scale = MklGetInput(context, 2);
-    const Tensor& saved_mean = MklGetInput(context, 3);
-    const Tensor& saved_var = MklGetInput(context, 4);
-
-    // Here scale, mean, and variance are 1D and considered
-    // those having same layout in MKL and TF
-    GetMklShape(context, 0, &(mkl_context.mkl_shape_out_backprop));
-    GetMklShape(context, 1, &(mkl_context.mkl_shape_input_shape));
-
-    bool input_in_mkl_format = mkl_context.mkl_shape_input_shape.IsMklTensor();
-    bool out_backprop_in_mkl_format =
-        mkl_context.mkl_shape_out_backprop.IsMklTensor();
-    if (!out_backprop_in_mkl_format) {
-      OP_REQUIRES(context, out_backprop.dims() == 4,
-                  errors::InvalidArgument("input must be 4-dimensional",
-                                          out_backprop.shape().DebugString()));
-    }
-    if (!input_in_mkl_format) {
-      OP_REQUIRES(context, input.dims() == 4,
-                  errors::InvalidArgument("input must be 4-dimensional",
-                                          input.shape().DebugString()));
-    }
-    OP_REQUIRES(context, scale.dims() == 1,
-                errors::InvalidArgument("scale must be 1-dimensional",
-                                        scale.shape().DebugString()));
-    OP_REQUIRES(context, saved_mean.dims() == 1,
-                errors::InvalidArgument("saved mean must be 1-dimensional",
-                                        saved_mean.shape().DebugString()));
-    OP_REQUIRES(context, saved_var.dims() == 1,
-                errors::InvalidArgument("saved variance must be 1-dimensional",
-                                        saved_var.shape().DebugString()));
-
-    mkl_context.MklExtractParams(context, tensor_format_);
-
-    mkl_context.MklCreateInputLayout(context);
-
-    unsigned int flag_batch_norm_grad = dnnUseScaleShift;
-
-    // Create Backward Op primitive.
-    CHECK_EQ(dnnBatchNormalizationCreateBackward_v2_F32(
-                 &(mkl_context.mkl_prim_batchnorm_bwd), nullptr,
-                 mkl_context.mkl_lt_input, static_cast<float>(epsilon_),
-                 flag_batch_norm_grad),
-             E_SUCCESS);
-
-    // Temporary tensors and their buffers if conversion is required
-    Tensor mkl_tmp_input_buf_tensor, mkl_tmp_outbackprop_buf_tensor,
-        mkl_tmp_scaleshift_buf_tensor;
-    mkl_context.MklPrepareContextInputs(context, &mkl_tmp_input_buf_tensor,
-                                        &mkl_tmp_outbackprop_buf_tensor,
-                                        &mkl_tmp_scaleshift_buf_tensor);
-
-    // Allocate tensor for grad w.r.t. input(x)
-    Tensor* in_backprop = nullptr;
-    TensorShape tf_shape_in_backprop;
-    MklShape mkl_shape_in_backprop;
-    mkl_shape_in_backprop.SetMklTensor(true);
-    mkl_shape_in_backprop.SetMklLayout(mkl_context.mkl_prim_batchnorm_bwd,
-                                       dnnResourceDiffSrc);
-    mkl_shape_in_backprop.SetTfLayout(mkl_context.mkl_params.in_dims,
-                                      mkl_context.mkl_params.in_sizes,
-                                      mkl_context.mkl_params.in_strides);
-    mkl_shape_in_backprop.SetTfDimOrder(mkl_context.mkl_params.in_dims,
-                                        tensor_format_);
-    tf_shape_in_backprop.AddDim(
-        dnnLayoutGetMemorySize_F32(
-            static_cast<dnnLayout_t>(mkl_shape_in_backprop.GetMklLayout())) /
-        sizeof(T));
-    AllocateOutputSetMklShape(context, 0, &in_backprop, tf_shape_in_backprop,
-                              mkl_shape_in_backprop);
-    mkl_context.mkl_res_batchnorm_bwd[dnnResourceDiffSrc] =
-        static_cast<void*>(in_backprop->flat<T>().data());
-
-    // grad_scale and grad_shift are combined together in MKL
-    // So create a single temporary buffer for those.
-    // Also set dnnResourceDiffScaleShift to the temporary buffer
-    Tensor mkl_tmp_grad_scale_shift_buf_tensor;
-    mkl_context.MklPrepareGradScaleShift(context,
-                                         &mkl_tmp_grad_scale_shift_buf_tensor);
-
-    // All dnn resources are set now, ready to execute
-    CHECK_EQ(dnnExecute_F32(mkl_context.mkl_prim_batchnorm_bwd,
-                            mkl_context.mkl_res_batchnorm_bwd),
-             E_SUCCESS);
-
-    // Now separate out scale and shift grad and copy to individual tensors
-    const TensorShape& tf_shape_scale_shift = scale.shape();
-    // Allocate tensor for grad w.r.t. scale (beta)
-    Tensor* scale_backprop = nullptr;
-    MklShape mkl_shape_scale_backprop;
-    AllocateOutputSetMklShape(context, 1, &scale_backprop, tf_shape_scale_shift,
-                              mkl_shape_scale_backprop);
-
-    // Allocate tensor for grad w.r.t. shift(gamma)
-    Tensor* shift_backprop = nullptr;
-    MklShape mkl_shape_shift_backprop;
-    AllocateOutputSetMklShape(context, 2, &shift_backprop, tf_shape_scale_shift,
-                              mkl_shape_shift_backprop);
-
-    // copy scale and shift grads to tensors
-    float* mkl_buf_scale_shift = const_cast<float*>(static_cast<const float*>(
-        mkl_tmp_grad_scale_shift_buf_tensor.flat<T>().data()));
-    float* tf_buf_scale = const_cast<float*>(
-        static_cast<const float*>(scale_backprop->flat<T>().data()));
-    float* tf_buf_shift = const_cast<float*>(
-        static_cast<const float*>(shift_backprop->flat<T>().data()));
-    auto depth = mkl_context.mkl_params.depth;
-    for (int i = 0; i < depth; i++) {
-      tf_buf_scale[i] = mkl_buf_scale_shift[i];
-      tf_buf_shift[i] = mkl_buf_scale_shift[i + depth];
-    }
-
-    // Two placeholders for estimated_mean and estimated_variance, which are
-    // used for inference and thus not needed here for gradient computation.
-    Tensor* placeholder_1 = nullptr;
-    MklShape mkl_shape_placeholder_1;
-    AllocateOutputSetMklShape(context, 3, &placeholder_1, TensorShape({}),
-                              mkl_shape_placeholder_1);
-    Tensor* placeholder_2 = nullptr;
-    MklShape mkl_shape_placeholder_2;
-    AllocateOutputSetMklShape(context, 4, &placeholder_2, TensorShape({}),
-                              mkl_shape_placeholder_2);
-
-    mkl_context.MklCleanup();
-  }
-
- private:
-  T epsilon_;
-  TensorFormat tensor_format_;
-
-  // Structure containing all info for MklOp
-  typedef struct {
-    // Parameters used for input and output layouts
-    struct MklBatchNormParams {
-      // BatchNormOp src and
-      size_t in_dims;
-      size_t in_sizes[4];
-      size_t in_strides[4];
-      size_t depth;  // Batch normalization is done for per channel.
-    } mkl_params;
-
-    MklShape mkl_shape_out_backprop;
-    MklShape mkl_shape_input_shape;
-
-    // MKL primitive and resources for BatchNormOp
-    dnnPrimitive_t mkl_prim_batchnorm_bwd = nullptr;
-    void* mkl_res_batchnorm_bwd[dnnResourceNumber];
-
-    // MKL layouts for inputs in the context
-    dnnLayout_t mkl_lt_out_backprop = nullptr;
-    dnnLayout_t mkl_lt_input = nullptr;
-
-    void MklCleanup() {
-      bool input_in_mkl_format = mkl_shape_input_shape.IsMklTensor();
-      bool out_backprop_in_mkl_format = mkl_shape_out_backprop.IsMklTensor();
-      if (!input_in_mkl_format) dnnLayoutDelete_F32(mkl_lt_input);
-      if (!out_backprop_in_mkl_format) dnnLayoutDelete_F32(mkl_lt_out_backprop);
-
-      dnnDelete_F32(mkl_prim_batchnorm_bwd);
-    }
-
-    void MklExtractParams(OpKernelContext* context,
-                          const TensorFormat& tensor_format) {
-      const Tensor& input = MklGetInput(context, 1);
-      bool input_in_mkl_format = mkl_shape_input_shape.IsMklTensor();
-      mkl_params.in_dims = input_in_mkl_format
-                               ? mkl_shape_input_shape.GetDimension()
-                               : input.dims();
-      mkl_params.in_sizes[0] = static_cast<size_t>(
-          input_in_mkl_format ? mkl_shape_input_shape.GetSizes()[0]
-                              : GetTensorDim(input, tensor_format, 'W'));
-      mkl_params.in_sizes[1] = static_cast<size_t>(
-          input_in_mkl_format ? mkl_shape_input_shape.GetSizes()[1]
-                              : GetTensorDim(input, tensor_format, 'H'));
-      mkl_params.in_sizes[2] = static_cast<size_t>(
-          input_in_mkl_format ? mkl_shape_input_shape.GetSizes()[2]
-                              : GetTensorDim(input, tensor_format, 'C'));
-      mkl_params.in_sizes[3] = static_cast<size_t>(
-          input_in_mkl_format ? mkl_shape_input_shape.GetSizes()[3]
-                              : GetTensorDim(input, tensor_format, 'N'));
-      mkl_params.depth = mkl_params.in_sizes[2];
-      GetStridesFromSizes(tensor_format, mkl_params.in_strides,
-                          mkl_params.in_sizes);
-    }
-
-    void MklCreateInputLayout(OpKernelContext* context) {
-      const Tensor& input = MklGetInput(context, 0);
-      bool input_in_mkl_format = mkl_shape_input_shape.IsMklTensor();
-      if (input_in_mkl_format) {
-        mkl_lt_input =
-            static_cast<dnnLayout_t>(mkl_shape_input_shape.GetCurLayout());
-      } else {
-        CHECK_EQ(
-            dnnLayoutCreate_F32(&mkl_lt_input, mkl_params.in_dims,
-                                mkl_params.in_sizes, mkl_params.in_strides),
-            E_SUCCESS);
-      }
-
-      bool out_backprop_in_mkl_format = mkl_shape_out_backprop.IsMklTensor();
-      if (out_backprop_in_mkl_format) {
-        mkl_lt_out_backprop =
-            static_cast<dnnLayout_t>(mkl_shape_out_backprop.GetCurLayout());
-      } else {
-        CHECK_EQ(
-            dnnLayoutCreate_F32(&mkl_lt_out_backprop, mkl_params.in_dims,
-                                mkl_params.in_sizes, mkl_params.in_strides),
-            E_SUCCESS);
-      }
-    }
-
-    void MklPrepareContextInputs(OpKernelContext* context,
-                                 Tensor* mkl_tmp_input_buf_tensor,
-                                 Tensor* mkl_tmp_outbackprop_buf_tensor,
-                                 Tensor* mkl_tmp_scaleshift_buf_tensor) {
-      bool mkl_convert_input;
-      dnnPrimitive_t mkl_prim_convert_input = nullptr;
-      dnnLayout_t mkl_lt_internal_input = nullptr;
-      void* mkl_buf_converted_input = nullptr;
-      // Compare with internal layouts and convert if needed
-      const Tensor& input = MklGetInput(context, 1);
-      void* mkl_buf_input =
-          const_cast<void*>(static_cast<const void*>(input.flat<T>().data()));
-      CHECK_EQ(
-          dnnLayoutCreateFromPrimitive_F32(
-              &mkl_lt_internal_input, mkl_prim_batchnorm_bwd, dnnResourceSrc),
-          E_SUCCESS);
-      mkl_convert_input =
-          !dnnLayoutCompare_F32(mkl_lt_internal_input, mkl_lt_input);
-      if (mkl_convert_input) {
-        CHECK_EQ(dnnConversionCreate_F32(&mkl_prim_convert_input, mkl_lt_input,
-                                         mkl_lt_internal_input),
-                 E_SUCCESS);
-        AllocTmpBuffer(context, mkl_tmp_input_buf_tensor, mkl_lt_internal_input,
-                       &mkl_buf_converted_input);
-        CHECK_EQ(dnnConversionExecute_F32(mkl_prim_convert_input, mkl_buf_input,
-                                          mkl_buf_converted_input),
-                 E_SUCCESS);
-        dnnDelete_F32(mkl_prim_convert_input);
-      }
-      dnnLayoutDelete_F32(mkl_lt_internal_input);
-      mkl_res_batchnorm_bwd[dnnResourceSrc] =
-          (mkl_convert_input) ? mkl_buf_converted_input : mkl_buf_input;
-
-      bool mkl_convert_out_backprop;
-      dnnPrimitive_t mkl_prim_convert_out_backprop = nullptr;
-      dnnLayout_t mkl_lt_internal_out_backprop = nullptr;
-      void* mkl_buf_converted_out_backprop = nullptr;
-      // Compare with internal layouts and convert if needed
-      const Tensor& out_backprop = MklGetInput(context, 0);
-      void* mkl_buf_out_backprop = const_cast<void*>(
-          static_cast<const void*>(out_backprop.flat<T>().data()));
-      CHECK_EQ(dnnLayoutCreateFromPrimitive_F32(&mkl_lt_internal_out_backprop,
-                                                mkl_prim_batchnorm_bwd,
-                                                dnnResourceDiffDst),
-               E_SUCCESS);
-      mkl_convert_out_backprop = !dnnLayoutCompare_F32(
-          mkl_lt_internal_out_backprop, mkl_lt_out_backprop);
-      if (mkl_convert_out_backprop) {
-        CHECK_EQ(dnnConversionCreate_F32(&mkl_prim_convert_out_backprop,
-                                         mkl_lt_out_backprop,
-                                         mkl_lt_internal_out_backprop),
-                 E_SUCCESS);
-        AllocTmpBuffer(context, mkl_tmp_outbackprop_buf_tensor,
-                       mkl_lt_internal_out_backprop,
-                       &mkl_buf_converted_out_backprop);
-        CHECK_EQ(dnnConversionExecute_F32(mkl_prim_convert_out_backprop,
-                                          mkl_buf_out_backprop,
-                                          mkl_buf_converted_out_backprop),
-                 E_SUCCESS);
-        dnnDelete_F32(mkl_prim_convert_out_backprop);
-      }
-      dnnLayoutDelete_F32(mkl_lt_internal_out_backprop);
-      mkl_res_batchnorm_bwd[dnnResourceDiffDst] =
-          (mkl_convert_out_backprop) ? mkl_buf_converted_out_backprop
-                                     : mkl_buf_out_backprop;
-
-      // Set dnnResourceMean and dnnResourceVariance
-      const Tensor& saved_mean = MklGetInput(context, 3);
-      const Tensor& saved_var = MklGetInput(context, 4);
-      void* mkl_buf_saved_mean = const_cast<void*>(
-          static_cast<const void*>(saved_mean.flat<T>().data()));
-      void* mkl_buf_saved_var = const_cast<void*>(
-          static_cast<const void*>(saved_var.flat<T>().data()));
-      mkl_res_batchnorm_bwd[dnnResourceMean] = mkl_buf_saved_mean;
-      mkl_res_batchnorm_bwd[dnnResourceVariance] = mkl_buf_saved_var;
-
-      // Set dnnResourceScaleShift
-      // Note backward Op needs only current values of scale parameters,
-      // shift parameters could be garbage and won't be used
-      const Tensor& scale = MklGetInput(context, 2);
-      dnnLayout_t mkl_lt_scale_shift = nullptr;
-      void* mkl_buf_scale_shift = nullptr;
-      CHECK_EQ(dnnLayoutCreateFromPrimitive_F32(&mkl_lt_scale_shift,
-                                                mkl_prim_batchnorm_bwd,
-                                                dnnResourceScaleShift),
-               E_SUCCESS);
-      AllocTmpBuffer(context, mkl_tmp_scaleshift_buf_tensor, mkl_lt_scale_shift,
-                     &mkl_buf_scale_shift);
-      float* pscale =
-          const_cast<float*>(static_cast<const float*>(scale.flat<T>().data()));
-      float* pscale_shift = static_cast<float*>(mkl_buf_scale_shift);
-      auto depth = mkl_params.depth;
-      for (int i = 0; i < depth; i++) pscale_shift[i] = pscale[i];
-      mkl_res_batchnorm_bwd[dnnResourceScaleShift] = mkl_buf_scale_shift;
-      dnnLayoutDelete_F32(mkl_lt_scale_shift);
-    }
-
-    void MklPrepareGradScaleShift(OpKernelContext* context,
-                                  Tensor* mkl_tmp_grad_scale_shift_buf_tensor) {
-      dnnLayout_t mkl_lt_grad_scaleshift = nullptr;
-      void* mkl_buf_grad_scaleshift = nullptr;
-      CHECK_EQ(dnnLayoutCreateFromPrimitive_F32(&mkl_lt_grad_scaleshift,
-                                                mkl_prim_batchnorm_bwd,
-                                                dnnResourceDiffScaleShift),
-               E_SUCCESS);
-      AllocTmpBuffer(context, mkl_tmp_grad_scale_shift_buf_tensor,
-                     mkl_lt_grad_scaleshift, &mkl_buf_grad_scaleshift);
-      mkl_res_batchnorm_bwd[dnnResourceDiffScaleShift] =
-          mkl_buf_grad_scaleshift;
-      dnnLayoutDelete_F32(mkl_lt_grad_scaleshift);
-    }
-  } MklFusedBatchNormGradOpContext;
-};
-#endif
-
-#ifndef INTEL_MKL_ML_ONLY
-
 struct MklBatchNormFwdParams {
   memory::dims src_dims;
   int depth;
@@ -1765,8 +1113,6 @@ class MklFusedBatchNormGradOp : public OpKernel {
   memory::dims GetMeanVarianceDims() { return memory::dims({1, depth_}); }
 };
 
-#endif
-
 #define REGISTER_MKL_CPU(T)                                         \
   REGISTER_KERNEL_BUILDER(Name("_MklFusedBatchNorm")                \
                               .Device(DEVICE_CPU)                   \
-- 
GitLab


From cb9dccae23f34edb15cdbe58ad6fceab702138fb Mon Sep 17 00:00:00 2001
From: Guozhong Zhuang <guozhong.zhuang@intel.com>
Date: Fri, 12 Oct 2018 10:41:32 -0700
Subject: [PATCH 0009/2345] adjust headers inclusion order per review
 suggestion of another PR

---
 tensorflow/core/kernels/mkl_fused_batch_norm_op.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc b/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
index 4b8c066902..ff46e75a36 100644
--- a/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
+++ b/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
@@ -19,8 +19,8 @@ limitations under the License.
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
-#include "tensorflow/core/util/tensor_format.h"
 #include "tensorflow/core/util/mkl_util.h"
+#include "tensorflow/core/util/tensor_format.h"
 #include "mkldnn.hpp"
 
 using mkldnn::batch_normalization_backward;
-- 
GitLab


From bae74d26f93872374b48c60a73d189df148a6f99 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?=
 <facai.yan@gmail.com>
Date: Tue, 16 Oct 2018 09:45:45 +0800
Subject: [PATCH 0010/2345] CLN: remove reshape in conv3d, becasue bias_add has
 supported 5-dim

---
 .../python/keras/layers/convolutional.py      | 33 ++-----------------
 .../python/keras/layers/convolutional_test.py | 16 +++++++++
 2 files changed, 19 insertions(+), 30 deletions(-)

diff --git a/tensorflow/python/keras/layers/convolutional.py b/tensorflow/python/keras/layers/convolutional.py
index 58024677ee..f8afa0d430 100644
--- a/tensorflow/python/keras/layers/convolutional.py
+++ b/tensorflow/python/keras/layers/convolutional.py
@@ -199,21 +199,8 @@ class Conv(Layer):
           # nn.bias_add does not accept a 1D input tensor.
           bias = array_ops.reshape(self.bias, (1, self.filters, 1))
           outputs += bias
-        if self.rank == 2:
+        else:
           outputs = nn.bias_add(outputs, self.bias, data_format='NCHW')
-        if self.rank == 3:
-          # As of Mar 2017, direct addition is significantly slower than
-          # bias_add when computing gradients. To use bias_add, we collapse Z
-          # and Y into a single dimension to obtain a 4D input tensor.
-          outputs_shape = outputs.shape.as_list()
-          if outputs_shape[0] is None:
-            outputs_shape[0] = -1
-          outputs_4d = array_ops.reshape(outputs,
-                                         [outputs_shape[0], outputs_shape[1],
-                                          outputs_shape[2] * outputs_shape[3],
-                                          outputs_shape[4]])
-          outputs_4d = nn.bias_add(outputs_4d, self.bias, data_format='NCHW')
-          outputs = array_ops.reshape(outputs_4d, outputs_shape)
       else:
         outputs = nn.bias_add(outputs, self.bias, data_format='NHWC')
 
@@ -1127,24 +1114,10 @@ class Conv3DTranspose(Conv3D):
       outputs.set_shape(out_shape)
 
     if self.use_bias:
-      outputs_shape = outputs.shape.as_list()
-      if outputs_shape[0] is None:
-        outputs_shape[0] = -1
-      if self.data_format == 'channels_first':
-        outputs_4d = array_ops.reshape(outputs, [
-            outputs_shape[0], outputs_shape[1],
-            outputs_shape[2] * outputs_shape[3], outputs_shape[4]
-        ])
-      else:
-        outputs_4d = array_ops.reshape(outputs, [
-            outputs_shape[0], outputs_shape[1] * outputs_shape[2],
-            outputs_shape[3], outputs_shape[4]
-        ])
-      outputs_4d = nn.bias_add(
-          outputs_4d,
+      outputs = nn.bias_add(
+          outputs,
           self.bias,
           data_format=conv_utils.convert_data_format(self.data_format, ndim=4))
-      outputs = array_ops.reshape(outputs_4d, outputs_shape)
 
     if self.activation is not None:
       return self.activation(outputs)
diff --git a/tensorflow/python/keras/layers/convolutional_test.py b/tensorflow/python/keras/layers/convolutional_test.py
index 4afddbc8cc..63fb60ebaf 100644
--- a/tensorflow/python/keras/layers/convolutional_test.py
+++ b/tensorflow/python/keras/layers/convolutional_test.py
@@ -336,6 +336,14 @@ class Conv3DTransposeTest(test.TestCase):
       self.assertEqual(layer.kernel.constraint, k_constraint)
       self.assertEqual(layer.bias.constraint, b_constraint)
 
+  def test_conv3dtranspose_dynamic_shape(self):
+    with self.session(use_gpu=True):
+      # Won't raise error here.
+      layer = keras.layers.Conv3DTranspose(3, 3, data_format='channels_last')
+      layer.build((None, None, None, None, 1))
+      layer1 = keras.layers.Conv3DTranspose(3, 3, data_format='channels_first')
+      layer1.build((None, 1, None, None, None))
+
 
 class SeparableConv1DTest(test.TestCase):
 
@@ -557,6 +565,14 @@ class Conv3DTest(test.TestCase):
       self.assertEqual(layer.kernel.constraint, k_constraint)
       self.assertEqual(layer.bias.constraint, b_constraint)
 
+  def test_conv3d_dynamic_shape(self):
+    with self.session(use_gpu=True):
+      # Won't raise error here.
+      layer = keras.layers.Conv3D(3, 3, data_format='channels_last')
+      layer.build((None, None, None, None, 1))
+      layer1 = keras.layers.Conv3D(3, 3, data_format='channels_first')
+      layer1.build((None, 1, None, None, None))
+
 
 class ZeroPaddingTest(test.TestCase):
 
-- 
GitLab


From 02e52ca65a91f12b9f352ffa1d0ed86a1c6dd276 Mon Sep 17 00:00:00 2001
From: Palmer Lao <palmer.lao@gmail.com>
Date: Thu, 18 Oct 2018 16:42:15 -0700
Subject: [PATCH 0011/2345] Make shared S3 lib linkable

---
 tensorflow/core/platform/s3/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/platform/s3/BUILD b/tensorflow/core/platform/s3/BUILD
index 41184b6fd9..7bc4d80db5 100644
--- a/tensorflow/core/platform/s3/BUILD
+++ b/tensorflow/core/platform/s3/BUILD
@@ -14,7 +14,7 @@ load(
 )
 
 tf_cc_binary(
-    name = "s3_file_system.so",
+    name = "libs3_file_system_shared.so",
     srcs = [
         "aws_crypto.cc",
         "aws_crypto.h",
-- 
GitLab


From 4182fe444d0eed807403482a165c1b547d1c25ce Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Fri, 19 Oct 2018 03:16:59 +0000
Subject: [PATCH 0012/2345] Add uint8 support for Pad with GPU

This fix tries to address some of the issue raised in 17823 where
Pad with GPU does not support uint8. This fix adds the uint8 support.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/kernels/pad_op.cc        | 2 ++
 tensorflow/core/kernels/pad_op_gpu.cu.cc | 1 +
 2 files changed, 3 insertions(+)

diff --git a/tensorflow/core/kernels/pad_op.cc b/tensorflow/core/kernels/pad_op.cc
index 3b9133ed7e..691430ebaf 100644
--- a/tensorflow/core/kernels/pad_op.cc
+++ b/tensorflow/core/kernels/pad_op.cc
@@ -322,6 +322,7 @@ namespace functor {
 
 TF_CALL_GPU_ALL_TYPES(DECLARE_GPU_SPECS);
 TF_CALL_int8(DECLARE_GPU_SPECS);
+TF_CALL_uint8(DECLARE_GPU_SPECS);
 }  // namespace functor
 
 // Registration of the GPU implementations.
@@ -355,6 +356,7 @@ TF_CALL_int8(DECLARE_GPU_SPECS);
 
 TF_CALL_GPU_ALL_TYPES(REGISTER_GPU_KERNEL);
 TF_CALL_int8(REGISTER_GPU_KERNEL);
+TF_CALL_uint8(REGISTER_GPU_KERNEL);
 
 // A special GPU kernel for int32.
 // TODO(b/25387198): Also enable int32 in device memory. This kernel
diff --git a/tensorflow/core/kernels/pad_op_gpu.cu.cc b/tensorflow/core/kernels/pad_op_gpu.cu.cc
index 00ec44adc2..0cd8ef17ba 100644
--- a/tensorflow/core/kernels/pad_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/pad_op_gpu.cu.cc
@@ -41,6 +41,7 @@ typedef Eigen::GpuDevice GPUDevice;
 
 TF_CALL_GPU_ALL_TYPES(DEFINE_GPU_SPECS);
 TF_CALL_int8(DEFINE_GPU_SPECS);
+TF_CALL_uint8(DEFINE_GPU_SPECS);
 
 }  // namespace tensorflow
 
-- 
GitLab


From e18fba2d7eb0ba9e01ff998ee413932ead712eb7 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Fri, 19 Oct 2018 03:18:06 +0000
Subject: [PATCH 0013/2345] Adds test case of uint8 support for Pad with GPU

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/kernel_tests/pad_op_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/pad_op_test.py b/tensorflow/python/kernel_tests/pad_op_test.py
index fc302c4141..e875bb9d65 100644
--- a/tensorflow/python/kernel_tests/pad_op_test.py
+++ b/tensorflow/python/kernel_tests/pad_op_test.py
@@ -215,7 +215,7 @@ class PadOpTest(test.TestCase):
   def testIntTypes(self):
     # TODO(touts): Figure out why the padding tests do not work on GPU
     # for int types and rank > 2.
-    for t in [np.int8, np.int32, np.int64]:
+    for t in [np.int8, np.uint8, np.int32, np.int64]:
       self._testAll(
           np.random.randint(-100, 100, (4, 4, 3)).astype(t),
           [[1, 0], [2, 3], [0, 2]], 0)
-- 
GitLab


From 5ec7c8c199368cf950cb31113c6820d872f45de5 Mon Sep 17 00:00:00 2001
From: frreiss <frreiss@us.ibm.com>
Date: Thu, 18 Oct 2018 19:50:42 -0700
Subject: [PATCH 0014/2345] Document undocumented ops

Fix missing carriage return
---
 tensorflow/python/ops/check_ops.py | 31 ++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/tensorflow/python/ops/check_ops.py b/tensorflow/python/ops/check_ops.py
index 40b111ea0c..b2511f1bfe 100644
--- a/tensorflow/python/ops/check_ops.py
+++ b/tensorflow/python/ops/check_ops.py
@@ -1136,6 +1136,25 @@ def _get_diff_for_monotonic_comparison(x):
     v1=['debugging.is_numeric_tensor', 'is_numeric_tensor'])
 @deprecation.deprecated_endpoints('is_numeric_tensor')
 def is_numeric_tensor(tensor):
+  """Returns `True` if the elements of `tensor` are numbers.
+  
+  Specifically, returns `True` if the dtype of `tensor` is one of the following:
+
+  * `tf.float32`
+  * `tf.float64`
+  * `tf.int8`
+  * `tf.int16`
+  * `tf.int32`
+  * `tf.int64`
+  * `tf.uint8`
+  * `tf.qint8`
+  * `tf.qint32`
+  * `tf.quint8`
+  * `tf.complex64`
+
+  Returns `False` if `tensor` is of a non-numeric type or if `tensor` is not 
+  a `tf.Tensor` object.
+  """
   return isinstance(tensor, ops.Tensor) and tensor.dtype in NUMERIC_TYPES
 
 
@@ -1283,6 +1302,18 @@ def assert_same_float_dtype(tensors=None, dtype=None):
     'debugging.assert_scalar', v1=['debugging.assert_scalar', 'assert_scalar'])
 @deprecation.deprecated_endpoints('assert_scalar')
 def assert_scalar(tensor, name=None):
+  """Statically checks whether a tensor is zero-dimensional.
+
+  Args:
+    tensor: Value to test.
+    name: A name for this operation (optional).  Defaults to "assert_scalar".
+
+  Raises:
+    ValueError: If `tensor`'s shape has more than zero dimensions.
+  
+  Returns:
+    The input `tensor`, possibly converted to a `tf.Tensor`
+  """
   with ops.name_scope(name, 'assert_scalar', [tensor]) as name_scope:
     tensor = ops.convert_to_tensor(tensor, name=name_scope)
     shape = tensor.get_shape()
-- 
GitLab


From d31e6ad81cf0c88e08a00a6ab16c2ac2419e7b10 Mon Sep 17 00:00:00 2001
From: Matt Conley <mconley@nvidia.com>
Date: Wed, 24 Oct 2018 13:57:20 -0700
Subject: [PATCH 0015/2345] Update Cast Op test for ARM behavior

---
 tensorflow/python/kernel_tests/cast_op_test.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/kernel_tests/cast_op_test.py b/tensorflow/python/kernel_tests/cast_op_test.py
index a5dff5df62..90c2bd0294 100644
--- a/tensorflow/python/kernel_tests/cast_op_test.py
+++ b/tensorflow/python/kernel_tests/cast_op_test.py
@@ -151,7 +151,7 @@ class CastOpTest(test.TestCase):
       # np.float64("np.inf").astype(np.int32) is negative on x86 but positive on ppc64le
       # Numpy link to relevant discussion - https://github.com/numpy/numpy/issues/9040
       # Tensorflow link to relevant discussion - https://github.com/tensorflow/tensorflow/issues/9360
-      if platform.machine() == "ppc64le":
+      if platform.machine() == "ppc64le" or platform.machine() == "aarch64":
         self._compare(-np.inf, np.int32, i4.min, False)
         self._compare(-np.inf, np.int64, i8.min, False)
       else:
@@ -163,8 +163,13 @@ class CastOpTest(test.TestCase):
     self._compare(-np.inf, np.int64, i8.min, False)
     self.assertAllEqual(np.isnan(self._cast(np.nan, np.float32, False)), True)
     self.assertAllEqual(np.isnan(self._cast(np.nan, np.float64, False)), True)
-    self._compare(np.nan, np.int32, i4.min, False)
-    self._compare(np.nan, np.int64, i8.min, False)
+    # np.float64(np.nan).astype(np.int32) is 0 on ARM
+    if platform.machine() == "aarch64":
+      self._compare(np.nan, np.int32, 0, False)
+      self._compare(np.nan, np.int64, 0, False)
+    else:
+      self._compare(np.nan, np.int32, i4.min, False)
+      self._compare(np.nan, np.int64, i8.min, False)
 
     self._compare(np.inf, np.float32, np.inf, True)
     self._compare(np.inf, np.float64, np.inf, True)
-- 
GitLab


From 65344d0a7ebd21c71b3f3ed7cb091541504b659a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?=
 <facai.yan@gmail.com>
Date: Thu, 1 Nov 2018 09:36:09 +0800
Subject: [PATCH 0016/2345] TST: use testing_utils.layer_test instead

---
 .../python/keras/layers/convolutional_test.py | 28 +++++++++++++------
 1 file changed, 20 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/keras/layers/convolutional_test.py b/tensorflow/python/keras/layers/convolutional_test.py
index 63fb60ebaf..7d8051e596 100644
--- a/tensorflow/python/keras/layers/convolutional_test.py
+++ b/tensorflow/python/keras/layers/convolutional_test.py
@@ -339,10 +339,16 @@ class Conv3DTransposeTest(test.TestCase):
   def test_conv3dtranspose_dynamic_shape(self):
     with self.session(use_gpu=True):
       # Won't raise error here.
-      layer = keras.layers.Conv3DTranspose(3, 3, data_format='channels_last')
-      layer.build((None, None, None, None, 1))
-      layer1 = keras.layers.Conv3DTranspose(3, 3, data_format='channels_first')
-      layer1.build((None, 1, None, None, None))
+      testing_utils.layer_test(
+          keras.layers.Conv3DTranspose,
+          kwargs={'data_format': 'channels_first',
+                  'filters': 3, 'kernel_size': 3},
+          input_shape=(None, 1, None, None, None))
+      testing_utils.layer_test(
+          keras.layers.Conv3DTranspose,
+          kwargs={'data_format': 'channels_last',
+                  'filters': 3, 'kernel_size': 3},
+          input_shape=(None, None, None, None, 1))
 
 
 class SeparableConv1DTest(test.TestCase):
@@ -568,10 +574,16 @@ class Conv3DTest(test.TestCase):
   def test_conv3d_dynamic_shape(self):
     with self.session(use_gpu=True):
       # Won't raise error here.
-      layer = keras.layers.Conv3D(3, 3, data_format='channels_last')
-      layer.build((None, None, None, None, 1))
-      layer1 = keras.layers.Conv3D(3, 3, data_format='channels_first')
-      layer1.build((None, 1, None, None, None))
+      testing_utils.layer_test(
+          keras.layers.Conv3D,
+          kwargs={'data_format': 'channels_first',
+                  'filters': 3, 'kernel_size': 3},
+          input_shape=(None, 1, None, None, None))
+      testing_utils.layer_test(
+          keras.layers.Conv3D,
+          kwargs={'data_format': 'channels_last',
+                  'filters': 3, 'kernel_size': 3},
+          input_shape=(None, None, None, None, 1))
 
 
 class ZeroPaddingTest(test.TestCase):
-- 
GitLab


From 35f20ca9e794b455777a52bf70b5f7d79ae60455 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?=
 <facai.yan@gmail.com>
Date: Thu, 1 Nov 2018 10:11:14 +0800
Subject: [PATCH 0017/2345] TST: add input_data

---
 .../python/keras/layers/convolutional_test.py      | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/keras/layers/convolutional_test.py b/tensorflow/python/keras/layers/convolutional_test.py
index 7d8051e596..737e12a2bd 100644
--- a/tensorflow/python/keras/layers/convolutional_test.py
+++ b/tensorflow/python/keras/layers/convolutional_test.py
@@ -337,18 +337,21 @@ class Conv3DTransposeTest(test.TestCase):
       self.assertEqual(layer.bias.constraint, b_constraint)
 
   def test_conv3dtranspose_dynamic_shape(self):
+    input_data = np.random.random((1, 3, 3, 3, 3))
     with self.session(use_gpu=True):
       # Won't raise error here.
       testing_utils.layer_test(
           keras.layers.Conv3DTranspose,
           kwargs={'data_format': 'channels_first',
                   'filters': 3, 'kernel_size': 3},
-          input_shape=(None, 1, None, None, None))
+          input_shape=(None, 3, None, None, None),
+          input_data=input_data)
       testing_utils.layer_test(
           keras.layers.Conv3DTranspose,
           kwargs={'data_format': 'channels_last',
                   'filters': 3, 'kernel_size': 3},
-          input_shape=(None, None, None, None, 1))
+          input_shape=(None, None, None, None, 3),
+          input_data=input_data)
 
 
 class SeparableConv1DTest(test.TestCase):
@@ -572,18 +575,21 @@ class Conv3DTest(test.TestCase):
       self.assertEqual(layer.bias.constraint, b_constraint)
 
   def test_conv3d_dynamic_shape(self):
+    input_data = np.random.random((1, 3, 3, 3, 3))
     with self.session(use_gpu=True):
       # Won't raise error here.
       testing_utils.layer_test(
           keras.layers.Conv3D,
           kwargs={'data_format': 'channels_first',
                   'filters': 3, 'kernel_size': 3},
-          input_shape=(None, 1, None, None, None))
+          input_shape=(None, 3, None, None, None),
+          input_data=input_data)
       testing_utils.layer_test(
           keras.layers.Conv3D,
           kwargs={'data_format': 'channels_last',
                   'filters': 3, 'kernel_size': 3},
-          input_shape=(None, None, None, None, 1))
+          input_shape=(None, None, None, None, 3),
+          input_data=input_data)
 
 
 class ZeroPaddingTest(test.TestCase):
-- 
GitLab


From 8de3a6ee6c7b498e32e8c22c6631a3c0a7a4af86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?=
 <facai.yan@gmail.com>
Date: Thu, 1 Nov 2018 10:35:48 +0800
Subject: [PATCH 0018/2345] TST: data_format=first only run for gpu

---
 .../python/keras/layers/convolutional_test.py | 30 ++++++++++---------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/tensorflow/python/keras/layers/convolutional_test.py b/tensorflow/python/keras/layers/convolutional_test.py
index 737e12a2bd..ecee080937 100644
--- a/tensorflow/python/keras/layers/convolutional_test.py
+++ b/tensorflow/python/keras/layers/convolutional_test.py
@@ -337,21 +337,22 @@ class Conv3DTransposeTest(test.TestCase):
       self.assertEqual(layer.bias.constraint, b_constraint)
 
   def test_conv3dtranspose_dynamic_shape(self):
-    input_data = np.random.random((1, 3, 3, 3, 3))
+    input_data = np.random.random((1, 3, 3, 3, 3)).astype(np.float32)
     with self.session(use_gpu=True):
       # Won't raise error here.
-      testing_utils.layer_test(
-          keras.layers.Conv3DTranspose,
-          kwargs={'data_format': 'channels_first',
-                  'filters': 3, 'kernel_size': 3},
-          input_shape=(None, 3, None, None, None),
-          input_data=input_data)
       testing_utils.layer_test(
           keras.layers.Conv3DTranspose,
           kwargs={'data_format': 'channels_last',
                   'filters': 3, 'kernel_size': 3},
           input_shape=(None, None, None, None, 3),
           input_data=input_data)
+      if test.is_gpu_available(cuda_only=True):
+        testing_utils.layer_test(
+            keras.layers.Conv3DTranspose,
+            kwargs={'data_format': 'channels_first',
+                    'filters': 3, 'kernel_size': 3},
+            input_shape=(None, 3, None, None, None),
+            input_data=input_data)
 
 
 class SeparableConv1DTest(test.TestCase):
@@ -575,21 +576,22 @@ class Conv3DTest(test.TestCase):
       self.assertEqual(layer.bias.constraint, b_constraint)
 
   def test_conv3d_dynamic_shape(self):
-    input_data = np.random.random((1, 3, 3, 3, 3))
+    input_data = np.random.random((1, 3, 3, 3, 3)).astype(np.float32)
     with self.session(use_gpu=True):
       # Won't raise error here.
-      testing_utils.layer_test(
-          keras.layers.Conv3D,
-          kwargs={'data_format': 'channels_first',
-                  'filters': 3, 'kernel_size': 3},
-          input_shape=(None, 3, None, None, None),
-          input_data=input_data)
       testing_utils.layer_test(
           keras.layers.Conv3D,
           kwargs={'data_format': 'channels_last',
                   'filters': 3, 'kernel_size': 3},
           input_shape=(None, None, None, None, 3),
           input_data=input_data)
+      if test.is_gpu_available(cuda_only=True):
+        testing_utils.layer_test(
+            keras.layers.Conv3D,
+            kwargs={'data_format': 'channels_first',
+                    'filters': 3, 'kernel_size': 3},
+            input_shape=(None, 3, None, None, None),
+            input_data=input_data)
 
 
 class ZeroPaddingTest(test.TestCase):
-- 
GitLab


From c8a0c22e3f94728af57ceaed2f53603d1d3d87a9 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Thu, 28 Jun 2018 15:10:41 +0000
Subject: [PATCH 0019/2345] Add int64 for ConcatGPU

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/kernels/concat_lib_gpu.cc         | 1 +
 tensorflow/core/kernels/concat_lib_gpu_impl.cu.cc | 4 ++++
 2 files changed, 5 insertions(+)

diff --git a/tensorflow/core/kernels/concat_lib_gpu.cc b/tensorflow/core/kernels/concat_lib_gpu.cc
index 93e392d303..278ef2b2e8 100644
--- a/tensorflow/core/kernels/concat_lib_gpu.cc
+++ b/tensorflow/core/kernels/concat_lib_gpu.cc
@@ -116,6 +116,7 @@ TF_CALL_GPU_NUMBER_TYPES(REGISTER);
 TF_CALL_complex64(REGISTER);
 TF_CALL_complex128(REGISTER);
 TF_CALL_int64(REGISTER);
+TF_CALL_int16(REGISTER);
 TF_CALL_bfloat16(REGISTER);
 TF_CALL_bool(REGISTER);
 TF_CALL_uint8(REGISTER);
diff --git a/tensorflow/core/kernels/concat_lib_gpu_impl.cu.cc b/tensorflow/core/kernels/concat_lib_gpu_impl.cu.cc
index a561d918bd..752e0ed3b7 100644
--- a/tensorflow/core/kernels/concat_lib_gpu_impl.cu.cc
+++ b/tensorflow/core/kernels/concat_lib_gpu_impl.cu.cc
@@ -202,6 +202,7 @@ TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPUCONCAT32);
 TF_CALL_complex64(REGISTER_GPUCONCAT32);
 TF_CALL_complex128(REGISTER_GPUCONCAT32);
 TF_CALL_int64(REGISTER_GPUCONCAT32);
+TF_CALL_int16(REGISTER_GPUCONCAT32);
 TF_CALL_uint8(REGISTER_GPUCONCAT32);
 REGISTER_GPUCONCAT32(bfloat16);
 REGISTER_GPUCONCAT32(bool);
@@ -210,6 +211,7 @@ TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPUCONCAT64);
 TF_CALL_complex64(REGISTER_GPUCONCAT64);
 TF_CALL_complex128(REGISTER_GPUCONCAT64);
 TF_CALL_int64(REGISTER_GPUCONCAT64);
+TF_CALL_int16(REGISTER_GPUCONCAT64);
 TF_CALL_uint8(REGISTER_GPUCONCAT64);
 REGISTER_GPUCONCAT64(bfloat16);
 REGISTER_GPUCONCAT64(bool);
@@ -218,6 +220,7 @@ TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU32);
 TF_CALL_complex64(REGISTER_GPU32);
 TF_CALL_complex128(REGISTER_GPU32);
 TF_CALL_int64(REGISTER_GPU32);
+TF_CALL_int16(REGISTER_GPU32);
 TF_CALL_uint8(REGISTER_GPU32);
 REGISTER_GPU32(bfloat16);
 REGISTER_GPU32(bool);
@@ -226,6 +229,7 @@ TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU64);
 TF_CALL_complex64(REGISTER_GPU64);
 TF_CALL_complex128(REGISTER_GPU64);
 TF_CALL_int64(REGISTER_GPU64);
+TF_CALL_int16(REGISTER_GPU64);
 TF_CALL_uint8(REGISTER_GPU64);
 REGISTER_GPU64(bfloat16);
 REGISTER_GPU64(bool);
-- 
GitLab


From 42321707242771cf28deb1d577dfdd6a17e9eae9 Mon Sep 17 00:00:00 2001
From: Guozhong Zhuang <guozhong.zhuang@intel.com>
Date: Mon, 12 Nov 2018 15:24:22 -0800
Subject: [PATCH 0020/2345] fix issues related to clang format check

---
 .../core/kernels/mkl_fused_batch_norm_op.cc     | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc b/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
index ff46e75a36..685db657e2 100644
--- a/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
+++ b/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
@@ -13,15 +13,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 #ifdef INTEL_MKL
-
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "mkldnn.hpp"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/util/mkl_util.h"
 #include "tensorflow/core/util/tensor_format.h"
-#include "mkldnn.hpp"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
 using mkldnn::batch_normalization_backward;
 using mkldnn::batch_normalization_forward;
@@ -705,9 +704,9 @@ class MklFusedBatchNormOp : public OpKernel {
         std::memcpy(batch_variance_data, variance_data, depth_ * sizeof(T));
       }
     } catch (mkldnn::error& e) {
-      string error_msg = "Status: " + std::to_string(e.status) +
-                         ", message: " + string(e.message) + ", in file " +
-                         string(__FILE__) + ":" + std::to_string(__LINE__);
+      string error_msg = "Status: " + std::to_string(e.status) + ", message: " +
+                         string(e.message) + ", in file " + string(__FILE__) +
+                         ":" + std::to_string(__LINE__);
       OP_REQUIRES_OK(
           context,
           errors::Aborted("Operation received an exception:", error_msg));
@@ -1029,9 +1028,9 @@ class MklFusedBatchNormGradOp : public OpKernel {
                   reinterpret_cast<char*>(diff_weights_data + depth_),
                   depth_ * sizeof(T));
     } catch (mkldnn::error& e) {
-      string error_msg = "Status: " + std::to_string(e.status) +
-                         ", message: " + string(e.message) + ", in file " +
-                         string(__FILE__) + ":" + std::to_string(__LINE__);
+      string error_msg = "Status: " + std::to_string(e.status) + ", message: " +
+                         string(e.message) + ", in file " + string(__FILE__) +
+                         ":" + std::to_string(__LINE__);
       OP_REQUIRES_OK(
           context,
           errors::Aborted("Operation received an exception:", error_msg));
-- 
GitLab


From 898bb2ad207d2b4cee188ce8b868c76bda8f0ad2 Mon Sep 17 00:00:00 2001
From: zhaoyongke <zhaoyongke@yeah.net>
Date: Sun, 18 Nov 2018 01:00:26 +0800
Subject: [PATCH 0021/2345] Take channel_multiplier into considerations. Thanks
 to smillius, drpngx, HuiyangFei

---
 .../tools/graph_transforms/fold_batch_norms.cc      | 13 ++++++++++---
 .../tools/graph_transforms/fold_old_batch_norms.cc  | 12 ++++++++++--
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/tensorflow/tools/graph_transforms/fold_batch_norms.cc b/tensorflow/tools/graph_transforms/fold_batch_norms.cc
index cb4230dd82..2d7bb65430 100644
--- a/tensorflow/tools/graph_transforms/fold_batch_norms.cc
+++ b/tensorflow/tools/graph_transforms/fold_batch_norms.cc
@@ -73,9 +73,16 @@ Status FoldBatchNorms(const GraphDef& input_graph_def,
 
         // Make sure all the inputs really are vectors, with as many entries as
         // there are columns in the weights.
-        const int weights_cols_index = conv_node.op() == "Conv2D" ? 3 : \
-                            (conv_node.op() == "DepthwiseConv2dNative" ? 2 : 1);
-        const int64 weights_cols = weights.shape().dim_size(weights_cols_index);
+        int64 weights_cols;
+        if (conv_node.op() == "Conv2D") {
+          weights_cols = weights.shape().dim_size(3);
+        }
+        else if (conv_node.op() == "DepthwiseConv2dNative") {
+          weights_cols = weights.shape().dim_size(2) * weights.shape().dim_size(3);
+        }
+        else {
+          weights_cols = weights.shape().dim_size(1);
+        }
         if ((mul_values.shape().dims() != 1) ||
             (mul_values.shape().dim_size(0) != weights_cols)) {
           return errors::InvalidArgument(
diff --git a/tensorflow/tools/graph_transforms/fold_old_batch_norms.cc b/tensorflow/tools/graph_transforms/fold_old_batch_norms.cc
index 1a4b141d0e..413361b616 100644
--- a/tensorflow/tools/graph_transforms/fold_old_batch_norms.cc
+++ b/tensorflow/tools/graph_transforms/fold_old_batch_norms.cc
@@ -116,8 +116,16 @@ Status FuseScaleOffsetToConvWeights(const std::vector<float>& scale_values,
   CHECK_EQ("Const", weights_node.op());
 
   Tensor weights = GetNodeTensorAttr(weights_node, "value");
-  const int weights_cols_idx = conv_node.op() == "Conv2D" ? 3 : 2;
-  const int64 weights_cols = weights.shape().dim_size(weights_cols_idx);
+  int64 weights_cols;
+  if (conv_node.op() == "Conv2D") {
+    weights_cols = weights.shape().dim_size(3);
+  }
+  else if (conv_node.op() == "DepthwiseConv2dNative") {
+    weights_cols = weights.shape().dim_size(2) * weights.shape().dim_size(3);
+  }
+  else {
+    weights_cols = weights.shape().dim_size(1);
+  }
   CHECK_EQ(weights_cols, scale_values.size());
 
   // Multiply the original weights by the scale vector.
-- 
GitLab


From c1d9c390847471dc233a4a91725f98fdb861fd5a Mon Sep 17 00:00:00 2001
From: zhaoyongke <zhaoyongke@yeah.net>
Date: Sun, 18 Nov 2018 01:46:07 +0800
Subject: [PATCH 0022/2345] Add test cases for depthwise conv

---
 .../graph_transforms/fold_batch_norms_test.cc |  54 ++++++
 .../fold_old_batch_norms_test.cc              | 164 ++++++++++++++++++
 2 files changed, 218 insertions(+)

diff --git a/tensorflow/tools/graph_transforms/fold_batch_norms_test.cc b/tensorflow/tools/graph_transforms/fold_batch_norms_test.cc
index a5d541feb6..2b5326799e 100644
--- a/tensorflow/tools/graph_transforms/fold_batch_norms_test.cc
+++ b/tensorflow/tools/graph_transforms/fold_batch_norms_test.cc
@@ -87,6 +87,57 @@ class FoldBatchNormsTest : public ::testing::Test {
     }
   }
 
+  void TestFoldBatchNormsDepthwiseConv2dNative() {
+    auto root = tensorflow::Scope::NewRootScope();
+    using namespace ::tensorflow::ops;  // NOLINT(build/namespaces)
+
+    Tensor input_data(DT_FLOAT, TensorShape({1, 1, 6, 2}));
+    test::FillValues<float>(
+        &input_data, {1.0f, 4.0f, 2.0f, 5.0f, 3.0f, 6.0f, -1.0f, -4.0f, -2.0f,
+                      -5.0f, -3.0f, -6.0f});
+    Output input_op =
+        Const(root.WithOpName("input_op"), Input::Initializer(input_data));
+
+    Tensor weights_data(DT_FLOAT, TensorShape({1, 2, 2, 2}));
+    test::FillValues<float>(&weights_data,
+                            {1.0f, 2.0f, 3.0f, 4.0f, 0.1f, 0.2f, 0.3f, 0.4f});
+    Output weights_op =
+        Const(root.WithOpName("weights_op"), Input::Initializer(weights_data));
+
+    Output conv_op = DepthwiseConv2dNative(root.WithOpName("conv_op"), input_op, weights_op,
+                            {1, 1, 1, 1}, "VALID");
+
+    Tensor mul_values_data(DT_FLOAT, TensorShape({4})); 
+    test::FillValues<float>(&mul_values_data, {2.0f, 3.0f, 4.0f, 5.0f});
+    Output mul_values_op = Const(root.WithOpName("mul_values"),
+                                 Input::Initializer(mul_values_data));
+
+    Output mul_op = Mul(root.WithOpName("output"), conv_op, mul_values_op);
+
+    GraphDef original_graph_def;
+    TF_ASSERT_OK(root.ToGraphDef(&original_graph_def));
+
+    std::unique_ptr<Session> original_session(NewSession(SessionOptions()));
+    TF_ASSERT_OK(original_session->Create(original_graph_def));
+    std::vector<Tensor> original_outputs;
+    TF_ASSERT_OK(original_session->Run({}, {"output"}, {}, &original_outputs));
+
+    GraphDef fused_graph_def;
+    TF_ASSERT_OK(
+        FoldBatchNorms(original_graph_def, {{}, {"output"}}, &fused_graph_def));
+
+    std::unique_ptr<Session> fused_session(NewSession(SessionOptions()));
+    TF_ASSERT_OK(fused_session->Create(fused_graph_def));
+    std::vector<Tensor> fused_outputs;
+    TF_ASSERT_OK(fused_session->Run({}, {"output"}, {}, &fused_outputs));
+
+    test::ExpectTensorNear<float>(original_outputs[0], fused_outputs[0], 1e-5);
+
+    for (const NodeDef& node : fused_graph_def.node()) {
+      EXPECT_NE("Mul", node.op());
+    }
+  }    
+
   void TestFoldBatchNormsConv2DShared() {
     auto root = tensorflow::Scope::NewRootScope();
     using namespace ::tensorflow::ops;  // NOLINT(build/namespaces)
@@ -202,6 +253,9 @@ TEST_F(FoldBatchNormsTest, TestFoldBatchNormsConv2D) {
 TEST_F(FoldBatchNormsTest, TestFoldBatchNormsMatMul) {
   TestFoldBatchNormsMatMul();
 }
+TEST_F(FoldBatchNormsTest, TestFoldBatchNormsDepthwiseConv2dNative) {
+  TestFoldBatchNormsDepthwiseConv2dNative();
+}
 
 }  // namespace graph_transforms
 }  // namespace tensorflow
diff --git a/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc b/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc
index 435f46c107..45637cf9d1 100644
--- a/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc
+++ b/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc
@@ -121,6 +121,85 @@ class FoldOldBatchNormsTest : public ::testing::Test {
     }
   }
 
+  void TestFoldOldBatchNormsAfterDepthwiseConv2dNative() {
+    auto root = tensorflow::Scope::NewRootScope();
+    using namespace ::tensorflow::ops;  // NOLINT(build/namespaces)
+
+    Tensor input_data(DT_FLOAT, TensorShape({1, 1, 6, 2}));
+    test::FillValues<float>(
+        &input_data, {1.0f, 4.0f, 2.0f, 5.0f, 3.0f, 6.0f, -1.0f, -4.0f, -2.0f,
+                      -5.0f, -3.0f, -6.0f});
+    Output input_op =
+        Const(root.WithOpName("input_op"), Input::Initializer(input_data));
+
+    Tensor weights_data(DT_FLOAT, TensorShape({1, 2, 2, 2}));
+    test::FillValues<float>(&weights_data,
+                            {1.0f, 2.0f, 3.0f, 4.0f, 0.1f, 0.2f, 0.3f, 0.4f});
+    Output weights_op =
+        Const(root.WithOpName("weights_op"), Input::Initializer(weights_data));
+
+    Output conv_op = DepthwiseConv2dNative(root.WithOpName("conv_op"), 
+                            input_op, weights_op, {1, 1, 1, 1}, "VALID");
+
+    Tensor mean_data(DT_FLOAT, TensorShape({4}));
+    test::FillValues<float>(&mean_data, {10.0f, 20.0f, 30.0f, 40.0f});
+    Output mean_op =
+        Const(root.WithOpName("mean_op"), Input::Initializer(mean_data));
+
+    Tensor variance_data(DT_FLOAT, TensorShape({4}));
+    test::FillValues<float>(&variance_data, {0.25f, 0.5f, 0.75f, 1.0f});
+    Output variance_op = Const(root.WithOpName("variance_op"),
+                               Input::Initializer(variance_data));
+
+    Tensor beta_data(DT_FLOAT, TensorShape({4}));
+    test::FillValues<float>(&beta_data, {0.1f, 0.6f, 1.1f, 1.6f});
+    Output beta_op =
+        Const(root.WithOpName("beta_op"), Input::Initializer(beta_data));
+
+    Tensor gamma_data(DT_FLOAT, TensorShape({4}));
+    test::FillValues<float>(&gamma_data, {1.0f, 2.0f, 3.0f, 4.0f});
+    Output gamma_op =
+        Const(root.WithOpName("gamma_op"), Input::Initializer(gamma_data));
+
+    GraphDef original_graph_def;
+    TF_ASSERT_OK(root.ToGraphDef(&original_graph_def));
+
+
+    NodeDef batch_norm_node;
+    batch_norm_node.set_op("BatchNormWithGlobalNormalization");
+    batch_norm_node.set_name("output");
+    AddNodeInput("conv_op", &batch_norm_node);
+    AddNodeInput("mean_op", &batch_norm_node);
+    AddNodeInput("variance_op", &batch_norm_node);
+    AddNodeInput("beta_op", &batch_norm_node);
+    AddNodeInput("gamma_op", &batch_norm_node);
+    SetNodeAttr("T", DT_FLOAT, &batch_norm_node);
+    SetNodeAttr("variance_epsilon", 0.00001f, &batch_norm_node);
+    SetNodeAttr("scale_after_normalization", false, &batch_norm_node);
+    *(original_graph_def.mutable_node()->Add()) = batch_norm_node;
+    original_graph_def.mutable_versions()->set_producer(8);
+
+    std::unique_ptr<Session> original_session(NewSession(SessionOptions()));
+    TF_ASSERT_OK(original_session->Create(original_graph_def));
+    std::vector<Tensor> original_outputs;
+    TF_ASSERT_OK(original_session->Run({}, {"output"}, {}, &original_outputs));
+
+    GraphDef fused_graph_def;
+    TF_ASSERT_OK(FoldOldBatchNorms(original_graph_def, {{}, {"output"}},
+                                   &fused_graph_def));
+
+    std::unique_ptr<Session> fused_session(NewSession(SessionOptions()));
+    TF_ASSERT_OK(fused_session->Create(fused_graph_def));
+    std::vector<Tensor> fused_outputs;
+    TF_ASSERT_OK(fused_session->Run({}, {"output"}, {}, &fused_outputs));
+
+    test::ExpectTensorNear<float>(original_outputs[0], fused_outputs[0], 1e-5);
+
+    for (const NodeDef& node : fused_graph_def.node()) {
+      EXPECT_NE("BatchNormWithGlobalNormalization", node.op());
+    }
+  }    
+
   void TestFoldFusedBatchNorms() {
     auto root = tensorflow::Scope::NewRootScope();
     using namespace ::tensorflow::ops;  // NOLINT(build/namespaces)
@@ -198,6 +277,83 @@ class FoldOldBatchNormsTest : public ::testing::Test {
     }
   }
 
+  void TestFoldFusedBatchNormsAfterDepthwiseConv2dNative() {
+    auto root = tensorflow::Scope::NewRootScope();
+    using namespace ::tensorflow::ops;  // NOLINT(build/namespaces)
+
+    Tensor input_data(DT_FLOAT, TensorShape({1, 1, 6, 2}));
+    test::FillValues<float>(
+        &input_data, {1.0f, 4.0f, 2.0f, 5.0f, 3.0f, 6.0f, -1.0f, -4.0f, -2.0f,
+                      -5.0f, -3.0f, -6.0f});
+    Output input_op =
+        Const(root.WithOpName("input_op"), Input::Initializer(input_data));
+
+    Tensor weights_data(DT_FLOAT, TensorShape({1, 2, 2, 2}));
+    test::FillValues<float>(&weights_data,
+                            {1.0f, 2.0f, 3.0f, 4.0f, 0.1f, 0.2f, 0.3f, 0.4f});
+    Output weights_op =
+        Const(root.WithOpName("weights_op"), Input::Initializer(weights_data));
+
+    Output conv_op = DepthwiseConv2dNative(root.WithOpName("conv_op"), 
+                            input_op, weights_op, {1, 1, 1, 1}, "VALID");
+
+    Tensor mean_data(DT_FLOAT, TensorShape({4}));
+    test::FillValues<float>(&mean_data, {10.0f, 20.0f, 30.0f, 40.0f});
+    Output mean_op =
+        Const(root.WithOpName("mean_op"), Input::Initializer(mean_data));
+
+    Tensor variance_data(DT_FLOAT, TensorShape({4}));
+    test::FillValues<float>(&variance_data, {0.25f, 0.5f, 0.75f, 1.0f});
+    Output variance_op = Const(root.WithOpName("variance_op"),
+                               Input::Initializer(variance_data));
+
+    Tensor beta_data(DT_FLOAT, TensorShape({4}));
+    test::FillValues<float>(&beta_data, {0.1f, 0.6f, 1.1f, 1.6f});
+    Output beta_op =
+        Const(root.WithOpName("beta_op"), Input::Initializer(beta_data));
+
+    Tensor gamma_data(DT_FLOAT, TensorShape({4}));
+    test::FillValues<float>(&gamma_data, {1.0f, 2.0f, 3.0f, 4.0f});
+    Output gamma_op =
+        Const(root.WithOpName("gamma_op"), Input::Initializer(gamma_data));
+
+    GraphDef original_graph_def;
+    TF_ASSERT_OK(root.ToGraphDef(&original_graph_def));
+
+    NodeDef batch_norm_node;
+    batch_norm_node.set_op("FusedBatchNorm");
+    batch_norm_node.set_name("output");
+    AddNodeInput("conv_op", &batch_norm_node);
+    AddNodeInput("gamma_op", &batch_norm_node);
+    AddNodeInput("beta_op", &batch_norm_node);
+    AddNodeInput("mean_op", &batch_norm_node);
+    AddNodeInput("variance_op", &batch_norm_node);
+    SetNodeAttr("T", DT_FLOAT, &batch_norm_node);
+    SetNodeAttr("epsilon", 0.00001f, &batch_norm_node);
+    SetNodeAttr("is_training", false, &batch_norm_node);
+    *(original_graph_def.mutable_node()->Add()) = batch_norm_node;
+
+    std::unique_ptr<Session> original_session(NewSession(SessionOptions()));
+    TF_ASSERT_OK(original_session->Create(original_graph_def));
+    std::vector<Tensor> original_outputs;
+    TF_ASSERT_OK(original_session->Run({}, {"output"}, {}, &original_outputs));
+
+    GraphDef fused_graph_def;
+    TF_ASSERT_OK(FoldOldBatchNorms(original_graph_def, {{}, {"output"}},
+                                   &fused_graph_def));
+
+    std::unique_ptr<Session> fused_session(NewSession(SessionOptions()));
+    TF_ASSERT_OK(fused_session->Create(fused_graph_def));
+    std::vector<Tensor> fused_outputs;
+    TF_ASSERT_OK(fused_session->Run({}, {"output"}, {}, &fused_outputs));
+
+    test::ExpectTensorNear<float>(original_outputs[0], fused_outputs[0], 2e-5);
+
+    for (const NodeDef& node : fused_graph_def.node()) {
+      EXPECT_NE("FusedBatchNorm", node.op());
+    }
+  }
+
   void TestFoldFusedBatchNormsWithConcat(const bool split) {
     auto root = tensorflow::Scope::NewRootScope();
     using namespace ::tensorflow::ops;  // NOLINT(build/namespaces)
@@ -410,5 +566,13 @@ TEST_F(FoldOldBatchNormsTest, TestFoldFusedBatchNormsWithBatchToSpace) {
   TestFoldFusedBatchNormsWithBatchToSpace();
 }
 
+TEST_F(FoldOldBatchNormsTest, TestFoldOldBatchNormsAfterDepthwiseConv2dNative) {
+  TestFoldOldBatchNormsAfterDepthwiseConv2dNative();
+}
+
+TEST_F(FoldOldBatchNormsTest, TestFoldFusedBatchNormsAfterDepthwiseConv2dNative) {
+  TestFoldFusedBatchNormsAfterDepthwiseConv2dNative();
+}
+
 }  // namespace graph_transforms
 }  // namespace tensorflow
-- 
GitLab


From 47a7c6ec8880a34c00d9ba336e4d75ba4a9cbb85 Mon Sep 17 00:00:00 2001
From: Shahzad Lone <shahzadlone@gmail.com>
Date: Thu, 29 Nov 2018 01:20:19 -0500
Subject: [PATCH 0023/2345] Reserve because we know the size before, helps save
 reallocation cost.

---
 tensorflow/cc/tools/freeze_saved_model.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/cc/tools/freeze_saved_model.cc b/tensorflow/cc/tools/freeze_saved_model.cc
index 23e9dc40d2..9469c60ec8 100644
--- a/tensorflow/cc/tools/freeze_saved_model.cc
+++ b/tensorflow/cc/tools/freeze_saved_model.cc
@@ -124,7 +124,9 @@ Status GetVariableNameToTensorMap(
     return Status::OK();
   }
   std::vector<string> variable_names;
+  variable_names.reserve(variable_names.size());
   std::vector<string> tensor_names;
+  tensor_names.reserve(variable_names.size());
   for (const string& node_name : variable_names_set) {
     variable_names.push_back(node_name);
     NodeDef* node_def = name_to_node_map.at(node_name);
-- 
GitLab


From d45cd461242e3a27e505a67c17d5ddb7f4a84641 Mon Sep 17 00:00:00 2001
From: Yves-Noel Weweler <y.weweler@gmail.com>
Date: Fri, 30 Nov 2018 14:46:12 +0100
Subject: [PATCH 0024/2345]  Add drop_remainder argument to
 bucket_by_sequence_length

`tf.data.experimental.bucket_by_sequence_length` does not allow to drop
the last batch in case it has fewer than `batch_size` elements.
This patch does implement `drop_remainder` for `bucket_by_sequence_length`
to enable thhis behaviour.

`drop_remainder` is optinal and set to `False` by default to maintain
compatibility.
---
 tensorflow/python/data/experimental/ops/grouping.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/data/experimental/ops/grouping.py b/tensorflow/python/data/experimental/ops/grouping.py
index db10ea3b7f..71e4b3391f 100644
--- a/tensorflow/python/data/experimental/ops/grouping.py
+++ b/tensorflow/python/data/experimental/ops/grouping.py
@@ -130,7 +130,8 @@ def bucket_by_sequence_length(element_length_func,
                               padded_shapes=None,
                               padding_values=None,
                               pad_to_bucket_boundary=False,
-                              no_padding=False):
+                              no_padding=False,
+                              drop_remainder=False):
   """A transformation that buckets elements in a `Dataset` by length.
 
   Elements of the `Dataset` are grouped together by length and then are padded
@@ -160,6 +161,10 @@ def bucket_by_sequence_length(element_length_func,
       any elements with length longer than `max(bucket_boundaries)`.
     no_padding: `bool`, indicates whether to pad the batch features (features
       need to be either of type `tf.SparseTensor` or of same shape).
+    drop_remainder: (Optional.) A `tf.bool` scalar `tf.Tensor`, representing
+      whether the last batch should be dropped in the case its has fewer than
+      batch_size` elements; the default behavior is not to drop the smaller
+      batch.
 
   Returns:
     A `Dataset` transformation function, which can be passed to
@@ -209,7 +214,7 @@ def bucket_by_sequence_length(element_length_func,
       """Batch elements in dataset."""
       batch_size = window_size_fn(bucket_id)
       if no_padding:
-        return grouped_dataset.batch(batch_size)
+        return grouped_dataset.batch(batch_size, drop_remainder=drop_remainder)
       none_filler = None
       if pad_to_bucket_boundary:
         err_msg = ("When pad_to_bucket_boundary=True, elements must have "
@@ -227,7 +232,8 @@ def bucket_by_sequence_length(element_length_func,
       shapes = make_padded_shapes(
           padded_shapes or grouped_dataset.output_shapes,
           none_filler=none_filler)
-      return grouped_dataset.padded_batch(batch_size, shapes, padding_values)
+      return grouped_dataset.padded_batch(batch_size, shapes, padding_values,
+                                          drop_remainder=drop_remainder)
 
     def _apply_fn(dataset):
       return dataset.apply(
-- 
GitLab


From 916c5238d864e04861a58b3f77b9d29cca03b0e0 Mon Sep 17 00:00:00 2001
From: Yves-Noel Weweler <y.weweler@gmail.com>
Date: Sun, 2 Dec 2018 18:27:55 +0100
Subject: [PATCH 0025/2345] Add drop_reminder support to the test of bucketing
 of sparse tensors

---
 .../bucket_by_sequence_length_test.py         | 31 ++++++++++++-------
 1 file changed, 19 insertions(+), 12 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
index af20e50fb9..fb2d1cf63c 100644
--- a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
@@ -274,11 +274,16 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
       dataset = dataset.map(_to_sparse_tensor)
       return dataset
 
-    def _compute_expected_batches():
+    def _compute_expected_batches(drop_remainder):
       """Computes expected batch outputs and stores in a set."""
       all_expected_sparse_tensors = set()
       for bucket_start_len in range(min_len, max_len, bucket_size):
-        for batch_offset in range(0, bucket_size, batch_size):
+        if drop_remainder:
+          batch_offsets = [0]
+        else:
+          batch_offsets = range(0, bucket_size, batch_size)
+
+        for batch_offset in batch_offsets:
           batch_start_len = bucket_start_len + batch_offset
           batch_end_len = min(batch_start_len + batch_size,
                               bucket_start_len + bucket_size)
@@ -306,16 +311,18 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
             all_sparse_tensors.add(sprs_tensor)
       return all_sparse_tensors
 
-    dataset = _build_dataset()
-    boundaries = range(min_len + bucket_size + 1, max_len, bucket_size)
-    dataset = dataset.apply(grouping.bucket_by_sequence_length(
-        _element_length_fn,
-        boundaries,
-        [batch_size] * (len(boundaries) + 1),
-        no_padding=True))
-    batches = _compute_batches(dataset)
-    expected_batches = _compute_expected_batches()
-    self.assertEqual(batches, expected_batches)
+    for drop_remainder in (True, False):
+      dataset = _build_dataset()
+      boundaries = range(min_len + bucket_size + 1, max_len, bucket_size)
+      dataset = dataset.apply(grouping.bucket_by_sequence_length(
+          _element_length_fn,
+          boundaries,
+          [batch_size] * (len(boundaries) + 1),
+          no_padding=True,
+          drop_remainder=drop_remainder))
+      batches = _compute_batches(dataset)
+      expected_batches = _compute_expected_batches()
+      self.assertEqual(batches, expected_batches)
 
 
 if __name__ == "__main__":
-- 
GitLab


From d372f19f132e8f037390a9a2a6fde7bafa2620b9 Mon Sep 17 00:00:00 2001
From: Yves-Noel Weweler <y.weweler@gmail.com>
Date: Sun, 2 Dec 2018 18:29:03 +0100
Subject: [PATCH 0026/2345] Add a drop_reminder version of the testBucket test

Squeezing both cases into a single test would make it way to
complicated. Therefore, I created a separate test.
---
 .../bucket_by_sequence_length_test.py         | 111 ++++++++++++++++++
 1 file changed, 111 insertions(+)

diff --git a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
index fb2d1cf63c..7b2e922d55 100644
--- a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
@@ -71,6 +71,117 @@ def _get_record_shape(sparse):
 
 class BucketBySequenceLengthTest(test_base.DatasetTestBase):
 
+  def testBucketDropReminder(self):
+
+    boundaries = [10, 20, 30]
+    batch_sizes = [10, 8, 4, 2]
+    lengths = [8, 13, 25, 35]
+
+    n_bucket_elements = [28, 7, 6, 5]
+    n_expected_batches = 5
+
+    # Expected sequence lengths of the individual batches.
+    expected_lengths = []
+
+    # Expected sum of all batches with an equal sequence length.
+    # <seq-length>: <expected-total-sum>
+    expected_sums = dict()
+
+    # Expected batch sizes of batches depending on the sequence length.
+    # <seq-length>: [batch1_size, ..., batchN_size]
+    expected_batch_sizes = dict()
+
+    for length, batch_size, bucket_elements in zip(lengths, batch_sizes, n_bucket_elements):
+      # Calculate the expected sum across all batches of a specific sequence length.
+      expected_sums[length] = (bucket_elements - bucket_elements % batch_size) * length
+      # Calculate the expected occurrence of individual batch sizes.
+      expected_batch_sizes[length] = [batch_size] * (bucket_elements // batch_size)
+      # Calculate the expected occurence of individual sequence lengths.
+      expected_lengths.extend([length] * (bucket_elements // batch_size))
+
+    def build_dataset(sparse):
+      def _generator():
+        # Produce 1 batch for each bucket
+        elements = []
+        for bucket_elements, length in zip(n_bucket_elements, lengths):
+          # Using only full sequences (opposed to the strategy employed in `testBucket`) makes
+          # checking the sum a lot easier.
+          record_len = length
+          for _ in range(bucket_elements):
+            elements.append([1] * record_len)
+        random.shuffle(elements)
+        for el in elements:
+          yield (_format_record(el, sparse),)
+      dataset = dataset_ops.Dataset.from_generator(
+          _generator,
+          (_get_record_type(sparse),),
+          (_get_record_shape(sparse),))
+      if sparse:
+        dataset = dataset.map(lambda x: (_to_sparse_tensor(x),))
+      return dataset
+
+    def _test_bucket_by_padding(no_padding):
+      dataset = build_dataset(sparse=no_padding)
+      dataset = dataset.apply(
+          grouping.bucket_by_sequence_length(
+              _element_length_fn,
+              boundaries,
+              batch_sizes,
+              no_padding=no_padding,
+              drop_remainder=True))
+      batch, = dataset.make_one_shot_iterator().get_next()
+
+      with self.cached_session() as sess:
+        batches = []
+        for _ in range(n_expected_batches):
+          batches.append(self.evaluate(batch))
+        with self.assertRaises(errors.OutOfRangeError):
+          self.evaluate(batch)
+
+      generated_lengths = []
+
+      # <seq-length>: <total-sum>
+      generated_sums = dict()
+
+      # <seq-length>: [<batch_size>, ...]
+      generated_batch_sizes = dict()
+
+      for length, batch_size, bucket_elements in zip(lengths, batch_sizes, n_bucket_elements):
+          # Initialize the sum across all batches.
+          expected_sums[length] = 0
+          # Initialize the individual batch sizes.
+          expected_batch_sizes[length] = []
+
+      for batch in batches:
+        shape = batch.dense_shape if no_padding else batch.shape
+        length = shape[1]
+        generated_lengths.append(length)
+
+        batch_size = shape[0]
+        generated_batch_sizes[length].append(batch_size)
+
+        batch_sum = batch.values.sum() if no_padding else batch.sum()
+        generated_sums[length] += batch_sum
+
+      for l in lengths:
+        # Make sure the sum of the batch contents is correct for the individual sequence lengths.
+        self.assertEqual(generated_sums[l], expected_sums[l],
+                         'Tensor sums did not match! expected: {}, generated: {}'
+                         .format(expected_sums, generated_sums))
+
+        # Make sure the individual batch sizes are generated as expected.
+        self.assertEqual(sorted(generated_batch_sizes[l]), sorted(expected_batch_sizes[l]),
+                         'Batch-sizes did not match! expected: {}, generated: {}'
+                         .format(sorted(expected_batch_sizes[l]), sorted(generated_batch_sizes[l])))
+
+      # Make sure the generated sequence lengths appear as often as expected.
+      self.assertEqual(sorted(generated_lengths), sorted(expected_lengths),
+                       'The generated sequence lengths did not match! expected: {}, generated: {}'
+                       .format(sorted(expected_lengths), sorted(generated_lengths)))
+
+    for no_padding in (True, False):
+      _test_bucket_by_padding(no_padding)
+
   def testBucket(self):
 
     boundaries = [10, 20, 30]
-- 
GitLab


From 68426789549cb9d2edc8726fc5edabf4f221bd9b Mon Sep 17 00:00:00 2001
From: Yves-Noel Weweler <y.weweler@gmail.com>
Date: Sun, 2 Dec 2018 18:40:12 +0100
Subject: [PATCH 0027/2345] Fix pylint warnings

* Missing argument drop_remainder in the testBucketSparse test.
* Insert line breaks
---
 .../bucket_by_sequence_length_test.py         | 45 ++++++++++++-------
 1 file changed, 29 insertions(+), 16 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
index 7b2e922d55..5b96101b7c 100644
--- a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
@@ -91,11 +91,15 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
     # <seq-length>: [batch1_size, ..., batchN_size]
     expected_batch_sizes = dict()
 
-    for length, batch_size, bucket_elements in zip(lengths, batch_sizes, n_bucket_elements):
+    for length, batch_size, bucket_elements in zip(lengths,
+                                                   batch_sizes,
+                                                   n_bucket_elements):
       # Calculate the expected sum across all batches of a specific sequence length.
-      expected_sums[length] = (bucket_elements - bucket_elements % batch_size) * length
+      expected_sums[length] = \
+          (bucket_elements - bucket_elements % batch_size) * length
       # Calculate the expected occurrence of individual batch sizes.
-      expected_batch_sizes[length] = [batch_size] * (bucket_elements // batch_size)
+      expected_batch_sizes[length] = \
+          [batch_size] * (bucket_elements // batch_size)
       # Calculate the expected occurence of individual sequence lengths.
       expected_lengths.extend([length] * (bucket_elements // batch_size))
 
@@ -146,11 +150,13 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
       # <seq-length>: [<batch_size>, ...]
       generated_batch_sizes = dict()
 
-      for length, batch_size, bucket_elements in zip(lengths, batch_sizes, n_bucket_elements):
-          # Initialize the sum across all batches.
-          expected_sums[length] = 0
-          # Initialize the individual batch sizes.
-          expected_batch_sizes[length] = []
+      for length, batch_size, bucket_elements in zip(lengths,
+                                                     batch_sizes,
+                                                     n_bucket_elements):
+        # Initialize the sum across all batches.
+        expected_sums[length] = 0
+        # Initialize the individual batch sizes.
+        expected_batch_sizes[length] = []
 
       for batch in batches:
         shape = batch.dense_shape if no_padding else batch.shape
@@ -165,19 +171,26 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
 
       for l in lengths:
         # Make sure the sum of the batch contents is correct for the individual sequence lengths.
-        self.assertEqual(generated_sums[l], expected_sums[l],
-                         'Tensor sums did not match! expected: {}, generated: {}'
+        self.assertEqual(generated_sums[l],
+                         expected_sums[l],
+                         'Tensor sums did not match! '
+                         'expected: {}, generated: {}'
                          .format(expected_sums, generated_sums))
 
         # Make sure the individual batch sizes are generated as expected.
-        self.assertEqual(sorted(generated_batch_sizes[l]), sorted(expected_batch_sizes[l]),
-                         'Batch-sizes did not match! expected: {}, generated: {}'
-                         .format(sorted(expected_batch_sizes[l]), sorted(generated_batch_sizes[l])))
+        self.assertEqual(sorted(generated_batch_sizes[l]),
+                         sorted(expected_batch_sizes[l]),
+                         'Batch-sizes did not match! '
+                         'expected: {}, generated: {}'
+                         .format(sorted(expected_batch_sizes[l]),
+                                 sorted(generated_batch_sizes[l])))
 
       # Make sure the generated sequence lengths appear as often as expected.
       self.assertEqual(sorted(generated_lengths), sorted(expected_lengths),
-                       'The generated sequence lengths did not match! expected: {}, generated: {}'
-                       .format(sorted(expected_lengths), sorted(generated_lengths)))
+                       'The generated sequence lengths did not match! '
+                       'expected: {}, generated: {}'
+                       .format(sorted(expected_lengths),
+                               sorted(generated_lengths)))
 
     for no_padding in (True, False):
       _test_bucket_by_padding(no_padding)
@@ -432,7 +445,7 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
           no_padding=True,
           drop_remainder=drop_remainder))
       batches = _compute_batches(dataset)
-      expected_batches = _compute_expected_batches()
+      expected_batches = _compute_expected_batches(drop_remainder)
       self.assertEqual(batches, expected_batches)
 
 
-- 
GitLab


From 4ddd2ab07fdfea81a53d110fee19bb88c41251a0 Mon Sep 17 00:00:00 2001
From: Yves-Noel Weweler <y.weweler@gmail.com>
Date: Mon, 3 Dec 2018 13:58:58 +0100
Subject: [PATCH 0028/2345] Fix copy paste bug

Pasted the wrong variable names from the docker test image.
---
 .../kernel_tests/bucket_by_sequence_length_test.py            | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
index 5b96101b7c..fab79619a0 100644
--- a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
@@ -154,9 +154,9 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
                                                      batch_sizes,
                                                      n_bucket_elements):
         # Initialize the sum across all batches.
-        expected_sums[length] = 0
+        generated_sums[length] = 0
         # Initialize the individual batch sizes.
-        expected_batch_sizes[length] = []
+        generated_batch_sizes[length] = []
 
       for batch in batches:
         shape = batch.dense_shape if no_padding else batch.shape
-- 
GitLab


From c425f34ad4ef83055bb6ca5a9542d7320558c598 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=8D=9C=E5=B1=85?= <zhaoyongke@yeah.net>
Date: Thu, 6 Dec 2018 22:04:17 +0800
Subject: [PATCH 0029/2345] clang-format-3.6 regenerated files in this PR

---
 .../graph_transforms/fold_batch_norms.cc      | 12 ++++-----
 .../graph_transforms/fold_batch_norms_test.cc |  8 +++---
 .../graph_transforms/fold_old_batch_norms.cc  | 21 +++++++--------
 .../fold_old_batch_norms_test.cc              | 27 ++++++++++---------
 4 files changed, 33 insertions(+), 35 deletions(-)

diff --git a/tensorflow/tools/graph_transforms/fold_batch_norms.cc b/tensorflow/tools/graph_transforms/fold_batch_norms.cc
index 2d7bb65430..0a37620700 100644
--- a/tensorflow/tools/graph_transforms/fold_batch_norms.cc
+++ b/tensorflow/tools/graph_transforms/fold_batch_norms.cc
@@ -76,11 +76,10 @@ Status FoldBatchNorms(const GraphDef& input_graph_def,
         int64 weights_cols;
         if (conv_node.op() == "Conv2D") {
           weights_cols = weights.shape().dim_size(3);
-        }
-        else if (conv_node.op() == "DepthwiseConv2dNative") {
-          weights_cols = weights.shape().dim_size(2) * weights.shape().dim_size(3);
-        }
-        else {
+        } else if (conv_node.op() == "DepthwiseConv2dNative") {
+          weights_cols =
+              weights.shape().dim_size(2) * weights.shape().dim_size(3);
+        } else {
           weights_cols = weights.shape().dim_size(1);
         }
         if ((mul_values.shape().dims() != 1) ||
@@ -96,7 +95,8 @@ Status FoldBatchNorms(const GraphDef& input_graph_def,
         auto scaled_weights_vector = scaled_weights.flat<float>();
         for (int64 row = 0; row < weights_vector.dimension(0); ++row) {
           scaled_weights_vector(row) =
-                weights_vector(row) * mul_values.flat<float>()(row % weights_cols);
+              weights_vector(row) *
+              mul_values.flat<float>()(row % weights_cols);
         }
 
         // Construct the new nodes.
diff --git a/tensorflow/tools/graph_transforms/fold_batch_norms_test.cc b/tensorflow/tools/graph_transforms/fold_batch_norms_test.cc
index 2b5326799e..885fbd59b7 100644
--- a/tensorflow/tools/graph_transforms/fold_batch_norms_test.cc
+++ b/tensorflow/tools/graph_transforms/fold_batch_norms_test.cc
@@ -104,10 +104,10 @@ class FoldBatchNormsTest : public ::testing::Test {
     Output weights_op =
         Const(root.WithOpName("weights_op"), Input::Initializer(weights_data));
 
-    Output conv_op = DepthwiseConv2dNative(root.WithOpName("conv_op"), input_op, weights_op,
-                            {1, 1, 1, 1}, "VALID");
+    Output conv_op = DepthwiseConv2dNative(root.WithOpName("conv_op"), input_op,
+                                           weights_op, {1, 1, 1, 1}, "VALID");
 
-    Tensor mul_values_data(DT_FLOAT, TensorShape({4})); 
+    Tensor mul_values_data(DT_FLOAT, TensorShape({4}));
     test::FillValues<float>(&mul_values_data, {2.0f, 3.0f, 4.0f, 5.0f});
     Output mul_values_op = Const(root.WithOpName("mul_values"),
                                  Input::Initializer(mul_values_data));
@@ -136,7 +136,7 @@ class FoldBatchNormsTest : public ::testing::Test {
     for (const NodeDef& node : fused_graph_def.node()) {
       EXPECT_NE("Mul", node.op());
     }
-  }    
+  }
 
   void TestFoldBatchNormsConv2DShared() {
     auto root = tensorflow::Scope::NewRootScope();
diff --git a/tensorflow/tools/graph_transforms/fold_old_batch_norms.cc b/tensorflow/tools/graph_transforms/fold_old_batch_norms.cc
index 413361b616..8c67bd23b5 100644
--- a/tensorflow/tools/graph_transforms/fold_old_batch_norms.cc
+++ b/tensorflow/tools/graph_transforms/fold_old_batch_norms.cc
@@ -32,9 +32,9 @@ Status ErrorIfNotVector(const Tensor& input, const string& input_name,
                         int expected_width) {
   if ((input.shape().dims() != 1) ||
       (input.shape().dim_size(0) != expected_width)) {
-    return errors::InvalidArgument(
-        input_name,
-        " input to batch norm has bad shape: ", input.shape().DebugString());
+    return errors::InvalidArgument(input_name,
+                                   " input to batch norm has bad shape: ",
+                                   input.shape().DebugString());
   }
   return Status::OK();
 }
@@ -119,11 +119,9 @@ Status FuseScaleOffsetToConvWeights(const std::vector<float>& scale_values,
   int64 weights_cols;
   if (conv_node.op() == "Conv2D") {
     weights_cols = weights.shape().dim_size(3);
-  }
-  else if (conv_node.op() == "DepthwiseConv2dNative") {
+  } else if (conv_node.op() == "DepthwiseConv2dNative") {
     weights_cols = weights.shape().dim_size(2) * weights.shape().dim_size(3);
-  }
-  else {
+  } else {
     weights_cols = weights.shape().dim_size(1);
   }
   CHECK_EQ(weights_cols, scale_values.size());
@@ -134,7 +132,7 @@ Status FuseScaleOffsetToConvWeights(const std::vector<float>& scale_values,
   auto scaled_weights_vector = scaled_weights.flat<float>();
   for (int64 row = 0; row < weights_vector.dimension(0); ++row) {
     scaled_weights_vector(row) =
-          weights_vector(row) * scale_values[row % weights_cols];
+        weights_vector(row) * scale_values[row % weights_cols];
   }
   // Figure out the remaining bias to add on.
   Tensor bias_offset(DT_FLOAT, {weights_cols});
@@ -193,7 +191,7 @@ Status FuseBatchNormWithConv(const NodeMatch& match,
 }
 
 Status FuseBatchNormWithBatchToSpace(const NodeMatch& match,
-                             std::vector<NodeDef>* new_nodes) {
+                                     std::vector<NodeDef>* new_nodes) {
   // Calculate the scale and offset values to apply.
   std::vector<float> scale_values;
   std::vector<float> offset_values;
@@ -208,9 +206,8 @@ Status FuseBatchNormWithBatchToSpace(const NodeMatch& match,
   const NodeDef& conv_node = conv_node_match.node;
 
   string biasadd_name = conv_node.name() + "/biasadd";
-  TF_RETURN_IF_ERROR(
-      FuseScaleOffsetToConvWeights(scale_values, offset_values, conv_node_match,
-                                   biasadd_name , new_nodes));
+  TF_RETURN_IF_ERROR(FuseScaleOffsetToConvWeights(
+      scale_values, offset_values, conv_node_match, biasadd_name, new_nodes));
 
   NodeDef new_batch_to_space_node = batch_to_space_node;
   // reuse batch_norm node name
diff --git a/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc b/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc
index 45637cf9d1..925f37745c 100644
--- a/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc
+++ b/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc
@@ -138,8 +138,8 @@ class FoldOldBatchNormsTest : public ::testing::Test {
     Output weights_op =
         Const(root.WithOpName("weights_op"), Input::Initializer(weights_data));
 
-    Output conv_op = DepthwiseConv2dNative(root.WithOpName("conv_op"), 
-                            input_op, weights_op, {1, 1, 1, 1}, "VALID");
+    Output conv_op = DepthwiseConv2dNative(root.WithOpName("conv_op"), input_op,
+                                           weights_op, {1, 1, 1, 1}, "VALID");
 
     Tensor mean_data(DT_FLOAT, TensorShape({4}));
     test::FillValues<float>(&mean_data, {10.0f, 20.0f, 30.0f, 40.0f});
@@ -164,7 +164,6 @@ class FoldOldBatchNormsTest : public ::testing::Test {
     GraphDef original_graph_def;
     TF_ASSERT_OK(root.ToGraphDef(&original_graph_def));
 
-
     NodeDef batch_norm_node;
     batch_norm_node.set_op("BatchNormWithGlobalNormalization");
     batch_norm_node.set_name("output");
@@ -198,7 +197,7 @@ class FoldOldBatchNormsTest : public ::testing::Test {
     for (const NodeDef& node : fused_graph_def.node()) {
       EXPECT_NE("BatchNormWithGlobalNormalization", node.op());
     }
-  }    
+  }
 
   void TestFoldFusedBatchNorms() {
     auto root = tensorflow::Scope::NewRootScope();
@@ -294,8 +293,8 @@ class FoldOldBatchNormsTest : public ::testing::Test {
     Output weights_op =
         Const(root.WithOpName("weights_op"), Input::Initializer(weights_data));
 
-    Output conv_op = DepthwiseConv2dNative(root.WithOpName("conv_op"), 
-                            input_op, weights_op, {1, 1, 1, 1}, "VALID");
+    Output conv_op = DepthwiseConv2dNative(root.WithOpName("conv_op"), input_op,
+                                           weights_op, {1, 1, 1, 1}, "VALID");
 
     Tensor mean_data(DT_FLOAT, TensorShape({4}));
     test::FillValues<float>(&mean_data, {10.0f, 20.0f, 30.0f, 40.0f});
@@ -477,16 +476,17 @@ void TestFoldFusedBatchNormsWithBatchToSpace() {
 
   Tensor block_shape_data(DT_INT32, TensorShape({2}));
   test::FillValues<int32>(&block_shape_data, {1, 2});
-  Output block_shape_op =
-      Const(root.WithOpName("block_shape_op"), Input::Initializer(block_shape_data));
+  Output block_shape_op = Const(root.WithOpName("block_shape_op"),
+                                Input::Initializer(block_shape_data));
 
   Tensor crops_data(DT_INT32, TensorShape({2, 2}));
   test::FillValues<int32>(&crops_data, {0, 0, 0, 1});
   Output crops_op =
       Const(root.WithOpName("crops_op"), Input::Initializer(crops_data));
 
-  Output batch_to_space_op = BatchToSpaceND(root.WithOpName("batch_to_space_op"),
-                                            conv_op, block_shape_op, crops_data);
+  Output batch_to_space_op =
+      BatchToSpaceND(root.WithOpName("batch_to_space_op"), conv_op,
+                     block_shape_op, crops_data);
 
   Tensor mean_data(DT_FLOAT, TensorShape({2}));
   test::FillValues<float>(&mean_data, {10.0f, 20.0f});
@@ -495,8 +495,8 @@ void TestFoldFusedBatchNormsWithBatchToSpace() {
 
   Tensor variance_data(DT_FLOAT, TensorShape({2}));
   test::FillValues<float>(&variance_data, {0.25f, 0.5f});
-  Output variance_op = Const(root.WithOpName("variance_op"),
-                             Input::Initializer(variance_data));
+  Output variance_op =
+      Const(root.WithOpName("variance_op"), Input::Initializer(variance_data));
 
   Tensor beta_data(DT_FLOAT, TensorShape({2}));
   test::FillValues<float>(&beta_data, {0.1f, 0.6f});
@@ -570,7 +570,8 @@ TEST_F(FoldOldBatchNormsTest, TestFoldOldBatchNormsAfterDepthwiseConv2dNative) {
   TestFoldOldBatchNormsAfterDepthwiseConv2dNative();
 }
 
-TEST_F(FoldOldBatchNormsTest, TestFoldFusedBatchNormsAfterDepthwiseConv2dNative) {
+TEST_F(FoldOldBatchNormsTest,
+       TestFoldFusedBatchNormsAfterDepthwiseConv2dNative) {
   TestFoldFusedBatchNormsAfterDepthwiseConv2dNative();
 }
 
-- 
GitLab


From 562b078b836b215b761fc91a177937cfcbdd0ea0 Mon Sep 17 00:00:00 2001
From: Nayana-ibm <nayana.thorat@gmail.com>
Date: Thu, 6 Dec 2018 11:13:12 -0500
Subject: [PATCH 0030/2345] resolve ImportError: cannot import name cloud on
 s390x - skip import clound

---
 tensorflow/contrib/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/__init__.py b/tensorflow/contrib/__init__.py
index 4f1a2a5693..a23e331f4f 100644
--- a/tensorflow/contrib/__init__.py
+++ b/tensorflow/contrib/__init__.py
@@ -20,13 +20,14 @@ from __future__ import division
 from __future__ import print_function
 
 import os
+import platform
 
 # Add projects here, they will show up under tf.contrib.
 from tensorflow.contrib import autograph
 from tensorflow.contrib import batching
 from tensorflow.contrib import bayesflow
 from tensorflow.contrib import checkpoint
-if os.name != "nt":
+if os.name != "nt" and platform.machine() != "s390x":
   from tensorflow.contrib import cloud
 from tensorflow.contrib import cluster_resolver
 from tensorflow.contrib import coder
-- 
GitLab


From aff52125a0d172f4c3ae8ea7c170ca0f6d97e97d Mon Sep 17 00:00:00 2001
From: Andy Craze <accraze@gmail.com>
Date: Fri, 7 Dec 2018 20:37:37 -0800
Subject: [PATCH 0031/2345] Update tables_initializer to link to guide

Resolves  #20629
---
 tensorflow/python/ops/lookup_ops.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/python/ops/lookup_ops.py b/tensorflow/python/ops/lookup_ops.py
index 758cb8041d..9302696a45 100644
--- a/tensorflow/python/ops/lookup_ops.py
+++ b/tensorflow/python/ops/lookup_ops.py
@@ -64,6 +64,8 @@ def initialize_all_tables(name="init_all_tables"):
 @tf_export(v1=["initializers.tables_initializer", "tables_initializer"])
 def tables_initializer(name="init_all_tables"):
   """Returns an Op that initializes all tables of the default graph.
+  See the [Low Level Intro](https://www.tensorflow.org/guide/low_level_intro#feature_columns)
+  guide, for an example of usage.
 
   Args:
     name: Optional name for the initialization op.
-- 
GitLab


From 035955852708fba07565ead298cd54ad64ab1a55 Mon Sep 17 00:00:00 2001
From: Jacky Ko <jackkykokoko@gmail.com>
Date: Sun, 9 Dec 2018 18:33:50 +0800
Subject: [PATCH 0032/2345] static lib name change

---
 tensorflow/contrib/cmake/external/abseil_cpp.cmake | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/cmake/external/abseil_cpp.cmake b/tensorflow/contrib/cmake/external/abseil_cpp.cmake
index 8b76f37858..539c5cbb76 100644
--- a/tensorflow/contrib/cmake/external/abseil_cpp.cmake
+++ b/tensorflow/contrib/cmake/external/abseil_cpp.cmake
@@ -48,7 +48,7 @@ else (systemlib_ABSEIL_CPP)
       set(abseil_cpp_STATIC_LIBRARIES
           ${abseil_cpp_BUILD}/absl/base/Release/absl_base.lib
           ${abseil_cpp_BUILD}/absl/base/Release/absl_dynamic_annotations.lib
-          ${abseil_cpp_BUILD}/absl/base/Release/absl_internal_malloc_internal.lib
+          ${abseil_cpp_BUILD}/absl/base/Release/absl_malloc_internal.lib
           ${abseil_cpp_BUILD}/absl/strings/Release/absl_strings.lib
           ${abseil_cpp_BUILD}/absl/strings/Release/str_format_internal.lib
           ${abseil_cpp_BUILD}/absl/types/Release/absl_bad_optional_access.lib)
@@ -94,6 +94,8 @@ else (systemlib_ABSEIL_CPP)
   )
 
   include_directories(${abseil_cpp_INCLUDE_DIR})
+  message(STATUS ${abseil_cpp_INCLUDE_DIR})
+
   list(APPEND tensorflow_EXTERNAL_LIBRARIES ${abseil_cpp_STATIC_LIBRARIES})
 
   list(APPEND tensorflow_EXTERNAL_DEPENDENCIES abseil_cpp_build)
-- 
GitLab


From 5a4871af05392c2e5744515a4c67bc6f0f420943 Mon Sep 17 00:00:00 2001
From: fo40225 <fo40225@hotmail.com>
Date: Sat, 1 Sep 2018 00:58:14 +0800
Subject: [PATCH 0033/2345] fix AttributeError: 'module' object has no
 attribute '???'on windows python 2.7

---
 tensorflow/api_template_v1.__init__.py      | 3 ++-
 tensorflow/contrib/cmake/python_modules.txt | 3 +++
 tensorflow/contrib/cmake/tf_python.cmake    | 1 +
 3 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/tensorflow/api_template_v1.__init__.py b/tensorflow/api_template_v1.__init__.py
index 65bdb6cb1b..b9b21bad50 100644
--- a/tensorflow/api_template_v1.__init__.py
+++ b/tensorflow/api_template_v1.__init__.py
@@ -40,7 +40,8 @@ if '__all__' in vars():
   vars()['__all__'].append('contrib')
 
 from tensorflow.python.platform import flags  # pylint: disable=g-import-not-at-top
-app.flags = flags  # pylint: disable=undefined-variable
+from tensorflow.python.platform import app  # pylint: disable=g-import-not-at-top
+app.flags = flags
 
 # Make sure directory containing top level submodules is in
 # the __path__ so that "from tensorflow.foo import bar" works.
diff --git a/tensorflow/contrib/cmake/python_modules.txt b/tensorflow/contrib/cmake/python_modules.txt
index 96160568fa..21ae9a08a6 100644
--- a/tensorflow/contrib/cmake/python_modules.txt
+++ b/tensorflow/contrib/cmake/python_modules.txt
@@ -1,6 +1,9 @@
 # python_sanity_test.py will complain about invalid or missing entries
 # problematic entries can be commented for temporary whitelisting
 tensorflow
+tensorflow/compiler
+tensorflow/compiler/xla
+tensorflow/compiler/xla/service
 tensorflow/core
 tensorflow/core/example
 tensorflow/core/framework
diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake
index 8faccf8d55..1fe8795ddf 100755
--- a/tensorflow/contrib/cmake/tf_python.cmake
+++ b/tensorflow/contrib/cmake/tf_python.cmake
@@ -802,6 +802,7 @@ add_custom_command(
       # tensorflow/__init__.py depends on files generated in this step. So, remove it while
       # this step is running since the files aren't there yet.
       COMMAND ${CMAKE_COMMAND} -E remove -f ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/__init__.py
+      COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/__init__.py
 
       # Run create_python_api.py to generate API init files.
       COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}/tf_python "${PY_RUNTIME_ENV}" ${PYTHON_EXECUTABLE}
-- 
GitLab


From cc69b82bbcc68c21db839cca4f6fec043f6005aa Mon Sep 17 00:00:00 2001
From: Jacky Ko <jackkykokoko@gmail.com>
Date: Sun, 9 Dec 2018 19:08:32 +0800
Subject: [PATCH 0034/2345] abseil use master branch

---
 tensorflow/contrib/cmake/external/abseil_cpp.cmake | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/cmake/external/abseil_cpp.cmake b/tensorflow/contrib/cmake/external/abseil_cpp.cmake
index 539c5cbb76..f64b60fd5e 100644
--- a/tensorflow/contrib/cmake/external/abseil_cpp.cmake
+++ b/tensorflow/contrib/cmake/external/abseil_cpp.cmake
@@ -39,8 +39,8 @@ else (systemlib_ABSEIL_CPP)
   include (ExternalProject)
 
   set(abseil_cpp_INCLUDE_DIR ${CMAKE_BINARY_DIR}/abseil_cpp/src/abseil_cpp_build)
-  set(abseil_cpp_URL https://github.com/abseil/abseil-cpp/archive/e01d95528ea2137a4a27a88d1f57c6cb260aafed.tar.gz)
-  set(abseil_cpp_HASH SHA256=84043ed402d2a2a6ba4cdddb7e85118b1158fd81fe4ac3a14adc343d054c1e2e)
+  set(abseil_cpp_URL https://github.com/abseil/abseil-cpp.git)
+  set(abseil_cpp_TAG master)
   set(abseil_cpp_BUILD ${CMAKE_BINARY_DIR}/abseil_cpp/src/abseil_cpp_build)
 
   if(WIN32)
@@ -48,7 +48,7 @@ else (systemlib_ABSEIL_CPP)
       set(abseil_cpp_STATIC_LIBRARIES
           ${abseil_cpp_BUILD}/absl/base/Release/absl_base.lib
           ${abseil_cpp_BUILD}/absl/base/Release/absl_dynamic_annotations.lib
-          ${abseil_cpp_BUILD}/absl/base/Release/absl_malloc_internal.lib
+          ${abseil_cpp_BUILD}/absl/base/Release/absl_internal_malloc_internal.lib
           ${abseil_cpp_BUILD}/absl/strings/Release/absl_strings.lib
           ${abseil_cpp_BUILD}/absl/strings/Release/str_format_internal.lib
           ${abseil_cpp_BUILD}/absl/types/Release/absl_bad_optional_access.lib)
@@ -79,8 +79,7 @@ else (systemlib_ABSEIL_CPP)
 
   ExternalProject_Add(abseil_cpp_build
       PREFIX abseil_cpp
-      URL ${abseil_cpp_URL}
-      URL_HASH ${abseil_cpp_HASH}
+      GIT_REPOSITORY ${abseil_cpp_URL}
       DOWNLOAD_DIR "${DOWNLOAD_LOCATION}"
       BUILD_IN_SOURCE 1
       BUILD_BYPRODUCTS ${abseil_cpp_STATIC_LIBRARIES}
-- 
GitLab


From 956b77a2a8f8db7a57b818bc0c06669fd395d56d Mon Sep 17 00:00:00 2001
From: Jacky Ko <jackkykokoko@gmail.com>
Date: Sun, 9 Dec 2018 19:31:05 +0800
Subject: [PATCH 0035/2345] abseil lib linkage update

---
 tensorflow/contrib/cmake/external/abseil_cpp.cmake | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/contrib/cmake/external/abseil_cpp.cmake b/tensorflow/contrib/cmake/external/abseil_cpp.cmake
index f64b60fd5e..eefa7d3f03 100644
--- a/tensorflow/contrib/cmake/external/abseil_cpp.cmake
+++ b/tensorflow/contrib/cmake/external/abseil_cpp.cmake
@@ -49,6 +49,8 @@ else (systemlib_ABSEIL_CPP)
           ${abseil_cpp_BUILD}/absl/base/Release/absl_base.lib
           ${abseil_cpp_BUILD}/absl/base/Release/absl_dynamic_annotations.lib
           ${abseil_cpp_BUILD}/absl/base/Release/absl_internal_malloc_internal.lib
+          ${abseil_cpp_BUILD}/absl/base/Release/absl_internal_throw_delegate.lib
+          ${abseil_cpp_BUILD}/absl/numeric/Release/absl_int128.lib
           ${abseil_cpp_BUILD}/absl/strings/Release/absl_strings.lib
           ${abseil_cpp_BUILD}/absl/strings/Release/str_format_internal.lib
           ${abseil_cpp_BUILD}/absl/types/Release/absl_bad_optional_access.lib)
-- 
GitLab


From 7578e120de2a3a5282ced8d41881f19363f83466 Mon Sep 17 00:00:00 2001
From: Dan Jarvis <daj@users.noreply.github.com>
Date: Thu, 23 Nov 2017 13:06:02 -0500
Subject: [PATCH 0036/2345] Fix crash on closing the app when classifier failed
 to initialize

When testing on an API 21 emulator, the classifier fails to initialize.
`E/TfLiteCameraDemo: Failed to initialize an image classifier.`

In this situation, the app crashes when pressing Back to exit.  Here's the cause:
```
java.lang.NullPointerException: Attempt to invoke virtual method 'void com.example.android.tflitecamerademo.ImageClassifier.close()' on a null object reference
                                                                                        at com.example.android.tflitecamerademo.Camera2BasicFragment.onDestroy(Camera2BasicFragment.java:331)
                                                                                        at android.app.Fragment.performDestroy(Fragment.java:2266)
```
The fix is to check for null before calling `.close()`.

I'll investigate why the classifier is failing to initialize separately. :-)
---
 .../android/tflitecamerademo/Camera2BasicFragment.java        | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java b/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java
index 165d335101..a7b3440536 100644
--- a/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java
+++ b/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java
@@ -476,7 +476,9 @@ public class Camera2BasicFragment extends Fragment
 
   @Override
   public void onDestroy() {
-    classifier.close();
+    if (classifier != null) {
+      classifier.close();
+    }
     super.onDestroy();
   }
 
-- 
GitLab


From 16380e05087b41c39547e3f05e1ce85cea44efb2 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Tue, 11 Dec 2018 00:40:26 +0000
Subject: [PATCH 0037/2345] Fix warning caused by to_int32

While running tf.keras I noticed the following warning:
```
WARNING:tensorflow:From /usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/math_ops.py:3064: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
```

This fix fixes the warning caused by deprecated to_int32.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/ops/math_ops.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index e2b634ee8f..1d85fb0193 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -3061,8 +3061,8 @@ def reduced_shape(input_shape, axes):
     input_shape[axes] = 1
     return input_shape
 
-  input_shape = to_int32(input_shape)  # [2, 3, 5, 7]
-  axes = to_int32(axes)  # [1, 2]
+  input_shape = cast(input_shape, dtypes.int32)  # [2, 3, 5, 7]
+  axes = cast(axes, dtypes.int32)  # [1, 2]
 
   input_rank = array_ops.size(input_shape)  # 4
   axes = (axes + input_rank) % input_rank
-- 
GitLab


From 16f454f95e9d9823145d6b00a7e007afd0ea569b Mon Sep 17 00:00:00 2001
From: Luke Han <sungyuk@gmail.com>
Date: Tue, 11 Dec 2018 11:03:05 +0900
Subject: [PATCH 0038/2345] fix typo in scatter_nd_add docstring

---
 tensorflow/python/ops/state_ops.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/ops/state_ops.py b/tensorflow/python/ops/state_ops.py
index 3ac69c1c20..25b31c698e 100644
--- a/tensorflow/python/ops/state_ops.py
+++ b/tensorflow/python/ops/state_ops.py
@@ -462,9 +462,8 @@ def scatter_nd_add(ref, indices, updates, use_locking=False, name=None):
     updates: A `Tensor`. Must have the same type as `ref`.
       A tensor of updated values to add to ref.
     use_locking: An optional `bool`. Defaults to `False`.
-      An optional bool. Defaults to True. If True, the assignment will
-      be protected by a lock; otherwise the behavior is undefined,
-      but may exhibit less contention.
+      If True, the assignment will be protected by a lock;
+      otherwise the behavior is undefined, but may exhibit less contention.
     name: A name for the operation (optional).
 
   Returns:
-- 
GitLab


From f0f7ed323983ab2cb157bf782950b6e2238b9f6b Mon Sep 17 00:00:00 2001
From: Jacky Ko <jackkykokoko@gmail.com>
Date: Tue, 11 Dec 2018 20:35:50 +0800
Subject: [PATCH 0039/2345] add abseil time library linking

---
 tensorflow/contrib/cmake/external/abseil_cpp.cmake | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/cmake/external/abseil_cpp.cmake b/tensorflow/contrib/cmake/external/abseil_cpp.cmake
index b85fd48f0f..6c6a5df7f7 100644
--- a/tensorflow/contrib/cmake/external/abseil_cpp.cmake
+++ b/tensorflow/contrib/cmake/external/abseil_cpp.cmake
@@ -31,8 +31,8 @@ if (systemlib_ABSEIL_CPP)
   message(STATUS "  abseil_cpp includes: ${ABSEIL_CPP_INCLUDE_DIR}")
   message(STATUS "  abseil_cpp libraries: ${ABSEIL_CPP_LIBRARIES}")
 
-  add_custom_target(abseil_cpp)
-  list(APPEND tensorflow_EXTERNAL_DEPENDENCIES abseil_cpp)
+  add_custom_target(abseil_cpp_build)
+  list(APPEND tensorflow_EXTERNAL_DEPENDENCIES abseil_cpp_build)
 
 else (systemlib_ABSEIL_CPP)
 
@@ -53,6 +53,7 @@ else (systemlib_ABSEIL_CPP)
           ${abseil_cpp_BUILD}/absl/numeric/Release/absl_int128.lib
           ${abseil_cpp_BUILD}/absl/strings/Release/absl_strings.lib
           ${abseil_cpp_BUILD}/absl/strings/Release/str_format_internal.lib
+          ${abseil_cpp_BUILD}/absl/time/Release/absl_time.lib
           ${abseil_cpp_BUILD}/absl/types/Release/absl_bad_optional_access.lib)
     else()
       set(abseil_cpp_STATIC_LIBRARIES
@@ -64,6 +65,7 @@ else (systemlib_ABSEIL_CPP)
           ${abseil_cpp_BUILD}/absl/numeric/absl_int128.lib
           ${abseil_cpp_BUILD}/absl/strings/absl_strings.lib
           ${abseil_cpp_BUILD}/absl/strings/str_format_internal.lib
+          ${abseil_cpp_BUILD}/absl/time/absl_time.lib
           ${abseil_cpp_BUILD}/absl/types/absl_bad_optional_access.lib)
     endif()
   else()
@@ -76,14 +78,18 @@ else (systemlib_ABSEIL_CPP)
         ${abseil_cpp_BUILD}/absl/numeric/libabsl_int128.a
         ${abseil_cpp_BUILD}/absl/strings/libabsl_strings.a
         ${abseil_cpp_BUILD}/absl/strings/libstr_format_internal.a
+        ${abseil_cpp_BUILD}/absl/time/libabsl_time.a
         ${abseil_cpp_BUILD}/absl/types/libabsl_bad_optional_access.a)
   endif()
 
-  ExternalProject_Add(abseil_cpp
+  ExternalProject_Add(abseil_cpp_build
       PREFIX abseil_cpp
       GIT_REPOSITORY ${abseil_cpp_URL}
       DOWNLOAD_DIR "${DOWNLOAD_LOCATION}"
+      BUILD_IN_SOURCE 1
       BUILD_BYPRODUCTS ${abseil_cpp_STATIC_LIBRARIES}
+      BUILD_COMMAND ${CMAKE_COMMAND} --build . --config Release
+      COMMAND ${CMAKE_COMMAND} --build . --config Release
       INSTALL_COMMAND ""
       CMAKE_CACHE_ARGS
           -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=${tensorflow_ENABLE_POSITION_INDEPENDENT_CODE}
@@ -96,6 +102,6 @@ else (systemlib_ABSEIL_CPP)
 
   list(APPEND tensorflow_EXTERNAL_LIBRARIES ${abseil_cpp_STATIC_LIBRARIES})
 
-  list(APPEND tensorflow_EXTERNAL_DEPENDENCIES abseil_cpp)
+  list(APPEND tensorflow_EXTERNAL_DEPENDENCIES abseil_cpp_build)
 
 endif (systemlib_ABSEIL_CPP)
\ No newline at end of file
-- 
GitLab


From 568db5fb164092eaf2863f0e4a2fdb48533d76f9 Mon Sep 17 00:00:00 2001
From: Siju Samuel <siju.samuel@huawei.com>
Date: Mon, 19 Nov 2018 07:20:36 +0530
Subject: [PATCH 0040/2345] Abs, Ceil ops initial

---
 tensorflow/lite/build_def.bzl                 |   1 +
 tensorflow/lite/builtin_ops.h                 |   1 +
 .../lite/core/api/flatbuffer_conversions.cc   |   2 +
 .../lite/delegates/nnapi/nnapi_delegate.cc    |  20 ++++
 .../delegates/nnapi/nnapi_delegate_test.cc    | 109 ++++++++++++++++++
 .../writer/option_writer_generator.cc         |   2 +
 tensorflow/lite/g3doc/tf_ops_compatibility.md |  22 ++++
 tensorflow/lite/kernels/BUILD                 |  34 ++++++
 tensorflow/lite/kernels/abs.cc                |  59 ++++++++++
 tensorflow/lite/kernels/abs_test.cc           |  94 +++++++++++++++
 tensorflow/lite/kernels/ceil.cc               |  59 ++++++++++
 tensorflow/lite/kernels/ceil_test.cc          |  83 +++++++++++++
 .../internal/optimized/legacy_optimized_ops.h |  12 ++
 .../internal/optimized/optimized_ops.h        |  16 +++
 .../internal/reference/legacy_reference_ops.h |  12 ++
 .../internal/reference/reference_ops.h        |  20 ++++
 tensorflow/lite/kernels/register.cc           |   4 +
 tensorflow/lite/nnapi/NeuralNetworksShim.h    |   2 +
 tensorflow/lite/nnapi_delegate.cc             |   6 +
 tensorflow/lite/schema/schema.fbs             |   1 +
 tensorflow/lite/schema/schema_generated.h     |  10 +-
 tensorflow/lite/testing/generate_examples.py  |  52 +++++++++
 tensorflow/lite/toco/export_tensorflow.cc     |  26 +++++
 .../propagate_fixed_sizes.cc                  |   2 +
 .../reorder_elementwise_unary.cc              |   2 +
 tensorflow/lite/toco/import_tensorflow.cc     |  29 +++++
 tensorflow/lite/toco/model.h                  |  22 ++++
 tensorflow/lite/toco/tflite/operator.cc       |   4 +
 tensorflow/lite/toco/tflite/operator_test.cc  |   2 +
 .../lite/toco/tflite/whitelisted_flex_ops.cc  |   1 +
 tensorflow/lite/toco/tooling_util.cc          |   2 +
 31 files changed, 708 insertions(+), 3 deletions(-)
 create mode 100644 tensorflow/lite/kernels/abs.cc
 create mode 100644 tensorflow/lite/kernels/abs_test.cc
 create mode 100644 tensorflow/lite/kernels/ceil.cc
 create mode 100644 tensorflow/lite/kernels/ceil_test.cc

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index c17eddf47b..1539ac788a 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -228,6 +228,7 @@ def generated_test_models():
         "arg_min_max",
         "avg_pool",
         "batch_to_space_nd",
+        "ceil",
         "concat",
         "constant",
         "control_dep",
diff --git a/tensorflow/lite/builtin_ops.h b/tensorflow/lite/builtin_ops.h
index f97d3ac4bf..0077fade40 100644
--- a/tensorflow/lite/builtin_ops.h
+++ b/tensorflow/lite/builtin_ops.h
@@ -128,6 +128,7 @@ typedef enum {
   kTfLiteBuiltinMirrorPad = 100,
   kTfLiteBuiltinAbs = 101,
   kTfLiteBuiltinSplitV = 102,
+  kTfLiteBuiltinCeil = 103,
 } TfLiteBuiltinOperator;
 
 #ifdef __cplusplus
diff --git a/tensorflow/lite/core/api/flatbuffer_conversions.cc b/tensorflow/lite/core/api/flatbuffer_conversions.cc
index c00a0a3a54..8a436c440f 100644
--- a/tensorflow/lite/core/api/flatbuffer_conversions.cc
+++ b/tensorflow/lite/core/api/flatbuffer_conversions.cc
@@ -664,6 +664,8 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type,
     case BuiltinOperator_EQUAL:
     case BuiltinOperator_EXP:
     case BuiltinOperator_EXPAND_DIMS:
+    case BuiltinOperator_ABS:
+    case BuiltinOperator_CEIL:
     case BuiltinOperator_FLOOR:
     case BuiltinOperator_GREATER:
     case BuiltinOperator_GREATER_EQUAL:
diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
index 4fe07004a8..1a95ac1891 100644
--- a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
+++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
@@ -638,6 +638,26 @@ class NNAPIDelegateKernel {
           return nullptr;
         }
         break;
+      case kTfLiteBuiltinCeil:
+        if (version == 1) {
+          return [](const NNAPIOpMappingArgs& mapping_args)
+                     -> ANeuralNetworksOperationType {
+            return ANEURALNETWORKS_CEIL;
+          };
+        } else {
+          return nullptr;
+        }
+        break;
+      case kTfLiteBuiltinAbs:
+        if (version == 1) {
+          return [](const NNAPIOpMappingArgs& mapping_args)
+                     -> ANeuralNetworksOperationType {
+            return ANEURALNETWORKS_ABS;
+          };
+        } else {
+          return nullptr;
+        }
+        break;
       case kTfLiteBuiltinRelu:
         if (version == 1) {
           return [](const NNAPIOpMappingArgs& mapping_args)
diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
index ca48af0c95..3050b69028 100644
--- a/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
+++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
@@ -1024,6 +1024,115 @@ TEST(NNAPIDelegate, FloorMultiDims) {
   EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 1, 1, 5}));
 }
 
+class CeilOpModel : public SingleOpModelWithNNAPI {
+ public:
+  CeilOpModel(std::initializer_list<int> input_shape, TensorType input_type) {
+    input_ = AddInput(TensorType_FLOAT32);
+    output_ = AddOutput(TensorType_FLOAT32);
+    SetBuiltinOp(BuiltinOperator_CEIL, BuiltinOptions_NONE, 0);
+    BuildInterpreter({
+        input_shape,
+    });
+  }
+
+  int input() { return input_; }
+
+  std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
+  std::vector<int> GetOutputShape() { return GetTensorShape(output_); }
+
+ private:
+  int input_;
+  int output_;
+};
+
+TEST(NNAPIDelegate, CeilSingleDim) {
+  CeilOpModel model({2}, TensorType_FLOAT32);
+  model.PopulateTensor<float>(model.input(), {8.5, 0.0});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({8, 0}));
+  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2}));
+}
+
+TEST(NNAPIDelegate, CeilMultiDims) {
+  CeilOpModel model({2, 1, 1, 5}, TensorType_FLOAT32);
+  model.PopulateTensor<float>(model.input(), {
+                                                 0.0001,
+                                                 8.0001,
+                                                 0.9999,
+                                                 9.9999,
+                                                 0.5,
+                                                 -0.0001,
+                                                 -8.0001,
+                                                 -0.9999,
+                                                 -9.9999,
+                                                 -0.5,
+                                             });
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(),
+              ElementsAreArray({1, 9, 1, 10, 1, 0, -8, 0, -9, 0}));
+  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 1, 1, 5}));
+}
+
+class AbsOpModel : public SingleOpModelWithNNAPI {
+ public:
+  AbsOpModel(std::initializer_list<int> input_shape, TensorType input_type) {
+    input_ = AddInput(TensorType_FLOAT32);
+    output_ = AddOutput(TensorType_FLOAT32);
+    SetBuiltinOp(BuiltinOperator_ABS, BuiltinOptions_NONE, 0);
+    BuildInterpreter({
+        input_shape,
+    });
+  }
+
+  int input() { return input_; }
+
+  std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
+  std::vector<int> GetOutputShape() { return GetTensorShape(output_); }
+
+ private:
+  int input_;
+  int output_;
+};
+
+TEST(NNAPIDelegate, AbsSingleDim) {
+  AbsOpModel model({2}, TensorType_FLOAT32);
+  model.PopulateTensor<float>(model.input(), {8.5, -2.0});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({8.5, 2.0}));
+  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2}));
+}
+
+TEST(NNAPIDelegate, AbsMultiDims) {
+  AbsOpModel model({2, 1, 1, 5}, TensorType_FLOAT32);
+  model.PopulateTensor<float>(model.input(), {
+                                                 0.0001,
+                                                 8.0001,
+                                                 0.9999,
+                                                 9.9999,
+                                                 0.5,
+                                                 -0.0001,
+                                                 -8.0001,
+                                                 -0.9999,
+                                                 -9.9999,
+                                                 -0.5,
+                                             });
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(),
+              ElementsAreArray({
+                                                 0.0001,
+                                                 8.0001,
+                                                 0.9999,
+                                                 9.9999,
+                                                 0.5,
+                                                 0.0001,
+                                                 8.0001,
+                                                 0.9999,
+                                                 9.9999,
+                                                 0.5,
+                                             }));
+  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 1, 1, 5}));
+}
+
 class LocalResponseNormOpModel : public SingleOpModelWithNNAPI {
  public:
   LocalResponseNormOpModel(std::initializer_list<int> input_shape, int radius,
diff --git a/tensorflow/lite/experimental/writer/option_writer_generator.cc b/tensorflow/lite/experimental/writer/option_writer_generator.cc
index fa360a2f47..d79b6e8ca9 100644
--- a/tensorflow/lite/experimental/writer/option_writer_generator.cc
+++ b/tensorflow/lite/experimental/writer/option_writer_generator.cc
@@ -160,6 +160,8 @@ class OpOptionData {
     op_to_option_["EMBEDDING_LOOKUP"] =
         "";  // TODO(aselle): maybe something else.
     op_to_option_["FLOOR"] = "";
+    op_to_option_["CEIL"] = "";
+    op_to_option_["ABS"] = "";
     op_to_option_["HASHTABLE_LOOKUP"] =
         "";  // TODO(aselle): maybe something else.
     op_to_option_["LOGISTIC"] = "";
diff --git a/tensorflow/lite/g3doc/tf_ops_compatibility.md b/tensorflow/lite/g3doc/tf_ops_compatibility.md
index dcfda72137..dc6ba8a463 100644
--- a/tensorflow/lite/g3doc/tf_ops_compatibility.md
+++ b/tensorflow/lite/g3doc/tf_ops_compatibility.md
@@ -362,6 +362,28 @@ Outputs {
 }
 ```
 
+**CEIL**
+
+```
+inputs {
+  0: tensor
+}
+outputs: {
+  0: result of computing element-wise ceil of the input tensor
+}
+```
+
+**ABS**
+
+```
+inputs {
+  0: tensor
+}
+outputs: {
+  0: result of computing element-wise absolute value of the input tensor
+}
+```
+
 **FULLY_CONNECTED**
 
 ```
diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index bad1c4aebf..f66f69870f 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -158,6 +158,7 @@ cc_library(
 cc_library(
     name = "builtin_op_kernels",
     srcs = [
+        "abs.cc",
         "activations.cc",
         "add.cc",
         "arg_min_max.cc",
@@ -167,6 +168,7 @@ cc_library(
         "bidirectional_sequence_lstm.cc",
         "bidirectional_sequence_rnn.cc",
         "cast.cc",
+        "ceil.cc",
         "comparisons.cc",
         "concatenation.cc",
         "conv.cc",
@@ -599,6 +601,38 @@ tf_cc_test(
     ],
 )
 
+tf_cc_test(
+    name = "abs_test",
+    size = "small",
+    srcs = ["abs_test.cc"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
+    deps = [
+        ":builtin_ops",
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite/kernels:test_util",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
+tf_cc_test(
+    name = "ceil_test",
+    size = "small",
+    srcs = ["ceil_test.cc"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
+    deps = [
+        ":builtin_ops",
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite/kernels:test_util",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
 tf_cc_test(
     name = "elementwise_test",
     size = "small",
diff --git a/tensorflow/lite/kernels/abs.cc b/tensorflow/lite/kernels/abs.cc
new file mode 100644
index 0000000000..b44f9de630
--- /dev/null
+++ b/tensorflow/lite/kernels/abs.cc
@@ -0,0 +1,59 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/kernels/internal/optimized/optimized_ops.h"
+#include "tensorflow/lite/kernels/internal/tensor.h"
+#include "tensorflow/lite/kernels/kernel_util.h"
+
+namespace tflite {
+namespace ops {
+namespace builtin {
+namespace abs {
+
+constexpr int kInputTensor = 0;
+constexpr int kOutputTensor = 0;
+
+TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
+  const TfLiteTensor* input = GetInput(context, node, kInputTensor);
+  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+  TF_LITE_ENSURE_EQ(context, NumInputs(node), 1);
+  TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);
+  TF_LITE_ENSURE_EQ(context, input->type, kTfLiteFloat32);
+  output->type = input->type;
+  TfLiteIntArray* output_size = TfLiteIntArrayCopy(input->dims);
+  return context->ResizeTensor(context, output, output_size);
+}
+
+TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
+  const TfLiteTensor* input = GetInput(context, node, kInputTensor);
+  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+
+  optimized_ops::Abs(GetTensorShape(input), GetTensorData<float>(input),
+                       GetTensorShape(output), GetTensorData<float>(output));
+
+  return kTfLiteOk;
+}
+}  // namespace abs
+
+TfLiteRegistration* Register_ABS() {
+  static TfLiteRegistration r = {/*init=*/nullptr,
+                                 /*free=*/nullptr, abs::Prepare, abs::Eval};
+  return &r;
+}
+
+}  // namespace builtin
+}  // namespace ops
+}  // namespace tflite
diff --git a/tensorflow/lite/kernels/abs_test.cc b/tensorflow/lite/kernels/abs_test.cc
new file mode 100644
index 0000000000..5e3f95c43a
--- /dev/null
+++ b/tensorflow/lite/kernels/abs_test.cc
@@ -0,0 +1,94 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <gtest/gtest.h>
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/kernels/test_util.h"
+#include "tensorflow/lite/model.h"
+
+namespace tflite {
+namespace {
+
+using ::testing::ElementsAreArray;
+
+class AbsOpModel : public SingleOpModel {
+ public:
+  AbsOpModel(std::initializer_list<int> input_shape, TensorType input_type) {
+    input_ = AddInput(TensorType_FLOAT32);
+    output_ = AddOutput(TensorType_FLOAT32);
+    SetBuiltinOp(BuiltinOperator_ABS, BuiltinOptions_NONE, 0);
+    BuildInterpreter({
+        input_shape,
+    });
+  }
+
+  int input() { return input_; }
+
+  std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
+  std::vector<int> GetOutputShape() { return GetTensorShape(output_); }
+
+ private:
+  int input_;
+  int output_;
+};
+
+TEST(AbsOpTest, SingleDim) {
+  AbsOpModel model({2}, TensorType_FLOAT32);
+  model.PopulateTensor<float>(model.input(), {8.5, -2.0});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({8.5, 2.0}));
+  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2}));
+}
+
+TEST(AbsOpTest, MultiDims) {
+  AbsOpModel model({2, 1, 1, 5}, TensorType_FLOAT32);
+  model.PopulateTensor<float>(model.input(), {
+                                                 0.0001,
+                                                 8.0001,
+                                                 0.9999,
+                                                 9.9999,
+                                                 0.5,
+                                                 -0.0001,
+                                                 -8.0001,
+                                                 -0.9999,
+                                                 -9.9999,
+                                                 -0.5,
+                                             });
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(),
+              ElementsAreArray({
+                                                 0.0001,
+                                                 8.0001,
+                                                 0.9999,
+                                                 9.9999,
+                                                 0.5,
+                                                 0.0001,
+                                                 8.0001,
+                                                 0.9999,
+                                                 9.9999,
+                                                 0.5,
+                                             }));
+  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 1, 1, 5}));
+}
+
+}  // namespace
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/lite/kernels/ceil.cc b/tensorflow/lite/kernels/ceil.cc
new file mode 100644
index 0000000000..e0ea061f25
--- /dev/null
+++ b/tensorflow/lite/kernels/ceil.cc
@@ -0,0 +1,59 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/kernels/internal/optimized/optimized_ops.h"
+#include "tensorflow/lite/kernels/internal/tensor.h"
+#include "tensorflow/lite/kernels/kernel_util.h"
+
+namespace tflite {
+namespace ops {
+namespace builtin {
+namespace ceil {
+
+constexpr int kInputTensor = 0;
+constexpr int kOutputTensor = 0;
+
+TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
+  const TfLiteTensor* input = GetInput(context, node, kInputTensor);
+  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+  TF_LITE_ENSURE_EQ(context, NumInputs(node), 1);
+  TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);
+  TF_LITE_ENSURE_EQ(context, input->type, kTfLiteFloat32);
+  output->type = input->type;
+  TfLiteIntArray* output_size = TfLiteIntArrayCopy(input->dims);
+  return context->ResizeTensor(context, output, output_size);
+}
+
+TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
+  const TfLiteTensor* input = GetInput(context, node, kInputTensor);
+  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+
+  optimized_ops::Ceil(GetTensorShape(input), GetTensorData<float>(input),
+                       GetTensorShape(output), GetTensorData<float>(output));
+
+  return kTfLiteOk;
+}
+}  // namespace ceil
+
+TfLiteRegistration* Register_CEIL() {
+  static TfLiteRegistration r = {/*init=*/nullptr,
+                                 /*free=*/nullptr, ceil::Prepare, ceil::Eval};
+  return &r;
+}
+
+}  // namespace builtin
+}  // namespace ops
+}  // namespace tflite
diff --git a/tensorflow/lite/kernels/ceil_test.cc b/tensorflow/lite/kernels/ceil_test.cc
new file mode 100644
index 0000000000..e120105082
--- /dev/null
+++ b/tensorflow/lite/kernels/ceil_test.cc
@@ -0,0 +1,83 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <gtest/gtest.h>
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/kernels/test_util.h"
+#include "tensorflow/lite/model.h"
+
+namespace tflite {
+namespace {
+
+using ::testing::ElementsAreArray;
+
+class CeilOpModel : public SingleOpModel {
+ public:
+  CeilOpModel(std::initializer_list<int> input_shape, TensorType input_type) {
+    input_ = AddInput(TensorType_FLOAT32);
+    output_ = AddOutput(TensorType_FLOAT32);
+    SetBuiltinOp(BuiltinOperator_CEIL, BuiltinOptions_NONE, 0);
+    BuildInterpreter({
+        input_shape,
+    });
+  }
+
+  int input() { return input_; }
+
+  std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
+  std::vector<int> GetOutputShape() { return GetTensorShape(output_); }
+
+ private:
+  int input_;
+  int output_;
+};
+
+TEST(CeilOpTest, SingleDim) {
+  CeilOpModel model({2}, TensorType_FLOAT32);
+  model.PopulateTensor<float>(model.input(), {8.5, 0.0});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({9, 0}));
+  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2}));
+}
+
+TEST(CeilOpTest, MultiDims) {
+  CeilOpModel model({2, 1, 1, 5}, TensorType_FLOAT32);
+  model.PopulateTensor<float>(model.input(), {
+                                                 0.0001,
+                                                 8.0001,
+                                                 0.9999,
+                                                 9.9999,
+                                                 0.5,
+                                                 -0.0001,
+                                                 -8.0001,
+                                                 -0.9999,
+                                                 -9.9999,
+                                                 -0.5,
+                                             });
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(),
+              ElementsAreArray({1, 9, 1, 10, 1, 0, -8, 0, -9, 0}));
+  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 1, 1, 5}));
+}
+
+}  // namespace
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h b/tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h
index 5485d907c2..c24b7faa21 100644
--- a/tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h
+++ b/tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h
@@ -1740,6 +1740,18 @@ inline void Floor(const float* input_data, const Dims<4>& input_dims,
         output_data);
 }
 
+inline void Ceil(const float* input_data, const Dims<4>& input_dims,
+                 float* output_data, const Dims<4>& output_dims) {
+  Ceil(DimsToShape(input_dims), input_data, DimsToShape(output_dims),
+       output_data);
+}
+
+inline void Abs(const float* input_data, const Dims<4>& input_dims,
+                float* output_data, const Dims<4>& output_dims) {
+  Abs(DimsToShape(input_dims), input_data, DimsToShape(output_dims),
+      output_data);
+}
+
 inline void ResizeBilinear(const float* input_data, const Dims<4>& input_dims,
                            const int32* output_size_data,
                            const Dims<4>& output_size_dims, float* output_data,
diff --git a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
index c79b69a22e..7f7296ae97 100644
--- a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
+++ b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
@@ -4898,6 +4898,22 @@ inline void Floor(const RuntimeShape& input_shape, const float* input_data,
   output_map.array() = Eigen::floor(input_map.array());
 }
 
+inline void Ceil(const RuntimeShape& input_shape, const float* input_data,
+                 const RuntimeShape& output_shape, float* output_data) {
+  gemmlowp::ScopedProfilingLabel label("Ceil");
+  auto input_map = MapAsVector(input_data, input_shape);
+  auto output_map = MapAsVector(output_data, output_shape);
+  output_map.array() = Eigen::ceil(input_map.array());
+}
+
+inline void Abs(const RuntimeShape& input_shape, const float* input_data,
+                const RuntimeShape& output_shape, float* output_data) {
+  gemmlowp::ScopedProfilingLabel label("Abs");
+  auto input_map = MapAsVector(input_data, input_shape);
+  auto output_map = MapAsVector(output_data, output_shape);
+  output_map.array() = Eigen::abs(input_map.array());
+}
+
 #ifdef USE_NEON
 inline void ResizeBilinearKernel(const float* input_ptr, int32 depth,
                                  float scale, float* output_ptr) {
diff --git a/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h b/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h
index 380fc8f98e..30bb92c8b3 100644
--- a/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h
+++ b/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h
@@ -1883,6 +1883,18 @@ inline void Floor(const float* input_data, const Dims<4>& input_dims,
         output_data);
 }
 
+inline void Ceil(const float* input_data, const Dims<4>& input_dims,
+                 float* output_data, const Dims<4>& output_dims) {
+  Ceil(DimsToShape(input_dims), input_data, DimsToShape(output_dims),
+       output_data);
+}
+
+inline void Abs(const float* input_data, const Dims<4>& input_dims,
+                float* output_data, const Dims<4>& output_dims) {
+  Abs(DimsToShape(input_dims), input_data, DimsToShape(output_dims),
+      output_data);
+}
+
 template <typename T>
 inline void ResizeBilinear(const T* input_data, const Dims<4>& input_dims,
                            const int32* output_size_data,
diff --git a/tensorflow/lite/kernels/internal/reference/reference_ops.h b/tensorflow/lite/kernels/internal/reference/reference_ops.h
index ea3ab06da1..133a455e9e 100644
--- a/tensorflow/lite/kernels/internal/reference/reference_ops.h
+++ b/tensorflow/lite/kernels/internal/reference/reference_ops.h
@@ -3040,6 +3040,26 @@ inline void Floor(const RuntimeShape& input_shape, const float* input_data,
   }
 }
 
+inline void Ceil(const RuntimeShape& input_shape, const float* input_data,
+                 const RuntimeShape& output_shape, float* output_data) {
+  const int flat_size = MatchingFlatSize(input_shape, output_shape);
+
+  for (int i = 0; i < flat_size; i++) {
+    int offset = i;
+    output_data[offset] = std::ceil(input_data[offset]);
+  }
+}
+
+inline void Abs(const RuntimeShape& input_shape, const float* input_data,
+                const RuntimeShape& output_shape, float* output_data) {
+  const int flat_size = MatchingFlatSize(input_shape, output_shape);
+
+  for (int i = 0; i < flat_size; i++) {
+    int offset = i;
+    output_data[offset] = std::fabs(input_data[offset]);
+  }
+}
+
 template <typename T, typename CoordsT = int32>
 inline void Gather(const tflite::GatherParams& op_params,
                    const RuntimeShape& input_shape, const T* input_data,
diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index c0e6f6994f..caf61e9003 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -94,6 +94,8 @@ TfLiteRegistration* Register_GREATER_EQUAL();
 TfLiteRegistration* Register_LESS();
 TfLiteRegistration* Register_LESS_EQUAL();
 TfLiteRegistration* Register_FLOOR();
+TfLiteRegistration* Register_CEIL();
+TfLiteRegistration* Register_ABS();
 TfLiteRegistration* Register_TILE();
 TfLiteRegistration* Register_NEG();
 TfLiteRegistration* Register_SUM();
@@ -235,6 +237,8 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_LESS, Register_LESS());
   AddBuiltin(BuiltinOperator_LESS_EQUAL, Register_LESS_EQUAL());
   AddBuiltin(BuiltinOperator_FLOOR, Register_FLOOR());
+  AddBuiltin(BuiltinOperator_CEIL, Register_CEIL());
+  AddBuiltin(BuiltinOperator_ABS, Register_ABS());
   AddBuiltin(BuiltinOperator_NEG, Register_NEG());
   AddBuiltin(BuiltinOperator_SELECT, Register_SELECT());
   AddBuiltin(BuiltinOperator_SLICE, Register_SLICE());
diff --git a/tensorflow/lite/nnapi/NeuralNetworksShim.h b/tensorflow/lite/nnapi/NeuralNetworksShim.h
index c39502f4ac..037d364c3b 100644
--- a/tensorflow/lite/nnapi/NeuralNetworksShim.h
+++ b/tensorflow/lite/nnapi/NeuralNetworksShim.h
@@ -143,6 +143,8 @@ enum {
   ANEURALNETWORKS_STRIDED_SLICE = 35,
   ANEURALNETWORKS_SUB = 36,
   ANEURALNETWORKS_TRANSPOSE = 37,
+  ANEURALNETWORKS_CEIL = 38,
+  ANEURALNETWORKS_ABS = 39,
 };
 
 /**
diff --git a/tensorflow/lite/nnapi_delegate.cc b/tensorflow/lite/nnapi_delegate.cc
index 26d75696a1..6da803ce7b 100644
--- a/tensorflow/lite/nnapi_delegate.cc
+++ b/tensorflow/lite/nnapi_delegate.cc
@@ -489,6 +489,12 @@ TfLiteStatus AddOpsAndParams(
       case tflite::BuiltinOperator_FLOOR:
         nn_op_type = ANEURALNETWORKS_FLOOR;
         break;
+      case tflite::BuiltinOperator_CEIL:
+        nn_op_type = ANEURALNETWORKS_CEIL;
+        break;
+      case tflite::BuiltinOperator_ABS:
+        nn_op_type = ANEURALNETWORKS_ABS;
+        break;
       case tflite::BuiltinOperator_LOGISTIC:
         nn_op_type = ANEURALNETWORKS_LOGISTIC;
         break;
diff --git a/tensorflow/lite/schema/schema.fbs b/tensorflow/lite/schema/schema.fbs
index 980f13b19b..2d27722415 100644
--- a/tensorflow/lite/schema/schema.fbs
+++ b/tensorflow/lite/schema/schema.fbs
@@ -205,6 +205,7 @@ enum BuiltinOperator : byte {
   MIRROR_PAD = 100,
   ABS = 101,
   SPLIT_V = 102,
+  CEIL = 103,
 }
 
 // Options for the builtin operators.
diff --git a/tensorflow/lite/schema/schema_generated.h b/tensorflow/lite/schema/schema_generated.h
index 637cbafabd..a605c19b52 100755
--- a/tensorflow/lite/schema/schema_generated.h
+++ b/tensorflow/lite/schema/schema_generated.h
@@ -520,11 +520,12 @@ enum BuiltinOperator {
   BuiltinOperator_MIRROR_PAD = 100,
   BuiltinOperator_ABS = 101,
   BuiltinOperator_SPLIT_V = 102,
+  BuiltinOperator_CEIL = 103,
   BuiltinOperator_MIN = BuiltinOperator_ADD,
-  BuiltinOperator_MAX = BuiltinOperator_SPLIT_V
+  BuiltinOperator_MAX = BuiltinOperator_CEIL
 };
 
-inline const BuiltinOperator (&EnumValuesBuiltinOperator())[102] {
+inline const BuiltinOperator (&EnumValuesBuiltinOperator())[103] {
   static const BuiltinOperator values[] = {
     BuiltinOperator_ADD,
     BuiltinOperator_AVERAGE_POOL_2D,
@@ -627,7 +628,9 @@ inline const BuiltinOperator (&EnumValuesBuiltinOperator())[102] {
     BuiltinOperator_SQUARED_DIFFERENCE,
     BuiltinOperator_MIRROR_PAD,
     BuiltinOperator_ABS,
-    BuiltinOperator_SPLIT_V
+    BuiltinOperator_SPLIT_V,
+    BuiltinOperator_CEIL
+
   };
   return values;
 }
@@ -737,6 +740,7 @@ inline const char * const *EnumNamesBuiltinOperator() {
     "MIRROR_PAD",
     "ABS",
     "SPLIT_V",
+    "CEIL",
     nullptr
   };
   return names;
diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index dd7b3d0745..825873a884 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -3021,6 +3021,58 @@ def make_floor_tests(zip_path):
   make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
 
 
+def make_ceil_tests(zip_path):
+  """Make a set of tests to do ceil."""
+
+  test_parameters = [{
+      "input_dtype": [tf.float32],
+      "input_shape": [[1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+  }]
+
+  def build_graph(parameters):
+    """Build the ceil op testing graph."""
+    input_value = tf.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input1",
+        shape=parameters["input_shape"])
+    out = tf.ceil(input_value)
+    return [input_value], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_value = create_tensor_data(parameters["input_dtype"],
+                                     parameters["input_shape"])
+    return [input_value], sess.run(
+        outputs, feed_dict={inputs[0]: input_value})
+
+  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+
+
+def make_abs_tests(zip_path):
+  """Make a set of tests to do abs."""
+
+  test_parameters = [{
+      "input_dtype": [tf.float32],
+      "input_shape": [[1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+  }]
+
+  def build_graph(parameters):
+    """Build the abs op testing graph."""
+    input_value = tf.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input1",
+        shape=parameters["input_shape"])
+    out = tf.abs(input_value)
+    return [input_value], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_value = create_tensor_data(parameters["input_dtype"],
+                                     parameters["input_shape"])
+    return [input_value], sess.run(
+        outputs, feed_dict={inputs[0]: input_value})
+
+  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+
+
 def make_neg_tests(zip_path):
   """Make a set of tests to do neg."""
 
diff --git a/tensorflow/lite/toco/export_tensorflow.cc b/tensorflow/lite/toco/export_tensorflow.cc
index 9fff001552..9dfeaad164 100644
--- a/tensorflow/lite/toco/export_tensorflow.cc
+++ b/tensorflow/lite/toco/export_tensorflow.cc
@@ -1205,6 +1205,26 @@ void ConvertFloorOperator(const Model& model, const FloorOperator& src_op,
   (*floor_op->mutable_attr())["T"].set_type(DT_FLOAT);
 }
 
+void ConvertCeilOperator(const Model& model, const CeilOperator& src_op,
+                         GraphDef* tensorflow_graph) {
+  tensorflow::NodeDef* ceil_op = tensorflow_graph->add_node();
+  ceil_op->set_op("Ceil");
+  ceil_op->set_name(src_op.outputs[0]);
+  CHECK_EQ(src_op.inputs.size(), 1);
+  *ceil_op->add_input() = src_op.inputs[0];
+  (*ceil_op->mutable_attr())["T"].set_type(DT_FLOAT);
+}
+
+void ConvertAbsOperator(const Model& model, const AbsOperator& src_op,
+                        GraphDef* tensorflow_graph) {
+  tensorflow::NodeDef* abs_op = tensorflow_graph->add_node();
+  abs_op->set_op("Abs");
+  abs_op->set_name(src_op.outputs[0]);
+  CHECK_EQ(src_op.inputs.size(), 1);
+  *abs_op->add_input() = src_op.inputs[0];
+  (*abs_op->mutable_attr())["T"].set_type(DT_FLOAT);
+}
+
 void ConvertGatherOperator(const Model& model, const GatherOperator& src_op,
                            GraphDef* tensorflow_graph) {
   tensorflow::NodeDef* gather_op = tensorflow_graph->add_node();
@@ -2169,6 +2189,12 @@ void ConvertOperator(const Model& model, const Operator& src_op,
   } else if (src_op.type == OperatorType::kFloor) {
     ConvertFloorOperator(model, static_cast<const FloorOperator&>(src_op),
                          tensorflow_graph);
+  } else if (src_op.type == OperatorType::kCeil) {
+    ConvertCeilOperator(model, static_cast<const CeilOperator&>(src_op),
+                        tensorflow_graph);
+  } else if (src_op.type == OperatorType::kAbs) {
+    ConvertAbsOperator(model, static_cast<const AbsOperator&>(src_op),
+                       tensorflow_graph);
   } else if (src_op.type == OperatorType::kGather) {
     ConvertGatherOperator(model, static_cast<const GatherOperator&>(src_op),
                           tensorflow_graph);
diff --git a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
index 0e653f08a0..c6c7681c03 100644
--- a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
+++ b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
@@ -1869,6 +1869,8 @@ void ProcessMirrorPadOperator(Model* model, MirrorPadOperator* op) {
     case OperatorType::kAssert:
     case OperatorType::kCast:
     case OperatorType::kFloor:
+    case OperatorType::kCeil:
+    case OperatorType::kAbs:
     case OperatorType::kExp:
     case OperatorType::kSin:
     case OperatorType::kLogicalAnd:
diff --git a/tensorflow/lite/toco/graph_transformations/reorder_elementwise_unary.cc b/tensorflow/lite/toco/graph_transformations/reorder_elementwise_unary.cc
index 6a4b919854..90e64816c8 100644
--- a/tensorflow/lite/toco/graph_transformations/reorder_elementwise_unary.cc
+++ b/tensorflow/lite/toco/graph_transformations/reorder_elementwise_unary.cc
@@ -29,7 +29,9 @@ namespace {
 
 bool IsElementwiseOperator(OperatorType optype) {
   switch (optype) {
+    case OperatorType::kAbs:
     case OperatorType::kCast:
+    case OperatorType::kCeil:
     case OperatorType::kExp:
     case OperatorType::kFloor:
     case OperatorType::kNeg:
diff --git a/tensorflow/lite/toco/import_tensorflow.cc b/tensorflow/lite/toco/import_tensorflow.cc
index 0b2f810394..de8ae19a71 100644
--- a/tensorflow/lite/toco/import_tensorflow.cc
+++ b/tensorflow/lite/toco/import_tensorflow.cc
@@ -1491,6 +1491,34 @@ tensorflow::Status ConvertFloorOperator(
   return tensorflow::Status::OK();
 }
 
+tensorflow::Status ConvertCeilOperator(
+    const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
+    Model* model) {
+  CHECK_EQ(node.op(), "Ceil");
+  TF_QCHECK_OK(CheckInputsCount(node, tf_import_flags, 1));
+  const auto data_type = GetDataTypeAttr(node, "T");
+  CHECK(data_type == DT_FLOAT);
+  auto* op = new CeilOperator;
+  op->inputs.push_back(node.input(0));
+  op->outputs.push_back(node.name());
+  model->operators.emplace_back(op);
+  return tensorflow::Status::OK();
+}
+
+tensorflow::Status ConvertAbsOperator(
+    const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
+    Model* model) {
+  CHECK_EQ(node.op(), "Abs");
+  TF_QCHECK_OK(CheckInputsCount(node, tf_import_flags, 1));
+  const auto data_type = GetDataTypeAttr(node, "T");
+  CHECK(data_type == DT_FLOAT);
+  auto* op = new AbsOperator;
+  op->inputs.push_back(node.input(0));
+  op->outputs.push_back(node.name());
+  model->operators.emplace_back(op);
+  return tensorflow::Status::OK();
+}
+
 tensorflow::Status ConvertGatherOperator(
     const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
     Model* model) {
@@ -2314,6 +2342,7 @@ ConverterMapType GetTensorFlowNodeConverterMap() {
       {"BatchToSpaceND", ConvertBatchToSpaceNDOperator},
       {"BiasAdd", ConvertBiasAddOperator},
       {"Cast", ConvertCastOperator},
+      {"Ceil", ConvertCeilOperator},
       {"CheckNumerics", ConvertIdentityOperator},
       {"Concat", ConvertConcatOperator},
       {"ConcatV2", ConvertConcatOperator},
diff --git a/tensorflow/lite/toco/model.h b/tensorflow/lite/toco/model.h
index d392535f5c..69c6d7608a 100644
--- a/tensorflow/lite/toco/model.h
+++ b/tensorflow/lite/toco/model.h
@@ -37,11 +37,13 @@ using tflite::QuantizationParams;
 enum class OperatorType : uint8 {
   kNone,
   // General-purpose neural network operators.
+  kAbs,
   kAdd,
   kAddN,
   kAveragePool,
   kBatchMatMul,
   kBatchNormalization,
+  kCeil,
   kConv,
   kConcatenation,
   kDepthwiseConv,
@@ -1658,6 +1660,26 @@ struct FloorOperator : Operator {
   FloorOperator() : Operator(OperatorType::kFloor) {}
 };
 
+// Ceil operator.
+//
+// Inputs:
+//   inputs[0]: required: the input array
+//
+// TensorFlow equivalent: Ceil
+struct CeilOperator : Operator {
+  CeilOperator() : Operator(OperatorType::kCeil) {}
+};
+
+// Abs operator.
+//
+// Inputs:
+//   inputs[0]: required: the input array
+//
+// TensorFlow equivalent: Abs
+struct AbsOperator : Operator {
+  AbsOperator() : Operator(OperatorType::kAbs) {}
+};
+
 // Gather operator. It gathers slices from params according to indices.
 // Only 1-D indices are supported at the moment.
 //
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index 205af23da5..2122d04958 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -1649,6 +1649,10 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
       "DEQUANTIZE", OperatorType::kDequantize));
   ops.push_back(
       MakeUnique<SimpleOperator<FloorOperator>>("FLOOR", OperatorType::kFloor));
+  ops.push_back(
+      MakeUnique<SimpleOperator<CeilOperator>>("CEIL", OperatorType::kCeil));
+  ops.push_back(
+      MakeUnique<SimpleOperator<AbsOperator>>("ABS", OperatorType::kAbs));
   ops.push_back(
       MakeUnique<SimpleOperator<ReluOperator>>("RELU", OperatorType::kRelu));
   ops.push_back(MakeUnique<SimpleOperator<Relu1Operator>>(
diff --git a/tensorflow/lite/toco/tflite/operator_test.cc b/tensorflow/lite/toco/tflite/operator_test.cc
index 14ec89cd73..e284619903 100644
--- a/tensorflow/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/lite/toco/tflite/operator_test.cc
@@ -114,6 +114,8 @@ TEST_F(OperatorTest, SimpleOperators) {
   CheckSimpleOperator<DequantizeOperator>("DEQUANTIZE",
                                           OperatorType::kDequantize);
   CheckSimpleOperator<FloorOperator>("FLOOR", OperatorType::kFloor);
+  CheckSimpleOperator<CeilOperator>("CEIL", OperatorType::kCeil);
+  CheckSimpleOperator<AbsOperator>("ABS", OperatorType::kAbs);
   CheckSimpleOperator<ReluOperator>("RELU", OperatorType::kRelu);
   CheckSimpleOperator<Relu1Operator>("RELU_N1_TO_1", OperatorType::kRelu1);
   CheckSimpleOperator<Relu6Operator>("RELU6", OperatorType::kRelu6);
diff --git a/tensorflow/lite/toco/tflite/whitelisted_flex_ops.cc b/tensorflow/lite/toco/tflite/whitelisted_flex_ops.cc
index 039a918af1..a96222a8a2 100644
--- a/tensorflow/lite/toco/tflite/whitelisted_flex_ops.cc
+++ b/tensorflow/lite/toco/tflite/whitelisted_flex_ops.cc
@@ -68,6 +68,7 @@ bool IsWhitelistedFlexOp(const std::string& tensorflow_op_name) {
           "BroadcastArgs",
           "BroadcastGradientArgs",
           "Cast",
+          "Ceil",
           "CheckNumerics",
           "ComplexAbs",
           "Concat",
diff --git a/tensorflow/lite/toco/tooling_util.cc b/tensorflow/lite/toco/tooling_util.cc
index af4cd386a2..0887dca9b5 100644
--- a/tensorflow/lite/toco/tooling_util.cc
+++ b/tensorflow/lite/toco/tooling_util.cc
@@ -385,6 +385,8 @@ const char* OperatorTypeName(OperatorType type) {
     HANDLE_OPERATORTYPENAME_CASE(ConcatV2)
     HANDLE_OPERATORTYPENAME_CASE(Cast)
     HANDLE_OPERATORTYPENAME_CASE(Floor)
+    HANDLE_OPERATORTYPENAME_CASE(Ceil)
+    HANDLE_OPERATORTYPENAME_CASE(Abs)
     HANDLE_OPERATORTYPENAME_CASE(Gather)
     HANDLE_OPERATORTYPENAME_CASE(ResizeBilinear)
     HANDLE_OPERATORTYPENAME_CASE(SpaceToBatchND)
-- 
GitLab


From 9e91a0899f116e8bfec46922221c45765346ce2e Mon Sep 17 00:00:00 2001
From: Yves-Noel Weweler <y.weweler@gmail.com>
Date: Wed, 12 Dec 2018 15:27:09 +0100
Subject: [PATCH 0041/2345] Update tensorflow.data.experimental goldens for v1

Add the drop_remainder argument for bucket_by_sequence_length to the goldens.
---
 .../tools/api/golden/v1/tensorflow.data.experimental.pbtxt      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
index ad10b82283..b8ba3e341f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
@@ -54,7 +54,7 @@ tf_module {
   }
   member_method {
     name: "bucket_by_sequence_length"
-    argspec: "args=[\'element_length_func\', \'bucket_boundaries\', \'bucket_batch_sizes\', \'padded_shapes\', \'padding_values\', \'pad_to_bucket_boundary\', \'no_padding\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'False\'], "
+    argspec: "args=[\'element_length_func\', \'bucket_boundaries\', \'bucket_batch_sizes\', \'padded_shapes\', \'padding_values\', \'pad_to_bucket_boundary\', \'no_padding\', \'drop_remainder\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'False\', \'False\'], "
   }
   member_method {
     name: "choose_from_datasets"
-- 
GitLab


From c266087d4d086913194bd621cad125853eca8fd6 Mon Sep 17 00:00:00 2001
From: Yves-Noel Weweler <y.weweler@gmail.com>
Date: Wed, 12 Dec 2018 15:29:25 +0100
Subject: [PATCH 0042/2345] Update tensorflow.data.experimental goldens for v2

Add the drop_remainder argument for bucket_by_sequence_length to the goldens.
---
 .../tools/api/golden/v2/tensorflow.data.experimental.pbtxt      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
index ad10b82283..b8ba3e341f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
@@ -54,7 +54,7 @@ tf_module {
   }
   member_method {
     name: "bucket_by_sequence_length"
-    argspec: "args=[\'element_length_func\', \'bucket_boundaries\', \'bucket_batch_sizes\', \'padded_shapes\', \'padding_values\', \'pad_to_bucket_boundary\', \'no_padding\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'False\'], "
+    argspec: "args=[\'element_length_func\', \'bucket_boundaries\', \'bucket_batch_sizes\', \'padded_shapes\', \'padding_values\', \'pad_to_bucket_boundary\', \'no_padding\', \'drop_remainder\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'False\', \'False\'], "
   }
   member_method {
     name: "choose_from_datasets"
-- 
GitLab


From e30a0a1de0b0b51da69b3d4bf5c545142accb3c2 Mon Sep 17 00:00:00 2001
From: Siju Samuel <siju.samuel@huawei.com>
Date: Wed, 12 Dec 2018 20:03:36 +0530
Subject: [PATCH 0043/2345] Removed abs since already committed in
 5916a9f0e4b5b2c4f80767ff83a001a6f86b4395

---
 .../lite/core/api/flatbuffer_conversions.cc   |  1 -
 .../lite/delegates/nnapi/nnapi_delegate.cc    | 10 --
 .../delegates/nnapi/nnapi_delegate_test.cc    | 60 ------------
 .../writer/option_writer_generator.cc         |  1 -
 tensorflow/lite/g3doc/tf_ops_compatibility.md | 11 ---
 tensorflow/lite/kernels/BUILD                 | 17 ----
 tensorflow/lite/kernels/abs.cc                | 59 ------------
 tensorflow/lite/kernels/abs_test.cc           | 94 -------------------
 .../internal/optimized/legacy_optimized_ops.h |  6 --
 .../internal/optimized/optimized_ops.h        |  8 --
 .../internal/reference/legacy_reference_ops.h |  6 --
 .../internal/reference/reference_ops.h        | 10 --
 tensorflow/lite/kernels/register.cc           |  2 -
 tensorflow/lite/nnapi/NeuralNetworksShim.h    |  1 -
 tensorflow/lite/nnapi_delegate.cc             |  3 -
 tensorflow/lite/testing/generate_examples.py  | 26 -----
 tensorflow/lite/toco/export_tensorflow.cc     | 13 ---
 .../propagate_fixed_sizes.cc                  |  1 -
 .../reorder_elementwise_unary.cc              |  1 -
 tensorflow/lite/toco/import_tensorflow.cc     | 14 ---
 tensorflow/lite/toco/model.h                  | 11 ---
 tensorflow/lite/toco/tflite/operator.cc       |  2 -
 tensorflow/lite/toco/tflite/operator_test.cc  |  1 -
 tensorflow/lite/toco/tooling_util.cc          |  1 -
 24 files changed, 359 deletions(-)
 delete mode 100644 tensorflow/lite/kernels/abs.cc
 delete mode 100644 tensorflow/lite/kernels/abs_test.cc

diff --git a/tensorflow/lite/core/api/flatbuffer_conversions.cc b/tensorflow/lite/core/api/flatbuffer_conversions.cc
index 8a436c440f..9c8eb5a2d8 100644
--- a/tensorflow/lite/core/api/flatbuffer_conversions.cc
+++ b/tensorflow/lite/core/api/flatbuffer_conversions.cc
@@ -664,7 +664,6 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type,
     case BuiltinOperator_EQUAL:
     case BuiltinOperator_EXP:
     case BuiltinOperator_EXPAND_DIMS:
-    case BuiltinOperator_ABS:
     case BuiltinOperator_CEIL:
     case BuiltinOperator_FLOOR:
     case BuiltinOperator_GREATER:
diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
index 1a95ac1891..cac98ae3da 100644
--- a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
+++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
@@ -648,16 +648,6 @@ class NNAPIDelegateKernel {
           return nullptr;
         }
         break;
-      case kTfLiteBuiltinAbs:
-        if (version == 1) {
-          return [](const NNAPIOpMappingArgs& mapping_args)
-                     -> ANeuralNetworksOperationType {
-            return ANEURALNETWORKS_ABS;
-          };
-        } else {
-          return nullptr;
-        }
-        break;
       case kTfLiteBuiltinRelu:
         if (version == 1) {
           return [](const NNAPIOpMappingArgs& mapping_args)
diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
index 3050b69028..f6a04e36cd 100644
--- a/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
+++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
@@ -1073,66 +1073,6 @@ TEST(NNAPIDelegate, CeilMultiDims) {
   EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 1, 1, 5}));
 }
 
-class AbsOpModel : public SingleOpModelWithNNAPI {
- public:
-  AbsOpModel(std::initializer_list<int> input_shape, TensorType input_type) {
-    input_ = AddInput(TensorType_FLOAT32);
-    output_ = AddOutput(TensorType_FLOAT32);
-    SetBuiltinOp(BuiltinOperator_ABS, BuiltinOptions_NONE, 0);
-    BuildInterpreter({
-        input_shape,
-    });
-  }
-
-  int input() { return input_; }
-
-  std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
-  std::vector<int> GetOutputShape() { return GetTensorShape(output_); }
-
- private:
-  int input_;
-  int output_;
-};
-
-TEST(NNAPIDelegate, AbsSingleDim) {
-  AbsOpModel model({2}, TensorType_FLOAT32);
-  model.PopulateTensor<float>(model.input(), {8.5, -2.0});
-  model.Invoke();
-  EXPECT_THAT(model.GetOutput(), ElementsAreArray({8.5, 2.0}));
-  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2}));
-}
-
-TEST(NNAPIDelegate, AbsMultiDims) {
-  AbsOpModel model({2, 1, 1, 5}, TensorType_FLOAT32);
-  model.PopulateTensor<float>(model.input(), {
-                                                 0.0001,
-                                                 8.0001,
-                                                 0.9999,
-                                                 9.9999,
-                                                 0.5,
-                                                 -0.0001,
-                                                 -8.0001,
-                                                 -0.9999,
-                                                 -9.9999,
-                                                 -0.5,
-                                             });
-  model.Invoke();
-  EXPECT_THAT(model.GetOutput(),
-              ElementsAreArray({
-                                                 0.0001,
-                                                 8.0001,
-                                                 0.9999,
-                                                 9.9999,
-                                                 0.5,
-                                                 0.0001,
-                                                 8.0001,
-                                                 0.9999,
-                                                 9.9999,
-                                                 0.5,
-                                             }));
-  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 1, 1, 5}));
-}
-
 class LocalResponseNormOpModel : public SingleOpModelWithNNAPI {
  public:
   LocalResponseNormOpModel(std::initializer_list<int> input_shape, int radius,
diff --git a/tensorflow/lite/experimental/writer/option_writer_generator.cc b/tensorflow/lite/experimental/writer/option_writer_generator.cc
index d79b6e8ca9..ba0303f9db 100644
--- a/tensorflow/lite/experimental/writer/option_writer_generator.cc
+++ b/tensorflow/lite/experimental/writer/option_writer_generator.cc
@@ -161,7 +161,6 @@ class OpOptionData {
         "";  // TODO(aselle): maybe something else.
     op_to_option_["FLOOR"] = "";
     op_to_option_["CEIL"] = "";
-    op_to_option_["ABS"] = "";
     op_to_option_["HASHTABLE_LOOKUP"] =
         "";  // TODO(aselle): maybe something else.
     op_to_option_["LOGISTIC"] = "";
diff --git a/tensorflow/lite/g3doc/tf_ops_compatibility.md b/tensorflow/lite/g3doc/tf_ops_compatibility.md
index dc6ba8a463..d20320a3b5 100644
--- a/tensorflow/lite/g3doc/tf_ops_compatibility.md
+++ b/tensorflow/lite/g3doc/tf_ops_compatibility.md
@@ -373,17 +373,6 @@ outputs: {
 }
 ```
 
-**ABS**
-
-```
-inputs {
-  0: tensor
-}
-outputs: {
-  0: result of computing element-wise absolute value of the input tensor
-}
-```
-
 **FULLY_CONNECTED**
 
 ```
diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index f66f69870f..71d06ba4d7 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -158,7 +158,6 @@ cc_library(
 cc_library(
     name = "builtin_op_kernels",
     srcs = [
-        "abs.cc",
         "activations.cc",
         "add.cc",
         "arg_min_max.cc",
@@ -601,22 +600,6 @@ tf_cc_test(
     ],
 )
 
-tf_cc_test(
-    name = "abs_test",
-    size = "small",
-    srcs = ["abs_test.cc"],
-    tags = [
-        "no_oss",
-        "tflite_not_portable_ios",
-    ],
-    deps = [
-        ":builtin_ops",
-        "//tensorflow/lite:framework",
-        "//tensorflow/lite/kernels:test_util",
-        "@com_google_googletest//:gtest",
-    ],
-)
-
 tf_cc_test(
     name = "ceil_test",
     size = "small",
diff --git a/tensorflow/lite/kernels/abs.cc b/tensorflow/lite/kernels/abs.cc
deleted file mode 100644
index b44f9de630..0000000000
--- a/tensorflow/lite/kernels/abs.cc
+++ /dev/null
@@ -1,59 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/lite/c/c_api_internal.h"
-#include "tensorflow/lite/kernels/internal/optimized/optimized_ops.h"
-#include "tensorflow/lite/kernels/internal/tensor.h"
-#include "tensorflow/lite/kernels/kernel_util.h"
-
-namespace tflite {
-namespace ops {
-namespace builtin {
-namespace abs {
-
-constexpr int kInputTensor = 0;
-constexpr int kOutputTensor = 0;
-
-TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
-  const TfLiteTensor* input = GetInput(context, node, kInputTensor);
-  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
-  TF_LITE_ENSURE_EQ(context, NumInputs(node), 1);
-  TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);
-  TF_LITE_ENSURE_EQ(context, input->type, kTfLiteFloat32);
-  output->type = input->type;
-  TfLiteIntArray* output_size = TfLiteIntArrayCopy(input->dims);
-  return context->ResizeTensor(context, output, output_size);
-}
-
-TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
-  const TfLiteTensor* input = GetInput(context, node, kInputTensor);
-  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
-
-  optimized_ops::Abs(GetTensorShape(input), GetTensorData<float>(input),
-                       GetTensorShape(output), GetTensorData<float>(output));
-
-  return kTfLiteOk;
-}
-}  // namespace abs
-
-TfLiteRegistration* Register_ABS() {
-  static TfLiteRegistration r = {/*init=*/nullptr,
-                                 /*free=*/nullptr, abs::Prepare, abs::Eval};
-  return &r;
-}
-
-}  // namespace builtin
-}  // namespace ops
-}  // namespace tflite
diff --git a/tensorflow/lite/kernels/abs_test.cc b/tensorflow/lite/kernels/abs_test.cc
deleted file mode 100644
index 5e3f95c43a..0000000000
--- a/tensorflow/lite/kernels/abs_test.cc
+++ /dev/null
@@ -1,94 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include <gtest/gtest.h>
-#include "tensorflow/lite/interpreter.h"
-#include "tensorflow/lite/kernels/register.h"
-#include "tensorflow/lite/kernels/test_util.h"
-#include "tensorflow/lite/model.h"
-
-namespace tflite {
-namespace {
-
-using ::testing::ElementsAreArray;
-
-class AbsOpModel : public SingleOpModel {
- public:
-  AbsOpModel(std::initializer_list<int> input_shape, TensorType input_type) {
-    input_ = AddInput(TensorType_FLOAT32);
-    output_ = AddOutput(TensorType_FLOAT32);
-    SetBuiltinOp(BuiltinOperator_ABS, BuiltinOptions_NONE, 0);
-    BuildInterpreter({
-        input_shape,
-    });
-  }
-
-  int input() { return input_; }
-
-  std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
-  std::vector<int> GetOutputShape() { return GetTensorShape(output_); }
-
- private:
-  int input_;
-  int output_;
-};
-
-TEST(AbsOpTest, SingleDim) {
-  AbsOpModel model({2}, TensorType_FLOAT32);
-  model.PopulateTensor<float>(model.input(), {8.5, -2.0});
-  model.Invoke();
-  EXPECT_THAT(model.GetOutput(), ElementsAreArray({8.5, 2.0}));
-  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2}));
-}
-
-TEST(AbsOpTest, MultiDims) {
-  AbsOpModel model({2, 1, 1, 5}, TensorType_FLOAT32);
-  model.PopulateTensor<float>(model.input(), {
-                                                 0.0001,
-                                                 8.0001,
-                                                 0.9999,
-                                                 9.9999,
-                                                 0.5,
-                                                 -0.0001,
-                                                 -8.0001,
-                                                 -0.9999,
-                                                 -9.9999,
-                                                 -0.5,
-                                             });
-  model.Invoke();
-  EXPECT_THAT(model.GetOutput(),
-              ElementsAreArray({
-                                                 0.0001,
-                                                 8.0001,
-                                                 0.9999,
-                                                 9.9999,
-                                                 0.5,
-                                                 0.0001,
-                                                 8.0001,
-                                                 0.9999,
-                                                 9.9999,
-                                                 0.5,
-                                             }));
-  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 1, 1, 5}));
-}
-
-}  // namespace
-}  // namespace tflite
-
-int main(int argc, char** argv) {
-  ::tflite::LogToStderr();
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}
diff --git a/tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h b/tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h
index c24b7faa21..a76649f934 100644
--- a/tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h
+++ b/tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h
@@ -1746,12 +1746,6 @@ inline void Ceil(const float* input_data, const Dims<4>& input_dims,
        output_data);
 }
 
-inline void Abs(const float* input_data, const Dims<4>& input_dims,
-                float* output_data, const Dims<4>& output_dims) {
-  Abs(DimsToShape(input_dims), input_data, DimsToShape(output_dims),
-      output_data);
-}
-
 inline void ResizeBilinear(const float* input_data, const Dims<4>& input_dims,
                            const int32* output_size_data,
                            const Dims<4>& output_size_dims, float* output_data,
diff --git a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
index 7f7296ae97..0bd779bf3a 100644
--- a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
+++ b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
@@ -4906,14 +4906,6 @@ inline void Ceil(const RuntimeShape& input_shape, const float* input_data,
   output_map.array() = Eigen::ceil(input_map.array());
 }
 
-inline void Abs(const RuntimeShape& input_shape, const float* input_data,
-                const RuntimeShape& output_shape, float* output_data) {
-  gemmlowp::ScopedProfilingLabel label("Abs");
-  auto input_map = MapAsVector(input_data, input_shape);
-  auto output_map = MapAsVector(output_data, output_shape);
-  output_map.array() = Eigen::abs(input_map.array());
-}
-
 #ifdef USE_NEON
 inline void ResizeBilinearKernel(const float* input_ptr, int32 depth,
                                  float scale, float* output_ptr) {
diff --git a/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h b/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h
index 30bb92c8b3..431e2413e8 100644
--- a/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h
+++ b/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h
@@ -1889,12 +1889,6 @@ inline void Ceil(const float* input_data, const Dims<4>& input_dims,
        output_data);
 }
 
-inline void Abs(const float* input_data, const Dims<4>& input_dims,
-                float* output_data, const Dims<4>& output_dims) {
-  Abs(DimsToShape(input_dims), input_data, DimsToShape(output_dims),
-      output_data);
-}
-
 template <typename T>
 inline void ResizeBilinear(const T* input_data, const Dims<4>& input_dims,
                            const int32* output_size_data,
diff --git a/tensorflow/lite/kernels/internal/reference/reference_ops.h b/tensorflow/lite/kernels/internal/reference/reference_ops.h
index 133a455e9e..6b8d817ca1 100644
--- a/tensorflow/lite/kernels/internal/reference/reference_ops.h
+++ b/tensorflow/lite/kernels/internal/reference/reference_ops.h
@@ -3050,16 +3050,6 @@ inline void Ceil(const RuntimeShape& input_shape, const float* input_data,
   }
 }
 
-inline void Abs(const RuntimeShape& input_shape, const float* input_data,
-                const RuntimeShape& output_shape, float* output_data) {
-  const int flat_size = MatchingFlatSize(input_shape, output_shape);
-
-  for (int i = 0; i < flat_size; i++) {
-    int offset = i;
-    output_data[offset] = std::fabs(input_data[offset]);
-  }
-}
-
 template <typename T, typename CoordsT = int32>
 inline void Gather(const tflite::GatherParams& op_params,
                    const RuntimeShape& input_shape, const T* input_data,
diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index caf61e9003..47ae934c4b 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -95,7 +95,6 @@ TfLiteRegistration* Register_LESS();
 TfLiteRegistration* Register_LESS_EQUAL();
 TfLiteRegistration* Register_FLOOR();
 TfLiteRegistration* Register_CEIL();
-TfLiteRegistration* Register_ABS();
 TfLiteRegistration* Register_TILE();
 TfLiteRegistration* Register_NEG();
 TfLiteRegistration* Register_SUM();
@@ -238,7 +237,6 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_LESS_EQUAL, Register_LESS_EQUAL());
   AddBuiltin(BuiltinOperator_FLOOR, Register_FLOOR());
   AddBuiltin(BuiltinOperator_CEIL, Register_CEIL());
-  AddBuiltin(BuiltinOperator_ABS, Register_ABS());
   AddBuiltin(BuiltinOperator_NEG, Register_NEG());
   AddBuiltin(BuiltinOperator_SELECT, Register_SELECT());
   AddBuiltin(BuiltinOperator_SLICE, Register_SLICE());
diff --git a/tensorflow/lite/nnapi/NeuralNetworksShim.h b/tensorflow/lite/nnapi/NeuralNetworksShim.h
index 037d364c3b..82c5840952 100644
--- a/tensorflow/lite/nnapi/NeuralNetworksShim.h
+++ b/tensorflow/lite/nnapi/NeuralNetworksShim.h
@@ -144,7 +144,6 @@ enum {
   ANEURALNETWORKS_SUB = 36,
   ANEURALNETWORKS_TRANSPOSE = 37,
   ANEURALNETWORKS_CEIL = 38,
-  ANEURALNETWORKS_ABS = 39,
 };
 
 /**
diff --git a/tensorflow/lite/nnapi_delegate.cc b/tensorflow/lite/nnapi_delegate.cc
index 6da803ce7b..dfbb4813ad 100644
--- a/tensorflow/lite/nnapi_delegate.cc
+++ b/tensorflow/lite/nnapi_delegate.cc
@@ -492,9 +492,6 @@ TfLiteStatus AddOpsAndParams(
       case tflite::BuiltinOperator_CEIL:
         nn_op_type = ANEURALNETWORKS_CEIL;
         break;
-      case tflite::BuiltinOperator_ABS:
-        nn_op_type = ANEURALNETWORKS_ABS;
-        break;
       case tflite::BuiltinOperator_LOGISTIC:
         nn_op_type = ANEURALNETWORKS_LOGISTIC;
         break;
diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index 825873a884..1dea01d59b 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -3047,32 +3047,6 @@ def make_ceil_tests(zip_path):
   make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
 
 
-def make_abs_tests(zip_path):
-  """Make a set of tests to do abs."""
-
-  test_parameters = [{
-      "input_dtype": [tf.float32],
-      "input_shape": [[1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
-  }]
-
-  def build_graph(parameters):
-    """Build the abs op testing graph."""
-    input_value = tf.placeholder(
-        dtype=parameters["input_dtype"],
-        name="input1",
-        shape=parameters["input_shape"])
-    out = tf.abs(input_value)
-    return [input_value], [out]
-
-  def build_inputs(parameters, sess, inputs, outputs):
-    input_value = create_tensor_data(parameters["input_dtype"],
-                                     parameters["input_shape"])
-    return [input_value], sess.run(
-        outputs, feed_dict={inputs[0]: input_value})
-
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
-
-
 def make_neg_tests(zip_path):
   """Make a set of tests to do neg."""
 
diff --git a/tensorflow/lite/toco/export_tensorflow.cc b/tensorflow/lite/toco/export_tensorflow.cc
index 9dfeaad164..816cb9bc28 100644
--- a/tensorflow/lite/toco/export_tensorflow.cc
+++ b/tensorflow/lite/toco/export_tensorflow.cc
@@ -1215,16 +1215,6 @@ void ConvertCeilOperator(const Model& model, const CeilOperator& src_op,
   (*ceil_op->mutable_attr())["T"].set_type(DT_FLOAT);
 }
 
-void ConvertAbsOperator(const Model& model, const AbsOperator& src_op,
-                        GraphDef* tensorflow_graph) {
-  tensorflow::NodeDef* abs_op = tensorflow_graph->add_node();
-  abs_op->set_op("Abs");
-  abs_op->set_name(src_op.outputs[0]);
-  CHECK_EQ(src_op.inputs.size(), 1);
-  *abs_op->add_input() = src_op.inputs[0];
-  (*abs_op->mutable_attr())["T"].set_type(DT_FLOAT);
-}
-
 void ConvertGatherOperator(const Model& model, const GatherOperator& src_op,
                            GraphDef* tensorflow_graph) {
   tensorflow::NodeDef* gather_op = tensorflow_graph->add_node();
@@ -2192,9 +2182,6 @@ void ConvertOperator(const Model& model, const Operator& src_op,
   } else if (src_op.type == OperatorType::kCeil) {
     ConvertCeilOperator(model, static_cast<const CeilOperator&>(src_op),
                         tensorflow_graph);
-  } else if (src_op.type == OperatorType::kAbs) {
-    ConvertAbsOperator(model, static_cast<const AbsOperator&>(src_op),
-                       tensorflow_graph);
   } else if (src_op.type == OperatorType::kGather) {
     ConvertGatherOperator(model, static_cast<const GatherOperator&>(src_op),
                           tensorflow_graph);
diff --git a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
index c6c7681c03..0aae68c9c4 100644
--- a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
+++ b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
@@ -1870,7 +1870,6 @@ void ProcessMirrorPadOperator(Model* model, MirrorPadOperator* op) {
     case OperatorType::kCast:
     case OperatorType::kFloor:
     case OperatorType::kCeil:
-    case OperatorType::kAbs:
     case OperatorType::kExp:
     case OperatorType::kSin:
     case OperatorType::kLogicalAnd:
diff --git a/tensorflow/lite/toco/graph_transformations/reorder_elementwise_unary.cc b/tensorflow/lite/toco/graph_transformations/reorder_elementwise_unary.cc
index 90e64816c8..98105d384e 100644
--- a/tensorflow/lite/toco/graph_transformations/reorder_elementwise_unary.cc
+++ b/tensorflow/lite/toco/graph_transformations/reorder_elementwise_unary.cc
@@ -29,7 +29,6 @@ namespace {
 
 bool IsElementwiseOperator(OperatorType optype) {
   switch (optype) {
-    case OperatorType::kAbs:
     case OperatorType::kCast:
     case OperatorType::kCeil:
     case OperatorType::kExp:
diff --git a/tensorflow/lite/toco/import_tensorflow.cc b/tensorflow/lite/toco/import_tensorflow.cc
index de8ae19a71..53defed6cb 100644
--- a/tensorflow/lite/toco/import_tensorflow.cc
+++ b/tensorflow/lite/toco/import_tensorflow.cc
@@ -1505,20 +1505,6 @@ tensorflow::Status ConvertCeilOperator(
   return tensorflow::Status::OK();
 }
 
-tensorflow::Status ConvertAbsOperator(
-    const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
-    Model* model) {
-  CHECK_EQ(node.op(), "Abs");
-  TF_QCHECK_OK(CheckInputsCount(node, tf_import_flags, 1));
-  const auto data_type = GetDataTypeAttr(node, "T");
-  CHECK(data_type == DT_FLOAT);
-  auto* op = new AbsOperator;
-  op->inputs.push_back(node.input(0));
-  op->outputs.push_back(node.name());
-  model->operators.emplace_back(op);
-  return tensorflow::Status::OK();
-}
-
 tensorflow::Status ConvertGatherOperator(
     const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
     Model* model) {
diff --git a/tensorflow/lite/toco/model.h b/tensorflow/lite/toco/model.h
index 69c6d7608a..c5cb215114 100644
--- a/tensorflow/lite/toco/model.h
+++ b/tensorflow/lite/toco/model.h
@@ -37,7 +37,6 @@ using tflite::QuantizationParams;
 enum class OperatorType : uint8 {
   kNone,
   // General-purpose neural network operators.
-  kAbs,
   kAdd,
   kAddN,
   kAveragePool,
@@ -1670,16 +1669,6 @@ struct CeilOperator : Operator {
   CeilOperator() : Operator(OperatorType::kCeil) {}
 };
 
-// Abs operator.
-//
-// Inputs:
-//   inputs[0]: required: the input array
-//
-// TensorFlow equivalent: Abs
-struct AbsOperator : Operator {
-  AbsOperator() : Operator(OperatorType::kAbs) {}
-};
-
 // Gather operator. It gathers slices from params according to indices.
 // Only 1-D indices are supported at the moment.
 //
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index 2122d04958..dd47808046 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -1651,8 +1651,6 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
       MakeUnique<SimpleOperator<FloorOperator>>("FLOOR", OperatorType::kFloor));
   ops.push_back(
       MakeUnique<SimpleOperator<CeilOperator>>("CEIL", OperatorType::kCeil));
-  ops.push_back(
-      MakeUnique<SimpleOperator<AbsOperator>>("ABS", OperatorType::kAbs));
   ops.push_back(
       MakeUnique<SimpleOperator<ReluOperator>>("RELU", OperatorType::kRelu));
   ops.push_back(MakeUnique<SimpleOperator<Relu1Operator>>(
diff --git a/tensorflow/lite/toco/tflite/operator_test.cc b/tensorflow/lite/toco/tflite/operator_test.cc
index e284619903..ba713edcb6 100644
--- a/tensorflow/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/lite/toco/tflite/operator_test.cc
@@ -115,7 +115,6 @@ TEST_F(OperatorTest, SimpleOperators) {
                                           OperatorType::kDequantize);
   CheckSimpleOperator<FloorOperator>("FLOOR", OperatorType::kFloor);
   CheckSimpleOperator<CeilOperator>("CEIL", OperatorType::kCeil);
-  CheckSimpleOperator<AbsOperator>("ABS", OperatorType::kAbs);
   CheckSimpleOperator<ReluOperator>("RELU", OperatorType::kRelu);
   CheckSimpleOperator<Relu1Operator>("RELU_N1_TO_1", OperatorType::kRelu1);
   CheckSimpleOperator<Relu6Operator>("RELU6", OperatorType::kRelu6);
diff --git a/tensorflow/lite/toco/tooling_util.cc b/tensorflow/lite/toco/tooling_util.cc
index 0887dca9b5..9d6f554c59 100644
--- a/tensorflow/lite/toco/tooling_util.cc
+++ b/tensorflow/lite/toco/tooling_util.cc
@@ -386,7 +386,6 @@ const char* OperatorTypeName(OperatorType type) {
     HANDLE_OPERATORTYPENAME_CASE(Cast)
     HANDLE_OPERATORTYPENAME_CASE(Floor)
     HANDLE_OPERATORTYPENAME_CASE(Ceil)
-    HANDLE_OPERATORTYPENAME_CASE(Abs)
     HANDLE_OPERATORTYPENAME_CASE(Gather)
     HANDLE_OPERATORTYPENAME_CASE(ResizeBilinear)
     HANDLE_OPERATORTYPENAME_CASE(SpaceToBatchND)
-- 
GitLab


From 9d5ac5c511dca4a959014b6f9882309bc9bd703f Mon Sep 17 00:00:00 2001
From: Yves-Noel Weweler <y.weweler@gmail.com>
Date: Thu, 13 Dec 2018 22:40:14 +0100
Subject: [PATCH 0044/2345] Fix typos

---
 tensorflow/python/data/experimental/ops/grouping.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/data/experimental/ops/grouping.py b/tensorflow/python/data/experimental/ops/grouping.py
index 71e4b3391f..2eef36d0c8 100644
--- a/tensorflow/python/data/experimental/ops/grouping.py
+++ b/tensorflow/python/data/experimental/ops/grouping.py
@@ -162,8 +162,8 @@ def bucket_by_sequence_length(element_length_func,
     no_padding: `bool`, indicates whether to pad the batch features (features
       need to be either of type `tf.SparseTensor` or of same shape).
     drop_remainder: (Optional.) A `tf.bool` scalar `tf.Tensor`, representing
-      whether the last batch should be dropped in the case its has fewer than
-      batch_size` elements; the default behavior is not to drop the smaller
+      whether the last batch should be dropped in the case it has fewer than
+      `batch_size` elements; the default behavior is not to drop the smaller
       batch.
 
   Returns:
-- 
GitLab


From a06497bce4e91b267850b8b82ee4fe684c01e94b Mon Sep 17 00:00:00 2001
From: Steve Lang <stevelang789@users.noreply.github.com>
Date: Fri, 14 Dec 2018 09:01:27 +1100
Subject: [PATCH 0045/2345] .numpy() was missing in code example: tf.add(1,
 2).numpy()

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 68d7e180d1..c512aeb06c 100644
--- a/README.md
+++ b/README.md
@@ -65,7 +65,7 @@ $ python
 ```python
 >>> import tensorflow as tf
 >>> tf.enable_eager_execution()
->>> tf.add(1, 2)
+>>> tf.add(1, 2).numpy()
 3
 >>> hello = tf.constant('Hello, TensorFlow!')
 >>> hello.numpy()
-- 
GitLab


From c8c5f69e9a4d2424ba4cc603daebd4c24e5128d4 Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Thu, 13 Dec 2018 14:45:48 -0800
Subject: [PATCH 0046/2345] [Go]: Fixup paths to protocol buffers.

Without this: go generate github.com/tensorflow/tensorflow/tensorflow/go/op
would fail with:
../genop/internal/api_def_map.go:34:2: cannot find package "github.com/tensorflow/tensorflow/tensorflow/go/genop/internal/proto/tensorflow/core/framework_go_proto" in any of:
        /usr/local/go/src/github.com/tensorflow/tensorflow/tensorflow/go/genop/internal/proto/tensorflow/core/framework_go_proto (from $GOROOT)
        /go/src/github.com/tensorflow/tensorflow/tensorflow/go/genop/internal/proto/tensorflow/core/framework_go_proto (from $GOPATH)

This breakage was probably introduced by https://github.com/tensorflow/tensorflow/pull/17262
---
 tensorflow/go/genop/internal/api_def_map.go | 2 +-
 tensorflow/go/genop/internal/genop.go       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/go/genop/internal/api_def_map.go b/tensorflow/go/genop/internal/api_def_map.go
index 8600452b47..0bbd88b61c 100644
--- a/tensorflow/go/genop/internal/api_def_map.go
+++ b/tensorflow/go/genop/internal/api_def_map.go
@@ -31,7 +31,7 @@ import (
 	"unsafe"
 
 	"github.com/golang/protobuf/proto"
-	pb "github.com/tensorflow/tensorflow/tensorflow/go/genop/internal/proto/tensorflow/core/framework_go_proto"
+	pb "github.com/tensorflow/tensorflow/tensorflow/go/genop/internal/proto/github.com/tensorflow/tensorflow/tensorflow/go/core/framework"
 )
 
 // Encapsulates a collection of API definitions.
diff --git a/tensorflow/go/genop/internal/genop.go b/tensorflow/go/genop/internal/genop.go
index fb81631218..1c05715a1a 100644
--- a/tensorflow/go/genop/internal/genop.go
+++ b/tensorflow/go/genop/internal/genop.go
@@ -47,7 +47,7 @@ import (
 	"unsafe"
 
 	"github.com/golang/protobuf/proto"
-	pb "github.com/tensorflow/tensorflow/tensorflow/go/genop/internal/proto/tensorflow/core/framework_go_proto"
+	pb "github.com/tensorflow/tensorflow/tensorflow/go/genop/internal/proto/github.com/tensorflow/tensorflow/tensorflow/go/core/framework"
 )
 
 // GenerateFunctionsForRegisteredOps writes a Go source code file to w
-- 
GitLab


From c1fc5d5036d850be852ec473f32292ca973f5bed Mon Sep 17 00:00:00 2001
From: Yves-Noel Weweler <y.weweler@gmail.com>
Date: Thu, 13 Dec 2018 22:57:22 +0100
Subject: [PATCH 0047/2345] Rewrite grouping tests to be @parameterized tests

---
 .../data/experimental/kernel_tests/BUILD      |  1 +
 .../bucket_by_sequence_length_test.py         | 60 ++++++++++++-------
 2 files changed, 38 insertions(+), 23 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/BUILD b/tensorflow/python/data/experimental/kernel_tests/BUILD
index d7ca5a70e4..897e949b0f 100644
--- a/tensorflow/python/data/experimental/kernel_tests/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/BUILD
@@ -23,6 +23,7 @@ py_test(
         "//tensorflow/python/data/kernel_tests:test_base",
         "//tensorflow/python/data/ops:dataset_ops",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
index fab79619a0..bcb7ef9496 100644
--- a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
@@ -19,6 +19,8 @@ from __future__ import print_function
 
 import random
 
+from absl.testing import parameterized
+
 from tensorflow.python.data.experimental.ops import grouping
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
@@ -69,9 +71,13 @@ def _get_record_shape(sparse):
   return tensor_shape.TensorShape([None])
 
 
-class BucketBySequenceLengthTest(test_base.DatasetTestBase):
+class BucketBySequenceLengthTest(test_base.DatasetTestBase, parameterized.TestCase):
 
-  def testBucketDropReminder(self):
+  @parameterized.named_parameters(
+    ("WithoutPadding", True),
+    ("WithPadding", False),
+  )
+  def testBucketDropReminder(self, param_no_padding):
 
     boundaries = [10, 20, 30]
     batch_sizes = [10, 8, 4, 2]
@@ -192,10 +198,13 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
                        .format(sorted(expected_lengths),
                                sorted(generated_lengths)))
 
-    for no_padding in (True, False):
-      _test_bucket_by_padding(no_padding)
+    _test_bucket_by_padding(param_no_padding)
 
-  def testBucket(self):
+  @parameterized.named_parameters(
+      ("WithoutPadding", True),
+      ("WithPadding", False),
+  )
+  def testBucket(self, param_no_padding):
 
     boundaries = [10, 20, 30]
     batch_sizes = [10, 8, 4, 2]
@@ -251,8 +260,7 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
       self.assertEqual(sorted(batch_sizes), sorted(batch_sizes_val))
       self.assertEqual(sorted(lengths), sorted(lengths_val))
 
-    for no_padding in (True, False):
-      _test_bucket_by_padding(no_padding)
+    _test_bucket_by_padding(param_no_padding)
 
   def testPadToBoundary(self):
 
@@ -336,7 +344,11 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
     self.assertAllEqual(batches[4], [[1, 1, 1, 1, 1, 1, 1, 1, 1, 0],
                                      [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])
 
-  def testTupleElements(self):
+  @parameterized.named_parameters(
+      ("WithoutPadding", True),
+      ("WithPadding", False),
+  )
+  def testTupleElements(self, param_no_padding):
 
     def build_dataset(sparse):
       def _generator():
@@ -364,10 +376,13 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
       self.assertEqual([None, None], shapes[0].as_list())
       self.assertEqual([None], shapes[1].as_list())
 
-    for no_padding in (True, False):
-      _test_tuple_elements_by_padding(no_padding)
+    _test_tuple_elements_by_padding(param_no_padding)
 
-  def testBucketSparse(self):
+  @parameterized.named_parameters(
+      ("DoDropRemainder", True),
+      ("DoNotDropRemainder", False),
+  )
+  def testBucketSparse(self, param_drop_remainder):
     """Tests bucketing of sparse tensors (case where `no_padding` == True).
 
     Test runs on following dataset:
@@ -435,18 +450,17 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
             all_sparse_tensors.add(sprs_tensor)
       return all_sparse_tensors
 
-    for drop_remainder in (True, False):
-      dataset = _build_dataset()
-      boundaries = range(min_len + bucket_size + 1, max_len, bucket_size)
-      dataset = dataset.apply(grouping.bucket_by_sequence_length(
-          _element_length_fn,
-          boundaries,
-          [batch_size] * (len(boundaries) + 1),
-          no_padding=True,
-          drop_remainder=drop_remainder))
-      batches = _compute_batches(dataset)
-      expected_batches = _compute_expected_batches(drop_remainder)
-      self.assertEqual(batches, expected_batches)
+    dataset = _build_dataset()
+    boundaries = range(min_len + bucket_size + 1, max_len, bucket_size)
+    dataset = dataset.apply(grouping.bucket_by_sequence_length(
+        _element_length_fn,
+        boundaries,
+        [batch_size] * (len(boundaries) + 1),
+        no_padding=True,
+        drop_remainder=param_drop_remainder))
+    batches = _compute_batches(dataset)
+    expected_batches = _compute_expected_batches(param_drop_remainder)
+    self.assertEqual(batches, expected_batches)
 
 
 if __name__ == "__main__":
-- 
GitLab


From 2cad3742600f3e903bc5a2d1a60ef4b571e7c8f9 Mon Sep 17 00:00:00 2001
From: WeijieSun <luckyweijie@gmail.com>
Date: Mon, 17 Dec 2018 10:23:31 +0800
Subject: [PATCH 0048/2345] remove KERB_TICKET_CACHE_PATH env

---
 tensorflow/core/platform/hadoop/hadoop_file_system.cc | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/tensorflow/core/platform/hadoop/hadoop_file_system.cc b/tensorflow/core/platform/hadoop/hadoop_file_system.cc
index eb35531e9f..9a7d98e6a0 100644
--- a/tensorflow/core/platform/hadoop/hadoop_file_system.cc
+++ b/tensorflow/core/platform/hadoop/hadoop_file_system.cc
@@ -59,8 +59,6 @@ class LibHDFS {
   std::function<hdfsBuilder*()> hdfsNewBuilder;
   std::function<void(hdfsBuilder*, const char*)> hdfsBuilderSetNameNode;
   std::function<int(const char*, char**)> hdfsConfGetStr;
-  std::function<void(hdfsBuilder*, const char* kerbTicketCachePath)>
-      hdfsBuilderSetKerbTicketCachePath;
   std::function<int(hdfsFS, hdfsFile)> hdfsCloseFile;
   std::function<tSize(hdfsFS, hdfsFile, tOffset, void*, tSize)> hdfsPread;
   std::function<tSize(hdfsFS, hdfsFile, const void*, tSize)> hdfsWrite;
@@ -87,7 +85,6 @@ class LibHDFS {
       BIND_HDFS_FUNC(hdfsNewBuilder);
       BIND_HDFS_FUNC(hdfsBuilderSetNameNode);
       BIND_HDFS_FUNC(hdfsConfGetStr);
-      BIND_HDFS_FUNC(hdfsBuilderSetKerbTicketCachePath);
       BIND_HDFS_FUNC(hdfsCloseFile);
       BIND_HDFS_FUNC(hdfsPread);
       BIND_HDFS_FUNC(hdfsWrite);
@@ -166,13 +163,6 @@ Status HadoopFileSystem::Connect(StringPiece fname, hdfsFS* fs) {
   } else {
     hdfs_->hdfsBuilderSetNameNode(builder, nn.c_str());
   }
-  // KERB_TICKET_CACHE_PATH will be deleted in the future, Because KRB5CCNAME is
-  // the build in environment variable of Kerberos, so KERB_TICKET_CACHE_PATH
-  // and related code are unnecessary.
-  char* ticket_cache_path = getenv("KERB_TICKET_CACHE_PATH");
-  if (ticket_cache_path != nullptr) {
-    hdfs_->hdfsBuilderSetKerbTicketCachePath(builder, ticket_cache_path);
-  }
   *fs = hdfs_->hdfsBuilderConnect(builder);
   if (*fs == nullptr) {
     return errors::NotFound(strerror(errno));
-- 
GitLab


From 8db73e896985d043f24592abf405fed4c867aee8 Mon Sep 17 00:00:00 2001
From: Pavel Samolysov <samolisov@gmail.com>
Date: Mon, 17 Dec 2018 18:32:45 +0300
Subject: [PATCH 0049/2345] [OpenMP] Fix undeclared identifier in
 eigen_support.cc

When OpenMP is enabled, the following error occurs during a
compilation of the `tensorflow/lite/kernels/eigen_support.cc` unit:

tensorflow/lite/kernels/eigen_support.cc:42:23: error: use of undeclared
identifier 'context'
  Eigen::setNbThreads(context->recommended_num_threads);

The `SetEigenNbThreads` method already gets the number of threads as the
`threads` parameter and doesn't need to calculate it using a method of
the `context` variable, so the invocation of the `Eigen::setNbThreads`
member must be changed a little bit.

Signed-off-by: Pavel Samolysov <samolisov@gmail.com>
---
 tensorflow/lite/kernels/eigen_support.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/kernels/eigen_support.cc b/tensorflow/lite/kernels/eigen_support.cc
index bad5975a7c..e2a2c4aac9 100644
--- a/tensorflow/lite/kernels/eigen_support.cc
+++ b/tensorflow/lite/kernels/eigen_support.cc
@@ -39,7 +39,7 @@ void SetEigenNbThreads(int threads) {
 #if defined(EIGEN_HAS_OPENMP)
   // The global Eigen thread count is only used when OpenMP is enabled. As this
   // call causes problems with tsan, make it only when OpenMP is available.
-  Eigen::setNbThreads(context->recommended_num_threads);
+  Eigen::setNbThreads(threads);
 #endif  // defined(EIGEN_HAS_OPENMP)
 }
 
-- 
GitLab


From 46288142246f2d5130e7c3ba5787f616d53e53a9 Mon Sep 17 00:00:00 2001
From: Joe Quadrino <jquadrin@gmail.com>
Date: Fri, 30 Nov 2018 19:42:36 -0500
Subject: [PATCH 0050/2345] Configurable AWS logging for S3 filesystem

AWS logging for S3 is too verbose and should be configurable.

export AWS_LOG_LEVEL=FATAL

Using TF_CPP_MIN_LOG_LEVEL for the AWS log handler can lead to a high
rate of aws logging which drowns out less frequent logs

Related commits 7bb0592, f67a7a4
Issue #21898
---
 tensorflow/core/platform/s3/aws_logging.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/platform/s3/aws_logging.cc b/tensorflow/core/platform/s3/aws_logging.cc
index 44317f1a3e..8eeb2c9492 100644
--- a/tensorflow/core/platform/s3/aws_logging.cc
+++ b/tensorflow/core/platform/s3/aws_logging.cc
@@ -74,7 +74,9 @@ static const char* kAWSLoggingTag = "AWSLogging";
 Aws::Utils::Logging::LogLevel ParseLogLevelFromEnv() {
   Aws::Utils::Logging::LogLevel log_level = Aws::Utils::Logging::LogLevel::Info;
 
-  const int64_t level = tensorflow::internal::MinLogLevelFromEnv();
+  const int64_t level = 
+      getenv("AWS_LOG_LEVEL") ? tensorflow::internal::LogLevelStrToInt(getenv("AWS_LOG_LEVEL")) 
+                              : tensorflow::internal::MinLogLevelFromEnv();
 
   switch (level) {
     case INFO:
-- 
GitLab


From c5720d1626c3c7e85069b0436bffacc8c6ca8122 Mon Sep 17 00:00:00 2001
From: Yves-Noel Weweler <y.weweler@gmail.com>
Date: Wed, 19 Dec 2018 11:54:28 +0100
Subject: [PATCH 0051/2345] Update testBucketDropReminder test case

See #24071
---
 .../bucket_by_sequence_length_test.py         | 20 ++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
index e0978676fd..a95d8e1049 100644
--- a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
@@ -76,11 +76,12 @@ def _get_record_shape(sparse):
 @test_util.run_all_in_graph_and_eager_modes
 class BucketBySequenceLengthTest(test_base.DatasetTestBase, parameterized.TestCase):
 
+  # TODO(b/117581999): add eager coverage.
   @parameterized.named_parameters(
     ("WithoutPadding", True),
     ("WithPadding", False),
   )
-  def testBucketDropReminder(self, param_no_padding):
+  def testSkipEagerBucketDropReminder(self, param_no_padding):
 
     boundaries = [10, 20, 30]
     batch_sizes = [10, 8, 4, 2]
@@ -142,14 +143,15 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase, parameterized.TestCa
               batch_sizes,
               no_padding=no_padding,
               drop_remainder=True))
-      batch, = dataset.make_one_shot_iterator().get_next()
-
-      with self.cached_session() as sess:
-        batches = []
-        for _ in range(n_expected_batches):
-          batches.append(self.evaluate(batch))
-        with self.assertRaises(errors.OutOfRangeError):
-          self.evaluate(batch)
+
+      get_next = self.getNext(dataset)
+      batches = []
+      for _ in range(n_expected_batches):
+        batch, = self.evaluate(get_next())
+        batches.append(batch)
+
+      with self.assertRaises(errors.OutOfRangeError):
+        self.evaluate(get_next())
 
       generated_lengths = []
 
-- 
GitLab


From c30e6b769f8f6be9d93372a000ee330d7a5ec56a Mon Sep 17 00:00:00 2001
From: Ashwin Ramaswami <aramaswamis@gmail.com>
Date: Wed, 19 Dec 2018 09:59:49 -0500
Subject: [PATCH 0052/2345] Update doc references to use tfp.distributions

---
 tensorflow/python/ops/distributions/bijector_impl.py      | 2 +-
 .../python/ops/distributions/transformed_distribution.py  | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/ops/distributions/bijector_impl.py b/tensorflow/python/ops/distributions/bijector_impl.py
index 9c63385dd0..a347cfdec1 100644
--- a/tensorflow/python/ops/distributions/bijector_impl.py
+++ b/tensorflow/python/ops/distributions/bijector_impl.py
@@ -462,7 +462,7 @@ class Bijector(object):
 
 
   ```python
-  abs = tf.contrib.distributions.bijectors.AbsoluteValue()
+  abs = tfp.distributions.bijectors.AbsoluteValue()
 
   abs.forward(-1.)
   ==> 1.
diff --git a/tensorflow/python/ops/distributions/transformed_distribution.py b/tensorflow/python/ops/distributions/transformed_distribution.py
index 1becfc1877..3c6476864a 100644
--- a/tensorflow/python/ops/distributions/transformed_distribution.py
+++ b/tensorflow/python/ops/distributions/transformed_distribution.py
@@ -167,7 +167,7 @@ class TransformedDistribution(distribution_lib.Distribution):
   distribution:
 
   ```python
-  ds = tf.contrib.distributions
+  ds = tfp.distributions
   log_normal = ds.TransformedDistribution(
     distribution=ds.Normal(loc=0., scale=1.),
     bijector=ds.bijectors.Exp(),
@@ -177,7 +177,7 @@ class TransformedDistribution(distribution_lib.Distribution):
   A `LogNormal` made from callables:
 
   ```python
-  ds = tf.contrib.distributions
+  ds = tfp.distributions
   log_normal = ds.TransformedDistribution(
     distribution=ds.Normal(loc=0., scale=1.),
     bijector=ds.bijectors.Inline(
@@ -191,7 +191,7 @@ class TransformedDistribution(distribution_lib.Distribution):
   Another example constructing a Normal from a StandardNormal:
 
   ```python
-  ds = tf.contrib.distributions
+  ds = tfp.distributions
   normal = ds.TransformedDistribution(
     distribution=ds.Normal(loc=0., scale=1.),
     bijector=ds.bijectors.Affine(
@@ -209,7 +209,7 @@ class TransformedDistribution(distribution_lib.Distribution):
   multivariate Normal as a `TransformedDistribution`.
 
   ```python
-  ds = tf.contrib.distributions
+  ds = tfp.distributions
   # We will create two MVNs with batch_shape = event_shape = 2.
   mean = [[-1., 0],      # batch:0
           [0., 1]]       # batch:1
-- 
GitLab


From 5bbdac505efbc4decaf7f15b247a26060fe25d46 Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Wed, 19 Dec 2018 12:39:03 -0800
Subject: [PATCH 0053/2345] Update lookup_ops.py

---
 tensorflow/python/ops/lookup_ops.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/ops/lookup_ops.py b/tensorflow/python/ops/lookup_ops.py
index 9302696a45..70bbbb72e6 100644
--- a/tensorflow/python/ops/lookup_ops.py
+++ b/tensorflow/python/ops/lookup_ops.py
@@ -64,6 +64,7 @@ def initialize_all_tables(name="init_all_tables"):
 @tf_export(v1=["initializers.tables_initializer", "tables_initializer"])
 def tables_initializer(name="init_all_tables"):
   """Returns an Op that initializes all tables of the default graph.
+  
   See the [Low Level Intro](https://www.tensorflow.org/guide/low_level_intro#feature_columns)
   guide, for an example of usage.
 
-- 
GitLab


From 9e92c2fe264debad1548554182dcc9af06d2e6ee Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 19 Dec 2018 13:48:25 -0800
Subject: [PATCH 0054/2345] Fix bug with renaming output bindings

---
 tensorflow/contrib/tensorrt/BUILD             |  1 +
 .../contrib/tensorrt/convert/convert_nodes.cc | 17 +++++
 .../tensorrt/test/identity_output_test.py     | 72 +++++++++++++++++++
 3 files changed, 90 insertions(+)
 create mode 100644 tensorflow/contrib/tensorrt/test/identity_output_test.py

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 784acce444..3e8c486d8c 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -491,6 +491,7 @@ cuda_py_tests(
         "test/binary_tensor_weight_broadcast_test.py",
         "test/concatenation_test.py",
         "test/const_broadcast_test.py",
+        "test/identity_output_test.py",
         "test/manual_test.py",
         "test/memory_alignment_test.py",
         "test/multi_connection_neighbor_engine_test.py",
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index adf8831b96..729afe6c64 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -879,6 +879,8 @@ Status Converter::ConvertNode(const NodeDef& node_def) {
     // We need to check the name before setting it. If the input is one of the
     // engine input, setting the name here will overwrite engine input
     // bindings which will cause runtime error.
+    // TODO(tmorris): Remove this work-around once we use TRT's IIdentityLayer
+    // in ConvertIdentity.
     if (output.is_tensor()) {
       const char* tensor_name = output.tensor()->getName();
       if (!tensorflow::str_util::StartsWith(tensor_name, kInputPHName)) {
@@ -939,6 +941,21 @@ Status Converter::RenameAndMarkOutputTensors(
     if (tensor == nullptr) {
       return errors::NotFound("Output tensor not found: ", output.first);
     }
+    // Check if this tensor has already been marked as an output.
+    // ConvertIdentity can cause the same tensor to be repeated in
+    // output_tensors, which can cause us to overwrite the name of the output
+    // tensor binding. For example, if we rename OutputPH_0 to OutputPH_1 then
+    // we won't be able to locate OutputPH_0 during runtime. To fix this,
+    // duplicate the tensor using no-op shuffle.
+    // TODO(tmorris): Remove this work-around once we use TRT's IIdentityLayer
+    // in ConvertIdentity.
+    if (tensorflow::str_util::StartsWith(tensor->getName(), kOutputPHName)) {
+      nvinfer1::IShuffleLayer* layer = network()->addShuffle(*tensor);
+      TFTRT_RETURN_ERROR_IF_NULLPTR(
+          layer, StrCat("Output Copy for ", tensor->getName()));
+      MarkQuantizationRangesAsInferrable(tensor, layer->getOutput(0));
+      tensor = layer->getOutput(0);
+    }
     tensor->setName(output.second.c_str());
     VLOG(1) << "Marking output tensor " << output.first << ", as output tensor "
             << output.second;
diff --git a/tensorflow/contrib/tensorrt/test/identity_output_test.py b/tensorflow/contrib/tensorrt/test/identity_output_test.py
new file mode 100644
index 0000000000..391434ba83
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/test/identity_output_test.py
@@ -0,0 +1,72 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Model script to test TF-TensorRT integration."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gen_array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn
+from tensorflow.python.platform import test
+
+
+class IdentityTest(trt_test.TfTrtIntegrationTestBase):
+
+  def _ConstOp(self, shape):
+    return constant_op.constant(np.random.randn(*shape), dtype=dtypes.float32)
+
+  def GetParams(self):
+    """Testing conversion of BiasAdd MatMul in TF-TRT conversion."""
+    input_name = "input"
+    input_dims = [100, 32]
+    g = ops.Graph()
+    with g.as_default():
+      x = array_ops.placeholder(
+          dtype=dtypes.float32, shape=input_dims, name=input_name)
+
+      b = self._ConstOp((32, 4))
+      x1 = math_ops.matmul(x, b)
+      b = self._ConstOp((1, 4))
+      x1 = x1 + b
+
+      out1 = array_ops.identity(x1, name='output1')
+      out2 = array_ops.identity(x1, name='output2')
+      iden1 = array_ops.identity(x1)
+      out3 = array_ops.identity(iden1, name='output3')
+
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=[input_name],
+        input_dims=[input_dims],
+        output_names=['output1', 'output2', 'output3'],
+        expected_output_dims=[(100, 4), (100, 4), (100, 4)])
+
+
+  def ExpectedEnginesToBuild(self, run_params):
+    """Return the expected engines to build."""
+    return ["TRTEngineOp_0"]
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From cfe7df2cf064f9eacdf8ab30574ebe9ec6438544 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 19 Dec 2018 14:14:28 -0800
Subject: [PATCH 0055/2345] Update comment in test

---
 tensorflow/contrib/tensorrt/test/identity_output_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tensorrt/test/identity_output_test.py b/tensorflow/contrib/tensorrt/test/identity_output_test.py
index 391434ba83..1aa05999bd 100644
--- a/tensorflow/contrib/tensorrt/test/identity_output_test.py
+++ b/tensorflow/contrib/tensorrt/test/identity_output_test.py
@@ -37,7 +37,7 @@ class IdentityTest(trt_test.TfTrtIntegrationTestBase):
     return constant_op.constant(np.random.randn(*shape), dtype=dtypes.float32)
 
   def GetParams(self):
-    """Testing conversion of BiasAdd MatMul in TF-TRT conversion."""
+    """Testing engine with the same tensor repeated as output via identity."""
     input_name = "input"
     input_dims = [100, 32]
     g = ops.Graph()
-- 
GitLab


From f03cf64b1295ea4e9e431a7a524cc23b2c1f3306 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 19 Dec 2018 15:10:58 -0800
Subject: [PATCH 0056/2345] Explain usage of shuffle layer

---
 tensorflow/contrib/tensorrt/convert/convert_nodes.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 729afe6c64..2a402fe699 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -950,6 +950,7 @@ Status Converter::RenameAndMarkOutputTensors(
     // TODO(tmorris): Remove this work-around once we use TRT's IIdentityLayer
     // in ConvertIdentity.
     if (tensorflow::str_util::StartsWith(tensor->getName(), kOutputPHName)) {
+      // Using shuffle layer for identity by not setting reshape or transpose.
       nvinfer1::IShuffleLayer* layer = network()->addShuffle(*tensor);
       TFTRT_RETURN_ERROR_IF_NULLPTR(
           layer, StrCat("Output Copy for ", tensor->getName()));
-- 
GitLab


From d489a943a0fbb995d48ee70558270c4a5f0f9c39 Mon Sep 17 00:00:00 2001
From: Taylor Thornton <thornton.tn@gmail.com>
Date: Tue, 18 Dec 2018 22:19:59 -0800
Subject: [PATCH 0057/2345] upgrade aws-sdk-cpp to 1.5.8 in order to pick up
 the auth retry changeset that landed in that version

---
 third_party/aws/BUILD.bazel   |  5 +++++
 third_party/aws/workspace.bzl | 11 +++++++----
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/third_party/aws/BUILD.bazel b/third_party/aws/BUILD.bazel
index 5426f79e46..1f61858777 100644
--- a/third_party/aws/BUILD.bazel
+++ b/third_party/aws/BUILD.bazel
@@ -80,6 +80,11 @@ cc_library(
     deps = [
         "@curl",
     ],
+    copts = [
+        "-DAWS_SDK_VERSION_MAJOR=1", 
+        "-DAWS_SDK_VERSION_MINOR=5", 
+        "-DAWS_SDK_VERSION_PATCH=8"
+    ],
 )
 
 template_rule(
diff --git a/third_party/aws/workspace.bzl b/third_party/aws/workspace.bzl
index c216638154..10799b5153 100644
--- a/third_party/aws/workspace.bzl
+++ b/third_party/aws/workspace.bzl
@@ -2,14 +2,17 @@
 
 load("//third_party:repo.bzl", "third_party_http_archive")
 
+# NOTE: version updates here should also update the major, minor, and patch variables declared in 
+# the  copts field of the //third_party/aws:aws target
+
 def repo():
     third_party_http_archive(
         name = "aws",
         urls = [
-            "https://mirror.bazel.build/github.com/aws/aws-sdk-cpp/archive/1.3.15.tar.gz",
-            "https://github.com/aws/aws-sdk-cpp/archive/1.3.15.tar.gz",
+            "https://mirror.bazel.build/github.com/aws/aws-sdk-cpp/archive/1.5.8.tar.gz",
+            "https://github.com/aws/aws-sdk-cpp/archive/1.5.8.tar.gz",
         ],
-        sha256 = "b888d8ce5fc10254c3dd6c9020c7764dd53cf39cf011249d0b4deda895de1b7c",
-        strip_prefix = "aws-sdk-cpp-1.3.15",
+        sha256 = "89905075fe50aa13e0337ff905c2e8c1ce9caf77a3504484a7cda39179120ffc",
+        strip_prefix = "aws-sdk-cpp-1.5.8",
         build_file = "//third_party/aws:BUILD.bazel",
     )
-- 
GitLab


From c75023169590ef0534579758285d6a1d8ad54ad2 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Wed, 19 Dec 2018 23:38:51 +0000
Subject: [PATCH 0058/2345] Replace deprecated FastGFile with GFile

FastGFile has been deprecated and replaced with GFile, though
the example in speech_commands still uses FastGFile. This fix
fix the issue to remove the deprecated warning:
```
WARNING:tensorflow:From <stdin>:1: __init__ (from tensorflow.python.platform.gfile) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.gfile.GFile.
```

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/examples/speech_commands/label_wav.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/examples/speech_commands/label_wav.py b/tensorflow/examples/speech_commands/label_wav.py
index 0017aec3a5..eb8323454c 100644
--- a/tensorflow/examples/speech_commands/label_wav.py
+++ b/tensorflow/examples/speech_commands/label_wav.py
@@ -45,7 +45,7 @@ FLAGS = None
 
 def load_graph(filename):
   """Unpersists graph from file as default graph."""
-  with tf.gfile.FastGFile(filename, 'rb') as f:
+  with tf.gfile.GFile(filename, 'rb') as f:
     graph_def = tf.GraphDef()
     graph_def.ParseFromString(f.read())
     tf.import_graph_def(graph_def, name='')
-- 
GitLab


From 4696da4bf9a586cf250b32a062da833af3ffd4a9 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Wed, 19 Dec 2018 23:41:52 +0000
Subject: [PATCH 0059/2345] Also replace FastGFile to GFile in label_wav_dir.py

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/examples/speech_commands/label_wav_dir.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/examples/speech_commands/label_wav_dir.py b/tensorflow/examples/speech_commands/label_wav_dir.py
index a34db512dd..2e1890c3e8 100644
--- a/tensorflow/examples/speech_commands/label_wav_dir.py
+++ b/tensorflow/examples/speech_commands/label_wav_dir.py
@@ -46,7 +46,7 @@ FLAGS = None
 
 def load_graph(filename):
   """Unpersists graph from file as default graph."""
-  with tf.gfile.FastGFile(filename, 'rb') as f:
+  with tf.gfile.GFile(filename, 'rb') as f:
     graph_def = tf.GraphDef()
     graph_def.ParseFromString(f.read())
     tf.import_graph_def(graph_def, name='')
-- 
GitLab


From 0f10d9fcd0c47b497d1bd5069f28d21f94fedc60 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 19 Dec 2018 15:56:19 -0800
Subject: [PATCH 0060/2345] Improve comment

---
 tensorflow/contrib/tensorrt/test/identity_output_test.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tensorrt/test/identity_output_test.py b/tensorflow/contrib/tensorrt/test/identity_output_test.py
index 1aa05999bd..2be9da9ede 100644
--- a/tensorflow/contrib/tensorrt/test/identity_output_test.py
+++ b/tensorflow/contrib/tensorrt/test/identity_output_test.py
@@ -12,7 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Model script to test TF-TensorRT integration."""
+"""This test checks a situation where the same tensor is considered as an output
+multiple times because it has been duplicated by 2+ indentity ops. Previously,
+the tensor would be renamed multiple times, overwriting the output binding name
+which resulted in a runtime error when the binding would not be found.
+"""
 
 from __future__ import absolute_import
 from __future__ import division
-- 
GitLab


From 7faefa4bb665e115cc744d7895a407338624993f Mon Sep 17 00:00:00 2001
From: Clayne Robison <clayne.b.robison@intel.com>
Date: Wed, 19 Dec 2018 19:08:29 -0700
Subject: [PATCH 0061/2345] This PR tempers the unused CPU instruction warning
 when TensorFlow is built with support for Intel(R) MKL-DNN. Many customers
 have complained about this warning because they don't understand that MKL-DNN
 has support for JIT compilation that detects CPU features at runtime.

---
 tensorflow/core/platform/cpu_feature_guard.cc | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tensorflow/core/platform/cpu_feature_guard.cc b/tensorflow/core/platform/cpu_feature_guard.cc
index 2efe0c0876..3f0d1cdb51 100644
--- a/tensorflow/core/platform/cpu_feature_guard.cc
+++ b/tensorflow/core/platform/cpu_feature_guard.cc
@@ -139,7 +139,19 @@ void InfoAboutUnusedCPUFeatures() {
 #endif  // else of if defined(_MSC_VER) && !defined(__clang__)
     if (!missing_instructions.empty()) {
       LOG(INFO) << "Your CPU supports instructions that this TensorFlow "
+#ifndef INTEL_MKL
                 << "binary was not compiled to use:" << missing_instructions;
+#else
+                << "binary was not compiled to use:" << missing_instructions
+                << ". When TensorFlow is compiled with support for these "
+                << "instructions, performance can improve. However, because "
+                << "this version of TensorFlow has been compiled with support "
+                << "for Intel(R) MKL-DNN, which supports just-in-time "
+                << "compilation, many performance-critical codepaths will use "
+                << "these instructions, even though TensorFlow was compiled "
+                << "without explicit support them. See "
+                << "https://github.com/intel/mkl-dnn/issues/3";
+#endif
     }
   });
 }
-- 
GitLab


From 915b8783a05b0da7c30ba36531ce03c811930852 Mon Sep 17 00:00:00 2001
From: Clayne Robison <clayne.b.robison@intel.com>
Date: Wed, 19 Dec 2018 19:13:27 -0700
Subject: [PATCH 0062/2345] [Intel MKL] Remove TensorFlow lite test from the
 public CI. MKL does not support TF lite.

---
 tensorflow/tools/ci_build/linux/cpu/run_mkl.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/linux/cpu/run_mkl.sh b/tensorflow/tools/ci_build/linux/cpu/run_mkl.sh
index 7be5f454ec..a8b73cbe0c 100755
--- a/tensorflow/tools/ci_build/linux/cpu/run_mkl.sh
+++ b/tensorflow/tools/ci_build/linux/cpu/run_mkl.sh
@@ -36,4 +36,4 @@ yes "" | $PYTHON_BIN_PATH configure.py
 bazel test --test_tag_filters=-no_oss,-oss_serial,-gpu,-benchmark-test --test_lang_filters=cc,py -k \
     --jobs=${N_JOBS} --test_timeout 300,450,1200,3600 --build_tests_only \
     --config=mkl --test_env=KMP_BLOCKTIME=0 --config=opt --test_output=errors -- \
-    //tensorflow/... -//tensorflow/compiler/... -//tensorflow/contrib/...
+    //tensorflow/... -//tensorflow/compiler/... -//tensorflow/contrib/... -//tensorflow/lite/...
-- 
GitLab


From f4867d3e4fecdb1ca8a445addad383aa0292a3b3 Mon Sep 17 00:00:00 2001
From: Sami Kama <skama@nvidia.com>
Date: Wed, 19 Dec 2018 19:40:22 -0800
Subject: [PATCH 0063/2345] Initialize only CPU devices in optimize_dataset_op

---
 tensorflow/core/grappler/grappler_item_builder.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/grappler/grappler_item_builder.cc b/tensorflow/core/grappler/grappler_item_builder.cc
index 9224ee7849..a984efd10d 100644
--- a/tensorflow/core/grappler/grappler_item_builder.cc
+++ b/tensorflow/core/grappler/grappler_item_builder.cc
@@ -103,7 +103,11 @@ Status OptimizeGraph(const GraphDef& graph_def_arg, GraphDef* output_graph_def,
 
   // Instantiate all variables for function library runtime creation.
   std::vector<std::unique_ptr<Device>> devices;
-  TF_RETURN_IF_ERROR(DeviceFactory::AddDevices(
+  // Only CPU device is used so instead of calling DeviceFactory::AddDevices()
+  // with dummy session config, which will conflict with user defined options and
+  // create unwanted devices, call cpu_factory->CreateDevices() to get CPU only devices.
+  DeviceFactory* cpu_factory = DeviceFactory::GetFactory("CPU");
+  TF_RETURN_IF_ERROR(cpu_factory->CreateDevices(
       options, "/job:localhost/replica:0/task:0", &devices));
   Device* cpu_device = devices[0].get();
   std::unique_ptr<DeviceMgr> dvc_mgr(new DeviceMgr(std::move(devices)));
-- 
GitLab


From 85ef6de9fcb5977bd738e10264f0641869594b83 Mon Sep 17 00:00:00 2001
From: Sami Kama <skama@nvidia.com>
Date: Wed, 19 Dec 2018 19:52:47 -0800
Subject: [PATCH 0064/2345] Fix clang format

---
 tensorflow/core/grappler/grappler_item_builder.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/grappler/grappler_item_builder.cc b/tensorflow/core/grappler/grappler_item_builder.cc
index a984efd10d..fc55fb5b3d 100644
--- a/tensorflow/core/grappler/grappler_item_builder.cc
+++ b/tensorflow/core/grappler/grappler_item_builder.cc
@@ -104,8 +104,9 @@ Status OptimizeGraph(const GraphDef& graph_def_arg, GraphDef* output_graph_def,
   // Instantiate all variables for function library runtime creation.
   std::vector<std::unique_ptr<Device>> devices;
   // Only CPU device is used so instead of calling DeviceFactory::AddDevices()
-  // with dummy session config, which will conflict with user defined options and
-  // create unwanted devices, call cpu_factory->CreateDevices() to get CPU only devices.
+  // with dummy session config, which will conflict with user defined options
+  // and create unwanted devices, call cpu_factory->CreateDevices() to get CPU
+  // only devices.
   DeviceFactory* cpu_factory = DeviceFactory::GetFactory("CPU");
   TF_RETURN_IF_ERROR(cpu_factory->CreateDevices(
       options, "/job:localhost/replica:0/task:0", &devices));
-- 
GitLab


From 2565842e71e1665d8a4fc126edc40b37becfed71 Mon Sep 17 00:00:00 2001
From: Innovimax <innovimax@gmail.com>
Date: Thu, 20 Dec 2018 14:11:27 +0100
Subject: [PATCH 0065/2345] fix typo

---
 tensorflow/java/src/gen/cc/op_gen_main.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/java/src/gen/cc/op_gen_main.cc b/tensorflow/java/src/gen/cc/op_gen_main.cc
index 0d9e0883af..cf4bb03dad 100644
--- a/tensorflow/java/src/gen/cc/op_gen_main.cc
+++ b/tensorflow/java/src/gen/cc/op_gen_main.cc
@@ -35,7 +35,7 @@ const char kUsageHeader[] =
     "graph.\n\n"
     "Operation wrappers are generated under the path specified by the "
     "'--output_dir' argument. This path can be absolute or relative to the\n"
-    "current working directory and will be created if it does not exists.\n\n"
+    "current working directory and will be created if it does not exist.\n\n"
     "Note that the operations will not be available through the "
     "'org.tensorflow.op.Ops' API until the generated classes are compiled\n"
     "using an appropriate annotation processor.\n\n"
-- 
GitLab


From 8cf1cf01615e68b4946ebad5a49d8096f0366e7f Mon Sep 17 00:00:00 2001
From: Mahmoud Abuzaina <mahmoud.abuzaina@intel.com>
Date: Thu, 20 Dec 2018 08:58:52 -0800
Subject: [PATCH 0066/2345] Hiding Mkl Quantized Ops

---
 .../api_def/base_api/api_def_QuantizedConv2DAndRelu.pbtxt     | 4 ++++
 .../api_def_QuantizedConv2DAndReluAndRequantize.pbtxt         | 4 ++++
 .../base_api/api_def_QuantizedConv2DAndRequantize.pbtxt       | 4 ++++
 .../api_def/base_api/api_def_QuantizedConv2DWithBias.pbtxt    | 4 ++++
 .../base_api/api_def_QuantizedConv2DWithBiasAndRelu.pbtxt     | 4 ++++
 .../api_def_QuantizedConv2DWithBiasAndReluAndRequantize.pbtxt | 4 ++++
 .../api_def_QuantizedConv2DWithBiasAndRequantize.pbtxt        | 4 ++++
 ...QuantizedConv2DWithBiasSignedSumAndReluAndRequantize.pbtxt | 4 ++++
 .../base_api/api_def_QuantizedConv2DWithBiasSumAndRelu.pbtxt  | 4 ++++
 ...i_def_QuantizedConv2DWithBiasSumAndReluAndRequantize.pbtxt | 4 ++++
 10 files changed, 40 insertions(+)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_QuantizedConv2DAndRelu.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_QuantizedConv2DAndReluAndRequantize.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_QuantizedConv2DAndRequantize.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBias.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasAndRelu.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasAndReluAndRequantize.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasAndRequantize.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasSignedSumAndReluAndRequantize.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasSumAndRelu.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasSumAndReluAndRequantize.pbtxt

diff --git a/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DAndRelu.pbtxt b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DAndRelu.pbtxt
new file mode 100644
index 0000000000..17ff15378c
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DAndRelu.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "QuantizedConv2DAndRelu"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DAndReluAndRequantize.pbtxt b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DAndReluAndRequantize.pbtxt
new file mode 100644
index 0000000000..b3ab3eba2c
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DAndReluAndRequantize.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "QuantizedConv2DAndReluAndRequantize"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DAndRequantize.pbtxt b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DAndRequantize.pbtxt
new file mode 100644
index 0000000000..8b00c2b7f6
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DAndRequantize.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "QuantizedConv2DAndRequantize"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBias.pbtxt b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBias.pbtxt
new file mode 100644
index 0000000000..f309f648ca
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBias.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "QuantizedConv2DWithBias"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasAndRelu.pbtxt b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasAndRelu.pbtxt
new file mode 100644
index 0000000000..b6b73eaae3
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasAndRelu.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "QuantizedConv2DWithBiasAndRelu"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasAndReluAndRequantize.pbtxt b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasAndReluAndRequantize.pbtxt
new file mode 100644
index 0000000000..101f72708a
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasAndReluAndRequantize.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "QuantizedConv2DWithBiasAndReluAndRequantize"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasAndRequantize.pbtxt b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasAndRequantize.pbtxt
new file mode 100644
index 0000000000..697e268415
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasAndRequantize.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "QuantizedConv2DWithBiasAndRequantize"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasSignedSumAndReluAndRequantize.pbtxt b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasSignedSumAndReluAndRequantize.pbtxt
new file mode 100644
index 0000000000..0cf52d6c89
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasSignedSumAndReluAndRequantize.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "QuantizedConv2DWithBiasSignedSumAndReluAndRequantize"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasSumAndRelu.pbtxt b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasSumAndRelu.pbtxt
new file mode 100644
index 0000000000..e91a2b8dc0
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasSumAndRelu.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "QuantizedConv2DWithBiasSumAndRelu"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasSumAndReluAndRequantize.pbtxt b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasSumAndReluAndRequantize.pbtxt
new file mode 100644
index 0000000000..fe3ec528bf
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasSumAndReluAndRequantize.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "QuantizedConv2DWithBiasSumAndReluAndRequantize"
+  visibility: HIDDEN
+}
-- 
GitLab


From a9ba4a47efb80d2951ee6deea8e2d8de56dbaaf9 Mon Sep 17 00:00:00 2001
From: vitor-alves <vitoralves842@gmail.com>
Date: Thu, 20 Dec 2018 17:26:45 -0200
Subject: [PATCH 0067/2345] Typo

---
 tensorflow/lite/examples/label_image/label_image.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/examples/label_image/label_image.md b/tensorflow/lite/examples/label_image/label_image.md
index fd9f49918b..debe8ffbc5 100644
--- a/tensorflow/lite/examples/label_image/label_image.md
+++ b/tensorflow/lite/examples/label_image/label_image.md
@@ -51,7 +51,7 @@ average time: 100.986 ms
 0.0235294: 514 cornet
 0.0196078: 835 suit
 ```
-Run `interpreter->Invoker()` 100 times:
+Run `interpreter->Invoke()` 100 times:
 ```
 > ./label_image   -c 100                               
 Loaded model ./mobilenet_quant_v1_224.tflite
-- 
GitLab


From f8dd4143788857cd6655ce983937908dc4ecd232 Mon Sep 17 00:00:00 2001
From: Guozhong Zhuang <guozhong.zhuang@intel.com>
Date: Thu, 20 Dec 2018 11:38:29 -0800
Subject: [PATCH 0068/2345] Limit total number of MKL primitive caching

---
 tensorflow/core/util/mkl_util.h       | 173 ++++++++++++++++++++------
 tensorflow/core/util/mkl_util_test.cc |  36 ++++++
 2 files changed, 168 insertions(+), 41 deletions(-)

diff --git a/tensorflow/core/util/mkl_util.h b/tensorflow/core/util/mkl_util.h
index 928807458a..34284f1e1e 100644
--- a/tensorflow/core/util/mkl_util.h
+++ b/tensorflow/core/util/mkl_util.h
@@ -17,6 +17,7 @@ limitations under the License.
 #define TENSORFLOW_CORE_UTIL_MKL_UTIL_H_
 #ifdef INTEL_MKL
 
+#include <list>
 #include <memory>
 #include <string>
 #include <unordered_map>
@@ -34,8 +35,7 @@ limitations under the License.
 #endif
 
 #ifdef INTEL_MKL_ML_ONLY
-#error \
-    "Compiling for INTEL MKL ML only is no longer supported.Please use MKL DNN (the default option for --config=mkl)"
+#error "Please use INTEL MKL DNN (the default option for --config=mkl)."
 #endif
 
 #ifdef INTEL_MKL_ML_ONLY
@@ -111,7 +111,7 @@ typedef enum {
   Dim3d_I = 1
 } MklDnnDims3D;
 
-// Enum used to templatize MklOp kernel implementations
+// Enum used to templatize MklOp kernel implementations
 // that support both fp32 and int8 versions.
 enum class MklQuantization {
   QUANTIZED_VERSION,
@@ -266,32 +266,32 @@ class MklShape {
     CHECK_EQ(dnnDelete_F32(convert), E_SUCCESS);
   }
 
-  // The following methods are used for serializing and de-serializing the
-  // contents of the mklshape object.
-  // The data is serialized in this order
-  // isMklTensor_
-  // dimension_
-  // sizes_
-  // strides_
-  // mklLayout_
-  // tfLayout_
-  // tf_to_mkl_dim_map_
+// The following methods are used for serializing and de-serializing the
+// contents of the mklshape object.
+// The data is serialized in this order
+// isMklTensor_
+// dimension_
+// sizes_
+// strides_
+// mklLayout_
+// tfLayout_
+// tf_to_mkl_dim_map_
 
 #define SIZE_OF_MKL_DNN_BUF \
   (dnnLayoutSerializationBufferSize_F32())  // Size of buffer needed to
                                             // serialize dnn_layout pointer
 
-  // Size of buffer to hold the serialized object, the size is computed as
-  // follows sizeof(isMklTensor_) + sizeof(dimension_) + sizeof(sizes_) +
-  // sizeof(strides_)
-  // + sizeof(mklLayout_ buffer) + sizeof(tfLayout_ buffer)
-  // + sizeof(tf_to_mkl_dim_map_)
+// Size of buffer to hold the serialized object, the size is computed as
+// follows sizeof(isMklTensor_) + sizeof(dimension_) + sizeof(sizes_) +
+// sizeof(strides_)
+// + sizeof(mklLayout_ buffer) + sizeof(tfLayout_ buffer)
+// + sizeof(tf_to_mkl_dim_map_)
 
 #define SIZE_OF_MKL_SERIAL_DATA(dims) \
   (2 * sizeof(size_t) + 3 * dims * sizeof(size_t) + 2 * SIZE_OF_MKL_DNN_BUF)
 
-  // First we need to define some macro for offsets into the serial buffer where
-  // different elements of Mklshape is written/read from
+// First we need to define some macro for offsets into the serial buffer where
+// different elements of Mklshape is written/read from
 
 #define IS_MKL_TENSOR_OFFSET 0
 // Location from start of buffer where isMklTensor_ is serialized
@@ -808,7 +808,6 @@ inline Tensor ConvertMklToTF(OpKernelContext* context, const Tensor& mkl_tensor,
       return mkl_tensor;  // return input since it is already TF tensor
 
     TensorShape output_shape = mkl_shape.GetTfShape();
-    ;
 
     // Allocate output tensor.
     context->allocate_temp(DataTypeToEnum<T>::v(), output_shape,
@@ -834,9 +833,9 @@ inline Tensor ConvertMklToTF(OpKernelContext* context, const Tensor& mkl_tensor,
       CHECK(output_tensor.CopyFrom(mkl_tensor, output_shape));
     }
   } catch (mkldnn::error& e) {
-    string error_msg = "Status: " + std::to_string(e.status) +
-                       ", message: " + string(e.message) + ", in file " +
-                       string(__FILE__) + ":" + std::to_string(__LINE__);
+    string error_msg = "Status: " + std::to_string(e.status) + ", message: " +
+                       string(e.message) + ", in file " + string(__FILE__) +
+                       ":" + std::to_string(__LINE__);
     LOG(FATAL) << "Operation received an exception: " << error_msg;
   }
   return output_tensor;
@@ -2031,6 +2030,107 @@ class MklPrimitive {
 
 const mkldnn::memory::dims NONE_DIMS = {};
 
+//
+// LRUCache is a class which implements LRU (Least Recently Use) cache.
+// The implementation is similar to that of
+//    tensorflow/core/platform/cloud/expiring_lru_cache.h
+// without its thread-safe part because the cache is supposed to be
+// used as thread local (for instance, MKLPrimitive caching).
+//
+// The LRU list maintains objects in chronological order based on
+// creation time, with the least recently accessed object at the
+// tail of LRU list, while the most recently accessed object
+// at the head of LRU list.
+//
+// This class is used to maintain an upper bound on the total number of
+// cached items. When the cache reaches its capacity, the LRU item will
+// be removed and replaced by a new one from SetOp call.
+//
+template <typename T>
+class LRUCache {
+ public:
+  explicit LRUCache(size_t capacity) {
+    capacity_ = capacity;
+    Clear();
+  }
+
+  T* GetOp(const string& key) {
+    auto it = cache_.find(key);
+    if (it == cache_.end()) {
+      return nullptr;
+    }
+
+    // Move to the front of LRU list as the most recently accessed.
+    lru_list_.erase(it->second.lru_iterator);
+    lru_list_.push_front(it->first);
+    it->second.lru_iterator = lru_list_.begin();
+    return it->second.op;
+  }
+
+  void SetOp(const string& key, T* op) {
+    if (lru_list_.size() >= capacity_) {
+      Delete();
+    }
+
+    // Insert an entry to the front of the LRU list
+    lru_list_.push_front(key);
+    Entry entry(op, lru_list_.begin());
+    cache_.insert(std::make_pair(key, entry));
+  }
+
+  void Clear() {
+    if (lru_list_.empty()) return;
+
+    // delete the cached objects
+    for (auto& key : lru_list_) {
+      auto it = cache_.find(key);
+      DCHECK(it == cache_.end());
+      delete it->second.op;
+    }
+
+    // clean up the cache
+    cache_.clear();
+    lru_list_.clear();
+  }
+
+ private:
+  struct Entry {
+    // The entry's value.
+    T* op;
+
+    // A list iterator pointing to the entry's position in the LRU list.
+    std::list<string>::iterator lru_iterator;
+    Entry(T* op, std::list<string>::iterator it) {
+      this->op = op;
+      this->lru_iterator = it;
+    }
+  };
+
+  // Remove the least recently accessed entry from LRU list, which
+  // is the tail of lru_list_. Correspondingly cache_ is updated.
+  bool Delete() {
+    if (lru_list_.empty()) return false;
+    string key = lru_list_.back();
+    auto it = cache_.find(key);
+    DCHECK(it == cache_.end());
+    lru_list_.pop_back();
+    delete it->second.op;  // delete the object
+    cache_.erase(it);
+    return true;
+  }
+
+  // cache capacity
+  size_t capacity_;
+
+  // The cache, a map from string key to a LRU entry.
+  std::unordered_map<string, Entry> cache_;
+
+  // The LRU list of entries.
+  // The front of the list identifies the most recently accessed entry,
+  // while the back of the list is the least recently accessed entry.
+  std::list<string> lru_list_;
+};
+
 template <typename T>
 class MklPrimitiveFactory {
  public:
@@ -2039,23 +2139,13 @@ class MklPrimitiveFactory {
   ~MklPrimitiveFactory() {}
 
   MklPrimitive* GetOp(const string& key) {
-    auto& map = MklPrimitiveFactory<T>::GetHashMap();
-    auto stream_iter = map.find(key);
-    if (stream_iter == map.end()) {
-      return nullptr;
-    } else {
-      CHECK(stream_iter->second != nullptr) << "nullptr present in map";
-      return stream_iter->second;
-    }
+    auto& lru_cache = MklPrimitiveFactory<T>::GetLRUCache();
+    return lru_cache.GetOp(key);
   }
 
   void SetOp(const string& key, MklPrimitive* op) {
-    auto& map = MklPrimitiveFactory<T>::GetHashMap();
-    auto stream_iter = map.find(key);
-
-    CHECK(stream_iter == map.end());
-
-    map[key] = op;
+    auto& lru_cache = MklPrimitiveFactory<T>::GetLRUCache();
+    lru_cache.SetOp(key, op);
   }
 
   /// Function to decide whether HW has AVX512 or AVX2
@@ -2075,9 +2165,10 @@ class MklPrimitiveFactory {
   }
 
  private:
-  static inline std::unordered_map<string, MklPrimitive*>& GetHashMap() {
-    static thread_local std::unordered_map<string, MklPrimitive*> map_;
-    return map_;
+  static inline LRUCache<MklPrimitive>& GetLRUCache() {
+    static const int kCapacity = 1024;  // cache capacity
+    static thread_local LRUCache<MklPrimitive> lru_cache_(kCapacity);
+    return lru_cache_;
   }
 };
 
diff --git a/tensorflow/core/util/mkl_util_test.cc b/tensorflow/core/util/mkl_util_test.cc
index 4f837f105d..7ac2e510fc 100644
--- a/tensorflow/core/util/mkl_util_test.cc
+++ b/tensorflow/core/util/mkl_util_test.cc
@@ -84,6 +84,42 @@ TEST(MklUtilTest, MklDnnBlockedFormatTest) {
   EXPECT_EQ(b_md2.data.format, mkldnn_blocked);
 }
 
+TEST(MklUtilTest, LRUCacheTest) {
+  // The cached objects are of type int*
+  size_t capacity = 100;
+  size_t num_objects = capacity + 10;
+  LRUCache<int> lru_cache(capacity);
+
+  // Test SetOp: be able to set more ops than the capacity
+  for (int k = 0; k < num_objects; k++) {
+    lru_cache.SetOp(std::to_string(k), new int(k));
+  }
+
+  // Test GetOp and capacity:
+  //   -- Less recently accessed objects should not be
+  //      in cache any more.
+  for (int k = 0; k < num_objects - capacity; k++) {
+    EXPECT_EQ(nullptr, lru_cache.GetOp(std::to_string(k)));
+  }
+
+  // Test GetOp and capacity:
+  //   -- most recently accessed objects should still
+  //      be in cache.
+  for (int k = num_objects - capacity; k < num_objects; k++) {
+    int* pint = lru_cache.GetOp(std::to_string(k));
+    EXPECT_NE(nullptr, pint);
+    EXPECT_EQ(*pint, k);
+  }
+
+  // Clean up the cache
+  lru_cache.Clear();
+
+  // After clean up, there should be no cached object.
+  for (int k = 0; k < num_objects; k++) {
+    EXPECT_EQ(nullptr, lru_cache.GetOp(std::to_string(k)));
+  }
+}
+
 #endif  // INTEL_MKL_ML_ONLY
 }  // namespace
 }  // namespace tensorflow
-- 
GitLab


From beed5ef6fc98f513f368e68ae08357a8db623ef0 Mon Sep 17 00:00:00 2001
From: Pooya Davoodi <pdavoodi@nvidia.com>
Date: Thu, 20 Dec 2018 11:38:58 -0800
Subject: [PATCH 0069/2345] TFTRT: Convert between str and unicode in py2

Also check for types before doing conversion.
---
 tensorflow/contrib/tensorrt/python/trt_convert.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/python/trt_convert.py b/tensorflow/contrib/tensorrt/python/trt_convert.py
index 203b2697ba..3d2b6b499d 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert.py
@@ -46,11 +46,13 @@ from tensorflow.python.saved_model import tag_constants
 from tensorflow.python.training import saver
 
 if _six.PY2:
-  _to_bytes = lambda s: s
-  _to_string = lambda s: s
+  _to_bytes = lambda s: s.encode("utf-8", errors="surrogateescape") \
+    if isinstance(s, unicode) else s
+  _to_string = lambda s: s.decode("utf-8") if isinstance(s, str) else s
 else:
-  _to_bytes = lambda s: s.encode("utf-8", errors="surrogateescape")
-  _to_string = lambda s: s.decode("utf-8")
+  _to_bytes = lambda s: s.encode("utf-8", errors="surrogateescape") \
+    if isinstance(s, str) else s
+  _to_string = lambda s: s.decode("utf-8") if isinstance(s, bytes) else s
 
 
 class TrtPrecisionMode(object):
-- 
GitLab


From 466711c40783a907cf6867cec5c13c16ed0bf257 Mon Sep 17 00:00:00 2001
From: Taylor Jakobson <tjakobs@us.ibm.com>
Date: Fri, 30 Nov 2018 13:50:27 -0600
Subject: [PATCH 0070/2345] Add support for ppc64le_dockerfiles

Add support for ppc64le dockerfiles with newest assembler changes.
---
 tensorflow/tools/dockerfiles/assembler.py     |   8 +
 .../dockerfiles/devel-cpu-jupyter.Dockerfile  |   9 +-
 .../dockerfiles/devel-cpu.Dockerfile          |   9 +-
 .../dockerfiles/devel-gpu-jupyter.Dockerfile  |  53 +++---
 .../dockerfiles/devel-gpu.Dockerfile          |  51 +++---
 .../dockerfiles/gpu-jupyter.Dockerfile        |  31 ++--
 .../dockerfiles/dockerfiles/gpu.Dockerfile    |  31 ++--
 .../ppc64le/cpu-ppc64le-jupyter.Dockerfile    |  92 +++++++++++
 .../ppc64le/cpu-ppc64le.Dockerfile            |  75 +++++++++
 .../devel-cpu-ppc64le-jupyter.Dockerfile      | 125 +++++++++++++++
 .../ppc64le/devel-cpu-ppc64le.Dockerfile      | 108 +++++++++++++
 .../devel-gpu-ppc64le-jupyter.Dockerfile      | 151 ++++++++++++++++++
 .../ppc64le/devel-gpu-ppc64le.Dockerfile      | 134 ++++++++++++++++
 .../ppc64le/gpu-ppc64le-jupyter.Dockerfile    | 125 +++++++++++++++
 .../ppc64le/gpu-ppc64le.Dockerfile            | 108 +++++++++++++
 .../tensorflow-ppc64le.partial.Dockerfile     |  28 ++++
 .../ubuntu/bazelbuild.partial.Dockerfile      |  33 ++++
 .../ubuntu/devel-cpu.partial.Dockerfile       |   9 +-
 .../ubuntu/devel-nvidia.partial.Dockerfile    |  51 +++---
 .../partials/ubuntu/nvidia.partial.Dockerfile |  31 ++--
 tensorflow/tools/dockerfiles/spec.yml         |  71 ++++++++
 tensorflow/tools/dockerfiles/tools.Dockerfile |   2 +-
 22 files changed, 1216 insertions(+), 119 deletions(-)
 create mode 100644 tensorflow/tools/dockerfiles/dockerfiles/ppc64le/cpu-ppc64le-jupyter.Dockerfile
 create mode 100644 tensorflow/tools/dockerfiles/dockerfiles/ppc64le/cpu-ppc64le.Dockerfile
 create mode 100644 tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le-jupyter.Dockerfile
 create mode 100644 tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le.Dockerfile
 create mode 100644 tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le-jupyter.Dockerfile
 create mode 100644 tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le.Dockerfile
 create mode 100644 tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le-jupyter.Dockerfile
 create mode 100644 tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le.Dockerfile
 create mode 100644 tensorflow/tools/dockerfiles/partials/tensorflow-ppc64le.partial.Dockerfile
 create mode 100644 tensorflow/tools/dockerfiles/partials/ubuntu/bazelbuild.partial.Dockerfile

diff --git a/tensorflow/tools/dockerfiles/assembler.py b/tensorflow/tools/dockerfiles/assembler.py
index 09537b7314..83b72cb5bb 100644
--- a/tensorflow/tools/dockerfiles/assembler.py
+++ b/tensorflow/tools/dockerfiles/assembler.py
@@ -34,6 +34,7 @@ import errno
 import itertools
 import multiprocessing
 import os
+import platform
 import re
 import shutil
 import sys
@@ -552,6 +553,13 @@ def main(argv):
       if not FLAGS.build_images:
         continue
 
+      # Only build images for host architecture
+      proc_arch = platform.processor()
+      is_x86 = proc_arch.startswith('x86')
+      if (is_x86 and any([arch in tag for arch in ['ppc64le']]) or
+          not is_x86 and proc_arch not in tag):
+        continue
+
       # Generate a temporary Dockerfile to use to build, since docker-py
       # needs a filepath relative to the build context (i.e. the current
       # directory)
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu-jupyter.Dockerfile
index c1f6dafbe0..4657f8b4c7 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu-jupyter.Dockerfile
@@ -30,7 +30,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         libcurl3-dev \
         libfreetype6-dev \
         libhdf5-serial-dev \
-        libpng12-dev \
         libzmq3-dev \
         pkg-config \
         rsync \
@@ -43,12 +42,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
- 
+
 ENV CI_BUILD_PYTHON python
 
-# Check out TensorFlow source code if --build_arg CHECKOUT_TENSORFLOW=1
+# CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version
+ARG CACHE_STOP=1
+# Check out TensorFlow source code if --build_arg CHECKOUT_TF_SRC=1
 ARG CHECKOUT_TF_SRC=0
-RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src
+RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src || true
 
 ARG USE_PYTHON_3_NOT_2
 ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3}
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu.Dockerfile
index b4dfc8b099..ce69ee0650 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu.Dockerfile
@@ -30,7 +30,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         libcurl3-dev \
         libfreetype6-dev \
         libhdf5-serial-dev \
-        libpng12-dev \
         libzmq3-dev \
         pkg-config \
         rsync \
@@ -43,12 +42,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
- 
+
 ENV CI_BUILD_PYTHON python
 
-# Check out TensorFlow source code if --build_arg CHECKOUT_TENSORFLOW=1
+# CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version
+ARG CACHE_STOP=1
+# Check out TensorFlow source code if --build_arg CHECKOUT_TF_SRC=1
 ARG CHECKOUT_TF_SRC=0
-RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src
+RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src || true
 
 ARG USE_PYTHON_3_NOT_2
 ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3}
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile
index 6d76c06332..e41fc18e42 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile
@@ -21,23 +21,28 @@
 
 ARG UBUNTU_VERSION=16.04
 
-FROM nvidia/cuda:10.0-base-ubuntu${UBUNTU_VERSION} as base
-
+ARG ARCH=
+ARG CUDA=10.0
+FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}-base-ubuntu${UBUNTU_VERSION} as base
+ARG CUDNN=7.4.1.5-1
+ARG LIB_DIR_PREFIX=x84_64
+
+# Needed for string substitution 
+SHELL ["/bin/bash", "-c"]
 RUN apt-get update && apt-get install -y --no-install-recommends \
         build-essential \
-        cuda-command-line-tools-10-0 \
-        cuda-cublas-dev-10-0 \
-        cuda-cudart-dev-10-0 \
-        cuda-cufft-dev-10-0 \
-        cuda-curand-dev-10-0 \
-        cuda-cusolver-dev-10-0 \
-        cuda-cusparse-dev-10-0 \
-        libcudnn7=7.4.1.5-1+cuda10.0 \
-        libcudnn7-dev=7.4.1.5-1+cuda10.0 \
+        cuda-command-line-tools-${CUDA/./-} \
+        cuda-cublas-dev-${CUDA/./-} \
+        cuda-cudart-dev-${CUDA/./-} \
+        cuda-cufft-dev-${CUDA/./-} \
+        cuda-curand-dev-${CUDA/./-} \
+        cuda-cusolver-dev-${CUDA/./-} \
+        cuda-cusparse-dev-${CUDA/./-} \
+        libcudnn7=${CUDNN}+cuda${CUDA} \
+        libcudnn7-dev=${CUDNN}+cuda${CUDA} \
         libcurl3-dev \
         libfreetype6-dev \
         libhdf5-serial-dev \
-        libpng12-dev \
         libzmq3-dev \
         pkg-config \
         rsync \
@@ -48,14 +53,15 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         wget \
         git \
         && \
-    find /usr/local/cuda-10.0/lib64/ -type f -name 'lib*_static.a' -not -name 'libcudart_static.a' -delete && \
-    rm /usr/lib/x86_64-linux-gnu/libcudnn_static_v7.a
+    find /usr/local/cuda-${CUDA}/lib64/ -type f -name 'lib*_static.a' -not -name 'libcudart_static.a' -delete && \
+    rm /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libcudnn_static_v7.a
 
-RUN apt-get update && \
-        apt-get install nvinfer-runtime-trt-repo-ubuntu1604-5.0.2-ga-cuda10.0 \
+RUN [ ${ARCH} = ppc64le ] || (apt-get update && \
+        apt-get install nvinfer-runtime-trt-repo-ubuntu1604-5.0.2-ga-cuda${CUDA} \
         && apt-get update \
-        && apt-get install -y --no-install-recommends libnvinfer-dev=5.0.2-1+cuda10.0 \
-        && rm -rf /var/lib/apt/lists/*
+        && apt-get install -y --no-install-recommends libnvinfer5=5.0.2-1+cuda${CUDA} \
+        && apt-get clean \
+        && rm -rf /var/lib/apt/lists/*)
 
 # Configure the build for our CUDA configuration.
 ENV CI_BUILD_PYTHON python
@@ -63,12 +69,13 @@ ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
 ENV TF_NEED_CUDA 1
 ENV TF_NEED_TENSORRT 1
 ENV TF_CUDA_COMPUTE_CAPABILITIES=3.5,5.2,6.0,6.1,7.0
-ENV TF_CUDA_VERSION=10.0
-ENV TF_CUDNN_VERSION=7
-
-# Check out TensorFlow source code if --build_arg CHECKOUT_TENSORFLOW=1
+ENV TF_CUDA_VERSION=${CUDA}
+ENV TF_CUDNN_VERSION=${CUDNN%%.*}
+# CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version
+ARG CACHE_STOP=1
+# Check out TensorFlow source code if --build_arg CHECKOUT_TF_SRC=1
 ARG CHECKOUT_TF_SRC=0
-RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src
+RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src || true
 
 ARG USE_PYTHON_3_NOT_2
 ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3}
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile
index 160abc8763..7ae5010079 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile
@@ -21,23 +21,28 @@
 
 ARG UBUNTU_VERSION=16.04
 
-FROM nvidia/cuda:10.0-base-ubuntu${UBUNTU_VERSION} as base
+ARG ARCH=
+ARG CUDA=10.0
+FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}-base-ubuntu${UBUNTU_VERSION} as base
+ARG CUDNN=7.4.1.5-1
+ARG LIB_DIR_PREFIX=x84_64
 
+# Needed for string substitution 
+SHELL ["/bin/bash", "-c"]
 RUN apt-get update && apt-get install -y --no-install-recommends \
         build-essential \
-        cuda-command-line-tools-10-0 \
-        cuda-cublas-dev-10-0 \
-        cuda-cudart-dev-10-0 \
-        cuda-cufft-dev-10-0 \
-        cuda-curand-dev-10-0 \
-        cuda-cusolver-dev-10-0 \
-        cuda-cusparse-dev-10-0 \
-        libcudnn7=7.4.1.5-1+cuda10.0 \
-        libcudnn7-dev=7.4.1.5-1+cuda10.0 \
+        cuda-command-line-tools-${CUDA/./-} \
+        cuda-cublas-dev-${CUDA/./-} \
+        cuda-cudart-dev-${CUDA/./-} \
+        cuda-cufft-dev-${CUDA/./-} \
+        cuda-curand-dev-${CUDA/./-} \
+        cuda-cusolver-dev-${CUDA/./-} \
+        cuda-cusparse-dev-${CUDA/./-} \
+        libcudnn7=${CUDNN}+cuda${CUDA} \
+        libcudnn7-dev=${CUDNN}+cuda${CUDA} \
         libcurl3-dev \
         libfreetype6-dev \
         libhdf5-serial-dev \
-        libpng12-dev \
         libzmq3-dev \
         pkg-config \
         rsync \
@@ -48,14 +53,15 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         wget \
         git \
         && \
-    find /usr/local/cuda-10.0/lib64/ -type f -name 'lib*_static.a' -not -name 'libcudart_static.a' -delete && \
-    rm /usr/lib/x86_64-linux-gnu/libcudnn_static_v7.a
+    find /usr/local/cuda-${CUDA}/lib64/ -type f -name 'lib*_static.a' -not -name 'libcudart_static.a' -delete && \
+    rm /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libcudnn_static_v7.a
 
-RUN apt-get update && \
-        apt-get install nvinfer-runtime-trt-repo-ubuntu1604-5.0.2-ga-cuda10.0 \
+RUN [ ${ARCH} = ppc64le ] || (apt-get update && \
+        apt-get install nvinfer-runtime-trt-repo-ubuntu1604-5.0.2-ga-cuda${CUDA} \
         && apt-get update \
-        && apt-get install -y --no-install-recommends libnvinfer-dev=5.0.2-1+cuda10.0 \
-        && rm -rf /var/lib/apt/lists/*
+        && apt-get install -y --no-install-recommends libnvinfer5=5.0.2-1+cuda${CUDA} \
+        && apt-get clean \
+        && rm -rf /var/lib/apt/lists/*)
 
 # Configure the build for our CUDA configuration.
 ENV CI_BUILD_PYTHON python
@@ -63,12 +69,13 @@ ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
 ENV TF_NEED_CUDA 1
 ENV TF_NEED_TENSORRT 1
 ENV TF_CUDA_COMPUTE_CAPABILITIES=3.5,5.2,6.0,6.1,7.0
-ENV TF_CUDA_VERSION=10.0
-ENV TF_CUDNN_VERSION=7
-
-# Check out TensorFlow source code if --build_arg CHECKOUT_TENSORFLOW=1
+ENV TF_CUDA_VERSION=${CUDA}
+ENV TF_CUDNN_VERSION=${CUDNN%%.*}
+# CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version
+ARG CACHE_STOP=1
+# Check out TensorFlow source code if --build_arg CHECKOUT_TF_SRC=1
 ARG CHECKOUT_TF_SRC=0
-RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src
+RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src || true
 
 ARG USE_PYTHON_3_NOT_2
 ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3}
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile
index 46252c5413..12eb5afa8a 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile
@@ -21,32 +21,37 @@
 
 ARG UBUNTU_VERSION=16.04
 
-FROM nvidia/cuda:10.0-base-ubuntu${UBUNTU_VERSION} as base
+ARG ARCH=
+ARG CUDA=10.0
+FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}-base-ubuntu${UBUNTU_VERSION} as base
+ARG CUDNN=7.4.1.5-1
 
+# Needed for string substitution 
+SHELL ["/bin/bash", "-c"]
 # Pick up some TF dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
         build-essential \
-        cuda-command-line-tools-10-0 \
-        cuda-cublas-10-0 \
-        cuda-cufft-10-0 \
-        cuda-curand-10-0 \
-        cuda-cusolver-10-0 \
-        cuda-cusparse-10-0 \
-        libcudnn7=7.4.1.5-1+cuda10.0 \
+        cuda-command-line-tools-${CUDA/./-} \
+        cuda-cublas-${CUDA/./-} \
+        cuda-cufft-${CUDA/./-} \
+        cuda-curand-${CUDA/./-} \
+        cuda-cusolver-${CUDA/./-} \
+        cuda-cusparse-${CUDA/./-} \
+        curl \
+        libcudnn7=${CUDNN}+cuda${CUDA} \
         libfreetype6-dev \
         libhdf5-serial-dev \
-        libpng12-dev \
         libzmq3-dev \
         pkg-config \
         software-properties-common \
         unzip
 
-RUN apt-get update && \
-        apt-get install nvinfer-runtime-trt-repo-ubuntu1604-5.0.2-ga-cuda10.0 \
+RUN [ ${ARCH} = ppc64le ] || (apt-get update && \
+        apt-get install nvinfer-runtime-trt-repo-ubuntu1604-5.0.2-ga-cuda${CUDA} \
         && apt-get update \
-        && apt-get install -y --no-install-recommends libnvinfer5=5.0.2-1+cuda10.0 \
+        && apt-get install -y --no-install-recommends libnvinfer5=5.0.2-1+cuda${CUDA} \
         && apt-get clean \
-        && rm -rf /var/lib/apt/lists/*
+        && rm -rf /var/lib/apt/lists/*)
 
 # For CUDA profiling, TensorFlow requires CUPTI.
 ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile
index 80e427f824..00664b6b73 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile
@@ -21,32 +21,37 @@
 
 ARG UBUNTU_VERSION=16.04
 
-FROM nvidia/cuda:10.0-base-ubuntu${UBUNTU_VERSION} as base
+ARG ARCH=
+ARG CUDA=10.0
+FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}-base-ubuntu${UBUNTU_VERSION} as base
+ARG CUDNN=7.4.1.5-1
 
+# Needed for string substitution 
+SHELL ["/bin/bash", "-c"]
 # Pick up some TF dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
         build-essential \
-        cuda-command-line-tools-10-0 \
-        cuda-cublas-10-0 \
-        cuda-cufft-10-0 \
-        cuda-curand-10-0 \
-        cuda-cusolver-10-0 \
-        cuda-cusparse-10-0 \
-        libcudnn7=7.4.1.5-1+cuda10.0 \
+        cuda-command-line-tools-${CUDA/./-} \
+        cuda-cublas-${CUDA/./-} \
+        cuda-cufft-${CUDA/./-} \
+        cuda-curand-${CUDA/./-} \
+        cuda-cusolver-${CUDA/./-} \
+        cuda-cusparse-${CUDA/./-} \
+        curl \
+        libcudnn7=${CUDNN}+cuda${CUDA} \
         libfreetype6-dev \
         libhdf5-serial-dev \
-        libpng12-dev \
         libzmq3-dev \
         pkg-config \
         software-properties-common \
         unzip
 
-RUN apt-get update && \
-        apt-get install nvinfer-runtime-trt-repo-ubuntu1604-5.0.2-ga-cuda10.0 \
+RUN [ ${ARCH} = ppc64le ] || (apt-get update && \
+        apt-get install nvinfer-runtime-trt-repo-ubuntu1604-5.0.2-ga-cuda${CUDA} \
         && apt-get update \
-        && apt-get install -y --no-install-recommends libnvinfer5=5.0.2-1+cuda10.0 \
+        && apt-get install -y --no-install-recommends libnvinfer5=5.0.2-1+cuda${CUDA} \
         && apt-get clean \
-        && rm -rf /var/lib/apt/lists/*
+        && rm -rf /var/lib/apt/lists/*)
 
 # For CUDA profiling, TensorFlow requires CUPTI.
 ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/cpu-ppc64le-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/cpu-ppc64le-jupyter.Dockerfile
new file mode 100644
index 0000000000..beb3292a9d
--- /dev/null
+++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/cpu-ppc64le-jupyter.Dockerfile
@@ -0,0 +1,92 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+#
+# THIS IS A GENERATED DOCKERFILE.
+#
+# This file was assembled from multiple pieces, whose use is documented
+# throughout. Please refer to the TensorFlow dockerfiles documentation
+# for more information.
+
+ARG UBUNTU_VERSION=16.04
+
+FROM ubuntu:${UBUNTU_VERSION} as base
+
+ARG USE_PYTHON_3_NOT_2
+ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3}
+ARG PYTHON=python${_PY_SUFFIX}
+ARG PIP=pip${_PY_SUFFIX}
+
+# See http://bugs.python.org/issue19846
+ENV LANG C.UTF-8
+
+RUN apt-get update && apt-get install -y \
+    ${PYTHON} \
+    ${PYTHON}-pip
+
+RUN ${PIP} --no-cache-dir install --upgrade \
+    pip \
+    setuptools
+
+# Some TF tools expect a "python" binary
+RUN ln -s $(which ${PYTHON}) /usr/local/bin/python 
+
+# Options:
+#   tensorflow
+#   tensorflow-gpu
+#   tf-nightly
+#   tf-nightly-gpu
+ARG TF_PACKAGE=tensorflow
+RUN apt-get update && apt-get install -y wget libhdf5-dev
+RUN ${PIP} install --global-option=build_ext \
+            --global-option=-I/usr/include/hdf5/serial/ \
+            --global-option=-L/usr/lib/powerpc64le-linux-gnu/hdf5/serial \
+            h5py
+
+# CACHE_STOP is used to rerun future commands, otherwise downloading the .whl will be cached and will not pull the most recent version
+ARG CACHE_STOP=1
+RUN if [ ${TF_PACKAGE} = tensorflow-gpu ]; then \
+        BASE=https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Release_Build/lastSuccessfulBuild/; \
+    elif [ ${TF_PACKAGE} = tf-nightly-gpu ]; then \
+        BASE=https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Nightly_Artifact/lastSuccessfulBuild/; \
+    elif [ ${TF_PACKAGE} = tensorflow ]; then \
+        BASE=https://powerci.osuosl.org/job/TensorFlow_PPC64LE_CPU_Release_Build/lastSuccessfulBuild/; \
+    elif [ ${TF_PACKAGE} = tf-nightly ]; then \
+        BASE=https://powerci.osuosl.org/job/TensorFlow_PPC64LE_CPU_Nightly_Artifact/lastSuccessfulBuild/; \
+    fi; \
+    MAJOR=`${PYTHON} -c 'import sys; print(sys.version_info[0])'`; \
+    MINOR=`${PYTHON} -c 'import sys; print(sys.version_info[1])'`; \
+    PACKAGE=$(wget -qO- ${BASE}"api/xml?xpath=//fileName&wrapper=artifacts" | grep -o "[^<>]*cp${MAJOR}${MINOR}[^<>]*.whl"); \
+    wget ${BASE}"artifact/tensorflow_pkg/"${PACKAGE}; \
+    ${PIP} install ${PACKAGE}
+
+COPY bashrc /etc/bash.bashrc
+RUN chmod a+rwx /etc/bash.bashrc
+
+RUN ${PIP} install jupyter matplotlib
+
+RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/
+RUN mkdir /.local && chmod a+rwx /.local
+RUN apt-get install -y --no-install-recommends wget
+WORKDIR /tf/tensorflow-tutorials
+RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_classification.ipynb
+RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_text_classification.ipynb
+COPY readme-for-jupyter.md README.md
+RUN apt-get autoremove -y && apt-get remove -y wget
+WORKDIR /tf
+EXPOSE 8888
+
+RUN ${PYTHON} -m ipykernel.kernelspec
+
+CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/tf --ip 0.0.0.0 --no-browser --allow-root"]
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/cpu-ppc64le.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/cpu-ppc64le.Dockerfile
new file mode 100644
index 0000000000..083d61bf9a
--- /dev/null
+++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/cpu-ppc64le.Dockerfile
@@ -0,0 +1,75 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+#
+# THIS IS A GENERATED DOCKERFILE.
+#
+# This file was assembled from multiple pieces, whose use is documented
+# throughout. Please refer to the TensorFlow dockerfiles documentation
+# for more information.
+
+ARG UBUNTU_VERSION=16.04
+
+FROM ubuntu:${UBUNTU_VERSION} as base
+
+ARG USE_PYTHON_3_NOT_2
+ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3}
+ARG PYTHON=python${_PY_SUFFIX}
+ARG PIP=pip${_PY_SUFFIX}
+
+# See http://bugs.python.org/issue19846
+ENV LANG C.UTF-8
+
+RUN apt-get update && apt-get install -y \
+    ${PYTHON} \
+    ${PYTHON}-pip
+
+RUN ${PIP} --no-cache-dir install --upgrade \
+    pip \
+    setuptools
+
+# Some TF tools expect a "python" binary
+RUN ln -s $(which ${PYTHON}) /usr/local/bin/python 
+
+# Options:
+#   tensorflow
+#   tensorflow-gpu
+#   tf-nightly
+#   tf-nightly-gpu
+ARG TF_PACKAGE=tensorflow
+RUN apt-get update && apt-get install -y wget libhdf5-dev
+RUN ${PIP} install --global-option=build_ext \
+            --global-option=-I/usr/include/hdf5/serial/ \
+            --global-option=-L/usr/lib/powerpc64le-linux-gnu/hdf5/serial \
+            h5py
+
+# CACHE_STOP is used to rerun future commands, otherwise downloading the .whl will be cached and will not pull the most recent version
+ARG CACHE_STOP=1
+RUN if [ ${TF_PACKAGE} = tensorflow-gpu ]; then \
+        BASE=https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Release_Build/lastSuccessfulBuild/; \
+    elif [ ${TF_PACKAGE} = tf-nightly-gpu ]; then \
+        BASE=https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Nightly_Artifact/lastSuccessfulBuild/; \
+    elif [ ${TF_PACKAGE} = tensorflow ]; then \
+        BASE=https://powerci.osuosl.org/job/TensorFlow_PPC64LE_CPU_Release_Build/lastSuccessfulBuild/; \
+    elif [ ${TF_PACKAGE} = tf-nightly ]; then \
+        BASE=https://powerci.osuosl.org/job/TensorFlow_PPC64LE_CPU_Nightly_Artifact/lastSuccessfulBuild/; \
+    fi; \
+    MAJOR=`${PYTHON} -c 'import sys; print(sys.version_info[0])'`; \
+    MINOR=`${PYTHON} -c 'import sys; print(sys.version_info[1])'`; \
+    PACKAGE=$(wget -qO- ${BASE}"api/xml?xpath=//fileName&wrapper=artifacts" | grep -o "[^<>]*cp${MAJOR}${MINOR}[^<>]*.whl"); \
+    wget ${BASE}"artifact/tensorflow_pkg/"${PACKAGE}; \
+    ${PIP} install ${PACKAGE}
+
+COPY bashrc /etc/bash.bashrc
+RUN chmod a+rwx /etc/bash.bashrc
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le-jupyter.Dockerfile
new file mode 100644
index 0000000000..1f32849735
--- /dev/null
+++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le-jupyter.Dockerfile
@@ -0,0 +1,125 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+#
+# THIS IS A GENERATED DOCKERFILE.
+#
+# This file was assembled from multiple pieces, whose use is documented
+# throughout. Please refer to the TensorFlow dockerfiles documentation
+# for more information.
+
+ARG UBUNTU_VERSION=16.04
+
+FROM ubuntu:${UBUNTU_VERSION} AS base
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        build-essential \
+        curl \
+        git \
+        libcurl3-dev \
+        libfreetype6-dev \
+        libhdf5-serial-dev \
+        libzmq3-dev \
+        pkg-config \
+        rsync \
+        software-properties-common \
+        unzip \
+        zip \
+        zlib1g-dev \
+        openjdk-8-jdk \
+        openjdk-8-jre-headless \
+        && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV CI_BUILD_PYTHON python
+
+# CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version
+ARG CACHE_STOP=1
+# Check out TensorFlow source code if --build_arg CHECKOUT_TF_SRC=1
+ARG CHECKOUT_TF_SRC=0
+RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src || true
+
+ARG USE_PYTHON_3_NOT_2
+ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3}
+ARG PYTHON=python${_PY_SUFFIX}
+ARG PIP=pip${_PY_SUFFIX}
+
+# See http://bugs.python.org/issue19846
+ENV LANG C.UTF-8
+
+RUN apt-get update && apt-get install -y \
+    ${PYTHON} \
+    ${PYTHON}-pip
+
+RUN ${PIP} --no-cache-dir install --upgrade \
+    pip \
+    setuptools
+
+# Some TF tools expect a "python" binary
+RUN ln -s $(which ${PYTHON}) /usr/local/bin/python 
+
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    curl \
+    git \
+    openjdk-8-jdk \
+    ${PYTHON}-dev \
+    swig
+
+RUN ${PIP} --no-cache-dir install \
+    Pillow \
+    h5py \
+    keras_applications \
+    keras_preprocessing \
+    matplotlib \
+    mock \
+    numpy \
+    scipy \
+    sklearn \
+    pandas \
+    && test "${USE_PYTHON_3_NOT_2}" -eq 1 && true || ${PIP} --no-cache-dir install \
+    enum34
+
+ # Build and install bazel
+ENV BAZEL_VERSION 0.15.0
+WORKDIR /
+RUN mkdir /bazel && \
+    cd /bazel && \
+    curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-dist.zip && \
+    unzip bazel-$BAZEL_VERSION-dist.zip && \
+    bash ./compile.sh && \
+    cp output/bazel /usr/local/bin/ && \
+    rm -rf /bazel && \
+    cd -
+
+COPY bashrc /etc/bash.bashrc
+RUN chmod a+rwx /etc/bash.bashrc
+
+RUN ${PIP} install jupyter matplotlib
+
+RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/
+RUN mkdir /.local && chmod a+rwx /.local
+RUN apt-get install -y --no-install-recommends wget
+WORKDIR /tf/tensorflow-tutorials
+RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_classification.ipynb
+RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_text_classification.ipynb
+COPY readme-for-jupyter.md README.md
+RUN apt-get autoremove -y && apt-get remove -y wget
+WORKDIR /tf
+EXPOSE 8888
+
+RUN ${PYTHON} -m ipykernel.kernelspec
+
+CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/tf --ip 0.0.0.0 --no-browser --allow-root"]
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le.Dockerfile
new file mode 100644
index 0000000000..cda51c371d
--- /dev/null
+++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le.Dockerfile
@@ -0,0 +1,108 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+#
+# THIS IS A GENERATED DOCKERFILE.
+#
+# This file was assembled from multiple pieces, whose use is documented
+# throughout. Please refer to the TensorFlow dockerfiles documentation
+# for more information.
+
+ARG UBUNTU_VERSION=16.04
+
+FROM ubuntu:${UBUNTU_VERSION} AS base
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        build-essential \
+        curl \
+        git \
+        libcurl3-dev \
+        libfreetype6-dev \
+        libhdf5-serial-dev \
+        libzmq3-dev \
+        pkg-config \
+        rsync \
+        software-properties-common \
+        unzip \
+        zip \
+        zlib1g-dev \
+        openjdk-8-jdk \
+        openjdk-8-jre-headless \
+        && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV CI_BUILD_PYTHON python
+
+# CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version
+ARG CACHE_STOP=1
+# Check out TensorFlow source code if --build_arg CHECKOUT_TF_SRC=1
+ARG CHECKOUT_TF_SRC=0
+RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src || true
+
+ARG USE_PYTHON_3_NOT_2
+ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3}
+ARG PYTHON=python${_PY_SUFFIX}
+ARG PIP=pip${_PY_SUFFIX}
+
+# See http://bugs.python.org/issue19846
+ENV LANG C.UTF-8
+
+RUN apt-get update && apt-get install -y \
+    ${PYTHON} \
+    ${PYTHON}-pip
+
+RUN ${PIP} --no-cache-dir install --upgrade \
+    pip \
+    setuptools
+
+# Some TF tools expect a "python" binary
+RUN ln -s $(which ${PYTHON}) /usr/local/bin/python 
+
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    curl \
+    git \
+    openjdk-8-jdk \
+    ${PYTHON}-dev \
+    swig
+
+RUN ${PIP} --no-cache-dir install \
+    Pillow \
+    h5py \
+    keras_applications \
+    keras_preprocessing \
+    matplotlib \
+    mock \
+    numpy \
+    scipy \
+    sklearn \
+    pandas \
+    && test "${USE_PYTHON_3_NOT_2}" -eq 1 && true || ${PIP} --no-cache-dir install \
+    enum34
+
+ # Build and install bazel
+ENV BAZEL_VERSION 0.15.0
+WORKDIR /
+RUN mkdir /bazel && \
+    cd /bazel && \
+    curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-dist.zip && \
+    unzip bazel-$BAZEL_VERSION-dist.zip && \
+    bash ./compile.sh && \
+    cp output/bazel /usr/local/bin/ && \
+    rm -rf /bazel && \
+    cd -
+
+COPY bashrc /etc/bash.bashrc
+RUN chmod a+rwx /etc/bash.bashrc
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le-jupyter.Dockerfile
new file mode 100644
index 0000000000..d8ee19f66e
--- /dev/null
+++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le-jupyter.Dockerfile
@@ -0,0 +1,151 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+#
+# THIS IS A GENERATED DOCKERFILE.
+#
+# This file was assembled from multiple pieces, whose use is documented
+# throughout. Please refer to the TensorFlow dockerfiles documentation
+# for more information.
+
+ARG UBUNTU_VERSION=16.04
+
+ARG ARCH=
+ARG CUDA=10.0
+FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}-base-ubuntu${UBUNTU_VERSION} as base
+ARG CUDNN=7.4.1.5-1
+ARG LIB_DIR_PREFIX=x84_64
+
+# Needed for string substitution 
+SHELL ["/bin/bash", "-c"]
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        build-essential \
+        cuda-command-line-tools-${CUDA/./-} \
+        cuda-cublas-dev-${CUDA/./-} \
+        cuda-cudart-dev-${CUDA/./-} \
+        cuda-cufft-dev-${CUDA/./-} \
+        cuda-curand-dev-${CUDA/./-} \
+        cuda-cusolver-dev-${CUDA/./-} \
+        cuda-cusparse-dev-${CUDA/./-} \
+        libcudnn7=${CUDNN}+cuda${CUDA} \
+        libcudnn7-dev=${CUDNN}+cuda${CUDA} \
+        libcurl3-dev \
+        libfreetype6-dev \
+        libhdf5-serial-dev \
+        libzmq3-dev \
+        pkg-config \
+        rsync \
+        software-properties-common \
+        unzip \
+        zip \
+        zlib1g-dev \
+        wget \
+        git \
+        && \
+    find /usr/local/cuda-${CUDA}/lib64/ -type f -name 'lib*_static.a' -not -name 'libcudart_static.a' -delete && \
+    rm /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libcudnn_static_v7.a
+
+RUN [ ${ARCH} = ppc64le ] || (apt-get update && \
+        apt-get install nvinfer-runtime-trt-repo-ubuntu1604-5.0.2-ga-cuda${CUDA} \
+        && apt-get update \
+        && apt-get install -y --no-install-recommends libnvinfer5=5.0.2-1+cuda${CUDA} \
+        && apt-get clean \
+        && rm -rf /var/lib/apt/lists/*)
+
+# Configure the build for our CUDA configuration.
+ENV CI_BUILD_PYTHON python
+ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
+ENV TF_NEED_CUDA 1
+ENV TF_NEED_TENSORRT 1
+ENV TF_CUDA_COMPUTE_CAPABILITIES=3.5,5.2,6.0,6.1,7.0
+ENV TF_CUDA_VERSION=${CUDA}
+ENV TF_CUDNN_VERSION=${CUDNN%%.*}
+# CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version
+ARG CACHE_STOP=1
+# Check out TensorFlow source code if --build_arg CHECKOUT_TF_SRC=1
+ARG CHECKOUT_TF_SRC=0
+RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src || true
+
+ARG USE_PYTHON_3_NOT_2
+ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3}
+ARG PYTHON=python${_PY_SUFFIX}
+ARG PIP=pip${_PY_SUFFIX}
+
+# See http://bugs.python.org/issue19846
+ENV LANG C.UTF-8
+
+RUN apt-get update && apt-get install -y \
+    ${PYTHON} \
+    ${PYTHON}-pip
+
+RUN ${PIP} --no-cache-dir install --upgrade \
+    pip \
+    setuptools
+
+# Some TF tools expect a "python" binary
+RUN ln -s $(which ${PYTHON}) /usr/local/bin/python 
+
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    curl \
+    git \
+    openjdk-8-jdk \
+    ${PYTHON}-dev \
+    swig
+
+RUN ${PIP} --no-cache-dir install \
+    Pillow \
+    h5py \
+    keras_applications \
+    keras_preprocessing \
+    matplotlib \
+    mock \
+    numpy \
+    scipy \
+    sklearn \
+    pandas \
+    && test "${USE_PYTHON_3_NOT_2}" -eq 1 && true || ${PIP} --no-cache-dir install \
+    enum34
+
+ # Build and install bazel
+ENV BAZEL_VERSION 0.15.0
+WORKDIR /
+RUN mkdir /bazel && \
+    cd /bazel && \
+    curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-dist.zip && \
+    unzip bazel-$BAZEL_VERSION-dist.zip && \
+    bash ./compile.sh && \
+    cp output/bazel /usr/local/bin/ && \
+    rm -rf /bazel && \
+    cd -
+
+COPY bashrc /etc/bash.bashrc
+RUN chmod a+rwx /etc/bash.bashrc
+
+RUN ${PIP} install jupyter matplotlib
+
+RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/
+RUN mkdir /.local && chmod a+rwx /.local
+RUN apt-get install -y --no-install-recommends wget
+WORKDIR /tf/tensorflow-tutorials
+RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_classification.ipynb
+RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_text_classification.ipynb
+COPY readme-for-jupyter.md README.md
+RUN apt-get autoremove -y && apt-get remove -y wget
+WORKDIR /tf
+EXPOSE 8888
+
+RUN ${PYTHON} -m ipykernel.kernelspec
+
+CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/tf --ip 0.0.0.0 --no-browser --allow-root"]
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le.Dockerfile
new file mode 100644
index 0000000000..966070634b
--- /dev/null
+++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le.Dockerfile
@@ -0,0 +1,134 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+#
+# THIS IS A GENERATED DOCKERFILE.
+#
+# This file was assembled from multiple pieces, whose use is documented
+# throughout. Please refer to the TensorFlow dockerfiles documentation
+# for more information.
+
+ARG UBUNTU_VERSION=16.04
+
+ARG ARCH=
+ARG CUDA=10.0
+FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}-base-ubuntu${UBUNTU_VERSION} as base
+ARG CUDNN=7.4.1.5-1
+ARG LIB_DIR_PREFIX=x84_64
+
+# Needed for string substitution 
+SHELL ["/bin/bash", "-c"]
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        build-essential \
+        cuda-command-line-tools-${CUDA/./-} \
+        cuda-cublas-dev-${CUDA/./-} \
+        cuda-cudart-dev-${CUDA/./-} \
+        cuda-cufft-dev-${CUDA/./-} \
+        cuda-curand-dev-${CUDA/./-} \
+        cuda-cusolver-dev-${CUDA/./-} \
+        cuda-cusparse-dev-${CUDA/./-} \
+        libcudnn7=${CUDNN}+cuda${CUDA} \
+        libcudnn7-dev=${CUDNN}+cuda${CUDA} \
+        libcurl3-dev \
+        libfreetype6-dev \
+        libhdf5-serial-dev \
+        libzmq3-dev \
+        pkg-config \
+        rsync \
+        software-properties-common \
+        unzip \
+        zip \
+        zlib1g-dev \
+        wget \
+        git \
+        && \
+    find /usr/local/cuda-${CUDA}/lib64/ -type f -name 'lib*_static.a' -not -name 'libcudart_static.a' -delete && \
+    rm /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libcudnn_static_v7.a
+
+RUN [ ${ARCH} = ppc64le ] || (apt-get update && \
+        apt-get install nvinfer-runtime-trt-repo-ubuntu1604-5.0.2-ga-cuda${CUDA} \
+        && apt-get update \
+        && apt-get install -y --no-install-recommends libnvinfer5=5.0.2-1+cuda${CUDA} \
+        && apt-get clean \
+        && rm -rf /var/lib/apt/lists/*)
+
+# Configure the build for our CUDA configuration.
+ENV CI_BUILD_PYTHON python
+ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
+ENV TF_NEED_CUDA 1
+ENV TF_NEED_TENSORRT 1
+ENV TF_CUDA_COMPUTE_CAPABILITIES=3.5,5.2,6.0,6.1,7.0
+ENV TF_CUDA_VERSION=${CUDA}
+ENV TF_CUDNN_VERSION=${CUDNN%%.*}
+# CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version
+ARG CACHE_STOP=1
+# Check out TensorFlow source code if --build_arg CHECKOUT_TF_SRC=1
+ARG CHECKOUT_TF_SRC=0
+RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src || true
+
+ARG USE_PYTHON_3_NOT_2
+ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3}
+ARG PYTHON=python${_PY_SUFFIX}
+ARG PIP=pip${_PY_SUFFIX}
+
+# See http://bugs.python.org/issue19846
+ENV LANG C.UTF-8
+
+RUN apt-get update && apt-get install -y \
+    ${PYTHON} \
+    ${PYTHON}-pip
+
+RUN ${PIP} --no-cache-dir install --upgrade \
+    pip \
+    setuptools
+
+# Some TF tools expect a "python" binary
+RUN ln -s $(which ${PYTHON}) /usr/local/bin/python 
+
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    curl \
+    git \
+    openjdk-8-jdk \
+    ${PYTHON}-dev \
+    swig
+
+RUN ${PIP} --no-cache-dir install \
+    Pillow \
+    h5py \
+    keras_applications \
+    keras_preprocessing \
+    matplotlib \
+    mock \
+    numpy \
+    scipy \
+    sklearn \
+    pandas \
+    && test "${USE_PYTHON_3_NOT_2}" -eq 1 && true || ${PIP} --no-cache-dir install \
+    enum34
+
+ # Build and install bazel
+ENV BAZEL_VERSION 0.15.0
+WORKDIR /
+RUN mkdir /bazel && \
+    cd /bazel && \
+    curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-dist.zip && \
+    unzip bazel-$BAZEL_VERSION-dist.zip && \
+    bash ./compile.sh && \
+    cp output/bazel /usr/local/bin/ && \
+    rm -rf /bazel && \
+    cd -
+
+COPY bashrc /etc/bash.bashrc
+RUN chmod a+rwx /etc/bash.bashrc
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le-jupyter.Dockerfile
new file mode 100644
index 0000000000..449a8d8aa8
--- /dev/null
+++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le-jupyter.Dockerfile
@@ -0,0 +1,125 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+#
+# THIS IS A GENERATED DOCKERFILE.
+#
+# This file was assembled from multiple pieces, whose use is documented
+# throughout. Please refer to the TensorFlow dockerfiles documentation
+# for more information.
+
+ARG UBUNTU_VERSION=16.04
+
+ARG ARCH=
+ARG CUDA=10.0
+FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}-base-ubuntu${UBUNTU_VERSION} as base
+ARG CUDNN=7.4.1.5-1
+
+# Needed for string substitution 
+SHELL ["/bin/bash", "-c"]
+# Pick up some TF dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        build-essential \
+        cuda-command-line-tools-${CUDA/./-} \
+        cuda-cublas-${CUDA/./-} \
+        cuda-cufft-${CUDA/./-} \
+        cuda-curand-${CUDA/./-} \
+        cuda-cusolver-${CUDA/./-} \
+        cuda-cusparse-${CUDA/./-} \
+        curl \
+        libcudnn7=${CUDNN}+cuda${CUDA} \
+        libfreetype6-dev \
+        libhdf5-serial-dev \
+        libzmq3-dev \
+        pkg-config \
+        software-properties-common \
+        unzip
+
+RUN [ ${ARCH} = ppc64le ] || (apt-get update && \
+        apt-get install nvinfer-runtime-trt-repo-ubuntu1604-5.0.2-ga-cuda${CUDA} \
+        && apt-get update \
+        && apt-get install -y --no-install-recommends libnvinfer5=5.0.2-1+cuda${CUDA} \
+        && apt-get clean \
+        && rm -rf /var/lib/apt/lists/*)
+
+# For CUDA profiling, TensorFlow requires CUPTI.
+ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
+
+ARG USE_PYTHON_3_NOT_2
+ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3}
+ARG PYTHON=python${_PY_SUFFIX}
+ARG PIP=pip${_PY_SUFFIX}
+
+# See http://bugs.python.org/issue19846
+ENV LANG C.UTF-8
+
+RUN apt-get update && apt-get install -y \
+    ${PYTHON} \
+    ${PYTHON}-pip
+
+RUN ${PIP} --no-cache-dir install --upgrade \
+    pip \
+    setuptools
+
+# Some TF tools expect a "python" binary
+RUN ln -s $(which ${PYTHON}) /usr/local/bin/python 
+
+# Options:
+#   tensorflow
+#   tensorflow-gpu
+#   tf-nightly
+#   tf-nightly-gpu
+ARG TF_PACKAGE=tensorflow
+RUN apt-get update && apt-get install -y wget libhdf5-dev
+RUN ${PIP} install --global-option=build_ext \
+            --global-option=-I/usr/include/hdf5/serial/ \
+            --global-option=-L/usr/lib/powerpc64le-linux-gnu/hdf5/serial \
+            h5py
+
+# CACHE_STOP is used to rerun future commands, otherwise downloading the .whl will be cached and will not pull the most recent version
+ARG CACHE_STOP=1
+RUN if [ ${TF_PACKAGE} = tensorflow-gpu ]; then \
+        BASE=https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Release_Build/lastSuccessfulBuild/; \
+    elif [ ${TF_PACKAGE} = tf-nightly-gpu ]; then \
+        BASE=https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Nightly_Artifact/lastSuccessfulBuild/; \
+    elif [ ${TF_PACKAGE} = tensorflow ]; then \
+        BASE=https://powerci.osuosl.org/job/TensorFlow_PPC64LE_CPU_Release_Build/lastSuccessfulBuild/; \
+    elif [ ${TF_PACKAGE} = tf-nightly ]; then \
+        BASE=https://powerci.osuosl.org/job/TensorFlow_PPC64LE_CPU_Nightly_Artifact/lastSuccessfulBuild/; \
+    fi; \
+    MAJOR=`${PYTHON} -c 'import sys; print(sys.version_info[0])'`; \
+    MINOR=`${PYTHON} -c 'import sys; print(sys.version_info[1])'`; \
+    PACKAGE=$(wget -qO- ${BASE}"api/xml?xpath=//fileName&wrapper=artifacts" | grep -o "[^<>]*cp${MAJOR}${MINOR}[^<>]*.whl"); \
+    wget ${BASE}"artifact/tensorflow_pkg/"${PACKAGE}; \
+    ${PIP} install ${PACKAGE}
+
+COPY bashrc /etc/bash.bashrc
+RUN chmod a+rwx /etc/bash.bashrc
+
+RUN ${PIP} install jupyter matplotlib
+
+RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/
+RUN mkdir /.local && chmod a+rwx /.local
+RUN apt-get install -y --no-install-recommends wget
+WORKDIR /tf/tensorflow-tutorials
+RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_classification.ipynb
+RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_text_classification.ipynb
+COPY readme-for-jupyter.md README.md
+RUN apt-get autoremove -y && apt-get remove -y wget
+WORKDIR /tf
+EXPOSE 8888
+
+RUN ${PYTHON} -m ipykernel.kernelspec
+
+CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/tf --ip 0.0.0.0 --no-browser --allow-root"]
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le.Dockerfile
new file mode 100644
index 0000000000..f01a47f1c0
--- /dev/null
+++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le.Dockerfile
@@ -0,0 +1,108 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+#
+# THIS IS A GENERATED DOCKERFILE.
+#
+# This file was assembled from multiple pieces, whose use is documented
+# throughout. Please refer to the TensorFlow dockerfiles documentation
+# for more information.
+
+ARG UBUNTU_VERSION=16.04
+
+ARG ARCH=
+ARG CUDA=10.0
+FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}-base-ubuntu${UBUNTU_VERSION} as base
+ARG CUDNN=7.4.1.5-1
+
+# Needed for string substitution 
+SHELL ["/bin/bash", "-c"]
+# Pick up some TF dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        build-essential \
+        cuda-command-line-tools-${CUDA/./-} \
+        cuda-cublas-${CUDA/./-} \
+        cuda-cufft-${CUDA/./-} \
+        cuda-curand-${CUDA/./-} \
+        cuda-cusolver-${CUDA/./-} \
+        cuda-cusparse-${CUDA/./-} \
+        curl \
+        libcudnn7=${CUDNN}+cuda${CUDA} \
+        libfreetype6-dev \
+        libhdf5-serial-dev \
+        libzmq3-dev \
+        pkg-config \
+        software-properties-common \
+        unzip
+
+RUN [ ${ARCH} = ppc64le ] || (apt-get update && \
+        apt-get install nvinfer-runtime-trt-repo-ubuntu1604-5.0.2-ga-cuda${CUDA} \
+        && apt-get update \
+        && apt-get install -y --no-install-recommends libnvinfer5=5.0.2-1+cuda${CUDA} \
+        && apt-get clean \
+        && rm -rf /var/lib/apt/lists/*)
+
+# For CUDA profiling, TensorFlow requires CUPTI.
+ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
+
+ARG USE_PYTHON_3_NOT_2
+ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3}
+ARG PYTHON=python${_PY_SUFFIX}
+ARG PIP=pip${_PY_SUFFIX}
+
+# See http://bugs.python.org/issue19846
+ENV LANG C.UTF-8
+
+RUN apt-get update && apt-get install -y \
+    ${PYTHON} \
+    ${PYTHON}-pip
+
+RUN ${PIP} --no-cache-dir install --upgrade \
+    pip \
+    setuptools
+
+# Some TF tools expect a "python" binary
+RUN ln -s $(which ${PYTHON}) /usr/local/bin/python 
+
+# Options:
+#   tensorflow
+#   tensorflow-gpu
+#   tf-nightly
+#   tf-nightly-gpu
+ARG TF_PACKAGE=tensorflow
+RUN apt-get update && apt-get install -y wget libhdf5-dev
+RUN ${PIP} install --global-option=build_ext \
+            --global-option=-I/usr/include/hdf5/serial/ \
+            --global-option=-L/usr/lib/powerpc64le-linux-gnu/hdf5/serial \
+            h5py
+
+# CACHE_STOP is used to rerun future commands, otherwise downloading the .whl will be cached and will not pull the most recent version
+ARG CACHE_STOP=1
+RUN if [ ${TF_PACKAGE} = tensorflow-gpu ]; then \
+        BASE=https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Release_Build/lastSuccessfulBuild/; \
+    elif [ ${TF_PACKAGE} = tf-nightly-gpu ]; then \
+        BASE=https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Nightly_Artifact/lastSuccessfulBuild/; \
+    elif [ ${TF_PACKAGE} = tensorflow ]; then \
+        BASE=https://powerci.osuosl.org/job/TensorFlow_PPC64LE_CPU_Release_Build/lastSuccessfulBuild/; \
+    elif [ ${TF_PACKAGE} = tf-nightly ]; then \
+        BASE=https://powerci.osuosl.org/job/TensorFlow_PPC64LE_CPU_Nightly_Artifact/lastSuccessfulBuild/; \
+    fi; \
+    MAJOR=`${PYTHON} -c 'import sys; print(sys.version_info[0])'`; \
+    MINOR=`${PYTHON} -c 'import sys; print(sys.version_info[1])'`; \
+    PACKAGE=$(wget -qO- ${BASE}"api/xml?xpath=//fileName&wrapper=artifacts" | grep -o "[^<>]*cp${MAJOR}${MINOR}[^<>]*.whl"); \
+    wget ${BASE}"artifact/tensorflow_pkg/"${PACKAGE}; \
+    ${PIP} install ${PACKAGE}
+
+COPY bashrc /etc/bash.bashrc
+RUN chmod a+rwx /etc/bash.bashrc
diff --git a/tensorflow/tools/dockerfiles/partials/tensorflow-ppc64le.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/tensorflow-ppc64le.partial.Dockerfile
new file mode 100644
index 0000000000..1e79574a34
--- /dev/null
+++ b/tensorflow/tools/dockerfiles/partials/tensorflow-ppc64le.partial.Dockerfile
@@ -0,0 +1,28 @@
+# Options:
+#   tensorflow
+#   tensorflow-gpu
+#   tf-nightly
+#   tf-nightly-gpu
+ARG TF_PACKAGE=tensorflow
+RUN apt-get update && apt-get install -y wget libhdf5-dev
+RUN ${PIP} install --global-option=build_ext \
+            --global-option=-I/usr/include/hdf5/serial/ \
+            --global-option=-L/usr/lib/powerpc64le-linux-gnu/hdf5/serial \
+            h5py
+
+# CACHE_STOP is used to rerun future commands, otherwise downloading the .whl will be cached and will not pull the most recent version
+ARG CACHE_STOP=1
+RUN if [ ${TF_PACKAGE} = tensorflow-gpu ]; then \
+        BASE=https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Release_Build/lastSuccessfulBuild/; \
+    elif [ ${TF_PACKAGE} = tf-nightly-gpu ]; then \
+        BASE=https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Nightly_Artifact/lastSuccessfulBuild/; \
+    elif [ ${TF_PACKAGE} = tensorflow ]; then \
+        BASE=https://powerci.osuosl.org/job/TensorFlow_PPC64LE_CPU_Release_Build/lastSuccessfulBuild/; \
+    elif [ ${TF_PACKAGE} = tf-nightly ]; then \
+        BASE=https://powerci.osuosl.org/job/TensorFlow_PPC64LE_CPU_Nightly_Artifact/lastSuccessfulBuild/; \
+    fi; \
+    MAJOR=`${PYTHON} -c 'import sys; print(sys.version_info[0])'`; \
+    MINOR=`${PYTHON} -c 'import sys; print(sys.version_info[1])'`; \
+    PACKAGE=$(wget -qO- ${BASE}"api/xml?xpath=//fileName&wrapper=artifacts" | grep -o "[^<>]*cp${MAJOR}${MINOR}[^<>]*.whl"); \
+    wget ${BASE}"artifact/tensorflow_pkg/"${PACKAGE}; \
+    ${PIP} install ${PACKAGE}
diff --git a/tensorflow/tools/dockerfiles/partials/ubuntu/bazelbuild.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/bazelbuild.partial.Dockerfile
new file mode 100644
index 0000000000..0397ab5fa8
--- /dev/null
+++ b/tensorflow/tools/dockerfiles/partials/ubuntu/bazelbuild.partial.Dockerfile
@@ -0,0 +1,33 @@
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    curl \
+    git \
+    openjdk-8-jdk \
+    ${PYTHON}-dev \
+    swig
+
+RUN ${PIP} --no-cache-dir install \
+    Pillow \
+    h5py \
+    keras_applications \
+    keras_preprocessing \
+    matplotlib \
+    mock \
+    numpy \
+    scipy \
+    sklearn \
+    pandas \
+    && test "${USE_PYTHON_3_NOT_2}" -eq 1 && true || ${PIP} --no-cache-dir install \
+    enum34
+
+ # Build and install bazel
+ENV BAZEL_VERSION 0.15.0
+WORKDIR /
+RUN mkdir /bazel && \
+    cd /bazel && \
+    curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-dist.zip && \
+    unzip bazel-$BAZEL_VERSION-dist.zip && \
+    bash ./compile.sh && \
+    cp output/bazel /usr/local/bin/ && \
+    rm -rf /bazel && \
+    cd -
diff --git a/tensorflow/tools/dockerfiles/partials/ubuntu/devel-cpu.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/devel-cpu.partial.Dockerfile
index 0652ac4151..aaeda0b207 100644
--- a/tensorflow/tools/dockerfiles/partials/ubuntu/devel-cpu.partial.Dockerfile
+++ b/tensorflow/tools/dockerfiles/partials/ubuntu/devel-cpu.partial.Dockerfile
@@ -7,7 +7,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         libcurl3-dev \
         libfreetype6-dev \
         libhdf5-serial-dev \
-        libpng12-dev \
         libzmq3-dev \
         pkg-config \
         rsync \
@@ -20,9 +19,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
- 
+
 ENV CI_BUILD_PYTHON python
 
-# Check out TensorFlow source code if --build-arg CHECKOUT_TF_SRC=1
+# CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version
+ARG CACHE_STOP=1
+# Check out TensorFlow source code if --build_arg CHECKOUT_TF_SRC=1
 ARG CHECKOUT_TF_SRC=0
-RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src
+RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src || true
diff --git a/tensorflow/tools/dockerfiles/partials/ubuntu/devel-nvidia.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/devel-nvidia.partial.Dockerfile
index 2b4494ac59..8ce4a1879d 100644
--- a/tensorflow/tools/dockerfiles/partials/ubuntu/devel-nvidia.partial.Dockerfile
+++ b/tensorflow/tools/dockerfiles/partials/ubuntu/devel-nvidia.partial.Dockerfile
@@ -1,20 +1,25 @@
-FROM nvidia/cuda:10.0-base-ubuntu${UBUNTU_VERSION} as base
+ARG ARCH=
+ARG CUDA=10.0
+FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}-base-ubuntu${UBUNTU_VERSION} as base
+ARG CUDNN=7.4.1.5-1
+ARG LIB_DIR_PREFIX=x84_64
 
+# Needed for string substitution 
+SHELL ["/bin/bash", "-c"]
 RUN apt-get update && apt-get install -y --no-install-recommends \
         build-essential \
-        cuda-command-line-tools-10-0 \
-        cuda-cublas-dev-10-0 \
-        cuda-cudart-dev-10-0 \
-        cuda-cufft-dev-10-0 \
-        cuda-curand-dev-10-0 \
-        cuda-cusolver-dev-10-0 \
-        cuda-cusparse-dev-10-0 \
-        libcudnn7=7.4.1.5-1+cuda10.0 \
-        libcudnn7-dev=7.4.1.5-1+cuda10.0 \
+        cuda-command-line-tools-${CUDA/./-} \
+        cuda-cublas-dev-${CUDA/./-} \
+        cuda-cudart-dev-${CUDA/./-} \
+        cuda-cufft-dev-${CUDA/./-} \
+        cuda-curand-dev-${CUDA/./-} \
+        cuda-cusolver-dev-${CUDA/./-} \
+        cuda-cusparse-dev-${CUDA/./-} \
+        libcudnn7=${CUDNN}+cuda${CUDA} \
+        libcudnn7-dev=${CUDNN}+cuda${CUDA} \
         libcurl3-dev \
         libfreetype6-dev \
         libhdf5-serial-dev \
-        libpng12-dev \
         libzmq3-dev \
         pkg-config \
         rsync \
@@ -25,14 +30,15 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         wget \
         git \
         && \
-    find /usr/local/cuda-10.0/lib64/ -type f -name 'lib*_static.a' -not -name 'libcudart_static.a' -delete && \
-    rm /usr/lib/x86_64-linux-gnu/libcudnn_static_v7.a
+    find /usr/local/cuda-${CUDA}/lib64/ -type f -name 'lib*_static.a' -not -name 'libcudart_static.a' -delete && \
+    rm /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libcudnn_static_v7.a
 
-RUN apt-get update && \
-        apt-get install nvinfer-runtime-trt-repo-ubuntu1604-5.0.2-ga-cuda10.0 \
+RUN [ ${ARCH} = ppc64le ] || (apt-get update && \
+        apt-get install nvinfer-runtime-trt-repo-ubuntu1604-5.0.2-ga-cuda${CUDA} \
         && apt-get update \
-        && apt-get install -y --no-install-recommends libnvinfer-dev=5.0.2-1+cuda10.0 \
-        && rm -rf /var/lib/apt/lists/*
+        && apt-get install -y --no-install-recommends libnvinfer5=5.0.2-1+cuda${CUDA} \
+        && apt-get clean \
+        && rm -rf /var/lib/apt/lists/*)
 
 # Configure the build for our CUDA configuration.
 ENV CI_BUILD_PYTHON python
@@ -40,9 +46,10 @@ ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
 ENV TF_NEED_CUDA 1
 ENV TF_NEED_TENSORRT 1
 ENV TF_CUDA_COMPUTE_CAPABILITIES=3.5,5.2,6.0,6.1,7.0
-ENV TF_CUDA_VERSION=10.0
-ENV TF_CUDNN_VERSION=7
-
-# Check out TensorFlow source code if --build_arg CHECKOUT_TENSORFLOW=1
+ENV TF_CUDA_VERSION=${CUDA}
+ENV TF_CUDNN_VERSION=${CUDNN%%.*}
+# CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version
+ARG CACHE_STOP=1
+# Check out TensorFlow source code if --build_arg CHECKOUT_TF_SRC=1
 ARG CHECKOUT_TF_SRC=0
-RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src
+RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src || true
diff --git a/tensorflow/tools/dockerfiles/partials/ubuntu/nvidia.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/nvidia.partial.Dockerfile
index a6393a3280..1d40ed5f98 100644
--- a/tensorflow/tools/dockerfiles/partials/ubuntu/nvidia.partial.Dockerfile
+++ b/tensorflow/tools/dockerfiles/partials/ubuntu/nvidia.partial.Dockerfile
@@ -1,29 +1,34 @@
-FROM nvidia/cuda:10.0-base-ubuntu${UBUNTU_VERSION} as base
+ARG ARCH=
+ARG CUDA=10.0
+FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}-base-ubuntu${UBUNTU_VERSION} as base
+ARG CUDNN=7.4.1.5-1
 
+# Needed for string substitution 
+SHELL ["/bin/bash", "-c"]
 # Pick up some TF dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
         build-essential \
-        cuda-command-line-tools-10-0 \
-        cuda-cublas-10-0 \
-        cuda-cufft-10-0 \
-        cuda-curand-10-0 \
-        cuda-cusolver-10-0 \
-        cuda-cusparse-10-0 \
-        libcudnn7=7.4.1.5-1+cuda10.0 \
+        cuda-command-line-tools-${CUDA/./-} \
+        cuda-cublas-${CUDA/./-} \
+        cuda-cufft-${CUDA/./-} \
+        cuda-curand-${CUDA/./-} \
+        cuda-cusolver-${CUDA/./-} \
+        cuda-cusparse-${CUDA/./-} \
+        curl \
+        libcudnn7=${CUDNN}+cuda${CUDA} \
         libfreetype6-dev \
         libhdf5-serial-dev \
-        libpng12-dev \
         libzmq3-dev \
         pkg-config \
         software-properties-common \
         unzip
 
-RUN apt-get update && \
-        apt-get install nvinfer-runtime-trt-repo-ubuntu1604-5.0.2-ga-cuda10.0 \
+RUN [ ${ARCH} = ppc64le ] || (apt-get update && \
+        apt-get install nvinfer-runtime-trt-repo-ubuntu1604-5.0.2-ga-cuda${CUDA} \
         && apt-get update \
-        && apt-get install -y --no-install-recommends libnvinfer5=5.0.2-1+cuda10.0 \
+        && apt-get install -y --no-install-recommends libnvinfer5=5.0.2-1+cuda${CUDA} \
         && apt-get clean \
-        && rm -rf /var/lib/apt/lists/*
+        && rm -rf /var/lib/apt/lists/*)
 
 # For CUDA profiling, TensorFlow requires CUPTI.
 ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
diff --git a/tensorflow/tools/dockerfiles/spec.yml b/tensorflow/tools/dockerfiles/spec.yml
index 19d96e7a3d..d19b1d15fc 100644
--- a/tensorflow/tools/dockerfiles/spec.yml
+++ b/tensorflow/tools/dockerfiles/spec.yml
@@ -56,6 +56,13 @@ releases:
             - "{ubuntu}{jupyter}"
             - "{ubuntu-devel}{jupyter}"
 
+    ppc64le-dockerfiles:
+        is_dockerfiles: true
+        upload_images: false
+        tag_specs:
+            - "{ubuntu-ppc64le}{jupyter}"
+            - "{ubuntu-devel-ppc64le}{jupyter}"
+
 slice_sets:
 
     py:
@@ -122,6 +129,70 @@ slice_sets:
           args:
               - CHECKOUT_TF_SRC=1
 
+    ubuntu-ppc64le:
+        - add_to_name: "-ppc64le"
+          dockerfile_exclusive_name: "cpu-ppc64le"
+          dockerfile_subdirectory: "ppc64le"
+          args:
+              - UBUNTU_VERSION=18.04
+          partials:
+              - ubuntu/version
+              - ubuntu/cpu
+              - ubuntu/python
+              - tensorflow-ppc64le
+              - shell
+        - add_to_name: "-gpu-ppc64le"
+          dockerfile_exclusive_name: "gpu-ppc64le"
+          dockerfile_subdirectory: "ppc64le"
+          args:
+              - UBUNTU_VERSION=18.04
+              - ARCH=ppc64le
+              - CUDA=10.0
+              - TF_PACKAGE=tensorflow-gpu
+          partials:
+              - ubuntu/version
+              - ubuntu/nvidia
+              - ubuntu/python
+              - tensorflow-ppc64le
+              - shell
+          tests:
+              - import-gpu.sh
+          test_runtime: nvidia
+
+    ubuntu-devel-ppc64le:
+        - add_to_name: "devel-ppc64le"
+          dockerfile_exclusive_name: "devel-cpu-ppc64le"
+          dockerfile_subdirectory: "ppc64le"
+          partials:
+              - ubuntu/version
+              - ubuntu/devel-cpu
+              - ubuntu/python
+              - ubuntu/bazelbuild
+              - shell
+          tests:
+              - build-cpu.sh
+          args:
+              - UBUNTU_VERSION=18.04
+              - CHECKOUT_TF_SRC=1
+        - add_to_name: "devel-gpu-ppc64le"
+          dockerfile_exclusive_name: "devel-gpu-ppc64le"
+          dockerfile_subdirectory: "ppc64le"
+          args:
+              - UBUNTU_VERSION=18.04
+              - ARCH=ppc64le
+              - CUDA=10.0
+              - LIB_DIR_PREFIX=powerpc64le
+              - CHECKOUT_TF_SRC=1
+          partials:
+              - ubuntu/version
+              - ubuntu/devel-nvidia
+              - ubuntu/python
+              - ubuntu/bazelbuild
+              - shell
+          tests:
+              - build-gpu.sh
+          test_runtime: nvidia
+
     nightly:
         - add_to_name: "nightly"
           partials:
diff --git a/tensorflow/tools/dockerfiles/tools.Dockerfile b/tensorflow/tools/dockerfiles/tools.Dockerfile
index e8929295a5..a96b2578cb 100644
--- a/tensorflow/tools/dockerfiles/tools.Dockerfile
+++ b/tensorflow/tools/dockerfiles/tools.Dockerfile
@@ -17,7 +17,7 @@
 #
 # You can use this image to quickly develop changes to the Dockerfile assembler
 # or set of TF Docker partials. See README.md for usage instructions.
-FROM debian:stretch
+FROM ubuntu:16.04
 LABEL maintainer="Austin Anderson <angerson@google.com>"
 
 RUN apt-get update && apt-get install -y python3 python3-pip bash curl
-- 
GitLab


From 79bc4f4bf914d4b94eef89e8df2e19ff54c2db71 Mon Sep 17 00:00:00 2001
From: Siju Samuel <siju.samuel@huawei.com>
Date: Fri, 21 Dec 2018 10:31:19 +0530
Subject: [PATCH 0071/2345] Review comments updated, nnapi removed for ceil

---
 .../lite/delegates/nnapi/nnapi_delegate.cc    | 10 ----
 .../delegates/nnapi/nnapi_delegate_test.cc    | 49 -------------------
 tensorflow/lite/kernels/BUILD                 |  1 -
 .../internal/optimized/legacy_optimized_ops.h |  6 ---
 .../internal/reference/legacy_reference_ops.h |  6 ---
 tensorflow/lite/nnapi/NeuralNetworksShim.h    |  1 -
 tensorflow/lite/nnapi_delegate.cc             |  3 --
 7 files changed, 76 deletions(-)

diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
index cac98ae3da..4fe07004a8 100644
--- a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
+++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
@@ -638,16 +638,6 @@ class NNAPIDelegateKernel {
           return nullptr;
         }
         break;
-      case kTfLiteBuiltinCeil:
-        if (version == 1) {
-          return [](const NNAPIOpMappingArgs& mapping_args)
-                     -> ANeuralNetworksOperationType {
-            return ANEURALNETWORKS_CEIL;
-          };
-        } else {
-          return nullptr;
-        }
-        break;
       case kTfLiteBuiltinRelu:
         if (version == 1) {
           return [](const NNAPIOpMappingArgs& mapping_args)
diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
index f6a04e36cd..ca48af0c95 100644
--- a/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
+++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
@@ -1024,55 +1024,6 @@ TEST(NNAPIDelegate, FloorMultiDims) {
   EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 1, 1, 5}));
 }
 
-class CeilOpModel : public SingleOpModelWithNNAPI {
- public:
-  CeilOpModel(std::initializer_list<int> input_shape, TensorType input_type) {
-    input_ = AddInput(TensorType_FLOAT32);
-    output_ = AddOutput(TensorType_FLOAT32);
-    SetBuiltinOp(BuiltinOperator_CEIL, BuiltinOptions_NONE, 0);
-    BuildInterpreter({
-        input_shape,
-    });
-  }
-
-  int input() { return input_; }
-
-  std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
-  std::vector<int> GetOutputShape() { return GetTensorShape(output_); }
-
- private:
-  int input_;
-  int output_;
-};
-
-TEST(NNAPIDelegate, CeilSingleDim) {
-  CeilOpModel model({2}, TensorType_FLOAT32);
-  model.PopulateTensor<float>(model.input(), {8.5, 0.0});
-  model.Invoke();
-  EXPECT_THAT(model.GetOutput(), ElementsAreArray({8, 0}));
-  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2}));
-}
-
-TEST(NNAPIDelegate, CeilMultiDims) {
-  CeilOpModel model({2, 1, 1, 5}, TensorType_FLOAT32);
-  model.PopulateTensor<float>(model.input(), {
-                                                 0.0001,
-                                                 8.0001,
-                                                 0.9999,
-                                                 9.9999,
-                                                 0.5,
-                                                 -0.0001,
-                                                 -8.0001,
-                                                 -0.9999,
-                                                 -9.9999,
-                                                 -0.5,
-                                             });
-  model.Invoke();
-  EXPECT_THAT(model.GetOutput(),
-              ElementsAreArray({1, 9, 1, 10, 1, 0, -8, 0, -9, 0}));
-  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 1, 1, 5}));
-}
-
 class LocalResponseNormOpModel : public SingleOpModelWithNNAPI {
  public:
   LocalResponseNormOpModel(std::initializer_list<int> input_shape, int radius,
diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index 71d06ba4d7..1d53022e03 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -605,7 +605,6 @@ tf_cc_test(
     size = "small",
     srcs = ["ceil_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
diff --git a/tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h b/tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h
index a76649f934..5485d907c2 100644
--- a/tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h
+++ b/tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h
@@ -1740,12 +1740,6 @@ inline void Floor(const float* input_data, const Dims<4>& input_dims,
         output_data);
 }
 
-inline void Ceil(const float* input_data, const Dims<4>& input_dims,
-                 float* output_data, const Dims<4>& output_dims) {
-  Ceil(DimsToShape(input_dims), input_data, DimsToShape(output_dims),
-       output_data);
-}
-
 inline void ResizeBilinear(const float* input_data, const Dims<4>& input_dims,
                            const int32* output_size_data,
                            const Dims<4>& output_size_dims, float* output_data,
diff --git a/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h b/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h
index 431e2413e8..380fc8f98e 100644
--- a/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h
+++ b/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h
@@ -1883,12 +1883,6 @@ inline void Floor(const float* input_data, const Dims<4>& input_dims,
         output_data);
 }
 
-inline void Ceil(const float* input_data, const Dims<4>& input_dims,
-                 float* output_data, const Dims<4>& output_dims) {
-  Ceil(DimsToShape(input_dims), input_data, DimsToShape(output_dims),
-       output_data);
-}
-
 template <typename T>
 inline void ResizeBilinear(const T* input_data, const Dims<4>& input_dims,
                            const int32* output_size_data,
diff --git a/tensorflow/lite/nnapi/NeuralNetworksShim.h b/tensorflow/lite/nnapi/NeuralNetworksShim.h
index 82c5840952..c39502f4ac 100644
--- a/tensorflow/lite/nnapi/NeuralNetworksShim.h
+++ b/tensorflow/lite/nnapi/NeuralNetworksShim.h
@@ -143,7 +143,6 @@ enum {
   ANEURALNETWORKS_STRIDED_SLICE = 35,
   ANEURALNETWORKS_SUB = 36,
   ANEURALNETWORKS_TRANSPOSE = 37,
-  ANEURALNETWORKS_CEIL = 38,
 };
 
 /**
diff --git a/tensorflow/lite/nnapi_delegate.cc b/tensorflow/lite/nnapi_delegate.cc
index dfbb4813ad..26d75696a1 100644
--- a/tensorflow/lite/nnapi_delegate.cc
+++ b/tensorflow/lite/nnapi_delegate.cc
@@ -489,9 +489,6 @@ TfLiteStatus AddOpsAndParams(
       case tflite::BuiltinOperator_FLOOR:
         nn_op_type = ANEURALNETWORKS_FLOOR;
         break;
-      case tflite::BuiltinOperator_CEIL:
-        nn_op_type = ANEURALNETWORKS_CEIL;
-        break;
       case tflite::BuiltinOperator_LOGISTIC:
         nn_op_type = ANEURALNETWORKS_LOGISTIC;
         break;
-- 
GitLab


From 8da29925c65dc72b49f693942923519b38dd0242 Mon Sep 17 00:00:00 2001
From: Yves-Noel Weweler <y.weweler@gmail.com>
Date: Fri, 21 Dec 2018 15:58:22 +0100
Subject: [PATCH 0072/2345] Fix pylint warnings from Ubuntu Sanity test

---
 .../kernel_tests/bucket_by_sequence_length_test.py         | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
index a95d8e1049..4f1ea9f2ee 100644
--- a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
@@ -74,12 +74,13 @@ def _get_record_shape(sparse):
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class BucketBySequenceLengthTest(test_base.DatasetTestBase, parameterized.TestCase):
+class BucketBySequenceLengthTest(test_base.DatasetTestBase,
+                                 parameterized.TestCase):
 
   # TODO(b/117581999): add eager coverage.
   @parameterized.named_parameters(
-    ("WithoutPadding", True),
-    ("WithPadding", False),
+      ("WithoutPadding", True),
+      ("WithPadding", False),
   )
   def testSkipEagerBucketDropReminder(self, param_no_padding):
 
-- 
GitLab


From 7859702e8a7093f242c167ea027cc227f1f9d048 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 21 Dec 2018 08:57:29 -0800
Subject: [PATCH 0073/2345] Fix failing test in TRT4.0 - TRT bug with tensors
 rank < 3 in INT8 mode

---
 tensorflow/contrib/tensorrt/test/identity_output_test.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tensorflow/contrib/tensorrt/test/identity_output_test.py b/tensorflow/contrib/tensorrt/test/identity_output_test.py
index 2be9da9ede..da7d70876d 100644
--- a/tensorflow/contrib/tensorrt/test/identity_output_test.py
+++ b/tensorflow/contrib/tensorrt/test/identity_output_test.py
@@ -71,6 +71,11 @@ class IdentityTest(trt_test.TfTrtIntegrationTestBase):
     """Return the expected engines to build."""
     return ["TRTEngineOp_0"]
 
+  def ShouldRunTest(self, run_params):
+    """Whether to run the test."""
+    # TODO(aaroey): Trt 4.0 forbids conversion for tensors with rank <3 in int8
+    # mode, which is a bug. Re-enable this when trt library is fixed.
+    return not trt_test.IsQuantizationMode(run_params.precision_mode)
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From ef1fd4ab6cb076aee9af7801f5b66fb66a768fc4 Mon Sep 17 00:00:00 2001
From: Mahmoud Abuzaina <mahmoud.abuzaina@intel.com>
Date: Fri, 21 Dec 2018 09:37:24 -0800
Subject: [PATCH 0074/2345] Moved #endif of MKL

---
 tensorflow/core/ops/nn_ops.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc
index c7cd3140be..24d774ef59 100644
--- a/tensorflow/core/ops/nn_ops.cc
+++ b/tensorflow/core/ops/nn_ops.cc
@@ -2515,6 +2515,7 @@ NOTE Do not invoke this operator directly in Python. Graph rewrite pass is
 expected to invoke these operators.
 )doc");
 
+#endif  // INTEL_MKL
 REGISTER_OP("QuantizedConv2DAndRequantize")
     .Input("input: Tinput")
     .Input("filter: Tfilter")
@@ -2851,6 +2852,5 @@ REGISTER_OP("QuantizedConv2DWithBiasSignedSumAndReluAndRequantize")
       return Status::OK();
     });
 
-#endif  // INTEL_MKL
 
 }  // namespace tensorflow
-- 
GitLab


From 6bd27704c472b8d1cbd64b600fe00f223e74b4f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E7=8E=8B=E6=8C=AF=E5=8D=8E=20=28WANG=20Zhenhua=29?=
 <i@jackwish.net>
Date: Sat, 22 Dec 2018 10:35:47 +0800
Subject: [PATCH 0075/2345] lite: remove memset in resize bilinear opt op

memset() is not necessarily needed here since every element
in output_data memory will be updated.
---
 tensorflow/lite/kernels/internal/optimized/optimized_ops.h | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
index 7bc6b324c5..a5374827e1 100644
--- a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
+++ b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
@@ -5387,9 +5387,6 @@ inline void ResizeBilinearGenericSmallChannel(
     int32 output_height, int32 output_width, float height_scale,
     float width_scale, const RuntimeShape& input_shape, const T* input_data,
     const RuntimeShape& output_shape, T* output_data) {
-  memset(output_data, 0,
-         batches * output_height * output_width * depth * sizeof(T));
-
   T* output_ptr = &output_data[0];
   for (int b = 0; b < batches; ++b) {
     for (int y = 0; y < output_height; ++y) {
-- 
GitLab


From 315e009f917aebe04a754a046a57cb3ac1d10557 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E7=8E=8B=E6=8C=AF=E5=8D=8E=20=28WANG=20Zhenhua=29?=
 <i@jackwish.net>
Date: Sat, 22 Dec 2018 10:36:50 +0800
Subject: [PATCH 0076/2345] lite: perform std::floor before cast float to int

This won't change the resulted value, but to have same
code style in the context and reference_ops.
---
 tensorflow/lite/kernels/internal/optimized/optimized_ops.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
index a5374827e1..8f09fab4ca 100644
--- a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
+++ b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
@@ -5395,7 +5395,7 @@ inline void ResizeBilinearGenericSmallChannel(
       int32 y1 = std::min(y0 + 1, input_height - 1);
       for (int x = 0; x < output_width; ++x) {
         float input_x = x * width_scale;
-        int32 x0 = static_cast<int32>(input_x);
+        int32 x0 = static_cast<int32>(std::floor((input_x)));
         int32 x1 = std::min(x0 + 1, input_width - 1);
 
         int32 input_offset[4] = {Offset(input_shape, b, y0, x0, 0),
-- 
GitLab


From 20af3560823fcab4843c2bbabbf8414b96a1022c Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sat, 22 Dec 2018 21:49:31 +0000
Subject: [PATCH 0077/2345] Use rate instead of keep_prob calling dropout in
 keras

While running tf.keras I noticed the following warning:
```
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/layers/core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.
Instructions for updating:
Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.
```

This fix fixes the warning by replacing keep_prob with rate.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/keras/layers/core.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/layers/core.py b/tensorflow/python/keras/layers/core.py
index dfbab80be3..a5e1d3aab9 100644
--- a/tensorflow/python/keras/layers/core.py
+++ b/tensorflow/python/keras/layers/core.py
@@ -138,9 +138,10 @@ class Dropout(Layer):
       training = K.learning_phase()
 
     def dropped_inputs():
-      return nn.dropout(inputs, 1  - self.rate,
+      return nn.dropout(inputs,
                         noise_shape=self._get_noise_shape(inputs),
-                        seed=self.seed)
+                        seed=self.seed,
+                        rate = self.rate)
     output = tf_utils.smart_cond(training,
                                  dropped_inputs,
                                  lambda: array_ops.identity(inputs))
-- 
GitLab


From cb4b9c007b0e2f792ae921feda8f3f7956397d8e Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sat, 22 Dec 2018 21:52:40 +0000
Subject: [PATCH 0078/2345] Pylint fix of `(bad-whitespace)`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/keras/layers/core.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/layers/core.py b/tensorflow/python/keras/layers/core.py
index a5e1d3aab9..7c292c819d 100644
--- a/tensorflow/python/keras/layers/core.py
+++ b/tensorflow/python/keras/layers/core.py
@@ -141,7 +141,7 @@ class Dropout(Layer):
       return nn.dropout(inputs,
                         noise_shape=self._get_noise_shape(inputs),
                         seed=self.seed,
-                        rate = self.rate)
+                        rate=self.rate)
     output = tf_utils.smart_cond(training,
                                  dropped_inputs,
                                  lambda: array_ops.identity(inputs))
-- 
GitLab


From afae41cb416acff7a1a3ec41e2d91a95721e4f88 Mon Sep 17 00:00:00 2001
From: "Li, Guizi" <guizi.li@intel.com>
Date: Mon, 24 Dec 2018 09:48:10 +0800
Subject: [PATCH 0079/2345] [Intel MKL] Enable pad+fusedconv fusion

---
 tensorflow/core/graph/mkl_layout_pass.cc      | 182 ++++++++-
 tensorflow/core/graph/mkl_layout_pass_test.cc | 334 +++++++++++++++++
 tensorflow/core/kernels/mkl_conv_ops.cc       | 109 ++++--
 tensorflow/core/kernels/mkl_fused_ops_test.cc | 345 +++++++++++++++---
 tensorflow/core/ops/mkl_nn_ops.cc             |  57 +++
 5 files changed, 934 insertions(+), 93 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 9495132f4a..e69b1ad62e 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -280,8 +280,10 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
         "_MklConv2DBackpropFilterWithBias";
     csinfo_.mkl_fused_conv2d = "_MklFusedConv2D";
     csinfo_.mkl_pad_with_conv2d = "_MklPadWithConv2D";
+    csinfo_.mkl_pad_with_fused_conv2d = "_MklPadWithFusedConv2D";
     csinfo_.pad = "Pad";
     csinfo_.pad_with_conv2d = "__MklDummyPadWithConv2D";
+    csinfo_.pad_with_fused_conv2d = "__MklDummyPadWithFusedConv2D";
 // Temporarily don't convert quantized operators into MKL versions for now.
 // TODO(Intel-tf) Once all the relevant PRs have been merged then remove
 // the ifdef.
@@ -423,6 +425,9 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
                       CopyAttrsDataType, AlwaysRewrite});
     rinfo_.push_back({csinfo_.pad_with_conv2d, csinfo_.mkl_pad_with_conv2d,
                       CopyAttrsPadWithConv2D, AlwaysRewrite});
+    rinfo_.push_back({csinfo_.pad_with_fused_conv2d,
+                      csinfo_.mkl_pad_with_fused_conv2d,
+                      CopyAttrsPadWithFusedConv2D, AlwaysRewrite});
 #ifdef INTEL_MKL_QUANTIZED
     rinfo_.push_back({csinfo_.quantized_avg_pool,
                       mkl_op_registry::GetMklOpName(csinfo_.quantized_avg_pool),
@@ -538,6 +543,9 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     // Merge Pad and Conv2d, only if the pad op is "Pad"
     // Doesn't merge if pad op is "PadV2" or "MirrorPad"
 
+    minfo_.push_back({csinfo_.pad, csinfo_.fused_conv2d,
+                      csinfo_.pad_with_fused_conv2d, GetPadOrFusedConv2D});
+
     // The fusion patterns in "finfo_" that show up first will get applied
     // first, for example, graph "A->B->C-D" and finfo_ is {A->B->C to ABC,
     // A->B->C->D to ABCD}, since the first gets applied first, the final
@@ -701,9 +709,11 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     string mkl_conv2d_with_bias;
     string mkl_fused_conv2d;
     string mkl_pad_with_conv2d;
+    string mkl_pad_with_fused_conv2d;
     string mul;
     string pad;
     string pad_with_conv2d;
+    string pad_with_fused_conv2d;
     string quantized_avg_pool;
     string quantized_conv2d;
     string quantized_conv2d_with_requantize;
@@ -917,6 +927,59 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
 
     return n;
   }
+
+  // Find Pad or _FusedConv2D node that can be merged with input node 'm'.
+  // If input 'm' is Pad, then check if there exists _FusedConv2D node that can
+  // be  merged with 'm'. If input 'm' is _FusedConv2D, then check if there
+  // exists Pad  node that can be merged with 'm'.
+  static Node* GetPadOrFusedConv2D(const Node* m) {
+    DCHECK(m);
+    Node* n = nullptr;
+
+    const Node* conv_node;
+    if (m->type_string() == csinfo_.pad) {
+      // If m is Pad, then _FusedConv2D is the output of Pad.
+      for (const Edge* e : m->out_edges()) {
+        if (!e->IsControlEdge() &&
+            e->dst()->type_string() == csinfo_.fused_conv2d) {
+          n = e->dst();
+          conv_node = n;
+          break;
+        }
+      }
+    } else {
+      DCHECK_EQ(m->type_string(), csinfo_.fused_conv2d);
+      // If m is _FusedConv2D, Go over all input edges
+      // and search for Pad  Node.
+      for (const Edge* e : m->in_edges()) {
+        if (!e->IsControlEdge() && e->src()->type_string() == csinfo_.pad) {
+          n = e->src();
+          conv_node = m;
+          break;
+        }
+      }
+    }
+    // Check if only VALID type of padding is used
+    // or not.
+    if (n != nullptr) {
+      string padding;
+      TF_CHECK_OK(GetNodeAttr(conv_node->def(), "padding", &padding));
+      if (padding != "VALID") {
+        // Then do not merge.
+        n = nullptr;
+        VLOG(1) << "MklLayoutRewritePass: Could match Pad and _FusedConv2D "
+                << "node for merging. Only VALID type of padding in conv op "
+                << "can be merged with Pad op Input node: " << m->DebugString();
+      }
+    } else {
+      VLOG(1) << "MklLayoutRewritePass: Could not find matching "
+              << "Pad and _FusedConv2D node for merging. Input node: "
+              << m->DebugString();
+    }
+
+    return n;
+  }
+
   // Find Conv2DBackpropFilter or BiasAddGrad node that can be merged with input
   // node 'm'. If input 'm' is Conv2DBackpropFilter, then check if there exists
   // BiasAddGrad node that can be merged with 'm'. If input 'm' is BiasAddGrad,
@@ -1407,9 +1470,16 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
                            bool change_format = false);
   static void CopyAttrsPadWithConv2D(const Node* orig_node, NodeBuilder* nb,
                                      bool change_format = false);
+  static void CopyAttrsPadWithFusedConv2D(const Node* orig_node,
+                                          NodeBuilder* nb,
+                                          bool change_format = false);
   static void CopyAttrsFromPadAndConv2D(const Node* orig_node1,
                                         const Node* orig_node2, NodeBuilder* nb,
                                         bool change_format = false);
+  static void CopyAttrsFromPadAndFusedConv2D(const Node* orig_node1,
+                                             const Node* orig_node2,
+                                             NodeBuilder* nb,
+                                             bool change_format = false);
   static void CopyAttrsPooling(const Node* orig_node, NodeBuilder* nb,
                                bool change_format = false);
   static void CopyAttrsQuantizedPooling(const Node* orig_node, NodeBuilder* nb,
@@ -1627,6 +1697,7 @@ int MklLayoutRewritePass::SetUpContiguousInputs(
     for (const Edge* e : filter_node->out_edges()) {
       if ((e->dst()->type_string() == csinfo_.mkl_conv2d ||
            e->dst()->type_string() == csinfo_.mkl_pad_with_conv2d ||
+           e->dst()->type_string() == csinfo_.mkl_pad_with_fused_conv2d ||
            e->dst()->type_string() == csinfo_.mkl_conv2d_with_bias ||
            e->dst()->type_string() == csinfo_.mkl_fused_conv2d) &&
           e->dst_input() == kConv2DFilterInputSlotIdx
@@ -2015,6 +2086,26 @@ void MklLayoutRewritePass::CopyAttrsPadWithConv2D(const Node* orig_node,
   nb->Attr("Tpaddings", Tpaddings);
 }
 
+void MklLayoutRewritePass::CopyAttrsPadWithFusedConv2D(const Node* orig_node,
+                                                       NodeBuilder* nb,
+                                                       bool change_format) {
+  DataType Tpaddings;
+  bool is_filter_const;
+
+  CopyAttrsFusedConv2D(orig_node, nb, change_format);
+
+  // Get attributes from old node.
+  TF_CHECK_OK(GetNodeAttr(orig_node->def(), "Tpaddings", &Tpaddings));
+  // Check if filter is a constant.
+  Node* filter_node = nullptr;
+  orig_node->input_node(1, &filter_node);
+  is_filter_const = filter_node->IsConstant();
+
+  // Add attributes to new node.
+  nb->Attr("Tpaddings", Tpaddings);
+  nb->Attr("is_filter_const", is_filter_const);
+}
+
 // Used with MergePadWithConv2D
 void MklLayoutRewritePass::CopyAttrsFromPadAndConv2D(const Node* orig_node1,
                                                      const Node* orig_node2,
@@ -2049,6 +2140,42 @@ void MklLayoutRewritePass::CopyAttrsFromPadAndConv2D(const Node* orig_node1,
   nb->Attr("Tpaddings", Tpaddings);
 }
 
+void MklLayoutRewritePass::CopyAttrsFromPadAndFusedConv2D(
+    const Node* orig_node1, const Node* orig_node2, NodeBuilder* nb,
+    bool change_format) {
+  DataType T;
+  int num_args;
+  string data_format;
+  string padding;
+  std::vector<int32> strides;
+  std::vector<int32> dilations;
+  float epsilon;
+  std::vector<string> fused_ops;
+  DataType Tpaddings;
+
+  // Get all attributes from old node.
+  TF_CHECK_OK(GetNodeAttr(orig_node1->def(), "T", &T));
+  TF_CHECK_OK(GetNodeAttr(orig_node1->def(), "num_args", &num_args));
+  TF_CHECK_OK(GetNodeAttr(orig_node1->def(), "strides", &strides));
+  TF_CHECK_OK(GetNodeAttr(orig_node1->def(), "padding", &padding));
+  TF_CHECK_OK(GetNodeAttr(orig_node1->def(), "data_format", &data_format));
+  TF_CHECK_OK(GetNodeAttr(orig_node1->def(), "dilations", &dilations));
+  TF_CHECK_OK(GetNodeAttr(orig_node1->def(), "fused_ops", &fused_ops));
+  TF_CHECK_OK(GetNodeAttr(orig_node1->def(), "epsilon", &epsilon));
+  TF_CHECK_OK(GetNodeAttr(orig_node2->def(), "Tpaddings", &Tpaddings));
+
+  // Add attributes to new node.
+  nb->Attr("T", T);
+  nb->Attr("num_args", num_args);
+  nb->Attr("strides", strides);
+  nb->Attr("padding", padding);
+  nb->Attr("data_format", data_format);
+  nb->Attr("dilations", dilations);
+  nb->Attr("epsilon", epsilon);
+  nb->Attr("Tpaddings", Tpaddings);
+  nb->Attr("fused_ops", fused_ops);
+}
+
 void MklLayoutRewritePass::CopyAttrsConv2DDepthwise(const Node* orig_node,
                                                     NodeBuilder* nb,
                                                     bool change_format) {
@@ -2575,10 +2702,14 @@ Status MklLayoutRewritePass::MergeConv2DWithBiasAdd(std::unique_ptr<Graph>* g,
 Status MklLayoutRewritePass::MergePadWithConv2D(std::unique_ptr<Graph>* g,
                                                 Node* m, Node* n) {
   DCHECK(((m->type_string() == csinfo_.pad &&
-           n->type_string() == csinfo_.conv2d)) ||
+           (n->type_string() == csinfo_.conv2d ||
+            n->type_string() == csinfo_.fused_conv2d))) ||
          ((n->type_string() == csinfo_.pad &&
-           m->type_string() == csinfo_.conv2d)));
+           (m->type_string() == csinfo_.conv2d ||
+            m->type_string() == csinfo_.fused_conv2d))));
 
+  bool is_fused_conv2d = n->type_string() == csinfo_.fused_conv2d ||
+                         m->type_string() == csinfo_.fused_conv2d;
   // Conv2D is successor node, and Pad predecessor node.
   Node* pred = m->type_string() == csinfo_.pad ? m : n;
   Node* succ = m->type_string() == csinfo_.pad ? n : m;
@@ -2589,7 +2720,6 @@ Status MklLayoutRewritePass::MergePadWithConv2D(std::unique_ptr<Graph>* g,
   std::vector<int32> strides;
   std::vector<int32> dilations;
   string data_format_pred, data_format_succ;
-  bool use_cudnn_on_gnu;
   TF_CHECK_OK(GetNodeAttr(pred->def(), "T", &T_pred));
   TF_CHECK_OK(GetNodeAttr(succ->def(), "T", &T_succ));
   TF_CHECK_OK(GetNodeAttr(succ->def(), "padding", &padding));
@@ -2598,7 +2728,6 @@ Status MklLayoutRewritePass::MergePadWithConv2D(std::unique_ptr<Graph>* g,
   // Data format for pad is not available and not necessary, thus
   // dont need to match data format for Pad
   TF_CHECK_OK(GetNodeAttr(succ->def(), "data_format", &data_format_succ));
-  TF_CHECK_OK(GetNodeAttr(succ->def(), "use_cudnn_on_gpu", &use_cudnn_on_gnu));
   // Check if the data types and devices of both succ and pred are the same.
   // Assert is not used,  because it can be too strict.
   // Don't need to check for data formats because it is not available in Pad.
@@ -2644,29 +2773,54 @@ Status MklLayoutRewritePass::MergePadWithConv2D(std::unique_ptr<Graph>* g,
   }
   DCHECK_EQ(PadDataInputEdges, 2);
 
-  // Conv2D must have 2 data inputs: pad output and Filter
+  // Conv2D must have 2 data inputs: Pad output and Filter
+  // FusedConv2D have 3 data inputs: Pad output, Filter and Args;
   int ConvDataInputEdges = 0;
   for (const Edge* e : succ->in_edges()) {
     if (!e->IsControlEdge()) {
       ConvDataInputEdges++;
     }
   }
-  DCHECK_EQ(ConvDataInputEdges, 2);
+
+  if (is_fused_conv2d) {
+    DCHECK_EQ(ConvDataInputEdges, 3);
+  } else {
+    DCHECK_EQ(ConvDataInputEdges, 2);
+  }
 
   // We will use the node name of Conv2D as the name of new node
   // Build new node. We use same name as original node, but change the op
   // name.
-  NodeBuilder nb(succ->name(), csinfo_.pad_with_conv2d);
+
+  auto fused_op_name = csinfo_.pad_with_conv2d;
+  if (is_fused_conv2d) {
+    fused_op_name = csinfo_.pad_with_fused_conv2d;
+  }
+  NodeBuilder nb(succ->name(), fused_op_name);
   nb.Input(pred_in[0].first, pred_in[0].second);  // In1 (input data)  of Pad
   // pred_in[1] will be 2nd Tensorflow tensor for Conv2D.
   nb.Input(succ_in[1].first, succ_in[1].second);  // In2 (filter) of conv2d
   // In1 of Conv2D is same as output of Pad.
   // Thus, only need to add In2 of Conv2D
+
+  // FusedConv2D has one additional input, args
+  if (is_fused_conv2d) {
+    std::vector<NodeBuilder::NodeOut> args;
+    args.emplace_back(succ_in[2].first, succ_in[2].second);
+    nb.Input(gtl::ArraySlice<NodeBuilder::NodeOut>{
+        args});  // In3 (args) of FusedConv2D
+  }
   nb.Input(pred_in[1].first, pred_in[1].second);  // In2 (paddings) of Pad
 
-  // Copy attributes from Pad and conv2D to PadWithConv2D.
-  CopyAttrsFromPadAndConv2D(const_cast<const Node*>(succ),
-                            const_cast<const Node*>(pred), &nb);
+  if (is_fused_conv2d) {
+    // Copy attributes from Pad and FusedConv2D to PadWithFusedConv2D.
+    CopyAttrsFromPadAndFusedConv2D(const_cast<const Node*>(succ),
+                                   const_cast<const Node*>(pred), &nb);
+  } else {
+    // Copy attributes from Pad and conv2D to PadWithConv2D.
+    CopyAttrsFromPadAndConv2D(const_cast<const Node*>(succ),
+                              const_cast<const Node*>(pred), &nb);
+  }
 
   // Copy the device assigned to old node to new node.
   nb.Device(succ->def().device());
@@ -2871,6 +3025,13 @@ Status MklLayoutRewritePass::MergeNode(std::unique_ptr<Graph>* g, Node* m,
     return this->MergePadWithConv2D(g, m, n);
   }
 
+  if (((m->type_string() == csinfo_.pad &&
+        n->type_string() == csinfo_.fused_conv2d && FusedConv2DRewrite(n))) ||
+      ((n->type_string() == csinfo_.pad &&
+        m->type_string() == csinfo_.fused_conv2d && FusedConv2DRewrite(m)))) {
+    return this->MergePadWithConv2D(g, m, n);
+  }
+
   if (((m->type_string() == csinfo_.bias_add_grad &&
         n->type_string() == csinfo_.conv2d_grad_filter)) ||
       ((n->type_string() == csinfo_.bias_add_grad &&
@@ -3021,6 +3182,7 @@ MklLayoutRewritePass::CheckForNodeRewrite(const Node* n) const {
   // names do not match Mkl node names.
   if (n->type_string() != csinfo_.conv2d_with_bias &&
       n->type_string() != csinfo_.pad_with_conv2d &&
+      n->type_string() != csinfo_.pad_with_fused_conv2d &&
       n->type_string() != csinfo_.conv2d_grad_filter_with_bias &&
       n->type_string() != csinfo_.fused_conv2d &&
       !mkl_op_registry::IsMklOp(mkl_op_registry::GetMklOpName(n->type_string()),
diff --git a/tensorflow/core/graph/mkl_layout_pass_test.cc b/tensorflow/core/graph/mkl_layout_pass_test.cc
index 6e73ed1b9f..5544e2bc36 100644
--- a/tensorflow/core/graph/mkl_layout_pass_test.cc
+++ b/tensorflow/core/graph/mkl_layout_pass_test.cc
@@ -1243,6 +1243,340 @@ TEST_F(MklLayoutPassTest, NodeRewrite_FusedConv2D_Negative2) {
             "D(_FusedConv2D);E(Zeta)|A->D;B->D:1;C->D:2;C->E:1;D->E");
 }
 
+// Merge test for PadWithFusedConv2D Op with BiasAdd fusion
+// padding is VALID type
+// A = input(image), B = input(paddings), C= Pad = input of conv2D,
+// D=input(filter), E = input(bias), F = _FusedConv2D
+// G = Zeta C=Pad(A,B); F=_FusedConv2D(C,D,E); G=Zeta(F,E)
+// After layout pass
+// _MklPadWithFusedConv2D(A, D, E, B, DMT/_0, DMT/_1, DMT/_2, DMT/_3)
+TEST_F(MklLayoutPassTest, NodeMerge_PadWithFusedConv2D_Positive1) {
+  InitGraph(
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Int32Input'}"
+      "node { name: 'C' op: 'Pad'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'Tpaddings'        value { type: DT_INT32 } }"
+      " input: ['A', 'B']}"
+      "node { name: 'D' op: 'Input'}"
+      "node { name: 'E' op: 'Input'}"
+      "node { name: 'F' op: '_FusedConv2D'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'num_args'         value { i: 1 } }"
+      " attr { key: 'data_format'      value { s: 'NCHW' } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'padding'          value { s: 'VALID' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'fused_ops'        value { list: {s: 'BiasAdd'} } }"
+      " attr { key: 'epsilon'          value { f: 0.001 }}"
+      " input: ['C', 'D', 'E']}"
+      "node { name: 'G' op: 'Zeta' attr { key: 'T' value { type: DT_FLOAT } }"
+      " input: ['F', 'E'] }");
+  EXPECT_EQ(
+      DoMklLayoutOptimizationPass(),
+      "A(Input);B(Int32Input);D(Input);DMT/_0(Const);DMT/_1(Const);DMT/"
+      "_2(Const);DMT/_3(Const);E(Input);F(_MklPadWithFusedConv2D);"
+      "G(Zeta)|A->F;A:control->DMT/_0:control;A:control->DMT/_1:control;"
+      "A:control->DMT/_2:control;A:control->DMT/_3:control;B->F:3;D->F:1;DMT/"
+      "_0->F:4;DMT/_1->F:5;DMT/_2->F:6;DMT/_3->F:7;E->F:2;E->G:1;F->G");
+}
+
+// Merge test for PadWithFusedConv2D Op with BiasAdd+Relu fusion
+// padding is VALID type
+// A = input(image), B = input(paddings), C= Pad = input of conv2D,
+// D=input(filter), E = input(bias), F = _FusedConv2D(With relu)
+// G = Zeta C=Pad(A,B); F=_FusedConv2D(C,D,E); G=Zeta(F,E)
+// After layout pass
+// _MklPadWithFusedConv2D(A, D, E, B, DMT/_0, DMT/_1, DMT/_2, DMT/_3)
+TEST_F(MklLayoutPassTest, NodeMerge_PadWithFusedConv2D_Positive2) {
+  InitGraph(
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Int32Input'}"
+      "node { name: 'C' op: 'Pad'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'Tpaddings'        value { type: DT_INT32 } }"
+      " input: ['A', 'B']}"
+      "node { name: 'D' op: 'Input'}"
+      "node { name: 'E' op: 'Input'}"
+      "node { name: 'F' op: '_FusedConv2D'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'num_args'         value { i: 1 } }"
+      " attr { key: 'data_format'      value { s: 'NCHW' } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'padding'          value { s: 'VALID' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'fused_ops'"
+      "             value { list: {s: 'BiasAdd', s: 'Relu'} } }"
+      " attr { key: 'epsilon'          value { f: 0.001 }}"
+      " input: ['C', 'D', 'E']}"
+      "node { name: 'G' op: 'Zeta' attr { key: 'T' value { type: DT_FLOAT } }"
+      " input: ['F', 'E'] }");
+  EXPECT_EQ(DoMklLayoutOptimizationPass(),
+            "A(Input);B(Int32Input);D(Input);DMT/_0(Const);DMT/_1(Const);DMT/"
+            "_2(Const);DMT/_3(Const);E(Input);F(_MklPadWithFusedConv2D);"
+            "G(Zeta)|A->F;A:control->DMT/_0:control;A:control->DMT/_1:control;"
+            "A:control->DMT/_2:control;A:control->DMT/_3:control;B->F:3;"
+            "D->F:1;DMT/_0->F:4;DMT/_1->F:5;DMT/_2->F:6;DMT/"
+            "_3->F:7;E->F:2;E->G:1;F->G");
+}
+
+// Merge test for PadWithFusedConv2D Op with unsupported fusion
+// padding is VALID type
+// A = input(image), B = input(paddings), C= Pad = input of conv2D,
+// D=input(filter), E = input(bias), F = _FusedConv2D(With Unsupported)
+// G = Zeta C=Pad(A,B); F=_FusedConv2D(C,D,E); G=Zeta(F,E)
+// After layout pass - No merging
+TEST_F(MklLayoutPassTest, NodeMerge_PadWithFusedConv2D_Negative1) {
+  InitGraph(
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Int32Input'}"
+      "node { name: 'C' op: 'Pad'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'Tpaddings'        value { type: DT_INT32 } }"
+      " input: ['A', 'B']}"
+      "node { name: 'D' op: 'Input'}"
+      "node { name: 'E' op: 'Input'}"
+      "node { name: 'F' op: '_FusedConv2D'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'num_args'         value { i: 1 } }"
+      " attr { key: 'data_format'      value { s: 'NCHW' } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'padding'          value { s: 'VALID' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'fused_ops'        value { list: {s: 'Unsupported'} } }"
+      " attr { key: 'epsilon'          value { f: 0.001 }}"
+      " input: ['C', 'D', 'E']}"
+      "node { name: 'G' op: 'Zeta' attr { key: 'T' value { type: DT_FLOAT } }"
+      " input: ['F', 'E'] }");
+  EXPECT_EQ(DoMklLayoutOptimizationPass(),
+            "A(Input);B(Int32Input);C(Pad);D(Input);E(Input);F(_FusedConv2D);G("
+            "Zeta)|A->C;B->C:1;C->F;D->F:1;E->F:2;E->G:1;F->G");
+}
+
+// Merge test for PadWithFusedConv2D Op with BiasAdd fusion
+// padding is SAME type
+// A = input(image), B = input(paddings), C= Pad = input of conv2D,
+// D=input(filter), E = input(bias), F = _FusedConv2D
+// G = Zeta C=Pad(A,B); F=_FusedConv2D(C,D,E); G=Zeta(F,E)
+// After layout pass - No merging
+TEST_F(MklLayoutPassTest, NodeMerge_PadWithFusedConv2D_Negative2) {
+  InitGraph(
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Int32Input'}"
+      "node { name: 'C' op: 'Pad'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'Tpaddings'        value { type: DT_INT32 } }"
+      " input: ['A', 'B']}"
+      "node { name: 'D' op: 'Input'}"
+      "node { name: 'E' op: 'Input'}"
+      "node { name: 'F' op: '_FusedConv2D'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'num_args'         value { i: 1 } }"
+      " attr { key: 'data_format'      value { s: 'NCHW' } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'padding'          value { s: 'SAME' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'fused_ops'        value { list: {s: 'BiasAdd'} } }"
+      " attr { key: 'epsilon'          value { f: 0.001 }}"
+      " input: ['C', 'D', 'E']}"
+      "node { name: 'G' op: 'Zeta' attr { key: 'T' value { type: DT_FLOAT } }"
+      " input: ['F', 'E'] }");
+  EXPECT_EQ(DoMklLayoutOptimizationPass(),
+            "A(Input);B(Int32Input);C(Pad);D(Input);DMT/_0(Const);DMT/"
+            "_1(Const);DMT/_2(Const);E(Input);F(_MklFusedConv2D);G(Zeta)|A->C;"
+            "B->C:1;C->F;C:control->DMT/_0:control;C:control->DMT/_1:control;"
+            "C:control->DMT/_2:control;D->F:1;DMT/_0->F:3;DMT/_1->F:4;DMT/"
+            "_2->F:5;E->F:2;E->G:1;F->G");
+}
+
+// Merge test for PadWithFusedConv2D Op with BiasAdd+Relu fusion
+// padding is SAME type
+// A = input(image), B = input(paddings), C= Pad = input of conv2D,
+// D=input(filter), E = input(bias), F = _FusedConv2D(With relu)
+// G = Zeta C=Pad(A,B); F=_FusedConv2D(C,D,E); G=Zeta(F,E)
+// After layout pass - No merging
+TEST_F(MklLayoutPassTest, NodeMerge_PadWithFusedConv2D_Negative3) {
+  InitGraph(
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Int32Input'}"
+      "node { name: 'C' op: 'Pad'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'Tpaddings'        value { type: DT_INT32 } }"
+      " input: ['A', 'B']}"
+      "node { name: 'D' op: 'Input'}"
+      "node { name: 'E' op: 'Input'}"
+      "node { name: 'F' op: '_FusedConv2D'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'num_args'         value { i: 1 } }"
+      " attr { key: 'data_format'      value { s: 'NCHW' } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'padding'          value { s: 'SAME' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'fused_ops'"
+      "             value { list: {s: 'BiasAdd', s: 'Relu'} } }"
+      " attr { key: 'epsilon'          value { f: 0.001 }}"
+      " input: ['C', 'D', 'E']}"
+      "node { name: 'G' op: 'Zeta' attr { key: 'T' value { type: DT_FLOAT } }"
+      " input: ['F', 'E'] }");
+  EXPECT_EQ(DoMklLayoutOptimizationPass(),
+            "A(Input);B(Int32Input);C(Pad);D(Input);DMT/_0(Const);DMT/"
+            "_1(Const);DMT/_2(Const);E(Input);F(_MklFusedConv2D);G(Zeta)|A->C;"
+            "B->C:1;C->F;C:control->DMT/_0:control;C:control->DMT/_1:control;"
+            "C:control->DMT/_2:control;D->F:1;DMT/_0->F:3;DMT/_1->F:4;DMT/"
+            "_2->F:5;E->F:2;E->G:1;F->G");
+}
+
+// Test if input control edges do not duplicate after merge.
+// If both the merging ops have input control edge from a common op
+// then, the merged op will have only one control edge from that
+// common op.
+// padding is VALID type
+// A = input(image), A1 = input, B = input(paddings),
+// C= Pad = input of conv2D,
+// D=input(filter), F=input(bias), E = _FusedConv2D, Z = Zeta
+// C=Pad(A,B); E=_FusedConv2D(C,D,F); Z=Zeta(E,Y)
+// A1:control->C:control
+// A1:control->E:control
+// After layout pass:
+// _MklPadWithFusedConv2D(A, D, B, F, DMT/_0, DMT/_1, DMT/_2, DMT/_3)
+// A1:control->E:control (only one control edge)
+TEST_F(MklLayoutPassTest, Input_ControlEdge_PadWithFusedConv2D_Positive) {
+  DCHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
+  InitGraph(
+      "node { name: 'A1' op: 'Input'}"
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Int32Input'}"
+      "node { name: 'C' op: 'Pad'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'Tpaddings'        value { type: DT_INT32 } }"
+      " input: ['A', 'B']}"
+      "node { name: 'D' op: 'Input'}"
+      "node { name: 'F' op: 'Input'}"
+      "node { name: 'E' op: '_FusedConv2D'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'num_args'         value { i: 1 } }"
+      " attr { key: 'data_format'      value { s: 'NCHW' } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'padding'          value { s: 'VALID' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'fused_ops'        value { list: {s: 'BiasAdd'} } }"
+      " attr { key: 'epsilon'          value { f: 0.001 }}"
+      " input: ['C', 'D', 'F']}"
+      "node { name: 'Y' op: 'Input'}"
+      "node { name: 'Z' op: 'Zeta'"
+      " attr {key: 'T'                 value { type: DT_FLOAT } }"
+      " input: ['E', 'Y']}");
+  Node* a1 = FindNode("A1");
+  Node* c = FindNode("C");
+  Node* e = FindNode("E");
+  const Edge* edge = graph_.AddControlEdge(a1, c);
+  const Edge* edge_1 = graph_.AddControlEdge(a1, e);
+  ASSERT_NE(edge, nullptr);
+  ASSERT_NE(edge_1, nullptr);
+  EXPECT_EQ(DoMklLayoutOptimizationPass(),
+            "A(Input);A1(Input);B(Int32Input);D(Input);DMT/_0(Const);DMT/"
+            "_1(Const);DMT/_2(Const);DMT/"
+            "_3(Const);E(_MklPadWithFusedConv2D);F(Input);Y(Input);Z(Zeta)|A->"
+            "E;A1:control->E:control;A:control->DMT/_0:control;A:control->DMT/"
+            "_1:control;A:control->DMT/_2:control;A:control->DMT/"
+            "_3:control;B->E:3;D->E:1;DMT/_0->E:4;DMT/_1->E:5;DMT/_2->E:6;DMT/"
+            "_3->E:7;E->Z;F->E:2;Y->Z:1");
+}
+
+// Test if output control edges does not duplicate after merge.
+// If both the merging ops have output control edge to a common op,
+// then after merge, the merged op will have only one control edge
+// to that commom op.
+// padding is VALID type
+// A = input(image), B = input(paddings), C= Pad = input of conv2D,
+// D=input(filter), F=input(bias), E = _FusedConv2D, Z = Zeta
+// C=Pad(A,B); E=_FusedConv2D(C,D,F); Z=Zeta(E,Y)
+// C:control->A1:control
+// E:control->A1:control
+// After layout pass:
+// _MklPadWithFusedConv2D(A, D, B, F, DMT/_0, DMT/_1, DMT/_2, DMT/_2)
+// E:control->A1:control (only one control edge)
+TEST_F(MklLayoutPassTest, Output_ControlEdge_PadWithFusedConv2D_Positive) {
+  DCHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
+  InitGraph(
+      "node { name: 'A1' op: 'Input'}"
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Int32Input'}"
+      "node { name: 'C' op: 'Pad'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'Tpaddings'        value { type: DT_INT32 } }"
+      " input: ['A', 'B']}"
+      "node { name: 'D' op: 'Input'}"
+      "node { name: 'F' op: 'Input'}"
+      "node { name: 'E' op: '_FusedConv2D'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'num_args'         value { i: 1 } }"
+      " attr { key: 'data_format'      value { s: 'NCHW' } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'padding'          value { s: 'VALID' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'fused_ops'        value { list: {s: 'BiasAdd'} } }"
+      " attr { key: 'epsilon'          value { f: 0.001 }}"
+      " input: ['C', 'D', 'F']}"
+      "node { name: 'Y' op: 'Input'}"
+      "node { name: 'Z' op: 'Zeta'"
+      " attr {key: 'T'                 value { type: DT_FLOAT } }"
+      " input: ['E', 'Y']}");
+  Node* a1 = FindNode("A1");
+  Node* c = FindNode("C");
+  Node* e = FindNode("E");
+  const Edge* edge = graph_.AddControlEdge(c, a1);
+  const Edge* edge_1 = graph_.AddControlEdge(e, a1);
+  ASSERT_NE(edge, nullptr);
+  ASSERT_NE(edge_1, nullptr);
+  EXPECT_EQ(DoMklLayoutOptimizationPass(),
+            "A(Input);A1(Input);B(Int32Input);D(Input);DMT/_0(Const);DMT/"
+            "_1(Const);DMT/_2(Const);DMT/"
+            "_3(Const);E(_MklPadWithFusedConv2D);F(Input);Y(Input);Z(Zeta)|A->"
+            "E;A:control->DMT/_0:control;A:control->DMT/"
+            "_1:control;A:control->DMT/_2:control;A:control->DMT/"
+            "_3:control;B->E:3;D->E:1;DMT/_0->E:4;DMT/_1->E:5;DMT/_2->E:6;DMT/"
+            "_3->E:7;E->Z;E:control->A1:control;F->E:2;Y->Z:1");
+}
+
+// Pad + _FusedConv2D with padding is VALID,
+// Input node pointing to both Pad and _FusedConv2D
+// Output of both Pad and _FusedConv2D feeds one node (Z as Output2)
+// A = input(as image), B = input(as paddings), C= Pad
+// F = input(as bias), E = _FusedConv2D, Z = Output2
+// C=Pad(A,B); E=_FusedConv2D(C,A,F); Z=Output(C,E)
+// After layout pass - No merging, since Pad and _FusedConv2D both
+// feed to the same node (Z)
+TEST_F(MklLayoutPassTest, NodeMerge_PadWithFusedConv2D_Common_InOutput) {
+  DCHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
+  InitGraph(
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Int32Input'}"
+      "node { name: 'C' op: 'Pad'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'Tpaddings'        value { type: DT_INT32 } }"
+      " input: ['A', 'B']}"
+      "node { name: 'F' op: 'Input'}"
+      "node { name: 'E' op: '_FusedConv2D'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'num_args'         value { i: 1 } }"
+      " attr { key: 'data_format'      value { s: 'NCHW' } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'padding'          value { s: 'VALID' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'fused_ops'        value { list: {s: 'BiasAdd'} } }"
+      " attr { key: 'epsilon'          value { f: 0.001 }}"
+      " input: ['C', 'A', 'F']}"
+      "node { name: 'Z' op: 'Output2'"
+      " input: ['C', 'E']}");
+  EXPECT_EQ(DoMklLayoutOptimizationPass(),
+            "A(Input);B(Int32Input);C(Pad);DMT/_0(Const);DMT/_1(Const);DMT/"
+            "_2(Const);E(_MklFusedConv2D);F(Input);Z(Output2)|A->C;A->E:1;B->C:"
+            "1;C->E;C->Z;C:control->DMT/_0:control;C:control->DMT/"
+            "_1:control;C:control->DMT/_2:control;DMT/_0->E:3;DMT/_1->E:4;DMT/"
+            "_2->E:5;E->Z:1;F->E:2");
+}
+
 TEST_F(MklLayoutPassTest, NodeRewrite_Conv2DGradFilter_Positive) {
   InitGraph(
       "node { name: 'A' op: 'Input'}"
diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc
index 8c585fb48c..ead2f1e09e 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_ops.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include <map>
 #include <vector>
 
+#include "absl/strings/str_join.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
@@ -465,19 +466,18 @@ class MklConvOp : public OpKernel {
                                         filter.shape().DebugString()));
 
     for (int i = 0; i < 3; i++) {
-      OP_REQUIRES(
-          context,
-          FastBoundsCheck(filter.dim_size(i), std::numeric_limits<int>::max()),
-          errors::InvalidArgument("filter too large"));
+      OP_REQUIRES(context, FastBoundsCheck(filter.dim_size(i),
+                                           std::numeric_limits<int>::max()),
+                  errors::InvalidArgument("filter too large"));
     }
 
     const int64 input_depth =
         input_in_mkl_format ? GetMklTensorDim(mkl_context.input_shape, 'C')
                             : GetTensorDim(input, data_format_, 'C');
-    OP_REQUIRES(context, input_depth == filter.dim_size(2),
-                errors::InvalidArgument(
-                    "input and filter must have the same depth: ", input_depth,
-                    " vs ", filter.dim_size(2)));
+    OP_REQUIRES(
+        context, input_depth == filter.dim_size(2),
+        errors::InvalidArgument("input and filter must have the same depth: ",
+                                input_depth, " vs ", filter.dim_size(2)));
     // The last dimension for filter is out_depth.
     const int out_depth = static_cast<int>(filter.dim_size(3));
 
@@ -486,10 +486,9 @@ class MklConvOp : public OpKernel {
     const int64 input_rows_raw =
         input_in_mkl_format ? GetMklTensorDim(mkl_context.input_shape, 'H')
                             : GetTensorDim(input, data_format_, 'H');
-    OP_REQUIRES(
-        context,
-        FastBoundsCheck(input_rows_raw, std::numeric_limits<int>::max()),
-        errors::InvalidArgument("Input rows too large"));
+    OP_REQUIRES(context, FastBoundsCheck(input_rows_raw,
+                                         std::numeric_limits<int>::max()),
+                errors::InvalidArgument("Input rows too large"));
     const int input_rows = static_cast<int>(input_rows_raw);
     const int filter_rows = static_cast<int>(filter.dim_size(0));
 
@@ -498,10 +497,9 @@ class MklConvOp : public OpKernel {
     const int64 input_cols_raw =
         input_in_mkl_format ? GetMklTensorDim(mkl_context.input_shape, 'W')
                             : GetTensorDim(input, data_format_, 'W');
-    OP_REQUIRES(
-        context,
-        FastBoundsCheck(input_cols_raw, std::numeric_limits<int>::max()),
-        errors::InvalidArgument("Input cols too large"));
+    OP_REQUIRES(context, FastBoundsCheck(input_cols_raw,
+                                         std::numeric_limits<int>::max()),
+                errors::InvalidArgument("Input cols too large"));
     const int input_cols = static_cast<int>(input_cols_raw);
     const int filter_cols = static_cast<int>(filter.dim_size(1));
 
@@ -509,10 +507,9 @@ class MklConvOp : public OpKernel {
     const int64 input_batch_raw =
         input_in_mkl_format ? GetMklTensorDim(mkl_context.input_shape, 'N')
                             : GetTensorDim(input, data_format_, 'N');
-    OP_REQUIRES(
-        context,
-        FastBoundsCheck(input_batch_raw, std::numeric_limits<int>::max()),
-        errors::InvalidArgument("batch is too large"));
+    OP_REQUIRES(context, FastBoundsCheck(input_batch_raw,
+                                         std::numeric_limits<int>::max()),
+                errors::InvalidArgument("batch is too large"));
     const int batch = static_cast<int>(input_batch_raw);
 
     // For now we take the stride from the second and third dimensions only (we
@@ -894,17 +891,15 @@ class MklConvOp : public OpKernel {
       OP_REQUIRES(context, dilations_.size() == 5,
                   errors::InvalidArgument("Dilation rates field must "
                                           "specify 5 dimensions"));
-      OP_REQUIRES(context,
-                  (GetTensorDim(dilations_, data_format_, 'N') == 1 &&
-                   GetTensorDim(dilations_, data_format_, 'C') == 1),
+      OP_REQUIRES(context, (GetTensorDim(dilations_, data_format_, 'N') == 1 &&
+                            GetTensorDim(dilations_, data_format_, 'C') == 1),
                   errors::InvalidArgument(
                       "Current implementation does not yet support "
                       "dilations rates in the batch and depth dimensions."));
       OP_REQUIRES(
-          context,
-          (GetTensorDim(dilations_, data_format_, '0') > 0 &&
-           GetTensorDim(dilations_, data_format_, '1') > 0 &&
-           GetTensorDim(dilations_, data_format_, '2') > 0),
+          context, (GetTensorDim(dilations_, data_format_, '0') > 0 &&
+                    GetTensorDim(dilations_, data_format_, '1') > 0 &&
+                    GetTensorDim(dilations_, data_format_, '2') > 0),
           errors::InvalidArgument("Dilated rates should be larger than 0."));
     }
   }
@@ -930,7 +925,7 @@ class MklConvOp : public OpKernel {
       memory::dims dst_dims_tf_order, dst_dims_mkl_order;
 
       // If pad with conv2d fusion is enabled
-      if (pad_enabled) {
+      if (fuse_pad_) {
         PadWithConvFusion(context, padding_left, padding_right);
       }
 
@@ -942,7 +937,7 @@ class MklConvOp : public OpKernel {
       conv_utl.GetConvFwdSizesInMklOrder(
           src_tf_shape, filter_tf_shape, &src_dims, &filter_dims, &strides,
           &dilations, &dst_dims_tf_order, &dst_dims_mkl_order, &padding_left,
-          &padding_right, pad_enabled, is_depthwise);
+          &padding_right, fuse_pad_, is_depthwise);
       if (!context->status().ok()) return;
 
       // Check for corner case - if there is nothing to compute, return.
@@ -984,7 +979,7 @@ class MklConvOp : public OpKernel {
       // TODO(Intel-tf) Add check to make sure pad_enabled is true only for 2D
       if (!is_conv2d) {
         OP_REQUIRES(
-            context, !pad_enabled,
+            context, !fuse_pad_,
             errors::InvalidArgument("Pad+Conv fusion only works for 2D"));
       }
       // Create memory for user data.
@@ -1131,7 +1126,7 @@ class MklConvOp : public OpKernel {
 
   void PadWithConvFusion(OpKernelContext* context, memory::dims& padding_left,
                          memory::dims& padding_right) {
-    const Tensor& paddings_tf = MklGetInput(context, 2);
+    const Tensor& paddings_tf = MklGetInput(context, input_index_pad);
     OP_REQUIRES(context, paddings_tf.dims() == 2,
                 errors::InvalidArgument("paddings must be 2-dimensional: ",
                                         paddings_tf.shape().DebugString()));
@@ -1170,6 +1165,11 @@ class MklConvOp : public OpKernel {
  protected:
   void set_fuse_biasadd(bool fuse_biasadd) { fuse_biasadd_ = fuse_biasadd; }
   void set_fuse_relu(bool fuse_relu) { fuse_relu_ = fuse_relu; }
+  void set_fuse_pad(bool fuse_pad) {
+    fuse_pad_ = fuse_pad;
+    // In PadwithFusedConv OP, pad is the fourth index.
+    input_index_pad = 3;
+  }
 
   // This method is for the base class MklConvOp, which handles the
   // floating point implementation of Conv. The quantized conv implementations
@@ -1243,9 +1243,11 @@ class MklConvOp : public OpKernel {
   // Initialize to values the template is instantiated with
   bool fuse_biasadd_ = bias_enabled;
   bool fuse_relu_ = false;
+  bool fuse_pad_ = pad_enabled;
+
+  int input_index_pad = 2;
 
   const int kInputIndex_Src = 0, kInputIndex_Filter = 1, kInputIndex_Bias = 2;
-  const int kInputIndex_Pad = 2;
   const int kOutputIndex_Dst = 0, kOutputIndex_Filter = 1;
   const int kDilationH = 0, kDilationW = 1;
 
@@ -1314,14 +1316,15 @@ class MklConvOp : public OpKernel {
 
 // Base class for fused convolution forward operations
 template <typename Device, typename Tinput, typename Tfilter, typename Tbias,
-          typename Toutput, typename Ttemp_output>
+          typename Toutput, typename Ttemp_output, typename Tpadding,
+          bool pad_enabled>
 class MklFusedConvOp
     : public MklConvOp<Device, Tinput, Tfilter, Tbias, Toutput, Ttemp_output,
-                       int32, false, false, false> {
+                       Tpadding, false, false, false> {
  public:
   explicit MklFusedConvOp(OpKernelConstruction* context)
-      : MklConvOp<Device, Tinput, Tfilter, Tbias, Toutput, Ttemp_output, int32,
-                  false, false, false>(context) {
+      : MklConvOp<Device, Tinput, Tfilter, Tbias, Toutput, Ttemp_output,
+                  Tpadding, false, false, false>(context) {
     // Since we came here through the registration of _MklFusedConv2D, get
     // all information from 'fused_ops' and 'num_args'
     std::vector<string> fused_ops;
@@ -1351,6 +1354,10 @@ class MklFusedConvOp
                   errors::Unimplemented("Fusion is not implemented: [",
                                         str_util::Join(fused_ops, ","), "]"));
     }
+
+    if (pad_enabled) {
+      this->set_fuse_pad(true);
+    }
   }
 
   virtual ~MklFusedConvOp() {}
@@ -1663,8 +1670,8 @@ class MklQuantizedConv2DSumReluOp
     const float max_filter =
         context->input(5 + bias_index_offset).flat<float>()(0);
 
-    reorder_sum_scale = 255.0 * 127.0 /
-                        (std::max(std::abs(max_input), std::abs(min_input)) *
+    reorder_sum_scale =
+        255.0 * 127.0 / (std::max(std::abs(max_input), std::abs(min_input)) *
                          std::max(std::abs(max_filter), std::abs(min_filter)));
     std::vector<float> scales;
     scales.push_back(reorder_sum_scale);
@@ -1966,11 +1973,33 @@ TF_CALL_float(REGISTER_MKL_CPU_2D);
 TF_CALL_float(REGISTER_MKL_CPU_2D_DEPTHWISE);
 
 #define REGISTER_MKL_CPU_2D_FUSED(T)                                \
-  REGISTER_KERNEL_BUILDER(Name("_MklFusedConv2D")                   \
+  REGISTER_KERNEL_BUILDER(                                          \
+      Name("_MklFusedConv2D")                                       \
+          .Device(DEVICE_CPU)                                       \
+          .TypeConstraint<T>("T")                                   \
+          .Label(mkl_op_registry::kMklOpLabel),                     \
+      MklFusedConvOp<CPUDevice, T, T, T, T, T, int32, false>);      \
+  REGISTER_KERNEL_BUILDER(                                          \
+      Name("_MklPadWithFusedConv2D")                                \
+          .Device(DEVICE_CPU)                                       \
+          .TypeConstraint<int32>("Tpaddings")                       \
+          .TypeConstraint<T>("T")                                   \
+          .Label(mkl_op_registry::kMklOpLabel),                     \
+      MklFusedConvOp<CPUDevice, T, T, T, T, T, int32, true>);       \
+  REGISTER_KERNEL_BUILDER(                                          \
+      Name("_MklPadWithFusedConv2D")                                \
+          .Device(DEVICE_CPU)                                       \
+          .TypeConstraint<T>("T")                                   \
+          .TypeConstraint<int64>("Tpaddings")                       \
+          .Label(mkl_op_registry::kMklOpLabel),                     \
+      MklFusedConvOp<CPUDevice, T, T, T, T, T, int64, true>);       \
+  REGISTER_KERNEL_BUILDER(Name("__MklDummyPadWithFusedConv2D")      \
                               .Device(DEVICE_CPU)                   \
                               .TypeConstraint<T>("T")               \
+                              .TypeConstraint<int32>("Tpaddings")   \
                               .Label(mkl_op_registry::kMklOpLabel), \
-                          MklFusedConvOp<CPUDevice, T, T, T, T, T>);
+                          MklDummyOp<CPUDevice, T>);
+// Note we are registering _MklFusedConv2D.
 // We check the fused_ops attributes to decide if bias is enabled or not.
 
 TF_CALL_float(REGISTER_MKL_CPU_2D_FUSED);
diff --git a/tensorflow/core/kernels/mkl_fused_ops_test.cc b/tensorflow/core/kernels/mkl_fused_ops_test.cc
index 258cca9332..f1243c5b68 100644
--- a/tensorflow/core/kernels/mkl_fused_ops_test.cc
+++ b/tensorflow/core/kernels/mkl_fused_ops_test.cc
@@ -59,6 +59,23 @@ class ConvMklToTF : public OpsTestBase {
     *output = *GetOutput(0);
   }
 
+  // Runs a Tensorflow graph defined by the root scope, and fetches the result
+  // of 'fetch' node into the output Tensor.
+  void RunAndFetch(const tensorflow::Scope& root, const string& fetch,
+                   Tensor* output) {
+    tensorflow::GraphDef graph;
+    TF_ASSERT_OK(root.ToGraphDef(&graph));
+
+    std::unique_ptr<tensorflow::Session> session(
+        tensorflow::NewSession(tensorflow::SessionOptions()));
+    TF_ASSERT_OK(session->Create(graph));
+
+    std::vector<Tensor> unfused_tensors;
+    TF_ASSERT_OK(session->Run({}, {fetch}, {}, &unfused_tensors));
+
+    *output = unfused_tensors[0];
+  }
+
   void ConvertAndCompare(DataType dtype, const Tensor& tensor,
                          const Tensor& mkl_meta_tensor,
                          const Tensor& expected) {
@@ -83,23 +100,6 @@ class MklFusedConv2DOpTest : public OpsTestBase {
       std::function<void(const Tensor& input_data, const Tensor& filter_data,
                          const Tensor& bias_data, Tensor* out)>;
 
-  // Runs a Tensorflow graph defined by the root scope, and fetches the result
-  // of 'fetch' node into the output Tensor.
-  void RunAndFetch(const tensorflow::Scope& root, const string& fetch,
-                   Tensor* output) {
-    tensorflow::GraphDef graph;
-    TF_ASSERT_OK(root.ToGraphDef(&graph));
-
-    std::unique_ptr<tensorflow::Session> session(
-        tensorflow::NewSession(tensorflow::SessionOptions()));
-    TF_ASSERT_OK(session->Create(graph));
-
-    std::vector<Tensor> unfused_tensors;
-    TF_ASSERT_OK(session->Run({}, {fetch}, {}, &unfused_tensors));
-
-    *output = unfused_tensors[0];
-  }
-
   void RunConv2DWithBias(const Tensor& input_data, const Tensor& filter_data,
                          const Tensor& bias_data, Tensor* output,
                          int stride = 1) {
@@ -115,7 +115,8 @@ class MklFusedConv2DOpTest : public OpsTestBase {
         root.WithOpName("with_bias"), conv,
         ops::Const(root.WithOpName("bias"), Input::Initializer(bias_data)));
 
-    RunAndFetch(root, "with_bias", output);
+    ConvMklToTF<T> conv_comp;
+    conv_comp.RunAndFetch(root, "with_bias", output);
   }
 
   void RunConv2DWithBiasAndRelu(const Tensor& input_data,
@@ -136,7 +137,8 @@ class MklFusedConv2DOpTest : public OpsTestBase {
 
     auto with_relu = ops::Relu(root.WithOpName("with_relu"), with_bias);
 
-    RunAndFetch(root, "with_relu", output);
+    ConvMklToTF<T> conv_comp;
+    conv_comp.RunAndFetch(root, "with_relu", output);
   }
 
   void RunMklFusedConv2DOp(const Tensor& image, const Tensor& filter,
@@ -218,18 +220,18 @@ class MklFusedConv2DOpTest : public OpsTestBase {
                             int depth = kDepth, int image_width = kImageWidth,
                             int image_height = kImageHeight,
                             int image_batch_count = kImageBatchCount) {
-    const BiasAddGraphRunner run_default =
-        [this](const Tensor& input_data, const Tensor& filter_data,
-               const Tensor& bias_data, Tensor* out) {
-          RunConv2DWithBias(input_data, filter_data, bias_data, out);
-        };
-
-    const BiasAddGraphRunner run_fused =
-        [this](const Tensor& input_data, const Tensor& filter_data,
-               const Tensor& bias_data, Tensor* out) {
-          RunMklFusedConv2DOp(input_data, filter_data, {bias_data}, {"BiasAdd"},
-                              out);
-        };
+    const BiasAddGraphRunner run_default = [this](
+        const Tensor& input_data, const Tensor& filter_data,
+        const Tensor& bias_data, Tensor* out) {
+      RunConv2DWithBias(input_data, filter_data, bias_data, out);
+    };
+
+    const BiasAddGraphRunner run_fused = [this](
+        const Tensor& input_data, const Tensor& filter_data,
+        const Tensor& bias_data, Tensor* out) {
+      RunMklFusedConv2DOp(input_data, filter_data, {bias_data}, {"BiasAdd"},
+                          out);
+    };
 
     VerifyBiasAddTensorsNear(depth, image_width, image_height,
                              image_batch_count, filter_size, filter_count,
@@ -243,18 +245,18 @@ class MklFusedConv2DOpTest : public OpsTestBase {
                                    int image_width = kImageWidth,
                                    int image_height = kImageHeight,
                                    int image_batch_count = kImageBatchCount) {
-    const BiasAddGraphRunner run_default =
-        [this](const Tensor& input_data, const Tensor& filter_data,
-               const Tensor& bias_data, Tensor* out) {
-          RunConv2DWithBiasAndRelu(input_data, filter_data, bias_data, out);
-        };
-
-    const BiasAddGraphRunner run_fused =
-        [this](const Tensor& input_data, const Tensor& filter_data,
-               const Tensor& bias_data, Tensor* out) {
-          RunMklFusedConv2DOp(input_data, filter_data, {bias_data},
-                              {"BiasAdd", "Relu"}, out);
-        };
+    const BiasAddGraphRunner run_default = [this](
+        const Tensor& input_data, const Tensor& filter_data,
+        const Tensor& bias_data, Tensor* out) {
+      RunConv2DWithBiasAndRelu(input_data, filter_data, bias_data, out);
+    };
+
+    const BiasAddGraphRunner run_fused = [this](
+        const Tensor& input_data, const Tensor& filter_data,
+        const Tensor& bias_data, Tensor* out) {
+      RunMklFusedConv2DOp(input_data, filter_data, {bias_data},
+                          {"BiasAdd", "Relu"}, out);
+    };
 
     VerifyBiasAddTensorsNear(depth, image_width, image_height,
                              image_batch_count, filter_size, filter_count,
@@ -401,5 +403,262 @@ TEST_F(FusedPadConvOpTest, PaddingConvTestNchw) {
 
   Run<float>(DT_FLOAT, image, filter, padding, expected, "NCHW");
 }
+
+// Testing fusion of pad and fusedconv2d
+
+template <typename T>
+class MklPadWithFusedConv2DOpTest : public OpsTestBase {
+ protected:
+  static constexpr int kDepth = 3;
+  static constexpr int kImageWidth = 30;
+  static constexpr int kImageHeight = 28;
+  static constexpr int kImageBatchCount = 8;
+
+  // 0: top pad, 1 bottom pad, 2 left pad, 3 right pad
+  int padding_list[4];
+
+  using BiasAddGraphRunner =
+      std::function<void(const Tensor& input_data, const Tensor& filter_data,
+                         const Tensor& bias_data, Tensor* out)>;
+
+  void VerifyBiasAddTensorsNear(int depth, int image_width, int image_height,
+                                int image_batch_count, int filter_size,
+                                int filter_count,
+                                const BiasAddGraphRunner& run_default,
+                                const BiasAddGraphRunner& run_fused) {
+    DataType dtype = DataTypeToEnum<T>::v();
+
+    Tensor image(dtype, {image_batch_count, image_height, image_width, depth});
+    image.flat<T>() = image.flat<T>().setRandom();
+
+    Tensor filter(dtype, {filter_size, filter_size, depth, filter_count});
+    filter.flat<T>() = filter.flat<T>().setRandom();
+
+    const int bias_size = filter_count;
+    Tensor bias(dtype, {bias_size});
+    bias.flat<T>() = bias.flat<T>().setRandom();
+
+    Tensor conv_2d;
+    Tensor fused_conv_2d;
+
+    run_default(image, filter, bias, &conv_2d);
+    run_fused(image, filter, bias, &fused_conv_2d);
+
+    ASSERT_EQ(conv_2d.dtype(), fused_conv_2d.dtype());
+    ASSERT_EQ(conv_2d.shape(), fused_conv_2d.shape());
+
+    test::ExpectClose(conv_2d, fused_conv_2d);
+  }
+
+  // Verifies that computing Pad+Conv2D+BiasAdd in a graph is identical to
+  // FusedConv2D.
+  void VerifyConv2DWithBiasAndPad(int filter_size, int filter_count,
+                                  int depth = kDepth,
+                                  int image_width = kImageWidth,
+                                  int image_height = kImageHeight,
+                                  int image_batch_count = kImageBatchCount) {
+    const BiasAddGraphRunner run_default = [this](
+        const Tensor& input_data, const Tensor& filter_data,
+        const Tensor& bias_data, Tensor* out) {
+      RunMklFusedConv2DWithPadAndBias(input_data, filter_data, bias_data, out);
+    };
+
+    const BiasAddGraphRunner run_fused = [this](
+        const Tensor& input_data, const Tensor& filter_data,
+        const Tensor& bias_data, Tensor* out) {
+      RunMklFusedConv2DWithPadOp(input_data, filter_data, {bias_data},
+                                 {"BiasAdd"}, out);
+    };
+
+    VerifyBiasAddTensorsNear(depth, image_width, image_height,
+                             image_batch_count, filter_size, filter_count,
+                             run_default, run_fused);
+  }
+
+  // Verifies that computing Pad+Conv2D+BiasAdd+Relu in a graph is identical to
+  // FusedConv2D.
+  void VerifyConv2DWithBiasReluAndPad(
+      int filter_size, int filter_count, int depth = kDepth,
+      int image_width = kImageWidth, int image_height = kImageHeight,
+      int image_batch_count = kImageBatchCount) {
+    const BiasAddGraphRunner run_default = [this](
+        const Tensor& input_data, const Tensor& filter_data,
+        const Tensor& bias_data, Tensor* out) {
+      RunMklFusedConv2DWithPadAndBiasRelu(input_data, filter_data, bias_data,
+                                          out);
+    };
+
+    const BiasAddGraphRunner run_fused = [this](
+        const Tensor& input_data, const Tensor& filter_data,
+        const Tensor& bias_data, Tensor* out) {
+      RunMklFusedConv2DWithPadOp(input_data, filter_data, {bias_data},
+                                 {"BiasAdd", "Relu"}, out);
+    };
+
+    VerifyBiasAddTensorsNear(depth, image_width, image_height,
+                             image_batch_count, filter_size, filter_count,
+                             run_default, run_fused);
+  }
+
+  void RunMklFusedConv2DWithPadAndBias(const Tensor& input_data,
+                                       const Tensor& filter_data,
+                                       const Tensor& bias_data, Tensor* output,
+                                       int stride = 1) {
+    auto root = tensorflow::Scope::NewRootScope();
+
+    // As FusedConv2D only support NHWC format, so here use NHWC format.
+    auto padding = ops::Const(root.WithOpName("padding"),
+                              {0, 0, padding_list[0], padding_list[1],
+                               padding_list[2], padding_list[3], 0, 0},
+                              {4, 2});
+    auto pad = ops::Pad(
+        root.WithOpName("pad"),
+        ops::Const(root.WithOpName("input"), Input::Initializer(input_data)),
+        padding);
+
+    auto conv = ops::Conv2D(
+        root.WithOpName("conv"), pad,
+        ops::Const(root.WithOpName("filter"), Input::Initializer(filter_data)),
+        {1, stride, stride, 1}, "VALID");
+
+    auto with_bias = ops::BiasAdd(
+        root.WithOpName("with_bias"), conv,
+        ops::Const(root.WithOpName("bias"), Input::Initializer(bias_data)));
+
+    ConvMklToTF<T> conv_comp;
+    conv_comp.RunAndFetch(root, "with_bias", output);
+  }
+
+  void RunMklFusedConv2DWithPadAndBiasRelu(const Tensor& input_data,
+                                           const Tensor& filter_data,
+                                           const Tensor& bias_data,
+                                           Tensor* output, int stride = 1) {
+    auto root = tensorflow::Scope::NewRootScope();
+
+    // As FusedConv2D only support NHWC format, so here use NHWC format.
+    auto padding = ops::Const(root.WithOpName("padding"),
+                              {0, 0, padding_list[0], padding_list[1],
+                               padding_list[2], padding_list[3], 0, 0},
+                              {4, 2});
+    auto pad = ops::Pad(
+        root.WithOpName("pad"),
+        ops::Const(root.WithOpName("input"), Input::Initializer(input_data)),
+        padding);
+
+    auto conv = ops::Conv2D(
+        root.WithOpName("conv"), pad,
+        ops::Const(root.WithOpName("filter"), Input::Initializer(filter_data)),
+        {1, stride, stride, 1}, "VALID");
+
+    auto with_bias = ops::BiasAdd(
+        root.WithOpName("with_bias"), conv,
+        ops::Const(root.WithOpName("bias"), Input::Initializer(bias_data)));
+
+    auto with_relu = ops::Relu(root.WithOpName("with_relu"), with_bias);
+
+    ConvMklToTF<T> conv_comp;
+    conv_comp.RunAndFetch(root, "with_relu", output);
+  }
+
+  void RunMklFusedConv2DWithPadOp(const Tensor& image, const Tensor& filter,
+                                  const std::vector<Tensor>& args,
+                                  const std::vector<string>& fused_ops,
+                                  Tensor* output, int stride = 1) {
+    DataType dtype = DataTypeToEnum<T>::v();
+    const int num_args = static_cast<int>(args.size());
+    Tensor padding(DT_INT32, {4, 2});
+    test::FillValues<int32>(&padding, {0, 0, padding_list[0], padding_list[1],
+                                       padding_list[2], padding_list[3], 0, 0});
+
+    TF_EXPECT_OK(NodeDefBuilder("pad_fused_conv_op", "_MklPadWithFusedConv2D")
+                     .Input(FakeInput(dtype))
+                     .Input(FakeInput(dtype))
+                     .Attr("num_args", num_args)
+                     .Input(FakeInput(num_args, dtype))
+                     .Input(FakeInput(DT_INT32))
+                     .Input(FakeInput(DT_UINT8))
+                     .Input(FakeInput(DT_UINT8))
+                     .Input(FakeInput(num_args, DT_UINT8))
+                     .Input(FakeInput(DT_UINT8))
+                     .Attr("T", dtype)
+                     .Attr("strides", {1, stride, stride, 1})
+                     .Attr("padding", "VALID")
+                     .Attr("fused_ops", fused_ops)
+                     .Attr("_kernel", "MklOp")
+                     .Finalize(node_def()));
+
+    TF_EXPECT_OK(InitOp());
+
+    AddInputFromArray<T>(image.shape(), image.flat<T>());
+    AddInputFromArray<T>(filter.shape(), filter.flat<T>());
+    for (const Tensor& arg : args)
+      AddInputFromArray<T>(arg.shape(), arg.flat<T>());
+    AddInputFromArray<int32>(padding.shape(), padding.flat<int32>());
+    AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+    AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+    AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+    for (const Tensor& arg : args)
+      AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+    TF_ASSERT_OK(RunOpKernel());
+
+    // Compare output to expected results
+    const Tensor& output_tensor = *GetOutput(0);
+    // Index 2 will need to be changed if the number of outputs produced
+    // by MklConv2D change.
+    const Tensor& output_meta_tensor = *GetOutput(2);
+    ConvMklToTF<T> conv_comp;
+    conv_comp.PerformConversion(dtype, output_tensor, output_meta_tensor,
+                                output);
+  }
+
+ public:
+  void SetPaddingList(int top, int bottom, int left, int right) {
+    padding_list[0] = top;
+    padding_list[1] = bottom;
+    padding_list[2] = left;
+    padding_list[3] = right;
+  }
+};
+
+TYPED_TEST_CASE_P(MklPadWithFusedConv2DOpTest);
+
+TYPED_TEST_P(MklPadWithFusedConv2DOpTest, WithBiasAndRoundPad) {
+  const int filter_size = 1;
+  const int filter_count = 12;
+  this->SetPaddingList(2, 2, 1, 1);
+  this->VerifyConv2DWithBiasAndPad(filter_size, filter_count);
+}
+
+TYPED_TEST_P(MklPadWithFusedConv2DOpTest, WithBiasAndPartialPad) {
+  const int filter_size = 1;
+  const int filter_count = 12;
+  this->SetPaddingList(4, 0, 2, 0);
+  this->VerifyConv2DWithBiasAndPad(filter_size, filter_count);
+}
+
+TYPED_TEST_P(MklPadWithFusedConv2DOpTest, WithBiasReluAndRoundPad) {
+  const int filter_size = 1;
+  const int filter_count = 12;
+  this->SetPaddingList(2, 2, 1, 1);
+  this->VerifyConv2DWithBiasReluAndPad(filter_size, filter_count);
+}
+
+TYPED_TEST_P(MklPadWithFusedConv2DOpTest, WithBiasReluAndPartialPad) {
+  const int filter_size = 1;
+  const int filter_count = 12;
+  this->SetPaddingList(4, 0, 2, 0);
+  this->VerifyConv2DWithBiasReluAndPad(filter_size, filter_count);
+}
+
+REGISTER_TYPED_TEST_CASE_P(MklPadWithFusedConv2DOpTest,  //
+                           WithBiasAndRoundPad,          //
+                           WithBiasAndPartialPad,        //
+                           WithBiasReluAndRoundPad,      //
+                           WithBiasReluAndPartialPad);
+
+using MklPadWithFusedConv2DDataTypes = ::testing::Types<float>;
+INSTANTIATE_TYPED_TEST_CASE_P(Test, MklPadWithFusedConv2DOpTest,
+                              MklPadWithFusedConv2DDataTypes);
+
 }  // namespace tensorflow
 #endif  // INTEL_MKL
diff --git a/tensorflow/core/ops/mkl_nn_ops.cc b/tensorflow/core/ops/mkl_nn_ops.cc
index 658afd9901..6386bbff40 100644
--- a/tensorflow/core/ops/mkl_nn_ops.cc
+++ b/tensorflow/core/ops/mkl_nn_ops.cc
@@ -59,6 +59,63 @@ REGISTER_OP("_MklFusedConv2D")
  is expected to create these operators.
 )doc");
 
+REGISTER_OP("__MklDummyPadWithFusedConv2D")
+    .Input("input: T")
+    .Input("filter: T")
+    .Input("args: num_args * T")
+    .Input("paddings: Tpaddings")
+    .Output("output: T")
+    .Output("filter_output: T")
+    .Output("mkl_output: uint8")
+    .Output("mkl_filter_output: uint8")
+    .Attr("T: {float}")
+    .Attr("num_args: int >= 0")
+    .Attr("strides: list(int)")
+    .Attr(GetPaddingAttrString())
+    .Attr(GetConvnetDataFormatAttrString())
+    .Attr("dilations: list(int) = [1, 1, 1, 1]")
+    .Attr("fused_ops: list(string) = []")
+    .Attr("Tpaddings: {int32, int64} = DT_INT32")
+    // Attributes for the FusedBatchNorm ------------------------------------ //
+    .Attr("epsilon: float = 0.0001")
+    // ---------------------------------------------------------------------- //
+    .SetShapeFn(shape_inference::Conv2DShape)
+    .Doc(R"doc(
+*NOTE*: Do not invoke this operator directly in Python. MKL DNN graph transformer
+ is expected to create these operators.
+)doc");
+
+REGISTER_OP("_MklPadWithFusedConv2D")
+    .Input("input: T")
+    .Input("filter: T")
+    .Input("args: num_args * T")
+    .Input("paddings: Tpaddings")
+    .Input("mkl_input: uint8")
+    .Input("mkl_filter: uint8")
+    .Input("mkl_args: num_args * uint8")
+    .Input("mkl_paddings: uint8")
+    .Output("output: T")
+    .Output("filter_output: T")
+    .Output("mkl_output: uint8")
+    .Output("mkl_filter_output: uint8")
+    .Attr("T: {float}")
+    .Attr("num_args: int >= 0")
+    .Attr("strides: list(int)")
+    .Attr("is_filter_const: bool = false")
+    .Attr(GetPaddingAttrString())
+    .Attr(GetConvnetDataFormatAttrString())
+    .Attr("dilations: list(int) = [1, 1, 1, 1]")
+    .Attr("fused_ops: list(string) = []")
+    .Attr("Tpaddings: {int32, int64} = DT_INT32")
+    // Attributes for the FusedBatchNorm ------------------------------------ //
+    .Attr("epsilon: float = 0.0001")
+    // ---------------------------------------------------------------------- //
+    .SetShapeFn(shape_inference::Conv2DShape)
+    .Doc(R"doc(
+*NOTE*: Do not invoke this operator directly in Python. MKL DNN graph transformer
+ is expected to create these operators.
+)doc");
+
 REGISTER_OP("_MklQuantizedMaxPool")
     .Input("input:         T")
     .Input("min_input:     float")
-- 
GitLab


From 6ecb77b9bd800e2beda7a7fcefaf0ac3a6363e1f Mon Sep 17 00:00:00 2001
From: Samantha Andow <samdow@fb.com>
Date: Wed, 26 Dec 2018 08:58:44 -0800
Subject: [PATCH 0080/2345] Fix iterable bug

---
 tensorflow/java/src/gen/cc/op_specs.cc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/java/src/gen/cc/op_specs.cc b/tensorflow/java/src/gen/cc/op_specs.cc
index 4f5a491d25..4024efedef 100644
--- a/tensorflow/java/src/gen/cc/op_specs.cc
+++ b/tensorflow/java/src/gen/cc/op_specs.cc
@@ -91,11 +91,6 @@ class TypeResolver {
 
 Type TypeResolver::TypeOf(const OpDef_ArgDef& arg_def, bool* iterable_out) {
   *iterable_out = false;
-  if (!arg_def.number_attr().empty()) {
-    // when number_attr is set, argument has to be a list of tensors
-    *iterable_out = true;
-    visited_attrs_.insert(std::make_pair(arg_def.number_attr(), Type::Int()));
-  }
   Type type = Type::Wildcard();
   if (arg_def.type() != DataType::DT_INVALID) {
     type = Type::ForDataType(arg_def.type());
@@ -122,6 +117,11 @@ Type TypeResolver::TypeOf(const OpDef_ArgDef& arg_def, bool* iterable_out) {
     LOG(FATAL) << "Cannot resolve data type of argument \"" << arg_def.name()
                << "\" in operation \"" << op_def_.name() << "\"";
   }
+  if (!arg_def.number_attr().empty()) {
+    // when number_attr is set, argument has to be a list of tensors
+    *iterable_out = true;
+    visited_attrs_.insert(std::make_pair(arg_def.number_attr(), Type::Int()));
+  }
   return type;
 }
 
-- 
GitLab


From 1ae58b7732b5b5c390bdec5499497a87222d6fcd Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Wed, 26 Dec 2018 20:07:56 +0000
Subject: [PATCH 0081/2345] Fix tf.version.GIT_VERSION to return str (instead
 of bytes)

This fix fixes the issue raised in 24578 where
`tf.version.GIT_VERSION` returns byte instead of string:
```
Python 3.6.7 (default, Oct 22 2018, 11:32:17)
[GCC 8.2.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import tensorflow as tf
>>> print(tf.version.GIT_VERSION)
b'v1.12.0-5133-gc343196842'
>>> print(tf.version.COMPILER_VERSION)
4.8.5
>>> print(tf.version.VERSION)
1.13.0-dev20181226
```

This fix fixes 24578.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/tools/git/gen_git_source.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/git/gen_git_source.py b/tensorflow/tools/git/gen_git_source.py
index 645d817d9f..0d0a641074 100755
--- a/tensorflow/tools/git/gen_git_source.py
+++ b/tensorflow/tools/git/gen_git_source.py
@@ -216,7 +216,7 @@ const int tf_monolithic_build() {
   return 0;
 #endif
 }
-""" % git_version
+""" % git_version.decode('utf-8')
   open(filename, "w").write(contents)
 
 
-- 
GitLab


From d0913801f880d19cdd62b0374f7a1ede2cb60416 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Wed, 26 Dec 2018 20:17:35 +0000
Subject: [PATCH 0082/2345] Update git_version to be the same type of byte
 before final conversion

"git_version_is_invalid" should be b"git_version_is_invalid" so
that `git_version` is always byte (conversion will happen later)

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/tools/git/gen_git_source.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/git/gen_git_source.py b/tensorflow/tools/git/gen_git_source.py
index 0d0a641074..f97cc8d1d6 100755
--- a/tensorflow/tools/git/gen_git_source.py
+++ b/tensorflow/tools/git/gen_git_source.py
@@ -189,7 +189,7 @@ def write_version_info(filename, git_version):
     git_version: the result of a git describe.
   """
   if b"\"" in git_version or b"\\" in git_version:
-    git_version = "git_version_is_invalid"  # do not cause build to fail!
+    git_version = b"git_version_is_invalid"  # do not cause build to fail!
   contents = """/*  Generated by gen_git_source.py  */
 #include <string>
 const char* tf_git_version() {return "%s";}
-- 
GitLab


From a044d650c4d7b731a435cfb216d64629e20b6202 Mon Sep 17 00:00:00 2001
From: Bhavani Subramanian <bhavani1.subramanian@intel.com>
Date: Wed, 26 Dec 2018 14:00:58 -0800
Subject: [PATCH 0083/2345] Enabled filter caching for convolution when filter
 is a constant.

---
 tensorflow/core/graph/mkl_layout_pass.cc      | 131 ++++--
 tensorflow/core/graph/mkl_layout_pass_test.cc | 333 +++++++++++++
 tensorflow/core/kernels/mkl_conv_ops.cc       | 442 +++++++++++-------
 tensorflow/core/kernels/mkl_fused_ops_test.cc | 113 ++++-
 tensorflow/core/ops/mkl_nn_ops.cc             |  12 +
 tensorflow/core/ops/nn_ops.cc                 |  19 +-
 6 files changed, 831 insertions(+), 219 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 9495132f4a..035b3a46ba 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -357,9 +357,9 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
                       CopyAttrsConcatV2, AlwaysRewrite});
     rinfo_.push_back({csinfo_.conv2d,
                       mkl_op_registry::GetMklOpName(csinfo_.conv2d),
-                      CopyAttrsConv, AlwaysRewrite});
+                      CopyAttrsConvCheckConstFilter, AlwaysRewrite});
     rinfo_.push_back({csinfo_.conv2d_with_bias, csinfo_.mkl_conv2d_with_bias,
-                      CopyAttrsConv, AlwaysRewrite});
+                      CopyAttrsConvCheckConstFilter, AlwaysRewrite});
     rinfo_.push_back({csinfo_.conv2d_grad_filter,
                       mkl_op_registry::GetMklOpName(csinfo_.conv2d_grad_filter),
                       CopyAttrsConv, AlwaysRewrite});
@@ -371,7 +371,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
                       CopyAttrsConv, AlwaysRewrite});
     rinfo_.push_back({csinfo_.conv3d,
                       mkl_op_registry::GetMklOpName(csinfo_.conv3d),
-                      CopyAttrsConv, AlwaysRewrite});
+                      CopyAttrsConvCheckConstFilter, AlwaysRewrite});
     rinfo_.push_back({csinfo_.conv3d_grad_filter,
                       mkl_op_registry::GetMklOpName(csinfo_.conv3d_grad_filter),
                       CopyAttrsConv, AlwaysRewrite});
@@ -1391,10 +1391,13 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
                               bool change_format = false);
   static void CopyAttrsConcatV2(const Node* orig_node, NodeBuilder* nb,
                                 bool change_format = false);
-  static void CopyAttrsConv2DDepthwise(const Node* orig_node, NodeBuilder* nb,
-                                       bool change_format = false);
+  static void CopyAttrsConvCheckConstFilter(const Node* orig_node,
+                                            NodeBuilder* nb,
+                                            bool change_format = false);
   static void CopyAttrsConv(const Node* orig_node, NodeBuilder* nb,
                             bool change_format = false);
+  static void CopyAttrsConv2DDepthwise(const Node* orig_node, NodeBuilder* nb,
+                                       bool change_format = false);
   static void CopyAttrsDataType(const Node* orig_node, NodeBuilder* nb,
                                 bool change_format = false);
   static void CopyAttrsFusedBatchNorm(const Node* orig_node, NodeBuilder* nb,
@@ -1426,6 +1429,10 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
                              bool change_format = false);
   static void CopyAttrsSplit(const Node* orig_node, NodeBuilder* nb,
                              bool change_format = false);
+  static void CopyFormatAttrsConv(const Node* orig_node, NodeBuilder* nb,
+                                  const std::vector<int32>& strides,
+                                  const std::vector<int32>& dilations,
+                                  bool change_format = false);
 
   // Generate a graph node in graph 'g' representing a dummy Mkl tensor node,
   // using node for original node 'orig_node' and return it in '*out'.
@@ -1929,23 +1936,10 @@ void MklLayoutRewritePass::AddWorkSpaceEdgeIfNeeded(
 // Op-specific functions to copy attributes from old node to new node
 //////////////////////////////////////////////////////////////////////////
 
-void MklLayoutRewritePass::CopyAttrsConv(const Node* orig_node, NodeBuilder* nb,
-                                         bool change_format) {
-  DataType T;
+void MklLayoutRewritePass::CopyFormatAttrsConv(
+    const Node* orig_node, NodeBuilder* nb, const std::vector<int32>& strides,
+    const std::vector<int32>& dilations, bool change_format) {
   string data_format;
-  string padding;
-  std::vector<int32> strides;
-  std::vector<int32> dilations;
-
-  // Get all attributes from old node.
-  TF_CHECK_OK(GetNodeAttr(orig_node->def(), "T", &T));
-  TF_CHECK_OK(GetNodeAttr(orig_node->def(), "strides", &strides));
-  TF_CHECK_OK(GetNodeAttr(orig_node->def(), "dilations", &dilations));
-  TF_CHECK_OK(GetNodeAttr(orig_node->def(), "padding", &padding));
-
-  // Add attributes to new node.
-  nb->Attr("T", T);
-  nb->Attr("padding", padding);
 
   if (!change_format) {
     nb->Attr("strides", strides);
@@ -1957,9 +1951,8 @@ void MklLayoutRewritePass::CopyAttrsConv(const Node* orig_node, NodeBuilder* nb,
     std::vector<int32> new_strides;
     std::vector<int32> new_dilations;
     if (strides.size() == 5) {
-      // "strides" and "dilations" also need to be changed according to
-      // "data_format",
-      // in this case, is "NDHWC" to "NCDHW".
+      // `strides` and `dilations` also need to be changed according to
+      // `data_format`. In this case, from `NDHWC` to `NCDHW`.
       new_strides = {strides[NDHWC::dim::N], strides[NDHWC::dim::C],
                      strides[NDHWC::dim::D], strides[NDHWC::dim::H],
                      strides[NDHWC::dim::W]};
@@ -1968,9 +1961,8 @@ void MklLayoutRewritePass::CopyAttrsConv(const Node* orig_node, NodeBuilder* nb,
                        dilations[NDHWC::dim::D], dilations[NDHWC::dim::H],
                        dilations[NDHWC::dim::W]};
     } else {
-      // "strides" and "dilations" also need to be changed according to
-      // "data_format",
-      // in this case, is "NHWC" to "NCHW".
+      // `strides` and `dilations` also need to be changed according to
+      // `data_format`. In this case, from `NHWC` to `NCHW`.
 
       new_strides = {strides[NHWC::dim::N], strides[NHWC::dim::C],
                      strides[NHWC::dim::H], strides[NHWC::dim::W]};
@@ -1983,6 +1975,56 @@ void MklLayoutRewritePass::CopyAttrsConv(const Node* orig_node, NodeBuilder* nb,
   }
 }
 
+void MklLayoutRewritePass::CopyAttrsConvCheckConstFilter(const Node* orig_node,
+                                                         NodeBuilder* nb,
+                                                         bool change_format) {
+  DataType T;
+  string padding;
+  std::vector<int32> strides;
+  std::vector<int32> dilations;
+  bool is_filter_const;
+
+  // Get all attributes from old node.
+  TF_CHECK_OK(GetNodeAttr(orig_node->def(), "T", &T));
+  TF_CHECK_OK(GetNodeAttr(orig_node->def(), "strides", &strides));
+  TF_CHECK_OK(GetNodeAttr(orig_node->def(), "dilations", &dilations));
+  TF_CHECK_OK(GetNodeAttr(orig_node->def(), "padding", &padding));
+
+  // Check if filter is a constant.
+  Node* filter_node = nullptr;
+  orig_node->input_node(1, &filter_node);
+  is_filter_const = filter_node->IsConstant();
+
+  // Add attributes to new node.
+  nb->Attr("T", T);
+  nb->Attr("padding", padding);
+  nb->Attr("is_filter_const", is_filter_const);
+
+  // Add attributes related to `data_format`.
+  CopyFormatAttrsConv(orig_node, nb, strides, dilations, change_format);
+}
+
+void MklLayoutRewritePass::CopyAttrsConv(const Node* orig_node, NodeBuilder* nb,
+                                         bool change_format) {
+  DataType T;
+  string padding;
+  std::vector<int32> strides;
+  std::vector<int32> dilations;
+
+  // Get all attributes from old node.
+  TF_CHECK_OK(GetNodeAttr(orig_node->def(), "T", &T));
+  TF_CHECK_OK(GetNodeAttr(orig_node->def(), "strides", &strides));
+  TF_CHECK_OK(GetNodeAttr(orig_node->def(), "dilations", &dilations));
+  TF_CHECK_OK(GetNodeAttr(orig_node->def(), "padding", &padding));
+
+  // Add attributes to new node.
+  nb->Attr("T", T);
+  nb->Attr("padding", padding);
+
+  // Add attributes related to `data_format`.
+  CopyFormatAttrsConv(orig_node, nb, strides, dilations, change_format);
+}
+
 // Used in rinfo when replacing __MklDummyPadWithConv2D by _MklPadWithConv2D
 void MklLayoutRewritePass::CopyAttrsPadWithConv2D(const Node* orig_node,
                                                   NodeBuilder* nb,
@@ -1993,6 +2035,7 @@ void MklLayoutRewritePass::CopyAttrsPadWithConv2D(const Node* orig_node,
   string padding;
   std::vector<int32> strides;
   std::vector<int32> dilations;
+  bool is_filter_const;
   bool use_cudnn_on_gpu;
 
   // Get all attributes from old node.
@@ -2005,8 +2048,14 @@ void MklLayoutRewritePass::CopyAttrsPadWithConv2D(const Node* orig_node,
       GetNodeAttr(orig_node->def(), "use_cudnn_on_gpu", &use_cudnn_on_gpu));
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "Tpaddings", &Tpaddings));
 
+  // Check if filter is a constant.
+  Node* filter_node = nullptr;
+  orig_node->input_node(1, &filter_node);
+  is_filter_const = filter_node->IsConstant();
+
   // Add attributes to new node.
   nb->Attr("T", T);
+  nb->Attr("is_filter_const", is_filter_const);
   nb->Attr("strides", strides);
   nb->Attr("dilations", dilations);
   nb->Attr("padding", padding);
@@ -2057,6 +2106,7 @@ void MklLayoutRewritePass::CopyAttrsConv2DDepthwise(const Node* orig_node,
   string padding;
   std::vector<int32> strides;
   std::vector<int32> dilations;
+  bool is_filter_const;
 
   // Get all attributes from old node.
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "T", &T));
@@ -2065,12 +2115,18 @@ void MklLayoutRewritePass::CopyAttrsConv2DDepthwise(const Node* orig_node,
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "padding", &padding));
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "data_format", &data_format));
 
+  // Check if filter is a constant.
+  Node* filter_node = nullptr;
+  orig_node->input_node(1, &filter_node);
+  is_filter_const = filter_node->IsConstant();
+
   // Add attributes to new node.
   nb->Attr("T", T);
   nb->Attr("strides", strides);
   nb->Attr("dilations", dilations);
   nb->Attr("padding", padding);
   nb->Attr("data_format", data_format);
+  nb->Attr("is_filter_const", is_filter_const);
 }
 
 void MklLayoutRewritePass::CopyAttrsAddN(const Node* orig_node, NodeBuilder* nb,
@@ -2206,6 +2262,7 @@ void MklLayoutRewritePass::CopyAttrsQuantizedConv2D(const Node* orig_node,
   string padding;
   string data_format("NHWC");
   std::vector<int32> strides, dilations;
+  bool is_filter_const;
 
   // Get all attributes from old node.
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "Tinput", &Tinput));
@@ -2215,6 +2272,11 @@ void MklLayoutRewritePass::CopyAttrsQuantizedConv2D(const Node* orig_node,
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "strides", &strides));
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "dilations", &dilations));
 
+  // Check if filter is a constant.
+  Node* filter_node = nullptr;
+  orig_node->input_node(1, &filter_node);
+  is_filter_const = filter_node->IsConstant();
+
   // Add attributes to new node.
   nb->Attr("Tinput", Tinput);
   nb->Attr("Tfilter", Tfilter);
@@ -2224,7 +2286,9 @@ void MklLayoutRewritePass::CopyAttrsQuantizedConv2D(const Node* orig_node,
   nb->Attr("dilations", dilations);
   nb->Attr("T", out_type);  // added "T" for facilitating MklToTf conversion.
   nb->Attr("data_format", data_format);
-  // Requantization attr Tbias
+  nb->Attr("is_filter_const", is_filter_const);
+
+  // Requantization attr Tbias.
   DataType Tbias;
   Status bias_status = GetNodeAttr(orig_node->def(), "Tbias", &Tbias);
   if (bias_status.ToString() == "OK") nb->Attr("Tbias", Tbias);
@@ -2253,6 +2317,7 @@ void MklLayoutRewritePass::CopyAttrsReshape(const Node* orig_node,
   // Get all attributes from old node.
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "T", &T));
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "Tshape", &Tshape));
+
   // Add attributes to new node.
   nb->Attr("T", T);
   nb->Attr("Tshape", Tshape);
@@ -2266,6 +2331,7 @@ void MklLayoutRewritePass::CopyAttrsSlice(const Node* orig_node,
   // Get all attributes from old node.
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "T", &T));
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "Index", &Index));
+
   // Add attributes to new node.
   nb->Attr("T", T);
   nb->Attr("Index", Index);
@@ -2353,6 +2419,7 @@ void MklLayoutRewritePass::CopyAttrsFusedConv2D(const Node* orig_node,
   std::vector<int32> strides;
   std::vector<int32> dilations;
   std::vector<string> fused_ops;
+  bool is_filter_const;
 
   // Get all attributes from old node.
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "T", &T));
@@ -2364,6 +2431,11 @@ void MklLayoutRewritePass::CopyAttrsFusedConv2D(const Node* orig_node,
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "fused_ops", &fused_ops));
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "epsilon", &epsilon));
 
+  // Check if filter is a constant.
+  Node* filter_node = nullptr;
+  orig_node->input_node(1, &filter_node);
+  is_filter_const = filter_node->IsConstant();
+
   // Add attributes to new node.
   nb->Attr("T", T);
   nb->Attr("num_args", num_args);
@@ -2373,6 +2445,7 @@ void MklLayoutRewritePass::CopyAttrsFusedConv2D(const Node* orig_node,
   nb->Attr("dilations", dilations);
   nb->Attr("fused_ops", fused_ops);
   nb->Attr("epsilon", epsilon);
+  nb->Attr("is_filter_const", is_filter_const);
 }
 
 //////////////////////////////////////////////////////////////////////////
@@ -2505,7 +2578,7 @@ Status MklLayoutRewritePass::MergeConv2DWithBiasAdd(std::unique_ptr<Graph>* g,
   nb.Input(succ_in[1].first, succ_in[1].second);  // In2 of BiasAdd
 
   // Copy attributes from Conv2D to Conv2DWithBias.
-  CopyAttrsConv(const_cast<const Node*>(pred), &nb);
+  CopyAttrsConvCheckConstFilter(const_cast<const Node*>(pred), &nb);
 
   // Copy the device assigned to old node to new node.
   nb.Device(succ->def().device());
diff --git a/tensorflow/core/graph/mkl_layout_pass_test.cc b/tensorflow/core/graph/mkl_layout_pass_test.cc
index 6e73ed1b9f..02b8e23927 100644
--- a/tensorflow/core/graph/mkl_layout_pass_test.cc
+++ b/tensorflow/core/graph/mkl_layout_pass_test.cc
@@ -123,6 +123,29 @@ class MklLayoutPassTest : public ::testing::Test {
     return result;
   }
 
+  // Returns the attribute value only from the first node
+  template <typename T>
+  T DoMklLayoutOptimizationPassGetAttrVal(const string& attr,
+                                          const string& node_name) {
+    string before = CanonicalGraphString(&graph_);
+    LOG(ERROR) << "Before MKL layout rewrite pass: " << before;
+
+    std::unique_ptr<Graph>* ug = new std::unique_ptr<Graph>(&graph_);
+    RunMklLayoutRewritePass(ug);
+
+    string result = CanonicalGraphString(&graph_);
+    LOG(ERROR) << "After MKL layout rewrite pass:  " << result;
+
+    T attr_val{};
+    for (const Node* n : graph_.nodes()) {
+      if (IncludeNode(n) && n->type_string() == node_name) {
+        TF_CHECK_OK(GetNodeAttr(n->def(), attr, &attr_val));
+        break;
+      }
+    }
+    return attr_val;
+  }
+
   const string& OriginalGraph() const { return original_; }
 
   Graph graph_;
@@ -2561,6 +2584,316 @@ TEST_F(MklLayoutPassTest, PostRewriteFixUpPass) {
 
 /////////////////////////////////////////////////////////////////////
 
+/////////////////////////////////////////////////////////////////////
+// Unit tests related to filter caching.
+//
+// These tests check if the attribute `is_filter_const` is set to true
+// when filter is a constant and false otherwise for various operators
+// such as Conv2D, Conv2DWithBias, Conv3D etc.
+
+// Conv2D op where filter is a constant.
+TEST_F(MklLayoutPassTest, Conv2D_FilterCaching_Positive) {
+  InitGraph(
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Const' "  // Filter
+      " attr { key: 'dtype' value { type: DT_FLOAT } }"
+      " attr { key: 'value' value { "
+      "    tensor { dtype: DT_FLOAT tensor_shape { dim { size: 1 } } "
+      "    int_val: 0 } } } }"
+      "node { name: 'C' op: 'Conv2D'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'data_format'      value { s: 'NCHW' } }"
+      " attr { key: 'use_cudnn_on_gpu' value { b: false } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'padding'          value { s: 'SAME' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
+      " input: ['A', 'B']}"
+      "node { name: 'D' op: 'Zeta' attr { key: 'T' value { type: DT_FLOAT } }"
+      " input: ['B', 'C'] }");
+  EXPECT_EQ(DoMklLayoutOptimizationPassGetAttrVal<bool>("is_filter_const",
+                                                        "_MklConv2D"),
+            true);
+}
+
+// Conv2D op where filter is NOT a constant.
+TEST_F(MklLayoutPassTest, Conv2D_FilterCaching_Negative) {
+  InitGraph(
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Input'}"  // Filter
+      "node { name: 'C' op: 'Conv2D'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'data_format'      value { s: 'NCHW' } }"
+      " attr { key: 'use_cudnn_on_gpu' value { b: false } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'padding'          value { s: 'SAME' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
+      " input: ['A', 'B']}"
+      "node { name: 'D' op: 'Zeta' attr { key: 'T' value { type: DT_FLOAT } }"
+      " input: ['B', 'C'] }");
+  EXPECT_EQ(DoMklLayoutOptimizationPassGetAttrVal<bool>("is_filter_const",
+                                                        "_MklConv2D"),
+            false);
+}
+
+// Conv2D + BiasAdd fusion where filter is a constant.
+TEST_F(MklLayoutPassTest, Conv2DWithBias_FilterCaching_Positive) {
+  InitGraph(
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Const'"  // Filter
+      " attr { key: 'dtype' value { type: DT_FLOAT } }"
+      " attr { key: 'value' value { "
+      "    tensor { dtype: DT_FLOAT tensor_shape { dim { size: 1 } } "
+      "    int_val: 0 } } } }"
+      "node { name: 'C' op: 'Conv2D'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'data_format'      value { s: 'NCHW' } }"
+      " attr { key: 'use_cudnn_on_gpu' value { b: false } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'padding'          value { s: 'SAME' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
+      " input: ['A', 'B']}"
+      "node { name: 'D' op: 'Input'}"
+      "node { name: 'E' op: 'BiasAdd'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'data_format'      value { s: 'NCHW' } }"
+      " input: ['C', 'D'] }"
+      "node { name: 'Y' op: 'Input'}"
+      "node { name: 'Z' op: 'Zeta'"
+      " attr {key: 'T'                 value { type: DT_FLOAT } }"
+      " input: ['E', 'Y']}");
+  EXPECT_EQ(DoMklLayoutOptimizationPassGetAttrVal<bool>("is_filter_const",
+                                                        "_MklConv2DWithBias"),
+            true);
+}
+
+// Conv2D + BiasAdd fusion where filter is NOT a constant.
+TEST_F(MklLayoutPassTest, Conv2DWithBias_FilterCaching_Negative) {
+  InitGraph(
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Input'}"  // Filter
+      "node { name: 'C' op: 'Conv2D'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'data_format'      value { s: 'NCHW' } }"
+      " attr { key: 'use_cudnn_on_gpu' value { b: false } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'padding'          value { s: 'SAME' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
+      " input: ['A', 'B']}"
+      "node { name: 'D' op: 'Input'}"
+      "node { name: 'E' op: 'BiasAdd'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'data_format'      value { s: 'NCHW' } }"
+      " input: ['C', 'D'] }"
+      "node { name: 'Y' op: 'Input'}"
+      "node { name: 'Z' op: 'Zeta'"
+      " attr {key: 'T'                 value { type: DT_FLOAT } }"
+      " input: ['E', 'Y']}");
+  EXPECT_EQ(DoMklLayoutOptimizationPassGetAttrVal<bool>("is_filter_const",
+                                                        "_MklConv2DWithBias"),
+            false);
+}
+
+// Conv3D op where filter is a constant.
+TEST_F(MklLayoutPassTest, Conv3D_FilterCaching_Positive) {
+  InitGraph(
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Const' "  // Filter
+      " attr { key: 'dtype' value { type: DT_FLOAT } }"
+      " attr { key: 'value' value { "
+      "    tensor { dtype: DT_FLOAT tensor_shape { dim { size: 1 } } "
+      "    int_val: 0 } } } }"
+      "node { name: 'C' op: 'Conv3D'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'data_format'      value { s: 'NCDHW' } }"
+      " attr { key: 'use_cudnn_on_gpu' value { b: false } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1, "
+      "i:1} } }"
+      " attr { key: 'padding'          value { s: 'SAME' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1, "
+      "i:1} } }"
+      " input: ['A', 'B']}"
+      "node { name: 'D' op: 'Zeta' attr { key: 'T' value { type: DT_FLOAT } }"
+      " input: ['B', 'C'] }");
+  EXPECT_EQ(DoMklLayoutOptimizationPassGetAttrVal<bool>("is_filter_const",
+                                                        "_MklConv3D"),
+            true);
+}
+
+// Conv3D op where filter is NOT a constant.
+TEST_F(MklLayoutPassTest, Conv3D_FilterCaching_Negative) {
+  InitGraph(
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Input'}"  // Filter
+      "node { name: 'C' op: 'Conv3D'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'data_format'      value { s: 'NCDHW' } }"
+      " attr { key: 'use_cudnn_on_gpu' value { b: false } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1, "
+      "i:1} } }"
+      " attr { key: 'padding'          value { s: 'SAME' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1, "
+      "i:1} } }"
+      " input: ['A', 'B']}"
+      "node { name: 'D' op: 'Zeta' attr { key: 'T' value { type: DT_FLOAT } }"
+      " input: ['B', 'C'] }");
+  EXPECT_EQ(DoMklLayoutOptimizationPassGetAttrVal<bool>("is_filter_const",
+                                                        "_MklConv3D"),
+            false);
+}
+
+// Pad + Conv2D fusion where filter is a constant.
+TEST_F(MklLayoutPassTest, PadWithConv2D_FilterCaching_Positive) {
+  DCHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
+  InitGraph(
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Int32Input'}"
+      "node { name: 'C' op: 'Pad'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'Tpaddings'        value { type: DT_INT32 } }"
+      " input: ['A', 'B']}"
+      "node { name: 'D' op: 'Const'"  // Filter
+      " attr { key: 'dtype' value { type: DT_FLOAT } }"
+      " attr { key: 'value' value { "
+      "    tensor { dtype: DT_FLOAT tensor_shape { dim { size: 1 } } "
+      "    int_val: 0 } } } }"
+      "node { name: 'E' op: 'Conv2D'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'data_format'      value { s: 'NHWC' } }"
+      " attr { key: 'use_cudnn_on_gpu' value { b: false } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'padding'          value { s: 'VALID' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
+      " input: ['C', 'D'] }"
+      "node { name: 'Y' op: 'Input'}"
+      "node { name: 'Z' op: 'Zeta'"
+      " attr {key: 'T'                 value { type: DT_FLOAT } }"
+      " input: ['E', 'Y']}");
+  EXPECT_EQ(DoMklLayoutOptimizationPassGetAttrVal<bool>("is_filter_const",
+                                                        "_MklPadWithConv2D"),
+            true);
+}
+
+// Pad + Conv2D fusion where filter is NOT a constant.
+TEST_F(MklLayoutPassTest, PadWithConv2D_FilterCaching_Negative) {
+  DCHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
+  InitGraph(
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Int32Input'}"
+      "node { name: 'C' op: 'Pad'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'Tpaddings'        value { type: DT_INT32 } }"
+      " input: ['A', 'B']}"
+      "node { name: 'D' op: 'Input'}"  // Filter
+      "node { name: 'E' op: 'Conv2D'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'data_format'      value { s: 'NHWC' } }"
+      " attr { key: 'use_cudnn_on_gpu' value { b: false } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'padding'          value { s: 'VALID' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
+      " input: ['C', 'D'] }"
+      "node { name: 'Y' op: 'Input'}"
+      "node { name: 'Z' op: 'Zeta'"
+      " attr {key: 'T'                 value { type: DT_FLOAT } }"
+      " input: ['E', 'Y']}");
+  EXPECT_EQ(DoMklLayoutOptimizationPassGetAttrVal<bool>("is_filter_const",
+                                                        "_MklPadWithConv2D"),
+            false);
+}
+
+// _FusedConv2D + BiasAdd fusion where filter is a constant.
+TEST_F(MklLayoutPassTest, FusedConv2DWithBias_FilterCaching_Positive) {
+  InitGraph(
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Const'"  // Filter
+      " attr { key: 'dtype' value { type: DT_FLOAT } }"
+      " attr { key: 'value' value { "
+      "    tensor { dtype: DT_FLOAT tensor_shape { dim { size: 1 } } "
+      "    int_val: 0 } } } }"
+      "node { name: 'C' op: 'Input'}"
+      "node { name: 'D' op: '_FusedConv2D'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'num_args'         value { i: 1 } }"
+      " attr { key: 'data_format'      value { s: 'NCHW' } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'padding'          value { s: 'SAME' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'fused_ops'        value { list: {s: 'BiasAdd'} } }"
+      " attr { key: 'epsilon'          value { f: 0.001 }}"
+      " input: ['A', 'B', 'C']}"
+      "node { name: 'E' op: 'Zeta' attr { key: 'T' value { type: DT_FLOAT } }"
+      " input: ['D', 'C'] }");
+  EXPECT_EQ(DoMklLayoutOptimizationPassGetAttrVal<bool>("is_filter_const",
+                                                        "_MklFusedConv2D"),
+            true);
+}
+
+// _FusedConv2D + BiasAdd fusion where filter is NOT a constant.
+TEST_F(MklLayoutPassTest, FusedConv2DWithBias_FilterCaching_Negative) {
+  InitGraph(
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Input'}"  // Filter
+      "node { name: 'C' op: 'Input'}"
+      "node { name: 'D' op: '_FusedConv2D'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'num_args'         value { i: 1 } }"
+      " attr { key: 'data_format'      value { s: 'NCHW' } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'padding'          value { s: 'SAME' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'fused_ops'        value { list: {s: 'BiasAdd'} } }"
+      " attr { key: 'epsilon'          value { f: 0.001 }}"
+      " input: ['A', 'B', 'C']}"
+      "node { name: 'E' op: 'Zeta' attr { key: 'T' value { type: DT_FLOAT } }"
+      " input: ['D', 'C'] }");
+  EXPECT_EQ(DoMklLayoutOptimizationPassGetAttrVal<bool>("is_filter_const",
+                                                        "_MklFusedConv2D"),
+            false);
+}
+
+// Depthwise Conv2D op where filter is a constant.
+TEST_F(MklLayoutPassTest, DepthwiseConv2dNative_FilterCaching_Positive) {
+  InitGraph(
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Const'"  // Filter
+      " attr { key: 'dtype' value { type: DT_FLOAT } }"
+      " attr { key: 'value' value { "
+      "    tensor { dtype: DT_FLOAT tensor_shape { dim { size: 1 } } "
+      "    int_val: 0 } } } }"
+      "node { name: 'C' op: 'DepthwiseConv2dNative'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'data_format'      value { s: 'NCHW' } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'padding'          value { s: 'SAME' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
+      " input: ['A', 'B']}"
+      "node { name: 'D' op: 'Zeta' attr { key: 'T' value { type: DT_FLOAT } }"
+      " input: ['B', 'C'] }");
+  EXPECT_EQ(DoMklLayoutOptimizationPassGetAttrVal<bool>(
+                "is_filter_const", "_MklDepthwiseConv2dNative"),
+            true);
+}
+
+// Depthwise Conv2D op where filter is NOT a constant.
+TEST_F(MklLayoutPassTest, DepthwiseConv2dNative_FilterCaching_Negative) {
+  InitGraph(
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Input'}"  // Filter
+      "node { name: 'C' op: 'DepthwiseConv2dNative'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'data_format'      value { s: 'NCHW' } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'padding'          value { s: 'SAME' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
+      " input: ['A', 'B']}"
+      "node { name: 'D' op: 'Zeta' attr { key: 'T' value { type: DT_FLOAT } }"
+      " input: ['B', 'C'] }");
+  EXPECT_EQ(DoMklLayoutOptimizationPassGetAttrVal<bool>(
+                "is_filter_const", "_MklDepthwiseConv2dNative"),
+            false);
+}
+/////////////////////////////////////////////////////////////////////
+
 static void BM_MklLayoutRewritePass(int iters, int op_nodes) {
   testing::StopTiming();
   string s;
diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc
index 8c585fb48c..ae3ba50133 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_ops.cc
@@ -91,6 +91,9 @@ struct MklConvFwdParams {
         padding_left(padding_left),
         padding_right(padding_right) {}
 };
+
+typedef mkldnn::convolution_forward::primitive_desc ConvFwdPd;
+
 // With quantization, input, filter, and output can have different types
 // so we use differnt template parameter for each type
 template <typename T, typename Tinput, typename Tfilter, typename Tbias,
@@ -100,7 +103,7 @@ class MklConvFwdPrimitive : public MklPrimitive {
   explicit MklConvFwdPrimitive(const MklConvFwdParams& convFwdDims)
       : cpu_engine_(engine::cpu, 0) {
     context_.fwd_stream.reset(new stream(stream::kind::eager));
-    // create conv primitive
+    // Create conv primitive
     if (context_.conv_fwd == nullptr) {
       Setup(convFwdDims);
     }
@@ -125,7 +128,7 @@ class MklConvFwdPrimitive : public MklPrimitive {
         static_cast<void*>(const_cast<Toutput*>(dst_data)));
     context_.fwd_stream->submit(context_.fwd_primitives);
 
-    // after exec, set data handle back
+    // After exec, set data handle back
     context_.src_mem->set_data_handle(DummyData);
     context_.filter_mem->set_data_handle(DummyData);
     context_.bias_mem->set_data_handle(DummyData);
@@ -148,7 +151,7 @@ class MklConvFwdPrimitive : public MklPrimitive {
         static_cast<void*>(const_cast<Toutput*>(dst_data)));
     context_.fwd_stream->submit(context_.fwd_primitives);
 
-    // after execution, set data handle back
+    // After execution, set data handle back
     context_.src_mem->set_data_handle(DummyData);
     context_.filter_mem->set_data_handle(DummyData);
     context_.dst_mem->set_data_handle(DummyData);
@@ -158,15 +161,14 @@ class MklConvFwdPrimitive : public MklPrimitive {
 
   memory::format GetFilterMemoryFormat() const { return context_.filter_fmt; }
 
-  std::shared_ptr<mkldnn::convolution_forward::primitive_desc>
-  GetPrimitiveDesc() const {
+  std::shared_ptr<ConvFwdPd> GetPrimitiveDesc() const {
     return context_.fwd_pd;
   }
 
  private:
   // Primitive reuse context for Conv2D Fwd op
   struct ConvFwdContext {
-    // expected memory format for this primitive instance
+    // Expected memory format for this primitive instance
     memory::format src_fmt;
     memory::format filter_fmt;
 
@@ -176,17 +178,17 @@ class MklConvFwdPrimitive : public MklPrimitive {
     std::shared_ptr<mkldnn::memory> bias_mem;
     std::shared_ptr<mkldnn::memory> dst_mem;
 
-    // desc & prmitive desc
+    // Desc & prmitive desc
     std::shared_ptr<mkldnn::convolution_forward::desc> fwd_desc;
 
-    // memory desc
+    // Memory desc
     std::shared_ptr<mkldnn::memory::desc> src_md;
     std::shared_ptr<mkldnn::memory::desc> filter_md;
     std::shared_ptr<mkldnn::memory::desc> bias_md;
     std::shared_ptr<mkldnn::memory::desc> dst_md;
 
-    // convolution primitive
-    std::shared_ptr<mkldnn::convolution_forward::primitive_desc> fwd_pd;
+    // Convolution primitive
+    std::shared_ptr<ConvFwdPd> fwd_pd;
     std::shared_ptr<mkldnn::primitive> conv_fwd;
 
     std::shared_ptr<mkldnn::stream> fwd_stream;
@@ -209,7 +211,7 @@ class MklConvFwdPrimitive : public MklPrimitive {
   };
 
   void Setup(const MklConvFwdParams& convFwdDims) {
-    // create memory descriptors for convolution data w/ no specified format
+    // Create memory descriptors for convolution data w/ no specified format
     context_.src_md.reset(new memory::desc(
         {convFwdDims.src_dims}, MklDnnType<Tinput>(), memory::format::any));
 
@@ -223,7 +225,7 @@ class MklConvFwdPrimitive : public MklPrimitive {
       context_.bias_md.reset(new memory::desc(
           {convFwdDims.bias_dims}, MklDnnType<Tbias>(), memory::format::any));
 
-    // create a convolution
+    // Create a convolution
     if (!convFwdDims.bias_dims.empty()) {
       context_.fwd_desc.reset(new convolution_forward::desc(
           prop_kind::forward, convolution_direct, *context_.src_md,
@@ -238,8 +240,7 @@ class MklConvFwdPrimitive : public MklPrimitive {
           convFwdDims.padding_right, padding_kind::zero));
     }
 
-    context_.fwd_pd.reset(new convolution_forward::primitive_desc(
-        *context_.fwd_desc, cpu_engine_));
+    context_.fwd_pd.reset(new ConvFwdPd(*context_.fwd_desc, cpu_engine_));
 
     // Check if there is any fusions as post-ops
     auto const& post_op_params = convFwdDims.post_op_params;
@@ -270,21 +271,20 @@ class MklConvFwdPrimitive : public MklPrimitive {
         }
       }
       post_ops_attr.set_post_ops(post_ops);
-      context_.fwd_pd.reset(new convolution_forward::primitive_desc(
-          *context_.fwd_desc, post_ops_attr, cpu_engine_));
+      context_.fwd_pd.reset(
+          new ConvFwdPd(*context_.fwd_desc, post_ops_attr, cpu_engine_));
     } else {
-      context_.fwd_pd.reset(new convolution_forward::primitive_desc(
-          *context_.fwd_desc, cpu_engine_));
+      context_.fwd_pd.reset(new ConvFwdPd(*context_.fwd_desc, cpu_engine_));
     }
 
-    // store the expected memory format
+    // Store the expected memory format
     context_.src_fmt = static_cast<mkldnn::memory::format>(
         context_.fwd_pd.get()->src_primitive_desc().desc().data.format);
 
     context_.filter_fmt = static_cast<mkldnn::memory::format>(
         context_.fwd_pd.get()->weights_primitive_desc().desc().data.format);
 
-    // create memory primitive based on dummy data
+    // Create memory primitive based on dummy data
     context_.src_mem.reset(
         new memory(context_.fwd_pd.get()->src_primitive_desc(), DummyData));
     context_.filter_mem.reset(
@@ -292,7 +292,7 @@ class MklConvFwdPrimitive : public MklPrimitive {
     context_.dst_mem.reset(
         new memory(context_.fwd_pd.get()->dst_primitive_desc(), DummyData));
 
-    // create convolution primitive and add it to net
+    // Create convolution primitive and add it to net
     if (!convFwdDims.bias_dims.empty()) {
       context_.bias_mem.reset(new memory(
           {{{convFwdDims.bias_dims}, MklDnnType<T>(), memory::format::x},
@@ -323,11 +323,12 @@ class MklConvFwdPrimitiveFactory : public MklPrimitiveFactory<T> {
       const MklConvFwdParams& convFwdDims, bool do_not_cache) {
     MklConvFwdPrimitive<T, Tinput, Tfilter, Tbias, Toutput>* conv_fwd = nullptr;
 
-    if (do_not_cache) { /* Always create new primitive */
+    if (do_not_cache) {
+      // Always create a new primitive
       conv_fwd = new MklConvFwdPrimitive<T, Tinput, Tfilter, Tbias, Toutput>(
           convFwdDims);
     } else {
-      // try to find a suitable one in pool
+      // Try to find a suitable one in pool
       conv_fwd = dynamic_cast<
           MklConvFwdPrimitive<T, Tinput, Tfilter, Tbias, Toutput>*>(
           MklConvFwdPrimitiveFactory<T, Tinput, Tfilter, Tbias,
@@ -465,19 +466,18 @@ class MklConvOp : public OpKernel {
                                         filter.shape().DebugString()));
 
     for (int i = 0; i < 3; i++) {
-      OP_REQUIRES(
-          context,
-          FastBoundsCheck(filter.dim_size(i), std::numeric_limits<int>::max()),
-          errors::InvalidArgument("filter too large"));
+      OP_REQUIRES(context, FastBoundsCheck(filter.dim_size(i),
+                                           std::numeric_limits<int>::max()),
+                  errors::InvalidArgument("filter too large"));
     }
 
     const int64 input_depth =
         input_in_mkl_format ? GetMklTensorDim(mkl_context.input_shape, 'C')
                             : GetTensorDim(input, data_format_, 'C');
-    OP_REQUIRES(context, input_depth == filter.dim_size(2),
-                errors::InvalidArgument(
-                    "input and filter must have the same depth: ", input_depth,
-                    " vs ", filter.dim_size(2)));
+    OP_REQUIRES(
+        context, input_depth == filter.dim_size(2),
+        errors::InvalidArgument("input and filter must have the same depth: ",
+                                input_depth, " vs ", filter.dim_size(2)));
     // The last dimension for filter is out_depth.
     const int out_depth = static_cast<int>(filter.dim_size(3));
 
@@ -486,10 +486,9 @@ class MklConvOp : public OpKernel {
     const int64 input_rows_raw =
         input_in_mkl_format ? GetMklTensorDim(mkl_context.input_shape, 'H')
                             : GetTensorDim(input, data_format_, 'H');
-    OP_REQUIRES(
-        context,
-        FastBoundsCheck(input_rows_raw, std::numeric_limits<int>::max()),
-        errors::InvalidArgument("Input rows too large"));
+    OP_REQUIRES(context, FastBoundsCheck(input_rows_raw,
+                                         std::numeric_limits<int>::max()),
+                errors::InvalidArgument("Input rows too large"));
     const int input_rows = static_cast<int>(input_rows_raw);
     const int filter_rows = static_cast<int>(filter.dim_size(0));
 
@@ -498,10 +497,9 @@ class MklConvOp : public OpKernel {
     const int64 input_cols_raw =
         input_in_mkl_format ? GetMklTensorDim(mkl_context.input_shape, 'W')
                             : GetTensorDim(input, data_format_, 'W');
-    OP_REQUIRES(
-        context,
-        FastBoundsCheck(input_cols_raw, std::numeric_limits<int>::max()),
-        errors::InvalidArgument("Input cols too large"));
+    OP_REQUIRES(context, FastBoundsCheck(input_cols_raw,
+                                         std::numeric_limits<int>::max()),
+                errors::InvalidArgument("Input cols too large"));
     const int input_cols = static_cast<int>(input_cols_raw);
     const int filter_cols = static_cast<int>(filter.dim_size(1));
 
@@ -509,10 +507,9 @@ class MklConvOp : public OpKernel {
     const int64 input_batch_raw =
         input_in_mkl_format ? GetMklTensorDim(mkl_context.input_shape, 'N')
                             : GetTensorDim(input, data_format_, 'N');
-    OP_REQUIRES(
-        context,
-        FastBoundsCheck(input_batch_raw, std::numeric_limits<int>::max()),
-        errors::InvalidArgument("batch is too large"));
+    OP_REQUIRES(context, FastBoundsCheck(input_batch_raw,
+                                         std::numeric_limits<int>::max()),
+                errors::InvalidArgument("batch is too large"));
     const int batch = static_cast<int>(input_batch_raw);
 
     // For now we take the stride from the second and third dimensions only (we
@@ -874,6 +871,9 @@ class MklConvOp : public OpKernel {
         errors::InvalidArgument("Current implementation does not yet support "
                                 "strides in the batch and depth dimensions."));
     OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
+    is_filter_const_ = false;
+    OP_REQUIRES_OK(context,
+                   context->GetAttr("is_filter_const", &is_filter_const_));
 
     if (strides_.size() == 4) {
       OP_REQUIRES(context, dilations_.size() == 4,
@@ -894,17 +894,15 @@ class MklConvOp : public OpKernel {
       OP_REQUIRES(context, dilations_.size() == 5,
                   errors::InvalidArgument("Dilation rates field must "
                                           "specify 5 dimensions"));
-      OP_REQUIRES(context,
-                  (GetTensorDim(dilations_, data_format_, 'N') == 1 &&
-                   GetTensorDim(dilations_, data_format_, 'C') == 1),
+      OP_REQUIRES(context, (GetTensorDim(dilations_, data_format_, 'N') == 1 &&
+                            GetTensorDim(dilations_, data_format_, 'C') == 1),
                   errors::InvalidArgument(
                       "Current implementation does not yet support "
                       "dilations rates in the batch and depth dimensions."));
       OP_REQUIRES(
-          context,
-          (GetTensorDim(dilations_, data_format_, '0') > 0 &&
-           GetTensorDim(dilations_, data_format_, '1') > 0 &&
-           GetTensorDim(dilations_, data_format_, '2') > 0),
+          context, (GetTensorDim(dilations_, data_format_, '0') > 0 &&
+                    GetTensorDim(dilations_, data_format_, '1') > 0 &&
+                    GetTensorDim(dilations_, data_format_, '2') > 0),
           errors::InvalidArgument("Dilated rates should be larger than 0."));
     }
   }
@@ -915,6 +913,10 @@ class MklConvOp : public OpKernel {
       const Tensor& src_tensor = MklGetInput(context, kInputIndex_Src);
       const Tensor& filter_tensor = MklGetInput(context, kInputIndex_Filter);
 
+      // Data from persistent (cached) filter tensor
+      const Tensor& cached_filter_data_tensor =
+          *cached_filter_data_ptensor_.AccessTensor(context);
+
       MklDnnShape src_mkl_shape, filter_mkl_shape;
       GetMklShape(context, kInputIndex_Src, &src_mkl_shape);
       GetMklShape(context, kInputIndex_Filter, &filter_mkl_shape);
@@ -956,11 +958,9 @@ class MklConvOp : public OpKernel {
         AllocateOutputSetMklShape(context, kOutputIndex_Dst, &dst_tensor,
                                   src_tf_shape, dst_mkl_shape);
 
-        // MklConv2D/3D also outputs converted filter
-        // as 2nd output of Conv2D/3D.
+        // MklConv2D/3D also outputs converted filter as 2nd output.
         filter_mkl_shape.SetMklTensor(false);
         Tensor* output_filter_tensor = nullptr;
-        // MklConv2D also outputs converted filter as 2nd output.
         if (typeid(Tinput) == typeid(float) &&
             typeid(Tfilter) == typeid(float) &&
             typeid(Toutput) == typeid(float)) {
@@ -974,6 +974,12 @@ class MklConvOp : public OpKernel {
 
       bool is_conv2d = (strides_.size() == 4);
 
+      if (!is_conv2d) {
+        OP_REQUIRES(
+            context, !pad_enabled,
+            errors::InvalidArgument("Pad + Conv fusion only works for 2D"));
+      }
+
       // TODO 3-D support for Depthwise is not there
       if (is_depthwise) {
         OP_REQUIRES(context, is_conv2d,
@@ -993,10 +999,11 @@ class MklConvOp : public OpKernel {
       auto tf_fmt = is_conv2d ? TFDataFormatToMklDnnDataFormat(data_format_)
                               : TFDataFormatToMklDnn3DDataFormat(data_format_);
 
-      // If input is in MKL layout, then simply grab input layout; otherwise,
-      // construct input Tf layout. For TF layout, although input shape
-      // (src_dims) required is in MKL-DNN order, the layout is Tensorflow's
-      // layout depending on data format:
+      // If input is in MKL layout, then simply grab the layout; otherwise,
+      // construct TF layout for input.
+      // For constructing TF layout for input, although input shape (src_dims)
+      // is required to be in MKL-DNN order, the input layout is actually in
+      // TF layout depending on the data format:
       //     Conv2D: NHWC or NCHW
       //     Conv3D: NDHWC or NCDHW
       auto src_md = src_mkl_shape.IsMklTensor()
@@ -1005,8 +1012,8 @@ class MklConvOp : public OpKernel {
       src.SetUsrMem(src_md, &src_tensor);
 
       // Although filter shape (filter_dims) required is in MKL-DNN order,
-      // the layout is Tensorflow's layout (HWIO) and (HWIGO)for depthwise/group
-      // convolutions
+      // the layout is Tensorflow's layout (HWIO) and (HWIGO) for
+      // depthwise/group convolutions.
 
       auto filter_format = is_conv2d ? (is_depthwise ? memory::format::hwigo
                                                      : memory::format::hwio)
@@ -1018,54 +1025,42 @@ class MklConvOp : public OpKernel {
               ? filter_mkl_shape.GetMklLayout()
               : memory::desc(filter_dims, MklDnnType<Tfilter>(), filter_format);
       filter.SetUsrMem(filter_md, &filter_tensor);
-      // MKLDNN dilation starts from 0.
-      for (int i = 0; i < dilations.size(); i++) dilations[i] -= 1;
 
-      // In some cases, primitve descriptor includes potentialy large buffers,
-      // we don't cache those primitves if the env variable
-      // TF_MKL_OPTIMIZE_PRIMITIVE_MEMUSE is true. MKL DNN allocates buffers
-      // in the following cases
+      // MKLDNN dilations start from 0.
+      for (int i = 0; i < dilations.size(); i++) --dilations[i];
+
+      // In some cases, primitive descriptor could potentially contain
+      // large buffers. As a result, we don't cache these primitives if the
+      // environment variable `TF_MKL_OPTIMIZE_PRIMITIVE_MEMUSE` is set to True.
+      // MKL-DNN allocates buffers in the following cases:
       //   1. Legacy CPU without AVX512/AVX2, or
-      //   2. 1x1 convolution with stride != 1
+      //   2. 1x1 convolution with strides != 1
       bool do_not_cache =
           MklPrimitiveFactory<Tinput>::IsPrimitiveMemOptEnabled() &&
           (src_dims[MklDnnDims::Dim_N] > kSmallBatchSize) &&
           (MklPrimitiveFactory<Tinput>::IsLegacyPlatform() ||
            IsConv1x1StrideNot1(filter_dims, strides));
 
-      // get a conv2d fwd from primitive pool
+      // Get a conv2d fwd from primitive pool
       MklConvFwdPrimitive<float, Tinput, Tfilter, Tbias, Ttemp_output>*
           conv_fwd = nullptr;
+      memory::dims bias_dims = {};
       if (fuse_biasadd_) {
-        memory::dims bias_dims = {};
         conv_utl.GetBiasSizeInMklOrder(kInputIndex_Bias, &bias_dims);
-        MklConvFwdParams convFwdDims(src_dims, filter_dims, bias_dims,
-                                     dst_dims_mkl_order, strides, dilations,
-                                     padding_left, padding_right);
-
-        // TODO(mdfaijul):  Extend the basic parameters for data types and
-        // fusions
-        this->ExtendConvFwdParams(context, convFwdDims);
-
-        conv_fwd = MklConvFwdPrimitiveFactory<float, Tinput, Tfilter, Tbias,
-                                              Ttemp_output>::Get(convFwdDims,
-                                                                 do_not_cache);
-      } else {
-        MklConvFwdParams convFwdDims(src_dims, filter_dims, NONE_DIMS,
-                                     dst_dims_mkl_order, strides, dilations,
-                                     padding_left, padding_right);
+      }
+      MklConvFwdParams convFwdDims(
+          src_dims, filter_dims, fuse_biasadd_ ? bias_dims : NONE_DIMS,
+          dst_dims_mkl_order, strides, dilations, padding_left, padding_right);
 
-        // Extend the basic parameters for data types and fusions
-        this->ExtendConvFwdParams(context, convFwdDims);
+      // TODO(mdfaijul): Extend the basic parameters for data types and fusions
+      this->ExtendConvFwdParams(context, convFwdDims);
 
-        conv_fwd = MklConvFwdPrimitiveFactory<float, Tinput, Tfilter, Tbias,
-                                              Ttemp_output>::Get(convFwdDims,
-                                                                 do_not_cache);
-      }
+      conv_fwd = MklConvFwdPrimitiveFactory<float, Tinput, Tfilter, Tbias,
+                                            Ttemp_output>::Get(convFwdDims,
+                                                               do_not_cache);
 
-      // allocate output tensors output_tensor and filter_out_tensor
-      std::shared_ptr<mkldnn::convolution_forward::primitive_desc> conv_fwd_pd =
-          conv_fwd->GetPrimitiveDesc();
+      // Allocate output tensors `output_tensor` and `filter_out_tensor`
+      std::shared_ptr<ConvFwdPd> conv_fwd_pd = conv_fwd->GetPrimitiveDesc();
       AllocateOutputTensor(context, *conv_fwd_pd, dst_dims_mkl_order, tf_fmt,
                            &dst_tensor);
       Tensor* filter_out_tensor = nullptr;
@@ -1079,9 +1074,10 @@ class MklConvOp : public OpKernel {
       Ttemp_output* dst_data =
           reinterpret_cast<Ttemp_output*>(dst_tensor->flat<Toutput>().data());
 
-      // check whether src/filter need reorder
+      // Check whether src and filter needs to be reordered
       Tinput* src_data = nullptr;
       if (src_md.data.format != conv_fwd->GetSrcMemoryFormat()) {
+        // Reorder src
         src.SetUsrMem(src_md, &src_tensor);
         src.CheckReorderToOpMem(conv_fwd_pd.get()->src_primitive_desc());
         src_data = static_cast<Tinput*>(src.GetOpMem().get_data_handle());
@@ -1089,25 +1085,43 @@ class MklConvOp : public OpKernel {
         src_data = static_cast<Tinput*>(
             const_cast<Tinput*>(src_tensor.flat<Tinput>().data()));
       }
+
       Tfilter* filter_data = nullptr;
       if (filter_md.data.format != conv_fwd->GetFilterMemoryFormat()) {
-        filter.SetUsrMem(filter_md, &filter_tensor);
-        if (filter_out_tensor == nullptr) {
-          filter.CheckReorderToOpMem(
-              conv_fwd_pd.get()->weights_primitive_desc());
-        } else {
-          filter.CheckReorderToOpMem(
-              conv_fwd_pd.get()->weights_primitive_desc(),
-              filter.GetTensorBuffer(filter_out_tensor));
+        bool is_filter_cached = false;
+        // If filter is a constant, we can avoid the conversion of filter from
+        // Tensorflow format to MKL format by caching the filter when it is
+        // converted for the first time. This cached filter can then be reused
+        // in subsequent iterations.
+        if (is_filter_const_) {
+          if (IsFilterCacheEmpty(context)) {
+            // Cache filter if it is not already cached.
+            CacheFilter(context, conv_fwd_pd, filter_data, filter_tensor,
+                        filter, filter_md);
+          }
+          filter_data =
+              GetCachedFilter(context, conv_fwd->GetFilterMemoryFormat());
+          is_filter_cached = (filter_data != nullptr);
+        }
+        if (!is_filter_cached) {
+          filter.SetUsrMem(filter_md, &filter_tensor);
+          if (filter_out_tensor == nullptr) {
+            filter.CheckReorderToOpMem(
+                conv_fwd_pd.get()->weights_primitive_desc());
+          } else {
+            filter.CheckReorderToOpMem(
+                conv_fwd_pd.get()->weights_primitive_desc(),
+                filter.GetTensorBuffer(filter_out_tensor));
+          }
+          filter_data =
+              static_cast<Tfilter*>(filter.GetOpMem().get_data_handle());
         }
-        filter_data =
-            static_cast<Tfilter*>(filter.GetOpMem().get_data_handle());
       } else {
         filter_data = static_cast<Tfilter*>(
             const_cast<Tfilter*>(filter_tensor.flat<Tfilter>().data()));
       }
 
-      // execute convolution
+      // Execute convolution
       if (fuse_biasadd_) {
         const Tensor& bias_tensor = MklGetInput(context, kInputIndex_Bias);
         Tbias* bias_data =
@@ -1117,7 +1131,7 @@ class MklConvOp : public OpKernel {
         conv_fwd->Execute(src_data, filter_data, dst_data);
       }
 
-      // delete primitive since it is not cached.
+      // Delete primitive since it is not cached.
       if (do_not_cache) delete conv_fwd;
     } catch (mkldnn::error& e) {
       string error_msg = tensorflow::strings::StrCat(
@@ -1135,18 +1149,22 @@ class MklConvOp : public OpKernel {
     OP_REQUIRES(context, paddings_tf.dims() == 2,
                 errors::InvalidArgument("paddings must be 2-dimensional: ",
                                         paddings_tf.shape().DebugString()));
-    Tpadding* paddings = nullptr;
-    // To get individual pad, need to flatten the tensor
-    paddings = static_cast<Tpadding*>(
+
+    // Flatten tensor to get individual paddings.
+    Tpadding* paddings = static_cast<Tpadding*>(
         const_cast<Tpadding*>(paddings_tf.flat<Tpadding>().data()));
-    // For NHWC format:
-    // paddings[0], paddings[1], paddings[6], paddings[7] should be zero
-    // if the paddings_tf is [ [0, 0] [1,2] [3,4] [0,0] ]
-    // paddings = {0, 0, 1, 2, 3, 4, 0, 0} ; flat method is row major
-    // then, values are: top = 1, bottom =2, left=3, right=4
-    // For NCHW format:
-    // paddings[0], paddings[1], paddings[2], paddings[3] should be zero
-    // similar explanation as NHWC format will apply.
+
+    // If the data format is NHWC, indices 0, 1, 6 and 7 of paddings(_tf)
+    // will be zero.
+    // Example:
+    // paddings_tf = [ [0, 0] [1, 2] [3, 4] [0, 0] ],
+    // flat method = row-major, then:
+    // paddings = {0, 0, 1, 2, 3, 4, 0, 0}.
+    // Hence, the values are: top = 1, bottom = 2, left = 3, right = 4.
+    //
+    // Similarly, if the data format is NCHW, indices 0, 1, 2 and 3 of
+    // paddings(_tf) will be zero.
+    // i.e. for the above example, paddings = {0, 0, 0, 0, 1, 2, 3, 4}.
     int64 pad_top, pad_left;
     int64 pad_bottom, pad_right;
     string data_format = ToString(data_format_);
@@ -1161,6 +1179,7 @@ class MklConvOp : public OpKernel {
       pad_left = paddings[6];
       pad_right = paddings[7];
     }
+
     // Create padding arrays for MKL DNN convolutions.
     // MKL-DNN uses asymetric padding.
     padding_left = {static_cast<int>(pad_top), static_cast<int>(pad_left)};
@@ -1183,30 +1202,26 @@ class MklConvOp : public OpKernel {
     params.dtypes.append(typeid(Toutput).name());
 
     // Add fusions as post ops
-    // Note: Fusion of BiasAdd is handled directly inside MklConvOp by
-    // checking fuse_biasadd_ flag.
+    // NOTE: Fusion of BiasAdd is handled directly inside MklConvOp by
+    // checking `fuse_biasadd_` flag.
     if (fuse_relu_) params.post_op_params.push_back({"relu", {1.0, 0.0, 0.0}});
   }
 
-  virtual Tbias* GetBiasHandle(
-      OpKernelContext* context,
-      std::shared_ptr<mkldnn::convolution_forward::primitive_desc>&
-          conv2d_fwd_pd,
-      const Tensor& bias_tensor) {
+  virtual Tbias* GetBiasHandle(OpKernelContext* context,
+                               std::shared_ptr<ConvFwdPd>& conv2d_fwd_pd,
+                               const Tensor& bias_tensor) {
     if (fuse_biasadd_) {
       return static_cast<Tbias*>(
           const_cast<Tbias*>(bias_tensor.flat<Tbias>().data()));
-    } else {
-      return nullptr;
     }
+    return nullptr;
   }
 
-  // Allocate output tensor.
-  virtual void AllocateOutputTensor(
-      OpKernelContext* context,
-      const convolution_forward::primitive_desc& conv_prim_desc,
-      const memory::dims& output_dims_mkl_order,
-      memory::format output_tf_format, Tensor** output_tensor) {
+  virtual void AllocateOutputTensor(OpKernelContext* context,
+                                    const ConvFwdPd& conv_prim_desc,
+                                    const memory::dims& output_dims_mkl_order,
+                                    memory::format output_tf_format,
+                                    Tensor** output_tensor) {
     CHECK_NOTNULL(output_tensor);
     auto dst_pd = conv_prim_desc.dst_primitive_desc();
 
@@ -1237,8 +1252,12 @@ class MklConvOp : public OpKernel {
  private:
   std::vector<int32> strides_;
   std::vector<int32> dilations_;
+  bool is_filter_const_;
+  mutex mu_;
   Padding padding_;
   TensorFormat data_format_;
+  PersistentTensor cached_filter_data_ptensor_ GUARDED_BY(mu_);
+  PersistentTensor cached_filter_md_ptensor_ GUARDED_BY(mu_);
 
   // Initialize to values the template is instantiated with
   bool fuse_biasadd_ = bias_enabled;
@@ -1249,11 +1268,35 @@ class MklConvOp : public OpKernel {
   const int kOutputIndex_Dst = 0, kOutputIndex_Filter = 1;
   const int kDilationH = 0, kDilationW = 1;
 
-  // Allocate filter output tensor.
-  void AllocateFilterOutputTensor(
-      OpKernelContext* context,
-      const convolution_forward::primitive_desc& conv_prim_desc,
-      const memory::dims& filter_dims_tf_order, Tensor** filter_tensor) {
+  // Allocate persistent tensors for cached filter data and
+  // cached filter memory descriptor (data format)
+  void AllocatePersistTensor(OpKernelContext* context,
+                             const ConvFwdPd& conv_prim_desc,
+                             Tensor** filter_tensor) {
+    CHECK_NOTNULL(filter_tensor);
+    TensorShape filter_tf_shape;
+    filter_tf_shape.AddDim(
+        (conv_prim_desc.weights_primitive_desc().get_size() / sizeof(Tfilter)));
+    OP_REQUIRES_OK(context, context->allocate_persistent(
+                                DataTypeToEnum<Tfilter>::value, filter_tf_shape,
+                                &cached_filter_data_ptensor_, filter_tensor));
+
+    Tensor* second_tensor = nullptr;
+    TensorShape filter_mkl_format;
+    filter_mkl_format.AddDim(
+        sizeof(conv_prim_desc.weights_primitive_desc().desc().data.format) /
+        sizeof(DT_INT32));
+    OP_REQUIRES_OK(context, context->allocate_persistent(
+                                DT_INT32, filter_mkl_format,
+                                &cached_filter_md_ptensor_, &second_tensor));
+    second_tensor->scalar<int32>()() =
+        conv_prim_desc.weights_primitive_desc().desc().data.format;
+  }
+
+  void AllocateFilterOutputTensor(OpKernelContext* context,
+                                  const ConvFwdPd& conv_prim_desc,
+                                  const memory::dims& filter_dims_tf_order,
+                                  Tensor** filter_tensor) {
     CHECK_NOTNULL(filter_tensor);
     auto filter_pd = conv_prim_desc.weights_primitive_desc();
 
@@ -1276,12 +1319,14 @@ class MklConvOp : public OpKernel {
     AllocateOutputSetMklShape(context, kOutputIndex_Filter, filter_tensor,
                               filter_tf_shape, filter_mkl_shape);
   }
+
   // Prepare and execute net - checks for input and output reorders.
-  void PrepareAndExecuteNet(
-      const convolution_forward::primitive_desc& conv_prim_desc,
-      MklDnnData<Tinput>* src, MklDnnData<Tfilter>* filter,
-      MklDnnData<Tbias>* bias, MklDnnData<Toutput>* output,
-      Tensor* filter_out_tensor) {
+  void PrepareAndExecuteNet(const ConvFwdPd& conv_prim_desc,
+                            MklDnnData<Tinput>* src,
+                            MklDnnData<Tfilter>* filter,
+                            MklDnnData<Tbias>* bias,
+                            MklDnnData<Toutput>* output,
+                            Tensor* filter_out_tensor) {
     CHECK_NOTNULL(filter_out_tensor);
 
     // Create reorders between user layout and MKL layout if it is needed and
@@ -1310,6 +1355,67 @@ class MklConvOp : public OpKernel {
 
     stream(stream::kind::eager).submit(net).wait();
   }
+
+  inline bool IsFilterCacheEmpty(OpKernelContext* context) {
+    tf_shared_lock lock(mu_);
+    SHARED_LOCKS_REQUIRED(mu_) {
+      const Tensor& cached_filter_data_tensor =
+          *cached_filter_data_ptensor_.AccessTensor(context);
+      return (cached_filter_data_tensor.NumElements() == 0);
+    }
+  }
+
+  // Cache the converted filter in a persistent tensor.
+  // Only one thread can execute this method at any given time.
+  void CacheFilter(OpKernelContext* context,
+                   const std::shared_ptr<ConvFwdPd>& conv_fwd_pd,
+                   Tfilter* filter_data, const Tensor& filter_tensor,
+                   MklDnnData<Tfilter>& filter, const memory::desc& filter_md) {
+    mutex_lock lock(mu_);
+    EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+      const Tensor& cached_filter_data_tensor =
+          *cached_filter_data_ptensor_.AccessTensor(context);
+
+      // If filter is already cached, there's nothing to do.
+      if (cached_filter_data_tensor.NumElements() > 0) {
+        return;
+      }
+
+      // Otherwise, cache filter
+      filter.SetUsrMem(filter_md, &filter_tensor);
+      filter.CheckReorderToOpMem(conv_fwd_pd.get()->weights_primitive_desc());
+      filter_data = static_cast<Tfilter*>(filter.GetOpMem().get_data_handle());
+
+      Tensor* filter_tensor_ptr = nullptr;
+      AllocatePersistTensor(context, *conv_fwd_pd, &filter_tensor_ptr);
+      void* cached_filter_data = filter.GetTensorBuffer(filter_tensor_ptr);
+      size_t cached_filter_data_size =
+          filter.GetOpMem().get_primitive_desc().get_size();
+      memcpy(cached_filter_data, filter_data, cached_filter_data_size);
+    }
+  }
+
+  Tfilter* GetCachedFilter(OpKernelContext* context,
+                           const memory::format& filter_mf) {
+    tf_shared_lock lock(mu_);
+    SHARED_LOCKS_REQUIRED(mu_) {
+      const Tensor& cached_filter_data =
+          *cached_filter_data_ptensor_.AccessTensor(context);
+      const Tensor& cached_filter_md =
+          *cached_filter_md_ptensor_.AccessTensor(context);
+
+      // Check if the memory descriptor of the cached weights is same as
+      // filter_mf. If so, we can used the cached weights; otherwise
+      // return NULL.
+      // TODO (bhavanis): Do we need to cast filter_mf before the check?
+      if (cached_filter_md.scalar<int32>().size() &&
+          cached_filter_md.scalar<int32>()() == filter_mf) {
+        return static_cast<Tfilter*>(
+            const_cast<Tfilter*>(cached_filter_data.flat<Tfilter>().data()));
+      }
+      return nullptr;
+    }
+  }
 };
 
 // Base class for fused convolution forward operations
@@ -1356,7 +1462,7 @@ class MklFusedConvOp
   virtual ~MklFusedConvOp() {}
 };
 
-// We create new class for each verison of Quantized Convolution and inherit
+// We create new class for each version of Quantized Convolution and inherit
 // from the FP32 version of the base class
 template <typename Device, typename Tbias, typename Toutput,
           typename Ttemp_output, bool bias_enabled>
@@ -1378,7 +1484,13 @@ class MklQuantizedConv2DOp
 
   explicit MklQuantizedConv2DOp(OpKernelConstruction* context)
       : MklConvOp<Device, quint8, qint8, Tbias, Toutput, Ttemp_output, int32,
-                  bias_enabled, false, false>(context) {}
+                  bias_enabled, false, false>(context) {
+    bool is_filter_const;
+    OP_REQUIRES_OK(context,
+                   context->GetAttr("is_filter_const", &is_filter_const));
+    OP_REQUIRES(context, is_filter_const == true,
+                errors::InvalidArgument("Filter must be a constant"));
+  }
 
   void Compute(OpKernelContext* context) override {
     // Compute int32 output tensor
@@ -1402,9 +1514,9 @@ class MklQuantizedConv2DOp
     float max_output_value;
     if (std::is_same<Toutput, quint8>::value ||
         std::is_same<Toutput, qint8>::value) {
-      // This is the case the convolution and requantization are fused.
-      // min_freezed_output and max_freezed_output are the actual range
-      // for the output
+      // This is the case when convolution and requantization is fused.
+      // min_freezed_output and max_freezed_output are the actual ranges
+      // for the output.
       min_output_value = context->input(6 + bias_index_offset).flat<float>()(0);
       max_output_value = context->input(7 + bias_index_offset).flat<float>()(0);
     } else {
@@ -1473,10 +1585,9 @@ class MklQuantizedConv2DOp
     }
   }
 
-  Tbias* GetBiasHandle(
-      OpKernelContext* context,
-      std::shared_ptr<mkldnn::convolution_forward::primitive_desc>& conv_fwd_pd,
-      const Tensor& bias_tensor) override {
+  Tbias* GetBiasHandle(OpKernelContext* context,
+                       std::shared_ptr<ConvFwdPd>& conv_fwd_pd,
+                       const Tensor& bias_tensor) override {
     int bias_index_offset;
     bias_index_offset = bias_enabled ? 1 : 0;
 
@@ -1534,7 +1645,13 @@ class MklQuantizedConv2DReluOp
 
   explicit MklQuantizedConv2DReluOp(OpKernelConstruction* context)
       : MklQuantizedConv2DOp<Device, Tbias, Toutput, Ttemp_output,
-                             bias_enabled>(context) {}
+                             bias_enabled>(context) {
+    bool is_filter_const;
+    OP_REQUIRES_OK(context,
+                   context->GetAttr("is_filter_const", &is_filter_const));
+    OP_REQUIRES(context, is_filter_const == true,
+                errors::InvalidArgument("Filter must be a constant"));
+  }
 
  protected:
   void ExtendConvFwdParams(OpKernelContext* context,
@@ -1565,7 +1682,13 @@ class MklQuantizedConv2DSumReluOp
 
   explicit MklQuantizedConv2DSumReluOp(OpKernelConstruction* context)
       : MklQuantizedConv2DOp<Device, Tbias, Toutput, Ttemp_output,
-                             bias_enabled>(context) {}
+                             bias_enabled>(context) {
+    bool is_filter_const;
+    OP_REQUIRES_OK(context,
+                   context->GetAttr("is_filter_const", &is_filter_const));
+    OP_REQUIRES(context, is_filter_const == true,
+                errors::InvalidArgument("Filter must be a constant"));
+  }
 
  protected:
   void ExtendConvFwdParams(OpKernelContext* context,
@@ -1605,12 +1728,11 @@ class MklQuantizedConv2DSumReluOp
     params.post_op_params.push_back({"relu", {1.0, 0.0, 0.0}});
   }
 
-  // Allocate output tensor.
-  void AllocateOutputTensor(
-      OpKernelContext* context,
-      const convolution_forward::primitive_desc& conv_prim_desc,
-      const memory::dims& output_dims_mkl_order,
-      memory::format output_tf_format, Tensor** output_tensor) override {
+  void AllocateOutputTensor(OpKernelContext* context,
+                            const ConvFwdPd& conv_prim_desc,
+                            const memory::dims& output_dims_mkl_order,
+                            memory::format output_tf_format,
+                            Tensor** output_tensor) override {
     int summand_idx = context->num_inputs() / 2 - 1;
     float reorder_sum_scale = 1.0;
     if (std::is_same<Toutput, quint8>::value) {
@@ -1663,8 +1785,8 @@ class MklQuantizedConv2DSumReluOp
     const float max_filter =
         context->input(5 + bias_index_offset).flat<float>()(0);
 
-    reorder_sum_scale = 255.0 * 127.0 /
-                        (std::max(std::abs(max_input), std::abs(min_input)) *
+    reorder_sum_scale =
+        255.0 * 127.0 / (std::max(std::abs(max_input), std::abs(min_input)) *
                          std::max(std::abs(max_filter), std::abs(min_filter)));
     std::vector<float> scales;
     scales.push_back(reorder_sum_scale);
diff --git a/tensorflow/core/kernels/mkl_fused_ops_test.cc b/tensorflow/core/kernels/mkl_fused_ops_test.cc
index 258cca9332..6d2e4fae7c 100644
--- a/tensorflow/core/kernels/mkl_fused_ops_test.cc
+++ b/tensorflow/core/kernels/mkl_fused_ops_test.cc
@@ -218,18 +218,18 @@ class MklFusedConv2DOpTest : public OpsTestBase {
                             int depth = kDepth, int image_width = kImageWidth,
                             int image_height = kImageHeight,
                             int image_batch_count = kImageBatchCount) {
-    const BiasAddGraphRunner run_default =
-        [this](const Tensor& input_data, const Tensor& filter_data,
-               const Tensor& bias_data, Tensor* out) {
-          RunConv2DWithBias(input_data, filter_data, bias_data, out);
-        };
-
-    const BiasAddGraphRunner run_fused =
-        [this](const Tensor& input_data, const Tensor& filter_data,
-               const Tensor& bias_data, Tensor* out) {
-          RunMklFusedConv2DOp(input_data, filter_data, {bias_data}, {"BiasAdd"},
-                              out);
-        };
+    const BiasAddGraphRunner run_default = [this](
+        const Tensor& input_data, const Tensor& filter_data,
+        const Tensor& bias_data, Tensor* out) {
+      RunConv2DWithBias(input_data, filter_data, bias_data, out);
+    };
+
+    const BiasAddGraphRunner run_fused = [this](
+        const Tensor& input_data, const Tensor& filter_data,
+        const Tensor& bias_data, Tensor* out) {
+      RunMklFusedConv2DOp(input_data, filter_data, {bias_data}, {"BiasAdd"},
+                          out);
+    };
 
     VerifyBiasAddTensorsNear(depth, image_width, image_height,
                              image_batch_count, filter_size, filter_count,
@@ -243,18 +243,18 @@ class MklFusedConv2DOpTest : public OpsTestBase {
                                    int image_width = kImageWidth,
                                    int image_height = kImageHeight,
                                    int image_batch_count = kImageBatchCount) {
-    const BiasAddGraphRunner run_default =
-        [this](const Tensor& input_data, const Tensor& filter_data,
-               const Tensor& bias_data, Tensor* out) {
-          RunConv2DWithBiasAndRelu(input_data, filter_data, bias_data, out);
-        };
-
-    const BiasAddGraphRunner run_fused =
-        [this](const Tensor& input_data, const Tensor& filter_data,
-               const Tensor& bias_data, Tensor* out) {
-          RunMklFusedConv2DOp(input_data, filter_data, {bias_data},
-                              {"BiasAdd", "Relu"}, out);
-        };
+    const BiasAddGraphRunner run_default = [this](
+        const Tensor& input_data, const Tensor& filter_data,
+        const Tensor& bias_data, Tensor* out) {
+      RunConv2DWithBiasAndRelu(input_data, filter_data, bias_data, out);
+    };
+
+    const BiasAddGraphRunner run_fused = [this](
+        const Tensor& input_data, const Tensor& filter_data,
+        const Tensor& bias_data, Tensor* out) {
+      RunMklFusedConv2DOp(input_data, filter_data, {bias_data},
+                          {"BiasAdd", "Relu"}, out);
+    };
 
     VerifyBiasAddTensorsNear(depth, image_width, image_height,
                              image_batch_count, filter_size, filter_count,
@@ -401,5 +401,70 @@ TEST_F(FusedPadConvOpTest, PaddingConvTestNchw) {
 
   Run<float>(DT_FLOAT, image, filter, padding, expected, "NCHW");
 }
+
+class FilterCacheTest : public OpsTestBase {
+ public:
+  template <typename T>
+  void Run(DataType dtype, Tensor& image, Tensor& filter, Tensor& expected,
+           const bool is_filter_const) {
+    const int stride = 1;
+
+    TF_EXPECT_OK(NodeDefBuilder("conv2d_filter_cache", "_MklConv2D")
+                     .Input(FakeInput(dtype))     // Input
+                     .Input(FakeInput(dtype))     // Filter
+                     .Input(FakeInput(DT_UINT8))  // MKl second tensor
+                     .Input(FakeInput(DT_UINT8))  // MKl second tensor
+                     .Attr("padding", "VALID")
+                     .Attr("data_format", "NHWC")
+                     .Attr("is_filter_const", is_filter_const)
+                     .Attr("T", dtype)
+                     .Attr("strides", {1, stride, stride, 1})
+                     .Attr("_kernel", "MklOp")
+                     .Finalize(node_def()));
+    TF_EXPECT_OK(InitOp());
+
+    // Setting up inputs and execute
+    AddInputFromArray<T>(image.shape(), image.flat<T>());
+    AddInputFromArray<T>(filter.shape(), filter.flat<T>());
+    AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+    AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+
+    TF_ASSERT_OK(RunOpKernel());
+
+    // Compare output to expected results
+    const Tensor& first = *GetOutput(0);
+    const Tensor& second = *GetOutput(2);
+    ConvMklToTF<T> conv_comp;
+    conv_comp.ConvertAndCompare(dtype, first, second, expected);
+
+    // Run kernel again to ensure cached data is being reused
+    TF_ASSERT_OK(RunOpKernel());
+
+    // Compare output to expected results
+    const Tensor& first_new = *GetOutput(0);
+    const Tensor& second_new = *GetOutput(2);
+    ConvMklToTF<T> conv_comp_new;
+    conv_comp_new.ConvertAndCompare(dtype, first_new, second_new, expected);
+  }
+};
+
+TEST_F(FilterCacheTest, Conv2DFilterCacheTest) {
+  const int depth = 1;
+  const int image_width = 4;
+  const int image_height = 3;
+  const int image_batch_count = 1;
+  Tensor image(DT_FLOAT, {image_batch_count, image_height, image_width, depth});
+  test::FillValues<float>(&image, {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
+
+  const int filter_size = 3;
+  const int filter_count = 1;
+  Tensor filter(DT_FLOAT, {filter_size, filter_size, depth, filter_count});
+  test::FillValues<float>(&filter, {1, 4, 7, 2, 5, 8, 3, 6, 9});
+
+  Tensor expected(DT_FLOAT, TensorShape({1, 1, 2, 1}));
+  test::FillValues<float>(&expected, {312, 357});
+
+  Run<float>(DT_FLOAT, image, filter, expected, true);
+}
 }  // namespace tensorflow
 #endif  // INTEL_MKL
diff --git a/tensorflow/core/ops/mkl_nn_ops.cc b/tensorflow/core/ops/mkl_nn_ops.cc
index 658afd9901..f7b81912b9 100644
--- a/tensorflow/core/ops/mkl_nn_ops.cc
+++ b/tensorflow/core/ops/mkl_nn_ops.cc
@@ -46,6 +46,7 @@ REGISTER_OP("_MklFusedConv2D")
     .Attr("T: {float}")
     .Attr("num_args: int >= 0")
     .Attr("strides: list(int)")
+    .Attr("is_filter_const: bool = false")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
     .Attr("dilations: list(int) = [1, 1, 1, 1]")
@@ -145,6 +146,7 @@ REGISTER_OP("_MklQuantizedConv2D")
     .Attr("out_type: quantizedtype = DT_QINT32")
     .Attr("data_format: string = 'NHWC'")
     .Attr("strides: list(int)")
+    .Attr("is_filter_const: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn([](InferenceContext* c) {
@@ -189,6 +191,7 @@ REGISTER_OP("_MklQuantizedConv2DAndRequantize")
     .Attr("out_type: quantizedtype = DT_QINT8")
     .Attr("data_format: string = 'NHWC'")
     .Attr("strides: list(int)")
+    .Attr("is_filter_const: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn([](InferenceContext* c) {
@@ -233,6 +236,7 @@ REGISTER_OP("_MklQuantizedConv2DWithBias")
     .Attr("out_type: quantizedtype = DT_QINT32")
     .Attr("data_format: string = 'NHWC'")
     .Attr("strides: list(int)")
+    .Attr("is_filter_const: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn([](InferenceContext* c) {
@@ -281,6 +285,7 @@ REGISTER_OP("_MklQuantizedConv2DWithBiasAndRequantize")
     .Attr("out_type: quantizedtype = DT_QINT8")
     .Attr("data_format: string = 'NHWC'")
     .Attr("strides: list(int)")
+    .Attr("is_filter_const: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn([](InferenceContext* c) {
@@ -322,6 +327,7 @@ REGISTER_OP("_MklQuantizedConv2DAndRelu")
     .Attr("out_type: quantizedtype = DT_QINT32")
     .Attr("data_format: string = 'NHWC'")
     .Attr("strides: list(int)")
+    .Attr("is_filter_const: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn([](InferenceContext* c) {
@@ -366,6 +372,7 @@ REGISTER_OP("_MklQuantizedConv2DAndReluAndRequantize")
     .Attr("out_type: quantizedtype = DT_QUINT8")
     .Attr("data_format: string = 'NHWC'")
     .Attr("strides: list(int)")
+    .Attr("is_filter_const: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn([](InferenceContext* c) {
@@ -410,6 +417,7 @@ REGISTER_OP("_MklQuantizedConv2DWithBiasAndRelu")
     .Attr("out_type: quantizedtype = DT_QINT32")
     .Attr("data_format: string = 'NHWC'")
     .Attr("strides: list(int)")
+    .Attr("is_filter_const: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn([](InferenceContext* c) {
@@ -458,6 +466,7 @@ REGISTER_OP("_MklQuantizedConv2DWithBiasAndReluAndRequantize")
     .Attr("out_type: quantizedtype = DT_QUINT8")
     .Attr("data_format: string = 'NHWC'")
     .Attr("strides: list(int)")
+    .Attr("is_filter_const: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn([](InferenceContext* c) {
@@ -505,6 +514,7 @@ REGISTER_OP("_MklQuantizedConv2DWithBiasSumAndRelu")
     .Attr("out_type: quantizedtype = DT_QINT32")
     .Attr("data_format: string = 'NHWC'")
     .Attr("strides: list(int)")
+    .Attr("is_filter_const: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn([](InferenceContext* c) {
@@ -560,6 +570,7 @@ REGISTER_OP("_MklQuantizedConv2DWithBiasSumAndReluAndRequantize")
     .Attr("out_type: quantizedtype = DT_QUINT8")
     .Attr("data_format: string = 'NHWC'")
     .Attr("strides: list(int)")
+    .Attr("is_filter_const: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn([](InferenceContext* c) {
@@ -617,6 +628,7 @@ REGISTER_OP("_MklQuantizedConv2DWithBiasSignedSumAndReluAndRequantize")
     .Attr("out_type: quantizedtype = DT_QUINT8")
     .Attr("data_format: string = 'NHWC'")
     .Attr("strides: list(int)")
+    .Attr("is_filter_const: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn([](InferenceContext* c) {
diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc
index 0e94259f75..2bdd02a76c 100644
--- a/tensorflow/core/ops/nn_ops.cc
+++ b/tensorflow/core/ops/nn_ops.cc
@@ -1210,9 +1210,9 @@ Status TopKShapeFn(InferenceContext* c) {
   DimensionHandle last_dim = c->Dim(input, -1);
   if (c->ValueKnown(last_dim) && c->ValueKnown(k_dim) &&
       c->Value(last_dim) < c->Value(k_dim)) {
-    return errors::InvalidArgument(
-        "input must have last dimension >= k = ", c->Value(k_dim), " but is ",
-        c->Value(last_dim));
+    return errors::InvalidArgument("input must have last dimension >= k = ",
+                                   c->Value(k_dim), " but is ",
+                                   c->Value(last_dim));
   }
 
   // Replace last_dim with k_dim.
@@ -1266,9 +1266,9 @@ REGISTER_OP("NthElement")
       DimensionHandle last_dim = c->Dim(input, -1);
       if (c->ValueKnown(last_dim) && c->ValueKnown(n_dim) &&
           c->Value(last_dim) <= c->Value(n_dim)) {
-        return errors::InvalidArgument(
-            "Input must have last dimension > n = ", c->Value(n_dim),
-            " but is ", c->Value(last_dim));
+        return errors::InvalidArgument("Input must have last dimension > n = ",
+                                       c->Value(n_dim), " but is ",
+                                       c->Value(last_dim));
       }
 
       // Reduce last_dim for output tensor
@@ -1551,6 +1551,7 @@ REGISTER_OP("_MklDepthwiseConv2dNative")
     .Output("mkl_filter_output: uint8")
     .Attr("T: {half, bfloat16, float, double}")
     .Attr("strides: list(int)")
+    .Attr("is_filter_const: bool = false")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
     .Attr("dilations: list(int) = [1, 1, 1, 1]")
@@ -1568,6 +1569,7 @@ REGISTER_OP("_MklConv2D")
     .Attr("T: {half, float, double}")
     .Attr("strides: list(int)")
     .Attr("use_cudnn_on_gpu: bool = true")
+    .Attr("is_filter_const: bool = false")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
     .Attr("dilations: list(int) = [1, 1, 1, 1]")
@@ -1587,6 +1589,7 @@ REGISTER_OP("__MklDummyConv2DWithBias")
     .Attr("T: {half, float, double}")
     .Attr("strides: list(int)")
     .Attr("use_cudnn_on_gpu: bool = true")
+    .Attr("is_filter_const: bool = false")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
     .Attr("dilations: list(int) = [1, 1, 1, 1]")
@@ -1614,6 +1617,7 @@ REGISTER_OP("_MklConv2DWithBias")
     .Attr("T: {half, float, double}")
     .Attr("strides: list(int)")
     .Attr("use_cudnn_on_gpu: bool = true")
+    .Attr("is_filter_const: bool = false")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
     .Attr("dilations: list(int) = [1, 1, 1, 1]")
@@ -1634,6 +1638,7 @@ REGISTER_OP("__MklDummyPadWithConv2D")
     .Attr("T: {half, float, double}")
     .Attr("strides: list(int)")
     .Attr("use_cudnn_on_gpu: bool = true")
+    .Attr("is_filter_const: bool = false")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
     .Attr("dilations: list(int) = [1, 1, 1, 1]")
@@ -1664,6 +1669,7 @@ REGISTER_OP("_MklPadWithConv2D")
     .Attr("use_cudnn_on_gpu: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
+    .Attr("is_filter_const: bool = false")
     .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .Attr("Tpaddings: {int32, int64} = DT_INT32")
     .SetShapeFn(shape_inference::Conv2DShape)
@@ -1849,6 +1855,7 @@ REGISTER_OP("_MklConv3D")
     .Output("mkl_filter_output: uint8")
     .Attr("T: {half, float, double}")
     .Attr("strides: list(int) >= 5")
+    .Attr("is_filter_const: bool = false")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnet3dDataFormatAttrString())
     .Attr("dilations: list(int) = [1, 1, 1, 1, 1]")
-- 
GitLab


From 17bc7e61e566db5b3288b2afe6dd8cbc1152c2f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?=
 <facai.yan@gmail.com>
Date: Tue, 18 Dec 2018 08:51:25 +0800
Subject: [PATCH 0084/2345] ENH: The gradient op of bias_add supports 3/4/5D
 NCHW format

---
 tensorflow/core/kernels/bias_op.cc            | 46 +++++++++----------
 tensorflow/core/kernels/bias_op_gpu.cu.cc     |  4 +-
 tensorflow/core/kernels/bias_op_gpu.h         |  2 +-
 .../python/kernel_tests/bias_op_test.py       | 38 ++++++++++++---
 tensorflow/python/ops/nn_grad.py              |  6 +--
 5 files changed, 58 insertions(+), 38 deletions(-)

diff --git a/tensorflow/core/kernels/bias_op.cc b/tensorflow/core/kernels/bias_op.cc
index d4f4b43d63..df12dafdeb 100644
--- a/tensorflow/core/kernels/bias_op.cc
+++ b/tensorflow/core/kernels/bias_op.cc
@@ -18,13 +18,13 @@ limitations under the License.
 #define EIGEN_USE_THREADS
 
 #include "tensorflow/core/kernels/bias_op.h"
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/util/tensor_format.h"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
 #if GOOGLE_CUDA
 #include "tensorflow/core/kernels/bias_op_gpu.h"
@@ -153,13 +153,13 @@ class BiasOp : public BinaryOp<T> {
               bias.tensor<T, 1>().reshape(four_dims).broadcast(broad_cast_dims);
         } break;
         case 5: {
-          Eigen::DSizes<int32, 5> four_dims(1, channel, 1, 1, 1);
+          Eigen::DSizes<int32, 5> five_dims(1, channel, 1, 1, 1);
           Eigen::DSizes<int32, 5> broad_cast_dims(batch, 1, height, width,
                                                   depth);
           const Device& d = context->eigen_device<Device>();
           output->tensor<T, 5>().device(d) =
               input.tensor<T, 5>() +
-              bias.tensor<T, 1>().reshape(four_dims).broadcast(broad_cast_dims);
+              bias.tensor<T, 1>().reshape(five_dims).broadcast(broad_cast_dims);
         } break;
         default:
           OP_REQUIRES(context, false,
@@ -269,28 +269,24 @@ class BiasGradOp : public OpKernel {
       output->template flat<T>().setZero();
     } else {
       // Added by intel_tf to support NCHW on CPU regardless of MKL used or not.
-      // TODO(yongtang): Add 3/4/5 dimensional data support for NCHW format.
       if (data_format_ == FORMAT_NCHW) {
-        OP_REQUIRES(context, output_backprop.dims() == 4,
-                    errors::InvalidArgument(
-                        "NCHW format supports only 4D input/output tensor."));
-        Eigen::DSizes<Eigen::Index, 4> four_dims(batch, channel, height, width);
+        Eigen::DSizes<Eigen::Index, 3> three_dims(batch, channel,
+                                                  height * width * depth);
 #ifdef EIGEN_HAS_INDEX_LIST
         using idx0 = Eigen::type2index<0>;
         using idx2 = Eigen::type2index<2>;
-        using idx3 = Eigen::type2index<3>;
-        Eigen::IndexList<idx0, idx2, idx3> reduction_axes;
+        Eigen::IndexList<idx0, idx2> reduction_axes;
 #else
-        Eigen::array<Eigen::Index, 3> reduction_axes = {0, 2, 3};
+        Eigen::array<Eigen::Index, 2> reduction_axes = {0, 2};
 #endif
         output->template flat<T>().device(context->eigen_device<Device>()) =
             output_backprop.flat<T>()
                 .template cast<typename AccumulatorType<T>::type>()
-                .reshape(four_dims)
+                .reshape(three_dims)
                 .sum(reduction_axes)
                 .template cast<T>();  // End of code by intel_tf.
       } else {
-        Eigen::DSizes<Eigen::Index, 2> two_dims(batch * height * width,
+        Eigen::DSizes<Eigen::Index, 2> two_dims(batch * height * width * depth,
                                                 channel);
 #ifdef EIGEN_HAS_INDEX_LIST
         Eigen::IndexList<Eigen::type2index<0> > reduction_axis;
@@ -496,21 +492,21 @@ class BiasGradOp<GPUDevice, T> : public OpKernel {
 
   void ComputeWithCustomKernel(OpKernelContext* context,
                                const Tensor& output_backprop, int32 batch,
-                               int32 width, int32 height, int32 channel,
-                               Tensor* output) {
+                               int32 width, int32 height, int32 depth,
+                               int32 channel, Tensor* output) {
     BiasGradGPU<T>::compute(context->template eigen_device<Device>(),
                             output_backprop.template flat<T>().data(),
                             output->flat<T>().data(), batch, width, height,
-                            channel, data_format_);
+                            depth, channel, data_format_);
   }
 
   void ComputeWithReduceSum(OpKernelContext* context,
                             const Tensor& output_backprop, int32 batch,
-                            int32 width, int32 height, int32 channel,
-                            Tensor* output) {
+                            int32 width, int32 height, int32 depth,
+                            int32 channel, Tensor* output) {
     if (data_format_ == FORMAT_NCHW) {
       int32 row_count = batch * channel;
-      int32 col_count = height * width;
+      int32 col_count = height * width * depth;
       Tensor temp_grad_outputs;
       // For 'NCHW' format, we perform reduction twice: first HW, then N.
       TensorShape temp_grad_output_shape{row_count, col_count};
@@ -528,7 +524,7 @@ class BiasGradOp<GPUDevice, T> : public OpKernel {
                                      row_count, col_count);
     } else {
       // For 'NHWC', we simply apply reduction once on NHW.
-      int32 row_count = batch * height * width;
+      int32 row_count = batch * height * width * depth;
       int32 col_count = channel;
       BiasGradGPU<T>::DoColReduction(
           context, const_cast<T*>(output->flat<T>().data()),
@@ -561,7 +557,7 @@ class BiasGradOp<GPUDevice, T> : public OpKernel {
     int device_id = stream->parent()->device_ordinal();
     DataType dtype = output_backprop.dtype();
     BiasAddParams bias_parameters = {
-        {batch, height * width, channel},
+        {batch, height * width * depth, channel},
         data_format_,
         dtype,
         device_id,
@@ -576,7 +572,7 @@ class BiasGradOp<GPUDevice, T> : public OpKernel {
       stream->InitTimer(&timer);
       stream->ThenStartTimer(&timer);
       ComputeWithCustomKernel(context, output_backprop, batch, width, height,
-                              channel, output);
+                              depth, channel, output);
       stream->ThenStopTimer(&timer);
       uint64 elapsed_microseconds = timer.Microseconds();
       VLOG(1) << "BiasAddGrad " << bias_parameters.ToString()
@@ -589,7 +585,7 @@ class BiasGradOp<GPUDevice, T> : public OpKernel {
       // Try reduction and profile.
       stream->ThenStartTimer(&timer);
       ComputeWithReduceSum(context, output_backprop, batch, width, height,
-                           channel, output);
+                           depth, channel, output);
       stream->ThenStopTimer(&timer);
 
       elapsed_microseconds = timer.Microseconds();
@@ -610,11 +606,11 @@ class BiasGradOp<GPUDevice, T> : public OpKernel {
     // Choose the best algorithm based on autotune results.
     if (algo_config.get_mode() == BiasAddGradGPUMode::kReduction) {
       ComputeWithReduceSum(context, output_backprop, batch, width, height,
-                           channel, output);
+                           depth, channel, output);
     } else {
       // Default to the customized kernel.
       ComputeWithCustomKernel(context, output_backprop, batch, width, height,
-                              channel, output);
+                              depth, channel, output);
     }
   }
 
diff --git a/tensorflow/core/kernels/bias_op_gpu.cu.cc b/tensorflow/core/kernels/bias_op_gpu.cu.cc
index 24fea8a8e6..006fa1dc71 100644
--- a/tensorflow/core/kernels/bias_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/bias_op_gpu.cu.cc
@@ -195,10 +195,10 @@ __global__ void BiasGradNCHW_SharedAtomics(const T* output_backprop,
 template <typename T>
 void BiasGradGPU<T>::compute(const GPUDevice& d, const T* output_backprop,
                              T* bias_backprop, int32 batch, int32 height,
-                             int32 width, int32 channel,
+                             int32 width, int32 depth, int32 channel,
                              TensorFormat data_format) {
   const int32 bias_size = channel;
-  const int32 image_size = height * width;
+  const int32 image_size = height * width * depth;
   const int32 total_count = batch * bias_size * image_size;
   if (total_count == 0) {
     return;
diff --git a/tensorflow/core/kernels/bias_op_gpu.h b/tensorflow/core/kernels/bias_op_gpu.h
index a0b2ce4f9b..372a403e68 100644
--- a/tensorflow/core/kernels/bias_op_gpu.h
+++ b/tensorflow/core/kernels/bias_op_gpu.h
@@ -39,7 +39,7 @@ template <typename T>
 struct BiasGradGPU {
   static void compute(const GPUDevice& device, const T* output_backprop,
                       T* bias_backprop, int32 batch, int32 height, int32 width,
-                      int32 channel, TensorFormat data_format);
+                      int32 depth, int32 channel, TensorFormat data_format);
 
   static void DoRowReduction(OpKernelContext* context, T* output,
                              const T* input, int rows, int cols);
diff --git a/tensorflow/python/kernel_tests/bias_op_test.py b/tensorflow/python/kernel_tests/bias_op_test.py
index 66f442dbdd..c3976194a0 100644
--- a/tensorflow/python/kernel_tests/bias_op_test.py
+++ b/tensorflow/python/kernel_tests/bias_op_test.py
@@ -196,9 +196,7 @@ class BiasAddTest(test.TestCase):
       self.assertAllClose(grad_jacob_t, grad_jacob_n, threshold, threshold)
 
   @test_util.run_deprecated_v1
-  def testGradientTensor(self):
-    # TODO(yongtang): BiasAddGrad with NCHW only works 4D. Reenable once
-    # all dimensions are supported.
+  def testGradientTensor2D(self):
     for (data_format, use_gpu) in ("NHWC", False), ("NHWC", True):
       for dtype in (dtypes.float16, dtypes.float32, dtypes.float64):
         np_input = np.array(
@@ -207,9 +205,19 @@ class BiasAddTest(test.TestCase):
         bias = np.array([1.3, 2.4], dtype=dtype.as_numpy_dtype)
         self._testGradient(np_input, bias, dtype, data_format, use_gpu)
 
+  @test_util.run_deprecated_v1
+  def testGradientTensor3D(self):
+    for (data_format, use_gpu) in [("NHWC", False), ("NHWC", True),
+                                   ("NCHW", False), ("NCHW", True)]:
+      for dtype in (dtypes.float16, dtypes.float32, dtypes.float64):
+        np_input = np.array(
+            [1.0, 2.0, 3.0, 4.0, 5.0, 6.0],
+            dtype=dtype.as_numpy_dtype).reshape(1, 3, 2)
+        bias = np.array([1.3, 2.4], dtype=dtype.as_numpy_dtype)
+        self._testGradient(np_input, bias, dtype, data_format, use_gpu)
+
   @test_util.run_deprecated_v1
   def testGradientTensor4D(self):
-    # BiasAddGrad with NCHW support 4D so all are enabled.
     for (data_format, use_gpu) in [("NHWC", False), ("NHWC", True),
                                    ("NCHW", False), ("NCHW", True)]:
       for dtype in (dtypes.float16, dtypes.float32, dtypes.float64):
@@ -219,6 +227,17 @@ class BiasAddTest(test.TestCase):
         bias = np.array([1.3, 2.4], dtype=dtype.as_numpy_dtype)
         self._testGradient(np_input, bias, dtype, data_format, use_gpu)
 
+  @test_util.run_deprecated_v1
+  def testGradientTensor5D(self):
+    for (data_format, use_gpu) in [("NHWC", False), ("NHWC", True),
+                                   ("NCHW", False), ("NCHW", True)]:
+      for dtype in (dtypes.float16, dtypes.float32, dtypes.float64):
+        np_input = np.arange(
+            1.0, 49.0, dtype=dtype.as_numpy_dtype).reshape(
+                [1, 2, 3, 4, 2]).astype(np.float32)
+        bias = np.array([1.3, 2.4], dtype=dtype.as_numpy_dtype)
+        self._testGradient(np_input, bias, dtype, data_format, use_gpu)
+
   @test_util.run_deprecated_v1
   def testEmpty(self):
     np.random.seed(7)
@@ -227,10 +246,15 @@ class BiasAddTest(test.TestCase):
 
   @test_util.run_deprecated_v1
   def testEmptyGradient(self):
-    # TODO(yongtang): BiasAddGrad with NCHW only works 4D. Reenable once
-    # all dimensions are supported.
     for (data_format, use_gpu) in ("NHWC", False), ("NHWC", True):
-      for shape in (0, 0), (2, 0), (0, 2), (4, 3, 0), (4, 0, 3), (0, 4, 3):
+      for shape in (0, 0), (2, 0), (0, 2):
+        self._testGradient(
+            np.random.randn(*shape),
+            np.random.randn(shape[-1]), dtypes.float64, data_format, use_gpu)
+
+    for (data_format, use_gpu) in [("NHWC", False), ("NHWC", True),
+                                   ("NCHW", False), ("NCHW", True)]:
+      for shape in (4, 3, 0), (4, 0, 3), (0, 4, 3):
         self._testGradient(
             np.random.randn(*shape),
             np.random.randn(shape[-1]), dtypes.float64, data_format, use_gpu)
diff --git a/tensorflow/python/ops/nn_grad.py b/tensorflow/python/ops/nn_grad.py
index 34404edc9a..7131e4abc4 100644
--- a/tensorflow/python/ops/nn_grad.py
+++ b/tensorflow/python/ops/nn_grad.py
@@ -314,10 +314,10 @@ def _BiasAddGradGrad(op, received_grad):
 
   if data_format == b"NCHW":
     expanded_shape = array_ops.concat([
-        array_ops.ones_like(shape[:-3]), bias_shape,
-        array_ops.ones_like(shape[-2:])
+        array_ops.ones_like(shape[:1]), bias_shape,
+        array_ops.ones_like(shape[2:])
     ], 0)
-    tile_mults = array_ops.concat([shape[:-3], [1], shape[-2:]], 0)
+    tile_mults = array_ops.concat([shape[:1], [1], shape[2:]], 0)
   else:
     expanded_shape = array_ops.concat(
         [array_ops.ones_like(shape[:-1]), bias_shape], 0)
-- 
GitLab


From 96f41397357f96c41459cf351f8853caa8d724c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lien=20Geron?= <ageron@users.noreply.github.com>
Date: Fri, 28 Dec 2018 15:18:06 +0800
Subject: [PATCH 0085/2345] Fix typo in the documentation of tf.function

"`add_noise()` will return a different output every time it is invoked. However, `add_noise` will return the same value every time it is called..."
=> the second `add_noise` should be `traced`
---
 tensorflow/python/eager/def_function.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py
index ebc47d1566..b25df5a47c 100644
--- a/tensorflow/python/eager/def_function.py
+++ b/tensorflow/python/eager/def_function.py
@@ -765,7 +765,7 @@ def function(func=None,
   ```
 
   `add_noise()` will return a different output every time it is invoked.
-  However, `add_noise` will return the same value every time it is called,
+  However, `traced()` will return the same value every time it is called,
   since a particular random value generated by the `np.random.randn` call will
   be inserted in the traced/staged TensorFlow graph as a constant. In this
   particular example, replacing `np.random.randn(5, 5)` with
-- 
GitLab


From 9fb415c6eac3cd245de736e7ebc54094416c1013 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 30 Dec 2018 01:07:37 +0000
Subject: [PATCH 0086/2345] Fix TypeError when using tf.keras.utils.plot_model

This fix fixes the issue raised in 24622
where a TypeError was raised when using tf.keras.utils.plot_model.

This fix fixes 24622.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/keras/utils/vis_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/utils/vis_utils.py b/tensorflow/python/keras/utils/vis_utils.py
index 82bc2755bd..fa848ae8a7 100644
--- a/tensorflow/python/keras/utils/vis_utils.py
+++ b/tensorflow/python/keras/utils/vis_utils.py
@@ -120,7 +120,7 @@ def model_to_dot(model, show_shapes=False, show_layer_names=True, rankdir='TB'):
     for i, node in enumerate(layer._inbound_nodes):
       node_key = layer.name + '_ib-' + str(i)
       if node_key in model._network_nodes:  # pylint: disable=protected-access
-        for inbound_layer in node.inbound_layers:
+        for inbound_layer in nest.flatten(node.inbound_layers):
           inbound_layer_id = str(id(inbound_layer))
           layer_id = str(id(layer))
           dot.add_edge(pydot.Edge(inbound_layer_id, layer_id))
-- 
GitLab


From 129bf6c79a6793951d0110d31b1b2c0198479fff Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 30 Dec 2018 01:09:11 +0000
Subject: [PATCH 0087/2345] Add missing python import

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/keras/utils/vis_utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/keras/utils/vis_utils.py b/tensorflow/python/keras/utils/vis_utils.py
index fa848ae8a7..c7c45f381e 100644
--- a/tensorflow/python/keras/utils/vis_utils.py
+++ b/tensorflow/python/keras/utils/vis_utils.py
@@ -67,6 +67,7 @@ def model_to_dot(model, show_shapes=False, show_layer_names=True, rankdir='TB'):
   """
   from tensorflow.python.keras.layers.wrappers import Wrapper
   from tensorflow.python.keras.models import Sequential
+  from tensorflow.python.util import nest
 
   _check_pydot()
   dot = pydot.Dot()
-- 
GitLab


From 53e677de105c2edbfdc2d86ceb31d06765b08512 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 30 Dec 2018 10:07:06 +0000
Subject: [PATCH 0088/2345] Fix incorrect display of keras model summary

This fix fixes the issue raised in 24627 where the
display of keras model summary is incorrect (regression
from 1.12). The reason was that `layer.name`
(vs. layer object itself) should be used,

This fix fixes 24627.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/keras/utils/layer_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/utils/layer_utils.py b/tensorflow/python/keras/utils/layer_utils.py
index ead5afd1ae..1d85e8a25f 100644
--- a/tensorflow/python/keras/utils/layer_utils.py
+++ b/tensorflow/python/keras/utils/layer_utils.py
@@ -196,7 +196,7 @@ def print_summary(model, line_length=None, positions=None, print_fn=None):
         continue
 
       for inbound_layer, node_index, tensor_index, _ in node.iterate_inbound():
-        connections.append('{}[{}][{}]'.format(inbound_layer, node_index,
+        connections.append('{}[{}][{}]'.format(inbound_layer.name, node_index,
                                                tensor_index))
 
     name = layer.name
-- 
GitLab


From 85f3821cf8c4cf8354e2aff6e8a4b7c1c5f2fd88 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Mon, 31 Dec 2018 00:39:14 +0000
Subject: [PATCH 0089/2345] Replace deprecated test_session with cached_session
 in quantile_ops_test

This fix replace deprecated test_session with cached_session in
quantile_ops_test.py to remove warnings during tests

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .../python/kernel_tests/boosted_trees/quantile_ops_test.py    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py b/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py
index 37a60fa0e3..12eb58536a 100644
--- a/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py
+++ b/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py
@@ -145,7 +145,7 @@ class QuantileOpsTest(test_util.TensorFlowTestCase):
     save_dir = os.path.join(self.get_temp_dir(), "save_restore")
     save_path = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash")
 
-    with self.test_session() as sess:
+    with self.cached_session() as sess:
       accumulator = boosted_trees_ops.QuantileAccumulator(
           num_streams=2, num_quantiles=3, epsilon=self.eps, name="q0")
 
@@ -177,7 +177,7 @@ class QuantileOpsTest(test_util.TensorFlowTestCase):
     save_dir = os.path.join(self.get_temp_dir(), "save_restore")
     save_path = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash")
 
-    with self.test_session() as sess:
+    with self.cached_session() as sess:
       accumulator = boosted_trees_ops.QuantileAccumulator(
           num_streams=2, num_quantiles=3, epsilon=self.eps, name="q0")
 
-- 
GitLab


From 1f6a9fc270bd356e817f208b9386732f03824c65 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Mon, 31 Dec 2018 00:40:43 +0000
Subject: [PATCH 0090/2345] Fix test_session warnings during tests by replacing
 it with session()

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .../python/kernel_tests/boosted_trees/quantile_ops_test.py    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py b/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py
index 12eb58536a..0315456447 100644
--- a/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py
+++ b/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py
@@ -164,7 +164,7 @@ class QuantileOpsTest(test_util.TensorFlowTestCase):
       self.assertAllClose(self._feature_1_boundaries, buckets[1].eval())
       save.save(sess, save_path)
 
-    with self.test_session(graph=ops.Graph()) as sess:
+    with self.session(graph=ops.Graph()) as sess:
       accumulator = boosted_trees_ops.QuantileAccumulator(
           num_streams=2, num_quantiles=3, epsilon=self.eps, name="q0")
       save = saver.Saver()
@@ -195,7 +195,7 @@ class QuantileOpsTest(test_util.TensorFlowTestCase):
       self.assertAllClose(self._feature_0_boundaries, buckets[0].eval())
       self.assertAllClose(self._feature_1_boundaries, buckets[1].eval())
 
-    with self.test_session(graph=ops.Graph()) as sess:
+    with self.session(graph=ops.Graph()) as sess:
       accumulator = boosted_trees_ops.QuantileAccumulator(
           num_streams=2, num_quantiles=3, epsilon=self.eps, name="q0")
       save = saver.Saver()
-- 
GitLab


From 1f31207ab85639eea00e2fa04586a6529de4784f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lien=20Geron?= <ageron@users.noreply.github.com>
Date: Mon, 31 Dec 2018 11:50:57 +0800
Subject: [PATCH 0091/2345] Fix model_to_dot() for Sequential models

The model_to_dot() function does not display the input layer of Sequential models correctly. See https://github.com/keras-team/keras/issues/10638
This fix seems to have been applied already in keras-team/keras, but not in tf.keras.
---
 tensorflow/python/keras/utils/vis_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/utils/vis_utils.py b/tensorflow/python/keras/utils/vis_utils.py
index 82bc2755bd..f56b2720b5 100644
--- a/tensorflow/python/keras/utils/vis_utils.py
+++ b/tensorflow/python/keras/utils/vis_utils.py
@@ -77,7 +77,7 @@ def model_to_dot(model, show_shapes=False, show_layer_names=True, rankdir='TB'):
   if isinstance(model, Sequential):
     if not model.built:
       model.build()
-  layers = model.layers
+  layers = model._layers
 
   # Create graph nodes.
   for layer in layers:
-- 
GitLab


From 68eb3b63b0eae2dc1eab5697872d54f5bc472f82 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lien=20Geron?= <ageron@users.noreply.github.com>
Date: Mon, 31 Dec 2018 19:11:17 +0800
Subject: [PATCH 0092/2345] Add model_to_dot and print_summary to
 tf.keras.utils

Fixes #24639
---
 tensorflow/python/keras/utils/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/python/keras/utils/__init__.py b/tensorflow/python/keras/utils/__init__.py
index 61940ad789..66d9817a6a 100644
--- a/tensorflow/python/keras/utils/__init__.py
+++ b/tensorflow/python/keras/utils/__init__.py
@@ -34,10 +34,12 @@ from tensorflow.python.keras.utils.generic_utils import serialize_keras_object
 from tensorflow.python.keras.utils.io_utils import HDF5Matrix
 from tensorflow.python.keras.utils.layer_utils import convert_all_kernels_in_model
 from tensorflow.python.keras.utils.layer_utils import get_source_inputs
+from tensorflow.python.keras.utils.layer_utils import print_summary
 from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions
 from tensorflow.python.keras.utils.multi_gpu_utils import multi_gpu_model
 from tensorflow.python.keras.utils.np_utils import normalize
 from tensorflow.python.keras.utils.np_utils import to_categorical
+from tensorflow.python.keras.utils.vis_utils import model_to_dot
 from tensorflow.python.keras.utils.vis_utils import plot_model
 
 del absolute_import
-- 
GitLab


From ceb64c15db61855183d3b88504b5469af0f423c7 Mon Sep 17 00:00:00 2001
From: Dheeraj Rajaram Reddy <dheeraj98reddy@gmail.com>
Date: Tue, 1 Jan 2019 01:25:55 +0530
Subject: [PATCH 0093/2345] Add take_while experimental dataset op (tests
 pending)

---
 .../core/kernels/data/experimental/BUILD      |  14 +
 .../experimental/take_while_dataset_op.cc     | 242 ++++++++++++++++++
 .../core/ops/experimental_dataset_ops.cc      |  11 +
 .../python/data/experimental/__init__.py      |   1 +
 tensorflow/python/data/experimental/ops/BUILD |  14 +
 .../data/experimental/ops/take_while_ops.py   |  79 ++++++
 6 files changed, 361 insertions(+)
 create mode 100644 tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
 create mode 100644 tensorflow/python/data/experimental/ops/take_while_ops.py

diff --git a/tensorflow/core/kernels/data/experimental/BUILD b/tensorflow/core/kernels/data/experimental/BUILD
index 7433303f77..4ce14b0140 100644
--- a/tensorflow/core/kernels/data/experimental/BUILD
+++ b/tensorflow/core/kernels/data/experimental/BUILD
@@ -293,6 +293,19 @@ tf_kernel_library(
     ],
 )
 
+tf_kernel_library(
+    name = "take_while_dataset_op",
+    srcs = ["take_while_dataset_op.cc"],
+    deps = [
+        "//tensorflow/core:core_cpu_internal",
+        "//tensorflow/core:dataset_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core/kernels/data:captured_function",
+    ],
+)
+
 tf_kernel_library(
     name = "to_tf_record_op",
     srcs = ["to_tf_record_op.cc"],
@@ -365,6 +378,7 @@ tf_kernel_library(
         ":sql_dataset_op",
         ":stats_aggregator_ops",
         ":stats_dataset_ops",
+        ":take_while_dataset_op",
         ":threadpool_dataset_op",
         ":to_tf_record_op",
         ":unbatch_dataset_op",
diff --git a/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc b/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
new file mode 100644
index 0000000000..271b919bc9
--- /dev/null
+++ b/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
@@ -0,0 +1,242 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <iterator>
+#include <vector>
+
+#include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/framework/dataset.h"
+#include "tensorflow/core/framework/partial_tensor_shape.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/kernels/data/captured_function.h"
+
+namespace tensorflow {
+namespace data {
+namespace {
+
+// See documentation in ../../ops/dataset_ops.cc for a high-level
+// description of the following op.
+
+class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
+ public:
+  explicit TakeWhileDatasetOp(OpKernelConstruction* ctx)
+      : UnaryDatasetOpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("predicate", &func_));
+    OP_REQUIRES_OK(
+        ctx, ctx->GetAttr("preserve_cardinality", &preserve_cardinality_));
+  }
+
+  void MakeDataset(OpKernelContext* ctx, DatasetBase* input,
+                   DatasetBase** output) override {
+    std::unique_ptr<CapturedFunction> captured_func;
+    OP_REQUIRES_OK(ctx, CapturedFunction::Create(func_, ctx, "other_arguments",
+                                                 &captured_func));
+    
+    // TODO (squadrick): check short-circuit
+    *output = new Dataset(ctx, input, func_, std::move(captured_func), 
+                          preserve_cardinality_);
+  }
+
+ private:
+  class Dataset : public DatasetBase {
+   public:
+    Dataset(OpKernelContext* ctx, const DatasetBase* input,
+            const NameAttrList& func,
+            std::unique_ptr<CapturedFunction> captured_func,
+            bool preserve_cardinality)
+        : DatasetBase(DatasetContext(ctx)),
+          input_(input),
+          func_(func),
+          captured_func_(std::move(captured_func)),
+          preserve_cardinality_(preserve_cardinality) {
+      input_->Ref();
+    }
+
+    ~Dataset() override { input_->Unref(); }
+
+    std::unique_ptr<IteratorBase> MakeIteratorInternal(
+        const string& prefix) const override {
+      return std::unique_ptr<IteratorBase>(
+          new Iterator({this, strings::StrCat(prefix, "::TakeWhile")}));
+    }
+
+    const DataTypeVector& output_dtypes() const override {
+      return input_->output_dtypes();
+    }
+    const std::vector<PartialTensorShape>& output_shapes() const override {
+      return input_->output_shapes();
+    }
+
+    string DebugString() const override { return "TakeWhileDatasetOp::Dataset"; }
+
+    int64 Cardinality() const override { return input_->Cardinality(); }
+
+   protected:
+    Status AsGraphDefInternal(SerializationContext* ctx,
+                              DatasetGraphDefBuilder* b,
+                              Node** output) const override {
+      TF_RETURN_IF_ERROR(b->AddFunction(ctx, func_.name()));
+      Node* input_node;
+      TF_RETURN_IF_ERROR(b->AddInputDataset(ctx, input_, &input_node));
+
+      std::vector<Node*> other_arguments;
+      other_arguments.reserve(captured_func_->captured_inputs().size());
+      DataTypeVector other_arguments_types;
+      other_arguments_types.reserve(captured_func_->captured_inputs().size());
+      for (const Tensor& t : captured_func_->captured_inputs()) {
+        Node* node;
+        DatasetBase* input;
+        Status s = GetDatasetFromVariantTensor(t, &input);
+        if (s.ok()) {
+          TF_RETURN_IF_ERROR(b->AddInputDataset(ctx, input, &node));
+        } else {
+          TF_RETURN_IF_ERROR(b->AddTensor(t, &node));
+        }
+        other_arguments.emplace_back(node);
+        other_arguments_types.emplace_back(t.dtype());
+      }
+      AttrValue f_attr;
+      b->BuildAttrValue(func_, &f_attr);
+
+      AttrValue other_arguments_types_attr;
+      b->BuildAttrValue(other_arguments_types, &other_arguments_types_attr);
+
+      AttrValue preserve_cardinality_attr;
+      b->BuildAttrValue(preserve_cardinality_, &preserve_cardinality_attr);
+
+      TF_RETURN_IF_ERROR(b->AddDataset(
+            this, {std::make_pair(0, input_node)},
+            {std::make_pair(1, other_arguments)},
+            {std::make_pair("predicate", f_attr),
+             std::make_pair("Targuments", other_arguments_types_attr),
+             std::make_pair("preserve_cardinality", 
+                            preserve_cardinality_attr)},
+             output));
+      return Status::OK();
+    }
+
+   private:
+    class Iterator : public DatasetIterator<Dataset> {
+     public:
+      explicit Iterator(const Params& params)
+          : DatasetIterator<Dataset>(params) {}
+
+      Status Initialize(IteratorContext* ctx) override {
+        TF_RETURN_IF_ERROR(
+            dataset()->input_->MakeIterator(ctx, prefix(), &input_impl_));
+        return dataset()->captured_func_->Instantiate(
+            ctx, &instantiated_captured_func_);
+      }
+
+      Status GetNextInternal(IteratorContext* ctx,
+                             std::vector<Tensor>* out_tensors,
+                             bool* end_of_sequence) override {
+        {
+          tf_shared_lock l(mu_);
+          if(!input_impl_) {
+            *end_of_sequence = true;
+            return Status::OK();
+          }
+          TF_RETURN_IF_ERROR(
+              input_impl_->GetNext(ctx, out_tensors, end_of_sequence));
+        }
+        if (*end_of_sequence) {
+          mutex_lock l(mu_);
+          input_impl_.reset();
+          return Status::OK();
+        }
+
+        std::vector<Tensor> bool_output;
+
+        Status s = instantiated_captured_func_->RunWithBorrowedArgs(
+            ctx, *out_tensors, &bool_output);
+
+        if (s.ok()) {
+          if(bool_output.size() != 1 || bool_output[0].dtype() != DT_BOOL ||
+             bool_output[0].NumElements() != 1) {
+            return errors::InvalidArgument(
+                "`predicate` must returns a scalar bool tensor.");
+          }
+          auto cond = bool_output[0].scalar<bool>()();
+          if (!cond) { // predicate is false
+            *end_of_sequence = true;
+            return Status::OK();
+          }
+        } else if (errors::IsOutOfRange(s)) {
+          if (dataset()->preserve_cardinality_) {
+            // To guarantee that the transformation preserves the cardinality of
+            // the dataset, we convert `OutOfRange` to `InvalidArgument` as the
+            // former may be interpreted by a caller as the end of sequence.
+            return errors::InvalidArgument(
+                "Function invocation produced OutOfRangeError: ",
+                s.error_message());
+          } else {
+            // `f` may deliberately raise `errors::OutOfRange` to indicate
+            // that we should terminate the iteration early.
+            *end_of_sequence = true;
+            return Status::OK();
+          }
+        }
+        return s;
+      }
+
+     protected:
+      std::shared_ptr<model::Node> CreateNode(
+          IteratorContext* ctx, model::Node::Args args) const override {
+        return model::MakeKnownRatioNode(std::move(args),
+                                         /*ratio=*/1);
+      }
+
+      Status SaveInternal(IteratorStateWriter* writer) override {
+        mutex_lock l(mu_);
+        if (input_impl_)
+          TF_RETURN_IF_ERROR(SaveInput(writer, input_impl_));
+        else
+          TF_RETURN_IF_ERROR(
+              writer->WriteScalar(full_name("input_impls_empty"), ""));
+        return Status::OK();
+      }
+
+      Status RestoreInternal(IteratorContext* ctx,
+                             IteratorStateReader* reader) override {
+        mutex_lock l(mu_);
+        if (reader->Contains(full_name("input_impls_empty")))
+          input_impl_.reset();
+        else
+          TF_RETURN_IF_ERROR(RestoreInput(ctx, reader, input_impl_));
+        return Status::OK();
+      }
+
+     private:
+      mutex mu_;
+      std::unique_ptr<IteratorBase> input_impl_ GUARDED_BY(mu_);
+      std::unique_ptr<InstantiatedCapturedFunction> instantiated_captured_func_;
+    };
+
+    const DatasetBase* const input_;
+    const NameAttrList func_;
+    const std::unique_ptr<CapturedFunction> captured_func_;
+    const bool preserve_cardinality_;
+  };
+
+  NameAttrList func_;
+  bool preserve_cardinality_;
+};
+
+REGISTER_KERNEL_BUILDER(Name("ExperimentalTakeWhileDataset").Device(DEVICE_CPU),
+                        TakeWhileDatasetOp);
+
+}  // namespace
+}  // namespace data
+}  // namespace tensorflow
diff --git a/tensorflow/core/ops/experimental_dataset_ops.cc b/tensorflow/core/ops/experimental_dataset_ops.cc
index f904e2536d..d10a13f06f 100644
--- a/tensorflow/core/ops/experimental_dataset_ops.cc
+++ b/tensorflow/core/ops/experimental_dataset_ops.cc
@@ -352,6 +352,17 @@ REGISTER_OP("ExperimentalStatsAggregatorSummary")
     .Output("summary: string")
     .SetShapeFn(shape_inference::ScalarShape);
 
+REGISTER_OP("ExperimentalTakeWhileDataset")
+    .Input("input_dataset: variant")
+    .Input("other_arguments: Targuments")
+    .Output("handle: variant")
+    .Attr("predicate: func")
+    .Attr("Targuments: list(type) >= 0")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")
+    .Attr("preserve_cardinality: bool = false")
+    .SetShapeFn(shape_inference::ScalarShape);
+
 REGISTER_OP("ExperimentalUnbatchDataset")
     .Input("input_dataset: variant")
     .Output("handle: variant")
diff --git a/tensorflow/python/data/experimental/__init__.py b/tensorflow/python/data/experimental/__init__.py
index ffc2e5ef5f..016acefa1f 100644
--- a/tensorflow/python/data/experimental/__init__.py
+++ b/tensorflow/python/data/experimental/__init__.py
@@ -115,6 +115,7 @@ from tensorflow.python.data.experimental.ops.shuffle_ops import shuffle_and_repe
 from tensorflow.python.data.experimental.ops.stats_aggregator import StatsAggregator
 from tensorflow.python.data.experimental.ops.stats_ops import latency_stats
 from tensorflow.python.data.experimental.ops.stats_options import StatsOptions
+from tensorflow.python.data.experimental.ops.take_while_ops import take_while
 from tensorflow.python.data.experimental.ops.threading_options import ThreadingOptions
 from tensorflow.python.data.experimental.ops.unique import unique
 from tensorflow.python.data.experimental.ops.writers import TFRecordWriter
diff --git a/tensorflow/python/data/experimental/ops/BUILD b/tensorflow/python/data/experimental/ops/BUILD
index 60c20e0bcf..95e9509678 100644
--- a/tensorflow/python/data/experimental/ops/BUILD
+++ b/tensorflow/python/data/experimental/ops/BUILD
@@ -354,6 +354,19 @@ py_library(
     ],
 )
 
+py_library(
+    name = "take_while_ops",
+    srcs = ["take_while_ops.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:experimental_dataset_ops_gen",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:function",
+        "//tensorflow/python/data/ops:dataset_ops",
+    ],
+)
+
 py_library(
     name = "threading_options",
     srcs = ["threading_options.py"],
@@ -454,6 +467,7 @@ py_library(
         ":shuffle_ops",
         ":sleep",
         ":stats_ops",
+        "take_while_ops",
         ":threadpool",
         ":unique",
         ":writers",
diff --git a/tensorflow/python/data/experimental/ops/take_while_ops.py b/tensorflow/python/data/experimental/ops/take_while_ops.py
new file mode 100644
index 0000000000..edbecd0666
--- /dev/null
+++ b/tensorflow/python/data/experimental/ops/take_while_ops.py
@@ -0,0 +1,79 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""take-while dataset transformation."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import collections
+
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.util import structure as structure_lib
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import gen_experimental_dataset_ops
+from tensorflow.python.util.tf_export import tf_export
+
+
+class _TakeWhileDataset(dataset_ops.UnaryUnchangedStructureDataset):
+  """A dataset that stops iteration when `predicate` returns false."""
+
+  def __init__(self, input_dataset, predicate):
+    """See `take_while()` for details."""
+    
+    self._input_dataset = input_dataset
+    wrapped_func = dataset_ops.StructuredFunctionWrapper(
+            predicate,
+            self._transformation_name(),
+            dataset=self._input_dataset)
+    
+    if not wrapped_func.output_structure.is_compatible_with(
+            structure_lib.TensorStructure(dtypes.bool, [])):
+        raise ValueError("`predicate` must return a scalar boolean tensor.")
+     
+    self._predicate = wrapped_func
+
+    variant_tensor = gen_experimental_dataset_ops.experimental_take_while_dataset(
+            self._input_dataset._variant_tensor,
+            other_arguments=self._predicate.function.captured_inputs,
+            predicate=self._predicate.function,
+            preserve_cardinality=True,
+            **dataset_ops.flat_structure(self))
+    super(_TakeWhileDataset, self).__init__(input_dataset, variant_tensor)
+
+  def _functions(self):
+      return [self._predicate]
+
+  def _transformation_name(self):
+      return "tf.data.experimental.take_while()"
+
+
+@tf_export("data.experimental.take_while")
+def take_while(predicate):
+  """A transformation that stops dataset iteration based on a `predicate` condition
+
+  Args:
+    predicate: A function that maps a nested structure of tensors 
+    (having shapes and types defined by `self.output_shapes` and 
+    `self.output_types`) to a scalar `tf.bool` tensor.
+
+  Returns:
+    A `Dataset` transformation function, which can be passed to
+    `tf.data.Dataset.apply`.
+  """
+  def _apply_fn(dataset):
+      return _TakeWhileDataset(dataset, predicate)
+
+  return _apply_fn
-- 
GitLab


From 15d49deb3c5d4048718af19a2e0e0279e8f204d5 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Tue, 1 Jan 2019 00:16:30 +0000
Subject: [PATCH 0094/2345] Update eigen library to 88fc23324517 to fix 24457

This fix updates eigen library to 88fc23324517 so that the issue
raised in 24457 could be fixed.

This fix fixes 24457.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 11ce55feda..d32caceb9e 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -136,11 +136,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "eigen_archive",
         build_file = clean_dep("//third_party:eigen.BUILD"),
-        sha256 = "753fbb58d0a49b6bcbcfb126ebfa2e21fc97f7471529ba835a096008ce588d8a",
-        strip_prefix = "eigen-eigen-9f48e814419e",
+        sha256 = "9de38f2d162c51599b802f7c36d9f3773980d19ac908c61638f8344d2c10e1ca",
+        strip_prefix = "eigen-eigen-88fc23324517",
         urls = [
-            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/9f48e814419e.tar.gz",
-            "https://bitbucket.org/eigen/eigen/get/9f48e814419e.tar.gz",
+            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/88fc23324517..tar.gz",
+            "https://bitbucket.org/eigen/eigen/get/88fc23324517.tar.gz",
         ],
     )
 
-- 
GitLab


From fcffdde4d6a918a7971631b444e8fc0cb8b8244b Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Tue, 1 Jan 2019 19:19:48 +0530
Subject: [PATCH 0095/2345] SynchronousMemcpy changed to SynchronousMemcpyD2H
 and SynchronousMemcpyH2D

---
 .../core/common_runtime/gpu/gpu_debug_allocator.cc | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc b/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc
index 989ddbe4af..7268d6cf6a 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc
@@ -44,7 +44,8 @@ bool CheckMask(se::StreamExecutor* exec, void* ptr, int64* mask) {
   se::DeviceMemory<int64> gpu_ptr{se::DeviceMemoryBase{ptr, MASK_BYTES}};
   int64 tmp[MASK_WORDS];
 
-  if (!exec->SynchronousMemcpy(&tmp, gpu_ptr, MASK_BYTES)) {
+  Status result = exec->SynchronousMemcpyD2H(gpu_ptr, MASK_BYTES, tmp);
+  if (!result.ok()) {
     LOG(FATAL) << "Could not copy debug mask";
   }
 
@@ -63,7 +64,8 @@ bool CheckMask(se::StreamExecutor* exec, void* ptr, int64* mask) {
 
 void InitMask(se::StreamExecutor* exec, void* ptr, int64* mask) {
   se::DeviceMemory<int64> gpu_ptr{se::DeviceMemoryBase{ptr, MASK_BYTES}};
-  if (!exec->SynchronousMemcpy(&gpu_ptr, mask, MASK_BYTES)) {
+  Status result = exec->SynchronousMemcpyH2D(mask, MASK_BYTES, &gpu_ptr);
+  if (!result.ok()) {
     LOG(FATAL) << "Could not copy debug mask";
   }
 }
@@ -171,7 +173,9 @@ void* GPUNanResetAllocator::AllocateRaw(size_t alignment, size_t num_bytes) {
   se::DeviceMemory<float> nan_ptr{
       se::DeviceMemoryBase{static_cast<float*>(allocated_ptr), req_size}};
 
-  if (!stream_exec_->SynchronousMemcpy(&nan_ptr, &nans[0], req_size)) {
+  Status result = stream_exec_->SynchronousMemcpyH2D(&nans[0], req_size, 
+                                                     &nan_ptr);
+  if (!result.ok()) {
     LOG(ERROR) << "Could not initialize to NaNs";
   }
 
@@ -185,7 +189,9 @@ void GPUNanResetAllocator::DeallocateRaw(void* ptr) {
                             std::nanf(""));
     se::DeviceMemory<float> nan_ptr{
         se::DeviceMemoryBase{static_cast<float*>(ptr), req_size}};
-    if (!stream_exec_->SynchronousMemcpy(&nan_ptr, &nans[0], req_size)) {
+    Status result = stream_exec_->SynchronousMemcpyH2D(&nans[0], req_size, 
+                                                       &nan_ptr);
+    if (!result.ok()) {
       LOG(ERROR) << "Could not initialize to NaNs";
     }
   }
-- 
GitLab


From 3c80e17e79c1920dbba1201d0c50a818772f7e1f Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Tue, 1 Jan 2019 19:23:23 +0530
Subject: [PATCH 0096/2345] Removed unnecessary spaces

---
 tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc b/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc
index 7268d6cf6a..77737a76b6 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc
@@ -173,7 +173,7 @@ void* GPUNanResetAllocator::AllocateRaw(size_t alignment, size_t num_bytes) {
   se::DeviceMemory<float> nan_ptr{
       se::DeviceMemoryBase{static_cast<float*>(allocated_ptr), req_size}};
 
-  Status result = stream_exec_->SynchronousMemcpyH2D(&nans[0], req_size, 
+  Status result = stream_exec_->SynchronousMemcpyH2D(&nans[0], req_size,
                                                      &nan_ptr);
   if (!result.ok()) {
     LOG(ERROR) << "Could not initialize to NaNs";
@@ -189,7 +189,7 @@ void GPUNanResetAllocator::DeallocateRaw(void* ptr) {
                             std::nanf(""));
     se::DeviceMemory<float> nan_ptr{
         se::DeviceMemoryBase{static_cast<float*>(ptr), req_size}};
-    Status result = stream_exec_->SynchronousMemcpyH2D(&nans[0], req_size, 
+    Status result = stream_exec_->SynchronousMemcpyH2D(&nans[0], req_size,
                                                        &nan_ptr);
     if (!result.ok()) {
       LOG(ERROR) << "Could not initialize to NaNs";
-- 
GitLab


From 5356b6898b512d3202088299ab422208fc5f8bb5 Mon Sep 17 00:00:00 2001
From: Dheeraj Rajaram Reddy <dheeraj98reddy@gmail.com>
Date: Tue, 1 Jan 2019 22:26:40 +0530
Subject: [PATCH 0097/2345] Make required changes

Remove preserve_cardinality
Update license year
Propagate predicate's error to the caller
Inline _transformation_name
---
 .../experimental/take_while_dataset_op.cc     | 46 ++++---------------
 .../core/ops/experimental_dataset_ops.cc      |  1 -
 .../data/experimental/ops/take_while_ops.py   |  8 +---
 3 files changed, 12 insertions(+), 43 deletions(-)

diff --git a/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc b/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
index 271b919bc9..b18cc32051 100644
--- a/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
@@ -1,4 +1,4 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -33,8 +33,6 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
   explicit TakeWhileDatasetOp(OpKernelConstruction* ctx)
       : UnaryDatasetOpKernel(ctx) {
     OP_REQUIRES_OK(ctx, ctx->GetAttr("predicate", &func_));
-    OP_REQUIRES_OK(
-        ctx, ctx->GetAttr("preserve_cardinality", &preserve_cardinality_));
   }
 
   void MakeDataset(OpKernelContext* ctx, DatasetBase* input,
@@ -44,22 +42,19 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
                                                  &captured_func));
     
     // TODO (squadrick): check short-circuit
-    *output = new Dataset(ctx, input, func_, std::move(captured_func), 
-                          preserve_cardinality_);
+    *output = new Dataset(ctx, input, func_, std::move(captured_func));
   }
 
  private:
   class Dataset : public DatasetBase {
    public:
     Dataset(OpKernelContext* ctx, const DatasetBase* input,
-            const NameAttrList& func,
-            std::unique_ptr<CapturedFunction> captured_func,
-            bool preserve_cardinality)
+            const NameAttrList& func, 
+            std::unique_ptr<CapturedFunction> captured_func)
         : DatasetBase(DatasetContext(ctx)),
           input_(input),
           func_(func),
-          captured_func_(std::move(captured_func)),
-          preserve_cardinality_(preserve_cardinality) {
+          captured_func_(std::move(captured_func)) {
       input_->Ref();
     }
 
@@ -74,13 +69,14 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
     const DataTypeVector& output_dtypes() const override {
       return input_->output_dtypes();
     }
+
     const std::vector<PartialTensorShape>& output_shapes() const override {
       return input_->output_shapes();
     }
 
     string DebugString() const override { return "TakeWhileDatasetOp::Dataset"; }
 
-    int64 Cardinality() const override { return input_->Cardinality(); }
+    int64 Cardinality() const override { return kUnknownCardinality; }
 
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
@@ -112,16 +108,11 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
       AttrValue other_arguments_types_attr;
       b->BuildAttrValue(other_arguments_types, &other_arguments_types_attr);
 
-      AttrValue preserve_cardinality_attr;
-      b->BuildAttrValue(preserve_cardinality_, &preserve_cardinality_attr);
-
       TF_RETURN_IF_ERROR(b->AddDataset(
             this, {std::make_pair(0, input_node)},
             {std::make_pair(1, other_arguments)},
             {std::make_pair("predicate", f_attr),
-             std::make_pair("Targuments", other_arguments_types_attr),
-             std::make_pair("preserve_cardinality", 
-                            preserve_cardinality_attr)},
+             std::make_pair("Targuments", other_arguments_types_attr)},
              output));
       return Status::OK();
     }
@@ -168,27 +159,12 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
             return errors::InvalidArgument(
                 "`predicate` must returns a scalar bool tensor.");
           }
-          auto cond = bool_output[0].scalar<bool>()();
-          if (!cond) { // predicate is false
-            *end_of_sequence = true;
-            return Status::OK();
-          }
-        } else if (errors::IsOutOfRange(s)) {
-          if (dataset()->preserve_cardinality_) {
-            // To guarantee that the transformation preserves the cardinality of
-            // the dataset, we convert `OutOfRange` to `InvalidArgument` as the
-            // former may be interpreted by a caller as the end of sequence.
-            return errors::InvalidArgument(
-                "Function invocation produced OutOfRangeError: ",
-                s.error_message());
-          } else {
-            // `f` may deliberately raise `errors::OutOfRange` to indicate
-            // that we should terminate the iteration early.
+          if (!bool_output[0].scalar<bool>()()) {
             *end_of_sequence = true;
             return Status::OK();
           }
         }
-        return s;
+        return s; // propagate error to caller
       }
 
      protected:
@@ -227,11 +203,9 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
     const DatasetBase* const input_;
     const NameAttrList func_;
     const std::unique_ptr<CapturedFunction> captured_func_;
-    const bool preserve_cardinality_;
   };
 
   NameAttrList func_;
-  bool preserve_cardinality_;
 };
 
 REGISTER_KERNEL_BUILDER(Name("ExperimentalTakeWhileDataset").Device(DEVICE_CPU),
diff --git a/tensorflow/core/ops/experimental_dataset_ops.cc b/tensorflow/core/ops/experimental_dataset_ops.cc
index d10a13f06f..9ac50ffc72 100644
--- a/tensorflow/core/ops/experimental_dataset_ops.cc
+++ b/tensorflow/core/ops/experimental_dataset_ops.cc
@@ -360,7 +360,6 @@ REGISTER_OP("ExperimentalTakeWhileDataset")
     .Attr("Targuments: list(type) >= 0")
     .Attr("output_types: list(type) >= 1")
     .Attr("output_shapes: list(shape) >= 1")
-    .Attr("preserve_cardinality: bool = false")
     .SetShapeFn(shape_inference::ScalarShape);
 
 REGISTER_OP("ExperimentalUnbatchDataset")
diff --git a/tensorflow/python/data/experimental/ops/take_while_ops.py b/tensorflow/python/data/experimental/ops/take_while_ops.py
index edbecd0666..224eb8aa77 100644
--- a/tensorflow/python/data/experimental/ops/take_while_ops.py
+++ b/tensorflow/python/data/experimental/ops/take_while_ops.py
@@ -1,4 +1,4 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -36,7 +36,7 @@ class _TakeWhileDataset(dataset_ops.UnaryUnchangedStructureDataset):
     self._input_dataset = input_dataset
     wrapped_func = dataset_ops.StructuredFunctionWrapper(
             predicate,
-            self._transformation_name(),
+            "tf.data.experimental.take_while()",
             dataset=self._input_dataset)
     
     if not wrapped_func.output_structure.is_compatible_with(
@@ -49,16 +49,12 @@ class _TakeWhileDataset(dataset_ops.UnaryUnchangedStructureDataset):
             self._input_dataset._variant_tensor,
             other_arguments=self._predicate.function.captured_inputs,
             predicate=self._predicate.function,
-            preserve_cardinality=True,
             **dataset_ops.flat_structure(self))
     super(_TakeWhileDataset, self).__init__(input_dataset, variant_tensor)
 
   def _functions(self):
       return [self._predicate]
 
-  def _transformation_name(self):
-      return "tf.data.experimental.take_while()"
-
 
 @tf_export("data.experimental.take_while")
 def take_while(predicate):
-- 
GitLab


From 4e07d4ae73de5632bb49514a5b486140bdd8516e Mon Sep 17 00:00:00 2001
From: Dheeraj Rajaram Reddy <dheeraj98reddy@gmail.com>
Date: Tue, 1 Jan 2019 23:47:28 +0530
Subject: [PATCH 0098/2345] Add take_while serialization test

---
 .../take_while_dataset_serialization_test.py  | 45 +++++++++++++++++++
 1 file changed, 45 insertions(+)
 create mode 100644 tensorflow/python/data/experimental/kernel_tests/serialization/take_while_dataset_serialization_test.py

diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/take_while_dataset_serialization_test.py b/tensorflow/python/data/experimental/kernel_tests/serialization/take_while_dataset_serialization_test.py
new file mode 100644
index 0000000000..9b5498cca5
--- /dev/null
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/take_while_dataset_serialization_test.py
@@ -0,0 +1,45 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for the TakeWhileDataset serialization."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.data.experimental.kernel_tests.serialization import dataset_serialization_test_base
+from tensorflow.python.data.experimental.ops import take_while_ops
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.platform import test
+
+
+class TakeWhileDatasetSerializationTest(
+    dataset_serialization_test_base.DatasetSerializationTestBase):
+
+  def _build_dataset(self, num_elements, upper_bound):
+    return dataset_ops.Dataset.range(num_elements).apply(
+              take_while_ops.take_while(lambda x: x < upper_bound))
+
+  def testCore(self):
+    def run_test(num_elem1, num_elem2, upper_bound):
+        self.run_core_tests(lambda: self._build_dataset(num_elem1, upper_bound),
+                            lambda: self._build_dataset(num_elem2, upper_bound), 
+                            upper_bound)
+
+    run_test(23, 10, 7)
+    run_test(10, 50, 0)
+    run_test(25, 30, 25)
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From ab7465e957d2f5b6f8ada982cf91266629092677 Mon Sep 17 00:00:00 2001
From: Dheeraj Rajaram Reddy <dheeraj98reddy@gmail.com>
Date: Wed, 2 Jan 2019 19:23:57 +0530
Subject: [PATCH 0099/2345] Add take_while tests

---
 .../data/experimental/kernel_tests/BUILD      | 22 +++++
 .../kernel_tests/serialization/BUILD          | 18 ++++
 .../kernel_tests/take_while_test.py           | 94 +++++++++++++++++++
 3 files changed, 134 insertions(+)
 create mode 100644 tensorflow/python/data/experimental/kernel_tests/take_while_test.py

diff --git a/tensorflow/python/data/experimental/kernel_tests/BUILD b/tensorflow/python/data/experimental/kernel_tests/BUILD
index 9362a3e8eb..b4196685e2 100644
--- a/tensorflow/python/data/experimental/kernel_tests/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/BUILD
@@ -638,6 +638,28 @@ py_library(
     ],
 )
 
+py_test(
+    name = "take_while_test",
+    size = "small",
+    srcs = ["take_while_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:errors",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:script_ops",
+        "//tensorflow/python:sparse_tensor",
+        "//tensorflow/python/data/experimental/ops:take_while_ops",
+        "//tensorflow/python/data/kernel_tests:test_base",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/eager:context",
+        "//third_party/py/numpy",
+    ],
+)
+
 py_test(
     name = "tf_record_writer_test",
     size = "small",
diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD b/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
index 4a2e28f496..00b05e4d30 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
@@ -666,6 +666,24 @@ py_test(
     ],
 )
 
+py_test(
+    name = "take_while_dataset_serialization_test",
+    size = "small",
+    srcs = ["take_while_dataset_serialization_test.py"],
+    srcs_version = "PY2AND3",
+    tags = [
+        "no_oss",
+        "no_pip",
+        "no_windows",
+    ],
+    deps = [
+        ":dataset_serialization_test_base",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python/data/experimental/ops:take_while_ops",
+        "//tensorflow/python/data/ops:dataset_ops",
+    ],
+)
+
 py_test(
     name = "textline_dataset_serialization_test",
     size = "medium",
diff --git a/tensorflow/python/data/experimental/kernel_tests/take_while_test.py b/tensorflow/python/data/experimental/kernel_tests/take_while_test.py
new file mode 100644
index 0000000000..d561a3cd35
--- /dev/null
+++ b/tensorflow/python/data/experimental/kernel_tests/take_while_test.py
@@ -0,0 +1,94 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for `tf.data.experimental.Dataset.take_while()`."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.data.kernel_tests import test_base
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.experimental.ops import take_while_ops
+from tensorflow.python.framework import errors
+from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
+from tensorflow.python.framework import constant_op
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import functional_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.platform import test
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class TakeWhileTest(test_base.DatasetTestBase):
+
+  def testTakeWhileDataset(self):
+    def do_test(num_elements, window_size):
+      def predicate_func(elem):
+        return array_ops.shape(elem)[0] > (window_size - 1)
+
+      flatten_func = lambda x: dataset_ops.Dataset.from_tensor_slices(x)
+      take_while = take_while_ops.take_while(predicate_func)
+
+      dataset = dataset_ops.Dataset.range(num_elements).batch(window_size)
+      dataset = dataset.apply(take_while).flat_map(flatten_func)
+      
+      self.assertDatasetProduces(dataset, 
+              np.arange(int(num_elements / window_size) * window_size))
+
+    do_test(14, 2)
+    do_test(15, 2)
+    do_test(100, 3)
+
+  def testTakeWhileDatasetRange(self):
+    def get_dataset(num_elemets, upper_bound):
+      return dataset_ops.Dataset.range(num_elemets).apply(
+              take_while_ops.take_while(lambda x: x < upper_bound))
+        
+    def do_test(num_elemets, upper_bound):  
+      self.assertDatasetProduces(get_dataset(num_elemets, upper_bound), 
+              np.arange(upper_bound))
+    
+    def out_of_bounds(num_elemets, upper_bound):
+      with self.assertRaises(errors.OutOfRangeError):
+        self.assertDatasetProduces(get_dataset(num_elemets, upper_bound), 
+                np.arange(upper_bound))
+
+    do_test(10, 2)
+    do_test(16, 7)
+    do_test(100, 99)
+    out_of_bounds(100, 101)
+    out_of_bounds(0, 1)
+
+  def testTakeWhileDatasetString(self):
+    def stringNotEquals(string):
+        return lambda x: math_ops.not_equal(x, constant_op.constant(string))
+
+    string = ["this", "is", "the", "test", "for", "strings"]
+    dataset = dataset_ops.Dataset.from_tensor_slices(string).apply(
+            take_while_ops.take_while(stringNotEquals("test")))
+    
+    next_element = self.getNext(dataset)
+    self.assertEqual(b"this", self.evaluate(next_element()))
+    self.assertEqual(b"is", self.evaluate(next_element()))
+    self.assertEqual(b"the", self.evaluate(next_element()))
+
+    with self.assertRaises(errors.OutOfRangeError):
+        self.assertEqual(b"test", self.evaluate(next_element()))
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From 276a456b3551a329881e0104edf8ea848d446033 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 2 Jan 2019 13:25:11 -0800
Subject: [PATCH 0100/2345] Remove unused imports

---
 tensorflow/contrib/tensorrt/test/identity_output_test.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/test/identity_output_test.py b/tensorflow/contrib/tensorrt/test/identity_output_test.py
index da7d70876d..b376e30476 100644
--- a/tensorflow/contrib/tensorrt/test/identity_output_test.py
+++ b/tensorflow/contrib/tensorrt/test/identity_output_test.py
@@ -29,9 +29,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import nn
 from tensorflow.python.platform import test
 
 
-- 
GitLab


From f92510eb61a469c0ea4652f8cd2582f71a821d80 Mon Sep 17 00:00:00 2001
From: Clayne Robison <clayne.b.robison@intel.com>
Date: Wed, 2 Jan 2019 15:25:54 -0700
Subject: [PATCH 0101/2345] [Intel MKL] Updating the public MKL Dockerfiles to
 reflect Ubuntu 18.04 use of python 3.6; also updating the default TensorFlow
 branch.

---
 tensorflow/tools/docker/Dockerfile.devel-mkl  | 41 ++++++-------------
 .../tools/docker/Dockerfile.devel-mkl-horovod | 30 +++++++-------
 tensorflow/tools/docker/Dockerfile.mkl        | 18 ++++----
 .../tools/docker/Dockerfile.mkl-horovod       | 26 ++++++------
 4 files changed, 47 insertions(+), 68 deletions(-)

diff --git a/tensorflow/tools/docker/Dockerfile.devel-mkl b/tensorflow/tools/docker/Dockerfile.devel-mkl
index 4eefd31d00..32aa00bdff 100755
--- a/tensorflow/tools/docker/Dockerfile.devel-mkl
+++ b/tensorflow/tools/docker/Dockerfile.devel-mkl
@@ -3,13 +3,18 @@ FROM ubuntu:18.04
 LABEL maintainer="Clayne Robison <clayne.b.robison@intel.com>"
 
 # These parameters can be overridden by parameterized_docker_build.sh
-ARG TF_BUILD_VERSION=r1.12
+ARG TF_BUILD_VERSION=r1.13
 ARG PYTHON="python"
 ARG PYTHON3_DEV=""
 ARG WHL_DIR="/tmp/pip"
 ARG PIP="pip"
 
-RUN apt-get update && apt-get install -y --no-install-recommends \
+RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
+        ${PYTHON} \
+        ${PYTHON}-dev \
+        ${PYTHON}-pip \
+        ${PYTHON}-setuptools \
+        ${PYTHON}-wheel \
         build-essential \
         curl \
         git \
@@ -17,35 +22,20 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         libfreetype6-dev \
         libhdf5-serial-dev \
         libpng-dev \
-        libzmq3-dev \
         libssl-dev \
+        libzmq3-dev \
+        openjdk-8-jdk \
+        openjdk-8-jre-headless \
         pkg-config \
         rsync \
         software-properties-common \
         unzip \
         zip \
         zlib1g-dev \
-        openjdk-8-jdk \
-        openjdk-8-jre-headless
-
-#install Python 3
-RUN if [ ${PYTHON} = "python3.6" ]; then \
-      curl https://www.python.org/ftp/python/3.6.5/Python-3.6.5.tar.xz -o /opt/python.tar.xz && \
-      cd /opt && tar xvf python.tar.xz && \
-      cd /opt/*/ && ./configure && \
-      make && make install; \
-    else \
-      apt-get install -y --no-install-recommends \
-        python-dev \
-        ${PYTHON3_DEV}; \
-    fi
-
-RUN    apt-get clean && \
+        && \
+    apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py && \
-    ${PYTHON} get-pip.py && \
-    rm get-pip.py
 
 RUN ${PIP} --no-cache-dir install \
         Pillow \
@@ -57,17 +47,12 @@ RUN ${PIP} --no-cache-dir install \
         matplotlib \
         mock \
         numpy \
+        pandas \
         scipy \
         sklearn \
-        pandas \
         && \
     ${PYTHON} -m ipykernel.kernelspec
 
-RUN if [ "${PYTHON}" = "python3" ]; then \
-      ln -s -f /usr/bin/python3 /usr/bin/python; \
-  elif [ "${PYTHON}" = "python3.6" ]; then \
-      ln -s -f /usr/local/bin/python3.6 /usr/bin/python; \
-  fi
 
 # Set up our notebook config.
 COPY jupyter_notebook_config.py /root/.jupyter/
diff --git a/tensorflow/tools/docker/Dockerfile.devel-mkl-horovod b/tensorflow/tools/docker/Dockerfile.devel-mkl-horovod
index 3810daefa5..21140918aa 100755
--- a/tensorflow/tools/docker/Dockerfile.devel-mkl-horovod
+++ b/tensorflow/tools/docker/Dockerfile.devel-mkl-horovod
@@ -3,42 +3,43 @@ FROM ubuntu:18.04
 LABEL maintainer="Cong Xu <cong.xu@intel.com>"
 
 # These parameters can be overridden by parameterized_docker_build.sh
-ARG TF_BUILD_VERSION=r1.11
+ARG TF_BUILD_VERSION=r1.13
 ARG PYTHON="python"
 ARG PYTHON3_DEV=""
 ARG WHL_DIR="/tmp/pip"
 ARG PIP="pip"
 
-RUN apt-get update && apt-get install -y --no-install-recommends \
+
+RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
+        ${PYTHON} \
+        ${PYTHON}-dev \
+        ${PYTHON}-pip \
+        ${PYTHON}-setuptools \
+        ${PYTHON}-wheel \
         build-essential \
         curl \
         git \
         libcurl3-dev \
         libfreetype6-dev \
         libhdf5-serial-dev \
+        libnuma-dev \
         libpng-dev \
         libzmq3-dev \
+        openjdk-8-jdk \
+        openjdk-8-jre-headless \
+        openssh-client \
+        openssh-server \
         pkg-config \
-        python-dev \
-        ${PYTHON3_DEV} \
         rsync \
         software-properties-common \
         unzip \
+        wget \
         zip \
         zlib1g-dev \
-        openjdk-8-jdk \
-        openjdk-8-jre-headless \
-        wget \
-        libnuma-dev \
-        openssh-client \
-        openssh-server \
         && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py && \
-    ${PYTHON} get-pip.py && \
-    rm get-pip.py
 
 RUN ${PIP} --no-cache-dir install \
         Pillow \
@@ -56,9 +57,6 @@ RUN ${PIP} --no-cache-dir install \
         && \
     ${PYTHON} -m ipykernel.kernelspec
 
-RUN if [ "${PYTHON}" = "python3" ]; then \
-  ln -s -f /usr/bin/python3 /usr/bin/python; \
-  fi
 
 # Set up our notebook config.
 COPY jupyter_notebook_config.py /root/.jupyter/
diff --git a/tensorflow/tools/docker/Dockerfile.mkl b/tensorflow/tools/docker/Dockerfile.mkl
index dad27697fa..3f7729ba59 100755
--- a/tensorflow/tools/docker/Dockerfile.mkl
+++ b/tensorflow/tools/docker/Dockerfile.mkl
@@ -6,13 +6,18 @@ LABEL maintainer="Clayne Robison <clayne.b.robison@intel.com>"
 ARG TF_WHL_URL
 
 # Optional parameters
-ARG TF_BUILD_VERSION=r1.9
+ARG TF_BUILD_VERSION=r1.13
 ARG PYTHON="python"
 ARG PYTHON_DEV="python-dev"
 ARG PIP="pip"
 
 # Pick up some TF dependencies
-RUN apt-get update && apt-get install -y --no-install-recommends \
+RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
+        ${PYTHON} \
+        ${PYTHON}-dev \
+        ${PYTHON}-pip \
+        ${PYTHON}-setuptools \
+        ${PYTHON}-wheel \
         build-essential \
         curl \
         libfreetype6-dev \
@@ -20,8 +25,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         libpng-dev \
         libzmq3-dev \
         pkg-config \
-        ${PYTHON} \
-        ${PYTHON_DEV} \
         rsync \
         software-properties-common \
         unzip \
@@ -29,9 +32,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-RUN curl -O https://bootstrap.pypa.io/get-pip.py && \
-    ${PYTHON} get-pip.py && \
-    rm get-pip.py
 
 RUN ${PIP} --no-cache-dir install \
         Pillow \
@@ -48,13 +48,11 @@ RUN ${PIP} --no-cache-dir install \
         && \
     ${PYTHON} -m ipykernel.kernelspec
 
+
 COPY ${TF_WHL_URL} /
 RUN ${PIP} install --no-cache-dir --force-reinstall /${TF_WHL_URL} && \
     rm -rf /${TF_WHL_URL}
 
-RUN if [ "${PYTHON}" = "python3" ]; then \
-  ln -s -f /usr/bin/python3 /usr/bin/python; \
-  fi
 
 # Set up our notebook config.
 COPY jupyter_notebook_config.py /root/.jupyter/
diff --git a/tensorflow/tools/docker/Dockerfile.mkl-horovod b/tensorflow/tools/docker/Dockerfile.mkl-horovod
index 19dc45c62c..b0afd63727 100755
--- a/tensorflow/tools/docker/Dockerfile.mkl-horovod
+++ b/tensorflow/tools/docker/Dockerfile.mkl-horovod
@@ -6,36 +6,36 @@ LABEL maintainer="Cong Xu <cong.xu@intel.com>"
 ARG TF_WHL_URL
 
 # Optional parameters
-ARG TF_BUILD_VERSION=r1.11
+ARG TF_BUILD_VERSION=r1.13
 ARG PYTHON="python"
 ARG PYTHON_DEV="python-dev"
 ARG PIP="pip"
 
 # Pick up some TF dependencies
-RUN apt-get update && apt-get install -y --no-install-recommends \
+# RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
+        ${PYTHON} \
+        ${PYTHON}-dev \
+        ${PYTHON}-pip \
+        ${PYTHON}-setuptools \
+        ${PYTHON}-wheel \
         build-essential \
         curl \
         libfreetype6-dev \
         libhdf5-serial-dev \
+        libnuma-dev \
         libpng-dev \
         libzmq3-dev \
+        openssh-client \
+        openssh-server \
         pkg-config \
-        python \
-        ${PYTHON_DEV} \
         rsync \
         software-properties-common \
         unzip \
         wget \
-        libnuma-dev \
-        openssh-client \
-        openssh-server \
         && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-RUN curl -O https://bootstrap.pypa.io/get-pip.py && \
-    python get-pip.py && \
-    rm get-pip.py
 
 RUN ${PIP} --no-cache-dir install \
         Pillow \
@@ -50,15 +50,13 @@ RUN ${PIP} --no-cache-dir install \
         scipy \
         sklearn \
         && \
-    python -m ipykernel.kernelspec
+    ${PYTHON} -m ipykernel.kernelspec
+
 
 COPY ${TF_WHL_URL} /
 RUN ${PIP} install --no-cache-dir --force-reinstall /${TF_WHL_URL} && \
     rm -rf /${TF_WHL_URL}
 
-RUN if [ "${PYTHON}" = "python3" ]; then \
-  ln -s -f /usr/bin/python3 /usr/bin/python; \
-  fi
 
 # Set up our notebook config.
 COPY jupyter_notebook_config.py /root/.jupyter/
-- 
GitLab


From 85ca0e9b815fc8438f1c10379c18393783021e15 Mon Sep 17 00:00:00 2001
From: "Joshua V. Dillon" <jvdillon@google.com>
Date: Wed, 2 Jan 2019 14:31:31 -0800
Subject: [PATCH 0102/2345] Remove tf.contrib.timeseries dependency on TF
 distributions.

PiperOrigin-RevId: 227582617
---
 .../timeseries/python/timeseries/BUILD        |  3 +-
 .../timeseries/python/timeseries/ar_model.py  | 19 ++++---------
 .../python/timeseries/math_utils.py           | 28 +++++++++++++++++++
 .../timeseries/state_space_models/BUILD       |  2 --
 .../filtering_postprocessor.py                |  8 ++----
 .../state_space_models/kalman_filter.py       |  9 +++---
 6 files changed, 43 insertions(+), 26 deletions(-)

diff --git a/tensorflow/contrib/timeseries/python/timeseries/BUILD b/tensorflow/contrib/timeseries/python/timeseries/BUILD
index 4b90b596b2..a0c3204d41 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/BUILD
+++ b/tensorflow/contrib/timeseries/python/timeseries/BUILD
@@ -361,9 +361,10 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":feature_keys",
+        ":math_utils",
         ":model",
         ":model_utils",
-        "//tensorflow/contrib/distributions:distributions_py",
+        "//tensorflow/contrib/rnn:rnn_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:check_ops",
         "//tensorflow/python:constant_op",
diff --git a/tensorflow/contrib/timeseries/python/timeseries/ar_model.py b/tensorflow/contrib/timeseries/python/timeseries/ar_model.py
index bcadf4094e..a8d5e1a49d 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/ar_model.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/ar_model.py
@@ -18,9 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib import distributions
-
 from tensorflow.contrib.rnn.python.ops import lstm_ops
+from tensorflow.contrib.timeseries.python.timeseries import math_utils
 from tensorflow.contrib.timeseries.python.timeseries import model
 from tensorflow.contrib.timeseries.python.timeseries import model_utils
 from tensorflow.contrib.timeseries.python.timeseries.feature_keys import PredictionFeatures
@@ -462,8 +461,8 @@ class ARModel(model.TimeSeriesModel):
     if self.loss == ARModel.NORMAL_LIKELIHOOD_LOSS:
       covariance = prediction_ops["covariance"]
       sigma = math_ops.sqrt(gen_math_ops.maximum(covariance, 1e-5))
-      normal = distributions.Normal(loc=targets, scale=sigma)
-      loss_op = -math_ops.reduce_sum(normal.log_prob(prediction))
+      loss_op = -math_ops.reduce_sum(
+          math_utils.normal_log_prob(targets, sigma, prediction))
     else:
       assert self.loss == ARModel.SQUARED_LOSS, self.loss
       loss_op = math_ops.reduce_sum(math_ops.square(prediction - targets))
@@ -965,16 +964,11 @@ class AnomalyMixtureARModel(ARModel):
       anomaly_variance = prediction_ops["anomaly_params"]
       anomaly_sigma = math_ops.sqrt(
           gen_math_ops.maximum(anomaly_variance, 1e-5))
-      normal = distributions.Normal(loc=targets, scale=anomaly_sigma)
-      log_prob = normal.log_prob(prediction)
+      log_prob = math_utils.normal_log_prob(targets, anomaly_sigma, prediction)
     else:
       assert self._anomaly_distribution == AnomalyMixtureARModel.CAUCHY_ANOMALY
       anomaly_scale = prediction_ops["anomaly_params"]
-      cauchy = distributions.StudentT(
-          df=array_ops.ones([], dtype=anomaly_scale.dtype),
-          loc=targets,
-          scale=anomaly_scale)
-      log_prob = cauchy.log_prob(prediction)
+      log_prob = math_utils.cauchy_log_prob(targets, anomaly_scale, prediction)
     return log_prob
 
   def loss_op(self, targets, prediction_ops):
@@ -983,8 +977,7 @@ class AnomalyMixtureARModel(ARModel):
     covariance = prediction_ops["covariance"]
     # Normal data log probability.
     sigma = math_ops.sqrt(gen_math_ops.maximum(covariance, 1e-5))
-    normal1 = distributions.Normal(loc=targets, scale=sigma)
-    log_prob1 = normal1.log_prob(prediction)
+    log_prob1 = math_utils.normal_log_prob(targets, sigma, prediction)
     log_prob1 += math_ops.log(1 - self._anomaly_prior_probability)
     # Anomaly log probability.
     log_prob2 = self._anomaly_log_prob(targets, prediction_ops)
diff --git a/tensorflow/contrib/timeseries/python/timeseries/math_utils.py b/tensorflow/contrib/timeseries/python/timeseries/math_utils.py
index aab3306438..b7375e5055 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/math_utils.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/math_utils.py
@@ -21,6 +21,8 @@ from __future__ import print_function
 import collections
 import math
 
+import numpy as np
+
 from tensorflow.contrib import lookup
 from tensorflow.contrib.layers.python.layers import layers
 
@@ -43,6 +45,32 @@ from tensorflow.python.ops import variable_scope
 from tensorflow.python.util import nest
 
 
+def normal_log_prob(loc, scale, x):
+  """Computes the Normal log pdf."""
+  z = (x - loc) / scale
+  return -0.5 * (math_ops.square(z)
+                 + np.log(2. * np.pi) + math_ops.log(scale))
+
+
+def cauchy_log_prob(loc, scale, x):
+  """Computes the Cauchy log pdf."""
+  z = (x - loc) / scale
+  return (-np.log(np.pi) - math_ops.log(scale) -
+          math_ops.log1p(math_ops.square(z)))
+
+
+def mvn_tril_log_prob(loc, scale_tril, x):
+  """Computes the MVN log pdf under tril scale. Doesn't handle batches."""
+  x0 = x - loc
+  z = linalg_ops.matrix_triangular_solve(
+      scale_tril, x0[..., array_ops.newaxis])[..., 0]
+  log_det_cov = 2. * math_ops.reduce_sum(math_ops.log(
+      array_ops.matrix_diag_part(scale_tril)), axis=-1)
+  d = math_ops.cast(array_ops.shape(scale_tril)[-1], log_det_cov.dtype)
+  return -0.5 * (math_ops.reduce_sum(math_ops.square(z), axis=-1)
+                 + d * np.log(2. * np.pi) + log_det_cov)
+
+
 def clip_covariance(
     covariance_matrix, maximum_variance_ratio, minimum_variance):
   """Enforce constraints on a covariance matrix to improve numerical stability.
diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD
index 125750e763..cf5e749042 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD
+++ b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD
@@ -78,7 +78,6 @@ py_library(
     srcs = ["kalman_filter.py"],
     srcs_version = "PY2AND3",
     deps = [
-        "//tensorflow/contrib/distributions:distributions_py",
         "//tensorflow/contrib/timeseries/python/timeseries:math_utils",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:control_flow_ops",
@@ -235,7 +234,6 @@ py_library(
     srcs = ["filtering_postprocessor.py"],
     srcs_version = "PY2AND3",
     deps = [
-        "//tensorflow/contrib/distributions:distributions_py",
         "//tensorflow/contrib/timeseries/python/timeseries:math_utils",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:check_ops",
diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/filtering_postprocessor.py b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/filtering_postprocessor.py
index e9e2ac0aaf..3fa2fbd9f7 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/filtering_postprocessor.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/filtering_postprocessor.py
@@ -22,8 +22,6 @@ import abc
 
 import six
 
-from tensorflow.contrib import distributions
-
 from tensorflow.contrib.timeseries.python.timeseries import math_utils
 
 from tensorflow.python.framework import dtypes
@@ -91,10 +89,10 @@ def cauchy_alternative_to_gaussian(current_times, current_values, outputs):
   """
   del current_times  # unused
   cauchy_scale = math_utils.entropy_matched_cauchy_scale(outputs["covariance"])
-  individual_log_pdfs = distributions.StudentT(
-      df=array_ops.ones([], dtype=current_values.dtype),
+  individual_log_pdfs = math_utils.cauchy_log_prob(
       loc=outputs["mean"],
-      scale=cauchy_scale).log_prob(current_values)
+      scale=cauchy_scale,
+      x=current_values)
   return math_ops.reduce_sum(individual_log_pdfs, axis=1)
 
 
diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/kalman_filter.py b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/kalman_filter.py
index a614386121..c0ec797bc5 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/kalman_filter.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/kalman_filter.py
@@ -18,8 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib import distributions
-
 from tensorflow.contrib.timeseries.python.timeseries import math_utils
 
 from tensorflow.python.framework import dtypes
@@ -137,9 +135,10 @@ class KalmanFilter(object):
     with ops.control_dependencies([non_negative_assert]):
       observation_covariance_cholesky = linalg_ops.cholesky(
           symmetrized_observation_covariance)
-    log_prediction_prob = distributions.MultivariateNormalTriL(
-        predicted_observation, observation_covariance_cholesky).log_prob(
-            observation)
+    log_prediction_prob = math_utils.mvn_tril_log_prob(
+        loc=predicted_observation,
+        scale_tril=observation_covariance_cholesky,
+        x=observation)
     (posterior_state,
      posterior_state_var) = self.posterior_from_prior_state(
          prior_state=estimated_state,
-- 
GitLab


From 790a635a044c9b33103f1e02f57faf08c86eee36 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Wed, 2 Jan 2019 14:53:55 -0800
Subject: [PATCH 0103/2345] Stop Model.save_weights from printing a deprication
 warning (TF format)

Also has it save relative paths in the checkpoint proto, which removes a behavior difference between it and CheckpointManager.

PiperOrigin-RevId: 227586345
---
 tensorflow/python/keras/engine/network.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/engine/network.py b/tensorflow/python/keras/engine/network.py
index 0837ce6780..8e130aef40 100644
--- a/tensorflow/python/keras/engine/network.py
+++ b/tensorflow/python/keras/engine/network.py
@@ -1382,9 +1382,10 @@ class Network(base_layer.Layer):
             % (optimizer,))
       self._checkpointable_saver.save(filepath, session=session)
       # Record this checkpoint so it's visible from tf.train.latest_checkpoint.
-      checkpoint_management.update_checkpoint_state(
+      checkpoint_management.update_checkpoint_state_internal(
           save_dir=os.path.dirname(filepath),
           model_checkpoint_path=filepath,
+          save_relative_paths=True,
           all_model_checkpoint_paths=[filepath])
 
   def load_weights(self, filepath, by_name=False):
-- 
GitLab


From a7a3bbfcbf53381631dded0e293031e93d056ebc Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Wed, 2 Jan 2019 15:00:14 -0800
Subject: [PATCH 0104/2345] [Grappler] Cleanup function/function_optimizer

1. Migrate to absl containers
2. Compute function signature hash without allocating temporary sorted containers

PiperOrigin-RevId: 227587343
---
 tensorflow/core/grappler/optimizers/BUILD     |   2 +
 .../grappler/optimizers/function_optimizer.cc | 152 +++++++++---------
 .../grappler/optimizers/meta_optimizer.cc     |  19 +--
 tensorflow/core/grappler/utils/BUILD          |   4 +
 tensorflow/core/grappler/utils/functions.cc   |  80 ++++-----
 tensorflow/core/grappler/utils/functions.h    |  52 +++---
 .../core/grappler/utils/functions_test.cc     |   6 +-
 7 files changed, 155 insertions(+), 160 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 3a1ab28dae..512c3b07b4 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -151,6 +151,8 @@ cc_library(
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/utils:functions",
         "@com_google_absl//absl/algorithm:container",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
     ],
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index 73c950b3fc..d074676c3d 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -15,10 +15,11 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/function_optimizer.h"
 
-#include <unordered_map>
 #include <vector>
 
 #include "absl/algorithm/container.h"
+#include "absl/container/flat_hash_map.h"
+#include "absl/container/flat_hash_set.h"
 #include "absl/memory/memory.h"
 #include "absl/strings/str_replace.h"
 #include "absl/strings/substitute.h"
@@ -163,10 +164,10 @@ struct FunctionSpecializationSignature {
 
   string func_name;
   bool is_in_fetch_set;
-  gtl::FlatSet<OutputPort> active_outputs;
-  std::unordered_map<string, DataType> type_parameters;
-  std::unordered_map<string, AttrValue> body_parameters;
-  std::unordered_map<InputPort, string> const_inputs;
+  absl::flat_hash_set<OutputPort> active_outputs;
+  absl::flat_hash_map<string, DataType> type_parameters;
+  absl::flat_hash_map<string, AttrValue> body_parameters;
+  absl::flat_hash_map<InputPort, string> const_inputs;
 
   bool operator==(const FunctionSpecializationSignature& other) const {
     bool equals = func_name == other.func_name &&
@@ -189,48 +190,45 @@ struct FunctionSpecializationSignature {
     return true;
   }
 
-  // TODO(ezhulenev): Migrate to AbslHashValue.
-  // TODO(ezhulenev): Optimize performance by computing hashes of unordered
-  // values first, and then compute a hash of sorted hashes.
-  struct Hash {
-    uint64 operator()(FunctionSpecializationSignature const& s) const {
-      uint64 h = Hash64(s.func_name);
-      h = Hash64Combine(std::hash<bool>()(s.is_in_fetch_set), h);
-
-      // Use std::set/std::map for deterministic iteration order.
-
-      std::set<OutputPort> active_outputs(s.active_outputs.begin(),
-                                          s.active_outputs.end());
-      for (const auto& active_output : active_outputs) {
-        h = Hash64Combine(std::hash<int>()(active_output), h);
-      }
-
-      std::map<string, DataType> types(s.type_parameters.begin(),
-                                       s.type_parameters.end());
-      for (const auto& pair : types) {
-        AttrValue attr_value;
-        attr_value.set_type(pair.second);
-        h = Hash64Combine(Hash64(pair.first), h);
-        h = Hash64Combine(AttrValueHash(attr_value), h);
-      }
-
-      std::map<string, AttrValue> body(s.body_parameters.begin(),
-                                       s.body_parameters.end());
-      for (const auto& pair : body) {
-        h = Hash64Combine(Hash64(pair.first), h);
-        h = Hash64Combine(FastAttrValueHash(pair.second), h);
-      }
-
-      std::map<InputPort, string> inputs(s.const_inputs.begin(),
-                                         s.const_inputs.end());
-      for (const auto& pair : inputs) {
-        h = Hash64Combine(std::hash<int>()(pair.first), h);
-        h = Hash64Combine(Hash64(pair.second), h);
-      }
-
-      return h;
-    }
-  };
+  template <typename H>
+  friend H AbslHashValue(H h, const FunctionSpecializationSignature& s) {
+    H base = H::combine(std::move(h), s.func_name, s.is_in_fetch_set);
+
+    // First pre-compute hashes for all values in collections with
+    // non-deterministic iteration order.
+    std::vector<uint64> hashes;
+    hashes.reserve(s.active_outputs.size()         //
+                   + s.type_parameters.size() * 2  //
+                   + s.body_parameters.size() * 2  //
+                   + s.const_inputs.size() * 2);
+
+    absl::c_transform(s.active_outputs, std::back_inserter(hashes),
+                      hash<OutputPort>());
+
+    using TypeParam = std::pair<const string, DataType>;
+    absl::c_for_each(s.type_parameters, [&hashes](const TypeParam& type_param) {
+      AttrValue attr_value;
+      attr_value.set_type(type_param.second);
+      hashes.push_back(Hash64(type_param.first));
+      hashes.push_back(AttrValueHash(attr_value));
+    });
+
+    using BodyParam = std::pair<const string, AttrValue>;
+    absl::c_for_each(s.body_parameters, [&hashes](const BodyParam& body_param) {
+      hashes.push_back(Hash64(body_param.first));
+      hashes.push_back(FastAttrValueHash(body_param.second));
+    });
+
+    using ConstInput = std::pair<const InputPort, string>;
+    absl::c_for_each(s.const_inputs, [&hashes](const ConstInput& const_input) {
+      hashes.push_back(hash<InputPort>()(const_input.first));
+      hashes.push_back(Hash64(const_input.second));
+    });
+
+    // Combine all pre-computed hashes in a deterministic order.
+    absl::c_sort(hashes);
+    return H::combine_contiguous(std::move(base), hashes.data(), hashes.size());
+  }
 };
 
 struct FunctionSpecialization {
@@ -238,13 +236,13 @@ struct FunctionSpecialization {
   // True if the function caller node is in GrapplerItem fetch set.
   bool is_in_fetch_set;
   // Names of the tensors that were pushed down into the function body.
-  gtl::FlatSet<string> const_inputs;
+  absl::flat_hash_set<string> const_inputs;
   // Control dependencies of pushed down const inputs have to be attached to
   // function caller node.
-  gtl::FlatSet<string> control_deps;
+  absl::flat_hash_set<string> control_deps;
   // Output tensors (ports) that consumed by other nodes in the graph or in a
   // GrapplerItem fetch set.
-  gtl::FlatSet<int> active_outputs;
+  absl::flat_hash_set<int> active_outputs;
   // Mapping from original function output port to the output port of
   // specialized function. If function specialization changes the number of
   // function outputs it's required to update all node consumers.
@@ -285,12 +283,13 @@ class FunctionOptimizerContext {
     return flr_;
   }
 
-  const gtl::FlatMap<SafeTensorId, SafeTensorId, SafeTensorId::Hasher>&
+  const absl::flat_hash_map<SafeTensorId, SafeTensorId, SafeTensorId::Hasher>&
   tensor_mapping() const {
     return tensor_mapping_;
   }
 
-  const gtl::FlatMap<string, std::vector<string>>& control_overrides() const {
+  const absl::flat_hash_map<string, std::vector<string>>& control_overrides()
+      const {
     return control_overrides_;
   }
 
@@ -298,7 +297,9 @@ class FunctionOptimizerContext {
 
   const string& grappler_item_id() const { return grappler_item_id_; }
 
-  const gtl::FlatSet<string>& fetch_tensors() const { return fetch_tensors_; }
+  const absl::flat_hash_set<string>& fetch_tensors() const {
+    return fetch_tensors_;
+  }
 
   const DeviceSet* devices() const {
     // Create fake devices lazily only if we need a DeviceSet.
@@ -365,7 +366,7 @@ class FunctionOptimizerContext {
 
  private:
   void InitializeTrulyConstNodes(const GrapplerItem& item) {
-    gtl::FlatSet<string> feed_nodes;
+    absl::flat_hash_set<string> feed_nodes;
     for (const auto& feed : item.feed) {
       feed_nodes.insert(NodeName(feed.first));
     }
@@ -411,7 +412,7 @@ class FunctionOptimizerContext {
   FunctionLibraryRuntime* flr_ = nullptr;
 
   // Fully defined names of the devices available to the GrapplerItem.
-  const gtl::FlatSet<string> available_device_names_;
+  const absl::flat_hash_set<string> available_device_names_;
 
   // List of available `FakedDevices` (lazily initialized, see devices()).
   mutable std::vector<std::unique_ptr<Device>> available_devices_;
@@ -421,16 +422,15 @@ class FunctionOptimizerContext {
   mutable DeviceSet available_device_set_;
 
   // Nodes that are Const and not in feed.
-  std::unordered_map<string, const NodeDef*> truly_const_nodes_;
+  absl::flat_hash_map<string, const NodeDef*> truly_const_nodes_;
   // Specialized functions.
-  std::unordered_map<FunctionSpecializationSignature,
-                     const FunctionSpecialization,
-                     FunctionSpecializationSignature::Hash>
+  absl::flat_hash_map<FunctionSpecializationSignature,
+                      const FunctionSpecialization>
       specialized_functions_;
 
   // GrapplerItem.fetch is a vector of tensors.
-  gtl::FlatSet<string> fetch_tensors_;  // format: node_name:port
-  gtl::FlatSet<string> fetch_nodes_;    // format: node_name
+  absl::flat_hash_set<string> fetch_tensors_;  // format: node_name:port
+  absl::flat_hash_set<string> fetch_nodes_;    // format: node_name
 
   // After function inlining and specialization, the optimized graph might be in
   // invalid state, nodes can read from non-existing function call nodes that
@@ -439,7 +439,7 @@ class FunctionOptimizerContext {
   //
   // Tensor mapping that has to be applied to the graph after all functions
   // optimizations (invalidated tensor id -> optimized graph tensor id).
-  gtl::FlatMap<SafeTensorId, SafeTensorId, SafeTensorId::Hasher>
+  absl::flat_hash_map<SafeTensorId, SafeTensorId, SafeTensorId::Hasher>
       tensor_mapping_;
 
   // When we inline a function into the optimized graph, we no longer have the
@@ -448,7 +448,7 @@ class FunctionOptimizerContext {
   // to all side-effectful ops inside the function body.
   //
   // Invalidated function call node name -> Inlined side-effectful nodes
-  gtl::FlatMap<string, std::vector<string>> control_overrides_;
+  absl::flat_hash_map<string, std::vector<string>> control_overrides_;
 
   // Use graph view to find active outputs of the function caller nodes.
   GraphView graph_view_;
@@ -472,10 +472,10 @@ const FunctionDef* FindFunctionCall(const FunctionOptimizerContext& ctx,
   return ctx.function_library().Find(node.op());
 }
 
-gtl::FlatSet<int> GetActiveOutputs(const NodeDef& node,
-                                   const FunctionOptimizerContext& ctx,
-                                   int size_hint = 0) {
-  gtl::FlatSet<int> active_outputs;
+absl::flat_hash_set<int> GetActiveOutputs(const NodeDef& node,
+                                          const FunctionOptimizerContext& ctx,
+                                          int size_hint = 0) {
+  absl::flat_hash_set<int> active_outputs;
   active_outputs.reserve(static_cast<size_t>(size_hint));
 
   // 1. Output can be consumed by the other graph node.
@@ -508,7 +508,7 @@ bool HasUnusedOutputs(const NodeDef& func_node, const FunctionDef& func,
   // number of output args is the same as number of possible function caller
   // node outputs.
   int num_outputs = func.signature().output_arg_size();
-  const gtl::FlatSet<int> active_outputs =
+  const absl::flat_hash_set<int> active_outputs =
       GetActiveOutputs(func_node, ctx, /*size_hind*/ num_outputs);
 
   return active_outputs.size() != num_outputs;
@@ -519,7 +519,7 @@ bool HasUnusedOutputs(const NodeDef& func_node, const FunctionDef& func,
 FunctionDefLibrary PruneFunctionLibrary(const FunctionLibraryDefinition& flib,
                                         const GraphDef& optimized_graph) {
   FunctionLibraryDefinition pruned_flib =
-      ReachableFunctionLibraryDefinition(flib, optimized_graph);
+      flib.ReachableDefinitions(optimized_graph);
 
   int pruned_functions = static_cast<int>(pruned_flib.num_functions()) -
                          static_cast<int>(flib.num_functions());
@@ -534,8 +534,8 @@ FunctionDefLibrary PruneFunctionLibrary(const FunctionLibraryDefinition& flib,
 Status PushDownConstInputs(const NodeDef& func_node,
                            const FunctionOptimizerContext& ctx,
                            GrapplerFunctionItem* item,
-                           gtl::FlatSet<string>* const_inputs,
-                           gtl::FlatSet<string>* control_deps) {
+                           absl::flat_hash_set<string>* const_inputs,
+                           absl::flat_hash_set<string>* control_deps) {
   // Record node control dependencies in the control_deps set.
   const auto record_control_deps = [&](const NodeDef* const_input) {
     for (int i = const_input->input_size() - 1; i >= 0; --i) {
@@ -585,7 +585,7 @@ void RemovePushedDownConstInputs(const FunctionSpecialization& specialization,
 
   // Attach control dependencies of pushed down const input to the caller node.
   if (!specialization.control_deps.empty()) {
-    gtl::FlatSet<string> existing_control_deps;
+    absl::flat_hash_set<string> existing_control_deps;
 
     for (const string& input : keep_inputs) {
       existing_control_deps.insert(AsControlDependency(NodeName(input)));
@@ -797,8 +797,8 @@ Status SpecializeFunction(const NodeDef& func_node, const FunctionDef& func,
 
   // Push const inputs into the function body, and keep track of their control
   // dependencies.
-  gtl::FlatSet<string> const_inputs;
-  gtl::FlatSet<string> control_deps;
+  absl::flat_hash_set<string> const_inputs;
+  absl::flat_hash_set<string> control_deps;
   TF_RETURN_IF_ERROR(PushDownConstInputs(func_node, *ctx, &item, &const_inputs,
                                          &control_deps));
 
@@ -1005,7 +1005,7 @@ Status InlineDirectFunctionCall(const NodeDef& func_node,
 
   // Mapping from input placeholder name to function input position.
   int idx = 0;
-  std::unordered_map<string, int> input_placeholders_idx;
+  absl::flat_hash_map<string, int> input_placeholders_idx;
   for (const InputArgExpansion& input_arg : item.inputs()) {
     for (const string& placeholder : input_arg.placeholders) {
       input_placeholders_idx[placeholder] = idx++;
@@ -1699,7 +1699,7 @@ Status FunctionOptimizer::Optimize(Cluster*, const GrapplerItem& item,
   if (!ctx.control_overrides().empty()) {
     for (NodeDef& node : *optimized_graph->mutable_node()) {
       // Keep track of new control inputs to the node.
-      gtl::FlatSet<string> add_ctrl_inputs;
+      absl::flat_hash_set<string> add_ctrl_inputs;
 
       // Remove all invalidated control inputs.
       for (int idx = 0; idx < node.input_size(); /* see below */) {
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
index 67699b093d..a84bb1d62f 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
@@ -427,6 +427,14 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
   VLOG(1) << "Starting optimization for grappler item: " << item.id;
   optimization_results_.clear();
 
+  // Constructs a FunctionLibraryDefinition with functions that are reachable
+  // from the nodes of the graph.
+  const auto minimized_flib =
+      [](const GraphDef& graph) -> FunctionLibraryDefinition {
+    return FunctionLibraryDefinition(OpRegistry::Global(), graph.library())
+        .ReachableDefinitions(graph);
+  };
+
   // 0. Original graph might contain a huge function library, that is mostly
   // unused. This library copied over by each individual Grappler optimizer,
   // which adds a huge overhead. Before starting optimization passes we just
@@ -436,11 +444,7 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
   GraphDef trimmed_graph;  // do not copy graph with a potentially huge library
   *trimmed_graph.mutable_node() = item.graph.node();
   *trimmed_graph.mutable_versions() = item.graph.versions();
-  *trimmed_graph.mutable_library() =
-      grappler::ReachableFunctionLibraryDefinition(
-          FunctionLibraryDefinition(OpRegistry::Global(), item.graph.library()),
-          item.graph)
-          .ToProto();
+  *trimmed_graph.mutable_library() = minimized_flib(item.graph).ToProto();
 
   GrapplerItem trimmed_item = item.WithGraph(std::move(trimmed_graph));
 
@@ -472,10 +476,7 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
   }
 
   // 2. Optimize functions reachable from the optimized graph.
-  FunctionLibraryDefinition flib = ReachableFunctionLibraryDefinition(
-      FunctionLibraryDefinition(OpRegistry::Global(),
-                                optimized_graph->library()),
-      *optimized_graph);
+  FunctionLibraryDefinition flib = minimized_flib(*optimized_graph);
 
   // Find functions for which we might need to compute a gradient at runtime.
   absl::flat_hash_set<string> differentiable_functions;
diff --git a/tensorflow/core/grappler/utils/BUILD b/tensorflow/core/grappler/utils/BUILD
index cd69cf895c..89417f85c2 100644
--- a/tensorflow/core/grappler/utils/BUILD
+++ b/tensorflow/core/grappler/utils/BUILD
@@ -178,6 +178,9 @@ cc_library(
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
+        "@com_google_absl//absl/algorithm:container",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/strings",
     ],
 )
@@ -196,6 +199,7 @@ tf_cc_test(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
+        "@com_google_absl//absl/container:flat_hash_map",
     ],
 )
 
diff --git a/tensorflow/core/grappler/utils/functions.cc b/tensorflow/core/grappler/utils/functions.cc
index f2894a942b..7c2180ae40 100644
--- a/tensorflow/core/grappler/utils/functions.cc
+++ b/tensorflow/core/grappler/utils/functions.cc
@@ -14,8 +14,9 @@ limitations under the License.
 ==============================================================================*/
 #include "tensorflow/core/grappler/utils/functions.h"
 
-#include <unordered_map>
-
+#include "absl/container/flat_hash_map.h"
+#include "absl/container/flat_hash_set.h"
+#include "absl/strings/str_cat.h"
 #include "absl/strings/substitute.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/function.h"
@@ -28,7 +29,6 @@ limitations under the License.
 #include "tensorflow/core/framework/versions.pb.h"
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/grappler/utils.h"
-#include "tensorflow/core/lib/gtl/map_util.h"
 #include "tensorflow/core/lib/strings/scanner.h"
 
 namespace tensorflow {
@@ -76,16 +76,6 @@ Status ResolveFunctionBodyNodeAttrPlaceholders(
 
 }  // namespace
 
-FunctionLibraryDefinition ReachableFunctionLibraryDefinition(
-    const FunctionLibraryDefinition& flib, const GraphDef& graph) {
-  return flib.ReachableDefinitions(graph);
-}
-
-FunctionLibraryDefinition ReachableFunctionLibraryDefinition(
-    const FunctionLibraryDefinition& flib, const FunctionDef& func) {
-  return flib.ReachableDefinitions(func);
-}
-
 void GrapplerFunctionConnectivity::RegisterInputArgExpansion(
     InputArgExpansion input_arg_expansion) {
   string input_name = input_arg_expansion.input_name;
@@ -94,7 +84,7 @@ void GrapplerFunctionConnectivity::RegisterInputArgExpansion(
   for (int i = 0; i < placeholders.size(); ++i) {
     const string& placeholder = input_arg_expansion.placeholders[i];
     input_arg_placeholders_.insert(
-        {placeholder, InputArgPlaceholder{input_name, /*input_position=*/i}});
+        {placeholder, InputArgPlaceholder{input_name, /*input_index=*/i}});
   }
   input_arg_expansions_.insert(
       {std::move(input_name), std::move(input_arg_expansion)});
@@ -193,7 +183,7 @@ Status GrapplerFunctionConnectivity::ExpandFunctionDefInput(
           // If position is not defined expand node output range
           for (int i = output_range.first; i < output_range.second; ++i) {
             graph_def_inputs->push_back(
-                i == 0 ? node_name : strings::StrCat(node_name, ":", i));
+                i == 0 ? node_name : absl::StrCat(node_name, ":", i));
           }
         } else {
           if (position > (output_range.second - output_range.first)) {
@@ -203,7 +193,7 @@ Status GrapplerFunctionConnectivity::ExpandFunctionDefInput(
           }
           int pos = output_range.first + position;
           graph_def_inputs->push_back(
-              pos == 0 ? node_name : strings::StrCat(node_name, ":", pos));
+              pos == 0 ? node_name : absl::StrCat(node_name, ":", pos));
         }
 
         return Status::OK();
@@ -232,39 +222,39 @@ Status GrapplerFunctionConnectivity::ExpandNodeInputs(
 
 Status GrapplerFunctionConnectivity::AsFunctionDefInput(
     const string& graph_def_input, string* func_def_input) const {
-  using gtl::FindOrNull;
-
   if (IsControlInput(graph_def_input)) {
     *func_def_input = graph_def_input;
     return Status::OK();
   }
 
-  int position;
-  string node_name = ParseNodeName(graph_def_input, &position);
-  CHECK_GE(position, 0);
+  const TensorId tensor = ParseTensorName(graph_def_input);
+  DCHECK_GE(tensor.index(), 0);
+
+  const absl::string_view node_name = tensor.node();
+  const int index = tensor.index();
 
   // Check if it's an input arg placeholder
-  if (position == 0) {
-    const InputArgPlaceholder* placeholder =
-        FindOrNull(input_arg_placeholders_, node_name);
-    if (placeholder != nullptr) {
-      *func_def_input = strings::StrCat(placeholder->input_name, ":",
-                                        placeholder->input_position);
+  if (tensor.index() == 0) {
+    const auto is_input_placeholder = input_arg_placeholders_.find(node_name);
+    if (is_input_placeholder != input_arg_placeholders_.end()) {
+      const InputArgPlaceholder& placeholder = is_input_placeholder->second;
+      *func_def_input =
+          absl::StrCat(placeholder.input_name, ":", placeholder.input_index);
       return Status::OK();
     }
   }
 
   // It must be output from one of the function body nodes
-  const tensorflow::NameRangeMap* outputs_range_map =
-      FindOrNull(function_body_outputs_, node_name);
-  if (outputs_range_map != nullptr) {
-    for (const auto& el : *outputs_range_map) {
+  const auto is_body_output = function_body_outputs_.find(tensor.node());
+  if (is_body_output != function_body_outputs_.end()) {
+    const tensorflow::NameRangeMap& outputs_range_map = is_body_output->second;
+
+    for (const auto& el : outputs_range_map) {
       const auto& output_name = el.first;
       const auto& output_range = el.second;
-      if (position >= output_range.first && position < output_range.second) {
-        int pos = position - output_range.first;
-        *func_def_input =
-            strings::StrCat(node_name, ":", output_name, ":", pos);
+      if (index >= output_range.first && index < output_range.second) {
+        int pos = index - output_range.first;
+        *func_def_input = absl::StrCat(node_name, ":", output_name, ":", pos);
         return Status::OK();
       }
     }
@@ -426,7 +416,7 @@ bool IsParametrized(const FunctionDef& func) {
 
 Status InstantiationTypeParameters(
     const FunctionDef& func, const AttrSlice& func_instantiation_attr,
-    std::unordered_map<string, DataType>* type_parameters) {
+    absl::flat_hash_map<string, DataType>* type_parameters) {
   if (!type_parameters->empty()) {
     return errors::InvalidArgument("Type parameters output map must be empty");
   }
@@ -454,7 +444,7 @@ Status InstantiationTypeParameters(
 
 Status InstantiationBodyParameters(
     const FunctionDef& func, const AttrSlice& func_instantiation_attr,
-    std::unordered_map<string, AttrValue>* body_parameters) {
+    absl::flat_hash_map<string, AttrValue>* body_parameters) {
   if (!body_parameters->empty()) {
     return errors::InvalidArgument("Body parameters output map must be empty");
   }
@@ -514,8 +504,7 @@ Status MakeGrapplerFunctionItem(const FunctionDef& func,
 
   // Function body shares the library with the graph that instantiated it. We do
   // not need a full copy of the function library, just the reachable subset.
-  *function_body.mutable_library() =
-      ReachableFunctionLibraryDefinition(flib, func).ToProto();
+  *function_body.mutable_library() = flib.ReachableDefinitions(func).ToProto();
 
   VLOG(3) << absl::Substitute(
       "Deleted $0 unreachable functions from the Grappler function item "
@@ -645,7 +634,7 @@ Status RegisterGrapplerFunctionConnectivity(
   return Status::OK();
 }
 
-Status ReplaceInputWithConst(const NodeDef& input_const, int input_position,
+Status ReplaceInputWithConst(const NodeDef& input_const, int input_index,
                              GrapplerFunctionItem* item) {
   if (!IsConstant(input_const)) {
     return errors::InvalidArgument("Input node ", input_const.name(),
@@ -657,7 +646,7 @@ Status ReplaceInputWithConst(const NodeDef& input_const, int input_position,
   // Find input arg expansion and input placeholder position in it for the
   // given function input position.
   InputArgExpansion* input_arg_expansion = nullptr;
-  int placeholder_idx = input_position;
+  int placeholder_idx = input_index;
 
   for (InputArgExpansion& input : inputs) {
     if (placeholder_idx < input.placeholders.size()) {
@@ -668,9 +657,8 @@ Status ReplaceInputWithConst(const NodeDef& input_const, int input_position,
   }
 
   if (input_arg_expansion == nullptr) {
-    return errors::InvalidArgument(
-        "Input placeholder not found: input_position=", input_position,
-        " function=", item->id);
+    return errors::InvalidArgument("Input placeholder not found: input_index=",
+                                   input_index, " function=", item->id);
   }
 
   // Delete placeholder from input expansion.
@@ -699,7 +687,7 @@ Status ReplaceInputWithConst(const NodeDef& input_const, int input_position,
   return Status::OK();
 }
 
-Status RemoveUnusedOutputs(const gtl::FlatSet<int>& active_outputs,
+Status RemoveUnusedOutputs(const absl::flat_hash_set<int>& active_outputs,
                            GrapplerFunctionItem* item,
                            std::vector<std::pair<int, int>>* output_mapping) {
   DCHECK(output_mapping->empty());
@@ -713,7 +701,7 @@ Status RemoveUnusedOutputs(const gtl::FlatSet<int>& active_outputs,
     }
   }
 
-  gtl::FlatSet<const OutputArgExpansion*> unused_output_args;
+  absl::flat_hash_set<const OutputArgExpansion*> unused_output_args;
 
   const auto is_unused_output_arg = [&](const OutputArgExpansion& output) {
     return unused_output_args.find(&output) != unused_output_args.end();
diff --git a/tensorflow/core/grappler/utils/functions.h b/tensorflow/core/grappler/utils/functions.h
index 038cf5f527..ce8a3e5ac7 100644
--- a/tensorflow/core/grappler/utils/functions.h
+++ b/tensorflow/core/grappler/utils/functions.h
@@ -18,7 +18,9 @@ limitations under the License.
 
 #include <memory>
 #include <string>
-#include <unordered_map>
+
+#include "absl/container/flat_hash_map.h"
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/framework/function.pb.h"
@@ -30,13 +32,6 @@ limitations under the License.
 namespace tensorflow {
 namespace grappler {
 
-// Returns a copy of FunctionLibraryDefinition with subset of functions that are
-// reachable from the nodes of the graph.
-FunctionLibraryDefinition ReachableFunctionLibraryDefinition(
-    const FunctionLibraryDefinition& flib, const GraphDef& graph);
-FunctionLibraryDefinition ReachableFunctionLibraryDefinition(
-    const FunctionLibraryDefinition& flib, const FunctionDef& func);
-
 // Depending on the function instantiation attributes, input argument to the
 // function might be a single tensor, list of tensors of the same type, or a
 // list of tensors of different types.
@@ -81,12 +76,12 @@ class GrapplerFunctionConnectivity {
   void RegisterFunctionBodyOutputs(const string& node_name,
                                    tensorflow::NameRangeMap&& outputs);
 
-  // Expand input encoded in FunctionDef format (name[:output][:position]) into
+  // Expands input encoded in FunctionDef format (name[:output][:position]) into
   // multiple inputs in GraphDef format (name[:position]).
   Status ExpandFunctionDefInput(const string& func_def_input,
                                 std::vector<string>* graph_def_inputs) const;
 
-  // Update Node inputs from FunctionDef to GraphDef format.
+  // Updates Node inputs from FunctionDef to GraphDef format.
   Status ExpandNodeInputs(NodeDef* function_body_node) const;
 
   // When expanding inputs in function def format, single input might be
@@ -96,29 +91,31 @@ class GrapplerFunctionConnectivity {
   // instantiation attributes and length of input args (and node def outputs) is
   // known.
 
-  // Map from GraphDef input format to FunctionDef input format using registered
-  // input arg expansion and function body outputs.
+  // Converts input name from GraphDef format (name[:position]) to the
+  // FunctionDef input format (name[:output][:position]) using registered input
+  // arg expansion and function body outputs.
   Status AsFunctionDefInput(const string& graph_def_input,
                             string* func_def_input) const;
 
-  // Update Node inputs from GraphDef to FunctionDef format.
+  // Updates Node inputs from GraphDef to FunctionDef format.
   Status AsFunctionDefNode(NodeDef* function_body_node) const;
 
  private:
   // Mapping from input name to input arg expansion.
-  std::unordered_map<string, InputArgExpansion> input_arg_expansions_;
+  absl::flat_hash_map<string, InputArgExpansion> input_arg_expansions_;
   // Mapping from function body node name to output names range map.
-  std::unordered_map<string, tensorflow::NameRangeMap> function_body_outputs_;
+  absl::flat_hash_map<string, tensorflow::NameRangeMap> function_body_outputs_;
 
+  // For each placeholder added to the function instantiation graph, we keep a
+  // mapping back to the function input argument name and index.
   struct InputArgPlaceholder {
-    string input_name;   // Name of the function input argument.
-    int input_position;  // Index of a tensor in the function input argument
-                         // expansion, it can be greater than `0` if input
-                         // argument is a list of tensors (aka list(type)).
+    string input_name;  // Name of the function input argument.
+    int input_index;    // Index of a tensor in the function input argument
+                        // expansion, it can be greater than `0` if input
+                        // argument is a list of tensors (aka list(type)).
   };
-
   // Mapping from input arg placeholder to the function input tensor.
-  std::unordered_map<string, InputArgPlaceholder> input_arg_placeholders_;
+  absl::flat_hash_map<string, InputArgPlaceholder> input_arg_placeholders_;
 };
 
 // Get Function type attributes using attributes of a node that instantiated
@@ -172,7 +169,8 @@ class GrapplerFunctionItem : public GrapplerItem {
   friend Status ReplaceInputWithConst(const NodeDef&, int,
                                       GrapplerFunctionItem*);
   friend Status RemoveUnusedOutputs(
-      const gtl::FlatSet<int>& active_outputs, GrapplerFunctionItem* item,
+      const absl::flat_hash_set<int>& active_outputs,
+      GrapplerFunctionItem* item,
       std::vector<std::pair<int, int>>* output_mapping);
 
   GrapplerFunctionItem(string func_name, string description,
@@ -191,7 +189,7 @@ class GrapplerFunctionItem : public GrapplerItem {
 
   std::set<string> input_arg_placeholders_;
 
-  bool is_stateful_;
+  bool is_stateful_ = false;
 };
 
 // Check if function input/output types are fully defined only at instantiation
@@ -210,14 +208,14 @@ bool IsParametrized(const FunctionDef& func);
 // caller node. Return error if type can't be resolved.
 Status InstantiationTypeParameters(
     const FunctionDef& func, const AttrSlice& func_instantiation_attr,
-    std::unordered_map<string, DataType>* type_parameters);
+    absl::flat_hash_map<string, DataType>* type_parameters);
 
 // Resolve function instantiation body parameters (values for the function body
 // attr placeholders) from the attributes of the caller node. Return error if
 // type can't be resolved.
 Status InstantiationBodyParameters(
     const FunctionDef& func, const AttrSlice& func_instantiation_attr,
-    std::unordered_map<string, AttrValue>* body_parameters);
+    absl::flat_hash_map<string, AttrValue>* body_parameters);
 
 // Register GrapplerFunctionItem input arg expansion and function body outputs
 // in the GrapplerFunctionConnectivity. Use function library definition to
@@ -227,7 +225,7 @@ Status RegisterGrapplerFunctionConnectivity(
     GrapplerFunctionConnectivity* connectivity);
 
 // Replace one of the function inputs with a constant.
-Status ReplaceInputWithConst(const NodeDef& input_const, int input_position,
+Status ReplaceInputWithConst(const NodeDef& input_const, int input_index,
                              GrapplerFunctionItem* item);
 
 // Remove function output arguments that do not have any active outputs (output
@@ -236,7 +234,7 @@ Status ReplaceInputWithConst(const NodeDef& input_const, int input_position,
 // potentially be connected to the same output argument (in case of tensor list
 // outputs). Add output mapping for all active outputs that changed it's output
 // position (std::pair<old position, new position>).
-Status RemoveUnusedOutputs(const gtl::FlatSet<int>& active_outputs,
+Status RemoveUnusedOutputs(const absl::flat_hash_set<int>& active_outputs,
                            GrapplerFunctionItem* item,
                            std::vector<std::pair<int, int>>* output_mapping);
 
diff --git a/tensorflow/core/grappler/utils/functions_test.cc b/tensorflow/core/grappler/utils/functions_test.cc
index 5923850eca..29d6100d23 100644
--- a/tensorflow/core/grappler/utils/functions_test.cc
+++ b/tensorflow/core/grappler/utils/functions_test.cc
@@ -14,6 +14,8 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/grappler/utils/functions.h"
+
+#include "absl/container/flat_hash_map.h"
 #include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/core/framework/function_testlib.h"
 #include "tensorflow/core/framework/node_def.pb.h"
@@ -77,7 +79,7 @@ TEST_F(FunctionsTest, InstantiationParameters) {
   func_instantiation_attr["B"].set_type(DT_INT32);
   func_instantiation_attr["C"].set_type(DT_DOUBLE);
 
-  std::unordered_map<string, DataType> type_parameters;
+  absl::flat_hash_map<string, DataType> type_parameters;
   TF_EXPECT_OK(InstantiationTypeParameters(
       func, AttrSlice(&func_instantiation_attr), &type_parameters));
 
@@ -86,7 +88,7 @@ TEST_F(FunctionsTest, InstantiationParameters) {
   EXPECT_EQ(DT_INT32, type_parameters["B"]);
   EXPECT_EQ(DT_DOUBLE, type_parameters["C"]);
 
-  std::unordered_map<string, AttrValue> body_parameters;
+  absl::flat_hash_map<string, AttrValue> body_parameters;
   TF_EXPECT_OK(InstantiationBodyParameters(
       func, AttrSlice(&func_instantiation_attr), &body_parameters));
 
-- 
GitLab


From 26d8486baa67d2c545aa4384f9d666dc7b9ad530 Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Wed, 2 Jan 2019 15:11:44 -0800
Subject: [PATCH 0105/2345] documentation fix

PiperOrigin-RevId: 227589407
---
 tensorflow/python/data/experimental/ops/optimization_options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/data/experimental/ops/optimization_options.py b/tensorflow/python/data/experimental/ops/optimization_options.py
index 41a819d94b..eb95484ac5 100644
--- a/tensorflow/python/data/experimental/ops/optimization_options.py
+++ b/tensorflow/python/data/experimental/ops/optimization_options.py
@@ -33,7 +33,7 @@ class OptimizationOptions(options.OptionsBase):
   ```python
   options = tf.data.Options()
   options.experimental_optimization.map_vectorization = True
-  options.apply_default_optimizations = False
+  options.experimental_optimization.apply_default_optimizations = False
   dataset = dataset.with_options(options)
   ```
   """
-- 
GitLab


From 29bc2e98eaf1dd730574886ffe1b317e982323d4 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Wed, 2 Jan 2019 15:47:20 -0800
Subject: [PATCH 0106/2345] Validate that all variables under the model were
 created in the distribution strategy scope.

PiperOrigin-RevId: 227595107
---
 .../contrib/distribute/python/keras_test.py   | 44 +++++++++++++++++++
 tensorflow/python/keras/engine/training.py    | 14 ++++++
 2 files changed, 58 insertions(+)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index b99d11b923..ba5a5e6400 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -245,6 +245,18 @@ def all_strategy_combinations():
   return strategy_minus_tpu_combinations() + tpu_strategy_combinations()
 
 
+def all_strategy_combinations_minus_default():
+  strategy_minus_default_combinations = combinations.combine(
+      distribution=[
+          combinations.one_device_strategy,
+          combinations.mirrored_strategy_with_gpu_and_cpu,
+          combinations.mirrored_strategy_with_two_gpus,
+          combinations.core_mirrored_strategy_with_gpu_and_cpu,
+          combinations.core_mirrored_strategy_with_two_gpus],
+      mode=['graph', 'eager'])
+  return strategy_minus_default_combinations + tpu_strategy_combinations()
+
+
 # TODO(priyag): Add v2 optimizers here.
 def strategy_and_optimizer_combinations():
   return combinations.times(
@@ -1149,5 +1161,37 @@ class TestDistributionStrategyWithNormalizationLayer(
       np.testing.assert_allclose(out.std(), 1.0, atol=1e-1)
 
 
+class TestDistributionStrategyVariableValidation(test.TestCase,
+                                                 parameterized.TestCase):
+
+  @combinations.generate(all_strategy_combinations_minus_default())
+  def test_layer_outside_scope(self, distribution):
+    with self.cached_session():
+      with self.assertRaisesRegexp(
+          ValueError, 'was not created in the distribution strategy'):
+        x = keras.layers.Input(shape=(3,), name='input')
+        y = keras.layers.Dense(4, name='dense')(x)
+        with distribution.scope():
+          model = keras.Model(x, y)
+          optimizer = gradient_descent.GradientDescentOptimizer(0.001)
+          loss = 'mse'
+          metrics = ['mae', keras.metrics.CategoricalAccuracy()]
+          model.compile(optimizer, loss, metrics=metrics)
+
+  @combinations.generate(all_strategy_combinations_minus_default())
+  def test_model_outside_scope(self, distribution):
+    with self.cached_session():
+      with self.assertRaisesRegexp(
+          ValueError, 'was not created in the distribution strategy'):
+        x = keras.layers.Input(shape=(3,), name='input')
+        y = keras.layers.Dense(4, name='dense')(x)
+        model = keras.Model(x, y)
+        with distribution.scope():
+          optimizer = gradient_descent.GradientDescentOptimizer(0.001)
+          loss = 'mse'
+          metrics = ['mae', keras.metrics.CategoricalAccuracy()]
+          model.compile(optimizer, loss, metrics=metrics)
+
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index be3afafd3e..1dc3b81cfc 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -548,6 +548,20 @@ class Model(Network):
       trainable_weights = self.trainable_weights
       self._collected_trainable_weights = trainable_weights
 
+      # Validate all variables were correctly created in distribution scope.
+      if self._distribution_strategy and not self._compile_distribution:
+        for v in self.variables:
+          if v.distribute_strategy is not self._distribution_strategy:
+            raise ValueError(
+                'Variable (%s) was not created in the distribution strategy '
+                'scope of (%s). It is most likely due to not all layers or '
+                'the model or optimizer being created outside the distribution '
+                'strategy scope. Try to make sure your code looks similar '
+                'to the following.\n'
+                'with strategy.scope():\n'
+                '  model=_create_model()\n'
+                '  model.compile(...)'% (v, self._distribution_strategy))
+
   @property
   def metrics(self):
     """Returns the model's metrics added using `compile`, `add_metric` APIs."""
-- 
GitLab


From 6af0243ba6842330d7bcce6499287f6e9a640c9f Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <penporn@google.com>
Date: Wed, 2 Jan 2019 15:52:17 -0800
Subject: [PATCH 0107/2345] Fix a typo in Near. Before this change, when the
 values are not 'Near', the program reports the address of b instead of the
 value of b[i].

PiperOrigin-RevId: 227595818
---
 tensorflow/core/framework/tensor_testutil.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/framework/tensor_testutil.h b/tensorflow/core/framework/tensor_testutil.h
index 3163002851..b58292b3b0 100644
--- a/tensorflow/core/framework/tensor_testutil.h
+++ b/tensorflow/core/framework/tensor_testutil.h
@@ -206,7 +206,7 @@ struct Expector<T, true> {
     const T* b = y.flat<T>().data();
     for (int i = 0; i < size; ++i) {
       EXPECT_TRUE(Near(a[i], b[i], abs_err))
-          << "a = " << a[i] << " b = " << b << " index = " << i;
+          << "a = " << a[i] << " b = " << b[i] << " index = " << i;
     }
   }
 };
-- 
GitLab


From f0dde544f263013867e8c66c82db85e66c42d530 Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Wed, 2 Jan 2019 16:00:27 -0800
Subject: [PATCH 0108/2345] [XLA] Don't go out of bounds in
 Shape::dynamic_dimensions_.

PiperOrigin-RevId: 227596997
---
 tensorflow/compiler/xla/shape.cc | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/shape.cc b/tensorflow/compiler/xla/shape.cc
index c04a916154..f05d816268 100644
--- a/tensorflow/compiler/xla/shape.cc
+++ b/tensorflow/compiler/xla/shape.cc
@@ -27,7 +27,19 @@ Shape::Shape(const ShapeProto& shape_proto) {
   for (const int64 dimension : shape_proto.dimensions()) {
     add_dimensions(dimension);
   }
-  for (int i = 0; i < shape_proto.is_dynamic_dimension_size(); i++) {
+  // A malformed proto may have different is_dynamic_dimension_size and
+  // dimensions_size. Since C++ is evil, and we have no good way of bailing out
+  // in a constructor, conservatively trim the is_dynamic_dimension size.
+  // TODO(b/120111794): Make this a hard error when we have a factory method
+  // instead of a constructor.
+  if (shape_proto.dimensions_size() !=
+      shape_proto.is_dynamic_dimension_size()) {
+    LOG(ERROR) << "Malformed shape proto: number of is_dynamic_dimension "
+                  "fields does not match number of dimension fields";
+  }
+  int64 num_dynamic_dimension_fields = std::min(
+      shape_proto.dimensions_size(), shape_proto.is_dynamic_dimension_size());
+  for (int i = 0; i < num_dynamic_dimension_fields; i++) {
     dynamic_dimensions_[i] = shape_proto.is_dynamic_dimension(i);
   }
   tuple_shapes_.reserve(shape_proto.tuple_shapes_size());
-- 
GitLab


From 0b6177c2fa509d951d4086eab88fc506792cfe43 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 2 Jan 2019 16:08:42 -0800
Subject: [PATCH 0109/2345] Fix make_initializable_iterator comment

PiperOrigin-RevId: 227598505
---
 tensorflow/python/data/ops/dataset_ops.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index 1ba00a8e69..61210f3a7f 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -1756,8 +1756,8 @@ def make_initializable_iterator(dataset):
     RuntimeError: If eager execution is enabled.
   """
   try:
-    # Call the defined `make_one_shot_iterator()` if there is one, because some
-    # datasets (e.g. for prefetching) override its behavior.
+    # Call the defined `make_initializable_iterator()` if there is one, because
+    # some datasets (e.g. for prefetching) override its behavior.
     return dataset.make_initializable_iterator()
   except AttributeError:
     return DatasetV1Adapter(dataset).make_initializable_iterator()
-- 
GitLab


From dd3adf935a800e25132b187d65aa62e97504f50c Mon Sep 17 00:00:00 2001
From: Kay Zhu <kayzhu@google.com>
Date: Wed, 2 Jan 2019 16:11:52 -0800
Subject: [PATCH 0110/2345] [XLA] Enable interpreter backend test by default
 for xla/tests without having to using additional backend tags
 "enable_for_xla_interpreter".

PiperOrigin-RevId: 227598976
---
 tensorflow/compiler/xla/client/lib/BUILD      |   6 -
 .../compiler/xla/service/hlo_evaluator.cc     |   5 +
 .../compiler/xla/service/hlo_evaluator.h      |  23 +++
 .../xla/service/hlo_evaluator_typed_visitor.h |  32 ++--
 .../compiler/xla/service/interpreter/BUILD    |   3 +
 .../xla/service/interpreter/compiler.cc       |   6 +
 tensorflow/compiler/xla/tests/BUILD           | 144 ++++--------------
 .../xla/tests/array_elementwise_ops_test.cc   |  13 ++
 .../compiler/xla/tests/bfloat16_test.cc       |   8 +-
 tensorflow/compiler/xla/tests/client_test.cc  |   5 +-
 .../xla/tests/gather_operation_test.cc        |   4 +-
 .../xla/tests/local_client_execute_test.cc    |   6 +-
 12 files changed, 115 insertions(+), 140 deletions(-)

diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD
index 966a581981..d863ebfea1 100644
--- a/tensorflow/compiler/xla/client/lib/BUILD
+++ b/tensorflow/compiler/xla/client/lib/BUILD
@@ -93,7 +93,6 @@ cc_library(
 xla_test(
     name = "constants_test",
     srcs = ["constants_test.cc"],
-    tags = ["enable_for_xla_interpreter"],
     deps = [
         ":constants",
         "//tensorflow/compiler/xla:test",
@@ -147,7 +146,6 @@ cc_library(
 xla_test(
     name = "math_test",
     srcs = ["math_test.cc"],
-    tags = ["enable_for_xla_interpreter"],
     deps = [
         ":math",
         "//tensorflow/compiler/xla:literal_util",
@@ -181,7 +179,6 @@ cc_library(
 xla_test(
     name = "matrix_test",
     srcs = ["matrix_test.cc"],
-    tags = ["enable_for_xla_interpreter"],
     deps = [
         ":matrix",
         ":slicing",
@@ -295,7 +292,6 @@ cc_library(
 xla_test(
     name = "slicing_test",
     srcs = ["slicing_test.cc"],
-    tags = ["enable_for_xla_interpreter"],
     deps = [
         ":slicing",
         "//tensorflow/compiler/xla:literal_util",
@@ -324,7 +320,6 @@ cc_library(
 xla_test(
     name = "sorting_test",
     srcs = ["sorting_test.cc"],
-    tags = ["enable_for_xla_interpreter"],
     deps = [
         ":sorting",
         "//tensorflow/compiler/xla:test",
@@ -354,7 +349,6 @@ xla_test(
     srcs = ["quantize_test.cc"],
     # TODO(b/122119490): re-enable TAP after fixing.
     tags = [
-        "enable_for_xla_interpreter",
         "notap",
     ],
     deps = [
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index 794b97a7e0..4f5484293c 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -138,6 +138,11 @@ StatusOr<Literal> Compare<complex64>(const Shape& shape, HloOpcode opcode,
 
 }  // namespace
 
+// Note that unsupported types by the typed visitor does not necessarily imply
+// the non-typed HloEvaluator (parent evaluator) would not support them either
+// in the type-agnostic handler. For e.g., HandleGetTupleElement in the parent
+// type-agnostic evaluator will be able to accept Tuple primitive type, whereas
+// HloEvaluatorTypedVisitor cannot.
 HloEvaluator::HloEvaluator(int64 max_loop_iterations)
     : max_loop_iterations_(max_loop_iterations) {
   typed_visitors_[PRED] =
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.h b/tensorflow/compiler/xla/service/hlo_evaluator.h
index 44d5e5c030..d8e3019576 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.h
@@ -211,6 +211,29 @@ class HloEvaluator : public DfsHloVisitorWithDefault {
 
   Status HandleReduce(HloInstruction* reduce) override;
 
+  // Unsupported HLOs, note some of them (such as BatchNorm*) are typically
+  // expanded in a semantic-preserving way into other HLOs by adding exanpsion
+  // HLO pass to the HLO optimization pass during compilation, which can then be
+  // handled by the evaluator.
+  Status HandleBatchNormGrad(HloInstruction* batch_norm_grad) override {
+    return Unimplemented("BatchNormGrad HLO is unsupported by the evaluator.");
+  };
+  Status HandleBatchNormInference(
+      HloInstruction* batch_norm_inference) override {
+    return Unimplemented(
+        "BatchNormInference HLO is unsupported by the evaluator.");
+  };
+  Status HandleBatchNormTraining(HloInstruction* batch_norm_training) override {
+    return Unimplemented(
+        "BatchNormTraining HLO is unsupported by the evaluator.");
+  };
+  Status HandleInfeed(HloInstruction* infeed) override {
+    return Unimplemented("Infeed HLO is unsupported by the evaluator.");
+  };
+  Status HandleOutfeed(HloInstruction* outfeed) override {
+    return Unimplemented("Outfeed HLO is unsupported by the evaluator.");
+  };
+
   // Returns the already-evaluated literal result for the instruction.
   // A Constant instruction is considered evaluated and its literal will be
   // returned directly without looking up the cache.
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
index 6f3208be13..b1bbf49d4a 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
@@ -917,7 +917,11 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
   Status HandleClamp(HloInstruction* clamp) {
     std::function<ElementwiseT(ElementwiseT, ElementwiseT, ElementwiseT)>
         clamp_op = [](ElementwiseT low, ElementwiseT value, ElementwiseT high) {
-          return std::fmin(high, std::fmax(value, low));
+          if (std::isnan(low) || std::isnan(high)) {
+            return static_cast<ElementwiseT>(NAN);
+          }
+          return static_cast<ElementwiseT>(
+              std::fmin(high, std::fmax(value, low)));
         };
     TF_ASSIGN_OR_RETURN(
         parent_->evaluated_[clamp],
@@ -2664,12 +2668,13 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
     return HandleReducePrecision<ElementwiseT>(reduce_precision);
   }
 
-  template <typename NativeT,
-            typename std::enable_if<
-                std::is_same<NativeT, bfloat16>::value ||
-                std::is_same<NativeT, Eigen::half>::value ||
-                std::is_integral<NativeT>::value ||
-                std::is_floating_point<NativeT>::value>::type* = nullptr>
+  template <
+      typename NativeT,
+      typename std::enable_if<
+          std::is_same<NativeT, bfloat16>::value ||
+          std::is_same<NativeT, Eigen::half>::value ||
+          std::is_integral<NativeT>::value || is_complex_t<NativeT>::value ||
+          std::is_floating_point<NativeT>::value>::type* = nullptr>
   Status HandleIota(HloInstruction* instruction) {
     auto* iota = Cast<HloIotaInstruction>(instruction);
     const int64 iota_size = iota->shape().dimensions(iota->iota_dimension());
@@ -2700,12 +2705,13 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
 
     return Status::OK();
   }
-  template <typename NativeT,
-            typename std::enable_if<
-                !(std::is_same<NativeT, bfloat16>::value ||
-                  std::is_same<NativeT, Eigen::half>::value ||
-                  std::is_integral<NativeT>::value ||
-                  std::is_floating_point<NativeT>::value)>::type* = nullptr>
+  template <
+      typename NativeT,
+      typename std::enable_if<
+          !(std::is_same<NativeT, bfloat16>::value ||
+            std::is_same<NativeT, Eigen::half>::value ||
+            std::is_integral<NativeT>::value || is_complex_t<NativeT>::value ||
+            std::is_floating_point<NativeT>::value)>::type* = nullptr>
   Status HandleIota(HloInstruction* iota) {
     return UnsupportedTypeError(iota);
   }
diff --git a/tensorflow/compiler/xla/service/interpreter/BUILD b/tensorflow/compiler/xla/service/interpreter/BUILD
index a981d94a99..8999f38ccd 100644
--- a/tensorflow/compiler/xla/service/interpreter/BUILD
+++ b/tensorflow/compiler/xla/service/interpreter/BUILD
@@ -32,6 +32,7 @@ cc_library(
         "//tensorflow/compiler/xla:status_macros",
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla/service:algebraic_simplifier",
+        "//tensorflow/compiler/xla/service:batchnorm_expander",
         "//tensorflow/compiler/xla/service:compiler",
         "//tensorflow/compiler/xla/service:computation_placer",
         "//tensorflow/compiler/xla/service:executable",
@@ -41,12 +42,14 @@ cc_library(
         "//tensorflow/compiler/xla/service:hlo_cost_analysis",
         "//tensorflow/compiler/xla/service:hlo_cse",
         "//tensorflow/compiler/xla/service:hlo_dce",
+        "//tensorflow/compiler/xla/service:hlo_get_dimension_size_rewriter",
         "//tensorflow/compiler/xla/service:hlo_module_config",
         "//tensorflow/compiler/xla/service:hlo_pass",
         "//tensorflow/compiler/xla/service:hlo_pass_pipeline",
         "//tensorflow/compiler/xla/service:hlo_subcomputation_unification",
         "//tensorflow/compiler/xla/service:layout_assignment",
         "//tensorflow/compiler/xla/service:map_inliner",
+        "//tensorflow/compiler/xla/service:reduce_precision_insertion",
         "//tensorflow/compiler/xla/service:reshape_mover",
         "//tensorflow/compiler/xla/service:while_loop_simplifier",
         "//tensorflow/core:lib",
diff --git a/tensorflow/compiler/xla/service/interpreter/compiler.cc b/tensorflow/compiler/xla/service/interpreter/compiler.cc
index d37ae94bf6..69f611a094 100644
--- a/tensorflow/compiler/xla/service/interpreter/compiler.cc
+++ b/tensorflow/compiler/xla/service/interpreter/compiler.cc
@@ -31,6 +31,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/interpreter/executable.h"
 #include "tensorflow/compiler/xla/service/layout_assignment.h"
 #include "tensorflow/compiler/xla/service/map_inliner.h"
+#include "tensorflow/compiler/xla/service/reduce_precision_insertion.h"
 #include "tensorflow/compiler/xla/service/reshape_mover.h"
 #include "tensorflow/compiler/xla/service/while_loop_simplifier.h"
 #include "tensorflow/compiler/xla/status_macros.h"
@@ -46,6 +47,11 @@ Status InterpreterCompiler::RunHloOptimization(HloModule* hlo_module) {
   pipeline.AddPass<LayoutAssignment>(
       hlo_module->mutable_entry_computation_layout(),
       LayoutAssignment::InstructionCanChangeLayout);
+
+  ReducePrecisionInsertion::AddPasses(
+      &pipeline, hlo_module->config().debug_options(),
+      ReducePrecisionInsertion::PassTiming::BEFORE_OPTIMIZATION);
+
   return pipeline.Run(hlo_module).status();
 }
 
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index ee24d4d99c..578e716b59 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -276,9 +276,6 @@ cc_library(
 xla_test(
     name = "bad_rng_shape_validation_test",
     srcs = ["bad_rng_shape_validation_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:test",
@@ -344,9 +341,6 @@ xla_test(
 xla_test(
     name = "check_execution_arity_test",
     srcs = ["check_execution_arity_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:shape_util",
@@ -367,9 +361,6 @@ xla_test(
 xla_test(
     name = "query_inferred_shape_test",
     srcs = ["query_inferred_shape_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:statusor",
@@ -387,9 +378,6 @@ xla_test(
 xla_test(
     name = "while_test",
     srcs = ["while_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:shape_util",
@@ -413,6 +401,10 @@ xla_test(
 xla_test(
     name = "xla_hlo_profile_test",
     srcs = ["xla_hlo_profile_test.cc"],
+    blacklisted_backends = [
+        # Hlo profiles are not supported on the interpreter backend.
+        "interpreter",
+    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:shape_util",
@@ -436,9 +428,6 @@ xla_test(
 xla_test(
     name = "axpy_simple_test",
     srcs = ["axpy_simple_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla/client:local_client",
         "//tensorflow/compiler/xla/client:xla_builder",
@@ -453,7 +442,6 @@ xla_test(
 xla_test(
     name = "map_test",
     srcs = ["map_test.cc"],
-    tags = ["enable_for_xla_interpreter"],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:literal",
@@ -506,9 +494,6 @@ xla_test(
 xla_test(
     name = "pred_test",
     srcs = ["pred_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla/client:local_client",
@@ -524,9 +509,6 @@ xla_test(
 xla_test(
     name = "select_test",
     srcs = ["select_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:global_data",
@@ -544,7 +526,6 @@ xla_test(
 xla_test(
     name = "conditional_test",
     srcs = ["conditional_test.cc"],
-    tags = ["enable_for_xla_interpreter"],
     deps = [
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:global_data",
@@ -562,7 +543,6 @@ xla_test(
 xla_test(
     name = "unary_op_test",
     srcs = ["unary_op_test.cc"],
-    tags = ["enable_for_xla_interpreter"],
     deps = [
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:global_data",
@@ -623,9 +603,6 @@ xla_test(
 xla_test(
     name = "deconstruct_tuple_test",
     srcs = ["deconstruct_tuple_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:shape_util",
@@ -648,7 +625,6 @@ xla_test(
     name = "array_elementwise_ops_test",
     srcs = ["array_elementwise_ops_test.cc"],
     shard_count = 25,
-    tags = ["enable_for_xla_interpreter"],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:array3d",
@@ -698,7 +674,6 @@ xla_test(
 xla_test(
     name = "reduce_precision_test",
     srcs = ["reduce_precision_test.cc"],
-    tags = ["enable_for_xla_interpreter"],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:literal",
@@ -725,7 +700,6 @@ xla_test(
     srcs = ["dot_operation_test.cc"],
     shard_count = 20,
     tags = [
-        "enable_for_xla_interpreter",
         "optonly",
     ],
     deps = [
@@ -806,9 +780,6 @@ xla_test(
 xla_test(
     name = "transpose_test",
     srcs = ["transpose_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:reference_util",
@@ -828,9 +799,6 @@ xla_test(
 xla_test(
     name = "constants_test",
     srcs = ["constants_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:array3d",
@@ -951,6 +919,11 @@ xla_test(
 xla_test(
     name = "batch_normalization_test",
     srcs = ["batch_normalization_test.cc"],
+    blacklisted_backends = [
+        # BatchNorm HLOs are not handled by the interpreter backend, and the
+        # BatchNorm expander is not run on the interpreter.
+        "interpreter",
+    ],
     shard_count = 40,
     deps = [
         ":test_utils",
@@ -1042,9 +1015,6 @@ xla_test(
     name = "slice_test",
     srcs = ["slice_test.cc"],
     shard_count = 40,
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:reference_util",
@@ -1065,9 +1035,6 @@ xla_test(
 xla_test(
     name = "multidimensional_slice_test",
     srcs = ["multidimensional_slice_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:array3d",
@@ -1085,9 +1052,6 @@ xla_test(
     name = "dynamic_ops_test",
     timeout = "moderate",
     srcs = ["dynamic_ops_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:reference_util",
@@ -1113,9 +1077,6 @@ xla_test(
 xla_test(
     name = "tuple_test",
     srcs = ["tuple_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:literal",
@@ -1139,9 +1100,6 @@ xla_test(
 xla_test(
     name = "vector_ops_reduce_test",
     srcs = ["vector_ops_reduce_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:array3d",
@@ -1162,7 +1120,6 @@ xla_test(
     srcs = ["reduce_test.cc"],
     shard_count = 40,
     tags = [
-        "enable_for_xla_interpreter",
         "optonly",
     ],
     deps = [
@@ -1229,7 +1186,6 @@ xla_test(
     srcs = [],
     shard_count = 20,
     tags = [
-        "enable_for_xla_interpreter",
         "optonly",
     ],
     xla_test_library_deps = [":reduce_window_test_library"],
@@ -1241,7 +1197,6 @@ xla_test(
     timeout = "long",
     srcs = ["select_and_scatter_test.cc"],
     tags = [
-        "enable_for_xla_interpreter",
         "optonly",
     ],
     deps = [
@@ -1267,9 +1222,6 @@ xla_test(
 xla_test(
     name = "copy_test",
     srcs = ["copy_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         ":client_library_test_base",
         "//tensorflow/compiler/xla:array2d",
@@ -1290,9 +1242,6 @@ xla_test(
 xla_test(
     name = "reduce_hlo_test",
     srcs = ["reduce_hlo_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla/service:hlo_parser",
         "//tensorflow/compiler/xla/tests:hlo_test_base",
@@ -1306,9 +1255,6 @@ xla_test(
 xla_test(
     name = "token_hlo_test",
     srcs = ["token_hlo_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla/service:hlo_parser",
         "//tensorflow/compiler/xla/service:hlo_verifier",
@@ -1323,9 +1269,6 @@ xla_test(
 xla_test(
     name = "call_test",
     srcs = ["call_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:literal_util",
@@ -1368,9 +1311,6 @@ xla_test(
 xla_test(
     name = "binop_scaling_test",
     srcs = ["binop_scaling_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:array4d",
@@ -1388,9 +1328,6 @@ xla_test(
 xla_test(
     name = "broadcast_simple_test",
     srcs = ["broadcast_simple_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:array4d",
@@ -1410,9 +1347,6 @@ xla_test(
 xla_test(
     name = "pad_test",
     srcs = ["pad_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:array4d",
@@ -1434,9 +1368,6 @@ xla_test(
 xla_test(
     name = "fmax_test",
     srcs = ["fmax_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla/client:local_client",
         "//tensorflow/compiler/xla/client:xla_builder",
@@ -1451,9 +1382,6 @@ xla_test(
 xla_test(
     name = "log_test",
     srcs = ["log_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla/client:local_client",
         "//tensorflow/compiler/xla/client:xla_builder",
@@ -1468,9 +1396,6 @@ xla_test(
 xla_test(
     name = "matrix_ops_simple_test",
     srcs = ["matrix_ops_simple_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:literal",
@@ -1497,6 +1422,10 @@ xla_test(
 xla_test(
     name = "prng_test",
     srcs = ["prng_test.cc"],
+    blacklisted_backends = [
+        # TODO(b/122047800) support RNGs on the interpreter backend.
+        "interpreter",
+    ],
     deps = [
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:shape_util",
@@ -1517,9 +1446,6 @@ xla_test(
     name = "reshape_test",
     srcs = ["reshape_test.cc"],
     shard_count = 30,
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:array4d",
@@ -1545,9 +1471,6 @@ xla_test(
 xla_test(
     name = "reverse_test",
     srcs = ["reverse_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:array4d",
@@ -1566,9 +1489,6 @@ xla_test(
 xla_test(
     name = "vector_ops_simple_test",
     srcs = ["vector_ops_simple_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array4d",
         "//tensorflow/compiler/xla:shape_util",
@@ -1592,9 +1512,6 @@ xla_test(
 xla_test(
     name = "concat_test",
     srcs = ["concat_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:array3d",
@@ -1615,9 +1532,6 @@ xla_test(
 xla_test(
     name = "convert_test",
     srcs = ["convert_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:xla_data_proto",
@@ -1637,6 +1551,10 @@ xla_test(
 xla_test(
     name = "all_reduce_test",
     srcs = ["all_reduce_test.cc"],
+    blacklisted_backends = [
+        # All reduce is not supported on the interpreter backend.
+        "interpreter",
+    ],
     deps = [
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:shape_util",
@@ -1661,9 +1579,6 @@ xla_test(
 xla_test(
     name = "bitcast_convert_test",
     srcs = ["bitcast_convert_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:xla_data_proto",
@@ -1703,9 +1618,6 @@ xla_test(
 xla_test(
     name = "floor_ceil_test",
     srcs = ["floor_ceil_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla/client:local_client",
         "//tensorflow/compiler/xla/client:xla_builder",
@@ -1767,6 +1679,10 @@ xla_test(
 xla_test(
     name = "execution_profile_test",
     srcs = ["execution_profile_test.cc"],
+    blacklisted_backends = [
+        # Execution profiles are not supported on the interpreter backend.
+        "interpreter",
+    ],
     deps = [
         ":client_library_test_base",
         "//tensorflow/compiler/xla/client:global_data",
@@ -1781,6 +1697,10 @@ xla_test(
     name = "execution_profile_test_with_xla_hlo_profile",
     srcs = ["execution_profile_test.cc"],
     args = ["--xla_hlo_profile"],
+    blacklisted_backends = [
+        # Hlo profiles are not supported on the interpreter backend.
+        "interpreter",
+    ],
     deps = [
         ":client_library_test_base",
         "//tensorflow/compiler/xla/client:global_data",
@@ -1794,9 +1714,6 @@ xla_test(
 xla_test(
     name = "replay_test",
     srcs = ["replay_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:protobuf_util",
@@ -1819,9 +1736,6 @@ xla_test(
 xla_test(
     name = "broadcast_test",
     srcs = ["broadcast_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:shape_util",
@@ -1883,9 +1797,6 @@ xla_test(
 xla_test(
     name = "fusion_test",
     srcs = ["fusion_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:literal",
@@ -2003,6 +1914,10 @@ xla_test(
 xla_test(
     name = "outfeed_in_nested_computation_test",
     srcs = ["outfeed_in_nested_computation_test.cc"],
+    blacklisted_backends = [
+        # Outfeed ops are not supported on the interpreter backend.
+        "interpreter",
+    ],
     deps = [
         "//tensorflow/compiler/xla/tests:local_client_test_base",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
@@ -2179,7 +2094,6 @@ xla_test(
     srcs = ["iota_test.cc"],
     shard_count = 30,
     tags = [
-        "enable_for_xla_interpreter",
         # Require optimized builds, iota_test_cpu is very slow in fastbuild.
         "optonly",
     ],
diff --git a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
index 915b456b52..e4cc8b4199 100644
--- a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
+++ b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
@@ -2047,6 +2047,19 @@ XLA_TEST_F(ArrayElementwiseOpTest, NonNanClampF32) {
                              error_spec_);
 }
 
+XLA_TEST_F(ArrayElementwiseOpTest, ClampF32) {
+  SetFastMathDisabled(true);
+  XlaBuilder builder(TestName());
+  auto minimum = ConstantR1<float>(&builder, {1.0f, -6.5f, 1.0f, 2.25f, NAN});
+  auto argument =
+      ConstantR1<float>(&builder, {2.0f, 10.0f, -5.0f, 1.0f, 10.0f});
+  auto maximum = ConstantR1<float>(&builder, {3.0f, 0.5f, 25.5f, NAN, 123.0f});
+  Clamp(minimum, argument, maximum);
+
+  ComputeAndCompareR1<float>(&builder, {2.0f, 0.5f, 1.0f, NAN, NAN}, {},
+                             error_spec_);
+}
+
 XLA_TEST_F(ArrayElementwiseOpTest, ClampF32Scalar) {
   XlaBuilder builder(TestName());
   auto minimum = ConstantR0<float>(&builder, 0.0f);
diff --git a/tensorflow/compiler/xla/tests/bfloat16_test.cc b/tensorflow/compiler/xla/tests/bfloat16_test.cc
index e9728e636f..63e4811705 100644
--- a/tensorflow/compiler/xla/tests/bfloat16_test.cc
+++ b/tensorflow/compiler/xla/tests/bfloat16_test.cc
@@ -76,7 +76,9 @@ XLA_TEST_F(Bfloat16Test, NegateScalarF16) {
                                 error_spec_);
 }
 
-XLA_TEST_F(Bfloat16Test, BatchNormTraining) {
+// Disabled on interpreter since BatchNormExanper is not run by default on the
+// intepreter backend.
+XLA_TEST_F(Bfloat16Test, DISABLED_ON_INTERPRETER(BatchNormTraining)) {
   const int kFeatureIndex = 2;
   XlaBuilder builder(TestName());
 
@@ -110,7 +112,9 @@ XLA_TEST_F(Bfloat16Test, BatchNormTraining) {
   ComputeAndCompareTuple(&builder, expected, {}, ErrorSpec(0.01, 0.02));
 }
 
-XLA_TEST_F(Bfloat16Test, BatchNormGrad) {
+// Disabled on interpreter since BatchNormExanper is not run by default on the
+// intepreter backend.
+XLA_TEST_F(Bfloat16Test, DISABLED_ON_INTERPRETER(BatchNormGrad)) {
   const int kFeatureIndex = 2;
   XlaBuilder builder(TestName());
 
diff --git a/tensorflow/compiler/xla/tests/client_test.cc b/tensorflow/compiler/xla/tests/client_test.cc
index d5b3a4d14f..247328b730 100644
--- a/tensorflow/compiler/xla/tests/client_test.cc
+++ b/tensorflow/compiler/xla/tests/client_test.cc
@@ -109,7 +109,10 @@ XLA_TEST_F(ClientTest, ExecuteWithTupleLayout) {
                                      /*minor_to_major=*/{1, 0})));
 }
 
-XLA_TEST_F(ClientTest, DISABLED_ON_GPU(ExecuteParallel)) {
+// Disabled for interpreter since ExecuteAsyncOnStream is not implemented on
+// interpreter backend.
+XLA_TEST_F(ClientTest,
+           DISABLED_ON_INTERPRETER(DISABLED_ON_GPU(ExecuteParallel))) {
   XlaComputation add_with_one_arg, mul_with_two_args, dot_with_one_arg;
   Shape shape = ShapeUtil::MakeShape(S32, {2, 2});
 
diff --git a/tensorflow/compiler/xla/tests/gather_operation_test.cc b/tensorflow/compiler/xla/tests/gather_operation_test.cc
index daa89398a6..d65b67a535 100644
--- a/tensorflow/compiler/xla/tests/gather_operation_test.cc
+++ b/tensorflow/compiler/xla/tests/gather_operation_test.cc
@@ -600,7 +600,9 @@ ENTRY main {
 
 class GatherClientLibraryTest : public ClientLibraryTestBase {};
 
-XLA_TEST_F(GatherClientLibraryTest, DISABLED_ON_GPU(Basic)) {
+// Disabled on interpreter since ExectuteAsyncOnStream is not supported.
+XLA_TEST_F(GatherClientLibraryTest,
+           DISABLED_ON_INTERPRETER(DISABLED_ON_GPU(Basic))) {
   // We create this HLO, but using the XlaBuilder API.
   //
   // ENTRY main {
diff --git a/tensorflow/compiler/xla/tests/local_client_execute_test.cc b/tensorflow/compiler/xla/tests/local_client_execute_test.cc
index 6522c56325..96527886b7 100644
--- a/tensorflow/compiler/xla/tests/local_client_execute_test.cc
+++ b/tensorflow/compiler/xla/tests/local_client_execute_test.cc
@@ -842,7 +842,8 @@ XLA_TEST_F(LocalClientExecuteTest, ShapeBufferToLiteralConversion64bit) {
        LiteralUtil::CreateR0<int64>(123456789000LL)}));
 }
 
-XLA_TEST_F(LocalClientExecuteTest, InfeedTest) {
+// Disabled on interpreter backend since infeed HLO is unsupported.
+XLA_TEST_F(LocalClientExecuteTest, DISABLED_ON_INTERPRETER(InfeedTest)) {
   XlaBuilder builder(TestName());
   const Shape shape = ShapeUtil::MakeShape(F32, {3});
   auto in = Infeed(&builder, shape);
@@ -867,7 +868,8 @@ XLA_TEST_F(LocalClientExecuteTest, InfeedTest) {
   LiteralTestUtil::ExpectR1Equal<float>({-4.0, 125.0, 45.0}, result);
 }
 
-XLA_TEST_F(LocalClientExecuteTest, InfeedOutfeedTest) {
+// Disabled on interpreter backend since infeed/outfeed HLOs are unsupported.
+XLA_TEST_F(LocalClientExecuteTest, DISABLED_ON_INTERPRETER(InfeedOutfeedTest)) {
   XlaBuilder builder(TestName());
   const Shape shape = ShapeUtil::MakeShape(F32, {3});
   auto in = Infeed(&builder, shape);
-- 
GitLab


From 4435d81822bef10437b3ec7a73300db62a5a53ad Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Wed, 2 Jan 2019 16:16:35 -0800
Subject: [PATCH 0111/2345] Throw a more helpful error message when trying to
 export a subclassed Model

Previously it said to call _set_inputs. More directly people should just specify signatures=...

PiperOrigin-RevId: 227599645
---
 tensorflow/python/saved_model/save_test.py | 25 ++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/tensorflow/python/saved_model/save_test.py b/tensorflow/python/saved_model/save_test.py
index 4573495998..e643ff3dbf 100644
--- a/tensorflow/python/saved_model/save_test.py
+++ b/tensorflow/python/saved_model/save_test.py
@@ -31,6 +31,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_spec
 from tensorflow.python.framework import test_util
+from tensorflow.python.keras.engine import training
 from tensorflow.python.keras.layers import core
 from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import lookup_ops
@@ -306,6 +307,30 @@ class SaveTest(test.TestCase):
       self.assertNotIn("T", complex_node.attr)
       self.assertNotIn("Tout", complex_node.attr)
 
+  def test_subclassed_no_signature(self):
+
+    class Subclassed(training.Model):
+
+      def call(self, inputs):
+        return inputs * 2.
+
+    save_dir = os.path.join(self.get_temp_dir(), "saved_model")
+    model = Subclassed()
+    with self.assertRaisesRegexp(
+        ValueError, "no @tf.function-decorated methods"):
+      save.save(model, save_dir)
+
+    traced_call = def_function.function(
+        model.call,
+        input_signature=(tensor_spec.TensorSpec(
+            (None, None),
+            dtype=dtypes.float32),))
+    save.save(model, save_dir, traced_call)
+    self.assertAllClose({"output_0": [[8., 10.], [10., 12.]]},
+                        _import_and_infer(
+                            save_dir,
+                            {"inputs": [[4., 5.], [5., 6.]]}))
+
 
 class AssetTests(test.TestCase):
 
-- 
GitLab


From da7b71f67147ff4795c5c0168d1f225ba2b4b522 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 2 Jan 2019 16:32:13 -0800
Subject: [PATCH 0112/2345] Automated rollback of commit
 7d2be9eef5f4b74da5b5c6f473ea2e6ad57398dc

PiperOrigin-RevId: 227601883
---
 .../lite/delegates/nnapi/nnapi_delegate.cc    |  161 ++-
 tensorflow/lite/nnapi/BUILD                   |   12 -
 tensorflow/lite/nnapi/NeuralNetworksShim.cc   |  137 --
 tensorflow/lite/nnapi/NeuralNetworksShim.h    | 1205 +++++++++--------
 tensorflow/lite/nnapi/nnapi_lib_test.cc       |   90 --
 tensorflow/lite/nnapi_delegate.cc             |  135 +-
 6 files changed, 809 insertions(+), 931 deletions(-)
 delete mode 100644 tensorflow/lite/nnapi/NeuralNetworksShim.cc
 delete mode 100644 tensorflow/lite/nnapi/nnapi_lib_test.cc

diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
index 9487d3331f..8af159e6fb 100644
--- a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
+++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
@@ -48,39 +48,57 @@ namespace {
   } while (0)
 
 namespace {
+int32_t GetAndroidSdkVersion() {
+#ifdef __ANDROID__
+  const char* sdkProp = "ro.build.version.sdk";
+  char sdkVersion[PROP_VALUE_MAX];
+  int length = __system_property_get(sdkProp, sdkVersion);
+  if (length != 0) {
+    for (int i = 0; i < length; ++i) {
+      int digit = sdkVersion[i] - '0';
+      if (digit < 0 || digit > 9) {
+        // Non-numeric SDK version, assume it's higher then expected;
+        return std::numeric_limits<int32_t>::max();
+      }
+    }
+    return atoi(sdkVersion);
+  }
+#endif  // __ANDROID__
+  return 0;
+}
+
 constexpr int32_t kMinSdkVersionForNNAPI = 27;
 constexpr int32_t kMinSdkVersionForNNAPI11 = 28;
+static const int32_t kAndroidSdkVersion = GetAndroidSdkVersion();
+
 }  // namespace
 
 // RAII NN API Model Destructor for use with std::unique_ptr
 struct NNFreeModel {
   void operator()(ANeuralNetworksModel* model) {
-    NnApiImplementation()->ANeuralNetworksModel_free(model);
+    ANeuralNetworksModel_free(model);
   }
 };
 // RAII NN API Compilation Destructor for use with std::unique_ptr
 struct NNFreeCompilation {
   void operator()(ANeuralNetworksCompilation* model) {
-    NnApiImplementation()->ANeuralNetworksCompilation_free(model);
+    ANeuralNetworksCompilation_free(model);
   }
 };
 
 // Manage NNAPI shared memory handle
 class NNMemory {
  public:
+  NNMemory(const char* name, size_t size) {
 #ifdef __ANDROID__
-  NNMemory(const NnApi* nnapi, const char* name, size_t size) {
-    nnapi_ = nnapi;
     byte_size_ = size;
-    fd_ = nnapi_->ASharedMemory_create(name, size);
+    fd_ = ASharedMemory_create(name, size);
     data_ptr_ = reinterpret_cast<uint8_t*>(
         mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, 0));
-    nnapi_->ANeuralNetworksMemory_createFromFd(size, PROT_READ | PROT_WRITE,
-                                               fd_, 0, &nn_memory_handle_);
-  }
-#else
-  NNMemory(const NnApi* /*nnapi*/, const char* /*name*/, size_t /*size*/) {}
+    ANeuralNetworksMemory_createFromFd(size, PROT_READ | PROT_WRITE, fd_, 0,
+                                       &nn_memory_handle_);
 #endif
+  }
 
   ~NNMemory() {
 #ifdef __ANDROID__
@@ -88,7 +106,7 @@ class NNMemory {
       munmap(data_ptr_, byte_size_);
     }
     if (nn_memory_handle_) {
-      nnapi_->ANeuralNetworksMemory_free(nn_memory_handle_);
+      ANeuralNetworksMemory_free(nn_memory_handle_);
     }
     if (fd_ > 0) close(fd_);
 #endif
@@ -99,7 +117,6 @@ class NNMemory {
 
  private:
 #ifdef __ANDROID__
-  const NnApi* nnapi_;
   int fd_ = 0;
   size_t byte_size_ = 0;
 #endif
@@ -149,10 +166,9 @@ class OperandMapping {
 // operands for both tensors and parameters, and TFLite separates the two.
 class NNAPIOpBuilder {
  public:
-  NNAPIOpBuilder(const NnApi* nnapi, TfLiteContext* context,
-                 OperandMapping* tensor_mapping, ANeuralNetworksModel* nn_model)
-      : nnapi_(nnapi),
-        context_(context),
+  NNAPIOpBuilder(TfLiteContext* context, OperandMapping* tensor_mapping,
+                 ANeuralNetworksModel* nn_model)
+      : context_(context),
         operand_mapping_(tensor_mapping),
         nn_model_(nn_model) {}
 
@@ -208,8 +224,7 @@ class NNAPIOpBuilder {
         .dimensionCount = dimension_count,
         .dimensions = dims.data()};
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_,
-        nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
+        context_, ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
     int ann_operand = operand_mapping_->add_new_non_tensor_operand();
     augmented_outputs_.push_back(ann_operand);
     return kTfLiteOk;
@@ -226,8 +241,7 @@ class NNAPIOpBuilder {
         reinterpret_cast<uint32_t*>(tensor->dims->data), tensor->params.scale,
         tensor->params.zero_point};
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_,
-        nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
+        context_, ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
     augmented_outputs_.push_back(ann_index);
 
     *ann_tensor_index_out = ann_index;
@@ -284,14 +298,13 @@ class NNAPIOpBuilder {
         nn_type, static_cast<uint32_t>(tensor->dims->size),
         reinterpret_cast<uint32_t*>(tensor->dims->data), scale, zeroPoint};
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_,
-        nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
+        context_, ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
 
     if (tensor->allocation_type == kTfLiteMmapRo) {
       // TODO(b/80630405): Use NNAPIAllocation.
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
           context_,
-          nnapi_->ANeuralNetworksModel_setOperandValue(
+          ANeuralNetworksModel_setOperandValue(
               nn_model_, ann_tensor_index, tensor->data.raw, tensor->bytes));
     }
 
@@ -304,7 +317,7 @@ class NNAPIOpBuilder {
     // Actually add a NN API operation
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
         context_,
-        nnapi_->ANeuralNetworksModel_addOperation(
+        ANeuralNetworksModel_addOperation(
             nn_model_, type, static_cast<uint32_t>(augmented_inputs_.size()),
             augmented_inputs_.data(),
             static_cast<uint32_t>(augmented_outputs_.size()),
@@ -319,12 +332,11 @@ class NNAPIOpBuilder {
   TfLiteStatus AddScalarOperand(T value, int32_t nn_type) {
     ANeuralNetworksOperandType operand_type{.type = nn_type};
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_,
-        nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
+        context_, ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
     int ann_operand = operand_mapping_->add_new_non_tensor_operand();
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_, nnapi_->ANeuralNetworksModel_setOperandValue(
-                      nn_model_, ann_operand, &value, sizeof(T)));
+        context_, ANeuralNetworksModel_setOperandValue(nn_model_, ann_operand,
+                                                       &value, sizeof(T)));
     augmented_inputs_.push_back(ann_operand);
     return kTfLiteOk;
   }
@@ -335,19 +347,15 @@ class NNAPIOpBuilder {
     ANeuralNetworksOperandType operand_type{
         .type = nn_type, .dimensionCount = 1, .dimensions = &num_values};
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_,
-        nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
+        context_, ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
     int ann_operand = operand_mapping_->add_new_non_tensor_operand();
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_, nnapi_->ANeuralNetworksModel_setOperandValue(
+        context_, ANeuralNetworksModel_setOperandValue(
                       nn_model_, ann_operand, values, sizeof(T) * num_values));
     augmented_inputs_.push_back(ann_operand);
     return kTfLiteOk;
   }
 
-  // Access to NNAPI.
-  const NnApi* const nnapi_;
-
   // TfLiteContext for error handling.
   TfLiteContext* const context_;
 
@@ -383,7 +391,7 @@ ANeuralNetworksOperationType BasicMappingFn(
 // The kernel that represents the node sub set of TF Lite being run on NN API.
 class NNAPIDelegateKernel {
  public:
-  NNAPIDelegateKernel() { nnapi_ = NnApiImplementation(); }
+  NNAPIDelegateKernel() = default;
 
   typedef ANeuralNetworksOperationType (*MappingFn)(
       const NNAPIOpMappingArgs& mapping_args);
@@ -392,7 +400,7 @@ class NNAPIDelegateKernel {
   // when called. You can use this function to see if a node is supported
   // (i.e. that MappingFn is not nullptr).
   static MappingFn Map(TfLiteContext* context, int builtin_code, int version,
-                       int android_sdk_version, TfLiteNode* node) {
+                       TfLiteNode* node) {
     switch (builtin_code) {
       case kTfLiteBuiltinAdd:
         if (version == 1) {
@@ -511,7 +519,7 @@ class NNAPIDelegateKernel {
         }
         break;
       case kTfLiteBuiltinSqueeze:
-        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11) {
+        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11) {
           return [](const NNAPIOpMappingArgs& mapping_args)
                      -> ANeuralNetworksOperationType {
             auto builtin = reinterpret_cast<TfLiteSqueezeParams*>(
@@ -627,7 +635,7 @@ class NNAPIDelegateKernel {
         }
         break;
       case kTfLiteBuiltinSub:
-        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11 &&
+        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11 &&
             context->tensors[node->inputs->data[0]].type == kTfLiteFloat32) {
           // NNAPI only support float sub.
           return [](const NNAPIOpMappingArgs& mapping_args)
@@ -640,7 +648,7 @@ class NNAPIDelegateKernel {
         }
         break;
       case kTfLiteBuiltinDiv:
-        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11 &&
+        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11 &&
             context->tensors[node->inputs->data[0]].type == kTfLiteFloat32) {
           // NNAPI only support float div.
           return [](const NNAPIOpMappingArgs& mapping_args)
@@ -653,7 +661,7 @@ class NNAPIDelegateKernel {
         }
         break;
       case kTfLiteBuiltinPad:
-        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11 &&
+        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11 &&
             node->inputs->size == 2 &&
             context->tensors[node->inputs->data[0]].type == kTfLiteFloat32) {
           // NNAPI does not support specifying the padding value.
@@ -663,12 +671,12 @@ class NNAPIDelegateKernel {
         }
         break;
       case kTfLiteBuiltinSpaceToBatchNd:
-        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11) {
+        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11) {
           return BasicMappingFn<ANEURALNETWORKS_SPACE_TO_BATCH_ND>;
         }
         break;
       case kTfLiteBuiltinStridedSlice:
-        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11) {
+        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11) {
           return [](const NNAPIOpMappingArgs& mapping_args)
                      -> ANeuralNetworksOperationType {
             auto builtin = reinterpret_cast<TfLiteStridedSliceParams*>(
@@ -686,7 +694,7 @@ class NNAPIDelegateKernel {
         // dimensions.
         // TODO(b/110888333): Support dynamically-sized tensors in delegates.
         if ((version == 1) &&
-            (android_sdk_version >= kMinSdkVersionForNNAPI11) &&
+            (kAndroidSdkVersion >= kMinSdkVersionForNNAPI11) &&
             (node->inputs->size > 1) &&
             (context->tensors[node->inputs->data[1]].allocation_type ==
              kTfLiteMmapRo)) {
@@ -784,7 +792,7 @@ class NNAPIDelegateKernel {
         break;
       case kTfLiteBuiltinMean:
         // NNAPI does not support generating a scalar as output for MEAN.
-        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11 &&
+        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11 &&
             context->tensors[node->inputs->data[0]].type == kTfLiteFloat32 &&
             context->tensors[node->outputs->data[0]].dims->size > 0) {
           return [](const NNAPIOpMappingArgs& mapping_args)
@@ -828,8 +836,8 @@ class NNAPIDelegateKernel {
 
     if (!nn_model_) {
       ANeuralNetworksModel* model;
-      RETURN_TFLITE_ERROR_IF_NN_ERROR(
-          context, nnapi_->ANeuralNetworksModel_create(&model));
+      RETURN_TFLITE_ERROR_IF_NN_ERROR(context,
+                                      ANeuralNetworksModel_create(&model));
       nn_model_.reset(model);
 
       TF_LITE_ENSURE_STATUS(
@@ -839,10 +847,10 @@ class NNAPIDelegateKernel {
     if (!nn_compilation_) {
       ANeuralNetworksCompilation* compilation;
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
-          context, nnapi_->ANeuralNetworksCompilation_create(nn_model_.get(),
-                                                             &compilation));
+          context,
+          ANeuralNetworksCompilation_create(nn_model_.get(), &compilation));
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
-          context, nnapi_->ANeuralNetworksCompilation_finish(compilation));
+          context, ANeuralNetworksCompilation_finish(compilation));
       nn_compilation_.reset(compilation);
     }
     return kTfLiteOk;
@@ -851,8 +859,8 @@ class NNAPIDelegateKernel {
   TfLiteStatus Invoke(TfLiteContext* context, TfLiteNode* node) {
     ANeuralNetworksExecution* execution = nullptr;
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context, nnapi_->ANeuralNetworksExecution_create(nn_compilation_.get(),
-                                                         &execution));
+        context,
+        ANeuralNetworksExecution_create(nn_compilation_.get(), &execution));
 
     // Set the input tensor buffers. Note: we access tflite tensors using
     // absolute indices but NN api indices inputs by relative indices.
@@ -872,7 +880,7 @@ class NNAPIDelegateKernel {
                tensor->data.raw, tensor->bytes);
         RETURN_TFLITE_ERROR_IF_NN_ERROR(
             context,
-            nnapi_->ANeuralNetworksExecution_setInputFromMemory(
+            ANeuralNetworksExecution_setInputFromMemory(
                 execution, relative_input_index, nullptr,
                 nn_input_memory_->get_handle(), input_offset, tensor->bytes));
         input_offset += tensor->bytes;
@@ -887,7 +895,7 @@ class NNAPIDelegateKernel {
       TfLiteTensor* tensor = &context->tensors[output_index];
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
           context,
-          nnapi_->ANeuralNetworksExecution_setOutputFromMemory(
+          ANeuralNetworksExecution_setOutputFromMemory(
               execution, relative_output_index, nullptr,
               nn_output_memory_->get_handle(), output_offset, tensor->bytes));
       output_offset += tensor->bytes;
@@ -903,7 +911,7 @@ class NNAPIDelegateKernel {
       // reading and writing into the same buffer during a invocation.
       // TODO(110369471): using double shared buffer to minimize the copies.
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
-          context, nnapi_->ANeuralNetworksExecution_setOutput(
+          context, ANeuralNetworksExecution_setOutput(
                        execution, relative_output_index, nullptr,
                        tensor->data.raw, tensor->bytes));
       relative_output_index++;
@@ -911,12 +919,10 @@ class NNAPIDelegateKernel {
     // Invoke ANN in blocking fashion.
     ANeuralNetworksEvent* event = nullptr;
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context,
-        nnapi_->ANeuralNetworksExecution_startCompute(execution, &event));
-    RETURN_TFLITE_ERROR_IF_NN_ERROR(context,
-                                    nnapi_->ANeuralNetworksEvent_wait(event));
-    nnapi_->ANeuralNetworksEvent_free(event);
-    nnapi_->ANeuralNetworksExecution_free(execution);
+        context, ANeuralNetworksExecution_startCompute(execution, &event));
+    RETURN_TFLITE_ERROR_IF_NN_ERROR(context, ANeuralNetworksEvent_wait(event));
+    ANeuralNetworksEvent_free(event);
+    ANeuralNetworksExecution_free(execution);
 
     // copy results from shared memory to the destination.
     output_offset = 0;
@@ -931,8 +937,6 @@ class NNAPIDelegateKernel {
   }
 
  private:
-  // Access to NNApi.
-  const NnApi* nnapi_;
   // ANN API state.
   std::unique_ptr<ANeuralNetworksModel, NNFreeModel> nn_model_;
   std::unique_ptr<ANeuralNetworksCompilation, NNFreeCompilation>
@@ -953,7 +957,7 @@ class NNAPIDelegateKernel {
     // The operand builder allows creating a single op. We create it at this
     // reduced power position rather than in the for loop to avoid reallocating
     // the vectors.
-    NNAPIOpBuilder builder(nnapi_, context, &operand_mapping_, nn_model_.get());
+    NNAPIOpBuilder builder(context, &operand_mapping_, nn_model_.get());
     // Add Tensors
     // allocate outside to avoid realloc
     for (auto node_index : nodes_) {
@@ -976,10 +980,9 @@ class NNAPIDelegateKernel {
         }
       }
       // Get op type and operands
-      int nn_op_type = Map(
-          context, reg->builtin_code, reg->version, nnapi_->android_sdk_version,
-          node)({context, &builder, node, &model_state_outputs_,
-                 &model_state_tfl_inputs_});
+      int nn_op_type = Map(context, reg->builtin_code, reg->version, node)(
+          {context, &builder, node, &model_state_outputs_,
+           &model_state_tfl_inputs_});
       // Map outputs to NN API tensor indices.
       for (auto output_index : TfLiteIntArrayView(node->outputs)) {
         TF_LITE_ENSURE_STATUS(builder.AddTensorOutput(output_index));
@@ -1025,27 +1028,25 @@ class NNAPIDelegateKernel {
 
     // Tell ANN to declare inputs/outputs
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context, nnapi_->ANeuralNetworksModel_identifyInputsAndOutputs(
+        context, ANeuralNetworksModel_identifyInputsAndOutputs(
                      nn_model_.get(), inputs.size(), inputs.data(),
                      outputs.size(), outputs.data()));
 
     // Set relaxed computation mode for fp32 if possible.
-    if (nnapi_->android_sdk_version >= kMinSdkVersionForNNAPI11) {
+    if (kAndroidSdkVersion >= kMinSdkVersionForNNAPI11) {
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
-          context,
-          nnapi_->ANeuralNetworksModel_relaxComputationFloat32toFloat16(
-              nn_model_.get(), context->allow_fp32_relax_to_fp16));
+          context, ANeuralNetworksModel_relaxComputationFloat32toFloat16(
+                       nn_model_.get(), context->allow_fp32_relax_to_fp16));
     }
 
     // Finalize the model
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context, nnapi_->ANeuralNetworksModel_finish(nn_model_.get()));
+        context, ANeuralNetworksModel_finish(nn_model_.get()));
 
     // Create shared memory pool for inputs and outputs.
-    nn_input_memory_.reset(
-        new NNMemory(nnapi_, "input_pool", total_input_byte_size));
+    nn_input_memory_.reset(new NNMemory("input_pool", total_input_byte_size));
     nn_output_memory_.reset(
-        new NNMemory(nnapi_, "output_pool", total_output_byte_size));
+        new NNMemory("output_pool", total_output_byte_size));
 
     return kTfLiteOk;
   }
@@ -1061,9 +1062,7 @@ TfLiteDelegate* NnApiDelegate() {
       .Prepare = [](TfLiteContext* context,
                     TfLiteDelegate* delegate) -> TfLiteStatus {
         // Do not check nodes_ if NN API is unavailable.
-        const NnApi* nnapi = NnApiImplementation();
-        if (nnapi->android_sdk_version < kMinSdkVersionForNNAPI ||
-            !nnapi->nnapi_exists) {
+        if (kAndroidSdkVersion < kMinSdkVersionForNNAPI || !NNAPIExists()) {
           return kTfLiteOk;
         }
 
@@ -1076,7 +1075,6 @@ TfLiteDelegate* NnApiDelegate() {
         TfLiteIntArray* plan;
         TF_LITE_ENSURE_STATUS(context->GetExecutionPlan(context, &plan));
 
-        int android_sdk_version = NnApiImplementation()->android_sdk_version;
         // Check for every node if it is supported
         // TODO(b/80625235): Fix this to do more careful checking of versioning.
         for (int node_index : TfLiteIntArrayView(plan)) {
@@ -1085,8 +1083,7 @@ TfLiteDelegate* NnApiDelegate() {
           TF_LITE_ENSURE_STATUS(context->GetNodeAndRegistration(
               context, node_index, &node, &registration));
           if (NNAPIDelegateKernel::Map(context, registration->builtin_code,
-                                       registration->version,
-                                       android_sdk_version, node)) {
+                                       registration->version, node)) {
             supported_nodes.push_back(node_index);
           }
         }
diff --git a/tensorflow/lite/nnapi/BUILD b/tensorflow/lite/nnapi/BUILD
index 390c3730cb..467a2b7a7b 100644
--- a/tensorflow/lite/nnapi/BUILD
+++ b/tensorflow/lite/nnapi/BUILD
@@ -6,20 +6,8 @@ package(default_visibility = [
 
 cc_library(
     name = "nnapi_lib",
-    srcs = [
-        "NeuralNetworksShim.cc",
-    ],
     hdrs = [
         "NeuralNetworksShim.h",
     ],
     linkopts = ["-ldl"],
 )
-
-cc_test(
-    name = "nnapi_lib_test",
-    srcs = ["nnapi_lib_test.cc"],
-    deps = [
-        "//tensorflow/lite/nnapi:nnapi_lib",
-        "@com_google_googletest//:gtest_main",
-    ],
-)
diff --git a/tensorflow/lite/nnapi/NeuralNetworksShim.cc b/tensorflow/lite/nnapi/NeuralNetworksShim.cc
deleted file mode 100644
index c063525cae..0000000000
--- a/tensorflow/lite/nnapi/NeuralNetworksShim.cc
+++ /dev/null
@@ -1,137 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include "tensorflow/lite/nnapi/NeuralNetworksShim.h"
-
-#include <cstdlib>
-
-#ifdef __ANDROID__
-#include <dlfcn.h>
-#include <sys/mman.h>
-#include <sys/system_properties.h>
-#include <unistd.h>
-#endif
-
-#define NNAPI_LOG(format, ...) fprintf(stderr, format "\n", __VA_ARGS__);
-
-namespace {
-
-#ifdef __ANDROID__
-int32_t GetAndroidSdkVersion() {
-  const char* sdkProp = "ro.build.version.sdk";
-  char sdkVersion[PROP_VALUE_MAX];
-  int length = __system_property_get(sdkProp, sdkVersion);
-  if (length != 0) {
-    int32_t result = 0;
-    for (int i = 0; i < length; ++i) {
-      int digit = sdkVersion[i] - '0';
-      if (digit < 0 || digit > 9) {
-        // Non-numeric SDK version, assume it's higher than expected;
-        return 0xffff;
-      }
-      result = result * 10 + digit;
-    }
-    return result;
-  }
-  return 0;
-}
-
-void* LoadFunction(void* handle, const char* name) {
-  if (handle == nullptr) {
-    return nullptr;
-  }
-  void* fn = dlsym(handle, name);
-  if (fn == nullptr) {
-    NNAPI_LOG("nnapi error: unable to open function %s", name);
-  }
-  return fn;
-}
-
-#define LOAD_FUNCTION(handle, name) \
-  nnapi.name = reinterpret_cast<name##_fn>(LoadFunction(handle, #name));
-
-#else
-
-#define LOAD_FUNCTION(handle, name) nnapi.name = nullptr;
-
-#endif
-
-const NnApi LoadNnApi() {
-  NnApi nnapi = {};
-
-#ifdef __ANDROID__
-  void* libneuralnetworks = nullptr;
-  void* libandroid = nullptr;
-  nnapi.android_sdk_version = GetAndroidSdkVersion();
-  if (nnapi.android_sdk_version < 27) {
-    NNAPI_LOG("nnapi error: requires android sdk version to be at least %d",
-              27);
-  } else {
-    // TODO: change RTLD_LOCAL? Assumes there can be multiple instances of nn
-    // api RT
-    libneuralnetworks = dlopen("libneuralnetworks.so", RTLD_LAZY | RTLD_LOCAL);
-    if (libneuralnetworks == nullptr) {
-      NNAPI_LOG("nnapi error: unable to open library %s",
-                "libneuralnetworks.so");
-    }
-    libandroid = dlopen("libandroid.so", RTLD_LAZY | RTLD_LOCAL);
-    if (libneuralnetworks == nullptr) {
-      NNAPI_LOG("nnapi error: unable to open library %s", "libandroid.so");
-    }
-  }
-  nnapi.nnapi_exists = libneuralnetworks != nullptr;
-#else
-  nnapi.nnapi_exists = false;
-  nnapi.android_sdk_version = 0;
-#endif
-
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksMemory_createFromFd);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksMemory_free);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_create);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_free);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_finish);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_addOperand);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_setOperandValue);
-  LOAD_FUNCTION(libneuralnetworks,
-                ANeuralNetworksModel_setOperandValueFromMemory);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_addOperation);
-  LOAD_FUNCTION(libneuralnetworks,
-                ANeuralNetworksModel_identifyInputsAndOutputs);
-  LOAD_FUNCTION(libneuralnetworks,
-                ANeuralNetworksModel_relaxComputationFloat32toFloat16);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksCompilation_create);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksCompilation_free);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksCompilation_setPreference);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksCompilation_finish);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_create);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_free);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_setInput);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_setInputFromMemory);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_setOutput);
-  LOAD_FUNCTION(libneuralnetworks,
-                ANeuralNetworksExecution_setOutputFromMemory);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_startCompute);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksEvent_wait);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksEvent_free);
-  LOAD_FUNCTION(libandroid, ASharedMemory_create);
-
-  return nnapi;
-}
-
-}  // namespace
-
-const NnApi* NnApiImplementation() {
-  static const NnApi nnapi = LoadNnApi();
-  return &nnapi;
-}
diff --git a/tensorflow/lite/nnapi/NeuralNetworksShim.h b/tensorflow/lite/nnapi/NeuralNetworksShim.h
index 3fd869e4af..c39502f4ac 100644
--- a/tensorflow/lite/nnapi/NeuralNetworksShim.h
+++ b/tensorflow/lite/nnapi/NeuralNetworksShim.h
@@ -15,10 +15,69 @@ limitations under the License.
 #ifndef TENSORFLOW_LITE_NNAPI_NEURALNETWORKSSHIM_H_
 #define TENSORFLOW_LITE_NNAPI_NEURALNETWORKSSHIM_H_
 
+#include <dlfcn.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 
+// helpers
+
+#define NNAPI_LOG(format, ...) fprintf(stderr, format "\n", __VA_ARGS__);
+#define LOAD_FUNCTION(name) \
+  static name##_fn fn = reinterpret_cast<name##_fn>(loadFunction(#name));
+#define EXECUTE_FUNCTION(...) \
+  if (fn != nullptr) {        \
+    fn(__VA_ARGS__);          \
+  }
+#define EXECUTE_FUNCTION_RETURN(...) return fn != nullptr ? fn(__VA_ARGS__) : 0;
+
+inline void* loadLibrary(const char* name) {
+  // TODO: change RTLD_LOCAL? Assumes there can be multiple instances of nn
+  // api RT
+  void* handle = nullptr;
+#ifdef __ANDROID__
+  handle = dlopen(name, RTLD_LAZY | RTLD_LOCAL);
+  if (handle == nullptr) {
+    NNAPI_LOG("nnapi error: unable to open library %s", name);
+  }
+#endif
+  return handle;
+}
+
+typedef int (*ASharedMemory_create_fn)(const char* name, size_t size);
+
+// ASharedMemory_create was added in Android 8.0, so safe to use with NNAPI
+// which was added in 8.1.
+inline int ASharedMemory_create(const char* name, size_t size) {
+  static void* handle = loadLibrary("libandroid.so");
+  static ASharedMemory_create_fn fn =
+      handle != nullptr ? reinterpret_cast<ASharedMemory_create_fn>(
+                              dlsym(handle, "ASharedMemory_create"))
+                        : nullptr;
+  return fn(name, size);
+}
+
+inline void* getLibraryHandle() {
+  static void* handle = loadLibrary("libneuralnetworks.so");
+  return handle;
+}
+
+inline void* loadFunction(const char* name) {
+  void* fn = nullptr;
+  if (getLibraryHandle() != nullptr) {
+    fn = dlsym(getLibraryHandle(), name);
+  }
+  if (fn == nullptr) {
+    NNAPI_LOG("nnapi error: unable to open function %s", name);
+  }
+  return fn;
+}
+
+inline bool NNAPIExists() {
+  static bool nnapi_is_available = getLibraryHandle();
+  return nnapi_is_available;
+}
+
 // NN api types based on NNAPI header file
 // https://developer.android.com/ndk/reference/group/neural-networks
 
@@ -348,564 +407,606 @@ typedef int (*ANeuralNetworksEvent_wait_fn)(ANeuralNetworksEvent* event);
 
 typedef void (*ANeuralNetworksEvent_free_fn)(ANeuralNetworksEvent* event);
 
-typedef int (*ASharedMemory_create_fn)(const char* name, size_t size);
+/**
+ * Creates a shared memory object from a file descriptor.
+ *
+ * The shared memory is backed by a file descriptor via mmap.
+ * See {@link ANeuralNetworksMemory} for a description on how to use
+ * this shared memory.
+ *
+ * @param size The requested size in bytes.
+ *             Must not be larger than the file size.
+ * @param prot The desired memory protection for the mapping.
+ *             It is either PROT_NONE or the bitwise OR of one or
+ *             more of the following flags: PROT_READ, PROT_WRITE.
+ * @param fd The requested file descriptor.
+ *           The file descriptor has to be mmap-able. The file
+ *           descriptor will be duplicated.
+ * @param offset The offset to the beginning of the file of the area to map.
+ *               The offset has to be aligned to a page size.
+ * @param memory The memory object to be created.
+ *               Set to NULL if unsuccessful.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if the request completed normally.
+ */
+inline int ANeuralNetworksMemory_createFromFd(size_t size, int protect, int fd,
+                                              size_t offset,
+                                              ANeuralNetworksMemory** memory) {
+  LOAD_FUNCTION(ANeuralNetworksMemory_createFromFd);
+  EXECUTE_FUNCTION_RETURN(size, protect, fd, offset, memory);
+}
 
-struct NnApi {
-  bool nnapi_exists;
-  int32_t android_sdk_version;
-
-  /**
-   * Creates a shared memory object from a file descriptor.
-   *
-   * The shared memory is backed by a file descriptor via mmap.
-   * See {@link ANeuralNetworksMemory} for a description on how to use
-   * this shared memory.
-   *
-   * @param size The requested size in bytes.
-   *             Must not be larger than the file size.
-   * @param prot The desired memory protection for the mapping.
-   *             It is either PROT_NONE or the bitwise OR of one or
-   *             more of the following flags: PROT_READ, PROT_WRITE.
-   * @param fd The requested file descriptor.
-   *           The file descriptor has to be mmap-able. The file
-   *           descriptor will be duplicated.
-   * @param offset The offset to the beginning of the file of the area to map.
-   *               The offset has to be aligned to a page size.
-   * @param memory The memory object to be created.
-   *               Set to NULL if unsuccessful.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if the request completed normally.
-   */
-  int (*ANeuralNetworksMemory_createFromFd)(size_t size, int protect, int fd,
-                                            size_t offset,
-                                            ANeuralNetworksMemory** memory);
-
-  /**
-   * Delete a memory object.
-   *
-   * Destroys the object used by the run time to keep track of the memory.
-   * This will free the underlying actual memory if no other code has open
-   * handles to this memory.
-   *
-   * @param memory The memory object to be freed.
-   */
-  void (*ANeuralNetworksMemory_free)(ANeuralNetworksMemory* memory);
-
-  /**
-   * Create an empty {@link ANeuralNetworksModel}.
-   *
-   * <p>This only creates the object. Computation is performed once
-   * {@link ANeuralNetworksExecution_startCompute} is invoked.
-   *
-   * The model should be constructed with calls to
-   * {@link ANeuralNetworksModel_addOperation} and
-   * {@link ANeuralNetworksModel_addOperand}
-   *
-   * <p>{@link ANeuralNetworksModel_finish} should be called once the model
-   * has been fully constructed.</p>
-   *
-   * <p>{@link ANeuralNetworksModel_free} should be called once the model
-   * is no longer needed.</p>
-   *
-   * @param model The {@link ANeuralNetworksModel} to be created.
-   *              Set to NULL if unsuccessful.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if successful.
-   */
-  int (*ANeuralNetworksModel_create)(ANeuralNetworksModel** model);
-
-  /**
-   * Destroy a model.
-   *
-   * The model need not have been finished by a call to
-   * {@link ANeuralNetworksModel_finish}.
-   *
-   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
-   *
-   * @param model The model to be destroyed. Passing NULL is acceptable and
-   *              results in no operation.
-   */
-  void (*ANeuralNetworksModel_free)(ANeuralNetworksModel* model);
-
-  /**
-   * Indicate that we have finished modifying a model. Required before
-   * calling {@link ANeuralNetworksCompilation_compile}.
-   *
-   * An application is responsible to make sure that no other thread uses
-   * the model at the same time.
-   *
-   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
-   *
-   * @param model The model to be finished.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if successful.
-   */
-  int (*ANeuralNetworksModel_finish)(ANeuralNetworksModel* model);
-
-  /**
-   * Add an operand to a model.
-   *
-   * The order in which the operands are added is important. The first one added
-   * to a model will have the index value 0, the second 1, etc. These indexes
-   * are used as operand identifiers in
-   * {@link ANeuralNetworksModel_addOperation},
-   * {@link ANeuralNetworksExecution_setInput},
-   * {@link ANeuralNetworksExecution_setInputFromMemory},
-   * {@link ANeuralNetworksExecution_setOutput},
-   * {@link ANeuralNetworksExecution_setOutputFromMemory} and
-   * {@link ANeuralNetworksExecution_setOperandValue}.
-   *
-   * To build a model that can accommodate inputs of various sizes, as you may
-   * want to do for a CNN, set the size of the dimensions that will vary at run
-   * time to 0. If you do so, provide the full dimensions when calling
-   * {@link ANeuralNetworksExecution_setInput} or {@link
-   * ANeuralNetworksExecution_setInputFromMemory}.
-   *
-   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
-   * been called will return an error.
-   *
-   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
-   *
-   * @param model The model to be modified.
-   * @param type The {@link ANeuralNetworksOperandType} that describes the shape
-   * of the operand.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if successful.
-   */
-  int (*ANeuralNetworksModel_addOperand)(
-      ANeuralNetworksModel* model, const ANeuralNetworksOperandType* type);
-
-  /**
-   * Sets an operand to a constant value.
-   *
-   * For scalar values, the content of buffer is copied into the model.
-   *
-   * For tensor values, a pointer to the buffer is stored within the model.
-   * The application is responsible for not changing the content of this region
-   * until all executions using this model have completed. As the data may
-   * be copied during processing, modifying the data after this call yields
-   * undefined results.
-   *
-   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
-   * been called will return an error.
-   *
-   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
-   *
-   * @param model The model to be modified.
-   * @param index The index of the model operand we're setting.
-   * @param buffer A pointer to the data to use.
-   * @param length The size in bytes of the data value.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if successful.
-   */
-  int (*ANeuralNetworksModel_setOperandValue)(ANeuralNetworksModel* model,
-                                              int32_t index, const void* buffer,
-                                              size_t length);
-
-  /**
-   * Sets an operand to a value stored in a memory object.
-   *
-   * The content of the memory is not copied. A reference to that memory is
-   * stored inside the model. The application is responsible for not changing
-   * the content of the memory region until all executions using this model have
-   * completed.
-   * As the data may be copied during processing, modifying the data after this
-   * call yields undefined results.
-   *
-   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
-   * been called will return an error.
-   *
-   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
-   *
-   * @param model The model to be modified.
-   * @param index The index of the model operand we're setting.
-   * @param buffer A pointer to the data to use.
-   * @param memory The memory containing the data.
-   * @param offset This specifies the location of the data within the memory.
-   *               The offset is in bytes from the start of memory.
-   * @param length The size in bytes of the data value.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if successful.
-   */
-  int (*ANeuralNetworksModel_setOperandValueFromMemory)(
-      ANeuralNetworksModel* model, int32_t index,
-      const ANeuralNetworksMemory* memory, size_t offset, size_t length);
-
-  /**
-   * Add an operation to a model.
-   *
-   * @param model The model to be modified.
-   * @param type The type of the operation.
-   * @param inputCount The number of entries in the inputs array.
-   * @param inputs An array of indexes identifying each operand.
-   * @param outputCount The number of entries in the outputs array.
-   * @param outputs An array of indexes identifying each operand.
-   *
-   * The operands specified by inputs and outputs must have been
-   * previously added by calls to {@link ANeuralNetworksModel_addOperand}.
-   *
-   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
-   * been called will return an error.
-   *
-   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if successful.
-   */
-  int (*ANeuralNetworksModel_addOperation)(ANeuralNetworksModel* model,
-                                           ANeuralNetworksOperationType type,
-                                           uint32_t inputCount,
-                                           const uint32_t* inputs,
-                                           uint32_t outputCount,
-                                           const uint32_t* outputs);
-
-  /**
-   * Specifies which operands will be the model's inputs and outputs.
-   *
-   * An operand cannot be used for both input and output. Doing so will
-   * return an error.
-   *
-   * @param model The model to be modified.
-   * @param inputCount The number of entries in the inputs array.
-   * @param inputs An array of indexes identifying the input operands.
-   * @param outputCount The number of entries in the outputs array.
-   * @param outputs An array of indexes identifying the output operands.
-   *
-   * The operands specified by inputs and outputs must have been
-   * previously added by calls to {@link ANeuralNetworksModel_addOperand}.
-   *
-   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
-   * been called will return an error.
-   *
-   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
-   *
-   */
-  int (*ANeuralNetworksModel_identifyInputsAndOutputs)(
-      ANeuralNetworksModel* model, uint32_t inputCount, const uint32_t* inputs,
-      uint32_t outputCount, const uint32_t* outputs);
-
-  /**
-   * Specifies whether {@link ANEURALNETWORKS_TENSOR_FLOAT32} is allowed to be
-   * calculated with range and/or precision as low as that of the
-   * IEEE 754 16-bit floating-point format. By default,
-   * {@link ANEURALNETWORKS_TENSOR_FLOAT32} must be calculated using at least
-   * the range and precision of the IEEE 754 32-bit floating-point format.
-   *
-   * @param model The model to be modified.
-   * @param allow 'true' indicates {@link ANEURALNETWORKS_TENSOR_FLOAT32} may be
-   *              calculated with range and/or precision as low as that of the
-   *              IEEE 754 16-bit floating point format. 'false' indicates
-   *              {@link ANEURALNETWORKS_TENSOR_FLOAT32} must be calculated
-   *              using at least the range and precision of the IEEE 754 32-bit
-   *              floating point format.
-   *
-   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
-   * been called will return an error.
-   *
-   * Available since API level 28.
-   *
-   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
-   */
-  int (*ANeuralNetworksModel_relaxComputationFloat32toFloat16)(
-      ANeuralNetworksModel* model, bool allow);
-
-  /**
-   * Create a {@link ANeuralNetworksCompilation} to compile the given model.
-   * This only creates the object. Compilation is only performed once
-   * {@link ANeuralNetworksCompilation_start} is invoked.
-   *
-   * <p>The provided model must outlive the compilation.</p>
-   *
-   * The model must already have been finished by a call to
-   * {@link ANeuralNetworksModel_finish}.
-   *
-   * See {@link ANeuralNetworksCompilation} for information on multithreaded
-   * usage.
-   *
-   * @param model The {@link ANeuralNetworksModel} to be compiled.
-   * @param compilation The newly created object or NULL if unsuccessful.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA
-   *         if the model is invalid.
-   */
-  int (*ANeuralNetworksCompilation_create)(
-      ANeuralNetworksModel* model, ANeuralNetworksCompilation** compilation);
-
-  /**
-   * Destroy a compilation.
-   *
-   * <p>If called on a compilation for which
-   * {@link ANeuralNetworksCompilation_start} has been called, the
-   * function will return immediately but will mark the compilation to be
-   * deleted once the compilation completes. The
-   * {@link ANeuralNetworksCompilation_wait} will return ERROR_DELETED.
-   *
-   * See {@link ANeuralNetworksCompilation} for information on multithreaded
-   * usage.
-   *
-   * @param compilation The compilation to be destroyed. Passing NULL is
-   * acceptable and results in no operation.
-   */
-  void (*ANeuralNetworksCompilation_free)(
-      ANeuralNetworksCompilation* compilation);
-
-  /**
-   * Sets the execution preference.
-   *
-   * <p>Provides guidance to the runtime when trade-offs are possible.</p>
-   *
-   * See {@link ANeuralNetworksCompilation} for information on multithreaded
-   * usage.
-   *
-   * @param compilation The compilation to be modified.
-   * @param preference Either {@link PREFER_LOW_POWER},
-   *                  {@link PREFER_SINGLE_FAST_ANSWER}, or
-   *                  {@link PREFER_SUSTAINED_SPEED}.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if successful.
-   */
-  int (*ANeuralNetworksCompilation_setPreference)(
-      ANeuralNetworksCompilation* compilation, int32_t preference);
-
-  /**
-   * Waits until the compilation completes.
-   *
-   * More than one thread can wait on a compilation. When the compilation
-   * completes, all threads will be released.
-   *
-   * See {@link ANeuralNetworksCompilation} for information on multithreaded
-   * usage.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if the compilation completed normally.
-   */
-  int (*ANeuralNetworksCompilation_finish)(
-      ANeuralNetworksCompilation* compilation);
-
-  /**
-   * Create a {@link ANeuralNetworksExecution} to apply the given compilation.
-   * This only creates the object. Computation is only performed once
-   * {@link ANeuralNetworksExecution_startCompute} is invoked.
-   *
-   * <p>The provided compilation must outlive the execution.</p>
-   *
-   * See {@link ANeuralNetworksExecution} for information on multithreaded
-   * usage.
-   *
-   * @param compilation The {@link ANeuralNetworksCompilation} to be evaluated.
-   * @param execution The newly created object or NULL if unsuccessful.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA
-   *         if the compilation is invalid.
-   */
-  int (*ANeuralNetworksExecution_create)(
-      ANeuralNetworksCompilation* compilation,
-      ANeuralNetworksExecution** execution);
-
-  /**
-   * Destroy an execution.
-   *
-   * <p>If called on an execution for which
-   * {@link ANeuralNetworksExecution_startCompute} has been called, the
-   * function will return immediately but will mark the execution to be deleted
-   * once the computation completes.   The {link ANeuralNetworksExecution_wait}
-   * will return ANEURALNETWORKS_ERROR_DELETED.
-   *
-   * See {@link ANeuralNetworksExecution} for information on multithreaded
-   * usage.
-   *
-   * @param execution The execution to be destroyed. Passing NULL is acceptable
-   * and results in no operation.
-   */
-  void (*ANeuralNetworksExecution_free)(ANeuralNetworksExecution* execution);
-
-  /**
-   * Associate a user buffer with an input of the model of the
-   * {@link ANeuralNetworksExecution}.
-   *
-   * <p>The provided buffer must outlive the execution.</p>
-   *
-   * See {@link ANeuralNetworksExecution} for information on multithreaded
-   * usage.
-   *
-   * @param execution The execution to be modified.
-   * @param index The index of the input argument we are setting. It is
-   *              an index into the lists passed to
-   *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is
-   *              not the index associated with {@link
-   * ANeuralNetworksModel_addOperand}.
-   * @param type The type of the operand. This should be used to specify the
-   *             dimensions that were set to 0 when the operand was added to the
-   *             model. All other properties of the type must be the same as
-   *             specified in the model. If the type is the same as specified
-   *             when the model was built, NULL can be passed.
-   * @param buffer The buffer containing the data.
-   * @param length The length in bytes of the buffer.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
-   * the name is not recognized or the buffer is too small for the input.
-   */
-  int (*ANeuralNetworksExecution_setInput)(
-      ANeuralNetworksExecution* execution, int32_t index,
-      const ANeuralNetworksOperandType* type, const void* buffer,
-      size_t length);
-
-  /**
-   * Associate part of a memory object with an input of the model of the
-   * {@link ANeuralNetworksExecution}.
-   *
-   * <p>The provided memory must outlive the execution.</p>
-   *
-   * See {@link ANeuralNetworksExecution} for information on multithreaded
-   * usage.
-   *
-   * @param execution The execution to be modified.
-   * @param index The index of the input argument we are setting. It is
-   *              an index into the lists passed to
-   *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is
-   *              not the index associated with {@link
-   * ANeuralNetworksModel_addOperand}.
-   * @param type The type of the operand. This can be used to specify the
-   *             dimensions that were set to 0 when the operand was added to the
-   *             model. All other values must be the same as specified in the
-   *             model. If the type is the same as specified when the model
-   *             was built, NULL can be passed.
-   * @param memory The memory containing the data.
-   * @param offset This specifies the location of the data within the memory.
-   *               The offset is in bytes from the start of memory.
-   * @param length The size in bytes of the data value.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
-   * the name is not recognized or the buffer is too small for the input.
-   */
-  int (*ANeuralNetworksExecution_setInputFromMemory)(
-      ANeuralNetworksExecution* execution, int32_t index,
-      const ANeuralNetworksOperandType* type,
-      const ANeuralNetworksMemory* memory, size_t offset, size_t length);
-
-  /**
-   * Associate a user buffer with an output of the model of the
-   * {@link ANeuralNetworksExecution}.
-   *
-   * <p>The provided buffer must outlive the execution.</p>
-   *
-   * See {@link ANeuralNetworksExecution} for information on multithreaded
-   * usage.
-   *
-   * @param execution The execution to be modified.
-   * @param index The index of the output argument we are setting. It is
-   *              an index into the lists passed to
-   *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is
-   *              not the index associated with {@link
-   * ANeuralNetworksModel_addOperand}.
-   * @param type The type of the operand. This can be used to specify the
-   *             dimensions that were set to 0 when the operand was added to the
-   *             model. All other values must be the same as specified in the
-   *             model. If the type is the same as specified when the model
-   *             was built, NULL can be passed.
-   * @param buffer The buffer where the data is to be written.
-   * @param length The length in bytes of the buffer.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
-   * the name is not recognized or the buffer is too small for the output.
-   */
-  int (*ANeuralNetworksExecution_setOutput)(
-      ANeuralNetworksExecution* execution, int32_t index,
-      const ANeuralNetworksOperandType* type, void* buffer, size_t length);
-
-  /**
-   * Associate part of a memory object with an output of the model of the
-   * {@link ANeuralNetworksExecution}.
-   *
-   * <p>The provided memory must outlive the execution.</p>
-   *
-   * See {@link ANeuralNetworksExecution} for information on multithreaded
-   * usage.
-   *
-   * @param execution The execution to be modified.
-   * @param index The index of the output argument we are setting. It is
-   *              an index into the lists passed to
-   *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is
-   *              not the index associated with {@link
-   * ANeuralNetworksModel_addOperand}.
-   * @param type The type of the operand. This can be used to specify the
-   *             dimensions that were set to 0 when the operand was added to the
-   *             model. All other values must be the same as specified in the
-   *             model. If the type is the same as specified when the model
-   *             was built, NULL can be passed.
-   * @param memory The memory where the data is to be stored.
-   * @param offset This specifies the location of the data within the memory.
-   *               The offset is in bytes from the start of memory.
-   * @param length The length in bytes of the data value.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
-   * the name is not recognized or the buffer is too small for the output.
-   */
-  int (*ANeuralNetworksExecution_setOutputFromMemory)(
-      ANeuralNetworksExecution* execution, int32_t index,
-      const ANeuralNetworksOperandType* type,
-      const ANeuralNetworksMemory* memory, size_t offset, size_t length);
-
-  /**
-   * Schedule evaluation of the execution.
-   *
-   * <p>Schedules evaluation of the execution. Once the model has been
-   * applied and the outputs are ready to be consumed, the execution will be
-   * signaled. Use {@link ANeuralNetworksExecution_wait} to wait for that
-   * signal.
-   * </p>
-   *
-   * Multiple executions can be scheduled and evaluated concurrently, and
-   * compilations can be performed concurrently with executions. The runtime
-   * makes no guarantee on the ordering of the completion of compilations and
-   * executions. If it's important to the application, the application should
-   * enforce the ordering by using {@link ANeuralNetworksCompilation_wait} and
-   * {@link ANeuralNetworksExecution_wait}.
-   *
-   * ANeuralNetworksExecution_wait must be called to recuperate the resources
-   * used by the execution.
-   *
-   * See {@link ANeuralNetworksExecution} for information on multithreaded
-   * usage.
-   *
-   * @param execution The execution to be scheduled and executed.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if successful.
-   */
-  int (*ANeuralNetworksExecution_startCompute)(
-      ANeuralNetworksExecution* execution, ANeuralNetworksEvent** event);
-
-  /**
-   * Waits until the execution completes.
-   *
-   * More than one thread can wait on an event. When the execution completes,
-   * all threads will be released.
-   *
-   * See {@link ANeuralNetworksExecution} for information on multithreaded
-   * usage.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if the execution completed normally.
-   */
-  int (*ANeuralNetworksEvent_wait)(ANeuralNetworksEvent* event);
+/**
+ * Delete a memory object.
+ *
+ * Destroys the object used by the run time to keep track of the memory.
+ * This will free the underlying actual memory if no other code has open
+ * handles to this memory.
+ *
+ * @param memory The memory object to be freed.
+ */
+inline void ANeuralNetworksMemory_free(ANeuralNetworksMemory* memory) {
+  LOAD_FUNCTION(ANeuralNetworksMemory_free);
+  EXECUTE_FUNCTION(memory);
+}
 
-  /**
-   * Destroys the event.
-   *
-   * See {@link ANeuralNetworksExecution} for information on multithreaded
-   * usage.
-   */
-  void (*ANeuralNetworksEvent_free)(ANeuralNetworksEvent* event);
+/**
+ * Create an empty {@link ANeuralNetworksModel}.
+ *
+ * <p>This only creates the object. Computation is performed once
+ * {@link ANeuralNetworksExecution_startCompute} is invoked.
+ *
+ * The model should be constructed with calls to
+ * {@link ANeuralNetworksModel_addOperation} and
+ * {@link ANeuralNetworksModel_addOperand}
+ *
+ * <p>{@link ANeuralNetworksModel_finish} should be called once the model
+ * has been fully constructed.</p>
+ *
+ * <p>{@link ANeuralNetworksModel_free} should be called once the model
+ * is no longer needed.</p>
+ *
+ * @param model The {@link ANeuralNetworksModel} to be created.
+ *              Set to NULL if unsuccessful.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful.
+ */
+inline int ANeuralNetworksModel_create(ANeuralNetworksModel** model) {
+  LOAD_FUNCTION(ANeuralNetworksModel_create);
+  EXECUTE_FUNCTION_RETURN(model);
+}
 
-  // ASharedMemory_create was added in Android 8.0, so safe to use with NNAPI
-  // which was added in 8.1.
-  int (*ASharedMemory_create)(const char* name, size_t size);
+/**
+ * Destroy a model.
+ *
+ * The model need not have been finished by a call to
+ * {@link ANeuralNetworksModel_finish}.
+ *
+ * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+ *
+ * @param model The model to be destroyed. Passing NULL is acceptable and
+ *              results in no operation.
+ */
+inline void ANeuralNetworksModel_free(ANeuralNetworksModel* model) {
+  LOAD_FUNCTION(ANeuralNetworksModel_free);
+  EXECUTE_FUNCTION(model);
+}
 
-  /**/
-};
+/**
+ * Indicate that we have finished modifying a model. Required before
+ * calling {@link ANeuralNetworksCompilation_compile}.
+ *
+ * An application is responsible to make sure that no other thread uses
+ * the model at the same time.
+ *
+ * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+ *
+ * @param model The model to be finished.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful.
+ */
+inline int ANeuralNetworksModel_finish(ANeuralNetworksModel* model) {
+  LOAD_FUNCTION(ANeuralNetworksModel_finish);
+  EXECUTE_FUNCTION_RETURN(model);
+}
+
+/**
+ * Add an operand to a model.
+ *
+ * The order in which the operands are added is important. The first one added
+ * to a model will have the index value 0, the second 1, etc. These indexes are
+ * used as operand identifiers in {@link ANeuralNetworksModel_addOperation},
+ * {@link ANeuralNetworksExecution_setInput},
+ * {@link ANeuralNetworksExecution_setInputFromMemory},
+ * {@link ANeuralNetworksExecution_setOutput},
+ * {@link ANeuralNetworksExecution_setOutputFromMemory} and
+ * {@link ANeuralNetworksExecution_setOperandValue}.
+ *
+ * To build a model that can accommodate inputs of various sizes, as you may
+ * want to do for a CNN, set the size of the dimensions that will vary at run
+ * time to 0. If you do so, provide the full dimensions when calling
+ * {@link ANeuralNetworksExecution_setInput} or {@link
+ * ANeuralNetworksExecution_setInputFromMemory}.
+ *
+ * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+ * been called will return an error.
+ *
+ * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+ *
+ * @param model The model to be modified.
+ * @param type The {@link ANeuralNetworksOperandType} that describes the shape
+ * of the operand.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful.
+ */
+inline int ANeuralNetworksModel_addOperand(
+    ANeuralNetworksModel* model, const ANeuralNetworksOperandType* type) {
+  LOAD_FUNCTION(ANeuralNetworksModel_addOperand);
+  EXECUTE_FUNCTION_RETURN(model, type);
+}
+
+/**
+ * Sets an operand to a constant value.
+ *
+ * For scalar values, the content of buffer is copied into the model.
+ *
+ * For tensor values, a pointer to the buffer is stored within the model.
+ * The application is responsible for not changing the content of this region
+ * until all executions using this model have completed. As the data may
+ * be copied during processing, modifying the data after this call yields
+ * undefined results.
+ *
+ * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+ * been called will return an error.
+ *
+ * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+ *
+ * @param model The model to be modified.
+ * @param index The index of the model operand we're setting.
+ * @param buffer A pointer to the data to use.
+ * @param length The size in bytes of the data value.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful.
+ */
+inline int ANeuralNetworksModel_setOperandValue(ANeuralNetworksModel* model,
+                                                int32_t index,
+                                                const void* buffer,
+                                                size_t length) {
+  LOAD_FUNCTION(ANeuralNetworksModel_setOperandValue);
+  EXECUTE_FUNCTION_RETURN(model, index, buffer, length);
+}
 
 /**
- * Load the NNAPI implementation from the shared libraries.
- * The NnApi structure is filled with all the pointers. If one function doesn't
- * exist, a null pointer is stored.
+ * Sets an operand to a value stored in a memory object.
+ *
+ * The content of the memory is not copied. A reference to that memory is stored
+ * inside the model. The application is responsible for not changing the content
+ * of the memory region until all executions using this model have completed.
+ * As the data may be copied during processing, modifying the data after this
+ * call yields undefined results.
+ *
+ * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+ * been called will return an error.
+ *
+ * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+ *
+ * @param model The model to be modified.
+ * @param index The index of the model operand we're setting.
+ * @param buffer A pointer to the data to use.
+ * @param memory The memory containing the data.
+ * @param offset This specifies the location of the data within the memory.
+ *               The offset is in bytes from the start of memory.
+ * @param length The size in bytes of the data value.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful.
  */
-const NnApi* NnApiImplementation();
+inline int ANeuralNetworksModel_setOperandValueFromMemory(
+    ANeuralNetworksModel* model, int32_t index,
+    const ANeuralNetworksMemory* memory, size_t offset, size_t length) {
+  LOAD_FUNCTION(ANeuralNetworksModel_setOperandValueFromMemory);
+  EXECUTE_FUNCTION_RETURN(model, index, memory, offset, length);
+}
+
+/**
+ * Add an operation to a model.
+ *
+ * @param model The model to be modified.
+ * @param type The type of the operation.
+ * @param inputCount The number of entries in the inputs array.
+ * @param inputs An array of indexes identifying each operand.
+ * @param outputCount The number of entries in the outputs array.
+ * @param outputs An array of indexes identifying each operand.
+ *
+ * The operands specified by inputs and outputs must have been
+ * previously added by calls to {@link ANeuralNetworksModel_addOperand}.
+ *
+ * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+ * been called will return an error.
+ *
+ * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful.
+ */
+inline int ANeuralNetworksModel_addOperation(ANeuralNetworksModel* model,
+                                             ANeuralNetworksOperationType type,
+                                             uint32_t inputCount,
+                                             const uint32_t* inputs,
+                                             uint32_t outputCount,
+                                             const uint32_t* outputs) {
+  LOAD_FUNCTION(ANeuralNetworksModel_addOperation);
+  EXECUTE_FUNCTION_RETURN(model, type, inputCount, inputs, outputCount,
+                          outputs);
+}
+
+/**
+ * Specifies which operands will be the model's inputs and outputs.
+ *
+ * An operand cannot be used for both input and output. Doing so will
+ * return an error.
+ *
+ * @param model The model to be modified.
+ * @param inputCount The number of entries in the inputs array.
+ * @param inputs An array of indexes identifying the input operands.
+ * @param outputCount The number of entries in the outputs array.
+ * @param outputs An array of indexes identifying the output operands.
+ *
+ * The operands specified by inputs and outputs must have been
+ * previously added by calls to {@link ANeuralNetworksModel_addOperand}.
+ *
+ * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+ * been called will return an error.
+ *
+ * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+ *
+ */
+inline int ANeuralNetworksModel_identifyInputsAndOutputs(
+    ANeuralNetworksModel* model, uint32_t inputCount, const uint32_t* inputs,
+    uint32_t outputCount, const uint32_t* outputs) {
+  LOAD_FUNCTION(ANeuralNetworksModel_identifyInputsAndOutputs);
+  EXECUTE_FUNCTION_RETURN(model, inputCount, inputs, outputCount, outputs);
+}
+
+/**
+ * Specifies whether {@link ANEURALNETWORKS_TENSOR_FLOAT32} is allowed to be
+ * calculated with range and/or precision as low as that of the IEEE 754 16-bit
+ * floating-point format. By default, {@link ANEURALNETWORKS_TENSOR_FLOAT32}
+ * must be calculated using at least the range and precision of the IEEE 754
+ * 32-bit floating-point format.
+ *
+ * @param model The model to be modified.
+ * @param allow 'true' indicates {@link ANEURALNETWORKS_TENSOR_FLOAT32} may be
+ *              calculated with range and/or precision as low as that of the
+ *              IEEE 754 16-bit floating point format. 'false' indicates
+ *              {@link ANEURALNETWORKS_TENSOR_FLOAT32} must be calculated using
+ *              at least the range and precision of the IEEE 754 32-bit floating
+ *              point format.
+ *
+ * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+ * been called will return an error.
+ *
+ * Available since API level 28.
+ *
+ * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+ */
+inline int ANeuralNetworksModel_relaxComputationFloat32toFloat16(
+    ANeuralNetworksModel* model, bool allow) {
+  LOAD_FUNCTION(ANeuralNetworksModel_relaxComputationFloat32toFloat16);
+  EXECUTE_FUNCTION_RETURN(model, allow);
+}
+
+/**
+ * Create a {@link ANeuralNetworksCompilation} to compile the given model.
+ * This only creates the object. Compilation is only performed once
+ * {@link ANeuralNetworksCompilation_start} is invoked.
+ *
+ * <p>The provided model must outlive the compilation.</p>
+ *
+ * The model must already have been finished by a call to
+ * {@link ANeuralNetworksModel_finish}.
+ *
+ * See {@link ANeuralNetworksCompilation} for information on multithreaded
+ * usage.
+ *
+ * @param model The {@link ANeuralNetworksModel} to be compiled.
+ * @param compilation The newly created object or NULL if unsuccessful.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA
+ *         if the model is invalid.
+ */
+inline int ANeuralNetworksCompilation_create(
+    ANeuralNetworksModel* model, ANeuralNetworksCompilation** compilation) {
+  LOAD_FUNCTION(ANeuralNetworksCompilation_create);
+  EXECUTE_FUNCTION_RETURN(model, compilation);
+}
+
+/**
+ * Destroy a compilation.
+ *
+ * <p>If called on a compilation for which
+ * {@link ANeuralNetworksCompilation_start} has been called, the
+ * function will return immediately but will mark the compilation to be deleted
+ * once the compilation completes. The {@link ANeuralNetworksCompilation_wait}
+ * will return ERROR_DELETED.
+ *
+ * See {@link ANeuralNetworksCompilation} for information on multithreaded
+ * usage.
+ *
+ * @param compilation The compilation to be destroyed. Passing NULL is
+ * acceptable and results in no operation.
+ */
+inline void ANeuralNetworksCompilation_free(
+    ANeuralNetworksCompilation* compilation) {
+  LOAD_FUNCTION(ANeuralNetworksCompilation_free);
+  EXECUTE_FUNCTION(compilation);
+}
+
+/**
+ * Sets the execution preference.
+ *
+ * <p>Provides guidance to the runtime when trade-offs are possible.</p>
+ *
+ * See {@link ANeuralNetworksCompilation} for information on multithreaded
+ * usage.
+ *
+ * @param compilation The compilation to be modified.
+ * @param preference Either {@link PREFER_LOW_POWER},
+ *                  {@link PREFER_SINGLE_FAST_ANSWER}, or
+ *                  {@link PREFER_SUSTAINED_SPEED}.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful.
+ */
+inline int ANeuralNetworksCompilation_setPreference(
+    ANeuralNetworksCompilation* compilation, int32_t preference) {
+  LOAD_FUNCTION(ANeuralNetworksCompilation_setPreference);
+  EXECUTE_FUNCTION_RETURN(compilation, preference);
+}
+
+/**
+ * Waits until the compilation completes.
+ *
+ * More than one thread can wait on a compilation. When the compilation
+ * completes, all threads will be released.
+ *
+ * See {@link ANeuralNetworksCompilation} for information on multithreaded
+ * usage.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if the compilation completed normally.
+ */
+inline int ANeuralNetworksCompilation_finish(
+    ANeuralNetworksCompilation* compilation) {
+  LOAD_FUNCTION(ANeuralNetworksCompilation_finish);
+  EXECUTE_FUNCTION_RETURN(compilation);
+}
+/**
+ * Create a {@link ANeuralNetworksExecution} to apply the given compilation.
+ * This only creates the object. Computation is only performed once
+ * {@link ANeuralNetworksExecution_startCompute} is invoked.
+ *
+ * <p>The provided compilation must outlive the execution.</p>
+ *
+ * See {@link ANeuralNetworksExecution} for information on multithreaded usage.
+ *
+ * @param compilation The {@link ANeuralNetworksCompilation} to be evaluated.
+ * @param execution The newly created object or NULL if unsuccessful.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA
+ *         if the compilation is invalid.
+ */
+inline int ANeuralNetworksExecution_create(
+    ANeuralNetworksCompilation* compilation,
+    ANeuralNetworksExecution** execution) {
+  LOAD_FUNCTION(ANeuralNetworksExecution_create);
+  EXECUTE_FUNCTION_RETURN(compilation, execution);
+}
+
+/**
+ * Destroy an execution.
+ *
+ * <p>If called on an execution for which
+ * {@link ANeuralNetworksExecution_startCompute} has been called, the
+ * function will return immediately but will mark the execution to be deleted
+ * once the computation completes.   The {link ANeuralNetworksExecution_wait}
+ * will return ANEURALNETWORKS_ERROR_DELETED.
+ *
+ * See {@link ANeuralNetworksExecution} for information on multithreaded usage.
+ *
+ * @param execution The execution to be destroyed. Passing NULL is acceptable
+ * and results in no operation.
+ */
+inline void ANeuralNetworksExecution_free(ANeuralNetworksExecution* execution) {
+  LOAD_FUNCTION(ANeuralNetworksExecution_free);
+  EXECUTE_FUNCTION(execution);
+}
+
+/**
+ * Associate a user buffer with an input of the model of the
+ * {@link ANeuralNetworksExecution}.
+ *
+ * <p>The provided buffer must outlive the execution.</p>
+ *
+ * See {@link ANeuralNetworksExecution} for information on multithreaded usage.
+ *
+ * @param execution The execution to be modified.
+ * @param index The index of the input argument we are setting. It is
+ *              an index into the lists passed to
+ *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is not
+ *              the index associated with {@link
+ * ANeuralNetworksModel_addOperand}.
+ * @param type The type of the operand. This should be used to specify the
+ *             dimensions that were set to 0 when the operand was added to the
+ *             model. All other properties of the type must be the same as
+ *             specified in the model. If the type is the same as specified
+ *             when the model was built, NULL can be passed.
+ * @param buffer The buffer containing the data.
+ * @param length The length in bytes of the buffer.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
+ * the name is not recognized or the buffer is too small for the input.
+ */
+inline int ANeuralNetworksExecution_setInput(
+    ANeuralNetworksExecution* execution, int32_t index,
+    const ANeuralNetworksOperandType* type, const void* buffer, size_t length) {
+  LOAD_FUNCTION(ANeuralNetworksExecution_setInput);
+  EXECUTE_FUNCTION_RETURN(execution, index, type, buffer, length);
+}
+
+/**
+ * Associate part of a memory object with an input of the model of the
+ * {@link ANeuralNetworksExecution}.
+ *
+ * <p>The provided memory must outlive the execution.</p>
+ *
+ * See {@link ANeuralNetworksExecution} for information on multithreaded usage.
+ *
+ * @param execution The execution to be modified.
+ * @param index The index of the input argument we are setting. It is
+ *              an index into the lists passed to
+ *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is not
+ *              the index associated with {@link
+ * ANeuralNetworksModel_addOperand}.
+ * @param type The type of the operand. This can be used to specify the
+ *             dimensions that were set to 0 when the operand was added to the
+ *             model. All other values must be the same as specified in the
+ *             model. If the type is the same as specified when the model
+ *             was built, NULL can be passed.
+ * @param memory The memory containing the data.
+ * @param offset This specifies the location of the data within the memory.
+ *               The offset is in bytes from the start of memory.
+ * @param length The size in bytes of the data value.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
+ * the name is not recognized or the buffer is too small for the input.
+ */
+inline int ANeuralNetworksExecution_setInputFromMemory(
+    ANeuralNetworksExecution* execution, int32_t index,
+    const ANeuralNetworksOperandType* type, const ANeuralNetworksMemory* memory,
+    size_t offset, size_t length) {
+  LOAD_FUNCTION(ANeuralNetworksExecution_setInputFromMemory);
+  EXECUTE_FUNCTION_RETURN(execution, index, type, memory, offset, length);
+}
+
+/**
+ * Associate a user buffer with an output of the model of the
+ * {@link ANeuralNetworksExecution}.
+ *
+ * <p>The provided buffer must outlive the execution.</p>
+ *
+ * See {@link ANeuralNetworksExecution} for information on multithreaded usage.
+ *
+ * @param execution The execution to be modified.
+ * @param index The index of the output argument we are setting. It is
+ *              an index into the lists passed to
+ *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is not
+ *              the index associated with {@link
+ * ANeuralNetworksModel_addOperand}.
+ * @param type The type of the operand. This can be used to specify the
+ *             dimensions that were set to 0 when the operand was added to the
+ *             model. All other values must be the same as specified in the
+ *             model. If the type is the same as specified when the model
+ *             was built, NULL can be passed.
+ * @param buffer The buffer where the data is to be written.
+ * @param length The length in bytes of the buffer.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
+ * the name is not recognized or the buffer is too small for the output.
+ */
+inline int ANeuralNetworksExecution_setOutput(
+    ANeuralNetworksExecution* execution, int32_t index,
+    const ANeuralNetworksOperandType* type, void* buffer, size_t length) {
+  LOAD_FUNCTION(ANeuralNetworksExecution_setOutput);
+  EXECUTE_FUNCTION_RETURN(execution, index, type, buffer, length);
+}
+
+/**
+ * Associate part of a memory object with an output of the model of the
+ * {@link ANeuralNetworksExecution}.
+ *
+ * <p>The provided memory must outlive the execution.</p>
+ *
+ * See {@link ANeuralNetworksExecution} for information on multithreaded usage.
+ *
+ * @param execution The execution to be modified.
+ * @param index The index of the output argument we are setting. It is
+ *              an index into the lists passed to
+ *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is not
+ *              the index associated with {@link
+ * ANeuralNetworksModel_addOperand}.
+ * @param type The type of the operand. This can be used to specify the
+ *             dimensions that were set to 0 when the operand was added to the
+ *             model. All other values must be the same as specified in the
+ *             model. If the type is the same as specified when the model
+ *             was built, NULL can be passed.
+ * @param memory The memory where the data is to be stored.
+ * @param offset This specifies the location of the data within the memory.
+ *               The offset is in bytes from the start of memory.
+ * @param length The length in bytes of the data value.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
+ * the name is not recognized or the buffer is too small for the output.
+ */
+inline int ANeuralNetworksExecution_setOutputFromMemory(
+    ANeuralNetworksExecution* execution, int32_t index,
+    const ANeuralNetworksOperandType* type, const ANeuralNetworksMemory* memory,
+    size_t offset, size_t length) {
+  LOAD_FUNCTION(ANeuralNetworksExecution_setOutputFromMemory);
+  EXECUTE_FUNCTION_RETURN(execution, index, type, memory, offset, length);
+}
+
+/**
+ * Schedule evaluation of the execution.
+ *
+ * <p>Schedules evaluation of the execution. Once the model has been
+ * applied and the outputs are ready to be consumed, the execution will be
+ * signaled. Use {@link ANeuralNetworksExecution_wait} to wait for that signal.
+ * </p>
+ *
+ * Multiple executions can be scheduled and evaluated concurrently, and
+ * compilations can be performed concurrently with executions. The runtime makes
+ * no guarantee on the ordering of the completion of compilations and
+ * executions. If it's important to the application, the application should
+ * enforce the ordering by using {@link ANeuralNetworksCompilation_wait} and
+ * {@link ANeuralNetworksExecution_wait}.
+ *
+ * ANeuralNetworksExecution_wait must be called to recuperate the resources used
+ * by the execution.
+ *
+ * See {@link ANeuralNetworksExecution} for information on multithreaded usage.
+ *
+ * @param execution The execution to be scheduled and executed.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful.
+ */
+inline int ANeuralNetworksExecution_startCompute(
+    ANeuralNetworksExecution* execution, ANeuralNetworksEvent** event) {
+  LOAD_FUNCTION(ANeuralNetworksExecution_startCompute);
+  EXECUTE_FUNCTION_RETURN(execution, event);
+}
+
+/**
+ * Waits until the execution completes.
+ *
+ * More than one thread can wait on an event. When the execution completes,
+ * all threads will be released.
+ *
+ * See {@link ANeuralNetworksExecution} for information on multithreaded usage.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if the execution completed normally.
+ */
+inline int ANeuralNetworksEvent_wait(ANeuralNetworksEvent* event) {
+  LOAD_FUNCTION(ANeuralNetworksEvent_wait);
+  EXECUTE_FUNCTION_RETURN(event);
+}
+
+/**
+ * Destroys the event.
+ *
+ * See {@link ANeuralNetworksExecution} for information on multithreaded usage.
+ */
+inline void ANeuralNetworksEvent_free(ANeuralNetworksEvent* event) {
+  LOAD_FUNCTION(ANeuralNetworksEvent_free);
+  EXECUTE_FUNCTION(event);
+}
+
+/**/
 
 #endif  // TENSORFLOW_LITE_NNAPI_NEURALNETWORKSSHIM_H_
diff --git a/tensorflow/lite/nnapi/nnapi_lib_test.cc b/tensorflow/lite/nnapi/nnapi_lib_test.cc
deleted file mode 100644
index aaf0ffaa8e..0000000000
--- a/tensorflow/lite/nnapi/nnapi_lib_test.cc
+++ /dev/null
@@ -1,90 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include <gtest/gtest.h>
-#include "tensorflow/lite/nnapi/NeuralNetworksShim.h"
-
-namespace {
-
-TEST(NnapiLibTest, NnApiImplementation) {
-  const NnApi* nnapi = NnApiImplementation();
-  EXPECT_NE(nnapi, nullptr);
-#ifdef __ANDROID__
-  EXPECT_TRUE(nnapi->nnapi_exists);
-  EXPECT_GT(nnapi->android_sdk_version, 0);
-  EXPECT_NE(nnapi->ANeuralNetworksMemory_createFromFd, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksMemory_free, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksModel_create, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksModel_free, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksModel_finish, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksModel_addOperand, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksModel_setOperandValue, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksModel_setOperandValueFromMemory, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksModel_addOperation, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksModel_identifyInputsAndOutputs, nullptr);
-  if (nnapi->android_sdk_version >= 28) {
-    // relaxComputationFloat32toFloat16 only available with Android 9.0 (P).
-    EXPECT_NE(nnapi->ANeuralNetworksModel_relaxComputationFloat32toFloat16,
-              nullptr);
-  } else {
-    EXPECT_EQ(nnapi->ANeuralNetworksModel_relaxComputationFloat32toFloat16,
-              nullptr);
-  }
-  EXPECT_NE(nnapi->ANeuralNetworksCompilation_create, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksCompilation_free, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksCompilation_setPreference, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksCompilation_finish, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksExecution_create, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksExecution_free, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksExecution_setInput, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksExecution_setInputFromMemory, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksExecution_setOutput, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksExecution_setOutputFromMemory, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksExecution_startCompute, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksEvent_wait, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksEvent_free, nullptr);
-  EXPECT_NE(nnapi->ASharedMemory_create, nullptr);
-#else
-  EXPECT_FALSE(nnapi->nnapi_exists);
-  EXPECT_EQ(nnapi->android_sdk_version, 0);
-  EXPECT_EQ(nnapi->ANeuralNetworksMemory_createFromFd, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksMemory_free, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksModel_create, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksModel_free, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksModel_finish, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksModel_addOperand, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksModel_setOperandValue, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksModel_setOperandValueFromMemory, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksModel_addOperation, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksModel_identifyInputsAndOutputs, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksModel_relaxComputationFloat32toFloat16,
-            nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksCompilation_create, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksCompilation_free, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksCompilation_setPreference, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksCompilation_finish, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksExecution_create, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksExecution_free, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksExecution_setInput, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksExecution_setInputFromMemory, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksExecution_setOutput, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksExecution_setOutputFromMemory, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksExecution_startCompute, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksEvent_wait, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksEvent_free, nullptr);
-  EXPECT_EQ(nnapi->ASharedMemory_create, nullptr);
-#endif
-}
-
-}  // namespace
diff --git a/tensorflow/lite/nnapi_delegate.cc b/tensorflow/lite/nnapi_delegate.cc
index 44b9930acc..dc8e81cde7 100644
--- a/tensorflow/lite/nnapi_delegate.cc
+++ b/tensorflow/lite/nnapi_delegate.cc
@@ -84,27 +84,56 @@ void logError(const char* format, ...) {
 static const int64_t kOperandIdNotSet = -1;
 static const int64_t kOperandNotNeeded = -2;
 
+namespace {
+
+int32_t GetAndroidSdkVersion() {
+#ifdef __ANDROID__
+  const char* sdkProp = "ro.build.version.sdk";
+  char sdkVersion[PROP_VALUE_MAX];
+  int length = __system_property_get(sdkProp, sdkVersion);
+  if (length != 0) {
+    for (int i = 0; i < length; ++i) {
+      int digit = sdkVersion[i] - '0';
+      if (digit < 0 || digit > 9) {
+        // Non-numeric SDK version, assume it's higher then expected;
+        return 0xFFFF;
+      }
+    }
+    return atoi(sdkVersion);
+  }
+  FATAL("No %s prop", sdkProp);
+#endif  // __ANDROID__
+  return 0;
+}
+
+int32_t GetAndroidSdkVersionCached() {
+  static int32_t androidSdkVersion = GetAndroidSdkVersion();
+  return androidSdkVersion;
+}
+
+}  // namespace
+
 NNAPIAllocation::NNAPIAllocation(const char* filename,
                                  ErrorReporter* error_reporter)
     : MMAPAllocation(filename, error_reporter) {
   if (mmapped_buffer_ != MAP_FAILED)
-    CHECK_NN(NnApiImplementation()->ANeuralNetworksMemory_createFromFd(
-        buffer_size_bytes_, PROT_READ, mmap_fd_, 0, &handle_));
+    CHECK_NN(ANeuralNetworksMemory_createFromFd(buffer_size_bytes_, PROT_READ,
+                                                mmap_fd_, 0, &handle_));
 }
 
 NNAPIAllocation::~NNAPIAllocation() {
   if (handle_) {
-    NnApiImplementation()->ANeuralNetworksMemory_free(handle_);
+    ANeuralNetworksMemory_free(handle_);
   }
 }
 
 NNAPIDelegate::~NNAPIDelegate() {
   if (nn_compiled_model_) {
-    NnApiImplementation()->ANeuralNetworksCompilation_free(nn_compiled_model_);
+    ANeuralNetworksCompilation_free(nn_compiled_model_);
     nn_compiled_model_ = nullptr;
   }
   if (nn_model_) {
-    NnApiImplementation()->ANeuralNetworksModel_free(nn_model_);
+    ANeuralNetworksModel_free(nn_model_);
     nn_model_ = nullptr;
     // TODO(aselle): Is this thread-safe and callable multiple times?
   }
@@ -116,7 +145,6 @@ TfLiteStatus addTensorOperands(tflite::Subgraph* subgraph,
                                ANeuralNetworksModel* nn_model,
                                uint32_t* no_of_operands_added,
                                std::vector<int64_t>* nnapi_ids) {
-  const NnApi* nnapi = NnApiImplementation();
   uint32_t next_id = 0;
   for (size_t i = 0; i < subgraph->tensors_size(); i++) {
     // Skip temporaries and RNN back-edges.
@@ -170,24 +198,24 @@ TfLiteStatus addTensorOperands(tflite::Subgraph* subgraph,
         nn_type, static_cast<uint32_t>(tensor->dims->size),
         reinterpret_cast<uint32_t*>(tensor->dims->data), scale, zeroPoint};
     RETURN_ERROR_IF_NN_FAILED(
-        nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type));
+        ANeuralNetworksModel_addOperand(nn_model, &operand_type));
     // TODO(aselle): Based on Michael's suggestion, limiting this to read
     // only memory
     if (tensor->allocation_type == kTfLiteMmapRo) {
       if (const NNAPIAllocation* alloc = dynamic_cast<const NNAPIAllocation*>(
               static_cast<const Allocation*>(tensor->allocation))) {
         RETURN_ERROR_IF_NN_FAILED(
-            nnapi->ANeuralNetworksModel_setOperandValueFromMemory(
+            ANeuralNetworksModel_setOperandValueFromMemory(
                 nn_model, next_id, alloc->memory(),
                 alloc->offset(tensor->data.raw), tensor->bytes));
       } else {
-        RETURN_ERROR_IF_NN_FAILED(nnapi->ANeuralNetworksModel_setOperandValue(
+        RETURN_ERROR_IF_NN_FAILED(ANeuralNetworksModel_setOperandValue(
             nn_model, next_id, tensor->data.raw, tensor->bytes));
       }
     } else if (tensor->bytes == 0) {
       // These size 0 tensors are optional tensors reserved.
-      RETURN_ERROR_IF_NN_FAILED(nnapi->ANeuralNetworksModel_setOperandValue(
-          nn_model, next_id, nullptr, 0));
+      RETURN_ERROR_IF_NN_FAILED(
+          ANeuralNetworksModel_setOperandValue(nn_model, next_id, nullptr, 0));
     }
 
     ++next_id;
@@ -216,7 +244,6 @@ TfLiteStatus AddOpsAndParams(
     uint32_t next_id, std::vector<int>* model_state_inputs,
     std::vector<int>* model_state_outputs,
     const std::vector<int64_t>& tensor_id_to_nnapi_id) {
-  const NnApi* nnapi = NnApiImplementation();
   for (size_t i = 0; i < subgraph->nodes_size(); i++) {
     const auto* node_and_registration = subgraph->node_and_registration(i);
     const TfLiteNode& node = node_and_registration->first;
@@ -231,21 +258,21 @@ TfLiteStatus AddOpsAndParams(
     MapAndAddTensorIds(node.outputs->data, node.outputs->size,
                        &augmented_outputs, tensor_id_to_nnapi_id);
 
-    auto add_scalar_int32 = [nnapi, &nn_model, &augmented_inputs,
+    auto add_scalar_int32 = [&nn_model, &augmented_inputs,
                              &next_id](int value) {
       ANeuralNetworksOperandType operand_type{.type = ANEURALNETWORKS_INT32};
-      CHECK_NN(nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type))
-      CHECK_NN(nnapi->ANeuralNetworksModel_setOperandValue(
-          nn_model, next_id, &value, sizeof(int32_t)))
+      CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type))
+      CHECK_NN(ANeuralNetworksModel_setOperandValue(nn_model, next_id, &value,
+                                                    sizeof(int32_t)))
       augmented_inputs.push_back(next_id++);
     };
 
-    auto add_scalar_float32 = [nnapi, &nn_model, &augmented_inputs,
+    auto add_scalar_float32 = [&nn_model, &augmented_inputs,
                                &next_id](float value) {
       ANeuralNetworksOperandType operand_type{.type = ANEURALNETWORKS_FLOAT32};
-      CHECK_NN(nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type))
-      CHECK_NN(nnapi->ANeuralNetworksModel_setOperandValue(
-          nn_model, next_id, &value, sizeof(float)))
+      CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type))
+      CHECK_NN(ANeuralNetworksModel_setOperandValue(nn_model, next_id, &value,
+                                                    sizeof(float)))
       augmented_inputs.push_back(next_id++);
     };
 
@@ -254,8 +281,8 @@ TfLiteStatus AddOpsAndParams(
           .type = ANEURALNETWORKS_TENSOR_INT32,
           .dimensionCount = 1,
           .dimensions = &num_values};
-      CHECK_NN(nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type))
-      CHECK_NN(nnapi->ANeuralNetworksModel_setOperandValue(
+      CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type))
+      CHECK_NN(ANeuralNetworksModel_setOperandValue(
           nn_model, next_id, values, sizeof(int32_t) * num_values));
       augmented_inputs.push_back(next_id++);
     };
@@ -264,16 +291,15 @@ TfLiteStatus AddOpsAndParams(
     // For each state_out tensor, a corresponding state_in operand needs to be
     // created for NNAPI.
     auto duplicate_state_tensor_float32 =
-        [nnapi, subgraph, &nn_model, &next_id, &augmented_inputs,
-         &model_state_inputs, &model_state_outputs](int tensor_id) {
+        [subgraph, &nn_model, &next_id, &augmented_inputs, &model_state_inputs,
+         &model_state_outputs](int tensor_id) {
           const TfLiteTensor* tensor = subgraph->tensor(tensor_id);
           ANeuralNetworksOperandType operand_type{
               ANEURALNETWORKS_TENSOR_FLOAT32,
               static_cast<uint32_t>(tensor->dims->size),
               reinterpret_cast<uint32_t*>(tensor->dims->data),
               tensor->params.scale, tensor->params.zero_point};
-          CHECK_NN(
-              nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type));
+          CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type));
           augmented_inputs.push_back(next_id);
           model_state_inputs->push_back(next_id);
           model_state_outputs->push_back(tensor_id);
@@ -362,7 +388,7 @@ TfLiteStatus AddOpsAndParams(
     };
 
     // LSTM in NNAPI requires scratch tensor as an output operand.
-    auto add_lstm_scratch_tensor_float32 = [nnapi, subgraph, &node, &nn_model,
+    auto add_lstm_scratch_tensor_float32 = [subgraph, &node, &nn_model,
                                             &next_id, &augmented_outputs]() {
       if (node.temporaries->size == 0) return;
       int scratch_buffer_index = node.temporaries->data[0];
@@ -372,7 +398,7 @@ TfLiteStatus AddOpsAndParams(
           static_cast<uint32_t>(tensor->dims->size),
           reinterpret_cast<uint32_t*>(tensor->dims->data), tensor->params.scale,
           tensor->params.zero_point};
-      CHECK_NN(nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type));
+      CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type));
       augmented_outputs.insert(augmented_outputs.begin(), next_id++);
     };
 
@@ -401,16 +427,15 @@ TfLiteStatus AddOpsAndParams(
     };
 
     // Handle optional input tensors.
-    auto add_optional_tensors = [nnapi, &nn_model, &augmented_inputs,
+    auto add_optional_tensors = [&nn_model, &augmented_inputs,
                                  &next_id](int nn_type) {
       for (size_t idx = 0; idx < augmented_inputs.size(); idx++) {
         if (augmented_inputs[idx] == kOptionalTensor) {
           const std::vector<uint32_t> dim = {0, 0};
           ANeuralNetworksOperandType operand_type{nn_type, 2, dim.data(), 0, 0};
-          CHECK_NN(
-              nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type))
-          CHECK_NN(nnapi->ANeuralNetworksModel_setOperandValue(
-              nn_model, next_id, nullptr, 0))
+          CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type))
+          CHECK_NN(ANeuralNetworksModel_setOperandValue(nn_model, next_id,
+                                                        nullptr, 0))
           augmented_inputs[idx] = next_id++;
         }
       }
@@ -671,13 +696,13 @@ TfLiteStatus AddOpsAndParams(
         break;
     }
 
-    if (nnapi_version == 11 && nnapi->android_sdk_version < 28) {
+    if (nnapi_version == 11 && GetAndroidSdkVersionCached() < 28) {
       logError("Op %d needs NNAPI1.1", builtin);
       return kTfLiteError;
     }
 
     // Add the operation.
-    RETURN_ERROR_IF_NN_FAILED(nnapi->ANeuralNetworksModel_addOperation(
+    RETURN_ERROR_IF_NN_FAILED(ANeuralNetworksModel_addOperation(
         nn_model, nn_op_type, static_cast<uint32_t>(augmented_inputs.size()),
         augmented_inputs.data(),
         static_cast<uint32_t>(augmented_outputs.size()),
@@ -689,10 +714,9 @@ TfLiteStatus AddOpsAndParams(
 TfLiteStatus NNAPIDelegate::BuildGraph(Subgraph* subgraph) {
   if (nn_model_ && nn_compiled_model_) return model_status_;
 
-  const NnApi* nnapi = NnApiImplementation();
   // TODO(aselle): This is not correct. need to handle resize invalidation.
   if (!nn_model_) {
-    CHECK_NN(nnapi->ANeuralNetworksModel_create(&nn_model_));
+    CHECK_NN(ANeuralNetworksModel_create(&nn_model_));
 
     // Find which tensors should be added to NNAPI. TFLite has temporaries
     // and RNN back-edges which are are not valid for NNAPI. We look through all
@@ -739,22 +763,21 @@ TfLiteStatus NNAPIDelegate::BuildGraph(Subgraph* subgraph) {
                        model_states_outputs_.size(), &augmented_outputs,
                        tensor_id_to_nnapi_id);
 
-    CHECK_NN(nnapi->ANeuralNetworksModel_identifyInputsAndOutputs(
+    CHECK_NN(ANeuralNetworksModel_identifyInputsAndOutputs(
         nn_model_, static_cast<uint32_t>(augmented_inputs.size()),
         reinterpret_cast<const uint32_t*>(augmented_inputs.data()),
         static_cast<uint32_t>(augmented_outputs.size()),
         reinterpret_cast<const uint32_t*>(augmented_outputs.data())));
 
-    if (nnapi->android_sdk_version >= 28) {
-      CHECK_NN(nnapi->ANeuralNetworksModel_relaxComputationFloat32toFloat16(
+    if (GetAndroidSdkVersionCached() >= 28) {
+      CHECK_NN(ANeuralNetworksModel_relaxComputationFloat32toFloat16(
           nn_model_, subgraph->GetAllowFp16PrecisionForFp32()));
     }
-    CHECK_NN(nnapi->ANeuralNetworksModel_finish(nn_model_));
+    CHECK_NN(ANeuralNetworksModel_finish(nn_model_));
   }
   if (!nn_compiled_model_) {
-    CHECK_NN(nnapi->ANeuralNetworksCompilation_create(nn_model_,
-                                                      &nn_compiled_model_));
-    CHECK_NN(nnapi->ANeuralNetworksCompilation_finish(nn_compiled_model_));
+    CHECK_NN(ANeuralNetworksCompilation_create(nn_model_, &nn_compiled_model_));
+    CHECK_NN(ANeuralNetworksCompilation_finish(nn_compiled_model_));
   }
   return kTfLiteOk;
 }
@@ -770,10 +793,8 @@ TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
     return model_status_;
   }
 
-  const NnApi* nnapi = NnApiImplementation();
   ANeuralNetworksExecution* execution = nullptr;
-  CHECK_NN(
-      nnapi->ANeuralNetworksExecution_create(nn_compiled_model_, &execution));
+  CHECK_NN(ANeuralNetworksExecution_create(nn_compiled_model_, &execution));
 
   // Currently perform deep copy of input buffer
   for (size_t i = 0; i < subgraph->inputs().size(); i++) {
@@ -781,7 +802,7 @@ TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
     // TODO(aselle): Is this what we want or do we want input instead?
     // TODO(aselle): This should be called setInputValue maybe to be cons.
     TfLiteTensor* tensor = subgraph->tensor(input);
-    CHECK_NN(nnapi->ANeuralNetworksExecution_setInput(
+    CHECK_NN(ANeuralNetworksExecution_setInput(
         execution, i, nullptr, tensor->data.raw, tensor->bytes));
   }
 
@@ -789,7 +810,7 @@ TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
   for (size_t i = 0; i < subgraph->outputs().size(); i++) {
     int output = subgraph->outputs()[i];
     TfLiteTensor* tensor = subgraph->tensor(output);
-    CHECK_NN(nnapi->ANeuralNetworksExecution_setOutput(
+    CHECK_NN(ANeuralNetworksExecution_setOutput(
         execution, i, nullptr, tensor->data.raw, tensor->bytes));
   }
 
@@ -801,21 +822,21 @@ TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
     // Here we are using a deep copy for state_in tensors so that we are not
     // reading and writing into the same buffer during a invocation.
     // TODO(miaowang): using double shared buffer to minimize the copies.
-    CHECK_NN(nnapi->ANeuralNetworksExecution_setInput(
+    CHECK_NN(ANeuralNetworksExecution_setInput(
         execution, i + subgraph->inputs().size(), nullptr, tensor->data.raw,
         tensor->bytes));
     // Tell NNAPI where to output the state_out.
-    CHECK_NN(nnapi->ANeuralNetworksExecution_setOutput(
+    CHECK_NN(ANeuralNetworksExecution_setOutput(
         execution, i + subgraph->outputs().size(), nullptr, tensor->data.raw,
         tensor->bytes));
   }
 
   // Currently use blocking compute.
   ANeuralNetworksEvent* event = nullptr;
-  CHECK_NN(nnapi->ANeuralNetworksExecution_startCompute(execution, &event));
-  CHECK_NN(nnapi->ANeuralNetworksEvent_wait(event));
-  nnapi->ANeuralNetworksEvent_free(event);
-  nnapi->ANeuralNetworksExecution_free(execution);
+  CHECK_NN(ANeuralNetworksExecution_startCompute(execution, &event));
+  CHECK_NN(ANeuralNetworksEvent_wait(event));
+  ANeuralNetworksEvent_free(event);
+  ANeuralNetworksExecution_free(execution);
 
 #if 0
   printf("From the NN API:\n");
@@ -833,8 +854,6 @@ TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
   return kTfLiteOk;
 }
 
-bool NNAPIDelegate::IsSupported() {
-  return NnApiImplementation()->nnapi_exists;
-}
+bool NNAPIDelegate::IsSupported() { return NNAPIExists(); }
 
 }  // namespace tflite
-- 
GitLab


From a460c9525d416e442c245c87ccb6da9032cc5fd6 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Wed, 2 Jan 2019 16:58:33 -0800
Subject: [PATCH 0113/2345] Fix masking generate logic in sequence model.

sequence model was incorrectly using the layer output instead of input to generate mask, and compute_mask of the layer usually does not have shape check.

PiperOrigin-RevId: 227605298
---
 tensorflow/python/keras/engine/sequential.py  |   9 +-
 tensorflow/python/keras/layers/gru_test.py    | 112 ++++++++--------
 tensorflow/python/keras/layers/lstm_test.py   |   8 +-
 .../python/keras/layers/simplernn_test.py     |   8 +-
 .../python/keras/layers/unified_gru_test.py   |  64 +++++----
 .../python/keras/layers/unified_lstm_test.py  | 125 +++++++++---------
 6 files changed, 153 insertions(+), 173 deletions(-)

diff --git a/tensorflow/python/keras/engine/sequential.py b/tensorflow/python/keras/engine/sequential.py
index dffd3c8a69..3627091198 100644
--- a/tensorflow/python/keras/engine/sequential.py
+++ b/tensorflow/python/keras/engine/sequential.py
@@ -262,16 +262,17 @@ class Sequential(Model):
           with ops.name_scope(layer._name_scope()):
             layer._maybe_build(x)
           layer.built = True
+        if layer.supports_masking:
+          mask = layer.compute_mask(x, mask)
+        else:
+          mask = None
+
         if context.executing_eagerly():
           x = layer(x, **kwargs)
         elif layer.dynamic:
           x = layer._symbolic_call(x)
         else:
           x = layer.call(x, **kwargs)
-        if layer.supports_masking:
-          mask = layer.compute_mask(x, mask)
-        else:
-          mask = None
       if not context.executing_eagerly():
         x._keras_mask = mask
     return x, mask
diff --git a/tensorflow/python/keras/layers/gru_test.py b/tensorflow/python/keras/layers/gru_test.py
index 52a7944d60..e2b65de661 100644
--- a/tensorflow/python/keras/layers/gru_test.py
+++ b/tensorflow/python/keras/layers/gru_test.py
@@ -117,63 +117,6 @@ class GRULayerTest(keras_parameterized.TestCase):
                   run_eagerly=testing_utils.should_run_eagerly())
     model.fit(inputs, targets, epochs=1, batch_size=2, verbose=1)
 
-
-@tf_test_util.run_all_in_graph_and_eager_modes
-class GRULayerGenericTest(test.TestCase):
-
-  def test_constraints_GRU(self):
-    embedding_dim = 4
-    layer_class = keras.layers.GRU
-    k_constraint = keras.constraints.max_norm(0.01)
-    r_constraint = keras.constraints.max_norm(0.01)
-    b_constraint = keras.constraints.max_norm(0.01)
-    layer = layer_class(
-        5,
-        return_sequences=False,
-        weights=None,
-        input_shape=(None, embedding_dim),
-        kernel_constraint=k_constraint,
-        recurrent_constraint=r_constraint,
-        bias_constraint=b_constraint)
-    layer.build((None, None, embedding_dim))
-    self.assertEqual(layer.cell.kernel.constraint, k_constraint)
-    self.assertEqual(layer.cell.recurrent_kernel.constraint, r_constraint)
-    self.assertEqual(layer.cell.bias.constraint, b_constraint)
-
-  def test_from_config_GRU(self):
-    layer_class = keras.layers.GRU
-    for stateful in (False, True):
-      l1 = layer_class(units=1, stateful=stateful)
-      l2 = layer_class.from_config(l1.get_config())
-      assert l1.get_config() == l2.get_config()
-
-  def test_regularizers_GRU(self):
-    embedding_dim = 4
-    layer_class = keras.layers.GRU
-    layer = layer_class(
-        5,
-        return_sequences=False,
-        weights=None,
-        input_shape=(None, embedding_dim),
-        kernel_regularizer=keras.regularizers.l1(0.01),
-        recurrent_regularizer=keras.regularizers.l1(0.01),
-        bias_regularizer='l2',
-        activity_regularizer='l1')
-    layer.build((None, None, 2))
-    self.assertEqual(len(layer.losses), 3)
-
-    x = keras.backend.variable(np.ones((2, 3, 2)))
-    layer(x)
-    if context.executing_eagerly():
-      self.assertEqual(len(layer.losses), 4)
-    else:
-      self.assertEqual(len(layer.get_losses_for(x)), 1)
-
-
-class GRULayerV1OnlyTest(test.TestCase):
-
-  @tf_test_util.run_v1_only('b/120941292')
-  @tf_test_util.run_in_graph_and_eager_modes
   def test_statefulness_GRU(self):
     num_samples = 2
     timesteps = 3
@@ -192,7 +135,8 @@ class GRULayerV1OnlyTest(test.TestCase):
     layer = layer_class(
         units, return_sequences=False, stateful=True, weights=None)
     model.add(layer)
-    model.compile(optimizer='sgd', loss='mse')
+    model.compile(optimizer='sgd', loss='mse',
+                  run_eagerly=testing_utils.should_run_eagerly())
     out1 = model.predict(np.ones((num_samples, timesteps)))
     self.assertEqual(out1.shape, (num_samples, units))
 
@@ -237,5 +181,57 @@ class GRULayerV1OnlyTest(test.TestCase):
     np.testing.assert_allclose(out7, out6, atol=1e-5)
 
 
+@tf_test_util.run_all_in_graph_and_eager_modes
+class GRULayerGenericTest(test.TestCase):
+
+  def test_constraints_GRU(self):
+    embedding_dim = 4
+    layer_class = keras.layers.GRU
+    k_constraint = keras.constraints.max_norm(0.01)
+    r_constraint = keras.constraints.max_norm(0.01)
+    b_constraint = keras.constraints.max_norm(0.01)
+    layer = layer_class(
+        5,
+        return_sequences=False,
+        weights=None,
+        input_shape=(None, embedding_dim),
+        kernel_constraint=k_constraint,
+        recurrent_constraint=r_constraint,
+        bias_constraint=b_constraint)
+    layer.build((None, None, embedding_dim))
+    self.assertEqual(layer.cell.kernel.constraint, k_constraint)
+    self.assertEqual(layer.cell.recurrent_kernel.constraint, r_constraint)
+    self.assertEqual(layer.cell.bias.constraint, b_constraint)
+
+  def test_from_config_GRU(self):
+    layer_class = keras.layers.GRU
+    for stateful in (False, True):
+      l1 = layer_class(units=1, stateful=stateful)
+      l2 = layer_class.from_config(l1.get_config())
+      assert l1.get_config() == l2.get_config()
+
+  def test_regularizers_GRU(self):
+    embedding_dim = 4
+    layer_class = keras.layers.GRU
+    layer = layer_class(
+        5,
+        return_sequences=False,
+        weights=None,
+        input_shape=(None, embedding_dim),
+        kernel_regularizer=keras.regularizers.l1(0.01),
+        recurrent_regularizer=keras.regularizers.l1(0.01),
+        bias_regularizer='l2',
+        activity_regularizer='l1')
+    layer.build((None, None, 2))
+    self.assertEqual(len(layer.losses), 3)
+
+    x = keras.backend.variable(np.ones((2, 3, 2)))
+    layer(x)
+    if context.executing_eagerly():
+      self.assertEqual(len(layer.losses), 4)
+    else:
+      self.assertEqual(len(layer.get_losses_for(x)), 1)
+
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/layers/lstm_test.py b/tensorflow/python/keras/layers/lstm_test.py
index e8c55438c9..38c0177e39 100644
--- a/tensorflow/python/keras/layers/lstm_test.py
+++ b/tensorflow/python/keras/layers/lstm_test.py
@@ -23,7 +23,6 @@ import numpy as np
 
 from tensorflow.python import keras
 from tensorflow.python.eager import context
-from tensorflow.python.framework import test_util
 from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.platform import test
@@ -340,11 +339,6 @@ class LSTMLayerTest(keras_parameterized.TestCase):
     else:
       self.assertEqual(len(layer.get_losses_for(x)), 1)
 
-
-class LSTMLayerV1OnlyTest(test.TestCase):
-
-  @test_util.run_v1_only('b/120941292')
-  @test_util.run_in_graph_and_eager_modes
   def test_statefulness_LSTM(self):
     num_samples = 2
     timesteps = 3
@@ -363,7 +357,7 @@ class LSTMLayerV1OnlyTest(test.TestCase):
         units, return_sequences=False, stateful=True, weights=None)
     model.add(layer)
     model.compile(optimizer=gradient_descent.GradientDescentOptimizer(0.01),
-                  loss='mse')
+                  loss='mse', run_eagerly=testing_utils.should_run_eagerly())
     out1 = model.predict(np.ones((num_samples, timesteps)))
     self.assertEqual(out1.shape, (num_samples, units))
 
diff --git a/tensorflow/python/keras/layers/simplernn_test.py b/tensorflow/python/keras/layers/simplernn_test.py
index 390ae789e1..0e599dade9 100644
--- a/tensorflow/python/keras/layers/simplernn_test.py
+++ b/tensorflow/python/keras/layers/simplernn_test.py
@@ -22,7 +22,6 @@ import numpy as np
 
 from tensorflow.python import keras
 from tensorflow.python.eager import context
-from tensorflow.python.framework import test_util
 from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.platform import test
@@ -141,11 +140,6 @@ class SimpleRNNLayerTest(keras_parameterized.TestCase):
     else:
       self.assertEqual(len(layer.get_losses_for(x)), 1)
 
-
-class SimpleRNNLayerV1OnlyTest(test.TestCase):
-
-  @test_util.run_v1_only('b/120941292')
-  @test_util.run_in_graph_and_eager_modes
   def test_statefulness_SimpleRNN(self):
     num_samples = 2
     timesteps = 3
@@ -164,7 +158,7 @@ class SimpleRNNLayerV1OnlyTest(test.TestCase):
         units, return_sequences=False, stateful=True, weights=None)
     model.add(layer)
     model.compile(optimizer=gradient_descent.GradientDescentOptimizer(0.01),
-                  loss='mse')
+                  loss='mse', run_eagerly=testing_utils.should_run_eagerly())
     out1 = model.predict(np.ones((num_samples, timesteps)))
     self.assertEqual(out1.shape, (num_samples, units))
 
diff --git a/tensorflow/python/keras/layers/unified_gru_test.py b/tensorflow/python/keras/layers/unified_gru_test.py
index 244ffdb8b6..11322764ac 100644
--- a/tensorflow/python/keras/layers/unified_gru_test.py
+++ b/tensorflow/python/keras/layers/unified_gru_test.py
@@ -402,39 +402,6 @@ class UnifiedGRUTest(keras_parameterized.TestCase):
     else:
       self.assertEqual(len(layer.get_losses_for(x)), 1)
 
-
-class GRULayerGradientTapeTest(test.TestCase):
-
-  @test_util.run_in_graph_and_eager_modes(config=_config)
-  def test_in_tape(self):
-    if not context.executing_eagerly():
-      self.skipTest('bloo')
-    time_steps = 10
-    embedding_size = 11
-    gru_unit_size = 12
-
-    gru = keras.layers.UnifiedGRU(gru_unit_size,
-                                  return_sequences=True,
-                                  return_state=True,
-                                  recurrent_activation='sigmoid',
-                                  recurrent_initializer='glorot_uniform')
-
-    x = random_ops.random_uniform([1, time_steps, embedding_size])
-    y = random_ops.random_uniform([1, gru_unit_size])
-
-    with backprop.GradientTape() as tape:
-      hidden_state = array_ops.zeros([1, gru_unit_size], dtype=dtypes.float32)
-      _, state = gru(x, initial_state=hidden_state)
-
-      loss = math_ops.reduce_mean(math_ops.square(state - y))
-
-    tape.gradient(loss, gru.variables)
-
-
-class GRULayerV1OnlyTest(test.TestCase, parameterized.TestCase):
-
-  @test_util.run_v1_only('b/120941292')
-  @test_util.run_in_graph_and_eager_modes(config=_config)
   def test_statefulness_GRU(self):
     num_samples = 2
     timesteps = 3
@@ -452,7 +419,8 @@ class GRULayerV1OnlyTest(test.TestCase, parameterized.TestCase):
     layer = layer_class(
         units, return_sequences=False, stateful=True, weights=None)
     model.add(layer)
-    model.compile(optimizer='sgd', loss='mse')
+    model.compile(optimizer=gradient_descent.GradientDescentOptimizer(0.01),
+                  loss='mse', run_eagerly=testing_utils.should_run_eagerly())
     out1 = model.predict(np.ones((num_samples, timesteps)))
     self.assertEqual(out1.shape, (num_samples, units))
 
@@ -497,6 +465,34 @@ class GRULayerV1OnlyTest(test.TestCase, parameterized.TestCase):
     np.testing.assert_allclose(out7, out6, atol=1e-5)
 
 
+class GRULayerGradientTapeTest(test.TestCase):
+
+  @test_util.run_in_graph_and_eager_modes(config=_config)
+  def test_in_tape(self):
+    if not context.executing_eagerly():
+      self.skipTest('bloo')
+    time_steps = 10
+    embedding_size = 11
+    gru_unit_size = 12
+
+    gru = keras.layers.UnifiedGRU(gru_unit_size,
+                                  return_sequences=True,
+                                  return_state=True,
+                                  recurrent_activation='sigmoid',
+                                  recurrent_initializer='glorot_uniform')
+
+    x = random_ops.random_uniform([1, time_steps, embedding_size])
+    y = random_ops.random_uniform([1, gru_unit_size])
+
+    with backprop.GradientTape() as tape:
+      hidden_state = array_ops.zeros([1, gru_unit_size], dtype=dtypes.float32)
+      _, state = gru(x, initial_state=hidden_state)
+
+      loss = math_ops.reduce_mean(math_ops.square(state - y))
+
+    tape.gradient(loss, gru.variables)
+
+
 class GRULayerGraphOnlyTest(test.TestCase):
 
   # Need session for test
diff --git a/tensorflow/python/keras/layers/unified_lstm_test.py b/tensorflow/python/keras/layers/unified_lstm_test.py
index 811b8d128e..375894b166 100644
--- a/tensorflow/python/keras/layers/unified_lstm_test.py
+++ b/tensorflow/python/keras/layers/unified_lstm_test.py
@@ -571,6 +571,68 @@ class UnifiedLSTMTest(keras_parameterized.TestCase):
     else:
       self.assertEqual(len(layer.get_losses_for(x)), 1)
 
+  def test_statefulness_LSTM(self):
+    num_samples = 2
+    timesteps = 3
+    embedding_dim = 4
+    units = 2
+    layer_class = keras.layers.UnifiedLSTM
+    model = keras.models.Sequential()
+    model.add(
+        keras.layers.Embedding(
+            4,
+            embedding_dim,
+            mask_zero=True,
+            input_length=timesteps,
+            batch_input_shape=(num_samples, timesteps)))
+    layer = layer_class(
+        units, return_sequences=False, stateful=True, weights=None)
+    model.add(layer)
+    model.compile(optimizer=gradient_descent.GradientDescentOptimizer(0.01),
+                  loss='mse', run_eagerly=testing_utils.should_run_eagerly())
+    out1 = model.predict(np.ones((num_samples, timesteps)))
+    self.assertEqual(out1.shape, (num_samples, units))
+
+    # train once so that the states change
+    model.train_on_batch(
+        np.ones((num_samples, timesteps)), np.ones((num_samples, units)))
+    out2 = model.predict(np.ones((num_samples, timesteps)))
+
+    # if the state is not reset, output should be different
+    self.assertNotEqual(out1.max(), out2.max())
+
+    # check that output changes after states are reset
+    # (even though the model itself didn't change)
+    layer.reset_states()
+    out3 = model.predict(np.ones((num_samples, timesteps)))
+    self.assertNotEqual(out2.max(), out3.max())
+
+    # check that container-level reset_states() works
+    model.reset_states()
+    out4 = model.predict(np.ones((num_samples, timesteps)))
+    self.assertAllClose(out3, out4, atol=1e-5)
+
+    # check that the call to `predict` updated the states
+    out5 = model.predict(np.ones((num_samples, timesteps)))
+    self.assertNotEqual(out4.max(), out5.max())
+
+    # Check masking
+    layer.reset_states()
+
+    left_padded_input = np.ones((num_samples, timesteps))
+    left_padded_input[0, :1] = 0
+    left_padded_input[1, :2] = 0
+    out6 = model.predict(left_padded_input)
+
+    layer.reset_states()
+
+    right_padded_input = np.ones((num_samples, timesteps))
+    right_padded_input[0, -1:] = 0
+    right_padded_input[1, -2:] = 0
+    out7 = model.predict(right_padded_input)
+
+    self.assertAllClose(out7, out6, atol=1e-5)
+
 
 class LSTMLayerGraphOnlyTest(test.TestCase):
 
@@ -697,69 +759,6 @@ class LSTMLayerV1OnlyTest(test.TestCase, parameterized.TestCase):
         },
         input_shape=(num_samples, timesteps, embedding_dim))
 
-  @test_util.run_in_graph_and_eager_modes(config=_config)
-  def test_statefulness_LSTM(self):
-    num_samples = 2
-    timesteps = 3
-    embedding_dim = 4
-    units = 2
-    layer_class = keras.layers.UnifiedLSTM
-    model = keras.models.Sequential()
-    model.add(
-        keras.layers.Embedding(
-            4,
-            embedding_dim,
-            mask_zero=True,
-            input_length=timesteps,
-            batch_input_shape=(num_samples, timesteps)))
-    layer = layer_class(
-        units, return_sequences=False, stateful=True, weights=None)
-    model.add(layer)
-    model.compile(
-        optimizer=gradient_descent.GradientDescentOptimizer(0.01), loss='mse')
-    out1 = model.predict(np.ones((num_samples, timesteps)))
-    self.assertEqual(out1.shape, (num_samples, units))
-
-    # train once so that the states change
-    model.train_on_batch(
-        np.ones((num_samples, timesteps)), np.ones((num_samples, units)))
-    out2 = model.predict(np.ones((num_samples, timesteps)))
-
-    # if the state is not reset, output should be different
-    self.assertNotEqual(out1.max(), out2.max())
-
-    # check that output changes after states are reset
-    # (even though the model itself didn't change)
-    layer.reset_states()
-    out3 = model.predict(np.ones((num_samples, timesteps)))
-    self.assertNotEqual(out2.max(), out3.max())
-
-    # check that container-level reset_states() works
-    model.reset_states()
-    out4 = model.predict(np.ones((num_samples, timesteps)))
-    self.assertAllClose(out3, out4, atol=1e-5)
-
-    # check that the call to `predict` updated the states
-    out5 = model.predict(np.ones((num_samples, timesteps)))
-    self.assertNotEqual(out4.max(), out5.max())
-
-    # Check masking
-    layer.reset_states()
-
-    left_padded_input = np.ones((num_samples, timesteps))
-    left_padded_input[0, :1] = 0
-    left_padded_input[1, :2] = 0
-    out6 = model.predict(left_padded_input)
-
-    layer.reset_states()
-
-    right_padded_input = np.ones((num_samples, timesteps))
-    right_padded_input[0, -1:] = 0
-    right_padded_input[1, -2:] = 0
-    out7 = model.predict(right_padded_input)
-
-    self.assertAllClose(out7, out6, atol=1e-5)
-
 
 class UnifiedLSTMPerformanceTest(test.Benchmark):
 
-- 
GitLab


From 563ff22e7f6c9431072d2d9af6e876b3b4843920 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 2 Jan 2019 17:06:04 -0800
Subject: [PATCH 0114/2345] Fix bug where sample weights were being ignored in
 the tf.distribute.Strategy / numpy input case.

PiperOrigin-RevId: 227606473
---
 tensorflow/python/keras/engine/training.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 1dc3b81cfc..8263467c37 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -2191,7 +2191,6 @@ class Model(Network):
         else:
           x = dataset_ops.Dataset.from_tensor_slices((var_x, var_y))
 
-        x = dataset_ops.Dataset.from_tensor_slices((var_x, var_y))
         if shuffle:
           # 1024 is a good buffer size since it is much larger than the average
           # batch size provided by the user and provides sufficient randomness.
-- 
GitLab


From b38e065943271461bd9afce7ae8f1bd00c3d6bf9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 2 Jan 2019 17:34:44 -0800
Subject: [PATCH 0115/2345] use packages@tensorflow.org as author email for
 python package

PiperOrigin-RevId: 227609414
---
 tensorflow/lite/tools/pip_package/setup.py | 2 +-
 tensorflow/tools/pip_package/setup.py      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/tools/pip_package/setup.py b/tensorflow/lite/tools/pip_package/setup.py
index 64d62ee1f2..82f9ac5b8f 100644
--- a/tensorflow/lite/tools/pip_package/setup.py
+++ b/tensorflow/lite/tools/pip_package/setup.py
@@ -136,7 +136,7 @@ setup(
     long_description='\n'.join(DOCLINES[2:]),
     url='https://www.tensorflow.org/lite/',
     author='Google Inc.',
-    author_email='opensource@google.com',
+    author_email='packages@tensorflow.org',
     license='Apache 2.0',
     include_package_data=True,
     keywords='tflite tensorflow tensor machine learning',
diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index 3927540cc7..9567e0aff7 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -248,7 +248,7 @@ setup(
     url='https://www.tensorflow.org/',
     download_url='https://github.com/tensorflow/tensorflow/tags',
     author='Google Inc.',
-    author_email='opensource@google.com',
+    author_email='packages@tensorflow.org',
     # Contained modules and scripts.
     packages=find_packages(),
     entry_points={
-- 
GitLab


From 894278658bf6da29831371670862688f46153be0 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Wed, 2 Jan 2019 17:56:16 -0800
Subject: [PATCH 0116/2345] Add top k categorical accuracy and sparse top k
 categorical accuracy v2 metrics.

PiperOrigin-RevId: 227611439
---
 tensorflow/python/keras/metrics.py      | 78 +++++++++++++++++++++++++
 tensorflow/python/keras/metrics_test.py | 76 ++++++++++++++++++++++++
 2 files changed, 154 insertions(+)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 6b81151d5c..73c116a745 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -590,6 +590,84 @@ class SparseCategoricalAccuracy(MeanMetricWrapper):
     return super(SparseCategoricalAccuracy, cls).from_config(config)
 
 
+class TopKCategoricalAccuracy(MeanMetricWrapper):
+  """Computes how often targets are in the top `K` predictions.
+
+  Usage:
+
+  ```python
+  m = tf.keras.metrics.TopKCategoricalAccuracy()
+  m.update_state([[0, 0, 1], [0, 1, 0]], [[0.1, 0.9, 0.8], [0.05, 0.95, 0]])
+  print('Final result: ', m.result().numpy())  # Final result: 1.0
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', metrics=[tf.keras.metrics.TopKCategoricalAccuracy()])
+  ```
+  """
+
+  def __init__(self, k=5, name='top_k_categorical_accuracy', dtype=None):
+    """Creates a `TopKCategoricalAccuracy` instance.
+
+    Args:
+      k: (Optional) Number of top elements to look at for computing accuracy.
+        Defaults to 5.
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+    """
+    super(TopKCategoricalAccuracy, self).__init__(
+        top_k_categorical_accuracy, name, dtype=dtype, k=k)
+
+  @classmethod
+  def from_config(cls, config):
+    if 'fn' in config:
+      config.pop('fn')
+    return super(TopKCategoricalAccuracy, cls).from_config(config)
+
+
+class SparseTopKCategoricalAccuracy(MeanMetricWrapper):
+  """Computes how often integer targets are in the top `K` predictions.
+
+  Usage:
+
+  ```python
+  m = tf.keras.metrics.SparseTopKCategoricalAccuracy()
+  m.update_state([2, 1], [[0.1, 0.9, 0.8], [0.05, 0.95, 0]])
+  print('Final result: ', m.result().numpy())  # Final result: 1.0
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile(
+    'sgd',
+    metrics=[tf.keras.metrics.SparseTopKCategoricalAccuracy()])
+  ```
+  """
+
+  def __init__(self, k=5, name='sparse_top_k_categorical_accuracy', dtype=None):
+    """Creates a `SparseTopKCategoricalAccuracy` instance.
+
+    Args:
+      k: (Optional) Number of top elements to look at for computing accuracy.
+        Defaults to 5.
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+    """
+    super(SparseTopKCategoricalAccuracy, self).__init__(
+        sparse_top_k_categorical_accuracy, name, dtype=dtype, k=k)
+
+  @classmethod
+  def from_config(cls, config):
+    if 'fn' in config:
+      config.pop('fn')
+    return super(SparseTopKCategoricalAccuracy, cls).from_config(config)
+
+
 class _ConfusionMatrixConditionCount(Metric):
   """Calculates the number of the given confusion matrix condition."""
 
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index 8f8befd413..fb635e7055 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -1391,6 +1391,82 @@ class RootMeanSquaredErrorTest(test.TestCase):
     self.assertAllClose(math.sqrt(13), self.evaluate(result), atol=1e-3)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class TopKCategoricalAccuracyTest(test.TestCase):
+
+  def test_config(self):
+    a_obj = metrics.TopKCategoricalAccuracy(name='topkca', dtype=dtypes.int32)
+    self.assertEqual(a_obj.name, 'topkca')
+    self.assertEqual(a_obj._dtype, dtypes.int32)
+
+    a_obj2 = metrics.TopKCategoricalAccuracy.from_config(a_obj.get_config())
+    self.assertEqual(a_obj2.name, 'topkca')
+    self.assertEqual(a_obj2._dtype, dtypes.int32)
+
+  def test_correctness(self):
+    a_obj = metrics.TopKCategoricalAccuracy()
+    self.evaluate(variables.variables_initializer(a_obj.variables))
+    y_true = constant_op.constant([[0, 0, 1], [0, 1, 0]])
+    y_pred = constant_op.constant([[0.1, 0.9, 0.8], [0.05, 0.95, 0]])
+
+    result = a_obj(y_true, y_pred)
+    self.assertEqual(1, self.evaluate(result))  # both the samples match
+
+    # With `k` < 5.
+    a_obj = metrics.TopKCategoricalAccuracy(k=1)
+    self.evaluate(variables.variables_initializer(a_obj.variables))
+    result = a_obj(y_true, y_pred)
+    self.assertEqual(0.5, self.evaluate(result))  # only sample #2 matches
+
+    # With `k` > 5.
+    y_true = constant_op.constant([[0, 0, 1, 0, 0, 0, 0],
+                                   [0, 1, 0, 0, 0, 0, 0]])
+    y_pred = constant_op.constant([[0.5, 0.9, 0.1, 0.7, 0.6, 0.5, 0.4],
+                                   [0.05, 0.95, 0, 0, 0, 0, 0]])
+    a_obj = metrics.TopKCategoricalAccuracy(k=6)
+    self.evaluate(variables.variables_initializer(a_obj.variables))
+    result = a_obj(y_true, y_pred)
+    self.assertEqual(0.5, self.evaluate(result))  # only 1 sample matches.
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class SparseTopKCategoricalAccuracyTest(test.TestCase):
+
+  def test_config(self):
+    a_obj = metrics.SparseTopKCategoricalAccuracy(
+        name='stopkca', dtype=dtypes.int32)
+    self.assertEqual(a_obj.name, 'stopkca')
+    self.assertEqual(a_obj._dtype, dtypes.int32)
+
+    a_obj2 = metrics.SparseTopKCategoricalAccuracy.from_config(
+        a_obj.get_config())
+    self.assertEqual(a_obj2.name, 'stopkca')
+    self.assertEqual(a_obj2._dtype, dtypes.int32)
+
+  def test_correctness(self):
+    a_obj = metrics.SparseTopKCategoricalAccuracy()
+    self.evaluate(variables.variables_initializer(a_obj.variables))
+    y_true = constant_op.constant([2, 1])
+    y_pred = constant_op.constant([[0.1, 0.9, 0.8], [0.05, 0.95, 0]])
+
+    result = a_obj(y_true, y_pred)
+    self.assertEqual(1, self.evaluate(result))  # both the samples match
+
+    # With `k` < 5.
+    a_obj = metrics.SparseTopKCategoricalAccuracy(k=1)
+    self.evaluate(variables.variables_initializer(a_obj.variables))
+    result = a_obj(y_true, y_pred)
+    self.assertEqual(0.5, self.evaluate(result))  # only sample #2 matches
+
+    # With `k` > 5.
+    y_pred = constant_op.constant([[0.5, 0.9, 0.1, 0.7, 0.6, 0.5, 0.4],
+                                   [0.05, 0.95, 0, 0, 0, 0, 0]])
+    a_obj = metrics.SparseTopKCategoricalAccuracy(k=6)
+    self.evaluate(variables.variables_initializer(a_obj.variables))
+    result = a_obj(y_true, y_pred)
+    self.assertEqual(0.5, self.evaluate(result))  # only 1 sample matches.
+
+
 def _get_model(compile_metrics):
   model_layers = [
       layers.Dense(3, activation='relu', kernel_initializer='ones'),
-- 
GitLab


From a8bf983c60f3684220996cd518d2b86e31dd6e1e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 2 Jan 2019 21:49:05 -0800
Subject: [PATCH 0117/2345] use packages@tensorflow.org as author email for
 python package

PiperOrigin-RevId: 227630041
---
 tensorflow/contrib/tpu/profiler/pip_package/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tpu/profiler/pip_package/setup.py b/tensorflow/contrib/tpu/profiler/pip_package/setup.py
index f27ae38e04..807cf26fe9 100644
--- a/tensorflow/contrib/tpu/profiler/pip_package/setup.py
+++ b/tensorflow/contrib/tpu/profiler/pip_package/setup.py
@@ -33,7 +33,7 @@ setup(
     long_description='Tools for capture TPU profile',
     url='https://www.tensorflow.org/tfrc/',
     author='Google Inc.',
-    author_email='opensource@google.com',
+    author_email='packages@tensorflow.org',
     packages=['cloud_tpu_profiler'],
     package_data={
         'cloud_tpu_profiler': ['data/*'],
-- 
GitLab


From 7e71784d707cee4e14689b55a59bb1e23ed1a3f5 Mon Sep 17 00:00:00 2001
From: Heungsub Lee <sub@subl.ee>
Date: Thu, 3 Jan 2019 17:12:21 +0900
Subject: [PATCH 0118/2345] Fix quotation typo around "task"

---
 tensorflow/contrib/distribute/python/mirrored_strategy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index db8fd98307..e635190b50 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -48,7 +48,7 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
   distributed environment.
 
   There are several important concepts for distributed TensorFlow, e.g.
-  `client`, `job`, 'task', `cluster`, `in-graph replication` and
+  `client`, `job`, `task`, `cluster`, `in-graph replication` and
   'synchronous training' and they have already been defined in the
   [TensorFlow's documentation](https://www.tensorflow.org/deploy/distributed).
   The distribution strategy inherits these concepts as well and in addition to
-- 
GitLab


From df0bd2904e2ace75c53e0e7663a2f9cff23f261d Mon Sep 17 00:00:00 2001
From: Kay Zhu <kayzhu@google.com>
Date: Thu, 3 Jan 2019 00:30:28 -0800
Subject: [PATCH 0119/2345] [XLA] Support RNG in HloEvaluator.

PiperOrigin-RevId: 227642508
---
 .../xla/service/hlo_constant_folding.cc       |  5 +-
 .../compiler/xla/service/hlo_evaluator.cc     |  8 ++
 .../compiler/xla/service/hlo_evaluator.h      |  5 +
 .../xla/service/hlo_evaluator_typed_visitor.h | 99 ++++++++++++++++++-
 tensorflow/compiler/xla/tests/BUILD           |  4 -
 tensorflow/compiler/xla/tests/prng_test.cc    | 41 +++++++-
 6 files changed, 153 insertions(+), 9 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_constant_folding.cc b/tensorflow/compiler/xla/service/hlo_constant_folding.cc
index c58d00ff50..799cc9fd91 100644
--- a/tensorflow/compiler/xla/service/hlo_constant_folding.cc
+++ b/tensorflow/compiler/xla/service/hlo_constant_folding.cc
@@ -52,7 +52,7 @@ StatusOr<bool> HloConstantFolding::Run(HloModule* module) {
           computation->root_instruction() != instruction) {
         continue;
       }
-      // Skip Constant, Parameter, Tuple, AfterAll operation.
+      // Skip Constant, Parameter, Tuple, AfterAll, Rng operations.
       // Tuple constants are not directly supported by any backends, hence
       // folding Tuple is not useful and would in fact be expanded back into
       // kTuple by Algebraic Simplifier.
@@ -62,7 +62,8 @@ StatusOr<bool> HloConstantFolding::Run(HloModule* module) {
       if (instruction->opcode() == HloOpcode::kParameter ||
           instruction->opcode() == HloOpcode::kConstant ||
           instruction->opcode() == HloOpcode::kTuple ||
-          instruction->opcode() == HloOpcode::kAfterAll) {
+          instruction->opcode() == HloOpcode::kAfterAll ||
+          instruction->opcode() == HloOpcode::kRng) {
         continue;
       }
 
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index 4f5484293c..e897d09e1b 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -233,6 +233,14 @@ StatusOr<Literal> HloEvaluator::Evaluate(
   for (const auto& literal_ptr : arg_literals) {
     arg_literals_.push_back(&*literal_ptr);
   }
+  if (computation.parent()->config().seed()) {
+    seed_ = computation.parent()->config().seed();
+  } else {
+    std::random_device rd;
+    seed_ = rd();
+  }
+
+  engine_ = std::minstd_rand0(seed_);
 
   TF_RETURN_IF_ERROR(computation.Accept(this));
   return GetEvaluatedLiteralFor(computation.root_instruction()).Clone();
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.h b/tensorflow/compiler/xla/service/hlo_evaluator.h
index d8e3019576..4fc5d10aed 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.h
@@ -290,6 +290,11 @@ class HloEvaluator : public DfsHloVisitorWithDefault {
   // Max loop iterations to execute with no maximum if negative.
   int64 max_loop_iterations_;
 
+  // Module-level seed handle.
+  uint64 seed_;
+  // RNG engine.
+  std::minstd_rand0 engine_;
+
   TF_DISALLOW_COPY_AND_ASSIGN(HloEvaluator);
 };
 
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
index b1bbf49d4a..cf3509d0aa 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
@@ -2653,7 +2653,7 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
   template <typename NativeT, typename std::enable_if<std::is_same<
                                   double, NativeT>::value>::type* = nullptr>
   Status HandleReducePrecision(HloInstruction* reduce_precision) {
-    return InvalidArgument("Double not supported for reduce precision");
+    return InvalidArgument("Double is not supported for reduce precision");
   }
 
   template <
@@ -2719,6 +2719,103 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
     return HandleIota<ReturnT>(iota);
   }
 
+  template <typename NativeT,
+            typename std::enable_if<
+                !(std::is_integral<NativeT>::value ||
+                  std::is_floating_point<NativeT>::value)>::type* = nullptr>
+  Status HandleRng(HloInstruction* random) {
+    return UnsupportedTypeError(random);
+  }
+  template <typename NativeT,
+            typename std::enable_if<
+                (std::is_floating_point<NativeT>::value)>::type* = nullptr>
+  Status HandleRng(HloInstruction* random) {
+    RandomDistribution distribution = random->random_distribution();
+    const auto result_shape = random->shape();
+    Literal result(result_shape);
+
+    switch (distribution) {
+      case RNG_UNIFORM: {
+        const Literal& low =
+            parent_->GetEvaluatedLiteralFor(random->operand(0));
+        const Literal& high =
+            parent_->GetEvaluatedLiteralFor(random->operand(1));
+
+        std::uniform_real_distribution<NativeT> generator(
+            low.Get<NativeT>({}), high.Get<NativeT>({}));
+
+        TF_RETURN_IF_ERROR(
+            result.Populate<NativeT>([&](absl::Span<const int64> /*indexes*/) {
+              return generator(parent_->engine_);
+            }));
+        break;
+      }
+      case RNG_NORMAL: {
+        const Literal& mean =
+            parent_->GetEvaluatedLiteralFor(random->operand(0));
+        const Literal& stddev =
+            parent_->GetEvaluatedLiteralFor(random->operand(1));
+
+        std::normal_distribution<NativeT> generator(mean.Get<NativeT>({}),
+                                                    stddev.Get<NativeT>({}));
+
+        TF_RETURN_IF_ERROR(
+            result.Populate<NativeT>([&](absl::Span<const int64> /*indexes*/) {
+              return generator(parent_->engine_);
+            }));
+        break;
+      }
+      default:
+        return UnimplementedStrCat("The distribution ",
+                                   RandomDistribution_Name(distribution),
+                                   " is not implemented.");
+    }
+    parent_->evaluated_[random] = std::move(result);
+    return Status::OK();
+  }
+  template <typename NativeT,
+            typename std::enable_if<(std::is_integral<NativeT>::value)>::type* =
+                nullptr>
+  Status HandleRng(HloInstruction* random) {
+    RandomDistribution distribution = random->random_distribution();
+    const auto result_shape = random->shape();
+    Literal result(result_shape);
+
+    switch (distribution) {
+      case RNG_UNIFORM: {
+        const Literal& low =
+            parent_->GetEvaluatedLiteralFor(random->operand(0));
+        const Literal& high =
+            parent_->GetEvaluatedLiteralFor(random->operand(1));
+
+        // Note std::uniform_int_distribution assumes interval is closed, i.e.,
+        // [low, high], but we want [low, high) instead. Hence high-1 is used as
+        // the upper range.
+        std::uniform_int_distribution<int64> generator(
+            low.Get<NativeT>({}), high.Get<NativeT>({}) - 1);
+
+        TF_RETURN_IF_ERROR(
+            result.Populate<NativeT>([&](absl::Span<const int64> /*indexes*/) {
+              return static_cast<NativeT>(generator(parent_->engine_));
+            }));
+        break;
+      }
+      case RNG_NORMAL: {
+        return Unimplemented(
+            "Normal distribution is not supported for integral types.");
+      }
+      default:
+        return UnimplementedStrCat("The distribution ",
+                                   RandomDistribution_Name(distribution),
+                                   " is not implemented.");
+    }
+    parent_->evaluated_[random] = std::move(result);
+    return Status::OK();
+  }
+  Status HandleRng(HloInstruction* random) override {
+    return HandleRng<ReturnT>(random);
+  }
+
  private:
   // Creates a vector of multipliers which can be used to create a linear index
   // into shape.
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 578e716b59..2c11ba4283 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -1422,10 +1422,6 @@ xla_test(
 xla_test(
     name = "prng_test",
     srcs = ["prng_test.cc"],
-    blacklisted_backends = [
-        # TODO(b/122047800) support RNGs on the interpreter backend.
-        "interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:shape_util",
diff --git a/tensorflow/compiler/xla/tests/prng_test.cc b/tensorflow/compiler/xla/tests/prng_test.cc
index 8f2c26f0ee..e49bcf26bd 100644
--- a/tensorflow/compiler/xla/tests/prng_test.cc
+++ b/tensorflow/compiler/xla/tests/prng_test.cc
@@ -80,7 +80,9 @@ XLA_TEST_F(PrngTest, LargeU01) { UniformTest<float>(0, 1, {0x100, 0x100}); }
 XLA_TEST_F(PrngTest, TwelveValuesU524) { UniformTest<int32>(5, 24, {12}); }
 
 // TODO(b/71543667): Fix Rng ops on LLVM backends.
-XLA_TEST_F(PrngTest, DISABLED_ON_GPU(DISABLED_ON_CPU(ScalarBF16Tests))) {
+// TODO(b/122047800): Interpreter does not support BF16 for RNG ops.
+XLA_TEST_F(PrngTest, DISABLED_ON_INTERPRETER(
+                         DISABLED_ON_GPU(DISABLED_ON_CPU(ScalarBF16Tests)))) {
   for (int64 seed = 0; seed < 100; ++seed) {
     // The largest negative number smaller than zero in bf16 that's not
     // denormalized.
@@ -103,7 +105,9 @@ XLA_TEST_F(PrngTest, DISABLED_ON_GPU(DISABLED_ON_CPU(ScalarBF16Tests))) {
 }
 
 // TODO(b/71543667): Fix Rng ops on LLVM backends.
-XLA_TEST_F(PrngTest, DISABLED_ON_GPU(DISABLED_ON_CPU(ScalarBF16CountTests))) {
+// TODO(b/122047800): Interpreter does not support BF16 for RNG ops.
+XLA_TEST_F(PrngTest, DISABLED_ON_INTERPRETER(DISABLED_ON_GPU(
+                         DISABLED_ON_CPU(ScalarBF16CountTests)))) {
   // There are 3 BF16 values in the range of [32.25, 33): 32.25, 32.5, 32.75,
   // they should get similar counts.
   bfloat16 low = static_cast<bfloat16>(32.25);
@@ -276,6 +280,39 @@ XLA_TEST_F(PrngTest, PassInGlobalRngSeed) {
   EXPECT_FALSE(LiteralTestUtil::Equal(result5, result6));
 }
 
+// This test verifies that the two RNG instructions with the same parameters in
+// the same HloComputation produces different values.
+XLA_TEST_F(PrngTest, DifferentValuesForIdenticalRngNodesInSameComputation) {
+  // Build a U[0,1) computation.
+  auto build_computation = [this]() {
+    XlaBuilder builder(TestName());
+    auto a = RngUniform(ConstantR0<int32>(&builder, 0),
+                        ConstantR0<int32>(&builder, 100),
+                        ShapeUtil::MakeShape(S32, {10}));
+    auto b = RngUniform(ConstantR0<int32>(&builder, 0),
+                        ConstantR0<int32>(&builder, 100),
+                        ShapeUtil::MakeShape(S32, {10}));
+    Tuple(&builder, {a, b});
+    return builder.Build();
+  };
+
+  ExecutionOptions execution_options = execution_options_;
+  execution_options.set_seed(42);
+
+  Literal result_tuple;
+  {
+    TF_ASSERT_OK_AND_ASSIGN(auto computation, build_computation());
+    TF_ASSERT_OK_AND_ASSIGN(
+        result_tuple, client_->ExecuteAndTransfer(computation, /*arguments=*/{},
+                                                  &execution_options));
+  }
+
+  auto results = result_tuple.DecomposeTuple();
+  ASSERT_EQ(results.size(), 2);
+
+  EXPECT_FALSE(LiteralTestUtil::Equal(results[0], results[1]));
+}
+
 XLA_TEST_F(PrngTest, TenValuesN01) {
   XlaBuilder builder(TestName());
   RngNormal(ConstantR0<float>(&builder, 0), ConstantR0<float>(&builder, 1),
-- 
GitLab


From 37326d1497de6de3ea32a1b121f99b2e50ac2dbc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 01:02:58 -0800
Subject: [PATCH 0120/2345] compat: Update forward compatibility horizon to
 2019-01-03

PiperOrigin-RevId: 227646086
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 949bb0dc0c..c60c42ee65 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 2)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 3)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 61fde9e1e3c5d995aa20f7bf2781ba60db5bf246 Mon Sep 17 00:00:00 2001
From: Heungsub Lee <sub@subl.ee>
Date: Thu, 3 Jan 2019 18:26:13 +0900
Subject: [PATCH 0121/2345] Single quote to backtick for consistency

---
 tensorflow/contrib/distribute/python/mirrored_strategy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index e635190b50..e3ab2bf19e 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -49,7 +49,7 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
 
   There are several important concepts for distributed TensorFlow, e.g.
   `client`, `job`, `task`, `cluster`, `in-graph replication` and
-  'synchronous training' and they have already been defined in the
+  `synchronous training` and they have already been defined in the
   [TensorFlow's documentation](https://www.tensorflow.org/deploy/distributed).
   The distribution strategy inherits these concepts as well and in addition to
   that we also clarify several more concepts:
-- 
GitLab


From eb54349cb4274ab797917a9e0699decec0b9794c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 03:11:44 -0800
Subject: [PATCH 0122/2345] Remove NCCL from RBE dockers. NCCL is built from
 source now.

PiperOrigin-RevId: 227661008
---
 .../Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04         |  9 ---------
 .../ci_build/Dockerfile.rbe.cuda9.0-cudnn7-ubuntu14.04 |  9 ---------
 tensorflow/tools/docker/Dockerfile.devel-gpu           | 10 ----------
 3 files changed, 28 deletions(-)

diff --git a/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04 b/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04
index 553fbb2860..d08d31d913 100644
--- a/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04
+++ b/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04
@@ -19,7 +19,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates
 ENV CUDA_VERSION 10.0.130
 ENV CUDA_PKG_VERSION 10-0=$CUDA_VERSION-1
 ENV CUDNN_VERSION 7.3.1.20
-ENV NCCL_VERSION 2.3.5
 ENV TENSORRT_VERSION 5.0.2
 ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
 ENV NVIDIA_REQUIRE_CUDA "cuda>=10.0,driver>=410"
@@ -48,26 +47,18 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         libcudnn7=$CUDNN_VERSION-1+cuda10.0 \
         libcudnn7=$CUDNN_VERSION-1+cuda10.0 \
         libcudnn7-dev=$CUDNN_VERSION-1+cuda10.0 \
-        libnccl2=$NCCL_VERSION-2+cuda10.0 \
-        libnccl-dev=$NCCL_VERSION-2+cuda10.0 \
         nvinfer-runtime-trt-repo-ubuntu1604-$TENSORRT_VERSION-ga-cuda10.0 && \
     apt-get update && apt-get install -y --no-install-recommends \
         libnvinfer5=$TENSORRT_VERSION-1+cuda10.0 \
         libnvinfer-dev=$TENSORRT_VERSION-1+cuda10.0 && \
     ln -s cuda-10.0 /usr/local/cuda && \
     apt-mark hold libcudnn7 && \
-    apt-mark hold libnccl2 && \
     rm -rf /var/lib/apt/lists/*
 
 # TODO(b/110903506): Provide a link to the SONAME of libcuda.so.
 # https://github.com/NVIDIA/nvidia-docker/issues/775
 RUN ln -s libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1
 
-# TODO(klimek): Once the TODO in tensorflow's configure.py to correctly find
-# libnccl is resolved, delete this block.
-RUN ln -s /usr/lib/x86_64-linux-gnu/libnccl.so /usr/lib/libnccl.so \
- && ln -s /usr/lib/x86_64-linux-gnu/libnccl.so /usr/lib/libnccl.so.2
-
 # Install a newer version of libstdc++, as new clang versions do not work
 # with the stock ubuntu 14.04 libstdc++.
 RUN apt-get update && \
diff --git a/tensorflow/tools/ci_build/Dockerfile.rbe.cuda9.0-cudnn7-ubuntu14.04 b/tensorflow/tools/ci_build/Dockerfile.rbe.cuda9.0-cudnn7-ubuntu14.04
index 60a23e1edb..9d5d51447c 100644
--- a/tensorflow/tools/ci_build/Dockerfile.rbe.cuda9.0-cudnn7-ubuntu14.04
+++ b/tensorflow/tools/ci_build/Dockerfile.rbe.cuda9.0-cudnn7-ubuntu14.04
@@ -25,7 +25,6 @@ ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
 ENV NVIDIA_VISIBLE_DEVICES all
 ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
 ENV NVIDIA_REQUIRE_CUDA "cuda>=9.0"
-ENV NCCL_VERSION 2.2.13
 ENV TENSORRT_VERSION 5.0.2
 ENV CUDNN_VERSION 7.1.4.18
 
@@ -45,14 +44,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         cuda-cudart-$CUDA_PKG_VERSION \
         cuda-libraries-$CUDA_PKG_VERSION \
         cuda-cublas-9-0=9.0.176.4-1 \
-        libnccl2=$NCCL_VERSION-1+cuda9.0 \
         cuda-libraries-dev-$CUDA_PKG_VERSION \
         cuda-nvml-dev-$CUDA_PKG_VERSION \
         cuda-minimal-build-$CUDA_PKG_VERSION \
         cuda-command-line-tools-$CUDA_PKG_VERSION \
         cuda-core-9-0=9.0.176.3-1 \
         cuda-cublas-dev-9-0=9.0.176.4-1 \
-        libnccl-dev=$NCCL_VERSION-1+cuda9.0 \
         libcudnn7-dev=$CUDNN_VERSION-1+cuda9.0 \
         libcudnn7=$CUDNN_VERSION-1+cuda9.0 \
         nvinfer-runtime-trt-repo-ubuntu1604-$TENSORRT_VERSION-ga-cuda9.0 && \
@@ -60,7 +57,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         libnvinfer5=$TENSORRT_VERSION-1+cuda9.0 \
         libnvinfer-dev=$TENSORRT_VERSION-1+cuda9.0 && \
     ln -s cuda-9.0 /usr/local/cuda && \
-    apt-mark hold libnccl2 && \
     apt-mark hold libcudnn7 libcudnn7-dev && \
     rm -rf /var/lib/apt/lists/*
 
@@ -71,11 +67,6 @@ RUN echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf && \
 # https://github.com/NVIDIA/nvidia-docker/issues/775
 RUN ln -s libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1
 
-# TODO(klimek): Once the TODO in tensorflow's configure.py to correctly find
-# libnccl is resolved, delete this block.
-RUN ln -s /usr/lib/x86_64-linux-gnu/libnccl.so /usr/lib/libnccl.so \
- && ln -s /usr/lib/x86_64-linux-gnu/libnccl.so /usr/lib/libnccl.so.2
-
 # Install a newer version of libstdc++, as new clang versions do not work
 # with the stock ubuntu 14.04 libstdc++.
 RUN apt-get update && \
diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu b/tensorflow/tools/docker/Dockerfile.devel-gpu
index 1ad359ddcc..e085ee7170 100644
--- a/tensorflow/tools/docker/Dockerfile.devel-gpu
+++ b/tensorflow/tools/docker/Dockerfile.devel-gpu
@@ -15,8 +15,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         git \
         libcudnn7=7.2.1.38-1+cuda9.0 \
         libcudnn7-dev=7.2.1.38-1+cuda9.0 \
-        libnccl2=2.2.13-1+cuda9.0 \
-        libnccl-dev=2.2.13-1+cuda9.0 \
         libcurl3-dev \
         libfreetype6-dev \
         libhdf5-serial-dev \
@@ -41,11 +39,6 @@ RUN apt-get update && \
         apt-get install libnvinfer4=4.1.2-1+cuda9.0 && \
         apt-get install libnvinfer-dev=4.1.2-1+cuda9.0
 
-# Link NCCL libray and header where the build script expects them.
-RUN mkdir /usr/local/cuda-9.0/lib &&  \
-    ln -s /usr/lib/x86_64-linux-gnu/libnccl.so.2 /usr/local/cuda/lib/libnccl.so.2 && \
-    ln -s /usr/include/nccl.h /usr/local/cuda/include/nccl.h
-
 RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py && \
     python get-pip.py && \
     rm get-pip.py
@@ -111,9 +104,6 @@ ENV TF_CUDA_COMPUTE_CAPABILITIES=3.5,5.2,6.0,6.1,7.0
 ENV TF_CUDA_VERSION=9.0
 ENV TF_CUDNN_VERSION=7
 
-# NCCL 2.x
-ENV TF_NCCL_VERSION=2
-
 RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 && \
     LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:${LD_LIBRARY_PATH} \
     tensorflow/tools/ci_build/builds/configured GPU \
-- 
GitLab


From 31cf2b97f0f3b1a90d26a56016c4b984f62dea3e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 06:51:42 -0800
Subject: [PATCH 0123/2345] Add all unary operations to list of
 autograph-supported ops.

PiperOrigin-RevId: 227681912
---
 .../converters/logical_expressions.py         | 44 ++++++-------
 .../converters/logical_expressions_test.py    |  7 ++
 .../python/autograph/operators/__init__.py    |  2 +
 .../python/autograph/operators/logical.py     | 65 +++++--------------
 4 files changed, 46 insertions(+), 72 deletions(-)

diff --git a/tensorflow/python/autograph/converters/logical_expressions.py b/tensorflow/python/autograph/converters/logical_expressions.py
index dfcaafdc9e..ea9740a22e 100644
--- a/tensorflow/python/autograph/converters/logical_expressions.py
+++ b/tensorflow/python/autograph/converters/logical_expressions.py
@@ -38,29 +38,29 @@ from tensorflow.python.autograph.pyct import templates
 SAFE_BOOLEAN_OPERAND = 'SAFE_BOOLEAN_OPERAND'
 
 
+OP_MAPPING = {
+    gast.And: 'ag__.and_',
+    gast.Eq: 'ag__.eq',
+    gast.NotEq: 'ag__.not_eq',
+    gast.Lt: 'ag__.lt',
+    gast.LtE: 'ag__.lt_e',
+    gast.Gt: 'ag__.gt',
+    gast.GtE: 'ag__.gt_e',
+    gast.Is: 'ag__.is_',
+    gast.IsNot: 'ag__.is_not',
+    gast.In: 'ag__.in_',
+    gast.Not: 'ag__.not_',
+    gast.NotIn: 'ag__.not_in',
+    gast.Or: 'ag__.or_',
+    gast.UAdd: 'ag__.u_add',
+    gast.USub: 'ag__.u_sub',
+    gast.Invert: 'ag__.invert',
+}
+
+
 class LogicalExpressionTransformer(converter.Base):
   """Converts logical expressions to corresponding TF calls."""
 
-  def __init__(self, ctx):
-    super(LogicalExpressionTransformer, self).__init__(ctx)
-    # TODO(mdan): For completeness and consistency, overload everything.
-    self.op_mapping = {
-        gast.And: 'ag__.and_',
-        gast.Eq: 'ag__.eq',
-        gast.NotEq: 'ag__.not_eq',
-        gast.Lt: 'ag__.lt',
-        gast.LtE: 'ag__.lt_e',
-        gast.Gt: 'ag__.gt',
-        gast.GtE: 'ag__.gt_e',
-        gast.Is: 'ag__.is_',
-        gast.IsNot: 'ag__.is_not',
-        gast.In: 'ag__.in_',
-        gast.Not: 'ag__.not_',
-        gast.NotIn: 'ag__.not_in',
-        gast.Or: 'ag__.or_',
-        gast.USub: 'ag__.u_sub',
-    }
-
   def _expect_simple_symbol(self, operand):
     if isinstance(operand, gast.Name):
       return
@@ -74,11 +74,11 @@ class LogicalExpressionTransformer(converter.Base):
 
   def _has_matching_func(self, operator):
     op_type = type(operator)
-    return op_type in self.op_mapping
+    return op_type in OP_MAPPING
 
   def _matching_func(self, operator):
     op_type = type(operator)
-    return self.op_mapping[op_type]
+    return OP_MAPPING[op_type]
 
   def _as_function(self, func_name, args, args_as_lambda=False):
     if args_as_lambda:
diff --git a/tensorflow/python/autograph/converters/logical_expressions_test.py b/tensorflow/python/autograph/converters/logical_expressions_test.py
index 687412750e..67ccd1fb47 100644
--- a/tensorflow/python/autograph/converters/logical_expressions_test.py
+++ b/tensorflow/python/autograph/converters/logical_expressions_test.py
@@ -77,6 +77,13 @@ class LogicalExpressionTest(converter_testing.TestCase):
     with self.converted(test_fn, logical_expressions, {}) as result:
       self.assertTrue(result.test_fn('a', ('a',)))
 
+  def test_unary_ops(self):
+    def test_fn(a):
+      return ~a, -a, +a
+
+    with self.converted(test_fn, logical_expressions, {}) as result:
+      self.assertEqual(result.test_fn(1), (-2, -1, 1))
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/autograph/operators/__init__.py b/tensorflow/python/autograph/operators/__init__.py
index 7a580fe324..35f8028c29 100644
--- a/tensorflow/python/autograph/operators/__init__.py
+++ b/tensorflow/python/autograph/operators/__init__.py
@@ -52,6 +52,7 @@ from tensorflow.python.autograph.operators.logical import eq
 from tensorflow.python.autograph.operators.logical import gt
 from tensorflow.python.autograph.operators.logical import gt_e
 from tensorflow.python.autograph.operators.logical import in_
+from tensorflow.python.autograph.operators.logical import invert
 from tensorflow.python.autograph.operators.logical import is_
 from tensorflow.python.autograph.operators.logical import is_not
 from tensorflow.python.autograph.operators.logical import lt
@@ -60,6 +61,7 @@ from tensorflow.python.autograph.operators.logical import not_
 from tensorflow.python.autograph.operators.logical import not_eq
 from tensorflow.python.autograph.operators.logical import not_in
 from tensorflow.python.autograph.operators.logical import or_
+from tensorflow.python.autograph.operators.logical import u_add
 from tensorflow.python.autograph.operators.logical import u_sub
 from tensorflow.python.autograph.operators.py_builtins import float_
 from tensorflow.python.autograph.operators.py_builtins import int_
diff --git a/tensorflow/python/autograph/operators/logical.py b/tensorflow/python/autograph/operators/logical.py
index 569db5b91b..dadb0daf1a 100644
--- a/tensorflow/python/autograph/operators/logical.py
+++ b/tensorflow/python/autograph/operators/logical.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import operator
+
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gen_math_ops
@@ -35,7 +37,7 @@ def and_(a, b):
   a_val = a()
   if tensor_util.is_tensor(a_val):
     return _tf_lazy_and(a_val, b)
-  return _py_lazy_and(a_val, b)
+  return a_val and b()
 
 
 def _tf_lazy_and(cond, b):
@@ -44,17 +46,12 @@ def _tf_lazy_and(cond, b):
   return control_flow_ops.cond(cond, b, lambda: cond)
 
 
-def _py_lazy_and(cond, b):
-  """Lazy-eval equivalent of "and" in Python."""
-  return cond and b()
-
-
 def or_(a, b):
   """Functional form of "or". Uses lazy evaluation semantics."""
   a_val = a()
   if tensor_util.is_tensor(a_val):
     return _tf_lazy_or(a_val, b)
-  return _py_lazy_or(a_val, b)
+  return a_val or b()
 
 
 def _tf_lazy_or(cond, b):
@@ -63,16 +60,11 @@ def _tf_lazy_or(cond, b):
   return control_flow_ops.cond(cond, lambda: cond, b)
 
 
-def _py_lazy_or(cond, b):
-  """Lazy-eval equivalent of "or" in Python."""
-  return cond or b()
-
-
 def eq(a, b):
   """Functional form of "equal"."""
   if tensor_util.is_tensor(a) or tensor_util.is_tensor(b):
     return _tf_equal(a, b)
-  return _py_equal(a, b)
+  return a == b
 
 
 def _tf_equal(a, b):
@@ -80,11 +72,6 @@ def _tf_equal(a, b):
   return gen_math_ops.equal(a, b)
 
 
-def _py_equal(a, b):
-  """Overload of "equal" that falls back to Python's default implementation."""
-  return a == b
-
-
 def not_eq(a, b):
   """Functional form of "not-equal"."""
   return not_(eq(a, b))
@@ -92,25 +79,8 @@ def not_eq(a, b):
 
 # Default implementation for the remainings.
 
-
-def gt(a, b):
-  """Functional form of "less-than"."""
-  return a > b
-
-
-def gt_e(a, b):
-  """Functional form of "less-than"."""
-  return a >= b
-
-
-def is_(a, b):
-  """Functional form of "less-than"."""
-  return a is b
-
-
-def is_not(a, b):
-  """Functional form of "less-than"."""
-  return a is not b
+is_ = operator.is_
+is_not = operator.is_not
 
 
 def in_(a, b):
@@ -119,21 +89,16 @@ def in_(a, b):
   return a in b
 
 
-def lt(a, b):
-  """Functional form of "less-than"."""
-  return a < b
-
-
-def lt_e(a, b):
-  """Functional form of "less-than"."""
-  return a <= b
-
-
 def not_in(a, b):
   """Functional form of "less-than"."""
   return a not in b
 
+gt = operator.gt
+gt_e = operator.ge
+lt = operator.lt
+lt_e = operator.le
+
 
-def u_sub(a):
-  """Functional form of "unary-sub"."""
-  return -a
+u_add = operator.pos
+u_sub = operator.neg
+invert = operator.invert
-- 
GitLab


From dfad4e97b2d00df768e9e8df16179a6f7ceda43c Mon Sep 17 00:00:00 2001
From: James Keeling <jtkeeling@google.com>
Date: Thu, 3 Jan 2019 09:08:14 -0800
Subject: [PATCH 0124/2345] Fix odd number of backticks in docstring.

This was causing the documentation to render incorrectly.

PiperOrigin-RevId: 227698008
---
 tensorflow/python/ops/linalg/linear_operator_kronecker.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/linalg/linear_operator_kronecker.py b/tensorflow/python/ops/linalg/linear_operator_kronecker.py
index f7e785caa5..005b9b429b 100644
--- a/tensorflow/python/ops/linalg/linear_operator_kronecker.py
+++ b/tensorflow/python/ops/linalg/linear_operator_kronecker.py
@@ -71,7 +71,7 @@ class LinearOperatorKronecker(linear_operator.LinearOperator):
   `op1 x op2 x .. opJ` (we omit parentheses as the Kronecker product is
   associative).
 
-  If `opj` has shape `batch_shape_j` + [M_j, N_j`, then the composed operator
+  If `opj` has shape `batch_shape_j + [M_j, N_j]`, then the composed operator
   will have shape equal to `broadcast_batch_shape + [prod M_j, prod N_j]`,
   where the product is over all operators.
 
-- 
GitLab


From d04b09d65653bb78ad3b14448bf9ef867d9041d4 Mon Sep 17 00:00:00 2001
From: Blake Hechtman <blakehechtman@google.com>
Date: Thu, 3 Jan 2019 09:19:36 -0800
Subject: [PATCH 0125/2345] [XLA] Propagate the layout of transposes and
 reshapes that add one sized dimensions in a depth first order.

PiperOrigin-RevId: 227699358
---
 .../compiler/xla/service/layout_assignment.cc | 23 +++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/service/layout_assignment.cc b/tensorflow/compiler/xla/service/layout_assignment.cc
index 1c4ac178f0..ebf7a88fae 100644
--- a/tensorflow/compiler/xla/service/layout_assignment.cc
+++ b/tensorflow/compiler/xla/service/layout_assignment.cc
@@ -1236,6 +1236,23 @@ Status LayoutAssignment::PropagateUseConstraintToDefs(
       });
 }
 
+namespace {
+// A transpose or a reshape that only changes trivial dimensions have meaningful
+// layouts that are valuable to propagate in a depthfirst manner to avoid
+// unassinged layouts in the graph.
+bool InstructionShouldPropagateDepthFirst(const HloInstruction& hlo) {
+  switch (hlo.opcode()) {
+    case HloOpcode::kReshape:
+      return std::get<0>(hlo.ReshapeMerelyInsertsOrDeletes1SizedDimensions());
+    case HloOpcode::kTranspose:
+      return true;
+    default:
+      return false;
+  }
+}
+
+}  // namespace
+
 Status LayoutAssignment::PropagateOperandConstraint(
     const OperandLayoutConstraint& operand_constraint,
     LayoutConstraints* constraints) {
@@ -1370,7 +1387,7 @@ Status LayoutAssignment::PropagateOperandConstraint(
             TF_RETURN_IF_ERROR(constraints->SetBufferLayout(
                 *layout, *buffer,
                 /*mandatory=*/user->opcode() == HloOpcode::kReduce,
-                /*dfs=*/false));
+                /*dfs=*/InstructionShouldPropagateDepthFirst(*user)));
           }
         }
         return Status::OK();
@@ -1420,11 +1437,9 @@ Status LayoutAssignment::PropagateBufferConstraintToOperands(
             ChooseOperandLayoutFromOutputLayout(buffer_constraint.layout(),
                                                 instruction, operand_no);
         if (operand_layout != nullptr) {
-          // Do not propagate operand constraints of transposes and reshapes, it
-          // tends to create really bad layouts.
           TF_RETURN_IF_ERROR(constraints->SetArrayOperandLayout(
               *operand_layout, instruction, operand_no, /*mandatory=*/false,
-              /*dfs=*/false));
+              /*dfs=*/InstructionShouldPropagateDepthFirst(*instruction)));
         }
       } else {
         VLOG(6) << "Operand already has a constraint "
-- 
GitLab


From a5b70b7cb46e18846ba8875a13da08e7444c5136 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Thu, 3 Jan 2019 09:20:12 -0800
Subject: [PATCH 0126/2345] Fix for an optimizer.get_config() exception when
 exporting a SavedModel

Tests tf.saved_model.save() with Optimizerv2 instead of v1

PiperOrigin-RevId: 227699428
---
 tensorflow/python/keras/backend.py         | 5 +++++
 tensorflow/python/saved_model/save_test.py | 6 +++---
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index 42d94e77a0..6693243a68 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -2794,6 +2794,11 @@ def get_value(x):
   """
   if context.executing_eagerly():
     return x.numpy()
+  elif not getattr(x, '_in_graph_mode', True):
+    # This is a variable which was created in an eager context, but is being
+    # evaluated from a Graph.
+    with context.eager_mode():
+      return x.numpy()
   elif ops.inside_function():
     raise RuntimeError('Cannot get value inside Tensorflow graph function.')
   return x.eval(session=get_session())
diff --git a/tensorflow/python/saved_model/save_test.py b/tensorflow/python/saved_model/save_test.py
index e643ff3dbf..533b954190 100644
--- a/tensorflow/python/saved_model/save_test.py
+++ b/tensorflow/python/saved_model/save_test.py
@@ -41,7 +41,7 @@ from tensorflow.python.saved_model import loader
 from tensorflow.python.saved_model import save
 from tensorflow.python.saved_model import signature_constants
 from tensorflow.python.saved_model import tag_constants
-from tensorflow.python.training import adam
+from tensorflow.python.keras.optimizer_v2 import adam
 from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.training.checkpointable import util
 
@@ -50,7 +50,7 @@ class _ModelWithOptimizer(util.Checkpoint):
 
   def __init__(self):
     self.dense = core.Dense(1)
-    self.optimizer = adam.AdamOptimizer(0.01)
+    self.optimizer = adam.Adam(0.01)
 
   @def_function.function(
       input_signature=(tensor_spec.TensorSpec([None, 2], dtypes.float32),
@@ -401,7 +401,7 @@ class _ModelWithOptimizerUsingDefun(util.Checkpoint):
 
   def __init__(self):
     self.dense = core.Dense(1)
-    self.optimizer = adam.AdamOptimizer(0.01)
+    self.optimizer = adam.Adam(0.01)
 
   # Using defun due to control flow v2 cycles, b/121159261. def_function uses
   # conds to gate variable initialization and so triggers cond reference cycles,
-- 
GitLab


From 78acbc6ee7fa557504599d7847bcd96a9a977d40 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Thu, 3 Jan 2019 09:21:25 -0800
Subject: [PATCH 0127/2345] TFTS: Remove flaky assertions

PiperOrigin-RevId: 227699591
---
 tensorflow/contrib/timeseries/examples/predict_test.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/tensorflow/contrib/timeseries/examples/predict_test.py b/tensorflow/contrib/timeseries/examples/predict_test.py
index 678fd71cd8..b353f85cb5 100644
--- a/tensorflow/contrib/timeseries/examples/predict_test.py
+++ b/tensorflow/contrib/timeseries/examples/predict_test.py
@@ -43,10 +43,6 @@ class PeriodTrendExampleTest(test.TestCase):
     self.assertAllEqual([700], mean.shape)
     self.assertAllEqual([700], upper_limit.shape)
     self.assertAllEqual([700], lower_limit.shape)
-    # Check that variance hasn't blown up too much. This is a relatively good
-    # indication that training was successful.
-    self.assertLess(upper_limit[-1] - lower_limit[-1],
-                    1.5 * (upper_limit[0] - lower_limit[0]))
 
   def test_ar(self):
     (times, observed, all_times, mean,
@@ -55,7 +51,6 @@ class PeriodTrendExampleTest(test.TestCase):
     self.assertAllEqual(all_times.shape, mean.shape)
     self.assertAllEqual(all_times.shape, upper_limit.shape)
     self.assertAllEqual(all_times.shape, lower_limit.shape)
-    self.assertLess((upper_limit - lower_limit).mean(), 4.)
 
 
 if __name__ == "__main__":
-- 
GitLab


From c2ec7217a8b122d6a561da5d7d96b94ede9b7684 Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Thu, 3 Jan 2019 09:53:37 -0800
Subject: [PATCH 0128/2345] Add model saving in Resnet50 code.

In addition,
1. Optimized the weights saving logic in Keras for TPU.
2. Fixed the bug in optimizer for TPU mirrored variable.

PiperOrigin-RevId: 227704010
---
 tensorflow/python/keras/engine/saving.py             | 11 ++++++++++-
 tensorflow/python/keras/optimizer_v2/BUILD           |  1 +
 tensorflow/python/keras/optimizer_v2/optimizer_v2.py |  4 +++-
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/engine/saving.py b/tensorflow/python/keras/engine/saving.py
index 91eba0acab..0604a389a9 100644
--- a/tensorflow/python/keras/engine/saving.py
+++ b/tensorflow/python/keras/engine/saving.py
@@ -25,6 +25,7 @@ import os
 import numpy as np
 from six.moves import zip  # pylint: disable=redefined-builtin
 
+from tensorflow.python.framework import ops
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras import optimizers
 from tensorflow.python.keras.utils import conv_utils
@@ -736,9 +737,17 @@ def save_weights_to_hdf5_group(f, layers):
   f.attrs['backend'] = K.backend().encode('utf8')
   f.attrs['keras_version'] = str(keras_version).encode('utf8')
 
+  # On TPUs, modifying the graph between session.runs() triggers some expensive
+  # recompilation overhead. To avoid this, we build up the full set of tensors
+  # to save before fetching weights, thus only modifying the graph once.
+  layer_weights_dict = {}
+  for layer in layers:
+    layer_weights_dict[layer.name] = [ops.convert_to_tensor(w)
+                                      for w in layer.weights]
+
   for layer in layers:
     g = f.create_group(layer.name)
-    symbolic_weights = layer.weights
+    symbolic_weights = layer_weights_dict[layer.name]
     weight_values = K.batch_get_value(symbolic_weights)
     weight_names = []
     for i, (w, val) in enumerate(zip(symbolic_weights, weight_values)):
diff --git a/tensorflow/python/keras/optimizer_v2/BUILD b/tensorflow/python/keras/optimizer_v2/BUILD
index 05f6e5ac48..964cf5fcba 100644
--- a/tensorflow/python/keras/optimizer_v2/BUILD
+++ b/tensorflow/python/keras/optimizer_v2/BUILD
@@ -34,6 +34,7 @@ py_library(
         "//tensorflow/python:variable_scope",
         "//tensorflow/python:variables",
         "//tensorflow/python/distribute:reduce_util",
+        "//tensorflow/python/distribute:values",
     ],
 )
 
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
index b740b2f05b..98f87a41ae 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
@@ -28,6 +28,7 @@ import six
 from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import distribution_strategy_context as distribute_ctx
 from tensorflow.python.distribute import reduce_util as ds_reduce_util
+from tensorflow.python.distribute import values as distributed_values
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
@@ -576,7 +577,8 @@ class OptimizerV2(checkpointable.CheckpointableBase):
     value = self._get_hyper(hyperparameter_name)
     if callable(value):
       return value()
-    if isinstance(value, (ops.Tensor, tf_variables.Variable)):
+    if isinstance(value, (ops.Tensor, tf_variables.Variable,
+                          distributed_values.TPUMirroredVariable)):
       return backend.get_value(value)
     return value
 
-- 
GitLab


From 3e2408a52822439b04de0a0a070a200c8f15509b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 10:34:07 -0800
Subject: [PATCH 0129/2345] Modify NNAPI delegate SVDF test to use 0 bias.

PiperOrigin-RevId: 227710985
---
 tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
index 2da14cc1de..5da052eb42 100644
--- a/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
+++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
@@ -2003,7 +2003,9 @@ class BaseSVDFOpModel : public SingleOpModelWithNNAPI {
     input_ = AddInput(TensorType_FLOAT32);
     weights_feature_ = AddInput(weights_feature_type);
     weights_time_ = AddInput(weights_time_type);
-    bias_ = AddNullInput();
+    // TODO(b/121383394) : figure out why optional bias causes TFLite segfault
+    // when using NNAPI delegate.
+    bias_ = AddInput(TensorType_FLOAT32);
     const int num_filters = units * rank;
     activation_state_ = AddInput(
         TensorData{TensorType_FLOAT32, {batches, memory_size * num_filters}},
@@ -2019,6 +2021,8 @@ class BaseSVDFOpModel : public SingleOpModelWithNNAPI {
         {units_},                             // bias tensor
         {batches, memory_size * num_filters}  // activation_state tensor
     });
+    // TODO(b/121383394) : remove once the optional bias bug is fixed.
+    PopulateTensor(bias_, std::vector<float>(units_));
   }
 
   // Populates the weights_feature tensor.
-- 
GitLab


From 140f0a3cde3696ca5128322cffda178563c8d04f Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Thu, 3 Jan 2019 10:35:02 -0800
Subject: [PATCH 0130/2345] Finish migration to the new scope based API for
 using distribution strategy.

PiperOrigin-RevId: 227711127
---
 tensorflow/contrib/distribute/README.md       | 30 ++++++++-----------
 .../distribute/python/examples/keras_mnist.py | 19 ++++++------
 2 files changed, 21 insertions(+), 28 deletions(-)

diff --git a/tensorflow/contrib/distribute/README.md b/tensorflow/contrib/distribute/README.md
index 8a8dc159ad..dbcaf8185f 100644
--- a/tensorflow/contrib/distribute/README.md
+++ b/tensorflow/contrib/distribute/README.md
@@ -43,28 +43,19 @@ the workers.
 
 Let's see how to scale to multiple GPUs on one machine using `MirroredStrategy` with [tf.keras] (https://www.tensorflow.org/guide/keras).
 
-Take a very simple model consisting of a single layer:
+Let's define a simple input dataset for training this model. Note that currently we require using
+[`tf.data.Dataset`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset)
+with `DistributionStrategy`.
 
 ```python
 import tensorflow as tf
 from tensorflow import keras
 
-inputs = tf.keras.layers.Input(shape=(1,))
-predictions = tf.keras.layers.Dense(1)(inputs)
-model = tf.keras.models.Model(inputs=inputs, outputs=predictions)
-```
-
-Let's also define a simple input dataset for training this model. Note that currently we require using
-[`tf.data.Dataset`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset)
-with `DistributionStrategy`.
-
-```python
 features = tf.data.Dataset.from_tensors([1.]).repeat(10000).batch(10)
 labels = tf.data.Dataset.from_tensors([1.]).repeat(10000).batch(10)
 train_dataset = tf.data.Dataset.zip((features, labels))
 ```
 
-
 To distribute this Keras model on multiple GPUs using `MirroredStrategy` we
 first instantiate a `MirroredStrategy` object.
 
@@ -72,14 +63,17 @@ first instantiate a `MirroredStrategy` object.
 distribution = tf.contrib.distribute.MirroredStrategy()
 ```
 
-We then compile the Keras model and pass the `MirroredStrategy` object in the
-`distribute` argument (apart from other usual arguments like `loss` and
-`optimizer`).
+Take a very simple model consisting of a single layer. We need to create and compile
+the model under the distribution strategy scope.
 
 ```python
-model.compile(loss='mean_squared_error',
-              optimizer=tf.train.GradientDescentOptimizer(learning_rate=0.2),
-              distribute=distribution)
+with distribution.scope():
+  inputs = tf.keras.layers.Input(shape=(1,))
+  predictions = tf.keras.layers.Dense(1)(inputs)
+  model = tf.keras.models.Model(inputs=inputs, outputs=predictions)
+
+  model.compile(loss='mean_squared_error',
+                optimizer=tf.train.GradientDescentOptimizer(learning_rate=0.2))
 ```
 
 To train the model we call Keras `fit` API using the input dataset that we
diff --git a/tensorflow/contrib/distribute/python/examples/keras_mnist.py b/tensorflow/contrib/distribute/python/examples/keras_mnist.py
index 60fda99664..1ce91ecaf2 100644
--- a/tensorflow/contrib/distribute/python/examples/keras_mnist.py
+++ b/tensorflow/contrib/distribute/python/examples/keras_mnist.py
@@ -109,22 +109,21 @@ def main(_):
   tf.enable_eager_execution()
 
   train_ds, eval_ds, input_shape = get_input_datasets()
-  model = get_model(input_shape)
 
   # Instantiate the MirroredStrategy object. If we don't specify `num_gpus` or
   # the `devices` argument then all the GPUs available on the machine are used.
   # TODO(priyag): Use `tf.distribute.MirroredStrategy` once available.
   strategy = mirrored_strategy.MirroredStrategy(['/gpu:0', '/cpu:0'])
 
-  optimizer = rmsprop.RMSProp(learning_rate=0.001)
-
-  # Compile the model by passing the distribution strategy object to the
-  # `distribute` argument. `fit`, `evaluate` and `predict` will be distributed
-  # based on the strategy instantiated.
-  model.compile(loss=tf.keras.losses.categorical_crossentropy,
-                optimizer=optimizer,
-                metrics=['accuracy'],
-                distribute=strategy)
+  # Create and compile the model under Distribution strategy scope.
+  # `fit`, `evaluate` and `predict` will be distributed based on the strategy
+  # model was compiled with.
+  with strategy.scope():
+    model = get_model(input_shape)
+    optimizer = rmsprop.RMSProp(learning_rate=0.001)
+    model.compile(loss=tf.keras.losses.categorical_crossentropy,
+                  optimizer=optimizer,
+                  metrics=['accuracy'])
 
   # Train the model with the train dataset.
   model.fit(x=train_ds, epochs=20, steps_per_epoch=468)
-- 
GitLab


From 64596a68235709e7f3246d0b120a4146521a998b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 10:37:35 -0800
Subject: [PATCH 0131/2345] No public changes.

PiperOrigin-RevId: 227711548
---
 tensorflow/lite/experimental/swift/BUILD      | 100 +++++
 tensorflow/lite/experimental/swift/LICENSE    | 202 ++++++++++
 tensorflow/lite/experimental/swift/README.md  |  54 +++
 .../swift/Sources/Interpreter.swift           | 265 ++++++++++++++
 .../swift/Sources/InterpreterError.swift      |  99 +++++
 .../swift/Sources/InterpreterOptions.swift    |  29 ++
 .../experimental/swift/Sources/Model.swift    |  40 ++
 .../Sources/QuantizationParameters.swift      |  38 ++
 .../experimental/swift/Sources/Tensor.swift   | 138 +++++++
 .../Configs/TensorFlowLite.tulsigen           |  57 +++
 .../project.tulsiconf                         |  14 +
 .../project.pbxproj                           | 345 ++++++++++++++++++
 .../TensorFlowLiteApp/AppDelegate.swift       |  24 ++
 .../Array+TensorFlowLite.swift                |  22 ++
 .../AppIcon.appiconset/Contents.json          |  98 +++++
 .../Assets.xcassets/Contents.json             |   6 +
 .../Base.lproj/LaunchScreen.storyboard        |  44 +++
 .../Base.lproj/Main.storyboard                |  95 +++++
 .../Data+TensorFlowLite.swift                 |  13 +
 .../TensorFlowLiteApp/Info.plist              |  46 +++
 .../TensorFlowLiteApp/ViewController.swift    | 299 +++++++++++++++
 .../swift/Tests/InterpreterOptionsTests.swift |  54 +++
 .../swift/Tests/InterpreterTests.swift        | 315 ++++++++++++++++
 .../experimental/swift/Tests/ModelTests.swift |  59 +++
 .../Tests/QuantizationParametersTests.swift   |  43 +++
 .../swift/Tests/TensorTests.swift             |  83 +++++
 .../tools/pip_package/pip_smoke_test.py       |  36 +-
 27 files changed, 2603 insertions(+), 15 deletions(-)
 create mode 100644 tensorflow/lite/experimental/swift/BUILD
 create mode 100644 tensorflow/lite/experimental/swift/LICENSE
 create mode 100644 tensorflow/lite/experimental/swift/README.md
 create mode 100644 tensorflow/lite/experimental/swift/Sources/Interpreter.swift
 create mode 100644 tensorflow/lite/experimental/swift/Sources/InterpreterError.swift
 create mode 100644 tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift
 create mode 100644 tensorflow/lite/experimental/swift/Sources/Model.swift
 create mode 100644 tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift
 create mode 100644 tensorflow/lite/experimental/swift/Sources/Tensor.swift
 create mode 100644 tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
 create mode 100644 tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift
 create mode 100644 tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift
 create mode 100644 tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift
 create mode 100644 tensorflow/lite/experimental/swift/Tests/ModelTests.swift
 create mode 100644 tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift
 create mode 100644 tensorflow/lite/experimental/swift/Tests/TensorTests.swift

diff --git a/tensorflow/lite/experimental/swift/BUILD b/tensorflow/lite/experimental/swift/BUILD
new file mode 100644
index 0000000000..3bd288a1e1
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/BUILD
@@ -0,0 +1,100 @@
+# TensorFlow Lite for Swift.
+
+package(default_visibility = ["//visibility:private"])
+
+licenses(["notice"])  # Apache 2.0
+
+exports_files(["LICENSE"])
+
+load("@build_bazel_rules_apple//apple:ios.bzl", "ios_application", "ios_unit_test")
+load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library")
+
+MINIMUM_OS_VERSION = "9.0"
+
+SWIFT_COPTS = [
+    "-wmo",
+]
+
+swift_library(
+    name = "TensorFlowLite",
+    srcs = glob(["Sources/*.swift"]),
+    copts = SWIFT_COPTS,
+    module_name = "TensorFlowLite",
+    tags = ["manual"],
+    deps = [
+        "//tensorflow/lite/experimental/c:c_api",
+    ],
+)
+
+ios_unit_test(
+    name = "TensorFlowLiteTests",
+    size = "small",
+    minimum_os_version = MINIMUM_OS_VERSION,
+    tags = [
+        "manual",
+        # DISABLED: Following sanitizer tests are not supported by iOS test targets.
+        "noasan",
+        "nomsan",
+        "notsan",
+    ],
+    deps = [":TensorFlowLiteTestsLib"],
+)
+
+swift_library(
+    name = "TensorFlowLiteTestsLib",
+    testonly = 1,
+    srcs = glob(["Tests/*.swift"]),
+    copts = SWIFT_COPTS,
+    tags = ["manual"],
+    deps = [
+        ":TensorFlowLite",
+        ":TestResources",
+    ],
+)
+
+objc_library(
+    name = "TestResources",
+    resources = [
+        "//tensorflow/lite:testdata/add.bin",
+        "//tensorflow/lite:testdata/add_quantized.bin",
+        "//tensorflow/lite:testdata/multi_add.bin",
+    ],
+    tags = ["manual"],
+)
+
+ios_application(
+    name = "TensorFlowLiteApp",
+    app_icons = glob(["TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/**"]),
+    bundle_id = "com.tensorflow.lite.swift.TensorFlowLite",
+    families = [
+        "ipad",
+        "iphone",
+    ],
+    infoplists = ["TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist"],
+    launch_storyboard = "TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard",
+    minimum_os_version = MINIMUM_OS_VERSION,
+    sdk_frameworks = [
+        "CoreGraphics",
+    ],
+    tags = ["manual"],
+    deps = [":TensorFlowLiteAppLib"],
+)
+
+swift_library(
+    name = "TensorFlowLiteAppLib",
+    srcs = glob(["TestApps/TensorFlowLiteApp/TensorFlowLiteApp/*.swift"]),
+    tags = ["manual"],
+    deps = [
+        ":TensorFlowLite",
+        ":TensorFlowLiteAppResources",
+    ],
+)
+
+objc_library(
+    name = "TensorFlowLiteAppResources",
+    storyboards = glob([
+        "TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/*.storyboard",
+    ]),
+    tags = ["manual"],
+    deps = [":TestResources"],
+)
diff --git a/tensorflow/lite/experimental/swift/LICENSE b/tensorflow/lite/experimental/swift/LICENSE
new file mode 100644
index 0000000000..d645695673
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/LICENSE
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/tensorflow/lite/experimental/swift/README.md b/tensorflow/lite/experimental/swift/README.md
new file mode 100644
index 0000000000..deb16c1e91
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/README.md
@@ -0,0 +1,54 @@
+# TensorFlow Lite for Swift
+
+[TensorFlow Lite](https://www.tensorflow.org/lite/) is TensorFlow's lightweight
+solution for Swift developers. It enables low-latency inference of on-device
+machine learning models with a small binary size and fast performance supporting
+hardware acceleration.
+
+## Getting Started
+
+### Bazel
+
+In your `BUILD` file, add the `TensorFlowLite` dependency:
+
+```python
+swift_library(
+  # ...
+  deps = [
+      "//tensorflow/lite/swift:TensorFlowLite",
+  ],
+)
+```
+
+In your Swift files, import the module:
+
+```swift
+import TensorFlowLite
+```
+
+### Tulsi
+
+Open the `TensorFlowLite.tulsiproj` using the [TulsiApp](https://github.com/bazelbuild/tulsi) or by
+running the [`generate_xcodeproj.sh`](https://github.com/bazelbuild/tulsi/blob/master/src/tools/generate_xcodeproj.sh)
+script:
+
+```shell
+generate_xcodeproj.sh --genconfig tensorflow/lite/swift/TensorFlowLite.tulsiproj:TensorFlowLite --outputfolder ~/path/to/generated/TensorFlowLite.xcodeproj
+```
+
+### CocoaPods
+
+Add the following to your `Podfile`:
+
+```ruby
+use_frameworks!
+pod 'TensorFlowLiteSwift'
+```
+
+Then, run `pod install`.
+
+In your Swift files, import the module:
+
+```swift
+import TensorFlowLite
+```
diff --git a/tensorflow/lite/experimental/swift/Sources/Interpreter.swift b/tensorflow/lite/experimental/swift/Sources/Interpreter.swift
new file mode 100644
index 0000000000..a14b5966b1
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Sources/Interpreter.swift
@@ -0,0 +1,265 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+import TensorFlowLiteCAPI
+
+/// A TensorFlow Lite interpreter that performs inference from a given model.
+public final class Interpreter {
+
+  /// The `TFL_Interpreter` C pointer type represented as an `UnsafePointer<TFL_Interpreter>`.
+  private typealias CInterpreter = OpaquePointer
+
+  /// Total number of input tensors associated with the model.
+  public var inputTensorCount: Int {
+    return Int(TFL_InterpreterGetInputTensorCount(cInterpreter))
+  }
+
+  /// Total number of output tensors associated with the model.
+  public var outputTensorCount: Int {
+    return Int(TFL_InterpreterGetOutputTensorCount(cInterpreter))
+  }
+
+  /// The underlying `TFL_Interpreter` C pointer.
+  private var cInterpreter: CInterpreter?
+
+  /// Creates a new model interpreter instance.
+  ///
+  /// - Parameters:
+  ///   - modelPath: Local file path to a TensorFlow Lite model.
+  ///   - options: Custom configurations for the interpreter. The default is `nil` indicating that
+  ///       interpreter will determine the configuration options.
+  /// - Throws: An error if the model could not be loaded or the interpreter could not be created.
+  public init(modelPath: String, options: InterpreterOptions? = nil) throws {
+    guard let model = Model(filePath: modelPath) else { throw InterpreterError.failedToLoadModel }
+
+    let cInterpreterOptions: OpaquePointer? = try options.map { options in
+      guard let cOptions = TFL_NewInterpreterOptions() else {
+        throw InterpreterError.failedToCreateInterpreter
+      }
+      if let threadCount = options.threadCount, threadCount > 0 {
+        TFL_InterpreterOptionsSetNumThreads(cOptions, Int32(threadCount))
+      }
+      if options.isErrorLoggingEnabled {
+        TFL_InterpreterOptionsSetErrorReporter(
+          cOptions,
+          { (_, format, arguments) in
+            guard let cFormat = format,
+                  let message = String(cFormat: cFormat, arguments: arguments)
+            else {
+              return
+            }
+            print(String(describing: InterpreterError.tensorFlowLiteError(message)))
+          },
+          nil
+        )
+      }
+      return cOptions
+    }
+    defer { TFL_DeleteInterpreterOptions(cInterpreterOptions) }
+
+    guard let cInterpreter = TFL_NewInterpreter(model.cModel, cInterpreterOptions) else {
+      throw InterpreterError.failedToCreateInterpreter
+    }
+    self.cInterpreter = cInterpreter
+  }
+
+  deinit {
+    TFL_DeleteInterpreter(cInterpreter)
+  }
+
+  /// Invokes the interpreter to perform inference from the loaded graph.
+  ///
+  /// - Throws: An error if the model was not ready because tensors were not allocated.
+  public func invoke() throws {
+    guard TFL_InterpreterInvoke(cInterpreter) == kTfLiteOk else {
+      // TODO(b/117510052): Determine which error to throw.
+      throw InterpreterError.allocateTensorsRequired
+    }
+  }
+
+  /// Returns the input tensor at the given index.
+  ///
+  /// - Parameters:
+  ///   - index: The index for the input tensor.
+  /// - Throws: An error if the index is invalid or the tensors have not been allocated.
+  /// - Returns: The input tensor at the given index.
+  public func input(at index: Int) throws -> Tensor {
+    let maxIndex = inputTensorCount - 1
+    guard case 0...maxIndex = index else {
+      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
+    }
+    guard let cTensor = TFL_InterpreterGetInputTensor(cInterpreter, Int32(index)),
+          let bytes = TFL_TensorData(cTensor),
+          let nameCString = TFL_TensorName(cTensor)
+    else {
+      throw InterpreterError.allocateTensorsRequired
+    }
+    guard let dataType = TensorDataType(type: TFL_TensorType(cTensor)) else {
+      throw InterpreterError.invalidTensorDataType
+    }
+
+    let name = String(cString: nameCString)
+    let rank = TFL_TensorNumDims(cTensor)
+    let dimensions = (0..<rank).map { Int(TFL_TensorDim(cTensor, $0)) }
+    let shape = TensorShape(dimensions)
+    let byteCount = TFL_TensorByteSize(cTensor)
+    let data = Data(bytes: bytes, count: byteCount)
+    let cQuantizationParams = TFL_TensorQuantizationParams(cTensor)
+    let scale = cQuantizationParams.scale
+    let zeroPoint = Int(cQuantizationParams.zero_point)
+    var quantizationParameters: QuantizationParameters? = nil
+    if scale != 0.0 {
+      // TODO(b/117510052): Update this check once the TfLiteQuantizationParams struct has a mode.
+      quantizationParameters = QuantizationParameters(scale: scale, zeroPoint: zeroPoint)
+    }
+    let tensor = Tensor(
+      name: name,
+      dataType: dataType,
+      shape: shape,
+      data: data,
+      quantizationParameters: quantizationParameters
+    )
+    return tensor
+  }
+
+  /// Returns the output tensor at the given index.
+  ///
+  /// - Parameters:
+  ///   - index: The index for the output tensor.
+  /// - Throws: An error if the index is invalid, tensors haven't been allocated, or interpreter
+  ///     hasn't been invoked for models that dynamically compute output tensors based on the values
+  ///     of its input tensors.
+  /// - Returns: The output tensor at the given index.
+  public func output(at index: Int) throws -> Tensor {
+    let maxIndex = outputTensorCount - 1
+    guard case 0...maxIndex = index else {
+      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
+    }
+    guard let cTensor = TFL_InterpreterGetOutputTensor(cInterpreter, Int32(index)),
+          let bytes = TFL_TensorData(cTensor),
+          let nameCString = TFL_TensorName(cTensor)
+    else {
+      // TODO(b/117510052): Determine which error to throw.
+      throw InterpreterError.invokeInterpreterRequired
+    }
+    guard let dataType = TensorDataType(type: TFL_TensorType(cTensor)) else {
+      throw InterpreterError.invalidTensorDataType
+    }
+
+    let name = String(cString: nameCString)
+    let rank = TFL_TensorNumDims(cTensor)
+    let dimensions = (0..<rank).map { Int(TFL_TensorDim(cTensor, $0)) }
+    let shape = TensorShape(dimensions)
+    let byteCount = TFL_TensorByteSize(cTensor)
+    let data = Data(bytes: bytes, count: byteCount)
+    let cQuantizationParams = TFL_TensorQuantizationParams(cTensor)
+    let scale = cQuantizationParams.scale
+    let zeroPoint = Int(cQuantizationParams.zero_point)
+    var quantizationParameters: QuantizationParameters? = nil
+    if scale != 0.0 {
+      // TODO(b/117510052): Update this check once the TfLiteQuantizationParams struct has a mode.
+      quantizationParameters = QuantizationParameters(scale: scale, zeroPoint: zeroPoint)
+    }
+    let tensor = Tensor(
+      name: name,
+      dataType: dataType,
+      shape: shape,
+      data: data,
+      quantizationParameters: quantizationParameters
+    )
+    return tensor
+  }
+
+  /// Resizes the input tensor at the given index to the specified tensor shape.
+  ///
+  /// - Note: After resizing an input tensor, the client **must** explicitly call
+  ///     `allocateTensors()` before attempting to access the resized tensor data or invoking the
+  ///     interpreter to perform inference.
+  /// - Parameters:
+  ///   - index: The index for the input tensor.
+  ///   - shape: The shape that the input tensor should be resized to.
+  /// - Throws: An error if the input tensor at the given index could not be resized.
+  public func resizeInput(at index: Int, to shape: TensorShape) throws {
+    let maxIndex = inputTensorCount - 1
+    guard case 0...maxIndex = index else {
+      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
+    }
+    guard TFL_InterpreterResizeInputTensor(
+            cInterpreter,
+            Int32(index),
+            shape.int32Dimensions,
+            Int32(shape.rank)
+          ) == kTfLiteOk
+    else {
+      throw InterpreterError.failedToResizeInputTensor(index: index)
+    }
+  }
+
+  /// Copies the given data to the input tensor at the given index.
+  ///
+  /// - Parameters:
+  ///   - data: The data to be copied to the input tensor's data buffer.
+  ///   - index: The index for the input tensor.
+  /// - Throws: An error if the `data.count` does not match the input tensor's `data.count` or if
+  ///     the given index is invalid.
+  /// - Returns: The input tensor with the copied data.
+  @discardableResult
+  public func copy(_ data: Data, toInputAt index: Int) throws -> Tensor {
+    let maxIndex = inputTensorCount - 1
+    guard case 0...maxIndex = index else {
+      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
+    }
+    guard let cTensor = TFL_InterpreterGetInputTensor(cInterpreter, Int32(index)) else {
+      throw InterpreterError.allocateTensorsRequired
+    }
+
+    let byteCount = TFL_TensorByteSize(cTensor)
+    guard data.count == byteCount else {
+      throw InterpreterError.invalidTensorDataCount(provided: data.count, required: byteCount)
+    }
+
+    let status = data.withUnsafeBytes { TFL_TensorCopyFromBuffer(cTensor, $0, data.count) }
+    guard status == kTfLiteOk else { throw InterpreterError.failedToCopyDataToInputTensor }
+    return try input(at: index)
+  }
+
+  /// Allocates memory for all input tensors based on their `TensorShape`s.
+  ///
+  /// - Note: This is a relatively expensive operation and should only be called after creating the
+  ///     interpreter and/or resizing any input tensors.
+  /// - Throws: An error if memory could not be allocated for the input tensors.
+  public func allocateTensors() throws {
+    guard TFL_InterpreterAllocateTensors(cInterpreter) == kTfLiteOk else {
+      throw InterpreterError.failedToAllocateTensors
+    }
+  }
+}
+
+// MARK: - Extensions
+
+extension String {
+  /// Returns a new `String` initialized by using the given format C array as a template into which
+  /// the remaining argument values are substituted according to the user’s default locale.
+  ///
+  /// - Note: Returns `nil` if a new `String` could not be constructed from the given values.
+  /// - Parameters:
+  ///   - cFormat: The format C array as a template for substituting values.
+  ///   - arguments: A C pointer to a `va_list` of arguments to substitute into `cFormat`.
+  init?(cFormat: UnsafePointer<CChar>, arguments: CVaListPointer) {
+    var buffer: UnsafeMutablePointer<CChar>?
+    guard vasprintf(&buffer, cFormat, arguments) != 0, let cString = buffer else { return nil }
+    self.init(validatingUTF8: cString)
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/Sources/InterpreterError.swift b/tensorflow/lite/experimental/swift/Sources/InterpreterError.swift
new file mode 100644
index 0000000000..5de58b997a
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Sources/InterpreterError.swift
@@ -0,0 +1,99 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+
+/// TensorFlow Lite interpreter errors.
+public enum InterpreterError: Error {
+  case invalidTensorIndex(index: Int, maxIndex: Int)
+  case invalidTensorDataCount(provided: Int, required: Int)
+  case invalidTensorDataType
+  case failedToLoadModel
+  case failedToCreateInterpreter
+  case failedToResizeInputTensor(index: Int)
+  case failedToCopyDataToInputTensor
+  case failedToAllocateTensors
+  case allocateTensorsRequired
+  case invokeInterpreterRequired
+  case tensorFlowLiteError(String)
+}
+
+// MARK: - Extensions
+
+extension InterpreterError: LocalizedError {
+  /// Localized description of the interpreter error.
+  public var errorDescription: String? {
+    switch self {
+    case .invalidTensorIndex(let index, let maxIndex):
+      return "Invalid tensor index \(index), max index is \(maxIndex)."
+    case .invalidTensorDataCount(let providedCount, let requiredCount):
+      return "Provided data count \(providedCount) must match the required count \(requiredCount)."
+    case .invalidTensorDataType:
+      return "Tensor data type is unsupported or could not be determined because of a model error."
+    case .failedToLoadModel:
+      return "Failed to load the given model."
+    case .failedToCreateInterpreter:
+      return "Failed to create the interpreter."
+    case .failedToResizeInputTensor(let index):
+      return "Failed to resize input tesnor at index \(index)."
+    case .failedToCopyDataToInputTensor:
+      return "Failed to copy data to input tensor."
+    case .failedToAllocateTensors:
+      return "Failed to allocate memory for input tensors."
+    case .allocateTensorsRequired:
+      return "Must call allocateTensors()."
+    case .invokeInterpreterRequired:
+      return "Must call invoke()."
+    case .tensorFlowLiteError(let message):
+      return "TensorFlow Lite Error: \(message)"
+    }
+  }
+}
+
+extension InterpreterError: CustomStringConvertible {
+  /// Textual representation of the TensorFlow Lite interpreter error.
+  public var description: String {
+    return errorDescription ?? "Unknown error."
+  }
+}
+
+#if swift(>=4.2)
+extension InterpreterError: Equatable {}
+#else
+extension InterpreterError: Equatable {
+  public static func == (lhs: InterpreterError, rhs: InterpreterError) -> Bool {
+    switch (lhs, rhs) {
+    case (.invalidTensorDataType, .invalidTensorDataType),
+         (.failedToLoadModel, .failedToLoadModel),
+         (.failedToCreateInterpreter, .failedToCreateInterpreter),
+         (.failedToAllocateTensors, .failedToAllocateTensors),
+         (.allocateTensorsRequired, .allocateTensorsRequired),
+         (.invokeInterpreterRequired, .invokeInterpreterRequired):
+      return true
+    case (.invalidTensorIndex(let lhsIndex, let lhsMaxIndex),
+          .invalidTensorIndex(let rhsIndex, let rhsMaxIndex)):
+      return lhsIndex == rhsIndex && lhsMaxIndex == rhsMaxIndex
+    case (.invalidTensorDataCount(let lhsProvidedCount, let lhsRequiredCount),
+          .invalidTensorDataCount(let rhsProvidedCount, let rhsRequiredCount)):
+      return lhsProvidedCount == rhsProvidedCount && lhsRequiredCount == rhsRequiredCount
+    case (.failedToResizeInputTensor(let lhsIndex), .failedToResizeInputTensor(let rhsIndex)):
+      return lhsIndex == rhsIndex
+    case (.tensorFlowLiteError(let lhsMessage), .tensorFlowLiteError(let rhsMessage)):
+      return lhsMessage == rhsMessage
+    default:
+      return false
+    }
+  }
+}
+#endif  // swift(>=4.2)
diff --git a/tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift b/tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift
new file mode 100644
index 0000000000..2365fd7ade
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift
@@ -0,0 +1,29 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+
+/// Custom configuration options for a TensorFlow Lite interpreter.
+public struct InterpreterOptions: Equatable {
+
+  /// Maximum number of CPU threads that the interpreter should run on. Default is `nil` which
+  /// indicates that the `Interpreter` will decide the number of threads to use.
+  public var threadCount: Int? = nil
+
+  /// Whether error logging to the console is enabled. The default is `false`.
+  public var isErrorLoggingEnabled = false
+
+  /// Creates a new instance of interpreter options.
+  public init() {}
+}
diff --git a/tensorflow/lite/experimental/swift/Sources/Model.swift b/tensorflow/lite/experimental/swift/Sources/Model.swift
new file mode 100644
index 0000000000..e8c49ff1ae
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Sources/Model.swift
@@ -0,0 +1,40 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+import TensorFlowLiteCAPI
+
+/// A TensorFlow Lite model used by the 'Interpreter` to perform inference.
+final class Model {
+
+  /// The `TFL_Model` C pointer type represented as an `UnsafePointer<TFL_Model>`.
+  typealias CModel = OpaquePointer
+
+  /// The underlying `TFL_Model` C pointer.
+  let cModel: CModel?
+
+  /// Creates a new model instance.
+  ///
+  /// - Precondition: Initialization can fail if the given `filePath` is invalid.
+  /// - Parameters:
+  ///   - filePath: Local file path to a TensorFlow Lite model.
+  init?(filePath: String) {
+    guard !filePath.isEmpty, let cModel = TFL_NewModelFromFile(filePath) else { return nil }
+    self.cModel = cModel
+  }
+
+  deinit {
+    TFL_DeleteModel(cModel)
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift b/tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift
new file mode 100644
index 0000000000..f367875644
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift
@@ -0,0 +1,38 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+
+/// Parameters that determine the mapping of quantized values to real values. Quantized values can
+/// be mapped to float values using the following conversion:
+/// `realValue = scale * (quantizedValue - zeroPoint)`.
+public struct QuantizationParameters {
+
+  /// Difference between real values corresponding to consecutive quantized values differing by 1.
+  /// For example, the range of quantized values for `UInt8` data type is [0, 255].
+  public let scale: Float
+
+  /// Quantized value that corresponds to the real 0 value.
+  public let zeroPoint: Int
+
+  /// Creates a new quantization parameters instance.
+  ///
+  /// - Parameters:
+  ///   - scale: Scale value for asymmetric quantization.
+  ///   - zeroPoint: Zero point for asymmetric quantization.
+  init(scale: Float, zeroPoint: Int) {
+    self.scale = scale
+    self.zeroPoint = zeroPoint
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/Sources/Tensor.swift b/tensorflow/lite/experimental/swift/Sources/Tensor.swift
new file mode 100644
index 0000000000..b738d87549
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Sources/Tensor.swift
@@ -0,0 +1,138 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+import TensorFlowLiteCAPI
+
+/// An input or output tensor in a TensorFlow Lite graph.
+public struct Tensor {
+
+  /// Name of the tensor.
+  public let name: String
+
+  /// Data type of the tensor.
+  public let dataType: TensorDataType
+
+  /// Shape of the tensor.
+  public let shape: TensorShape
+
+  /// Data in the input or output tensor.
+  public let data: Data
+
+  /// Quantization parameters for the tensor if using a quantized model.
+  public let quantizationParameters: QuantizationParameters?
+
+  /// Creates a new input or output tensor instance.
+  ///
+  /// - Parameters:
+  ///   - name: Name of the tensor.
+  ///   - dataType: Data type of the tensor.
+  ///   - data: Data in the input tensor.
+  ///   - quantizationParameters Quantization parameters for the tensor if using a quantized model.
+  ///       The default is `nil`.
+  init(
+    name: String,
+    dataType: TensorDataType,
+    shape: TensorShape,
+    data: Data,
+    quantizationParameters: QuantizationParameters? = nil
+  ) {
+    self.name = name
+    self.dataType = dataType
+    self.shape = shape
+    self.data = data
+    self.quantizationParameters = quantizationParameters
+  }
+}
+
+/// Supported TensorFlow Lite tensor data types.
+public enum TensorDataType: Equatable {
+  /// 32-bit single precision floating point tensor data type.
+  case float32
+  /// 8-bit unsigned integer tensor data type.
+  case uInt8
+  /// 16-bit signed integer tensor data type.
+  case int16
+  /// 32-bit signed integer tensor data type.
+  case int32
+  /// 64-bit signed integer tensor data type.
+  case int64
+  /// Boolean tensor data type.
+  case bool
+
+  /// Creates a new tensor data type from the given `TFL_Type` or `nil` if the data type is
+  /// unsupported or could not be determined because there was an error.
+  ///
+  /// - Parameter type: A data type supported by a tensor.
+  init?(type: TFL_Type) {
+    switch type {
+    case kTfLiteFloat32:
+      self = .float32
+    case kTfLiteUInt8:
+      self = .uInt8
+    case kTfLiteInt16:
+      self = .int16
+    case kTfLiteInt32:
+      self = .int32
+    case kTfLiteInt64:
+      self = .int64
+    case kTfLiteBool:
+      self = .bool
+    case kTfLiteNoType:
+      fallthrough
+    default:
+      return nil
+    }
+  }
+}
+
+/// The shape of a TensorFlow Lite tensor.
+public struct TensorShape {
+
+  /// The number of dimensions of the tensor.
+  public let rank: Int
+
+  /// Array of dimensions for the tensor.
+  public let dimensions: [Int]
+
+  /// Array of `Int32` dimensions for the tensor.
+  var int32Dimensions: [Int32] { return dimensions.map(Int32.init) }
+
+  /// Creates a new tensor shape instance with the given array of dimensions.
+  ///
+  /// - Parameters:
+  ///   - dimensions: Dimensions for the tensor.
+  public init(_ dimensions: [Int]) {
+    self.rank = dimensions.count
+    self.dimensions = dimensions
+  }
+
+  /// Creates a new tensor shape instance with the given elements representing the dimensions.
+  ///
+  /// - Parameters:
+  ///   - elements: Dimensions for the tensor.
+  public init(_ elements: Int...) {
+    self.init(elements)
+  }
+}
+
+extension TensorShape: ExpressibleByArrayLiteral {
+  /// Creates a new tensor shape instance with the given array literal representing the dimensions.
+  ///
+  /// - Parameters:
+  ///   - arrayLiteral: Dimensions for the tensor.
+  public init(arrayLiteral: Int...) {
+    self.init(arrayLiteral)
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
new file mode 100644
index 0000000000..4010fab49e
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
@@ -0,0 +1,57 @@
+{
+  "sourceFilters" : [
+    "third_party/tensorflow/lite/experimental/c",
+    "third_party/tensorflow/lite/experimental/swift",
+    "third_party/tensorflow/lite/experimental/swift/Sources",
+    "third_party/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp",
+    "third_party/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj",
+    "third_party/tensorflow/lite/experimental/swift/Tests",
+  ],
+  "buildTargets" : [
+    "//third_party/tensorflow/lite/experimental/swift:TensorFlowLite",
+    "//third_party/tensorflow/lite/experimental/swift:TensorFlowLiteApp",
+    "//third_party/tensorflow/lite/experimental/swift:TensorFlowLiteTests",
+  ],
+  "projectName" : "TensorFlowLite",
+  "optionSet" : {
+    "LaunchActionPreActionScript" : {
+      "p" : "$(inherited)"
+    },
+    "BazelBuildStartupOptionsRelease" : {
+      "p" : "$(inherited)"
+    },
+    "BazelBuildOptionsRelease" : {
+      "p" : "$(inherited)"
+    },
+    "BazelBuildOptionsDebug" : {
+      "p" : "$(inherited)"
+    },
+    "EnvironmentVariables" : {
+      "p" : "$(inherited)"
+    },
+    "BuildActionPreActionScript" : {
+      "p" : "$(inherited)"
+    },
+    "CommandlineArguments" : {
+      "p" : "$(inherited)"
+    },
+    "TestActionPreActionScript" : {
+      "p" : "$(inherited)"
+    },
+    "BazelBuildStartupOptionsDebug" : {
+      "p" : "$(inherited)"
+    },
+    "BuildActionPostActionScript" : {
+      "p" : "$(inherited)"
+    },
+    "TestActionPostActionScript" : {
+      "p" : "$(inherited)"
+    },
+    "LaunchActionPostActionScript" : {
+      "p" : "$(inherited)"
+    }
+  },
+  "additionalFilePaths" : [
+    "third_party/tensorflow/lite/experimental/swift/BUILD"
+  ]
+}
diff --git a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
new file mode 100644
index 0000000000..14cff94453
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
@@ -0,0 +1,14 @@
+{
+  "configDefaults" : {
+    "optionSet" : {
+      "ProjectPrioritizesSwift" : {
+        "p" : "YES"
+      }
+    }
+  },
+  "projectName" : "TensorFlowLite",
+  "packages" : [
+    "third_party/tensorflow/lite/experimental/swift"
+  ],
+  "workspaceRoot" : "../../../../../.."
+}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj
new file mode 100644
index 0000000000..fbbf9a1de2
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj
@@ -0,0 +1,345 @@
+// !$*UTF8*$!
+{
+	archiveVersion = 1;
+	classes = {
+	};
+	objectVersion = 50;
+	objects = {
+
+/* Begin PBXBuildFile section */
+		4A7304B421500B8400C90B21 /* Data+TensorFlowLite.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4A7304B321500B8300C90B21 /* Data+TensorFlowLite.swift */; };
+		4AA72B732146ED64006C3AEF /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4AA72B722146ED64006C3AEF /* AppDelegate.swift */; };
+		4AA72B752146ED64006C3AEF /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4AA72B742146ED64006C3AEF /* ViewController.swift */; };
+		4AA72B782146ED64006C3AEF /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 4AA72B762146ED64006C3AEF /* Main.storyboard */; };
+		4AA72B7A2146ED66006C3AEF /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 4AA72B792146ED66006C3AEF /* Assets.xcassets */; };
+		4AA72B7D2146ED66006C3AEF /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 4AA72B7B2146ED66006C3AEF /* LaunchScreen.storyboard */; };
+		4ADDE0CE2176600E00FF07A2 /* Array+TensorFlowLite.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4ADDE0CD2176600900FF07A2 /* Array+TensorFlowLite.swift */; };
+/* End PBXBuildFile section */
+
+/* Begin PBXFileReference section */
+		4A7304B321500B8300C90B21 /* Data+TensorFlowLite.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "Data+TensorFlowLite.swift"; sourceTree = "<group>"; };
+		4AA72B6F2146ED64006C3AEF /* TensorFlowLiteApp.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = TensorFlowLiteApp.app; sourceTree = BUILT_PRODUCTS_DIR; };
+		4AA72B722146ED64006C3AEF /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = "<group>"; };
+		4AA72B742146ED64006C3AEF /* ViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewController.swift; sourceTree = "<group>"; };
+		4AA72B772146ED64006C3AEF /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = "<group>"; };
+		4AA72B792146ED66006C3AEF /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = "<group>"; };
+		4AA72B7C2146ED66006C3AEF /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = "<group>"; };
+		4AA72B7E2146ED66006C3AEF /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
+		4ADDE0CD2176600900FF07A2 /* Array+TensorFlowLite.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "Array+TensorFlowLite.swift"; sourceTree = "<group>"; };
+/* End PBXFileReference section */
+
+/* Begin PBXFrameworksBuildPhase section */
+		4AA72B6C2146ED64006C3AEF /* Frameworks */ = {
+			isa = PBXFrameworksBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXFrameworksBuildPhase section */
+
+/* Begin PBXGroup section */
+		4AA72B662146ED64006C3AEF = {
+			isa = PBXGroup;
+			children = (
+				4AA72B712146ED64006C3AEF /* TensorFlowLiteApp */,
+				4AA72B702146ED64006C3AEF /* Products */,
+			);
+			sourceTree = "<group>";
+		};
+		4AA72B702146ED64006C3AEF /* Products */ = {
+			isa = PBXGroup;
+			children = (
+				4AA72B6F2146ED64006C3AEF /* TensorFlowLiteApp.app */,
+			);
+			name = Products;
+			sourceTree = "<group>";
+		};
+		4AA72B712146ED64006C3AEF /* TensorFlowLiteApp */ = {
+			isa = PBXGroup;
+			children = (
+				4AA72B722146ED64006C3AEF /* AppDelegate.swift */,
+				4ADDE0CD2176600900FF07A2 /* Array+TensorFlowLite.swift */,
+				4A7304B321500B8300C90B21 /* Data+TensorFlowLite.swift */,
+				4AA72B742146ED64006C3AEF /* ViewController.swift */,
+				4AA72B762146ED64006C3AEF /* Main.storyboard */,
+				4AA72B792146ED66006C3AEF /* Assets.xcassets */,
+				4AA72B7B2146ED66006C3AEF /* LaunchScreen.storyboard */,
+				4AA72B7E2146ED66006C3AEF /* Info.plist */,
+			);
+			path = TensorFlowLiteApp;
+			sourceTree = "<group>";
+		};
+/* End PBXGroup section */
+
+/* Begin PBXNativeTarget section */
+		4AA72B6E2146ED64006C3AEF /* TensorFlowLiteApp */ = {
+			isa = PBXNativeTarget;
+			buildConfigurationList = 4AA72B812146ED66006C3AEF /* Build configuration list for PBXNativeTarget "TensorFlowLiteApp" */;
+			buildPhases = (
+				4AA72B6B2146ED64006C3AEF /* Sources */,
+				4AA72B6C2146ED64006C3AEF /* Frameworks */,
+				4AA72B6D2146ED64006C3AEF /* Resources */,
+			);
+			buildRules = (
+			);
+			dependencies = (
+			);
+			name = TensorFlowLiteApp;
+			productName = TensorFlowLiteApp;
+			productReference = 4AA72B6F2146ED64006C3AEF /* TensorFlowLiteApp.app */;
+			productType = "com.apple.product-type.application";
+		};
+/* End PBXNativeTarget section */
+
+/* Begin PBXProject section */
+		4AA72B672146ED64006C3AEF /* Project object */ = {
+			isa = PBXProject;
+			attributes = {
+				LastSwiftUpdateCheck = 0940;
+				LastUpgradeCheck = 0940;
+				ORGANIZATIONNAME = Google;
+				TargetAttributes = {
+					4AA72B6E2146ED64006C3AEF = {
+						CreatedOnToolsVersion = 9.4.1;
+					};
+				};
+			};
+			buildConfigurationList = 4AA72B6A2146ED64006C3AEF /* Build configuration list for PBXProject "TensorFlowLiteApp" */;
+			compatibilityVersion = "Xcode 9.3";
+			developmentRegion = en;
+			hasScannedForEncodings = 0;
+			knownRegions = (
+				en,
+				Base,
+			);
+			mainGroup = 4AA72B662146ED64006C3AEF;
+			productRefGroup = 4AA72B702146ED64006C3AEF /* Products */;
+			projectDirPath = "";
+			projectRoot = "";
+			targets = (
+				4AA72B6E2146ED64006C3AEF /* TensorFlowLiteApp */,
+			);
+		};
+/* End PBXProject section */
+
+/* Begin PBXResourcesBuildPhase section */
+		4AA72B6D2146ED64006C3AEF /* Resources */ = {
+			isa = PBXResourcesBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+				4AA72B7D2146ED66006C3AEF /* LaunchScreen.storyboard in Resources */,
+				4AA72B7A2146ED66006C3AEF /* Assets.xcassets in Resources */,
+				4AA72B782146ED64006C3AEF /* Main.storyboard in Resources */,
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXResourcesBuildPhase section */
+
+/* Begin PBXSourcesBuildPhase section */
+		4AA72B6B2146ED64006C3AEF /* Sources */ = {
+			isa = PBXSourcesBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+				4AA72B732146ED64006C3AEF /* AppDelegate.swift in Sources */,
+				4ADDE0CE2176600E00FF07A2 /* Array+TensorFlowLite.swift in Sources */,
+				4A7304B421500B8400C90B21 /* Data+TensorFlowLite.swift in Sources */,
+				4AA72B752146ED64006C3AEF /* ViewController.swift in Sources */,
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXSourcesBuildPhase section */
+
+/* Begin PBXVariantGroup section */
+		4AA72B762146ED64006C3AEF /* Main.storyboard */ = {
+			isa = PBXVariantGroup;
+			children = (
+				4AA72B772146ED64006C3AEF /* Base */,
+			);
+			name = Main.storyboard;
+			sourceTree = "<group>";
+		};
+		4AA72B7B2146ED66006C3AEF /* LaunchScreen.storyboard */ = {
+			isa = PBXVariantGroup;
+			children = (
+				4AA72B7C2146ED66006C3AEF /* Base */,
+			);
+			name = LaunchScreen.storyboard;
+			sourceTree = "<group>";
+		};
+/* End PBXVariantGroup section */
+
+/* Begin XCBuildConfiguration section */
+		4AA72B7F2146ED66006C3AEF /* Debug */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ALWAYS_SEARCH_USER_PATHS = NO;
+				CLANG_ANALYZER_NONNULL = YES;
+				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
+				CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
+				CLANG_CXX_LIBRARY = "libc++";
+				CLANG_ENABLE_MODULES = YES;
+				CLANG_ENABLE_OBJC_ARC = YES;
+				CLANG_ENABLE_OBJC_WEAK = YES;
+				CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
+				CLANG_WARN_BOOL_CONVERSION = YES;
+				CLANG_WARN_COMMA = YES;
+				CLANG_WARN_CONSTANT_CONVERSION = YES;
+				CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
+				CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
+				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
+				CLANG_WARN_EMPTY_BODY = YES;
+				CLANG_WARN_ENUM_CONVERSION = YES;
+				CLANG_WARN_INFINITE_RECURSION = YES;
+				CLANG_WARN_INT_CONVERSION = YES;
+				CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
+				CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
+				CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
+				CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
+				CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
+				CLANG_WARN_STRICT_PROTOTYPES = YES;
+				CLANG_WARN_SUSPICIOUS_MOVE = YES;
+				CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
+				CLANG_WARN_UNREACHABLE_CODE = YES;
+				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
+				CODE_SIGN_IDENTITY = "iPhone Developer";
+				COPY_PHASE_STRIP = NO;
+				DEBUG_INFORMATION_FORMAT = dwarf;
+				ENABLE_STRICT_OBJC_MSGSEND = YES;
+				ENABLE_TESTABILITY = YES;
+				GCC_C_LANGUAGE_STANDARD = gnu11;
+				GCC_DYNAMIC_NO_PIC = NO;
+				GCC_NO_COMMON_BLOCKS = YES;
+				GCC_OPTIMIZATION_LEVEL = 0;
+				GCC_PREPROCESSOR_DEFINITIONS = (
+					"DEBUG=1",
+					"$(inherited)",
+				);
+				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
+				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
+				GCC_WARN_UNDECLARED_SELECTOR = YES;
+				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
+				GCC_WARN_UNUSED_FUNCTION = YES;
+				GCC_WARN_UNUSED_VARIABLE = YES;
+				IPHONEOS_DEPLOYMENT_TARGET = 11.4;
+				MTL_ENABLE_DEBUG_INFO = YES;
+				ONLY_ACTIVE_ARCH = YES;
+				SDKROOT = iphoneos;
+				SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;
+				SWIFT_OPTIMIZATION_LEVEL = "-Onone";
+			};
+			name = Debug;
+		};
+		4AA72B802146ED66006C3AEF /* Release */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ALWAYS_SEARCH_USER_PATHS = NO;
+				CLANG_ANALYZER_NONNULL = YES;
+				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
+				CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
+				CLANG_CXX_LIBRARY = "libc++";
+				CLANG_ENABLE_MODULES = YES;
+				CLANG_ENABLE_OBJC_ARC = YES;
+				CLANG_ENABLE_OBJC_WEAK = YES;
+				CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
+				CLANG_WARN_BOOL_CONVERSION = YES;
+				CLANG_WARN_COMMA = YES;
+				CLANG_WARN_CONSTANT_CONVERSION = YES;
+				CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
+				CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
+				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
+				CLANG_WARN_EMPTY_BODY = YES;
+				CLANG_WARN_ENUM_CONVERSION = YES;
+				CLANG_WARN_INFINITE_RECURSION = YES;
+				CLANG_WARN_INT_CONVERSION = YES;
+				CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
+				CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
+				CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
+				CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
+				CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
+				CLANG_WARN_STRICT_PROTOTYPES = YES;
+				CLANG_WARN_SUSPICIOUS_MOVE = YES;
+				CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
+				CLANG_WARN_UNREACHABLE_CODE = YES;
+				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
+				CODE_SIGN_IDENTITY = "iPhone Developer";
+				COPY_PHASE_STRIP = NO;
+				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
+				ENABLE_NS_ASSERTIONS = NO;
+				ENABLE_STRICT_OBJC_MSGSEND = YES;
+				GCC_C_LANGUAGE_STANDARD = gnu11;
+				GCC_NO_COMMON_BLOCKS = YES;
+				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
+				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
+				GCC_WARN_UNDECLARED_SELECTOR = YES;
+				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
+				GCC_WARN_UNUSED_FUNCTION = YES;
+				GCC_WARN_UNUSED_VARIABLE = YES;
+				IPHONEOS_DEPLOYMENT_TARGET = 11.4;
+				MTL_ENABLE_DEBUG_INFO = NO;
+				SDKROOT = iphoneos;
+				SWIFT_COMPILATION_MODE = wholemodule;
+				SWIFT_OPTIMIZATION_LEVEL = "-O";
+				VALIDATE_PRODUCT = YES;
+			};
+			name = Release;
+		};
+		4AA72B822146ED66006C3AEF /* Debug */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
+				CODE_SIGN_STYLE = Automatic;
+				INFOPLIST_FILE = TensorFlowLiteApp/Info.plist;
+				LD_RUNPATH_SEARCH_PATHS = (
+					"$(inherited)",
+					"@executable_path/Frameworks",
+				);
+				PRODUCT_BUNDLE_IDENTIFIER = com.tensorflow.lite.swift.TensorFlowLite;
+				PRODUCT_NAME = "$(TARGET_NAME)";
+				SWIFT_VERSION = 4.0;
+				TARGETED_DEVICE_FAMILY = "1,2";
+			};
+			name = Debug;
+		};
+		4AA72B832146ED66006C3AEF /* Release */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
+				CODE_SIGN_STYLE = Automatic;
+				INFOPLIST_FILE = TensorFlowLiteApp/Info.plist;
+				LD_RUNPATH_SEARCH_PATHS = (
+					"$(inherited)",
+					"@executable_path/Frameworks",
+				);
+				PRODUCT_BUNDLE_IDENTIFIER = com.tensorflow.lite.swift.TensorFlowLite;
+				PRODUCT_NAME = "$(TARGET_NAME)";
+				SWIFT_VERSION = 4.0;
+				TARGETED_DEVICE_FAMILY = "1,2";
+			};
+			name = Release;
+		};
+/* End XCBuildConfiguration section */
+
+/* Begin XCConfigurationList section */
+		4AA72B6A2146ED64006C3AEF /* Build configuration list for PBXProject "TensorFlowLiteApp" */ = {
+			isa = XCConfigurationList;
+			buildConfigurations = (
+				4AA72B7F2146ED66006C3AEF /* Debug */,
+				4AA72B802146ED66006C3AEF /* Release */,
+			);
+			defaultConfigurationIsVisible = 0;
+			defaultConfigurationName = Release;
+		};
+		4AA72B812146ED66006C3AEF /* Build configuration list for PBXNativeTarget "TensorFlowLiteApp" */ = {
+			isa = XCConfigurationList;
+			buildConfigurations = (
+				4AA72B822146ED66006C3AEF /* Debug */,
+				4AA72B832146ED66006C3AEF /* Release */,
+			);
+			defaultConfigurationIsVisible = 0;
+			defaultConfigurationName = Release;
+		};
+/* End XCConfigurationList section */
+	};
+	rootObject = 4AA72B672146ED64006C3AEF /* Project object */;
+}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift
new file mode 100644
index 0000000000..ffa90a06ad
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift
@@ -0,0 +1,24 @@
+import UIKit
+
+@UIApplicationMain
+
+final class AppDelegate: UIResponder, UIApplicationDelegate {
+
+  /// The main window of the app.
+  var window: UIWindow?
+
+  func application(
+    _ application: UIApplication,
+    didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? = nil
+  ) -> Bool {
+    return true
+  }
+}
+
+// MARK: - Extensions
+
+#if !swift(>=4.2)
+extension UIApplication {
+  typealias LaunchOptionsKey = UIApplicationLaunchOptionsKey
+}
+#endif  // !swift(>=4.2)
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift
new file mode 100644
index 0000000000..56df1ce659
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift
@@ -0,0 +1,22 @@
+import Foundation
+
+extension Array {
+  /// Creates a new array from the bytes of the given unsafe data.
+  ///
+  /// - Warning: The array's `Element` type must be trivial in that it can be copied bit for bit
+  ///     with no indirection or reference-counting operations; otherwise, copying the raw bytes in
+  ///     the `unsafeData`'s buffer to a new array returns an unsafe copy.
+  /// - Note: Returns `nil` if `unsafeData.count` is not a multiple of
+  ///     `MemoryLayout<Element>.stride`.
+  /// - Parameter unsafeData: The data containing the bytes to turn into an array.
+  init?(unsafeData: Data) {
+    guard unsafeData.count % MemoryLayout<Element>.stride == 0 else { return nil }
+    let elements = unsafeData.withUnsafeBytes {
+      UnsafeBufferPointer<Element>(
+        start: $0,
+        count: unsafeData.count / MemoryLayout<Element>.stride
+      )
+    }
+    self.init(elements)
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json
new file mode 100644
index 0000000000..d8db8d65fd
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json
@@ -0,0 +1,98 @@
+{
+  "images" : [
+    {
+      "idiom" : "iphone",
+      "size" : "20x20",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "20x20",
+      "scale" : "3x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "29x29",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "29x29",
+      "scale" : "3x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "40x40",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "40x40",
+      "scale" : "3x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "60x60",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "60x60",
+      "scale" : "3x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "20x20",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "20x20",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "29x29",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "29x29",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "40x40",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "40x40",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "76x76",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "76x76",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "83.5x83.5",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ios-marketing",
+      "size" : "1024x1024",
+      "scale" : "1x"
+    }
+  ],
+  "info" : {
+    "version" : 1,
+    "author" : "xcode"
+  }
+}
\ No newline at end of file
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json
new file mode 100644
index 0000000000..da4a164c91
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json
@@ -0,0 +1,6 @@
+{
+  "info" : {
+    "version" : 1,
+    "author" : "xcode"
+  }
+}
\ No newline at end of file
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard
new file mode 100644
index 0000000000..a07a1321be
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="14109" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" launchScreen="YES" useTraitCollections="YES" colorMatched="YES" initialViewController="01J-lp-oVM">
+    <device id="retina4_7" orientation="portrait">
+        <adaptation id="fullscreen"/>
+    </device>
+    <dependencies>
+        <deployment identifier="iOS"/>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="14088"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--View Controller-->
+        <scene sceneID="EHf-IW-A2E">
+            <objects>
+                <viewController id="01J-lp-oVM" sceneMemberID="viewController">
+                    <layoutGuides>
+                        <viewControllerLayoutGuide type="top" id="Llm-lL-Icb"/>
+                        <viewControllerLayoutGuide type="bottom" id="xb3-aO-Qok"/>
+                    </layoutGuides>
+                    <view key="view" contentMode="scaleToFill" id="Ze5-6b-2t3">
+                        <rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <subviews>
+                            <label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" text="TensorFlowLite" textAlignment="center" lineBreakMode="tailTruncation" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="3Gq-PV-hia">
+                                <rect key="frame" x="16" y="315" width="343" height="38.5"/>
+                                <fontDescription key="fontDescription" type="boldSystem" pointSize="32"/>
+                                <nil key="textColor"/>
+                                <nil key="highlightedColor"/>
+                            </label>
+                        </subviews>
+                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                        <constraints>
+                            <constraint firstItem="3Gq-PV-hia" firstAttribute="leading" secondItem="Ze5-6b-2t3" secondAttribute="leading" constant="16" id="aXL-9T-5Pf"/>
+                            <constraint firstItem="3Gq-PV-hia" firstAttribute="centerY" secondItem="Ze5-6b-2t3" secondAttribute="centerY" id="cDf-Go-1FR"/>
+                            <constraint firstAttribute="trailing" secondItem="3Gq-PV-hia" secondAttribute="trailing" constant="16" id="fB9-BX-A3B"/>
+                        </constraints>
+                    </view>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="iYj-Kq-Ea1" userLabel="First Responder" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="52" y="374.66266866566718"/>
+        </scene>
+    </scenes>
+</document>
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
new file mode 100644
index 0000000000..d0e91d63c4
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
@@ -0,0 +1,95 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="14313.18" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" colorMatched="YES" initialViewController="BYZ-38-t0r">
+    <device id="retina4_7" orientation="portrait">
+        <adaptation id="fullscreen"/>
+    </device>
+    <dependencies>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="14283.14"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--View Controller-->
+        <scene sceneID="tne-QT-ifu">
+            <objects>
+                <viewController storyboardIdentifier="viewController" useStoryboardIdentifierAsRestorationIdentifier="YES" id="BYZ-38-t0r" customClass="ViewController" customModule="third_party_tensorflow_lite_experimental_swift_TensorFlowLiteAppLib" sceneMemberID="viewController">
+                    <layoutGuides>
+                        <viewControllerLayoutGuide type="top" id="y3c-jy-aDJ"/>
+                        <viewControllerLayoutGuide type="bottom" id="wfy-db-euE"/>
+                    </layoutGuides>
+                    <view key="view" contentMode="scaleToFill" id="8bC-Xf-vdC">
+                        <rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <subviews>
+                            <textView clipsSubviews="YES" multipleTouchEnabled="YES" contentMode="scaleToFill" editable="NO" selectable="NO" translatesAutoresizingMaskIntoConstraints="NO" id="7Mj-sL-hrd">
+                                <rect key="frame" x="0.0" y="367" width="375" height="300"/>
+                                <color key="backgroundColor" red="0.0" green="0.47843137250000001" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                                <constraints>
+                                    <constraint firstAttribute="height" constant="300" id="YUb-MC-D5w"/>
+                                </constraints>
+                                <color key="textColor" cocoaTouchSystemColor="tableCellGroupedBackgroundColor"/>
+                                <fontDescription key="fontDescription" type="system" pointSize="14"/>
+                                <textInputTraits key="textInputTraits" autocapitalizationType="sentences"/>
+                            </textView>
+                            <toolbar opaque="NO" clearsContextBeforeDrawing="NO" contentMode="scaleToFill" translatesAutoresizingMaskIntoConstraints="NO" id="Qwg-EP-bd6" userLabel="Bottom Toolbar">
+                                <rect key="frame" x="0.0" y="323" width="375" height="44"/>
+                                <constraints>
+                                    <constraint firstAttribute="height" constant="44" id="jhT-Q0-E9N"/>
+                                </constraints>
+                                <items>
+                                    <barButtonItem style="plain" systemItem="flexibleSpace" id="P3q-uA-YUa"/>
+                                    <barButtonItem title="Invoke Interpreter" id="A4J-Mg-nmd" userLabel="Invoke Button">
+                                        <connections>
+                                            <action selector="invokeInterpreter:" destination="BYZ-38-t0r" id="lZU-x7-PsJ"/>
+                                        </connections>
+                                    </barButtonItem>
+                                    <barButtonItem style="plain" systemItem="flexibleSpace" id="Qad-Pa-ySg"/>
+                                </items>
+                            </toolbar>
+                            <toolbar opaque="NO" clearsContextBeforeDrawing="NO" contentMode="scaleToFill" translatesAutoresizingMaskIntoConstraints="NO" id="Gkb-TR-PCB" userLabel="Top Toolbar">
+                                <rect key="frame" x="0.0" y="28" width="375" height="44"/>
+                                <constraints>
+                                    <constraint firstAttribute="height" constant="44" id="hSD-2q-fUE"/>
+                                </constraints>
+                                <items>
+                                    <barButtonItem style="plain" id="LKw-TX-bbH">
+                                        <segmentedControl key="customView" opaque="NO" contentMode="scaleToFill" contentHorizontalAlignment="left" contentVerticalAlignment="top" segmentControlStyle="bar" selectedSegmentIndex="0" id="rhA-nW-xzT">
+                                            <rect key="frame" x="16" y="7" width="343" height="30"/>
+                                            <autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
+                                            <segments>
+                                                <segment title="Add"/>
+                                                <segment title="AddQuantized"/>
+                                                <segment title="MultiAdd"/>
+                                            </segments>
+                                            <connections>
+                                                <action selector="modelChanged:" destination="BYZ-38-t0r" eventType="valueChanged" id="YnG-Ov-B5D"/>
+                                            </connections>
+                                        </segmentedControl>
+                                    </barButtonItem>
+                                </items>
+                            </toolbar>
+                        </subviews>
+                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                        <constraints>
+                            <constraint firstAttribute="trailing" secondItem="Gkb-TR-PCB" secondAttribute="trailing" id="4Cr-Sf-I7n"/>
+                            <constraint firstItem="7Mj-sL-hrd" firstAttribute="bottom" secondItem="wfy-db-euE" secondAttribute="top" id="6ot-zD-sze"/>
+                            <constraint firstItem="7Mj-sL-hrd" firstAttribute="top" secondItem="Qwg-EP-bd6" secondAttribute="bottom" id="ELA-C6-NiG"/>
+                            <constraint firstAttribute="trailing" secondItem="7Mj-sL-hrd" secondAttribute="trailing" id="HDO-xr-mBl"/>
+                            <constraint firstItem="Gkb-TR-PCB" firstAttribute="leading" secondItem="8bC-Xf-vdC" secondAttribute="leading" id="Kmo-6K-gS4"/>
+                            <constraint firstItem="Qwg-EP-bd6" firstAttribute="leading" secondItem="8bC-Xf-vdC" secondAttribute="leading" id="hGu-lm-fMG"/>
+                            <constraint firstAttribute="trailing" secondItem="Qwg-EP-bd6" secondAttribute="trailing" id="iXR-LK-nTO"/>
+                            <constraint firstItem="7Mj-sL-hrd" firstAttribute="leading" secondItem="8bC-Xf-vdC" secondAttribute="leading" id="nr7-jW-ZYf"/>
+                            <constraint firstItem="Gkb-TR-PCB" firstAttribute="top" secondItem="y3c-jy-aDJ" secondAttribute="bottom" constant="8" id="uCF-VW-rR0"/>
+                        </constraints>
+                    </view>
+                    <connections>
+                        <outlet property="invokeButton" destination="A4J-Mg-nmd" id="UxZ-Ft-E45"/>
+                        <outlet property="modelControl" destination="rhA-nW-xzT" id="KKf-TT-BQ2"/>
+                        <outlet property="resultsTextView" destination="7Mj-sL-hrd" id="T4I-z4-tYA"/>
+                    </connections>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="dkx-z0-nzr" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="125.59999999999999" y="133.5832083958021"/>
+        </scene>
+    </scenes>
+</document>
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift
new file mode 100644
index 0000000000..bc8a70c848
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift
@@ -0,0 +1,13 @@
+import Foundation
+
+extension Data {
+  /// Creates a new buffer by copying the buffer pointer of the given array.
+  ///
+  /// - Warning: The given array's element type `T` must be trivial in that it can be copied bit
+  ///     for bit with no indirection or reference-counting operations; otherwise, reinterpreting
+  ///     data from the resulting buffer has undefined behavior.
+  /// - Parameter array: An array with elements of type `T`.
+  init<T>(copyingBufferOf array: [T]) {
+    self = array.withUnsafeBufferPointer(Data.init)
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist
new file mode 100644
index 0000000000..3ca3875f04
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>CFBundleDevelopmentRegion</key>
+	<string>en</string>
+	<key>CFBundleExecutable</key>
+	<string>$(EXECUTABLE_NAME)</string>
+	<key>CFBundleIdentifier</key>
+	<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
+	<key>CFBundleInfoDictionaryVersion</key>
+	<string>6.0</string>
+	<key>CFBundleName</key>
+	<string>$(PRODUCT_NAME)</string>
+	<key>CFBundlePackageType</key>
+	<string>APPL</string>
+	<key>CFBundleShortVersionString</key>
+	<string>1.0</string>
+	<key>CFBundleVersion</key>
+	<string>0.0.1</string>
+	<key>LSRequiresIPhoneOS</key>
+	<true/>
+	<key>NSCameraUsageDescription</key>
+	<string>NSCameraUsageDescription</string>
+	<key>NSPhotoLibraryUsageDescription</key>
+	<string>Select a photo to detect objects in.</string>
+	<key>UILaunchStoryboardName</key>
+	<string>LaunchScreen</string>
+	<key>UIMainStoryboardFile</key>
+	<string>Main</string>
+	<key>UIRequiredDeviceCapabilities</key>
+	<array>
+		<string>armv7</string>
+	</array>
+	<key>UISupportedInterfaceOrientations</key>
+	<array>
+		<string>UIInterfaceOrientationPortrait</string>
+		<string>UIInterfaceOrientationPortraitUpsideDown</string>
+	</array>
+	<key>UISupportedInterfaceOrientations~ipad</key>
+	<array>
+		<string>UIInterfaceOrientationPortrait</string>
+		<string>UIInterfaceOrientationPortraitUpsideDown</string>
+	</array>
+</dict>
+</plist>
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift
new file mode 100644
index 0000000000..73c74fd19c
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift
@@ -0,0 +1,299 @@
+import TensorFlowLite
+import UIKit
+
+class ViewController: UIViewController {
+
+  // MARK: - Properties
+
+  /// TensorFlowLite interpreter object for performing inference from a given model.
+  private var interpreter: Interpreter?
+
+  /// Serial dispatch queue for managing `Interpreter` calls.
+  private let interpreterQueue = DispatchQueue(
+    label: Constant.dispatchQueueLabel,
+    qos: .userInitiated
+  )
+
+  /// The currently selected model.
+  private var currentModel: Model {
+    guard let currentModel = Model(rawValue: modelControl.selectedSegmentIndex) else {
+      preconditionFailure("Invalid model for selected segment index.")
+    }
+    return currentModel
+  }
+
+  /// A description of the current model.
+  private var modelDescription: String {
+    guard let interpreter = interpreter else { return "" }
+    let inputCount = interpreter.inputTensorCount
+    let outputCount = interpreter.outputTensorCount
+    let inputTensors = (0..<inputCount).map { index in
+      var tensorInfo = "  Input \(index + 1): "
+      do {
+        let tensor = try interpreter.input(at: index)
+        tensorInfo += "\(tensor)"
+      } catch let error {
+        tensorInfo += "\(error.localizedDescription)"
+      }
+      return tensorInfo
+    }.joined(separator: "\n")
+    let outputTensors = (0..<outputCount).map { index in
+      var tensorInfo = "  Output \(index + 1): "
+      do {
+        let tensor = try interpreter.output(at: index)
+        tensorInfo += "\(tensor)"
+      } catch let error {
+        tensorInfo += "\(error.localizedDescription)"
+      }
+      return tensorInfo
+    }.joined(separator: "\n")
+    return "Model Description:\n" +
+             "  Input Tensor Count = \(inputCount)\n\(inputTensors)\n\n" +
+             "  Output Tensor Count = \(outputCount)\n\(outputTensors)"
+  }
+
+  // MARK: - IBOutlets
+
+  /// A segmented control for changing models. See the `Model` enum for available models.
+  @IBOutlet private var modelControl: UISegmentedControl!
+
+  @IBOutlet private var resultsTextView: UITextView!
+  @IBOutlet private var invokeButton: UIBarButtonItem!
+
+  // MARK: - UIViewController
+
+  override func viewDidLoad() {
+    super.viewDidLoad()
+
+    invokeButton.isEnabled = false
+    loadModel()
+  }
+
+  // MARK: - IBActions
+
+  @IBAction func modelChanged(_ sender: Any) {
+    invokeButton.isEnabled = false
+    updateResultsText("Switched to the \(currentModel.description) model.")
+    loadModel()
+  }
+
+  @IBAction func invokeInterpreter(_ sender: Any) {
+    switch currentModel {
+    case .add:
+      invokeAdd()
+    case .addQuantized:
+      invokeAddQuantized()
+    case .multiAdd:
+      invokeMultiAdd()
+    }
+  }
+
+  // MARK: - Private
+
+  private func loadModel() {
+    let fileInfo = currentModel.fileInfo
+    guard let modelPath = Bundle.main.path(forResource: fileInfo.name, ofType: fileInfo.extension)
+    else {
+      updateResultsText("Failed to load the \(currentModel.description) model.")
+      return
+    }
+    setUpInterpreter(withModelPath: modelPath)
+  }
+
+  private func setUpInterpreter(withModelPath modelPath: String) {
+    interpreterQueue.async {
+      do {
+        var options = InterpreterOptions()
+        options.isErrorLoggingEnabled = true
+        self.interpreter = try Interpreter(modelPath: modelPath, options: options)
+      } catch let error {
+        self.updateResultsText(
+          "Failed to create the interpreter with error: \(error.localizedDescription)"
+        )
+        return
+      }
+      safeDispatchOnMain { self.invokeButton.isEnabled = true }
+    }
+  }
+
+  private func invokeAdd() {
+    interpreterQueue.async {
+      guard let interpreter = self.interpreter else {
+        self.updateResultsText(Constant.nilInterpreterErrorMessage)
+        return
+      }
+      do {
+        try interpreter.resizeInput(at: 0, to: [2])
+        try interpreter.allocateTensors()
+        let input: [Float32] = [1, 3]
+        let resultsText = self.modelDescription + "\n\n" +
+          "Performing 2 add operations on input \(input.description) equals: "
+        self.updateResultsText(resultsText)
+        let data = Data(copyingBufferOf: input)
+        try interpreter.copy(data, toInputAt: 0)
+        try interpreter.invoke()
+        let outputTensor = try interpreter.output(at: 0)
+        let results: () -> String = {
+          guard let results = [Float32](unsafeData: outputTensor.data) else { return "No results." }
+          return resultsText + results.description
+        }
+        self.updateResultsText(results())
+      } catch let error {
+        self.updateResultsText(
+          "Failed to invoke the interpreter with error: \(error.localizedDescription)"
+        )
+        return
+      }
+    }
+  }
+
+  private func invokeAddQuantized() {
+    interpreterQueue.async {
+      guard let interpreter = self.interpreter else {
+        self.updateResultsText(Constant.nilInterpreterErrorMessage)
+        return
+      }
+      do {
+        try interpreter.resizeInput(at: 0, to: [2])
+        try interpreter.allocateTensors()
+        let input: [UInt8] = [1, 3]
+        let resultsText = self.modelDescription + "\n\n" +
+          "Performing 2 add operations on quantized input \(input.description) equals: "
+        self.updateResultsText(resultsText)
+        let data = Data(input)
+        try interpreter.copy(data, toInputAt: 0)
+        try interpreter.invoke()
+        let outputTensor = try interpreter.output(at: 0)
+        let results: () -> String = {
+          guard let quantizationParameters = outputTensor.quantizationParameters else {
+            return "No results."
+          }
+          let quantizedResults = [UInt8](outputTensor.data)
+          let dequantizedResults = quantizedResults.map {
+            quantizationParameters.scale * Float(Int($0) - quantizationParameters.zeroPoint)
+          }
+          return resultsText + quantizedResults.description +
+                   ", dequantized results: " + dequantizedResults.description
+        }
+        self.updateResultsText(results())
+      } catch let error {
+        self.updateResultsText(
+          "Failed to invoke the interpreter with error: \(error.localizedDescription)"
+        )
+        return
+      }
+    }
+  }
+
+  private func invokeMultiAdd() {
+    interpreterQueue.async {
+      guard let interpreter = self.interpreter else {
+        self.updateResultsText(Constant.nilInterpreterErrorMessage)
+        return
+      }
+      do {
+        let shape = TensorShape(2)
+        try (0..<interpreter.inputTensorCount).forEach { index in
+          try interpreter.resizeInput(at: index, to: shape)
+        }
+        try interpreter.allocateTensors()
+        let inputs = try (0..<interpreter.inputTensorCount).map { index -> [Float32] in
+          let input = [Float32(index + 1), Float32(index + 2)]
+          let data = Data(copyingBufferOf: input)
+          try interpreter.copy(data, toInputAt: index)
+          return input
+        }
+        let resultsText = self.modelDescription + "\n\n" +
+          "Performing 3 add operations on inputs \(inputs.description) equals: "
+        self.updateResultsText(resultsText)
+        try interpreter.invoke()
+        let results = try (0..<interpreter.outputTensorCount).map { index -> [Float32] in
+          let tensor = try interpreter.output(at: index)
+          return [Float32](unsafeData: tensor.data) ?? []
+        }
+        self.updateResultsText(resultsText + results.description)
+      } catch let error {
+        self.updateResultsText(
+          "Failed to invoke the interpreter with error: \(error.localizedDescription)"
+        )
+        return
+      }
+    }
+  }
+
+  private func updateResultsText(_ text: String? = nil) {
+    safeDispatchOnMain { self.resultsTextView.text = text }
+  }
+}
+
+// MARK: - Constants
+
+private enum Constant {
+  static let dispatchQueueLabel = "TensorFlowLiteInterpreterQueue"
+  static let nilInterpreterErrorMessage =
+    "Failed to invoke the interpreter because the interpreter was nil."
+}
+
+/// Models that can be loaded by the TensorFlow Lite `Interpreter`.
+private enum Model: Int, CustomStringConvertible {
+  /// A float model that performs two add operations on one input tensor and returns the result in
+  /// one output tensor.
+  case add = 0
+  /// A quantized model that performs two add operations on one input tensor and returns the result
+  /// in one output tensor.
+  case addQuantized = 1
+  /// A float model that performs three add operations on four input tensors and returns the results
+  /// in 2 output tensors.
+  case multiAdd = 2
+
+  var fileInfo: (name: String, extension: String) {
+    switch self {
+    case .add:
+      return Add.fileInfo
+    case .addQuantized:
+      return AddQuantized.fileInfo
+    case .multiAdd:
+      return MultiAdd.fileInfo
+    }
+  }
+
+  // MARK: - CustomStringConvertible
+
+  var description: String {
+    switch self {
+    case .add:
+      return Add.name
+    case .addQuantized:
+      return AddQuantized.name
+    case .multiAdd:
+      return MultiAdd.name
+    }
+  }
+}
+
+/// Values for the `Add` model.
+private enum Add {
+  static let name = "Add"
+  static let fileInfo = (name: "add", extension: "bin")
+}
+
+/// Values for the `AddQuantized` model.
+private enum AddQuantized {
+  static let name = "AddQuantized"
+  static let fileInfo = (name: "add_quantized", extension: "bin")
+}
+
+/// Values for the `MultiAdd` model.
+private enum MultiAdd {
+  static let name = "MultiAdd"
+  static let fileInfo = (name: "multi_add", extension: "bin")
+}
+
+// MARK: - Fileprivate
+
+/// Safely dispatches the given block on the main queue. If the current thread is `main`, the block
+/// is executed synchronously; otherwise, the block is executed asynchronously on the main thread.
+fileprivate func safeDispatchOnMain(_ block: @escaping () -> Void) {
+  if Thread.isMainThread { block(); return }
+  DispatchQueue.main.async { block() }
+}
diff --git a/tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift b/tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift
new file mode 100644
index 0000000000..54b4f59b28
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift
@@ -0,0 +1,54 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+@testable import TensorFlowLite
+import XCTest
+
+class InterpreterOptionsTests: XCTestCase {
+
+  func testInterpreterOptions_InitWithDefaultValues() {
+    let options = InterpreterOptions()
+    XCTAssertNil(options.threadCount)
+    XCTAssertFalse(options.isErrorLoggingEnabled)
+  }
+
+  func testInterpreterOptions_InitWithCustomValues() {
+    var options = InterpreterOptions()
+    options.threadCount = 2
+    XCTAssertEqual(options.threadCount, 2)
+    options.isErrorLoggingEnabled = true
+    XCTAssertTrue(options.isErrorLoggingEnabled)
+  }
+
+  func testInterpreterOptions_Equatable() {
+    var options1 = InterpreterOptions()
+    var options2 = InterpreterOptions()
+    XCTAssertEqual(options1, options2)
+
+    options1.threadCount = 2
+    options2.threadCount = 2
+    XCTAssertEqual(options1, options2)
+
+    options2.threadCount = 3
+    XCTAssertNotEqual(options1, options2)
+    options2.threadCount = 2
+
+    options1.isErrorLoggingEnabled = true
+    options2.isErrorLoggingEnabled = true
+    XCTAssertEqual(options1, options2)
+
+    options2.isErrorLoggingEnabled = false
+    XCTAssertNotEqual(options1, options2)
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift b/tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift
new file mode 100644
index 0000000000..e98da5f951
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift
@@ -0,0 +1,315 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+@testable import TensorFlowLite
+import XCTest
+
+class InterpreterTests: XCTestCase {
+
+  var interpreter: Interpreter!
+
+  override func setUp() {
+    super.setUp()
+
+    interpreter = try! Interpreter(modelPath: AddModel.path)
+  }
+
+  override func tearDown() {
+    interpreter = nil
+
+    super.tearDown()
+  }
+
+  func testInterpreter_InitWithModelPath() {
+    XCTAssertNoThrow(try Interpreter(modelPath: AddModel.path))
+  }
+
+  func testInterpreter_Init_ThrowsFailedToLoadModel() {
+    XCTAssertThrowsError(try Interpreter(modelPath: "/invalid/path")) { error in
+      self.assertEqualErrors(actual: error, expected: .failedToLoadModel)
+    }
+  }
+
+  func testInterpreter_InitWithModelPathAndOptions() {
+    var options = InterpreterOptions()
+    options.threadCount = 2
+    XCTAssertNoThrow(try Interpreter(modelPath: AddModel.path, options: options))
+  }
+
+  func testInterpreter_InputTensorCount() {
+    XCTAssertEqual(interpreter.inputTensorCount, AddModel.inputTensorCount)
+  }
+
+  func testInterpreter_OutputTensorCount() {
+    XCTAssertEqual(interpreter.outputTensorCount, AddModel.outputTensorCount)
+  }
+
+  func testInterpreter_Invoke() throws {
+    try interpreter.allocateTensors()
+    XCTAssertNoThrow(try interpreter.invoke())
+  }
+
+  func testInterpreter_Invoke_ThrowsAllocateTensorsRequired_ModelNotReady() {
+    XCTAssertThrowsError(try interpreter.invoke()) { error in
+      self.assertEqualErrors(actual: error, expected: .allocateTensorsRequired)
+    }
+  }
+
+  func testInterpreter_InputTensorAtIndex() throws {
+    try setUpAddModelInputTensor()
+    let inputTensor = try interpreter.input(at: AddModel.validIndex)
+    XCTAssertEqual(inputTensor, AddModel.inputTensor)
+  }
+
+  func testInterpreter_InputTensorAtIndex_QuantizedModel() throws {
+    interpreter = try Interpreter(modelPath: AddQuantizedModel.path)
+    try setUpAddQuantizedModelInputTensor()
+    let inputTensor = try interpreter.input(at: AddQuantizedModel.inputOutputIndex)
+    XCTAssertEqual(inputTensor, AddQuantizedModel.inputTensor)
+  }
+
+  func testInterpreter_InputTensorAtIndex_ThrowsInvalidIndex() throws {
+    try interpreter.allocateTensors()
+    XCTAssertThrowsError(try interpreter.input(at: AddModel.invalidIndex)) { error in
+      let maxIndex = AddModel.inputTensorCount - 1
+      self.assertEqualErrors(
+        actual: error,
+        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
+      )
+    }
+  }
+
+  func testInterpreter_InputTensorAtIndex_ThrowsAllocateTensorsRequired() {
+    XCTAssertThrowsError(try interpreter.input(at: AddModel.validIndex)) { error in
+      self.assertEqualErrors(actual: error, expected: .allocateTensorsRequired)
+    }
+  }
+
+  func testInterpreter_OutputTensorAtIndex() throws {
+    try setUpAddModelInputTensor()
+    try interpreter.invoke()
+    let outputTensor = try interpreter.output(at: AddModel.validIndex)
+    XCTAssertEqual(outputTensor, AddModel.outputTensor)
+    let expectedResults = [Float32](unsafeData: outputTensor.data)
+    XCTAssertEqual(expectedResults, AddModel.results)
+  }
+
+  func testInterpreter_OutputTensorAtIndex_QuantizedModel() throws {
+    interpreter = try Interpreter(modelPath: AddQuantizedModel.path)
+    try setUpAddQuantizedModelInputTensor()
+    try interpreter.invoke()
+    let outputTensor = try interpreter.output(at: AddQuantizedModel.inputOutputIndex)
+    XCTAssertEqual(outputTensor, AddQuantizedModel.outputTensor)
+    let expectedResults = [UInt8](outputTensor.data)
+    XCTAssertEqual(expectedResults, AddQuantizedModel.results)
+  }
+
+  func testInterpreter_OutputTensorAtIndex_ThrowsInvalidIndex() throws {
+    try interpreter.allocateTensors()
+    try interpreter.invoke()
+    XCTAssertThrowsError(try interpreter.output(at: AddModel.invalidIndex)) { error in
+      let maxIndex = AddModel.outputTensorCount - 1
+      self.assertEqualErrors(
+        actual: error,
+        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
+      )
+    }
+  }
+
+  func testInterpreter_OutputTensorAtIndex_ThrowsInvokeInterpreterRequired() {
+    XCTAssertThrowsError(try interpreter.output(at: AddModel.validIndex)) { error in
+      self.assertEqualErrors(actual: error, expected: .invokeInterpreterRequired)
+    }
+  }
+
+  func testInterpreter_ResizeInputTensorAtIndexToShape() {
+    XCTAssertNoThrow(try interpreter.resizeInput(at: AddModel.validIndex, to: [2, 2, 3]))
+    XCTAssertNoThrow(try interpreter.allocateTensors())
+  }
+
+  func testInterpreter_ResizeInputTensorAtIndexToShape_ThrowsInvalidIndex() {
+    XCTAssertThrowsError(try interpreter.resizeInput(
+      at: AddModel.invalidIndex,
+      to: [2, 2, 3]
+    )) { error in
+      let maxIndex = AddModel.inputTensorCount - 1
+      self.assertEqualErrors(
+        actual: error,
+        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
+      )
+    }
+  }
+
+  func testInterpreter_CopyDataToInputTensorAtIndex() throws {
+    try interpreter.resizeInput(at: AddModel.validIndex, to: AddModel.shape)
+    try interpreter.allocateTensors()
+    let inputTensor = try interpreter.copy(AddModel.inputData, toInputAt: AddModel.validIndex)
+    XCTAssertEqual(inputTensor.data, AddModel.inputData)
+  }
+
+  func testInterpreter_CopyDataToInputTensorAtIndex_ThrowsInvalidIndex() {
+    XCTAssertThrowsError(try interpreter.copy(
+      AddModel.inputData,
+      toInputAt: AddModel.invalidIndex
+    )) { error in
+      let maxIndex = AddModel.inputTensorCount - 1
+      self.assertEqualErrors(
+        actual: error,
+        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
+      )
+    }
+  }
+
+  func testInterpreter_CopyDataToInputTensorAtIndex_ThrowsInvalidDataCount() throws {
+    try interpreter.resizeInput(at: AddModel.validIndex, to: AddModel.shape)
+    try interpreter.allocateTensors()
+    let invalidData = Data(count: AddModel.dataCount - 1)
+    XCTAssertThrowsError(try interpreter.copy(
+      invalidData,
+      toInputAt: AddModel.validIndex
+    )) { error in
+      self.assertEqualErrors(
+        actual: error,
+        expected: .invalidTensorDataCount(provided: invalidData.count, required: AddModel.dataCount)
+      )
+    }
+  }
+
+  func testInterpreter_AllocateTensors() {
+    XCTAssertNoThrow(try interpreter.allocateTensors())
+  }
+
+  // MARK: - Private
+
+  private func setUpAddModelInputTensor() throws {
+    precondition(interpreter != nil)
+    try interpreter.resizeInput(at: AddModel.validIndex, to: AddModel.shape)
+    try interpreter.allocateTensors()
+    try interpreter.copy(AddModel.inputData, toInputAt: AddModel.validIndex)
+  }
+
+  private func setUpAddQuantizedModelInputTensor() throws {
+    precondition(interpreter != nil)
+    try interpreter.resizeInput(at: AddQuantizedModel.inputOutputIndex, to: AddQuantizedModel.shape)
+    try interpreter.allocateTensors()
+    try interpreter.copy(AddQuantizedModel.inputData, toInputAt: AddQuantizedModel.inputOutputIndex)
+  }
+
+  private func assertEqualErrors(actual: Error, expected: InterpreterError) {
+    guard let actual = actual as? InterpreterError else {
+      XCTFail("Actual error should be of type InterpreterError.")
+      return
+    }
+    XCTAssertEqual(actual, expected)
+  }
+}
+
+// MARK: - Constants
+
+/// Values for the `add.bin` model.
+private enum AddModel {
+  static let info = (name: "add", extension: "bin")
+  static let inputTensorCount = 1
+  static let outputTensorCount = 1
+  static let invalidIndex = 1
+  static let validIndex = 0
+  static let shape: TensorShape = [2]
+  static let dataCount = inputData.count
+  static let inputData = Data(copyingBufferOf: [Float32(1.0), Float32(3.0)])
+  static let outputData = Data(copyingBufferOf: [Float32(3.0), Float32(9.0)])
+  static let results = [Float32(3.0), Float32(9.0)]
+
+  static let inputTensor = Tensor(
+    name: "input",
+    dataType: .float32,
+    shape: shape,
+    data: inputData
+  )
+  static let outputTensor = Tensor(
+    name: "output",
+    dataType: .float32,
+    shape: shape,
+    data: outputData
+  )
+
+  static var path: String = {
+    let bundle = Bundle(for: InterpreterTests.self)
+    guard let path = bundle.path(forResource: info.name, ofType: info.extension) else { return "" }
+    return path
+  }()
+}
+
+/// Values for the `add_quantized.bin` model.
+private enum AddQuantizedModel {
+  static let info = (name: "add_quantized", extension: "bin")
+  static let inputOutputIndex = 0
+  static let shape: TensorShape = [2]
+  static let inputData = Data([1, 3])
+  static let outputData = Data([3, 9])
+  static let quantizationParameters = QuantizationParameters(scale: 0.003922, zeroPoint: 0)
+  static let results: [UInt8] = [3, 9]
+
+  static let inputTensor = Tensor(
+    name: "input",
+    dataType: .uInt8,
+    shape: shape,
+    data: inputData,
+    quantizationParameters: quantizationParameters
+  )
+  static let outputTensor = Tensor(
+    name: "output",
+    dataType: .uInt8,
+    shape: shape,
+    data: outputData,
+    quantizationParameters: quantizationParameters
+  )
+
+  static var path: String = {
+    let bundle = Bundle(for: InterpreterTests.self)
+    guard let path = bundle.path(forResource: info.name, ofType: info.extension) else { return "" }
+    return path
+  }()
+}
+
+// MARK: - Extensions
+
+extension Array {
+  /// Creates a new array from the bytes of the given unsafe data.
+  ///
+  /// - Note: Returns `nil` if `unsafeData.count` is not a multiple of
+  ///     `MemoryLayout<Element>.stride`.
+  /// - Parameter unsafeData: The data containing the bytes to turn into an array.
+  init?(unsafeData: Data) {
+    guard unsafeData.count % MemoryLayout<Element>.stride == 0 else { return nil }
+    let elements = unsafeData.withUnsafeBytes {
+      UnsafeBufferPointer<Element>(
+        start: $0,
+        count: unsafeData.count / MemoryLayout<Element>.stride
+      )
+    }
+    self.init(elements)
+  }
+}
+
+extension Data {
+  /// Creates a new buffer by copying the buffer pointer of the given array.
+  ///
+  /// - Warning: The given array's element type `T` must be trivial in that it can be copied bit
+  ///     for bit with no indirection or reference-counting operations; otherwise, reinterpreting
+  ///     data from the resulting buffer has undefined behavior.
+  /// - Parameter array: An array with elements of type `T`.
+  init<T>(copyingBufferOf array: [T]) {
+    self = array.withUnsafeBufferPointer(Data.init)
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/Tests/ModelTests.swift b/tensorflow/lite/experimental/swift/Tests/ModelTests.swift
new file mode 100644
index 0000000000..025db18906
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Tests/ModelTests.swift
@@ -0,0 +1,59 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+@testable import TensorFlowLite
+import XCTest
+
+class ModelTests: XCTestCase {
+
+  var modelPath: String!
+
+  override func setUp() {
+    super.setUp()
+
+    let bundle = Bundle(for: type(of: self))
+    guard let modelPath = bundle.path(
+            forResource: Constant.modelInfo.name,
+            ofType: Constant.modelInfo.extension)
+    else {
+      XCTFail("Failed to get the model file path.")
+      return
+    }
+    self.modelPath = modelPath
+  }
+
+  override func tearDown() {
+    modelPath = nil
+
+    super.tearDown()
+  }
+
+  func testModel_InitWithFilePath() {
+    XCTAssertNotNil(Model(filePath: modelPath))
+  }
+
+  func testModel_InitWithEmptyFilePath_FailsInitialization() {
+    XCTAssertNil(Model(filePath: ""))
+  }
+
+  func testModel_InitWithInvalidFilePath_FailsInitialization() {
+    XCTAssertNil(Model(filePath: "invalid/path"))
+  }
+}
+
+// MARK: - Constants
+
+private enum Constant {
+  static let modelInfo = (name: "add", extension: "bin")
+}
diff --git a/tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift b/tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift
new file mode 100644
index 0000000000..65648c2698
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift
@@ -0,0 +1,43 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+@testable import TensorFlowLite
+import XCTest
+
+class QuantizationParametersTests: XCTestCase {
+
+  func testQuantizationParameters_InitWithCustomValues() {
+    let parameters = QuantizationParameters(scale: 0.5, zeroPoint: 1)
+    XCTAssertEqual(parameters.scale, 0.5)
+    XCTAssertEqual(parameters.zeroPoint, 1)
+  }
+
+  func testQuantizationParameters_Equatable() {
+    let parameters1 = QuantizationParameters(scale: 0.5, zeroPoint: 1)
+    let parameters2 = QuantizationParameters(scale: 0.5, zeroPoint: 1)
+    XCTAssertEqual(parameters1, parameters2)
+
+    let parameters3 = QuantizationParameters(scale: 0.4, zeroPoint: 1)
+    XCTAssertNotEqual(parameters1, parameters3)
+    XCTAssertNotEqual(parameters2, parameters3)
+  }
+}
+
+// MARK: - Extensions
+
+extension QuantizationParameters: Equatable {
+  public static func == (lhs: QuantizationParameters, rhs: QuantizationParameters) -> Bool {
+    return lhs.scale == rhs.scale && lhs.zeroPoint == rhs.zeroPoint
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/Tests/TensorTests.swift b/tensorflow/lite/experimental/swift/Tests/TensorTests.swift
new file mode 100644
index 0000000000..4540043a16
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Tests/TensorTests.swift
@@ -0,0 +1,83 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+@testable import TensorFlowLite
+import XCTest
+
+class TensorTests: XCTestCase {
+
+  // MARK: - Tensor
+
+  func testTensor_Init() {
+    let name = "InputTensor"
+    let dataType: TensorDataType = .uInt8
+    let shape = TensorShape(Constant.dimensions)
+    guard let data = name.data(using: .utf8) else { XCTFail("Data should not be nil."); return }
+    let quantizationParameters = QuantizationParameters(scale: 0.5, zeroPoint: 1)
+    let inputTensor = Tensor(
+      name: name,
+      dataType: dataType,
+      shape: shape,
+      data: data,
+      quantizationParameters: quantizationParameters
+    )
+    XCTAssertEqual(inputTensor.name, name)
+    XCTAssertEqual(inputTensor.dataType, dataType)
+    XCTAssertEqual(inputTensor.shape, shape)
+    XCTAssertEqual(inputTensor.data, data)
+    XCTAssertEqual(inputTensor.quantizationParameters, quantizationParameters)
+  }
+
+  // MARK: - TensorShape
+
+  func testTensorShape_InitWithArray() {
+    let shape = TensorShape(Constant.dimensions)
+    XCTAssertEqual(shape.rank, Constant.dimensions.count)
+    XCTAssertEqual(shape.dimensions, Constant.dimensions)
+  }
+
+  func testTensorShape_InitWithElements() {
+    let shape = TensorShape(2, 2, 3)
+    XCTAssertEqual(shape.rank, Constant.dimensions.count)
+    XCTAssertEqual(shape.dimensions, Constant.dimensions)
+  }
+
+  func testTensorShape_InitWithArrayLiteral() {
+    let shape: TensorShape = [2, 2, 3]
+    XCTAssertEqual(shape.rank, Constant.dimensions.count)
+    XCTAssertEqual(shape.dimensions, Constant.dimensions)
+  }
+}
+
+// MARK: - Constants
+
+private enum Constant {
+  /// Array of 2 arrays of 2 arrays of 3 numbers: [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]].
+  static let dimensions = [2, 2, 3]
+}
+
+// MARK: - Extensions
+
+extension TensorShape: Equatable {
+  public static func == (lhs: TensorShape, rhs: TensorShape) -> Bool {
+    return lhs.rank == rhs.rank && lhs.dimensions == rhs.dimensions
+  }
+}
+
+extension Tensor: Equatable {
+  public static func == (lhs: Tensor, rhs: Tensor) -> Bool {
+    return lhs.name == rhs.name && lhs.dataType == rhs.dataType && lhs.shape == rhs.shape &&
+           lhs.data == rhs.data && lhs.quantizationParameters == rhs.quantizationParameters
+  }
+}
diff --git a/tensorflow/tools/pip_package/pip_smoke_test.py b/tensorflow/tools/pip_package/pip_smoke_test.py
index 51d010c9e1..952c71c615 100644
--- a/tensorflow/tools/pip_package/pip_smoke_test.py
+++ b/tensorflow/tools/pip_package/pip_smoke_test.py
@@ -30,14 +30,19 @@ os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../..")))
 PIP_PACKAGE_QUERY_EXPRESSION = (
     "deps(//tensorflow/tools/pip_package:build_pip_package)")
 
+# List of file paths containing BUILD files that should not be included for the
+# pip smoke test.
+BUILD_BLACKLIST = [
+    "tensorflow/lite/examples/android",
+    "tensorflow/lite/experimental/swift",
+]
 
 def GetBuild(dir_base):
   """Get the list of BUILD file all targets recursively startind at dir_base."""
   items = []
   for root, _, files in os.walk(dir_base):
     for name in files:
-      if (name == "BUILD" and
-          root.find("tensorflow/lite/examples/android") == -1):
+      if (name == "BUILD" and root not in BUILD_BLACKLIST):
         items.append("//" + root + ":all")
   return items
 
@@ -67,9 +72,9 @@ def BuildPyTestDependencies():
 
 PYTHON_TARGETS, PY_TEST_QUERY_EXPRESSION = BuildPyTestDependencies()
 
-# Hard-coded blacklist of files if not included in pip package
 # TODO(amitpatankar): Clean up blacklist.
-BLACKLIST = [
+# List of dependencies that should not included in the pip package.
+DEPENDENCY_BLACKLIST = [
     "//tensorflow/python:extra_py_tests_deps",
     "//tensorflow/cc/saved_model:saved_model_half_plus_two",
     "//tensorflow:no_tensorflow_py_deps",
@@ -82,9 +87,7 @@ BLACKLIST = [
     "//tensorflow/core/kernels/cloud:bigquery_reader_ops",
     "//tensorflow/python/feature_column:vocabulary_testdata",
     "//tensorflow/python:framework/test_file_system.so",
-    # contrib
-    "//tensorflow/contrib/session_bundle:session_bundle_half_plus_two",
-    "//tensorflow/contrib/keras:testing_utils",
+    # lite
     "//tensorflow/lite/experimental/examples/lstm:tflite_lstm",
     "//tensorflow/lite/experimental/examples/lstm:tflite_lstm.py",
     "//tensorflow/lite/experimental/examples/lstm:unidirectional_sequence_lstm_test",  # pylint:disable=line-too-long
@@ -93,6 +96,9 @@ BLACKLIST = [
     "//tensorflow/lite/python:interpreter_test",
     "//tensorflow/lite/python:interpreter.py",
     "//tensorflow/lite/python:interpreter_test.py",
+    # contrib
+    "//tensorflow/contrib/session_bundle:session_bundle_half_plus_two",
+    "//tensorflow/contrib/keras:testing_utils",
     "//tensorflow/contrib/ffmpeg:test_data",
     "//tensorflow/contrib/fused_conv:fused_conv2d_bias_activation_op_test_base",
     "//tensorflow/contrib/hadoop:test_data",
@@ -149,8 +155,8 @@ def main():
   # File extensions and endings to ignore
   ignore_extensions = ["_test", "_test.py", "_test_gpu", "_test_gpu.py"]
 
-  ignored_files = 0
-  blacklisted_files = len(BLACKLIST)
+  ignored_files_count = 0
+  blacklisted_dependencies_count = len(DEPENDENCY_BLACKLIST)
   # Compare dependencies
   for dependency in tf_py_test_dependencies_list:
     if dependency and dependency.startswith("//tensorflow"):
@@ -158,16 +164,16 @@ def main():
       # Ignore extensions
       if any(dependency.endswith(ext) for ext in ignore_extensions):
         ignore = True
-        ignored_files += 1
+        ignored_files_count += 1
 
-      # Check if the dependency is in the pip package, the blacklist, or
-      # should be ignored because of its file extension
+      # Check if the dependency is in the pip package, the dependency blacklist,
+      # or should be ignored because of its file extension.
       if not (ignore or dependency in pip_package_dependencies_list or
-              dependency in BLACKLIST):
+              dependency in DEPENDENCY_BLACKLIST):
         missing_dependencies.append(dependency)
 
-  print("Ignored files: %d" % ignored_files)
-  print("Blacklisted files: %d" % blacklisted_files)
+  print("Ignored files count: %d" % ignored_files_count)
+  print("Blacklisted dependencies count: %d" % blacklisted_dependencies_count)
   if missing_dependencies:
     print("Missing the following dependencies from pip_packages:")
     for missing_dependency in missing_dependencies:
-- 
GitLab


From e5b4b8daf1db2f49173a0eace291172cc9b1e0d3 Mon Sep 17 00:00:00 2001
From: Ben Zinberg <bzinberg@google.com>
Date: Thu, 3 Jan 2019 10:37:47 -0800
Subject: [PATCH 0132/2345] Minor correction to the docs: `tf.Dimension(None)`
 does not `==` itself.

PiperOrigin-RevId: 227711581
---
 tensorflow/python/framework/tensor_shape.py | 49 +++++++++++----------
 1 file changed, 25 insertions(+), 24 deletions(-)

diff --git a/tensorflow/python/framework/tensor_shape.py b/tensorflow/python/framework/tensor_shape.py
index a4c626c64c..a7537bb5f1 100644
--- a/tensorflow/python/framework/tensor_shape.py
+++ b/tensorflow/python/framework/tensor_shape.py
@@ -271,10 +271,11 @@ class Dimension(object):
     Dimensions are combined as follows:
 
     ```python
-    tf.Dimension(n)   .merge_with(tf.Dimension(n))    == tf.Dimension(n)
-    tf.Dimension(n)   .merge_with(tf.Dimension(None)) == tf.Dimension(n)
-    tf.Dimension(None).merge_with(tf.Dimension(n))    == tf.Dimension(n)
-    tf.Dimension(None).merge_with(tf.Dimension(None)) == tf.Dimension(None)
+    tf.Dimension(n)   .merge_with(tf.Dimension(n))     == tf.Dimension(n)
+    tf.Dimension(n)   .merge_with(tf.Dimension(None))  == tf.Dimension(n)
+    tf.Dimension(None).merge_with(tf.Dimension(n))     == tf.Dimension(n)
+    # equivalent to tf.Dimension(None)
+    tf.Dimension(None).merge_with(tf.Dimension(None))
 
     # raises ValueError for n != m
     tf.Dimension(n)   .merge_with(tf.Dimension(m))
@@ -304,10 +305,10 @@ class Dimension(object):
     Dimensions are summed as follows:
 
     ```python
-    tf.Dimension(m)    + tf.Dimension(n)    == tf.Dimension(m + n)
-    tf.Dimension(m)    + tf.Dimension(None) == tf.Dimension(None)
-    tf.Dimension(None) + tf.Dimension(n)    == tf.Dimension(None)
-    tf.Dimension(None) + tf.Dimension(None) == tf.Dimension(None)
+    tf.Dimension(m)    + tf.Dimension(n)     == tf.Dimension(m + n)
+    tf.Dimension(m)    + tf.Dimension(None)  # equiv. to tf.Dimension(None)
+    tf.Dimension(None) + tf.Dimension(n)     # equiv. to tf.Dimension(None)
+    tf.Dimension(None) + tf.Dimension(None)  # equiv. to tf.Dimension(None)
     ```
 
     Args:
@@ -339,10 +340,10 @@ class Dimension(object):
     Dimensions are subtracted as follows:
 
     ```python
-    tf.Dimension(m)    - tf.Dimension(n)    == tf.Dimension(m - n)
-    tf.Dimension(m)    - tf.Dimension(None) == tf.Dimension(None)
-    tf.Dimension(None) - tf.Dimension(n)    == tf.Dimension(None)
-    tf.Dimension(None) - tf.Dimension(None) == tf.Dimension(None)
+    tf.Dimension(m)    - tf.Dimension(n)     == tf.Dimension(m - n)
+    tf.Dimension(m)    - tf.Dimension(None)  # equiv. to tf.Dimension(None)
+    tf.Dimension(None) - tf.Dimension(n)     # equiv. to tf.Dimension(None)
+    tf.Dimension(None) - tf.Dimension(None)  # equiv. to tf.Dimension(None)
     ```
 
     Args:
@@ -378,10 +379,10 @@ class Dimension(object):
     Dimensions are summed as follows:
 
     ```python
-    tf.Dimension(m)    * tf.Dimension(n)    == tf.Dimension(m * n)
-    tf.Dimension(m)    * tf.Dimension(None) == tf.Dimension(None)
-    tf.Dimension(None) * tf.Dimension(n)    == tf.Dimension(None)
-    tf.Dimension(None) * tf.Dimension(None) == tf.Dimension(None)
+    tf.Dimension(m)    * tf.Dimension(n)     == tf.Dimension(m * n)
+    tf.Dimension(m)    * tf.Dimension(None)  # equiv. to tf.Dimension(None)
+    tf.Dimension(None) * tf.Dimension(n)     # equiv. to tf.Dimension(None)
+    tf.Dimension(None) * tf.Dimension(None)  # equiv. to tf.Dimension(None)
     ```
 
     Args:
@@ -417,10 +418,10 @@ class Dimension(object):
     Dimensions are divided as follows:
 
     ```python
-    tf.Dimension(m)    // tf.Dimension(n)    == tf.Dimension(m // n)
-    tf.Dimension(m)    // tf.Dimension(None) == tf.Dimension(None)
-    tf.Dimension(None) // tf.Dimension(n)    == tf.Dimension(None)
-    tf.Dimension(None) // tf.Dimension(None) == tf.Dimension(None)
+    tf.Dimension(m)    // tf.Dimension(n)     == tf.Dimension(m // n)
+    tf.Dimension(m)    // tf.Dimension(None)  # equiv. to tf.Dimension(None)
+    tf.Dimension(None) // tf.Dimension(n)     # equiv. to tf.Dimension(None)
+    tf.Dimension(None) // tf.Dimension(None)  # equiv. to tf.Dimension(None)
     ```
 
     Args:
@@ -475,10 +476,10 @@ class Dimension(object):
     Dimension moduli are computed as follows:
 
     ```python
-    tf.Dimension(m)    % tf.Dimension(n)    == tf.Dimension(m % n)
-    tf.Dimension(m)    % tf.Dimension(None) == tf.Dimension(None)
-    tf.Dimension(None) % tf.Dimension(n)    == tf.Dimension(None)
-    tf.Dimension(None) % tf.Dimension(None) == tf.Dimension(None)
+    tf.Dimension(m)    % tf.Dimension(n)     == tf.Dimension(m % n)
+    tf.Dimension(m)    % tf.Dimension(None)  # equiv. to tf.Dimension(None)
+    tf.Dimension(None) % tf.Dimension(n)     # equiv. to tf.Dimension(None)
+    tf.Dimension(None) % tf.Dimension(None)  # equiv. to tf.Dimension(None)
     ```
 
     Args:
-- 
GitLab


From 0d969989e0e4fbddb8c57eadf8f4c72d975fbd6b Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Thu, 3 Jan 2019 10:41:14 -0800
Subject: [PATCH 0133/2345] Automated rollback of commit
 4d9173668fd3ba410532efd901467312b7418752

PiperOrigin-RevId: 227712180
---
 tensorflow/cc/BUILD              |  1 +
 tensorflow/cc/saved_model/BUILD  |  2 +
 tensorflow/compiler/tf2xla/BUILD |  1 +
 tensorflow/core/BUILD            |  3 +-
 tensorflow/core/kernels/BUILD    | 83 +-------------------------------
 tensorflow/tensorflow.bzl        |  2 +
 6 files changed, 10 insertions(+), 82 deletions(-)

diff --git a/tensorflow/cc/BUILD b/tensorflow/cc/BUILD
index a09becc49b..cf6d6050fa 100644
--- a/tensorflow/cc/BUILD
+++ b/tensorflow/cc/BUILD
@@ -150,6 +150,7 @@ cc_library_with_android_deps(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:ops",
         "//tensorflow/core:protos_all_cc",
     ],
 )
diff --git a/tensorflow/cc/saved_model/BUILD b/tensorflow/cc/saved_model/BUILD
index 52345a376c..836678d364 100644
--- a/tensorflow/cc/saved_model/BUILD
+++ b/tensorflow/cc/saved_model/BUILD
@@ -81,6 +81,7 @@ cc_library(
     ] + if_not_mobile([
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:lib",
+        "//tensorflow/core:ops",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:tensorflow",
     ]) + if_android([
@@ -100,6 +101,7 @@ cc_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:ops",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/util/tensor_bundle:naming",
         # mobile not supported yet
diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD
index d8123e956f..dfb6b57161 100644
--- a/tensorflow/compiler/tf2xla/BUILD
+++ b/tensorflow/compiler/tf2xla/BUILD
@@ -281,6 +281,7 @@ tf_cc_test(
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
     ],
 )
 
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 1afac057d9..f9552d5965 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1526,6 +1526,7 @@ cc_library(
         ":framework_internal",
         ":lib",
         ":lib_internal",
+        ":ops",
         ":protos_all_cc",
         ":shape_inference_testutil",
         ":tensor_testutil",
@@ -3072,6 +3073,7 @@ tf_cuda_library(
         ":lib",
         ":lib_internal",
         ":metrics",
+        ":ops",
         ":proto_text",
         ":protos_all_cc",
         "//tensorflow/core/debug:debug_graph_utils",
@@ -3854,7 +3856,6 @@ tf_cc_test(
         "ops/cudnn_rnn_ops_test.cc",
     ],
     deps = [
-        ":cudnn_rnn_ops",
         "//tensorflow/core",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index c72d04d3af..fd1df63c95 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -157,7 +157,6 @@ tf_kernel_library(
     name = "collective_ops",
     prefix = "collective_ops",
     deps = [
-        "//tensorflow/core:collective_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
@@ -222,7 +221,6 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:nn_ops_op_lib",
         "//third_party/eigen3",
     ],
     alwayslink = 1,
@@ -313,7 +311,6 @@ tf_kernel_library(
         "//tensorflow/core/nccl:nccl_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:gpu_headers_lib",
-        "//tensorflow/core:nccl_ops_op_lib",
     ]),
 )
 
@@ -459,7 +456,6 @@ cc_library(
     name = "batch_kernels",
     srcs = ["batch_kernels.cc"],
     deps = [
-        "//tensorflow/core:batch_ops_op_lib",
         "//tensorflow/core:framework_headers_lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/kernels:concat_lib_hdrs",
@@ -686,7 +682,6 @@ ARRAY_DEPS = [
     ":ops_util",
     ":transpose_functor",
     "//tensorflow/core:array_grad",
-    "//tensorflow/core:array_ops_op_lib",
     "//tensorflow/core:core_cpu",
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
@@ -715,7 +710,6 @@ tf_kernel_library(
     deps = [
         "//tensorflow/core:framework_headers_lib",
         "//tensorflow/core:lib",
-        "//tensorflow/core:set_ops_op_lib",
         "//third_party/eigen3",
     ],
 )
@@ -1079,7 +1073,6 @@ tf_kernel_library(
     srcs = ["ragged_gather_op.cc"],
     deps = [
         "//tensorflow/core:framework",
-        "//tensorflow/core:ragged_array_ops_op_lib",
     ],
 )
 
@@ -1090,7 +1083,6 @@ tf_cc_test(
     deps = [
         ":ragged_gather_op",
         "//tensorflow/core:framework",
-        "//tensorflow/core:ragged_array_ops_op_lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
@@ -1103,7 +1095,6 @@ tf_kernel_library(
     srcs = ["ragged_range_op.cc"],
     deps = [
         "//tensorflow/core:framework",
-        "//tensorflow/core:ragged_math_ops_op_lib",
     ],
 )
 
@@ -1113,7 +1104,6 @@ tf_cc_test(
     deps = [
         ":ragged_range_op",
         "//tensorflow/core:framework",
-        "//tensorflow/core:ragged_math_ops_op_lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
@@ -1126,7 +1116,6 @@ tf_kernel_library(
     srcs = ["ragged_tensor_to_sparse_kernel.cc"],
     deps = [
         "//tensorflow/core:framework",
-        "//tensorflow/core:ragged_conversion_ops_op_lib",
     ],
 )
 
@@ -1138,7 +1127,6 @@ tf_cc_test(
         ":ragged_tensor_to_sparse_kernel",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
-        "//tensorflow/core:ragged_conversion_ops_op_lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
@@ -1152,7 +1140,6 @@ tf_kernel_library(
     visibility = ["//visibility:public"],
     deps = [
         ":gpu_util_hdrs",
-        "//tensorflow/core:cudnn_rnn_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
@@ -1229,7 +1216,6 @@ tf_cuda_cc_test(
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
-        "//tensorflow/core:math_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
@@ -1754,7 +1740,6 @@ tf_kernel_library(
     prefix = "candidate_sampler_ops",
     deps = [
         ":range_sampler",
-        "//tensorflow/core:candidate_sampling_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
     ],
@@ -1788,7 +1773,6 @@ tf_kernel_library(
     name = "control_flow_ops",
     prefix = "control_flow_ops",
     deps = [
-        "//tensorflow/core:control_flow_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
     ],
@@ -1800,7 +1784,6 @@ tf_kernel_library(
     deps = [
         ":bounds_check",
         ":ops_util",
-        "//tensorflow/core:ctc_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core/util/ctc:ctc_beam_search_lib",
@@ -1881,7 +1864,6 @@ DATA_FLOW_DEPS = [
     ":typed_queue",
     "//third_party/eigen3",
     "//tensorflow/core:core_cpu",
-    "//tensorflow/core:data_flow_ops_op_lib",
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
@@ -1946,7 +1928,6 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:scoped_allocator_ops_op_lib",
     ],
 )
 
@@ -1965,7 +1946,6 @@ tf_cuda_cc_test(
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
-        "//tensorflow/core:math_ops_op_lib",
         "//tensorflow/core:proto_text",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
@@ -2013,7 +1993,6 @@ tf_kernel_library(
 DYNAMIC_DEPS = [
     ":bounds_check",
     "//tensorflow/core:core_cpu",
-    "//tensorflow/core:data_flow_ops_op_lib",
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
@@ -2046,7 +2025,6 @@ LOOKUP_DEPS = [
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
-    "//tensorflow/core:lookup_ops_op_lib",
 ]
 
 tf_kernel_library(
@@ -2075,7 +2053,6 @@ tf_kernel_library(
     deps = [
         ":lookup_table_init_op",
         ":lookup_table_op",
-        "//tensorflow/core:checkpoint_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//third_party/eigen3",
@@ -2086,7 +2063,6 @@ tf_kernel_library(
     name = "load_and_remap_matrix_op",
     srcs = ["load_and_remap_matrix_op.cc"],
     deps = [
-        "//tensorflow/core:checkpoint_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
@@ -2233,7 +2209,6 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:resource_variable_ops_op_lib",
         "@com_google_absl//absl/strings",
     ],
 )
@@ -2250,7 +2225,6 @@ tf_kernel_library(
         ":concat_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
-        "//tensorflow/core:list_ops_op_lib",
         "//third_party/eigen3",
     ],
 )
@@ -2261,7 +2235,6 @@ tf_kernel_library(
     deps = [
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
-        "//tensorflow/core:user_ops_op_lib",
     ],
 )
 
@@ -2284,7 +2257,6 @@ tf_kernel_library(
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
-        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//third_party/eigen3",
@@ -2297,7 +2269,6 @@ tf_kernel_library(
     deps = [
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
-        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/grappler:grappler_item",
@@ -2340,7 +2311,6 @@ IMAGE_DEPS = [
     "//third_party/eigen3",
     "//tensorflow/core:framework",
     "//tensorflow/core:gif_internal",
-    "//tensorflow/core:image_ops_op_lib",
     "//tensorflow/core:jpeg_internal",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
@@ -2680,7 +2650,6 @@ cc_library(
 IO_DEPS = [
     ":ops_util",
     "//tensorflow/core:framework",
-    "//tensorflow/core:io_ops_op_lib",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
     "//tensorflow/core:protos_all_cc",
@@ -2724,7 +2693,6 @@ SAVE_RESTORE_DEPS = [
     ":bounds_check_lib",
     ":save_restore_tensor",
     "//tensorflow/core:framework",
-    "//tensorflow/core:io_ops_op_lib",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
     "//tensorflow/core:protos_all_cc",
@@ -2843,7 +2811,6 @@ LINALG_DEPS = [
     "//third_party/eigen3",
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
-    "//tensorflow/core:linalg_ops_op_lib",
 ] + if_cuda([
     ":cuda_solvers",
     ":transpose_functor",
@@ -2987,7 +2954,6 @@ LOGGING_DEPS = [
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
-    "//tensorflow/core:logging_ops_op_lib",
     "//tensorflow/core:protos_all_cc",
 ]
 
@@ -3059,7 +3025,6 @@ tf_kernel_library(
         ":bounds_check",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
-        "//tensorflow/core:manip_ops_op_lib",
         "//third_party/eigen3",
     ],
 )
@@ -3091,7 +3056,6 @@ MATH_DEPS = [
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
     "//tensorflow/core:math_grad",
-    "//tensorflow/core:math_ops_op_lib",
     "//third_party/eigen3",
 ]
 
@@ -3202,7 +3166,7 @@ tf_kernel_library(
 tf_kernel_library(
     name = "cwise_op",
     prefix = "cwise_op",
-    deps = MATH_DEPS + ["//tensorflow/core:bitwise_ops_op_lib"],
+    deps = MATH_DEPS,
 )
 
 tf_kernel_library(
@@ -3221,7 +3185,6 @@ tf_kernel_library(
     name = "fft_ops",
     prefix = "fft_ops",
     deps = MATH_DEPS + [
-        "//tensorflow/core:spectral_ops_op_lib",
     ] + if_cuda([
         "//tensorflow/core/platform/default/build_config:cufft_plugin",
     ]),
@@ -3420,10 +3383,7 @@ tf_cuda_cc_test(
         ":quantized_ops",
         "//tensorflow/cc:cc_ops",
         "//tensorflow/cc:client_session",
-        "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:framework",
-        "//tensorflow/core:math_ops_op_lib",
-        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
@@ -3681,7 +3641,6 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:nn_ops_op_lib",
     ] + select({
         ":xsmm_convolutions": [
             "@libxsmm_archive//:xsmm_avx",
@@ -3703,7 +3662,6 @@ tf_kernel_library(
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
-        "//tensorflow/core:nn_ops_op_lib",
     ] + if_cuda([
         "@cub_archive//:cub",
         "@local_config_cuda//cuda:cudnn_header",
@@ -3723,7 +3681,6 @@ tf_kernel_library(
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
-        "//tensorflow/core:nn_ops_op_lib",
     ] + if_cuda([
         "@local_config_cuda//cuda:cudnn_header",
     ]),
@@ -3771,9 +3728,8 @@ NN_DEPS = [
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
     "//tensorflow/core:nn_grad",
-    "//tensorflow/core:nn_ops_op_lib",
     "//third_party/eigen3",
-] + if_mkl(["//tensorflow/core:mkl_nn_ops_op_lib"])
+]
 
 tf_kernel_library(
     name = "batch_norm_op",
@@ -3893,7 +3849,6 @@ tf_kernel_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:nn_grad",
-        "//tensorflow/core:nn_ops_op_lib",
     ] + if_cuda(["@cub_archive//:cub"]),
 )
 
@@ -4001,7 +3956,6 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:stream_executor",
         "//third_party/eigen3",
     ],
@@ -4045,7 +3999,6 @@ tf_kernel_library(
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
-        "//tensorflow/core:nn_ops_op_lib",
         "//third_party/eigen3",
     ],
 )
@@ -4127,7 +4080,6 @@ cc_library(
 PARSING_DEPS = [
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
-    "//tensorflow/core:parsing_ops_op_lib",
     "//tensorflow/core:proto_text",
     "//tensorflow/core:protos_all_cc",
 ]
@@ -4196,7 +4148,6 @@ RANDOM_OPS_DEPS = [
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
-    "//tensorflow/core:random_ops_op_lib",
 ]
 
 tf_kernel_library(
@@ -4235,7 +4186,6 @@ tf_kernel_library(
         ":random_op",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
-        "//tensorflow/core:stateless_random_ops_op_lib",
     ],
 )
 
@@ -4250,8 +4200,6 @@ cc_library(
 REQUIRED_DEPS = [
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
-    "//tensorflow/core:no_op_op_lib",
-    "//tensorflow/core:sendrecv_ops_op_lib",
 ]
 
 tf_kernel_library(
@@ -4312,7 +4260,6 @@ cc_library(
 SPARSE_DEPS = [
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
-    "//tensorflow/core:sparse_ops_op_lib",
 ]
 
 tf_kernel_library(
@@ -4557,7 +4504,6 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:sdca_ops_op_lib",
         "//third_party/eigen3",
         "@farmhash_archive//:farmhash",
     ],
@@ -4597,7 +4543,6 @@ STATE_DEPS = [
     "//third_party/eigen3",
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
-    "//tensorflow/core:state_ops_op_lib",
 ] + if_sycl(["//tensorflow/core:sycl_runtime"])
 
 tf_kernel_library(
@@ -4735,7 +4680,6 @@ STRING_DEPS = [
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
-    "//tensorflow/core:string_ops_op_lib",
 ]
 
 tf_kernel_library(
@@ -4886,7 +4830,6 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:string_ops_op_lib",
         "//third_party/eigen3",
         "//third_party/icu/data:conversion_data",
         "@icu//:common",
@@ -4908,7 +4851,6 @@ tf_kernel_library(
         ":variable_ops",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
-        "//tensorflow/core:training_ops_op_lib",
         "//third_party/eigen3",
     ],
 )
@@ -4967,7 +4909,6 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:random_ops_op_lib",
     ],
 )
 
@@ -4995,7 +4936,6 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:random_ops_op_lib",
     ],
 )
 
@@ -5904,12 +5844,9 @@ tf_kernel_library(
         ":ops_util",
         ":pooling_ops",
         ":quantization_utils",
-        "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
-        "//tensorflow/core:math_ops_op_lib",
-        "//tensorflow/core:nn_ops_op_lib",
         "//third_party/eigen3",
         "@gemmlowp",
     ],
@@ -6479,7 +6416,6 @@ tf_kernel_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
-        "//tensorflow/core:remote_fused_graph_ops_op_lib",
     ],
 )
 
@@ -6634,8 +6570,6 @@ tf_mkl_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:mkl_nn_ops_op_lib",
-        "//tensorflow/core:nn_ops_op_lib",
     ] + mkl_deps(),
 )
 
@@ -6685,8 +6619,6 @@ tf_mkl_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:mkl_nn_ops_op_lib",
-        "//tensorflow/core:nn_ops_op_lib",
     ] + mkl_deps(),
 )
 
@@ -6705,8 +6637,6 @@ tf_mkl_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:mkl_nn_ops_op_lib",
-        "//tensorflow/core:nn_ops_op_lib",
     ] + mkl_deps(),
 )
 
@@ -6720,8 +6650,6 @@ tf_mkl_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:mkl_nn_ops_op_lib",
-        "//tensorflow/core:nn_ops_op_lib",
         "//third_party/eigen3",
     ] + mkl_deps(),
 )
@@ -6736,8 +6664,6 @@ tf_mkl_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:mkl_nn_ops_op_lib",
-        "//tensorflow/core:nn_ops_op_lib",
         "//third_party/eigen3",
     ] + mkl_deps(),
 )
@@ -6878,7 +6804,6 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
-        "//tensorflow/core:summary_ops_op_lib",
         "//tensorflow/core/lib/db:sqlite",
     ] + if_not_v2([
         "//tensorflow/contrib/tensorboard/db:schema",
@@ -6893,7 +6818,6 @@ tf_kernel_library(
         "decode_proto_op.cc",
     ],
     deps = [
-        "//tensorflow/core:decode_proto_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core/util/proto:decode",
@@ -6907,7 +6831,6 @@ tf_kernel_library(
     name = "encode_proto_op",
     srcs = ["encode_proto_op.cc"],
     deps = [
-        "//tensorflow/core:encode_proto_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core/util/proto:descriptors",
@@ -6925,7 +6848,6 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:rpc_ops_op_lib",
         "//tensorflow/core/util/rpc:call_container",
         "//tensorflow/core/util/rpc:rpc_factory",
         "//tensorflow/core/util/rpc:rpc_factory_registry",
@@ -6938,7 +6860,6 @@ tf_kernel_library(
     srcs = ["unicode_script_op.cc"],
     deps = [
         "//tensorflow/core:framework",
-        "//tensorflow/core:string_ops_op_lib",
         "@icu//:common",
     ],
 )
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index 1024b686eb..b9069768e8 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -590,6 +590,7 @@ def tf_gen_op_wrappers_cc(
             clean_dep("//tensorflow/core:core_cpu"),
             clean_dep("//tensorflow/core:framework"),
             clean_dep("//tensorflow/core:lib"),
+            clean_dep("//tensorflow/core:ops"),
             clean_dep("//tensorflow/core:protos_all_cc"),
         ]) + if_android([
             clean_dep("//tensorflow/core:android_tensorflow_lib"),
@@ -606,6 +607,7 @@ def tf_gen_op_wrappers_cc(
             clean_dep("//tensorflow/core:core_cpu"),
             clean_dep("//tensorflow/core:framework"),
             clean_dep("//tensorflow/core:lib"),
+            clean_dep("//tensorflow/core:ops"),
             clean_dep("//tensorflow/core:protos_all_cc"),
         ]) + if_android([
             clean_dep("//tensorflow/core:android_tensorflow_lib"),
-- 
GitLab


From f6fa3515db311b666dd4bd529f9b8b87575ce978 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Thu, 3 Jan 2019 10:44:43 -0800
Subject: [PATCH 0134/2345] Silence a noisy __del__

Popped up sometimes, looking something like:

Exception ignored in: <bound method BaseSession.__del__ of <tensorflow.python.client.session.Session object at 0x7ff8f47827f0>>
Traceback (most recent call last):
  File "[...]tensorflow/python/client/session.py", line 738, in __del__
TypeError: 'NoneType' object is not callable

Which I'm guessing is TF_DeleteSession being None.

PiperOrigin-RevId: 227712751
---
 tensorflow/python/client/session.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/client/session.py b/tensorflow/python/client/session.py
index 87a200ed33..b97eb884b3 100644
--- a/tensorflow/python/client/session.py
+++ b/tensorflow/python/client/session.py
@@ -736,10 +736,11 @@ class BaseSession(SessionInterface):
     if self._session is not None:
       try:
         tf_session.TF_DeleteSession(self._session)
-      except AttributeError:
-        # At shutdown, `c_api_util` or `tf_session` may have been garbage
-        # collected, causing the above method calls to fail. In this case,
-        # silently leak since the program is about to terminate anyway.
+      except (AttributeError, TypeError):
+        # At shutdown, `c_api_util`, `tf_session`, or
+        # `tf_session.TF_DeleteSession` may have been garbage collected, causing
+        # the above method calls to fail. In this case, silently leak since the
+        # program is about to terminate anyway.
         pass
       self._session = None
 
-- 
GitLab


From e518527f10048b10956c7192de1411c36a3a1938 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 11:59:51 -0800
Subject: [PATCH 0135/2345] Spectral Normalization implementation in TFGAN

PiperOrigin-RevId: 227725854
---
 tensorflow/contrib/gan/BUILD                  |  43 +++
 .../contrib/gan/python/features/__init__.py   |   3 +
 .../features/python/spectral_normalization.py |  32 ++
 .../python/spectral_normalization_impl.py     | 315 ++++++++++++++++
 .../python/spectral_normalization_test.py     | 354 ++++++++++++++++++
 5 files changed, 747 insertions(+)
 create mode 100644 tensorflow/contrib/gan/python/features/python/spectral_normalization.py
 create mode 100644 tensorflow/contrib/gan/python/features/python/spectral_normalization_impl.py
 create mode 100644 tensorflow/contrib/gan/python/features/python/spectral_normalization_test.py

diff --git a/tensorflow/contrib/gan/BUILD b/tensorflow/contrib/gan/BUILD
index 97184dabb0..0626875b76 100644
--- a/tensorflow/contrib/gan/BUILD
+++ b/tensorflow/contrib/gan/BUILD
@@ -132,6 +132,7 @@ py_library(
         ":clip_weights",
         ":conditioning_utils",
         ":random_tensor_pool",
+        ":spectral_normalization",
         ":virtual_batchnorm",
         "//tensorflow/python:util",
     ],
@@ -676,3 +677,45 @@ py_test(
         "//third_party/py/numpy",
     ],
 )
+
+py_library(
+    name = "spectral_normalization",
+    srcs = [
+        "python/features/python/spectral_normalization.py",
+        "python/features/python/spectral_normalization_impl.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:init_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:standard_ops",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python/keras:engine",
+    ],
+)
+
+py_test(
+    name = "spectral_normalization_test",
+    srcs = ["python/features/python/spectral_normalization_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":spectral_normalization",
+        "//tensorflow/contrib/layers:layers_py",
+        "//tensorflow/contrib/slim",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:init_ops",
+        "//tensorflow/python:layers",
+        "//tensorflow/python:linalg_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python:variables",
+        "//tensorflow/python/keras:layers",
+        "//third_party/py/numpy",
+    ],
+)
diff --git a/tensorflow/contrib/gan/python/features/__init__.py b/tensorflow/contrib/gan/python/features/__init__.py
index 4816daf760..410c3a0205 100644
--- a/tensorflow/contrib/gan/python/features/__init__.py
+++ b/tensorflow/contrib/gan/python/features/__init__.py
@@ -27,11 +27,13 @@ from __future__ import print_function
 from tensorflow.contrib.gan.python.features.python import clip_weights
 from tensorflow.contrib.gan.python.features.python import conditioning_utils
 from tensorflow.contrib.gan.python.features.python import random_tensor_pool
+from tensorflow.contrib.gan.python.features.python import spectral_normalization
 from tensorflow.contrib.gan.python.features.python import virtual_batchnorm
 
 from tensorflow.contrib.gan.python.features.python.clip_weights import *
 from tensorflow.contrib.gan.python.features.python.conditioning_utils import *
 from tensorflow.contrib.gan.python.features.python.random_tensor_pool import *
+from tensorflow.contrib.gan.python.features.python.spectral_normalization import *
 from tensorflow.contrib.gan.python.features.python.virtual_batchnorm import *
 # pylint: enable=unused-import,wildcard-import
 
@@ -40,5 +42,6 @@ from tensorflow.python.util.all_util import remove_undocumented
 _allowed_symbols = clip_weights.__all__
 _allowed_symbols += conditioning_utils.__all__
 _allowed_symbols += random_tensor_pool.__all__
+_allowed_symbols += spectral_normalization.__all__
 _allowed_symbols += virtual_batchnorm.__all__
 remove_undocumented(__name__, _allowed_symbols)
diff --git a/tensorflow/contrib/gan/python/features/python/spectral_normalization.py b/tensorflow/contrib/gan/python/features/python/spectral_normalization.py
new file mode 100644
index 0000000000..54d3d0a218
--- /dev/null
+++ b/tensorflow/contrib/gan/python/features/python/spectral_normalization.py
@@ -0,0 +1,32 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Keras-like layers and utilities that implement Spectral Normalization.
+
+Based on "Spectral Normalization for Generative Adversarial Networks" by Miyato,
+et al in ICLR 2018. https://openreview.net/pdf?id=B1QRgziT-
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.gan.python.features.python import spectral_normalization_impl
+# pylint: disable=wildcard-import
+from tensorflow.contrib.gan.python.features.python.spectral_normalization_impl import *
+# pylint: enable=wildcard-import
+from tensorflow.python.util.all_util import remove_undocumented
+
+__all__ = spectral_normalization_impl.__all__
+remove_undocumented(__name__, __all__)
diff --git a/tensorflow/contrib/gan/python/features/python/spectral_normalization_impl.py b/tensorflow/contrib/gan/python/features/python/spectral_normalization_impl.py
new file mode 100644
index 0000000000..0cc653f0a7
--- /dev/null
+++ b/tensorflow/contrib/gan/python/features/python/spectral_normalization_impl.py
@@ -0,0 +1,315 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Keras-like layers and utilities that implement Spectral Normalization.
+
+Based on "Spectral Normalization for Generative Adversarial Networks" by Miyato,
+et al in ICLR 2018. https://openreview.net/pdf?id=B1QRgziT-
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import contextlib
+import numbers
+import re
+
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.keras.engine import base_layer_utils as keras_base_layer_utils
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import init_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.platform import tf_logging as logging
+
+__all__ = [
+    'compute_spectral_norm', 'spectral_normalize', 'spectral_norm_regularizer',
+    'spectral_normalization_custom_getter', 'keras_spectral_normalization'
+]
+
+# tf.bfloat16 should work, but tf.matmul converts those to tf.float32 which then
+# can't directly be assigned back to the tf.bfloat16 variable.
+_OK_DTYPES_FOR_SPECTRAL_NORM = (dtypes.float16, dtypes.float32, dtypes.float64)
+_PERSISTED_U_VARIABLE_SUFFIX = 'spectral_norm_u'
+
+
+def compute_spectral_norm(w_tensor, power_iteration_rounds=1, name=None):
+  """Estimates the largest singular value in the weight tensor.
+
+  Args:
+    w_tensor: The weight matrix whose spectral norm should be computed.
+    power_iteration_rounds: The number of iterations of the power method to
+      perform. A higher number yeilds a better approximation.
+    name: An optional scope name.
+
+  Returns:
+    The largest singular value (the spectral norm) of w.
+  """
+  with variable_scope.variable_scope(name, 'spectral_norm'):
+    # The paper says to flatten convnet kernel weights from
+    # (C_out, C_in, KH, KW) to (C_out, C_in * KH * KW). But TensorFlow's Conv2D
+    # kernel weight shape is (KH, KW, C_in, C_out), so it should be reshaped to
+    # (KH * KW * C_in, C_out), and similarly for other layers that put output
+    # channels as last dimension.
+    # n.b. this means that w here is equivalent to w.T in the paper.
+    w = array_ops.reshape(w_tensor, (-1, w_tensor.get_shape()[-1]))
+
+    # Persisted approximation of first left singular vector of matrix `w`.
+    u_var = variable_scope.get_variable(
+        _PERSISTED_U_VARIABLE_SUFFIX,
+        shape=(w.shape[0], 1),
+        dtype=w.dtype,
+        initializer=init_ops.random_normal_initializer(),
+        trainable=False)
+    u = u_var
+
+    # Use power iteration method to approximate spectral norm.
+    for _ in range(power_iteration_rounds):
+      # `v` approximates the first right singular vector of matrix `w`.
+      v = nn.l2_normalize(math_ops.matmul(array_ops.transpose(w), u))
+      u = nn.l2_normalize(math_ops.matmul(w, v))
+
+    # Update persisted approximation.
+    with ops.control_dependencies([u_var.assign(u, name='update_u')]):
+      u = array_ops.identity(u)
+
+    u = array_ops.stop_gradient(u)
+    v = array_ops.stop_gradient(v)
+
+    # Largest singular value of `w`.
+    spectral_norm = math_ops.matmul(
+        math_ops.matmul(array_ops.transpose(u), w), v)
+    spectral_norm.shape.assert_is_fully_defined()
+    spectral_norm.shape.assert_is_compatible_with([1, 1])
+
+    return spectral_norm[0][0]
+
+
+def spectral_normalize(w, power_iteration_rounds=1, name=None):
+  """Normalizes a weight matrix by its spectral norm.
+
+  Args:
+    w: The weight matrix to be normalized.
+    power_iteration_rounds: The number of iterations of the power method to
+      perform. A higher number yeilds a better approximation.
+    name: An optional scope name.
+
+  Returns:
+    A normalized weight matrix tensor.
+  """
+  with variable_scope.variable_scope(name, 'spectral_normalize'):
+    w_normalized = w / compute_spectral_norm(
+        w, power_iteration_rounds=power_iteration_rounds)
+    return array_ops.reshape(w_normalized, w.get_shape())
+
+
+def spectral_norm_regularizer(scale, power_iteration_rounds=1, scope=None):
+  """Returns a functions that can be used to apply spectral norm regularization.
+
+  Small spectral norms enforce a small Lipschitz constant, which is necessary
+  for Wasserstein GANs.
+
+  Args:
+    scale: A scalar multiplier. 0.0 disables the regularizer.
+    power_iteration_rounds: The number of iterations of the power method to
+      perform. A higher number yeilds a better approximation.
+    scope: An optional scope name.
+
+  Returns:
+    A function with the signature `sn(weights)` that applies spectral norm
+    regularization.
+
+  Raises:
+    ValueError: If scale is negative or if scale is not a float.
+  """
+  if isinstance(scale, numbers.Integral):
+    raise ValueError('scale cannot be an integer: %s' % scale)
+  if isinstance(scale, numbers.Real):
+    if scale < 0.0:
+      raise ValueError(
+          'Setting a scale less than 0 on a regularizer: %g' % scale)
+    if scale == 0.0:
+      logging.info('Scale of 0 disables regularizer.')
+      return lambda _: None
+
+  def sn(weights, name=None):
+    """Applies spectral norm regularization to weights."""
+    with ops.name_scope(scope, 'SpectralNormRegularizer', [weights]) as name:
+      scale_t = ops.convert_to_tensor(
+          scale, dtype=weights.dtype.base_dtype, name='scale')
+      return math_ops.multiply(
+          scale_t,
+          compute_spectral_norm(
+              weights, power_iteration_rounds=power_iteration_rounds),
+          name=name)
+
+  return sn
+
+
+def _default_name_filter(name):
+  """A filter function to identify common names of weight variables.
+
+  Args:
+    name: The variable name.
+
+  Returns:
+    Whether `name` is a standard name for a weight/kernel variables used in the
+    Keras, tf.layers, tf.contrib.layers or tf.contrib.slim libraries.
+  """
+  match = re.match(r'(.*\/)?(depthwise_|pointwise_)?(weights|kernel)$', name)
+  return match is not None
+
+
+def spectral_normalization_custom_getter(name_filter=_default_name_filter,
+                                         power_iteration_rounds=1):
+  """Custom getter that performs Spectral Normalization on a weight tensor.
+
+  Specifically it divides the weight tensor by its largest singular value. This
+  is intended to stabilize GAN training, by making the discriminator satisfy a
+  local 1-Lipschitz constraint.
+
+  Based on [Spectral Normalization for Generative Adversarial Networks][sn-gan].
+
+  [sn-gan]: https://openreview.net/forum?id=B1QRgziT-
+
+  To reproduce an SN-GAN, apply this custom_getter to every weight tensor of
+  your discriminator. The last dimension of the weight tensor must be the number
+  of output channels.
+
+  Apply this to layers by supplying this as the `custom_getter` of a
+  `tf.variable_scope`. For example:
+
+    with tf.variable_scope('discriminator',
+                           custom_getter=spectral_norm_getter()):
+      net = discriminator_fn(net)
+
+  IMPORTANT: Keras does not respect the custom_getter supplied by the
+  VariableScope, so Keras users should use `keras_spectral_normalization`
+  instead of (or in addition to) this approach.
+
+  It is important to carefully select to which weights you want to apply
+  Spectral Normalization. In general you want to normalize the kernels of
+  convolution and dense layers, but you do not want to normalize biases. You
+  also want to avoid normalizing batch normalization (and similar) variables,
+  but in general such layers play poorly with Spectral Normalization, since the
+  gamma can cancel out the normalization in other layers. By default we supply a
+  filter that matches the kernel variable names of the dense and convolution
+  layers of the tf.layers, tf.contrib.layers, tf.keras and tf.contrib.slim
+  libraries. If you are using anything else you'll need a custom `name_filter`.
+
+  This custom getter internally creates a variable used to compute the spectral
+  norm by power iteration. It will update every time the variable is accessed,
+  which means the normalized discriminator weights may change slightly whilst
+  training the generator. Whilst unusual, this matches how the paper's authors
+  implement it, and in general additional rounds of power iteration can't hurt.
+
+  Args:
+    name_filter: Optionally, a method that takes a Variable name as input and
+      returns whether this Variable should be normalized.
+    power_iteration_rounds: The number of iterations of the power method to
+      perform per step. A higher number yeilds a better approximation of the
+      true spectral norm.
+
+  Returns:
+    A custom getter function that applies Spectral Normalization to all
+    Variables whose names match `name_filter`.
+
+  Raises:
+    ValueError: If name_filter is not callable.
+  """
+  if not callable(name_filter):
+    raise ValueError('name_filter must be callable')
+
+  def _internal_getter(getter, name, *args, **kwargs):
+    """A custom getter function that applies Spectral Normalization.
+
+    Args:
+      getter: The true getter to call.
+      name: Name of new/existing variable, in the same format as
+        tf.get_variable.
+      *args: Other positional arguments, in the same format as tf.get_variable.
+      **kwargs: Keyword arguments, in the same format as tf.get_variable.
+
+    Returns:
+      The return value of `getter(name, *args, **kwargs)`, spectrally
+      normalized.
+
+    Raises:
+      ValueError: If used incorrectly, or if `dtype` is not supported.
+    """
+    if not name_filter(name):
+      return getter(name, *args, **kwargs)
+
+    if name.endswith(_PERSISTED_U_VARIABLE_SUFFIX):
+      raise ValueError(
+          'Cannot apply Spectral Normalization to internal variables created '
+          'for Spectral Normalization. Tried to normalized variable [%s]' %
+          name)
+
+    if kwargs['dtype'] not in _OK_DTYPES_FOR_SPECTRAL_NORM:
+      raise ValueError('Disallowed data type {}'.format(kwargs['dtype']))
+
+    # This layer's weight Variable/PartitionedVariable.
+    w_tensor = getter(name, *args, **kwargs)
+
+    if len(w_tensor.get_shape()) < 2:
+      raise ValueError(
+          'Spectral norm can only be applied to multi-dimensional tensors')
+
+    return spectral_normalize(
+        w_tensor,
+        power_iteration_rounds=power_iteration_rounds,
+        name=(name + '/spectral_normalize'))
+
+  return _internal_getter
+
+
+@contextlib.contextmanager
+def keras_spectral_normalization(name_filter=_default_name_filter,
+                                 power_iteration_rounds=1):
+  """A context manager that enables Spectral Normalization for Keras.
+
+  Keras doesn't respect the `custom_getter` in the VariableScope, so this is a
+  bit of a hack to make things work.
+
+  Usage:
+    with keras_spectral_normalization():
+      net = discriminator_fn(net)
+
+  Args:
+    name_filter: Optionally, a method that takes a Variable name as input and
+      returns whether this Variable should be normalized.
+    power_iteration_rounds: The number of iterations of the power method to
+      perform per step. A higher number yeilds a better approximation of the
+      true spectral norm.
+
+  Yields:
+    A context manager that wraps the standard Keras variable creation method
+    with the `spectral_normalization_custom_getter`.
+  """
+  original_make_variable = keras_base_layer_utils.make_variable
+  sn_getter = spectral_normalization_custom_getter(
+      name_filter=name_filter, power_iteration_rounds=power_iteration_rounds)
+
+  def make_variable_wrapper(name, *args, **kwargs):
+    return sn_getter(original_make_variable, name, *args, **kwargs)
+
+  keras_base_layer_utils.make_variable = make_variable_wrapper
+
+  yield
+
+  keras_base_layer_utils.make_variable = original_make_variable
diff --git a/tensorflow/contrib/gan/python/features/python/spectral_normalization_test.py b/tensorflow/contrib/gan/python/features/python/spectral_normalization_test.py
new file mode 100644
index 0000000000..4ea21f70ec
--- /dev/null
+++ b/tensorflow/contrib/gan/python/features/python/spectral_normalization_test.py
@@ -0,0 +1,354 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for features.spectral_normalization."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib import slim
+from tensorflow.contrib.gan.python.features.python import spectral_normalization_impl as spectral_normalization
+from tensorflow.contrib.layers.python.layers import layers as contrib_layers
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.keras.layers import convolutional as keras_convolutional
+from tensorflow.python.keras.layers import core as keras_core
+from tensorflow.python.layers import convolutional as layers_convolutional
+from tensorflow.python.layers import core as layers_core
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import init_ops
+from tensorflow.python.ops import linalg_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+
+
+class SpectralNormalizationTest(test.TestCase):
+
+  def testComputeSpectralNorm(self):
+    weights = variable_scope.get_variable(
+        'w', dtype=dtypes.float32, shape=[2, 3, 50, 100])
+    weights = math_ops.multiply(weights, 10.0)
+    s = linalg_ops.svd(
+        array_ops.reshape(weights, [-1, weights.shape[-1]]), compute_uv=False)
+    true_sn = s[..., 0]
+    estimated_sn = spectral_normalization.compute_spectral_norm(weights)
+
+    with self.cached_session() as sess:
+      sess.run(variables.global_variables_initializer())
+      np_true_sn = sess.run(true_sn)
+      for i in range(50):
+        est = sess.run(estimated_sn)
+        if i < 1:
+          np_est_1 = est
+        if i < 4:
+          np_est_5 = est
+        if i < 9:
+          np_est_10 = est
+        np_est_50 = est
+
+      # Check that the estimate improves with more iterations.
+      self.assertAlmostEqual(np_true_sn, np_est_50, 0)
+      self.assertGreater(
+          abs(np_true_sn - np_est_10), abs(np_true_sn - np_est_50))
+      self.assertGreater(
+          abs(np_true_sn - np_est_5), abs(np_true_sn - np_est_10))
+      self.assertGreater(abs(np_true_sn - np_est_1), abs(np_true_sn - np_est_5))
+
+  def testSpectralNormalize(self):
+    weights = variable_scope.get_variable(
+        'w', dtype=dtypes.float32, shape=[2, 3, 50, 100])
+    weights = math_ops.multiply(weights, 10.0)
+    normalized_weights = spectral_normalization.spectral_normalize(
+        weights, power_iteration_rounds=1)
+
+    unnormalized_sigma = linalg_ops.svd(
+        array_ops.reshape(weights, [-1, weights.shape[-1]]),
+        compute_uv=False)[..., 0]
+    normalized_sigma = linalg_ops.svd(
+        array_ops.reshape(normalized_weights, [-1, weights.shape[-1]]),
+        compute_uv=False)[..., 0]
+
+    with self.cached_session() as sess:
+      sess.run(variables.global_variables_initializer())
+      s0 = sess.run(unnormalized_sigma)
+
+      for i in range(50):
+        sigma = sess.run(normalized_sigma)
+        if i < 1:
+          s1 = sigma
+        if i < 5:
+          s5 = sigma
+        if i < 10:
+          s10 = sigma
+        s50 = sigma
+
+      self.assertAlmostEqual(1., s50, 0)
+      self.assertGreater(abs(s10 - 1.), abs(s50 - 1.))
+      self.assertGreater(abs(s5 - 1.), abs(s10 - 1.))
+      self.assertGreater(abs(s1 - 1.), abs(s5 - 1.))
+      self.assertGreater(abs(s0 - 1.), abs(s1 - 1.))
+
+  def _testLayerHelper(self, build_layer_fn, w_shape, b_shape, is_keras=False):
+    x = array_ops.placeholder(dtypes.float32, shape=[2, 10, 10, 3])
+
+    w_initial = np.random.randn(*w_shape) * 10
+    w_initializer = init_ops.constant_initializer(w_initial)
+    b_initial = np.random.randn(*b_shape)
+    b_initializer = init_ops.constant_initializer(b_initial)
+
+    if is_keras:
+      context_manager = spectral_normalization.keras_spectral_normalization()
+    else:
+      getter = spectral_normalization.spectral_normalization_custom_getter()
+      context_manager = variable_scope.variable_scope('', custom_getter=getter)
+
+    with context_manager:
+      (net,
+       expected_normalized_vars, expected_not_normalized_vars) = build_layer_fn(
+           x, w_initializer, b_initializer)
+
+    x_data = np.random.rand(*x.shape)
+
+    with self.cached_session() as sess:
+      sess.run(variables.global_variables_initializer())
+
+      # Before running a forward pass we still expect the variables values to
+      # differ from the initial value because of the normalizer.
+      w_befores = []
+      for name, var in expected_normalized_vars.items():
+        w_before = sess.run(var)
+        w_befores.append(w_before)
+        self.assertFalse(
+            np.allclose(w_initial, w_before),
+            msg=('%s appears not to be normalized. Before: %s After: %s' %
+                 (name, w_initial, w_before)))
+
+      # Not true for the unnormalized variables.
+      for name, var in expected_not_normalized_vars.items():
+        b_before = sess.run(var)
+        self.assertTrue(
+            np.allclose(b_initial, b_before),
+            msg=('%s appears to be unexpectedly normalized. '
+                 'Before: %s After: %s' % (name, b_initial, b_before)))
+
+      # Run a bunch of forward passes.
+      for _ in range(1000):
+        _ = sess.run(net, feed_dict={x: x_data})
+
+      # We expect this to have improved the estimate of the spectral norm,
+      # which should have changed the variable values and brought them close
+      # to the true Spectral Normalized values.
+      _, s, _ = np.linalg.svd(w_initial.reshape([-1, 3]))
+      exactly_normalized = w_initial / s[0]
+      for w_before, (name, var) in zip(w_befores,
+                                       expected_normalized_vars.items()):
+        w_after = sess.run(var)
+        self.assertFalse(
+            np.allclose(w_before, w_after, rtol=1e-8, atol=1e-8),
+            msg=('%s did not improve over many iterations. '
+                 'Before: %s After: %s' % (name, w_before, w_after)))
+        self.assertAllClose(
+            exactly_normalized,
+            w_after,
+            rtol=1e-4,
+            atol=1e-4,
+            msg=('Estimate of spectral norm for %s was innacurate. '
+                 'Normalized matrices do not match.'
+                 'Estimate: %s Actual: %s' % (name, w_after,
+                                              exactly_normalized)))
+
+  def testConv2D_Layers(self):
+
+    def build_layer_fn(x, w_initializer, b_initializer):
+      layer = layers_convolutional.Conv2D(
+          filters=3,
+          kernel_size=3,
+          padding='same',
+          kernel_initializer=w_initializer,
+          bias_initializer=b_initializer)
+      net = layer.apply(x)
+      expected_normalized_vars = {'tf.layers.Conv2d.kernel': layer.kernel}
+      expected_not_normalized_vars = {'tf.layers.Conv2d.bias': layer.bias}
+
+      return net, expected_normalized_vars, expected_not_normalized_vars
+
+    self._testLayerHelper(build_layer_fn, (3, 3, 3, 3), (3,))
+
+  def testConv2D_ContribLayers(self):
+
+    def build_layer_fn(x, w_initializer, b_initializer):
+      var_collection = {
+          'weights': ['CONTRIB_LAYERS_CONV2D_WEIGHTS'],
+          'biases': ['CONTRIB_LAYERS_CONV2D_BIASES']
+      }
+      net = contrib_layers.conv2d(
+          x,
+          3,
+          3,
+          weights_initializer=w_initializer,
+          biases_initializer=b_initializer,
+          variables_collections=var_collection)
+      weight_vars = ops.get_collection('CONTRIB_LAYERS_CONV2D_WEIGHTS')
+      self.assertEquals(1, len(weight_vars))
+      bias_vars = ops.get_collection('CONTRIB_LAYERS_CONV2D_BIASES')
+      self.assertEquals(1, len(bias_vars))
+      expected_normalized_vars = {
+          'contrib.layers.conv2d.weights': weight_vars[0]
+      }
+      expected_not_normalized_vars = {
+          'contrib.layers.conv2d.bias': bias_vars[0]
+      }
+
+      return net, expected_normalized_vars, expected_not_normalized_vars
+
+    self._testLayerHelper(build_layer_fn, (3, 3, 3, 3), (3,))
+
+  def testConv2D_Slim(self):
+
+    def build_layer_fn(x, w_initializer, b_initializer):
+      var_collection = {
+          'weights': ['SLIM_CONV2D_WEIGHTS'],
+          'biases': ['SLIM_CONV2D_BIASES']
+      }
+      net = slim.conv2d(
+          x,
+          3,
+          3,
+          weights_initializer=w_initializer,
+          biases_initializer=b_initializer,
+          variables_collections=var_collection)
+      weight_vars = ops.get_collection('SLIM_CONV2D_WEIGHTS')
+      self.assertEquals(1, len(weight_vars))
+      bias_vars = ops.get_collection('SLIM_CONV2D_BIASES')
+      self.assertEquals(1, len(bias_vars))
+      expected_normalized_vars = {'slim.conv2d.weights': weight_vars[0]}
+      expected_not_normalized_vars = {'slim.conv2d.bias': bias_vars[0]}
+
+      return net, expected_normalized_vars, expected_not_normalized_vars
+
+    self._testLayerHelper(build_layer_fn, (3, 3, 3, 3), (3,))
+
+  def testConv2D_Keras(self):
+
+    def build_layer_fn(x, w_initializer, b_initializer):
+      layer = keras_convolutional.Conv2D(
+          filters=3,
+          kernel_size=3,
+          padding='same',
+          kernel_initializer=w_initializer,
+          bias_initializer=b_initializer)
+      net = layer.apply(x)
+      expected_normalized_vars = {'keras.layers.Conv2d.kernel': layer.kernel}
+      expected_not_normalized_vars = {'keras.layers.Conv2d.bias': layer.bias}
+
+      return net, expected_normalized_vars, expected_not_normalized_vars
+
+    self._testLayerHelper(build_layer_fn, (3, 3, 3, 3), (3,), is_keras=True)
+
+  def testFC_Layers(self):
+
+    def build_layer_fn(x, w_initializer, b_initializer):
+      x = layers_core.Flatten()(x)
+      layer = layers_core.Dense(
+          units=3,
+          kernel_initializer=w_initializer,
+          bias_initializer=b_initializer)
+      net = layer.apply(x)
+      expected_normalized_vars = {'tf.layers.Dense.kernel': layer.kernel}
+      expected_not_normalized_vars = {'tf.layers.Dense.bias': layer.bias}
+
+      return net, expected_normalized_vars, expected_not_normalized_vars
+
+    self._testLayerHelper(build_layer_fn, (300, 3), (3,))
+
+  def testFC_ContribLayers(self):
+
+    def build_layer_fn(x, w_initializer, b_initializer):
+      var_collection = {
+          'weights': ['CONTRIB_LAYERS_FC_WEIGHTS'],
+          'biases': ['CONTRIB_LAYERS_FC_BIASES']
+      }
+      x = contrib_layers.flatten(x)
+      net = contrib_layers.fully_connected(
+          x,
+          3,
+          weights_initializer=w_initializer,
+          biases_initializer=b_initializer,
+          variables_collections=var_collection)
+      weight_vars = ops.get_collection('CONTRIB_LAYERS_FC_WEIGHTS')
+      self.assertEquals(1, len(weight_vars))
+      bias_vars = ops.get_collection('CONTRIB_LAYERS_FC_BIASES')
+      self.assertEquals(1, len(bias_vars))
+      expected_normalized_vars = {
+          'contrib.layers.fully_connected.weights': weight_vars[0]
+      }
+      expected_not_normalized_vars = {
+          'contrib.layers.fully_connected.bias': bias_vars[0]
+      }
+
+      return net, expected_normalized_vars, expected_not_normalized_vars
+
+    self._testLayerHelper(build_layer_fn, (300, 3), (3,))
+
+  def testFC_Slim(self):
+
+    def build_layer_fn(x, w_initializer, b_initializer):
+      var_collection = {
+          'weights': ['SLIM_FC_WEIGHTS'],
+          'biases': ['SLIM_FC_BIASES']
+      }
+      x = slim.flatten(x)
+      net = slim.fully_connected(
+          x,
+          3,
+          weights_initializer=w_initializer,
+          biases_initializer=b_initializer,
+          variables_collections=var_collection)
+      weight_vars = ops.get_collection('SLIM_FC_WEIGHTS')
+      self.assertEquals(1, len(weight_vars))
+      bias_vars = ops.get_collection('SLIM_FC_BIASES')
+      self.assertEquals(1, len(bias_vars))
+      expected_normalized_vars = {
+          'slim.fully_connected.weights': weight_vars[0]
+      }
+      expected_not_normalized_vars = {'slim.fully_connected.bias': bias_vars[0]}
+
+      return net, expected_normalized_vars, expected_not_normalized_vars
+
+    self._testLayerHelper(build_layer_fn, (300, 3), (3,))
+
+  def testFC_Keras(self):
+
+    def build_layer_fn(x, w_initializer, b_initializer):
+      x = keras_core.Flatten()(x)
+      layer = keras_core.Dense(
+          units=3,
+          kernel_initializer=w_initializer,
+          bias_initializer=b_initializer)
+      net = layer.apply(x)
+      expected_normalized_vars = {'keras.layers.Dense.kernel': layer.kernel}
+      expected_not_normalized_vars = {'keras.layers.Dense.bias': layer.bias}
+
+      return net, expected_normalized_vars, expected_not_normalized_vars
+
+    self._testLayerHelper(build_layer_fn, (300, 3), (3,), is_keras=True)
+
+
+if __name__ == '__main__':
+  test.main()
-- 
GitLab


From 41c30afbfb6a752f09f9b26db807b1a1ac494e8a Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Thu, 3 Jan 2019 12:07:50 -0800
Subject: [PATCH 0136/2345] Enable kernel tests for ios.

PiperOrigin-RevId: 227727388
---
 tensorflow/core/BUILD                         |   9 +-
 tensorflow/lite/kernels/BUILD                 | 163 ------------------
 .../lite/kernels/batch_to_space_nd_test.cc    |   2 +
 tensorflow/lite/kernels/pad_test.cc           |   8 +
 tensorflow/lite/kernels/reshape_test.cc       |  10 ++
 .../lite/kernels/space_to_batch_nd_test.cc    |   4 +
 .../lite/kernels/space_to_depth_test.cc       |   2 +
 tensorflow/lite/kernels/strided_slice_test.cc |   2 +
 tensorflow/lite/kernels/transpose_test.cc     |   4 +
 9 files changed, 39 insertions(+), 165 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index f9552d5965..6b32e1fead 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -2348,7 +2348,12 @@ cc_library(
 
 cc_library(
     name = "tflite_portable_logging",
-    srcs = [],
+    srcs = [
+    ] + if_ios([
+        "platform/default/logging.cc",
+        "platform/env_time.cc",
+        "platform/posix/env_time.cc",
+    ]),
     hdrs = [
         "lib/bfloat16/bfloat16.h",
         "platform/default/integral_types.h",
@@ -2357,7 +2362,7 @@ cc_library(
         "platform/macros.h",
         "platform/platform.h",
         "platform/types.h",
-    ] + if_windows(["platform/windows/integral_types.h"]),
+    ] + if_windows(["platform/windows/integral_types.h"]) + if_ios(["platform/env_time.h"]),
     copts = tf_copts(),
     linkopts = ["-ldl"],
     deps = [
diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index 5cc06c7a63..9abc33d297 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -25,9 +25,6 @@ tf_cc_test(
     name = "optional_tensor_test",
     size = "small",
     srcs = ["optional_tensor_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -122,9 +119,6 @@ tf_cc_test(
     name = "kernel_util_test",
     size = "small",
     srcs = ["kernel_util_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":kernel_util",
         "//tensorflow/lite/testing:util",
@@ -136,9 +130,6 @@ tf_cc_test(
     name = "test_util_test",
     size = "small",
     srcs = ["test_util_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",  # TODO(b/117786830)
-    ],
     deps = [
         ":test_util",
         "//tensorflow/lite/testing:util",
@@ -304,9 +295,6 @@ tf_cc_test(
     name = "audio_spectrogram_test",
     size = "small",
     srcs = ["audio_spectrogram_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -320,9 +308,6 @@ tf_cc_test(
     name = "mfcc_test",
     size = "small",
     srcs = ["mfcc_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -336,9 +321,6 @@ tf_cc_test(
     name = "detection_postprocess_test",
     size = "small",
     srcs = ["detection_postprocess_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -352,9 +334,6 @@ tf_cc_test(
     name = "relu1_test",
     size = "small",
     srcs = ["relu1_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -368,9 +347,6 @@ tf_cc_test(
     name = "sparse_output_fully_connected_test",
     size = "small",
     srcs = ["sparse_output_fully_connected_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -384,7 +360,6 @@ tf_cc_test(
     name = "activations_test",
     size = "small",
     srcs = ["activations_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -397,7 +372,6 @@ tf_cc_test(
     name = "add_test",
     size = "small",
     srcs = ["add_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -410,9 +384,6 @@ tf_cc_test(
     name = "arg_min_max_test",
     size = "small",
     srcs = ["arg_min_max_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -425,9 +396,6 @@ tf_cc_test(
     name = "div_test",
     size = "small",
     srcs = ["div_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -440,9 +408,6 @@ tf_cc_test(
     name = "sub_test",
     size = "small",
     srcs = ["sub_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -455,9 +420,6 @@ tf_cc_test(
     name = "transpose_test",
     size = "small",
     srcs = ["transpose_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -472,9 +434,6 @@ tf_cc_test(
     name = "space_to_batch_nd_test",
     size = "small",
     srcs = ["space_to_batch_nd_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -487,9 +446,6 @@ tf_cc_test(
     name = "batch_to_space_nd_test",
     size = "small",
     srcs = ["batch_to_space_nd_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -502,9 +458,6 @@ tf_cc_test(
     name = "cast_test",
     size = "small",
     srcs = ["cast_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -517,7 +470,6 @@ tf_cc_test(
     name = "concatenation_test",
     size = "small",
     srcs = ["concatenation_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -530,7 +482,6 @@ tf_cc_test(
     name = "conv_test",
     size = "small",
     srcs = ["conv_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -544,7 +495,6 @@ tf_cc_test(
     name = "depthwise_conv_test",
     size = "small",
     srcs = ["depthwise_conv_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -558,9 +508,6 @@ tf_cc_test(
     name = "dequantize_test",
     size = "small",
     srcs = ["dequantize_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -574,7 +521,6 @@ tf_cc_test(
     name = "basic_rnn_test",
     size = "small",
     srcs = ["basic_rnn_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -587,9 +533,6 @@ tf_cc_test(
     name = "bidirectional_sequence_lstm_test",
     size = "small",
     srcs = ["bidirectional_sequence_lstm_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -603,9 +546,6 @@ tf_cc_test(
     name = "floor_test",
     size = "small",
     srcs = ["floor_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -618,9 +558,6 @@ tf_cc_test(
     name = "elementwise_test",
     size = "small",
     srcs = ["elementwise_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -633,9 +570,6 @@ tf_cc_test(
     name = "unidirectional_sequence_lstm_test",
     size = "small",
     srcs = ["unidirectional_sequence_lstm_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -663,9 +597,6 @@ tf_cc_test(
     name = "unidirectional_sequence_rnn_test",
     size = "small",
     srcs = ["unidirectional_sequence_rnn_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -678,7 +609,6 @@ tf_cc_test(
     name = "l2norm_test",
     size = "small",
     srcs = ["l2norm_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -691,9 +621,6 @@ tf_cc_test(
     name = "exp_test",
     size = "small",
     srcs = ["exp_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -706,9 +633,6 @@ tf_cc_test(
     name = "fake_quant_test",
     size = "small",
     srcs = ["fake_quant_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -721,9 +645,6 @@ tf_cc_test(
     name = "maximum_minimum_test",
     size = "small",
     srcs = ["maximum_minimum_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -736,9 +657,6 @@ tf_cc_test(
     name = "reduce_test",
     size = "small",
     srcs = ["reduce_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -751,7 +669,6 @@ tf_cc_test(
     name = "mul_test",
     size = "small",
     srcs = ["mul_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -764,9 +681,6 @@ tf_cc_test(
     name = "pad_test",
     size = "small",
     srcs = ["pad_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -779,7 +693,6 @@ tf_cc_test(
     name = "reshape_test",
     size = "small",
     srcs = ["reshape_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -792,9 +705,6 @@ tf_cc_test(
     name = "gather_test",
     size = "small",
     srcs = ["gather_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -808,9 +718,6 @@ tf_cc_test(
     name = "topk_v2_test",
     size = "small",
     srcs = ["topk_v2_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -824,7 +731,6 @@ tf_cc_test(
     name = "resize_bilinear_test",
     size = "small",
     srcs = ["resize_bilinear_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -837,7 +743,6 @@ tf_cc_test(
     name = "resize_nearest_neighbor_test",
     size = "small",
     srcs = ["resize_nearest_neighbor_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -850,7 +755,6 @@ tf_cc_test(
     name = "svdf_test",
     size = "small",
     srcs = ["svdf_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -863,7 +767,6 @@ tf_cc_test(
     name = "embedding_lookup_test",
     size = "small",
     srcs = ["embedding_lookup_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -876,7 +779,6 @@ tf_cc_test(
     name = "embedding_lookup_sparse_test",
     size = "small",
     srcs = ["embedding_lookup_sparse_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -889,7 +791,6 @@ tf_cc_test(
     name = "fully_connected_test",
     size = "small",
     srcs = ["fully_connected_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -904,7 +805,6 @@ tf_cc_test(
     name = "local_response_norm_test",
     size = "small",
     srcs = ["local_response_norm_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -917,7 +817,6 @@ tf_cc_test(
     name = "pooling_test",
     size = "small",
     srcs = ["pooling_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -930,7 +829,6 @@ tf_cc_test(
     name = "softmax_test",
     size = "small",
     srcs = ["softmax_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -944,9 +842,6 @@ tf_cc_test(
     name = "log_softmax_test",
     size = "small",
     srcs = ["log_softmax_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -960,7 +855,6 @@ tf_cc_test(
     name = "lsh_projection_test",
     size = "small",
     srcs = ["lsh_projection_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -973,7 +867,6 @@ tf_cc_test(
     name = "hashtable_lookup_test",
     size = "small",
     srcs = ["hashtable_lookup_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -987,7 +880,6 @@ tf_cc_test(
     name = "layer_norm_lstm_test",
     size = "small",
     srcs = ["layer_norm_lstm_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1001,7 +893,6 @@ tf_cc_test(
     name = "lstm_test",
     size = "small",
     srcs = ["lstm_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1014,7 +905,6 @@ tf_cc_test(
     name = "skip_gram_test",
     size = "small",
     srcs = ["skip_gram_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1028,7 +918,6 @@ tf_cc_test(
     name = "space_to_depth_test",
     size = "small",
     srcs = ["space_to_depth_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1041,9 +930,6 @@ tf_cc_test(
     name = "split_test",
     size = "small",
     srcs = ["split_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1056,9 +942,6 @@ tf_cc_test(
     name = "split_v_test",
     size = "small",
     srcs = ["split_v_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1071,9 +954,6 @@ tf_cc_test(
     name = "squeeze_test",
     size = "small",
     srcs = ["squeeze_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1086,9 +966,6 @@ tf_cc_test(
     name = "strided_slice_test",
     size = "small",
     srcs = ["strided_slice_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1101,9 +978,6 @@ tf_cc_test(
     name = "tile_test",
     size = "small",
     srcs = ["tile_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1119,9 +993,6 @@ tf_cc_test(
     srcs = [
         "comparisons_test.cc",
     ],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1134,9 +1005,6 @@ tf_cc_test(
     name = "neg_test",
     size = "small",
     srcs = ["neg_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1151,9 +1019,6 @@ tf_cc_test(
     srcs = [
         "select_test.cc",
     ],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1168,9 +1033,6 @@ tf_cc_test(
     srcs = [
         "slice_test.cc",
     ],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1183,9 +1045,6 @@ tf_cc_test(
     name = "transpose_conv_test",
     size = "small",
     srcs = ["transpose_conv_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1199,9 +1058,6 @@ tf_cc_test(
     name = "expand_dims_test",
     size = "small",
     srcs = ["expand_dims_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1215,9 +1071,6 @@ tf_cc_test(
     name = "sparse_to_dense_test",
     size = "small",
     srcs = ["sparse_to_dense_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1231,9 +1084,6 @@ tf_cc_test(
     name = "shape_test",
     size = "small",
     srcs = ["shape_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1247,9 +1097,6 @@ tf_cc_test(
     name = "pow_test",
     size = "small",
     srcs = ["pow_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1263,7 +1110,6 @@ tf_cc_test(
     name = "pack_test",
     size = "small",
     srcs = ["pack_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1277,7 +1123,6 @@ tf_cc_test(
     name = "one_hot_test",
     size = "small",
     srcs = ["one_hot_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1290,7 +1135,6 @@ tf_cc_test(
     name = "logical_test",
     size = "small",
     srcs = ["logical_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1304,7 +1148,6 @@ tf_cc_test(
     name = "unpack_test",
     size = "small",
     srcs = ["unpack_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:builtin_op_data",
@@ -1318,7 +1161,6 @@ tf_cc_test(
     name = "floor_div_test",
     size = "small",
     srcs = ["floor_div_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:builtin_op_data",
@@ -1332,7 +1174,6 @@ tf_cc_test(
     name = "zeros_like_test",
     size = "small",
     srcs = ["zeros_like_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:builtin_op_data",
@@ -1346,7 +1187,6 @@ tf_cc_test(
     name = "floor_mod_test",
     size = "small",
     srcs = ["floor_mod_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:builtin_op_data",
@@ -1360,7 +1200,6 @@ tf_cc_test(
     name = "range_test",
     size = "small",
     srcs = ["range_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:builtin_op_data",
@@ -1374,7 +1213,6 @@ tf_cc_test(
     name = "squared_difference_test",
     size = "small",
     srcs = ["squared_difference_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1387,7 +1225,6 @@ tf_cc_test(
     name = "fill_test",
     size = "small",
     srcs = ["fill_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
diff --git a/tensorflow/lite/kernels/batch_to_space_nd_test.cc b/tensorflow/lite/kernels/batch_to_space_nd_test.cc
index a3e06d4c89..f330895599 100644
--- a/tensorflow/lite/kernels/batch_to_space_nd_test.cc
+++ b/tensorflow/lite/kernels/batch_to_space_nd_test.cc
@@ -114,6 +114,7 @@ TEST(BatchToSpaceNDOpTest, SimpleDynamicTest) {
                                                4, 8, 11, 15, 12, 16}));
 }
 
+#ifdef GTEST_HAS_DEATH_TEST
 TEST(BatchToSpaceNDOpTest, InvalidShapeTest) {
   EXPECT_DEATH(BatchToSpaceNDOpConstModel({3, 2, 2, 1}, {2, 2}, {0, 0, 0, 0}),
                "Cannot allocate tensors");
@@ -131,6 +132,7 @@ TEST(BatchToSpaceNDOpTest, InvalidCropsDynamicTest) {
   m.SetCrops({0, 0, -1, 0});
   EXPECT_DEATH(m.Invoke(), "crops.2. >= 0 was not true.");
 }
+#endif
 
 }  // namespace
 }  // namespace tflite
diff --git a/tensorflow/lite/kernels/pad_test.cc b/tensorflow/lite/kernels/pad_test.cc
index 415a285c70..3caa4065dc 100644
--- a/tensorflow/lite/kernels/pad_test.cc
+++ b/tensorflow/lite/kernels/pad_test.cc
@@ -175,6 +175,7 @@ class PadOpDynamicModel : public PadOpModel<float> {
   }
 };
 
+#ifdef GTEST_HAS_DEATH_TEST
 TEST(PadOpTest, TooManyDimensions) {
   EXPECT_DEATH(
       PadOpConstModel({TensorType_FLOAT32, {1, 2, 3, 4, 5, 6, 7, 8, 9}}, {9, 2},
@@ -195,6 +196,7 @@ TEST(PadOpTest, InvalidPadValue) {
                       {0, 0, 1, -1, 2, -1, 0, 0}, {TensorType_FLOAT32}),
       "Pad value has to be greater than equal to 0.");
 }
+#endif
 
 TEST(PadOpTest, SimpleConstTest) {
   // Padding is represented as four 2-D lists representing above padding and
@@ -306,6 +308,7 @@ class QuantizedPadOpTest : public ::testing::Test {
   }
 };
 
+#ifdef GTEST_HAS_DEATH_TEST
 TEST_F(QuantizedPadOpTest, ZeroNotInQuantizationRange) {
   // The test_util and actual quantization code currently ensure that the range
   // must include zero, but if that ever changes, this test will catch it.
@@ -314,6 +317,7 @@ TEST_F(QuantizedPadOpTest, ZeroNotInQuantizationRange) {
                                  {TensorType_UINT8, {}, 1.0, 2.0}),
                ".*Check failed: f_min <= 0.*");
 }
+#endif
 
 TEST_F(QuantizedPadOpTest, SimpleConstTest) {
   // Padding is represented as four 2-D lists representing above padding and
@@ -371,6 +375,7 @@ TEST_F(QuantizedPadOpTest, AdvancedDynamicTest) {
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 4, 7, 1}));
 }
 
+#ifdef GTEST_HAS_DEATH_TEST
 TEST(PadV2OpTest, TooManyDimensions) {
   EXPECT_DEATH(PadV2OpConstModel<float>(
                    {TensorType_FLOAT32, {1, 2, 3, 4, 5, 6, 7, 8, 9}}, {9, 2},
@@ -392,6 +397,7 @@ TEST(PadV2OpTest, InvalidPadValue) {
                                         {TensorType_FLOAT32}),
                "Pad value has to be greater than equal to 0.");
 }
+#endif
 
 TEST(PadV2OpTest, SimpleConstTest) {
   // Padding is represented as four 2-D lists representing above padding and
@@ -495,6 +501,7 @@ class QuantizedPadV2OpTest : public ::testing::Test {
   }
 };
 
+#ifdef GTEST_HAS_DEATH_TEST
 TEST_F(QuantizedPadV2OpTest, ZeroNotInQuantizationRange) {
   // The test_util and actual quantization code currently ensure that the range
   // must include zero, but if that ever changes, this test will catch it.
@@ -504,6 +511,7 @@ TEST_F(QuantizedPadV2OpTest, ZeroNotInQuantizationRange) {
                                  {TensorType_UINT8, {}, 1.0, 2.0}),
       ".*Check failed: f_min <= 0.*");
 }
+#endif
 
 TEST_F(QuantizedPadV2OpTest, SimpleConstTest) {
   // Padding is represented as four 2-D lists representing above padding and
diff --git a/tensorflow/lite/kernels/reshape_test.cc b/tensorflow/lite/kernels/reshape_test.cc
index 00bbbef57e..f98f3eb9ae 100644
--- a/tensorflow/lite/kernels/reshape_test.cc
+++ b/tensorflow/lite/kernels/reshape_test.cc
@@ -123,6 +123,7 @@ class ReshapeOpModel : public SingleOpModel {
   int output_;
 };
 
+#ifdef GTEST_HAS_DEATH_TEST
 TEST_P(ReshapeOpTest, MismatchedDimensions) {
   if (GetParam() == kAsTensor) {
     ReshapeOpModel<float> m({1, 2, 4, 1}, {2}, {2, 1}, GetParam());
@@ -133,12 +134,15 @@ TEST_P(ReshapeOpTest, MismatchedDimensions) {
                  "num_input_elements != num_output_elements");
   }
 }
+#endif
 
 TEST_P(ReshapeOpTest, TooManyDimensions) {
   if (GetParam() == kAsReshapeOption) {
+#ifdef GTEST_HAS_DEATH_TEST
     EXPECT_DEATH(ReshapeOpModel<float>({1, 1, 2, 1, 1, 1, 1, 1, 1}, {9},
                                        {1, 1, 1, 1, 1, 1, 1, 1, 2}, GetParam()),
                  "Found too many dimensions");
+#endif
   } else {
     ReshapeOpModel<float> m({1, 1, 2, 1, 1, 1, 1, 1, 1}, {9},
                             {1, 1, 1, 1, 1, 1, 1, 1, 2}, GetParam());
@@ -150,6 +154,7 @@ TEST_P(ReshapeOpTest, TooManyDimensions) {
   }
 }
 
+#ifdef GTEST_HAS_DEATH_TEST
 TEST_P(ReshapeOpTest, TooManySpecialDimensions) {
   if (GetParam() != kAsTensor) {
     EXPECT_DEATH(
@@ -160,6 +165,7 @@ TEST_P(ReshapeOpTest, TooManySpecialDimensions) {
     EXPECT_DEATH(m.Invoke(), "stretch_dim != -1");
   }
 }
+#endif
 
 // Create the model with a 2x2 shape. Processing still works because the new
 // shape ends up being hardcoded as a flat vector.
@@ -202,12 +208,16 @@ TEST_P(ReshapeOpTest, ScalarOutput) {
 // and output are scalars.
 TEST_P(ReshapeOpTest, LegacyScalarOutput) {
   if (GetParam() == kAsConstantTensor) {
+#ifdef GTEST_HAS_DEATH_TEST
     EXPECT_DEATH(ReshapeOpModel<float>({1}, {1}, {0}, GetParam()),
                  "num_input_elements != num_output_elements");
+#endif
   } else if (GetParam() == kAsTensor) {
+#ifdef GTEST_HAS_DEATH_TEST
     ReshapeOpModel<float> m({1}, {1}, {0}, GetParam());
     m.SetInput({3});
     EXPECT_DEATH(m.Invoke(), "num_input_elements != num_output_elements");
+#endif
   } else {
     ReshapeOpModel<float> m({1}, {1}, {0}, GetParam());
     m.SetInput({3});
diff --git a/tensorflow/lite/kernels/space_to_batch_nd_test.cc b/tensorflow/lite/kernels/space_to_batch_nd_test.cc
index 4d55ba56b7..c5d6e9a530 100644
--- a/tensorflow/lite/kernels/space_to_batch_nd_test.cc
+++ b/tensorflow/lite/kernels/space_to_batch_nd_test.cc
@@ -106,12 +106,14 @@ class SpaceToBatchNDOpDynamicModel : public SpaceToBatchNDOpModel {
   }
 };
 
+#ifdef GTEST_HAS_DEATH_TEST
 TEST(SpaceToBatchNDOpTest, InvalidShapeTest) {
   EXPECT_DEATH(
       SpaceToBatchNDOpConstModel({TensorType_FLOAT32, {1, 3, 3, 1}}, {2, 2},
                                  {0, 0, 0, 0}, {TensorType_FLOAT32}),
       "Cannot allocate tensors");
 }
+#endif
 
 TEST(SpaceToBatchNDOpTest, SimpleConstTest) {
   SpaceToBatchNDOpConstModel m({TensorType_FLOAT32, {1, 4, 4, 1}}, {2, 2},
@@ -220,6 +222,7 @@ class QuantizedSpaceToBatchNDOpTest : public ::testing::Test {
   }
 };
 
+#ifdef GTEST_HAS_DEATH_TEST
 TEST_F(QuantizedSpaceToBatchNDOpTest, ZeroNotInQuantizationRange) {
   // The test_util and actual quantization code currently ensure that the range
   // must include zero, but if that ever changes, this test will catch it.
@@ -228,6 +231,7 @@ TEST_F(QuantizedSpaceToBatchNDOpTest, ZeroNotInQuantizationRange) {
                    {0, 0, 1, 1, 1, 1, 0, 0}, {TensorType_UINT8, {}, 1.0, 2.0}),
                ".*Check failed: f_min <= 0.*");
 }
+#endif
 
 TEST_F(QuantizedSpaceToBatchNDOpTest, SimplePaddingConstTest) {
   SpaceToBatchNDOpConstModel m({TensorType_UINT8, {1, 5, 2, 1}, -1.0, 1.0},
diff --git a/tensorflow/lite/kernels/space_to_depth_test.cc b/tensorflow/lite/kernels/space_to_depth_test.cc
index 5744669b6d..3fa8d86348 100644
--- a/tensorflow/lite/kernels/space_to_depth_test.cc
+++ b/tensorflow/lite/kernels/space_to_depth_test.cc
@@ -50,10 +50,12 @@ class SpaceToDepthOpModel : public SingleOpModel {
   int output_;
 };
 
+#ifdef GTEST_HAS_DEATH_TEST
 TEST(SpaceToDepthOpModel, BadBlockSize) {
   EXPECT_DEATH(SpaceToDepthOpModel({TensorType_FLOAT32, {1, 2, 2, 1}}, 3),
                "Cannot allocate tensors");
 }
+#endif
 
 TEST(SpaceToDepthOpModel, Float32) {
   SpaceToDepthOpModel m({TensorType_FLOAT32, {1, 2, 2, 2}}, 2);
diff --git a/tensorflow/lite/kernels/strided_slice_test.cc b/tensorflow/lite/kernels/strided_slice_test.cc
index 122e01b99e..34875bf049 100644
--- a/tensorflow/lite/kernels/strided_slice_test.cc
+++ b/tensorflow/lite/kernels/strided_slice_test.cc
@@ -72,6 +72,7 @@ class StridedSliceOpModel : public SingleOpModel {
   int output_;
 };
 
+#ifdef GTEST_HAS_DEATH_TEST
 TEST(StridedSliceOpTest, UnsupportedInputSize) {
   EXPECT_DEATH(
       StridedSliceOpModel<>({2, 2, 2, 2, 2}, {5}, {5}, {5}, 0, 0, 0, 0, 0),
@@ -84,6 +85,7 @@ TEST(StridedSliceOpTest, UnssupportedArgs) {
   EXPECT_DEATH(StridedSliceOpModel<>({3, 2}, {2}, {2}, {2}, 0, 0, 0, 1, 0),
                "new_axis_mask is not implemented yet.");
 }
+#endif
 
 TEST(StridedSliceOpTest, In1D) {
   StridedSliceOpModel<> m({4}, {1}, {1}, {1}, 0, 0, 0, 0, 0);
diff --git a/tensorflow/lite/kernels/transpose_test.cc b/tensorflow/lite/kernels/transpose_test.cc
index 3ebaf3ca27..93df2c81db 100644
--- a/tensorflow/lite/kernels/transpose_test.cc
+++ b/tensorflow/lite/kernels/transpose_test.cc
@@ -184,6 +184,7 @@ class TransposeOpDynamicModel : public TransposeOpModel {
   }
 };
 
+#ifdef GTEST_HAS_DEATH_TEST
 TEST(TransposeTest, TestUnequalPermSize) {
   EXPECT_DEATH(TransposeOpConstModel({1, 3, 3, 1}, {2}, {2, 2}), "2 != 4");
 }
@@ -194,6 +195,7 @@ TEST(TransposeTest, TestPermOutOfBounds) {
   EXPECT_DEATH(TransposeOpConstModel({1, 3, 3, 1}, {4}, {0, 1, 2, 4}),
                "Transpose op permutations array is out of bounds.");
 }
+#endif
 
 TEST(TransposeTest, Test1DInputConstTensor) {
   TransposeOpConstModel m({3}, {1}, {0});
@@ -252,10 +254,12 @@ TEST(TransposeTest, Test3DInputDynamicTensor) {
                                 2, 6, 10, 14, 18, 22, 3, 7, 11, 15, 19, 23}));
 }
 
+#ifdef GTEST_HAS_DEATH_TEST
 TEST(TransposeTest, Test5DInputTensor) {
   EXPECT_DEATH(TransposeOpConstModel({1, 2, 3, 4, 5}, {5}, {0, 1, 2, 3, 4}),
                "Transpose op only supports 1D-4D input arrays.");
 }
+#endif
 
 TEST(TransposeTest, SimpleTestNoReorderConstTensor) {
   TransposeOpConstModel m({1, 2, 3, 1}, {4}, {0, 1, 2, 3});
-- 
GitLab


From aa36aa0a912a4addfb387e5fc0f759db225d03cc Mon Sep 17 00:00:00 2001
From: Davide Libenzi <dlibenzi@google.com>
Date: Thu, 3 Jan 2019 12:26:05 -0800
Subject: [PATCH 0137/2345] Allow XRT release operations to support vectors of
 keys.

PiperOrigin-RevId: 227730040
---
 .../compiler/xrt/kernels/xrt_compile_ops.cc   | 15 ++-
 .../compiler/xrt/kernels/xrt_state_ops.h      | 16 +--
 .../compiler/xrt/ops/xrt_compile_ops.cc       |  6 +-
 tensorflow/compiler/xrt/ops/xrt_state_ops.cc  |  5 +-
 tensorflow/compiler/xrt/tests/raw_api_test.cc | 98 ++++++++++++++++++-
 5 files changed, 117 insertions(+), 23 deletions(-)

diff --git a/tensorflow/compiler/xrt/kernels/xrt_compile_ops.cc b/tensorflow/compiler/xrt/kernels/xrt_compile_ops.cc
index 2ccdf0f02d..2ee1a6cd1a 100644
--- a/tensorflow/compiler/xrt/kernels/xrt_compile_ops.cc
+++ b/tensorflow/compiler/xrt/kernels/xrt_compile_ops.cc
@@ -215,11 +215,6 @@ XRTReleaseCompilationRefOp::~XRTReleaseCompilationRefOp() = default;
 void XRTReleaseCompilationRefOp::Compute(OpKernelContext* ctx) {
   VLOG(1) << "XRTReleaseCompilationRefOp::Compute";
 
-  const Tensor& key_tensor = ctx->input(0);
-  OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(key_tensor.shape()),
-              errors::Internal("computation key should be a string scalar"));
-  int64 uid = key_tensor.scalar<int64>()();
-
   ResourceMgr* rm;
   OP_REQUIRES_OK(ctx, XRTGenericDeviceAccessor::GetResourceManager(ctx, &rm));
 
@@ -230,9 +225,13 @@ void XRTReleaseCompilationRefOp::Compute(OpKernelContext* ctx) {
                           kXRTCompilationCacheResourceName, &cache));
   core::ScopedUnref cache_unref(cache);
 
-  OP_REQUIRES_OK(ctx, cache->Release(uid));
-
-  VLOG(2) << "Released computation handle " << uid;
+  const Tensor& keys_tensor = ctx->input(0);
+  auto flat_keys = keys_tensor.flat<int64>();
+  for (int64 i = 0; i < flat_keys.size(); ++i) {
+    int64 key = flat_keys(i);
+    OP_REQUIRES_OK(ctx, cache->Release(key));
+    VLOG(2) << "Released computation handle " << key;
+  }
 }
 
 }  // namespace
diff --git a/tensorflow/compiler/xrt/kernels/xrt_state_ops.h b/tensorflow/compiler/xrt/kernels/xrt_state_ops.h
index 2e2f3ff116..c8def16bbc 100644
--- a/tensorflow/compiler/xrt/kernels/xrt_state_ops.h
+++ b/tensorflow/compiler/xrt/kernels/xrt_state_ops.h
@@ -453,17 +453,17 @@ class XRTReleaseAllocationOp : public OpKernel {
   void Compute(OpKernelContext* ctx) override {
     VLOG(1) << "XRTReleaseAllocationOp::Compute";
 
-    const Tensor& allocation_handle = ctx->input(0);
-    OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(allocation_handle.shape()),
-                errors::Internal("handle input should be an int64 scalar"));
-    int64 key = allocation_handle.scalar<int64>()();
-
     ResourceMgr* rm;
     OP_REQUIRES_OK(ctx, DeviceAccessor::GetResourceManager(ctx, &rm));
 
-    OP_REQUIRES_OK(ctx, XRTTupleAllocation::DeleteFromResourceManager(rm, key));
-
-    VLOG(2) << "Released allocation handle " << key;
+    const Tensor& allocation_handle = ctx->input(0);
+    auto flat_keys = allocation_handle.flat<int64>();
+    for (int64 i = 0; i < flat_keys.size(); ++i) {
+      int64 key = flat_keys(i);
+      OP_REQUIRES_OK(ctx,
+                     XRTTupleAllocation::DeleteFromResourceManager(rm, key));
+      VLOG(2) << "Released allocation handle " << key;
+    }
   }
 };
 
diff --git a/tensorflow/compiler/xrt/ops/xrt_compile_ops.cc b/tensorflow/compiler/xrt/ops/xrt_compile_ops.cc
index 7b3b50c695..9dd964e546 100644
--- a/tensorflow/compiler/xrt/ops/xrt_compile_ops.cc
+++ b/tensorflow/compiler/xrt/ops/xrt_compile_ops.cc
@@ -44,10 +44,10 @@ REGISTER_OP("XRTReleaseCompilationHandle")
     .SetShapeFn(tensorflow::shape_inference::NoOutputs)
     .Doc(
         R"(
-Discards a computation from the compilation cache. The handle cannot be
-subsequently used.
+Discards one or more computation handles from the compilation cache.
+The handle(s) cannot be subsequently used.
 
-'handle' is an id returned from a XRTCompile Op.
+'handle' is an ID (or vector of IDs) returned from a XRTCompile Op.
 )");
 
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/xrt/ops/xrt_state_ops.cc b/tensorflow/compiler/xrt/ops/xrt_state_ops.cc
index fe6bee0dac..db58f0797d 100644
--- a/tensorflow/compiler/xrt/ops/xrt_state_ops.cc
+++ b/tensorflow/compiler/xrt/ops/xrt_state_ops.cc
@@ -127,10 +127,11 @@ REGISTER_OP("XRTReleaseAllocationHandle")
     .SetShapeFn(tensorflow::shape_inference::NoOutputs)
     .Doc(
         R"(
-Discards an allocation from device memory. The handle cannot be subsequently
+Discards one or more device memory handles. The handle(s) cannot be subsequently
 used.
 
-'handle' is the id returned from the Op that produced the on-device allocation.
+'handle' is the ID (or a vector of IDs) returned from the Op that produced the
+on-device allocation.
 )");
 
 REGISTER_OP("XRTReleaseAllAllocations")
diff --git a/tensorflow/compiler/xrt/tests/raw_api_test.cc b/tensorflow/compiler/xrt/tests/raw_api_test.cc
index 5f8121703e..c8479cb778 100644
--- a/tensorflow/compiler/xrt/tests/raw_api_test.cc
+++ b/tensorflow/compiler/xrt/tests/raw_api_test.cc
@@ -258,8 +258,102 @@ TEST(RawApiTest, AllocAndRewrite) {
   EXPECT_TRUE(new_response.ParseFromString(outputs[0].scalar<string>()()));
   EXPECT_TRUE(CompareLiteralProtos(new_literal, new_response));
 
-  auto release =
-      ops::XRTReleaseAllocationHandle(root, Input(allocation_handle));
+  Tensor release_tensor(DT_INT64, TensorShape({1}));
+  release_tensor.flat<int64>()(0) = allocation_handle;
+
+  auto release = ops::XRTReleaseAllocationHandle(root, release_tensor);
+  TF_EXPECT_OK(session.Run(tensorflow::ClientSession::FeedType(), {}, {release},
+                           &outputs));
+}
+
+TEST(RawApiTest, AllocReleaseMany) {
+  xrt::XLAAllocation alloc1;
+  *alloc1.mutable_value() =
+      xla::LiteralUtil::CreateR2({{4, 5}, {6, 7}}).ToProto();
+  xrt::XLAAllocation alloc2;
+  *alloc2.mutable_value() =
+      xla::LiteralUtil::CreateR2({{6, 7}, {4, 5}}).ToProto();
+
+  Scope root = Scope::NewRootScope().WithDevice(DeviceFromFlag());
+  auto value1 =
+      ops::Const(root.WithDevice("/device:CPU:0"), alloc1.SerializeAsString());
+  auto value2 =
+      ops::Const(root.WithDevice("/device:CPU:0"), alloc2.SerializeAsString());
+  auto handle1 = ops::XRTAllocate(root, value1);
+  auto handle2 = ops::XRTAllocate(root, value2);
+  TF_ASSERT_OK(root.status());
+
+  tensorflow::ClientSession session(root);
+  std::vector<tensorflow::Tensor> outputs;
+  TF_EXPECT_OK(session.Run({handle1, handle2}, &outputs));
+  EXPECT_EQ(outputs.size(), 2);
+
+  int64 allocation_handle1 = outputs[0].scalar<int64>()();
+  int64 allocation_handle2 = outputs[1].scalar<int64>()();
+
+  Tensor release_tensor(DT_INT64, TensorShape({2}));
+  release_tensor.flat<int64>()(0) = allocation_handle1;
+  release_tensor.flat<int64>()(1) = allocation_handle2;
+
+  auto release = ops::XRTReleaseAllocationHandle(root, release_tensor);
+  outputs.clear();
+  TF_EXPECT_OK(session.Run(tensorflow::ClientSession::FeedType(), {}, {release},
+                           &outputs));
+}
+
+TEST(RawApiTest, CompileAndReleaseMany) {
+  xrt::XLAComputation c1;
+  auto config1 = c1.mutable_config();
+  auto shapes1 = config1->mutable_program_shape();
+  *shapes1->add_parameters() =
+      xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto();
+  *shapes1->add_parameters() =
+      xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto();
+  *shapes1->mutable_result() =
+      xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto();
+  StoreComputationSnapshot(AddAndScale(), c1.mutable_hlo_snapshot());
+
+  xrt::XLAComputation c2;
+  auto config2 = c2.mutable_config();
+  auto shapes2 = config2->mutable_program_shape();
+  *shapes2->add_parameters() =
+      xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto();
+  *shapes2->add_parameters() =
+      xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto();
+  *shapes2->mutable_result() =
+      xla::ShapeUtil::MakeTupleShape({xla::ShapeUtil::MakeShape(xla::F32, {2})})
+          .ToProto();
+  StoreComputationSnapshot(AddAndTuple(), c2.mutable_hlo_snapshot());
+
+  xrt::XRTExecutionConfig e;
+  e.set_release_input_handles(true);
+  e.set_release_compilation_handle(false);
+
+  Scope root = Scope::NewRootScope().WithDevice(DeviceFromFlag());
+  auto e_config =
+      ops::Const(root.WithDevice("/device:CPU:0"), e.SerializeAsString());
+  auto computation1 =
+      ops::Const(root.WithDevice("/device:CPU:0"), c1.SerializeAsString());
+  auto c_handle1 = ops::XRTCompile(root, computation1);
+  auto computation2 =
+      ops::Const(root.WithDevice("/device:CPU:0"), c2.SerializeAsString());
+  auto c_handle2 = ops::XRTCompile(root, computation2);
+  TF_ASSERT_OK(root.status());
+
+  ClientSession session(root);
+  std::vector<Tensor> outputs;
+  TF_EXPECT_OK(session.Run({c_handle1.handle, c_handle2.handle}, &outputs));
+  EXPECT_EQ(outputs.size(), 2);
+
+  int64 compilation_handle1 = outputs[0].scalar<int64>()();
+  int64 compilation_handle2 = outputs[1].scalar<int64>()();
+
+  Tensor release_tensor(DT_INT64, TensorShape({2}));
+  release_tensor.flat<int64>()(0) = compilation_handle1;
+  release_tensor.flat<int64>()(1) = compilation_handle2;
+
+  auto release = ops::XRTReleaseCompilationHandle(root, release_tensor);
+  outputs.clear();
   TF_EXPECT_OK(session.Run(tensorflow::ClientSession::FeedType(), {}, {release},
                            &outputs));
 }
-- 
GitLab


From cb4f2d41f22b6df49c80555a911d7dc2f08b7b90 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 12:27:49 -0800
Subject: [PATCH 0138/2345] Fix autograph detection of tensors that are wrapped
 in python structures.

A dict with tensor values did not register as containing a Tensor, causing the python branch to be taken instead of the tf branch.

PiperOrigin-RevId: 227730234
---
 tensorflow/python/autograph/operators/BUILD   |  1 +
 .../autograph/operators/control_flow.py       |  4 ++-
 .../autograph/operators/control_flow_test.py  | 30 +++++++++++++------
 3 files changed, 25 insertions(+), 10 deletions(-)

diff --git a/tensorflow/python/autograph/operators/BUILD b/tensorflow/python/autograph/operators/BUILD
index aedb901845..21a66c86b7 100644
--- a/tensorflow/python/autograph/operators/BUILD
+++ b/tensorflow/python/autograph/operators/BUILD
@@ -38,6 +38,7 @@ py_library(
         "//tensorflow/python:list_ops",
         "//tensorflow/python:tensor_array_ops",
         "//tensorflow/python:tensor_util",
+        "//tensorflow/python:util",
         "//tensorflow/python:variables",
         "//tensorflow/python/autograph/utils",
         "//tensorflow/python/data/ops:dataset_ops",
diff --git a/tensorflow/python/autograph/operators/control_flow.py b/tensorflow/python/autograph/operators/control_flow.py
index afa3787d42..035ea1bd92 100644
--- a/tensorflow/python/autograph/operators/control_flow.py
+++ b/tensorflow/python/autograph/operators/control_flow.py
@@ -23,6 +23,7 @@ from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gen_math_ops
+from tensorflow.python.util import nest
 
 
 def for_stmt(iter_, extra_test, body, init_state):
@@ -160,7 +161,8 @@ def while_stmt(test, body, init_state, extra_deps, opts=None):
   # TODO(mdan): Consider adding a generic mechanism for dynamic dispatch.
   # That could be something as simple as a collection of dispatch rules, with
   # some prioritization.
-  if any(tensor_util.is_tensor(v) for v in init_state + extra_deps):
+  if any(tensor_util.is_tensor(v)
+         for v in nest.flatten(init_state + extra_deps)):
     return _tf_while_stmt(test, body, init_state, opts)
   else:
     return _py_while_stmt(test, body, init_state, opts)
diff --git a/tensorflow/python/autograph/operators/control_flow_test.py b/tensorflow/python/autograph/operators/control_flow_test.py
index 0a7d4b6402..f9e006f7ad 100644
--- a/tensorflow/python/autograph/operators/control_flow_test.py
+++ b/tensorflow/python/autograph/operators/control_flow_test.py
@@ -36,7 +36,7 @@ class ForLoopTest(test.TestCase):
         extra_test=lambda s: True,
         body=lambda i, s: (s + i,),
         init_state=(0,))
-    with self.cached_session() as sess:
+    with self.cached_session():
       self.assertEqual((10,), self.evaluate(s))
 
   def test_python(self):
@@ -55,7 +55,7 @@ class ForLoopTest(test.TestCase):
         extra_test=lambda s: True,
         body=lambda i, s: (s + i,),
         init_state=(0,))
-    with self.cached_session() as sess:
+    with self.cached_session():
       self.assertEqual((10,), self.evaluate(s))
 
 
@@ -65,18 +65,30 @@ class WhileLoopTest(test.TestCase):
   def test_tensor(self):
     n = constant_op.constant(5)
     results = control_flow.while_stmt(
-        test=lambda i, s: i < n,
-        body=lambda i, s: (i + 1, s + i,),
+        test=lambda i, sum: i < n,
+        body=lambda i, sum: (i + 1, sum + i,),
         init_state=(0, 0),
         extra_deps=(n,))
-    with self.cached_session() as sess:
+    with self.cached_session():
       self.assertEqual((5, 10), self.evaluate(results))
 
+  @test_util.run_deprecated_v1
+  def test_tensor_dict_state(self):
+    n = 5
+    init_state = {'i': constant_op.constant(0), 'sum': constant_op.constant(0)}
+    results = control_flow.while_stmt(
+        test=lambda s: s['i'] < n,
+        body=lambda s: ({'i': s['i'] + 1, 'sum': s['sum'] + s['i']},),
+        init_state=(init_state,),
+        extra_deps=())
+    with self.cached_session():
+      self.assertEqual(({'i': 5, 'sum': 10},), self.evaluate(results))
+
   def test_python(self):
     n = 5
     results = control_flow.while_stmt(
-        test=lambda i, s: i < n,
-        body=lambda i, s: (i + 1, s + i),
+        test=lambda i, sum: i < n,
+        body=lambda i, sum: (i + 1, sum + i),
         init_state=(0, 0),
         extra_deps=(n,))
     self.assertEqual((5, 10), results)
@@ -93,7 +105,7 @@ class IfStmtTest(test.TestCase):
 
   @test_util.run_deprecated_v1
   def test_tensor(self):
-    with self.cached_session() as sess:
+    with self.cached_session():
       t = self.single_return_if_stmt(constant_op.constant(True))
       self.assertEqual(1, self.evaluate(t))
       t = self.single_return_if_stmt(constant_op.constant(False))
@@ -105,7 +117,7 @@ class IfStmtTest(test.TestCase):
 
   @test_util.run_deprecated_v1
   def test_tensor_multiple_returns(self):
-    with self.cached_session() as sess:
+    with self.cached_session():
       t = self.multi_return_if_stmt(constant_op.constant(True))
       self.assertAllEqual([1, 2], self.evaluate(t))
       t = self.multi_return_if_stmt(constant_op.constant(False))
-- 
GitLab


From 7e2d53c1c371f38c7f0ef13c1c06336b22a195c0 Mon Sep 17 00:00:00 2001
From: Shivani Agrawal <shivaniagrawal@google.com>
Date: Thu, 3 Jan 2019 12:46:47 -0800
Subject: [PATCH 0139/2345] [tf.data] Adds the expected check for better
 debugging.

PiperOrigin-RevId: 227733165
---
 tensorflow/core/kernels/data/experimental/scan_dataset_op.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc b/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc
index 76ab33fe98..87263b5606 100644
--- a/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc
@@ -186,6 +186,8 @@ class ScanDatasetOp : public UnaryDatasetOpKernel {
 
         Status s = instantiated_captured_func_->Run(ctx, std::move(args),
                                                     &state_and_output);
+        DCHECK(state_and_output.size() <=
+               dataset()->state_types_.size() + output_dtypes().size());
         if (s.ok()) {
           state_.clear();
           size_t i = 0;
-- 
GitLab


From f9d4786119cda668d9b05d4ae49f2a4fe8d5738f Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Thu, 3 Jan 2019 13:01:24 -0800
Subject: [PATCH 0140/2345] Make tests all have their own main so
 initialization works.

PiperOrigin-RevId: 227735303
---
 tensorflow/lite/toco/BUILD                     | 12 ++++++++----
 tensorflow/lite/toco/import_tensorflow_test.cc |  7 +++++++
 tensorflow/lite/toco/toco_convert_test.cc      |  7 +++++++
 tensorflow/lite/toco/toco_port_test.cc         |  7 +++++++
 tensorflow/lite/toco/tooling_util_test.cc      |  9 ++++++++-
 5 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/tensorflow/lite/toco/BUILD b/tensorflow/lite/toco/BUILD
index 93d41fcae1..d77234c80a 100644
--- a/tensorflow/lite/toco/BUILD
+++ b/tensorflow/lite/toco/BUILD
@@ -348,7 +348,8 @@ tf_cc_test(
         "//tensorflow/core:lib",
         "//tensorflow/core:ops",
         "//tensorflow/core:protos_all_cc",
-        "@com_google_googletest//:gtest_main",
+        "//tensorflow/lite/testing:util",
+        "@com_google_googletest//:gtest",
     ],
 )
 
@@ -387,7 +388,8 @@ tf_cc_test(
         ":model",
         ":tooling_util",
         "//tensorflow/core:lib",
-        "@com_google_googletest//:gtest_main",
+        "//tensorflow/lite/testing:util",
+        "@com_google_googletest//:gtest",
     ],
 )
 
@@ -451,12 +453,13 @@ tf_cc_test(
         ":toco_port",
         ":toco_tooling",
         ":types_proto_cc",
-        "@com_google_googletest//:gtest_main",
+        "@com_google_googletest//:gtest",
         "@com_google_absl//absl/strings",
         "//tensorflow/core:lib",
         # We cannot embed the core:ops dependency directly into :toco_tooling as
         # it can conflict with downstream deps when toco is used as a library.
         "//tensorflow/core:ops",
+        "//tensorflow/lite/testing:util",
     ],
 )
 
@@ -468,6 +471,7 @@ tf_cc_test(
     ],
     deps = [
         ":toco_port",
-        "@com_google_googletest//:gtest_main",
+        "//tensorflow/lite/testing:util",
+        "@com_google_googletest//:gtest",
     ],
 )
diff --git a/tensorflow/lite/toco/import_tensorflow_test.cc b/tensorflow/lite/toco/import_tensorflow_test.cc
index 260704fd2a..70382f546b 100644
--- a/tensorflow/lite/toco/import_tensorflow_test.cc
+++ b/tensorflow/lite/toco/import_tensorflow_test.cc
@@ -23,6 +23,7 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor.pb.h"
 #include "tensorflow/core/framework/tensor_shape.pb.h"
 #include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/lite/testing/util.h"
 
 namespace toco {
 
@@ -564,3 +565,9 @@ TEST(ImportTest, UnsupportedOpWithMultipleOutputs) {
 
 }  // namespace
 }  // namespace toco
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/lite/toco/toco_convert_test.cc b/tensorflow/lite/toco/toco_convert_test.cc
index c3c440db94..3730c53ae1 100644
--- a/tensorflow/lite/toco/toco_convert_test.cc
+++ b/tensorflow/lite/toco/toco_convert_test.cc
@@ -15,6 +15,7 @@ limitations under the License.
 #include "tensorflow/lite/toco/toco_convert.h"
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
+#include "tensorflow/lite/testing/util.h"
 
 namespace toco {
 namespace {
@@ -171,3 +172,9 @@ TEST(TocoTest, TransientStringTensors) {
 
 }  // namespace
 }  // namespace toco
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/lite/toco/toco_port_test.cc b/tensorflow/lite/toco/toco_port_test.cc
index f5fbb4caeb..d80d423ed7 100644
--- a/tensorflow/lite/toco/toco_port_test.cc
+++ b/tensorflow/lite/toco/toco_port_test.cc
@@ -13,6 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 #include "tensorflow/lite/toco/toco_port.h"
+#include "tensorflow/lite/testing/util.h"
 #include "tensorflow/lite/toco/toco_types.h"
 
 #include <gmock/gmock.h>
@@ -56,3 +57,9 @@ TEST(TocoPortTest, JoinPath) {
 }  // namespace
 }  // namespace port
 }  // namespace toco
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/lite/toco/tooling_util_test.cc b/tensorflow/lite/toco/tooling_util_test.cc
index 6f1c9c563a..e44b94b712 100644
--- a/tensorflow/lite/toco/tooling_util_test.cc
+++ b/tensorflow/lite/toco/tooling_util_test.cc
@@ -16,9 +16,10 @@ limitations under the License.
 #include <vector>
 
 #include <gtest/gtest.h>
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/lite/testing/util.h"
 #include "tensorflow/lite/toco/model.h"
 #include "tensorflow/lite/toco/tooling_util.h"
-#include "tensorflow/core/lib/core/status.h"
 
 namespace toco {
 
@@ -203,3 +204,9 @@ TEST(FusedActivationTest, DefaultsToUnfused) {
 }
 
 }  // namespace toco
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
-- 
GitLab


From 689db4ac81b13279677ceefd84700de3d0ec9925 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Thu, 3 Jan 2019 13:29:58 -0800
Subject: [PATCH 0141/2345] Replace SimpleGraphView with GraphTopologyView.

+ added separate enter/advance predicates for DFS traversal

PiperOrigin-RevId: 227740260
---
 tensorflow/core/grappler/optimizers/BUILD     |   7 +
 .../optimizers/arithmetic_optimizer.cc        |  47 ++---
 .../grappler/optimizers/loop_optimizer.cc     |  40 +++--
 .../grappler/optimizers/memory_optimizer.cc   |  67 ++++---
 tensorflow/core/grappler/utils.cc             | 169 ------------------
 tensorflow/core/grappler/utils.h              |  73 --------
 tensorflow/core/grappler/utils/traversal.cc   |  19 +-
 tensorflow/core/grappler/utils/traversal.h    |  39 +++-
 .../core/grappler/utils/traversal_test.cc     |  46 ++++-
 9 files changed, 184 insertions(+), 323 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 512c3b07b4..e2b9de7cc5 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -254,12 +254,16 @@ cc_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core/grappler:graph_topology_view",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/costs:graph_properties",
         "//tensorflow/core/grappler/utils:symbolic_shapes",
         "//tensorflow/core/grappler/utils:topological_sort",
+        "//tensorflow/core/grappler/utils:traversal",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/container:flat_hash_set",
     ],
 )
 
@@ -411,6 +415,7 @@ cc_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core/grappler:graph_topology_view",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:op_types",
@@ -613,11 +618,13 @@ cc_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core/grappler:graph_topology_view",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler/costs:graph_properties",
         "//tensorflow/core/grappler/utils:frame",
+        "//tensorflow/core/grappler/utils:traversal",
         "@com_google_absl//absl/container:flat_hash_set",
     ],
 )
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index 94a87c3ff2..e85e0473b3 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -22,6 +22,8 @@ limitations under the License.
 #include <unordered_set>
 #include <vector>
 
+#include "absl/container/flat_hash_map.h"
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/attr_value_util.h"
 #include "tensorflow/core/framework/node_def.pb.h"
@@ -31,6 +33,7 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_shape.pb.h"
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/grappler/costs/graph_properties.h"
+#include "tensorflow/core/grappler/graph_topology_view.h"
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/grappler/optimizers/constant_folding.h"
@@ -38,6 +41,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/utils.h"
 #include "tensorflow/core/grappler/utils/symbolic_shapes.h"
 #include "tensorflow/core/grappler/utils/topological_sort.h"
+#include "tensorflow/core/grappler/utils/traversal.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
 #include "tensorflow/core/lib/hash/hash.h"
@@ -3334,36 +3338,37 @@ bool ArithmeticOptimizer::CanDedup(const NodeDef& node) const {
 }
 
 void ArithmeticOptimizer::DedupComputations() {
-  bool stop = true;
-  SimpleGraphView graph_view;
-  if (!graph_view.Initialize(*optimized_graph_).ok()) {
-    LOG(WARNING) << "Failed to build SimpleGraphView.";
+  GraphTopologyView graph_view;
+  if (!graph_view.InitializeFromGraph(*optimized_graph_).ok()) {
+    LOG(WARNING) << "Failed to initialize GraphTopologyView.";
     return;
   }
-  std::set<int> duplicates;
+
+  const absl::flat_hash_set<string> ops_to_traverse = {
+      "Identity", "IdentityN", "Reshape", "ExpandDims",
+      "Enter",    "Switch",    "Merge"};
+
   // Populate feed_inplace_op;
-  std::unordered_set<const NodeDef*> feeds_inplace_op;
-  for (int i = 0; i < optimized_graph_->node_size(); ++i) {
-    const NodeDef& root = optimized_graph_->node(i);
-    if (feeds_inplace_op.find(&root) != feeds_inplace_op.end()) {
-      continue;
-    }
+  absl::flat_hash_set<const NodeDef*> feeds_inplace_op;
+
+  for (const NodeDef& root : optimized_graph_->node()) {
+    if (feeds_inplace_op.find(&root) != feeds_inplace_op.end()) continue;
 
     if (ModifiesInputsInPlace(root)) {
-      const std::unordered_set<string> op_types_to_traverse = {
-          root.op(),    "Identity", "IdentityN", "Reshape",
-          "ExpandDims", "Enter",    "Switch",    "Merge"};
+      const auto is_continue_traversal = [&](const NodeDef* node) -> bool {
+        return node->op() == root.op() || ops_to_traverse.count(node->op()) > 0;
+      };
 
-      graph_view.DepthFirstSearchWithCallback(
-          op_types_to_traverse, i,
-          [&](const NodeDef& node) {
-            feeds_inplace_op.insert(&node);
-            return false;
-          },
-          SimpleGraphView::kFollowInputs /* search through inputs */);
+      DfsTraversal(graph_view, {&root}, TraversalDirection::kFollowInputs,
+                   DfsPredicates::Advance(is_continue_traversal),
+                   DfsCallbacks::PreOrder([&](const NodeDef* node) {
+                     feeds_inplace_op.insert(node);
+                   }));
     }
   }
 
+  bool stop = true;
+  std::set<int> duplicates;
   do {
     stop = true;
     UniqueNodes nodes;
diff --git a/tensorflow/core/grappler/optimizers/loop_optimizer.cc b/tensorflow/core/grappler/optimizers/loop_optimizer.cc
index 3606473840..cf5e4db29f 100644
--- a/tensorflow/core/grappler/optimizers/loop_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/loop_optimizer.cc
@@ -30,12 +30,14 @@ limitations under the License.
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/tensor.pb.h"
 #include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/grappler/graph_topology_view.h"
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/mutable_graph_view.h"
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/grappler/optimizers/constant_folding.h"
 #include "tensorflow/core/grappler/optimizers/evaluation_utils.h"
 #include "tensorflow/core/grappler/utils/frame.h"
+#include "tensorflow/core/grappler/utils/traversal.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
 #include "tensorflow/core/lib/gtl/inlined_vector.h"
@@ -451,16 +453,29 @@ Status LoopInvariantNodeMotionOptimizer::Optimize() {
 }
 
 std::vector<int> GetStackPushNodesToConvert(
-    const SimpleGraphView& graph_view,
+    const GraphTopologyView& graph_view,
     const std::unordered_set<string>& nodes_to_preserve, int stack_node_idx) {
   VLOG(1) << "Stack node: " << graph_view.graph()->node(stack_node_idx).name();
+
   const std::unordered_set<string> op_types_to_traverse(
       {"Stack", "StackV2", "Enter", "RefEnter", "Switch", "RefSwitch",
        "Identity", "RefIdentity"});
+  const auto is_op_to_traverse = [&](const NodeDef* node) -> bool {
+    return op_types_to_traverse.find(node->op()) != op_types_to_traverse.end();
+  };
+
   std::vector<int> nodes_to_convert;
-  std::set<int> fanout;
-  graph_view.DepthFirstSearch(op_types_to_traverse, stack_node_idx, &fanout);
-  for (int fanout_idx : fanout) {
+  std::vector<int> fanouts;
+
+  DfsTraversal(graph_view, {graph_view.GetNode(stack_node_idx)},
+               TraversalDirection::kFollowOutputs,
+               DfsPredicates::Advance(is_op_to_traverse),
+               DfsCallbacks::PreOrder([&](const NodeDef* node) {
+                 const absl::optional<int> idx = graph_view.GetNodeIndex(*node);
+                 fanouts.push_back(idx.value());
+               }));
+
+  for (int fanout_idx : fanouts) {
     const NodeDef& fanout_node = graph_view.graph()->node(fanout_idx);
     VLOG(1) << "Fanout " << fanout_idx << " : " << fanout_node.name();
     if (IsStackPushOp(fanout_node)) {
@@ -468,13 +483,12 @@ std::vector<int> GetStackPushNodesToConvert(
       // happen when the graph we have contains only the forward pass for a loop
       // (as when the forward and backward passes are split across different
       // functions).
-      if (graph_view.has_node(fanout_node.input(0))) {
-        const NodeDef* stack_node =
-            &graph_view.node(graph_view.index(fanout_node.input(0)));
+      if (graph_view.HasNode(fanout_node.input(0))) {
+        const NodeDef* stack_node = graph_view.GetNode(fanout_node.input(0));
         while (stack_node->op() != "Stack" && stack_node->op() != "StackV2" &&
                stack_node->input_size() > 0 &&
-               graph_view.has_node(stack_node->input(0))) {
-          stack_node = &graph_view.node(graph_view.index(stack_node->input(0)));
+               graph_view.HasNode(stack_node->input(0))) {
+          stack_node = graph_view.GetNode(stack_node->input(0));
         }
         if (nodes_to_preserve.find(stack_node->name()) ==
             nodes_to_preserve.end()) {
@@ -488,7 +502,7 @@ std::vector<int> GetStackPushNodesToConvert(
                    op_types_to_traverse.end()) {
       continue;
     } else if (!IsStackPopOp(fanout_node) ||
-               (!graph_view.outputs(fanout_idx).empty() ||
+               (!graph_view.GetFanout(fanout_idx).empty() ||
                 nodes_to_preserve.find(fanout_node.name()) !=
                     nodes_to_preserve.end())) {
       // The node is either a stack pop with consumers or something unexpected
@@ -497,14 +511,16 @@ std::vector<int> GetStackPushNodesToConvert(
       break;
     }
   }
+
   return nodes_to_convert;
 }
 
 Status RemoveStackOps(const std::unordered_set<string>& nodes_to_preserve,
                       GraphDef* optimized_graph) {
   NodeMap node_map(optimized_graph);
-  SimpleGraphView graph_view;
-  TF_RETURN_IF_ERROR(graph_view.Initialize(*optimized_graph));
+  GraphTopologyView graph_view;
+  TF_RETURN_IF_ERROR(graph_view.InitializeFromGraph(*optimized_graph));
+
   for (int node_idx = 0; node_idx < optimized_graph->node_size(); ++node_idx) {
     if (IsStackOp(optimized_graph->node(node_idx))) {
       for (int push_node_idx : GetStackPushNodesToConvert(
diff --git a/tensorflow/core/grappler/optimizers/memory_optimizer.cc b/tensorflow/core/grappler/optimizers/memory_optimizer.cc
index 6e33645537..b50d50f842 100644
--- a/tensorflow/core/grappler/optimizers/memory_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/memory_optimizer.cc
@@ -30,6 +30,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/costs/graph_memory.h"
 #include "tensorflow/core/grappler/costs/graph_properties.h"
 #include "tensorflow/core/grappler/costs/utils.h"
+#include "tensorflow/core/grappler/graph_topology_view.h"
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/mutable_graph_view.h"
 #include "tensorflow/core/grappler/op_types.h"
@@ -188,13 +189,14 @@ std::vector<RecomputedSubGraph> GetOpGroupsToRecompute(
       }
     }
     // Recompute only nodes which eventually feed into a target node.
-    connected_subgraph(node_map,
-                       true,   // Collect inputs
-                       false,  // Collect outputs
-                       [&unpruned_recompute_nodes](const NodeDef& node) {
-                         return unpruned_recompute_nodes.count(&node) != 0;
-                       },
-                       &current_recomputation.recomputed_source_nodes);
+    connected_subgraph(
+        node_map,
+        true,   // Collect inputs
+        false,  // Collect outputs
+        [&unpruned_recompute_nodes](const NodeDef& node) {
+          return unpruned_recompute_nodes.count(&node) != 0;
+        },
+        &current_recomputation.recomputed_source_nodes);
     if (current_recomputation.target_nodes.empty()) {
       continue;
     }
@@ -1268,46 +1270,55 @@ Status RelaxAllocatorConstraints(GraphDef* optimized_graph) {
     return Status::OK();
   }
 
-  std::unordered_set<int> optimized_nodes;
-  SimpleGraphView graph_view;
-  TF_RETURN_IF_ERROR(graph_view.Initialize(*optimized_graph));
+  GraphTopologyView graph_view;
+  TF_RETURN_IF_ERROR(graph_view.InitializeFromGraph(*optimized_graph));
+  std::unordered_set<const NodeDef*> optimized_nodes;
+
   for (int i : assign_nodes) {
-    if (optimized_nodes.find(i) == optimized_nodes.end()) {
-      const NodeDef& assign_node = optimized_graph->node(i);
-      optimized_nodes.insert(i);
-      std::vector<int> assign_nodes_in_fanout;
-      assign_nodes_in_fanout.push_back(i);
-      std::set<int> transitive_fanout;
-      graph_view.DepthFirstSearch(std::unordered_set<string>{}, i,
-                                  &transitive_fanout);
+    const NodeDef& assign_node = optimized_graph->node(i);
+
+    if (optimized_nodes.find(&assign_node) == optimized_nodes.end()) {
+      std::vector<const NodeDef*> assign_nodes_in_fanout;
+      optimized_nodes.insert(&assign_node);
+      assign_nodes_in_fanout.push_back(&assign_node);
+
+      std::vector<const NodeDef*> transitive_fanout;
+      DfsTraversal(graph_view, {graph_view.GetNode(i)},
+                   TraversalDirection::kFollowOutputs,
+                   DfsCallbacks::PreOrder([&](const NodeDef* node) {
+                     transitive_fanout.push_back(node);
+                   }));
+
       bool relax_constraint = true;
       // If all nodes in the transitive fanout are on the same device as the
       // assign node, there is no need to allocate the output in pinned memory.
-      for (int fanout : transitive_fanout) {
-        const NodeDef& fanout_node = optimized_graph->node(fanout);
+      for (const NodeDef* fanout_node : transitive_fanout) {
+        // const NodeDef& fanout_node = optimized_graph->node(fanout);
         if (relax_constraint &&
-            (IsSend(fanout_node) ||
-             CrossesTaskOrCpuGpuBoundary(fanout_node, assign_node))) {
+            (IsSend(*fanout_node) ||
+             CrossesTaskOrCpuGpuBoundary(*fanout_node, assign_node))) {
           relax_constraint = false;
           break;
         }
-        if (optimized_nodes.find(fanout) == optimized_nodes.end() &&
-            IsAssign(fanout_node)) {
-          assign_nodes_in_fanout.push_back(fanout);
+        if (optimized_nodes.find(fanout_node) == optimized_nodes.end() &&
+            IsAssign(*fanout_node)) {
+          assign_nodes_in_fanout.push_back(fanout_node);
         }
       }
 
       if (relax_constraint) {
-        for (int assign_idx : assign_nodes_in_fanout) {
+        for (const NodeDef* assign_node_in_fanout : assign_nodes_in_fanout) {
           // If all devices match in fanout of node(i) then, by transitivity,
           // they must also match in the fanout of other assign nodes
           // in the fanout of node(i), so we can process them here,
           // and save computing their transitive fanout later.
-          optimized_nodes.insert(assign_idx);
+          optimized_nodes.insert(assign_node_in_fanout);
 
           // Set an attribute telling AssignOp to ignore allocator constraints.
+          const absl::optional<int> assign_node_idx =
+              graph_view.GetNodeIndex(*assign_node_in_fanout);
           NodeDef* assign_node_to_relax =
-              optimized_graph->mutable_node(assign_idx);
+              optimized_graph->mutable_node(assign_node_idx.value());
           (*assign_node_to_relax
                 ->mutable_attr())["_grappler_relax_allocator_constraints"]
               .set_b(true);
diff --git a/tensorflow/core/grappler/utils.cc b/tensorflow/core/grappler/utils.cc
index d8fb88d95f..cc9a38b2d2 100644
--- a/tensorflow/core/grappler/utils.cc
+++ b/tensorflow/core/grappler/utils.cc
@@ -402,175 +402,6 @@ void EraseNodesFromGraph(const std::set<string>& nodes_to_delete,
   EraseNodesFromGraphImpl(nodes_idx_to_delete, graph);
 }
 
-Status SimpleGraphView::Initialize(
-    const GraphDef& graph,
-    const std::vector<std::pair<const NodeDef*, const NodeDef*>>*
-        extra_dependencies,
-    bool dedup_inputs, bool dedup_outputs) {
-  graph_ = &graph;
-  const int num_nodes = graph.node_size();
-  inputs_.clear();
-  inputs_.resize(num_nodes);
-  outputs_.clear();
-  outputs_.resize(num_nodes);
-  name_to_index_.clear();
-  name_to_index_.reserve(num_nodes);
-  index_to_name_.clear();
-  index_to_name_.reserve(num_nodes);
-
-  // Build map from name to index and vice versa.
-  for (int node_idx = 0; node_idx < num_nodes; ++node_idx) {
-    const NodeDef& node = graph.node(node_idx);
-    name_to_index_.emplace(node.name(), node_idx);
-    index_to_name_.push_back(node.name());
-  }
-
-  if (extra_dependencies) {
-    for (const auto& dep : *extra_dependencies) {
-      auto itr_src = name_to_index_.find(dep.first->name());
-      if (itr_src == name_to_index_.end()) {
-        return errors::InvalidArgument("Non-existent src ", dep.first->name());
-      }
-      auto itr_tgt = name_to_index_.find(dep.second->name());
-      if (itr_tgt == name_to_index_.end()) {
-        return errors::InvalidArgument("Non-existent tgt ", dep.second->name());
-      }
-      const int src_idx = itr_src->second;
-      const int tgt_idx = itr_tgt->second;
-      inputs_[tgt_idx].push_back(src_idx);
-      outputs_[src_idx].push_back(tgt_idx);
-    }
-  }
-
-  // Build forward and reverse adjacency lists.
-  for (int node_idx = 0; node_idx < num_nodes; ++node_idx) {
-    const NodeDef& node = graph.node(node_idx);
-    inputs_[node_idx].reserve(node.input_size());
-    for (const string& input : node.input()) {
-      auto it = name_to_index_.find(NodeName(input));
-      if (it == name_to_index_.end()) {
-        return errors::InvalidArgument("Non-existent input ", input,
-                                       " for node ", node.name());
-      }
-      const int input_idx = it->second;
-      inputs_[node_idx].push_back(input_idx);
-      outputs_[input_idx].push_back(node_idx);
-    }
-    if (dedup_inputs) {
-      // Dedup the input list while it's still hot in cache.
-      STLSortAndRemoveDuplicates(&inputs_[node_idx]);
-    }
-  }
-
-  // Dedup outputs.
-  if (dedup_outputs) {
-    for (int node_idx = 0; node_idx < num_nodes; ++node_idx) {
-      STLSortAndRemoveDuplicates(&outputs_[node_idx]);
-    }
-  }
-  return Status::OK();
-}
-
-void SimpleGraphView::DepthFirstSearch(
-    const std::unordered_set<string>& op_types_to_traverse, int root_node,
-    std::set<int>* nodes_found,
-    SimpleGraphView::SearchDirection direction) const {
-  nodes_found->clear();
-  const string& op_type = graph_->node(root_node).op();
-  if (!op_types_to_traverse.empty() &&
-      op_types_to_traverse.find(op_type) == op_types_to_traverse.end()) {
-    return;
-  }
-  std::vector<int> stack;
-  stack.reserve(32);
-  stack.push_back(root_node);
-
-  auto push_neighbors = [&stack,
-                         &nodes_found](absl::Span<const int> neighbors) {
-    for (auto output_idx : neighbors) {
-      if (nodes_found->find(output_idx) == nodes_found->end()) {
-        stack.push_back(output_idx);
-      }
-    }
-  };
-
-  while (!stack.empty()) {
-    const int node_idx = stack.back();
-    stack.pop_back();
-    nodes_found->insert(node_idx);
-    const string& op_type = graph_->node(node_idx).op();
-    if (op_types_to_traverse.empty() ||
-        op_types_to_traverse.find(op_type) != op_types_to_traverse.end()) {
-      if (direction == kFollowOutputs) {
-        push_neighbors(this->outputs(node_idx));
-      } else {
-        push_neighbors(this->inputs(node_idx));
-      }
-    }
-  }
-}
-
-void SimpleGraphView::DepthFirstSearchWithCallback(
-    const std::unordered_set<string>& op_types_to_traverse, int node_idx,
-    SimpleGraphView::DFSCallback callback,
-    SimpleGraphView::SearchDirection direction) const {
-  std::set<int> nodes_found;
-  nodes_found.clear();
-  const string& op_type = graph_->node(node_idx).op();
-  if (!op_types_to_traverse.empty() &&
-      op_types_to_traverse.find(op_type) == op_types_to_traverse.end()) {
-    return;
-  }
-  std::vector<int> stack;
-  stack.reserve(32);
-  stack.push_back(node_idx);
-  auto push_neighbors = [&stack,
-                         &nodes_found](absl::Span<const int> neighbors) {
-    for (auto output_idx : neighbors) {
-      if (nodes_found.find(output_idx) == nodes_found.end()) {
-        stack.push_back(output_idx);
-      }
-    }
-  };
-  while (!stack.empty()) {
-    const int node_idx = stack.back();
-    stack.pop_back();
-    if (callback(graph_->node(node_idx))) {
-      return;
-    }
-    nodes_found.insert(node_idx);
-    const string& op_type = graph_->node(node_idx).op();
-    if (op_types_to_traverse.empty() ||
-        op_types_to_traverse.find(op_type) != op_types_to_traverse.end()) {
-      if (direction == kFollowOutputs) {
-        push_neighbors(this->outputs(node_idx));
-      } else {
-        push_neighbors(this->inputs(node_idx));
-      }
-    }
-  }
-}
-
-string SimpleGraphView::PrintToString() const {
-  string str;
-  for (int i = 0; i < num_nodes(); ++i) {
-    strings::StrAppend(&str, "Node ", i, "'", node_name(i), "'\n", "Inputs: [");
-    for (int input : inputs(i)) {
-      strings::StrAppend(&str, input, " '", node_name(input), "', ");
-    }
-    strings::StrAppend(&str, "]\n", "Outputs: [");
-    for (int j = 0; j < outputs(i).size(); ++j) {
-      const int output = outputs(i)[j];
-      if (j > 0) {
-        strings::StrAppend(&str, ", ");
-      }
-      strings::StrAppend(&str, output, " '", node_name(output), "'");
-    }
-    strings::StrAppend(&str, "]\n");
-  }
-  return str;
-}
-
 #define HANDLE_CASE(DTYPE)                                          \
   case DTYPE:                                                       \
     if (!SafeSetScalarTensorValue<EnumToDataType<DTYPE>::Type>(     \
diff --git a/tensorflow/core/grappler/utils.h b/tensorflow/core/grappler/utils.h
index a9d6abd6db..1e820977ae 100644
--- a/tensorflow/core/grappler/utils.h
+++ b/tensorflow/core/grappler/utils.h
@@ -302,79 +302,6 @@ void EraseNodesFromGraph(std::vector<int>&& nodes_to_delete, GraphDef* graph);
 void EraseNodesFromGraph(const std::set<string>& nodes_to_delete,
                          GraphDef* graph);
 
-class SimpleGraphView {
- public:
-  // Build a graph view for the specified graphdef.
-  Status Initialize(const GraphDef& graph) {
-    return Initialize(graph, nullptr, true, true);
-  }
-  // Build a graph view for the specified graphdef augmented with the additional
-  // edges specified in 'extra_dependencies' if any. Note that
-  // extra_dependencies can be null.
-  Status Initialize(
-      const GraphDef& graph,
-      const std::vector<std::pair<const NodeDef*, const NodeDef*>>*
-          extra_dependencies) {
-    return Initialize(graph, extra_dependencies, true, true);
-  }
-  Status Initialize(
-      const GraphDef& graph,
-      const std::vector<std::pair<const NodeDef*, const NodeDef*>>*
-          extra_dependencies,
-      bool dedup_inputs, bool dedup_outputs);
-
-  const GraphDef* graph() const { return graph_; }
-  inline int num_nodes() const { return index_to_name_.size(); }
-  inline bool has_node(const string& node_name) const {
-    return name_to_index_.find(node_name) != name_to_index_.end();
-  }
-  inline const int index(const string& node_name) const {
-    const auto& it = name_to_index_.find(node_name);
-    DCHECK(it != name_to_index_.end());
-    return it == name_to_index_.end() ? -1 : it->second;
-  }
-  inline const NodeDef& node(int node_idx) const {
-    return graph_->node(node_idx);
-  }
-  inline const string& node_name(int node_idx) const {
-    return index_to_name_[node_idx];
-  }
-  inline const gtl::InlinedVector<int, 4>& inputs(int node_idx) const {
-    return inputs_[node_idx];
-  }
-  inline const gtl::InlinedVector<int, 2>& outputs(int node_idx) const {
-    return outputs_[node_idx];
-  }
-
-  enum SearchDirection { kFollowOutputs = 1, kFollowInputs = 2 };
-
-  // Traverse the graph starting at `node_idx`, collecting indices of nodes
-  // visited in nodes_found. If a node has an op in `op_types_to_traverse`, the
-  // walk continues to its children. It is assumed that *graph_ was not modified
-  // after the call to Initialize().
-  // If `op_types_to_traverse` is empty the DFS will traverse any node type.
-  void DepthFirstSearch(const std::unordered_set<string>& op_types_to_traverse,
-                        int node_idx, std::set<int>* nodes_found,
-                        SearchDirection direction = kFollowOutputs) const;
-
-  typedef std::function<bool(const NodeDef&)> DFSCallback;
-
-  // Like DepthFirstSearch, but invoke `callback` as each node is discovered. If
-  // `callback` returns true, the search is terminated early.
-  void DepthFirstSearchWithCallback(
-      const std::unordered_set<string>& op_types_to_traverse, int node_idx,
-      DFSCallback callback, SearchDirection direction = kFollowOutputs) const;
-
-  string PrintToString() const;
-
- private:
-  const GraphDef* graph_;  // Not owned.
-  std::vector<string> index_to_name_;
-  gtl::FlatMap<string, int> name_to_index_;
-  std::vector<gtl::InlinedVector<int, 4>> inputs_;
-  std::vector<gtl::InlinedVector<int, 2>> outputs_;
-};
-
 }  // end namespace grappler
 }  // end namespace tensorflow
 
diff --git a/tensorflow/core/grappler/utils/traversal.cc b/tensorflow/core/grappler/utils/traversal.cc
index 3231094fd4..c602e8c0e4 100644
--- a/tensorflow/core/grappler/utils/traversal.cc
+++ b/tensorflow/core/grappler/utils/traversal.cc
@@ -44,8 +44,8 @@ enum class NodeState { kNotVisited, kVisiting, kDone };
 
 void DfsTraversal(const GraphTopologyView& graph_view,
                   const absl::Span<const NodeDef* const> from,
-                  TraversalDirection direction,
-                  const std::function<bool(const NodeDef*)>& should_visit,
+                  const TraversalDirection direction,
+                  const DfsPredicates& predicates,
                   const DfsCallbacks& callbacks) {
   std::vector<DfsStackElem> stack;
   stack.reserve(from.size());
@@ -66,8 +66,8 @@ void DfsTraversal(const GraphTopologyView& graph_view,
     NodeState& state = node_state[w.node];
     if (state == NodeState::kDone) continue;
 
-    // Skip nodes that do not match predicate.
-    if (should_visit && !should_visit(graph_view.GetNode(w.node))) {
+    // Skip nodes that we should not enter.
+    if (predicates.enter && !predicates.enter(graph_view.GetNode(w.node))) {
       state = NodeState::kDone;
       continue;
     }
@@ -98,6 +98,11 @@ void DfsTraversal(const GraphTopologyView& graph_view,
     // Enqueue the node again with the children_visited flag set to true.
     stack.emplace_back(w.node, true, w.src);
 
+    // Check if we can continue traversal from the current node.
+    if (predicates.advance && !predicates.advance(graph_view.GetNode(w.node))) {
+      continue;
+    }
+
     // Now enqueue the fanin/fanout nodes.
     if (direction == TraversalDirection::kFollowInputs) {
       for (const int fanin : graph_view.GetFanin(w.node)) {
@@ -111,14 +116,10 @@ void DfsTraversal(const GraphTopologyView& graph_view,
   }
 }
 
-// Traverse the graph in DFS order in the given direction, starting from the
-// list of nodes specified in the `from` argument. Call corresponding
-// callbacks for each visited node.
 void DfsTraversal(const GraphTopologyView& graph_view,
                   const absl::Span<const NodeDef* const> from,
                   TraversalDirection direction, const DfsCallbacks& callbacks) {
-  const auto visit_all = [](const NodeDef*) { return true; };
-  DfsTraversal(graph_view, from, direction, visit_all, callbacks);
+  DfsTraversal(graph_view, from, direction, {}, callbacks);
 }
 
 }  // namespace grappler
diff --git a/tensorflow/core/grappler/utils/traversal.h b/tensorflow/core/grappler/utils/traversal.h
index ec6009096d..5c9dada493 100644
--- a/tensorflow/core/grappler/utils/traversal.h
+++ b/tensorflow/core/grappler/utils/traversal.h
@@ -33,6 +33,7 @@ enum class TraversalDirection { kFollowInputs, kFollowOutputs };
 // corresponding back edges. Moreover, the pre and post order will assume that
 // these back edges will be cut.
 struct DfsCallbacks {
+  DfsCallbacks() = default;
   DfsCallbacks(std::function<void(const NodeDef*)> pre,
                std::function<void(const NodeDef*)> post,
                std::function<void(const NodeDef*, const NodeDef*)> back_edge)
@@ -53,16 +54,40 @@ struct DfsCallbacks {
   std::function<void(const NodeDef*, const NodeDef*)> on_back_edge;
 };
 
+// Encapsulate DFS predicates for traversing the graph.
+//
+// The `enter` predicate decides if traversal should enter the node, and the
+// `advance` predicate decides if the traversal should follow inputs/outputs
+// from the node.
+//
+// If predicates are empty (default initialized), it's assumed that we can enter
+// into any node and advance from any node respectively.
+struct DfsPredicates {
+  DfsPredicates() = default;
+  DfsPredicates(std::function<bool(const NodeDef*)> enter,
+                std::function<bool(const NodeDef*)> advance)
+      : enter(std::move(enter)), advance(std::move(advance)) {}
+
+  static DfsPredicates Enter(std::function<bool(const NodeDef*)> enter) {
+    return DfsPredicates(std::move(enter), nullptr);
+  }
+
+  static DfsPredicates Advance(std::function<bool(const NodeDef*)> advance) {
+    return DfsPredicates(nullptr, std::move(advance));
+  }
+
+  std::function<bool(const NodeDef*)> enter;
+  std::function<bool(const NodeDef*)> advance;
+};
+
 // Traverse the graph in DFS order in the given direction, starting from the
-// list of nodes specified in the `from` argument. Use `should_visit` to decide
-// if the node should be visited. This predicate also applied to the `from`
-// nodes. Call corresponding callbacks for each visited node. If `should_visit`
-// is empty (default initialized) it's assumed that traversal can visit all the
-// nodes.
+// list of nodes specified in the `from` argument. Use `predicates` to decide if
+// traversal should enter/advance to/from the graph node. These predicates also
+// applied to the `from` nodes. Call corresponding callbacks for each visited
+// node.
 void DfsTraversal(const GraphTopologyView& graph_view,
                   absl::Span<const NodeDef* const> from,
-                  TraversalDirection direction,
-                  const std::function<bool(const NodeDef*)>& should_visit,
+                  TraversalDirection direction, const DfsPredicates& predicates,
                   const DfsCallbacks& callbacks);
 
 // Traverse the graph in DFS order in the given direction, starting from the
diff --git a/tensorflow/core/grappler/utils/traversal_test.cc b/tensorflow/core/grappler/utils/traversal_test.cc
index 5a68fa201e..7b36d328e9 100644
--- a/tensorflow/core/grappler/utils/traversal_test.cc
+++ b/tensorflow/core/grappler/utils/traversal_test.cc
@@ -159,7 +159,7 @@ TEST(TraversalTest, OutputDfsWithLoop) {
   EXPECT_EQ(back_edges, expected_edges);
 }
 
-TEST(TraversalTest, DfsWithPredicate) {
+TEST(TraversalTest, DfsWithEnterPredicate) {
   const string op = "OpIsNotImportantInThisTest";
 
   GraphDef graph = ::tensorflow::test::function::GDef(  //
@@ -171,8 +171,8 @@ TEST(TraversalTest, DfsWithPredicate) {
        NDef("6", op, {"3", "5"}, {})},                  //
       /*funcs=*/{});
 
-  // Do not go through nodes '2' and '3'.
-  const auto predicate = [](const NodeDef* node) {
+  // Do not enter the nodes '2' and '3'.
+  const auto enter = [](const NodeDef* node) {
     return node->name() != "2" && node->name() != "3";
   };
 
@@ -185,7 +185,8 @@ TEST(TraversalTest, DfsWithPredicate) {
   GraphTopologyView graph_view;
   TF_CHECK_OK(graph_view.InitializeFromGraph(graph));
   DfsTraversal(graph_view, start_nodes, TraversalDirection::kFollowOutputs,
-               predicate, MkCallbacks(&pre_order, &post_order, &back_edges));
+               DfsPredicates::Enter(enter),
+               MkCallbacks(&pre_order, &post_order, &back_edges));
 
   const std::vector<string> expected_pre = {"1", "4", "5", "6"};
   const std::vector<string> expected_post = {"6", "5", "4", "1"};
@@ -195,6 +196,43 @@ TEST(TraversalTest, DfsWithPredicate) {
   EXPECT_TRUE(back_edges.empty());
 }
 
+TEST(TraversalTest, DfsWithAdvancePredicate) {
+  const string op = "OpIsNotImportantInThisTest";
+
+  GraphDef graph = ::tensorflow::test::function::GDef(  //
+      {NDef("1", op, {}, {}),                           //       2 -> 3
+       NDef("2", op, {"1"}, {}),                        // 1 -> /      \ -> 6
+       NDef("3", op, {"2"}, {}),                        //      \      /
+       NDef("4", op, {"1"}, {}),                        //       4 -> 5
+       NDef("5", op, {"4"}, {}),                        //
+       NDef("6", op, {"3", "5"}, {})},                  //
+      {} /* empty function library*/);
+
+  // Do not advance from the nodes '2' and '3'.
+  const auto advance = [](const NodeDef* node) {
+    return node->name() != "2" && node->name() != "3";
+  };
+
+  std::vector<const NodeDef*> start_nodes = {&graph.node(0)};
+
+  std::vector<string> pre_order;
+  std::vector<string> post_order;
+  std::vector<string> back_edges;
+
+  GraphTopologyView graph_view;
+  TF_CHECK_OK(graph_view.InitializeFromGraph(graph));
+  DfsTraversal(graph_view, start_nodes, TraversalDirection::kFollowOutputs,
+               DfsPredicates::Advance(advance),
+               MkCallbacks(&pre_order, &post_order, &back_edges));
+
+  const std::vector<string> expected_pre = {"1", "4", "5", "6", "2"};
+  const std::vector<string> expected_post = {"6", "5", "4", "2", "1"};
+
+  EXPECT_EQ(pre_order, expected_pre);
+  EXPECT_EQ(post_order, expected_post);
+  EXPECT_TRUE(back_edges.empty());
+}
+
 }  // namespace
 }  // namespace grappler
 }  // namespace tensorflow
-- 
GitLab


From 75f12a50203ca31370c5edc02d650ee4a47fdf03 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Schoentgen?= <contact@tiger-222.fr>
Date: Thu, 3 Jan 2019 22:36:43 +0100
Subject: [PATCH 0142/2345] Fix several DeprecationWarning: invlid escape
 sequence
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Mickaël Schoentgen <contact@tiger-222.fr>
---
 .../contrib/kernel_methods/python/losses.py      |  2 +-
 .../python/learn/learn_io/generator_io_test.py   |  4 ++--
 .../opt/python/training/adam_gs_optimizer.py     |  2 +-
 tensorflow/contrib/optimizer_v2/adam.py          |  2 +-
 tensorflow/python/client/session_test.py         |  2 +-
 .../python/feature_column/feature_column_test.py | 12 ++++++------
 .../feature_column/feature_column_v2_test.py     | 16 ++++++++--------
 .../python/kernel_tests/confusion_matrix_test.py |  4 ++--
 tensorflow/python/training/adam.py               |  2 +-
 tensorflow/tools/ci_build/copy_binary.py         |  2 +-
 .../scripts_allreduce/k8s_generate_yaml_lib.py   |  2 +-
 .../windows/msvc_wrapper_for_nvcc.py             |  2 +-
 .../windows/msvc_wrapper_for_nvcc.py             |  2 +-
 .../gcc-nvcc/windows/msvc_wrapper_for_nvcc.py    |  2 +-
 14 files changed, 28 insertions(+), 28 deletions(-)

diff --git a/tensorflow/contrib/kernel_methods/python/losses.py b/tensorflow/contrib/kernel_methods/python/losses.py
index 4ef0a66a52..294a7d69a7 100644
--- a/tensorflow/contrib/kernel_methods/python/losses.py
+++ b/tensorflow/contrib/kernel_methods/python/losses.py
@@ -34,7 +34,7 @@ def sparse_multiclass_hinge_loss(
     scope=None,
     loss_collection=ops.GraphKeys.LOSSES,
     reduction=losses.Reduction.SUM_BY_NONZERO_WEIGHTS):
-  """Adds Ops for computing the multiclass hinge loss.
+  r"""Adds Ops for computing the multiclass hinge loss.
 
   The implementation is based on the following paper:
   On the Algorithmic Implementation of Multiclass Kernel-based Vector Machines
diff --git a/tensorflow/contrib/learn/python/learn/learn_io/generator_io_test.py b/tensorflow/contrib/learn/python/learn/learn_io/generator_io_test.py
index 5e90d1fa20..318046733b 100644
--- a/tensorflow/contrib/learn/python/learn/learn_io/generator_io_test.py
+++ b/tensorflow/contrib/learn/python/learn/learn_io/generator_io_test.py
@@ -174,7 +174,7 @@ class GeneratorIoTest(test.TestCase):
       return np.arange(32, 36)
 
     with self.cached_session():
-      with self.assertRaisesRegexp(TypeError, 'x\(\) must be generator'):
+      with self.assertRaisesRegexp(TypeError, r'x\(\) must be generator'):
         failing_input_fn = generator_io.generator_input_fn(
             generator, batch_size=2, shuffle=False, num_epochs=1)
         failing_input_fn()
@@ -185,7 +185,7 @@ class GeneratorIoTest(test.TestCase):
       yield np.arange(32, 36)
 
     with self.cached_session():
-      with self.assertRaisesRegexp(TypeError, 'x\(\) must yield dict'):
+      with self.assertRaisesRegexp(TypeError, r'x\(\) must yield dict'):
         failing_input_fn = generator_io.generator_input_fn(
             generator, batch_size=2, shuffle=False, num_epochs=1)
         failing_input_fn()
diff --git a/tensorflow/contrib/opt/python/training/adam_gs_optimizer.py b/tensorflow/contrib/opt/python/training/adam_gs_optimizer.py
index 3fb649ea82..855fbf58bf 100644
--- a/tensorflow/contrib/opt/python/training/adam_gs_optimizer.py
+++ b/tensorflow/contrib/opt/python/training/adam_gs_optimizer.py
@@ -41,7 +41,7 @@ class AdamGSOptimizer(optimizer.Optimizer):
   def __init__(self, global_step=0, learning_rate=0.001,
                beta1=0.9, beta2=0.999, epsilon=1e-8,
                use_locking=False, name="Adam"):
-    """Construct a new Adam optimizer.
+    r"""Construct a new Adam optimizer.
 
     Branched from tf.train.AdamOptimizer. The only difference is to pass
     global step for computing beta1 and beta2 accumulators, instead of having
diff --git a/tensorflow/contrib/optimizer_v2/adam.py b/tensorflow/contrib/optimizer_v2/adam.py
index 248ffb1f7e..1b7800f324 100644
--- a/tensorflow/contrib/optimizer_v2/adam.py
+++ b/tensorflow/contrib/optimizer_v2/adam.py
@@ -36,7 +36,7 @@ class AdamOptimizer(optimizer_v2.OptimizerV2):
 
   def __init__(self, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8,
                use_locking=False, name="Adam"):
-    """Construct a new Adam optimizer.
+    r"""Construct a new Adam optimizer.
 
     Initialization:
 
diff --git a/tensorflow/python/client/session_test.py b/tensorflow/python/client/session_test.py
index c4a118a414..da6218663d 100644
--- a/tensorflow/python/client/session_test.py
+++ b/tensorflow/python/client/session_test.py
@@ -2036,7 +2036,7 @@ class SessionTest(test_util.TensorFlowTestCase):
     with self.cached_session() as sess:
       a = array_ops.placeholder(dtype=dtypes.string)
       with self.assertRaisesRegexp(
-          TypeError, 'Type of feed value 1 with type <(\w+) \'int\'> is not'):
+          TypeError, r'Type of feed value 1 with type <(\w+) \'int\'> is not'):
         sess.run(a, feed_dict={a: 1})
 
 
diff --git a/tensorflow/python/feature_column/feature_column_test.py b/tensorflow/python/feature_column/feature_column_test.py
index daa0a3b3a4..0ded2bf8c9 100644
--- a/tensorflow/python/feature_column/feature_column_test.py
+++ b/tensorflow/python/feature_column/feature_column_test.py
@@ -1832,7 +1832,7 @@ class LinearModelTest(test.TestCase):
       }
     with self.assertRaisesRegexp(
         ValueError,
-        'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+        r'Batch size \(first dimension\) of each feature must be same.'):
       fc.linear_model(features, [price1, price2])
 
   def test_subset_of_static_batch_size_mismatch(self):
@@ -1847,7 +1847,7 @@ class LinearModelTest(test.TestCase):
       }
       with self.assertRaisesRegexp(
           ValueError,
-          'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+          r'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
         fc.linear_model(features, [price1, price2, price3])
 
   def test_runtime_batch_size_mismatch(self):
@@ -2467,7 +2467,7 @@ class _LinearModelTest(test.TestCase):
       }
     with self.assertRaisesRegexp(
         ValueError,
-        'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+        r'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
       get_keras_linear_model_predictions(features, [price1, price2])
 
   def test_subset_of_static_batch_size_mismatch(self):
@@ -2482,7 +2482,7 @@ class _LinearModelTest(test.TestCase):
       }
       with self.assertRaisesRegexp(
           ValueError,
-          'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+          r'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
         get_keras_linear_model_predictions(features, [price1, price2, price3])
 
   def test_runtime_batch_size_mismatch(self):
@@ -2974,7 +2974,7 @@ class FunctionalInputLayerTest(test.TestCase):
       }
       with self.assertRaisesRegexp(
           ValueError,
-          'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+          r'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
         fc.input_layer(features, [price1, price2])
 
   def test_subset_of_static_batch_size_mismatch(self):
@@ -2989,7 +2989,7 @@ class FunctionalInputLayerTest(test.TestCase):
       }
       with self.assertRaisesRegexp(
           ValueError,
-          'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+          r'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
         fc.input_layer(features, [price1, price2, price3])
 
   def test_runtime_batch_size_mismatch(self):
diff --git a/tensorflow/python/feature_column/feature_column_v2_test.py b/tensorflow/python/feature_column/feature_column_v2_test.py
index a247425369..2a864a0573 100644
--- a/tensorflow/python/feature_column/feature_column_v2_test.py
+++ b/tensorflow/python/feature_column/feature_column_v2_test.py
@@ -2052,7 +2052,7 @@ class LinearModelTest(test.TestCase):
       }
     with self.assertRaisesRegexp(
         ValueError,
-        'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+        r'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
       model = fc.LinearModel([price1, price2])
       model(features)
 
@@ -2068,7 +2068,7 @@ class LinearModelTest(test.TestCase):
       }
       with self.assertRaisesRegexp(
           ValueError,
-          'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+          r'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
         model = fc.LinearModel([price1, price2, price3])
         model(features)
 
@@ -2818,7 +2818,7 @@ class OldLinearModelTest(test.TestCase):
       }
     with self.assertRaisesRegexp(
         ValueError,
-        'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+        r'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
       fc_old.linear_model(features, [price1, price2])
 
   def test_subset_of_static_batch_size_mismatch(self):
@@ -2833,7 +2833,7 @@ class OldLinearModelTest(test.TestCase):
       }
       with self.assertRaisesRegexp(
           ValueError,
-          'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+          r'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
         fc_old.linear_model(features, [price1, price2, price3])
 
   def test_runtime_batch_size_mismatch(self):
@@ -3435,7 +3435,7 @@ class DenseFeaturesTest(test.TestCase):
       }
       with self.assertRaisesRegexp(
           ValueError,
-          'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+          r'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
         fc.DenseFeatures([price1, price2])(features)
 
   def test_subset_of_static_batch_size_mismatch(self):
@@ -3450,7 +3450,7 @@ class DenseFeaturesTest(test.TestCase):
       }
       with self.assertRaisesRegexp(
           ValueError,
-          'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+          r'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
         fc.DenseFeatures([price1, price2, price3])(features)
 
   def test_runtime_batch_size_mismatch(self):
@@ -4141,7 +4141,7 @@ class FunctionalInputLayerTest(test.TestCase):
       }
       with self.assertRaisesRegexp(
           ValueError,
-          'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+          r'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
         fc_old.input_layer(features, [price1, price2])
 
   def test_subset_of_static_batch_size_mismatch(self):
@@ -4156,7 +4156,7 @@ class FunctionalInputLayerTest(test.TestCase):
       }
       with self.assertRaisesRegexp(
           ValueError,
-          'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+          r'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
         fc_old.input_layer(features, [price1, price2, price3])
 
   def test_runtime_batch_size_mismatch(self):
diff --git a/tensorflow/python/kernel_tests/confusion_matrix_test.py b/tensorflow/python/kernel_tests/confusion_matrix_test.py
index ae13c8e32e..6670f0326c 100644
--- a/tensorflow/python/kernel_tests/confusion_matrix_test.py
+++ b/tensorflow/python/kernel_tests/confusion_matrix_test.py
@@ -472,7 +472,7 @@ class RemoveSqueezableDimensionsTest(test.TestCase):
       }
       with self.assertRaisesRegexp(
           errors_impl.InvalidArgumentError,
-          "Can not squeeze dim\[2\]"):
+          r"Can not squeeze dim\[2\]"):
         dynamic_labels.eval(feed_dict=feed_dict)
       self.assertAllEqual(
           prediction_values, dynamic_predictions.eval(feed_dict=feed_dict))
@@ -500,7 +500,7 @@ class RemoveSqueezableDimensionsTest(test.TestCase):
           label_values, dynamic_labels.eval(feed_dict=feed_dict))
       with self.assertRaisesRegexp(
           errors_impl.InvalidArgumentError,
-          "Can not squeeze dim\[2\]"):
+          r"Can not squeeze dim\[2\]"):
         dynamic_predictions.eval(feed_dict=feed_dict)
 
 
diff --git a/tensorflow/python/training/adam.py b/tensorflow/python/training/adam.py
index 0c701f4712..c204a5c1ee 100644
--- a/tensorflow/python/training/adam.py
+++ b/tensorflow/python/training/adam.py
@@ -39,7 +39,7 @@ class AdamOptimizer(optimizer.Optimizer):
 
   def __init__(self, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8,
                use_locking=False, name="Adam"):
-    """Construct a new Adam optimizer.
+    r"""Construct a new Adam optimizer.
 
     Initialization:
 
diff --git a/tensorflow/tools/ci_build/copy_binary.py b/tensorflow/tools/ci_build/copy_binary.py
index 148526492d..40a7443745 100755
--- a/tensorflow/tools/ci_build/copy_binary.py
+++ b/tensorflow/tools/ci_build/copy_binary.py
@@ -33,7 +33,7 @@ import tempfile
 import zipfile
 
 TF_NIGHTLY_REGEX = (r"(.+)tf_nightly(|_gpu)-(\d\.[\d]{1,2}"
-                    "\.\d.dev[\d]{0,8})-(.+)\.whl")
+                    r"\.\d.dev[\d]{0,8})-(.+)\.whl")
 BINARY_STRING_TEMPLATE = "%s-%s-%s.whl"
 
 
diff --git a/tensorflow/tools/dist_test/scripts_allreduce/k8s_generate_yaml_lib.py b/tensorflow/tools/dist_test/scripts_allreduce/k8s_generate_yaml_lib.py
index c570d1a9f8..038a712d53 100644
--- a/tensorflow/tools/dist_test/scripts_allreduce/k8s_generate_yaml_lib.py
+++ b/tensorflow/tools/dist_test/scripts_allreduce/k8s_generate_yaml_lib.py
@@ -195,7 +195,7 @@ def generate_RSA(bits=2048, exponent=65537):
 
 def get_change_ssh_port(use_hostnet, port):
   if use_hostnet == 1:
-    return "sed -i '/Port 22/c\Port {}' /etc/ssh/sshd_config".format(port)
+    return r"sed -i '/Port 22/c\Port {}' /etc/ssh/sshd_config".format(port)
 
   return ''
 
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/windows/msvc_wrapper_for_nvcc.py b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/windows/msvc_wrapper_for_nvcc.py
index 00483951af..2ea20b8b53 100755
--- a/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/windows/msvc_wrapper_for_nvcc.py
+++ b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/windows/msvc_wrapper_for_nvcc.py
@@ -104,7 +104,7 @@ def InvokeNvcc(argv, log=False):
   """
 
   src_files = [f for f in argv if
-               re.search('\.cpp$|\.cc$|\.c$|\.cxx$|\.C$', f)]
+               re.search(r'\.cpp$|\.cc$|\.c$|\.cxx$|\.C$', f)]
   if len(src_files) == 0:
     raise Error('No source files found for cuda compilation.')
 
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/windows/msvc_wrapper_for_nvcc.py b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/windows/msvc_wrapper_for_nvcc.py
index 859b3196d5..2d0898e9cb 100755
--- a/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/windows/msvc_wrapper_for_nvcc.py
+++ b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/windows/msvc_wrapper_for_nvcc.py
@@ -104,7 +104,7 @@ def InvokeNvcc(argv, log=False):
   """
 
   src_files = [f for f in argv if
-               re.search('\.cpp$|\.cc$|\.c$|\.cxx$|\.C$', f)]
+               re.search(r'\.cpp$|\.cc$|\.c$|\.cxx$|\.C$', f)]
   if len(src_files) == 0:
     raise Error('No source files found for cuda compilation.')
 
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc/windows/msvc_wrapper_for_nvcc.py b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc/windows/msvc_wrapper_for_nvcc.py
index 859b3196d5..2d0898e9cb 100755
--- a/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc/windows/msvc_wrapper_for_nvcc.py
+++ b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc/windows/msvc_wrapper_for_nvcc.py
@@ -104,7 +104,7 @@ def InvokeNvcc(argv, log=False):
   """
 
   src_files = [f for f in argv if
-               re.search('\.cpp$|\.cc$|\.c$|\.cxx$|\.C$', f)]
+               re.search(r'\.cpp$|\.cc$|\.c$|\.cxx$|\.C$', f)]
   if len(src_files) == 0:
     raise Error('No source files found for cuda compilation.')
 
-- 
GitLab


From 4ee52f8b0751004e76164464938cafed9f1f3623 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Thu, 3 Jan 2019 13:33:29 -0800
Subject: [PATCH 0143/2345] [TF:XLA] Bump open source llvm revision to r350327

PiperOrigin-RevId: 227740939
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 11ce55feda..7792824dbf 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -498,11 +498,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "65a1aeb29e5940f9f480a41e904659d944e738458afd139caa7bde14bd6aab8a",
-        strip_prefix = "llvm-331ffd31b3dd49b3f02a27556938b836b679f564",
+        sha256 = "3a1c8a817d2d9a85b85e1ad776a90faa51f475bb924fbf70383892bb1faceac9",
+        strip_prefix = "llvm-6da47bee848304d215e3ca08b2a4a4dc9c954310",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/331ffd31b3dd49b3f02a27556938b836b679f564.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/331ffd31b3dd49b3f02a27556938b836b679f564.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/6da47bee848304d215e3ca08b2a4a4dc9c954310.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/6da47bee848304d215e3ca08b2a4a4dc9c954310.tar.gz",
         ],
     )
 
-- 
GitLab


From 6f4410f580ff48078f33c343b72d742095535fae Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Thu, 3 Jan 2019 14:05:20 -0800
Subject: [PATCH 0144/2345] Disable on mac a flaky test.

PiperOrigin-RevId: 227746737
---
 tensorflow/lite/kernels/internal/BUILD | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/lite/kernels/internal/BUILD b/tensorflow/lite/kernels/internal/BUILD
index 5eb8a353b7..92b4705908 100644
--- a/tensorflow/lite/kernels/internal/BUILD
+++ b/tensorflow/lite/kernels/internal/BUILD
@@ -654,6 +654,8 @@ cc_test(
         "logsoftmax_quantized_test.cc",
     ],
     tags = [
+        # TODO(b/122242739): Reenable after fixing the flakiness?
+        "nomac",
         "tflite_not_portable",
     ],
     deps = [
-- 
GitLab


From 4dc0ae2baf454ba3d3ee531d889079b0ff05bc95 Mon Sep 17 00:00:00 2001
From: Blake Hechtman <blakehechtman@google.com>
Date: Thu, 3 Jan 2019 14:13:01 -0800
Subject: [PATCH 0145/2345] Fix typo.

PiperOrigin-RevId: 227748088
---
 tensorflow/compiler/xla/service/layout_assignment.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/layout_assignment.cc b/tensorflow/compiler/xla/service/layout_assignment.cc
index ebf7a88fae..09246db68e 100644
--- a/tensorflow/compiler/xla/service/layout_assignment.cc
+++ b/tensorflow/compiler/xla/service/layout_assignment.cc
@@ -1239,7 +1239,7 @@ Status LayoutAssignment::PropagateUseConstraintToDefs(
 namespace {
 // A transpose or a reshape that only changes trivial dimensions have meaningful
 // layouts that are valuable to propagate in a depthfirst manner to avoid
-// unassinged layouts in the graph.
+// unassigned layouts in the graph.
 bool InstructionShouldPropagateDepthFirst(const HloInstruction& hlo) {
   switch (hlo.opcode()) {
     case HloOpcode::kReshape:
-- 
GitLab


From 7c9323bedc48c98be3c07b72ec1d6f4dccdefb35 Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <penporn@google.com>
Date: Thu, 3 Jan 2019 14:22:17 -0800
Subject: [PATCH 0146/2345] Turn on MKL-DNN contraction kernels by default. To
 disable them, build with --define=tensorflow_mkldnn_contraction_kernel=0.
 Make TestFoldFusedBatchNormsWithConcat use ExpectClose which is more suitable
 for floating-point comparison than ExpectNear.

PiperOrigin-RevId: 227749705
---
 .bazelrc                                                       | 3 ---
 tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc | 2 +-
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/.bazelrc b/.bazelrc
index ceba7bfdba..cd7e13ddfc 100644
--- a/.bazelrc
+++ b/.bazelrc
@@ -93,9 +93,6 @@ build --define=PREFIX=/usr
 build --define=LIBDIR=$(PREFIX)/lib
 build --define=INCLUDEDIR=$(PREFIX)/include
 
-# Disable MKL-DNN contraction kernels by default.
-build --define=tensorflow_mkldnn_contraction_kernel=0
-
 # Default options should come above this line
 
 # Options from ./configure
diff --git a/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc b/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc
index 435f46c107..6c7174926d 100644
--- a/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc
+++ b/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc
@@ -291,7 +291,7 @@ class FoldOldBatchNormsTest : public ::testing::Test {
     std::vector<Tensor> fused_outputs;
     TF_ASSERT_OK(fused_session->Run({}, {"output"}, {}, &fused_outputs));
 
-    test::ExpectTensorNear<float>(original_outputs[0], fused_outputs[0], 1e-5);
+    test::ExpectClose(original_outputs[0], fused_outputs[0]);
 
     for (const NodeDef& node : fused_graph_def.node()) {
       EXPECT_NE("FusedBatchNorm", node.op());
-- 
GitLab


From 9d78259894c0b533ac6454c9eee27f8eb8bef638 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Thu, 3 Jan 2019 14:29:35 -0800
Subject: [PATCH 0147/2345] Add "element_shape" attr to TensorListConcat and
 use it in _GraphTensorArrayV2.

When using while_v2, this change prevents errors like this:
  All except the first dimension must be fully defined when concating an empty tensor list. element_shape: <unknown>

Previously, TensorListConcat op could only determine the element size
from the TensorList object at runtime. In the case of a while loop
that executes zero times, it wouldn't be able to determine the size
since no element was ever seen by the runtime. However, we may have
determined the element size during graph construction, so we use the
attribute to tell the runtime what the element size is.

Note that the original TensorArray implementation already does this.

PiperOrigin-RevId: 227750868
---
 tensorflow/core/kernels/list_kernels.h        | 44 +++++++++++++------
 tensorflow/core/ops/list_ops.cc               | 14 ++++--
 tensorflow/python/ops/list_ops.py             |  6 ++-
 .../ops/parallel_for/control_flow_ops_test.py |  5 ---
 tensorflow/python/ops/tensor_array_ops.py     | 12 +++--
 5 files changed, 53 insertions(+), 28 deletions(-)

diff --git a/tensorflow/core/kernels/list_kernels.h b/tensorflow/core/kernels/list_kernels.h
index 686679474c..99b4f42084 100644
--- a/tensorflow/core/kernels/list_kernels.h
+++ b/tensorflow/core/kernels/list_kernels.h
@@ -158,6 +158,17 @@ class TensorListConcat : public OpKernel {
       std::vector<std::unique_ptr<typename TTypes<T, 2>::ConstMatrix>>;
   explicit TensorListConcat(OpKernelConstruction* c) : OpKernel(c) {
     OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_));
+    // TODO(skyewm): the HasAttr check can be removed once the
+    // element_shape_except_first_dim attr has been checked in for 2 weeks
+    // (around 1/14/2019).
+    if (c->HasAttr("element_shape")) {
+      PartialTensorShape element_shape;
+      OP_REQUIRES_OK(c, c->GetAttr("element_shape", &element_shape));
+      if (!element_shape.unknown_rank()) {
+        element_shape_except_first_dim_ = PartialTensorShape(
+            gtl::ArraySlice<int64>(element_shape.dim_sizes()).subspan(1));
+      }
+    }
   }
 
   ~TensorListConcat() {}
@@ -178,29 +189,33 @@ class TensorListConcat : public OpKernel {
             " but list elements ", DataTypeString(tensor_list->element_dtype)));
     // If the TensorList is empty, its element_shape must be fully defined
     // except for the first dimension.
-    PartialTensorShape shape_except_first_dim;
-    if (!tensor_list->element_shape.unknown_rank()) {
-      OP_REQUIRES(c, tensor_list->element_shape.dims() >= 1,
-                  errors::InvalidArgument(
-                      "Concat requires elements to be at least vectors, ",
-                      "found scalars instead."));
-      shape_except_first_dim = PartialTensorShape(
-          gtl::ArraySlice<int64>(tensor_list->element_shape.dim_sizes())
-              .subspan(1));
+    if (!element_shape_except_first_dim_.IsFullyDefined()) {
+      if (!tensor_list->element_shape.unknown_rank()) {
+        OP_REQUIRES(c, tensor_list->element_shape.dims() >= 1,
+                    errors::InvalidArgument(
+                        "Concat requires elements to be at least vectors, ",
+                        "found scalars instead."));
+        PartialTensorShape shape_except_first_dim(
+            gtl::ArraySlice<int64>(tensor_list->element_shape.dim_sizes())
+                .subspan(1));
+        PartialTensorShape tmp = element_shape_except_first_dim_;
+        OP_REQUIRES_OK(c, tmp.MergeWith(shape_except_first_dim,
+                                        &element_shape_except_first_dim_));
+      }
     }
     OP_REQUIRES(c,
                 !tensor_list->tensors.empty() ||
-                    shape_except_first_dim.IsFullyDefined(),
+                    element_shape_except_first_dim_.IsFullyDefined(),
                 errors::InvalidArgument(
                     "All except the first dimension must be fully defined ",
                     "when concating an empty tensor list. element_shape: ",
                     tensor_list->element_shape.DebugString()));
     // 1. Compute the shape of the output tensor.
-    // If `shape_except_first_dim` is fully-defined we just prepend the leading
-    // dim to it. Otherwise we use the shape of the first element tensor and
-    // check to make sure shapes of all tensors are compatible.
+    // If `element_shape_except_first_dim_` is fully-defined we just prepend the
+    // leading dim to it. Otherwise we use the shape of the first element tensor
+    // and check to make sure shapes of all tensors are compatible.
     TensorShape output_shape;
-    if (!shape_except_first_dim.AsTensorShape(&output_shape)) {
+    if (!element_shape_except_first_dim_.AsTensorShape(&output_shape)) {
       const Tensor& element_tensor = tensor_list->tensors[0];
       OP_REQUIRES(
           c, TensorShapeUtils::IsVectorOrHigher(element_tensor.shape()),
@@ -268,6 +283,7 @@ class TensorListConcat : public OpKernel {
 
  private:
   DataType element_dtype_;
+  PartialTensorShape element_shape_except_first_dim_;
 };
 
 template <typename Device, typename T>
diff --git a/tensorflow/core/ops/list_ops.cc b/tensorflow/core/ops/list_ops.cc
index 5722170071..cbc9c7a2f4 100644
--- a/tensorflow/core/ops/list_ops.cc
+++ b/tensorflow/core/ops/list_ops.cc
@@ -212,10 +212,16 @@ REGISTER_OP("TensorListConcat")
     .Output("tensor: element_dtype")
     .Output("lengths: int64")
     .Attr("element_dtype: type")
+    .Attr("element_shape: shape = { unknown_rank: true }")
     .SetShapeFn([](shape_inference::InferenceContext* c) {
       DataType element_dtype;
       TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype));
-      shape_inference::ShapeHandle element_shape = c->UnknownShape();
+      PartialTensorShape raw_element_shape;
+      TF_RETURN_IF_ERROR(c->GetAttr("element_shape", &raw_element_shape));
+      shape_inference::ShapeHandle element_shape;
+      TF_RETURN_IF_ERROR(c->MakeShapeFromPartialTensorShape(raw_element_shape,
+                                                            &element_shape));
+
       auto* handle_data = c->input_handle_shapes_and_types(0);
       if (handle_data != nullptr && handle_data->size() != 1) {
         return errors::InvalidArgument(
@@ -231,10 +237,10 @@ REGISTER_OP("TensorListConcat")
               DataTypeString(list_shape_type.dtype), " but expected type ",
               DataTypeString(element_dtype));
         }
-        shape_inference::ShapeHandle ignored;
+        shape_inference::ShapeHandle merged;
         TF_RETURN_IF_ERROR(
-            c->Merge(element_shape, list_shape_type.shape, &ignored));
-        element_shape = list_shape_type.shape;
+            c->Merge(element_shape, list_shape_type.shape, &merged));
+        element_shape = merged;
       }
       if (c->RankKnown(element_shape)) {
         shape_inference::ShapeHandle result;
diff --git a/tensorflow/python/ops/list_ops.py b/tensorflow/python/ops/list_ops.py
index 3eac97655a..7bb9ca89e0 100644
--- a/tensorflow/python/ops/list_ops.py
+++ b/tensorflow/python/ops/list_ops.py
@@ -71,11 +71,13 @@ def tensor_list_from_tensor(tensor, element_shape, name=None):
       name=name)
 
 
-def tensor_list_concat(input_handle, element_dtype, name=None):
+def tensor_list_concat(input_handle, element_dtype, element_shape=None,
+                       name=None):
   # Ignore the lengths output of TensorListConcat. It is only used during
   # gradient computation.
   return gen_list_ops.tensor_list_concat(
-      input_handle=input_handle, element_dtype=element_dtype, name=name)[0]
+      input_handle=input_handle, element_dtype=element_dtype,
+      element_shape=element_shape, name=name)[0]
 
 
 def tensor_list_split(tensor, element_shape, lengths, name=None):
diff --git a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
index 8acb0d839c..933bddd8cc 100644
--- a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
+++ b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
@@ -27,7 +27,6 @@ from tensorflow.core.example import example_pb2
 from tensorflow.core.example import feature_pb2
 from tensorflow.python.client import session
 from tensorflow.python.eager import backprop
-from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -105,10 +104,6 @@ class PForTest(test.TestCase):
     flags.FLAGS.op_conversion_fallback_to_while_loop = False
 
   def test_parallel_iterations(self):
-    # TODO(b/121334512): Remove this check once this passes in Eager mode.
-    if context.executing_eagerly():
-      return
-
     for parallel_iterations in [2, 3, 8, 10]:
       x = random_ops.random_uniform([8, 3])
 
diff --git a/tensorflow/python/ops/tensor_array_ops.py b/tensorflow/python/ops/tensor_array_ops.py
index 9b8bd2c8d3..90a8b0af46 100644
--- a/tensorflow/python/ops/tensor_array_ops.py
+++ b/tensorflow/python/ops/tensor_array_ops.py
@@ -588,10 +588,16 @@ class _GraphTensorArrayV2(object):
 
   def concat(self, name=None):
     """See TensorArray."""
-    value = list_ops.tensor_list_concat(
-        input_handle=self._flow, element_dtype=self._dtype, name=name)
     if self._element_shape and self._element_shape[0].dims is not None:
-      value.set_shape([None] + self._element_shape[0].dims[1:])
+      element_shape = [None] + self._element_shape[0].dims[1:]
+    else:
+      element_shape = None
+
+    value = list_ops.tensor_list_concat(
+        input_handle=self._flow,
+        element_dtype=self._dtype,
+        element_shape=element_shape,
+        name=name)
     return value
 
   @tf_should_use.should_use_result
-- 
GitLab


From 000dc6e0face09fd2a2886b611e6d1d3d7776e5d Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Thu, 3 Jan 2019 14:36:59 -0800
Subject: [PATCH 0148/2345] Update svdf test case.

PiperOrigin-RevId: 227752127
---
 tensorflow/lite/kernels/svdf_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/kernels/svdf_test.cc b/tensorflow/lite/kernels/svdf_test.cc
index fc2bb49223..c420260bf5 100644
--- a/tensorflow/lite/kernels/svdf_test.cc
+++ b/tensorflow/lite/kernels/svdf_test.cc
@@ -209,7 +209,7 @@ class HybridSVDFOpModel : public BaseSVDFOpModel {
     tensor_type_ = tensor_type;
   }
 
-  void SetWeights(int weights_idx, std::vector<float> f) {
+  void SetWeights(int weights_idx, const std::vector<float>& f) {
     if (tensor_type_ == TensorType_UINT8) {
       SymmetricQuantizeAndPopulate(weights_idx, f);
     } else {
-- 
GitLab


From 7d57d32e440bc4f1654fb217fd701531a82c3587 Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Thu, 3 Jan 2019 14:45:53 -0800
Subject: [PATCH 0149/2345] Fix dso_loader.cc's includes.

PiperOrigin-RevId: 227753675
---
 tensorflow/contrib/mpi_collectives/BUILD                  | 2 +-
 tensorflow/stream_executor/platform/default/dso_loader.cc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/mpi_collectives/BUILD b/tensorflow/contrib/mpi_collectives/BUILD
index ecac06354d..d943ae6880 100644
--- a/tensorflow/contrib/mpi_collectives/BUILD
+++ b/tensorflow/contrib/mpi_collectives/BUILD
@@ -52,7 +52,7 @@ tf_custom_op_library(
     deps = [
         ":mpi_defines",
         ":mpi_message_proto_cc",
-        "//tensorflow/stream_executor:stream_executor_headers_lib",
+        "//tensorflow/core:stream_executor_headers_lib",
         "//third_party/mpi",
     ],
 )
diff --git a/tensorflow/stream_executor/platform/default/dso_loader.cc b/tensorflow/stream_executor/platform/default/dso_loader.cc
index 0f0bce3253..668eeee3f3 100644
--- a/tensorflow/stream_executor/platform/default/dso_loader.cc
+++ b/tensorflow/stream_executor/platform/default/dso_loader.cc
@@ -28,7 +28,7 @@ limitations under the License.
 #include "tensorflow/stream_executor/lib/path.h"
 #include "tensorflow/stream_executor/lib/str_util.h"
 #include "tensorflow/stream_executor/lib/stringprintf.h"
-#include "tensorflow/stream_executor/platform/dso_loader.h"
+#include "tensorflow/stream_executor/platform/default/dso_loader.h"
 #include "tensorflow/stream_executor/platform/logging.h"
 #include "tensorflow/stream_executor/platform/port.h"
 
-- 
GitLab


From 1051c377051b2ee24a495318737358d9ccf7280f Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 2 Jan 2019 14:34:42 -0800
Subject: [PATCH 0150/2345] Copy ben's changes

---
 .../contrib/tensorrt/convert/convert_nodes.cc |  25 ++-
 .../contrib/tensorrt/test/conv2d_test.py      | 172 ++++++++++++++++++
 2 files changed, 194 insertions(+), 3 deletions(-)
 create mode 100644 tensorflow/contrib/tensorrt/test/conv2d_test.py

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 2a402fe699..84d7f1aee5 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -62,14 +62,14 @@ limitations under the License.
 
 #define TFTRT_RETURN_ERROR_IF_FALSE(status, node) \
   do {                                            \
-    if (status == false) {                        \
+    if ((status) == false) {                        \
       TFTRT_INTERNAL_ERROR_AT_NODE(node);         \
     }                                             \
   } while (0)
 
 #define TFTRT_RETURN_ERROR_IF_NULLPTR(ptr, node) \
   do {                                           \
-    if (ptr == nullptr) {                        \
+    if ((ptr) == nullptr) {                        \
       TFTRT_INTERNAL_ERROR_AT_NODE(node);        \
     }                                            \
   } while (0)
@@ -1577,12 +1577,14 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
   const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
   TFAttrs attrs(node_def);
 
+  int c_index = 1;
   int h_index = 2;
   int w_index = 3;
   auto data_format = attrs.get<string>("data_format");
   if (data_format == "NHWC") {
     TF_RETURN_IF_ERROR(params->converter->TransposeTensor(
         const_cast<nvinfer1::ITensor*>(tensor), {0, 3, 1, 2}, &tensor));
+    c_index = 3;
     h_index = 1;
     w_index = 2;
     // TODO(jie): transpose it
@@ -1618,14 +1620,30 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
           << tf_stride[3];
   const nvinfer1::DimsHW stride(tf_stride[h_index], tf_stride[w_index]);
 
+  auto tf_dilations = attrs.get<std::vector<int>>("dilations");
+  if ((int)tf_dilations.size() != 4) {
+    return tensorflow::errors::InvalidArgument(
+        "Convolution dilations field must specify 4 dimensions " +
+        node_def.name());
+  }
+  if (tf_dilations[0] != 1 || tf_dilations[c_index] != 1) {
+    return tensorflow::errors::Unimplemented(
+        "Dilation rate must be 1 for batch and channel dimensions, at ",
+        node_def.name());
+  }
+  nvinfer1::DimsHW dilation(tf_dilations[h_index], tf_dilations[w_index]);
+
   std::vector<std::pair<int, int>> padding;
   // TODO(jie): padding.
   if (attrs.get<string>("padding") == "SAME") {
     // This is NCHW tensor with no batch dimension.
     //  1 -> h
     //  2 -> w
+    nvinfer1::DimsHW effective_kernel_size = kernel_size;
+    effective_kernel_size.h() += (kernel_size.h() - 1) * (dilation.h() - 1);
+    effective_kernel_size.w() += (kernel_size.w() - 1) * (dilation.w() - 1);
     padding = CreateSamePadding(
-        stride, kernel_size,
+        stride, effective_kernel_size,
         {static_cast<int>(tensor_dim.d[1]), static_cast<int>(tensor_dim.d[2])});
   } else {
     padding = {{0, 0}, {0, 0}};
@@ -1659,6 +1677,7 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
   layer->setPadding({padding[0].first, padding[1].first});
   layer->setName(node_def.name().c_str());
   layer->setNbGroups(num_groups);
+  layer->setDilation(dilation);
   const nvinfer1::ITensor* output_tensor = layer->getOutput(0);
   VLOG(2) << "TENSOR out: " << DebugString(output_tensor->getDimensions());
   VLOG(2) << "data_format: " << data_format;
diff --git a/tensorflow/contrib/tensorrt/test/conv2d_test.py b/tensorflow/contrib/tensorrt/test/conv2d_test.py
new file mode 100644
index 0000000000..bcf20fc504
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/test/conv2d_test.py
@@ -0,0 +1,172 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Model script to test TF-TensorRT integration."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gen_array_ops
+from tensorflow.python.ops import gen_nn_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn_ops
+from tensorflow.python.platform import test
+
+
+def conv2d_layer(inputs, filters, kernel_size, strides=(1, 1), padding='valid',
+                 data_format='channels_last', dilation_rate=(1, 1), name=None):
+  dtype = inputs.dtype
+  c_axis = -1 if data_format == 'channels_last' else 1
+  nchan = inputs.shape[c_axis]
+  weights_shape = (kernel_size[0], kernel_size[1], nchan, filters)
+  weights = constant_op.constant(np.random.randn(*weights_shape), dtype=dtype)
+  padding = padding.upper()
+  if data_format == 'channels_last':
+    strides = [1] + list(strides) + [1]
+    dilations = [1] + list(dilation_rate) + [1]
+    data_format = 'NHWC'
+  else:
+    strides = [1, 1] + list(strides)
+    dilations = [1, 1] + list(dilation_rate)
+    data_format = 'NCHW'
+  return gen_nn_ops.conv2d(inputs, weights, strides=strides, padding=padding,
+                           dilations=dilations, data_format=data_format)
+
+def div_round_up(n, d):
+  return (n - 1) // d + 1
+
+class Conv2DNCHWTest(trt_test.TfTrtIntegrationTestBase):
+
+  def GetParams(self):
+    """Testing conversion of Conv2D (data_format=NCHW) in TF-TRT conversion."""
+    np.random.seed(1234)
+    dtype = dtypes.float32
+    input_name = "input"
+    n, c, h, w = 13, 3, 7, 11
+    num_filters = 5
+    input_dims = [n, c, h, w]
+    output_name = "output"
+    g = ops.Graph()
+    with g.as_default():
+      inp = array_ops.placeholder(
+          dtype=dtype, shape=[None] + input_dims[1:], name=input_name)
+      with g.device("/GPU:0"):
+        results = []
+        for kernel_size in [(3, 3), (3, 2)]:
+          for dilation_rate in [(1, 1), (2, 3)]:
+            result = conv2d_layer(inp, num_filters, kernel_size,
+                                  dilation_rate=dilation_rate, padding='same',
+                                  data_format='channels_first')
+            results.append(result)
+        output = sum(results)
+        output = array_ops.identity(output, name=output_name)
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=[input_name],
+        input_dims=[input_dims],
+        output_names=[output_name],
+        expected_output_dims=[(n, num_filters, h, w)])
+
+  def ExpectedEnginesToBuild(self, run_params):
+    """Return the expected engines to build."""
+    return ["my_trt_op_0"]
+
+
+class Conv2DStridedNCHWTest(trt_test.TfTrtIntegrationTestBase):
+
+  def GetParams(self):
+    """Testing conversion of strided Conv2D (data_format=NCHW) in TF-TRT
+    conversion."""
+    np.random.seed(1234)
+    dtype = dtypes.float32
+    input_name = "input"
+    n, c, h, w = 13, 3, 7, 11
+    num_filters = 5
+    input_dims = [n, c, h, w]
+    output_name = "output"
+    g = ops.Graph()
+    with g.as_default():
+      inp = array_ops.placeholder(
+          dtype=dtype, shape=[None] + input_dims[1:], name=input_name)
+      with g.device("/GPU:0"):
+        output = inp
+        output = conv2d_layer(output, num_filters, (3, 2), strides=(2, 2),
+                              padding='same', data_format='channels_first')
+        h = div_round_up(h, 2)
+        w = div_round_up(w, 2)
+        output = conv2d_layer(output, num_filters, (3, 3), strides=(2, 2),
+                              dilation_rate=(2, 3), padding='same',
+                              data_format='channels_first')
+        h = div_round_up(h, 2)
+        w = div_round_up(w, 2)
+        output = array_ops.identity(output, name=output_name)
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=[input_name],
+        input_dims=[input_dims],
+        output_names=[output_name],
+        expected_output_dims=[(n, num_filters, h, w)])
+
+  def ExpectedEnginesToBuild(self, run_params):
+    """Return the expected engines to build."""
+    return ["my_trt_op_0"]
+
+
+class Conv2DNHWCTest(trt_test.TfTrtIntegrationTestBase):
+
+  def GetParams(self):
+    """Testing conversion of Conv2D (data_format=NHWC) in TF-TRT conversion."""
+    np.random.seed(1234)
+    dtype = dtypes.float32
+    input_name = "input"
+    n, h, w, c = 13, 7, 11, 3
+    num_filters = 5
+    input_dims = [n, h, w, c]
+    output_name = "output"
+    g = ops.Graph()
+    with g.as_default():
+      inp = array_ops.placeholder(
+          dtype=dtype, shape=[None] + input_dims[1:], name=input_name)
+      with g.device("/GPU:0"):
+        results = []
+        for kernel_size in [(3, 3), (3, 2)]:
+          for dilation_rate in [(1, 1), (2, 3)]:
+            result = conv2d_layer(inp, num_filters, kernel_size,
+                                  dilation_rate=dilation_rate, padding='same',
+                                  data_format='channels_last')
+            results.append(result)
+        output = sum(results)
+        output = array_ops.identity(output, name=output_name)
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=[input_name],
+        input_dims=[input_dims],
+        output_names=[output_name],
+        expected_output_dims=[(n, h, w, num_filters)])
+
+  def ExpectedEnginesToBuild(self, run_params):
+    """Return the expected engines to build."""
+    return ["my_trt_op_0"]
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From db95d28671c790de369a84c891f9bc971663e0e4 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 2 Jan 2019 15:12:40 -0800
Subject: [PATCH 0151/2345] Tidy up Conv2D converter

---
 tensorflow/contrib/tensorrt/BUILD             |  1 +
 .../contrib/tensorrt/convert/convert_nodes.cc | 94 ++++++++-----------
 .../contrib/tensorrt/test/conv2d_test.py      |  6 +-
 3 files changed, 43 insertions(+), 58 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 089c9ad97c..c17dea02f1 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -491,6 +491,7 @@ cuda_py_tests(
         "test/binary_tensor_weight_broadcast_test.py",
         "test/concatenation_test.py",
         "test/const_broadcast_test.py",
+        "test/conv2d_test.py",
         "test/identity_output_test.py",
         "test/manual_test.py",
         "test/memory_alignment_test.py",
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 84d7f1aee5..614cc47130 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -62,14 +62,14 @@ limitations under the License.
 
 #define TFTRT_RETURN_ERROR_IF_FALSE(status, node) \
   do {                                            \
-    if ((status) == false) {                        \
+    if ((status) == false) {                      \
       TFTRT_INTERNAL_ERROR_AT_NODE(node);         \
     }                                             \
   } while (0)
 
 #define TFTRT_RETURN_ERROR_IF_NULLPTR(ptr, node) \
   do {                                           \
-    if ((ptr) == nullptr) {                        \
+    if ((ptr) == nullptr) {                      \
       TFTRT_INTERNAL_ERROR_AT_NODE(node);        \
     }                                            \
   } while (0)
@@ -1567,41 +1567,56 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
                                              node_def.name());
   }
   TRT_ShapedWeights weights_rsck = inputs.at(1).weights();
-  VLOG(2) << "weight shape: " << weights_rsck.DebugString();
   if (weights_rsck.shape_.nbDims != 4) {
     return tensorflow::errors::Internal(
         "Conv2D expects kernel of dimension 4, at: " + node_def.name());
   }
+  TFAttrs attrs(node_def);
+  auto data_format = attrs.get<string>("data_format");
+  int c_index = (data_format == "NHWC") ? 3 : 1;
+  int h_index = (data_format == "NHWC") ? 1 : 2;
+  int w_index = (data_format == "NHWC") ? 2 : 3;
+  auto tf_dilations = attrs.get<std::vector<int>>("dilations");
+  if (tf_dilations.size() != 4) {
+    return tensorflow::errors::InvalidArgument(
+        "Convolution dilations field must specify 4 dimensions, at ",
+        node_def.name());
+  }
+  if (tf_dilations[0] != 1 || tf_dilations[c_index] != 1) {
+    return tensorflow::errors::Unimplemented(
+        "Dilation rate must be 1 for batch and channel dimensions, at ",
+        node_def.name());
+  }
+  nvinfer1::DimsHW dilation(tf_dilations[h_index], tf_dilations[w_index]);
+  const auto tf_stride = attrs.get<std::vector<int>>("strides");
+  if (tf_stride[0] != 1 || tf_stride[c_index] != 1) {
+    return tensorflow::errors::Unimplemented(
+        "Stride must be 1 for batch and channel dimensions, at ",
+        node_def.name());
+  }
+  const nvinfer1::DimsHW stride(tf_stride[h_index], tf_stride[w_index]);
   if (params->validation_only) return tensorflow::Status::OK();
 
   const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
-  TFAttrs attrs(node_def);
 
-  int c_index = 1;
-  int h_index = 2;
-  int w_index = 3;
-  auto data_format = attrs.get<string>("data_format");
-  if (data_format == "NHWC") {
+  // Transpose to NCHW (NCHW is required for IConvLayer).
+  const bool need_transpose = (data_format == "NHWC");
+  if (need_transpose) {
     TF_RETURN_IF_ERROR(params->converter->TransposeTensor(
         const_cast<nvinfer1::ITensor*>(tensor), {0, 3, 1, 2}, &tensor));
-    c_index = 3;
-    h_index = 1;
-    w_index = 2;
-    // TODO(jie): transpose it
   }
-
-  // tensor after transpose (NCHW)
+  // Dimensions of transposed tensor.
   const auto tensor_dim = tensor->getDimensions();
 
+  // This is a depthwise convolution when num_groups is 0. Otherwise, num_groups
+  // will be 1.
   int num_groups = group;
-  if (num_groups == 0) num_groups = tensor_dim.d[0];  // depthwise convolution
-  VLOG(2) << "groups count: " << num_groups;
+  if (num_groups == 0) num_groups = tensor_dim.d[0];
 
   if (params->converter->precision_mode() == FP16MODE) {
     weights_rsck =
         ConvertFP32ToFP16(params->weight_store, inputs.at(1).weights());
   }
-
   TRT_ShapedWeights weights =
       params->weight_store->GetTempWeights(weights_rsck);
   ReorderRSCKToKCRS(weights_rsck, &weights, num_groups);
@@ -1610,35 +1625,10 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
   nvinfer1::DimsHW kernel_size;
   kernel_size.h() = weights.shape_.d[2];
   kernel_size.w() = weights.shape_.d[3];
-  VLOG(2) << "RSCK: " << weights.DebugString();
-  VLOG(2) << "kernel size: " << kernel_size.h() << ", " << kernel_size.w();
-
-  // TODO(jie): stride. (NHWC/NCHW)
-  const auto tf_stride = attrs.get<std::vector<int>>("strides");
-  VLOG(2) << "h_INDEX" << h_index << ", w_index " << w_index;
-  VLOG(2) << "stride: " << tf_stride[0] << tf_stride[1] << tf_stride[2]
-          << tf_stride[3];
-  const nvinfer1::DimsHW stride(tf_stride[h_index], tf_stride[w_index]);
-
-  auto tf_dilations = attrs.get<std::vector<int>>("dilations");
-  if ((int)tf_dilations.size() != 4) {
-    return tensorflow::errors::InvalidArgument(
-        "Convolution dilations field must specify 4 dimensions " +
-        node_def.name());
-  }
-  if (tf_dilations[0] != 1 || tf_dilations[c_index] != 1) {
-    return tensorflow::errors::Unimplemented(
-        "Dilation rate must be 1 for batch and channel dimensions, at ",
-        node_def.name());
-  }
-  nvinfer1::DimsHW dilation(tf_dilations[h_index], tf_dilations[w_index]);
 
+  // Add padding.
   std::vector<std::pair<int, int>> padding;
-  // TODO(jie): padding.
   if (attrs.get<string>("padding") == "SAME") {
-    // This is NCHW tensor with no batch dimension.
-    //  1 -> h
-    //  2 -> w
     nvinfer1::DimsHW effective_kernel_size = kernel_size;
     effective_kernel_size.h() += (kernel_size.h() - 1) * (dilation.h() - 1);
     effective_kernel_size.w() += (kernel_size.w() - 1) * (dilation.w() - 1);
@@ -1648,13 +1638,9 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
   } else {
     padding = {{0, 0}, {0, 0}};
   }
-
   if (padding[0].first != padding[0].second ||
       padding[1].first != padding[1].second) {
-    // TODO(jie): handle asymmetric padding
-    VLOG(2) << "Padding!!!: " << padding[0].first << padding[0].second
-            << padding[1].first << padding[1].second;
-    VLOG(2) << "TENSOR before: " << DebugString(tensor->getDimensions());
+    // Handle asymmetric padding.
     auto pad_layer = params->converter->network()->addPadding(
         *const_cast<nvinfer1::ITensor*>(tensor),
         nvinfer1::DimsHW(padding[0].first, padding[1].first),
@@ -1664,25 +1650,23 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
         const_cast<nvinfer1::ITensor*>(tensor), pad_layer->getOutput(0));
     padding = {{0, 0}, {0, 0}};
     tensor = pad_layer->getOutput(0);
-    VLOG(2) << "TENSOR after: " << DebugString(tensor->getDimensions());
   }
 
+  // Add convolution.
   nvinfer1::IConvolutionLayer* layer =
       params->converter->network()->addConvolution(
           *const_cast<nvinfer1::ITensor*>(tensor), noutput, kernel_size,
           weights.GetTrtWeights(), biases.GetTrtWeights());
   TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name());
-
   layer->setStride(stride);
   layer->setPadding({padding[0].first, padding[1].first});
   layer->setName(node_def.name().c_str());
   layer->setNbGroups(num_groups);
   layer->setDilation(dilation);
   const nvinfer1::ITensor* output_tensor = layer->getOutput(0);
-  VLOG(2) << "TENSOR out: " << DebugString(output_tensor->getDimensions());
-  VLOG(2) << "data_format: " << data_format;
-  if (data_format == "NHWC") {
-    // TODO(jie): transpose it back!
+
+  // Restore transpose.
+  if (need_transpose) {
     TF_RETURN_IF_ERROR(params->converter->TransposeTensor(
         const_cast<nvinfer1::ITensor*>(output_tensor), {0, 2, 3, 1},
         &output_tensor));
diff --git a/tensorflow/contrib/tensorrt/test/conv2d_test.py b/tensorflow/contrib/tensorrt/test/conv2d_test.py
index bcf20fc504..c5228dea33 100644
--- a/tensorflow/contrib/tensorrt/test/conv2d_test.py
+++ b/tensorflow/contrib/tensorrt/test/conv2d_test.py
@@ -88,7 +88,7 @@ class Conv2DNCHWTest(trt_test.TfTrtIntegrationTestBase):
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
-    return ["my_trt_op_0"]
+    return ["TRTEngineOp_0"]
 
 
 class Conv2DStridedNCHWTest(trt_test.TfTrtIntegrationTestBase):
@@ -128,7 +128,7 @@ class Conv2DStridedNCHWTest(trt_test.TfTrtIntegrationTestBase):
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
-    return ["my_trt_op_0"]
+    return ["TRTEngineOp_0"]
 
 
 class Conv2DNHWCTest(trt_test.TfTrtIntegrationTestBase):
@@ -165,7 +165,7 @@ class Conv2DNHWCTest(trt_test.TfTrtIntegrationTestBase):
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
-    return ["my_trt_op_0"]
+    return ["TRTEngineOp_0"]
 
 
 if __name__ == "__main__":
-- 
GitLab


From a42746772faea0ab58d421dc68220ebc98c053bc Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 2 Jan 2019 16:13:58 -0800
Subject: [PATCH 0152/2345] Add unit test for Conv2D validator

---
 .../contrib/tensorrt/convert/convert_nodes.cc |  23 ++--
 .../tensorrt/convert/convert_nodes_test.cc    | 102 ++++++++++++++++++
 2 files changed, 118 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 614cc47130..34ba2aeaa4 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -1556,6 +1556,11 @@ enum class ConvolutionType { DEFAULT, DEPTHWISE_CONV };
 tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
+  if (inputs.size() != 2) {
+    return tensorflow::errors::InvalidArgument("Two inputs are expected for ",
+                                               node_def.op(), ", at ",
+                                               node_def.name());
+  }
   if (inputs.at(0).is_weights()) {
     return tensorflow::errors::Unimplemented(
         node_def.op(), " is only implemented for tensors, not weights, at ",
@@ -1568,8 +1573,8 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
   }
   TRT_ShapedWeights weights_rsck = inputs.at(1).weights();
   if (weights_rsck.shape_.nbDims != 4) {
-    return tensorflow::errors::Internal(
-        "Conv2D expects kernel of dimension 4, at: " + node_def.name());
+    return tensorflow::errors::InvalidArgument(
+        "Conv2D expects kernel of dimension 4, at " + node_def.name());
   }
   TFAttrs attrs(node_def);
   auto data_format = attrs.get<string>("data_format");
@@ -1587,8 +1592,13 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
         "Dilation rate must be 1 for batch and channel dimensions, at ",
         node_def.name());
   }
-  nvinfer1::DimsHW dilation(tf_dilations[h_index], tf_dilations[w_index]);
+  const nvinfer1::DimsHW dilation(tf_dilations[h_index], tf_dilations[w_index]);
   const auto tf_stride = attrs.get<std::vector<int>>("strides");
+  if (tf_stride.size() != 4) {
+    return tensorflow::errors::InvalidArgument(
+        "Convolution strides field must specify 4 dimensions, at ",
+        node_def.name());
+  }
   if (tf_stride[0] != 1 || tf_stride[c_index] != 1) {
     return tensorflow::errors::Unimplemented(
         "Stride must be 1 for batch and channel dimensions, at ",
@@ -1608,10 +1618,9 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
   // Dimensions of transposed tensor.
   const auto tensor_dim = tensor->getDimensions();
 
-  // This is a depthwise convolution when num_groups is 0. Otherwise, num_groups
-  // will be 1.
-  int num_groups = group;
-  if (num_groups == 0) num_groups = tensor_dim.d[0];
+  // For depthwise convolution, group will be 0 so set num_groups to size of
+  // input's channel dim. For a non-depthwise conv, num_groups will be 1.
+  const int num_groups = (group == 0) ? tensor_dim.d[0] : group;
 
   if (params->converter->precision_mode() == FP16MODE) {
     weights_rsck =
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index a2ddfbffa5..cb73d2e07b 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -2378,6 +2378,8 @@ TEST_F(OpConverterTest, ConvertStridedSlice) {
   };
 
   {
+    // Input is weights, should fail.
+    Reset();
     NodeDef node_def = get_strided_slice_nodedef();
     AddTestWeights<int32>("input", {1, 2, 3}, {1, 2, 3, 4, 5, 6});
     AddTestWeights<int32>("begin", {4}, {0, 0, 0, 0});
@@ -2619,6 +2621,106 @@ TEST_F(OpConverterTest, ConvertStridedSlice) {
   }
 }
 
+TEST_F(OpConverterTest, ConvertConv2D) {
+  {
+    // Input list is empty, should fail.
+    NodeDef node_def = MakeNodeDef("my_conv2d", "Conv2D", {});
+    RunValidationAndConversion(
+        node_def, error::INVALID_ARGUMENT,
+        "Two inputs are expected for Conv2D, at my_conv2d");
+  }
+
+  // Get nodedef for Conv2D layer.
+  auto get_conv2d_nodedef = [](
+      std::vector<int> strides = {1, 1, 1, 1}, string padding = "SAME",
+      string data_format = "NCHW",
+      std::vector<int> dilations = {1, 1, 1, 1}) -> NodeDef {
+    Scope s = Scope::NewRootScope();
+    auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
+    auto filter = ops::Placeholder(s.WithOpName("weights"), DT_FLOAT);
+    ops::Conv2D::Attrs attrs =
+        ops::Conv2D::Attrs().DataFormat(data_format).Dilations(dilations);
+    auto conv2d = ops::Conv2D(s.WithOpName("my_conv2d"), input, filter, strides,
+                              padding, attrs);
+    return conv2d.operation.node()->def();
+  };
+
+  {
+    // Input is weights, should fail.
+    Reset();
+    NodeDef node_def = get_conv2d_nodedef();
+    AddTestWeights<float>("input", {1, 2, 3}, {1, 2, 3, 4, 5, 6});
+    AddTestWeights<float>("weights", {3, 3, 1, 1}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
+    RunValidationAndConversion(
+        node_def, error::UNIMPLEMENTED,
+        "Conv2D is only implemented for tensors, not weights, at my_conv2d");
+  }
+  {
+    // Filter is tensor, should fail.
+    Reset();
+    NodeDef node_def = get_conv2d_nodedef();
+    AddTestTensor("input", {1, 2, 3});
+    AddTestTensor("weights", {3, 3, 1, 1});
+    RunValidationAndConversion(
+        node_def, error::UNIMPLEMENTED,
+        "Kernel for Conv2D must be constant weights, at my_conv2d");
+  }
+  {
+    // Filter is not 4D, should fail.
+    Reset();
+    NodeDef node_def = get_conv2d_nodedef();
+    AddTestTensor("input", {1, 2, 3});
+    AddTestWeights<float>("weights", {3, 3, 1}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
+    RunValidationAndConversion(
+        node_def, error::INVALID_ARGUMENT,
+        "Conv2D expects kernel of dimension 4, at my_conv2d");
+  }
+  {
+    // Dilations is not 4D, should fail.
+    Reset();
+    NodeDef node_def =
+        get_conv2d_nodedef({1, 1, 1, 1}, "SAME", "NCHW", {1, 1, 1});
+    AddTestTensor("input", {1, 2, 3});
+    AddTestWeights<float>("weights", {3, 3, 1, 1}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
+    RunValidationAndConversion(
+        node_def, error::INVALID_ARGUMENT,
+        "Convolution dilations field must specify 4 dimensions, at my_conv2d");
+  }
+  {
+    // Dilation value is not 1 for channel, should fail.
+    Reset();
+    NodeDef node_def =
+        get_conv2d_nodedef({1, 1, 1, 1}, "SAME", "NCHW", {1, 2, 1, 1});
+    AddTestTensor("input", {1, 2, 3});
+    AddTestWeights<float>("weights", {3, 3, 1, 1}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
+    RunValidationAndConversion(node_def, error::UNIMPLEMENTED,
+                               "Dilation rate must be 1 for batch and channel "
+                               "dimensions, at my_conv2d");
+  }
+  {
+    // Strides is not 4D, should fail.
+    Reset();
+    NodeDef node_def =
+        get_conv2d_nodedef({1, 1, 1}, "SAME", "NCHW", {1, 1, 1, 1});
+    AddTestTensor("input", {1, 2, 3});
+    AddTestWeights<float>("weights", {3, 3, 1, 1}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
+    RunValidationAndConversion(
+        node_def, error::INVALID_ARGUMENT,
+        "Convolution strides field must specify 4 dimensions, at my_conv2d");
+  }
+  {
+    // Stride value is not 1 for channel, should fail.
+    Reset();
+    NodeDef node_def =
+        get_conv2d_nodedef({1, 2, 1, 1}, "SAME", "NCHW", {1, 1, 1, 1});
+    AddTestTensor("input", {1, 2, 3});
+    AddTestWeights<float>("weights", {3, 3, 1, 1}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
+    RunValidationAndConversion(
+        node_def, error::UNIMPLEMENTED,
+        "Stride must be 1 for batch and channel dimensions, at my_conv2d");
+  }
+}
+
 }  // namespace convert
 }  // namespace tensorrt
 }  // namespace tensorflow
-- 
GitLab


From 560e4dd67fa8f9b0b99b940e35d039cf6c40194a Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 2 Jan 2019 16:20:44 -0800
Subject: [PATCH 0153/2345] Remove unused imports

---
 tensorflow/contrib/tensorrt/test/conv2d_test.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/test/conv2d_test.py b/tensorflow/contrib/tensorrt/test/conv2d_test.py
index c5228dea33..f7d9ca6826 100644
--- a/tensorflow/contrib/tensorrt/test/conv2d_test.py
+++ b/tensorflow/contrib/tensorrt/test/conv2d_test.py
@@ -25,10 +25,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import gen_nn_ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import nn_ops
 from tensorflow.python.platform import test
 
 
-- 
GitLab


From cf471f0c15b0c9c78c44a238fe60fe1e31289503 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Thu, 3 Jan 2019 14:56:44 -0800
Subject: [PATCH 0154/2345] Add positive test cases for Conv2D. Reuse
 build_graph in conv2d_test.

---
 .../contrib/tensorrt/convert/convert_nodes.cc |   5 +-
 .../tensorrt/convert/convert_nodes_test.cc    | 134 ++++++++++++++++++
 .../contrib/tensorrt/test/conv2d_test.py      | 106 ++++++--------
 3 files changed, 183 insertions(+), 62 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 34ba2aeaa4..179d631a3e 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -62,14 +62,14 @@ limitations under the License.
 
 #define TFTRT_RETURN_ERROR_IF_FALSE(status, node) \
   do {                                            \
-    if ((status) == false) {                      \
+    if (status == false) {                        \
       TFTRT_INTERNAL_ERROR_AT_NODE(node);         \
     }                                             \
   } while (0)
 
 #define TFTRT_RETURN_ERROR_IF_NULLPTR(ptr, node) \
   do {                                           \
-    if ((ptr) == nullptr) {                      \
+    if (ptr == nullptr) {                        \
       TFTRT_INTERNAL_ERROR_AT_NODE(node);        \
     }                                            \
   } while (0)
@@ -1593,6 +1593,7 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
         node_def.name());
   }
   const nvinfer1::DimsHW dilation(tf_dilations[h_index], tf_dilations[w_index]);
+
   const auto tf_stride = attrs.get<std::vector<int>>("strides");
   if (tf_stride.size() != 4) {
     return tensorflow::errors::InvalidArgument(
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index cb73d2e07b..1e49026a34 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -2697,6 +2697,17 @@ TEST_F(OpConverterTest, ConvertConv2D) {
                                "Dilation rate must be 1 for batch and channel "
                                "dimensions, at my_conv2d");
   }
+  {
+    // Dilation value is not 1 for channel (NHWC), should fail.
+    Reset();
+    NodeDef node_def =
+        get_conv2d_nodedef({1, 1, 1, 1}, "SAME", "NHWC", {1, 1, 1, 2});
+    AddTestTensor("input", {2, 3, 1});
+    AddTestWeights<float>("weights", {3, 3, 1, 1}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
+    RunValidationAndConversion(node_def, error::UNIMPLEMENTED,
+                               "Dilation rate must be 1 for batch and channel "
+                               "dimensions, at my_conv2d");
+  }
   {
     // Strides is not 4D, should fail.
     Reset();
@@ -2719,6 +2730,129 @@ TEST_F(OpConverterTest, ConvertConv2D) {
         node_def, error::UNIMPLEMENTED,
         "Stride must be 1 for batch and channel dimensions, at my_conv2d");
   }
+
+  struct TestParams {
+    TestParams(const std::vector<int>& input_dims,
+               const std::vector<float>& input,
+               const std::vector<int>& filter_dims,
+               const std::vector<float>& filter,
+               const std::vector<int>& strides, const string& padding,
+               const string& data_format, const std::vector<int>& dilations,
+               const std::vector<int>& expected_output_dims,
+               const std::vector<float>& expected_output)
+        : input_dims(input_dims),
+          input(input),
+          filter_dims(filter_dims),
+          filter(filter),
+          strides(strides),
+          padding(padding),
+          data_format(data_format),
+          dilations(dilations),
+          expected_output_dims(expected_output_dims),
+          expected_output(expected_output) {}
+
+    std::vector<int> input_dims;
+    std::vector<float> input;
+    std::vector<int> filter_dims;
+    std::vector<float> filter;
+    std::vector<int> strides;
+    string padding;
+    string data_format;
+    std::vector<int> dilations;
+    std::vector<int> expected_output_dims;
+    std::vector<float> expected_output;
+  };
+
+  // Ok.
+  const int kConv2DOKCases = 6;
+  TestParams ok_params[kConv2DOKCases] = {
+      // Basic
+      TestParams{/*input_dims=*/{1, 2, 3},
+                 /*input=*/{0, 1, 2, 3, 3, 4},
+                 /*filter_dims=*/{1, 2, 1, 1},
+                 /*filter=*/{-1, 1},
+                 /*strides=*/{1, 1, 1, 1},
+                 /*padding=*/"VALID",
+                 /*data_format=*/"NCHW",
+                 /*dilations=*/{1, 1, 1, 1},
+                 /*expected_output_dims=*/{1, 2, 2},
+                 /*expected_output=*/{1, 1, 0, 1}},
+      // SAME padding (Asymmetric)
+      TestParams{/*input_dims=*/{1, 2, 3},
+                 /*input=*/{0, 1, 2, 3, 3, 4},
+                 /*filter_dims=*/{1, 2, 1, 1},
+                 /*filter=*/{-1, 1},
+                 /*strides=*/{1, 1, 1, 1},
+                 /*padding=*/"SAME",
+                 /*data_format=*/"NCHW",
+                 /*dilations=*/{1, 1, 1, 1},
+                 /*expected_output_dims=*/{1, 2, 3},
+                 /*expected_output=*/{1, 1, -2, 0, 1, -4}},
+      // SAME padding (Symmetric)
+      TestParams{/*input_dims=*/{1, 2, 3},
+                 /*input=*/{0, 1, 2, 3, 3, 4},
+                 /*filter_dims=*/{1, 3, 1, 1},
+                 /*filter=*/{-1, 0, 1},
+                 /*strides=*/{1, 1, 1, 1},
+                 /*padding=*/"SAME",
+                 /*data_format=*/"NCHW",
+                 /*dilations=*/{1, 1, 1, 1},
+                 /*expected_output_dims=*/{1, 2, 3},
+                 /*expected_output=*/{1, 2, -1, 3, 1, -3}},
+      // NHWC
+      TestParams{/*input_dims=*/{2, 3, 1},
+                 /*input=*/{0, 1, 2, 3, 3, 4},
+                 /*filter_dims=*/{1, 2, 1, 1},
+                 /*filter=*/{-1, 1},
+                 /*strides=*/{1, 1, 1, 1},
+                 /*padding=*/"VALID",
+                 /*data_format=*/"NHWC",
+                 /*dilations=*/{1, 1, 1, 1},
+                 /*expected_output_dims=*/{2, 2, 1},
+                 /*expected_output=*/{1, 1, 0, 1}},
+      // Dilated
+      TestParams{/*input_dims=*/{1, 2, 3},
+                 /*input=*/{0, 1, 2, 3, 3, 4},
+                 /*filter_dims=*/{1, 2, 1, 1},
+                 /*filter=*/{-1, 1},
+                 /*strides=*/{1, 1, 1, 1},
+                 /*padding=*/"VALID",
+                 /*data_format=*/"NCHW",
+                 /*dilations=*/{1, 1, 1, 2},
+                 /*expected_output_dims=*/{1, 2, 1},
+                 /*expected_output=*/{2, 1}},
+      // Strided
+      TestParams{/*input_dims=*/{1, 2, 4},
+                 /*input=*/{0, 1, 2, 2, 3, 4, 4, 7},
+                 /*filter_dims=*/{1, 2, 1, 1},
+                 /*filter=*/{-1, 1},
+                 /*strides=*/{1, 1, 1, 2},
+                 /*padding=*/"VALID",
+                 /*data_format=*/"NCHW",
+                 /*dilations=*/{1, 1, 1, 1},
+                 /*expected_output_dims=*/{1, 2, 2},
+                 /*expected_output=*/{1, 0, 1, 3}},
+  };
+
+  for (int i = 0; i < kConv2DOKCases; i++) {
+    Reset();
+    NodeDef node_def =
+        get_conv2d_nodedef(ok_params[i].strides, ok_params[i].padding,
+                           ok_params[i].data_format, ok_params[i].dilations);
+    AddTestTensor("input", ok_params[i].input_dims);
+    AddTestWeights<float>("weights", ok_params[i].filter_dims,
+                          ok_params[i].filter);
+    RunValidationAndConversion(node_def);
+    TRT_TensorOrWeights output;
+    TF_EXPECT_OK(GetTensorOrWeights("my_conv2d", &output));
+    EXPECT_TRUE(output.is_tensor());
+    ExpectTrtDimsEqualsArray(ok_params[i].expected_output_dims,
+                             output.tensor()->getDimensions());
+    std::vector<float> output_data(ok_params[i].expected_output.size());
+    BuildAndRun<float>({{"input", ok_params[i].input}}, "my_conv2d",
+                       &output_data);
+    EXPECT_THAT(output_data, ElementsAreArray(ok_params[i].expected_output));
+  }
 }
 
 }  // namespace convert
diff --git a/tensorflow/contrib/tensorrt/test/conv2d_test.py b/tensorflow/contrib/tensorrt/test/conv2d_test.py
index f7d9ca6826..68b6534823 100644
--- a/tensorflow/contrib/tensorrt/test/conv2d_test.py
+++ b/tensorflow/contrib/tensorrt/test/conv2d_test.py
@@ -51,37 +51,60 @@ def conv2d_layer(inputs, filters, kernel_size, strides=(1, 1), padding='valid',
 def div_round_up(n, d):
   return (n - 1) // d + 1
 
+def build_graph(input_dims, dtype, num_filters, data_format, kernel_sizes,
+                dilation_rates, padding='same'):
+  g = ops.Graph()
+  with g.as_default():
+    inp = array_ops.placeholder(
+        dtype=dtype, shape=[None] + input_dims[1:], name="input")
+    with g.device("/GPU:0"):
+      results = []
+      for kernel_size in kernel_sizes:
+        for dilation_rate in dilation_rates:
+          result = conv2d_layer(inp, num_filters, kernel_size, (1, 1),
+                                padding, data_format, dilation_rate)
+          results.append(result)
+      output = sum(results)
+      output = array_ops.identity(output, name="output")
+  return g
+
+
 class Conv2DNCHWTest(trt_test.TfTrtIntegrationTestBase):
 
   def GetParams(self):
     """Testing conversion of Conv2D (data_format=NCHW) in TF-TRT conversion."""
     np.random.seed(1234)
-    dtype = dtypes.float32
-    input_name = "input"
-    n, c, h, w = 13, 3, 7, 11
-    num_filters = 5
-    input_dims = [n, c, h, w]
-    output_name = "output"
-    g = ops.Graph()
-    with g.as_default():
-      inp = array_ops.placeholder(
-          dtype=dtype, shape=[None] + input_dims[1:], name=input_name)
-      with g.device("/GPU:0"):
-        results = []
-        for kernel_size in [(3, 3), (3, 2)]:
-          for dilation_rate in [(1, 1), (2, 3)]:
-            result = conv2d_layer(inp, num_filters, kernel_size,
-                                  dilation_rate=dilation_rate, padding='same',
-                                  data_format='channels_first')
-            results.append(result)
-        output = sum(results)
-        output = array_ops.identity(output, name=output_name)
+    input_dims = [13, 3, 7, 11]
+    g = build_graph(input_dims=input_dims, dtype=dtypes.float32, num_filters=5,
+                    data_format="channels_first", kernel_sizes=[(3, 3), (3, 2)],
+                    dilation_rates=[(1, 1), (2, 3)])
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
-        input_names=[input_name],
+        input_names=["input"],
         input_dims=[input_dims],
-        output_names=[output_name],
-        expected_output_dims=[(n, num_filters, h, w)])
+        output_names=["output"],
+        expected_output_dims=[(13, 5, 7, 11)])
+
+  def ExpectedEnginesToBuild(self, run_params):
+    """Return the expected engines to build."""
+    return ["TRTEngineOp_0"]
+
+
+class Conv2DNHWCTest(trt_test.TfTrtIntegrationTestBase):
+
+  def GetParams(self):
+    """Testing conversion of Conv2D (data_format=NCHW) in TF-TRT conversion."""
+    np.random.seed(1234)
+    input_dims = [13, 7, 11, 3]
+    g = build_graph(input_dims=input_dims, dtype=dtypes.float32, num_filters=5,
+                    data_format="channels_last", kernel_sizes=[(3, 3), (3, 2)],
+                    dilation_rates=[(1, 1), (2, 3)])
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=["input"],
+        input_dims=[input_dims],
+        output_names=["output"],
+        expected_output_dims=[(13, 7, 11, 5)])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -128,42 +151,5 @@ class Conv2DStridedNCHWTest(trt_test.TfTrtIntegrationTestBase):
     return ["TRTEngineOp_0"]
 
 
-class Conv2DNHWCTest(trt_test.TfTrtIntegrationTestBase):
-
-  def GetParams(self):
-    """Testing conversion of Conv2D (data_format=NHWC) in TF-TRT conversion."""
-    np.random.seed(1234)
-    dtype = dtypes.float32
-    input_name = "input"
-    n, h, w, c = 13, 7, 11, 3
-    num_filters = 5
-    input_dims = [n, h, w, c]
-    output_name = "output"
-    g = ops.Graph()
-    with g.as_default():
-      inp = array_ops.placeholder(
-          dtype=dtype, shape=[None] + input_dims[1:], name=input_name)
-      with g.device("/GPU:0"):
-        results = []
-        for kernel_size in [(3, 3), (3, 2)]:
-          for dilation_rate in [(1, 1), (2, 3)]:
-            result = conv2d_layer(inp, num_filters, kernel_size,
-                                  dilation_rate=dilation_rate, padding='same',
-                                  data_format='channels_last')
-            results.append(result)
-        output = sum(results)
-        output = array_ops.identity(output, name=output_name)
-    return trt_test.TfTrtIntegrationTestParams(
-        gdef=g.as_graph_def(),
-        input_names=[input_name],
-        input_dims=[input_dims],
-        output_names=[output_name],
-        expected_output_dims=[(n, h, w, num_filters)])
-
-  def ExpectedEnginesToBuild(self, run_params):
-    """Return the expected engines to build."""
-    return ["TRTEngineOp_0"]
-
-
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 2ae28cb35f8060a1e7168b216adb13ce665cb349 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Thu, 3 Jan 2019 14:57:47 -0800
Subject: [PATCH 0155/2345] Add v2 log loss.

PiperOrigin-RevId: 227755606
---
 tensorflow/python/keras/losses.py      | 46 +++++++++++++
 tensorflow/python/keras/losses_test.py | 91 ++++++++++++++++++++++++++
 2 files changed, 137 insertions(+)

diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index d05dfb03f1..e6db795bb8 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -500,6 +500,46 @@ class CategoricalHinge(Loss):
     return categorical_hinge(y_true, y_pred)
 
 
+class LogLoss(Loss):
+  """Computes the log loss between `y_true` and `y_pred`.
+
+  logloss = -y log(p) - (1-y) log(1-p)
+
+  Usage:
+
+  ```python
+  l = tf.losses.LogLoss()
+  loss = l([0., 1., 1.], [1., 0., 1.])
+  print('Loss: ', loss.numpy())  # Loss: 10.745
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', loss=tf.losses.LogLoss())
+  ```
+
+  Args:
+    epsilon: A small increment to add to avoid taking a log of zero.
+    reduction: Type of `tf.losses.Reduction` to apply to loss. Default value is
+      `SUM_OVER_BATCH_SIZE`.
+    name: Optional name for the op.
+  """
+
+  def __init__(self,
+               epsilon=1e-7,
+               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               name=None):
+    super(LogLoss, self).__init__(reduction=reduction, name=name)
+    self.epsilon = epsilon
+
+  def call(self, y_true, y_pred):
+    y_pred = ops.convert_to_tensor(y_pred)
+    y_true = math_ops.cast(y_true, y_pred.dtype)
+    return logloss(y_true, y_pred, epsilon=self.epsilon)
+
+
 @keras_export('keras.metrics.mean_squared_error',
               'keras.metrics.mse',
               'keras.metrics.MSE',
@@ -562,6 +602,12 @@ def categorical_hinge(y_true, y_pred):
   return math_ops.maximum(0., neg - pos + 1.)
 
 
+def logloss(y_true, y_pred, epsilon=1e-7):
+  losses = math_ops.multiply(y_true, math_ops.log(y_pred + epsilon))
+  losses += math_ops.multiply((1 - y_true), math_ops.log(1 - y_pred + epsilon))
+  return K.mean(-losses, axis=-1)
+
+
 @keras_export('keras.losses.logcosh')
 def logcosh(y_true, y_pred):
   """Logarithm of the hyperbolic cosine of the prediction error.
diff --git a/tensorflow/python/keras/losses_test.py b/tensorflow/python/keras/losses_test.py
index 19ed7c8ed9..76fadc4a24 100644
--- a/tensorflow/python/keras/losses_test.py
+++ b/tensorflow/python/keras/losses_test.py
@@ -1003,5 +1003,96 @@ class CategoricalHingeTest(test.TestCase):
     self.assertAlmostEqual(self.evaluate(loss), 0., 3)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class LogLossTest(test.TestCase):
+
+  def setup(self):
+    # TODO(psv): Change to setUp() after b/122319309 is fixed.
+    y_pred = np.asarray([.9, .2, .2, .8, .4, .6]).reshape((2, 3))
+    y_true = np.asarray([1., 0., 1., 1., 0., 0.]).reshape((2, 3))
+    epsilon = 1e-7  # to avoid log 0
+
+    self.batch_size = 6
+    self.expected_losses = np.multiply(y_true, np.log(y_pred + epsilon))
+    self.expected_losses += np.multiply(1 - y_true,
+                                        np.log(1 - y_pred + epsilon))
+    self.expected_losses = -self.expected_losses
+
+    self.y_pred = constant_op.constant(y_pred)
+    self.y_true = constant_op.constant(y_true)
+
+  def test_config(self):
+    log_loss_obj = keras.losses.LogLoss(
+        reduction=losses_impl.ReductionV2.SUM, name='log')
+    self.assertEqual(log_loss_obj.name, 'log')
+    self.assertEqual(log_loss_obj.reduction, losses_impl.ReductionV2.SUM)
+
+  def test_all_correct(self):
+    self.setup()
+    log_loss_obj = keras.losses.LogLoss()
+    loss = log_loss_obj(self.y_true, self.y_true)
+    self.assertAlmostEqual(self.evaluate(loss), 0.0, 3)
+
+  def test_unweighted(self):
+    self.setup()
+    log_loss_obj = keras.losses.LogLoss()
+    loss = log_loss_obj(self.y_true, self.y_pred)
+    actual_loss = np.sum(self.expected_losses) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), actual_loss, 3)
+
+  def test_scalar_weighted(self):
+    self.setup()
+    log_loss_obj = keras.losses.LogLoss()
+    sample_weight = 2.3
+    loss = log_loss_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    actual_loss = sample_weight * np.sum(self.expected_losses) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), actual_loss, 3)
+
+    # Verify we get the same output when the same input is given
+    loss_2 = log_loss_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), self.evaluate(loss_2), 3)
+
+  def test_sample_weighted(self):
+    self.setup()
+    log_loss_obj = keras.losses.LogLoss()
+    sample_weight = constant_op.constant((1.2, 3.4), shape=(2, 1))
+
+    loss = log_loss_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    actual_loss = np.multiply(
+        self.expected_losses,
+        np.asarray([1.2, 1.2, 1.2, 3.4, 3.4, 3.4]).reshape((2, 3)))
+    actual_loss = np.sum(actual_loss) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), actual_loss, 3)
+
+  def test_timestep_weighted(self):
+    log_loss_obj = keras.losses.LogLoss()
+
+    y_pred = np.asarray([.9, .2, .2, .8, .4, .6]).reshape((2, 3, 1))
+    y_true = np.asarray([1., 0., 1., 1., 0., 0.]).reshape((2, 3, 1))
+    epsilon = 1e-7  # to avoid log 0
+    batch_size = 6
+
+    expected_losses = np.multiply(y_true, np.log(y_pred + epsilon))
+    expected_losses += np.multiply(1 - y_true, np.log(1 - y_pred + epsilon))
+
+    y_pred = constant_op.constant(y_pred)
+    y_true = constant_op.constant(y_true)
+    sample_weight = np.array([3, 6, 5, 0, 4, 2]).reshape((2, 3, 1))
+    loss = log_loss_obj(
+        y_true,
+        y_pred,
+        sample_weight=constant_op.constant(sample_weight, shape=(2, 3)))
+    actual_loss = np.multiply(-expected_losses, sample_weight)
+    actual_loss = np.sum(actual_loss) / batch_size
+    self.assertAlmostEqual(self.evaluate(loss), actual_loss, 3)
+
+  def test_zero_weighted(self):
+    self.setup()
+    log_loss_obj = keras.losses.LogLoss()
+    sample_weight = 0
+    loss = log_loss_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), 0., 3)
+
+
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 76df46cc71916d7027334b26cc2b390e4c432251 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 15:05:44 -0800
Subject: [PATCH 0156/2345] Automated rollback of commit
 4d9173668fd3ba410532efd901467312b7418752

PiperOrigin-RevId: 227757156
---
 tensorflow/cc/BUILD              |  1 -
 tensorflow/cc/saved_model/BUILD  |  2 -
 tensorflow/compiler/tf2xla/BUILD |  1 -
 tensorflow/core/BUILD            |  3 +-
 tensorflow/core/kernels/BUILD    | 83 +++++++++++++++++++++++++++++++-
 tensorflow/tensorflow.bzl        |  2 -
 6 files changed, 82 insertions(+), 10 deletions(-)

diff --git a/tensorflow/cc/BUILD b/tensorflow/cc/BUILD
index cf6d6050fa..a09becc49b 100644
--- a/tensorflow/cc/BUILD
+++ b/tensorflow/cc/BUILD
@@ -150,7 +150,6 @@ cc_library_with_android_deps(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:ops",
         "//tensorflow/core:protos_all_cc",
     ],
 )
diff --git a/tensorflow/cc/saved_model/BUILD b/tensorflow/cc/saved_model/BUILD
index 836678d364..52345a376c 100644
--- a/tensorflow/cc/saved_model/BUILD
+++ b/tensorflow/cc/saved_model/BUILD
@@ -81,7 +81,6 @@ cc_library(
     ] + if_not_mobile([
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:lib",
-        "//tensorflow/core:ops",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:tensorflow",
     ]) + if_android([
@@ -101,7 +100,6 @@ cc_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:ops",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/util/tensor_bundle:naming",
         # mobile not supported yet
diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD
index dfb6b57161..d8123e956f 100644
--- a/tensorflow/compiler/tf2xla/BUILD
+++ b/tensorflow/compiler/tf2xla/BUILD
@@ -281,7 +281,6 @@ tf_cc_test(
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
-        "//tensorflow/core:testlib",
     ],
 )
 
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 6b32e1fead..7c9decbd09 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1526,7 +1526,6 @@ cc_library(
         ":framework_internal",
         ":lib",
         ":lib_internal",
-        ":ops",
         ":protos_all_cc",
         ":shape_inference_testutil",
         ":tensor_testutil",
@@ -3078,7 +3077,6 @@ tf_cuda_library(
         ":lib",
         ":lib_internal",
         ":metrics",
-        ":ops",
         ":proto_text",
         ":protos_all_cc",
         "//tensorflow/core/debug:debug_graph_utils",
@@ -3861,6 +3859,7 @@ tf_cc_test(
         "ops/cudnn_rnn_ops_test.cc",
     ],
     deps = [
+        ":cudnn_rnn_ops",
         "//tensorflow/core",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index fd1df63c95..c72d04d3af 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -157,6 +157,7 @@ tf_kernel_library(
     name = "collective_ops",
     prefix = "collective_ops",
     deps = [
+        "//tensorflow/core:collective_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
@@ -221,6 +222,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:nn_ops_op_lib",
         "//third_party/eigen3",
     ],
     alwayslink = 1,
@@ -311,6 +313,7 @@ tf_kernel_library(
         "//tensorflow/core/nccl:nccl_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:gpu_headers_lib",
+        "//tensorflow/core:nccl_ops_op_lib",
     ]),
 )
 
@@ -456,6 +459,7 @@ cc_library(
     name = "batch_kernels",
     srcs = ["batch_kernels.cc"],
     deps = [
+        "//tensorflow/core:batch_ops_op_lib",
         "//tensorflow/core:framework_headers_lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/kernels:concat_lib_hdrs",
@@ -682,6 +686,7 @@ ARRAY_DEPS = [
     ":ops_util",
     ":transpose_functor",
     "//tensorflow/core:array_grad",
+    "//tensorflow/core:array_ops_op_lib",
     "//tensorflow/core:core_cpu",
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
@@ -710,6 +715,7 @@ tf_kernel_library(
     deps = [
         "//tensorflow/core:framework_headers_lib",
         "//tensorflow/core:lib",
+        "//tensorflow/core:set_ops_op_lib",
         "//third_party/eigen3",
     ],
 )
@@ -1073,6 +1079,7 @@ tf_kernel_library(
     srcs = ["ragged_gather_op.cc"],
     deps = [
         "//tensorflow/core:framework",
+        "//tensorflow/core:ragged_array_ops_op_lib",
     ],
 )
 
@@ -1083,6 +1090,7 @@ tf_cc_test(
     deps = [
         ":ragged_gather_op",
         "//tensorflow/core:framework",
+        "//tensorflow/core:ragged_array_ops_op_lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
@@ -1095,6 +1103,7 @@ tf_kernel_library(
     srcs = ["ragged_range_op.cc"],
     deps = [
         "//tensorflow/core:framework",
+        "//tensorflow/core:ragged_math_ops_op_lib",
     ],
 )
 
@@ -1104,6 +1113,7 @@ tf_cc_test(
     deps = [
         ":ragged_range_op",
         "//tensorflow/core:framework",
+        "//tensorflow/core:ragged_math_ops_op_lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
@@ -1116,6 +1126,7 @@ tf_kernel_library(
     srcs = ["ragged_tensor_to_sparse_kernel.cc"],
     deps = [
         "//tensorflow/core:framework",
+        "//tensorflow/core:ragged_conversion_ops_op_lib",
     ],
 )
 
@@ -1127,6 +1138,7 @@ tf_cc_test(
         ":ragged_tensor_to_sparse_kernel",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:ragged_conversion_ops_op_lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
@@ -1140,6 +1152,7 @@ tf_kernel_library(
     visibility = ["//visibility:public"],
     deps = [
         ":gpu_util_hdrs",
+        "//tensorflow/core:cudnn_rnn_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
@@ -1216,6 +1229,7 @@ tf_cuda_cc_test(
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:math_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
@@ -1740,6 +1754,7 @@ tf_kernel_library(
     prefix = "candidate_sampler_ops",
     deps = [
         ":range_sampler",
+        "//tensorflow/core:candidate_sampling_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
     ],
@@ -1773,6 +1788,7 @@ tf_kernel_library(
     name = "control_flow_ops",
     prefix = "control_flow_ops",
     deps = [
+        "//tensorflow/core:control_flow_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
     ],
@@ -1784,6 +1800,7 @@ tf_kernel_library(
     deps = [
         ":bounds_check",
         ":ops_util",
+        "//tensorflow/core:ctc_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core/util/ctc:ctc_beam_search_lib",
@@ -1864,6 +1881,7 @@ DATA_FLOW_DEPS = [
     ":typed_queue",
     "//third_party/eigen3",
     "//tensorflow/core:core_cpu",
+    "//tensorflow/core:data_flow_ops_op_lib",
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
@@ -1928,6 +1946,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:scoped_allocator_ops_op_lib",
     ],
 )
 
@@ -1946,6 +1965,7 @@ tf_cuda_cc_test(
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:math_ops_op_lib",
         "//tensorflow/core:proto_text",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
@@ -1993,6 +2013,7 @@ tf_kernel_library(
 DYNAMIC_DEPS = [
     ":bounds_check",
     "//tensorflow/core:core_cpu",
+    "//tensorflow/core:data_flow_ops_op_lib",
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
@@ -2025,6 +2046,7 @@ LOOKUP_DEPS = [
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
+    "//tensorflow/core:lookup_ops_op_lib",
 ]
 
 tf_kernel_library(
@@ -2053,6 +2075,7 @@ tf_kernel_library(
     deps = [
         ":lookup_table_init_op",
         ":lookup_table_op",
+        "//tensorflow/core:checkpoint_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//third_party/eigen3",
@@ -2063,6 +2086,7 @@ tf_kernel_library(
     name = "load_and_remap_matrix_op",
     srcs = ["load_and_remap_matrix_op.cc"],
     deps = [
+        "//tensorflow/core:checkpoint_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
@@ -2209,6 +2233,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:resource_variable_ops_op_lib",
         "@com_google_absl//absl/strings",
     ],
 )
@@ -2225,6 +2250,7 @@ tf_kernel_library(
         ":concat_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:list_ops_op_lib",
         "//third_party/eigen3",
     ],
 )
@@ -2235,6 +2261,7 @@ tf_kernel_library(
     deps = [
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:user_ops_op_lib",
     ],
 )
 
@@ -2257,6 +2284,7 @@ tf_kernel_library(
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
+        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//third_party/eigen3",
@@ -2269,6 +2297,7 @@ tf_kernel_library(
     deps = [
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
+        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/grappler:grappler_item",
@@ -2311,6 +2340,7 @@ IMAGE_DEPS = [
     "//third_party/eigen3",
     "//tensorflow/core:framework",
     "//tensorflow/core:gif_internal",
+    "//tensorflow/core:image_ops_op_lib",
     "//tensorflow/core:jpeg_internal",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
@@ -2650,6 +2680,7 @@ cc_library(
 IO_DEPS = [
     ":ops_util",
     "//tensorflow/core:framework",
+    "//tensorflow/core:io_ops_op_lib",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
     "//tensorflow/core:protos_all_cc",
@@ -2693,6 +2724,7 @@ SAVE_RESTORE_DEPS = [
     ":bounds_check_lib",
     ":save_restore_tensor",
     "//tensorflow/core:framework",
+    "//tensorflow/core:io_ops_op_lib",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
     "//tensorflow/core:protos_all_cc",
@@ -2811,6 +2843,7 @@ LINALG_DEPS = [
     "//third_party/eigen3",
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
+    "//tensorflow/core:linalg_ops_op_lib",
 ] + if_cuda([
     ":cuda_solvers",
     ":transpose_functor",
@@ -2954,6 +2987,7 @@ LOGGING_DEPS = [
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
+    "//tensorflow/core:logging_ops_op_lib",
     "//tensorflow/core:protos_all_cc",
 ]
 
@@ -3025,6 +3059,7 @@ tf_kernel_library(
         ":bounds_check",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:manip_ops_op_lib",
         "//third_party/eigen3",
     ],
 )
@@ -3056,6 +3091,7 @@ MATH_DEPS = [
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
     "//tensorflow/core:math_grad",
+    "//tensorflow/core:math_ops_op_lib",
     "//third_party/eigen3",
 ]
 
@@ -3166,7 +3202,7 @@ tf_kernel_library(
 tf_kernel_library(
     name = "cwise_op",
     prefix = "cwise_op",
-    deps = MATH_DEPS,
+    deps = MATH_DEPS + ["//tensorflow/core:bitwise_ops_op_lib"],
 )
 
 tf_kernel_library(
@@ -3185,6 +3221,7 @@ tf_kernel_library(
     name = "fft_ops",
     prefix = "fft_ops",
     deps = MATH_DEPS + [
+        "//tensorflow/core:spectral_ops_op_lib",
     ] + if_cuda([
         "//tensorflow/core/platform/default/build_config:cufft_plugin",
     ]),
@@ -3383,7 +3420,10 @@ tf_cuda_cc_test(
         ":quantized_ops",
         "//tensorflow/cc:cc_ops",
         "//tensorflow/cc:client_session",
+        "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:framework",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
@@ -3641,6 +3681,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:nn_ops_op_lib",
     ] + select({
         ":xsmm_convolutions": [
             "@libxsmm_archive//:xsmm_avx",
@@ -3662,6 +3703,7 @@ tf_kernel_library(
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:nn_ops_op_lib",
     ] + if_cuda([
         "@cub_archive//:cub",
         "@local_config_cuda//cuda:cudnn_header",
@@ -3681,6 +3723,7 @@ tf_kernel_library(
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:nn_ops_op_lib",
     ] + if_cuda([
         "@local_config_cuda//cuda:cudnn_header",
     ]),
@@ -3728,8 +3771,9 @@ NN_DEPS = [
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
     "//tensorflow/core:nn_grad",
+    "//tensorflow/core:nn_ops_op_lib",
     "//third_party/eigen3",
-]
+] + if_mkl(["//tensorflow/core:mkl_nn_ops_op_lib"])
 
 tf_kernel_library(
     name = "batch_norm_op",
@@ -3849,6 +3893,7 @@ tf_kernel_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:nn_grad",
+        "//tensorflow/core:nn_ops_op_lib",
     ] + if_cuda(["@cub_archive//:cub"]),
 )
 
@@ -3956,6 +4001,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:stream_executor",
         "//third_party/eigen3",
     ],
@@ -3999,6 +4045,7 @@ tf_kernel_library(
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//third_party/eigen3",
     ],
 )
@@ -4080,6 +4127,7 @@ cc_library(
 PARSING_DEPS = [
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
+    "//tensorflow/core:parsing_ops_op_lib",
     "//tensorflow/core:proto_text",
     "//tensorflow/core:protos_all_cc",
 ]
@@ -4148,6 +4196,7 @@ RANDOM_OPS_DEPS = [
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
+    "//tensorflow/core:random_ops_op_lib",
 ]
 
 tf_kernel_library(
@@ -4186,6 +4235,7 @@ tf_kernel_library(
         ":random_op",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:stateless_random_ops_op_lib",
     ],
 )
 
@@ -4200,6 +4250,8 @@ cc_library(
 REQUIRED_DEPS = [
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
+    "//tensorflow/core:no_op_op_lib",
+    "//tensorflow/core:sendrecv_ops_op_lib",
 ]
 
 tf_kernel_library(
@@ -4260,6 +4312,7 @@ cc_library(
 SPARSE_DEPS = [
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
+    "//tensorflow/core:sparse_ops_op_lib",
 ]
 
 tf_kernel_library(
@@ -4504,6 +4557,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:sdca_ops_op_lib",
         "//third_party/eigen3",
         "@farmhash_archive//:farmhash",
     ],
@@ -4543,6 +4597,7 @@ STATE_DEPS = [
     "//third_party/eigen3",
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
+    "//tensorflow/core:state_ops_op_lib",
 ] + if_sycl(["//tensorflow/core:sycl_runtime"])
 
 tf_kernel_library(
@@ -4680,6 +4735,7 @@ STRING_DEPS = [
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
+    "//tensorflow/core:string_ops_op_lib",
 ]
 
 tf_kernel_library(
@@ -4830,6 +4886,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:string_ops_op_lib",
         "//third_party/eigen3",
         "//third_party/icu/data:conversion_data",
         "@icu//:common",
@@ -4851,6 +4908,7 @@ tf_kernel_library(
         ":variable_ops",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:training_ops_op_lib",
         "//third_party/eigen3",
     ],
 )
@@ -4909,6 +4967,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:random_ops_op_lib",
     ],
 )
 
@@ -4936,6 +4995,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:random_ops_op_lib",
     ],
 )
 
@@ -5844,9 +5904,12 @@ tf_kernel_library(
         ":ops_util",
         ":pooling_ops",
         ":quantization_utils",
+        "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//third_party/eigen3",
         "@gemmlowp",
     ],
@@ -6416,6 +6479,7 @@ tf_kernel_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:remote_fused_graph_ops_op_lib",
     ],
 )
 
@@ -6570,6 +6634,8 @@ tf_mkl_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:mkl_nn_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
     ] + mkl_deps(),
 )
 
@@ -6619,6 +6685,8 @@ tf_mkl_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:mkl_nn_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
     ] + mkl_deps(),
 )
 
@@ -6637,6 +6705,8 @@ tf_mkl_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:mkl_nn_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
     ] + mkl_deps(),
 )
 
@@ -6650,6 +6720,8 @@ tf_mkl_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:mkl_nn_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//third_party/eigen3",
     ] + mkl_deps(),
 )
@@ -6664,6 +6736,8 @@ tf_mkl_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:mkl_nn_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//third_party/eigen3",
     ] + mkl_deps(),
 )
@@ -6804,6 +6878,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:summary_ops_op_lib",
         "//tensorflow/core/lib/db:sqlite",
     ] + if_not_v2([
         "//tensorflow/contrib/tensorboard/db:schema",
@@ -6818,6 +6893,7 @@ tf_kernel_library(
         "decode_proto_op.cc",
     ],
     deps = [
+        "//tensorflow/core:decode_proto_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core/util/proto:decode",
@@ -6831,6 +6907,7 @@ tf_kernel_library(
     name = "encode_proto_op",
     srcs = ["encode_proto_op.cc"],
     deps = [
+        "//tensorflow/core:encode_proto_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core/util/proto:descriptors",
@@ -6848,6 +6925,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:rpc_ops_op_lib",
         "//tensorflow/core/util/rpc:call_container",
         "//tensorflow/core/util/rpc:rpc_factory",
         "//tensorflow/core/util/rpc:rpc_factory_registry",
@@ -6860,6 +6938,7 @@ tf_kernel_library(
     srcs = ["unicode_script_op.cc"],
     deps = [
         "//tensorflow/core:framework",
+        "//tensorflow/core:string_ops_op_lib",
         "@icu//:common",
     ],
 )
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index b9069768e8..1024b686eb 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -590,7 +590,6 @@ def tf_gen_op_wrappers_cc(
             clean_dep("//tensorflow/core:core_cpu"),
             clean_dep("//tensorflow/core:framework"),
             clean_dep("//tensorflow/core:lib"),
-            clean_dep("//tensorflow/core:ops"),
             clean_dep("//tensorflow/core:protos_all_cc"),
         ]) + if_android([
             clean_dep("//tensorflow/core:android_tensorflow_lib"),
@@ -607,7 +606,6 @@ def tf_gen_op_wrappers_cc(
             clean_dep("//tensorflow/core:core_cpu"),
             clean_dep("//tensorflow/core:framework"),
             clean_dep("//tensorflow/core:lib"),
-            clean_dep("//tensorflow/core:ops"),
             clean_dep("//tensorflow/core:protos_all_cc"),
         ]) + if_android([
             clean_dep("//tensorflow/core:android_tensorflow_lib"),
-- 
GitLab


From 8f1e280350d1a0a6b4134327ecd029195438e767 Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Thu, 3 Jan 2019 15:08:10 -0800
Subject: [PATCH 0157/2345] Add a clang-specific warning -Wmismatched-tags.

PiperOrigin-RevId: 227757550
---
 tensorflow/stream_executor/cuda/cuda_dnn.cc | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index 7c05701895..01ff248150 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -51,6 +51,12 @@ limitations under the License.
 #include "absl/strings/string_view.h"
 // clang-format on
 
+#pragma clang diagnostic push
+
+// Make sure that Eigen::half forward declaration in dnn.h matches the
+// declaration in Eigen.
+#pragma clang diagnostic warning "-Wmismatched-tags"
+
 namespace stream_executor {
 namespace cuda {
 
@@ -4531,5 +4537,7 @@ void initialize_cudnn() {
 
 }  // namespace stream_executor
 
+#pragma clang diagnostic pop
+
 REGISTER_MODULE_INITIALIZER(register_cudnn,
                             { stream_executor::initialize_cudnn(); });
-- 
GitLab


From 2c65e8b01301d3d001952b3fc92d4a06c1f08bf5 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Thu, 3 Jan 2019 15:23:43 -0800
Subject: [PATCH 0158/2345] Throw an error when running loop inside scope

PiperOrigin-RevId: 227760023
---
 .../contrib/distribute/python/keras_test.py   | 19 +++++++++++++++++--
 .../engine/distributed_training_utils.py      | 11 +++++++++++
 .../keras/engine/training_distributed.py      |  6 ++++++
 3 files changed, 34 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index ba5a5e6400..4cfcc0375f 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -1161,8 +1161,8 @@ class TestDistributionStrategyWithNormalizationLayer(
       np.testing.assert_allclose(out.std(), 1.0, atol=1e-1)
 
 
-class TestDistributionStrategyVariableValidation(test.TestCase,
-                                                 parameterized.TestCase):
+class TestDistributionStrategyValidation(test.TestCase,
+                                         parameterized.TestCase):
 
   @combinations.generate(all_strategy_combinations_minus_default())
   def test_layer_outside_scope(self, distribution):
@@ -1192,6 +1192,21 @@ class TestDistributionStrategyVariableValidation(test.TestCase,
           metrics = ['mae', keras.metrics.CategoricalAccuracy()]
           model.compile(optimizer, loss, metrics=metrics)
 
+  @combinations.generate(all_strategy_combinations_minus_default())
+  def test_loop_in_scope(self, distribution):
+    with self.cached_session():
+      with self.assertRaisesRegexp(
+          RuntimeError, 'should not be run inside the distribution strategy'):
+        with distribution.scope():
+          x = keras.layers.Input(shape=(3,), name='input')
+          y = keras.layers.Dense(4, name='dense')(x)
+          model = keras.Model(x, y)
+          optimizer = gradient_descent.GradientDescentOptimizer(0.001)
+          loss = 'mse'
+          model.compile(optimizer, loss)
+          input_array = np.zeros((3, 3), dtype=np.float32)
+          model.predict(input_array)
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index 231c59ba93..9e3bdd26cb 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -38,10 +38,21 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.training import distribution_strategy_context
 from tensorflow.python.training.mode_keys import ModeKeys
 from tensorflow.python.util import nest
 
 
+def validate_not_in_strategy_scope():
+  """Validate fit/eval/predict are not running in DS scope."""
+  if distribution_strategy_context.has_distribution_strategy():
+    if distribution_strategy_context.in_cross_replica_context():
+      raise RuntimeError(
+          'Fit/Eval/Predict should not be run inside the distribution strategy '
+          'scope. Only model creation and compilation should be in '
+          'distribution strategy scope.')
+
+
 def set_weights(distribution_strategy, dist_model, weights):
   """Sets the weights of the replicated models.
 
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 0bd79f2b47..9f63f7bada 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -53,6 +53,8 @@ def fit_distributed(model,
                     steps_per_epoch=None,
                     validation_steps=None):
   """Fit loop for Distribution Strategies."""
+  # TODO(b/122314600): Remove the scope validate.
+  distributed_training_utils.validate_not_in_strategy_scope()
   distributed_training_utils.validate_callbacks(callbacks, model.optimizer)
   distributed_training_utils.validate_inputs(
       x, y, model._distribution_strategy)
@@ -136,6 +138,8 @@ def evaluate_distributed(model,
                          steps=None,
                          callbacks=None):
   """Evaluate loop for Distribution Strategies."""
+  # TODO(b/122314600): Remove the scope validate.
+  distributed_training_utils.validate_not_in_strategy_scope()
   distributed_training_utils.validate_inputs(x, y, model._distribution_strategy)
   first_x_value = nest.flatten(x)[0]
   if isinstance(first_x_value, np.ndarray):
@@ -171,6 +175,8 @@ def predict_distributed(model,
                         steps=None,
                         callbacks=None):
   """Predict loop for Distribution Strategies."""
+  # TODO(b/122314600): Remove the scope validate.
+  distributed_training_utils.validate_not_in_strategy_scope()
   distributed_training_utils.validate_inputs(
       x, None, model._distribution_strategy)
   first_x_value = nest.flatten(x)[0]
-- 
GitLab


From 6932e795621a76a7e520bee529d915265b40e9bd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 15:26:41 -0800
Subject: [PATCH 0159/2345] In tf.name_scope, throw an exception when the
 `default_name` arg is passed in but not a string and warn the user that they
 likely meant to pass it in as the `values` kwarg.

PiperOrigin-RevId: 227760445
---
 tensorflow/python/framework/ops.py      | 8 ++++++++
 tensorflow/python/framework/ops_test.py | 3 +++
 2 files changed, 11 insertions(+)

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 5dc8e418a2..079fad4210 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -6029,7 +6029,15 @@ class name_scope(object):  # pylint: disable=invalid-name
       name: The name argument that is passed to the op function.
       default_name: The default name to use if the `name` argument is `None`.
       values: The list of `Tensor` arguments that are passed to the op function.
+
+    Raises:
+      TypeError: if `default_name` is passed in but not a string.
     """
+    if not (default_name is None or isinstance(default_name, six.string_types)):
+      raise TypeError(
+          "`default_name` type (%s) is not a string type. You likely meant to "
+          "pass this into the `values` kwarg."
+          % type(default_name))
     self._name = default_name if name is None else name
     self._default_name = default_name
     self._values = values
diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py
index 2d7ee1a99e..58d311fe4e 100644
--- a/tensorflow/python/framework/ops_test.py
+++ b/tensorflow/python/framework/ops_test.py
@@ -2052,6 +2052,9 @@ class OpScopeTest(test_util.TensorFlowTestCase):
     with ops.name_scope(None, default_scope_name, [a, b]) as scope:
       self.assertEqual("%s/" % default_scope_name, scope)
       self.assertEqual(g0, ops.get_default_graph())
+    with self.assertRaises(TypeError):
+      with ops.name_scope(scope_name, [a, b]):
+        pass
 
   def _testGraphElements(self, graph_elements):
     scope_name = "my_scope"
-- 
GitLab


From 88c658a1bc737385e2edcbb7068406321151d875 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 16:18:43 -0800
Subject: [PATCH 0160/2345] Update ops-related pbtxt files.

PiperOrigin-RevId: 227768824
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 28 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 |  9 ++++++
 2 files changed, 37 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index d27f7a5eac..28d085b2d2 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -76625,6 +76625,34 @@ op {
     type: "type"
   }
 }
+op {
+  name: "TensorListConcat"
+  input_arg {
+    name: "input_handle"
+    type: DT_VARIANT
+  }
+  output_arg {
+    name: "tensor"
+    type_attr: "element_dtype"
+  }
+  output_arg {
+    name: "lengths"
+    type: DT_INT64
+  }
+  attr {
+    name: "element_dtype"
+    type: "type"
+  }
+  attr {
+    name: "element_shape"
+    type: "shape"
+    default_value {
+      shape {
+        unknown_rank: true
+      }
+    }
+  }
+}
 op {
   name: "TensorListConcatLists"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 412b4160d9..b4767d35d2 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -36756,6 +36756,15 @@ op {
     name: "element_dtype"
     type: "type"
   }
+  attr {
+    name: "element_shape"
+    type: "shape"
+    default_value {
+      shape {
+        unknown_rank: true
+      }
+    }
+  }
 }
 op {
   name: "TensorListConcatLists"
-- 
GitLab


From be57fb3a05ccfe01089fc2926e885c35ad5bda00 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 16:34:45 -0800
Subject: [PATCH 0161/2345] Small refactoring in preparation for a larger
 change.

PiperOrigin-RevId: 227771218
---
 tensorflow/lite/delegates/flex/kernel.cc      | 116 +++++++++++-------
 tensorflow/lite/delegates/flex/kernel_test.cc |   5 +-
 2 files changed, 70 insertions(+), 51 deletions(-)

diff --git a/tensorflow/lite/delegates/flex/kernel.cc b/tensorflow/lite/delegates/flex/kernel.cc
index d6e12ef650..250fc31043 100644
--- a/tensorflow/lite/delegates/flex/kernel.cc
+++ b/tensorflow/lite/delegates/flex/kernel.cc
@@ -52,11 +52,11 @@ namespace flex {
 namespace kernel {
 
 // Controls the lifetime of tensor handles in a vector.
-class VectorOfHandles {
+class OpOutputs {
  public:
-  explicit VectorOfHandles(int num_elements) : vector_(num_elements, nullptr) {}
+  explicit OpOutputs(int num_elements) : vector_(num_elements, nullptr) {}
 
-  ~VectorOfHandles() {
+  ~OpOutputs() {
     for (auto* handle : vector_) {
       if (handle) handle->Unref();
     }
@@ -72,6 +72,21 @@ class VectorOfHandles {
   tensorflow::gtl::InlinedVector<tensorflow::TensorHandle*, 2> vector_;
 };
 
+// A single node within the larger 'op'. Note that this kernel executes many
+// TensorFlow ops within a single TF Lite op.
+struct OpNode {
+  // The name of the TensorFlow op to execute.
+  string name;
+  // Index of this node into TF Lite's operator list.
+  int index;
+  // The corresponding NodeDef, containing the attributes for the op.
+  tensorflow::NodeDef nodedef;
+  // List of inputs, as TF Lite tensor indices.
+  std::vector<int> inputs;
+  // List of outputs, as TF Lite tensor indices.
+  std::vector<int> outputs;
+};
+
 // Executes the TensorFlow op given by 'op_name', with the attributes specified
 // in 'nodedef'. Inputs and outputs are given as indices into the 'buffer_map'.
 tensorflow::Status ExecuteFlexOp(tensorflow::EagerContext* eager_context,
@@ -114,7 +129,7 @@ tensorflow::Status ExecuteFlexOp(tensorflow::EagerContext* eager_context,
   }
 
   int num_retvals = outputs.size();
-  VectorOfHandles retvals(num_retvals);
+  OpOutputs retvals(num_retvals);
   TF_RETURN_WITH_CONTEXT_IF_ERROR(
       EagerExecute(&op, retvals.GetVector(), &num_retvals),
       " (while executing '", op_name, "' via Eager)");
@@ -133,22 +148,38 @@ tensorflow::Status ExecuteFlexOp(tensorflow::EagerContext* eager_context,
   return tensorflow::Status::OK();
 }
 
-// A single node within the larger 'op'. Note that this kernel executes many
-// TensorFlow ops within a single TF Lite op.
-struct OpNode {
-  // The name of the TensorFlow op to execute.
-  string name;
-  // Index of this node into TF Lite's operator list.
-  int index;
-  // The corresponding NodeDef, containing the attributes for the op.
-  tensorflow::NodeDef nodedef;
-  // List of inputs, as TF Lite tensor indices.
-  std::vector<int> inputs;
-  // List of outputs, as TF Lite tensor indices.
-  std::vector<int> outputs;
-};
+tensorflow::Status InitializeNodeDef(const void* custom_initial_data,
+                                     int custom_initial_data_size,
+                                     OpNode* node_data) {
+  if (!custom_initial_data) {
+    return tensorflow::errors::Internal(
+        "Cannot convert empty data into a valid NodeDef");
+  }
+  // The flexbuffer contains a vector where the first elements is the
+  // op name and the second is a serialized NodeDef.
+  const flexbuffers::Vector& v =
+      flexbuffers::GetRoot(
+          reinterpret_cast<const uint8_t*>(custom_initial_data),
+          custom_initial_data_size)
+          .AsVector();
+
+  node_data->name = v[0].AsString().str();
+  if (!node_data->nodedef.ParseFromString(v[1].AsString().str())) {
+    node_data->nodedef.Clear();
+    return tensorflow::errors::Internal(
+        "Failed to parse data into a valid NodeDef");
+  }
+
+  // Fill NodeDef with defaults if it's a valid op.
+  const tensorflow::OpRegistrationData* op_reg_data;
+  TF_RETURN_IF_ERROR(tensorflow::OpRegistry::Global()->LookUp(
+      node_data->nodedef.op(), &op_reg_data));
+  AddDefaultsToNodeDef(op_reg_data->op_def, &node_data->nodedef);
 
-// The Larger 'op', which contains all the nodes in a supported subgraph.
+  return tensorflow::Status::OK();
+}
+
+// The larger 'op', which contains all the nodes in a supported subgraph.
 struct OpData {
   tensorflow::EagerContext* eager_context;
   BufferMap* buffer_map;
@@ -182,6 +213,7 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) {
   }
 
   CHECK(params->nodes_to_replace);
+  tensorflow::Status status;
   for (auto node_index : TfLiteIntArrayView(params->nodes_to_replace)) {
     TfLiteNode* node;
     TfLiteRegistration* reg;
@@ -192,29 +224,10 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) {
 
     node_data.index = node_index;
     node_data.name = "";
-    if (node->custom_initial_data) {
-      // The flexbuffer contains a vector where the first elements is the
-      // op name and the second is a serialized NodeDef.
-      const flexbuffers::Vector& v =
-          flexbuffers::GetRoot(
-              reinterpret_cast<const uint8_t*>(node->custom_initial_data),
-              node->custom_initial_data_size)
-              .AsVector();
-
-      node_data.name = v[0].AsString().str();
-      if (!node_data.nodedef.ParseFromString(v[1].AsString().str())) {
-        // We will just leave the nodedef empty and error out in Eval().
-        node_data.nodedef.Clear();
-      }
-    }
 
-    // Fill NodeDef with defaults if it's a valid op.
-    const tensorflow::OpRegistrationData* op_reg_data;
-    auto tf_status = tensorflow::OpRegistry::Global()->LookUp(
-        node_data.nodedef.op(), &op_reg_data);
-    if (tf_status.ok()) {
-      AddDefaultsToNodeDef(op_reg_data->op_def, &node_data.nodedef);
-    }
+    status = InitializeNodeDef(node->custom_initial_data,
+                               node->custom_initial_data_size, &node_data);
+    if (!status.ok()) break;
 
     for (auto input_index : TfLiteIntArrayView(node->inputs)) {
       node_data.inputs.push_back(input_index);
@@ -224,6 +237,13 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) {
     }
   }
 
+  if (ConvertStatus(context, status) != kTfLiteOk) {
+    // We can't return an error from this function but ConvertStatus will
+    // report them and we will stop processing in Prepare() if anything went
+    // wrong.
+    return op_data;
+  }
+
   return op_data;
 }
 
@@ -263,6 +283,12 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   }
 
   for (const auto& node_data : op_data->nodes) {
+    if (node_data.nodedef.op().empty()) {
+      context->ReportError(context, "Invalid NodeDef in Flex op '%s'",
+                           node_data.name.c_str());
+      return kTfLiteError;
+    }
+
     for (int tensor_index : node_data.inputs) {
       ++tensor_ref_count[tensor_index];
     }
@@ -303,12 +329,8 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   for (const auto& node_data : op_data->nodes) {
     SCOPED_TAGGED_OPERATOR_PROFILE(
         reinterpret_cast<profiling::Profiler*>(context->profiler),
-        node_data.name.c_str(), node_data.index);
-    if (node_data.nodedef.op().empty()) {
-      context->ReportError(context, "Invalid NodeDef in Flex op '%s'",
-                           node_data.name.c_str());
-      return kTfLiteError;
-    }
+        node_data->name.c_str(), node_data->index);
+
     auto status =
         ExecuteFlexOp(eager_context, buffer_map, node_data.name,
                       node_data.nodedef, node_data.inputs, node_data.outputs);
diff --git a/tensorflow/lite/delegates/flex/kernel_test.cc b/tensorflow/lite/delegates/flex/kernel_test.cc
index cc5c8b32a0..647c199d4f 100644
--- a/tensorflow/lite/delegates/flex/kernel_test.cc
+++ b/tensorflow/lite/delegates/flex/kernel_test.cc
@@ -166,10 +166,7 @@ TEST_F(KernelTest, WrongSetOfNodes) {
     return GenericPrepare(context, delegate, {0, 1});
   });
 
-  SetShape(0, {2, 2, 1});
-  SetValues(0, {1.1f, 2.2f, 3.3f, 4.4f});
-
-  ASSERT_FALSE(Invoke());
+  ASSERT_NE(interpreter_->AllocateTensors(), kTfLiteOk);
   ASSERT_THAT(error_reporter().error_messages(),
               ContainsRegex("Invalid NodeDef in Flex op"));
 }
-- 
GitLab


From 03258acbf396a1241dee813e86848dc768394afd Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Thu, 3 Jan 2019 17:15:41 -0800
Subject: [PATCH 0162/2345] [tf.data] Replacing active / total parallel calls
 metric with thread utilization metric.

PiperOrigin-RevId: 227776910
---
 .../experimental/map_and_batch_dataset_op.cc  | 22 ++++++++-----------
 .../data/parallel_interleave_dataset_op.cc    | 19 +++++++---------
 .../kernels/data/parallel_map_iterator.cc     | 19 +++++++---------
 .../kernel_tests/stats_dataset_test_base.py   |  4 +---
 4 files changed, 26 insertions(+), 38 deletions(-)

diff --git a/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc b/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
index ef75c84456..a6348e4647 100644
--- a/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
@@ -259,7 +259,7 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
                                             params.dataset->batch_size_)) {
         std::vector<string> components =
             str_util::Split(params.prefix, "::", str_util::SkipEmpty());
-        prefix_end_ = components.back();
+        key_prefix_ = components.back();
       }
 
       ~Iterator() override {
@@ -397,8 +397,9 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
         const auto& stats_aggregator = ctx->stats_aggregator();
         if (stats_aggregator) {
           stats_aggregator->AddScalar(
-              strings::StrCat(prefix_end_, "::active_parallel_calls"),
-              static_cast<float>(num_calls_));
+              strings::StrCat(key_prefix_, "::thread_utilization"),
+              static_cast<float>(num_calls_) /
+                  static_cast<float>(num_parallel_calls_->value));
         }
         cond_var_->notify_all();
       }
@@ -637,18 +638,13 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
               num_calls_++;
             }
           }
-          const std::shared_ptr<StatsAggregator>& stats_aggregator =
-              ctx->stats_aggregator();
+          const auto& stats_aggregator = ctx->stats_aggregator();
           if (stats_aggregator) {
             mutex_lock l(*mu_);
-            // TODO(shivaniagrawal): add `parallel_calls_utilization` in the
-            // monitoring code or as histogram at fixed time intervals.
             stats_aggregator->AddScalar(
-                strings::StrCat(prefix_end_, "::active_parallel_calls"),
-                static_cast<float>(num_calls_));
-            stats_aggregator->AddScalar(
-                strings::StrCat(prefix_end_, "::num_parallel_calls"),
-                static_cast<float>(num_parallel_calls_->value));
+                strings::StrCat(key_prefix_, "::thread_utilization"),
+                static_cast<float>(num_calls_) /
+                    static_cast<float>(num_parallel_calls_->value));
           }
           for (const auto& call : new_calls) {
             CallFunction(ctx, call.first, call.second);
@@ -803,7 +799,7 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
       int64 waiting_ GUARDED_BY(*mu_) = 0;
       // Identifies the maximum number of batch results to store.
       int64 max_batch_results_ GUARDED_BY(*mu_);
-      string prefix_end_;
+      string key_prefix_;
       std::unique_ptr<InstantiatedCapturedFunction> instantiated_captured_func_;
     };
 
diff --git a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
index 74791669ce..315f96d4b5 100644
--- a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
+++ b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
@@ -209,7 +209,7 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
                 false /* low_latency_hint */)) {
         std::vector<string> components =
             str_util::Split(params.prefix, "::", str_util::SkipEmpty());
-        prefix_end_ = components.back();
+        key_prefix_ = components.back();
       }
 
       ~ParallelInterleaveIterator() override {
@@ -430,8 +430,9 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
         const auto& stats_aggregator = ctx->stats_aggregator();
         if (stats_aggregator) {
           stats_aggregator->AddScalar(
-              strings::StrCat(prefix_end_, "::active_parallel_calls"),
-              static_cast<float>(num_calls_));
+              strings::StrCat(key_prefix_, "::thread_utilization"),
+              static_cast<float>(num_calls_) /
+                  static_cast<float>(num_parallel_calls_->value));
         }
         cond_var_->notify_all();
       }
@@ -515,14 +516,10 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
           }
           const auto& stats_aggregator = ctx->stats_aggregator();
           if (stats_aggregator) {
-            // TODO(shivaniagrawal): add `parallel_calls_utilization` in the
-            // monitoring code or as histogram at fixed time intervals.
             stats_aggregator->AddScalar(
-                strings::StrCat(prefix_end_, "::active_parallel_calls"),
-                static_cast<float>(num_calls_));
-            stats_aggregator->AddScalar(
-                strings::StrCat(prefix_end_, "::num_parallel_calls"),
-                static_cast<float>(num_parallel_calls_->value));
+                strings::StrCat(key_prefix_, "::thread_utilization"),
+                static_cast<float>(num_calls_) /
+                    static_cast<float>(num_parallel_calls_->value));
           }
           cond_var_->notify_all();
         }
@@ -693,7 +690,7 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
 
       // Identifies whether background activity should be cancelled.
       bool cancelled_ GUARDED_BY(*mu_) = false;
-      string prefix_end_;
+      string key_prefix_;
       std::unique_ptr<InstantiatedCapturedFunction> instantiated_captured_func_;
     };
 
diff --git a/tensorflow/core/kernels/data/parallel_map_iterator.cc b/tensorflow/core/kernels/data/parallel_map_iterator.cc
index b62e7059ba..9a4ce981d0 100644
--- a/tensorflow/core/kernels/data/parallel_map_iterator.cc
+++ b/tensorflow/core/kernels/data/parallel_map_iterator.cc
@@ -60,7 +60,7 @@ class ParallelMapIterator : public DatasetBaseIterator {
         preserve_cardinality_(params.preserve_cardinality) {
     std::vector<string> components =
         str_util::Split(base_params.prefix, "::", str_util::SkipEmpty());
-    prefix_end_ = components.back();
+    key_prefix_ = components.back();
   }
 
   ~ParallelMapIterator() override {
@@ -207,8 +207,9 @@ class ParallelMapIterator : public DatasetBaseIterator {
     const auto& stats_aggregator = ctx->stats_aggregator();
     if (stats_aggregator) {
       stats_aggregator->AddScalar(
-          strings::StrCat(prefix_end_, "::active_parallel_calls"),
-          static_cast<float>(num_calls_));
+          strings::StrCat(key_prefix_, "::thread_utilization"),
+          static_cast<float>(num_calls_) /
+              static_cast<float>(num_parallel_calls_->value));
     }
     RecordBufferEnqueue(ctx.get(), result->return_values);
     result->notification.Notify();
@@ -300,14 +301,10 @@ class ParallelMapIterator : public DatasetBaseIterator {
         }
         const auto& stats_aggregator = ctx->stats_aggregator();
         if (stats_aggregator) {
-          // TODO(shivaniagrawal): add `parallel_calls_utilization` in the
-          // monitoring code or as histogram at fixed time intervals.
-          stats_aggregator->AddScalar(
-              strings::StrCat(prefix_end_, "::active_parallel_calls"),
-              static_cast<float>(num_calls_));
           stats_aggregator->AddScalar(
-              strings::StrCat(prefix_end_, "::num_parallel_calls"),
-              static_cast<float>(num_parallel_calls_->value));
+              strings::StrCat(key_prefix_, "::thread_utilization"),
+              static_cast<float>(num_calls_) /
+                  static_cast<float>(num_parallel_calls_->value));
         }
         cond_var_->notify_all();
       }
@@ -403,7 +400,7 @@ class ParallelMapIterator : public DatasetBaseIterator {
       GUARDED_BY(*mu_);
   std::unique_ptr<Thread> runner_thread_ GUARDED_BY(*mu_);
   bool cancelled_ GUARDED_BY(*mu_) = false;
-  string prefix_end_;
+  string key_prefix_;
 };
 
 }  // namespace
diff --git a/tensorflow/python/data/experimental/kernel_tests/stats_dataset_test_base.py b/tensorflow/python/data/experimental/kernel_tests/stats_dataset_test_base.py
index b80aab994e..f5a15f4c84 100644
--- a/tensorflow/python/data/experimental/kernel_tests/stats_dataset_test_base.py
+++ b/tensorflow/python/data/experimental/kernel_tests/stats_dataset_test_base.py
@@ -104,9 +104,7 @@ class StatsDatasetTestBase(test_base.DatasetTestBase):
         self._assertSummaryHasCountMoreOrEqualGeneralisedTag(
             summary_str, "::execution_time", float(i + 1))
       self._assertSummaryContains(summary_str,
-                                  dataset_name + "::num_parallel_calls")
-      self._assertSummaryContains(summary_str,
-                                  dataset_name + "::active_parallel_calls")
+                                  dataset_name + "::thread_utilization")
     with self.assertRaises(errors.OutOfRangeError):
       self.evaluate(next_element())
     if function_processing_time:
-- 
GitLab


From 8cfb8d2c11c93e36d7db2635518427260fcc4d41 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 17:16:06 -0800
Subject: [PATCH 0163/2345] Turn OpNode into a proper class, in preparation for
 a larger change.

PiperOrigin-RevId: 227776959
---
 tensorflow/lite/delegates/flex/kernel.cc | 118 +++++++++++++----------
 1 file changed, 68 insertions(+), 50 deletions(-)

diff --git a/tensorflow/lite/delegates/flex/kernel.cc b/tensorflow/lite/delegates/flex/kernel.cc
index 250fc31043..7d18c39098 100644
--- a/tensorflow/lite/delegates/flex/kernel.cc
+++ b/tensorflow/lite/delegates/flex/kernel.cc
@@ -74,17 +74,66 @@ class OpOutputs {
 
 // A single node within the larger 'op'. Note that this kernel executes many
 // TensorFlow ops within a single TF Lite op.
-struct OpNode {
+class OpNode {
+ public:
+  OpNode() {}
+  ~OpNode() {}
+
+  const string& name() const { return name_; }
+  void set_name(const string& name) { name_ = name; }
+
+  int index() const { return index_; }
+  void set_index(int index) { index_ = index; }
+
+  const tensorflow::NodeDef& nodedef() const { return nodedef_; }
+
+  const std::vector<int>& inputs() const { return inputs_; }
+  std::vector<int>* mutable_inputs() { return &inputs_; }
+
+  const std::vector<int>& outputs() const { return outputs_; }
+  std::vector<int>* mutable_outputs() { return &outputs_; }
+
+  tensorflow::Status InitializeNodeDef(const void* custom_initial_data,
+                                       int custom_initial_data_size) {
+    if (!custom_initial_data) {
+      return tensorflow::errors::Internal(
+          "Cannot convert empty data into a valid NodeDef");
+    }
+    // The flexbuffer contains a vector where the first elements is the
+    // op name and the second is a serialized NodeDef.
+    const flexbuffers::Vector& v =
+        flexbuffers::GetRoot(
+            reinterpret_cast<const uint8_t*>(custom_initial_data),
+            custom_initial_data_size)
+            .AsVector();
+
+    name_ = v[0].AsString().str();
+    if (!nodedef_.ParseFromString(v[1].AsString().str())) {
+      nodedef_.Clear();
+      return tensorflow::errors::Internal(
+          "Failed to parse data into a valid NodeDef");
+    }
+
+    // Fill NodeDef with defaults if it's a valid op.
+    const tensorflow::OpRegistrationData* op_reg_data;
+    TF_RETURN_IF_ERROR(
+        tensorflow::OpRegistry::Global()->LookUp(nodedef_.op(), &op_reg_data));
+    AddDefaultsToNodeDef(op_reg_data->op_def, &nodedef_);
+
+    return tensorflow::Status::OK();
+  }
+
+ private:
   // The name of the TensorFlow op to execute.
-  string name;
+  string name_;
   // Index of this node into TF Lite's operator list.
-  int index;
+  int index_;
   // The corresponding NodeDef, containing the attributes for the op.
-  tensorflow::NodeDef nodedef;
+  tensorflow::NodeDef nodedef_;
   // List of inputs, as TF Lite tensor indices.
-  std::vector<int> inputs;
+  std::vector<int> inputs_;
   // List of outputs, as TF Lite tensor indices.
-  std::vector<int> outputs;
+  std::vector<int> outputs_;
 };
 
 // Executes the TensorFlow op given by 'op_name', with the attributes specified
@@ -148,37 +197,6 @@ tensorflow::Status ExecuteFlexOp(tensorflow::EagerContext* eager_context,
   return tensorflow::Status::OK();
 }
 
-tensorflow::Status InitializeNodeDef(const void* custom_initial_data,
-                                     int custom_initial_data_size,
-                                     OpNode* node_data) {
-  if (!custom_initial_data) {
-    return tensorflow::errors::Internal(
-        "Cannot convert empty data into a valid NodeDef");
-  }
-  // The flexbuffer contains a vector where the first elements is the
-  // op name and the second is a serialized NodeDef.
-  const flexbuffers::Vector& v =
-      flexbuffers::GetRoot(
-          reinterpret_cast<const uint8_t*>(custom_initial_data),
-          custom_initial_data_size)
-          .AsVector();
-
-  node_data->name = v[0].AsString().str();
-  if (!node_data->nodedef.ParseFromString(v[1].AsString().str())) {
-    node_data->nodedef.Clear();
-    return tensorflow::errors::Internal(
-        "Failed to parse data into a valid NodeDef");
-  }
-
-  // Fill NodeDef with defaults if it's a valid op.
-  const tensorflow::OpRegistrationData* op_reg_data;
-  TF_RETURN_IF_ERROR(tensorflow::OpRegistry::Global()->LookUp(
-      node_data->nodedef.op(), &op_reg_data));
-  AddDefaultsToNodeDef(op_reg_data->op_def, &node_data->nodedef);
-
-  return tensorflow::Status::OK();
-}
-
 // The larger 'op', which contains all the nodes in a supported subgraph.
 struct OpData {
   tensorflow::EagerContext* eager_context;
@@ -222,18 +240,18 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) {
     op_data->nodes.push_back(OpNode());
     OpNode& node_data = op_data->nodes.back();
 
-    node_data.index = node_index;
-    node_data.name = "";
+    node_data.set_index(node_index);
+    node_data.set_name("");
 
-    status = InitializeNodeDef(node->custom_initial_data,
-                               node->custom_initial_data_size, &node_data);
+    status = node_data.InitializeNodeDef(node->custom_initial_data,
+                                         node->custom_initial_data_size);
     if (!status.ok()) break;
 
     for (auto input_index : TfLiteIntArrayView(node->inputs)) {
-      node_data.inputs.push_back(input_index);
+      node_data.mutable_inputs()->push_back(input_index);
     }
     for (auto output_index : TfLiteIntArrayView(node->outputs)) {
-      node_data.outputs.push_back(output_index);
+      node_data.mutable_outputs()->push_back(output_index);
     }
   }
 
@@ -283,13 +301,13 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   }
 
   for (const auto& node_data : op_data->nodes) {
-    if (node_data.nodedef.op().empty()) {
+    if (node_data.nodedef().op().empty()) {
       context->ReportError(context, "Invalid NodeDef in Flex op '%s'",
-                           node_data.name.c_str());
+                           node_data.name().c_str());
       return kTfLiteError;
     }
 
-    for (int tensor_index : node_data.inputs) {
+    for (int tensor_index : node_data.inputs()) {
       ++tensor_ref_count[tensor_index];
     }
   }
@@ -329,11 +347,11 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   for (const auto& node_data : op_data->nodes) {
     SCOPED_TAGGED_OPERATOR_PROFILE(
         reinterpret_cast<profiling::Profiler*>(context->profiler),
-        node_data->name.c_str(), node_data->index);
+        node_data->name().c_str(), node_data->index());
 
-    auto status =
-        ExecuteFlexOp(eager_context, buffer_map, node_data.name,
-                      node_data.nodedef, node_data.inputs, node_data.outputs);
+    auto status = ExecuteFlexOp(eager_context, buffer_map, node_data.name(),
+                                node_data.nodedef(), node_data.inputs(),
+                                node_data.outputs());
     TF_LITE_ENSURE_OK(context, ConvertStatus(context, status));
   }
 
-- 
GitLab


From 1bad2986b24e520b6b1558d5970b4b25abff0ab3 Mon Sep 17 00:00:00 2001
From: Tong Shen <endlessroad@google.com>
Date: Thu, 3 Jan 2019 17:22:28 -0800
Subject: [PATCH 0164/2345] Support outside compilation in TPUPartitionedCall.

PiperOrigin-RevId: 227777744
---
 tensorflow/compiler/tf2xla/BUILD              |  1 +
 .../compiler/tf2xla/side_effect_util.cc       | 39 +++++++++++++++++++
 tensorflow/compiler/tf2xla/side_effect_util.h |  4 ++
 3 files changed, 44 insertions(+)

diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD
index d8123e956f..0366ec45fb 100644
--- a/tensorflow/compiler/tf2xla/BUILD
+++ b/tensorflow/compiler/tf2xla/BUILD
@@ -669,6 +669,7 @@ cc_library(
     name = "side_effect_util",
     srcs = ["side_effect_util.cc"],
     hdrs = ["side_effect_util.h"],
+    visibility = [":friends"],
     deps = [
         "//tensorflow/core:core_cpu",
         "@com_google_absl//absl/strings",
diff --git a/tensorflow/compiler/tf2xla/side_effect_util.cc b/tensorflow/compiler/tf2xla/side_effect_util.cc
index b62f8e9115..a9144cfc4f 100644
--- a/tensorflow/compiler/tf2xla/side_effect_util.cc
+++ b/tensorflow/compiler/tf2xla/side_effect_util.cc
@@ -26,6 +26,45 @@ const char kXlaTokenArgNodeName[] = "_xla_token_arg_node";
 
 const char kXlaHasHostTransferAttrName[] = "_xla_has_host_transfer";
 
+Status SetDeviceOrdinalAttributeForNode(Node* node, int device_ordinal) {
+  if (!HasNodeAttr(node->def(), kXlaHasHostTransferAttrName)) {
+    return errors::InvalidArgument("Node ", node->DebugString(),
+                                   " does not have attribute ",
+                                   kXlaHasHostTransferAttrName);
+  }
+
+  if (node->type_string() == "_XlaRecvAtHost" ||
+      node->type_string() == "_XlaSendFromHost") {
+    node->ClearAttr("device_ordinal");
+    node->AddAttr("device_ordinal", device_ordinal);
+  } else if (node->type_string() == "If") {
+    AttrValue device_ordinal_value;
+    device_ordinal_value.set_i(device_ordinal);
+    for (const string& attr_name :
+         std::vector<string>{"then_branch", "else_branch"}) {
+      NameAttrList branch_func;
+      TF_RETURN_IF_ERROR(GetNodeAttr(node->attrs(), attr_name, &branch_func));
+      (*branch_func.mutable_attr())["device_ordinal"] = device_ordinal_value;
+      node->ClearAttr(attr_name);
+      node->AddAttr(attr_name, branch_func);
+    }
+  } else if (node->type_string() == "While") {
+    AttrValue device_ordinal_value;
+    device_ordinal_value.set_i(device_ordinal);
+    for (const string& attr_name : std::vector<string>{"cond", "body"}) {
+      NameAttrList branch_func;
+      TF_RETURN_IF_ERROR(GetNodeAttr(node->attrs(), attr_name, &branch_func));
+      (*branch_func.mutable_attr())["device_ordinal"] = device_ordinal_value;
+      node->ClearAttr(attr_name);
+      node->AddAttr(attr_name, branch_func);
+    }
+  } else {
+    return errors::Internal("Unknown node type to set 'device_ordinal': ",
+                            node->DebugString());
+  }
+  return Status::OK();
+}
+
 std::set<std::string> CalculateTokenInputsForOutputToken(const Graph& g) {
   std::set<std::string> results;
   Node* first_side_effecting_node_on_path = nullptr;
diff --git a/tensorflow/compiler/tf2xla/side_effect_util.h b/tensorflow/compiler/tf2xla/side_effect_util.h
index 7081b362c3..75e1f253fb 100644
--- a/tensorflow/compiler/tf2xla/side_effect_util.h
+++ b/tensorflow/compiler/tf2xla/side_effect_util.h
@@ -38,6 +38,10 @@ extern const char kXlaTokenArgNodeName[];
 // This node have XlaRecvAtHost/XlaSendFromHost in its associated functions.
 extern const char kXlaHasHostTransferAttrName[];
 
+// Sets device ordinal attribute for nodes with attribute
+// `kXlaHasHostTransferAttrName`.
+Status SetDeviceOrdinalAttributeForNode(Node* node, int device_ordinal);
+
 // Calculates side-effect dependencies for the graph's token output.
 // Returns a set of node names representing these dependencies.
 std::set<std::string> CalculateTokenInputsForOutputToken(const Graph& g);
-- 
GitLab


From 7f9d4f0b93f256440662fa12e5508b552537803a Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Thu, 3 Jan 2019 17:31:51 -0800
Subject: [PATCH 0165/2345] Enable Tensorflow custom contraction kernel support
 in XLA single threaded matmul.

PiperOrigin-RevId: 227778830
---
 tensorflow/compiler/xla/service/cpu/BUILD                     | 1 +
 .../xla/service/cpu/runtime_single_threaded_matmul.cc         | 4 ++++
 2 files changed, 5 insertions(+)

diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index f49b5110be..a4ca772c7c 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -631,6 +631,7 @@ cc_library(
     deps = [
         ":runtime_matvec",
         "//tensorflow/core:framework_lite",
+        "//tensorflow/core/kernels:eigen_contraction_kernel",
         "//third_party/eigen3",
     ],
 )
diff --git a/tensorflow/compiler/xla/service/cpu/runtime_single_threaded_matmul.cc b/tensorflow/compiler/xla/service/cpu/runtime_single_threaded_matmul.cc
index 1ed743afc3..1f7204e67a 100644
--- a/tensorflow/compiler/xla/service/cpu/runtime_single_threaded_matmul.cc
+++ b/tensorflow/compiler/xla/service/cpu/runtime_single_threaded_matmul.cc
@@ -20,6 +20,10 @@ limitations under the License.
 #include "tensorflow/core/platform/dynamic_annotations.h"
 #include "tensorflow/core/platform/types.h"
 
+#if defined(TENSORFLOW_USE_CUSTOM_CONTRACTION_KERNEL)
+#include "tensorflow/core/kernels/eigen_contraction_kernel.h"
+#endif
+
 using tensorflow::int32;
 using tensorflow::int64;
 
-- 
GitLab


From 846dba8e81fde8a47196c9b04d31f847e5cc336f Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Thu, 3 Jan 2019 18:05:00 -0800
Subject: [PATCH 0166/2345] Automated rollback of commit
 64596a68235709e7f3246d0b120a4146521a998b

PiperOrigin-RevId: 227782472
---
 tensorflow/lite/experimental/swift/BUILD      | 100 -----
 tensorflow/lite/experimental/swift/LICENSE    | 202 ----------
 tensorflow/lite/experimental/swift/README.md  |  54 ---
 .../swift/Sources/Interpreter.swift           | 265 --------------
 .../swift/Sources/InterpreterError.swift      |  99 -----
 .../swift/Sources/InterpreterOptions.swift    |  29 --
 .../experimental/swift/Sources/Model.swift    |  40 --
 .../Sources/QuantizationParameters.swift      |  38 --
 .../experimental/swift/Sources/Tensor.swift   | 138 -------
 .../Configs/TensorFlowLite.tulsigen           |  57 ---
 .../project.tulsiconf                         |  14 -
 .../project.pbxproj                           | 345 ------------------
 .../TensorFlowLiteApp/AppDelegate.swift       |  24 --
 .../Array+TensorFlowLite.swift                |  22 --
 .../AppIcon.appiconset/Contents.json          |  98 -----
 .../Assets.xcassets/Contents.json             |   6 -
 .../Base.lproj/LaunchScreen.storyboard        |  44 ---
 .../Base.lproj/Main.storyboard                |  95 -----
 .../Data+TensorFlowLite.swift                 |  13 -
 .../TensorFlowLiteApp/Info.plist              |  46 ---
 .../TensorFlowLiteApp/ViewController.swift    | 299 ---------------
 .../swift/Tests/InterpreterOptionsTests.swift |  54 ---
 .../swift/Tests/InterpreterTests.swift        | 315 ----------------
 .../experimental/swift/Tests/ModelTests.swift |  59 ---
 .../Tests/QuantizationParametersTests.swift   |  43 ---
 .../swift/Tests/TensorTests.swift             |  83 -----
 .../tools/pip_package/pip_smoke_test.py       |  36 +-
 27 files changed, 15 insertions(+), 2603 deletions(-)
 delete mode 100644 tensorflow/lite/experimental/swift/BUILD
 delete mode 100644 tensorflow/lite/experimental/swift/LICENSE
 delete mode 100644 tensorflow/lite/experimental/swift/README.md
 delete mode 100644 tensorflow/lite/experimental/swift/Sources/Interpreter.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Sources/InterpreterError.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Sources/Model.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Sources/Tensor.swift
 delete mode 100644 tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
 delete mode 100644 tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Tests/ModelTests.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Tests/TensorTests.swift

diff --git a/tensorflow/lite/experimental/swift/BUILD b/tensorflow/lite/experimental/swift/BUILD
deleted file mode 100644
index 3bd288a1e1..0000000000
--- a/tensorflow/lite/experimental/swift/BUILD
+++ /dev/null
@@ -1,100 +0,0 @@
-# TensorFlow Lite for Swift.
-
-package(default_visibility = ["//visibility:private"])
-
-licenses(["notice"])  # Apache 2.0
-
-exports_files(["LICENSE"])
-
-load("@build_bazel_rules_apple//apple:ios.bzl", "ios_application", "ios_unit_test")
-load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library")
-
-MINIMUM_OS_VERSION = "9.0"
-
-SWIFT_COPTS = [
-    "-wmo",
-]
-
-swift_library(
-    name = "TensorFlowLite",
-    srcs = glob(["Sources/*.swift"]),
-    copts = SWIFT_COPTS,
-    module_name = "TensorFlowLite",
-    tags = ["manual"],
-    deps = [
-        "//tensorflow/lite/experimental/c:c_api",
-    ],
-)
-
-ios_unit_test(
-    name = "TensorFlowLiteTests",
-    size = "small",
-    minimum_os_version = MINIMUM_OS_VERSION,
-    tags = [
-        "manual",
-        # DISABLED: Following sanitizer tests are not supported by iOS test targets.
-        "noasan",
-        "nomsan",
-        "notsan",
-    ],
-    deps = [":TensorFlowLiteTestsLib"],
-)
-
-swift_library(
-    name = "TensorFlowLiteTestsLib",
-    testonly = 1,
-    srcs = glob(["Tests/*.swift"]),
-    copts = SWIFT_COPTS,
-    tags = ["manual"],
-    deps = [
-        ":TensorFlowLite",
-        ":TestResources",
-    ],
-)
-
-objc_library(
-    name = "TestResources",
-    resources = [
-        "//tensorflow/lite:testdata/add.bin",
-        "//tensorflow/lite:testdata/add_quantized.bin",
-        "//tensorflow/lite:testdata/multi_add.bin",
-    ],
-    tags = ["manual"],
-)
-
-ios_application(
-    name = "TensorFlowLiteApp",
-    app_icons = glob(["TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/**"]),
-    bundle_id = "com.tensorflow.lite.swift.TensorFlowLite",
-    families = [
-        "ipad",
-        "iphone",
-    ],
-    infoplists = ["TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist"],
-    launch_storyboard = "TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard",
-    minimum_os_version = MINIMUM_OS_VERSION,
-    sdk_frameworks = [
-        "CoreGraphics",
-    ],
-    tags = ["manual"],
-    deps = [":TensorFlowLiteAppLib"],
-)
-
-swift_library(
-    name = "TensorFlowLiteAppLib",
-    srcs = glob(["TestApps/TensorFlowLiteApp/TensorFlowLiteApp/*.swift"]),
-    tags = ["manual"],
-    deps = [
-        ":TensorFlowLite",
-        ":TensorFlowLiteAppResources",
-    ],
-)
-
-objc_library(
-    name = "TensorFlowLiteAppResources",
-    storyboards = glob([
-        "TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/*.storyboard",
-    ]),
-    tags = ["manual"],
-    deps = [":TestResources"],
-)
diff --git a/tensorflow/lite/experimental/swift/LICENSE b/tensorflow/lite/experimental/swift/LICENSE
deleted file mode 100644
index d645695673..0000000000
--- a/tensorflow/lite/experimental/swift/LICENSE
+++ /dev/null
@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
diff --git a/tensorflow/lite/experimental/swift/README.md b/tensorflow/lite/experimental/swift/README.md
deleted file mode 100644
index deb16c1e91..0000000000
--- a/tensorflow/lite/experimental/swift/README.md
+++ /dev/null
@@ -1,54 +0,0 @@
-# TensorFlow Lite for Swift
-
-[TensorFlow Lite](https://www.tensorflow.org/lite/) is TensorFlow's lightweight
-solution for Swift developers. It enables low-latency inference of on-device
-machine learning models with a small binary size and fast performance supporting
-hardware acceleration.
-
-## Getting Started
-
-### Bazel
-
-In your `BUILD` file, add the `TensorFlowLite` dependency:
-
-```python
-swift_library(
-  # ...
-  deps = [
-      "//tensorflow/lite/swift:TensorFlowLite",
-  ],
-)
-```
-
-In your Swift files, import the module:
-
-```swift
-import TensorFlowLite
-```
-
-### Tulsi
-
-Open the `TensorFlowLite.tulsiproj` using the [TulsiApp](https://github.com/bazelbuild/tulsi) or by
-running the [`generate_xcodeproj.sh`](https://github.com/bazelbuild/tulsi/blob/master/src/tools/generate_xcodeproj.sh)
-script:
-
-```shell
-generate_xcodeproj.sh --genconfig tensorflow/lite/swift/TensorFlowLite.tulsiproj:TensorFlowLite --outputfolder ~/path/to/generated/TensorFlowLite.xcodeproj
-```
-
-### CocoaPods
-
-Add the following to your `Podfile`:
-
-```ruby
-use_frameworks!
-pod 'TensorFlowLiteSwift'
-```
-
-Then, run `pod install`.
-
-In your Swift files, import the module:
-
-```swift
-import TensorFlowLite
-```
diff --git a/tensorflow/lite/experimental/swift/Sources/Interpreter.swift b/tensorflow/lite/experimental/swift/Sources/Interpreter.swift
deleted file mode 100644
index a14b5966b1..0000000000
--- a/tensorflow/lite/experimental/swift/Sources/Interpreter.swift
+++ /dev/null
@@ -1,265 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import Foundation
-import TensorFlowLiteCAPI
-
-/// A TensorFlow Lite interpreter that performs inference from a given model.
-public final class Interpreter {
-
-  /// The `TFL_Interpreter` C pointer type represented as an `UnsafePointer<TFL_Interpreter>`.
-  private typealias CInterpreter = OpaquePointer
-
-  /// Total number of input tensors associated with the model.
-  public var inputTensorCount: Int {
-    return Int(TFL_InterpreterGetInputTensorCount(cInterpreter))
-  }
-
-  /// Total number of output tensors associated with the model.
-  public var outputTensorCount: Int {
-    return Int(TFL_InterpreterGetOutputTensorCount(cInterpreter))
-  }
-
-  /// The underlying `TFL_Interpreter` C pointer.
-  private var cInterpreter: CInterpreter?
-
-  /// Creates a new model interpreter instance.
-  ///
-  /// - Parameters:
-  ///   - modelPath: Local file path to a TensorFlow Lite model.
-  ///   - options: Custom configurations for the interpreter. The default is `nil` indicating that
-  ///       interpreter will determine the configuration options.
-  /// - Throws: An error if the model could not be loaded or the interpreter could not be created.
-  public init(modelPath: String, options: InterpreterOptions? = nil) throws {
-    guard let model = Model(filePath: modelPath) else { throw InterpreterError.failedToLoadModel }
-
-    let cInterpreterOptions: OpaquePointer? = try options.map { options in
-      guard let cOptions = TFL_NewInterpreterOptions() else {
-        throw InterpreterError.failedToCreateInterpreter
-      }
-      if let threadCount = options.threadCount, threadCount > 0 {
-        TFL_InterpreterOptionsSetNumThreads(cOptions, Int32(threadCount))
-      }
-      if options.isErrorLoggingEnabled {
-        TFL_InterpreterOptionsSetErrorReporter(
-          cOptions,
-          { (_, format, arguments) in
-            guard let cFormat = format,
-                  let message = String(cFormat: cFormat, arguments: arguments)
-            else {
-              return
-            }
-            print(String(describing: InterpreterError.tensorFlowLiteError(message)))
-          },
-          nil
-        )
-      }
-      return cOptions
-    }
-    defer { TFL_DeleteInterpreterOptions(cInterpreterOptions) }
-
-    guard let cInterpreter = TFL_NewInterpreter(model.cModel, cInterpreterOptions) else {
-      throw InterpreterError.failedToCreateInterpreter
-    }
-    self.cInterpreter = cInterpreter
-  }
-
-  deinit {
-    TFL_DeleteInterpreter(cInterpreter)
-  }
-
-  /// Invokes the interpreter to perform inference from the loaded graph.
-  ///
-  /// - Throws: An error if the model was not ready because tensors were not allocated.
-  public func invoke() throws {
-    guard TFL_InterpreterInvoke(cInterpreter) == kTfLiteOk else {
-      // TODO(b/117510052): Determine which error to throw.
-      throw InterpreterError.allocateTensorsRequired
-    }
-  }
-
-  /// Returns the input tensor at the given index.
-  ///
-  /// - Parameters:
-  ///   - index: The index for the input tensor.
-  /// - Throws: An error if the index is invalid or the tensors have not been allocated.
-  /// - Returns: The input tensor at the given index.
-  public func input(at index: Int) throws -> Tensor {
-    let maxIndex = inputTensorCount - 1
-    guard case 0...maxIndex = index else {
-      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
-    }
-    guard let cTensor = TFL_InterpreterGetInputTensor(cInterpreter, Int32(index)),
-          let bytes = TFL_TensorData(cTensor),
-          let nameCString = TFL_TensorName(cTensor)
-    else {
-      throw InterpreterError.allocateTensorsRequired
-    }
-    guard let dataType = TensorDataType(type: TFL_TensorType(cTensor)) else {
-      throw InterpreterError.invalidTensorDataType
-    }
-
-    let name = String(cString: nameCString)
-    let rank = TFL_TensorNumDims(cTensor)
-    let dimensions = (0..<rank).map { Int(TFL_TensorDim(cTensor, $0)) }
-    let shape = TensorShape(dimensions)
-    let byteCount = TFL_TensorByteSize(cTensor)
-    let data = Data(bytes: bytes, count: byteCount)
-    let cQuantizationParams = TFL_TensorQuantizationParams(cTensor)
-    let scale = cQuantizationParams.scale
-    let zeroPoint = Int(cQuantizationParams.zero_point)
-    var quantizationParameters: QuantizationParameters? = nil
-    if scale != 0.0 {
-      // TODO(b/117510052): Update this check once the TfLiteQuantizationParams struct has a mode.
-      quantizationParameters = QuantizationParameters(scale: scale, zeroPoint: zeroPoint)
-    }
-    let tensor = Tensor(
-      name: name,
-      dataType: dataType,
-      shape: shape,
-      data: data,
-      quantizationParameters: quantizationParameters
-    )
-    return tensor
-  }
-
-  /// Returns the output tensor at the given index.
-  ///
-  /// - Parameters:
-  ///   - index: The index for the output tensor.
-  /// - Throws: An error if the index is invalid, tensors haven't been allocated, or interpreter
-  ///     hasn't been invoked for models that dynamically compute output tensors based on the values
-  ///     of its input tensors.
-  /// - Returns: The output tensor at the given index.
-  public func output(at index: Int) throws -> Tensor {
-    let maxIndex = outputTensorCount - 1
-    guard case 0...maxIndex = index else {
-      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
-    }
-    guard let cTensor = TFL_InterpreterGetOutputTensor(cInterpreter, Int32(index)),
-          let bytes = TFL_TensorData(cTensor),
-          let nameCString = TFL_TensorName(cTensor)
-    else {
-      // TODO(b/117510052): Determine which error to throw.
-      throw InterpreterError.invokeInterpreterRequired
-    }
-    guard let dataType = TensorDataType(type: TFL_TensorType(cTensor)) else {
-      throw InterpreterError.invalidTensorDataType
-    }
-
-    let name = String(cString: nameCString)
-    let rank = TFL_TensorNumDims(cTensor)
-    let dimensions = (0..<rank).map { Int(TFL_TensorDim(cTensor, $0)) }
-    let shape = TensorShape(dimensions)
-    let byteCount = TFL_TensorByteSize(cTensor)
-    let data = Data(bytes: bytes, count: byteCount)
-    let cQuantizationParams = TFL_TensorQuantizationParams(cTensor)
-    let scale = cQuantizationParams.scale
-    let zeroPoint = Int(cQuantizationParams.zero_point)
-    var quantizationParameters: QuantizationParameters? = nil
-    if scale != 0.0 {
-      // TODO(b/117510052): Update this check once the TfLiteQuantizationParams struct has a mode.
-      quantizationParameters = QuantizationParameters(scale: scale, zeroPoint: zeroPoint)
-    }
-    let tensor = Tensor(
-      name: name,
-      dataType: dataType,
-      shape: shape,
-      data: data,
-      quantizationParameters: quantizationParameters
-    )
-    return tensor
-  }
-
-  /// Resizes the input tensor at the given index to the specified tensor shape.
-  ///
-  /// - Note: After resizing an input tensor, the client **must** explicitly call
-  ///     `allocateTensors()` before attempting to access the resized tensor data or invoking the
-  ///     interpreter to perform inference.
-  /// - Parameters:
-  ///   - index: The index for the input tensor.
-  ///   - shape: The shape that the input tensor should be resized to.
-  /// - Throws: An error if the input tensor at the given index could not be resized.
-  public func resizeInput(at index: Int, to shape: TensorShape) throws {
-    let maxIndex = inputTensorCount - 1
-    guard case 0...maxIndex = index else {
-      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
-    }
-    guard TFL_InterpreterResizeInputTensor(
-            cInterpreter,
-            Int32(index),
-            shape.int32Dimensions,
-            Int32(shape.rank)
-          ) == kTfLiteOk
-    else {
-      throw InterpreterError.failedToResizeInputTensor(index: index)
-    }
-  }
-
-  /// Copies the given data to the input tensor at the given index.
-  ///
-  /// - Parameters:
-  ///   - data: The data to be copied to the input tensor's data buffer.
-  ///   - index: The index for the input tensor.
-  /// - Throws: An error if the `data.count` does not match the input tensor's `data.count` or if
-  ///     the given index is invalid.
-  /// - Returns: The input tensor with the copied data.
-  @discardableResult
-  public func copy(_ data: Data, toInputAt index: Int) throws -> Tensor {
-    let maxIndex = inputTensorCount - 1
-    guard case 0...maxIndex = index else {
-      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
-    }
-    guard let cTensor = TFL_InterpreterGetInputTensor(cInterpreter, Int32(index)) else {
-      throw InterpreterError.allocateTensorsRequired
-    }
-
-    let byteCount = TFL_TensorByteSize(cTensor)
-    guard data.count == byteCount else {
-      throw InterpreterError.invalidTensorDataCount(provided: data.count, required: byteCount)
-    }
-
-    let status = data.withUnsafeBytes { TFL_TensorCopyFromBuffer(cTensor, $0, data.count) }
-    guard status == kTfLiteOk else { throw InterpreterError.failedToCopyDataToInputTensor }
-    return try input(at: index)
-  }
-
-  /// Allocates memory for all input tensors based on their `TensorShape`s.
-  ///
-  /// - Note: This is a relatively expensive operation and should only be called after creating the
-  ///     interpreter and/or resizing any input tensors.
-  /// - Throws: An error if memory could not be allocated for the input tensors.
-  public func allocateTensors() throws {
-    guard TFL_InterpreterAllocateTensors(cInterpreter) == kTfLiteOk else {
-      throw InterpreterError.failedToAllocateTensors
-    }
-  }
-}
-
-// MARK: - Extensions
-
-extension String {
-  /// Returns a new `String` initialized by using the given format C array as a template into which
-  /// the remaining argument values are substituted according to the user’s default locale.
-  ///
-  /// - Note: Returns `nil` if a new `String` could not be constructed from the given values.
-  /// - Parameters:
-  ///   - cFormat: The format C array as a template for substituting values.
-  ///   - arguments: A C pointer to a `va_list` of arguments to substitute into `cFormat`.
-  init?(cFormat: UnsafePointer<CChar>, arguments: CVaListPointer) {
-    var buffer: UnsafeMutablePointer<CChar>?
-    guard vasprintf(&buffer, cFormat, arguments) != 0, let cString = buffer else { return nil }
-    self.init(validatingUTF8: cString)
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/Sources/InterpreterError.swift b/tensorflow/lite/experimental/swift/Sources/InterpreterError.swift
deleted file mode 100644
index 5de58b997a..0000000000
--- a/tensorflow/lite/experimental/swift/Sources/InterpreterError.swift
+++ /dev/null
@@ -1,99 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import Foundation
-
-/// TensorFlow Lite interpreter errors.
-public enum InterpreterError: Error {
-  case invalidTensorIndex(index: Int, maxIndex: Int)
-  case invalidTensorDataCount(provided: Int, required: Int)
-  case invalidTensorDataType
-  case failedToLoadModel
-  case failedToCreateInterpreter
-  case failedToResizeInputTensor(index: Int)
-  case failedToCopyDataToInputTensor
-  case failedToAllocateTensors
-  case allocateTensorsRequired
-  case invokeInterpreterRequired
-  case tensorFlowLiteError(String)
-}
-
-// MARK: - Extensions
-
-extension InterpreterError: LocalizedError {
-  /// Localized description of the interpreter error.
-  public var errorDescription: String? {
-    switch self {
-    case .invalidTensorIndex(let index, let maxIndex):
-      return "Invalid tensor index \(index), max index is \(maxIndex)."
-    case .invalidTensorDataCount(let providedCount, let requiredCount):
-      return "Provided data count \(providedCount) must match the required count \(requiredCount)."
-    case .invalidTensorDataType:
-      return "Tensor data type is unsupported or could not be determined because of a model error."
-    case .failedToLoadModel:
-      return "Failed to load the given model."
-    case .failedToCreateInterpreter:
-      return "Failed to create the interpreter."
-    case .failedToResizeInputTensor(let index):
-      return "Failed to resize input tesnor at index \(index)."
-    case .failedToCopyDataToInputTensor:
-      return "Failed to copy data to input tensor."
-    case .failedToAllocateTensors:
-      return "Failed to allocate memory for input tensors."
-    case .allocateTensorsRequired:
-      return "Must call allocateTensors()."
-    case .invokeInterpreterRequired:
-      return "Must call invoke()."
-    case .tensorFlowLiteError(let message):
-      return "TensorFlow Lite Error: \(message)"
-    }
-  }
-}
-
-extension InterpreterError: CustomStringConvertible {
-  /// Textual representation of the TensorFlow Lite interpreter error.
-  public var description: String {
-    return errorDescription ?? "Unknown error."
-  }
-}
-
-#if swift(>=4.2)
-extension InterpreterError: Equatable {}
-#else
-extension InterpreterError: Equatable {
-  public static func == (lhs: InterpreterError, rhs: InterpreterError) -> Bool {
-    switch (lhs, rhs) {
-    case (.invalidTensorDataType, .invalidTensorDataType),
-         (.failedToLoadModel, .failedToLoadModel),
-         (.failedToCreateInterpreter, .failedToCreateInterpreter),
-         (.failedToAllocateTensors, .failedToAllocateTensors),
-         (.allocateTensorsRequired, .allocateTensorsRequired),
-         (.invokeInterpreterRequired, .invokeInterpreterRequired):
-      return true
-    case (.invalidTensorIndex(let lhsIndex, let lhsMaxIndex),
-          .invalidTensorIndex(let rhsIndex, let rhsMaxIndex)):
-      return lhsIndex == rhsIndex && lhsMaxIndex == rhsMaxIndex
-    case (.invalidTensorDataCount(let lhsProvidedCount, let lhsRequiredCount),
-          .invalidTensorDataCount(let rhsProvidedCount, let rhsRequiredCount)):
-      return lhsProvidedCount == rhsProvidedCount && lhsRequiredCount == rhsRequiredCount
-    case (.failedToResizeInputTensor(let lhsIndex), .failedToResizeInputTensor(let rhsIndex)):
-      return lhsIndex == rhsIndex
-    case (.tensorFlowLiteError(let lhsMessage), .tensorFlowLiteError(let rhsMessage)):
-      return lhsMessage == rhsMessage
-    default:
-      return false
-    }
-  }
-}
-#endif  // swift(>=4.2)
diff --git a/tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift b/tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift
deleted file mode 100644
index 2365fd7ade..0000000000
--- a/tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift
+++ /dev/null
@@ -1,29 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import Foundation
-
-/// Custom configuration options for a TensorFlow Lite interpreter.
-public struct InterpreterOptions: Equatable {
-
-  /// Maximum number of CPU threads that the interpreter should run on. Default is `nil` which
-  /// indicates that the `Interpreter` will decide the number of threads to use.
-  public var threadCount: Int? = nil
-
-  /// Whether error logging to the console is enabled. The default is `false`.
-  public var isErrorLoggingEnabled = false
-
-  /// Creates a new instance of interpreter options.
-  public init() {}
-}
diff --git a/tensorflow/lite/experimental/swift/Sources/Model.swift b/tensorflow/lite/experimental/swift/Sources/Model.swift
deleted file mode 100644
index e8c49ff1ae..0000000000
--- a/tensorflow/lite/experimental/swift/Sources/Model.swift
+++ /dev/null
@@ -1,40 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import Foundation
-import TensorFlowLiteCAPI
-
-/// A TensorFlow Lite model used by the 'Interpreter` to perform inference.
-final class Model {
-
-  /// The `TFL_Model` C pointer type represented as an `UnsafePointer<TFL_Model>`.
-  typealias CModel = OpaquePointer
-
-  /// The underlying `TFL_Model` C pointer.
-  let cModel: CModel?
-
-  /// Creates a new model instance.
-  ///
-  /// - Precondition: Initialization can fail if the given `filePath` is invalid.
-  /// - Parameters:
-  ///   - filePath: Local file path to a TensorFlow Lite model.
-  init?(filePath: String) {
-    guard !filePath.isEmpty, let cModel = TFL_NewModelFromFile(filePath) else { return nil }
-    self.cModel = cModel
-  }
-
-  deinit {
-    TFL_DeleteModel(cModel)
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift b/tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift
deleted file mode 100644
index f367875644..0000000000
--- a/tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift
+++ /dev/null
@@ -1,38 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import Foundation
-
-/// Parameters that determine the mapping of quantized values to real values. Quantized values can
-/// be mapped to float values using the following conversion:
-/// `realValue = scale * (quantizedValue - zeroPoint)`.
-public struct QuantizationParameters {
-
-  /// Difference between real values corresponding to consecutive quantized values differing by 1.
-  /// For example, the range of quantized values for `UInt8` data type is [0, 255].
-  public let scale: Float
-
-  /// Quantized value that corresponds to the real 0 value.
-  public let zeroPoint: Int
-
-  /// Creates a new quantization parameters instance.
-  ///
-  /// - Parameters:
-  ///   - scale: Scale value for asymmetric quantization.
-  ///   - zeroPoint: Zero point for asymmetric quantization.
-  init(scale: Float, zeroPoint: Int) {
-    self.scale = scale
-    self.zeroPoint = zeroPoint
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/Sources/Tensor.swift b/tensorflow/lite/experimental/swift/Sources/Tensor.swift
deleted file mode 100644
index b738d87549..0000000000
--- a/tensorflow/lite/experimental/swift/Sources/Tensor.swift
+++ /dev/null
@@ -1,138 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import Foundation
-import TensorFlowLiteCAPI
-
-/// An input or output tensor in a TensorFlow Lite graph.
-public struct Tensor {
-
-  /// Name of the tensor.
-  public let name: String
-
-  /// Data type of the tensor.
-  public let dataType: TensorDataType
-
-  /// Shape of the tensor.
-  public let shape: TensorShape
-
-  /// Data in the input or output tensor.
-  public let data: Data
-
-  /// Quantization parameters for the tensor if using a quantized model.
-  public let quantizationParameters: QuantizationParameters?
-
-  /// Creates a new input or output tensor instance.
-  ///
-  /// - Parameters:
-  ///   - name: Name of the tensor.
-  ///   - dataType: Data type of the tensor.
-  ///   - data: Data in the input tensor.
-  ///   - quantizationParameters Quantization parameters for the tensor if using a quantized model.
-  ///       The default is `nil`.
-  init(
-    name: String,
-    dataType: TensorDataType,
-    shape: TensorShape,
-    data: Data,
-    quantizationParameters: QuantizationParameters? = nil
-  ) {
-    self.name = name
-    self.dataType = dataType
-    self.shape = shape
-    self.data = data
-    self.quantizationParameters = quantizationParameters
-  }
-}
-
-/// Supported TensorFlow Lite tensor data types.
-public enum TensorDataType: Equatable {
-  /// 32-bit single precision floating point tensor data type.
-  case float32
-  /// 8-bit unsigned integer tensor data type.
-  case uInt8
-  /// 16-bit signed integer tensor data type.
-  case int16
-  /// 32-bit signed integer tensor data type.
-  case int32
-  /// 64-bit signed integer tensor data type.
-  case int64
-  /// Boolean tensor data type.
-  case bool
-
-  /// Creates a new tensor data type from the given `TFL_Type` or `nil` if the data type is
-  /// unsupported or could not be determined because there was an error.
-  ///
-  /// - Parameter type: A data type supported by a tensor.
-  init?(type: TFL_Type) {
-    switch type {
-    case kTfLiteFloat32:
-      self = .float32
-    case kTfLiteUInt8:
-      self = .uInt8
-    case kTfLiteInt16:
-      self = .int16
-    case kTfLiteInt32:
-      self = .int32
-    case kTfLiteInt64:
-      self = .int64
-    case kTfLiteBool:
-      self = .bool
-    case kTfLiteNoType:
-      fallthrough
-    default:
-      return nil
-    }
-  }
-}
-
-/// The shape of a TensorFlow Lite tensor.
-public struct TensorShape {
-
-  /// The number of dimensions of the tensor.
-  public let rank: Int
-
-  /// Array of dimensions for the tensor.
-  public let dimensions: [Int]
-
-  /// Array of `Int32` dimensions for the tensor.
-  var int32Dimensions: [Int32] { return dimensions.map(Int32.init) }
-
-  /// Creates a new tensor shape instance with the given array of dimensions.
-  ///
-  /// - Parameters:
-  ///   - dimensions: Dimensions for the tensor.
-  public init(_ dimensions: [Int]) {
-    self.rank = dimensions.count
-    self.dimensions = dimensions
-  }
-
-  /// Creates a new tensor shape instance with the given elements representing the dimensions.
-  ///
-  /// - Parameters:
-  ///   - elements: Dimensions for the tensor.
-  public init(_ elements: Int...) {
-    self.init(elements)
-  }
-}
-
-extension TensorShape: ExpressibleByArrayLiteral {
-  /// Creates a new tensor shape instance with the given array literal representing the dimensions.
-  ///
-  /// - Parameters:
-  ///   - arrayLiteral: Dimensions for the tensor.
-  public init(arrayLiteral: Int...) {
-    self.init(arrayLiteral)
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
deleted file mode 100644
index 4010fab49e..0000000000
--- a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
+++ /dev/null
@@ -1,57 +0,0 @@
-{
-  "sourceFilters" : [
-    "third_party/tensorflow/lite/experimental/c",
-    "third_party/tensorflow/lite/experimental/swift",
-    "third_party/tensorflow/lite/experimental/swift/Sources",
-    "third_party/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp",
-    "third_party/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj",
-    "third_party/tensorflow/lite/experimental/swift/Tests",
-  ],
-  "buildTargets" : [
-    "//third_party/tensorflow/lite/experimental/swift:TensorFlowLite",
-    "//third_party/tensorflow/lite/experimental/swift:TensorFlowLiteApp",
-    "//third_party/tensorflow/lite/experimental/swift:TensorFlowLiteTests",
-  ],
-  "projectName" : "TensorFlowLite",
-  "optionSet" : {
-    "LaunchActionPreActionScript" : {
-      "p" : "$(inherited)"
-    },
-    "BazelBuildStartupOptionsRelease" : {
-      "p" : "$(inherited)"
-    },
-    "BazelBuildOptionsRelease" : {
-      "p" : "$(inherited)"
-    },
-    "BazelBuildOptionsDebug" : {
-      "p" : "$(inherited)"
-    },
-    "EnvironmentVariables" : {
-      "p" : "$(inherited)"
-    },
-    "BuildActionPreActionScript" : {
-      "p" : "$(inherited)"
-    },
-    "CommandlineArguments" : {
-      "p" : "$(inherited)"
-    },
-    "TestActionPreActionScript" : {
-      "p" : "$(inherited)"
-    },
-    "BazelBuildStartupOptionsDebug" : {
-      "p" : "$(inherited)"
-    },
-    "BuildActionPostActionScript" : {
-      "p" : "$(inherited)"
-    },
-    "TestActionPostActionScript" : {
-      "p" : "$(inherited)"
-    },
-    "LaunchActionPostActionScript" : {
-      "p" : "$(inherited)"
-    }
-  },
-  "additionalFilePaths" : [
-    "third_party/tensorflow/lite/experimental/swift/BUILD"
-  ]
-}
diff --git a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
deleted file mode 100644
index 14cff94453..0000000000
--- a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
+++ /dev/null
@@ -1,14 +0,0 @@
-{
-  "configDefaults" : {
-    "optionSet" : {
-      "ProjectPrioritizesSwift" : {
-        "p" : "YES"
-      }
-    }
-  },
-  "projectName" : "TensorFlowLite",
-  "packages" : [
-    "third_party/tensorflow/lite/experimental/swift"
-  ],
-  "workspaceRoot" : "../../../../../.."
-}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj
deleted file mode 100644
index fbbf9a1de2..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj
+++ /dev/null
@@ -1,345 +0,0 @@
-// !$*UTF8*$!
-{
-	archiveVersion = 1;
-	classes = {
-	};
-	objectVersion = 50;
-	objects = {
-
-/* Begin PBXBuildFile section */
-		4A7304B421500B8400C90B21 /* Data+TensorFlowLite.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4A7304B321500B8300C90B21 /* Data+TensorFlowLite.swift */; };
-		4AA72B732146ED64006C3AEF /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4AA72B722146ED64006C3AEF /* AppDelegate.swift */; };
-		4AA72B752146ED64006C3AEF /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4AA72B742146ED64006C3AEF /* ViewController.swift */; };
-		4AA72B782146ED64006C3AEF /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 4AA72B762146ED64006C3AEF /* Main.storyboard */; };
-		4AA72B7A2146ED66006C3AEF /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 4AA72B792146ED66006C3AEF /* Assets.xcassets */; };
-		4AA72B7D2146ED66006C3AEF /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 4AA72B7B2146ED66006C3AEF /* LaunchScreen.storyboard */; };
-		4ADDE0CE2176600E00FF07A2 /* Array+TensorFlowLite.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4ADDE0CD2176600900FF07A2 /* Array+TensorFlowLite.swift */; };
-/* End PBXBuildFile section */
-
-/* Begin PBXFileReference section */
-		4A7304B321500B8300C90B21 /* Data+TensorFlowLite.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "Data+TensorFlowLite.swift"; sourceTree = "<group>"; };
-		4AA72B6F2146ED64006C3AEF /* TensorFlowLiteApp.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = TensorFlowLiteApp.app; sourceTree = BUILT_PRODUCTS_DIR; };
-		4AA72B722146ED64006C3AEF /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = "<group>"; };
-		4AA72B742146ED64006C3AEF /* ViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewController.swift; sourceTree = "<group>"; };
-		4AA72B772146ED64006C3AEF /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = "<group>"; };
-		4AA72B792146ED66006C3AEF /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = "<group>"; };
-		4AA72B7C2146ED66006C3AEF /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = "<group>"; };
-		4AA72B7E2146ED66006C3AEF /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
-		4ADDE0CD2176600900FF07A2 /* Array+TensorFlowLite.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "Array+TensorFlowLite.swift"; sourceTree = "<group>"; };
-/* End PBXFileReference section */
-
-/* Begin PBXFrameworksBuildPhase section */
-		4AA72B6C2146ED64006C3AEF /* Frameworks */ = {
-			isa = PBXFrameworksBuildPhase;
-			buildActionMask = 2147483647;
-			files = (
-			);
-			runOnlyForDeploymentPostprocessing = 0;
-		};
-/* End PBXFrameworksBuildPhase section */
-
-/* Begin PBXGroup section */
-		4AA72B662146ED64006C3AEF = {
-			isa = PBXGroup;
-			children = (
-				4AA72B712146ED64006C3AEF /* TensorFlowLiteApp */,
-				4AA72B702146ED64006C3AEF /* Products */,
-			);
-			sourceTree = "<group>";
-		};
-		4AA72B702146ED64006C3AEF /* Products */ = {
-			isa = PBXGroup;
-			children = (
-				4AA72B6F2146ED64006C3AEF /* TensorFlowLiteApp.app */,
-			);
-			name = Products;
-			sourceTree = "<group>";
-		};
-		4AA72B712146ED64006C3AEF /* TensorFlowLiteApp */ = {
-			isa = PBXGroup;
-			children = (
-				4AA72B722146ED64006C3AEF /* AppDelegate.swift */,
-				4ADDE0CD2176600900FF07A2 /* Array+TensorFlowLite.swift */,
-				4A7304B321500B8300C90B21 /* Data+TensorFlowLite.swift */,
-				4AA72B742146ED64006C3AEF /* ViewController.swift */,
-				4AA72B762146ED64006C3AEF /* Main.storyboard */,
-				4AA72B792146ED66006C3AEF /* Assets.xcassets */,
-				4AA72B7B2146ED66006C3AEF /* LaunchScreen.storyboard */,
-				4AA72B7E2146ED66006C3AEF /* Info.plist */,
-			);
-			path = TensorFlowLiteApp;
-			sourceTree = "<group>";
-		};
-/* End PBXGroup section */
-
-/* Begin PBXNativeTarget section */
-		4AA72B6E2146ED64006C3AEF /* TensorFlowLiteApp */ = {
-			isa = PBXNativeTarget;
-			buildConfigurationList = 4AA72B812146ED66006C3AEF /* Build configuration list for PBXNativeTarget "TensorFlowLiteApp" */;
-			buildPhases = (
-				4AA72B6B2146ED64006C3AEF /* Sources */,
-				4AA72B6C2146ED64006C3AEF /* Frameworks */,
-				4AA72B6D2146ED64006C3AEF /* Resources */,
-			);
-			buildRules = (
-			);
-			dependencies = (
-			);
-			name = TensorFlowLiteApp;
-			productName = TensorFlowLiteApp;
-			productReference = 4AA72B6F2146ED64006C3AEF /* TensorFlowLiteApp.app */;
-			productType = "com.apple.product-type.application";
-		};
-/* End PBXNativeTarget section */
-
-/* Begin PBXProject section */
-		4AA72B672146ED64006C3AEF /* Project object */ = {
-			isa = PBXProject;
-			attributes = {
-				LastSwiftUpdateCheck = 0940;
-				LastUpgradeCheck = 0940;
-				ORGANIZATIONNAME = Google;
-				TargetAttributes = {
-					4AA72B6E2146ED64006C3AEF = {
-						CreatedOnToolsVersion = 9.4.1;
-					};
-				};
-			};
-			buildConfigurationList = 4AA72B6A2146ED64006C3AEF /* Build configuration list for PBXProject "TensorFlowLiteApp" */;
-			compatibilityVersion = "Xcode 9.3";
-			developmentRegion = en;
-			hasScannedForEncodings = 0;
-			knownRegions = (
-				en,
-				Base,
-			);
-			mainGroup = 4AA72B662146ED64006C3AEF;
-			productRefGroup = 4AA72B702146ED64006C3AEF /* Products */;
-			projectDirPath = "";
-			projectRoot = "";
-			targets = (
-				4AA72B6E2146ED64006C3AEF /* TensorFlowLiteApp */,
-			);
-		};
-/* End PBXProject section */
-
-/* Begin PBXResourcesBuildPhase section */
-		4AA72B6D2146ED64006C3AEF /* Resources */ = {
-			isa = PBXResourcesBuildPhase;
-			buildActionMask = 2147483647;
-			files = (
-				4AA72B7D2146ED66006C3AEF /* LaunchScreen.storyboard in Resources */,
-				4AA72B7A2146ED66006C3AEF /* Assets.xcassets in Resources */,
-				4AA72B782146ED64006C3AEF /* Main.storyboard in Resources */,
-			);
-			runOnlyForDeploymentPostprocessing = 0;
-		};
-/* End PBXResourcesBuildPhase section */
-
-/* Begin PBXSourcesBuildPhase section */
-		4AA72B6B2146ED64006C3AEF /* Sources */ = {
-			isa = PBXSourcesBuildPhase;
-			buildActionMask = 2147483647;
-			files = (
-				4AA72B732146ED64006C3AEF /* AppDelegate.swift in Sources */,
-				4ADDE0CE2176600E00FF07A2 /* Array+TensorFlowLite.swift in Sources */,
-				4A7304B421500B8400C90B21 /* Data+TensorFlowLite.swift in Sources */,
-				4AA72B752146ED64006C3AEF /* ViewController.swift in Sources */,
-			);
-			runOnlyForDeploymentPostprocessing = 0;
-		};
-/* End PBXSourcesBuildPhase section */
-
-/* Begin PBXVariantGroup section */
-		4AA72B762146ED64006C3AEF /* Main.storyboard */ = {
-			isa = PBXVariantGroup;
-			children = (
-				4AA72B772146ED64006C3AEF /* Base */,
-			);
-			name = Main.storyboard;
-			sourceTree = "<group>";
-		};
-		4AA72B7B2146ED66006C3AEF /* LaunchScreen.storyboard */ = {
-			isa = PBXVariantGroup;
-			children = (
-				4AA72B7C2146ED66006C3AEF /* Base */,
-			);
-			name = LaunchScreen.storyboard;
-			sourceTree = "<group>";
-		};
-/* End PBXVariantGroup section */
-
-/* Begin XCBuildConfiguration section */
-		4AA72B7F2146ED66006C3AEF /* Debug */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				ALWAYS_SEARCH_USER_PATHS = NO;
-				CLANG_ANALYZER_NONNULL = YES;
-				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
-				CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
-				CLANG_CXX_LIBRARY = "libc++";
-				CLANG_ENABLE_MODULES = YES;
-				CLANG_ENABLE_OBJC_ARC = YES;
-				CLANG_ENABLE_OBJC_WEAK = YES;
-				CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
-				CLANG_WARN_BOOL_CONVERSION = YES;
-				CLANG_WARN_COMMA = YES;
-				CLANG_WARN_CONSTANT_CONVERSION = YES;
-				CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
-				CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
-				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
-				CLANG_WARN_EMPTY_BODY = YES;
-				CLANG_WARN_ENUM_CONVERSION = YES;
-				CLANG_WARN_INFINITE_RECURSION = YES;
-				CLANG_WARN_INT_CONVERSION = YES;
-				CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
-				CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
-				CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
-				CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
-				CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
-				CLANG_WARN_STRICT_PROTOTYPES = YES;
-				CLANG_WARN_SUSPICIOUS_MOVE = YES;
-				CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
-				CLANG_WARN_UNREACHABLE_CODE = YES;
-				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
-				CODE_SIGN_IDENTITY = "iPhone Developer";
-				COPY_PHASE_STRIP = NO;
-				DEBUG_INFORMATION_FORMAT = dwarf;
-				ENABLE_STRICT_OBJC_MSGSEND = YES;
-				ENABLE_TESTABILITY = YES;
-				GCC_C_LANGUAGE_STANDARD = gnu11;
-				GCC_DYNAMIC_NO_PIC = NO;
-				GCC_NO_COMMON_BLOCKS = YES;
-				GCC_OPTIMIZATION_LEVEL = 0;
-				GCC_PREPROCESSOR_DEFINITIONS = (
-					"DEBUG=1",
-					"$(inherited)",
-				);
-				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
-				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
-				GCC_WARN_UNDECLARED_SELECTOR = YES;
-				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
-				GCC_WARN_UNUSED_FUNCTION = YES;
-				GCC_WARN_UNUSED_VARIABLE = YES;
-				IPHONEOS_DEPLOYMENT_TARGET = 11.4;
-				MTL_ENABLE_DEBUG_INFO = YES;
-				ONLY_ACTIVE_ARCH = YES;
-				SDKROOT = iphoneos;
-				SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;
-				SWIFT_OPTIMIZATION_LEVEL = "-Onone";
-			};
-			name = Debug;
-		};
-		4AA72B802146ED66006C3AEF /* Release */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				ALWAYS_SEARCH_USER_PATHS = NO;
-				CLANG_ANALYZER_NONNULL = YES;
-				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
-				CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
-				CLANG_CXX_LIBRARY = "libc++";
-				CLANG_ENABLE_MODULES = YES;
-				CLANG_ENABLE_OBJC_ARC = YES;
-				CLANG_ENABLE_OBJC_WEAK = YES;
-				CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
-				CLANG_WARN_BOOL_CONVERSION = YES;
-				CLANG_WARN_COMMA = YES;
-				CLANG_WARN_CONSTANT_CONVERSION = YES;
-				CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
-				CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
-				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
-				CLANG_WARN_EMPTY_BODY = YES;
-				CLANG_WARN_ENUM_CONVERSION = YES;
-				CLANG_WARN_INFINITE_RECURSION = YES;
-				CLANG_WARN_INT_CONVERSION = YES;
-				CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
-				CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
-				CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
-				CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
-				CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
-				CLANG_WARN_STRICT_PROTOTYPES = YES;
-				CLANG_WARN_SUSPICIOUS_MOVE = YES;
-				CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
-				CLANG_WARN_UNREACHABLE_CODE = YES;
-				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
-				CODE_SIGN_IDENTITY = "iPhone Developer";
-				COPY_PHASE_STRIP = NO;
-				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
-				ENABLE_NS_ASSERTIONS = NO;
-				ENABLE_STRICT_OBJC_MSGSEND = YES;
-				GCC_C_LANGUAGE_STANDARD = gnu11;
-				GCC_NO_COMMON_BLOCKS = YES;
-				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
-				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
-				GCC_WARN_UNDECLARED_SELECTOR = YES;
-				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
-				GCC_WARN_UNUSED_FUNCTION = YES;
-				GCC_WARN_UNUSED_VARIABLE = YES;
-				IPHONEOS_DEPLOYMENT_TARGET = 11.4;
-				MTL_ENABLE_DEBUG_INFO = NO;
-				SDKROOT = iphoneos;
-				SWIFT_COMPILATION_MODE = wholemodule;
-				SWIFT_OPTIMIZATION_LEVEL = "-O";
-				VALIDATE_PRODUCT = YES;
-			};
-			name = Release;
-		};
-		4AA72B822146ED66006C3AEF /* Debug */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
-				CODE_SIGN_STYLE = Automatic;
-				INFOPLIST_FILE = TensorFlowLiteApp/Info.plist;
-				LD_RUNPATH_SEARCH_PATHS = (
-					"$(inherited)",
-					"@executable_path/Frameworks",
-				);
-				PRODUCT_BUNDLE_IDENTIFIER = com.tensorflow.lite.swift.TensorFlowLite;
-				PRODUCT_NAME = "$(TARGET_NAME)";
-				SWIFT_VERSION = 4.0;
-				TARGETED_DEVICE_FAMILY = "1,2";
-			};
-			name = Debug;
-		};
-		4AA72B832146ED66006C3AEF /* Release */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
-				CODE_SIGN_STYLE = Automatic;
-				INFOPLIST_FILE = TensorFlowLiteApp/Info.plist;
-				LD_RUNPATH_SEARCH_PATHS = (
-					"$(inherited)",
-					"@executable_path/Frameworks",
-				);
-				PRODUCT_BUNDLE_IDENTIFIER = com.tensorflow.lite.swift.TensorFlowLite;
-				PRODUCT_NAME = "$(TARGET_NAME)";
-				SWIFT_VERSION = 4.0;
-				TARGETED_DEVICE_FAMILY = "1,2";
-			};
-			name = Release;
-		};
-/* End XCBuildConfiguration section */
-
-/* Begin XCConfigurationList section */
-		4AA72B6A2146ED64006C3AEF /* Build configuration list for PBXProject "TensorFlowLiteApp" */ = {
-			isa = XCConfigurationList;
-			buildConfigurations = (
-				4AA72B7F2146ED66006C3AEF /* Debug */,
-				4AA72B802146ED66006C3AEF /* Release */,
-			);
-			defaultConfigurationIsVisible = 0;
-			defaultConfigurationName = Release;
-		};
-		4AA72B812146ED66006C3AEF /* Build configuration list for PBXNativeTarget "TensorFlowLiteApp" */ = {
-			isa = XCConfigurationList;
-			buildConfigurations = (
-				4AA72B822146ED66006C3AEF /* Debug */,
-				4AA72B832146ED66006C3AEF /* Release */,
-			);
-			defaultConfigurationIsVisible = 0;
-			defaultConfigurationName = Release;
-		};
-/* End XCConfigurationList section */
-	};
-	rootObject = 4AA72B672146ED64006C3AEF /* Project object */;
-}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift
deleted file mode 100644
index ffa90a06ad..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift
+++ /dev/null
@@ -1,24 +0,0 @@
-import UIKit
-
-@UIApplicationMain
-
-final class AppDelegate: UIResponder, UIApplicationDelegate {
-
-  /// The main window of the app.
-  var window: UIWindow?
-
-  func application(
-    _ application: UIApplication,
-    didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? = nil
-  ) -> Bool {
-    return true
-  }
-}
-
-// MARK: - Extensions
-
-#if !swift(>=4.2)
-extension UIApplication {
-  typealias LaunchOptionsKey = UIApplicationLaunchOptionsKey
-}
-#endif  // !swift(>=4.2)
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift
deleted file mode 100644
index 56df1ce659..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift
+++ /dev/null
@@ -1,22 +0,0 @@
-import Foundation
-
-extension Array {
-  /// Creates a new array from the bytes of the given unsafe data.
-  ///
-  /// - Warning: The array's `Element` type must be trivial in that it can be copied bit for bit
-  ///     with no indirection or reference-counting operations; otherwise, copying the raw bytes in
-  ///     the `unsafeData`'s buffer to a new array returns an unsafe copy.
-  /// - Note: Returns `nil` if `unsafeData.count` is not a multiple of
-  ///     `MemoryLayout<Element>.stride`.
-  /// - Parameter unsafeData: The data containing the bytes to turn into an array.
-  init?(unsafeData: Data) {
-    guard unsafeData.count % MemoryLayout<Element>.stride == 0 else { return nil }
-    let elements = unsafeData.withUnsafeBytes {
-      UnsafeBufferPointer<Element>(
-        start: $0,
-        count: unsafeData.count / MemoryLayout<Element>.stride
-      )
-    }
-    self.init(elements)
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json
deleted file mode 100644
index d8db8d65fd..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json
+++ /dev/null
@@ -1,98 +0,0 @@
-{
-  "images" : [
-    {
-      "idiom" : "iphone",
-      "size" : "20x20",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "iphone",
-      "size" : "20x20",
-      "scale" : "3x"
-    },
-    {
-      "idiom" : "iphone",
-      "size" : "29x29",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "iphone",
-      "size" : "29x29",
-      "scale" : "3x"
-    },
-    {
-      "idiom" : "iphone",
-      "size" : "40x40",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "iphone",
-      "size" : "40x40",
-      "scale" : "3x"
-    },
-    {
-      "idiom" : "iphone",
-      "size" : "60x60",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "iphone",
-      "size" : "60x60",
-      "scale" : "3x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "20x20",
-      "scale" : "1x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "20x20",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "29x29",
-      "scale" : "1x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "29x29",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "40x40",
-      "scale" : "1x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "40x40",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "76x76",
-      "scale" : "1x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "76x76",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "83.5x83.5",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "ios-marketing",
-      "size" : "1024x1024",
-      "scale" : "1x"
-    }
-  ],
-  "info" : {
-    "version" : 1,
-    "author" : "xcode"
-  }
-}
\ No newline at end of file
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json
deleted file mode 100644
index da4a164c91..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
-  "info" : {
-    "version" : 1,
-    "author" : "xcode"
-  }
-}
\ No newline at end of file
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard
deleted file mode 100644
index a07a1321be..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard
+++ /dev/null
@@ -1,44 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="14109" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" launchScreen="YES" useTraitCollections="YES" colorMatched="YES" initialViewController="01J-lp-oVM">
-    <device id="retina4_7" orientation="portrait">
-        <adaptation id="fullscreen"/>
-    </device>
-    <dependencies>
-        <deployment identifier="iOS"/>
-        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="14088"/>
-        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
-    </dependencies>
-    <scenes>
-        <!--View Controller-->
-        <scene sceneID="EHf-IW-A2E">
-            <objects>
-                <viewController id="01J-lp-oVM" sceneMemberID="viewController">
-                    <layoutGuides>
-                        <viewControllerLayoutGuide type="top" id="Llm-lL-Icb"/>
-                        <viewControllerLayoutGuide type="bottom" id="xb3-aO-Qok"/>
-                    </layoutGuides>
-                    <view key="view" contentMode="scaleToFill" id="Ze5-6b-2t3">
-                        <rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
-                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
-                        <subviews>
-                            <label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" text="TensorFlowLite" textAlignment="center" lineBreakMode="tailTruncation" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="3Gq-PV-hia">
-                                <rect key="frame" x="16" y="315" width="343" height="38.5"/>
-                                <fontDescription key="fontDescription" type="boldSystem" pointSize="32"/>
-                                <nil key="textColor"/>
-                                <nil key="highlightedColor"/>
-                            </label>
-                        </subviews>
-                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
-                        <constraints>
-                            <constraint firstItem="3Gq-PV-hia" firstAttribute="leading" secondItem="Ze5-6b-2t3" secondAttribute="leading" constant="16" id="aXL-9T-5Pf"/>
-                            <constraint firstItem="3Gq-PV-hia" firstAttribute="centerY" secondItem="Ze5-6b-2t3" secondAttribute="centerY" id="cDf-Go-1FR"/>
-                            <constraint firstAttribute="trailing" secondItem="3Gq-PV-hia" secondAttribute="trailing" constant="16" id="fB9-BX-A3B"/>
-                        </constraints>
-                    </view>
-                </viewController>
-                <placeholder placeholderIdentifier="IBFirstResponder" id="iYj-Kq-Ea1" userLabel="First Responder" sceneMemberID="firstResponder"/>
-            </objects>
-            <point key="canvasLocation" x="52" y="374.66266866566718"/>
-        </scene>
-    </scenes>
-</document>
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
deleted file mode 100644
index d0e91d63c4..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
+++ /dev/null
@@ -1,95 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="14313.18" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" colorMatched="YES" initialViewController="BYZ-38-t0r">
-    <device id="retina4_7" orientation="portrait">
-        <adaptation id="fullscreen"/>
-    </device>
-    <dependencies>
-        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="14283.14"/>
-        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
-    </dependencies>
-    <scenes>
-        <!--View Controller-->
-        <scene sceneID="tne-QT-ifu">
-            <objects>
-                <viewController storyboardIdentifier="viewController" useStoryboardIdentifierAsRestorationIdentifier="YES" id="BYZ-38-t0r" customClass="ViewController" customModule="third_party_tensorflow_lite_experimental_swift_TensorFlowLiteAppLib" sceneMemberID="viewController">
-                    <layoutGuides>
-                        <viewControllerLayoutGuide type="top" id="y3c-jy-aDJ"/>
-                        <viewControllerLayoutGuide type="bottom" id="wfy-db-euE"/>
-                    </layoutGuides>
-                    <view key="view" contentMode="scaleToFill" id="8bC-Xf-vdC">
-                        <rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
-                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
-                        <subviews>
-                            <textView clipsSubviews="YES" multipleTouchEnabled="YES" contentMode="scaleToFill" editable="NO" selectable="NO" translatesAutoresizingMaskIntoConstraints="NO" id="7Mj-sL-hrd">
-                                <rect key="frame" x="0.0" y="367" width="375" height="300"/>
-                                <color key="backgroundColor" red="0.0" green="0.47843137250000001" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
-                                <constraints>
-                                    <constraint firstAttribute="height" constant="300" id="YUb-MC-D5w"/>
-                                </constraints>
-                                <color key="textColor" cocoaTouchSystemColor="tableCellGroupedBackgroundColor"/>
-                                <fontDescription key="fontDescription" type="system" pointSize="14"/>
-                                <textInputTraits key="textInputTraits" autocapitalizationType="sentences"/>
-                            </textView>
-                            <toolbar opaque="NO" clearsContextBeforeDrawing="NO" contentMode="scaleToFill" translatesAutoresizingMaskIntoConstraints="NO" id="Qwg-EP-bd6" userLabel="Bottom Toolbar">
-                                <rect key="frame" x="0.0" y="323" width="375" height="44"/>
-                                <constraints>
-                                    <constraint firstAttribute="height" constant="44" id="jhT-Q0-E9N"/>
-                                </constraints>
-                                <items>
-                                    <barButtonItem style="plain" systemItem="flexibleSpace" id="P3q-uA-YUa"/>
-                                    <barButtonItem title="Invoke Interpreter" id="A4J-Mg-nmd" userLabel="Invoke Button">
-                                        <connections>
-                                            <action selector="invokeInterpreter:" destination="BYZ-38-t0r" id="lZU-x7-PsJ"/>
-                                        </connections>
-                                    </barButtonItem>
-                                    <barButtonItem style="plain" systemItem="flexibleSpace" id="Qad-Pa-ySg"/>
-                                </items>
-                            </toolbar>
-                            <toolbar opaque="NO" clearsContextBeforeDrawing="NO" contentMode="scaleToFill" translatesAutoresizingMaskIntoConstraints="NO" id="Gkb-TR-PCB" userLabel="Top Toolbar">
-                                <rect key="frame" x="0.0" y="28" width="375" height="44"/>
-                                <constraints>
-                                    <constraint firstAttribute="height" constant="44" id="hSD-2q-fUE"/>
-                                </constraints>
-                                <items>
-                                    <barButtonItem style="plain" id="LKw-TX-bbH">
-                                        <segmentedControl key="customView" opaque="NO" contentMode="scaleToFill" contentHorizontalAlignment="left" contentVerticalAlignment="top" segmentControlStyle="bar" selectedSegmentIndex="0" id="rhA-nW-xzT">
-                                            <rect key="frame" x="16" y="7" width="343" height="30"/>
-                                            <autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
-                                            <segments>
-                                                <segment title="Add"/>
-                                                <segment title="AddQuantized"/>
-                                                <segment title="MultiAdd"/>
-                                            </segments>
-                                            <connections>
-                                                <action selector="modelChanged:" destination="BYZ-38-t0r" eventType="valueChanged" id="YnG-Ov-B5D"/>
-                                            </connections>
-                                        </segmentedControl>
-                                    </barButtonItem>
-                                </items>
-                            </toolbar>
-                        </subviews>
-                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
-                        <constraints>
-                            <constraint firstAttribute="trailing" secondItem="Gkb-TR-PCB" secondAttribute="trailing" id="4Cr-Sf-I7n"/>
-                            <constraint firstItem="7Mj-sL-hrd" firstAttribute="bottom" secondItem="wfy-db-euE" secondAttribute="top" id="6ot-zD-sze"/>
-                            <constraint firstItem="7Mj-sL-hrd" firstAttribute="top" secondItem="Qwg-EP-bd6" secondAttribute="bottom" id="ELA-C6-NiG"/>
-                            <constraint firstAttribute="trailing" secondItem="7Mj-sL-hrd" secondAttribute="trailing" id="HDO-xr-mBl"/>
-                            <constraint firstItem="Gkb-TR-PCB" firstAttribute="leading" secondItem="8bC-Xf-vdC" secondAttribute="leading" id="Kmo-6K-gS4"/>
-                            <constraint firstItem="Qwg-EP-bd6" firstAttribute="leading" secondItem="8bC-Xf-vdC" secondAttribute="leading" id="hGu-lm-fMG"/>
-                            <constraint firstAttribute="trailing" secondItem="Qwg-EP-bd6" secondAttribute="trailing" id="iXR-LK-nTO"/>
-                            <constraint firstItem="7Mj-sL-hrd" firstAttribute="leading" secondItem="8bC-Xf-vdC" secondAttribute="leading" id="nr7-jW-ZYf"/>
-                            <constraint firstItem="Gkb-TR-PCB" firstAttribute="top" secondItem="y3c-jy-aDJ" secondAttribute="bottom" constant="8" id="uCF-VW-rR0"/>
-                        </constraints>
-                    </view>
-                    <connections>
-                        <outlet property="invokeButton" destination="A4J-Mg-nmd" id="UxZ-Ft-E45"/>
-                        <outlet property="modelControl" destination="rhA-nW-xzT" id="KKf-TT-BQ2"/>
-                        <outlet property="resultsTextView" destination="7Mj-sL-hrd" id="T4I-z4-tYA"/>
-                    </connections>
-                </viewController>
-                <placeholder placeholderIdentifier="IBFirstResponder" id="dkx-z0-nzr" sceneMemberID="firstResponder"/>
-            </objects>
-            <point key="canvasLocation" x="125.59999999999999" y="133.5832083958021"/>
-        </scene>
-    </scenes>
-</document>
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift
deleted file mode 100644
index bc8a70c848..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift
+++ /dev/null
@@ -1,13 +0,0 @@
-import Foundation
-
-extension Data {
-  /// Creates a new buffer by copying the buffer pointer of the given array.
-  ///
-  /// - Warning: The given array's element type `T` must be trivial in that it can be copied bit
-  ///     for bit with no indirection or reference-counting operations; otherwise, reinterpreting
-  ///     data from the resulting buffer has undefined behavior.
-  /// - Parameter array: An array with elements of type `T`.
-  init<T>(copyingBufferOf array: [T]) {
-    self = array.withUnsafeBufferPointer(Data.init)
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist
deleted file mode 100644
index 3ca3875f04..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist
+++ /dev/null
@@ -1,46 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
-<plist version="1.0">
-<dict>
-	<key>CFBundleDevelopmentRegion</key>
-	<string>en</string>
-	<key>CFBundleExecutable</key>
-	<string>$(EXECUTABLE_NAME)</string>
-	<key>CFBundleIdentifier</key>
-	<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
-	<key>CFBundleInfoDictionaryVersion</key>
-	<string>6.0</string>
-	<key>CFBundleName</key>
-	<string>$(PRODUCT_NAME)</string>
-	<key>CFBundlePackageType</key>
-	<string>APPL</string>
-	<key>CFBundleShortVersionString</key>
-	<string>1.0</string>
-	<key>CFBundleVersion</key>
-	<string>0.0.1</string>
-	<key>LSRequiresIPhoneOS</key>
-	<true/>
-	<key>NSCameraUsageDescription</key>
-	<string>NSCameraUsageDescription</string>
-	<key>NSPhotoLibraryUsageDescription</key>
-	<string>Select a photo to detect objects in.</string>
-	<key>UILaunchStoryboardName</key>
-	<string>LaunchScreen</string>
-	<key>UIMainStoryboardFile</key>
-	<string>Main</string>
-	<key>UIRequiredDeviceCapabilities</key>
-	<array>
-		<string>armv7</string>
-	</array>
-	<key>UISupportedInterfaceOrientations</key>
-	<array>
-		<string>UIInterfaceOrientationPortrait</string>
-		<string>UIInterfaceOrientationPortraitUpsideDown</string>
-	</array>
-	<key>UISupportedInterfaceOrientations~ipad</key>
-	<array>
-		<string>UIInterfaceOrientationPortrait</string>
-		<string>UIInterfaceOrientationPortraitUpsideDown</string>
-	</array>
-</dict>
-</plist>
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift
deleted file mode 100644
index 73c74fd19c..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift
+++ /dev/null
@@ -1,299 +0,0 @@
-import TensorFlowLite
-import UIKit
-
-class ViewController: UIViewController {
-
-  // MARK: - Properties
-
-  /// TensorFlowLite interpreter object for performing inference from a given model.
-  private var interpreter: Interpreter?
-
-  /// Serial dispatch queue for managing `Interpreter` calls.
-  private let interpreterQueue = DispatchQueue(
-    label: Constant.dispatchQueueLabel,
-    qos: .userInitiated
-  )
-
-  /// The currently selected model.
-  private var currentModel: Model {
-    guard let currentModel = Model(rawValue: modelControl.selectedSegmentIndex) else {
-      preconditionFailure("Invalid model for selected segment index.")
-    }
-    return currentModel
-  }
-
-  /// A description of the current model.
-  private var modelDescription: String {
-    guard let interpreter = interpreter else { return "" }
-    let inputCount = interpreter.inputTensorCount
-    let outputCount = interpreter.outputTensorCount
-    let inputTensors = (0..<inputCount).map { index in
-      var tensorInfo = "  Input \(index + 1): "
-      do {
-        let tensor = try interpreter.input(at: index)
-        tensorInfo += "\(tensor)"
-      } catch let error {
-        tensorInfo += "\(error.localizedDescription)"
-      }
-      return tensorInfo
-    }.joined(separator: "\n")
-    let outputTensors = (0..<outputCount).map { index in
-      var tensorInfo = "  Output \(index + 1): "
-      do {
-        let tensor = try interpreter.output(at: index)
-        tensorInfo += "\(tensor)"
-      } catch let error {
-        tensorInfo += "\(error.localizedDescription)"
-      }
-      return tensorInfo
-    }.joined(separator: "\n")
-    return "Model Description:\n" +
-             "  Input Tensor Count = \(inputCount)\n\(inputTensors)\n\n" +
-             "  Output Tensor Count = \(outputCount)\n\(outputTensors)"
-  }
-
-  // MARK: - IBOutlets
-
-  /// A segmented control for changing models. See the `Model` enum for available models.
-  @IBOutlet private var modelControl: UISegmentedControl!
-
-  @IBOutlet private var resultsTextView: UITextView!
-  @IBOutlet private var invokeButton: UIBarButtonItem!
-
-  // MARK: - UIViewController
-
-  override func viewDidLoad() {
-    super.viewDidLoad()
-
-    invokeButton.isEnabled = false
-    loadModel()
-  }
-
-  // MARK: - IBActions
-
-  @IBAction func modelChanged(_ sender: Any) {
-    invokeButton.isEnabled = false
-    updateResultsText("Switched to the \(currentModel.description) model.")
-    loadModel()
-  }
-
-  @IBAction func invokeInterpreter(_ sender: Any) {
-    switch currentModel {
-    case .add:
-      invokeAdd()
-    case .addQuantized:
-      invokeAddQuantized()
-    case .multiAdd:
-      invokeMultiAdd()
-    }
-  }
-
-  // MARK: - Private
-
-  private func loadModel() {
-    let fileInfo = currentModel.fileInfo
-    guard let modelPath = Bundle.main.path(forResource: fileInfo.name, ofType: fileInfo.extension)
-    else {
-      updateResultsText("Failed to load the \(currentModel.description) model.")
-      return
-    }
-    setUpInterpreter(withModelPath: modelPath)
-  }
-
-  private func setUpInterpreter(withModelPath modelPath: String) {
-    interpreterQueue.async {
-      do {
-        var options = InterpreterOptions()
-        options.isErrorLoggingEnabled = true
-        self.interpreter = try Interpreter(modelPath: modelPath, options: options)
-      } catch let error {
-        self.updateResultsText(
-          "Failed to create the interpreter with error: \(error.localizedDescription)"
-        )
-        return
-      }
-      safeDispatchOnMain { self.invokeButton.isEnabled = true }
-    }
-  }
-
-  private func invokeAdd() {
-    interpreterQueue.async {
-      guard let interpreter = self.interpreter else {
-        self.updateResultsText(Constant.nilInterpreterErrorMessage)
-        return
-      }
-      do {
-        try interpreter.resizeInput(at: 0, to: [2])
-        try interpreter.allocateTensors()
-        let input: [Float32] = [1, 3]
-        let resultsText = self.modelDescription + "\n\n" +
-          "Performing 2 add operations on input \(input.description) equals: "
-        self.updateResultsText(resultsText)
-        let data = Data(copyingBufferOf: input)
-        try interpreter.copy(data, toInputAt: 0)
-        try interpreter.invoke()
-        let outputTensor = try interpreter.output(at: 0)
-        let results: () -> String = {
-          guard let results = [Float32](unsafeData: outputTensor.data) else { return "No results." }
-          return resultsText + results.description
-        }
-        self.updateResultsText(results())
-      } catch let error {
-        self.updateResultsText(
-          "Failed to invoke the interpreter with error: \(error.localizedDescription)"
-        )
-        return
-      }
-    }
-  }
-
-  private func invokeAddQuantized() {
-    interpreterQueue.async {
-      guard let interpreter = self.interpreter else {
-        self.updateResultsText(Constant.nilInterpreterErrorMessage)
-        return
-      }
-      do {
-        try interpreter.resizeInput(at: 0, to: [2])
-        try interpreter.allocateTensors()
-        let input: [UInt8] = [1, 3]
-        let resultsText = self.modelDescription + "\n\n" +
-          "Performing 2 add operations on quantized input \(input.description) equals: "
-        self.updateResultsText(resultsText)
-        let data = Data(input)
-        try interpreter.copy(data, toInputAt: 0)
-        try interpreter.invoke()
-        let outputTensor = try interpreter.output(at: 0)
-        let results: () -> String = {
-          guard let quantizationParameters = outputTensor.quantizationParameters else {
-            return "No results."
-          }
-          let quantizedResults = [UInt8](outputTensor.data)
-          let dequantizedResults = quantizedResults.map {
-            quantizationParameters.scale * Float(Int($0) - quantizationParameters.zeroPoint)
-          }
-          return resultsText + quantizedResults.description +
-                   ", dequantized results: " + dequantizedResults.description
-        }
-        self.updateResultsText(results())
-      } catch let error {
-        self.updateResultsText(
-          "Failed to invoke the interpreter with error: \(error.localizedDescription)"
-        )
-        return
-      }
-    }
-  }
-
-  private func invokeMultiAdd() {
-    interpreterQueue.async {
-      guard let interpreter = self.interpreter else {
-        self.updateResultsText(Constant.nilInterpreterErrorMessage)
-        return
-      }
-      do {
-        let shape = TensorShape(2)
-        try (0..<interpreter.inputTensorCount).forEach { index in
-          try interpreter.resizeInput(at: index, to: shape)
-        }
-        try interpreter.allocateTensors()
-        let inputs = try (0..<interpreter.inputTensorCount).map { index -> [Float32] in
-          let input = [Float32(index + 1), Float32(index + 2)]
-          let data = Data(copyingBufferOf: input)
-          try interpreter.copy(data, toInputAt: index)
-          return input
-        }
-        let resultsText = self.modelDescription + "\n\n" +
-          "Performing 3 add operations on inputs \(inputs.description) equals: "
-        self.updateResultsText(resultsText)
-        try interpreter.invoke()
-        let results = try (0..<interpreter.outputTensorCount).map { index -> [Float32] in
-          let tensor = try interpreter.output(at: index)
-          return [Float32](unsafeData: tensor.data) ?? []
-        }
-        self.updateResultsText(resultsText + results.description)
-      } catch let error {
-        self.updateResultsText(
-          "Failed to invoke the interpreter with error: \(error.localizedDescription)"
-        )
-        return
-      }
-    }
-  }
-
-  private func updateResultsText(_ text: String? = nil) {
-    safeDispatchOnMain { self.resultsTextView.text = text }
-  }
-}
-
-// MARK: - Constants
-
-private enum Constant {
-  static let dispatchQueueLabel = "TensorFlowLiteInterpreterQueue"
-  static let nilInterpreterErrorMessage =
-    "Failed to invoke the interpreter because the interpreter was nil."
-}
-
-/// Models that can be loaded by the TensorFlow Lite `Interpreter`.
-private enum Model: Int, CustomStringConvertible {
-  /// A float model that performs two add operations on one input tensor and returns the result in
-  /// one output tensor.
-  case add = 0
-  /// A quantized model that performs two add operations on one input tensor and returns the result
-  /// in one output tensor.
-  case addQuantized = 1
-  /// A float model that performs three add operations on four input tensors and returns the results
-  /// in 2 output tensors.
-  case multiAdd = 2
-
-  var fileInfo: (name: String, extension: String) {
-    switch self {
-    case .add:
-      return Add.fileInfo
-    case .addQuantized:
-      return AddQuantized.fileInfo
-    case .multiAdd:
-      return MultiAdd.fileInfo
-    }
-  }
-
-  // MARK: - CustomStringConvertible
-
-  var description: String {
-    switch self {
-    case .add:
-      return Add.name
-    case .addQuantized:
-      return AddQuantized.name
-    case .multiAdd:
-      return MultiAdd.name
-    }
-  }
-}
-
-/// Values for the `Add` model.
-private enum Add {
-  static let name = "Add"
-  static let fileInfo = (name: "add", extension: "bin")
-}
-
-/// Values for the `AddQuantized` model.
-private enum AddQuantized {
-  static let name = "AddQuantized"
-  static let fileInfo = (name: "add_quantized", extension: "bin")
-}
-
-/// Values for the `MultiAdd` model.
-private enum MultiAdd {
-  static let name = "MultiAdd"
-  static let fileInfo = (name: "multi_add", extension: "bin")
-}
-
-// MARK: - Fileprivate
-
-/// Safely dispatches the given block on the main queue. If the current thread is `main`, the block
-/// is executed synchronously; otherwise, the block is executed asynchronously on the main thread.
-fileprivate func safeDispatchOnMain(_ block: @escaping () -> Void) {
-  if Thread.isMainThread { block(); return }
-  DispatchQueue.main.async { block() }
-}
diff --git a/tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift b/tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift
deleted file mode 100644
index 54b4f59b28..0000000000
--- a/tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift
+++ /dev/null
@@ -1,54 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-@testable import TensorFlowLite
-import XCTest
-
-class InterpreterOptionsTests: XCTestCase {
-
-  func testInterpreterOptions_InitWithDefaultValues() {
-    let options = InterpreterOptions()
-    XCTAssertNil(options.threadCount)
-    XCTAssertFalse(options.isErrorLoggingEnabled)
-  }
-
-  func testInterpreterOptions_InitWithCustomValues() {
-    var options = InterpreterOptions()
-    options.threadCount = 2
-    XCTAssertEqual(options.threadCount, 2)
-    options.isErrorLoggingEnabled = true
-    XCTAssertTrue(options.isErrorLoggingEnabled)
-  }
-
-  func testInterpreterOptions_Equatable() {
-    var options1 = InterpreterOptions()
-    var options2 = InterpreterOptions()
-    XCTAssertEqual(options1, options2)
-
-    options1.threadCount = 2
-    options2.threadCount = 2
-    XCTAssertEqual(options1, options2)
-
-    options2.threadCount = 3
-    XCTAssertNotEqual(options1, options2)
-    options2.threadCount = 2
-
-    options1.isErrorLoggingEnabled = true
-    options2.isErrorLoggingEnabled = true
-    XCTAssertEqual(options1, options2)
-
-    options2.isErrorLoggingEnabled = false
-    XCTAssertNotEqual(options1, options2)
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift b/tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift
deleted file mode 100644
index e98da5f951..0000000000
--- a/tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift
+++ /dev/null
@@ -1,315 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-@testable import TensorFlowLite
-import XCTest
-
-class InterpreterTests: XCTestCase {
-
-  var interpreter: Interpreter!
-
-  override func setUp() {
-    super.setUp()
-
-    interpreter = try! Interpreter(modelPath: AddModel.path)
-  }
-
-  override func tearDown() {
-    interpreter = nil
-
-    super.tearDown()
-  }
-
-  func testInterpreter_InitWithModelPath() {
-    XCTAssertNoThrow(try Interpreter(modelPath: AddModel.path))
-  }
-
-  func testInterpreter_Init_ThrowsFailedToLoadModel() {
-    XCTAssertThrowsError(try Interpreter(modelPath: "/invalid/path")) { error in
-      self.assertEqualErrors(actual: error, expected: .failedToLoadModel)
-    }
-  }
-
-  func testInterpreter_InitWithModelPathAndOptions() {
-    var options = InterpreterOptions()
-    options.threadCount = 2
-    XCTAssertNoThrow(try Interpreter(modelPath: AddModel.path, options: options))
-  }
-
-  func testInterpreter_InputTensorCount() {
-    XCTAssertEqual(interpreter.inputTensorCount, AddModel.inputTensorCount)
-  }
-
-  func testInterpreter_OutputTensorCount() {
-    XCTAssertEqual(interpreter.outputTensorCount, AddModel.outputTensorCount)
-  }
-
-  func testInterpreter_Invoke() throws {
-    try interpreter.allocateTensors()
-    XCTAssertNoThrow(try interpreter.invoke())
-  }
-
-  func testInterpreter_Invoke_ThrowsAllocateTensorsRequired_ModelNotReady() {
-    XCTAssertThrowsError(try interpreter.invoke()) { error in
-      self.assertEqualErrors(actual: error, expected: .allocateTensorsRequired)
-    }
-  }
-
-  func testInterpreter_InputTensorAtIndex() throws {
-    try setUpAddModelInputTensor()
-    let inputTensor = try interpreter.input(at: AddModel.validIndex)
-    XCTAssertEqual(inputTensor, AddModel.inputTensor)
-  }
-
-  func testInterpreter_InputTensorAtIndex_QuantizedModel() throws {
-    interpreter = try Interpreter(modelPath: AddQuantizedModel.path)
-    try setUpAddQuantizedModelInputTensor()
-    let inputTensor = try interpreter.input(at: AddQuantizedModel.inputOutputIndex)
-    XCTAssertEqual(inputTensor, AddQuantizedModel.inputTensor)
-  }
-
-  func testInterpreter_InputTensorAtIndex_ThrowsInvalidIndex() throws {
-    try interpreter.allocateTensors()
-    XCTAssertThrowsError(try interpreter.input(at: AddModel.invalidIndex)) { error in
-      let maxIndex = AddModel.inputTensorCount - 1
-      self.assertEqualErrors(
-        actual: error,
-        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
-      )
-    }
-  }
-
-  func testInterpreter_InputTensorAtIndex_ThrowsAllocateTensorsRequired() {
-    XCTAssertThrowsError(try interpreter.input(at: AddModel.validIndex)) { error in
-      self.assertEqualErrors(actual: error, expected: .allocateTensorsRequired)
-    }
-  }
-
-  func testInterpreter_OutputTensorAtIndex() throws {
-    try setUpAddModelInputTensor()
-    try interpreter.invoke()
-    let outputTensor = try interpreter.output(at: AddModel.validIndex)
-    XCTAssertEqual(outputTensor, AddModel.outputTensor)
-    let expectedResults = [Float32](unsafeData: outputTensor.data)
-    XCTAssertEqual(expectedResults, AddModel.results)
-  }
-
-  func testInterpreter_OutputTensorAtIndex_QuantizedModel() throws {
-    interpreter = try Interpreter(modelPath: AddQuantizedModel.path)
-    try setUpAddQuantizedModelInputTensor()
-    try interpreter.invoke()
-    let outputTensor = try interpreter.output(at: AddQuantizedModel.inputOutputIndex)
-    XCTAssertEqual(outputTensor, AddQuantizedModel.outputTensor)
-    let expectedResults = [UInt8](outputTensor.data)
-    XCTAssertEqual(expectedResults, AddQuantizedModel.results)
-  }
-
-  func testInterpreter_OutputTensorAtIndex_ThrowsInvalidIndex() throws {
-    try interpreter.allocateTensors()
-    try interpreter.invoke()
-    XCTAssertThrowsError(try interpreter.output(at: AddModel.invalidIndex)) { error in
-      let maxIndex = AddModel.outputTensorCount - 1
-      self.assertEqualErrors(
-        actual: error,
-        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
-      )
-    }
-  }
-
-  func testInterpreter_OutputTensorAtIndex_ThrowsInvokeInterpreterRequired() {
-    XCTAssertThrowsError(try interpreter.output(at: AddModel.validIndex)) { error in
-      self.assertEqualErrors(actual: error, expected: .invokeInterpreterRequired)
-    }
-  }
-
-  func testInterpreter_ResizeInputTensorAtIndexToShape() {
-    XCTAssertNoThrow(try interpreter.resizeInput(at: AddModel.validIndex, to: [2, 2, 3]))
-    XCTAssertNoThrow(try interpreter.allocateTensors())
-  }
-
-  func testInterpreter_ResizeInputTensorAtIndexToShape_ThrowsInvalidIndex() {
-    XCTAssertThrowsError(try interpreter.resizeInput(
-      at: AddModel.invalidIndex,
-      to: [2, 2, 3]
-    )) { error in
-      let maxIndex = AddModel.inputTensorCount - 1
-      self.assertEqualErrors(
-        actual: error,
-        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
-      )
-    }
-  }
-
-  func testInterpreter_CopyDataToInputTensorAtIndex() throws {
-    try interpreter.resizeInput(at: AddModel.validIndex, to: AddModel.shape)
-    try interpreter.allocateTensors()
-    let inputTensor = try interpreter.copy(AddModel.inputData, toInputAt: AddModel.validIndex)
-    XCTAssertEqual(inputTensor.data, AddModel.inputData)
-  }
-
-  func testInterpreter_CopyDataToInputTensorAtIndex_ThrowsInvalidIndex() {
-    XCTAssertThrowsError(try interpreter.copy(
-      AddModel.inputData,
-      toInputAt: AddModel.invalidIndex
-    )) { error in
-      let maxIndex = AddModel.inputTensorCount - 1
-      self.assertEqualErrors(
-        actual: error,
-        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
-      )
-    }
-  }
-
-  func testInterpreter_CopyDataToInputTensorAtIndex_ThrowsInvalidDataCount() throws {
-    try interpreter.resizeInput(at: AddModel.validIndex, to: AddModel.shape)
-    try interpreter.allocateTensors()
-    let invalidData = Data(count: AddModel.dataCount - 1)
-    XCTAssertThrowsError(try interpreter.copy(
-      invalidData,
-      toInputAt: AddModel.validIndex
-    )) { error in
-      self.assertEqualErrors(
-        actual: error,
-        expected: .invalidTensorDataCount(provided: invalidData.count, required: AddModel.dataCount)
-      )
-    }
-  }
-
-  func testInterpreter_AllocateTensors() {
-    XCTAssertNoThrow(try interpreter.allocateTensors())
-  }
-
-  // MARK: - Private
-
-  private func setUpAddModelInputTensor() throws {
-    precondition(interpreter != nil)
-    try interpreter.resizeInput(at: AddModel.validIndex, to: AddModel.shape)
-    try interpreter.allocateTensors()
-    try interpreter.copy(AddModel.inputData, toInputAt: AddModel.validIndex)
-  }
-
-  private func setUpAddQuantizedModelInputTensor() throws {
-    precondition(interpreter != nil)
-    try interpreter.resizeInput(at: AddQuantizedModel.inputOutputIndex, to: AddQuantizedModel.shape)
-    try interpreter.allocateTensors()
-    try interpreter.copy(AddQuantizedModel.inputData, toInputAt: AddQuantizedModel.inputOutputIndex)
-  }
-
-  private func assertEqualErrors(actual: Error, expected: InterpreterError) {
-    guard let actual = actual as? InterpreterError else {
-      XCTFail("Actual error should be of type InterpreterError.")
-      return
-    }
-    XCTAssertEqual(actual, expected)
-  }
-}
-
-// MARK: - Constants
-
-/// Values for the `add.bin` model.
-private enum AddModel {
-  static let info = (name: "add", extension: "bin")
-  static let inputTensorCount = 1
-  static let outputTensorCount = 1
-  static let invalidIndex = 1
-  static let validIndex = 0
-  static let shape: TensorShape = [2]
-  static let dataCount = inputData.count
-  static let inputData = Data(copyingBufferOf: [Float32(1.0), Float32(3.0)])
-  static let outputData = Data(copyingBufferOf: [Float32(3.0), Float32(9.0)])
-  static let results = [Float32(3.0), Float32(9.0)]
-
-  static let inputTensor = Tensor(
-    name: "input",
-    dataType: .float32,
-    shape: shape,
-    data: inputData
-  )
-  static let outputTensor = Tensor(
-    name: "output",
-    dataType: .float32,
-    shape: shape,
-    data: outputData
-  )
-
-  static var path: String = {
-    let bundle = Bundle(for: InterpreterTests.self)
-    guard let path = bundle.path(forResource: info.name, ofType: info.extension) else { return "" }
-    return path
-  }()
-}
-
-/// Values for the `add_quantized.bin` model.
-private enum AddQuantizedModel {
-  static let info = (name: "add_quantized", extension: "bin")
-  static let inputOutputIndex = 0
-  static let shape: TensorShape = [2]
-  static let inputData = Data([1, 3])
-  static let outputData = Data([3, 9])
-  static let quantizationParameters = QuantizationParameters(scale: 0.003922, zeroPoint: 0)
-  static let results: [UInt8] = [3, 9]
-
-  static let inputTensor = Tensor(
-    name: "input",
-    dataType: .uInt8,
-    shape: shape,
-    data: inputData,
-    quantizationParameters: quantizationParameters
-  )
-  static let outputTensor = Tensor(
-    name: "output",
-    dataType: .uInt8,
-    shape: shape,
-    data: outputData,
-    quantizationParameters: quantizationParameters
-  )
-
-  static var path: String = {
-    let bundle = Bundle(for: InterpreterTests.self)
-    guard let path = bundle.path(forResource: info.name, ofType: info.extension) else { return "" }
-    return path
-  }()
-}
-
-// MARK: - Extensions
-
-extension Array {
-  /// Creates a new array from the bytes of the given unsafe data.
-  ///
-  /// - Note: Returns `nil` if `unsafeData.count` is not a multiple of
-  ///     `MemoryLayout<Element>.stride`.
-  /// - Parameter unsafeData: The data containing the bytes to turn into an array.
-  init?(unsafeData: Data) {
-    guard unsafeData.count % MemoryLayout<Element>.stride == 0 else { return nil }
-    let elements = unsafeData.withUnsafeBytes {
-      UnsafeBufferPointer<Element>(
-        start: $0,
-        count: unsafeData.count / MemoryLayout<Element>.stride
-      )
-    }
-    self.init(elements)
-  }
-}
-
-extension Data {
-  /// Creates a new buffer by copying the buffer pointer of the given array.
-  ///
-  /// - Warning: The given array's element type `T` must be trivial in that it can be copied bit
-  ///     for bit with no indirection or reference-counting operations; otherwise, reinterpreting
-  ///     data from the resulting buffer has undefined behavior.
-  /// - Parameter array: An array with elements of type `T`.
-  init<T>(copyingBufferOf array: [T]) {
-    self = array.withUnsafeBufferPointer(Data.init)
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/Tests/ModelTests.swift b/tensorflow/lite/experimental/swift/Tests/ModelTests.swift
deleted file mode 100644
index 025db18906..0000000000
--- a/tensorflow/lite/experimental/swift/Tests/ModelTests.swift
+++ /dev/null
@@ -1,59 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-@testable import TensorFlowLite
-import XCTest
-
-class ModelTests: XCTestCase {
-
-  var modelPath: String!
-
-  override func setUp() {
-    super.setUp()
-
-    let bundle = Bundle(for: type(of: self))
-    guard let modelPath = bundle.path(
-            forResource: Constant.modelInfo.name,
-            ofType: Constant.modelInfo.extension)
-    else {
-      XCTFail("Failed to get the model file path.")
-      return
-    }
-    self.modelPath = modelPath
-  }
-
-  override func tearDown() {
-    modelPath = nil
-
-    super.tearDown()
-  }
-
-  func testModel_InitWithFilePath() {
-    XCTAssertNotNil(Model(filePath: modelPath))
-  }
-
-  func testModel_InitWithEmptyFilePath_FailsInitialization() {
-    XCTAssertNil(Model(filePath: ""))
-  }
-
-  func testModel_InitWithInvalidFilePath_FailsInitialization() {
-    XCTAssertNil(Model(filePath: "invalid/path"))
-  }
-}
-
-// MARK: - Constants
-
-private enum Constant {
-  static let modelInfo = (name: "add", extension: "bin")
-}
diff --git a/tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift b/tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift
deleted file mode 100644
index 65648c2698..0000000000
--- a/tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift
+++ /dev/null
@@ -1,43 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-@testable import TensorFlowLite
-import XCTest
-
-class QuantizationParametersTests: XCTestCase {
-
-  func testQuantizationParameters_InitWithCustomValues() {
-    let parameters = QuantizationParameters(scale: 0.5, zeroPoint: 1)
-    XCTAssertEqual(parameters.scale, 0.5)
-    XCTAssertEqual(parameters.zeroPoint, 1)
-  }
-
-  func testQuantizationParameters_Equatable() {
-    let parameters1 = QuantizationParameters(scale: 0.5, zeroPoint: 1)
-    let parameters2 = QuantizationParameters(scale: 0.5, zeroPoint: 1)
-    XCTAssertEqual(parameters1, parameters2)
-
-    let parameters3 = QuantizationParameters(scale: 0.4, zeroPoint: 1)
-    XCTAssertNotEqual(parameters1, parameters3)
-    XCTAssertNotEqual(parameters2, parameters3)
-  }
-}
-
-// MARK: - Extensions
-
-extension QuantizationParameters: Equatable {
-  public static func == (lhs: QuantizationParameters, rhs: QuantizationParameters) -> Bool {
-    return lhs.scale == rhs.scale && lhs.zeroPoint == rhs.zeroPoint
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/Tests/TensorTests.swift b/tensorflow/lite/experimental/swift/Tests/TensorTests.swift
deleted file mode 100644
index 4540043a16..0000000000
--- a/tensorflow/lite/experimental/swift/Tests/TensorTests.swift
+++ /dev/null
@@ -1,83 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-@testable import TensorFlowLite
-import XCTest
-
-class TensorTests: XCTestCase {
-
-  // MARK: - Tensor
-
-  func testTensor_Init() {
-    let name = "InputTensor"
-    let dataType: TensorDataType = .uInt8
-    let shape = TensorShape(Constant.dimensions)
-    guard let data = name.data(using: .utf8) else { XCTFail("Data should not be nil."); return }
-    let quantizationParameters = QuantizationParameters(scale: 0.5, zeroPoint: 1)
-    let inputTensor = Tensor(
-      name: name,
-      dataType: dataType,
-      shape: shape,
-      data: data,
-      quantizationParameters: quantizationParameters
-    )
-    XCTAssertEqual(inputTensor.name, name)
-    XCTAssertEqual(inputTensor.dataType, dataType)
-    XCTAssertEqual(inputTensor.shape, shape)
-    XCTAssertEqual(inputTensor.data, data)
-    XCTAssertEqual(inputTensor.quantizationParameters, quantizationParameters)
-  }
-
-  // MARK: - TensorShape
-
-  func testTensorShape_InitWithArray() {
-    let shape = TensorShape(Constant.dimensions)
-    XCTAssertEqual(shape.rank, Constant.dimensions.count)
-    XCTAssertEqual(shape.dimensions, Constant.dimensions)
-  }
-
-  func testTensorShape_InitWithElements() {
-    let shape = TensorShape(2, 2, 3)
-    XCTAssertEqual(shape.rank, Constant.dimensions.count)
-    XCTAssertEqual(shape.dimensions, Constant.dimensions)
-  }
-
-  func testTensorShape_InitWithArrayLiteral() {
-    let shape: TensorShape = [2, 2, 3]
-    XCTAssertEqual(shape.rank, Constant.dimensions.count)
-    XCTAssertEqual(shape.dimensions, Constant.dimensions)
-  }
-}
-
-// MARK: - Constants
-
-private enum Constant {
-  /// Array of 2 arrays of 2 arrays of 3 numbers: [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]].
-  static let dimensions = [2, 2, 3]
-}
-
-// MARK: - Extensions
-
-extension TensorShape: Equatable {
-  public static func == (lhs: TensorShape, rhs: TensorShape) -> Bool {
-    return lhs.rank == rhs.rank && lhs.dimensions == rhs.dimensions
-  }
-}
-
-extension Tensor: Equatable {
-  public static func == (lhs: Tensor, rhs: Tensor) -> Bool {
-    return lhs.name == rhs.name && lhs.dataType == rhs.dataType && lhs.shape == rhs.shape &&
-           lhs.data == rhs.data && lhs.quantizationParameters == rhs.quantizationParameters
-  }
-}
diff --git a/tensorflow/tools/pip_package/pip_smoke_test.py b/tensorflow/tools/pip_package/pip_smoke_test.py
index 952c71c615..51d010c9e1 100644
--- a/tensorflow/tools/pip_package/pip_smoke_test.py
+++ b/tensorflow/tools/pip_package/pip_smoke_test.py
@@ -30,19 +30,14 @@ os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../..")))
 PIP_PACKAGE_QUERY_EXPRESSION = (
     "deps(//tensorflow/tools/pip_package:build_pip_package)")
 
-# List of file paths containing BUILD files that should not be included for the
-# pip smoke test.
-BUILD_BLACKLIST = [
-    "tensorflow/lite/examples/android",
-    "tensorflow/lite/experimental/swift",
-]
 
 def GetBuild(dir_base):
   """Get the list of BUILD file all targets recursively startind at dir_base."""
   items = []
   for root, _, files in os.walk(dir_base):
     for name in files:
-      if (name == "BUILD" and root not in BUILD_BLACKLIST):
+      if (name == "BUILD" and
+          root.find("tensorflow/lite/examples/android") == -1):
         items.append("//" + root + ":all")
   return items
 
@@ -72,9 +67,9 @@ def BuildPyTestDependencies():
 
 PYTHON_TARGETS, PY_TEST_QUERY_EXPRESSION = BuildPyTestDependencies()
 
+# Hard-coded blacklist of files if not included in pip package
 # TODO(amitpatankar): Clean up blacklist.
-# List of dependencies that should not included in the pip package.
-DEPENDENCY_BLACKLIST = [
+BLACKLIST = [
     "//tensorflow/python:extra_py_tests_deps",
     "//tensorflow/cc/saved_model:saved_model_half_plus_two",
     "//tensorflow:no_tensorflow_py_deps",
@@ -87,7 +82,9 @@ DEPENDENCY_BLACKLIST = [
     "//tensorflow/core/kernels/cloud:bigquery_reader_ops",
     "//tensorflow/python/feature_column:vocabulary_testdata",
     "//tensorflow/python:framework/test_file_system.so",
-    # lite
+    # contrib
+    "//tensorflow/contrib/session_bundle:session_bundle_half_plus_two",
+    "//tensorflow/contrib/keras:testing_utils",
     "//tensorflow/lite/experimental/examples/lstm:tflite_lstm",
     "//tensorflow/lite/experimental/examples/lstm:tflite_lstm.py",
     "//tensorflow/lite/experimental/examples/lstm:unidirectional_sequence_lstm_test",  # pylint:disable=line-too-long
@@ -96,9 +93,6 @@ DEPENDENCY_BLACKLIST = [
     "//tensorflow/lite/python:interpreter_test",
     "//tensorflow/lite/python:interpreter.py",
     "//tensorflow/lite/python:interpreter_test.py",
-    # contrib
-    "//tensorflow/contrib/session_bundle:session_bundle_half_plus_two",
-    "//tensorflow/contrib/keras:testing_utils",
     "//tensorflow/contrib/ffmpeg:test_data",
     "//tensorflow/contrib/fused_conv:fused_conv2d_bias_activation_op_test_base",
     "//tensorflow/contrib/hadoop:test_data",
@@ -155,8 +149,8 @@ def main():
   # File extensions and endings to ignore
   ignore_extensions = ["_test", "_test.py", "_test_gpu", "_test_gpu.py"]
 
-  ignored_files_count = 0
-  blacklisted_dependencies_count = len(DEPENDENCY_BLACKLIST)
+  ignored_files = 0
+  blacklisted_files = len(BLACKLIST)
   # Compare dependencies
   for dependency in tf_py_test_dependencies_list:
     if dependency and dependency.startswith("//tensorflow"):
@@ -164,16 +158,16 @@ def main():
       # Ignore extensions
       if any(dependency.endswith(ext) for ext in ignore_extensions):
         ignore = True
-        ignored_files_count += 1
+        ignored_files += 1
 
-      # Check if the dependency is in the pip package, the dependency blacklist,
-      # or should be ignored because of its file extension.
+      # Check if the dependency is in the pip package, the blacklist, or
+      # should be ignored because of its file extension
       if not (ignore or dependency in pip_package_dependencies_list or
-              dependency in DEPENDENCY_BLACKLIST):
+              dependency in BLACKLIST):
         missing_dependencies.append(dependency)
 
-  print("Ignored files count: %d" % ignored_files_count)
-  print("Blacklisted dependencies count: %d" % blacklisted_dependencies_count)
+  print("Ignored files: %d" % ignored_files)
+  print("Blacklisted files: %d" % blacklisted_files)
   if missing_dependencies:
     print("Missing the following dependencies from pip_packages:")
     for missing_dependency in missing_dependencies:
-- 
GitLab


From 2e55da567448374f9f7e2498c1e338d07baf812f Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Thu, 3 Jan 2019 18:15:14 -0800
Subject: [PATCH 0167/2345] [Grappler] MutableGraphView dedup control
 dependencies on init and on mutations. Update max regular output port on
 mutations.

PiperOrigin-RevId: 227783394
---
 tensorflow/core/grappler/graph_view.h         |   5 +-
 .../core/grappler/mutable_graph_view.cc       | 275 ++++++++----
 tensorflow/core/grappler/mutable_graph_view.h |  72 ++--
 .../core/grappler/mutable_graph_view_test.cc  | 395 ++++++++++++++----
 .../optimizers/data/graph_test_utils.cc       |   4 +-
 .../optimizers/data/latency_all_edges_test.cc |   4 +-
 6 files changed, 552 insertions(+), 203 deletions(-)

diff --git a/tensorflow/core/grappler/graph_view.h b/tensorflow/core/grappler/graph_view.h
index 16156d0f20..a17d17524a 100644
--- a/tensorflow/core/grappler/graph_view.h
+++ b/tensorflow/core/grappler/graph_view.h
@@ -172,9 +172,8 @@ class GraphViewInternal {
     if (fanin.index() < -1) {
       return false;
     }
-    string fanin_string = TensorIdToString(fanin);
-    for (int i = 0; i < node.input_size(); ++i) {
-      if (node.input(i) == fanin_string) {
+    for (const string& input : node.input()) {
+      if (ParseTensorName(input) == fanin) {
         return true;
       }
     }
diff --git a/tensorflow/core/grappler/mutable_graph_view.cc b/tensorflow/core/grappler/mutable_graph_view.cc
index ca4d5255c0..5df8bd8113 100644
--- a/tensorflow/core/grappler/mutable_graph_view.cc
+++ b/tensorflow/core/grappler/mutable_graph_view.cc
@@ -39,8 +39,103 @@ bool IsTensorIdPortValid(const TensorId& tensor_id) {
   return tensor_id.index() >= Graph::kControlSlot;
 }
 
+// Determines if node is an Identity where it's first regular input is a Switch
+// node.
+bool IsIdentityConsumingSwitch(const MutableGraphView& graph,
+                               const NodeDef& node) {
+  if ((IsIdentity(node) || IsIdentityNSingleInput(node)) &&
+      node.input_size() > 0) {
+    TensorId tensor_id = ParseTensorName(node.input(0));
+    if (tensor_id.index() == Graph::kControlSlot) return false;
+    NodeDef* input_node = graph.GetNode(tensor_id.node());
+    return IsSwitch(*input_node);
+  }
+  return false;
+}
+
+// Determines if node input can be deduped by regular inputs when used as a
+// control dependency. Specifically, if a node is an Identity that leads to a
+// Switch node, when used as a control dependency, that control dependency
+// should not be deduped even though the same node is used as a regular input.
+bool CanDedupControlWithRegularInput(const MutableGraphView& graph,
+                                     const NodeDef& control_node) {
+  return !IsIdentityConsumingSwitch(graph, control_node);
+}
+
+// Determines if node input can be deduped by regular inputs when used as a
+// control dependency. Specifically, if a node is an Identity that leads to a
+// Switch node, when used as a control dependency, that control dependency
+// should not be deduped even though the same node is used as a regular input.
+bool CanDedupControlWithRegularInput(const MutableGraphView& graph,
+                                     absl::string_view control_node_name) {
+  NodeDef* control_node = graph.GetNode(control_node_name);
+  return CanDedupControlWithRegularInput(graph, *control_node);
+}
+
 }  // namespace
 
+void MutableGraphView::AddAndDedupFanouts(NodeDef* node) {
+  absl::flat_hash_set<absl::string_view> fanins;
+  absl::flat_hash_set<absl::string_view> controlling_fanins;
+  int pos = 0;
+  const int last_idx = node->input_size() - 1;
+  int last_pos = last_idx;
+  while (pos <= last_pos) {
+    TensorId tensor_id = ParseTensorName(node->input(pos));
+    absl::string_view input_node_name = tensor_id.node();
+    bool is_control_input = IsControlInput(tensor_id);
+    bool can_dedup_control_with_regular_input =
+        CanDedupControlWithRegularInput(*this, input_node_name);
+    bool can_dedup_control =
+        is_control_input && (can_dedup_control_with_regular_input ||
+                             (!can_dedup_control_with_regular_input &&
+                              controlling_fanins.contains(input_node_name)));
+    if (!gtl::InsertIfNotPresent(&fanins, input_node_name) &&
+        can_dedup_control) {
+      node->mutable_input()->SwapElements(pos, last_pos--);
+    } else {
+      OutputPort output(nodes()[input_node_name], tensor_id.index());
+
+      if (is_control_input) {
+        fanouts()[output].emplace(node, Graph::kControlSlot);
+      } else {
+        max_regular_output_port()[output.node] =
+            std::max(max_regular_output_port()[output.node], output.port_id);
+        fanouts()[output].emplace(node, pos);
+      }
+      ++pos;
+    }
+    if (is_control_input) {
+      controlling_fanins.insert(input_node_name);
+    }
+  }
+
+  if (last_pos < last_idx) {
+    node->mutable_input()->DeleteSubrange(last_pos + 1, last_idx - last_pos);
+  }
+}
+
+void MutableGraphView::UpdateMaxRegularOutputPortForRemovedFanin(
+    const OutputPort& fanin,
+    const absl::flat_hash_set<InputPort>& fanin_fanouts) {
+  int max_port = max_regular_output_port()[fanin.node];
+  if (!fanin_fanouts.empty() || max_port != fanin.port_id) {
+    return;
+  }
+  bool updated_max_port = false;
+  for (int i = fanin.port_id - 1; i >= 0; --i) {
+    OutputPort fanin_port(fanin.node, i);
+    if (!fanouts()[fanin_port].empty()) {
+      max_regular_output_port()[fanin.node] = i;
+      updated_max_port = true;
+      break;
+    }
+  }
+  if (!updated_max_port) {
+    max_regular_output_port().erase(fanin.node);
+  }
+}
+
 const absl::flat_hash_set<MutableGraphView::InputPort>&
 MutableGraphView::GetFanout(const GraphView::OutputPort& port) const {
   return GetFanout(MutableGraphView::OutputPort(const_cast<NodeDef*>(port.node),
@@ -65,16 +160,16 @@ NodeDef* MutableGraphView::AddNode(NodeDef&& node) {
 
   AddUniqueNodeOrDie(node_in_graph);
 
-  AddFanouts(node_in_graph);
+  AddAndDedupFanouts(node_in_graph);
   return node_in_graph;
 }
 
-void MutableGraphView::UpdateFanouts(absl::string_view from_node,
+bool MutableGraphView::UpdateFanouts(absl::string_view from_node,
                                      absl::string_view to_node) {
   NodeDef* from_node_ptr = GetNode(from_node);
   NodeDef* to_node_ptr = GetNode(to_node);
   if (from_node_ptr && to_node_ptr) {
-    UpdateFanouts(from_node_ptr, to_node_ptr);
+    return UpdateFanoutsInternal(from_node_ptr, to_node_ptr);
   } else if (!from_node_ptr) {
     LOG(WARNING) << absl::Substitute(
         "Can't update fanouts from '$0' to '$1', from node was not found.",
@@ -84,9 +179,11 @@ void MutableGraphView::UpdateFanouts(absl::string_view from_node,
         "Can't update fanouts from '$0' to '$1', to node was not found.",
         from_node, to_node);
   }
+  return false;
 }
 
-void MutableGraphView::UpdateFanouts(NodeDef* from_node, NodeDef* to_node) {
+bool MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
+                                             NodeDef* to_node) {
   VLOG(2) << absl::Substitute("Update fanouts from '$0' to '$1'.",
                               from_node->name(), to_node->name());
 
@@ -112,6 +209,7 @@ void MutableGraphView::UpdateFanouts(NodeDef* from_node, NodeDef* to_node) {
   // input to some other node.
   int keep_max_regular_output_port = -1;
 
+  bool modified = false;
   for (const Edge& edge : regular_edges) {
     const OutputPort output_port = edge.src;
     const InputPort input_port = edge.dst;
@@ -120,7 +218,7 @@ void MutableGraphView::UpdateFanouts(NodeDef* from_node, NodeDef* to_node) {
     // AddAndUpdateFanoutsWithoutSelfLoops test for an example).
     if (input_port.node == to_node) {
       keep_max_regular_output_port =
-          std::max(keep_max_regular_output_port, input_port.port_id);
+          std::max(keep_max_regular_output_port, output_port.port_id);
       continue;
     }
 
@@ -135,6 +233,11 @@ void MutableGraphView::UpdateFanouts(NodeDef* from_node, NodeDef* to_node) {
     remove_edge(output_port, input_port);
     // Add an edge between the `to_node` and new fanout node.
     add_edge(OutputPort(to_node, output_port.port_id), input_port);
+    // Dedup control dependency.
+    if (CanDedupControlWithRegularInput(*this, *to_node)) {
+      RemoveControllingFaninInternal(input_port.node, to_node);
+    }
+    modified = true;
   }
 
   // For the control fanouts we do not know the input index in a NodeDef,
@@ -142,7 +245,6 @@ void MutableGraphView::UpdateFanouts(NodeDef* from_node, NodeDef* to_node) {
 
   auto control_fanouts =
       GetFanout(GraphView::OutputPort(from_node, Graph::kControlSlot));
-  if (control_fanouts.empty()) return;
 
   const string from_control_input = absl::StrCat("^", from_node->name());
   const string to_control_input = absl::StrCat("^", to_node->name());
@@ -151,20 +253,9 @@ void MutableGraphView::UpdateFanouts(NodeDef* from_node, NodeDef* to_node) {
     // Node can't be control dependency of itself.
     if (control_port.node == to_node) continue;
 
-    // Find and update input corresponding to control dependency.
     NodeDef* node = control_port.node;
-    for (int i = node->input_size() - 1; i >= 0; --i) {
-      const string& input = node->input(i);
-      if (!IsControlInput(input)) break;  // we reached regular inputs
-      if (input == from_control_input) {
-        node->set_input(i, to_control_input);
-      }
-    }
-
-    // Remove old edge between the `from_node` and the fanout node.
-    remove_edge(OutputPort(from_node, Graph::kControlSlot), control_port);
-    // Add an edge between the `to_node` and new fanout node.
-    add_edge(OutputPort(to_node, Graph::kControlSlot), control_port);
+    modified |= RemoveControllingFaninInternal(node, from_node);
+    modified |= AddFaninInternal(node, {to_node, Graph::kControlSlot});
   }
 
   // Because we update all regular fanouts of `from_node`, we can just copy
@@ -177,21 +268,33 @@ void MutableGraphView::UpdateFanouts(NodeDef* from_node, NodeDef* to_node) {
   } else {
     max_regular_output_port().erase(from_node);
   }
+
+  return modified;
 }
 
 bool MutableGraphView::AddFaninInternal(NodeDef* node,
                                         const OutputPort& fanin) {
   int num_non_controlling_fanins =
       NumFanins(*node, /*include_controlling_nodes=*/false);
+  bool input_is_control = fanin.port_id == Graph::kControlSlot;
+  bool can_dedup_control_with_regular_input =
+      CanDedupControlWithRegularInput(*this, *fanin.node);
+  // Don't add duplicate control dependencies.
+  if (input_is_control) {
+    const int start =
+        can_dedup_control_with_regular_input ? 0 : num_non_controlling_fanins;
+    for (int i = start; i < node->input_size(); ++i) {
+      if (ParseTensorName(node->input(i)).node() == fanin.node->name()) {
+        return false;
+      }
+    }
+  }
+
   InputPort input;
   input.node = node;
-  input.port_id = fanin.port_id == Graph::kControlSlot
-                      ? Graph::kControlSlot
-                      : num_non_controlling_fanins;
+  input.port_id =
+      input_is_control ? Graph::kControlSlot : num_non_controlling_fanins;
 
-  if (!gtl::InsertIfNotPresent(&fanouts()[fanin], input)) {
-    return false;
-  }
   node->add_input(TensorIdToString({fanin.node->name(), fanin.port_id}));
   if (fanin.port_id > Graph::kControlSlot) {
     int node_input_size = node->input_size() - 1;
@@ -202,6 +305,17 @@ bool MutableGraphView::AddFaninInternal(NodeDef* node,
                                           num_non_controlling_fanins);
     }
   }
+
+  fanouts()[fanin].insert(input);
+  if (max_regular_output_port()[fanin.node] < fanin.port_id) {
+    max_regular_output_port()[fanin.node] = fanin.port_id;
+  }
+
+  // Dedup control dependencies.
+  if (!input_is_control && can_dedup_control_with_regular_input) {
+    RemoveControllingFaninInternal(node, fanin.node);
+  }
+
   return true;
 }
 
@@ -244,16 +358,18 @@ bool MutableGraphView::RemoveFanins(NodeDef* node,
       input.port_id =
           tensor_id.index() == Graph::kControlSlot ? Graph::kControlSlot : i;
 
+      auto& fanouts_set = fanouts()[fanin];
       if (remove_fanin) {
-        fanouts()[fanin].erase(input);
+        fanouts_set.erase(input);
       } else {
         // Shift inputs to be retained.
         if (tensor_id.index() > Graph::kControlSlot) {
-          fanouts()[fanin].erase(input);
-          fanouts()[fanin].insert(InputPort(node, i));
+          fanouts_set.erase(input);
+          fanouts_set.insert(InputPort(node, i));
         }
         mutable_inputs->SwapElements(i, curr_pos++);
       }
+      UpdateMaxRegularOutputPortForRemovedFanin(fanin, fanouts_set);
 
       modified = true;
     } else {
@@ -314,15 +430,12 @@ bool MutableGraphView::UpdateFanin(absl::string_view node_name,
     return false;
   }
 
-  bool is_from_fanin_control = from_fanin.index() == Graph::kControlSlot;
-  bool is_to_fanin_control = to_fanin.index() == Graph::kControlSlot;
   // When replacing a non control dependency fanin with a control dependency, or
   // vice versa, remove and add, so ports can be updated properly in fanout(s).
-  if (is_from_fanin_control || is_to_fanin_control) {
+  if (from_fanin.index() == Graph::kControlSlot ||
+      to_fanin.index() == Graph::kControlSlot) {
     bool modified = RemoveFanins(node, {from_fanin});
-    if (!HasFanin(*node, to_fanin)) {
-      modified |= AddFaninInternal(node, to_fanin);
-    }
+    modified |= AddFaninInternal(node, to_fanin);
     return modified;
   }
 
@@ -336,79 +449,47 @@ bool MutableGraphView::UpdateFanin(absl::string_view node_name,
   string to_fanin_string = TensorIdToString(to_fanin);
   int num_inputs = node->input_size();
   bool modified = false;
+  absl::flat_hash_set<InputPort>* from_fanin_port_fanouts = nullptr;
+  absl::flat_hash_set<InputPort>* to_fanin_port_fanouts = nullptr;
   for (int i = 0; i < num_inputs; ++i) {
     if (ParseTensorName(node->input(i)) == from_fanin) {
-      OutputPort from_fanin_port(from_fanin_node, from_fanin.index());
       InputPort old_input;
       old_input.node = node;
       old_input.port_id =
           from_fanin.index() == Graph::kControlSlot ? Graph::kControlSlot : i;
-      fanouts()[from_fanin_port].erase(old_input);
+      if (from_fanin_port_fanouts == nullptr) {
+        OutputPort from_fanin_port(from_fanin_node, from_fanin.index());
+        from_fanin_port_fanouts = &fanouts()[from_fanin_port];
+      }
+      from_fanin_port_fanouts->erase(old_input);
 
-      OutputPort to_fanin_port(to_fanin_node, to_fanin.index());
       InputPort new_input;
       new_input.node = node;
       new_input.port_id =
           to_fanin.index() == Graph::kControlSlot ? Graph::kControlSlot : i;
-      fanouts()[to_fanin_port].insert(new_input);
+      if (to_fanin_port_fanouts == nullptr) {
+        OutputPort to_fanin_port(to_fanin_node, to_fanin.index());
+        to_fanin_port_fanouts = &fanouts()[to_fanin_port];
+      }
+      to_fanin_port_fanouts->insert(new_input);
 
       node->set_input(i, to_fanin_string);
       modified = true;
     }
   }
 
-  return modified;
-}
-
-bool MutableGraphView::DedupControllingFanins(NodeDef* node) {
-  absl::flat_hash_set<absl::string_view> fanins;
-  absl::flat_hash_set<string> removed_fanins;
-  int pos = 0;
-  const int last_idx = node->input_size() - 1;
-  int last_pos = last_idx;
-  while (pos <= last_pos) {
-    const string& input = node->input(pos);
-    TensorId tensor_id = ParseTensorName(input);
-    if (!gtl::InsertIfNotPresent(&fanins, tensor_id.node()) &&
-        IsControlInput(tensor_id)) {
-      node->mutable_input()->SwapElements(pos, last_pos--);
-      removed_fanins.insert(input);
-    } else {
-      ++pos;
+  // Dedup control dependencies and update max regular output ports.
+  if (modified) {
+    UpdateMaxRegularOutputPortForRemovedFanin(
+        {from_fanin_node, from_fanin.index()}, *from_fanin_port_fanouts);
+    if (max_regular_output_port()[to_fanin_node] < to_fanin.index()) {
+      max_regular_output_port()[to_fanin_node] = to_fanin.index();
     }
-  }
-
-  if (last_pos < last_idx) {
-    absl::flat_hash_set<string> retained_fanins(
-        node->input().begin(), node->input().begin() + last_pos + 1);
-    for (const auto& removed : removed_fanins) {
-      if (!retained_fanins.contains(removed)) {
-        OutputPort fanin(nodes()[ParseTensorName(removed).node()],
-                         Graph::kControlSlot);
-        fanouts()[fanin].erase({node, Graph::kControlSlot});
-      }
+    if (CanDedupControlWithRegularInput(*this, *to_fanin_node)) {
+      RemoveControllingFaninInternal(node, to_fanin_node);
     }
-    node->mutable_input()->DeleteSubrange(last_pos + 1, last_idx - last_pos);
-    return true;
   }
 
-  return false;
-}
-
-bool MutableGraphView::DedupControllingFanins(absl::string_view node_name) {
-  NodeDef* node = GetNode(node_name);
-  if (node == nullptr) {
-    return false;
-  }
-  return DedupControllingFanins(node);
-}
-
-bool MutableGraphView::DedupControllingFanins() {
-  const int num_nodes = graph()->node_size();
-  bool modified = false;
-  for (int i = 0; i < num_nodes; ++i) {
-    modified |= DedupControllingFanins(graph()->mutable_node(i));
-  }
   return modified;
 }
 
@@ -463,6 +544,24 @@ bool MutableGraphView::AddControllingFanin(absl::string_view node_name,
   }
 }
 
+bool MutableGraphView::RemoveControllingFaninInternal(NodeDef* node,
+                                                      NodeDef* fanin_node) {
+  for (int i = node->input_size() - 1; i >= 0; --i) {
+    TensorId tensor_id = ParseTensorName(node->input(i));
+    if (tensor_id.index() > Graph::kControlSlot) {
+      break;
+    }
+    if (tensor_id.node() == fanin_node->name()) {
+      fanouts()[{fanin_node, Graph::kControlSlot}].erase(
+          {node, Graph::kControlSlot});
+      node->mutable_input()->SwapElements(i, node->input_size() - 1);
+      node->mutable_input()->RemoveLast();
+      return true;
+    }
+  }
+  return false;
+}
+
 void MutableGraphView::DeleteNodes(const std::set<string>& nodes_to_delete) {
   for (const string& node_name_to_delete : nodes_to_delete)
     RemoveFaninsInternal(nodes().at(node_name_to_delete),
@@ -488,7 +587,9 @@ void MutableGraphView::RemoveFaninsInternal(NodeDef* deleted_node,
     else
       input.port_id = i;
 
-    fanouts()[fanin].erase(input);
+    auto& fanouts_set = fanouts()[fanin];
+    fanouts_set.erase(input);
+    UpdateMaxRegularOutputPortForRemovedFanin(fanin, fanouts_set);
   }
 }
 
diff --git a/tensorflow/core/grappler/mutable_graph_view.h b/tensorflow/core/grappler/mutable_graph_view.h
index f7c2a1118e..93f0958d5e 100644
--- a/tensorflow/core/grappler/mutable_graph_view.h
+++ b/tensorflow/core/grappler/mutable_graph_view.h
@@ -24,8 +24,10 @@ limitations under the License.
 #include "absl/types/span.h"
 #include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
+#include "tensorflow/core/graph/graph.h"
 #include "tensorflow/core/graph/tensor_id.h"
 #include "tensorflow/core/grappler/graph_view.h"
+#include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
@@ -41,7 +43,7 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
  public:
   explicit MutableGraphView(GraphDef* graph) : GraphViewInternal(graph) {
     for (NodeDef& node : *graph->mutable_node()) AddUniqueNodeOrDie(&node);
-    for (NodeDef& node : *graph->mutable_node()) AddFanouts(&node);
+    for (NodeDef& node : *graph->mutable_node()) AddAndDedupFanouts(&node);
   }
 
   // Lookup fanouts/fanins using immutable ports.
@@ -63,16 +65,20 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // Updates all fanouts (input ports fetching output tensors) from `from_node`
   // to the `to_node`, including control dependencies.
   //
-  // Example: We have 2 nodes that use `bar` node output tensors as inputs:
-  //   1. foo1(bar:0, bar:1, other:0, ^bar)
+  // Example: We have 3 nodes that use `bar` node output tensors as inputs:
+  //   1. foo1(bar:0, bar:1, other:0)
   //   2. foo2(bar:1, other:1)
+  //   3. foo3(other:2, ^bar)
   //
   // After calling ForwardOutputs(bar, new_bar):
-  //   1. foo1(new_bar:0, new_bar:1, other:0, ^new_bar)
+  //   1. foo1(new_bar:0, new_bar:1, other:0)
   //   2. foo2(new_bar:1, other:1)
-  void UpdateFanouts(absl::string_view from_node, absl::string_view to_node);
+  //   3. foo3(other:2, ^new_bar)
+  //
+  // This will return true iff the nodes are modified.
+  bool UpdateFanouts(absl::string_view from_node, absl::string_view to_node);
 
-  // Add fanin to node `node_name`. If the node or fanin do not exist in the
+  // Adds fanin to node `node_name`. If the node or fanin do not exist in the
   // graph, nothing will be modified in the graph. If fanin is a control
   // dependency, existing control dependencies will be checked first before
   // adding. Otherwise fanin will be added after existing non control dependency
@@ -82,7 +88,7 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // already exists, the node will not be modified.
   bool AddFanin(absl::string_view node_name, const TensorId& fanin);
 
-  // Remove fanin from node `node_name`. If the node or fanin do not exist in
+  // Removes fanin from node `node_name`. If the node or fanin do not exist in
   // the graph, nothing will be modified in the graph. If there are multiple
   // inputs that match the fanin, all of them will be removed.
   //
@@ -90,32 +96,20 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // fanin, the node will not be modified.
   bool RemoveFanin(absl::string_view node_name, const TensorId& fanin);
 
-  // Remove all fanins from node `node_name`. Control dependencies will be
+  // Removes all fanins from node `node_name`. Control dependencies will be
   // retained if keep_controlling_fanins is true.
   //
   // This will return true iff the node is modified.
   bool RemoveAllFanins(absl::string_view node_name,
                        bool keep_controlling_fanins);
 
-  // Replace all fanins `from_fanin` with `to_fanin` in node `node_name`. If
+  // Replaces all fanins `from_fanin` with `to_fanin` in node `node_name`. If
   // the fanins or node do not exist, nothing will be modified in the graph.
   //
   // This will return true iff the node is modified.
   bool UpdateFanin(absl::string_view node_name, const TensorId& from_fanin,
                    const TensorId& to_fanin);
 
-  // Removes redundant control fanins from node `node_name`.
-  //
-  // This will return true iff the node is modified.
-  // TODO(lyandy): Measure performance of deduping on every AddFanin compared to
-  // deduping once at the end.
-  bool DedupControllingFanins(absl::string_view node_name);
-
-  // Removes redundant control fanins from all nodes in the graph.
-  //
-  // This will return true iff the node is modified.
-  bool DedupControllingFanins();
-
   // Adds a control dependency to the target node named `node_name`.
   //
   // Case 1: If the fanin is not a Switch node, the control dependency is simply
@@ -140,27 +134,42 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   void DeleteNodes(const std::set<string>& nodes_to_delete);
 
  private:
+  // Adds fanouts for fanins of node to graph, while deduping control
+  // dependencies from existing control dependencies and regular fanins. Note,
+  // node inputs will be mutated if control dependencies can be deduped.
+  void AddAndDedupFanouts(NodeDef* node);
+
+  // Finds next output port smaller than fanin.port_id and update. The
+  // max_regular_output_port is only updated if fanin.port_id is the same as the
+  // current max_regular_output_port and if the fanouts set is empty. If there
+  // are no regular outputs, max_regular_output_port will be erased.
+  void UpdateMaxRegularOutputPortForRemovedFanin(
+      const OutputPort& fanin,
+      const absl::flat_hash_set<InputPort>& fanin_fanouts);
+
   // Updates all fanouts (input ports fetching output tensors) from `from_node`
   // to the `to_node`, including control dependencies.
   //
-  // Example: We have 2 nodes that use `bar` node output tensors as inputs:
-  //   1. foo1(bar:0, bar:1, other:0, ^bar)
+  // Example: We have 3 nodes that use `bar` node output tensors as inputs:
+  //   1. foo1(bar:0, bar:1, other:0)
   //   2. foo2(bar:1, other:1)
+  //   3. foo3(other:2, ^bar)
   //
   // After calling ForwardOutputs(bar, new_bar):
-  //   1. foo1(new_bar:0, new_bar:1, other:0, ^new_bar)
+  //   1. foo1(new_bar:0, new_bar:1, other:0)
   //   2. foo2(new_bar:1, other:1)
+  //   3. foo3(other:2, ^new_bar)
   //
   // IMPORTANT: If `from_node` or `to_node` is not in the underlying graph, the
   // behavior is undefined.
-  void UpdateFanouts(NodeDef* from_node, NodeDef* to_node);
+  bool UpdateFanoutsInternal(NodeDef* from_node, NodeDef* to_node);
 
   // Removes fanins of the deleted node from internal state. Control
   // dependencies are retained iff keep_controlling_fanins is true.
   void RemoveFaninsInternal(NodeDef* deleted_node,
                             bool keep_controlling_fanins);
 
-  // Add fanin to node. If fanin is a control dependency, existing control
+  // Adds fanin to node. If fanin is a control dependency, existing control
   // dependencies will be checked first before adding. Otherwise fanin will be
   // added after existing non control dependency inputs.
   //
@@ -168,7 +177,7 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // already exists, the node will not be modified.
   bool AddFaninInternal(NodeDef* node, const OutputPort& fanin);
 
-  // Add fanin to node. If the node or fanin do not exist in the graph, nothing
+  // Adds fanin to node. If the node or fanin do not exist in the graph, nothing
   // will be modified in the graph. If fanin is a control dependency, existing
   // control dependencies will be checked first before adding. Otherwise fanin
   // will be added after existing non control dependency inputs.
@@ -178,10 +187,15 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   bool AddFaninInternal(NodeDef* node, const TensorId& fanin);
 
   // Removes any fanin in node that matches to a fanin in fanins.
+  //
+  // This will return true iff the node is modified.
   bool RemoveFanins(NodeDef* node, absl::Span<const TensorId> fanins);
 
-  // Removes redundant control fanins from node.
-  bool DedupControllingFanins(NodeDef* node);
+  // Removes controlling fanin `fanin_node` from node if such controlling fanin
+  // exists.
+  //
+  // This will return true iff the node is modified.
+  bool RemoveControllingFaninInternal(NodeDef* node, NodeDef* fanin_node);
 };
 
 }  // end namespace grappler
diff --git a/tensorflow/core/grappler/mutable_graph_view_test.cc b/tensorflow/core/grappler/mutable_graph_view_test.cc
index cdc212f6f9..526ca85ec0 100644
--- a/tensorflow/core/grappler/mutable_graph_view_test.cc
+++ b/tensorflow/core/grappler/mutable_graph_view_test.cc
@@ -35,7 +35,8 @@ TEST(MutableGraphViewTest, AddAndUpdateFanouts) {
       {NDef("bar", "NotImportant", {}, {}),
        NDef("other", "NotImportant", {}, {}),
        NDef("foo_1", "NotImportant", {"bar", "other", "bar:1", "^bar"}),
-       NDef("foo_2", "NotImportant", {"other:1", "bar:2", "^bar"})},
+       NDef("foo_2", "NotImportant", {"other:1", "bar:2", "^bar"}),
+       NDef("foo_3", "NotImportant", {"other:2", "^bar"})},
       /*funcs=*/{});
 
   MutableGraphView graph(&graph_def);
@@ -43,7 +44,56 @@ TEST(MutableGraphViewTest, AddAndUpdateFanouts) {
   NodeDef* new_bar = graph.AddNode(NDef("new_bar", "NotImportant", {}, {}));
   NodeDef* bar = graph.GetNode("bar");
 
-  graph.UpdateFanouts(bar->name(), new_bar->name());
+  EXPECT_TRUE(graph.UpdateFanouts(bar->name(), new_bar->name()));
+
+  // Fanout nodes must have their inputs updated.
+  NodeDef* foo_1 = graph.GetNode("foo_1");
+  ASSERT_NE(foo_1, nullptr);
+  ASSERT_EQ(foo_1->input_size(), 3);
+  EXPECT_EQ(foo_1->input(0), "new_bar");
+  EXPECT_EQ(foo_1->input(1), "other");
+  EXPECT_EQ(foo_1->input(2), "new_bar:1");
+
+  NodeDef* foo_2 = graph.GetNode("foo_2");
+  ASSERT_NE(foo_2, nullptr);
+  ASSERT_EQ(foo_2->input_size(), 2);
+  EXPECT_EQ(foo_2->input(0), "other:1");
+  EXPECT_EQ(foo_2->input(1), "new_bar:2");
+
+  NodeDef* foo_3 = graph.GetNode("foo_3");
+  ASSERT_NE(foo_3, nullptr);
+  ASSERT_EQ(foo_3->input_size(), 2);
+  EXPECT_EQ(foo_3->input(0), "other:2");
+  EXPECT_EQ(foo_3->input(1), "^new_bar");
+
+  // And fanouts mapping must be also updated for both nodes.
+  bool include_control_fanouts = true;
+  auto old_node_fanouts = graph.GetFanouts(*bar, include_control_fanouts);
+  auto new_node_fanouts = graph.GetFanouts(*new_bar, include_control_fanouts);
+
+  EXPECT_TRUE(old_node_fanouts.empty());
+
+  EXPECT_EQ(new_node_fanouts.size(), 4);
+  EXPECT_EQ(new_node_fanouts.count(MutableGraphView::InputPort(foo_1, 0)), 1);
+  EXPECT_EQ(new_node_fanouts.count(MutableGraphView::InputPort(foo_1, 2)), 1);
+  EXPECT_EQ(new_node_fanouts.count(MutableGraphView::InputPort(foo_2, 1)), 1);
+  EXPECT_EQ(new_node_fanouts.count(MutableGraphView::InputPort(foo_3, -1)), 1);
+}
+
+TEST(MutableGraphViewTest, AddAndUpdateFanoutsKeepControls) {
+  GraphDef graph_def = test::function::GDef(
+      {NDef("bar_1", "Switch", {}, {}), NDef("bar_2", "Identity", {"bar_1:1"}),
+       NDef("other", "NotImportant", {}, {}),
+       NDef("foo_1", "NotImportant", {"bar_2", "other", "bar_2:1", "^bar_2"}),
+       NDef("foo_2", "NotImportant", {"other:1", "bar_2:2", "^bar_2"})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  NodeDef* new_bar = graph.AddNode(NDef("new_bar", "Identity", {"bar_1:2"}));
+  NodeDef* bar_2 = graph.GetNode("bar_2");
+
+  EXPECT_TRUE(graph.UpdateFanouts(bar_2->name(), new_bar->name()));
 
   // Fanout nodes must have their inputs updated.
   NodeDef* foo_1 = graph.GetNode("foo_1");
@@ -63,7 +113,7 @@ TEST(MutableGraphViewTest, AddAndUpdateFanouts) {
 
   // And fanouts mapping must be also updated for both nodes.
   bool include_control_fanouts = true;
-  auto old_node_fanouts = graph.GetFanouts(*bar, include_control_fanouts);
+  auto old_node_fanouts = graph.GetFanouts(*bar_2, include_control_fanouts);
   auto new_node_fanouts = graph.GetFanouts(*new_bar, include_control_fanouts);
 
   EXPECT_TRUE(old_node_fanouts.empty());
@@ -78,7 +128,8 @@ TEST(MutableGraphViewTest, AddAndUpdateFanoutsWithoutSelfLoops) {
   // Actual node.op() is not important in this test.
   GraphDef graph_def =
       test::function::GDef({NDef("bar", "NotImportant", {}, {}),
-                            NDef("foo", "NotImportant", {"bar", "^bar"})},
+                            NDef("foo_1", "NotImportant", {"bar", "^bar"}),
+                            NDef("foo_2", "NotImportant", {"^bar"})},
                            /*funcs=*/{});
 
   MutableGraphView graph(&graph_def);
@@ -87,14 +138,18 @@ TEST(MutableGraphViewTest, AddAndUpdateFanoutsWithoutSelfLoops) {
   NodeDef* new_bar = graph.AddNode(NDef("new_bar", "NewBar", {"bar"}, {}));
   NodeDef* bar = graph.GetNode("bar");
 
-  graph.UpdateFanouts("bar", new_bar->name());
+  EXPECT_TRUE(graph.UpdateFanouts("bar", new_bar->name()));
 
   // Foo node must read from `new_bar`.
-  NodeDef* foo = graph.GetNode("foo");
-  ASSERT_NE(foo, nullptr);
-  ASSERT_EQ(foo->input_size(), 2);
-  EXPECT_EQ(foo->input(0), "new_bar");
-  EXPECT_EQ(foo->input(1), "^new_bar");
+  NodeDef* foo_1 = graph.GetNode("foo_1");
+  ASSERT_NE(foo_1, nullptr);
+  ASSERT_EQ(foo_1->input_size(), 1);
+  EXPECT_EQ(foo_1->input(0), "new_bar");
+
+  NodeDef* foo_2 = graph.GetNode("foo_2");
+  ASSERT_NE(foo_2, nullptr);
+  ASSERT_EQ(foo_2->input_size(), 1);
+  EXPECT_EQ(foo_2->input(0), "^new_bar");
 
   // And the `new_bar` should read from the original `bar`.
   ASSERT_EQ(new_bar->input_size(), 1);
@@ -109,8 +164,8 @@ TEST(MutableGraphViewTest, AddAndUpdateFanoutsWithoutSelfLoops) {
   EXPECT_EQ(bar_fanouts.count(MutableGraphView::InputPort(new_bar, 0)), 1);
 
   EXPECT_EQ(new_bar_fanouts.size(), 2);
-  EXPECT_EQ(new_bar_fanouts.count(MutableGraphView::InputPort(foo, 0)), 1);
-  EXPECT_EQ(new_bar_fanouts.count(MutableGraphView::InputPort(foo, -1)), 1);
+  EXPECT_EQ(new_bar_fanouts.count(MutableGraphView::InputPort(foo_1, 0)), 1);
+  EXPECT_EQ(new_bar_fanouts.count(MutableGraphView::InputPort(foo_2, -1)), 1);
 }
 
 GraphDef SimpleMutateFaninGraph() {
@@ -176,7 +231,7 @@ TEST(MutableGraphViewTest, AddFanin) {
   expected_node = NDef("", "", {"b", "a:1", "a:1", "b:2"});
   TestAddFanin("foo_3", {"b", 2}, /*modified=*/true, &expected_node);
   // Add input to node with 1 input multiple controls.
-  expected_node = NDef("", "", {"b", "a", "^c", "^a"});
+  expected_node = NDef("", "", {"b", "a", "^c"});
   TestAddFanin("foo_2", {"a", 0}, /*modified=*/true, &expected_node);
   // Add input to node with multiple inputs and controls.
   expected_node = NDef("", "", {"a", "b:2", "b:2", "a:1", "^d", "^c"});
@@ -201,8 +256,8 @@ TEST(MutableGraphViewTest, AddFanin) {
   TestAddFanin("foo_2", {"d", Graph::kControlSlot}, /*modified=*/true,
                &expected_node);
   // Add control to node with multiple input multiple controls.
-  expected_node = NDef("", "", {"a", "b:2", "b:2", "^c", "^d", "^a"});
-  TestAddFanin("foo_4", {"a", Graph::kControlSlot}, /*modified=*/true,
+  expected_node = NDef("", "", {"a", "b:2", "b:2", "^c", "^d"});
+  TestAddFanin("foo_4", {"a", Graph::kControlSlot}, /*modified=*/false,
                &expected_node);
   // Add control to node with 0 inputs 0 controls.
   expected_node = NDef("", "", {"^a"});
@@ -431,7 +486,7 @@ TEST(MutableGraphViewTest, UpdateFanin) {
   TestUpdateFanin("foo_4", {"d", Graph::kControlSlot}, {"d", 1},
                   /*modified=*/true, &expected_node);
   // Update fanin from control to control.
-  expected_node = NDef("", "", {"a", "b:2", "b:2", "^d", "^b"});
+  expected_node = NDef("", "", {"a", "b:2", "b:2", "^d"});
   TestUpdateFanin("foo_4", {"c", Graph::kControlSlot},
                   {"b", Graph::kControlSlot}, /*modified=*/true,
                   &expected_node);
@@ -463,85 +518,266 @@ TEST(MutableGraphViewTest, UpdateFanin) {
                   /*modified=*/false, /*expected_node=*/nullptr);
 }
 
-GraphDef SimpleDuplicateControllingFaninsGraph() {
+TEST(MutableGraphViewTest, DedupControllingFaninsOnGraphInit) {
+  // Actual node.op() is not important in this test.
+  GraphDef graph_def = test::function::GDef(
+      {
+          NDef("a", "NotImportant", {}, {}),
+          NDef("b", "NotImportant", {}, {}),
+          NDef("c", "Switch", {}, {}),
+          NDef("d", "Identity", {"c:1"}),
+          NDef("foo_1", "IdentityN", {"a", "b:1", "^b"}),
+          NDef("foo_2", "IdentityN", {"a", "^b", "^b"}),
+          NDef("foo_3", "IdentityN", {"a", "b:1", "^b", "^b"}),
+          NDef("foo_4", "IdentityN", {"a:2", "b:1", "^b", "^b", "^a", "^a"}),
+          NDef("foo_5", "NotImportant", {"a:2", "b:1", "^b", "^b", "^a", "^a"}),
+          NDef("foo_6", "Identity", {"d", "^d"}),
+          NDef("foo_7", "NotImportant",
+               {"a:3", "b:2", "d", "^d", "^d", "^a", "^b", "^a", "^b"}),
+      },
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_EQ(graph.graph()->node_size(), 11);
+  NodeDef* a = graph.GetNode("a");
+  ASSERT_NE(a, nullptr);
+  ASSERT_EQ(a->input_size(), 0);
+  NodeDef* b = graph.GetNode("b");
+  ASSERT_NE(b, nullptr);
+  ASSERT_EQ(b->input_size(), 0);
+  NodeDef* c = graph.GetNode("c");
+  ASSERT_NE(c, nullptr);
+  ASSERT_EQ(c->input_size(), 0);
+  NodeDef* d = graph.GetNode("d");
+  ASSERT_NE(d, nullptr);
+  ASSERT_EQ(d->input_size(), 1);
+  EXPECT_EQ(d->input(0), "c:1");
+  NodeDef* foo_1 = graph.GetNode("foo_1");
+  ASSERT_NE(foo_1, nullptr);
+  ASSERT_EQ(foo_1->input_size(), 2);
+  EXPECT_EQ(foo_1->input(0), "a");
+  EXPECT_EQ(foo_1->input(1), "b:1");
+  NodeDef* foo_2 = graph.GetNode("foo_2");
+  ASSERT_NE(foo_2, nullptr);
+  ASSERT_EQ(foo_2->input_size(), 2);
+  EXPECT_EQ(foo_2->input(0), "a");
+  EXPECT_EQ(foo_2->input(1), "^b");
+  NodeDef* foo_3 = graph.GetNode("foo_3");
+  ASSERT_NE(foo_3, nullptr);
+  ASSERT_EQ(foo_3->input_size(), 2);
+  EXPECT_EQ(foo_3->input(0), "a");
+  EXPECT_EQ(foo_3->input(1), "b:1");
+  NodeDef* foo_4 = graph.GetNode("foo_4");
+  ASSERT_NE(foo_4, nullptr);
+  ASSERT_EQ(foo_4->input_size(), 2);
+  EXPECT_EQ(foo_4->input(0), "a:2");
+  EXPECT_EQ(foo_4->input(1), "b:1");
+  NodeDef* foo_5 = graph.GetNode("foo_5");
+  ASSERT_NE(foo_5, nullptr);
+  ASSERT_EQ(foo_5->input_size(), 2);
+  EXPECT_EQ(foo_5->input(0), "a:2");
+  EXPECT_EQ(foo_5->input(1), "b:1");
+  NodeDef* foo_6 = graph.GetNode("foo_6");
+  ASSERT_NE(foo_6, nullptr);
+  ASSERT_EQ(foo_6->input_size(), 2);
+  EXPECT_EQ(foo_6->input(0), "d");
+  EXPECT_EQ(foo_6->input(1), "^d");
+  NodeDef* foo_7 = graph.GetNode("foo_7");
+  ASSERT_NE(foo_7, nullptr);
+  ASSERT_EQ(foo_7->input_size(), 4);
+  EXPECT_EQ(foo_7->input(0), "a:3");
+  EXPECT_EQ(foo_7->input(1), "b:2");
+  EXPECT_EQ(foo_7->input(2), "d");
+  EXPECT_EQ(foo_7->input(3), "^d");
+}
+
+TEST(MutableGraphViewTest, DedupControllingFaninsOnAddFanin) {
+  // Actual node.op() is not important in this test.
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {"^a"}),
+       NDef("c", "NotImportant", {"a:1"})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_TRUE(graph.AddFanin("b", {"a", 2}));
+  NodeDef* b = graph.GetNode("b");
+  ASSERT_NE(b, nullptr);
+  ASSERT_EQ(b->input_size(), 1);
+  EXPECT_EQ(b->input(0), "a:2");
+
+  EXPECT_FALSE(graph.AddFanin("c", {"a", Graph::kControlSlot}));
+  NodeDef* c = graph.GetNode("c");
+  ASSERT_NE(c, nullptr);
+  ASSERT_EQ(c->input_size(), 1);
+  EXPECT_EQ(c->input(0), "a:1");
+}
+
+TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnAddFanin) {
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "Switch", {}, {}), NDef("b", "Identity", {"a:1"}),
+       NDef("c", "", {}, {}), NDef("d", "", {}, {})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_TRUE(graph.AddFanin("c", {"b", 2}));
+  NodeDef* c = graph.GetNode("c");
+  ASSERT_NE(c, nullptr);
+  ASSERT_EQ(c->input_size(), 1);
+  EXPECT_EQ(c->input(0), "b:2");
+  EXPECT_TRUE(graph.AddFanin("c", {"b", Graph::kControlSlot}));
+  ASSERT_EQ(c->input_size(), 2);
+  EXPECT_EQ(c->input(0), "b:2");
+  EXPECT_EQ(c->input(1), "^b");
+  EXPECT_FALSE(graph.AddFanin("c", {"b", Graph::kControlSlot}));
+  ASSERT_EQ(c->input_size(), 2);
+  EXPECT_EQ(c->input(0), "b:2");
+  EXPECT_EQ(c->input(1), "^b");
+
+  EXPECT_TRUE(graph.AddFanin("d", {"b", Graph::kControlSlot}));
+  NodeDef* d = graph.GetNode("d");
+  ASSERT_NE(d, nullptr);
+  ASSERT_EQ(d->input_size(), 1);
+  EXPECT_EQ(d->input(0), "^b");
+  EXPECT_FALSE(graph.AddFanin("d", {"b", Graph::kControlSlot}));
+  ASSERT_EQ(d->input_size(), 1);
+  EXPECT_EQ(d->input(0), "^b");
+  EXPECT_TRUE(graph.AddFanin("d", {"b", 3}));
+  ASSERT_EQ(d->input_size(), 2);
+  EXPECT_EQ(d->input(0), "b:3");
+  EXPECT_EQ(d->input(1), "^b");
+}
+
+TEST(MutableGraphViewTest, DedupControllingFaninsOnUpdateFanin) {
   // Actual node.op() is not important in this test.
   GraphDef graph_def = test::function::GDef(
       {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {}, {}),
-       NDef("foo_1", "NotImportant", {"a", "b:1", "^b"}),
-       NDef("foo_2", "NotImportant", {"a", "^b", "^b"}),
-       NDef("foo_3", "NotImportant", {"a", "b:1", "^b", "^b"}),
-       NDef("foo_4", "NotImportant", {"a:2", "b:1", "^b", "^b", "^a", "^a"})},
+       NDef("c", "NotImportant", {"a:1", "^b"})},
       /*funcs=*/{});
-  return graph_def;
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_TRUE(graph.UpdateFanin("c", {"a", 1}, {"b", 2}));
+  NodeDef* c = graph.GetNode("c");
+  ASSERT_NE(c, nullptr);
+  ASSERT_EQ(c->input_size(), 1);
+  EXPECT_EQ(c->input(0), "b:2");
 }
 
-void CheckDedupControllingFaninsForNode(MutableGraphView* graph,
-                                        absl::string_view node_name,
-                                        const NodeDef* expected_node) {
-  // Deduping again should result in no change.
-  EXPECT_FALSE(graph->DedupControllingFanins(node_name));
-  NodeDef* node = graph->GetNode(node_name);
-  ASSERT_NE(node, nullptr);
-  ASSERT_EQ(node->input_size(), expected_node->input_size());
-  CompareNodeInputs(*graph, expected_node, node);
-  for (int i = 0; i < node->input_size(); ++i) {
-    TensorId tensor_id = ParseTensorName(node->input(i));
-    if (tensor_id.index() > Graph::kControlSlot) {
-      CheckFanout(*graph, {tensor_id.node(), Graph::kControlSlot}, node_name);
-    }
-  }
+TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnUpdateFanin) {
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "Switch", {}, {}), NDef("b", "Identity", {"a:1"}),
+       NDef("c", "Identity", {"a:2"}), NDef("d", "NotImportant", {"c", "^b"}),
+       NDef("e", "NotImportant", {"b", "^c"})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_TRUE(graph.UpdateFanin("d", {"b", Graph::kControlSlot},
+                                {"c", Graph::kControlSlot}));
+  NodeDef* d = graph.GetNode("d");
+  ASSERT_NE(d, nullptr);
+  ASSERT_EQ(d->input_size(), 2);
+  EXPECT_EQ(d->input(0), "c");
+  EXPECT_EQ(d->input(1), "^c");
+
+  EXPECT_TRUE(graph.UpdateFanin("e", {"b", 0}, {"c", 3}));
+  NodeDef* e = graph.GetNode("e");
+  ASSERT_NE(e, nullptr);
+  ASSERT_EQ(e->input_size(), 2);
+  EXPECT_EQ(e->input(0), "c:3");
+  EXPECT_EQ(e->input(1), "^c");
+
+  EXPECT_TRUE(graph.UpdateFanin("e", {"c", 3}, {"c", Graph::kControlSlot}));
+  ASSERT_NE(e, nullptr);
+  ASSERT_EQ(e->input_size(), 1);
+  EXPECT_EQ(e->input(0), "^c");
+}
+
+TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnAddFanin) {
+  // Actual node.op() is not important in this test.
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {"a:1"}),
+       NDef("c", "NotImportant", {"^b"})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_TRUE(graph.AddFanin("c", {"a", 3}));
+  NodeDef* a = graph.GetNode("a");
+  ASSERT_NE(a, nullptr);
+  auto fanouts = graph.GetFanouts(*a, /*include_controlled_nodes=*/true);
+  EXPECT_EQ(fanouts.size(), 2);
+  NodeDef* b = graph.GetNode("b");
+  ASSERT_NE(b, nullptr);
+  EXPECT_EQ(fanouts.count(MutableGraphView::InputPort(b, 0)), 1);
+  NodeDef* c = graph.GetNode("c");
+  ASSERT_NE(c, nullptr);
+  EXPECT_EQ(fanouts.count(MutableGraphView::InputPort(c, 0)), 1);
 }
 
-void TestDedupControllingFaninsForNode(MutableGraphView* graph,
-                                       absl::string_view node_name,
-                                       const NodeDef* expected_node) {
-  EXPECT_TRUE(graph->DedupControllingFanins(node_name));
-  CheckDedupControllingFaninsForNode(graph, node_name, expected_node);
+TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnRemoveFanin) {
+  // Actual node.op() is not important in this test.
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {"a:1"}),
+       NDef("c", "NotImportant", {"a:2"})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_TRUE(graph.RemoveFanin("c", {"a", 2}));
+  NodeDef* a = graph.GetNode("a");
+  ASSERT_NE(a, nullptr);
+  auto fanouts = graph.GetFanouts(*a, /*include_controlled_nodes=*/true);
+  EXPECT_EQ(fanouts.size(), 1);
+  NodeDef* b = graph.GetNode("b");
+  ASSERT_NE(b, nullptr);
+  EXPECT_EQ(fanouts.count(MutableGraphView::InputPort(b, 0)), 1);
 }
 
-TEST(MutableGraphViewTest, DedupControllingFaninsForNode) {
-  GraphDef graph_def = SimpleDuplicateControllingFaninsGraph();
+TEST(MutableGraphViewTest, KeepMaxRegularOutputPortOnRemoveFanin) {
+  // Actual node.op() is not important in this test.
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {"a:1"}),
+       NDef("c", "NotImportant", {"a:2"})},
+      /*funcs=*/{});
 
   MutableGraphView graph(&graph_def);
 
-  NodeDef expected_node;
-  // Remove redundant control dependency '^b'.
-  expected_node = NDef("", "", {"a", "b:1"});
-  TestDedupControllingFaninsForNode(&graph, "foo_1", &expected_node);
-  // Remove extra control dependency '^b'.
-  expected_node = NDef("", "", {"a", "^b"});
-  TestDedupControllingFaninsForNode(&graph, "foo_2", &expected_node);
-  // Remove redundant and extra control dependencies '^b'.
-  expected_node = NDef("", "", {"a", "b:1"});
-  TestDedupControllingFaninsForNode(&graph, "foo_3", &expected_node);
-  // Remove multiple redundant control dependencies.
-  expected_node = NDef("", "", {"a:2", "b:1"});
-  TestDedupControllingFaninsForNode(&graph, "foo_4", &expected_node);
-  // Missing node.
-  EXPECT_FALSE(graph.DedupControllingFanins("missing"));
+  EXPECT_TRUE(graph.RemoveFanin("b", {"a", 1}));
+  NodeDef* a = graph.GetNode("a");
+  ASSERT_NE(a, nullptr);
+  auto fanouts = graph.GetFanouts(*a, /*include_controlled_nodes=*/true);
+  EXPECT_EQ(fanouts.size(), 1);
+  NodeDef* c = graph.GetNode("c");
+  ASSERT_NE(c, nullptr);
+  EXPECT_EQ(fanouts.count(MutableGraphView::InputPort(c, 0)), 1);
 }
 
-TEST(MutableGraphViewTest, DedupControllingFaninsForGraph) {
-  GraphDef graph_def = SimpleDuplicateControllingFaninsGraph();
+TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnUpdateFanin) {
+  // Actual node.op() is not important in this test.
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {"a:1"}),
+       NDef("c", "NotImportant", {"a:2"})},
+      /*funcs=*/{});
 
   MutableGraphView graph(&graph_def);
-  EXPECT_TRUE(graph.DedupControllingFanins());
-  // Deduping again should result in no change.
-  EXPECT_FALSE(graph.DedupControllingFanins());
 
-  NodeDef expected_node;
-  // Remove redundant control dependency '^b'.
-  expected_node = NDef("", "", {"a", "b:1"});
-  CheckDedupControllingFaninsForNode(&graph, "foo_1", &expected_node);
-  // Remove extra control dependency '^b'.
-  expected_node = NDef("", "", {"a", "^b"});
-  CheckDedupControllingFaninsForNode(&graph, "foo_2", &expected_node);
-  // Remove redundant and extra control dependencies '^b'.
-  expected_node = NDef("", "", {"a", "b:1"});
-  CheckDedupControllingFaninsForNode(&graph, "foo_3", &expected_node);
-  // Remove multiple redundant control dependencies.
-  expected_node = NDef("", "", {"a:2", "b:1"});
-  CheckDedupControllingFaninsForNode(&graph, "foo_4", &expected_node);
+  EXPECT_TRUE(graph.UpdateFanin("c", {"a", 2}, {"b", 3}));
+  NodeDef* a = graph.GetNode("a");
+  ASSERT_NE(a, nullptr);
+  auto a_fanouts = graph.GetFanouts(*a, /*include_controlled_nodes=*/true);
+  EXPECT_EQ(a_fanouts.size(), 1);
+  NodeDef* b = graph.GetNode("b");
+  ASSERT_NE(b, nullptr);
+  EXPECT_EQ(a_fanouts.count(MutableGraphView::InputPort(b, 0)), 1);
+  auto b_fanouts = graph.GetFanouts(*b, /*include_controlled_nodes=*/true);
+  EXPECT_EQ(b_fanouts.size(), 1);
+  NodeDef* c = graph.GetNode("c");
+  ASSERT_NE(c, nullptr);
+  EXPECT_EQ(b_fanouts.count(MutableGraphView::InputPort(c, 0)), 1);
 }
 
 TEST(MutableGraphViewTest, AddControllingFaninMissing) {
@@ -655,7 +891,7 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithNoExistingIdentity) {
 TEST(MutableGraphViewTest, AddControllingFaninSwitchWithExistingAddedIdentity) {
   GraphDef graph_def = test::function::GDef(
       {NDef("a", "NotImportant", {}, {}), NDef("switch", "Switch", {}, {}),
-       NDef("ConstantFoldingCtrl/switch_0", "Identity", {}, {})},
+       NDef("ConstantFoldingCtrl/switch_0", "Identity", {"switch"})},
       /*funcs=*/{});
 
   MutableGraphView graph(&graph_def);
@@ -694,9 +930,8 @@ TEST(MutableGraphViewTest, DeleteNodes) {
   auto bar_fanouts = graph.GetFanouts(*bar, include_control_fanouts);
   auto other_fanouts = graph.GetFanouts(*other, include_control_fanouts);
 
-  EXPECT_EQ(bar_fanouts.size(), 2);
+  EXPECT_EQ(bar_fanouts.size(), 1);
   EXPECT_EQ(bar_fanouts.count(MutableGraphView::InputPort(foo_2, 1)), 1);
-  EXPECT_EQ(bar_fanouts.count(MutableGraphView::InputPort(foo_2, -1)), 1);
 
   EXPECT_EQ(other_fanouts.size(), 1);
   EXPECT_EQ(other_fanouts.count(MutableGraphView::InputPort(foo_2, 0)), 1);
diff --git a/tensorflow/core/grappler/optimizers/data/graph_test_utils.cc b/tensorflow/core/grappler/optimizers/data/graph_test_utils.cc
index 9d8b388a3a..202dfb5ac8 100644
--- a/tensorflow/core/grappler/optimizers/data/graph_test_utils.cc
+++ b/tensorflow/core/grappler/optimizers/data/graph_test_utils.cc
@@ -42,7 +42,7 @@ NodeDef MakeMapAndBatchNode(StringPiece name, StringPiece input_node_name,
                             StringPiece function_name) {
   return test::function::NDef(
       name, "ExperimentalMapAndBatchDataset",
-      {string(input_node_name), "", string(batch_size_node_name),
+      {string(input_node_name), string(batch_size_node_name),
        string(num_parallel_calls_node_name), string(drop_remainder_node_name)},
       {{"f", FunctionDefHelper::FunctionRef(string(function_name))},
        {"Targuments", {}},
@@ -68,7 +68,7 @@ NodeDef MakeParallelInterleaveNode(StringPiece name,
                                    StringPiece function_name, bool sloppy) {
   return test::function::NDef(
       name, "ParallelInterleaveDatasetV2",
-      {string(input_node_name), "", string(cycle_length_node_name),
+      {string(input_node_name), string(cycle_length_node_name),
        string(block_length_node_name), string(num_parallel_calls_node_name)},
       {
           {"f", FunctionDefHelper::FunctionRef(string(function_name))},
diff --git a/tensorflow/core/grappler/optimizers/data/latency_all_edges_test.cc b/tensorflow/core/grappler/optimizers/data/latency_all_edges_test.cc
index d428d04a66..426c1dca5b 100644
--- a/tensorflow/core/grappler/optimizers/data/latency_all_edges_test.cc
+++ b/tensorflow/core/grappler/optimizers/data/latency_all_edges_test.cc
@@ -30,9 +30,9 @@ TEST(LatencyAllEdgesTest, AddLatenciesAfterTensorMapPrefetch) {
   using test::function::NDef;
   GrapplerItem item;
   NodeDef component_node =
-      NDef("component_nodes", "Const", {}, {{"value", 1}, {"dtype", DT_INT32}});
+      NDef("component_node", "Const", {}, {{"value", 1}, {"dtype", DT_INT32}});
   NodeDef from_tensor_node =
-      NDef("from_tensor_nodes", "TensorDataset", {"component_nodes"},
+      NDef("from_tensor_node", "TensorDataset", {"component_node"},
            {{"Toutput_types", {}}, {"output_shapes", {}}});
 
   NodeDef captured_input_node = NDef("captured_input_node", "Const", {},
-- 
GitLab


From db48d17b671623a24dcee7e90f8b18d87f51d009 Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Thu, 3 Jan 2019 18:17:13 -0800
Subject: [PATCH 0168/2345] [XLA] Make the verifier check bitcasts don't change
 the element type

This is what BitcastConvert is for.

PiperOrigin-RevId: 227783600
---
 .../compiler/xla/service/hlo_verifier.cc      |  8 ++++++
 .../compiler/xla/service/hlo_verifier_test.cc | 18 +++++++++++++
 .../while_loop_invariant_code_motion_test.cc  | 27 ++++++++++---------
 3 files changed, 41 insertions(+), 12 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc
index 14e29533c2..9274d42280 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier.cc
@@ -387,6 +387,14 @@ Status ShapeVerifier::HandleReduce(HloInstruction* reduce) {
 
 Status ShapeVerifier::HandleBitcast(HloInstruction* bitcast) {
   TF_RETURN_IF_ERROR(CheckOperandCount(bitcast, 1));
+  // Bitcasts are not allowed to change the element type.
+  if (bitcast->operand(0)->shape().element_type() !=
+      bitcast->shape().element_type()) {
+    return InternalError(
+        "Bitcast can not change the element type from %s to %s",
+        PrimitiveType_Name(bitcast->operand(0)->shape().element_type()),
+        PrimitiveType_Name(bitcast->shape().element_type()));
+  }
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/xla/service/hlo_verifier_test.cc b/tensorflow/compiler/xla/service/hlo_verifier_test.cc
index 91f247a9bb..9b2d884f79 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier_test.cc
@@ -480,5 +480,23 @@ TEST_F(HloVerifierTestLayoutSensitive, ConcatWithLayoutChangeNotAllowed) {
   EXPECT_THAT(status.error_message(),
               HasSubstr("Instruction shouldn't change layouts"));
 }
+
+TEST_F(HloVerifierTest, BitcastCanNotChangeElementType) {
+  const char* const hlo_string = R"(
+  HloModule Module
+
+  ENTRY BitcastCanNotChangeElementType {
+   constant.0 = f32[2] constant({0.0, 0.0})
+   ROOT bitcast = s32[2] bitcast(constant.0)
+  }
+  )";
+  TF_ASSERT_OK_AND_ASSIGN(auto module, ParseHloString(hlo_string));
+
+  auto status = verifier().Run(module.get()).status();
+  ASSERT_FALSE(status.ok());
+  EXPECT_THAT(status.error_message(),
+              HasSubstr("Bitcast can not change the element type"));
+}
+
 }  // namespace
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/while_loop_invariant_code_motion_test.cc b/tensorflow/compiler/xla/service/while_loop_invariant_code_motion_test.cc
index 8e7c4bc882..3587c016b4 100644
--- a/tensorflow/compiler/xla/service/while_loop_invariant_code_motion_test.cc
+++ b/tensorflow/compiler/xla/service/while_loop_invariant_code_motion_test.cc
@@ -299,7 +299,7 @@ TEST_F(WhileLoopInvariantCodeMotionTest, DontHoistBitcastAlone) {
   // bitcast either.
   auto m = CreateNewVerifiedModule();
   auto scalar_s32 = ShapeUtil::MakeShape(S32, {});
-  auto scalar_f32 = ShapeUtil::MakeShape(F32, {});
+  auto effective_scalar_s32 = ShapeUtil::MakeShape(S32, {1});
   auto token_shape = ShapeUtil::MakeTokenShape();
   Shape while_shape =
       ShapeUtil::MakeTupleShape({scalar_s32, scalar_s32, token_shape});
@@ -314,10 +314,12 @@ TEST_F(WhileLoopInvariantCodeMotionTest, DontHoistBitcastAlone) {
         HloInstruction::CreateGetTupleElement(scalar_s32, param, 1));
     HloInstruction* in_token = builder.AddInstruction(
         HloInstruction::CreateGetTupleElement(token_shape, param, 2));
-    HloInstruction* bitcast_inst = builder.AddInstruction(
-        HloInstruction::CreateUnary(scalar_f32, HloOpcode::kBitcast, gte_0));
-    HloInstruction* out_token = builder.AddInstruction(
-        HloInstruction::CreateOutfeed(scalar_f32, bitcast_inst, in_token, ""));
+    HloInstruction* bitcast_inst =
+        builder.AddInstruction(HloInstruction::CreateUnary(
+            effective_scalar_s32, HloOpcode::kBitcast, gte_0));
+    HloInstruction* out_token =
+        builder.AddInstruction(HloInstruction::CreateOutfeed(
+            effective_scalar_s32, bitcast_inst, in_token, ""));
     builder.AddInstruction(
         HloInstruction::CreateTuple({gte_0, gte_1, out_token}));
 
@@ -352,9 +354,9 @@ TEST_F(WhileLoopInvariantCodeMotionTest, HoistBitcastIfNeeded) {
   // The bitcast's user can be hoisted, so hoist the bitcast too.
   auto m = CreateNewVerifiedModule();
   auto scalar_s32 = ShapeUtil::MakeShape(S32, {});
-  auto scalar_f32 = ShapeUtil::MakeShape(F32, {});
-  Shape while_shape =
-      ShapeUtil::MakeTupleShape({scalar_s32, scalar_f32, scalar_f32});
+  auto effective_scalar_s32 = ShapeUtil::MakeShape(S32, {1});
+  Shape while_shape = ShapeUtil::MakeTupleShape(
+      {scalar_s32, effective_scalar_s32, effective_scalar_s32});
 
   HloComputation* while_body = [&]() {
     HloComputation::Builder builder(TestName() + ".while_body");
@@ -363,12 +365,13 @@ TEST_F(WhileLoopInvariantCodeMotionTest, HoistBitcastIfNeeded) {
     HloInstruction* gte_0 = builder.AddInstruction(
         HloInstruction::CreateGetTupleElement(scalar_s32, param, 0));
     HloInstruction* gte_1 = builder.AddInstruction(
-        HloInstruction::CreateGetTupleElement(scalar_f32, param, 1));
-    HloInstruction* bitcast_inst = builder.AddInstruction(
-        HloInstruction::CreateUnary(scalar_f32, HloOpcode::kBitcast, gte_0));
+        HloInstruction::CreateGetTupleElement(effective_scalar_s32, param, 1));
+    HloInstruction* bitcast_inst =
+        builder.AddInstruction(HloInstruction::CreateUnary(
+            effective_scalar_s32, HloOpcode::kBitcast, gte_0));
     HloInstruction* add_inst =
         builder.AddInstruction(HloInstruction::CreateBinary(
-            scalar_f32, HloOpcode::kAdd, bitcast_inst, gte_1));
+            effective_scalar_s32, HloOpcode::kAdd, bitcast_inst, gte_1));
     builder.AddInstruction(
         HloInstruction::CreateTuple({gte_0, gte_1, add_inst}));
 
-- 
GitLab


From 328ca94210ba39e13b51228200c8d2da27246f7f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 18:19:05 -0800
Subject: [PATCH 0169/2345] Refactor ExecuteFlexOp into smaller methods of
 OpNode.

PiperOrigin-RevId: 227783768
---
 tensorflow/lite/delegates/flex/kernel.cc | 145 +++++++++++++----------
 1 file changed, 85 insertions(+), 60 deletions(-)

diff --git a/tensorflow/lite/delegates/flex/kernel.cc b/tensorflow/lite/delegates/flex/kernel.cc
index 7d18c39098..3012241f4f 100644
--- a/tensorflow/lite/delegates/flex/kernel.cc
+++ b/tensorflow/lite/delegates/flex/kernel.cc
@@ -88,10 +88,18 @@ class OpNode {
   const tensorflow::NodeDef& nodedef() const { return nodedef_; }
 
   const std::vector<int>& inputs() const { return inputs_; }
-  std::vector<int>* mutable_inputs() { return &inputs_; }
+  void InitializeInputs(const TfLiteIntArray* inputs) {
+    for (int index : TfLiteIntArrayView(inputs)) {
+      inputs_.push_back(index);
+    }
+  }
 
   const std::vector<int>& outputs() const { return outputs_; }
-  std::vector<int>* mutable_outputs() { return &outputs_; }
+  void InitializeOutputs(const TfLiteIntArray* outputs) {
+    for (int index : TfLiteIntArrayView(outputs)) {
+      outputs_.push_back(index);
+    }
+  }
 
   tensorflow::Status InitializeNodeDef(const void* custom_initial_data,
                                        int custom_initial_data_size) {
@@ -123,6 +131,66 @@ class OpNode {
     return tensorflow::Status::OK();
   }
 
+  // Build thew new EagerOperation. In case of error, the returned 'op' is
+  // guaranteed to be 'nullptr'.
+  tensorflow::Status BuildEagerOp(
+      tensorflow::EagerContext* eager_context,
+      std::unique_ptr<tensorflow::EagerOperation>* op) {
+    op->reset();
+
+    const tensorflow::AttrTypeMap* attr_types;
+    bool is_function = false;
+    TF_RETURN_WITH_CONTEXT_IF_ERROR(
+        tensorflow::AttrTypeMapForOp(name_.c_str(), &attr_types, &is_function),
+        " (while processing attributes of '", name_, "')");
+    if (is_function) {
+      return tensorflow::errors::NotFound(
+          "Operation '", name_,
+          "' is not registered.  (while processing attributes of '", name_,
+          "')");
+    }
+
+    op->reset(new tensorflow::EagerOperation(eager_context, name_.c_str(),
+                                             /*is_function=*/false,
+                                             attr_types));
+    for (const auto& attr : nodedef_.attr()) {
+      (*op)->MutableAttrs()->Set(attr.first, attr.second);
+    }
+
+    return tensorflow::Status::OK();
+  }
+
+  tensorflow::Status BuildEagerInputs(BufferMap* buffer_map,
+                                      tensorflow::EagerOperation* op) {
+    for (int input_index : inputs_) {
+      if (!buffer_map->HasTensor(input_index)) {
+        return tensorflow::errors::Internal(
+            "Cannot read from invalid tensor index ", input_index);
+      }
+      auto* handle = new tensorflow::TensorHandle(
+          buffer_map->GetTensor(input_index), nullptr, nullptr, nullptr);
+      op->AddInput(handle);
+      handle->Unref();
+
+      if (buffer_map->IsForwardable(input_index)) {
+        // Take it out of the map, so Eager/TF can reuse the buffer for an
+        // output tensor of the op.
+        buffer_map->RemoveTensor(input_index);
+      }
+    }
+    return tensorflow::Status::OK();
+  }
+
+  tensorflow::Status PersistEagerOutputs(BufferMap* buffer_map,
+                                         OpOutputs* retvals) {
+    for (int i = 0; i < outputs_.size(); ++i) {
+      const tensorflow::Tensor* tensor = nullptr;
+      TF_RETURN_IF_ERROR(retvals->GetHandle(i)->Tensor(&tensor));
+      buffer_map->SetFromTensorFlow(outputs_[i], *tensor);
+    }
+    return tensorflow::Status::OK();
+  }
+
  private:
   // The name of the TensorFlow op to execute.
   string name_;
@@ -139,60 +207,23 @@ class OpNode {
 // Executes the TensorFlow op given by 'op_name', with the attributes specified
 // in 'nodedef'. Inputs and outputs are given as indices into the 'buffer_map'.
 tensorflow::Status ExecuteFlexOp(tensorflow::EagerContext* eager_context,
-                                 BufferMap* buffer_map, const string& op_name,
-                                 const tensorflow::NodeDef& nodedef,
-                                 const std::vector<int>& inputs,
-                                 const std::vector<int>& outputs) {
-  const tensorflow::AttrTypeMap* attr_types;
-  bool is_function = false;
-  TF_RETURN_WITH_CONTEXT_IF_ERROR(
-      tensorflow::AttrTypeMapForOp(op_name.c_str(), &attr_types, &is_function),
-      " (while processing attributes of '", op_name, "')");
-  if (is_function) {
-    return tensorflow::errors::NotFound(
-        "Operation '", op_name,
-        "' is not registered.  (while processing attributes of '", op_name,
-        "')");
-  }
-  tensorflow::EagerOperation op(eager_context, op_name.c_str(),
-                                /*is_function=*/false, attr_types);
-  for (const auto& attr : nodedef.attr()) {
-    op.MutableAttrs()->Set(attr.first, attr.second);
-  }
+                                 BufferMap* buffer_map, OpNode* node_data) {
+  std::unique_ptr<tensorflow::EagerOperation> op;
+  TF_RETURN_IF_ERROR(node_data->BuildEagerOp(eager_context, &op));
+  TF_RETURN_IF_ERROR(node_data->BuildEagerInputs(buffer_map, op.get()));
 
-  for (int input_index : inputs) {
-    if (!buffer_map->HasTensor(input_index)) {
-      return tensorflow::errors::Internal(
-          "Cannot read from invalid tensor index ", input_index);
-    }
-    auto* handle = new tensorflow::TensorHandle(
-        buffer_map->GetTensor(input_index), nullptr, nullptr, nullptr);
-    op.AddInput(handle);
-    handle->Unref();
-
-    if (buffer_map->IsForwardable(input_index)) {
-      // Take it out of the map, so Eager/TF can reuse the buffer for an output
-      // tensor of the op.
-      buffer_map->RemoveTensor(input_index);
-    }
-  }
-
-  int num_retvals = outputs.size();
+  int num_retvals = node_data->outputs().size();
   OpOutputs retvals(num_retvals);
   TF_RETURN_WITH_CONTEXT_IF_ERROR(
-      EagerExecute(&op, retvals.GetVector(), &num_retvals),
-      " (while executing '", op_name, "' via Eager)");
+      EagerExecute(op.get(), retvals.GetVector(), &num_retvals),
+      " (while executing '", node_data->name(), "' via Eager)");
 
-  if (num_retvals != outputs.size()) {
+  if (num_retvals != node_data->outputs().size()) {
     return tensorflow::errors::Internal(
         "Unexpected number of outputs from EagerExecute");
   }
 
-  for (int i = 0; i < num_retvals; ++i) {
-    const tensorflow::Tensor* tensor = nullptr;
-    TF_RETURN_IF_ERROR(retvals.GetHandle(i)->Tensor(&tensor));
-    buffer_map->SetFromTensorFlow(outputs[i], *tensor);
-  }
+  TF_RETURN_IF_ERROR(node_data->PersistEagerOutputs(buffer_map, &retvals));
 
   return tensorflow::Status::OK();
 }
@@ -247,12 +278,8 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) {
                                          node->custom_initial_data_size);
     if (!status.ok()) break;
 
-    for (auto input_index : TfLiteIntArrayView(node->inputs)) {
-      node_data.mutable_inputs()->push_back(input_index);
-    }
-    for (auto output_index : TfLiteIntArrayView(node->outputs)) {
-      node_data.mutable_outputs()->push_back(output_index);
-    }
+    node_data.InitializeInputs(node->inputs);
+    node_data.InitializeOutputs(node->outputs);
   }
 
   if (ConvertStatus(context, status) != kTfLiteOk) {
@@ -325,7 +352,7 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
 }
 
 TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
-  const auto* op_data = reinterpret_cast<OpData*>(node->user_data);
+  auto* op_data = reinterpret_cast<OpData*>(node->user_data);
   BufferMap* buffer_map = op_data->buffer_map;
   tensorflow::EagerContext* eager_context = op_data->eager_context;
 
@@ -344,14 +371,12 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   }
 
   // Execute the TensorFlow Ops sequentially.
-  for (const auto& node_data : op_data->nodes) {
+  for (OpNode& node_data : op_data->nodes) {
     SCOPED_TAGGED_OPERATOR_PROFILE(
         reinterpret_cast<profiling::Profiler*>(context->profiler),
-        node_data->name().c_str(), node_data->index());
+        node_data.name().c_str(), node_data.index());
 
-    auto status = ExecuteFlexOp(eager_context, buffer_map, node_data.name(),
-                                node_data.nodedef(), node_data.inputs(),
-                                node_data.outputs());
+    auto status = ExecuteFlexOp(eager_context, buffer_map, &node_data);
     TF_LITE_ENSURE_OK(context, ConvertStatus(context, status));
   }
 
-- 
GitLab


From d0ce85e772fa17afcc7914c63bcab1b90df6664e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 18:26:08 -0800
Subject: [PATCH 0170/2345] Internal change.

PiperOrigin-RevId: 227784390
---
 tensorflow/compiler/xla/service/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index eed958dc6b..b84792cfc3 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -3576,6 +3576,7 @@ cc_library(
 tf_cc_test(
     name = "indexed_array_analysis_test",
     srcs = ["indexed_array_analysis_test.cc"],
+    extra_copts = ["-Wno-string-plus-int"],
     deps = [
         ":hlo_matchers",
         ":indexed_array_analysis",
-- 
GitLab


From a2f7f39d982682fa8de050e001522581570c510f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 19:50:14 -0800
Subject: [PATCH 0171/2345] Add support non-stacking(cross-links) but connected
 to other bidi-lstm ops case, tensorflow equivalent:
 tf.nn.static_bidirectional_rnn

PiperOrigin-RevId: 227791656
---
 .../kernels/bidirectional_sequence_lstm.cc    |  54 +++--
 .../bidirectional_sequence_lstm_test.cc       | 202 +++++++++++++++++-
 2 files changed, 233 insertions(+), 23 deletions(-)

diff --git a/tensorflow/lite/kernels/bidirectional_sequence_lstm.cc b/tensorflow/lite/kernels/bidirectional_sequence_lstm.cc
index 1ddfe7201e..31c6e3f44c 100644
--- a/tensorflow/lite/kernels/bidirectional_sequence_lstm.cc
+++ b/tensorflow/lite/kernels/bidirectional_sequence_lstm.cc
@@ -105,7 +105,10 @@ constexpr int kBwInputActivationStateTensor = 37;
 // Cell state tensors of size {n_batch, n_cell}
 constexpr int kBwInputCellStateTensor = 38;
 
-// Auxiliary input and weights when stacking.
+// Used as auxiliary input and weights when stacking for
+// tf.contrib.rnn.stack_bidirectional_rnn case (with cross links); Used as input
+// to the backward cell when stacking for tf.nn.static_bidirectional_rnn case
+// (without cross links).
 constexpr int kAuxInputTensor = 39;  // Optional
 // Forward weights.
 constexpr int kFwAuxInputToInputWeightsTensor = 40;   // Optional
@@ -459,8 +462,8 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   const TfLiteTensor* bw_aux_input_to_output_weights =
       GetOptionalInputTensor(context, node, kBwAuxInputToOutputWeightsTensor);
 
-  const bool aux_inputs_all_or_none =
-      ((aux_input != nullptr) && (fw_aux_input_to_cell_weights != nullptr) &&
+  const bool aux_inputs_weights_all_or_none =
+      ((fw_aux_input_to_cell_weights != nullptr) &&
        (fw_aux_input_to_forget_weights != nullptr) &&
        (fw_aux_input_to_output_weights != nullptr) &&
        (bw_aux_input_to_cell_weights != nullptr) &&
@@ -472,8 +475,9 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
        (bw_aux_input_to_cell_weights == nullptr) &&
        (bw_aux_input_to_forget_weights == nullptr) &&
        (bw_aux_input_to_output_weights == nullptr));
-  TF_LITE_ENSURE(context, aux_inputs_all_or_none);
-  const bool has_aux_input = (aux_input != nullptr);
+  TF_LITE_ENSURE(context, aux_inputs_weights_all_or_none);
+
+  const bool has_aux_input = (fw_aux_input_to_forget_weights != nullptr);
 
   if (has_aux_input) {
     // Check that aux_input has the same dimensions (except last) as the input.
@@ -870,6 +874,9 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   const TfLiteTensor* bw_aux_input_to_output_weights =
       GetOptionalInputTensor(context, node, kBwAuxInputToOutputWeightsTensor);
 
+  const bool has_previous_bw_output = (aux_input != nullptr);
+  const bool use_aux_input = (fw_aux_input_to_forget_weights != nullptr);
+
   // Populate a TfLiteLSTMParams struct for the evaluation functions.
   TfLiteLSTMParams lstm_params = {params->activation, params->cell_clip,
                                   params->proj_clip, kTfLiteLSTMFullKernel};
@@ -879,6 +886,26 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   const auto actual_bw_output = params->merge_outputs ? fw_output : bw_output;
 
   const bool time_major = params->time_major;
+
+  // We want to cover the following cases:
+  //
+  // If not stacking (not connected after other bidi lstms):
+  //   both fw & bw will just use `input`; aux_input will be null.
+  //
+  // If stacking with cross_links, TensorFlow equivalent
+  // (tf.contrib.rnn.stack_bidirectional_rnn):
+  //   both fw & bw will use `input`, but aux_input will be none null.
+  //   Note, this time, whether connected after other bidi lstms both works.
+  //
+  // If stacking without cross_links, but connected after other bidi lstms,
+  // TensorFlow equivalent (tf.nn.static_bidirectional_rnn):
+  //   fw will use `input`, bw will use aux_input, and the `real aux_input`
+  //   will be null.
+
+  const bool non_stacking_mode = !use_aux_input && has_previous_bw_output;
+  const TfLiteTensor* bw_input = non_stacking_mode ? aux_input : input;
+  const TfLiteTensor* real_aux_input = non_stacking_mode ? nullptr : aux_input;
+
   switch (fw_input_to_output_weights->type) {
     case kTfLiteFloat32: {
       TfLiteStatus fw_pass_status = lstm_eval::EvalFloat(
@@ -891,7 +918,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
           /*input_layer_norm_coefficients=*/nullptr,
           /*forget_layer_norm_coefficients=*/nullptr,
           /*cell_layer_norm_coefficients=*/nullptr,
-          /*output_layer_norm_coefficients=*/nullptr, aux_input,
+          /*output_layer_norm_coefficients=*/nullptr, real_aux_input,
           fw_aux_input_to_input_weights, fw_aux_input_to_forget_weights,
           fw_aux_input_to_cell_weights, fw_aux_input_to_output_weights,
           fw_input_gate_bias, fw_forget_gate_bias, fw_cell_bias,
@@ -902,7 +929,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
       TF_LITE_ENSURE_OK(context, fw_pass_status);
 
       TfLiteStatus bw_pass_status = lstm_eval::EvalFloat(
-          input, bw_input_to_input_weights, bw_input_to_forget_weights,
+          bw_input, bw_input_to_input_weights, bw_input_to_forget_weights,
           bw_input_to_cell_weights, bw_input_to_output_weights,
           bw_recurrent_to_input_weights, bw_recurrent_to_forget_weights,
           bw_recurrent_to_cell_weights, bw_recurrent_to_output_weights,
@@ -911,7 +938,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
           /*input_layer_norm_coefficients=*/nullptr,
           /*forget_layer_norm_coefficients=*/nullptr,
           /*cell_layer_norm_coefficients=*/nullptr,
-          /*output_layer_norm_coefficients=*/nullptr, aux_input,
+          /*output_layer_norm_coefficients=*/nullptr, real_aux_input,
           bw_aux_input_to_input_weights, bw_aux_input_to_forget_weights,
           bw_aux_input_to_cell_weights, bw_aux_input_to_output_weights,
           bw_input_gate_bias, bw_forget_gate_bias, bw_cell_bias,
@@ -942,9 +969,8 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
       TfLiteTensor* recovered_cell_weights =
           GetTemporary(context, node, kRecoveredCellWeights);
       TfLiteTensor* aux_input_quantized =
-          (aux_input == nullptr)
-              ? nullptr
-              : GetTemporary(context, node, kAuxInputQuantized);
+          use_aux_input ? GetTemporary(context, node, kAuxInputQuantized)
+                        : nullptr;
 
       TfLiteStatus fw_pass_status = lstm_eval::EvalHybrid(
           input, fw_input_to_input_weights, fw_input_to_forget_weights,
@@ -956,7 +982,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
           /*input_layer_norm_coefficients=*/nullptr,
           /*forget_layer_norm_coefficients=*/nullptr,
           /*cell_layer_norm_coefficients=*/nullptr,
-          /*output_layer_norm_coefficients=*/nullptr, aux_input,
+          /*output_layer_norm_coefficients=*/nullptr, real_aux_input,
           fw_aux_input_to_input_weights, fw_aux_input_to_forget_weights,
           fw_aux_input_to_cell_weights, fw_aux_input_to_output_weights,
           fw_input_gate_bias, fw_forget_gate_bias, fw_cell_bias,
@@ -970,7 +996,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
       TF_LITE_ENSURE_OK(context, fw_pass_status);
 
       TfLiteStatus bw_pass_status = lstm_eval::EvalHybrid(
-          input, bw_input_to_input_weights, bw_input_to_forget_weights,
+          bw_input, bw_input_to_input_weights, bw_input_to_forget_weights,
           bw_input_to_cell_weights, bw_input_to_output_weights,
           bw_recurrent_to_input_weights, bw_recurrent_to_forget_weights,
           bw_recurrent_to_cell_weights, bw_recurrent_to_output_weights,
@@ -979,7 +1005,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
           /*input_layer_norm_coefficients=*/nullptr,
           /*forget_layer_norm_coefficients=*/nullptr,
           /*cell_layer_norm_coefficients=*/nullptr,
-          /*output_layer_norm_coefficients=*/nullptr, aux_input,
+          /*output_layer_norm_coefficients=*/nullptr, real_aux_input,
           bw_aux_input_to_input_weights, bw_aux_input_to_forget_weights,
           bw_aux_input_to_cell_weights, bw_aux_input_to_output_weights,
           bw_input_gate_bias, bw_forget_gate_bias, bw_cell_bias,
diff --git a/tensorflow/lite/kernels/bidirectional_sequence_lstm_test.cc b/tensorflow/lite/kernels/bidirectional_sequence_lstm_test.cc
index f5df6d15af..59ea47a2a2 100644
--- a/tensorflow/lite/kernels/bidirectional_sequence_lstm_test.cc
+++ b/tensorflow/lite/kernels/bidirectional_sequence_lstm_test.cc
@@ -38,7 +38,7 @@ class BidirectionalLSTMOpModel : public SingleOpModel {
                            int sequence_length, bool use_cifg,
                            bool use_peephole, bool use_projection_weights,
                            bool use_projection_bias, bool merge_outputs,
-                           float cell_clip, float proj_clip,
+                           bool use_aux_input, float cell_clip, float proj_clip,
                            bool quantize_weights, bool time_major,
                            const std::vector<std::vector<int>>& input_shapes)
       : n_batch_(n_batch),
@@ -185,7 +185,11 @@ class BidirectionalLSTMOpModel : public SingleOpModel {
       bw_output_ = AddOutput(TensorType_FLOAT32);
     }
 
-    aux_input_ = AddNullInput();
+    if (use_aux_input) {
+      aux_input_ = AddInput(TensorType_FLOAT32);
+    } else {
+      aux_input_ = AddNullInput();
+    }
     fw_aux_input_to_input_weights_ = AddNullInput();
     fw_aux_input_to_forget_weights_ = AddNullInput();
     fw_aux_input_to_cell_weights_ = AddNullInput();
@@ -302,6 +306,10 @@ class BidirectionalLSTMOpModel : public SingleOpModel {
     PopulateTensor(input_, offset, begin, end);
   }
 
+  void SetAuxInput(int offset, float* begin, float* end) {
+    PopulateTensor(aux_input_, offset, begin, end);
+  }
+
   std::vector<float> GetFwOutput() { return ExtractVector<float>(fw_output_); }
   std::vector<float> GetBwOutput() { return ExtractVector<float>(bw_output_); }
 
@@ -406,7 +414,8 @@ TEST_P(LSTMOpTest, BlackBoxTestNoCifgNoPeepholeNoProjectionNoClipping) {
   BidirectionalLSTMOpModel lstm(
       n_batch, n_input, n_cell, n_output, sequence_length, /*use_cifg=*/false,
       /*use_peephole=*/false, /*use_projection_weights=*/false,
-      /*use_projection_bias=*/false, /*merge_outputs=*/false, /*cell_clip=*/0.0,
+      /*use_projection_bias=*/false, /*merge_outputs=*/false,
+      /*use_aux_input=*/false, /*cell_clip=*/0.0,
       /*proj_clip=*/0.0, quantize_weights, /*time_major=*/true,
       {
           {sequence_length, n_batch, n_input},  // input tensor
@@ -570,7 +579,8 @@ TEST_P(LSTMOpTest, BlackBoxTestMergedOutput) {
   BidirectionalLSTMOpModel lstm(
       n_batch, n_input, n_cell, n_output, sequence_length, /*use_cifg=*/false,
       /*use_peephole=*/false, /*use_projection_weights=*/false,
-      /*use_projection_bias=*/false, /*merge_outputs=*/true, /*cell_clip=*/0.0,
+      /*use_projection_bias=*/false, /*merge_outputs=*/true,
+      /*use_aux_input=*/false, /*cell_clip=*/0.0,
       /*proj_clip=*/0.0, quantize_weights, /*time_major=*/true,
       {
           {sequence_length, n_batch, n_input},  // input tensor
@@ -733,7 +743,8 @@ TEST(LSTMOpTest, BlackBoxTestNoCifgNoPeepholeNoProjectionNoClippingReverse) {
   BidirectionalLSTMOpModel lstm(
       n_batch, n_input, n_cell, n_output, sequence_length, /*use_cifg=*/false,
       /*use_peephole=*/false, /*use_projection_weights=*/false,
-      /*use_projection_bias=*/false, /*merge_outputs=*/false, /*cell_clip=*/0.0,
+      /*use_projection_bias=*/false, /*merge_outputs=*/false,
+      /*use_aux_input=*/false, /*cell_clip=*/0.0,
       /*proj_clip=*/0.0, /*quantize_weights=*/false, /*time_major=*/true,
       {
           {sequence_length, n_batch, n_input},  // input tensor
@@ -895,7 +906,8 @@ TEST(LSTMOpTest, BlackBoxTestWithCifgWithPeepholeNoProjectionNoClipping) {
   BidirectionalLSTMOpModel lstm(
       n_batch, n_input, n_cell, n_output, sequence_length, /*use_cifg=*/true,
       /*use_peephole=*/true, /*use_projection_weights=*/false,
-      /*use_projection_bias=*/false, /*merge_outputs=*/false, /*cell_clip=*/0.0,
+      /*use_projection_bias=*/false, /*merge_outputs=*/false,
+      /*use_aux_input=*/false, /*cell_clip=*/0.0,
       /*proj_clip=*/0.0, /*quantize_weights=*/false, /*time_major=*/true,
       {
           {sequence_length, n_batch, n_input},  // input tensor
@@ -1047,7 +1059,8 @@ TEST(LSTMOpTest,
   BidirectionalLSTMOpModel lstm(
       n_batch, n_input, n_cell, n_output, sequence_length, /*use_cifg=*/true,
       /*use_peephole=*/true, /*use_projection_weights=*/false,
-      /*use_projection_bias=*/false, /*merge_outputs=*/false, /*cell_clip=*/0.0,
+      /*use_projection_bias=*/false, /*merge_outputs=*/false,
+      /*use_aux_input=*/false, /*cell_clip=*/0.0,
       /*proj_clip=*/0.0, /*quantize_weights=*/false, /*time_major=*/true,
       {
           {sequence_length, n_batch, n_input},  // input tensor
@@ -1199,7 +1212,8 @@ TEST(LSTMOpTest, BlackBoxTestWithPeepholeWithProjectionNoClipping) {
   BidirectionalLSTMOpModel lstm(
       n_batch, n_input, n_cell, n_output, sequence_length, /*use_cifg=*/false,
       /*use_peephole=*/true, /*use_projection_weights=*/true,
-      /*use_projection_bias=*/false, /*merge_outputs=*/false, /*cell_clip=*/0.0,
+      /*use_projection_bias=*/false, /*merge_outputs=*/false,
+      /*use_aux_input=*/false, /*cell_clip=*/0.0,
       /*proj_clip=*/0.0, /*quantize_weights=*/false, /*time_major=*/true,
       {
           {sequence_length, n_batch, n_input},  // input tensor
@@ -1903,7 +1917,8 @@ TEST(LSTMOpTest, BlackBoxTestWithPeepholeWithProjectionNoClippingBatchMajor) {
   BidirectionalLSTMOpModel lstm(
       n_batch, n_input, n_cell, n_output, sequence_length, /*use_cifg=*/false,
       /*use_peephole=*/true, /*use_projection_weights=*/true,
-      /*use_projection_bias=*/false, /*merge_outputs=*/false, /*cell_clip=*/0.0,
+      /*use_projection_bias=*/false, /*merge_outputs=*/false,
+      /*use_aux_input=*/false, /*cell_clip=*/0.0,
       /*proj_clip=*/0.0, /*quantize_weights=*/false, /*time_major=*/false,
       {
           {n_batch, sequence_length, n_input},  // input tensor
@@ -2590,6 +2605,175 @@ TEST(LSTMOpTest, BlackBoxTestWithPeepholeWithProjectionNoClippingBatchMajor) {
   EXPECT_THAT(combined, ElementsAreArray(ArrayFloatNear(expected)));
 }
 
+// Same as the no cifg no peephole no projection no clipping test, but have an
+// aux input (without aux input weights), this is the case when stacking but no
+// cross-links.
+TEST_P(LSTMOpTest, BlackBoxTestWithAuxInput) {
+  const int n_batch = 1;
+  const int n_input = 2;
+  // n_cell and n_output have the same size when there is no projection.
+  const int n_cell = 4;
+  const int n_output = 4;
+  const int sequence_length = 3;
+  const bool quantize_weights = GetParam();
+
+  BidirectionalLSTMOpModel lstm(
+      n_batch, n_input, n_cell, n_output, sequence_length, /*use_cifg=*/false,
+      /*use_peephole=*/false, /*use_projection_weights=*/false,
+      /*use_projection_bias=*/false, /*merge_outputs=*/false,
+      /*use_aux_input=*/true, /*cell_clip=*/0.0,
+      /*proj_clip=*/0.0, quantize_weights, /*time_major=*/true,
+      {
+          {sequence_length, n_batch, n_input},  // input tensor
+
+          // Forward cell
+          {n_cell, n_input},  // input_to_input_weight tensor
+          {n_cell, n_input},  // input_to_forget_weight tensor
+          {n_cell, n_input},  // input_to_cell_weight tensor
+          {n_cell, n_input},  // input_to_output_weight tensor
+
+          {n_cell, n_output},  // recurrent_to_input_weight tensor
+          {n_cell, n_output},  // recurrent_to_forget_weight tensor
+          {n_cell, n_output},  // recurrent_to_cell_weight tensor
+          {n_cell, n_output},  // recurrent_to_output_weight tensor
+
+          {0},  // cell_to_input_weight tensor
+          {0},  // cell_to_forget_weight tensor
+          {0},  // cell_to_output_weight tensor
+
+          {n_cell},  // input_gate_bias tensor
+          {n_cell},  // forget_gate_bias tensor
+          {n_cell},  // cell_bias tensor
+          {n_cell},  // output_gate_bias tensor
+
+          {0, 0},  // projection_weight tensor
+          {0},     // projection_bias tensor
+
+          // Backward cell
+          {n_cell, n_input},  // input_to_input_weight tensor
+          {n_cell, n_input},  // input_to_forget_weight tensor
+          {n_cell, n_input},  // input_to_cell_weight tensor
+          {n_cell, n_input},  // input_to_output_weight tensor
+
+          {n_cell, n_output},  // recurrent_to_input_weight tensor
+          {n_cell, n_output},  // recurrent_to_forget_weight tensor
+          {n_cell, n_output},  // recurrent_to_cell_weight tensor
+          {n_cell, n_output},  // recurrent_to_output_weight tensor
+
+          {0},  // cell_to_input_weight tensor
+          {0},  // cell_to_forget_weight tensor
+          {0},  // cell_to_output_weight tensor
+
+          {n_cell},  // input_gate_bias tensor
+          {n_cell},  // forget_gate_bias tensor
+          {n_cell},  // cell_bias tensor
+          {n_cell},  // output_gate_bias tensor
+
+          {0, 0},  // projection_weight tensor
+          {0},     // projection_bias tensor
+
+          {n_batch, n_output},  // activation_state tensor
+          {n_batch, n_cell},    // cell_state tensor
+
+          {n_batch, n_output},  // activation_state tensor
+          {n_batch, n_cell},    // cell_state tensor
+
+          // TODO(b/121134029): Update tests so tensor shapes after state tensor
+          // are used. They are currently ignored by test_util.
+          {sequence_length, n_batch, n_input},  // aux_input tensor
+          {n_cell, 0},                          // aux_fw_input_to_input tensor
+          {n_cell, 0},                          // aux_fw_input_to_forget tensor
+          {n_cell, 0},                          // aux_fw_input_to_cell tensor
+          {n_cell, 0},                          // aux_fw_input_to_output tensor
+          {n_cell, 0},                          // aux_bw_input_to_input tensor
+          {n_cell, 0},                          // aux_bw_input_to_forget tensor
+          {n_cell, 0},                          // aux_bw_input_to_cell tensor
+          {n_cell, 0},                          // aux_bw_input_to_output tensor
+      });
+
+  lstm.SetInputToInputWeights({-0.45018822, -0.02338299, -0.0870589,
+                               -0.34550029, 0.04266912, -0.15680569,
+                               -0.34856534, 0.43890524});
+
+  lstm.SetInputToCellWeights({-0.50013041, 0.1370284, 0.11810488, 0.2013163,
+                              -0.20583314, 0.44344562, 0.22077113,
+                              -0.29909778});
+
+  lstm.SetInputToForgetWeights({0.09701663, 0.20334584, -0.50592935,
+                                -0.31343272, -0.40032279, 0.44781327,
+                                0.01387155, -0.35593212});
+
+  lstm.SetInputToOutputWeights({-0.25065863, -0.28290087, 0.04613829,
+                                0.40525138, 0.44272184, 0.03897077, -0.1556896,
+                                0.19487578});
+
+  lstm.SetInputGateBias({0., 0., 0., 0.});
+
+  lstm.SetCellBias({0., 0., 0., 0.});
+
+  lstm.SetForgetGateBias({1., 1., 1., 1.});
+
+  lstm.SetOutputGateBias({0., 0., 0., 0.});
+
+  lstm.SetRecurrentToInputWeights(
+      {-0.0063535, -0.2042388, 0.31454784, -0.35746509, 0.28902304, 0.08183324,
+       -0.16555229, 0.02286911, -0.13566875, 0.03034258, 0.48091322,
+       -0.12528998, 0.24077177, -0.51332325, -0.33502164, 0.10629296});
+
+  lstm.SetRecurrentToCellWeights(
+      {-0.3407414, 0.24443203, -0.2078532, 0.26320225, 0.05695659, -0.00123841,
+       -0.4744786, -0.35869038, -0.06418842, -0.13502428, -0.501764, 0.22830659,
+       -0.46367589, 0.26016325, -0.03894562, -0.16368064});
+
+  lstm.SetRecurrentToForgetWeights(
+      {-0.48684245, -0.06655136, 0.42224967, 0.2112639, 0.27654213, 0.20864892,
+       -0.07646349, 0.45877004, 0.00141793, -0.14609534, 0.36447752, 0.09196436,
+       0.28053468, 0.01560611, -0.20127171, -0.01140004});
+
+  lstm.SetRecurrentToOutputWeights(
+      {0.43385774, -0.17194885, 0.2718237, 0.09215671, 0.24107647, -0.39835793,
+       0.18212086, 0.01301402, 0.48572797, -0.50656658, 0.20047462, -0.20607421,
+       -0.51818722, -0.15390486, 0.0468148, 0.39922136});
+
+  // Input should have n_input * sequence_length many values.
+  static float lstm_input[] = {2., 3., 3., 4., 1., 1.};
+  static float lstm_fw_golden_output[] = {
+      -0.02973187, 0.1229473,  0.20885126, -0.15358765,
+      -0.03716109, 0.12507336, 0.41193449, -0.20860538,
+      -0.15053082, 0.09120187, 0.24278517, -0.12222792};
+  static float lstm_bw_golden_output[] = {
+      -0.0806187, 0.139077, 0.400476,   -0.197842, -0.0332076, 0.123838,
+      0.309777,   -0.17621, -0.0490733, 0.0739237, 0.067706,   -0.0208124};
+
+  float* batch0_start = lstm_input;
+  float* batch0_end = batch0_start + lstm.num_inputs() * lstm.sequence_length();
+
+  lstm.SetInput(0, batch0_start, batch0_end);
+  // Aux input and input are the same, so we should observe the same outputs
+  // as there's no aux input.
+  lstm.SetAuxInput(0, batch0_start, batch0_end);
+
+  lstm.Invoke();
+
+  float* fw_golden_start = lstm_fw_golden_output;
+  float* fw_golden_end =
+      fw_golden_start + lstm.num_fw_outputs() * lstm.sequence_length();
+  std::vector<float> fw_expected;
+  fw_expected.insert(fw_expected.end(), fw_golden_start, fw_golden_end);
+  EXPECT_THAT(lstm.GetFwOutput(),
+              ElementsAreArray(
+                  ArrayFloatNear(fw_expected, quantize_weights ? 1e-2 : 1e-5)));
+
+  float* bw_golden_start = lstm_bw_golden_output;
+  float* bw_golden_end =
+      bw_golden_start + lstm.num_bw_outputs() * lstm.sequence_length();
+  std::vector<float> bw_expected;
+  bw_expected.insert(bw_expected.end(), bw_golden_start, bw_golden_end);
+  EXPECT_THAT(lstm.GetBwOutput(),
+              ElementsAreArray(
+                  ArrayFloatNear(bw_expected, quantize_weights ? 1e-2 : 1e-5)));
+}
+
 }  // namespace
 }  // namespace tflite
 
-- 
GitLab


From cbb4b0bcd794d40a29196f0670396415c24b2881 Mon Sep 17 00:00:00 2001
From: manhyuk <manhyuk@kw.ac.kr>
Date: Fri, 4 Jan 2019 13:27:45 +0900
Subject: [PATCH 0172/2345] fix typo

---
 tensorflow/contrib/cmake/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/cmake/README.md b/tensorflow/contrib/cmake/README.md
index b2badc5785..b4f4b028f6 100644
--- a/tensorflow/contrib/cmake/README.md
+++ b/tensorflow/contrib/cmake/README.md
@@ -147,7 +147,7 @@ suitable interface for project configuration and dependency setting.
     *   Go (required if you need ssl support, optional)
     *   NASM/YASM (required by grpc for ssl support, optional)
 2.  Start CMake GUI
-3.  Click on `Browse Source` and direct to the the folder
+3.  Click on `Browse Source` and direct to the folder
     `<tensorflow-source>/tensorflow/contrib/cmake`
 4.  Click on `Browse Build` and spectify a location that you want tensorflow to
     be build
-- 
GitLab


From e913092464cc5376343063d7905d491d29808c09 Mon Sep 17 00:00:00 2001
From: manhyuk <manhyuk@kw.ac.kr>
Date: Fri, 4 Jan 2019 13:28:05 +0900
Subject: [PATCH 0173/2345] fix typo

---
 tensorflow/contrib/ignite/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/ignite/README.md b/tensorflow/contrib/ignite/README.md
index 5a8c650fb9..7493ea6759 100644
--- a/tensorflow/contrib/ignite/README.md
+++ b/tensorflow/contrib/ignite/README.md
@@ -30,7 +30,7 @@ system based on Apache Ignite.
 
 ## Features
 
-Ignite Dataset provides features that that you can use in a wide range of cases. The most important and interesting features are described below.
+Ignite Dataset provides features that you can use in a wide range of cases. The most important and interesting features are described below.
 
 ### Distributed In-Memory Datasource
 [Apache Ignite](https://ignite.apache.org/) is a distributed in-memory database, caching, and processing platform that provides fast data access. It allows you to avoid limitations of hard drive and store and operate with as much data as you need in distributed cluster. You can utilize
-- 
GitLab


From f332be115267be60ed7e16fed661555c15f7d990 Mon Sep 17 00:00:00 2001
From: Pariksheet Pinjari <pariksheet.pinjari@huawei.com>
Date: Fri, 4 Jan 2019 12:35:09 +0530
Subject: [PATCH 0174/2345] Update api_def_RegexReplace.pbtxt

Documentation issue fixed
---
 tensorflow/core/api_def/base_api/api_def_RegexReplace.pbtxt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/api_def/base_api/api_def_RegexReplace.pbtxt b/tensorflow/core/api_def/base_api/api_def_RegexReplace.pbtxt
index 70ad521926..2cc1a55676 100644
--- a/tensorflow/core/api_def/base_api/api_def_RegexReplace.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_RegexReplace.pbtxt
@@ -10,7 +10,7 @@ op {
   }
   in_arg {
     name: "rewrite"
-    description: "The rewrite to be applied to the matched expresion."
+    description: "The rewrite to be applied to the matched expression."
   }
   out_arg {
     name: "output"
-- 
GitLab


From 72f0dadfffd8c2f635ec58ded5c38154644c6f3f Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Thu, 3 Jan 2019 23:49:44 -0800
Subject: [PATCH 0175/2345] Automated rollback of commit
 7d57d32e440bc4f1654fb217fd701531a82c3587

PiperOrigin-RevId: 227809861
---
 tensorflow/contrib/mpi_collectives/BUILD                  | 2 +-
 tensorflow/stream_executor/platform/default/dso_loader.cc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/mpi_collectives/BUILD b/tensorflow/contrib/mpi_collectives/BUILD
index d943ae6880..ecac06354d 100644
--- a/tensorflow/contrib/mpi_collectives/BUILD
+++ b/tensorflow/contrib/mpi_collectives/BUILD
@@ -52,7 +52,7 @@ tf_custom_op_library(
     deps = [
         ":mpi_defines",
         ":mpi_message_proto_cc",
-        "//tensorflow/core:stream_executor_headers_lib",
+        "//tensorflow/stream_executor:stream_executor_headers_lib",
         "//third_party/mpi",
     ],
 )
diff --git a/tensorflow/stream_executor/platform/default/dso_loader.cc b/tensorflow/stream_executor/platform/default/dso_loader.cc
index 668eeee3f3..0f0bce3253 100644
--- a/tensorflow/stream_executor/platform/default/dso_loader.cc
+++ b/tensorflow/stream_executor/platform/default/dso_loader.cc
@@ -28,7 +28,7 @@ limitations under the License.
 #include "tensorflow/stream_executor/lib/path.h"
 #include "tensorflow/stream_executor/lib/str_util.h"
 #include "tensorflow/stream_executor/lib/stringprintf.h"
-#include "tensorflow/stream_executor/platform/default/dso_loader.h"
+#include "tensorflow/stream_executor/platform/dso_loader.h"
 #include "tensorflow/stream_executor/platform/logging.h"
 #include "tensorflow/stream_executor/platform/port.h"
 
-- 
GitLab


From 6965d80c135a7c0008d2e17ce74529ab5798a5e4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 01:02:44 -0800
Subject: [PATCH 0176/2345] compat: Update forward compatibility horizon to
 2019-01-04

PiperOrigin-RevId: 227817111
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index c60c42ee65..66c4fa4893 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 3)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 4)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From f76dc80f344440637b0b8ae84754c50bb4868b97 Mon Sep 17 00:00:00 2001
From: Dheeraj Rajaram Reddy <dheeraj98reddy@gmail.com>
Date: Fri, 4 Jan 2019 15:19:33 +0530
Subject: [PATCH 0177/2345] Add api_def for ExperimentalTakeWhileDataset

---
 ...api_def_ExperimentalTakeWhileDataset.pbtxt | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_ExperimentalTakeWhileDataset.pbtxt

diff --git a/tensorflow/core/api_def/base_api/api_def_ExperimentalTakeWhileDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalTakeWhileDataset.pbtxt
new file mode 100644
index 0000000000..699e0c2e39
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_ExperimentalTakeWhileDataset.pbtxt
@@ -0,0 +1,25 @@
+op {
+  graph_op_name: "ExperimentalTakeWhileDataset"
+  visibility: HIDDEN
+  in_arg {
+    name: "other_arguments"
+    description: <<END
+A list of tensors, typically values that were captured when
+building a closure for `predicate`.
+END
+  }
+  attr {
+    name: "predicate"
+    description: <<END
+A function returning a scalar boolean.
+END
+  }
+  summary: "Creates a dataset that stops iteration when predicate` is false."
+  description: <<END
+The `predicate` function must return a scalar boolean and accept the
+following arguments:
+
+* One tensor for each component of an element of `input_dataset`.
+* One tensor for each value in `other_arguments`.
+END
+}
-- 
GitLab


From 2456898958279c1c75ac24da473665f418f2ec94 Mon Sep 17 00:00:00 2001
From: Dheeraj Rajaram Reddy <dheeraj98reddy@gmail.com>
Date: Fri, 4 Jan 2019 15:25:21 +0530
Subject: [PATCH 0178/2345] Format files according to TF style guide and minor
 fixes

---
 .../experimental/take_while_dataset_op.cc     | 32 ++++----
 .../kernel_tests/take_while_test.py           | 82 +++++++++----------
 .../data/experimental/ops/take_while_ops.py   | 40 ++++-----
 3 files changed, 72 insertions(+), 82 deletions(-)

diff --git a/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc b/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
index b18cc32051..cf9c5f0cb6 100644
--- a/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
@@ -40,7 +40,7 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
     std::unique_ptr<CapturedFunction> captured_func;
     OP_REQUIRES_OK(ctx, CapturedFunction::Create(func_, ctx, "other_arguments",
                                                  &captured_func));
-    
+
     // TODO (squadrick): check short-circuit
     *output = new Dataset(ctx, input, func_, std::move(captured_func));
   }
@@ -49,7 +49,7 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
   class Dataset : public DatasetBase {
    public:
     Dataset(OpKernelContext* ctx, const DatasetBase* input,
-            const NameAttrList& func, 
+            const NameAttrList& func,
             std::unique_ptr<CapturedFunction> captured_func)
         : DatasetBase(DatasetContext(ctx)),
           input_(input),
@@ -74,7 +74,9 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
       return input_->output_shapes();
     }
 
-    string DebugString() const override { return "TakeWhileDatasetOp::Dataset"; }
+    string DebugString() const override {
+      return "TakeWhileDatasetOp::Dataset";
+    }
 
     int64 Cardinality() const override { return kUnknownCardinality; }
 
@@ -109,11 +111,11 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
       b->BuildAttrValue(other_arguments_types, &other_arguments_types_attr);
 
       TF_RETURN_IF_ERROR(b->AddDataset(
-            this, {std::make_pair(0, input_node)},
-            {std::make_pair(1, other_arguments)},
-            {std::make_pair("predicate", f_attr),
-             std::make_pair("Targuments", other_arguments_types_attr)},
-             output));
+          this, {std::make_pair(0, input_node)},
+          {std::make_pair(1, other_arguments)},
+          {std::make_pair("predicate", f_attr),
+           std::make_pair("Targuments", other_arguments_types_attr)},
+          output));
       return Status::OK();
     }
 
@@ -135,7 +137,7 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
                              bool* end_of_sequence) override {
         {
           tf_shared_lock l(mu_);
-          if(!input_impl_) {
+          if (!input_impl_) {
             *end_of_sequence = true;
             return Status::OK();
           }
@@ -154,17 +156,15 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
             ctx, *out_tensors, &bool_output);
 
         if (s.ok()) {
-          if(bool_output.size() != 1 || bool_output[0].dtype() != DT_BOOL ||
-             bool_output[0].NumElements() != 1) {
+          if (bool_output.size() != 1 || bool_output[0].dtype() != DT_BOOL ||
+              bool_output[0].NumElements() != 1) {
             return errors::InvalidArgument(
                 "`predicate` must returns a scalar bool tensor.");
           }
-          if (!bool_output[0].scalar<bool>()()) {
-            *end_of_sequence = true;
-            return Status::OK();
-          }
+          *end_of_sequence = !bool_output[0].scalar<bool>()();
+          return Status::OK();
         }
-        return s; // propagate error to caller
+        return s;  // propagate error to caller
       }
 
      protected:
diff --git a/tensorflow/python/data/experimental/kernel_tests/take_while_test.py b/tensorflow/python/data/experimental/kernel_tests/take_while_test.py
index d561a3cd35..b1e7974471 100644
--- a/tensorflow/python/data/experimental/kernel_tests/take_while_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/take_while_test.py
@@ -18,76 +18,70 @@ from __future__ import division
 from __future__ import print_function
 
 import numpy as np
+from absl.testing import parameterized
 
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.experimental.ops import take_while_ops
 from tensorflow.python.framework import errors
-from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import test_util
 from tensorflow.python.framework import constant_op
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import functional_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class TakeWhileTest(test_base.DatasetTestBase):
-
-  def testTakeWhileDataset(self):
-    def do_test(num_elements, window_size):
-      def predicate_func(elem):
-        return array_ops.shape(elem)[0] > (window_size - 1)
-
-      flatten_func = lambda x: dataset_ops.Dataset.from_tensor_slices(x)
-      take_while = take_while_ops.take_while(predicate_func)
-
-      dataset = dataset_ops.Dataset.range(num_elements).batch(window_size)
-      dataset = dataset.apply(take_while).flat_map(flatten_func)
-      
-      self.assertDatasetProduces(dataset, 
-              np.arange(int(num_elements / window_size) * window_size))
-
-    do_test(14, 2)
-    do_test(15, 2)
-    do_test(100, 3)
-
-  def testTakeWhileDatasetRange(self):
-    def get_dataset(num_elemets, upper_bound):
-      return dataset_ops.Dataset.range(num_elemets).apply(
-              take_while_ops.take_while(lambda x: x < upper_bound))
-        
-    def do_test(num_elemets, upper_bound):  
-      self.assertDatasetProduces(get_dataset(num_elemets, upper_bound), 
-              np.arange(upper_bound))
-    
-    def out_of_bounds(num_elemets, upper_bound):
+class TakeWhileTest(test_base.DatasetTestBase, parameterized.TestCase):
+  @parameterized.parameters(
+      (14, 2),
+      (15, 2),
+      (100, 3))
+  def testTakeWhileDataset(self, num_elements, window_size):
+    def _predicate_func(elem):
+      return array_ops.shape(elem)[0] > (window_size - 1)
+
+    take_while = take_while_ops.take_while(_predicate_func)
+
+    dataset = dataset_ops.Dataset.range(num_elements).batch(window_size)
+    dataset = dataset.apply(take_while).flat_map(
+        lambda x: dataset_ops.Dataset.from_tensor_slices(x))
+
+    expected_num_elements = int(num_elements / window_size) * window_size
+    self.assertDatasetProduces(dataset, np.arange(expected_num_elements))
+
+  @parameterized.parameters(
+      (10, 2, False),
+      (16, 7, False),
+      (100, 99, False),
+      (100, 101, True),
+      (0, 1, True))
+  def testTakeWhileDatasetRange(self, num_elements, upper_bound, out_of_bounds):
+    dataset = dataset_ops.Dataset.range(num_elements).apply(
+        take_while_ops.take_while(lambda x: x < upper_bound))
+
+    if out_of_bounds:
       with self.assertRaises(errors.OutOfRangeError):
-        self.assertDatasetProduces(get_dataset(num_elemets, upper_bound), 
-                np.arange(upper_bound))
+        self.assertDatasetProduces(dataset, np.arange(upper_bound))
 
-    do_test(10, 2)
-    do_test(16, 7)
-    do_test(100, 99)
-    out_of_bounds(100, 101)
-    out_of_bounds(0, 1)
+    else:
+      self.assertDatasetProduces(dataset, np.arange(upper_bound))
 
   def testTakeWhileDatasetString(self):
-    def stringNotEquals(string):
-        return lambda x: math_ops.not_equal(x, constant_op.constant(string))
+    def not_equal(string):
+      return lambda x: math_ops.not_equal(x, constant_op.constant(string))
 
     string = ["this", "is", "the", "test", "for", "strings"]
     dataset = dataset_ops.Dataset.from_tensor_slices(string).apply(
-            take_while_ops.take_while(stringNotEquals("test")))
-    
+        take_while_ops.take_while(not_equal("test")))
+
     next_element = self.getNext(dataset)
     self.assertEqual(b"this", self.evaluate(next_element()))
     self.assertEqual(b"is", self.evaluate(next_element()))
     self.assertEqual(b"the", self.evaluate(next_element()))
 
     with self.assertRaises(errors.OutOfRangeError):
-        self.assertEqual(b"test", self.evaluate(next_element()))
+      self.assertEqual(b"test", self.evaluate(next_element()))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/ops/take_while_ops.py b/tensorflow/python/data/experimental/ops/take_while_ops.py
index 224eb8aa77..c9518cc9ad 100644
--- a/tensorflow/python/data/experimental/ops/take_while_ops.py
+++ b/tensorflow/python/data/experimental/ops/take_while_ops.py
@@ -17,12 +17,9 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import collections
-
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import structure as structure_lib
 from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops
 from tensorflow.python.ops import gen_experimental_dataset_ops
 from tensorflow.python.util.tf_export import tf_export
 
@@ -32,28 +29,27 @@ class _TakeWhileDataset(dataset_ops.UnaryUnchangedStructureDataset):
 
   def __init__(self, input_dataset, predicate):
     """See `take_while()` for details."""
-    
+
     self._input_dataset = input_dataset
     wrapped_func = dataset_ops.StructuredFunctionWrapper(
-            predicate,
-            "tf.data.experimental.take_while()",
-            dataset=self._input_dataset)
-    
+        predicate,
+        "tf.data.experimental.take_while()",
+        dataset=self._input_dataset)
+
     if not wrapped_func.output_structure.is_compatible_with(
-            structure_lib.TensorStructure(dtypes.bool, [])):
-        raise ValueError("`predicate` must return a scalar boolean tensor.")
-     
+        structure_lib.TensorStructure(dtypes.bool, [])):
+      raise ValueError("`predicate` must return a scalar boolean tensor.")
+    
     self._predicate = wrapped_func
-
-    variant_tensor = gen_experimental_dataset_ops.experimental_take_while_dataset(
-            self._input_dataset._variant_tensor,
-            other_arguments=self._predicate.function.captured_inputs,
-            predicate=self._predicate.function,
-            **dataset_ops.flat_structure(self))
-    super(_TakeWhileDataset, self).__init__(input_dataset, variant_tensor)
+    var_tensor = gen_experimental_dataset_ops.experimental_take_while_dataset(
+        self._input_dataset._variant_tensor, # pylint: disable=protected-access
+        other_arguments=self._predicate.function.captured_inputs,
+        predicate=self._predicate.function,
+        **dataset_ops.flat_structure(self))
+    super(_TakeWhileDataset, self).__init__(input_dataset, var_tensor)
 
   def _functions(self):
-      return [self._predicate]
+    return [self._predicate]
 
 
 @tf_export("data.experimental.take_while")
@@ -61,8 +57,8 @@ def take_while(predicate):
   """A transformation that stops dataset iteration based on a `predicate` condition
 
   Args:
-    predicate: A function that maps a nested structure of tensors 
-    (having shapes and types defined by `self.output_shapes` and 
+    predicate: A function that maps a nested structure of tensors
+    (having shapes and types defined by `self.output_shapes` and
     `self.output_types`) to a scalar `tf.bool` tensor.
 
   Returns:
@@ -70,6 +66,6 @@ def take_while(predicate):
     `tf.data.Dataset.apply`.
   """
   def _apply_fn(dataset):
-      return _TakeWhileDataset(dataset, predicate)
+    return _TakeWhileDataset(dataset, predicate)
 
   return _apply_fn
-- 
GitLab


From 28edbb813a3a94892d57ce29354e69c0a960a23e Mon Sep 17 00:00:00 2001
From: Dheeraj Rajaram Reddy <dheeraj98reddy@gmail.com>
Date: Fri, 4 Jan 2019 15:32:40 +0530
Subject: [PATCH 0179/2345] Use parameterized test case

---
 .../take_while_dataset_serialization_test.py  | 26 ++++++++++---------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/take_while_dataset_serialization_test.py b/tensorflow/python/data/experimental/kernel_tests/serialization/take_while_dataset_serialization_test.py
index 9b5498cca5..a96e37f0b4 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/take_while_dataset_serialization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/take_while_dataset_serialization_test.py
@@ -17,6 +17,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from absl.testing import parameterized
+
 from tensorflow.python.data.experimental.kernel_tests.serialization import dataset_serialization_test_base
 from tensorflow.python.data.experimental.ops import take_while_ops
 from tensorflow.python.data.ops import dataset_ops
@@ -24,21 +26,21 @@ from tensorflow.python.platform import test
 
 
 class TakeWhileDatasetSerializationTest(
-    dataset_serialization_test_base.DatasetSerializationTestBase):
+    dataset_serialization_test_base.DatasetSerializationTestBase,
+    parameterized.TestCase):
 
   def _build_dataset(self, num_elements, upper_bound):
     return dataset_ops.Dataset.range(num_elements).apply(
-              take_while_ops.take_while(lambda x: x < upper_bound))
-
-  def testCore(self):
-    def run_test(num_elem1, num_elem2, upper_bound):
-        self.run_core_tests(lambda: self._build_dataset(num_elem1, upper_bound),
-                            lambda: self._build_dataset(num_elem2, upper_bound), 
-                            upper_bound)
-
-    run_test(23, 10, 7)
-    run_test(10, 50, 0)
-    run_test(25, 30, 25)
+        take_while_ops.take_while(lambda x: x < upper_bound))
+
+  @parameterized.parameters(
+      (23, 10, 7),
+      (10, 50, 0),
+      (25, 30, 25))
+  def testCore(self, num_elem1, num_elem2, upper_bound):
+    self.run_core_tests(lambda: self._build_dataset(num_elem1, upper_bound),
+                        lambda: self._build_dataset(num_elem2, upper_bound),
+                        upper_bound)
 
 
 if __name__ == "__main__":
-- 
GitLab


From 8e4ca33c8b5861526bcdc43eb1af6d73e1670e92 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 04:10:23 -0800
Subject: [PATCH 0180/2345] Correctly handle S32 and U32 in the AR-CRS combiner

Some of the transforms we are doing are only valid on floating point types
so we have to condition them on the element type of the operations.

PiperOrigin-RevId: 227836047
---
 tensorflow/compiler/xla/service/BUILD         |  1 +
 .../compiler/xla/service/ar_crs_combiner.cc   | 24 +++++++++++++++----
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index b84792cfc3..755c477a12 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -3678,6 +3678,7 @@ cc_library(
         ":pattern_matcher",
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:literal_util",
+        "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:status_macros",
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:types",
diff --git a/tensorflow/compiler/xla/service/ar_crs_combiner.cc b/tensorflow/compiler/xla/service/ar_crs_combiner.cc
index 47d2c7e357..4a227d3b5c 100644
--- a/tensorflow/compiler/xla/service/ar_crs_combiner.cc
+++ b/tensorflow/compiler/xla/service/ar_crs_combiner.cc
@@ -26,6 +26,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
 #include "tensorflow/compiler/xla/service/pattern_matcher.h"
+#include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
@@ -44,11 +45,24 @@ bool MatchesArCrsPattern(HloInstruction* instruction) {
     if (instruction->user_count() != 1) {
       return false;
     }
-    auto opcode = instruction->opcode();
-    return opcode == HloOpcode::kBitcast || opcode == HloOpcode::kTranspose ||
-           opcode == HloOpcode::kReshape || opcode == HloOpcode::kConvert ||
-           opcode == HloOpcode::kAdd || opcode == HloOpcode::kSubtract ||
-           opcode == HloOpcode::kMultiply;
+    switch (instruction->opcode()) {
+      case HloOpcode::kBitcast:
+      case HloOpcode::kTranspose:
+      case HloOpcode::kReshape:
+        return true;
+      case HloOpcode::kConvert:
+        // Can be moved across if both input and output is either float or
+        // integer (e.g. S32<->U32 or F32<->BF16)
+        return ShapeUtil::ElementIsFloating(instruction->shape()) ==
+               ShapeUtil::ElementIsFloating(instruction->operand(0)->shape());
+      case HloOpcode::kAdd:
+      case HloOpcode::kSubtract:
+      case HloOpcode::kMultiply:
+        // Only supported for floating point operands.
+        return ShapeUtil::ElementIsFloating(instruction->shape());
+      default:
+        return false;
+    }
   };
 
   auto computation_is_addition = [](HloComputation* c) {
-- 
GitLab


From 0bc20afeec2b06437a7196b0d191d61f181cd81a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 07:27:10 -0800
Subject: [PATCH 0181/2345] Update RBE CPU Dockerfile with instructions on how
 to build/push.

PiperOrigin-RevId: 227853285
---
 tensorflow/tools/ci_build/Dockerfile.rbe.cpu | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tensorflow/tools/ci_build/Dockerfile.rbe.cpu b/tensorflow/tools/ci_build/Dockerfile.rbe.cpu
index 7e5860aeec..500fb6e0b3 100644
--- a/tensorflow/tools/ci_build/Dockerfile.rbe.cpu
+++ b/tensorflow/tools/ci_build/Dockerfile.rbe.cpu
@@ -1,3 +1,8 @@
+# To push a new version, run:
+# $ docker build -f Dockerfile.rbe.cpu \
+#       --tag "gcr.io/tensorflow-testing/nosla-ubuntu16.04" .
+# $ docker push gcr.io/tensorflow-testing/nosla-ubuntu16.04
+
 FROM launcher.gcr.io/google/rbe-ubuntu16-04:r327695
 LABEL maintainer="Yu Yi <yiyu@google.com>"
 
-- 
GitLab


From bd2f78fd205faa739c657d26156bd27e02223df0 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Fri, 4 Jan 2019 08:31:47 -0800
Subject: [PATCH 0182/2345] Make contrib.seq2seq more compatible with tf 2.0.
 (1/6)

1. Add a new loss object which wrap around the existing loss function, which
use can use it keras model.
2. Slightly update the code for readability and reduce duplication.
3. Update the test case to be eager compatible.

PiperOrigin-RevId: 227860442
---
 .../seq2seq/python/kernel_tests/loss_test.py  | 290 ++++++++++++++----
 tensorflow/contrib/seq2seq/python/ops/loss.py | 109 +++++--
 2 files changed, 315 insertions(+), 84 deletions(-)

diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/loss_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/loss_test.py
index 5aa32b532f..41b2a53ca5 100644
--- a/tensorflow/contrib/seq2seq/python/kernel_tests/loss_test.py
+++ b/tensorflow/contrib/seq2seq/python/kernel_tests/loss_test.py
@@ -14,80 +14,254 @@
 # ==============================================================================
 
 """Tests for contrib.seq2seq.python.seq2seq.loss_ops."""
-# pylint: disable=unused-import,g-bad-import-order
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
-# pylint: enable=unused-import
 
 import numpy as np
 
 from tensorflow.contrib.seq2seq.python.ops import loss
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import init_ops
-from tensorflow.python.ops import variable_scope
 from tensorflow.python.platform import test
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class LossTest(test.TestCase):
 
+  def setUp(self):
+    self.batch_size = 2
+    self.sequence_length = 3
+    self.number_of_classes = 5
+    logits = [
+        constant_op.constant(i + 0.5, shape=[self.batch_size,
+                                             self.number_of_classes])
+        for i in range(self.sequence_length)
+    ]
+    self.logits = array_ops.stack(logits, axis=1)
+    targets = [
+        constant_op.constant(i, dtypes.int32, shape=[self.batch_size])
+        for i in range(self.sequence_length)
+    ]
+    self.targets = array_ops.stack(targets, axis=1)
+    weights = [
+        constant_op.constant(1.0, shape=[self.batch_size])
+        for _ in range(self.sequence_length)
+    ]
+    self.weights = array_ops.stack(weights, axis=1)
+    # expected_loss = sparse_softmax_cross_entropy_with_logits(targets, logits)
+    # where targets = [0, 1, 2], and logits = [[0.5] * 5, [1.5] * 5, [2.5] * 5]
+    self.expected_loss = 1.60944
+
   def testSequenceLoss(self):
-    with self.session(use_gpu=True) as sess:
-      with variable_scope.variable_scope(
-          'root', initializer=init_ops.constant_initializer(0.5)):
-        batch_size = 2
-        sequence_length = 3
-        number_of_classes = 5
-        logits = [
-            constant_op.constant(
-                i + 0.5, shape=[batch_size, number_of_classes])
-            for i in range(sequence_length)
-        ]
-        logits = array_ops.stack(logits, axis=1)
-        targets = [
-            constant_op.constant(
-                i, dtypes.int32, shape=[batch_size])
-            for i in range(sequence_length)
-        ]
-        targets = array_ops.stack(targets, axis=1)
-        weights = [
-            constant_op.constant(
-                1.0, shape=[batch_size]) for i in range(sequence_length)
-        ]
-        weights = array_ops.stack(weights, axis=1)
-
-        average_loss_per_example = loss.sequence_loss(
-            logits, targets, weights,
-            average_across_timesteps=True,
-            average_across_batch=True)
-        res = sess.run(average_loss_per_example)
-        self.assertAllClose(1.60944, res)
-
-        average_loss_per_sequence = loss.sequence_loss(
-            logits, targets, weights,
-            average_across_timesteps=False,
-            average_across_batch=True)
-        res = sess.run(average_loss_per_sequence)
-        compare_per_sequence = np.ones((sequence_length)) * 1.60944
-        self.assertAllClose(compare_per_sequence, res)
-
-        average_loss_per_batch = loss.sequence_loss(
-            logits, targets, weights,
-            average_across_timesteps=True,
-            average_across_batch=False)
-        res = sess.run(average_loss_per_batch)
-        compare_per_batch = np.ones((batch_size)) * 1.60944
-        self.assertAllClose(compare_per_batch, res)
-
-        total_loss = loss.sequence_loss(
-            logits, targets, weights,
-            average_across_timesteps=False,
-            average_across_batch=False)
-        res = sess.run(total_loss)
-        compare_total = np.ones((batch_size, sequence_length)) * 1.60944
-        self.assertAllClose(compare_total, res)
+    with self.test_session(use_gpu=True):
+      average_loss_per_example = loss.sequence_loss(
+          self.logits, self.targets, self.weights,
+          average_across_timesteps=True,
+          average_across_batch=True)
+      res = self.evaluate(average_loss_per_example)
+      self.assertAllClose(self.expected_loss, res)
+
+      average_loss_per_sequence = loss.sequence_loss(
+          self.logits, self.targets, self.weights,
+          average_across_timesteps=False,
+          average_across_batch=True)
+      res = self.evaluate(average_loss_per_sequence)
+      compare_per_sequence = np.full((self.sequence_length), self.expected_loss)
+      self.assertAllClose(compare_per_sequence, res)
+
+      average_loss_per_batch = loss.sequence_loss(
+          self.logits, self.targets, self.weights,
+          average_across_timesteps=True,
+          average_across_batch=False)
+      res = self.evaluate(average_loss_per_batch)
+      compare_per_batch = np.full((self.batch_size), self.expected_loss)
+      self.assertAllClose(compare_per_batch, res)
+
+      total_loss = loss.sequence_loss(
+          self.logits, self.targets, self.weights,
+          average_across_timesteps=False,
+          average_across_batch=False)
+      res = self.evaluate(total_loss)
+      compare_total = np.full((self.batch_size, self.sequence_length),
+                              self.expected_loss)
+      self.assertAllClose(compare_total, res)
+
+  def testSequenceLossClass(self):
+    with self.test_session(use_gpu=True):
+      seq_loss = loss.SequenceLoss(average_across_timesteps=True,
+                                   average_across_batch=True,
+                                   sum_over_timesteps=False,
+                                   sum_over_batch=False)
+      average_loss_per_example = seq_loss(
+          self.targets, self.logits, self.weights)
+      res = self.evaluate(average_loss_per_example)
+      self.assertAllClose(self.expected_loss, res)
+
+      seq_loss = loss.SequenceLoss(average_across_timesteps=False,
+                                   average_across_batch=True,
+                                   sum_over_timesteps=False,
+                                   sum_over_batch=False)
+      average_loss_per_sequence = seq_loss(
+          self.targets, self.logits, self.weights)
+      res = self.evaluate(average_loss_per_sequence)
+      compare_per_sequence = np.full((self.sequence_length), self.expected_loss)
+      self.assertAllClose(compare_per_sequence, res)
+
+      seq_loss = loss.SequenceLoss(average_across_timesteps=True,
+                                   average_across_batch=False,
+                                   sum_over_timesteps=False,
+                                   sum_over_batch=False)
+      average_loss_per_batch = seq_loss(
+          self.targets, self.logits, self.weights)
+      res = self.evaluate(average_loss_per_batch)
+      compare_per_batch = np.full((self.batch_size), self.expected_loss)
+      self.assertAllClose(compare_per_batch, res)
+
+      seq_loss = loss.SequenceLoss(average_across_timesteps=False,
+                                   average_across_batch=False,
+                                   sum_over_timesteps=False,
+                                   sum_over_batch=False)
+      total_loss = seq_loss(
+          self.targets, self.logits, self.weights)
+      res = self.evaluate(total_loss)
+      compare_total = np.full((self.batch_size, self.sequence_length),
+                              self.expected_loss)
+      self.assertAllClose(compare_total, res)
+
+  def testSumReduction(self):
+    with self.test_session(use_gpu=True):
+      seq_loss = loss.SequenceLoss(average_across_timesteps=False,
+                                   average_across_batch=False,
+                                   sum_over_timesteps=True,
+                                   sum_over_batch=True)
+      average_loss_per_example = seq_loss(
+          self.targets, self.logits, self.weights)
+      res = self.evaluate(average_loss_per_example)
+      self.assertAllClose(self.expected_loss, res)
+
+      seq_loss = loss.SequenceLoss(average_across_timesteps=False,
+                                   average_across_batch=False,
+                                   sum_over_timesteps=False,
+                                   sum_over_batch=True)
+      average_loss_per_sequence = seq_loss(
+          self.targets, self.logits, self.weights)
+      res = self.evaluate(average_loss_per_sequence)
+      compare_per_sequence = np.full((self.sequence_length), self.expected_loss)
+      self.assertAllClose(compare_per_sequence, res)
+
+      seq_loss = loss.SequenceLoss(average_across_timesteps=False,
+                                   average_across_batch=False,
+                                   sum_over_timesteps=True,
+                                   sum_over_batch=False)
+      average_loss_per_batch = seq_loss(
+          self.targets, self.logits, self.weights)
+      res = self.evaluate(average_loss_per_batch)
+      compare_per_batch = np.full((self.batch_size), self.expected_loss)
+      self.assertAllClose(compare_per_batch, res)
+
+      seq_loss = loss.SequenceLoss(average_across_timesteps=False,
+                                   average_across_batch=False,
+                                   sum_over_timesteps=False,
+                                   sum_over_batch=False)
+      total_loss = seq_loss(
+          self.targets, self.logits, self.weights)
+      res = self.evaluate(total_loss)
+      compare_total = np.full((self.batch_size, self.sequence_length),
+                              self.expected_loss)
+      self.assertAllClose(compare_total, res)
+
+  def testWeightedSumReduction(self):
+    weights = [
+        constant_op.constant(1.0, shape=[self.batch_size])
+        for _ in range(self.sequence_length)
+    ]
+    # Make the last element in the sequence to have zero weights.
+    weights[-1] = constant_op.constant(0.0, shape=[self.batch_size])
+    self.weights = array_ops.stack(weights, axis=1)
+    with self.test_session(use_gpu=True):
+      seq_loss = loss.SequenceLoss(average_across_timesteps=False,
+                                   average_across_batch=False,
+                                   sum_over_timesteps=True,
+                                   sum_over_batch=True)
+      average_loss_per_example = seq_loss(
+          self.targets, self.logits, self.weights)
+      res = self.evaluate(average_loss_per_example)
+      self.assertAllClose(self.expected_loss, res)
+
+      seq_loss = loss.SequenceLoss(average_across_timesteps=False,
+                                   average_across_batch=False,
+                                   sum_over_timesteps=False,
+                                   sum_over_batch=True)
+      average_loss_per_sequence = seq_loss(
+          self.targets, self.logits, self.weights)
+      res = self.evaluate(average_loss_per_sequence)
+      compare_per_sequence = np.full((self.sequence_length), self.expected_loss)
+      # The last element in every sequence are zeros, which will be filtered.
+      compare_per_sequence[-1] = 0.
+      self.assertAllClose(compare_per_sequence, res)
+
+      seq_loss = loss.SequenceLoss(average_across_timesteps=False,
+                                   average_across_batch=False,
+                                   sum_over_timesteps=True,
+                                   sum_over_batch=False)
+      average_loss_per_batch = seq_loss(self.targets, self.logits, self.weights)
+      res = self.evaluate(average_loss_per_batch)
+      compare_per_batch = np.full((self.batch_size), self.expected_loss)
+      self.assertAllClose(compare_per_batch, res)
+
+      seq_loss = loss.SequenceLoss(average_across_timesteps=False,
+                                   average_across_batch=False,
+                                   sum_over_timesteps=False,
+                                   sum_over_batch=False)
+      total_loss = seq_loss(self.targets, self.logits, self.weights)
+      res = self.evaluate(total_loss)
+      compare_total = np.full((self.batch_size, self.sequence_length),
+                              self.expected_loss)
+      # The last element in every sequence are zeros, which will be filtered.
+      compare_total[:, -1] = 0
+      self.assertAllClose(compare_total, res)
+
+  def testZeroWeights(self):
+    weights = [
+        constant_op.constant(0.0, shape=[self.batch_size])
+        for _ in range(self.sequence_length)
+    ]
+    weights = array_ops.stack(weights, axis=1)
+    with self.test_session(use_gpu=True):
+      average_loss_per_example = loss.sequence_loss(
+          self.logits, self.targets, weights,
+          average_across_timesteps=True,
+          average_across_batch=True)
+      res = self.evaluate(average_loss_per_example)
+      self.assertAllClose(0.0, res)
+
+      average_loss_per_sequence = loss.sequence_loss(
+          self.logits, self.targets, weights,
+          average_across_timesteps=False,
+          average_across_batch=True)
+      res = self.evaluate(average_loss_per_sequence)
+      compare_per_sequence = np.zeros((self.sequence_length))
+      self.assertAllClose(compare_per_sequence, res)
+
+      average_loss_per_batch = loss.sequence_loss(
+          self.logits, self.targets, weights,
+          average_across_timesteps=True,
+          average_across_batch=False)
+      res = self.evaluate(average_loss_per_batch)
+      compare_per_batch = np.zeros((self.batch_size))
+      self.assertAllClose(compare_per_batch, res)
+
+      total_loss = loss.sequence_loss(
+          self.logits, self.targets, weights,
+          average_across_timesteps=False,
+          average_across_batch=False)
+      res = self.evaluate(total_loss)
+      compare_total = np.zeros((self.batch_size, self.sequence_length))
+      self.assertAllClose(compare_total, res)
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/seq2seq/python/ops/loss.py b/tensorflow/contrib/seq2seq/python/ops/loss.py
index 39a6d2f58b..0fbfd61870 100644
--- a/tensorflow/contrib/seq2seq/python/ops/loss.py
+++ b/tensorflow/contrib/seq2seq/python/ops/loss.py
@@ -20,11 +20,12 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.framework import ops
+from tensorflow.python.keras.losses import Loss
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn_ops
 
-__all__ = ["sequence_loss"]
+__all__ = ["sequence_loss", "SequenceLoss"]
 
 
 def sequence_loss(logits,
@@ -32,16 +33,26 @@ def sequence_loss(logits,
                   weights,
                   average_across_timesteps=True,
                   average_across_batch=True,
+                  sum_over_timesteps=False,
+                  sum_over_batch=False,
                   softmax_loss_function=None,
                   name=None):
   """Weighted cross-entropy loss for a sequence of logits.
 
-  Depending on the values of `average_across_timesteps` and
-  `average_across_batch`, the return Tensor will have rank 0, 1, or 2 as these
-  arguments reduce the cross-entropy at each target, which has shape
-  `[batch_size, sequence_length]`, over their respective dimensions. For
-  example, if `average_across_timesteps` is `True` and `average_across_batch`
-  is `False`, then the return Tensor will have shape `[batch_size]`.
+  Depending on the values of `average_across_timesteps` / `sum_over_timesteps`
+  and `average_across_batch` / `sum_over_batch`, the return Tensor will have
+  rank 0, 1, or 2 as these arguments reduce the cross-entropy at each target,
+  which has shape `[batch_size, sequence_length]`, over their respective
+  dimensions. For example, if `average_across_timesteps` is `True` and
+  `average_across_batch` is `False`, then the return Tensor will have shape
+  `[batch_size]`.
+
+  Note that `average_across_timesteps` and `sum_over_timesteps` cannot be True
+  at same time. Same for `average_across_batch` and `sum_over_batch`.
+
+  The recommended loss reduction in tf 2.0 has been changed to sum_over, instead
+  of weighted average. User are recommend to use `sum_over_timesteps` and
+  `sum_over_batch` for reduction.
 
   Args:
     logits: A Tensor of shape
@@ -58,6 +69,12 @@ def sequence_loss(logits,
       dimension and divide the cost by the total label weight across timesteps.
     average_across_batch: If set, sum the cost across the batch dimension and
       divide the returned cost by the batch size.
+    sum_over_timesteps: If set, sum the cost across the sequence dimension and
+      divide the size of the sequence. Note that any element with 0 weights will
+      be excluded from size calculation.
+    sum_over_batch: if set, sum the cost across the batch dimension and divide
+      the total cost by the batch size. Not that any element with 0 weights will
+      be excluded from size calculation.
     softmax_loss_function: Function (labels, logits) -> loss-batch
       to be used instead of the standard softmax (the default if this is None).
       **Note that to avoid confusion, it is required for the function to accept
@@ -78,11 +95,15 @@ def sequence_loss(logits,
     raise ValueError("Logits must be a "
                      "[batch_size x sequence_length x logits] tensor")
   if len(targets.get_shape()) != 2:
-    raise ValueError("Targets must be a [batch_size x sequence_length] "
-                     "tensor")
+    raise ValueError("Targets must be a [batch_size x sequence_length] tensor")
   if len(weights.get_shape()) != 2:
-    raise ValueError("Weights must be a [batch_size x sequence_length] "
-                     "tensor")
+    raise ValueError("Weights must be a [batch_size x sequence_length] tensor")
+  if average_across_timesteps and sum_over_timesteps:
+    raise ValueError("average_across_timesteps and sum_over_timesteps cannot "
+                     "be set to True at same time.")
+  if average_across_batch and sum_over_batch:
+    raise ValueError("average_across_batch and sum_over_batch cannot be set "
+                     "to True at same time.")
   with ops.name_scope(name, "sequence_loss", [logits, targets, weights]):
     num_classes = array_ops.shape(logits)[2]
     logits_flat = array_ops.reshape(logits, [-1, num_classes])
@@ -96,20 +117,56 @@ def sequence_loss(logits,
     if average_across_timesteps and average_across_batch:
       crossent = math_ops.reduce_sum(crossent)
       total_size = math_ops.reduce_sum(weights)
-      total_size += 1e-12  # to avoid division by 0 for all-0 weights
-      crossent /= total_size
+      crossent = math_ops.div_no_nan(crossent, total_size)
+    elif sum_over_timesteps and sum_over_batch:
+      crossent = math_ops.reduce_sum(crossent)
+      total_count = math_ops.cast(math_ops.count_nonzero(weights),
+                                  crossent.dtype)
+      crossent = math_ops.div_no_nan(crossent, total_count)
     else:
-      batch_size = array_ops.shape(logits)[0]
-      sequence_length = array_ops.shape(logits)[1]
-      crossent = array_ops.reshape(crossent, [batch_size, sequence_length])
-    if average_across_timesteps and not average_across_batch:
-      crossent = math_ops.reduce_sum(crossent, axis=[1])
-      total_size = math_ops.reduce_sum(weights, axis=[1])
-      total_size += 1e-12  # to avoid division by 0 for all-0 weights
-      crossent /= total_size
-    if not average_across_timesteps and average_across_batch:
-      crossent = math_ops.reduce_sum(crossent, axis=[0])
-      total_size = math_ops.reduce_sum(weights, axis=[0])
-      total_size += 1e-12  # to avoid division by 0 for all-0 weights
-      crossent /= total_size
+      crossent = array_ops.reshape(crossent, array_ops.shape(logits)[0:2])
+      if average_across_timesteps or average_across_batch:
+        reduce_axis = [0] if average_across_batch else [1]
+        crossent = math_ops.reduce_sum(crossent, axis=reduce_axis)
+        total_size = math_ops.reduce_sum(weights, axis=reduce_axis)
+        crossent = math_ops.div_no_nan(crossent, total_size)
+      elif sum_over_timesteps or sum_over_batch:
+        reduce_axis = [0] if sum_over_batch else [1]
+        crossent = math_ops.reduce_sum(crossent, axis=reduce_axis)
+        total_count = math_ops.cast(
+            math_ops.count_nonzero(weights, axis=reduce_axis),
+            dtype=crossent.dtype)
+        crossent = math_ops.div_no_nan(crossent, total_count)
     return crossent
+
+
+class SequenceLoss(Loss):
+  """Weighted cross-entropy loss for a sequence of logits."""
+
+  def __init__(self,
+               average_across_timesteps=False,
+               average_across_batch=False,
+               sum_over_timesteps=True,
+               sum_over_batch=True,
+               softmax_loss_function=None,
+               name=None):
+    super(SequenceLoss, self).__init__(name=name)
+    self.average_across_timesteps = average_across_timesteps
+    self.average_across_batch = average_across_batch
+    self.sum_over_timesteps = sum_over_timesteps
+    self.sum_over_batch = sum_over_batch
+    self.softmax_loss_function = softmax_loss_function
+
+  def __call__(self, y_true, y_pred, sample_weight=None):
+    """Override the parent __call__ to have a customized reduce behavior."""
+    return sequence_loss(y_pred, y_true, sample_weight,
+                         average_across_timesteps=self.average_across_timesteps,
+                         average_across_batch=self.average_across_batch,
+                         sum_over_timesteps=self.sum_over_timesteps,
+                         sum_over_batch=self.sum_over_batch,
+                         softmax_loss_function=self.softmax_loss_function,
+                         name=self.name)
+
+  def call(self, y_true, y_pred):
+    # Skip this method since the __call__ contains real implementation.
+    pass
-- 
GitLab


From 7a830dab71dc530226936306b109fecead4366d6 Mon Sep 17 00:00:00 2001
From: Dheeraj Rajaram Reddy <dheeraj98reddy@gmail.com>
Date: Fri, 4 Jan 2019 22:42:38 +0530
Subject: [PATCH 0183/2345] Add ShortCircuit optimization for take_while and
 corresponding test

---
 .../core/kernels/data/experimental/BUILD      |  1 +
 .../experimental/take_while_dataset_op.cc     | 80 +++++++++++++------
 .../kernel_tests/take_while_test.py           | 16 ++++
 3 files changed, 73 insertions(+), 24 deletions(-)

diff --git a/tensorflow/core/kernels/data/experimental/BUILD b/tensorflow/core/kernels/data/experimental/BUILD
index 4ce14b0140..53d1f02a3a 100644
--- a/tensorflow/core/kernels/data/experimental/BUILD
+++ b/tensorflow/core/kernels/data/experimental/BUILD
@@ -303,6 +303,7 @@ tf_kernel_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core/kernels/data:captured_function",
+        "//tensorflow/core/kernels/data:dataset_utils"
     ],
 )
 
diff --git a/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc b/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
index cf9c5f0cb6..b670e4efaa 100644
--- a/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/kernels/data/captured_function.h"
+#include "tensorflow/core/kernels/data/dataset_utils.h"
 
 namespace tensorflow {
 namespace data {
@@ -30,6 +31,10 @@ namespace {
 
 class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
  public:
+  using LoopIteratorPredicate =
+      std::function<Status(IteratorContext*, InstantiatedCapturedFunction*,
+                           std::vector<Tensor>, bool*)>;
+
   explicit TakeWhileDatasetOp(OpKernelConstruction* ctx)
       : UnaryDatasetOpKernel(ctx) {
     OP_REQUIRES_OK(ctx, ctx->GetAttr("predicate", &func_));
@@ -41,8 +46,45 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
     OP_REQUIRES_OK(ctx, CapturedFunction::Create(func_, ctx, "other_arguments",
                                                  &captured_func));
 
-    // TODO (squadrick): check short-circuit
-    *output = new Dataset(ctx, input, func_, std::move(captured_func));
+    std::vector<int> indices;
+    OP_REQUIRES_OK(ctx, ComputeShortCircuitIndices(ctx, func_, &indices));
+    OP_REQUIRES(
+        ctx, indices.size() <= 1,
+        errors::InvalidArgument("`predicate` has more than one return value."));
+
+    LoopIteratorPredicate loop_pred;
+    if (indices.empty()) {
+      loop_pred = [](IteratorContext* ctx,
+                     InstantiatedCapturedFunction* inst_captured_func,
+                     const std::vector<Tensor>& args, bool* end_of_sequence) {
+        std::vector<Tensor> result;
+        TF_RETURN_IF_ERROR(
+            inst_captured_func->RunWithBorrowedArgs(ctx, args, &result));
+
+        if (result.size() != 1 || result[0].dtype() != DT_BOOL ||
+            result[0].NumElements() != 1) {
+          return errors::InvalidArgument(
+              "`predicate` must returns a scalar bool tensor.");
+        }
+        *end_of_sequence = !result[0].scalar<bool>()();
+        return Status::OK();
+      };
+    } else {
+      loop_pred = [indices](IteratorContext* ctx,
+                            InstantiatedCapturedFunction* inst_captured_func,
+                            const std::vector<Tensor>& args,
+                            bool* end_of_sequence) {
+        const Tensor& predicate = args[indices[0]];
+        if (predicate.dtype() != DT_BOOL || predicate.NumElements() != 1) {
+          return errors::InvalidArgument(
+              "`predicate` must returns a scalar bool tensor.");
+        }
+        *end_of_sequence = !predicate.scalar<bool>()();
+        return Status::OK();
+      };
+    }
+    *output = new Dataset(ctx, input, func_, std::move(captured_func),
+                          std::move(loop_pred));
   }
 
  private:
@@ -50,11 +92,13 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
    public:
     Dataset(OpKernelContext* ctx, const DatasetBase* input,
             const NameAttrList& func,
-            std::unique_ptr<CapturedFunction> captured_func)
+            std::unique_ptr<CapturedFunction> captured_func,
+            LoopIteratorPredicate loop_pred)
         : DatasetBase(DatasetContext(ctx)),
           input_(input),
           func_(func),
-          captured_func_(std::move(captured_func)) {
+          captured_func_(std::move(captured_func)),
+          loop_pred_(std::move(loop_pred)) {
       input_->Ref();
     }
 
@@ -62,8 +106,8 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::TakeWhile")}));
+      return std::unique_ptr<IteratorBase>(new Iterator(
+          {this, strings::StrCat(prefix, "::TakeWhile")}, loop_pred_));
     }
 
     const DataTypeVector& output_dtypes() const override {
@@ -122,8 +166,8 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
    private:
     class Iterator : public DatasetIterator<Dataset> {
      public:
-      explicit Iterator(const Params& params)
-          : DatasetIterator<Dataset>(params) {}
+      explicit Iterator(const Params& params, LoopIteratorPredicate loop_pred)
+          : DatasetIterator<Dataset>(params), loop_pred_(loop_pred) {}
 
       Status Initialize(IteratorContext* ctx) override {
         TF_RETURN_IF_ERROR(
@@ -149,22 +193,8 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
           input_impl_.reset();
           return Status::OK();
         }
-
-        std::vector<Tensor> bool_output;
-
-        Status s = instantiated_captured_func_->RunWithBorrowedArgs(
-            ctx, *out_tensors, &bool_output);
-
-        if (s.ok()) {
-          if (bool_output.size() != 1 || bool_output[0].dtype() != DT_BOOL ||
-              bool_output[0].NumElements() != 1) {
-            return errors::InvalidArgument(
-                "`predicate` must returns a scalar bool tensor.");
-          }
-          *end_of_sequence = !bool_output[0].scalar<bool>()();
-          return Status::OK();
-        }
-        return s;  // propagate error to caller
+        return loop_pred_(ctx, instantiated_captured_func_.get(), *out_tensors,
+                          end_of_sequence);
       }
 
      protected:
@@ -198,11 +228,13 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
       mutex mu_;
       std::unique_ptr<IteratorBase> input_impl_ GUARDED_BY(mu_);
       std::unique_ptr<InstantiatedCapturedFunction> instantiated_captured_func_;
+      const LoopIteratorPredicate loop_pred_;
     };
 
     const DatasetBase* const input_;
     const NameAttrList func_;
     const std::unique_ptr<CapturedFunction> captured_func_;
+    const LoopIteratorPredicate loop_pred_;
   };
 
   NameAttrList func_;
diff --git a/tensorflow/python/data/experimental/kernel_tests/take_while_test.py b/tensorflow/python/data/experimental/kernel_tests/take_while_test.py
index b1e7974471..3faee0639f 100644
--- a/tensorflow/python/data/experimental/kernel_tests/take_while_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/take_while_test.py
@@ -83,6 +83,22 @@ class TakeWhileTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.assertRaises(errors.OutOfRangeError):
       self.assertEqual(b"test", self.evaluate(next_element()))
 
+  def testTakewhileDatasetShortCircuit(self):
+    def _predicate_func(data_elem):
+      return data_elem
+
+    boolean_array = [True, True, True, True, False]
+    dataset = dataset_ops.Dataset.from_tensor_slices(boolean_array).apply(
+        take_while_ops.take_while(_predicate_func))
+
+    next_element = self.getNext(dataset)
+    self.assertEqual(True, self.evaluate(next_element()))
+    self.assertEqual(True, self.evaluate(next_element()))
+    self.assertEqual(True, self.evaluate(next_element()))
+    self.assertEqual(True, self.evaluate(next_element()))
+
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(next_element())
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 533d43705aa018d3bd07dc448ad63625776be63c Mon Sep 17 00:00:00 2001
From: "Joshua V. Dillon" <jvdillon@google.com>
Date: Fri, 4 Jan 2019 09:13:05 -0800
Subject: [PATCH 0184/2345] Remove tf.contrib.distributions seq2seq dependecy.

PiperOrigin-RevId: 227865533
---
 tensorflow/contrib/seq2seq/BUILD              |  2 -
 .../python/kernel_tests/basic_decoder_test.py |  6 +-
 .../contrib/seq2seq/python/ops/helper.py      | 86 ++++++++++++++++---
 3 files changed, 76 insertions(+), 18 deletions(-)

diff --git a/tensorflow/contrib/seq2seq/BUILD b/tensorflow/contrib/seq2seq/BUILD
index 18b56cd219..89176180ae 100644
--- a/tensorflow/contrib/seq2seq/BUILD
+++ b/tensorflow/contrib/seq2seq/BUILD
@@ -33,7 +33,6 @@ tf_custom_op_py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":beam_search_ops",
-        "//tensorflow/contrib/distributions:distributions_py",
         "//tensorflow/contrib/layers:layers_py",
         "//tensorflow/contrib/rnn:rnn_py",
         "//tensorflow/contrib/util:util_py",
@@ -59,7 +58,6 @@ tf_custom_op_py_library(
         "//tensorflow/python:tensor_util",
         "//tensorflow/python:util",
         "//tensorflow/python:variable_scope",
-        "//tensorflow/python/ops/distributions",
         "//third_party/py/numpy",
         "@six_archive//:six",
     ],
diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/basic_decoder_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/basic_decoder_test.py
index b7f9f3fb09..abcf71c61b 100644
--- a/tensorflow/contrib/seq2seq/python/kernel_tests/basic_decoder_test.py
+++ b/tensorflow/contrib/seq2seq/python/kernel_tests/basic_decoder_test.py
@@ -34,8 +34,6 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import rnn_cell
 from tensorflow.python.ops import variables
 from tensorflow.python.ops import variable_scope
-from tensorflow.python.ops.distributions import bernoulli
-from tensorflow.python.ops.distributions import categorical
 from tensorflow.python.platform import test
 # pylint: enable=g-import-not-at-top
 
@@ -517,7 +515,7 @@ class BasicDecoderTest(test.TestCase):
         vocabulary_size)
 
     # The sample function samples categorically from the logits.
-    sample_fn = lambda x: categorical.Categorical(logits=x).sample()
+    sample_fn = lambda x: helper_py.categorical_sample(logits=x)
     # The next inputs are a one-hot encoding of the sampled labels.
     next_inputs_fn = (
         lambda x: array_ops.one_hot(x, vocabulary_size, dtype=dtypes.float32))
@@ -599,7 +597,7 @@ class BasicDecoderTest(test.TestCase):
 
     # The sample function samples independent bernoullis from the logits.
     sample_fn = (
-        lambda x: bernoulli.Bernoulli(logits=x, dtype=dtypes.bool).sample())
+        lambda x: helper_py.bernoulli_sample(logits=x, dtype=dtypes.bool))
     # The next inputs are a one-hot encoding of the sampled labels.
     next_inputs_fn = math_ops.to_float
     end_fn = lambda sample_ids: sample_ids[:, end_token]
diff --git a/tensorflow/contrib/seq2seq/python/ops/helper.py b/tensorflow/contrib/seq2seq/python/ops/helper.py
index 3245cc5e72..033c2eb080 100644
--- a/tensorflow/contrib/seq2seq/python/ops/helper.py
+++ b/tensorflow/contrib/seq2seq/python/ops/helper.py
@@ -32,9 +32,8 @@ from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import embedding_ops
 from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import tensor_array_ops
-from tensorflow.python.ops.distributions import bernoulli
-from tensorflow.python.ops.distributions import categorical
 from tensorflow.python.util import nest
 
 __all__ = [
@@ -51,6 +50,68 @@ __all__ = [
 _transpose_batch_time = decoder._transpose_batch_time  # pylint: disable=protected-access
 
 
+# The following sample functions (_call_sampler, bernoulli_sample,
+# categorical_sample) mimic TensorFlow Probability distribution semantics.
+
+
+def _call_sampler(sample_n_fn, sample_shape, name=None):
+  """Reshapes vector of samples."""
+  with ops.name_scope(name, "call_sampler", values=[sample_shape]):
+    sample_shape = ops.convert_to_tensor(
+        sample_shape, dtype=dtypes.int32, name="sample_shape")
+    # Ensure sample_shape is a vector (vs just a scalar).
+    pad = math_ops.cast(math_ops.equal(array_ops.rank(sample_shape), 0),
+                        dtypes.int32)
+    sample_shape = array_ops.reshape(
+        sample_shape,
+        array_ops.pad(array_ops.shape(sample_shape),
+                      paddings=[[pad, 0]],
+                      constant_values=1))
+    samples = sample_n_fn(math_ops.reduce_prod(sample_shape))
+    batch_event_shape = array_ops.shape(samples)[1:]
+    final_shape = array_ops.concat([sample_shape, batch_event_shape], 0)
+    return array_ops.reshape(samples, final_shape)
+
+
+def bernoulli_sample(probs=None, logits=None, dtype=dtypes.int32,
+                     sample_shape=(), seed=None):
+  """Samples from Bernoulli distribution."""
+  if probs is None:
+    probs = math_ops.sigmoid(logits, name="probs")
+  else:
+    probs = ops.convert_to_tensor(probs, name="probs")
+  batch_shape_tensor = array_ops.shape(probs)
+  def _sample_n(n):
+    """Sample vector of Bernoullis."""
+    new_shape = array_ops.concat([[n], batch_shape_tensor], 0)
+    uniform = random_ops.random_uniform(
+        new_shape, seed=seed, dtype=probs.dtype)
+    return math_ops.cast(math_ops.less(uniform, probs), dtype)
+  return _call_sampler(_sample_n, sample_shape)
+
+
+def categorical_sample(logits, dtype=dtypes.int32,
+                       sample_shape=(), seed=None):
+  """Samples from categorical distribution."""
+  logits = ops.convert_to_tensor(logits, name="logits")
+  event_size = array_ops.shape(logits)[-1]
+  batch_shape_tensor = array_ops.shape(logits)[:-1]
+  def _sample_n(n):
+    """Sample vector of categoricals."""
+    if logits.shape.ndims == 2:
+      logits_2d = logits
+    else:
+      logits_2d = array_ops.reshape(logits, [-1, event_size])
+    sample_dtype = dtypes.int64 if logits.dtype.size > 4 else dtypes.int32
+    draws = random_ops.multinomial(
+        logits_2d, n, seed=seed, output_dtype=sample_dtype)
+    draws = array_ops.reshape(
+        array_ops.transpose(draws),
+        array_ops.concat([[n], batch_shape_tensor], 0))
+    return math_ops.cast(draws, dtype)
+  return _call_sampler(_sample_n, sample_shape)
+
+
 def _unstack_ta(inp):
   return tensor_array_ops.TensorArray(
       dtype=inp.dtype, size=array_ops.shape(inp)[0],
@@ -307,14 +368,14 @@ class ScheduledEmbeddingTrainingHelper(TrainingHelper):
     with ops.name_scope(name, "ScheduledEmbeddingTrainingHelperSample",
                         [time, outputs, state]):
       # Return -1s where we did not sample, and sample_ids elsewhere
-      select_sampler = bernoulli.Bernoulli(
-          probs=self._sampling_probability, dtype=dtypes.bool)
-      select_sample = select_sampler.sample(
-          sample_shape=self.batch_size, seed=self._scheduling_seed)
-      sample_id_sampler = categorical.Categorical(logits=outputs)
+      select_sample = bernoulli_sample(
+          probs=self._sampling_probability,
+          dtype=dtypes.bool,
+          sample_shape=self.batch_size,
+          seed=self._scheduling_seed)
       return array_ops.where(
           select_sample,
-          sample_id_sampler.sample(seed=self._seed),
+          categorical_sample(logits=outputs, seed=self._seed),
           gen_array_ops.fill([self.batch_size], -1))
 
   def next_inputs(self, time, outputs, state, sample_ids, name=None):
@@ -425,8 +486,10 @@ class ScheduledOutputTrainingHelper(TrainingHelper):
   def sample(self, time, outputs, state, name=None):
     with ops.name_scope(name, "ScheduledOutputTrainingHelperSample",
                         [time, outputs, state]):
-      sampler = bernoulli.Bernoulli(probs=self._sampling_probability)
-      return sampler.sample(sample_shape=self.batch_size, seed=self._seed)
+      return bernoulli_sample(
+          probs=self._sampling_probability,
+          sample_shape=self.batch_size,
+          seed=self._seed)
 
   def next_inputs(self, time, outputs, state, sample_ids, name=None):
     with ops.name_scope(name, "ScheduledOutputTrainingHelperNextInputs",
@@ -610,8 +673,7 @@ class SampleEmbeddingHelper(GreedyEmbeddingHelper):
     else:
       logits = outputs / self._softmax_temperature
 
-    sample_id_sampler = categorical.Categorical(logits=logits)
-    sample_ids = sample_id_sampler.sample(seed=self._seed)
+    sample_ids = categorical_sample(logits=logits, seed=self._seed)
 
     return sample_ids
 
-- 
GitLab


From 586c507c879c816eb001bfb32a14deb7053640a6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 09:58:03 -0800
Subject: [PATCH 0185/2345] [tfgan] Adds GANEstimator wrapper that allows
 backpropagation to input latent space.

PiperOrigin-RevId: 227871387
---
 tensorflow/contrib/gan/BUILD                  |  36 +++
 .../contrib/gan/python/estimator/__init__.py  |   5 +-
 .../estimator/python/latent_gan_estimator.py  |  28 +++
 .../python/latent_gan_estimator_impl.py       | 205 ++++++++++++++++++
 .../python/latent_gan_estimator_test.py       | 119 ++++++++++
 5 files changed, 392 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/contrib/gan/python/estimator/python/latent_gan_estimator.py
 create mode 100644 tensorflow/contrib/gan/python/estimator/python/latent_gan_estimator_impl.py
 create mode 100644 tensorflow/contrib/gan/python/estimator/python/latent_gan_estimator_test.py

diff --git a/tensorflow/contrib/gan/BUILD b/tensorflow/contrib/gan/BUILD
index 0626875b76..08df38ad27 100644
--- a/tensorflow/contrib/gan/BUILD
+++ b/tensorflow/contrib/gan/BUILD
@@ -107,6 +107,7 @@ py_library(
     deps = [
         ":gan_estimator",
         ":head",
+        ":latent_gan_estimator",
         ":stargan_estimator",
         ":tpu_gan_estimator",
         "//tensorflow/python:util",
@@ -643,6 +644,41 @@ py_test(
     ],
 )
 
+py_library(
+    name = "latent_gan_estimator",
+    srcs = [
+        "python/estimator/python/latent_gan_estimator.py",
+        "python/estimator/python/latent_gan_estimator_impl.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":train",
+        "//tensorflow/python:clip_ops",
+        "//tensorflow/python:gradients",
+        "//tensorflow/python:random_ops",
+        "//tensorflow/python:summary",
+        "//tensorflow/python:training_util",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python/estimator:estimator_py",
+    ],
+)
+
+py_test(
+    name = "latent_gan_estimator_test",
+    srcs = [
+        "python/estimator/python/latent_gan_estimator_test.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":latent_gan_estimator",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:training",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python/estimator:run_config",
+        "//tensorflow/python/ops/losses",
+    ],
+)
+
 py_library(
     name = "sliced_wasserstein",
     srcs = [
diff --git a/tensorflow/contrib/gan/python/estimator/__init__.py b/tensorflow/contrib/gan/python/estimator/__init__.py
index 03a639165b..430266555b 100644
--- a/tensorflow/contrib/gan/python/estimator/__init__.py
+++ b/tensorflow/contrib/gan/python/estimator/__init__.py
@@ -26,11 +26,13 @@ from __future__ import print_function
 # pylint: disable=unused-import,wildcard-import
 from tensorflow.contrib.gan.python.estimator.python import gan_estimator
 from tensorflow.contrib.gan.python.estimator.python import head
+from tensorflow.contrib.gan.python.estimator.python import latent_gan_estimator
 from tensorflow.contrib.gan.python.estimator.python import stargan_estimator
 from tensorflow.contrib.gan.python.estimator.python import tpu_gan_estimator
 
 from tensorflow.contrib.gan.python.estimator.python.gan_estimator import *
 from tensorflow.contrib.gan.python.estimator.python.head import *
+from tensorflow.contrib.gan.python.estimator.python.latent_gan_estimator import *
 from tensorflow.contrib.gan.python.estimator.python.stargan_estimator import *
 from tensorflow.contrib.gan.python.estimator.python.tpu_gan_estimator import *
 # pylint: enable=unused-import,wildcard-import
@@ -41,7 +43,8 @@ _allowed_symbols = ([
     'gan_estimator',
     'stargan_estimator',
     'tpu_gan_estimator',
+    'latent_gan_estimator',
     'head',
 ] + gan_estimator.__all__ + stargan_estimator.__all__ + head.__all__ +
-                    tpu_gan_estimator.__all__)
+                    tpu_gan_estimator.__all__ + latent_gan_estimator.__all__)
 remove_undocumented(__name__, _allowed_symbols)
diff --git a/tensorflow/contrib/gan/python/estimator/python/latent_gan_estimator.py b/tensorflow/contrib/gan/python/estimator/python/latent_gan_estimator.py
new file mode 100644
index 0000000000..4e164e2416
--- /dev/null
+++ b/tensorflow/contrib/gan/python/estimator/python/latent_gan_estimator.py
@@ -0,0 +1,28 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""`tf.Learn` components for `Train Input Estimator`."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.gan.python.estimator.python import latent_gan_estimator_impl
+# pylint: disable=wildcard-import
+from tensorflow.contrib.gan.python.estimator.python.latent_gan_estimator_impl import *
+# pylint: enable=wildcard-import
+from tensorflow.python.util.all_util import remove_undocumented
+
+__all__ = latent_gan_estimator_impl.__all__
+remove_undocumented(__name__, __all__)
diff --git a/tensorflow/contrib/gan/python/estimator/python/latent_gan_estimator_impl.py b/tensorflow/contrib/gan/python/estimator/python/latent_gan_estimator_impl.py
new file mode 100644
index 0000000000..f5afc77319
--- /dev/null
+++ b/tensorflow/contrib/gan/python/estimator/python/latent_gan_estimator_impl.py
@@ -0,0 +1,205 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Implements an estimator wrapper that allows training the input latent space.
+
+This file implements a latent gan estimator that wraps around a previously
+trained GAN. The latent gan estimator trains a single variable z, representing
+the hidden latent distribution that is the 'noise' input to the GAN. By training
+z, the inpainting estimator can move around the latent z space towards
+minimizing a specific loss function.
+
+The latent gan estimator has a few key differences from a normal estimator.
+
+First: the variables in the estimator should not be saved, as we are not
+updating the original GAN and are only adding a new z variable that is meant
+to be different for each run. In order to do distributed training using
+train_and_evaluate, the Tensorflow RunConfig is expected to save checkpoints
+by having either save_checkpoints_steps or save_checkpoints_secs saved.
+To avoid this conflict, we purposely set the save_checkpoints_steps value in
+the RunConfig to be one step more than the total number of steps that the
+inpainter estimator will run.
+
+Second: we need to specify warm start settings, as we are reloading the
+GAN model into a different graph (specifically, one with a new z variable).
+The warm start settings defined below reload all GAN variables and ignore the
+new z variable (and the optimizer).
+
+Usage:
+
+  def _generator(net, mode):
+    ...
+
+  def _discriminator(net, condition, mode):
+    ...
+
+  def _loss(gan_model, features, labels, add_summaries):
+    ...
+
+  def optimizer():
+    ...
+
+  params = {<required params>}
+  config = tf.estimator.RunConfig()
+  tmp_dir = path/to/output/storage
+
+  estimator = latent_gan_estimator.get_latent_gan_estimator(
+      _generator, _discriminator, _loss, optimizer, params, config, tmp_dir)
+
+  def input_fn():
+    ...
+
+  estimator.train(input_fn=input_fn)
+
+See latent_gan_estimator_test.py or tensorflow_models/gan/face_inpainting for
+further examples.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import functools
+from tensorflow.contrib.gan.python import train as tfgan_train
+from tensorflow.python.estimator import estimator
+from tensorflow.python.estimator import model_fn as model_fn_lib
+from tensorflow.python.ops import clip_ops
+from tensorflow.python.ops import gradients_impl
+from tensorflow.python.ops import random_ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.summary import summary
+from tensorflow.python.training import training_util
+
+
+INPUT_NAME = 'new_var_z_input'  # The name for the new z space input variable.
+OPTIMIZER_NAME = 'latent_gan_optimizer'  # The name for the new optimizer vars.
+
+__all__ = [
+    'get_latent_gan_estimator',
+]
+
+
+def _get_latent_gan_model_fn(generator_fn, discriminator_fn, loss_fn,
+                             optimizer):
+  """Sets up a model function that wraps around a given GAN."""
+  def model_fn(features, labels, mode, params):
+    """Model function defining an inpainting estimator."""
+    batch_size = params['batch_size']
+    z_shape = [batch_size] + params['z_shape']
+    add_summaries = params['add_summaries']
+    input_clip = params['input_clip']
+
+    z = variable_scope.get_variable(
+        name=INPUT_NAME, initializer=random_ops.truncated_normal(z_shape),
+        constraint=lambda x: clip_ops.clip_by_value(x, -input_clip, input_clip))
+
+    generator = functools.partial(generator_fn, mode=mode)
+    discriminator = functools.partial(discriminator_fn, mode=mode)
+    gan_model = tfgan_train.gan_model(generator_fn=generator,
+                                      discriminator_fn=discriminator,
+                                      real_data=labels,
+                                      generator_inputs=z,
+                                      check_shapes=False)
+
+    loss = loss_fn(gan_model, features, labels, add_summaries)
+
+    # Use a variable scope to make sure that estimator variables dont cause
+    # save/load problems when restoring from ckpts.
+    with variable_scope.variable_scope(OPTIMIZER_NAME):
+      opt = optimizer(learning_rate=params['learning_rate'],
+                      **params['opt_kwargs'])
+      train_op = opt.minimize(
+          loss=loss, global_step=training_util.get_or_create_global_step(),
+          var_list=[z])
+
+    if add_summaries:
+      z_grads = gradients_impl.gradients(loss, z)
+      summary.scalar('z_loss/z_grads', clip_ops.global_norm(z_grads))
+      summary.scalar('z_loss/loss', loss)
+
+    return model_fn_lib.EstimatorSpec(mode=mode,
+                                      predictions=gan_model.generated_data,
+                                      loss=loss,
+                                      train_op=train_op)
+  return model_fn
+
+
+def get_latent_gan_estimator(generator_fn, discriminator_fn, loss_fn,
+                             optimizer, params, config, ckpt_dir,
+                             warmstart_options=True):
+  """Gets an estimator that passes gradients to the input.
+
+  This function takes in a generator and adds a trainable z variable that is
+  used as input to this generator_fn. The generator itself is treated as a black
+  box through which gradients can pass through without updating any weights. The
+  result is a trainable way to traverse the GAN latent space. The loss_fn is
+  used to actually train the z variable. The generator_fn and discriminator_fn
+  should be previously trained by the tfgan library (on reload, the variables
+  are expected to follow the tfgan format. It may be possible to use the
+  latent gan estimator with entirely custom GANs that do not use the tfgan
+  library as long as the appropriate variables are wired properly).
+
+  Args:
+    generator_fn: a function defining a Tensorflow graph for a GAN generator.
+      The weights defined in this graph should already be defined in the given
+      checkpoint location. Should have 'mode' as an argument.
+    discriminator_fn: a function defining a Tensorflow graph for a GAN
+      discriminator. Should have 'mode' as an argument.
+    loss_fn: a function defining a Tensorflow graph for a GAN loss. Takes in a
+      GANModel tuple, features, labels, and add_summaries as inputs.
+    optimizer: a tf.Optimizer or a function that returns a tf.Optimizer with no
+      inputs.
+   params: An object containing the following parameters:
+      - batch_size: an int indicating the size of the training batch.
+      - z_shape: the desired shape of the input z values (not counting batch).
+      - learning_rate: a scalar or function defining a learning rate applied to
+        optimizer.
+      - input_clip: the amount to clip the x training variable by.
+      - add_summaries: whether or not to add summaries.
+      - opt_kwargs: optimizer kwargs.
+    config: tf.RunConfig. Should point model to output dir and should indicate
+     whether to save checkpoints (to avoid saving checkpoints, set
+     save_checkpoints_steps to a number larger than the number of train steps).
+     The model_dir field in the RunConfig should point to a directory WITHOUT
+     any saved checkpoints.
+    ckpt_dir: the directory where the model checkpoints live. The checkpoint is
+     used to warm start the underlying GAN. This should NOT be the same as
+     config.model_dir.
+    warmstart_options: boolean, None, or a WarmStartSettings object. If set to
+      True, uses a default WarmStartSettings object. If set to False or None,
+      does not use warm start. If using a custom WarmStartSettings object, make
+      sure that new variables are properly accounted for when reloading the
+      underlying GAN. Defaults to True.
+  Returns:
+    An estimator spec defining a GAN input training estimator.
+  """
+  model_fn = _get_latent_gan_model_fn(generator_fn, discriminator_fn,
+                                      loss_fn, optimizer)
+
+  if isinstance(warmstart_options, estimator.WarmStartSettings):
+    ws = warmstart_options
+  elif warmstart_options:
+    # Default WarmStart loads all variable names except INPUT_NAME and
+    # OPTIMIZER_NAME.
+    var_regex = '^(?!.*(%s|%s).*)' % (INPUT_NAME, OPTIMIZER_NAME)
+    ws = estimator.WarmStartSettings(ckpt_to_initialize_from=ckpt_dir,
+                                     vars_to_warm_start=var_regex)
+  else:
+    ws = None
+
+  if 'opt_kwargs' not in params:
+    params['opt_kwargs'] = {}
+
+  return estimator.Estimator(model_fn=model_fn, config=config, params=params,
+                             warm_start_from=ws)
diff --git a/tensorflow/contrib/gan/python/estimator/python/latent_gan_estimator_test.py b/tensorflow/contrib/gan/python/estimator/python/latent_gan_estimator_test.py
new file mode 100644
index 0000000000..ac139e532e
--- /dev/null
+++ b/tensorflow/contrib/gan/python/estimator/python/latent_gan_estimator_test.py
@@ -0,0 +1,119 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for latent_gan_estimator.
+
+See g3.tp.tensorflow.contrib.gan.python.estimator.python.latent_gan_estimator.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import tempfile
+import numpy as np
+from tensorflow.contrib.gan.python.estimator.python import latent_gan_estimator
+from tensorflow.python.estimator import run_config as run_config
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.ops.losses import losses
+from tensorflow.python.platform import test
+from tensorflow.python.training import training
+
+
+class TrainInputEstimatorTest(test.TestCase):
+
+  def test_get_input_training_estimator(self):
+    """Integration test to make sure the input_training_estimator works."""
+
+    # Create dummy test input tensors.
+    true_features = np.reshape(np.random.uniform(size=100), (10, 10))
+    true_labels = np.reshape(np.random.uniform(size=100), (5, 20))
+    expected_z_output = [[1, -1], [-1, 1]]
+
+    # Fill out required parameters randomly, includes optimizer kwargs.
+    params = {
+        'batch_size': 2,
+        'z_shape': [2],
+        'learning_rate': 1.0,
+        'input_clip': 1.0,
+        'add_summaries': False,
+        'opt_kwargs': {
+            'beta1': 0.1
+        }
+    }
+
+    input_z_shape = [params['batch_size']] + params['z_shape']
+
+    # Create dummy model functions that represent an underlying GANEstimator and
+    # the input training wrapper. Make sure that everything is wired up
+    # correctly in the internals of each dummy function.
+    def _generator(net, mode):
+      """The generator function will get the newly created z variable."""
+      del mode
+      self.assertSequenceEqual(net.shape, input_z_shape)
+      gen_dummy_var = variable_scope.get_variable(
+          name='generator_dummy_variable',
+          initializer=array_ops.ones(input_z_shape))
+      return net * gen_dummy_var
+
+    def _discriminator(net, condition, mode):
+      """The discriminator function will get either the z variable or labels."""
+      del condition, mode
+      try:
+        self.assertSequenceEqual(net.shape, true_labels.shape)
+      except AssertionError:
+        self.assertSequenceEqual(net.shape, input_z_shape)
+      return net
+
+    def _loss(gan_model, features, labels, _):
+      """Make sure that features and labels are passed in from input."""
+      self.assertTrue(np.array_equal(features, true_features))
+      self.assertTrue(np.array_equal(labels, true_labels))
+      return losses.absolute_difference(expected_z_output,
+                                        gan_model.generated_data)
+
+    optimizer = training.AdamOptimizer
+
+    # We are not loading checkpoints, so set the corresponding directory to a
+    # dummy directories.
+    tmp_dir = tempfile.mkdtemp()
+    config = run_config.RunConfig(model_dir=tmp_dir,
+                                  save_summary_steps=None,
+                                  save_checkpoints_steps=1,
+                                  save_checkpoints_secs=None)
+
+    # Get the estimator. Disable warm start so that there is no attempted
+    # checkpoint reloading.
+    estimator = latent_gan_estimator.get_latent_gan_estimator(
+        _generator, _discriminator, _loss, optimizer, params, config, tmp_dir,
+        warmstart_options=None)
+
+    # Train for a few steps.
+    def dummy_input():
+      return true_features, true_labels
+    estimator.train(input_fn=dummy_input, steps=10)
+
+    # Make sure the generator variables did not change, but the z variables did
+    # change.
+    self.assertTrue(np.array_equal(
+        estimator.get_variable_value('Generator/generator_dummy_variable'),
+        np.ones(input_z_shape)))
+    self.assertTrue(np.array_equal(
+        estimator.get_variable_value('new_var_z_input'),
+        expected_z_output))
+
+
+if __name__ == '__main__':
+  test.main()
-- 
GitLab


From df01a441b477b20346c5cbf8248018798b75ff0f Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Fri, 4 Jan 2019 10:04:36 -0800
Subject: [PATCH 0186/2345] Decoupled ResourceVariable from RefVariable

* RefVariable._try_guard_against_uninitialized_dependencies and related
  methods are now functions used by both variables subclasses
* Removed implicit chaining of ResourceVariable ->Tensor convertor with
  that of RefVariable. Previously, ResourceVariable._dense_var_to_tensor
  would return NotImplemented if called with a different dtype. This
  effectively delegated the conversion to RefVariable._TensorConversionFunction
  which allowed compatible but not exactly matching dtypes. The change
  makes this behavior explicit in ResourceVariable._dense_var_to_tensor.

PiperOrigin-RevId: 227872550
---
 .../feature_column/feature_column_v2_test.py  |   8 +-
 .../python/ops/resource_variable_ops.py       | 109 ++++--
 tensorflow/python/ops/variables.py            | 316 +++++++++---------
 3 files changed, 251 insertions(+), 182 deletions(-)

diff --git a/tensorflow/python/feature_column/feature_column_v2_test.py b/tensorflow/python/feature_column/feature_column_v2_test.py
index a247425369..253e4c322b 100644
--- a/tensorflow/python/feature_column/feature_column_v2_test.py
+++ b/tensorflow/python/feature_column/feature_column_v2_test.py
@@ -2015,7 +2015,7 @@ class LinearModelTest(test.TestCase):
       }
       model(features)
       for var in model.variables:
-        self.assertTrue(isinstance(var, variables_lib.RefVariable))
+        self.assertIsInstance(var, variables_lib.VariableV1)
       variable_names = [var.name for var in model.variables]
       self.assertItemsEqual([
           'linear_model/dense_feature_bucketized/weights:0',
@@ -4010,7 +4010,7 @@ class FunctionalInputLayerTest(test.TestCase):
       self.assertEqual(0, len(cols_to_vars[dense_feature_bucketized]))
       self.assertEqual(1, len(cols_to_vars[some_embedding_column]))
       self.assertIsInstance(cols_to_vars[some_embedding_column][0],
-                            variables_lib.Variable)
+                            variables_lib.VariableV1)
       self.assertAllEqual(cols_to_vars[some_embedding_column][0].shape, [5, 10])
 
   @test_util.run_deprecated_v1
@@ -6839,7 +6839,7 @@ class EmbeddingColumnTest(test.TestCase):
     self.assertItemsEqual(('dense_features/aaa_embedding/embedding_weights:0',),
                           tuple([v.name for v in global_vars]))
     for v in global_vars:
-      self.assertTrue(isinstance(v, variables_lib.RefVariable))
+      self.assertIsInstance(v, variables_lib.Variable)
     trainable_vars = ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)
     self.assertItemsEqual(('dense_features/aaa_embedding/embedding_weights:0',),
                           tuple([v.name for v in trainable_vars]))
@@ -7732,7 +7732,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
         ['aaa_bbb_shared_embedding:0', 'ccc_ddd_shared_embedding:0'],
         tuple([v.name for v in global_vars]))
     for v in global_vars:
-      self.assertTrue(isinstance(v, variables_lib.RefVariable))
+      self.assertIsInstance(v, variables_lib.Variable)
     trainable_vars = ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)
     if trainable:
       self.assertItemsEqual(
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 3e26463ddd..ede4bc0078 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -36,6 +36,7 @@ from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import gen_resource_variable_ops
 from tensorflow.python.ops import gen_state_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variables
 # go/tf-wildcard-import
 # pylint: disable=wildcard-import
@@ -160,8 +161,7 @@ def shape_safe_assign_variable_handle(handle, shape, value, name=None):
                                                       name=name)
 
 
-# TODO(apassos) make this be variables.Variable
-class ResourceVariable(variables.RefVariable):
+class ResourceVariable(variables.VariableV1):
   """Variable based on resource handles.
 
   See the [Variables How To](https://tensorflow.org/guide/variables)
@@ -297,6 +297,15 @@ class ResourceVariable(variables.RefVariable):
           dtype=dtype,
           constraint=constraint)
 
+  def __repr__(self):
+    if context.executing_eagerly() and not self._in_graph_mode:
+      return "<tf.Variable '%s' shape=%s dtype=%s, numpy=%s>" % (
+          self.name, self.get_shape(), self.dtype.name,
+          ops.numpy_text(self.read_value(), is_repr=True))
+    else:
+      return "<tf.Variable '%s' shape=%s dtype=%s>" % (
+          self.name, self.get_shape(), self.dtype.name)
+
   # pylint: disable=unused-argument
   def _init_from_args(self,
                       initial_value=None,
@@ -437,12 +446,15 @@ class ResourceVariable(variables.RefVariable):
                 gen_resource_variable_ops.var_is_initialized_op(self._handle))
           if initial_value is not None:
             with ops.name_scope("Assign") as n, ops.colocate_with(self._handle):
+              # pylint: disable=protected-access
               self._initializer_op = (
                   gen_resource_variable_ops.assign_variable_op(
                       self._handle,
-                      self._try_guard_against_uninitialized_dependencies(
+                      variables._try_guard_against_uninitialized_dependencies(
+                          name,
                           initial_value),
                       name=n))
+              # pylint: enable=protected-access
           with ops.name_scope("Read"), ops.colocate_with(self._handle):
             # Manually assign reads to the handle's device to avoid log
             # messages.
@@ -683,6 +695,10 @@ class ResourceVariable(variables.RefVariable):
     """The op for this variable."""
     return self._handle.op
 
+  @property
+  def trainable(self):
+    return self._trainable
+
   def eval(self, session=None):
     """Evaluates and returns the value of this variable."""
     if context.executing_eagerly():
@@ -818,10 +834,6 @@ class ResourceVariable(variables.RefVariable):
     return ResourceVariable(
         variable_def=variable_def, import_scope=import_scope)
 
-  def _ref(self):
-    """Unsupported."""
-    raise NotImplementedError("ResourceVariable does not implement _ref()")
-
   def set_shape(self, shape):
     """Unsupported."""
     raise NotImplementedError("ResourceVariable does not implement set_shape()")
@@ -994,6 +1006,55 @@ class ResourceVariable(variables.RefVariable):
         self.handle, sparse_delta.indices,
         ops.convert_to_tensor(sparse_delta.values, self.dtype), name=name))
 
+  def batch_scatter_update(self, sparse_delta, use_locking=False, name=None):
+    """Assigns `IndexedSlices` to this variable batch-wise.
+
+    Analogous to `batch_gather`. This assumes that this variable and the
+    sparse_delta IndexedSlices have a series of leading dimensions that are the
+    same for all of them, and the updates are performed on the last dimension of
+    indices. In other words, the dimensions should be the following:
+
+    `num_prefix_dims = sparse_delta.indices.ndims - 1`
+    `batch_dim = num_prefix_dims + 1`
+    `sparse_delta.updates.shape = sparse_delta.indices.shape + var.shape[
+         batch_dim:]`
+
+    where
+
+    `sparse_delta.updates.shape[:num_prefix_dims]`
+    `== sparse_delta.indices.shape[:num_prefix_dims]`
+    `== var.shape[:num_prefix_dims]`
+
+    And the operation performed can be expressed as:
+
+    `var[i_1, ..., i_n,
+         sparse_delta.indices[i_1, ..., i_n, j]] = sparse_delta.updates[
+            i_1, ..., i_n, j]`
+
+    When sparse_delta.indices is a 1D tensor, this operation is equivalent to
+    `scatter_update`.
+
+    To avoid this operation one can looping over the first `ndims` of the
+    variable and using `scatter_update` on the subtensors that result of slicing
+    the first dimension. This is a valid option for `ndims = 1`, but less
+    efficient than this implementation.
+
+    Args:
+      sparse_delta: `IndexedSlices` to be assigned to this variable.
+      use_locking: If `True`, use locking during the operation.
+      name: the name of the operation.
+
+    Returns:
+      A `Tensor` that will hold the new value of this variable after
+      the scattered subtraction has completed.
+
+    Raises:
+      ValueError: if `sparse_delta` is not an `IndexedSlices`.
+    """
+    return self._lazy_read(state_ops.batch_scatter_update(
+        self, sparse_delta.indices, sparse_delta.values,
+        use_locking=use_locking, name=name))
+
   def scatter_nd_sub(self, indices, updates, name=None):
     """Applies sparse subtraction to individual values or slices in a Variable.
 
@@ -1178,8 +1239,10 @@ class ResourceVariable(variables.RefVariable):
 
   def _dense_var_to_tensor(self, dtype=None, name=None, as_ref=False):
     del name
-    if dtype is not None and dtype != self.dtype:
-      return NotImplemented
+    if dtype is not None and not dtype.is_compatible_with(self.dtype):
+      raise ValueError(
+          "Incompatible type conversion requested to type {!r} for variable "
+          "of type {!r}".format(dtype.name, self.dtype.name))
     if as_ref:
       return self.read_value().op.inputs[0]
     else:
@@ -1231,6 +1294,12 @@ def _dense_var_to_tensor(var, dtype=None, name=None, as_ref=False):
   return var._dense_var_to_tensor(dtype=dtype, name=name, as_ref=as_ref)  # pylint: disable=protected-access
 
 
+# Register a conversion function which reads the value of the variable,
+# allowing instances of the class to be used as tensors.
+ops.register_tensor_conversion_function(ResourceVariable, _dense_var_to_tensor)
+ops.register_dense_tensor_like_type(ResourceVariable)
+
+
 class _UnreadVariable(ResourceVariable):
   """Represents a future for a read of a variable.
 
@@ -1291,7 +1360,7 @@ class _UnreadVariable(ResourceVariable):
     """The op for this variable."""
     return self._parent_op
 
-ops.register_tensor_conversion_function(_UnreadVariable, _dense_var_to_tensor)
+
 ops.register_dense_tensor_like_type(_UnreadVariable)
 
 
@@ -1390,29 +1459,15 @@ class _MixedPrecisionVariable(ResourceVariable):
 
   def _dense_var_to_tensor(self, dtype=None, name=None, as_ref=False):
     del name
-    dtype = dtype or self.read_dtype
-    if dtype != self.read_dtype or as_ref:
+    if (dtype is not None and
+        not dtype.is_compatible_with(self.read_dtype) or as_ref):
       return NotImplemented
-    else:
-      res = self.value()
-    return res
+    return self.value()
 
   def _should_act_as_resource_variable(self):
     """To pass resource_variable_ops.is_resource_variable check."""
     pass
 
-# Register a conversion function which reads the value of the variable,
-# allowing instances of the class to be used as tensors.
-
-# Note: registering for Variable after ResourceVariable because inheritance will
-# otherwise lead to the wrong behavior.
-ops.register_tensor_conversion_function(ResourceVariable, _dense_var_to_tensor)
-ops.register_tensor_conversion_function(
-    variables.Variable, variables.Variable._TensorConversionFunction)  # pylint: disable=protected-access
-
-# pylint: disable=protected-access
-ops.register_dense_tensor_like_type(ResourceVariable)
-
 
 @ops.RegisterGradient("ReadVariableOp")
 def _ReadGrad(_, grad):
diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index 657f64deaa..a4967e1c0f 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -59,21 +59,6 @@ def _make_getter(captured_getter, captured_previous):
   return getter
 
 
-def _has_cycle(op, path):
-  """Detect cycles in the dependencies of `initial_value`."""
-  if op.name in path:
-    return True
-  path.add(op.name)
-  for op_input in op.inputs:
-    if _has_cycle(op_input.op, path):
-      return True
-  for op_control_input in op.control_inputs:
-    if _has_cycle(op_control_input, path):
-      return True
-  path.remove(op.name)
-  return False
-
-
 @tf_export("VariableSynchronization")
 class VariableSynchronization(enum.Enum):
   """Indicates when a distributed variable will be synced.
@@ -1041,6 +1026,10 @@ class Variable(six.with_metaclass(VariableMetaclass,
     """Alias of `Variable.shape`."""
     return self.shape
 
+  def _gather_saveables_for_checkpoint(self):
+    """For implementing `Checkpointable`. This object is saveable on its own."""
+    return {checkpointable.VARIABLE_VALUE_KEY: self}
+
   def to_proto(self, export_scope=None):
     """Converts a `Variable` to a `VariableDef` protocol buffer.
 
@@ -1144,6 +1133,9 @@ class Variable(six.with_metaclass(VariableMetaclass,
         return None
 
 
+Variable._OverloadAllOperators()  # pylint: disable=protected-access
+
+
 @tf_export(v1=["Variable"])
 class VariableV1(Variable):
   """See the [Variables Guide](https://tensorflow.org/guide/variables).
@@ -1582,7 +1574,8 @@ class RefVariable(VariableV1):
         # using their initialized_value() method.
         self._initializer_op = state_ops.assign(
             self._variable,
-            self._try_guard_against_uninitialized_dependencies(
+            _try_guard_against_uninitialized_dependencies(
+                name,
                 self._initial_value),
             validate_shape=validate_shape).op
 
@@ -2161,134 +2154,6 @@ class RefVariable(VariableV1):
     else:
       return v.value()
 
-  def _gather_saveables_for_checkpoint(self):
-    """For implementing `Checkpointable`. This object is saveable on its own."""
-    return {checkpointable.VARIABLE_VALUE_KEY: self}
-
-  def _try_guard_against_uninitialized_dependencies(self, initial_value):
-    """Attempt to guard against dependencies on uninitialized variables.
-
-    Replace references to variables in `initial_value` with references to the
-    variable's initialized values. The initialized values are essentially
-    conditional TensorFlow graphs that return a variable's value if it is
-    initialized or its `initial_value` if it hasn't been initialized. This
-    replacement is done on a best effort basis:
-
-    - If the `initial_value` graph contains cycles, we don't do any
-      replacements for that graph.
-    - If the variables that `initial_value` depends on are not present in the
-      `GLOBAL_VARIABLES` or `LOCAL_VARIABLES` we don't replace them.
-
-    In these cases, it is up to the caller to ensure that the `initial_value`
-    graph uses initialized variables or that they guard access to variables
-    using their `initialized_value` method.
-
-    Args:
-      initial_value: `Tensor`. The initial value.
-    Returns:
-      A `Tensor` suitable to initialize a variable.
-    Raises:
-      TypeError: If `initial_value` is not a `Tensor`.
-    """
-    if not isinstance(initial_value, ops.Tensor):
-      raise TypeError("initial_value needs to be a Tensor: %s" % initial_value)
-
-    # Don't modify initial_value if it contains any cyclic dependencies.
-    if _has_cycle(initial_value.op, path=set()):
-      return initial_value
-
-    return self._safe_initial_value_from_tensor(initial_value, op_cache={})
-
-  def _safe_initial_value_from_tensor(self, tensor, op_cache):
-    """Replace dependencies on variables with their initialized values.
-
-    Args:
-      tensor: A `Tensor`. The tensor to replace.
-      op_cache: A dict mapping operation names to `Operation`s. Used to memoize
-        the results so as to avoid creating redundant operations.
-    Returns:
-      A `Tensor` compatible with `tensor`. Any inputs that lead to variable
-      values will be replaced with a corresponding graph that uses the
-      variable's initialized values. This is done on a best-effort basis. If no
-      modifications need to be made then `tensor` will be returned unchanged.
-    """
-    op = tensor.op
-    new_op = op_cache.get(op.name)
-    if new_op is None:
-      new_op = self._safe_initial_value_from_op(op, op_cache)
-      op_cache[op.name] = new_op
-    return new_op.outputs[tensor.value_index]
-
-  def _safe_initial_value_from_op(self, op, op_cache):
-    """Replace dependencies on variables with their initialized values.
-
-    Args:
-      op: An `Operation`. The operation to replace.
-      op_cache: A dict mapping operation names to `Operation`s. Used to memoize
-        the results so as to avoid creating redundant operations.
-    Returns:
-      An `Operation` compatible with `op`. Any inputs that lead to variable
-      values will be replaced with a corresponding graph that uses the
-      variable's initialized values. This is done on a best-effort basis. If no
-      modifications need to be made then `op` will be returned unchanged.
-    """
-    op_type = op.node_def.op
-    if op_type in ("IsVariableInitialized", "VarIsInitializedOp",
-                   "ReadVariableOp"):
-      return op
-
-    # Attempt to find the initialized_value of any variable reference / handles.
-    # TODO(b/70206927): Fix handling of ResourceVariables.
-    if op_type in ("Variable", "VariableV2", "VarHandleOp"):
-      initialized_value = self._find_initialized_value_for_variable(op)
-      return op if initialized_value is None else initialized_value.op
-
-    # Recursively build initializer expressions for inputs.
-    modified = False
-    new_op_inputs = []
-    for op_input in op.inputs:
-      new_op_input = self._safe_initial_value_from_tensor(op_input, op_cache)
-      new_op_inputs.append(new_op_input)
-      modified = modified or (new_op_input != op_input)
-
-    # If at least one input was modified, replace the op.
-    if modified:
-      new_op_type = op_type
-      if new_op_type == "RefSwitch":
-        new_op_type = "Switch"
-      new_op_name = op.node_def.name + "_" + self.name
-      new_op_name = new_op_name.replace(":", "_")
-      return self.graph.create_op(
-          new_op_type, new_op_inputs,
-          op._output_types,  # pylint: disable=protected-access
-          name=new_op_name, attrs=op.node_def.attr)
-
-    return op
-
-  def _find_initialized_value_for_variable(self, variable_op):
-    """Find the initialized value for a variable op.
-
-    To do so, lookup the variable op in the variables collection.
-
-    Args:
-      variable_op: A variable `Operation`.
-    Returns:
-      A `Tensor` representing the initialized value for the variable or `None`
-      if the initialized value could not be found.
-    """
-    try:
-      var_names = [variable_op.node_def.name, variable_op.node_def.name + ":0"]
-      for collection_name in (ops.GraphKeys.GLOBAL_VARIABLES,
-                              ops.GraphKeys.LOCAL_VARIABLES):
-        for var in self.graph.get_collection(collection_name):
-          if var.name in var_names:
-            return var.initialized_value()
-    except AttributeError:
-      # Return None when an incomplete user-defined variable type was put in
-      # the collection.
-      return None
-    return None
-
   # NOTE(mrry): This enables the Variable's overloaded "right" binary
   # operators to run when the left operand is an ndarray, because it
   # accords the Variable class higher priority than an ndarray, or a
@@ -2441,6 +2306,151 @@ class RefVariable(VariableV1):
     return self._save_slice_info
 
 
+def _try_guard_against_uninitialized_dependencies(name, initial_value):
+  """Attempt to guard against dependencies on uninitialized variables.
+
+  Replace references to variables in `initial_value` with references to the
+  variable's initialized values. The initialized values are essentially
+  conditional TensorFlow graphs that return a variable's value if it is
+  initialized or its `initial_value` if it hasn't been initialized. This
+  replacement is done on a best effort basis:
+
+  - If the `initial_value` graph contains cycles, we don't do any
+    replacements for that graph.
+  - If the variables that `initial_value` depends on are not present in the
+    `GLOBAL_VARIABLES` or `LOCAL_VARIABLES` we don't replace them.
+
+  In these cases, it is up to the caller to ensure that the `initial_value`
+  graph uses initialized variables or that they guard access to variables
+  using their `initialized_value` method.
+
+  Args:
+    name: Variable name.
+    initial_value: `Tensor`. The initial value.
+  Returns:
+    A `Tensor` suitable to initialize a variable.
+  Raises:
+    TypeError: If `initial_value` is not a `Tensor`.
+  """
+  if not isinstance(initial_value, ops.Tensor):
+    raise TypeError("initial_value needs to be a Tensor: %s" % initial_value)
+
+  # Don't modify initial_value if it contains any cyclic dependencies.
+  if _has_cycle(initial_value.op, path=set()):
+    return initial_value
+  return _safe_initial_value_from_tensor(name, initial_value, op_cache={})
+
+
+def _has_cycle(op, path):
+  """Detect cycles in the dependencies of `initial_value`."""
+  if op.name in path:
+    return True
+  path.add(op.name)
+  for op_input in op.inputs:
+    if _has_cycle(op_input.op, path):
+      return True
+  for op_control_input in op.control_inputs:
+    if _has_cycle(op_control_input, path):
+      return True
+  path.remove(op.name)
+  return False
+
+
+def _safe_initial_value_from_tensor(name, tensor, op_cache):
+  """Replace dependencies on variables with their initialized values.
+
+  Args:
+    name: Variable name.
+    tensor: A `Tensor`. The tensor to replace.
+    op_cache: A dict mapping operation names to `Operation`s. Used to memoize
+      the results so as to avoid creating redundant operations.
+  Returns:
+    A `Tensor` compatible with `tensor`. Any inputs that lead to variable
+    values will be replaced with a corresponding graph that uses the
+    variable's initialized values. This is done on a best-effort basis. If no
+    modifications need to be made then `tensor` will be returned unchanged.
+  """
+  op = tensor.op
+  new_op = op_cache.get(op.name)
+  if new_op is None:
+    new_op = _safe_initial_value_from_op(name, op, op_cache)
+    op_cache[op.name] = new_op
+  return new_op.outputs[tensor.value_index]
+
+
+def _safe_initial_value_from_op(name, op, op_cache):
+  """Replace dependencies on variables with their initialized values.
+
+  Args:
+    name: Variable name.
+    op: An `Operation`. The operation to replace.
+    op_cache: A dict mapping operation names to `Operation`s. Used to memoize
+      the results so as to avoid creating redundant operations.
+  Returns:
+    An `Operation` compatible with `op`. Any inputs that lead to variable
+    values will be replaced with a corresponding graph that uses the
+    variable's initialized values. This is done on a best-effort basis. If no
+    modifications need to be made then `op` will be returned unchanged.
+  """
+  op_type = op.node_def.op
+  if op_type in ("IsVariableInitialized", "VarIsInitializedOp",
+                 "ReadVariableOp"):
+    return op
+
+  # Attempt to find the initialized_value of any variable reference / handles.
+  # TODO(b/70206927): Fix handling of ResourceVariables.
+  if op_type in ("Variable", "VariableV2", "VarHandleOp"):
+    initialized_value = _find_initialized_value_for_variable(op)
+    return op if initialized_value is None else initialized_value.op
+
+  # Recursively build initializer expressions for inputs.
+  modified = False
+  new_op_inputs = []
+  for op_input in op.inputs:
+    new_op_input = _safe_initial_value_from_tensor(name, op_input, op_cache)
+    new_op_inputs.append(new_op_input)
+    modified = modified or (new_op_input != op_input)
+
+  # If at least one input was modified, replace the op.
+  if modified:
+    new_op_type = op_type
+    if new_op_type == "RefSwitch":
+      new_op_type = "Switch"
+    new_op_name = op.node_def.name + "_" + name
+    new_op_name = new_op_name.replace(":", "_")
+    return op.graph.create_op(
+        new_op_type, new_op_inputs,
+        op._output_types,  # pylint: disable=protected-access
+        name=new_op_name, attrs=op.node_def.attr)
+
+  return op
+
+
+def _find_initialized_value_for_variable(variable_op):
+  """Find the initialized value for a variable op.
+
+  To do so, lookup the variable op in the variables collection.
+
+  Args:
+    variable_op: A variable `Operation`.
+  Returns:
+    A `Tensor` representing the initialized value for the variable or `None`
+    if the initialized value could not be found.
+  """
+  try:
+    var_names = [variable_op.node_def.name, variable_op.node_def.name + ":0"]
+    for collection_name in (ops.GraphKeys.GLOBAL_VARIABLES,
+                            ops.GraphKeys.LOCAL_VARIABLES):
+      for var in variable_op.graph.get_collection(collection_name):
+        if var.name in var_names:
+          return var.initialized_value()
+  except AttributeError:
+    # Return None when an incomplete user-defined variable type was put in
+    # the collection.
+    return None
+  return None
+
+
 class PartitionedVariable(object):
   """A container for partitioned `Variable` objects.
 
@@ -2659,6 +2669,15 @@ class PartitionedVariable(object):
       return assign_list
     return [assign.op for assign in assign_list]
 
+
+# Register a conversion function which reads the value of the variable,
+# allowing instances of the class to be used as tensors.
+ops.register_tensor_conversion_function(
+    RefVariable,
+    RefVariable._TensorConversionFunction)  # pylint: disable=protected-access
+ops.register_dense_tensor_like_type(RefVariable)
+
+
 @tf_export(v1=["global_variables"])
 def global_variables(scope=None):
   """Returns global variables.
@@ -2982,12 +3001,7 @@ def report_uninitialized_variables(var_list=None,
         # uninitialized variables.
         return array_ops.boolean_mask(variable_names_tensor, variables_mask)
 
-# pylint: disable=protected-access
-Variable._OverloadAllOperators()
 
 ops.register_tensor_conversion_function(
-    PartitionedVariable, PartitionedVariable._TensorConversionFunction)
-# pylint: enable=protected-access
-
-
-ops.register_dense_tensor_like_type(Variable)
+    PartitionedVariable,
+    PartitionedVariable._TensorConversionFunction)  # pylint: disable=protected-access
-- 
GitLab


From 2a3d6401a3f21e4c9d043d90da48444d6e24e057 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 10:09:12 -0800
Subject: [PATCH 0187/2345] Remove NHWC format restriction while using
 batch_group_count for convolutions of depthwise backprop filters

PiperOrigin-RevId: 227873314
---
 tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc b/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
index b0bc764030..6b049f653c 100644
--- a/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
+++ b/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
@@ -428,11 +428,8 @@ xla::StatusOr<xla::XlaOp> MakeXlaBackpropFilterConvOp(
   int n_dim = GetTensorBatchDimIndex(num_dims, attrs.data_format);
   int c_dim = GetTensorFeatureDimIndex(num_dims, attrs.data_format);
 
-  // The conversion logic below assumes that the data format is NHWC, so we also
-  // check that here.
   bool use_batch_group_count =
-      filter_tensor_shape.dim_size(num_dims - 1) == 1 && attrs.depthwise &&
-      attrs.data_format == FORMAT_NHWC;
+      filter_tensor_shape.dim_size(num_dims - 1) == 1 && attrs.depthwise;
 
   std::vector<std::pair<int64, int64>> padding(attrs.num_spatial_dims);
   std::vector<int64> rhs_dilation(attrs.num_spatial_dims);
-- 
GitLab


From 07816cc1e13c02d7ad579d1279a5fca883126c35 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 10:09:49 -0800
Subject: [PATCH 0188/2345] Fix markdown rendering.

PiperOrigin-RevId: 227873426
---
 tensorflow/python/ops/variables.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index a4967e1c0f..f9fc72a6da 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -306,6 +306,7 @@ class Variable(six.with_metaclass(VariableMetaclass,
 
   Here replacing adding `use_resource=True` when constructing the variable will
   fix any nondeterminism issues:
+
   ```
   v = tf.Variable(True, use_resource=True)
   tf.cond(v, lambda: v.assign(False), my_false_fn)
-- 
GitLab


From daa75aff18fd42598d1fb68f13da13042e886c07 Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Fri, 4 Jan 2019 10:17:31 -0800
Subject: [PATCH 0189/2345] Allows TensorListScatter to scatter at
 non-contiguous indices to make it consistent with TensorArray.scatter.

PiperOrigin-RevId: 227874653
---
 tensorflow/core/kernels/list_kernels.h        | 31 ++++++++++----
 .../python/kernel_tests/list_ops_test.py      | 41 +++++++++++++++++++
 .../kernel_tests/tensor_array_ops_test.py     |  5 +--
 tensorflow/python/ops/list_ops.py             | 14 +++++--
 4 files changed, 77 insertions(+), 14 deletions(-)

diff --git a/tensorflow/core/kernels/list_kernels.h b/tensorflow/core/kernels/list_kernels.h
index 99b4f42084..559e14b03c 100644
--- a/tensorflow/core/kernels/list_kernels.h
+++ b/tensorflow/core/kernels/list_kernels.h
@@ -521,14 +521,31 @@ class TensorListScatter : public OpKernel {
                     "Specified a list with shape ", element_shape.DebugString(),
                     " from a tensor with shape ", output_shape.DebugString()));
     output_list.element_shape = element_shape;
-    output_list.tensors.reserve(indices.NumElements());
+
+    OP_REQUIRES(c, indices.NumElements() == input_tensor.shape().dim_size(0),
+                errors::InvalidArgument(
+                    "Invalid number of rows in input tensor. Expected: ",
+                    indices.NumElements(),
+                    " Actual: ", input_tensor.shape().dim_size(0)));
+
+    // Validate indices and resize output_list.tensors to fit the highest index.
+    {
+      size_t list_size = 0;
+      for (int index = 0; index < indices.NumElements(); ++index) {
+        const int i = indices.flat<int32>()(index);
+        OP_REQUIRES(c, i >= 0,
+                    errors::InvalidArgument(
+                        "Indices in TensorListScatter must all be positive."));
+        if (i >= list_size) {
+          list_size = i + 1;
+        }
+      }
+      output_list.tensors.resize(list_size, Tensor(DT_INVALID));
+    }
+
     for (int index = 0; index < indices.NumElements(); ++index) {
       const int i = indices.flat<int32>()(index);
-      OP_REQUIRES(c, i < input_tensor.shape().dim_size(0),
-                  errors::InvalidArgument(
-                      "Trying to scatter index ", i, " from tensor with ",
-                      input_tensor.shape().dim_size(0), " rows."));
-      Tensor tmp = input_tensor.Slice(i, i + 1);
+      Tensor tmp = input_tensor.Slice(index, index + 1);
       TensorShape tmp_shape = tmp.shape();
       tmp_shape.RemoveDim(0);
       OP_REQUIRES(c, tmp.CopyFrom(tmp, tmp_shape),
@@ -541,7 +558,7 @@ class TensorListScatter : public OpKernel {
       // many small ondes.
       aligned.flat<T>().device(c->eigen_device<Device>()) =
           tmp.unaligned_flat<T>();
-      output_list.tensors.push_back(aligned);
+      std::swap(output_list.tensors[i], aligned);
     }
     output_tensor->scalar<Variant>()() = std::move(output_list);
   }
diff --git a/tensorflow/python/kernel_tests/list_ops_test.py b/tensorflow/python/kernel_tests/list_ops_test.py
index 054ac36755..fc0e270667 100644
--- a/tensorflow/python/kernel_tests/list_ops_test.py
+++ b/tensorflow/python/kernel_tests/list_ops_test.py
@@ -290,6 +290,47 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       t = list_ops.tensor_list_gather(l, [], element_dtype=dtypes.float32)
       self.evaluate(t)
 
+  def testGatherGradWithNonContiguousIndices(self):
+    with backprop.GradientTape(persistent=True) as tape:
+      t = constant_op.constant([1.0, 2.0, 3.0])
+      l = list_ops.tensor_list_from_tensor(t, element_shape=[])
+      c = constant_op.constant(5.0)
+      tape.watch(c)
+      l = list_ops.tensor_list_set_item(l, 1, c)
+      t = list_ops.tensor_list_gather(l, [1], element_dtype=dtypes.float32)
+      self.assertAllEqual(self.evaluate(t), [5.0])
+      s = t[0] * t[0]
+    dt = tape.gradient(s, c)
+    self.assertAllEqual(self.evaluate(dt), 10.0)
+    dl = tape.gradient(t, l)
+    dl_length = list_ops.tensor_list_length(dl)
+    self.assertAllEqual(self.evaluate(dl_length), 3)
+
+  def testScatterOutputListSize(self):
+    c0 = constant_op.constant([1.0, 2.0])
+    l = list_ops.tensor_list_scatter(
+        c0, [1, 3], ops.convert_to_tensor([], dtype=dtypes.int32))
+    # TensorListScatter should return a list with size largest index + 1.
+    self.assertEqual(self.evaluate(list_ops.tensor_list_length(l)), 4)
+
+  def testScatterWithInvalidRowsInInputTensorFails(self):
+    c0 = constant_op.constant([1.0, 2.0])
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        "Invalid number of rows in input tensor. Expected: 3 Actual: 2"):
+      l = list_ops.tensor_list_scatter(
+          c0, [1, 0, 2], ops.convert_to_tensor([], dtype=dtypes.int32))
+      self.evaluate(l)
+
+  def testScatterWithNegativeIndicesFails(self):
+    c0 = constant_op.constant([1.0, 2.0])
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        "Indices in TensorListScatter must all be positive."):
+      l = list_ops.tensor_list_scatter(
+          c0, [-1, -2], ops.convert_to_tensor([], dtype=dtypes.int32))
+      self.evaluate(l)
+
   def testScatterGrad(self):
     with backprop.GradientTape() as tape:
       c0 = constant_op.constant([1.0, 2.0])
diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
index 9fb5791048..303e9a006a 100644
--- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py
+++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
@@ -1359,7 +1359,6 @@ class TensorArrayTest(test.TestCase):
   def testSkipEagerTensorArrayEvalEmptyWithDefault(self):
     self._testTensorArrayEvalEmptyWithDefault()
 
-  @test_util.disable_control_flow_v2("b/117943286")
   @test_util.run_v1_only("b/117943489")
   def testSkipEagerTensorArrayScatterReadAndGradients(self):
     with self.session(use_gpu=True) as session:
@@ -1387,8 +1386,8 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual([10.0, -10.0], read_vals[1])
       self.assertAllEqual([[2.0, 3.0], [4.0, 5.0]], grad_vals[0])
 
-  @test_util.disable_control_flow_v2("b/117943286")
-  @test_util.run_v1_only("b/117943286")
+  @test_util.disable_control_flow_v2("b/118890905")
+  @test_util.run_v1_only("b/118890905")
   def testTensorArrayWriteGatherAndGradients(self):
     with self.session(use_gpu=True) as session:
       ta = tensor_array_ops.TensorArray(
diff --git a/tensorflow/python/ops/list_ops.py b/tensorflow/python/ops/list_ops.py
index 7bb9ca89e0..df928ea85d 100644
--- a/tensorflow/python/ops/list_ops.py
+++ b/tensorflow/python/ops/list_ops.py
@@ -200,10 +200,16 @@ def _TensorListResizeGrad(op, dlist):
 
 @ops.RegisterGradient("TensorListGather")
 def _TensorListGatherGrad(op, dtensor):
-  _, indices = op.inputs
-  return gen_list_ops.tensor_list_scatter(
-      tensor=dtensor, indices=indices,
-      element_shape=ops.convert_to_tensor(-1, dtype=dtypes.int32)), None
+  input_list, indices = op.inputs
+  dlist = gen_list_ops.tensor_list_scatter(
+      tensor=dtensor,
+      indices=indices,
+      element_shape=ops.convert_to_tensor(-1, dtype=dtypes.int32))
+  # TensorListScatter returns a list with size `max(indices) + 1`
+  # so we manually resize it to match the size of the input list.
+  input_list_size = gen_list_ops.tensor_list_length(input_list)
+  dlist = gen_list_ops.tensor_list_resize(dlist, input_list_size)
+  return dlist, None
 
 
 @ops.RegisterGradient("TensorListScatter")
-- 
GitLab


From d183818d4d5e85a8f469402d273f8f03fa21f4b6 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Fri, 4 Jan 2019 10:24:02 -0800
Subject: [PATCH 0190/2345] Update the math formula for LogLoss to match the
 parameters.

PiperOrigin-RevId: 227875597
---
 tensorflow/python/keras/losses.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index e6db795bb8..6a428b28a9 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -503,7 +503,7 @@ class CategoricalHinge(Loss):
 class LogLoss(Loss):
   """Computes the log loss between `y_true` and `y_pred`.
 
-  logloss = -y log(p) - (1-y) log(1-p)
+  logloss = - y_true * log(y_pred) - (1 - y_true) * log(1 - y_pred)
 
   Usage:
 
-- 
GitLab


From 59d5fbcc1a5e677387aac2bbf919faddf0e43377 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 4 Jan 2019 10:36:25 -0800
Subject: [PATCH 0191/2345] Automated rollback of commit
 64596a68235709e7f3246d0b120a4146521a998b

PiperOrigin-RevId: 227877471
---
 tensorflow/lite/experimental/swift/BUILD      | 100 +++++
 tensorflow/lite/experimental/swift/LICENSE    | 202 ++++++++++
 tensorflow/lite/experimental/swift/README.md  |  54 +++
 .../swift/Sources/Interpreter.swift           | 265 ++++++++++++++
 .../swift/Sources/InterpreterError.swift      |  99 +++++
 .../swift/Sources/InterpreterOptions.swift    |  29 ++
 .../experimental/swift/Sources/Model.swift    |  40 ++
 .../Sources/QuantizationParameters.swift      |  38 ++
 .../experimental/swift/Sources/Tensor.swift   | 138 +++++++
 .../Configs/TensorFlowLite.tulsigen           |  57 +++
 .../project.tulsiconf                         |  14 +
 .../project.pbxproj                           | 345 ++++++++++++++++++
 .../TensorFlowLiteApp/AppDelegate.swift       |  24 ++
 .../Array+TensorFlowLite.swift                |  22 ++
 .../AppIcon.appiconset/Contents.json          |  98 +++++
 .../Assets.xcassets/Contents.json             |   6 +
 .../Base.lproj/LaunchScreen.storyboard        |  44 +++
 .../Base.lproj/Main.storyboard                |  95 +++++
 .../Data+TensorFlowLite.swift                 |  13 +
 .../TensorFlowLiteApp/Info.plist              |  46 +++
 .../TensorFlowLiteApp/ViewController.swift    | 299 +++++++++++++++
 .../swift/Tests/InterpreterOptionsTests.swift |  54 +++
 .../swift/Tests/InterpreterTests.swift        | 315 ++++++++++++++++
 .../experimental/swift/Tests/ModelTests.swift |  59 +++
 .../Tests/QuantizationParametersTests.swift   |  43 +++
 .../swift/Tests/TensorTests.swift             |  83 +++++
 .../tools/pip_package/pip_smoke_test.py       |  36 +-
 27 files changed, 2603 insertions(+), 15 deletions(-)
 create mode 100644 tensorflow/lite/experimental/swift/BUILD
 create mode 100644 tensorflow/lite/experimental/swift/LICENSE
 create mode 100644 tensorflow/lite/experimental/swift/README.md
 create mode 100644 tensorflow/lite/experimental/swift/Sources/Interpreter.swift
 create mode 100644 tensorflow/lite/experimental/swift/Sources/InterpreterError.swift
 create mode 100644 tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift
 create mode 100644 tensorflow/lite/experimental/swift/Sources/Model.swift
 create mode 100644 tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift
 create mode 100644 tensorflow/lite/experimental/swift/Sources/Tensor.swift
 create mode 100644 tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
 create mode 100644 tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift
 create mode 100644 tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift
 create mode 100644 tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift
 create mode 100644 tensorflow/lite/experimental/swift/Tests/ModelTests.swift
 create mode 100644 tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift
 create mode 100644 tensorflow/lite/experimental/swift/Tests/TensorTests.swift

diff --git a/tensorflow/lite/experimental/swift/BUILD b/tensorflow/lite/experimental/swift/BUILD
new file mode 100644
index 0000000000..3bd288a1e1
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/BUILD
@@ -0,0 +1,100 @@
+# TensorFlow Lite for Swift.
+
+package(default_visibility = ["//visibility:private"])
+
+licenses(["notice"])  # Apache 2.0
+
+exports_files(["LICENSE"])
+
+load("@build_bazel_rules_apple//apple:ios.bzl", "ios_application", "ios_unit_test")
+load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library")
+
+MINIMUM_OS_VERSION = "9.0"
+
+SWIFT_COPTS = [
+    "-wmo",
+]
+
+swift_library(
+    name = "TensorFlowLite",
+    srcs = glob(["Sources/*.swift"]),
+    copts = SWIFT_COPTS,
+    module_name = "TensorFlowLite",
+    tags = ["manual"],
+    deps = [
+        "//tensorflow/lite/experimental/c:c_api",
+    ],
+)
+
+ios_unit_test(
+    name = "TensorFlowLiteTests",
+    size = "small",
+    minimum_os_version = MINIMUM_OS_VERSION,
+    tags = [
+        "manual",
+        # DISABLED: Following sanitizer tests are not supported by iOS test targets.
+        "noasan",
+        "nomsan",
+        "notsan",
+    ],
+    deps = [":TensorFlowLiteTestsLib"],
+)
+
+swift_library(
+    name = "TensorFlowLiteTestsLib",
+    testonly = 1,
+    srcs = glob(["Tests/*.swift"]),
+    copts = SWIFT_COPTS,
+    tags = ["manual"],
+    deps = [
+        ":TensorFlowLite",
+        ":TestResources",
+    ],
+)
+
+objc_library(
+    name = "TestResources",
+    resources = [
+        "//tensorflow/lite:testdata/add.bin",
+        "//tensorflow/lite:testdata/add_quantized.bin",
+        "//tensorflow/lite:testdata/multi_add.bin",
+    ],
+    tags = ["manual"],
+)
+
+ios_application(
+    name = "TensorFlowLiteApp",
+    app_icons = glob(["TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/**"]),
+    bundle_id = "com.tensorflow.lite.swift.TensorFlowLite",
+    families = [
+        "ipad",
+        "iphone",
+    ],
+    infoplists = ["TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist"],
+    launch_storyboard = "TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard",
+    minimum_os_version = MINIMUM_OS_VERSION,
+    sdk_frameworks = [
+        "CoreGraphics",
+    ],
+    tags = ["manual"],
+    deps = [":TensorFlowLiteAppLib"],
+)
+
+swift_library(
+    name = "TensorFlowLiteAppLib",
+    srcs = glob(["TestApps/TensorFlowLiteApp/TensorFlowLiteApp/*.swift"]),
+    tags = ["manual"],
+    deps = [
+        ":TensorFlowLite",
+        ":TensorFlowLiteAppResources",
+    ],
+)
+
+objc_library(
+    name = "TensorFlowLiteAppResources",
+    storyboards = glob([
+        "TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/*.storyboard",
+    ]),
+    tags = ["manual"],
+    deps = [":TestResources"],
+)
diff --git a/tensorflow/lite/experimental/swift/LICENSE b/tensorflow/lite/experimental/swift/LICENSE
new file mode 100644
index 0000000000..d645695673
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/LICENSE
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/tensorflow/lite/experimental/swift/README.md b/tensorflow/lite/experimental/swift/README.md
new file mode 100644
index 0000000000..deb16c1e91
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/README.md
@@ -0,0 +1,54 @@
+# TensorFlow Lite for Swift
+
+[TensorFlow Lite](https://www.tensorflow.org/lite/) is TensorFlow's lightweight
+solution for Swift developers. It enables low-latency inference of on-device
+machine learning models with a small binary size and fast performance supporting
+hardware acceleration.
+
+## Getting Started
+
+### Bazel
+
+In your `BUILD` file, add the `TensorFlowLite` dependency:
+
+```python
+swift_library(
+  # ...
+  deps = [
+      "//tensorflow/lite/swift:TensorFlowLite",
+  ],
+)
+```
+
+In your Swift files, import the module:
+
+```swift
+import TensorFlowLite
+```
+
+### Tulsi
+
+Open the `TensorFlowLite.tulsiproj` using the [TulsiApp](https://github.com/bazelbuild/tulsi) or by
+running the [`generate_xcodeproj.sh`](https://github.com/bazelbuild/tulsi/blob/master/src/tools/generate_xcodeproj.sh)
+script:
+
+```shell
+generate_xcodeproj.sh --genconfig tensorflow/lite/swift/TensorFlowLite.tulsiproj:TensorFlowLite --outputfolder ~/path/to/generated/TensorFlowLite.xcodeproj
+```
+
+### CocoaPods
+
+Add the following to your `Podfile`:
+
+```ruby
+use_frameworks!
+pod 'TensorFlowLiteSwift'
+```
+
+Then, run `pod install`.
+
+In your Swift files, import the module:
+
+```swift
+import TensorFlowLite
+```
diff --git a/tensorflow/lite/experimental/swift/Sources/Interpreter.swift b/tensorflow/lite/experimental/swift/Sources/Interpreter.swift
new file mode 100644
index 0000000000..a14b5966b1
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Sources/Interpreter.swift
@@ -0,0 +1,265 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+import TensorFlowLiteCAPI
+
+/// A TensorFlow Lite interpreter that performs inference from a given model.
+public final class Interpreter {
+
+  /// The `TFL_Interpreter` C pointer type represented as an `UnsafePointer<TFL_Interpreter>`.
+  private typealias CInterpreter = OpaquePointer
+
+  /// Total number of input tensors associated with the model.
+  public var inputTensorCount: Int {
+    return Int(TFL_InterpreterGetInputTensorCount(cInterpreter))
+  }
+
+  /// Total number of output tensors associated with the model.
+  public var outputTensorCount: Int {
+    return Int(TFL_InterpreterGetOutputTensorCount(cInterpreter))
+  }
+
+  /// The underlying `TFL_Interpreter` C pointer.
+  private var cInterpreter: CInterpreter?
+
+  /// Creates a new model interpreter instance.
+  ///
+  /// - Parameters:
+  ///   - modelPath: Local file path to a TensorFlow Lite model.
+  ///   - options: Custom configurations for the interpreter. The default is `nil` indicating that
+  ///       interpreter will determine the configuration options.
+  /// - Throws: An error if the model could not be loaded or the interpreter could not be created.
+  public init(modelPath: String, options: InterpreterOptions? = nil) throws {
+    guard let model = Model(filePath: modelPath) else { throw InterpreterError.failedToLoadModel }
+
+    let cInterpreterOptions: OpaquePointer? = try options.map { options in
+      guard let cOptions = TFL_NewInterpreterOptions() else {
+        throw InterpreterError.failedToCreateInterpreter
+      }
+      if let threadCount = options.threadCount, threadCount > 0 {
+        TFL_InterpreterOptionsSetNumThreads(cOptions, Int32(threadCount))
+      }
+      if options.isErrorLoggingEnabled {
+        TFL_InterpreterOptionsSetErrorReporter(
+          cOptions,
+          { (_, format, arguments) in
+            guard let cFormat = format,
+                  let message = String(cFormat: cFormat, arguments: arguments)
+            else {
+              return
+            }
+            print(String(describing: InterpreterError.tensorFlowLiteError(message)))
+          },
+          nil
+        )
+      }
+      return cOptions
+    }
+    defer { TFL_DeleteInterpreterOptions(cInterpreterOptions) }
+
+    guard let cInterpreter = TFL_NewInterpreter(model.cModel, cInterpreterOptions) else {
+      throw InterpreterError.failedToCreateInterpreter
+    }
+    self.cInterpreter = cInterpreter
+  }
+
+  deinit {
+    TFL_DeleteInterpreter(cInterpreter)
+  }
+
+  /// Invokes the interpreter to perform inference from the loaded graph.
+  ///
+  /// - Throws: An error if the model was not ready because tensors were not allocated.
+  public func invoke() throws {
+    guard TFL_InterpreterInvoke(cInterpreter) == kTfLiteOk else {
+      // TODO(b/117510052): Determine which error to throw.
+      throw InterpreterError.allocateTensorsRequired
+    }
+  }
+
+  /// Returns the input tensor at the given index.
+  ///
+  /// - Parameters:
+  ///   - index: The index for the input tensor.
+  /// - Throws: An error if the index is invalid or the tensors have not been allocated.
+  /// - Returns: The input tensor at the given index.
+  public func input(at index: Int) throws -> Tensor {
+    let maxIndex = inputTensorCount - 1
+    guard case 0...maxIndex = index else {
+      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
+    }
+    guard let cTensor = TFL_InterpreterGetInputTensor(cInterpreter, Int32(index)),
+          let bytes = TFL_TensorData(cTensor),
+          let nameCString = TFL_TensorName(cTensor)
+    else {
+      throw InterpreterError.allocateTensorsRequired
+    }
+    guard let dataType = TensorDataType(type: TFL_TensorType(cTensor)) else {
+      throw InterpreterError.invalidTensorDataType
+    }
+
+    let name = String(cString: nameCString)
+    let rank = TFL_TensorNumDims(cTensor)
+    let dimensions = (0..<rank).map { Int(TFL_TensorDim(cTensor, $0)) }
+    let shape = TensorShape(dimensions)
+    let byteCount = TFL_TensorByteSize(cTensor)
+    let data = Data(bytes: bytes, count: byteCount)
+    let cQuantizationParams = TFL_TensorQuantizationParams(cTensor)
+    let scale = cQuantizationParams.scale
+    let zeroPoint = Int(cQuantizationParams.zero_point)
+    var quantizationParameters: QuantizationParameters? = nil
+    if scale != 0.0 {
+      // TODO(b/117510052): Update this check once the TfLiteQuantizationParams struct has a mode.
+      quantizationParameters = QuantizationParameters(scale: scale, zeroPoint: zeroPoint)
+    }
+    let tensor = Tensor(
+      name: name,
+      dataType: dataType,
+      shape: shape,
+      data: data,
+      quantizationParameters: quantizationParameters
+    )
+    return tensor
+  }
+
+  /// Returns the output tensor at the given index.
+  ///
+  /// - Parameters:
+  ///   - index: The index for the output tensor.
+  /// - Throws: An error if the index is invalid, tensors haven't been allocated, or interpreter
+  ///     hasn't been invoked for models that dynamically compute output tensors based on the values
+  ///     of its input tensors.
+  /// - Returns: The output tensor at the given index.
+  public func output(at index: Int) throws -> Tensor {
+    let maxIndex = outputTensorCount - 1
+    guard case 0...maxIndex = index else {
+      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
+    }
+    guard let cTensor = TFL_InterpreterGetOutputTensor(cInterpreter, Int32(index)),
+          let bytes = TFL_TensorData(cTensor),
+          let nameCString = TFL_TensorName(cTensor)
+    else {
+      // TODO(b/117510052): Determine which error to throw.
+      throw InterpreterError.invokeInterpreterRequired
+    }
+    guard let dataType = TensorDataType(type: TFL_TensorType(cTensor)) else {
+      throw InterpreterError.invalidTensorDataType
+    }
+
+    let name = String(cString: nameCString)
+    let rank = TFL_TensorNumDims(cTensor)
+    let dimensions = (0..<rank).map { Int(TFL_TensorDim(cTensor, $0)) }
+    let shape = TensorShape(dimensions)
+    let byteCount = TFL_TensorByteSize(cTensor)
+    let data = Data(bytes: bytes, count: byteCount)
+    let cQuantizationParams = TFL_TensorQuantizationParams(cTensor)
+    let scale = cQuantizationParams.scale
+    let zeroPoint = Int(cQuantizationParams.zero_point)
+    var quantizationParameters: QuantizationParameters? = nil
+    if scale != 0.0 {
+      // TODO(b/117510052): Update this check once the TfLiteQuantizationParams struct has a mode.
+      quantizationParameters = QuantizationParameters(scale: scale, zeroPoint: zeroPoint)
+    }
+    let tensor = Tensor(
+      name: name,
+      dataType: dataType,
+      shape: shape,
+      data: data,
+      quantizationParameters: quantizationParameters
+    )
+    return tensor
+  }
+
+  /// Resizes the input tensor at the given index to the specified tensor shape.
+  ///
+  /// - Note: After resizing an input tensor, the client **must** explicitly call
+  ///     `allocateTensors()` before attempting to access the resized tensor data or invoking the
+  ///     interpreter to perform inference.
+  /// - Parameters:
+  ///   - index: The index for the input tensor.
+  ///   - shape: The shape that the input tensor should be resized to.
+  /// - Throws: An error if the input tensor at the given index could not be resized.
+  public func resizeInput(at index: Int, to shape: TensorShape) throws {
+    let maxIndex = inputTensorCount - 1
+    guard case 0...maxIndex = index else {
+      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
+    }
+    guard TFL_InterpreterResizeInputTensor(
+            cInterpreter,
+            Int32(index),
+            shape.int32Dimensions,
+            Int32(shape.rank)
+          ) == kTfLiteOk
+    else {
+      throw InterpreterError.failedToResizeInputTensor(index: index)
+    }
+  }
+
+  /// Copies the given data to the input tensor at the given index.
+  ///
+  /// - Parameters:
+  ///   - data: The data to be copied to the input tensor's data buffer.
+  ///   - index: The index for the input tensor.
+  /// - Throws: An error if the `data.count` does not match the input tensor's `data.count` or if
+  ///     the given index is invalid.
+  /// - Returns: The input tensor with the copied data.
+  @discardableResult
+  public func copy(_ data: Data, toInputAt index: Int) throws -> Tensor {
+    let maxIndex = inputTensorCount - 1
+    guard case 0...maxIndex = index else {
+      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
+    }
+    guard let cTensor = TFL_InterpreterGetInputTensor(cInterpreter, Int32(index)) else {
+      throw InterpreterError.allocateTensorsRequired
+    }
+
+    let byteCount = TFL_TensorByteSize(cTensor)
+    guard data.count == byteCount else {
+      throw InterpreterError.invalidTensorDataCount(provided: data.count, required: byteCount)
+    }
+
+    let status = data.withUnsafeBytes { TFL_TensorCopyFromBuffer(cTensor, $0, data.count) }
+    guard status == kTfLiteOk else { throw InterpreterError.failedToCopyDataToInputTensor }
+    return try input(at: index)
+  }
+
+  /// Allocates memory for all input tensors based on their `TensorShape`s.
+  ///
+  /// - Note: This is a relatively expensive operation and should only be called after creating the
+  ///     interpreter and/or resizing any input tensors.
+  /// - Throws: An error if memory could not be allocated for the input tensors.
+  public func allocateTensors() throws {
+    guard TFL_InterpreterAllocateTensors(cInterpreter) == kTfLiteOk else {
+      throw InterpreterError.failedToAllocateTensors
+    }
+  }
+}
+
+// MARK: - Extensions
+
+extension String {
+  /// Returns a new `String` initialized by using the given format C array as a template into which
+  /// the remaining argument values are substituted according to the user’s default locale.
+  ///
+  /// - Note: Returns `nil` if a new `String` could not be constructed from the given values.
+  /// - Parameters:
+  ///   - cFormat: The format C array as a template for substituting values.
+  ///   - arguments: A C pointer to a `va_list` of arguments to substitute into `cFormat`.
+  init?(cFormat: UnsafePointer<CChar>, arguments: CVaListPointer) {
+    var buffer: UnsafeMutablePointer<CChar>?
+    guard vasprintf(&buffer, cFormat, arguments) != 0, let cString = buffer else { return nil }
+    self.init(validatingUTF8: cString)
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/Sources/InterpreterError.swift b/tensorflow/lite/experimental/swift/Sources/InterpreterError.swift
new file mode 100644
index 0000000000..5de58b997a
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Sources/InterpreterError.swift
@@ -0,0 +1,99 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+
+/// TensorFlow Lite interpreter errors.
+public enum InterpreterError: Error {
+  case invalidTensorIndex(index: Int, maxIndex: Int)
+  case invalidTensorDataCount(provided: Int, required: Int)
+  case invalidTensorDataType
+  case failedToLoadModel
+  case failedToCreateInterpreter
+  case failedToResizeInputTensor(index: Int)
+  case failedToCopyDataToInputTensor
+  case failedToAllocateTensors
+  case allocateTensorsRequired
+  case invokeInterpreterRequired
+  case tensorFlowLiteError(String)
+}
+
+// MARK: - Extensions
+
+extension InterpreterError: LocalizedError {
+  /// Localized description of the interpreter error.
+  public var errorDescription: String? {
+    switch self {
+    case .invalidTensorIndex(let index, let maxIndex):
+      return "Invalid tensor index \(index), max index is \(maxIndex)."
+    case .invalidTensorDataCount(let providedCount, let requiredCount):
+      return "Provided data count \(providedCount) must match the required count \(requiredCount)."
+    case .invalidTensorDataType:
+      return "Tensor data type is unsupported or could not be determined because of a model error."
+    case .failedToLoadModel:
+      return "Failed to load the given model."
+    case .failedToCreateInterpreter:
+      return "Failed to create the interpreter."
+    case .failedToResizeInputTensor(let index):
+      return "Failed to resize input tesnor at index \(index)."
+    case .failedToCopyDataToInputTensor:
+      return "Failed to copy data to input tensor."
+    case .failedToAllocateTensors:
+      return "Failed to allocate memory for input tensors."
+    case .allocateTensorsRequired:
+      return "Must call allocateTensors()."
+    case .invokeInterpreterRequired:
+      return "Must call invoke()."
+    case .tensorFlowLiteError(let message):
+      return "TensorFlow Lite Error: \(message)"
+    }
+  }
+}
+
+extension InterpreterError: CustomStringConvertible {
+  /// Textual representation of the TensorFlow Lite interpreter error.
+  public var description: String {
+    return errorDescription ?? "Unknown error."
+  }
+}
+
+#if swift(>=4.2)
+extension InterpreterError: Equatable {}
+#else
+extension InterpreterError: Equatable {
+  public static func == (lhs: InterpreterError, rhs: InterpreterError) -> Bool {
+    switch (lhs, rhs) {
+    case (.invalidTensorDataType, .invalidTensorDataType),
+         (.failedToLoadModel, .failedToLoadModel),
+         (.failedToCreateInterpreter, .failedToCreateInterpreter),
+         (.failedToAllocateTensors, .failedToAllocateTensors),
+         (.allocateTensorsRequired, .allocateTensorsRequired),
+         (.invokeInterpreterRequired, .invokeInterpreterRequired):
+      return true
+    case (.invalidTensorIndex(let lhsIndex, let lhsMaxIndex),
+          .invalidTensorIndex(let rhsIndex, let rhsMaxIndex)):
+      return lhsIndex == rhsIndex && lhsMaxIndex == rhsMaxIndex
+    case (.invalidTensorDataCount(let lhsProvidedCount, let lhsRequiredCount),
+          .invalidTensorDataCount(let rhsProvidedCount, let rhsRequiredCount)):
+      return lhsProvidedCount == rhsProvidedCount && lhsRequiredCount == rhsRequiredCount
+    case (.failedToResizeInputTensor(let lhsIndex), .failedToResizeInputTensor(let rhsIndex)):
+      return lhsIndex == rhsIndex
+    case (.tensorFlowLiteError(let lhsMessage), .tensorFlowLiteError(let rhsMessage)):
+      return lhsMessage == rhsMessage
+    default:
+      return false
+    }
+  }
+}
+#endif  // swift(>=4.2)
diff --git a/tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift b/tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift
new file mode 100644
index 0000000000..2365fd7ade
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift
@@ -0,0 +1,29 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+
+/// Custom configuration options for a TensorFlow Lite interpreter.
+public struct InterpreterOptions: Equatable {
+
+  /// Maximum number of CPU threads that the interpreter should run on. Default is `nil` which
+  /// indicates that the `Interpreter` will decide the number of threads to use.
+  public var threadCount: Int? = nil
+
+  /// Whether error logging to the console is enabled. The default is `false`.
+  public var isErrorLoggingEnabled = false
+
+  /// Creates a new instance of interpreter options.
+  public init() {}
+}
diff --git a/tensorflow/lite/experimental/swift/Sources/Model.swift b/tensorflow/lite/experimental/swift/Sources/Model.swift
new file mode 100644
index 0000000000..e8c49ff1ae
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Sources/Model.swift
@@ -0,0 +1,40 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+import TensorFlowLiteCAPI
+
+/// A TensorFlow Lite model used by the 'Interpreter` to perform inference.
+final class Model {
+
+  /// The `TFL_Model` C pointer type represented as an `UnsafePointer<TFL_Model>`.
+  typealias CModel = OpaquePointer
+
+  /// The underlying `TFL_Model` C pointer.
+  let cModel: CModel?
+
+  /// Creates a new model instance.
+  ///
+  /// - Precondition: Initialization can fail if the given `filePath` is invalid.
+  /// - Parameters:
+  ///   - filePath: Local file path to a TensorFlow Lite model.
+  init?(filePath: String) {
+    guard !filePath.isEmpty, let cModel = TFL_NewModelFromFile(filePath) else { return nil }
+    self.cModel = cModel
+  }
+
+  deinit {
+    TFL_DeleteModel(cModel)
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift b/tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift
new file mode 100644
index 0000000000..f367875644
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift
@@ -0,0 +1,38 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+
+/// Parameters that determine the mapping of quantized values to real values. Quantized values can
+/// be mapped to float values using the following conversion:
+/// `realValue = scale * (quantizedValue - zeroPoint)`.
+public struct QuantizationParameters {
+
+  /// Difference between real values corresponding to consecutive quantized values differing by 1.
+  /// For example, the range of quantized values for `UInt8` data type is [0, 255].
+  public let scale: Float
+
+  /// Quantized value that corresponds to the real 0 value.
+  public let zeroPoint: Int
+
+  /// Creates a new quantization parameters instance.
+  ///
+  /// - Parameters:
+  ///   - scale: Scale value for asymmetric quantization.
+  ///   - zeroPoint: Zero point for asymmetric quantization.
+  init(scale: Float, zeroPoint: Int) {
+    self.scale = scale
+    self.zeroPoint = zeroPoint
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/Sources/Tensor.swift b/tensorflow/lite/experimental/swift/Sources/Tensor.swift
new file mode 100644
index 0000000000..b738d87549
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Sources/Tensor.swift
@@ -0,0 +1,138 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+import TensorFlowLiteCAPI
+
+/// An input or output tensor in a TensorFlow Lite graph.
+public struct Tensor {
+
+  /// Name of the tensor.
+  public let name: String
+
+  /// Data type of the tensor.
+  public let dataType: TensorDataType
+
+  /// Shape of the tensor.
+  public let shape: TensorShape
+
+  /// Data in the input or output tensor.
+  public let data: Data
+
+  /// Quantization parameters for the tensor if using a quantized model.
+  public let quantizationParameters: QuantizationParameters?
+
+  /// Creates a new input or output tensor instance.
+  ///
+  /// - Parameters:
+  ///   - name: Name of the tensor.
+  ///   - dataType: Data type of the tensor.
+  ///   - data: Data in the input tensor.
+  ///   - quantizationParameters Quantization parameters for the tensor if using a quantized model.
+  ///       The default is `nil`.
+  init(
+    name: String,
+    dataType: TensorDataType,
+    shape: TensorShape,
+    data: Data,
+    quantizationParameters: QuantizationParameters? = nil
+  ) {
+    self.name = name
+    self.dataType = dataType
+    self.shape = shape
+    self.data = data
+    self.quantizationParameters = quantizationParameters
+  }
+}
+
+/// Supported TensorFlow Lite tensor data types.
+public enum TensorDataType: Equatable {
+  /// 32-bit single precision floating point tensor data type.
+  case float32
+  /// 8-bit unsigned integer tensor data type.
+  case uInt8
+  /// 16-bit signed integer tensor data type.
+  case int16
+  /// 32-bit signed integer tensor data type.
+  case int32
+  /// 64-bit signed integer tensor data type.
+  case int64
+  /// Boolean tensor data type.
+  case bool
+
+  /// Creates a new tensor data type from the given `TFL_Type` or `nil` if the data type is
+  /// unsupported or could not be determined because there was an error.
+  ///
+  /// - Parameter type: A data type supported by a tensor.
+  init?(type: TFL_Type) {
+    switch type {
+    case kTfLiteFloat32:
+      self = .float32
+    case kTfLiteUInt8:
+      self = .uInt8
+    case kTfLiteInt16:
+      self = .int16
+    case kTfLiteInt32:
+      self = .int32
+    case kTfLiteInt64:
+      self = .int64
+    case kTfLiteBool:
+      self = .bool
+    case kTfLiteNoType:
+      fallthrough
+    default:
+      return nil
+    }
+  }
+}
+
+/// The shape of a TensorFlow Lite tensor.
+public struct TensorShape {
+
+  /// The number of dimensions of the tensor.
+  public let rank: Int
+
+  /// Array of dimensions for the tensor.
+  public let dimensions: [Int]
+
+  /// Array of `Int32` dimensions for the tensor.
+  var int32Dimensions: [Int32] { return dimensions.map(Int32.init) }
+
+  /// Creates a new tensor shape instance with the given array of dimensions.
+  ///
+  /// - Parameters:
+  ///   - dimensions: Dimensions for the tensor.
+  public init(_ dimensions: [Int]) {
+    self.rank = dimensions.count
+    self.dimensions = dimensions
+  }
+
+  /// Creates a new tensor shape instance with the given elements representing the dimensions.
+  ///
+  /// - Parameters:
+  ///   - elements: Dimensions for the tensor.
+  public init(_ elements: Int...) {
+    self.init(elements)
+  }
+}
+
+extension TensorShape: ExpressibleByArrayLiteral {
+  /// Creates a new tensor shape instance with the given array literal representing the dimensions.
+  ///
+  /// - Parameters:
+  ///   - arrayLiteral: Dimensions for the tensor.
+  public init(arrayLiteral: Int...) {
+    self.init(arrayLiteral)
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
new file mode 100644
index 0000000000..4010fab49e
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
@@ -0,0 +1,57 @@
+{
+  "sourceFilters" : [
+    "third_party/tensorflow/lite/experimental/c",
+    "third_party/tensorflow/lite/experimental/swift",
+    "third_party/tensorflow/lite/experimental/swift/Sources",
+    "third_party/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp",
+    "third_party/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj",
+    "third_party/tensorflow/lite/experimental/swift/Tests",
+  ],
+  "buildTargets" : [
+    "//third_party/tensorflow/lite/experimental/swift:TensorFlowLite",
+    "//third_party/tensorflow/lite/experimental/swift:TensorFlowLiteApp",
+    "//third_party/tensorflow/lite/experimental/swift:TensorFlowLiteTests",
+  ],
+  "projectName" : "TensorFlowLite",
+  "optionSet" : {
+    "LaunchActionPreActionScript" : {
+      "p" : "$(inherited)"
+    },
+    "BazelBuildStartupOptionsRelease" : {
+      "p" : "$(inherited)"
+    },
+    "BazelBuildOptionsRelease" : {
+      "p" : "$(inherited)"
+    },
+    "BazelBuildOptionsDebug" : {
+      "p" : "$(inherited)"
+    },
+    "EnvironmentVariables" : {
+      "p" : "$(inherited)"
+    },
+    "BuildActionPreActionScript" : {
+      "p" : "$(inherited)"
+    },
+    "CommandlineArguments" : {
+      "p" : "$(inherited)"
+    },
+    "TestActionPreActionScript" : {
+      "p" : "$(inherited)"
+    },
+    "BazelBuildStartupOptionsDebug" : {
+      "p" : "$(inherited)"
+    },
+    "BuildActionPostActionScript" : {
+      "p" : "$(inherited)"
+    },
+    "TestActionPostActionScript" : {
+      "p" : "$(inherited)"
+    },
+    "LaunchActionPostActionScript" : {
+      "p" : "$(inherited)"
+    }
+  },
+  "additionalFilePaths" : [
+    "third_party/tensorflow/lite/experimental/swift/BUILD"
+  ]
+}
diff --git a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
new file mode 100644
index 0000000000..14cff94453
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
@@ -0,0 +1,14 @@
+{
+  "configDefaults" : {
+    "optionSet" : {
+      "ProjectPrioritizesSwift" : {
+        "p" : "YES"
+      }
+    }
+  },
+  "projectName" : "TensorFlowLite",
+  "packages" : [
+    "third_party/tensorflow/lite/experimental/swift"
+  ],
+  "workspaceRoot" : "../../../../../.."
+}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj
new file mode 100644
index 0000000000..fbbf9a1de2
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj
@@ -0,0 +1,345 @@
+// !$*UTF8*$!
+{
+	archiveVersion = 1;
+	classes = {
+	};
+	objectVersion = 50;
+	objects = {
+
+/* Begin PBXBuildFile section */
+		4A7304B421500B8400C90B21 /* Data+TensorFlowLite.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4A7304B321500B8300C90B21 /* Data+TensorFlowLite.swift */; };
+		4AA72B732146ED64006C3AEF /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4AA72B722146ED64006C3AEF /* AppDelegate.swift */; };
+		4AA72B752146ED64006C3AEF /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4AA72B742146ED64006C3AEF /* ViewController.swift */; };
+		4AA72B782146ED64006C3AEF /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 4AA72B762146ED64006C3AEF /* Main.storyboard */; };
+		4AA72B7A2146ED66006C3AEF /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 4AA72B792146ED66006C3AEF /* Assets.xcassets */; };
+		4AA72B7D2146ED66006C3AEF /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 4AA72B7B2146ED66006C3AEF /* LaunchScreen.storyboard */; };
+		4ADDE0CE2176600E00FF07A2 /* Array+TensorFlowLite.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4ADDE0CD2176600900FF07A2 /* Array+TensorFlowLite.swift */; };
+/* End PBXBuildFile section */
+
+/* Begin PBXFileReference section */
+		4A7304B321500B8300C90B21 /* Data+TensorFlowLite.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "Data+TensorFlowLite.swift"; sourceTree = "<group>"; };
+		4AA72B6F2146ED64006C3AEF /* TensorFlowLiteApp.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = TensorFlowLiteApp.app; sourceTree = BUILT_PRODUCTS_DIR; };
+		4AA72B722146ED64006C3AEF /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = "<group>"; };
+		4AA72B742146ED64006C3AEF /* ViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewController.swift; sourceTree = "<group>"; };
+		4AA72B772146ED64006C3AEF /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = "<group>"; };
+		4AA72B792146ED66006C3AEF /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = "<group>"; };
+		4AA72B7C2146ED66006C3AEF /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = "<group>"; };
+		4AA72B7E2146ED66006C3AEF /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
+		4ADDE0CD2176600900FF07A2 /* Array+TensorFlowLite.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "Array+TensorFlowLite.swift"; sourceTree = "<group>"; };
+/* End PBXFileReference section */
+
+/* Begin PBXFrameworksBuildPhase section */
+		4AA72B6C2146ED64006C3AEF /* Frameworks */ = {
+			isa = PBXFrameworksBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXFrameworksBuildPhase section */
+
+/* Begin PBXGroup section */
+		4AA72B662146ED64006C3AEF = {
+			isa = PBXGroup;
+			children = (
+				4AA72B712146ED64006C3AEF /* TensorFlowLiteApp */,
+				4AA72B702146ED64006C3AEF /* Products */,
+			);
+			sourceTree = "<group>";
+		};
+		4AA72B702146ED64006C3AEF /* Products */ = {
+			isa = PBXGroup;
+			children = (
+				4AA72B6F2146ED64006C3AEF /* TensorFlowLiteApp.app */,
+			);
+			name = Products;
+			sourceTree = "<group>";
+		};
+		4AA72B712146ED64006C3AEF /* TensorFlowLiteApp */ = {
+			isa = PBXGroup;
+			children = (
+				4AA72B722146ED64006C3AEF /* AppDelegate.swift */,
+				4ADDE0CD2176600900FF07A2 /* Array+TensorFlowLite.swift */,
+				4A7304B321500B8300C90B21 /* Data+TensorFlowLite.swift */,
+				4AA72B742146ED64006C3AEF /* ViewController.swift */,
+				4AA72B762146ED64006C3AEF /* Main.storyboard */,
+				4AA72B792146ED66006C3AEF /* Assets.xcassets */,
+				4AA72B7B2146ED66006C3AEF /* LaunchScreen.storyboard */,
+				4AA72B7E2146ED66006C3AEF /* Info.plist */,
+			);
+			path = TensorFlowLiteApp;
+			sourceTree = "<group>";
+		};
+/* End PBXGroup section */
+
+/* Begin PBXNativeTarget section */
+		4AA72B6E2146ED64006C3AEF /* TensorFlowLiteApp */ = {
+			isa = PBXNativeTarget;
+			buildConfigurationList = 4AA72B812146ED66006C3AEF /* Build configuration list for PBXNativeTarget "TensorFlowLiteApp" */;
+			buildPhases = (
+				4AA72B6B2146ED64006C3AEF /* Sources */,
+				4AA72B6C2146ED64006C3AEF /* Frameworks */,
+				4AA72B6D2146ED64006C3AEF /* Resources */,
+			);
+			buildRules = (
+			);
+			dependencies = (
+			);
+			name = TensorFlowLiteApp;
+			productName = TensorFlowLiteApp;
+			productReference = 4AA72B6F2146ED64006C3AEF /* TensorFlowLiteApp.app */;
+			productType = "com.apple.product-type.application";
+		};
+/* End PBXNativeTarget section */
+
+/* Begin PBXProject section */
+		4AA72B672146ED64006C3AEF /* Project object */ = {
+			isa = PBXProject;
+			attributes = {
+				LastSwiftUpdateCheck = 0940;
+				LastUpgradeCheck = 0940;
+				ORGANIZATIONNAME = Google;
+				TargetAttributes = {
+					4AA72B6E2146ED64006C3AEF = {
+						CreatedOnToolsVersion = 9.4.1;
+					};
+				};
+			};
+			buildConfigurationList = 4AA72B6A2146ED64006C3AEF /* Build configuration list for PBXProject "TensorFlowLiteApp" */;
+			compatibilityVersion = "Xcode 9.3";
+			developmentRegion = en;
+			hasScannedForEncodings = 0;
+			knownRegions = (
+				en,
+				Base,
+			);
+			mainGroup = 4AA72B662146ED64006C3AEF;
+			productRefGroup = 4AA72B702146ED64006C3AEF /* Products */;
+			projectDirPath = "";
+			projectRoot = "";
+			targets = (
+				4AA72B6E2146ED64006C3AEF /* TensorFlowLiteApp */,
+			);
+		};
+/* End PBXProject section */
+
+/* Begin PBXResourcesBuildPhase section */
+		4AA72B6D2146ED64006C3AEF /* Resources */ = {
+			isa = PBXResourcesBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+				4AA72B7D2146ED66006C3AEF /* LaunchScreen.storyboard in Resources */,
+				4AA72B7A2146ED66006C3AEF /* Assets.xcassets in Resources */,
+				4AA72B782146ED64006C3AEF /* Main.storyboard in Resources */,
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXResourcesBuildPhase section */
+
+/* Begin PBXSourcesBuildPhase section */
+		4AA72B6B2146ED64006C3AEF /* Sources */ = {
+			isa = PBXSourcesBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+				4AA72B732146ED64006C3AEF /* AppDelegate.swift in Sources */,
+				4ADDE0CE2176600E00FF07A2 /* Array+TensorFlowLite.swift in Sources */,
+				4A7304B421500B8400C90B21 /* Data+TensorFlowLite.swift in Sources */,
+				4AA72B752146ED64006C3AEF /* ViewController.swift in Sources */,
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXSourcesBuildPhase section */
+
+/* Begin PBXVariantGroup section */
+		4AA72B762146ED64006C3AEF /* Main.storyboard */ = {
+			isa = PBXVariantGroup;
+			children = (
+				4AA72B772146ED64006C3AEF /* Base */,
+			);
+			name = Main.storyboard;
+			sourceTree = "<group>";
+		};
+		4AA72B7B2146ED66006C3AEF /* LaunchScreen.storyboard */ = {
+			isa = PBXVariantGroup;
+			children = (
+				4AA72B7C2146ED66006C3AEF /* Base */,
+			);
+			name = LaunchScreen.storyboard;
+			sourceTree = "<group>";
+		};
+/* End PBXVariantGroup section */
+
+/* Begin XCBuildConfiguration section */
+		4AA72B7F2146ED66006C3AEF /* Debug */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ALWAYS_SEARCH_USER_PATHS = NO;
+				CLANG_ANALYZER_NONNULL = YES;
+				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
+				CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
+				CLANG_CXX_LIBRARY = "libc++";
+				CLANG_ENABLE_MODULES = YES;
+				CLANG_ENABLE_OBJC_ARC = YES;
+				CLANG_ENABLE_OBJC_WEAK = YES;
+				CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
+				CLANG_WARN_BOOL_CONVERSION = YES;
+				CLANG_WARN_COMMA = YES;
+				CLANG_WARN_CONSTANT_CONVERSION = YES;
+				CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
+				CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
+				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
+				CLANG_WARN_EMPTY_BODY = YES;
+				CLANG_WARN_ENUM_CONVERSION = YES;
+				CLANG_WARN_INFINITE_RECURSION = YES;
+				CLANG_WARN_INT_CONVERSION = YES;
+				CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
+				CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
+				CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
+				CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
+				CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
+				CLANG_WARN_STRICT_PROTOTYPES = YES;
+				CLANG_WARN_SUSPICIOUS_MOVE = YES;
+				CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
+				CLANG_WARN_UNREACHABLE_CODE = YES;
+				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
+				CODE_SIGN_IDENTITY = "iPhone Developer";
+				COPY_PHASE_STRIP = NO;
+				DEBUG_INFORMATION_FORMAT = dwarf;
+				ENABLE_STRICT_OBJC_MSGSEND = YES;
+				ENABLE_TESTABILITY = YES;
+				GCC_C_LANGUAGE_STANDARD = gnu11;
+				GCC_DYNAMIC_NO_PIC = NO;
+				GCC_NO_COMMON_BLOCKS = YES;
+				GCC_OPTIMIZATION_LEVEL = 0;
+				GCC_PREPROCESSOR_DEFINITIONS = (
+					"DEBUG=1",
+					"$(inherited)",
+				);
+				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
+				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
+				GCC_WARN_UNDECLARED_SELECTOR = YES;
+				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
+				GCC_WARN_UNUSED_FUNCTION = YES;
+				GCC_WARN_UNUSED_VARIABLE = YES;
+				IPHONEOS_DEPLOYMENT_TARGET = 11.4;
+				MTL_ENABLE_DEBUG_INFO = YES;
+				ONLY_ACTIVE_ARCH = YES;
+				SDKROOT = iphoneos;
+				SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;
+				SWIFT_OPTIMIZATION_LEVEL = "-Onone";
+			};
+			name = Debug;
+		};
+		4AA72B802146ED66006C3AEF /* Release */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ALWAYS_SEARCH_USER_PATHS = NO;
+				CLANG_ANALYZER_NONNULL = YES;
+				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
+				CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
+				CLANG_CXX_LIBRARY = "libc++";
+				CLANG_ENABLE_MODULES = YES;
+				CLANG_ENABLE_OBJC_ARC = YES;
+				CLANG_ENABLE_OBJC_WEAK = YES;
+				CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
+				CLANG_WARN_BOOL_CONVERSION = YES;
+				CLANG_WARN_COMMA = YES;
+				CLANG_WARN_CONSTANT_CONVERSION = YES;
+				CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
+				CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
+				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
+				CLANG_WARN_EMPTY_BODY = YES;
+				CLANG_WARN_ENUM_CONVERSION = YES;
+				CLANG_WARN_INFINITE_RECURSION = YES;
+				CLANG_WARN_INT_CONVERSION = YES;
+				CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
+				CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
+				CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
+				CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
+				CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
+				CLANG_WARN_STRICT_PROTOTYPES = YES;
+				CLANG_WARN_SUSPICIOUS_MOVE = YES;
+				CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
+				CLANG_WARN_UNREACHABLE_CODE = YES;
+				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
+				CODE_SIGN_IDENTITY = "iPhone Developer";
+				COPY_PHASE_STRIP = NO;
+				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
+				ENABLE_NS_ASSERTIONS = NO;
+				ENABLE_STRICT_OBJC_MSGSEND = YES;
+				GCC_C_LANGUAGE_STANDARD = gnu11;
+				GCC_NO_COMMON_BLOCKS = YES;
+				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
+				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
+				GCC_WARN_UNDECLARED_SELECTOR = YES;
+				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
+				GCC_WARN_UNUSED_FUNCTION = YES;
+				GCC_WARN_UNUSED_VARIABLE = YES;
+				IPHONEOS_DEPLOYMENT_TARGET = 11.4;
+				MTL_ENABLE_DEBUG_INFO = NO;
+				SDKROOT = iphoneos;
+				SWIFT_COMPILATION_MODE = wholemodule;
+				SWIFT_OPTIMIZATION_LEVEL = "-O";
+				VALIDATE_PRODUCT = YES;
+			};
+			name = Release;
+		};
+		4AA72B822146ED66006C3AEF /* Debug */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
+				CODE_SIGN_STYLE = Automatic;
+				INFOPLIST_FILE = TensorFlowLiteApp/Info.plist;
+				LD_RUNPATH_SEARCH_PATHS = (
+					"$(inherited)",
+					"@executable_path/Frameworks",
+				);
+				PRODUCT_BUNDLE_IDENTIFIER = com.tensorflow.lite.swift.TensorFlowLite;
+				PRODUCT_NAME = "$(TARGET_NAME)";
+				SWIFT_VERSION = 4.0;
+				TARGETED_DEVICE_FAMILY = "1,2";
+			};
+			name = Debug;
+		};
+		4AA72B832146ED66006C3AEF /* Release */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
+				CODE_SIGN_STYLE = Automatic;
+				INFOPLIST_FILE = TensorFlowLiteApp/Info.plist;
+				LD_RUNPATH_SEARCH_PATHS = (
+					"$(inherited)",
+					"@executable_path/Frameworks",
+				);
+				PRODUCT_BUNDLE_IDENTIFIER = com.tensorflow.lite.swift.TensorFlowLite;
+				PRODUCT_NAME = "$(TARGET_NAME)";
+				SWIFT_VERSION = 4.0;
+				TARGETED_DEVICE_FAMILY = "1,2";
+			};
+			name = Release;
+		};
+/* End XCBuildConfiguration section */
+
+/* Begin XCConfigurationList section */
+		4AA72B6A2146ED64006C3AEF /* Build configuration list for PBXProject "TensorFlowLiteApp" */ = {
+			isa = XCConfigurationList;
+			buildConfigurations = (
+				4AA72B7F2146ED66006C3AEF /* Debug */,
+				4AA72B802146ED66006C3AEF /* Release */,
+			);
+			defaultConfigurationIsVisible = 0;
+			defaultConfigurationName = Release;
+		};
+		4AA72B812146ED66006C3AEF /* Build configuration list for PBXNativeTarget "TensorFlowLiteApp" */ = {
+			isa = XCConfigurationList;
+			buildConfigurations = (
+				4AA72B822146ED66006C3AEF /* Debug */,
+				4AA72B832146ED66006C3AEF /* Release */,
+			);
+			defaultConfigurationIsVisible = 0;
+			defaultConfigurationName = Release;
+		};
+/* End XCConfigurationList section */
+	};
+	rootObject = 4AA72B672146ED64006C3AEF /* Project object */;
+}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift
new file mode 100644
index 0000000000..ffa90a06ad
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift
@@ -0,0 +1,24 @@
+import UIKit
+
+@UIApplicationMain
+
+final class AppDelegate: UIResponder, UIApplicationDelegate {
+
+  /// The main window of the app.
+  var window: UIWindow?
+
+  func application(
+    _ application: UIApplication,
+    didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? = nil
+  ) -> Bool {
+    return true
+  }
+}
+
+// MARK: - Extensions
+
+#if !swift(>=4.2)
+extension UIApplication {
+  typealias LaunchOptionsKey = UIApplicationLaunchOptionsKey
+}
+#endif  // !swift(>=4.2)
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift
new file mode 100644
index 0000000000..56df1ce659
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift
@@ -0,0 +1,22 @@
+import Foundation
+
+extension Array {
+  /// Creates a new array from the bytes of the given unsafe data.
+  ///
+  /// - Warning: The array's `Element` type must be trivial in that it can be copied bit for bit
+  ///     with no indirection or reference-counting operations; otherwise, copying the raw bytes in
+  ///     the `unsafeData`'s buffer to a new array returns an unsafe copy.
+  /// - Note: Returns `nil` if `unsafeData.count` is not a multiple of
+  ///     `MemoryLayout<Element>.stride`.
+  /// - Parameter unsafeData: The data containing the bytes to turn into an array.
+  init?(unsafeData: Data) {
+    guard unsafeData.count % MemoryLayout<Element>.stride == 0 else { return nil }
+    let elements = unsafeData.withUnsafeBytes {
+      UnsafeBufferPointer<Element>(
+        start: $0,
+        count: unsafeData.count / MemoryLayout<Element>.stride
+      )
+    }
+    self.init(elements)
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json
new file mode 100644
index 0000000000..d8db8d65fd
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json
@@ -0,0 +1,98 @@
+{
+  "images" : [
+    {
+      "idiom" : "iphone",
+      "size" : "20x20",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "20x20",
+      "scale" : "3x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "29x29",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "29x29",
+      "scale" : "3x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "40x40",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "40x40",
+      "scale" : "3x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "60x60",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "60x60",
+      "scale" : "3x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "20x20",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "20x20",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "29x29",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "29x29",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "40x40",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "40x40",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "76x76",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "76x76",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "83.5x83.5",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ios-marketing",
+      "size" : "1024x1024",
+      "scale" : "1x"
+    }
+  ],
+  "info" : {
+    "version" : 1,
+    "author" : "xcode"
+  }
+}
\ No newline at end of file
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json
new file mode 100644
index 0000000000..da4a164c91
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json
@@ -0,0 +1,6 @@
+{
+  "info" : {
+    "version" : 1,
+    "author" : "xcode"
+  }
+}
\ No newline at end of file
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard
new file mode 100644
index 0000000000..a07a1321be
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="14109" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" launchScreen="YES" useTraitCollections="YES" colorMatched="YES" initialViewController="01J-lp-oVM">
+    <device id="retina4_7" orientation="portrait">
+        <adaptation id="fullscreen"/>
+    </device>
+    <dependencies>
+        <deployment identifier="iOS"/>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="14088"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--View Controller-->
+        <scene sceneID="EHf-IW-A2E">
+            <objects>
+                <viewController id="01J-lp-oVM" sceneMemberID="viewController">
+                    <layoutGuides>
+                        <viewControllerLayoutGuide type="top" id="Llm-lL-Icb"/>
+                        <viewControllerLayoutGuide type="bottom" id="xb3-aO-Qok"/>
+                    </layoutGuides>
+                    <view key="view" contentMode="scaleToFill" id="Ze5-6b-2t3">
+                        <rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <subviews>
+                            <label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" text="TensorFlowLite" textAlignment="center" lineBreakMode="tailTruncation" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="3Gq-PV-hia">
+                                <rect key="frame" x="16" y="315" width="343" height="38.5"/>
+                                <fontDescription key="fontDescription" type="boldSystem" pointSize="32"/>
+                                <nil key="textColor"/>
+                                <nil key="highlightedColor"/>
+                            </label>
+                        </subviews>
+                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                        <constraints>
+                            <constraint firstItem="3Gq-PV-hia" firstAttribute="leading" secondItem="Ze5-6b-2t3" secondAttribute="leading" constant="16" id="aXL-9T-5Pf"/>
+                            <constraint firstItem="3Gq-PV-hia" firstAttribute="centerY" secondItem="Ze5-6b-2t3" secondAttribute="centerY" id="cDf-Go-1FR"/>
+                            <constraint firstAttribute="trailing" secondItem="3Gq-PV-hia" secondAttribute="trailing" constant="16" id="fB9-BX-A3B"/>
+                        </constraints>
+                    </view>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="iYj-Kq-Ea1" userLabel="First Responder" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="52" y="374.66266866566718"/>
+        </scene>
+    </scenes>
+</document>
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
new file mode 100644
index 0000000000..d0e91d63c4
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
@@ -0,0 +1,95 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="14313.18" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" colorMatched="YES" initialViewController="BYZ-38-t0r">
+    <device id="retina4_7" orientation="portrait">
+        <adaptation id="fullscreen"/>
+    </device>
+    <dependencies>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="14283.14"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--View Controller-->
+        <scene sceneID="tne-QT-ifu">
+            <objects>
+                <viewController storyboardIdentifier="viewController" useStoryboardIdentifierAsRestorationIdentifier="YES" id="BYZ-38-t0r" customClass="ViewController" customModule="third_party_tensorflow_lite_experimental_swift_TensorFlowLiteAppLib" sceneMemberID="viewController">
+                    <layoutGuides>
+                        <viewControllerLayoutGuide type="top" id="y3c-jy-aDJ"/>
+                        <viewControllerLayoutGuide type="bottom" id="wfy-db-euE"/>
+                    </layoutGuides>
+                    <view key="view" contentMode="scaleToFill" id="8bC-Xf-vdC">
+                        <rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <subviews>
+                            <textView clipsSubviews="YES" multipleTouchEnabled="YES" contentMode="scaleToFill" editable="NO" selectable="NO" translatesAutoresizingMaskIntoConstraints="NO" id="7Mj-sL-hrd">
+                                <rect key="frame" x="0.0" y="367" width="375" height="300"/>
+                                <color key="backgroundColor" red="0.0" green="0.47843137250000001" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                                <constraints>
+                                    <constraint firstAttribute="height" constant="300" id="YUb-MC-D5w"/>
+                                </constraints>
+                                <color key="textColor" cocoaTouchSystemColor="tableCellGroupedBackgroundColor"/>
+                                <fontDescription key="fontDescription" type="system" pointSize="14"/>
+                                <textInputTraits key="textInputTraits" autocapitalizationType="sentences"/>
+                            </textView>
+                            <toolbar opaque="NO" clearsContextBeforeDrawing="NO" contentMode="scaleToFill" translatesAutoresizingMaskIntoConstraints="NO" id="Qwg-EP-bd6" userLabel="Bottom Toolbar">
+                                <rect key="frame" x="0.0" y="323" width="375" height="44"/>
+                                <constraints>
+                                    <constraint firstAttribute="height" constant="44" id="jhT-Q0-E9N"/>
+                                </constraints>
+                                <items>
+                                    <barButtonItem style="plain" systemItem="flexibleSpace" id="P3q-uA-YUa"/>
+                                    <barButtonItem title="Invoke Interpreter" id="A4J-Mg-nmd" userLabel="Invoke Button">
+                                        <connections>
+                                            <action selector="invokeInterpreter:" destination="BYZ-38-t0r" id="lZU-x7-PsJ"/>
+                                        </connections>
+                                    </barButtonItem>
+                                    <barButtonItem style="plain" systemItem="flexibleSpace" id="Qad-Pa-ySg"/>
+                                </items>
+                            </toolbar>
+                            <toolbar opaque="NO" clearsContextBeforeDrawing="NO" contentMode="scaleToFill" translatesAutoresizingMaskIntoConstraints="NO" id="Gkb-TR-PCB" userLabel="Top Toolbar">
+                                <rect key="frame" x="0.0" y="28" width="375" height="44"/>
+                                <constraints>
+                                    <constraint firstAttribute="height" constant="44" id="hSD-2q-fUE"/>
+                                </constraints>
+                                <items>
+                                    <barButtonItem style="plain" id="LKw-TX-bbH">
+                                        <segmentedControl key="customView" opaque="NO" contentMode="scaleToFill" contentHorizontalAlignment="left" contentVerticalAlignment="top" segmentControlStyle="bar" selectedSegmentIndex="0" id="rhA-nW-xzT">
+                                            <rect key="frame" x="16" y="7" width="343" height="30"/>
+                                            <autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
+                                            <segments>
+                                                <segment title="Add"/>
+                                                <segment title="AddQuantized"/>
+                                                <segment title="MultiAdd"/>
+                                            </segments>
+                                            <connections>
+                                                <action selector="modelChanged:" destination="BYZ-38-t0r" eventType="valueChanged" id="YnG-Ov-B5D"/>
+                                            </connections>
+                                        </segmentedControl>
+                                    </barButtonItem>
+                                </items>
+                            </toolbar>
+                        </subviews>
+                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                        <constraints>
+                            <constraint firstAttribute="trailing" secondItem="Gkb-TR-PCB" secondAttribute="trailing" id="4Cr-Sf-I7n"/>
+                            <constraint firstItem="7Mj-sL-hrd" firstAttribute="bottom" secondItem="wfy-db-euE" secondAttribute="top" id="6ot-zD-sze"/>
+                            <constraint firstItem="7Mj-sL-hrd" firstAttribute="top" secondItem="Qwg-EP-bd6" secondAttribute="bottom" id="ELA-C6-NiG"/>
+                            <constraint firstAttribute="trailing" secondItem="7Mj-sL-hrd" secondAttribute="trailing" id="HDO-xr-mBl"/>
+                            <constraint firstItem="Gkb-TR-PCB" firstAttribute="leading" secondItem="8bC-Xf-vdC" secondAttribute="leading" id="Kmo-6K-gS4"/>
+                            <constraint firstItem="Qwg-EP-bd6" firstAttribute="leading" secondItem="8bC-Xf-vdC" secondAttribute="leading" id="hGu-lm-fMG"/>
+                            <constraint firstAttribute="trailing" secondItem="Qwg-EP-bd6" secondAttribute="trailing" id="iXR-LK-nTO"/>
+                            <constraint firstItem="7Mj-sL-hrd" firstAttribute="leading" secondItem="8bC-Xf-vdC" secondAttribute="leading" id="nr7-jW-ZYf"/>
+                            <constraint firstItem="Gkb-TR-PCB" firstAttribute="top" secondItem="y3c-jy-aDJ" secondAttribute="bottom" constant="8" id="uCF-VW-rR0"/>
+                        </constraints>
+                    </view>
+                    <connections>
+                        <outlet property="invokeButton" destination="A4J-Mg-nmd" id="UxZ-Ft-E45"/>
+                        <outlet property="modelControl" destination="rhA-nW-xzT" id="KKf-TT-BQ2"/>
+                        <outlet property="resultsTextView" destination="7Mj-sL-hrd" id="T4I-z4-tYA"/>
+                    </connections>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="dkx-z0-nzr" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="125.59999999999999" y="133.5832083958021"/>
+        </scene>
+    </scenes>
+</document>
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift
new file mode 100644
index 0000000000..bc8a70c848
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift
@@ -0,0 +1,13 @@
+import Foundation
+
+extension Data {
+  /// Creates a new buffer by copying the buffer pointer of the given array.
+  ///
+  /// - Warning: The given array's element type `T` must be trivial in that it can be copied bit
+  ///     for bit with no indirection or reference-counting operations; otherwise, reinterpreting
+  ///     data from the resulting buffer has undefined behavior.
+  /// - Parameter array: An array with elements of type `T`.
+  init<T>(copyingBufferOf array: [T]) {
+    self = array.withUnsafeBufferPointer(Data.init)
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist
new file mode 100644
index 0000000000..3ca3875f04
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>CFBundleDevelopmentRegion</key>
+	<string>en</string>
+	<key>CFBundleExecutable</key>
+	<string>$(EXECUTABLE_NAME)</string>
+	<key>CFBundleIdentifier</key>
+	<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
+	<key>CFBundleInfoDictionaryVersion</key>
+	<string>6.0</string>
+	<key>CFBundleName</key>
+	<string>$(PRODUCT_NAME)</string>
+	<key>CFBundlePackageType</key>
+	<string>APPL</string>
+	<key>CFBundleShortVersionString</key>
+	<string>1.0</string>
+	<key>CFBundleVersion</key>
+	<string>0.0.1</string>
+	<key>LSRequiresIPhoneOS</key>
+	<true/>
+	<key>NSCameraUsageDescription</key>
+	<string>NSCameraUsageDescription</string>
+	<key>NSPhotoLibraryUsageDescription</key>
+	<string>Select a photo to detect objects in.</string>
+	<key>UILaunchStoryboardName</key>
+	<string>LaunchScreen</string>
+	<key>UIMainStoryboardFile</key>
+	<string>Main</string>
+	<key>UIRequiredDeviceCapabilities</key>
+	<array>
+		<string>armv7</string>
+	</array>
+	<key>UISupportedInterfaceOrientations</key>
+	<array>
+		<string>UIInterfaceOrientationPortrait</string>
+		<string>UIInterfaceOrientationPortraitUpsideDown</string>
+	</array>
+	<key>UISupportedInterfaceOrientations~ipad</key>
+	<array>
+		<string>UIInterfaceOrientationPortrait</string>
+		<string>UIInterfaceOrientationPortraitUpsideDown</string>
+	</array>
+</dict>
+</plist>
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift
new file mode 100644
index 0000000000..73c74fd19c
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift
@@ -0,0 +1,299 @@
+import TensorFlowLite
+import UIKit
+
+class ViewController: UIViewController {
+
+  // MARK: - Properties
+
+  /// TensorFlowLite interpreter object for performing inference from a given model.
+  private var interpreter: Interpreter?
+
+  /// Serial dispatch queue for managing `Interpreter` calls.
+  private let interpreterQueue = DispatchQueue(
+    label: Constant.dispatchQueueLabel,
+    qos: .userInitiated
+  )
+
+  /// The currently selected model.
+  private var currentModel: Model {
+    guard let currentModel = Model(rawValue: modelControl.selectedSegmentIndex) else {
+      preconditionFailure("Invalid model for selected segment index.")
+    }
+    return currentModel
+  }
+
+  /// A description of the current model.
+  private var modelDescription: String {
+    guard let interpreter = interpreter else { return "" }
+    let inputCount = interpreter.inputTensorCount
+    let outputCount = interpreter.outputTensorCount
+    let inputTensors = (0..<inputCount).map { index in
+      var tensorInfo = "  Input \(index + 1): "
+      do {
+        let tensor = try interpreter.input(at: index)
+        tensorInfo += "\(tensor)"
+      } catch let error {
+        tensorInfo += "\(error.localizedDescription)"
+      }
+      return tensorInfo
+    }.joined(separator: "\n")
+    let outputTensors = (0..<outputCount).map { index in
+      var tensorInfo = "  Output \(index + 1): "
+      do {
+        let tensor = try interpreter.output(at: index)
+        tensorInfo += "\(tensor)"
+      } catch let error {
+        tensorInfo += "\(error.localizedDescription)"
+      }
+      return tensorInfo
+    }.joined(separator: "\n")
+    return "Model Description:\n" +
+             "  Input Tensor Count = \(inputCount)\n\(inputTensors)\n\n" +
+             "  Output Tensor Count = \(outputCount)\n\(outputTensors)"
+  }
+
+  // MARK: - IBOutlets
+
+  /// A segmented control for changing models. See the `Model` enum for available models.
+  @IBOutlet private var modelControl: UISegmentedControl!
+
+  @IBOutlet private var resultsTextView: UITextView!
+  @IBOutlet private var invokeButton: UIBarButtonItem!
+
+  // MARK: - UIViewController
+
+  override func viewDidLoad() {
+    super.viewDidLoad()
+
+    invokeButton.isEnabled = false
+    loadModel()
+  }
+
+  // MARK: - IBActions
+
+  @IBAction func modelChanged(_ sender: Any) {
+    invokeButton.isEnabled = false
+    updateResultsText("Switched to the \(currentModel.description) model.")
+    loadModel()
+  }
+
+  @IBAction func invokeInterpreter(_ sender: Any) {
+    switch currentModel {
+    case .add:
+      invokeAdd()
+    case .addQuantized:
+      invokeAddQuantized()
+    case .multiAdd:
+      invokeMultiAdd()
+    }
+  }
+
+  // MARK: - Private
+
+  private func loadModel() {
+    let fileInfo = currentModel.fileInfo
+    guard let modelPath = Bundle.main.path(forResource: fileInfo.name, ofType: fileInfo.extension)
+    else {
+      updateResultsText("Failed to load the \(currentModel.description) model.")
+      return
+    }
+    setUpInterpreter(withModelPath: modelPath)
+  }
+
+  private func setUpInterpreter(withModelPath modelPath: String) {
+    interpreterQueue.async {
+      do {
+        var options = InterpreterOptions()
+        options.isErrorLoggingEnabled = true
+        self.interpreter = try Interpreter(modelPath: modelPath, options: options)
+      } catch let error {
+        self.updateResultsText(
+          "Failed to create the interpreter with error: \(error.localizedDescription)"
+        )
+        return
+      }
+      safeDispatchOnMain { self.invokeButton.isEnabled = true }
+    }
+  }
+
+  private func invokeAdd() {
+    interpreterQueue.async {
+      guard let interpreter = self.interpreter else {
+        self.updateResultsText(Constant.nilInterpreterErrorMessage)
+        return
+      }
+      do {
+        try interpreter.resizeInput(at: 0, to: [2])
+        try interpreter.allocateTensors()
+        let input: [Float32] = [1, 3]
+        let resultsText = self.modelDescription + "\n\n" +
+          "Performing 2 add operations on input \(input.description) equals: "
+        self.updateResultsText(resultsText)
+        let data = Data(copyingBufferOf: input)
+        try interpreter.copy(data, toInputAt: 0)
+        try interpreter.invoke()
+        let outputTensor = try interpreter.output(at: 0)
+        let results: () -> String = {
+          guard let results = [Float32](unsafeData: outputTensor.data) else { return "No results." }
+          return resultsText + results.description
+        }
+        self.updateResultsText(results())
+      } catch let error {
+        self.updateResultsText(
+          "Failed to invoke the interpreter with error: \(error.localizedDescription)"
+        )
+        return
+      }
+    }
+  }
+
+  private func invokeAddQuantized() {
+    interpreterQueue.async {
+      guard let interpreter = self.interpreter else {
+        self.updateResultsText(Constant.nilInterpreterErrorMessage)
+        return
+      }
+      do {
+        try interpreter.resizeInput(at: 0, to: [2])
+        try interpreter.allocateTensors()
+        let input: [UInt8] = [1, 3]
+        let resultsText = self.modelDescription + "\n\n" +
+          "Performing 2 add operations on quantized input \(input.description) equals: "
+        self.updateResultsText(resultsText)
+        let data = Data(input)
+        try interpreter.copy(data, toInputAt: 0)
+        try interpreter.invoke()
+        let outputTensor = try interpreter.output(at: 0)
+        let results: () -> String = {
+          guard let quantizationParameters = outputTensor.quantizationParameters else {
+            return "No results."
+          }
+          let quantizedResults = [UInt8](outputTensor.data)
+          let dequantizedResults = quantizedResults.map {
+            quantizationParameters.scale * Float(Int($0) - quantizationParameters.zeroPoint)
+          }
+          return resultsText + quantizedResults.description +
+                   ", dequantized results: " + dequantizedResults.description
+        }
+        self.updateResultsText(results())
+      } catch let error {
+        self.updateResultsText(
+          "Failed to invoke the interpreter with error: \(error.localizedDescription)"
+        )
+        return
+      }
+    }
+  }
+
+  private func invokeMultiAdd() {
+    interpreterQueue.async {
+      guard let interpreter = self.interpreter else {
+        self.updateResultsText(Constant.nilInterpreterErrorMessage)
+        return
+      }
+      do {
+        let shape = TensorShape(2)
+        try (0..<interpreter.inputTensorCount).forEach { index in
+          try interpreter.resizeInput(at: index, to: shape)
+        }
+        try interpreter.allocateTensors()
+        let inputs = try (0..<interpreter.inputTensorCount).map { index -> [Float32] in
+          let input = [Float32(index + 1), Float32(index + 2)]
+          let data = Data(copyingBufferOf: input)
+          try interpreter.copy(data, toInputAt: index)
+          return input
+        }
+        let resultsText = self.modelDescription + "\n\n" +
+          "Performing 3 add operations on inputs \(inputs.description) equals: "
+        self.updateResultsText(resultsText)
+        try interpreter.invoke()
+        let results = try (0..<interpreter.outputTensorCount).map { index -> [Float32] in
+          let tensor = try interpreter.output(at: index)
+          return [Float32](unsafeData: tensor.data) ?? []
+        }
+        self.updateResultsText(resultsText + results.description)
+      } catch let error {
+        self.updateResultsText(
+          "Failed to invoke the interpreter with error: \(error.localizedDescription)"
+        )
+        return
+      }
+    }
+  }
+
+  private func updateResultsText(_ text: String? = nil) {
+    safeDispatchOnMain { self.resultsTextView.text = text }
+  }
+}
+
+// MARK: - Constants
+
+private enum Constant {
+  static let dispatchQueueLabel = "TensorFlowLiteInterpreterQueue"
+  static let nilInterpreterErrorMessage =
+    "Failed to invoke the interpreter because the interpreter was nil."
+}
+
+/// Models that can be loaded by the TensorFlow Lite `Interpreter`.
+private enum Model: Int, CustomStringConvertible {
+  /// A float model that performs two add operations on one input tensor and returns the result in
+  /// one output tensor.
+  case add = 0
+  /// A quantized model that performs two add operations on one input tensor and returns the result
+  /// in one output tensor.
+  case addQuantized = 1
+  /// A float model that performs three add operations on four input tensors and returns the results
+  /// in 2 output tensors.
+  case multiAdd = 2
+
+  var fileInfo: (name: String, extension: String) {
+    switch self {
+    case .add:
+      return Add.fileInfo
+    case .addQuantized:
+      return AddQuantized.fileInfo
+    case .multiAdd:
+      return MultiAdd.fileInfo
+    }
+  }
+
+  // MARK: - CustomStringConvertible
+
+  var description: String {
+    switch self {
+    case .add:
+      return Add.name
+    case .addQuantized:
+      return AddQuantized.name
+    case .multiAdd:
+      return MultiAdd.name
+    }
+  }
+}
+
+/// Values for the `Add` model.
+private enum Add {
+  static let name = "Add"
+  static let fileInfo = (name: "add", extension: "bin")
+}
+
+/// Values for the `AddQuantized` model.
+private enum AddQuantized {
+  static let name = "AddQuantized"
+  static let fileInfo = (name: "add_quantized", extension: "bin")
+}
+
+/// Values for the `MultiAdd` model.
+private enum MultiAdd {
+  static let name = "MultiAdd"
+  static let fileInfo = (name: "multi_add", extension: "bin")
+}
+
+// MARK: - Fileprivate
+
+/// Safely dispatches the given block on the main queue. If the current thread is `main`, the block
+/// is executed synchronously; otherwise, the block is executed asynchronously on the main thread.
+fileprivate func safeDispatchOnMain(_ block: @escaping () -> Void) {
+  if Thread.isMainThread { block(); return }
+  DispatchQueue.main.async { block() }
+}
diff --git a/tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift b/tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift
new file mode 100644
index 0000000000..54b4f59b28
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift
@@ -0,0 +1,54 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+@testable import TensorFlowLite
+import XCTest
+
+class InterpreterOptionsTests: XCTestCase {
+
+  func testInterpreterOptions_InitWithDefaultValues() {
+    let options = InterpreterOptions()
+    XCTAssertNil(options.threadCount)
+    XCTAssertFalse(options.isErrorLoggingEnabled)
+  }
+
+  func testInterpreterOptions_InitWithCustomValues() {
+    var options = InterpreterOptions()
+    options.threadCount = 2
+    XCTAssertEqual(options.threadCount, 2)
+    options.isErrorLoggingEnabled = true
+    XCTAssertTrue(options.isErrorLoggingEnabled)
+  }
+
+  func testInterpreterOptions_Equatable() {
+    var options1 = InterpreterOptions()
+    var options2 = InterpreterOptions()
+    XCTAssertEqual(options1, options2)
+
+    options1.threadCount = 2
+    options2.threadCount = 2
+    XCTAssertEqual(options1, options2)
+
+    options2.threadCount = 3
+    XCTAssertNotEqual(options1, options2)
+    options2.threadCount = 2
+
+    options1.isErrorLoggingEnabled = true
+    options2.isErrorLoggingEnabled = true
+    XCTAssertEqual(options1, options2)
+
+    options2.isErrorLoggingEnabled = false
+    XCTAssertNotEqual(options1, options2)
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift b/tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift
new file mode 100644
index 0000000000..e98da5f951
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift
@@ -0,0 +1,315 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+@testable import TensorFlowLite
+import XCTest
+
+class InterpreterTests: XCTestCase {
+
+  var interpreter: Interpreter!
+
+  override func setUp() {
+    super.setUp()
+
+    interpreter = try! Interpreter(modelPath: AddModel.path)
+  }
+
+  override func tearDown() {
+    interpreter = nil
+
+    super.tearDown()
+  }
+
+  func testInterpreter_InitWithModelPath() {
+    XCTAssertNoThrow(try Interpreter(modelPath: AddModel.path))
+  }
+
+  func testInterpreter_Init_ThrowsFailedToLoadModel() {
+    XCTAssertThrowsError(try Interpreter(modelPath: "/invalid/path")) { error in
+      self.assertEqualErrors(actual: error, expected: .failedToLoadModel)
+    }
+  }
+
+  func testInterpreter_InitWithModelPathAndOptions() {
+    var options = InterpreterOptions()
+    options.threadCount = 2
+    XCTAssertNoThrow(try Interpreter(modelPath: AddModel.path, options: options))
+  }
+
+  func testInterpreter_InputTensorCount() {
+    XCTAssertEqual(interpreter.inputTensorCount, AddModel.inputTensorCount)
+  }
+
+  func testInterpreter_OutputTensorCount() {
+    XCTAssertEqual(interpreter.outputTensorCount, AddModel.outputTensorCount)
+  }
+
+  func testInterpreter_Invoke() throws {
+    try interpreter.allocateTensors()
+    XCTAssertNoThrow(try interpreter.invoke())
+  }
+
+  func testInterpreter_Invoke_ThrowsAllocateTensorsRequired_ModelNotReady() {
+    XCTAssertThrowsError(try interpreter.invoke()) { error in
+      self.assertEqualErrors(actual: error, expected: .allocateTensorsRequired)
+    }
+  }
+
+  func testInterpreter_InputTensorAtIndex() throws {
+    try setUpAddModelInputTensor()
+    let inputTensor = try interpreter.input(at: AddModel.validIndex)
+    XCTAssertEqual(inputTensor, AddModel.inputTensor)
+  }
+
+  func testInterpreter_InputTensorAtIndex_QuantizedModel() throws {
+    interpreter = try Interpreter(modelPath: AddQuantizedModel.path)
+    try setUpAddQuantizedModelInputTensor()
+    let inputTensor = try interpreter.input(at: AddQuantizedModel.inputOutputIndex)
+    XCTAssertEqual(inputTensor, AddQuantizedModel.inputTensor)
+  }
+
+  func testInterpreter_InputTensorAtIndex_ThrowsInvalidIndex() throws {
+    try interpreter.allocateTensors()
+    XCTAssertThrowsError(try interpreter.input(at: AddModel.invalidIndex)) { error in
+      let maxIndex = AddModel.inputTensorCount - 1
+      self.assertEqualErrors(
+        actual: error,
+        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
+      )
+    }
+  }
+
+  func testInterpreter_InputTensorAtIndex_ThrowsAllocateTensorsRequired() {
+    XCTAssertThrowsError(try interpreter.input(at: AddModel.validIndex)) { error in
+      self.assertEqualErrors(actual: error, expected: .allocateTensorsRequired)
+    }
+  }
+
+  func testInterpreter_OutputTensorAtIndex() throws {
+    try setUpAddModelInputTensor()
+    try interpreter.invoke()
+    let outputTensor = try interpreter.output(at: AddModel.validIndex)
+    XCTAssertEqual(outputTensor, AddModel.outputTensor)
+    let expectedResults = [Float32](unsafeData: outputTensor.data)
+    XCTAssertEqual(expectedResults, AddModel.results)
+  }
+
+  func testInterpreter_OutputTensorAtIndex_QuantizedModel() throws {
+    interpreter = try Interpreter(modelPath: AddQuantizedModel.path)
+    try setUpAddQuantizedModelInputTensor()
+    try interpreter.invoke()
+    let outputTensor = try interpreter.output(at: AddQuantizedModel.inputOutputIndex)
+    XCTAssertEqual(outputTensor, AddQuantizedModel.outputTensor)
+    let expectedResults = [UInt8](outputTensor.data)
+    XCTAssertEqual(expectedResults, AddQuantizedModel.results)
+  }
+
+  func testInterpreter_OutputTensorAtIndex_ThrowsInvalidIndex() throws {
+    try interpreter.allocateTensors()
+    try interpreter.invoke()
+    XCTAssertThrowsError(try interpreter.output(at: AddModel.invalidIndex)) { error in
+      let maxIndex = AddModel.outputTensorCount - 1
+      self.assertEqualErrors(
+        actual: error,
+        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
+      )
+    }
+  }
+
+  func testInterpreter_OutputTensorAtIndex_ThrowsInvokeInterpreterRequired() {
+    XCTAssertThrowsError(try interpreter.output(at: AddModel.validIndex)) { error in
+      self.assertEqualErrors(actual: error, expected: .invokeInterpreterRequired)
+    }
+  }
+
+  func testInterpreter_ResizeInputTensorAtIndexToShape() {
+    XCTAssertNoThrow(try interpreter.resizeInput(at: AddModel.validIndex, to: [2, 2, 3]))
+    XCTAssertNoThrow(try interpreter.allocateTensors())
+  }
+
+  func testInterpreter_ResizeInputTensorAtIndexToShape_ThrowsInvalidIndex() {
+    XCTAssertThrowsError(try interpreter.resizeInput(
+      at: AddModel.invalidIndex,
+      to: [2, 2, 3]
+    )) { error in
+      let maxIndex = AddModel.inputTensorCount - 1
+      self.assertEqualErrors(
+        actual: error,
+        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
+      )
+    }
+  }
+
+  func testInterpreter_CopyDataToInputTensorAtIndex() throws {
+    try interpreter.resizeInput(at: AddModel.validIndex, to: AddModel.shape)
+    try interpreter.allocateTensors()
+    let inputTensor = try interpreter.copy(AddModel.inputData, toInputAt: AddModel.validIndex)
+    XCTAssertEqual(inputTensor.data, AddModel.inputData)
+  }
+
+  func testInterpreter_CopyDataToInputTensorAtIndex_ThrowsInvalidIndex() {
+    XCTAssertThrowsError(try interpreter.copy(
+      AddModel.inputData,
+      toInputAt: AddModel.invalidIndex
+    )) { error in
+      let maxIndex = AddModel.inputTensorCount - 1
+      self.assertEqualErrors(
+        actual: error,
+        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
+      )
+    }
+  }
+
+  func testInterpreter_CopyDataToInputTensorAtIndex_ThrowsInvalidDataCount() throws {
+    try interpreter.resizeInput(at: AddModel.validIndex, to: AddModel.shape)
+    try interpreter.allocateTensors()
+    let invalidData = Data(count: AddModel.dataCount - 1)
+    XCTAssertThrowsError(try interpreter.copy(
+      invalidData,
+      toInputAt: AddModel.validIndex
+    )) { error in
+      self.assertEqualErrors(
+        actual: error,
+        expected: .invalidTensorDataCount(provided: invalidData.count, required: AddModel.dataCount)
+      )
+    }
+  }
+
+  func testInterpreter_AllocateTensors() {
+    XCTAssertNoThrow(try interpreter.allocateTensors())
+  }
+
+  // MARK: - Private
+
+  private func setUpAddModelInputTensor() throws {
+    precondition(interpreter != nil)
+    try interpreter.resizeInput(at: AddModel.validIndex, to: AddModel.shape)
+    try interpreter.allocateTensors()
+    try interpreter.copy(AddModel.inputData, toInputAt: AddModel.validIndex)
+  }
+
+  private func setUpAddQuantizedModelInputTensor() throws {
+    precondition(interpreter != nil)
+    try interpreter.resizeInput(at: AddQuantizedModel.inputOutputIndex, to: AddQuantizedModel.shape)
+    try interpreter.allocateTensors()
+    try interpreter.copy(AddQuantizedModel.inputData, toInputAt: AddQuantizedModel.inputOutputIndex)
+  }
+
+  private func assertEqualErrors(actual: Error, expected: InterpreterError) {
+    guard let actual = actual as? InterpreterError else {
+      XCTFail("Actual error should be of type InterpreterError.")
+      return
+    }
+    XCTAssertEqual(actual, expected)
+  }
+}
+
+// MARK: - Constants
+
+/// Values for the `add.bin` model.
+private enum AddModel {
+  static let info = (name: "add", extension: "bin")
+  static let inputTensorCount = 1
+  static let outputTensorCount = 1
+  static let invalidIndex = 1
+  static let validIndex = 0
+  static let shape: TensorShape = [2]
+  static let dataCount = inputData.count
+  static let inputData = Data(copyingBufferOf: [Float32(1.0), Float32(3.0)])
+  static let outputData = Data(copyingBufferOf: [Float32(3.0), Float32(9.0)])
+  static let results = [Float32(3.0), Float32(9.0)]
+
+  static let inputTensor = Tensor(
+    name: "input",
+    dataType: .float32,
+    shape: shape,
+    data: inputData
+  )
+  static let outputTensor = Tensor(
+    name: "output",
+    dataType: .float32,
+    shape: shape,
+    data: outputData
+  )
+
+  static var path: String = {
+    let bundle = Bundle(for: InterpreterTests.self)
+    guard let path = bundle.path(forResource: info.name, ofType: info.extension) else { return "" }
+    return path
+  }()
+}
+
+/// Values for the `add_quantized.bin` model.
+private enum AddQuantizedModel {
+  static let info = (name: "add_quantized", extension: "bin")
+  static let inputOutputIndex = 0
+  static let shape: TensorShape = [2]
+  static let inputData = Data([1, 3])
+  static let outputData = Data([3, 9])
+  static let quantizationParameters = QuantizationParameters(scale: 0.003922, zeroPoint: 0)
+  static let results: [UInt8] = [3, 9]
+
+  static let inputTensor = Tensor(
+    name: "input",
+    dataType: .uInt8,
+    shape: shape,
+    data: inputData,
+    quantizationParameters: quantizationParameters
+  )
+  static let outputTensor = Tensor(
+    name: "output",
+    dataType: .uInt8,
+    shape: shape,
+    data: outputData,
+    quantizationParameters: quantizationParameters
+  )
+
+  static var path: String = {
+    let bundle = Bundle(for: InterpreterTests.self)
+    guard let path = bundle.path(forResource: info.name, ofType: info.extension) else { return "" }
+    return path
+  }()
+}
+
+// MARK: - Extensions
+
+extension Array {
+  /// Creates a new array from the bytes of the given unsafe data.
+  ///
+  /// - Note: Returns `nil` if `unsafeData.count` is not a multiple of
+  ///     `MemoryLayout<Element>.stride`.
+  /// - Parameter unsafeData: The data containing the bytes to turn into an array.
+  init?(unsafeData: Data) {
+    guard unsafeData.count % MemoryLayout<Element>.stride == 0 else { return nil }
+    let elements = unsafeData.withUnsafeBytes {
+      UnsafeBufferPointer<Element>(
+        start: $0,
+        count: unsafeData.count / MemoryLayout<Element>.stride
+      )
+    }
+    self.init(elements)
+  }
+}
+
+extension Data {
+  /// Creates a new buffer by copying the buffer pointer of the given array.
+  ///
+  /// - Warning: The given array's element type `T` must be trivial in that it can be copied bit
+  ///     for bit with no indirection or reference-counting operations; otherwise, reinterpreting
+  ///     data from the resulting buffer has undefined behavior.
+  /// - Parameter array: An array with elements of type `T`.
+  init<T>(copyingBufferOf array: [T]) {
+    self = array.withUnsafeBufferPointer(Data.init)
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/Tests/ModelTests.swift b/tensorflow/lite/experimental/swift/Tests/ModelTests.swift
new file mode 100644
index 0000000000..025db18906
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Tests/ModelTests.swift
@@ -0,0 +1,59 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+@testable import TensorFlowLite
+import XCTest
+
+class ModelTests: XCTestCase {
+
+  var modelPath: String!
+
+  override func setUp() {
+    super.setUp()
+
+    let bundle = Bundle(for: type(of: self))
+    guard let modelPath = bundle.path(
+            forResource: Constant.modelInfo.name,
+            ofType: Constant.modelInfo.extension)
+    else {
+      XCTFail("Failed to get the model file path.")
+      return
+    }
+    self.modelPath = modelPath
+  }
+
+  override func tearDown() {
+    modelPath = nil
+
+    super.tearDown()
+  }
+
+  func testModel_InitWithFilePath() {
+    XCTAssertNotNil(Model(filePath: modelPath))
+  }
+
+  func testModel_InitWithEmptyFilePath_FailsInitialization() {
+    XCTAssertNil(Model(filePath: ""))
+  }
+
+  func testModel_InitWithInvalidFilePath_FailsInitialization() {
+    XCTAssertNil(Model(filePath: "invalid/path"))
+  }
+}
+
+// MARK: - Constants
+
+private enum Constant {
+  static let modelInfo = (name: "add", extension: "bin")
+}
diff --git a/tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift b/tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift
new file mode 100644
index 0000000000..65648c2698
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift
@@ -0,0 +1,43 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+@testable import TensorFlowLite
+import XCTest
+
+class QuantizationParametersTests: XCTestCase {
+
+  func testQuantizationParameters_InitWithCustomValues() {
+    let parameters = QuantizationParameters(scale: 0.5, zeroPoint: 1)
+    XCTAssertEqual(parameters.scale, 0.5)
+    XCTAssertEqual(parameters.zeroPoint, 1)
+  }
+
+  func testQuantizationParameters_Equatable() {
+    let parameters1 = QuantizationParameters(scale: 0.5, zeroPoint: 1)
+    let parameters2 = QuantizationParameters(scale: 0.5, zeroPoint: 1)
+    XCTAssertEqual(parameters1, parameters2)
+
+    let parameters3 = QuantizationParameters(scale: 0.4, zeroPoint: 1)
+    XCTAssertNotEqual(parameters1, parameters3)
+    XCTAssertNotEqual(parameters2, parameters3)
+  }
+}
+
+// MARK: - Extensions
+
+extension QuantizationParameters: Equatable {
+  public static func == (lhs: QuantizationParameters, rhs: QuantizationParameters) -> Bool {
+    return lhs.scale == rhs.scale && lhs.zeroPoint == rhs.zeroPoint
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/Tests/TensorTests.swift b/tensorflow/lite/experimental/swift/Tests/TensorTests.swift
new file mode 100644
index 0000000000..4540043a16
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Tests/TensorTests.swift
@@ -0,0 +1,83 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+@testable import TensorFlowLite
+import XCTest
+
+class TensorTests: XCTestCase {
+
+  // MARK: - Tensor
+
+  func testTensor_Init() {
+    let name = "InputTensor"
+    let dataType: TensorDataType = .uInt8
+    let shape = TensorShape(Constant.dimensions)
+    guard let data = name.data(using: .utf8) else { XCTFail("Data should not be nil."); return }
+    let quantizationParameters = QuantizationParameters(scale: 0.5, zeroPoint: 1)
+    let inputTensor = Tensor(
+      name: name,
+      dataType: dataType,
+      shape: shape,
+      data: data,
+      quantizationParameters: quantizationParameters
+    )
+    XCTAssertEqual(inputTensor.name, name)
+    XCTAssertEqual(inputTensor.dataType, dataType)
+    XCTAssertEqual(inputTensor.shape, shape)
+    XCTAssertEqual(inputTensor.data, data)
+    XCTAssertEqual(inputTensor.quantizationParameters, quantizationParameters)
+  }
+
+  // MARK: - TensorShape
+
+  func testTensorShape_InitWithArray() {
+    let shape = TensorShape(Constant.dimensions)
+    XCTAssertEqual(shape.rank, Constant.dimensions.count)
+    XCTAssertEqual(shape.dimensions, Constant.dimensions)
+  }
+
+  func testTensorShape_InitWithElements() {
+    let shape = TensorShape(2, 2, 3)
+    XCTAssertEqual(shape.rank, Constant.dimensions.count)
+    XCTAssertEqual(shape.dimensions, Constant.dimensions)
+  }
+
+  func testTensorShape_InitWithArrayLiteral() {
+    let shape: TensorShape = [2, 2, 3]
+    XCTAssertEqual(shape.rank, Constant.dimensions.count)
+    XCTAssertEqual(shape.dimensions, Constant.dimensions)
+  }
+}
+
+// MARK: - Constants
+
+private enum Constant {
+  /// Array of 2 arrays of 2 arrays of 3 numbers: [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]].
+  static let dimensions = [2, 2, 3]
+}
+
+// MARK: - Extensions
+
+extension TensorShape: Equatable {
+  public static func == (lhs: TensorShape, rhs: TensorShape) -> Bool {
+    return lhs.rank == rhs.rank && lhs.dimensions == rhs.dimensions
+  }
+}
+
+extension Tensor: Equatable {
+  public static func == (lhs: Tensor, rhs: Tensor) -> Bool {
+    return lhs.name == rhs.name && lhs.dataType == rhs.dataType && lhs.shape == rhs.shape &&
+           lhs.data == rhs.data && lhs.quantizationParameters == rhs.quantizationParameters
+  }
+}
diff --git a/tensorflow/tools/pip_package/pip_smoke_test.py b/tensorflow/tools/pip_package/pip_smoke_test.py
index 51d010c9e1..952c71c615 100644
--- a/tensorflow/tools/pip_package/pip_smoke_test.py
+++ b/tensorflow/tools/pip_package/pip_smoke_test.py
@@ -30,14 +30,19 @@ os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../..")))
 PIP_PACKAGE_QUERY_EXPRESSION = (
     "deps(//tensorflow/tools/pip_package:build_pip_package)")
 
+# List of file paths containing BUILD files that should not be included for the
+# pip smoke test.
+BUILD_BLACKLIST = [
+    "tensorflow/lite/examples/android",
+    "tensorflow/lite/experimental/swift",
+]
 
 def GetBuild(dir_base):
   """Get the list of BUILD file all targets recursively startind at dir_base."""
   items = []
   for root, _, files in os.walk(dir_base):
     for name in files:
-      if (name == "BUILD" and
-          root.find("tensorflow/lite/examples/android") == -1):
+      if (name == "BUILD" and root not in BUILD_BLACKLIST):
         items.append("//" + root + ":all")
   return items
 
@@ -67,9 +72,9 @@ def BuildPyTestDependencies():
 
 PYTHON_TARGETS, PY_TEST_QUERY_EXPRESSION = BuildPyTestDependencies()
 
-# Hard-coded blacklist of files if not included in pip package
 # TODO(amitpatankar): Clean up blacklist.
-BLACKLIST = [
+# List of dependencies that should not included in the pip package.
+DEPENDENCY_BLACKLIST = [
     "//tensorflow/python:extra_py_tests_deps",
     "//tensorflow/cc/saved_model:saved_model_half_plus_two",
     "//tensorflow:no_tensorflow_py_deps",
@@ -82,9 +87,7 @@ BLACKLIST = [
     "//tensorflow/core/kernels/cloud:bigquery_reader_ops",
     "//tensorflow/python/feature_column:vocabulary_testdata",
     "//tensorflow/python:framework/test_file_system.so",
-    # contrib
-    "//tensorflow/contrib/session_bundle:session_bundle_half_plus_two",
-    "//tensorflow/contrib/keras:testing_utils",
+    # lite
     "//tensorflow/lite/experimental/examples/lstm:tflite_lstm",
     "//tensorflow/lite/experimental/examples/lstm:tflite_lstm.py",
     "//tensorflow/lite/experimental/examples/lstm:unidirectional_sequence_lstm_test",  # pylint:disable=line-too-long
@@ -93,6 +96,9 @@ BLACKLIST = [
     "//tensorflow/lite/python:interpreter_test",
     "//tensorflow/lite/python:interpreter.py",
     "//tensorflow/lite/python:interpreter_test.py",
+    # contrib
+    "//tensorflow/contrib/session_bundle:session_bundle_half_plus_two",
+    "//tensorflow/contrib/keras:testing_utils",
     "//tensorflow/contrib/ffmpeg:test_data",
     "//tensorflow/contrib/fused_conv:fused_conv2d_bias_activation_op_test_base",
     "//tensorflow/contrib/hadoop:test_data",
@@ -149,8 +155,8 @@ def main():
   # File extensions and endings to ignore
   ignore_extensions = ["_test", "_test.py", "_test_gpu", "_test_gpu.py"]
 
-  ignored_files = 0
-  blacklisted_files = len(BLACKLIST)
+  ignored_files_count = 0
+  blacklisted_dependencies_count = len(DEPENDENCY_BLACKLIST)
   # Compare dependencies
   for dependency in tf_py_test_dependencies_list:
     if dependency and dependency.startswith("//tensorflow"):
@@ -158,16 +164,16 @@ def main():
       # Ignore extensions
       if any(dependency.endswith(ext) for ext in ignore_extensions):
         ignore = True
-        ignored_files += 1
+        ignored_files_count += 1
 
-      # Check if the dependency is in the pip package, the blacklist, or
-      # should be ignored because of its file extension
+      # Check if the dependency is in the pip package, the dependency blacklist,
+      # or should be ignored because of its file extension.
       if not (ignore or dependency in pip_package_dependencies_list or
-              dependency in BLACKLIST):
+              dependency in DEPENDENCY_BLACKLIST):
         missing_dependencies.append(dependency)
 
-  print("Ignored files: %d" % ignored_files)
-  print("Blacklisted files: %d" % blacklisted_files)
+  print("Ignored files count: %d" % ignored_files_count)
+  print("Blacklisted dependencies count: %d" % blacklisted_dependencies_count)
   if missing_dependencies:
     print("Missing the following dependencies from pip_packages:")
     for missing_dependency in missing_dependencies:
-- 
GitLab


From d28fddf859f0d9093a1279ceda52ebfb2d915788 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Fri, 4 Jan 2019 10:57:17 -0800
Subject: [PATCH 0192/2345] Fix some metric doc strings.

PiperOrigin-RevId: 227880840
---
 tensorflow/python/keras/metrics.py | 42 +++++++++++++++---------------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 73c116a745..7f13cc46e3 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -1382,8 +1382,8 @@ class MeanAbsoluteError(MeanMetricWrapper):
 
   Usage:
   ```python
-  mae = tf.metrics.MeanAbsoluteError()
-  mae.update_state([0., 0., 1., 1.], [1., 1., 1., 0.])
+  m = tf.metrics.MeanAbsoluteError()
+  m.update_state([0., 0., 1., 1.], [1., 1., 1., 0.])
   print('Final result: ', m.result().numpy())  # Final result: 0.75
   ```
 
@@ -1391,7 +1391,7 @@ class MeanAbsoluteError(MeanMetricWrapper):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.keras.losses.MeanAbsoluteError())
+  model.compile('sgd', metrics=[tf.keras.metrics.MeanAbsoluteError()])
   ```
   """
 
@@ -1416,8 +1416,8 @@ class MeanAbsolutePercentageError(MeanMetricWrapper):
   Usage:
 
   ```python
-  mape = tf.keras.losses.MeanAbsolutePercentageError()
-  mape.update_state([0., 0., 1., 1.], [1., 1., 1., 0.])
+  m = tf.keras.metrics.MeanAbsolutePercentageError()
+  m.update_state([0., 0., 1., 1.], [1., 1., 1., 0.])
   print('Final result: ', m.result().numpy())  # Final result: 5e+08
   ```
 
@@ -1425,7 +1425,7 @@ class MeanAbsolutePercentageError(MeanMetricWrapper):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.keras.losses.MeanAbsolutePercentageError())
+  model.compile('sgd', metrics=[tf.keras.metrics.MeanAbsolutePercentageError()])
   ```
   """
 
@@ -1450,8 +1450,8 @@ class MeanSquaredError(MeanMetricWrapper):
   Usage:
 
   ```python
-  mape = tf.keras.losses.MeanSquaredError()
-  mape.update_state([0., 0., 1., 1.], [1., 1., 1., 0.])
+  m = tf.keras.metrics.MeanSquaredError()
+  m.update_state([0., 0., 1., 1.], [1., 1., 1., 0.])
   print('Final result: ', m.result().numpy())  # Final result: 0.75
   ```
 
@@ -1459,7 +1459,7 @@ class MeanSquaredError(MeanMetricWrapper):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.keras.losses.MeanSquaredError())
+  model.compile('sgd', metrics=[tf.keras.metrics.MeanSquaredError()])
   ```
   """
 
@@ -1484,8 +1484,8 @@ class MeanSquaredLogarithmicError(MeanMetricWrapper):
   Usage:
 
   ```python
-  msle = tf.keras.losses.MeanSquaredLogarithmicError()
-  msle.update_state([0., 0., 1., 1.], [1., 1., 1., 0.])
+  m = tf.keras.metrics.MeanSquaredLogarithmicError()
+  m.update_state([0., 0., 1., 1.], [1., 1., 1., 0.])
   print('Final result: ', m.result().numpy())  # Final result: 0.36034
   ```
 
@@ -1493,7 +1493,7 @@ class MeanSquaredLogarithmicError(MeanMetricWrapper):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.keras.losses.MeanSquaredLogarithmicError())
+  model.compile('sgd', metrics=[tf.keras.metrics.MeanSquaredLogarithmicError()])
   ```
   """
 
@@ -1518,8 +1518,8 @@ class Hinge(MeanMetricWrapper):
   Usage:
 
   ```python
-  h = tf.keras.metrics.Hinge()
-  h.update_state([0., 1., 1.], [1., 0., 1.])
+  m = tf.keras.metrics.Hinge()
+  m.update_state([0., 1., 1.], [1., 0., 1.])
   print('Final result: ', m.result().numpy())  # Final result: 0.66
   ```
 
@@ -1527,7 +1527,7 @@ class Hinge(MeanMetricWrapper):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.keras.metrics.Hinge())
+  model.compile('sgd', metrics=[tf.keras.metrics.Hinge()])
   ```
   """
 
@@ -1551,8 +1551,8 @@ class SquaredHinge(MeanMetricWrapper):
   Usage:
 
   ```python
-  h = tf.keras.metrics.SquaredHinge()
-  h.update_state([0., 1., 1.], [1., 0., 1.])
+  m = tf.keras.metrics.SquaredHinge()
+  m.update_state([0., 1., 1.], [1., 0., 1.])
   print('Final result: ', m.result().numpy())  # Final result: 0.66
   ```
 
@@ -1560,7 +1560,7 @@ class SquaredHinge(MeanMetricWrapper):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.keras.metrics.SquaredHinge())
+  model.compile('sgd', metrics=[tf.keras.metrics.SquaredHinge()])
   ```
   """
 
@@ -1584,8 +1584,8 @@ class CategoricalHinge(MeanMetricWrapper):
   Usage:
 
   ```python
-  h = tf.keras.metrics.CategoricalHinge()
-  h.update_state([0., 1., 1.], [1., 0., 1.])
+  m = tf.keras.metrics.CategoricalHinge()
+  m.update_state([0., 1., 1.], [1., 0., 1.])
   print('Final result: ', m.result().numpy())  # Final result: 1.0
   ```
 
@@ -1593,7 +1593,7 @@ class CategoricalHinge(MeanMetricWrapper):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.keras.metrics.CategoricalHinge())
+  model.compile('sgd', metrics=[tf.keras.metrics.CategoricalHinge()])
   ```
   """
 
-- 
GitLab


From 49d70dff868661bc7aa999227dad64bde2538899 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 10:58:35 -0800
Subject: [PATCH 0193/2345] Make tf.where documentation clearer.

PiperOrigin-RevId: 227881065
---
 tensorflow/python/ops/array_ops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 07a7e180fb..81558db04f 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -3191,7 +3191,7 @@ def where(condition, x=None, y=None, name=None):
 
   Returns:
     A `Tensor` with the same type and shape as `x`, `y` if they are non-None.
-    A `Tensor` with shape `(num_true, dim_size(condition))`.
+    Otherwise, a `Tensor` with shape `(num_true, rank(condition))`.
 
   Raises:
     ValueError: When exactly one of `x` or `y` is non-None.
-- 
GitLab


From 9d72bb94c054b9d3a00f39e6a99c02995b207ae6 Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Fri, 4 Jan 2019 11:04:06 -0800
Subject: [PATCH 0194/2345] [XLA] The HLO evaluator should preserve the layouts
 for instructions inside a fusion.

When evaluating a fusion, the HLO evaluator makes a copy of the fusion and
resets the layout of each instruction in the copy to default layout before
evaluating the copy of the fusion. In doing that, it changes the layout of the
fusion intputs and outputs and causes incorrect results. This change fixes the
evaluator to not overwrite the layouts for the instructions inside a fusion.

Add test cases for fusion inputs, single output fusion and multiple output
fusion.

PiperOrigin-RevId: 227882142
---
 .../compiler/xla/service/hlo_evaluator.cc     |  4 +-
 .../xla/service/hlo_evaluator_test.cc         | 66 +++++++++++++++++++
 2 files changed, 69 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index e897d09e1b..9754b8906b 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -1097,7 +1097,9 @@ Status HloEvaluator::HandleFusion(HloInstruction* fusion) {
       fusion->fused_instructions_computation()->Clone(
           /*suffix=*/"clone_with_layout", &context);
   for (auto* instruction : cloned_fused_computation->instructions()) {
-    LayoutUtil::SetToDefaultLayout(instruction->mutable_shape());
+    if (!LayoutUtil::HasLayout(instruction->shape())) {
+      LayoutUtil::SetToDefaultLayout(instruction->mutable_shape());
+    }
   }
   auto readded_computation =
       empty_hlo_module.AddEntryComputation(std::move(cloned_fused_computation));
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
index 0298da2174..8056193d99 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
@@ -2924,5 +2924,71 @@ ENTRY main {
             "Expected 1 argument, but got 2.");
 }
 
+TEST_F(HloEvaluatorTest, PreserveFusionInputLayout) {
+  constexpr absl::string_view hlo_text = R"(
+    HloModule FusionInputLayout
+
+    fused_computation {
+      param_0 = f32[20,20]{0,1} parameter(0)
+      ROOT bitcast = f32[20,20]{1,0} bitcast(param_0)
+    }
+
+    ENTRY kernel_entry {
+      parameter.0 = f32[20,20]{0,1} parameter(0)
+      ROOT fusion = f32[20,20]{1,0} fusion(parameter.0),
+        kind=kLoop, calls=fused_computation
+    })";
+
+  TF_ASSERT_OK_AND_ASSIGN(m_, ParseAndReturnVerifiedModule(hlo_text));
+  auto args = MakeFakeArguments(m_.get()).ConsumeValueOrDie();
+  Literal actual = Evaluate({&args[0]});
+  EXPECT_TRUE(absl::c_equal(args[0].data<float>(), actual.data<float>()));
+}
+
+TEST_F(HloEvaluatorTest, PreserveFusionOutputLayout) {
+  constexpr absl::string_view hlo_text = R"(
+    HloModule FusionOutputLayout
+
+    fused_computation {
+      param_0 = f32[20,20]{1,0} parameter(0)
+      ROOT bitcast = f32[20,20]{0,1} bitcast(param_0)
+    }
+
+    ENTRY kernel_entry {
+      parameter.0 = f32[20,20]{1,0} parameter(0)
+      ROOT fusion = f32[20,20]{0,1} fusion(parameter.0),
+        kind=kLoop, calls=fused_computation
+    })";
+
+  TF_ASSERT_OK_AND_ASSIGN(m_, ParseAndReturnVerifiedModule(hlo_text));
+  auto args = MakeFakeArguments(m_.get()).ConsumeValueOrDie();
+  Literal actual = Evaluate({&args[0]});
+  EXPECT_TRUE(absl::c_equal(args[0].data<float>(), actual.data<float>()));
+}
+
+TEST_F(HloEvaluatorTest, PreserveMOFusionOutputLayout) {
+  constexpr absl::string_view hlo_text = R"(
+    HloModule MOFusionOutputLayout
+
+    fused_computation {
+      param_0 = f32[20,20]{1,0} parameter(0)
+      bitcast = f32[20,20]{0,1} bitcast(param_0)
+      ROOT tuple = (f32[20,20]{0,1}) tuple(bitcast)
+    }
+
+    ENTRY kernel_entry {
+      parameter.0 = f32[20,20]{1,0} parameter(0)
+      ROOT fusion = (f32[20,20]{0,1}) fusion(parameter.0),
+        kind=kLoop, calls=fused_computation
+    })";
+
+  TF_ASSERT_OK_AND_ASSIGN(m_, ParseAndReturnVerifiedModule(hlo_text));
+  auto args = MakeFakeArguments(m_.get()).ConsumeValueOrDie();
+  Literal actual_tuple = Evaluate({&args[0]});
+  std::vector<Literal> actual_literals = actual_tuple.DecomposeTuple();
+  EXPECT_TRUE(
+      absl::c_equal(args[0].data<float>(), actual_literals[0].data<float>()));
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From d76c2d0a570495854617c318a2708636f5530f27 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Fri, 4 Jan 2019 11:10:12 -0800
Subject: [PATCH 0195/2345] Improve the error message for model
 fit/eval/predict in scope

PiperOrigin-RevId: 227883249
---
 tensorflow/contrib/distribute/python/keras_test.py          | 2 +-
 .../python/keras/engine/distributed_training_utils.py       | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 4cfcc0375f..1cb5fa30a3 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -1196,7 +1196,7 @@ class TestDistributionStrategyValidation(test.TestCase,
   def test_loop_in_scope(self, distribution):
     with self.cached_session():
       with self.assertRaisesRegexp(
-          RuntimeError, 'should not be run inside the distribution strategy'):
+          RuntimeError, 'should not be run inside the tf.distribute.Strategy'):
         with distribution.scope():
           x = keras.layers.Input(shape=(3,), name='input')
           y = keras.layers.Dense(4, name='dense')(x)
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index 9e3bdd26cb..1678d84307 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -48,9 +48,9 @@ def validate_not_in_strategy_scope():
   if distribution_strategy_context.has_distribution_strategy():
     if distribution_strategy_context.in_cross_replica_context():
       raise RuntimeError(
-          'Fit/Eval/Predict should not be run inside the distribution strategy '
-          'scope. Only model creation and compilation should be in '
-          'distribution strategy scope.')
+          'Fit/Eval/Predict should not be run inside the tf.distribute.Strategy'
+          ' scope. Only model creation and compilation should be in '
+          'tf.distribute.Strategy scope.')
 
 
 def set_weights(distribution_strategy, dist_model, weights):
-- 
GitLab


From 735d26a821d9f69f58ad12ea841da9bc8c5a65ee Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 11:18:24 -0800
Subject: [PATCH 0196/2345] Make session_handle of DirectSession an unique
 identifier and accessible from OptimizationPassRegistry::PRE_PLACEMENT pass.

PiperOrigin-RevId: 227884596
---
 tensorflow/core/common_runtime/direct_session.cc      | 11 +++++++----
 tensorflow/core/common_runtime/direct_session.h       |  1 +
 tensorflow/core/common_runtime/executor.cc            |  3 +++
 tensorflow/core/common_runtime/executor.h             |  2 ++
 .../core/common_runtime/graph_execution_state.cc      |  3 +++
 .../core/common_runtime/graph_execution_state.h       |  4 ++++
 .../core/common_runtime/optimization_registry.h       |  1 +
 tensorflow/core/framework/op_kernel.h                 |  6 ++++++
 8 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/common_runtime/direct_session.cc b/tensorflow/core/common_runtime/direct_session.cc
index 51b2c68c76..36f1a92aab 100644
--- a/tensorflow/core/common_runtime/direct_session.cc
+++ b/tensorflow/core/common_runtime/direct_session.cc
@@ -59,6 +59,7 @@ limitations under the License.
 #include "tensorflow/core/lib/gtl/array_slice.h"
 #include "tensorflow/core/lib/gtl/stl_util.h"
 #include "tensorflow/core/lib/monitoring/counter.h"
+#include "tensorflow/core/lib/random/random.h"
 #include "tensorflow/core/lib/strings/numbers.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
@@ -303,10 +304,8 @@ DirectSession::DirectSession(const SessionOptions& options,
   if (!status.ok()) {
     LOG(ERROR) << status.error_message();
   }
-  // NOTE(mrry): We do not need to use a unique string for the session
-  // handle, because DirectSession owns its devices. This may change
-  // in future versions.
-  session_handle_ = "direct";
+  session_handle_ =
+      strings::StrCat("direct", strings::FpToString(random::New64()));
   int devices_added = 0;
   if (options.config.log_device_placement()) {
     const string mapping_str = device_mgr_->DeviceMappingString();
@@ -371,6 +370,7 @@ Status DirectSession::MaybeInitializeExecutionState(
   GraphExecutionStateOptions options;
   options.device_set = &device_set_;
   options.session_options = &options_;
+  options.session_handle = session_handle_;
   // TODO(mrry,suharshs): We explicitly copy `graph` so that
   // `MakeForBaseGraph()` can take ownership of its
   // contents. Previously this happened implicitly in calls to the
@@ -533,6 +533,7 @@ Status DirectSession::RunInternal(int64 step_id, const RunOptions& run_options,
   CancellationManager step_cancellation_manager;
   args.cancellation_manager = &step_cancellation_manager;
   args.session_state = &session_state_;
+  args.session_handle = session_handle_;
   args.tensor_store = &run_state.tensor_store;
   args.step_container = &run_state.step_container;
   args.sync_on_finish = sync_on_finish_;
@@ -888,6 +889,7 @@ Status DirectSession::PRunSetup(const std::vector<string>& input_names,
     SchedClosure(pool, std::move(c));
   };
   args.session_state = &session_state_;
+  args.session_handle = session_handle_;
   args.tensor_store = &run_state->tensor_store;
   args.step_container = &run_state->step_container;
   if (LogMemory::IsEnabled()) {
@@ -1465,6 +1467,7 @@ Status DirectSession::CreateGraphs(
     prune_options.device_set = &device_set_;
     prune_options.session_options = &options_;
     prune_options.stateful_placements = stateful_placements_;
+    prune_options.session_handle = session_handle_;
     TF_RETURN_IF_ERROR(GraphExecutionState::MakeForPrunedGraph(
         execution_state_->original_graph_def().library(), prune_options,
         execution_state_->original_graph_def(), subgraph_options,
diff --git a/tensorflow/core/common_runtime/direct_session.h b/tensorflow/core/common_runtime/direct_session.h
index 6754e9cfb7..bcac341544 100644
--- a/tensorflow/core/common_runtime/direct_session.h
+++ b/tensorflow/core/common_runtime/direct_session.h
@@ -317,6 +317,7 @@ class DirectSession : public Session {
   std::vector<Device*> devices_;  // not owned
   DeviceSet device_set_;
 
+  // Unique session identifier.
   string session_handle_;
   mutex graph_state_lock_;
   bool graph_created_ GUARDED_BY(graph_state_lock_) = false;
diff --git a/tensorflow/core/common_runtime/executor.cc b/tensorflow/core/common_runtime/executor.cc
index df2ee6c618..07c8c4a5d4 100644
--- a/tensorflow/core/common_runtime/executor.cc
+++ b/tensorflow/core/common_runtime/executor.cc
@@ -1244,6 +1244,7 @@ class ExecutorState {
   Rendezvous* rendezvous_;
   CollectiveExecutor* collective_executor_ = nullptr;
   SessionState* session_state_;
+  string session_handle_;
   TensorStore* tensor_store_;
   // Step-local container.
   ScopedStepContainer* step_container_;
@@ -1371,6 +1372,7 @@ ExecutorState::ExecutorState(const Executor::Args& args, ExecutorImpl* impl)
       rendezvous_(args.rendezvous),
       collective_executor_(args.collective_executor),
       session_state_(args.session_state),
+      session_handle_(args.session_handle),
       tensor_store_(args.tensor_store),
       step_container_(args.step_container),
       stats_collector_(args.stats_collector),
@@ -1616,6 +1618,7 @@ void ExecutorState::Process(TaggedNode tagged_node, int64 scheduled_nsec) {
   params.rendezvous = rendezvous_;
   params.collective_executor = collective_executor_;
   params.session_state = session_state_;
+  params.session_handle = session_handle_;
   params.tensor_store = tensor_store_;
   params.cancellation_manager = cancellation_manager_;
   params.call_frame = call_frame_;
diff --git a/tensorflow/core/common_runtime/executor.h b/tensorflow/core/common_runtime/executor.h
index 02930168a4..4be60c6771 100644
--- a/tensorflow/core/common_runtime/executor.h
+++ b/tensorflow/core/common_runtime/executor.h
@@ -88,6 +88,8 @@ class Executor {
     CallFrameInterface* call_frame = nullptr;
     CancellationManager* cancellation_manager = nullptr;
     SessionState* session_state = nullptr;
+    // Unique session identifier. Can be empty.
+    string session_handle;
     TensorStore* tensor_store = nullptr;
     ScopedStepContainer* step_container = nullptr;
     CollectiveExecutor* collective_executor = nullptr;
diff --git a/tensorflow/core/common_runtime/graph_execution_state.cc b/tensorflow/core/common_runtime/graph_execution_state.cc
index 04d658f047..9ecbc34f5f 100644
--- a/tensorflow/core/common_runtime/graph_execution_state.cc
+++ b/tensorflow/core/common_runtime/graph_execution_state.cc
@@ -59,6 +59,7 @@ GraphExecutionState::GraphExecutionState(
     : stateful_placements_(options.stateful_placements),
       device_set_(options.device_set),
       session_options_(options.session_options),
+      session_handle_(options.session_handle),
       flib_def_(new FunctionLibraryDefinition(OpRegistry::Global(),
                                               graph_def->library())),
       graph_(nullptr) {
@@ -198,6 +199,7 @@ Status GraphExecutionState::Extend(
   GraphExecutionStateOptions combined_options;
   combined_options.device_set = device_set_;
   combined_options.session_options = session_options_;
+  combined_options.session_handle = session_handle_;
   combined_options.stateful_placements = stateful_placements_;
 
   // NOTE(mrry): `gdef` is no longer valid after the constructor
@@ -558,6 +560,7 @@ Status GraphExecutionState::InitBaseGraph(const BuildGraphOptions& options) {
   RestoreStatefulNodes(new_graph.get());
 
   GraphOptimizationPassOptions optimization_options;
+  optimization_options.session_handle = session_handle_;
   optimization_options.session_options = session_options_;
   optimization_options.graph = &new_graph;
   optimization_options.flib_def = flib_def_.get();
diff --git a/tensorflow/core/common_runtime/graph_execution_state.h b/tensorflow/core/common_runtime/graph_execution_state.h
index 9cabe478a6..56315bb1ef 100644
--- a/tensorflow/core/common_runtime/graph_execution_state.h
+++ b/tensorflow/core/common_runtime/graph_execution_state.h
@@ -41,6 +41,8 @@ struct RewriteGraphMetadata;
 struct GraphExecutionStateOptions {
   const DeviceSet* device_set = nullptr;
   const SessionOptions* session_options = nullptr;
+  // Unique session identifier. Can be empty.
+  string session_handle;
   // A map from node name to device name, representing the unchangeable
   // placement of stateful nodes.
   std::unordered_map<string, string> stateful_placements;
@@ -192,6 +194,8 @@ class GraphExecutionState {
   GraphDef original_graph_def_;            // Immutable after ctor.
   const DeviceSet* device_set_;            // Not owned
   const SessionOptions* session_options_;  // Not owned
+  // Unique session identifier. Can be empty.
+  string session_handle_;
 
   // Map from name to Node for the full graph in placed_.
   NodeNameToCostIdMap node_name_to_cost_id_map_;
diff --git a/tensorflow/core/common_runtime/optimization_registry.h b/tensorflow/core/common_runtime/optimization_registry.h
index 6fcd2afd27..19a76529bb 100644
--- a/tensorflow/core/common_runtime/optimization_registry.h
+++ b/tensorflow/core/common_runtime/optimization_registry.h
@@ -35,6 +35,7 @@ struct SessionOptions;
 // as a key into a state dictionary if it wants to keep state across
 // calls.
 struct GraphOptimizationPassOptions {
+  // Filled in by DirectSession for PRE_PLACEMENT optimizations. Can be empty.
   string session_handle;
   const SessionOptions* session_options = nullptr;
   const CostModel* cost_model = nullptr;
diff --git a/tensorflow/core/framework/op_kernel.h b/tensorflow/core/framework/op_kernel.h
index bccb2bf3c7..aa07cbd380 100644
--- a/tensorflow/core/framework/op_kernel.h
+++ b/tensorflow/core/framework/op_kernel.h
@@ -606,6 +606,9 @@ class OpKernelContext {
     // The session state for this op.
     SessionState* session_state = nullptr;
 
+    // Unique session identifier. Can be empty.
+    string session_handle;
+
     // The tensor store for this op.
     TensorStore* tensor_store = nullptr;
 
@@ -1034,6 +1037,9 @@ class OpKernelContext {
   // An op kernel can access the session state it belongs to.
   SessionState* session_state() const { return params_->session_state; }
 
+  // Unique identifier of the session it belongs to. Can be empty.
+  string session_handle() const { return params_->session_handle; }
+
   // An op kernel can access the tensor store of the run it belongs to.
   TensorStore* tensor_store() const { return params_->tensor_store; }
 
-- 
GitLab


From 8c22259497e9914c37a7adcd33aebaf754473a02 Mon Sep 17 00:00:00 2001
From: Laurent Le Brun <laurentlb@gmail.com>
Date: Fri, 4 Jan 2019 20:37:31 +0100
Subject: [PATCH 0197/2345] Update dependency on rules_closure

This makes this command succeed:
  bazel build //tensorflow/tools/pip_package:build_pip_package --incompatible_disallow_data_transition

This is needed for a future change in Bazel.
---
 WORKSPACE | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/WORKSPACE b/WORKSPACE
index 2277e83a3f..1c59686f16 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -4,11 +4,11 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file"
 
 http_archive(
     name = "io_bazel_rules_closure",
-    sha256 = "a38539c5b5c358548e75b44141b4ab637bba7c4dc02b46b1f62a96d6433f56ae",
-    strip_prefix = "rules_closure-dbb96841cc0a5fb2664c37822803b06dab20c7d1",
+    sha256 = "43c9b882fa921923bcba764453f4058d102bece35a37c9f6383c713004aacff1",
+    strip_prefix = "rules_closure-9889e2348259a5aad7e805547c1a0cf311cfcd91",
     urls = [
-        "https://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/dbb96841cc0a5fb2664c37822803b06dab20c7d1.tar.gz",
-        "https://github.com/bazelbuild/rules_closure/archive/dbb96841cc0a5fb2664c37822803b06dab20c7d1.tar.gz",  # 2018-04-13
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/9889e2348259a5aad7e805547c1a0cf311cfcd91.tar.gz",
+        "https://github.com/bazelbuild/rules_closure/archive/9889e2348259a5aad7e805547c1a0cf311cfcd91.tar.gz",  # 2018-12-21
     ],
 )
 
-- 
GitLab


From 3e37a58b6615582cca0006be22d95edf4945f078 Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Fri, 4 Jan 2019 11:44:16 -0800
Subject: [PATCH 0198/2345] Update documentation for Apollo3 board

PiperOrigin-RevId: 227889248
---
 tensorflow/lite/experimental/micro/README.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/experimental/micro/README.md b/tensorflow/lite/experimental/micro/README.md
index dab7ae80f3..464c7b6ad7 100644
--- a/tensorflow/lite/experimental/micro/README.md
+++ b/tensorflow/lite/experimental/micro/README.md
@@ -187,7 +187,7 @@ You should see the following log with the magic string `~~~ALL TEST PASSED~~~`:
 02:25:22.4253 [DEBUG] uart0: [+0.16ms host +0s virt 0.28s virt from start]   Progam has exited with code:0x00000000
 ```
 
-## Building for Apollo3
+## Building for Ambiq Micro Apollo3Blue EVB
 
 Follow these steps to get the pushbutton yes/no example working on Apollo 3:
 
@@ -201,7 +201,8 @@ Follow these steps to get the pushbutton yes/no example working on Apollo 3:
     tensorflow/lite/experimental/micro/tools/make/Makefile TARGET=apollo3evb
     pushbutton_cmsis_speech_test_bin
 4.  Install [Segger JLink tools](https://www.segger.com/downloads/jlink/)
-5.  Connect the Apollo3 EVB (with mic shield) to the computer and power it on
+5.  Connect the Apollo3 EVB (with mic shield in slot 3 of Microbus Shield board)
+    to the computer and power it on.
 6.  Start the GDB server in a new terminal with the following command:
     JLinkGDBServer -select USB -device AMA3B1KK-KBR -endian little -if SWD
     -speed 1000 -noir -noLocalhostOnly
-- 
GitLab


From 37afc40b6f33073193114c081bcda939289deca0 Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Fri, 4 Jan 2019 11:57:01 -0800
Subject: [PATCH 0199/2345] Roll-forward with fix: Fix dso_loader.cc's
 includes.

tf_custom_op_library already adds core:stream_executor_headers_lib as a dep, so we must not add a duplicated dep here.

PiperOrigin-RevId: 227891141
---
 tensorflow/contrib/mpi_collectives/BUILD                  | 1 -
 tensorflow/stream_executor/platform/default/dso_loader.cc | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/contrib/mpi_collectives/BUILD b/tensorflow/contrib/mpi_collectives/BUILD
index ecac06354d..a7be92a35e 100644
--- a/tensorflow/contrib/mpi_collectives/BUILD
+++ b/tensorflow/contrib/mpi_collectives/BUILD
@@ -52,7 +52,6 @@ tf_custom_op_library(
     deps = [
         ":mpi_defines",
         ":mpi_message_proto_cc",
-        "//tensorflow/stream_executor:stream_executor_headers_lib",
         "//third_party/mpi",
     ],
 )
diff --git a/tensorflow/stream_executor/platform/default/dso_loader.cc b/tensorflow/stream_executor/platform/default/dso_loader.cc
index 0f0bce3253..668eeee3f3 100644
--- a/tensorflow/stream_executor/platform/default/dso_loader.cc
+++ b/tensorflow/stream_executor/platform/default/dso_loader.cc
@@ -28,7 +28,7 @@ limitations under the License.
 #include "tensorflow/stream_executor/lib/path.h"
 #include "tensorflow/stream_executor/lib/str_util.h"
 #include "tensorflow/stream_executor/lib/stringprintf.h"
-#include "tensorflow/stream_executor/platform/dso_loader.h"
+#include "tensorflow/stream_executor/platform/default/dso_loader.h"
 #include "tensorflow/stream_executor/platform/logging.h"
 #include "tensorflow/stream_executor/platform/port.h"
 
-- 
GitLab


From 45b2b27f6f30ec981300fb6987736f14caa45f29 Mon Sep 17 00:00:00 2001
From: Yuxin Wu <ppwwyyxx@users.noreply.github.com>
Date: Fri, 4 Jan 2019 12:17:10 -0800
Subject: [PATCH 0200/2345] improve docs of depthwise conv

---
 tensorflow/python/ops/nn_impl.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/ops/nn_impl.py b/tensorflow/python/ops/nn_impl.py
index ec5ea44526..7abfde5149 100644
--- a/tensorflow/python/ops/nn_impl.py
+++ b/tensorflow/python/ops/nn_impl.py
@@ -467,7 +467,7 @@ def depthwise_conv2d(input,
   to `channel_multiplier` channels for each), then concatenates the results
   together.  The output has `in_channels * channel_multiplier` channels.
 
-  In detail,
+  In detail, with the default NHWC format,
 
       output[b, i, j, k * channel_multiplier + q] = sum_{di, dj}
            filter[di, dj, k, q] * input[b, strides[1] * i + rate[0] * di,
@@ -540,7 +540,7 @@ def depthwise_conv2d_v2(input,
   to `channel_multiplier` channels for each), then concatenates the results
   together.  The output has `in_channels * channel_multiplier` channels.
 
-  In detail,
+  In detail, with the default NHWC format,
 
       output[b, i, j, k * channel_multiplier + q] = sum_{di, dj}
            filter[di, dj, k, q] * input[b, strides[1] * i + rate[0] * di,
@@ -599,7 +599,7 @@ def separable_conv2d(input,
   between dimensions `[1, 2]` and `3`, not spatial separability between
   dimensions `1` and `2`.
 
-  In detail,
+  In detail, with the default NHWC format,
 
       output[b, i, j, k] = sum_{di, dj, q, r}
           input[b, strides[1] * i + di, strides[2] * j + dj, q] *
@@ -699,7 +699,7 @@ def separable_conv2d_v2(
   between dimensions `[1, 2]` and `3`, not spatial separability between
   dimensions `1` and `2`.
 
-  In detail,
+  In detail, with the default NHWC format,
 
       output[b, i, j, k] = sum_{di, dj, q, r}
           input[b, strides[1] * i + di, strides[2] * j + dj, q] *
-- 
GitLab


From f1d4d18a622a3bb030ebc92b60763fe35a755bd5 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Fri, 4 Jan 2019 12:17:54 -0800
Subject: [PATCH 0201/2345] Add logcosh v2 loss and metric.

PiperOrigin-RevId: 227894520
---
 tensorflow/python/keras/losses.py       | 27 +++++++++
 tensorflow/python/keras/losses_test.py  | 81 +++++++++++++++++++++++++
 tensorflow/python/keras/metrics.py      | 31 ++++++++++
 tensorflow/python/keras/metrics_test.py | 43 +++++++++++++
 4 files changed, 182 insertions(+)

diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index 6a428b28a9..1ff564054b 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -540,6 +540,33 @@ class LogLoss(Loss):
     return logloss(y_true, y_pred, epsilon=self.epsilon)
 
 
+class Logcosh(Loss):
+  """Computes the logarithm of the hyperbolic cosine of the prediction error.
+
+  logcosh = log((exp(x) + exp(-x))/2) where x is the error `y_pred` - `y_true`.
+
+  Usage:
+
+  ```python
+  l = tf.losses.Logcosh()
+  loss = l([0., 1., 1.], [1., 0., 1.])
+  print('Loss: ', loss.numpy())  # Loss: 0.289
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', loss=tf.losses.Logcosh())
+  ```
+  """
+
+  def call(self, y_true, y_pred):
+    y_pred = ops.convert_to_tensor(y_pred)
+    y_true = math_ops.cast(y_true, y_pred.dtype)
+    return logcosh(y_true, y_pred)
+
+
 @keras_export('keras.metrics.mean_squared_error',
               'keras.metrics.mse',
               'keras.metrics.MSE',
diff --git a/tensorflow/python/keras/losses_test.py b/tensorflow/python/keras/losses_test.py
index 76fadc4a24..aa00ba5a91 100644
--- a/tensorflow/python/keras/losses_test.py
+++ b/tensorflow/python/keras/losses_test.py
@@ -1094,5 +1094,86 @@ class LogLossTest(test.TestCase):
     self.assertAlmostEqual(self.evaluate(loss), 0., 3)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class LogcoshTest(test.TestCase):
+
+  def setup(self):
+    y_pred = np.asarray([1, 9, 2, -5, -2, 6]).reshape((2, 3))
+    y_true = np.asarray([4, 8, 12, 8, 1, 3]).reshape((2, 3))
+
+    self.batch_size = 6
+    error = y_pred - y_true
+    self.expected_losses = np.log((np.exp(error) + np.exp(-error)) / 2)
+
+    self.y_pred = constant_op.constant(y_pred, dtype=dtypes.float32)
+    self.y_true = constant_op.constant(y_true)
+
+  def test_config(self):
+    logcosh_obj = keras.losses.Logcosh(
+        reduction=losses_impl.ReductionV2.SUM, name='logcosh_loss')
+    self.assertEqual(logcosh_obj.name, 'logcosh_loss')
+    self.assertEqual(logcosh_obj.reduction, losses_impl.ReductionV2.SUM)
+
+  def test_unweighted(self):
+    self.setup()
+    logcosh_obj = keras.losses.Logcosh()
+
+    loss = logcosh_obj(self.y_true, self.y_pred)
+    expected_loss = np.sum(self.expected_losses) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+
+  def test_scalar_weighted(self):
+    self.setup()
+    logcosh_obj = keras.losses.Logcosh()
+    sample_weight = 2.3
+
+    loss = logcosh_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    expected_loss = sample_weight * np.sum(
+        self.expected_losses) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+
+    # Verify we get the same output when the same input is given
+    loss_2 = logcosh_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), self.evaluate(loss_2), 3)
+
+  def test_sample_weighted(self):
+    self.setup()
+    logcosh_obj = keras.losses.Logcosh()
+
+    sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
+    loss = logcosh_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+
+    expected_loss = np.multiply(
+        self.expected_losses,
+        np.asarray([1.2, 1.2, 1.2, 3.4, 3.4, 3.4]).reshape((2, 3)))
+    expected_loss = np.sum(expected_loss) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+
+  def test_timestep_weighted(self):
+    self.setup()
+    logcosh_obj = keras.losses.Logcosh()
+    y_true = np.asarray([1, 9, 2, -5, -2, 6]).reshape(2, 3, 1)
+    y_pred = np.asarray([4, 8, 12, 8, 1, 3]).reshape(2, 3, 1)
+    error = y_pred - y_true
+    expected_losses = np.log((np.exp(error) + np.exp(-error)) / 2)
+    sample_weight = np.array([3, 6, 5, 0, 4, 2]).reshape((2, 3, 1))
+
+    y_pred = constant_op.constant(y_pred, dtype=dtypes.float32)
+    y_true = constant_op.constant(y_true)
+    loss = logcosh_obj(
+        y_true,
+        y_pred,
+        sample_weight=constant_op.constant(sample_weight, shape=(2, 3)))
+    expected_loss = np.sum(expected_losses * sample_weight) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+
+  def test_zero_weighted(self):
+    self.setup()
+    logcosh_obj = keras.losses.Logcosh()
+    sample_weight = 0
+    loss = logcosh_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), 0., 3)
+
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 7f13cc46e3..2de0bc474c 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -1654,6 +1654,37 @@ class RootMeanSquaredError(Mean):
     return math_ops.sqrt(math_ops.div_no_nan(self.total, self.count))
 
 
+class Logcosh(MeanMetricWrapper):
+  """Computes the logarithm of the hyperbolic cosine of the prediction error.
+
+  logcosh = log((exp(x) + exp(-x))/2) where x is the error `y_pred` - `y_true`.
+
+  Usage:
+
+  ```python
+  m = tf.keras.metrics.Logcosh()
+  m.update_state([0., 1., 1.], [1., 0., 1.])
+  print('Final result: ', m.result().numpy())  # Final result: 0.289
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', metrics=[tf.keras.metrics.Logcosh()])
+  ```
+  """
+
+  def __init__(self, name='logcosh', dtype=None):
+    super(Logcosh, self).__init__(logcosh, name, dtype=dtype)
+
+  @classmethod
+  def from_config(cls, config):
+    if 'fn' in config:
+      config.pop('fn')
+    return super(Logcosh, cls).from_config(config)
+
+
 def accuracy(y_true, y_pred):
   y_pred.get_shape().assert_is_compatible_with(y_true.get_shape())
   if y_true.dtype != y_pred.dtype:
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index fb635e7055..a9d88789ed 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -1467,6 +1467,49 @@ class SparseTopKCategoricalAccuracyTest(test.TestCase):
     self.assertEqual(0.5, self.evaluate(result))  # only 1 sample matches.
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class LogcoshTest(test.TestCase):
+
+  def setup(self):
+    y_pred = np.asarray([1, 9, 2, -5, -2, 6]).reshape((2, 3))
+    y_true = np.asarray([4, 8, 12, 8, 1, 3]).reshape((2, 3))
+
+    self.batch_size = 6
+    error = y_pred - y_true
+    self.expected_results = np.log((np.exp(error) + np.exp(-error)) / 2)
+
+    self.y_pred = constant_op.constant(y_pred, dtype=dtypes.float32)
+    self.y_true = constant_op.constant(y_true)
+
+  def test_config(self):
+    logcosh_obj = metrics.Logcosh(name='logcosh', dtype=dtypes.int32)
+    self.assertEqual(logcosh_obj.name, 'logcosh')
+    self.assertEqual(logcosh_obj._dtype, dtypes.int32)
+
+  def test_unweighted(self):
+    self.setup()
+    logcosh_obj = metrics.Logcosh()
+    self.evaluate(variables.variables_initializer(logcosh_obj.variables))
+
+    update_op = logcosh_obj.update_state(self.y_true, self.y_pred)
+    self.evaluate(update_op)
+    result = logcosh_obj.result()
+    expected_result = np.sum(self.expected_results) / self.batch_size
+    self.assertAllClose(result, expected_result, atol=1e-3)
+
+  def test_weighted(self):
+    self.setup()
+    logcosh_obj = metrics.Logcosh()
+    self.evaluate(variables.variables_initializer(logcosh_obj.variables))
+    sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
+    result = logcosh_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+
+    sample_weight = np.asarray([1.2, 1.2, 1.2, 3.4, 3.4, 3.4]).reshape((2, 3))
+    expected_result = np.multiply(self.expected_results, sample_weight)
+    expected_result = np.sum(expected_result) / np.sum(sample_weight)
+    self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)
+
+
 def _get_model(compile_metrics):
   model_layers = [
       layers.Dense(3, activation='relu', kernel_initializer='ones'),
-- 
GitLab


From f9bd1568aa6f0e0b8c40c10d73caeca80c3c19ab Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 4 Jan 2019 12:18:26 -0800
Subject: [PATCH 0202/2345] [XLA] Use absl::flat_hash_{map,set}::contains()
 instead of count().

contains() is a much better description of what we're trying to do!

Also change a few std containers over to absl containers so we can advantage of
this.

PiperOrigin-RevId: 227894609
---
 tensorflow/compiler/xla/BUILD                 |  1 +
 tensorflow/compiler/xla/client/xla_builder.cc |  8 +++--
 tensorflow/compiler/xla/client/xla_builder.h  |  3 +-
 tensorflow/compiler/xla/reference_util.cc     | 25 ++++++++------
 tensorflow/compiler/xla/service/BUILD         |  9 +++++
 .../xla/service/algebraic_simplifier.cc       |  5 +--
 tensorflow/compiler/xla/service/backend.cc    |  8 ++---
 tensorflow/compiler/xla/service/backend.h     |  4 ++-
 .../compiler/xla/service/buffer_assignment.cc | 34 +++++++------------
 .../compiler/xla/service/buffer_assignment.h  |  7 ++--
 .../xla/service/buffer_assignment_test.cc     |  7 ++--
 .../compiler/xla/service/channel_tracker.cc   |  4 +--
 .../compiler/xla/service/channel_tracker.h    |  4 ++-
 .../xla/service/cpu/cpu_layout_assignment.cc  |  3 +-
 .../compiler/xla/service/cpu/ir_emitter.cc    | 12 +++----
 .../compiler/xla/service/cpu/ir_emitter.h     |  2 +-
 tensorflow/compiler/xla/service/gpu/BUILD     |  7 +++-
 .../xla/service/gpu/buffer_allocations.cc     | 11 +++---
 .../xla/service/gpu/buffer_allocations.h      |  4 ++-
 .../xla/service/gpu/hlo_to_ir_bindings.cc     |  7 ++--
 .../xla/service/gpu/hlo_to_ir_bindings.h      |  6 ++--
 .../xla/service/gpu/multi_output_fusion.cc    |  2 +-
 .../xla/service/gpu/stream_assignment.cc      |  9 ++---
 .../xla/service/gpu/thunk_schedule.cc         | 16 +++++----
 .../compiler/xla/service/gpu/thunk_schedule.h | 12 ++++---
 .../compiler/xla/service/heap_simulator.cc    | 22 ++++++------
 .../compiler/xla/service/hlo_computation.cc   | 15 ++++----
 .../xla/service/hlo_computation_test.cc       |  5 +--
 .../xla/service/hlo_dataflow_analysis.cc      |  2 +-
 tensorflow/compiler/xla/service/hlo_dce.cc    |  5 +--
 .../compiler/xla/service/hlo_graph_dumper.cc  | 16 ++++-----
 .../xla/service/hlo_instruction_test.cc       | 33 +++++++++---------
 .../xla/service/hlo_liveness_analysis.cc      |  5 +--
 tensorflow/compiler/xla/service/hlo_module.cc | 29 +++++++---------
 .../xla/service/hlo_module_group_metadata.cc  |  4 +--
 .../xla/service/hlo_module_group_metadata.h   |  2 +-
 .../compiler/xla/service/hlo_ordering.cc      |  2 +-
 .../compiler/xla/service/hlo_pass_pipeline.cc |  2 +-
 .../compiler/xla/service/hlo_schedule.cc      | 10 +++---
 .../compiler/xla/service/hlo_schedule.h       |  2 +-
 .../compiler/xla/service/hlo_sharding.cc      |  5 +--
 .../xla/service/hlo_sharding_metadata.cc      |  4 +--
 .../xla/service/indexed_array_analysis.cc     |  6 ++--
 .../compiler/xla/service/layout_assignment.cc |  2 +-
 .../compiler/xla/service/layout_assignment.h  |  4 +--
 tensorflow/compiler/xla/service/llvm_ir/BUILD |  1 +
 .../xla/service/llvm_ir/alias_analysis.h      |  7 ++--
 .../xla/service/llvm_ir/fused_ir_emitter.cc   |  2 +-
 .../xla/service/llvm_ir/fused_ir_emitter.h    |  6 ++--
 .../xla/service/multi_output_fusion.cc        |  2 +-
 .../xla/service/tuple_points_to_analysis.cc   |  8 ++---
 .../while_loop_invariant_code_motion.cc       |  4 +--
 .../xla/service/while_loop_simplifier.cc      |  2 +-
 53 files changed, 225 insertions(+), 192 deletions(-)

diff --git a/tensorflow/compiler/xla/BUILD b/tensorflow/compiler/xla/BUILD
index 722d137668..8a0e2db2b5 100644
--- a/tensorflow/compiler/xla/BUILD
+++ b/tensorflow/compiler/xla/BUILD
@@ -741,6 +741,7 @@ cc_library(
         "//tensorflow/compiler/xla/service:hlo_evaluator",
         "//tensorflow/compiler/xla/service:shape_inference",
         "//tensorflow/core:lib",
+        "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/types:span",
     ],
diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc
index 4b752def1a..433033cae6 100644
--- a/tensorflow/compiler/xla/client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_builder.cc
@@ -22,6 +22,8 @@ limitations under the License.
 #include <utility>
 
 #include "absl/algorithm/container.h"
+#include "absl/container/flat_hash_map.h"
+#include "absl/container/flat_hash_set.h"
 #include "absl/memory/memory.h"
 #include "absl/strings/match.h"
 #include "absl/strings/str_cat.h"
@@ -193,9 +195,9 @@ StatusOr<ProgramShape> XlaBuilder::GetProgramShape(XlaOp root) const {
 }
 
 void XlaBuilder::IsConstantVisitor(const int64 op_handle,
-                                   std::set<int64>* visited,
+                                   absl::flat_hash_set<int64>* visited,
                                    bool* is_constant) const {
-  if (visited->count(op_handle) != 0 || !*is_constant) {
+  if (visited->contains(op_handle) || !*is_constant) {
     return;
   }
 
@@ -2415,7 +2417,7 @@ StatusOr<bool> XlaBuilder::IsConstant(const XlaOp& operand) const {
   TF_RETURN_IF_ERROR(LookUpInstruction(operand).status());
 
   bool is_constant = true;
-  std::set<int64> visited;
+  absl::flat_hash_set<int64> visited;
   IsConstantVisitor(operand.handle(), &visited, &is_constant);
   return is_constant;
 }
diff --git a/tensorflow/compiler/xla/client/xla_builder.h b/tensorflow/compiler/xla/client/xla_builder.h
index 68ddf2cdd8..ebef8e0879 100644
--- a/tensorflow/compiler/xla/client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_builder.h
@@ -727,7 +727,8 @@ class XlaBuilder {
   // operation such as `RngNormal` or `Infeed`. The visitor walks the
   // computation starting at a given operation and sets is_constant to false iff
   // a parameter or stateful operation is encountered.
-  void IsConstantVisitor(const int64 op_handle, std::set<int64>* visited,
+  void IsConstantVisitor(const int64 op_handle,
+                         absl::flat_hash_set<int64>* visited,
                          bool* is_constant) const;
 
   // Checks bounds for convolution parameters.
diff --git a/tensorflow/compiler/xla/reference_util.cc b/tensorflow/compiler/xla/reference_util.cc
index 033f6c9b0f..08b78ee244 100644
--- a/tensorflow/compiler/xla/reference_util.cc
+++ b/tensorflow/compiler/xla/reference_util.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include <array>
 #include <utility>
 
+#include "absl/container/flat_hash_set.h"
 #include "absl/memory/memory.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal_util.h"
@@ -605,24 +606,26 @@ ReferenceUtil::ReduceToRowArray2D(
     const std::function<float(float, float)>& reduce_function) {
   std::vector<float> result;
   CHECK_EQ(dims.size(), 3);
-  const std::set<int64> dim_set(dims.begin(), dims.end());
+  const absl::flat_hash_set<int64> dim_set(dims.begin(), dims.end());
   CHECK_EQ(dim_set.size(), 3);
-  for (int64 a0 = 0; a0 == 0 || (!dim_set.count(0) && a0 < array.n1()); ++a0) {
-    for (int64 a1 = 0; a1 == 0 || (!dim_set.count(1) && a1 < array.n2());
+  for (int64 a0 = 0; a0 == 0 || (!dim_set.contains(0) && a0 < array.n1());
+       ++a0) {
+    for (int64 a1 = 0; a1 == 0 || (!dim_set.contains(1) && a1 < array.n2());
          ++a1) {
-      for (int64 a2 = 0; a2 == 0 || (!dim_set.count(2) && a2 < array.n3());
+      for (int64 a2 = 0; a2 == 0 || (!dim_set.contains(2) && a2 < array.n3());
            ++a2) {
-        for (int64 a3 = 0; a3 == 0 || (!dim_set.count(3) && a3 < array.n4());
+        for (int64 a3 = 0; a3 == 0 || (!dim_set.contains(3) && a3 < array.n4());
              ++a3) {
           float accumulator = init;
-          for (int64 i0 = 0; i0 == 0 || (dim_set.count(0) && i0 < array.n1());
-               ++i0) {
-            for (int64 i1 = 0; i1 == 0 || (dim_set.count(1) && i1 < array.n2());
-                 ++i1) {
+          for (int64 i0 = 0;
+               i0 == 0 || (dim_set.contains(0) && i0 < array.n1()); ++i0) {
+            for (int64 i1 = 0;
+                 i1 == 0 || (dim_set.contains(1) && i1 < array.n2()); ++i1) {
               for (int64 i2 = 0;
-                   i2 == 0 || (dim_set.count(2) && i2 < array.n3()); ++i2) {
+                   i2 == 0 || (dim_set.contains(2) && i2 < array.n3()); ++i2) {
                 for (int64 i3 = 0;
-                     i3 == 0 || (dim_set.count(3) && i3 < array.n4()); ++i3) {
+                     i3 == 0 || (dim_set.contains(3) && i3 < array.n4());
+                     ++i3) {
                   // Handle zero-sized arrays.
                   if (array.n1() > 0 && array.n2() > 0 && array.n3() > 0 &&
                       array.n4() > 0) {
diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 755c477a12..85a21a8518 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -515,6 +515,7 @@ tf_cc_test(
         "//tensorflow/compiler/xla:window_util",
         "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+        "@com_google_absl//absl/container:flat_hash_map",
     ],
 )
 
@@ -677,6 +678,7 @@ cc_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:stream_executor_no_cuda",
         "//third_party/eigen3",
+        "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/types:span",
@@ -1002,6 +1004,7 @@ cc_library(
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/types:span",
@@ -1136,6 +1139,7 @@ tf_cc_test(
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
         "//tensorflow/core:lib",
         "//tensorflow/core:test",
+        "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/memory",
     ],
 )
@@ -1580,6 +1584,7 @@ cc_library(
         "//tensorflow/core:lib",
         "@com_google_absl//absl/algorithm:container",
         "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/container:inlined_vector",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
@@ -2116,6 +2121,7 @@ tf_cc_test(
         "//tensorflow/compiler/xla:test_helpers",
         "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+        "@com_google_absl//absl/container:flat_hash_set",
     ],
 )
 
@@ -2288,6 +2294,7 @@ cc_library(
         "//tensorflow/compiler/xla:types",
         "//tensorflow/compiler/xla:util",
         "//tensorflow/core:lib",
+        "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
     ],
@@ -2548,6 +2555,7 @@ cc_library(
         "//tensorflow/compiler/xla:types",
         "//tensorflow/compiler/xla:util",
         "//tensorflow/core:lib",
+        "@com_google_absl//absl/container:flat_hash_set",
     ],
 )
 
@@ -3188,6 +3196,7 @@ cc_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:regexp_internal",
+        "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/strings:str_format",
         "@com_google_absl//absl/types:optional",
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index c573b60161..3d269dc04f 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -27,6 +27,7 @@ limitations under the License.
 
 #include "absl/algorithm/container.h"
 #include "absl/container/flat_hash_map.h"
+#include "absl/container/flat_hash_set.h"
 #include "absl/container/inlined_vector.h"
 #include "absl/memory/memory.h"
 #include "absl/strings/str_cat.h"
@@ -2539,7 +2540,7 @@ Status AlgebraicSimplifierVisitor::HandleReduce(HloInstruction* reduce) {
     }
     if (can_move_reshape_into_reduce) {
       changed_ = true;
-      std::unordered_set<int64> dimensions_not_to_reduce;
+      absl::flat_hash_set<int64> dimensions_not_to_reduce;
       for (auto dim_pair : unmodified_dims) {
         if (arg_dim_in_output[dim_pair.second]) {
           dimensions_not_to_reduce.insert(dim_pair.first);
@@ -2547,7 +2548,7 @@ Status AlgebraicSimplifierVisitor::HandleReduce(HloInstruction* reduce) {
       }
       std::vector<int64> new_reduce_dimensions;
       for (int64 i = 0; i < arg->operand(0)->shape().rank(); ++i) {
-        if (dimensions_not_to_reduce.count(i) == 0) {
+        if (!dimensions_not_to_reduce.contains(i)) {
           new_reduce_dimensions.push_back(i);
         }
       }
diff --git a/tensorflow/compiler/xla/service/backend.cc b/tensorflow/compiler/xla/service/backend.cc
index 2cf24a9dd5..215e8ced4b 100644
--- a/tensorflow/compiler/xla/service/backend.cc
+++ b/tensorflow/compiler/xla/service/backend.cc
@@ -115,12 +115,10 @@ StatusOr<StreamPool::Ptr> Backend::BorrowStream(int device_ordinal) {
 
 StatusOr<StreamPool::Ptr> Backend::BorrowStream(se::StreamExecutor* executor) {
   tensorflow::mutex_lock l(mu_);
-  if (0 == stream_pools_.count(executor)) {
-    stream_pools_.emplace(std::piecewise_construct,
-                          std::forward_as_tuple(executor),
-                          std::forward_as_tuple());
+  if (!stream_pools_.contains(executor)) {
+    stream_pools_.emplace(executor, absl::make_unique<StreamPool>());
   }
-  return stream_pools_.at(executor).BorrowStream(executor);
+  return stream_pools_.at(executor)->BorrowStream(executor);
 }
 
 Backend::Backend(se::Platform* platform, Compiler* compiler,
diff --git a/tensorflow/compiler/xla/service/backend.h b/tensorflow/compiler/xla/service/backend.h
index 7ca993fb26..c35f033dc0 100644
--- a/tensorflow/compiler/xla/service/backend.h
+++ b/tensorflow/compiler/xla/service/backend.h
@@ -22,6 +22,7 @@ limitations under the License.
 #include <string>
 #include <vector>
 
+#include "absl/container/flat_hash_map.h"
 #include "absl/strings/str_cat.h"
 #include "absl/types/span.h"
 #include "tensorflow/compiler/xla/service/compiler.h"
@@ -175,7 +176,8 @@ class Backend {
   tensorflow::mutex mu_;
 
   // Mapping from stream executor to stream pools, used by `BorrowStream` above.
-  std::map<se::StreamExecutor*, StreamPool> stream_pools_ GUARDED_BY(mu_);
+  absl::flat_hash_map<se::StreamExecutor*, std::unique_ptr<StreamPool>>
+      stream_pools_ GUARDED_BY(mu_);
 
   // The default memory allocator to use.
   std::unique_ptr<StreamExecutorMemoryAllocator> memory_allocator_;
diff --git a/tensorflow/compiler/xla/service/buffer_assignment.cc b/tensorflow/compiler/xla/service/buffer_assignment.cc
index 6f4c1104f3..c997d594c4 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment.cc
+++ b/tensorflow/compiler/xla/service/buffer_assignment.cc
@@ -138,8 +138,8 @@ Status GatherComputationsByAllocationType(
     worklist.pop_front();
     const HloComputation* computation = worklist_front.first;
     bool is_thread_local = worklist_front.second;
-    bool in_thread_local_set = thread_local_set.count(computation) > 0;
-    bool in_global_set = global_set.count(computation) > 0;
+    bool in_thread_local_set = thread_local_set.contains(computation);
+    bool in_global_set = global_set.contains(computation);
 
     // If the computation has already been added to the respective set, then
     // nothing to do.
@@ -207,9 +207,9 @@ Status GatherComputationsByAllocationType(
 
   // Add the computations to the vectors in post order.
   for (auto* computation : module->MakeComputationPostOrder()) {
-    if (thread_local_set.count(computation) > 0) {
+    if (thread_local_set.contains(computation)) {
       thread_local_computations->push_back(computation);
-    } else if (global_set.count(computation) > 0) {
+    } else if (global_set.contains(computation)) {
       global_computations->push_back(computation);
     }
     // If the computation is not reachable from the entry computation, then it
@@ -219,13 +219,6 @@ Status GatherComputationsByAllocationType(
   return Status::OK();
 }
 
-size_t BufferAllocation::Slice::Hasher::operator()(Slice s) const {
-  uint64 h = std::hash<int64>()(s.index());
-  h = tensorflow::Hash64Combine(h, std::hash<int64>()(s.offset()));
-  h = tensorflow::Hash64Combine(h, std::hash<int64>()(s.size()));
-  return h;
-}
-
 string BufferAllocation::Slice::ToString() const {
   return absl::StrCat("{index:", index(), ", offset:", offset_,
                       ", size:", size_, "}");
@@ -240,7 +233,7 @@ BufferAllocation::Slice BufferAllocation::GetSlice(
 void BufferAllocation::AddAssignment(const LogicalBuffer& buffer, int64 offset,
                                      int64 size) {
   VLOG(4) << "Trying to add " << buffer << " to allocation #" << index();
-  CHECK(assigned_buffers_.count(&buffer) == 0)
+  CHECK(!assigned_buffers_.contains(&buffer))
       << "LogicalBuffer " << buffer << " already assigned to allocation "
       << index_;
   CHECK_LE(offset, size_) << "LogicalBuffer " << buffer
@@ -346,7 +339,7 @@ const PointsToSet& BufferAssignment::GetPointsToSet(
 
 bool BufferAssignment::HasAllocation(const LogicalBuffer& buffer) const {
   TF_CHECK_OK(points_to_analysis().VerifyBuffer(buffer));
-  return allocation_index_for_buffer_.count(&buffer) > 0;
+  return allocation_index_for_buffer_.contains(&buffer);
 }
 
 const BufferAllocation& BufferAssignment::GetAssignedAllocation(
@@ -401,7 +394,7 @@ bool BufferAssignment::HasAllocationAt(const HloInstruction* instruction,
                                        const ShapeIndex& index) const {
   for (const LogicalBuffer* buffer :
        GetPointsToSet(instruction).element(index)) {
-    if (allocation_index_for_buffer_.count(buffer) > 0) {
+    if (allocation_index_for_buffer_.contains(buffer)) {
       return true;
     }
   }
@@ -459,8 +452,7 @@ bool BufferAssignment::SharesSliceAtIndex(
 
 bool BufferAssignment::HaveDisjointSlices(const HloInstruction* hlo_a,
                                           const HloInstruction* hlo_b) const {
-  using SliceSet =
-      flat_hash_set<BufferAllocation::Slice, BufferAllocation::Slice::Hasher>;
+  using SliceSet = flat_hash_set<BufferAllocation::Slice>;
   // Gets the slices all of instr's subshapes.  If any subshape doesn't have an
   // assigned slice, returns the empty set.
   auto collect_slices = [&](const HloInstruction* instr) -> SliceSet {
@@ -519,7 +511,7 @@ BufferAllocation* BufferAssignment::NewAllocation(const LogicalBuffer& buffer,
 void BufferAssignment::AddAssignment(BufferAllocation* allocation,
                                      const LogicalBuffer& buffer, int64 offset,
                                      int64 size) {
-  CHECK_EQ(0, allocation_index_for_buffer_.count(&buffer))
+  CHECK(!allocation_index_for_buffer_.contains(&buffer))
       << "LogicalBuffer " << buffer << " already has an allocation.";
   CHECK(allocation->is_reusable() || allocation->assigned_buffers().empty())
       << "Non-reusable allocation already assigned a buffer: "
@@ -988,7 +980,7 @@ Status BufferAssigner::AssignBuffersForComputation(
   std::vector<BufferAllocation::Index> allocation_indices;
   for (const LogicalBuffer* buffer : sorted_buffers) {
     VLOG(3) << "Assigning allocation to: " << *buffer;
-    if (colocated_buffers.count(buffer) > 0) {
+    if (colocated_buffers.contains(buffer)) {
       // Colocated buffers are currently assigned in an earlier pass.
       VLOG(3) << "Skipping colocated buffer: " << *buffer;
       continue;
@@ -1056,7 +1048,7 @@ Status BufferAssigner::AssignBuffersForComputation(
              assignment->GetAllSlices(operand, /*index=*/{})) {
           BufferAllocation* allocation =
               assignment->GetMutableAllocation(operand_slice.index());
-          if (colocated_allocations.count(allocation->index()) == 0) {
+          if (!colocated_allocations.contains(allocation->index())) {
             // TODO(b/32491382) Colocated buffers are currently assigned in an
             // earlier pass, and so can break the "increasing allocation size"
             // invariant in this function (causing this CHECK to fail). However,
@@ -1087,7 +1079,7 @@ Status BufferAssigner::AssignBuffersForComputation(
         // Instructions are iterated in increasing buffer size, so any
         // previously create allocation must be large enough to hold this
         // instruction's output (with the exception of colocated buffers).
-        if (colocated_allocations.count(allocation->index()) == 0) {
+        if (!colocated_allocations.contains(allocation->index())) {
           // TODO(b/32491382) Colocated buffers are currently assigned in an
           // earlier pass, and so can break the "increasing allocation size"
           // invariant in this function (causing this CHECK to fail). However,
@@ -1376,7 +1368,7 @@ void BufferAssigner::AddSetToColocatedBufferSets(
   std::vector<size_t> overlap_set_indices;
   for (size_t index = 0; index < colocated_buffer_sets->size(); ++index) {
     for (const LogicalBuffer* buffer : colocated_set) {
-      if ((*colocated_buffer_sets)[index].count(buffer) > 0) {
+      if ((*colocated_buffer_sets)[index].contains(buffer)) {
         VLOG(5) << "Found overlap with existing set on buffer "
                 << buffer->ToString() << "\n"
                 << ColocatedBufferSetsToString((*colocated_buffer_sets)[index],
diff --git a/tensorflow/compiler/xla/service/buffer_assignment.h b/tensorflow/compiler/xla/service/buffer_assignment.h
index 0a9fdede80..4baab9b6ad 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment.h
+++ b/tensorflow/compiler/xla/service/buffer_assignment.h
@@ -186,9 +186,10 @@ class BufferAllocation {
              end > other.offset_;
     }
 
-    struct Hasher {
-      size_t operator()(Slice s) const;
-    };
+    template <typename H>
+    friend H AbslHashValue(H h, const Slice& s) {
+      return H::combine(std::move(h), s.index(), s.offset(), s.size());
+    }
 
     string ToString() const;
 
diff --git a/tensorflow/compiler/xla/service/buffer_assignment_test.cc b/tensorflow/compiler/xla/service/buffer_assignment_test.cc
index 8f482e6ba8..370fad7ffb 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment_test.cc
+++ b/tensorflow/compiler/xla/service/buffer_assignment_test.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include <utility>
 #include <vector>
 
+#include "absl/container/flat_hash_set.h"
 #include "absl/memory/memory.h"
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/service/buffer_value.h"
@@ -309,7 +310,7 @@ class BufferAssignmentTest : public HloTestBase {
 static bool BuffersDistinct(const std::vector<const HloInstruction*>& a,
                             const std::vector<const HloInstruction*>& b,
                             const BufferAssignment& assignment) {
-  std::set<BufferAllocation::Slice> a_slices;
+  absl::flat_hash_set<BufferAllocation::Slice> a_slices;
   for (const HloInstruction* instruction : a) {
     if (assignment.HasTopLevelAllocation(instruction)) {
       a_slices.insert(
@@ -319,8 +320,8 @@ static bool BuffersDistinct(const std::vector<const HloInstruction*>& a,
 
   for (const HloInstruction* instruction : b) {
     if (assignment.HasTopLevelAllocation(instruction)) {
-      if (a_slices.count(assignment.GetUniqueTopLevelSlice(instruction)
-                             .ConsumeValueOrDie())) {
+      if (a_slices.contains(assignment.GetUniqueTopLevelSlice(instruction)
+                                .ConsumeValueOrDie())) {
         return false;
       }
     }
diff --git a/tensorflow/compiler/xla/service/channel_tracker.cc b/tensorflow/compiler/xla/service/channel_tracker.cc
index 3c2d1ae6d8..b517495f2e 100644
--- a/tensorflow/compiler/xla/service/channel_tracker.cc
+++ b/tensorflow/compiler/xla/service/channel_tracker.cc
@@ -72,7 +72,7 @@ ChannelHandle ChannelTracker::AllocateHandle(ChannelHandle::ChannelType type) {
 }
 
 Status ChannelTracker::RegisterSendInternal(const ChannelHandle& handle) {
-  if (opaque_to_channel_.count(handle.handle()) == 0) {
+  if (!opaque_to_channel_.contains(handle.handle())) {
     return NotFound("channel handle not found: %d", handle.handle());
   }
   Channel& channel = opaque_to_channel_[handle.handle()];
@@ -94,7 +94,7 @@ Status ChannelTracker::RegisterSendInternal(const ChannelHandle& handle) {
 }
 
 Status ChannelTracker::RegisterRecvInternal(const ChannelHandle& handle) {
-  if (opaque_to_channel_.count(handle.handle()) == 0) {
+  if (!opaque_to_channel_.contains(handle.handle())) {
     return NotFound("channel handle not found: %d", handle.handle());
   }
   Channel& channel = opaque_to_channel_[handle.handle()];
diff --git a/tensorflow/compiler/xla/service/channel_tracker.h b/tensorflow/compiler/xla/service/channel_tracker.h
index 52037bf9b5..89e17eba36 100644
--- a/tensorflow/compiler/xla/service/channel_tracker.h
+++ b/tensorflow/compiler/xla/service/channel_tracker.h
@@ -18,6 +18,7 @@ limitations under the License.
 
 #include <map>
 
+#include "absl/container/flat_hash_map.h"
 #include "absl/types/span.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
 #include "tensorflow/compiler/xla/status.h"
@@ -83,7 +84,8 @@ class ChannelTracker {
 
   // Mapping from ChannelHandle value to the corresponding registered
   // Channel object.
-  std::map<int64, Channel> opaque_to_channel_ GUARDED_BY(channel_mutex_);
+  absl::flat_hash_map<int64, Channel> opaque_to_channel_
+      GUARDED_BY(channel_mutex_);
 
   TF_DISALLOW_COPY_AND_ASSIGN(ChannelTracker);
 };
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_layout_assignment.cc b/tensorflow/compiler/xla/service/cpu/cpu_layout_assignment.cc
index 1cb154e9ca..ff11409482 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_layout_assignment.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_layout_assignment.cc
@@ -46,8 +46,7 @@ static bool ShouldMakeAllUsersColMajor(const HloInstruction* instruction) {
   for (auto* user : instruction->users()) {
     optional<int64> operand_idx = ProfitableToMakeDotOperandColumnMajor(*user);
     if (!operand_idx || user->operand(*operand_idx) != instruction ||
-        std::count(user->operands().begin(), user->operands().end(),
-                   instruction) != 1) {
+        absl::c_count(user->operands(), instruction) != 1) {
       return false;
     }
   }
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index 169d628923..e4760aedf1 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -24,11 +24,9 @@ limitations under the License.
 #include <utility>
 #include <vector>
 
+// IWYU pragma: no_include "llvm/IR/Intrinsics.gen.inc"
 #include "absl/container/flat_hash_map.h"
 #include "absl/container/flat_hash_set.h"
-#include "tensorflow/core/lib/math/math_util.h"
-#include "tensorflow/core/platform/logging.h"
-// IWYU pragma: no_include "llvm/IR/Intrinsics.gen.inc"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_format.h"
 #include "absl/types/span.h"
@@ -70,6 +68,8 @@ limitations under the License.
 #include "tensorflow/compiler/xla/window_util.h"
 #include "tensorflow/core/lib/core/bits.h"
 #include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/math/math_util.h"
+#include "tensorflow/core/platform/logging.h"
 
 namespace xla {
 
@@ -1399,7 +1399,7 @@ static bool ReductionPreservesLayout(const HloInstruction& reduce) {
 
   int64 delta = 0;
   for (int64 i = 0; i < operand_shape.dimensions_size(); i++) {
-    if (reduced_dims.count(i)) {
+    if (reduced_dims.contains(i)) {
       delta++;
     } else {
       InsertOrDie(&unreduced_dim_map, i, i - delta);
@@ -1412,7 +1412,7 @@ static bool ReductionPreservesLayout(const HloInstruction& reduce) {
   for (int64 operand_dim_idx = 0;
        operand_dim_idx < operand_shape.dimensions_size(); operand_dim_idx++) {
     int64 operand_dim = operand_shape.layout().minor_to_major(operand_dim_idx);
-    if (!reduced_dims.count(operand_dim)) {
+    if (!reduced_dims.contains(operand_dim)) {
       if (FindOrDie(unreduced_dim_map, operand_dim) !=
           result_shape.layout().minor_to_major(result_dim_idx++)) {
         return false;
@@ -1990,7 +1990,7 @@ Status IrEmitter::HandleSlice(HloInstruction* slice) {
   // The memcpy will copy elements that are logically this shape (allowed to be
   // scalar).
   const Shape logical_element_shape = ShapeUtil::FilterDimensions(
-      [&inner_dims](int64 dim) -> bool { return inner_dims.count(dim); },
+      [&inner_dims](int64 dim) { return inner_dims.contains(dim); },
       operand->shape());
 
   const int64 primitive_elements_per_logical_element =
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.h b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
index db76de4bb2..a6fb11dcbf 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.h
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
@@ -448,7 +448,7 @@ class IrEmitter : public DfsHloVisitorWithDefault,
       computation_to_profile_idx_;
 
   // Maps HLOs to Values emitted for them.
-  std::unordered_map<const HloInstruction*, llvm::Value*> emitted_value_;
+  absl::flat_hash_map<const HloInstruction*, llvm::Value*> emitted_value_;
 
   llvm_ir::AliasAnalysis alias_analysis_;
 
diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD
index 6c23f921f4..d10a11d096 100644
--- a/tensorflow/compiler/xla/service/gpu/BUILD
+++ b/tensorflow/compiler/xla/service/gpu/BUILD
@@ -94,8 +94,8 @@ cc_library(
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla/service:hlo",
         "//tensorflow/compiler/xla/service:hlo_reachability",
-        "//tensorflow/core:lib",
         "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/memory",
     ],
 )
@@ -135,6 +135,8 @@ cc_library(
         "//tensorflow/compiler/xla/service/llvm_ir:llvm_util",
         "//tensorflow/compiler/xla/service/llvm_ir:tuple_ops",
         "//tensorflow/core:lib",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/types:span",
         "@llvm//:core",
@@ -263,7 +265,9 @@ cc_library(
         "//tensorflow/compiler/xla/service:buffer_assignment",
         "//tensorflow/compiler/xla/service:device_memory_allocator",
         "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
         "//tensorflow/core:stream_executor_no_cuda",
+        "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/types:span",
     ],
@@ -362,6 +366,7 @@ cc_library(
         "//tensorflow/core/platform/default/build_config:stream_executor_cuda",  # build_cleaner: keep
         "//tensorflow/stream_executor",
         "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/strings:str_format",
diff --git a/tensorflow/compiler/xla/service/gpu/buffer_allocations.cc b/tensorflow/compiler/xla/service/gpu/buffer_allocations.cc
index 528209abc7..eb59ee5a1d 100644
--- a/tensorflow/compiler/xla/service/gpu/buffer_allocations.cc
+++ b/tensorflow/compiler/xla/service/gpu/buffer_allocations.cc
@@ -24,6 +24,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/util.h"
 #include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/gtl/map_util.h"
 #include "tensorflow/core/lib/strings/numbers.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/types.h"
@@ -57,16 +58,16 @@ StatusOr<std::unique_ptr<BufferAllocations>> BufferAllocations::Builder::Build(
 
     // If buffer #i's address is already registered (e.g. external arguments or
     // result buffers), use that registered buffer.
-    if (registered_buffers_.count(i)) {
-      se::DeviceMemoryBase address = FindOrDie(registered_buffers_, i);
-      if (reinterpret_cast<uintptr_t>(address.opaque()) % expected_alignment !=
+    if (se::DeviceMemoryBase* address =
+            tensorflow::gtl::FindOrNull(registered_buffers_, i)) {
+      if (reinterpret_cast<uintptr_t>(address->opaque()) % expected_alignment !=
           0) {
         return InternalError(
             "Address of registered buffer %d must be a multiple of %x, but "
             "was %p",
-            i, kEntryParameterAlignBytes, address.opaque());
+            i, kEntryParameterAlignBytes, address->opaque());
       }
-      buffer_allocations->SetBuffer(i, FindOrDie(registered_buffers_, i));
+      buffer_allocations->SetBuffer(i, *address);
       continue;
     }
 
diff --git a/tensorflow/compiler/xla/service/gpu/buffer_allocations.h b/tensorflow/compiler/xla/service/gpu/buffer_allocations.h
index 14186b8faa..9413ac2cff 100644
--- a/tensorflow/compiler/xla/service/gpu/buffer_allocations.h
+++ b/tensorflow/compiler/xla/service/gpu/buffer_allocations.h
@@ -20,6 +20,7 @@ limitations under the License.
 #include <set>
 #include <vector>
 
+#include "absl/container/flat_hash_map.h"
 #include "absl/types/span.h"
 #include "tensorflow/compiler/xla/service/buffer_assignment.h"
 #include "tensorflow/compiler/xla/service/device_memory_allocator.h"
@@ -52,7 +53,8 @@ class BufferAllocations {
         DeviceMemoryAllocator* memory_allocator);
 
    private:
-    std::map<BufferAllocation::Index, se::DeviceMemoryBase> registered_buffers_;
+    absl::flat_hash_map<BufferAllocation::Index, se::DeviceMemoryBase>
+        registered_buffers_;
   };
 
   ~BufferAllocations();
diff --git a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc
index 9634c92786..69aaaceca1 100644
--- a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc
+++ b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h"
 
+#include "absl/container/flat_hash_set.h"
 #include "absl/strings/str_cat.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Function.h"
@@ -45,10 +46,10 @@ void HloToIrBindings::EmitBasePointersForHlos(
 
   // An HLO can have duplicated operands. This data structure remembers which
   // operand HLOs are already bound to avoid rebinding the same HLO.
-  std::set<const HloInstruction*> already_bound_for_this_function;
+  absl::flat_hash_set<const HloInstruction*> already_bound_for_this_function;
   auto arg_iter = function->arg_begin();
   for (const HloInstruction* io_hlo : io_hlos) {
-    if (!already_bound_for_this_function.count(io_hlo)) {
+    if (!already_bound_for_this_function.contains(io_hlo)) {
       if (!is_nested_ && io_hlo->opcode() == HloOpcode::kGetTupleElement) {
         BindHloToIrValue(*io_hlo, EmitGetTupleElement(io_hlo, &*arg_iter));
       } else {
@@ -63,7 +64,7 @@ void HloToIrBindings::EmitBasePointersForHlos(
   temp_buffer_base_->setName("temp_buffer");
 
   for (const HloInstruction* non_io_hlo : non_io_hlos) {
-    if (already_bound_for_this_function.count(non_io_hlo)) {
+    if (already_bound_for_this_function.contains(non_io_hlo)) {
       continue;
     }
     already_bound_for_this_function.insert(non_io_hlo);
diff --git a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h
index c0edae530c..f57b594e9c 100644
--- a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h
+++ b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h
@@ -18,6 +18,7 @@ limitations under the License.
 
 #include <unordered_map>
 
+#include "absl/container/flat_hash_map.h"
 #include "absl/types/span.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Value.h"
@@ -61,7 +62,7 @@ class HloToIrBindings {
 
   // Returns whether `hlo` is bound to an LLVM IR value.
   bool BoundToIrValue(const HloInstruction& hlo) const {
-    return base_ptrs_.count(&hlo);
+    return base_ptrs_.contains(&hlo);
   }
 
   llvm::Value* GetTempBufferBase() const { return temp_buffer_base_; }
@@ -110,7 +111,8 @@ class HloToIrBindings {
   // For an instruction that generates multiple outputs, the root will be a
   // tuple shape. The IrArray for each element output is stored in the subnode
   // in the ShapeTree.
-  std::unordered_map<const HloInstruction*, ShapeTree<llvm::Value*>> base_ptrs_;
+  absl::flat_hash_map<const HloInstruction*, ShapeTree<llvm::Value*>>
+      base_ptrs_;
 
   // The address of the memory block that contains all temporary buffers.
   llvm::Value* temp_buffer_base_ = nullptr;
diff --git a/tensorflow/compiler/xla/service/gpu/multi_output_fusion.cc b/tensorflow/compiler/xla/service/gpu/multi_output_fusion.cc
index 01fddcede6..02e1207f37 100644
--- a/tensorflow/compiler/xla/service/gpu/multi_output_fusion.cc
+++ b/tensorflow/compiler/xla/service/gpu/multi_output_fusion.cc
@@ -67,7 +67,7 @@ int64 GpuMultiOutputFusion::GetProfit(HloInstruction* instr1,
   }
   int64 profit = 0;
   for (auto instr : instr2->operands()) {
-    if (!IsProfitableOperand(instr) || in_list.count(instr) == 0) {
+    if (!IsProfitableOperand(instr) || !in_list.contains(instr)) {
       continue;
     }
     profit += ShapeUtil::ByteSizeOf(instr->shape());
diff --git a/tensorflow/compiler/xla/service/gpu/stream_assignment.cc b/tensorflow/compiler/xla/service/gpu/stream_assignment.cc
index 4775baf44a..1dedbd3bef 100644
--- a/tensorflow/compiler/xla/service/gpu/stream_assignment.cc
+++ b/tensorflow/compiler/xla/service/gpu/stream_assignment.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/service/gpu/stream_assignment.h"
 
+#include "absl/container/flat_hash_set.h"
 #include "absl/memory/memory.h"
 #include "tensorflow/compiler/xla/map_util.h"
 #include "tensorflow/compiler/xla/service/gpu/ir_emission_utils.h"
@@ -25,7 +26,7 @@ namespace xla {
 namespace gpu {
 
 bool StreamAssignment::HasStreamAssigned(const HloInstruction& hlo) const {
-  return hlo_to_stream_number_.count(&hlo);
+  return hlo_to_stream_number_.contains(&hlo);
 }
 
 int StreamAssignment::StreamNumberForHlo(const HloInstruction& hlo) const {
@@ -98,10 +99,10 @@ int ComputeStreamToAssign(
   // greedy approach. First, we compute as forbidden_stream_numbers the
   // streams assigned to GEMMs that are concurrent with `hlo`. Then, we assign
   // `hlo` a different stream.
-  std::set<int> forbidden_stream_numbers;
+  absl::flat_hash_set<int> forbidden_stream_numbers;
   for (const auto* seen_gemm : seen_gemms) {
     int stream_num = stream_assignment.StreamNumberForHlo(*seen_gemm);
-    if (!forbidden_stream_numbers.count(stream_num) &&
+    if (!forbidden_stream_numbers.contains(stream_num) &&
         CanRunConcurrently(*seen_gemm, hlo, reachability)) {
       forbidden_stream_numbers.insert(stream_num);
     }
@@ -109,7 +110,7 @@ int ComputeStreamToAssign(
 
   for (int stream_num = 0; stream_num < stream_assignment.StreamCount();
        ++stream_num) {
-    if (!forbidden_stream_numbers.count(stream_num)) {
+    if (!forbidden_stream_numbers.contains(stream_num)) {
       return stream_num;
     }
   }
diff --git a/tensorflow/compiler/xla/service/gpu/thunk_schedule.cc b/tensorflow/compiler/xla/service/gpu/thunk_schedule.cc
index 6b2d76764a..25bad67bab 100644
--- a/tensorflow/compiler/xla/service/gpu/thunk_schedule.cc
+++ b/tensorflow/compiler/xla/service/gpu/thunk_schedule.cc
@@ -14,17 +14,19 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/compiler/xla/service/gpu/thunk_schedule.h"
+#include "absl/container/flat_hash_map.h"
 #include "tensorflow/compiler/xla/array2d.h"
 #include "tensorflow/compiler/xla/map_util.h"
 #include "tensorflow/compiler/xla/types.h"
+#include "tensorflow/core/lib/gtl/map_util.h"
 
 namespace xla {
 namespace gpu {
 
 void ThunkSchedule::AddDependenciesOnTransitiveOperands(
     const Thunk& thunk, const HloInstruction& operand,
-    const std::unordered_map<const HloInstruction*, Thunk*>& hlo_to_thunk) {
-  if (hlo_to_thunk.count(&operand)) {
+    const absl::flat_hash_map<const HloInstruction*, Thunk*>& hlo_to_thunk) {
+  if (hlo_to_thunk.contains(&operand)) {
     // If `operand` is mapped to a thunk, adds `operand` to `thunk`'s dependency
     // list if `operand` is assigned to a different stream. As an optimization,
     // we skip `operand`'s operands because `operand` depends on them already.
@@ -48,14 +50,14 @@ ThunkSchedule::ThunkSchedule(
     const std::vector<HloInstruction*>& hlo_total_order)
     : thunks_(std::move(thunks)),
       stream_assignment_(std::move(stream_assignment)) {
-  std::unordered_map<const HloInstruction*, Thunk*> hlo_to_thunk;
+  absl::flat_hash_map<const HloInstruction*, Thunk*> hlo_to_thunk;
   for (const auto& thunk : *thunks_) {
     InsertOrDie(&hlo_to_thunk, thunk->hlo_instruction(), thunk.get());
   }
 
   for (HloInstruction* hlo : hlo_total_order) {
-    if (hlo_to_thunk.count(hlo)) {
-      thunk_total_order_.push_back(FindOrDie(hlo_to_thunk, hlo));
+    if (Thunk** thunk = tensorflow::gtl::FindOrNull(hlo_to_thunk, hlo)) {
+      thunk_total_order_.push_back(*thunk);
     }
   }
 
@@ -106,7 +108,7 @@ void ThunkSchedule::RemoveRedundantDependencyEdges() {
   // redundant dependency edge.
   Array2D<int> last_dependency(stream_count, stream_count, -1);
   for (const Thunk* dst : thunk_total_order_) {
-    if (!depends_on_.count(dst)) {
+    if (!depends_on_.contains(dst)) {
       continue;
     }
 
@@ -134,7 +136,7 @@ void ThunkSchedule::RemoveRedundantDependencyEdges() {
 
 const std::list<const Thunk*>& ThunkSchedule::DependsOn(
     const Thunk* thunk) const {
-  if (depends_on_.count(thunk)) {
+  if (depends_on_.contains(thunk)) {
     return FindOrDie(depends_on_, thunk);
   } else {
     return empty_thunk_list_;
diff --git a/tensorflow/compiler/xla/service/gpu/thunk_schedule.h b/tensorflow/compiler/xla/service/gpu/thunk_schedule.h
index 43b628a1ba..549378debd 100644
--- a/tensorflow/compiler/xla/service/gpu/thunk_schedule.h
+++ b/tensorflow/compiler/xla/service/gpu/thunk_schedule.h
@@ -21,6 +21,8 @@ limitations under the License.
 #include <unordered_map>
 #include <vector>
 
+#include "absl/container/flat_hash_map.h"
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/compiler/xla/service/gpu/stream_assignment.h"
 #include "tensorflow/compiler/xla/service/gpu/thunk.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
@@ -54,7 +56,9 @@ class ThunkSchedule {
   // Thunks that `thunk` depends on.
   const std::list<const Thunk*>& DependsOn(const Thunk* thunk) const;
   // Whether `thunk` is depended by another thunk.
-  bool Depended(const Thunk* thunk) const { return depended_by_.count(thunk); }
+  bool Depended(const Thunk* thunk) const {
+    return depended_by_.contains(thunk);
+  }
 
   // Delegates to StreamAssignment.
   int StreamCount() const { return stream_assignment_->StreamCount(); }
@@ -75,13 +79,13 @@ class ThunkSchedule {
   // thunk.hlo_instruction().
   void AddDependenciesOnTransitiveOperands(
       const Thunk& thunk, const HloInstruction& operand,
-      const std::unordered_map<const HloInstruction*, Thunk*>& hlo_to_thunk);
+      const absl::flat_hash_map<const HloInstruction*, Thunk*>& hlo_to_thunk);
 
   std::unique_ptr<ThunkSequence> thunks_;
   std::vector<Thunk*> thunk_total_order_;
 
-  std::unordered_map<const Thunk*, std::list<const Thunk*>> depends_on_;
-  std::set<const Thunk*> depended_by_;
+  absl::flat_hash_map<const Thunk*, std::list<const Thunk*>> depends_on_;
+  absl::flat_hash_set<const Thunk*> depended_by_;
   std::list<const Thunk*> empty_thunk_list_;
 
   std::unique_ptr<StreamAssignment> stream_assignment_;
diff --git a/tensorflow/compiler/xla/service/heap_simulator.cc b/tensorflow/compiler/xla/service/heap_simulator.cc
index 9220865867..70ae7078ae 100644
--- a/tensorflow/compiler/xla/service/heap_simulator.cc
+++ b/tensorflow/compiler/xla/service/heap_simulator.cc
@@ -199,7 +199,7 @@ Status HeapSimulator::RunComputation(
 
       // If the buffer has no users and isn't an entry parameter or output, it
       // must be a dead value.
-      if (live_buffers.count(buffer) == 0) {
+      if (!live_buffers.contains(buffer)) {
         dead_buffers_to_free.push_back(buffer);
       }
     }
@@ -253,7 +253,7 @@ Status HeapSimulator::RunComputation(
       bool shared = false;
       if (options_.may_reuse_operand_buffers) {
         for (const BufferValue* operand_buffer : operand_buffers_to_free) {
-          if (reused_buffers.count(operand_buffer) != 0) {
+          if (reused_buffers.contains(operand_buffer)) {
             continue;
           }
           if (buffer->instruction()->IsUserOf(operand_buffer->instruction()) &&
@@ -374,15 +374,15 @@ bool HeapSimulator::IgnoreBuffer(const BufferValue* buffer) const {
     return true;
   }
   return options_.buffers_to_assign != nullptr &&
-         options_.buffers_to_assign->count(buffer) == 0;
+         !options_.buffers_to_assign->contains(buffer);
 }
 
 // Alloc always calls the underlying heap algorithm.
 void HeapSimulator::Alloc(const BufferValue* buffer,
                           const HloInstruction* instruction) {
-  CHECK(allocated_buffers_.count(buffer) == 0)
+  CHECK(!allocated_buffers_.contains(buffer))
       << "Alloc called on allocated buffer: " << *buffer;
-  CHECK(freed_buffers_.count(buffer) == 0)
+  CHECK(!freed_buffers_.contains(buffer))
       << "Alloc called on freed buffer: " << *buffer;
 
   allocated_buffers_.insert(buffer);
@@ -411,9 +411,9 @@ void HeapSimulator::Free(const BufferValue* buffer,
     buffer = group->canonical;
   }
 
-  CHECK(allocated_buffers_.count(buffer) > 0)
+  CHECK(allocated_buffers_.contains(buffer))
       << "Free called on non-allocated buffer: " << *buffer;
-  CHECK(freed_buffers_.count(buffer) == 0)
+  CHECK(!freed_buffers_.contains(buffer))
       << "Free called on freed buffer: " << *buffer;
 
   freed_buffers_.insert(buffer);
@@ -433,11 +433,11 @@ void HeapSimulator::ShareBuffer(const BufferValue* buffer,
                                 const HloInstruction* instruction) {
   CHECK_LE(size_fn_(*buffer), size_fn_(*shared))
       << "ShareBuffer oversized buffer" << *buffer << " shared: " << *shared;
-  CHECK(allocated_buffers_.count(buffer) == 0)
+  CHECK(!allocated_buffers_.contains(buffer))
       << "ShareBuffer called on allocated buffer: " << *buffer;
-  CHECK(freed_buffers_.count(buffer) == 0)
+  CHECK(!freed_buffers_.contains(buffer))
       << "ShareBuffer called on freed buffer: " << *buffer;
-  CHECK(freed_buffers_.count(shared) == 0)
+  CHECK(!freed_buffers_.contains(shared))
       << "ShareBuffer called on freed shared buffer: " << *shared;
 
   const BufferValue* canonical = nullptr;
@@ -452,7 +452,7 @@ void HeapSimulator::ShareBuffer(const BufferValue* buffer,
   } else {
     // The 'shared' buffer doesn't have a group; it must be the canonical.  Add
     // both 'buffer' and 'shared' to a new group.
-    CHECK(allocated_buffers_.count(shared) > 0)
+    CHECK(allocated_buffers_.contains(shared))
         << "ShareBuffer called on non-allocated shared buffer: " << *shared;
     auto group = std::make_shared<SharedGroup>();
     canonical = shared;
diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc
index 52ca67afb8..10f0cf3a34 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation.cc
@@ -207,14 +207,14 @@ Status HloComputation::RemoveInstructionAndUnusedOperands(
   TF_RET_CHECK(instruction->user_count() == 0);
   TF_RET_CHECK(IsRemovable(instruction))
       << "Cannot remove instruction: " << instruction->ToString();
-  std::unordered_set<HloInstruction*> removed;
+  absl::flat_hash_set<HloInstruction*> removed;
   std::queue<HloInstruction*> worklist;
   worklist.push(instruction);
   while (!worklist.empty()) {
     HloInstruction* item = worklist.front();
     worklist.pop();
 
-    if (removed.count(item) != 0 || item->user_count() != 0 ||
+    if (removed.contains(item) || item->user_count() != 0 ||
         item == root_instruction() || !IsRemovable(item) ||
         (item->HasSideEffect() && item != instruction)) {
       continue;
@@ -694,13 +694,14 @@ bool HloComputation::operator==(const HloComputation& other) const {
   if (this == &other) {
     return true;
   }
-  std::set<std::pair<const HloInstruction*, const HloInstruction*>> visited;
+  absl::flat_hash_set<std::pair<const HloInstruction*, const HloInstruction*>>
+      visited;
   std::function<bool(const HloInstruction*, const HloInstruction*)> eq =
       [&visited, &eq](const HloInstruction* a, const HloInstruction* b) {
         // If <a,b> are visited but not identical, the recursion should have
         // been aborted. So, if <a,b> are visited at this point, they must be
         // identical.
-        if (visited.count(std::make_pair(a, b)) > 0) {
+        if (visited.contains(std::make_pair(a, b))) {
           return true;
         }
         visited.emplace(a, b);
@@ -803,13 +804,13 @@ Status HloComputation::AcceptOrdered(
         << root->ToString();
   }
   TF_RET_CHECK(order.size() == instruction_count());
-  std::unordered_set<const HloInstruction*> visited;
+  absl::flat_hash_set<const HloInstruction*> visited;
   for (const HloInstruction* instruction : order) {
     VLOG(3) << "Visiting ordered: " << instruction->ToString();
-    TF_RET_CHECK(instruction_iterators_.count(instruction) == 1)
+    TF_RET_CHECK(instruction_iterators_.contains(instruction))
         << "Instruction " << instruction->name() << " is not in computation "
         << name();
-    TF_RET_CHECK(visited.count(instruction) == 0)
+    TF_RET_CHECK(!visited.contains(instruction))
         << "Instruction " << instruction->name()
         << " appears more than once in order";
     HloInstruction* mutable_instruction =
diff --git a/tensorflow/compiler/xla/service/hlo_computation_test.cc b/tensorflow/compiler/xla/service/hlo_computation_test.cc
index 0361c87428..216d56b868 100644
--- a/tensorflow/compiler/xla/service/hlo_computation_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation_test.cc
@@ -17,6 +17,7 @@ limitations under the License.
 
 #include <set>
 
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
@@ -226,7 +227,7 @@ TEST_F(HloComputationTest, VisitWithMultipleRoots) {
         : computation_(computation) {}
 
     Status DefaultAction(HloInstruction* hlo_instruction) override {
-      EXPECT_EQ(0, visited_set_.count(hlo_instruction));
+      EXPECT_FALSE(visited_set_.contains(hlo_instruction));
       visited_set_.insert(hlo_instruction);
       last_visited_ = hlo_instruction;
       return Status::OK();
@@ -239,7 +240,7 @@ TEST_F(HloComputationTest, VisitWithMultipleRoots) {
     }
 
     HloComputation* computation_;
-    std::set<HloInstruction*> visited_set_;
+    absl::flat_hash_set<HloInstruction*> visited_set_;
     int64 finish_visit_calls_ = 0;
     HloInstruction* last_visited_ = nullptr;
   };
diff --git a/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc b/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc
index 1924204df0..0db452b85f 100644
--- a/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc
+++ b/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc
@@ -107,7 +107,7 @@ bool HloDataflowAnalysis::AreTransitiveUsesElementwiseOrTuple(
           return false;
         }
       }
-      if (!visited.count(user)) {
+      if (!visited.contains(user)) {
         stack.push_back(user);
       }
     }
diff --git a/tensorflow/compiler/xla/service/hlo_dce.cc b/tensorflow/compiler/xla/service/hlo_dce.cc
index 7d35e251ca..a5a11f09cf 100644
--- a/tensorflow/compiler/xla/service/hlo_dce.cc
+++ b/tensorflow/compiler/xla/service/hlo_dce.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include <utility>
 #include <vector>
 
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
@@ -65,7 +66,7 @@ StatusOr<bool> HloDCE::Run(HloModule* module) {
 
   // Now DCE HloComputations.  First, collect the computations that are
   // referenced by some remaining instruction.
-  std::unordered_set<HloComputation*> live_computations;
+  absl::flat_hash_set<HloComputation*> live_computations;
   if (HloComputation* entry_computation = module->entry_computation()) {
     live_computations.insert(entry_computation);
   }
@@ -79,7 +80,7 @@ StatusOr<bool> HloDCE::Run(HloModule* module) {
 
   // Remove dead computations.
   for (auto* computation : module->MakeComputationPostOrder()) {
-    if (live_computations.count(computation) == 0) {
+    if (!live_computations.contains(computation)) {
       TF_RETURN_IF_ERROR(module->RemoveEmbeddedComputation(computation));
       changed = true;
     }
diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
index c6eaead8dd..22bcb2030b 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
@@ -24,9 +24,9 @@ limitations under the License.
 #include <queue>
 #include <string>
 #include <tuple>
-#include <unordered_map>
 #include <vector>
 
+#include "absl/container/flat_hash_map.h"
 #include "absl/strings/match.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_format.h"
@@ -380,7 +380,7 @@ class HloDotDumper {
   // Each HloInstruction dumped gets a monotically-increasing node ID.  This
   // must start at 1, because that's where graphviz's accounting starts.
   int64 next_node_id_ = 1;
-  std::unordered_map<const HloInstruction*, int64> node_ids_;
+  absl::flat_hash_map<const HloInstruction*, int64> node_ids_;
 
   // The "root" tag doesn't have an associated HloInstruction pointer, so we
   // need to store it outside the map.
@@ -397,7 +397,7 @@ class HloDotDumper {
 
   // Each HloComputation that's emitted gets a monotonically-increasing ID.
   int64 next_cluster_id_ = 1;
-  std::unordered_map<const HloComputation*, int64> cluster_ids_;
+  absl::flat_hash_map<const HloComputation*, int64> cluster_ids_;
 
   // Edges to print from Footer().  Edges come at the end because graphviz is
   // unhappy if an edge from a subcomputation to a node in the outer computation
@@ -407,7 +407,7 @@ class HloDotDumper {
 
   // When coloring by sharding information, we track the sharding string
   // representation to color association, by round-robin the color schemes.
-  std::unordered_map<HloSharding, ColorScheme, HloSharding::Hasher>
+  absl::flat_hash_map<HloSharding, ColorScheme, HloSharding::Hasher>
       sharding_colors_;
   int64 next_shard_color_ = 0;
 };
@@ -1286,7 +1286,7 @@ NodeFilter MakeNodeRadiusAroundFilter(const HloInstruction* root,
                                       int64 radius) {
   // First, find the neighborhood of nodes with distance from root <= radius.
   // These nodes are our initial set of "normal" nodes.
-  std::unordered_map<const HloInstruction*, NodeFilterResult> nodes;
+  absl::flat_hash_map<const HloInstruction*, NodeFilterResult> nodes;
   std::deque<std::pair<const HloInstruction*, /*depth*/ int64>> worklist;
   worklist.push_back({root, 0});
   while (!worklist.empty()) {
@@ -1307,7 +1307,7 @@ NodeFilter MakeNodeRadiusAroundFilter(const HloInstruction* root,
     // are not interesting to the graph at hand.
     if (instr == root || instr->opcode() != HloOpcode::kTuple) {
       for (const HloInstruction* operand : instr->operands()) {
-        if (!nodes.count(operand)) {
+        if (!nodes.contains(operand)) {
           worklist.push_back({operand, depth + 1});
         }
       }
@@ -1335,7 +1335,7 @@ NodeFilter MakeNodeRadiusAroundFilter(const HloInstruction* root,
       continue;
     }
     for (const HloInstruction* user : instr->users()) {
-      if (!nodes.count(user)) {
+      if (!nodes.contains(user)) {
         worklist.push_back({user, depth + 1});
       }
     }
@@ -1344,7 +1344,7 @@ NodeFilter MakeNodeRadiusAroundFilter(const HloInstruction* root,
   auto is_displayed = [&](const HloInstruction* instr) {
     // Constants are displayed inline with their users; they're never omitted.
     // Nodes in subcomputations are always shown.
-    return nodes.count(instr) > 0 || instr->opcode() == HloOpcode::kConstant ||
+    return nodes.contains(instr) || instr->opcode() == HloOpcode::kConstant ||
            instr->parent() != root->parent();
   };
 
diff --git a/tensorflow/compiler/xla/service/hlo_instruction_test.cc b/tensorflow/compiler/xla/service/hlo_instruction_test.cc
index 8048e332cb..35f031f29a 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction_test.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include <utility>
 #include <vector>
 
+#include "absl/container/flat_hash_map.h"
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/protobuf_util.h"
 #include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h"
@@ -55,13 +56,13 @@ class OpAndUserCollectingVisitor : public DfsHloVisitorWithDefault {
   }
 
   Status HandleParameter(HloInstruction* parameter) override {
-    EXPECT_EQ(0, count_.count(parameter));
+    EXPECT_FALSE(count_.contains(parameter));
     count_[parameter] = GetCountsForNode(parameter);
     return Status::OK();
   }
 
   Status HandleConstant(HloInstruction* constant) override {
-    EXPECT_EQ(0, count_.count(constant));
+    EXPECT_FALSE(count_.contains(constant));
     count_[constant] = GetCountsForNode(constant);
     return Status::OK();
   }
@@ -69,25 +70,25 @@ class OpAndUserCollectingVisitor : public DfsHloVisitorWithDefault {
   Status HandleAdd(HloInstruction* add) override {
     auto lhs = add->operand(0);
     auto rhs = add->operand(1);
-    EXPECT_EQ(0, count_.count(add));
-    EXPECT_GT(count_.count(lhs), 0);
-    EXPECT_GT(count_.count(rhs), 0);
+    EXPECT_FALSE(count_.contains(add));
+    EXPECT_TRUE(count_.contains(lhs));
+    EXPECT_TRUE(count_.contains(rhs));
     count_[add] = GetCountsForNode(add);
     return Status::OK();
   }
 
   Status HandleNegate(HloInstruction* negate) override {
     auto operand = negate->operand(0);
-    EXPECT_EQ(0, count_.count(negate));
-    EXPECT_GT(count_.count(operand), 0);
+    EXPECT_FALSE(count_.contains(negate));
+    EXPECT_TRUE(count_.contains(operand));
     count_[negate] = GetCountsForNode(negate);
     return Status::OK();
   }
 
   Status HandleMap(HloInstruction* map) override {
-    EXPECT_EQ(0, count_.count(map));
+    EXPECT_FALSE(count_.contains(map));
     for (HloInstruction* arg : map->operands()) {
-      EXPECT_GT(count_.count(arg), 0);
+      EXPECT_TRUE(count_.contains(arg));
     }
     count_[map] = GetCountsForNode(map);
     return Status::OK();
@@ -96,9 +97,9 @@ class OpAndUserCollectingVisitor : public DfsHloVisitorWithDefault {
   Status HandleReduce(HloInstruction* reduce) override {
     auto arg = reduce->operand(0);
     auto init_value = reduce->operand(1);
-    EXPECT_EQ(0, count_.count(reduce));
-    EXPECT_GT(count_.count(arg), 0);
-    EXPECT_GT(count_.count(init_value), 0);
+    EXPECT_FALSE(count_.contains(reduce));
+    EXPECT_TRUE(count_.contains(arg));
+    EXPECT_TRUE(count_.contains(init_value));
     count_[reduce] = GetCountsForNode(reduce);
     return Status::OK();
   }
@@ -128,7 +129,7 @@ class OpAndUserCollectingVisitor : public DfsHloVisitorWithDefault {
   }
 
   // Counters for HLOs. Maps HLO to a NumOpsAndUsers.
-  std::unordered_map<const HloInstruction*, NumOpsAndUsers> count_;
+  absl::flat_hash_map<const HloInstruction*, NumOpsAndUsers> count_;
 };
 
 TEST_F(HloInstructionTest, BasicProperties) {
@@ -137,7 +138,7 @@ TEST_F(HloInstructionTest, BasicProperties) {
   EXPECT_EQ(HloOpcode::kParameter, parameter->opcode());
   EXPECT_TRUE(ShapeUtil::IsScalarWithElementType(parameter->shape(), F32));
   EXPECT_FALSE(ShapeUtil::IsScalarWithElementType(parameter->shape(), S32));
-  EXPECT_EQ(0, parameter->operand_count());
+  EXPECT_FALSE(parameter->operand_count());
 }
 
 TEST_F(HloInstructionTest, UserWithTwoOperands) {
@@ -981,9 +982,9 @@ TEST_F(HloInstructionTest, FunctionVisitor) {
   module->AddEntryComputation(builder.Build());
 
   int visit_num = 0;
-  std::unordered_map<HloInstruction*, int> visit_order;
+  absl::flat_hash_map<HloInstruction*, int> visit_order;
   EXPECT_IS_OK(add->Accept([&visit_num, &visit_order](HloInstruction* inst) {
-    EXPECT_EQ(0, visit_order.count(inst));
+    EXPECT_FALSE(visit_order.contains(inst));
     visit_order[inst] = visit_num;
     visit_num++;
     return Status::OK();
diff --git a/tensorflow/compiler/xla/service/hlo_liveness_analysis.cc b/tensorflow/compiler/xla/service/hlo_liveness_analysis.cc
index 5bf055f3c0..e14bcfa7f6 100644
--- a/tensorflow/compiler/xla/service/hlo_liveness_analysis.cc
+++ b/tensorflow/compiler/xla/service/hlo_liveness_analysis.cc
@@ -17,6 +17,7 @@ limitations under the License.
 
 #include <deque>
 
+#include "absl/container/flat_hash_set.h"
 #include "absl/memory/memory.h"
 #include "absl/strings/str_cat.h"
 #include "tensorflow/compiler/xla/map_util.h"
@@ -36,11 +37,11 @@ namespace xla {
 namespace {
 
 using Worklist = std::deque<const HloInstruction*>;
-using Workset = std::unordered_set<const HloInstruction*>;
+using Workset = absl::flat_hash_set<const HloInstruction*>;
 
 void AddToWorklist(const HloInstruction* instruction, Worklist* worklist,
                    Workset* workset) {
-  if (workset->count(instruction) == 0) {
+  if (!workset->contains(instruction)) {
     worklist->push_back(instruction);
     workset->insert(instruction);
     VLOG(3) << "ADD instruction: " << instruction->name();
diff --git a/tensorflow/compiler/xla/service/hlo_module.cc b/tensorflow/compiler/xla/service/hlo_module.cc
index fe8371384c..ed78189fed 100644
--- a/tensorflow/compiler/xla/service/hlo_module.cc
+++ b/tensorflow/compiler/xla/service/hlo_module.cc
@@ -392,15 +392,12 @@ namespace {
 // Returns whether `hlo` is used outside the given subcomputation.
 // `instructions_in_subcomputation` is the instruction set of the given
 // subcomputation.
-bool IsUsedOutsideSubcomputation(
-    const HloInstruction& hlo,
-    const std::unordered_set<HloInstruction*>& instructions_in_subcomputation) {
-  for (HloInstruction* user : hlo.users()) {
-    if (!instructions_in_subcomputation.count(user)) {
-      return true;
-    }
-  }
-  return false;
+bool IsUsedOutsideSubcomputation(const HloInstruction& hlo,
+                                 const absl::flat_hash_set<HloInstruction*>&
+                                     instructions_in_subcomputation) {
+  return absl::c_any_of(hlo.users(), [&](HloInstruction* user) {
+    return !instructions_in_subcomputation.contains(user);
+  });
 }
 }  // anonymous namespace
 
@@ -411,9 +408,9 @@ HloInstruction* HloModule::OutlineExpressionFromComputation(
 
   // A map from original instructions to their counterparts in the new outlined
   // function.
-  std::unordered_map<HloInstruction*, HloInstruction*> outlined_instructions;
+  absl::flat_hash_map<HloInstruction*, HloInstruction*> outlined_instructions;
   // A set that contains all instructions to be outlined.
-  std::unordered_set<HloInstruction*> instruction_set_to_outline(
+  absl::flat_hash_set<HloInstruction*> instruction_set_to_outline(
       instructions_to_outline.begin(), instructions_to_outline.end());
   std::vector<HloInstruction*> arguments;
   std::vector<HloInstruction*> outputs;
@@ -502,7 +499,7 @@ std::vector<HloComputation*> HloModule::MakeComputationPostOrder() const {
   // First determine all root computations by building a set of nonroot
   // computations (computations which are called by an instruction in the
   // module).
-  std::set<HloComputation*> nonroot_computations;
+  absl::flat_hash_set<HloComputation*> nonroot_computations;
   for (auto& computation : computations_) {
     for (auto* instruction : computation->instructions()) {
       for (HloComputation* called_computation :
@@ -515,19 +512,19 @@ std::vector<HloComputation*> HloModule::MakeComputationPostOrder() const {
   // Keep track of computations which have already been added to the post
   // order. This prevents duplication as an embedded computation may be called
   // from two different root computations.
-  std::set<HloComputation*> added_computations;
+  absl::flat_hash_set<HloComputation*> added_computations;
   std::vector<HloComputation*> post_order;
   for (auto& computation : computations_) {
-    if (nonroot_computations.count(computation.get()) == 0) {
+    if (!nonroot_computations.contains(computation.get())) {
       for (HloComputation* embedded_computation :
            computation->MakeEmbeddedComputationsList()) {
-        if (added_computations.count(embedded_computation) == 0) {
+        if (!added_computations.contains(embedded_computation)) {
           post_order.push_back(embedded_computation);
           added_computations.insert(embedded_computation);
         }
       }
       // Root computations should only be encountered once.
-      CHECK_EQ(0, added_computations.count(computation.get()));
+      CHECK(!added_computations.contains(computation.get()));
       post_order.push_back(computation.get());
       added_computations.insert(computation.get());
     }
diff --git a/tensorflow/compiler/xla/service/hlo_module_group_metadata.cc b/tensorflow/compiler/xla/service/hlo_module_group_metadata.cc
index 80f8ca2226..47734bc55c 100644
--- a/tensorflow/compiler/xla/service/hlo_module_group_metadata.cc
+++ b/tensorflow/compiler/xla/service/hlo_module_group_metadata.cc
@@ -199,7 +199,7 @@ bool HloModuleGroupMetadata::IsChannelInstruction(
 }
 
 bool HloModuleGroupMetadata::IsCompanionInstruction(HloInstruction* hlo) const {
-  return companion_set_index_.count(hlo) > 0;
+  return companion_set_index_.contains(hlo);
 }
 
 bool HloModuleGroupMetadata::InstructionCommunicates(
@@ -510,7 +510,7 @@ Status HloModuleGroupMetadata::CheckCommunicatingInstruction(
   HloComputation* computation = instruction->parent();
   const HloModule* module = computation->parent();
   if (module->entry_computation() == computation ||
-      tracked_instructions_.count(computation) > 0) {
+      tracked_instructions_.contains(computation)) {
     return Status::OK();
   }
   return FailedPrecondition("channel is used in disallowed computation");
diff --git a/tensorflow/compiler/xla/service/hlo_module_group_metadata.h b/tensorflow/compiler/xla/service/hlo_module_group_metadata.h
index 5cd0e38f38..3ed95c1050 100644
--- a/tensorflow/compiler/xla/service/hlo_module_group_metadata.h
+++ b/tensorflow/compiler/xla/service/hlo_module_group_metadata.h
@@ -178,7 +178,7 @@ class HloModuleGroupMetadata {
   // Precondition: IsCompanionWhile(instruction) is true.
   const std::vector<HloInstruction*>& Companions(
       const HloInstruction* instruction) const {
-    CHECK_EQ(companion_set_index_.count(instruction), 1);
+    CHECK(companion_set_index_.contains(instruction));
     return companion_set(companion_set_index_.at(instruction));
   }
 
diff --git a/tensorflow/compiler/xla/service/hlo_ordering.cc b/tensorflow/compiler/xla/service/hlo_ordering.cc
index ca6a154809..0cec61c257 100644
--- a/tensorflow/compiler/xla/service/hlo_ordering.cc
+++ b/tensorflow/compiler/xla/service/hlo_ordering.cc
@@ -367,7 +367,7 @@ bool SequentialHloOrdering::ExecutesBeforeInSameComputation(
     const HloInstruction* a, const HloInstruction* b) const {
   CHECK_EQ(a->parent(), b->parent());
   // If either instruction is not in the order, then 'a' and 'b' are unordered.
-  if (order_position_.count(a) == 0 || order_position_.count(b) == 0) {
+  if (!order_position_.contains(a) || !order_position_.contains(b)) {
     return false;
   }
   return order_position_.at(a) < order_position_.at(b);
diff --git a/tensorflow/compiler/xla/service/hlo_pass_pipeline.cc b/tensorflow/compiler/xla/service/hlo_pass_pipeline.cc
index 33ce7e23a8..ae8c08cf1d 100644
--- a/tensorflow/compiler/xla/service/hlo_pass_pipeline.cc
+++ b/tensorflow/compiler/xla/service/hlo_pass_pipeline.cc
@@ -89,7 +89,7 @@ std::vector<HloPassInterface*> HloPassPipeline::GetEnabledPasses(
 
   std::vector<HloPassInterface*> enabled_passes;
   for (auto& pass : passes_) {
-    if (disabled_pass_names.count(string(pass->name())) == 0) {
+    if (!disabled_pass_names.contains(pass->name())) {
       enabled_passes.push_back(pass.get());
     }
   }
diff --git a/tensorflow/compiler/xla/service/hlo_schedule.cc b/tensorflow/compiler/xla/service/hlo_schedule.cc
index 8f6eb974c5..e75373501c 100644
--- a/tensorflow/compiler/xla/service/hlo_schedule.cc
+++ b/tensorflow/compiler/xla/service/hlo_schedule.cc
@@ -140,7 +140,7 @@ Status HloSchedule::UpdateComputationSchedule(
   std::queue<HloInstruction*> worklist;
 
   for (HloInstruction* instruction : computation->instructions()) {
-    if (ids_in_schedule.count(instruction->unique_id()) == 0) {
+    if (!ids_in_schedule.contains(instruction->unique_id())) {
       // This is a newly added instruction which is not in the schedule.
       if (instruction->operands().empty()) {
         worklist.push(instruction);
@@ -204,7 +204,7 @@ Status HloSchedule::Update() {
   std::vector<HloComputation*> nonfusion_computations =
       module_->MakeNonfusionComputations();
   for (const HloComputation* computation : nonfusion_computations) {
-    TF_RET_CHECK(sequences_.count(computation->unique_id()) == 1)
+    TF_RET_CHECK(sequences_.contains(computation->unique_id()))
         << "Computation " << computation->name() << " not in HloSchedule.";
   }
   if (sequences_.size() > nonfusion_computations.size()) {
@@ -215,7 +215,7 @@ Status HloSchedule::Update() {
       nonfusion_computations_ids.insert(computation->unique_id());
     }
     for (auto it = sequences_.begin(); it != sequences_.end();) {
-      if (nonfusion_computations_ids.count(it->first) == 0) {
+      if (!nonfusion_computations_ids.contains(it->first)) {
         sequences_.erase(it++);
       } else {
         ++it;
@@ -244,7 +244,7 @@ Status HloSchedule::Verify() const {
       << "Schedule has " << sequences_.size() << " sequences, but module has "
       << nonfusion_computations.size() << " non-fusion computations";
   for (const HloComputation* computation : nonfusion_computations) {
-    TF_RET_CHECK(sequences_.count(computation->unique_id()) == 1)
+    TF_RET_CHECK(sequences_.contains(computation->unique_id()))
         << "Computation " << computation->name()
         << " missing from HLO schedule.";
   }
@@ -268,7 +268,7 @@ Status HloSchedule::Verify() const {
         << instruction_position.size() << " instructions, expected "
         << computation->instruction_count();
     for (const HloInstruction* instruction : computation->instructions()) {
-      TF_RET_CHECK(instruction_position.count(instruction) == 1)
+      TF_RET_CHECK(instruction_position.contains(instruction))
           << "Instruction " << instruction->name() << " is not in schedule";
     }
 
diff --git a/tensorflow/compiler/xla/service/hlo_schedule.h b/tensorflow/compiler/xla/service/hlo_schedule.h
index 486ddbf499..a5f54ae2c3 100644
--- a/tensorflow/compiler/xla/service/hlo_schedule.h
+++ b/tensorflow/compiler/xla/service/hlo_schedule.h
@@ -110,7 +110,7 @@ class HloSchedule {
 
   // Returns true if the schedule has a sequence for the given computation.
   bool is_computation_scheduled(const HloComputation* computation) const {
-    return sequences_.count(computation->unique_id()) == 1;
+    return sequences_.contains(computation->unique_id());
   }
 
   // Updates the schedule such that it is (again) a valid schedule for the
diff --git a/tensorflow/compiler/xla/service/hlo_sharding.cc b/tensorflow/compiler/xla/service/hlo_sharding.cc
index bdd6ec1169..6c9c32f179 100644
--- a/tensorflow/compiler/xla/service/hlo_sharding.cc
+++ b/tensorflow/compiler/xla/service/hlo_sharding.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/service/hlo_sharding.h"
 
+#include "absl/container/flat_hash_set.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_join.h"
 #include "tensorflow/compiler/xla/overflow_util.h"
@@ -316,7 +317,7 @@ Status HloSharding::ValidateNonTuple(const Shape& shape,
   // All tile assignments must be less than the number of available cores and
   // unique.
   Status status = Status::OK();
-  std::set<int64> seen_cores;
+  absl::flat_hash_set<int64> seen_cores;
   tile_assignment_.Each(
       [&](absl::Span<const int64> indices, int32 core) {
         // Don't overwrite a bad status, so we report the first error.
@@ -324,7 +325,7 @@ Status HloSharding::ValidateNonTuple(const Shape& shape,
           if (core >= num_devices) {
             status = tensorflow::errors::InvalidArgument(StrCat(
                 "core ", core, " > ", num_devices, " in tile assignment"));
-          } else if (seen_cores.count(core) != 0) {
+          } else if (seen_cores.contains(core)) {
             status = tensorflow::errors::InvalidArgument(
                 StrCat("core ", core, " is not unique in tile assignment"));
           }
diff --git a/tensorflow/compiler/xla/service/hlo_sharding_metadata.cc b/tensorflow/compiler/xla/service/hlo_sharding_metadata.cc
index b414d2a663..094d98bc6e 100644
--- a/tensorflow/compiler/xla/service/hlo_sharding_metadata.cc
+++ b/tensorflow/compiler/xla/service/hlo_sharding_metadata.cc
@@ -99,7 +99,7 @@ std::vector<PassThrough> LocatePassThroughDomainLinks(
         << "Instruction is not a kDomain: " << instruction->ToString();
     for (HloInstruction* user : instruction->users()) {
       if (user->opcode() == HloOpcode::kDomain &&
-          domain.exit_domains.count(user) != 0) {
+          domain.exit_domains.contains(user)) {
         pass_through.emplace_back(user, instruction);
         VLOG(2) << "Found passthrough domain link:";
         VLOG(2) << "  " << user->ToString();
@@ -253,7 +253,7 @@ StatusOr<bool> ApplyShardingFromUsers(HloInstruction* instruction,
       instruction->shape(), HloSharding::AssignDevice(kUnassignedDevice));
   for (HloInstruction* user : instruction->users()) {
     if (user->opcode() == HloOpcode::kDomain &&
-        domain.exit_domains.count(user) > 0) {
+        domain.exit_domains.contains(user)) {
       // If a user is a domain and it is registered in the domain exits, then
       // the instruction sharding is taken directly from the domain, and no
       // further users need to be visited.
diff --git a/tensorflow/compiler/xla/service/indexed_array_analysis.cc b/tensorflow/compiler/xla/service/indexed_array_analysis.cc
index a41cf714c5..76bf48870d 100644
--- a/tensorflow/compiler/xla/service/indexed_array_analysis.cc
+++ b/tensorflow/compiler/xla/service/indexed_array_analysis.cc
@@ -103,7 +103,7 @@ Status IndexedArrayAnalysis::TraverseAndPopulateCache(
 
   do {
     const HloInstruction* instr = stack.back();
-    if (cache_.count(instr)) {
+    if (cache_.contains(instr)) {
       stack.pop_back();
       continue;
     }
@@ -111,9 +111,9 @@ Status IndexedArrayAnalysis::TraverseAndPopulateCache(
     switch (FindOrDie(dfs_state_map, instr)) {
       case kDiscovered: {
         for (const HloInstruction* operand : instr->operands()) {
-          if (!cache_.count(operand)) {
+          if (!cache_.contains(operand)) {
             stack.push_back(operand);
-            CHECK(!dfs_state_map.count(operand) ||
+            CHECK(!dfs_state_map.contains(operand) ||
                   dfs_state_map[operand] == kDiscovered);
             dfs_state_map[operand] = kDiscovered;
           }
diff --git a/tensorflow/compiler/xla/service/layout_assignment.cc b/tensorflow/compiler/xla/service/layout_assignment.cc
index 09246db68e..406cd0a219 100644
--- a/tensorflow/compiler/xla/service/layout_assignment.cc
+++ b/tensorflow/compiler/xla/service/layout_assignment.cc
@@ -2135,7 +2135,7 @@ Status LayoutAssignment::ClearPreviousPassSideEffects(HloModule* module) {
     for (HloInstruction* instruction :
          computation->MakeInstructionPostOrder()) {
       if (instruction->opcode() == HloOpcode::kCopy &&
-          added_copies_.count(instruction) > 0) {
+          added_copies_.contains(instruction)) {
         VLOG(5) << "Removing added copy: " << instruction->ToString();
         TF_RETURN_IF_ERROR(
             instruction->ReplaceAllUsesWith(instruction->mutable_operand(0)));
diff --git a/tensorflow/compiler/xla/service/layout_assignment.h b/tensorflow/compiler/xla/service/layout_assignment.h
index 3b081de3c7..5701cb5b02 100644
--- a/tensorflow/compiler/xla/service/layout_assignment.h
+++ b/tensorflow/compiler/xla/service/layout_assignment.h
@@ -243,7 +243,7 @@ class ChannelLayoutConstraints {
 
   // Returns true if channel_id has a layout constraint.
   bool IsChannelConstrained(int64 channel_id) const {
-    return constraints_.count(channel_id) > 0;
+    return constraints_.contains(channel_id);
   }
 
   // Given `shape`, apply the layout for `channel_id`. `channel_id` must already
@@ -276,7 +276,7 @@ class ChannelLayoutConstraints {
   }
 
  private:
-  std::unordered_map<int64, Layout> constraints_;
+  absl::flat_hash_map<int64, Layout> constraints_;
 };
 
 // HLO pass which assigns layouts to all instructions in the HLO module while
diff --git a/tensorflow/compiler/xla/service/llvm_ir/BUILD b/tensorflow/compiler/xla/service/llvm_ir/BUILD
index 728a66b388..54e8fe1947 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/BUILD
+++ b/tensorflow/compiler/xla/service/llvm_ir/BUILD
@@ -169,6 +169,7 @@ cc_library(
         "//tensorflow/compiler/xla/service:elemental_ir_emitter",
         "//tensorflow/compiler/xla/service:hlo",
         "//tensorflow/core:lib",
+        "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/types:optional",
         "@com_google_absl//absl/types:span",
         "@llvm//:core",
diff --git a/tensorflow/compiler/xla/service/llvm_ir/alias_analysis.h b/tensorflow/compiler/xla/service/llvm_ir/alias_analysis.h
index 2b46b3c396..12e2f449e2 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/alias_analysis.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/alias_analysis.h
@@ -76,15 +76,12 @@ class AliasAnalysis {
   // A map from a buffer slice to metadata corresponding to its alias.scope
   // metadata.  The index kParameterAliasSet is used to hold aliasing
   // information for parameters.
-  absl::flat_hash_map<BufferAllocation::Slice, llvm::MDNode*,
-                      BufferAllocation::Slice::Hasher>
+  absl::flat_hash_map<BufferAllocation::Slice, llvm::MDNode*>
       alias_scope_metadata_;
 
   // A map from a buffer slice to metadata corresponding to its noalias
   // metadata.
-  absl::flat_hash_map<BufferAllocation::Slice, llvm::MDNode*,
-                      BufferAllocation::Slice::Hasher>
-      noalias_metadata_;
+  absl::flat_hash_map<BufferAllocation::Slice, llvm::MDNode*> noalias_metadata_;
 };
 
 }  // namespace llvm_ir
diff --git a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc
index 03a475b40b..e440f05e2b 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc
@@ -35,7 +35,7 @@ using llvm_ir::IrArray;
 Status FusedIrEmitter::DefaultAction(HloInstruction* hlo) {
   indexed_generators_[hlo] =
       [=](const IrArray::Index& index) -> StatusOr<llvm::Value*> {
-    if (generated_value_cache_[hlo].count(index.multidim()) > 0) {
+    if (generated_value_cache_[hlo].contains(index.multidim())) {
       llvm::Value* generated_value =
           generated_value_cache_[hlo][index.multidim()];
       llvm::BasicBlock* generated_value_bb = nullptr;
diff --git a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h
index 1b9c61f670..e6d52a580c 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h
@@ -19,6 +19,7 @@ limitations under the License.
 #include <map>
 #include <unordered_map>
 
+#include "absl/container/flat_hash_map.h"
 #include "absl/types/optional.h"
 #include "absl/types/span.h"
 #include "llvm/IR/IRBuilder.h"
@@ -134,8 +135,9 @@ class FusedIrEmitter : public DfsHloVisitorWithDefault {
 
   // Cache of generated values, lest we regenerate an element of a node with
   // multiple outgoing edges
-  std::unordered_map<const HloInstruction*,
-                     std::map<std::vector<llvm::Value*>, llvm::Value*>>
+  absl::flat_hash_map<
+      const HloInstruction*,
+      absl::flat_hash_map<std::vector<llvm::Value*>, llvm::Value*>>
       generated_value_cache_;
 };
 
diff --git a/tensorflow/compiler/xla/service/multi_output_fusion.cc b/tensorflow/compiler/xla/service/multi_output_fusion.cc
index 9ccdd7d8d8..53d52d9a3d 100644
--- a/tensorflow/compiler/xla/service/multi_output_fusion.cc
+++ b/tensorflow/compiler/xla/service/multi_output_fusion.cc
@@ -198,7 +198,7 @@ void MultiOutputFusion::Update(HloInstruction* instr1, HloInstruction* instr2) {
     if (instr == fusion || is_fused(instr) || is_connected(fusion, instr)) {
       continue;
     }
-    if (in_list.count(instr) > 0) {
+    if (in_list.contains(instr)) {
       continue;
     }
     int64 profit = GetProfit(instr, fusion);
diff --git a/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc b/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc
index b1f0672c60..dec719b04a 100644
--- a/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc
+++ b/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include <utility>
 #include <vector>
 
+#include "absl/container/flat_hash_set.h"
 #include "absl/memory/memory.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_format.h"
@@ -55,11 +56,10 @@ bool PointsToSet::IsAmbiguous() const {
 
 bool PointsToSet::IsDistinct() const {
   bool distinct = true;
-  std::set<const LogicalBuffer*> all_points_to;
-  ForEachElement([&distinct, &all_points_to](const ShapeIndex& /*index*/,
-                                             const BufferList& points_to) {
+  absl::flat_hash_set<const LogicalBuffer*> all_points_to;
+  ForEachElement([&](const ShapeIndex& /*index*/, const BufferList& points_to) {
     for (auto& buffer : points_to) {
-      if (all_points_to.count(buffer) != 0) {
+      if (all_points_to.contains(buffer)) {
         distinct = false;
       }
       all_points_to.insert(buffer);
diff --git a/tensorflow/compiler/xla/service/while_loop_invariant_code_motion.cc b/tensorflow/compiler/xla/service/while_loop_invariant_code_motion.cc
index a1c627a319..69cc8feb3f 100644
--- a/tensorflow/compiler/xla/service/while_loop_invariant_code_motion.cc
+++ b/tensorflow/compiler/xla/service/while_loop_invariant_code_motion.cc
@@ -89,7 +89,7 @@ static void CreateLoopInvariantCopy(
 
     HloInstruction* next_operand =
         frame->instruction->mutable_operand(frame->operand_index++);
-    if (hoisted_instructions->count(next_operand) ||
+    if (hoisted_instructions->contains(next_operand) ||
         next_operand == while_body_param) {
       continue;
     }
@@ -241,7 +241,7 @@ WhileLoopInvariantCodeMotion::TryHoistingInvariantInstructionsFromWhileBody(
 
     auto is_invariant = [&](HloInstruction* op) {
       return hoisted_instructions.find(op) != hoisted_instructions.end() ||
-             unhoisted_invariant_instructions.count(op) ||
+             unhoisted_invariant_instructions.contains(op) ||
              op->opcode() == HloOpcode::kConstant;
     };
 
diff --git a/tensorflow/compiler/xla/service/while_loop_simplifier.cc b/tensorflow/compiler/xla/service/while_loop_simplifier.cc
index 772faa25e2..de884b037d 100644
--- a/tensorflow/compiler/xla/service/while_loop_simplifier.cc
+++ b/tensorflow/compiler/xla/service/while_loop_simplifier.cc
@@ -127,7 +127,7 @@ static StatusOr<bool> TryRemoveDeadWhileParams(HloInstruction* while_op) {
   // through to the while body's root, count that element as "used", since
   // removing that element would be observable.
   for (int64 i = 0; i < while_body_root->operand_count(); ++i) {
-    if (used_tuple_indices.count(i)) {
+    if (used_tuple_indices.contains(i)) {
       continue;
     }
 
-- 
GitLab


From b4813a0cff9f7ad1278aee376a5302f31c4558a2 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 4 Jan 2019 12:29:13 -0800
Subject: [PATCH 0203/2345] [XLA] Use absl::c_foo rather than std::foo.

No functional change.

PiperOrigin-RevId: 227896034
---
 tensorflow/compiler/xla/BUILD                 |  1 +
 .../compiler/xla/client/lib/sorting_test.cc   |  2 +-
 tensorflow/compiler/xla/layout_util.cc        |  6 +-
 .../compiler/xla/metric_table_report.cc       |  4 +-
 tensorflow/compiler/xla/service/BUILD         |  1 +
 .../xla/service/algebraic_simplifier.cc       |  7 +-
 .../compiler/xla/service/buffer_assignment.cc | 85 +++++++++----------
 .../compiler/xla/service/call_graph_test.cc   |  2 +-
 .../xla/service/computation_layout.cc         |  4 +-
 .../compiler/xla/service/copy_insertion.cc    | 18 ++--
 .../compiler/xla/service/cpu/disassembler.cc  | 21 +++--
 .../compiler/xla/service/cpu/ir_emitter.cc    | 12 +--
 .../xla/service/gpu/ir_emission_utils.cc      | 15 ++--
 .../xla/service/gpu/ir_emitter_unnested.cc    |  8 +-
 .../compiler/xla/service/heap_simulator.cc    | 41 +++++----
 .../xla/service/hlo_alias_analysis.cc         | 30 +++----
 tensorflow/compiler/xla/service/hlo_buffer.cc |  2 +-
 .../compiler/xla/service/hlo_computation.cc   | 12 ++-
 .../xla/service/hlo_dataflow_analysis.cc      | 36 +++-----
 .../compiler/xla/service/hlo_dce_test.cc      |  4 +-
 .../compiler/xla/service/hlo_domain_map.cc    |  8 +-
 .../compiler/xla/service/hlo_evaluator.cc     |  3 +-
 .../xla/service/hlo_evaluator_typed_visitor.h |  3 +-
 .../compiler/xla/service/hlo_graph_dumper.cc  | 42 +++++----
 .../compiler/xla/service/hlo_instruction.cc   | 37 ++++----
 .../compiler/xla/service/hlo_instructions.cc  | 18 ++--
 tensorflow/compiler/xla/service/hlo_module.cc | 18 ++--
 .../xla/service/hlo_module_dce_test.cc        |  4 +-
 tensorflow/compiler/xla/service/hlo_parser.cc |  2 +-
 .../xla/service/hlo_profile_printer.cc        | 10 +--
 .../compiler/xla/service/hlo_reachability.cc  |  2 +-
 .../xla/service/hlo_rematerialization.cc      | 25 +++---
 .../compiler/xla/service/hlo_sharding.cc      |  9 +-
 .../compiler/xla/service/hlo_sharding.h       |  9 +-
 .../xla/service/hlo_tfgraph_builder.cc        |  3 +-
 tensorflow/compiler/xla/service/hlo_value.cc  |  2 +-
 .../service/human_readable_profile_builder.cc |  6 +-
 .../xla/service/instruction_fusion.cc         | 30 +++----
 .../compiler/xla/service/layout_assignment.cc |  9 +-
 .../xla/service/llvm_ir/alias_analysis.cc     |  4 +-
 .../compiler/xla/service/name_uniquer.cc      |  2 +-
 .../compiler/xla/service/platform_util.cc     |  4 +-
 .../compiler/xla/service/shape_inference.cc   | 22 +++--
 .../compiler/xla/service/transpose_folding.cc |  6 +-
 .../xla/service/tuple_points_to_analysis.cc   | 26 +++---
 .../service/tuple_points_to_analysis_test.cc  |  5 +-
 .../xla/service/while_loop_simplifier.cc      |  5 +-
 .../xla/service/while_loop_simplifier_test.cc | 11 ++-
 tensorflow/compiler/xla/shape.cc              |  3 +-
 tensorflow/compiler/xla/shape_util.cc         |  3 +-
 tensorflow/compiler/xla/sparse_index_array.h  |  2 +-
 .../compiler/xla/tests/convolution_test.cc    |  4 +-
 tensorflow/compiler/xla/util.cc               |  2 +-
 tensorflow/compiler/xla/util.h                |  3 +-
 tensorflow/compiler/xla/window_util.cc        | 32 ++++---
 55 files changed, 305 insertions(+), 380 deletions(-)

diff --git a/tensorflow/compiler/xla/BUILD b/tensorflow/compiler/xla/BUILD
index 8a0e2db2b5..45623c2718 100644
--- a/tensorflow/compiler/xla/BUILD
+++ b/tensorflow/compiler/xla/BUILD
@@ -717,6 +717,7 @@ cc_library(
         ":types",
         ":xla_data_proto",
         "//tensorflow/core:lib",
+        "@com_google_absl//absl/algorithm:container",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/types:span",
     ],
diff --git a/tensorflow/compiler/xla/client/lib/sorting_test.cc b/tensorflow/compiler/xla/client/lib/sorting_test.cc
index 27ff36c749..0fbd138aca 100644
--- a/tensorflow/compiler/xla/client/lib/sorting_test.cc
+++ b/tensorflow/compiler/xla/client/lib/sorting_test.cc
@@ -77,7 +77,7 @@ XLA_TEST_F(SortingTest, TopKFullSort) {
   auto x = ConstantR1<float>(&builder, inputs);
   xla::GetTupleElement(xla::TopK(x, kSize), 0);
 
-  std::sort(inputs.begin(), inputs.end(), std::greater<float>());
+  absl::c_sort(inputs, std::greater<float>());
   ComputeAndCompareR1<float>(&builder, inputs, {});
 }
 
diff --git a/tensorflow/compiler/xla/layout_util.cc b/tensorflow/compiler/xla/layout_util.cc
index 42649d6692..2fe9b56c6b 100644
--- a/tensorflow/compiler/xla/layout_util.cc
+++ b/tensorflow/compiler/xla/layout_util.cc
@@ -290,8 +290,8 @@ Layout CreateDefaultLayoutForRank(int64 rank) {
 /* static */ bool LayoutUtil::HasLayout(const Shape& shape) {
   if (shape.IsTuple()) {
     // Tuple shape: all subshapes must have a layout.
-    return std::all_of(shape.tuple_shapes().begin(), shape.tuple_shapes().end(),
-                       [](const Shape& s) { return HasLayout(s); });
+    return absl::c_all_of(shape.tuple_shapes(),
+                          [](const Shape& s) { return HasLayout(s); });
   } else if (!shape.IsArray()) {
     // Opaque, token types etc. ignore layout.
     return true;
@@ -424,7 +424,7 @@ Status LayoutUtil::CopyLayoutBetweenShapes(const Shape& src, Shape* dst) {
     positions_in_layout.push_back(
         PositionInContainer(layout.minor_to_major(), dim));
   }
-  std::sort(positions_in_layout.begin(), positions_in_layout.end());
+  absl::c_sort(positions_in_layout);
   for (size_t i = 1; i < positions_in_layout.size(); ++i) {
     if (1 != positions_in_layout[i] - positions_in_layout[i - 1]) {
       return false;
diff --git a/tensorflow/compiler/xla/metric_table_report.cc b/tensorflow/compiler/xla/metric_table_report.cc
index 4eab4fa429..ad1699a1ae 100644
--- a/tensorflow/compiler/xla/metric_table_report.cc
+++ b/tensorflow/compiler/xla/metric_table_report.cc
@@ -55,7 +55,7 @@ string MetricTableReport::MakeReport(double expected_metric_sum) {
   const auto metric_greater = [](const Entry& a, const Entry& b) {
     return a.metric > b.metric;
   };
-  std::sort(entries_.begin(), entries_.end(), metric_greater);
+  absl::c_sort(entries_, metric_greater);
 
   // Create the report
   AppendLine();
@@ -117,7 +117,7 @@ std::vector<MetricTableReport::Category> MetricTableReport::MakeCategories(
   auto metric_sum_greater = [](const Category& a, const Category& b) {
     return a.metric_sum > b.metric_sum;
   };
-  std::sort(categories.begin(), categories.end(), metric_sum_greater);
+  absl::c_sort(categories, metric_sum_greater);
 
   return categories;
 }
diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 85a21a8518..3c18c450e6 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -3414,6 +3414,7 @@ cc_library(
         ":hlo_profile_printer_data",
         ":human_readable_profile_builder",
         "//tensorflow/compiler/xla:types",
+        "@com_google_absl//absl/algorithm:container",
         "@com_google_absl//absl/strings",
     ],
 )
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index 3d269dc04f..003cb7e791 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -2216,8 +2216,7 @@ Status AlgebraicSimplifierVisitor::HandleReverse(HloInstruction* reverse) {
   auto dim_is_one = [&](int64 i) -> bool {
     return reverse->shape().dimensions(i) == 1;
   };
-  if (std::all_of(reverse->dimensions().begin(), reverse->dimensions().end(),
-                  dim_is_one)) {
+  if (absl::c_all_of(reverse->dimensions(), dim_is_one)) {
     return ReplaceInstruction(reverse, reverse->mutable_operand(0));
   }
   return Status::OK();
@@ -2492,9 +2491,9 @@ Status AlgebraicSimplifierVisitor::HandleReduce(HloInstruction* reduce) {
     // Create a new reduce with the combined reduction dimensions of both
     // reduces.
     std::vector<int64> arg_dims = arg->dimensions();
-    std::sort(arg_dims.begin(), arg_dims.end());
+    absl::c_sort(arg_dims);
     std::vector<int64> reduce_dims = reduce->dimensions();
-    std::sort(reduce_dims.begin(), reduce_dims.end());
+    absl::c_sort(reduce_dims);
     // Transform reduce_dims to the same rank as the operand of the operand.
     for (int64 arg_dim : arg_dims) {
       for (int64& dim : reduce_dims) {
diff --git a/tensorflow/compiler/xla/service/buffer_assignment.cc b/tensorflow/compiler/xla/service/buffer_assignment.cc
index c997d594c4..c3285f516b 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment.cc
+++ b/tensorflow/compiler/xla/service/buffer_assignment.cc
@@ -86,10 +86,9 @@ std::vector<int64> ColorInterferenceGraph(
   // first, but it would be good to investigate other ordering heuristics too.
   std::vector<int64> nodes(node_count);
   std::iota(nodes.begin(), nodes.end(), 0);
-  std::sort(nodes.begin(), nodes.end(),
-            [&interference_map](const int64 i, const int64 j) {
-              return interference_map[i].size() > interference_map[j].size();
-            });
+  absl::c_sort(nodes, [&interference_map](const int64 i, const int64 j) {
+    return interference_map[i].size() > interference_map[j].size();
+  });
 
   const int64 kColorUnassigned = -1;
   std::vector<int64> assigned_colors(node_count, kColorUnassigned);
@@ -272,11 +271,12 @@ BufferAllocationProto BufferAllocation::ToProto() const {
     proto_assigned->set_offset(buffer_offset_size.second.offset);
     proto_assigned->set_size(buffer_offset_size.second.size);
   }
-  std::sort(proto.mutable_assigned()->begin(), proto.mutable_assigned()->end(),
-            [](const BufferAllocationProto::Assigned& assign1,
-               const BufferAllocationProto::Assigned& assign2) {
-              return assign1.logical_buffer_id() < assign2.logical_buffer_id();
-            });
+  absl::c_sort(*proto.mutable_assigned(),
+               [](const BufferAllocationProto::Assigned& assign1,
+                  const BufferAllocationProto::Assigned& assign2) {
+                 return assign1.logical_buffer_id() <
+                        assign2.logical_buffer_id();
+               });
   return proto;
 }
 
@@ -308,10 +308,10 @@ string BufferAllocation::ToString() const {
   for (const auto& buffer_offset_size : assigned_buffers_) {
     sorted_buffers.push_back(buffer_offset_size.first);
   }
-  std::sort(sorted_buffers.begin(), sorted_buffers.end(),
-            [](const LogicalBuffer* a, const LogicalBuffer* b) {
-              return a->id() < b->id();
-            });
+  absl::c_sort(sorted_buffers,
+               [](const LogicalBuffer* a, const LogicalBuffer* b) {
+                 return a->id() < b->id();
+               });
   for (const LogicalBuffer* buffer : sorted_buffers) {
     const OffsetSize& offset_size = FindOrDie(assigned_buffers_, buffer);
     StrAppend(&output, absl::StrFormat(
@@ -479,10 +479,9 @@ bool BufferAssignment::HaveDisjointSlices(const HloInstruction* hlo_a,
   // didn't return the empty set) for both HLOs, and the two resulting sets of
   // slices are disjoint.
   return !slices_a.empty() && !slices_b.empty() &&
-         std::none_of(slices_a.begin(), slices_a.end(),
-                      [&](const BufferAllocation::Slice& slice) {
-                        return slices_b.count(slice) > 0;
-                      });
+         absl::c_none_of(slices_a, [&](const BufferAllocation::Slice& slice) {
+           return slices_b.contains(slice);
+         });
 }
 
 StatusOr<BufferAllocation::Slice>
@@ -952,28 +951,28 @@ Status BufferAssigner::AssignBuffersForComputation(
   // operands (assuming operands are the same/larger size) enabling the
   // important reuse case where an elementwise instruction reuses one of its
   // operand's buffer. This improves locality.
-  std::sort(sorted_buffers.begin(), sorted_buffers.end(),
-            [has_sequential_order, &liveness, &post_order_position, assignment](
-                const LogicalBuffer* a, const LogicalBuffer* b) {
-              // Primary sort is by decreasing buffer size.
-              const int64 a_size = assignment->buffer_size_(*a);
-              const int64 b_size = assignment->buffer_size_(*b);
-              if (a_size != b_size) {
-                return a_size > b_size;  // use ">" for decreasing size.
-              }
-              // Otherwise live out buffers come before others, if the
-              // instructions are sequentially ordered.
-              if (has_sequential_order) {
-                const bool a_live_out = liveness.MaybeLiveOut(*a);
-                const bool b_live_out = liveness.MaybeLiveOut(*b);
-                if (a_live_out != b_live_out) {
-                  return a_live_out;
-                }
-              }
-              // Final tiebreaker is in instruction post order.
-              return post_order_position.at(a->instruction()) <
-                     post_order_position.at(b->instruction());
-            });
+  absl::c_sort(sorted_buffers,
+               [has_sequential_order, &liveness, &post_order_position,
+                assignment](const LogicalBuffer* a, const LogicalBuffer* b) {
+                 // Primary sort is by decreasing buffer size.
+                 const int64 a_size = assignment->buffer_size_(*a);
+                 const int64 b_size = assignment->buffer_size_(*b);
+                 if (a_size != b_size) {
+                   return a_size > b_size;  // use ">" for decreasing size.
+                 }
+                 // Otherwise live out buffers come before others, if the
+                 // instructions are sequentially ordered.
+                 if (has_sequential_order) {
+                   const bool a_live_out = liveness.MaybeLiveOut(*a);
+                   const bool b_live_out = liveness.MaybeLiveOut(*b);
+                   if (a_live_out != b_live_out) {
+                     return a_live_out;
+                   }
+                 }
+                 // Final tiebreaker is in instruction post order.
+                 return post_order_position.at(a->instruction()) <
+                        post_order_position.at(b->instruction());
+               });
 
   // BufferAllocations are necessarily created in decreasing size order. Keep
   // indices of previously created BufferAllocations in allocation_indices.
@@ -1305,10 +1304,10 @@ std::vector<const LogicalBuffer*> ComputePeakMemoryLogicalBuffers(
                              live_buffers.end());
 
   // Stabily sort the live buffers.
-  std::sort(live_buffers_vector.begin(), live_buffers_vector.end(),
-            [](const LogicalBuffer* a, const LogicalBuffer* b) {
-              return a->id() < b->id();
-            });
+  absl::c_sort(live_buffers_vector,
+               [](const LogicalBuffer* a, const LogicalBuffer* b) {
+                 return a->id() < b->id();
+               });
   return live_buffers_vector;
 }
 
diff --git a/tensorflow/compiler/xla/service/call_graph_test.cc b/tensorflow/compiler/xla/service/call_graph_test.cc
index 7924134200..5de724f892 100644
--- a/tensorflow/compiler/xla/service/call_graph_test.cc
+++ b/tensorflow/compiler/xla/service/call_graph_test.cc
@@ -384,7 +384,7 @@ TEST_F(CallGraphTest, ComplexGraph) {
 
   // Verify visitation order of some computations in the graph.
   auto index_of = [&visited](const HloComputation* comp) {
-    auto it = std::find(visited.begin(), visited.end(), comp);
+    auto it = absl::c_find(visited, comp);
     EXPECT_NE(it, visited.end());
     return std::distance(visited.begin(), it);
   };
diff --git a/tensorflow/compiler/xla/service/computation_layout.cc b/tensorflow/compiler/xla/service/computation_layout.cc
index efc893818d..92d1ca4ba5 100644
--- a/tensorflow/compiler/xla/service/computation_layout.cc
+++ b/tensorflow/compiler/xla/service/computation_layout.cc
@@ -42,8 +42,8 @@ void ComputationLayout::SetToDefaultLayout() {
 }
 
 bool ComputationLayout::LayoutIsSet() const {
-  return std::all_of(parameter_layouts_.begin(), parameter_layouts_.end(),
-                     [](const ShapeLayout& s) { return s.LayoutIsSet(); }) &&
+  return absl::c_all_of(parameter_layouts_,
+                        [](const ShapeLayout& s) { return s.LayoutIsSet(); }) &&
          result_layout_.LayoutIsSet();
 }
 
diff --git a/tensorflow/compiler/xla/service/copy_insertion.cc b/tensorflow/compiler/xla/service/copy_insertion.cc
index e0165d557d..2b837901f0 100644
--- a/tensorflow/compiler/xla/service/copy_insertion.cc
+++ b/tensorflow/compiler/xla/service/copy_insertion.cc
@@ -539,10 +539,9 @@ class CopyRemover {
         }
 
         std::vector<const HloValue*> values = buffer.values();
-        std::sort(values.begin(), values.end(),
-                  [this](const HloValue* a, const HloValue* b) {
-                    return ordering_.IsDefinedBefore(*a, *b);
-                  });
+        absl::c_sort(values, [this](const HloValue* a, const HloValue* b) {
+          return ordering_.IsDefinedBefore(*a, *b);
+        });
 
         // Create a list containing all of the values in the buffer.
         AddValueList(values, &value_to_node);
@@ -842,12 +841,11 @@ class CopyRemover {
       copy_value_node->next->prev = operand_node;
 
       // Patch up uses. Remove use of copy from operand_node uses.
-      auto it =
-          std::find_if(operand_node->uses.begin(), operand_node->uses.end(),
-                       [copy_value_node](const HloUse* use) {
-                         return use->instruction ==
-                                copy_value_node->value->defining_instruction();
-                       });
+      auto it = absl::c_find_if(
+          operand_node->uses, [copy_value_node](const HloUse* use) {
+            return use->instruction ==
+                   copy_value_node->value->defining_instruction();
+          });
       CHECK(it != operand_node->uses.end());
       operand_node->uses.erase(it);
 
diff --git a/tensorflow/compiler/xla/service/cpu/disassembler.cc b/tensorflow/compiler/xla/service/cpu/disassembler.cc
index 3ae64142cd..c3c6847b7b 100644
--- a/tensorflow/compiler/xla/service/cpu/disassembler.cc
+++ b/tensorflow/compiler/xla/service/cpu/disassembler.cc
@@ -77,17 +77,16 @@ StatusOr<DisassemblerResult> Disassembler::DisassembleObjectFile(
     }
 
     // Sort the symbols in increasing address order.
-    std::sort(
-        symbols.begin(), symbols.end(),
-        [](const llvm::object::SymbolRef& a, const llvm::object::SymbolRef& b) {
-          // getAddress returns a Expected object. Assert there is no error
-          // before extracting the address.
-          llvm::Expected<uint64_t> a_address_or_error = a.getAddress();
-          CHECK(a_address_or_error);
-          llvm::Expected<uint64_t> b_address_or_error = b.getAddress();
-          CHECK(b_address_or_error);
-          return a_address_or_error.get() < b_address_or_error.get();
-        });
+    absl::c_sort(symbols, [](const llvm::object::SymbolRef& a,
+                             const llvm::object::SymbolRef& b) {
+      // getAddress returns a Expected object. Assert there is no error
+      // before extracting the address.
+      llvm::Expected<uint64_t> a_address_or_error = a.getAddress();
+      CHECK(a_address_or_error);
+      llvm::Expected<uint64_t> b_address_or_error = b.getAddress();
+      CHECK(b_address_or_error);
+      return a_address_or_error.get() < b_address_or_error.get();
+    });
 
     // Construct ArrayRef pointing to section contents.
     llvm::StringRef section_content_string;
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index e4760aedf1..b352848824 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -1709,10 +1709,8 @@ StatusOr<bool> IrEmitter::EmitVectorizedReduce(
       vectorization_factor_in_bytes /
       ShapeUtil::ByteSizeOfPrimitiveType(reduce->shape().element_type());
 
-  bool is_reduction_over_minor_dimension =
-      std::find(dimensions.begin(), dimensions.end(),
-                LayoutUtil::Minor(arg->shape().layout(), 0)) !=
-      dimensions.end();
+  bool is_reduction_over_minor_dimension = absl::c_linear_search(
+      dimensions, LayoutUtil::Minor(arg->shape().layout(), 0));
 
   unsigned element_alignment = tensorflow::MathUtil::GCD<unsigned>(
       ShapeUtil::ByteSizeOfPrimitiveType(reduce->shape().element_type()),
@@ -2401,8 +2399,7 @@ StatusOr<bool> IrEmitter::EmitFastConcatenate(
   int64 concat_dim = concatenate->dimensions(0);
   const Layout& output_layout = output_shape.layout();
   auto output_min2maj = LayoutUtil::MinorToMajor(output_layout);
-  auto concat_dim_layout_itr =
-      std::find(output_min2maj.begin(), output_min2maj.end(), concat_dim);
+  auto concat_dim_layout_itr = absl::c_find(output_min2maj, concat_dim);
 
   std::vector<int64> inner_dims(output_min2maj.begin(), concat_dim_layout_itr);
   std::vector<int64> outer_dims(std::next(concat_dim_layout_itr),
@@ -2956,8 +2953,7 @@ Status IrEmitter::ElementTypesSameAndSupported(
 
   TF_RET_CHECK(!operands.empty());
   PrimitiveType primitive_type = operands[0]->shape().element_type();
-  if (std::find(supported_types.begin(), supported_types.end(),
-                primitive_type) == supported_types.end()) {
+  if (!absl::c_linear_search(supported_types, primitive_type)) {
     return Unimplemented("unsupported operand type %s in op %s",
                          PrimitiveType_Name(primitive_type),
                          HloOpcodeString(instruction.opcode()));
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc b/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc
index 5d25a032a9..caccb18899 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc
@@ -154,20 +154,17 @@ bool IsReductionToVector(const HloInstruction& reduce) {
   const HloInstruction* input = reduce.operand(0);
   std::vector<int64> dims_to_keep;
   for (int64 dim = 0; dim < input->shape().dimensions().size(); ++dim) {
-    if (!std::count(reduce.dimensions().begin(), reduce.dimensions().end(),
-                    dim)) {
+    if (!absl::c_linear_search(reduce.dimensions(), dim)) {
       dims_to_keep.push_back(dim);
     }
   }
   return LayoutUtil::AreDimensionsConsecutive(input->shape().layout(),
                                               dims_to_keep) &&
-         ShapeUtil::Equal(reduce.shape(), ShapeUtil::FilterDimensions(
-                                              [&dims_to_keep](int64 dim) {
-                                                return std::count(
-                                                    dims_to_keep.begin(),
-                                                    dims_to_keep.end(), dim);
-                                              },
-                                              input->shape()));
+         ShapeUtil::Equal(
+             reduce.shape(),
+             ShapeUtil::FilterDimensions(
+                 [&](int64 dim) { return absl::c_count(dims_to_keep, dim); },
+                 input->shape()));
 }
 
 // This emits a device-side call to
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index 063d493d90..923b3d5945 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -1506,10 +1506,10 @@ std::unique_ptr<KernelThunk> IrEmitterUnnested::BuildKernelThunk(
                     return !allocation->is_constant();
                   });
 
-  std::sort(non_constant_buffers.begin(), non_constant_buffers.end(),
-            [](const BufferAllocation* a, const BufferAllocation* b) {
-              return a->index() < b->index();
-            });
+  absl::c_sort(non_constant_buffers,
+               [](const BufferAllocation* a, const BufferAllocation* b) {
+                 return a->index() < b->index();
+               });
 
   llvm::Function* kernel = BuildKernelPrototype(*inst, non_constant_buffers);
 
diff --git a/tensorflow/compiler/xla/service/heap_simulator.cc b/tensorflow/compiler/xla/service/heap_simulator.cc
index 70ae7078ae..4fca981c6a 100644
--- a/tensorflow/compiler/xla/service/heap_simulator.cc
+++ b/tensorflow/compiler/xla/service/heap_simulator.cc
@@ -225,10 +225,10 @@ Status HeapSimulator::RunComputation(
       }
     }
     // Sort to get a deterministic iteration order.
-    std::sort(operand_buffers_to_free.begin(), operand_buffers_to_free.end(),
-              [](const BufferValue* x, const BufferValue* y) {
-                return x->id() < y->id();
-              });
+    absl::c_sort(operand_buffers_to_free,
+                 [](const BufferValue* x, const BufferValue* y) {
+                   return x->id() < y->id();
+                 });
 
     // Allocate buffers defined by this instruction.  This is the latest point
     // that we can allocate; right before the buffer is first used.  This must
@@ -335,10 +335,9 @@ Status HeapSimulator::RunComputation(
     to_free.push_back(buffer);
   }
 
-  std::sort(to_free.begin(), to_free.end(),
-            [](const BufferValue* x, const BufferValue* y) {
-              return x->id() < y->id();
-            });
+  absl::c_sort(to_free, [](const BufferValue* x, const BufferValue* y) {
+    return x->id() < y->id();
+  });
   for (const BufferValue* buffer : to_free) {
     VLOG(3) << "Freeing pending: " << buffer->ToString();
     Free(buffer, root);
@@ -596,7 +595,7 @@ void DecreasingSizeRunsHeap::CallAndDrainRun() {
   }
 
   // Call ops in the run sorted by decreasing size, breaking ties by buffer id.
-  std::sort(run_.begin(), run_.end(), [](const Op& a, const Op& b) {
+  absl::c_sort(run_, [](const Op& a, const Op& b) {
     if (a.size != b.size) {
       return a.size > b.size;
     }
@@ -866,23 +865,23 @@ HeapSimulator::Result GlobalDecreasingSizeBestFitHeap::Finish() {
   for (auto& entry : buffer_intervals_) {
     sorted_buffer_intervals.push_back(entry.second);
   }
-  std::sort(sorted_buffer_intervals.begin(), sorted_buffer_intervals.end(),
-            [](const BufferInterval& x, const BufferInterval& y) {
-              if (x.size != y.size) {
-                return x.size > y.size;
-              }
-              if (x.end - x.start != y.end - y.start) {
-                return x.end - x.start > y.end - y.start;
-              }
-              return x.buffer->id() < y.buffer->id();
-            });
+  absl::c_sort(sorted_buffer_intervals,
+               [](const BufferInterval& x, const BufferInterval& y) {
+                 if (x.size != y.size) {
+                   return x.size > y.size;
+                 }
+                 if (x.end - x.start != y.end - y.start) {
+                   return x.end - x.start > y.end - y.start;
+                 }
+                 return x.buffer->id() < y.buffer->id();
+               });
 
   BufferIntervalTree interval_tree(sorted_buffer_intervals.size());
   for (auto& buffer_interval : sorted_buffer_intervals) {
     auto chunks_overlapping_in_time = interval_tree.ChunksOverlappingInTime(
         buffer_interval.start, buffer_interval.end);
-    std::sort(
-        chunks_overlapping_in_time.begin(), chunks_overlapping_in_time.end(),
+    absl::c_sort(
+        chunks_overlapping_in_time,
         [](const Chunk& x, const Chunk& y) { return x.offset < y.offset; });
 
     // Find the minimum free chunk that can hold this buffer.
diff --git a/tensorflow/compiler/xla/service/hlo_alias_analysis.cc b/tensorflow/compiler/xla/service/hlo_alias_analysis.cc
index 68094d8907..4c46ea595e 100644
--- a/tensorflow/compiler/xla/service/hlo_alias_analysis.cc
+++ b/tensorflow/compiler/xla/service/hlo_alias_analysis.cc
@@ -117,7 +117,7 @@ class BufferValueMap {
     for (const auto& pair : buffers_) {
       buffer_numbers.push_back(pair.first);
     }
-    std::sort(buffer_numbers.begin(), buffer_numbers.end());
+    absl::c_sort(buffer_numbers);
     return buffer_numbers;
   }
 
@@ -319,7 +319,7 @@ class BufferValueMap {
     ComputeWhileAliasedBuffers(value, &aliased_buffers);
     ComputeConditionalAliasedBuffers(value, &aliased_buffers);
     // Uniquify aliased buffers.
-    std::sort(aliased_buffers.begin(), aliased_buffers.end());
+    absl::c_sort(aliased_buffers);
     aliased_buffers.erase(
         std::unique(aliased_buffers.begin(), aliased_buffers.end()),
         aliased_buffers.end());
@@ -367,7 +367,7 @@ std::vector<const HloBuffer*> HloAliasAnalysis::ComputeBuffersAt(
   }
 
   // Sort and uniquify vector before returning.
-  std::sort(buffers.begin(), buffers.end(), HloBuffer::IdLessThan);
+  absl::c_sort(buffers, HloBuffer::IdLessThan);
   buffers.erase(std::unique(buffers.begin(), buffers.end()), buffers.end());
 
   return buffers;
@@ -430,8 +430,7 @@ Status HloAliasAnalysis::Verify() const {
   for (const auto& pair : value_to_buffer_) {
     const HloValue* value = pair.first;
     const HloBuffer& buffer = *pair.second;
-    TF_RET_CHECK(std::find(buffer.values().begin(), buffer.values().end(),
-                           value) != buffer.values().end());
+    TF_RET_CHECK(absl::c_linear_search(buffer.values(), value));
   }
 
   for (HloBuffer::Id id = 0; id < buffers_.size(); ++id) {
@@ -515,7 +514,7 @@ StatusOr<std::unique_ptr<HloAliasAnalysis>> HloAliasAnalysis::Run(
     auto& value_set = buffer_map.GetValuesInBuffer(buffer_number);
     std::vector<const HloValue*> sorted_values(value_set.begin(),
                                                value_set.end());
-    std::sort(sorted_values.begin(), sorted_values.end(), HloValue::IdLessThan);
+    absl::c_sort(sorted_values, HloValue::IdLessThan);
     alias_analysis->buffers_.emplace_back(next_id++, sorted_values);
     for (const HloValue* value : sorted_values) {
       alias_analysis->value_to_buffer_[value] =
@@ -547,16 +546,15 @@ bool HloAliasAnalysis::HasLiveRangeInterference(
     // tie-break using value ID. The tie-break is necessary because we need a
     // strict weak order for std::sort.
     std::vector<const HloValue*> values = buffer.values();
-    std::sort(values.begin(), values.end(),
-              [&ordering](const HloValue* a, const HloValue* b) {
-                if (ordering.IsDefinedBefore(*a, *b)) {
-                  return true;
-                } else if (ordering.IsDefinedBefore(*b, *a)) {
-                  return false;
-                } else {
-                  return a->id() < b->id();
-                }
-              });
+    absl::c_sort(values, [&ordering](const HloValue* a, const HloValue* b) {
+      if (ordering.IsDefinedBefore(*a, *b)) {
+        return true;
+      } else if (ordering.IsDefinedBefore(*b, *a)) {
+        return false;
+      } else {
+        return a->id() < b->id();
+      }
+    });
 
     // Walk through the ordered vector of values. First verify that the values
     // are totally ordered with respect to 'ordering', then check that no
diff --git a/tensorflow/compiler/xla/service/hlo_buffer.cc b/tensorflow/compiler/xla/service/hlo_buffer.cc
index 9c3aa0e64d..32e48651b3 100644
--- a/tensorflow/compiler/xla/service/hlo_buffer.cc
+++ b/tensorflow/compiler/xla/service/hlo_buffer.cc
@@ -49,7 +49,7 @@ std::vector<HloPosition> HloBuffer::ComputePositions() const {
                      value->positions().end());
   }
   // Remove duplicates and sort positions.
-  std::sort(positions.begin(), positions.end());
+  absl::c_sort(positions);
   positions.erase(std::unique(positions.begin(), positions.end()),
                   positions.end());
   return positions;
diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc
index 10f0cf3a34..9e15722633 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation.cc
@@ -531,11 +531,10 @@ HloComputation::CreateFromProto(
   HloInstruction* root = instruction_map.at(proto.root_id());
 
   // Sort the instructions in the proto id's order.
-  std::sort(instructions.begin(), instructions.end(),
-            [&](const std::unique_ptr<HloInstruction>& a,
-                const std::unique_ptr<HloInstruction>& b) {
-              return to_proto_id[a.get()] < to_proto_id[b.get()];
-            });
+  absl::c_sort(instructions, [&](const std::unique_ptr<HloInstruction>& a,
+                                 const std::unique_ptr<HloInstruction>& b) {
+    return to_proto_id[a.get()] < to_proto_id[b.get()];
+  });
 
   TF_RETURN_IF_ERROR([&]() -> Status {
     std::vector<bool> parameters_seen(parameter_count);
@@ -800,8 +799,7 @@ Status HloComputation::AcceptOrdered(
     absl::Span<HloInstruction* const> order) const {
   VLOG(3) << "Accepting visitor with order.";
   for (HloInstruction* root : CollectUnreachableRoots()) {
-    TF_RET_CHECK(std::find(order.begin(), order.end(), root) != order.end())
-        << root->ToString();
+    TF_RET_CHECK(absl::c_linear_search(order, root)) << root->ToString();
   }
   TF_RET_CHECK(order.size() == instruction_count());
   absl::flat_hash_set<const HloInstruction*> visited;
diff --git a/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc b/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc
index 0db452b85f..3144a84805 100644
--- a/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc
+++ b/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc
@@ -256,7 +256,7 @@ bool HloDataflowAnalysis::Phi(
         input_value_ids.push_back(value->id());
       }
     }
-    std::sort(input_value_ids.begin(), input_value_ids.end());
+    absl::c_sort(input_value_ids);
     input_value_ids.erase(
         std::unique(input_value_ids.begin(), input_value_ids.end()),
         input_value_ids.end());
@@ -271,8 +271,7 @@ bool HloDataflowAnalysis::Phi(
     if (current_value_defined_here) {
       VLOG(5) << "current_value_defined_here: " << current_value->ToString();
       CHECK(current_value->is_phi());
-      auto it = std::find(input_value_ids.begin(), input_value_ids.end(),
-                          current_value->id());
+      auto it = absl::c_find(input_value_ids, current_value->id());
       if (it != input_value_ids.end()) {
         input_value_ids.erase(it);
       }
@@ -921,8 +920,7 @@ StatusOr<std::unique_ptr<HloDataflowAnalysis>> HloDataflowAnalysis::Run(
   for (auto& pair : dataflow_analysis->values_) {
     dataflow_analysis->values_vector_.push_back(&pair.second);
   }
-  std::sort(dataflow_analysis->values_vector_.begin(),
-            dataflow_analysis->values_vector_.end(), HloValue::IdLessThan);
+  absl::c_sort(dataflow_analysis->values_vector_, HloValue::IdLessThan);
 
   TF_DCHECK_OK(dataflow_analysis->Verify());
 
@@ -937,9 +935,7 @@ Status HloDataflowAnalysis::Verify() const {
   for (const HloValue* value : values()) {
     for (const HloPosition& position : value->positions()) {
       const HloValueSet& value_set = GetValueSet(position);
-      TF_RET_CHECK(std::find(value_set.values().begin(),
-                             value_set.values().end(),
-                             value) != value_set.values().end())
+      TF_RET_CHECK(absl::c_linear_search(value_set.values(), value))
           << "Value set at position " << position << " does not contain value "
           << value->ToShortString();
     }
@@ -954,9 +950,7 @@ Status HloDataflowAnalysis::Verify() const {
         const HloValueSet& value_set = pair.second;
         const HloPosition position{instruction, index};
         for (const HloValue* value : value_set.values()) {
-          TF_RET_CHECK(std::find(value->positions().begin(),
-                                 value->positions().end(),
-                                 position) != value->positions().end())
+          TF_RET_CHECK(absl::c_linear_search(value->positions(), position))
               << "Value set at position " << position
               << " unexpectedly contains value " << value->ToShortString();
         }
@@ -1041,11 +1035,10 @@ bool HloDataflowAnalysis::CanShareOperandBufferWithUser(
       // Check if one operand of kAdd fused root is kDot or kConvolution.
       auto* add = user->fused_expression_root();
       auto add_operand_it =
-          std::find_if(add->operands().begin(), add->operands().end(),
-                       [&](HloInstruction* operand) {
-                         return operand->opcode() == HloOpcode::kConvolution ||
-                                operand->opcode() == HloOpcode::kDot;
-                       });
+          absl::c_find_if(add->operands(), [&](HloInstruction* operand) {
+            return operand->opcode() == HloOpcode::kConvolution ||
+                   operand->opcode() == HloOpcode::kDot;
+          });
       if (add_operand_it == add->operands().end()) {
         return false;
       }
@@ -1100,16 +1093,15 @@ bool HloDataflowAnalysis::CanShareOperandBufferWithUser(
     // *) The root instruction of the called computation is element-wise on
     //    'operand'.
     const bool found_caller_use =
-        std::find_if(uses.begin(), uses.end(), [user](const HloUse& use) {
+        absl::c_find_if(uses, [user](const HloUse& use) {
           return use.instruction == user;
         }) != uses.end();
     auto* callee_root = user->to_apply()->root_instruction();
     const bool found_elementwise_callee_use =
-        std::find_if(
-            uses.begin(), uses.end(), [callee_root](const HloUse& use) {
-              return use.instruction == callee_root &&
-                     callee_root->IsElementwiseOnOperand(use.operand_number);
-            }) != uses.end();
+        absl::c_find_if(uses, [callee_root](const HloUse& use) {
+          return use.instruction == callee_root &&
+                 callee_root->IsElementwiseOnOperand(use.operand_number);
+        }) != uses.end();
     return uses.size() == 2 && found_caller_use && found_elementwise_callee_use;
   }
 
diff --git a/tensorflow/compiler/xla/service/hlo_dce_test.cc b/tensorflow/compiler/xla/service/hlo_dce_test.cc
index 1fa4259a3e..b5d72b386f 100644
--- a/tensorflow/compiler/xla/service/hlo_dce_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_dce_test.cc
@@ -43,9 +43,7 @@ class HloDceTest : public HloTestBase {
   // Returns whether the given instruction exists in the given computation.
   bool HasInstruction(const HloComputation& computation,
                       const HloInstruction* instruction) {
-    return std::find(computation.instructions().begin(),
-                     computation.instructions().end(),
-                     instruction) != computation.instructions().end();
+    return absl::c_linear_search(computation.instructions(), instruction);
   }
 };
 
diff --git a/tensorflow/compiler/xla/service/hlo_domain_map.cc b/tensorflow/compiler/xla/service/hlo_domain_map.cc
index c6d02f9f67..7cdb7f6bdf 100644
--- a/tensorflow/compiler/xla/service/hlo_domain_map.cc
+++ b/tensorflow/compiler/xla/service/hlo_domain_map.cc
@@ -230,10 +230,10 @@ HloDomainMap::MakeNonDomainInstructions(
     }
   }
   // sort instructions according to instructions_order
-  std::sort(instructions.begin(), instructions.end(),
-            [&instructions_order](HloInstruction* a, HloInstruction* b) {
-              return instructions_order.at(a) < instructions_order.at(b);
-            });
+  absl::c_sort(instructions,
+               [&instructions_order](HloInstruction* a, HloInstruction* b) {
+                 return instructions_order.at(a) < instructions_order.at(b);
+               });
   return instructions;
 }
 
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index 9754b8906b..3a97b56c66 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -1248,8 +1248,7 @@ StatusOr<Literal> EvaluateSortInternal(HloInstruction* sort,
         // Extract a slice from the keys and values literals that correspond to
         // exactly the row in dimension 'sort_dim'.
         std::vector<int64> limit_indices(indices.begin(), indices.end());
-        std::for_each(limit_indices.begin(), limit_indices.end(),
-                      [](int64& index) { ++index; });
+        absl::c_for_each(limit_indices, [](int64& index) { ++index; });
         limit_indices[sort_dim] = sort_dim_elements;
         TF_ASSIGN_OR_RETURN(auto keys_to_sort,
                             keys_literal.Slice(indices, limit_indices)
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
index cf3509d0aa..cc5f6cc682 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
@@ -1673,8 +1673,7 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
           // Extract a slice from the literal that corresponds to exactly the
           // row in dimension 'sort_dim'.
           std::vector<int64> limit_indices(indices.begin(), indices.end());
-          std::for_each(limit_indices.begin(), limit_indices.end(),
-                        [](int64& index) { ++index; });
+          absl::c_for_each(limit_indices, [](int64& index) { ++index; });
           limit_indices[sort_dim] = sort_dim_elements;
           TF_ASSIGN_OR_RETURN(auto row_to_sort,
                               keys_literal.Slice(indices, limit_indices)
diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
index 22bcb2030b..4c7f5e9e7d 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
@@ -561,8 +561,8 @@ bool HloDotDumper::ShouldShowSubcomputation(const HloComputation* subcomp) {
   }
 
   // Show the subcomputation if we're showing any of its members.
-  return std::any_of(
-      subcomp->instructions().begin(), subcomp->instructions().end(),
+  return absl::c_any_of(
+      subcomp->instructions(),
       [&](const HloInstruction* instr) { return filter_.Show(instr); });
 }
 
@@ -735,15 +735,14 @@ bool HloDotDumper::ShouldMergeIntoUsers(const HloInstruction* instr) const {
   const int kMinUsersToOmit = 3;
   return instr->opcode() == HloOpcode::kParameter && instr->shape().IsTuple() &&
          !instr->IsFused() &&
-         std::count_if(instr->users().begin(), instr->users().end(),
-                       [&](const HloInstruction* user) {
-                         return filter_.Show(user);
-                       }) > kMinUsersToOmit &&
-         std::all_of(instr->users().begin(), instr->users().end(),
-                     [&](const HloInstruction* user) {
-                       return !filter_.Show(user) ||
-                              user->opcode() == HloOpcode::kGetTupleElement;
-                     });
+         absl::c_count_if(instr->users(),
+                          [&](const HloInstruction* user) {
+                            return filter_.Show(user);
+                          }) > kMinUsersToOmit &&
+         absl::c_all_of(instr->users(), [&](const HloInstruction* user) {
+           return !filter_.Show(user) ||
+                  user->opcode() == HloOpcode::kGetTupleElement;
+         });
 }
 
 string HloDotDumper::DumpInstruction(const HloInstruction* instr) {
@@ -900,12 +899,11 @@ ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) {
   // the same color as a parameter.  Unless the merged-in parameter is a
   // parameter to a fusion node that is bound to a constant -- these aren't
   // "real" parameters from the user's perspective.
-  if (std::any_of(instr->operands().begin(), instr->operands().end(),
-                  [&](const HloInstruction* operand) {
-                    return operand->opcode() == HloOpcode::kParameter &&
-                           ShouldMergeIntoUsers(operand) &&
-                           TryGetFusionParameterConstant(operand) == nullptr;
-                  })) {
+  if (absl::c_any_of(instr->operands(), [&](const HloInstruction* operand) {
+        return operand->opcode() == HloOpcode::kParameter &&
+               ShouldMergeIntoUsers(operand) &&
+               TryGetFusionParameterConstant(operand) == nullptr;
+      })) {
     return parameter_color;
   }
 
@@ -1355,12 +1353,11 @@ NodeFilter MakeNodeRadiusAroundFilter(const HloInstruction* root,
     NodeFilterResult& filter_result = kv.second;
     const auto& operands = instr->operands();
 
-    if (std::any_of(operands.begin(), operands.end(), is_displayed) &&
-        !std::all_of(operands.begin(), operands.end(), is_displayed)) {
+    if (absl::c_any_of(operands, is_displayed) &&
+        !absl::c_all_of(operands, is_displayed)) {
       // Mark nodes with some operands omitted appropriately.
       filter_result = kSomeOperandsOmitted;
-    } else if (!operands.empty() &&
-               std::none_of(operands.begin(), operands.end(), is_displayed)) {
+    } else if (!operands.empty() && absl::c_none_of(operands, is_displayed)) {
       // Mark nodes with *all* operands omitted appropriately.
       filter_result = kOmitNodeOperands;
     }
@@ -1368,8 +1365,7 @@ NodeFilter MakeNodeRadiusAroundFilter(const HloInstruction* root,
     // Promote nodes with type kSomeUsersOmitted to kNormalNode if all of their
     // users made it into the graph.
     if (filter_result == kSomeUsersOmitted &&
-        std::all_of(instr->users().begin(), instr->users().end(),
-                    is_displayed)) {
+        absl::c_all_of(instr->users(), is_displayed)) {
       filter_result = kNormalNode;
     }
   }
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index 66bc73740f..029d170317 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -83,15 +83,14 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
     return computation_map.at(proto.called_computation_ids(index));
   };
 
-  TF_RET_CHECK(std::all_of(
-      proto.operand_ids().begin(), proto.operand_ids().end(),
-      [&instruction_map](int64 id) { return instruction_map.contains(id); }))
+  TF_RET_CHECK(
+      absl::c_all_of(proto.operand_ids(),
+                     [&](int64 id) { return instruction_map.contains(id); }))
       << proto.name() << " instruction contains invalid operand id(s)";
 
-  TF_RET_CHECK(std::all_of(
-      proto.called_computation_ids().begin(),
-      proto.called_computation_ids().end(),
-      [&computation_map](int64 id) { return computation_map.contains(id); }))
+  TF_RET_CHECK(
+      absl::c_all_of(proto.called_computation_ids(),
+                     [&](int64 id) { return computation_map.contains(id); }))
       << proto.name() << " instruction references invalid computation id(s)";
 
   Shape shape(proto.shape());
@@ -1599,12 +1598,10 @@ HloInstruction::InstructionVector HloInstruction::unique_operands() const {
 
 Status HloInstruction::AddControlDependencyTo(HloInstruction* instruction) {
   TF_RET_CHECK(instruction->parent() == parent());
-  if (std::find(control_successors_.begin(), control_successors_.end(),
-                instruction) == control_successors_.end()) {
+  if (!absl::c_linear_search(control_successors_, instruction)) {
     control_successors_.push_back(instruction);
-    TF_RET_CHECK(std::find(instruction->control_predecessors_.begin(),
-                           instruction->control_predecessors_.end(),
-                           this) == instruction->control_predecessors_.end());
+    TF_RET_CHECK(
+        !absl::c_linear_search(instruction->control_predecessors_, this));
     instruction->control_predecessors_.push_back(this);
   }
   return Status::OK();
@@ -1853,7 +1850,7 @@ void HloInstruction::RemoveUser(HloInstruction* user) {
   user_set_.erase(set_it);
   // This is linear in the number of the users, but a vector provides a stable
   // iteration order and much faster traversal.
-  auto vec_it = std::find(users_.begin(), users_.end(), user);
+  auto vec_it = absl::c_find(users_, user);
   CHECK(vec_it != users_.end());
   users_.erase(vec_it);
 }
@@ -1871,8 +1868,7 @@ Status HloInstruction::ReplaceUseWith(HloInstruction* user,
 
   RemoveUser(user);
 
-  TF_RET_CHECK(
-      std::count(user->operands_.begin(), user->operands_.end(), this) >= 0);
+  TF_RET_CHECK(absl::c_count(user->operands_, this) >= 0);
   std::replace(user->operands_.begin(), user->operands_.end(), this,
                new_producer);
   new_producer->AddUser(user);
@@ -1907,8 +1903,7 @@ Status HloInstruction::ReplaceOperandWithDifferentShape(
   VLOG(3) << "Replacing operand " << operand_num << " of " << name() << " with "
           << new_operand->name() << ", was " << old_operand->name();
 
-  if (std::find(operands_.begin(), operands_.end(), old_operand) ==
-      operands_.end()) {
+  if (!absl::c_linear_search(operands_, old_operand)) {
     old_operand->RemoveUser(this);
   }
   new_operand->AddUser(this);
@@ -2945,10 +2940,10 @@ StatusOr<HloInstruction::FusionKind> StringToFusionKind(
 
 string PaddingConfigToString(const PaddingConfig& padding) {
   bool has_interior_padding =
-      std::any_of(padding.dimensions().begin(), padding.dimensions().end(),
-                  [](const PaddingConfig::PaddingConfigDimension& dim) {
-                    return dim.interior_padding() != 0;
-                  });
+      absl::c_any_of(padding.dimensions(),
+                     [](const PaddingConfig::PaddingConfigDimension& dim) {
+                       return dim.interior_padding() != 0;
+                     });
   return StrJoin(
       padding.dimensions(), "x",
       [&](string* out, const PaddingConfig::PaddingConfigDimension& dim) {
diff --git a/tensorflow/compiler/xla/service/hlo_instructions.cc b/tensorflow/compiler/xla/service/hlo_instructions.cc
index 7170dd7d81..c24bf41ff8 100644
--- a/tensorflow/compiler/xla/service/hlo_instructions.cc
+++ b/tensorflow/compiler/xla/service/hlo_instructions.cc
@@ -42,11 +42,9 @@ using absl::StrJoin;
 bool IsInstructionElementwiseOnOperand(const HloInstruction* instruction,
                                        const HloInstruction* operand) {
   std::vector<int64> operand_indices = instruction->OperandIndices(operand);
-  return std::all_of(
-      operand_indices.begin(), operand_indices.end(),
-      [instruction](int64 operand_index) {
-        return instruction->IsElementwiseOnOperand(operand_index);
-      });
+  return absl::c_all_of(operand_indices, [instruction](int64 operand_index) {
+    return instruction->IsElementwiseOnOperand(operand_index);
+  });
 }
 
 string PrecisionConfigToString(const PrecisionConfig& precision_config) {
@@ -814,8 +812,7 @@ std::vector<string> HloSliceInstruction::ExtraAttributesToStringImpl(
   std::vector<string> bounds;
   bounds.reserve(slice_starts_.size());
   const bool omit_stride =
-      std::all_of(slice_strides_.begin(), slice_strides_.end(),
-                  [](int64 stride) { return stride == 1; });
+      absl::c_all_of(slice_strides_, [](int64 stride) { return stride == 1; });
   for (int i = 0; i < slice_starts_.size(); ++i) {
     string stride_str = omit_stride ? "" : StrCat(":", slice_strides_[i]);
     bounds.push_back(
@@ -1051,8 +1048,7 @@ HloInstruction* HloFusionInstruction::AddFusionOperand(
 
 void HloFusionInstruction::MergeFusionInstruction(
     HloFusionInstruction* instruction_to_merge) {
-  CHECK(std::find(operands().begin(), operands().end(), instruction_to_merge) !=
-        operands().end());
+  CHECK(absl::c_linear_search(operands(), instruction_to_merge));
   // Clone the instruction from which to merge fused instructions.
   std::unique_ptr<HloInstruction> cloned = instruction_to_merge->Clone();
   HloFusionInstruction* cloned_fusion =
@@ -1219,8 +1215,8 @@ HloInstruction* HloFusionInstruction::CloneAndFuseInternal(
     // corresponding fused parameter instruction. Renumber parameters as
     // necessary to make parameter numbers consistent with their index in the
     // fused_parameter_ vector.
-    bool in_operand_list = std::find(operands().begin(), operands().end(),
-                                     instruction_to_fuse) != operands().end();
+    bool in_operand_list =
+        absl::c_linear_search(operands(), instruction_to_fuse);
     CHECK(add_output || in_operand_list);
     if (instruction_to_fuse->opcode() == HloOpcode::kTuple) {
       // We assume all uses of a kTuple operation are GTE ops, not another
diff --git a/tensorflow/compiler/xla/service/hlo_module.cc b/tensorflow/compiler/xla/service/hlo_module.cc
index ed78189fed..258f918f47 100644
--- a/tensorflow/compiler/xla/service/hlo_module.cc
+++ b/tensorflow/compiler/xla/service/hlo_module.cc
@@ -107,11 +107,10 @@ HloComputation* HloModule::AddEntryComputation(
 }
 
 Status HloModule::RemoveEmbeddedComputation(HloComputation* to_remove) {
-  auto it =
-      std::find_if(computations_.begin(), computations_.end(),
-                   [&to_remove](const std::unique_ptr<HloComputation>& comp) {
-                     return comp.get() == to_remove;
-                   });
+  auto it = absl::c_find_if(
+      computations_, [&to_remove](const std::unique_ptr<HloComputation>& comp) {
+        return comp.get() == to_remove;
+      });
   TF_RET_CHECK(it->get() == to_remove);
   computations_.erase(it);
   return Status::OK();
@@ -304,11 +303,10 @@ StatusOr<std::unique_ptr<HloModule>> HloModule::CreateFromProto(
   auto module = absl::make_unique<HloModule>(proto.name(), module_config);
 
   // Sort the computations in the proto id's order.
-  std::sort(computations.begin(), computations.end(),
-            [&](const std::unique_ptr<HloComputation>& a,
-                const std::unique_ptr<HloComputation>& b) {
-              return to_proto_id[a.get()] < to_proto_id[b.get()];
-            });
+  absl::c_sort(computations, [&](const std::unique_ptr<HloComputation>& a,
+                                 const std::unique_ptr<HloComputation>& b) {
+    return to_proto_id[a.get()] < to_proto_id[b.get()];
+  });
 
   // Add sorted computations to the module.
   for (auto& computation : computations) {
diff --git a/tensorflow/compiler/xla/service/hlo_module_dce_test.cc b/tensorflow/compiler/xla/service/hlo_module_dce_test.cc
index e535b7d749..f6e2866204 100644
--- a/tensorflow/compiler/xla/service/hlo_module_dce_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_module_dce_test.cc
@@ -38,9 +38,7 @@ class HloModuleDceTest : public HloTestBase {
   // Returns whether the given instruction exists in the given computation.
   bool HasInstruction(const HloComputation& computation,
                       const HloInstruction* instruction) {
-    return std::find(computation.instructions().begin(),
-                     computation.instructions().end(),
-                     instruction) != computation.instructions().end();
+    return absl::c_linear_search(computation.instructions(), instruction);
   }
 
   // Returns whether the while instruction with name 'while_name' in
diff --git a/tensorflow/compiler/xla/service/hlo_parser.cc b/tensorflow/compiler/xla/service/hlo_parser.cc
index 730d39500b..638396308c 100644
--- a/tensorflow/compiler/xla/service/hlo_parser.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser.cc
@@ -2746,7 +2746,7 @@ bool HloParser::ParseConvolutionDimensionNumbers(
   }
 
   auto is_unique = [](string str) -> bool {
-    std::sort(str.begin(), str.end());
+    absl::c_sort(str);
     return std::unique(str.begin(), str.end()) == str.end();
   };
 
diff --git a/tensorflow/compiler/xla/service/hlo_profile_printer.cc b/tensorflow/compiler/xla/service/hlo_profile_printer.cc
index 5eb707a957..9cc202aa9f 100644
--- a/tensorflow/compiler/xla/service/hlo_profile_printer.cc
+++ b/tensorflow/compiler/xla/service/hlo_profile_printer.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/service/hlo_profile_printer.h"
 
+#include "absl/algorithm/container.h"
 #include "absl/strings/str_cat.h"
 #include "tensorflow/compiler/xla/service/human_readable_profile_builder.h"
 
@@ -34,11 +35,10 @@ string PrintHloProfile(const HloProfilePrinterData& hlo_profile_printer_data,
   for (const HloComputationInfo& computation_info :
        hlo_profile_printer_data.computation_infos()) {
     const auto& instruction_infos = computation_info.instruction_infos();
-    bool any_instruction_profiled =
-        std::any_of(instruction_infos.begin(), instruction_infos.end(),
-                    [&](const HloInstructionInfo& instruction_info) {
-                      return counters[instruction_info.profile_index()] != 0;
-                    });
+    bool any_instruction_profiled = absl::c_any_of(
+        instruction_infos, [&](const HloInstructionInfo& instruction_info) {
+          return counters[instruction_info.profile_index()] != 0;
+        });
 
     if (!any_instruction_profiled) {
       continue;
diff --git a/tensorflow/compiler/xla/service/hlo_reachability.cc b/tensorflow/compiler/xla/service/hlo_reachability.cc
index edaa4c59e2..0fced7f15b 100644
--- a/tensorflow/compiler/xla/service/hlo_reachability.cc
+++ b/tensorflow/compiler/xla/service/hlo_reachability.cc
@@ -49,7 +49,7 @@ void HloReachabilityMap::SetReachabilityToUnionHelper(
     absl::Span<const HloInstruction* const> inputs,
     const HloInstruction* instruction, BitVector* bit_vector) {
   // If instruction is part of inputs, don't reset the bit_vector.
-  if (std::find(inputs.begin(), inputs.end(), instruction) == inputs.end()) {
+  if (!absl::c_linear_search(inputs, instruction)) {
     bit_vector->SetToZero();
   }
   bit_vector->Set(GetIndex(instruction));
diff --git a/tensorflow/compiler/xla/service/hlo_rematerialization.cc b/tensorflow/compiler/xla/service/hlo_rematerialization.cc
index ac74e2432f..9ca14ca18a 100644
--- a/tensorflow/compiler/xla/service/hlo_rematerialization.cc
+++ b/tensorflow/compiler/xla/service/hlo_rematerialization.cc
@@ -235,8 +235,7 @@ class InstructionList {
     }
 
     // Now scan forwards until we find one of the before_instructions.
-    while (std::find(before_instructions.begin(), before_instructions.end(),
-                     min_position_item) == before_instructions.end()) {
+    while (!absl::c_linear_search(before_instructions, min_position_item)) {
       min_position_item = min_position_item->next;
     }
     return InsertBefore(to_insert, min_position_item);
@@ -302,7 +301,7 @@ ItemList GetUsers(const InstructionList& instruction_list,
       // A buffer may be used by the instruction via more than one alias. For
       // example, a buffer which appears in more than one element of a tuple.
       Item* user_item = instruction_list.GetItem(user);
-      if (std::find(users.begin(), users.end(), user_item) == users.end()) {
+      if (!absl::c_linear_search(users, user_item)) {
         users.push_back(user_item);
       }
     }
@@ -456,8 +455,7 @@ class MemoryUsageTracker {
       return false;
     }
     const BufferIdList& in_progress_uses = in_progress_item_->buffers_used;
-    return std::find(in_progress_uses.begin(), in_progress_uses.end(),
-                     buffer_id) != in_progress_uses.end();
+    return absl::c_linear_search(in_progress_uses, buffer_id);
   }
 
   // Returns whether the given instruction is live at the current program
@@ -535,8 +533,7 @@ MemoryUsageTracker::MemoryUsageTracker(
         bool unused;
         for (Item* user_item : GetUsers(instruction_list_, logical_buffer,
                                         points_to_analysis, &unused)) {
-          if (std::find(buffer->users.begin(), buffer->users.end(),
-                        user_item) == buffer->users.end()) {
+          if (!absl::c_linear_search(buffer->users, user_item)) {
             buffer->users.push_back(user_item);
             buffer->unfinished_user_count++;
             user_item->buffers_used.push_back(buffer->id);
@@ -784,8 +781,7 @@ bool MemoryUsageTracker::Check() const {
 
     for (const Buffer& buffer : buffers_) {
       if (buffer.defining_instruction->instruction == instruction) {
-        CHECK(std::find(defined_buffers.begin(), defined_buffers.end(),
-                        buffer.id) != defined_buffers.end())
+        CHECK(absl::c_linear_search(defined_buffers, buffer.id))
             << "Instruction " << instruction->name()
             << " defined buffers is missing: " << buffer.ToString();
       }
@@ -808,8 +804,7 @@ bool MemoryUsageTracker::Check() const {
     int64 unfinished_uses = 0;
     for (Item* user : buffer.users) {
       const BufferIdList& used_buffers = user->buffers_used;
-      CHECK(std::find(used_buffers.begin(), used_buffers.end(), buffer.id) !=
-            used_buffers.end())
+      CHECK(absl::c_linear_search(used_buffers, buffer.id))
           << "Instruction " << user->instruction->name()
           << " used buffers is missing " << buffer.ToString();
       if (!IsFinished(user)) {
@@ -836,10 +831,10 @@ int64 RematerializationCost(const HloInstruction* instruction,
   // If none of the users of 'instruction' have been placed in the sequence (as
   // tracked by memory_tracker), then rematerialization of 'instruction' is a
   // zero-cost move of 'instruction' in the sequence.
-  if (!std::any_of(instruction->users().begin(), instruction->users().end(),
-                   [&memory_tracker](const HloInstruction* inst) {
-                     return memory_tracker.IsPlaced(inst);
-                   })) {
+  if (!absl::c_any_of(instruction->users(),
+                      [&memory_tracker](const HloInstruction* inst) {
+                        return memory_tracker.IsPlaced(inst);
+                      })) {
     return 0;
   }
 
diff --git a/tensorflow/compiler/xla/service/hlo_sharding.cc b/tensorflow/compiler/xla/service/hlo_sharding.cc
index 6c9c32f179..37cc146bd7 100644
--- a/tensorflow/compiler/xla/service/hlo_sharding.cc
+++ b/tensorflow/compiler/xla/service/hlo_sharding.cc
@@ -107,13 +107,12 @@ string HloSharding::ToString() const {
 
 bool HloSharding::UsesDevice(int64 device) const {
   if (IsTuple()) {
-    return std::any_of(
-        tuple_elements_.begin(), tuple_elements_.end(),
-        [&](const HloSharding& s) { return s.UsesDevice(device); });
+    return absl::c_any_of(tuple_elements_, [&](const HloSharding& s) {
+      return s.UsesDevice(device);
+    });
   }
   const auto& devices = tile_assignment_;
-  return replicated_ ||
-         std::find(devices.begin(), devices.end(), device) != devices.end();
+  return replicated_ || absl::c_linear_search(devices, device);
 }
 
 std::map<int64, int64> HloSharding::UsedDevices(int64* count) const {
diff --git a/tensorflow/compiler/xla/service/hlo_sharding.h b/tensorflow/compiler/xla/service/hlo_sharding.h
index 9775505f86..5789ae0998 100644
--- a/tensorflow/compiler/xla/service/hlo_sharding.h
+++ b/tensorflow/compiler/xla/service/hlo_sharding.h
@@ -101,8 +101,8 @@ class HloSharding {
     if (!IsTuple()) {
       return replicated_;
     }
-    return std::all_of(tuple_elements_.begin(), tuple_elements_.end(),
-                       [](const HloSharding& s) { return s.IsReplicated(); });
+    return absl::c_all_of(
+        tuple_elements_, [](const HloSharding& s) { return s.IsReplicated(); });
   }
 
   // Returns true if the tile size is the same as the input size.
@@ -110,8 +110,9 @@ class HloSharding {
     if (!IsTuple()) {
       return maximal_;
     }
-    return std::all_of(tuple_elements_.begin(), tuple_elements_.end(),
-                       [](const HloSharding& s) { return s.IsTileMaximal(); });
+    return absl::c_all_of(tuple_elements_, [](const HloSharding& s) {
+      return s.IsTileMaximal();
+    });
   }
 
   // Returns true if the sharding defines an operation on the given device.
diff --git a/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc b/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc
index 4ea39a7628..c1f69db74e 100644
--- a/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc
+++ b/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc
@@ -61,8 +61,7 @@ void CleanNodeName(string* name) {
   name->erase(std::remove(name->begin(), name->end(), '%'), name->end());
   const string chars_to_replace = "<>[]";
   auto pred = [&](char c) {
-    return std::find(chars_to_replace.begin(), chars_to_replace.end(), c) !=
-           chars_to_replace.end();
+    return absl::c_linear_search(chars_to_replace, c);
   };
   std::replace_if(name->begin(), name->end(), pred, '_');
 }
diff --git a/tensorflow/compiler/xla/service/hlo_value.cc b/tensorflow/compiler/xla/service/hlo_value.cc
index d409df06be..218b33b2ac 100644
--- a/tensorflow/compiler/xla/service/hlo_value.cc
+++ b/tensorflow/compiler/xla/service/hlo_value.cc
@@ -209,7 +209,7 @@ std::ostream& operator<<(std::ostream& out, const HloValue& value) {
 }
 
 void HloValueSet::SortAndUniquifyValues() {
-  std::sort(values_.begin(), values_.end(), HloValue::IdLessThan);
+  absl::c_sort(values_, HloValue::IdLessThan);
   values_.erase(std::unique(values_.begin(), values_.end(), HloValue::IdEqual),
                 values_.end());
 }
diff --git a/tensorflow/compiler/xla/service/human_readable_profile_builder.cc b/tensorflow/compiler/xla/service/human_readable_profile_builder.cc
index 90904ac001..88fc62bd1e 100644
--- a/tensorflow/compiler/xla/service/human_readable_profile_builder.cc
+++ b/tensorflow/compiler/xla/service/human_readable_profile_builder.cc
@@ -128,9 +128,9 @@ string HumanReadableProfileBuilder::ToString() const {
 
   // Sort ops in decreasing order of cycles, and print them.
   std::vector<OpInfo> sorted_ops(op_infos_);
-  std::sort(
-      sorted_ops.begin(), sorted_ops.end(),
-      [](const OpInfo& a, const OpInfo& b) { return a.cycles > b.cycles; });
+  absl::c_sort(sorted_ops, [](const OpInfo& a, const OpInfo& b) {
+    return a.cycles > b.cycles;
+  });
   for (const auto& op : sorted_ops) {
     print_op(op);
   }
diff --git a/tensorflow/compiler/xla/service/instruction_fusion.cc b/tensorflow/compiler/xla/service/instruction_fusion.cc
index f6f25c4413..b97060535d 100644
--- a/tensorflow/compiler/xla/service/instruction_fusion.cc
+++ b/tensorflow/compiler/xla/service/instruction_fusion.cc
@@ -178,19 +178,18 @@ bool InstructionFusion::EffectivelyAtMostUnary(HloInstruction* hlo) {
           output_rank = std::max(output_rank, ShapeUtil::TrueRank(subshape));
         }
       });
-  return std::count_if(hlo->operands().begin(), hlo->operands().end(),
-                       [output_rank](HloInstruction* operand) {
-                         if (operand->opcode() == HloOpcode::kBroadcast ||
-                             operand->opcode() == HloOpcode::kIota) {
-                           return false;
-                         }
-                         if (operand->opcode() == HloOpcode::kConstant &&
-                             ShapeUtil::IsEffectiveScalar(operand->shape())) {
-                           return false;
-                         }
-                         return ShapeUtil::TrueRank(operand->shape()) >=
-                                output_rank;
-                       }) <= 1;
+  return absl::c_count_if(
+             hlo->operands(), [output_rank](HloInstruction* operand) {
+               if (operand->opcode() == HloOpcode::kBroadcast ||
+                   operand->opcode() == HloOpcode::kIota) {
+                 return false;
+               }
+               if (operand->opcode() == HloOpcode::kConstant &&
+                   ShapeUtil::IsEffectiveScalar(operand->shape())) {
+                 return false;
+               }
+               return ShapeUtil::TrueRank(operand->shape()) >= output_rank;
+             }) <= 1;
 }
 
 bool InstructionFusion::CanFuseOnAllPaths(
@@ -409,9 +408,8 @@ class ReversePostOrderFusionQueue : public FusionQueue {
       }
       sorted_operand_numbers.push_back(i);
     }
-    std::sort(
-        sorted_operand_numbers.begin(), sorted_operand_numbers.end(),
-        [&](int64 i, int64 j) {
+    absl::c_sort(
+        sorted_operand_numbers, [&](int64 i, int64 j) {
           // Instructions with higher priority in the queue come first.
           return (
               FindOrDie(post_order_index_, instruction->mutable_operand(i)) >
diff --git a/tensorflow/compiler/xla/service/layout_assignment.cc b/tensorflow/compiler/xla/service/layout_assignment.cc
index 406cd0a219..10ff7bb6d4 100644
--- a/tensorflow/compiler/xla/service/layout_assignment.cc
+++ b/tensorflow/compiler/xla/service/layout_assignment.cc
@@ -147,12 +147,9 @@ bool LayoutConstraints::OperandBufferForwarded(
   PointsToSet::BufferSet* output_buffers = GetBufferSet(instruction);
   PointsToSet::BufferSet* operand_buffers =
       GetBufferSet(instruction->operand(operand_no));
-  for (const LogicalBuffer* output_buffer : *output_buffers) {
-    if (operand_buffers->count(output_buffer) > 0) {
-      return true;
-    }
-  }
-  return false;
+  return absl::c_any_of(*output_buffers, [&](const LogicalBuffer* b) {
+    return operand_buffers->count(b) > 0;
+  });
 }
 
 Status LayoutConstraints::SetBufferLayout(const Layout& layout,
diff --git a/tensorflow/compiler/xla/service/llvm_ir/alias_analysis.cc b/tensorflow/compiler/xla/service/llvm_ir/alias_analysis.cc
index 643ecd0fba..ce3d922ca7 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/alias_analysis.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/alias_analysis.cc
@@ -81,9 +81,7 @@ void AliasAnalysis::AddAliasingInformationToIrArray(const HloInstruction& hlo,
     if (hlo.opcode() == HloOpcode::kParameter) {
       const std::vector<HloInstruction*>& parameter_instructions =
           module_.entry_computation()->parameter_instructions();
-      if (std::find(parameter_instructions.begin(),
-                    parameter_instructions.end(),
-                    &hlo) != parameter_instructions.end()) {
+      if (absl::c_linear_search(parameter_instructions, &hlo)) {
         array->MarkInvariantOverWholeProgram(context_);
       }
     }
diff --git a/tensorflow/compiler/xla/service/name_uniquer.cc b/tensorflow/compiler/xla/service/name_uniquer.cc
index daa718879d..f6feed2993 100644
--- a/tensorflow/compiler/xla/service/name_uniquer.cc
+++ b/tensorflow/compiler/xla/service/name_uniquer.cc
@@ -34,7 +34,7 @@ bool IsAllowed(char character) {
 }  // namespace
 
 NameUniquer::NameUniquer(const string& separator) {
-  CHECK(std::all_of(separator.begin(), separator.end(), IsAllowed))
+  CHECK(absl::c_all_of(separator, IsAllowed))
       << "separator should comprises allowed characters only";
   separator_ = separator;
 }
diff --git a/tensorflow/compiler/xla/service/platform_util.cc b/tensorflow/compiler/xla/service/platform_util.cc
index 896b73cda4..0491f641fc 100644
--- a/tensorflow/compiler/xla/service/platform_util.cc
+++ b/tensorflow/compiler/xla/service/platform_util.cc
@@ -260,8 +260,8 @@ PlatformUtil::GetStreamExecutors(
     // Block here in thread_pool destructor until all devices are initialized.
   }
   VLOG(1) << "Device initialization complete";
-  if (std::all_of(stream_executors.begin(), stream_executors.end(),
-                  [](se::StreamExecutor* s) { return s == nullptr; })) {
+  if (absl::c_all_of(stream_executors,
+                     [](se::StreamExecutor* s) { return s == nullptr; })) {
     return InternalError("no supported devices found for platform %s",
                          platform->Name());
   }
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index 4a2dd09742..d711c8ef0d 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -534,9 +534,8 @@ Status ValidateDotDimensionNumbers(
                           absl::Span<const int64> contracting_dims,
                           absl::Span<const int64> batch_dims) -> bool {
     auto in_range = [&rank](int64 i) -> bool { return 0 <= i && i < rank; };
-    return std::all_of(contracting_dims.begin(), contracting_dims.end(),
-                       in_range) &&
-           std::all_of(batch_dims.begin(), batch_dims.end(), in_range);
+    return absl::c_all_of(contracting_dims, in_range) &&
+           absl::c_all_of(batch_dims, in_range);
   };
 
   absl::Span<const int64> lhs_contracting_dimensions =
@@ -563,9 +562,8 @@ Status ValidateDotDimensionNumbers(
     auto is_unique = [&dim_set](int64 i) -> bool {
       return dim_set.insert(i).second;
     };
-    return std::all_of(contracting_dims.begin(), contracting_dims.end(),
-                       is_unique) &&
-           std::all_of(batch_dims.begin(), batch_dims.end(), is_unique);
+    return absl::c_all_of(contracting_dims, is_unique) &&
+           absl::c_all_of(batch_dims, is_unique);
   };
 
   if (!dims_unique(lhs_contracting_dimensions, lhs_batch_dimensions) ||
@@ -1589,29 +1587,29 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
   input_dnums[1] = dnums.input_feature_dimension();
   std::copy(dnums.input_spatial_dimensions().begin(),
             dnums.input_spatial_dimensions().end(), input_dnums.begin() + 2);
-  std::sort(input_dnums.begin(), input_dnums.end());
+  absl::c_sort(input_dnums);
 
   std::vector<int64> window_dnums(num_dims);
   window_dnums[0] = dnums.kernel_input_feature_dimension();
   window_dnums[1] = dnums.kernel_output_feature_dimension();
   std::copy(dnums.kernel_spatial_dimensions().begin(),
             dnums.kernel_spatial_dimensions().end(), window_dnums.begin() + 2);
-  std::sort(window_dnums.begin(), window_dnums.end());
+  absl::c_sort(window_dnums);
 
   std::vector<int64> output_dnums(num_dims);
   output_dnums[0] = dnums.output_batch_dimension();
   output_dnums[1] = dnums.output_feature_dimension();
   std::copy(dnums.output_spatial_dimensions().begin(),
             dnums.output_spatial_dimensions().end(), output_dnums.begin() + 2);
-  std::sort(output_dnums.begin(), output_dnums.end());
+  absl::c_sort(output_dnums);
 
   std::vector<int64> expected_dnums(num_dims);
   std::iota(expected_dnums.begin(), expected_dnums.end(), 0);
 
   const auto in_range = [num_dims](int64 i) { return 0 <= i && i < num_dims; };
-  if (!std::all_of(input_dnums.begin(), input_dnums.end(), in_range) ||
-      !std::all_of(window_dnums.begin(), window_dnums.end(), in_range) ||
-      !std::all_of(output_dnums.begin(), output_dnums.end(), in_range)) {
+  if (!absl::c_all_of(input_dnums, in_range) ||
+      !absl::c_all_of(window_dnums, in_range) ||
+      !absl::c_all_of(output_dnums, in_range)) {
     return InvalidArgument(
         "A dimension number is out of range in convolution: %s.",
         dnums.DebugString());
diff --git a/tensorflow/compiler/xla/service/transpose_folding.cc b/tensorflow/compiler/xla/service/transpose_folding.cc
index 15eb46bac0..a95ca2bf2a 100644
--- a/tensorflow/compiler/xla/service/transpose_folding.cc
+++ b/tensorflow/compiler/xla/service/transpose_folding.cc
@@ -130,8 +130,7 @@ bool FoldTransposeIntoConvolution(InstructionOperandsPair pair) {
 
   HloInstruction* new_lhs;
   const int64 kLhsIdx = 0;
-  if (std::find(operand_indices.begin(), operand_indices.end(), kLhsIdx) !=
-      operand_indices.end()) {
+  if (absl::c_linear_search(operand_indices, kLhsIdx)) {
     HloInstruction& transpose = *convolution.mutable_operand(kLhsIdx);
     const auto& transpose_dimensions = transpose.dimensions();
     HloInstruction& transpose_operand = *transpose.mutable_operand(0);
@@ -154,8 +153,7 @@ bool FoldTransposeIntoConvolution(InstructionOperandsPair pair) {
 
   HloInstruction* new_rhs;
   const int64 kRhsIdx = 1;
-  if (std::find(operand_indices.begin(), operand_indices.end(), kRhsIdx) !=
-      operand_indices.end()) {
+  if (absl::c_linear_search(operand_indices, kRhsIdx)) {
     HloInstruction& transpose = *convolution.mutable_operand(kRhsIdx);
     const auto& transpose_dimensions = transpose.dimensions();
     HloInstruction& transpose_operand = *transpose.mutable_operand(0);
diff --git a/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc b/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc
index dec719b04a..5e505aaf02 100644
--- a/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc
+++ b/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc
@@ -87,9 +87,7 @@ bool PointsToSet::ContainsBuffer(const LogicalBuffer& buffer) const {
   bool found = false;
   ForEachElement([&found, &buffer](const ShapeIndex& /*index*/,
                                    const BufferList& pointed_to_buffers) {
-    if (!found &&
-        std::find(pointed_to_buffers.begin(), pointed_to_buffers.end(),
-                  &buffer) != pointed_to_buffers.end()) {
+    if (!found && absl::c_linear_search(pointed_to_buffers, &buffer)) {
       found = true;
     }
   });
@@ -99,8 +97,7 @@ bool PointsToSet::ContainsBuffer(const LogicalBuffer& buffer) const {
 bool PointsToSet::ContainsBufferAtIndex(const LogicalBuffer& buffer,
                                         const ShapeIndex& index) const {
   const auto& pointed_to_buffers = element(index);
-  return std::find(pointed_to_buffers.begin(), pointed_to_buffers.end(),
-                   &buffer) != pointed_to_buffers.end();
+  return absl::c_linear_search(pointed_to_buffers, &buffer);
 }
 
 void PointsToSet::AddPointedToBuffer(const LogicalBuffer& buffer,
@@ -604,9 +601,8 @@ bool TuplePointsToAnalysis::DoesNotUseOperandBuffer(
   } else if (user->opcode() == HloOpcode::kFusion &&
              user->fusion_kind() == HloInstruction::FusionKind::kLoop) {
     // Find fusion parameter associated with 'operand'.
-    auto it = std::find_if(
-        user->fused_parameters().begin(), user->fused_parameters().end(),
-        [=](HloInstruction* fused_param) {
+    auto it = absl::c_find_if(
+        user->fused_parameters(), [&](HloInstruction* fused_param) {
           return user->operand(fused_param->parameter_number()) == operand;
         });
     CHECK(it != user->fused_parameters().end());
@@ -672,9 +668,8 @@ bool TuplePointsToAnalysis::HasUniqueFusedUseOfOperandAt(
   }
   // Find fusion parameter associated with 'operand'.
   const auto& fused_params = fusion->fused_parameters();
-  auto fused_param_it = std::find_if(
-      fused_params.begin(), fused_params.end(),
-      [&](HloInstruction* fused_param) {
+  auto fused_param_it =
+      absl::c_find_if(fused_params, [&](HloInstruction* fused_param) {
         return fusion->operand(fused_param->parameter_number()) == operand;
       });
   if (fused_param_it == fused_params.end()) {
@@ -743,11 +738,10 @@ bool TuplePointsToAnalysis::CanShareOperandBufferWithUser(
       // Check if one operand of kAdd fused root is kDot or kConvolution.
       auto* add = user->fused_expression_root();
       auto add_operand_it =
-          std::find_if(add->operands().begin(), add->operands().end(),
-                       [&](HloInstruction* operand) {
-                         return operand->opcode() == HloOpcode::kConvolution ||
-                                operand->opcode() == HloOpcode::kDot;
-                       });
+          absl::c_find_if(add->operands(), [&](HloInstruction* operand) {
+            return operand->opcode() == HloOpcode::kConvolution ||
+                   operand->opcode() == HloOpcode::kDot;
+          });
       if (add_operand_it == add->operands().end()) {
         return false;
       }
diff --git a/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc b/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc
index d8875ca747..7599e1e6ad 100644
--- a/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc
+++ b/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc
@@ -721,9 +721,8 @@ class FusionPointsToAnalysisTest : public TuplePointsToAnalysisTest {
   // to fusion 'operand'.
   HloInstruction* GetFusionParameterForOperand(HloInstruction* fusion,
                                                HloInstruction* operand) {
-    auto it = std::find_if(
-        fusion->fused_instructions().begin(),
-        fusion->fused_instructions().end(), [=](const HloInstruction* fused) {
+    auto it = absl::c_find_if(
+        fusion->fused_instructions(), [&](const HloInstruction* fused) {
           return fused->opcode() == HloOpcode::kParameter &&
                  fusion->operand(fused->parameter_number()) == operand;
         });
diff --git a/tensorflow/compiler/xla/service/while_loop_simplifier.cc b/tensorflow/compiler/xla/service/while_loop_simplifier.cc
index de884b037d..09d5409571 100644
--- a/tensorflow/compiler/xla/service/while_loop_simplifier.cc
+++ b/tensorflow/compiler/xla/service/while_loop_simplifier.cc
@@ -109,8 +109,7 @@ static StatusOr<bool> TryRemoveDeadWhileParams(HloInstruction* while_op) {
       // operand appears in, but it may appear more than once!
       if (user->user_count() == 1 && user->users().front() == while_body_root &&
           while_body_root->operand_index(user) == user->tuple_index() &&
-          std::count(while_body_root->operands().begin(),
-                     while_body_root->operands().end(), user) == 1) {
+          absl::c_count(while_body_root->operands(), user) == 1) {
         continue;
       }
 
@@ -158,7 +157,7 @@ static StatusOr<bool> TryRemoveDeadWhileParams(HloInstruction* while_op) {
   // Build up maps from the old/new to the new/old tuple indices.
   std::vector<int64> new_to_old_tuple_idx(used_tuple_indices.begin(),
                                           used_tuple_indices.end());
-  std::sort(new_to_old_tuple_idx.begin(), new_to_old_tuple_idx.end());
+  absl::c_sort(new_to_old_tuple_idx);
 
   absl::flat_hash_map<int64, int64> old_to_new_tuple_idx;
   for (int64 new_idx = 0; new_idx < new_to_old_tuple_idx.size(); ++new_idx) {
diff --git a/tensorflow/compiler/xla/service/while_loop_simplifier_test.cc b/tensorflow/compiler/xla/service/while_loop_simplifier_test.cc
index 3713989ca2..ecca76b1e8 100644
--- a/tensorflow/compiler/xla/service/while_loop_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/while_loop_simplifier_test.cc
@@ -407,13 +407,12 @@ TEST_F(WhileLoopSimplifierTest, RemoveUnusedLoopOperands) {
   // The original while instruction is still left in the module as a dead
   // instruction, find a while instruction with a different name as the new
   // while instruction.
+  const auto& instrs = m->entry_computation()->instructions();
   HloInstruction* new_while_op =
-      *std::find_if(m->entry_computation()->instructions().begin(),
-                    m->entry_computation()->instructions().end(),
-                    [&](const HloInstruction* instr) {
-                      return (instr->opcode() == HloOpcode::kWhile &&
-                              instr->name() != "while");
-                    });
+      *absl::c_find_if(instrs, [&](const HloInstruction* instr) {
+        return (instr->opcode() == HloOpcode::kWhile &&
+                instr->name() != "while");
+      });
 
   auto scalar_s32 = ShapeUtil::MakeShape(S32, {});
   EXPECT_TRUE(
diff --git a/tensorflow/compiler/xla/shape.cc b/tensorflow/compiler/xla/shape.cc
index f05d816268..ea4f722ef4 100644
--- a/tensorflow/compiler/xla/shape.cc
+++ b/tensorflow/compiler/xla/shape.cc
@@ -87,8 +87,7 @@ bool Shape::is_static() const {
       }
     }
   }
-  return !std::any_of(dynamic_dimensions_.begin(), dynamic_dimensions_.end(),
-                      [](bool b) { return b; });
+  return !absl::c_any_of(dynamic_dimensions_, [](bool b) { return b; });
 }
 
 void Shape::DeleteDimension(int64 dim_to_delete) {
diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc
index 7f68c0ae1f..4726ee6a5b 100644
--- a/tensorflow/compiler/xla/shape_util.cc
+++ b/tensorflow/compiler/xla/shape_util.cc
@@ -405,8 +405,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
 }
 
 /* static */ bool ShapeUtil::IsNestedTuple(const Shape& shape) {
-  return IsTuple(shape) && std::any_of(shape.tuple_shapes().begin(),
-                                       shape.tuple_shapes().end(), IsTuple);
+  return IsTuple(shape) && absl::c_any_of(shape.tuple_shapes(), IsTuple);
 }
 
 /* static */ bool ShapeUtil::IsEmptyTuple(const Shape& shape) {
diff --git a/tensorflow/compiler/xla/sparse_index_array.h b/tensorflow/compiler/xla/sparse_index_array.h
index a96d483462..0c25355467 100644
--- a/tensorflow/compiler/xla/sparse_index_array.h
+++ b/tensorflow/compiler/xla/sparse_index_array.h
@@ -135,7 +135,7 @@ void SparseIndexArray::SortWithValues(absl::Span<NativeT> values) {
   auto sort_order_less = [this](int64 lhs, int64 rhs) {
     return IndexUtil::CompareIndices(At(lhs), At(rhs)) < 0;
   };
-  std::sort(sort_order.begin(), sort_order.end(), sort_order_less);
+  absl::c_sort(sort_order, sort_order_less);
 
   // Reorder the array elements according to sort_order.  Work through the array
   // and follow cycles so we can do the reorder in-place.
diff --git a/tensorflow/compiler/xla/tests/convolution_test.cc b/tensorflow/compiler/xla/tests/convolution_test.cc
index 2496938912..9db9f2563b 100644
--- a/tensorflow/compiler/xla/tests/convolution_test.cc
+++ b/tensorflow/compiler/xla/tests/convolution_test.cc
@@ -467,8 +467,8 @@ XLA_TEST_F(ConvolutionTest, Convolve3D_1x4x2x3x3_2x2x2x3x3_Valid) {
 // servers. The error message is missing the operator ++.
 template <typename T>
 void iota_int_init_value(std::vector<T>& values, int init_value) {
-  std::for_each(values.begin(), values.end(),
-                [&](T& value) { value = static_cast<T>(init_value++); });
+  absl::c_for_each(values,
+                   [&](T& value) { value = static_cast<T>(init_value++); });
 }
 
 template <typename T>
diff --git a/tensorflow/compiler/xla/util.cc b/tensorflow/compiler/xla/util.cc
index 68cab7387c..34b73b5206 100644
--- a/tensorflow/compiler/xla/util.cc
+++ b/tensorflow/compiler/xla/util.cc
@@ -86,7 +86,7 @@ bool IsPermutation(absl::Span<const int64> permutation, int64 rank) {
     CHECK_LT(index, rank);
     output[index] = 0;
   }
-  return std::find(output.begin(), output.end(), -1) == output.end();
+  return !absl::c_linear_search(output, -1);
 }
 
 std::vector<int64> InversePermutation(
diff --git a/tensorflow/compiler/xla/util.h b/tensorflow/compiler/xla/util.h
index 6722641e9d..f2fd17dc99 100644
--- a/tensorflow/compiler/xla/util.h
+++ b/tensorflow/compiler/xla/util.h
@@ -324,8 +324,7 @@ bool IsIdentityPermutation(absl::Span<const int64> permutation);
 
 template <typename Container>
 int64 PositionInContainer(const Container& container, int64 value) {
-  return std::distance(container.begin(),
-                       std::find(container.begin(), container.end(), value));
+  return std::distance(container.begin(), absl::c_find(container, value));
 }
 
 // Formats the container as a comma-separated string. StrAppend must support
diff --git a/tensorflow/compiler/xla/window_util.cc b/tensorflow/compiler/xla/window_util.cc
index 51c73b3d17..e001cc35f9 100644
--- a/tensorflow/compiler/xla/window_util.cc
+++ b/tensorflow/compiler/xla/window_util.cc
@@ -17,6 +17,7 @@ limitations under the License.
 
 #include <vector>
 
+#include "absl/algorithm/container.h"
 #include "absl/strings/str_cat.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
@@ -137,25 +138,23 @@ bool HasPadding(const Window& window) {
 }
 
 bool HasSymmetricPadding(const Window& window) {
-  return std::all_of(window.dimensions().begin(), window.dimensions().end(),
-                     [](const WindowDimension& dim) {
-                       return dim.padding_low() == dim.padding_high();
-                     });
+  return absl::c_all_of(window.dimensions(), [](const WindowDimension& dim) {
+    return dim.padding_low() == dim.padding_high();
+  });
 }
 
 bool HasSymmetricPadding(const PaddingConfig& padding_config) {
-  return std::all_of(padding_config.dimensions().begin(),
-                     padding_config.dimensions().end(),
-                     [](const PaddingConfig::PaddingConfigDimension& dim) {
-                       return dim.edge_padding_low() == dim.edge_padding_high();
-                     });
+  return absl::c_all_of(padding_config.dimensions(),
+                        [](const PaddingConfig::PaddingConfigDimension& dim) {
+                          return dim.edge_padding_low() ==
+                                 dim.edge_padding_high();
+                        });
 }
 
 bool HasNegativePadding(const Window& window) {
-  return std::any_of(window.dimensions().begin(), window.dimensions().end(),
-                     [](const WindowDimension& dim) {
-                       return dim.padding_low() < 0 || dim.padding_high() < 0;
-                     });
+  return absl::c_any_of(window.dimensions(), [](const WindowDimension& dim) {
+    return dim.padding_low() < 0 || dim.padding_high() < 0;
+  });
 }
 
 bool HasBaseDilation(const Window& window) {
@@ -190,10 +189,9 @@ bool AllOrNoneReversed(const Window& window) {
     return true;
   }
   bool reversed = window.dimensions()[0].window_reversal();
-  return std::all_of(window.dimensions().begin(), window.dimensions().end(),
-                     [&](const WindowDimension& dim) {
-                       return dim.window_reversal() == reversed;
-                     });
+  return absl::c_all_of(window.dimensions(), [&](const WindowDimension& dim) {
+    return dim.window_reversal() == reversed;
+  });
 }
 
 bool HasDilation(const Window& window) {
-- 
GitLab


From e2071f6a0502a5bc81abd0cb533f3b52ba0e7855 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Fri, 4 Jan 2019 12:59:45 -0800
Subject: [PATCH 0204/2345] Add poisson v2 loss and metric.

PiperOrigin-RevId: 227900433
---
 tensorflow/python/keras/losses.py       | 47 +++++++++-----
 tensorflow/python/keras/losses_test.py  | 81 +++++++++++++++++++++++++
 tensorflow/python/keras/metrics.py      | 31 ++++++++++
 tensorflow/python/keras/metrics_test.py | 46 ++++++++++++++
 4 files changed, 189 insertions(+), 16 deletions(-)

diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index 1ff564054b..46e729c197 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -519,25 +519,39 @@ class LogLoss(Loss):
   model = keras.models.Model(inputs, outputs)
   model.compile('sgd', loss=tf.losses.LogLoss())
   ```
-
-  Args:
-    epsilon: A small increment to add to avoid taking a log of zero.
-    reduction: Type of `tf.losses.Reduction` to apply to loss. Default value is
-      `SUM_OVER_BATCH_SIZE`.
-    name: Optional name for the op.
   """
 
-  def __init__(self,
-               epsilon=1e-7,
-               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
-               name=None):
-    super(LogLoss, self).__init__(reduction=reduction, name=name)
-    self.epsilon = epsilon
+  def call(self, y_true, y_pred):
+    y_pred = ops.convert_to_tensor(y_pred)
+    y_true = math_ops.cast(y_true, y_pred.dtype)
+    return logloss(y_true, y_pred)
+
+
+class Poisson(Loss):
+  """Computes the poisson loss between `y_true` and `y_pred`.
+
+  loss = y_pred - y_true * log(y_pred)
+
+  Usage:
+
+  ```python
+  p = tf.losses.Poisson()
+  loss = p([1, 9, 2], [4, 8, 12])
+  print('Loss: ', loss.numpy())  # Loss: -4.63
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', loss=tf.losses.Poisson())
+  ```
+  """
 
   def call(self, y_true, y_pred):
     y_pred = ops.convert_to_tensor(y_pred)
     y_true = math_ops.cast(y_true, y_pred.dtype)
-    return logloss(y_true, y_pred, epsilon=self.epsilon)
+    return poisson(y_true, y_pred)
 
 
 class Logcosh(Loss):
@@ -629,9 +643,10 @@ def categorical_hinge(y_true, y_pred):
   return math_ops.maximum(0., neg - pos + 1.)
 
 
-def logloss(y_true, y_pred, epsilon=1e-7):
-  losses = math_ops.multiply(y_true, math_ops.log(y_pred + epsilon))
-  losses += math_ops.multiply((1 - y_true), math_ops.log(1 - y_pred + epsilon))
+def logloss(y_true, y_pred):
+  losses = math_ops.multiply(y_true, math_ops.log(y_pred + K.epsilon()))
+  losses += math_ops.multiply((1 - y_true),
+                              math_ops.log(1 - y_pred + K.epsilon()))
   return K.mean(-losses, axis=-1)
 
 
diff --git a/tensorflow/python/keras/losses_test.py b/tensorflow/python/keras/losses_test.py
index aa00ba5a91..f34a6caa18 100644
--- a/tensorflow/python/keras/losses_test.py
+++ b/tensorflow/python/keras/losses_test.py
@@ -1175,5 +1175,86 @@ class LogcoshTest(test.TestCase):
     self.assertAlmostEqual(self.evaluate(loss), 0., 3)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class PoissonTest(test.TestCase):
+
+  def setup(self):
+    self.np_y_pred = np.asarray([1, 9, 2, 5, 2, 6]).reshape((2, 3))
+    self.np_y_true = np.asarray([4, 8, 12, 8, 1, 3]).reshape((2, 3))
+
+    self.batch_size = 6
+    self.expected_losses = self.np_y_pred - np.multiply(self.np_y_true,
+                                                        np.log(self.np_y_pred))
+
+    self.y_pred = constant_op.constant(self.np_y_pred, dtype=dtypes.float32)
+    self.y_true = constant_op.constant(self.np_y_true)
+
+  def test_config(self):
+    poisson_obj = keras.losses.Poisson(
+        reduction=losses_impl.ReductionV2.SUM, name='poisson')
+    self.assertEqual(poisson_obj.name, 'poisson')
+    self.assertEqual(poisson_obj.reduction, losses_impl.ReductionV2.SUM)
+
+  def test_unweighted(self):
+    self.setup()
+    poisson_obj = keras.losses.Poisson()
+
+    loss = poisson_obj(self.y_true, self.y_pred)
+    expected_loss = np.sum(self.expected_losses) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+
+  def test_scalar_weighted(self):
+    self.setup()
+    poisson_obj = keras.losses.Poisson()
+    sample_weight = 2.3
+    loss = poisson_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+
+    expected_loss = sample_weight * np.sum(
+        self.expected_losses) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+
+    # Verify we get the same output when the same input is given
+    loss_2 = poisson_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), self.evaluate(loss_2), 3)
+
+  def test_sample_weighted(self):
+    self.setup()
+    poisson_obj = keras.losses.Poisson()
+
+    sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
+    loss = poisson_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+
+    expected_loss = np.multiply(
+        self.expected_losses,
+        np.asarray([1.2, 1.2, 1.2, 3.4, 3.4, 3.4]).reshape((2, 3)))
+    expected_loss = np.sum(expected_loss) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+
+  def test_timestep_weighted(self):
+    self.setup()
+    poisson_obj = keras.losses.Poisson()
+    y_true = self.np_y_true.reshape(2, 3, 1)
+    y_pred = self.np_y_pred.reshape(2, 3, 1)
+    sample_weight = np.asarray([3, 6, 5, 0, 4, 2]).reshape(2, 3, 1)
+    expected_losses = y_pred - np.multiply(y_true, np.log(y_pred))
+
+    y_pred = constant_op.constant(y_pred, dtype=dtypes.float32)
+    y_true = constant_op.constant(y_true)
+
+    loss = poisson_obj(
+        y_true,
+        y_pred,
+        sample_weight=constant_op.constant(sample_weight, shape=(2, 3)))
+    expected_loss = np.sum(expected_losses * sample_weight) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+
+  def test_zero_weighted(self):
+    self.setup()
+    poisson_obj = keras.losses.Poisson()
+    loss = poisson_obj(self.y_true, self.y_pred, sample_weight=0)
+    self.assertAlmostEqual(self.evaluate(loss), 0., 3)
+
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 2de0bc474c..71568e3fc5 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -1685,6 +1685,37 @@ class Logcosh(MeanMetricWrapper):
     return super(Logcosh, cls).from_config(config)
 
 
+class Poisson(MeanMetricWrapper):
+  """Computes the poisson metric between `y_true` and `y_pred`.
+
+  metric = y_pred - y_true * log(y_pred)
+
+  Usage:
+
+  ```python
+  m = tf.keras.metrics.Poisson()
+  m.update_state([1, 9, 2], [4, 8, 12])
+  print('Final result: ', m.result().numpy())  # Final result: -4.63
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', metrics=[tf.keras.metrics.Poisson()])
+  ```
+  """
+
+  def __init__(self, name='poisson', dtype=None):
+    super(Poisson, self).__init__(poisson, name, dtype=dtype)
+
+  @classmethod
+  def from_config(cls, config):
+    if 'fn' in config:
+      config.pop('fn')
+    return super(Poisson, cls).from_config(config)
+
+
 def accuracy(y_true, y_pred):
   y_pred.get_shape().assert_is_compatible_with(y_true.get_shape())
   if y_true.dtype != y_pred.dtype:
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index a9d88789ed..88b763d569 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -1510,6 +1510,52 @@ class LogcoshTest(test.TestCase):
     self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class PoissonTest(test.TestCase):
+
+  def setup(self):
+    y_pred = np.asarray([1, 9, 2, 5, 2, 6]).reshape((2, 3))
+    y_true = np.asarray([4, 8, 12, 8, 1, 3]).reshape((2, 3))
+
+    self.batch_size = 6
+    self.expected_results = y_pred - np.multiply(y_true, np.log(y_pred))
+
+    self.y_pred = constant_op.constant(y_pred, dtype=dtypes.float32)
+    self.y_true = constant_op.constant(y_true)
+
+  def test_config(self):
+    poisson_obj = metrics.Poisson(name='poisson', dtype=dtypes.int32)
+    self.assertEqual(poisson_obj.name, 'poisson')
+    self.assertEqual(poisson_obj._dtype, dtypes.int32)
+
+    poisson_obj2 = metrics.Poisson.from_config(poisson_obj.get_config())
+    self.assertEqual(poisson_obj2.name, 'poisson')
+    self.assertEqual(poisson_obj2._dtype, dtypes.int32)
+
+  def test_unweighted(self):
+    self.setup()
+    poisson_obj = metrics.Poisson()
+    self.evaluate(variables.variables_initializer(poisson_obj.variables))
+
+    update_op = poisson_obj.update_state(self.y_true, self.y_pred)
+    self.evaluate(update_op)
+    result = poisson_obj.result()
+    expected_result = np.sum(self.expected_results) / self.batch_size
+    self.assertAllClose(result, expected_result, atol=1e-3)
+
+  def test_weighted(self):
+    self.setup()
+    poisson_obj = metrics.Poisson()
+    self.evaluate(variables.variables_initializer(poisson_obj.variables))
+    sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
+
+    result = poisson_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    sample_weight = np.asarray([1.2, 1.2, 1.2, 3.4, 3.4, 3.4]).reshape((2, 3))
+    expected_result = np.multiply(self.expected_results, sample_weight)
+    expected_result = np.sum(expected_result) / np.sum(sample_weight)
+    self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)
+
+
 def _get_model(compile_metrics):
   model_layers = [
       layers.Dense(3, activation='relu', kernel_initializer='ones'),
-- 
GitLab


From d198ec5f99258f443af8e16825ae9c77e26c5db6 Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Fri, 4 Jan 2019 13:11:46 -0800
Subject: [PATCH 0205/2345] Move ParameterServerStrategy to core.

PiperOrigin-RevId: 227902368
---
 tensorflow/contrib/distribute/python/BUILD    |  15 +-
 .../python/parameter_server_strategy.py       | 467 +---------------
 .../python/parameter_server_strategy_test.py  | 339 +++++++----
 tensorflow/python/distribute/BUILD            |  22 +
 .../distribute/parameter_server_strategy.py   | 528 ++++++++++++++++++
 5 files changed, 813 insertions(+), 558 deletions(-)
 create mode 100644 tensorflow/python/distribute/parameter_server_strategy.py

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 78ad3214fb..de6b6f1f84 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -80,19 +80,10 @@ py_library(
     srcs = ["parameter_server_strategy.py"],
     visibility = ["//tensorflow:internal"],
     deps = [
-        ":mirrored_strategy",
-        "//tensorflow/core:protos_all_py",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:resource_variable_ops",
-        "//tensorflow/python:training",
-        "//tensorflow/python:util",
-        "//tensorflow/python/distribute:cross_device_ops",
-        "//tensorflow/python/distribute:input_lib",
-        "//tensorflow/python/distribute:multi_worker_util",
-        "//tensorflow/python/distribute:reduce_util",
+        "//tensorflow/python/distribute:distribute_lib",
+        "//tensorflow/python/distribute:parameter_server_strategy",
         "//tensorflow/python/distribute:values",
-        "//tensorflow/python/eager:context",
+        "//tensorflow/python/distribute/cluster_resolver:cluster_resolver_lib",
     ],
 )
 
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy.py b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
index 461e1bca21..bb0b8eb992 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
@@ -18,35 +18,24 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import copy
-
-from tensorflow.contrib.distribute.python import mirrored_strategy
-from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib
-from tensorflow.python.distribute import device_util
 from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import input_lib
-from tensorflow.python.distribute import multi_worker_util
-from tensorflow.python.distribute import values
-from tensorflow.python.eager import context
-from tensorflow.python.framework import device as tf_device
-from tensorflow.python.framework import ops
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import resource_variable_ops
-from tensorflow.python.ops import variable_scope as vs
-from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.training import device_setter
-from tensorflow.python.util import nest
+from tensorflow.python.distribute import parameter_server_strategy
+from tensorflow.python.distribute.cluster_resolver import SimpleClusterResolver
+from tensorflow.python.distribute.cluster_resolver import TFConfigClusterResolver
+
+# pylint: disable=protected-access,invalid-name,line-too-long
+CoreParameterServerStrategy = parameter_server_strategy.ParameterServerStrategy
+CoreParameterServerExtended = parameter_server_strategy.ParameterServerStrategyExtended
 
-_LOCAL_CPU = "/device:CPU:0"
-_LOCAL_GPU_0 = "/device:GPU:0"
+# pylint: enable=protected-access,invalid-name,line-too-long
 
 
-# TODO(yuefengz): maybe cache variables on local CPU.
-# TODO(yuefengz): we may want to set session options to disallow communication
-# between workers.
 class ParameterServerStrategy(distribute_lib.DistributionStrategy):
   """A parameter server DistributionStrategy.
 
+  *** contrib version ***
+
   This strategy class works for both local training and between-graph replicated
   training for multiple workers. If `cluster_spec` is specified, either passed
   in to __init__() method or parsed from the
@@ -101,434 +90,24 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy):
         ParameterServerExtended(self, num_gpus_per_worker))
 
 
-class ParameterServerExtended(distribute_lib.DistributionStrategyExtended):
+class ParameterServerExtended(CoreParameterServerExtended):
   """Implementation of ParameterServerStrategy."""
 
   def __init__(self, container_strategy, num_gpus_per_worker):
-    super(ParameterServerExtended, self).__init__(container_strategy)
-    self._num_gpus_per_worker = num_gpus_per_worker
-    self._initialize_local(num_gpus_per_worker)
-
-    # We typically don't need to do all-reduce in this strategy.
-    self._cross_device_ops = (
-        cross_device_ops_lib.ReductionToOneDeviceCrossDeviceOps(
-            reduce_to_device=_LOCAL_CPU))
-
-  def _initialize_multi_worker(self, num_gpus_per_worker, cluster_spec,
-                               task_type, task_id):
-    """Initialize devices for multiple workers.
-
-    It creates variable devices and compute devices. Variables and operations
-    will be assigned to them respectively. We have one compute device per
-    replica. The variable device is a device function or device string. The
-    default variable device assigns variables to parameter servers in a
-    round-robin fashion.
-
-    Args:
-      num_gpus_per_worker: number of local GPUs or GPUs per worker.
-      cluster_spec: a dict, ClusterDef or ClusterSpec object specifying the
-        cluster configurations.
-      task_type: the current task type.
-      task_id: the current task id.
-
-    Raises:
-      ValueError: if the cluster_spec doesn't have ps jobs.
-    """
-    assert cluster_spec
-    if not task_type or task_id is None:
-      raise ValueError("When `cluster_spec` is given, you must also specify "
-                       "`task_type` and `task_id`")
-    cluster_spec = multi_worker_util.normalize_cluster_spec(cluster_spec)
-
-    worker_device = "/job:%s/task:%d" % (self._task_type, self._task_id)
-
-    # Define compute devices which is a list of device strings and one for each
-    # replica. When there are GPUs, replicate operations on these GPUs.
-    # Otherwise, place operations on CPU.
-    if num_gpus_per_worker > 0:
-      compute_devices = tuple(
-          "%s/device:GPU:%d" % (worker_device, i)
-          for i in range(num_gpus_per_worker)
-      )
-    else:
-      compute_devices = (worker_device,)
-
-    self._device_map = values.ReplicaDeviceMap(compute_devices)
-    self._input_workers = input_lib.InputWorkers(
-        self._device_map, [(worker_device, compute_devices)])
-
-    # In distributed mode, place variables on ps jobs in a round-robin fashion.
-    # Note that devices returned from `replica_device_setter` are not
-    # canonical and therefore we don't canonicalize all variable devices to
-    # make them consistent.
-    # TODO(yuefengz): support passing a strategy object to control variable
-    # assignment.
-    # TODO(yuefengz): merge the logic of replica_device_setter into this
-    # class.
-    num_ps_replicas = len(cluster_spec.as_dict().get("ps", []))
-    if num_ps_replicas == 0:
-      raise ValueError("The cluster spec needs to have `ps` jobs.")
-    self._variable_device = device_setter.replica_device_setter(
-        ps_tasks=num_ps_replicas,
-        worker_device=worker_device,
-        merge_devices=True,
-        cluster=cluster_spec)
-
-    # The `_parameter_devices` is needed for the `parameter_devices` property
-    # and is a list of all variable devices. Here parameter devices are all
-    # tasks of the "ps" job.
-    self._parameter_devices = tuple(map("/job:ps/task:{}".format,
-                                        range(num_ps_replicas)))
-
-    # Add a default device so that ops without specified devices will not end up
-    # on other workers.
-    self._default_device = worker_device
-
-    self._is_chief = multi_worker_util.is_chief(cluster_spec, task_type,
-                                                task_id)
-    self._cluster_spec = cluster_spec
-    self._task_type = task_type
-    self._task_id = task_id
-
-    logging.info(
-        "Multi-worker ParameterServerStrategy with "
-        "cluster_spec = %r, task_type = %r, task_id = %r, "
-        "num_ps_replicas = %r, is_chief = %r, device_map = %r, "
-        "variable_device = %r", cluster_spec.as_dict(), task_type, task_id,
-        num_ps_replicas, self._is_chief, self._device_map,
-        self._variable_device)
-
-  def _initialize_local(self, num_gpus_per_worker):
-    """Initialize internal devices for local training."""
-    worker_device = device_util.canonicalize("/device:CPU:0")
-    # Define compute devices which is a list of device strings and one for each
-    # replica. When there are GPUs, replicate operations on these GPUs.
-    # Otherwise, place operations on CPU.
-    if num_gpus_per_worker > 0:
-      compute_devices = tuple(
-          map("/device:GPU:{}".format, range(num_gpus_per_worker)))
-    else:
-      compute_devices = (_LOCAL_CPU,)
-
-    self._device_map = values.ReplicaDeviceMap(compute_devices)
-    self._input_workers = input_lib.InputWorkers(
-        self._device_map, [(worker_device, compute_devices)])
-
-    # If there is only one GPU, put everything on that GPU. Otherwise, place
-    # variables on CPU.
-    if num_gpus_per_worker == 1:
-      assert len(compute_devices) == 1
-      self._variable_device = _LOCAL_GPU_0
-      self._parameter_devices = (_LOCAL_GPU_0,)
-    else:
-      self._variable_device = _LOCAL_CPU
-      self._parameter_devices = (_LOCAL_CPU,)
-
-    self._is_chief = True
-    self._cluster_spec = None
-    self._task_type = None
-    self._task_id = None
-
-    logging.info(
-        "ParameterServerStrategy with compute_devices = %r, "
-        "variable_device = %r", compute_devices, self._variable_device)
-
-  def _validate_colocate_with_variable(self, colocate_with_variable):
-    values.validate_colocate(colocate_with_variable, self)
-
-  def _distribute_dataset(self, dataset_fn):
-    """Distributes the dataset to each local GPU."""
-    return input_lib.PerReplicaDataset(
-        self._call_dataset_fn(dataset_fn), self._input_workers, 0,
-        prefetch_on_device=True)
+    # Use TFConfigClusterResolver to parse TF_CONFIG. We don't want to change
+    # the constructor's interface to allow customized cluster resolver. Use
+    # SimpleClusterResolver to override num_accelerators.
+    tfconfig = TFConfigClusterResolver()
+    cluster_resolver = SimpleClusterResolver(
+        cluster_spec=tfconfig.cluster_spec(),
+        task_type=tfconfig.task_type,
+        task_index=tfconfig.task_index,
+        num_accelerators=num_gpus_per_worker)
+    super(ParameterServerExtended, self).__init__(
+        container_strategy, cluster_resolver=cluster_resolver)
 
   def _make_dataset_iterator(self, dataset):
-    return input_lib.DatasetIterator(dataset, self._input_workers,
-                                     self._num_replicas_in_sync)
-
-  def _make_input_fn_iterator(
-      self,
-      input_fn,
-      replication_mode=distribute_lib.InputReplicationMode.PER_WORKER):
-    """Distributes the dataset to each local GPU."""
-    if self._cluster_spec:
-      input_pipeline_id = multi_worker_util.id_in_cluster(
-          self._cluster_spec, self._task_type, self._task_id)
-      num_input_pipelines = multi_worker_util.worker_count(
-          self._cluster_spec, self._task_type)
-    else:
-      input_pipeline_id = 0
-      num_input_pipelines = 1
-    input_context = distribute_lib.InputContext(
-        num_input_pipelines=num_input_pipelines,
-        input_pipeline_id=input_pipeline_id,
-        num_replicas_in_sync=self._num_replicas_in_sync)
-    return input_lib.InputFunctionIterator(
-        input_fn, self._input_workers, [input_context])
-
-  def _broadcast_to(self, tensor, destinations):
-    # This is both a fast path for Python constants, and a way to delay
-    # converting Python values to a tensor until we know what type it
-    # should be converted to. Otherwise we have trouble with:
-    #   global_step.assign_add(1)
-    # since the `1` gets broadcast as an int32 but global_step is int64.
-    if isinstance(tensor, (float, int)):
-      return tensor
-    if not cross_device_ops_lib.check_destinations(destinations):
-      # TODO(josh11b): Use current logical device instead of 0 here.
-      destinations = values.LogicalDeviceSpec(
-          device_map=self._device_map, logical_device=0)
-    return self._cross_device_ops.broadcast(tensor, destinations)
-
-  def _allow_variable_partition(self):
-    return not context.executing_eagerly()
-
-  # TODO(yuefengz): not all ops in device_setter.STANDARD_PS_OPS will go through
-  # this creator, such as "MutableHashTable".
-  def _create_variable(self, next_creator, *args, **kwargs):
-    if self._num_replicas_in_sync > 1:
-      aggregation = kwargs.pop("aggregation", vs.VariableAggregation.NONE)
-      if aggregation not in (
-          vs.VariableAggregation.NONE,
-          vs.VariableAggregation.SUM,
-          vs.VariableAggregation.MEAN,
-          vs.VariableAggregation.ONLY_FIRST_REPLICA
-      ):
-        raise ValueError("Invalid variable aggregation mode: " + aggregation +
-                         " for variable: " + kwargs["name"])
-
-      def var_creator(*args, **kwargs):
-        """Create an AggregatingVariable and fix up collections."""
-        # Record what collections this variable should be added to.
-        collections = kwargs.pop("collections", None)
-        if collections is None:
-          collections = [ops.GraphKeys.GLOBAL_VARIABLES]
-        kwargs["collections"] = []
-
-        # Create and wrap the variable.
-        v = next_creator(*args, **kwargs)
-        wrapped = values.AggregatingVariable(
-            self._container_strategy(), v, aggregation)
-
-        # Add the wrapped variable to the requested collections.
-        # The handling of eager mode and the global step matches
-        # ResourceVariable._init_from_args().
-        if not context.executing_eagerly():
-          g = ops.get_default_graph()
-          # If "trainable" is True, next_creator() will add the contained
-          # variable to the TRAINABLE_VARIABLES collection, so we manually
-          # remove it and replace with the wrapper. We can't set "trainable"
-          # to False for next_creator() since that causes functions like
-          # implicit_gradients to skip those variables.
-          if kwargs.get("trainable", True):
-            collections.append(ops.GraphKeys.TRAINABLE_VARIABLES)
-            l = g.get_collection_ref(ops.GraphKeys.TRAINABLE_VARIABLES)
-            l.remove(v)
-          g.add_to_collections(collections, wrapped)
-        elif ops.GraphKeys.GLOBAL_STEP in collections:
-          ops.add_to_collections(ops.GraphKeys.GLOBAL_STEP, wrapped)
-
-        return wrapped
-    else:
-      var_creator = next_creator
-
-    if "colocate_with" in kwargs:
-      with ops.device(None):
-        with ops.colocate_with(kwargs["colocate_with"]):
-          return var_creator(*args, **kwargs)
-
-    with ops.colocate_with(None, ignore_existing=True):
-      with ops.device(self._variable_device):
-        return var_creator(*args, **kwargs)
-
-  def _call_for_each_replica(self, fn, args, kwargs):
-    # pylint: disable=protected-access
-    return mirrored_strategy._call_for_each_replica(
-        self._container_strategy(), self._device_map, fn, args, kwargs)
-
-  def _verify_destinations_not_different_worker(self, destinations):
-    if not self._cluster_spec:
-      return
-    if destinations is None:
-      return
-    for d in cross_device_ops_lib.get_devices_from(destinations):
-      d_spec = tf_device.DeviceSpec.from_string(d)
-      if d_spec.job == self._task_type and d_spec.task != self._task_id:
-        raise ValueError(
-            "Cannot reduce to another worker: %r, current worker is %r" %
-            (d, self._input_workers.worker_devices[0]))
-
-  def _reduce_to(self, reduce_op, value, destinations):
-    self._verify_destinations_not_different_worker(destinations)
-    if not isinstance(value, values.DistributedValues):
-      # pylint: disable=protected-access
-      return cross_device_ops_lib.reduce_non_distributed_value(
-          reduce_op, self._device_map, value, destinations)
-    return self._cross_device_ops.reduce(
-        reduce_op, value, destinations=destinations)
-
-  def _batch_reduce_to(self, reduce_op, value_destination_pairs):
-    for _, destinations in value_destination_pairs:
-      self._verify_destinations_not_different_worker(destinations)
-    return self._cross_device_ops.batch_reduce(reduce_op,
-                                               value_destination_pairs)
-
-  def _select_single_value(self, structured):
-    """Select any single values in `structured`."""
-
-    def _select_fn(x):  # pylint: disable=g-missing-docstring
-      if isinstance(x, values.Mirrored):
-        if len(x.devices) == 1:
-          return x.primary
-        else:
-          raise ValueError(
-              "You cannot update variable with a Mirrored object with multiple "
-              "components %r when using ParameterServerStrategy. You must "
-              "specify a single value or a Mirrored with a single value." % x)
-      elif isinstance(x, values.PerReplica):
-        raise ValueError(
-            "You cannot update variable with a PerReplica object %r when using "
-            "ParameterServerStrategy. You must specify a single value or a "
-            "Mirrored with a single value" % x)
-      else:
-        return x
-
-    return nest.map_structure(_select_fn, structured)
-
-  def _update(self, var, fn, args, kwargs, group):
-    if isinstance(var, values.AggregatingVariable):
-      var = var.get()
-    if not isinstance(var, resource_variable_ops.ResourceVariable):
-      raise ValueError(
-          "You can not update `var` %r. It must be a Variable." % var)
-    with ops.colocate_with(var), distribute_lib.UpdateContext(var.device):
-      result = fn(var, *self._select_single_value(args),
-                  **self._select_single_value(kwargs))
-      if group:
-        return result
-      else:
-        return nest.map_structure(self._unwrap, result)
-
-  # TODO(yuefengz): does it need to call _select_single_value?
-  def _update_non_slot(self, colocate_with, fn, args, kwargs, group):
-    with ops.device(
-        colocate_with.device), distribute_lib.UpdateContext(colocate_with):
-      result = fn(*args, **kwargs)
-      if group:
-        return result
-      else:
-        return nest.map_structure(self._unwrap, result)
-
-  def _unwrap(self, val):
-    if isinstance(val, values.DistributedValues):
-      return val.values
-    return (val,)
-
-  def value_container(self, val):
-    if (hasattr(val, "_aggregating_container") and
-        not isinstance(val, values.AggregatingVariable)):
-      wrapper = val._aggregating_container()  # pylint: disable=protected-access
-      if wrapper is not None:
-        return wrapper
-    return val
-
-  def read_var(self, var):
-    # No need to distinguish between normal variables and replica-local
-    # variables.
-    return array_ops.identity(var)
-
-  def _configure(self,
-                 session_config=None,
-                 cluster_spec=None,
-                 task_type=None,
-                 task_id=None):
-    """Configures the strategy class.
-
-    The strategy object will be re-initialized if `cluster_spec` is given but
-    was not passed in the constructor.
-
-    Args:
-      session_config: not used currently.
-      cluster_spec: a dict, ClusterDef or ClusterSpec object specifying the
-        cluster configurations.
-      task_type: the current task type.
-      task_id: the current task id.
-
-    Raises:
-      ValueError: if `cluster_spec` is given but `task_type` or `task_id` is
-        not.
-    """
-    if not self._cluster_spec and cluster_spec:
-      # If a `cluster_spec` is already passed in, do nothing here.
-      # TODO(yuefengz): check `cluster_spec` is the same if this object has
-      # already been initialized with a `cluster_spec`.
-      if task_type is None or task_id is None:
-        raise ValueError("When `cluster_spec` is given, must also specify "
-                         "`task_type` and `task_id`.")
-      self._cluster_spec = multi_worker_util.normalize_cluster_spec(
-          cluster_spec)
-      self._task_type = task_type
-      self._task_id = task_id
-      self._initialize_multi_worker(self._num_gpus_per_worker,
-                                    self._cluster_spec, task_type, task_id)
-
-    if session_config:
-      session_config.CopyFrom(self._update_config_proto(session_config))
-
-  def _update_config_proto(self, config_proto):
-    updated_config = copy.deepcopy(config_proto)
-    if not self._cluster_spec:
-      updated_config.isolate_session_state = True
-      return updated_config
-
-    updated_config.isolate_session_state = False
-
-    assert self._task_type
-    assert self._task_id is not None
-
-    # The device filters prevent communication between workers.
-    if self._task_type not in ["chief", "worker"]:
-      return updated_config
-    del updated_config.device_filters[:]
-    updated_config.device_filters.extend(
-        ["/job:%s/task:%d" % (self._task_type, self._task_id), "/job:ps"])
-    return updated_config
-
-  @property
-  def _num_replicas_in_sync(self):
-    return self._device_map.num_replicas_in_graph
-
-  @property
-  def worker_devices(self):
-    return self._device_map.all_devices
-
-  @property
-  def worker_devices_by_replica(self):
-    return self._device_map.devices_by_replica
-
-  @property
-  def parameter_devices(self):
-    return self._parameter_devices
-
-  def non_slot_devices(self, var_list):
-    return min(var_list, key=lambda x: x.name)
-
-  @property
-  def experimental_between_graph(self):
-    # TODO(yuefengz): Should this return False in the local case?
-    return True
-
-  @property
-  def experimental_should_init(self):
-    return self._is_chief
-
-  @property
-  def should_checkpoint(self):
-    return self._is_chief
-
-  @property
-  def should_save_summary(self):
-    return self._is_chief
+    return input_lib.DatasetIterator(dataset, self._input_workers)
 
   # TODO(priyag): Delete this once all strategies use global batch size.
   @property
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
index ce7065f220..d8c6c50db2 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
@@ -29,10 +29,13 @@ from tensorflow.contrib.distribute.python import strategy_test_lib
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import device_util
+from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import distribution_strategy_context as ds_context
 from tensorflow.python.distribute import multi_worker_util
+from tensorflow.python.distribute import parameter_server_strategy as core_parameter_server_strategy
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.distribute import values
+from tensorflow.python.distribute.cluster_resolver import SimpleClusterResolver
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
 from tensorflow.python.estimator import run_config
@@ -50,6 +53,7 @@ from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 from tensorflow.python.training import training_util
+from tensorflow.python.training.server_lib import ClusterSpec
 
 CHIEF = run_config.TaskType.CHIEF
 WORKER = run_config.TaskType.WORKER
@@ -63,6 +67,57 @@ def _get_replica_id_integer():
   return replica_id
 
 
+class MockCoreParameterServerStrategy(distribute_lib.DistributionStrategy):
+  """Mock the strategy to allow cluster resolver as an argument."""
+
+  def __init__(self, cluster_resolver):
+    super(MockCoreParameterServerStrategy, self).__init__(
+        core_parameter_server_strategy.ParameterServerStrategyExtended(
+            self, cluster_resolver=cluster_resolver))
+
+
+def create_test_objects(cluster_spec=None,
+                        task_type=None,
+                        task_id=None,
+                        num_gpus=None,
+                        sess_config=None,
+                        use_core_strategy=False):
+  sess_config = sess_config or config_pb2.ConfigProto()
+  if num_gpus is None:
+    num_gpus = context.num_gpus()
+  if use_core_strategy:
+    if cluster_spec and task_type and task_id is not None:
+      cluster_resolver = SimpleClusterResolver(
+          cluster_spec=multi_worker_util.normalize_cluster_spec(cluster_spec),
+          task_type=task_type,
+          task_index=task_id,
+          num_accelerators=num_gpus)
+      target = 'grpc://' + cluster_spec[WORKER][task_id]
+    else:
+      cluster_resolver = SimpleClusterResolver(
+          ClusterSpec({}), num_accelerators=num_gpus)
+      target = ''
+
+    distribution = MockCoreParameterServerStrategy(cluster_resolver)
+    sess_config = copy.deepcopy(sess_config)
+    sess_config = distribution.update_config_proto(sess_config)
+  else:
+    distribution = parameter_server_strategy.ParameterServerStrategy(
+        num_gpus_per_worker=num_gpus)
+    if task_type:
+      sess_config = copy.deepcopy(sess_config)
+      distribution.configure(
+          session_config=sess_config,
+          cluster_spec=cluster_spec,
+          task_type=task_type,
+          task_id=task_id)
+      target = 'grpc://' + cluster_spec[WORKER][task_id]
+    else:
+      target = ''
+
+  return distribution, target, sess_config
+
+
 class ParameterServerStrategyTestBase(
     multi_worker_test_base.MultiWorkerTestBase):
 
@@ -76,24 +131,27 @@ class ParameterServerStrategyTestBase(
     self._sess_config = config_pb2.ConfigProto(allow_soft_placement=True)
     super(ParameterServerStrategyTestBase, self).setUp()
 
-  def _get_test_objects(self, task_type, task_id, num_gpus):
-    distribution = parameter_server_strategy.ParameterServerStrategy(
-        num_gpus_per_worker=num_gpus)
-    if not task_type:
-      return distribution, '', self._sess_config
-
-    sess_config = copy.deepcopy(self._sess_config)
-    distribution.configure(
-        session_config=sess_config,
+  def _get_test_objects(self,
+                        task_type,
+                        task_id,
+                        num_gpus,
+                        use_core_strategy=False):
+    return create_test_objects(
         cluster_spec=self._cluster_spec,
         task_type=task_type,
-        task_id=task_id)
-    return (distribution, 'grpc://' + self._cluster_spec[WORKER][task_id],
-            sess_config)
-
-  def _test_device_assignment_distributed(self, task_type, task_id, num_gpus):
+        task_id=task_id,
+        num_gpus=num_gpus,
+        sess_config=self._sess_config,
+        use_core_strategy=use_core_strategy)
+
+  def _test_device_assignment_distributed(self,
+                                          task_type,
+                                          task_id,
+                                          num_gpus,
+                                          use_core_strategy=False):
     worker_device = '/job:%s/replica:0/task:%d' % (task_type, task_id)
-    d, _, sess_config = self._get_test_objects(task_type, task_id, num_gpus)
+    d, _, sess_config = self._get_test_objects(
+        task_type, task_id, num_gpus, use_core_strategy=use_core_strategy)
     with ops.Graph().as_default(), \
          self.cached_session(target=self._default_target,
                              config=sess_config) as sess, \
@@ -191,8 +249,9 @@ class ParameterServerStrategyTestBase(
         self.assertEqual(f_val, 46.0)
 
   def _test_device_assignment_distributed_enable_partitioner(
-      self, task_type, task_id, num_gpus):
-    d, _, sess_config = self._get_test_objects(task_type, task_id, num_gpus)
+      self, task_type, task_id, num_gpus, use_core_strategy=False):
+    d, _, sess_config = self._get_test_objects(
+        task_type, task_id, num_gpus, use_core_strategy=use_core_strategy)
     num_shards = len(d.parameter_devices)
     partitioner = partitioned_variables.fixed_size_partitioner(num_shards)
     with ops.Graph().as_default(), \
@@ -340,9 +399,13 @@ class ParameterServerStrategyTestBase(
         self.assertEqual(z_val, 43.0)
         self.assertEqual(f_val, 46.0)
 
-  def _test_simple_increment(self, task_type, task_id, num_gpus):
+  def _test_simple_increment(self,
+                             task_type,
+                             task_id,
+                             num_gpus,
+                             use_core_strategy=False):
     d, master_target, sess_config = self._get_test_objects(
-        task_type, task_id, num_gpus)
+        task_type, task_id, num_gpus, use_core_strategy=use_core_strategy)
     if d.extended._cluster_spec:
       num_workers = len(d.extended._cluster_spec.as_dict().get(WORKER))
       if 'chief' in d.extended._cluster_spec.as_dict():
@@ -410,9 +473,13 @@ class ParameterServerStrategyTestBase(
               y_val == 20.0 + 1.0 * num_workers * d.num_replicas_in_sync and
               z_val == 30.0 + 1.0 * num_workers)
 
-  def _test_minimize_loss_graph(self, task_type, task_id, num_gpus):
+  def _test_minimize_loss_graph(self,
+                                task_type,
+                                task_id,
+                                num_gpus,
+                                use_core_strategy=False):
     d, master_target, sess_config = self._get_test_objects(
-        task_type, task_id, num_gpus)
+        task_type, task_id, num_gpus, use_core_strategy=use_core_strategy)
     if task_type:
       # Multi-worker
       assert hasattr(d.extended, '_cluster_spec') and d.extended._cluster_spec
@@ -498,10 +565,15 @@ class ParameterServerStrategyTestBase(
       self.assertLess(error_after, error_before)
       return error_after < error_before
 
-  def _test_input_fn_iterator(self, task_type, task_id, num_gpus, input_fn,
-                              expected_values):
+  def _test_input_fn_iterator(self,
+                              task_type,
+                              task_id,
+                              num_gpus,
+                              input_fn,
+                              expected_values,
+                              use_core_strategy=False):
     distribution, master_target, config = self._get_test_objects(
-        task_type, task_id, num_gpus)
+        task_type, task_id, num_gpus, use_core_strategy=use_core_strategy)
     devices = distribution.extended.worker_devices
 
     with ops.Graph().as_default(), \
@@ -541,66 +613,93 @@ class ParameterServerStrategyTest(ParameterServerStrategyTestBase,
         num_workers=3, num_ps=2)
     cls._default_target = 'grpc://' + cls._cluster_spec[WORKER][0]
 
-  def test_num_replicas_in_sync(self):
-    distribution = parameter_server_strategy.ParameterServerStrategy(
-        num_gpus_per_worker=2)
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def test_num_replicas_in_sync(self, use_core_strategy):
+    strategy, _, _ = create_test_objects(
+        num_gpus=2, use_core_strategy=use_core_strategy)
     # All the devices on a given worker are in sync which in this case is the
     # number of gpus on each worker.
-    self.assertEqual(2, distribution.num_replicas_in_sync)
+    self.assertEqual(2, strategy.num_replicas_in_sync)
 
-  def testDeviceAssignmentLocalCPU(self):
-    distribution = parameter_server_strategy.ParameterServerStrategy(
-        num_gpus_per_worker=0)
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testDeviceAssignmentLocalCPU(self, use_core_strategy):
+    strategy, _, _ = create_test_objects(
+        num_gpus=0, use_core_strategy=use_core_strategy)
     self._test_device_assignment_local(
-        distribution, compute_device='CPU', variable_device='CPU', num_gpus=0)
+        strategy, compute_device='CPU', variable_device='CPU', num_gpus=0)
 
-  def testDeviceAssignmentLocalOneGPU(self):
-    distribution = parameter_server_strategy.ParameterServerStrategy(
-        num_gpus_per_worker=1)
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testDeviceAssignmentLocalOneGPU(self, use_core_strategy):
+    strategy, _, _ = create_test_objects(
+        num_gpus=1, use_core_strategy=use_core_strategy)
     self._test_device_assignment_local(
-        distribution, compute_device='GPU', variable_device='GPU', num_gpus=1)
+        strategy, compute_device='GPU', variable_device='GPU', num_gpus=1)
 
-  def testDeviceAssignmentLocalTwoGPUs(self):
-    distribution = parameter_server_strategy.ParameterServerStrategy(
-        num_gpus_per_worker=2)
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testDeviceAssignmentLocalTwoGPUs(self, use_core_strategy):
+    strategy, _, _ = create_test_objects(
+        num_gpus=2, use_core_strategy=use_core_strategy)
     self._test_device_assignment_local(
-        distribution, compute_device='GPU', variable_device='CPU', num_gpus=2)
+        strategy, compute_device='GPU', variable_device='CPU', num_gpus=2)
 
   @combinations.generate(
-      combinations.combine(mode=['graph'], num_gpus=[0, 1, 2]))
-  def testDeviceAssignmentDistributed(self, num_gpus):
-    self._test_device_assignment_distributed('worker', 1, num_gpus)
+      combinations.combine(
+          mode=['graph'], num_gpus=[0, 1, 2], use_core_strategy=[True, False]))
+  def testDeviceAssignmentDistributed(self, num_gpus, use_core_strategy):
+    self._test_device_assignment_distributed(
+        'worker', 1, num_gpus, use_core_strategy=use_core_strategy)
 
   @combinations.generate(
-      combinations.combine(mode=['graph'], num_gpus=[0, 1, 2]))
-  def testDeviceAssignmentDistributedEnablePartitioner(self, num_gpus):
+      combinations.combine(
+          mode=['graph'], num_gpus=[0, 1, 2], use_core_strategy=[True, False]))
+  def testDeviceAssignmentDistributedEnablePartitioner(self, num_gpus,
+                                                       use_core_strategy):
     self._test_device_assignment_distributed_enable_partitioner(
-        'worker', 1, num_gpus)
+        'worker', 1, num_gpus, use_core_strategy=use_core_strategy)
 
-  def testSimpleBetweenGraph(self):
-    self._run_between_graph_clients(self._test_simple_increment,
-                                    self._cluster_spec, context.num_gpus())
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testSimpleBetweenGraph(self, use_core_strategy):
+    self._run_between_graph_clients(
+        self._test_simple_increment,
+        self._cluster_spec,
+        context.num_gpus(),
+        use_core_strategy=use_core_strategy)
 
   @combinations.generate(
-      combinations.combine(mode=['graph'], num_gpus=[0, 1, 2]))
-  def testLocalSimpleIncrement(self, num_gpus):
-    self._test_simple_increment(None, 0, num_gpus)
+      combinations.combine(
+          mode=['graph'], num_gpus=[0, 1, 2], use_core_strategy=[True, False]))
+  def testLocalSimpleIncrement(self, num_gpus, use_core_strategy):
+    self._test_simple_increment(None, 0, num_gpus, use_core_strategy)
 
   @combinations.generate(
-      combinations.combine(mode=['graph'], num_gpus=[0, 1, 2]))
-  def testMinimizeLossGraphDistributed(self, num_gpus):
-    self._run_between_graph_clients(self._test_minimize_loss_graph,
-                                    self._cluster_spec, num_gpus)
+      combinations.combine(
+          mode=['graph'], num_gpus=[0, 1, 2], use_core_strategy=[True, False]))
+  def testMinimizeLossGraphDistributed(self, num_gpus, use_core_strategy):
+    self._run_between_graph_clients(
+        self._test_minimize_loss_graph,
+        self._cluster_spec,
+        num_gpus,
+        use_core_strategy=use_core_strategy)
 
   @combinations.generate(
-      combinations.combine(mode=['graph'], num_gpus=[0, 1, 2]))
-  def testMinimizeLossGraphLocal(self, num_gpus):
-    self._test_minimize_loss_graph(None, None, num_gpus)
+      combinations.combine(
+          mode=['graph'], num_gpus=[0, 1, 2], use_core_strategy=[True, False]))
+  def testMinimizeLossGraphLocal(self, num_gpus, use_core_strategy):
+    self._test_minimize_loss_graph(None, None, num_gpus, use_core_strategy)
 
   # TODO(priyag): Refactor this and other multi worker tests.
   @combinations.generate(
-      combinations.combine(mode=['graph'], num_gpus=[1, 2], required_gpus=1))
-  def testMakeInputFnIteratorDistributed(self, num_gpus):
+      combinations.combine(
+          mode=['graph'],
+          num_gpus=[1, 2],
+          required_gpus=1,
+          use_core_strategy=[True, False]))
+  def testMakeInputFnIteratorDistributed(self, num_gpus, use_core_strategy):
     if context.num_gpus() < num_gpus:
       self.skipTest('Not enough GPUs')
     dataset_fn = lambda: dataset_ops.Dataset.range(100)
@@ -612,12 +711,21 @@ class ParameterServerStrategyTest(ParameterServerStrategyTestBase,
         expected_num_replicas_in_sync=num_gpus,
         expected_num_input_pipelines=3,
         expected_input_pipeline_id=1)  # because task_id = 1
-    self._test_input_fn_iterator('worker', 1, num_gpus,
-                                 input_fn, expected_values)
+    self._test_input_fn_iterator(
+        'worker',
+        1,
+        num_gpus,
+        input_fn,
+        expected_values,
+        use_core_strategy=use_core_strategy)
 
   @combinations.generate(
-      combinations.combine(mode=['graph'], num_gpus=[1, 2], required_gpus=1))
-  def testMakeInputFnIteratorLocal(self, num_gpus):
+      combinations.combine(
+          mode=['graph'],
+          num_gpus=[1, 2],
+          required_gpus=1,
+          use_core_strategy=[True, False]))
+  def testMakeInputFnIteratorLocal(self, num_gpus, use_core_strategy):
     if context.num_gpus() < num_gpus:
       self.skipTest('Not enough GPUs')
     dataset_fn = lambda: dataset_ops.Dataset.range(100)
@@ -629,23 +737,31 @@ class ParameterServerStrategyTest(ParameterServerStrategyTestBase,
         expected_num_replicas_in_sync=num_gpus,
         expected_num_input_pipelines=1,
         expected_input_pipeline_id=0)  # only one worker and pipeline for local.
-    self._test_input_fn_iterator(None, None, num_gpus,
-                                 input_fn, expected_values)
+    self._test_input_fn_iterator(
+        None,
+        None,
+        num_gpus,
+        input_fn,
+        expected_values,
+        use_core_strategy=use_core_strategy)
 
-  def testGlobalStepUpdate(self):
-    strategy = parameter_server_strategy.ParameterServerStrategy(
-        num_gpus_per_worker=context.num_gpus())
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testGlobalStepUpdate(self, use_core_strategy):
+    strategy, _, _ = create_test_objects(use_core_strategy=use_core_strategy)
     self._test_global_step_update(strategy)
 
-  def testUpdateConfigProtoMultiWorker(self):
-    distribution = parameter_server_strategy.ParameterServerStrategy(
-        num_gpus_per_worker=2)
-    distribution.configure(
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testUpdateConfigProtoMultiWorker(self, use_core_strategy):
+    strategy, _, _ = create_test_objects(
+        num_gpus=2, use_core_strategy=use_core_strategy)
+    strategy.configure(
         cluster_spec=self._cluster_spec, task_type='worker', task_id=1)
 
     config_proto = config_pb2.ConfigProto(device_filters=['to_be_overridden'])
 
-    new_config = distribution.update_config_proto(config_proto)
+    new_config = strategy.update_config_proto(config_proto)
 
     # Verify device filters.
     self.assertEqual(['/job:worker/task:1', '/job:ps'],
@@ -654,12 +770,14 @@ class ParameterServerStrategyTest(ParameterServerStrategyTestBase,
     # Verify isolate_session_state
     self.assertFalse(new_config.isolate_session_state)
 
-  def testUpdateConfigProtoLocal(self):
-    distribution = parameter_server_strategy.ParameterServerStrategy(
-        num_gpus_per_worker=2)
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testUpdateConfigProtoLocal(self, use_core_strategy):
+    strategy, _, _ = create_test_objects(
+        num_gpus=2, use_core_strategy=use_core_strategy)
 
     config_proto = config_pb2.ConfigProto()
-    new_config = distribution.update_config_proto(config_proto)
+    new_config = strategy.update_config_proto(config_proto)
 
     # Verify isolate_session_state
     self.assertTrue(new_config.isolate_session_state)
@@ -674,20 +792,31 @@ class ParameterServerStrategyWithChiefTest(ParameterServerStrategyTestBase,
         num_workers=3, num_ps=2, has_chief=True)
     cls._default_target = 'grpc://' + cls._cluster_spec[CHIEF][0]
 
-  def testSimpleBetweenGraph(self):
-    self._run_between_graph_clients(self._test_simple_increment,
-                                    self._cluster_spec, context.num_gpus())
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testSimpleBetweenGraph(self, use_core_strategy):
+    self._run_between_graph_clients(
+        self._test_simple_increment,
+        self._cluster_spec,
+        context.num_gpus(),
+        use_core_strategy=use_core_strategy)
 
   @combinations.generate(
-      combinations.combine(mode=['graph'], num_gpus=[0, 1, 2]))
-  def testMinimizeLossGraph(self, num_gpus):
-    self._run_between_graph_clients(self._test_minimize_loss_graph,
-                                    self._cluster_spec, num_gpus)
+      combinations.combine(
+          mode=['graph'], num_gpus=[0, 1, 2], use_core_strategy=[True, False]))
+  def testMinimizeLossGraph(self, num_gpus, use_core_strategy):
+    self._run_between_graph_clients(
+        self._test_minimize_loss_graph,
+        self._cluster_spec,
+        num_gpus,
+        use_core_strategy=use_core_strategy)
 
-  def testGlobalStepIsWrappedOnTwoGPUs(self):
-    distribution = parameter_server_strategy.ParameterServerStrategy(
-        num_gpus_per_worker=2)
-    with ops.Graph().as_default(), distribution.scope():
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testGlobalStepIsWrappedOnTwoGPUs(self, use_core_strategy):
+    strategy, _, _ = create_test_objects(
+        num_gpus=2, use_core_strategy=use_core_strategy)
+    with ops.Graph().as_default(), strategy.scope():
       created_step = training_util.create_global_step()
       get_step = training_util.get_global_step()
       self.assertEqual(created_step, get_step,
@@ -696,12 +825,14 @@ class ParameterServerStrategyWithChiefTest(ParameterServerStrategyTestBase,
                              id(get_step), get_step.__class__.__name__)))
       self.assertIs(values.AggregatingVariable, type(created_step))
       self.assertIs(values.AggregatingVariable, type(get_step))
-      self.assertIs(distribution, created_step.distribute_strategy)
+      self.assertIs(strategy, created_step.distribute_strategy)
 
-  def testGlobalStepIsNotWrappedOnOneGPU(self):
-    distribution = parameter_server_strategy.ParameterServerStrategy(
-        num_gpus_per_worker=1)
-    with ops.Graph().as_default(), distribution.scope():
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testGlobalStepIsNotWrappedOnOneGPU(self, use_core_strategy):
+    strategy, _, _ = create_test_objects(
+        num_gpus=1, use_core_strategy=use_core_strategy)
+    with ops.Graph().as_default(), strategy.scope():
       created_step = training_util.create_global_step()
       get_step = training_util.get_global_step()
       self.assertEqual(created_step, get_step,
@@ -710,20 +841,24 @@ class ParameterServerStrategyWithChiefTest(ParameterServerStrategyTestBase,
                              id(get_step), get_step.__class__.__name__)))
       self.assertIs(resource_variable_ops.ResourceVariable, type(created_step))
       self.assertIs(resource_variable_ops.ResourceVariable, type(get_step))
-      self.assertIs(distribution, created_step.distribute_strategy)
+      self.assertIs(strategy, created_step.distribute_strategy)
+
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testValueContainer(self, use_core_strategy):
+    strategy, _, _ = create_test_objects(
+        num_gpus=2, use_core_strategy=use_core_strategy)
+    with ops.Graph().as_default(), strategy.scope():
 
-  def testValueContainer(self):
-    distribution = parameter_server_strategy.ParameterServerStrategy(
-        num_gpus_per_worker=2)
-    with ops.Graph().as_default(), distribution.scope():
       def f():
         with backprop.GradientTape() as tape:
           v = variable_scope.get_variable('v', initializer=10.0)
           _ = v * v
         v, = tape.watched_variables()
-        w = distribution.extended.value_container(v)
+        w = strategy.extended.value_container(v)
         self.assertIs(values.AggregatingVariable, type(w))
-      distribution.extended.call_for_each_replica(f)
+
+      strategy.extended.call_for_each_replica(f)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/distribute/BUILD b/tensorflow/python/distribute/BUILD
index 987fb00454..faaa61934a 100644
--- a/tensorflow/python/distribute/BUILD
+++ b/tensorflow/python/distribute/BUILD
@@ -242,6 +242,28 @@ py_library(
     ],
 )
 
+py_library(
+    name = "parameter_server_strategy",
+    srcs = ["parameter_server_strategy.py"],
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        ":input_lib",
+        ":mirrored_strategy",
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:resource_variable_ops",
+        "//tensorflow/python:training",
+        "//tensorflow/python:util",
+        "//tensorflow/python/distribute:cross_device_ops",
+        "//tensorflow/python/distribute:multi_worker_util",
+        "//tensorflow/python/distribute:reduce_util",
+        "//tensorflow/python/distribute:values",
+        "//tensorflow/python/distribute/cluster_resolver:cluster_resolver_lib",
+        "//tensorflow/python/eager:context",
+    ],
+)
+
 py_library(
     name = "multi_worker_util",
     srcs = [
diff --git a/tensorflow/python/distribute/parameter_server_strategy.py b/tensorflow/python/distribute/parameter_server_strategy.py
new file mode 100644
index 0000000000..71fbffdc0d
--- /dev/null
+++ b/tensorflow/python/distribute/parameter_server_strategy.py
@@ -0,0 +1,528 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Classes implementing a multi-worker ps DistributionStrategy."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import copy
+
+from tensorflow.contrib.distribute.python import mirrored_strategy
+from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib
+from tensorflow.python.distribute import device_util
+from tensorflow.python.distribute import distribute_lib
+from tensorflow.python.distribute import input_lib
+from tensorflow.python.distribute import multi_worker_util
+from tensorflow.python.distribute import values
+from tensorflow.python.distribute.cluster_resolver import SimpleClusterResolver
+from tensorflow.python.distribute.cluster_resolver import TFConfigClusterResolver
+from tensorflow.python.eager import context
+from tensorflow.python.framework import device as tf_device
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import variable_scope as vs
+from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.training import device_setter
+from tensorflow.python.util import nest
+
+_LOCAL_CPU = "/device:CPU:0"
+_LOCAL_GPU_0 = "/device:GPU:0"
+
+
+# TODO(yuefengz): maybe cache variables on local CPU.
+class ParameterServerStrategy(distribute_lib.DistributionStrategy):
+  """A parameter server DistributionStrategy.
+
+  This strategy class works for both local training and between-graph replicated
+  training for multiple workers. It uses `TFConfigClusterResolver` to detect
+  configurations for multi-worker training. In multi-worker training mode, i.e.
+  `TFConfigClusterResolver` has detected 'TF_CONFIG' environment variable and
+  'TF_CONFIG' has a cluster spec, variables and updates to those variables are
+  assigned to parameter servers and other operations are assigned to workers.
+  In local training mode, variables are assigned to local CPU or the only GPU.
+  When each worker has more than one GPU, operations will be replicated on these
+  GPUs. In both cases, operations are replicated but variables are not and these
+  workers share a common view for which paramater server a variable is assigned
+  to.
+
+  This class assumes between-graph replication will be used and works on a graph
+  for a particular worker. Note that each graph and worker is independent.
+  This means that while each worker will synchronously compute a single gradient
+  update across all GPUs, updates between workers proceed asynchronously.
+  Operations that occur only on the first replica (such as incrementing the
+  global step), will occur on the first replica *of every worker*.
+
+  It is expected to call `call_for_each_replica(fn, ...)` for any
+  operations which potentially can be replicated across replicas (i.e. multiple
+  GPUs) even if there is only CPU or one GPU. When defining the `fn`, extra
+  caution needs to be taken:
+
+  1) It is generally not recommended to open a device scope under the strategy's
+  scope. A device scope (i.e. calling `tf.device`) will be merged with or
+  override the device for operations but will not change the device for
+  variables.
+
+  2) It is also not recommended to open a colocation scope (i.e. calling
+  `tf.colocate_with`) under the strategy's scope. For colocating variables,
+  use `distribution.colocate_vars_with` instead. Colocation of ops will possibly
+  create conflicts of device assignment.
+  """
+
+  def __init__(self):
+    """Initializes this strategy with default TFConfigClusterResolver."""
+    super(ParameterServerStrategy, self).__init__(
+        ParameterServerStrategyExtended(self))
+
+
+class ParameterServerStrategyExtended(
+    distribute_lib.DistributionStrategyExtended):
+  """Implementation of ParameterServerStrategy."""
+
+  def __init__(self,
+               container_strategy,
+               cluster_resolver=TFConfigClusterResolver()):
+    super(ParameterServerStrategyExtended, self).__init__(container_strategy)
+    self._initialize_strategy(cluster_resolver)
+
+    # We typically don't need to do all-reduce in this strategy.
+    self._cross_device_ops = (
+        cross_device_ops_lib.ReductionToOneDeviceCrossDeviceOps(
+            reduce_to_device=_LOCAL_CPU))
+
+  def _initialize_strategy(self, cluster_resolver):
+    if cluster_resolver.cluster_spec().as_dict():
+      self._initialize_multi_worker(cluster_resolver)
+    else:
+      self._initialize_local(cluster_resolver)
+    # Save the num_gpus_per_worker for configure method.
+    self._num_gpus_per_worker = cluster_resolver.num_accelerators()
+
+  def _initialize_multi_worker(self, cluster_resolver):
+    """Initialize devices for multiple workers.
+
+    It creates variable devices and compute devices. Variables and operations
+    will be assigned to them respectively. We have one compute device per
+    replica. The variable device is a device function or device string. The
+    default variable device assigns variables to parameter servers in a
+    round-robin fashion.
+
+    Args:
+      cluster_resolver: a descendant of `ClusterResolver` object.
+
+    Raises:
+      ValueError: if the cluster doesn't have ps jobs.
+    """
+    num_gpus = cluster_resolver.num_accelerators()
+    cluster_spec = cluster_resolver.cluster_spec()
+    task_type = cluster_resolver.task_type
+    task_id = cluster_resolver.task_index
+    if not task_type or task_id is None:
+      raise ValueError("When `cluster_spec` is given, you must also specify "
+                       "`task_type` and `task_id`")
+    cluster_spec = multi_worker_util.normalize_cluster_spec(cluster_spec)
+    assert cluster_spec.as_dict()
+
+    worker_device = "/job:%s/task:%d" % (task_type, task_id)
+
+    # Define compute devices which is a list of device strings and one for each
+    # replica. When there are GPUs, replicate operations on these GPUs.
+    # Otherwise, place operations on CPU.
+    if num_gpus > 0:
+      compute_devices = tuple(
+          "%s/device:GPU:%d" % (worker_device, i) for i in range(num_gpus))
+    else:
+      compute_devices = (worker_device,)
+
+    self._device_map = values.ReplicaDeviceMap(compute_devices)
+    self._input_workers = input_lib.InputWorkers(
+        self._device_map, [(worker_device, compute_devices)])
+
+    # In distributed mode, place variables on ps jobs in a round-robin fashion.
+    # Note that devices returned from `replica_device_setter` are not
+    # canonical and therefore we don't canonicalize all variable devices to
+    # make them consistent.
+    # TODO(yuefengz): support passing a strategy object to control variable
+    # assignment.
+    # TODO(yuefengz): merge the logic of replica_device_setter into this
+    # class.
+    num_ps_replicas = len(cluster_spec.as_dict().get("ps", []))
+    if num_ps_replicas == 0:
+      raise ValueError("The cluster spec needs to have `ps` jobs.")
+    self._variable_device = device_setter.replica_device_setter(
+        ps_tasks=num_ps_replicas,
+        worker_device=worker_device,
+        merge_devices=True,
+        cluster=cluster_spec)
+
+    # The `_parameter_devices` is needed for the `parameter_devices` property
+    # and is a list of all variable devices. Here parameter devices are all
+    # tasks of the "ps" job.
+    self._parameter_devices = tuple(map("/job:ps/task:{}".format,
+                                        range(num_ps_replicas)))
+
+    # Add a default device so that ops without specified devices will not end up
+    # on other workers.
+    self._default_device = worker_device
+
+    self._is_chief = multi_worker_util.is_chief(cluster_spec, task_type,
+                                                task_id)
+    self._cluster_spec = cluster_spec
+    self._task_type = task_type
+    self._task_id = task_id
+
+    logging.info(
+        "Multi-worker ParameterServerStrategy with "
+        "cluster_spec = %r, task_type = %r, task_id = %r, "
+        "num_ps_replicas = %r, is_chief = %r, device_map = %r, "
+        "variable_device = %r", cluster_spec.as_dict(), task_type, task_id,
+        num_ps_replicas, self._is_chief, self._device_map,
+        self._variable_device)
+
+  def _initialize_local(self, cluster_resolver):
+    """Initialize internal devices for local training."""
+    worker_device = device_util.canonicalize("/device:CPU:0")
+    num_gpus = cluster_resolver.num_accelerators()
+    # Define compute devices which is a list of device strings and one for each
+    # replica. When there are GPUs, replicate operations on these GPUs.
+    # Otherwise, place operations on CPU.
+    if num_gpus > 0:
+      compute_devices = tuple(map("/device:GPU:{}".format, range(num_gpus)))
+    else:
+      compute_devices = (_LOCAL_CPU,)
+
+    self._device_map = values.ReplicaDeviceMap(compute_devices)
+    self._input_workers = input_lib.InputWorkers(
+        self._device_map, [(worker_device, compute_devices)])
+
+    # If there is only one GPU, put everything on that GPU. Otherwise, place
+    # variables on CPU.
+    if num_gpus == 1:
+      assert len(compute_devices) == 1
+      self._variable_device = _LOCAL_GPU_0
+      self._parameter_devices = (_LOCAL_GPU_0,)
+    else:
+      self._variable_device = _LOCAL_CPU
+      self._parameter_devices = (_LOCAL_CPU,)
+
+    self._is_chief = True
+    self._cluster_spec = None
+    self._task_type = None
+    self._task_id = None
+
+    logging.info(
+        "ParameterServerStrategy with compute_devices = %r, "
+        "variable_device = %r", compute_devices, self._variable_device)
+
+  def _validate_colocate_with_variable(self, colocate_with_variable):
+    values.validate_colocate(colocate_with_variable, self)
+
+  def _distribute_dataset(self, dataset_fn):
+    """Distributes the dataset to each local GPU."""
+    return input_lib.PerReplicaDataset(
+        self._call_dataset_fn(dataset_fn),
+        self._input_workers,
+        0,
+        prefetch_on_device=True)
+
+  def _make_dataset_iterator(self, dataset):
+    return input_lib.DatasetIterator(dataset, self._input_workers,
+                                     self._num_replicas_in_sync)
+
+  def _make_input_fn_iterator(
+      self,
+      input_fn,
+      replication_mode=distribute_lib.InputReplicationMode.PER_WORKER):
+    """Distributes the dataset to each local GPU."""
+    if self._cluster_spec:
+      input_pipeline_id = multi_worker_util.id_in_cluster(
+          self._cluster_spec, self._task_type, self._task_id)
+      num_input_pipelines = multi_worker_util.worker_count(
+          self._cluster_spec, self._task_type)
+    else:
+      input_pipeline_id = 0
+      num_input_pipelines = 1
+    input_context = distribute_lib.InputContext(
+        num_input_pipelines=num_input_pipelines,
+        input_pipeline_id=input_pipeline_id,
+        num_replicas_in_sync=self._num_replicas_in_sync)
+    return input_lib.InputFunctionIterator(input_fn, self._input_workers,
+                                           [input_context])
+
+  def _broadcast_to(self, tensor, destinations):
+    # This is both a fast path for Python constants, and a way to delay
+    # converting Python values to a tensor until we know what type it
+    # should be converted to. Otherwise we have trouble with:
+    #   global_step.assign_add(1)
+    # since the `1` gets broadcast as an int32 but global_step is int64.
+    if isinstance(tensor, (float, int)):
+      return tensor
+    if not cross_device_ops_lib.check_destinations(destinations):
+      # TODO(josh11b): Use current logical device instead of 0 here.
+      destinations = values.LogicalDeviceSpec(
+          device_map=self._device_map, logical_device=0)
+    return self._cross_device_ops.broadcast(tensor, destinations)
+
+  def _allow_variable_partition(self):
+    return not context.executing_eagerly()
+
+  # TODO(yuefengz): not all ops in device_setter.STANDARD_PS_OPS will go through
+  # this creator, such as "MutableHashTable".
+  def _create_variable(self, next_creator, *args, **kwargs):
+    if self._num_replicas_in_sync > 1:
+      aggregation = kwargs.pop("aggregation", vs.VariableAggregation.NONE)
+      if aggregation not in (
+          vs.VariableAggregation.NONE,
+          vs.VariableAggregation.SUM,
+          vs.VariableAggregation.MEAN,
+          vs.VariableAggregation.ONLY_FIRST_REPLICA
+      ):
+        raise ValueError("Invalid variable aggregation mode: " + aggregation +
+                         " for variable: " + kwargs["name"])
+
+      def var_creator(*args, **kwargs):
+        """Create an AggregatingVariable and fix up collections."""
+        # Record what collections this variable should be added to.
+        collections = kwargs.pop("collections", None)
+        if collections is None:
+          collections = [ops.GraphKeys.GLOBAL_VARIABLES]
+        kwargs["collections"] = []
+
+        # Create and wrap the variable.
+        v = next_creator(*args, **kwargs)
+        wrapped = values.AggregatingVariable(
+            self._container_strategy(), v, aggregation)
+
+        # Add the wrapped variable to the requested collections.
+        # The handling of eager mode and the global step matches
+        # ResourceVariable._init_from_args().
+        if not context.executing_eagerly():
+          g = ops.get_default_graph()
+          # If "trainable" is True, next_creator() will add the contained
+          # variable to the TRAINABLE_VARIABLES collection, so we manually
+          # remove it and replace with the wrapper. We can't set "trainable"
+          # to False for next_creator() since that causes functions like
+          # implicit_gradients to skip those variables.
+          if kwargs.get("trainable", True):
+            collections.append(ops.GraphKeys.TRAINABLE_VARIABLES)
+            l = g.get_collection_ref(ops.GraphKeys.TRAINABLE_VARIABLES)
+            l.remove(v)
+          g.add_to_collections(collections, wrapped)
+        elif ops.GraphKeys.GLOBAL_STEP in collections:
+          ops.add_to_collections(ops.GraphKeys.GLOBAL_STEP, wrapped)
+
+        return wrapped
+    else:
+      var_creator = next_creator
+
+    if "colocate_with" in kwargs:
+      with ops.device(None):
+        with ops.colocate_with(kwargs["colocate_with"]):
+          return var_creator(*args, **kwargs)
+
+    with ops.colocate_with(None, ignore_existing=True):
+      with ops.device(self._variable_device):
+        return var_creator(*args, **kwargs)
+
+  def _call_for_each_replica(self, fn, args, kwargs):
+    # pylint: disable=protected-access
+    return mirrored_strategy._call_for_each_replica(
+        self._container_strategy(), self._device_map, fn, args, kwargs)
+
+  def _verify_destinations_not_different_worker(self, destinations):
+    if not self._cluster_spec:
+      return
+    if destinations is None:
+      return
+    for d in cross_device_ops_lib.get_devices_from(destinations):
+      d_spec = tf_device.DeviceSpec.from_string(d)
+      if d_spec.job == self._task_type and d_spec.task != self._task_id:
+        raise ValueError(
+            "Cannot reduce to another worker: %r, current worker is %r" %
+            (d, self._input_workers.worker_devices[0]))
+
+  def _reduce_to(self, reduce_op, value, destinations):
+    self._verify_destinations_not_different_worker(destinations)
+    if not isinstance(value, values.DistributedValues):
+      # pylint: disable=protected-access
+      return cross_device_ops_lib.reduce_non_distributed_value(
+          reduce_op, self._device_map, value, destinations)
+    return self._cross_device_ops.reduce(
+        reduce_op, value, destinations=destinations)
+
+  def _batch_reduce_to(self, reduce_op, value_destination_pairs):
+    for _, destinations in value_destination_pairs:
+      self._verify_destinations_not_different_worker(destinations)
+    return self._cross_device_ops.batch_reduce(reduce_op,
+                                               value_destination_pairs)
+
+  def _select_single_value(self, structured):
+    """Select any single values in `structured`."""
+
+    def _select_fn(x):  # pylint: disable=g-missing-docstring
+      if isinstance(x, values.Mirrored):
+        if len(x.devices) == 1:
+          return x.primary
+        else:
+          raise ValueError(
+              "You cannot update variable with a Mirrored object with multiple "
+              "components %r when using ParameterServerStrategy. You must "
+              "specify a single value or a Mirrored with a single value." % x)
+      elif isinstance(x, values.PerReplica):
+        raise ValueError(
+            "You cannot update variable with a PerReplica object %r when using "
+            "ParameterServerStrategy. You must specify a single value or a "
+            "Mirrored with a single value" % x)
+      else:
+        return x
+
+    return nest.map_structure(_select_fn, structured)
+
+  def _update(self, var, fn, args, kwargs, group):
+    if isinstance(var, values.AggregatingVariable):
+      var = var.get()
+    if not isinstance(var, resource_variable_ops.ResourceVariable):
+      raise ValueError(
+          "You can not update `var` %r. It must be a Variable." % var)
+    with ops.colocate_with(var), distribute_lib.UpdateContext(var.device):
+      result = fn(var, *self._select_single_value(args),
+                  **self._select_single_value(kwargs))
+      if group:
+        return result
+      else:
+        return nest.map_structure(self._unwrap, result)
+
+  # TODO(yuefengz): does it need to call _select_single_value?
+  def _update_non_slot(self, colocate_with, fn, args, kwargs, group):
+    with ops.device(
+        colocate_with.device), distribute_lib.UpdateContext(colocate_with):
+      result = fn(*args, **kwargs)
+      if group:
+        return result
+      else:
+        return nest.map_structure(self._unwrap, result)
+
+  def _unwrap(self, val):
+    if isinstance(val, values.DistributedValues):
+      return val.values
+    return (val,)
+
+  def value_container(self, val):
+    if (hasattr(val, "_aggregating_container") and
+        not isinstance(val, values.AggregatingVariable)):
+      wrapper = val._aggregating_container()  # pylint: disable=protected-access
+      if wrapper is not None:
+        return wrapper
+    return val
+
+  def read_var(self, var):
+    # No need to distinguish between normal variables and replica-local
+    # variables.
+    return array_ops.identity(var)
+
+  def _configure(self,
+                 session_config=None,
+                 cluster_spec=None,
+                 task_type=None,
+                 task_id=None):
+    """Configures the strategy class.
+
+    The strategy object will be re-initialized if `cluster_spec` is given but
+    was not passed in the constructor.
+
+    Args:
+      session_config: not used currently.
+      cluster_spec: a dict, ClusterDef or ClusterSpec object specifying the
+        cluster configurations.
+      task_type: the current task type.
+      task_id: the current task id.
+
+    Raises:
+      ValueError: if `cluster_spec` is given but `task_type` or `task_id` is
+        not.
+    """
+    if cluster_spec:
+      # Use the num_gpus_per_worker recorded in constructor since _configure
+      # doesn't take num_gpus.
+      cluster_resolver = SimpleClusterResolver(
+          cluster_spec=multi_worker_util.normalize_cluster_spec(cluster_spec),
+          task_type=task_type,
+          task_index=task_id,
+          num_accelerators=self._num_gpus_per_worker)
+      self._initialize_multi_worker(cluster_resolver)
+
+    if session_config:
+      session_config.CopyFrom(self._update_config_proto(session_config))
+
+  def _update_config_proto(self, config_proto):
+    updated_config = copy.deepcopy(config_proto)
+    if not self._cluster_spec:
+      updated_config.isolate_session_state = True
+      return updated_config
+
+    updated_config.isolate_session_state = False
+
+    assert self._task_type
+    assert self._task_id is not None
+
+    # The device filters prevent communication between workers.
+    if self._task_type not in ["chief", "worker"]:
+      return updated_config
+    del updated_config.device_filters[:]
+    updated_config.device_filters.extend(
+        ["/job:%s/task:%d" % (self._task_type, self._task_id), "/job:ps"])
+    return updated_config
+
+  @property
+  def _num_replicas_in_sync(self):
+    return self._device_map.num_replicas_in_graph
+
+  @property
+  def worker_devices(self):
+    return self._device_map.all_devices
+
+  @property
+  def worker_devices_by_replica(self):
+    return self._device_map.devices_by_replica
+
+  @property
+  def parameter_devices(self):
+    return self._parameter_devices
+
+  def non_slot_devices(self, var_list):
+    return min(var_list, key=lambda x: x.name)
+
+  @property
+  def experimental_between_graph(self):
+    # TODO(yuefengz): Should this return False in the local case?
+    return True
+
+  @property
+  def experimental_should_init(self):
+    return self._is_chief
+
+  @property
+  def should_checkpoint(self):
+    return self._is_chief
+
+  @property
+  def should_save_summary(self):
+    return self._is_chief
+
+  # TODO(priyag): Delete this once all strategies use global batch size.
+  @property
+  def _global_batch_size(self):
+    return True
-- 
GitLab


From cea2e9c4fab95890873114bf60a5f19facca0c2f Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Fri, 4 Jan 2019 13:17:37 -0800
Subject: [PATCH 0206/2345] Removed set_shape overrides from ResourceVariable
 subclasses.

These are essentially no-ops and allow for reads returning tensors inconsistent
with variable shape:

    >>> import tensorflow as tf
    >>> tf.enable_eager_execution()
    >>> v = tf.Variable(0)
    >>> t = v.assign(42)
    >>> t.set_shape((100500, ))
    >>> t.shape
    (100500, )
    >>> tf.convert_to_tensor(t)
    <tf.Tensor: id=9, shape=(), dtype=int32, numpy=42>

PiperOrigin-RevId: 227903155
---
 tensorflow/python/ops/resource_variable_ops.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index ede4bc0078..d2e5dd9cfe 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -1351,10 +1351,6 @@ class _UnreadVariable(ResourceVariable):
       return gen_resource_variable_ops.read_variable_op(self._handle,
                                                         self._dtype)
 
-  def set_shape(self, shape):
-    self._shape = shape
-    self._cached_shape_as_list = None
-
   @property
   def op(self):
     """The op for this variable."""
@@ -1443,10 +1439,6 @@ class _MixedPrecisionVariable(ResourceVariable):
       else:
         return res
 
-  def set_shape(self, shape):
-    self._shape = shape
-    self._cached_shape_as_list = None
-
   @property
   def op(self):
     """The op for this variable."""
-- 
GitLab


From 4614b04cc076c543501124670da8b8da417cbf9f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 13:49:49 -0800
Subject: [PATCH 0207/2345] One more refactoring of kernel.cc. This introduces
 OpInputs to match OpOutputs, which is slightly enhanced.

PiperOrigin-RevId: 227908106
---
 tensorflow/lite/delegates/flex/kernel.cc | 127 ++++++++++++++---------
 1 file changed, 80 insertions(+), 47 deletions(-)

diff --git a/tensorflow/lite/delegates/flex/kernel.cc b/tensorflow/lite/delegates/flex/kernel.cc
index 3012241f4f..0005b6e5b0 100644
--- a/tensorflow/lite/delegates/flex/kernel.cc
+++ b/tensorflow/lite/delegates/flex/kernel.cc
@@ -51,24 +51,57 @@ namespace tflite {
 namespace flex {
 namespace kernel {
 
-// Controls the lifetime of tensor handles in a vector.
+// A list of inputs of a given node of the TensorFlow/Eager graph.
+class OpInputs {
+ public:
+  explicit OpInputs(const TfLiteIntArray* indexes) {
+    for (int index : TfLiteIntArrayView(indexes)) {
+      inputs_.push_back(index);
+    }
+  }
+  ~OpInputs() {}
+
+  int Size() const { return inputs_.size(); }
+
+  int TfLiteIndex(int i) const { return inputs_[i]; }
+
+ private:
+  std::vector<int> inputs_;
+};
+
+// A list of outputs of a given node of the TensorFlow/Eager graph, along with
+// the actual outputs of the EagerOperation.
 class OpOutputs {
  public:
-  explicit OpOutputs(int num_elements) : vector_(num_elements, nullptr) {}
+  explicit OpOutputs(const TfLiteIntArray* indexes) {
+    for (int index : TfLiteIntArrayView(indexes)) {
+      outputs_.push_back(index);
+    }
+    vector_.resize(outputs_.size());
+  }
+  ~OpOutputs() { ResetTensorHandles(); }
+
+  int Size() const { return outputs_.size(); }
+
+  int TfLiteIndex(int i) const { return outputs_[i]; }
 
-  ~OpOutputs() {
-    for (auto* handle : vector_) {
-      if (handle) handle->Unref();
+  // Carefully unreference all the handles in the eager output vector.
+  void ResetTensorHandles() {
+    for (int i = 0; i < vector_.size(); ++i) {
+      if (vector_[i]) {
+        vector_[i]->Unref();
+        vector_[i] = nullptr;
+      }
     }
   }
 
-  tensorflow::gtl::InlinedVector<tensorflow::TensorHandle*, 2>* GetVector() {
+  tensorflow::gtl::InlinedVector<tensorflow::TensorHandle*, 2>*
+  GetTensorHandles() {
     return &vector_;
   }
 
-  tensorflow::TensorHandle* GetHandle(int index) { return vector_[index]; }
-
  private:
+  std::vector<int> outputs_;
   tensorflow::gtl::InlinedVector<tensorflow::TensorHandle*, 2> vector_;
 };
 
@@ -76,7 +109,8 @@ class OpOutputs {
 // TensorFlow ops within a single TF Lite op.
 class OpNode {
  public:
-  OpNode() {}
+  OpNode(const TfLiteIntArray* inputs, const TfLiteIntArray* outputs)
+      : inputs_(inputs), outputs_(outputs) {}
   ~OpNode() {}
 
   const string& name() const { return name_; }
@@ -87,19 +121,14 @@ class OpNode {
 
   const tensorflow::NodeDef& nodedef() const { return nodedef_; }
 
-  const std::vector<int>& inputs() const { return inputs_; }
-  void InitializeInputs(const TfLiteIntArray* inputs) {
-    for (int index : TfLiteIntArrayView(inputs)) {
-      inputs_.push_back(index);
-    }
-  }
+  const OpInputs& inputs() const { return inputs_; }
+  OpInputs* mutable_inputs() { return &inputs_; }
 
-  const std::vector<int>& outputs() const { return outputs_; }
-  void InitializeOutputs(const TfLiteIntArray* outputs) {
-    for (int index : TfLiteIntArrayView(outputs)) {
-      outputs_.push_back(index);
-    }
-  }
+  const OpOutputs& outputs() const { return outputs_; }
+  OpOutputs* mutable_outputs() { return &outputs_; }
+
+  int NumInputs() const { return inputs_.Size(); }
+  int NumOutputs() const { return outputs_.Size(); }
 
   tensorflow::Status InitializeNodeDef(const void* custom_initial_data,
                                        int custom_initial_data_size) {
@@ -162,7 +191,8 @@ class OpNode {
 
   tensorflow::Status BuildEagerInputs(BufferMap* buffer_map,
                                       tensorflow::EagerOperation* op) {
-    for (int input_index : inputs_) {
+    for (int i = 0; i < inputs_.Size(); ++i) {
+      int input_index = inputs_.TfLiteIndex(i);
       if (!buffer_map->HasTensor(input_index)) {
         return tensorflow::errors::Internal(
             "Cannot read from invalid tensor index ", input_index);
@@ -181,17 +211,21 @@ class OpNode {
     return tensorflow::Status::OK();
   }
 
-  tensorflow::Status PersistEagerOutputs(BufferMap* buffer_map,
-                                         OpOutputs* retvals) {
-    for (int i = 0; i < outputs_.size(); ++i) {
+  tensorflow::Status PersistEagerOutputs(BufferMap* buffer_map) {
+    auto* handles = outputs_.GetTensorHandles();
+    for (int i = 0; i < outputs_.Size(); ++i) {
       const tensorflow::Tensor* tensor = nullptr;
-      TF_RETURN_IF_ERROR(retvals->GetHandle(i)->Tensor(&tensor));
-      buffer_map->SetFromTensorFlow(outputs_[i], *tensor);
+      TF_RETURN_IF_ERROR(handles->at(i)->Tensor(&tensor));
+      buffer_map->SetFromTensorFlow(outputs_.TfLiteIndex(i), *tensor);
     }
+    outputs_.ResetTensorHandles();
     return tensorflow::Status::OK();
   }
 
  private:
+  OpNode(const OpNode&) = delete;
+  OpNode& operator=(const OpNode&) = delete;
+
   // The name of the TensorFlow op to execute.
   string name_;
   // Index of this node into TF Lite's operator list.
@@ -199,9 +233,9 @@ class OpNode {
   // The corresponding NodeDef, containing the attributes for the op.
   tensorflow::NodeDef nodedef_;
   // List of inputs, as TF Lite tensor indices.
-  std::vector<int> inputs_;
+  OpInputs inputs_;
   // List of outputs, as TF Lite tensor indices.
-  std::vector<int> outputs_;
+  OpOutputs outputs_;
 };
 
 // Executes the TensorFlow op given by 'op_name', with the attributes specified
@@ -212,18 +246,18 @@ tensorflow::Status ExecuteFlexOp(tensorflow::EagerContext* eager_context,
   TF_RETURN_IF_ERROR(node_data->BuildEagerOp(eager_context, &op));
   TF_RETURN_IF_ERROR(node_data->BuildEagerInputs(buffer_map, op.get()));
 
-  int num_retvals = node_data->outputs().size();
-  OpOutputs retvals(num_retvals);
+  int num_retvals = node_data->NumOutputs();
   TF_RETURN_WITH_CONTEXT_IF_ERROR(
-      EagerExecute(op.get(), retvals.GetVector(), &num_retvals),
+      EagerExecute(op.get(), node_data->mutable_outputs()->GetTensorHandles(),
+                   &num_retvals),
       " (while executing '", node_data->name(), "' via Eager)");
 
-  if (num_retvals != node_data->outputs().size()) {
+  if (num_retvals != node_data->NumOutputs()) {
     return tensorflow::errors::Internal(
         "Unexpected number of outputs from EagerExecute");
   }
 
-  TF_RETURN_IF_ERROR(node_data->PersistEagerOutputs(buffer_map, &retvals));
+  TF_RETURN_IF_ERROR(node_data->PersistEagerOutputs(buffer_map));
 
   return tensorflow::Status::OK();
 }
@@ -232,7 +266,7 @@ tensorflow::Status ExecuteFlexOp(tensorflow::EagerContext* eager_context,
 struct OpData {
   tensorflow::EagerContext* eager_context;
   BufferMap* buffer_map;
-  std::vector<OpNode> nodes;
+  std::vector<std::unique_ptr<OpNode>> nodes;
   std::vector<int> subgraph_inputs;
   std::vector<int> subgraph_outputs;
 };
@@ -261,6 +295,8 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) {
     op_data->subgraph_inputs.push_back(tensor_index);
   }
 
+  op_data->nodes.reserve(params->nodes_to_replace->size);
+
   CHECK(params->nodes_to_replace);
   tensorflow::Status status;
   for (auto node_index : TfLiteIntArrayView(params->nodes_to_replace)) {
@@ -268,8 +304,8 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) {
     TfLiteRegistration* reg;
     context->GetNodeAndRegistration(context, node_index, &node, &reg);
 
-    op_data->nodes.push_back(OpNode());
-    OpNode& node_data = op_data->nodes.back();
+    op_data->nodes.emplace_back(new OpNode(node->inputs, node->outputs));
+    OpNode& node_data = *op_data->nodes.back();
 
     node_data.set_index(node_index);
     node_data.set_name("");
@@ -277,9 +313,6 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) {
     status = node_data.InitializeNodeDef(node->custom_initial_data,
                                          node->custom_initial_data_size);
     if (!status.ok()) break;
-
-    node_data.InitializeInputs(node->inputs);
-    node_data.InitializeOutputs(node->outputs);
   }
 
   if (ConvertStatus(context, status) != kTfLiteOk) {
@@ -328,14 +361,14 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   }
 
   for (const auto& node_data : op_data->nodes) {
-    if (node_data.nodedef().op().empty()) {
+    if (node_data->nodedef().op().empty()) {
       context->ReportError(context, "Invalid NodeDef in Flex op '%s'",
-                           node_data.name().c_str());
+                           node_data->name().c_str());
       return kTfLiteError;
     }
 
-    for (int tensor_index : node_data.inputs()) {
-      ++tensor_ref_count[tensor_index];
+    for (int i = 0; i < node_data->inputs().Size(); ++i) {
+      ++tensor_ref_count[node_data->inputs().TfLiteIndex(i)];
     }
   }
 
@@ -371,12 +404,12 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   }
 
   // Execute the TensorFlow Ops sequentially.
-  for (OpNode& node_data : op_data->nodes) {
+  for (auto& node_data : op_data->nodes) {
     SCOPED_TAGGED_OPERATOR_PROFILE(
         reinterpret_cast<profiling::Profiler*>(context->profiler),
-        node_data.name().c_str(), node_data.index());
+        node_data->name().c_str(), node_data->index());
 
-    auto status = ExecuteFlexOp(eager_context, buffer_map, &node_data);
+    auto status = ExecuteFlexOp(eager_context, buffer_map, node_data.get());
     TF_LITE_ENSURE_OK(context, ConvertStatus(context, status));
   }
 
-- 
GitLab


From 962d8821ebb23e918bae3680bab5a7f6df32cd7f Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Fri, 4 Jan 2019 13:52:30 -0800
Subject: [PATCH 0208/2345] [XLA] Adapt HLO pipeline to scalar-index DS and DUS

This and adapts HLO to work with the scalar DynamicSlice / DynamicUpdateSlice
form. This means it also effectively switches the backends to use the new form.

Also a bunch of test churn - tests that don't run optimizations (and, hence,
don't trigger the splitter pass) needed to be converted.

PiperOrigin-RevId: 227908542
---
 .../xla/service/algebraic_simplifier.cc       | 27 ++++-----
 .../xla/service/algebraic_simplifier_test.cc  | 26 +++++----
 .../xla/service/buffer_assignment_test.cc     |  6 +-
 tensorflow/compiler/xla/service/cpu/BUILD     | 18 +++---
 .../compiler/xla/service/cpu/cpu_compiler.cc  |  2 +
 tensorflow/compiler/xla/service/gpu/BUILD     | 12 ++--
 .../service/gpu/multi_output_fusion_test.cc   |  5 +-
 .../xla/service/gpu/nvptx_compiler.cc         |  2 +
 .../xla/service/hlo_creation_utils.cc         | 37 ++++++++++--
 .../xla/service/hlo_evaluator_typed_visitor.h | 22 +-------
 .../compiler/xla/service/hlo_verifier.cc      | 19 +------
 .../compiler/xla/service/hlo_verifier_test.cc |  5 +-
 .../compiler/xla/service/interpreter/BUILD    |  9 +--
 .../xla/service/interpreter/compiler.cc       |  2 +
 .../xla/service/layout_assignment_test.cc     |  7 ++-
 .../compiler/xla/service/pattern_matcher.h    |  2 +-
 .../compiler/xla/service/shape_inference.cc   | 10 ++--
 .../compiler/xla/service/shape_inference.h    |  4 +-
 tensorflow/compiler/xla/shape_util.cc         |  1 +
 tensorflow/compiler/xla/tests/BUILD           |  1 +
 tensorflow/compiler/xla/tests/test_utils.cc   | 52 +++++++----------
 .../compiler/xla/tests/test_utils_test.cc     | 56 ++++++++++---------
 22 files changed, 162 insertions(+), 163 deletions(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index 003cb7e791..5ac746c9f3 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -1380,6 +1380,9 @@ StatusOr<HloInstruction*> AlgebraicSimplifierVisitor::OptimizeDotOfGather(
   // => output dimensions: DS ({M x N}, {0, start}, {M, 1}) => {M x 1}.
 
   bool lhs_is_dynamic_slice = lhs->opcode() == HloOpcode::kDynamicSlice;
+  HloDynamicSliceInstruction* dynamic_slice =
+      lhs_is_dynamic_slice ? Cast<HloDynamicSliceInstruction>(lhs)
+                           : Cast<HloDynamicSliceInstruction>(rhs);
 
   // ctA:
   HloInstruction* left_operand =
@@ -1397,8 +1400,6 @@ StatusOr<HloInstruction*> AlgebraicSimplifierVisitor::OptimizeDotOfGather(
       HloInstruction::CreateDot(memoized_shape, left_operand, right_operand,
                                 dnums, dot->precision_config()));
   // Get pair {start, 0} or {0, start}.
-  HloInstruction* original_start_indices =
-      lhs_is_dynamic_slice ? lhs->mutable_operand(1) : rhs->mutable_operand(1);
   // Position of start:
   int index_of_non_zero_start = lhs_is_dynamic_slice
                                     ? 1 - lhs_contracting_dimension
@@ -1407,23 +1408,19 @@ StatusOr<HloInstruction*> AlgebraicSimplifierVisitor::OptimizeDotOfGather(
   int index_of_zero_start = 1 - index_of_non_zero_start;
 
   // Slice out start and 0 components and reorder if necessary.
-  auto indices_type = original_start_indices->shape().element_type();
+  auto indices_type = dynamic_slice->operand(1)->shape().element_type();
   Shape s_shape = ShapeUtil::MakeShape(indices_type, {1});
   Shape d_shape = ShapeUtil::MakeShape(indices_type, {2});
   HloInstruction* non_zero_start =
-      computation_->AddInstruction(HloInstruction::CreateSlice(
-          s_shape, original_start_indices, {index_of_non_zero_start},
-          {index_of_non_zero_start + 1}, {1}));
+      dynamic_slice->mutable_operand(1 + index_of_non_zero_start);
   HloInstruction* zero_start =
-      computation_->AddInstruction(HloInstruction::CreateSlice(
-          s_shape, original_start_indices, {index_of_zero_start},
-          {index_of_zero_start + 1}, {1}));
-  HloInstruction* new_start_indices =
-      lhs_is_dynamic_slice
-          ? computation_->AddInstruction(HloInstruction::CreateConcatenate(
-                d_shape, {non_zero_start, zero_start}, 0))
-          : computation_->AddInstruction(HloInstruction::CreateConcatenate(
-                d_shape, {zero_start, non_zero_start}, 0));
+      dynamic_slice->mutable_operand(1 + index_of_zero_start);
+  std::vector<HloInstruction*> new_start_indices;
+  if (lhs_is_dynamic_slice) {
+    new_start_indices = {non_zero_start, zero_start};
+  } else {
+    new_start_indices = {zero_start, non_zero_start};
+  }
 
   // Build DynamicSlice(ctA x ctB).
   const int new_slice_m = lhs_is_dynamic_slice ? 1 : m;
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index 211c5bf05a..e39321f1f3 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -4535,14 +4535,17 @@ TEST_P(DotOfGatherSimplificationTest, ConstantRHS) {
 
   int32 start_row = (spec.lcd == 0) ? 0 : spec.s;
   int32 start_col = (spec.lcd == 0) ? spec.s : 0;
-  const auto start_indices =
+  std::vector<HloInstruction*> start_indices = {
       builder.AddInstruction(HloInstruction::CreateConstant(
-          LiteralUtil::CreateR1<int32>({start_row, start_col})));
+          LiteralUtil::CreateR0<int32>(start_row))),
+      builder.AddInstruction(HloInstruction::CreateConstant(
+          LiteralUtil::CreateR0<int32>(start_col)))};
   int64 slice_row_size = (spec.lcd == 0) ? spec.k : 1;
   int64 slice_col_size = (spec.lcd == 0) ? 1 : spec.k;
-  Shape ds_shape = ShapeUtil::MakeShape(F32, {slice_row_size, slice_col_size});
+  std::vector<int64> slice_sizes = {slice_row_size, slice_col_size};
+  Shape ds_shape = ShapeUtil::MakeShape(F32, slice_sizes);
   auto* ds = builder.AddInstruction(HloInstruction::CreateDynamicSlice(
-      ds_shape, lhs, start_indices, {slice_row_size, slice_col_size}));
+      ds_shape, lhs, start_indices, slice_sizes));
 
   int64 rhs_rows = (spec.rcd == 0) ? spec.k : spec.n;
   int64 rhs_cols = (spec.rcd == 0) ? spec.n : spec.k;
@@ -4575,7 +4578,7 @@ TEST_P(DotOfGatherSimplificationTest, ConstantRHS) {
   } else {
     EXPECT_THAT(computation->root_instruction(),
                 GmockMatch(m::DynamicSlice(m::Dot(m::Constant(), m::Constant()),
-                                           m::Concatenate())));
+                                           m::Constant(), m::Constant())));
   }
 }
 
@@ -4613,14 +4616,17 @@ TEST_P(DotOfGatherSimplificationTest, ConstantLHS) {
 
   int32 start_row = (spec.rcd == 0) ? 0 : spec.s;
   int32 start_col = (spec.rcd == 0) ? spec.s : 0;
-  const auto start_indices =
+  std::vector<HloInstruction*> start_indices = {
+      builder.AddInstruction(HloInstruction::CreateConstant(
+          LiteralUtil::CreateR0<int32>(start_row))),
       builder.AddInstruction(HloInstruction::CreateConstant(
-          LiteralUtil::CreateR1<int32>({start_row, start_col})));
+          LiteralUtil::CreateR0<int32>(start_col)))};
   int64 slice_row_size = (spec.rcd == 0) ? spec.k : 1;
   int64 slice_col_size = (spec.rcd == 0) ? 1 : spec.k;
-  Shape ds_shape = ShapeUtil::MakeShape(F32, {slice_row_size, slice_col_size});
+  std::vector<int64> slice_sizes = {slice_row_size, slice_col_size};
+  Shape ds_shape = ShapeUtil::MakeShape(F32, slice_sizes);
   auto* ds = builder.AddInstruction(HloInstruction::CreateDynamicSlice(
-      ds_shape, rhs, start_indices, {slice_row_size, slice_col_size}));
+      ds_shape, rhs, start_indices, slice_sizes));
 
   DotDimensionNumbers dot_dnums;
   dot_dnums.add_lhs_contracting_dimensions(spec.lcd);
@@ -4645,7 +4651,7 @@ TEST_P(DotOfGatherSimplificationTest, ConstantLHS) {
   } else {
     EXPECT_THAT(computation->root_instruction(),
                 GmockMatch(m::DynamicSlice(m::Dot(m::Constant(), m::Constant()),
-                                           m::Concatenate())));
+                                           m::Constant(), m::Constant())));
   }
 }
 
diff --git a/tensorflow/compiler/xla/service/buffer_assignment_test.cc b/tensorflow/compiler/xla/service/buffer_assignment_test.cc
index 370fad7ffb..1b4e93a2f3 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment_test.cc
+++ b/tensorflow/compiler/xla/service/buffer_assignment_test.cc
@@ -2486,9 +2486,9 @@ while_body {
   get-tuple-element.3 = s32[] get-tuple-element(state), index=0
   constant.2 = s32[] constant(128)
   add.5 = s32[] add(get-tuple-element.3, constant.2)
-  constant.3 = s32[3]{0} constant({0, 0, 0})
-  dynamic-update-slice.5 = f32[1280,1,128]{2,1,0} dynamic-update-slice(get-tuple-element.4, broadcast.6, constant.3)
-  dynamic-update-slice.9 = f32[1280,1,128]{2,1,0} dynamic-update-slice(dynamic-update-slice.5, broadcast.6, constant.3)
+  constant.3 = s32[] constant(0)
+  dynamic-update-slice.5 = f32[1280,1,128]{2,1,0} dynamic-update-slice(get-tuple-element.4, broadcast.6, constant.3, constant.3, constant.3)
+  dynamic-update-slice.9 = f32[1280,1,128]{2,1,0} dynamic-update-slice(dynamic-update-slice.5, broadcast.6, constant.3, constant.3, constant.3)
   ROOT tuple.85 = (s32[], s32[], s32[2]{0}, f32[1280,1,128]{2,1,0}) tuple(add.5, dynamic-update-slice.9)
 }
 
diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index a4ca772c7c..5c72e1ff71 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -1,6 +1,14 @@
 # Description:
 #    LLVM-based CPU backend for XLA.
 
+load("//tensorflow/compiler/xla:xla.bzl", "ORC_JIT_MEMORY_MAPPER_TARGETS")
+load(
+    "//third_party/mkl:build_defs.bzl",
+    "mkl_deps",
+)
+load("//tensorflow:tensorflow.bzl", "tf_cc_binary", "tf_cc_test")
+load(":build_defs.bzl", "runtime_copts")
+
 licenses(["notice"])  # Apache 2.0
 
 package(
@@ -14,15 +22,6 @@ package_group(
     ],
 )
 
-load(":build_defs.bzl", "runtime_copts")
-load("//tensorflow:tensorflow.bzl", "tf_cc_test")
-load("//tensorflow:tensorflow.bzl", "tf_cc_binary")
-load("//tensorflow/compiler/xla:xla.bzl", "ORC_JIT_MEMORY_MAPPER_TARGETS")
-load(
-    "//third_party/mkl:build_defs.bzl",
-    "mkl_deps",
-)
-
 # Filegroup used to collect source files for dependency checking.
 filegroup(
     name = "c_srcs",
@@ -114,6 +113,7 @@ cc_library(
         "//tensorflow/compiler/xla/service:conditional_simplifier",
         "//tensorflow/compiler/xla/service:convolution_group_converter",
         "//tensorflow/compiler/xla/service:dot_decomposer",
+        "//tensorflow/compiler/xla/service:dynamic_index_splitter",
         "//tensorflow/compiler/xla/service:executable",
         "//tensorflow/compiler/xla/service:flatten_call_graph",
         "//tensorflow/compiler/xla/service:hlo",
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
index dbab839aee..91c64e45b9 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
@@ -69,6 +69,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/cpu/simple_orc_jit.h"
 #include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h"
 #include "tensorflow/compiler/xla/service/dot_decomposer.h"
+#include "tensorflow/compiler/xla/service/dynamic_index_splitter.h"
 #include "tensorflow/compiler/xla/service/flatten_call_graph.h"
 #include "tensorflow/compiler/xla/service/hlo.pb.h"
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
@@ -244,6 +245,7 @@ Status CpuCompiler::RunHloPassesThroughLayoutAssn(
   HloPassPipeline pipeline("HLO passes through layout assignment");
   pipeline.AddInvariantChecker<HloVerifier>(/*layout_sensitive=*/false,
                                             /*allow_mixed_precision=*/false);
+  pipeline.AddPass<DynamicIndexSplitter>();
   pipeline.AddPass<CpuHloSupportChecker>();
 
   ReducePrecisionInsertion::AddPasses(
diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD
index d10a11d096..5e81281bae 100644
--- a/tensorflow/compiler/xla/service/gpu/BUILD
+++ b/tensorflow/compiler/xla/service/gpu/BUILD
@@ -3,6 +3,11 @@
 
 load("//tensorflow/compiler/xla/tests:build_defs.bzl", "xla_test")
 load("//tensorflow/compiler/xla:xla.bzl", "xla_proto_library")
+load(
+    "//tensorflow/core:platform/default/build_config_root.bzl",
+    "tf_cuda_tests_tags",
+)
+load("//tensorflow:tensorflow.bzl", "tf_cc_test")
 
 licenses(["notice"])  # Apache 2.0
 
@@ -24,12 +29,6 @@ filegroup(
     ]),
 )
 
-load("//tensorflow:tensorflow.bzl", "tf_cc_test")
-load(
-    "//tensorflow/core:platform/default/build_config_root.bzl",
-    "tf_cuda_tests_tags",
-)
-
 xla_proto_library(
     name = "backend_configs",
     srcs = ["backend_configs.proto"],
@@ -700,6 +699,7 @@ cc_library(
         "//tensorflow/compiler/xla/service:call_inliner",
         "//tensorflow/compiler/xla/service:conditional_simplifier",
         "//tensorflow/compiler/xla/service:convolution_group_converter",
+        "//tensorflow/compiler/xla/service:dynamic_index_splitter",
         "//tensorflow/compiler/xla/service:executable",
         "//tensorflow/compiler/xla/service:flatten_call_graph",
         "//tensorflow/compiler/xla/service:hlo",
diff --git a/tensorflow/compiler/xla/service/gpu/multi_output_fusion_test.cc b/tensorflow/compiler/xla/service/gpu/multi_output_fusion_test.cc
index d16c87ba5c..40b87b16a1 100644
--- a/tensorflow/compiler/xla/service/gpu/multi_output_fusion_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/multi_output_fusion_test.cc
@@ -628,8 +628,7 @@ TEST_F(MultiOutputFusionTest, MultiOutputFusionDUS) {
       p.1 = s32[1]{0} parameter(1)
       p.2 = f16[1,96,1024]{2,1,0} parameter(2)
       c.0 = s32[] constant(0)
-      pad = s32[3]{0} pad(p.1, c.0), padding=0_2
-      ROOT %dynamic-update-slice = f16[50,96,1024]{2,1,0} dynamic-update-slice(p.0, p.2, pad)
+      ROOT %dynamic-update-slice = f16[50,96,1024]{2,1,0} dynamic-update-slice(p.0, p.2, p.1, c.0, c.0)
     }
 
     fusion.2 {
@@ -638,7 +637,7 @@ TEST_F(MultiOutputFusionTest, MultiOutputFusionDUS) {
       p.2 = f16[1,96,1024]{2,1,0} parameter(2)
       c.0 = s32[] constant(0)
       pad = s32[3]{0} pad(p.1, c.0), padding=0_2
-      ROOT %dynamic-update-slice = f16[50,96,1024]{2,1,0} dynamic-update-slice(p.0, p.2, pad)
+      ROOT %dynamic-update-slice = f16[50,96,1024]{2,1,0} dynamic-update-slice(p.0, p.2, p.1, c.0, c.0)
     }
 
     ENTRY entry {
diff --git a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
index cd369d5598..9be7609508 100644
--- a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
+++ b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
@@ -37,6 +37,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/call_inliner.h"
 #include "tensorflow/compiler/xla/service/conditional_simplifier.h"
 #include "tensorflow/compiler/xla/service/convolution_group_converter.h"
+#include "tensorflow/compiler/xla/service/dynamic_index_splitter.h"
 #include "tensorflow/compiler/xla/service/flatten_call_graph.h"
 #include "tensorflow/compiler/xla/service/gpu/cudnn_batchnorm_rewriter.h"
 #include "tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.h"
@@ -152,6 +153,7 @@ Status OptimizeHloModule(HloModule* hlo_module, se::StreamExecutor* stream_exec,
     HloPassPipeline pipeline("optimization");
     pipeline.AddInvariantChecker<HloVerifier>(/*layout_sensitive=*/false,
                                               /*allow_mixed_precision=*/false);
+    pipeline.AddPass<DynamicIndexSplitter>();
     pipeline.AddPass<GpuHloSupportChecker>();
     ReducePrecisionInsertion::AddPasses(
         &pipeline, hlo_module->config().debug_options(),
diff --git a/tensorflow/compiler/xla/service/hlo_creation_utils.cc b/tensorflow/compiler/xla/service/hlo_creation_utils.cc
index 8cea95a73e..bb5d21c654 100644
--- a/tensorflow/compiler/xla/service/hlo_creation_utils.cc
+++ b/tensorflow/compiler/xla/service/hlo_creation_utils.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include "absl/strings/str_cat.h"
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
 #include "tensorflow/compiler/xla/service/shape_inference.h"
 #include "tensorflow/compiler/xla/util.h"
@@ -105,12 +106,26 @@ StatusOr<HloInstruction*> MakeDynamicSliceHlo(
     absl::Span<const int64> slice_sizes) {
   HloComputation* computation = operand->parent();
   CHECK_EQ(computation, start_indices->parent());
+  int64 rank = start_indices->shape().dimensions(0);
+  std::vector<HloInstruction*> scalar_start_indices;
+  for (int i = 0; i < rank; ++i) {
+    // TODO(b/118437727): Update callers to provide scalars directly.
+    auto slice = computation->AddInstruction(HloInstruction::CreateSlice(
+        ShapeUtil::MakeShape(start_indices->shape().element_type(), {1}),
+        start_indices, {i}, {i + 1}, {1}));
+    scalar_start_indices.push_back(
+        computation->AddInstruction(HloInstruction::CreateReshape(
+            ShapeUtil::MakeShape(start_indices->shape().element_type(), {}),
+            slice)));
+  }
+  std::vector<Shape> scalar_start_indices_shapes(
+      rank, ShapeUtil::MakeShape(start_indices->shape().element_type(), {}));
   TF_ASSIGN_OR_RETURN(
       Shape dynamic_slice_shape,
       ShapeInference::InferDynamicSliceShape(
-          operand->shape(), {start_indices->shape()}, slice_sizes));
+          operand->shape(), scalar_start_indices_shapes, slice_sizes));
   return computation->AddInstruction(HloInstruction::CreateDynamicSlice(
-      dynamic_slice_shape, operand, start_indices, slice_sizes));
+      dynamic_slice_shape, operand, scalar_start_indices, slice_sizes));
 }
 
 StatusOr<HloInstruction*> MakeDynamicUpdateSliceHlo(
@@ -119,12 +134,26 @@ StatusOr<HloInstruction*> MakeDynamicUpdateSliceHlo(
   HloComputation* computation = operand->parent();
   CHECK_EQ(computation, update->parent());
   CHECK_EQ(computation, start_indices->parent());
+  int64 rank = start_indices->shape().dimensions(0);
+  std::vector<HloInstruction*> scalar_start_indices;
+  for (int i = 0; i < rank; ++i) {
+    // TODO(b/118437727): Update callers to provide scalars directly.
+    auto slice = computation->AddInstruction(HloInstruction::CreateSlice(
+        ShapeUtil::MakeShape(start_indices->shape().element_type(), {1}),
+        start_indices, {i}, {i + 1}, {1}));
+    scalar_start_indices.push_back(
+        computation->AddInstruction(HloInstruction::CreateReshape(
+            ShapeUtil::MakeShape(start_indices->shape().element_type(), {}),
+            slice)));
+  }
+  std::vector<Shape> scalar_start_indices_shapes(
+      rank, ShapeUtil::MakeShape(start_indices->shape().element_type(), {}));
   TF_ASSIGN_OR_RETURN(
       Shape dynamic_update_slice_shape,
       ShapeInference::InferDynamicUpdateSliceShape(
-          operand->shape(), update->shape(), {start_indices->shape()}));
+          operand->shape(), update->shape(), scalar_start_indices_shapes));
   return computation->AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
-      dynamic_update_slice_shape, operand, update, start_indices));
+      dynamic_update_slice_shape, operand, update, scalar_start_indices));
 }
 
 StatusOr<HloInstruction*> MakeBroadcastHlo(
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
index cc5f6cc682..698b177310 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
@@ -1410,22 +1410,12 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
     auto operand = dynamic_slice->operand(0);
     auto start_indices = dynamic_slice->operand(1);
     auto result_shape = dynamic_slice->shape();
-    // TODO(b/118437727): Remove all of this nonsense.
-    // We may get an instruction without a parent module. In this case, assume
-    // scalar indices are not allowed.
-    bool allow_scalar_index = false;
-    if (dynamic_slice->GetModule() != nullptr) {
-      allow_scalar_index = dynamic_slice->GetModule()
-                               ->config()
-                               .debug_options()
-                               .xla_allow_scalar_index_dynamic_ops();
-    }
     TF_ASSIGN_OR_RETURN(
         auto inferred_return_shape,
         ShapeInference::InferDynamicSliceShape(
             operand->shape(),
             Cast<HloDynamicSliceInstruction>(dynamic_slice)->index_shapes(),
-            dynamic_slice->dynamic_slice_sizes(), allow_scalar_index));
+            dynamic_slice->dynamic_slice_sizes()));
     TF_RET_CHECK(ShapeUtil::Compatible(result_shape, inferred_return_shape))
         << "return shape is set to: " << ShapeUtil::HumanString(result_shape)
         << " but is inferred to be: "
@@ -1483,20 +1473,12 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
     auto update = dynamic_update_slice->operand(1);
     auto start_indices = dynamic_update_slice->operand(2);
     auto result_shape = dynamic_update_slice->shape();
-    bool allow_scalar_index = false;
-    if (dynamic_update_slice->GetModule() != nullptr) {
-      allow_scalar_index = dynamic_update_slice->GetModule()
-                               ->config()
-                               .debug_options()
-                               .xla_allow_scalar_index_dynamic_ops();
-    }
     TF_ASSIGN_OR_RETURN(
         auto inferred_return_shape,
         ShapeInference::InferDynamicUpdateSliceShape(
             operand->shape(), update->shape(),
             Cast<HloDynamicUpdateSliceInstruction>(dynamic_update_slice)
-                ->index_shapes(),
-            allow_scalar_index));
+                ->index_shapes()));
     TF_RET_CHECK(ShapeUtil::Compatible(result_shape, inferred_return_shape))
         << "return shape is set to: " << ShapeUtil::HumanString(result_shape)
         << " but is inferred to be: "
diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc
index 9274d42280..2c69c27ce5 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier.cc
@@ -504,38 +504,23 @@ Status ShapeVerifier::HandleSlice(HloInstruction* slice) {
 }
 
 Status ShapeVerifier::HandleDynamicSlice(HloInstruction* dynamic_slice) {
-  const DebugOptions& debug_options =
-      dynamic_slice->GetModule()->config().debug_options();
-  const bool allow_scalar_indices =
-      debug_options.xla_allow_scalar_index_dynamic_ops();
-  if (!allow_scalar_indices) {
-    TF_RETURN_IF_ERROR(CheckOperandCount(dynamic_slice, 2));
-  }
   return CheckShape(
       dynamic_slice,
       ShapeInference::InferDynamicSliceShape(
           dynamic_slice->operand(0)->shape(),
           Cast<HloDynamicSliceInstruction>(dynamic_slice)->index_shapes(),
-          dynamic_slice->dynamic_slice_sizes(), allow_scalar_indices));
+          dynamic_slice->dynamic_slice_sizes()));
 }
 
 Status ShapeVerifier::HandleDynamicUpdateSlice(
     HloInstruction* dynamic_update_slice) {
-  const DebugOptions& debug_options =
-      dynamic_update_slice->GetModule()->config().debug_options();
-  const bool allow_scalar_indices =
-      debug_options.xla_allow_scalar_index_dynamic_ops();
-  if (!allow_scalar_indices) {
-    TF_RETURN_IF_ERROR(CheckOperandCount(dynamic_update_slice, 3));
-  }
   return CheckShape(
       dynamic_update_slice,
       ShapeInference::InferDynamicUpdateSliceShape(
           dynamic_update_slice->operand(0)->shape(),
           dynamic_update_slice->operand(1)->shape(),
           Cast<HloDynamicUpdateSliceInstruction>(dynamic_update_slice)
-              ->index_shapes(),
-          allow_scalar_indices));
+              ->index_shapes()));
 }
 
 Status ShapeVerifier::HandleTuple(HloInstruction* tuple) {
diff --git a/tensorflow/compiler/xla/service/hlo_verifier_test.cc b/tensorflow/compiler/xla/service/hlo_verifier_test.cc
index 9b2d884f79..d27c62a879 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier_test.cc
@@ -450,8 +450,9 @@ TEST_F(HloVerifierTestLayoutSensitive, SliceWithLayoutChangeNotAllowed) {
    HloModule SliceWithLayoutChange
     ENTRY SliceWithLayoutChange {
       par0 = f32[4,5]{0,1} parameter(0)
-      par1 = s32[2] parameter(1)
-      ROOT dslice0 = f32[3,4]{1,0} dynamic-slice(par0, par1),
+      par1 = s32[] parameter(1)
+      par2 = s32[] parameter(2)
+      ROOT dslice0 = f32[3,4]{1,0} dynamic-slice(par0, par1, par2),
         dynamic_slice_sizes={3,4}
     }
   )";
diff --git a/tensorflow/compiler/xla/service/interpreter/BUILD b/tensorflow/compiler/xla/service/interpreter/BUILD
index 8999f38ccd..2e1d2cfbda 100644
--- a/tensorflow/compiler/xla/service/interpreter/BUILD
+++ b/tensorflow/compiler/xla/service/interpreter/BUILD
@@ -1,12 +1,12 @@
-licenses(["notice"])  # Apache 2.0
-
-package(default_visibility = ["//visibility:public"])
-
 load(
     "//tensorflow/core:platform/default/build_config_root.bzl",
     "if_static",
 )
 
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//visibility:public"])
+
 cc_library(
     name = "interpreter_transfer_manager",
     srcs = ["interpreter_transfer_manager.cc"],
@@ -35,6 +35,7 @@ cc_library(
         "//tensorflow/compiler/xla/service:batchnorm_expander",
         "//tensorflow/compiler/xla/service:compiler",
         "//tensorflow/compiler/xla/service:computation_placer",
+        "//tensorflow/compiler/xla/service:dynamic_index_splitter",
         "//tensorflow/compiler/xla/service:executable",
         "//tensorflow/compiler/xla/service:flatten_call_graph",
         "//tensorflow/compiler/xla/service:hlo",
diff --git a/tensorflow/compiler/xla/service/interpreter/compiler.cc b/tensorflow/compiler/xla/service/interpreter/compiler.cc
index 69f611a094..4818b2dae0 100644
--- a/tensorflow/compiler/xla/service/interpreter/compiler.cc
+++ b/tensorflow/compiler/xla/service/interpreter/compiler.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include "absl/memory/memory.h"
 #include "tensorflow/compiler/xla/service/algebraic_simplifier.h"
 #include "tensorflow/compiler/xla/service/computation_placer.h"
+#include "tensorflow/compiler/xla/service/dynamic_index_splitter.h"
 #include "tensorflow/compiler/xla/service/flatten_call_graph.h"
 #include "tensorflow/compiler/xla/service/hlo_constant_folding.h"
 #include "tensorflow/compiler/xla/service/hlo_cse.h"
@@ -44,6 +45,7 @@ namespace interpreter {
 Status InterpreterCompiler::RunHloOptimization(HloModule* hlo_module) {
   HloPassPipeline pipeline("Interpreter");
 
+  pipeline.AddPass<DynamicIndexSplitter>();
   pipeline.AddPass<LayoutAssignment>(
       hlo_module->mutable_entry_computation_layout(),
       LayoutAssignment::InstructionCanChangeLayout);
diff --git a/tensorflow/compiler/xla/service/layout_assignment_test.cc b/tensorflow/compiler/xla/service/layout_assignment_test.cc
index 387b385157..c8cf3c47d3 100644
--- a/tensorflow/compiler/xla/service/layout_assignment_test.cc
+++ b/tensorflow/compiler/xla/service/layout_assignment_test.cc
@@ -960,8 +960,9 @@ TEST_F(LayoutAssignmentTest, CopyDSliceOperandToAvoidImplicitLayoutChange) {
     ENTRY CopyDSliceOperandToAvoidImplicitLayoutChange {
       par0 = f32[3,4]{1,0} parameter(0)
       par1 = f32[4,5]{0,1} parameter(1)
-      par2 = s32[2] parameter(2)
-      dslice0 = f32[3,4] dynamic-slice(par1, par2), dynamic_slice_sizes={3,4}
+      par2 = s32[] parameter(2)
+      par3 = s32[] parameter(3)
+      dslice0 = f32[3,4] dynamic-slice(par1, par2, par3), dynamic_slice_sizes={3,4}
       ROOT add0 = f32[3,4]{1,0} add(par0,dslice0)
     }
   )";
@@ -982,7 +983,7 @@ TEST_F(LayoutAssignmentTest, CopyDSliceOperandToAvoidImplicitLayoutChange) {
                   m::Parameter(),
                   m::DynamicSlice(
                       m::Copy(m::Parameter(1)).WithShapeEqualTo(&shape_copy),
-                      m::Parameter(2)))));
+                      m::Parameter(2), m::Parameter(3)))));
 }
 
 TEST_F(LayoutAssignmentTest, CopyConcatOperandToAvoidImplicitLayoutChange) {
diff --git a/tensorflow/compiler/xla/service/pattern_matcher.h b/tensorflow/compiler/xla/service/pattern_matcher.h
index 802b2c0c85..9e3d106021 100644
--- a/tensorflow/compiler/xla/service/pattern_matcher.h
+++ b/tensorflow/compiler/xla/service/pattern_matcher.h
@@ -2118,7 +2118,6 @@ XLA_BINOP_PATTERN(Divide)
 XLA_BINOP_PATTERN(Complex)
 XLA_BINOP_PATTERN(Convolution)
 XLA_BINOP_PATTERN(Dot)
-XLA_BINOP_PATTERN(DynamicSlice)
 XLA_COMMUTATIVE_BINOP_PATTERN(Eq)
 XLA_BINOP_PATTERN(Gather)
 XLA_BINOP_PATTERN(Ge)
@@ -2235,6 +2234,7 @@ inline auto WithOperands(Matcher&& m, int64 operand_num, FirstArg&& first_arg,
 XLA_VARIADIC_OP_PATTERN(AfterAll);
 XLA_VARIADIC_OP_PATTERN(Concatenate);
 XLA_VARIADIC_OP_PATTERN(CustomCall);
+XLA_VARIADIC_OP_PATTERN(DynamicSlice)
 XLA_VARIADIC_OP_PATTERN(Map)
 XLA_VARIADIC_OP_PATTERN(Reduce);
 XLA_VARIADIC_OP_PATTERN(Sort);
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index d711c8ef0d..b729dd7660 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -2099,15 +2099,15 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
     }
 
     const Shape& start_indices_shape = start_index_shapes[0];
-    TF_RETURN_IF_ERROR(
-        ExpectArray(start_indices_shape, "start indices of dynamic slice"));
-
     VLOG(2) << StrFormat(
         "slicing shape %s at dynamic start_indices %s with slice_sizes={%s}",
         ShapeUtil::HumanString(operand_shape),
         ShapeUtil::HumanString(start_indices_shape),
         StrJoin(slice_sizes, ", "));
 
+    TF_RETURN_IF_ERROR(
+        ExpectArray(start_indices_shape, "start indices of dynamic slice"));
+
     if (start_indices_shape.rank() != 1) {
       return InvalidArgument(
           "Dynamic slice start indices of rank %d must be rank1.",
@@ -2151,7 +2151,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
             "Dynamic slice start indices must be of integral type.");
       }
       for (const Shape& index_shape : start_index_shapes) {
-        if (!ShapeUtil::Equal(first_index_shape, index_shape)) {
+        if (!ShapeUtil::Compatible(first_index_shape, index_shape)) {
           return InvalidArgument(
               "Dynamic slice start indices must all have the same shape, got "
               "mismatching indices with shapes %s and %s.",
@@ -2258,7 +2258,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
             "Dynamic update slice start indices must be of integral type.");
       }
       for (const Shape& index_shape : start_index_shapes) {
-        if (!ShapeUtil::Equal(first_index_shape, index_shape)) {
+        if (!ShapeUtil::Compatible(first_index_shape, index_shape)) {
           return InvalidArgument(
               "Dynamic update slice start indices must all have the same "
               "shape, got mismatching indices with shapes %s and %s.",
diff --git a/tensorflow/compiler/xla/service/shape_inference.h b/tensorflow/compiler/xla/service/shape_inference.h
index e440e364c8..7d39ef38e0 100644
--- a/tensorflow/compiler/xla/service/shape_inference.h
+++ b/tensorflow/compiler/xla/service/shape_inference.h
@@ -177,14 +177,14 @@ class ShapeInference {
   // in 'slice_sizes', with dynamic start indices shape 'start_indices_shape'.
   static StatusOr<Shape> InferDynamicSliceShape(
       const Shape& operand_shape, absl::Span<const Shape> start_index_shapes,
-      absl::Span<const int64> slice_sizes, bool allow_scalar_indices = false);
+      absl::Span<const int64> slice_sizes, bool allow_scalar_indices = true);
 
   // Infers the shape produced by a dynamic update slice operation based
   // on the shape of operand and update.
   static StatusOr<Shape> InferDynamicUpdateSliceShape(
       const Shape& operand_shape, const Shape& update_shape,
       absl::Span<const Shape> start_index_shapes,
-      bool allow_scalar_indices = false);
+      bool allow_scalar_indices = true);
 
   // Infers the shape produced by doing a compile-time-constant indexing into
   // the given input shape. This is essential for operations on tuples, because
diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc
index 4726ee6a5b..7f359311c3 100644
--- a/tensorflow/compiler/xla/shape_util.cc
+++ b/tensorflow/compiler/xla/shape_util.cc
@@ -112,6 +112,7 @@ bool CompareShapes(const Shape& lhs, const Shape& rhs, bool compare_layouts,
 
   if (compare_layouts) {
     if (lhs.layout().format() != rhs.layout().format()) {
+      VLOG(3) << "CompareShapes: lhs layout format != rhs layout format";
       return false;
     }
     if (LayoutUtil::IsDenseArray(lhs)) {
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 2c11ba4283..9433264917 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -71,6 +71,7 @@ cc_library(
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/compiler/xla/service:hlo_casting_utils",
         "//tensorflow/compiler/xla/service:hlo_dataflow_analysis",
         "//tensorflow/compiler/xla/service:hlo_verifier",
         "//tensorflow/compiler/xla/service:transfer_manager",
diff --git a/tensorflow/compiler/xla/tests/test_utils.cc b/tensorflow/compiler/xla/tests/test_utils.cc
index 18df41c117..95c89b0ba6 100644
--- a/tensorflow/compiler/xla/tests/test_utils.cc
+++ b/tensorflow/compiler/xla/tests/test_utils.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include "absl/memory/memory.h"
 #include "tensorflow/compiler/xla/literal_util.h"
 #include "tensorflow/compiler/xla/primitive_util.h"
+#include "tensorflow/compiler/xla/service/hlo_casting_utils.h"
 #include "tensorflow/compiler/xla/service/hlo_dataflow_analysis.h"
 #include "tensorflow/compiler/xla/service/hlo_verifier.h"
 #include "tensorflow/compiler/xla/service/transfer_manager.h"
@@ -274,16 +275,9 @@ bool NeedsInitValue(const HloUse& use) {
 
 // Generate random values that are constrained to the input_shape minus the
 // output_shape so as not to produce wrapping slices, for instance.
-Literal MakeRandomIndex(absl::Span<const int64> index_space,
-                        std::minstd_rand0* engine) {
-  std::vector<int32> start_indices(index_space.size());
-  if (engine != nullptr) {
-    for (int i = 0; i < index_space.size(); ++i) {
-      std::uniform_int_distribution<int32> generator(0, index_space[i]);
-      start_indices[i] = generator(*engine);
-    }
-  }
-  return LiteralUtil::CreateR1<int32>(start_indices);
+Literal MakeRandomIndex(int64 index_bound, std::minstd_rand0* engine) {
+  std::uniform_int_distribution<int32> generator(0, index_bound);
+  return LiteralUtil::CreateR0<int32>(generator(*engine));
 }
 
 // Use dataflow analysis on each parameter to see if there are uses that would
@@ -300,8 +294,8 @@ std::vector<HloInstruction*> FindConstrainedUses(
       HloInstruction* instruction = use.instruction;
       const HloOpcode opcode = instruction->opcode();
       const int64 op_num = use.operand_number;
-      if ((opcode == HloOpcode::kDynamicSlice && op_num == 1) ||
-          (opcode == HloOpcode::kDynamicUpdateSlice && op_num == 2)) {
+      if ((opcode == HloOpcode::kDynamicSlice && op_num >= 1) ||
+          (opcode == HloOpcode::kDynamicUpdateSlice && op_num >= 2)) {
         constrained_uses.push_back(instruction);
       } else if (opcode == HloOpcode::kFusion) {
         const HloInstruction* const to_analyze =
@@ -336,7 +330,7 @@ std::vector<HloInstruction*> FindConstrainedUses(
 StatusOr<Literal> CreateLiteralForConstrainedUses(
     const absl::Span<HloInstruction* const> constrained_uses,
     const HloInstruction& param, std::minstd_rand0* engine) {
-  std::vector<int64> index_space;
+  int64 index_bound = INT64_MAX;
   bool no_duplicates = false;
   bool needs_constant = false;
   ConstantType constant_type = ConstantType::kUnknown;
@@ -348,19 +342,16 @@ StatusOr<Literal> CreateLiteralForConstrainedUses(
         const Shape& slice_shape = use->opcode() == HloOpcode::kDynamicSlice
                                        ? use->shape()
                                        : use->operand(1)->shape();
-        const int64 rank = indexed_shape.rank();
-        if (!index_space.empty()) {
-          TF_RET_CHECK(rank == index_space.size());
-          for (int64 i = 0; i < rank; ++i) {
-            index_space[i] = std::min(
-                index_space[i], ShapeUtil::GetDimension(indexed_shape, i) -
-                                    ShapeUtil::GetDimension(slice_shape, i));
-          }
-        } else {
-          index_space.resize(rank);
-          for (int64 i = 0; i < rank; ++i) {
-            index_space[i] = ShapeUtil::GetDimension(indexed_shape, i) -
-                             ShapeUtil::GetDimension(slice_shape, i);
+        const int64 first_index =
+            Cast<HloDynamicIndexInstruction>(use)->first_index_operand_number();
+        for (int64 operand = first_index; operand < use->operand_count();
+             ++operand) {
+          if (use->operand(operand) == &param) {
+            index_bound = std::min(
+                index_bound,
+                ShapeUtil::GetDimension(indexed_shape, operand - first_index) -
+                    ShapeUtil::GetDimension(slice_shape,
+                                            operand - first_index));
           }
         }
         break;
@@ -388,16 +379,13 @@ StatusOr<Literal> CreateLiteralForConstrainedUses(
   }
   int constraint_count = 0;
   constraint_count += no_duplicates ? 1 : 0;
-  constraint_count += !index_space.empty() ? 1 : 0;
+  constraint_count += (index_bound != INT64_MAX) ? 1 : 0;
   constraint_count += needs_constant ? 1 : 0;
   if (constraint_count > 1) {
     return Unimplemented("Conflicting operand generation constraints.");
   }
-  if (!index_space.empty()) {
-    // constrained_uses looks through bitcasts, so param and indexed_space may
-    // not have the same shape.  (For example, param might be an R0 while
-    // indexed_space might have size 1.)
-    return MakeRandomIndex(index_space, engine)
+  if (index_bound != INT64_MAX) {
+    return MakeRandomIndex(index_bound, engine)
         .Reshape(param.shape().dimensions());
   } else if (needs_constant) {
     switch (constant_type) {
diff --git a/tensorflow/compiler/xla/tests/test_utils_test.cc b/tensorflow/compiler/xla/tests/test_utils_test.cc
index 4b4f164d1b..591d6c1922 100644
--- a/tensorflow/compiler/xla/tests/test_utils_test.cc
+++ b/tensorflow/compiler/xla/tests/test_utils_test.cc
@@ -79,25 +79,26 @@ XLA_TEST_F(TestUtilsTest, MultipleIndexSpacesForDynamicSlices) {
                     R"(HloModule index_space_module
 
     ENTRY IndexSpace {
-      index_param = s32[3]{0} parameter(0)
-      array_param.1 = f32[123,4,789]{0,1,2} parameter(1)
-      array_param.2 = f32[3,3000,5]{0,1,2} parameter(2)
-      dynamic-slice.1 = f32[1,2,3] dynamic-slice(array_param.1, index_param), dynamic_slice_sizes={1,2,3}
-      ROOT dynamic-slice.2 = f32[3,2,2] dynamic-slice(array_param.2, index_param), dynamic_slice_sizes={3,2,2}
+      index_param.0 = s32[] parameter(0)
+      index_param.1 = s32[] parameter(1)
+      index_param.2 = s32[] parameter(2)
+      array_param.1 = f32[123,4,789]{0,1,2} parameter(3)
+      array_param.2 = f32[3,3000,5]{0,1,2} parameter(4)
+      dynamic-slice.1 = f32[1,2,3] dynamic-slice(array_param.1, index_param.0, index_param.1, index_param.2), dynamic_slice_sizes={1,2,3}
+      ROOT dynamic-slice.2 = f32[3,2,2] dynamic-slice(array_param.2, index_param.0, index_param.1, index_param.2), dynamic_slice_sizes={3,2,2}
     })")
                     .ValueOrDie();
   TF_ASSERT_OK_AND_ASSIGN(std::vector<Literal> args,
                           MakeFakeArguments(module.get()));
-  ASSERT_EQ(args.size(), 3);
-  const Literal& index_arg = args[0];
+  ASSERT_EQ(args.size(), 5);
 
-  EXPECT_EQ(index_arg.Get<int32>({0}), 0);
+  EXPECT_EQ(args[0].Get<int32>({}), 0);
 
-  EXPECT_GE(index_arg.Get<int32>({1}), 0);
-  EXPECT_LE(index_arg.Get<int32>({1}), 2);
+  EXPECT_GE(args[1].Get<int32>({}), 0);
+  EXPECT_LE(args[0].Get<int32>({}), 2);
 
-  EXPECT_GE(index_arg.Get<int32>({2}), 0);
-  EXPECT_LE(index_arg.Get<int32>({2}), 3);
+  EXPECT_GE(args[2].Get<int32>({}), 0);
+  EXPECT_LE(args[2].Get<int32>({}), 3);
 }
 
 XLA_TEST_F(TestUtilsTest, MultipleIndexSpacesForDynamicUpdateSlices) {
@@ -105,28 +106,29 @@ XLA_TEST_F(TestUtilsTest, MultipleIndexSpacesForDynamicUpdateSlices) {
                     R"(HloModule index_space_module
 
     ENTRY IndexSpace {
-      index_param = s32[3]{0} parameter(0)
-      array_param.1 = f32[123,4,789]{0,1,2} parameter(1)
-      array_param.2 = f32[3,3000,5]{0,1,2} parameter(2)
-      update_param.1 = f32[1,2,3]{0,1,2} parameter(3)
-      update_param.2 = f32[3,2,2]{0,1,2} parameter(4)
-
-      dynamic-update-slice.1 = f32[123,4,789] dynamic-update-slice(array_param.1, update_param.1, index_param)
-      ROOT dynamic-update-slice.2 = f32[3,3000,5] dynamic-update-slice(array_param.2, update_param.2, index_param)
+      index_param.0 = s32[] parameter(0)
+      index_param.1 = s32[] parameter(1)
+      index_param.2 = s32[] parameter(2)
+      array_param.1 = f32[123,4,789]{0,1,2} parameter(3)
+      array_param.2 = f32[3,3000,5]{0,1,2} parameter(4)
+      update_param.1 = f32[1,2,3]{0,1,2} parameter(5)
+      update_param.2 = f32[3,2,2]{0,1,2} parameter(6)
+
+      dynamic-update-slice.1 = f32[123,4,789] dynamic-update-slice(array_param.1, update_param.1, index_param.0, index_param.1, index_param.2)
+      ROOT dynamic-update-slice.2 = f32[3,3000,5] dynamic-update-slice(array_param.2, update_param.2, index_param.0, index_param.1, index_param.2)
     })")
                     .ValueOrDie();
   TF_ASSERT_OK_AND_ASSIGN(std::vector<Literal> args,
                           MakeFakeArguments(module.get()));
-  ASSERT_EQ(args.size(), 5);
-  const Literal& index_arg = args[0];
+  ASSERT_EQ(args.size(), 7);
 
-  EXPECT_EQ(index_arg.Get<int32>({0}), 0);
+  EXPECT_EQ(args[0].Get<int32>({}), 0);
 
-  EXPECT_GE(index_arg.Get<int32>({1}), 0);
-  EXPECT_LE(index_arg.Get<int32>({1}), 2);
+  EXPECT_GE(args[1].Get<int32>({}), 0);
+  EXPECT_LE(args[0].Get<int32>({}), 2);
 
-  EXPECT_GE(index_arg.Get<int32>({2}), 0);
-  EXPECT_LE(index_arg.Get<int32>({2}), 3);
+  EXPECT_GE(args[2].Get<int32>({}), 0);
+  EXPECT_LE(args[2].Get<int32>({}), 3);
 }
 
 XLA_TEST_F(TestUtilsTest, NoDuplicatesFloats) {
-- 
GitLab


From efe565bc0981e80a52a97f3961cfba3e87023b42 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Fri, 4 Jan 2019 14:08:26 -0800
Subject: [PATCH 0209/2345] Make the initial tf.train.Checkpoint.restore() read
 Tensors in a batch

Should be faster when reading from distributed file systems. Does not affect cases where restore-on-create is necessary, but as long as variable objects have been created and tracked before restore() their reads should be batched together.

PiperOrigin-RevId: 227911381
---
 .../python/training/checkpointable/util.py    | 22 +++------
 .../training/saving/functional_saver.py       | 45 +++++++++++--------
 2 files changed, 31 insertions(+), 36 deletions(-)

diff --git a/tensorflow/python/training/checkpointable/util.py b/tensorflow/python/training/checkpointable/util.py
index c890a7f440..a45263f5c6 100644
--- a/tensorflow/python/training/checkpointable/util.py
+++ b/tensorflow/python/training/checkpointable/util.py
@@ -176,25 +176,13 @@ class _CheckpointRestoreCoordinator(object):
         raise AssertionError(
             ("Saveable keys changed when validating. Got back %s, was "
              "expecting %s") % (tensor_saveables.keys(), validated_names))
-      for saveable in validated_saveables:
-        if saveable.device:
-          device = saveable_object_util.set_cpu0(saveable.device)
-        else:
-          device = None
-        with ops.device(device):
-          tensors = []
-          for spec in saveable.specs:
-            tensors.append(
-                io_ops.restore_v2(
-                    self.save_path_tensor,
-                    [spec.name],
-                    [spec.slice_spec],
-                    [spec.dtype])[0])
-          restore_op = saveable.restore(tensors, restored_shapes=None)
-        if not context.executing_eagerly():
+      new_restore_ops = functional_saver.restore_from_saveable_objects(
+          self.save_path_tensor, validated_saveables)
+      if not context.executing_eagerly():
+        restore_ops.extend(new_restore_ops)
+        for saveable, restore_op in zip(validated_saveables, new_restore_ops):
           assert saveable.name not in self.restore_ops_by_name
           self.restore_ops_by_name[saveable.name] = restore_op
-          restore_ops.append(restore_op)
     return restore_ops
 
 
diff --git a/tensorflow/python/training/saving/functional_saver.py b/tensorflow/python/training/saving/functional_saver.py
index 51f618ddd3..4ff2742c2f 100644
--- a/tensorflow/python/training/saving/functional_saver.py
+++ b/tensorflow/python/training/saving/functional_saver.py
@@ -107,25 +107,32 @@ class Saver(object):
       A scalar string Tensor containing `file_prefix` with control dependencies
       on the restore ops.
     """
-    restore_specs = []
-    tensor_structure = []
-    for saveable in self._saveable_objects:
-      saveable_tensor_structure = []
-      tensor_structure.append(saveable_tensor_structure)
-      for spec in saveable.specs:
-        saveable_tensor_structure.append(spec.name)
-        restore_specs.append((spec.name, spec.slice_spec, spec.dtype))
-    tensor_names, tensor_slices, tensor_dtypes = zip(*restore_specs)
-    with ops.device("cpu:0"):
-      restored_tensors = io_ops.restore_v2(
-          file_prefix, tensor_names, tensor_slices, tensor_dtypes)
-    structured_restored_tensors = nest.pack_sequence_as(
-        tensor_structure, restored_tensors)
-    restore_ops = []
-    for saveable, restored_tensors in zip(self._saveable_objects,
-                                          structured_restored_tensors):
-      restore_ops.append(saveable.restore(restored_tensors,
-                                          restored_shapes=None))
+    restore_ops = restore_from_saveable_objects(
+        file_prefix, self._saveable_objects)
     with ops.device("cpu:0"):
       with ops.control_dependencies(restore_ops):
         return array_ops.identity(file_prefix)
+
+
+def restore_from_saveable_objects(file_prefix, saveable_objects):
+  """Reads from a checkpoint and returns restore ops for `saveable_objects`s."""
+  restore_specs = []
+  tensor_structure = []
+  for saveable in saveable_objects:
+    saveable_tensor_structure = []
+    tensor_structure.append(saveable_tensor_structure)
+    for spec in saveable.specs:
+      saveable_tensor_structure.append(spec.name)
+      restore_specs.append((spec.name, spec.slice_spec, spec.dtype))
+  tensor_names, tensor_slices, tensor_dtypes = zip(*restore_specs)
+  with ops.device("cpu:0"):
+    restored_tensors = io_ops.restore_v2(
+        file_prefix, tensor_names, tensor_slices, tensor_dtypes)
+  structured_restored_tensors = nest.pack_sequence_as(
+      tensor_structure, restored_tensors)
+  restore_ops = []
+  for saveable, restored_tensors in zip(saveable_objects,
+                                        structured_restored_tensors):
+    restore_ops.append(saveable.restore(restored_tensors,
+                                        restored_shapes=None))
+  return restore_ops
-- 
GitLab


From 41e333f0193f89bc889a31f2a32b4119261aa222 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Fri, 4 Jan 2019 14:10:40 -0800
Subject: [PATCH 0210/2345] [tf.data] Adding prefetching of input iterators for
 parallel interleave.

PiperOrigin-RevId: 227911701
---
 .../data/parallel_interleave_dataset_op.cc    | 752 +++++++++++-------
 .../data/kernel_tests/interleave_test.py      | 123 +--
 2 files changed, 511 insertions(+), 364 deletions(-)

diff --git a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
index 315f96d4b5..dc2663d1e0 100644
--- a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
+++ b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 #include <atomic>
 #include <deque>
+#include <memory>
 #include <utility>
 
 #include "tensorflow/core/common_runtime/function.h"
@@ -21,6 +22,7 @@ limitations under the License.
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/stats_aggregator.h"
 #include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/kernels/data/captured_function.h"
 #include "tensorflow/core/kernels/data/dataset_utils.h"
 #include "tensorflow/core/lib/core/threadpool.h"
@@ -43,12 +45,7 @@ namespace {
 //
 // Furthermore, this class favors modularity over extended functionality. In
 // particular, it refrains from implementing configurable buffering of output
-// elements and prefetching of input iterators, relying on other parts of
-// tf.data to provide this functionality if necessary.
-//
-// The above design choices were made with automated optimizations in mind,
-// isolating the degree of parallelism as the single tunable knob of this
-// implementation.
+// elements and prefetching of input iterators.
 class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
  public:
   explicit ParallelInterleaveDatasetOp(OpKernelConstruction* ctx)
@@ -237,27 +234,20 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
       Status GetNextInternal(IteratorContext* ctx,
                              std::vector<Tensor>* out_tensors,
                              bool* end_of_sequence) override {
-        std::shared_ptr<InvocationResult> result;
-        do {
-          result.reset();
-          {
-            mutex_lock l(*mu_);
-            EnsureRunnerThreadStarted(ctx);
-            while (ShouldWait(&result)) {
-              RecordStop(ctx);
-              cond_var_->wait(l);
-              RecordStart(ctx);
-            }
-            if (!result) {
-              *end_of_sequence = true;
-              return Status::OK();
-            }
+        std::shared_ptr<Result> result;
+        {
+          mutex_lock l(*mu_);
+          EnsureThreadsStarted(ctx);
+          while (!Consume(&result)) {
+            RecordStop(ctx);
+            cond_var_->wait(l);
+            RecordStart(ctx);
           }
-          RecordStop(ctx);
-          result->notification.WaitForNotification();
-          RecordStart(ctx);
-        } while (result->skip);
-
+        }
+        if (!result) {
+          *end_of_sequence = true;
+          return Status::OK();
+        }
         if (result->status.ok()) {
           *out_tensors = std::move(result->return_values);
           RecordBufferDequeue(ctx, *out_tensors);
@@ -281,37 +271,22 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
         while (num_calls_ > 0) {
           cond_var_->wait(l);
         }
-        CHECK_EQ(num_calls_, 0);
+        DCHECK_EQ(num_calls_, 0);
         TF_RETURN_IF_ERROR(SaveInput(writer, input_impl_));
-        TF_RETURN_IF_ERROR(writer->WriteScalar(
-            full_name("invocation_results.size"), invocation_results_.size()));
-        for (size_t i = 0; i < invocation_results_.size(); i++) {
-          std::shared_ptr<InvocationResult> result = invocation_results_[i];
-          TF_RETURN_IF_ERROR(WriteStatusLocked(writer, i, result->status));
-          TF_RETURN_IF_ERROR(writer->WriteScalar(
-              full_name(strings::StrCat("invocation_results[", i, "].size")),
-              result->return_values.size()));
-          for (size_t j = 0; j < result->return_values.size(); j++) {
-            TF_RETURN_IF_ERROR(writer->WriteTensor(
-                full_name(
-                    strings::StrCat("invocation_results[", i, "][", j, "]")),
-                result->return_values[j]));
-          }
-          if (result->skip) {
-            TF_RETURN_IF_ERROR(writer->WriteScalar(
-                full_name(strings::StrCat("invocation_results[", i, "].skip")),
-                ""));
-          }
-        }
+        TF_RETURN_IF_ERROR(
+            writer->WriteScalar(full_name("block_index"), block_index_));
         TF_RETURN_IF_ERROR(
             writer->WriteScalar(full_name("cycle_index"), cycle_index_));
         if (end_of_input_) {
           TF_RETURN_IF_ERROR(
               writer->WriteScalar(full_name("end_of_input"), ""));
         }
+        TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("element_id_counter"),
+                                               element_id_counter_));
         TF_RETURN_IF_ERROR(
             writer->WriteScalar(full_name("num_open"), num_open_));
         TF_RETURN_IF_ERROR(WriteCurrentElements(writer));
+        TF_RETURN_IF_ERROR(WriteFutureElements(writer));
         return Status::OK();
       }
 
@@ -319,114 +294,268 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
                              IteratorStateReader* reader) override {
         mutex_lock l(*mu_);
         TF_RETURN_IF_ERROR(RestoreInput(ctx, reader, input_impl_));
-        int64 invocation_results_size;
-        TF_RETURN_IF_ERROR(reader->ReadScalar(
-            full_name("invocation_results.size"), &invocation_results_size));
-        for (size_t i = 0; i < invocation_results_size; i++) {
-          std::shared_ptr<InvocationResult> result(new InvocationResult());
-          invocation_results_.push_back(result);
-          TF_RETURN_IF_ERROR(ReadStatusLocked(reader, i, &result->status));
-          size_t num_return_values;
-          {
-            int64 size;
-            TF_RETURN_IF_ERROR(reader->ReadScalar(
-                full_name(strings::StrCat("invocation_results[", i, "].size")),
-                &size));
-            num_return_values = static_cast<size_t>(size);
-            if (num_return_values != size) {
-              return errors::InvalidArgument(strings::StrCat(
-                  full_name(
-                      strings::StrCat("invocation_results[", i, "].size")),
-                  ": ", size, " is not a valid value of type size_t."));
-            }
-          }
-          result->return_values.reserve(num_return_values);
-          for (size_t j = 0; j < num_return_values; j++) {
-            result->return_values.emplace_back();
-            TF_RETURN_IF_ERROR(
-                reader->ReadTensor(full_name(strings::StrCat(
-                                       "invocation_results[", i, "][", j, "]")),
-                                   &result->return_values.back()));
-          }
-          result->skip = reader->Contains(
-              full_name(strings::StrCat("invocation_results[", i, "].skip")));
-          result->notification.Notify();
-        }
+        TF_RETURN_IF_ERROR(
+            reader->ReadScalar(full_name("block_index"), &block_index_));
         TF_RETURN_IF_ERROR(
             reader->ReadScalar(full_name("cycle_index"), &cycle_index_));
+        TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("element_id_counter"),
+                                              &element_id_counter_));
         if (reader->Contains(full_name("end_of_input"))) end_of_input_ = true;
         TF_RETURN_IF_ERROR(
             reader->ReadScalar(full_name("num_open"), &num_open_));
         TF_RETURN_IF_ERROR(ReadCurrentElements(ctx, reader));
+        TF_RETURN_IF_ERROR(ReadFutureElements(ctx, reader));
         return Status::OK();
       }
 
      private:
+      // Represents the result of fetching an element from a dataset.
+      struct Result {
+        Status status;
+        std::vector<Tensor> return_values;
+        // Indicates whether the result is ready to be consumed.
+        bool is_ready = false;
+      };
+
+      // The interleave transformation repeatedly inputs elements, applies the
+      // user-provided function to transform the input elements to datasets, and
+      // interleaves the elements of these datasets as its output.
+      //
+      // This structure represents an input element and derived state.
       struct Element {
+        // Unique identifier, needed to support checkpointing.
+        int64 id;
+        // The actual input element.
+        std::vector<Tensor> inputs;
+        // Iterator created from the input element.
         std::unique_ptr<IteratorBase> iterator;
-        std::vector<Tensor> inputs;  // inputs for creating the iterator
-        bool in_use;
+        mutex mu;
+        // Buffer for storing the outputs of `iterator`.
+        std::deque<std::shared_ptr<Result>> results GUARDED_BY(mu);
+        // Indicates whether the element is used by a worker thread.
+        bool in_use = false;
       };
 
-      struct InvocationResult {
-        Notification notification;  // used for coordination with the consumer
-        Status status;              // the invocation status
-        std::vector<Tensor> return_values;  // the invocation result values
-        bool skip;  // if set the result should be skipped
-      };
+      // Advances the position in the interleave cycle to the next cycle
+      // element.
+      void AdvanceToNextInCycle() EXCLUSIVE_LOCKS_REQUIRED(*mu_) {
+        block_index_ = 0;
+        cycle_index_ = (cycle_index_ + 1) % dataset()->cycle_length_;
+      }
 
-      void EnsureRunnerThreadStarted(IteratorContext* ctx)
+      // Advances the position in the interleave cycle by one.
+      void AdvancePosition() EXCLUSIVE_LOCKS_REQUIRED(*mu_) {
+        ++block_index_;
+        if (block_index_ == dataset()->block_length_) {
+          AdvanceToNextInCycle();
+        }
+      }
+
+      // Consumes a result (if available), returning an indication of whether
+      // a result is available. If `true` is returned, `result` either
+      // points to a valid result or is null if end of input has been reached.
+      bool Consume(std::shared_ptr<Result>* result)
           EXCLUSIVE_LOCKS_REQUIRED(*mu_) {
-        if (!runner_thread_) {
-          std::shared_ptr<IteratorContext> new_ctx(new IteratorContext(*ctx));
-          runner_thread_.reset(ctx->env()->StartThread(
-              {}, "tf_data_parallel_interleave_runner",
-              [this, new_ctx]() { RunnerThread(new_ctx); }));
+        if (!sloppy_) {
+          return ConsumeHelper(result);
+        }
+        // If we are allowed to be sloppy (i.e. return results out of order),
+        // try to find an element in the cycle that has a result available.
+        for (int i = 0; i < dataset()->cycle_length_; ++i) {
+          if (ConsumeHelper(result)) {
+            return true;
+          }
+          AdvanceToNextInCycle();
+        }
+        return false;
+      }
+
+      bool ConsumeHelper(std::shared_ptr<Result>* result)
+          EXCLUSIVE_LOCKS_REQUIRED(*mu_) {
+        while (true) {
+          std::shared_ptr<Element> element = current_elements_[cycle_index_];
+          if (element) {
+            mutex_lock l(element->mu);
+            if (!element->results.empty()) {
+              if (element->results.front()->is_ready) {
+                // We found a result.
+                std::swap(*result, element->results.front());
+                element->results.pop_front();
+                AdvancePosition();
+                cond_var_->notify_all();
+                return true;
+              } else {
+                // Wait for the result to become ready.
+                return false;
+              }
+            } else if (!element->iterator) {
+              // We reached the end of input for this element. Reset
+              // it and move on to the next cycle element.
+              current_elements_[cycle_index_].reset();
+              AdvanceToNextInCycle();
+              cond_var_->notify_all();
+              continue;
+            } else {
+              // Wait for the iterator to produce a result.
+              return false;
+            }
+          } else {
+            if (!future_elements_.empty() || !end_of_input_) {
+              // Wait for an element to be created.
+              return false;
+            }
+            // No new elements will be created; try to find a
+            // non-empty element in the cycle.
+            for (int i = 0; i < dataset()->cycle_length_; ++i) {
+              AdvanceToNextInCycle();
+              if (current_elements_[cycle_index_]) {
+                break;
+              }
+            }
+            if (current_elements_[cycle_index_]) {
+              continue;
+            }
+            // End of input has been reached.
+            return true;
+          }
         }
       }
 
-      // Fetches up to `results.size()` outputs from the cycle element at
-      // position `cycle_index`.
+      // Manages current cycle elements, creating new iterators as needed and
+      // asynchronously fetching results from existing iterators.
       //
-      // If end of input is encountered, the `skip` field of the invocation
-      // result is used to identify results that should be skipped.
-      void FetchOutputs(
-          const std::shared_ptr<IteratorContext>& ctx, IteratorBase* iterator,
-          int64 cycle_index,
-          const std::vector<std::shared_ptr<InvocationResult>>& results)
-          LOCKS_EXCLUDED(*mu_) {
+      // This method runs in the `current_elements_manager_` background thread.
+      void CurrentElementsManager(const std::shared_ptr<IteratorContext>& ctx) {
         RecordStart(ctx.get());
         auto cleanup = gtl::MakeCleanup([this, ctx] { RecordStop(ctx.get()); });
-        bool end_of_input = false;
-        for (auto& result : results) {
-          if (!end_of_input) {
-            result->status = iterator->GetNext(
-                ctx.get(), &result->return_values, &end_of_input);
+        auto busy = [this]() EXCLUSIVE_LOCKS_REQUIRED(*mu_) -> bool {
+          const bool has_more_elements =
+              !future_elements_.empty() || !end_of_input_;
+          const int block_length = dataset()->block_length_;
+          bool all_elements_busy = true;
+          for (auto& element : current_elements_) {
+            if (!element) {
+              if (has_more_elements) {
+                all_elements_busy = false;
+                break;
+              }
+            } else {
+              mutex_lock l(element->mu);
+              if (!element->in_use && element->iterator &&
+                  element->results.size() < block_length) {
+                all_elements_busy = false;
+                break;
+              }
+            }
           }
-          if (end_of_input) {
-            result->skip = true;
+          return all_elements_busy || num_calls_ >= num_parallel_calls_->value;
+        };
+        while (true) {
+          mutex_lock l(*mu_);
+
+          // Wait until this thread is cancelled, the end of input has been
+          // reached.
+          while (!cancelled_ && (!end_of_input_ || num_open_ > 0) && busy()) {
+            RecordStop(ctx.get());
+            cond_var_->wait(l);
+            RecordStart(ctx.get());
           }
-          RecordBufferEnqueue(ctx.get(), result->return_values);
-          {
-            mutex_lock l(*mu_);
-            result->notification.Notify();
-            cond_var_->notify_all();
+
+          if (cancelled_ ||
+              (future_elements_.empty() && end_of_input_ && num_open_ == 0)) {
+            return;
+          }
+
+          for (int i = 0; i < dataset()->cycle_length_; ++i) {
+            int idx = (cycle_index_ + i) % dataset()->cycle_length_;
+            if (!current_elements_[idx]) {
+              if (!future_elements_.empty()) {
+                current_elements_[idx] = std::move(future_elements_.back());
+                future_elements_.pop_back();
+              } else {
+                current_elements_[idx] = MakeElement(ctx);
+                if (!current_elements_[idx]) {
+                  continue;
+                }
+              }
+            }
+            std::shared_ptr<Element> element = current_elements_[idx];
+            if (!element->in_use && element->iterator) {
+              int64 num_results;
+              {
+                mutex_lock l(element->mu);
+                num_results =
+                    dataset()->block_length_ - element->results.size();
+              }
+              if (num_results > 0) {
+                num_calls_++;
+                element->in_use = true;
+                thread_pool_->Schedule(
+                    std::bind(&ParallelInterleaveIterator::FetchResults, this,
+                              ctx, std::move(element), num_results));
+              }
+            }
+          }
+          const auto& stats_aggregator = ctx->stats_aggregator();
+          if (stats_aggregator) {
+            stats_aggregator->AddScalar(
+                strings::StrCat(key_prefix_, "::thread_utilization"),
+                static_cast<float>(num_calls_) /
+                    static_cast<float>(num_parallel_calls_->value));
           }
-          if (!result->status.ok()) {
+          cond_var_->notify_all();
+        }
+      }
+
+      void EnsureThreadsStarted(IteratorContext* ctx)
+          EXCLUSIVE_LOCKS_REQUIRED(*mu_) {
+        if (!current_elements_manager_) {
+          auto new_ctx = std::make_shared<IteratorContext>(*ctx);
+          current_elements_manager_ =
+              WrapUnique<Thread>(ctx->env()->StartThread(
+                  {}, "tf_data_parallel_interleave_current",
+                  [this, new_ctx]() { CurrentElementsManager(new_ctx); }));
+        }
+        if (!future_elements_manager_) {
+          auto new_ctx = std::make_shared<IteratorContext>(*ctx);
+          future_elements_manager_ = WrapUnique<Thread>(ctx->env()->StartThread(
+              {}, "tf_data_parallel_interleave_future",
+              [this, new_ctx]() { FutureElementsManager(new_ctx); }));
+        }
+      }
+
+      // Fetches up to `dataset()->block_length_` results from `element`.
+      void FetchResults(const std::shared_ptr<IteratorContext>& ctx,
+                        const std::shared_ptr<Element>& element,
+                        int64 num_results) LOCKS_EXCLUDED(*mu_) {
+        RecordStart(ctx.get());
+        auto cleanup = gtl::MakeCleanup([this, ctx] { RecordStop(ctx.get()); });
+        bool end_of_input = false;
+        for (int64 i = 0; i < num_results; ++i) {
+          auto result = std::make_shared<Result>();
+          result->status = element->iterator->GetNext(
+              ctx.get(), &result->return_values, &end_of_input);
+          if (end_of_input) {
             break;
           }
+          RecordBufferEnqueue(ctx.get(), result->return_values);
+          mutex_lock l(*mu_);
+          mutex_lock l2(element->mu);
+          element->results.push_back(result);
+          result->is_ready = true;
+          cond_var_->notify_all();
         }
 
         mutex_lock l(*mu_);
-        current_elements_[cycle_index].in_use = false;
+        // Release the ownership of the cycle element iterator.
+        element->in_use = false;
         if (end_of_input) {
-          // Release the ownership of the cycle element iterator, closing the
-          // iterator if end of input was encountered.
-          current_elements_[cycle_index].iterator.reset();
-          current_elements_[cycle_index].inputs.clear();
-          num_open_--;
+          // Close the iterator if end of input was encountered.
+          element->iterator.reset();
+          element->inputs.clear();
+          --num_open_;
         }
-        num_calls_--;
+        --num_calls_;
         const auto& stats_aggregator = ctx->stats_aggregator();
         if (stats_aggregator) {
           stats_aggregator->AddScalar(
@@ -437,82 +566,47 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
         cond_var_->notify_all();
       }
 
-      // Method responsible for 1) creating iterators out of input elements, 2)
-      // determining the order in which elements are fetched from the iterators,
-      // and 3) scheduling the fetching of the elements to a threadpool.
+      // Manages futures cycle elements, creating new iterators as needed and
+      // asynchronously fetching results from existing iterators.
       //
-      // This method runs in the `runner_thread` background thread.
-      void RunnerThread(const std::shared_ptr<IteratorContext>& ctx) {
+      // This method runs in the `future_elements_manager_` background thread.
+      void FutureElementsManager(const std::shared_ptr<IteratorContext>& ctx) {
         RecordStart(ctx.get());
         auto cleanup = gtl::MakeCleanup([this, ctx] { RecordStop(ctx.get()); });
         auto busy = [this]() EXCLUSIVE_LOCKS_REQUIRED(*mu_) -> bool {
-          return current_elements_[cycle_index_].in_use ||
-                 num_calls_ >= num_parallel_calls_->value ||
-                 invocation_results_.size() >=
-                     dataset()->cycle_length_ * dataset()->block_length_;
+          return num_calls_ >= num_parallel_calls_->value ||
+                 future_elements_.size() >= dataset()->cycle_length_;
         };
         while (true) {
           mutex_lock l(*mu_);
+
           // Wait until this thread is cancelled, the end of input has been
           // reached, or the cycle element at the `cycle_index_` position is
-          // not in use and there is space in the `invocation_results_` queue.
-          while (!cancelled_ && (!end_of_input_ || num_open_ > 0) && busy()) {
+          // not in use.
+          while (!cancelled_ && !end_of_input_ && busy()) {
             RecordStop(ctx.get());
             cond_var_->wait(l);
             RecordStart(ctx.get());
           }
 
-          if (cancelled_ || (end_of_input_ && num_open_ == 0)) {
+          if (cancelled_ || end_of_input_) {
             return;
           }
 
-          while ((!end_of_input_ || num_open_ > 0) && !busy()) {
-            if (!current_elements_[cycle_index_].iterator) {
-              // Try to create a new iterator from the next input element.
-              Status status = input_impl_->GetNext(
-                  ctx.get(), &current_elements_[cycle_index_].inputs,
-                  &end_of_input_);
-              if (!status.ok()) {
-                invocation_results_.emplace_back(new InvocationResult());
-                std::shared_ptr<InvocationResult>& result =
-                    invocation_results_.back();
-                result->status.Update(status);
-                result->notification.Notify();
-                break;
-              }
-              if (!end_of_input_) {
-                Status status = MakeIteratorFromInputElement(
-                    ctx.get(), current_elements_[cycle_index_].inputs,
-                    cycle_index_, *instantiated_captured_func_, prefix(),
-                    &current_elements_[cycle_index_].iterator);
-                if (!status.ok()) {
-                  invocation_results_.emplace_back(new InvocationResult());
-                  std::shared_ptr<InvocationResult>& result =
-                      invocation_results_.back();
-                  result->status.Update(status);
-                  result->notification.Notify();
-                  break;
-                }
-                ++num_open_;
-              }
+          while (!end_of_input_ && !busy()) {
+            std::shared_ptr<Element> element = MakeElement(ctx);
+            if (!element) {
+              break;
             }
-            if (current_elements_[cycle_index_].iterator) {
-              // Pre-allocate invocation results for outputs to be fetched
-              // and then fetch the outputs asynchronously.
-              std::vector<std::shared_ptr<InvocationResult>> results;
-              results.reserve(dataset()->block_length_);
-              for (int i = 0; i < dataset()->block_length_; ++i) {
-                invocation_results_.emplace_back(new InvocationResult());
-                results.push_back(invocation_results_.back());
-              }
-              num_calls_++;
-              current_elements_[cycle_index_].in_use = true;
-              thread_pool_->Schedule(
-                  std::bind(&ParallelInterleaveIterator::FetchOutputs, this,
-                            ctx, current_elements_[cycle_index_].iterator.get(),
-                            cycle_index_, std::move(results)));
+            future_elements_.push_front(element);
+            if (!element->iterator) {
+              continue;
             }
-            cycle_index_ = (cycle_index_ + 1) % dataset()->cycle_length_;
+            ++num_calls_;
+            element->in_use = true;
+            thread_pool_->Schedule(
+                std::bind(&ParallelInterleaveIterator::FetchResults, this, ctx,
+                          std::move(element), dataset()->block_length_));
           }
           const auto& stats_aggregator = ctx->stats_aggregator();
           if (stats_aggregator) {
@@ -525,56 +619,66 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
         }
       }
 
-      // Determines whether the caller needs to wait for a result. Upon
-      // returning false, `result` will either be NULL if end of input has been
-      // reached or point to the result.
-      bool ShouldWait(std::shared_ptr<InvocationResult>* result)
+      // Creates a new element.
+      std::shared_ptr<Element> MakeElement(
+          const std::shared_ptr<IteratorContext>& ctx)
           EXCLUSIVE_LOCKS_REQUIRED(*mu_) {
-        if (sloppy_) {
-          for (auto it = invocation_results_.begin();
-               it != invocation_results_.end(); ++it) {
-            if ((*it)->notification.HasBeenNotified()) {
-              std::swap(*result, *it);
-              invocation_results_.erase(it);
-              cond_var_->notify_all();
-              return false;
-            }
+        auto element = std::make_shared<Element>();
+        element->id = element_id_counter_++;
+        Status status =
+            input_impl_->GetNext(ctx.get(), &element->inputs, &end_of_input_);
+        if (!status.ok()) {
+          auto result = std::make_shared<Result>();
+          result->is_ready = true;
+          result->status = status;
+          mutex_lock l(element->mu);
+          element->results.push_back(std::move(result));
+          return element;
+        }
+        if (!end_of_input_) {
+          Status status = MakeIteratorFromInputElement(
+              ctx.get(), element->inputs, element->id,
+              *instantiated_captured_func_, prefix(), &element->iterator);
+          if (!status.ok()) {
+            auto result = std::make_shared<Result>();
+            result->is_ready = true;
+            result->status = status;
+            mutex_lock l(element->mu);
+            element->results.push_back(std::move(result));
+            return element;
           }
-          return !invocation_results_.empty() ||
-                 (!end_of_input_ || num_open_ > 0);
+          ++num_open_;
         } else {
-          if (!invocation_results_.empty()) {
-            std::swap(*result, invocation_results_.front());
-            invocation_results_.pop_front();
-            cond_var_->notify_all();
-            return false;
-          }
-          return (!end_of_input_ || num_open_ > 0);
+          element.reset();
         }
+        return element;
       }
 
-      Status WriteStatusLocked(IteratorStateWriter* writer, size_t index,
+      Status WriteStatusLocked(IteratorStateWriter* writer,
+                               const string& key_prefix, size_t idx,
                                const Status& status)
           EXCLUSIVE_LOCKS_REQUIRED(*mu_) {
         TF_RETURN_IF_ERROR(writer->WriteScalar(
-            CodeKey(index), static_cast<int64>(status.code())));
+            CodeKey(key_prefix, idx), static_cast<int64>(status.code())));
         if (!status.ok()) {
-          TF_RETURN_IF_ERROR(writer->WriteScalar(ErrorMessageKey(index),
-                                                 status.error_message()));
+          TF_RETURN_IF_ERROR(writer->WriteScalar(
+              ErrorMessageKey(key_prefix, idx), status.error_message()));
         }
         return Status::OK();
       }
 
-      Status ReadStatusLocked(IteratorStateReader* reader, size_t index,
+      Status ReadStatusLocked(IteratorStateReader* reader,
+                              const string& key_prefix, size_t idx,
                               Status* status) EXCLUSIVE_LOCKS_REQUIRED(*mu_) {
         int64 code_int;
-        TF_RETURN_IF_ERROR(reader->ReadScalar(CodeKey(index), &code_int));
+        TF_RETURN_IF_ERROR(
+            reader->ReadScalar(CodeKey(key_prefix, idx), &code_int));
         error::Code code = static_cast<error::Code>(code_int);
 
         if (code != error::Code::OK) {
           string error_message;
-          TF_RETURN_IF_ERROR(
-              reader->ReadScalar(ErrorMessageKey(index), &error_message));
+          TF_RETURN_IF_ERROR(reader->ReadScalar(
+              ErrorMessageKey(key_prefix, idx), &error_message));
           *status = Status(code, error_message);
         } else {
           *status = Status::OK();
@@ -582,64 +686,178 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
         return Status::OK();
       }
 
-      string CodeKey(size_t index) {
+      string CodeKey(const string& key_prefix, size_t idx) {
         return full_name(
-            strings::StrCat("invocation_results[", index, "].code"));
+            strings::StrCat(key_prefix, ".results[", idx, "].code"));
       }
 
-      string ErrorMessageKey(size_t index) {
+      string ErrorMessageKey(const string& key_prefix, size_t idx) {
         return full_name(
-            strings::StrCat("invocation_results[", index, "].error_message"));
+            strings::StrCat(key_prefix, ".results[", idx, "].error_message"));
+      }
+
+      Status WriteElement(std::shared_ptr<Element> element, int idx,
+                          const string& key_prefix, IteratorStateWriter* writer)
+          EXCLUSIVE_LOCKS_REQUIRED(*mu_) {
+        if (element->iterator) {
+          TF_RETURN_IF_ERROR(SaveInput(writer, element->iterator));
+          TF_RETURN_IF_ERROR(writer->WriteScalar(
+              full_name(strings::StrCat(key_prefix, "[", idx, "].id")),
+              element->id));
+          TF_RETURN_IF_ERROR(writer->WriteScalar(
+              full_name(strings::StrCat(key_prefix, "[", idx, "].inputs.size")),
+              element->inputs.size()));
+          for (int i = 0; i < element->inputs.size(); i++) {
+            TF_RETURN_IF_ERROR(writer->WriteTensor(
+                full_name(
+                    strings::StrCat(key_prefix, "[", idx, "].inputs[", i, "]")),
+                element->inputs[i]));
+          }
+        }
+        mutex_lock l(element->mu);
+        TF_RETURN_IF_ERROR(writer->WriteScalar(
+            full_name(strings::StrCat(key_prefix, "[", idx, "].results.size")),
+            element->results.size()));
+        for (size_t i = 0; i < element->results.size(); i++) {
+          std::shared_ptr<Result> result = element->results[i];
+          TF_RETURN_IF_ERROR(WriteStatusLocked(
+              writer, strings::StrCat(key_prefix, "[", idx, "]"), i,
+              result->status));
+          TF_RETURN_IF_ERROR(writer->WriteScalar(
+              full_name(strings::StrCat(key_prefix, "[", idx, "].results[", i,
+                                        "].size")),
+              result->return_values.size()));
+          for (size_t j = 0; j < result->return_values.size(); j++) {
+            TF_RETURN_IF_ERROR(writer->WriteTensor(
+                full_name(strings::StrCat(key_prefix, "[", idx, "].results[", i,
+                                          "][", j, "]")),
+                result->return_values[j]));
+          }
+          if (result->is_ready) {
+            TF_RETURN_IF_ERROR(writer->WriteScalar(
+                full_name(strings::StrCat(key_prefix, "[", idx, "].results[", i,
+                                          "].is_ready")),
+                ""));
+          }
+        }
+        return Status::OK();
       }
 
       Status WriteCurrentElements(IteratorStateWriter* writer)
           EXCLUSIVE_LOCKS_REQUIRED(*mu_) {
+        TF_RETURN_IF_ERROR(writer->WriteScalar(
+            full_name("current_elements.size"), current_elements_.size()));
         for (int idx = 0; idx < current_elements_.size(); idx++) {
-          if (current_elements_[idx].iterator) {
-            TF_RETURN_IF_ERROR(
-                SaveInput(writer, current_elements_[idx].iterator));
-            TF_RETURN_IF_ERROR(writer->WriteScalar(
-                full_name(
-                    strings::StrCat("current_elements[", idx, "].inputs.size")),
-                current_elements_[idx].inputs.size()));
-            for (int i = 0; i < current_elements_[idx].inputs.size(); i++) {
-              TF_RETURN_IF_ERROR(writer->WriteTensor(
-                  full_name(strings::StrCat("current_elements[", idx,
-                                            "].inputs[", i, "]")),
-                  current_elements_[idx].inputs[i]));
-            }
+          if (current_elements_[idx]) {
+            TF_RETURN_IF_ERROR(WriteElement(current_elements_[idx], idx,
+                                            "current_elements", writer));
           }
         }
         return Status::OK();
       }
 
+      Status WriteFutureElements(IteratorStateWriter* writer)
+          EXCLUSIVE_LOCKS_REQUIRED(*mu_) {
+        TF_RETURN_IF_ERROR(writer->WriteScalar(
+            full_name("future_elements.size"), future_elements_.size()));
+        for (int idx = 0; idx < future_elements_.size(); idx++) {
+          if (future_elements_[idx]) {
+            TF_RETURN_IF_ERROR(WriteElement(future_elements_[idx], idx,
+                                            "future_elements", writer));
+          }
+        }
+        return Status::OK();
+      }
+
+      Status ReadElement(IteratorContext* ctx, IteratorStateReader* reader,
+                         int idx, const string& key_prefix,
+                         std::shared_ptr<Element>* out)
+          EXCLUSIVE_LOCKS_REQUIRED(*mu_) {
+        if (!reader->Contains(full_name(
+                strings::StrCat(key_prefix, "[", idx, "].results.size")))) {
+          return Status::OK();
+        }
+        auto element = std::make_shared<Element>();
+        mutex_lock l(element->mu);
+        int64 results_size;
+        TF_RETURN_IF_ERROR(reader->ReadScalar(
+            full_name(strings::StrCat(key_prefix, "[", idx, "].results.size")),
+            &results_size));
+        element->results.resize(results_size);
+        for (size_t i = 0; i < results_size; i++) {
+          auto result = std::make_shared<Result>();
+          TF_RETURN_IF_ERROR(ReadStatusLocked(
+              reader, strings::StrCat(key_prefix, "[", idx, "]"), i,
+              &result->status));
+          int64 num_return_values;
+          TF_RETURN_IF_ERROR(reader->ReadScalar(
+              full_name(strings::StrCat(key_prefix, "[", idx, "].results[", i,
+                                        "].size")),
+              &num_return_values));
+          result->return_values.reserve(num_return_values);
+          for (size_t j = 0; j < num_return_values; j++) {
+            result->return_values.emplace_back();
+            TF_RETURN_IF_ERROR(reader->ReadTensor(
+                full_name(strings::StrCat(key_prefix, "[", idx, "].results[", i,
+                                          "][", j, "]")),
+                &result->return_values.back()));
+          }
+          result->is_ready = reader->Contains(full_name(strings::StrCat(
+              key_prefix, "[", idx, "].results[", i, "].is_ready")));
+          element->results[i] = std::move(result);
+        }
+        if (!reader->Contains(full_name(
+                strings::StrCat(key_prefix, "[", idx, "].inputs.size")))) {
+          element->iterator.reset();
+          *out = std::move(element);
+          return Status::OK();
+        }
+        int64 inputs_size;
+        TF_RETURN_IF_ERROR(reader->ReadScalar(
+            full_name(strings::StrCat(key_prefix, "[", idx, "].inputs.size")),
+            &inputs_size));
+        element->inputs.resize(inputs_size);
+        for (int i = 0; i < inputs_size; i++) {
+          TF_RETURN_IF_ERROR(reader->ReadTensor(
+              full_name(
+                  strings::StrCat(key_prefix, "[", idx, "].inputs[", i, "]")),
+              &element->inputs[i]));
+        }
+        TF_RETURN_IF_ERROR(reader->ReadScalar(
+            full_name(strings::StrCat(key_prefix, "[", idx, "].id")),
+            &element->id));
+        TF_RETURN_IF_ERROR(MakeIteratorFromInputElement(
+            ctx, element->inputs, element->id,
+            *instantiated_captured_func_.get(), prefix(), &element->iterator));
+        TF_RETURN_IF_ERROR(RestoreInput(ctx, reader, element->iterator));
+        *out = std::move(element);
+        return Status::OK();
+      }
+
       Status ReadCurrentElements(IteratorContext* ctx,
                                  IteratorStateReader* reader)
           EXCLUSIVE_LOCKS_REQUIRED(*mu_) {
+        int64 size;
+        TF_RETURN_IF_ERROR(
+            reader->ReadScalar(full_name("current_elements.size"), &size));
+        DCHECK_EQ(current_elements_.size(), size);
         for (int idx = 0; idx < current_elements_.size(); idx++) {
-          if (reader->Contains(full_name(strings::StrCat(
-                  "current_elements[", idx, "].inputs.size")))) {
-            int64 inputs_size;
-            TF_RETURN_IF_ERROR(reader->ReadScalar(
-                full_name(
-                    strings::StrCat("current_elements[", idx, "].inputs.size")),
-                &inputs_size));
-            current_elements_[idx].inputs.resize(inputs_size);
-            for (int i = 0; i < inputs_size; i++) {
-              TF_RETURN_IF_ERROR(reader->ReadTensor(
-                  full_name(strings::StrCat("current_elements[", idx,
-                                            "].inputs[", i, "]")),
-                  &current_elements_[idx].inputs[i]));
-            }
-            TF_RETURN_IF_ERROR(MakeIteratorFromInputElement(
-                ctx, current_elements_[idx].inputs, idx,
-                *instantiated_captured_func_.get(), prefix(),
-                &current_elements_[idx].iterator));
-            TF_RETURN_IF_ERROR(
-                RestoreInput(ctx, reader, current_elements_[idx].iterator));
-          } else {
-            current_elements_[idx].iterator.reset();
-          }
+          TF_RETURN_IF_ERROR(ReadElement(ctx, reader, idx, "current_elements",
+                                         &current_elements_[idx]));
+        }
+        return Status::OK();
+      }
+
+      Status ReadFutureElements(IteratorContext* ctx,
+                                IteratorStateReader* reader)
+          EXCLUSIVE_LOCKS_REQUIRED(*mu_) {
+        int64 size;
+        TF_RETURN_IF_ERROR(
+            reader->ReadScalar(full_name("future_elements.size"), &size));
+        future_elements_.resize(size);
+        for (int idx = 0; idx < future_elements_.size(); idx++) {
+          TF_RETURN_IF_ERROR(ReadElement(ctx, reader, idx, "future_elements",
+                                         &future_elements_[idx]));
         }
         return Status::OK();
       }
@@ -648,12 +866,11 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
       // the worker threads.
       const std::shared_ptr<mutex> mu_;
 
-      // Used for coordination between the main thread, the runner thread, and
-      // the worker threads. In particular, the runner thread should only
-      // schedule new calls when the number of in-flight calls is less than the
-      // user specified level of parallelism, there are slots available in the
-      // `invocation_results_` buffer, the current cycle element is not in use,
-      // and there are elements left to be fetched.
+      // Used for coordination between the main thread, the manager threads, and
+      // the threadpool threads. In particular, the managers thread should only
+      // schedule new calls into the threadpool when the number of in-flight
+      // calls is less than the user specified level of parallelism and there
+      // are slots available in the element `results` buffer.
       const std::shared_ptr<condition_variable> cond_var_;
 
       // Identifies the maximum number of parallel calls.
@@ -665,18 +882,17 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
       // Iterator for input elements.
       std::unique_ptr<IteratorBase> input_impl_ GUARDED_BY(*mu_);
 
-      // Identifies current cycle element.
-      int64 cycle_index_ = 0;
+      // Identifies position in the interleave cycle.
+      int64 block_index_ GUARDED_BY(*mu_) = 0;
+      int64 cycle_index_ GUARDED_BY(*mu_) = 0;
 
-      // Iterators for the current cycle elements. Concurrent access is
-      // protected by `element_in_use_`.
-      std::vector<Element> current_elements_ GUARDED_BY(*mu_);
+      // Elements of the current interleave cycle.
+      std::vector<std::shared_ptr<Element>> current_elements_ GUARDED_BY(*mu_);
 
-      // Buffer for storing the invocation results.
-      std::deque<std::shared_ptr<InvocationResult>> invocation_results_
-          GUARDED_BY(*mu_);
+      // Elements to be used in the interleave cycle in the future.
+      std::deque<std::shared_ptr<Element>> future_elements_ GUARDED_BY(*mu_);
 
-      // Identifies whether end of input has been reached.
+      // Identifies whether the global end of input has been reached.
       bool end_of_input_ GUARDED_BY(*mu_) = false;
 
       // Identifies the number of open iterators.
@@ -686,9 +902,11 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
       int64 num_calls_ GUARDED_BY(*mu_) = 0;
 
       std::unique_ptr<thread::ThreadPool> thread_pool_;
-      std::unique_ptr<Thread> runner_thread_ GUARDED_BY(*mu_);
+      std::unique_ptr<Thread> current_elements_manager_ GUARDED_BY(*mu_);
+      std::unique_ptr<Thread> future_elements_manager_ GUARDED_BY(*mu_);
+      int64 element_id_counter_ GUARDED_BY(*mu_) = 0;
 
-      // Identifies whether background activity should be cancelled.
+      // Identifies whether background threads should be cancelled.
       bool cancelled_ GUARDED_BY(*mu_) = false;
       string key_prefix_;
       std::unique_ptr<InstantiatedCapturedFunction> instantiated_captured_func_;
diff --git a/tensorflow/python/data/kernel_tests/interleave_test.py b/tensorflow/python/data/kernel_tests/interleave_test.py
index 4fb61b2daf..4b427ff5a4 100644
--- a/tensorflow/python/data/kernel_tests/interleave_test.py
+++ b/tensorflow/python/data/kernel_tests/interleave_test.py
@@ -17,19 +17,15 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import threading
-
 from absl.testing import parameterized
 import numpy as np
 
-from tensorflow.python.data.experimental.ops import threading_options
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import script_ops
 from tensorflow.python.ops import sparse_ops
 from tensorflow.python.platform import test
 
@@ -78,47 +74,6 @@ def _interleave(lists, cycle_length, block_length):
           break
 
 
-def _make_coordinated_sloppy_dataset(input_values, cycle_length, block_length,
-                                     num_parallel_calls):
-  """Produces a dataset iterator and events to control the order of elements.
-
-  Args:
-    input_values: the values to generate lists to interleave from
-    cycle_length: the length of the interleave cycle
-    block_length: the length of the interleave block
-    num_parallel_calls: the degree of interleave parallelism
-
-  Returns:
-    A dataset iterator (represented as `get_next` op) and events that can be
-    used to control the order of output elements.
-  """
-
-  # Set up threading events used to sequence when items are produced that
-  # are subsequently interleaved. These events allow us to deterministically
-  # simulate slowdowns and force sloppiness.
-  coordination_events = {i: threading.Event() for i in input_values}
-
-  def map_py_fn(x):
-    coordination_events[x].wait()
-    coordination_events[x].clear()
-    return x * x
-
-  def map_fn(x):
-    return script_ops.py_func(map_py_fn, [x], x.dtype)
-
-  def interleave_fn(x):
-    dataset = dataset_ops.Dataset.from_tensors(x)
-    dataset = dataset.repeat(x)
-    return dataset.map(map_fn)
-
-  options = dataset_ops.Options()
-  options.experimental_deterministic = False
-  dataset = dataset_ops.Dataset.from_tensor_slices(input_values).repeat(
-      2).interleave(interleave_fn, cycle_length, block_length,
-                    num_parallel_calls).with_options(options)
-  return dataset, coordination_events
-
-
 def _repeat(values, count):
   """Produces a list of lists suitable for testing interleave.
 
@@ -252,63 +207,37 @@ class InterleaveTest(test_base.DatasetTestBase, parameterized.TestCase):
       self.evaluate(get_next())
 
   @parameterized.named_parameters(
-      ("1", np.int64([4, 5, 6]), 2, 1, 1),
-      ("2", np.int64([4, 5, 6]), 2, 1, 2),
-      ("3", np.int64([4, 5, 6]), 2, 3, 1),
-      ("4", np.int64([4, 5, 6]), 2, 3, 2),
-      ("5", np.int64([4, 5, 6]), 3, 2, 1),
-      ("6", np.int64([4, 5, 6]), 3, 2, 2),
-      ("7", np.int64([4, 5, 6]), 3, 2, 3),
-      ("8", np.int64([4, 0, 6]), 2, 3, 1),
-      ("9", np.int64([4, 0, 6]), 2, 3, 2),
+      ("1", np.int64([4, 5, 6]), 1, 3, 1),
+      ("2", np.int64([4, 5, 6]), 2, 1, 1),
+      ("3", np.int64([4, 5, 6]), 2, 1, 2),
+      ("4", np.int64([4, 5, 6]), 2, 3, 1),
+      ("5", np.int64([4, 5, 6]), 2, 3, 2),
+      ("6", np.int64([4, 5, 6]), 7, 2, 1),
+      ("7", np.int64([4, 5, 6]), 7, 2, 3),
+      ("8", np.int64([4, 5, 6]), 7, 2, 5),
+      ("9", np.int64([4, 5, 6]), 7, 2, 7),
+      ("10", np.int64([4, 0, 6]), 2, 3, 1),
+      ("11", np.int64([4, 0, 6]), 2, 3, 2),
   )
-  def testSloppyInterleaveInOrder(self, input_values, cycle_length,
+  def testSloppyInterleaveDataset(self, input_values, cycle_length,
                                   block_length, num_parallel_calls):
-    dataset, coordination_events = _make_coordinated_sloppy_dataset(
-        input_values, cycle_length, block_length, num_parallel_calls)
-    options = dataset_ops.Options()
-    options.experimental_threading = threading_options.ThreadingOptions()
-    options.experimental_threading.private_threadpool_size = (
-        num_parallel_calls + 1)
-    dataset = dataset.with_options(options)
-
-    get_next = self.getNext(dataset, requires_initialization=True)
-    for expected_element in _interleave(
-        _repeat(input_values, 2), cycle_length, block_length):
-      coordination_events[expected_element].set()
-      self.assertEqual(expected_element * expected_element,
-                       self.evaluate(get_next()))
-    with self.assertRaises(errors.OutOfRangeError):
-      self.evaluate(get_next())
-
-  @parameterized.named_parameters(
-      ("1", np.int64([4, 5, 6]), 2, 1, 2),
-      ("2", np.int64([4, 5, 6]), 2, 3, 2),
-      ("3", np.int64([4, 5, 6]), 3, 2, 3),
-      ("4", np.int64([4, 0, 6]), 2, 3, 2),
-  )
-  def testSloppyInterleaveOutOfOrder(self, input_values, cycle_length,
-                                     block_length, num_parallel_calls):
-    dataset, coordination_events = _make_coordinated_sloppy_dataset(
-        input_values, cycle_length, block_length, num_parallel_calls)
+    count = 2
+    dataset = dataset_ops.Dataset.from_tensor_slices(input_values).repeat(
+        count).interleave(
+            lambda x: dataset_ops.Dataset.from_tensors(x).repeat(x),
+            cycle_length, block_length, num_parallel_calls)
     options = dataset_ops.Options()
-    options.experimental_threading = threading_options.ThreadingOptions()
-    options.experimental_threading.private_threadpool_size = (
-        num_parallel_calls + 1)
+    options.experimental_deterministic = False
     dataset = dataset.with_options(options)
-    get_next = self.getNext(dataset, requires_initialization=True)
-    elements = [
-        x for x in _interleave(
-            _repeat(input_values, 2), cycle_length, block_length)
+    expected_output = [
+        element for element in _interleave(
+            _repeat(input_values, count), cycle_length, block_length)
     ]
-    for i in [1, 4, 7]:
-      elements[i], elements[i + 1] = elements[i + 1], elements[i]
-
-    for element in elements:
-      coordination_events[element].set()
-      self.assertEqual(element * element, self.evaluate(get_next()))
-    with self.assertRaises(errors.OutOfRangeError):
-      self.evaluate(get_next())
+    get_next = self.getNext(dataset)
+    actual_output = []
+    for _ in range(len(expected_output)):
+      actual_output.append(self.evaluate(get_next()))
+    self.assertAllEqual(expected_output.sort(), actual_output.sort())
 
 
 if __name__ == "__main__":
-- 
GitLab


From dba2134638812e6375a26012a485b55a24c45a19 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 14:26:00 -0800
Subject: [PATCH 0211/2345] Switch RBE image location for CUDA 9 to
 tensorflow-testing.

PiperOrigin-RevId: 227914044
---
 .../tools/ci_build/Dockerfile.rbe.cuda9.0-cudnn7-ubuntu14.04  | 4 ++--
 third_party/toolchains/BUILD                                  | 2 +-
 third_party/toolchains/preconfig/generate/containers.bzl      | 2 +-
 third_party/toolchains/preconfig/generate/workspace.bzl       | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/tools/ci_build/Dockerfile.rbe.cuda9.0-cudnn7-ubuntu14.04 b/tensorflow/tools/ci_build/Dockerfile.rbe.cuda9.0-cudnn7-ubuntu14.04
index 9d5d51447c..4ce4214065 100644
--- a/tensorflow/tools/ci_build/Dockerfile.rbe.cuda9.0-cudnn7-ubuntu14.04
+++ b/tensorflow/tools/ci_build/Dockerfile.rbe.cuda9.0-cudnn7-ubuntu14.04
@@ -1,7 +1,7 @@
 # To push a new version, run:
 # $ docker build -f Dockerfile.rbe.cuda9.0-cudnn7-ubuntu14.04 \
-#       --tag "gcr.io/asci-toolchain/nosla-cuda9.0-cudnn7-ubuntu14.04" .
-# $ docker push gcr.io/asci-toolchain/nosla-cuda9.0-cudnn7-ubuntu14.04
+#       --tag "gcr.io/tensorflow-testing/nosla-cuda9.0-cudnn7-ubuntu14.04" .
+# $ docker push gcr.io/tensorflow-testing/nosla-cuda9.0-cudnn7-ubuntu14.04
 #
 # TODO(klimek): Include clang in this image so we can also target clang
 # builds.
diff --git a/third_party/toolchains/BUILD b/third_party/toolchains/BUILD
index 28f1e6979c..f2248341c4 100644
--- a/third_party/toolchains/BUILD
+++ b/third_party/toolchains/BUILD
@@ -32,7 +32,7 @@ platform(
     remote_execution_properties = """
         properties: {
             name: "container-image"
-            value:"docker://gcr.io/asci-toolchain/nosla-cuda9.0-cudnn7-ubuntu14.04@%s"
+            value:"docker://gcr.io/tensorflow-testing/nosla-cuda9.0-cudnn7-ubuntu14.04@%s"
         }""" % container_digests["cuda9.0-cudnn7-ubuntu14.04"],
 )
 
diff --git a/third_party/toolchains/preconfig/generate/containers.bzl b/third_party/toolchains/preconfig/generate/containers.bzl
index 32d5f29b52..67e43485ed 100644
--- a/third_party/toolchains/preconfig/generate/containers.bzl
+++ b/third_party/toolchains/preconfig/generate/containers.bzl
@@ -1,4 +1,4 @@
 container_digests = {
-    "cuda9.0-cudnn7-ubuntu14.04": "sha256:c43ed5341dd765042e0bbd1bf50fadeedd649d1e0c34d81999cb6ce30916cb95",
+    "cuda9.0-cudnn7-ubuntu14.04": "sha256:006a76ee1838122ff7f21ebac85f24c1ef350d4dd79b3ceff0e4fe649ed90d33",
     "cuda10.0-cudnn7-ubuntu14.04": "sha256:e36f05f1ff39e39ddf07122e37f2b1895948bb6f7acc3db37a3c496be5e66228",
 }
diff --git a/third_party/toolchains/preconfig/generate/workspace.bzl b/third_party/toolchains/preconfig/generate/workspace.bzl
index a3268585e1..eb74022c24 100644
--- a/third_party/toolchains/preconfig/generate/workspace.bzl
+++ b/third_party/toolchains/preconfig/generate/workspace.bzl
@@ -11,7 +11,7 @@ def _remote_config_workspace():
     container_pull(
         name = "cuda9.0-cudnn7-ubuntu14.04",
         registry = "gcr.io",
-        repository = "asci-toolchain/nosla-cuda9.0-cudnn7-ubuntu14.04",
+        repository = "tensorflow-testing/nosla-cuda9.0-cudnn7-ubuntu14.04",
         digest = container_digests["cuda9.0-cudnn7-ubuntu14.04"],
     )
 
-- 
GitLab


From 2f246ef177ad3cb8aba3b4de0a628822cbeab71d Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 4 Jan 2019 14:27:22 -0800
Subject: [PATCH 0212/2345] Upgrade ci docker images to cuda 10.

These really should be gone already...

PiperOrigin-RevId: 227914258
---
 tensorflow/tools/ci_build/Dockerfile.gpu     | 4 ++--
 tensorflow/tools/ci_build/Dockerfile.rbe.gpu | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/tools/ci_build/Dockerfile.gpu b/tensorflow/tools/ci_build/Dockerfile.gpu
index a4cad4b6c6..ff1fefe892 100644
--- a/tensorflow/tools/ci_build/Dockerfile.gpu
+++ b/tensorflow/tools/ci_build/Dockerfile.gpu
@@ -1,4 +1,4 @@
-FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04
+FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04
 
 LABEL maintainer="Jan Prach <jendap@google.com>"
 
@@ -24,7 +24,7 @@ COPY install/.bazelrc /etc/bazel.bazelrc
 ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
 
 # Link NCCL libray and header where the build script expects them.
-RUN mkdir /usr/local/cuda-9.0/lib &&  \
+RUN mkdir /usr/local/cuda/lib &&  \
     ln -s /usr/lib/x86_64-linux-gnu/libnccl.so.2 /usr/local/cuda/lib/libnccl.so.2 && \
     ln -s /usr/include/nccl.h /usr/local/cuda/include/nccl.h
 
diff --git a/tensorflow/tools/ci_build/Dockerfile.rbe.gpu b/tensorflow/tools/ci_build/Dockerfile.rbe.gpu
index b656205836..c4912a65b6 100644
--- a/tensorflow/tools/ci_build/Dockerfile.rbe.gpu
+++ b/tensorflow/tools/ci_build/Dockerfile.rbe.gpu
@@ -1,4 +1,4 @@
-FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04
+FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04
 
 LABEL maintainer="Nick Lopez <ngiraldo@google.com>"
 
-- 
GitLab


From 7cc1ff9072ce4c2c7f36b7011e24486071ad13ee Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Fri, 4 Jan 2019 14:35:17 -0800
Subject: [PATCH 0213/2345] Better exception for tf.constant(tf.constant(x))

PiperOrigin-RevId: 227915529
---
 tensorflow/python/framework/constant_op.py         |  4 ++++
 tensorflow/python/kernel_tests/constant_op_test.py | 11 +++++++++++
 2 files changed, 15 insertions(+)

diff --git a/tensorflow/python/framework/constant_op.py b/tensorflow/python/framework/constant_op.py
index ade0797dcd..62a25f3fea 100644
--- a/tensorflow/python/framework/constant_op.py
+++ b/tensorflow/python/framework/constant_op.py
@@ -277,6 +277,10 @@ def _constant_impl(
                     (num_t, shape, shape.num_elements()))
   g = ops.get_default_graph()
   tensor_value = attr_value_pb2.AttrValue()
+  if tensor_util.is_tensor(value):
+    raise ValueError(
+        ("Cannot create a tf.constant from symbolic Tensor %s. Did you mean "
+         "tf.convert_to_tensor?") % (value,))
   tensor_value.tensor.CopyFrom(
       tensor_util.make_tensor_proto(
           value, dtype=dtype, shape=shape, verify_shape=verify_shape,
diff --git a/tensorflow/python/kernel_tests/constant_op_test.py b/tensorflow/python/kernel_tests/constant_op_test.py
index 583082c2aa..2b743ed4c4 100644
--- a/tensorflow/python/kernel_tests/constant_op_test.py
+++ b/tensorflow/python/kernel_tests/constant_op_test.py
@@ -24,6 +24,7 @@ from google.protobuf import text_format
 
 from tensorflow.core.framework import graph_pb2
 from tensorflow.core.framework import tensor_pb2
+from tensorflow.python.eager import def_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes as dtypes_lib
 from tensorflow.python.framework import errors_impl
@@ -630,6 +631,16 @@ class OnesTest(test.TestCase):
         self.assertEqual([2, 3], z.get_shape())
         self.assertAllEqual(z.eval(), np.ones([2, 3]))
 
+  @test_util.run_in_graph_and_eager_modes
+  def testIteratedConstantSensibleException(self):
+
+    @def_function.function
+    def constant_on_constant(x):
+      return constant_op.constant(x)
+
+    with self.assertRaisesRegexp(ValueError, "convert_to_tensor"):
+      constant_on_constant(constant_op.constant(2))
+
 
 class OnesLikeTest(test.TestCase):
 
-- 
GitLab


From 73b6e92b4b3faf0c20687af3ba63989fc5234f13 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Fri, 4 Jan 2019 14:35:23 -0800
Subject: [PATCH 0214/2345] Remove usage of the initialize and finalize api

PiperOrigin-RevId: 227915547
---
 .../distribute/python/metrics_v1_test.py      |  3 ---
 .../distribute/python/minimize_loss_test.py   | 19 +------------------
 .../contrib/distribute/python/step_fn_test.py |  4 ----
 3 files changed, 1 insertion(+), 25 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/metrics_v1_test.py b/tensorflow/contrib/distribute/python/metrics_v1_test.py
index 32a0d19943..7472f6dde4 100644
--- a/tensorflow/contrib/distribute/python/metrics_v1_test.py
+++ b/tensorflow/contrib/distribute/python/metrics_v1_test.py
@@ -122,7 +122,6 @@ class MetricsV1Test(test.TestCase, parameterized.TestCase):
         batches_per_update = distribution.num_replicas_in_sync
 
       self.evaluate(iterator.initializer)
-      self.evaluate(distribution.initialize())
       self.evaluate(variables.local_variables_initializer())
 
       batches_consumed = 0
@@ -136,8 +135,6 @@ class MetricsV1Test(test.TestCase, parameterized.TestCase):
         if batches_consumed >= 4:  # Consume 4 input batches in total.
           break
 
-      self.evaluate(distribution.finalize())
-
   @combinations.generate(all_combinations() + tpu_combinations())
   def testMean(self, distribution):
     def _dataset_fn():
diff --git a/tensorflow/contrib/distribute/python/minimize_loss_test.py b/tensorflow/contrib/distribute/python/minimize_loss_test.py
index 824c4b0937..b0e24a53f6 100644
--- a/tensorflow/contrib/distribute/python/minimize_loss_test.py
+++ b/tensorflow/contrib/distribute/python/minimize_loss_test.py
@@ -75,7 +75,6 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
         return distribution.run_steps_on_dataset(
             step_fn, iterator, iterations=2).run_op
 
-      self.evaluate(distribution.initialize())
       if not context.executing_eagerly():
         with self.cached_session() as sess:
           run_step = sess.make_callable(run_step())
@@ -84,12 +83,9 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
       weights, biases = [], []
       for _ in range(5):
         run_step()
-
         weights.append(self.evaluate(layer.kernel))
         biases.append(self.evaluate(layer.bias))
 
-      self.evaluate(distribution.finalize())
-
       error = abs(numpy.add(numpy.squeeze(weights), numpy.squeeze(biases)) - 1)
       is_not_increasing = all(y <= x for x, y in zip(error, error[1:]))
       self.assertTrue(is_not_increasing)
@@ -152,7 +148,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
     # `distribution.scope`.
     with variable_scope.variable_creator_scope(
         appending_creator), distribution.scope():
-      model_fn, dataset_fn, layer = minimize_loss_example(
+      model_fn, dataset_fn, _ = minimize_loss_example(
           optimizer_fn,
           use_bias=True,
           use_callable_loss=True,
@@ -169,16 +165,12 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
         return distribution.run_steps_on_dataset(
             step_fn, iterator, iterations=1).run_op
 
-      self.evaluate(distribution.initialize())
       if not context.executing_eagerly():
         with self.cached_session() as sess:
           run_step = sess.make_callable(run_step())
       self.evaluate(variables_lib.global_variables_initializer())
-
       run_step()
 
-      self.evaluate(distribution.finalize())
-
       def get_expected_variables(optimizer_fn, num_parameter_devices):
         variables_map = {
             "GradientDescent": ["dense/kernel", "dense/bias"],
@@ -241,7 +233,6 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
         return distribution.run_steps_on_dataset(
             step_fn, iterator, iterations=1).run_op
 
-      self.evaluate(distribution.initialize())
       if not context.executing_eagerly():
         with self.cached_session() as sess:
           run_step = sess.make_callable(run_step())
@@ -267,8 +258,6 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
               expected_moving_mean - averaged_batch_mean(i)) * (1.0 - momentum))
           self.assertNear(expected_moving_means[i], moving_means[i], 0.0001)
 
-      self.evaluate(distribution.finalize())
-
   @combinations.generate(
       combinations.times(
           combinations.combine(
@@ -335,7 +324,6 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
         return distribution.run_steps_on_dataset(
             step_fn, iterator, iterations=1).run_op
 
-      self.evaluate(distribution.initialize())
       if not context.executing_eagerly():
         with self.cached_session() as sess:
           run_step = sess.make_callable(run_step())
@@ -370,8 +358,6 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
         # One of the mean loss reductions.
         self.assertNear(weight, 2 + 10.6, 0.0001)
 
-      self.evaluate(distribution.finalize())
-
   @combinations.generate(
       combinations.times(
           combinations.distributions_and_v1_optimizers(),
@@ -458,7 +444,6 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
             reduced=False, distribution=distribution)
         return (ctx.run_op, ctx.last_step_outputs["replica_loss_reduced"])
 
-      self.evaluate(distribution.initialize())
       if not context.executing_eagerly():
         with self.cached_session() as sess:
           run_step = sess.make_callable(run_step())
@@ -471,8 +456,6 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
         weights.append(self.evaluate(layer.kernel))
         biases.append(self.evaluate(layer.bias))
 
-      self.evaluate(distribution.finalize())
-
       loss_is_not_increasing = all(y <= x for x, y in zip(losses, losses[1:]))
       self.assertTrue(loss_is_not_increasing)
 
diff --git a/tensorflow/contrib/distribute/python/step_fn_test.py b/tensorflow/contrib/distribute/python/step_fn_test.py
index 1ff9b9ceec..a77d6d0bec 100644
--- a/tensorflow/contrib/distribute/python/step_fn_test.py
+++ b/tensorflow/contrib/distribute/python/step_fn_test.py
@@ -45,7 +45,6 @@ class SingleLossStepTest(test.TestCase, parameterized.TestCase):
       single_loss_step, layer = single_loss_example(
           optimizer_fn, distribution, use_bias=True, iterations_per_step=2)
 
-      self.evaluate(distribution.initialize())
       if context.executing_eagerly():
         run_step = single_loss_step
       else:
@@ -57,12 +56,9 @@ class SingleLossStepTest(test.TestCase, parameterized.TestCase):
       weights, biases = [], []
       for _ in range(5):
         run_step()
-
         weights.append(self.evaluate(layer.kernel))
         biases.append(self.evaluate(layer.bias))
 
-      self.evaluate(distribution.finalize())
-
       error = abs(numpy.add(numpy.squeeze(weights), numpy.squeeze(biases)) - 1)
       is_not_increasing = all(y <= x for x, y in zip(error, error[1:]))
       self.assertTrue(is_not_increasing)
-- 
GitLab


From 1dee496b6addb67c269151d4a96b81608b1eb463 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 14:43:02 -0800
Subject: [PATCH 0215/2345] TOCO's default_range parameters are floats, not
 integers.

PiperOrigin-RevId: 227916837
---
 tensorflow/lite/python/tflite_convert.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/python/tflite_convert.py b/tensorflow/lite/python/tflite_convert.py
index 341b539bea..401a592273 100644
--- a/tensorflow/lite/python/tflite_convert.py
+++ b/tensorflow/lite/python/tflite_convert.py
@@ -343,13 +343,13 @@ def run_main(_):
             "floats. Used for quantized input tensors. (default None)"))
   parser.add_argument(
       "--default_ranges_min",
-      type=int,
+      type=float,
       help=("Default value for min bound of min/max range values used for all "
             "arrays without a specified range, Intended for experimenting with "
             "quantization via \"dummy quantization\". (default None)"))
   parser.add_argument(
       "--default_ranges_max",
-      type=int,
+      type=float,
       help=("Default value for max bound of min/max range values used for all "
             "arrays without a specified range, Intended for experimenting with "
             "quantization via \"dummy quantization\". (default None)"))
-- 
GitLab


From 58edbcb97b90596ee5f4cd907dfa7dd748c1843a Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Fri, 4 Jan 2019 15:03:28 -0800
Subject: [PATCH 0216/2345] Internal change.

PiperOrigin-RevId: 227920137
---
 tensorflow/tools/ci_build/builds/run_pip_tests.sh    | 3 ++-
 tensorflow/tools/ci_build/ci_parameterized_build.sh  | 9 ++++++---
 tensorflow/tools/ci_build/osx/cpu/run_contrib.sh     | 1 +
 tensorflow/tools/ci_build/osx/cpu/run_py2_cc_core.sh | 1 +
 4 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/tensorflow/tools/ci_build/builds/run_pip_tests.sh b/tensorflow/tools/ci_build/builds/run_pip_tests.sh
index 7d5cf3f843..a095633a22 100755
--- a/tensorflow/tools/ci_build/builds/run_pip_tests.sh
+++ b/tensorflow/tools/ci_build/builds/run_pip_tests.sh
@@ -88,7 +88,8 @@ if [[ ${IS_GPU} == "1" ]]; then
   PIP_TEST_FILTER_TAG="-no_gpu,-no_pip_gpu,${PIP_TEST_FILTER_TAG}"
 fi
 if [[ ${IS_MAC} == "1" ]]; then
-  PIP_TEST_FILTER_TAG="-nomac,${PIP_TEST_FILTER_TAG}"
+  # TODO(b/122370901): Fix nomac, no_mac inconsistency.
+  PIP_TEST_FILTER_TAG="-nomac,-no_mac,${PIP_TEST_FILTER_TAG}"
 fi
 
 # Bazel flags we need for all tests:
diff --git a/tensorflow/tools/ci_build/ci_parameterized_build.sh b/tensorflow/tools/ci_build/ci_parameterized_build.sh
index 435ec7ca68..41b9f241d5 100755
--- a/tensorflow/tools/ci_build/ci_parameterized_build.sh
+++ b/tensorflow/tools/ci_build/ci_parameterized_build.sh
@@ -398,7 +398,8 @@ if [[ "${TF_BUILD_APPEND_ARGUMENTS}" == *"--test_tag_filters="* ]]; then
         NEW_ITEM="${NEW_ITEM},-benchmark-test"
       fi
       if [[ ${IS_MAC} == "1" ]] && [[ ${NEW_ITEM} != *"nomac"* ]]; then
-        NEW_ITEM="${NEW_ITEM},-nomac"
+        # TODO(b/122370901): Fix nomac, no_mac inconsistency.
+        NEW_ITEM="${NEW_ITEM},-nomac,-no_mac"
       fi
       EXTRA_ARGS="${EXTRA_ARGS} ${NEW_ITEM}"
     else
@@ -408,11 +409,13 @@ if [[ "${TF_BUILD_APPEND_ARGUMENTS}" == *"--test_tag_filters="* ]]; then
 else
   EXTRA_ARGS="${EXTRA_ARGS} ${TF_BUILD_APPEND_ARGUMENTS} --test_tag_filters=-no_oss,-oss_serial,-benchmark-test"
   if [[ ${IS_MAC} == "1" ]]; then
-    EXTRA_ARGS="${EXTRA_ARGS},-nomac"
+    # TODO(b/122370901): Fix nomac, no_mac inconsistency.
+    EXTRA_ARGS="${EXTRA_ARGS},-nomac,-no_mac"
   fi
   EXTRA_ARGS="${EXTRA_ARGS} --build_tag_filters=-no_oss,-oss_serial,-benchmark-test"
   if [[ ${IS_MAC} == "1" ]]; then
-    EXTRA_ARGS="${EXTRA_ARGS},-nomac"
+    # TODO(b/122370901): Fix nomac, no_mac inconsistency.
+    EXTRA_ARGS="${EXTRA_ARGS},-nomac,-no_mac"
   fi
 fi
 
diff --git a/tensorflow/tools/ci_build/osx/cpu/run_contrib.sh b/tensorflow/tools/ci_build/osx/cpu/run_contrib.sh
index 3efd994d78..1184d4acec 100755
--- a/tensorflow/tools/ci_build/osx/cpu/run_contrib.sh
+++ b/tensorflow/tools/ci_build/osx/cpu/run_contrib.sh
@@ -31,6 +31,7 @@ export CC_OPT_FLAGS='-mavx'
 export PYTHON_BIN_PATH=$(which python2)
 yes "" | $PYTHON_BIN_PATH configure.py
 which bazel
+# TODO(b/122370901): Fix nomac, no_mac inconsistency.
 bazel test --test_tag_filters=-no_oss,-gpu,-benchmark-test,-nomac,-no_mac \
     --test_timeout 300,450,1200,3600 \
     --test_size_filters=small,medium --config=opt \
diff --git a/tensorflow/tools/ci_build/osx/cpu/run_py2_cc_core.sh b/tensorflow/tools/ci_build/osx/cpu/run_py2_cc_core.sh
index adee0d3171..d39340b1d8 100755
--- a/tensorflow/tools/ci_build/osx/cpu/run_py2_cc_core.sh
+++ b/tensorflow/tools/ci_build/osx/cpu/run_py2_cc_core.sh
@@ -32,6 +32,7 @@ export CC_OPT_FLAGS='-mavx'
 export PYTHON_BIN_PATH=$(which python2)
 yes "" | $PYTHON_BIN_PATH configure.py
 which bazel
+# TODO(b/122370901): Fix nomac, no_mac inconsistency.
 bazel test --test_tag_filters=-no_oss,-gpu,-benchmark-test,-nomac,-no_mac \
     --test_timeout 300,450,1200,3600 --config=opt \
     --announce_rc \
-- 
GitLab


From 07777005d9b3feff4c8d74348739ad17de510131 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 15:07:06 -0800
Subject: [PATCH 0217/2345] Implement memory prefetching for PresizedCuckooMap

PiperOrigin-RevId: 227920727
---
 tensorflow/core/BUILD                         |  1 +
 tensorflow/core/util/presized_cuckoo_map.h    | 10 ++++
 .../core/util/presized_cuckoo_map_test.cc     | 49 ++++++++++++++++++-
 3 files changed, 59 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 7c9decbd09..460353df41 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -3813,6 +3813,7 @@ tf_cc_tests(
         "//tensorflow/core/kernels:ops_util",
         "//third_party/eigen3",
         "@com_google_absl//absl/base",
+        "@com_google_absl//absl/time",
     ],
 )
 
diff --git a/tensorflow/core/util/presized_cuckoo_map.h b/tensorflow/core/util/presized_cuckoo_map.h
index f88ad2faaf..1cdde34562 100644
--- a/tensorflow/core/util/presized_cuckoo_map.h
+++ b/tensorflow/core/util/presized_cuckoo_map.h
@@ -20,6 +20,7 @@ limitations under the License.
 #include <vector>
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/platform/macros.h"
+#include "tensorflow/core/platform/prefetch.h"
 
 namespace tensorflow {
 
@@ -132,6 +133,15 @@ class PresizedCuckooMap {
            FindInBucket(k, fast_map_to_buckets(h2(tk)), out);
   }
 
+  // Prefetch memory associated with the key k into cache levels specified by
+  // hint.
+  template <port::PrefetchHint hint = port::PREFETCH_HINT_T0>
+  void PrefetchKey(const key_type k) const {
+    const uint64 tk = key_transform(k);
+    port::prefetch<hint>(&buckets_[fast_map_to_buckets(tk)].keys);
+    port::prefetch<hint>(&buckets_[fast_map_to_buckets(h2(tk))].keys);
+  }
+
   int64 MemoryUsed() const {
     return sizeof(PresizedCuckooMap<value>) + sizeof(CuckooPathQueue);
   }
diff --git a/tensorflow/core/util/presized_cuckoo_map_test.cc b/tensorflow/core/util/presized_cuckoo_map_test.cc
index f2be1e8a2f..8dbb2ec065 100644
--- a/tensorflow/core/util/presized_cuckoo_map_test.cc
+++ b/tensorflow/core/util/presized_cuckoo_map_test.cc
@@ -13,12 +13,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/core/util/presized_cuckoo_map.h"
 #include <array>
+
+#include "absl/time/clock.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/fingerprint.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/platform/test_benchmark.h"
+#include "tensorflow/core/util/presized_cuckoo_map.h"
 
 namespace tensorflow {
 namespace {
@@ -50,6 +52,51 @@ TEST(PresizedCuckooMapTest, Basic) {
   EXPECT_EQ(out, 2);
 }
 
+TEST(PresizedCuckooMapTest, Prefetch) {
+  {
+    PresizedCuckooMap<int64> pscm(2);
+    EXPECT_TRUE(pscm.InsertUnique(1, 2));
+    // Works for both present and absent keys.
+    pscm.PrefetchKey(1);
+    pscm.PrefetchKey(2);
+  }
+
+  // Do not run in debug mode, when prefetch is not implemented, or when
+  // sanitizers are enabled.
+#if defined(NDEBUG) && defined(__GNUC__) && !defined(ADDRESS_SANITIZER) && \
+    !defined(MEMORY_SANITIZER) && !defined(THREAD_SANITIZER) &&            \
+    !defined(UNDEFINED_BEHAVIOR_SANITIZER)
+  const auto now = [] { return absl::Now(); };
+
+  // Make size enough to not fit in L2 cache (16.7 Mb)
+  static constexpr int size = 1 << 22;
+  PresizedCuckooMap<int64> pscm(size);
+  for (int i = 0; i < size; ++i) {
+    pscm.InsertUnique(i, i);
+  }
+
+  absl::Duration no_prefetch, prefetch;
+  int64 out;
+  for (int iter = 0; iter < 10; ++iter) {
+    auto time = now();
+    for (int i = 0; i < size; ++i) {
+      testing::DoNotOptimize(pscm.Find(i, &out));
+    }
+    no_prefetch += now() - time;
+
+    time = now();
+    for (int i = 0; i < size; ++i) {
+      pscm.PrefetchKey(i + 20);
+      testing::DoNotOptimize(pscm.Find(i, &out));
+    }
+    prefetch += now() - time;
+  }
+
+  // no_prefetch is at least 30% slower.
+  EXPECT_GE(1.0 * no_prefetch / prefetch, 1.3);
+#endif
+}
+
 TEST(PresizedCuckooMapTest, TooManyItems) {
   static constexpr int kTableSize = 1000;
   PresizedCuckooMap<int> pscm(kTableSize);
-- 
GitLab


From 620fa70346ef6cd2c7f13c9986625850d523cf0b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 15:23:23 -0800
Subject: [PATCH 0218/2345] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 227923029

---
 tensorflow/go/op/wrappers.go | 1844 ++++++++++++++++++++++++++++++++--
 1 file changed, 1782 insertions(+), 62 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 6fe61f512f..208c15d75a 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -327,6 +327,192 @@ func FakeQuantWithMinMaxArgs(scope *Scope, inputs tf.Output, optional ...FakeQua
 	return op.Output(0)
 }
 
+// Subtracts sparse `updates` from an existing tensor according to `indices`.
+//
+// This operation creates a new tensor by subtracting sparse `updates` from the
+// passed in `tensor`.
+// This operation is very similar to `tf.scatter_nd_sub`, except that the updates
+// are subtracted from an existing tensor (as opposed to a variable). If the memory
+// for the existing tensor cannot be re-used, a copy is made and updated.
+//
+// `indices` is an integer tensor containing indices into a new tensor of shape
+// `shape`.  The last dimension of `indices` can be at most the rank of `shape`:
+//
+//     indices.shape[-1] <= shape.rank
+//
+// The last dimension of `indices` corresponds to indices into elements
+// (if `indices.shape[-1] = shape.rank`) or slices
+// (if `indices.shape[-1] < shape.rank`) along dimension `indices.shape[-1]` of
+// `shape`.  `updates` is a tensor with shape
+//
+//     indices.shape[:-1] + shape[indices.shape[-1]:]
+//
+// The simplest form of tensor_scatter_sub is to subtract individual elements
+// from a tensor by index. For example, say we want to insert 4 scattered elements
+// in a rank-1 tensor with 8 elements.
+//
+// In Python, this scatter subtract operation would look like this:
+//
+// ```python
+//     indices = tf.constant([[4], [3], [1], [7]])
+//     updates = tf.constant([9, 10, 11, 12])
+//     tensor = tf.ones([8], dtype=tf.int32)
+//     updated = tf.tensor_scatter_sub(tensor, indices, updates)
+//     with tf.Session() as sess:
+//       print(sess.run(scatter))
+// ```
+//
+// The resulting tensor would look like this:
+//
+//     [1, -10, 1, -9, -8, 1, 1, -11]
+//
+// We can also, insert entire slices of a higher rank tensor all at once. For
+// example, if we wanted to insert two slices in the first dimension of a
+// rank-3 tensor with two matrices of new values.
+//
+// In Python, this scatter add operation would look like this:
+//
+// ```python
+//     indices = tf.constant([[0], [2]])
+//     updates = tf.constant([[[5, 5, 5, 5], [6, 6, 6, 6],
+//                             [7, 7, 7, 7], [8, 8, 8, 8]],
+//                            [[5, 5, 5, 5], [6, 6, 6, 6],
+//                             [7, 7, 7, 7], [8, 8, 8, 8]]])
+//     tensor = tf.ones([4, 4, 4])
+//     updated = tf.tensor_scatter_sub(tensor, indices, updates)
+//     with tf.Session() as sess:
+//       print(sess.run(scatter))
+// ```
+//
+// The resulting tensor would look like this:
+//
+//     [[[-4, -4, -4, -4], [-5, -5, -5, -5], [-6, -6, -6, -6], [-7, -7, -7, -7]],
+//      [[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]],
+//      [[-4, -4, -4, -4], [-5, -5, -5, -5], [-6, -6, -6, -6], [-7, -7, -7, -7]],
+//      [[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]]]
+//
+// Note that on CPU, if an out of bound index is found, an error is returned.
+// On GPU, if an out of bound index is found, the index is ignored.
+//
+// Arguments:
+//	tensor: Tensor to copy/update.
+//	indices: Index tensor.
+//	updates: Updates to scatter into output.
+//
+// Returns A new tensor copied from tensor and updates subtracted according to the indices.
+func TensorScatterSub(scope *Scope, tensor tf.Output, indices tf.Output, updates tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorScatterSub",
+		Input: []tf.Input{
+			tensor, indices, updates,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Scatter `updates` into an existing tensor according to `indices`.
+//
+// This operation creates a new tensor by applying sparse `updates` to the passed
+// in `tensor`.
+// This operation is very similar to `tf.scatter_nd`, except that the updates are
+// scattered onto an existing tensor (as opposed to a zero-tensor). If the memory
+// for the existing tensor cannot be re-used, a copy is made and updated.
+//
+// If `indices` contains duplicates, then their updates are accumulated (summed).
+//
+// **WARNING**: The order in which updates are applied is nondeterministic, so the
+// output will be nondeterministic if `indices` contains duplicates -- because
+// of some numerical approximation issues, numbers summed in different order
+// may yield different results.
+//
+// `indices` is an integer tensor containing indices into a new tensor of shape
+// `shape`.  The last dimension of `indices` can be at most the rank of `shape`:
+//
+//     indices.shape[-1] <= shape.rank
+//
+// The last dimension of `indices` corresponds to indices into elements
+// (if `indices.shape[-1] = shape.rank`) or slices
+// (if `indices.shape[-1] < shape.rank`) along dimension `indices.shape[-1]` of
+// `shape`.  `updates` is a tensor with shape
+//
+//     indices.shape[:-1] + shape[indices.shape[-1]:]
+//
+// The simplest form of scatter is to insert individual elements in a tensor by
+// index. For example, say we want to insert 4 scattered elements in a rank-1
+// tensor with 8 elements.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/ScatterNd1.png" alt>
+// </div>
+//
+// In Python, this scatter operation would look like this:
+//
+// ```python
+//     indices = tf.constant([[4], [3], [1], [7]])
+//     updates = tf.constant([9, 10, 11, 12])
+//     tensor = tf.ones([8], dtype=tf.int32)
+//     updated = tf.tensor_scatter_update(tensor, indices, updates)
+//     with tf.Session() as sess:
+//       print(sess.run(scatter))
+// ```
+//
+// The resulting tensor would look like this:
+//
+//     [1, 11, 1, 10, 9, 1, 1, 12]
+//
+// We can also, insert entire slices of a higher rank tensor all at once. For
+// example, if we wanted to insert two slices in the first dimension of a
+// rank-3 tensor with two matrices of new values.
+//
+// In Python, this scatter operation would look like this:
+//
+// ```python
+//     indices = tf.constant([[0], [2]])
+//     updates = tf.constant([[[5, 5, 5, 5], [6, 6, 6, 6],
+//                             [7, 7, 7, 7], [8, 8, 8, 8]],
+//                            [[5, 5, 5, 5], [6, 6, 6, 6],
+//                             [7, 7, 7, 7], [8, 8, 8, 8]]])
+//     tensor = tf.ones([4, 4, 4])
+//     updated = tf.tensor_scatter_update(tensor, indices, updates)
+//     with tf.Session() as sess:
+//       print(sess.run(scatter))
+// ```
+//
+// The resulting tensor would look like this:
+//
+//     [[[5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]],
+//      [[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]],
+//      [[5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]],
+//      [[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]]]
+//
+// Note that on CPU, if an out of bound index is found, an error is returned.
+// On GPU, if an out of bound index is found, the index is ignored.
+//
+// Arguments:
+//	tensor: Tensor to copy/update.
+//	indices: Index tensor.
+//	updates: Updates to scatter into output.
+//
+// Returns A new tensor with the given shape and updates applied according
+// to the indices.
+func TensorScatterUpdate(scope *Scope, tensor tf.Output, indices tf.Output, updates tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorScatterUpdate",
+		Input: []tf.Input{
+			tensor, indices, updates,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Scatter `updates` into a new tensor according to `indices`.
 //
 // Creates a new tensor by applying sparse `updates` to individual values or
@@ -334,6 +520,10 @@ func FakeQuantWithMinMaxArgs(scope *Scope, inputs tf.Output, optional ...FakeQua
 // the given `shape` according to indices.  This operator is the inverse of the
 // `tf.gather_nd` operator which extracts values or slices from a given tensor.
 //
+// This operation is similar to tensor_scatter_add, except that the tensor is
+// zero-initialized. Calling `tf.scatter_nd(indices, values, shape)` is identical
+// to `tensor_scatter_add(tf.zeros(shape, values.dtype), indices, values)`
+//
 // If `indices` contains duplicates, then their updates are accumulated (summed).
 //
 // **WARNING**: The order in which updates are applied is nondeterministic, so the
@@ -464,6 +654,15 @@ func QuantizeAndDequantizeV2RangeGiven(value bool) QuantizeAndDequantizeV2Attr {
 }
 
 // QuantizeAndDequantizeV2RoundMode sets the optional round_mode attribute to value.
+//
+// value: The 'round_mode' attribute controls which rounding tie-breaking algorithm is
+// used when rounding float values to their quantized equivalents. The following
+// rounding modes are currently supported:
+//
+// *   HALF_TO_EVEN: this is the default round_mode.
+// *   HALF_UP: round towards positive. In this mode 7.5 rounds up to 8 and -7.5
+//     rounds up to -7.
+//
 // If not specified, defaults to "HALF_TO_EVEN"
 func QuantizeAndDequantizeV2RoundMode(value string) QuantizeAndDequantizeV2Attr {
 	return func(m optionalAttr) {
@@ -523,7 +722,7 @@ func QuantizeAndDequantizeV2RoundMode(value string) QuantizeAndDequantizeV2Attr
 //
 // output = round(clamp(value, input_min, input_max) * scale_factor) / scale_factor.
 //
-// The above round function uses half to even rounding.
+// The above round function rounds the value based on the given round_mode.
 //
 //
 // Arguments:
@@ -3422,11 +3621,11 @@ func PopulationCount(scope *Scope, x tf.Output) (y tf.Output) {
 // bucketized values for a single feature.
 //
 // Arguments:
-//	float_values: float; List of Rank 2 Tensor each containing float values for a single feature.
+//	float_values: float; List of Rank 1 Tensor each containing float values for a single feature.
 //	bucket_boundaries: float; List of Rank 1 Tensors each containing the bucket boundaries for a single
 // feature.
 //
-// Returns int; List of Rank 2 Tensors each containing the bucketized values for a single feature.
+// Returns int; List of Rank 1 Tensors each containing the bucketized values for a single feature.
 func BoostedTreesBucketize(scope *Scope, float_values []tf.Output, bucket_boundaries []tf.Output) (buckets []tf.Output) {
 	if scope.Err() != nil {
 		return
@@ -3497,15 +3696,16 @@ func BoostedTreesQuantileStreamResourceFlush(scope *Scope, quantile_stream_resou
 
 // Makes the summary of quantiles for the batch.
 //
-// An op that takes a list of tensors and outputs the quantile summaries for each tensor.
+// An op that takes a list of tensors (one tensor per feature) and outputs the
+// quantile summaries for each tensor.
 //
 // Arguments:
-//	float_values: float; List of Rank 2 Tensors each containing values for a single feature.
+//	float_values: float; List of Rank 1 Tensors each containing values for a single feature.
 //	example_weights: float; Rank 1 Tensor with weights per instance.
 //	epsilon: float; The required maximum approximation error.
 //
-// Returns float; List of Rank 2 Tensors each containing the quantile summary (value, weight,
-// min_rank, max_rank) of a single feature.
+// Returns float; List of Rank 2 Tensors each containing the quantile summary
+// (value, weight, min_rank, max_rank) of a single feature.
 func BoostedTreesMakeQuantileSummaries(scope *Scope, float_values []tf.Output, example_weights tf.Output, epsilon tf.Output) (summaries []tf.Output) {
 	if scope.Err() != nil {
 		return
@@ -3806,6 +4006,70 @@ func BoostedTreesEnsembleResourceHandleOp(scope *Scope, optional ...BoostedTrees
 	return op.Output(0)
 }
 
+// Output the logits for the given input data
+//
+// Arguments:
+//	tree_handle: Handle to the tree resource.
+//	dense_features: Rank 2 dense features tensor.
+//	logits_dimension: Scalar, dimension of the logits.
+//
+// Returns The logits predictions from the tree for each instance in the batch.
+func TensorForestTreePredict(scope *Scope, tree_handle tf.Output, dense_features tf.Output, logits_dimension int64) (logits tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"logits_dimension": logits_dimension}
+	opspec := tf.OpSpec{
+		Type: "TensorForestTreePredict",
+		Input: []tf.Input{
+			tree_handle, dense_features,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Get the number of nodes in a tree
+//
+// Arguments:
+//	tree_handle: Handle to the tree resource.
+//
+// Returns The size of the tree.
+func TensorForestTreeSize(scope *Scope, tree_handle tf.Output) (tree_size tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorForestTreeSize",
+		Input: []tf.Input{
+			tree_handle,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Creates a tree resource and returns a handle to it.
+//
+// Arguments:
+//	tree_handle: Handle to the tree resource to be created.
+//	tree_config: Serialized proto string of the boosted_trees.Tree.
+//
+// Returns the created operation.
+func TensorForestCreateTreeVariable(scope *Scope, tree_handle tf.Output, tree_config tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorForestCreateTreeVariable",
+		Input: []tf.Input{
+			tree_handle, tree_config,
+		},
+	}
+	return scope.AddOperation(opspec)
+}
+
 // ComputeAccidentalHitsAttr is an optional argument to ComputeAccidentalHits.
 type ComputeAccidentalHitsAttr func(optionalAttr)
 
@@ -4829,6 +5093,119 @@ func CudnnRNNParamsToCanonical(scope *Scope, num_layers tf.Output, num_units tf.
 	return weights, biases
 }
 
+// CudnnRNNBackpropV3Attr is an optional argument to CudnnRNNBackpropV3.
+type CudnnRNNBackpropV3Attr func(optionalAttr)
+
+// CudnnRNNBackpropV3RnnMode sets the optional rnn_mode attribute to value.
+// If not specified, defaults to "lstm"
+func CudnnRNNBackpropV3RnnMode(value string) CudnnRNNBackpropV3Attr {
+	return func(m optionalAttr) {
+		m["rnn_mode"] = value
+	}
+}
+
+// CudnnRNNBackpropV3InputMode sets the optional input_mode attribute to value.
+// If not specified, defaults to "linear_input"
+func CudnnRNNBackpropV3InputMode(value string) CudnnRNNBackpropV3Attr {
+	return func(m optionalAttr) {
+		m["input_mode"] = value
+	}
+}
+
+// CudnnRNNBackpropV3Direction sets the optional direction attribute to value.
+// If not specified, defaults to "unidirectional"
+func CudnnRNNBackpropV3Direction(value string) CudnnRNNBackpropV3Attr {
+	return func(m optionalAttr) {
+		m["direction"] = value
+	}
+}
+
+// CudnnRNNBackpropV3Dropout sets the optional dropout attribute to value.
+// If not specified, defaults to 0
+func CudnnRNNBackpropV3Dropout(value float32) CudnnRNNBackpropV3Attr {
+	return func(m optionalAttr) {
+		m["dropout"] = value
+	}
+}
+
+// CudnnRNNBackpropV3Seed sets the optional seed attribute to value.
+// If not specified, defaults to 0
+func CudnnRNNBackpropV3Seed(value int64) CudnnRNNBackpropV3Attr {
+	return func(m optionalAttr) {
+		m["seed"] = value
+	}
+}
+
+// CudnnRNNBackpropV3Seed2 sets the optional seed2 attribute to value.
+// If not specified, defaults to 0
+func CudnnRNNBackpropV3Seed2(value int64) CudnnRNNBackpropV3Attr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// Backprop step of CudnnRNNV3.
+//
+// Compute the backprop of both data and weights in a RNN. Takes an extra
+//     "sequence_lengths" input than CudnnRNNBackprop.
+//
+// rnn_mode: Indicates the type of the RNN model.
+// input_mode: Indicates whether there is a linear projection between the input and
+//     the actual computation before the first layer. 'skip_input' is only allowed
+//     when input_size == num_units; 'auto_select' implies 'skip_input' when
+//     input_size == num_units; otherwise, it implies 'linear_input'.
+// direction: Indicates whether a bidirectional model will be used. Should be
+//   "unidirectional" or "bidirectional".
+// dropout: Dropout probability. When set to 0., dropout is disabled.
+// seed: The 1st part of a seed to initialize dropout.
+// seed2: The 2nd part of a seed to initialize dropout.
+// input: A 3-D tensor with the shape of [seq_length, batch_size, input_size].
+// input_h: A 3-D tensor with the shape of [num_layer * dir, batch_size,
+//     num_units].
+// input_c: For LSTM, a 3-D tensor with the shape of
+//     [num_layer * dir, batch, num_units]. For other models, it is ignored.
+// params: A 1-D tensor that contains the weights and biases in an opaque layout.
+//     The size must be created through CudnnRNNParamsSize, and initialized
+//     separately. Note that they might not be compatible across different
+//     generations. So it is a good idea to save and restore
+// sequence_lengths: a vector of lengths of each input sequence.
+// output: A 3-D tensor with the shape of [seq_length, batch_size,
+//     dir * num_units].
+// output_h: The same shape has input_h.
+// output_c: The same shape as input_c for LSTM. An empty tensor for other models.
+// output_backprop: A 3-D tensor with the same shape as output in the forward pass.
+// output_h_backprop: A 3-D tensor with the same shape as output_h in the forward
+//     pass.
+// output_c_backprop: A 3-D tensor with the same shape as output_c in the forward
+//     pass.
+// reserve_space: The same reserve_space produced in the forward operation.
+// input_backprop: The backprop to input in the forward pass. Has the same shape
+//     as input.
+// input_h_backprop: The backprop to input_h in the forward pass. Has the same
+//     shape as input_h.
+// input_c_backprop: The backprop to input_c in the forward pass. Has the same
+//     shape as input_c.
+// params_backprop: The backprop to the params buffer in the forward pass. Has the
+//     same shape as params.
+func CudnnRNNBackpropV3(scope *Scope, input tf.Output, input_h tf.Output, input_c tf.Output, params tf.Output, sequence_lengths tf.Output, output tf.Output, output_h tf.Output, output_c tf.Output, output_backprop tf.Output, output_h_backprop tf.Output, output_c_backprop tf.Output, reserve_space tf.Output, host_reserved tf.Output, optional ...CudnnRNNBackpropV3Attr) (input_backprop tf.Output, input_h_backprop tf.Output, input_c_backprop tf.Output, params_backprop tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "CudnnRNNBackpropV3",
+		Input: []tf.Input{
+			input, input_h, input_c, params, sequence_lengths, output, output_h, output_c, output_backprop, output_h_backprop, output_c_backprop, reserve_space, host_reserved,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2), op.Output(3)
+}
+
 // CudnnRNNBackpropV2Attr is an optional argument to CudnnRNNBackpropV2.
 type CudnnRNNBackpropV2Attr func(optionalAttr)
 
@@ -6697,6 +7074,34 @@ func Sign(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
+// Creates a dataset that passes a sliding window over `input_dataset`.
+//
+// Arguments:
+//
+//	window_size: A scalar representing the number of elements in the
+// sliding window.
+//	window_shift: A scalar representing the steps moving the sliding window
+// forward in one iteration. It must be positive.
+//	window_stride: A scalar representing the stride of the input elements of the sliding window.
+// It must be positive.
+//
+//
+func ExperimentalSlidingWindowDataset(scope *Scope, input_dataset tf.Output, window_size tf.Output, window_shift tf.Output, window_stride tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalSlidingWindowDataset",
+		Input: []tf.Input{
+			input_dataset, window_size, window_shift, window_stride,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Returns which elements of x are finite.
 //
 // @compatibility(numpy)
@@ -7608,6 +8013,33 @@ func Inv(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
+// Creates a dataset that batches input elements into a SparseTensor.
+//
+// Arguments:
+//	input_dataset: A handle to an input dataset. Must have a single component.
+//	batch_size: A scalar representing the number of elements to accumulate in a
+// batch.
+//	row_shape: A vector representing the dense shape of each row in the produced
+// SparseTensor. The shape may be partially specified, using `-1` to indicate
+// that a particular dimension should use the maximum size of all batch elements.
+//
+//
+func ExperimentalDenseToSparseBatchDataset(scope *Scope, input_dataset tf.Output, batch_size tf.Output, row_shape tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalDenseToSparseBatchDataset",
+		Input: []tf.Input{
+			input_dataset, batch_size, row_shape,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // ComplexAbsAttr is an optional argument to ComplexAbs.
 type ComplexAbsAttr func(optionalAttr)
 
@@ -8442,6 +8874,26 @@ func ResourceSparseApplyAdadelta(scope *Scope, var_ tf.Output, accum tf.Output,
 	return scope.AddOperation(opspec)
 }
 
+// Checks whether a tree has been initialized.
+//
+// Arguments:
+//	tree_handle: Handle to the tree.
+//
+// Returns Whether the tree is initialized.
+func TensorForestTreeIsInitializedOp(scope *Scope, tree_handle tf.Output) (is_initialized tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorForestTreeIsInitializedOp",
+		Input: []tf.Input{
+			tree_handle,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Gets next element for the provided shard number.
 //
 // Arguments:
@@ -10977,7 +11429,6 @@ func OneHotAxis(value int64) OneHotAttr {
 // =========
 //
 // Suppose that
-//
 // ```
 //   indices = [0, 2, -1, 1]
 //   depth = 3
@@ -10987,16 +11438,15 @@ func OneHotAxis(value int64) OneHotAttr {
 // ```
 //
 // Then output is `[4 x 3]`:
-//
-//     ```output =
-//       [5.0 0.0 0.0]  // one_hot(0)
-//       [0.0 0.0 5.0]  // one_hot(2)
-//       [0.0 0.0 0.0]  // one_hot(-1)
-//       [0.0 5.0 0.0]  // one_hot(1)
-//     ```
+// ```
+// output =
+//   [5.0 0.0 0.0]  // one_hot(0)
+//   [0.0 0.0 5.0]  // one_hot(2)
+//   [0.0 0.0 0.0]  // one_hot(-1)
+//   [0.0 5.0 0.0]  // one_hot(1)
+// ```
 //
 // Suppose that
-//
 // ```
 //   indices = [0, 2, -1, 1]
 //   depth = 3
@@ -11006,19 +11456,19 @@ func OneHotAxis(value int64) OneHotAttr {
 // ```
 //
 // Then output is `[3 x 4]`:
+// ```
+// output =
+//   [0.0 3.0 3.0 3.0]
+//   [3.0 3.0 3.0 0.0]
+//   [3.0 3.0 3.0 3.0]
+//   [3.0 0.0 3.0 3.0]
+// //  ^                one_hot(0)
+// //      ^            one_hot(2)
+// //          ^        one_hot(-1)
+// //              ^    one_hot(1)
+// ```
 //
-//     ```output =
-//       [0.0 3.0 3.0 3.0]
-//       [3.0 3.0 3.0 0.0]
-//       [3.0 3.0 3.0 3.0]
-//       [3.0 0.0 3.0 3.0]
-//     //  ^                one_hot(0)
-//     //      ^            one_hot(2)
-//     //          ^        one_hot(-1)
-//     //              ^    one_hot(1)
-//     ```
 // Suppose that
-//
 // ```
 //   indices = [[0, 2], [1, -1]]
 //   depth = 3
@@ -11028,15 +11478,16 @@ func OneHotAxis(value int64) OneHotAttr {
 // ```
 //
 // Then output is `[2 x 2 x 3]`:
-//
-//     ```output =
-//       [
-//         [1.0, 0.0, 0.0]  // one_hot(0)
-//         [0.0, 0.0, 1.0]  // one_hot(2)
-//       ][
-//         [0.0, 1.0, 0.0]  // one_hot(1)
-//         [0.0, 0.0, 0.0]  // one_hot(-1)
-//       ]```
+// ```
+// output =
+//   [
+//     [1.0, 0.0, 0.0]  // one_hot(0)
+//     [0.0, 0.0, 1.0]  // one_hot(2)
+//   ][
+//     [0.0, 1.0, 0.0]  // one_hot(1)
+//     [0.0, 0.0, 0.0]  // one_hot(-1)
+//   ]
+// ```
 //
 // Arguments:
 //	indices: A tensor of indices.
@@ -11594,8 +12045,8 @@ func ResizeBicubic(scope *Scope, images tf.Output, size tf.Output, optional ...R
 // Arguments:
 //	params_nested_splits: The `nested_row_splits` tensors that define the row-partitioning for the
 // `params` RaggedTensor input.
-//	params_dense_values: The `inner_values` for the `params` RaggedTensor. There was a terminology change
-// at the python level from dense_values to inner_values, so dense_values is the
+//	params_dense_values: The `flat_values` for the `params` RaggedTensor. There was a terminology change
+// at the python level from dense_values to flat_values, so dense_values is the
 // deprecated name.
 //	indices: Indices in the outermost dimension of `params` of the values that should be
 // gathered.
@@ -11604,7 +12055,7 @@ func ResizeBicubic(scope *Scope, images tf.Output, size tf.Output, optional ...R
 // `indices.shape.ndims + params.ragged_rank - 1`.
 //
 // Returns The `nested_row_splits` tensors that define the row-partitioning for the
-// returned RaggedTensor.The `inner_values` for the returned RaggedTensor.
+// returned RaggedTensor.The `flat_values` for the returned RaggedTensor.
 func RaggedGather(scope *Scope, params_nested_splits []tf.Output, params_dense_values tf.Output, indices tf.Output, OUTPUT_RAGGED_RANK int64) (output_nested_splits []tf.Output, output_dense_values tf.Output) {
 	if scope.Err() != nil {
 		return
@@ -11685,7 +12136,7 @@ func NonMaxSuppressionV2(scope *Scope, boxes tf.Output, scores tf.Output, max_ou
 //
 // Arguments:
 //	rt_nested_splits: The `row_splits` for the `RaggedTensor`.
-//	rt_dense_values: The `inner_values` for the `RaggedTensor`.
+//	rt_dense_values: The `flat_values` for the `RaggedTensor`.
 //
 // Returns The indices for the `SparseTensor`.The values of the `SparseTensor`.`sparse_dense_shape` is a tight bounding box of the input `RaggedTensor`.
 func RaggedTensorToSparse(scope *Scope, rt_nested_splits []tf.Output, rt_dense_values tf.Output) (sparse_indices tf.Output, sparse_values tf.Output, sparse_dense_shape tf.Output) {
@@ -12936,9 +13387,7 @@ func ResourceScatterNdAddUseLocking(value bool) ResourceScatterNdAddAttr {
 	}
 }
 
-// Adds sparse `updates` to individual values or slices within a given
-//
-// variable according to `indices`.
+// Applies sparse addition to individual values or slices in a Variable.
 //
 // `ref` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`.
 //
@@ -12952,24 +13401,24 @@ func ResourceScatterNdAddUseLocking(value bool) ResourceScatterNdAddAttr {
 // `updates` is `Tensor` of rank `Q-1+P-K` with shape:
 //
 // ```
-// [d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]].
+// [d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]]
 // ```
 //
-// For example, say we want to update 4 scattered elements to a rank-1 tensor to
-// 8 elements. In Python, that update would look like this:
+// For example, say we want to add 4 scattered elements to a rank-1 tensor to
+// 8 elements. In Python, that addition would look like this:
 //
 // ```python
-//     ref = tf.Variable([1, 2, 3, 4, 5, 6, 7, 8], use_resource=True)
-//     indices = tf.constant([[4], [3], [1] ,[7]])
-//     updates = tf.constant([9, 10, 11, 12])
-//     update = tf.scatter_nd_add(ref, indices, updates)
-//     with tf.Session() as sess:
-//       print sess.run(update)
+// ref = tf.Variable([1, 2, 3, 4, 5, 6, 7, 8], use_resource=True)
+// indices = tf.constant([[4], [3], [1], [7]])
+// updates = tf.constant([9, 10, 11, 12])
+// add = tf.scatter_nd_add(ref, indices, updates)
+// with tf.Session() as sess:
+//   print sess.run(add)
 // ```
 //
 // The resulting update to ref would look like this:
 //
-//     [1, 12, 3, 14, 14, 6, 7, 20]
+//     [1, 13, 3, 14, 14, 6, 7, 20]
 //
 // See `tf.scatter_nd` for more details about how to make updates to
 // slices.
@@ -13274,6 +13723,91 @@ func StatelessRandomNormal(scope *Scope, shape tf.Output, seed tf.Output, option
 	return op.Output(0)
 }
 
+// UnicodeDecodeAttr is an optional argument to UnicodeDecode.
+type UnicodeDecodeAttr func(optionalAttr)
+
+// UnicodeDecodeErrors sets the optional errors attribute to value.
+//
+// value: Error handling policy when there is invalid formatting found in the input.
+// The value of 'strict' will cause the operation to produce a InvalidArgument
+// error on any invalid input formatting. A value of 'replace' (the default) will
+// cause the operation to replace any invalid formatting in the input with the
+// `replacement_char` codepoint. A value of 'ignore' will cause the operation to
+// skip any invalid formatting in the input and produce no corresponding output
+// character.
+// If not specified, defaults to "replace"
+func UnicodeDecodeErrors(value string) UnicodeDecodeAttr {
+	return func(m optionalAttr) {
+		m["errors"] = value
+	}
+}
+
+// UnicodeDecodeReplacementChar sets the optional replacement_char attribute to value.
+//
+// value: The replacement character codepoint to be used in place of any invalid
+// formatting in the input when `errors='replace'`. Any valid unicode codepoint may
+// be used. The default value is the default unicode replacement character is
+// 0xFFFD or U+65533.)
+// If not specified, defaults to 65533
+func UnicodeDecodeReplacementChar(value int64) UnicodeDecodeAttr {
+	return func(m optionalAttr) {
+		m["replacement_char"] = value
+	}
+}
+
+// UnicodeDecodeReplaceControlCharacters sets the optional replace_control_characters attribute to value.
+//
+// value: Whether to replace the C0 control characters (00-1F) with the
+// `replacement_char`. Default is false.
+// If not specified, defaults to false
+func UnicodeDecodeReplaceControlCharacters(value bool) UnicodeDecodeAttr {
+	return func(m optionalAttr) {
+		m["replace_control_characters"] = value
+	}
+}
+
+// Decodes each string in `input` into a sequence of Unicode code points.
+//
+// The character codepoints for all strings are returned using a single vector
+// `char_values`, with strings expanded to characters in row-major order.
+//
+// The `row_splits` tensor indicates where the codepoints for
+// each input string begin and end within the `char_values` tensor.
+// In particular, the values for the `i`th
+// string (in row-major order) are stored in the slice
+// `[row_splits[i]:row_splits[i+1]]`. Thus:
+//
+// * `char_values[row_splits[i]+j]` is the Unicode codepoint for the `j`th
+//   character in the `i`th string (in row-major order).
+// * `row_splits[i+1] - row_splits[i]` is the number of characters in the `i`th
+//   string (in row-major order).
+//
+// Arguments:
+//	input: The text to be decoded. Can have any shape. Note that the output is flattened
+// to a vector of char values.
+//	input_encoding: Text encoding of the input strings. This is any of the encodings supported
+// by ICU ucnv algorithmic converters. Examples: `"UTF-16", "US ASCII", "UTF-8"`.
+//
+// Returns A 1D int32 tensor containing the row splits.A 1D int32 Tensor containing the decoded codepoints.
+func UnicodeDecode(scope *Scope, input tf.Output, input_encoding string, optional ...UnicodeDecodeAttr) (row_splits tf.Output, char_values tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"input_encoding": input_encoding}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "UnicodeDecode",
+		Input: []tf.Input{
+			input,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
 // Adds up a SparseTensor and a dense Tensor, using these special rules:
 //
 // (1) Broadcasts the dense side to have the same shape as the sparse side, if
@@ -13322,6 +13856,84 @@ func Erfc(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
+// UnicodeEncodeAttr is an optional argument to UnicodeEncode.
+type UnicodeEncodeAttr func(optionalAttr)
+
+// UnicodeEncodeErrors sets the optional errors attribute to value.
+//
+// value: Error handling policy when there is invalid formatting found in the input.
+// The value of 'strict' will cause the operation to produce a InvalidArgument
+// error on any invalid input formatting. A value of 'replace' (the default) will
+// cause the operation to replace any invalid formatting in the input with the
+// `replacement_char` codepoint. A value of 'ignore' will cause the operation to
+// skip any invalid formatting in the input and produce no corresponding output
+// character.
+// If not specified, defaults to "replace"
+func UnicodeEncodeErrors(value string) UnicodeEncodeAttr {
+	return func(m optionalAttr) {
+		m["errors"] = value
+	}
+}
+
+// UnicodeEncodeReplacementChar sets the optional replacement_char attribute to value.
+//
+// value: The replacement character codepoint to be used in place of any invalid
+// formatting in the input when `errors='replace'`. Any valid unicode codepoint may
+// be used. The default value is the default unicode replacement character is
+// 0xFFFD (U+65533).
+// If not specified, defaults to 65533
+func UnicodeEncodeReplacementChar(value int64) UnicodeEncodeAttr {
+	return func(m optionalAttr) {
+		m["replacement_char"] = value
+	}
+}
+
+// Encode a tensor of ints into unicode strings.
+//
+// Returns a vector of strings, where `output[i]` is constructed by encoding the
+// Unicode codepoints in `input_values[input_splits[i]:input_splits[i+1]]`
+// using `output_encoding`.
+//
+// ---
+//
+// Example:
+//
+// ```
+// input_values = [72, 101, 108, 108, 111, 87, 111, 114, 108, 100]
+// input_splits = [0, 5, 10]
+// output_encoding = 'UTF-8'
+//
+// output = ['Hello', 'World']
+// ```
+//
+// Arguments:
+//	input_values: A 1D tensor containing the unicode codepoints that should be encoded.
+//	input_splits: A 1D tensor specifying how the unicode codepoints should be split into strings.
+// In particular, `output[i]` is constructed by encoding the codepoints in the
+// slice `input_values[input_splits[i]:input_splits[i+1]]`.
+//	output_encoding: Unicode encoding of the output strings. Valid encodings are: `"UTF-8",
+// "UTF-16-BE", and "UTF-32-BE"`.
+//
+// Returns The 1-D Tensor of strings encoded from the provided unicode codepoints.
+func UnicodeEncode(scope *Scope, input_values tf.Output, input_splits tf.Output, output_encoding string, optional ...UnicodeEncodeAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_encoding": output_encoding}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "UnicodeEncode",
+		Input: []tf.Input{
+			input_values, input_splits,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Returns the number of tensors in the input tensor list.
 //
 // input_handle: the input list
@@ -14644,6 +15256,117 @@ func MultiDeviceIteratorToStringHandle(scope *Scope, multi_device_iterator tf.Ou
 	return op.Output(0)
 }
 
+// CudnnRNNV3Attr is an optional argument to CudnnRNNV3.
+type CudnnRNNV3Attr func(optionalAttr)
+
+// CudnnRNNV3RnnMode sets the optional rnn_mode attribute to value.
+// If not specified, defaults to "lstm"
+func CudnnRNNV3RnnMode(value string) CudnnRNNV3Attr {
+	return func(m optionalAttr) {
+		m["rnn_mode"] = value
+	}
+}
+
+// CudnnRNNV3InputMode sets the optional input_mode attribute to value.
+// If not specified, defaults to "linear_input"
+func CudnnRNNV3InputMode(value string) CudnnRNNV3Attr {
+	return func(m optionalAttr) {
+		m["input_mode"] = value
+	}
+}
+
+// CudnnRNNV3Direction sets the optional direction attribute to value.
+// If not specified, defaults to "unidirectional"
+func CudnnRNNV3Direction(value string) CudnnRNNV3Attr {
+	return func(m optionalAttr) {
+		m["direction"] = value
+	}
+}
+
+// CudnnRNNV3Dropout sets the optional dropout attribute to value.
+// If not specified, defaults to 0
+func CudnnRNNV3Dropout(value float32) CudnnRNNV3Attr {
+	return func(m optionalAttr) {
+		m["dropout"] = value
+	}
+}
+
+// CudnnRNNV3Seed sets the optional seed attribute to value.
+// If not specified, defaults to 0
+func CudnnRNNV3Seed(value int64) CudnnRNNV3Attr {
+	return func(m optionalAttr) {
+		m["seed"] = value
+	}
+}
+
+// CudnnRNNV3Seed2 sets the optional seed2 attribute to value.
+// If not specified, defaults to 0
+func CudnnRNNV3Seed2(value int64) CudnnRNNV3Attr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// CudnnRNNV3IsTraining sets the optional is_training attribute to value.
+// If not specified, defaults to true
+func CudnnRNNV3IsTraining(value bool) CudnnRNNV3Attr {
+	return func(m optionalAttr) {
+		m["is_training"] = value
+	}
+}
+
+// A RNN backed by cuDNN.
+//
+// Computes the RNN from the input and initial states, with respect to the params
+// buffer. Accepts one extra input "sequence_lengths" than CudnnRNN.
+//
+// rnn_mode: Indicates the type of the RNN model.
+// input_mode: Indicates whether there is a linear projection between the input and
+//   the actual computation before the first layer. 'skip_input' is only allowed
+//   when input_size == num_units; 'auto_select' implies 'skip_input' when
+//   input_size == num_units; otherwise, it implies 'linear_input'.
+// direction: Indicates whether a bidirectional model will be used. Should be
+//   "unidirectional" or "bidirectional".
+// dropout: Dropout probability. When set to 0., dropout is disabled.
+// seed: The 1st part of a seed to initialize dropout.
+// seed2: The 2nd part of a seed to initialize dropout.
+// input: A 3-D tensor with the shape of [seq_length, batch_size, input_size].
+// input_h: A 3-D tensor with the shape of [num_layer * dir, batch_size,
+//     num_units].
+// input_c: For LSTM, a 3-D tensor with the shape of
+//     [num_layer * dir, batch, num_units]. For other models, it is ignored.
+// params: A 1-D tensor that contains the weights and biases in an opaque layout.
+//     The size must be created through CudnnRNNParamsSize, and initialized
+//     separately. Note that they might not be compatible across different
+//     generations. So it is a good idea to save and restore
+// sequence_lengths: a vector of lengths of each input sequence.
+// output: A 3-D tensor with the shape of [seq_length, batch_size,
+//     dir * num_units].
+// output_h: The same shape has input_h.
+// output_c: The same shape as input_c for LSTM. An empty tensor for other models.
+// is_training: Indicates whether this operation is used for inferenece or
+//   training.
+// reserve_space: An opaque tensor that can be used in backprop calculation. It
+//   is only produced if is_training is true.
+func CudnnRNNV3(scope *Scope, input tf.Output, input_h tf.Output, input_c tf.Output, params tf.Output, sequence_lengths tf.Output, optional ...CudnnRNNV3Attr) (output tf.Output, output_h tf.Output, output_c tf.Output, reserve_space tf.Output, host_reserved tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "CudnnRNNV3",
+		Input: []tf.Input{
+			input, input_h, input_c, params, sequence_lengths,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4)
+}
+
 // Applies softmax to a batched N-D `SparseTensor`.
 //
 // The inputs represent an N-D SparseTensor  with logical shape `[..., B, C]`
@@ -16873,6 +17596,23 @@ func SparseReduceSum(scope *Scope, input_indices tf.Output, input_values tf.Outp
 	return op.Output(0)
 }
 
+// Records the latency of producing `input_dataset` elements in a StatsAggregator.
+func ExperimentalLatencyStatsDataset(scope *Scope, input_dataset tf.Output, tag tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalLatencyStatsDataset",
+		Input: []tf.Input{
+			input_dataset, tag,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // SparseTensorDenseMatMulAttr is an optional argument to SparseTensorDenseMatMul.
 type SparseTensorDenseMatMulAttr func(optionalAttr)
 
@@ -17555,6 +18295,69 @@ func Timestamp(scope *Scope) (ts tf.Output) {
 	return op.Output(0)
 }
 
+// ResourceSparseApplyKerasMomentumAttr is an optional argument to ResourceSparseApplyKerasMomentum.
+type ResourceSparseApplyKerasMomentumAttr func(optionalAttr)
+
+// ResourceSparseApplyKerasMomentumUseLocking sets the optional use_locking attribute to value.
+//
+// value: If `True`, updating of the var and accum tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceSparseApplyKerasMomentumUseLocking(value bool) ResourceSparseApplyKerasMomentumAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
+	}
+}
+
+// ResourceSparseApplyKerasMomentumUseNesterov sets the optional use_nesterov attribute to value.
+//
+// value: If `True`, the tensor passed to compute grad will be
+// var + momentum * accum, so in the end, the var you get is actually
+// var + momentum * accum.
+// If not specified, defaults to false
+func ResourceSparseApplyKerasMomentumUseNesterov(value bool) ResourceSparseApplyKerasMomentumAttr {
+	return func(m optionalAttr) {
+		m["use_nesterov"] = value
+	}
+}
+
+// Update relevant entries in '*var' and '*accum' according to the momentum scheme.
+//
+// Set use_nesterov = True if you want to use Nesterov momentum.
+//
+// That is for rows we have grad for, we update var and accum as follows:
+//
+// accum = accum * momentum - lr * grad
+// var += accum
+//
+// Arguments:
+//	var_: Should be from a Variable().
+//	accum: Should be from a Variable().
+//	lr: Learning rate. Must be a scalar.
+//	grad: The gradient.
+//	indices: A vector of indices into the first dimension of var and accum.
+//	momentum: Momentum. Must be a scalar.
+//
+// Returns the created operation.
+func ResourceSparseApplyKerasMomentum(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, indices tf.Output, momentum tf.Output, optional ...ResourceSparseApplyKerasMomentumAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceSparseApplyKerasMomentum",
+		Input: []tf.Input{
+			var_, accum, lr, grad, indices, momentum,
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
 // VariableShapeAttr is an optional argument to VariableShape.
 type VariableShapeAttr func(optionalAttr)
 
@@ -18293,6 +19096,96 @@ func DivNoNan(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	return op.Output(0)
 }
 
+// UnicodeDecodeWithOffsetsAttr is an optional argument to UnicodeDecodeWithOffsets.
+type UnicodeDecodeWithOffsetsAttr func(optionalAttr)
+
+// UnicodeDecodeWithOffsetsErrors sets the optional errors attribute to value.
+//
+// value: Error handling policy when there is invalid formatting found in the input.
+// The value of 'strict' will cause the operation to produce a InvalidArgument
+// error on any invalid input formatting. A value of 'replace' (the default) will
+// cause the operation to replace any invalid formatting in the input with the
+// `replacement_char` codepoint. A value of 'ignore' will cause the operation to
+// skip any invalid formatting in the input and produce no corresponding output
+// character.
+// If not specified, defaults to "replace"
+func UnicodeDecodeWithOffsetsErrors(value string) UnicodeDecodeWithOffsetsAttr {
+	return func(m optionalAttr) {
+		m["errors"] = value
+	}
+}
+
+// UnicodeDecodeWithOffsetsReplacementChar sets the optional replacement_char attribute to value.
+//
+// value: The replacement character codepoint to be used in place of any invalid
+// formatting in the input when `errors='replace'`. Any valid unicode codepoint may
+// be used. The default value is the default unicode replacement character is
+// 0xFFFD or U+65533.)
+// If not specified, defaults to 65533
+func UnicodeDecodeWithOffsetsReplacementChar(value int64) UnicodeDecodeWithOffsetsAttr {
+	return func(m optionalAttr) {
+		m["replacement_char"] = value
+	}
+}
+
+// UnicodeDecodeWithOffsetsReplaceControlCharacters sets the optional replace_control_characters attribute to value.
+//
+// value: Whether to replace the C0 control characters (00-1F) with the
+// `replacement_char`. Default is false.
+// If not specified, defaults to false
+func UnicodeDecodeWithOffsetsReplaceControlCharacters(value bool) UnicodeDecodeWithOffsetsAttr {
+	return func(m optionalAttr) {
+		m["replace_control_characters"] = value
+	}
+}
+
+// Decodes each string in `input` into a sequence of Unicode code points.
+//
+// The character codepoints for all strings are returned using a single vector
+// `char_values`, with strings expanded to characters in row-major order.
+// Similarly, the character start byte offsets are returned using a single vector
+// `char_to_byte_starts`, with strings expanded in row-major order.
+//
+// The `row_splits` tensor indicates where the codepoints and start offsets for
+// each input string begin and end within the `char_values` and
+// `char_to_byte_starts` tensors.  In particular, the values for the `i`th
+// string (in row-major order) are stored in the slice
+// `[row_splits[i]:row_splits[i+1]]`. Thus:
+//
+// * `char_values[row_splits[i]+j]` is the Unicode codepoint for the `j`th
+//   character in the `i`th string (in row-major order).
+// * `char_to_bytes_starts[row_splits[i]+j]` is the start byte offset for the `j`th
+//   character in the `i`th string (in row-major order).
+// * `row_splits[i+1] - row_splits[i]` is the number of characters in the `i`th
+//   string (in row-major order).
+//
+// Arguments:
+//	input: The text to be decoded. Can have any shape. Note that the output is flattened
+// to a vector of char values.
+//	input_encoding: Text encoding of the input strings. This is any of the encodings supported
+// by ICU ucnv algorithmic converters. Examples: `"UTF-16", "US ASCII", "UTF-8"`.
+//
+// Returns A 1D int32 tensor containing the row splits.A 1D int32 Tensor containing the decoded codepoints.A 1D int32 Tensor containing the byte index in the input string where each
+// character in `char_values` starts.
+func UnicodeDecodeWithOffsets(scope *Scope, input tf.Output, input_encoding string, optional ...UnicodeDecodeWithOffsetsAttr) (row_splits tf.Output, char_values tf.Output, char_to_byte_starts tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"input_encoding": input_encoding}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "UnicodeDecodeWithOffsets",
+		Input: []tf.Input{
+			input,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2)
+}
+
 // Returns x - y element-wise.
 //
 // *NOTE*: `Subtract` supports broadcasting. More about broadcasting
@@ -20606,15 +21499,62 @@ func Zeta(scope *Scope, x tf.Output, q tf.Output) (z tf.Output) {
 	return op.Output(0)
 }
 
+// Returns the cardinality of `input_dataset`.
+//
+// Returns the cardinality of `input_dataset`.
+//
+// Arguments:
+//	input_dataset: A variant tensor representing the dataset to return cardinality for.
+//
+// Returns The cardinality of `input_dataset`. Named constants are used to represent
+// infinite and unknown cardinality.
+func ExperimentalDatasetCardinality(scope *Scope, input_dataset tf.Output) (cardinality tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalDatasetCardinality",
+		Input: []tf.Input{
+			input_dataset,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Creates a dataset that executes a SQL query and emits rows of the result set.
+//
+// Arguments:
+//	driver_name: The database type. Currently, the only supported type is 'sqlite'.
+//	data_source_name: A connection string to connect to the database.
+//	query: A SQL query to execute.
+//
+//
+func ExperimentalSqlDataset(scope *Scope, driver_name tf.Output, data_source_name tf.Output, query tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalSqlDataset",
+		Input: []tf.Input{
+			driver_name, data_source_name, query,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Inverse fast Fourier transform.
 //
 // Computes the inverse 1-dimensional discrete Fourier transform over the
 // inner-most dimension of `input`.
 //
 // Arguments:
-//	input: A complex64 tensor.
+//	input: A complex tensor.
 //
-// Returns A complex64 tensor of the same shape as `input`. The inner-most
+// Returns A complex tensor of the same shape as `input`. The inner-most
 //   dimension of `input` is replaced with its inverse 1D Fourier transform.
 //
 // @compatibility(numpy)
@@ -20640,9 +21580,9 @@ func IFFT(scope *Scope, input tf.Output) (output tf.Output) {
 // 2 dimensions of `input`.
 //
 // Arguments:
-//	input: A complex64 tensor.
+//	input: A complex tensor.
 //
-// Returns A complex64 tensor of the same shape as `input`. The inner-most 2
+// Returns A complex tensor of the same shape as `input`. The inner-most 2
 //   dimensions of `input` are replaced with their 2D Fourier transform.
 //
 // @compatibility(numpy)
@@ -20668,9 +21608,9 @@ func FFT2D(scope *Scope, input tf.Output) (output tf.Output) {
 // inner-most 2 dimensions of `input`.
 //
 // Arguments:
-//	input: A complex64 tensor.
+//	input: A complex tensor.
 //
-// Returns A complex64 tensor of the same shape as `input`. The inner-most 2
+// Returns A complex tensor of the same shape as `input`. The inner-most 2
 //   dimensions of `input` are replaced with their inverse 2D Fourier transform.
 //
 // @compatibility(numpy)
@@ -21010,6 +21950,44 @@ func ReaderNumRecordsProducedV2(scope *Scope, reader_handle tf.Output) (records_
 	return op.Output(0)
 }
 
+// TensorListConcatAttr is an optional argument to TensorListConcat.
+type TensorListConcatAttr func(optionalAttr)
+
+// TensorListConcatElementShape sets the optional element_shape attribute to value.
+// If not specified, defaults to <unknown_rank:true >
+func TensorListConcatElementShape(value tf.Shape) TensorListConcatAttr {
+	return func(m optionalAttr) {
+		m["element_shape"] = value
+	}
+}
+
+// Concats all tensors in the list along the 0th dimension.
+//
+// Requires that all tensors have the same shape except the first dimension.
+//
+// input_handle: The input list.
+// tensor: The concated result.
+// lengths: Output tensor containing sizes of the 0th dimension of tensors in the list, used for computing the gradient.
+//
+func TensorListConcat(scope *Scope, input_handle tf.Output, element_dtype tf.DataType, optional ...TensorListConcatAttr) (tensor tf.Output, lengths tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"element_dtype": element_dtype}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorListConcat",
+		Input: []tf.Input{
+			input_handle,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
 // Returns the set of files matching one or more glob patterns.
 //
 // Note that this routine only supports wildcard characters in the
@@ -22051,6 +23029,61 @@ func SparseSparseMinimum(scope *Scope, a_indices tf.Output, a_values tf.Output,
 	return op.Output(0), op.Output(1)
 }
 
+// ResourceApplyAdamWithAmsgradAttr is an optional argument to ResourceApplyAdamWithAmsgrad.
+type ResourceApplyAdamWithAmsgradAttr func(optionalAttr)
+
+// ResourceApplyAdamWithAmsgradUseLocking sets the optional use_locking attribute to value.
+//
+// value: If `True`, updating of the var, m, and v tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceApplyAdamWithAmsgradUseLocking(value bool) ResourceApplyAdamWithAmsgradAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
+	}
+}
+
+// Update '*var' according to the Adam algorithm.
+//
+// $$lr_t := \text{learning\_rate} * \sqrt{1 - beta_2^t} / (1 - beta_1^t)$$
+// $$m_t := beta_1 * m_{t-1} + (1 - beta_1) * g$$
+// $$v_t := beta_2 * v_{t-1} + (1 - beta_2) * g * g$$
+// $$vhat_t := max{vhat_{t-1}, v_t}$$
+// $$variable := variable - lr_t * m_t / (\sqrt{vhat_t} + \epsilon)$$
+//
+// Arguments:
+//	var_: Should be from a Variable().
+//	m: Should be from a Variable().
+//	v: Should be from a Variable().
+//	vhat: Should be from a Variable().
+//	beta1_power: Must be a scalar.
+//	beta2_power: Must be a scalar.
+//	lr: Scaling factor. Must be a scalar.
+//	beta1: Momentum factor. Must be a scalar.
+//	beta2: Momentum factor. Must be a scalar.
+//	epsilon: Ridge term. Must be a scalar.
+//	grad: The gradient.
+//
+// Returns the created operation.
+func ResourceApplyAdamWithAmsgrad(scope *Scope, var_ tf.Output, m tf.Output, v tf.Output, vhat tf.Output, beta1_power tf.Output, beta2_power tf.Output, lr tf.Output, beta1 tf.Output, beta2 tf.Output, epsilon tf.Output, grad tf.Output, optional ...ResourceApplyAdamWithAmsgradAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceApplyAdamWithAmsgrad",
+		Input: []tf.Input{
+			var_, m, v, vhat, beta1_power, beta2_power, lr, beta1, beta2, epsilon, grad,
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
 // MapUnstageNoKeyAttr is an optional argument to MapUnstageNoKey.
 type MapUnstageNoKeyAttr func(optionalAttr)
 
@@ -22452,6 +23485,93 @@ func SetSize(scope *Scope, set_indices tf.Output, set_values tf.Output, set_shap
 	return op.Output(0)
 }
 
+// Adds sparse `updates` to an existing tensor according to `indices`.
+//
+// This operation creates a new tensor by adding sparse `updates` to the passed
+// in `tensor`.
+// This operation is very similar to `tf.scatter_nd_add`, except that the updates
+// are added onto an existing tensor (as opposed to a variable). If the memory
+// for the existing tensor cannot be re-used, a copy is made and updated.
+//
+// `indices` is an integer tensor containing indices into a new tensor of shape
+// `shape`.  The last dimension of `indices` can be at most the rank of `shape`:
+//
+//     indices.shape[-1] <= shape.rank
+//
+// The last dimension of `indices` corresponds to indices into elements
+// (if `indices.shape[-1] = shape.rank`) or slices
+// (if `indices.shape[-1] < shape.rank`) along dimension `indices.shape[-1]` of
+// `shape`.  `updates` is a tensor with shape
+//
+//     indices.shape[:-1] + shape[indices.shape[-1]:]
+//
+// The simplest form of tensor_scatter_add is to add individual elements to a
+// tensor by index. For example, say we want to add 4 elements in a rank-1
+// tensor with 8 elements.
+//
+// In Python, this scatter add operation would look like this:
+//
+// ```python
+//     indices = tf.constant([[4], [3], [1], [7]])
+//     updates = tf.constant([9, 10, 11, 12])
+//     tensor = tf.ones([8], dtype=tf.int32)
+//     updated = tf.tensor_scatter_add(tensor, indices, updates)
+//     with tf.Session() as sess:
+//       print(sess.run(scatter))
+// ```
+//
+// The resulting tensor would look like this:
+//
+//     [1, 12, 1, 11, 10, 1, 1, 13]
+//
+// We can also, insert entire slices of a higher rank tensor all at once. For
+// example, if we wanted to insert two slices in the first dimension of a
+// rank-3 tensor with two matrices of new values.
+//
+// In Python, this scatter add operation would look like this:
+//
+// ```python
+//     indices = tf.constant([[0], [2]])
+//     updates = tf.constant([[[5, 5, 5, 5], [6, 6, 6, 6],
+//                             [7, 7, 7, 7], [8, 8, 8, 8]],
+//                            [[5, 5, 5, 5], [6, 6, 6, 6],
+//                             [7, 7, 7, 7], [8, 8, 8, 8]]])
+//     tensor = tf.ones([4, 4, 4])
+//     updated = tf.tensor_scatter_add(tensor, indices, updates)
+//     with tf.Session() as sess:
+//       print(sess.run(scatter))
+// ```
+//
+// The resulting tensor would look like this:
+//
+//     [[[6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8], [9, 9, 9, 9]],
+//      [[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]],
+//      [[6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8], [9, 9, 9, 9]],
+//      [[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]]]
+//
+// Note that on CPU, if an out of bound index is found, an error is returned.
+// On GPU, if an out of bound index is found, the index is ignored.
+//
+// Arguments:
+//	tensor: Tensor to copy/update.
+//	indices: Index tensor.
+//	updates: Updates to scatter into output.
+//
+// Returns A new tensor copied from tensor and updates added according to the indices.
+func TensorScatterAdd(scope *Scope, tensor tf.Output, indices tf.Output, updates tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorScatterAdd",
+		Input: []tf.Input{
+			tensor, indices, updates,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Computes the sign and the log of the absolute value of the determinant of
 //
 // one or more square matrices.
@@ -22676,6 +23796,84 @@ func QuantizedResizeBilinear(scope *Scope, images tf.Output, size tf.Output, min
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
+// Creates a dataset that uses a custom thread pool to compute `input_dataset`.
+//
+// Arguments:
+//
+//	num_threads: Identifies the number of threads to use for the private threadpool.
+//
+//
+func ExperimentalPrivateThreadPoolDataset(scope *Scope, input_dataset tf.Output, num_threads tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalPrivateThreadPoolDataset",
+		Input: []tf.Input{
+			input_dataset, num_threads,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// ExperimentalParseExampleDatasetAttr is an optional argument to ExperimentalParseExampleDataset.
+type ExperimentalParseExampleDatasetAttr func(optionalAttr)
+
+// ExperimentalParseExampleDatasetSloppy sets the optional sloppy attribute to value.
+// If not specified, defaults to false
+func ExperimentalParseExampleDatasetSloppy(value bool) ExperimentalParseExampleDatasetAttr {
+	return func(m optionalAttr) {
+		m["sloppy"] = value
+	}
+}
+
+// Transforms `input_dataset` containing `Example` protos as vectors of DT_STRING into a dataset of `Tensor` or `SparseTensor` objects representing the parsed features.
+//
+// Arguments:
+//
+//
+//	dense_defaults: A dict mapping string keys to `Tensor`s.
+// The keys of the dict must match the dense_keys of the feature.
+//	sparse_keys: A list of string keys in the examples features.
+// The results for these keys will be returned as `SparseTensor` objects.
+//	dense_keys: A list of Ndense string Tensors (scalars).
+// The keys expected in the Examples features associated with dense values.
+//	sparse_types: A list of `DTypes` of the same length as `sparse_keys`.
+// Only `tf.float32` (`FloatList`), `tf.int64` (`Int64List`),
+// and `tf.string` (`BytesList`) are supported.
+//	dense_shapes: List of tuples with the same length as `dense_keys`.
+// The shape of the data for each dense feature referenced by `dense_keys`.
+// Required for any input tensors identified by `dense_keys`.  Must be
+// either fully defined, or may contain an unknown first dimension.
+// An unknown first dimension means the feature is treated as having
+// a variable number of blocks, and the output shape along this dimension
+// is considered unknown at graph build time.  Padding is applied for
+// minibatch elements smaller than the maximum number of blocks for the
+// given feature along this dimension.
+//	output_types: The type list for the return values.
+//	output_shapes: The list of shapes being produced.
+func ExperimentalParseExampleDataset(scope *Scope, input_dataset tf.Output, num_parallel_calls tf.Output, dense_defaults []tf.Output, sparse_keys []string, dense_keys []string, sparse_types []tf.DataType, dense_shapes []tf.Shape, output_types []tf.DataType, output_shapes []tf.Shape, optional ...ExperimentalParseExampleDatasetAttr) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"sparse_keys": sparse_keys, "dense_keys": dense_keys, "sparse_types": sparse_types, "dense_shapes": dense_shapes, "output_types": output_types, "output_shapes": output_shapes}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalParseExampleDataset",
+		Input: []tf.Input{
+			input_dataset, num_parallel_calls, tf.OutputList(dense_defaults),
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // SdcaOptimizerAttr is an optional argument to SdcaOptimizer.
 type SdcaOptimizerAttr func(optionalAttr)
 
@@ -23420,6 +24618,26 @@ func MatMul(scope *Scope, a tf.Output, b tf.Output, optional ...MatMulAttr) (pro
 	return op.Output(0)
 }
 
+// Serializes the tree handle to a proto
+//
+// Arguments:
+//	tree_handle: Handle to the tree resource to be serialized.
+//
+// Returns Serialied proto string of the tree resource.
+func TensorForestTreeSerialize(scope *Scope, tree_handle tf.Output) (tree_config tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorForestTreeSerialize",
+		Input: []tf.Input{
+			tree_handle,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // SparseMatMulAttr is an optional argument to SparseMatMul.
 type SparseMatMulAttr func(optionalAttr)
 
@@ -26575,6 +27793,28 @@ func Invert(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
+// Deserialize bucket boundaries and ready flag into current QuantileAccumulator.
+//
+// An op that deserializes bucket boundaries and are boundaries ready flag into current QuantileAccumulator.
+//
+// Arguments:
+//	quantile_stream_resource_handle: resource handle referring to a QuantileStreamResource.
+//	bucket_boundaries: float; List of Rank 1 Tensors each containing the bucket boundaries for a feature.
+//
+// Returns the created operation.
+func BoostedTreesQuantileStreamResourceDeserialize(scope *Scope, quantile_stream_resource_handle tf.Output, bucket_boundaries []tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "BoostedTreesQuantileStreamResourceDeserialize",
+		Input: []tf.Input{
+			quantile_stream_resource_handle, tf.OutputList(bucket_boundaries),
+		},
+	}
+	return scope.AddOperation(opspec)
+}
+
 // Inverse 3D fast Fourier transform.
 //
 // Computes the inverse 3-dimensional discrete Fourier transform over the
@@ -27102,6 +28342,29 @@ func AudioSummaryV2(scope *Scope, tag tf.Output, tensor tf.Output, sample_rate t
 	return op.Output(0)
 }
 
+// Splits a tensor into a list.
+//
+// list[i] corresponds to lengths[i] tensors from the input tensor.
+// The tensor must have rank at least 1 and contain exactly sum(lengths) elements.
+//
+// tensor: The input tensor.
+// element_shape: A shape compatible with that of elements in the tensor.
+// lengths: Vector of sizes of the 0th dimension of tensors in the list.
+// output_handle: The list.
+func TensorListSplit(scope *Scope, tensor tf.Output, element_shape tf.Output, lengths tf.Output) (output_handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorListSplit",
+		Input: []tf.Input{
+			tensor, element_shape, lengths,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // AvgPoolAttr is an optional argument to AvgPool.
 type AvgPoolAttr func(optionalAttr)
 
@@ -27222,6 +28485,26 @@ func TensorListGetItem(scope *Scope, input_handle tf.Output, index tf.Output, el
 	return op.Output(0)
 }
 
+// Resizes the list.
+//
+//
+// input_handle: the input list
+// size: size of the output list
+//
+func TensorListResize(scope *Scope, input_handle tf.Output, size tf.Output) (output_handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorListResize",
+		Input: []tf.Input{
+			input_handle, size,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Returns a diagonal tensor with a given diagonal values.
 //
 // Given a `diagonal`, this operation returns a tensor with the `diagonal` and
@@ -27412,7 +28695,7 @@ func TensorListScatter(scope *Scope, tensor tf.Output, indices tf.Output, elemen
 //	limits: The limits of each range.
 //	deltas: The deltas of each range.
 //
-// Returns The `row_splits` for the returned `RaggedTensor`.The `inner_values` for the returned `RaggedTensor`.
+// Returns The `row_splits` for the returned `RaggedTensor`.The `flat_values` for the returned `RaggedTensor`.
 func RaggedRange(scope *Scope, starts tf.Output, limits tf.Output, deltas tf.Output) (rt_nested_splits tf.Output, rt_dense_values tf.Output) {
 	if scope.Err() != nil {
 		return
@@ -27744,6 +29027,66 @@ func AdjustSaturation(scope *Scope, images tf.Output, scale tf.Output) (output t
 	return op.Output(0)
 }
 
+// ResourceApplyKerasMomentumAttr is an optional argument to ResourceApplyKerasMomentum.
+type ResourceApplyKerasMomentumAttr func(optionalAttr)
+
+// ResourceApplyKerasMomentumUseLocking sets the optional use_locking attribute to value.
+//
+// value: If `True`, updating of the var and accum tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceApplyKerasMomentumUseLocking(value bool) ResourceApplyKerasMomentumAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
+	}
+}
+
+// ResourceApplyKerasMomentumUseNesterov sets the optional use_nesterov attribute to value.
+//
+// value: If `True`, the tensor passed to compute grad will be
+// var + momentum * accum, so in the end, the var you get is actually
+// var + momentum * accum.
+// If not specified, defaults to false
+func ResourceApplyKerasMomentumUseNesterov(value bool) ResourceApplyKerasMomentumAttr {
+	return func(m optionalAttr) {
+		m["use_nesterov"] = value
+	}
+}
+
+// Update '*var' according to the momentum scheme. Set use_nesterov = True if you
+//
+// want to use Nesterov momentum.
+//
+// accum = accum * momentum - lr * grad
+// var += accum
+//
+// Arguments:
+//	var_: Should be from a Variable().
+//	accum: Should be from a Variable().
+//	lr: Scaling factor. Must be a scalar.
+//	grad: The gradient.
+//	momentum: Momentum. Must be a scalar.
+//
+// Returns the created operation.
+func ResourceApplyKerasMomentum(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, momentum tf.Output, optional ...ResourceApplyKerasMomentumAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceApplyKerasMomentum",
+		Input: []tf.Input{
+			var_, accum, lr, grad, momentum,
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
 // MatrixSolveAttr is an optional argument to MatrixSolve.
 type MatrixSolveAttr func(optionalAttr)
 
@@ -27813,6 +29156,70 @@ func DatasetToGraph(scope *Scope, input_dataset tf.Output) (graph tf.Output) {
 	return op.Output(0)
 }
 
+// LuAttr is an optional argument to Lu.
+type LuAttr func(optionalAttr)
+
+// LuOutputIdxType sets the optional output_idx_type attribute to value.
+// If not specified, defaults to DT_INT32
+func LuOutputIdxType(value tf.DataType) LuAttr {
+	return func(m optionalAttr) {
+		m["output_idx_type"] = value
+	}
+}
+
+// Computes the LU decomposition of one or more square matrices.
+//
+// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
+// form square matrices.
+//
+// The input has to be invertible.
+//
+// The output consists of two tensors LU and P containing the LU decomposition
+// of all input submatrices `[..., :, :]`. LU encodes the lower triangular and
+// upper triangular factors.
+//
+// For each input submatrix of shape `[M, M]`, L is a lower triangular matrix of
+// shape `[M, M]` with unit diagonal whose entries correspond to the strictly lower
+// triangular part of LU. U is a upper triangular matrix of shape `[M, M]` whose
+// entries correspond to the upper triangular part, including the diagonal, of LU.
+//
+// P represents a permutation matrix encoded as a list of indices each between `0`
+// and `M-1`, inclusive. If P_mat denotes the permutation matrix corresponding to
+// P, then the L, U and P satisfies P_mat * input = L * U.
+//
+// Arguments:
+//	input: A tensor of shape `[..., M, M]` whose inner-most 2 dimensions form matrices of
+// size `[M, M]`.
+//
+// Returns A tensor of shape `[..., M, M]` whose strictly lower triangular part denotes the
+// lower triangular factor `L` with unit diagonal, and whose upper triangular part
+// denotes the upper triangular factor `U`.Permutation of the rows encoded as a list of indices in `0..M-1`. Shape is
+// `[..., M]`.
+// @compatibility(scipy)
+// Similar to `scipy.linalg.lu`, except the triangular factors `L` and `U` are
+// packed into a single tensor, the permutation is applied to `input` instead of
+// the right hand side and the permutation `P` is returned as a list of indices
+// instead of a permutation matrix.
+// @end_compatibility
+func Lu(scope *Scope, input tf.Output, optional ...LuAttr) (lu tf.Output, p tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "Lu",
+		Input: []tf.Input{
+			input,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
 // Computes the matrix square root of one or more square matrices:
 //
 // matmul(sqrtm(A), sqrtm(A)) = A
@@ -29948,6 +31355,43 @@ func Iterator(scope *Scope, shared_name string, container string, output_types [
 	return op.Output(0)
 }
 
+// TensorForestTreeResourceHandleOpAttr is an optional argument to TensorForestTreeResourceHandleOp.
+type TensorForestTreeResourceHandleOpAttr func(optionalAttr)
+
+// TensorForestTreeResourceHandleOpContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func TensorForestTreeResourceHandleOpContainer(value string) TensorForestTreeResourceHandleOpAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// TensorForestTreeResourceHandleOpSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func TensorForestTreeResourceHandleOpSharedName(value string) TensorForestTreeResourceHandleOpAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Creates a handle to a TensorForestTreeResource
+func TensorForestTreeResourceHandleOp(scope *Scope, optional ...TensorForestTreeResourceHandleOpAttr) (resource tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorForestTreeResourceHandleOp",
+
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // CropAndResizeGradImageAttr is an optional argument to CropAndResizeGradImage.
 type CropAndResizeGradImageAttr func(optionalAttr)
 
@@ -30312,6 +31756,29 @@ func FakeParam(scope *Scope, dtype tf.DataType, shape tf.Shape) (output tf.Outpu
 	return op.Output(0)
 }
 
+// Returns the next representable value of `x1` in the direction of `x2`, element-wise.
+//
+// This operation returns the same result as the C++ std::nextafter function.
+//
+// It can also return a subnormal number.
+//
+// @compatibility(cpp)
+// Equivalent to C++ std::nextafter function.
+// @end_compatibility
+func NextAfter(scope *Scope, x1 tf.Output, x2 tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "NextAfter",
+		Input: []tf.Input{
+			x1, x2,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Computes the gradient for the inverse of `x` wrt its input.
 //
 // Specifically, `grad = -dy * y*y`, where `y = 1/x`, and `dy`
@@ -30461,6 +31928,71 @@ func BoostedTreesQuantileStreamResourceAddSummaries(scope *Scope, quantile_strea
 	return scope.AddOperation(opspec)
 }
 
+// Creates a Dataset that returns pseudorandom numbers.
+//
+// Arguments:
+//	seed: A scalar seed for the random number generator. If either seed or
+// seed2 is set to be non-zero, the random number generator is seeded
+// by the given seed.  Otherwise, a random seed is used.
+//	seed2: A second scalar seed to avoid seed collision.
+//
+//
+func ExperimentalRandomDataset(scope *Scope, seed tf.Output, seed2 tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalRandomDataset",
+		Input: []tf.Input{
+			seed, seed2,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// A dataset that splits the elements of its input into multiple elements.
+func ExperimentalUnbatchDataset(scope *Scope, input_dataset tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalUnbatchDataset",
+		Input: []tf.Input{
+			input_dataset,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Creates a dataset that overrides the maximum intra-op parallelism.
+//
+// Arguments:
+//
+//	max_intra_op_parallelism: Identifies the maximum intra-op parallelism to use.
+//
+//
+func ExperimentalMaxIntraOpParallelismDataset(scope *Scope, input_dataset tf.Output, max_intra_op_parallelism tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalMaxIntraOpParallelismDataset",
+		Input: []tf.Input{
+			input_dataset, max_intra_op_parallelism,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // StringSplitV2Attr is an optional argument to StringSplitV2.
 type StringSplitV2Attr func(optionalAttr)
 
@@ -30823,6 +32355,83 @@ func DeserializeIterator(scope *Scope, resource_handle tf.Output, serialized tf.
 	return scope.AddOperation(opspec)
 }
 
+// ResourceScatterNdSubAttr is an optional argument to ResourceScatterNdSub.
+type ResourceScatterNdSubAttr func(optionalAttr)
+
+// ResourceScatterNdSubUseLocking sets the optional use_locking attribute to value.
+//
+// value: An optional bool. Defaults to True. If True, the assignment will
+// be protected by a lock; otherwise the behavior is undefined,
+// but may exhibit less contention.
+// If not specified, defaults to true
+func ResourceScatterNdSubUseLocking(value bool) ResourceScatterNdSubAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
+	}
+}
+
+// Applies sparse subtraction to individual values or slices in a Variable.
+//
+// `ref` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`.
+//
+// `indices` must be integer tensor, containing indices into `ref`.
+// It must be shape `[d_0, ..., d_{Q-2}, K]` where `0 < K <= P`.
+//
+// The innermost dimension of `indices` (with length `K`) corresponds to
+// indices into elements (if `K = P`) or slices (if `K < P`) along the `K`th
+// dimension of `ref`.
+//
+// `updates` is `Tensor` of rank `Q-1+P-K` with shape:
+//
+// ```
+// [d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]]
+// ```
+//
+// For example, say we want to subtract 4 scattered elements from a rank-1 tensor
+// with 8 elements. In Python, that subtraction would look like this:
+//
+// ```python
+// ref = tf.Variable([1, 2, 3, 4, 5, 6, 7, 8], use_resource=True)
+// indices = tf.constant([[4], [3], [1], [7]])
+// updates = tf.constant([9, 10, 11, 12])
+// sub = tf.scatter_nd_sub(ref, indices, updates)
+// with tf.Session() as sess:
+//   print sess.run(sub)
+// ```
+//
+// The resulting update to ref would look like this:
+//
+//     [1, -9, 3, -6, -4, 6, 7, -4]
+//
+// See `tf.scatter_nd` for more details about how to make updates to
+// slices.
+//
+// Arguments:
+//	ref: A resource handle. Must be from a VarHandleOp.
+//	indices: A Tensor. Must be one of the following types: int32, int64.
+// A tensor of indices into ref.
+//	updates: A Tensor. Must have the same type as ref. A tensor of
+// values to add to ref.
+//
+// Returns the created operation.
+func ResourceScatterNdSub(scope *Scope, ref tf.Output, indices tf.Output, updates tf.Output, optional ...ResourceScatterNdSubAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceScatterNdSub",
+		Input: []tf.Input{
+			ref, indices, updates,
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
 // TensorArrayConcatV2Attr is an optional argument to TensorArrayConcatV2.
 type TensorArrayConcatV2Attr func(optionalAttr)
 
@@ -31032,6 +32641,43 @@ func TFRecordDataset(scope *Scope, filenames tf.Output, compression_type tf.Outp
 	return op.Output(0)
 }
 
+// ExperimentalStatsAggregatorHandleAttr is an optional argument to ExperimentalStatsAggregatorHandle.
+type ExperimentalStatsAggregatorHandleAttr func(optionalAttr)
+
+// ExperimentalStatsAggregatorHandleContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func ExperimentalStatsAggregatorHandleContainer(value string) ExperimentalStatsAggregatorHandleAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// ExperimentalStatsAggregatorHandleSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func ExperimentalStatsAggregatorHandleSharedName(value string) ExperimentalStatsAggregatorHandleAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Creates a statistics manager resource.
+func ExperimentalStatsAggregatorHandle(scope *Scope, optional ...ExperimentalStatsAggregatorHandleAttr) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalStatsAggregatorHandle",
+
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // A container for an iterator resource.
 //
 // Returns A handle to the iterator that can be passed to a "MakeIterator" or
@@ -31157,6 +32803,21 @@ func BatchToSpace(scope *Scope, input tf.Output, crops tf.Output, block_size int
 	return op.Output(0)
 }
 
+// Produces a summary of any statistics recorded by the given statistics manager.
+func ExperimentalStatsAggregatorSummary(scope *Scope, iterator tf.Output) (summary tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalStatsAggregatorSummary",
+		Input: []tf.Input{
+			iterator,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Makes a new iterator from the given `dataset` and stores it in `iterator`.
 //
 // This operation may be executed multiple times. Each execution will reset the
@@ -31488,6 +33149,26 @@ func FIFOQueueV2(scope *Scope, component_types []tf.DataType, optional ...FIFOQu
 	return op.Output(0)
 }
 
+// Deserializes a proto into the tree handle
+//
+// Arguments:
+//	tree_handle: Handle to the tree resource to be restored.
+//	tree_config: Serialied proto string of the boosted_trees.Tree proto.
+//
+// Returns the created operation.
+func TensorForestTreeDeserialize(scope *Scope, tree_handle tf.Output, tree_config tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorForestTreeDeserialize",
+		Input: []tf.Input{
+			tree_handle, tree_config,
+		},
+	}
+	return scope.AddOperation(opspec)
+}
+
 // Constructs an Optional variant from a tuple of tensors.
 func OptionalFromValue(scope *Scope, components []tf.Output) (optional tf.Output) {
 	if scope.Err() != nil {
@@ -31705,9 +33386,9 @@ func IteratorGetNextAsOptional(scope *Scope, iterator tf.Output, output_types []
 // dimension of `input`.
 //
 // Arguments:
-//	input: A complex64 tensor.
+//	input: A complex tensor.
 //
-// Returns A complex64 tensor of the same shape as `input`. The inner-most
+// Returns A complex tensor of the same shape as `input`. The inner-most
 //   dimension of `input` is replaced with its 1D Fourier transform.
 //
 // @compatibility(numpy)
@@ -32418,6 +34099,28 @@ func Merge(scope *Scope, inputs []tf.Output) (output tf.Output, value_index tf.O
 	return op.Output(0), op.Output(1)
 }
 
+// Writes the given dataset to the given file using the TFRecord format.
+//
+// Arguments:
+//	input_dataset: A variant tensor representing the dataset to write.
+//	filename: A scalar string tensor representing the filename to use.
+//	compression_type: A scalar string tensor containing either (i) the empty string (no
+// compression), (ii) "ZLIB", or (iii) "GZIP".
+//
+// Returns the created operation.
+func ExperimentalDatasetToTFRecord(scope *Scope, input_dataset tf.Output, filename tf.Output, compression_type tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalDatasetToTFRecord",
+		Input: []tf.Input{
+			input_dataset, filename, compression_type,
+		},
+	}
+	return scope.AddOperation(opspec)
+}
+
 // QueueCloseV2Attr is an optional argument to QueueCloseV2.
 type QueueCloseV2Attr func(optionalAttr)
 
@@ -32754,6 +34457,23 @@ func Rpc(scope *Scope, address tf.Output, method tf.Output, request tf.Output, o
 	return op.Output(0)
 }
 
+// Records the bytes size of each element of `input_dataset` in a StatsAggregator.
+func ExperimentalBytesProducedStatsDataset(scope *Scope, input_dataset tf.Output, tag tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalBytesProducedStatsDataset",
+		Input: []tf.Input{
+			input_dataset, tag,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // StackPushV2Attr is an optional argument to StackPushV2.
 type StackPushV2Attr func(optionalAttr)
 
-- 
GitLab


From 55fda7fb66f1774ac5271e523259faa7d2a44d0c Mon Sep 17 00:00:00 2001
From: Davide Libenzi <dlibenzi@google.com>
Date: Fri, 4 Jan 2019 15:31:48 -0800
Subject: [PATCH 0219/2345] Allow XLA input/output buffer aliasing to work via
 XRT.

PiperOrigin-RevId: 227924255
---
 .../xla/service/cpu/cpu_executable.cc         |  29 +++-
 .../xla/service/gpu/gpu_executable.cc         |  26 +++-
 tensorflow/compiler/xrt/kernels/BUILD         |   1 +
 .../compiler/xrt/kernels/xrt_execute_op.cc    |  17 +++
 tensorflow/compiler/xrt/tests/raw_api_test.cc | 127 +++++++++++++++++-
 tensorflow/compiler/xrt/xrt_state.cc          |  43 +++++-
 tensorflow/compiler/xrt/xrt_state.h           |  23 +++-
 7 files changed, 249 insertions(+), 17 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/cpu_executable.cc b/tensorflow/compiler/xla/service/cpu/cpu_executable.cc
index 412c2715b9..23d0af3423 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_executable.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_executable.cc
@@ -213,6 +213,8 @@ StatusOr<ScopedShapedBuffer> CpuExecutable::CreateResultShapedBuffer(
       /*on_host_shape=*/result_shape(),
       /*on_device_shape=*/result_shape(), run_options->allocator(),
       stream->parent()->device_ordinal());
+  const HloInputOutputAliasConfig& input_output_alias =
+      module().input_output_alias_config();
 
   // Move OwningDeviceMemory values which contain the array(s) of the result
   // into the respective location in ScopedShapedBuffer which is returned to the
@@ -232,12 +234,31 @@ StatusOr<ScopedShapedBuffer> CpuExecutable::CreateResultShapedBuffer(
         TF_ASSIGN_OR_RETURN(
             const BufferAllocation::Slice slice,
             this->assignment_->GetUniqueSlice(src, buffer_source->index()));
-        CHECK(!slice.allocation()->is_entry_computation_parameter());
-
         const BufferAllocation::Index buffer_index = slice.index();
         OwningDeviceMemory& buffer = buffers[buffer_index];
-        CHECK(!buffer.is_null() || buffer.size() == 0);
-        *device_memory = buffer.Forget();
+        if (!slice.allocation()->is_entry_computation_parameter()) {
+          // If the buffer coming out of the result is from a parameter, the
+          // owning buffer will be null, and that means the caller aliased some
+          // parameter buffer to an output one (via the
+          // HloInputOutputAliasConfig API). If that is the case, the caller
+          // will receive a partially complete scoped shaped buffer, which they
+          // will have to fill up on return. Unfortunately the interface to the
+          // execute APIs are ShapedBuffer pointer based, which assumes caller
+          // ownership, and hence a buffer coming from there cannot be part of
+          // the new ScopedShapedBuffer we create for the result (which assumes
+          // ownership).
+          *device_memory = buffer.Forget();
+        } else {
+          auto output_alias = input_output_alias.GetAliasedOutput(
+              slice.allocation()->parameter_number(),
+              slice.allocation()->param_shape_index());
+          CHECK(output_alias)
+              << "Ouput buffer is coming from parameter "
+              << slice.allocation()->parameter_number() << " at index "
+              << slice.allocation()->param_shape_index()
+              << ", but no alias exists";
+          CHECK_EQ(*output_alias, index);
+        }
         return Status::OK();
       }));
   return std::move(result_buffer);
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_executable.cc b/tensorflow/compiler/xla/service/gpu/gpu_executable.cc
index 128cecd765..434060ad89 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_executable.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_executable.cc
@@ -310,12 +310,34 @@ StatusOr<ScopedShapedBuffer> GpuExecutable::ExecuteOnStream(
         TF_ASSIGN_OR_RETURN(
             const BufferAllocation::Slice slice,
             this->assignment_->GetUniqueSlice(src_hlo, sources[0]->index()));
-        CHECK(!slice.allocation()->is_entry_computation_parameter());
 
         se::DeviceMemoryBase src_base =
             buffer_allocations->GetDeviceAddress(slice.index());
         CHECK(!src_base.is_null() || src_base.size() == 0);
-        *device_memory = src_base;
+        if (!slice.allocation()->is_entry_computation_parameter()) {
+          // If the buffer coming out of the result is from a parameter, it
+          // means the caller aliased some parameter buffer to an output one
+          // (via the HloInputOutputAliasConfig API). If that is the case, the
+          // caller will receive a partially complete scoped shaped buffer,
+          // which they will have to fill up on return.
+          // Unfortunately the interface to the execute APIs are ShapedBuffer
+          // pointer based, which assumes caller ownership, and hence a buffer
+          // coming from there cannot be part of the new ScopedShapedBuffer we
+          // create for the result (which assumes ownership).
+          *device_memory = src_base;
+        } else {
+          const HloInputOutputAliasConfig& input_output_alias =
+              module().input_output_alias_config();
+          auto output_alias = input_output_alias.GetAliasedOutput(
+              slice.allocation()->parameter_number(),
+              slice.allocation()->param_shape_index());
+          CHECK(output_alias)
+              << "Ouput buffer is coming from parameter "
+              << slice.allocation()->parameter_number() << " at index "
+              << slice.allocation()->param_shape_index()
+              << ", but no alias exists";
+          CHECK_EQ(*output_alias, index);
+        }
         buffers_in_result.insert(src_base);
         return Status::OK();
       }));
diff --git a/tensorflow/compiler/xrt/kernels/BUILD b/tensorflow/compiler/xrt/kernels/BUILD
index 67f475846e..c44769dfe3 100644
--- a/tensorflow/compiler/xrt/kernels/BUILD
+++ b/tensorflow/compiler/xrt/kernels/BUILD
@@ -55,6 +55,7 @@ cc_library(
         "//tensorflow/compiler/xla/client:xla_computation",
         "//tensorflow/compiler/xla/service:compiler",
         "//tensorflow/compiler/xla/service:computation_placer",
+        "//tensorflow/compiler/xla/service:hlo",
         "//tensorflow/compiler/xrt:xrt_proto",
         "//tensorflow/compiler/xrt:xrt_utils",
         "//tensorflow/core:core_cpu_internal",
diff --git a/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc b/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc
index 7f0ac123a5..7544541ca4 100644
--- a/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc
+++ b/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc
@@ -19,6 +19,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/service/computation_placer.h"
+#include "tensorflow/compiler/xla/service/hlo_input_output_alias_config.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/statusor.h"
@@ -228,6 +229,22 @@ Status XRTExecuteOp::DoWork(OpKernelContext* context) {
   TF_RETURN_IF_ERROR(XRTTupleAllocation::CreateFromBuffer(
       shaped_buffer, device_ref.backend(), device_ref.device_ordinal(),
       &output_tuple));
+
+  // The ScopedShapedBuffer returned by the executable Run() API, in case of
+  // input/output buffer aliasing, might have holes in it, which need to be
+  // filled using the proper input tuples buffers which are the source of
+  // aliasing.
+  const xla::HloInputOutputAliasConfig& input_output_alias =
+      executable->executable()->module().input_output_alias_config();
+  auto alias_function = [&](const xla::ShapeIndex& output_index,
+                            int64 param_number,
+                            const xla::ShapeIndex& param_index) -> Status {
+    TF_RET_CHECK(param_number < input_tuples.size());
+    return output_tuple->AliasBufferFrom(*input_tuples[param_number],
+                                         param_index, output_index);
+  };
+  TF_RETURN_IF_ERROR(input_output_alias.ForEachAliasWithStatus(alias_function));
+
   if (config_proto.return_exploded_tuple() &&
       output_tuple->on_device_shape().IsTuple()) {
     int64 tuple_element_count =
diff --git a/tensorflow/compiler/xrt/tests/raw_api_test.cc b/tensorflow/compiler/xrt/tests/raw_api_test.cc
index c8479cb778..f81faf0e61 100644
--- a/tensorflow/compiler/xrt/tests/raw_api_test.cc
+++ b/tensorflow/compiler/xrt/tests/raw_api_test.cc
@@ -96,14 +96,21 @@ xla::LiteralProto FloatMatrix(
   return array.ToProto();
 }
 
+xla::Literal ReadOutputLiteral(const std::vector<Tensor>& outputs, size_t idx) {
+  xla::LiteralProto response;
+  CHECK(response.ParseFromString(outputs[idx].scalar<string>()()));
+  return xla::Literal::CreateFromProto(response).ValueOrDie();
+}
+
 bool CompareLiteralProtos(const xla::LiteralProto& a,
                           const xla::LiteralProto& b) {
   auto l_a = xla::Literal::CreateFromProto(a).ValueOrDie();
   auto l_b = xla::Literal::CreateFromProto(b).ValueOrDie();
   bool equal = l_a == l_b;
   if (!equal) {
-    LOG(INFO) << "LiteralProtos don't match: " << a.DebugString()
-              << " != " << b.DebugString();
+    LOG(INFO) << "LiteralProtos don't match:\n"
+              << a.DebugString() << "\n!=\n"
+              << b.DebugString();
   }
   return equal;
 }
@@ -113,8 +120,19 @@ bool CompareLiteralToLiteralProto(const xla::Literal& a,
   auto l_b = xla::Literal::CreateFromProto(b).ValueOrDie();
   bool equal = a == l_b;
   if (!equal) {
-    LOG(INFO) << "Literal and LiteralProto don't match "
-              << a.ToProto().DebugString() << " != " << b.DebugString();
+    LOG(INFO) << "Literal and LiteralProto don't match:\n"
+              << a.ToProto().DebugString() << "\n!=\n"
+              << b.DebugString();
+  }
+  return equal;
+}
+
+bool CompareLiterals(const xla::Literal& a, const xla::Literal& b) {
+  bool equal = a == b;
+  if (!equal) {
+    LOG(INFO) << "Literals don't match:\n"
+              << a.ToProto().DebugString() << "\n!=\n"
+              << b.ToProto().DebugString();
   }
   return equal;
 }
@@ -939,6 +957,107 @@ TEST(RawApiTest, LeakCompilationReference) {
   TF_EXPECT_OK(session.Run({c_handle.handle}, &outputs));
 }
 
+TEST(RawApiTest, CompileAndExecuteWithReusedBuffers) {
+  xla::Shape element_shape = xla::ShapeUtil::MakeShape(xla::F32, {2});
+  xla::Shape shape =
+      xla::ShapeUtil::MakeTupleShape({element_shape, element_shape});
+  xla::Shape return_shape = xla::ShapeUtil::MakeTupleShape(
+      {element_shape, element_shape, element_shape, element_shape});
+  xla::XlaBuilder builder("ReuseBuffer");
+  auto param = xla::Parameter(&builder, 0, shape, "param");
+  auto p0 = xla::GetTupleElement(param, 0);
+  auto p1 = xla::GetTupleElement(param, 1);
+  auto add = xla::Add(p0, p1);
+  auto sub = xla::Sub(p0, p1);
+  xla::Tuple(&builder, {add, sub, p0, p1});
+
+  // Flip the tuple literals in the input handle.
+  builder.SetUpAlias({1}, 0, {0});
+  builder.SetUpAlias({0}, 0, {1});
+
+  auto computation = builder.Build().ValueOrDie();
+
+  auto literal0 = xla::LiteralUtil::CreateR1<float>({1.0f, 2.0f});
+  auto literal1 = xla::LiteralUtil::CreateR1<float>({5.0f, 9.0f});
+  auto literal = xla::LiteralUtil::MakeTuple({&literal0, &literal1});
+
+  xrt::XLAAllocation param_alloc;
+  *param_alloc.mutable_value() = literal.ToProto();
+
+  xrt::XLAComputation c;
+  auto config = c.mutable_config();
+  auto shapes = config->mutable_program_shape();
+  *shapes->add_parameters() = shape.ToProto();
+  *shapes->mutable_result() = return_shape.ToProto();
+  StoreComputationSnapshot(computation, c.mutable_hlo_snapshot());
+
+  xrt::XRTExecutionConfig e;
+  e.set_release_input_handles(false);
+  e.set_release_compilation_handle(true);
+
+  Scope root = Scope::NewRootScope().WithDevice(DeviceFromFlag());
+  ClientSession session(root);
+  auto e_config =
+      ops::Const(root.WithDevice("/device:CPU:0"), e.SerializeAsString());
+  auto c_data =
+      ops::Const(root.WithDevice("/device:CPU:0"), c.SerializeAsString());
+  auto c_handle = ops::XRTCompile(root, c_data);
+  auto param_value = ops::Const(root.WithDevice("/device:CPU:0"),
+                                param_alloc.SerializeAsString());
+  auto param_handle = ops::XRTAllocate(root, param_value);
+  TF_ASSERT_OK(root.status());
+
+  std::vector<Tensor> outputs;
+  TF_EXPECT_OK(session.Run({param_handle}, &outputs));
+
+  int64 alloc_handle = outputs[0].scalar<int64>()();
+
+  // Note that we release the result handle immediately, but since we aliased
+  // the output buffers onto the input allocation ones (held in alloc_handle),
+  // we can fetch the result from there.
+  auto result =
+      ops::XRTExecute(root, c_handle.handle, e_config, {Input(alloc_handle)});
+  auto read_back = ops::XRTReadLiteral(root, result);
+  auto release = ops::XRTReleaseAllocationHandle(
+      root.WithControlDependencies(read_back), result);
+  TF_ASSERT_OK(root.status());
+
+  outputs.clear();
+  TF_EXPECT_OK(session.Run(tensorflow::ClientSession::FeedType(), {read_back},
+                           {release}, &outputs));
+
+  xla::Literal exec_literal = ReadOutputLiteral(outputs, 0);
+  auto exec_literal_parts = exec_literal.DecomposeTuple();
+  ASSERT_EQ(exec_literal_parts.size(), 4);
+
+  EXPECT_TRUE(CompareLiterals(exec_literal_parts[2], literal0));
+  EXPECT_TRUE(CompareLiterals(exec_literal_parts[3], literal1));
+
+  // Now we read back the original input handle values, which at this point
+  // should contain the result of the XLA computation.
+  auto read_handle = ops::XRTReadLiteral(root, Input(alloc_handle));
+  TF_ASSERT_OK(root.status());
+  auto release_handle = ops::XRTReleaseAllocationHandle(
+      root.WithControlDependencies(read_handle), Input(alloc_handle));
+  TF_ASSERT_OK(root.status());
+
+  outputs.clear();
+  TF_EXPECT_OK(session.Run(tensorflow::ClientSession::FeedType(), {read_handle},
+                           {release_handle}, &outputs));
+
+  xla::Literal return_literal = ReadOutputLiteral(outputs, 0);
+
+  auto expected_literal0 = xla::LiteralUtil::CreateR1<float>({6.0f, 11.0f});
+  auto expected_literal1 = xla::LiteralUtil::CreateR1<float>({-4.0f, -7.0f});
+  // The first element of the computation returned tuple would be the add
+  // (expected_literal0), but since we flipped the buffers, the sub
+  // (expected_literal1) should come first.
+  auto expected_literal =
+      xla::LiteralUtil::MakeTuple({&expected_literal1, &expected_literal0});
+
+  EXPECT_TRUE(CompareLiterals(return_literal, expected_literal));
+}
+
 TEST(RawApiTest, CompileAndExecuteWithS64Argument) {
   xrt::XLAAllocation p0;
   *p0.mutable_value() = xla::LiteralUtil::CreateR0<int64>(11031965).ToProto();
diff --git a/tensorflow/compiler/xrt/xrt_state.cc b/tensorflow/compiler/xrt/xrt_state.cc
index 343460ff10..13c275aaca 100644
--- a/tensorflow/compiler/xrt/xrt_state.cc
+++ b/tensorflow/compiler/xrt/xrt_state.cc
@@ -133,7 +133,8 @@ Status AllocateScopedShapedBuffer(
 XRTBufferAllocation::XRTBufferAllocation(const se::DeviceMemoryBase& allocation,
                                          int device_ordinal,
                                          xla::DeviceMemoryAllocator* allocator)
-    : allocation_(allocation),
+    : size_(allocation.size()),
+      allocation_(allocation),
       device_ordinal_(device_ordinal),
       allocator_(allocator) {
   if (VLOG_IS_ON(2)) {
@@ -223,8 +224,19 @@ Status XRTTupleAllocation::ToLiteral(xla::Backend* backend, int device_ordinal,
                                      xla::Literal* literal) {
   auto transfer_manager = backend->transfer_manager();
   TF_ASSIGN_OR_RETURN(auto stream, backend->BorrowStream(device_ordinal));
+
+  // Validate the allocation buffers as if nulls gets to
+  // TransferLiteralFromDevice() a CHECK is issued.
+  xla::ShapedBuffer shaped_buffer = ToShapedBuffer();
+  for (auto& index_buffer : shaped_buffer.buffers()) {
+    if (index_buffer.second.is_null()) {
+      return errors::InvalidArgument("Literal buffer at index ",
+                                     index_buffer.first.ToString(),
+                                     " has been released");
+    }
+  }
   TF_ASSIGN_OR_RETURN(*literal, transfer_manager->TransferLiteralFromDevice(
-                                    stream.get(), ToShapedBuffer()));
+                                    stream.get(), shaped_buffer));
   return Status::OK();
 }
 
@@ -505,11 +517,34 @@ xla::ShapedBuffer XRTTupleAllocation::ToShapedBuffer() {
   return shaped_buffer;
 }
 
+Status XRTTupleAllocation::AliasBufferFrom(const XRTTupleAllocation& source,
+                                           const xla::ShapeIndex& source_index,
+                                           const xla::ShapeIndex& dest_index) {
+  XRTBufferAllocation* source_buffer = source.buffers_.element(source_index);
+  XRTBufferAllocation* dest_buffer = buffers_.element(dest_index);
+  // We allow the destination size being zero, because there are cases where we
+  // are coming in later filling in null/uninitialized device buffers.
+  // In all other cases, the size of the new buffer must match.
+  if (source_buffer->size() != dest_buffer->size() &&
+      dest_buffer->size() != 0) {
+    return errors::InvalidArgument(
+        "Source buffer at index ", source_index.ToString(),
+        " does not match the size of destination buffer at index ",
+        dest_index.ToString(), ": ", source_buffer->size(), " vs ",
+        dest_buffer->size());
+  }
+  *buffers_.mutable_element(dest_index) = source_buffer;
+  source_buffer->Ref();
+  dest_buffer->Unref();
+  return Status::OK();
+}
+
 xla::ShapeTree<xla::MaybeOwningDeviceMemory>
-XRTTupleAllocation::ToDeviceMemoryTree(bool release) {
+XRTTupleAllocation::ToDeviceMemoryTree(
+    const std::function<bool(const xla::ShapeIndex&)>& release_checker) {
   xla::ShapeTree<xla::MaybeOwningDeviceMemory> shaped_tree(on_device_shape());
   for (const auto& buffer : buffers_) {
-    if (!release) {
+    if (!release_checker(buffer.first)) {
       *shaped_tree.mutable_element(buffer.first) = buffer.second->allocation();
     } else {
       *shaped_tree.mutable_element(buffer.first) = xla::OwningDeviceMemory(
diff --git a/tensorflow/compiler/xrt/xrt_state.h b/tensorflow/compiler/xrt/xrt_state.h
index 3e3d502412..ac4be3a064 100644
--- a/tensorflow/compiler/xrt/xrt_state.h
+++ b/tensorflow/compiler/xrt/xrt_state.h
@@ -18,6 +18,7 @@ limitations under the License.
 #ifndef TENSORFLOW_COMPILER_XRT_XRT_STATE_H_
 #define TENSORFLOW_COMPILER_XRT_XRT_STATE_H_
 
+#include <functional>
 #include <memory>
 #include <string>
 #include <vector>
@@ -58,7 +59,14 @@ class XRTBufferAllocation : public core::RefCounted {
   // freed when the reference count drops to zero.
   void DiscardAllocation();
 
+  // Returns the expected size of the allocation. Since DiscardAllocation() will
+  // set allocation_ to {null,0}, and since later we might want to replace the
+  // discarded buffer with a new one, we need to be able to verify the size
+  // compatibility.
+  uint64 size() const { return size_; }
+
  private:
+  uint64 size_ = 0;
   se::DeviceMemoryBase allocation_;
   int device_ordinal_;
   xla::DeviceMemoryAllocator* allocator_;
@@ -168,9 +176,18 @@ class XRTTupleAllocation : public ResourceBase {
   // the same shape as on_host_shape.
   xla::ShapedBuffer ToShapedBuffer();
 
-  // Returns the device memory tree of this allocation. If 'release' is set, the
-  // ownership of the device memory is transferred to the result.
-  xla::ShapeTree<xla::MaybeOwningDeviceMemory> ToDeviceMemoryTree(bool release);
+  // Aliases the source buffer at source_index into the current tuple allocation
+  // dest_index.
+  Status AliasBufferFrom(const XRTTupleAllocation& source,
+                         const xla::ShapeIndex& source_index,
+                         const xla::ShapeIndex& dest_index);
+
+  // Returns the device memory tree of this allocation. If the release_checker
+  // function returns true for a given index, the ownership of the device memory
+  // at that index is transferred to the result. Every attempt to read the value
+  // at that index will fail.
+  xla::ShapeTree<xla::MaybeOwningDeviceMemory> ToDeviceMemoryTree(
+      const std::function<bool(const xla::ShapeIndex&)>& release_checker);
 
   string DebugString() override { return "XLA allocation handle"; }
 
-- 
GitLab


From 02bfac424857fa45621efab5908a31ffd4ce799b Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Fri, 4 Jan 2019 15:32:34 -0800
Subject: [PATCH 0220/2345] Add kullback leibler divergence v2 loss and metric.

PiperOrigin-RevId: 227924402
---
 tensorflow/python/keras/losses.py       | 27 +++++++++
 tensorflow/python/keras/losses_test.py  | 80 +++++++++++++++++++++++++
 tensorflow/python/keras/metrics.py      | 32 ++++++++++
 tensorflow/python/keras/metrics_test.py | 47 +++++++++++++++
 4 files changed, 186 insertions(+)

diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index 46e729c197..de1605a09c 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -581,6 +581,33 @@ class Logcosh(Loss):
     return logcosh(y_true, y_pred)
 
 
+class KullbackLeiblerDivergence(Loss):
+  """Computes kullback leibler divergence loss between `y_true` and `y_pred`.
+
+  loss = y_true * log(y_true / y_pred)
+
+  Usage:
+
+  ```python
+  k = tf.losses.KullbackLeiblerDivergence()
+  loss = k([.4, .9, .2], [.5, .8, .12])
+  print('Loss: ', loss.numpy())  # Loss: -0.043
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', loss=tf.losses.KullbackLeiblerDivergence())
+  ```
+  """
+
+  def call(self, y_true, y_pred):
+    y_pred = ops.convert_to_tensor(y_pred)
+    y_true = math_ops.cast(y_true, y_pred.dtype)
+    return kullback_leibler_divergence(y_true, y_pred)
+
+
 @keras_export('keras.metrics.mean_squared_error',
               'keras.metrics.mse',
               'keras.metrics.MSE',
diff --git a/tensorflow/python/keras/losses_test.py b/tensorflow/python/keras/losses_test.py
index f34a6caa18..caa4ff4c2e 100644
--- a/tensorflow/python/keras/losses_test.py
+++ b/tensorflow/python/keras/losses_test.py
@@ -1256,5 +1256,85 @@ class PoissonTest(test.TestCase):
     self.assertAlmostEqual(self.evaluate(loss), 0., 3)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class KullbackLeiblerDivergenceTest(test.TestCase):
+
+  def setup(self):
+    self.np_y_pred = np.asarray([.4, .9, .12, .36, .3, .4]).reshape((2, 3))
+    self.np_y_true = np.asarray([.5, .8, .12, .7, .43, .8]).reshape((2, 3))
+
+    self.batch_size = 2
+    self.expected_losses = np.multiply(self.np_y_true,
+                                       np.log(self.np_y_true / self.np_y_pred))
+
+    self.y_pred = constant_op.constant(self.np_y_pred, dtype=dtypes.float32)
+    self.y_true = constant_op.constant(self.np_y_true)
+
+  def test_config(self):
+    k_obj = keras.losses.KullbackLeiblerDivergence(
+        reduction=losses_impl.ReductionV2.SUM, name='kld')
+    self.assertEqual(k_obj.name, 'kld')
+    self.assertEqual(k_obj.reduction, losses_impl.ReductionV2.SUM)
+
+  def test_unweighted(self):
+    self.setup()
+    k_obj = keras.losses.KullbackLeiblerDivergence()
+
+    loss = k_obj(self.y_true, self.y_pred)
+    expected_loss = np.sum(self.expected_losses) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+
+  def test_scalar_weighted(self):
+    self.setup()
+    k_obj = keras.losses.KullbackLeiblerDivergence()
+    sample_weight = 2.3
+
+    loss = k_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    expected_loss = sample_weight * np.sum(
+        self.expected_losses) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+
+    # Verify we get the same output when the same input is given
+    loss_2 = k_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), self.evaluate(loss_2), 3)
+
+  def test_sample_weighted(self):
+    self.setup()
+    k_obj = keras.losses.KullbackLeiblerDivergence()
+    sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
+    loss = k_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+
+    expected_loss = np.multiply(
+        self.expected_losses,
+        np.asarray([1.2, 1.2, 1.2, 3.4, 3.4, 3.4]).reshape(2, 3))
+    expected_loss = np.sum(expected_loss) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+
+  def test_timestep_weighted(self):
+    self.setup()
+    k_obj = keras.losses.KullbackLeiblerDivergence()
+    y_true = self.np_y_true.reshape(2, 3, 1)
+    y_pred = self.np_y_pred.reshape(2, 3, 1)
+    sample_weight = np.asarray([3, 6, 5, 0, 4, 2]).reshape(2, 3)
+    expected_losses = np.sum(
+        np.multiply(y_true, np.log(y_true / y_pred)), axis=-1)
+
+    y_pred = constant_op.constant(y_pred, dtype=dtypes.float32)
+    y_true = constant_op.constant(y_true)
+    loss = k_obj(
+        y_true, y_pred, sample_weight=constant_op.constant(sample_weight))
+
+    num_timesteps = 3
+    expected_loss = np.sum(expected_losses * sample_weight) / (
+        self.batch_size * num_timesteps)
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+
+  def test_zero_weighted(self):
+    self.setup()
+    k_obj = keras.losses.KullbackLeiblerDivergence()
+    loss = k_obj(self.y_true, self.y_pred, sample_weight=0)
+    self.assertAlmostEqual(self.evaluate(loss), 0., 3)
+
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 71568e3fc5..9a2e057780 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -1716,6 +1716,38 @@ class Poisson(MeanMetricWrapper):
     return super(Poisson, cls).from_config(config)
 
 
+class KullbackLeiblerDivergence(MeanMetricWrapper):
+  """Computes kullback leibler divergence metric between `y_true` and `y_pred`.
+
+  metric = y_true * log(y_true / y_pred)
+
+  Usage:
+
+  ```python
+  m = tf.keras.metrics.KullbackLeiblerDivergence()
+  m.update_state([.4, .9, .2], [.5, .8, .12])
+  print('Final result: ', m.result().numpy())  # Final result: -0.043
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', metrics=[tf.keras.metrics.KullbackLeiblerDivergence()])
+  ```
+  """
+
+  def __init__(self, name='kullback_leibler_divergence', dtype=None):
+    super(KullbackLeiblerDivergence, self).__init__(
+        kullback_leibler_divergence, name, dtype=dtype)
+
+  @classmethod
+  def from_config(cls, config):
+    if 'fn' in config:
+      config.pop('fn')
+    return super(KullbackLeiblerDivergence, cls).from_config(config)
+
+
 def accuracy(y_true, y_pred):
   y_pred.get_shape().assert_is_compatible_with(y_true.get_shape())
   if y_true.dtype != y_pred.dtype:
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index 88b763d569..a02b95bb46 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -1556,6 +1556,53 @@ class PoissonTest(test.TestCase):
     self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class KullbackLeiblerDivergenceTest(test.TestCase):
+
+  def setup(self):
+    y_pred = np.asarray([.4, .9, .12, .36, .3, .4]).reshape((2, 3))
+    y_true = np.asarray([.5, .8, .12, .7, .43, .8]).reshape((2, 3))
+
+    self.batch_size = 2
+    self.expected_results = np.multiply(y_true, np.log(y_true / y_pred))
+
+    self.y_pred = constant_op.constant(y_pred, dtype=dtypes.float32)
+    self.y_true = constant_op.constant(y_true)
+
+  def test_config(self):
+    k_obj = metrics.KullbackLeiblerDivergence(name='kld', dtype=dtypes.int32)
+    self.assertEqual(k_obj.name, 'kld')
+    self.assertEqual(k_obj._dtype, dtypes.int32)
+
+    k_obj2 = metrics.KullbackLeiblerDivergence.from_config(k_obj.get_config())
+    self.assertEqual(k_obj2.name, 'kld')
+    self.assertEqual(k_obj2._dtype, dtypes.int32)
+
+  def test_unweighted(self):
+    self.setup()
+    k_obj = metrics.KullbackLeiblerDivergence()
+    self.evaluate(variables.variables_initializer(k_obj.variables))
+
+    update_op = k_obj.update_state(self.y_true, self.y_pred)
+    self.evaluate(update_op)
+    result = k_obj.result()
+    expected_result = np.sum(self.expected_results) / self.batch_size
+    self.assertAllClose(result, expected_result, atol=1e-3)
+
+  def test_weighted(self):
+    self.setup()
+    k_obj = metrics.KullbackLeiblerDivergence()
+    self.evaluate(variables.variables_initializer(k_obj.variables))
+
+    sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
+    result = k_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+
+    sample_weight = np.asarray([1.2, 1.2, 1.2, 3.4, 3.4, 3.4]).reshape((2, 3))
+    expected_result = np.multiply(self.expected_results, sample_weight)
+    expected_result = np.sum(expected_result) / (1.2 + 3.4)
+    self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)
+
+
 def _get_model(compile_metrics):
   model_layers = [
       layers.Dense(3, activation='relu', kernel_initializer='ones'),
-- 
GitLab


From 984ace61cdef46145a63081151c7eeb8609c4133 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 15:45:38 -0800
Subject: [PATCH 0221/2345] Report the CUcontext in the cuda_driver log spew,
 not the CudaContext pointer. CUDA tools (like nvprof, cuda-memcheck) report
 the CUcontext, so this makes it possible to match them up during debugging.

PiperOrigin-RevId: 227926402
---
 .../stream_executor/cuda/cuda_driver.cc       | 28 ++++++++++---------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/cuda_driver.cc b/tensorflow/stream_executor/cuda/cuda_driver.cc
index b34d1f722e..ca7a717bdb 100644
--- a/tensorflow/stream_executor/cuda/cuda_driver.cc
+++ b/tensorflow/stream_executor/cuda/cuda_driver.cc
@@ -431,7 +431,8 @@ bool DeviceOptionsToContextFlags(const DeviceOptions &device_options,
     *context = CreatedContexts::Add(new_context);
     CHECK(*context != nullptr)
         << "success in this call must entail non-null result";
-    VLOG(2) << "created or reused context " << context << " for this thread";
+    VLOG(2) << "created or reused context " << new_context
+            << " for this thread";
     return port::Status::OK();
   }
 
@@ -769,13 +770,13 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   ScopedActivateContext activated{context};
   CUresult res = cuStreamCreate(out, 0);
   if (res != CUDA_SUCCESS) {
-    LOG(ERROR) << "could not allocate CUDA stream for context " << context
-               << ": " << ToString(res);
+    LOG(ERROR) << "could not allocate CUDA stream for context "
+               << context->context() << ": " << ToString(res);
     return false;
   }
 
   VLOG(2) << "successfully created stream " << *out << " for context "
-          << context << " on thread";
+          << context->context() << " on thread";
   return true;
 }
 
@@ -788,11 +789,11 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   ScopedActivateContext activated{context};
   CUresult res = cuStreamDestroy(*stream);
   if (res != CUDA_SUCCESS) {
-    LOG(ERROR) << "failed to destroy CUDA stream for context " << context
-               << ": " << ToString(res);
+    LOG(ERROR) << "failed to destroy CUDA stream for context "
+               << context->context() << ": " << ToString(res);
   } else {
     VLOG(2) << "successfully destroyed stream " << *stream << " for context "
-            << context;
+            << context->context();
     *stream = nullptr;
   }
 }
@@ -809,8 +810,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
     return nullptr;
   }
   void *ptr = reinterpret_cast<void *>(result);
-  VLOG(2) << "allocated " << ptr << " for context " << context << " of "
-          << bytes << " bytes";
+  VLOG(2) << "allocated " << ptr << " for context " << context->context()
+          << " of " << bytes << " bytes";
   return ptr;
 }
 
@@ -823,7 +824,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
     LOG(ERROR) << "failed to free device memory at " << location
                << "; result: " << ToString(res);
   } else {
-    VLOG(2) << "deallocated " << location << " for context " << context;
+    VLOG(2) << "deallocated " << location << " for context "
+            << context->context();
   }
 }
 
@@ -839,8 +841,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
     return nullptr;
   }
   void *ptr = reinterpret_cast<void *>(result);
-  VLOG(2) << "allocated " << ptr << " for context " << context << " of "
-          << bytes << " bytes in unified memory";
+  VLOG(2) << "allocated " << ptr << " for context " << context->context()
+          << " of " << bytes << " bytes in unified memory";
   return ptr;
 }
 
@@ -854,7 +856,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                << "; result: " << ToString(res);
   } else {
     VLOG(2) << "deallocated unified memory at " << location << " for context "
-            << context;
+            << context->context();
   }
 }
 
-- 
GitLab


From 57d501422c16dd73b391d19bcc1e203942cc6b24 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 15:55:26 -0800
Subject: [PATCH 0222/2345] Add Keras LSTM model to correctness test.

PiperOrigin-RevId: 227927675
---
 tensorflow/contrib/distribute/python/BUILD    |   3 +-
 .../contrib/distribute/python/combinations.py |  15 ++-
 .../python/keras_correctness_test.py          | 100 +++++++++++++++---
 3 files changed, 99 insertions(+), 19 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index de6b6f1f84..c620a0448e 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -670,11 +670,12 @@ py_library(
 
 cuda_py_test(
     name = "keras_correctness_test",
+    size = "medium",
     srcs = ["keras_correctness_test.py"],
     additional_deps = [
         ":keras_correctness_test_lib",
     ],
-    shard_count = 16,
+    shard_count = 20,
     tags = [
         "multi_and_single_gpu",
         "no_oss",  # TODO(b/117919883): Fix python error.
diff --git a/tensorflow/contrib/distribute/python/combinations.py b/tensorflow/contrib/distribute/python/combinations.py
index f6c4291659..8db0e9c316 100644
--- a/tensorflow/contrib/distribute/python/combinations.py
+++ b/tensorflow/contrib/distribute/python/combinations.py
@@ -321,11 +321,12 @@ class NamedDistribution(object):
     return self._required_tpu
 
 
-def _get_tpu_strategy_creator(steps_per_run):
+def _get_tpu_strategy_creator(steps_per_run, **kwargs):
   def _create_tpu_strategy():
     resolver = cluster_resolver.TPUClusterResolver("")
     tpu_lib.initialize_tpu_system(resolver)
-    strategy = tpu_lib.TPUStrategy(resolver, steps_per_run=steps_per_run)
+    strategy = tpu_lib.TPUStrategy(resolver,
+                                   steps_per_run=steps_per_run, **kwargs)
     return strategy
   return _create_tpu_strategy
 
@@ -344,7 +345,15 @@ tpu_strategy = NamedDistribution(
 tpu_strategy_one_step = NamedDistribution(
     "TPUOneStep", _get_tpu_strategy_creator(steps_per_run=1),
     required_tpu=True)
-
+# TODO(b/122327153): Remove below two NamedDistributions.
+tpu_strategy_loop_on_device = NamedDistribution(
+    "TPULoopOnDevice", _get_tpu_strategy_creator(
+        steps_per_run=2, _disable_training_loop_on_host=True),
+    required_tpu=True)
+tpu_strategy_one_step_loop_on_device = NamedDistribution(
+    "TPUOneStepLoopOnDevice", _get_tpu_strategy_creator(
+        steps_per_run=1, _disable_training_loop_on_host=True),
+    required_tpu=True)
 mirrored_strategy_with_one_cpu = NamedDistribution(
     "Mirrored1CPU",
     lambda: mirrored_lib.MirroredStrategy(["/cpu:0"]))
diff --git a/tensorflow/contrib/distribute/python/keras_correctness_test.py b/tensorflow/contrib/distribute/python/keras_correctness_test.py
index 3abdee2c0e..1c4519ef71 100644
--- a/tensorflow/contrib/distribute/python/keras_correctness_test.py
+++ b/tensorflow/contrib/distribute/python/keras_correctness_test.py
@@ -62,24 +62,41 @@ def all_strategy_combinations_with_graph_mode():
   return combinations.combine(distribution=all_strategies, mode=['graph'])
 
 
+def tpu_strategies_with_host_training_loop_disabled():
+  strategies = [s for s in all_strategies if not s.required_tpu]
+  strategies.append(combinations.tpu_strategy_loop_on_device)
+  strategies.append(combinations.tpu_strategy_one_step_loop_on_device)
+  return strategies
+
+
 def strategy_and_input_combinations():
   def cnn_model_with_batch_norm(**kwargs):
     return _create_cnn_model(with_batch_norm=True, **kwargs)
 
-  return (
-      combinations.times(
-          combinations.combine(distribution=all_strategies),
-          combinations.combine(mode=['graph', 'eager'],
-                               use_numpy=[True, False],
-                               use_validation_data=[True, False]),
-          combinations.combine(model_with_data=[
-              ModelWithData('dnn', _create_dnn_model, _dnn_training_data),
-              ModelWithData('cnn', _create_cnn_model, _cnn_training_data),
-              ModelWithData('cnn_batch_norm',
-                            cnn_model_with_batch_norm,
-                            _cnn_training_data,
-                            with_batch_norm=True),
-          ])))
+  combinations_without_embedding_model = combinations.times(
+      combinations.combine(
+          distribution=tpu_strategies_with_host_training_loop_disabled()),
+      combinations.combine(mode=['graph', 'eager'],
+                           use_numpy=[True, False],
+                           use_validation_data=[True, False]),
+      combinations.combine(model_with_data=
+                           [ModelWithData('lstm',
+                                          _create_lstm_model,
+                                          _lstm_training_data)]))
+
+  combinations_with_embedding_model = combinations.times(
+      combinations.combine(distribution=all_strategies),
+      combinations.combine(mode=['graph', 'eager'],
+                           use_numpy=[True, False],
+                           use_validation_data=[True, False]),
+      combinations.combine(model_with_data=[
+          ModelWithData('dnn', _create_dnn_model, _dnn_training_data),
+          ModelWithData('cnn', _create_cnn_model, _cnn_training_data),
+          ModelWithData('cnn_batch_norm', cnn_model_with_batch_norm,
+                        _cnn_training_data, with_batch_norm=True),]))
+
+  return (combinations_with_embedding_model +
+          combinations_without_embedding_model)
 
 
 class MaybeDistributionScope(object):
@@ -191,6 +208,59 @@ def _create_cnn_model(initial_weights=None, distribution=None,
   return model
 
 
+def _lstm_training_data(count=_GLOBAL_BATCH_SIZE * _EVAL_STEPS,
+                        min_words=10,
+                        max_words=20,
+                        max_word_id=99,
+                        num_classes=2):
+  distribution = []
+  for _ in range(num_classes):
+    dist = np.abs(np.random.randn(max_word_id))
+    dist /= np.sum(dist)
+    distribution.append(dist)
+
+  features = []
+  labels = []
+  for _ in range(count):
+    label = np.random.randint(0, num_classes, size=1)[0]
+    num_words = np.random.randint(min_words, max_words, size=1)[0]
+    word_ids = np.random.choice(
+        max_word_id, size=num_words, replace=True, p=distribution[label])
+    word_ids = word_ids
+    labels.append(label)
+    features.append(word_ids)
+
+  features = keras.preprocessing.sequence.pad_sequences(
+      features, maxlen=max_words)
+  x_train = np.asarray(features, dtype=np.float32)
+  y_train = np.asarray(labels, dtype=np.int32).reshape((count, 1))
+  x_predict = x_train
+  return x_train, y_train, x_predict
+
+
+def _create_lstm_model(max_words=20,
+                       initial_weights=None,
+                       distribution=None):
+  with MaybeDistributionScope(distribution):
+    word_ids = keras.layers.Input(
+        shape=(max_words,), dtype=np.int32, name='words')
+    word_embed = keras.layers.Embedding(input_dim=100, output_dim=10)(word_ids)
+    lstm_embed = keras.layers.LSTM(
+        units=8, return_sequences=False)(word_embed)
+
+    preds = keras.layers.Dense(2, activation='softmax')(lstm_embed)
+    model = keras.Model(inputs=[word_ids], outputs=[preds])
+
+    if initial_weights:
+      model.set_weights(initial_weights)
+
+    model.compile(
+        optimizer=gradient_descent.GradientDescentOptimizer(learning_rate=0.1),
+        loss='sparse_categorical_crossentropy',
+        metrics=['sparse_categorical_accuracy'])
+  return model
+
+
 def batch_wrapper(dataset, batch_size, distribution, repeat=None):
   if repeat:
     dataset = dataset.repeat(repeat)
@@ -243,7 +313,7 @@ def get_correctness_test_inputs(use_numpy, use_validation_data,
     }
   else:
     if len(x_train) < _GLOBAL_BATCH_SIZE * _EVAL_STEPS:
-      # Currently, we cannot detech the size of a dataset. So, the eval steps is
+      # Currently, we cannot detect the size of a dataset. So, the eval steps is
       # hard coded.
       raise ValueError('x_train must have at least '
                        '_GLOBAL_BATCH_SIZE * _EVAL_STEPS samples')
-- 
GitLab


From 571d0114eda553e2d1b5c9c71f77c2211b5914e3 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Fri, 4 Jan 2019 16:10:25 -0800
Subject: [PATCH 0223/2345] Internal Cleanup.

PiperOrigin-RevId: 227929845
---
 .../keras/optimizer_v2/optimizer_v2_test.py    | 18 ------------------
 tensorflow/python/keras/optimizers.py          |  5 +++--
 2 files changed, 3 insertions(+), 20 deletions(-)

diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
index 290bdcad14..58c8b1f343 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
@@ -27,7 +27,6 @@ import numpy as np
 from tensorflow.python import keras
 from tensorflow.python.eager import context
 from tensorflow.python.eager import def_function
-from tensorflow.python.eager import function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -262,23 +261,6 @@ class OptimizerTest(test.TestCase):
       self.evaluate(sgd.iterations.initializer)
       self.assertEqual(0, self.evaluate(sgd.iterations))
 
-  @test_util.run_in_graph_and_eager_modes
-  def testSerializationWithinDefun(self):
-    with self.cached_session():
-      sgd = gradient_descent.SGD(3.0)
-      var0 = resource_variable_ops.ResourceVariable([1.0, 2.0],
-                                                    dtype=dtypes.float32)
-      loss = lambda: 3 * var0
-      sgd.minimize(loss, [var0])
-
-      def serialize():
-        config = sgd.get_config()
-        gradient_descent.SGD.from_config(config)
-
-      compiled_serialize = function.defun(serialize)
-      with self.assertRaisesRegexp(RuntimeError, 'inside Tensorflow graph'):
-        compiled_serialize()
-
   @test_util.run_in_graph_and_eager_modes
   def testConfig(self):
     with self.cached_session():
diff --git a/tensorflow/python/keras/optimizers.py b/tensorflow/python/keras/optimizers.py
index 0ab52fd0a0..12168e5574 100644
--- a/tensorflow/python/keras/optimizers.py
+++ b/tensorflow/python/keras/optimizers.py
@@ -575,7 +575,7 @@ class Adamax(Optimizer):
 
   def get_updates(self, loss, params):
     grads = self.get_gradients(loss, params)
-    self.updates = [state_ops.assign_add(self.iterations, 1)]
+    self.updates = []
 
     lr = self.lr
     if self.initial_decay > 0:
@@ -583,7 +583,8 @@ class Adamax(Optimizer):
           1. / (1. + self.decay * math_ops.cast(self.iterations,
                                                 K.dtype(self.decay))))
 
-    t = math_ops.cast(self.iterations, K.floatx()) + 1
+    with ops.control_dependencies([state_ops.assign_add(self.iterations, 1)]):
+      t = math_ops.cast(self.iterations, K.floatx())
     lr_t = lr / (1. - math_ops.pow(self.beta_1, t))
 
     shapes = [K.int_shape(p) for p in params]
-- 
GitLab


From abdbe6ee3d81d22d70e0181cea2a3bb261f7c09f Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Fri, 4 Jan 2019 16:31:04 -0800
Subject: [PATCH 0224/2345] Add anonymous shared name GUID to avoid kernel
 cache

Fixes #19671
We generate unique shared names for variables in eager mode to avoid
unintended sharing. This results in a memory leak due to the kernel
cache expanding with each new shared_name encountered. To avoid this
we reserve a shared name to send in with the op. When the handle
creation code sees this shared name, it generates a unique name
internally and returns a fresh handle each time.

PiperOrigin-RevId: 227932460
---
 tensorflow/core/framework/resource_handle.cc  |  3 ++
 tensorflow/core/framework/resource_handle.h   |  4 +++
 tensorflow/core/framework/resource_mgr.cc     | 12 ++++++-
 tensorflow/core/framework/resource_mgr.h      | 33 ++++++++++++-------
 tensorflow/python/eager/context.py            | 21 ++++++++++++
 tensorflow/python/eager/def_function.py       |  9 ++---
 tensorflow/python/eager/memory_test.py        | 11 +++++++
 .../python/ops/resource_variable_ops.py       |  6 ++--
 8 files changed, 81 insertions(+), 18 deletions(-)

diff --git a/tensorflow/core/framework/resource_handle.cc b/tensorflow/core/framework/resource_handle.cc
index fc3a329b3b..a3c48d63d4 100644
--- a/tensorflow/core/framework/resource_handle.cc
+++ b/tensorflow/core/framework/resource_handle.cc
@@ -19,6 +19,9 @@ limitations under the License.
 
 namespace tensorflow {
 
+const char ResourceHandle::ANONYMOUS_NAME[] =
+    "cd2c89b7-88b7-44c8-ad83-06c2a9158347";
+
 ResourceHandle::ResourceHandle() {}
 
 ResourceHandle::ResourceHandle(const ResourceHandleProto& proto) {
diff --git a/tensorflow/core/framework/resource_handle.h b/tensorflow/core/framework/resource_handle.h
index db213669a3..c3beffb8ac 100644
--- a/tensorflow/core/framework/resource_handle.h
+++ b/tensorflow/core/framework/resource_handle.h
@@ -67,6 +67,10 @@ class ResourceHandle {
 
   string DebugString() const;
 
+  // GUID for anonymous resources. Resources with this shared_name will have
+  // their shared_name replaced with a GUID at creation time
+  static const char ANONYMOUS_NAME[];
+
  public:
   string device_;
   string container_;
diff --git a/tensorflow/core/framework/resource_mgr.cc b/tensorflow/core/framework/resource_mgr.cc
index 9f3204ab96..6a94ff6642 100644
--- a/tensorflow/core/framework/resource_mgr.cc
+++ b/tensorflow/core/framework/resource_mgr.cc
@@ -13,6 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include <atomic>
+
 #include "tensorflow/core/framework/resource_mgr.h"
 
 #include "tensorflow/core/framework/device_attributes.pb.h"
@@ -26,6 +28,10 @@ limitations under the License.
 #include "tensorflow/core/platform/demangle.h"
 
 namespace tensorflow {
+
+// Used to generate unique names for anonymous variables
+static std::atomic<int64> current_id_;
+
 ResourceHandle MakeResourceHandle(OpKernelContext* ctx, const string& container,
                                   const string& name,
                                   const TypeIndex& type_index) {
@@ -38,7 +44,11 @@ ResourceHandle MakeResourceHandle(OpKernelContext* ctx, const string& container,
     actual_container = ctx->resource_manager()->default_container();
   }
   result.set_container(actual_container);
-  result.set_name(name);
+  if (name == ResourceHandle::ANONYMOUS_NAME) {
+    result.set_name(strings::StrCat("_AnonymousVar", current_id_.fetch_add(1)));
+  } else {
+    result.set_name(name);
+  }
   result.set_hash_code(type_index.hash_code());
   result.set_maybe_type_name(type_index.name());
   return result;
diff --git a/tensorflow/core/framework/resource_mgr.h b/tensorflow/core/framework/resource_mgr.h
index 3195cd2e9d..8a7c25da92 100644
--- a/tensorflow/core/framework/resource_mgr.h
+++ b/tensorflow/core/framework/resource_mgr.h
@@ -619,20 +619,31 @@ ResourceHandleOp<T>::ResourceHandleOp(OpKernelConstruction* context)
 
 template <typename T>
 void ResourceHandleOp<T>::Compute(OpKernelContext* ctx) {
-  if (!initialized_.load()) {
-    mutex_lock ml(mutex_);
-    // Checking again to see if another thread has initialized the resource.
+  if (name_ == ResourceHandle::ANONYMOUS_NAME) {
+    AllocatorAttributes attr;
+    attr.set_on_host(true);
+    Tensor handle;
+    OP_REQUIRES_OK(
+        ctx, ctx->allocate_temp(DT_RESOURCE, TensorShape({}), &handle, attr));
+    handle.scalar<ResourceHandle>()() =
+        MakeResourceHandle<T>(ctx, container_, name_);
+    ctx->set_output(0, handle);
+  } else {
     if (!initialized_.load()) {
-      AllocatorAttributes attr;
-      attr.set_on_host(true);
-      OP_REQUIRES_OK(ctx, ctx->allocate_temp(DT_RESOURCE, TensorShape({}),
-                                             &resource_, attr));
-      resource_.scalar<ResourceHandle>()() =
-          MakeResourceHandle<T>(ctx, container_, name_);
-      initialized_.store(true);
+      mutex_lock ml(mutex_);
+      // Checking again to see if another thread has initialized the resource.
+      if (!initialized_.load()) {
+        AllocatorAttributes attr;
+        attr.set_on_host(true);
+        OP_REQUIRES_OK(ctx, ctx->allocate_temp(DT_RESOURCE, TensorShape({}),
+                                               &resource_, attr));
+        resource_.scalar<ResourceHandle>()() =
+            MakeResourceHandle<T>(ctx, container_, name_);
+        initialized_.store(true);
+      }
     }
+    ctx->set_output(0, resource_);
   }
-  ctx->set_output(0, resource_);
 }
 
 template <typename T>
diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py
index cd43dc7ab2..faaf742c26 100644
--- a/tensorflow/python/eager/context.py
+++ b/tensorflow/python/eager/context.py
@@ -788,6 +788,27 @@ def in_eager_mode():
   return executing_eagerly()
 
 
+def shared_name(name=None):
+  """Returns the anonymous shared name GUID if no shared name is specified.
+
+  In eager mode we need to use a unique shared name to avoid spurious sharing
+  issues. The runtime generates a unique name on our behalf when the reserved
+  GUID is used as a shared name.
+
+  Args:
+    name: Optional shared name
+
+  Returns:
+    Eager compatible shared name.
+  """
+  if name or not executing_eagerly():
+    return name
+
+  # Ensure a unique name when eager execution is enabled to avoid spurious
+  # sharing issues.
+  return "cd2c89b7-88b7-44c8-ad83-06c2a9158347"
+
+
 def graph_mode():
   """Context-manager to disable eager execution for the current thread."""
   return context()._mode(GRAPH_MODE)  # pylint: disable=protected-access
diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py
index ebc47d1566..aa4f20df49 100644
--- a/tensorflow/python/eager/def_function.py
+++ b/tensorflow/python/eager/def_function.py
@@ -130,8 +130,9 @@ class UnliftedInitializerVariable(resource_variable_ops.ResourceVariable):
                         if init_from_fn else [initial_value]) as name:
       # pylint: disable=protected-access
       with ops.init_scope():
-        shared_name = ops._name_from_scope_name(name)
-        shared_name = "%s_%d" % (shared_name, ops.uid())
+        handle_name = ops._name_from_scope_name(name)
+        unique_id = "%s_%d" % (handle_name, ops.uid())
+        shared_name = context.shared_name(unique_id)
       with ops.name_scope("Initializer"), ops.device(None):
         initial_value = ops.convert_to_tensor(
             initial_value() if init_from_fn else initial_value,
@@ -144,8 +145,8 @@ class UnliftedInitializerVariable(resource_variable_ops.ResourceVariable):
             name=name,
             graph_mode=self._in_graph_mode)
       self._shape = initial_value.shape
-      self._unique_id = shared_name
-      self._handle_name = shared_name + ":0"
+      self._unique_id = unique_id
+      self._handle_name = handle_name + ":0"
       self._dtype = initial_value.dtype.base_dtype
       self._constraint = constraint
       assert initial_value is not None
diff --git a/tensorflow/python/eager/memory_test.py b/tensorflow/python/eager/memory_test.py
index 5e4516239c..9d29180379 100644
--- a/tensorflow/python/eager/memory_test.py
+++ b/tensorflow/python/eager/memory_test.py
@@ -33,6 +33,7 @@ from tensorflow.python.eager import context
 from tensorflow.python.eager import test
 from tensorflow.python.framework import dtypes
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops.variables import Variable
 
 # memory_profiler might not be available in the OSS version of TensorFlow.
 try:
@@ -81,6 +82,16 @@ class MemoryTest(test.TestCase):
           "Maximum allowed increase: %f") % (initial, increase,
                                              increase_threshold_absolute_mb)
 
+  def testMemoryLeakAnonymousVariable(self):
+    if memory_profiler is None:
+      self.skipTest("memory_profiler required to run this test")
+
+    def f():
+      inputs = Variable(array_ops.zeros([32, 100], dtypes.float32))
+      del inputs
+
+    self.assertNotIncreasingMemory(f, num_iters=10000)
+
   def testMemoryLeakInSimpleModelForwardOnly(self):
     if memory_profiler is None:
       self.skipTest("memory_profiler required to run this test")
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index d2e5dd9cfe..1d98fb2c89 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -403,10 +403,12 @@ class ResourceVariable(variables.VariableV1):
         handle_name = ops._name_from_scope_name(name)
         if self._in_graph_mode:
           shared_name = handle_name
+          unique_id = shared_name
         else:
           # When in eager mode use a uid for the shared_name, to prevent
           # accidental sharing.
-          shared_name = "%s_%d" % (handle_name, ops.uid())
+          unique_id = "%s_%d" % (handle_name, ops.uid())
+          shared_name = context.shared_name()
         # Use attr_scope and device(None) to simulate the behavior of
         # colocate_with when the variable we want to colocate with doesn't
         # yet exist.
@@ -434,7 +436,7 @@ class ResourceVariable(variables.VariableV1):
               "variable inside a loop or conditional, use a lambda as the "
               "initializer." % name)
         # pylint: enable=protected-access
-        self._unique_id = shared_name
+        self._unique_id = unique_id
         self._initial_value = initial_value if self._in_graph_mode else None
         self._handle_name = handle_name + ":0"
         self._dtype = initial_value.dtype.base_dtype
-- 
GitLab


From 84bddea0109598e47fbe76ba1d443fdcb34c9bf4 Mon Sep 17 00:00:00 2001
From: Paul Donnelly <pauldonnelly@google.com>
Date: Fri, 4 Jan 2019 16:50:13 -0800
Subject: [PATCH 0225/2345] Improve Maxpool op to use cuDNN 7.3 improvements to
 cudnnPoolingForward()

PiperOrigin-RevId: 227934668
---
 tensorflow/core/kernels/maxpooling_op.cc      |  8 ++
 .../core/kernels/maxpooling_op_gpu.cu.cc      |  1 -
 tensorflow/core/kernels/pooling_ops_common.cc | 83 +++++++++++++++----
 tensorflow/stream_executor/cuda/cuda_dnn.cc   | 25 ++++++
 tensorflow/stream_executor/cuda/cuda_dnn.h    |  8 ++
 tensorflow/stream_executor/dnn.h              | 11 +++
 tensorflow/stream_executor/stream.cc          | 22 +++++
 tensorflow/stream_executor/stream.h           |  7 ++
 8 files changed, 149 insertions(+), 16 deletions(-)

diff --git a/tensorflow/core/kernels/maxpooling_op.cc b/tensorflow/core/kernels/maxpooling_op.cc
index 507fc99837..ab235843f7 100644
--- a/tensorflow/core/kernels/maxpooling_op.cc
+++ b/tensorflow/core/kernels/maxpooling_op.cc
@@ -41,6 +41,7 @@ limitations under the License.
 #include "tensorflow/core/util/use_cudnn.h"
 
 #if GOOGLE_CUDA
+#include "cuda/include/cudnn.h"
 #include "tensorflow/core/kernels/maxpooling_op_gpu.h"
 #include "tensorflow/core/kernels/pooling_ops_common_gpu.h"
 #include "tensorflow/core/platform/stream_executor.h"
@@ -1134,11 +1135,18 @@ class MaxPoolingNoMaskOp<GPUDevice, T> : public OpKernel {
                 errors::InvalidArgument(
                     "qint8 should be used with data_format NCHW_VECT_C."));
 
+#if CUDNN_VERSION >= 7300
+    if (use_dnn_) {
+      DnnPoolingOp<T>::Compute(context, se::dnn::PoolingMode::kMaximum, ksize_,
+                               stride_, padding_, data_format_, tensor_in,
+                               out_shape, propagate_nans_);
+#else
     // These is_int8x4 checks avoid linker errors for missing qint8 kernels.
     if (!is_int8x4 && use_dnn_ && data_format_ == FORMAT_NCHW) {
       DnnPoolingOp<T>::Compute(context, se::dnn::PoolingMode::kMaximum, ksize_,
                                stride_, padding_, data_format_, tensor_in,
                                out_shape, propagate_nans_);
+#endif
     } else {
       Tensor* output = nullptr;
       OP_REQUIRES_OK(context, context->allocate_output(0, out_shape, &output));
diff --git a/tensorflow/core/kernels/maxpooling_op_gpu.cu.cc b/tensorflow/core/kernels/maxpooling_op_gpu.cu.cc
index 56d0340547..f28811ffa4 100644
--- a/tensorflow/core/kernels/maxpooling_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/maxpooling_op_gpu.cu.cc
@@ -390,7 +390,6 @@ bool MaxPoolForwardNoMask_NCHW_VECT_C::operator()(
       0, d.stream()>>>(output_size, bottom_data, height, width, channels,
                        pooled_height, pooled_width, kernel_h, kernel_w,
                        stride_h, stride_w, pad_t, pad_l, top_data);
-  d.synchronize();
   return d.ok();
 }
 
diff --git a/tensorflow/core/kernels/pooling_ops_common.cc b/tensorflow/core/kernels/pooling_ops_common.cc
index e583f7feb4..69122f467c 100644
--- a/tensorflow/core/kernels/pooling_ops_common.cc
+++ b/tensorflow/core/kernels/pooling_ops_common.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor.h"
 
 #if GOOGLE_CUDA
+#include "cuda/include/cudnn.h"
 #include "tensorflow/core/kernels/conv_2d.h"
 #include "tensorflow/core/kernels/pooling_ops_common_gpu.h"
 #include "tensorflow/core/platform/stream_executor.h"
@@ -28,6 +29,20 @@ limitations under the License.
 
 namespace tensorflow {
 
+namespace {
+
+template <typename T>
+struct RawType {
+  using type = T;
+};
+
+template <>
+struct RawType<qint8> {
+  using type = int8;
+};
+
+}  // namespace
+
 PoolParameters::PoolParameters(OpKernelContext* context,
                                const std::vector<int32>& ksize,
                                const std::vector<int32>& stride,
@@ -156,7 +171,10 @@ void DnnPoolingOp<T>::Compute(OpKernelContext* context,
     return;
   }
 
-  /// For now, cudnn does not support NHWC format, so we need to convert it
+  int batch_size = params.tensor_in_batch;
+  int depth = params.depth;
+#if CUDNN_VERSION < 7300
+  /// Earlier versions do not support NHWC format, so we need to convert it
   /// to NCHW before calling cudnn. We need to get rid of this once it is done
   Tensor transformed_input;
   if (data_format == FORMAT_NHWC) {
@@ -181,7 +199,31 @@ void DnnPoolingOp<T>::Compute(OpKernelContext* context,
   } else {
     transformed_output = *tensor_out;
   }
-
+  se::dnn::DataLayout data_layout = se::dnn::DataLayout::kBatchDepthYX;
+#else
+  auto& transformed_input = tensor_in;
+  auto& transformed_output = *tensor_out;
+  se::dnn::DataLayout data_layout;
+  switch (data_format) {
+    case FORMAT_NHWC:
+      data_layout = se::dnn::DataLayout::kBatchYXDepth;
+      break;
+    case FORMAT_NCHW:
+      data_layout = se::dnn::DataLayout::kBatchDepthYX;
+      break;
+    case FORMAT_NCHW_VECT_C:
+      // NCHW_VECT_C is not supported by cudnnPoolingForward(), but can be
+      // emulated via NHWC.
+      data_layout = se::dnn::DataLayout::kBatchYXDepth;
+      batch_size *= depth;
+      depth = 4;
+      break;
+    default:
+      OP_REQUIRES(context, false,
+                  errors::InvalidArgument("Unsupported format: ",
+                                          ToString(data_format)));
+  }
+#endif
   /// Get ready to call cudnn
   se::dnn::PoolingDescriptor pooling_desc;
   pooling_desc.set_pooling_mode(pooling_mode)
@@ -194,23 +236,27 @@ void DnnPoolingOp<T>::Compute(OpKernelContext* context,
       .set_propagate_nans(propagate_nans);
 
   se::dnn::BatchDescriptor input_desc;
-  input_desc.set_count(params.tensor_in_batch)
+  input_desc.set_count(batch_size)
       .set_height(params.tensor_in_rows)
       .set_width(params.tensor_in_cols)
-      .set_feature_map_count(params.depth)
-      .set_layout(se::dnn::DataLayout::kBatchDepthYX);
+      .set_feature_map_count(depth)
+      .set_layout(data_layout);
 
   se::dnn::BatchDescriptor output_desc;
-  output_desc.set_count(params.tensor_in_batch)
+  output_desc.set_count(batch_size)
       .set_height(params.out_height)
       .set_width(params.out_width)
-      .set_feature_map_count(params.depth)
-      .set_layout(se::dnn::DataLayout::kBatchDepthYX);
+      .set_feature_map_count(depth)
+      .set_layout(data_layout);
+
+  auto input_data =
+      AsDeviceMemory(reinterpret_cast<const typename RawType<T>::type*>(
+                         transformed_input.template flat<T>().data()),
+                     transformed_input.template flat<T>().size());
 
-  auto input_data = AsDeviceMemory(transformed_input.template flat<T>().data(),
-                                   transformed_input.template flat<T>().size());
   auto output_data =
-      AsDeviceMemory(transformed_output.template flat<T>().data(),
+      AsDeviceMemory(reinterpret_cast<const typename RawType<T>::type*>(
+                         transformed_output.template flat<T>().data()),
                      transformed_output.template flat<T>().size());
 
   auto* stream = context->op_device_context()->stream();
@@ -222,15 +268,17 @@ void DnnPoolingOp<T>::Compute(OpKernelContext* context,
                     .ok();
   OP_REQUIRES(context, status,
               errors::Internal("cudnn PoolForward launch failed"));
-
+#if CUDNN_VERSION < 7300
   if (data_format == FORMAT_NHWC) {
     /// Transform the output data from NCHW back to NHWC
     auto toConstTensor = [](const Tensor& x) -> const Tensor { return x; };
-    functor::NCHWToNHWC<GPUDevice, T, 4>()(
+    using RT = typename RawType<T>::type;
+    functor::NCHWToNHWC<GPUDevice, RT, 4>()(
         context->eigen_device<Device>(),
-        toConstTensor(transformed_output).template tensor<T, 4>(),
-        tensor_out->tensor<T, 4>());
+        toConstTensor(transformed_output).template tensor<RT, 4>(),
+        tensor_out->tensor<RT, 4>());
   }
+#endif
 }
 
 template <typename T>
@@ -388,6 +436,11 @@ void DnnPoolingGradOp<T>::Compute(
   template class DnnPoolingOp<T>; \
   template class DnnPoolingGradOp<T>;
 TF_CALL_GPU_NUMBER_TYPES(DEFINE_DNN_OPS)
+
+#if CUDNN_VERSION >= 7300
+template class DnnPoolingOp<qint8>;
+#endif
+
 #undef DEFINE_DNN_OPS
 
 #endif  // GOOGLE_CUDA
diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index 01ff248150..a34aa9354d 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -4207,6 +4207,31 @@ bool CudnnSupport::DoPoolForward(
   return IsStatusOk(status, /*report_error=*/true);
 }
 
+bool CudnnSupport::DoPoolForward(
+    Stream* stream, const dnn::PoolingDescriptor& pooling_dimensions,
+    const dnn::BatchDescriptor& input_dimensions,
+    const DeviceMemory<int8>& input_data,
+    const dnn::BatchDescriptor& output_dimensions,
+    DeviceMemory<int8>* output_data, ScratchAllocator* workspace_allocator) {
+  // Alpha is the scaling factor for input.
+  float alpha = 1.0;
+  // Beta is the scaling factor for output.
+  float beta = 0.0;
+
+  CudnnTensorDescriptor src_desc(input_dimensions, CUDNN_DATA_INT8);
+  CudnnTensorDescriptor dest_desc(output_dimensions, CUDNN_DATA_INT8);
+  CudnnPoolingDescriptor pooling_desc(pooling_dimensions);
+
+  auto cudnn = cudnn_->GetHandle(parent_, stream);
+  auto status = [&] {
+    RETURN_IF_CUDNN_ERROR(cudnnPoolingForward(
+        cudnn.handle(), pooling_desc.handle(), &alpha, src_desc.handle(),
+        input_data.opaque(), &beta, dest_desc.handle(), output_data->opaque()));
+    return port::Status::OK();
+  }();
+  return IsStatusOk(status, /*report_error=*/true);
+}
+
 bool CudnnSupport::DoPoolBackward(
     Stream* stream, const dnn::PoolingDescriptor& pooling_dimensions,
     const dnn::BatchDescriptor& input_dimensions,
diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.h b/tensorflow/stream_executor/cuda/cuda_dnn.h
index 044ed54514..4cce3c5626 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.h
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.h
@@ -540,6 +540,14 @@ class CudnnSupport : public dnn::DnnSupport {
                      DeviceMemory<Eigen::half>* output_data,
                      ScratchAllocator* workspace_allocator) override;
 
+  bool DoPoolForward(Stream* stream,
+                     const dnn::PoolingDescriptor& pooling_dimensions,
+                     const dnn::BatchDescriptor& input_dimensions,
+                     const DeviceMemory<int8>& input_data,
+                     const dnn::BatchDescriptor& output_dimensions,
+                     DeviceMemory<int8>* output_data,
+                     ScratchAllocator* workspace_allocator) override;
+
   bool DoPoolBackward(Stream* stream,
                       const dnn::PoolingDescriptor& pooling_dimensions,
                       const dnn::BatchDescriptor& input_dimensions,
diff --git a/tensorflow/stream_executor/dnn.h b/tensorflow/stream_executor/dnn.h
index 1001824ed5..ff7b02cf6b 100644
--- a/tensorflow/stream_executor/dnn.h
+++ b/tensorflow/stream_executor/dnn.h
@@ -1617,6 +1617,17 @@ class DnnSupport {
     return false;
   }
 
+  virtual bool DoPoolForward(Stream* stream,
+                             const dnn::PoolingDescriptor& pooling_dimensions,
+                             const dnn::BatchDescriptor& input_dimensions,
+                             const DeviceMemory<int8>& input_data,
+                             const dnn::BatchDescriptor& output_dimensions,
+                             DeviceMemory<int8>* output_data,
+                             ScratchAllocator* workspace_allocator) {
+    LOG(FATAL) << "DoPoolForward not implemented for int8.";
+    return false;
+  }
+
   // Performs differentiation of the pooling operation.
   virtual bool DoPoolBackward(Stream* stream,
                               const dnn::PoolingDescriptor& pooling_dimensions,
diff --git a/tensorflow/stream_executor/stream.cc b/tensorflow/stream_executor/stream.cc
index 3edc66cde8..1b14e4c339 100644
--- a/tensorflow/stream_executor/stream.cc
+++ b/tensorflow/stream_executor/stream.cc
@@ -1490,6 +1490,28 @@ Stream &Stream::ThenPoolForward(
   return *this;
 }
 
+Stream &Stream::ThenPoolForward(
+    const dnn::PoolingDescriptor &pooling_dimensions,
+    const dnn::BatchDescriptor &input_dimensions,
+    const DeviceMemory<int8> &input_data,
+    const dnn::BatchDescriptor &output_dimensions,
+    DeviceMemory<int8> *output_data, ScratchAllocator *workspace_allocator) {
+  VLOG_CALL(PARAM(pooling_dimensions), PARAM(input_dimensions),
+            PARAM(input_data), PARAM(output_dimensions), PARAM(output_data),
+            PARAM(workspace_allocator));
+
+  if (ok()) {
+    if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
+      CheckError(dnn->DoPoolForward(this, pooling_dimensions, input_dimensions,
+                                    input_data, output_dimensions, output_data,
+                                    workspace_allocator));
+    } else {
+      SetErrorAndLogNoDnnSupport();
+    }
+  }
+  return *this;
+}
+
 Stream &Stream::ThenPoolBackward(
     const dnn::PoolingDescriptor &pooling_dimensions,
     const dnn::BatchDescriptor &input_dimensions,
diff --git a/tensorflow/stream_executor/stream.h b/tensorflow/stream_executor/stream.h
index 0fc90cf83d..d5e2fdf58d 100644
--- a/tensorflow/stream_executor/stream.h
+++ b/tensorflow/stream_executor/stream.h
@@ -650,6 +650,13 @@ class Stream {
                           DeviceMemory<Eigen::half> *output_data,
                           ScratchAllocator *workspace_allocator = nullptr);
 
+  Stream &ThenPoolForward(const dnn::PoolingDescriptor &pooling_dimensions,
+                          const dnn::BatchDescriptor &input_dimensions,
+                          const DeviceMemory<int8> &input_data,
+                          const dnn::BatchDescriptor &output_dimensions,
+                          DeviceMemory<int8> *output_data,
+                          ScratchAllocator *workspace_allocator = nullptr);
+
   Stream &ThenPoolBackward(const dnn::PoolingDescriptor &pooling_dimensions,
                            const dnn::BatchDescriptor &input_dimensions,
                            const DeviceMemory<double> &input_data,
-- 
GitLab


From 75f2e9c266ec68d202a11ea5514898ccd1735d86 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Fri, 4 Jan 2019 16:51:01 -0800
Subject: [PATCH 0226/2345] Automated rollback of commit
 7cc1ff9072ce4c2c7f36b7011e24486071ad13ee

PiperOrigin-RevId: 227934767
---
 tensorflow/python/framework/constant_op.py         |  4 ----
 tensorflow/python/kernel_tests/constant_op_test.py | 11 -----------
 2 files changed, 15 deletions(-)

diff --git a/tensorflow/python/framework/constant_op.py b/tensorflow/python/framework/constant_op.py
index 62a25f3fea..ade0797dcd 100644
--- a/tensorflow/python/framework/constant_op.py
+++ b/tensorflow/python/framework/constant_op.py
@@ -277,10 +277,6 @@ def _constant_impl(
                     (num_t, shape, shape.num_elements()))
   g = ops.get_default_graph()
   tensor_value = attr_value_pb2.AttrValue()
-  if tensor_util.is_tensor(value):
-    raise ValueError(
-        ("Cannot create a tf.constant from symbolic Tensor %s. Did you mean "
-         "tf.convert_to_tensor?") % (value,))
   tensor_value.tensor.CopyFrom(
       tensor_util.make_tensor_proto(
           value, dtype=dtype, shape=shape, verify_shape=verify_shape,
diff --git a/tensorflow/python/kernel_tests/constant_op_test.py b/tensorflow/python/kernel_tests/constant_op_test.py
index 2b743ed4c4..583082c2aa 100644
--- a/tensorflow/python/kernel_tests/constant_op_test.py
+++ b/tensorflow/python/kernel_tests/constant_op_test.py
@@ -24,7 +24,6 @@ from google.protobuf import text_format
 
 from tensorflow.core.framework import graph_pb2
 from tensorflow.core.framework import tensor_pb2
-from tensorflow.python.eager import def_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes as dtypes_lib
 from tensorflow.python.framework import errors_impl
@@ -631,16 +630,6 @@ class OnesTest(test.TestCase):
         self.assertEqual([2, 3], z.get_shape())
         self.assertAllEqual(z.eval(), np.ones([2, 3]))
 
-  @test_util.run_in_graph_and_eager_modes
-  def testIteratedConstantSensibleException(self):
-
-    @def_function.function
-    def constant_on_constant(x):
-      return constant_op.constant(x)
-
-    with self.assertRaisesRegexp(ValueError, "convert_to_tensor"):
-      constant_on_constant(constant_op.constant(2))
-
 
 class OnesLikeTest(test.TestCase):
 
-- 
GitLab


From fe66882827806db63761b593c70c43340f40750e Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Fri, 4 Jan 2019 16:51:20 -0800
Subject: [PATCH 0227/2345] Refactor compatibility upgrade tool to use pasta
 (an AST based refactoring tool) instead of line based edits.

This allows for some simplification (e.g., we can now remove arguments with a dict entry) and for much more powerful transformations.

This passes all tests with no destructive test changes (though there are cosmetic test changes).

Also adds transformations for tf.to_dtype -> tf.cast(..., dtype=...).

PiperOrigin-RevId: 227934801
---
 tensorflow/tools/compatibility/BUILD          |  12 +-
 tensorflow/tools/compatibility/ast_edits.py   | 668 +++++++++---------
 .../tools/compatibility/ast_edits_test.py     |  27 +-
 tensorflow/tools/compatibility/tf_upgrade.py  |  26 +-
 .../tools/compatibility/tf_upgrade_test.py    |   2 +-
 .../tools/compatibility/tf_upgrade_v2.py      | 288 +++++---
 .../tools/compatibility/tf_upgrade_v2_test.py | 105 +--
 tensorflow/tools/pip_package/BUILD            |   1 +
 tensorflow/workspace.bzl                      |   2 +
 third_party/pasta/BUILD                       |   1 +
 third_party/pasta/BUILD.bazel                 |  29 +
 third_party/pasta/BUILD.system                |  13 +
 third_party/pasta/workspace.bzl               |  16 +
 13 files changed, 687 insertions(+), 503 deletions(-)
 create mode 100644 third_party/pasta/BUILD
 create mode 100644 third_party/pasta/BUILD.bazel
 create mode 100644 third_party/pasta/BUILD.system
 create mode 100644 third_party/pasta/workspace.bzl

diff --git a/tensorflow/tools/compatibility/BUILD b/tensorflow/tools/compatibility/BUILD
index a9902d77f5..452e6db941 100644
--- a/tensorflow/tools/compatibility/BUILD
+++ b/tensorflow/tools/compatibility/BUILD
@@ -1,17 +1,21 @@
-licenses(["notice"])  # Apache 2.0
-
-package(default_visibility = ["//tensorflow:internal"])
-
 load(
     "//tensorflow:tensorflow.bzl",
     "tf_copts",  # @unused
     "tf_cc_test",  # @unused
 )
 
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//tensorflow:internal"])
+
 py_library(
     name = "ast_edits",
     srcs = ["ast_edits.py"],
     srcs_version = "PY2AND3",
+    deps = [
+        "@pasta",
+        "@six_archive//:six",
+    ],
 )
 
 py_test(
diff --git a/tensorflow/tools/compatibility/ast_edits.py b/tensorflow/tools/compatibility/ast_edits.py
index 9106ec97c8..0ec6c2ac8c 100644
--- a/tensorflow/tools/compatibility/ast_edits.py
+++ b/tensorflow/tools/compatibility/ast_edits.py
@@ -19,7 +19,6 @@ from __future__ import division
 from __future__ import print_function
 
 import ast
-import collections
 import os
 import re
 import shutil
@@ -27,6 +26,9 @@ import sys
 import tempfile
 import traceback
 
+import pasta
+import six
+
 # Some regular expressions we will need for parsing
 FIND_OPEN = re.compile(r"^\s*(\[).*$")
 FIND_STRING_CHARS = re.compile(r"['\"]")
@@ -44,264 +46,294 @@ class APIChangeSpec(object):
     notifications)
   * `function_reorders`: maps functions whose argument order has changed to the
     list of arguments in the new order
-  * `function_handle`: maps function names to custom handlers for the function
   * `function_warnings`: maps full names of functions to warnings that will be
     printed out if the function is used. (e.g. tf.nn.convolution())
-  * `unrestricted_function_warnings`: maps names of functions to warnings that
-    will be printed out when the function is used (e.g. foo.convolution()).
-  * `function_keyword_additions`: maps function names to a map of arg->value
-    names that should be passed to the function.
+  * `function_transformers`: maps function names to custom handlers
 
   For an example, see `TFAPIChangeSpec`.
   """
 
 
-class _FileEditTuple(
-    collections.namedtuple("_FileEditTuple",
-                           ["comment", "line", "start", "old", "new"])):
-  """Each edit that is recorded by a _FileEditRecorder.
+class _PastaEditVisitor(ast.NodeVisitor):
+  """AST Visitor that processes function calls.
 
-  Fields:
-    comment: A description of the edit and why it was made.
-    line: The line number in the file where the edit occurs (1-indexed).
-    start: The column number in the file where the edit occurs (0-indexed).
-    old: text string to remove (this must match what was in file).
-    new: text string to add in place of `old`.
+  Updates function calls from old API version to new API version using a given
+  change spec.
   """
 
-  __slots__ = ()
+  def __init__(self, api_change_spec):
+    self._api_change_spec = api_change_spec
+    self._log = []   # Holds 3-tuples: line, col, msg.
+    self._errors = []  # Same structure as _log.
+    self._stack = []  # Allow easy access to parents.
 
+  # Overridden to maintain a stack of nodes to allow for parent access
+  def visit(self, node):
+    self._stack.append(node)
+    super(_PastaEditVisitor, self).visit(node)
+    self._stack.pop()
 
-class _FileEditRecorder(object):
-  """Record changes that need to be done to the file."""
+  @property
+  def errors(self):
+    return self._errors
 
-  def __init__(self, filename):
-    # all edits are lists of chars
-    self._filename = filename
+  @property
+  def log(self):
+    return self._log
 
-    self._line_to_edit = collections.defaultdict(list)
-    self._errors = []
+  def _format_log(self, log):
+    text = ""
+    for log_entry in log:
+      text += "Line %d:%d: %s\n" % log_entry
+    return text
 
-  def process(self, text):
-    """Process a list of strings, each corresponding to the recorded changes.
+  def log_text(self):
+    return self._format_log(self.log)
 
-    Args:
-      text: A list of lines of text (assumed to contain newlines)
-    Returns:
-      A tuple of the modified text and a textual description of what is done.
-    Raises:
-      ValueError: if substitution source location does not have expected text.
-    """
+  def add_log(self, lineno, col, msg):
+    self._log.append((lineno, col, msg))
+    print("Line %d:%d: %s" % (lineno, col, msg))
 
-    change_report = ""
-
-    # Iterate of each line
-    for line, edits in self._line_to_edit.items():
-      offset = 0
-      # sort by column so that edits are processed in order in order to make
-      # indexing adjustments cumulative for changes that change the string
-      # length
-      edits.sort(key=lambda x: x.start)
-
-      # Extract each line to a list of characters, because mutable lists
-      # are editable, unlike immutable strings.
-      char_array = list(text[line - 1])
-
-      # Record a description of the change
-      change_report += "%r Line %d\n" % (self._filename, line)
-      change_report += "-" * 80 + "\n\n"
-      for e in edits:
-        change_report += "%s\n" % e.comment
-      change_report += "\n    Old: %s" % (text[line - 1])
-
-      # Make underscore buffers for underlining where in the line the edit was
-      change_list = [" "] * len(text[line - 1])
-      change_list_new = [" "] * len(text[line - 1])
-
-      # Iterate for each edit
-      for e in edits:
-        # Create effective start, end by accounting for change in length due
-        # to previous edits
-        start_eff = e.start + offset
-        end_eff = start_eff + len(e.old)
-
-        # Make sure the edit is changing what it should be changing
-        old_actual = "".join(char_array[start_eff:end_eff])
-        if old_actual != e.old:
-          raise ValueError("Expected text %r but got %r" %
-                           ("".join(e.old), "".join(old_actual)))
-        # Make the edit
-        char_array[start_eff:end_eff] = list(e.new)
-
-        # Create the underline highlighting of the before and after
-        change_list[e.start:e.start + len(e.old)] = "~" * len(e.old)
-        change_list_new[start_eff:end_eff] = "~" * len(e.new)
-
-        # Keep track of how to generate effective ranges
-        offset += len(e.new) - len(e.old)
-
-      # Finish the report comment
-      change_report += "         %s\n" % "".join(change_list)
-      text[line - 1] = "".join(char_array)
-      change_report += "    New: %s" % (text[line - 1])
-      change_report += "         %s\n\n" % "".join(change_list_new)
-    return "".join(text), change_report, self._errors
-
-  def add(self, comment, line, start, old, new, error=None):
-    """Add a new change that is needed.
+  def add_error(self, lineno, col, msg):
+    # All errors are also added to the regular log.
+    self.add_log(lineno, col, msg)
+    self._errors.append((lineno, col, msg))
 
-    Args:
-      comment: A description of what was changed
-      line: Line number (1 indexed)
-      start: Column offset (0 indexed)
-      old: old text
-      new: new text
-      error: this "edit" is something that cannot be fixed automatically
-    Returns:
-      None
-    """
-
-    self._line_to_edit[line].append(
-        _FileEditTuple(comment, line, start, old, new))
-    if error:
-      self._errors.append("%s:%d: %s" % (self._filename, line, error))
-
-
-class _ASTCallVisitor(ast.NodeVisitor):
-  """AST Visitor that processes function calls.
+  def add_logs(self, logs):
+    """Record a log and print it.
 
-  Updates function calls from old API version to new API version using a given
-  change spec.
-  """
-
-  def __init__(self, filename, lines, api_change_spec):
-    self._filename = filename
-    self._file_edit = _FileEditRecorder(filename)
-    self._lines = lines
-    self._api_change_spec = api_change_spec
+    The log should be a tuple (lineno, col_offset, msg), which will be printed
+    and then recorded. It is part of the log available in the self.log property.
 
-  def process(self, lines):
-    return self._file_edit.process(lines)
-
-  def generic_visit(self, node):
-    ast.NodeVisitor.generic_visit(self, node)
-
-  def _rename_functions(self, node, full_name):
-    symbol_renames = self._api_change_spec.symbol_renames
-    try:
-      new_name = symbol_renames[full_name]
-      self._file_edit.add("Renamed function %r to %r" % (full_name, new_name),
-                          node.lineno, node.col_offset, full_name, new_name)
-    except KeyError:
-      pass
-
-  def _print_warning_for_function(self, node, full_name):
-    function_warnings = self._api_change_spec.function_warnings
-    try:
-      warning_message = function_warnings[full_name]
-      warning_message = warning_message.replace("<function name>", full_name)
-      self._file_edit.add(warning_message,
-                          node.lineno, node.col_offset, full_name, full_name,
-                          error="%s requires manual check." % full_name)
-    except KeyError:
-      pass
+    Args:
+      logs: The log to add. Must be a tuple (lineno, col_offset, msg).
+    """
+    self._log.extend(logs)
+    for log in logs:
+      print("Line %d:%d: %s" % log)
 
-  def _print_warning_for_function_unrestricted(self, node):
-    """Print a warning when specific functions are called.
+  def add_errors(self, errors):
+    """Record an error and print it.
 
-    The function _print_warning_for_function matches the full name of the called
-    function, e.g., tf.foo.bar(). This function matches the function name that
-    is called, as long as the function is an attribute. For example,
-    `tf.foo.bar()` and `foo.bar()` are matched, but not `bar()`.
+    The error must be a tuple (lineno, col_offset, msg), which will be printed
+    and then recorded as both a log and an error. It is therefore part of the
+    log available in the self.log as well as the self.errors property.
 
     Args:
-      node: ast.Call object
+      errors: The log to add. Must be a tuple (lineno, col_offset, msg).
     """
-    function_warnings = getattr(
-        self._api_change_spec, "unrestricted_function_warnings", {})
-    if isinstance(node.func, ast.Attribute):
-      function_name = node.func.attr
-      try:
-        warning_message = function_warnings[function_name]
-        self._file_edit.add(warning_message,
-                            node.lineno, node.col_offset, "", "",
-                            error="%s requires manual check." % function_name)
-      except KeyError:
-        pass
-
-  def _get_attribute_full_path(self, node):
-    """Traverse an attribute to generate a full name e.g. tf.foo.bar.
+    self.add_logs(errors)
+    self._errors.extend(errors)
+
+  def _get_applicable_entries(self, transformer_field, full_name, name):
+    """Get all list entries indexed by name that apply to full_name or name."""
+    # Transformers are indexed to full name, name, or no name
+    # as a performance optimization.
+    function_transformers = getattr(self._api_change_spec,
+                                    transformer_field, {})
+
+    glob_name = "*." + name if name else None
+    transformers = []
+    if full_name in function_transformers:
+      transformers.append(function_transformers[full_name])
+    if glob_name in function_transformers:
+      transformers.append(function_transformers[glob_name])
+    if "*" in function_transformers:
+      transformers.append(function_transformers["*"])
+    return transformers
+
+  def _get_applicable_dict(self, transformer_field, full_name, name):
+    """Get all dict entries indexed by name that apply to full_name or name."""
+    # Transformers are indexed to full name, name, or no name
+    # as a performance optimization.
+    function_transformers = getattr(self._api_change_spec,
+                                    transformer_field, {})
+
+    glob_name = "*." + name if name else None
+    transformers = function_transformers.get("*", {}).copy()
+    transformers.update(function_transformers.get(glob_name, {}))
+    transformers.update(function_transformers.get(full_name, {}))
+    return transformers
+
+  def _get_full_name(self, node):
+    """Traverse an Attribute node to generate a full name, e.g., "tf.foo.bar".
+
+    This is the inverse of _full_name_node.
 
     Args:
       node: A Node of type Attribute.
 
     Returns:
-      a '.'-delimited full-name or None if the tree was not a simple form.
+      a '.'-delimited full-name or None if node was not Attribute or Name.
       i.e. `foo()+b).bar` returns None, while `a.b.c` would return "a.b.c".
     """
     curr = node
     items = []
     while not isinstance(curr, ast.Name):
       if not isinstance(curr, ast.Attribute):
-        return None, None
+        return None
       items.append(curr.attr)
       curr = curr.value
     items.append(curr.id)
-    return ".".join(reversed(items)), items[0]
+    return ".".join(reversed(items))
 
-  def _find_true_position(self, node):
-    """Return correct line number and column offset for a given node.
+  def _full_name_node(self, name, ctx=ast.Load()):
+    """Make an Attribute or Name node for name.
 
-    This is necessary mainly because ListComp's location reporting reports
-    the next token after the list comprehension list opening.
+    Translate a qualified name into nested Attribute nodes (and a Name node).
+
+    Args:
+      name: The name to translate to a node.
+      ctx: What context this name is used in. Defaults to Load()
 
     Returns:
-      lineno, offset for the given node
+      A Name or Attribute node.
+    """
+    names = name.split(".")
+    names.reverse()
+    node = ast.Name(id=names.pop(), ctx=ast.Load())
+    while names:
+      node = ast.Attribute(value=node, attr=names.pop(), ctx=ast.Load())
+
+    # Change outermost ctx to the one given to us (inner ones should be Load).
+    node.ctx = ctx
+    return node
+
+  def _maybe_add_warning(self, node, full_name):
+    """Adds an error to be printed about full_name at node."""
+    function_warnings = self._api_change_spec.function_warnings
+    if full_name in function_warnings:
+      warning_message = function_warnings[full_name]
+      warning_message = warning_message.replace("<function name>", full_name)
+      self.add_error(node.lineno, node.col_offset,
+                     "%s requires manual check: %s." % (full_name,
+                                                        warning_message))
+      return True
+    else:
+      return False
+
+  def _maybe_add_call_warning(self, node, full_name, name):
+    """Print a warning when specific functions are called.
+
+    The function _print_warning_for_function matches the full name of the called
+    function, e.g., tf.foo.bar(). This function matches the function name that
+    is called, as long as the function is an attribute. For example,
+    `tf.foo.bar()` and `foo.bar()` are matched, but not `bar()`.
 
     Args:
-      node: Node for which we wish to know the lineno and col_offset
+      node: ast.Call object
+      full_name: The precomputed full name of the callable, if one exists, None
+        otherwise.
+      name: The precomputed name of the callable, if one exists, None otherwise.
+
+    Returns:
+      Whether an error was recorded.
     """
-    if isinstance(node, ast.ListComp):
-      # Strangely, ast.ListComp returns the col_offset of the first token
-      # after the '[' token which appears to be a bug. Workaround by
-      # explicitly finding the real start of the list comprehension.
-      line = node.lineno
-      col = node.col_offset
-      # loop over lines
-      while 1:
-        # Reverse the text to and regular expression search for whitespace
-        text = self._lines[line - 1]
-        reversed_preceding_text = text[:col][::-1]
-        # First find if a [ can be found with only whitespace between it and
-        # col.
-        m = FIND_OPEN.match(reversed_preceding_text)
-        if m:
-          new_col_offset = col - m.start(1) - 1
-          return line, new_col_offset
+    # Only look for *.-warnings here, the other will be handled by the Attribute
+    # visitor. Also, do not warn for bare functions, only if the call func is
+    # an attribute.
+    warned = False
+    if isinstance(node.func, ast.Attribute):
+      warned = self._maybe_add_warning(node, "*." + name)
+
+    # All arg warnings are handled here, since only we have the args
+    arg_warnings = self._get_applicable_dict("function_arg_warnings",
+                                             full_name, name)
+
+    used_args = [kw.arg for kw in node.keywords]
+    for arg, warning in arg_warnings.items():
+      if arg in used_args:
+        warned = True
+        warning_message = warning.replace("<function name>", full_name or name)
+        self.add_error(node.lineno, node.col_offset,
+                       "%s called with %s argument requires manual check: %s." %
+                       (full_name or name, arg, warning_message))
+
+    return warned
+
+  def _maybe_rename(self, parent, node, full_name):
+    """Replace node (Attribute or Name) with a node representing full_name."""
+    new_name = self._api_change_spec.symbol_renames.get(full_name, None)
+    if new_name:
+      self.add_log(node.lineno, node.col_offset,
+                   "Renamed %r to %r" % (full_name, new_name))
+      new_node = self._full_name_node(new_name, node.ctx)
+      ast.copy_location(new_node, node)
+      pasta.ast_utils.replace_child(parent, node, new_node)
+      return True
+    else:
+      return False
+
+  def _maybe_change_to_function_call(self, parent, node, full_name):
+    """Wraps node (typically, an Attribute or Expr) in a Call."""
+    if full_name in self._api_change_spec.change_to_function:
+      if not isinstance(parent, ast.Call):
+        # ast.Call's constructor is really picky about how many arguments it
+        # wants, and also, it changed between Py2 and Py3.
+        if six.PY2:
+          new_node = ast.Call(node, [], [], None, None)
         else:
-          if (reversed_preceding_text == "" or
-              reversed_preceding_text.isspace()):
-            line = line - 1
-            prev_line = self._lines[line - 1]
-            # TODO(aselle):
-            # this is poor comment detection, but it is good enough for
-            # cases where the comment does not contain string literal starting/
-            # ending characters. If ast gave us start and end locations of the
-            # ast nodes rather than just start, we could use string literal
-            # node ranges to filter out spurious #'s that appear in string
-            # literals.
-            comment_start = prev_line.find("#")
-            if comment_start == -1:
-              col = len(prev_line) - 1
-            elif FIND_STRING_CHARS.search(prev_line[comment_start:]) is None:
-              col = comment_start
-            else:
-              return None, None
-          else:
-            return None, None
-    # Most other nodes return proper locations (with notably does not), but
-    # it is not possible to use that in an argument.
-    return node.lineno, node.col_offset
+          new_node = ast.Call(node, [], [])
+        pasta.ast_utils.replace_child(parent, node, new_node)
+        ast.copy_location(new_node, node)
+        self.add_log(node.lineno, node.col_offset,
+                     "Changed %r to a function call" % full_name)
+        return True
+    return False
+
+  def _maybe_add_arg_names(self, node, full_name):
+    """Make args into keyword args if function called full_name requires it."""
+    function_reorders = self._api_change_spec.function_reorders
+
+    if full_name in function_reorders:
+      reordered = function_reorders[full_name]
+      new_keywords = []
+      for idx, arg in enumerate(node.args):
+        keyword_arg = reordered[idx]
+        new_keywords.append(ast.keyword(arg=keyword_arg, value=arg))
+
+      if new_keywords:
+        self.add_log(node.lineno, node.col_offset,
+                     "Added keywords to args of function %r" % full_name)
+        node.args = []
+        node.keywords = new_keywords + (node.keywords or [])
+        return True
+    return False
+
+  def _maybe_modify_args(self, node, full_name, name):
+    """Rename keyword args if the function called full_name requires it."""
+    renamed_keywords = self._get_applicable_dict("function_keyword_renames",
+                                                 full_name, name)
+
+    if not renamed_keywords:
+      return False
+
+    modified = False
+    new_keywords = []
+    for keyword in node.keywords:
+      argkey = keyword.arg
+      if argkey in renamed_keywords:
+        modified = True
+        if renamed_keywords[argkey] is None:
+          lineno = getattr(keyword, "lineno", node.lineno)
+          col_offset = getattr(keyword, "col_offset", node.col_offset)
+          self.add_log(lineno, col_offset,
+                       "Removed argument %s for function %s" % (
+                           argkey, full_name or name))
+        else:
+          keyword.arg = renamed_keywords[argkey]
+          lineno = getattr(keyword, "lineno", node.lineno)
+          col_offset = getattr(keyword, "col_offset", node.col_offset)
+          self.add_log(lineno, col_offset,
+                       "Renamed keyword argument for %s from %s to %s" % (
+                           full_name, argkey, renamed_keywords[argkey]))
+          new_keywords.append(keyword)
+      else:
+        new_keywords.append(keyword)
+
+    if modified:
+      node.keywords = new_keywords
+    return modified
 
   def visit_Call(self, node):  # pylint: disable=invalid-name
     """Handle visiting a call node in the AST.
@@ -309,104 +341,74 @@ class _ASTCallVisitor(ast.NodeVisitor):
     Args:
       node: Current Node
     """
-    self._print_warning_for_function_unrestricted(node)
-
-    # Find a simple attribute name path e.g. "tf.foo.bar"
-    full_name, name = self._get_attribute_full_path(node.func)
-
-    # Make sure the func is marked as being part of a call
-    node.func.is_function_for_call = True
+    assert self._stack[-1] is node
 
+    # Get the name for this call, so we can index stuff with it.
+    full_name = self._get_full_name(node.func)
     if full_name:
-      # Call special handlers
-      function_handles = self._api_change_spec.function_handle
-      glob_name = "*.{}".format(name)
-      if glob_name in function_handles:
-        function_handles[glob_name](self._file_edit, node, self._lines)
-      if full_name in function_handles:
-        function_handles[full_name](self._file_edit, node, self._lines)
-
-      # Examine any non-keyword argument and make it into a keyword argument
-      # if reordering required.
-      function_reorders = self._api_change_spec.function_reorders
-      function_keyword_renames = (
-          self._api_change_spec.function_keyword_renames)
-
-      if full_name in function_reorders:
-        reordered = function_reorders[full_name]
-        for idx, arg in enumerate(node.args):
-          lineno, col_offset = self._find_true_position(arg)
-          if lineno is None or col_offset is None:
-            self._file_edit.add(
-                "Failed to add keyword %r to reordered function %r" %
-                (reordered[idx], full_name),
-                arg.lineno,
-                arg.col_offset,
-                "",
-                "",
-                error="A necessary keyword argument failed to be inserted.")
-          else:
-            keyword_arg = reordered[idx]
-            if (full_name in function_keyword_renames and
-                keyword_arg in function_keyword_renames[full_name]):
-              keyword_arg = function_keyword_renames[full_name][keyword_arg]
-            self._file_edit.add("Added keyword %r to reordered function %r" %
-                                (reordered[idx], full_name), lineno, col_offset,
-                                "", keyword_arg + "=")
-
-      # Examine each keyword argument and convert it to the final renamed form
-      renamed_keywords = ({} if full_name not in function_keyword_renames else
-                          function_keyword_renames[full_name])
-      for keyword in node.keywords:
-        argkey = keyword.arg
-        argval = keyword.value
-
-        if argkey in renamed_keywords:
-          argval_lineno, argval_col_offset = self._find_true_position(argval)
-          if argval_lineno is not None and argval_col_offset is not None:
-            # TODO(aselle): We should scan backward to find the start of the
-            # keyword key. Unfortunately ast does not give you the location of
-            # keyword keys, so we are forced to infer it from the keyword arg
-            # value.
-            key_start = argval_col_offset - len(argkey) - 1
-            key_end = key_start + len(argkey) + 1
-            if (self._lines[argval_lineno - 1][key_start:key_end] == argkey +
-                "="):
-              self._file_edit.add("Renamed keyword argument from %r to %r" %
-                                  (argkey,
-                                   renamed_keywords[argkey]), argval_lineno,
-                                  argval_col_offset - len(argkey) - 1,
-                                  argkey + "=", renamed_keywords[argkey] + "=")
-              continue
-          self._file_edit.add(
-              "Failed to rename keyword argument from %r to %r" %
-              (argkey, renamed_keywords[argkey]),
-              argval.lineno,
-              argval.col_offset - len(argkey) - 1,
-              "",
-              "",
-              error="Failed to find keyword lexographically. Fix manually.")
-
-    ast.NodeVisitor.generic_visit(self, node)
+      name = full_name.split(".")[-1]
+    elif isinstance(node.func, ast.Name):
+      name = node.func.id
+    elif isinstance(node.func, ast.Attribute):
+      name = node.func.attr
+    else:
+      name = None
+
+    # Call standard transformers for this node.
+    # Make sure warnings come first, since args or names triggering warnings
+    # may be removed by the other transformations.
+    self._maybe_add_call_warning(node, full_name, name)
+    # Make all args into kwargs
+    self._maybe_add_arg_names(node, full_name)
+    # Argument name changes or deletions
+    self._maybe_modify_args(node, full_name, name)
+
+    # Call transformers. These have the ability to modify the node, and if they
+    # do, will return the new node they created (or the same node if they just
+    # changed it). The are given the parent, but we will take care of
+    # integrating their changes into the parent if they return a new node.
+    #
+    # These are matched on the old name, since renaming is performed by the
+    # Attribute visitor, which happens later.
+    transformers = self._get_applicable_entries("function_transformers",
+                                                full_name, name)
+
+    parent = self._stack[-2]
+
+    for transformer in transformers:
+      logs = []
+      errors = []
+      new_node = transformer(parent, node, full_name, name, logs, errors)
+      self.add_logs(logs)
+      self.add_errors(errors)
+      if new_node:
+        if new_node is not node:
+          pasta.ast_utils.replace_child(parent, node, new_node)
+          node = new_node
+          self._stack[-1] = node
+
+    self.generic_visit(node)
 
   def visit_Attribute(self, node):  # pylint: disable=invalid-name
-    """Handle bare Attributes i.e. [tf.foo, tf.bar].
+    """Handle bare Attributes i.e. [tf.foo, tf.bar]."""
+    assert self._stack[-1] is node
 
-    Args:
-      node: Node that is of type ast.Attribute
-    """
-    full_name, _ = self._get_attribute_full_path(node)
+    full_name = self._get_full_name(node)
     if full_name:
+      parent = self._stack[-2]
+
       # Make sure the warning comes first, otherwise the name may have changed
-      self._print_warning_for_function(node, full_name)
-      self._rename_functions(node, full_name)
-    if full_name in self._api_change_spec.change_to_function:
-      if not hasattr(node, "is_function_for_call"):
-        new_text = full_name + "()"
-        self._file_edit.add("Changed %r to %r" % (full_name, new_text),
-                            node.lineno, node.col_offset, full_name, new_text)
+      self._maybe_add_warning(node, full_name)
 
-    ast.NodeVisitor.generic_visit(self, node)
+      # Once we did a modification, node is invalid and not worth inspecting
+      # further. Also, we only perform modifications for simple nodes, so
+      # There'd be no point in descending further.
+      if self._maybe_rename(parent, node, full_name):
+        return
+      if self._maybe_change_to_function_call(parent, node, full_name):
+        return
+
+    self.generic_visit(node)
 
 
 class ASTCodeUpgrader(object):
@@ -429,16 +431,42 @@ class ASTCodeUpgrader(object):
     """
 
     # Write to a temporary file, just in case we are doing an implace modify.
+    # pylint: disable=g-backslash-continuation
     with open(in_filename, "r") as in_file, \
         tempfile.NamedTemporaryFile("w", delete=False) as temp_file:
       ret = self.process_opened_file(in_filename, in_file, out_filename,
                                      temp_file)
+    # pylint: enable=g-backslash-continuation
 
     shutil.move(temp_file.name, out_filename)
     return ret
 
-  # Broad exceptions are required here because ast throws whatever it wants.
-  # pylint: disable=broad-except
+  def _format_errors(self, errors, in_filename):
+    return ["%s:%d:%d: %s" % ((in_filename,) + error) for error in errors]
+
+  def update_string_pasta(self, text, in_filename):
+    """Updates a file using pasta."""
+    try:
+      t = pasta.parse(text)
+    except (SyntaxError, ValueError, TypeError):
+      log = "Failed to parse.\n\n" + traceback.format_exc()
+      return 0, "", log, []
+
+    visitor = _PastaEditVisitor(self._api_change_spec)
+    visitor.visit(t)
+
+    errors = self._format_errors(visitor.errors, in_filename)
+    return 1, pasta.dump(t), visitor.log_text(), errors
+
+  def _format_log(self, log, in_filename, out_filename):
+    text = "-" * 80 + "\n"
+    text += "Processing file %r\n outputting to %r\n" % (in_filename,
+                                                         out_filename)
+    text += "-" * 80 + "\n\n"
+    text += log
+    text += "-" * 80 + "\n\n"
+    return text
+
   def process_opened_file(self, in_filename, in_file, out_filename, out_file):
     """Process the given python file for incompatible changes.
 
@@ -453,30 +481,16 @@ class ASTCodeUpgrader(object):
     Returns:
       A tuple representing number of files processed, log of actions, errors
     """
-    process_errors = []
-    text = "-" * 80 + "\n"
-    text += "Processing file %r\n outputting to %r\n" % (in_filename,
-                                                         out_filename)
-    text += "-" * 80 + "\n\n"
-
-    parsed_ast = None
     lines = in_file.readlines()
-    try:
-      parsed_ast = ast.parse("".join(lines))
-    except Exception:
-      text += "Failed to parse %r\n\n" % in_filename
-      text += traceback.format_exc()
-    if parsed_ast:
-      visitor = _ASTCallVisitor(in_filename, lines, self._api_change_spec)
-      visitor.visit(parsed_ast)
-      out_text, new_text, process_errors = visitor.process(lines)
-      text += new_text
-      if out_file:
-        out_file.write(out_text)
-    text += "\n"
-    return 1, text, process_errors
-
-  # pylint: enable=broad-except
+    processed_file, new_file_content, log, process_errors = (
+        self.update_string_pasta("".join(lines), in_filename))
+
+    if out_file and processed_file:
+      out_file.write(new_file_content)
+
+    return (processed_file,
+            self._format_log(log, in_filename, out_filename),
+            process_errors)
 
   def process_tree(self, root_directory, output_root_directory,
                    copy_other_files):
diff --git a/tensorflow/tools/compatibility/ast_edits_test.py b/tensorflow/tools/compatibility/ast_edits_test.py
index 99f20a026f..b2e9f67638 100644
--- a/tensorflow/tools/compatibility/ast_edits_test.py
+++ b/tensorflow/tools/compatibility/ast_edits_test.py
@@ -39,6 +39,8 @@ following new APIs:
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+import ast
+import pasta
 import six
 from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test as test_lib
@@ -54,7 +56,6 @@ class NoUpdateSpec(ast_edits.APIChangeSpec):
     self.function_keyword_renames = {}
     self.symbol_renames = {}
     self.function_warnings = {}
-    self.unrestricted_function_warnings = {}
     self.change_to_function = {}
 
 
@@ -401,7 +402,8 @@ class TestAstEdits(test_util.TensorFlowTestCase):
 
       def __init__(self):
         NoUpdateSpec.__init__(self)
-        self.unrestricted_function_warnings = {"foo": "not good"}
+        self.function_warnings = {"*.foo": "not good"}
+
     texts = ["object.foo()", "get_object().foo()",
              "get_object().foo()", "object.foo().bar()"]
     for text in texts:
@@ -416,5 +418,26 @@ class TestAstEdits(test_util.TensorFlowTestCase):
       self.assertNotIn("not good", report)
 
 
+class ManualEditsTest(test_util.TensorFlowTestCase):
+
+  def disabled_test_arg_order(self):
+    """Tests that generated arg order is sane."""
+    text = "f(a)"
+    t = pasta.parse(text)
+    node = pasta.ast_utils.find_nodes_by_type(t, (ast.Call,))[0]
+    arg = ast.keyword(arg="b", value=ast.Num(n=0))
+    node.keywords.append(arg)
+
+    # This is only needed in Python3, and I think it's a bug (but maybe in ast).
+    arg.value.lineno = 0
+    arg.value.col_offset = 0
+
+    # pasta.dump should never put kwargs before args, even if the col_offset is
+    # messed up.
+    # This fails if run with python3, but works find for python2.
+    # In python3, the dump yields "f(b=0, a)".
+    self.assertEqual(pasta.dump(t), "f(a, b=0)")
+
+
 if __name__ == "__main__":
   test_lib.main()
diff --git a/tensorflow/tools/compatibility/tf_upgrade.py b/tensorflow/tools/compatibility/tf_upgrade.py
index 68d2c02570..241b08510f 100644
--- a/tensorflow/tools/compatibility/tf_upgrade.py
+++ b/tensorflow/tools/compatibility/tf_upgrade.py
@@ -175,27 +175,13 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.op_scope": ["values", "name", "default_name"],
     }
 
-    # Specially handled functions.
-    self.function_handle = {"tf.reverse": self._reverse_handler}
-
     # Warnings that should be printed if corresponding functions are used.
-    self.function_warnings = {}
-
-  @staticmethod
-  def _reverse_handler(file_edit_recorder, node, lines):
-    del lines
-    # TODO(aselle): Could check for a literal list of bools and try to convert
-    # them to indices.
-    comment = ("ERROR: tf.reverse has had its argument semantics changed "
-               "significantly the converter cannot detect this reliably, so "
-               "you need to inspect this usage manually.\n")
-    file_edit_recorder.add(
-        comment,
-        node.lineno,
-        node.col_offset,
-        "tf.reverse",
-        "tf.reverse",
-        error="tf.reverse requires manual check.")
+    self.function_warnings = {
+        "tf.reverse":
+            "ERROR: tf.reverse has had its argument semantics changed "
+            "significantly. The converter cannot detect this reliably, so "
+            "you need to inspect this usage manually.\n",
+    }
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/tools/compatibility/tf_upgrade_test.py b/tensorflow/tools/compatibility/tf_upgrade_test.py
index 66325ea2ad..cf05575a9d 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_test.py
@@ -112,7 +112,7 @@ class TestUpgrade(test_util.TensorFlowTestCase):
     text = "tf.reverse(a, b)\n"
     _, unused_report, errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, new_text)
-    self.assertEqual(errors, ["test.py:1: tf.reverse requires manual check."])
+    self.assertIn("tf.reverse requires manual check", errors[0])
 
   def testListComprehension(self):
     def _test(input, output):  # pylint: disable=redefined-builtin
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 82839afc3d..9d9c5878f7 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -18,7 +18,9 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import re
+import ast
+
+import pasta
 
 from tensorflow.tools.compatibility import ast_edits
 from tensorflow.tools.compatibility import renames_v2
@@ -31,7 +33,22 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
   def __init__(self):
     # Maps from a function name to a dictionary that describes how to
     # map from an old argument keyword to the new argument keyword.
+    # If the new argument is None, it will be removed.
+    # Only keyword args are handled, so make sure to also put any function in
+    # function_reorders to ensure that all args are made into keywords first.
     self.function_keyword_renames = {
+        "tf.gradients": {
+            "colocate_gradients_with_ops": None,
+        },
+        "tf.hessians": {
+            "colocate_gradients_with_ops": None,
+        },
+        "*.minimize": {
+            "colocate_gradients_with_ops": None,
+        },
+        "*.compute_gradients": {
+            "colocate_gradients_with_ops": None,
+        },
         "tf.argmin": {
             "dimension": "axis",
         },
@@ -672,14 +689,31 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     # positional arguments yourself, this could do the wrong thing.
     self.function_reorders = reorders_v2.reorders
 
-    # Specially handled functions.
-    self.function_handle = {
-        "tf.batch_gather": self._batch_gather_handler,
-        "tf.nn.dropout": self._dropout_handler,
-        "tf.gradients": self._colocate_handler("tf.gradients"),
-        "*.minimize": self._colocate_handler("Optimizer.minimize"),
-        "*.compute_gradients":
-            self._colocate_handler("Optimizer.compute_gradients"),
+    # Specially handled functions (pasta version)
+    # Each transformer is a callable which will be called with the arguments
+    #   transformer(parent, node, full_name, name, logs, errors)
+    # Where logs and errors are lists to which (line, col, msg) tuples can be
+    # appended, full_name is the FQN of the function called (or None if that is
+    # unknown), name is the name of the function called (or None is that is
+    # unknown). node is an ast.Call node representing this function call, and
+    # parent is its parent in the AST.
+    # The function may modify node (but not parent), and must return
+    # - none, if nothing was modified
+    # - node, if node was modified in place (make sure to use
+    #   pasta.ast_utils.replace_child to swap out children, otherwise formatting
+    #   may get messy)
+    # - a replacement for node, if the whole call node was replaced. The caller
+    #   will take care of changing parent.
+    self.function_transformers = {
+        "tf.nn.dropout": self._dropout_transformer,
+        "tf.batch_gather": self._batch_gather_transformer,
+        "tf.to_bfloat16": self._cast_transformer,
+        "tf.to_complex128": self._cast_transformer,
+        "tf.to_complex64": self._cast_transformer,
+        "tf.to_double": self._cast_transformer,
+        "tf.to_float": self._cast_transformer,
+        "tf.to_int32": self._cast_transformer,
+        "tf.to_int64": self._cast_transformer,
     }
 
     decay_function_comment = (
@@ -748,9 +782,45 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "compat.v1 for backward compatibility. Please update these calls to "
         "the TF 2.0 versions.")
 
+    export_saved_model_renamed = (
+        "(Manual edit required) Please rename the method export_savedmodel() "
+        "to export_saved_model(). Two things to note:\n\t(1) The argument "
+        "strip_default_attributes has been removed. The function will always "
+        "strip the default attributes from ops. If this breaks your code, "
+        "please switch to tf.compat.v1.estimator.Estimator.\n\t(2) This change "
+        "only effects core estimator. If you are using "
+        "tf.contrib.learn.Estimator, please switch to using core estimator.")
+
+    make_initializable_iterator_deprecation = (
+        "(Manual edit required) The "
+        "`tf.data.Dataset.make_initializable_iterator()` method has been "
+        "removed. If you are using the Estimator API, you can return a dataset "
+        "directly from your input functions without creating an iterator. "
+        "As a last resort, please replace calls to that method on `dataset` "
+        "with a call to "
+        "`tf.compat.v1.data.make_initializable_iterator(dataset)`.")
+
+    make_one_shot_iterator_deprecation = (
+        "(Manual edit required) The "
+        "`tf.data.Dataset.make_one_shot_iterator()` method has been "
+        "removed. If you are using eager execution, you can iterate over "
+        "`dataset` using a Python `for` loop. If you are using the Estimator "
+        "API, you can return a dataset directly from your input functions "
+        "without creating an iterator. As a last resort, please replace calls "
+        "to that method on `dataset` with a call to "
+        "`tf.compat.v1.data.make_one_shot_iterator(dataset)`.")
+
     # Function warnings. <function name> placeholder inside warnings will be
     # replaced by function name.
+    # You can use *. to add items which do not check the FQN, and apply to e.g.,
+    # methods.
     self.function_warnings = {
+        "*.export_savedmodel":
+            export_saved_model_renamed,
+        "*.make_initializable_iterator":
+            make_initializable_iterator_deprecation,
+        "*.make_one_shot_iterator":
+            make_one_shot_iterator_deprecation,
         "tf.assert_greater":
             assert_return_type_comment,
         "tf.assert_equal":
@@ -834,11 +904,6 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             default_loss_reduction_changed,
         "tf.estimator.BaselineRegressor":
             default_loss_reduction_changed,
-        "tf.hessians":
-            "tf.hessians no longer takes "
-            "'colocate_gradients_with_ops' argument. Also, "
-            "arguments have been reordered so that 'name' is the "
-            "last argument.",
         "tf.nn.conv1d":
             "WARNING: use_cudnn_on_gpu argument has been removed and \"value\""
             " was renamed to \"input\"",
@@ -1064,111 +1129,118 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             metrics_comment,
     }
 
+    # Warnings that are emitted only if a specific arg is found.
+    self.function_arg_warnings = {
+        "tf.gradients": {
+            "colocate_gradients_with_ops":
+                "tf.gradients no longer takes "
+                "'colocate_gradients_with_ops' argument, it behaves as if it "
+                "was set to True.",
+        },
+        "*.minimize": {
+            "colocate_gradients_with_ops":
+                "Optimizer.minimize no longer takes "
+                "'colocate_gradients_with_ops' argument, it behaves as if it "
+                "was set to True.",
+        },
+        "*.compute_gradients": {
+            "colocate_gradients_with_ops":
+                "Optimizer.compute_gradients no "
+                "longer takes 'colocate_gradients_with_ops' argument, it "
+                "behaves as if it was set to True.",
+        },
+    }
+
     self.symbol_renames = {
         name: new_name
         for name, new_name in self.symbol_renames.items()
     }
 
-    export_saved_model_renamed = (
-        "(Manual edit required) Please rename the method export_savedmodel() "
-        "to export_saved_model(). Two things to note:\n\t(1) The argument "
-        "strip_default_attributes has been removed. The function will always "
-        "strip the default attributes from ops. If this breaks your code, "
-        "please switch to tf.compat.v1.estimator.Estimator.\n\t(2) This change "
-        "only effects core estimator. If you are using "
-        "tf.contrib.learn.Estimator, please switch to using core estimator.")
-
-    make_initializable_iterator_deprecation = (
-        "(Manual edit required) The "
-        "`tf.data.Dataset.make_initializable_iterator()` method has been "
-        "removed. If you are using the Estimator API, you can return a dataset "
-        "directly from your input functions without creating an iterator. "
-        "As a last resort, please replace calls to that method on `dataset` "
-        "with a call to "
-        "`tf.compat.v1.data.make_initializable_iterator(dataset)`.")
-
-    make_one_shot_iterator_deprecation = (
-        "(Manual edit required) The "
-        "`tf.data.Dataset.make_one_shot_iterator()` method has been "
-        "removed. If you are using eager execution, you can iterate over "
-        "`dataset` using a Python `for` loop. If you are using the Estimator "
-        "API, you can return a dataset directly from your input functions "
-        "without creating an iterator. As a last resort, please replace calls "
-        "to that method on `dataset` with a call to "
-        "`tf.compat.v1.data.make_one_shot_iterator(dataset)`.")
+  @staticmethod
+  def _dropout_transformer(parent, node, full_name, name, logs, errors):
+    def _replace_keep_prob_node(parent, old_value):
+      """Replaces old_value with 1-(old_value)."""
+      one = ast.Num(n=1)
+      one.lineno = 0
+      one.col_offset = 0
+      new_value = ast.BinOp(left=one, op=ast.Sub(),
+                            right=old_value)
+      # This copies the prefix and suffix on old_value to new_value.
+      pasta.ast_utils.replace_child(parent, old_value, new_value)
+      ast.copy_location(new_value, old_value)
+      # Put parentheses around keep_prob.value (and remove the old prefix/
+      # suffix, they should only be around new_value).
+      pasta.base.formatting.set(old_value, "prefix", "(")
+      pasta.base.formatting.set(old_value, "suffix", ")")
 
-    # Specify warnings for functions that aren't restricted to the tf.x.y.z
-    # format. This should only be used for methods with unique names, e.g.
-    # export_savedmodel, which is only defined in Estimator objects.
-    self.unrestricted_function_warnings = {
-        "export_savedmodel": export_saved_model_renamed,
-        "make_initializable_iterator": make_initializable_iterator_deprecation,
-        "make_one_shot_iterator": make_one_shot_iterator_deprecation,
-    }
+    # Check if we have a keep_prob keyword arg
+    for keep_prob in node.keywords:
+      if keep_prob.arg == "keep_prob":
+        logs.append((node.lineno, node.col_offset,
+                     "Changing keep_prob arg of tf.nn.dropout to rate, and "
+                     "recomputing value. Please check this transformation.\n"))
+        keep_prob.arg = "rate"
+        _replace_keep_prob_node(keep_prob, keep_prob.value)
+        return node
 
-  @staticmethod
-  def _dropout_handler(file_edit_recorder, node, lines):
-    del lines
+    # Maybe it was a positional arg
     if len(node.args) < 2:
-      comment = ("ERROR: tf.nn.dropout did not take arguments, so automatic "
-                 "transformation was disabled. tf.nn.dropout has changed "
-                 "the semantics of the second argument.")
-      file_edit_recorder.add(
-          comment,
-          node.lineno,
-          node.col_offset,
-          "tf.nn.dropout",
-          "tf.nn.dropout",
-          error="tf.nn.dropout requires manual check.")
+      errors.append((node.lineno, node.col_offset,
+                     "ERROR: tf.nn.dropout called without arguments, so "
+                     "automatic fix was disabled. tf.nn.dropout has changed "
+                     "the semantics of the second argument."))
     else:
-      comment = ("WARNING: tf.nn.dropout has changed the semantics of the "
-                 "second argument. Please check the transformation.\n")
-      file_edit_recorder.add(
-          comment,
-          node.args[1].lineno,
-          node.args[1].col_offset,
-          "",
-          "1 - ")
+      _replace_keep_prob_node(node, node.args[1])
+      logs.append((node.lineno, node.col_offset,
+                   "Changing keep_prob arg of tf.nn.dropout to rate, and "
+                   "recomputing value.\n"))
+      errors.append((node.lineno, node.col_offset,
+                     "WARNING: tf.nn.dropout has changed the semantics of the "
+                     "second argument. Please check the applied transformation."
+                    ))
+      return node
 
   @staticmethod
-  def _colocate_handler(name):
-    def _helper(file_edit_recorder, node, lines):
-      """Handler for updating colocate arguments."""
-      del lines
-      for keyword in node.keywords:
-        if keyword.arg == "colocate_gradients_with_ops":
-          # TODO(jhseu): Since ast_edit.py does string replacement, there's no
-          # straightforward way to remove the argument. Try to fix before 2.0 is
-          # final.
-          comment = ("For tf.gradients and tf.Optimizer.minimize, "
-                     "colocate_gradients_with_op has been removed and now "
-                     "defaults to True.")
-          file_edit_recorder.add(
-              comment,
-              node.lineno,
-              node.col_offset,
-              "",
-              "",
-              error="{} requires manual check.".format(name))
-    return _helper
+  def _cast_transformer(parent, node, full_name, name, logs, errors):
+    """Transforms to_int and to_float to cast(..., dtype=...)."""
 
-  @staticmethod
-  def _batch_gather_handler(file_edit_recorder, node, lines):
-    lineno = node.lineno
-    column = node.col_offset
+    # Find out the dtype to cast to from the function name
+    dtype_str = name[3:]
+    new_arg = ast.keyword(arg="dtype",
+                          value=ast.Attribute(value=ast.Name(id="tf",
+                                                             ctx=ast.Load()),
+                                              attr=dtype_str, ctx=ast.Load()))
+
+    # Python3 ast requires the args for the Attribute, but codegen will mess up
+    # the arg order if we just set them to 0.
+    new_arg.value.lineno = node.lineno
+    new_arg.value.col_offset = node.col_offset+100
 
-    # Find the position to add the batch_dims argument.  We add it as the
-    # first argument, since that's easiest.  This is safe because we included
-    # batch_gather in self.reordered_function_names, so it will have all
-    # of its arguments changed to keyword arguments.
-    m = re.match(r"tf\s*\.\s*batch_gather\s*\(", lines[lineno - 1][column:])
-    if m is not None:
-      file_edit_recorder.add(
-          "Added keyword argument 'batch_dims=-1' to 'tf.batch_gather'",
-          lineno, column + m.end(), "", "batch_dims=-1, ")
+    node.keywords.append(new_arg)
+    if isinstance(node.func, ast.Attribute):
+      node.func.attr = "cast"
     else:
-      file_edit_recorder.add(
-          "Unable to add keyword argument 'batch_dims=-1' to 'tf.batch_gather'",
-          lineno, column, "", "",
-          error="Unable to add keyword argument batch_dims=-1 to "
-          "tf.batch_gather; please add it manually.")
+      assert isinstance(node.func, ast.Name)
+      node.func.id = "cast"
+
+    logs.append((node.lineno, node.col_offset,
+                 "Changed %s call to tf.cast(..., dtype=tf.%s)." % (full_name,
+                                                                    dtype_str)))
+    return node
+
+  @staticmethod
+  def _batch_gather_transformer(parent, node, full_name, name, logs, errors):
+    # Check if the call already has a batch_dims argument
+    if any([kw.arg == "batch_dims" for kw in node.keywords]):
+      logs.append((node.lineno, node.col_offset, "tf.batch_gather already has "
+                   "batch_dims argument. Neat."))
+      return None
+
+    minus_one = ast.Num(n=-1)
+    minus_one.lineno = 0
+    minus_one.col_offset = 0
+    new_arg = ast.keyword("batch_dims", minus_one)
+    node.keywords.append(new_arg)
+    logs.append((node.lineno, node.col_offset,
+                 "Added keyword argument batch_dims=-1 to tf.batch_gather."))
+    return node
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 54120cfdd7..d8bc0ba75a 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -239,8 +239,8 @@ class TestUpgrade(test_util.TensorFlowTestCase):
     }
     function_warnings = (
         tf_upgrade_v2.TFAPIChangeSpec().function_warnings)
-    function_handles = (
-        tf_upgrade_v2.TFAPIChangeSpec().function_handle)
+    function_transformers = (
+        tf_upgrade_v2.TFAPIChangeSpec().function_transformers)
     keyword_renames = (
         tf_upgrade_v2.TFAPIChangeSpec().function_keyword_renames)
 
@@ -255,7 +255,7 @@ class TestUpgrade(test_util.TensorFlowTestCase):
 
         for name in names_v1:
           tf_name = "tf.%s" % name
-          if tf_name in function_warnings or tf_name in function_handles:
+          if tf_name in function_warnings or tf_name in function_transformers:
             continue  # These require manual change
           if tf_name in v1_name_exceptions:
             continue
@@ -362,15 +362,14 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
 
       text = "%s(a, b)\n" % decay
       _, report, errors, _ = self._upgrade(text)
-      self.assertEqual(errors, ["test.py:1: %s requires manual check." % decay])
+      self.assertIn("%s requires manual check" % decay, errors[0])
       self.assertIn("%s has been changed" % decay, report)
 
   def testPiecewiseDecay(self):
     text = "tf.train.piecewise_constant_decay(a, b)\n"
     _, report, errors, _ = self._upgrade(text)
-    self.assertEqual(
-        errors,
-        ["test.py:1: tf.train.piecewise_constant_decay requires manual check."])
+    self.assertIn("tf.train.piecewise_constant_decay requires manual check",
+                  errors[0])
     self.assertIn("tf.train.piecewise_constant_decay has been changed", report)
 
   def testMetrics(self):
@@ -414,7 +413,7 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
       text = ns + "(a, b)"
       _, report, errors, new_text = self._upgrade(text)
       self.assertEqual("tf.compat.v1.metrics." + m + "(a, b)", new_text)
-      self.assertEqual(errors, ["test.py:1: %s requires manual check." % ns])
+      self.assertIn("test.py:1:0: %s requires manual check" % ns, errors[0])
       self.assertIn(
           "WARNING: tf.metrics have been converted to object oriented"
           " versions in TF 2.0 and after. The metric function calls have been "
@@ -445,7 +444,7 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
       text = ns + "(a, b)"
       _, report, errors, new_text = self._upgrade(text)
       self.assertEqual("tf.compat.v1.losses." + l + "(a, b)", new_text)
-      self.assertEqual(errors, ["test.py:1: %s requires manual check." % ns])
+      self.assertIn("test.py:1:0: %s requires manual check" % ns, errors[0])
       self.assertIn(
           "WARNING: tf.losses have been converted to object oriented"
           " versions in TF 2.0 and after. The loss function calls have been "
@@ -463,7 +462,7 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
       text = ns + "(a, b)"
       _, report, errors, new_text = self._upgrade(text)
       self.assertEqual(text, new_text)
-      self.assertEqual(errors, ["test.py:1: %s requires manual check." % ns])
+      self.assertIn("%s requires manual check" % ns, errors[0])
       self.assertIn("loss_reduction has been changed", report)
 
   def testDropout(self):
@@ -471,15 +470,40 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
     _, unused_report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual(
         new_text,
-        "tf.nn.dropout(x, 1 - keep_prob, name=\"foo\")\n",
+        "tf.nn.dropout(x, 1 - (keep_prob), name=\"foo\")\n",
+    )
+
+    text = "tf.nn.dropout(x, keep_prob=.4, name=\"foo\")\n"
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(
+        new_text,
+        "tf.nn.dropout(x, rate=1 - (.4), name=\"foo\")\n",
+    )
+
+    text = (
+        "tf.nn.dropout(x,  # Stuff before\n"
+        "              keep_prob=.4,  # Stuff after\n"
+        "              name=\"foo\")\n"
+    )
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(
+        new_text,
+        "tf.nn.dropout(x,  # Stuff before\n"
+        "              rate=1 - (.4),  # Stuff after\n"
+        "              name=\"foo\")\n",
     )
 
     text = "tf.nn.dropout(x)\n"
     _, unused_report, errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, text)
+    self.assertIn("tf.nn.dropout called without arguments", errors[0])
+
+  def testDropoutExpr(self):
+    text = "tf.nn.dropout(x, 1 - func(3 + 4.), name=\"foo\")\n"
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual(
-        errors,
-        ["test.py:1: tf.nn.dropout requires manual check."]
+        new_text,
+        "tf.nn.dropout(x, 1 - (1 - func(3 + 4.)), name=\"foo\")\n",
     )
 
   def testCountNonZeroChanges(self):
@@ -543,9 +567,11 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
 
     text = "tf.gradients(a, colocate_gradients_with_ops=False)\n"
     _, unused_report, errors, new_text = self._upgrade(text)
-    self.assertEqual(text, new_text)
-    self.assertEqual(errors, ["test.py:1: tf.gradients requires manual check."])
+    self.assertEqual("tf.gradients(a)\n", new_text)
+    self.assertIn("tf.gradients", errors[0])
+    self.assertIn("requires manual check", errors[0])
 
+  def testColocateGradientsWithOpsMinimize(self):
     text = "optimizer.minimize(a, foo=False)\n"
     _, unused_report, errors, new_text = self._upgrade(text)
     self.assertEqual(text, new_text)
@@ -553,10 +579,11 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
 
     text = "optimizer.minimize(a, colocate_gradients_with_ops=False)\n"
     _, unused_report, errors, new_text = self._upgrade(text)
-    self.assertEqual(text, new_text)
-    self.assertEqual(errors,
-                     ["test.py:1: Optimizer.minimize requires manual check."])
+    self.assertEqual("optimizer.minimize(a)\n", new_text)
+    self.assertIn("requires manual check", errors[0])
+    self.assertIn("minimize", errors[0])
 
+  def testColocateGradientsWithOpsComputeGradients(self):
     text = "optimizer.compute_gradients(a, foo=False)\n"
     _, unused_report, errors, new_text = self._upgrade(text)
     self.assertEqual(text, new_text)
@@ -564,10 +591,9 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
 
     text = "optimizer.compute_gradients(a, colocate_gradients_with_ops=False)\n"
     _, unused_report, errors, new_text = self._upgrade(text)
-    self.assertEqual(text, new_text)
-    self.assertEqual(errors,
-                     ["test.py:1: Optimizer.compute_gradients "
-                      "requires manual check."])
+    self.assertEqual("optimizer.compute_gradients(a)\n", new_text)
+    self.assertIn("requires manual check", errors[0])
+    self.assertIn("compute_gradients", errors[0])
 
   def testExportSavedModelRename(self):
     text = "self.est.export_savedmodel(path)"
@@ -673,7 +699,7 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
     _, report, errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, expected_text)
     self.assertIn(
-        "tf.nn.softmax_cross_entropy_with_logits requires manual check.",
+        "tf.nn.softmax_cross_entropy_with_logits requires manual check",
         errors[0])
     self.assertIn(
         "tf.nn.softmax_cross_entropy_with_logits behavior has changed. ",
@@ -817,27 +843,24 @@ tf.print('abc')
 
   def testBatchGather(self):
     text = "tf.batch_gather(foo, bar)"
-    expected_text = "tf.gather(batch_dims=-1, params=foo, indices=bar)"
+    expected_text1 = "tf.gather(params=foo, indices=bar, batch_dims=-1)"
+    expected_text2 = "tf.gather(batch_dims=-1, params=foo, indices=bar)"
     _, unused_report, unused_errors, new_text = self._upgrade(text)
-    self.assertEqual(new_text, expected_text)
+    self.assertIn(new_text, [expected_text1, expected_text2])
 
     text = "tf.batch_gather(params=foo, indices=bar)"
-    expected_text = "tf.gather(batch_dims=-1, params=foo, indices=bar)"
-    _, unused_report, unused_errors, new_text = self._upgrade(text)
-    self.assertEqual(new_text, expected_text)
-
-    text = "tf.batch_gather  (  foo, bar)"
-    expected_text = "tf.gather  (batch_dims=-1,   params=foo, indices=bar)"
-    _, unused_report, unused_errors, new_text = self._upgrade(text)
-    self.assertEqual(new_text, expected_text)
-
-    text = "(tf.batch_gather\n(foo, bar))"
-    expected_text = "(tf.gather\n(params=foo, indices=bar))"
-    expected_errors = ["test.py:1: Unable to add keyword argument batch_dims=-1"
-                       " to tf.batch_gather; please add it manually."]
-    _, unused_report, errors, new_text = self._upgrade(text)
-    self.assertEqual(errors, expected_errors)
-    self.assertEqual(new_text, expected_text)
+    expected_text1 = "tf.gather(params=foo, indices=bar, batch_dims=-1)"
+    expected_text2 = "tf.gather(batch_dims=-1, params=foo, indices=bar)"
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertIn(new_text, [expected_text1, expected_text2])
+
+  def testCast(self):
+    for dtype in ["int32", "int64", "float", "double",
+                  "complex64", "complex128", "bfloat16"]:
+      text = "tf.to_%s(x, name='test')" % dtype
+      expected_text = "tf.cast(x, name='test', dtype=tf.%s)" % dtype
+      _, unused_report, unused_errors, new_text = self._upgrade(text)
+      self.assertEqual(expected_text, new_text)
 
 
 class TestUpgradeFiles(test_util.TensorFlowTestCase):
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index 93a85763f5..c51b45a49c 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -169,6 +169,7 @@ filegroup(
         "@local_config_sycl//sycl:LICENSE.text",
         "@nasm//:LICENSE",
         "@nsync//:LICENSE",
+        "@pasta//:LICENSE",
         "@pcre//:LICENCE",
         "@png_archive//:LICENSE",
         "@protobuf_archive//:LICENSE",
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 729eb8069f..b43312abb2 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -29,6 +29,7 @@ load("//third_party/jpeg:workspace.bzl", jpeg = "repo")
 load("//third_party/nasm:workspace.bzl", nasm = "repo")
 load("//third_party/kissfft:workspace.bzl", kissfft = "repo")
 load("//third_party/keras_applications_archive:workspace.bzl", keras_applications = "repo")
+load("//third_party/pasta:workspace.bzl", pasta = "repo")
 
 def initialize_third_party():
     """ Load third party repositories.  See above load() statements. """
@@ -41,6 +42,7 @@ def initialize_third_party():
     kissfft()
     jpeg()
     nasm()
+    pasta()
 
 # Sanitize a dependency so that it works correctly from code that includes
 # TensorFlow as a submodule.
diff --git a/third_party/pasta/BUILD b/third_party/pasta/BUILD
new file mode 100644
index 0000000000..9bd256a579
--- /dev/null
+++ b/third_party/pasta/BUILD
@@ -0,0 +1 @@
+# Empty BUILD file to force build system to see this directory at all.
diff --git a/third_party/pasta/BUILD.bazel b/third_party/pasta/BUILD.bazel
new file mode 100644
index 0000000000..6be33511e4
--- /dev/null
+++ b/third_party/pasta/BUILD.bazel
@@ -0,0 +1,29 @@
+# Description:
+#   AST-based python refactoring.
+
+licenses(["notice"])  # Apache2
+
+exports_files(["LICENSE"])
+
+py_library(
+    name = "pasta",
+    srcs = [
+        "__init__.py",
+        "augment/__init__.py",
+        "augment/errors.py",
+        "augment/import_utils.py",
+        "augment/inline.py",
+        "augment/rename.py",
+        "base/__init__.py",
+        "base/annotate.py",
+        "base/ast_constants.py",
+        "base/ast_utils.py",
+        "base/codegen.py",
+        "base/formatting.py",
+        "base/scope.py",
+        "base/test_utils.py",
+        "base/token_generator.py",
+    ],
+    srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
+)
diff --git a/third_party/pasta/BUILD.system b/third_party/pasta/BUILD.system
new file mode 100644
index 0000000000..6adc953c5a
--- /dev/null
+++ b/third_party/pasta/BUILD.system
@@ -0,0 +1,13 @@
+# Description: Pasta, AST based python refactoring.
+
+licenses(["notice"])  # Apache2
+
+filegroup(
+    name = "LICENSE",
+    visibility = ["//visibility:public"],
+)
+
+py_library(
+    name = "pasta",
+    visibility = ["//visibility:public"],
+)
diff --git a/third_party/pasta/workspace.bzl b/third_party/pasta/workspace.bzl
new file mode 100644
index 0000000000..063bdb7b98
--- /dev/null
+++ b/third_party/pasta/workspace.bzl
@@ -0,0 +1,16 @@
+"""Loads pasta python package."""
+
+load("//third_party:repo.bzl", "third_party_http_archive")
+
+def repo():
+    third_party_http_archive(
+        name = "pasta",
+        urls = [
+            "https://mirror.bazel.build/github.com/google/pasta/archive/c3d72cdee6fc806251949e912510444d58d7413c.tar.gz",
+            "https://github.com/google/pasta/archive/c3d72cdee6fc806251949e912510444d58d7413c.tar.gz",
+        ],
+        strip_prefix = "pasta-c3d72cdee6fc806251949e912510444d58d7413c/pasta",
+        sha256 = "b5905f9cecc4b28363c563f3c4cb0545288bd35f7cc72c55066e97e53befc084",
+        build_file = "//third_party/pasta:BUILD.bazel",
+        system_build_file = "//third_party/pasta:BUILD.system",
+    )
-- 
GitLab


From 4b90409c6b9e11d0bf0517a8fc49f9826167cb0a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 17:04:27 -0800
Subject: [PATCH 0228/2345] Several fixes/improvements to wrap_function: 1.
 Makes gradient tape work on captures post-pruning 2. Allows users to prune
 with wrapped_func.inputs even when the func includes captures 3. Makes sure
 to write variables to collections for better legacy compatibility in
 compat.v1 (currently if the outer init_scope is eager, variables won't be
 written to the global_variables / trainable_variables collections)

PiperOrigin-RevId: 227936352
---
 tensorflow/python/eager/wrap_function.py      | 23 +++++++++-
 tensorflow/python/eager/wrap_function_test.py | 44 +++++++++++++++++++
 2 files changed, 66 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/eager/wrap_function.py b/tensorflow/python/eager/wrap_function.py
index 0930b6116d..2c65e97871 100644
--- a/tensorflow/python/eager/wrap_function.py
+++ b/tensorflow/python/eager/wrap_function.py
@@ -38,8 +38,20 @@ class VariableHolder(object):
     self._variables = []
 
   def variable_creator_scope(self, next_creator, **kwargs):
+    """Creates variables & adds them to collections to match legacy code."""
     v = next_creator(**kwargs)
     self._variables.append(v)
+
+    collections = kwargs.get("collections")
+    trainable = v.trainable
+
+    if collections is None:
+      collections = [ops.GraphKeys.GLOBAL_VARIABLES]
+    if trainable and ops.GraphKeys.TRAINABLE_VARIABLES not in collections:
+      collections = list(collections) + [ops.GraphKeys.TRAINABLE_VARIABLES]
+
+    ops.add_to_collections(collections, v)
+
     return v
 
   def __call__(self, *args, **kwargs):
@@ -61,6 +73,13 @@ class WrappedFunction(function.Function):
     for f in flat_feeds:
       if not isinstance(f, ops.Tensor):
         raise ValueError("Feeds must be tensors.")
+
+    # Ignoring all feeds that are captures allows prune to be called
+    # using wrapped_func.inputs even when it uses variables
+    internal_captures = self.graph.internal_captures
+    flat_feeds = [f for f in flat_feeds
+                  if f not in internal_captures]
+
     tensor_fetches = []
     operation_fetches = []
     for f in flat_fetches:
@@ -87,7 +106,7 @@ class WrappedFunction(function.Function):
           sink_tensor = control_flow_ops.no_op()
     lift_map = lift_to_graph.lift_to_graph(
         sink_tensor, pruned_graph,
-        sources=flat_feeds + self.graph.internal_captures)
+        sources=flat_feeds + internal_captures)
     for original_fetch, identity_fetch in zip(
         tensor_fetches, identity_fetches):
       lift_map[original_fetch] = lift_map[identity_fetch]
@@ -98,6 +117,8 @@ class WrappedFunction(function.Function):
     pruned_graph.inputs.extend(lift_map[x] for x in flat_feeds)
     pruned_graph.inputs.extend(pruned_graph.captures.values())
 
+    pruned_graph.variables = self.graph.variables
+
     def _structured_output_mapping(fetched):
       lifted = lift_map[fetched]
       if isinstance(lifted, ops.Operation):
diff --git a/tensorflow/python/eager/wrap_function_test.py b/tensorflow/python/eager/wrap_function_test.py
index 65dd73aafc..17e204b4d8 100644
--- a/tensorflow/python/eager/wrap_function_test.py
+++ b/tensorflow/python/eager/wrap_function_test.py
@@ -18,6 +18,7 @@ from __future__ import division
 from __future__ import print_function
 
 
+from tensorflow.python.eager import backprop
 from tensorflow.python.eager import wrap_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -90,11 +91,54 @@ class WrapFunctionTest(test.TestCase):
 
     f_wrapped = wrap_function.wrap_function(f, [])
     self.assertAllEqual(6.0, f_wrapped())
+
+    # Test pruning directly on the inputs
+    pruned = f_wrapped.prune(
+        feeds=f_wrapped.inputs,
+        fetches=f_wrapped.graph.get_tensor_by_name('fetch:0'))
+    self.assertAllEqual(6.0, pruned())
+
+    # Test pruning with no inputs
     pruned = f_wrapped.prune(
         feeds=(),
         fetches=f_wrapped.graph.get_tensor_by_name('fetch:0'))
     self.assertAllEqual(6.0, pruned())
 
+  def testGradientsOfPrune(self):
+
+    v1 = variables.Variable(2.)
+    v2_holder = []
+
+    def f(z):
+      v2 = variables.Variable(3.)
+      v2_holder.append(v2)
+      return array_ops.identity(v1 * v2 * z, 'fetch')
+
+    f_wrapped = wrap_function.wrap_function(
+        f, [tensor_spec.TensorSpec((), dtype=dtypes.float32)])
+
+    x = constant_op.constant(1.)
+    with backprop.GradientTape() as tape:
+      tape.watch(x)
+      out = f_wrapped(x)
+    grads = tape.gradient(out, [x, v1, v2_holder[0]])
+
+    self.assertAllEqual(6.0, out)
+    self.assertAllEqual([6.0, 3.0, 2.0], grads)
+
+    pruned = f_wrapped.prune(
+        feeds=f_wrapped.inputs,
+        fetches=f_wrapped.graph.get_tensor_by_name('fetch:0'))
+
+    x = constant_op.constant(1.)
+    with backprop.GradientTape() as tape:
+      tape.watch(x)
+      out = pruned(x)
+    grads = tape.gradient(out, [x, v1, v2_holder[0]])
+
+    self.assertAllEqual(6.0, out)
+    self.assertAllEqual([6.0, 3.0, 2.0], grads)
+
   def testPruneOperations(self):
 
     v = variables.Variable(0)
-- 
GitLab


From 542e30bab143034047ee18a0b060648be78df58c Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Fri, 4 Jan 2019 17:11:01 -0800
Subject: [PATCH 0229/2345] Enable nnapi_delegate_test in oss builds

PiperOrigin-RevId: 227937061
---
 tensorflow/lite/delegates/nnapi/BUILD | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/delegates/nnapi/BUILD b/tensorflow/lite/delegates/nnapi/BUILD
index fd954ba222..dda3c02567 100644
--- a/tensorflow/lite/delegates/nnapi/BUILD
+++ b/tensorflow/lite/delegates/nnapi/BUILD
@@ -3,6 +3,7 @@ package(default_visibility = [
 ])
 
 load("//tensorflow:tensorflow.bzl", "tf_cc_test")
+load("//tensorflow/lite:special_rules.bzl", "tflite_portable_test_suite")
 
 licenses(["notice"])  # Apache 2.0
 
@@ -23,7 +24,7 @@ tf_cc_test(
     name = "nnapi_delegate_test",
     size = "small",
     srcs = ["nnapi_delegate_test.cc"],
-    tags = ["no_oss"],
+    tags = ["tflite_not_portable_ios"],
     deps = [
         ":nnapi_delegate",
         "//tensorflow/lite:framework",
@@ -32,3 +33,5 @@ tf_cc_test(
         "@com_google_googletest//:gtest",
     ],
 )
+
+tflite_portable_test_suite()
-- 
GitLab


From 774ede5790c6cf11da05ca1082e9870db7b843f9 Mon Sep 17 00:00:00 2001
From: Karmel Allison <karmel@google.com>
Date: Fri, 4 Jan 2019 17:23:06 -0800
Subject: [PATCH 0230/2345] Consolidated duplicate tests for model cloning and
 added Keras test decorators where appropriate.

Updated models_tests to run in v1/v2. This required several other changes:
* Added a testing utility to get v2 optimizers
* Ensured that subclassed models cleared key compilation attrs upon clone_reset (_is_compiled; optimizer)
* Modified train/eval loops to check self._is_compiled rather than the existence of other attrs created during compiling, as these other attrs would not be reset during cloning.

PiperOrigin-RevId: 227938218
---
 tensorflow/python/keras/BUILD              |   7 +-
 tensorflow/python/keras/engine/training.py |   4 +-
 tensorflow/python/keras/models.py          |  44 +-
 tensorflow/python/keras/models_test.py     | 734 ++++++++-------------
 tensorflow/python/keras/testing_utils.py   |  56 +-
 5 files changed, 375 insertions(+), 470 deletions(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 914568f652..3ffff61c2b 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -1,5 +1,7 @@
 # Description:
 #   Contains the Keras API (internal TensorFlow version).
+load("//tensorflow:tensorflow.bzl", "tf_py_test")
+load("//tensorflow:tensorflow.bzl", "cuda_py_test")
 
 licenses(["notice"])  # Apache 2.0
 
@@ -7,9 +9,6 @@ package(default_visibility = ["//visibility:public"])
 
 exports_files(["LICENSE"])
 
-load("//tensorflow:tensorflow.bzl", "tf_py_test")
-load("//tensorflow:tensorflow.bzl", "cuda_py_test")
-
 config_setting(
     name = "empty_condition",
     values = {"define": "UNUSED=unused"},
@@ -1022,7 +1021,7 @@ tf_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:training",
     ],
-    shard_count = 2,
+    shard_count = 8,
     tags = ["notsan"],  # b/67509773
 )
 
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 8263467c37..bd06751428 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -1991,7 +1991,7 @@ class Model(Network):
           ' without calling `model.compile` after ?', 1)
 
   def _make_train_function_helper(self, fn_name, outputs, metric_updates=None):
-    if not hasattr(self, fn_name):
+    if not self._is_compiled:
       raise RuntimeError('You must compile your model before using it.')
     self._check_trainable_weights_consistency()
     if getattr(self, fn_name) is None:
@@ -2040,7 +2040,7 @@ class Model(Network):
         '_fit_function', [self.total_loss] + metrics_tensors)
 
   def _make_test_function_helper(self, fn_name, outputs, metric_updates=None):
-    if not hasattr(self, fn_name):
+    if not self._is_compiled:
       raise RuntimeError('You must compile your model before using it.')
     if getattr(self, fn_name) is None:
       inputs = (self._feed_inputs +
diff --git a/tensorflow/python/keras/models.py b/tensorflow/python/keras/models.py
index f36c89041c..851de6e71f 100644
--- a/tensorflow/python/keras/models.py
+++ b/tensorflow/python/keras/models.py
@@ -281,8 +281,6 @@ def clone_model(model, input_tensors=None):
 
 
 # "Clone" a subclassed model by reseting all of the attributes.
-
-
 def _in_place_subclassed_model_reset(model):
   """Substitute for model cloning that works for subclassed models.
 
@@ -382,11 +380,30 @@ def _in_place_subclassed_model_reset(model):
       for name in attributes_to_cache:
         attributes_cache[name] = getattr(model, name)
   model._original_attributes_cache = attributes_cache
-  # Reset built state
+  _reset_build_compile_trackers(model)
+  model._setattr_tracking = setattr_tracking
+
+
+def _reset_build_compile_trackers(model):
+  """Reset state trackers for model.
+
+  Note that we do not actually zero out attributes such as optimizer,
+  but instead rely on the expectation that all of the attrs will be
+  over-written on calling build/compile/etc. This is somewhat fragile,
+  insofar as we check elsewhere for the presence of these attributes as
+  evidence of having been built/compiled/etc. Pending a better way to do this,
+  we reset key attributes here to allow building and compiling.
+
+  Args:
+    model: the model that is being reset
+  """
+  # Reset build state
   model.built = False
   model.inputs = None
   model.outputs = None
-  model._setattr_tracking = setattr_tracking
+  # Reset compile state
+  model._is_compiled = False  # pylint:disable=protected-access
+  model.optimizer = None
 
 
 def in_place_subclassed_model_state_restoration(model):
@@ -418,9 +435,7 @@ def in_place_subclassed_model_state_restoration(model):
     model._setattr_tracking = setattr_tracking
   else:
     # Restore to the state of a never-called model.
-    model.built = False
-    model.inputs = None
-    model.outputs = None
+    _reset_build_compile_trackers(model)
 
 
 def clone_and_build_model(
@@ -462,7 +477,10 @@ def clone_and_build_model(
       - cloning a subclassed model with `in_place_reset` set to False.
       - compiling the clone when the original model has not been compiled.
   """
-  if compile_clone and not model.optimizer:
+  # Grab optimizer now, as we reset-in-place for subclassed models, but
+  # want to maintain access to the original optimizer.
+  orig_optimizer = model.optimizer
+  if compile_clone and not orig_optimizer:
     raise ValueError(
         'Error when cloning model: compile_clone was set to True, but the '
         'original model has not been compiled.')
@@ -498,14 +516,14 @@ def clone_and_build_model(
         input_tensors = input_tensors[0]
       clone._set_inputs(input_tensors)
 
-  if compile_clone and model.optimizer:
-    if isinstance(model.optimizer, optimizers.TFOptimizer):
+  if compile_clone:
+    if isinstance(orig_optimizer, optimizers.TFOptimizer):
       optimizer = optimizers.TFOptimizer(
-          model.optimizer.optimizer, optimizer_iterations)
+          orig_optimizer.optimizer, optimizer_iterations)
       K.track_tf_optimizer(optimizer)
     else:
-      optimizer_config = model.optimizer.get_config()
-      optimizer = model.optimizer.__class__.from_config(optimizer_config)
+      optimizer_config = orig_optimizer.get_config()
+      optimizer = orig_optimizer.__class__.from_config(optimizer_config)
       if optimizer_iterations is not None:
         optimizer.iterations = optimizer_iterations
 
diff --git a/tensorflow/python/keras/models_test.py b/tensorflow/python/keras/models_test.py
index 0a5f9a7bea..3eab10f624 100644
--- a/tensorflow/python/keras/models_test.py
+++ b/tensorflow/python/keras/models_test.py
@@ -19,18 +19,21 @@ from __future__ import division
 from __future__ import print_function
 
 import copy
+import functools
 import os
 
+from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.python import keras
 from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import test_util
 from tensorflow.python.keras import backend as K
+from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import metrics
 from tensorflow.python.keras import models
+from tensorflow.python.keras import testing_utils
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import resource_variable_ops
@@ -52,158 +55,181 @@ class TestModel(keras.Model):
     return self.layer1(x)
 
 
-def sequential_model(add_input_layer, include_input_shape=True):
-  model = keras.models.Sequential()
+def _get_layers(input_shape=(4,), add_input_layer=False):
   if add_input_layer:
-    model.add(keras.layers.InputLayer(input_shape=(4,)))
-    model.add(keras.layers.Dense(4))
-  elif include_input_shape:
-    model.add(keras.layers.Dense(4, input_shape=(4,)))
+    model_layers = [keras.layers.InputLayer(input_shape=input_shape),
+                    keras.layers.Dense(4)]
+  elif input_shape:
+    model_layers = [keras.layers.Dense(4, input_shape=input_shape)]
   else:
-    model.add(keras.layers.Dense(4))
-  model.add(keras.layers.BatchNormalization())
-  model.add(keras.layers.Dropout(0.5))
-  model.add(keras.layers.Dense(4))
-  return model
-
-
-class TestModelCloning(test.TestCase):
-
-  @test_util.run_v1_only('b/120545219')
-  def test_clone_sequential_model(self):
-    with self.cached_session():
-      val_a = np.random.random((10, 4))
-      val_out = np.random.random((10, 4))
-
-      model = sequential_model(False)
-
-    # Everything should work in a new session.
-    keras.backend.clear_session()
-
-    with self.cached_session():
-      # With placeholder creation
-      new_model = keras.models.clone_model(model)
+    model_layers = [keras.layers.Dense(4)]
+
+  model_layers += [
+      keras.layers.BatchNormalization(),
+      keras.layers.Dropout(0.5),
+      keras.layers.Dense(4)]
+
+  return model_layers
+
+
+def _get_model(input_shape=(4,)):
+  model_layers = _get_layers(input_shape=None, add_input_layer=False)
+  return testing_utils.get_model_from_layers(
+      model_layers, input_shape=input_shape)
+
+
+class TestModelCloning(keras_parameterized.TestCase):
+
+  @keras_parameterized.run_all_keras_modes
+  @parameterized.named_parameters([
+      {'testcase_name': 'has_input_layer',
+       'input_shape': (4,),
+       'add_input_layer': True,
+       'share_weights': False},
+      {'testcase_name': 'no_input_layer',
+       'input_shape': None,
+       'add_input_layer': False,
+       'share_weights': False},
+      {'testcase_name': 'has_input_layer_share_weights',
+       'input_shape': (4,),
+       'add_input_layer': True,
+       'share_weights': True},
+      {'testcase_name': 'no_input_layer_share_weights',
+       'input_shape': None,
+       'add_input_layer': False,
+       'share_weights': True},
+  ])
+  def test_clone_sequential_model(
+      self, input_shape, add_input_layer, share_weights):
+
+    if share_weights:
+      clone_fn = functools.partial(
+          keras.models._clone_sequential_model, share_weights=True)
+    else:
+      clone_fn = keras.models.clone_model
+
+    val_a = np.random.random((10, 4))
+    model = models.Sequential(_get_layers(input_shape, add_input_layer))
+    # Sanity check
+    self.assertEqual(
+        isinstance(model._layers[0], keras.layers.InputLayer),
+        add_input_layer)
+    self.assertEqual(model._is_graph_network, add_input_layer)
+
+    # With placeholder creation -- clone model should have an InputLayer
+    # if the original model has one.
+    new_model = clone_fn(model)
+    self.assertEqual(
+        isinstance(new_model._layers[0], keras.layers.InputLayer),
+        add_input_layer)
+    self.assertEqual(new_model._is_graph_network, model._is_graph_network)
+    if input_shape:
       # update ops from batch norm needs to be included
       self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
-      new_model.compile('rmsprop', 'mse')
-      new_model.train_on_batch(val_a, val_out)
-
-      # On top of new tensor
-      input_a = keras.Input(shape=(4,))
-      new_model = keras.models.clone_model(model, input_tensors=input_a)
-      self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
-      new_model.compile('rmsprop', 'mse')
-      new_model.train_on_batch(val_a, val_out)
 
-      # On top of new, non-Keras tensor
+    # On top of new tensor  -- clone model should always have an InputLayer.
+    input_a = keras.Input(shape=(4,))
+    new_model = clone_fn(model, input_tensors=input_a)
+    self.assertIsInstance(new_model._layers[0], keras.layers.InputLayer)
+    self.assertTrue(new_model._is_graph_network)
+
+    # On top of new, non-Keras tensor  -- clone model should always have an
+    # InputLayer.
+    if not context.executing_eagerly():
+      # TODO(b/121277734):Skip Eager contexts, as Input() layers raise an error
+      # saying they should not be used with EagerTensors
       input_a = keras.backend.variable(val_a)
-      new_model = keras.models.clone_model(model, input_tensors=input_a)
-      self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
-      new_model.compile('rmsprop', 'mse')
-      new_model.train_on_batch(None, val_out)
-
-  @test_util.run_v1_only('b/120545219')
-  def test_clone_sequential_model_input_layer(self):
-
-    def test_input_layer(include_inputs):
-      with self.cached_session():
-        val_a = np.random.random((10, 4))
-        model = sequential_model(include_inputs, include_inputs)
-        # Sanity check
-        self.assertEqual(
-            isinstance(model._layers[0], keras.layers.InputLayer),
-            include_inputs)
-        self.assertEqual(model._is_graph_network, include_inputs)
-
-      keras.backend.clear_session()
-      with self.cached_session():
-        # With placeholder creation -- clone model should have an InputLayer
-        # if the original model has one.
-        new_model = keras.models.clone_model(model)
-        self.assertEqual(
-            isinstance(new_model._layers[0], keras.layers.InputLayer),
-            include_inputs)
-        self.assertEqual(new_model._is_graph_network, model._is_graph_network)
-
-        # On top of new tensor  -- clone model should always have an InputLayer.
-        input_a = keras.Input(shape=(4,))
-        new_model = keras.models.clone_model(model, input_tensors=input_a)
-        self.assertIsInstance(new_model._layers[0], keras.layers.InputLayer)
-        self.assertTrue(new_model._is_graph_network)
-
-        # On top of new, non-Keras tensor  -- clone model should always have an
-        # InputLayer.
-        input_a = keras.backend.variable(val_a)
-        new_model = keras.models.clone_model(model, input_tensors=input_a)
-        self.assertIsInstance(new_model._layers[0], keras.layers.InputLayer)
-        self.assertTrue(new_model._is_graph_network)
-
-    test_input_layer(True)
-    test_input_layer(False)
-
-  @test_util.run_v1_only('b/120545219')
-  def test_clone_functional_model(self):
-    with self.cached_session():
-      val_a = np.random.random((10, 4))
-      val_b = np.random.random((10, 4))
-      val_out = np.random.random((10, 4))
-
-      input_a = keras.Input(shape=(4,))
-      input_b = keras.Input(shape=(4,))
-      dense_1 = keras.layers.Dense(4,)
-      dense_2 = keras.layers.Dense(4,)
-
-      x_a = dense_1(input_a)
-      x_a = keras.layers.Dropout(0.5)(x_a)
-      x_a = keras.layers.BatchNormalization()(x_a)
-      x_b = dense_1(input_b)
-      x_a = dense_2(x_a)
-      outputs = keras.layers.add([x_a, x_b])
-      model = keras.models.Model([input_a, input_b], outputs)
-
-    # Everything should work in a new session.
-    keras.backend.clear_session()
-
-    with self.cached_session():
-      # With placeholder creation
-      new_model = keras.models.clone_model(model)
-      self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
-      new_model.compile('rmsprop', 'mse')
-      new_model.train_on_batch([val_a, val_b], val_out)
-
-      # On top of new tensors
-      input_a = keras.Input(shape=(4,), name='a')
-      input_b = keras.Input(shape=(4,), name='b')
-      new_model = keras.models.clone_model(
-          model, input_tensors=[input_a, input_b])
-      self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
-      new_model.compile('rmsprop', 'mse')
-      new_model.train_on_batch([val_a, val_b], val_out)
-
-      # On top of new, non-Keras tensors
+      new_model = clone_fn(model, input_tensors=input_a)
+      self.assertIsInstance(new_model._layers[0], keras.layers.InputLayer)
+      self.assertTrue(new_model._is_graph_network)
+
+  @keras_parameterized.run_all_keras_modes
+  @parameterized.named_parameters([
+      {'testcase_name': 'clone_weights', 'share_weights': False},
+      {'testcase_name': 'share_weights', 'share_weights': True},
+  ])
+  def test_clone_functional_model(self, share_weights):
+    if share_weights:
+      clone_fn = functools.partial(
+          keras.models._clone_functional_model, share_weights=True)
+    else:
+      clone_fn = keras.models.clone_model
+
+    val_a = np.random.random((10, 4))
+    val_b = np.random.random((10, 4))
+    val_out = np.random.random((10, 4))
+
+    input_a = keras.Input(shape=(4,))
+    input_b = keras.Input(shape=(4,))
+    dense_1 = keras.layers.Dense(4,)
+    dense_2 = keras.layers.Dense(4,)
+
+    x_a = dense_1(input_a)
+    x_a = keras.layers.Dropout(0.5)(x_a)
+    x_a = keras.layers.BatchNormalization()(x_a)
+    x_b = dense_1(input_b)
+    x_a = dense_2(x_a)
+    outputs = keras.layers.add([x_a, x_b])
+    model = keras.models.Model([input_a, input_b], outputs)
+
+    # With placeholder creation
+    new_model = clone_fn(model)
+    self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
+    new_model.compile(
+        testing_utils.get_v2_optimizer('rmsprop'), 'mse',
+        run_eagerly=testing_utils.should_run_eagerly())
+    new_model.train_on_batch([val_a, val_b], val_out)
+
+    # On top of new tensors
+    input_a = keras.Input(shape=(4,), name='a')
+    input_b = keras.Input(shape=(4,), name='b')
+    new_model = keras.models.clone_model(
+        model, input_tensors=[input_a, input_b])
+    self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
+    new_model.compile(
+        testing_utils.get_v2_optimizer('rmsprop'), 'mse',
+        run_eagerly=testing_utils.should_run_eagerly())
+    new_model.train_on_batch([val_a, val_b], val_out)
+
+    # On top of new, non-Keras tensors
+    if not context.executing_eagerly():
+      # TODO(b/121277734):Skip Eager contexts, as Input() layers raise an error
+      # saying they should not be used with EagerTensors
       input_a = keras.backend.variable(val_a)
       input_b = keras.backend.variable(val_b)
-      new_model = keras.models.clone_model(
-          model, input_tensors=[input_a, input_b])
+      new_model = clone_fn(model, input_tensors=[input_a, input_b])
       self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
-      new_model.compile('rmsprop', 'mse')
+      new_model.compile(
+          testing_utils.get_v2_optimizer('rmsprop'), 'mse',
+          run_eagerly=testing_utils.should_run_eagerly())
       new_model.train_on_batch(None, val_out)
 
-  @test_util.run_in_graph_and_eager_modes
-  def test_clone_functional_model_with_masking(self):
-    with self.cached_session():
-      x = np.array([[[1], [1]], [[0], [0]]])
-      inputs = keras.Input((2, 1))
-      outputs = keras.layers.Masking(mask_value=0)(inputs)
-      outputs = keras.layers.TimeDistributed(
-          keras.layers.Dense(1, kernel_initializer='one'))(outputs)
-      model = keras.Model(inputs, outputs)
-
-      model = keras.models.clone_model(model)
-      model.compile(loss='mse', optimizer=adam.AdamOptimizer(0.01))
-      y = np.array([[[1], [1]], [[1], [1]]])
-      loss = model.train_on_batch(x, y)
-      self.assertEqual(float(loss), 0.)
+  @keras_parameterized.run_all_keras_modes
+  @parameterized.named_parameters([
+      {'testcase_name': 'clone_weights', 'share_weights': False},
+      {'testcase_name': 'share_weights', 'share_weights': True},
+  ])
+  def test_clone_functional_with_masking(self, share_weights):
+    if share_weights:
+      clone_fn = functools.partial(
+          keras.models._clone_functional_model, share_weights=True)
+    else:
+      clone_fn = keras.models.clone_model
+
+    x = np.array([[[1.], [1.]], [[0.], [0.]]])
+    inputs = keras.Input((2, 1))
+    outputs = keras.layers.Masking(mask_value=0)(inputs)
+    outputs = keras.layers.TimeDistributed(
+        keras.layers.Dense(1, kernel_initializer='one'))(outputs)
+    model = keras.Model(inputs, outputs)
+
+    model = clone_fn(model)
+    model.compile(
+        loss='mse', optimizer=testing_utils.get_v2_optimizer('adam'),
+        run_eagerly=testing_utils.should_run_eagerly())
+    y = np.array([[[1], [1]], [[1], [1]]])
+    loss = model.train_on_batch(x, y)
+    self.assertEqual(float(loss), 0.)
 
   def test_model_cloning_invalid_use_cases(self):
     seq_model = keras.models.Sequential()
@@ -249,168 +275,23 @@ class TestModelCloning(test.TestCase):
       self.assertFalse(has_placeholder)
 
 
-class TestModelCloningLayerPreserveWeights(test.TestCase):
-
-  @test_util.run_deprecated_v1
-  def test_clone_sequential_model(self):
-    with self.cached_session():
-      val_a = np.random.random((10, 4))
-      val_out = np.random.random((10, 4))
-
-      model = sequential_model(False)
-
-    # Everything should work in a new session.
-    keras.backend.clear_session()
-
-    with self.cached_session():
-      # With placeholder creation
-      new_model = keras.models._clone_sequential_model(
-          model, share_weights=True)
-      # update ops from batch norm needs to be included
-      self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
-      new_model.compile('rmsprop', 'mse')
-      new_model.train_on_batch(val_a, val_out)
-
-      # On top of new tensor
-      input_a = keras.Input(shape=(4,))
-      new_model = keras.models._clone_sequential_model(
-          model, input_tensors=input_a, share_weights=True)
-      self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
-      new_model.compile('rmsprop', 'mse')
-      new_model.train_on_batch(val_a, val_out)
-
-      # On top of new, non-Keras tensor
-      input_a = keras.backend.variable(val_a)
-      new_model = keras.models._clone_sequential_model(
-          model, input_tensors=input_a, share_weights=True)
-      self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
-      new_model.compile('rmsprop', 'mse')
-      new_model.train_on_batch(None, val_out)
-
-  @test_util.run_deprecated_v1
-  def test_clone_sequential_model_input_layer(self):
-
-    @test_util.run_deprecated_v1
-    def test_input_layer(include_inputs):
-      with self.cached_session():
-        val_a = np.random.random((10, 4))
-        model = sequential_model(include_inputs, include_inputs)
-        # Sanity check
-        self.assertEqual(
-            isinstance(model._layers[0], keras.layers.InputLayer),
-            include_inputs)
-        self.assertEqual(model._is_graph_network, include_inputs)
-
-      keras.backend.clear_session()
-      with self.cached_session():
-        # With placeholder creation -- clone model should have an InputLayer
-        # if the original model has one.
-        new_model = keras.models._clone_sequential_model(
-            model, share_weights=True)
-        self.assertEqual(
-            isinstance(new_model._layers[0], keras.layers.InputLayer),
-            include_inputs)
-        self.assertEqual(new_model._is_graph_network, model._is_graph_network)
-
-        # On top of new tensor  -- clone model should always have an InputLayer.
-        input_a = keras.Input(shape=(4,))
-        new_model = keras.models._clone_sequential_model(
-            model, input_tensors=input_a, share_weights=True)
-        self.assertIsInstance(new_model._layers[0], keras.layers.InputLayer)
-        self.assertTrue(new_model._is_graph_network)
-
-        # On top of new, non-Keras tensor  -- clone model should always have an
-        # InputLayer.
-        input_a = keras.backend.variable(val_a)
-        new_model = keras.models._clone_sequential_model(
-            model, input_tensors=input_a, share_weights=True)
-        self.assertIsInstance(new_model._layers[0], keras.layers.InputLayer)
-        self.assertTrue(new_model._is_graph_network)
-
-    test_input_layer(True)
-    test_input_layer(False)
-
-  @test_util.run_deprecated_v1
-  def test_clone_functional_model(self):
-    with self.cached_session():
-      val_a = np.random.random((10, 4))
-      val_b = np.random.random((10, 4))
-      val_out = np.random.random((10, 4))
-
-      input_a = keras.Input(shape=(4,))
-      input_b = keras.Input(shape=(4,))
-      dense_1 = keras.layers.Dense(4,)
-      dense_2 = keras.layers.Dense(4,)
-
-      x_a = dense_1(input_a)
-      x_a = keras.layers.Dropout(0.5)(x_a)
-      x_a = keras.layers.BatchNormalization()(x_a)
-      x_b = dense_1(input_b)
-      x_a = dense_2(x_a)
-      outputs = keras.layers.add([x_a, x_b])
-      model = keras.models.Model([input_a, input_b], outputs)
-
-    # Everything should work in a new session.
-    keras.backend.clear_session()
-
-    with self.cached_session():
-      # With placeholder creation
-      new_model = keras.models._clone_functional_model(
-          model, share_weights=True)
-      self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
-      new_model.compile('rmsprop', 'mse')
-      new_model.train_on_batch([val_a, val_b], val_out)
-
-      # On top of new tensors
-      input_a = keras.Input(shape=(4,), name='a')
-      input_b = keras.Input(shape=(4,), name='b')
-      new_model = keras.models._clone_functional_model(
-          model, input_tensors=[input_a, input_b], share_weights=True)
-      self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
-      new_model.compile('rmsprop', 'mse')
-      new_model.train_on_batch([val_a, val_b], val_out)
-
-      # On top of new, non-Keras tensors
-      input_a = keras.backend.variable(val_a)
-      input_b = keras.backend.variable(val_b)
-      new_model = keras.models._clone_functional_model(
-          model, input_tensors=[input_a, input_b], share_weights=True)
-      self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
-      new_model.compile('rmsprop', 'mse')
-      new_model.train_on_batch(None, val_out)
-
-  @test_util.run_in_graph_and_eager_modes
-  def test_clone_functional_model_with_masking(self):
-    with self.cached_session():
-      x = np.array([[[1], [1]], [[0], [0]]])
-      inputs = keras.Input((2, 1))
-      outputs = keras.layers.Masking(mask_value=0)(inputs)
-      outputs = keras.layers.TimeDistributed(
-          keras.layers.Dense(1, kernel_initializer='one'))(outputs)
-      model = keras.Model(inputs, outputs)
-
-      model = keras.models._clone_functional_model(
-          model, share_weights=True)
-      model.compile(loss='mse', optimizer=adam.AdamOptimizer(0.01))
-      y = np.array([[[1], [1]], [[1], [1]]])
-      loss = model.train_on_batch(x, y)
-      self.assertEqual(float(loss), 0.)
-
-
 def _has_placeholder(graph):
   ops_types = [op.type for op in graph.get_operations()]
   return any('Placeholder' in s for s in ops_types)
 
 
-class CheckpointingTests(test.TestCase):
+@keras_parameterized.run_with_all_model_types
+@keras_parameterized.run_all_keras_modes
+class CheckpointingTests(keras_parameterized.TestCase):
 
-  @test_util.run_in_graph_and_eager_modes
   def test_optimizer_dependency(self):
-    model = keras.models.Sequential()
-    model.add(keras.layers.Dense(1, input_shape=(4,)))
-    opt = adam.AdamOptimizer(0.01)
-    model.compile(optimizer=opt, loss='mse')
-    model.fit(x=np.array([[1., 2., 3., 4.]]), y=[1.], epochs=2)
+    model = _get_model()
+    opt = adam.AdamOptimizer(.01)
+    model.compile(
+        optimizer=opt, loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
+
+    model.fit(x=np.array([[1., 2., 3., 4.]]), y=np.array([1.]), epochs=2)
     save_prefix = os.path.join(self.get_temp_dir(), 'ckpt')
     beta1_power, _ = opt._get_beta_accumulators()
     self.evaluate(beta1_power.assign(12.))
@@ -420,7 +301,8 @@ class CheckpointingTests(test.TestCase):
     self.assertEqual(12., self.evaluate(beta1_power))
 
 
-class TestModelBackend(test.TestCase):
+@keras_parameterized.run_all_keras_modes
+class TestModelBackend(keras_parameterized.TestCase):
 
   def test_model_backend_float64_use_cases(self):
     # Test case for GitHub issue 19318
@@ -430,7 +312,9 @@ class TestModelBackend(test.TestCase):
     x = keras.Input((5,))
     y = keras.layers.Dense(1)(x)
     model = keras.models.Model(x, y)
-    model.compile('rmsprop', 'mse')
+    model.compile(
+        testing_utils.get_v2_optimizer('rmsprop'), 'mse',
+        run_eagerly=testing_utils.should_run_eagerly())
 
     keras.backend.set_floatx(floatx)
 
@@ -465,48 +349,46 @@ class TestModelDeepCopy(test.TestCase):
                       model_copy.get_weights()[0]))
 
 
-@test_util.run_v1_only('b/120545219')
-class TestCloneAndBuildModel(test.TestCase):
+@keras_parameterized.run_all_keras_modes
+class TestCloneAndBuildModel(keras_parameterized.TestCase):
 
+  @keras_parameterized.run_with_all_model_types
   def test_clone_and_build_non_compiled_model(self):
-    with self.cached_session():
-      inp = np.random.random((10, 4))
-      out = np.random.random((10, 4))
+    inp = np.random.random((10, 4))
+    out = np.random.random((10, 4))
 
-      model = keras.models.Sequential()
-      model.add(keras.layers.Dense(4, input_shape=(4,)))
-      model.add(keras.layers.BatchNormalization())
-      model.add(keras.layers.Dropout(0.5))
-      model.add(keras.layers.Dense(4))
-
-    # Everything should work in a new session.
-    keras.backend.clear_session()
-
-    with self.cached_session():
-      with self.assertRaisesRegexp(ValueError, 'has not been compiled'):
-        models.clone_and_build_model(model, compile_clone=True)
-
-      # With placeholder creation
-      new_model = models.clone_and_build_model(model, compile_clone=False)
-      with self.assertRaisesRegexp(RuntimeError, 'must compile'):
-        new_model.evaluate(inp, out)
-      with self.assertRaisesRegexp(RuntimeError, 'must compile'):
-        new_model.train_on_batch(inp, out)
-      new_model.compile('rmsprop', 'mse')
-      new_model.train_on_batch(inp, out)
+    model = _get_model()
+
+    with self.assertRaisesRegexp(ValueError, 'has not been compiled'):
+      models.clone_and_build_model(model, compile_clone=True)
 
-      # Create new tensors for inputs and targets
-      input_a = keras.Input(shape=(4,))
-      target_a = keras.Input(shape=(4,))
-      new_model = models.clone_and_build_model(model, input_tensors=input_a,
-                                               target_tensors=[target_a],
-                                               compile_clone=False)
-      with self.assertRaisesRegexp(RuntimeError, 'must compile'):
-        new_model.evaluate(inp, out)
-      with self.assertRaisesRegexp(RuntimeError, 'must compile'):
-        new_model.train_on_batch(inp, out)
-      new_model.compile('rmsprop', 'mse')
+    is_subclassed = (testing_utils.get_model_type() == 'subclass')
+    # With placeholder creation
+    new_model = models.clone_and_build_model(
+        model, compile_clone=False, in_place_reset=is_subclassed)
+    with self.assertRaisesRegexp(RuntimeError, 'must compile'):
+      new_model.evaluate(inp, out)
+    with self.assertRaisesRegexp(RuntimeError, 'must compile'):
+      new_model.train_on_batch(inp, out)
+    new_model.compile(
+        testing_utils.get_v2_optimizer('rmsprop'), 'mse',
+        run_eagerly=testing_utils.should_run_eagerly())
+    new_model.train_on_batch(inp, out)
+
+    # Create new tensors for inputs and targets
+    input_a = keras.Input(shape=(4,))
+    target_a = keras.Input(shape=(4,))
+    new_model = models.clone_and_build_model(
+        model, input_tensors=input_a, target_tensors=[target_a],
+        compile_clone=False, in_place_reset=is_subclassed)
+    with self.assertRaisesRegexp(RuntimeError, 'must compile'):
+      new_model.evaluate(inp, out)
+    with self.assertRaisesRegexp(RuntimeError, 'must compile'):
       new_model.train_on_batch(inp, out)
+    new_model.compile(
+        testing_utils.get_v2_optimizer('rmsprop'), 'mse',
+        run_eagerly=testing_utils.should_run_eagerly())
+    new_model.train_on_batch(inp, out)
 
   def _assert_same_compile_params(self, model):
     """Assert that two models have the same compile parameters."""
@@ -519,134 +401,88 @@ class TestCloneAndBuildModel(test.TestCase):
     self.assertEqual(['acc', metrics.categorical_accuracy],
                      model._compile_metrics)
 
-  def _clone_and_build_test_helper(self, model, is_subclassed=False):
+  def _clone_and_build_test_helper(self, model, model_type):
     inp = np.random.random((10, 4))
     out = np.random.random((10, 4))
 
-    # Everything should work in a new session.
-    keras.backend.clear_session()
-
-    with self.cached_session():
-      # With placeholder creation
-      new_model = models.clone_and_build_model(
-          model, compile_clone=True, in_place_reset=is_subclassed)
+    is_subclassed = (model_type == 'subclass')
+
+    # With placeholder creation
+    new_model = models.clone_and_build_model(
+        model, compile_clone=True, in_place_reset=is_subclassed)
+
+    self._assert_same_compile_params(new_model)
+    new_model.train_on_batch(inp, out)
+    new_model.evaluate(inp, out)
+
+    # Create new tensors for inputs and targets
+    input_a = keras.Input(shape=(4,), name='a')
+    new_model = models.clone_and_build_model(
+        model, input_tensors=input_a, compile_clone=True,
+        in_place_reset=is_subclassed)
+    self._assert_same_compile_params(new_model)
+    new_model.train_on_batch(inp, out)
+    new_model.evaluate(inp, out)
+
+    target_a = keras.Input(shape=(4,), name='b')
+    new_model = models.clone_and_build_model(
+        model, input_tensors=input_a, target_tensors=[target_a],
+        compile_clone=True, in_place_reset=is_subclassed)
+    self._assert_same_compile_params(new_model)
+    new_model.train_on_batch(inp, out)
+    new_model.evaluate(inp, out)
+
+  @keras_parameterized.run_with_all_model_types
+  def test_clone_and_build_compiled(self):
+    model = _get_model()
+    model.compile(
+        testing_utils.get_v2_optimizer('rmsprop'), 'mse',
+        metrics=['acc', metrics.categorical_accuracy],
+        run_eagerly=testing_utils.should_run_eagerly())
+
+    self._clone_and_build_test_helper(model, testing_utils.get_model_type())
+
+  def test_clone_and_build_sequential_without_inputs_defined(self):
+    model = models.Sequential(_get_layers(input_shape=None))
+    model.compile(
+        testing_utils.get_v2_optimizer('rmsprop'),
+        'mse', metrics=['acc', metrics.categorical_accuracy],
+        run_eagerly=testing_utils.should_run_eagerly())
+    self._clone_and_build_test_helper(model, 'sequential')
 
-      self._assert_same_compile_params(new_model)
-      new_model.train_on_batch(inp, out)
-      new_model.evaluate(inp, out)
-
-      # Create new tensors for inputs and targets
-      input_a = keras.Input(shape=(4,), name='a')
-      new_model = models.clone_and_build_model(
-          model, input_tensors=input_a, compile_clone=True,
-          in_place_reset=is_subclassed)
-      self._assert_same_compile_params(new_model)
-      new_model.train_on_batch(inp, out)
-      new_model.evaluate(inp, out)
-
-      target_a = keras.Input(shape=(4,), name='b')
-      new_model = models.clone_and_build_model(
-          model, input_tensors=input_a, target_tensors=[target_a],
-          compile_clone=True, in_place_reset=is_subclassed)
-      self._assert_same_compile_params(new_model)
-      new_model.train_on_batch(inp, out)
-      new_model.evaluate(inp, out)
-
-  def test_clone_and_build_compiled_sequential_model(self):
-    with self.cached_session():
-      model = keras.models.Sequential()
-      model.add(keras.layers.Dense(4, input_shape=(4,)))
-      model.add(keras.layers.BatchNormalization())
-      model.add(keras.layers.Dropout(0.5))
-      model.add(keras.layers.Dense(4))
-      model.compile('rmsprop', 'mse',
-                    metrics=['acc', metrics.categorical_accuracy])
-
-    self._clone_and_build_test_helper(model)
-
-  def test_clone_and_build_functional_model(self):
-    with self.cached_session():
-      input_a = keras.Input(shape=(4,))
-      dense_1 = keras.layers.Dense(4,)
-      dense_2 = keras.layers.Dense(4,)
-
-      x_a = dense_1(input_a)
-      x_a = keras.layers.Dropout(0.5)(x_a)
-      x_a = keras.layers.BatchNormalization()(x_a)
-      x_a = dense_2(x_a)
-      model = keras.models.Model(input_a, x_a)
-      model.compile('rmsprop', 'mse',
-                    metrics=['acc', metrics.categorical_accuracy])
-
-    self._clone_and_build_test_helper(model)
-
-  def test_clone_and_build_subclassed_model(self):
-    class SubclassedModel(keras.Model):
-
-      def __init__(self):
-        super(SubclassedModel, self).__init__()
-        self.layer1 = keras.layers.Dense(4)
-        self.layer2 = keras.layers.Dense(4)
-
-      def call(self, inp):
-        out = self.layer1(inp)
-        out = keras.layers.BatchNormalization()(out)
-        out = keras.layers.Dropout(0.5)(out)
-        out = self.layer2(out)
-        return out
-
-    with self.cached_session():
-      model = SubclassedModel()
-      model.compile('rmsprop', 'mse',
-                    metrics=['acc', metrics.categorical_accuracy])
-    self._clone_and_build_test_helper(model, True)
+    inp = np.random.random((10, 4))
+    out = np.random.random((10, 4))
+    model.train_on_batch(inp, out)
+    self._clone_and_build_test_helper(model, 'sequential')
 
   def assert_optimizer_iterations_increases(self, optimizer):
-    with self.cached_session():
-      input_a = keras.Input(shape=(4,))
-      dense_1 = keras.layers.Dense(4,)
-      dense_2 = keras.layers.Dense(4,)
-
-      x_a = dense_1(input_a)
-      x_a = keras.layers.Dropout(0.5)(x_a)
-      x_a = keras.layers.BatchNormalization()(x_a)
-      x_a = dense_2(x_a)
-      model = keras.models.Model(input_a, x_a)
-      model.compile(optimizer, 'mse',
-                    metrics=['acc', metrics.categorical_accuracy])
+    model = _get_model()
+    model.compile(
+        optimizer, 'mse', metrics=['acc', metrics.categorical_accuracy],
+        run_eagerly=testing_utils.should_run_eagerly())
 
-      global_step = keras.backend.variable(123, dtype=dtypes.int64)
-      clone_model = models.clone_and_build_model(
-          model, compile_clone=True, optimizer_iterations=global_step)
+    global_step = keras.backend.variable(123, dtype=dtypes.int64)
+    clone_model = models.clone_and_build_model(
+        model, compile_clone=True, optimizer_iterations=global_step,
+        in_place_reset=(testing_utils.get_model_type() == 'subclass'))
 
-      inp = np.random.random((10, 4))
-      out = np.random.random((10, 4))
-      clone_model.train_on_batch(inp, out)
+    inp = np.random.random((10, 4))
+    out = np.random.random((10, 4))
+    clone_model.train_on_batch(inp, out)
 
-      self.assertEqual(K.eval(global_step), 124)
+    self.assertEqual(K.eval(global_step), 124)
 
+  @keras_parameterized.run_with_all_model_types
   def test_replace_tf_optimizer_iterations_variable(self):
     self.assert_optimizer_iterations_increases(adam.AdamOptimizer(0.01))
 
+  @keras_parameterized.run_with_all_model_types
   def test_replace_keras_optimizer_iterations_variable(self):
-    self.assert_optimizer_iterations_increases('adam')
+    if testing_utils.should_run_eagerly():
+      # This needs to be updated to run with v2 optimizers.
+      self.skipTest('b/120991591')
 
-  def test_replace_keras_optimizer_v2_iterations_variable(self):
-    self.assert_optimizer_iterations_increases(
-        keras.optimizer_v2.adam.Adam(0.01))
-
-  def test_clone_and_build_sequential_model_without_inputs_defined(self):
-    with self.cached_session():
-      model = sequential_model(False, False)
-      model.compile('rmsprop', 'mse',
-                    metrics=['acc', metrics.categorical_accuracy])
-    self._clone_and_build_test_helper(model, False)
-
-    with self.cached_session():
-      inp = np.random.random((10, 4))
-      out = np.random.random((10, 4))
-      model.train_on_batch(inp, out)
-    self._clone_and_build_test_helper(model, False)
+    self.assert_optimizer_iterations_increases('adam')
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/keras/testing_utils.py b/tensorflow/python/keras/testing_utils.py
index fd062b0ab3..6448f87d25 100644
--- a/tensorflow/python/keras/testing_utils.py
+++ b/tensorflow/python/keras/testing_utils.py
@@ -25,6 +25,13 @@ import numpy as np
 from tensorflow.python import keras
 from tensorflow.python.eager import context
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.keras.optimizer_v2 import adadelta as adadelta_v2
+from tensorflow.python.keras.optimizer_v2 import adagrad as adagrad_v2
+from tensorflow.python.keras.optimizer_v2 import adam as adam_v2
+from tensorflow.python.keras.optimizer_v2 import adamax as adamax_v2
+from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_descent_v2
+from tensorflow.python.keras.optimizer_v2 import nadam as nadam_v2
+from tensorflow.python.keras.optimizer_v2 import rmsprop as rmsprop_v2
 from tensorflow.python.training.rmsprop import RMSPropOptimizer
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util import tf_inspect
@@ -355,11 +362,20 @@ class _SubclassModel(keras.Model):
 
   def __init__(self, layers):
     super(_SubclassModel, self).__init__()
-    self.all_layers = layers
+    # Note that clone and build doesn't support lists of layers in subclassed
+    # models. Adding each layer directly here.
+    for i, layer in enumerate(layers):
+      setattr(self, self._layer_name_for_i(i), layer)
+
+    self.num_layers = len(layers)
+
+  def _layer_name_for_i(self, i):
+    return 'layer{}'.format(i)
 
   def call(self, inputs, **kwargs):
     x = inputs
-    for layer in self.all_layers:
+    for i in range(self.num_layers):
+      layer = getattr(self, self._layer_name_for_i(i))
       x = layer(x)
     return x
 
@@ -626,3 +642,39 @@ def get_multi_io_model(
     return keras.Model(inputs, outputs)
 
   raise ValueError('Unknown model type {}'.format(model_type))
+
+
+_V2_OPTIMIZER_MAP = {
+    'adadelta': adadelta_v2.Adadelta,
+    'adagrad': adagrad_v2.Adagrad,
+    'adam': adam_v2.Adam,
+    'adamax': adamax_v2.Adamax,
+    'nadam': nadam_v2.Nadam,
+    'rmsprop': rmsprop_v2.RMSprop,
+    'sgd': gradient_descent_v2.SGD
+}
+
+
+def get_v2_optimizer(name, **kwargs):
+  """Get the v2 optimizer requested.
+
+  This is only necessary until v2 are the default, as we are testing in Eager,
+  and Eager + v1 optimizers fail tests. When we are in v2, the strings alone
+  should be sufficient, and this mapping can theoretically be removed.
+
+  Args:
+    name: string name of Keras v2 optimizer.
+    **kwargs: any kwargs to pass to the optimizer constructor.
+
+  Returns:
+    Initialized Keras v2 optimizer.
+
+  Raises:
+    ValueError: if an unknown name was passed.
+  """
+  try:
+    return _V2_OPTIMIZER_MAP[name](**kwargs)
+  except KeyError:
+    raise ValueError(
+        'Could not find requested v2 optimizer: {}\nValid choices: {}'.format(
+            name, list(_V2_OPTIMIZER_MAP.keys())))
-- 
GitLab


From 17bf2b33a90017c8536ef4a242bb60636bfc5e8a Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Fri, 4 Jan 2019 17:45:29 -0800
Subject: [PATCH 0231/2345] Graph construction: skip testing for duplicate
 edges during source/sink setup.

When fixing up source/sink edges, we are only adding missing edges.  Skip the
test for duplicate edges in this case, which avoids a linear traversal over
sink edges.

PiperOrigin-RevId: 227940303
---
 tensorflow/core/graph/algorithm.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/graph/algorithm.cc b/tensorflow/core/graph/algorithm.cc
index 9b4200e0b4..25bd3516a1 100644
--- a/tensorflow/core/graph/algorithm.cc
+++ b/tensorflow/core/graph/algorithm.cc
@@ -222,11 +222,12 @@ bool FixupSourceAndSinkEdges(Graph* g) {
   bool changed = false;
   for (Node* n : g->nodes()) {
     if (!n->IsSource() && n->in_edges().empty()) {
-      g->AddControlEdge(g->source_node(), n);
+      g->AddControlEdge(g->source_node(), n,
+                        true /* skip test for duplicates */);
       changed = true;
     }
     if (!n->IsSink() && n->out_edges().empty()) {
-      g->AddControlEdge(n, g->sink_node());
+      g->AddControlEdge(n, g->sink_node(), true /* skip test for duplicates */);
       changed = true;
     }
   }
-- 
GitLab


From 1814e22d3cf78fd6e5b172c6de1a44cd6abc25f3 Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Fri, 4 Jan 2019 17:45:29 -0800
Subject: [PATCH 0232/2345] Fix some empty struct build warnings in TFLite

PiperOrigin-RevId: 227940304
---
 tensorflow/lite/c/builtin_op_data.h | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tensorflow/lite/c/builtin_op_data.h b/tensorflow/lite/c/builtin_op_data.h
index 46ac656ecf..332c2db145 100644
--- a/tensorflow/lite/c/builtin_op_data.h
+++ b/tensorflow/lite/c/builtin_op_data.h
@@ -25,6 +25,11 @@ extern "C" {
 
 // TODO(aselle): Consider using "if this then that" for testing.
 
+// Useful placeholder to put in otherwise empty structs to avoid size warnings.
+typedef struct {
+  char dummy;
+} EmptyStructPlaceholder;
+
 // IMPORTANT: All new members of structs must be added at the end to ensure
 // backwards compatibility.
 
@@ -152,9 +157,11 @@ typedef struct {
 } TfLiteAddParams;
 
 typedef struct {
+  EmptyStructPlaceholder placeholder;
 } TfLiteSpaceToBatchNDParams;
 
 typedef struct {
+  EmptyStructPlaceholder placeholder;
 } TfLiteBatchToSpaceNDParams;
 
 typedef struct {
@@ -230,9 +237,11 @@ typedef struct {
 } TfLiteResizeNearestNeighborParams;
 
 typedef struct {
+  EmptyStructPlaceholder placeholder;
 } TfLitePadParams;
 
 typedef struct {
+  EmptyStructPlaceholder placeholder;
 } TfLitePadV2Params;
 
 typedef struct {
@@ -272,6 +281,7 @@ typedef struct {
 } TfLiteGatherParams;
 
 typedef struct {
+  EmptyStructPlaceholder placeholder;
 } TfLiteTransposeParams;
 
 typedef struct {
-- 
GitLab


From 447189935cbc1ab3aeb2ac67f81f03d1a188e548 Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Fri, 4 Jan 2019 18:40:27 -0800
Subject: [PATCH 0233/2345] Make TRT tests respect RewriterConfig set by
 individual test case everywhere.

PiperOrigin-RevId: 227944638
---
 .../test/tf_trt_integration_test_base.py       | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
index 495a9391a1..671abba6a6 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
@@ -239,8 +239,8 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
 
   def _GetConfigProto(self, run_params, graph_state):
     """Get config proto based on specific settings."""
+    conversion_params = self.GetConversionParams(run_params)
     if graph_state != GraphState.ORIGINAL and run_params.use_optimizer:
-      conversion_params = self.GetConversionParams(run_params)
       rewriter_cfg = trt_convert.get_tensorrt_rewriter_config(
           conversion_params.rewriter_config, conversion_params.max_batch_size,
           conversion_params.max_workspace_size_bytes,
@@ -254,6 +254,9 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
       graph_options = config_pb2.GraphOptions(rewrite_options=rewriter_cfg)
     else:
       graph_options = config_pb2.GraphOptions()
+      if conversion_params.rewriter_config is not None:
+        graph_options.rewrite_options.CopyFrom(
+            conversion_params.rewriter_config)
 
     config = config_pb2.ConfigProto(
         gpu_options=self._GetGPUOptions(), graph_options=graph_options)
@@ -321,16 +324,13 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
     return self._RunGraph(
         run_params, gdef, input_data, config, GraphState.CALIBRATE, num_runs=5)
 
-  def _GetTrtGraphDef(self, run_params, gdef):
+  def _GetTrtGraphDef(self, run_params, graph_state, gdef):
     """Return trt converted graphdef."""
     params = self._GetParamsCached()
     conversion_params = self.GetConversionParams(run_params)
     logging.info(conversion_params)
 
-    config_for_trt = config_pb2.ConfigProto(gpu_options=self._GetGPUOptions())
-    if conversion_params.rewriter_config is not None:
-      config_for_trt.graph_options.rewrite_options.CopyFrom(
-          conversion_params.rewriter_config)
+    config_for_trt = self._GetConfigProto(run_params, graph_state)
     return trt_convert.create_inference_graph(
         input_graph_def=gdef,
         outputs=params.input_names + params.output_names,
@@ -506,7 +506,8 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
         result = self._RunCalibration(run_params, input_gdef, input_data,
                                       calib_config)
       else:
-        calib_gdef = self._GetTrtGraphDef(run_params, input_gdef)
+        calib_gdef = self._GetTrtGraphDef(run_params, GraphState.CALIBRATE,
+                                          input_gdef)
         self._VerifyGraphDef(run_params, calib_gdef, GraphState.CALIBRATE)
         result = self._RunCalibration(run_params, calib_gdef, input_data,
                                       calib_config)
@@ -527,7 +528,8 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
     logging.info("Running final inference graph, config:\n%s",
                  str(infer_config))
     if not run_params.use_optimizer:
-      infer_gdef = self._GetTrtGraphDef(run_params, infer_gdef)
+      infer_gdef = self._GetTrtGraphDef(run_params, GraphState.INFERENCE,
+                                        infer_gdef)
       self._VerifyGraphDef(run_params, infer_gdef, GraphState.INFERENCE)
 
     result = self._RunGraph(run_params, infer_gdef, input_data, infer_config,
-- 
GitLab


From d93409b9dfc3bfaacfa452fab603a089931b3335 Mon Sep 17 00:00:00 2001
From: Davide Libenzi <dlibenzi@google.com>
Date: Fri, 4 Jan 2019 19:00:39 -0800
Subject: [PATCH 0234/2345] Automated rollback of commit
 55fda7fb66f1774ac5271e523259faa7d2a44d0c

PiperOrigin-RevId: 227945898
---
 .../xla/service/cpu/cpu_executable.cc         |  29 +---
 .../xla/service/gpu/gpu_executable.cc         |  26 +---
 tensorflow/compiler/xrt/kernels/BUILD         |   1 -
 .../compiler/xrt/kernels/xrt_execute_op.cc    |  17 ---
 tensorflow/compiler/xrt/tests/raw_api_test.cc | 127 +-----------------
 tensorflow/compiler/xrt/xrt_state.cc          |  43 +-----
 tensorflow/compiler/xrt/xrt_state.h           |  23 +---
 7 files changed, 17 insertions(+), 249 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/cpu_executable.cc b/tensorflow/compiler/xla/service/cpu/cpu_executable.cc
index 23d0af3423..412c2715b9 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_executable.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_executable.cc
@@ -213,8 +213,6 @@ StatusOr<ScopedShapedBuffer> CpuExecutable::CreateResultShapedBuffer(
       /*on_host_shape=*/result_shape(),
       /*on_device_shape=*/result_shape(), run_options->allocator(),
       stream->parent()->device_ordinal());
-  const HloInputOutputAliasConfig& input_output_alias =
-      module().input_output_alias_config();
 
   // Move OwningDeviceMemory values which contain the array(s) of the result
   // into the respective location in ScopedShapedBuffer which is returned to the
@@ -234,31 +232,12 @@ StatusOr<ScopedShapedBuffer> CpuExecutable::CreateResultShapedBuffer(
         TF_ASSIGN_OR_RETURN(
             const BufferAllocation::Slice slice,
             this->assignment_->GetUniqueSlice(src, buffer_source->index()));
+        CHECK(!slice.allocation()->is_entry_computation_parameter());
+
         const BufferAllocation::Index buffer_index = slice.index();
         OwningDeviceMemory& buffer = buffers[buffer_index];
-        if (!slice.allocation()->is_entry_computation_parameter()) {
-          // If the buffer coming out of the result is from a parameter, the
-          // owning buffer will be null, and that means the caller aliased some
-          // parameter buffer to an output one (via the
-          // HloInputOutputAliasConfig API). If that is the case, the caller
-          // will receive a partially complete scoped shaped buffer, which they
-          // will have to fill up on return. Unfortunately the interface to the
-          // execute APIs are ShapedBuffer pointer based, which assumes caller
-          // ownership, and hence a buffer coming from there cannot be part of
-          // the new ScopedShapedBuffer we create for the result (which assumes
-          // ownership).
-          *device_memory = buffer.Forget();
-        } else {
-          auto output_alias = input_output_alias.GetAliasedOutput(
-              slice.allocation()->parameter_number(),
-              slice.allocation()->param_shape_index());
-          CHECK(output_alias)
-              << "Ouput buffer is coming from parameter "
-              << slice.allocation()->parameter_number() << " at index "
-              << slice.allocation()->param_shape_index()
-              << ", but no alias exists";
-          CHECK_EQ(*output_alias, index);
-        }
+        CHECK(!buffer.is_null() || buffer.size() == 0);
+        *device_memory = buffer.Forget();
         return Status::OK();
       }));
   return std::move(result_buffer);
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_executable.cc b/tensorflow/compiler/xla/service/gpu/gpu_executable.cc
index 434060ad89..128cecd765 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_executable.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_executable.cc
@@ -310,34 +310,12 @@ StatusOr<ScopedShapedBuffer> GpuExecutable::ExecuteOnStream(
         TF_ASSIGN_OR_RETURN(
             const BufferAllocation::Slice slice,
             this->assignment_->GetUniqueSlice(src_hlo, sources[0]->index()));
+        CHECK(!slice.allocation()->is_entry_computation_parameter());
 
         se::DeviceMemoryBase src_base =
             buffer_allocations->GetDeviceAddress(slice.index());
         CHECK(!src_base.is_null() || src_base.size() == 0);
-        if (!slice.allocation()->is_entry_computation_parameter()) {
-          // If the buffer coming out of the result is from a parameter, it
-          // means the caller aliased some parameter buffer to an output one
-          // (via the HloInputOutputAliasConfig API). If that is the case, the
-          // caller will receive a partially complete scoped shaped buffer,
-          // which they will have to fill up on return.
-          // Unfortunately the interface to the execute APIs are ShapedBuffer
-          // pointer based, which assumes caller ownership, and hence a buffer
-          // coming from there cannot be part of the new ScopedShapedBuffer we
-          // create for the result (which assumes ownership).
-          *device_memory = src_base;
-        } else {
-          const HloInputOutputAliasConfig& input_output_alias =
-              module().input_output_alias_config();
-          auto output_alias = input_output_alias.GetAliasedOutput(
-              slice.allocation()->parameter_number(),
-              slice.allocation()->param_shape_index());
-          CHECK(output_alias)
-              << "Ouput buffer is coming from parameter "
-              << slice.allocation()->parameter_number() << " at index "
-              << slice.allocation()->param_shape_index()
-              << ", but no alias exists";
-          CHECK_EQ(*output_alias, index);
-        }
+        *device_memory = src_base;
         buffers_in_result.insert(src_base);
         return Status::OK();
       }));
diff --git a/tensorflow/compiler/xrt/kernels/BUILD b/tensorflow/compiler/xrt/kernels/BUILD
index c44769dfe3..67f475846e 100644
--- a/tensorflow/compiler/xrt/kernels/BUILD
+++ b/tensorflow/compiler/xrt/kernels/BUILD
@@ -55,7 +55,6 @@ cc_library(
         "//tensorflow/compiler/xla/client:xla_computation",
         "//tensorflow/compiler/xla/service:compiler",
         "//tensorflow/compiler/xla/service:computation_placer",
-        "//tensorflow/compiler/xla/service:hlo",
         "//tensorflow/compiler/xrt:xrt_proto",
         "//tensorflow/compiler/xrt:xrt_utils",
         "//tensorflow/core:core_cpu_internal",
diff --git a/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc b/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc
index 7544541ca4..7f0ac123a5 100644
--- a/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc
+++ b/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc
@@ -19,7 +19,6 @@ limitations under the License.
 
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/service/computation_placer.h"
-#include "tensorflow/compiler/xla/service/hlo_input_output_alias_config.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/statusor.h"
@@ -229,22 +228,6 @@ Status XRTExecuteOp::DoWork(OpKernelContext* context) {
   TF_RETURN_IF_ERROR(XRTTupleAllocation::CreateFromBuffer(
       shaped_buffer, device_ref.backend(), device_ref.device_ordinal(),
       &output_tuple));
-
-  // The ScopedShapedBuffer returned by the executable Run() API, in case of
-  // input/output buffer aliasing, might have holes in it, which need to be
-  // filled using the proper input tuples buffers which are the source of
-  // aliasing.
-  const xla::HloInputOutputAliasConfig& input_output_alias =
-      executable->executable()->module().input_output_alias_config();
-  auto alias_function = [&](const xla::ShapeIndex& output_index,
-                            int64 param_number,
-                            const xla::ShapeIndex& param_index) -> Status {
-    TF_RET_CHECK(param_number < input_tuples.size());
-    return output_tuple->AliasBufferFrom(*input_tuples[param_number],
-                                         param_index, output_index);
-  };
-  TF_RETURN_IF_ERROR(input_output_alias.ForEachAliasWithStatus(alias_function));
-
   if (config_proto.return_exploded_tuple() &&
       output_tuple->on_device_shape().IsTuple()) {
     int64 tuple_element_count =
diff --git a/tensorflow/compiler/xrt/tests/raw_api_test.cc b/tensorflow/compiler/xrt/tests/raw_api_test.cc
index f81faf0e61..c8479cb778 100644
--- a/tensorflow/compiler/xrt/tests/raw_api_test.cc
+++ b/tensorflow/compiler/xrt/tests/raw_api_test.cc
@@ -96,21 +96,14 @@ xla::LiteralProto FloatMatrix(
   return array.ToProto();
 }
 
-xla::Literal ReadOutputLiteral(const std::vector<Tensor>& outputs, size_t idx) {
-  xla::LiteralProto response;
-  CHECK(response.ParseFromString(outputs[idx].scalar<string>()()));
-  return xla::Literal::CreateFromProto(response).ValueOrDie();
-}
-
 bool CompareLiteralProtos(const xla::LiteralProto& a,
                           const xla::LiteralProto& b) {
   auto l_a = xla::Literal::CreateFromProto(a).ValueOrDie();
   auto l_b = xla::Literal::CreateFromProto(b).ValueOrDie();
   bool equal = l_a == l_b;
   if (!equal) {
-    LOG(INFO) << "LiteralProtos don't match:\n"
-              << a.DebugString() << "\n!=\n"
-              << b.DebugString();
+    LOG(INFO) << "LiteralProtos don't match: " << a.DebugString()
+              << " != " << b.DebugString();
   }
   return equal;
 }
@@ -120,19 +113,8 @@ bool CompareLiteralToLiteralProto(const xla::Literal& a,
   auto l_b = xla::Literal::CreateFromProto(b).ValueOrDie();
   bool equal = a == l_b;
   if (!equal) {
-    LOG(INFO) << "Literal and LiteralProto don't match:\n"
-              << a.ToProto().DebugString() << "\n!=\n"
-              << b.DebugString();
-  }
-  return equal;
-}
-
-bool CompareLiterals(const xla::Literal& a, const xla::Literal& b) {
-  bool equal = a == b;
-  if (!equal) {
-    LOG(INFO) << "Literals don't match:\n"
-              << a.ToProto().DebugString() << "\n!=\n"
-              << b.ToProto().DebugString();
+    LOG(INFO) << "Literal and LiteralProto don't match "
+              << a.ToProto().DebugString() << " != " << b.DebugString();
   }
   return equal;
 }
@@ -957,107 +939,6 @@ TEST(RawApiTest, LeakCompilationReference) {
   TF_EXPECT_OK(session.Run({c_handle.handle}, &outputs));
 }
 
-TEST(RawApiTest, CompileAndExecuteWithReusedBuffers) {
-  xla::Shape element_shape = xla::ShapeUtil::MakeShape(xla::F32, {2});
-  xla::Shape shape =
-      xla::ShapeUtil::MakeTupleShape({element_shape, element_shape});
-  xla::Shape return_shape = xla::ShapeUtil::MakeTupleShape(
-      {element_shape, element_shape, element_shape, element_shape});
-  xla::XlaBuilder builder("ReuseBuffer");
-  auto param = xla::Parameter(&builder, 0, shape, "param");
-  auto p0 = xla::GetTupleElement(param, 0);
-  auto p1 = xla::GetTupleElement(param, 1);
-  auto add = xla::Add(p0, p1);
-  auto sub = xla::Sub(p0, p1);
-  xla::Tuple(&builder, {add, sub, p0, p1});
-
-  // Flip the tuple literals in the input handle.
-  builder.SetUpAlias({1}, 0, {0});
-  builder.SetUpAlias({0}, 0, {1});
-
-  auto computation = builder.Build().ValueOrDie();
-
-  auto literal0 = xla::LiteralUtil::CreateR1<float>({1.0f, 2.0f});
-  auto literal1 = xla::LiteralUtil::CreateR1<float>({5.0f, 9.0f});
-  auto literal = xla::LiteralUtil::MakeTuple({&literal0, &literal1});
-
-  xrt::XLAAllocation param_alloc;
-  *param_alloc.mutable_value() = literal.ToProto();
-
-  xrt::XLAComputation c;
-  auto config = c.mutable_config();
-  auto shapes = config->mutable_program_shape();
-  *shapes->add_parameters() = shape.ToProto();
-  *shapes->mutable_result() = return_shape.ToProto();
-  StoreComputationSnapshot(computation, c.mutable_hlo_snapshot());
-
-  xrt::XRTExecutionConfig e;
-  e.set_release_input_handles(false);
-  e.set_release_compilation_handle(true);
-
-  Scope root = Scope::NewRootScope().WithDevice(DeviceFromFlag());
-  ClientSession session(root);
-  auto e_config =
-      ops::Const(root.WithDevice("/device:CPU:0"), e.SerializeAsString());
-  auto c_data =
-      ops::Const(root.WithDevice("/device:CPU:0"), c.SerializeAsString());
-  auto c_handle = ops::XRTCompile(root, c_data);
-  auto param_value = ops::Const(root.WithDevice("/device:CPU:0"),
-                                param_alloc.SerializeAsString());
-  auto param_handle = ops::XRTAllocate(root, param_value);
-  TF_ASSERT_OK(root.status());
-
-  std::vector<Tensor> outputs;
-  TF_EXPECT_OK(session.Run({param_handle}, &outputs));
-
-  int64 alloc_handle = outputs[0].scalar<int64>()();
-
-  // Note that we release the result handle immediately, but since we aliased
-  // the output buffers onto the input allocation ones (held in alloc_handle),
-  // we can fetch the result from there.
-  auto result =
-      ops::XRTExecute(root, c_handle.handle, e_config, {Input(alloc_handle)});
-  auto read_back = ops::XRTReadLiteral(root, result);
-  auto release = ops::XRTReleaseAllocationHandle(
-      root.WithControlDependencies(read_back), result);
-  TF_ASSERT_OK(root.status());
-
-  outputs.clear();
-  TF_EXPECT_OK(session.Run(tensorflow::ClientSession::FeedType(), {read_back},
-                           {release}, &outputs));
-
-  xla::Literal exec_literal = ReadOutputLiteral(outputs, 0);
-  auto exec_literal_parts = exec_literal.DecomposeTuple();
-  ASSERT_EQ(exec_literal_parts.size(), 4);
-
-  EXPECT_TRUE(CompareLiterals(exec_literal_parts[2], literal0));
-  EXPECT_TRUE(CompareLiterals(exec_literal_parts[3], literal1));
-
-  // Now we read back the original input handle values, which at this point
-  // should contain the result of the XLA computation.
-  auto read_handle = ops::XRTReadLiteral(root, Input(alloc_handle));
-  TF_ASSERT_OK(root.status());
-  auto release_handle = ops::XRTReleaseAllocationHandle(
-      root.WithControlDependencies(read_handle), Input(alloc_handle));
-  TF_ASSERT_OK(root.status());
-
-  outputs.clear();
-  TF_EXPECT_OK(session.Run(tensorflow::ClientSession::FeedType(), {read_handle},
-                           {release_handle}, &outputs));
-
-  xla::Literal return_literal = ReadOutputLiteral(outputs, 0);
-
-  auto expected_literal0 = xla::LiteralUtil::CreateR1<float>({6.0f, 11.0f});
-  auto expected_literal1 = xla::LiteralUtil::CreateR1<float>({-4.0f, -7.0f});
-  // The first element of the computation returned tuple would be the add
-  // (expected_literal0), but since we flipped the buffers, the sub
-  // (expected_literal1) should come first.
-  auto expected_literal =
-      xla::LiteralUtil::MakeTuple({&expected_literal1, &expected_literal0});
-
-  EXPECT_TRUE(CompareLiterals(return_literal, expected_literal));
-}
-
 TEST(RawApiTest, CompileAndExecuteWithS64Argument) {
   xrt::XLAAllocation p0;
   *p0.mutable_value() = xla::LiteralUtil::CreateR0<int64>(11031965).ToProto();
diff --git a/tensorflow/compiler/xrt/xrt_state.cc b/tensorflow/compiler/xrt/xrt_state.cc
index 13c275aaca..343460ff10 100644
--- a/tensorflow/compiler/xrt/xrt_state.cc
+++ b/tensorflow/compiler/xrt/xrt_state.cc
@@ -133,8 +133,7 @@ Status AllocateScopedShapedBuffer(
 XRTBufferAllocation::XRTBufferAllocation(const se::DeviceMemoryBase& allocation,
                                          int device_ordinal,
                                          xla::DeviceMemoryAllocator* allocator)
-    : size_(allocation.size()),
-      allocation_(allocation),
+    : allocation_(allocation),
       device_ordinal_(device_ordinal),
       allocator_(allocator) {
   if (VLOG_IS_ON(2)) {
@@ -224,19 +223,8 @@ Status XRTTupleAllocation::ToLiteral(xla::Backend* backend, int device_ordinal,
                                      xla::Literal* literal) {
   auto transfer_manager = backend->transfer_manager();
   TF_ASSIGN_OR_RETURN(auto stream, backend->BorrowStream(device_ordinal));
-
-  // Validate the allocation buffers as if nulls gets to
-  // TransferLiteralFromDevice() a CHECK is issued.
-  xla::ShapedBuffer shaped_buffer = ToShapedBuffer();
-  for (auto& index_buffer : shaped_buffer.buffers()) {
-    if (index_buffer.second.is_null()) {
-      return errors::InvalidArgument("Literal buffer at index ",
-                                     index_buffer.first.ToString(),
-                                     " has been released");
-    }
-  }
   TF_ASSIGN_OR_RETURN(*literal, transfer_manager->TransferLiteralFromDevice(
-                                    stream.get(), shaped_buffer));
+                                    stream.get(), ToShapedBuffer()));
   return Status::OK();
 }
 
@@ -517,34 +505,11 @@ xla::ShapedBuffer XRTTupleAllocation::ToShapedBuffer() {
   return shaped_buffer;
 }
 
-Status XRTTupleAllocation::AliasBufferFrom(const XRTTupleAllocation& source,
-                                           const xla::ShapeIndex& source_index,
-                                           const xla::ShapeIndex& dest_index) {
-  XRTBufferAllocation* source_buffer = source.buffers_.element(source_index);
-  XRTBufferAllocation* dest_buffer = buffers_.element(dest_index);
-  // We allow the destination size being zero, because there are cases where we
-  // are coming in later filling in null/uninitialized device buffers.
-  // In all other cases, the size of the new buffer must match.
-  if (source_buffer->size() != dest_buffer->size() &&
-      dest_buffer->size() != 0) {
-    return errors::InvalidArgument(
-        "Source buffer at index ", source_index.ToString(),
-        " does not match the size of destination buffer at index ",
-        dest_index.ToString(), ": ", source_buffer->size(), " vs ",
-        dest_buffer->size());
-  }
-  *buffers_.mutable_element(dest_index) = source_buffer;
-  source_buffer->Ref();
-  dest_buffer->Unref();
-  return Status::OK();
-}
-
 xla::ShapeTree<xla::MaybeOwningDeviceMemory>
-XRTTupleAllocation::ToDeviceMemoryTree(
-    const std::function<bool(const xla::ShapeIndex&)>& release_checker) {
+XRTTupleAllocation::ToDeviceMemoryTree(bool release) {
   xla::ShapeTree<xla::MaybeOwningDeviceMemory> shaped_tree(on_device_shape());
   for (const auto& buffer : buffers_) {
-    if (!release_checker(buffer.first)) {
+    if (!release) {
       *shaped_tree.mutable_element(buffer.first) = buffer.second->allocation();
     } else {
       *shaped_tree.mutable_element(buffer.first) = xla::OwningDeviceMemory(
diff --git a/tensorflow/compiler/xrt/xrt_state.h b/tensorflow/compiler/xrt/xrt_state.h
index ac4be3a064..3e3d502412 100644
--- a/tensorflow/compiler/xrt/xrt_state.h
+++ b/tensorflow/compiler/xrt/xrt_state.h
@@ -18,7 +18,6 @@ limitations under the License.
 #ifndef TENSORFLOW_COMPILER_XRT_XRT_STATE_H_
 #define TENSORFLOW_COMPILER_XRT_XRT_STATE_H_
 
-#include <functional>
 #include <memory>
 #include <string>
 #include <vector>
@@ -59,14 +58,7 @@ class XRTBufferAllocation : public core::RefCounted {
   // freed when the reference count drops to zero.
   void DiscardAllocation();
 
-  // Returns the expected size of the allocation. Since DiscardAllocation() will
-  // set allocation_ to {null,0}, and since later we might want to replace the
-  // discarded buffer with a new one, we need to be able to verify the size
-  // compatibility.
-  uint64 size() const { return size_; }
-
  private:
-  uint64 size_ = 0;
   se::DeviceMemoryBase allocation_;
   int device_ordinal_;
   xla::DeviceMemoryAllocator* allocator_;
@@ -176,18 +168,9 @@ class XRTTupleAllocation : public ResourceBase {
   // the same shape as on_host_shape.
   xla::ShapedBuffer ToShapedBuffer();
 
-  // Aliases the source buffer at source_index into the current tuple allocation
-  // dest_index.
-  Status AliasBufferFrom(const XRTTupleAllocation& source,
-                         const xla::ShapeIndex& source_index,
-                         const xla::ShapeIndex& dest_index);
-
-  // Returns the device memory tree of this allocation. If the release_checker
-  // function returns true for a given index, the ownership of the device memory
-  // at that index is transferred to the result. Every attempt to read the value
-  // at that index will fail.
-  xla::ShapeTree<xla::MaybeOwningDeviceMemory> ToDeviceMemoryTree(
-      const std::function<bool(const xla::ShapeIndex&)>& release_checker);
+  // Returns the device memory tree of this allocation. If 'release' is set, the
+  // ownership of the device memory is transferred to the result.
+  xla::ShapeTree<xla::MaybeOwningDeviceMemory> ToDeviceMemoryTree(bool release);
 
   string DebugString() override { return "XLA allocation handle"; }
 
-- 
GitLab


From 47c68e37dee827d9c3fe95f39e5fb49d07a573a5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 19:47:17 -0800
Subject: [PATCH 0235/2345] Automated rollback of commit
 57d501422c16dd73b391d19bcc1e203942cc6b24

PiperOrigin-RevId: 227949022
---
 tensorflow/contrib/distribute/python/BUILD    |   3 +-
 .../contrib/distribute/python/combinations.py |  15 +--
 .../python/keras_correctness_test.py          | 100 +++---------------
 3 files changed, 19 insertions(+), 99 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index c620a0448e..de6b6f1f84 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -670,12 +670,11 @@ py_library(
 
 cuda_py_test(
     name = "keras_correctness_test",
-    size = "medium",
     srcs = ["keras_correctness_test.py"],
     additional_deps = [
         ":keras_correctness_test_lib",
     ],
-    shard_count = 20,
+    shard_count = 16,
     tags = [
         "multi_and_single_gpu",
         "no_oss",  # TODO(b/117919883): Fix python error.
diff --git a/tensorflow/contrib/distribute/python/combinations.py b/tensorflow/contrib/distribute/python/combinations.py
index 8db0e9c316..f6c4291659 100644
--- a/tensorflow/contrib/distribute/python/combinations.py
+++ b/tensorflow/contrib/distribute/python/combinations.py
@@ -321,12 +321,11 @@ class NamedDistribution(object):
     return self._required_tpu
 
 
-def _get_tpu_strategy_creator(steps_per_run, **kwargs):
+def _get_tpu_strategy_creator(steps_per_run):
   def _create_tpu_strategy():
     resolver = cluster_resolver.TPUClusterResolver("")
     tpu_lib.initialize_tpu_system(resolver)
-    strategy = tpu_lib.TPUStrategy(resolver,
-                                   steps_per_run=steps_per_run, **kwargs)
+    strategy = tpu_lib.TPUStrategy(resolver, steps_per_run=steps_per_run)
     return strategy
   return _create_tpu_strategy
 
@@ -345,15 +344,7 @@ tpu_strategy = NamedDistribution(
 tpu_strategy_one_step = NamedDistribution(
     "TPUOneStep", _get_tpu_strategy_creator(steps_per_run=1),
     required_tpu=True)
-# TODO(b/122327153): Remove below two NamedDistributions.
-tpu_strategy_loop_on_device = NamedDistribution(
-    "TPULoopOnDevice", _get_tpu_strategy_creator(
-        steps_per_run=2, _disable_training_loop_on_host=True),
-    required_tpu=True)
-tpu_strategy_one_step_loop_on_device = NamedDistribution(
-    "TPUOneStepLoopOnDevice", _get_tpu_strategy_creator(
-        steps_per_run=1, _disable_training_loop_on_host=True),
-    required_tpu=True)
+
 mirrored_strategy_with_one_cpu = NamedDistribution(
     "Mirrored1CPU",
     lambda: mirrored_lib.MirroredStrategy(["/cpu:0"]))
diff --git a/tensorflow/contrib/distribute/python/keras_correctness_test.py b/tensorflow/contrib/distribute/python/keras_correctness_test.py
index 1c4519ef71..3abdee2c0e 100644
--- a/tensorflow/contrib/distribute/python/keras_correctness_test.py
+++ b/tensorflow/contrib/distribute/python/keras_correctness_test.py
@@ -62,41 +62,24 @@ def all_strategy_combinations_with_graph_mode():
   return combinations.combine(distribution=all_strategies, mode=['graph'])
 
 
-def tpu_strategies_with_host_training_loop_disabled():
-  strategies = [s for s in all_strategies if not s.required_tpu]
-  strategies.append(combinations.tpu_strategy_loop_on_device)
-  strategies.append(combinations.tpu_strategy_one_step_loop_on_device)
-  return strategies
-
-
 def strategy_and_input_combinations():
   def cnn_model_with_batch_norm(**kwargs):
     return _create_cnn_model(with_batch_norm=True, **kwargs)
 
-  combinations_without_embedding_model = combinations.times(
-      combinations.combine(
-          distribution=tpu_strategies_with_host_training_loop_disabled()),
-      combinations.combine(mode=['graph', 'eager'],
-                           use_numpy=[True, False],
-                           use_validation_data=[True, False]),
-      combinations.combine(model_with_data=
-                           [ModelWithData('lstm',
-                                          _create_lstm_model,
-                                          _lstm_training_data)]))
-
-  combinations_with_embedding_model = combinations.times(
-      combinations.combine(distribution=all_strategies),
-      combinations.combine(mode=['graph', 'eager'],
-                           use_numpy=[True, False],
-                           use_validation_data=[True, False]),
-      combinations.combine(model_with_data=[
-          ModelWithData('dnn', _create_dnn_model, _dnn_training_data),
-          ModelWithData('cnn', _create_cnn_model, _cnn_training_data),
-          ModelWithData('cnn_batch_norm', cnn_model_with_batch_norm,
-                        _cnn_training_data, with_batch_norm=True),]))
-
-  return (combinations_with_embedding_model +
-          combinations_without_embedding_model)
+  return (
+      combinations.times(
+          combinations.combine(distribution=all_strategies),
+          combinations.combine(mode=['graph', 'eager'],
+                               use_numpy=[True, False],
+                               use_validation_data=[True, False]),
+          combinations.combine(model_with_data=[
+              ModelWithData('dnn', _create_dnn_model, _dnn_training_data),
+              ModelWithData('cnn', _create_cnn_model, _cnn_training_data),
+              ModelWithData('cnn_batch_norm',
+                            cnn_model_with_batch_norm,
+                            _cnn_training_data,
+                            with_batch_norm=True),
+          ])))
 
 
 class MaybeDistributionScope(object):
@@ -208,59 +191,6 @@ def _create_cnn_model(initial_weights=None, distribution=None,
   return model
 
 
-def _lstm_training_data(count=_GLOBAL_BATCH_SIZE * _EVAL_STEPS,
-                        min_words=10,
-                        max_words=20,
-                        max_word_id=99,
-                        num_classes=2):
-  distribution = []
-  for _ in range(num_classes):
-    dist = np.abs(np.random.randn(max_word_id))
-    dist /= np.sum(dist)
-    distribution.append(dist)
-
-  features = []
-  labels = []
-  for _ in range(count):
-    label = np.random.randint(0, num_classes, size=1)[0]
-    num_words = np.random.randint(min_words, max_words, size=1)[0]
-    word_ids = np.random.choice(
-        max_word_id, size=num_words, replace=True, p=distribution[label])
-    word_ids = word_ids
-    labels.append(label)
-    features.append(word_ids)
-
-  features = keras.preprocessing.sequence.pad_sequences(
-      features, maxlen=max_words)
-  x_train = np.asarray(features, dtype=np.float32)
-  y_train = np.asarray(labels, dtype=np.int32).reshape((count, 1))
-  x_predict = x_train
-  return x_train, y_train, x_predict
-
-
-def _create_lstm_model(max_words=20,
-                       initial_weights=None,
-                       distribution=None):
-  with MaybeDistributionScope(distribution):
-    word_ids = keras.layers.Input(
-        shape=(max_words,), dtype=np.int32, name='words')
-    word_embed = keras.layers.Embedding(input_dim=100, output_dim=10)(word_ids)
-    lstm_embed = keras.layers.LSTM(
-        units=8, return_sequences=False)(word_embed)
-
-    preds = keras.layers.Dense(2, activation='softmax')(lstm_embed)
-    model = keras.Model(inputs=[word_ids], outputs=[preds])
-
-    if initial_weights:
-      model.set_weights(initial_weights)
-
-    model.compile(
-        optimizer=gradient_descent.GradientDescentOptimizer(learning_rate=0.1),
-        loss='sparse_categorical_crossentropy',
-        metrics=['sparse_categorical_accuracy'])
-  return model
-
-
 def batch_wrapper(dataset, batch_size, distribution, repeat=None):
   if repeat:
     dataset = dataset.repeat(repeat)
@@ -313,7 +243,7 @@ def get_correctness_test_inputs(use_numpy, use_validation_data,
     }
   else:
     if len(x_train) < _GLOBAL_BATCH_SIZE * _EVAL_STEPS:
-      # Currently, we cannot detect the size of a dataset. So, the eval steps is
+      # Currently, we cannot detech the size of a dataset. So, the eval steps is
       # hard coded.
       raise ValueError('x_train must have at least '
                        '_GLOBAL_BATCH_SIZE * _EVAL_STEPS samples')
-- 
GitLab


From 5633ba3fc829a31c297a412929f5953e8c7a1f2f Mon Sep 17 00:00:00 2001
From: Jason Zaman <jason@perfinion.com>
Date: Wed, 2 Jan 2019 18:29:06 +0800
Subject: [PATCH 0236/2345] cloud oauth_client: update for OpenSSL 1.1.0
 compatibility

EVP_MD_CTX_cleanup was removed in OpenSSL 1.1.0. There is a call to
EVP_MD_CTX_destroy right after and _destroy will call _cleanup if
required. EVP_MD_CTX_destroy exists in OpenSSL 1.0, 1.1 and BoringSSL so
removing the call to _cleanup works everywhere.

Signed-off-by: Jason Zaman <jason@perfinion.com>
---
 tensorflow/core/platform/cloud/oauth_client.cc      | 6 +++++-
 tensorflow/core/platform/cloud/oauth_client_test.cc | 1 -
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/platform/cloud/oauth_client.cc b/tensorflow/core/platform/cloud/oauth_client.cc
index 9b85cae9b9..a8657359a3 100644
--- a/tensorflow/core/platform/cloud/oauth_client.cc
+++ b/tensorflow/core/platform/cloud/oauth_client.cc
@@ -95,6 +95,11 @@ Status CreateSignature(RSA* private_key, StringPiece to_sign,
   if (!md) {
     return errors::Internal("Could not get a sha256 encryptor.");
   }
+
+  // EVP_MD_CTX_destroy is renamed to EVP_MD_CTX_free in OpenSSL 1.1.0 but
+  // the old name is still retained as a compatibility macro.
+  // Keep this around until support is dropped for OpenSSL 1.0
+  // https://www.openssl.org/news/cl110.txt
   std::unique_ptr<EVP_MD_CTX, std::function<void(EVP_MD_CTX*)>> md_ctx(
       EVP_MD_CTX_create(), [](EVP_MD_CTX* ptr) { EVP_MD_CTX_destroy(ptr); });
   if (!md_ctx) {
@@ -119,7 +124,6 @@ Status CreateSignature(RSA* private_key, StringPiece to_sign,
   if (EVP_DigestSignFinal(md_ctx.get(), sig.get(), &sig_len) != 1) {
     return errors::Internal("DigestFinal (signature compute) failed.");
   }
-  EVP_MD_CTX_cleanup(md_ctx.get());
   return Base64Encode(StringPiece(reinterpret_cast<char*>(sig.get()), sig_len),
                       signature);
 }
diff --git a/tensorflow/core/platform/cloud/oauth_client_test.cc b/tensorflow/core/platform/cloud/oauth_client_test.cc
index 1cd0641cd3..ce3b9d79c8 100644
--- a/tensorflow/core/platform/cloud/oauth_client_test.cc
+++ b/tensorflow/core/platform/cloud/oauth_client_test.cc
@@ -166,7 +166,6 @@ TEST(OAuthClientTest, GetTokenFromServiceAccountJson) {
                 const_cast<unsigned char*>(
                     reinterpret_cast<const unsigned char*>(signature.data())),
                 signature.size()));
-  EVP_MD_CTX_cleanup(md_ctx);
 
   // Free all the crypto-related resources.
   EVP_PKEY_free(key);
-- 
GitLab


From 13fc6cfefb8204d5144f4c94b4b8cb50f69e8e8c Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Fri, 4 Jan 2019 21:25:17 -0800
Subject: [PATCH 0237/2345] Grappler micro-optimizations for virtual scheduler.

* Only collect additional memory information when VLOG(1) is active.
* Use const reference to avoid copy.

PiperOrigin-RevId: 227954880
---
 .../core/grappler/costs/virtual_scheduler.cc  | 26 ++++++++++++-------
 .../core/grappler/costs/virtual_scheduler.h   |  3 +++
 .../grappler/costs/virtual_scheduler_test.cc  |  4 ++-
 3 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.cc b/tensorflow/core/grappler/costs/virtual_scheduler.cc
index ae5200b359..7071c679dc 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler.cc
+++ b/tensorflow/core/grappler/costs/virtual_scheduler.cc
@@ -319,6 +319,7 @@ VirtualScheduler::VirtualScheduler(const GrapplerItem* grappler_item,
       placer_(cluster) {
   graph_costs_.num_ops_total = 0;
   initialized_ = false;
+  track_mem_usage_snapshot_ = VLOG_IS_ON(1);
 }
 
 VirtualScheduler::VirtualScheduler(const bool use_static_shapes,
@@ -331,6 +332,7 @@ VirtualScheduler::VirtualScheduler(const bool use_static_shapes,
       placer_(cluster) {
   graph_costs_.num_ops_total = 0;
   initialized_ = false;
+  track_mem_usage_snapshot_ = VLOG_IS_ON(1);
 }
 
 Status VirtualScheduler::Init(const GrapplerItem* item) {
@@ -562,7 +564,7 @@ void VirtualScheduler::MaybeUpdateInputOutput(const NodeDef* node) {
       inputs.push_back(control_message);
       outputs.push_back(control_message);
     } else {
-      auto output_properties =
+      const auto& output_properties =
           graph_properties_->GetOutputProperties(NodeName(input_source_name));
       // Like with HasInputProperties, if a node does not have output
       // properties, it's likely it was pruned during the shape inference run.
@@ -778,13 +780,16 @@ bool VirtualScheduler::MarkCurrNodeExecuted(const Costs& node_costs) {
   auto& op_cost = FindOrCreateZero(op_name, &op_to_cost_);
   op_cost = CombineCosts(op_cost, node_costs);
 
-  // Also keep track of op counts and costs per op (with their shapes).
-  OpContext op_context = GetCurrNode();
-  string node_description = GetOpDescription(op_context.op_info);
-  op_counts_[node_description] += 1;
-  op_costs_[node_description] =
-      std::make_pair(node_costs.execution_time.asMicroSeconds().count(),
-                     !node_costs.inaccurate);
+  if (VLOG_IS_ON(2)) {
+    // Also keep track of op counts and costs per op (with their shapes).
+    OpContext op_context = GetCurrNode();
+
+    string node_description = GetOpDescription(op_context.op_info);
+    op_counts_[node_description] += 1;
+    op_costs_[node_description] =
+        std::make_pair(node_costs.execution_time.asMicroSeconds().count(),
+                       !node_costs.inaccurate);
+  }
 
   // Update node and device states.
   auto& node_state = node_map_[node];
@@ -868,7 +873,10 @@ bool VirtualScheduler::MarkCurrNodeExecuted(const Costs& node_costs) {
     // check max memory usage.
     if (device.memory_usage > device.max_memory_usage) {
       device.max_memory_usage = device.memory_usage;
-      device.mem_usage_snapshot_at_peak = device.nodes_in_memory;
+
+      if (track_mem_usage_snapshot_) {
+        device.mem_usage_snapshot_at_peak = device.nodes_in_memory;
+      }
     }
   }
 
diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.h b/tensorflow/core/grappler/costs/virtual_scheduler.h
index 6a835f32d1..f500e4ab51 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler.h
+++ b/tensorflow/core/grappler/costs/virtual_scheduler.h
@@ -305,6 +305,8 @@ class VirtualScheduler {
     return &node_map_;
   }
 
+  void enable_mem_usage_tracking() { track_mem_usage_snapshot_ = true; }
+
  private:
   // Constants.
   const string kAttrInputSrc = "input_source_";
@@ -356,6 +358,7 @@ class VirtualScheduler {
   const GrapplerItem* grappler_item_;  // Not owned.
   bool use_static_shapes_;
   bool initialized_;
+  bool track_mem_usage_snapshot_;
 
   VirtualPlacer placer_;  // owned.
 };
diff --git a/tensorflow/core/grappler/costs/virtual_scheduler_test.cc b/tensorflow/core/grappler/costs/virtual_scheduler_test.cc
index 0a695458e1..c5651715f4 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler_test.cc
+++ b/tensorflow/core/grappler/costs/virtual_scheduler_test.cc
@@ -31,7 +31,9 @@ namespace grappler {
 class TestVirtualScheduler : public VirtualScheduler {
  public:
   TestVirtualScheduler(const bool use_static_shapes, Cluster* cluster)
-      : VirtualScheduler(use_static_shapes, cluster, &ready_node_manager_) {}
+      : VirtualScheduler(use_static_shapes, cluster, &ready_node_manager_) {
+    enable_mem_usage_tracking();
+  }
 
   FRIEND_TEST(VirtualSchedulerTest, MemoryUsage);
   FRIEND_TEST(VirtualSchedulerTest, ControlDependency);
-- 
GitLab


From 20dc97f0f11534adca40d66e5268eb74e012e254 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 21:37:11 -0800
Subject: [PATCH 0238/2345] export find_all_hinted_output_nodes for Ophint.

PiperOrigin-RevId: 227955652
---
 tensorflow/lite/python/convert_test.py | 23 ++++++++++++++++++
 tensorflow/lite/python/op_hint.py      | 32 +++++++++++++++++++++++++-
 2 files changed, 54 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/python/convert_test.py b/tensorflow/lite/python/convert_test.py
index cf49ee2b47..a29d431322 100644
--- a/tensorflow/lite/python/convert_test.py
+++ b/tensorflow/lite/python/convert_test.py
@@ -29,6 +29,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import test_util
 from tensorflow.python.framework.graph_util_impl import _bfs_for_reachable_nodes
 from tensorflow.python.framework.graph_util_impl import _extract_graph_summary
+from tensorflow.python.framework.graph_util_impl import _node_name
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
@@ -389,6 +390,28 @@ class ConvertTestOpHint(test_util.TensorFlowTestCase):
     with self.assertRaises(ValueError):
       convert.convert_dtype_to_tflite_type(dtypes.bool)
 
+  def testFindHintedOutputNodes(self):
+    """Test if all hinted output nodes are correctly found."""
+
+    def _build_ophinted_op(name, input1, input2):
+      custom_op = op_hint.OpHint(name)
+      input1 = custom_op.add_input(input1)
+      input2 = custom_op.add_input(input2)
+      output = math_ops.mul(input1, input2)
+      return custom_op.add_output(output)
+
+    output_1 = _build_ophinted_op("custom_op_1", array_ops.constant([1.]),
+                                  array_ops.constant([2.]))
+    output_2 = _build_ophinted_op("custom_op_2", array_ops.constant([3.]),
+                                  array_ops.constant([4.]))
+    with self.cached_session() as sess:
+      hinted_outputs_nodes = op_hint.find_all_hinted_output_nodes(sess)
+      expected_hinted_output_nodes = [
+          _node_name(output_1.name),
+          _node_name(output_2.name)
+      ]
+      self.assertCountEqual(hinted_outputs_nodes, expected_hinted_output_nodes)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/lite/python/op_hint.py b/tensorflow/lite/python/op_hint.py
index 8d7f9316bf..6ec050171f 100644
--- a/tensorflow/lite/python/op_hint.py
+++ b/tensorflow/lite/python/op_hint.py
@@ -964,6 +964,35 @@ def _convert_op_hints_to_stubs_helper(
   return curr_graph_def
 
 
+def find_all_hinted_output_nodes(session=None, graph_def=None):
+  """Find all Ophints output nodes in the graph.
+
+  This is used to get all the output nodes those are ophinted, it is important
+  for operation like convert_variables_to_constants keep all ophints structure.
+  Note: only one of session or graph_def should be used, not both.
+
+  Args:
+    session: A TensorFlow session that contains the graph to convert.
+    graph_def: A graph def that we should convert.
+
+  Returns:
+    A list of OpHints output nodes.
+  Raises:
+    ValueError: If both session and graph_def are provided.
+  """
+  if session is not None and graph_def is not None:
+    raise ValueError("Provide only one of session and graph_def.")
+  hinted_outputs_nodes = []
+  if session is not None:
+    hints = _find_all_hints_in_graph_def(session.graph_def)
+  elif graph_def is not None:
+    hints = _find_all_hints_in_graph_def(graph_def)
+  for hint in _six.itervalues(hints):
+    _, ouput_nodes = hint.flattened_inputs_and_outputs()
+    hinted_outputs_nodes.extend(ouput_nodes)
+  return hinted_outputs_nodes
+
+
 def convert_op_hints_to_stubs(session=None,
                               graph_def=None,
                               write_callback=lambda graph_def, comments: None):
@@ -996,6 +1025,7 @@ def convert_op_hints_to_stubs(session=None,
 
 
 _allowed_symbols = [
-    "OpHint", "convert_op_hints_to_stubs", "convert_op_hints_to_stubs_new"
+    "OpHint", "convert_op_hints_to_stubs", "convert_op_hints_to_stubs_new",
+    "find_all_hinted_output_nodes"
 ]
 remove_undocumented(__name__, _allowed_symbols)
-- 
GitLab


From 058bb7c3512455d2d83d2094a6f6e60b0ca6ee05 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 22:49:14 -0800
Subject: [PATCH 0239/2345] Rollback flaky test

PiperOrigin-RevId: 227959516
---
 tensorflow/core/BUILD                         |  1 -
 .../core/util/presized_cuckoo_map_test.cc     | 48 ++-----------------
 2 files changed, 5 insertions(+), 44 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 460353df41..7c9decbd09 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -3813,7 +3813,6 @@ tf_cc_tests(
         "//tensorflow/core/kernels:ops_util",
         "//third_party/eigen3",
         "@com_google_absl//absl/base",
-        "@com_google_absl//absl/time",
     ],
 )
 
diff --git a/tensorflow/core/util/presized_cuckoo_map_test.cc b/tensorflow/core/util/presized_cuckoo_map_test.cc
index 8dbb2ec065..f2c7904b00 100644
--- a/tensorflow/core/util/presized_cuckoo_map_test.cc
+++ b/tensorflow/core/util/presized_cuckoo_map_test.cc
@@ -15,7 +15,6 @@ limitations under the License.
 
 #include <array>
 
-#include "absl/time/clock.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/fingerprint.h"
 #include "tensorflow/core/platform/test.h"
@@ -53,48 +52,11 @@ TEST(PresizedCuckooMapTest, Basic) {
 }
 
 TEST(PresizedCuckooMapTest, Prefetch) {
-  {
-    PresizedCuckooMap<int64> pscm(2);
-    EXPECT_TRUE(pscm.InsertUnique(1, 2));
-    // Works for both present and absent keys.
-    pscm.PrefetchKey(1);
-    pscm.PrefetchKey(2);
-  }
-
-  // Do not run in debug mode, when prefetch is not implemented, or when
-  // sanitizers are enabled.
-#if defined(NDEBUG) && defined(__GNUC__) && !defined(ADDRESS_SANITIZER) && \
-    !defined(MEMORY_SANITIZER) && !defined(THREAD_SANITIZER) &&            \
-    !defined(UNDEFINED_BEHAVIOR_SANITIZER)
-  const auto now = [] { return absl::Now(); };
-
-  // Make size enough to not fit in L2 cache (16.7 Mb)
-  static constexpr int size = 1 << 22;
-  PresizedCuckooMap<int64> pscm(size);
-  for (int i = 0; i < size; ++i) {
-    pscm.InsertUnique(i, i);
-  }
-
-  absl::Duration no_prefetch, prefetch;
-  int64 out;
-  for (int iter = 0; iter < 10; ++iter) {
-    auto time = now();
-    for (int i = 0; i < size; ++i) {
-      testing::DoNotOptimize(pscm.Find(i, &out));
-    }
-    no_prefetch += now() - time;
-
-    time = now();
-    for (int i = 0; i < size; ++i) {
-      pscm.PrefetchKey(i + 20);
-      testing::DoNotOptimize(pscm.Find(i, &out));
-    }
-    prefetch += now() - time;
-  }
-
-  // no_prefetch is at least 30% slower.
-  EXPECT_GE(1.0 * no_prefetch / prefetch, 1.3);
-#endif
+  PresizedCuckooMap<int64> pscm(2);
+  EXPECT_TRUE(pscm.InsertUnique(1, 2));
+  // Works for both present and absent keys.
+  pscm.PrefetchKey(1);
+  pscm.PrefetchKey(2);
 }
 
 TEST(PresizedCuckooMapTest, TooManyItems) {
-- 
GitLab


From 9e8962b78b3c514cb1163c01fa7ab767592f952f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 5 Jan 2019 01:03:08 -0800
Subject: [PATCH 0240/2345] compat: Update forward compatibility horizon to
 2019-01-05

PiperOrigin-RevId: 227969226
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 66c4fa4893..3b808d71a5 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 4)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 5)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 186f573d7067b1df32e3bf2ffb7d34cffcdb4c64 Mon Sep 17 00:00:00 2001
From: Dheeraj Rajaram Reddy <dheeraj98reddy@gmail.com>
Date: Sat, 5 Jan 2019 17:31:21 +0530
Subject: [PATCH 0241/2345] Minor bug fixes

`take_while` ShortCircuit tests are parameterized
Fix errors in `BUILD` files using `buildifier`
LoopIteratorPredicate takes `vector<Tensor>&`
---
 tensorflow/core/kernels/data/experimental/BUILD |  2 +-
 .../data/experimental/take_while_dataset_op.cc  |  8 +++++---
 .../python/data/experimental/kernel_tests/BUILD |  4 ++--
 .../kernel_tests/take_while_test.py             | 17 +++++++++++------
 tensorflow/python/data/experimental/ops/BUILD   |  4 ++--
 5 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/tensorflow/core/kernels/data/experimental/BUILD b/tensorflow/core/kernels/data/experimental/BUILD
index 53d1f02a3a..d71b473b19 100644
--- a/tensorflow/core/kernels/data/experimental/BUILD
+++ b/tensorflow/core/kernels/data/experimental/BUILD
@@ -303,7 +303,7 @@ tf_kernel_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core/kernels/data:captured_function",
-        "//tensorflow/core/kernels/data:dataset_utils"
+        "//tensorflow/core/kernels/data:dataset_utils",
     ],
 )
 
diff --git a/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc b/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
index b670e4efaa..3a6f70e504 100644
--- a/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/kernels/data/captured_function.h"
 #include "tensorflow/core/kernels/data/dataset_utils.h"
+#include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
 namespace data {
@@ -33,7 +34,7 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
  public:
   using LoopIteratorPredicate =
       std::function<Status(IteratorContext*, InstantiatedCapturedFunction*,
-                           std::vector<Tensor>, bool*)>;
+                           std::vector<Tensor>&, bool*)>;
 
   explicit TakeWhileDatasetOp(OpKernelConstruction* ctx)
       : UnaryDatasetOpKernel(ctx) {
@@ -106,8 +107,9 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(new Iterator(
-          {this, strings::StrCat(prefix, "::TakeWhile")}, loop_pred_));
+      return MakeUnique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::TakeWhile")},
+          loop_pred_);
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/python/data/experimental/kernel_tests/BUILD b/tensorflow/python/data/experimental/kernel_tests/BUILD
index b4196685e2..e832231912 100644
--- a/tensorflow/python/data/experimental/kernel_tests/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/BUILD
@@ -650,13 +650,13 @@ py_test(
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:script_ops",
-        "//tensorflow/python:sparse_tensor",
+        "//tensorflow/python:math_ops",
         "//tensorflow/python/data/experimental/ops:take_while_ops",
         "//tensorflow/python/data/kernel_tests:test_base",
         "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python/eager:context",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/take_while_test.py b/tensorflow/python/data/experimental/kernel_tests/take_while_test.py
index 3faee0639f..7392f5420a 100644
--- a/tensorflow/python/data/experimental/kernel_tests/take_while_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/take_while_test.py
@@ -83,19 +83,24 @@ class TakeWhileTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.assertRaises(errors.OutOfRangeError):
       self.assertEqual(b"test", self.evaluate(next_element()))
 
-  def testTakewhileDatasetShortCircuit(self):
+  @parameterized.parameters(
+      (5, 3),
+      (10, 0),
+      (100, 5),
+      (8, 7))
+  def testTakewhileDatasetShortCircuit(self, size, index):
     def _predicate_func(data_elem):
       return data_elem
 
-    boolean_array = [True, True, True, True, False]
+    boolean_array = [True]*size
+    boolean_array[index] = False
     dataset = dataset_ops.Dataset.from_tensor_slices(boolean_array).apply(
         take_while_ops.take_while(_predicate_func))
 
     next_element = self.getNext(dataset)
-    self.assertEqual(True, self.evaluate(next_element()))
-    self.assertEqual(True, self.evaluate(next_element()))
-    self.assertEqual(True, self.evaluate(next_element()))
-    self.assertEqual(True, self.evaluate(next_element()))
+
+    for _ in range(index):
+      self.assertTrue(self.evaluate(next_element()))
 
     with self.assertRaises(errors.OutOfRangeError):
       self.evaluate(next_element())
diff --git a/tensorflow/python/data/experimental/ops/BUILD b/tensorflow/python/data/experimental/ops/BUILD
index 95e9509678..56bf59344f 100644
--- a/tensorflow/python/data/experimental/ops/BUILD
+++ b/tensorflow/python/data/experimental/ops/BUILD
@@ -359,9 +359,9 @@ py_library(
     srcs = ["take_while_ops.py"],
     srcs_version = "PY2AND3",
     deps = [
+        "//tensorflow/python:dtypes",
         "//tensorflow/python:experimental_dataset_ops_gen",
         "//tensorflow/python:framework_ops",
-        "//tensorflow/python:dtypes",
         "//tensorflow/python:function",
         "//tensorflow/python/data/ops:dataset_ops",
     ],
@@ -467,7 +467,7 @@ py_library(
         ":shuffle_ops",
         ":sleep",
         ":stats_ops",
-        "take_while_ops",
+        ":take_while_ops",
         ":threadpool",
         ":unique",
         ":writers",
-- 
GitLab


From f69eca8cef2e8536364bb5a6e5f966a5e071812f Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Sat, 5 Jan 2019 13:29:23 -0800
Subject: [PATCH 0242/2345] Enable test that works with control flow v2 now.

PiperOrigin-RevId: 228006449
---
 tensorflow/python/kernel_tests/tensor_array_ops_test.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
index 303e9a006a..3f1bd60d1a 100644
--- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py
+++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
@@ -1005,7 +1005,6 @@ class TensorArrayTest(test.TestCase):
     self._testWhileLoopWritePackGradients(
         dynamic_size=True, dtype=dtypes.float32)
 
-  @test_util.disable_control_flow_v2("b/119323158")
   def testGradSerialTwoLoops(self):
     with self.session(use_gpu=True):
       def loop(x):
-- 
GitLab


From 3e85e82fd5cd6b3100967cae899c2229636c2e95 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 6 Jan 2019 06:56:02 +0000
Subject: [PATCH 0243/2345] Fix incorrectly returned error type during
 RaggedTensor iteration

This fix tries to address the issue raised in 24679 where
RaggedTensor iteration does not stop correctly (eager mode):
```
import tensorflow as tf
r = tf.ragged.constant([[1., 2.], [3., 4., 5.], [6.]])
for elem in r:
    print(elem)
```

The reason was that `__getitem__()` should have thrown IndexError
(translated to StopIteration in next/__next__())
in order for python to correctly process iteration.

This fix fixes the issue.

This fix fixes 24679.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .../python/ops/ragged/ragged_getitem.py       | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/tensorflow/python/ops/ragged/ragged_getitem.py b/tensorflow/python/ops/ragged/ragged_getitem.py
index 001a400596..f5ac3e9a2a 100644
--- a/tensorflow/python/ops/ragged/ragged_getitem.py
+++ b/tensorflow/python/ops/ragged/ragged_getitem.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
@@ -150,6 +151,27 @@ def _ragged_getitem(rt_input, key_list):
   else:
     starts = rt_input.row_splits[:-1]
     limits = rt_input.row_splits[1:]
+    if context.executing_eagerly():
+      # In python, __getitem__ should throw IndexError for out of bound
+      # indices. This will allow iteration run correctly as python will
+      # translate IndexError into StopIteration for next()/__next__().
+      # Below is an example:
+      #    import tensorflow as tf
+      #    r = tf.ragged.constant([[1., 2.], [3., 4., 5.], [6.]])
+      #    for elem in r:
+      #      print(elem)
+      # In non eager mode, the exception is thrown when session runs
+      # so we don't know if out of bound happens before.
+      # In eager mode, however, it is possible to find out when to
+      # throw out of bound IndexError.
+      # In the following row_key >= len(starts) is checked. In case of
+      # TypeError which happens when row_key is not an integer, the exception
+      # will simply be ignored as it will be processed later anyway.
+      try:
+        if row_key >= len(starts):
+          raise IndexError('row key {} out of bounds'.format(row_key))
+      except TypeError:
+        pass
     row = rt_input.values[starts[row_key]:limits[row_key]]
     return row.__getitem__(inner_keys)
 
-- 
GitLab


From 1467dea7ee9550fed8e580cb627c05c62d126def Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 6 Jan 2019 06:59:28 +0000
Subject: [PATCH 0244/2345] Fix failed test cases due to returned IndexError
 change

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .../python/ops/ragged/ragged_tensor_test.py   | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/tensorflow/python/ops/ragged/ragged_tensor_test.py b/tensorflow/python/ops/ragged/ragged_tensor_test.py
index 89691b015d..ae2dda76bd 100644
--- a/tensorflow/python/ops/ragged/ragged_tensor_test.py
+++ b/tensorflow/python/ops/ragged/ragged_tensor_test.py
@@ -829,13 +829,13 @@ class RaggedTensorTest(ragged_test_util.RaggedTensorTestCase,
   @parameterized.parameters(
       # Tests for out-of-bound errors
       (SLICE_BUILDER[5],
-       (ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
+       (IndexError, ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
       (SLICE_BUILDER[-6],
-       (ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
+       (IndexError, ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
       (SLICE_BUILDER[0, 2],
-       (ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
+       (IndexError, ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
       (SLICE_BUILDER[3, 0],
-       (ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
+       (IndexError, ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
 
       # Indexing into an inner ragged dimension
       (SLICE_BUILDER[:, 3], ValueError,
@@ -954,13 +954,13 @@ class RaggedTensorTest(ragged_test_util.RaggedTensorTestCase,
 
       # Test for out-of-bounds errors.
       (SLICE_BUILDER[1, 0],
-       (ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
+       (IndexError, ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
       (SLICE_BUILDER[0, 0, 3],
-       (ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
+       (IndexError, ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
       (SLICE_BUILDER[5],
-       (ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
+       (IndexError, ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
       (SLICE_BUILDER[0, 5],
-       (ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
+       (IndexError, ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
   )
   def testRaggedTensorGetItemErrorsWithRaggedRank2(self, slice_spec, expected,
                                                    message):
@@ -983,9 +983,9 @@ class RaggedTensorTest(ragged_test_util.RaggedTensorTestCase,
 
   @parameterized.parameters(
       (SLICE_BUILDER[0],
-       (ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
+       (IndexError, ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
       (SLICE_BUILDER[-1],
-       (ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
+       (IndexError, ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
   )
   def testRaggedTensorGetItemErrorsWithEmptyTensor(self, slice_spec, expected,
                                                    message):
-- 
GitLab


From 453ebe01a3a1cc6382ec90539fc98e4554dece87 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 6 Jan 2019 07:00:16 +0000
Subject: [PATCH 0245/2345] Add additional test case for GitHub issue 24679

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/ops/ragged/ragged_tensor_test.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tensorflow/python/ops/ragged/ragged_tensor_test.py b/tensorflow/python/ops/ragged/ragged_tensor_test.py
index ae2dda76bd..706b6eccf2 100644
--- a/tensorflow/python/ops/ragged/ragged_tensor_test.py
+++ b/tensorflow/python/ops/ragged/ragged_tensor_test.py
@@ -1207,5 +1207,18 @@ class RaggedTensorTest(ragged_test_util.RaggedTensorTestCase,
       res2 = session.partial_run(handle, r2, feed_dict={c: c_val})
       self.assertAllEqual(res2, [15, 7])
 
+  # Test case for GitHub issue 24679.
+  def testEagerForLoop(self):
+    if not context.executing_eagerly():
+      return
+
+    values = [[1., 2.], [3., 4., 5.], [6.]]
+    r = ragged_factory_ops.constant(values)
+    i = 0
+    for elem in r:
+      value = values[i]
+      i += 1
+      self.assertAllEqual(elem, value)
+
 if __name__ == '__main__':
   googletest.main()
-- 
GitLab


From 929a9173c55597fb612d4aca75769bd9f8ac41a1 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 6 Jan 2019 07:04:00 +0000
Subject: [PATCH 0246/2345] Fix line too long issue

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .../python/ops/ragged/ragged_tensor_test.py   | 30 ++++++++++++-------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/tensorflow/python/ops/ragged/ragged_tensor_test.py b/tensorflow/python/ops/ragged/ragged_tensor_test.py
index 706b6eccf2..0c9506c731 100644
--- a/tensorflow/python/ops/ragged/ragged_tensor_test.py
+++ b/tensorflow/python/ops/ragged/ragged_tensor_test.py
@@ -829,13 +829,17 @@ class RaggedTensorTest(ragged_test_util.RaggedTensorTestCase,
   @parameterized.parameters(
       # Tests for out-of-bound errors
       (SLICE_BUILDER[5],
-       (IndexError, ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
+       (IndexError, ValueError, errors.InvalidArgumentError),
+       '.*out of bounds.*'),
       (SLICE_BUILDER[-6],
-       (IndexError, ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
+       (IndexError, ValueError, errors.InvalidArgumentError),
+       '.*out of bounds.*'),
       (SLICE_BUILDER[0, 2],
-       (IndexError, ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
+       (IndexError, ValueError, errors.InvalidArgumentError),
+       '.*out of bounds.*'),
       (SLICE_BUILDER[3, 0],
-       (IndexError, ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
+       (IndexError, ValueError, errors.InvalidArgumentError),
+       '.*out of bounds.*'),
 
       # Indexing into an inner ragged dimension
       (SLICE_BUILDER[:, 3], ValueError,
@@ -954,13 +958,17 @@ class RaggedTensorTest(ragged_test_util.RaggedTensorTestCase,
 
       # Test for out-of-bounds errors.
       (SLICE_BUILDER[1, 0],
-       (IndexError, ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
+       (IndexError, ValueError, errors.InvalidArgumentError),
+       '.*out of bounds.*'),
       (SLICE_BUILDER[0, 0, 3],
-       (IndexError, ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
+       (IndexError, ValueError, errors.InvalidArgumentError),
+       '.*out of bounds.*'),
       (SLICE_BUILDER[5],
-       (IndexError, ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
+       (IndexError, ValueError, errors.InvalidArgumentError),
+       '.*out of bounds.*'),
       (SLICE_BUILDER[0, 5],
-       (IndexError, ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
+       (IndexError, ValueError, errors.InvalidArgumentError),
+       '.*out of bounds.*'),
   )
   def testRaggedTensorGetItemErrorsWithRaggedRank2(self, slice_spec, expected,
                                                    message):
@@ -983,9 +991,11 @@ class RaggedTensorTest(ragged_test_util.RaggedTensorTestCase,
 
   @parameterized.parameters(
       (SLICE_BUILDER[0],
-       (IndexError, ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
+       (IndexError, ValueError, errors.InvalidArgumentError),
+       '.*out of bounds.*'),
       (SLICE_BUILDER[-1],
-       (IndexError, ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
+       (IndexError, ValueError, errors.InvalidArgumentError),
+       '.*out of bounds.*'),
   )
   def testRaggedTensorGetItemErrorsWithEmptyTensor(self, slice_spec, expected,
                                                    message):
-- 
GitLab


From 933109e06a36d1e85daa59214ee37f0e3d170cc8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 6 Jan 2019 01:03:37 -0800
Subject: [PATCH 0247/2345] compat: Update forward compatibility horizon to
 2019-01-06

PiperOrigin-RevId: 228038143
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 3b808d71a5..b779c80aa7 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 5)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 6)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From a0804de99207eb89b38a315b7baa0eb2976931c7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 6 Jan 2019 03:06:52 -0800
Subject: [PATCH 0248/2345] Internal change

PiperOrigin-RevId: 228045421
---
 tensorflow/contrib/BUILD       | 1 -
 tensorflow/contrib/__init__.py | 1 -
 2 files changed, 2 deletions(-)

diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD
index 832db0f4ab..307bb6eca3 100644
--- a/tensorflow/contrib/BUILD
+++ b/tensorflow/contrib/BUILD
@@ -63,7 +63,6 @@ py_library(
         "//tensorflow/contrib/libsvm",
         "//tensorflow/contrib/linear_optimizer:sdca_estimator_py",
         "//tensorflow/contrib/linear_optimizer:sdca_ops_py",
-        "//tensorflow/contrib/lite/python:lite",
         "//tensorflow/contrib/lookup:lookup_py",
         "//tensorflow/contrib/losses:losses_py",
         "//tensorflow/contrib/losses:metric_learning_py",
diff --git a/tensorflow/contrib/__init__.py b/tensorflow/contrib/__init__.py
index 4f1a2a5693..af59686120 100644
--- a/tensorflow/contrib/__init__.py
+++ b/tensorflow/contrib/__init__.py
@@ -91,7 +91,6 @@ from tensorflow.contrib import tpu
 from tensorflow.contrib import training
 from tensorflow.contrib import util
 from tensorflow.contrib.eager.python import tfe as eager
-from tensorflow.contrib.lite.python import lite
 from tensorflow.contrib.optimizer_v2 import optimizer_v2_symbols as optimizer_v2
 from tensorflow.contrib.receptive_field import receptive_field_api as receptive_field
 from tensorflow.contrib.recurrent.python import recurrent_api as recurrent
-- 
GitLab


From c67e5c39503c4ca2f1b383acfe55cd4fd8fba37d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 6 Jan 2019 03:15:34 -0800
Subject: [PATCH 0249/2345] Extend config generation to include a default CPU
 configuration and provide a platform definition for this configuration.

We create a new platform instead of changing the old one so we can transition
without the need to change all users atomically.

PiperOrigin-RevId: 228045771
---
 third_party/toolchains/BUILD                  | 28 ++++++++--
 .../toolchains/preconfig/generate/BUILD       |  6 ++
 .../preconfig/generate/containers.bzl         |  1 +
 .../preconfig/generate/generate.bzl           | 55 ++++++++++++-------
 .../toolchains/preconfig/generate/generate.sh | 31 ++++++++---
 .../preconfig/generate/workspace.bzl          |  7 +++
 6 files changed, 96 insertions(+), 32 deletions(-)

diff --git a/third_party/toolchains/BUILD b/third_party/toolchains/BUILD
index f2248341c4..6ed6e5c367 100644
--- a/third_party/toolchains/BUILD
+++ b/third_party/toolchains/BUILD
@@ -4,10 +4,9 @@ package(default_visibility = ["//visibility:public"])
 
 load("//third_party/toolchains/preconfig/generate:containers.bzl", "container_digests")
 
-# Platform for use with remote execution with
-# custom container based off RBE Ubuntu16_04
-# http://gcr.io/cloud-marketplace/google/rbe-ubuntu16-04
-# Built with //tensorflow/tools/ci_build/Dockerfile.rbe.cpu
+# TODO(b/122347293): This is the RBE config based on the CPU configuration / image provided
+# in the asci-toolchain setup. Delete this once we switched CPU remote builds to the
+# new platform below.
 platform(
     name = "rbe_ubuntu16_04-tf",
     constraint_values = [
@@ -23,6 +22,26 @@ platform(
         }""",
 )
 
+# Remote build platforms.
+# Each of the platform rules here provide a platform definition that is bound to a docker image.
+# The result of the skylark configuration is checked into
+# //tensorflow/third_party/toolchains/preconfig.
+
+# Built with //tensorflow/tools/ci_build/Dockerfile.rbe.cpu.
+platform(
+    name = "rbe_ubuntu16.04",
+    constraint_values = [
+        "@bazel_tools//platforms:x86_64",
+        "@bazel_tools//platforms:linux",
+    ],
+    remote_execution_properties = """
+        properties: {
+            name: "container-image"
+            value:"docker://gcr.io/tensorflow-testing/nosla-ubuntu16.04@%s"
+        }""" % container_digests["ubuntu16.04"],
+)
+
+# Built with //tensorflow/tools/ci_build/Dockerfile.rbe.cuda9.0-cudnn7-ubuntu14.04.
 platform(
     name = "rbe_cuda9.0-cudnn7-ubuntu14.04",
     constraint_values = [
@@ -36,6 +55,7 @@ platform(
         }""" % container_digests["cuda9.0-cudnn7-ubuntu14.04"],
 )
 
+# Built with //tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04.
 platform(
     name = "rbe_cuda10.0-cudnn7-ubuntu14.04",
     constraint_values = [
diff --git a/third_party/toolchains/preconfig/generate/BUILD b/third_party/toolchains/preconfig/generate/BUILD
index b4c98dc94d..ad79255251 100644
--- a/third_party/toolchains/preconfig/generate/BUILD
+++ b/third_party/toolchains/preconfig/generate/BUILD
@@ -2,6 +2,12 @@ licenses(["restricted"])
 
 load(":generate.bzl", "tensorflow_rbe_config")
 
+tensorflow_rbe_config(
+    name = "ubuntu16.04-py3-clang",
+    compiler = "clang",
+    python_version = "3",
+)
+
 tensorflow_rbe_config(
     name = "ubuntu14.04-py3-gcc-cuda9.0-cudnn7-tensorrt5",
     compiler = "gcc",
diff --git a/third_party/toolchains/preconfig/generate/containers.bzl b/third_party/toolchains/preconfig/generate/containers.bzl
index 67e43485ed..428208523b 100644
--- a/third_party/toolchains/preconfig/generate/containers.bzl
+++ b/third_party/toolchains/preconfig/generate/containers.bzl
@@ -1,4 +1,5 @@
 container_digests = {
+    "ubuntu16.04": "sha256:d0d98c53111c3ec071aa81632a2b0d6f210e5c2411c5172e31f99002125ec4de",
     "cuda9.0-cudnn7-ubuntu14.04": "sha256:006a76ee1838122ff7f21ebac85f24c1ef350d4dd79b3ceff0e4fe649ed90d33",
     "cuda10.0-cudnn7-ubuntu14.04": "sha256:e36f05f1ff39e39ddf07122e37f2b1895948bb6f7acc3db37a3c496be5e66228",
 }
diff --git a/third_party/toolchains/preconfig/generate/generate.bzl b/third_party/toolchains/preconfig/generate/generate.bzl
index 75deea41b8..fc485d43d2 100644
--- a/third_party/toolchains/preconfig/generate/generate.bzl
+++ b/third_party/toolchains/preconfig/generate/generate.bzl
@@ -3,30 +3,38 @@ load(
     "docker_toolchain_autoconfig",
 )
 
-def _tensorflow_rbe_config(name, cuda_version, cudnn_version, python_version, compiler, tensorrt_version):
-    docker_toolchain_autoconfig(
-        name = name,
-        base = "@cuda%s-cudnn%s-ubuntu14.04//image" % (cuda_version, cudnn_version),
-        bazel_version = "0.19.2",
+def _tensorflow_rbe_config(name, compiler, python_version, cuda_version = None, cudnn_version = None, tensorrt_version = None):
+    base = "@ubuntu16.04//image"
+    config_repos = [
+        "local_config_python",
+        "local_config_cc",
+    ]
+    env = {
+        "ABI_VERSION": "gcc",
+        "ABI_LIBC_VERSION": "glibc_2.19",
+        "BAZEL_COMPILER": compiler,
+        "BAZEL_HOST_SYSTEM": "i686-unknown-linux-gnu",
+        "BAZEL_TARGET_LIBC": "glibc_2.19",
+        "BAZEL_TARGET_CPU": "k8",
+        "BAZEL_TARGET_SYSTEM": "x86_64-unknown-linux-gnu",
+        "CC_TOOLCHAIN_NAME": "linux_gnu_x86",
+        "CC": compiler,
+        "PYTHON_BIN_PATH": "/usr/bin/python%s" % python_version,
+        "CLEAR_CACHE": "1",
+    }
+
+    if cuda_version != None:
+        base = "@cuda%s-cudnn%s-ubuntu14.04//image" % (cuda_version, cudnn_version)
+        # The cuda toolchain currently contains its own C++ toolchain definition,
+        # so we do not fetch local_config_cc.
         config_repos = [
-            "local_config_cuda",
             "local_config_python",
+            "local_config_cuda",
             "local_config_tensorrt",
-        ],
-        env = {
-            "ABI_VERSION": "gcc",
-            "ABI_LIBC_VERSION": "glibc_2.19",
-            "BAZEL_COMPILER": compiler,
-            "BAZEL_HOST_SYSTEM": "i686-unknown-linux-gnu",
-            "BAZEL_TARGET_LIBC": "glibc_2.19",
-            "BAZEL_TARGET_CPU": "k8",
-            "BAZEL_TARGET_SYSTEM": "x86_64-unknown-linux-gnu",
-            "CC_TOOLCHAIN_NAME": "linux_gnu_x86",
-            "CC": compiler,
-            "PYTHON_BIN_PATH": "/usr/bin/python%s" % python_version,
+        ]
+        env.update({
             "TF_NEED_CUDA": "1",
             "TF_CUDA_CLANG": "1" if compiler == "clang" else "0",
-            "CLEAR_CACHE": "1",
             "TF_CUDA_COMPUTE_CAPABILITIES": "3.0",
             "TF_ENABLE_XLA": "1",
             "TF_CUDNN_VERSION": cudnn_version,
@@ -35,7 +43,14 @@ def _tensorflow_rbe_config(name, cuda_version, cudnn_version, python_version, co
             "TF_NEED_TENSORRT" : "1",
             "TF_TENSORRT_VERSION": tensorrt_version,
             "TENSORRT_INSTALL_PATH": "/usr/lib/x86_64-linux-gnu",
-        },
+        })
+
+    docker_toolchain_autoconfig(
+        name = name,
+        base = base,
+        bazel_version = "0.21.0",
+        config_repos = config_repos,
+        env = env,
         mount_project = "$(mount_project)",
         tags = ["manual"],
         incompatible_changes_off = True,
diff --git a/third_party/toolchains/preconfig/generate/generate.sh b/third_party/toolchains/preconfig/generate/generate.sh
index 8e3a1e6ada..c05a4de6fb 100755
--- a/third_party/toolchains/preconfig/generate/generate.sh
+++ b/third_party/toolchains/preconfig/generate/generate.sh
@@ -37,8 +37,16 @@ TENSORRT_VERSION="${PLATFORM[5]}"
 
 # TODO(klimek): Put this into the name.
 
-if [[ "${COMPILER}" == "gcc" ]]; then
-  COMPILER="gcc-nvcc-${CUDA_VERSION}"
+if [[ -n "${CUDA_VERSION}" ]]; then
+  if [[ "${COMPILER}" == "gcc" ]]; then
+    COMPILER="gcc-nvcc-${CUDA_VERSION}"
+  fi
+  # Currently we create a special toolchain for clang when compiling with
+  # cuda enabled. We can get rid of this once the default toolchain bazel
+  # provides supports cuda.
+  if [[ "${COMPILER}" == "clang" ]]; then
+    COMPILER="cuda-clang"
+  fi
 fi
 
 echo "OS: ${OS}"
@@ -52,6 +60,8 @@ bazel build --define=mount_project="${PWD}" "${PKG}/generate:${TARGET}"
 cd "${TEMPDIR}"
 tar xvf "${ROOT}/bazel-bin/${PKG}/generate/${TARGET}_outputs.tar"
 
+# TODO(klimek): The skylark config rules should copy the files instead of
+# creating aliases.
 # Other than in @local_config_tensorrt, the header files in the remote config
 # repo are not relative to the repository root. Add a dummy include_prefix to
 # make them available as virtual includes.
@@ -74,14 +84,19 @@ mkdir "${OS}"
 # Python:
 mv local_config_python "${OS}/${PY_VERSION}"
 
-# Compiler:
-mv local_config_cuda/crosstool "${OS}/${COMPILER}"
+if [[ -n "${CUDA_VERSION}" ]]; then
+  # Compiler:
+  mv local_config_cuda/crosstool "${OS}/${COMPILER}"
 
-# CUDA:
-mv local_config_cuda "${OS}/${CUDA_VERSION}-${CUDNN_VERSION}"
+  # CUDA:
+  mv local_config_cuda "${OS}/${CUDA_VERSION}-${CUDNN_VERSION}"
 
-# TensorRT:
-mv local_config_tensorrt "${OS}/${TENSORRT_VERSION}"
+  # TensorRT:
+  mv local_config_tensorrt "${OS}/${TENSORRT_VERSION}"
+else
+  # Compiler:
+  mv local_config_cc "${OS}/${COMPILER}"
+fi
 
 # Cleanup for copybara.
 find "${OS}" -name 'BUILD' -o -name '*.bzl' |xargs buildifier
diff --git a/third_party/toolchains/preconfig/generate/workspace.bzl b/third_party/toolchains/preconfig/generate/workspace.bzl
index eb74022c24..0495173786 100644
--- a/third_party/toolchains/preconfig/generate/workspace.bzl
+++ b/third_party/toolchains/preconfig/generate/workspace.bzl
@@ -8,6 +8,13 @@ load(":containers.bzl", "container_digests")
 def _remote_config_workspace():
     container_repositories()
 
+    container_pull(
+        name = "ubuntu16.04",
+        registry = "gcr.io",
+        repository = "tensorflow-testing/nosla-ubuntu16.04",
+        digest = container_digests["ubuntu16.04"],
+    )
+
     container_pull(
         name = "cuda9.0-cudnn7-ubuntu14.04",
         registry = "gcr.io",
-- 
GitLab


From 92985e67564c119203f0aaca94b5a334d87c35e4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 6 Jan 2019 04:03:21 -0800
Subject: [PATCH 0250/2345] Add clang and python3 configurations for ubuntu
 16.04.

PiperOrigin-RevId: 228048061
---
 tensorflow/opensource_only.files              |    3 +
 .../preconfig/ubuntu16.04/clang/BUILD         |  111 ++
 .../preconfig/ubuntu16.04/clang/CROSSTOOL     | 1209 +++++++++++++++++
 .../preconfig/ubuntu16.04/clang/WORKSPACE     |    2 +
 .../preconfig/ubuntu16.04/clang/cc_wrapper.sh |   25 +
 .../ubuntu16.04/clang/dummy_toolchain.bzl     |   23 +
 .../ubuntu16.04/clang/tools/cpp/empty.cc      |    1 +
 .../preconfig/ubuntu16.04/py3/BUILD           |  205 +++
 .../preconfig/ubuntu16.04/py3/WORKSPACE       |    2 +
 9 files changed, 1581 insertions(+)
 create mode 100755 third_party/toolchains/preconfig/ubuntu16.04/clang/BUILD
 create mode 100755 third_party/toolchains/preconfig/ubuntu16.04/clang/CROSSTOOL
 create mode 100644 third_party/toolchains/preconfig/ubuntu16.04/clang/WORKSPACE
 create mode 100755 third_party/toolchains/preconfig/ubuntu16.04/clang/cc_wrapper.sh
 create mode 100755 third_party/toolchains/preconfig/ubuntu16.04/clang/dummy_toolchain.bzl
 create mode 100755 third_party/toolchains/preconfig/ubuntu16.04/clang/tools/cpp/empty.cc
 create mode 100755 third_party/toolchains/preconfig/ubuntu16.04/py3/BUILD
 create mode 100644 third_party/toolchains/preconfig/ubuntu16.04/py3/WORKSPACE

diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index 1054c285d6..e00d063cfe 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -53,6 +53,9 @@ tensorflow/third_party/toolchains/preconfig/generate/containers.bzl
 tensorflow/third_party/toolchains/preconfig/generate/generate.bzl
 tensorflow/third_party/toolchains/preconfig/generate/archives.bzl
 tensorflow/third_party/toolchains/preconfig/generate/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu16.04/clang/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu16.04/clang/dummy_toolchain.bzl
+tensorflow/third_party/toolchains/preconfig/ubuntu16.04/py3/BUILD
 tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/BUILD
 tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/dummy_toolchain.bzl
 tensorflow/third_party/toolchains/preconfig/win_1803/py36/BUILD
diff --git a/third_party/toolchains/preconfig/ubuntu16.04/clang/BUILD b/third_party/toolchains/preconfig/ubuntu16.04/clang/BUILD
new file mode 100755
index 0000000000..5a0c52f66a
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu16.04/clang/BUILD
@@ -0,0 +1,111 @@
+# Copyright 2016 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This becomes the BUILD file for @local_config_cc// under non-FreeBSD unixes.
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])  # Apache 2.0
+
+cc_library(
+    name = "malloc",
+)
+
+cc_library(
+    name = "stl",
+)
+
+filegroup(
+    name = "empty",
+    srcs = [],
+)
+
+filegroup(
+    name = "cc_wrapper",
+    srcs = ["cc_wrapper.sh"],
+)
+
+filegroup(
+    name = "compiler_deps",
+    srcs = glob(["extra_tools/**"]) + [":empty"],
+)
+
+# This is the entry point for --crosstool_top.  Toolchains are found
+# by lopping off the name of --crosstool_top and searching for
+# the "${CPU}" entry in the toolchains attribute.
+cc_toolchain_suite(
+    name = "toolchain",
+    toolchains = {
+        "k8|clang": ":cc-compiler-k8",
+        "k8": ":cc-compiler-k8",
+        "armeabi-v7a|compiler": ":cc-compiler-armeabi-v7a",
+        "armeabi-v7a": ":cc-compiler-armeabi-v7a",
+    },
+)
+
+cc_toolchain(
+    name = "cc-compiler-k8",
+    all_files = ":compiler_deps",
+    compiler_files = ":compiler_deps",
+    cpu = "k8",
+    dwp_files = ":empty",
+    dynamic_runtime_libs = [":empty"],
+    linker_files = ":compiler_deps",
+    objcopy_files = ":empty",
+    static_runtime_libs = [":empty"],
+    strip_files = ":empty",
+    supports_param_files = 1,
+    toolchain_identifier = "linux_gnu_x86",
+)
+
+toolchain(
+    name = "cc-toolchain-k8",
+    exec_compatible_with = [
+        # TODO(katre): add autodiscovered constraints for host CPU and OS.
+    ],
+    target_compatible_with = [
+        # TODO(katre): add autodiscovered constraints for host CPU and OS.
+    ],
+    toolchain = ":cc-compiler-k8",
+    toolchain_type = "@bazel_tools//tools/cpp:toolchain_type",
+)
+
+# Android tooling requires a default toolchain for the armeabi-v7a cpu.
+cc_toolchain(
+    name = "cc-compiler-armeabi-v7a",
+    all_files = ":empty",
+    compiler_files = ":empty",
+    cpu = "local",
+    dwp_files = ":empty",
+    dynamic_runtime_libs = [":empty"],
+    linker_files = ":empty",
+    objcopy_files = ":empty",
+    static_runtime_libs = [":empty"],
+    strip_files = ":empty",
+    supports_param_files = 1,
+    toolchain_identifier = "stub_armeabi-v7a",
+)
+
+toolchain(
+    name = "cc-toolchain-armeabi-v7a",
+    exec_compatible_with = [
+        # TODO(katre): add autodiscovered constraints for host CPU and OS.
+    ],
+    target_compatible_with = [
+        "@bazel_tools//platforms:arm",
+        "@bazel_tools//platforms:android",
+    ],
+    toolchain = ":cc-compiler-armabi-v7a",
+    toolchain_type = "@bazel_tools//tools/cpp:toolchain_type",
+)
diff --git a/third_party/toolchains/preconfig/ubuntu16.04/clang/CROSSTOOL b/third_party/toolchains/preconfig/ubuntu16.04/clang/CROSSTOOL
new file mode 100755
index 0000000000..48f82eb35d
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu16.04/clang/CROSSTOOL
@@ -0,0 +1,1209 @@
+# Copyright 2016 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+major_version: "local"
+minor_version: ""
+
+# Android tooling requires a default toolchain for the armeabi-v7a cpu.
+toolchain {
+  abi_version: "armeabi-v7a"
+  abi_libc_version: "armeabi-v7a"
+  builtin_sysroot: ""
+  compiler: "compiler"
+  host_system_name: "armeabi-v7a"
+  needsPic: true
+  supports_gold_linker: false
+  supports_incremental_linker: false
+  supports_fission: false
+  supports_interface_shared_objects: false
+  supports_normalizing_ar: false
+  supports_start_end_lib: false
+  target_libc: "armeabi-v7a"
+  target_cpu: "armeabi-v7a"
+  target_system_name: "armeabi-v7a"
+  toolchain_identifier: "stub_armeabi-v7a"
+
+  tool_path { name: "ar" path: "/bin/false" }
+  tool_path { name: "compat-ld" path: "/bin/false" }
+  tool_path { name: "cpp" path: "/bin/false" }
+  tool_path { name: "dwp" path: "/bin/false" }
+  tool_path { name: "gcc" path: "/bin/false" }
+  tool_path { name: "gcov" path: "/bin/false" }
+  tool_path { name: "ld" path: "/bin/false" }
+
+  tool_path { name: "nm" path: "/bin/false" }
+  tool_path { name: "objcopy" path: "/bin/false" }
+  tool_path { name: "objdump" path: "/bin/false" }
+  tool_path { name: "strip" path: "/bin/false" }
+  linking_mode_flags { mode: DYNAMIC }
+}
+
+toolchain {
+  toolchain_identifier: "linux_gnu_x86"
+  abi_version: "gcc"
+  abi_libc_version: "glibc_2.19"
+  builtin_sysroot: ""
+  compiler: "clang"
+  host_system_name: "i686-unknown-linux-gnu"
+  needsPic: true
+  supports_gold_linker: true
+  supports_incremental_linker: false
+  supports_fission: false
+  supports_interface_shared_objects: false
+  supports_normalizing_ar: false
+  supports_start_end_lib: true
+  target_libc: "glibc_2.19"
+  target_cpu: "k8"
+  target_system_name: "x86_64-unknown-linux-gnu"
+  cxx_flag: "-std=c++0x"
+  linker_flag: "-fuse-ld=gold"
+  linker_flag: "-Wl,-no-as-needed"
+  linker_flag: "-Wl,-z,relro,-z,now"
+  linker_flag: "-B/usr/local/bin"
+  linker_flag: "-lstdc++"
+  linker_flag: "-lm"
+  cxx_builtin_include_directory: "/usr/local/include"
+  cxx_builtin_include_directory: "/usr/local/lib/clang/7.0.0/include"
+  cxx_builtin_include_directory: "/usr/include/x86_64-linux-gnu"
+  cxx_builtin_include_directory: "/usr/include"
+  cxx_builtin_include_directory: "/usr/include/c++/4.9"
+  cxx_builtin_include_directory: "/usr/include/x86_64-linux-gnu/c++/4.9"
+  cxx_builtin_include_directory: "/usr/include/c++/4.9/backward"
+  objcopy_embed_flag: "-I"
+  objcopy_embed_flag: "binary"
+  unfiltered_cxx_flag: "-no-canonical-prefixes"
+  unfiltered_cxx_flag: "-Wno-builtin-macro-redefined"
+  unfiltered_cxx_flag: "-D__DATE__=\"redacted\""
+  unfiltered_cxx_flag: "-D__TIMESTAMP__=\"redacted\""
+  unfiltered_cxx_flag: "-D__TIME__=\"redacted\""
+  compiler_flag: "-U_FORTIFY_SOURCE"
+  compiler_flag: "-fstack-protector"
+  compiler_flag: "-Wall"
+  compiler_flag: "-Wthread-safety"
+  compiler_flag: "-Wself-assign"
+  compiler_flag: "-fcolor-diagnostics"
+  compiler_flag: "-fno-omit-frame-pointer"
+  tool_path {name: "ar" path: "/usr/bin/ar" }
+  tool_path {name: "ld" path: "/usr/bin/ld" }
+  tool_path {name: "cpp" path: "/usr/bin/cpp" }
+  tool_path {name: "gcc" path: "/usr/local/bin/clang" }
+  tool_path {name: "dwp" path: "/usr/bin/dwp" }
+  tool_path {name: "gcov" path: "None" }
+  tool_path {name: "nm" path: "/usr/bin/nm" }
+  tool_path {name: "objcopy" path: "/usr/bin/objcopy" }
+  tool_path {name: "objdump" path: "/usr/bin/objdump" }
+  tool_path {name: "strip" path: "/usr/bin/strip" }
+
+  compilation_mode_flags {
+    mode: DBG
+    compiler_flag: "-g"
+  }
+  compilation_mode_flags {
+    mode: OPT
+    compiler_flag: "-g0"
+    compiler_flag: "-O2"
+    compiler_flag: "-D_FORTIFY_SOURCE=1"
+    compiler_flag: "-DNDEBUG"
+    compiler_flag: "-ffunction-sections"
+    compiler_flag: "-fdata-sections"
+    linker_flag: "-Wl,--gc-sections"
+  }
+  linking_mode_flags { mode: DYNAMIC }
+
+
+    feature {
+      name: 'coverage'
+      provides: 'profile'
+      flag_set {
+        action: 'preprocess-assemble'
+        action: 'c-compile'
+        action: 'c++-compile'
+        action: 'c++-header-parsing'
+        action: 'c++-module-compile'
+        flag_group {
+        flag: '--coverage'
+      }
+      }
+      flag_set {
+        action: 'c++-link-dynamic-library'
+        action: 'c++-link-nodeps-dynamic-library'
+        action: 'c++-link-executable'
+        flag_group {
+        flag: '--coverage'
+      }
+      }
+    }
+  
+
+  feature {
+    name: 'fdo_optimize'
+    provides: 'profile'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      expand_if_all_available: 'fdo_profile_path'
+      flag_group {
+        flag: '-fprofile-use=%{fdo_profile_path}'
+        flag: '-fprofile-correction',
+      }
+    }
+  }
+}
+
+toolchain {
+  toolchain_identifier: "msys_x64_mingw"
+  abi_version: "local"
+  abi_libc_version: "local"
+  builtin_sysroot: ""
+  compiler: "mingw-gcc"
+  host_system_name: "local"
+  needsPic: false
+  target_libc: "mingw"
+  target_cpu: "x64_windows"
+  target_system_name: "local"
+
+  artifact_name_pattern {
+     category_name: 'executable'
+     prefix: ''
+     extension: '.exe'
+  }
+
+
+
+  linking_mode_flags { mode: DYNAMIC }
+}
+
+toolchain {
+  toolchain_identifier: "msvc_x64"
+  host_system_name: "local"
+  target_system_name: "local"
+
+  abi_version: "local"
+  abi_libc_version: "local"
+  target_cpu: "x64_windows"
+  compiler: "msvc-cl"
+  target_libc: "msvcrt"
+  default_python_version: "python2.7"
+
+
+
+  tool_path {
+    name: "ar"
+    path: ""
+  }
+  tool_path {
+    name: "ml"
+    path: ""
+  }
+  tool_path {
+    name: "cpp"
+    path: ""
+  }
+  tool_path {
+    name: "gcc"
+    path: ""
+  }
+  tool_path {
+    name: "gcov"
+    path: "wrapper/bin/msvc_nop.bat"
+  }
+  tool_path {
+    name: "ld"
+    path: ""
+  }
+  tool_path {
+    name: "nm"
+    path: "wrapper/bin/msvc_nop.bat"
+  }
+  tool_path {
+    name: "objcopy"
+    path: "wrapper/bin/msvc_nop.bat"
+  }
+  tool_path {
+    name: "objdump"
+    path: "wrapper/bin/msvc_nop.bat"
+  }
+  tool_path {
+    name: "strip"
+    path: "wrapper/bin/msvc_nop.bat"
+  }
+  supports_gold_linker: false
+  supports_start_end_lib: false
+  supports_interface_shared_objects: true
+  supports_incremental_linker: false
+  supports_normalizing_ar: true
+  needsPic: false
+
+  # TODO(pcloudy): Review those flags below, they should be defined by cl.exe
+  compiler_flag: "/DCOMPILER_MSVC"
+
+  # Don't define min/max macros in windows.h.
+  compiler_flag: "/DNOMINMAX"
+
+  # Platform defines.
+  compiler_flag: "/D_WIN32_WINNT=0x0601"
+  # Turn off warning messages.
+  compiler_flag: "/D_CRT_SECURE_NO_DEPRECATE"
+  compiler_flag: "/D_CRT_SECURE_NO_WARNINGS"
+
+  # Useful options to have on for compilation.
+  # Increase the capacity of object files to 2^32 sections.
+  compiler_flag: "/bigobj"
+  # Allocate 500MB for precomputed headers.
+  compiler_flag: "/Zm500"
+  # Catch C++ exceptions only and tell the compiler to assume that functions declared
+  # as extern "C" never throw a C++ exception.
+  compiler_flag: "/EHsc"
+
+  # Globally disabled warnings.
+  # Don't warn about elements of array being be default initialized.
+  compiler_flag: "/wd4351"
+  # Don't warn about no matching delete found.
+  compiler_flag: "/wd4291"
+  # Don't warn about diamond inheritance patterns.
+  compiler_flag: "/wd4250"
+  # Don't warn about insecure functions (e.g. non _s functions).
+  compiler_flag: "/wd4996"
+
+  linker_flag: "/MACHINE:X64"
+
+  feature {
+    name: "no_legacy_features"
+  }
+
+  artifact_name_pattern {
+     category_name: 'object_file'
+     prefix: ''
+     extension: '.obj'
+  }
+
+  artifact_name_pattern {
+     category_name: 'static_library'
+     prefix: ''
+     extension: '.lib'
+  }
+
+  artifact_name_pattern {
+     category_name: 'alwayslink_static_library'
+     prefix: ''
+     extension: '.lo.lib'
+  }
+
+  artifact_name_pattern {
+     category_name: 'executable'
+     prefix: ''
+     extension: '.exe'
+  }
+
+  artifact_name_pattern {
+     category_name: 'dynamic_library'
+     prefix: ''
+     extension: '.dll'
+  }
+
+  artifact_name_pattern {
+     category_name: 'interface_library'
+     prefix: ''
+     extension: '.if.lib'
+  }
+
+  # Suppress startup banner.
+  feature {
+    name: "nologo"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      action: "c++-module-compile"
+      action: "c++-module-codegen"
+      action: "c++-header-parsing"
+      action: "assemble"
+      action: "preprocess-assemble"
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      action: "c++-link-static-library"
+      flag_group {
+        flag: "/nologo"
+      }
+    }
+  }
+
+  feature {
+    name: 'has_configured_linker_path'
+  }
+
+  # This feature indicates strip is not supported, building stripped binary will just result a copy of orignial binary
+  feature {
+    name: 'no_stripping'
+  }
+
+  # This feature indicates this is a toolchain targeting Windows.
+  feature {
+    name: 'targets_windows'
+    implies: 'copy_dynamic_libraries_to_binary'
+    enabled: true
+  }
+
+  feature {
+    name: 'copy_dynamic_libraries_to_binary'
+  }
+
+  action_config {
+    config_name: 'assemble'
+    action_name: 'assemble'
+    tool {
+      tool_path: ''
+    }
+    implies: 'compiler_input_flags'
+    implies: 'compiler_output_flags'
+    implies: 'nologo'
+    implies: 'msvc_env'
+    implies: 'sysroot'
+  }
+
+  action_config {
+    config_name: 'preprocess-assemble'
+    action_name: 'preprocess-assemble'
+    tool {
+      tool_path: ''
+    }
+    implies: 'compiler_input_flags'
+    implies: 'compiler_output_flags'
+    implies: 'nologo'
+    implies: 'msvc_env'
+    implies: 'sysroot'
+  }
+
+  action_config {
+    config_name: 'c-compile'
+    action_name: 'c-compile'
+    tool {
+      tool_path: ''
+    }
+    implies: 'compiler_input_flags'
+    implies: 'compiler_output_flags'
+    implies: 'legacy_compile_flags'
+    implies: 'nologo'
+    implies: 'msvc_env'
+    implies: 'parse_showincludes'
+    implies: 'user_compile_flags'
+    implies: 'sysroot'
+    implies: 'unfiltered_compile_flags'
+  }
+
+  action_config {
+    config_name: 'c++-compile'
+    action_name: 'c++-compile'
+    tool {
+      tool_path: ''
+    }
+    implies: 'compiler_input_flags'
+    implies: 'compiler_output_flags'
+    implies: 'legacy_compile_flags'
+    implies: 'nologo'
+    implies: 'msvc_env'
+    implies: 'parse_showincludes'
+    implies: 'user_compile_flags'
+    implies: 'sysroot'
+    implies: 'unfiltered_compile_flags'
+  }
+
+  action_config {
+    config_name: 'c++-link-executable'
+    action_name: 'c++-link-executable'
+    tool {
+      tool_path: ''
+    }
+    implies: 'nologo'
+    implies: 'linkstamps'
+    implies: 'output_execpath_flags'
+    implies: 'input_param_flags'
+    implies: 'user_link_flags'
+    implies: 'legacy_link_flags'
+    implies: 'linker_subsystem_flag'
+    implies: 'linker_param_file'
+    implies: 'msvc_env'
+    implies: 'no_stripping'
+  }
+
+  action_config {
+    config_name: 'c++-link-dynamic-library'
+    action_name: 'c++-link-dynamic-library'
+    tool {
+      tool_path: ''
+    }
+    implies: 'nologo'
+    implies: 'shared_flag'
+    implies: 'linkstamps'
+    implies: 'output_execpath_flags'
+    implies: 'input_param_flags'
+    implies: 'user_link_flags'
+    implies: 'legacy_link_flags'
+    implies: 'linker_subsystem_flag'
+    implies: 'linker_param_file'
+    implies: 'msvc_env'
+    implies: 'no_stripping'
+    implies: 'has_configured_linker_path'
+    implies: 'def_file'
+  }
+
+  action_config {
+      config_name: 'c++-link-nodeps-dynamic-library'
+      action_name: 'c++-link-nodeps-dynamic-library'
+      tool {
+        tool_path: ''
+      }
+      implies: 'nologo'
+      implies: 'shared_flag'
+      implies: 'linkstamps'
+      implies: 'output_execpath_flags'
+      implies: 'input_param_flags'
+      implies: 'user_link_flags'
+      implies: 'legacy_link_flags'
+      implies: 'linker_subsystem_flag'
+      implies: 'linker_param_file'
+      implies: 'msvc_env'
+      implies: 'no_stripping'
+      implies: 'has_configured_linker_path'
+      implies: 'def_file'
+    }
+
+  action_config {
+    config_name: 'c++-link-static-library'
+    action_name: 'c++-link-static-library'
+    tool {
+      tool_path: ''
+    }
+    implies: 'nologo'
+    implies: 'archiver_flags'
+    implies: 'input_param_flags'
+    implies: 'linker_param_file'
+    implies: 'msvc_env'
+  }
+
+  # TODO(b/65151735): Remove legacy_compile_flags feature when legacy fields are
+  # not used in this crosstool
+  feature {
+    name: 'legacy_compile_flags'
+    flag_set {
+      expand_if_all_available: 'legacy_compile_flags'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      flag_group {
+        iterate_over: 'legacy_compile_flags'
+        flag: '%{legacy_compile_flags}'
+      }
+    }
+  }
+
+  feature {
+    name: "msvc_env"
+    env_set {
+      action: "c-compile"
+      action: "c++-compile"
+      action: "c++-module-compile"
+      action: "c++-module-codegen"
+      action: "c++-header-parsing"
+      action: "assemble"
+      action: "preprocess-assemble"
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      action: "c++-link-static-library"
+      env_entry {
+        key: "PATH"
+        value: ""
+      }
+      env_entry {
+        key: "TMP"
+        value: ""
+      }
+      env_entry {
+        key: "TEMP"
+        value: ""
+      }
+    }
+    implies: 'msvc_compile_env'
+    implies: 'msvc_link_env'
+  }
+
+  feature {
+    name: "msvc_compile_env"
+    env_set {
+      action: "c-compile"
+      action: "c++-compile"
+      action: "c++-module-compile"
+      action: "c++-module-codegen"
+      action: "c++-header-parsing"
+      action: "assemble"
+      action: "preprocess-assemble"
+      env_entry {
+        key: "INCLUDE"
+        value: ""
+      }
+    }
+  }
+
+  feature {
+    name: "msvc_link_env"
+    env_set {
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      action: "c++-link-static-library"
+      env_entry {
+        key: "LIB"
+        value: ""
+      }
+    }
+  }
+
+  feature {
+    name: 'include_paths'
+    flag_set {
+      action: "assemble"
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      flag_group {
+        iterate_over: 'quote_include_paths'
+        flag: '/I%{quote_include_paths}'
+      }
+      flag_group {
+        iterate_over: 'include_paths'
+        flag: '/I%{include_paths}'
+      }
+      flag_group {
+        iterate_over: 'system_include_paths'
+        flag: '/I%{system_include_paths}'
+      }
+    }
+  }
+
+  feature {
+    name: "preprocessor_defines"
+    flag_set {
+      action: "assemble"
+      action: "preprocess-assemble"
+      action: "c-compile"
+      action: "c++-compile"
+      action: "c++-header-parsing"
+      action: "c++-module-compile"
+      flag_group {
+        flag: "/D%{preprocessor_defines}"
+        iterate_over: "preprocessor_defines"
+      }
+    }
+  }
+
+  # Tell Bazel to parse the output of /showIncludes
+  feature {
+    name: 'parse_showincludes'
+    flag_set {
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-module-compile'
+      action: 'c++-header-parsing'
+      flag_group {
+        flag: "/showIncludes"
+      }
+    }
+  }
+
+
+  feature {
+    name: 'generate_pdb_file'
+    requires: {
+      feature: 'dbg'
+    }
+    requires: {
+      feature: 'fastbuild'
+    }
+  }
+
+  feature {
+    name: 'shared_flag'
+    flag_set {
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: '/DLL'
+      }
+    }
+  }
+
+  feature {
+    name: 'linkstamps'
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      expand_if_all_available: 'linkstamp_paths'
+      flag_group {
+        iterate_over: 'linkstamp_paths'
+        flag: '%{linkstamp_paths}'
+      }
+    }
+  }
+
+  feature {
+    name: 'output_execpath_flags'
+    flag_set {
+      expand_if_all_available: 'output_execpath'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: '/OUT:%{output_execpath}'
+      }
+    }
+  }
+
+  feature {
+    name: 'archiver_flags'
+    flag_set {
+      expand_if_all_available: 'output_execpath'
+      action: 'c++-link-static-library'
+      flag_group {
+        flag: '/OUT:%{output_execpath}'
+      }
+    }
+  }
+
+  feature {
+    name: 'input_param_flags'
+    flag_set {
+      expand_if_all_available: 'interface_library_output_path'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/IMPLIB:%{interface_library_output_path}"
+      }
+    }
+    flag_set {
+      expand_if_all_available: 'libopts'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        iterate_over: 'libopts'
+        flag: '%{libopts}'
+      }
+    }
+    flag_set {
+      expand_if_all_available: 'libraries_to_link'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      action: 'c++-link-static-library'
+      flag_group {
+        iterate_over: 'libraries_to_link'
+        flag_group {
+          expand_if_equal: {
+            variable: 'libraries_to_link.type'
+            value: 'object_file_group'
+          }
+          iterate_over: 'libraries_to_link.object_files'
+          flag_group {
+            flag: '%{libraries_to_link.object_files}'
+          }
+        }
+        flag_group {
+          expand_if_equal: {
+            variable: 'libraries_to_link.type'
+            value: 'object_file'
+          }
+          flag_group {
+            flag: '%{libraries_to_link.name}'
+          }
+        }
+        flag_group {
+          expand_if_equal: {
+            variable: 'libraries_to_link.type'
+            value: 'interface_library'
+          }
+          flag_group {
+            flag: '%{libraries_to_link.name}'
+          }
+        }
+        flag_group {
+          expand_if_equal: {
+            variable: 'libraries_to_link.type'
+            value: 'static_library'
+          }
+          flag_group {
+            expand_if_false: 'libraries_to_link.is_whole_archive'
+            flag: '%{libraries_to_link.name}'
+          }
+          flag_group {
+            expand_if_true: 'libraries_to_link.is_whole_archive'
+            flag: '/WHOLEARCHIVE:%{libraries_to_link.name}'
+          }
+        }
+      }
+    }
+  }
+
+  # Since this feature is declared earlier in the CROSSTOOL than
+  # "user_link_flags", this feature will be applied prior to it anwyhere they
+  # are both implied. And since "user_link_flags" contains the linkopts from
+  # the build rule, this allows the user to override the /SUBSYSTEM in the BUILD
+  # file.
+  feature {
+    name: 'linker_subsystem_flag'
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: '/SUBSYSTEM:CONSOLE'
+      }
+    }
+  }
+
+  # The "user_link_flags" contains user-defined linkopts (from build rules)
+  # so it should be defined after features that declare user-overridable flags.
+  # For example the "linker_subsystem_flag" defines a default "/SUBSYSTEM" flag
+  # but we want to let the user override it, therefore "link_flag_subsystem" is
+  # defined earlier in the CROSSTOOL file than "user_link_flags".
+  feature {
+    name: 'user_link_flags'
+    flag_set {
+      expand_if_all_available: 'user_link_flags'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        iterate_over: 'user_link_flags'
+        flag: '%{user_link_flags}'
+      }
+    }
+  }
+  feature {
+    name: 'legacy_link_flags'
+    flag_set {
+      expand_if_all_available: 'legacy_link_flags'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        iterate_over: 'legacy_link_flags'
+        flag: '%{legacy_link_flags}'
+      }
+    }
+  }
+
+  feature {
+    name: 'linker_param_file'
+    flag_set {
+      expand_if_all_available: 'linker_param_file'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      action: 'c++-link-static-library'
+      flag_group {
+        flag: '@%{linker_param_file}'
+      }
+    }
+  }
+
+  feature {
+    name: 'static_link_msvcrt'
+  }
+
+  feature {
+    name: 'static_link_msvcrt_no_debug'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/MT"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEFAULTLIB:libcmt.lib"
+      }
+    }
+    requires: { feature: 'fastbuild'}
+    requires: { feature: 'opt'}
+  }
+
+  feature {
+    name: 'dynamic_link_msvcrt_no_debug'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/MD"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEFAULTLIB:msvcrt.lib"
+      }
+    }
+    requires: { feature: 'fastbuild'}
+    requires: { feature: 'opt'}
+  }
+
+  feature {
+    name: 'static_link_msvcrt_debug'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/MTd"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEFAULTLIB:libcmtd.lib"
+      }
+    }
+    requires: { feature: 'dbg'}
+  }
+
+  feature {
+    name: 'dynamic_link_msvcrt_debug'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/MDd"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEFAULTLIB:msvcrtd.lib"
+      }
+    }
+    requires: { feature: 'dbg'}
+  }
+
+  feature {
+    name: 'dbg'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/Od"
+        flag: "/Z7"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: ""
+        flag: "/INCREMENTAL:NO"
+      }
+    }
+    implies: 'generate_pdb_file'
+  }
+
+  feature {
+    name: 'fastbuild'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/Od"
+        flag: "/Z7"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: ""
+        flag: "/INCREMENTAL:NO"
+      }
+    }
+    implies: 'generate_pdb_file'
+  }
+
+  feature {
+    name: 'opt'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/O2" # Implies /Og /Oi /Ot /Oy /Ob2 /Gs /GF /Gy
+      }
+    }
+    implies: 'frame_pointer'
+  }
+
+  # Keep stack frames for debugging, even in opt mode.
+  # Must come after /O1, /O2 and /Ox.
+  feature {
+    name: "frame_pointer"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        flag: "/Oy-"
+      }
+    }
+  }
+
+  # Remove assert/DCHECKs in opt mode.
+  # You can have them back with --features=-disable_assertions.
+  feature {
+    name: 'disable_assertions'
+    enabled: true
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      with_feature: {
+        feature: 'opt'
+      }
+      flag_group {
+        flag: "/DNDEBUG"
+      }
+    }
+  }
+
+  feature {
+    name: "determinism"
+    enabled: true
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        # Make C++ compilation deterministic. Use linkstamping instead of these
+        # compiler symbols.
+        # TODO: detect clang on Windows and use "-Wno-builtin-macro-redefined"
+        flag: "/wd4117" # Trying to define or undefine a predefined macro
+        flag: "-D__DATE__=\"redacted\""
+        flag: "-D__TIMESTAMP__=\"redacted\""
+        flag: "-D__TIME__=\"redacted\""
+      }
+    }
+  }
+
+  feature {
+    name: 'treat_warnings_as_errors'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/WX"
+      }
+    }
+  }
+
+  # Trade slower build time for smaller binary
+  feature {
+    name: 'smaller_binary'
+    enabled: true
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      with_feature: {
+        feature: 'opt'
+      }
+      flag_group {
+        flag: "/Gy" # Enable function-level linking (-ffunction-sections)
+        flag: "/Gw" # Optimize global data (-fdata-sections)
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library',
+      action: 'c++-link-nodeps-dynamic-library'
+      with_feature: {
+        feature: 'opt'
+      }
+      flag_group {
+        flag: '/OPT:ICF' # Fold identical functions
+        flag: '/OPT:REF' # Eliminate unreferenced functions and data
+      }
+    }
+  }
+
+  # Suppress warnings that most users do not care
+  feature {
+    name: 'ignore_noisy_warnings'
+    enabled: true
+    flag_set {
+      action: 'c++-link-static-library'
+      flag_group {
+        # Suppress 'object file does not define any public symbols' warning
+        flag: '/ignore:4221'
+      }
+    }
+  }
+
+  feature {
+    name: 'user_compile_flags'
+    flag_set {
+      expand_if_all_available: 'user_compile_flags'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      flag_group {
+        iterate_over: 'user_compile_flags'
+        flag: '%{user_compile_flags}'
+      }
+    }
+  }
+
+  feature {
+    name: 'sysroot'
+    flag_set {
+      expand_if_all_available: 'sysroot'
+      action: 'assemble'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        iterate_over: 'sysroot'
+        flag: '--sysroot=%{sysroot}'
+      }
+    }
+  }
+
+  feature {
+    name: 'unfiltered_compile_flags'
+    flag_set {
+      expand_if_all_available: 'unfiltered_compile_flags'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      flag_group {
+        iterate_over: 'unfiltered_compile_flags'
+        flag: '%{unfiltered_compile_flags}'
+      }
+    }
+  }
+
+  feature {
+    name: 'compiler_output_flags'
+    flag_set {
+      action: 'assemble'
+      flag_group {
+        expand_if_all_available: 'output_file'
+        expand_if_none_available: 'output_assembly_file'
+        expand_if_none_available: 'output_preprocess_file'
+        flag: '/Fo%{output_file}'
+        flag: '/Zi'
+      }
+    }
+    flag_set {
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      flag_group {
+        expand_if_all_available: 'output_file'
+        expand_if_none_available: 'output_assembly_file'
+        expand_if_none_available: 'output_preprocess_file'
+        flag: '/Fo%{output_file}'
+      }
+      flag_group {
+        expand_if_all_available: 'output_file'
+        expand_if_all_available: 'output_assembly_file'
+        flag: '/Fa%{output_file}'
+      }
+      flag_group {
+        expand_if_all_available: 'output_file'
+        expand_if_all_available: 'output_preprocess_file'
+        flag: '/P'
+        flag: '/Fi%{output_file}'
+      }
+    }
+  }
+
+  feature {
+    name: 'compiler_input_flags'
+    flag_set {
+      action: 'assemble'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      flag_group {
+        expand_if_all_available: 'source_file'
+        flag: '/c'
+        flag: '%{source_file}'
+      }
+    }
+  }
+
+  feature {
+    name : 'def_file',
+    flag_set {
+      expand_if_all_available: 'def_file_path'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEF:%{def_file_path}"
+        # We can specify a different DLL name in DEF file, /ignore:4070 suppresses
+        # the warning message about DLL name doesn't match the default one.
+        # See https://msdn.microsoft.com/en-us/library/sfkk2fz7.aspx
+        flag: "/ignore:4070"
+      }
+    }
+  }
+
+  feature {
+    name: 'windows_export_all_symbols'
+  }
+
+  feature {
+    name: 'no_windows_export_all_symbols'
+  }
+
+  linking_mode_flags { mode: DYNAMIC }
+}
diff --git a/third_party/toolchains/preconfig/ubuntu16.04/clang/WORKSPACE b/third_party/toolchains/preconfig/ubuntu16.04/clang/WORKSPACE
new file mode 100644
index 0000000000..bc05b4c36f
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu16.04/clang/WORKSPACE
@@ -0,0 +1,2 @@
+# DO NOT EDIT: automatically generated WORKSPACE file for cc_autoconf rule
+workspace(name = "local_config_cc")
diff --git a/third_party/toolchains/preconfig/ubuntu16.04/clang/cc_wrapper.sh b/third_party/toolchains/preconfig/ubuntu16.04/clang/cc_wrapper.sh
new file mode 100755
index 0000000000..42a751dccf
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu16.04/clang/cc_wrapper.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Copyright 2015 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Ship the environment to the C++ action
+#
+set -eu
+
+# Set-up the environment
+
+
+# Call the C++ compiler
+/usr/local/bin/clang "$@"
diff --git a/third_party/toolchains/preconfig/ubuntu16.04/clang/dummy_toolchain.bzl b/third_party/toolchains/preconfig/ubuntu16.04/clang/dummy_toolchain.bzl
new file mode 100755
index 0000000000..45c0285d23
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu16.04/clang/dummy_toolchain.bzl
@@ -0,0 +1,23 @@
+# pylint: disable=g-bad-file-header
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Skylark rule that stubs a toolchain."""
+
+def _dummy_toolchain_impl(ctx):
+    ctx = ctx  # unused argument
+    toolchain = platform_common.ToolchainInfo()
+    return [toolchain]
+
+dummy_toolchain = rule(_dummy_toolchain_impl, attrs = {})
diff --git a/third_party/toolchains/preconfig/ubuntu16.04/clang/tools/cpp/empty.cc b/third_party/toolchains/preconfig/ubuntu16.04/clang/tools/cpp/empty.cc
new file mode 100755
index 0000000000..c272dabaeb
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu16.04/clang/tools/cpp/empty.cc
@@ -0,0 +1 @@
+int main() {}
\ No newline at end of file
diff --git a/third_party/toolchains/preconfig/ubuntu16.04/py3/BUILD b/third_party/toolchains/preconfig/ubuntu16.04/py3/BUILD
new file mode 100755
index 0000000000..77eaa4d512
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu16.04/py3/BUILD
@@ -0,0 +1,205 @@
+licenses(["restricted"])
+
+package(default_visibility = ["//visibility:public"])
+
+# To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib
+# See https://docs.python.org/3/extending/windows.html
+cc_import(
+    name = "python_lib",
+    interface_library = select({
+        ":windows": ":python_import_lib",
+        # A placeholder for Unix platforms which makes --no_build happy.
+        "//conditions:default": "not-existing.lib",
+    }),
+    system_provided = 1,
+)
+
+cc_library(
+    name = "python_headers",
+    hdrs = [":python_include"],
+    includes = ["python_include"],
+    deps = select({
+        ":windows": [":python_lib"],
+        "//conditions:default": [],
+    }),
+)
+
+cc_library(
+    name = "numpy_headers",
+    hdrs = [":numpy_include"],
+    includes = ["numpy_include"],
+)
+
+config_setting(
+    name = "windows",
+    values = {"cpu": "x64_windows"},
+    visibility = ["//visibility:public"],
+)
+
+genrule(
+    name = "python_include",
+    outs = [
+        "python_include/Python-ast.h",
+        "python_include/Python.h",
+        "python_include/abstract.h",
+        "python_include/accu.h",
+        "python_include/asdl.h",
+        "python_include/ast.h",
+        "python_include/bitset.h",
+        "python_include/bltinmodule.h",
+        "python_include/boolobject.h",
+        "python_include/bytearrayobject.h",
+        "python_include/bytes_methods.h",
+        "python_include/bytesobject.h",
+        "python_include/cellobject.h",
+        "python_include/ceval.h",
+        "python_include/classobject.h",
+        "python_include/code.h",
+        "python_include/codecs.h",
+        "python_include/compile.h",
+        "python_include/complexobject.h",
+        "python_include/datetime.h",
+        "python_include/descrobject.h",
+        "python_include/dictobject.h",
+        "python_include/dtoa.h",
+        "python_include/dynamic_annotations.h",
+        "python_include/enumobject.h",
+        "python_include/errcode.h",
+        "python_include/eval.h",
+        "python_include/fileobject.h",
+        "python_include/fileutils.h",
+        "python_include/floatobject.h",
+        "python_include/frameobject.h",
+        "python_include/funcobject.h",
+        "python_include/genobject.h",
+        "python_include/graminit.h",
+        "python_include/grammar.h",
+        "python_include/import.h",
+        "python_include/intrcheck.h",
+        "python_include/iterobject.h",
+        "python_include/listobject.h",
+        "python_include/longintrepr.h",
+        "python_include/longobject.h",
+        "python_include/marshal.h",
+        "python_include/memoryobject.h",
+        "python_include/metagrammar.h",
+        "python_include/methodobject.h",
+        "python_include/modsupport.h",
+        "python_include/moduleobject.h",
+        "python_include/namespaceobject.h",
+        "python_include/node.h",
+        "python_include/numpy/__multiarray_api.h",
+        "python_include/numpy/__ufunc_api.h",
+        "python_include/numpy/_neighborhood_iterator_imp.h",
+        "python_include/numpy/_numpyconfig.h",
+        "python_include/numpy/arrayobject.h",
+        "python_include/numpy/arrayscalars.h",
+        "python_include/numpy/halffloat.h",
+        "python_include/numpy/multiarray_api.txt",
+        "python_include/numpy/ndarrayobject.h",
+        "python_include/numpy/ndarraytypes.h",
+        "python_include/numpy/noprefix.h",
+        "python_include/numpy/npy_1_7_deprecated_api.h",
+        "python_include/numpy/npy_3kcompat.h",
+        "python_include/numpy/npy_common.h",
+        "python_include/numpy/npy_cpu.h",
+        "python_include/numpy/npy_endian.h",
+        "python_include/numpy/npy_interrupt.h",
+        "python_include/numpy/npy_math.h",
+        "python_include/numpy/npy_no_deprecated_api.h",
+        "python_include/numpy/npy_os.h",
+        "python_include/numpy/numpyconfig.h",
+        "python_include/numpy/old_defines.h",
+        "python_include/numpy/oldnumeric.h",
+        "python_include/numpy/ufunc_api.txt",
+        "python_include/numpy/ufuncobject.h",
+        "python_include/numpy/utils.h",
+        "python_include/object.h",
+        "python_include/objimpl.h",
+        "python_include/odictobject.h",
+        "python_include/opcode.h",
+        "python_include/osdefs.h",
+        "python_include/parsetok.h",
+        "python_include/patchlevel.h",
+        "python_include/pgen.h",
+        "python_include/pgenheaders.h",
+        "python_include/py_curses.h",
+        "python_include/pyarena.h",
+        "python_include/pyatomic.h",
+        "python_include/pycapsule.h",
+        "python_include/pyconfig.h",
+        "python_include/pyctype.h",
+        "python_include/pydebug.h",
+        "python_include/pyerrors.h",
+        "python_include/pyexpat.h",
+        "python_include/pyfpe.h",
+        "python_include/pygetopt.h",
+        "python_include/pyhash.h",
+        "python_include/pylifecycle.h",
+        "python_include/pymacconfig.h",
+        "python_include/pymacro.h",
+        "python_include/pymath.h",
+        "python_include/pymem.h",
+        "python_include/pyport.h",
+        "python_include/pystate.h",
+        "python_include/pystrcmp.h",
+        "python_include/pystrhex.h",
+        "python_include/pystrtod.h",
+        "python_include/pythonrun.h",
+        "python_include/pythread.h",
+        "python_include/pytime.h",
+        "python_include/rangeobject.h",
+        "python_include/setobject.h",
+        "python_include/sliceobject.h",
+        "python_include/structmember.h",
+        "python_include/structseq.h",
+        "python_include/symtable.h",
+        "python_include/sysmodule.h",
+        "python_include/token.h",
+        "python_include/traceback.h",
+        "python_include/tupleobject.h",
+        "python_include/typeslots.h",
+        "python_include/ucnhash.h",
+        "python_include/unicodeobject.h",
+        "python_include/warnings.h",
+        "python_include/weakrefobject.h",
+    ],
+    cmd = """
+cp -f "/usr/include/python3.5m/Python-ast.h" "$(@D)/python_include/Python-ast.h" && cp -f "/usr/include/python3.5m/Python.h" "$(@D)/python_include/Python.h" && cp -f "/usr/include/python3.5m/abstract.h" "$(@D)/python_include/abstract.h" && cp -f "/usr/include/python3.5m/accu.h" "$(@D)/python_include/accu.h" && cp -f "/usr/include/python3.5m/asdl.h" "$(@D)/python_include/asdl.h" && cp -f "/usr/include/python3.5m/ast.h" "$(@D)/python_include/ast.h" && cp -f "/usr/include/python3.5m/bitset.h" "$(@D)/python_include/bitset.h" && cp -f "/usr/include/python3.5m/bltinmodule.h" "$(@D)/python_include/bltinmodule.h" && cp -f "/usr/include/python3.5m/boolobject.h" "$(@D)/python_include/boolobject.h" && cp -f "/usr/include/python3.5m/bytearrayobject.h" "$(@D)/python_include/bytearrayobject.h" && cp -f "/usr/include/python3.5m/bytes_methods.h" "$(@D)/python_include/bytes_methods.h" && cp -f "/usr/include/python3.5m/bytesobject.h" "$(@D)/python_include/bytesobject.h" && cp -f "/usr/include/python3.5m/cellobject.h" "$(@D)/python_include/cellobject.h" && cp -f "/usr/include/python3.5m/ceval.h" "$(@D)/python_include/ceval.h" && cp -f "/usr/include/python3.5m/classobject.h" "$(@D)/python_include/classobject.h" && cp -f "/usr/include/python3.5m/code.h" "$(@D)/python_include/code.h" && cp -f "/usr/include/python3.5m/codecs.h" "$(@D)/python_include/codecs.h" && cp -f "/usr/include/python3.5m/compile.h" "$(@D)/python_include/compile.h" && cp -f "/usr/include/python3.5m/complexobject.h" "$(@D)/python_include/complexobject.h" && cp -f "/usr/include/python3.5m/datetime.h" "$(@D)/python_include/datetime.h" && cp -f "/usr/include/python3.5m/descrobject.h" "$(@D)/python_include/descrobject.h" && cp -f "/usr/include/python3.5m/dictobject.h" "$(@D)/python_include/dictobject.h" && cp -f "/usr/include/python3.5m/dtoa.h" "$(@D)/python_include/dtoa.h" && cp -f "/usr/include/python3.5m/dynamic_annotations.h" "$(@D)/python_include/dynamic_annotations.h" && cp -f "/usr/include/python3.5m/enumobject.h" "$(@D)/python_include/enumobject.h" && cp -f "/usr/include/python3.5m/errcode.h" "$(@D)/python_include/errcode.h" && cp -f "/usr/include/python3.5m/eval.h" "$(@D)/python_include/eval.h" && cp -f "/usr/include/python3.5m/fileobject.h" "$(@D)/python_include/fileobject.h" && cp -f "/usr/include/python3.5m/fileutils.h" "$(@D)/python_include/fileutils.h" && cp -f "/usr/include/python3.5m/floatobject.h" "$(@D)/python_include/floatobject.h" && cp -f "/usr/include/python3.5m/frameobject.h" "$(@D)/python_include/frameobject.h" && cp -f "/usr/include/python3.5m/funcobject.h" "$(@D)/python_include/funcobject.h" && cp -f "/usr/include/python3.5m/genobject.h" "$(@D)/python_include/genobject.h" && cp -f "/usr/include/python3.5m/graminit.h" "$(@D)/python_include/graminit.h" && cp -f "/usr/include/python3.5m/grammar.h" "$(@D)/python_include/grammar.h" && cp -f "/usr/include/python3.5m/import.h" "$(@D)/python_include/import.h" && cp -f "/usr/include/python3.5m/intrcheck.h" "$(@D)/python_include/intrcheck.h" && cp -f "/usr/include/python3.5m/iterobject.h" "$(@D)/python_include/iterobject.h" && cp -f "/usr/include/python3.5m/listobject.h" "$(@D)/python_include/listobject.h" && cp -f "/usr/include/python3.5m/longintrepr.h" "$(@D)/python_include/longintrepr.h" && cp -f "/usr/include/python3.5m/longobject.h" "$(@D)/python_include/longobject.h" && cp -f "/usr/include/python3.5m/marshal.h" "$(@D)/python_include/marshal.h" && cp -f "/usr/include/python3.5m/memoryobject.h" "$(@D)/python_include/memoryobject.h" && cp -f "/usr/include/python3.5m/metagrammar.h" "$(@D)/python_include/metagrammar.h" && cp -f "/usr/include/python3.5m/methodobject.h" "$(@D)/python_include/methodobject.h" && cp -f "/usr/include/python3.5m/modsupport.h" "$(@D)/python_include/modsupport.h" && cp -f "/usr/include/python3.5m/moduleobject.h" "$(@D)/python_include/moduleobject.h" && cp -f "/usr/include/python3.5m/namespaceobject.h" "$(@D)/python_include/namespaceobject.h" && cp -f "/usr/include/python3.5m/node.h" "$(@D)/python_include/node.h" && cp -f "/usr/include/python3.5m/numpy/__multiarray_api.h" "$(@D)/python_include/numpy/__multiarray_api.h" && cp -f "/usr/include/python3.5m/numpy/__ufunc_api.h" "$(@D)/python_include/numpy/__ufunc_api.h" && cp -f "/usr/include/python3.5m/numpy/_neighborhood_iterator_imp.h" "$(@D)/python_include/numpy/_neighborhood_iterator_imp.h" && cp -f "/usr/include/python3.5m/numpy/_numpyconfig.h" "$(@D)/python_include/numpy/_numpyconfig.h" && cp -f "/usr/include/python3.5m/numpy/arrayobject.h" "$(@D)/python_include/numpy/arrayobject.h" && cp -f "/usr/include/python3.5m/numpy/arrayscalars.h" "$(@D)/python_include/numpy/arrayscalars.h" && cp -f "/usr/include/python3.5m/numpy/halffloat.h" "$(@D)/python_include/numpy/halffloat.h" && cp -f "/usr/include/python3.5m/numpy/multiarray_api.txt" "$(@D)/python_include/numpy/multiarray_api.txt" && cp -f "/usr/include/python3.5m/numpy/ndarrayobject.h" "$(@D)/python_include/numpy/ndarrayobject.h" && cp -f "/usr/include/python3.5m/numpy/ndarraytypes.h" "$(@D)/python_include/numpy/ndarraytypes.h" && cp -f "/usr/include/python3.5m/numpy/noprefix.h" "$(@D)/python_include/numpy/noprefix.h" && cp -f "/usr/include/python3.5m/numpy/npy_1_7_deprecated_api.h" "$(@D)/python_include/numpy/npy_1_7_deprecated_api.h" && cp -f "/usr/include/python3.5m/numpy/npy_3kcompat.h" "$(@D)/python_include/numpy/npy_3kcompat.h" && cp -f "/usr/include/python3.5m/numpy/npy_common.h" "$(@D)/python_include/numpy/npy_common.h" && cp -f "/usr/include/python3.5m/numpy/npy_cpu.h" "$(@D)/python_include/numpy/npy_cpu.h" && cp -f "/usr/include/python3.5m/numpy/npy_endian.h" "$(@D)/python_include/numpy/npy_endian.h" && cp -f "/usr/include/python3.5m/numpy/npy_interrupt.h" "$(@D)/python_include/numpy/npy_interrupt.h" && cp -f "/usr/include/python3.5m/numpy/npy_math.h" "$(@D)/python_include/numpy/npy_math.h" && cp -f "/usr/include/python3.5m/numpy/npy_no_deprecated_api.h" "$(@D)/python_include/numpy/npy_no_deprecated_api.h" && cp -f "/usr/include/python3.5m/numpy/npy_os.h" "$(@D)/python_include/numpy/npy_os.h" && cp -f "/usr/include/python3.5m/numpy/numpyconfig.h" "$(@D)/python_include/numpy/numpyconfig.h" && cp -f "/usr/include/python3.5m/numpy/old_defines.h" "$(@D)/python_include/numpy/old_defines.h" && cp -f "/usr/include/python3.5m/numpy/oldnumeric.h" "$(@D)/python_include/numpy/oldnumeric.h" && cp -f "/usr/include/python3.5m/numpy/ufunc_api.txt" "$(@D)/python_include/numpy/ufunc_api.txt" && cp -f "/usr/include/python3.5m/numpy/ufuncobject.h" "$(@D)/python_include/numpy/ufuncobject.h" && cp -f "/usr/include/python3.5m/numpy/utils.h" "$(@D)/python_include/numpy/utils.h" && cp -f "/usr/include/python3.5m/object.h" "$(@D)/python_include/object.h" && cp -f "/usr/include/python3.5m/objimpl.h" "$(@D)/python_include/objimpl.h" && cp -f "/usr/include/python3.5m/odictobject.h" "$(@D)/python_include/odictobject.h" && cp -f "/usr/include/python3.5m/opcode.h" "$(@D)/python_include/opcode.h" && cp -f "/usr/include/python3.5m/osdefs.h" "$(@D)/python_include/osdefs.h" && cp -f "/usr/include/python3.5m/parsetok.h" "$(@D)/python_include/parsetok.h" && cp -f "/usr/include/python3.5m/patchlevel.h" "$(@D)/python_include/patchlevel.h" && cp -f "/usr/include/python3.5m/pgen.h" "$(@D)/python_include/pgen.h" && cp -f "/usr/include/python3.5m/pgenheaders.h" "$(@D)/python_include/pgenheaders.h" && cp -f "/usr/include/python3.5m/py_curses.h" "$(@D)/python_include/py_curses.h" && cp -f "/usr/include/python3.5m/pyarena.h" "$(@D)/python_include/pyarena.h" && cp -f "/usr/include/python3.5m/pyatomic.h" "$(@D)/python_include/pyatomic.h" && cp -f "/usr/include/python3.5m/pycapsule.h" "$(@D)/python_include/pycapsule.h" && cp -f "/usr/include/python3.5m/pyconfig.h" "$(@D)/python_include/pyconfig.h" && cp -f "/usr/include/python3.5m/pyctype.h" "$(@D)/python_include/pyctype.h" && cp -f "/usr/include/python3.5m/pydebug.h" "$(@D)/python_include/pydebug.h" && cp -f "/usr/include/python3.5m/pyerrors.h" "$(@D)/python_include/pyerrors.h" && cp -f "/usr/include/python3.5m/pyexpat.h" "$(@D)/python_include/pyexpat.h" && cp -f "/usr/include/python3.5m/pyfpe.h" "$(@D)/python_include/pyfpe.h" && cp -f "/usr/include/python3.5m/pygetopt.h" "$(@D)/python_include/pygetopt.h" && cp -f "/usr/include/python3.5m/pyhash.h" "$(@D)/python_include/pyhash.h" && cp -f "/usr/include/python3.5m/pylifecycle.h" "$(@D)/python_include/pylifecycle.h" && cp -f "/usr/include/python3.5m/pymacconfig.h" "$(@D)/python_include/pymacconfig.h" && cp -f "/usr/include/python3.5m/pymacro.h" "$(@D)/python_include/pymacro.h" && cp -f "/usr/include/python3.5m/pymath.h" "$(@D)/python_include/pymath.h" && cp -f "/usr/include/python3.5m/pymem.h" "$(@D)/python_include/pymem.h" && cp -f "/usr/include/python3.5m/pyport.h" "$(@D)/python_include/pyport.h" && cp -f "/usr/include/python3.5m/pystate.h" "$(@D)/python_include/pystate.h" && cp -f "/usr/include/python3.5m/pystrcmp.h" "$(@D)/python_include/pystrcmp.h" && cp -f "/usr/include/python3.5m/pystrhex.h" "$(@D)/python_include/pystrhex.h" && cp -f "/usr/include/python3.5m/pystrtod.h" "$(@D)/python_include/pystrtod.h" && cp -f "/usr/include/python3.5m/pythonrun.h" "$(@D)/python_include/pythonrun.h" && cp -f "/usr/include/python3.5m/pythread.h" "$(@D)/python_include/pythread.h" && cp -f "/usr/include/python3.5m/pytime.h" "$(@D)/python_include/pytime.h" && cp -f "/usr/include/python3.5m/rangeobject.h" "$(@D)/python_include/rangeobject.h" && cp -f "/usr/include/python3.5m/setobject.h" "$(@D)/python_include/setobject.h" && cp -f "/usr/include/python3.5m/sliceobject.h" "$(@D)/python_include/sliceobject.h" && cp -f "/usr/include/python3.5m/structmember.h" "$(@D)/python_include/structmember.h" && cp -f "/usr/include/python3.5m/structseq.h" "$(@D)/python_include/structseq.h" && cp -f "/usr/include/python3.5m/symtable.h" "$(@D)/python_include/symtable.h" && cp -f "/usr/include/python3.5m/sysmodule.h" "$(@D)/python_include/sysmodule.h" && cp -f "/usr/include/python3.5m/token.h" "$(@D)/python_include/token.h" && cp -f "/usr/include/python3.5m/traceback.h" "$(@D)/python_include/traceback.h" && cp -f "/usr/include/python3.5m/tupleobject.h" "$(@D)/python_include/tupleobject.h" && cp -f "/usr/include/python3.5m/typeslots.h" "$(@D)/python_include/typeslots.h" && cp -f "/usr/include/python3.5m/ucnhash.h" "$(@D)/python_include/ucnhash.h" && cp -f "/usr/include/python3.5m/unicodeobject.h" "$(@D)/python_include/unicodeobject.h" && cp -f "/usr/include/python3.5m/warnings.h" "$(@D)/python_include/warnings.h" && cp -f "/usr/include/python3.5m/weakrefobject.h" "$(@D)/python_include/weakrefobject.h"
+   """,
+)
+
+genrule(
+    name = "numpy_include",
+    outs = [
+        "numpy_include/numpy/__multiarray_api.h",
+        "numpy_include/numpy/__ufunc_api.h",
+        "numpy_include/numpy/_neighborhood_iterator_imp.h",
+        "numpy_include/numpy/_numpyconfig.h",
+        "numpy_include/numpy/arrayobject.h",
+        "numpy_include/numpy/arrayscalars.h",
+        "numpy_include/numpy/halffloat.h",
+        "numpy_include/numpy/multiarray_api.txt",
+        "numpy_include/numpy/ndarrayobject.h",
+        "numpy_include/numpy/ndarraytypes.h",
+        "numpy_include/numpy/noprefix.h",
+        "numpy_include/numpy/npy_1_7_deprecated_api.h",
+        "numpy_include/numpy/npy_3kcompat.h",
+        "numpy_include/numpy/npy_common.h",
+        "numpy_include/numpy/npy_cpu.h",
+        "numpy_include/numpy/npy_endian.h",
+        "numpy_include/numpy/npy_interrupt.h",
+        "numpy_include/numpy/npy_math.h",
+        "numpy_include/numpy/npy_no_deprecated_api.h",
+        "numpy_include/numpy/npy_os.h",
+        "numpy_include/numpy/numpyconfig.h",
+        "numpy_include/numpy/old_defines.h",
+        "numpy_include/numpy/oldnumeric.h",
+        "numpy_include/numpy/ufunc_api.txt",
+        "numpy_include/numpy/ufuncobject.h",
+        "numpy_include/numpy/utils.h",
+    ],
+    cmd = """
+cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/__multiarray_api.h" "$(@D)/numpy_include/numpy/__multiarray_api.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/__ufunc_api.h" "$(@D)/numpy_include/numpy/__ufunc_api.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/_neighborhood_iterator_imp.h" "$(@D)/numpy_include/numpy/_neighborhood_iterator_imp.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/_numpyconfig.h" "$(@D)/numpy_include/numpy/_numpyconfig.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/arrayobject.h" "$(@D)/numpy_include/numpy/arrayobject.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/arrayscalars.h" "$(@D)/numpy_include/numpy/arrayscalars.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/halffloat.h" "$(@D)/numpy_include/numpy/halffloat.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/multiarray_api.txt" "$(@D)/numpy_include/numpy/multiarray_api.txt" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/ndarrayobject.h" "$(@D)/numpy_include/numpy/ndarrayobject.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/ndarraytypes.h" "$(@D)/numpy_include/numpy/ndarraytypes.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/noprefix.h" "$(@D)/numpy_include/numpy/noprefix.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h" "$(@D)/numpy_include/numpy/npy_1_7_deprecated_api.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/npy_3kcompat.h" "$(@D)/numpy_include/numpy/npy_3kcompat.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/npy_common.h" "$(@D)/numpy_include/numpy/npy_common.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/npy_cpu.h" "$(@D)/numpy_include/numpy/npy_cpu.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/npy_endian.h" "$(@D)/numpy_include/numpy/npy_endian.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/npy_interrupt.h" "$(@D)/numpy_include/numpy/npy_interrupt.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/npy_math.h" "$(@D)/numpy_include/numpy/npy_math.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/npy_no_deprecated_api.h" "$(@D)/numpy_include/numpy/npy_no_deprecated_api.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/npy_os.h" "$(@D)/numpy_include/numpy/npy_os.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/numpyconfig.h" "$(@D)/numpy_include/numpy/numpyconfig.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/old_defines.h" "$(@D)/numpy_include/numpy/old_defines.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/oldnumeric.h" "$(@D)/numpy_include/numpy/oldnumeric.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/ufunc_api.txt" "$(@D)/numpy_include/numpy/ufunc_api.txt" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/ufuncobject.h" "$(@D)/numpy_include/numpy/ufuncobject.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/utils.h" "$(@D)/numpy_include/numpy/utils.h"
+   """,
+)
diff --git a/third_party/toolchains/preconfig/ubuntu16.04/py3/WORKSPACE b/third_party/toolchains/preconfig/ubuntu16.04/py3/WORKSPACE
new file mode 100644
index 0000000000..1d298fefa3
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu16.04/py3/WORKSPACE
@@ -0,0 +1,2 @@
+# DO NOT EDIT: automatically generated WORKSPACE file for python_configure rule
+workspace(name = "local_config_python")
-- 
GitLab


From 1c7de3473c6b9eb6c5b3bd63ee75c203f7a5296c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 6 Jan 2019 13:52:15 -0800
Subject: [PATCH 0251/2345] Fix typo; remove extra space.

PiperOrigin-RevId: 228076497
---
 tensorflow/contrib/training/python/training/training.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/training/python/training/training.py b/tensorflow/contrib/training/python/training/training.py
index c272a2ac14..fc6e38ab4a 100644
--- a/tensorflow/contrib/training/python/training/training.py
+++ b/tensorflow/contrib/training/python/training/training.py
@@ -419,7 +419,7 @@ def create_train_op(total_loss,
     update_ops = set(update_ops)
   if not global_update_ops.issubset(update_ops):
     logging.warning('update_ops in create_train_op does not contain all the '
-                    ' update_ops in GraphKeys.UPDATE_OPS')
+                    'update_ops in GraphKeys.UPDATE_OPS')
 
   # Make sure update_ops are computed before total_loss.
   if update_ops:
-- 
GitLab


From ebd5477f13c8d9585788b9721f821095fca7352e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 6 Jan 2019 16:21:00 -0800
Subject: [PATCH 0252/2345] Make TF_DeleteServer check for __ANDROID__, too.

This should avoid a Wdelete-incomplete warning.

PiperOrigin-RevId: 228084360
---
 tensorflow/c/c_api.cc | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/tensorflow/c/c_api.cc b/tensorflow/c/c_api.cc
index 9580215a31..9f2f83920c 100644
--- a/tensorflow/c/c_api.cc
+++ b/tensorflow/c/c_api.cc
@@ -2881,6 +2881,9 @@ const char* TF_ServerTarget(TF_Server* server) {
 #endif
 }
 
-void TF_DeleteServer(TF_Server* server) { delete server; }
-
+void TF_DeleteServer(TF_Server* server) {
+#ifndef __ANDROID__
+  delete server;
+#endif
+}
 }  // end extern "C"
-- 
GitLab


From 38c91321421694432117b077294df43aa31d1193 Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Sun, 6 Jan 2019 16:30:18 -0800
Subject: [PATCH 0253/2345] [XLA:GPU] Selectively exclude 0-2-1 transposed
 input parameters from the shared memory transpose implementation.

Previously, we either use the shared memory tranpose implementation for all the
0-2-1 tranposed input parameters in a fusion or not to use the shared memory
tranpose implement for any input parameters, if the kernel contains instructions
such as kReverse and kGather which can make the shared memory tranpose
implemetation unsafe. There are two problems in such an approach. First, the
set of instructions that can make the shared memory tranpose implementation
unsafe is more than the aforementioned two instructions. Second, even though a
fusion contains such instructions, it may still be safe to use shared memory
tranpose to implement some, if not all, 0-2-1 tranposed input parameters that
are not involved in the computation of such instructions. This change adds an
analysis to inspect the transitive users of each 0-2-1 tranposed input
parameter to decide whether it is safe to use the shared memory tranpose
implementation for the parameter.

Add two test cases.

PiperOrigin-RevId: 228084822
---
 .../xla/service/gpu/ir_emitter_unnested.cc    | 118 +++++++++++++++---
 .../gpu/tests/gpu_kernel_tiling_test.cc       |  78 ++++++++++--
 2 files changed, 170 insertions(+), 26 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index 923b3d5945..f190af921e 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -2810,7 +2810,8 @@ void IrEmitterUnnested::EmitBlock(const TileGenerator& emit_one_tile,
 // unnested_hlo: The unnested hlo instruction for which the kernel is generated.
 //   Currently, these hlo instructions are supported: kLoop fusion, kCopy.
 // tiled_param_ids: The IDs for the parameters that are 0-2-1 transpose of
-//   other tensors with the same dimensions and need to be tiled and tranposed.
+//   other tensors with the same dimensions and are safe to be tranposed via
+//   the shared memory tranpose implementation.
 // mapping_scheme: The tiling scheme to use.
 // kernel_generator: Contains function objects for code generation, such as
 //   element generator, block prologue and epilogue generators.
@@ -2989,7 +2990,10 @@ LaunchDimensions IrEmitterUnnested::EmitKernel(
 
 // Emits a kernel for the given hlo instruction using a tiled 0-2-1 transpose
 // algorithm to improve the memory access patterns for the input parameters
-// with a shape that is a 0-2-1 transpose of the output tensor shape.
+// with a shape that is a 0-2-1 transpose of the output tensor shape. The caller
+// is responsible for making sure that it is safe to apply the shared memory
+// tranpose on the input parameters.
+//
 //
 // For the purpose of tiling, the output tensors have a logical shape of three
 // components 0-2-1 while the relevant input parameters have a logical shape
@@ -3040,26 +3044,99 @@ LaunchDimensions IrEmitterUnnested::EmitHlo021Tile(
 }
 
 namespace {
-// Returns true to indicate it is safe to use the tile based shared memory
-// transpose implementation to implement the kernel for the instruction.
+// A recursive function to inspect the users of a parameter to determine
+// whether it's safe for a parameter to participate in a shared-memory
+// transpose.
 //
-// An instruction is not safe for such an implementation if it can change the
-// element order of a tensor without changing the dimension of the tensor, and
-// the instruction has a corresponding elemental_ir_emitter.
-bool IsInstructionSafeForTileBasedTranspose(const HloInstruction* hlo) {
-  auto is_safe_for_tile_based_transpose = [&](const HloInstruction* instr) {
-    HloOpcode opcode = instr->opcode();
-    CHECK_NE(opcode, HloOpcode::kFusion);
-    return (opcode != HloOpcode::kReverse && opcode != HloOpcode::kGather);
-  };
+// Consider a fusion parameter P for which we might want to use a shmem
+// transpose.  If we do, we use a GPU thread block to preload a tile of P with
+// indices [z, y..y+31, x..x+31] to compute an output tile with the same indices
+// cooperatively, where z, y, x are the indices for the normalized input/output
+// tensor (see the document for FindTranspose021 for the definition of
+// normalized tensor for 0-2-1 transpose). This shmem transpose implementation
+// requires that the computation of the output tile only read elements within
+// the preload tile. If this is not true, we can't use a shmem transpose for P.
+//
+// If the computation of output element [z, y, x] only requires the element of
+// P with the same indices, the shmem tranpose implementation can be applied
+// to P safely. This is a sufficient but not necessary condition. We check all
+// the transitive users of P to see if we can find a user that may cause an
+// exception to the situation. If such a user is not found, we conclude that P
+// is safe for shmem transpose.
+//
+// This is trivially true for elementwise operations and some "data-movement"
+// ops like kTuple. However, it's not true for operations that can change the
+// dimensions of the inputs (e.g. pad, slice) and bitcast operation.
+// For example:
+//
+// fused_computation {
+//   param_0 = f32[64,64]{1,0} parameter(0)
+//   ROOT bitcast = f32[64,64]{0,1} bitcast(param_0)
+// }
+// The output element at logical address [0, 63] depends on the input element
+// at logical address [63, 0], which would not be within the shared-memory
+// block.
+//
+// TODO(bixia): In order to extend this for kInput fusion, that is reduction
+// with tranpose, we only need to end the use-chain checking with the input of
+// a reduce operations. In this case, the above description on "output" apply
+// to the result of such a use-chain, which provides the input to the reduce
+// operation.
+bool IsInstructionSafeForShmemTranspose(const HloInstruction* hlo) {
+  if (hlo->IsElementwise()) {
+    return absl::c_all_of(hlo->users(), [&](const HloInstruction* user) {
+      return IsInstructionSafeForShmemTranspose(user);
+    });
+  }
 
-  if (hlo->opcode() == HloOpcode::kFusion) {
-    return absl::c_all_of(hlo->fused_instructions_computation()->instructions(),
-                          is_safe_for_tile_based_transpose);
+  switch (hlo->opcode()) {
+    // Non-elementwise instructions that don't cause the shmem transpose
+    // to be unsafe, including the instructions that don't currently fuse.
+    case HloOpcode::kGetDimensionSize:
+      // The result of the operation doesn't rely on the content of the
+      // tensor. As such, there is no need to further inspect its users.
+      return true;
+    case HloOpcode::kGetTupleElement:
+    case HloOpcode::kMap:
+    case HloOpcode::kParameter:
+    case HloOpcode::kTuple:
+    case HloOpcode::kTupleSelect:
+      return absl::c_all_of(hlo->users(), [&](const HloInstruction* user) {
+        return IsInstructionSafeForShmemTranspose(user);
+      });
+
+    default:
+      return false;
   }
+}
 
-  return is_safe_for_tile_based_transpose(hlo);
+// Given a group of input parameters that are 0-2-1 tranpose of the outputs of
+// a fusion kernel, returns the input parameters that are safe for the shared
+// memory tranpose implementation.
+//
+// When a tile based shared memory transpose is used to implement an input with
+// 0-2-1 transpose, we preload a tile of the input elements
+// [z, y..y+31, x..x+31] to compute the output tile elements of the same
+// indices. Preloading the input tile this way is only safe when the computation
+// of the output tile elements do not need any input element outside the
+// preloaded tile. We inspect all the transitive users of the input parameter
+// up to the fusion root instruction to see if we can find any instruction
+// that can make preloading the input tile unsafe.
+std::vector<int64> FilterInputsForShmemTranspose(const HloInstruction* fusion,
+                                                 std::vector<int64> input_ids) {
+  std::vector<int64> filtered_input_ids;
+  for (int64 i = 0; i < input_ids.size(); ++i) {
+    const HloInstruction* input = fusion->fused_parameter(input_ids[i]);
+    if (IsInstructionSafeForShmemTranspose(input)) {
+      filtered_input_ids.push_back(input_ids[i]);
+    } else {
+      VLOG(10) << "Input not safe for shmem transpose " << input->ToString()
+               << "\n";
+    }
+  }
+  return filtered_input_ids;
 }
+
 }  // namespace
 
 bool IrEmitterUnnested::CheckAndEmitHloWithTile021(HloInstruction* hlo) {
@@ -3106,8 +3183,11 @@ bool IrEmitterUnnested::CheckAndEmitHloWithTile021(HloInstruction* hlo) {
     return false;
   }
 
-  if (!IsInstructionSafeForTileBasedTranspose(hlo)) {
-    return false;
+  if (opcode == HloOpcode::kFusion) {
+    params_012 = FilterInputsForShmemTranspose(hlo, params_012);
+    if (params_012.empty()) {
+      return false;
+    }
   }
 
   // Each of our shared memory tiles has 32*33 elements (so ~4kb, if the
diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_kernel_tiling_test.cc b/tensorflow/compiler/xla/service/gpu/tests/gpu_kernel_tiling_test.cc
index a302b582ed..8004ebdc8e 100644
--- a/tensorflow/compiler/xla/service/gpu/tests/gpu_kernel_tiling_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_kernel_tiling_test.cc
@@ -65,7 +65,7 @@ TEST_F(GpuKernelTilingTest, UnnestedTransposeWithProperDimensionsTiled) {
   CompileAndVerifyIr(std::move(hlo_module),
                      R"(
 ; CHECK-LABEL: define void @copy
-; CHECK: tail call void @llvm.nvvm.barrier0()
+; CHECK: call void @llvm.nvvm.barrier0()
 ; CHECK: }
 )",
                      /*match_optimized_ir=*/true);
@@ -91,7 +91,7 @@ TEST_F(GpuKernelTilingTest, UnnestedTransposeWithSmallDimensionsNotTiled) {
   CompileAndVerifyIr(std::move(hlo_module),
                      R"(
 ; CHECK-LABEL: define void @copy
-; CHECK-NOT: tail call void @llvm.nvvm.barrier0()
+; CHECK-NOT: call void @llvm.nvvm.barrier0()
 ; CHECK: }
 )",
                      /*match_optimized_ir=*/true);
@@ -118,7 +118,7 @@ TEST_F(GpuKernelTilingTest, SimpleFusionWithTransposeTiled) {
   CompileAndVerifyIr(std::move(hlo_module),
                      R"(
 ; CHECK-LABEL: define void @fusion
-; CHECK: tail call void @llvm.nvvm.barrier0()
+; CHECK: call void @llvm.nvvm.barrier0()
 ; CHECK: }
 )",
                      /*match_optimized_ir=*/true);
@@ -152,7 +152,7 @@ TEST_F(GpuKernelTilingTest, MultipleOutputFusionWithOnePossibleTransposeTiled) {
   CompileAndVerifyIr(std::move(hlo_module),
                      R"(
 ; CHECK-LABEL: define void @fusion
-; CHECK: tail call void @llvm.nvvm.barrier0()
+; CHECK: call void @llvm.nvvm.barrier0()
 ; CHECK: }
 )",
                      /*match_optimized_ir=*/true);
@@ -187,13 +187,13 @@ TEST_F(GpuKernelTilingTest,
   CompileAndVerifyIr(std::move(hlo_module),
                      R"(
 ; CHECK-LABEL: define void @fusion
-; CHECK-NOT: tail call void @llvm.nvvm.barrier0()
+; CHECK-NOT: call void @llvm.nvvm.barrier0()
 ; CHECK: }
 )",
                      /*match_optimized_ir=*/true);
 }
 
-TEST_F(GpuKernelTilingTest, FusionTransposeWithReverseNotTiled) {
+TEST_F(GpuKernelTilingTest, TransposedInputWithUserReverseNotTiled) {
   const char *const kHloString = R"(
     HloModule FusionTransposeWithReverseNotTiled
     fused_computation.1 {
@@ -214,12 +214,76 @@ TEST_F(GpuKernelTilingTest, FusionTransposeWithReverseNotTiled) {
   CompileAndVerifyIr(std::move(hlo_module),
                      R"(
 ; CHECK-LABEL: define void @fusion
-; CHECK-NOT: tail call void @llvm.nvvm.barrier0()
+; CHECK-NOT: call void @llvm.nvvm.barrier0()
 ; CHECK: }
 )",
                      /*match_optimized_ir=*/true);
 }
 
+TEST_F(GpuKernelTilingTest, TransposedInputWithUserBitcastNotTiled) {
+  const char *const kHloString = R"(
+    HloModule TransposedInputWithUserBitcast
+
+    fused_computation {
+      param_0 = f32[20,20]{1,0} parameter(0)
+      ROOT bitcast = f32[20,20]{0,1} bitcast(param_0)
+    }
+
+    ENTRY kernel_entry {
+      parameter.0 = f32[20,20]{1,0} parameter(0)
+      ROOT fusion = f32[20,20]{0,1} fusion(parameter.0),
+        kind=kLoop, calls=fused_computation
+    })";
+
+  // Check that a call to llvm.nvvm.barrier0 is not generated.
+  auto hlo_module =
+      ParseHloString(kHloString, ConfigWithoutLayoutAssignment()).ValueOrDie();
+  CompileAndVerifyIr(std::move(hlo_module),
+                     R"(
+; CHECK-LABEL: define void @fusion
+; CHECK-NOT: call void @llvm.nvvm.barrier0()
+; CHECK: }
+)",
+                     /*match_optimized_ir=*/true);
+
+  // Check that the kernel runs correctly.
+  EXPECT_TRUE(RunAndCompareNoHloPasses(kHloString, ErrorSpec{0.0}));
+}
+
+TEST_F(GpuKernelTilingTest, TransposedInputWithoutUnsafeUseTiled) {
+  const char *const kHloString = R"(
+    HloModule TwoTransposedInputs
+
+    fused_computation {
+      param_0 = f32[64,64]{1,0} parameter(0)
+      param_1 = f32[64,64]{1,0} parameter(1)
+      bitcast = f32[64,64]{0,1} bitcast(param_0)
+      copy = f32[64,64]{0,1} copy(param_1)
+      ROOT tuple = (f32[64,64]{0,1}, f32[64,64]{0,1}) tuple(bitcast, copy)
+    }
+
+    ENTRY kernel_entry {
+      parameter.0 = f32[64,64]{1,0} parameter(0)
+      parameter.1 = f32[64,64]{1,0} parameter(1)
+      ROOT fusion = (f32[64,64]{0,1}, f32[64,64]{0,1})
+        fusion(parameter.0, parameter.1),
+        kind=kLoop, calls=fused_computation
+    })";
+
+  // Check that a call to llvm.nvvm.barrier0 is not generated.
+  auto hlo_module =
+      ParseHloString(kHloString, ConfigWithoutLayoutAssignment()).ValueOrDie();
+  CompileAndVerifyIr(std::move(hlo_module),
+                     R"(
+; CHECK-LABEL: define void @fusion
+; CHECK: call void @llvm.nvvm.barrier0()
+; CHECK: }
+)",
+                     /*match_optimized_ir=*/true);
+  // Check that the kernel runs correctly.
+  EXPECT_TRUE(RunAndCompareNoHloPasses(kHloString, ErrorSpec{0.0}));
+}
+
 }  // namespace
 }  // namespace gpu
 }  // namespace xla
-- 
GitLab


From 187a8cba72aae56ed8dfbe5c2d5779664b7683f2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 6 Jan 2019 19:22:26 -0800
Subject: [PATCH 0254/2345] Change TensorFlow's resize_uninitialized wrapper to
 call libc++'s __resize_default_init extension when available.

PiperOrigin-RevId: 228095086
---
 tensorflow/core/BUILD              |  1 +
 tensorflow/core/lib/gtl/stl_util.h | 37 +++++++++++++++++++++++++-----
 2 files changed, 32 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 7c9decbd09..5e6c8d4dd8 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -2193,6 +2193,7 @@ cc_library(
         ],
     }),
     deps = tf_additional_lib_deps() + [
+        "@com_google_absl//absl/meta:type_traits",
         "@com_google_absl//absl/strings",
         "//third_party/eigen3",
         "@com_google_absl//absl/base:core_headers",
diff --git a/tensorflow/core/lib/gtl/stl_util.h b/tensorflow/core/lib/gtl/stl_util.h
index ffeca4e88a..853a290bf6 100644
--- a/tensorflow/core/lib/gtl/stl_util.h
+++ b/tensorflow/core/lib/gtl/stl_util.h
@@ -23,9 +23,12 @@ limitations under the License.
 #include <iterator>
 #include <memory>
 #include <string>
+#include <type_traits>
 #include <utility>
 #include <vector>
 
+#include "absl/meta/type_traits.h"
+
 namespace tensorflow {
 namespace gtl {
 
@@ -48,16 +51,38 @@ inline const T* vector_as_array(const std::vector<T, Allocator>* v) {
   return v->data();
 }
 
+namespace gtl_internal {
+
+// HasMember is true_type or false_type, depending on whether or not
+// T has a __resize_default_init member. Resize will call the
+// __resize_default_init member if it exists, and will call the resize
+// member otherwise.
+template <typename string_type, typename = void>
+struct ResizeUninitializedTraits {
+  using HasMember = std::false_type;
+  static void Resize(string_type* s, size_t new_size) { s->resize(new_size); }
+};
+
+// __resize_default_init is provided by libc++ >= 8.0 and by Google's internal
+// ::string implementation.
+template <typename string_type>
+struct ResizeUninitializedTraits<
+    string_type, absl::void_t<decltype(std::declval<string_type&>()
+                                           .__resize_default_init(237))> > {
+  using HasMember = std::true_type;
+  static void Resize(string_type* s, size_t new_size) {
+    s->__resize_default_init(new_size);
+  }
+};
+
+}  // namespace gtl_internal
+
 // Like str->resize(new_size), except any new characters added to "*str" as a
 // result of resizing may be left uninitialized, rather than being filled with
 // '0' bytes. Typically used when code is then going to overwrite the backing
-// store of the string with known data. Uses a Google extension to ::string.
+// store of the string with known data.
 inline void STLStringResizeUninitialized(string* s, size_t new_size) {
-#if __google_stl_resize_uninitialized_string
-  s->resize_uninitialized(new_size);
-#else
-  s->resize(new_size);
-#endif
+  gtl_internal::ResizeUninitializedTraits<string>::Resize(s, new_size);
 }
 
 // Calls delete (non-array version) on the SECOND item (pointer) in each pair in
-- 
GitLab


From b8f2207ead5c6ce6e7e98479a88a08e4977aa83d Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Sun, 6 Jan 2019 23:46:22 -0800
Subject: [PATCH 0255/2345] Increase length of stack trace for original
 variable definition.

PiperOrigin-RevId: 228112246
---
 tensorflow/python/ops/variable_scope.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py
index 64cdde3d85..35c00778ae 100644
--- a/tensorflow/python/ops/variable_scope.py
+++ b/tensorflow/python/ops/variable_scope.py
@@ -842,8 +842,11 @@ class _VariableStore(object):
         if isinstance(var, resource_variable_ops.ResourceVariable):
           raise ValueError(err_msg)
         tb = var.op.traceback[::-1]
-        # Throw away internal tf entries and only take a few lines.
-        tb = [x for x in tb if "tensorflow/python" not in x[0]][:3]
+        # Throw away internal tf entries and only take a few lines. In some
+        # cases the traceback can be longer (e.g. if someone uses factory
+        # functions to create variables) so we take more than needed in the
+        # default case.
+        tb = [x for x in tb if "tensorflow/python" not in x[0]][:5]
         raise ValueError("%s Originally defined at:\n\n%s" % (err_msg, "".join(
             traceback.format_list(tb))))
       found_var = self._vars[name]
-- 
GitLab


From 57d450732007e33db361ad8f16a1e8655a55dc39 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 00:42:55 -0800
Subject: [PATCH 0256/2345] Internal change.

PiperOrigin-RevId: 228117862
---
 tensorflow/contrib/gan/BUILD | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/contrib/gan/BUILD b/tensorflow/contrib/gan/BUILD
index 08df38ad27..db0868fb2c 100644
--- a/tensorflow/contrib/gan/BUILD
+++ b/tensorflow/contrib/gan/BUILD
@@ -2,7 +2,6 @@
 load("//tensorflow:tensorflow.bzl", "py_test")
 
 package(default_visibility = [
-    "//learning/brain/contrib/tfgan:__subpackages__",
     "//tensorflow:__subpackages__",
 ])
 
-- 
GitLab


From 134b9e580f639005f8e5986c53884183d6d9bde0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 01:03:09 -0800
Subject: [PATCH 0257/2345] compat: Update forward compatibility horizon to
 2019-01-07

PiperOrigin-RevId: 228120298
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index b779c80aa7..dbad494e40 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 6)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 7)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 56ab6a2a7b224cb4a30e545035639ab84d82c58f Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Mon, 7 Jan 2019 01:31:17 -0800
Subject: [PATCH 0258/2345] Fix Template.variables for templates with empty
 name.

PiperOrigin-RevId: 228123554
---
 .../python/kernel_tests/template_test.py      | 15 ++++++++++
 tensorflow/python/ops/template.py             | 30 ++++---------------
 2 files changed, 20 insertions(+), 25 deletions(-)

diff --git a/tensorflow/python/kernel_tests/template_test.py b/tensorflow/python/kernel_tests/template_test.py
index 3b2a56bd1f..f587a7ec43 100644
--- a/tensorflow/python/kernel_tests/template_test.py
+++ b/tensorflow/python/kernel_tests/template_test.py
@@ -160,6 +160,21 @@ class TemplateTest(test.TestCase):
     self.assertEqual(1, len(result))
     self.assertNotEqual(len(first), len(result))
 
+  @test_util.run_in_graph_and_eager_modes
+  def test_template_with_empty_name(self):
+    tpl = template.make_template("", variable_scoped_function)
+    with variable_scope.variable_scope("outer"):
+      x = variable_scope.get_variable("x", [])
+      v = tpl()
+    self.assertEqual("outer/", tpl.variable_scope_name)
+    self.assertEqual("outer//dummy:0", v.name)
+    if context.executing_eagerly():
+      # In eager mode `x` is not visible to the template since the template does
+      # not rely on global collections.
+      self.assertEqual([v], tpl.variables)
+    else:
+      self.assertEqual([x, v], tpl.variables)
+
   @test_util.run_in_graph_and_eager_modes
   def test_template_with_name(self):
     tmpl1 = template.make_template("s1", variable_scoped_function)
diff --git a/tensorflow/python/ops/template.py b/tensorflow/python/ops/template.py
index 7c2d3be338..4eaa16a22c 100644
--- a/tensorflow/python/ops/template.py
+++ b/tensorflow/python/ops/template.py
@@ -387,8 +387,11 @@ class Template(checkpointable.CheckpointableBase):
     """Returns the variable scope name created by this Template."""
     if self._variable_scope:
       name = self._variable_scope.name
-      # To prevent partial matches on the scope_name, we add '/' at the end.
-      return name if name[-1] == "/" else name + "/"
+      if not name or name[-1] == "/":
+        return name
+      else:
+        # To prevent partial matches on the scope_name, we add '/' at the end.
+        return name + "/"
 
   @property
   def variables(self):
@@ -646,29 +649,6 @@ class EagerTemplate(Template):
         with self._template_store.as_default():
           return self._call_func(args, kwargs)
 
-  @property
-  def name(self):
-    """Returns the name given to this Template."""
-    return self._name
-
-  @property
-  def func(self):
-    """Returns the func given to this Template."""
-    return self._func
-
-  @property
-  def variable_scope(self):
-    """Returns the variable scope object created by this Template."""
-    return self._variable_scope
-
-  @property
-  def variable_scope_name(self):
-    """Returns the variable scope name created by this Template."""
-    if self._variable_scope:
-      name = self._variable_scope.name
-      # To prevent partial matches on the scope_name, we add '/' at the end.
-      return name if name[-1] == "/" else name + "/"
-
   @property
   def variables(self):
     """Returns the list of variables created by the Template."""
-- 
GitLab


From 48d0e4d813b914fc480c51de5c48903b9bcc4cc7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 01:46:24 -0800
Subject: [PATCH 0259/2345] Fixing pasta by not stripping the subdirectory.

PiperOrigin-RevId: 228125275
---
 third_party/pasta/BUILD.bazel    |  5 +++--
 third_party/pasta/build_defs.bzl | 12 ++++++++++++
 third_party/pasta/workspace.bzl  |  2 +-
 3 files changed, 16 insertions(+), 3 deletions(-)
 create mode 100644 third_party/pasta/build_defs.bzl

diff --git a/third_party/pasta/BUILD.bazel b/third_party/pasta/BUILD.bazel
index 6be33511e4..ade681b606 100644
--- a/third_party/pasta/BUILD.bazel
+++ b/third_party/pasta/BUILD.bazel
@@ -1,5 +1,6 @@
 # Description:
 #   AST-based python refactoring.
+load("@//third_party/pasta:build_defs.bzl", "copy_srcs")
 
 licenses(["notice"])  # Apache2
 
@@ -7,7 +8,7 @@ exports_files(["LICENSE"])
 
 py_library(
     name = "pasta",
-    srcs = [
+    srcs = copy_srcs([
         "__init__.py",
         "augment/__init__.py",
         "augment/errors.py",
@@ -23,7 +24,7 @@ py_library(
         "base/scope.py",
         "base/test_utils.py",
         "base/token_generator.py",
-    ],
+    ]),
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
 )
diff --git a/third_party/pasta/build_defs.bzl b/third_party/pasta/build_defs.bzl
new file mode 100644
index 0000000000..0a5316de40
--- /dev/null
+++ b/third_party/pasta/build_defs.bzl
@@ -0,0 +1,12 @@
+"""Skylark makros for building pasta."""
+
+def copy_srcs(srcs):
+    """Copies srcs from 'pasta' to parent directory."""
+    for src in srcs:
+        native.genrule(
+            name = src.replace(".", "_"),
+            srcs = ["pasta/" + src],
+            outs = [src],
+            cmd = "mkdir -p $$(dirname $@); cp $< $@",
+        )
+    return srcs
diff --git a/third_party/pasta/workspace.bzl b/third_party/pasta/workspace.bzl
index 063bdb7b98..e46cc4a45e 100644
--- a/third_party/pasta/workspace.bzl
+++ b/third_party/pasta/workspace.bzl
@@ -9,7 +9,7 @@ def repo():
             "https://mirror.bazel.build/github.com/google/pasta/archive/c3d72cdee6fc806251949e912510444d58d7413c.tar.gz",
             "https://github.com/google/pasta/archive/c3d72cdee6fc806251949e912510444d58d7413c.tar.gz",
         ],
-        strip_prefix = "pasta-c3d72cdee6fc806251949e912510444d58d7413c/pasta",
+        strip_prefix = "pasta-c3d72cdee6fc806251949e912510444d58d7413c",
         sha256 = "b5905f9cecc4b28363c563f3c4cb0545288bd35f7cc72c55066e97e53befc084",
         build_file = "//third_party/pasta:BUILD.bazel",
         system_build_file = "//third_party/pasta:BUILD.system",
-- 
GitLab


From f64f7f787d3596cb7c9228f131f06c159a0ec188 Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Mon, 7 Jan 2019 02:38:36 -0800
Subject: [PATCH 0260/2345] Added option in conversion script to convert an
 entire directory in place when --inplace=True given as an argument

PiperOrigin-RevId: 228130782
---
 tensorflow/tools/compatibility/ast_edits.py   | 34 ++++++++++++++++++-
 .../tools/compatibility/tf_upgrade_v2_main.py | 10 +++++-
 2 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/compatibility/ast_edits.py b/tensorflow/tools/compatibility/ast_edits.py
index 0ec6c2ac8c..859fd95314 100644
--- a/tensorflow/tools/compatibility/ast_edits.py
+++ b/tensorflow/tools/compatibility/ast_edits.py
@@ -493,7 +493,7 @@ class ASTCodeUpgrader(object):
             process_errors)
 
   def process_tree(self, root_directory, output_root_directory,
-                   copy_other_files):
+                   copy_other_files, in_place):
     """Processes upgrades on an entire tree of python files in place.
 
     Note that only Python files. If you have custom code in other languages,
@@ -503,11 +503,20 @@ class ASTCodeUpgrader(object):
       root_directory: Directory to walk and process.
       output_root_directory: Directory to use as base.
       copy_other_files: Copy files that are not touched by this converter.
+      in_place: Allow the conversion of an entire directory in place.
 
     Returns:
       A tuple of files processed, the report string ofr all files, and errors
     """
 
+    if output_root_directory == root_directory:
+      if in_place:
+        return self.process_tree_inplace(root_directory)
+      else:
+        print("In order to copy a directory in place the `--inplace` input "
+              "arg must be set to `True`.")
+        sys.exit(1)
+
     # make sure output directory doesn't exist
     if output_root_directory and os.path.exists(output_root_directory):
       print("Output directory %r must not already exist." %
@@ -564,3 +573,26 @@ class ASTCodeUpgrader(object):
         os.makedirs(output_directory)
       shutil.copy(input_path, output_path)
     return file_count, report, tree_errors
+
+  def process_tree_inplace(self, root_directory):
+    """Process a directory of python files in place."""
+    files_to_process = []
+    for dir_name, _, file_list in os.walk(root_directory):
+      py_files = [os.path.join(dir_name,
+                               f) for f in file_list if f.endswith(".py")]
+      files_to_process += py_files
+
+    file_count = 0
+    tree_errors = []
+    report = ""
+    report += ("=" * 80) + "\n"
+    report += "Input tree: %r\n" % root_directory
+    report += ("=" * 80) + "\n"
+
+    for path in files_to_process:
+      file_count += 1
+      _, l_report, l_errors = self.process_file(path, path)
+      tree_errors += l_errors
+      report += l_report
+
+    return file_count, report, tree_errors
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_main.py b/tensorflow/tools/compatibility/tf_upgrade_v2_main.py
index 543d078642..870bc6f216 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_main.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_main.py
@@ -59,6 +59,14 @@ Simple usage:
             "copy the other files."),
       type=bool,
       default=True)
+  parser.add_argument(
+      "--inplace",
+      dest="in_place",
+      help=("If converting a whole tree of files, whether to "
+            "allow the conversion to be performed on the "
+            "files in the input tree."),
+      type=bool,
+      default=False)
   parser.add_argument(
       "--reportfile",
       dest="report_filename",
@@ -86,7 +94,7 @@ Simple usage:
           "--outtree=<output directory> argument is required when converting a "
           "file tree.")
     files_processed, report_text, errors = upgrade.process_tree(
-        args.input_tree, args.output_tree, args.copy_other_files)
+        args.input_tree, args.output_tree, args.copy_other_files, args.in_place)
   else:
     parser.print_help()
   if report_text:
-- 
GitLab


From 6612e1994c26ee4e40fab02b270bd79cdc7b0a74 Mon Sep 17 00:00:00 2001
From: Dayananda V <dayanandasiet@gmail.com>
Date: Mon, 7 Jan 2019 17:20:25 +0530
Subject: [PATCH 0261/2345] TF Android Example first time compile and run
 FileNotFoundException bug fix

java.lang.RuntimeException: java.io.FileNotFoundException: mobilenet_v1_1.0_224_quant.tflite
this error reported after application compile and run first time using Studio, this fix solves the exist problem.
---
 .../lite/examples/android/app/build.gradle       | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/tensorflow/lite/examples/android/app/build.gradle b/tensorflow/lite/examples/android/app/build.gradle
index e5f5c7efd1..fe8fa4daf9 100644
--- a/tensorflow/lite/examples/android/app/build.gradle
+++ b/tensorflow/lite/examples/android/app/build.gradle
@@ -1,5 +1,13 @@
 apply plugin: 'com.android.application'
 
+// import DownloadModels task
+project.ext.ASSET_DIR = projectDir.toString() + '/src/main/assets'
+project.ext.TMP_DIR   = project.buildDir.toString() + '/downloads'
+
+// Download default models; if you wish to use your own models then
+// place them in the "assets" directory and comment out this line.
+apply from: "download-models.gradle"
+
 android {
     compileSdkVersion 26
     buildToolsVersion '26.0.2'
@@ -36,14 +44,6 @@ repositories {
     }
 }
 
-// import DownloadModels task
-project.ext.ASSET_DIR = projectDir.toString() + '/src/main/assets'
-project.ext.TMP_DIR   = project.buildDir.toString() + '/downloads'
-
-// Download default models; if you wish to use your own models then
-// place them in the "assets" directory and comment out this line.
-apply from: "download-models.gradle"
-
 dependencies {
     compile fileTree(dir: 'libs', include: ['*.jar'])
     compile 'org.tensorflow:tensorflow-lite:0.0.0-nightly'
-- 
GitLab


From 2703d8e84aced78a05ec7ad1ea0e5f4d0fe70f9f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 07:03:10 -0800
Subject: [PATCH 0262/2345] Change //tensorflow/compiler/tf2xla/python:xla to a
 tf_custom_op_py_library

Previously it was a simple py_library what didn't linked in the op registration
code causing errors when no other dependency pulled in the actual
implementations.

PiperOrigin-RevId: 228158673
---
 tensorflow/compiler/tf2xla/ops/BUILD    | 17 ++++++++++++++++-
 tensorflow/compiler/tf2xla/python/BUILD |  7 ++++++-
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/ops/BUILD b/tensorflow/compiler/tf2xla/ops/BUILD
index 4dce0a2102..7140b6a122 100644
--- a/tensorflow/compiler/tf2xla/ops/BUILD
+++ b/tensorflow/compiler/tf2xla/ops/BUILD
@@ -4,7 +4,11 @@ package(
 
 licenses(["notice"])  # Apache 2.0
 
-load("//tensorflow:tensorflow.bzl", "tf_gen_op_wrapper_py")
+load(
+    "//tensorflow:tensorflow.bzl",
+    "tf_custom_op_library",
+    "tf_gen_op_wrapper_py",
+)
 
 cc_library(
     name = "xla_ops",
@@ -24,3 +28,14 @@ tf_gen_op_wrapper_py(
         ":xla_ops",
     ],
 )
+
+tf_custom_op_library(
+    name = "_xla_ops.so",
+    srcs = [
+        "xla_ops.cc",
+    ],
+    deps = [
+        "//tensorflow/core:lib",
+        "@com_google_absl//absl/algorithm:container",
+    ],
+)
diff --git a/tensorflow/compiler/tf2xla/python/BUILD b/tensorflow/compiler/tf2xla/python/BUILD
index fef97b98c3..9abdb04d77 100644
--- a/tensorflow/compiler/tf2xla/python/BUILD
+++ b/tensorflow/compiler/tf2xla/python/BUILD
@@ -15,6 +15,7 @@ load(
     "//tensorflow/core:platform/default/build_config.bzl",
     "tf_py_clif_cc",
 )
+load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library")
 
 tf_py_clif_cc(
     name = "xla_op_registry",
@@ -27,9 +28,13 @@ tf_py_clif_cc(
     ],
 )
 
-py_library(
+tf_custom_op_py_library(
     name = "xla",
     srcs = ["xla.py"],
+    dso = ["//tensorflow/compiler/tf2xla/ops:_xla_ops.so"],
+    kernels = [
+        "//tensorflow/compiler/tf2xla/ops:xla_ops",
+    ],
     deps = [
         "//tensorflow/compiler/tf2xla/ops:gen_xla_ops",
         "//tensorflow/compiler/xla:xla_data_proto_py",
-- 
GitLab


From e55ec99c9ef627d84750853f9ae83196723852d5 Mon Sep 17 00:00:00 2001
From: Adam Weiss <adam@signal11.com>
Date: Thu, 13 Dec 2018 15:22:36 -0500
Subject: [PATCH 0263/2345] [XLA] Wrap
 LocalComputationBuilder::GetOrCreateLocalClient with StatusOr<T>

Prevents crashes when GPUs present, but below minimum capability level
---
 .../xla/python/local_computation_builder.cc   | 21 ++++++++++---------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/tensorflow/compiler/xla/python/local_computation_builder.cc b/tensorflow/compiler/xla/python/local_computation_builder.cc
index 5d191f5a18..6611c87673 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.cc
+++ b/tensorflow/compiler/xla/python/local_computation_builder.cc
@@ -101,7 +101,7 @@ int GetReplicaCount() {
   return g_replica_count;
 }
 
-LocalClient* GetOrCreateLocalClient() {
+StatusOr<LocalClient*> GetOrCreateLocalClient() {
   string* platform_name = GetPlatformNameString();
   tensorflow::mutex_lock lock(g_local_client_mutex);
   if (g_local_client != nullptr) {
@@ -110,7 +110,8 @@ LocalClient* GetOrCreateLocalClient() {
   LocalClientOptions options;
   options.set_platform(PlatformUtil::GetPlatform(*platform_name).ValueOrDie());
   options.set_number_of_replicas(g_replica_count);
-  g_local_client = ClientLibrary::GetOrCreateLocalClient(options).ValueOrDie();
+  TF_ASSIGN_OR_RETURN(g_local_client,
+                      ClientLibrary::GetOrCreateLocalClient(options));
   CHECK(g_local_client != nullptr);
   return g_local_client;
 }
@@ -118,7 +119,7 @@ LocalClient* GetOrCreateLocalClient() {
 Status TransferToInfeedLocal(const Literal& literal) {
   VLOG(1) << "Infeeding literal without replica number; shape: "
           << literal.shape();
-  LocalClient* client = GetOrCreateLocalClient();
+  TF_ASSIGN_OR_RETURN(LocalClient * client, GetOrCreateLocalClient());
   return client->TransferToInfeedLocal(literal, /*device_ordinal=*/0);
 }
 
@@ -126,7 +127,7 @@ Status TransferToInfeedLocalReplica(const Literal& literal,
                                     int replica_number) {
   VLOG(1) << "Infeeding shape " << literal.shape()
           << " to replica number: " << replica_number;
-  LocalClient* client = GetOrCreateLocalClient();
+  TF_ASSIGN_OR_RETURN(LocalClient * client, GetOrCreateLocalClient());
   TF_ASSIGN_OR_RETURN(int device_ordinal,
                       client->ReplicaNumberToDeviceOrdinal(replica_number));
   return client->TransferToInfeedLocal(literal, device_ordinal);
@@ -136,7 +137,7 @@ StatusOr<Literal> TransferFromOutfeedLocalReplica(const Shape& shape,
                                                   int replica_number) {
   VLOG(1) << "Outfeeding literal from replica number: " << replica_number
           << " shape: " << shape;
-  LocalClient* client = GetOrCreateLocalClient();
+  TF_ASSIGN_OR_RETURN(LocalClient * client, GetOrCreateLocalClient());
   TF_ASSIGN_OR_RETURN(int device_ordinal,
                       client->ReplicaNumberToDeviceOrdinal(replica_number));
   return client->TransferFromOutfeedLocal(shape, device_ordinal);
@@ -153,7 +154,7 @@ static StatusOr<ScopedShapedBuffer> ToBuffer(LocalClient* client,
 StatusOr<LocalShapedBuffer*> LocalShapedBuffer::FromLiteral(
     const Literal& argument, const absl::optional<Shape>& shape_with_layout,
     int replica_number) {
-  LocalClient* client = GetOrCreateLocalClient();
+  TF_ASSIGN_OR_RETURN(LocalClient * client, GetOrCreateLocalClient());
   TF_ASSIGN_OR_RETURN(int device_ordinal,
                       client->ReplicaNumberToDeviceOrdinal(replica_number));
   VLOG(1) << "Creating shaped buffer from literal on replica/ordinal: "
@@ -183,7 +184,7 @@ const Shape& LocalShapedBuffer::shape() const {
 }
 
 StatusOr<Literal> LocalShapedBuffer::ToLiteral() const {
-  LocalClient* client = GetOrCreateLocalClient();
+  TF_ASSIGN_OR_RETURN(LocalClient * client, GetOrCreateLocalClient());
   return client->ShapedBufferToLiteral(*shaped_buffer());
 }
 
@@ -319,7 +320,7 @@ CompiledLocalComputation::CompiledLocalComputation(
 
 StatusOr<LocalShapedBuffer*> CompiledLocalComputation::Execute(
     absl::Span<LocalShapedBuffer* const> argument_handles) {
-  LocalClient* client = GetOrCreateLocalClient();
+  TF_ASSIGN_OR_RETURN(LocalClient * client, GetOrCreateLocalClient());
   StatusOr<int> device_ordinal_status = client->ReplicaNumberToDeviceOrdinal(0);
   StatusOr<ScopedShapedBuffer> result_buffer_status;
   if (!device_ordinal_status.ok()) {
@@ -362,7 +363,7 @@ StatusOr<LocalShapedBuffer*> CompiledLocalComputation::Execute(
 
 StatusOr<LocalShapedBufferTuple*> CompiledLocalComputation::ExecutePerReplica(
     absl::Span<const std::vector<LocalShapedBuffer*>> argument_handles) {
-  LocalClient* client = GetOrCreateLocalClient();
+  TF_ASSIGN_OR_RETURN(LocalClient * client, GetOrCreateLocalClient());
   const int num_replicas = GetReplicaCount();
 
   if (argument_handles.size() != num_replicas) {
@@ -535,7 +536,7 @@ StatusOr<CompiledLocalComputation*> LocalComputation::Compile(
     argument_shape_pointers.push_back(&argument_shape);
   }
 
-  LocalClient* client = GetOrCreateLocalClient();
+  TF_ASSIGN_OR_RETURN(LocalClient * client, GetOrCreateLocalClient());
   ExecutableBuildOptions options;
   if (build_options != nullptr) {
     options = *build_options;
-- 
GitLab


From 36c3466356546f09198ce01982e6049dbf1a1afe Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 10:31:04 -0800
Subject: [PATCH 0264/2345] Merged commit includes the following changes:
 228190834  by A. Unique TensorFlower<gardener@tensorflow.org>:

    Internal change

--
228187607  by A. Unique TensorFlower<gardener@tensorflow.org>:

    Split depthwise conv GPU code into multiple files

    This file was a bottleneck during compilation, often taking many minutes to compile. In local testing this change reduces the wall-clock build time for the depthwise conv GPU kernels from 163s to 105s.

--
228181904  by A. Unique TensorFlower<gardener@tensorflow.org>:

    Fix py2 and py3 of gen_git_version.py to fix 1.13 release.

    Byteify function to handle py2 and py3.

--
228161227  by A. Unique TensorFlower<gardener@tensorflow.org>:

    Create Tensorflow version of LLVM's opt.

    This is useful for iterating on individual passes in the OptimizationPassRegistry.
    This is run with a command of the form optimization_pass_runner --input_file_path=/tmp/input.pbtxt --output_file_path=/tmp/output.pbtxt --optimization_pass=NameOfGraphOptimizationPass

--

PiperOrigin-RevId: 228190834
---
 .../common_runtime/optimization_registry.h    |   4 +
 tensorflow/core/kernels/BUILD                 |  10 +-
 ...v_op_gpu.cu.cc => depthwise_conv_op_gpu.h} |  25 +-
 .../depthwise_conv_op_gpu_double.cu.cc        |  30 ++
 .../kernels/depthwise_conv_op_gpu_float.cu.cc |  30 ++
 .../kernels/depthwise_conv_op_gpu_half.cu.cc  |  30 ++
 tensorflow/opensource_only.files              | 416 +++++++++---------
 tensorflow/tools/git/gen_git_source.py        |   4 +-
 tensorflow/tools/optimization/BUILD           |  52 +++
 .../gpu_optimization_pass_runner_main.cc      |  60 +++
 .../optimization/optimization_pass_runner.cc  | 167 +++++++
 .../optimization/optimization_pass_runner.h   |  61 +++
 12 files changed, 661 insertions(+), 228 deletions(-)
 rename tensorflow/core/kernels/{depthwise_conv_op_gpu.cu.cc => depthwise_conv_op_gpu.h} (98%)
 create mode 100644 tensorflow/core/kernels/depthwise_conv_op_gpu_double.cu.cc
 create mode 100644 tensorflow/core/kernels/depthwise_conv_op_gpu_float.cu.cc
 create mode 100644 tensorflow/core/kernels/depthwise_conv_op_gpu_half.cu.cc
 create mode 100644 tensorflow/tools/optimization/BUILD
 create mode 100644 tensorflow/tools/optimization/gpu_optimization_pass_runner_main.cc
 create mode 100644 tensorflow/tools/optimization/optimization_pass_runner.cc
 create mode 100644 tensorflow/tools/optimization/optimization_pass_runner.h

diff --git a/tensorflow/core/common_runtime/optimization_registry.h b/tensorflow/core/common_runtime/optimization_registry.h
index 19a76529bb..0e31f389aa 100644
--- a/tensorflow/core/common_runtime/optimization_registry.h
+++ b/tensorflow/core/common_runtime/optimization_registry.h
@@ -95,6 +95,10 @@ class OptimizationPassRegistry {
   void Register(Grouping grouping, int phase,
                 std::unique_ptr<GraphOptimizationPass> pass);
 
+  const std::map<Grouping, GraphOptimizationPasses>& groups() {
+    return groups_;
+  }
+
   // Run all passes in grouping, ordered by phase, with the same
   // options.
   Status RunGrouping(Grouping grouping,
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index c72d04d3af..3891caf5d3 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -3695,7 +3695,15 @@ tf_kernel_library(
 
 tf_kernel_library(
     name = "depthwise_conv_op",
-    prefix = "depthwise_conv_op",
+    srcs = ["depthwise_conv_op.cc"],
+    hdrs = ["depthwise_conv_op.h"],
+    gpu_srcs = [
+        "depthwise_conv_op.h",
+        "depthwise_conv_op_gpu.h",
+        "depthwise_conv_op_gpu_double.cu.cc",
+        "depthwise_conv_op_gpu_float.cu.cc",
+        "depthwise_conv_op_gpu_half.cu.cc",
+    ],
     deps = [
         ":bounds_check",
         ":conv_ops",
diff --git a/tensorflow/core/kernels/depthwise_conv_op_gpu.cu.cc b/tensorflow/core/kernels/depthwise_conv_op_gpu.h
similarity index 98%
rename from tensorflow/core/kernels/depthwise_conv_op_gpu.cu.cc
rename to tensorflow/core/kernels/depthwise_conv_op_gpu.h
index e811968d27..098853e684 100644
--- a/tensorflow/core/kernels/depthwise_conv_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/depthwise_conv_op_gpu.h
@@ -13,6 +13,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#ifndef TENSORFLOW_CORE_KERNELS_DEPTHWISE_CONV_OP_GPU_H_
+#define TENSORFLOW_CORE_KERNELS_DEPTHWISE_CONV_OP_GPU_H_
+
 #if GOOGLE_CUDA
 #define EIGEN_USE_GPU
 
@@ -38,7 +41,7 @@ using Eigen::GpuDevice;
 
 // Returns whether depthwise convolution forward or backward input pass can be
 // performed using the faster ('Small') variant of the kernel.
-EIGEN_DEVICE_FUNC bool CanLaunchDepthwiseConv2dGPUSmall(
+inline EIGEN_DEVICE_FUNC bool CanLaunchDepthwiseConv2dGPUSmall(
     const DepthwiseArgs& args) {
   return args.depth_multiplier == 1 && args.stride == 1 && args.in_rows <= 32 &&
          args.in_cols <= 32 && args.in_rows == args.out_rows &&
@@ -51,7 +54,7 @@ EIGEN_DEVICE_FUNC bool CanLaunchDepthwiseConv2dGPUSmall(
 
 // Returns whether depthwise convolution backward filter pass can be performed
 // using the faster ('Small') variant of the kernel.
-EIGEN_DEVICE_FUNC bool CanLaunchDepthwiseConv2dBackpropFilterGPUSmall(
+inline EIGEN_DEVICE_FUNC bool CanLaunchDepthwiseConv2dBackpropFilterGPUSmall(
     const DepthwiseArgs& args, const int block_height) {
   return args.depth_multiplier == 1 && args.stride == 1 && args.in_rows <= 32 &&
          args.in_cols <= 32 && args.in_rows == args.out_rows &&
@@ -652,13 +655,12 @@ struct PseudoHalfType<Eigen::half> {
 };
 }  // namespace detail
 
-namespace {
 // Maps to float if T is __half, and to T otherwise.
 template <typename T>
 using PseudoHalfType = typename detail::PseudoHalfType<T>::Type;
 
 // Returns whether the context's GPU supports efficient fp16 math.
-bool HasFastHalfMath(OpKernelContext* ctx) {
+inline bool HasFastHalfMath(OpKernelContext* ctx) {
   int major, minor;
   ctx->op_device_context()
       ->stream()
@@ -669,7 +671,6 @@ bool HasFastHalfMath(OpKernelContext* ctx) {
   // GPUs before sm_53 don't support fp16 math, and sm_61's fp16 math is slow.
   return cuda_arch >= 530 && cuda_arch != 610;
 }
-}  // namespace
 
 template <typename T, DepthwiseConv2dDirection kDirection,
           int kKnownFilterWidth, int kKnownFilterHeight, int kBlockDepth,
@@ -808,10 +809,6 @@ void LaunchDepthwiseConvOp<GpuDevice, T>::operator()(OpKernelContext* ctx,
   }
 }
 
-template struct LaunchDepthwiseConvOp<GpuDevice, Eigen::half>;
-template struct LaunchDepthwiseConvOp<GpuDevice, float>;
-template struct LaunchDepthwiseConvOp<GpuDevice, double>;
-
 // A Cuda kernel to compute the depthwise convolution backprop w.r.t. input.
 template <typename T, int kKnownFilterWidth, int kKnownFilterHeight,
           int kKnownDepthMultiplier>
@@ -1030,10 +1027,6 @@ void LaunchDepthwiseConvBackpropInputOp<GpuDevice, T>::operator()(
   }
 }
 
-template struct LaunchDepthwiseConvBackpropInputOp<GpuDevice, Eigen::half>;
-template struct LaunchDepthwiseConvBackpropInputOp<GpuDevice, float>;
-template struct LaunchDepthwiseConvBackpropInputOp<GpuDevice, double>;
-
 // A Cuda kernel to compute the depthwise convolution backprop w.r.t. filter.
 template <typename T, int kKnownFilterWidth, int kKnownFilterHeight,
           int kKnownDepthMultiplier>
@@ -1803,9 +1796,7 @@ void LaunchDepthwiseConvBackpropFilterOp<GpuDevice, T>::operator()(
                  ctx, args, out_backprop, input, filter_backprop, data_format));
   }
 }
-
-template struct LaunchDepthwiseConvBackpropFilterOp<GpuDevice, Eigen::half>;
-template struct LaunchDepthwiseConvBackpropFilterOp<GpuDevice, float>;
-template struct LaunchDepthwiseConvBackpropFilterOp<GpuDevice, double>;
 }  // namespace tensorflow
 #endif  // GOOGLE_CUDA
+
+#endif  // TENSORFLOW_CORE_KERNELS_DEPTHWISE_CONV_OP_GPU_H_
diff --git a/tensorflow/core/kernels/depthwise_conv_op_gpu_double.cu.cc b/tensorflow/core/kernels/depthwise_conv_op_gpu_double.cu.cc
new file mode 100644
index 0000000000..073e7cf269
--- /dev/null
+++ b/tensorflow/core/kernels/depthwise_conv_op_gpu_double.cu.cc
@@ -0,0 +1,30 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/depthwise_conv_op.h"
+#include "tensorflow/core/kernels/depthwise_conv_op_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+
+template struct LaunchDepthwiseConvOp<GpuDevice, double>;
+template struct LaunchDepthwiseConvBackpropInputOp<GpuDevice, double>;
+template struct LaunchDepthwiseConvBackpropFilterOp<GpuDevice, double>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/depthwise_conv_op_gpu_float.cu.cc b/tensorflow/core/kernels/depthwise_conv_op_gpu_float.cu.cc
new file mode 100644
index 0000000000..4b0e15e476
--- /dev/null
+++ b/tensorflow/core/kernels/depthwise_conv_op_gpu_float.cu.cc
@@ -0,0 +1,30 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/depthwise_conv_op.h"
+#include "tensorflow/core/kernels/depthwise_conv_op_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+
+template struct LaunchDepthwiseConvOp<GpuDevice, float>;
+template struct LaunchDepthwiseConvBackpropInputOp<GpuDevice, float>;
+template struct LaunchDepthwiseConvBackpropFilterOp<GpuDevice, float>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/depthwise_conv_op_gpu_half.cu.cc b/tensorflow/core/kernels/depthwise_conv_op_gpu_half.cu.cc
new file mode 100644
index 0000000000..2db9fa4dff
--- /dev/null
+++ b/tensorflow/core/kernels/depthwise_conv_op_gpu_half.cu.cc
@@ -0,0 +1,30 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/depthwise_conv_op.h"
+#include "tensorflow/core/kernels/depthwise_conv_op_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+
+template struct LaunchDepthwiseConvOp<GpuDevice, Eigen::half>;
+template struct LaunchDepthwiseConvBackpropInputOp<GpuDevice, Eigen::half>;
+template struct LaunchDepthwiseConvBackpropFilterOp<GpuDevice, Eigen::half>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index e00d063cfe..40f87d6c36 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -1,248 +1,248 @@
-tensorflow/contrib/tpu/profiler/pip_package/BUILD
-tensorflow/contrib/tpu/profiler/pip_package/setup.py
-tensorflow/contrib/tpu/profiler/pip_package/README
-tensorflow/contrib/tpu/profiler/pip_package/build_pip_package.sh
-tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/main.py
-tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/__init__.py
-tensorflow/contrib/mpi/BUILD
-tensorflow/tools/ci_build/remote/BUILD
-tensorflow/tools/pip_package/README
-tensorflow/tools/pip_package/MANIFEST.in
-tensorflow/tools/pip_package/simple_console.py
-tensorflow/tools/pip_package/build_pip_package.sh
-tensorflow/tools/pip_package/check_load_py_test.py
-tensorflow/tools/pip_package/pip_smoke_test.py
-tensorflow/tools/pip_package/simple_console_for_windows.py
-tensorflow/tools/pip_package/setup.py
-tensorflow/tools/pip_package/BUILD
-tensorflow/tools/lib_package/concat_licenses.sh
-tensorflow/tools/lib_package/libtensorflow_test.c
-tensorflow/tools/lib_package/LibTensorFlowTest.java
-tensorflow/tools/lib_package/BUILD
-tensorflow/tools/lib_package/libtensorflow_test.sh
-tensorflow/tools/lib_package/README.md
-tensorflow/tools/lib_package/libtensorflow_java_test.sh
-tensorflow/tools/def_file_filter/def_file_filter_configure.bzl
-tensorflow/tools/def_file_filter/BUILD
-tensorflow/tools/def_file_filter/BUILD.tpl
-tensorflow/tools/def_file_filter/def_file_filter.py.tpl
-tensorflow/third_party/mkl/MKL_LICENSE
-tensorflow/third_party/mkl/LICENSE
-tensorflow/third_party/mkl/BUILD
-tensorflow/third_party/mkl/mkl.BUILD
-tensorflow/third_party/mkl/build_defs.bzl
-tensorflow/third_party/backports_weakref.BUILD
-tensorflow/third_party/toolchains/clang6/BUILD
+tensorflow/api_template_v1.__init__.py
+tensorflow/compat_template_v1.__init__.py
+tensorflow/stream_executor/BUILD
+tensorflow/third_party/curl.BUILD
+tensorflow/third_party/snappy.BUILD
+tensorflow/third_party/tflite_mobilenet_float.BUILD
+tensorflow/third_party/libxsmm.BUILD
+tensorflow/third_party/cub.BUILD
+tensorflow/third_party/repo.bzl
+tensorflow/third_party/systemlibs/curl.BUILD
+tensorflow/third_party/systemlibs/snappy.BUILD
+tensorflow/third_party/systemlibs/protobuf.BUILD
+tensorflow/third_party/systemlibs/absl_py.BUILD
+tensorflow/third_party/systemlibs/google_cloud_cpp.google.cloud.bigtable.BUILD
+tensorflow/third_party/systemlibs/astor.BUILD
+tensorflow/third_party/systemlibs/absl_py.absl.flags.BUILD
+tensorflow/third_party/systemlibs/png.BUILD
+tensorflow/third_party/systemlibs/BUILD.tpl
+tensorflow/third_party/systemlibs/boringssl.BUILD
+tensorflow/third_party/systemlibs/absl_py.absl.testing.BUILD
+tensorflow/third_party/systemlibs/swig.BUILD
+tensorflow/third_party/systemlibs/double_conversion.BUILD
+tensorflow/third_party/systemlibs/gif.BUILD
+tensorflow/third_party/systemlibs/cython.BUILD
+tensorflow/third_party/systemlibs/google_cloud_cpp.BUILD
+tensorflow/third_party/systemlibs/zlib.BUILD
+tensorflow/third_party/systemlibs/grpc.BUILD
+tensorflow/third_party/systemlibs/re2.BUILD
+tensorflow/third_party/systemlibs/build_defs.bzl.tpl
+tensorflow/third_party/systemlibs/protobuf.bzl
+tensorflow/third_party/systemlibs/gast.BUILD
+tensorflow/third_party/systemlibs/syslibs_configure.bzl
+tensorflow/third_party/systemlibs/nsync.BUILD
+tensorflow/third_party/systemlibs/jsoncpp.BUILD
+tensorflow/third_party/systemlibs/six.BUILD
+tensorflow/third_party/systemlibs/lmdb.BUILD
+tensorflow/third_party/systemlibs/BUILD
+tensorflow/third_party/systemlibs/googleapis.BUILD
+tensorflow/third_party/systemlibs/sqlite.BUILD
+tensorflow/third_party/systemlibs/termcolor.BUILD
+tensorflow/third_party/systemlibs/pcre.BUILD
+tensorflow/third_party/tflite_mobilenet_quant.BUILD
+tensorflow/third_party/common.bzl
+tensorflow/third_party/boringssl/BUILD
+tensorflow/third_party/kafka/config.patch
+tensorflow/third_party/kafka/BUILD
+tensorflow/third_party/android/android_configure.bzl
+tensorflow/third_party/android/android_configure.BUILD.tpl
+tensorflow/third_party/android/android.bzl.tpl
+tensorflow/third_party/android/BUILD
+tensorflow/third_party/astor.BUILD
+tensorflow/third_party/icu/udata.patch
+tensorflow/third_party/tflite_mobilenet.BUILD
+tensorflow/third_party/png.BUILD
+tensorflow/third_party/mpi/.gitignore
+tensorflow/third_party/mpi/BUILD
+tensorflow/third_party/nanopb.BUILD
+tensorflow/third_party/arm_neon_2_x86_sse.BUILD
+tensorflow/third_party/linenoise.BUILD
+tensorflow/third_party/llvm/llvm.autogenerated.BUILD
+tensorflow/third_party/llvm/llvm.bzl
+tensorflow/third_party/llvm/BUILD
+tensorflow/third_party/llvm/expand_cmake_vars.py
+tensorflow/third_party/swig.BUILD
+tensorflow/third_party/png_fix_rpi.patch
+tensorflow/third_party/double_conversion.BUILD
+tensorflow/third_party/gif.BUILD
+tensorflow/third_party/grpc/BUILD
+tensorflow/third_party/cython.BUILD
+tensorflow/third_party/eigen.BUILD
+tensorflow/third_party/codegen.BUILD
+tensorflow/third_party/zlib.BUILD
+tensorflow/third_party/tensorrt/BUILD.tpl
+tensorflow/third_party/tensorrt/tensorrt_configure.bzl
+tensorflow/third_party/tensorrt/build_defs.bzl.tpl
+tensorflow/third_party/tensorrt/remote.BUILD.tpl
+tensorflow/third_party/tensorrt/LICENSE
+tensorflow/third_party/tensorrt/BUILD
+tensorflow/third_party/tflite_ovic_testdata.BUILD
+tensorflow/third_party/fft2d/fft.h
+tensorflow/third_party/fft2d/fft2d.BUILD
+tensorflow/third_party/fft2d/LICENSE
+tensorflow/third_party/fft2d/BUILD
 tensorflow/third_party/toolchains/clang6/README.md
 tensorflow/third_party/toolchains/clang6/repo.bzl
-tensorflow/third_party/toolchains/clang6/CROSSTOOL.tpl
 tensorflow/third_party/toolchains/clang6/clang.BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/BUILD
+tensorflow/third_party/toolchains/clang6/CROSSTOOL.tpl
+tensorflow/third_party/toolchains/clang6/BUILD
+tensorflow/third_party/toolchains/preconfig/generate/archives.bzl
+tensorflow/third_party/toolchains/preconfig/generate/workspace.bzl
+tensorflow/third_party/toolchains/preconfig/generate/generate.bzl
+tensorflow/third_party/toolchains/preconfig/generate/containers.bzl
+tensorflow/third_party/toolchains/preconfig/generate/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/build_defs.bzl
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/py3/BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/build_defs.bzl
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/build_defs.bzl
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/build_defs.bzl
-tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/nccl2/BUILD
-tensorflow/third_party/toolchains/preconfig/generate/workspace.bzl
-tensorflow/third_party/toolchains/preconfig/generate/containers.bzl
-tensorflow/third_party/toolchains/preconfig/generate/generate.bzl
-tensorflow/third_party/toolchains/preconfig/generate/archives.bzl
-tensorflow/third_party/toolchains/preconfig/generate/BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu16.04/clang/BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu16.04/clang/dummy_toolchain.bzl
-tensorflow/third_party/toolchains/preconfig/ubuntu16.04/py3/BUILD
-tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/BUILD
-tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/dummy_toolchain.bzl
 tensorflow/third_party/toolchains/preconfig/win_1803/py36/BUILD
 tensorflow/third_party/toolchains/preconfig/win_1803/BUILD
-tensorflow/third_party/toolchains/gpus/cuda/build_defs.bzl
-tensorflow/third_party/toolchains/gpus/cuda/BUILD
-tensorflow/third_party/toolchains/gpus/cuda/cuda/cuda_config.h
-tensorflow/third_party/toolchains/gpus/crosstool/BUILD
-tensorflow/third_party/toolchains/gpus/crosstool/CROSSTOOL
-tensorflow/third_party/toolchains/gpus/py/BUILD
-tensorflow/third_party/toolchains/cpus/arm/arm_compiler_configure.bzl
+tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/dummy_toolchain.bzl
+tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu16.04/py3/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu16.04/clang/dummy_toolchain.bzl
+tensorflow/third_party/toolchains/preconfig/ubuntu16.04/clang/BUILD
+tensorflow/third_party/toolchains/cpus/py3/BUILD
 tensorflow/third_party/toolchains/cpus/arm/CROSSTOOL.tpl
 tensorflow/third_party/toolchains/cpus/arm/BUILD
-tensorflow/third_party/toolchains/cpus/py3/BUILD
+tensorflow/third_party/toolchains/cpus/arm/arm_compiler_configure.bzl
 tensorflow/third_party/toolchains/cpus/py/BUILD
+tensorflow/third_party/toolchains/gpus/crosstool/CROSSTOOL
+tensorflow/third_party/toolchains/gpus/crosstool/BUILD
+tensorflow/third_party/toolchains/gpus/cuda/build_defs.bzl
+tensorflow/third_party/toolchains/gpus/cuda/cuda/cuda_config.h
+tensorflow/third_party/toolchains/gpus/cuda/BUILD
+tensorflow/third_party/toolchains/gpus/py/BUILD
 tensorflow/third_party/toolchains/BUILD
-tensorflow/third_party/gpus/BUILD
+tensorflow/third_party/nccl/archive.BUILD
+tensorflow/third_party/nccl/nccl_configure.bzl
+tensorflow/third_party/nccl/system.BUILD.tpl
+tensorflow/third_party/nccl/build_defs.bzl.tpl
+tensorflow/third_party/nccl/LICENSE
+tensorflow/third_party/nccl/BUILD
+tensorflow/third_party/farmhash.BUILD
+tensorflow/third_party/sycl/crosstool/BUILD
+tensorflow/third_party/backports_weakref.BUILD
+tensorflow/third_party/gast.BUILD
+tensorflow/third_party/clang_toolchain/cc_configure_clang.bzl
+tensorflow/third_party/clang_toolchain/download_clang.bzl
+tensorflow/third_party/clang_toolchain/BUILD
+tensorflow/third_party/mpi_collectives/BUILD
+tensorflow/third_party/jsoncpp.BUILD
+tensorflow/third_party/mkl_dnn/mkldnn.BUILD
+tensorflow/third_party/mkl_dnn/LICENSE
+tensorflow/third_party/python_runtime/BUILD
+tensorflow/third_party/ngraph/ngraph.BUILD
+tensorflow/third_party/ngraph/nlohmann_json.BUILD
+tensorflow/third_party/ngraph/build_defs.bzl
+tensorflow/third_party/ngraph/ngraph_tf.BUILD
+tensorflow/third_party/ngraph/NGRAPH_LICENSE
+tensorflow/third_party/ngraph/tbb.BUILD
+tensorflow/third_party/ngraph/LICENSE
+tensorflow/third_party/ngraph/BUILD
+tensorflow/third_party/gpus/crosstool/windows/msvc_wrapper_for_nvcc.py.tpl
+tensorflow/third_party/gpus/crosstool/CROSSTOOL.tpl
+tensorflow/third_party/gpus/crosstool/BUILD.tpl
+tensorflow/third_party/gpus/crosstool/remote.BUILD.tpl
 tensorflow/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_rocm.tpl
 tensorflow/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_is_not_gcc.tpl
-tensorflow/third_party/gpus/crosstool/CROSSTOOL.tpl
 tensorflow/third_party/gpus/crosstool/CROSSTOOL_hipcc.tpl
 tensorflow/third_party/gpus/crosstool/LICENSE
-tensorflow/third_party/gpus/crosstool/remote.BUILD.tpl
-tensorflow/third_party/gpus/crosstool/windows/msvc_wrapper_for_nvcc.py.tpl
-tensorflow/third_party/gpus/crosstool/BUILD.tpl
 tensorflow/third_party/gpus/crosstool/BUILD
-tensorflow/third_party/gpus/cuda/LICENSE
+tensorflow/third_party/gpus/rocm/BUILD.tpl
+tensorflow/third_party/gpus/rocm/rocm_config.h.tpl
+tensorflow/third_party/gpus/rocm/build_defs.bzl.tpl
+tensorflow/third_party/gpus/rocm/BUILD
 tensorflow/third_party/gpus/cuda/BUILD.tpl
+tensorflow/third_party/gpus/cuda/build_defs.bzl.tpl
+tensorflow/third_party/gpus/cuda/remote.BUILD.tpl
 tensorflow/third_party/gpus/cuda/BUILD.windows.tpl
 tensorflow/third_party/gpus/cuda/cuda_config.h.tpl
-tensorflow/third_party/gpus/cuda/remote.BUILD.tpl
+tensorflow/third_party/gpus/cuda/LICENSE
 tensorflow/third_party/gpus/cuda/BUILD
-tensorflow/third_party/gpus/cuda/build_defs.bzl.tpl
-tensorflow/third_party/gpus/rocm/rocm_config.h.tpl
-tensorflow/third_party/gpus/rocm/BUILD
-tensorflow/third_party/gpus/rocm/BUILD.tpl
-tensorflow/third_party/gpus/rocm/build_defs.bzl.tpl
 tensorflow/third_party/gpus/cuda_configure.bzl
 tensorflow/third_party/gpus/rocm_configure.bzl
-tensorflow/third_party/snappy.BUILD
-tensorflow/third_party/cython.BUILD
-tensorflow/third_party/farmhash.BUILD
-tensorflow/third_party/eigen3/Eigen/Cholesky
-tensorflow/third_party/eigen3/Eigen/QR
-tensorflow/third_party/eigen3/Eigen/LU
+tensorflow/third_party/gpus/BUILD
+tensorflow/third_party/py/python_configure.bzl
+tensorflow/third_party/py/BUILD.tpl
+tensorflow/third_party/py/numpy/BUILD
+tensorflow/third_party/py/remote.BUILD.tpl
+tensorflow/third_party/py/BUILD
+tensorflow/third_party/mkl/build_defs.bzl
+tensorflow/third_party/mkl/mkl.BUILD
+tensorflow/third_party/mkl/MKL_LICENSE
+tensorflow/third_party/mkl/LICENSE
+tensorflow/third_party/mkl/BUILD
+tensorflow/third_party/com_google_absl.BUILD
+tensorflow/third_party/six.BUILD
+tensorflow/third_party/lmdb.BUILD
+tensorflow/third_party/BUILD
+tensorflow/third_party/googleapis.BUILD
+tensorflow/third_party/sqlite.BUILD
+tensorflow/third_party/termcolor.BUILD
+tensorflow/third_party/protobuf/BUILD
+tensorflow/third_party/pcre.BUILD
+tensorflow/third_party/git/BUILD.tpl
+tensorflow/third_party/git/git_configure.bzl
+tensorflow/third_party/git/BUILD
+tensorflow/third_party/pprof.BUILD
+tensorflow/third_party/tflite_smartreply.BUILD
 tensorflow/third_party/eigen3/Eigen/Core
-tensorflow/third_party/eigen3/Eigen/SVD
+tensorflow/third_party/eigen3/Eigen/Cholesky
 tensorflow/third_party/eigen3/Eigen/Eigenvalues
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/FixedPoint
+tensorflow/third_party/eigen3/Eigen/SVD
+tensorflow/third_party/eigen3/Eigen/LU
+tensorflow/third_party/eigen3/Eigen/QR
+tensorflow/third_party/eigen3/unsupported/Eigen/MatrixFunctions
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/Tensor
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX512.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/ThreadPool
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/TypeCastingAVX512.h
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX2.h
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatMatProduct.h
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/FixedPointTypes.h
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatVecProduct.h
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatMatProductNEON.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX512.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatVecProduct.h
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/TypeCastingAVX2.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/FixedPointTypes.h
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatMatProductAVX2.h
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/ThreadPool
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX2.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/FixedPoint
 tensorflow/third_party/eigen3/unsupported/Eigen/SpecialFunctions
-tensorflow/third_party/eigen3/unsupported/Eigen/MatrixFunctions
 tensorflow/third_party/eigen3/LICENSE
 tensorflow/third_party/eigen3/BUILD
-tensorflow/third_party/systemlibs/build_defs.bzl.tpl
-tensorflow/third_party/systemlibs/absl_py.BUILD
-tensorflow/third_party/systemlibs/curl.BUILD
-tensorflow/third_party/systemlibs/termcolor.BUILD
-tensorflow/third_party/systemlibs/absl_py.absl.flags.BUILD
-tensorflow/third_party/systemlibs/grpc.BUILD
-tensorflow/third_party/systemlibs/swig.BUILD
-tensorflow/third_party/systemlibs/protobuf.bzl
-tensorflow/third_party/systemlibs/protobuf.BUILD
-tensorflow/third_party/systemlibs/BUILD
-tensorflow/third_party/systemlibs/google_cloud_cpp.BUILD
-tensorflow/third_party/systemlibs/astor.BUILD
-tensorflow/third_party/systemlibs/six.BUILD
-tensorflow/third_party/systemlibs/absl_py.absl.testing.BUILD
-tensorflow/third_party/systemlibs/boringssl.BUILD
-tensorflow/third_party/systemlibs/nsync.BUILD
-tensorflow/third_party/systemlibs/google_cloud_cpp.google.cloud.bigtable.BUILD
-tensorflow/third_party/systemlibs/gif.BUILD
-tensorflow/third_party/systemlibs/pcre.BUILD
-tensorflow/third_party/systemlibs/BUILD.tpl
-tensorflow/third_party/systemlibs/snappy.BUILD
-tensorflow/third_party/systemlibs/gast.BUILD
-tensorflow/third_party/systemlibs/cython.BUILD
-tensorflow/third_party/systemlibs/double_conversion.BUILD
-tensorflow/third_party/systemlibs/zlib.BUILD
-tensorflow/third_party/systemlibs/jsoncpp.BUILD
-tensorflow/third_party/systemlibs/re2.BUILD
-tensorflow/third_party/systemlibs/lmdb.BUILD
-tensorflow/third_party/systemlibs/googleapis.BUILD
-tensorflow/third_party/systemlibs/png.BUILD
-tensorflow/third_party/systemlibs/syslibs_configure.bzl
-tensorflow/third_party/systemlibs/sqlite.BUILD
-tensorflow/third_party/python_runtime/BUILD
-tensorflow/third_party/sycl/crosstool/BUILD
-tensorflow/third_party/ngraph/LICENSE
-tensorflow/third_party/ngraph/tbb.BUILD
-tensorflow/third_party/ngraph/BUILD
-tensorflow/third_party/ngraph/ngraph.BUILD
-tensorflow/third_party/ngraph/build_defs.bzl
-tensorflow/third_party/ngraph/NGRAPH_LICENSE
-tensorflow/third_party/ngraph/ngraph_tf.BUILD
-tensorflow/third_party/ngraph/nlohmann_json.BUILD
-tensorflow/third_party/clang_toolchain/download_clang.bzl
-tensorflow/third_party/clang_toolchain/BUILD
-tensorflow/third_party/clang_toolchain/cc_configure_clang.bzl
-tensorflow/third_party/gast.BUILD
-tensorflow/third_party/llvm/BUILD
-tensorflow/third_party/llvm/expand_cmake_vars.py
-tensorflow/third_party/llvm/llvm.autogenerated.BUILD
-tensorflow/third_party/llvm/llvm.bzl
-tensorflow/third_party/icu/udata.patch
-tensorflow/third_party/nccl/archive.BUILD
-tensorflow/third_party/nccl/LICENSE
-tensorflow/third_party/nccl/system.BUILD.tpl
-tensorflow/third_party/nccl/nccl_configure.bzl
-tensorflow/third_party/nccl/build_defs.bzl.tpl
-tensorflow/third_party/nccl/BUILD
-tensorflow/third_party/fft2d/BUILD
-tensorflow/third_party/fft2d/fft.h
-tensorflow/third_party/fft2d/LICENSE
-tensorflow/third_party/fft2d/fft2d.BUILD
-tensorflow/third_party/boringssl/BUILD
-tensorflow/third_party/mpi/.gitignore
-tensorflow/third_party/mpi/BUILD
-tensorflow/third_party/tensorrt/LICENSE
-tensorflow/third_party/tensorrt/BUILD
-tensorflow/third_party/tensorrt/build_defs.bzl.tpl
-tensorflow/third_party/tensorrt/BUILD.tpl
-tensorflow/third_party/tensorrt/tensorrt_configure.bzl
-tensorflow/third_party/tensorrt/remote.BUILD.tpl
-tensorflow/third_party/kafka/config.patch
-tensorflow/third_party/kafka/BUILD
-tensorflow/third_party/android/BUILD
-tensorflow/third_party/android/android.bzl.tpl
-tensorflow/third_party/android/android_configure.bzl
-tensorflow/third_party/android/android_configure.BUILD.tpl
-tensorflow/third_party/tflite_smartreply.BUILD
-tensorflow/third_party/mkl_dnn/LICENSE
-tensorflow/third_party/mkl_dnn/mkldnn.BUILD
-tensorflow/third_party/pcre.BUILD
-tensorflow/third_party/linenoise.BUILD
-tensorflow/third_party/sqlite.BUILD
-tensorflow/third_party/common.bzl
-tensorflow/third_party/com_google_absl.BUILD
-tensorflow/third_party/pprof.BUILD
-tensorflow/third_party/BUILD
-tensorflow/third_party/tflite_mobilenet_quant.BUILD
-tensorflow/third_party/lmdb.BUILD
-tensorflow/third_party/git/BUILD.tpl
-tensorflow/third_party/git/BUILD
-tensorflow/third_party/git/git_configure.bzl
-tensorflow/third_party/protobuf/BUILD
-tensorflow/third_party/tflite_mobilenet.BUILD
-tensorflow/third_party/py/BUILD
-tensorflow/third_party/py/BUILD.tpl
-tensorflow/third_party/py/remote.BUILD.tpl
-tensorflow/third_party/py/numpy/BUILD
-tensorflow/third_party/py/python_configure.bzl
-tensorflow/third_party/termcolor.BUILD
-tensorflow/third_party/png_fix_rpi.patch
-tensorflow/third_party/swig.BUILD
-tensorflow/third_party/astor.BUILD
-tensorflow/third_party/grpc/BUILD
-tensorflow/third_party/curl.BUILD
-tensorflow/third_party/arm_neon_2_x86_sse.BUILD
-tensorflow/third_party/png.BUILD
-tensorflow/third_party/googleapis.BUILD
-tensorflow/third_party/mpi_collectives/BUILD
-tensorflow/third_party/nanopb.BUILD
-tensorflow/third_party/gif.BUILD
-tensorflow/third_party/double_conversion.BUILD
-tensorflow/third_party/six.BUILD
-tensorflow/third_party/tflite_mobilenet_float.BUILD
-tensorflow/third_party/repo.bzl
-tensorflow/third_party/codegen.BUILD
-tensorflow/third_party/cub.BUILD
-tensorflow/third_party/jsoncpp.BUILD
-tensorflow/third_party/tflite_ovic_testdata.BUILD
-tensorflow/third_party/libxsmm.BUILD
-tensorflow/third_party/zlib.BUILD
-tensorflow/third_party/eigen.BUILD
-tensorflow/stream_executor/BUILD
-tensorflow/api_template_v1.__init__.py
-tensorflow/compat_template_v1.__init__.py
+tensorflow/tools/pip_package/simple_console.py
+tensorflow/tools/pip_package/setup.py
+tensorflow/tools/pip_package/MANIFEST.in
+tensorflow/tools/pip_package/README
+tensorflow/tools/pip_package/check_load_py_test.py
+tensorflow/tools/pip_package/pip_smoke_test.py
+tensorflow/tools/pip_package/BUILD
+tensorflow/tools/pip_package/build_pip_package.sh
+tensorflow/tools/pip_package/simple_console_for_windows.py
+tensorflow/tools/lib_package/LibTensorFlowTest.java
+tensorflow/tools/lib_package/README.md
+tensorflow/tools/lib_package/libtensorflow_test.sh
+tensorflow/tools/lib_package/libtensorflow_java_test.sh
+tensorflow/tools/lib_package/libtensorflow_test.c
+tensorflow/tools/lib_package/concat_licenses.sh
+tensorflow/tools/lib_package/BUILD
+tensorflow/tools/ci_build/remote/BUILD
+tensorflow/tools/def_file_filter/BUILD.tpl
+tensorflow/tools/def_file_filter/def_file_filter_configure.bzl
+tensorflow/tools/def_file_filter/def_file_filter.py.tpl
+tensorflow/tools/def_file_filter/BUILD
+tensorflow/contrib/mpi/BUILD
+tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/main.py
+tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/__init__.py
+tensorflow/contrib/tpu/profiler/pip_package/setup.py
+tensorflow/contrib/tpu/profiler/pip_package/README
+tensorflow/contrib/tpu/profiler/pip_package/BUILD
+tensorflow/contrib/tpu/profiler/pip_package/build_pip_package.sh
 tensorflow/api_template.__init__.py
 tensorflow/__init__.py
\ No newline at end of file
diff --git a/tensorflow/tools/git/gen_git_source.py b/tensorflow/tools/git/gen_git_source.py
index 645d817d9f..0c37508650 100755
--- a/tensorflow/tools/git/gen_git_source.py
+++ b/tensorflow/tools/git/gen_git_source.py
@@ -174,8 +174,8 @@ def get_git_version(git_base_path, git_tag_override):
       # There might be "-" in the tag name. But we can be sure that the final
       # two "-" are those inserted by the git describe command.
       abbrev_commit = split_val[-1]
-      val = bytes(
-          version_separator.join([git_tag_override, "0", abbrev_commit]))
+      val = version_separator.join(
+          [bytes(git_tag_override), b"0", abbrev_commit])
     return val if val else unknown_label
   except (subprocess.CalledProcessError, OSError):
     return unknown_label
diff --git a/tensorflow/tools/optimization/BUILD b/tensorflow/tools/optimization/BUILD
new file mode 100644
index 0000000000..aa6c850b0b
--- /dev/null
+++ b/tensorflow/tools/optimization/BUILD
@@ -0,0 +1,52 @@
+# Description:
+#   Utilities that perform useful transformations on graphs
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])  # Apache 2.0
+
+load(
+    "//tensorflow:tensorflow.bzl",
+    "tf_cc_binary",
+    "tf_cuda_library",
+)
+
+exports_files(["LICENSE"])
+
+tf_cuda_library(
+    name = "optimization_pass_runner_lib",
+    srcs = ["optimization_pass_runner.cc"],
+    hdrs = ["optimization_pass_runner.h"],
+    deps = [
+        "//tensorflow/contrib:contrib_ops_op_lib",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:core_cpu_base",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:framework_internal",
+        "//tensorflow/core:framework_lite",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:tensorflow",
+    ],
+)
+
+tf_cc_binary(
+    name = "gpu_optimization_pass_runner",
+    srcs = ["gpu_optimization_pass_runner_main.cc"],
+    deps = [
+        ":optimization_pass_runner_lib",
+        "//tensorflow/compiler/jit:xla_cpu_jit",
+        "//tensorflow/compiler/jit:xla_gpu_jit",
+        "//tensorflow/compiler/tf2xla:xla_compiler",
+        "//tensorflow/contrib:contrib_ops_op_lib",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:core_cpu_base",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:framework_internal",
+        "//tensorflow/core:framework_lite",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:tensorflow",
+        "@com_google_absl//absl/strings",
+    ],
+)
diff --git a/tensorflow/tools/optimization/gpu_optimization_pass_runner_main.cc b/tensorflow/tools/optimization/gpu_optimization_pass_runner_main.cc
new file mode 100644
index 0000000000..0d9f26cd5a
--- /dev/null
+++ b/tensorflow/tools/optimization/gpu_optimization_pass_runner_main.cc
@@ -0,0 +1,60 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+// This file creates a binary that can run any registered optimization pass.
+// ./xla_gpu_opt  --input_file_path=/tmp/input.pbtxt
+// --output_file_path=/tmp/output.pbtxt
+// --optimization_pass=NameOfGraphOptimizationPass
+
+#include "absl/strings/str_cat.h"
+#include "tensorflow/compiler/tf2xla/xla_op_registry.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/protobuf/config.pb.h"
+#include "tensorflow/tools/optimization/optimization_pass_runner.h"
+
+int main(int argc, char** argv) {
+  tensorflow::OptimizationPassRunner runner;
+  // Add fake devices for CPU, GPU, and XLA to ensure we have all devices we
+  // need.
+  // Most machines in our servers currently use 8 gpus. There is nothing special
+  // about this number and it can be decreased or increased to test other
+  // configurations.
+  int num_gpus_per_machine = 8;
+  for (int i = 0; i < num_gpus_per_machine; i++) {
+    TF_CHECK_OK(runner.AddDevice(
+        absl::StrCat("/job:localhost/replica:0/task:0/device:CPU:", i),
+        tensorflow::DEVICE_CPU));
+    TF_CHECK_OK(runner.AddDevice(
+        absl::StrCat("/job:localhost/replica:0/task:0/device:GPU:", i),
+        tensorflow::DEVICE_GPU));
+    TF_CHECK_OK(runner.AddDevice(
+        absl::StrCat("/job:localhost/replica:0/task:0/device:XLA_CPU:", i),
+        tensorflow::DEVICE_XLA_CPU));
+    TF_CHECK_OK(runner.AddDevice(
+        absl::StrCat("/job:localhost/replica:0/task:0/device:XLA_GPU:", i),
+        tensorflow::DEVICE_XLA_GPU));
+    TF_CHECK_OK(runner.AddDevice(
+        absl::StrCat("/job:localhost/replica:0/task:0/device:CPU_XLA_JIT:", i),
+        tensorflow::DEVICE_CPU_XLA_JIT));
+    TF_CHECK_OK(runner.AddDevice(
+        absl::StrCat("/job:localhost/replica:0/task:0/device:GPU_XLA_JIT:", i),
+        tensorflow::DEVICE_GPU_XLA_JIT));
+  }
+  // This binary is used to test TF:XLA behavior, so turn on auto_jit.
+  TF_CHECK_OK(runner.SetJitLevel(tensorflow::OptimizerOptions::GlobalJitLevel::
+                                     OptimizerOptions_GlobalJitLevel_ON_2));
+  // Run the actual "main" function.
+  TF_CHECK_OK(runner.RunMain(argc, argv));
+}
diff --git a/tensorflow/tools/optimization/optimization_pass_runner.cc b/tensorflow/tools/optimization/optimization_pass_runner.cc
new file mode 100644
index 0000000000..231ff08381
--- /dev/null
+++ b/tensorflow/tools/optimization/optimization_pass_runner.cc
@@ -0,0 +1,167 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+// This file creates a library that can run any registered optimization pass.
+// The binary that uses this will be run in a form similar to:
+// ./optimization_pass_runner  --input_file_path=/tmp/input.pbtxt
+// --output_file_path=/tmp/output.pbtxt
+// --optimization_pass=NameOfGraphOptimizationPass
+#include "tensorflow/tools/optimization/optimization_pass_runner.h"
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/common_runtime/device_set.h"
+#include "tensorflow/core/common_runtime/optimization_registry.h"
+#include "tensorflow/core/framework/allocator.h"
+#include "tensorflow/core/framework/device_attributes.pb.h"
+#include "tensorflow/core/framework/function.h"
+#include "tensorflow/core/framework/function.pb.h"
+#include "tensorflow/core/framework/graph.pb.h"
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/graph/graph.h"
+#include "tensorflow/core/graph/graph_constructor.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/init_main.h"
+#include "tensorflow/core/protobuf/config.pb.h"
+#include "tensorflow/core/public/session_options.h"
+#include "tensorflow/core/util/command_line_flags.h"
+
+namespace tensorflow {
+
+namespace {
+// A fake device used to populate a DeviceSet.
+class FakeDevice : public Device {
+ private:
+  explicit FakeDevice(const DeviceAttributes& device_attributes)
+      : Device(nullptr, device_attributes) {}
+
+ public:
+  Status Sync() override;
+  static std::unique_ptr<Device> Make(const string& name, const string& type);
+};
+
+Status FakeDevice::Sync() {
+  return errors::Unimplemented("FakeDevice::Sync()");
+}
+
+std::unique_ptr<Device> FakeDevice::Make(const string& name,
+                                         const string& type) {
+  DeviceAttributes device_attributes;
+  device_attributes.set_name(name);
+  device_attributes.set_device_type(DeviceType(type).type());
+  return std::unique_ptr<Device>(new FakeDevice(device_attributes));
+}
+}  // namespace
+
+Status OptimizationPassRunner::RunMain(int argc, char** argv) {
+  string input_file_path;
+  string output_file_path;
+  string optimization_pass;
+
+  const std::vector<Flag> flag_list = {
+      Flag("input_file_path", &input_file_path, "Location of the input graph."),
+      Flag("output_file_path", &output_file_path,
+           "Location to write the resulting graph."),
+      // For now only a single optimization pass can be run.
+      Flag("optimization_pass", &optimization_pass,
+           "Which optimization pass to run."),
+  };
+  if (!Flags::Parse(&argc, argv, flag_list)) {
+    return errors::FailedPrecondition("Invalid flags passed");
+  }
+  port::InitMain(argv[0], &argc, &argv);
+
+  if (input_file_path.empty()) {
+    return errors::FailedPrecondition("input_file_path is a required flag.");
+  }
+  if (output_file_path.empty()) {
+    return errors::FailedPrecondition("output_file_path is a required flag.");
+  }
+  if (optimization_pass.empty()) {
+    return errors::FailedPrecondition("optimization_pass is a required flag.");
+  }
+
+  // Turn on XLA Auto-Jit.
+  auto session_options = absl::make_unique<SessionOptions>();
+  session_options->config.mutable_graph_options()
+      ->mutable_optimizer_options()
+      ->set_global_jit_level(jit_level_);
+  FunctionDefLibrary flib;
+  std::unique_ptr<Graph> graph = absl::make_unique<Graph>(OpRegistry::Global());
+
+  GraphOptimizationPassOptions options;
+  options.session_options = session_options.release();
+  options.graph = &graph;
+  options.flib_def =
+      new FunctionLibraryDefinition((*options.graph)->op_registry(), flib);
+
+  // Grab the data
+  GraphDef graphdef;
+  GraphConstructorOptions graph_opts;
+  graph_opts.expect_device_spec = true;
+  graph_opts.allow_internal_ops = true;
+  TF_RETURN_IF_ERROR(ReadTextProto(Env::Default(), input_file_path, &graphdef));
+  TF_RETURN_IF_ERROR(
+      ConvertGraphDefToGraph(graph_opts, graphdef, options.graph->get()));
+
+  // Add all devices that were previously configured with AddDevice.
+  DeviceSet device_set;
+  for (auto& device : devices_) {
+    device_set.AddDevice(device.get());
+  }
+  options.device_set = &device_set;
+
+  Status result = errors::NotFound(
+      "An OptimizationPass was not found with the desired name.");
+
+  // Run the optimization pass specified by the command line flag.
+  for (const auto& groups_and_passes :
+       OptimizationPassRegistry::Global()->groups()) {
+    for (const auto& phase_and_passes : groups_and_passes.second) {
+      for (const auto& pass : phase_and_passes.second) {
+        if (pass->name() == optimization_pass) {
+          result = pass->Run(options);
+        }
+      }
+    }
+  }
+
+  TF_RETURN_IF_ERROR(result);
+
+  // Write out the result.
+  options.graph->get()->ToGraphDef(&graphdef);
+  TF_RETURN_IF_ERROR(
+      WriteTextProto(Env::Default(), output_file_path, graphdef));
+  return Status::OK();
+}
+
+Status OptimizationPassRunner::SetJitLevel(
+    OptimizerOptions::GlobalJitLevel jit_level) {
+  jit_level_ = jit_level;
+  return Status::OK();
+}
+
+Status OptimizationPassRunner::AddDevice(const string& name,
+                                         const string& type) {
+  devices_.push_back(FakeDevice::Make(name, type));
+  return Status::OK();
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/tools/optimization/optimization_pass_runner.h b/tensorflow/tools/optimization/optimization_pass_runner.h
new file mode 100644
index 0000000000..3b26f64bcf
--- /dev/null
+++ b/tensorflow/tools/optimization/optimization_pass_runner.h
@@ -0,0 +1,61 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_TOOLS_OPTIMIZATION_OPTIMIZATION_PASS_RUNNER_H_
+#define TENSORFLOW_TOOLS_OPTIMIZATION_OPTIMIZATION_PASS_RUNNER_H_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/common_runtime/optimization_registry.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/protobuf/config.pb.h"
+
+namespace tensorflow {
+
+// OptimizationPassRunner can be initialized, populated with devices, then run
+// to test individual Tensorflow Optimization passes.
+class OptimizationPassRunner {
+ public:
+  explicit OptimizationPassRunner()
+      : jit_level_(OptimizerOptions::GlobalJitLevel::
+                       OptimizerOptions_GlobalJitLevel_DEFAULT) {}
+
+  // Add a fake device to the (initially empty) DeviceSet used for optimization.
+  // Names are of the form: "/job:localhost/replica:0/task:0/device:CPU:0"
+  Status AddDevice(const string& name, const string& type);
+
+  // Increasing the Jit level will cause XLA to compile parts of the tensorflow
+  // graph that it is able to.
+  Status SetJitLevel(OptimizerOptions::GlobalJitLevel jit_level);
+
+  // This can be called after adding devices and setting the jit level to parse
+  // command line flags and run the specified job. All 3 flags are required:
+  // input_file_path, output_file_path, optimization_pass.
+  //
+  // If this library becomes heavily used, the caller should be responsible for
+  // parsing any command line flags desired rather than this Method handling the
+  // work of a main() function.
+  Status RunMain(int argc, char** argv);
+
+ private:
+  OptimizerOptions::GlobalJitLevel jit_level_;
+  std::vector<std::unique_ptr<Device>> devices_;
+};
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_TOOLS_OPTIMIZATION_OPTIMIZATION_PASS_RUNNER_H_
-- 
GitLab


From 044b34b2819ec45ffc34c7b59e921ad4d8aaec9a Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <penporn@google.com>
Date: Mon, 7 Jan 2019 11:13:45 -0800
Subject: [PATCH 0265/2345] Disable MKL-DNN contraction kernels in Eigen when
 building with config=mkl to avoid duplicate MKL-DNN symbols. (config=mkl
 links to OpenMP MKL-DNN while tensorflow_mkldnn_contraction_kernel links to
 serial MKL-DNN.)

PiperOrigin-RevId: 228199406
---
 .bazelrc                         |   2 +
 tensorflow/opensource_only.files | 416 +++++++++++++++----------------
 2 files changed, 210 insertions(+), 208 deletions(-)

diff --git a/.bazelrc b/.bazelrc
index cd7e13ddfc..d5432aeb17 100644
--- a/.bazelrc
+++ b/.bazelrc
@@ -25,12 +25,14 @@ build --define framework_shared_object=true
 # If you would like to use a local MKL instead of downloading, please set the
 # environment variable "TF_MKL_ROOT" every time before build.
 build:mkl --define=build_with_mkl=true --define=enable_mkl=true
+build:mkl --define=tensorflow_mkldnn_contraction_kernel=0
 build:mkl -c opt
 
 # This config option is used to enable MKL-DNN open source library only,
 # without depending on MKL binary version.
 build:mkl_open_source_only --define=build_with_mkl_dnn_only=true
 build:mkl_open_source_only --define=build_with_mkl=true --define=enable_mkl=true
+build:mkl_open_source_only --define=tensorflow_mkldnn_contraction_kernel=0
 
 build:download_clang --crosstool_top=@local_config_download_clang//:toolchain
 build:download_clang --define=using_clang=true
diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index 40f87d6c36..e00d063cfe 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -1,248 +1,248 @@
-tensorflow/api_template_v1.__init__.py
-tensorflow/compat_template_v1.__init__.py
-tensorflow/stream_executor/BUILD
-tensorflow/third_party/curl.BUILD
-tensorflow/third_party/snappy.BUILD
-tensorflow/third_party/tflite_mobilenet_float.BUILD
-tensorflow/third_party/libxsmm.BUILD
-tensorflow/third_party/cub.BUILD
-tensorflow/third_party/repo.bzl
-tensorflow/third_party/systemlibs/curl.BUILD
-tensorflow/third_party/systemlibs/snappy.BUILD
-tensorflow/third_party/systemlibs/protobuf.BUILD
-tensorflow/third_party/systemlibs/absl_py.BUILD
-tensorflow/third_party/systemlibs/google_cloud_cpp.google.cloud.bigtable.BUILD
-tensorflow/third_party/systemlibs/astor.BUILD
-tensorflow/third_party/systemlibs/absl_py.absl.flags.BUILD
-tensorflow/third_party/systemlibs/png.BUILD
-tensorflow/third_party/systemlibs/BUILD.tpl
-tensorflow/third_party/systemlibs/boringssl.BUILD
-tensorflow/third_party/systemlibs/absl_py.absl.testing.BUILD
-tensorflow/third_party/systemlibs/swig.BUILD
-tensorflow/third_party/systemlibs/double_conversion.BUILD
-tensorflow/third_party/systemlibs/gif.BUILD
-tensorflow/third_party/systemlibs/cython.BUILD
-tensorflow/third_party/systemlibs/google_cloud_cpp.BUILD
-tensorflow/third_party/systemlibs/zlib.BUILD
-tensorflow/third_party/systemlibs/grpc.BUILD
-tensorflow/third_party/systemlibs/re2.BUILD
-tensorflow/third_party/systemlibs/build_defs.bzl.tpl
-tensorflow/third_party/systemlibs/protobuf.bzl
-tensorflow/third_party/systemlibs/gast.BUILD
-tensorflow/third_party/systemlibs/syslibs_configure.bzl
-tensorflow/third_party/systemlibs/nsync.BUILD
-tensorflow/third_party/systemlibs/jsoncpp.BUILD
-tensorflow/third_party/systemlibs/six.BUILD
-tensorflow/third_party/systemlibs/lmdb.BUILD
-tensorflow/third_party/systemlibs/BUILD
-tensorflow/third_party/systemlibs/googleapis.BUILD
-tensorflow/third_party/systemlibs/sqlite.BUILD
-tensorflow/third_party/systemlibs/termcolor.BUILD
-tensorflow/third_party/systemlibs/pcre.BUILD
-tensorflow/third_party/tflite_mobilenet_quant.BUILD
-tensorflow/third_party/common.bzl
-tensorflow/third_party/boringssl/BUILD
-tensorflow/third_party/kafka/config.patch
-tensorflow/third_party/kafka/BUILD
-tensorflow/third_party/android/android_configure.bzl
-tensorflow/third_party/android/android_configure.BUILD.tpl
-tensorflow/third_party/android/android.bzl.tpl
-tensorflow/third_party/android/BUILD
-tensorflow/third_party/astor.BUILD
-tensorflow/third_party/icu/udata.patch
-tensorflow/third_party/tflite_mobilenet.BUILD
-tensorflow/third_party/png.BUILD
-tensorflow/third_party/mpi/.gitignore
-tensorflow/third_party/mpi/BUILD
-tensorflow/third_party/nanopb.BUILD
-tensorflow/third_party/arm_neon_2_x86_sse.BUILD
-tensorflow/third_party/linenoise.BUILD
-tensorflow/third_party/llvm/llvm.autogenerated.BUILD
-tensorflow/third_party/llvm/llvm.bzl
-tensorflow/third_party/llvm/BUILD
-tensorflow/third_party/llvm/expand_cmake_vars.py
-tensorflow/third_party/swig.BUILD
-tensorflow/third_party/png_fix_rpi.patch
-tensorflow/third_party/double_conversion.BUILD
-tensorflow/third_party/gif.BUILD
-tensorflow/third_party/grpc/BUILD
-tensorflow/third_party/cython.BUILD
-tensorflow/third_party/eigen.BUILD
-tensorflow/third_party/codegen.BUILD
-tensorflow/third_party/zlib.BUILD
-tensorflow/third_party/tensorrt/BUILD.tpl
-tensorflow/third_party/tensorrt/tensorrt_configure.bzl
-tensorflow/third_party/tensorrt/build_defs.bzl.tpl
-tensorflow/third_party/tensorrt/remote.BUILD.tpl
-tensorflow/third_party/tensorrt/LICENSE
-tensorflow/third_party/tensorrt/BUILD
-tensorflow/third_party/tflite_ovic_testdata.BUILD
-tensorflow/third_party/fft2d/fft.h
-tensorflow/third_party/fft2d/fft2d.BUILD
-tensorflow/third_party/fft2d/LICENSE
-tensorflow/third_party/fft2d/BUILD
+tensorflow/contrib/tpu/profiler/pip_package/BUILD
+tensorflow/contrib/tpu/profiler/pip_package/setup.py
+tensorflow/contrib/tpu/profiler/pip_package/README
+tensorflow/contrib/tpu/profiler/pip_package/build_pip_package.sh
+tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/main.py
+tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/__init__.py
+tensorflow/contrib/mpi/BUILD
+tensorflow/tools/ci_build/remote/BUILD
+tensorflow/tools/pip_package/README
+tensorflow/tools/pip_package/MANIFEST.in
+tensorflow/tools/pip_package/simple_console.py
+tensorflow/tools/pip_package/build_pip_package.sh
+tensorflow/tools/pip_package/check_load_py_test.py
+tensorflow/tools/pip_package/pip_smoke_test.py
+tensorflow/tools/pip_package/simple_console_for_windows.py
+tensorflow/tools/pip_package/setup.py
+tensorflow/tools/pip_package/BUILD
+tensorflow/tools/lib_package/concat_licenses.sh
+tensorflow/tools/lib_package/libtensorflow_test.c
+tensorflow/tools/lib_package/LibTensorFlowTest.java
+tensorflow/tools/lib_package/BUILD
+tensorflow/tools/lib_package/libtensorflow_test.sh
+tensorflow/tools/lib_package/README.md
+tensorflow/tools/lib_package/libtensorflow_java_test.sh
+tensorflow/tools/def_file_filter/def_file_filter_configure.bzl
+tensorflow/tools/def_file_filter/BUILD
+tensorflow/tools/def_file_filter/BUILD.tpl
+tensorflow/tools/def_file_filter/def_file_filter.py.tpl
+tensorflow/third_party/mkl/MKL_LICENSE
+tensorflow/third_party/mkl/LICENSE
+tensorflow/third_party/mkl/BUILD
+tensorflow/third_party/mkl/mkl.BUILD
+tensorflow/third_party/mkl/build_defs.bzl
+tensorflow/third_party/backports_weakref.BUILD
+tensorflow/third_party/toolchains/clang6/BUILD
 tensorflow/third_party/toolchains/clang6/README.md
 tensorflow/third_party/toolchains/clang6/repo.bzl
-tensorflow/third_party/toolchains/clang6/clang.BUILD
 tensorflow/third_party/toolchains/clang6/CROSSTOOL.tpl
-tensorflow/third_party/toolchains/clang6/BUILD
-tensorflow/third_party/toolchains/preconfig/generate/archives.bzl
-tensorflow/third_party/toolchains/preconfig/generate/workspace.bzl
-tensorflow/third_party/toolchains/preconfig/generate/generate.bzl
-tensorflow/third_party/toolchains/preconfig/generate/containers.bzl
-tensorflow/third_party/toolchains/preconfig/generate/BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/build_defs.bzl
+tensorflow/third_party/toolchains/clang6/clang.BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/build_defs.bzl
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/py3/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc/BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/build_defs.bzl
-tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/build_defs.bzl
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/build_defs.bzl
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/nccl2/BUILD
+tensorflow/third_party/toolchains/preconfig/generate/workspace.bzl
+tensorflow/third_party/toolchains/preconfig/generate/containers.bzl
+tensorflow/third_party/toolchains/preconfig/generate/generate.bzl
+tensorflow/third_party/toolchains/preconfig/generate/archives.bzl
+tensorflow/third_party/toolchains/preconfig/generate/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu16.04/clang/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu16.04/clang/dummy_toolchain.bzl
+tensorflow/third_party/toolchains/preconfig/ubuntu16.04/py3/BUILD
+tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/BUILD
+tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/dummy_toolchain.bzl
 tensorflow/third_party/toolchains/preconfig/win_1803/py36/BUILD
 tensorflow/third_party/toolchains/preconfig/win_1803/BUILD
-tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/dummy_toolchain.bzl
-tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu16.04/py3/BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu16.04/clang/dummy_toolchain.bzl
-tensorflow/third_party/toolchains/preconfig/ubuntu16.04/clang/BUILD
-tensorflow/third_party/toolchains/cpus/py3/BUILD
-tensorflow/third_party/toolchains/cpus/arm/CROSSTOOL.tpl
-tensorflow/third_party/toolchains/cpus/arm/BUILD
-tensorflow/third_party/toolchains/cpus/arm/arm_compiler_configure.bzl
-tensorflow/third_party/toolchains/cpus/py/BUILD
-tensorflow/third_party/toolchains/gpus/crosstool/CROSSTOOL
-tensorflow/third_party/toolchains/gpus/crosstool/BUILD
 tensorflow/third_party/toolchains/gpus/cuda/build_defs.bzl
-tensorflow/third_party/toolchains/gpus/cuda/cuda/cuda_config.h
 tensorflow/third_party/toolchains/gpus/cuda/BUILD
+tensorflow/third_party/toolchains/gpus/cuda/cuda/cuda_config.h
+tensorflow/third_party/toolchains/gpus/crosstool/BUILD
+tensorflow/third_party/toolchains/gpus/crosstool/CROSSTOOL
 tensorflow/third_party/toolchains/gpus/py/BUILD
+tensorflow/third_party/toolchains/cpus/arm/arm_compiler_configure.bzl
+tensorflow/third_party/toolchains/cpus/arm/CROSSTOOL.tpl
+tensorflow/third_party/toolchains/cpus/arm/BUILD
+tensorflow/third_party/toolchains/cpus/py3/BUILD
+tensorflow/third_party/toolchains/cpus/py/BUILD
 tensorflow/third_party/toolchains/BUILD
-tensorflow/third_party/nccl/archive.BUILD
-tensorflow/third_party/nccl/nccl_configure.bzl
-tensorflow/third_party/nccl/system.BUILD.tpl
-tensorflow/third_party/nccl/build_defs.bzl.tpl
-tensorflow/third_party/nccl/LICENSE
-tensorflow/third_party/nccl/BUILD
-tensorflow/third_party/farmhash.BUILD
-tensorflow/third_party/sycl/crosstool/BUILD
-tensorflow/third_party/backports_weakref.BUILD
-tensorflow/third_party/gast.BUILD
-tensorflow/third_party/clang_toolchain/cc_configure_clang.bzl
-tensorflow/third_party/clang_toolchain/download_clang.bzl
-tensorflow/third_party/clang_toolchain/BUILD
-tensorflow/third_party/mpi_collectives/BUILD
-tensorflow/third_party/jsoncpp.BUILD
-tensorflow/third_party/mkl_dnn/mkldnn.BUILD
-tensorflow/third_party/mkl_dnn/LICENSE
-tensorflow/third_party/python_runtime/BUILD
-tensorflow/third_party/ngraph/ngraph.BUILD
-tensorflow/third_party/ngraph/nlohmann_json.BUILD
-tensorflow/third_party/ngraph/build_defs.bzl
-tensorflow/third_party/ngraph/ngraph_tf.BUILD
-tensorflow/third_party/ngraph/NGRAPH_LICENSE
-tensorflow/third_party/ngraph/tbb.BUILD
-tensorflow/third_party/ngraph/LICENSE
-tensorflow/third_party/ngraph/BUILD
-tensorflow/third_party/gpus/crosstool/windows/msvc_wrapper_for_nvcc.py.tpl
-tensorflow/third_party/gpus/crosstool/CROSSTOOL.tpl
-tensorflow/third_party/gpus/crosstool/BUILD.tpl
-tensorflow/third_party/gpus/crosstool/remote.BUILD.tpl
+tensorflow/third_party/gpus/BUILD
 tensorflow/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_rocm.tpl
 tensorflow/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_is_not_gcc.tpl
+tensorflow/third_party/gpus/crosstool/CROSSTOOL.tpl
 tensorflow/third_party/gpus/crosstool/CROSSTOOL_hipcc.tpl
 tensorflow/third_party/gpus/crosstool/LICENSE
+tensorflow/third_party/gpus/crosstool/remote.BUILD.tpl
+tensorflow/third_party/gpus/crosstool/windows/msvc_wrapper_for_nvcc.py.tpl
+tensorflow/third_party/gpus/crosstool/BUILD.tpl
 tensorflow/third_party/gpus/crosstool/BUILD
-tensorflow/third_party/gpus/rocm/BUILD.tpl
-tensorflow/third_party/gpus/rocm/rocm_config.h.tpl
-tensorflow/third_party/gpus/rocm/build_defs.bzl.tpl
-tensorflow/third_party/gpus/rocm/BUILD
+tensorflow/third_party/gpus/cuda/LICENSE
 tensorflow/third_party/gpus/cuda/BUILD.tpl
-tensorflow/third_party/gpus/cuda/build_defs.bzl.tpl
-tensorflow/third_party/gpus/cuda/remote.BUILD.tpl
 tensorflow/third_party/gpus/cuda/BUILD.windows.tpl
 tensorflow/third_party/gpus/cuda/cuda_config.h.tpl
-tensorflow/third_party/gpus/cuda/LICENSE
+tensorflow/third_party/gpus/cuda/remote.BUILD.tpl
 tensorflow/third_party/gpus/cuda/BUILD
+tensorflow/third_party/gpus/cuda/build_defs.bzl.tpl
+tensorflow/third_party/gpus/rocm/rocm_config.h.tpl
+tensorflow/third_party/gpus/rocm/BUILD
+tensorflow/third_party/gpus/rocm/BUILD.tpl
+tensorflow/third_party/gpus/rocm/build_defs.bzl.tpl
 tensorflow/third_party/gpus/cuda_configure.bzl
 tensorflow/third_party/gpus/rocm_configure.bzl
-tensorflow/third_party/gpus/BUILD
-tensorflow/third_party/py/python_configure.bzl
-tensorflow/third_party/py/BUILD.tpl
-tensorflow/third_party/py/numpy/BUILD
-tensorflow/third_party/py/remote.BUILD.tpl
-tensorflow/third_party/py/BUILD
-tensorflow/third_party/mkl/build_defs.bzl
-tensorflow/third_party/mkl/mkl.BUILD
-tensorflow/third_party/mkl/MKL_LICENSE
-tensorflow/third_party/mkl/LICENSE
-tensorflow/third_party/mkl/BUILD
-tensorflow/third_party/com_google_absl.BUILD
-tensorflow/third_party/six.BUILD
-tensorflow/third_party/lmdb.BUILD
-tensorflow/third_party/BUILD
-tensorflow/third_party/googleapis.BUILD
-tensorflow/third_party/sqlite.BUILD
-tensorflow/third_party/termcolor.BUILD
-tensorflow/third_party/protobuf/BUILD
-tensorflow/third_party/pcre.BUILD
-tensorflow/third_party/git/BUILD.tpl
-tensorflow/third_party/git/git_configure.bzl
-tensorflow/third_party/git/BUILD
-tensorflow/third_party/pprof.BUILD
-tensorflow/third_party/tflite_smartreply.BUILD
-tensorflow/third_party/eigen3/Eigen/Core
+tensorflow/third_party/snappy.BUILD
+tensorflow/third_party/cython.BUILD
+tensorflow/third_party/farmhash.BUILD
 tensorflow/third_party/eigen3/Eigen/Cholesky
-tensorflow/third_party/eigen3/Eigen/Eigenvalues
-tensorflow/third_party/eigen3/Eigen/SVD
-tensorflow/third_party/eigen3/Eigen/LU
 tensorflow/third_party/eigen3/Eigen/QR
-tensorflow/third_party/eigen3/unsupported/Eigen/MatrixFunctions
+tensorflow/third_party/eigen3/Eigen/LU
+tensorflow/third_party/eigen3/Eigen/Core
+tensorflow/third_party/eigen3/Eigen/SVD
+tensorflow/third_party/eigen3/Eigen/Eigenvalues
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/FixedPoint
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/Tensor
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/ThreadPool
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX512.h
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/TypeCastingAVX512.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX2.h
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatMatProduct.h
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatMatProductNEON.h
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX512.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/FixedPointTypes.h
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatVecProduct.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatMatProductNEON.h
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/TypeCastingAVX2.h
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/FixedPointTypes.h
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatMatProductAVX2.h
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX2.h
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/FixedPoint
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/ThreadPool
 tensorflow/third_party/eigen3/unsupported/Eigen/SpecialFunctions
+tensorflow/third_party/eigen3/unsupported/Eigen/MatrixFunctions
 tensorflow/third_party/eigen3/LICENSE
 tensorflow/third_party/eigen3/BUILD
-tensorflow/tools/pip_package/simple_console.py
-tensorflow/tools/pip_package/setup.py
-tensorflow/tools/pip_package/MANIFEST.in
-tensorflow/tools/pip_package/README
-tensorflow/tools/pip_package/check_load_py_test.py
-tensorflow/tools/pip_package/pip_smoke_test.py
-tensorflow/tools/pip_package/BUILD
-tensorflow/tools/pip_package/build_pip_package.sh
-tensorflow/tools/pip_package/simple_console_for_windows.py
-tensorflow/tools/lib_package/LibTensorFlowTest.java
-tensorflow/tools/lib_package/README.md
-tensorflow/tools/lib_package/libtensorflow_test.sh
-tensorflow/tools/lib_package/libtensorflow_java_test.sh
-tensorflow/tools/lib_package/libtensorflow_test.c
-tensorflow/tools/lib_package/concat_licenses.sh
-tensorflow/tools/lib_package/BUILD
-tensorflow/tools/ci_build/remote/BUILD
-tensorflow/tools/def_file_filter/BUILD.tpl
-tensorflow/tools/def_file_filter/def_file_filter_configure.bzl
-tensorflow/tools/def_file_filter/def_file_filter.py.tpl
-tensorflow/tools/def_file_filter/BUILD
-tensorflow/contrib/mpi/BUILD
-tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/main.py
-tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/__init__.py
-tensorflow/contrib/tpu/profiler/pip_package/setup.py
-tensorflow/contrib/tpu/profiler/pip_package/README
-tensorflow/contrib/tpu/profiler/pip_package/BUILD
-tensorflow/contrib/tpu/profiler/pip_package/build_pip_package.sh
+tensorflow/third_party/systemlibs/build_defs.bzl.tpl
+tensorflow/third_party/systemlibs/absl_py.BUILD
+tensorflow/third_party/systemlibs/curl.BUILD
+tensorflow/third_party/systemlibs/termcolor.BUILD
+tensorflow/third_party/systemlibs/absl_py.absl.flags.BUILD
+tensorflow/third_party/systemlibs/grpc.BUILD
+tensorflow/third_party/systemlibs/swig.BUILD
+tensorflow/third_party/systemlibs/protobuf.bzl
+tensorflow/third_party/systemlibs/protobuf.BUILD
+tensorflow/third_party/systemlibs/BUILD
+tensorflow/third_party/systemlibs/google_cloud_cpp.BUILD
+tensorflow/third_party/systemlibs/astor.BUILD
+tensorflow/third_party/systemlibs/six.BUILD
+tensorflow/third_party/systemlibs/absl_py.absl.testing.BUILD
+tensorflow/third_party/systemlibs/boringssl.BUILD
+tensorflow/third_party/systemlibs/nsync.BUILD
+tensorflow/third_party/systemlibs/google_cloud_cpp.google.cloud.bigtable.BUILD
+tensorflow/third_party/systemlibs/gif.BUILD
+tensorflow/third_party/systemlibs/pcre.BUILD
+tensorflow/third_party/systemlibs/BUILD.tpl
+tensorflow/third_party/systemlibs/snappy.BUILD
+tensorflow/third_party/systemlibs/gast.BUILD
+tensorflow/third_party/systemlibs/cython.BUILD
+tensorflow/third_party/systemlibs/double_conversion.BUILD
+tensorflow/third_party/systemlibs/zlib.BUILD
+tensorflow/third_party/systemlibs/jsoncpp.BUILD
+tensorflow/third_party/systemlibs/re2.BUILD
+tensorflow/third_party/systemlibs/lmdb.BUILD
+tensorflow/third_party/systemlibs/googleapis.BUILD
+tensorflow/third_party/systemlibs/png.BUILD
+tensorflow/third_party/systemlibs/syslibs_configure.bzl
+tensorflow/third_party/systemlibs/sqlite.BUILD
+tensorflow/third_party/python_runtime/BUILD
+tensorflow/third_party/sycl/crosstool/BUILD
+tensorflow/third_party/ngraph/LICENSE
+tensorflow/third_party/ngraph/tbb.BUILD
+tensorflow/third_party/ngraph/BUILD
+tensorflow/third_party/ngraph/ngraph.BUILD
+tensorflow/third_party/ngraph/build_defs.bzl
+tensorflow/third_party/ngraph/NGRAPH_LICENSE
+tensorflow/third_party/ngraph/ngraph_tf.BUILD
+tensorflow/third_party/ngraph/nlohmann_json.BUILD
+tensorflow/third_party/clang_toolchain/download_clang.bzl
+tensorflow/third_party/clang_toolchain/BUILD
+tensorflow/third_party/clang_toolchain/cc_configure_clang.bzl
+tensorflow/third_party/gast.BUILD
+tensorflow/third_party/llvm/BUILD
+tensorflow/third_party/llvm/expand_cmake_vars.py
+tensorflow/third_party/llvm/llvm.autogenerated.BUILD
+tensorflow/third_party/llvm/llvm.bzl
+tensorflow/third_party/icu/udata.patch
+tensorflow/third_party/nccl/archive.BUILD
+tensorflow/third_party/nccl/LICENSE
+tensorflow/third_party/nccl/system.BUILD.tpl
+tensorflow/third_party/nccl/nccl_configure.bzl
+tensorflow/third_party/nccl/build_defs.bzl.tpl
+tensorflow/third_party/nccl/BUILD
+tensorflow/third_party/fft2d/BUILD
+tensorflow/third_party/fft2d/fft.h
+tensorflow/third_party/fft2d/LICENSE
+tensorflow/third_party/fft2d/fft2d.BUILD
+tensorflow/third_party/boringssl/BUILD
+tensorflow/third_party/mpi/.gitignore
+tensorflow/third_party/mpi/BUILD
+tensorflow/third_party/tensorrt/LICENSE
+tensorflow/third_party/tensorrt/BUILD
+tensorflow/third_party/tensorrt/build_defs.bzl.tpl
+tensorflow/third_party/tensorrt/BUILD.tpl
+tensorflow/third_party/tensorrt/tensorrt_configure.bzl
+tensorflow/third_party/tensorrt/remote.BUILD.tpl
+tensorflow/third_party/kafka/config.patch
+tensorflow/third_party/kafka/BUILD
+tensorflow/third_party/android/BUILD
+tensorflow/third_party/android/android.bzl.tpl
+tensorflow/third_party/android/android_configure.bzl
+tensorflow/third_party/android/android_configure.BUILD.tpl
+tensorflow/third_party/tflite_smartreply.BUILD
+tensorflow/third_party/mkl_dnn/LICENSE
+tensorflow/third_party/mkl_dnn/mkldnn.BUILD
+tensorflow/third_party/pcre.BUILD
+tensorflow/third_party/linenoise.BUILD
+tensorflow/third_party/sqlite.BUILD
+tensorflow/third_party/common.bzl
+tensorflow/third_party/com_google_absl.BUILD
+tensorflow/third_party/pprof.BUILD
+tensorflow/third_party/BUILD
+tensorflow/third_party/tflite_mobilenet_quant.BUILD
+tensorflow/third_party/lmdb.BUILD
+tensorflow/third_party/git/BUILD.tpl
+tensorflow/third_party/git/BUILD
+tensorflow/third_party/git/git_configure.bzl
+tensorflow/third_party/protobuf/BUILD
+tensorflow/third_party/tflite_mobilenet.BUILD
+tensorflow/third_party/py/BUILD
+tensorflow/third_party/py/BUILD.tpl
+tensorflow/third_party/py/remote.BUILD.tpl
+tensorflow/third_party/py/numpy/BUILD
+tensorflow/third_party/py/python_configure.bzl
+tensorflow/third_party/termcolor.BUILD
+tensorflow/third_party/png_fix_rpi.patch
+tensorflow/third_party/swig.BUILD
+tensorflow/third_party/astor.BUILD
+tensorflow/third_party/grpc/BUILD
+tensorflow/third_party/curl.BUILD
+tensorflow/third_party/arm_neon_2_x86_sse.BUILD
+tensorflow/third_party/png.BUILD
+tensorflow/third_party/googleapis.BUILD
+tensorflow/third_party/mpi_collectives/BUILD
+tensorflow/third_party/nanopb.BUILD
+tensorflow/third_party/gif.BUILD
+tensorflow/third_party/double_conversion.BUILD
+tensorflow/third_party/six.BUILD
+tensorflow/third_party/tflite_mobilenet_float.BUILD
+tensorflow/third_party/repo.bzl
+tensorflow/third_party/codegen.BUILD
+tensorflow/third_party/cub.BUILD
+tensorflow/third_party/jsoncpp.BUILD
+tensorflow/third_party/tflite_ovic_testdata.BUILD
+tensorflow/third_party/libxsmm.BUILD
+tensorflow/third_party/zlib.BUILD
+tensorflow/third_party/eigen.BUILD
+tensorflow/stream_executor/BUILD
+tensorflow/api_template_v1.__init__.py
+tensorflow/compat_template_v1.__init__.py
 tensorflow/api_template.__init__.py
 tensorflow/__init__.py
\ No newline at end of file
-- 
GitLab


From 68340a644560241ca7098bb2d358a3cb6fb42a4d Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Mon, 7 Jan 2019 11:20:59 -0800
Subject: [PATCH 0266/2345] Move the fit-scope check to only error our when
 using numpy

PiperOrigin-RevId: 228200832
---
 .../python/keras/engine/training_distributed.py      | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 9f63f7bada..92a46e399a 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -53,14 +53,14 @@ def fit_distributed(model,
                     steps_per_epoch=None,
                     validation_steps=None):
   """Fit loop for Distribution Strategies."""
-  # TODO(b/122314600): Remove the scope validate.
-  distributed_training_utils.validate_not_in_strategy_scope()
   distributed_training_utils.validate_callbacks(callbacks, model.optimizer)
   distributed_training_utils.validate_inputs(
       x, y, model._distribution_strategy)
 
   first_x_value = nest.flatten(x)[0]
   if isinstance(first_x_value, np.ndarray):
+    # TODO(b/122314600): Remove the scope validate.
+    distributed_training_utils.validate_not_in_strategy_scope()
     steps_per_epoch, batch_size = (
         distributed_training_utils.get_input_params(
             model._distribution_strategy, first_x_value, steps_per_epoch,
@@ -138,11 +138,11 @@ def evaluate_distributed(model,
                          steps=None,
                          callbacks=None):
   """Evaluate loop for Distribution Strategies."""
-  # TODO(b/122314600): Remove the scope validate.
-  distributed_training_utils.validate_not_in_strategy_scope()
   distributed_training_utils.validate_inputs(x, y, model._distribution_strategy)
   first_x_value = nest.flatten(x)[0]
   if isinstance(first_x_value, np.ndarray):
+    # TODO(b/122314600): Remove the scope validate.
+    distributed_training_utils.validate_not_in_strategy_scope()
     steps, batch_size = distributed_training_utils.get_input_params(
         model._distribution_strategy, first_x_value, steps, batch_size)
   batch_size = model._validate_or_infer_batch_size(batch_size, steps, x)
@@ -175,12 +175,12 @@ def predict_distributed(model,
                         steps=None,
                         callbacks=None):
   """Predict loop for Distribution Strategies."""
-  # TODO(b/122314600): Remove the scope validate.
-  distributed_training_utils.validate_not_in_strategy_scope()
   distributed_training_utils.validate_inputs(
       x, None, model._distribution_strategy)
   first_x_value = nest.flatten(x)[0]
   if isinstance(first_x_value, np.ndarray):
+    # TODO(b/122314600): Remove the scope validate.
+    distributed_training_utils.validate_not_in_strategy_scope()
     steps, batch_size = distributed_training_utils.get_input_params(
         model._distribution_strategy, first_x_value, steps, batch_size)
   batch_size = model._validate_or_infer_batch_size(batch_size, steps, x)
-- 
GitLab


From 2c0a8c16477999cc791705112f501d4530649938 Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Mon, 7 Jan 2019 11:29:07 -0800
Subject: [PATCH 0267/2345] Add Unique Op implementation

PiperOrigin-RevId: 228202673
---
 tensorflow/lite/build_def.bzl                 |   1 +
 tensorflow/lite/kernels/BUILD                 |  12 ++
 tensorflow/lite/kernels/register.cc           |   2 +
 tensorflow/lite/kernels/unique.cc             | 164 ++++++++++++++++++
 tensorflow/lite/kernels/unique_test.cc        | 103 +++++++++++
 tensorflow/lite/testing/generate_examples.py  |  49 ++++++
 .../propagate_array_data_types.cc             |   8 +
 .../propagate_fixed_sizes.cc                  |  17 ++
 tensorflow/lite/toco/import_tensorflow.cc     | 113 ++++++------
 tensorflow/lite/toco/model.h                  |  14 +-
 tensorflow/lite/toco/tflite/operator.cc       |  28 +++
 tensorflow/lite/toco/tflite/operator_test.cc  |   9 +
 tensorflow/lite/toco/tooling_util.cc          |   1 +
 13 files changed, 467 insertions(+), 54 deletions(-)
 create mode 100644 tensorflow/lite/kernels/unique.cc
 create mode 100644 tensorflow/lite/kernels/unique_test.cc

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index c17eddf47b..343ec60f2a 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -311,6 +311,7 @@ def generated_test_models():
         "topk",
         "transpose",
         "transpose_conv",
+        "unique",
         "unpack",
         "unroll_batch_matmul",
         "where",
diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index 9abc33d297..6809a4a3d0 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -221,6 +221,7 @@ cc_library(
         "transpose_conv.cc",
         "unidirectional_sequence_lstm.cc",
         "unidirectional_sequence_rnn.cc",
+        "unique.cc",
         "unpack.cc",
         "zeros_like.cc",
     ],
@@ -1233,6 +1234,17 @@ tf_cc_test(
     ],
 )
 
+tf_cc_test(
+    name = "unique_test",
+    srcs = ["unique_test.cc"],
+    deps = [
+        ":builtin_ops",
+        ":test_util",
+        "//tensorflow/lite:framework",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
+
 filegroup(
     name = "all_files",
     srcs = glob(
diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index 47296fa105..f17f39fc2b 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -129,6 +129,7 @@ TfLiteRegistration* Register_LEAKY_RELU();
 TfLiteRegistration* Register_SQUARED_DIFFERENCE();
 TfLiteRegistration* Register_FILL();
 TfLiteRegistration* Register_MIRROR_PAD();
+TfLiteRegistration* Register_UNIQUE();
 
 TfLiteStatus UnsupportedTensorFlowOp(TfLiteContext* context, TfLiteNode* node) {
   context->ReportError(
@@ -284,6 +285,7 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_SQUARED_DIFFERENCE, Register_SQUARED_DIFFERENCE());
   AddBuiltin(BuiltinOperator_FILL, Register_FILL());
   AddBuiltin(BuiltinOperator_MIRROR_PAD, Register_MIRROR_PAD());
+  AddBuiltin(BuiltinOperator_UNIQUE, Register_UNIQUE());
 
   // TODO(andrewharp, ahentz): Move these somewhere more appropriate so that
   // custom ops aren't always included by default.
diff --git a/tensorflow/lite/kernels/unique.cc b/tensorflow/lite/kernels/unique.cc
new file mode 100644
index 0000000000..80c033aa5c
--- /dev/null
+++ b/tensorflow/lite/kernels/unique.cc
@@ -0,0 +1,164 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <map>
+
+#include "tensorflow/lite/c/builtin_op_data.h"
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/kernels/internal/reference/reference_ops.h"
+#include "tensorflow/lite/kernels/internal/tensor.h"
+#include "tensorflow/lite/kernels/kernel_util.h"
+
+namespace tflite {
+namespace ops {
+namespace builtin {
+namespace unique {
+
+void* Init(TfLiteContext* context, const char* buffer, size_t length) {
+  return nullptr;
+}
+
+void Free(TfLiteContext* context, void* buffer) {}
+
+TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
+  static const int kOutputUniqueTensor = 0;
+  static const int kOutputIndexTensor = 1;
+
+  TF_LITE_ENSURE_EQ(context, NumInputs(node), 1);
+  TF_LITE_ENSURE_EQ(context, NumOutputs(node), 2);
+  const TfLiteTensor* input = GetInput(context, node, 0);
+  TfLiteTensor* output_unique_tensor =
+      GetOutput(context, node, kOutputUniqueTensor);
+  TfLiteTensor* output_index_tensor =
+      GetOutput(context, node, kOutputIndexTensor);
+
+  // The op only supports 1D input.
+  TF_LITE_ENSURE_EQ(context, NumDimensions(input), 1);
+  TfLiteIntArray* output_index_shape = TfLiteIntArrayCopy(input->dims);
+  // The unique values are determined during evaluation, so we don't know yet
+  // the size of the output tensor.
+  SetTensorToDynamic(output_unique_tensor);
+  return context->ResizeTensor(context, output_index_tensor,
+                               output_index_shape);
+}
+
+namespace {
+
+// Actual evaluation for the unique op.
+template <typename T, typename I>
+TfLiteStatus EvalImpl(TfLiteContext* context, const TfLiteTensor* input,
+                      TfLiteNode* node) {
+  // Map from value, to index in the unique elements vector.
+  // Note that we prefer to use map than unordered_map as it showed less
+  // increase in the binary size.
+  std::map<T, int> unique_values;
+  TfLiteTensor* output_indexes = GetOutput(context, node, 1);
+  I* indexes = GetTensorData<I>(output_indexes);
+  const T* data = GetTensorData<T>(input);
+  const int num_elements = NumElements(input);
+
+  for (int i = 0; i < num_elements; ++i) {
+    const auto element_it = unique_values.find(data[i]);
+    if (element_it != unique_values.end()) {
+      indexes[i] = element_it->second;
+    } else {
+      const int unique_index = unique_values.size();
+      unique_values[data[i]] = unique_index;
+      indexes[i] = unique_index;
+    }
+  }
+  // Allocate output tensor.
+  TfLiteTensor* unique_output = GetOutput(context, node, 0);
+  std::unique_ptr<TfLiteIntArray, void (*)(TfLiteIntArray*)> shape(
+      TfLiteIntArrayCreate(NumDimensions(input)), TfLiteIntArrayFree);
+  shape->data[0] = unique_values.size();
+  TF_LITE_ENSURE_STATUS(
+      context->ResizeTensor(context, unique_output, shape.release()));
+  // Set the values in the output tensor.
+  T* output_unique_values = GetTensorData<T>(unique_output);
+  for (int i = 0; i < unique_values.size(); ++i) {
+    output_unique_values[i] = data[indexes[i]];
+  }
+  return kTfLiteOk;
+}
+
+template <typename T>
+TfLiteStatus EvalImpl(TfLiteContext* context, const TfLiteTensor* input,
+                      TfLiteNode* node) {
+  auto* params = reinterpret_cast<TfLiteUniqueParams*>(node->builtin_data);
+  if (params == nullptr) {
+    context->ReportError(context, "Null params passed");
+    return kTfLiteError;
+  }
+  switch (params->index_out_type) {
+    case kTfLiteInt32:
+      return EvalImpl<T, int32_t>(context, input, node);
+    case kTfLiteInt64:
+      return EvalImpl<T, int64_t>(context, input, node);
+    default:
+      context->ReportError(
+          context,
+          "Unique index output array can only be Int32 or In64, requested: ",
+          TfLiteTypeGetName(params->index_out_type));
+  }
+  return kTfLiteError;
+}
+
+}  // namespace
+
+TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
+  const TfLiteTensor* input = GetInput(context, node, 0);
+  TfLiteTensor* output_index_tensor = GetOutput(context, node, 1);
+  TF_LITE_ENSURE_EQ(context, NumElements(output_index_tensor),
+                    NumElements(input));
+
+  switch (input->type) {
+    case kTfLiteInt8:
+      TF_LITE_ENSURE_STATUS(EvalImpl<int8_t>(context, input, node));
+      break;
+    case kTfLiteInt16:
+      TF_LITE_ENSURE_STATUS(EvalImpl<int16_t>(context, input, node));
+      break;
+    case kTfLiteInt32:
+      TF_LITE_ENSURE_STATUS(EvalImpl<int32_t>(context, input, node));
+      break;
+    case kTfLiteInt64:
+      TF_LITE_ENSURE_STATUS(EvalImpl<int64_t>(context, input, node));
+      break;
+    case kTfLiteFloat32:
+      TF_LITE_ENSURE_STATUS(EvalImpl<float>(context, input, node));
+      break;
+    case kTfLiteUInt8:
+      TF_LITE_ENSURE_STATUS(EvalImpl<uint8_t>(context, input, node));
+      break;
+    default:
+      context->ReportError(context, "Currently Unique doesn't support type: %s",
+                           TfLiteTypeGetName(input->type));
+      return kTfLiteError;
+  }
+  return kTfLiteOk;
+}
+
+}  // namespace unique
+
+TfLiteRegistration* Register_UNIQUE() {
+  static TfLiteRegistration r = {unique::Init, unique::Free, unique::Prepare,
+                                 unique::Eval};
+  return &r;
+}
+
+}  // namespace builtin
+}  // namespace ops
+}  // namespace tflite
diff --git a/tensorflow/lite/kernels/unique_test.cc b/tensorflow/lite/kernels/unique_test.cc
new file mode 100644
index 0000000000..1df5e6b796
--- /dev/null
+++ b/tensorflow/lite/kernels/unique_test.cc
@@ -0,0 +1,103 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <gtest/gtest.h>
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/kernels/test_util.h"
+#include "tensorflow/lite/model.h"
+
+namespace tflite {
+namespace {
+
+using ::testing::ElementsAreArray;
+
+template <typename T, typename I>
+class UniqueOpModel : public SingleOpModel {
+ public:
+  UniqueOpModel(const TensorData& input, TensorType input_type,
+                TensorType index_out_type) {
+    input_id_ = AddInput(input);
+    output_id_ = AddOutput(input_type);
+    output_index_id_ = AddOutput(index_out_type);
+    SetBuiltinOp(BuiltinOperator_UNIQUE, BuiltinOptions_UniqueOptions,
+                 CreateUniqueOptions(builder_, index_out_type).Union());
+    BuildInterpreter({GetShape(input_id_)});
+  }
+
+  int input_tensor_id() { return input_id_; }
+
+  std::vector<T> GetOutput() { return ExtractVector<T>(output_id_); }
+  std::vector<I> GetIndexesOutput() {
+    return ExtractVector<I>(output_index_id_);
+  }
+
+ protected:
+  int input_id_;
+  int output_id_;
+  int output_index_id_;
+};
+
+TEST(UniqueOpModelTest, OneElement) {
+  UniqueOpModel<float, int32_t> model({TensorType_FLOAT32, {1}},
+                                      TensorType_FLOAT32, TensorType_INT32);
+  model.PopulateTensor<float>(model.input_tensor_id(), {5});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({5}));
+  EXPECT_THAT(model.GetIndexesOutput(), ElementsAreArray({0}));
+}
+
+TEST(UniqueOpModelTest, MultipleElements_AllUnique) {
+  UniqueOpModel<float, int32_t> model({TensorType_FLOAT32, {8}},
+                                      TensorType_FLOAT32, TensorType_INT32);
+  model.PopulateTensor<float>(model.input_tensor_id(),
+                              {5, 2, 3, 51, 6, 72, 7, 8});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({5, 2, 3, 51, 6, 72, 7, 8}));
+  EXPECT_THAT(model.GetIndexesOutput(),
+              ElementsAreArray({0, 1, 2, 3, 4, 5, 6, 7}));
+}
+
+TEST(UniqueOpModelTest, MultipleElements_AllDuplicates) {
+  UniqueOpModel<float, int32_t> model({TensorType_FLOAT32, {7}},
+                                      TensorType_FLOAT32, TensorType_INT32);
+  model.PopulateTensor<float>(model.input_tensor_id(), {5, 5, 5, 5, 5, 5, 5});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({5}));
+  EXPECT_THAT(model.GetIndexesOutput(),
+              ElementsAreArray({0, 0, 0, 0, 0, 0, 0}));
+}
+
+TEST(UniqueOpModelTest, MultipleElements_SomeDuplicates) {
+  UniqueOpModel<float, int32_t> model({TensorType_FLOAT32, {7}},
+                                      TensorType_FLOAT32, TensorType_INT32);
+  model.PopulateTensor<float>(model.input_tensor_id(), {2, 3, 5, 7, 2, 7, 3});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({2, 3, 5, 7}));
+  EXPECT_THAT(model.GetIndexesOutput(),
+              ElementsAreArray({0, 1, 2, 3, 0, 3, 1}));
+}
+
+TEST(UniqueOpModelTest, MultipleElements_SomeDuplicates_IndexInt64) {
+  UniqueOpModel<float, int64_t> model({TensorType_FLOAT32, {7}},
+                                      TensorType_FLOAT32, TensorType_INT64);
+  model.PopulateTensor<float>(model.input_tensor_id(), {2, 3, 5, 7, 2, 7, 3});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({2, 3, 5, 7}));
+  EXPECT_THAT(model.GetIndexesOutput(),
+              ElementsAreArray({0, 1, 2, 3, 0, 3, 1}));
+}
+
+}  // namespace
+}  // namespace tflite
diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index dd7b3d0745..87e7d7eb02 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -3749,6 +3749,55 @@ def make_placeholder_with_default_tests(zip_path):
                     expected_tf_success=3)
 
 
+def make_unique_tests(zip_path):
+  """Make a set of tests for Unique op."""
+
+  test_parameters = [
+      {
+          "input_shape": [[1]],
+          "index_type": [tf.int32, tf.int64, None],
+          "input_values": [3]
+      },
+      {
+          "input_shape": [[5]],
+          "index_type": [tf.int32, tf.int64],
+          "input_values": [[3, 2, 1, 2, 3]]
+      },
+      {
+          "input_shape": [[7]],
+          "index_type": [tf.int32, tf.int64],
+          "input_values": [[1, 1, 1, 1, 1, 1, 1]]
+      },
+      {
+          "input_shape": [[5]],
+          "index_type": [tf.int32, tf.int64],
+          "input_values": [[3, 2, 1, 0, -1]]
+      }]
+
+  def build_graph(parameters):
+    """Build the graph for the test case."""
+
+    input_tensor = tf.placeholder(
+        dtype=tf.int32, name="input", shape=parameters["input_shape"])
+    if parameters["index_type"] is None:
+      output = tf.unique(input_tensor)
+    else:
+      output = tf.unique(input_tensor, parameters["index_type"])
+
+    return [input_tensor], output
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_values = [create_tensor_data(tf.int32, parameters["input_shape"])]
+    return input_values, sess.run(
+        outputs, feed_dict=dict(zip(inputs, input_values)))
+
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_success=9)
+
 # Toco binary path provided by the generate rule.
 bin_path = None
 
diff --git a/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc b/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
index cbae6610d7..6d9aad66b6 100644
--- a/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
+++ b/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
@@ -252,6 +252,14 @@ void SetDataTypeForAllOutputs(Model* model, Operator* op,
       SetDataTypeForAllOutputs(model, op, data_type);
       break;
     }
+    case OperatorType::kUnique: {
+      CHECK_EQ(op->outputs.size(), 2);
+      const UniqueOperator* unique_op = static_cast<UniqueOperator*>(op);
+      const ArrayDataType data_type = model->GetArray(op->inputs[0]).data_type;
+      model->GetArray(op->outputs[0]).data_type = data_type;
+      model->GetArray(op->outputs[1]).data_type = unique_op->idx_out_type;
+      break;
+    }
     default: {
       // These operators produce outputs with the same type as their 1st input
       CHECK_GT(op->inputs.size(), 0);
diff --git a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
index 0e653f08a0..5185afd22e 100644
--- a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
+++ b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
@@ -1828,6 +1828,20 @@ void ProcessMirrorPadOperator(Model* model, MirrorPadOperator* op) {
   output_array.copy_shape(output_shape);
 }
 
+void ProcessUniqueOperator(Model* model, UniqueOperator* op) {
+  const auto& input_array = model->GetArray(op->inputs[0]);
+  // We have 2 outputs, the shape of the index tensor, is the same size
+  // as the input array. The unique values tensor, is unknown until runtime.
+  CHECK_EQ(op->outputs.size(), 2);
+  auto& idx_output_array = model->GetArray(op->outputs[1]);
+
+  // Yield until input dims have been resolved, or output already computed
+  if (!input_array.has_shape() || idx_output_array.has_shape()) {
+    return;
+  }
+  idx_output_array.copy_shape(input_array.shape());
+}
+
 }  // namespace
 
 ::tensorflow::Status PropagateFixedSizes::Run(Model* model,
@@ -2103,6 +2117,9 @@ void ProcessMirrorPadOperator(Model* model, MirrorPadOperator* op) {
     case OperatorType::kMirrorPad:
       ProcessMirrorPadOperator(model, static_cast<MirrorPadOperator*>(op));
       break;
+    case OperatorType::kUnique:
+      ProcessUniqueOperator(model, static_cast<UniqueOperator*>(op));
+      break;
     default:
       // Unimplemented, another graph transformation should drop it.
       LOG(FATAL) << "Unhandled operator type " << OperatorTypeName(op->type);
diff --git a/tensorflow/lite/toco/import_tensorflow.cc b/tensorflow/lite/toco/import_tensorflow.cc
index e1f7eb82ee..86e04b2393 100644
--- a/tensorflow/lite/toco/import_tensorflow.cc
+++ b/tensorflow/lite/toco/import_tensorflow.cc
@@ -1190,7 +1190,7 @@ enum FlexSupport { kFlexOk, kFlexNotOk };
 // taken from the given NodeDef, and its number must match NumInputs, unless
 // kAnyNumInputs is passed in. If kFlexOk is passed in the resulting operator
 // will be eligible for being exported as a flex op.
-template <typename Op, int NumInputs, FlexSupport flex>
+template <typename Op, int NumInputs, int NumOutputs, FlexSupport flex>
 tensorflow::Status ConvertSimpleOperatorGeneric(
     const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
     Model* model) {
@@ -1203,6 +1203,11 @@ tensorflow::Status ConvertSimpleOperatorGeneric(
     op->inputs.push_back(node.input(i));
   }
   op->outputs.push_back(node.name());
+  if (NumOutputs > 1) {
+    for (int i = 1; i < NumOutputs; ++i) {
+      op->outputs.push_back(node.name() + ":" + std::to_string(i));
+    }
+  }
 
   if (flex == kFlexOk) {
     RetainTensorFlowNodeDef(node, op);
@@ -1213,20 +1218,20 @@ tensorflow::Status ConvertSimpleOperatorGeneric(
 }
 
 // Convert a simple operator which is not valid as a flex op.
-template <typename Op, int NumInputs = kAnyNumInputs>
+template <typename Op, int NumInputs, int NumOutputs>
 tensorflow::Status ConvertSimpleOperator(
     const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
     Model* model) {
-  return ConvertSimpleOperatorGeneric<Op, NumInputs, kFlexNotOk>(
+  return ConvertSimpleOperatorGeneric<Op, NumInputs, NumOutputs, kFlexNotOk>(
       node, tf_import_flags, model);
 }
 
 // Convert a simple operator which is valid as a flex op.
-template <typename Op, int NumInputs = kAnyNumInputs>
+template <typename Op, int NumInputs, int NumOutputs>
 tensorflow::Status ConvertSimpleOperatorFlexOk(
     const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
     Model* model) {
-  return ConvertSimpleOperatorGeneric<Op, NumInputs, kFlexOk>(
+  return ConvertSimpleOperatorGeneric<Op, NumInputs, NumOutputs, kFlexOk>(
       node, tf_import_flags, model);
 }
 
@@ -2333,14 +2338,15 @@ ConverterMapType GetTensorFlowNodeConverterMapForFlex() {
 
 ConverterMapType GetTensorFlowNodeConverterMap() {
   return std::unordered_map<std::string, ConverterType>({
-      {"Abs", ConvertSimpleOperator<AbsOperator>},
-      {"Add", ConvertSimpleOperator<AddOperator, 2>},
-      {"AddN", ConvertSimpleOperatorFlexOk<AddNOperator>},
-      {"All", ConvertSimpleOperator<TensorFlowAllOperator>},
+      {"Abs", ConvertSimpleOperator<AbsOperator, kAnyNumInputs, 1>},
+      {"Add", ConvertSimpleOperator<AddOperator, 2, 1>},
+      {"AddN", ConvertSimpleOperatorFlexOk<AddNOperator, kAnyNumInputs, 1>},
+      {"All", ConvertSimpleOperator<TensorFlowAllOperator, kAnyNumInputs, 1>},
       {"Any", ConvertReduceOperator<TensorFlowAnyOperator>},
       {"ArgMax", ConvertArgMaxOperator},
       {"ArgMin", ConvertArgMinOperator},
-      {"Assert", ConvertSimpleOperator<TensorFlowAssertOperator>},
+      {"Assert",
+       ConvertSimpleOperator<TensorFlowAssertOperator, kAnyNumInputs, 1>},
       {"AvgPool", ConvertAvgPoolOperator},
       {"BatchMatMul", ConvertBatchMatMulOperator},
       {"BatchNormWithGlobalNormalization",
@@ -2357,98 +2363,99 @@ ConverterMapType GetTensorFlowNodeConverterMap() {
       {"CTCBeamSearchDecoder", ConvertCTCBeamSearchDecoderOperator},
       {"DepthToSpace", ConvertDepthToSpaceOperator},
       {"DepthwiseConv2dNative", ConvertDepthwiseConvOperator},
-      {"Div", ConvertSimpleOperator<DivOperator, 2>},
+      {"Div", ConvertSimpleOperator<DivOperator, 2, 1>},
       {"DynamicPartition", ConvertDynamicPartitionOperator},
       {"DynamicStitch", ConvertDynamicStitchOperator},
-      {"Equal", ConvertSimpleOperator<TensorFlowEqualOperator, 2>},
-      {"Exp", ConvertSimpleOperator<ExpOperator, 1>},
-      {"ExpandDims", ConvertSimpleOperator<ExpandDimsOperator, 2>},
+      {"Equal", ConvertSimpleOperator<TensorFlowEqualOperator, 2, 1>},
+      {"Exp", ConvertSimpleOperator<ExpOperator, 1, 1>},
+      {"ExpandDims", ConvertSimpleOperator<ExpandDimsOperator, 2, 1>},
       {"FakeQuantWithMinMaxArgs", ConvertFakeQuantWithMinMaxArgs},
       {"FakeQuantWithMinMaxVars", ConvertFakeQuantWithMinMaxVars},
-      {"Fill", ConvertSimpleOperator<FillOperator, 2>},
+      {"Fill", ConvertSimpleOperator<FillOperator, 2, 1>},
       {"Floor", ConvertFloorOperator},
-      {"FloorDiv", ConvertSimpleOperator<FloorDivOperator, 2>},
-      {"FloorMod", ConvertSimpleOperator<FloorModOperator, 2>},
+      {"FloorDiv", ConvertSimpleOperator<FloorDivOperator, 2, 1>},
+      {"FloorMod", ConvertSimpleOperator<FloorModOperator, 2, 1>},
       {"FusedBatchNorm", ConvertFusedBatchNormOperator},
       {"Gather", ConvertGatherOperator},
       {"GatherV2", ConvertGatherOperator},
-      {"Greater", ConvertSimpleOperator<TensorFlowGreaterOperator, 2>},
+      {"Greater", ConvertSimpleOperator<TensorFlowGreaterOperator, 2, 1>},
       {"GreaterEqual",
-       ConvertSimpleOperator<TensorFlowGreaterEqualOperator, 2>},
+       ConvertSimpleOperator<TensorFlowGreaterEqualOperator, 2, 1>},
       {"Identity", ConvertIdentityOperator},
       {"LRN", ConvertLRNOperator},
       {"LeakyRelu", ConvertLeakyReluOperator},
       {"LegacyFedInput", ConvertPlaceholderOperator},
-      {"Less", ConvertSimpleOperator<TensorFlowLessOperator, 2>},
-      {"LessEqual", ConvertSimpleOperator<TensorFlowLessEqualOperator, 2>},
-      {"Log", ConvertSimpleOperator<LogOperator, 1>},
-      {"LogicalAnd", ConvertSimpleOperator<LogicalAndOperator, 2>},
-      {"LogicalOr", ConvertSimpleOperator<LogicalOrOperator, 2>},
-      {"LogicalNot", ConvertSimpleOperator<LogicalNotOperator, 1>},
-      {"LogSoftmax", ConvertSimpleOperator<LogSoftmaxOperator, 1>},
+      {"Less", ConvertSimpleOperator<TensorFlowLessOperator, 2, 1>},
+      {"LessEqual", ConvertSimpleOperator<TensorFlowLessEqualOperator, 2, 1>},
+      {"Log", ConvertSimpleOperator<LogOperator, 1, 1>},
+      {"LogicalAnd", ConvertSimpleOperator<LogicalAndOperator, 2, 1>},
+      {"LogicalOr", ConvertSimpleOperator<LogicalOrOperator, 2, 1>},
+      {"LogicalNot", ConvertSimpleOperator<LogicalNotOperator, 1, 1>},
+      {"LogSoftmax", ConvertSimpleOperator<LogSoftmaxOperator, 1, 1>},
       {"MatMul", ConvertMatMulOperator},
       {"Max", ConvertReduceOperator<TensorFlowMaxOperator>},
       {"MaxPool", ConvertMaxPoolOperator},
-      {"Maximum", ConvertSimpleOperator<TensorFlowMaximumOperator, 2>},
+      {"Maximum", ConvertSimpleOperator<TensorFlowMaximumOperator, 2, 1>},
       {"Mean", ConvertReduceOperator<MeanOperator>},
-      {"Merge", ConvertSimpleOperator<TensorFlowMergeOperator, 2>},
+      {"Merge", ConvertSimpleOperator<TensorFlowMergeOperator, 2, 1>},
       {"Min", ConvertReduceOperator<TensorFlowMinOperator>},
-      {"Minimum", ConvertSimpleOperator<TensorFlowMinimumOperator, 2>},
-      {"Mul", ConvertSimpleOperator<MulOperator, 2>},
-      {"Neg", ConvertSimpleOperator<NegOperator, 1>},
+      {"Minimum", ConvertSimpleOperator<TensorFlowMinimumOperator, 2, 1>},
+      {"Mul", ConvertSimpleOperator<MulOperator, 2, 1>},
+      {"Neg", ConvertSimpleOperator<NegOperator, 1, 1>},
       {"NextIteration", ConvertOperatorSpecialCasedAsRNNBackEdge},
       {"NoOp", ConvertNoOpOperator},
-      {"NotEqual", ConvertSimpleOperator<TensorFlowNotEqualOperator, 2>},
+      {"NotEqual", ConvertSimpleOperator<TensorFlowNotEqualOperator, 2, 1>},
       {"OneHot", ConvertOneHotOperator},
       {"Pack", ConvertPackOperator},
-      {"Pad", ConvertSimpleOperator<PadOperator, 2>},
-      {"PadV2", ConvertSimpleOperator<PadV2Operator, 3>},
+      {"Pad", ConvertSimpleOperator<PadOperator, 2, 1>},
+      {"PadV2", ConvertSimpleOperator<PadV2Operator, 3, 1>},
       {"ParallelDynamicStitch", ConvertDynamicStitchOperator},
       {"Placeholder", ConvertPlaceholderOperator},
       {"PlaceholderWithDefault", ConvertIdentityOperator},
-      {"Pow", ConvertSimpleOperator<PowOperator, 2>},
+      {"Pow", ConvertSimpleOperator<PowOperator, 2, 1>},
       {"Prod", ConvertReduceOperator<TensorFlowProdOperator>},
       {"RandomUniform", ConvertRandomUniform},
       {"Range", ConvertRangeOperator},
-      {"Rank", ConvertSimpleOperator<RankOperator, 1>},
-      {"RealDiv", ConvertSimpleOperator<DivOperator, 2>},
-      {"Relu", ConvertSimpleOperator<ReluOperator, 1>},
-      {"Relu6", ConvertSimpleOperator<Relu6Operator, 1>},
-      {"Reshape", ConvertSimpleOperator<TensorFlowReshapeOperator, 2>},
+      {"Rank", ConvertSimpleOperator<RankOperator, 1, 1>},
+      {"RealDiv", ConvertSimpleOperator<DivOperator, 2, 1>},
+      {"Relu", ConvertSimpleOperator<ReluOperator, 1, 1>},
+      {"Relu6", ConvertSimpleOperator<Relu6Operator, 1, 1>},
+      {"Reshape", ConvertSimpleOperator<TensorFlowReshapeOperator, 2, 1>},
       {"ResizeBilinear", ConvertResizeBilinearOperator},
       {"ResizeNearestNeighbor", ConvertResizeNearestNeighborOperator},
-      {"Rsqrt", ConvertSimpleOperator<TensorFlowRsqrtOperator, 1>},
-      {"Select", ConvertSimpleOperator<SelectOperator, 3>},
+      {"Rsqrt", ConvertSimpleOperator<TensorFlowRsqrtOperator, 1, 1>},
+      {"Select", ConvertSimpleOperator<SelectOperator, 3, 1>},
       {"Shape", ConvertShapeOperator},
-      {"Sigmoid", ConvertSimpleOperator<LogisticOperator, 1>},
-      {"Sin", ConvertSimpleOperator<SinOperator, 1>},
-      {"Slice", ConvertSimpleOperator<SliceOperator, 3>},
+      {"Sigmoid", ConvertSimpleOperator<LogisticOperator, 1, 1>},
+      {"Sin", ConvertSimpleOperator<SinOperator, 1, 1>},
+      {"Slice", ConvertSimpleOperator<SliceOperator, 3, 1>},
       {"Softmax", ConvertSoftmaxOperator},
       {"SpaceToBatchND", ConvertSpaceToBatchNDOperator},
       {"SpaceToDepth", ConvertSpaceToDepthOperator},
       {"SparseToDense", ConvertSparseToDenseOperator},
       {"Split", ConvertSplitOperator},
       {"SplitV", ConvertSplitVOperator},
-      {"Sqrt", ConvertSimpleOperator<TensorFlowSqrtOperator, 1>},
-      {"Square", ConvertSimpleOperator<TensorFlowSquareOperator, 1>},
+      {"Sqrt", ConvertSimpleOperator<TensorFlowSqrtOperator, 1, 1>},
+      {"Square", ConvertSimpleOperator<TensorFlowSquareOperator, 1, 1>},
       {"SquaredDifference",
-       ConvertSimpleOperator<SquaredDifferenceOperator, 2>},
+       ConvertSimpleOperator<SquaredDifferenceOperator, 2, 1>},
       {"Squeeze", ConvertSqueezeOperator},
       {"StopGradient", ConvertIdentityOperator},
       {"StridedSlice", ConvertStridedSliceOperator},
-      {"Sub", ConvertSimpleOperator<SubOperator, 2>},
+      {"Sub", ConvertSimpleOperator<SubOperator, 2, 1>},
       {"Sum", ConvertReduceOperator<TensorFlowSumOperator>},
       {"Svdf", ConvertSvdfOperator},
       {"Switch", ConvertSwitchOperator},
-      {"Tanh", ConvertSimpleOperator<TanhOperator, 1>},
-      {"Tile", ConvertSimpleOperator<TensorFlowTileOperator, 2>},
+      {"Tanh", ConvertSimpleOperator<TanhOperator, 1, 1>},
+      {"Tile", ConvertSimpleOperator<TensorFlowTileOperator, 2, 1>},
       {"TopK", ConvertTopKV2Operator},
       {"TopKV2", ConvertTopKV2Operator},
-      {"Transpose", ConvertSimpleOperator<TransposeOperator, 2>},
+      {"Transpose", ConvertSimpleOperator<TransposeOperator, 2, 1>},
       {"Unpack", ConvertUnpackOperator},
-      {"ZerosLike", ConvertSimpleOperator<TensorFlowZerosLikeOperator, 1>},
+      {"ZerosLike", ConvertSimpleOperator<TensorFlowZerosLikeOperator, 1, 1>},
       {"UnidirectionalSequenceLstm", ConvertUnidirectionalSequenceLstm},
       {"MirrorPad", ConvertMirrorPadOperator},
+      {"Unique", ConvertSimpleOperator<UniqueOperator, 1, 2>},
   });
 }
 
diff --git a/tensorflow/lite/toco/model.h b/tensorflow/lite/toco/model.h
index e71d36583e..bfa86c8059 100644
--- a/tensorflow/lite/toco/model.h
+++ b/tensorflow/lite/toco/model.h
@@ -157,7 +157,8 @@ enum class OperatorType : uint8 {
   kResizeNearestNeighbor,
   kLeakyRelu,
   kAbs,
-  kMirrorPad
+  kMirrorPad,
+  kUnique
 };
 
 // Helper to deal with TensorFlow arrays using a different ordering of
@@ -1953,6 +1954,17 @@ struct MirrorPadOperator : Operator {
   MirrorPadMode mode;
 };
 
+// Unique Operator:
+//
+// Inputs:
+//   inputs[0]: required: the input array
+//
+// TensorFlow equivalent: Unique
+struct UniqueOperator : Operator {
+  UniqueOperator() : Operator(OperatorType::kUnique) {}
+  ArrayDataType idx_out_type = ArrayDataType::kInt32;
+};
+
 // Alloc's are used for transient arrays only. An Alloc specifies which interval
 // of the "transient_data" workspace buffer passed to inference functions, is to
 // be used for the transient array at hand. The 'start' and 'end' values are
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index d3fce6893f..8eb4d321ef 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include "tensorflow/core/util/ptr_util.h"
 // TODO(ycling): Consider refactoring to extract the LSTM definition out of
 // graph_transformation module.
+#include "tensorflow/lite/schema/schema_generated.h"
 #include "tensorflow/lite/toco/graph_transformations/lstm_utils.h"
 #include "tensorflow/lite/toco/model.h"
 #include "tensorflow/lite/toco/tflite/builtin_operator.h"
@@ -1478,6 +1479,31 @@ class MirrorPad
                    : MirrorPadMode::kSymmetric;
   }
 
+  int GetVersion(const OperatorSignature& op) const override { return 1; }
+};
+
+class Unique : public BuiltinOperator<UniqueOperator, ::tflite::UniqueOptions,
+                                      ::tflite::BuiltinOptions_UniqueOptions> {
+ public:
+  using BuiltinOperator::BuiltinOperator;
+  flatbuffers::Offset<TfLiteOptions> WriteOptions(
+      const TocoOperator& op,
+      flatbuffers::FlatBufferBuilder* builder) const override {
+    const UniqueOperator& unique_op = static_cast<const UniqueOperator&>(op);
+    return ::tflite::CreateUniqueOptions(
+        *builder, unique_op.idx_out_type == toco::ArrayDataType::kInt64
+                      ? ::tflite::TensorType::TensorType_INT64
+                      : ::tflite::TensorType_INT32);
+  }
+  void ReadOptions(const TfLiteOptions& options,
+                   TocoOperator* op) const override {
+    UniqueOperator* unique_op = static_cast<UniqueOperator*>(op);
+    unique_op->idx_out_type =
+        options.idx_out_type() == ::tflite::TensorType_INT64
+            ? toco::ArrayDataType::kInt64
+            : toco::ArrayDataType::kInt32;
+  }
+
   int GetVersion(const OperatorSignature& op_signature) const override {
     return 1;
   }
@@ -1819,6 +1845,8 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
       OperatorType::kSquaredDifference));
   ops.push_back(MakeUnique<MirrorPad>(::tflite::BuiltinOperator_MIRROR_PAD,
                                       OperatorType::kMirrorPad));
+  ops.push_back(MakeUnique<Unique>(::tflite::BuiltinOperator_UNIQUE,
+                                   OperatorType::kUnique));
 
   // Custom Operators.
   ops.push_back(
diff --git a/tensorflow/lite/toco/tflite/operator_test.cc b/tensorflow/lite/toco/tflite/operator_test.cc
index 849eace8cc..215eda82f6 100644
--- a/tensorflow/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/lite/toco/tflite/operator_test.cc
@@ -629,6 +629,15 @@ TEST_F(OperatorTest, BuiltinMirrorPad) {
   EXPECT_EQ(op.mode, output_toco_op->mode);
 }
 
+TEST_F(OperatorTest, BuiltinUnique) {
+  UniqueOperator op;
+  op.idx_out_type = ArrayDataType::kInt64;
+  auto output_toco_op =
+      SerializeAndDeserialize(GetOperator("UNIQUE", OperatorType::kUnique), op);
+  ASSERT_NE(nullptr, output_toco_op.get());
+  EXPECT_EQ(output_toco_op->idx_out_type, op.idx_out_type);
+}
+
 }  // namespace
 }  // namespace tflite
 
diff --git a/tensorflow/lite/toco/tooling_util.cc b/tensorflow/lite/toco/tooling_util.cc
index af4cd386a2..2396de1a3d 100644
--- a/tensorflow/lite/toco/tooling_util.cc
+++ b/tensorflow/lite/toco/tooling_util.cc
@@ -416,6 +416,7 @@ const char* OperatorTypeName(OperatorType type) {
     HANDLE_OPERATORTYPENAME_CASE(LeakyRelu)
     HANDLE_OPERATORTYPENAME_CASE(SquaredDifference)
     HANDLE_OPERATORTYPENAME_CASE(MirrorPad)
+    HANDLE_OPERATORTYPENAME_CASE(Unique)
     default:
       LOG(FATAL) << "Unhandled op type";
 #undef HANDLE_OPERATORTYPENAME_CASE
-- 
GitLab


From b2db5d4089f6cebfabd461401eeac4ed8a976fe1 Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Mon, 7 Jan 2019 11:30:46 -0800
Subject: [PATCH 0268/2345] Fix windows build

PiperOrigin-RevId: 228203086
---
 tensorflow/core/framework/resource_handle.cc | 3 ---
 tensorflow/core/framework/resource_handle.h  | 3 ++-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/framework/resource_handle.cc b/tensorflow/core/framework/resource_handle.cc
index a3c48d63d4..fc3a329b3b 100644
--- a/tensorflow/core/framework/resource_handle.cc
+++ b/tensorflow/core/framework/resource_handle.cc
@@ -19,9 +19,6 @@ limitations under the License.
 
 namespace tensorflow {
 
-const char ResourceHandle::ANONYMOUS_NAME[] =
-    "cd2c89b7-88b7-44c8-ad83-06c2a9158347";
-
 ResourceHandle::ResourceHandle() {}
 
 ResourceHandle::ResourceHandle(const ResourceHandleProto& proto) {
diff --git a/tensorflow/core/framework/resource_handle.h b/tensorflow/core/framework/resource_handle.h
index c3beffb8ac..d1f6771bf3 100644
--- a/tensorflow/core/framework/resource_handle.h
+++ b/tensorflow/core/framework/resource_handle.h
@@ -69,7 +69,8 @@ class ResourceHandle {
 
   // GUID for anonymous resources. Resources with this shared_name will have
   // their shared_name replaced with a GUID at creation time
-  static const char ANONYMOUS_NAME[];
+  static constexpr const char* ANONYMOUS_NAME =
+      "cd2c89b7-88b7-44c8-ad83-06c2a9158347";
 
  public:
   string device_;
-- 
GitLab


From 2325650d59511ad1e19e0102cf920b48cca0bfc5 Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Mon, 7 Jan 2019 11:31:30 -0800
Subject: [PATCH 0269/2345] Remove false comments in portable kernel.

PiperOrigin-RevId: 228203258
---
 .../lite/kernels/internal/reference/portable_tensor_utils.cc     | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/lite/kernels/internal/reference/portable_tensor_utils.cc b/tensorflow/lite/kernels/internal/reference/portable_tensor_utils.cc
index d692063a96..1acf0caad0 100644
--- a/tensorflow/lite/kernels/internal/reference/portable_tensor_utils.cc
+++ b/tensorflow/lite/kernels/internal/reference/portable_tensor_utils.cc
@@ -101,7 +101,6 @@ void PortableMatrixBatchVectorMultiplyAccumulate(
       __builtin_prefetch(row_ptr, 0 /* prefetch for read */,
                          3 /* temporal locality */);
 #endif
-      // For every block of 16 8-bit elements (128-bit register) from each row.
       for (col = 0; col < m_cols; ++col, ++row_ptr) {
         dotprod += (*row_ptr) * (vectors[col]);
       }  // for col
-- 
GitLab


From e740529d3f09071ef0691a8af24dc566f96cbbec Mon Sep 17 00:00:00 2001
From: Nick Felt <nickfelt@google.com>
Date: Mon, 7 Jan 2019 11:36:12 -0800
Subject: [PATCH 0270/2345] Ensure tf.linspace(start, stop, num)[-1] == stop

The contract of tf.linspace says "A sequence of num evenly-spaced values are generated beginning at start. If num > 1, the values in the sequence increase by stop - start / num - 1, so that the last one is exactly stop."

But the existing logic computes the last value as `start + (num - 1) * ((start - stop) / (num - 1))`.  This exactly equals `stop` for real numbers, but not for floating point numbers.  In particular, for start = 0.0, stop = 1.0, num = 42, this reduces to (1.0/41.0)*41.0, and for IEEE 754 32-bit floats, that resolves to ~0.99999994, not 1.0 (binary values 0x3F7FFFFF vs 0x3F800000).

After this change, `tf.linspace(start, stop, num)[-1] == stop` holds for all values assuming num > 1, because we just set it directly to `stop`.  This matches what numpy does here: https://github.com/numpy/numpy/blob/v1.15.0/numpy/core/function_base.py#L144-L145

PiperOrigin-RevId: 228204335
---
 tensorflow/core/kernels/sequence_ops.cc       |  9 ++++----
 tensorflow/core/kernels/sequence_ops_test.cc  | 21 +++++++++++++++++++
 .../python/kernel_tests/init_ops_test.py      | 16 ++++++++++++++
 3 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/kernels/sequence_ops.cc b/tensorflow/core/kernels/sequence_ops.cc
index 9db0bd4d98..21c3b89f54 100644
--- a/tensorflow/core/kernels/sequence_ops.cc
+++ b/tensorflow/core/kernels/sequence_ops.cc
@@ -143,11 +143,12 @@ class LinSpaceOp : public OpKernel {
     OP_REQUIRES_OK(context,
                    context->allocate_output(0, TensorShape({num}), &out));
     auto flat = out->flat<T>();
-    if (num == 1) {
-      flat(0) = start;
-    } else {
+    flat(0) = start;
+    if (num > 1) {
       const T step = (stop - start) / (num - 1);
-      for (Tnum i = 0; i < num; ++i) flat(i) = start + step * i;
+      for (Tnum i = 1; i < num - 1; ++i) flat(i) = start + step * i;
+      // Ensure final value == stop; float arithmetic won't guarantee this.
+      flat(num - 1) = stop;
     }
   }
 };
diff --git a/tensorflow/core/kernels/sequence_ops_test.cc b/tensorflow/core/kernels/sequence_ops_test.cc
index 5f0e0a69a8..2247c44750 100644
--- a/tensorflow/core/kernels/sequence_ops_test.cc
+++ b/tensorflow/core/kernels/sequence_ops_test.cc
@@ -114,6 +114,27 @@ TEST_F(LinSpaceOpTest, Simple_D32) {
   test::ExpectTensorEqual<float>(expected, *GetOutput(0));
 }
 
+TEST_F(LinSpaceOpTest, Exact_Endpoints) {
+  MakeOp(DT_FLOAT, DT_INT32);
+
+  // Feed and run. The particular values 0., 1., and 42 are chosen to test that
+  // the last value is not calculated via an intermediate delta as (1./41)*41,
+  // because for IEEE 32-bit floats that returns 0.99999994 != 1.0.
+  AddInputFromArray<float>(TensorShape({}), {0.0});
+  AddInputFromArray<float>(TensorShape({}), {1.0});
+  AddInputFromArray<int32>(TensorShape({}), {42});
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Check the output
+  Tensor output = *GetOutput(0);
+  float expected_start = 0.0;
+  float start = output.flat<float>()(0);
+  EXPECT_EQ(expected_start, start) << expected_start << " vs. " << start;
+  float expected_stop = 1.0;
+  float stop = output.flat<float>()(output.NumElements() - 1);
+  EXPECT_EQ(expected_stop, stop) << expected_stop << " vs. " << stop;
+}
+
 TEST_F(LinSpaceOpTest, Single_D64) {
   MakeOp(DT_FLOAT, DT_INT64);
 
diff --git a/tensorflow/python/kernel_tests/init_ops_test.py b/tensorflow/python/kernel_tests/init_ops_test.py
index 09b9944baa..4b9681afd2 100644
--- a/tensorflow/python/kernel_tests/init_ops_test.py
+++ b/tensorflow/python/kernel_tests/init_ops_test.py
@@ -592,6 +592,22 @@ class LinSpaceTest(test.TestCase):
       self.assertArrayNear(self._LinSpace(5., 5., 3), np.array([5.] * 3), 1e-5)
       self.assertArrayNear(self._LinSpace(5., 5., 4), np.array([5.] * 4), 1e-5)
 
+  def testEndpointsAreExact(self):
+    for self.force_gpu in self._gpu_modes():
+      # Test some cases that produce last values not equal to "stop" when
+      # computed via start + (num - 1) * ((stop - start) / (num - 1)), since
+      # float arithmetic will introduce error through precision loss.
+      self.assertAllEqual(
+          self._LinSpace(0., 1., 42)[[0, -1]], np.array([0., 1.], np.float32))
+      self.assertAllEqual(
+          self._LinSpace(-1., 0., 42)[[0, -1]], np.array([-1., 0.], np.float32))
+      self.assertAllEqual(
+          self._LinSpace(.1, .2, 4)[[0, -1]], np.array([.1, .2], np.float32))
+      # Check a case for float64 error too.
+      self.assertAllEqual(
+          self._LinSpace(np.array(0., np.float64), .1, 12)[[0, -1]],
+          np.array([0., .1], np.float64))
+
 
 class DeviceTest(test.TestCase):
 
-- 
GitLab


From 5167dd3495de7e857db8a2b578a08db8139c7d29 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Mon, 7 Jan 2019 11:53:00 -0800
Subject: [PATCH 0271/2345] Remove a remove_checkpoint deprecation warning
 showing up in a 2.x API

Inlines the only useful bits

PiperOrigin-RevId: 228207585
---
 tensorflow/python/training/checkpoint_management.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/training/checkpoint_management.py b/tensorflow/python/training/checkpoint_management.py
index a7ad1f70e5..21fa6b3b5d 100644
--- a/tensorflow/python/training/checkpoint_management.py
+++ b/tensorflow/python/training/checkpoint_management.py
@@ -621,7 +621,8 @@ class CheckpointManager(object):
                >= self._last_preserved_timestamp)):
         self._last_preserved_timestamp = timestamp
         continue
-      remove_checkpoint(filename)
+      _delete_file_if_exists(filename + ".index")
+      _delete_file_if_exists(filename + ".data-?????-of-?????")
 
   def _record_state(self):
     """Saves the `CheckpointManager`'s state in `directory`."""
-- 
GitLab


From f75b3374770d75b2f5b412ba4b32b497bec8b6aa Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 12:05:42 -0800
Subject: [PATCH 0272/2345] fix spacing

PiperOrigin-RevId: 228210397
---
 tensorflow/python/grappler/cluster.i | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/grappler/cluster.i b/tensorflow/python/grappler/cluster.i
index 87795ffcfb..b0c1f71a85 100644
--- a/tensorflow/python/grappler/cluster.i
+++ b/tensorflow/python/grappler/cluster.i
@@ -132,7 +132,7 @@ struct GCluster {
 
 static GCluster TF_NewCluster(bool allow_soft_placement,
                    bool disable_detailed_stats, TF_Status* out_status) {
-    int num_cpu_cores = tensorflow::grappler::GetNumAvailableLogicalCPUCores();
+  int num_cpu_cores = tensorflow::grappler::GetNumAvailableLogicalCPUCores();
   int num_gpus = tensorflow::grappler::GetNumAvailableGPUs();
   int timeout_s = 60 * 10;
   tensorflow::grappler::Cluster* cluster_ =
-- 
GitLab


From 27822e43c671c7705b2be24891cc40ad628d1c91 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 12:12:45 -0800
Subject: [PATCH 0273/2345] fix typos

PiperOrigin-RevId: 228211568
---
 tensorflow/core/grappler/clusters/virtual_cluster.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/grappler/clusters/virtual_cluster.cc b/tensorflow/core/grappler/clusters/virtual_cluster.cc
index dbd8f26c28..e80d6b54d9 100644
--- a/tensorflow/core/grappler/clusters/virtual_cluster.cc
+++ b/tensorflow/core/grappler/clusters/virtual_cluster.cc
@@ -67,8 +67,8 @@ Status VirtualCluster::Run(const GraphDef& graph,
                            const std::vector<string>& fetch,
                            RunMetadata* metadata) {
   // Initialize a virtual scheduler to process the graph. Make sure to use
-  // static shape inference to prevent the schedulrer from calling the Run
-  // method on the cluster, and create an infinite loop.
+  // static shape inference to prevent the scheduler from calling the Run
+  // method on the cluster and creating an infinite loop.
   GrapplerItem item;
   item.graph = graph;
   item.feed = feed;
-- 
GitLab


From 7ba8db889667a757339714253bb70f77bca5b2f0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 12:18:06 -0800
Subject: [PATCH 0274/2345] Add missing include for `std::fegetround()`

PiperOrigin-RevId: 228212438
---
 tensorflow/core/platform/setround.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/core/platform/setround.cc b/tensorflow/core/platform/setround.cc
index 592626bfa1..5573b2fc93 100644
--- a/tensorflow/core/platform/setround.cc
+++ b/tensorflow/core/platform/setround.cc
@@ -15,6 +15,8 @@ limitations under the License.
 
 #include "tensorflow/core/platform/setround.h"
 
+#include <cfenv>  // NOLINT
+
 namespace tensorflow {
 namespace port {
 
-- 
GitLab


From 7236c9bb7612fb1a0212dd81f934e0e68fddc0a2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 12:18:25 -0800
Subject: [PATCH 0275/2345] Fix issue with passing EagerTensors to fit when
 run_eagerly=False

PiperOrigin-RevId: 228212493
---
 .../python/keras/engine/training_arrays.py    |  4 ++
 .../keras/engine/training_eager_test.py       | 39 +++++++++++++------
 .../python/keras/engine/training_utils.py     | 19 +++++++++
 3 files changed, 51 insertions(+), 11 deletions(-)

diff --git a/tensorflow/python/keras/engine/training_arrays.py b/tensorflow/python/keras/engine/training_arrays.py
index 97025a9e18..9887fe34a8 100644
--- a/tensorflow/python/keras/engine/training_arrays.py
+++ b/tensorflow/python/keras/engine/training_arrays.py
@@ -200,6 +200,10 @@ def model_iteration(model,
   use_steps = steps_per_epoch is not None
   do_validation = val_inputs is not None
 
+  # Convert Eager Tensors to NumPy arrays to support batching/shuffling.
+  inputs, targets, sample_weights = training_utils. \
+      convert_eager_tensors_to_numpy((inputs, targets, sample_weights))
+
   # Prepare input data.
   ins = _prepare_feed_values(model, inputs, targets, sample_weights, mode)
   num_samples_or_steps = _get_num_samples_or_steps(ins, batch_size,
diff --git a/tensorflow/python/keras/engine/training_eager_test.py b/tensorflow/python/keras/engine/training_eager_test.py
index 27eaea23ba..a6e2c24ec2 100644
--- a/tensorflow/python/keras/engine/training_eager_test.py
+++ b/tensorflow/python/keras/engine/training_eager_test.py
@@ -28,13 +28,20 @@ from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import metrics as metrics_module
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.keras.optimizer_v2 import rmsprop
+from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
 
 class TrainingTest(keras_parameterized.TestCase):
 
   @keras_parameterized.run_with_all_model_types(exclude_models='sequential')
+  @keras_parameterized.run_all_keras_modes
   def test_model_methods_with_eager_tensors_multi_io(self):
+    if not context.executing_eagerly():
+      # Only test V2 Function and V2 Eager modes, as V1 Graph mode with
+      # symbolic tensors has different requirements.
+      return
+
     input_a = keras.layers.Input(shape=(3,), name='input_a')
     input_b = keras.layers.Input(shape=(3,), name='input_b')
 
@@ -53,13 +60,13 @@ class TrainingTest(keras_parameterized.TestCase):
         loss,
         metrics=metrics,
         loss_weights=loss_weights,
-        run_eagerly=True,
+        run_eagerly=testing_utils.should_run_eagerly(),
         sample_weight_mode=None)
 
-    input_a = keras.backend.zeros(shape=(10, 3))
-    input_b = keras.backend.zeros(shape=(10, 3))
-    target_a = keras.backend.zeros(shape=(10, 4))
-    target_b = keras.backend.zeros(shape=(10, 4))
+    input_a = array_ops.zeros(shape=(10, 3))
+    input_b = array_ops.zeros(shape=(10, 3))
+    target_a = array_ops.zeros(shape=(10, 4))
+    target_b = array_ops.zeros(shape=(10, 4))
 
     model.fit(
         [input_a, input_b], [target_a, target_b],
@@ -107,16 +114,26 @@ class TrainingTest(keras_parameterized.TestCase):
     model.test_on_batch([input_a, input_b], [target_a, target_b])
 
   @keras_parameterized.run_with_all_model_types
+  @keras_parameterized.run_all_keras_modes
   def test_model_methods_with_eager_tensors_single_io(self):
+    if not context.executing_eagerly():
+      # Only test V2 Function and V2 Eager modes, as V1 Graph mode with
+      # symbolic tensors has different requirements.
+      return
+
     model = testing_utils.get_small_mlp(10, 4, 3)
 
     optimizer = rmsprop.RMSprop(learning_rate=0.001)
     loss = 'mse'
     metrics = ['mae', metrics_module.CategoricalAccuracy()]
-    model.compile(optimizer, loss, metrics=metrics, run_eagerly=True)
+    model.compile(
+        optimizer,
+        loss,
+        metrics=metrics,
+        run_eagerly=testing_utils.should_run_eagerly())
 
-    inputs = keras.backend.zeros(shape=(10, 3))
-    targets = keras.backend.zeros(shape=(10, 4))
+    inputs = array_ops.zeros(shape=(10, 3))
+    targets = array_ops.zeros(shape=(10, 4))
 
     model.fit(inputs, targets, epochs=1, batch_size=2, verbose=0)
     model.fit(inputs, targets, epochs=1, batch_size=3, verbose=0, shuffle=False)
@@ -134,8 +151,8 @@ class TrainingTest(keras_parameterized.TestCase):
                   loss='mse',
                   run_eagerly=True)
 
-    x = keras.backend.zeros(shape=(10, 3))
-    y = keras.backend.zeros(shape=(10, 4))
+    x = array_ops.zeros(shape=(10, 3))
+    y = array_ops.zeros(shape=(10, 4))
     dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat(10).batch(5)
     iterator = dataset_ops.make_one_shot_iterator(dataset)
     validation_dataset = dataset_ops.Dataset.from_tensor_slices(
@@ -146,7 +163,7 @@ class TrainingTest(keras_parameterized.TestCase):
         ValueError, r'specify .* `steps_per_epoch`'):
       model.fit(iterator, epochs=1, verbose=0)
     if not context.executing_eagerly():
-      # In eager execution, `keras.backend.zeros` returns value tensors
+      # In eager execution, `array_ops.zeros` returns value tensors
       # which can be used for validation without a `validation_steps` argument.
       with self.assertRaisesRegexp(
           ValueError, r'provide either `batch_size` or `validation_steps`'):
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index d07e3cc4f7..5edf1de1b0 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -1402,3 +1402,22 @@ def set_run_eagerly_for_dict_structure(model, x):
       if isinstance(item, dict):
         model.run_eagerly = True
         return
+
+
+def convert_eager_tensors_to_numpy(structure):
+  """Convert every EagerTensor in `structure` to NumPy.
+
+  Arguments:
+    structure: An arbitrary structure of elements to be converted to NumPy
+      arrays.
+
+  Returns:
+    An identical structure with EagerTensors converted to NumPy arrays.
+  """
+
+  def _convert(element):
+    if isinstance(element, ops.EagerTensor):
+      return element.numpy()
+    return element
+
+  return nest.map_structure(_convert, structure)
-- 
GitLab


From e1a553ba1b10a6f39f01b364d7107d1c77a97776 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Mon, 7 Jan 2019 12:18:27 -0800
Subject: [PATCH 0276/2345] Add v2 huber loss.

PiperOrigin-RevId: 228212499
---
 tensorflow/python/keras/losses.py      |  80 ++++++++++++++++++++
 tensorflow/python/keras/losses_test.py | 101 +++++++++++++++++++++++++
 2 files changed, 181 insertions(+)

diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index de1605a09c..5f9278823c 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -608,6 +608,53 @@ class KullbackLeiblerDivergence(Loss):
     return kullback_leibler_divergence(y_true, y_pred)
 
 
+class HuberLoss(Loss):
+  """Computes the huber loss between `y_true` and `y_pred`.
+
+  For each value x in `error=y_true-y_pred`, the following is calculated:
+
+    ```
+    0.5 * x^2                  if |x| <= d
+    0.5 * d^2 + d * (|x| - d)  if |x| > d
+    ```
+  where d is `delta`. See: https://en.wikipedia.org/wiki/Huber_loss
+
+  Usage:
+
+  ```python
+  l = tf.losses.HuberLoss()
+  loss = l([0., 1., 1.], [1., 0., 1.])
+  print('Loss: ', loss.numpy())  # Loss: 0.333
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', loss=tf.losses.HuberLoss())
+  ```
+
+  Args:
+    delta: A float, the point where the huber loss function changes from a
+      quadratic to linear.
+    reduction: Type of `tf.losses.Reduction` to apply to loss. Default value is
+      `SUM_OVER_BATCH_SIZE`.
+    name: Optional name for the op.
+  """
+
+  def __init__(self,
+               delta=1.0,
+               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               name=None):
+    super(HuberLoss, self).__init__(reduction=reduction, name=name)
+    self.delta = delta
+
+  def call(self, y_true, y_pred):
+    y_pred = ops.convert_to_tensor(y_pred)
+    y_true = math_ops.cast(y_true, y_pred.dtype)
+    return huber_loss(y_true, y_pred, delta=self.delta)
+
+
 @keras_export('keras.metrics.mean_squared_error',
               'keras.metrics.mse',
               'keras.metrics.MSE',
@@ -677,6 +724,39 @@ def logloss(y_true, y_pred):
   return K.mean(-losses, axis=-1)
 
 
+def huber_loss(y_true, y_pred, delta=1.0):
+  """Computes huber loss value.
+
+  For each value x in `error=y_true-y_pred`, the following is calculated:
+
+    ```
+    0.5 * x^2                  if |x| <= d
+    0.5 * d^2 + d * (|x| - d)  if |x| > d
+    ```
+  where d is `delta`. See: https://en.wikipedia.org/wiki/Huber_loss
+
+  Args:
+    y_true: tensor of true targets.
+    y_pred: tensor of predicted targets.
+    delta: A float, the point where the huber loss function changes from a
+      quadratic to linear.
+
+  Returns:
+    Tensor with one scalar loss entry per sample.
+  """
+  y_pred = math_ops.cast(y_pred, dtype=K.floatx())
+  y_true = math_ops.cast(y_true, dtype=K.floatx())
+  error = math_ops.subtract(y_pred, y_true)
+  abs_error = math_ops.abs(error)
+  quadratic = math_ops.minimum(abs_error, delta)
+  linear = math_ops.subtract(abs_error, quadratic)
+  return math_ops.add(
+      math_ops.multiply(
+          ops.convert_to_tensor(0.5, dtype=quadratic.dtype),
+          math_ops.multiply(quadratic, quadratic)),
+      math_ops.multiply(delta, linear))
+
+
 @keras_export('keras.losses.logcosh')
 def logcosh(y_true, y_pred):
   """Logarithm of the hyperbolic cosine of the prediction error.
diff --git a/tensorflow/python/keras/losses_test.py b/tensorflow/python/keras/losses_test.py
index caa4ff4c2e..5d6cd1be55 100644
--- a/tensorflow/python/keras/losses_test.py
+++ b/tensorflow/python/keras/losses_test.py
@@ -1336,5 +1336,106 @@ class KullbackLeiblerDivergenceTest(test.TestCase):
     self.assertAlmostEqual(self.evaluate(loss), 0., 3)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class HuberLossTest(test.TestCase):
+
+  def huber_loss(self, y_true, y_pred, delta=1.0):
+    error = y_pred - y_true
+    abs_error = np.abs(error)
+
+    quadratic = np.minimum(abs_error, delta)
+    linear = np.subtract(abs_error, quadratic)
+    return np.add(
+        np.multiply(0.5, np.multiply(quadratic, quadratic)),
+        np.multiply(delta, linear))
+
+  def setup(self, delta=1.0):
+    self.np_y_pred = np.asarray([.9, .2, .2, .8, .4, .6]).reshape((2, 3))
+    self.np_y_true = np.asarray([1., 0., 1., 1., 0., 0.]).reshape((2, 3))
+
+    self.batch_size = 6
+    self.expected_losses = self.huber_loss(self.np_y_true, self.np_y_pred,
+                                           delta)
+
+    self.y_pred = constant_op.constant(self.np_y_pred)
+    self.y_true = constant_op.constant(self.np_y_true)
+
+  def test_config(self):
+    h_obj = keras.losses.HuberLoss(
+        reduction=losses_impl.ReductionV2.SUM, name='huber')
+    self.assertEqual(h_obj.name, 'huber')
+    self.assertEqual(h_obj.reduction, losses_impl.ReductionV2.SUM)
+
+  def test_all_correct(self):
+    self.setup()
+    h_obj = keras.losses.HuberLoss()
+    loss = h_obj(self.y_true, self.y_true)
+    self.assertAlmostEqual(self.evaluate(loss), 0.0, 3)
+
+  def test_unweighted(self):
+    self.setup()
+    h_obj = keras.losses.HuberLoss()
+    loss = h_obj(self.y_true, self.y_pred)
+    actual_loss = np.sum(self.expected_losses) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), actual_loss, 3)
+
+  def test_scalar_weighted(self):
+    self.setup()
+    h_obj = keras.losses.HuberLoss()
+    sample_weight = 2.3
+    loss = h_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    actual_loss = sample_weight * np.sum(self.expected_losses) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), actual_loss, 3)
+
+    # Verify we get the same output when the same input is given
+    loss_2 = h_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), self.evaluate(loss_2), 3)
+
+  def test_sample_weighted(self):
+    self.setup()
+    h_obj = keras.losses.HuberLoss()
+    sample_weight = constant_op.constant((1.2, 3.4), shape=(2, 1))
+
+    loss = h_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    actual_loss = np.multiply(
+        self.expected_losses,
+        np.asarray([1.2, 1.2, 1.2, 3.4, 3.4, 3.4]).reshape((2, 3)))
+    actual_loss = np.sum(actual_loss) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), actual_loss, 3)
+
+  def test_timestep_weighted(self):
+    self.setup()
+    h_obj = keras.losses.HuberLoss()
+    y_pred = self.np_y_pred.reshape((2, 3, 1))
+    y_true = self.np_y_true.reshape((2, 3, 1))
+    expected_losses = self.huber_loss(y_true, y_pred)
+
+    y_pred = constant_op.constant(y_pred)
+    y_true = constant_op.constant(y_true)
+    sample_weight = np.array([3, 6, 5, 0, 4, 2]).reshape((2, 3, 1))
+    loss = h_obj(
+        y_true,
+        y_pred,
+        sample_weight=constant_op.constant(sample_weight, shape=(2, 3)))
+    actual_loss = np.multiply(expected_losses, sample_weight)
+    actual_loss = np.sum(actual_loss) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), actual_loss, 3)
+
+  def test_zero_weighted(self):
+    self.setup()
+    h_obj = keras.losses.HuberLoss()
+    sample_weight = 0
+    loss = h_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), 0., 3)
+
+  def test_non_default_delta(self):
+    self.setup(delta=0.8)
+    h_obj = keras.losses.HuberLoss(delta=0.8)
+    sample_weight = 2.3
+    loss = h_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    actual_loss = sample_weight * np.sum(self.expected_losses) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), actual_loss, 3)
+
+
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 7ba04bba51ac7500afd5fe0d8e2f883082cdd5fb Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Mon, 7 Jan 2019 12:22:34 -0800
Subject: [PATCH 0277/2345] [XLA:GPU] Enhance column reduction implementation.

With KernelMappingScheme, a thread processes n elements, where
n=tile_size_x/num_thread_x. Previously, we only support n>1 for transpose and
row reduction. This change extends the kernel mapping scheme and code
generation to allow each thread to compute n partial results for column
reduction as follows:
.Add dilated_x to KernelMappingScheme, to indicate whether the multiple
elements processed by the same thread are contiguous or dilated. Dilated_x=true
is what we currently use for transpose while dilated_x=false is used to
support the vectorization of column reduction.
.Extend the stack storages that store the partial result address and current
output linear index address for each output tensor from a scalar to an array of
n elements.
.Add curr_iter_index_x to kernelCodegenInfo, to indicate which output element
that the compiler is currently generating code for. This information is used to
locate the partial result address and output linear index address for the
element.
.Modify the code generation to use n=2 for column reduction.
PiperOrigin-RevId: 228213135
---
 .../xla/service/gpu/ir_emitter_unnested.cc    | 330 +++++++++++-------
 .../xla/service/gpu/ir_emitter_unnested.h     |  20 +-
 .../compiler/xla/service/llvm_ir/ir_array.h   |   2 +
 .../xla/service/llvm_ir/kernel_tiling.cc      |   3 +-
 .../xla/service/llvm_ir/kernel_tiling.h       |  22 +-
 5 files changed, 248 insertions(+), 129 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index f190af921e..2bc4912155 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -88,6 +88,9 @@ namespace xla {
 namespace gpu {
 
 using llvm_ir::KernelMappingScheme;
+using EmitElementFunction =
+    std::function<void(const llvm_ir::IrArray::Index& index, llvm::Value* y_loc,
+                       llvm::Value* x_loc, int64 x_iter_num)>;
 
 namespace {
 
@@ -2133,53 +2136,86 @@ int IrEmitterUnnested::ConstructInputReducedShapeAndCastInputIrArrayToShape(
 
 namespace {
 
-void EmitFullElementalTile(
-    const KernelMappingScheme* mapping_scheme,
-    const IrArray::Index& tile_origin_index, const string& loop_name,
-    KernelSupportLibrary* ksl, llvm::IRBuilder<>* builder, llvm::Value* y,
-    llvm::Value* x, llvm::Type* index_ty,
-    const std::function<void(const IrArray::Index&, llvm::Value*,
-                             llvm::Value*)>& emit_elem_function) {
+std::tuple<llvm::Value*, int64> GetStartOffsetAndStepForX(
+    int64 tile_size_x, int64 num_threads_x,
+    const KernelMappingScheme* mapping_scheme, llvm::IRBuilder<>* builder,
+    llvm::Value* x, llvm::Type* index_ty) {
+  llvm::Value* start_offset_x;
+  int64 step_x;
+  if (mapping_scheme->DilatedX()) {
+    start_offset_x = x;
+    step_x = num_threads_x;
+  } else {
+    start_offset_x = builder->CreateMul(
+        x, llvm::ConstantInt::get(index_ty, tile_size_x / num_threads_x));
+    step_x = 1;
+  }
+  return std::make_tuple(start_offset_x, step_x);
+}
+
+void EmitFullElementalTile(const KernelMappingScheme* mapping_scheme,
+                           const IrArray::Index& tile_origin_index,
+                           const string& loop_name, KernelSupportLibrary* ksl,
+                           llvm::IRBuilder<>* builder, llvm::Value* y,
+                           llvm::Value* x, llvm::Type* index_ty,
+                           const EmitElementFunction& emit_elem_function) {
   int64 num_threads_x = mapping_scheme->GetNumberOfThreadsForDimensionX();
   int64 num_threads_y = mapping_scheme->GetNumberOfThreadsForDimensionY();
   int64 tile_size_x = mapping_scheme->GetTileSizeForDimensionX();
   int64 tile_size_y = mapping_scheme->GetTileSizeForDimensionY();
+
+  llvm::Value* start_offset_x;
+  int64 step_x;
+  std::tie(start_offset_x, step_x) = GetStartOffsetAndStepForX(
+      tile_size_x, num_threads_x, mapping_scheme, builder, x, index_ty);
+  IrArray::Index source_idx =
+      tile_origin_index.AddOffsetToDim(y, KernelMappingScheme::DimY, builder)
+          .AddOffsetToDim(start_offset_x, KernelMappingScheme::DimX, builder);
   ksl->For(loop_name + "_y", /*start=*/llvm::ConstantInt::get(index_ty, 0),
            /*end=*/llvm::ConstantInt::get(index_ty, tile_size_y),
            /*step=*/llvm::ConstantInt::get(index_ty, num_threads_y),
            [&](llvm::Value* y_indvar) {
-             IrArray::Index source_idx_y = tile_origin_index.AddOffsetToDim(
+             IrArray::Index source_idx_y = source_idx.AddOffsetToDim(
                  y_indvar, KernelMappingScheme::DimY, builder);
              llvm::Value* y_loc = builder->CreateAdd(y_indvar, y);
-             for (int64 j = 0; j < tile_size_x; j += num_threads_x) {
-               IrArray::Index source_idx = source_idx_y.AddOffsetToDim(
-                   llvm::ConstantInt::get(index_ty, j),
+
+             for (int64 j = 0; j < tile_size_x / num_threads_x; j++) {
+               IrArray::Index source_idx_y_x = source_idx_y.AddOffsetToDim(
+                   llvm::ConstantInt::get(index_ty, j * step_x),
                    KernelMappingScheme::DimX, builder);
-               llvm::Value* x_loc =
-                   builder->CreateAdd(llvm::ConstantInt::get(index_ty, j), x);
-               emit_elem_function(source_idx, y_loc, x_loc);
+               llvm::Value* x_loc = builder->CreateAdd(
+                   llvm::ConstantInt::get(index_ty, j * step_x),
+                   start_offset_x);
+               emit_elem_function(source_idx_y_x, y_loc, x_loc, j);
              }
            });
 }
 
-void EmitPartialElementalTile(
-    const KernelMappingScheme* mapping_scheme,
-    const IrArray::Index& tile_origin_index, const string& loop_name,
-    KernelSupportLibrary* ksl, llvm::IRBuilder<>* builder, llvm::Value* y,
-    llvm::Value* x, llvm::Value* tile_height, llvm::Value* tile_width,
-    llvm::Type* index_ty,
-    const std::function<void(const IrArray::Index&, llvm::Value*,
-                             llvm::Value*)>& emit_elem_function) {
+void EmitPartialElementalTile(const KernelMappingScheme* mapping_scheme,
+                              const IrArray::Index& tile_origin_index,
+                              const string& loop_name,
+                              KernelSupportLibrary* ksl,
+                              llvm::IRBuilder<>* builder, llvm::Value* y,
+                              llvm::Value* x, llvm::Value* tile_height,
+                              llvm::Value* tile_width, llvm::Type* index_ty,
+                              const EmitElementFunction& emit_elem_function) {
   int64 num_threads_x = mapping_scheme->GetNumberOfThreadsForDimensionX();
   int64 num_threads_y = mapping_scheme->GetNumberOfThreadsForDimensionY();
   int64 tile_size_x = mapping_scheme->GetTileSizeForDimensionX();
 
-  for (int64 j = 0; j < tile_size_x; j += num_threads_x) {
-    IrArray::Index source_idx =
-        tile_origin_index.AddOffsetToDim(llvm::ConstantInt::get(index_ty, j),
-                                         KernelMappingScheme::DimX, builder);
-    llvm::Value* x_loc =
-        builder->CreateAdd(llvm::ConstantInt::get(index_ty, j), x);
+  llvm::Value* start_offset_x;
+  int64 step_x;
+  std::tie(start_offset_x, step_x) = GetStartOffsetAndStepForX(
+      tile_size_x, num_threads_x, mapping_scheme, builder, x, index_ty);
+  IrArray::Index source_idx =
+      tile_origin_index.AddOffsetToDim(y, KernelMappingScheme::DimY, builder)
+          .AddOffsetToDim(start_offset_x, KernelMappingScheme::DimX, builder);
+  for (int64 j = 0; j < tile_size_x / num_threads_x; j++) {
+    IrArray::Index source_idx_x =
+        source_idx.AddOffsetToDim(llvm::ConstantInt::get(index_ty, j * step_x),
+                                  KernelMappingScheme::DimX, builder);
+    llvm::Value* x_loc = builder->CreateAdd(
+        llvm::ConstantInt::get(index_ty, j * step_x), start_offset_x);
 
     ksl->If(
         loop_name + "_x_in_tile", builder->CreateICmpULT(x_loc, tile_width),
@@ -2199,14 +2235,13 @@ void EmitPartialElementalTile(
               /*step=*/llvm::ConstantInt::get(index_ty, num_threads_y),
               [&](llvm::Value* y_indvar) {
                 llvm::Value* y_loc = builder->CreateAdd(y_indvar, y);
-                ksl->If(
-                    loop_name + "_y_in_tile",
-                    builder->CreateICmpULT(y_loc, tile_height), [&] {
-                      emit_elem_function(
-                          source_idx.AddOffsetToDim(
-                              y_indvar, KernelMappingScheme::DimY, builder),
-                          y_loc, x_loc);
-                    });
+                ksl->If(loop_name + "_y_in_tile",
+                        builder->CreateICmpULT(y_loc, tile_height), [&] {
+                          emit_elem_function(
+                              source_idx_x.AddOffsetToDim(
+                                  y_indvar, KernelMappingScheme::DimY, builder),
+                              y_loc, x_loc, j);
+                        });
               });
         });
   }
@@ -2225,8 +2260,7 @@ void EmitTiledElementalCodeWithBoundsCheck(
     const IrArray::Index& tile_origin_index, const string& loop_name,
     KernelSupportLibrary* ksl, llvm::IRBuilder<>* builder, llvm::Value* y,
     llvm::Value* x, llvm::Value* tile_height, llvm::Value* tile_width,
-    const std::function<void(const IrArray::Index&, llvm::Value*,
-                             llvm::Value*)>& emit_elem_function) {
+    const EmitElementFunction& emit_elem_function) {
   int64 tile_size_x = mapping_scheme->GetTileSizeForDimensionX();
   int64 tile_size_y = mapping_scheme->GetTileSizeForDimensionY();
   llvm::Type* index_ty = tile_width->getType();
@@ -2262,7 +2296,7 @@ void EmitTiledElementalCodeWithBoundsCheck(
 void IrEmitterUnnested::EmitTileElementForCopy(
     HloInstruction* hlo, const llvm_ir::IrArray::Index& index,
     const KernelCodegenInfo* kernel_info, llvm::Value* y_loc,
-    llvm::Value* x_loc) {
+    llvm::Value* x_loc, int64 /*x_iter_num*/) {
   llvm_ir::TiledParameterInfo* tiled_param_info =
       kernel_info->GetTiledParameterInfo();
   // TODO(jlebar): Add AA metadata to this load.
@@ -2292,7 +2326,7 @@ void IrEmitterUnnested::EmitTileElementForCopy(
 void IrEmitterUnnested::EmitTileElementForFusion(
     HloInstruction* hlo, const llvm_ir::IrArray::Index& index,
     const KernelCodegenInfo* kernel_info, llvm::Value* y_loc,
-    llvm::Value* x_loc) {
+    llvm::Value* x_loc, int64 /*x_iter_num*/) {
   llvm_ir::TiledParameterInfo* tiled_param_info =
       kernel_info->GetTiledParameterInfo();
   std::vector<IrArray> output_arrays = ConstructIrArrayForOutputs(*hlo);
@@ -2393,6 +2427,23 @@ class ReductionCodegenInfo : public IrEmitterUnnested::KernelCodegenInfo {
                             : llvm_ir::KernelMappingScheme::DimX;
   }
 
+  int GetNumberOfPartialResults() const {
+    if (IsRowReduction()) {
+      return 1;
+    }
+    int64 num_thread = mapping_scheme_->GetNumberOfThreadsForDimensionX();
+    int64 tile_size = mapping_scheme_->GetTileSizeForDimensionX();
+    CHECK_EQ(tile_size % num_thread, 0);
+    return tile_size / num_thread;
+  }
+
+  int GetPartialResultIndex(int64 x_iter_num) const {
+    if (IsRowReduction()) {
+      return 0;
+    }
+    return x_iter_num;
+  }
+
  private:
   AddressVector partial_result_addresses_;
   AddressVector reduction_input_addresses_;
@@ -2452,10 +2503,11 @@ void IrEmitterUnnested::EmitPrologueForOneReduction(
   llvm::AllocaInst* reduction_input_address = Alloca(element_type);
   reduction_input_addresses->push_back(reduction_input_address);
 
+  int num_partial_results = reduction_info->GetNumberOfPartialResults();
   AddressVector* partial_result_addresses =
       reduction_info->GetMutablePartialResultAddresses();
   llvm::AllocaInst* partial_result_address =
-      Alloca(element_type, /*ArraySize=*/nullptr,
+      Alloca(element_type, /*ArraySize=*/b_.getInt32(num_partial_results),
              "partial_reduction_result." + llvm::Twine(reduce_idx));
   partial_result_addresses->push_back(partial_result_address);
 
@@ -2478,7 +2530,9 @@ void IrEmitterUnnested::EmitPrologueForOneReduction(
             .EmitReadArrayElement(IrArray::Index(b_.getInt32Ty()), &b_);
   }
 
-  Store(init_ir_value, partial_result_address);
+  for (int i = 0; i < num_partial_results; ++i) {
+    Store(init_ir_value, InBoundsGEP(partial_result_address, {b_.getInt32(i)}));
+  }
 }
 
 void IrEmitterUnnested::EmitPrologueForReduction(
@@ -2516,10 +2570,14 @@ void IrEmitterUnnested::EmitPrologueForReduction(
                                 std::move(output_shape_index));
   }
 
-  // Allocate stack storage to store the current output linear index and record
-  // the address of the storage.
+  int num_partial_results = reduction_info->GetNumberOfPartialResults();
+
+  // Allocate stack storage to store the linear indices for the current output,
+  // and record the address of the storage.
   reduction_info->SetCurrentOutputLinearIndexAddress(
-      Alloca(reduction_info->GetIndexType()));
+      Alloca(reduction_info->GetIndexType(),
+             /*ArraySize=*/b_.getInt32(num_partial_results),
+             "current_output_linear_index_address"));
 
   if (!reduction_info->IsRowReduction()) {
     llvm::Type* bool_ty = b_.getInt1Ty();
@@ -2589,36 +2647,45 @@ void IrEmitterUnnested::EmitEpilogueForReduction(
     llvm_ir::SetToFirstInsertPoint(if_output_inbound_data.true_block, &b_);
   }
 
+  int num_partial_results = reduction_info->GetNumberOfPartialResults();
+
   // Emit an atomic operation that accumulates the partial reduction to the
   // output element. For row reduction, this is only for lane 0 due to the
   // if-statement emitted above.
   for (int i = 0; i != num_reduces; ++i) {
-    IrArray::Index element_index(
-        /*linear=*/Load(reduction_info->GetCurrentOutputLinearIndexAddress(),
-                        "output_linear_addr"),
-        ShapeUtil::GetSubshape(unnested_hlo->shape(),
-                               reduction_output_shape_indices[i]),
-        &b_);
-    llvm::Value* output_address =
-        GetIrArray(*unnested_hlo, *unnested_hlo,
-                   reduction_output_shape_indices[i])
-            .EmitArrayElementAddress(element_index, &b_,
-                                     "output_element_address");
-    // Do not emit atomic operations if each element in the reduction result is
-    // computed by one block, that is the dimension being reduced has only one
-    // block.
-    const llvm_ir::KernelMappingScheme* mapping_scheme =
-        reduction_info->GetKernelMappingScheme();
-    if (mapping_scheme->GetTileBlockSizeForDimension(
-            llvm_ir::KernelMappingScheme::DimZ) == 1 &&
-        mapping_scheme->GetTileBlockSizeForDimension(
-            reduction_info->GetReducedDimensionEnum()) == 1) {
-      TF_CHECK_OK(EmitCallToNestedComputation(
-          *reducers[i], {output_address, partial_result_addresses[i]},
-          output_address));
-    } else {
-      TF_CHECK_OK(EmitAtomicOperationForNestedComputation(
-          *reducers[i], output_address, partial_result_addresses[i]));
+    for (int j = 0; j < num_partial_results; ++j) {
+      IrArray::Index element_index(
+          /*linear=*/Load(
+              InBoundsGEP(reduction_info->GetCurrentOutputLinearIndexAddress(),
+                          {b_.getInt32(j)}),
+              "output_linear_addr"),
+          ShapeUtil::GetSubshape(unnested_hlo->shape(),
+                                 reduction_output_shape_indices[i]),
+          &b_);
+      llvm::Value* output_address =
+          GetIrArray(*unnested_hlo, *unnested_hlo,
+                     reduction_output_shape_indices[i])
+              .EmitArrayElementAddress(element_index, &b_,
+                                       "output_element_address");
+      // Do not emit atomic operations if each element in the reduction result
+      // is computed by one block, that is the dimension being reduced has only
+      // one block.
+      const llvm_ir::KernelMappingScheme* mapping_scheme =
+          reduction_info->GetKernelMappingScheme();
+      if (mapping_scheme->GetTileBlockSizeForDimension(
+              llvm_ir::KernelMappingScheme::DimZ) == 1 &&
+          mapping_scheme->GetTileBlockSizeForDimension(
+              reduction_info->GetReducedDimensionEnum()) == 1) {
+        TF_CHECK_OK(EmitCallToNestedComputation(
+            *reducers[i],
+            {output_address,
+             InBoundsGEP(partial_result_addresses[i], {b_.getInt32(j)})},
+            output_address));
+      } else {
+        TF_CHECK_OK(EmitAtomicOperationForNestedComputation(
+            *reducers[i], output_address,
+            InBoundsGEP(partial_result_addresses[i], {b_.getInt32(j)})));
+      }
     }
   }
 }
@@ -2626,7 +2693,7 @@ void IrEmitterUnnested::EmitEpilogueForReduction(
 void IrEmitterUnnested::EmitTileElementForReduction(
     HloInstruction* unnested_hlo, const llvm_ir::IrArray::Index& index,
     const KernelCodegenInfo* kernel_info, llvm::Value* y_loc,
-    llvm::Value* x_loc) {
+    llvm::Value* x_loc, int64 x_iter_num) {
   VLOG(10) << "Emit tile element for reduce " << unnested_hlo->ToString();
   HloInstruction* reduce_or_tuple = unnested_hlo->opcode() == HloOpcode::kFusion
                                         ? unnested_hlo->fused_expression_root()
@@ -2639,8 +2706,11 @@ void IrEmitterUnnested::EmitTileElementForReduction(
   // Record the linear address for the current reduction.
   const ReductionCodegenInfo* reduction_info =
       dynamic_cast<const ReductionCodegenInfo*>(kernel_info);
+  int partial_result_index = reduction_info->IsRowReduction() ? 0 : x_iter_num;
+
   Store(index[reduction_info->GetKeptDimensionEnum()],
-        reduction_info->GetCurrentOutputLinearIndexAddress());
+        InBoundsGEP(reduction_info->GetCurrentOutputLinearIndexAddress(),
+                    {b_.getInt32(partial_result_index)}));
   if (!reduction_info->IsRowReduction()) {
     llvm::Type* bool_ty = b_.getInt1Ty();
     llvm::AllocaInst* output_inbound_addr =
@@ -2687,6 +2757,13 @@ void IrEmitterUnnested::EmitTileElementForReduction(
       reduction_info->GetKernelMappingScheme()->GetUnnormalizedIndex(
           index,
           GetFirstReduceInstruction(output_instructions)->operand(0)->shape());
+  int num_partial_results = reduction_info->GetNumberOfPartialResults();
+  if (num_partial_results > 1) {
+    // Clear the linear index field of the IrArray::Index to enable the use of
+    // GetElementPointer with array types. This enables the vectorization of
+    // the computation for different partial results.
+    input_index.ClearLinearIndex();
+  }
   absl::Span<llvm::AllocaInst* const> partial_reduction_result_addresses =
       reduction_info->GetPartialResultAddresses();
   absl::Span<llvm::AllocaInst* const> reduction_input_addresses =
@@ -2699,10 +2776,12 @@ void IrEmitterUnnested::EmitTileElementForReduction(
   for (int i = 0; i != reducers.size(); ++i) {
     llvm::Value* const input_ir_value = input_gens[i](input_index).ValueOrDie();
     Store(input_ir_value, reduction_input_addresses[i]);
+    llvm::Value* partial_result_address =
+        InBoundsGEP(partial_reduction_result_addresses[i],
+                    {b_.getInt32(partial_result_index)});
     TF_CHECK_OK(EmitCallToNestedComputation(
-        *reducers[i],
-        {partial_reduction_result_addresses[i], reduction_input_addresses[i]},
-        partial_reduction_result_addresses[i]));
+        *reducers[i], {partial_result_address, reduction_input_addresses[i]},
+        partial_result_address));
   }
 
   // Emit code to generate the output for the non-reduction instructions in the
@@ -2713,8 +2792,8 @@ void IrEmitterUnnested::EmitTileElementForReduction(
 
 // Emits a kernel for the hlo instruction using the given tiling scheme.
 void IrEmitterUnnested::EmitBlock(const TileGenerator& emit_one_tile,
-                                  const KernelCodegenInfo* kernel_info,
-                                  KernelSupportLibrary& ksl,
+                                  KernelCodegenInfo* kernel_info,
+                                  KernelSupportLibrary* ksl,
                                   llvm::Type* index_ty) {
   KernelMappingScheme* mapping_scheme = kernel_info->GetKernelMappingScheme();
   absl::Span<const int64> dims_in_tile = mapping_scheme->GetDimensionsInTiles();
@@ -2747,15 +2826,14 @@ void IrEmitterUnnested::EmitBlock(const TileGenerator& emit_one_tile,
           llvm::Value* num_tiles_in_block =
               Select(ICmpEQ(last_block_for_dim, block_id_for_dim),
                      last_block_size_for_dim, block_size_for_dim);
-
-          ksl.For(loop_name,
-                  /*start=*/index_typed_constant(0),
-                  /*end=*/num_tiles_in_block,
-                  /*step=*/1, [&](llvm::Value* block_dim_induction_var) {
-                    IrArray::Index tile_index = starting_tile.AddOffsetToDim(
-                        block_dim_induction_var, dim_id, &b_);
-                    emit_next_block_dim(tile_index);
-                  });
+          ksl->For(loop_name,
+                   /*start=*/index_typed_constant(0),
+                   /*end=*/num_tiles_in_block,
+                   /*step=*/1, [&](llvm::Value* block_dim_induction_var) {
+                     IrArray::Index tile_index = starting_tile.AddOffsetToDim(
+                         block_dim_induction_var, dim_id, &b_);
+                     emit_next_block_dim(tile_index);
+                   });
         }
       };
 
@@ -2899,8 +2977,7 @@ LaunchDimensions IrEmitterUnnested::EmitKernel(
   auto emit_tiled_elemental_code_with_bounds_check =
       [&](const IrArray::Index& index, const string& loop_name,
           llvm::Value* tile_height, llvm::Value* tile_width,
-          const std::function<void(const IrArray::Index&, llvm::Value*,
-                                   llvm::Value*)>& emit_elem_function) {
+          const EmitElementFunction& emit_elem_function) {
         EmitTiledElementalCodeWithBoundsCheck(mapping_scheme, index, loop_name,
                                               &ksl, &b_, y, x, tile_height,
                                               tile_width, emit_elem_function);
@@ -2913,10 +2990,6 @@ LaunchDimensions IrEmitterUnnested::EmitKernel(
     const IrArray::Index input_tile_origin(
         Permute({0, 2, 1}, output_tile_origin.multidim()));
 
-    const IrArray::Index input_index =
-        input_tile_origin.AddOffsetToDim(x, KernelMappingScheme::DimX, &b_)
-            .AddOffsetToDim(y, KernelMappingScheme::DimY, &b_);
-
     // If shared memory transpose is needed, wait for all threads to reach this
     // point, lest we copy a value from tile to output before the other thread
     // copies it from input to tile. This is `__syncthreads` in CUDA.
@@ -2926,9 +2999,10 @@ LaunchDimensions IrEmitterUnnested::EmitKernel(
       // Note that tile_width and tile_height are flipped here because we are
       // reading a transposed tile.
       emit_tiled_elemental_code_with_bounds_check(
-          input_index, "input", output_tile_bounds[2], output_tile_bounds[1],
+          input_tile_origin, "input", output_tile_bounds[2],
+          output_tile_bounds[1],
           [&](const IrArray::Index& index, llvm::Value* y_loc,
-              llvm::Value* x_loc) {
+              llvm::Value* x_loc, int64 /*x_iter_num*/) {
             for (int64 id : tiled_param_ids) {
               IrArray& input_in_logical_shape =
                   param_in_reduced_shape_arrays[id];
@@ -2948,18 +3022,15 @@ LaunchDimensions IrEmitterUnnested::EmitKernel(
     llvm_ir::TiledParameterInfo tiled_param_info(param_shmem_buffers, y, x);
     kernel_info->SetTiledParamInfo(&tiled_param_info);
 
-    const IrArray::Index output_index =
-        output_tile_origin.AddOffsetToDim(x, KernelMappingScheme::DimX, &b_)
-            .AddOffsetToDim(y, KernelMappingScheme::DimY, &b_);
-
     // Write to output[index] by emitting code like normal, except that values
     // for the tiled parameters are read from the shmem buffers.
     emit_tiled_elemental_code_with_bounds_check(
-        output_index, "output", output_tile_bounds[1], output_tile_bounds[2],
-        [&](const IrArray::Index& index, llvm::Value* y_loc,
-            llvm::Value* x_loc) {
-          kernel_generator.GetTileElementGenerator()(unnested_hlo, index,
-                                                     kernel_info, y_loc, x_loc);
+        output_tile_origin, "output", output_tile_bounds[1],
+        output_tile_bounds[2],
+        [&](const IrArray::Index& index, llvm::Value* y_loc, llvm::Value* x_loc,
+            int64 x_iter_num) {
+          kernel_generator.GetTileElementGenerator()(
+              unnested_hlo, index, kernel_info, y_loc, x_loc, x_iter_num);
         });
 
     // If a tile block contains multiple tiles and shared memory buffers are
@@ -2977,7 +3048,7 @@ LaunchDimensions IrEmitterUnnested::EmitKernel(
     block_prologue_generator(unnested_hlo, kernel_info);
   }
 
-  EmitBlock(std::move(emit_one_tile), kernel_info, ksl, index_ty);
+  EmitBlock(std::move(emit_one_tile), kernel_info, &ksl, index_ty);
 
   const BlockEpilogueGenerator& block_epilogue_generator =
       kernel_generator.GetBlockEpilogueGenerator();
@@ -3026,17 +3097,19 @@ LaunchDimensions IrEmitterUnnested::EmitHlo021Tile(
     element_generator = [&](HloInstruction* hlo,
                             const llvm_ir::IrArray::Index& index,
                             const KernelCodegenInfo* kernel_info,
-                            llvm::Value* y_loc, llvm::Value* x_loc) {
-      EmitTileElementForCopy(hlo, index, kernel_info, y_loc, x_loc);
+                            llvm::Value* y_loc, llvm::Value* x_loc,
+                            int64 x_iter_num) {
+      EmitTileElementForCopy(hlo, index, kernel_info, y_loc, x_loc, x_iter_num);
     };
   } else {
     DCHECK_EQ(hlo->opcode(), HloOpcode::kFusion);
-    element_generator = [&](HloInstruction* hlo,
-                            const llvm_ir::IrArray::Index& index,
-                            const KernelCodegenInfo* kernel_info,
-                            llvm::Value* y_loc, llvm::Value* x_loc) {
-      EmitTileElementForFusion(hlo, index, kernel_info, y_loc, x_loc);
-    };
+    element_generator =
+        [&](HloInstruction* hlo, const llvm_ir::IrArray::Index& index,
+            const KernelCodegenInfo* kernel_info, llvm::Value* y_loc,
+            llvm::Value* x_loc, int64 x_iter_num) {
+          EmitTileElementForFusion(hlo, index, kernel_info, y_loc, x_loc,
+                                   x_iter_num);
+        };
   }
   KernelCodegenInfo kernel_info(&mapping_scheme);
   KernelCodeGenerator kernel_generator(std::move(element_generator));
@@ -3351,6 +3424,7 @@ IrEmitterUnnested::ComputeMappingSchemeAndReductionKind(
   std::tie(num_reduced_major, num_kept, num_reduced_minor) =
       GetReductionToVectorDimensions(input_shape, first_reduce->dimensions());
   CHECK_EQ(num_output_elems, num_kept);
+  bool dilated_x = true;
 
   if (num_kept == 1) {
     // Scalar reduction is a special row reduction with depth = height = 1.
@@ -3363,15 +3437,31 @@ IrEmitterUnnested::ComputeMappingSchemeAndReductionKind(
     height = num_reduced_major;
     width = num_kept;
     is_row_reduction = false;
+    // Assume unrolling is beneficial only when we can vectorize the loads
+    // of small data types.
+    auto is_unrolling_beneficial = [&] {
+      // TODO(b/122468062): Need further investigate to see whether we can
+      // remove the constraint on IsPowerOfTwo.
+      return IsPowerOfTwo(static_cast<uint64>(num_kept)) &&
+             primitive_util::BitWidth(
+                 first_reduce->operand(0)->shape().element_type()) <= 16;
+    };
     // Column reduction without transpose doesn't require communication among
     // threads processing elements in the same tile. The current implementation
-    // only support the use of on hardware thread block to process one block of
-    // tiles in the KernelMappingScheme. We try to maximize the values of
+    // only support the use of one hardware thread block to process one block of
+    // tiles in the KernelMappingScheme. We try to use one thread to compute
+    // the partial results for two tensor elements and to maximize the values of
     // num_threads_x and tile_size_x to allow a bigger hardware thread block.
     int64 hw_threads_per_block_limit =
         ThreadsPerBlockLimit(ir_emitter_context_->device_description());
-    tile_size_x = std::min(hw_threads_per_block_limit, num_kept);
-    num_threads_x = tile_size_x;
+    if (is_unrolling_beneficial()) {
+      tile_size_x = std::min(2 * hw_threads_per_block_limit, num_kept);
+      num_threads_x = tile_size_x / 2;
+      dilated_x = false;
+    } else {
+      tile_size_x = std::min(hw_threads_per_block_limit, num_kept);
+      num_threads_x = tile_size_x;
+    }
     int64 kNumElementsPerPartialSum = 128;
     tile_size_y = kNumElementsPerPartialSum;
   } else {
@@ -3400,6 +3490,7 @@ IrEmitterUnnested::ComputeMappingSchemeAndReductionKind(
   llvm_ir::KernelMappingScheme mapping_scheme(
       dims_in_elem, tile_size_y, tile_size_x, req_block_sizes, num_threads_y,
       num_threads_x, &b_);
+  mapping_scheme.SetDilatedX(dilated_x);
   return std::make_tuple(mapping_scheme, is_row_reduction);
 }
 
@@ -3454,8 +3545,9 @@ Status IrEmitterUnnested::EmitReductionToVector(HloInstruction* unnested_hlo) {
       /*tile_element_generator=*/
       [&](HloInstruction* hlo, const llvm_ir::IrArray::Index& index,
           const KernelCodegenInfo* kernel_info, llvm::Value* y_loc,
-          llvm::Value* x_loc) {
-        EmitTileElementForReduction(hlo, index, kernel_info, y_loc, x_loc);
+          llvm::Value* x_loc, int64 x_iter_num) {
+        EmitTileElementForReduction(hlo, index, kernel_info, y_loc, x_loc,
+                                    x_iter_num);
       },
       /*block_prologue_generator=*/
       [&](HloInstruction* hlo, KernelCodegenInfo* kernel_info) {
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
index d217ee36cf..eb0ea90e14 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
@@ -76,7 +76,6 @@ class IrEmitterUnnested : public IrEmitter {
     void SetLaneId(llvm::Value* v) { lane_id_ = v; }
     void SetIndexType(llvm::Type* t) { index_ty_ = t; }
     void SetTiledParamInfo(llvm_ir::TiledParameterInfo* tiled_param_info) {
-      CHECK_EQ(tiled_param_info_, nullptr);
       tiled_param_info_ = tiled_param_info;
     }
 
@@ -89,7 +88,7 @@ class IrEmitterUnnested : public IrEmitter {
     }
     llvm::Type* GetIndexType() const { return index_ty_; }
 
-   private:
+   protected:
     llvm_ir::KernelMappingScheme* mapping_scheme_;
     llvm_ir::TiledParameterInfo* tiled_param_info_;
     llvm::Value* lane_id_;
@@ -109,10 +108,12 @@ class IrEmitterUnnested : public IrEmitter {
   // y_loc: The y coordinate within a tile.
   // x_loc: The x coordinate within a tile.
   // kernel_info: Other information to support the kernel code generation.
+  // x_iter_num: When a thread process N elements in the X dimension, x_iter_num
+  //             has a value of 0..N-1 to identify the element being process.
   using TileElementGenerator = std::function<void(
       HloInstruction* hlo, const llvm_ir::IrArray::Index& index,
       const KernelCodegenInfo* kernel_info, llvm::Value* y_loc,
-      llvm::Value* x_loc)>;
+      llvm::Value* x_loc, int64 x_iter_num)>;
 
   // KernelCodeGenerator records the code generator objects that generate code
   // for tile elements or tile block prologue/epilogue.
@@ -243,26 +244,29 @@ class IrEmitterUnnested : public IrEmitter {
                               const KernelCodeGenerator& kernel_generator,
                               KernelCodegenInfo* kernel_info);
   void EmitBlock(const TileGenerator& emit_one_tile,
-                 const KernelCodegenInfo* kernel_info,
-                 KernelSupportLibrary& ksl, llvm::Type* index_ty);
+                 KernelCodegenInfo* kernel_info, KernelSupportLibrary* ksl,
+                 llvm::Type* index_ty);
   // Emits code to process a tensor element in a tile for the given kCopy HLO
   // that performs a 0-2-1 transpose.
   void EmitTileElementForCopy(HloInstruction* hlo,
                               const llvm_ir::IrArray::Index& index,
                               const KernelCodegenInfo* kernel_info,
-                              llvm::Value* y_loc, llvm::Value* x_loc);
+                              llvm::Value* y_loc, llvm::Value* x_loc,
+                              int64 x_iter_num);
   // Emits code to process a tensor element in a tile for the given kLoop fusion
   // HLO containing parameters that are 0-2-1 transpose of its outputs.
   void EmitTileElementForFusion(HloInstruction* hlo,
                                 const llvm_ir::IrArray::Index& index,
                                 const KernelCodegenInfo* kernel_info,
-                                llvm::Value* y_loc, llvm::Value* x_loc);
+                                llvm::Value* y_loc, llvm::Value* x_loc,
+                                int64 x_iter_num);
   // Emits code to process a tensor element in a tile for the given input hlo
   // that is either a unnested kReduce or a kInput fusion.
   void EmitTileElementForReduction(HloInstruction* unnested_hlo,
                                    const llvm_ir::IrArray::Index& index,
                                    const KernelCodegenInfo* kernel_info,
-                                   llvm::Value* y_loc, llvm::Value* x_loc);
+                                   llvm::Value* y_loc, llvm::Value* x_loc,
+                                   int64 x_iter_num);
   // Prepares for the code generation for a tile block of a reduction kernel.
   void EmitPrologueForReduction(HloInstruction* unnested_hlo,
                                 KernelCodegenInfo* kernel_info);
diff --git a/tensorflow/compiler/xla/service/llvm_ir/ir_array.h b/tensorflow/compiler/xla/service/llvm_ir/ir_array.h
index d6d84994ee..a483f7051f 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/ir_array.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/ir_array.h
@@ -189,6 +189,8 @@ class IrArray {
       return llvm::ConstantInt::get(index_type_, c);
     }
 
+    void ClearLinearIndex() { linear_ = nullptr; }
+
    private:
     // Changing the multi-dimensional index invalidates the linear index.
     std::vector<llvm::Value*>& mutable_multidim() {
diff --git a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc
index cebbc42901..cd8dd72cd7 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc
@@ -123,7 +123,8 @@ KernelMappingScheme::KernelMappingScheme(
       dims_in_elems_(dims_in_elems.begin(), dims_in_elems.end()),
       tile_sizes_{1, tile_size_y, tile_size_x},
       num_threads_x_(num_threads_x),
-      num_threads_y_(num_threads_y) {
+      num_threads_y_(num_threads_y),
+      dilated_x_(true) {
   DCHECK_EQ(dims_in_elems_.size(), 3);
   DCHECK_EQ(req_block_sizes.size(), 3);
 
diff --git a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h
index fb633b12e6..f802cc27d5 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h
@@ -117,7 +117,10 @@ class KernelMappingScheme {
   int64 GetNumberOfTilesInOneBlock() const {
     return absl::c_accumulate(block_sizes_, 1, std::multiplies<int64>());
   }
-
+  int64 GetNumberOfTilesInOneBlockForDimension(int d) const {
+    DCHECK(d >= DimZ && d <= DimX);
+    return block_sizes_[d];
+  }
   int64 GetNumberOfBlocks() const {
     return absl::c_accumulate(dims_in_blocks_, 1, std::multiplies<int64>());
   }
@@ -147,6 +150,16 @@ class KernelMappingScheme {
            GetNumberOfThreadsForDimensionY();
   }
 
+  bool DilatedX() const { return dilated_x_; }
+  void SetDilatedX(bool v) {
+    dilated_x_ = v;
+    if (!dilated_x_) {
+      // dilated_x_=false is for the purpose of vectorization, which requires
+      // GetTileSizeForDimension(DimX) to be a multiplier of num_threads_x_.
+      CHECK_EQ(GetTileSizeForDimension(DimX) % num_threads_x_, 0);
+    }
+  }
+
   IrArray::Index EmitBlockIndex(llvm::Type* index_ty);
   // Returns the index for the first tile in the block with the given block
   // index.
@@ -186,6 +199,13 @@ class KernelMappingScheme {
   int64 num_threads_x_;
   // Number of threads used to process elements in the Y direction of a tile.
   int64 num_threads_y_;
+
+  // When num_threads_x threads process a total of tile_size_x elements in the
+  // X dimension of a tile, each threads process n=tile_size_x/num_threads_x
+  // elements. When dilated_x=false, the n elements processed by a thread are
+  // contiguous. On the other hand, when dilated_x=true the n elements are
+  // dilated by a factor of num_threads_x.
+  bool dilated_x_;
 };
 
 // A class to represent information for tiled parameters to support IR emission
-- 
GitLab


From 021a8fe8a70b880031958f596b6c96e819e14386 Mon Sep 17 00:00:00 2001
From: Nick Felt <nickfelt@google.com>
Date: Mon, 7 Jan 2019 12:23:54 -0800
Subject: [PATCH 0278/2345] Move contrib/tensorboard/db to new core/summary
 directory

PiperOrigin-RevId: 228213314
---
 tensorflow/contrib/cmake/tf_core_framework.cmake     | 12 ++++++------
 tensorflow/core/kernels/BUILD                        | 10 ++++------
 tensorflow/core/kernels/summary_kernels.cc           |  6 +++---
 .../{contrib/tensorboard/db => core/summary}/BUILD   |  2 +-
 .../tensorboard/db => core/summary}/loader.cc        |  4 ++--
 .../tensorboard/db => core/summary}/schema.cc        |  2 +-
 .../tensorboard/db => core/summary}/schema.h         |  6 +++---
 .../tensorboard/db => core/summary}/schema_test.cc   |  2 +-
 .../db => core/summary}/summary_converter.cc         |  2 +-
 .../db => core/summary}/summary_converter.h          |  6 +++---
 .../db => core/summary}/summary_db_writer.cc         |  4 ++--
 .../db => core/summary}/summary_db_writer.h          |  6 +++---
 .../db => core/summary}/summary_db_writer_test.cc    |  4 ++--
 .../db => core/summary}/summary_file_writer.cc       |  4 ++--
 .../db => core/summary}/summary_file_writer.h        |  6 +++---
 .../db => core/summary}/summary_file_writer_test.cc  |  2 +-
 .../tensorboard/db => core/summary}/vacuum.cc        |  0
 17 files changed, 38 insertions(+), 40 deletions(-)
 rename tensorflow/{contrib/tensorboard/db => core/summary}/BUILD (98%)
 rename tensorflow/{contrib/tensorboard/db => core/summary}/loader.cc (97%)
 rename tensorflow/{contrib/tensorboard/db => core/summary}/schema.cc (99%)
 rename tensorflow/{contrib/tensorboard/db => core/summary}/schema.h (87%)
 rename tensorflow/{contrib/tensorboard/db => core/summary}/schema_test.cc (95%)
 rename tensorflow/{contrib/tensorboard/db => core/summary}/summary_converter.cc (99%)
 rename tensorflow/{contrib/tensorboard/db => core/summary}/summary_converter.h (89%)
 rename tensorflow/{contrib/tensorboard/db => core/summary}/summary_db_writer.cc (99%)
 rename tensorflow/{contrib/tensorboard/db => core/summary}/summary_db_writer.h (89%)
 rename tensorflow/{contrib/tensorboard/db => core/summary}/summary_db_writer_test.cc (99%)
 rename tensorflow/{contrib/tensorboard/db => core/summary}/summary_file_writer.cc (98%)
 rename tensorflow/{contrib/tensorboard/db => core/summary}/summary_file_writer.h (89%)
 rename tensorflow/{contrib/tensorboard/db => core/summary}/summary_file_writer_test.cc (99%)
 rename tensorflow/{contrib/tensorboard/db => core/summary}/vacuum.cc (100%)

diff --git a/tensorflow/contrib/cmake/tf_core_framework.cmake b/tensorflow/contrib/cmake/tf_core_framework.cmake
index d7b2a1339e..9ae5831f47 100644
--- a/tensorflow/contrib/cmake/tf_core_framework.cmake
+++ b/tensorflow/contrib/cmake/tf_core_framework.cmake
@@ -302,8 +302,8 @@ file(GLOB_RECURSE tf_core_framework_srcs
     "${tensorflow_source_dir}/tensorflow/core/common_runtime/session.cc"
     "${tensorflow_source_dir}/tensorflow/core/common_runtime/session_factory.cc"
     "${tensorflow_source_dir}/tensorflow/core/common_runtime/session_options.cc"
-    "${tensorflow_source_dir}/tensorflow/contrib/tensorboard/db/*.cc"
-    "${tensorflow_source_dir}/tensorflow/contrib/tensorboard/db/*.h"
+    "${tensorflow_source_dir}/tensorflow/core/summary/*.cc"
+    "${tensorflow_source_dir}/tensorflow/core/summary/*.h"
     "${tensorflow_source_dir}/public/*.h"
 )
 
@@ -317,14 +317,14 @@ file(GLOB_RECURSE tf_core_framework_exclude_srcs
     "${tensorflow_source_dir}/tensorflow/core/util/*test*.h"
     "${tensorflow_source_dir}/tensorflow/core/util/*test*.cc"
     "${tensorflow_source_dir}/tensorflow/core/util/*main.cc"
-    "${tensorflow_source_dir}/tensorflow/contrib/tensorboard/db/*test*.cc"
-    "${tensorflow_source_dir}/tensorflow/contrib/tensorboard/db/loader.cc"
-    "${tensorflow_source_dir}/tensorflow/contrib/tensorboard/db/vacuum.cc"
+    "${tensorflow_source_dir}/tensorflow/core/summary/*test*.cc"
+    "${tensorflow_source_dir}/tensorflow/core/summary/loader.cc"
+    "${tensorflow_source_dir}/tensorflow/core/summary/vacuum.cc"
 )
 
 # TODO(jart): Why doesn't this work?
 # set_source_files_properties(
-#     ${tensorflow_source_dir}/tensorflow/contrib/tensorboard/db/snapfn.cc
+#     ${tensorflow_source_dir}/tensorflow/core/lib/db/snapfn.cc
 #     PROPERTIES COMPILE_FLAGS -DSQLITE_OMIT_LOAD_EXTENSION)
 
 list(REMOVE_ITEM tf_core_framework_srcs ${tf_core_framework_exclude_srcs})
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 3891caf5d3..136902e249 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -31,7 +31,6 @@ load(
     "//tensorflow:tensorflow.bzl",
     "cc_header_only_library",
     "if_android",
-    "if_not_v2",
     "if_not_windows",
     "tf_cc_binary",
     "tf_cc_test",
@@ -6888,11 +6887,10 @@ tf_kernel_library(
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:summary_ops_op_lib",
         "//tensorflow/core/lib/db:sqlite",
-    ] + if_not_v2([
-        "//tensorflow/contrib/tensorboard/db:schema",
-        "//tensorflow/contrib/tensorboard/db:summary_db_writer",
-        "//tensorflow/contrib/tensorboard/db:summary_file_writer",
-    ]),
+        "//tensorflow/core/summary:schema",
+        "//tensorflow/core/summary:summary_db_writer",
+        "//tensorflow/core/summary:summary_file_writer",
+    ],
 )
 
 tf_kernel_library(
diff --git a/tensorflow/core/kernels/summary_kernels.cc b/tensorflow/core/kernels/summary_kernels.cc
index b287f0cc2f..5e3465d1dd 100644
--- a/tensorflow/core/kernels/summary_kernels.cc
+++ b/tensorflow/core/kernels/summary_kernels.cc
@@ -13,14 +13,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorboard/db/schema.h"
-#include "tensorflow/contrib/tensorboard/db/summary_db_writer.h"
-#include "tensorflow/contrib/tensorboard/db/summary_file_writer.h"
 #include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/lib/db/sqlite.h"
 #include "tensorflow/core/platform/protobuf.h"
+#include "tensorflow/core/summary/schema.h"
+#include "tensorflow/core/summary/summary_db_writer.h"
+#include "tensorflow/core/summary/summary_file_writer.h"
 #include "tensorflow/core/util/event.pb.h"
 
 namespace tensorflow {
diff --git a/tensorflow/contrib/tensorboard/db/BUILD b/tensorflow/core/summary/BUILD
similarity index 98%
rename from tensorflow/contrib/tensorboard/db/BUILD
rename to tensorflow/core/summary/BUILD
index 6507546ee9..a89175cdb1 100644
--- a/tensorflow/contrib/tensorboard/db/BUILD
+++ b/tensorflow/core/summary/BUILD
@@ -1,5 +1,5 @@
 # Description:
-#   TensorBoard database code.
+#   C++ implementation code for the summary writing APIs.
 
 package(default_visibility = ["//tensorflow:internal"])
 
diff --git a/tensorflow/contrib/tensorboard/db/loader.cc b/tensorflow/core/summary/loader.cc
similarity index 97%
rename from tensorflow/contrib/tensorboard/db/loader.cc
rename to tensorflow/core/summary/loader.cc
index 6439328022..68535feacf 100644
--- a/tensorflow/contrib/tensorboard/db/loader.cc
+++ b/tensorflow/core/summary/loader.cc
@@ -15,8 +15,8 @@ limitations under the License.
 #include <iostream>
 #include <vector>
 
-#include "tensorflow/contrib/tensorboard/db/schema.h"
-#include "tensorflow/contrib/tensorboard/db/summary_db_writer.h"
+#include "tensorflow/core/summary/schema.h"
+#include "tensorflow/core/summary/summary_db_writer.h"
 #include "tensorflow/core/lib/db/sqlite.h"
 #include "tensorflow/core/lib/io/record_reader.h"
 #include "tensorflow/core/platform/init_main.h"
diff --git a/tensorflow/contrib/tensorboard/db/schema.cc b/tensorflow/core/summary/schema.cc
similarity index 99%
rename from tensorflow/contrib/tensorboard/db/schema.cc
rename to tensorflow/core/summary/schema.cc
index 3c7bc87e4a..822e2fa3bf 100644
--- a/tensorflow/contrib/tensorboard/db/schema.cc
+++ b/tensorflow/core/summary/schema.cc
@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tensorflow/contrib/tensorboard/db/schema.h"
+#include "tensorflow/core/summary/schema.h"
 
 #include "tensorflow/core/lib/core/errors.h"
 
diff --git a/tensorflow/contrib/tensorboard/db/schema.h b/tensorflow/core/summary/schema.h
similarity index 87%
rename from tensorflow/contrib/tensorboard/db/schema.h
rename to tensorflow/core/summary/schema.h
index 3da4504225..6305f8eabd 100644
--- a/tensorflow/contrib/tensorboard/db/schema.h
+++ b/tensorflow/core/summary/schema.h
@@ -12,8 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#ifndef TENSORFLOW_CONTRIB_TENSORBOARD_DB_SCHEMA_H_
-#define TENSORFLOW_CONTRIB_TENSORBOARD_DB_SCHEMA_H_
+#ifndef TENSORFLOW_CORE_SUMMARY_SCHEMA_H_
+#define TENSORFLOW_CORE_SUMMARY_SCHEMA_H_
 
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/db/sqlite.h"
@@ -30,4 +30,4 @@ Status SetupTensorboardSqliteDb(Sqlite* db);
 
 }  // namespace tensorflow
 
-#endif  // TENSORFLOW_CONTRIB_TENSORBOARD_DB_SCHEMA_H_
+#endif  // TENSORFLOW_CORE_SUMMARY_SCHEMA_H_
diff --git a/tensorflow/contrib/tensorboard/db/schema_test.cc b/tensorflow/core/summary/schema_test.cc
similarity index 95%
rename from tensorflow/contrib/tensorboard/db/schema_test.cc
rename to tensorflow/core/summary/schema_test.cc
index 4d3f2880bd..fa21b45b62 100644
--- a/tensorflow/contrib/tensorboard/db/schema_test.cc
+++ b/tensorflow/core/summary/schema_test.cc
@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tensorflow/contrib/tensorboard/db/schema.h"
+#include "tensorflow/core/summary/schema.h"
 
 #include <memory>
 
diff --git a/tensorflow/contrib/tensorboard/db/summary_converter.cc b/tensorflow/core/summary/summary_converter.cc
similarity index 99%
rename from tensorflow/contrib/tensorboard/db/summary_converter.cc
rename to tensorflow/core/summary/summary_converter.cc
index 93c1183072..e6e34e9602 100644
--- a/tensorflow/contrib/tensorboard/db/summary_converter.cc
+++ b/tensorflow/core/summary/summary_converter.cc
@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tensorflow/contrib/tensorboard/db/summary_converter.h"
+#include "tensorflow/core/summary/summary_converter.h"
 
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/summary.pb.h"
diff --git a/tensorflow/contrib/tensorboard/db/summary_converter.h b/tensorflow/core/summary/summary_converter.h
similarity index 89%
rename from tensorflow/contrib/tensorboard/db/summary_converter.h
rename to tensorflow/core/summary/summary_converter.h
index 329c7f9f2f..dc005d2604 100644
--- a/tensorflow/contrib/tensorboard/db/summary_converter.h
+++ b/tensorflow/core/summary/summary_converter.h
@@ -12,8 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#ifndef TENSORFLOW_CONTRIB_TENSORBOARD_DB_SUMMARY_CONVERTER_H_
-#define TENSORFLOW_CONTRIB_TENSORBOARD_DB_SUMMARY_CONVERTER_H_
+#ifndef TENSORFLOW_CORE_SUMMARY_SUMMARY_CONVERTER_H_
+#define TENSORFLOW_CORE_SUMMARY_SUMMARY_CONVERTER_H_
 
 #include "tensorflow/core/framework/summary.pb.h"
 #include "tensorflow/core/framework/tensor.h"
@@ -35,4 +35,4 @@ Status AddTensorAsAudioToSummary(const Tensor& tensor, const string& tag,
 
 }  // namespace tensorflow
 
-#endif  // TENSORFLOW_CONTRIB_TENSORBOARD_DB_SUMMARY_CONVERTER_H_
+#endif  // TENSORFLOW_CORE_SUMMARY_SUMMARY_CONVERTER_H_
diff --git a/tensorflow/contrib/tensorboard/db/summary_db_writer.cc b/tensorflow/core/summary/summary_db_writer.cc
similarity index 99%
rename from tensorflow/contrib/tensorboard/db/summary_db_writer.cc
rename to tensorflow/core/summary/summary_db_writer.cc
index cfdc884277..7a5d796821 100644
--- a/tensorflow/contrib/tensorboard/db/summary_db_writer.cc
+++ b/tensorflow/core/summary/summary_db_writer.cc
@@ -12,11 +12,11 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tensorflow/contrib/tensorboard/db/summary_db_writer.h"
+#include "tensorflow/core/summary/summary_db_writer.h"
 
 #include <deque>
 
-#include "tensorflow/contrib/tensorboard/db/summary_converter.h"
+#include "tensorflow/core/summary/summary_converter.h"
 #include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/register_types.h"
diff --git a/tensorflow/contrib/tensorboard/db/summary_db_writer.h b/tensorflow/core/summary/summary_db_writer.h
similarity index 89%
rename from tensorflow/contrib/tensorboard/db/summary_db_writer.h
rename to tensorflow/core/summary/summary_db_writer.h
index 746da1533b..5669afe7f6 100644
--- a/tensorflow/contrib/tensorboard/db/summary_db_writer.h
+++ b/tensorflow/core/summary/summary_db_writer.h
@@ -12,8 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#ifndef TENSORFLOW_CONTRIB_TENSORBOARD_DB_SUMMARY_DB_WRITER_H_
-#define TENSORFLOW_CONTRIB_TENSORBOARD_DB_SUMMARY_DB_WRITER_H_
+#ifndef TENSORFLOW_CORE_SUMMARY_SUMMARY_DB_WRITER_H_
+#define TENSORFLOW_CORE_SUMMARY_SUMMARY_DB_WRITER_H_
 
 #include "tensorflow/core/kernels/summary_interface.h"
 #include "tensorflow/core/lib/core/status.h"
@@ -39,4 +39,4 @@ Status CreateSummaryDbWriter(Sqlite* db, const string& experiment_name,
 
 }  // namespace tensorflow
 
-#endif  // TENSORFLOW_CONTRIB_TENSORBOARD_DB_SUMMARY_DB_WRITER_H_
+#endif  // TENSORFLOW_CORE_SUMMARY_SUMMARY_DB_WRITER_H_
diff --git a/tensorflow/contrib/tensorboard/db/summary_db_writer_test.cc b/tensorflow/core/summary/summary_db_writer_test.cc
similarity index 99%
rename from tensorflow/contrib/tensorboard/db/summary_db_writer_test.cc
rename to tensorflow/core/summary/summary_db_writer_test.cc
index 2e8d4109dd..c4e9ddea2c 100644
--- a/tensorflow/contrib/tensorboard/db/summary_db_writer_test.cc
+++ b/tensorflow/core/summary/summary_db_writer_test.cc
@@ -12,9 +12,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tensorflow/contrib/tensorboard/db/summary_db_writer.h"
+#include "tensorflow/core/summary/summary_db_writer.h"
 
-#include "tensorflow/contrib/tensorboard/db/schema.h"
+#include "tensorflow/core/summary/schema.h"
 #include "tensorflow/core/framework/function.pb.h"
 #include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
diff --git a/tensorflow/contrib/tensorboard/db/summary_file_writer.cc b/tensorflow/core/summary/summary_file_writer.cc
similarity index 98%
rename from tensorflow/contrib/tensorboard/db/summary_file_writer.cc
rename to tensorflow/core/summary/summary_file_writer.cc
index 22b6f09d0c..593ccdd684 100644
--- a/tensorflow/contrib/tensorboard/db/summary_file_writer.cc
+++ b/tensorflow/core/summary/summary_file_writer.cc
@@ -12,9 +12,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tensorflow/contrib/tensorboard/db/summary_file_writer.h"
+#include "tensorflow/core/summary/summary_file_writer.h"
 
-#include "tensorflow/contrib/tensorboard/db/summary_converter.h"
+#include "tensorflow/core/summary/summary_converter.h"
 #include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/resource_mgr.h"
diff --git a/tensorflow/contrib/tensorboard/db/summary_file_writer.h b/tensorflow/core/summary/summary_file_writer.h
similarity index 89%
rename from tensorflow/contrib/tensorboard/db/summary_file_writer.h
rename to tensorflow/core/summary/summary_file_writer.h
index 73b0a5542b..7d964516da 100644
--- a/tensorflow/contrib/tensorboard/db/summary_file_writer.h
+++ b/tensorflow/core/summary/summary_file_writer.h
@@ -12,8 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#ifndef TENSORFLOW_CONTRIB_TENSORBOARD_DB_SUMMARY_FILE_WRITER_H_
-#define TENSORFLOW_CONTRIB_TENSORBOARD_DB_SUMMARY_FILE_WRITER_H_
+#ifndef TENSORFLOW_CORE_SUMMARY_SUMMARY_FILE_WRITER_H_
+#define TENSORFLOW_CORE_SUMMARY_SUMMARY_FILE_WRITER_H_
 
 #include "tensorflow/core/kernels/summary_interface.h"
 #include "tensorflow/core/lib/core/status.h"
@@ -40,4 +40,4 @@ Status CreateSummaryFileWriter(int max_queue, int flush_millis,
 
 }  // namespace tensorflow
 
-#endif  // TENSORFLOW_CONTRIB_TENSORBOARD_DB_SUMMARY_FILE_WRITER_H_
+#endif  // TENSORFLOW_CORE_SUMMARY_SUMMARY_FILE_WRITER_H_
diff --git a/tensorflow/contrib/tensorboard/db/summary_file_writer_test.cc b/tensorflow/core/summary/summary_file_writer_test.cc
similarity index 99%
rename from tensorflow/contrib/tensorboard/db/summary_file_writer_test.cc
rename to tensorflow/core/summary/summary_file_writer_test.cc
index ffbfb9533e..d3b19c3abd 100644
--- a/tensorflow/contrib/tensorboard/db/summary_file_writer_test.cc
+++ b/tensorflow/core/summary/summary_file_writer_test.cc
@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tensorflow/contrib/tensorboard/db/summary_file_writer.h"
+#include "tensorflow/core/summary/summary_file_writer.h"
 
 #include "tensorflow/core/framework/summary.pb.h"
 #include "tensorflow/core/framework/tensor.pb.h"
diff --git a/tensorflow/contrib/tensorboard/db/vacuum.cc b/tensorflow/core/summary/vacuum.cc
similarity index 100%
rename from tensorflow/contrib/tensorboard/db/vacuum.cc
rename to tensorflow/core/summary/vacuum.cc
-- 
GitLab


From bd2610f638bd35dca30bb00a6c16da775b3bfcdf Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Mon, 7 Jan 2019 12:31:43 -0800
Subject: [PATCH 0279/2345] [Grappler] Remapper for GPU
 Conv2d+BiasAdd+Activation

PiperOrigin-RevId: 228214620
---
 tensorflow/core/grappler/BUILD                |   1 +
 tensorflow/core/grappler/graph_view.cc        |  23 +-
 tensorflow/core/grappler/graph_view.h         |  10 +-
 tensorflow/core/grappler/optimizers/BUILD     |   2 +
 .../core/grappler/optimizers/remapper.cc      | 384 ++++++++++++------
 tensorflow/core/grappler/utils.cc             |  18 +-
 tensorflow/core/grappler/utils.h              |   3 +
 tensorflow/core/grappler/utils/BUILD          |   1 +
 .../core/grappler/utils/grappler_test.cc      |  27 ++
 .../core/grappler/utils/grappler_test.h       |  16 +-
 tensorflow/core/kernels/conv_ops_test.cc      |  20 +
 11 files changed, 353 insertions(+), 152 deletions(-)

diff --git a/tensorflow/core/grappler/BUILD b/tensorflow/core/grappler/BUILD
index 4d0f02f4d6..55d642612b 100644
--- a/tensorflow/core/grappler/BUILD
+++ b/tensorflow/core/grappler/BUILD
@@ -27,6 +27,7 @@ cc_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
+        "@com_google_absl//absl/strings",
         "@com_google_absl//absl/types:span",
     ],
 )
diff --git a/tensorflow/core/grappler/graph_view.cc b/tensorflow/core/grappler/graph_view.cc
index ba9d2eb321..be9b9c36c7 100644
--- a/tensorflow/core/grappler/graph_view.cc
+++ b/tensorflow/core/grappler/graph_view.cc
@@ -66,28 +66,27 @@ int OpInputPortIdToArgId(const NodeDef& node, const OpDef& op, int port_id) {
 bool HasSingleFanoutNode(const GraphView& graph_view, const NodeDef* node,
                          int port) {
   const auto output = GraphView::OutputPort(node, port);
-  const auto fanout = graph_view.GetFanout(output);
-  return fanout.size() <= 1;
+  return graph_view.GetFanout(output).size() <= 1;
 }
 
 bool HasFanouts(const GraphView& graph_view, const NodeDef* node, int port) {
   const auto output = GraphView::OutputPort(node, port);
-  const auto fanout = graph_view.GetFanout(output);
-  return !fanout.empty();
+  return !graph_view.GetFanout(output).empty();
 }
 
-bool NoControlFanin(const GraphView& graph_view, const NodeDef* node) {
-  const auto control_port = GraphView::InputPort(node, -1);
-  return graph_view.GetFanin(control_port).empty();
+bool HasControlFanin(const GraphView& graph_view, const NodeDef* node) {
+  const auto control_port = GraphView::InputPort(node, Graph::kControlSlot);
+  return !graph_view.GetFanin(control_port).empty();
 }
 
-bool NoControlFanout(const GraphView& graph_view, const NodeDef* node) {
-  const auto control_port = GraphView::OutputPort(node, -1);
-  return graph_view.GetFanout(control_port).empty();
+bool HasControlFanout(const GraphView& graph_view, const NodeDef* node) {
+  const auto control_port = GraphView::OutputPort(node, Graph::kControlSlot);
+  return !graph_view.GetFanout(control_port).empty();
 }
 
-bool NoControlFaninOrFanout(const GraphView& graph_view, const NodeDef* node) {
-  return NoControlFanin(graph_view, node) && NoControlFanout(graph_view, node);
+bool HasControlFaninOrFanout(const GraphView& graph_view, const NodeDef* node) {
+  return HasControlFanin(graph_view, node) ||
+         HasControlFanout(graph_view, node);
 }
 
 }  // end namespace grappler
diff --git a/tensorflow/core/grappler/graph_view.h b/tensorflow/core/grappler/graph_view.h
index a17d17524a..dc4ab93894 100644
--- a/tensorflow/core/grappler/graph_view.h
+++ b/tensorflow/core/grappler/graph_view.h
@@ -369,10 +369,12 @@ bool HasSingleFanoutNode(const GraphView& graph_view, const NodeDef* node,
 
 // Returns true if node has at least one fanout node at given output port.
 bool HasFanouts(const GraphView& graph_view, const NodeDef* node, int port = 0);
-
-bool NoControlFanin(const GraphView& graph_view, const NodeDef* node);
-bool NoControlFanout(const GraphView& graph_view, const NodeDef* node);
-bool NoControlFaninOrFanout(const GraphView& graph_view, const NodeDef* node);
+// Returns true if the node has at least one input control dependency.
+bool HasControlFanin(const GraphView& graph_view, const NodeDef* node);
+// Returns true if the node has at least one output control dependency.
+bool HasControlFanout(const GraphView& graph_view, const NodeDef* node);
+// Returns true if the node has at least one input or output control dependency.
+bool HasControlFaninOrFanout(const GraphView& graph_view, const NodeDef* node);
 
 }  // end namespace grappler
 }  // end namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index e2b9de7cc5..7e29cee86a 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -700,6 +700,7 @@ cc_library(
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/costs:graph_properties",
+        "//tensorflow/core/grappler/utils:symbolic_shapes",
         "//tensorflow/core/grappler/utils:topological_sort",
         "@com_google_absl//absl/container:flat_hash_set",
     ],
@@ -711,6 +712,7 @@ tf_cuda_cc_test(
     deps = [
         ":remapper",
         "//tensorflow/cc:cc_ops",
+        "//tensorflow/core:framework",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
diff --git a/tensorflow/core/grappler/optimizers/remapper.cc b/tensorflow/core/grappler/optimizers/remapper.cc
index f0c81f29e6..0869e3b49b 100644
--- a/tensorflow/core/grappler/optimizers/remapper.cc
+++ b/tensorflow/core/grappler/optimizers/remapper.cc
@@ -23,6 +23,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/grappler/optimizers/constant_folding.h"
 #include "tensorflow/core/grappler/utils.h"
+#include "tensorflow/core/grappler/utils/symbolic_shapes.h"
 #include "tensorflow/core/grappler/utils/topological_sort.h"
 #include "tensorflow/core/platform/logging.h"
 
@@ -60,17 +61,30 @@ struct RemapperContext {
 
 // FusedBatchNorm that can be replaced with a cheaper set of primitives.
 struct FusedBatchNorm {
+  FusedBatchNorm() = default;
+  explicit FusedBatchNorm(const NodeDef* fused_batch_norm)
+      : fused_batch_norm(fused_batch_norm) {}
+
   const NodeDef* fused_batch_norm = nullptr;
 };
 
 // Conv2D node followed by a BiasAdd.
 struct Conv2DWithBiasAdd {
+  Conv2DWithBiasAdd() = default;
+  Conv2DWithBiasAdd(const NodeDef* conv2d, const NodeDef* bias_add)
+      : conv2d(conv2d), bias_add(bias_add) {}
+
   const NodeDef* conv2d = nullptr;
   const NodeDef* bias_add = nullptr;
 };
 
 // Conv2D node followed by a BiasAdd and Relu.
 struct Conv2DWithBiasAddAndRelu {
+  Conv2DWithBiasAddAndRelu() = default;
+  Conv2DWithBiasAddAndRelu(const NodeDef* conv2d, const NodeDef* bias_add,
+                           const NodeDef* relu)
+      : conv2d(conv2d), bias_add(bias_add), relu(relu) {}
+
   const NodeDef* conv2d = nullptr;
   const NodeDef* bias_add = nullptr;
   const NodeDef* relu = nullptr;
@@ -78,6 +92,11 @@ struct Conv2DWithBiasAddAndRelu {
 
 // Conv2D node followed by a Squeeze and BiasAdd.
 struct Conv2DWithSqueezeAndBiasAdd {
+  Conv2DWithSqueezeAndBiasAdd() = default;
+  Conv2DWithSqueezeAndBiasAdd(const NodeDef* conv2d, const NodeDef* squeeze,
+                              const NodeDef* bias_add)
+      : conv2d(conv2d), squeeze(squeeze), bias_add(bias_add) {}
+
   const NodeDef* conv2d = nullptr;
   const NodeDef* squeeze = nullptr;
   const NodeDef* bias_add = nullptr;
@@ -85,6 +104,11 @@ struct Conv2DWithSqueezeAndBiasAdd {
 
 // Conv2D node followed by a FusedBatchNorm.
 struct Conv2DWithBatchNorm {
+  Conv2DWithBatchNorm() = default;
+  Conv2DWithBatchNorm(const NodeDef* conv2d, const NodeDef* fused_batch_norm,
+                      float epsilon = 0.0)
+      : conv2d(conv2d), fused_batch_norm(fused_batch_norm), epsilon(epsilon) {}
+
   const NodeDef* conv2d = nullptr;
   const NodeDef* fused_batch_norm = nullptr;
   float epsilon = 0.0;
@@ -92,16 +116,23 @@ struct Conv2DWithBatchNorm {
 
 // Conv2D node followed by a FusedBatchNorm and Relu.
 struct Conv2DWithBatchNormAndRelu {
+  Conv2DWithBatchNormAndRelu() = default;
+  Conv2DWithBatchNormAndRelu(const NodeDef* conv2d,
+                             const NodeDef* fused_batch_norm,
+                             const NodeDef* relu, float epsilon = 0.0)
+      : conv2d(conv2d),
+        fused_batch_norm(fused_batch_norm),
+        relu(relu),
+        epsilon(epsilon) {}
+
   const NodeDef* conv2d = nullptr;
   const NodeDef* fused_batch_norm = nullptr;
   const NodeDef* relu = nullptr;
   float epsilon = 0.0;
 };
 
-bool IsFloatOrDoubleDataType(const NodeDef* node,
-                             const string& type_attr = "T") {
-  DataType dtype = GetDataTypeFromAttr(*node, type_attr);
-  return dtype == DT_FLOAT || dtype == DT_DOUBLE;
+bool IsInPreserveSet(const RemapperContext& ctx, const NodeDef* node) {
+  return ctx.nodes_to_preserve.count(node->name()) > 0;
 }
 
 bool HaveSameDataType(const NodeDef* lhs, const NodeDef* rhs,
@@ -119,91 +150,165 @@ bool HasDataType(const NodeDef* node, const DataType& expected,
   return dtype == expected;
 }
 
-bool IsInPreserveSet(const RemapperContext& ctx, const NodeDef* node) {
-  return ctx.nodes_to_preserve.count(node->name()) > 0;
+bool IsCpuCompatibleDataType(const NodeDef* node,
+                             const string& type_attr = "T") {
+  DataType dtype = GetDataTypeFromAttr(*node, type_attr);
+  return dtype == DT_FLOAT || dtype == DT_DOUBLE;
+}
+
+bool IsGpuCompatibleDataType(const NodeDef* node,
+                             const string& type_attr = "T") {
+  DataType dtype = GetDataTypeFromAttr(*node, type_attr);
+  return dtype == DT_FLOAT;
+}
+
+bool IsCpuCompatibleDataFormat(const NodeDef* conv2d) {
+  DCHECK(IsConv2D(*conv2d)) << "Expected Conv2D op";
+  const string& data_format = conv2d->attr().at(kDataFormat).s();
+  return data_format == "NHWC";
+}
+
+bool IsGpuCompatibleDataFormat(const NodeDef* conv2d) {
+  DCHECK(IsConv2D(*conv2d)) << "Expected Conv2D op";
+  const string& data_format = conv2d->attr().at(kDataFormat).s();
+  return data_format == "NHWC" || data_format == "NCHW";
 }
 
-bool FindConv2DWithBias(const RemapperContext& ctx, const NodeDef* node,
-                        Conv2DWithBiasAdd* matched) {
+bool IsCpuCompatibleConv2D(const NodeDef* conv2d) {
+  DCHECK(IsConv2D(*conv2d)) << "Expected Conv2D op";
+  return NodeIsOnCpu(conv2d) && IsCpuCompatibleDataType(conv2d) &&
+         IsCpuCompatibleDataFormat(conv2d);
+}
+
+bool IsGpuCompatibleConv2D(const NodeDef* conv2d) {
+  DCHECK(IsConv2D(*conv2d)) << "Expected Conv2D op";
+  return NodeIsOnGpu(conv2d) && IsGpuCompatibleDataType(conv2d) &&
+         IsGpuCompatibleDataFormat(conv2d);
+}
+
+// Checks if we can rewrite a pattern to the `_FusedConv2D` on CPU device.
+template <typename Pattern>
+bool IsCpuCompatible(const Pattern& matched) {
+  return IsCpuCompatibleConv2D(matched.conv2d);
+}
+
+// Checks if we can rewrite a pattern to the `_FusedConv2D` on GPU device.
+bool IsGpuCompatible(const RemapperContext& ctx,
+                     const Conv2DWithBiasAddAndRelu& matched) {
+  const std::vector<OpInfo::TensorProperties>& input_props =
+      ctx.graph_properties.GetInputProperties(matched.conv2d->name());
+  const TensorShapeProto& filter_shape =
+      input_props.size() >= 2 ? input_props[1].shape() : TensorShapeProto();
+
+  // FusedConv2D on GPU with 1x1 convolution is marginally faster than
+  // in-graph computation in micro benchmarks (see kernels/conv_ops_test.cc),
+  // and significantly slower in large scale benchmarks.
+  bool is_spatial_conv = Rank(filter_shape) == 4 &&          //
+                         IsKnown(filter_shape.dim(1)) &&     //
+                         IsKnown(filter_shape.dim(2)) &&     //
+                         filter_shape.dim(1).size() != 1 &&  //
+                         filter_shape.dim(2).size() != 1;
+
+  return is_spatial_conv && IsGpuCompatibleConv2D(matched.conv2d);
+}
+bool IsGpuCompatible(const RemapperContext& ctx,
+                     const Conv2DWithBiasAdd& matched) {
+  return false;
+}
+bool IsGpuCompatible(const RemapperContext& ctx,
+                     const Conv2DWithSqueezeAndBiasAdd& matched) {
+  return false;
+}
+
+// Returns true if the given pattern is supported on the assigned device.
+template <typename Pattern>
+bool IsDeviceCompatible(const RemapperContext& ctx, Pattern& matched) {
+  return IsCpuCompatible(matched) || IsGpuCompatible(ctx, matched);
+}
+
+bool FindConv2DWithBias(const RemapperContext& ctx, const NodeDef* bias_add,
+                        Conv2DWithBiasAdd* matched,
+                        bool check_device_compatible = true) {
   if (!EigenSupportsContractionOutputKernel()) return false;
 
   // Root of the pattern must be a BiasAdd.
-  if (!node) return false;
-  if (!IsBiasAdd(*node)) return false;
-  if (!NodeIsOnCpu(node)) return false;
-  if (!IsFloatOrDoubleDataType(node)) return false;
-  if (!NoControlFaninOrFanout(ctx.graph_view, node)) return false;
+  if (bias_add == nullptr || !IsBiasAdd(*bias_add) ||
+      HasControlFaninOrFanout(ctx.graph_view, bias_add))
+    return false;
 
-  // Input to the BiasAdd must be a Conv2D in NHWC format.
-  const auto input_port = GraphView::InputPort(node, 0);
+  // Input to the BiasAdd must be a Conv2D.
+  const auto input_port = GraphView::InputPort(bias_add, 0);
   const auto conv2d = ctx.graph_view.GetRegularFanin(input_port);
-  if (!conv2d.node) return false;
-  if (!IsConv2D(*conv2d.node)) return false;
-  if (conv2d.node->attr().at(kDataFormat).s() != "NHWC") return false;
-  if (!NodeIsOnCpu(conv2d.node)) return false;
-  if (!HaveSameDataType(node, conv2d.node)) return false;
-  if (!NoControlFaninOrFanout(ctx.graph_view, conv2d.node)) return false;
-  if (!HasSingleFanoutNode(ctx.graph_view, conv2d.node)) return false;
-  if (IsInPreserveSet(ctx, conv2d.node)) return false;
+
+  if (!conv2d.node || !IsConv2D(*conv2d.node) ||
+      !HaveSameDataType(bias_add, conv2d.node) ||
+      HasControlFaninOrFanout(ctx.graph_view, conv2d.node) ||
+      !HasSingleFanoutNode(ctx.graph_view, conv2d.node) ||
+      IsInPreserveSet(ctx, conv2d.node))
+    return false;
+
+  // Check that data type and data format are supported on assigned device.
+  const Conv2DWithBiasAdd pattern{conv2d.node, bias_add};
+  if (check_device_compatible && !IsDeviceCompatible(ctx, pattern)) {
+    return false;
+  }
 
   // We successfully found a Conv2D+BiasAdd pattern.
-  matched->conv2d = conv2d.node;
-  matched->bias_add = node;
+  *matched = pattern;
 
   return true;
 }
 
-bool FindConv2DWithBiasAndRelu(const RemapperContext& ctx, const NodeDef* node,
+bool FindConv2DWithBiasAndRelu(const RemapperContext& ctx, const NodeDef* relu,
                                Conv2DWithBiasAddAndRelu* matched) {
   if (!EigenSupportsContractionOutputKernel()) return false;
 
   // Root of the pattern must be a Relu.
-  if (!node) return false;
-  if (!IsRelu(*node)) return false;
-  if (!NodeIsOnCpu(node)) return false;
-  if (!IsFloatOrDoubleDataType(node)) return false;
-  if (!NoControlFaninOrFanout(ctx.graph_view, node)) return false;
+  if (!relu || !IsRelu(*relu) || HasControlFaninOrFanout(ctx.graph_view, relu))
+    return false;
 
   // And input to Relu must match Conv2DWithBiasAdd pattern.
-  const auto input_port = GraphView::InputPort(node, 0);
+  const auto input_port = GraphView::InputPort(relu, 0);
   const auto bias_add = ctx.graph_view.GetRegularFanin(input_port);
 
   Conv2DWithBiasAdd base;
-  if (!FindConv2DWithBias(ctx, bias_add.node, &base)) return false;
-  if (!HasSingleFanoutNode(ctx.graph_view, base.bias_add)) return false;
-  if (!HaveSameDataType(node, base.bias_add)) return false;
-  if (IsInPreserveSet(ctx, base.bias_add)) return false;
+  if (!FindConv2DWithBias(ctx, bias_add.node, &base,
+                          /*check_device_compatible=*/false) ||
+      !HasSingleFanoutNode(ctx.graph_view, base.bias_add) ||
+      !HaveSameDataType(relu, base.bias_add) ||
+      IsInPreserveSet(ctx, base.bias_add))
+    return false;
+
+  // Check that data type and data format are supported on assigned device.
+  const Conv2DWithBiasAddAndRelu pattern{base.conv2d, base.bias_add, relu};
+  if (!IsDeviceCompatible(ctx, pattern)) return false;
 
   // We successfully found a Conv2D+BiasAdd+Relu pattern.
-  matched->conv2d = base.conv2d;
-  matched->bias_add = base.bias_add;
-  matched->relu = node;
+  *matched = pattern;
 
   return true;
 }
 
 bool FindConv2DWithSqueezeAndBias(const RemapperContext& ctx,
-                                  const NodeDef* node,
+                                  const NodeDef* bias_add,
                                   Conv2DWithSqueezeAndBiasAdd* matched) {
   if (!EigenSupportsContractionOutputKernel()) return false;
 
   // Root of the pattern must be a BiasAdd.
-  if (node == nullptr) return false;
-  if (node->op() != "BiasAdd") return false;
-  if (!NodeIsOnCpu(node)) return false;
-  if (!IsFloatOrDoubleDataType(node)) return false;
-  if (!NoControlFaninOrFanout(ctx.graph_view, node)) return false;
+  if (!bias_add || !IsBiasAdd(*bias_add) ||
+      HasControlFaninOrFanout(ctx.graph_view, bias_add))
+    return false;
 
   // Input to the BiasAdd must be a Squeeze.
-  const auto bias_input_port = GraphView::InputPort(node, 0);
+  const auto bias_input_port = GraphView::InputPort(bias_add, 0);
   const auto squeeze = ctx.graph_view.GetRegularFanin(bias_input_port);
-  if (squeeze.node == nullptr) return false;
-  if (squeeze.node->op() != "Squeeze") return false;
-  if (!NodeIsOnCpu(squeeze.node)) return false;
-  if (!HaveSameDataType(node, squeeze.node, "T")) return false;
-  if (!NoControlFaninOrFanout(ctx.graph_view, squeeze.node)) return false;
-  if (!HasSingleFanoutNode(ctx.graph_view, squeeze.node)) return false;
-  if (IsInPreserveSet(ctx, squeeze.node)) return false;
+
+  if (!squeeze.node || !IsSqueeze(*squeeze.node) ||
+      !HaveSameDataType(bias_add, squeeze.node, "T") ||
+      HasControlFaninOrFanout(ctx.graph_view, squeeze.node) ||
+      !HasSingleFanoutNode(ctx.graph_view, squeeze.node) ||
+      IsInPreserveSet(ctx, squeeze.node))
+    return false;
 
   // Squeeze must not squeeze output channel dimension.
   std::vector<int32> dims;
@@ -212,67 +317,72 @@ bool FindConv2DWithSqueezeAndBias(const RemapperContext& ctx,
     if (dim == 3) return false;
   }
 
-  // Input to the Squeeze must be a Conv2D in NHWC format.
+  // Input to the Squeeze must be a Conv2D.
   const auto squeeze_input_port = GraphView::InputPort(squeeze.node, 0);
   const auto conv2d = ctx.graph_view.GetRegularFanin(squeeze_input_port);
-  if (conv2d.node == nullptr) return false;
-  if (conv2d.node->op() != "Conv2D") return false;
-  if (conv2d.node->attr().at("data_format").s() != "NHWC") return false;
-  if (!NodeIsOnCpu(conv2d.node)) return false;
-  if (!HaveSameDataType(node, conv2d.node, "T")) return false;
-  if (!NoControlFaninOrFanout(ctx.graph_view, conv2d.node)) return false;
-  if (!HasSingleFanoutNode(ctx.graph_view, conv2d.node)) return false;
-  if (IsInPreserveSet(ctx, conv2d.node)) return false;
+
+  if (!conv2d.node || !IsConv2D(*conv2d.node) ||
+      !HaveSameDataType(bias_add, conv2d.node, "T") ||
+      HasControlFaninOrFanout(ctx.graph_view, conv2d.node) ||
+      !HasSingleFanoutNode(ctx.graph_view, conv2d.node) ||
+      IsInPreserveSet(ctx, conv2d.node))
+    return false;
+
+  // Check that data type and data format are supported on assigned device.
+  const Conv2DWithSqueezeAndBiasAdd pattern{conv2d.node, squeeze.node,
+                                            bias_add};
+  if (!IsDeviceCompatible(ctx, pattern)) return false;
 
   // We successfully found a Conv2D+Squeeze+BiasAdd pattern.
-  matched->conv2d = conv2d.node;
-  matched->squeeze = squeeze.node;
-  matched->bias_add = node;
+  *matched = pattern;
 
   return true;
 }
 
-bool FindConv2DWithBatchNorm(const RemapperContext& ctx, const NodeDef* node,
+bool FindConv2DWithBatchNorm(const RemapperContext& ctx,
+                             const NodeDef* batch_norm,
                              Conv2DWithBatchNorm* matched) {
   if (!EigenSupportsContractionOutputKernel()) return false;
 
   // Root of the pattern must be a FusedBatchNorm or a FusedBatchNormV2.
-  if (node == nullptr) return false;
-  if (!IsFusedBatchNorm(*node)) return false;
-  if (!NodeIsOnCpu(node)) return false;
-  if (!HasDataType(node, DT_FLOAT)) return false;
+  if (!batch_norm || !IsFusedBatchNorm(*batch_norm)) return false;
 
   // V2 has a separate data type for the scale/offset/mean/variance inputs.
-  if (node->op() == "FusedBatchNormV2" && !HasDataType(node, DT_FLOAT, "U"))
+  if (batch_norm->op() == "FusedBatchNormV2" &&
+      !HasDataType(batch_norm, DT_FLOAT, "U"))
     return false;
 
   // Check that batch normalization is in inference mode.
-  const auto& attr = node->attr();
+  const auto& attr = batch_norm->attr();
   if (attr.count(kIsTraining) > 0 && attr.at(kIsTraining).b()) return false;
 
   // Check that only 0th output is consumed by other nodes.
-  if (!NoControlFaninOrFanout(ctx.graph_view, node)) return false;
-  if (HasFanouts(ctx.graph_view, node, 1)) return false;  // batch_mean
-  if (HasFanouts(ctx.graph_view, node, 2)) return false;  // batch_variance
-  if (HasFanouts(ctx.graph_view, node, 3)) return false;  // reserve_space_1
-  if (HasFanouts(ctx.graph_view, node, 4)) return false;  // reserve_space_2
+  if (HasControlFaninOrFanout(ctx.graph_view, batch_norm) ||
+      HasFanouts(ctx.graph_view, batch_norm, 1) ||  // batch_mean
+      HasFanouts(ctx.graph_view, batch_norm, 2) ||  // batch_variance
+      HasFanouts(ctx.graph_view, batch_norm, 3) ||  // reserve_space_1
+      HasFanouts(ctx.graph_view, batch_norm, 4))    // reserve_space_2
+    return false;
 
-  // Input to the FusedBatchNorm must be a Conv2D in NHWC format.
-  const auto input_port = GraphView::InputPort(node, 0);
+  // Input to the FusedBatchNorm must be a Conv2D.
+  const auto input_port = GraphView::InputPort(batch_norm, 0);
   const auto conv2d = ctx.graph_view.GetRegularFanin(input_port);
-  if (conv2d.node == nullptr) return false;
-  if (!IsConv2D(*conv2d.node)) return false;
-  if (conv2d.node->attr().at(kDataFormat).s() != "NHWC") return false;
-  if (!NodeIsOnCpu(conv2d.node)) return false;
-  if (!HaveSameDataType(node, conv2d.node)) return false;
-  if (!NoControlFaninOrFanout(ctx.graph_view, conv2d.node)) return false;
-  if (!HasSingleFanoutNode(ctx.graph_view, conv2d.node)) return false;
-  if (IsInPreserveSet(ctx, conv2d.node)) return false;
+
+  if (!conv2d.node || !IsConv2D(*conv2d.node) ||               //
+      !NodeIsOnCpu(conv2d.node) ||                             //
+      !HaveSameDataType(batch_norm, conv2d.node) ||            //
+      !IsCpuCompatibleDataType(conv2d.node) ||                 //
+      !IsCpuCompatibleDataFormat(conv2d.node) ||               //
+      HasControlFaninOrFanout(ctx.graph_view, conv2d.node) ||  //
+      !HasSingleFanoutNode(ctx.graph_view, conv2d.node) ||     //
+      IsInPreserveSet(ctx, conv2d.node))
+    return false;
 
   // We successfully found a Conv2D+FusedBatchNorm pattern.
   matched->conv2d = conv2d.node;
-  matched->fused_batch_norm = node;
-  if (!GetNodeAttr(*node, "epsilon", &matched->epsilon).ok()) return false;
+  matched->fused_batch_norm = batch_norm;
+  if (!GetNodeAttr(*batch_norm, "epsilon", &matched->epsilon).ok())
+    return false;
 
   return true;
 }
@@ -283,21 +393,19 @@ bool FindConv2DWithBatchNormAndRelu(const RemapperContext& ctx,
   if (!EigenSupportsContractionOutputKernel()) return false;
 
   // Root of the pattern must be a Relu.
-  if (node == nullptr) return false;
-  if (!IsRelu(*node)) return false;
-  if (!NodeIsOnCpu(node)) return false;
-  if (!IsFloatOrDoubleDataType(node)) return false;
-  if (!NoControlFaninOrFanout(ctx.graph_view, node)) return false;
+  if (!node || !IsRelu(*node) || HasControlFaninOrFanout(ctx.graph_view, node))
+    return false;
 
   // And input to Relu must match Conv2DWithBatchNorm pattern.
   const auto input_port = GraphView::InputPort(node, 0);
   const auto batch_norm = ctx.graph_view.GetRegularFanin(input_port);
 
   Conv2DWithBatchNorm base;
-  if (!FindConv2DWithBatchNorm(ctx, batch_norm.node, &base)) return false;
-  if (!HasSingleFanoutNode(ctx.graph_view, base.fused_batch_norm)) return false;
-  if (!HaveSameDataType(node, base.fused_batch_norm)) return false;
-  if (IsInPreserveSet(ctx, base.fused_batch_norm)) return false;
+  if (!FindConv2DWithBatchNorm(ctx, batch_norm.node, &base) ||
+      !HasSingleFanoutNode(ctx.graph_view, base.fused_batch_norm) ||
+      !HaveSameDataType(node, base.fused_batch_norm) ||
+      IsInPreserveSet(ctx, base.fused_batch_norm))
+    return false;
 
   // We successfully found a Conv2D+FusedBatchNorm+Relu pattern.
   matched->conv2d = base.conv2d;
@@ -355,9 +463,7 @@ bool FindFusedBatchNorm(const RemapperContext& ctx, const NodeDef* node,
   return true;
 }
 
-void CopyConv2DAttributes(const NodeDef* conv2d, NodeDef* fused_conv2d,
-                          const std::vector<string>& fused_ops = {},
-                          int num_args = 1, float epsilon = 0.0) {
+void CopyConv2DAttributes(const NodeDef* conv2d, NodeDef* fused_conv2d) {
   auto* attr = fused_conv2d->mutable_attr();
   auto src_attr = conv2d->attr();
 
@@ -367,53 +473,65 @@ void CopyConv2DAttributes(const NodeDef* conv2d, NodeDef* fused_conv2d,
   (*attr)["dilations"] = src_attr.at("dilations");
   (*attr)["data_format"] = src_attr.at("data_format");
   (*attr)["use_cudnn_on_gpu"] = src_attr.at("use_cudnn_on_gpu");
+}
 
-  auto* fused_ops_attr = (*attr)["fused_ops"].mutable_list();
-  for (const string& fused_op : fused_ops) {
-    fused_ops_attr->add_s(fused_op);
-  }
-
+void SetFusedConv2DAttributes(
+    NodeDef* fused_conv2d, const absl::Span<const absl::string_view> fused_ops,
+    int num_args = 1, float epsilon = 0.0) {
+  auto* attr = fused_conv2d->mutable_attr();
+  SetAttrValue(fused_ops, &(*attr)["fused_ops"]);
   SetAttrValue(num_args, &(*attr)["num_args"]);
-  // Required only for FusedBatchNorm.
-  SetAttrValue(epsilon, &(*attr)["epsilon"]);
+  SetAttrValue(epsilon, &(*attr)["epsilon"]);  // required only for BatchNorm
 }
 
 void AddFusedConv2DNode(
-    const Conv2DWithBiasAdd& matched, GraphDef* optimized_graph,
+    const RemapperContext& ctx, const Conv2DWithBiasAdd& matched,
+    GraphDef* optimized_graph,
     absl::flat_hash_set<const NodeDef*>* invalidated_nodes) {
-  VLOG(2) << "Fuse Conv2D with BiasAdd: bias_add=" << matched.bias_add->name()
+  DCHECK(IsDeviceCompatible(ctx, matched))
+      << "Unsupported fused Conv2D pattern";
+
+  VLOG(2) << "Fuse Conv2D with BiasAdd: "
+          << " bias_add=" << matched.bias_add->name()
           << " conv2d=" << matched.conv2d->name();
 
   NodeDef* fused_conv2d = optimized_graph->add_node();
-  fused_conv2d->set_name(matched.bias_add->name());
   fused_conv2d->set_op(kFusedConv2D);
-  fused_conv2d->set_device(matched.bias_add->device());
+  fused_conv2d->set_name(matched.bias_add->name());
+  fused_conv2d->set_device(matched.conv2d->device());
   fused_conv2d->add_input(matched.conv2d->input(0));    // 0: input
   fused_conv2d->add_input(matched.conv2d->input(1));    // 1: filter
   fused_conv2d->add_input(matched.bias_add->input(1));  // 2: bias
 
-  CopyConv2DAttributes(matched.conv2d, fused_conv2d, {"BiasAdd"});
+  CopyConv2DAttributes(matched.conv2d, fused_conv2d);
+  SetFusedConv2DAttributes(fused_conv2d, {"BiasAdd"});
 
   invalidated_nodes->insert(matched.bias_add);
   invalidated_nodes->insert(matched.conv2d);
 }
 
 void AddFusedConv2DNode(
-    const Conv2DWithBiasAddAndRelu& matched, GraphDef* optimized_graph,
+    const RemapperContext& ctx, const Conv2DWithBiasAddAndRelu& matched,
+    GraphDef* optimized_graph,
     absl::flat_hash_set<const NodeDef*>* invalidated_nodes) {
-  VLOG(2) << "Fuse Conv2D with BiasAdd and Relu: relu=" << matched.relu->name()
+  DCHECK(IsDeviceCompatible(ctx, matched))
+      << "Unsupported fused Conv2D pattern";
+
+  VLOG(2) << "Fuse Conv2D with BiasAdd and Relu: "
+          << " relu=" << matched.relu->name()
           << " bias_add=" << matched.bias_add->name()
           << " conv2d=" << matched.conv2d->name();
 
   NodeDef* fused_conv2d = optimized_graph->add_node();
   fused_conv2d->set_name(matched.relu->name());
   fused_conv2d->set_op(kFusedConv2D);
-  fused_conv2d->set_device(matched.relu->device());
+  fused_conv2d->set_device(matched.conv2d->device());
   fused_conv2d->add_input(matched.conv2d->input(0));    // 0: input
   fused_conv2d->add_input(matched.conv2d->input(1));    // 1: filter
   fused_conv2d->add_input(matched.bias_add->input(1));  // 2: bias
 
-  CopyConv2DAttributes(matched.conv2d, fused_conv2d, {"BiasAdd", "Relu"});
+  CopyConv2DAttributes(matched.conv2d, fused_conv2d);
+  SetFusedConv2DAttributes(fused_conv2d, {"BiasAdd", "Relu"});
 
   invalidated_nodes->insert(matched.relu);
   invalidated_nodes->insert(matched.bias_add);
@@ -421,8 +539,12 @@ void AddFusedConv2DNode(
 }
 
 void AddFusedConv2DNode(
-    const Conv2DWithSqueezeAndBiasAdd& matched, GraphDef* optimized_graph,
+    const RemapperContext& ctx, const Conv2DWithSqueezeAndBiasAdd& matched,
+    GraphDef* optimized_graph,
     absl::flat_hash_set<const NodeDef*>* invalidated_nodes) {
+  DCHECK(IsDeviceCompatible(ctx, matched))
+      << "Unsupported fused Conv2D pattern";
+
   VLOG(2) << "Fuse Conv2D with Squeeze and BiasAdd: "
           << " bias_add=" << matched.bias_add->name()
           << " squeeze=" << matched.squeeze->name()
@@ -432,13 +554,14 @@ void AddFusedConv2DNode(
   // has single consumer (only the squeeze node).
   NodeDef* fused_conv2d = optimized_graph->add_node();
   fused_conv2d->set_name(matched.conv2d->name());
-  fused_conv2d->set_op("_FusedConv2D");
+  fused_conv2d->set_op(kFusedConv2D);
   fused_conv2d->set_device(matched.conv2d->device());
   fused_conv2d->add_input(matched.conv2d->input(0));    // 0: input
   fused_conv2d->add_input(matched.conv2d->input(1));    // 1: filter
   fused_conv2d->add_input(matched.bias_add->input(1));  // 2: bias
 
-  CopyConv2DAttributes(matched.conv2d, fused_conv2d, {"BiasAdd"});
+  CopyConv2DAttributes(matched.conv2d, fused_conv2d);
+  SetFusedConv2DAttributes(fused_conv2d, {"BiasAdd"});
 
   // Replace BiasAdd node with a Squeeze.
   NodeDef* remapped_squeeze = optimized_graph->add_node();
@@ -461,7 +584,7 @@ void AddFusedConv2DNode(
   NodeDef* fused_conv2d = optimized_graph->add_node();
   fused_conv2d->set_name(matched.fused_batch_norm->name());
   fused_conv2d->set_op(kFusedConv2D);
-  fused_conv2d->set_device(matched.fused_batch_norm->device());
+  fused_conv2d->set_device(matched.conv2d->device());
   fused_conv2d->add_input(matched.conv2d->input(0));            // 0: input
   fused_conv2d->add_input(matched.conv2d->input(1));            // 1: filter
   fused_conv2d->add_input(matched.fused_batch_norm->input(1));  // 2: scale
@@ -469,8 +592,9 @@ void AddFusedConv2DNode(
   fused_conv2d->add_input(matched.fused_batch_norm->input(3));  // 4: mean
   fused_conv2d->add_input(matched.fused_batch_norm->input(4));  // 5: variance
 
-  CopyConv2DAttributes(matched.conv2d, fused_conv2d, {"FusedBatchNorm"},
-                       /*num_args*/ 4, /*epsilon*/ matched.epsilon);
+  CopyConv2DAttributes(matched.conv2d, fused_conv2d);
+  SetFusedConv2DAttributes(fused_conv2d, {"FusedBatchNorm"},
+                           /*num_args=*/4, /*epsilon=*/matched.epsilon);
 
   invalidated_nodes->insert(matched.fused_batch_norm);
   invalidated_nodes->insert(matched.conv2d);
@@ -487,7 +611,7 @@ void AddFusedConv2DNode(
   NodeDef* fused_conv2d = optimized_graph->add_node();
   fused_conv2d->set_name(matched.relu->name());
   fused_conv2d->set_op(kFusedConv2D);
-  fused_conv2d->set_device(matched.fused_batch_norm->device());
+  fused_conv2d->set_device(matched.conv2d->device());
   fused_conv2d->add_input(matched.conv2d->input(0));            // 0: input
   fused_conv2d->add_input(matched.conv2d->input(1));            // 1: filter
   fused_conv2d->add_input(matched.fused_batch_norm->input(1));  // 2: scale
@@ -495,8 +619,9 @@ void AddFusedConv2DNode(
   fused_conv2d->add_input(matched.fused_batch_norm->input(3));  // 4: mean
   fused_conv2d->add_input(matched.fused_batch_norm->input(4));  // 5: variance
 
-  CopyConv2DAttributes(matched.conv2d, fused_conv2d, {"FusedBatchNorm", "Relu"},
-                       /*num_args*/ 4, /*epsilon*/ matched.epsilon);
+  CopyConv2DAttributes(matched.conv2d, fused_conv2d);
+  SetFusedConv2DAttributes(fused_conv2d, {"FusedBatchNorm", "Relu"},
+                           /*num_args=*/4, /*epsilon=*/matched.epsilon);
 
   invalidated_nodes->insert(matched.relu);
   invalidated_nodes->insert(matched.fused_batch_norm);
@@ -680,13 +805,14 @@ Status Remapper::Optimize(Cluster* /*cluster*/, const GrapplerItem& item,
 
     // Remap Conv2D+BiasAdd into the _FusedConv2D.
     if (FindConv2DWithBias(ctx, &node, &conv2d_with_bias)) {
-      AddFusedConv2DNode(conv2d_with_bias, optimized_graph, &invalidated_nodes);
+      AddFusedConv2DNode(ctx, conv2d_with_bias, optimized_graph,
+                         &invalidated_nodes);
       continue;
     }
 
     // Remap Conv2D+BiasAdd+Relu into the _FusedConv2D.
     if (FindConv2DWithBiasAndRelu(ctx, &node, &conv2d_with_bias_and_relu)) {
-      AddFusedConv2DNode(conv2d_with_bias_and_relu, optimized_graph,
+      AddFusedConv2DNode(ctx, conv2d_with_bias_and_relu, optimized_graph,
                          &invalidated_nodes);
       continue;
     }
@@ -694,7 +820,7 @@ Status Remapper::Optimize(Cluster* /*cluster*/, const GrapplerItem& item,
     // Remap Conv2D+Squeeze+BiasAdd into the _FusedConv2D+Squeeze.
     if (FindConv2DWithSqueezeAndBias(ctx, &node,
                                      &conv2d_with_squeeze_and_bias)) {
-      AddFusedConv2DNode(conv2d_with_squeeze_and_bias, optimized_graph,
+      AddFusedConv2DNode(ctx, conv2d_with_squeeze_and_bias, optimized_graph,
                          &invalidated_nodes);
       continue;
     }
diff --git a/tensorflow/core/grappler/utils.cc b/tensorflow/core/grappler/utils.cc
index cc9a38b2d2..375c3e56c8 100644
--- a/tensorflow/core/grappler/utils.cc
+++ b/tensorflow/core/grappler/utils.cc
@@ -20,6 +20,8 @@ limitations under the License.
 #include <queue>
 #include <vector>
 
+#include "absl/strings/match.h"
+#include "absl/strings/str_cat.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/framework/node_def_util.h"
@@ -166,10 +168,10 @@ string AddPrefixToNodeName(const string& name, const string& prefix,
                            const string& delimiter) {
   if (!name.empty()) {
     if (name[0] == '^') {
-      return strings::StrCat("^", prefix, delimiter, name.substr(1));
+      return absl::StrCat("^", prefix, delimiter, name.substr(1));
     }
   }
-  return strings::StrCat(prefix, delimiter, name);
+  return absl::StrCat(prefix, delimiter, name);
 }
 
 string AddPrefixToNodeName(const string& name, const string& prefix) {
@@ -193,20 +195,26 @@ bool ExecuteWithTimeout(std::function<void()> fn, const int64 timeout_in_ms,
 }
 
 string AsControlDependency(const NodeDef& node) {
-  return strings::StrCat("^", node.name());
+  return absl::StrCat("^", node.name());
 }
 
 string AsControlDependency(const string& node_name) {
   CHECK(!node_name.empty());
   return (!node_name.empty() && node_name[0] == '^')
              ? node_name
-             : strings::StrCat("^", node_name);
+             : absl::StrCat("^", node_name);
 }
 
 bool NodeIsOnCpu(const NodeDef* node) {
   string task, device;
   return DeviceNameUtils::SplitDeviceName(node->device(), &task, &device) &&
-         str_util::StartsWith(device, DEVICE_CPU);
+         absl::StartsWith(device, DEVICE_CPU);
+}
+
+bool NodeIsOnGpu(const NodeDef* node) {
+  string task, device;
+  return DeviceNameUtils::SplitDeviceName(node->device(), &task, &device) &&
+         absl::StartsWith(device, DEVICE_GPU);
 }
 
 int NumOutputs(const NodeDef& node, GraphDef* graph) {
diff --git a/tensorflow/core/grappler/utils.h b/tensorflow/core/grappler/utils.h
index 1e820977ae..9053ae4c07 100644
--- a/tensorflow/core/grappler/utils.h
+++ b/tensorflow/core/grappler/utils.h
@@ -242,6 +242,9 @@ string AsControlDependency(const string& node);
 // Returns true if the node is assigned to run on CPU device.
 bool NodeIsOnCpu(const NodeDef* node);
 
+// Returns true if the node is assigned to run on GPU device.
+bool NodeIsOnGpu(const NodeDef* node);
+
 // Returns the number of outputs of a node according to its OpDef. Note that
 // some of the outputs may be unconnected.
 int NumOutputs(const NodeDef& node, GraphDef* graph);
diff --git a/tensorflow/core/grappler/utils/BUILD b/tensorflow/core/grappler/utils/BUILD
index 89417f85c2..ff15541cdc 100644
--- a/tensorflow/core/grappler/utils/BUILD
+++ b/tensorflow/core/grappler/utils/BUILD
@@ -143,6 +143,7 @@ cc_library(
         "//tensorflow/core:test",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:utils",
+        "@com_google_absl//absl/algorithm:container",
     ],
 )
 
diff --git a/tensorflow/core/grappler/utils/grappler_test.cc b/tensorflow/core/grappler/utils/grappler_test.cc
index 576494cad5..5ef1cf444b 100644
--- a/tensorflow/core/grappler/utils/grappler_test.cc
+++ b/tensorflow/core/grappler/utils/grappler_test.cc
@@ -15,6 +15,8 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/utils/grappler_test.h"
 #include <memory>
+#include "absl/algorithm/container.h"
+#include "tensorflow/core/framework/attr_value_util.h"
 #include "tensorflow/core/grappler/utils.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/protobuf/rewriter_config.pb.h"
@@ -125,6 +127,31 @@ void GrapplerTest::CompareGraphs(GraphDef want, GraphDef got) const {
   }
 }
 
+void GrapplerTest::CompareNodes(const NodeDef& want, const NodeDef& got) const {
+  EXPECT_EQ(want.name(), got.name());
+  EXPECT_EQ(want.op(), got.op());
+
+  std::vector<string> want_inputs(want.input().begin(), want.input().end());
+  std::vector<string> got_inputs(got.input().begin(), got.input().end());
+  EXPECT_EQ(want_inputs, got_inputs);
+
+  const auto attr_name = [](const std::pair<const string, AttrValue>& attr) {
+    return attr.first;
+  };
+
+  std::vector<string> want_attrs;
+  std::vector<string> got_attrs;
+  absl::c_transform(want.attr(), std::back_inserter(want_attrs), attr_name);
+  absl::c_transform(got.attr(), std::back_inserter(got_attrs), attr_name);
+  absl::c_sort(want_attrs);
+  absl::c_sort(got_attrs);
+  EXPECT_EQ(want_attrs, got_attrs);
+
+  for (const string& attr : want_attrs) {
+    EXPECT_TRUE(AreAttrValuesEqual(want.attr().at(attr), got.attr().at(attr)));
+  }
+}
+
 bool GrapplerTest::IsNodesDirectlyConnected(const NodeMap& node_map,
                                             const string& src,
                                             const string& dst, int position) {
diff --git a/tensorflow/core/grappler/utils/grappler_test.h b/tensorflow/core/grappler/utils/grappler_test.h
index 0cfd740dcb..20fd04c1c6 100644
--- a/tensorflow/core/grappler/utils/grappler_test.h
+++ b/tensorflow/core/grappler/utils/grappler_test.h
@@ -49,13 +49,25 @@ class GrapplerTest : public ::testing::Test {
                    const std::vector<std::pair<string, AttrValue>>& attributes,
                    GraphDef* graph) const;
 
+  // Checks if two graphs are equal. Both graphs must have the same set of nodes
+  // with the same inputs and attributes. Nodes can be in different order.
+  //
+  // NOTE: This function uses EXPECT/ASSERT macros to check node properties
+  // equality, and adds all failuires to the current test.
   void CompareGraphs(GraphDef want, GraphDef got) const;
 
-  // Check if node 'src' is directly connected to the input($position) of 'dst'.
+  // Checks if two nodes have the same name, op, inputs and attributes.
+  //
+  // NOTE: This function uses EXPECT/ASSERT macros to check node properties
+  // equality, and adds all failuires to the current test.
+  void CompareNodes(const NodeDef& want, const NodeDef& got) const;
+
+  // Checks if node 'src' is directly connected to the input($position) of
+  // 'dst'.
   bool IsNodesDirectlyConnected(const NodeMap& node_map, const string& src,
                                 const string& dst, int position = 0);
 
-  // Count nodes of the given op-type in a graph.
+  // Counts nodes of the given op-type in a graph.
   int CountOpNodes(const GraphDef& graph, const string& op);
 
   // Get a random tensor with given shape.
diff --git a/tensorflow/core/kernels/conv_ops_test.cc b/tensorflow/core/kernels/conv_ops_test.cc
index 3e59219f8f..fc93915e16 100644
--- a/tensorflow/core/kernels/conv_ops_test.cc
+++ b/tensorflow/core/kernels/conv_ops_test.cc
@@ -1497,6 +1497,26 @@ BM_FusedConv2DWithBatchNormAndRelu(32, 32, 32, 128, 3, 3, 1024, cpu,
                                    "3x3 /b 32");
 
 #if GOOGLE_CUDA
+// -------------------------------------------------------------------------- //
+// 1x1 Convolution
+// -------------------------------------------------------------------------- //
+
+BM_Conv2D(8, 32, 32, 128, 1, 1, 1024, gpu, "1x1 /b 8");
+BM_Conv2D(16, 32, 32, 128, 1, 1, 1024, gpu, "1x1 /b 16");
+BM_Conv2D(32, 32, 32, 128, 1, 1, 1024, gpu, "1x1 /b 32");
+
+BM_Conv2DWithBiasAndRelu(8, 32, 32, 128, 1, 1, 1024, gpu, "1x1 /b 8");
+BM_Conv2DWithBiasAndRelu(16, 32, 32, 128, 1, 1, 1024, gpu, "1x1 /b 16");
+BM_Conv2DWithBiasAndRelu(32, 32, 32, 128, 1, 1, 1024, gpu, "1x1 /b 32");
+
+BM_FusedConv2DWithBiasAndRelu(8, 32, 32, 128, 1, 1, 1024, gpu, "1x1 /b 8");
+BM_FusedConv2DWithBiasAndRelu(16, 32, 32, 128, 1, 1, 1024, gpu, "1x1 /b 16");
+BM_FusedConv2DWithBiasAndRelu(32, 32, 32, 128, 1, 1, 1024, gpu, "1x1 /b 32");
+
+// -------------------------------------------------------------------------- //
+// 3x3 Convolution
+// -------------------------------------------------------------------------- //
+
 BM_Conv2D(8, 32, 32, 128, 3, 3, 1024, gpu, "3x3 /b 8");
 BM_Conv2D(16, 32, 32, 128, 3, 3, 1024, gpu, "3x3 /b 16");
 BM_Conv2D(32, 32, 32, 128, 3, 3, 1024, gpu, "3x3 /b 32");
-- 
GitLab


From c7c4a42c4372ca560ea415fe3a798e18286cedec Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 12:36:38 -0800
Subject: [PATCH 0280/2345] Fix an error in keras input_layer.Input() dtype
 type checking.

PiperOrigin-RevId: 228215444
---
 tensorflow/python/keras/engine/input_layer.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/engine/input_layer.py b/tensorflow/python/keras/engine/input_layer.py
index bc2cf2fb6e..c6dcedfce2 100644
--- a/tensorflow/python/keras/engine/input_layer.py
+++ b/tensorflow/python/keras/engine/input_layer.py
@@ -77,8 +77,9 @@ class InputLayer(base_layer.Layer):
         dtype = backend.floatx()
       else:
         dtype = backend.dtype(input_tensor)
-    elif input_tensor and input_tensor.dtype != dtype:
-      raise ValueError('`input_tensor.dtype` differs from `dtype`.')
+    elif input_tensor is not None and input_tensor.dtype != dtype:
+      raise ValueError('`input_tensor.dtype` differs from `dtype`: %s vs. %s' %
+                       (input_tensor.dtype, dtype))
     super(InputLayer, self).__init__(dtype=dtype, name=name)
     self.built = True
     self.sparse = sparse
-- 
GitLab


From e7f7f01352be4054a7fb2a872cf2ef335292600c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 12:48:37 -0800
Subject: [PATCH 0281/2345] Prevent the interpreter from continuing execution
 after an error occurred.

PiperOrigin-RevId: 228217675
---
 tensorflow/lite/core/subgraph.cc | 6 +++---
 tensorflow/lite/interpreter.cc   | 7 ++++---
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/tensorflow/lite/core/subgraph.cc b/tensorflow/lite/core/subgraph.cc
index 75212cac22..4be80d143e 100644
--- a/tensorflow/lite/core/subgraph.cc
+++ b/tensorflow/lite/core/subgraph.cc
@@ -670,7 +670,7 @@ TfLiteStatus Subgraph::Invoke() {
       TfLiteTensor* tensor = &tensors_[tensor_index];
       if (tensor->delegate && tensor->delegate != node.delegate &&
           tensor->data_is_stale) {
-        EnsureTensorDataIsReadable(tensor_index);
+        TF_LITE_ENSURE_STATUS(EnsureTensorDataIsReadable(tensor_index));
       }
     }
 
@@ -683,8 +683,8 @@ TfLiteStatus Subgraph::Invoke() {
     EnsureTensorsVectorCapacity();
     tensor_resized_since_op_invoke_ = false;
     if (OpInvoke(registration, &node) == kTfLiteError) {
-      status = ReportOpError(context_, node, registration, node_index,
-                             "failed to invoke");
+      return ReportOpError(context_, node, registration, node_index,
+                           "failed to invoke");
     }
 
     // Force execution prep for downstream ops if the latest op triggered the
diff --git a/tensorflow/lite/interpreter.cc b/tensorflow/lite/interpreter.cc
index 2abe062ec6..60fa2130fa 100644
--- a/tensorflow/lite/interpreter.cc
+++ b/tensorflow/lite/interpreter.cc
@@ -102,15 +102,16 @@ TfLiteStatus Interpreter::ResizeInputTensor(int tensor_index,
 }
 
 TfLiteStatus Interpreter::Invoke() {
-  TfLiteStatus status = primary_subgraph().Invoke();
+  TF_LITE_ENSURE_STATUS(primary_subgraph().Invoke());
 
   if (!allow_buffer_handle_output_) {
     for (int tensor_index : outputs()) {
-      primary_subgraph().EnsureTensorDataIsReadable(tensor_index);
+      TF_LITE_ENSURE_STATUS(
+          primary_subgraph().EnsureTensorDataIsReadable(tensor_index));
     }
   }
 
-  return status;
+  return kTfLiteOk;
 }
 
 TfLiteStatus Interpreter::AddTensors(int tensors_to_add,
-- 
GitLab


From 95d708d7b9e3f22c09ca77004e68f2cadb295226 Mon Sep 17 00:00:00 2001
From: Shanqing Cai <cais@google.com>
Date: Mon, 7 Jan 2019 13:12:01 -0800
Subject: [PATCH 0282/2345] tf.keras: Remove unnecessary backend check in
 PReLU.call()

PiperOrigin-RevId: 228222042
---
 tensorflow/python/keras/layers/advanced_activations.py | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/tensorflow/python/keras/layers/advanced_activations.py b/tensorflow/python/keras/layers/advanced_activations.py
index be1039a2ac..5095287430 100644
--- a/tensorflow/python/keras/layers/advanced_activations.py
+++ b/tensorflow/python/keras/layers/advanced_activations.py
@@ -121,11 +121,9 @@ class PReLU(Layer):
   @tf_utils.shape_type_conversion
   def build(self, input_shape):
     param_shape = list(input_shape[1:])
-    self.param_broadcast = [False] * len(param_shape)
     if self.shared_axes is not None:
       for i in self.shared_axes:
         param_shape[i - 1] = 1
-        self.param_broadcast[i - 1] = True
     self.alpha = self.add_weight(
         shape=param_shape,
         name='alpha',
@@ -143,12 +141,7 @@ class PReLU(Layer):
 
   def call(self, inputs, mask=None):
     pos = K.relu(inputs)
-    if K.backend() == 'theano':
-      neg = (
-          K.pattern_broadcast(self.alpha, self.param_broadcast) *
-          (inputs - math_ops.abs(inputs)) * 0.5)
-    else:
-      neg = -self.alpha * K.relu(-inputs)
+    neg = -self.alpha * K.relu(-inputs)
     return pos + neg
 
   def get_config(self):
-- 
GitLab


From 38491a84a9e38357e400457dbbe408b66e786672 Mon Sep 17 00:00:00 2001
From: Pooya Davoodi <pdavoodi@nvidia.com>
Date: Mon, 7 Jan 2019 14:21:21 -0800
Subject: [PATCH 0283/2345] TFTRT: Expand lambda and inline ifs to functions
 with if and else

---
 .../contrib/tensorrt/python/trt_convert.py    | 36 ++++++++++++++-----
 1 file changed, 27 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/python/trt_convert.py b/tensorflow/contrib/tensorrt/python/trt_convert.py
index 3d2b6b499d..f5d624ef64 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert.py
@@ -45,15 +45,33 @@ from tensorflow.python.saved_model import loader_impl
 from tensorflow.python.saved_model import tag_constants
 from tensorflow.python.training import saver
 
-if _six.PY2:
-  _to_bytes = lambda s: s.encode("utf-8", errors="surrogateescape") \
-    if isinstance(s, unicode) else s
-  _to_string = lambda s: s.decode("utf-8") if isinstance(s, str) else s
-else:
-  _to_bytes = lambda s: s.encode("utf-8", errors="surrogateescape") \
-    if isinstance(s, str) else s
-  _to_string = lambda s: s.decode("utf-8") if isinstance(s, bytes) else s
-
+def _to_bytes(s):
+    """Returns encoded of s if s is a sequence of chars otherwise returns s.
+    """
+    if _six.PY2:
+        if isinstance(s, unicode):
+            return s.encode("utf-8", errors="surrogateescape")
+        else:
+            return s
+    else:
+        if isinstance(s, str):
+            return s.encode("utf-8", errors="surrogateescape")
+        else:
+            return s
+
+def _to_string(s):
+    """Returns decoded of s if s is a sequence of bytes otherwise returns s.
+    """
+    if _six.PY2:
+        if isinstance(s, str):
+            return s.decode("utf-8")
+        else:
+            return s
+    else:
+        if isinstance(s, bytes):
+            return s.decode("utf-8")
+        else:
+            return s
 
 class TrtPrecisionMode(object):
   FP32 = "FP32"
-- 
GitLab


From 46f030b6e275bd9950da91e480bda5f2fa44ad80 Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Mon, 7 Jan 2019 13:26:16 -0800
Subject: [PATCH 0284/2345] Clean up example/learn/BUILD a bit

Some of the data deps are targets that no longer exist. Pretty sure
should be cleaned up.

PiperOrigin-RevId: 228224240
---
 tensorflow/examples/learn/BUILD | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/tensorflow/examples/learn/BUILD b/tensorflow/examples/learn/BUILD
index d6ec1f393b..a22d55e5af 100644
--- a/tensorflow/examples/learn/BUILD
+++ b/tensorflow/examples/learn/BUILD
@@ -28,17 +28,8 @@ sh_test(
     size = "large",
     srcs = ["examples_test.sh"],
     data = [
-        ":boston",
-        ":iris",
         ":iris_custom_decay_dnn",
         ":iris_custom_model",
-        ":iris_run_config",
-        ":random_forest_mnist",
-        ":resnet",
-        ":text_classification",
-        ":text_classification_character_cnn",
-        ":text_classification_character_rnn",
-        ":text_classification_cnn",
     ],
     tags = [
         "manual",
-- 
GitLab


From d2dd3bd43ebbfd2d14f2f7e658b5b5aae201607e Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Mon, 7 Jan 2019 13:35:32 -0800
Subject: [PATCH 0285/2345] while_v2 fixes for better TPU integration.

PiperOrigin-RevId: 228226029
---
 tensorflow/python/ops/while_v2.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/while_v2.py b/tensorflow/python/ops/while_v2.py
index aaed86c1ca..f9fea0e5bd 100644
--- a/tensorflow/python/ops/while_v2.py
+++ b/tensorflow/python/ops/while_v2.py
@@ -115,12 +115,15 @@ def while_loop(cond,
             loop_counter < maximum_iterations,
             cond(*_pack_sequence_as(orig_loop_vars, args)))
 
+    # NOTE(skyewm): we set read_only_collections=False for compatibility with
+    # TPUEstimator.
     cond_graph = func_graph_module.func_graph_from_py_func(
         cond_name,
         wrapped_cond,
         loop_vars, {},
         signature=_build_signature(loop_vars, shape_invariants),
-        func_graph=util.WhileCondFuncGraph(cond_name),
+        func_graph=util.WhileCondFuncGraph(cond_name,
+                                           read_only_collections=False),
         add_control_dependencies=add_control_dependencies)
 
     # Add external_captures of cond to the list of loop vars.
@@ -171,7 +174,8 @@ def while_loop(cond,
         wrapped_body,
         loop_vars, {},
         signature=_build_signature(loop_vars, shape_invariants),
-        func_graph=util.WhileBodyFuncGraph(body_name),
+        func_graph=util.WhileBodyFuncGraph(body_name,
+                                           read_only_collections=False),
         add_control_dependencies=add_control_dependencies)
     # Add external captures of body to the list of loop vars.
     # Note that external tensors will be treated as loop invariants, i.e.,
-- 
GitLab


From 879af3c38d2405db3ff4c96d6104cdf82a1ab368 Mon Sep 17 00:00:00 2001
From: Frank Chen <frankchn@google.com>
Date: Mon, 7 Jan 2019 13:38:36 -0800
Subject: [PATCH 0286/2345] Change dataset rewriting in distribution strategy
 to avoid a unbatch().batch() call.

PiperOrigin-RevId: 228226620
---
 .../python/data/experimental/ops/batching.py  |  8 ++++++--
 tensorflow/python/distribute/input_lib.py     | 20 ++++++++++++++++---
 2 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/data/experimental/ops/batching.py b/tensorflow/python/data/experimental/ops/batching.py
index f0cf7f0a99..ff8a9182c3 100644
--- a/tensorflow/python/data/experimental/ops/batching.py
+++ b/tensorflow/python/data/experimental/ops/batching.py
@@ -553,8 +553,12 @@ class _MapAndBatchDataset(dataset_ops.UnaryDataset):
     """See `Dataset.map()` for details."""
     self._input_dataset = input_dataset
 
-    self._map_func = dataset_ops.StructuredFunctionWrapper(
-        map_func, "tf.data.experimental.map_and_batch()", dataset=input_dataset)
+    if isinstance(map_func, dataset_ops.StructuredFunctionWrapper):
+      self._map_func = map_func
+    else:
+      self._map_func = dataset_ops.StructuredFunctionWrapper(
+          map_func, "tf.data.experimental.map_and_batch()",
+          dataset=input_dataset)
     self._batch_size_t = ops.convert_to_tensor(
         batch_size, dtype=dtypes.int64, name="batch_size")
     self._num_parallel_calls_t = ops.convert_to_tensor(
diff --git a/tensorflow/python/distribute/input_lib.py b/tensorflow/python/distribute/input_lib.py
index cbe6518e5c..022595146d 100644
--- a/tensorflow/python/distribute/input_lib.py
+++ b/tensorflow/python/distribute/input_lib.py
@@ -574,12 +574,19 @@ def _split_dataset_batch(dataset, split_batch_by):
         "The batch operations can be followed by a prefetch.")
 
   batched_dataset = _get_batch_dataset(dataset)
+  prev_dataset = batched_dataset._input_dataset
+
+  num_parallel_calls = None
+  map_func = None
+
   if isinstance(batched_dataset, dataset_ops.BatchDataset):
     batch_size = batched_dataset._batch_size
     drop_remainder = batched_dataset._drop_remainder
   elif isinstance(batched_dataset, batching._MapAndBatchDataset):
     batch_size = batched_dataset._batch_size_t
     drop_remainder = batched_dataset._drop_remainder_t
+    num_parallel_calls = batched_dataset._num_parallel_calls_t
+    map_func = batched_dataset._map_func
 
   prefetch_buffer = None
   if isinstance(dataset, dataset_ops.PrefetchDataset):
@@ -587,7 +594,6 @@ def _split_dataset_batch(dataset, split_batch_by):
   elif (isinstance(dataset, dataset_ops.DatasetV1Adapter)
         and isinstance(dataset._dataset, dataset_ops.PrefetchDataset)):
     prefetch_buffer = dataset._dataset._buffer_size
-  # pylint: enable=protected-access
 
   if tensor_util.is_tensor(batch_size):
     batch_size = tensor_util.constant_value(batch_size)
@@ -595,14 +601,22 @@ def _split_dataset_batch(dataset, split_batch_by):
   if tensor_util.is_tensor(drop_remainder):
     drop_remainder = tensor_util.constant_value(drop_remainder)
 
+  if num_parallel_calls is not None and tensor_util.is_tensor(drop_remainder):
+    num_parallel_calls = tensor_util.constant_value(num_parallel_calls)
+
   if batch_size % split_batch_by:
     raise ValueError(
         "Batch size %s cannot be sharded evenly across replicas %s" % (
             batch_size, split_batch_by))
   new_batch_size = batch_size // split_batch_by
 
-  dataset = dataset.apply(batching.unbatch())
-  dataset = dataset.batch(new_batch_size, drop_remainder=drop_remainder)
+  if isinstance(batched_dataset, dataset_ops.BatchDataset):
+    dataset = prev_dataset.batch(new_batch_size, drop_remainder=drop_remainder)
+  elif isinstance(batched_dataset, batching._MapAndBatchDataset):
+    dataset = prev_dataset.apply(batching.map_and_batch(
+        map_func, new_batch_size, num_parallel_calls, drop_remainder))
+  # pylint: enable=protected-access
+
   if prefetch_buffer is not None:
     dataset = dataset.prefetch(prefetch_buffer)
   return dataset
-- 
GitLab


From 7bc843afd6696fc802fa44f65f0fdf67b0bc56e6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 13:45:54 -0800
Subject: [PATCH 0287/2345] Unittests to demonstrate three bugs in flex mode: 
 - forwardability is a property of a subgraph, not of the global buffer map  -
 input tensors should never be forwardable, even when they    come from
 another subgraph. That's because they might be outputs of the    model, or
 inputs into another subgraph.  - input tensors shouldn't be removed at the
 end, because they might be    used by another subgraph.

PiperOrigin-RevId: 228228094
---
 tensorflow/lite/delegates/flex/kernel_test.cc | 175 +++++++++++++++++-
 tensorflow/lite/delegates/flex/test_util.cc   |   5 +
 tensorflow/lite/delegates/flex/test_util.h    |   3 +
 3 files changed, 179 insertions(+), 4 deletions(-)

diff --git a/tensorflow/lite/delegates/flex/kernel_test.cc b/tensorflow/lite/delegates/flex/kernel_test.cc
index 647c199d4f..9de3ca67de 100644
--- a/tensorflow/lite/delegates/flex/kernel_test.cc
+++ b/tensorflow/lite/delegates/flex/kernel_test.cc
@@ -36,13 +36,38 @@ TfLiteStatus GenericPrepare(TfLiteContext* context, TfLiteDelegate* delegate,
   return kTfLiteOk;
 }
 
+// There is no easy way to pass a parameter into the TfLiteDelegate's
+// 'prepare' function, so we keep a global map for testing purpused.
+// To avoid collisions use: GetPrepareFunction<__LINE__>().
+std::map<int, std::vector<int>>* GetGlobalOpLists() {
+  static auto* op_list = new std::map<int, std::vector<int>>;
+  return op_list;
+}
+
 class KernelTest : public testing::FlexModelTest {
  public:
+  static constexpr int kOnes = 1;  // This is the index of a tensor of 1's.
+  static constexpr int kTwos = 2;  // This is the index of a tensor of 2's.
+  static constexpr int kMaxTensors = 30;
+
+  static void SetUpTestSuite() { GetGlobalOpLists()->clear(); }
+
   KernelTest() {
     CHECK(delegate_data_.Prepare(tensorflow::SessionOptions{}).ok());
     interpreter_.reset(new Interpreter(&error_reporter_));
   }
 
+  typedef TfLiteStatus (*PrepareFunction)(TfLiteContext* context,
+                                          TfLiteDelegate* delegate);
+
+  template <int KEY>
+  PrepareFunction GetPrepareFunction() {
+    GetGlobalOpLists()->insert({KEY, tf_ops_});
+    return [](TfLiteContext* context, TfLiteDelegate* delegate) {
+      return GenericPrepare(context, delegate, GetGlobalOpLists()->at(KEY));
+    };
+  }
+
   template <typename T>
   void ConfigureDelegate(T prepare_function) {
     delegate_.data_ = &delegate_data_;
@@ -54,9 +79,13 @@ class KernelTest : public testing::FlexModelTest {
                                         TfLiteBufferHandle buffer_handle,
                                         TfLiteTensor* output) {
       auto* delegate_data = reinterpret_cast<DelegateData*>(delegate->data_);
-      tensorflow::StringPiece values = delegate_data->GetBufferMap(context)
-                                           ->GetTensor(buffer_handle)
-                                           .tensor_data();
+      auto* buffer_map = delegate_data->GetBufferMap(context);
+      if (!buffer_map->HasTensor(buffer_handle)) {
+        context->ReportError(context, "Tensor '%d' not found", buffer_handle);
+        return kTfLiteError;
+      }
+      tensorflow::StringPiece values =
+          buffer_map->GetTensor(buffer_handle).tensor_data();
       memcpy(output->data.raw, values.data(), values.size());
       return kTfLiteOk;
     };
@@ -225,7 +254,7 @@ TEST_F(KernelTest, SplitGraph) {
   AddTfOp(testing::kAdd, {9, 16}, {17});  // => 16
 
   ConfigureDelegate([](TfLiteContext* context, TfLiteDelegate* delegate) {
-    // All ops by #3 are TF ops, handled by the delegate. However, because #4
+    // All ops but #3 are TF ops, handled by the delegate. However, because #4
     // depends on the non-TF op, two subgraphs are necessary:
     //    TF subgraph 1: 0, 1, 2, 6, 7, 8, 9
     //    TF Lite Op: 3
@@ -260,6 +289,144 @@ TEST_F(KernelTest, SplitGraph) {
   ASSERT_THAT(GetValues(17), ElementsAre(18.0f));
 }
 
+class MultipleSubgraphsTest : public KernelTest {
+ public:
+  static constexpr int kInput = 0;
+
+  void PrepareInterpreter(PrepareFunction prepare,
+                          const std::vector<float>& input) {
+    ConfigureDelegate(prepare);
+
+    SetShape(kOnes, {3});
+    SetValues(kOnes, {1.0f, 1.0f, 1.0f});
+    SetShape(kTwos, {3});
+    SetValues(kTwos, {2.0f, 2.0f, 2.0f});
+
+    SetValues(kInput, input);
+  }
+
+  std::vector<float> Apply(const std::vector<float>& input,
+                           std::function<float(float)> function) {
+    std::vector<float> result;
+    for (float f : input) {
+      result.push_back(function(f));
+    }
+    return result;
+  }
+};
+
+TEST_F(MultipleSubgraphsTest, ForwardabilityIsLocal) {
+  AddTensors(kMaxTensors, {kInput, kOnes, kTwos}, {12}, kTfLiteFloat32, {3});
+
+  // Only TF tensors can be forwarded, so we build a small first graph
+  // to produce tensor #10. Here #10 is forwardable, because it is only
+  // used once, as an output.
+  AddTfOp(testing::kAdd, {0, kOnes}, {3});
+  AddTfOp(testing::kAdd, {0, kOnes}, {10});
+
+  // The second TF graph, separated from the former by a TF Lite
+  // multiplication, will consume tensor #10, which is not forwardable here
+  // since it is used by more than one op. The existing code will forward the
+  // tensor anyway, because it was deemed to be forwardable by the previous
+  // subgraph.
+  AddTfLiteMulOp({3, kTwos}, {4});
+  AddTfOp(testing::kAdd, {10, 4}, {11});
+  AddTfOp(testing::kAdd, {11, 10}, {7});
+
+  // And a simple TF Lite op trying to access tensor #10, which was removed
+  // from the buffer map. It will cause Invoke() to fail.
+  AddTfLiteMulOp({10, 7}, {12});
+
+  auto input = {3.0f, 4.0f, 5.0f};
+  PrepareInterpreter(GetPrepareFunction<__LINE__>(), input);
+
+  ASSERT_FALSE(Invoke());
+  ASSERT_THAT(error_reporter().error_messages(),
+              ContainsRegex("Cannot read from invalid tensor index 10"));
+
+  // TODO(b/122457581): expected result for this test:
+  // ASSERT_THAT(GetValues(12), ElementsAreArray(Apply(input, [](float in) {
+  //               return (4 * in + 4) * (in + 1);
+  //             })));
+}
+
+// Subgraphs should not remove input tensors from the buffer_map, since
+// they could be necessary for downstream graphs.
+TEST_F(MultipleSubgraphsTest, DoNotRemoveInputTensors) {
+  AddTensors(kMaxTensors, {kInput, kOnes, kTwos}, {12}, kTfLiteFloat32, {3});
+
+  // Only TF tensors can be removed, so we build a small first graph
+  // to produce tensor #10. We make sure it is used by more than one
+  // op, so it is not forwardable here.
+  AddTfOp(testing::kAdd, {0, kOnes}, {3});
+  AddTfOp(testing::kAdd, {0, kOnes}, {10});
+  AddTfOp(testing::kAdd, {10, kOnes}, {15});
+  AddTfOp(testing::kAdd, {10, kOnes}, {16});
+
+  // The second TF graph, separated from the former by a TF Lite
+  // multiplication, will consume tensor #10. The existing code will remove
+  // from the buffer_map all tensors that are not outputs, so #10 will
+  // disappear. Note that we are using #10 in two ops, so it is not forwardable
+  // either.
+  AddTfLiteMulOp({3, kTwos}, {4});
+  AddTfOp(testing::kAdd, {10, 4}, {11});
+  AddTfOp(testing::kAdd, {10, 11}, {7});
+
+  // And a simple TF Lite op trying to access tensor #10, which was removed
+  // from the buffer map. It will cause Invoke() to fail.
+  AddTfLiteMulOp({10, 7}, {12});
+
+  auto input = {3.0f, 4.0f, 5.0f};
+  PrepareInterpreter(GetPrepareFunction<__LINE__>(), input);
+
+  ASSERT_FALSE(Invoke());
+  ASSERT_THAT(error_reporter().error_messages(),
+              ContainsRegex("Tensor '10' not found"));
+
+  // TODO(b/122457581): expected result for this test:
+  // ASSERT_THAT(GetValues(12), ElementsAreArray(Apply(input, [](float in) {
+  //               return (4 * in + 4) * (in + 1);
+  //             })));
+}
+
+// A tensor is deemed forwardable but it happens to be the input to
+// more than one subgraph. It should not be forwarded, otherwise its
+// contents will be overwritten.
+TEST_F(MultipleSubgraphsTest, DoNotForwardInputTensors) {
+  AddTensors(kMaxTensors, {kInput, kOnes, kTwos}, {12}, kTfLiteFloat32, {3});
+
+  // Only TF tensors can be forwarded, so we build a small first graph
+  // to produce tensor #10.
+  AddTfOp(testing::kAdd, {0, kOnes}, {3});
+  AddTfOp(testing::kAdd, {0, kOnes}, {10});
+
+  // The second TF graph, separated from the former by a TF Lite
+  // multiplication, will consume tensor #10 and will think it is forwardable
+  // because it is used by a single op. However, the subgraph doesn't have
+  // enough information to make that judgment, as the input tensor could be
+  // used by another graph further downstream. The existing code will forward
+  // the tensor and remove it from the buffer_map, causing a failure later.
+  AddTfLiteMulOp({3, kTwos}, {4});
+  AddTfOp(testing::kAdd, {10, 4}, {11});
+  AddTfOp(testing::kAdd, {11, 4}, {7});
+
+  // And a simple TF Lite op trying to access tensor #10, which was removed
+  // from the buffer map. It will cause Invoke() to fail.
+  AddTfLiteMulOp({10, 7}, {12});
+
+  auto input = {3.0f, 4.0f, 5.0f};
+  PrepareInterpreter(GetPrepareFunction<__LINE__>(), input);
+
+  ASSERT_FALSE(Invoke());
+  ASSERT_THAT(error_reporter().error_messages(),
+              ContainsRegex("Tensor '10' not found"));
+
+  // TODO(b/122457581): expected result for this test:
+  // ASSERT_THAT(GetValues(12), ElementsAreArray(Apply(input, [](float in) {
+  //               return (5 * in + 5) * (in + 1);
+  //             })));
+}
+
 }  // namespace
 }  // namespace flex
 }  // namespace tflite
diff --git a/tensorflow/lite/delegates/flex/test_util.cc b/tensorflow/lite/delegates/flex/test_util.cc
index aa24675a7b..a67aeef231 100644
--- a/tensorflow/lite/delegates/flex/test_util.cc
+++ b/tensorflow/lite/delegates/flex/test_util.cc
@@ -90,6 +90,8 @@ void FlexModelTest::AddTensors(int num_tensors, const std::vector<int>& inputs,
 
 void FlexModelTest::AddTfLiteMulOp(const std::vector<int>& inputs,
                                    const std::vector<int>& outputs) {
+  ++next_op_index_;
+
   static TfLiteRegistration reg = {nullptr, nullptr, nullptr, nullptr};
   reg.builtin_code = BuiltinOperator_MUL;
   reg.prepare = [](TfLiteContext* context, TfLiteNode* node) {
@@ -114,6 +116,9 @@ void FlexModelTest::AddTfLiteMulOp(const std::vector<int>& inputs,
 
 void FlexModelTest::AddTfOp(TfOpType op, const std::vector<int>& inputs,
                             const std::vector<int>& outputs) {
+  tf_ops_.push_back(next_op_index_);
+  ++next_op_index_;
+
   auto attr = [](const string& key, const string& value) {
     return " attr{ key: '" + key + "' value {" + value + "}}";
   };
diff --git a/tensorflow/lite/delegates/flex/test_util.h b/tensorflow/lite/delegates/flex/test_util.h
index 2cc2dc30e9..1913a406e8 100644
--- a/tensorflow/lite/delegates/flex/test_util.h
+++ b/tensorflow/lite/delegates/flex/test_util.h
@@ -103,6 +103,7 @@ class FlexModelTest : public ::testing::Test {
  protected:
   std::unique_ptr<Interpreter> interpreter_;
   TestErrorReporter error_reporter_;
+  std::vector<int> tf_ops_;
 
  private:
   // Helper method to add a TensorFlow op. tflite_names needs to start with
@@ -112,6 +113,8 @@ class FlexModelTest : public ::testing::Test {
                const std::vector<int>& outputs);
 
   std::vector<std::vector<uint8_t>> flexbuffers_;
+
+  int next_op_index_ = 0;
 };
 
 }  // namespace testing
-- 
GitLab


From 2d8379badceae7beae681eb2321ed575ba8ec718 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 7 Jan 2019 14:17:08 -0800
Subject: [PATCH 0288/2345] [TF:XLA] Bump open source llvm revision to r350438

PiperOrigin-RevId: 228234219
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index b43312abb2..834ec28447 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -500,11 +500,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "3a1c8a817d2d9a85b85e1ad776a90faa51f475bb924fbf70383892bb1faceac9",
-        strip_prefix = "llvm-6da47bee848304d215e3ca08b2a4a4dc9c954310",
+        sha256 = "20c0ebd54f433d68652a54e328d70b0ec7cf37ce734645a0259cf904d14c2c53",
+        strip_prefix = "llvm-0e2aef924e02d00a4d3454d63c1921a243d316c6",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/6da47bee848304d215e3ca08b2a4a4dc9c954310.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/6da47bee848304d215e3ca08b2a4a4dc9c954310.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/0e2aef924e02d00a4d3454d63c1921a243d316c6.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/0e2aef924e02d00a4d3454d63c1921a243d316c6.tar.gz",
         ],
     )
 
-- 
GitLab


From 30e5f93366b7ef7f24f7f267d2ad1d75afe4583b Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 7 Jan 2019 14:24:44 -0800
Subject: [PATCH 0289/2345] [XLA:CPU] Split out the logic to emit tiled
 GEMM/GEMV into a separate file; NFC

PiperOrigin-RevId: 228235618
---
 tensorflow/compiler/xla/service/cpu/BUILD     |   16 +
 .../xla/service/cpu/dot_op_emitter.cc         |  993 +---------------
 .../xla/service/cpu/tiled_dot_emitter.cc      | 1014 +++++++++++++++++
 .../xla/service/cpu/tiled_dot_emitter.h       |   55 +
 4 files changed, 1107 insertions(+), 971 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/cpu/tiled_dot_emitter.cc
 create mode 100644 tensorflow/compiler/xla/service/cpu/tiled_dot_emitter.h

diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index 5c72e1ff71..86c5f71096 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -364,6 +364,21 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "tiled_dot_emitter",
+    srcs = ["tiled_dot_emitter.cc"],
+    hdrs = ["tiled_dot_emitter.h"],
+    deps = [
+        ":vector_support_library",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/compiler/xla/service/llvm_ir:kernel_support_library",
+        "//tensorflow/compiler/xla/service/llvm_ir:llvm_util",
+        "//tensorflow/core:lib",
+        "@llvm//:core",
+    ],
+)
+
 cc_library(
     name = "dot_op_emitter",
     srcs = ["dot_op_emitter.cc"],
@@ -373,6 +388,7 @@ cc_library(
         ":cpu_runtime",
         ":ir_emission_utils",
         ":target_machine_features",
+        ":tiled_dot_emitter",
         ":vector_support_library",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:status_macros",
diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
index 1525a33af7..bdd688c2b9 100644
--- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
@@ -26,6 +26,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/cpu/cpu_runtime.h"
 #include "tensorflow/compiler/xla/service/cpu/ir_emission_utils.h"
 #include "tensorflow/compiler/xla/service/cpu/target_machine_features.h"
+#include "tensorflow/compiler/xla/service/cpu/tiled_dot_emitter.h"
 #include "tensorflow/compiler/xla/service/cpu/vector_support_library.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/kernel_support_library.h"
@@ -41,930 +42,6 @@ namespace xla {
 using llvm_ir::SetToFirstInsertPoint;
 
 namespace cpu {
-
-namespace {
-// Provides tiled access to an in-memory rank 2 array.
-class MemoryTile {
- public:
-  // Constructs a MemoryTile that can operate on tiles consisting of
-  // `tile_size_along_major_dim` vectors from the matrix `matrix`, starting at
-  // `major_dim_offset` in the major dimension.  The tile size along the minor
-  // dimension is the vector size, and that is implicitly determined by `vsl`.
-  MemoryTile(VectorSupportLibrary* vsl, llvm::IRBuilder<>* b,
-             llvm::Value* matrix, int64 matrix_size_along_minor_dim,
-             llvm::Value* major_dim_offset, int64 tile_size_along_major_dim)
-      : vsl_(vsl), b_(b) {
-    pointers_.reserve(tile_size_along_major_dim);
-    for (int64 i = 0; i < tile_size_along_major_dim; i++) {
-      llvm::Value* total_offset =
-          b->CreateMul(b->getInt64(matrix_size_along_minor_dim),
-                       b->CreateAdd(b->getInt64(i), major_dim_offset));
-      pointers_.push_back(vsl_->ComputeOffsetPointer(matrix, total_offset));
-    }
-  }
-
-  // Load a tile consisting of `tile_size_along_major_dim` vectors from position
-  // {major: `major_dim_offset`, minor: `minor_dim_offset`}.
-  //
-  // Note: `major_dim_offset` is a parameter to the constructor.
-  std::vector<llvm::Value*> LoadTile(llvm::Value* minor_dim_offset) const {
-    std::vector<llvm::Value*> result;
-    result.reserve(pointers_.size());
-    for (const auto& pointer : pointers_) {
-      result.push_back(vsl_->LoadVector(pointer, minor_dim_offset));
-    }
-    return result;
-  }
-
-  // Stores `tile` to position {major: `major_dim_offset`, minor:
-  // `minor_dim_offset`}.
-  //
-  // Note: `major_dim_offset` is a parameter to the constructor.
-  void StoreTile(absl::Span<llvm::Value* const> tile,
-                 llvm::Value* minor_dim_offset) const {
-    CHECK_EQ(tile.size(), pointers_.size());
-    for (int64 i = 0; i < pointers_.size(); i++) {
-      vsl_->StoreVector(tile[i], pointers_[i], minor_dim_offset);
-    }
-  }
-
-  // Loads a tile of size [`tile_size_along_major_dim`,
-  // `tile_size_along_middle_dim`] from position {major: `major_dim_offset`,
-  // minor: `minor_dim_offset`} and then broadcasts each element into a vector
-  // of size vsl_.vector_size().  The (i,j)'th element of the return value is
-  // the (i,j)'th element in the tile broadcasted into an LLVM vector.
-  //
-  // Note: `major_dim_offset` is a parameter to the constructor.
-  std::vector<std::vector<llvm::Value*>> LoadBroadcastTile(
-      llvm::Value* minor_dim_offset, int64 tile_size_along_middle_dim) const {
-    std::vector<std::vector<llvm::Value*>> result;
-    result.resize(pointers_.size());
-    for (int64 i = 0; i < pointers_.size(); i++) {
-      for (int64 j = 0; j < tile_size_along_middle_dim; j++) {
-        result[i].push_back(vsl_->LoadBroadcast(
-            pointers_[i], b_->CreateAdd(minor_dim_offset, b_->getInt64(j))));
-      }
-    }
-    return result;
-  }
-
- private:
-  VectorSupportLibrary* vsl_;
-  llvm::IRBuilder<>* b_;
-  std::vector<llvm::Value*> pointers_;
-};
-
-// The base class for the classes representing the GEMV emitter configurations.
-//
-// The IR emitted (modulo the LLVM values representing the input and output
-// buffers) by the row major and column major GEMV emitters should be a function
-// of their configuration.  This is important because their configuration is
-// used as a key to cache the generated IR.
-class GemvConfig {
- public:
-  // Mixin for convenience.
-  template <typename T>
-  struct User {
-   public:
-    PrimitiveType scalar_type() const {
-      return derived().config().scalar_type();
-    }
-    int64 tile_rows() const { return derived().config().tile_rows(); }
-    int64 tile_cols() const { return derived().config().tile_cols(); }
-    int64 m() const { return derived().config().m(); }
-    int64 k() const { return derived().config().k(); }
-    int64 has_addend() const { return derived().config().has_addend(); }
-
-   private:
-    const T& derived() const { return *static_cast<const T*>(this); }
-  };
-
-  PrimitiveType scalar_type() const { return scalar_type_; }
-  int64 tile_rows() const { return tile_rows_; }
-  int64 tile_cols() const { return tile_cols_; }
-  int64 m() const { return m_; }
-  int64 k() const { return k_; }
-  bool has_addend() const { return has_addend_; }
-
-  string GetCacheKey() const {
-    return absl::StrCat(name_, "_", PrimitiveType_Name(scalar_type()), "_",
-                        tile_rows(), "_", tile_cols(), "_", m(), "_", k(),
-                        has_addend() ? "_with_addend" : "");
-  }
-
- protected:
-  explicit GemvConfig(string name, PrimitiveType scalar_type, int64 tile_rows,
-                      int64 tile_cols, int64 m, int64 k, bool has_addend)
-      : name_(std::move(name)),
-        scalar_type_(scalar_type),
-        tile_rows_(tile_rows),
-        tile_cols_(tile_cols),
-        m_(m),
-        k_(k),
-        has_addend_(has_addend) {}
-
- private:
-  string name_;
-  PrimitiveType scalar_type_;
-  int64 tile_rows_;
-  int64 tile_cols_;
-  int64 m_;
-  int64 k_;
-  bool has_addend_;
-};
-
-// Computes a dot product between "[M,K]{0,1} lhs" with a [K,1] vector (the
-// layout of the vector does not matter).  This implementation uses a tiling
-// scheme to improve performance.
-//
-// We logically separate the LHS matrix into four segments:
-//
-//   +----------------------+---+
-//   |                      |   |
-//   |                      |   |
-//   |         A            | B |
-//   |                      |   |
-//   |                      |   |
-//   |                      |   |
-//   +----------------------+---+
-//   |         C            | D |
-//   +----------------------+---+
-//
-// where A is the largest submatrix of the LHS that can be evenly dividied into
-// tiles.  For each tile in A, assuming tile_rows_ == tile_cols_ == 4, we have:
-//
-//   +---+---+---+---+       +--+--+--+--+
-//   |M00|M10|M20|M30|       |V0|V1|V2|V3|
-//   +---+---+---+---+       +--+--+--+--+
-//   |M01|M11|M21|M31| and   |V0|V1|V2|V3|
-//   +---+---+---+---+       +--+--+--+--+
-//   |M02|M12|M22|M32|       |V0|V1|V2|V3|
-//   +---+---+---+---+       +--+--+--+--+
-//   |M03|M13|M23|M33|       |V0|V1|V2|V3|
-//   +---+---+---+---+       +--+--+--+--+
-//
-// (Legend: rows are horizontal and columns are vertical; and each column is one
-// llvm::Value of a vector type)
-//
-// where:
-//
-//   a. The left tile is from the column major left matrix.
-//   b. The right tile is an elementwise broadcast of a [V0, V1, V2, V3]
-//      vector loaded from the RHS vector.
-//
-// As we iterate through the column dimension, we compute the change to the
-// result vector by an elementwise multiplication between the two tiles above
-// followed by a reduction along the major dimension:
-//
-//                     +-----------------------------------+
-//                     | M00*V0 + M10*V1 + M20*V2 + M30*V3 |
-//                     +-----------------------------------+
-//                     | M01*V0 + M11*V1 + M21*V2 + M31*V3 |
-// Result[R:R+4] +=    +-----------------------------------+
-//                     | M02*V0 + M12*V1 + M22*V2 + M32*V3 |
-//                     +-----------------------------------+
-//                     | M03*V0 + M13*V1 + M23*V2 + M33*V3 |
-//                     +-----------------------------------+
-//
-// Where R is the starting row for the tile.
-//
-// We have an inner epilogue loop to deal with the "C" submatrix and an outer
-// epilogue loop to deal with the B,D submarix.
-//
-// TODO(sanjoy): We should investigate if using gather loads and scatter stores
-// can be used here have the same inner loop for both column-major and row-major
-// matrix-vector products.
-class ColumnMajorMatrixVectorProductEmitter
-    : public GemvConfig::User<ColumnMajorMatrixVectorProductEmitter> {
- public:
-  class Config : public GemvConfig {
-   public:
-    explicit Config(PrimitiveType scalar_type, int64 tile_rows, int64 tile_cols,
-                    int64 m, int64 k, bool has_addend)
-        : GemvConfig(/*name=*/"col_major_gemv", scalar_type,
-                     /*tile_rows=*/tile_rows, /*tile_cols=*/tile_cols, /*m=*/m,
-                     /*k=*/k, /*has_addend=*/has_addend) {}
-  };
-
-  ColumnMajorMatrixVectorProductEmitter(const Config& config, llvm::Value* lhs,
-                                        llvm::Value* rhs, llvm::Value* addend,
-                                        llvm::Value* result,
-                                        llvm::IRBuilder<>* b)
-      : config_(config),
-        lhs_(lhs),
-        rhs_(rhs),
-        addend_(addend),
-        result_(result),
-        b_(b),
-        ksl_(b_),
-        vsl_(config.scalar_type(), /*vector_size=*/config.tile_rows(), b_, "") {
-    CHECK(tile_rows() > 0 && IsPowerOfTwo(static_cast<uint64>(tile_rows())));
-    CHECK(!has_addend() || addend != nullptr);
-  }
-
-  void Emit();
-
-  const Config& config() const { return config_; }
-
- private:
-  void EmitOuterLoopBody(llvm::Value* column, int64 column_count,
-                         bool is_first_column);
-
-  MemoryTile GetLhsMemoryTile(llvm::Value* column_start, int64 column_count) {
-    return MemoryTile(&vsl_, b_, /*matrix=*/lhs_,
-                      /*matrix_size_along_minor_dim=*/m(),
-                      /*major_dim_offset=*/column_start,
-                      /*tile_size_along_major_dim=*/column_count);
-  }
-
-  // Load a tile of values from the RHS.  For the RHS a "tile" is a contiguous
-  // sequence of `count` values, each one broadcasted to the vector width.
-  std::vector<llvm::Value*> LoadRhsTile(llvm::Value* offset, int64 count) {
-    llvm::Value* base_pointer = vsl_.ComputeOffsetPointer(rhs_, offset);
-    std::vector<llvm::Value*> result;
-    result.reserve(count);
-    for (int64 i = 0; i < count; i++) {
-      result.push_back(vsl_.LoadBroadcast(base_pointer, i));
-    }
-    return result;
-  }
-
-  void EmitInnerLoopTiled(MemoryTile* lhs_memory_tile,
-                          const std::vector<llvm::Value*>& rhs_tile,
-                          int64 columns, bool is_first_column);
-
-  void EmitInnerLoopEpilogue(llvm::Value* current_tile_col, int64 columns,
-                             bool is_first_tiled_column);
-
-  Config config_;
-  llvm::Value* lhs_;
-  llvm::Value* rhs_;
-  llvm::Value* addend_;
-  llvm::Value* result_;
-  llvm::IRBuilder<>* b_;
-  KernelSupportLibrary ksl_;
-  VectorSupportLibrary vsl_;
-};
-
-void ColumnMajorMatrixVectorProductEmitter::EmitOuterLoopBody(
-    llvm::Value* column, int64 column_count, bool is_first_column) {
-  MemoryTile lhs_memory_tile = GetLhsMemoryTile(/*column_start=*/column,
-                                                /*column_count=*/column_count);
-
-  std::vector<llvm::Value*> rhs_tile =
-      LoadRhsTile(column, /*count=*/column_count);
-  EmitInnerLoopTiled(&lhs_memory_tile, rhs_tile,
-                     /*columns=*/column_count, is_first_column);
-  EmitInnerLoopEpilogue(column, /*columns=*/column_count, is_first_column);
-}
-
-void ColumnMajorMatrixVectorProductEmitter::Emit() {
-  // See the comment on the class declaration for the algorithm used here.
-  int64 column_remainder = k() % tile_cols();
-  int64 column_limit = k() - column_remainder;
-
-  ksl_.For("dot.outer.tiled",
-           /*start=*/0, /*end=*/column_limit, /*step=*/tile_cols(),
-           [&](llvm::Value* column, bool is_first_column) {
-             EmitOuterLoopBody(column, tile_cols(), is_first_column);
-           });
-
-  if (column_remainder != 0) {
-    EmitOuterLoopBody(b_->getInt64(column_limit), column_remainder,
-                      column_limit == 0);
-  }
-}
-
-void ColumnMajorMatrixVectorProductEmitter::EmitInnerLoopTiled(
-    MemoryTile* lhs_memory_tile, const std::vector<llvm::Value*>& rhs_tile,
-    int64 columns, bool is_first_column) {
-  int64 row_limit = m() - (m() % tile_rows());
-
-  ksl_.For(
-      "dot.inner.tiled", /*start=*/0, /*end=*/row_limit,
-      /*step=*/tile_rows(), [&](llvm::Value* row) {
-        std::vector<llvm::Value*> lhs_tile =
-            lhs_memory_tile->LoadTile(/*minor_dim_offset=*/row);
-        llvm::Value* accumulator =
-            is_first_column ? (addend_ ? vsl_.LoadVector(addend_, row)
-                                       : vsl_.GetZeroVector())
-                            : vsl_.LoadVector(result_, row);
-        for (int i = 0; i < columns; i++) {
-          accumulator = vsl_.MulAdd(lhs_tile[i], rhs_tile[i], accumulator);
-        }
-        vsl_.StoreVector(accumulator, result_, row);
-      });
-}
-
-void ColumnMajorMatrixVectorProductEmitter::EmitInnerLoopEpilogue(
-    llvm::Value* current_tile_col, int64 columns, bool is_first_tiled_column) {
-  int64 row_start = m() - (m() % tile_rows());
-  if (row_start == m()) {
-    return;
-  }
-
-  llvm::Value* columns_llvm = b_->getInt64(columns);
-
-  // for (col = current_tile_col; col < (columns + current_tile_col); col++)
-  //   for (row = row_start, row < m_; row++) {
-  //     result[row] += lhs[row, col] * rhs[col]
-  //     // Also take into account that if col is 0 then result[row] is not
-  //     // initialized.
-  //   }
-
-  ksl_.For(
-      "dot.inner.epilg.outer", /*start=*/current_tile_col,
-      /*end=*/b_->CreateAdd(columns_llvm, current_tile_col),
-      /*step=*/1, /*peel_first_iteration=*/false,
-      [&](llvm::Value* col, llvm::Value* is_first_scalar_col) {
-        llvm::Value* rhs_element = vsl_.LoadScalar(rhs_, col);
-        llvm::Value* total_offset = b_->CreateMul(col, b_->getInt64(m()));
-        llvm::Value* lhs_base_pointer =
-            vsl_.ComputeOffsetPointer(lhs_, total_offset);
-        ksl_.For(
-            "dot.inner.epilg.inner", /*start=*/row_start, /*end=*/m(),
-            /*step=*/1, [&](llvm::Value* scalar_row) {
-              llvm::Value* product = vsl_.Mul(
-                  vsl_.LoadScalar(lhs_base_pointer, scalar_row), rhs_element);
-              llvm::Value* setting_result_first_time = b_->CreateAnd(
-                  is_first_scalar_col, b_->getInt1(is_first_tiled_column));
-              ksl_.If(
-                  setting_result_first_time,
-                  /*true_block_generator=*/
-                  [&]() {
-                    if (addend_) {
-                      vsl_.StoreScalar(
-                          vsl_.Add(vsl_.LoadScalar(addend_, scalar_row),
-                                   product),
-                          result_, scalar_row);
-                    } else {
-                      vsl_.StoreScalar(product, result_, scalar_row);
-                    }
-                  },
-                  /*false_block_generator=*/
-                  [&]() {
-                    vsl_.StoreScalar(
-                        vsl_.Add(vsl_.LoadScalar(result_, scalar_row), product),
-                        result_, scalar_row);
-                  });
-            });
-      });
-}
-
-// Computes a dot product between "[M,K]{1,0} lhs" with a [K,1] vector (the
-// layout of the vector does not matter).  This implementation uses a tiling
-// scheme to improve performance.
-//
-// We logically separate the LHS matrix into four segments:
-//
-//   +----------------------+---+
-//   |                      |   |
-//   |                      |   |
-//   |         A            | B |
-//   |                      |   |
-//   |                      |   |
-//   |                      |   |
-//   +----------------------+---+
-//   |         C            | D |
-//   +----------------------+---+
-//
-// where A is the largest submatrix of the LHS that can be evenly dividied into
-// tiles.  For each tile in A, assuming tile_rows_ == tile_cols_ == 4, we have:
-//
-//   +---+---+---+---+
-//   |M00|M10|M20|M30|
-//   +---+---+---+---+       +--+--+--+--+
-//   |M01|M11|M21|M31| and   |V0|V1|V2|V3|
-//   +---+---+---+---+       +--+--+--+--+
-//   |M02|M12|M22|M32|
-//   +---+---+---+---+
-//   |M03|M13|M23|M33|
-//   +---+---+---+---+
-//
-// (Legend: rows are horizontal and columns are vertical; and each row is one
-// llvm::Value of a vector type)
-//
-// where:
-//
-//   a. The left tile is loaded from the row major left matrix.
-//   b. The right vector is loaded from the RHS vector.
-//
-// We keep 4 vector accumulators accumulating the following four vector
-// expressions as we iterate over the row dimension:
-//
-//   +------+------+------+------+
-//   |M0I*V0|M1I*V1|M2I*V2|M3I*V3|  for I in [0,4)
-//   +------+------+------+------+
-//
-// In the end we do a horizontal reduction over these 4 vector accumulators to
-// get 4 values in the result vector.
-//
-// We have an inner epilogue loop to deal with the "B" sub-matrix and an outer
-// epilogue loop to deal with the C,D submatrix.
-class RowMajorMatrixVectorProductEmitter
-    : public GemvConfig::User<RowMajorMatrixVectorProductEmitter> {
- public:
-  class Config : public GemvConfig {
-   public:
-    explicit Config(PrimitiveType scalar_type, int64 tile_rows, int64 tile_cols,
-                    int64 m, int64 k, bool has_addend)
-        : GemvConfig(/*name=*/"row_major_gemv", scalar_type,
-                     /*tile_rows=*/tile_rows, /*tile_cols=*/tile_cols, /*m=*/m,
-                     /*k=*/k, /*has_addend=*/has_addend) {}
-  };
-
-  RowMajorMatrixVectorProductEmitter(const Config& config, llvm::Value* lhs,
-                                     llvm::Value* rhs, llvm::Value* addend,
-                                     llvm::Value* result, llvm::IRBuilder<>* b)
-      : config_(config),
-        lhs_(lhs),
-        rhs_(rhs),
-        addend_(addend),
-        result_(result),
-        b_(b),
-        ksl_(b_),
-        vsl_(scalar_type(), /*vector_size=*/tile_cols(), b_, "") {
-    CHECK(tile_cols() > 0 && IsPowerOfTwo(static_cast<uint64>(tile_cols())));
-    CHECK(!has_addend() || addend != nullptr);
-  }
-
-  void Emit();
-
-  const Config& config() const { return config_; }
-
- private:
-  MemoryTile GetLhsMemoryTile(llvm::Value* row_start, int64 row_count) {
-    return MemoryTile(&vsl_, b_, /*matrix=*/lhs_,
-                      /*matrix_size_along_minor_dim=*/k(),
-                      /*major_dim_offset=*/row_start,
-                      /*tile_size_along_major_dim=*/row_count);
-  }
-
-  void EmitOuterLoopBody(llvm::Value* row, int64 row_count);
-
-  void EmitInnerLoopTiled(MemoryTile* lhs_memory_tile, int64 rows,
-                          std::vector<VectorVariable>* vector_accumulators);
-
-  void EmitInnerLoopEpilogue(llvm::Value* current_tile_row, int64 rows,
-                             std::vector<ScalarVariable>* scalar_accumulators);
-
-  Config config_;
-  llvm::Value* lhs_;
-  llvm::Value* rhs_;
-  llvm::Value* addend_;
-  llvm::Value* result_;
-  llvm::IRBuilder<>* b_;
-  KernelSupportLibrary ksl_;
-  VectorSupportLibrary vsl_;
-};
-
-void RowMajorMatrixVectorProductEmitter::EmitOuterLoopBody(llvm::Value* row,
-                                                           int64 row_count) {
-  MemoryTile lhs_memory_tile = GetLhsMemoryTile(/*row_start=*/row,
-                                                /*row_count=*/row_count);
-  std::vector<VectorVariable> vector_accumulators;
-  std::vector<ScalarVariable> scalar_accumulators;
-  for (int i = 0; i < row_count; i++) {
-    vector_accumulators.emplace_back(&vsl_, vsl_.GetZeroVector());
-    scalar_accumulators.emplace_back(&vsl_, vsl_.GetZeroScalar());
-  }
-  EmitInnerLoopTiled(&lhs_memory_tile, /*rows=*/row_count,
-                     &vector_accumulators);
-  EmitInnerLoopEpilogue(/*current_tile_row=*/row, /*rows=*/row_count,
-                        &scalar_accumulators);
-
-  std::vector<llvm::Value*> accumulator_values;
-  std::transform(
-      vector_accumulators.begin(), vector_accumulators.end(),
-      std::back_inserter(accumulator_values),
-      [](const VectorVariable& vector_var) { return vector_var.Get(); });
-
-  std::vector<llvm::Value*> horizontal_sums;
-  if (row_count == vsl_.vector_size()) {
-    if (addend_) {
-      horizontal_sums = vsl_.ComputeHorizontalSums(
-          std::move(accumulator_values), vsl_.LoadVector(addend_, row));
-    } else {
-      horizontal_sums =
-          vsl_.ComputeHorizontalSums(std::move(accumulator_values));
-    }
-  } else {
-    horizontal_sums = vsl_.ComputeHorizontalSums(std::move(accumulator_values));
-  }
-
-  for (int i = 0; i < row_count; i++) {
-    llvm::Value* result_value =
-        vsl_.Add(horizontal_sums[i], scalar_accumulators[i].Get());
-    llvm::Value* offset = b_->CreateAdd(b_->getInt64(i), row);
-    if (addend_ && row_count != vsl_.vector_size()) {
-      result_value = vsl_.Add(vsl_.LoadScalar(addend_, offset), result_value);
-    }
-    vsl_.StoreScalar(result_value, result_, offset);
-  }
-}
-
-void RowMajorMatrixVectorProductEmitter::Emit() {
-  // See the comment on the class declaration for the algorithm used here.
-  int64 row_remainder = m() % tile_rows();
-  int64 row_limit = m() - row_remainder;
-
-  ksl_.For("dot.outer.tiled",
-           /*start=*/0, /*end=*/row_limit, /*step=*/tile_rows(),
-           [&](llvm::Value* row) { EmitOuterLoopBody(row, tile_rows()); });
-
-  if (row_remainder != 0) {
-    EmitOuterLoopBody(b_->getInt64(row_limit), row_remainder);
-  }
-}
-
-void RowMajorMatrixVectorProductEmitter::EmitInnerLoopTiled(
-    MemoryTile* lhs_memory_tile, int64 rows,
-    std::vector<VectorVariable>* vector_accumulators) {
-  int64 column_limit = k() - (k() % tile_cols());
-
-  ksl_.For("dot.inner.tiled", /*start=*/0, /*end=*/column_limit,
-           /*step=*/tile_cols(), [&](llvm::Value* col) {
-             std::vector<llvm::Value*> lhs_tile =
-                 lhs_memory_tile->LoadTile(/*minor_dim_offset=*/col);
-             llvm::Value* rhs_value = vsl_.LoadVector(rhs_, col);
-             for (int i = 0; i < rows; i++) {
-               llvm::Value* old_sum = (*vector_accumulators)[i].Get();
-               (*vector_accumulators)[i].Set(
-                   vsl_.Add(old_sum, vsl_.Mul(rhs_value, lhs_tile[i])));
-             }
-           });
-}
-
-void RowMajorMatrixVectorProductEmitter::EmitInnerLoopEpilogue(
-    llvm::Value* current_tile_row, int64 rows,
-    std::vector<ScalarVariable>* scalar_accumulators) {
-  int64 column_start = k() - (k() % tile_cols());
-  if (column_start == k()) {
-    return;
-  }
-
-  for (int r = 0; r < rows; r++) {
-    llvm::Value* total_offset = b_->CreateMul(
-        b_->CreateAdd(b_->getInt64(r), current_tile_row), b_->getInt64(k()));
-    llvm::Value* lhs_base_pointer =
-        vsl_.ComputeOffsetPointer(lhs_, total_offset);
-    ksl_.For(
-        "dot.inner.epilg.inner", /*start=*/column_start, /*end=*/k(),
-        /*step=*/1, [&](llvm::Value* scalar_col) {
-          llvm::Value* product =
-              vsl_.Mul(vsl_.LoadScalar(lhs_base_pointer, scalar_col),
-                       vsl_.LoadScalar(rhs_, scalar_col));
-          llvm::Value* old_value = (*scalar_accumulators)[r].Get();
-          (*scalar_accumulators)[r].Set(vsl_.Add(old_value, product));
-        });
-  }
-}
-
-// This class implements a tiled matrix multiplication algorithm, intended for
-// multiplying small matrices that don't need cache tiling.
-//
-// In the future this can be used as the innermost GEBP loop in a GEMM kernel as
-// described in "Goto, Kazushige, and Robert A. Geijn. "Anatomy of
-// high-performance matrix multiplication." ACM Transactions on Mathematical
-// Software (TOMS) 34.3 (2008): 12.".
-//
-// This only supports canonical dot operations (i.e. where the lhs contraction
-// dimension is 1 and the rhs contraction dimension is 0) over row major
-// matrices.
-class TiledSmallGemmEmitter {
- public:
-  // Describe the dimensions of the kernel.
-  class Dimensions {
-   public:
-    explicit Dimensions(int64 m, int64 k, int64 n) : m_(m), k_(k), n_(n) {}
-
-    int64 m() const { return m_; }
-    int64 k() const { return k_; }
-    int64 n() const { return n_; }
-
-    string ToString() const { return absl::StrCat(m(), "x", k(), "x", n()); }
-
-   private:
-    const int64 m_;
-    const int64 k_;
-    const int64 n_;
-  };
-
-  // Represents the configuration of the emitter.  The LLVM IR emitted by the
-  // emitter, modulo the LLVM values holding the input and output buffers, must
-  // be a function of the instance of `Config` passed to it.
-  //
-  // `dims` holds the matrix multiplication dimensions.
-  //
-  // `max_vectorization_width` is the maximum vector width (i.e. the width of
-  // the largest vector register we will use).  This can be larger than the
-  // largest vector register supported by the machine -- LLVM will legalize
-  // these large vector widths into legally sized vectors.
-  //
-  // `max_vector_count` is the maximum number of vectors of size
-  // `max_vectorization_width` that we will attempt to process at once.
-  //
-  // `min_vectorization_width` is the smallest vector width the emitter will use
-  // -- below that it will devolve to using a scalar loop.
-  //
-  // The innermost reduction loop executes the matrix multiply in tiles of size
-  // [`tile_size_m`, `tile_size_k`] from the LHS and [`tile_size_k`,
-  // <vectorization width>] in the RHS.
-  class Config {
-   public:
-    explicit Config(PrimitiveType scalar_type, Dimensions dims,
-                    int64 max_vectorization_width, int64 max_vector_count,
-                    int64 min_vectorization_width, int64 tile_size_m,
-                    int64 tile_size_k)
-        : scalar_type_(scalar_type),
-          dims_(dims),
-          max_vectorization_width_(max_vectorization_width),
-          max_vector_count_(max_vector_count),
-          min_vectorization_width_(min_vectorization_width),
-          tile_size_m_(tile_size_m),
-          tile_size_k_(tile_size_k) {}
-
-    string GetCacheKey() const {
-      return absl::StrCat("gemm_", PrimitiveType_Name(scalar_type()), "_",
-                          dims().ToString(), "_", max_vectorization_width(),
-                          "_", min_vectorization_width(), "_", tile_size_m(),
-                          "_", tile_size_k());
-    }
-
-    PrimitiveType scalar_type() const { return scalar_type_; }
-    Dimensions dims() const { return dims_; }
-    int64 max_vectorization_width() const { return max_vectorization_width_; }
-    int64 max_vector_count() const { return max_vector_count_; }
-    int64 min_vectorization_width() const { return min_vectorization_width_; }
-
-    int64 tile_size_m() const { return tile_size_m_; }
-    int64 tile_size_k() const { return tile_size_k_; }
-
-   private:
-    PrimitiveType scalar_type_;
-    Dimensions dims_;
-    int64 max_vectorization_width_;
-    int64 max_vector_count_;
-    int64 min_vectorization_width_;
-    int64 tile_size_m_;
-    int64 tile_size_k_;
-  };
-
-  // Creates an instance of TiledSmallGemmEmitter that matrix-multiplies
-  // `lhs` with `rhs` and stores the result in `result`.
-  explicit TiledSmallGemmEmitter(Config config, llvm::Value* lhs,
-                                 llvm::Value* rhs, llvm::Value* result,
-                                 llvm::IRBuilder<>* b)
-      : lhs_(lhs),
-        rhs_(rhs),
-        result_(result),
-        config_(config),
-        b_(b),
-        ksl_(b_) {
-    CHECK(max_vectorization_width() > 0 &&
-          IsPowerOfTwo(static_cast<uint64>(max_vectorization_width())));
-    CHECK_GT(max_vector_count(), 0);
-    CHECK(min_vectorization_width() > 0 &&
-          IsPowerOfTwo(static_cast<uint64>(min_vectorization_width())));
-    CHECK_GE(max_vectorization_width(), min_vectorization_width());
-    CHECK_GT(tile_size_k(), 0);
-  }
-
-  void Emit();
-
- private:
-  // The HandleResiduesOnX helpers split the iteration space for dimension X
-  // into a multiple of the tile size on dimension X and an epilogue.  These
-  // helpers ultimately call into `EmitTiledGemm` for emitting the
-  // tiled GEMM kernel.
-
-  void HandleResiduesOnN();
-  void HandleResiduesOnK(VectorSupportLibrary* vsl, llvm::Value* n_start,
-                         llvm::Value* n_end);
-  void HandleResiduesOnM(VectorSupportLibrary* vsl, int64 tile_size_k,
-                         llvm::Value* k_start, llvm::Value* k_end,
-                         llvm::Value* n_start, llvm::Value* n_end);
-
-  // This emits a tiled GEMM kernel.  For a detailed description see the comment
-  // on the implementation.
-  void EmitTiledGemm(VectorSupportLibrary* vsl, int64 tile_size_k,
-                     llvm::Value* k_start, llvm::Value* k_end,
-                     llvm::Value* n_start, llvm::Value* n_end,
-                     int64 tile_size_m, llvm::Value* m_start,
-                     llvm::Value* m_end);
-
-  llvm::Value* GetInt64(int64 value) { return b_->getInt64(value); }
-
-  Config config() const { return config_; }
-  Dimensions dims() const { return config().dims(); }
-
-  int64 max_vectorization_width() const {
-    return config().max_vectorization_width();
-  }
-  int64 max_vector_count() const { return config().max_vector_count(); }
-  int64 min_vectorization_width() const {
-    return config().min_vectorization_width();
-  }
-  int64 tile_size_m() const { return config().tile_size_m(); }
-  int64 tile_size_k() const { return config().tile_size_k(); }
-  PrimitiveType scalar_type() const { return config().scalar_type(); }
-
-  llvm::Value* lhs_;
-  llvm::Value* rhs_;
-  llvm::Value* result_;
-  Config config_;
-
-  llvm::IRBuilder<>* b_;
-  KernelSupportLibrary ksl_;
-};
-
-void TiledSmallGemmEmitter::Emit() { HandleResiduesOnN(); }
-
-void TiledSmallGemmEmitter::HandleResiduesOnN() {
-  // We can only iterate the `n` dimension for an extent that is divisible by
-  // the vectorization width.  So we emit an outer loop that first processes the
-  // largest extent in `n` that is divisible by max_vectorization_width, then
-  // the largest remaining extent that is divisible by max_vectorization_width /
-  // 2 etc.
-
-  int64 current_vectorization_width =
-      max_vector_count() * max_vectorization_width();
-  int64 current_vector_count = max_vector_count();
-
-  int64 n_start = 0;
-  while (n_start != dims().n() &&
-         current_vectorization_width >= min_vectorization_width()) {
-    int64 n_end = dims().n() - (dims().n() % current_vectorization_width);
-    if (n_start != n_end) {
-      VectorSupportLibrary vsl(scalar_type(), current_vectorization_width, b_,
-                               "gemm");
-      HandleResiduesOnK(&vsl, GetInt64(n_start), GetInt64(n_end));
-      n_start = n_end;
-    }
-    if (current_vector_count == 1) {
-      current_vectorization_width /= 2;
-    } else {
-      current_vector_count--;
-      current_vectorization_width =
-          current_vector_count * max_vectorization_width();
-    }
-  }
-
-  if (n_start != dims().n()) {
-    VectorSupportLibrary vsl(scalar_type(), 1, b_, "gemm");
-    ksl_.For("epi.n", n_start, dims().n(), 1, [&](llvm::Value* n_i) {
-      llvm::Value* n_i_next = b_->CreateAdd(n_i, b_->getInt64(1));
-      HandleResiduesOnK(&vsl, n_i, n_i_next);
-    });
-  }
-}
-
-void TiledSmallGemmEmitter::HandleResiduesOnK(VectorSupportLibrary* vsl,
-                                              llvm::Value* n_start,
-                                              llvm::Value* n_end) {
-  int64 k_start = 0;
-  int64 k_end = dims().k() - (dims().k() % tile_size_k());
-  if (k_end != k_start) {
-    HandleResiduesOnM(vsl, tile_size_k(), GetInt64(k_start), GetInt64(k_end),
-                      n_start, n_end);
-    k_start = k_end;
-  }
-
-  if (k_start != dims().k()) {
-    HandleResiduesOnM(vsl, dims().k() - k_start, GetInt64(k_start),
-                      GetInt64(dims().k()), n_start, n_end);
-  }
-}
-
-void TiledSmallGemmEmitter::HandleResiduesOnM(
-    VectorSupportLibrary* vsl, int64 tile_size_k, llvm::Value* k_start,
-    llvm::Value* k_end, llvm::Value* n_start, llvm::Value* n_end) {
-  const int64 m_end = dims().m() - dims().m() % tile_size_m();
-  EmitTiledGemm(vsl, tile_size_k, k_start, k_end, n_start, n_end, tile_size_m(),
-                GetInt64(0), GetInt64(m_end));
-
-  if (m_end != dims().m()) {
-    EmitTiledGemm(vsl, tile_size_k, k_start, k_end, n_start, n_end,
-                  dims().m() - m_end, GetInt64(m_end), GetInt64(dims().m()));
-  }
-}
-
-// The loop structure is:
-//
-// Iterate over dimension M as m:
-//   Iterate over dimension N as n:
-//     Iterate over dimension K as k:
-//       OutputTile[m,n] += Dot(LhsTile[m,k], RhsTile[k,n])
-//
-// I.e. a just a tiled version of a "naive" GEMM.
-//
-// The tiling scheme is as follows:
-//
-// Let the LHS be:
-//
-//   +----+----+----+
-//   | a0 | b0 | c0 | .
-//   +----+----+----+ .
-//   | a1 | b1 | c1 | .
-//   +----+----+----+
-//     ..     ..
-//
-// and the RHS be:
-//
-//   +----+----+----+----+
-//   | p0 | p1 | p2 | p3 | .
-//   +----+----+----+----+ .
-//   | q0 | q1 | q2 | q3 | .
-//   +----+----+----+----+
-//   | r0 | r1 | r2 | r3 | .
-//   +----+----+----+----+ .
-//     ......    ......
-//
-// and let tile_size_m=2, tile_size_k=3 and the vector width (implicitly denoted
-// by `vsl`) be 4.  Then we want to matrix multiply this tile to get a [2,4]
-// matrix that we can increment the result matrix by.
-//
-// First broadcast the rows row in LHS to 3 vectors of width 4, giving us a rank
-// 3 array, L, of dimension [2,3,4]:
-//
-//       L[0,_,_]           *      L[1,_,_]
-//                          *
-//   +----+----+----+----+  *  +----+----+----+----+
-//   | a0 | a0 | a0 | a0 |  *  | a1 | a1 | a1 | a1 |
-//   +----+----+----+----+  *  +----+----+----+----+
-//   | b0 | b0 | b0 | b0 |  *  | b1 | b1 | b1 | b1 |
-//   +----+----+----+----+  *  +----+----+----+----+
-//   | c0 | c0 | c0 | c0 |  *  | c1 | c1 | c1 | c1 |
-//   +----+----+----+----+  *  +----+----+----+----+
-//
-//
-// Then we FMA L[0,_,_] with the RHS to get the first row of the result and
-// L[1,_,_] with the RHS to get the second row of the result.  For example,
-// L[0,_,_] is computed as:
-//
-//   +----+----+----+----+   +----+----+----+----+
-//   | a0 | a0 | a0 | a0 | * | p0 | p1 | p2 | p3 |   +
-//   +----+----+----+----+   +----+----+----+----+
-//
-//   +----+----+----+----+   +----+----+----+----+
-//   | b0 | b0 | b0 | b0 | * | q0 | q1 | q2 | q3 |   +
-//   +----+----+----+----+   +----+----+----+----+
-//
-//   +----+----+----+----+   +----+----+----+----+
-//   | c0 | c0 | c0 | c0 | * | r0 | r1 | r2 | r3 |
-//   +----+----+----+----+   +----+----+----+----+
-//
-// to get:
-//
-//   +-------------------+-------------------+-------------------+---------
-//   | a0*p0+b0*q0+c0*r0 | a0*p1+b0*q1+c0*r1 | a0*p2+b0*q2+c0*r2 |  ...
-//   +-------------------+-------------------+-------------------+---------
-void TiledSmallGemmEmitter::EmitTiledGemm(
-    VectorSupportLibrary* vsl, int64 tile_size_k, llvm::Value* k_start,
-    llvm::Value* k_end, llvm::Value* n_start, llvm::Value* n_end,
-    int64 tile_size_m, llvm::Value* m_start, llvm::Value* m_end) {
-  ksl_.For(
-      "dot.m", m_start, m_end, tile_size_m, [&](llvm::Value* m_i) {
-        MemoryTile result_memory_tile(
-            vsl, b_, /*matrix=*/result_,
-            /*matrix_size_along_minor_dim=*/dims().n(),
-            /*major_dim_offset=*/m_i,
-            /*tile_size_along_major_dim=*/tile_size_m);
-        MemoryTile lhs_memory_tile(vsl, b_, /*matrix=*/lhs_,
-                                   /*matrix_size_along_minor_dim=*/dims().k(),
-                                   /*major_dim_offset=*/m_i,
-                                   /*tile_size_along_major_dim=*/tile_size_m);
-        ksl_.For(
-            "dot.n", n_start, n_end, vsl->vector_size(), [&](llvm::Value* n_i) {
-              TileVariable result_tile_var(vsl,
-                                           result_memory_tile.LoadTile(n_i));
-              ksl_.For(
-                  "dot.k", k_start, k_end, tile_size_k, [&](llvm::Value* k_i) {
-                    MemoryTile rhs_memory_tile(vsl, b_, rhs_, dims().n(), k_i,
-                                               tile_size_k);
-                    std::vector<std::vector<llvm::Value*>> lhs_tile =
-                        lhs_memory_tile.LoadBroadcastTile(k_i, tile_size_k);
-                    std::vector<llvm::Value*> rhs_tile =
-                        rhs_memory_tile.LoadTile(n_i);
-                    std::vector<llvm::Value*> result_tile =
-                        result_tile_var.Get();
-                    for (int64 r_m_i = 0; r_m_i < tile_size_m; r_m_i++) {
-                      for (int64 r_k_i = 0; r_k_i < tile_size_k; r_k_i++) {
-                        result_tile[r_m_i] =
-                            vsl->MulAdd(lhs_tile[r_m_i][r_k_i], rhs_tile[r_k_i],
-                                        result_tile[r_m_i]);
-                      }
-                    }
-                    result_tile_var.Set(result_tile);
-                  });
-
-              result_memory_tile.StoreTile(result_tile_var.Get(), n_i);
-            });
-      });
-}
-
-}  // namespace
-
 DotOpEmitter::DotOpEmitter(const HloInstruction& dot,
                            const llvm_ir::IrArray& target_array,
                            const llvm_ir::IrArray& lhs_array,
@@ -1062,32 +139,21 @@ bool DotOpEmitter::EmitSmallGemmIfProfitable(
   std::tie(tile_size_m, tile_size_k, tile_size_n_in_vector_width) =
       GetGemmTileSize();
 
-  TiledSmallGemmEmitter::Config config(
-      /*scalar_type=*/primitive_type,
-      TiledSmallGemmEmitter::Dimensions{/*m=*/m, /*k=*/k, /*n=*/n},
-      /*max_vectorization_width=*/max_target_vector_width,
-      /*max_vector_count=*/tile_size_n_in_vector_width,
-      /*min_vectorization_width=*/std::min<int64>(4, max_target_vector_width),
-      /*tile_size_m=*/tile_size_m, /*tile_size_k=*/tile_size_k);
-
-  VLOG(2) << "Emitting GEMM kernel in LLVM IR with config "
-          << config.GetCacheKey();
-
   const bool enable_fast_math =
       hlo_module_config_.debug_options().xla_cpu_enable_fast_math();
   const bool optimize_for_size =
       options::OptimizeForSizeRequested(hlo_module_config_);
 
-  KernelSupportLibrary::EmitAndCallOutlinedKernel(
+  EmitSmallGemm(
+      /*scalar_type=*/primitive_type,
+      /*m=*/m, /*k=*/k, /*n=*/n,
+      /*max_vectorization_width=*/max_target_vector_width,
+      /*max_vector_count=*/tile_size_n_in_vector_width,
+      /*min_vectorization_width=*/std::min<int64>(4, max_target_vector_width),
+      /*tile_size_m=*/tile_size_m, /*tile_size_k=*/tile_size_k, /*lhs=*/lhs,
+      /*rhs=*/rhs, /*result=*/target, b_,
       /*enable_fast_math=*/enable_fast_math,
-      /*optimize_for_size=*/optimize_for_size, b_, config.GetCacheKey(), lhs,
-      rhs, target,
-      [this, config](llvm::Value* lhs, llvm::Value* rhs, llvm::Value* target) {
-        TiledSmallGemmEmitter small_gemm_emitter(config, /*lhs=*/lhs,
-                                                 /*rhs=*/rhs,
-                                                 /*result=*/target, b_);
-        small_gemm_emitter.Emit();
-      });
+      /*optimize_for_size=*/optimize_for_size);
 
   return true;
 }
@@ -1177,41 +243,26 @@ bool DotOpEmitter::EmitLlvmIrDotIfProfitable() {
   if (is_column_major_matrix_vector) {
     VLOG(2) << "Emitting column major matrix-vector multiply with m = " << m
             << " and k = " << k;
-    ColumnMajorMatrixVectorProductEmitter::Config config(
+    EmitColumnMajorGemv(
         /*scalar_type=*/primitive_type,
         /*tile_rows=*/vector_register_element_size, /*tile_cols=*/tiling_factor,
-        /*m=*/m, /*k=*/k, /*has_addend=*/addend_array_ != nullptr);
-
-    KernelSupportLibrary::EmitAndCallOutlinedKernel(
+        /*m=*/m, /*k=*/k, /*lhs=*/lhs_op, /*rhs=*/rhs_op,
+        /*addend=*/addend_array_ ? addend_array_->GetBasePointer() : nullptr,
+        /*result=*/result_op, b_,
         /*enable_fast_math=*/enable_fast_math,
-        /*optimize_for_size=*/optimize_for_size, b_, config.GetCacheKey(),
-        lhs_op, rhs_op,
-        addend_array_ ? addend_array_->GetBasePointer() : nullptr, result_op,
-        [this, config](llvm::Value* lhs_op, llvm::Value* rhs_op,
-                       llvm::Value* addend_op, llvm::Value* result_op) {
-          ColumnMajorMatrixVectorProductEmitter emitter(
-              config, lhs_op, rhs_op, addend_op, result_op, b_);
-          emitter.Emit();
-        });
+        /*optimize_for_size=*/optimize_for_size);
   } else {
     VLOG(2) << "Emitting row major matrix-vector multiply with m = " << m
             << " and k = " << k;
-    RowMajorMatrixVectorProductEmitter::Config config(
+    EmitRowMajorGemv(
         /*scalar_type=*/primitive_type,
-        /*tile_rows=*/tiling_factor, /*tile_cols=*/vector_register_element_size,
-        /*m=*/m, /*k=*/k, /*has_addend=*/addend_array_ != nullptr);
-
-    KernelSupportLibrary::EmitAndCallOutlinedKernel(
+        /*tile_rows=*/tiling_factor,
+        /*tile_cols=*/vector_register_element_size,
+        /*m=*/m, /*k=*/k, /*lhs=*/lhs_op, /*rhs=*/rhs_op,
+        /*addend=*/addend_array_ ? addend_array_->GetBasePointer() : nullptr,
+        /*result=*/result_op, b_,
         /*enable_fast_math=*/enable_fast_math,
-        /*optimize_for_size=*/optimize_for_size, b_, config.GetCacheKey(),
-        lhs_op, rhs_op,
-        addend_array_ ? addend_array_->GetBasePointer() : nullptr, result_op,
-        [this, config](llvm::Value* lhs_op, llvm::Value* rhs_op,
-                       llvm::Value* addend_op, llvm::Value* result_op) {
-          RowMajorMatrixVectorProductEmitter emitter(config, lhs_op, rhs_op,
-                                                     addend_op, result_op, b_);
-          emitter.Emit();
-        });
+        /*optimize_for_size=*/optimize_for_size);
   }
 
   return true;
diff --git a/tensorflow/compiler/xla/service/cpu/tiled_dot_emitter.cc b/tensorflow/compiler/xla/service/cpu/tiled_dot_emitter.cc
new file mode 100644
index 0000000000..eb6c44b70a
--- /dev/null
+++ b/tensorflow/compiler/xla/service/cpu/tiled_dot_emitter.cc
@@ -0,0 +1,1014 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/cpu/tiled_dot_emitter.h"
+
+#include "tensorflow/compiler/xla/service/cpu/vector_support_library.h"
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/kernel_support_library.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h"
+
+namespace xla {
+namespace cpu {
+namespace {
+
+using tensorflow::int64;
+
+// Provides tiled access to an in-memory rank 2 array.
+class MemoryTile {
+ public:
+  // Constructs a MemoryTile that can operate on tiles consisting of
+  // `tile_size_along_major_dim` vectors from the matrix `matrix`, starting at
+  // `major_dim_offset` in the major dimension.  The tile size along the minor
+  // dimension is the vector size, and that is implicitly determined by `vsl`.
+  MemoryTile(VectorSupportLibrary* vsl, llvm::IRBuilder<>* b,
+             llvm::Value* matrix, int64 matrix_size_along_minor_dim,
+             llvm::Value* major_dim_offset, int64 tile_size_along_major_dim)
+      : vsl_(vsl), b_(b) {
+    pointers_.reserve(tile_size_along_major_dim);
+    for (int64 i = 0; i < tile_size_along_major_dim; i++) {
+      llvm::Value* total_offset =
+          b->CreateMul(b->getInt64(matrix_size_along_minor_dim),
+                       b->CreateAdd(b->getInt64(i), major_dim_offset));
+      pointers_.push_back(vsl_->ComputeOffsetPointer(matrix, total_offset));
+    }
+  }
+
+  // Load a tile consisting of `tile_size_along_major_dim` vectors from position
+  // {major: `major_dim_offset`, minor: `minor_dim_offset`}.
+  //
+  // Note: `major_dim_offset` is a parameter to the constructor.
+  std::vector<llvm::Value*> LoadTile(llvm::Value* minor_dim_offset) const {
+    std::vector<llvm::Value*> result;
+    result.reserve(pointers_.size());
+    for (const auto& pointer : pointers_) {
+      result.push_back(vsl_->LoadVector(pointer, minor_dim_offset));
+    }
+    return result;
+  }
+
+  // Stores `tile` to position {major: `major_dim_offset`, minor:
+  // `minor_dim_offset`}.
+  //
+  // Note: `major_dim_offset` is a parameter to the constructor.
+  void StoreTile(absl::Span<llvm::Value* const> tile,
+                 llvm::Value* minor_dim_offset) const {
+    CHECK_EQ(tile.size(), pointers_.size());
+    for (int64 i = 0; i < pointers_.size(); i++) {
+      vsl_->StoreVector(tile[i], pointers_[i], minor_dim_offset);
+    }
+  }
+
+  // Loads a tile of size [`tile_size_along_major_dim`,
+  // `tile_size_along_middle_dim`] from position {major: `major_dim_offset`,
+  // minor: `minor_dim_offset`} and then broadcasts each element into a vector
+  // of size vsl_.vector_size().  The (i,j)'th element of the return value is
+  // the (i,j)'th element in the tile broadcasted into an LLVM vector.
+  //
+  // Note: `major_dim_offset` is a parameter to the constructor.
+  std::vector<std::vector<llvm::Value*>> LoadBroadcastTile(
+      llvm::Value* minor_dim_offset, int64 tile_size_along_middle_dim) const {
+    std::vector<std::vector<llvm::Value*>> result;
+    result.resize(pointers_.size());
+    for (int64 i = 0; i < pointers_.size(); i++) {
+      for (int64 j = 0; j < tile_size_along_middle_dim; j++) {
+        result[i].push_back(vsl_->LoadBroadcast(
+            pointers_[i], b_->CreateAdd(minor_dim_offset, b_->getInt64(j))));
+      }
+    }
+    return result;
+  }
+
+ private:
+  VectorSupportLibrary* vsl_;
+  llvm::IRBuilder<>* b_;
+  std::vector<llvm::Value*> pointers_;
+};
+
+// The base class for the classes representing the GEMV emitter configurations.
+//
+// The IR emitted (modulo the LLVM values representing the input and output
+// buffers) by the row major and column major GEMV emitters should be a function
+// of their configuration.  This is important because their configuration is
+// used as a key to cache the generated IR.
+class GemvConfig {
+ public:
+  // Mixin for convenience.
+  template <typename T>
+  struct User {
+   public:
+    PrimitiveType scalar_type() const {
+      return derived().config().scalar_type();
+    }
+    int64 tile_rows() const { return derived().config().tile_rows(); }
+    int64 tile_cols() const { return derived().config().tile_cols(); }
+    int64 m() const { return derived().config().m(); }
+    int64 k() const { return derived().config().k(); }
+    int64 has_addend() const { return derived().config().has_addend(); }
+
+   private:
+    const T& derived() const { return *static_cast<const T*>(this); }
+  };
+
+  PrimitiveType scalar_type() const { return scalar_type_; }
+  int64 tile_rows() const { return tile_rows_; }
+  int64 tile_cols() const { return tile_cols_; }
+  int64 m() const { return m_; }
+  int64 k() const { return k_; }
+  bool has_addend() const { return has_addend_; }
+
+  string GetCacheKey() const {
+    return absl::StrCat(name_, "_", PrimitiveType_Name(scalar_type()), "_",
+                        tile_rows(), "_", tile_cols(), "_", m(), "_", k(),
+                        has_addend() ? "_with_addend" : "");
+  }
+
+ protected:
+  explicit GemvConfig(string name, PrimitiveType scalar_type, int64 tile_rows,
+                      int64 tile_cols, int64 m, int64 k, bool has_addend)
+      : name_(std::move(name)),
+        scalar_type_(scalar_type),
+        tile_rows_(tile_rows),
+        tile_cols_(tile_cols),
+        m_(m),
+        k_(k),
+        has_addend_(has_addend) {}
+
+ private:
+  string name_;
+  PrimitiveType scalar_type_;
+  int64 tile_rows_;
+  int64 tile_cols_;
+  int64 m_;
+  int64 k_;
+  bool has_addend_;
+};
+
+// Computes a dot product between "[M,K]{0,1} lhs" with a [K,1] vector (the
+// layout of the vector does not matter).  This implementation uses a tiling
+// scheme to improve performance.
+//
+// We logically separate the LHS matrix into four segments:
+//
+//   +----------------------+---+
+//   |                      |   |
+//   |                      |   |
+//   |         A            | B |
+//   |                      |   |
+//   |                      |   |
+//   |                      |   |
+//   +----------------------+---+
+//   |         C            | D |
+//   +----------------------+---+
+//
+// where A is the largest submatrix of the LHS that can be evenly dividied into
+// tiles.  For each tile in A, assuming tile_rows_ == tile_cols_ == 4, we have:
+//
+//   +---+---+---+---+       +--+--+--+--+
+//   |M00|M10|M20|M30|       |V0|V1|V2|V3|
+//   +---+---+---+---+       +--+--+--+--+
+//   |M01|M11|M21|M31| and   |V0|V1|V2|V3|
+//   +---+---+---+---+       +--+--+--+--+
+//   |M02|M12|M22|M32|       |V0|V1|V2|V3|
+//   +---+---+---+---+       +--+--+--+--+
+//   |M03|M13|M23|M33|       |V0|V1|V2|V3|
+//   +---+---+---+---+       +--+--+--+--+
+//
+// (Legend: rows are horizontal and columns are vertical; and each column is one
+// llvm::Value of a vector type)
+//
+// where:
+//
+//   a. The left tile is from the column major left matrix.
+//   b. The right tile is an elementwise broadcast of a [V0, V1, V2, V3]
+//      vector loaded from the RHS vector.
+//
+// As we iterate through the column dimension, we compute the change to the
+// result vector by an elementwise multiplication between the two tiles above
+// followed by a reduction along the major dimension:
+//
+//                     +-----------------------------------+
+//                     | M00*V0 + M10*V1 + M20*V2 + M30*V3 |
+//                     +-----------------------------------+
+//                     | M01*V0 + M11*V1 + M21*V2 + M31*V3 |
+// Result[R:R+4] +=    +-----------------------------------+
+//                     | M02*V0 + M12*V1 + M22*V2 + M32*V3 |
+//                     +-----------------------------------+
+//                     | M03*V0 + M13*V1 + M23*V2 + M33*V3 |
+//                     +-----------------------------------+
+//
+// Where R is the starting row for the tile.
+//
+// We have an inner epilogue loop to deal with the "C" submatrix and an outer
+// epilogue loop to deal with the B,D submarix.
+//
+// TODO(sanjoy): We should investigate if using gather loads and scatter stores
+// can be used here have the same inner loop for both column-major and row-major
+// matrix-vector products.
+class ColumnMajorMatrixVectorProductEmitter
+    : public GemvConfig::User<ColumnMajorMatrixVectorProductEmitter> {
+ public:
+  class Config : public GemvConfig {
+   public:
+    explicit Config(PrimitiveType scalar_type, int64 tile_rows, int64 tile_cols,
+                    int64 m, int64 k, bool has_addend)
+        : GemvConfig(/*name=*/"col_major_gemv", scalar_type,
+                     /*tile_rows=*/tile_rows, /*tile_cols=*/tile_cols, /*m=*/m,
+                     /*k=*/k, /*has_addend=*/has_addend) {}
+  };
+
+  ColumnMajorMatrixVectorProductEmitter(const Config& config, llvm::Value* lhs,
+                                        llvm::Value* rhs, llvm::Value* addend,
+                                        llvm::Value* result,
+                                        llvm::IRBuilder<>* b)
+      : config_(config),
+        lhs_(lhs),
+        rhs_(rhs),
+        addend_(addend),
+        result_(result),
+        b_(b),
+        ksl_(b_),
+        vsl_(config.scalar_type(), /*vector_size=*/config.tile_rows(), b_, "") {
+    CHECK(tile_rows() > 0 && IsPowerOfTwo(static_cast<uint64>(tile_rows())));
+    CHECK(!has_addend() || addend != nullptr);
+  }
+
+  void Emit();
+
+  const Config& config() const { return config_; }
+
+ private:
+  void EmitOuterLoopBody(llvm::Value* column, int64 column_count,
+                         bool is_first_column);
+
+  MemoryTile GetLhsMemoryTile(llvm::Value* column_start, int64 column_count) {
+    return MemoryTile(&vsl_, b_, /*matrix=*/lhs_,
+                      /*matrix_size_along_minor_dim=*/m(),
+                      /*major_dim_offset=*/column_start,
+                      /*tile_size_along_major_dim=*/column_count);
+  }
+
+  // Load a tile of values from the RHS.  For the RHS a "tile" is a contiguous
+  // sequence of `count` values, each one broadcasted to the vector width.
+  std::vector<llvm::Value*> LoadRhsTile(llvm::Value* offset, int64 count) {
+    llvm::Value* base_pointer = vsl_.ComputeOffsetPointer(rhs_, offset);
+    std::vector<llvm::Value*> result;
+    result.reserve(count);
+    for (int64 i = 0; i < count; i++) {
+      result.push_back(vsl_.LoadBroadcast(base_pointer, i));
+    }
+    return result;
+  }
+
+  void EmitInnerLoopTiled(MemoryTile* lhs_memory_tile,
+                          const std::vector<llvm::Value*>& rhs_tile,
+                          int64 columns, bool is_first_column);
+
+  void EmitInnerLoopEpilogue(llvm::Value* current_tile_col, int64 columns,
+                             bool is_first_tiled_column);
+
+  Config config_;
+  llvm::Value* lhs_;
+  llvm::Value* rhs_;
+  llvm::Value* addend_;
+  llvm::Value* result_;
+  llvm::IRBuilder<>* b_;
+  KernelSupportLibrary ksl_;
+  VectorSupportLibrary vsl_;
+};
+
+void ColumnMajorMatrixVectorProductEmitter::EmitOuterLoopBody(
+    llvm::Value* column, int64 column_count, bool is_first_column) {
+  MemoryTile lhs_memory_tile = GetLhsMemoryTile(/*column_start=*/column,
+                                                /*column_count=*/column_count);
+
+  std::vector<llvm::Value*> rhs_tile =
+      LoadRhsTile(column, /*count=*/column_count);
+  EmitInnerLoopTiled(&lhs_memory_tile, rhs_tile,
+                     /*columns=*/column_count, is_first_column);
+  EmitInnerLoopEpilogue(column, /*columns=*/column_count, is_first_column);
+}
+
+void ColumnMajorMatrixVectorProductEmitter::Emit() {
+  // See the comment on the class declaration for the algorithm used here.
+  int64 column_remainder = k() % tile_cols();
+  int64 column_limit = k() - column_remainder;
+
+  ksl_.For("dot.outer.tiled",
+           /*start=*/0, /*end=*/column_limit, /*step=*/tile_cols(),
+           [&](llvm::Value* column, bool is_first_column) {
+             EmitOuterLoopBody(column, tile_cols(), is_first_column);
+           });
+
+  if (column_remainder != 0) {
+    EmitOuterLoopBody(b_->getInt64(column_limit), column_remainder,
+                      column_limit == 0);
+  }
+}
+
+void ColumnMajorMatrixVectorProductEmitter::EmitInnerLoopTiled(
+    MemoryTile* lhs_memory_tile, const std::vector<llvm::Value*>& rhs_tile,
+    int64 columns, bool is_first_column) {
+  int64 row_limit = m() - (m() % tile_rows());
+
+  ksl_.For("dot.inner.tiled", /*start=*/0, /*end=*/row_limit,
+           /*step=*/tile_rows(), [&](llvm::Value* row) {
+             std::vector<llvm::Value*> lhs_tile =
+                 lhs_memory_tile->LoadTile(/*minor_dim_offset=*/row);
+             llvm::Value* accumulator =
+                 is_first_column ? (addend_ ? vsl_.LoadVector(addend_, row)
+                                            : vsl_.GetZeroVector())
+                                 : vsl_.LoadVector(result_, row);
+             for (int i = 0; i < columns; i++) {
+               accumulator = vsl_.MulAdd(lhs_tile[i], rhs_tile[i], accumulator);
+             }
+             vsl_.StoreVector(accumulator, result_, row);
+           });
+}
+
+void ColumnMajorMatrixVectorProductEmitter::EmitInnerLoopEpilogue(
+    llvm::Value* current_tile_col, int64 columns, bool is_first_tiled_column) {
+  int64 row_start = m() - (m() % tile_rows());
+  if (row_start == m()) {
+    return;
+  }
+
+  llvm::Value* columns_llvm = b_->getInt64(columns);
+
+  // for (col = current_tile_col; col < (columns + current_tile_col); col++)
+  //   for (row = row_start, row < m_; row++) {
+  //     result[row] += lhs[row, col] * rhs[col]
+  //     // Also take into account that if col is 0 then result[row] is not
+  //     // initialized.
+  //   }
+
+  ksl_.For(
+      "dot.inner.epilg.outer", /*start=*/current_tile_col,
+      /*end=*/b_->CreateAdd(columns_llvm, current_tile_col),
+      /*step=*/1, /*peel_first_iteration=*/false,
+      [&](llvm::Value* col, llvm::Value* is_first_scalar_col) {
+        llvm::Value* rhs_element = vsl_.LoadScalar(rhs_, col);
+        llvm::Value* total_offset = b_->CreateMul(col, b_->getInt64(m()));
+        llvm::Value* lhs_base_pointer =
+            vsl_.ComputeOffsetPointer(lhs_, total_offset);
+        ksl_.For(
+            "dot.inner.epilg.inner", /*start=*/row_start, /*end=*/m(),
+            /*step=*/1, [&](llvm::Value* scalar_row) {
+              llvm::Value* product = vsl_.Mul(
+                  vsl_.LoadScalar(lhs_base_pointer, scalar_row), rhs_element);
+              llvm::Value* setting_result_first_time = b_->CreateAnd(
+                  is_first_scalar_col, b_->getInt1(is_first_tiled_column));
+              ksl_.If(
+                  setting_result_first_time,
+                  /*true_block_generator=*/
+                  [&]() {
+                    if (addend_) {
+                      vsl_.StoreScalar(
+                          vsl_.Add(vsl_.LoadScalar(addend_, scalar_row),
+                                   product),
+                          result_, scalar_row);
+                    } else {
+                      vsl_.StoreScalar(product, result_, scalar_row);
+                    }
+                  },
+                  /*false_block_generator=*/
+                  [&]() {
+                    vsl_.StoreScalar(
+                        vsl_.Add(vsl_.LoadScalar(result_, scalar_row), product),
+                        result_, scalar_row);
+                  });
+            });
+      });
+}
+
+// Computes a dot product between "[M,K]{1,0} lhs" with a [K,1] vector (the
+// layout of the vector does not matter).  This implementation uses a tiling
+// scheme to improve performance.
+//
+// We logically separate the LHS matrix into four segments:
+//
+//   +----------------------+---+
+//   |                      |   |
+//   |                      |   |
+//   |         A            | B |
+//   |                      |   |
+//   |                      |   |
+//   |                      |   |
+//   +----------------------+---+
+//   |         C            | D |
+//   +----------------------+---+
+//
+// where A is the largest submatrix of the LHS that can be evenly dividied into
+// tiles.  For each tile in A, assuming tile_rows_ == tile_cols_ == 4, we have:
+//
+//   +---+---+---+---+
+//   |M00|M10|M20|M30|
+//   +---+---+---+---+       +--+--+--+--+
+//   |M01|M11|M21|M31| and   |V0|V1|V2|V3|
+//   +---+---+---+---+       +--+--+--+--+
+//   |M02|M12|M22|M32|
+//   +---+---+---+---+
+//   |M03|M13|M23|M33|
+//   +---+---+---+---+
+//
+// (Legend: rows are horizontal and columns are vertical; and each row is one
+// llvm::Value of a vector type)
+//
+// where:
+//
+//   a. The left tile is loaded from the row major left matrix.
+//   b. The right vector is loaded from the RHS vector.
+//
+// We keep 4 vector accumulators accumulating the following four vector
+// expressions as we iterate over the row dimension:
+//
+//   +------+------+------+------+
+//   |M0I*V0|M1I*V1|M2I*V2|M3I*V3|  for I in [0,4)
+//   +------+------+------+------+
+//
+// In the end we do a horizontal reduction over these 4 vector accumulators to
+// get 4 values in the result vector.
+//
+// We have an inner epilogue loop to deal with the "B" sub-matrix and an outer
+// epilogue loop to deal with the C,D submatrix.
+class RowMajorMatrixVectorProductEmitter
+    : public GemvConfig::User<RowMajorMatrixVectorProductEmitter> {
+ public:
+  class Config : public GemvConfig {
+   public:
+    explicit Config(PrimitiveType scalar_type, int64 tile_rows, int64 tile_cols,
+                    int64 m, int64 k, bool has_addend)
+        : GemvConfig(/*name=*/"row_major_gemv", scalar_type,
+                     /*tile_rows=*/tile_rows, /*tile_cols=*/tile_cols, /*m=*/m,
+                     /*k=*/k, /*has_addend=*/has_addend) {}
+  };
+
+  RowMajorMatrixVectorProductEmitter(const Config& config, llvm::Value* lhs,
+                                     llvm::Value* rhs, llvm::Value* addend,
+                                     llvm::Value* result, llvm::IRBuilder<>* b)
+      : config_(config),
+        lhs_(lhs),
+        rhs_(rhs),
+        addend_(addend),
+        result_(result),
+        b_(b),
+        ksl_(b_),
+        vsl_(scalar_type(), /*vector_size=*/tile_cols(), b_, "") {
+    CHECK(tile_cols() > 0 && IsPowerOfTwo(static_cast<uint64>(tile_cols())));
+    CHECK(!has_addend() || addend != nullptr);
+  }
+
+  void Emit();
+
+  const Config& config() const { return config_; }
+
+ private:
+  MemoryTile GetLhsMemoryTile(llvm::Value* row_start, int64 row_count) {
+    return MemoryTile(&vsl_, b_, /*matrix=*/lhs_,
+                      /*matrix_size_along_minor_dim=*/k(),
+                      /*major_dim_offset=*/row_start,
+                      /*tile_size_along_major_dim=*/row_count);
+  }
+
+  void EmitOuterLoopBody(llvm::Value* row, int64 row_count);
+
+  void EmitInnerLoopTiled(MemoryTile* lhs_memory_tile, int64 rows,
+                          std::vector<VectorVariable>* vector_accumulators);
+
+  void EmitInnerLoopEpilogue(llvm::Value* current_tile_row, int64 rows,
+                             std::vector<ScalarVariable>* scalar_accumulators);
+
+  Config config_;
+  llvm::Value* lhs_;
+  llvm::Value* rhs_;
+  llvm::Value* addend_;
+  llvm::Value* result_;
+  llvm::IRBuilder<>* b_;
+  KernelSupportLibrary ksl_;
+  VectorSupportLibrary vsl_;
+};
+
+void RowMajorMatrixVectorProductEmitter::EmitOuterLoopBody(llvm::Value* row,
+                                                           int64 row_count) {
+  MemoryTile lhs_memory_tile = GetLhsMemoryTile(/*row_start=*/row,
+                                                /*row_count=*/row_count);
+  std::vector<VectorVariable> vector_accumulators;
+  std::vector<ScalarVariable> scalar_accumulators;
+  for (int i = 0; i < row_count; i++) {
+    vector_accumulators.emplace_back(&vsl_, vsl_.GetZeroVector());
+    scalar_accumulators.emplace_back(&vsl_, vsl_.GetZeroScalar());
+  }
+  EmitInnerLoopTiled(&lhs_memory_tile, /*rows=*/row_count,
+                     &vector_accumulators);
+  EmitInnerLoopEpilogue(/*current_tile_row=*/row, /*rows=*/row_count,
+                        &scalar_accumulators);
+
+  std::vector<llvm::Value*> accumulator_values;
+  std::transform(
+      vector_accumulators.begin(), vector_accumulators.end(),
+      std::back_inserter(accumulator_values),
+      [](const VectorVariable& vector_var) { return vector_var.Get(); });
+
+  std::vector<llvm::Value*> horizontal_sums;
+  if (row_count == vsl_.vector_size()) {
+    if (addend_) {
+      horizontal_sums = vsl_.ComputeHorizontalSums(
+          std::move(accumulator_values), vsl_.LoadVector(addend_, row));
+    } else {
+      horizontal_sums =
+          vsl_.ComputeHorizontalSums(std::move(accumulator_values));
+    }
+  } else {
+    horizontal_sums = vsl_.ComputeHorizontalSums(std::move(accumulator_values));
+  }
+
+  for (int i = 0; i < row_count; i++) {
+    llvm::Value* result_value =
+        vsl_.Add(horizontal_sums[i], scalar_accumulators[i].Get());
+    llvm::Value* offset = b_->CreateAdd(b_->getInt64(i), row);
+    if (addend_ && row_count != vsl_.vector_size()) {
+      result_value = vsl_.Add(vsl_.LoadScalar(addend_, offset), result_value);
+    }
+    vsl_.StoreScalar(result_value, result_, offset);
+  }
+}
+
+void RowMajorMatrixVectorProductEmitter::Emit() {
+  // See the comment on the class declaration for the algorithm used here.
+  int64 row_remainder = m() % tile_rows();
+  int64 row_limit = m() - row_remainder;
+
+  ksl_.For("dot.outer.tiled",
+           /*start=*/0, /*end=*/row_limit, /*step=*/tile_rows(),
+           [&](llvm::Value* row) { EmitOuterLoopBody(row, tile_rows()); });
+
+  if (row_remainder != 0) {
+    EmitOuterLoopBody(b_->getInt64(row_limit), row_remainder);
+  }
+}
+
+void RowMajorMatrixVectorProductEmitter::EmitInnerLoopTiled(
+    MemoryTile* lhs_memory_tile, int64 rows,
+    std::vector<VectorVariable>* vector_accumulators) {
+  int64 column_limit = k() - (k() % tile_cols());
+
+  ksl_.For("dot.inner.tiled", /*start=*/0, /*end=*/column_limit,
+           /*step=*/tile_cols(), [&](llvm::Value* col) {
+             std::vector<llvm::Value*> lhs_tile =
+                 lhs_memory_tile->LoadTile(/*minor_dim_offset=*/col);
+             llvm::Value* rhs_value = vsl_.LoadVector(rhs_, col);
+             for (int i = 0; i < rows; i++) {
+               llvm::Value* old_sum = (*vector_accumulators)[i].Get();
+               (*vector_accumulators)[i].Set(
+                   vsl_.Add(old_sum, vsl_.Mul(rhs_value, lhs_tile[i])));
+             }
+           });
+}
+
+void RowMajorMatrixVectorProductEmitter::EmitInnerLoopEpilogue(
+    llvm::Value* current_tile_row, int64 rows,
+    std::vector<ScalarVariable>* scalar_accumulators) {
+  int64 column_start = k() - (k() % tile_cols());
+  if (column_start == k()) {
+    return;
+  }
+
+  for (int r = 0; r < rows; r++) {
+    llvm::Value* total_offset = b_->CreateMul(
+        b_->CreateAdd(b_->getInt64(r), current_tile_row), b_->getInt64(k()));
+    llvm::Value* lhs_base_pointer =
+        vsl_.ComputeOffsetPointer(lhs_, total_offset);
+    ksl_.For("dot.inner.epilg.inner", /*start=*/column_start, /*end=*/k(),
+             /*step=*/1, [&](llvm::Value* scalar_col) {
+               llvm::Value* product =
+                   vsl_.Mul(vsl_.LoadScalar(lhs_base_pointer, scalar_col),
+                            vsl_.LoadScalar(rhs_, scalar_col));
+               llvm::Value* old_value = (*scalar_accumulators)[r].Get();
+               (*scalar_accumulators)[r].Set(vsl_.Add(old_value, product));
+             });
+  }
+}
+
+// This class implements a tiled matrix multiplication algorithm, intended for
+// multiplying small matrices that don't need cache tiling.
+//
+// In the future this can be used as the innermost GEBP loop in a GEMM kernel as
+// described in "Goto, Kazushige, and Robert A. Geijn. "Anatomy of
+// high-performance matrix multiplication." ACM Transactions on Mathematical
+// Software (TOMS) 34.3 (2008): 12.".
+//
+// This only supports canonical dot operations (i.e. where the lhs contraction
+// dimension is 1 and the rhs contraction dimension is 0) over row major
+// matrices.
+class TiledSmallGemmEmitter {
+ public:
+  // Describe the dimensions of the kernel.
+  class Dimensions {
+   public:
+    explicit Dimensions(int64 m, int64 k, int64 n) : m_(m), k_(k), n_(n) {}
+
+    int64 m() const { return m_; }
+    int64 k() const { return k_; }
+    int64 n() const { return n_; }
+
+    string ToString() const { return absl::StrCat(m(), "x", k(), "x", n()); }
+
+   private:
+    const int64 m_;
+    const int64 k_;
+    const int64 n_;
+  };
+
+  // Represents the configuration of the emitter.  The LLVM IR emitted by the
+  // emitter, modulo the LLVM values holding the input and output buffers, must
+  // be a function of the instance of `Config` passed to it.
+  //
+  // `dims` holds the matrix multiplication dimensions.
+  //
+  // `max_vectorization_width` is the maximum vector width (i.e. the width of
+  // the largest vector register we will use).  This can be larger than the
+  // largest vector register supported by the machine -- LLVM will legalize
+  // these large vector widths into legally sized vectors.
+  //
+  // `max_vector_count` is the maximum number of vectors of size
+  // `max_vectorization_width` that we will attempt to process at once.
+  //
+  // `min_vectorization_width` is the smallest vector width the emitter will use
+  // -- below that it will devolve to using a scalar loop.
+  //
+  // The innermost reduction loop executes the matrix multiply in tiles of size
+  // [`tile_size_m`, `tile_size_k`] from the LHS and [`tile_size_k`,
+  // <vectorization width>] in the RHS.
+  class Config {
+   public:
+    explicit Config(PrimitiveType scalar_type, Dimensions dims,
+                    int64 max_vectorization_width, int64 max_vector_count,
+                    int64 min_vectorization_width, int64 tile_size_m,
+                    int64 tile_size_k)
+        : scalar_type_(scalar_type),
+          dims_(dims),
+          max_vectorization_width_(max_vectorization_width),
+          max_vector_count_(max_vector_count),
+          min_vectorization_width_(min_vectorization_width),
+          tile_size_m_(tile_size_m),
+          tile_size_k_(tile_size_k) {}
+
+    string GetCacheKey() const {
+      return absl::StrCat("gemm_", PrimitiveType_Name(scalar_type()), "_",
+                          dims().ToString(), "_", max_vectorization_width(),
+                          "_", min_vectorization_width(), "_", tile_size_m(),
+                          "_", tile_size_k());
+    }
+
+    PrimitiveType scalar_type() const { return scalar_type_; }
+    Dimensions dims() const { return dims_; }
+    int64 max_vectorization_width() const { return max_vectorization_width_; }
+    int64 max_vector_count() const { return max_vector_count_; }
+    int64 min_vectorization_width() const { return min_vectorization_width_; }
+
+    int64 tile_size_m() const { return tile_size_m_; }
+    int64 tile_size_k() const { return tile_size_k_; }
+
+   private:
+    PrimitiveType scalar_type_;
+    Dimensions dims_;
+    int64 max_vectorization_width_;
+    int64 max_vector_count_;
+    int64 min_vectorization_width_;
+    int64 tile_size_m_;
+    int64 tile_size_k_;
+  };
+
+  // Creates an instance of TiledSmallGemmEmitter that matrix-multiplies
+  // `lhs` with `rhs` and stores the result in `result`.
+  explicit TiledSmallGemmEmitter(Config config, llvm::Value* lhs,
+                                 llvm::Value* rhs, llvm::Value* result,
+                                 llvm::IRBuilder<>* b)
+      : lhs_(lhs),
+        rhs_(rhs),
+        result_(result),
+        config_(config),
+        b_(b),
+        ksl_(b_) {
+    CHECK(max_vectorization_width() > 0 &&
+          IsPowerOfTwo(static_cast<uint64>(max_vectorization_width())));
+    CHECK_GT(max_vector_count(), 0);
+    CHECK(min_vectorization_width() > 0 &&
+          IsPowerOfTwo(static_cast<uint64>(min_vectorization_width())));
+    CHECK_GE(max_vectorization_width(), min_vectorization_width());
+    CHECK_GT(tile_size_k(), 0);
+  }
+
+  void Emit();
+
+ private:
+  // The HandleResiduesOnX helpers split the iteration space for dimension X
+  // into a multiple of the tile size on dimension X and an epilogue.  These
+  // helpers ultimately call into `EmitTiledGemm` for emitting the
+  // tiled GEMM kernel.
+
+  void HandleResiduesOnN();
+  void HandleResiduesOnK(VectorSupportLibrary* vsl, llvm::Value* n_start,
+                         llvm::Value* n_end);
+  void HandleResiduesOnM(VectorSupportLibrary* vsl, int64 tile_size_k,
+                         llvm::Value* k_start, llvm::Value* k_end,
+                         llvm::Value* n_start, llvm::Value* n_end);
+
+  // This emits a tiled GEMM kernel.  For a detailed description see the comment
+  // on the implementation.
+  void EmitTiledGemm(VectorSupportLibrary* vsl, int64 tile_size_k,
+                     llvm::Value* k_start, llvm::Value* k_end,
+                     llvm::Value* n_start, llvm::Value* n_end,
+                     int64 tile_size_m, llvm::Value* m_start,
+                     llvm::Value* m_end);
+
+  llvm::Value* GetInt64(int64 value) { return b_->getInt64(value); }
+
+  Config config() const { return config_; }
+  Dimensions dims() const { return config().dims(); }
+
+  int64 max_vectorization_width() const {
+    return config().max_vectorization_width();
+  }
+  int64 max_vector_count() const { return config().max_vector_count(); }
+  int64 min_vectorization_width() const {
+    return config().min_vectorization_width();
+  }
+  int64 tile_size_m() const { return config().tile_size_m(); }
+  int64 tile_size_k() const { return config().tile_size_k(); }
+  PrimitiveType scalar_type() const { return config().scalar_type(); }
+
+  llvm::Value* lhs_;
+  llvm::Value* rhs_;
+  llvm::Value* result_;
+  Config config_;
+
+  llvm::IRBuilder<>* b_;
+  KernelSupportLibrary ksl_;
+};
+
+void TiledSmallGemmEmitter::Emit() { HandleResiduesOnN(); }
+
+void TiledSmallGemmEmitter::HandleResiduesOnN() {
+  // We can only iterate the `n` dimension for an extent that is divisible by
+  // the vectorization width.  So we emit an outer loop that first processes the
+  // largest extent in `n` that is divisible by max_vectorization_width, then
+  // the largest remaining extent that is divisible by max_vectorization_width /
+  // 2 etc.
+
+  int64 current_vectorization_width =
+      max_vector_count() * max_vectorization_width();
+  int64 current_vector_count = max_vector_count();
+
+  int64 n_start = 0;
+  while (n_start != dims().n() &&
+         current_vectorization_width >= min_vectorization_width()) {
+    int64 n_end = dims().n() - (dims().n() % current_vectorization_width);
+    if (n_start != n_end) {
+      VectorSupportLibrary vsl(scalar_type(), current_vectorization_width, b_,
+                               "gemm");
+      HandleResiduesOnK(&vsl, GetInt64(n_start), GetInt64(n_end));
+      n_start = n_end;
+    }
+    if (current_vector_count == 1) {
+      current_vectorization_width /= 2;
+    } else {
+      current_vector_count--;
+      current_vectorization_width =
+          current_vector_count * max_vectorization_width();
+    }
+  }
+
+  if (n_start != dims().n()) {
+    VectorSupportLibrary vsl(scalar_type(), 1, b_, "gemm");
+    ksl_.For("epi.n", n_start, dims().n(), 1, [&](llvm::Value* n_i) {
+      llvm::Value* n_i_next = b_->CreateAdd(n_i, b_->getInt64(1));
+      HandleResiduesOnK(&vsl, n_i, n_i_next);
+    });
+  }
+}
+
+void TiledSmallGemmEmitter::HandleResiduesOnK(VectorSupportLibrary* vsl,
+                                              llvm::Value* n_start,
+                                              llvm::Value* n_end) {
+  int64 k_start = 0;
+  int64 k_end = dims().k() - (dims().k() % tile_size_k());
+  if (k_end != k_start) {
+    HandleResiduesOnM(vsl, tile_size_k(), GetInt64(k_start), GetInt64(k_end),
+                      n_start, n_end);
+    k_start = k_end;
+  }
+
+  if (k_start != dims().k()) {
+    HandleResiduesOnM(vsl, dims().k() - k_start, GetInt64(k_start),
+                      GetInt64(dims().k()), n_start, n_end);
+  }
+}
+
+void TiledSmallGemmEmitter::HandleResiduesOnM(
+    VectorSupportLibrary* vsl, int64 tile_size_k, llvm::Value* k_start,
+    llvm::Value* k_end, llvm::Value* n_start, llvm::Value* n_end) {
+  const int64 m_end = dims().m() - dims().m() % tile_size_m();
+  EmitTiledGemm(vsl, tile_size_k, k_start, k_end, n_start, n_end, tile_size_m(),
+                GetInt64(0), GetInt64(m_end));
+
+  if (m_end != dims().m()) {
+    EmitTiledGemm(vsl, tile_size_k, k_start, k_end, n_start, n_end,
+                  dims().m() - m_end, GetInt64(m_end), GetInt64(dims().m()));
+  }
+}
+
+// The loop structure is:
+//
+// Iterate over dimension M as m:
+//   Iterate over dimension N as n:
+//     Iterate over dimension K as k:
+//       OutputTile[m,n] += Dot(LhsTile[m,k], RhsTile[k,n])
+//
+// I.e. a just a tiled version of a "naive" GEMM.
+//
+// The tiling scheme is as follows:
+//
+// Let the LHS be:
+//
+//   +----+----+----+
+//   | a0 | b0 | c0 | .
+//   +----+----+----+ .
+//   | a1 | b1 | c1 | .
+//   +----+----+----+
+//     ..     ..
+//
+// and the RHS be:
+//
+//   +----+----+----+----+
+//   | p0 | p1 | p2 | p3 | .
+//   +----+----+----+----+ .
+//   | q0 | q1 | q2 | q3 | .
+//   +----+----+----+----+
+//   | r0 | r1 | r2 | r3 | .
+//   +----+----+----+----+ .
+//     ......    ......
+//
+// and let tile_size_m=2, tile_size_k=3 and the vector width (implicitly denoted
+// by `vsl`) be 4.  Then we want to matrix multiply this tile to get a [2,4]
+// matrix that we can increment the result matrix by.
+//
+// First broadcast the rows row in LHS to 3 vectors of width 4, giving us a rank
+// 3 array, L, of dimension [2,3,4]:
+//
+//       L[0,_,_]           *      L[1,_,_]
+//                          *
+//   +----+----+----+----+  *  +----+----+----+----+
+//   | a0 | a0 | a0 | a0 |  *  | a1 | a1 | a1 | a1 |
+//   +----+----+----+----+  *  +----+----+----+----+
+//   | b0 | b0 | b0 | b0 |  *  | b1 | b1 | b1 | b1 |
+//   +----+----+----+----+  *  +----+----+----+----+
+//   | c0 | c0 | c0 | c0 |  *  | c1 | c1 | c1 | c1 |
+//   +----+----+----+----+  *  +----+----+----+----+
+//
+//
+// Then we FMA L[0,_,_] with the RHS to get the first row of the result and
+// L[1,_,_] with the RHS to get the second row of the result.  For example,
+// L[0,_,_] is computed as:
+//
+//   +----+----+----+----+   +----+----+----+----+
+//   | a0 | a0 | a0 | a0 | * | p0 | p1 | p2 | p3 |   +
+//   +----+----+----+----+   +----+----+----+----+
+//
+//   +----+----+----+----+   +----+----+----+----+
+//   | b0 | b0 | b0 | b0 | * | q0 | q1 | q2 | q3 |   +
+//   +----+----+----+----+   +----+----+----+----+
+//
+//   +----+----+----+----+   +----+----+----+----+
+//   | c0 | c0 | c0 | c0 | * | r0 | r1 | r2 | r3 |
+//   +----+----+----+----+   +----+----+----+----+
+//
+// to get:
+//
+//   +-------------------+-------------------+-------------------+---------
+//   | a0*p0+b0*q0+c0*r0 | a0*p1+b0*q1+c0*r1 | a0*p2+b0*q2+c0*r2 |  ...
+//   +-------------------+-------------------+-------------------+---------
+void TiledSmallGemmEmitter::EmitTiledGemm(
+    VectorSupportLibrary* vsl, int64 tile_size_k, llvm::Value* k_start,
+    llvm::Value* k_end, llvm::Value* n_start, llvm::Value* n_end,
+    int64 tile_size_m, llvm::Value* m_start, llvm::Value* m_end) {
+  ksl_.For("dot.m", m_start, m_end, tile_size_m, [&](llvm::Value* m_i) {
+    MemoryTile result_memory_tile(vsl, b_, /*matrix=*/result_,
+                                  /*matrix_size_along_minor_dim=*/dims().n(),
+                                  /*major_dim_offset=*/m_i,
+                                  /*tile_size_along_major_dim=*/tile_size_m);
+    MemoryTile lhs_memory_tile(vsl, b_, /*matrix=*/lhs_,
+                               /*matrix_size_along_minor_dim=*/dims().k(),
+                               /*major_dim_offset=*/m_i,
+                               /*tile_size_along_major_dim=*/tile_size_m);
+    ksl_.For(
+        "dot.n", n_start, n_end, vsl->vector_size(), [&](llvm::Value* n_i) {
+          TileVariable result_tile_var(vsl, result_memory_tile.LoadTile(n_i));
+          ksl_.For("dot.k", k_start, k_end, tile_size_k, [&](llvm::Value* k_i) {
+            MemoryTile rhs_memory_tile(vsl, b_, rhs_, dims().n(), k_i,
+                                       tile_size_k);
+            std::vector<std::vector<llvm::Value*>> lhs_tile =
+                lhs_memory_tile.LoadBroadcastTile(k_i, tile_size_k);
+            std::vector<llvm::Value*> rhs_tile = rhs_memory_tile.LoadTile(n_i);
+            std::vector<llvm::Value*> result_tile = result_tile_var.Get();
+            for (int64 r_m_i = 0; r_m_i < tile_size_m; r_m_i++) {
+              for (int64 r_k_i = 0; r_k_i < tile_size_k; r_k_i++) {
+                result_tile[r_m_i] =
+                    vsl->MulAdd(lhs_tile[r_m_i][r_k_i], rhs_tile[r_k_i],
+                                result_tile[r_m_i]);
+              }
+            }
+            result_tile_var.Set(result_tile);
+          });
+
+          result_memory_tile.StoreTile(result_tile_var.Get(), n_i);
+        });
+  });
+}
+
+}  // namespace
+
+void EmitRowMajorGemv(PrimitiveType scalar_type, int64 tile_rows,
+                      int64 tile_cols, int64 m, int64 k, llvm::Value* lhs,
+                      llvm::Value* rhs, llvm::Value* addend,
+                      llvm::Value* result, llvm::IRBuilder<>* b,
+                      bool enable_fast_math, bool optimize_for_size) {
+  RowMajorMatrixVectorProductEmitter::Config config(
+      /*scalar_type=*/scalar_type,
+      /*tile_rows=*/tile_rows, /*tile_cols=*/tile_cols,
+      /*m=*/m, /*k=*/k, /*has_addend=*/addend != nullptr);
+
+  KernelSupportLibrary::EmitAndCallOutlinedKernel(
+      /*enable_fast_math=*/enable_fast_math,
+      /*optimize_for_size=*/optimize_for_size, b, config.GetCacheKey(), lhs,
+      rhs, addend, result,
+      [&](llvm::Value* lhs, llvm::Value* rhs, llvm::Value* addend,
+          llvm::Value* result) {
+        RowMajorMatrixVectorProductEmitter emitter(config, lhs, rhs, addend,
+                                                   result, b);
+        emitter.Emit();
+      });
+}
+
+void EmitColumnMajorGemv(PrimitiveType scalar_type, int64 tile_rows,
+                         int64 tile_cols, int64 m, int64 k, llvm::Value* lhs,
+                         llvm::Value* rhs, llvm::Value* addend,
+                         llvm::Value* result, llvm::IRBuilder<>* b,
+                         bool enable_fast_math, bool optimize_for_size) {
+  ColumnMajorMatrixVectorProductEmitter::Config config(
+      /*scalar_type=*/scalar_type,
+      /*tile_rows=*/tile_rows, /*tile_cols=*/tile_cols,
+      /*m=*/m, /*k=*/k, /*has_addend=*/addend != nullptr);
+
+  KernelSupportLibrary::EmitAndCallOutlinedKernel(
+      /*enable_fast_math=*/enable_fast_math,
+      /*optimize_for_size=*/optimize_for_size, b, config.GetCacheKey(), lhs,
+      rhs, addend, result,
+      [&](llvm::Value* lhs, llvm::Value* rhs, llvm::Value* addend,
+          llvm::Value* result) {
+        ColumnMajorMatrixVectorProductEmitter emitter(config, lhs, rhs, addend,
+                                                      result, b);
+        emitter.Emit();
+      });
+}
+
+void EmitSmallGemm(PrimitiveType scalar_type, int64 m, int64 k, int64 n,
+                   int64 max_vectorization_width, int64 max_vector_count,
+                   int64 min_vectorization_width, int64 tile_size_m,
+                   int64 tile_size_k, llvm::Value* lhs, llvm::Value* rhs,
+                   llvm::Value* result, llvm::IRBuilder<>* b,
+                   bool enable_fast_math, bool optimize_for_size) {
+  TiledSmallGemmEmitter::Config config(
+      /*scalar_type=*/scalar_type,
+      TiledSmallGemmEmitter::Dimensions{/*m=*/m, /*k=*/k, /*n=*/n},
+      /*max_vectorization_width=*/max_vectorization_width,
+      /*max_vector_count=*/max_vector_count,
+      /*min_vectorization_width=*/min_vectorization_width,
+      /*tile_size_m=*/tile_size_m, /*tile_size_k=*/tile_size_k);
+
+  KernelSupportLibrary::EmitAndCallOutlinedKernel(
+      /*enable_fast_math=*/enable_fast_math,
+      /*optimize_for_size=*/optimize_for_size, b, config.GetCacheKey(), lhs,
+      rhs, result,
+      [&](llvm::Value* lhs, llvm::Value* rhs, llvm::Value* result) {
+        TiledSmallGemmEmitter small_gemm_emitter(config, /*lhs=*/lhs,
+                                                 /*rhs=*/rhs,
+                                                 /*result=*/result, b);
+        small_gemm_emitter.Emit();
+      });
+}
+
+}  // namespace cpu
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/cpu/tiled_dot_emitter.h b/tensorflow/compiler/xla/service/cpu/tiled_dot_emitter.h
new file mode 100644
index 0000000000..0a82326cc3
--- /dev/null
+++ b/tensorflow/compiler/xla/service/cpu/tiled_dot_emitter.h
@@ -0,0 +1,55 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_CPU_TILED_DOT_EMITTER_H_
+#define TENSORFLOW_COMPILER_XLA_SERVICE_CPU_TILED_DOT_EMITTER_H_
+
+#include "llvm/IR/IRBuilder.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace xla {
+namespace cpu {
+
+// These routines emit LLVM IR implementing tiled GEMM and GEMV routines.
+
+void EmitRowMajorGemv(PrimitiveType scalar_type, tensorflow::int64 tile_rows,
+                      tensorflow::int64 tile_cols, tensorflow::int64 m,
+                      tensorflow::int64 k, llvm::Value* lhs, llvm::Value* rhs,
+                      llvm::Value* addend, llvm::Value* result,
+                      llvm::IRBuilder<>* b, bool enable_fast_math,
+                      bool optimize_for_size);
+
+void EmitColumnMajorGemv(PrimitiveType scalar_type, tensorflow::int64 tile_rows,
+                         tensorflow::int64 tile_cols, tensorflow::int64 m,
+                         tensorflow::int64 k, llvm::Value* lhs,
+                         llvm::Value* rhs, llvm::Value* addend,
+                         llvm::Value* result, llvm::IRBuilder<>* b,
+                         bool enable_fast_math, bool optimize_for_size);
+
+void EmitSmallGemm(PrimitiveType scalar_type, tensorflow::int64 m,
+                   tensorflow::int64 k, tensorflow::int64 n,
+                   tensorflow::int64 max_vectorization_width,
+                   tensorflow::int64 max_vector_count,
+                   tensorflow::int64 min_vectorization_width,
+                   tensorflow::int64 tile_size_m, tensorflow::int64 tile_size_k,
+                   llvm::Value* lhs, llvm::Value* rhs, llvm::Value* result,
+                   llvm::IRBuilder<>* b, bool enable_fast_math,
+                   bool optimize_for_size);
+
+}  // namespace cpu
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_CPU_TILED_DOT_EMITTER_H_
-- 
GitLab


From 0ebce89605326c602b3dd4d8cfe3f3a29284b390 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 14:39:51 -0800
Subject: [PATCH 0290/2345] Optimize test combinations to prevent timeout
 during testing.

PiperOrigin-RevId: 228238454
---
 tensorflow/contrib/distribute/python/BUILD | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index de6b6f1f84..7ee12812af 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -674,7 +674,9 @@ cuda_py_test(
     additional_deps = [
         ":keras_correctness_test_lib",
     ],
-    shard_count = 16,
+    # Shard count is set to an odd number to distribute tasks across
+    # shards more evenly.
+    shard_count = 19,
     tags = [
         "multi_and_single_gpu",
         "no_oss",  # TODO(b/117919883): Fix python error.
-- 
GitLab


From 67a7f04d010c50fb3d6d1bde7793d3f8d350decf Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Mon, 7 Jan 2019 14:43:32 -0800
Subject: [PATCH 0291/2345] Graph partitioning: use a sensible hash function
 for device memory info.

PiperOrigin-RevId: 228239154
---
 tensorflow/core/BUILD                    |  2 ++
 tensorflow/core/graph/graph_partition.cc | 44 +++++++++---------------
 2 files changed, 18 insertions(+), 28 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 5e6c8d4dd8..e6af9211b5 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1750,6 +1750,7 @@ cc_library(
 cc_library(
     name = "mobile_additional_lib_deps",
     deps = tf_additional_lib_deps() + [
+        "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/strings",
     ],
@@ -2817,6 +2818,7 @@ tf_cuda_library(
         ":proto_text",
         ":protos_all_cc",
         "//third_party/eigen3",
+        "@com_google_absl//absl/container:flat_hash_map",
     ],
 )
 
diff --git a/tensorflow/core/graph/graph_partition.cc b/tensorflow/core/graph/graph_partition.cc
index f213eb7c10..be0cac50a1 100644
--- a/tensorflow/core/graph/graph_partition.cc
+++ b/tensorflow/core/graph/graph_partition.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include <utility>
 #include <vector>
 
+#include "absl/container/flat_hash_map.h"
 #include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/framework/memory_types.h"
 #include "tensorflow/core/framework/node_def_builder.h"
@@ -58,29 +59,24 @@ struct DupRecvKey {
   int src_output_slot;       // Edge's src node output slot
   GraphDef* dst_graph;       // Edge's dst node is in this subgraph
   bool recv_output_on_host;  // The output of recv is on host
-};
 
-struct DupRecvKeyHash {
-  size_t operator()(const DupRecvKey& k) const {
-    size_t h = Hash64(reinterpret_cast<const char*>(&k.src_node_id),
-                      sizeof(k.src_node_id), k.src_output_slot);
-    h = Hash64(reinterpret_cast<const char*>(&k.dst_graph), sizeof(k.dst_graph),
-               h);
-    h = Hash64(reinterpret_cast<const char*>(&k.recv_output_on_host),
-               sizeof(k.recv_output_on_host), h);
-    return h;
+  template <typename H>
+  friend H AbslHashValue(H h, const DupRecvKey& c) {
+    return H::combine(std::move(h), c.src_node_id, c.src_output_slot,
+                      reinterpret_cast<std::uintptr_t>(c.dst_graph),
+                      c.recv_output_on_host);
   }
-};
 
-struct DupRecvKeyEq {
-  bool operator()(const DupRecvKey& x, const DupRecvKey& y) const {
-    return (x.src_node_id == y.src_node_id) &&
-           (x.src_output_slot == y.src_output_slot) &&
-           (x.dst_graph == y.dst_graph) &&
-           (x.recv_output_on_host == y.recv_output_on_host);
-  }
+  friend bool operator==(const DupRecvKey& x, const DupRecvKey& y);
 };
 
+bool operator==(const DupRecvKey& x, const DupRecvKey& y) {
+  return (x.src_node_id == y.src_node_id) &&
+         (x.src_output_slot == y.src_output_slot) &&
+         (x.dst_graph == y.dst_graph) &&
+         (x.recv_output_on_host == y.recv_output_on_host);
+}
+
 // struct used to store the recvs, so that start times can be properly updated
 struct RecvInfo {
   NodeDef* recv;
@@ -88,19 +84,11 @@ struct RecvInfo {
   int64 start_time;
 };
 
-typedef std::unordered_map<DupRecvKey, RecvInfo, DupRecvKeyHash, DupRecvKeyEq>
-    DupRecvTable;
+typedef absl::flat_hash_map<DupRecvKey, RecvInfo> DupRecvTable;
 
-struct PairIntHash {
- public:
-  std::size_t operator()(const std::pair<int, int>& x) const {
-    return std::hash<int>()(x.first) ^ std::hash<int>()(x.second);
-  }
-};
 // A map used to store memory types for the inputs/outputs of every node.
 // The key is a pair of ints consisting of a node id and input/output index.
-typedef std::unordered_map<std::pair<int, int>, MemoryType, PairIntHash>
-    MemoryTypeMap;
+typedef absl::flat_hash_map<std::pair<int, int>, MemoryType> MemoryTypeMap;
 
 // We collect the following information about the graph before performing
 // graph partitioning.
-- 
GitLab


From 106b3c183546d2f4bde8c209ef9d01ba0fb2904b Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Mon, 7 Jan 2019 14:55:46 -0800
Subject: [PATCH 0292/2345] [tf.data] Implementation of a "standalone" C++ API
 that encapsulates TensorFlow runtime internals. The new API facilitates
 efficient execution of a dataset input pipeline (represented as a GraphDef)
 in C++.

PiperOrigin-RevId: 228241484
---
 tensorflow/core/common_runtime/data/BUILD     |  35 ++++
 .../core/common_runtime/data/standalone.cc    | 128 ++++++++++++
 .../core/common_runtime/data/standalone.h     | 122 ++++++++++++
 .../common_runtime/data/standalone_test.cc    | 188 ++++++++++++++++++
 4 files changed, 473 insertions(+)
 create mode 100644 tensorflow/core/common_runtime/data/BUILD
 create mode 100644 tensorflow/core/common_runtime/data/standalone.cc
 create mode 100644 tensorflow/core/common_runtime/data/standalone.h
 create mode 100644 tensorflow/core/common_runtime/data/standalone_test.cc

diff --git a/tensorflow/core/common_runtime/data/BUILD b/tensorflow/core/common_runtime/data/BUILD
new file mode 100644
index 0000000000..124862dbb7
--- /dev/null
+++ b/tensorflow/core/common_runtime/data/BUILD
@@ -0,0 +1,35 @@
+package(
+    default_visibility = [
+        "//tensorflow:internal",
+        "//tensorflow_models:__subpackages__",
+    ],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+load("//tensorflow:tensorflow.bzl", "tf_cc_test")
+load("//tensorflow/core:platform/default/build_config.bzl", "tf_protos_all")
+
+cc_library(
+    name = "standalone",
+    srcs = ["standalone.cc"],
+    hdrs = ["standalone.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tensorflow/core:core_cpu_internal",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:session_options",
+    ],
+)
+
+tf_cc_test(
+    name = "standalone_test",
+    srcs = ["standalone_test.cc"],
+    deps = [
+        ":standalone",
+        "//tensorflow/core:all_kernels",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ] + tf_protos_all(),
+)
diff --git a/tensorflow/core/common_runtime/data/standalone.cc b/tensorflow/core/common_runtime/data/standalone.cc
new file mode 100644
index 0000000000..b05bff566f
--- /dev/null
+++ b/tensorflow/core/common_runtime/data/standalone.cc
@@ -0,0 +1,128 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/common_runtime/data/standalone.h"
+
+#include <memory>
+
+#include "tensorflow/core/common_runtime/device_factory.h"
+#include "tensorflow/core/common_runtime/device_mgr.h"
+#include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/common_runtime/graph_runner.h"
+#include "tensorflow/core/common_runtime/process_util.h"
+#include "tensorflow/core/framework/dataset.h"
+#include "tensorflow/core/graph/graph.h"
+#include "tensorflow/core/graph/graph_constructor.h"
+#include "tensorflow/core/public/version.h"
+#include "tensorflow/core/util/ptr_util.h"
+
+namespace tensorflow {
+namespace data {
+namespace standalone {
+
+Status Iterator::GetNext(std::vector<Tensor>* outputs, bool* end_of_input) {
+  return iterator_->GetNext(ctx_.get(), outputs, end_of_input);
+}
+
+Iterator::Iterator(IteratorBase* iterator, IteratorContext* ctx)
+    : iterator_(iterator), ctx_(ctx) {}
+
+Status Dataset::FromGraph(Params params, const GraphDef& graph_def,
+                          const string& fetch_node,
+                          std::unique_ptr<Dataset>* result) {
+  Graph graph(OpRegistry::Global());
+  TF_RETURN_IF_ERROR(ImportGraphDef({}, graph_def, &graph, nullptr));
+
+  // Instantiate enough of the TensorFlow runtime to run `graph` on a single CPU
+  // device.
+  std::unique_ptr<DeviceMgr> device_mgr = MakeUnique<DeviceMgr>(
+      DeviceFactory::NewDevice("CPU", params.session_options, ""));
+  Device* device = device_mgr->ListDevices()[0];
+  // Clone the `FunctionLibraryDefinition` to extend its lifetime extends beyond
+  // the lifetime of `graph`.
+  std::unique_ptr<FunctionLibraryDefinition> flib_def =
+      MakeUnique<FunctionLibraryDefinition>(graph.flib_def());
+  std::unique_ptr<ProcessFunctionLibraryRuntime> pflr =
+      MakeUnique<ProcessFunctionLibraryRuntime>(
+          device_mgr.get(), Env::Default(), TF_GRAPH_DEF_VERSION,
+          flib_def.get(), OptimizerOptions{}, nullptr /* parent */);
+
+  // Run graph up to `output_node` and extract the `DatasetBase` stored in the
+  // DT_VARIANT output tensor.
+  data::DatasetBase* dataset;
+  {
+    std::vector<Tensor> outputs;
+    GraphRunner graph_runner(device);
+    TF_RETURN_IF_ERROR(graph_runner.Run(&graph, pflr->GetFLR("/device:CPU:0"),
+                                        {}, {fetch_node}, &outputs));
+    TF_RETURN_IF_ERROR(GetDatasetFromVariantTensor(outputs[0], &dataset));
+    // NOTE(mrry): The dataset is currently owned by `outputs[0]`, so acquire an
+    // additional reference.
+    dataset->Ref();
+  }
+
+  std::unique_ptr<thread::ThreadPool> pool(
+      NewThreadPoolFromSessionOptions(params.session_options));
+  *result =
+      WrapUnique(new Dataset(dataset, device_mgr.release(), pflr.release(),
+                             flib_def.release(), pool.release()));
+  return Status::OK();
+}  // static
+
+Status Dataset::MakeIterator(std::unique_ptr<Iterator>* result) {
+  // Create an `IteratorContext`, which bundles together the necessary runtime
+  // support to create and get elements from an iterator.
+  std::unique_ptr<IteratorContext> ctx;
+  {
+    // NOTE(mrry): In the current API, an `IteratorContext` is always initially
+    // created from an `OpKernelContext*`, so we need to create a fake
+    // `OpKernelContext` with the appropriate subset of parameters.
+    OpKernelContext::Params op_params;
+    op_params.function_library = pflr_->GetFLR("/device:CPU:0");
+    op_params.device = device_mgr_->ListDevices()[0];
+    op_params.runner = &runner_;
+    OpKernelContext op_ctx(&op_params, 0);
+    IteratorContext::Params params(&op_ctx);
+    params.function_handle_cache = function_handle_cache_.get();
+    ctx = MakeUnique<IteratorContext>(std::move(params));
+  }
+
+  // Create the iterator from the dataset.
+  std::unique_ptr<IteratorBase> iterator;
+  TF_RETURN_IF_ERROR(dataset_->MakeIterator(ctx.get(), "iterator", &iterator));
+
+  *result = WrapUnique(new Iterator(iterator.release(), ctx.release()));
+
+  return Status::OK();
+}
+
+Dataset::Dataset(DatasetBase* dataset, DeviceMgr* device_mgr,
+                 ProcessFunctionLibraryRuntime* pflr,
+                 FunctionLibraryDefinition* flib_def, thread::ThreadPool* pool)
+    : dataset_(dataset),
+      device_mgr_(device_mgr),
+      flib_def_(flib_def),
+      pflr_(pflr),
+      pool_(pool) {
+  runner_ = [this](std::function<void()> c) { pool_->Schedule(std::move(c)); };
+  function_handle_cache_ =
+      MakeUnique<FunctionHandleCache>(pflr_->GetFLR("/device:CPU:0"));
+}
+
+Dataset::~Dataset() { dataset_->Unref(); }
+
+}  // namespace standalone
+}  // namespace data
+}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/data/standalone.h b/tensorflow/core/common_runtime/data/standalone.h
new file mode 100644
index 0000000000..ecea5ba21d
--- /dev/null
+++ b/tensorflow/core/common_runtime/data/standalone.h
@@ -0,0 +1,122 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_DATA_STANDALONE_H_
+#define TENSORFLOW_CORE_COMMON_RUNTIME_DATA_STANDALONE_H_
+
+#include <memory>
+#include "tensorflow/core/common_runtime/device_mgr.h"
+#include "tensorflow/core/framework/dataset.h"
+#include "tensorflow/core/framework/function_handle_cache.h"
+#include "tensorflow/core/lib/core/threadpool.h"
+#include "tensorflow/core/public/session_options.h"
+
+namespace tensorflow {
+namespace data {
+namespace standalone {
+
+// The purpose of the API in this file is to facilitate standalone execution of
+// a tf.data input pipeline graph.
+//
+// The API exposes two abstractions -- a `Dataset` and an `Iterator` -- which
+// encapsulate TensorFlow runtime.
+//
+// The `Dataset` abstraction represents an input pipeline as a collection
+// of data sources and a logical plan of transformations that operate over the
+// data.
+//
+// The `Iterator` abstraction represents an execution of an input pipeline that
+// can be used to enumerate its elements.
+//
+// Example usage:
+//
+//   // Create a `Dataset` by running the `graph_def` graph and fetching the
+//   // output of the `fetch_node` node.
+//   tensorflow::data:standalone::Dataset::Params params;
+//   std::unique_ptr<tensorflow::data::standalone::Dataset> dataset;
+//   Status s = tensorflow::data::standalone::Dataset::FromGraph(
+//      params, graph_def, fetch_node, &dataset);
+//   if (!s.ok()) { /* error handling */ }
+//
+//   std::unique_ptr<tensorflow::data::standalone::Iterator> iterator;
+//   s = dataset->MakeIterator(&iterator);
+//   if (!s.ok()) { /* error handling */ }
+//
+//   bool end_of_input = false;
+//   while (!end_of_input) {
+//     std::vector<tensorflow::Tensor> outputs;
+//     s = iterator->GetNext(&outputs, &end_of_input);
+//     if (!s.ok()) { /* error handling */ }
+//     if (!end_of_input) { /* output handling */ }
+//   }
+
+class Dataset;
+
+// Represents an execution of an input pipeline that can be used to enumerate
+// its elements.
+class Iterator {
+ public:
+  // Returns the next element of the input pipeline (if there is one) and an
+  // indication of whether the end of the input pipeline has been reached.
+  Status GetNext(std::vector<Tensor>* outputs, bool* end_of_input);
+
+ private:
+  friend class Dataset;
+
+  Iterator(IteratorBase* iterator, IteratorContext* ctx);
+
+  std::unique_ptr<IteratorBase> iterator_;
+  std::unique_ptr<IteratorContext> ctx_;
+};
+
+// Represents an input pipeline as a collection of data sources and a logical
+// plan of transformations that operate over the data.
+class Dataset {
+ public:
+  // Parameters for `Dataset` creation (e.g. TensorFlow runtime configuration).
+  struct Params {
+    SessionOptions session_options;
+  };
+
+  // Creates a new `Dataset` instance by running the TensorFlow graph `graph`
+  // and fetching the output of the `fetch_node` node.
+  static Status FromGraph(Params params, const GraphDef& graph_def,
+                          const string& fetch_node,
+                          std::unique_ptr<Dataset>* result);
+
+  ~Dataset();
+
+  // Creates an iterator for this dataset.
+  Status MakeIterator(std::unique_ptr<Iterator>* result);
+
+ private:
+  Dataset(DatasetBase* dataset, DeviceMgr* device_mgr,
+          ProcessFunctionLibraryRuntime* pflr,
+          FunctionLibraryDefinition* flib_def, thread::ThreadPool* pool);
+
+  DatasetBase* dataset_;  // owned
+  std::unique_ptr<DeviceMgr> device_mgr_;
+  std::unique_ptr<FunctionLibraryDefinition> flib_def_;
+  std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_;
+  std::unique_ptr<thread::ThreadPool> pool_;
+  std::unique_ptr<FunctionHandleCache> function_handle_cache_;
+  std::function<void(std::function<void()>)> runner_;
+};
+
+}  // namespace standalone
+}  // namespace data
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_DATA_STANDALONE_H_
diff --git a/tensorflow/core/common_runtime/data/standalone_test.cc b/tensorflow/core/common_runtime/data/standalone_test.cc
new file mode 100644
index 0000000000..7e7a7a9b61
--- /dev/null
+++ b/tensorflow/core/common_runtime/data/standalone_test.cc
@@ -0,0 +1,188 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <memory>
+#include <vector>
+
+#include "tensorflow/core/common_runtime/data/standalone.h"
+#include "tensorflow/core/framework/graph.pb.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+namespace data {
+namespace standalone {
+namespace {
+
+constexpr const char* const kGraphProto = R"proto(
+  node {
+    name: "Const/_0"
+    op: "Const"
+    attr {
+      key: "dtype"
+      value { type: DT_INT64 }
+    }
+    attr {
+      key: "value"
+      value {
+        tensor {
+          dtype: DT_INT64
+          tensor_shape {}
+          int64_val: 0
+        }
+      }
+    }
+  }
+  node {
+    name: "Const/_1"
+    op: "Const"
+    attr {
+      key: "dtype"
+      value { type: DT_INT64 }
+    }
+    attr {
+      key: "value"
+      value {
+        tensor {
+          dtype: DT_INT64
+          tensor_shape {}
+          int64_val: 10
+        }
+      }
+    }
+  }
+  node {
+    name: "Const/_2"
+    op: "Const"
+    attr {
+      key: "dtype"
+      value { type: DT_INT64 }
+    }
+    attr {
+      key: "value"
+      value {
+        tensor {
+          dtype: DT_INT64
+          tensor_shape {}
+          int64_val: 1
+        }
+      }
+    }
+  }
+  node {
+    name: "RangeDataset/_3"
+    op: "RangeDataset"
+    input: "Const/_0"
+    input: "Const/_1"
+    input: "Const/_2"
+    attr {
+      key: "output_shapes"
+      value { list { shape { unknown_rank: true } } }
+    }
+    attr {
+      key: "output_types"
+      value { list { type: DT_INT64 } }
+    }
+  }
+  node {
+    name: "MapDataset/_4"
+    op: "MapDataset"
+    input: "RangeDataset/_3"
+    attr {
+      key: "Targuments"
+      value { list {} }
+    }
+    attr {
+      key: "f"
+      value { func { name: "Dataset_map_<lambda>_10" } }
+    }
+    attr {
+      key: "output_shapes"
+      value { list { shape {} } }
+    }
+    attr {
+      key: "output_types"
+      value { list { type: DT_INT64 } }
+    }
+    attr {
+      key: "preserve_cardinality"
+      value { b: false }
+    }
+    attr {
+      key: "use_inter_op_parallelism"
+      value { b: true }
+    }
+  }
+  library {
+    function {
+      signature {
+        name: "Dataset_map_<lambda>_10"
+        input_arg { name: "arg0" type: DT_INT64 }
+        output_arg { name: "mul" type: DT_INT64 }
+        description: "Wrapper for passing nested structures to and from tf.data functions."
+      }
+      node_def {
+        name: "mul_0"
+        op: "Mul"
+        input: "arg0"
+        input: "arg0"
+        attr {
+          key: "T"
+          value { type: DT_INT64 }
+        }
+      }
+      ret { key: "mul" value: "mul_0:z:0" }
+    }
+  }
+  versions { producer: 27 min_consumer: 12 }
+)proto";
+
+TEST(Scalar, Standalone) {
+  GraphDef graph_def;
+  protobuf::TextFormat::ParseFromString(kGraphProto, &graph_def);
+  struct TestCase {
+    string fetch_node;
+    std::vector<int64> expected_outputs;
+  };
+  auto test_cases = {
+      TestCase{"RangeDataset/_3", {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}},
+      TestCase{"MapDataset/_4", {0, 1, 4, 9, 16, 25, 36, 49, 64, 81}},
+  };
+  for (auto test_case : test_cases) {
+    std::unique_ptr<Dataset> dataset;
+    auto s = Dataset::FromGraph({}, graph_def, test_case.fetch_node, &dataset);
+    TF_EXPECT_OK(s);
+    std::unique_ptr<Iterator> iterator;
+    s = dataset->MakeIterator(&iterator);
+    TF_EXPECT_OK(s);
+    bool end_of_input = false;
+    for (int num_outputs = 0; !end_of_input; ++num_outputs) {
+      std::vector<tensorflow::Tensor> outputs;
+      s = iterator->GetNext(&outputs, &end_of_input);
+      TF_EXPECT_OK(s);
+      if (!end_of_input) {
+        EXPECT_EQ(outputs[0].scalar<int64>()(),
+                  test_case.expected_outputs[num_outputs]);
+      } else {
+        EXPECT_EQ(test_case.expected_outputs.size(), num_outputs);
+      }
+    }
+  }
+}
+
+}  // namespace
+}  // namespace standalone
+}  // namespace data
+}  // namespace tensorflow
-- 
GitLab


From 17efadd4556c242efaa68b30134d3e79d2412922 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 14:57:23 -0800
Subject: [PATCH 0293/2345] Internal copybara changes.

PiperOrigin-RevId: 228241768
---
 tensorflow/lite/experimental/swift/BUILD      |  1 +
 .../Configs/TensorFlowLite.tulsigen           | 20 +++++++++----------
 .../project.tulsiconf                         |  4 ++--
 .../Base.lproj/Main.storyboard                |  6 +++---
 4 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/tensorflow/lite/experimental/swift/BUILD b/tensorflow/lite/experimental/swift/BUILD
index 3bd288a1e1..53bcb0ecbd 100644
--- a/tensorflow/lite/experimental/swift/BUILD
+++ b/tensorflow/lite/experimental/swift/BUILD
@@ -83,6 +83,7 @@ ios_application(
 swift_library(
     name = "TensorFlowLiteAppLib",
     srcs = glob(["TestApps/TensorFlowLiteApp/TensorFlowLiteApp/*.swift"]),
+    module_name = "TensorFlowLiteAppLib",
     tags = ["manual"],
     deps = [
         ":TensorFlowLite",
diff --git a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
index 4010fab49e..16bc6cbfe8 100644
--- a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
+++ b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
@@ -1,16 +1,16 @@
 {
   "sourceFilters" : [
-    "third_party/tensorflow/lite/experimental/c",
-    "third_party/tensorflow/lite/experimental/swift",
-    "third_party/tensorflow/lite/experimental/swift/Sources",
-    "third_party/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp",
-    "third_party/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj",
-    "third_party/tensorflow/lite/experimental/swift/Tests",
+    "tensorflow/lite/experimental/c",
+    "tensorflow/lite/experimental/swift",
+    "tensorflow/lite/experimental/swift/Sources",
+    "tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp",
+    "tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj",
+    "tensorflow/lite/experimental/swift/Tests",
   ],
   "buildTargets" : [
-    "//third_party/tensorflow/lite/experimental/swift:TensorFlowLite",
-    "//third_party/tensorflow/lite/experimental/swift:TensorFlowLiteApp",
-    "//third_party/tensorflow/lite/experimental/swift:TensorFlowLiteTests",
+    "//tensorflow/lite/experimental/swift:TensorFlowLite",
+    "//tensorflow/lite/experimental/swift:TensorFlowLiteApp",
+    "//tensorflow/lite/experimental/swift:TensorFlowLiteTests",
   ],
   "projectName" : "TensorFlowLite",
   "optionSet" : {
@@ -52,6 +52,6 @@
     }
   },
   "additionalFilePaths" : [
-    "third_party/tensorflow/lite/experimental/swift/BUILD"
+    "tensorflow/lite/experimental/swift/BUILD"
   ]
 }
diff --git a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
index 14cff94453..82ac8aa381 100644
--- a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
+++ b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
@@ -8,7 +8,7 @@
   },
   "projectName" : "TensorFlowLite",
   "packages" : [
-    "third_party/tensorflow/lite/experimental/swift"
+    "tensorflow/lite/experimental/swift"
   ],
-  "workspaceRoot" : "../../../../../.."
+  "workspaceRoot" : "../../../../.."
 }
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
index d0e91d63c4..10cae6e855 100644
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
@@ -1,17 +1,17 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="14313.18" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" colorMatched="YES" initialViewController="BYZ-38-t0r">
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="14460.31" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" colorMatched="YES" initialViewController="BYZ-38-t0r">
     <device id="retina4_7" orientation="portrait">
         <adaptation id="fullscreen"/>
     </device>
     <dependencies>
-        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="14283.14"/>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="14460.20"/>
         <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
     </dependencies>
     <scenes>
         <!--View Controller-->
         <scene sceneID="tne-QT-ifu">
             <objects>
-                <viewController storyboardIdentifier="viewController" useStoryboardIdentifierAsRestorationIdentifier="YES" id="BYZ-38-t0r" customClass="ViewController" customModule="third_party_tensorflow_lite_experimental_swift_TensorFlowLiteAppLib" sceneMemberID="viewController">
+                <viewController storyboardIdentifier="viewController" useStoryboardIdentifierAsRestorationIdentifier="YES" id="BYZ-38-t0r" customClass="ViewController" customModule="TensorFlowLiteAppLib" sceneMemberID="viewController">
                     <layoutGuides>
                         <viewControllerLayoutGuide type="top" id="y3c-jy-aDJ"/>
                         <viewControllerLayoutGuide type="bottom" id="wfy-db-euE"/>
-- 
GitLab


From a0dfedcca7c766404735720b9d675e1a04f40350 Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Mon, 7 Jan 2019 15:02:16 -0800
Subject: [PATCH 0294/2345] [XLA] Use DynamicDimensionInference in HloEvaluator

- Use DynamicDimensionInference in HloEvaluator so that it can correctly evaluate GetDimensionSize.
- Change ComputeConstant to first calculate DynamicDimensionInference and pass it into evaluator.

PiperOrigin-RevId: 228242693
---
 tensorflow/compiler/xla/service/BUILD         |  2 +
 .../compiler/xla/service/hlo_evaluator.cc     | 34 ++++++++++++++++
 .../compiler/xla/service/hlo_evaluator.h      | 12 ++++++
 .../xla/service/hlo_evaluator_test.cc         | 40 +++++++++++++++++--
 tensorflow/compiler/xla/service/service.cc    |  5 +++
 5 files changed, 89 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 3c18c450e6..9bc6218f75 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -236,6 +236,7 @@ cc_library(
     ],
     hdrs = ["hlo_evaluator.h"],
     deps = [
+        ":dynamic_dimension_inference",
         ":hlo",
         ":hlo_casting_utils",
         ":hlo_query",
@@ -697,6 +698,7 @@ cc_library(
         ":compiler",
         ":computation_layout",
         ":device_memory_allocator",
+        ":dynamic_dimension_inference",
         ":executable",
         ":execution_tracker",
         ":hlo",
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index 3a97b56c66..b8d95fb244 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -361,6 +361,31 @@ Status HloEvaluator::HandleBitcast(HloInstruction* bitcast) {
   return Status::OK();
 }
 
+Status HloEvaluator::HandleGetDimensionSize(
+    HloInstruction* get_dimension_size) {
+  HloInstruction* operand = get_dimension_size->mutable_operand(0);
+  int64 dim = get_dimension_size->dimension();
+  if (dynamic_dimension_inference_ == nullptr) {
+    return InvalidArgument(
+        "Evaluator cannot evaluate get_dimension_size without "
+        "set_dynamic_dimension_inference.");
+  }
+  HloInstruction* dynamic_size =
+      dynamic_dimension_inference_->GetDynamicSize(operand, {}, dim);
+  if (dynamic_size != nullptr) {
+    evaluated_[get_dimension_size] =
+        GetEvaluatedLiteralFor(dynamic_size).Clone();
+    return Status::OK();
+  }
+
+  const Shape& shape = get_dimension_size->operand(0)->shape();
+  Literal output(ShapeUtil::MakeShape(U32, {}));
+  output.PopulateWithValue(
+      static_cast<uint32>(shape.dimensions(get_dimension_size->dimension())));
+  evaluated_[get_dimension_size] = std::move(output);
+  return Status::OK();
+}
+
 Status HloEvaluator::HandleParameter(HloInstruction* parameter) {
   CHECK_LT(parameter->parameter_number(), arg_literals_.size());
   const Literal* input_literal = arg_literals_[parameter->parameter_number()];
@@ -1080,6 +1105,8 @@ Status HloEvaluator::HandleCall(HloInstruction* call) {
   }
 
   HloEvaluator embedded_evaluator;
+  embedded_evaluator.set_dynamic_dimension_inference(
+      dynamic_dimension_inference_);
   Literal result = embedded_evaluator.Evaluate(*computation, arg_literals)
                        .ConsumeValueOrDie();
 
@@ -1113,6 +1140,8 @@ Status HloEvaluator::HandleFusion(HloInstruction* fusion) {
   }
 
   HloEvaluator embedded_evaluator;
+  embedded_evaluator.set_dynamic_dimension_inference(
+      dynamic_dimension_inference_);
   Literal result =
       embedded_evaluator.Evaluate(*readded_computation, arg_literals)
           .ConsumeValueOrDie();
@@ -1132,6 +1161,8 @@ Status HloEvaluator::HandleConditional(HloInstruction* conditional) {
   auto* false_computation = conditional->false_computation();
 
   HloEvaluator embedded_evaluator;
+  embedded_evaluator.set_dynamic_dimension_inference(
+      dynamic_dimension_inference_);
   Literal result;
   if (pred.Get<bool>({})) {
     result =
@@ -1186,7 +1217,10 @@ Status HloEvaluator::HandleWhile(HloInstruction* while_hlo) {
   bool keep_going = true;
   int64 iteration_count = 0;
   HloEvaluator cond_evaluator(max_loop_iterations_);
+  cond_evaluator.set_dynamic_dimension_inference(dynamic_dimension_inference_);
   HloEvaluator loop_body_evaluator(max_loop_iterations_);
+  loop_body_evaluator.set_dynamic_dimension_inference(
+      dynamic_dimension_inference_);
   while (keep_going) {
     if (max_loop_iterations_ >= 0 && iteration_count++ > max_loop_iterations_) {
       return InvalidArgument("Loop %s exceeded loop iteration limit (%d).",
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.h b/tensorflow/compiler/xla/service/hlo_evaluator.h
index 4fc5d10aed..604b861913 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.h
@@ -23,6 +23,7 @@ limitations under the License.
 #include "absl/types/span.h"
 #include "tensorflow/compiler/xla/array2d.h"
 #include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h"
+#include "tensorflow/compiler/xla/service/dynamic_dimension_inference.h"
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
@@ -123,6 +124,11 @@ class HloEvaluator : public DfsHloVisitorWithDefault {
                                   const PrecisionConfig& precision_config,
                                   const Literal& lhs, const Literal& rhs);
 
+  void set_dynamic_dimension_inference(
+      DynamicDimensionInference* dynamic_dimension_inference) {
+    dynamic_dimension_inference_ = dynamic_dimension_inference;
+  }
+
   // Enable the fast path for certain operations like dot or convolution.
   void set_use_fast_path(bool value) { use_fast_path_ = value; }
 
@@ -161,6 +167,8 @@ class HloEvaluator : public DfsHloVisitorWithDefault {
   //
   Status HandleBitcast(HloInstruction* bitcast) override;
 
+  Status HandleGetDimensionSize(HloInstruction* get_dimension_size) override;
+
   Status HandleParameter(HloInstruction* parameter) override;
 
   Status HandleConstant(HloInstruction* constant) override;
@@ -295,6 +303,10 @@ class HloEvaluator : public DfsHloVisitorWithDefault {
   // RNG engine.
   std::minstd_rand0 engine_;
 
+  // DynamicDimensionInference is used to evaluate GetDimensionSize, which
+  // returns the dynamic dimension size of its operand.
+  DynamicDimensionInference* dynamic_dimension_inference_;
+
   TF_DISALLOW_COPY_AND_ASSIGN(HloEvaluator);
 };
 
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
index 8056193d99..319f0db97a 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
@@ -62,8 +62,7 @@ class HloEvaluatorTest : public HloTestBase {
     if (use_bfloat16_) {
       HloElementTypeConverter(F32, BF16).Run(m_.get()).ValueOrDie();
     }
-    return HloEvaluator()
-        .Evaluate(*m_->entry_computation(), arg_literals)
+    return evaluator_.Evaluate(*m_->entry_computation(), arg_literals)
         .ConsumeValueOrDie();
   }
 
@@ -75,8 +74,7 @@ class HloEvaluatorTest : public HloTestBase {
     if (use_bfloat16_) {
       HloElementTypeConverter(F32, BF16).Run(m_.get()).ValueOrDie();
     }
-    return HloEvaluator()
-        .Evaluate(*module->entry_computation(), arg_literals)
+    return evaluator_.Evaluate(*module->entry_computation(), arg_literals)
         .ConsumeValueOrDie();
   }
 
@@ -115,6 +113,7 @@ class HloEvaluatorTest : public HloTestBase {
 
  protected:
   explicit HloEvaluatorTest(bool use_bfloat16) : use_bfloat16_(use_bfloat16) {}
+  HloEvaluator evaluator_;
 
   const bool use_bfloat16_;
   std::unique_ptr<HloModule> m_ = CreateNewVerifiedModule();
@@ -2874,6 +2873,39 @@ ENTRY main {
             static_cast<int32>(pow31 * pow31));
 }
 
+TEST_F(HloEvaluatorTest, GetDimensionSize) {
+  constexpr absl::string_view hlo_text = R"(
+HloModule Test
+
+ENTRY main {
+  size = u32[] parameter(0)
+
+  data = s32[4] parameter(1)
+
+  sum = s32[4] add(data, data)
+
+  ROOT dynamic_size = u32[] get-dimension-size(sum), dimensions={0}
+}
+)";
+  TF_ASSERT_OK_AND_ASSIGN(m_, ParseAndReturnVerifiedModule(hlo_text));
+
+  // Set up dynamic parameter binding.
+  TF_CHECK_OK(m_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{0, {}},
+      DynamicParameterBinding::DynamicDimension{1, {}, 0}));
+
+  TF_ASSERT_OK_AND_ASSIGN(DynamicDimensionInference dynamic_dimension_inference,
+                          DynamicDimensionInference::Run(m_.get()));
+
+  evaluator_.set_dynamic_dimension_inference(&dynamic_dimension_inference);
+  Literal size_arg = LiteralUtil::CreateR0<uint32>(3);
+  Literal data_arg = LiteralUtil::CreateR1<int32>({1, 2, 3, 4});
+
+  Literal actual = Evaluate({&size_arg, &data_arg});
+
+  EXPECT_EQ(actual.GetFirstElement<uint32>(), static_cast<uint32>(3));
+}
+
 // Check that we get a useful error if we pass inputs of the wrong shape.
 TEST_F(HloEvaluatorTest, EvaluateWithWrongInputShapes) {
   constexpr absl::string_view hlo_text = R"(
diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc
index efefe9003f..4a8d272261 100644
--- a/tensorflow/compiler/xla/service/service.cc
+++ b/tensorflow/compiler/xla/service/service.cc
@@ -29,6 +29,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/compiler.h"
 #include "tensorflow/compiler/xla/service/computation_layout.h"
 #include "tensorflow/compiler/xla/service/device_memory_allocator.h"
+#include "tensorflow/compiler/xla/service/dynamic_dimension_inference.h"
 #include "tensorflow/compiler/xla/service/executable.h"
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
 #include "tensorflow/compiler/xla/service/hlo_cost_analysis.h"
@@ -1097,7 +1098,11 @@ Status Service::ComputeConstantGraph(const ComputeConstantGraphRequest* arg,
   TF_ASSIGN_OR_RETURN(std::unique_ptr<HloModule> module,
                       CreateModuleFromProto(arg->computation(), config));
 
+  TF_ASSIGN_OR_RETURN(DynamicDimensionInference dynamic_dimension_inference,
+                      DynamicDimensionInference::Run(module.get()));
+
   HloEvaluator evaluator;
+  evaluator.set_dynamic_dimension_inference(&dynamic_dimension_inference);
   TF_ASSIGN_OR_RETURN(auto result_literal, evaluator.Evaluate(*module, {}));
 
   // Since the result layout is non-effective to the Evaluator results, explicit
-- 
GitLab


From b94da8cca6fc657a77ab920e7feeb05cd57f37d9 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 7 Jan 2019 15:04:18 -0800
Subject: [PATCH 0295/2345] Print a more ergonomic error message when the CPU
 JIT is not linked in

PiperOrigin-RevId: 228243176
---
 tensorflow/compiler/xla/service/compiler.cc | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/compiler.cc b/tensorflow/compiler/xla/service/compiler.cc
index 8f08c24490..653f4555a7 100644
--- a/tensorflow/compiler/xla/service/compiler.cc
+++ b/tensorflow/compiler/xla/service/compiler.cc
@@ -98,10 +98,17 @@ Compiler::GetPlatformCompilers() {
   auto* factories = GetPlatformCompilerFactories();
   auto it = factories->find(platform->id());
   if (it == factories->end()) {
+    string hint;
+    if (platform->Name() == "Host") {
+      hint = " (hint: try linking in tensorflow/compiler/jit:xla_cpu_jit)";
+    } else if (platform->Name() == "CUDA") {
+      hint = " (hint: try linking in tensorflow/compiler/jit:xla_gpu_jit)";
+    }
+
     return NotFound(
         "could not find registered compiler for platform %s -- check "
-        "target linkage",
-        platform->Name());
+        "target linkage%s",
+        platform->Name(), hint);
   }
 
   // And then we invoke the factory, placing the result into the mapping.
-- 
GitLab


From 35938dbf10a7695ace5fa1707b5e6ccdd22869a0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 15:34:00 -0800
Subject: [PATCH 0296/2345] Fixes for flex issues with multiple subgraphs.

PiperOrigin-RevId: 228248671
---
 tensorflow/lite/delegates/flex/buffer_map.h   |  3 ++
 .../lite/delegates/flex/buffer_map_test.cc    |  2 +
 tensorflow/lite/delegates/flex/kernel.cc      | 11 +++++-
 tensorflow/lite/delegates/flex/kernel_test.cc | 37 +++++++------------
 4 files changed, 27 insertions(+), 26 deletions(-)

diff --git a/tensorflow/lite/delegates/flex/buffer_map.h b/tensorflow/lite/delegates/flex/buffer_map.h
index 6c1df4c836..d4724a011d 100644
--- a/tensorflow/lite/delegates/flex/buffer_map.h
+++ b/tensorflow/lite/delegates/flex/buffer_map.h
@@ -66,6 +66,9 @@ class BufferMap {
   // be use by TF's forwarding optimizations.
   void SetForwardable(int tensor_index) { forwardable_.insert(tensor_index); }
 
+  // Removes all information about which tensors are forwardable.
+  void ClearForwardable() { forwardable_.clear(); }
+
   // Returns true if this tensor has been explicitly marks as forwardable by
   // a call to SetForwardable().
   bool IsForwardable(int tensor_index) const {
diff --git a/tensorflow/lite/delegates/flex/buffer_map_test.cc b/tensorflow/lite/delegates/flex/buffer_map_test.cc
index 2148bfe8e2..7b4acbd69d 100644
--- a/tensorflow/lite/delegates/flex/buffer_map_test.cc
+++ b/tensorflow/lite/delegates/flex/buffer_map_test.cc
@@ -278,6 +278,8 @@ TEST(BufferMapTest, Forwardable) {
   EXPECT_FALSE(buffer_map.IsForwardable(0));
   buffer_map.SetForwardable(0);
   EXPECT_TRUE(buffer_map.IsForwardable(0));
+  buffer_map.ClearForwardable();
+  EXPECT_FALSE(buffer_map.IsForwardable(0));
 }
 
 }  // namespace
diff --git a/tensorflow/lite/delegates/flex/kernel.cc b/tensorflow/lite/delegates/flex/kernel.cc
index 0005b6e5b0..84745ba9cb 100644
--- a/tensorflow/lite/delegates/flex/kernel.cc
+++ b/tensorflow/lite/delegates/flex/kernel.cc
@@ -350,7 +350,11 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
         buffer_map->SetFromTfLite(tensor_index, tensor);
       }
     }
-    ++tensor_ref_count[tensor_index];
+
+    // Input tensors should never be forwarded so we increment their ref counts
+    // twice: once for this graph and another for the possibility of them being
+    // used by another subgraph, or being an output of the full graph.
+    tensor_ref_count[tensor_index] += 2;
   }
 
   // All output tensors are allocated by TensorFlow/Eager, so we
@@ -372,6 +376,7 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
     }
   }
 
+  buffer_map->ClearForwardable();
   for (const auto& x : tensor_ref_count) {
     if (x.second == 1) {
       // This tensor is referenced once by a single op. We can allow the TF
@@ -429,10 +434,12 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   }
 
   // We don't need to keep track of internal TF tensors any longer, so take
-  // them out of the buffer_map, but make sure we keep all the one we might
+  // them out of the buffer_map, but make sure we keep all the ones we might
   // need for other subgraphs, or as final output of inference.
   const auto& outputs = op_data->subgraph_outputs;
   std::set<int> keep(outputs.begin(), outputs.end());
+  const auto& inputs = op_data->subgraph_inputs;
+  keep.insert(inputs.begin(), inputs.end());
   buffer_map->RemoveTensorsNotInSet(keep);
 
   return kTfLiteOk;
diff --git a/tensorflow/lite/delegates/flex/kernel_test.cc b/tensorflow/lite/delegates/flex/kernel_test.cc
index 9de3ca67de..cef8017e6e 100644
--- a/tensorflow/lite/delegates/flex/kernel_test.cc
+++ b/tensorflow/lite/delegates/flex/kernel_test.cc
@@ -25,6 +25,7 @@ namespace {
 
 using ::testing::ContainsRegex;
 using ::testing::ElementsAre;
+using ::testing::ElementsAreArray;
 
 TfLiteStatus GenericPrepare(TfLiteContext* context, TfLiteDelegate* delegate,
                             const std::vector<int>& supported_nodes) {
@@ -340,14 +341,10 @@ TEST_F(MultipleSubgraphsTest, ForwardabilityIsLocal) {
   auto input = {3.0f, 4.0f, 5.0f};
   PrepareInterpreter(GetPrepareFunction<__LINE__>(), input);
 
-  ASSERT_FALSE(Invoke());
-  ASSERT_THAT(error_reporter().error_messages(),
-              ContainsRegex("Cannot read from invalid tensor index 10"));
-
-  // TODO(b/122457581): expected result for this test:
-  // ASSERT_THAT(GetValues(12), ElementsAreArray(Apply(input, [](float in) {
-  //               return (4 * in + 4) * (in + 1);
-  //             })));
+  ASSERT_TRUE(Invoke());
+  ASSERT_THAT(GetValues(12), ElementsAreArray(Apply(input, [](float in) {
+                return (4 * in + 4) * (in + 1);
+              })));
 }
 
 // Subgraphs should not remove input tensors from the buffer_map, since
@@ -379,14 +376,10 @@ TEST_F(MultipleSubgraphsTest, DoNotRemoveInputTensors) {
   auto input = {3.0f, 4.0f, 5.0f};
   PrepareInterpreter(GetPrepareFunction<__LINE__>(), input);
 
-  ASSERT_FALSE(Invoke());
-  ASSERT_THAT(error_reporter().error_messages(),
-              ContainsRegex("Tensor '10' not found"));
-
-  // TODO(b/122457581): expected result for this test:
-  // ASSERT_THAT(GetValues(12), ElementsAreArray(Apply(input, [](float in) {
-  //               return (4 * in + 4) * (in + 1);
-  //             })));
+  ASSERT_TRUE(Invoke());
+  ASSERT_THAT(GetValues(12), ElementsAreArray(Apply(input, [](float in) {
+                return (4 * in + 4) * (in + 1);
+              })));
 }
 
 // A tensor is deemed forwardable but it happens to be the input to
@@ -417,14 +410,10 @@ TEST_F(MultipleSubgraphsTest, DoNotForwardInputTensors) {
   auto input = {3.0f, 4.0f, 5.0f};
   PrepareInterpreter(GetPrepareFunction<__LINE__>(), input);
 
-  ASSERT_FALSE(Invoke());
-  ASSERT_THAT(error_reporter().error_messages(),
-              ContainsRegex("Tensor '10' not found"));
-
-  // TODO(b/122457581): expected result for this test:
-  // ASSERT_THAT(GetValues(12), ElementsAreArray(Apply(input, [](float in) {
-  //               return (5 * in + 5) * (in + 1);
-  //             })));
+  ASSERT_TRUE(Invoke());
+  ASSERT_THAT(GetValues(12), ElementsAreArray(Apply(input, [](float in) {
+                return (5 * in + 5) * (in + 1);
+              })));
 }
 
 }  // namespace
-- 
GitLab


From 6d26622733a3f55861c135b857b2cbce981b2467 Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Mon, 7 Jan 2019 15:34:34 -0800
Subject: [PATCH 0297/2345] Fix some tests failures.

PiperOrigin-RevId: 228248776
---
 tensorflow/lite/kernels/BUILD                                 | 2 ++
 tensorflow/lite/kernels/dequantize_test.cc                    | 1 +
 tensorflow/lite/kernels/internal/BUILD                        | 4 ++++
 tensorflow/lite/kernels/sparse_output_fully_connected_test.cc | 1 +
 4 files changed, 8 insertions(+)

diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index 6809a4a3d0..50deac9bb6 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -352,6 +352,7 @@ tf_cc_test(
         ":builtin_ops",
         "//tensorflow/lite:framework",
         "//tensorflow/lite/kernels:test_util",
+        "//tensorflow/lite/kernels/internal:types",
         "@com_google_googletest//:gtest",
         "@flatbuffers",
     ],
@@ -513,6 +514,7 @@ tf_cc_test(
         ":builtin_ops",
         "//tensorflow/lite:framework",
         "//tensorflow/lite/kernels:test_util",
+        "//tensorflow/lite/kernels/internal:types",
         "@com_google_absl//absl/memory",
         "@com_google_googletest//:gtest",
     ],
diff --git a/tensorflow/lite/kernels/dequantize_test.cc b/tensorflow/lite/kernels/dequantize_test.cc
index bb5f1e74a8..6343745eef 100644
--- a/tensorflow/lite/kernels/dequantize_test.cc
+++ b/tensorflow/lite/kernels/dequantize_test.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 #include <gtest/gtest.h>
 #include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/kernels/internal/types.h"
 #include "tensorflow/lite/kernels/register.h"
 #include "tensorflow/lite/kernels/test_util.h"
 #include "tensorflow/lite/model.h"
diff --git a/tensorflow/lite/kernels/internal/BUILD b/tensorflow/lite/kernels/internal/BUILD
index 92b4705908..885f650dd5 100644
--- a/tensorflow/lite/kernels/internal/BUILD
+++ b/tensorflow/lite/kernels/internal/BUILD
@@ -545,6 +545,10 @@ cc_library(
     name = "test_util",
     srcs = ["test_util.cc"],
     hdrs = ["test_util.h"],
+    linkopts = select({
+        "//tensorflow:windows": [],
+        "//conditions:default": ["-lm"],
+    }),
     deps = [
         ":types",
         "//tensorflow/lite:string",
diff --git a/tensorflow/lite/kernels/sparse_output_fully_connected_test.cc b/tensorflow/lite/kernels/sparse_output_fully_connected_test.cc
index fb71cb7f7b..8152ae6685 100644
--- a/tensorflow/lite/kernels/sparse_output_fully_connected_test.cc
+++ b/tensorflow/lite/kernels/sparse_output_fully_connected_test.cc
@@ -19,6 +19,7 @@ limitations under the License.
 
 #include <gtest/gtest.h>
 #include "flatbuffers/flexbuffers.h"  // TF:flatbuffers
+#include "tensorflow/lite/kernels/internal/types.h"
 #include "tensorflow/lite/kernels/register.h"
 #include "tensorflow/lite/kernels/test_util.h"
 
-- 
GitLab


From 3b74e1c16ab280a4e0cbd5138a0c73710cb9804c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 15:46:39 -0800
Subject: [PATCH 0298/2345] Clarified that variables_to_restore function
 chooses whether to use a moving average version based on provided parameters
 rather than based on the existence of a given name.

PiperOrigin-RevId: 228250786
---
 tensorflow/python/training/moving_averages.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/training/moving_averages.py b/tensorflow/python/training/moving_averages.py
index 72670f0ca3..4b267dfb98 100644
--- a/tensorflow/python/training/moving_averages.py
+++ b/tensorflow/python/training/moving_averages.py
@@ -505,13 +505,13 @@ class ExponentialMovingAverage(object):
     ```
     Args:
       moving_avg_variables: a list of variables that require to use of the
-        moving variable name to be restored. If None, it will default to
+        moving average variable name to be restored. If None, it will default to
         variables.moving_average_variables() + variables.trainable_variables()
 
     Returns:
-      A map from restore_names to variables. The restore_name can be the
-      moving_average version of the variable name if it exist, or the original
-      variable name.
+      A map from restore_names to variables. The restore_name is either the
+      original or the moving average version of the variable name, depending
+      on whether the variable name is in the `moving_avg_variables`.
     """
     name_map = {}
     if moving_avg_variables is None:
-- 
GitLab


From 0de09568916f31d77ef0093285a058e901195919 Mon Sep 17 00:00:00 2001
From: Peter Ma <pcma@google.com>
Date: Mon, 7 Jan 2019 16:09:22 -0800
Subject: [PATCH 0299/2345] Replace op_features with op_info to correctly
 reflect the data type, and some minor fix on function argument names.

PiperOrigin-RevId: 228254456
---
 .../grappler/costs/op_level_cost_estimator.cc | 246 +++++++++---------
 .../grappler/costs/op_level_cost_estimator.h  |  34 ++-
 .../costs/op_level_cost_estimator_test.cc     |  17 +-
 3 files changed, 143 insertions(+), 154 deletions(-)

diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
index 55eb391d2b..96bac8d0cb 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
@@ -72,25 +72,25 @@ static const Costs::Duration kMinComputeTime(1);
 
 namespace {
 
-string GetDataFormat(const OpInfo& op_features) {
+string GetDataFormat(const OpInfo& op_info) {
   string data_format = "NHWC";  // Default format.
-  if (op_features.attr().find("data_format") != op_features.attr().end()) {
-    data_format = op_features.attr().at("data_format").s();
+  if (op_info.attr().find("data_format") != op_info.attr().end()) {
+    data_format = op_info.attr().at("data_format").s();
   }
   return data_format;
 }
 
-string GetFilterFormat(const OpInfo& op_features) {
+string GetFilterFormat(const OpInfo& op_info) {
   string filter_format = "HWIO";  // Default format.
-  if (op_features.attr().find("filter_format") != op_features.attr().end()) {
-    filter_format = op_features.attr().at("filter_format").s();
+  if (op_info.attr().find("filter_format") != op_info.attr().end()) {
+    filter_format = op_info.attr().at("filter_format").s();
   }
   return filter_format;
 }
 
-Padding GetPadding(const OpInfo& op_features) {
-  if (op_features.attr().find("padding") != op_features.attr().end() &&
-      op_features.attr().at("padding").s() == "VALID") {
+Padding GetPadding(const OpInfo& op_info) {
+  if (op_info.attr().find("padding") != op_info.attr().end() &&
+      op_info.attr().at("padding").s() == "VALID") {
     return Padding::VALID;
   }
   return Padding::SAME;  // Default padding.
@@ -107,11 +107,11 @@ bool IsTraining(const OpInfo& op_info) {
 // TODO(dyoon): support non-4D tensors in the c ost functions of convolution
 // related ops (Conv, Pool, BatchNorm, and their backprops) and the related
 // helper functions.
-std::vector<int64> GetStrides(const OpInfo& op_features) {
-  if (op_features.attr().find("strides") != op_features.attr().end()) {
-    const auto strides = op_features.attr().at("strides").list().i();
-    CHECK(strides.size() == 4) << "Attr strides is not a length-4 vector: "
-                               << op_features.DebugString();
+std::vector<int64> GetStrides(const OpInfo& op_info) {
+  if (op_info.attr().find("strides") != op_info.attr().end()) {
+    const auto strides = op_info.attr().at("strides").list().i();
+    CHECK(strides.size() == 4)
+        << "Attr strides is not a length-4 vector: " << op_info.DebugString();
     return {strides[0], strides[1], strides[2], strides[3]};
   }
   return {1, 1, 1, 1};
@@ -359,21 +359,21 @@ OpLevelCostEstimator::OpLevelCostEstimator() {
 }
 
 Costs OpLevelCostEstimator::PredictCosts(const OpContext& op_context) const {
-  const auto& op_features = op_context.op_info;
-  auto it = device_cost_impl_.find(op_features.op());
+  const auto& op_info = op_context.op_info;
+  auto it = device_cost_impl_.find(op_info.op());
   if (it == device_cost_impl_.end()) {
-    if (elementwise_ops_.find(op_features.op()) != elementwise_ops_.end()) {
+    if (elementwise_ops_.find(op_info.op()) != elementwise_ops_.end()) {
       return PredictCwiseOp(op_context);
     }
 
-    VLOG(1) << "Missing accurate estimator for op: " << op_features.op();
+    VLOG(1) << "Missing accurate estimator for op: " << op_info.op();
 
     return PredictCostOfAnUnknownOp(op_context);
   }
 
   std::function<Costs(const OpContext&)> estimator = it->second;
   Costs costs = estimator(op_context);
-  VLOG(1) << "Operation " << op_features.op() << " takes "
+  VLOG(1) << "Operation " << op_info.op() << " takes "
           << costs.execution_time.count() << " ns.";
   return costs;
 }
@@ -430,39 +430,38 @@ DeviceInfo OpLevelCostEstimator::GetDeviceInfo(
 }
 
 Costs OpLevelCostEstimator::PredictCwiseOp(const OpContext& op_context) const {
-  const auto& op_features = op_context.op_info;
+  const auto& op_info = op_context.op_info;
   bool found_unknown_shapes = false;
   // For unary or binary element-wise operations, op count is the element count
   // of any input. We use the count for the largest input here to be more robust
   // in case that the shape is unknown or partially known for other input.
-  int64 op_count =
-      CalculateLargestInputCount(op_features, &found_unknown_shapes);
+  int64 op_count = CalculateLargestInputCount(op_info, &found_unknown_shapes);
   // If output shape is available, try use the element count calcuated from
   // that.
-  if (op_features.outputs_size() > 0) {
-    op_count =
-        std::max(op_count, CalculateTensorElementCount(op_features.outputs(0),
-                                                       &found_unknown_shapes));
+  if (op_info.outputs_size() > 0) {
+    op_count = std::max(
+        op_count,
+        CalculateTensorElementCount(op_info.outputs(0), &found_unknown_shapes));
   }
   // For binary ops, calculate the output shape possibly resulting from
   // broadcasting.
-  if (op_features.inputs_size() >= 2) {
-    op_count = std::max(op_count,
-                        CwiseOutputElementCount(op_features.inputs(0).shape(),
-                                                op_features.inputs(1).shape()));
+  if (op_info.inputs_size() >= 2) {
+    op_count =
+        std::max(op_count, CwiseOutputElementCount(op_info.inputs(0).shape(),
+                                                   op_info.inputs(1).shape()));
   }
 
   int op_cost = 1;
   bool is_known_elementwise_op = false;
-  auto it = elementwise_ops_.find(op_features.op());
+  auto it = elementwise_ops_.find(op_info.op());
   if (it != elementwise_ops_.end()) {
     op_cost = it->second;
     is_known_elementwise_op = true;
   } else {
-    LOG(WARNING) << "Not a cwise op: " << op_features.op();
+    LOG(WARNING) << "Not a cwise op: " << op_info.op();
   }
 
-  Costs costs = PredictOpCountBasedCost(op_count * op_cost, op_features);
+  Costs costs = PredictOpCountBasedCost(op_count * op_cost, op_info);
   if (found_unknown_shapes || !is_known_elementwise_op) {
     costs.inaccurate = true;
   }
@@ -542,17 +541,17 @@ Costs OpLevelCostEstimator::PredictOpCountBasedCost(
 }
 
 int64 OpLevelCostEstimator::CountConv2DOperations(
-    const OpInfo& op_features, bool* found_unknown_shapes) const {
-  return CountConv2DOperations(op_features, nullptr, found_unknown_shapes);
+    const OpInfo& op_info, bool* found_unknown_shapes) const {
+  return CountConv2DOperations(op_info, nullptr, found_unknown_shapes);
 }
 
 // Helper to translate the positional arguments into named fields.
 OpLevelCostEstimator::ConvolutionDimensions
 OpLevelCostEstimator::ConvolutionDimensionsFromInputs(
     const TensorShapeProto& original_image_shape,
-    const TensorShapeProto& original_filter_shape, const OpInfo& op_features,
+    const TensorShapeProto& original_filter_shape, const OpInfo& op_info,
     bool* found_unknown_shapes) {
-  VLOG(2) << "op features: " << op_features.DebugString();
+  VLOG(2) << "op features: " << op_info.DebugString();
   VLOG(2) << "Original image shape: " << original_image_shape.DebugString();
   VLOG(2) << "Original filter shape: " << original_filter_shape.DebugString();
   auto image_shape =
@@ -563,7 +562,7 @@ OpLevelCostEstimator::ConvolutionDimensionsFromInputs(
   VLOG(2) << "Filter shape: " << filter_shape.DebugString();
 
   int x_index, y_index, channel_index;
-  const string& data_format = GetDataFormat(op_features);
+  const string& data_format = GetDataFormat(op_info);
   if (data_format == "NCHW") {
     x_index = 2;
     y_index = 3;
@@ -574,7 +573,7 @@ OpLevelCostEstimator::ConvolutionDimensionsFromInputs(
     y_index = 2;
     channel_index = 3;
   }
-  const string& filter_format = GetFilterFormat(op_features);
+  const string& filter_format = GetFilterFormat(op_info);
   int filter_x_index, filter_y_index, in_channel_index, out_channel_index;
   if (filter_format == "HWIO") {
     filter_x_index = 0;
@@ -594,8 +593,8 @@ OpLevelCostEstimator::ConvolutionDimensionsFromInputs(
   int64 iz = image_shape.dim(channel_index).size();
   int64 kx = filter_shape.dim(filter_x_index).size();
   int64 ky = filter_shape.dim(filter_y_index).size();
-  std::vector<int64> strides = GetStrides(op_features);
-  const auto padding = GetPadding(op_features);
+  std::vector<int64> strides = GetStrides(op_info);
+  const auto padding = GetPadding(op_info);
   int64 sx = strides[x_index];
   int64 sy = strides[y_index];
   int64 ox = GetOutputSize(ix, kx, sx, padding);
@@ -623,14 +622,13 @@ OpLevelCostEstimator::ConvolutionDimensionsFromInputs(
 }
 
 int64 OpLevelCostEstimator::CountConv2DOperations(
-    const OpInfo& op_features, ConvolutionDimensions* conv_info,
+    const OpInfo& op_info, ConvolutionDimensions* conv_info,
     bool* found_unknown_shapes) const {
-  DCHECK(op_features.op() == kConv2d ||
-         op_features.op() == kDepthwiseConv2dNative)
+  DCHECK(op_info.op() == kConv2d || op_info.op() == kDepthwiseConv2dNative)
       << "Invalid Operation: not Conv2D nor DepthwiseConv2dNative";
 
   ConvolutionDimensions conv_dims = ConvolutionDimensionsFromInputs(
-      op_features.inputs(0).shape(), op_features.inputs(1).shape(), op_features,
+      op_info.inputs(0).shape(), op_info.inputs(1).shape(), op_info,
       found_unknown_shapes);
 
   //  in DepthwiseConv2dNative conv_dims.oz is actually the channel depth
@@ -641,7 +639,7 @@ int64 OpLevelCostEstimator::CountConv2DOperations(
   int64 ops = conv_dims.batch;
   ops *= conv_dims.ox * conv_dims.oy;
   ops *= conv_dims.kx * conv_dims.ky;
-  if (op_features.op() == kConv2d) {
+  if (op_info.op() == kConv2d) {
     ops *= conv_dims.iz * conv_dims.oz;
   } else {
     // To ensure output tensor dims to be correct for DepthwiseConv2DNative,
@@ -658,32 +656,32 @@ int64 OpLevelCostEstimator::CountConv2DOperations(
 }
 
 int64 OpLevelCostEstimator::CountMatMulOperations(
-    const OpInfo& op_features, bool* found_unknown_shapes) const {
-  return CountMatMulOperations(op_features, nullptr, found_unknown_shapes);
+    const OpInfo& op_info, bool* found_unknown_shapes) const {
+  return CountMatMulOperations(op_info, nullptr, found_unknown_shapes);
 }
 
 // TODO(nishantpatil): Create separate estimator for Sparse Matmul
 int64 OpLevelCostEstimator::CountMatMulOperations(
-    const OpInfo& op_features, MatMulDimensions* mat_mul,
+    const OpInfo& op_info, MatMulDimensions* mat_mul,
     bool* found_unknown_shapes) const {
   double ops = 0;
 
-  if (op_features.inputs_size() < 2) {
-    LOG(ERROR) << "Need 2 inputs but got " << op_features.inputs_size();
+  if (op_info.inputs_size() < 2) {
+    LOG(ERROR) << "Need 2 inputs but got " << op_info.inputs_size();
     // TODO(pcma): Try to separate invalid inputs from unknown shapes
     *found_unknown_shapes = true;
     return 0;
   }
 
-  auto& a_matrix = op_features.inputs(0);
-  auto& b_matrix = op_features.inputs(1);
+  auto& a_matrix = op_info.inputs(0);
+  auto& b_matrix = op_info.inputs(1);
 
   bool transpose_a = false;
   bool transpose_b = false;
 
   double m_dim, n_dim, k_dim, k_dim_b = 0;
 
-  for (const auto& item : op_features.attr()) {
+  for (const auto& item : op_info.attr()) {
     VLOG(1) << "Key:" << item.first
             << " Value:" << SummarizeAttrValue(item.second);
     if (item.first == "transpose_a" && item.second.b() == true)
@@ -735,23 +733,23 @@ int64 OpLevelCostEstimator::CountMatMulOperations(
 }
 
 int64 OpLevelCostEstimator::CountBatchMatMulOperations(
-    const OpInfo& op_features, bool* found_unknown_shapes) const {
-  if (op_features.op() != kBatchMatMul) {
-    LOG(ERROR) << "Invalid Operation: " << op_features.op();
+    const OpInfo& op_info, bool* found_unknown_shapes) const {
+  if (op_info.op() != kBatchMatMul) {
+    LOG(ERROR) << "Invalid Operation: " << op_info.op();
     // TODO(pcma): Try to separate invalid inputs from unknown shapes
     *found_unknown_shapes = true;
     return 0;
   }
-  if (op_features.inputs_size() != 2) {
-    LOG(ERROR) << "Expected 2 inputs but got " << op_features.inputs_size();
+  if (op_info.inputs_size() != 2) {
+    LOG(ERROR) << "Expected 2 inputs but got " << op_info.inputs_size();
     // TODO(pcma): Try to separate invalid inputs from unknown shapes
     *found_unknown_shapes = true;
     return 0;
   }
 
   double ops = 0;
-  const auto& a_input = op_features.inputs(0);
-  const auto& b_input = op_features.inputs(1);
+  const auto& a_input = op_info.inputs(0);
+  const auto& b_input = op_info.inputs(1);
 
   // BatchMatMul requires inputs of at least matrix shape (rank 2).
   // The two most minor dimensions of each input are matrices that
@@ -801,24 +799,24 @@ int64 OpLevelCostEstimator::CountBatchMatMulOperations(
 
   // Build the MatMul. Note that values are ignored here since we are just
   // counting ops (e.g. only shapes matter).
-  OpInfo matmul_op_features;
-  matmul_op_features.set_op("MatMul");
+  OpInfo matmul_op_info;
+  matmul_op_info.set_op("MatMul");
 
   AttrValue transpose_a;
   transpose_a.set_b(false);
-  if (op_features.attr().find("adj_x") != op_features.attr().end()) {
-    transpose_a.set_b(op_features.attr().at("adj_x").b());
+  if (op_info.attr().find("adj_x") != op_info.attr().end()) {
+    transpose_a.set_b(op_info.attr().at("adj_x").b());
   }
-  (*matmul_op_features.mutable_attr())["transpose_a"] = transpose_a;
+  (*matmul_op_info.mutable_attr())["transpose_a"] = transpose_a;
 
   AttrValue transpose_b;
   transpose_b.set_b(false);
-  if (op_features.attr().find("adj_y") != op_features.attr().end()) {
-    transpose_b.set_b(op_features.attr().at("adj_y").b());
+  if (op_info.attr().find("adj_y") != op_info.attr().end()) {
+    transpose_b.set_b(op_info.attr().at("adj_y").b());
   }
-  (*matmul_op_features.mutable_attr())["transpose_b"] = transpose_b;
+  (*matmul_op_info.mutable_attr())["transpose_b"] = transpose_b;
 
-  OpInfo::TensorProperties* a_matrix = matmul_op_features.add_inputs();
+  OpInfo::TensorProperties* a_matrix = matmul_op_info.add_inputs();
   a_matrix->set_dtype(a_input.dtype());
   TensorShapeProto* a_matrix_shape = a_matrix->mutable_shape();
   for (int i = std::max(0, a_input_shape.dim_size() - matrix_rank);
@@ -826,7 +824,7 @@ int64 OpLevelCostEstimator::CountBatchMatMulOperations(
     *(a_matrix_shape->add_dim()) = a_input_shape.dim(i);
   }
 
-  OpInfo::TensorProperties* b_matrix = matmul_op_features.add_inputs();
+  OpInfo::TensorProperties* b_matrix = matmul_op_info.add_inputs();
   b_matrix->set_dtype(b_input.dtype());
   TensorShapeProto* b_matrix_shape = b_matrix->mutable_shape();
   for (int i = std::max(0, b_input_shape.dim_size() - matrix_rank);
@@ -836,7 +834,7 @@ int64 OpLevelCostEstimator::CountBatchMatMulOperations(
 
   for (int i = 0; i < num_matmuls; ++i) {
     bool matmul_unknown_shapes = false;
-    ops += CountMatMulOperations(matmul_op_features, &matmul_unknown_shapes);
+    ops += CountMatMulOperations(matmul_op_info, &matmul_unknown_shapes);
     *found_unknown_shapes |= matmul_unknown_shapes;
   }
   return ops;
@@ -894,16 +892,16 @@ bool GetTensorShapeProtoFromTensorProto(const TensorProto& tensor_proto,
 
 // TODO(cliffy): Dedup this method and CountConv2DBackpropFilterOperations.
 int64 OpLevelCostEstimator::CountConv2DBackpropInputOperations(
-    const OpInfo& op_features, ConvolutionDimensions* returned_conv_dims,
+    const OpInfo& op_info, ConvolutionDimensions* returned_conv_dims,
     bool* found_unknown_shapes) const {
   int64 ops = 0;
 
-  DCHECK(op_features.op() == kConv2dBackpropInput ||
-         op_features.op() == kDepthwiseConv2dNativeBackpropInput)
+  DCHECK(op_info.op() == kConv2dBackpropInput ||
+         op_info.op() == kDepthwiseConv2dNativeBackpropInput)
       << "Invalid Operation: not kConv2dBackpropInput nor"
          "kDepthwiseConv2dNativeBackpropInput";
 
-  if (op_features.inputs_size() < 2) {
+  if (op_info.inputs_size() < 2) {
     // TODO(pcma): Try to separate invalid inputs from unknown shapes
     *found_unknown_shapes = true;
     return ops;
@@ -911,12 +909,12 @@ int64 OpLevelCostEstimator::CountConv2DBackpropInputOperations(
 
   TensorShapeProto input_shape;
   bool shape_found = false;
-  if (op_features.inputs(0).has_value()) {
-    const TensorProto& value = op_features.inputs(0).value();
+  if (op_info.inputs(0).has_value()) {
+    const TensorProto& value = op_info.inputs(0).value();
     shape_found = GetTensorShapeProtoFromTensorProto(value, &input_shape);
   }
-  if (!shape_found && op_features.outputs_size() == 1) {
-    input_shape = op_features.outputs(0).shape();
+  if (!shape_found && op_info.outputs_size() == 1) {
+    input_shape = op_info.outputs(0).shape();
     shape_found = true;
   }
   if (!shape_found) {
@@ -929,13 +927,12 @@ int64 OpLevelCostEstimator::CountConv2DBackpropInputOperations(
   }
 
   ConvolutionDimensions conv_dims = ConvolutionDimensionsFromInputs(
-      input_shape, op_features.inputs(1).shape(), op_features,
-      found_unknown_shapes);
+      input_shape, op_info.inputs(1).shape(), op_info, found_unknown_shapes);
 
   ops = conv_dims.batch;
   ops *= conv_dims.ox * conv_dims.oy;
   ops *= conv_dims.kx * conv_dims.ky;
-  if (op_features.op() == kConv2dBackpropInput) {
+  if (op_info.op() == kConv2dBackpropInput) {
     ops *= conv_dims.iz * conv_dims.oz;
   } else {
     // conv_dims always use forward path definition regardless
@@ -944,7 +941,7 @@ int64 OpLevelCostEstimator::CountConv2DBackpropInputOperations(
   }
   ops *= kOpsPerMac;
 
-  VLOG(1) << "Operations for" << op_features.op() << "  " << ops;
+  VLOG(1) << "Operations for" << op_info.op() << "  " << ops;
 
   if (returned_conv_dims != nullptr) {
     *returned_conv_dims = conv_dims;
@@ -953,23 +950,23 @@ int64 OpLevelCostEstimator::CountConv2DBackpropInputOperations(
 }
 
 int64 OpLevelCostEstimator::CountConv2DBackpropFilterOperations(
-    const OpInfo& op_features, ConvolutionDimensions* returned_conv_dims,
+    const OpInfo& op_info, ConvolutionDimensions* returned_conv_dims,
     bool* found_unknown_shapes) const {
   int64 ops = 0;
 
-  DCHECK(op_features.op() == kConv2dBackpropFilter ||
-         op_features.op() == kDepthwiseConv2dNativeBackpropFilter)
+  DCHECK(op_info.op() == kConv2dBackpropFilter ||
+         op_info.op() == kDepthwiseConv2dNativeBackpropFilter)
       << "Invalid Operation: not kConv2dBackpropFilter nor"
          "kDepthwiseConv2dNativeBackpropFilter";
 
   TensorShapeProto filter_shape;
   bool shape_found = false;
-  if (op_features.inputs_size() >= 2 && op_features.inputs(1).has_value()) {
-    const TensorProto& value = op_features.inputs(1).value();
+  if (op_info.inputs_size() >= 2 && op_info.inputs(1).has_value()) {
+    const TensorProto& value = op_info.inputs(1).value();
     shape_found = GetTensorShapeProtoFromTensorProto(value, &filter_shape);
   }
-  if (!shape_found && op_features.outputs_size() == 1) {
-    filter_shape = op_features.outputs(0).shape();
+  if (!shape_found && op_info.outputs_size() == 1) {
+    filter_shape = op_info.outputs(0).shape();
     shape_found = true;
   }
   if (!shape_found) {
@@ -981,19 +978,18 @@ int64 OpLevelCostEstimator::CountConv2DBackpropFilterOperations(
     *found_unknown_shapes = true;
   }
 
-  if (op_features.inputs_size() < 1) {
+  if (op_info.inputs_size() < 1) {
     // TODO(pcma): Try to separate invalid inputs from unknown shapes
     *found_unknown_shapes = true;
     return ops;
   }
   ConvolutionDimensions conv_dims = ConvolutionDimensionsFromInputs(
-      op_features.inputs(0).shape(), filter_shape, op_features,
-      found_unknown_shapes);
+      op_info.inputs(0).shape(), filter_shape, op_info, found_unknown_shapes);
 
   ops = conv_dims.batch;
   ops *= conv_dims.ox * conv_dims.oy;
   ops *= conv_dims.kx * conv_dims.ky;
-  if (op_features.op() == kConv2dBackpropFilter) {
+  if (op_info.op() == kConv2dBackpropFilter) {
     ops *= conv_dims.iz * conv_dims.oz;
   } else {
     // conv_dims always use forward path definition regardless
@@ -1001,7 +997,7 @@ int64 OpLevelCostEstimator::CountConv2DBackpropFilterOperations(
     ops *= conv_dims.oz;
   }
   ops *= kOpsPerMac;
-  VLOG(1) << "Operations for" << op_features.op() << "  " << ops;
+  VLOG(1) << "Operations for" << op_info.op() << "  " << ops;
 
   if (returned_conv_dims != nullptr) {
     *returned_conv_dims = conv_dims;
@@ -1032,9 +1028,9 @@ int64 OpLevelCostEstimator::CalculateTensorSize(
 }
 
 int64 OpLevelCostEstimator::CalculateInputSize(
-    const OpInfo& op_features, bool* found_unknown_shapes) const {
+    const OpInfo& op_info, bool* found_unknown_shapes) const {
   int64 total_input_size = 0;
-  for (auto& input : op_features.inputs()) {
+  for (auto& input : op_info.inputs()) {
     int64 input_size = CalculateTensorSize(input, found_unknown_shapes);
     total_input_size += input_size;
     VLOG(1) << "Input Size: " << input_size
@@ -1044,9 +1040,9 @@ int64 OpLevelCostEstimator::CalculateInputSize(
 }
 
 int64 OpLevelCostEstimator::CalculateLargestInputCount(
-    const OpInfo& op_features, bool* found_unknown_shapes) const {
+    const OpInfo& op_info, bool* found_unknown_shapes) const {
   int64 largest_input_count = 0;
-  for (auto& input : op_features.inputs()) {
+  for (auto& input : op_info.inputs()) {
     int64 input_count =
         CalculateTensorElementCount(input, found_unknown_shapes);
     if (input_count > largest_input_count) {
@@ -1059,10 +1055,10 @@ int64 OpLevelCostEstimator::CalculateLargestInputCount(
 }
 
 int64 OpLevelCostEstimator::CalculateOutputSize(
-    const OpInfo& op_features, bool* found_unknown_shapes) const {
+    const OpInfo& op_info, bool* found_unknown_shapes) const {
   int64 total_output_size = 0;
   // use float as default for calculations
-  for (const auto& output : op_features.outputs()) {
+  for (const auto& output : op_info.outputs()) {
     DataType dt = output.dtype();
     const auto& original_output_shape = output.shape();
     int64 output_size = DataTypeSize(BaseType(dt));
@@ -1080,10 +1076,10 @@ int64 OpLevelCostEstimator::CalculateOutputSize(
 }
 
 Costs OpLevelCostEstimator::PredictConv2D(const OpContext& op_context) const {
-  const auto& op_features = op_context.op_info;
+  const auto& op_info = op_context.op_info;
   bool found_unknown_shapes = false;
   auto costs = PredictOpCountBasedCost(
-      CountConv2DOperations(op_features, &found_unknown_shapes), op_features);
+      CountConv2DOperations(op_info, &found_unknown_shapes), op_info);
   costs.inaccurate = found_unknown_shapes;
   costs.num_ops_with_unknown_shapes = found_unknown_shapes;
   return costs;
@@ -1091,12 +1087,12 @@ Costs OpLevelCostEstimator::PredictConv2D(const OpContext& op_context) const {
 
 Costs OpLevelCostEstimator::PredictConv2DBackpropInput(
     const OpContext& op_context) const {
-  const auto& op_features = op_context.op_info;
+  const auto& op_info = op_context.op_info;
   bool found_unknown_shapes = false;
   auto costs =
       PredictOpCountBasedCost(CountConv2DBackpropInputOperations(
-                                  op_features, nullptr, &found_unknown_shapes),
-                              op_features);
+                                  op_info, nullptr, &found_unknown_shapes),
+                              op_info);
   costs.inaccurate = found_unknown_shapes;
   costs.num_ops_with_unknown_shapes = found_unknown_shapes;
   return costs;
@@ -1104,12 +1100,12 @@ Costs OpLevelCostEstimator::PredictConv2DBackpropInput(
 
 Costs OpLevelCostEstimator::PredictConv2DBackpropFilter(
     const OpContext& op_context) const {
-  const auto& op_features = op_context.op_info;
+  const auto& op_info = op_context.op_info;
   bool found_unknown_shapes = false;
   auto costs =
       PredictOpCountBasedCost(CountConv2DBackpropFilterOperations(
-                                  op_features, nullptr, &found_unknown_shapes),
-                              op_features);
+                                  op_info, nullptr, &found_unknown_shapes),
+                              op_info);
   costs.inaccurate = found_unknown_shapes;
   costs.num_ops_with_unknown_shapes = found_unknown_shapes;
   return costs;
@@ -1204,26 +1200,26 @@ Costs OpLevelCostEstimator::PredictFusedConv2DBiasActivation(
 }
 
 Costs OpLevelCostEstimator::PredictMatMul(const OpContext& op_context) const {
-  const auto& op_features = op_context.op_info;
+  const auto& op_info = op_context.op_info;
   bool found_unknown_shapes = false;
   auto costs = PredictOpCountBasedCost(
-      CountMatMulOperations(op_features, &found_unknown_shapes), op_features);
+      CountMatMulOperations(op_info, &found_unknown_shapes), op_info);
   costs.inaccurate = found_unknown_shapes;
   costs.num_ops_with_unknown_shapes = found_unknown_shapes;
   return costs;
 }
 
 Costs OpLevelCostEstimator::PredictNoOp(const OpContext& op_context) const {
-  const auto& op_features = op_context.op_info;
-  VLOG(1) << "Op:" << op_features.op() << " Execution Time 0 (ns)";
+  const auto& op_info = op_context.op_info;
+  VLOG(1) << "Op:" << op_info.op() << " Execution Time 0 (ns)";
   return Costs::ZeroCosts();
 }
 
 Costs OpLevelCostEstimator::PredictIdentity(const OpContext& op_context) const {
-  const auto& op_features = op_context.op_info;
-  VLOG(1) << "Op:" << op_features.op() << " Execution Time 0 (ns)";
+  const auto& op_info = op_context.op_info;
+  VLOG(1) << "Op:" << op_info.op() << " Execution Time 0 (ns)";
   Costs result = Costs::ZeroCosts();
-  result.max_memory = CalculateOutputSize(op_features, &result.inaccurate);
+  result.max_memory = CalculateOutputSize(op_info, &result.inaccurate);
   result.num_ops_with_unknown_shapes = result.inaccurate;
   // Assign the minimum amount of time we can represent to the identity op since
   // it tends to be really cheap.
@@ -1233,11 +1229,10 @@ Costs OpLevelCostEstimator::PredictIdentity(const OpContext& op_context) const {
 }
 
 Costs OpLevelCostEstimator::PredictVariable(const OpContext& op_context) const {
-  const auto& op_features = op_context.op_info;
-  VLOG(1) << "Op:" << op_features.op() << " Execution Time 0 (ns)";
+  const auto& op_info = op_context.op_info;
+  VLOG(1) << "Op:" << op_info.op() << " Execution Time 0 (ns)";
   Costs result = Costs::ZeroCosts();
-  result.persistent_memory =
-      CalculateOutputSize(op_features, &result.inaccurate);
+  result.persistent_memory = CalculateOutputSize(op_info, &result.inaccurate);
   result.num_ops_with_unknown_shapes = result.inaccurate;
 
   result.compute_time = kMinComputeTime;
@@ -1247,20 +1242,19 @@ Costs OpLevelCostEstimator::PredictVariable(const OpContext& op_context) const {
 
 Costs OpLevelCostEstimator::PredictBatchMatMul(
     const OpContext& op_context) const {
-  const auto& op_features = op_context.op_info;
+  const auto& op_info = op_context.op_info;
   bool found_unknown_shapes = false;
   Costs costs = PredictOpCountBasedCost(
-      CountBatchMatMulOperations(op_features, &found_unknown_shapes),
-      op_features);
+      CountBatchMatMulOperations(op_info, &found_unknown_shapes), op_info);
   costs.inaccurate = found_unknown_shapes;
   costs.num_ops_with_unknown_shapes = found_unknown_shapes;
   return costs;
 }
 
 Costs OpLevelCostEstimator::PredictMetadata(const OpContext& op_context) const {
-  const auto& op_features = op_context.op_info;
+  const auto& op_info = op_context.op_info;
   Costs costs = Costs::ZeroCosts();
-  costs.max_memory = CalculateOutputSize(op_features, &costs.inaccurate);
+  costs.max_memory = CalculateOutputSize(op_info, &costs.inaccurate);
   costs.num_ops_with_unknown_shapes = costs.inaccurate;
   // Metadata operations are so cheap we assume they take the minimum amount of
   // time we can represent (1 ns).
diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator.h b/tensorflow/core/grappler/costs/op_level_cost_estimator.h
index 84dd9213f7..f8ba8c6637 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator.h
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator.h
@@ -16,10 +16,6 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_GRAPPLER_COSTS_OP_LEVEL_COST_ESTIMATOR_H_
 #define TENSORFLOW_CORE_GRAPPLER_COSTS_OP_LEVEL_COST_ESTIMATOR_H_
 
-#include <functional>
-#include <map>
-#include <string>
-
 #include "tensorflow/core/grappler/costs/cost_estimator.h"
 #include "tensorflow/core/grappler/costs/op_context.h"
 #include "tensorflow/core/grappler/costs/op_performance_data.pb.h"
@@ -79,24 +75,23 @@ class OpLevelCostEstimator {
     int64 sy;         // Stride y.
     Padding padding;  // SAME or VALID.
   };
-  int64 CountConv2DOperations(const OpInfo& op_features,
+  int64 CountConv2DOperations(const OpInfo& op_info,
                               bool* found_unknown_shapes) const;
-  int64 CountConv2DOperations(const OpInfo& op_features,
+  int64 CountConv2DOperations(const OpInfo& op_info,
                               ConvolutionDimensions* conv_info,
                               bool* found_unknown_shapes) const;
-  int64 CountMatMulOperations(const OpInfo& op_features,
+  int64 CountMatMulOperations(const OpInfo& op_info,
                               bool* found_unknown_shapes) const;
-  int64 CountMatMulOperations(const OpInfo& op_features,
-                              MatMulDimensions* mat_mul,
+  int64 CountMatMulOperations(const OpInfo& op_info, MatMulDimensions* mat_mul,
                               bool* found_unknown_shapes) const;
-  int64 CountBatchMatMulOperations(const OpInfo& op_features,
+  int64 CountBatchMatMulOperations(const OpInfo& op_info,
                                    bool* found_unknown_shapes) const;
-  int64 CountConv2DBackpropInputOperations(const OpInfo& op_features,
-                                           ConvolutionDimensions* conv_info,
-                                           bool* found_unknown_shapes) const;
-  int64 CountConv2DBackpropFilterOperations(const OpInfo& op_features,
-                                            ConvolutionDimensions* conv_info,
-                                            bool* found_unknown_shapes) const;
+  int64 CountConv2DBackpropInputOperations(
+      const OpInfo& op_info, ConvolutionDimensions* returned_conv_dims,
+      bool* found_unknown_shapes) const;
+  int64 CountConv2DBackpropFilterOperations(
+      const OpInfo& op_info, ConvolutionDimensions* returned_conv_dims,
+      bool* found_unknown_shapes) const;
 
   // Calculate the element count of an input/output tensor.
   int64 CalculateTensorElementCount(const OpInfo::TensorProperties& tensor,
@@ -108,17 +103,17 @@ class OpLevelCostEstimator {
 
   // Calculate the element count of the largest
   // input of specified TensorFlow op.
-  int64 CalculateLargestInputCount(const OpInfo& op_features,
+  int64 CalculateLargestInputCount(const OpInfo& op_info,
                                    bool* found_unknown_shapes) const;
 
   // Calculate the total size in bytes of the all
   // the inputs of specified TensorFlow op.
-  int64 CalculateInputSize(const OpInfo& op_features,
+  int64 CalculateInputSize(const OpInfo& op_info,
                            bool* found_unknown_shapes) const;
 
   // Calculate the total size in bytes of the all
   // the outputs of specified TensorFlow op.
-  int64 CalculateOutputSize(const OpInfo& op_features,
+  int64 CalculateOutputSize(const OpInfo& op_info,
                             bool* found_unknown_shapes) const;
 
   // This family of routines predicts the costs to
@@ -205,4 +200,5 @@ class OpLevelCostEstimator {
 
 }  // end namespace grappler
 }  // end namespace tensorflow
+
 #endif  // TENSORFLOW_CORE_GRAPPLER_COSTS_OP_LEVEL_COST_ESTIMATOR_H_
diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
index 9a59877ac5..6a9bf13b93 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
@@ -29,8 +29,8 @@ namespace grappler {
 
 namespace {
 // Wrangles the minimum number of proto fields to set up a matrix.
-void DescribeMatrix(int rows, int columns, OpInfo* op_features) {
-  auto input = op_features->add_inputs();
+void DescribeMatrix(int rows, int columns, OpInfo* op_info) {
+  auto input = op_info->add_inputs();
   auto shape = input->mutable_shape();
   auto shape_rows = shape->add_dim();
   shape_rows->set_size(rows);
@@ -39,8 +39,8 @@ void DescribeMatrix(int rows, int columns, OpInfo* op_features) {
   input->set_dtype(DT_FLOAT);
 }
 
-void SetCpuDevice(OpInfo* op_features) {
-  auto device = op_features->mutable_device();
+void SetCpuDevice(OpInfo* op_info) {
+  auto device = op_info->mutable_device();
   device->set_type("CPU");
   device->set_num_cores(10);
   device->set_bandwidth(10000000);  // 10000000 KB/s = 10 GB/s
@@ -413,15 +413,14 @@ class OpLevelCostEstimatorTest : public ::testing::Test {
     return estimator_.PredictCosts(op_context);
   }
 
-  int64 CountMatMulOperations(const OpInfo& op_features,
+  int64 CountMatMulOperations(const OpInfo& op_info,
                               bool* found_unknown_shapes) const {
-    return estimator_.CountMatMulOperations(op_features, found_unknown_shapes);
+    return estimator_.CountMatMulOperations(op_info, found_unknown_shapes);
   }
 
-  int64 CountBatchMatMulOperations(const OpInfo& op_features,
+  int64 CountBatchMatMulOperations(const OpInfo& op_info,
                                    bool* found_unknown_shapes) const {
-    return estimator_.CountBatchMatMulOperations(op_features,
-                                                 found_unknown_shapes);
+    return estimator_.CountBatchMatMulOperations(op_info, found_unknown_shapes);
   }
 
   void SetComputeMemoryOverlap(bool value) {
-- 
GitLab


From 7cadc29b011e2dd73f70034f280a13931c73900e Mon Sep 17 00:00:00 2001
From: Doe Hyun Yoon <dyoon@google.com>
Date: Mon, 7 Jan 2019 16:10:33 -0800
Subject: [PATCH 0300/2345] Add PredictCostsAndReturnRunMetadata method to
 CostEstimator.

PiperOrigin-RevId: 228254629
---
 tensorflow/core/grappler/costs/BUILD          |  1 +
 .../costs/analytical_cost_estimator.cc        | 37 +++++++++++--------
 .../costs/analytical_cost_estimator.h         | 14 +++----
 .../core/grappler/costs/cost_estimator.h      | 14 +++++++
 4 files changed, 42 insertions(+), 24 deletions(-)

diff --git a/tensorflow/core/grappler/costs/BUILD b/tensorflow/core/grappler/costs/BUILD
index 15dc7074b9..92294379b5 100644
--- a/tensorflow/core/grappler/costs/BUILD
+++ b/tensorflow/core/grappler/costs/BUILD
@@ -171,6 +171,7 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
     ],
 )
 
diff --git a/tensorflow/core/grappler/costs/analytical_cost_estimator.cc b/tensorflow/core/grappler/costs/analytical_cost_estimator.cc
index b7804ffaa5..eac6a0de3f 100644
--- a/tensorflow/core/grappler/costs/analytical_cost_estimator.cc
+++ b/tensorflow/core/grappler/costs/analytical_cost_estimator.cc
@@ -104,17 +104,15 @@ AnalyticalCostEstimator::AnalyticalCostEstimator(Cluster* cluster,
                                                  bool use_static_shapes)
     : AnalyticalCostEstimator(
           cluster, absl::make_unique<OpLevelCostEstimator>(),
-          ReadyNodeManagerFactory("FirstReady"), use_static_shapes, nullptr) {}
+          ReadyNodeManagerFactory("FirstReady"), use_static_shapes) {}
 
 AnalyticalCostEstimator::AnalyticalCostEstimator(
     Cluster* cluster, std::unique_ptr<OpLevelCostEstimator> node_estimator,
-    std::unique_ptr<ReadyNodeManager> node_manager, bool use_static_shapes,
-    RunMetadata* run_metadata)
+    std::unique_ptr<ReadyNodeManager> node_manager, bool use_static_shapes)
     : cluster_(cluster),
       node_estimator_(std::move(node_estimator)),
       node_manager_(std::move(node_manager)),
-      use_static_shapes_(use_static_shapes),
-      run_metadata_(run_metadata) {
+      use_static_shapes_(use_static_shapes) {
   scheduler_ = absl::make_unique<VirtualScheduler>(use_static_shapes_, cluster_,
                                                    node_manager_.get());
 }
@@ -128,6 +126,18 @@ Status AnalyticalCostEstimator::Initialize(const GrapplerItem& item) {
 Status AnalyticalCostEstimator::PredictCosts(const GraphDef& optimized_graph,
                                              CostGraphDef* cost_graph,
                                              Costs* costs) const {
+  RunMetadata run_metadata;
+  auto s =
+      PredictCostsAndReturnRunMetadata(optimized_graph, &run_metadata, costs);
+  if (s.ok() && cost_graph) {
+    cost_graph->Swap(run_metadata.mutable_cost_graph());
+  }
+  return s;
+}
+
+Status AnalyticalCostEstimator::PredictCostsAndReturnRunMetadata(
+    const GraphDef& optimized_graph, RunMetadata* run_metadata,
+    Costs* costs) const {
   GrapplerItem item = item_;
   item.graph = optimized_graph;
 
@@ -138,7 +148,9 @@ Status AnalyticalCostEstimator::PredictCosts(const GraphDef& optimized_graph,
   }
 
   gtl::FlatMap<string, CostGraphDef::Node*> name_to_cost_node;
-  if (cost_graph) {
+  CostGraphDef* cost_graph = nullptr;
+  if (run_metadata) {
+    cost_graph = run_metadata->mutable_cost_graph();
     // TODO(pcma): Clear nodes in cost_graph after we make sure we always pass
     // in an empty cost_graph (a non-empty but incomplete cost_graph will cause
     // problems, e.g., no node_id in cost_graph)
@@ -179,18 +191,13 @@ Status AnalyticalCostEstimator::PredictCosts(const GraphDef& optimized_graph,
     }
   }
 
-  *costs = scheduler_->Summary(run_metadata_);
-  // run_metadata_ gets step_stats and parition_graphs from Summary.
-  // Note that cost_graph could already point to the cost_graph field of
-  // run_metadata_, since both are set by the caller.
-  if (run_metadata_ && cost_graph &&
-      run_metadata_->mutable_cost_graph() != cost_graph)
-    *run_metadata_->mutable_cost_graph() = *cost_graph;
+  // run_metadata gets step_stats and partition_graphs from Summary.
+  *costs = scheduler_->Summary(run_metadata);
 
   if (VLOG_IS_ON(1)) {
     bool verbose = VLOG_IS_ON(2);
-    if (run_metadata_) {
-      VLOG(1) << GetStatsStringFromRunMetadata(*run_metadata_, verbose);
+    if (run_metadata) {
+      VLOG(1) << GetStatsStringFromRunMetadata(*run_metadata, verbose);
     } else {
       RunMetadata run_metadata;
       scheduler_->GenerateRunMetadata(&run_metadata);
diff --git a/tensorflow/core/grappler/costs/analytical_cost_estimator.h b/tensorflow/core/grappler/costs/analytical_cost_estimator.h
index 2629672459..ade61f4051 100644
--- a/tensorflow/core/grappler/costs/analytical_cost_estimator.h
+++ b/tensorflow/core/grappler/costs/analytical_cost_estimator.h
@@ -39,16 +39,10 @@ class AnalyticalCostEstimator : public CostEstimator {
  public:
   // Does not take ownership of cluster.
   AnalyticalCostEstimator(Cluster* cluster, bool use_static_shapes);
-  // Does not take ownership of cluster or run_metadata
-  //
-  // When metadata is provided, step_stats and partition_graphs fields will
-  // always be filled during PredictCosts, and the cost_graph field of metadata
-  // will be filled only when cost_graph is not nullptr when invoking
-  // PredictCosts.
   AnalyticalCostEstimator(Cluster* cluster,
                           std::unique_ptr<OpLevelCostEstimator> node_estimator,
                           std::unique_ptr<ReadyNodeManager> node_manager,
-                          bool use_static_shapes, RunMetadata* run_metadata);
+                          bool use_static_shapes);
   ~AnalyticalCostEstimator() override {}
 
   // Initializes the estimator for the specified grappler item.
@@ -61,6 +55,10 @@ class AnalyticalCostEstimator : public CostEstimator {
   Status PredictCosts(const GraphDef& optimized_graph, CostGraphDef* cost_graph,
                       Costs* cost) const override;
 
+  Status PredictCostsAndReturnRunMetadata(const GraphDef& optimized_graph,
+                                          RunMetadata* run_metadata,
+                                          Costs* cost) const override;
+
   const VirtualScheduler* GetScheduler() const { return scheduler_.get(); }
 
  private:
@@ -70,8 +68,6 @@ class AnalyticalCostEstimator : public CostEstimator {
   std::unique_ptr<ReadyNodeManager> node_manager_;
   bool use_static_shapes_;
   std::unique_ptr<VirtualScheduler> scheduler_;
-
-  RunMetadata* run_metadata_;
 };
 
 }  // end namespace grappler
diff --git a/tensorflow/core/grappler/costs/cost_estimator.h b/tensorflow/core/grappler/costs/cost_estimator.h
index e3b3a36b09..725d81a881 100644
--- a/tensorflow/core/grappler/costs/cost_estimator.h
+++ b/tensorflow/core/grappler/costs/cost_estimator.h
@@ -20,6 +20,7 @@ limitations under the License.
 #include <cmath>
 #include <unordered_map>
 #include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/protobuf/config.pb.h"
 
 namespace tensorflow {
 class GraphDef;
@@ -223,6 +224,19 @@ class CostEstimator {
   // not.
   virtual Status PredictCosts(const GraphDef& optimized_graph,
                               CostGraphDef* cost_graph, Costs* cost) const = 0;
+
+  // TODO(dyoon): Delete PredictCosts() with CostGraphDef as RunMetadata is a
+  // superset of CostGraphDef.
+  // Same method, but returns RunMetadata.
+  virtual Status PredictCostsAndReturnRunMetadata(
+      const GraphDef& optimized_graph, RunMetadata* run_metadata,
+      Costs* cost) const {
+    CostGraphDef* cost_graph = nullptr;
+    if (run_metadata) {
+      cost_graph = run_metadata->mutable_cost_graph();
+    }
+    return PredictCosts(optimized_graph, cost_graph, cost);
+  }
 };
 
 }  // end namespace grappler
-- 
GitLab


From acfc65f0110e272abc8bd1abce867f2f534ba263 Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Mon, 7 Jan 2019 16:11:20 -0800
Subject: [PATCH 0301/2345] [TF2XLA] Use dynamic dimension in SizeOp.

This is needed if the number of elements in a tensor can be dynamic.

PiperOrigin-RevId: 228254723
---
 tensorflow/compiler/tf2xla/kernels/shape_op.cc | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/shape_op.cc b/tensorflow/compiler/tf2xla/kernels/shape_op.cc
index 12830816ec..85b0367f73 100644
--- a/tensorflow/compiler/tf2xla/kernels/shape_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/shape_op.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_helpers.h"
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
+#include "tensorflow/compiler/xla/client/lib/constants.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
 #include "tensorflow/core/framework/tensor_shape.h"
@@ -91,14 +92,20 @@ class SizeOp : public XlaOpKernel {
 
   void Compile(XlaOpKernelContext* ctx) override {
     const TensorShape input_shape = ctx->InputShape(0);
-    const int64 size = input_shape.num_elements();
-    OP_REQUIRES(ctx, FastBoundsCheck(size, std::numeric_limits<int32>::max()),
+    OP_REQUIRES(ctx,
+                FastBoundsCheck(input_shape.num_elements(),
+                                std::numeric_limits<int32>::max()),
                 errors::InvalidArgument("Size does not work for tensors > "
                                         "int32 max."));
     Tensor size_constant(DT_INT32, TensorShape({}));
-    size_constant.scalar<int32>()() = static_cast<int32>(size);
-
-    ctx->SetConstantOutput(0, size_constant);
+    const int rank = input_shape.dims();
+    xla::XlaBuilder* builder = ctx->builder();
+    auto size = xla::One(builder, xla::U32);
+    for (int64 i = 0; i < rank; ++i) {
+      size = xla::Mul(size, xla::GetDimensionSize(ctx->Input(0), i));
+    }
+    size = xla::ConvertElementType(size, xla::S32);
+    ctx->SetOutput(0, size);
   }
 };
 
-- 
GitLab


From ec569e692435fb68191553fd7ffa5b633fdf1ef2 Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Mon, 7 Jan 2019 16:20:48 -0800
Subject: [PATCH 0302/2345] [Grappler] Replace NodeMap with MutableGraphView in
 DependencyOptimizer.

PiperOrigin-RevId: 228256183
---
 tensorflow/core/grappler/optimizers/BUILD     |   4 +
 .../optimizers/dependency_optimizer.cc        | 456 +++++++-----------
 .../optimizers/dependency_optimizer.h         |  24 +-
 .../optimizers/dependency_optimizer_test.cc   |   9 +-
 4 files changed, 202 insertions(+), 291 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 7e29cee86a..5a328a91c5 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -303,14 +303,18 @@ cc_library(
         ":constant_folding",
         ":graph_optimizer",
         "//tensorflow/core:framework",
+        "//tensorflow/core:graph",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/grappler:grappler_item",
+        "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/costs:graph_properties",
         "//tensorflow/core/grappler/utils:topological_sort",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/container:flat_hash_set",
     ],
 )
 
diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer.cc b/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
index 7fee3ae9d5..9bceb6b906 100644
--- a/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
@@ -18,10 +18,14 @@ limitations under the License.
 #include <unordered_map>
 #include <unordered_set>
 
+#include "absl/container/flat_hash_map.h"
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/graph/tensor_id.h"
 #include "tensorflow/core/grappler/costs/graph_properties.h"
 #include "tensorflow/core/grappler/grappler_item.h"
+#include "tensorflow/core/grappler/mutable_graph_view.h"
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/grappler/optimizers/constant_folding.h"
 #include "tensorflow/core/grappler/utils.h"
@@ -38,20 +42,15 @@ namespace grappler {
 
 namespace {
 
-bool RemoveInput(NodeDef* node, const string& input, NodeMap* node_map) {
-  bool removed_input = false;
-  int pos = 0;
-  while (pos < node->input_size()) {
-    if (node->input(pos) == input) {
-      node->mutable_input()->SwapElements(pos, node->input_size() - 1);
-      node->mutable_input()->RemoveLast();
-      node_map->RemoveOutput(NodeName(input), node->name());
-      removed_input = true;
-    } else {
-      ++pos;
-    }
+// Builds a map from the &graph->node(i) to i.
+absl::flat_hash_map<const NodeDef*, int> BuildNodeToIdx(const GraphDef& graph) {
+  // Set up &node -> index map.
+  absl::flat_hash_map<const NodeDef*, int> node_to_idx;
+  for (int i = 0; i < graph.node_size(); ++i) {
+    const NodeDef& node = graph.node(i);
+    node_to_idx[&node] = i;
   }
-  return removed_input;
+  return node_to_idx;
 }
 
 }  // namespace
@@ -68,7 +67,10 @@ bool DependencyOptimizer::SafeToRemoveIdentity(const NodeDef& node) const {
     // The output values of this node may be needed.
     return false;
   }
-  const NodeDef* input = node_map_->GetNode(NodeName(node.input(0)));
+
+  MutableGraphView::OutputPort port = graph_view_->GetRegularFanin(
+      MutableGraphView::InputPort(const_cast<NodeDef*>(&node), 0));
+  NodeDef* input = port.node;
   CHECK(input != nullptr) << "node = " << node.name()
                           << " input = " << node.input(0);
   // Don't remove Identity nodes corresponding to Variable reads or following
@@ -79,20 +81,23 @@ bool DependencyOptimizer::SafeToRemoveIdentity(const NodeDef& node) const {
     // Don't turn Identity nodes following Switch into NoOp or remove them
     // if it requires anchoring a control dependencies the Switch node, which
     // is not valid.
-    if (str_util::StartsWith(node.name(), kConstantFoldingCtrl)) {
-      // TODO(rmlarsen): Try to remove this artificial contraint.
+    MutableGraphView::OutputPort control_port(const_cast<NodeDef*>(&node),
+                                              Graph::kControlSlot);
+    if (!graph_view_->GetFanout(control_port).empty()) {
       return false;
     }
   }
-  for (auto consumer : node_map_->GetOutputs(node.name())) {
-    if (node.input_size() > 1 && IsMerge(*consumer)) {
+  bool node_has_multiple_inputs =
+      graph_view_->NumFanins(node, /*include_controlling_nodes=*/true) > 1;
+  for (auto consumer :
+       graph_view_->GetFanouts(node, /*include_controlled_nodes=*/true)) {
+    if (node_has_multiple_inputs && IsMerge(*consumer.node)) {
       return false;
     }
     if (IsSwitch(*input)) {
-      for (const string& consumer_input : consumer->input()) {
-        if (consumer_input == AsControlDependency(node.name())) {
-          return false;
-        }
+      if (graph_view_->HasFanin(*consumer.node,
+                                {node.name(), Graph::kControlSlot})) {
+        return false;
       }
     }
   }
@@ -126,7 +131,7 @@ bool DependencyOptimizer::SafeToConvertToNoOp(const NodeDef& node) const {
   if (!SafeToRemoveIdentity(node)) {
     return false;
   }
-  if (NumNonControlOutputs(node, *node_map_) > 0) {
+  if (graph_view_->NumFanouts(node, /*include_controlled_nodes=*/false) > 0) {
     // The output values of this node may be needed.
     return false;
   }
@@ -134,61 +139,56 @@ bool DependencyOptimizer::SafeToConvertToNoOp(const NodeDef& node) const {
 }
 
 int DependencyOptimizer::NumEdgesIfBypassed(
-    const NodeDef& node, const std::vector<NodeDef*>& output_nodes) const {
+    const NodeDef& node, int num_fanins,
+    const absl::flat_hash_set<MutableGraphView::Edge>& fanout_edges) const {
   const bool is_multi_input_identity_n =
       IsIdentityN(node) && !IsIdentityNSingleInput(node);
-  const int num_outputs = output_nodes.size();
-  const int num_inputs = node.input_size();
+  const int num_fanouts = fanout_edges.size();
 
   if (is_multi_input_identity_n) {
     // multi-input identity_n with input/output control dependencies will likely
     // increase number of edges after optimization.
     int num_edges_if_bypassed(0);
-    for (string input_node_name : node.input()) {
-      if (IsControlInput(input_node_name)) {
-        num_edges_if_bypassed += num_outputs;
+    int num_non_controlling_fanins =
+        graph_view_->NumFanins(node, /*include_controlling_nodes=*/false);
+    num_edges_if_bypassed += num_non_controlling_fanins;
+    num_edges_if_bypassed +=
+        (num_fanins - num_non_controlling_fanins) * num_fanouts;
+
+    for (const auto& fanout : fanout_edges) {
+      if (fanout.dst.port_id == Graph::kControlSlot) {
+        num_edges_if_bypassed += num_fanins;
       } else {
         ++num_edges_if_bypassed;
       }
     }
-
-    for (auto consumer : output_nodes) {
-      for (int j = 0; j < consumer->input_size(); ++j) {
-        const TensorId consumer_input = ParseTensorName(consumer->input(j));
-        if (consumer_input.node() == node.name()) {
-          if (IsControlInput(consumer_input)) {
-            num_edges_if_bypassed += num_inputs;
-          } else {
-            ++num_edges_if_bypassed;
-          }
-        }
-      }
-    }
     return num_edges_if_bypassed;
   } else {
-    return num_inputs * num_outputs;
+    return num_fanins * num_fanouts;
   }
 }
 
 bool DependencyOptimizer::BypassingNodeIsBeneficial(
-    const NodeDef& node, const std::vector<NodeDef*>& input_nodes,
-    const std::vector<NodeDef*>& output_nodes) const {
+    const NodeDef& node,
+    const absl::flat_hash_set<MutableGraphView::OutputPort>& fanins,
+    const absl::flat_hash_set<MutableGraphView::Edge>& fanout_edges) const {
   const bool is_identity = IsIdentity(node) || IsIdentityNSingleInput(node);
   const bool is_multi_input_identity_n =
       IsIdentityN(node) && !IsIdentityNSingleInput(node);
-  const int num_outputs = output_nodes.size();
-  const int num_inputs = node.input_size();
+  const int num_outputs = fanout_edges.size();
+  const int num_inputs = fanins.size();
 
-  if (NumEdgesIfBypassed(node, output_nodes) > num_inputs + num_outputs) {
+  if (NumEdgesIfBypassed(node, num_inputs, fanout_edges) >
+      num_inputs + num_outputs) {
     return false;
   }
 
   // Make sure that we don't increase the number of edges that cross
   // device boundaries.
   if ((num_inputs == 1 && num_outputs > 1 &&
-       input_nodes[0]->device() != node.device()) ||
+       fanins.begin()->node->device() != node.device()) ||
       (num_inputs > 1 && num_outputs == 1 &&
-       output_nodes[0]->device() != node.device())) {
+       fanout_edges.begin()->dst.node->device() != node.device())) {
     return false;
   }
 
@@ -197,12 +197,12 @@ bool DependencyOptimizer::BypassingNodeIsBeneficial(
   // cost before and after.
   const string& node_dev = node.device();
   int num_cross_in = 0;
-  for (NodeDef* input_node : input_nodes) {
-    num_cross_in += static_cast<int>(input_node->device() != node_dev);
+  for (const auto& fanin : fanins) {
+    num_cross_in += static_cast<int>(fanin.node->device() != node_dev);
   }
   int num_cross_out = 0;
-  for (NodeDef* output_node : output_nodes) {
-    num_cross_out += static_cast<int>(output_node->device() != node_dev);
+  for (const auto& fanout : fanout_edges) {
+    num_cross_out += static_cast<int>(fanout.dst.node->device() != node_dev);
   }
 
   if ((is_identity || is_multi_input_identity_n) && num_cross_in > 0 &&
@@ -216,10 +216,10 @@ bool DependencyOptimizer::BypassingNodeIsBeneficial(
   // Make sure we do not increase the number of device crossings.
   const int num_cross_before = num_cross_in + num_cross_out;
   int num_cross_after = 0;
-  for (NodeDef* input_node : input_nodes) {
-    for (NodeDef* output_node : output_nodes) {
+  for (const auto& fanin : fanins) {
+    for (const auto& fanout : fanout_edges) {
       num_cross_after +=
-          static_cast<int>(input_node->device() != output_node->device());
+          static_cast<int>(fanin.node->device() != fanout.dst.node->device());
     }
   }
   if (num_cross_after > num_cross_before) {
@@ -228,45 +228,32 @@ bool DependencyOptimizer::BypassingNodeIsBeneficial(
   return true;
 }
 
-void DependencyOptimizer::OptimizeNode(int node_idx,
-                                       SetVector<int>* nodes_to_simplify,
-                                       std::set<int>* nodes_to_delete) {
-  NodeDef* node = optimized_graph_->mutable_node(node_idx);
+void DependencyOptimizer::OptimizeNode(NodeDef* node,
+                                       SetVector<NodeDef*>* nodes_to_simplify,
+                                       std::set<string>* nodes_to_delete) {
+  const string node_name = node->name();
   const bool is_noop = IsNoOp(*node);
   const bool is_identity = IsIdentity(*node) || IsIdentityNSingleInput(*node);
   const bool is_multi_input_identity =
       IsIdentityN(*node) && !IsIdentityNSingleInput(*node);
-  const string node_name = node->name();
   // Constant nodes with no input control dependency are always executed early,
   // so we can prune all their output control dependencies.
-  if (IsConstant(*node) && node->input_size() == 0) {
-    const std::set<NodeDef*> output_nodes = node_map_->GetOutputs(node_name);
-    for (NodeDef* fanout : output_nodes) {
-      bool optimize_fanout = false;
-      bool data_connection = false;
-      for (int i = fanout->input_size() - 1; i >= 0; --i) {
-        const TensorId input_tensor = ParseTensorName(fanout->input(i));
-        if (input_tensor.node() == node_name) {
-          if (input_tensor.index() < 0) {
-            fanout->mutable_input()->SwapElements(i, fanout->input_size() - 1);
-            fanout->mutable_input()->RemoveLast();
-            optimize_fanout = true;
-          } else {
-            data_connection = true;
-          }
-        }
-      }
-      if (optimize_fanout) {
-        nodes_to_simplify->PushBack(node_to_idx_[fanout]);
-        if (!data_connection) {
-          node_map_->RemoveOutput(node_name, fanout->name());
-        }
+  if (IsConstant(*node) &&
+      graph_view_->NumFanins(*node, /*include_controlling_nodes=*/true) == 0) {
+    for (const auto& fanout :
+         graph_view_->GetFanouts(*node, /*include_controlled_nodes=*/true)) {
+      if (graph_view_->RemoveFanin(fanout.node->name(),
+                                   {node_name, Graph::kControlSlot})) {
+        nodes_to_simplify->PushBack(fanout.node);
       }
     }
-    if (node_map_->GetOutputs(node_name).empty() && fetch_nodes_known_ &&
+
+    if (graph_view_->NumFanouts(*node, /*include_controlled_nodes=*/true) ==
+            0 &&
+        fetch_nodes_known_ &&
         nodes_to_preserve_.find(node_name) == nodes_to_preserve_.end()) {
       // Mark the node for deletion.
-      nodes_to_delete->insert(node_to_idx_[node]);
+      nodes_to_delete->insert(node->name());
     }
     return;
   }
@@ -277,33 +264,23 @@ void DependencyOptimizer::OptimizeNode(int node_idx,
             << ") with NoOp.";
     // The outputs of this node are not consumed. Replace its inputs with
     // control dependencies and replace the op itself with the NoOp op.
-    std::unordered_set<string> ctrl_inputs;
-    int pos = 0;
-    while (pos < node->input_size()) {
-      const string old_input = node->input(pos);
-      if (IsControlInput(old_input)) {
-        if (!ctrl_inputs.insert(old_input).second) {
-          // We found a duplicate control input. Remove it.
-          node->mutable_input()->SwapElements(pos, node->input_size() - 1);
-          node->mutable_input()->RemoveLast();
-        } else {
-          ++pos;
-        }
-        continue;
+    int num_non_controlling_fanins =
+        graph_view_->NumFanins(*node, /*include_controlling_nodes=*/false);
+    if (num_non_controlling_fanins > 0) {
+      absl::flat_hash_set<string> non_controlling_fanins(
+          node->input().begin(),
+          node->input().begin() + num_non_controlling_fanins);
+      graph_view_->RemoveAllFanins(node_name,
+                                   /*keep_controlling_fanins=*/true);
+      for (const string& fanin : non_controlling_fanins) {
+        TensorId tensor_id = ParseTensorName(fanin);
+        graph_view_->AddControllingFanin(node_name, tensor_id);
+        nodes_to_simplify->PushBack(graph_view_->GetNode(tensor_id.node()));
       }
-      // Replace a normal input with a control input.
-      const string ctrl_input = ConstantFolding::AddControlDependency(
-          old_input, optimized_graph_, node_map_.get());
-      ctrl_inputs.insert(ctrl_input);
-      node->set_input(pos, ctrl_input);
-      node_map_->UpdateInput(node_name, old_input, ctrl_input);
-      const NodeDef* old_input_node = node_map_->GetNode(old_input);
-      nodes_to_simplify->PushBack(node_to_idx_[old_input_node]);
-      ++pos;
     }
     node->set_op("NoOp");
     node->clear_attr();
-    nodes_to_simplify->PushBack(node_to_idx_[node]);
+    nodes_to_simplify->PushBack(node);
     return;
   }
 
@@ -357,110 +334,80 @@ void DependencyOptimizer::OptimizeNode(int node_idx,
 
   if (is_noop || ((is_identity || is_multi_input_identity) &&
                   SafeToRemoveIdentity(*node))) {
-    const auto& output_node_set = node_map_->GetOutputs(node_name);
-    const std::vector<NodeDef*> output_nodes(output_node_set.begin(),
-                                             output_node_set.end());
-    const int num_inputs = node->input_size();
-    std::vector<NodeDef*> input_nodes;
-    for (int i = 0; i < num_inputs; ++i) {
-      NodeDef* input_node = node_map_->GetNode(node->input(i));
-      if (input_node == nullptr) {
-        LOG(ERROR) << "Invalid input " << node->input(i);
-        return;
-      }
-      input_nodes.push_back(input_node);
-    }
+    auto fanins =
+        graph_view_->GetFanins(*node, /*include_controlling_nodes=*/true);
+    auto fanout_edges =
+        graph_view_->GetFanoutEdges(*node, /*include_controlled_edges=*/true);
 
-    if (!BypassingNodeIsBeneficial(*node, input_nodes, output_nodes)) {
+    if (!BypassingNodeIsBeneficial(*node, fanins, fanout_edges)) {
       return;
     }
 
+    int num_non_controlling_fanins =
+        graph_view_->NumFanins(*node, /*include_controlling_nodes=*/false);
     VLOG(1) << "***** Rerouting input around\n" << node->DebugString();
     // Now remove the node and re-wire its inputs to its outputs.
-    for (auto consumer : output_nodes) {
+    for (auto fanout_edge : fanout_edges) {
       bool updated_consumer = false;
+      NodeDef* consumer = fanout_edge.dst.node;
       VLOG(1) << "consumer before:\n" << consumer->DebugString();
-      for (int i = 0; i < num_inputs; ++i) {
-        const NodeDef* input = input_nodes[i];
-        // Forward dependency from input to consumer if it doesn't already
-        // depend on it.
-        if ((is_identity && i == 0) ||
-            (is_multi_input_identity && !IsControlInput(node->input(i)))) {
-          // Replace regular input from Identity node.
-          string new_input;
-          const string& input_to_forward = node->input(i);
-          CHECK(!IsControlInput(input_to_forward));
-          for (int j = 0; j < consumer->input_size(); ++j) {
-            const TensorId old_input = ParseTensorName(consumer->input(j));
-            if (old_input.node() == node_name) {
-              if (old_input.index() == i) {
-                // Regular input
-                new_input = input_to_forward;
-                node_map_->UpdateInput(consumer->name(), old_input.ToString(),
-                                       new_input);
-                consumer->set_input(j, new_input);
-              } else if (old_input.index() == -1) {
-                // Control dependency
-                new_input = AsControlDependency(NodeName(input_to_forward));
-                node_map_->UpdateInput(consumer->name(), old_input.ToString(),
-                                       new_input);
-                consumer->set_input(j, new_input);
-              }
+      if (fanout_edge.src.port_id == Graph::kControlSlot) {
+        updated_consumer = graph_view_->RemoveFanin(
+            consumer->name(), {node_name, Graph::kControlSlot});
+        if (updated_consumer) {
+          // Add all non controlling fanins of node as controlling fanins to
+          // consumer.
+          for (int i = 0; i < num_non_controlling_fanins; ++i) {
+            TensorId input = ParseTensorName(node->input(i));
+            if (graph_view_->AddControllingFanin(consumer->name(), input)) {
+              nodes_to_simplify->PushBack(graph_view_->GetNode(input.node()));
             }
           }
-          updated_consumer = true;
-        } else {
-          // Forward dependency from input to consumer if it doesn't already
-          // depend on it.
-          if (node_map_->GetOutputs(input->name()).count(consumer) == 0) {
-            consumer->add_input(AsControlDependency(input->name()));
-            node_map_->AddOutput(input->name(), consumer->name());
-            nodes_to_simplify->PushBack(node_to_idx_[input]);
-            updated_consumer = true;
-          }
         }
+      } else {
+        updated_consumer = graph_view_->UpdateFanin(
+            consumer->name(), {node_name, fanout_edge.src.port_id},
+            ParseTensorName(node->input(fanout_edge.src.port_id)));
       }
-      // Remove dependency on node from consumer.
-      updated_consumer |= RemoveInput(consumer, AsControlDependency(node_name),
-                                      node_map_.get());
       if (updated_consumer) {
-        nodes_to_simplify->PushBack(node_to_idx_[consumer]);
+        // Forward all controlling fanins of node to consumer.
+        for (int i = num_non_controlling_fanins; i < node->input_size(); ++i) {
+          TensorId input = ParseTensorName(node->input(i));
+          if (graph_view_->AddFanin(consumer->name(), input)) {
+            nodes_to_simplify->PushBack(graph_view_->GetNode(input.node()));
+          }
+        }
+        nodes_to_simplify->PushBack(consumer);
       }
       VLOG(1) << "consumer after:\n" << consumer->DebugString();
     }
-    node_map_->RemoveOutputs(node_name);
     if (fetch_nodes_known_ &&
         nodes_to_preserve_.find(node_name) == nodes_to_preserve_.end()) {
       // Mark the node for deletion.
-      nodes_to_delete->insert(node_idx);
+      nodes_to_delete->insert(node_name);
 
       // Disconnect the node from its inputs to enable further optimizations.
-      node_map_->RemoveInputs(node_name);
-      node->clear_input();
+      graph_view_->RemoveAllFanins(node_name,
+                                   /*keep_controlling_fanins=*/false);
     }
   }
 }
 
-void DependencyOptimizer::CleanControlInputs() {
-  for (int i = 0; i < optimized_graph_->node_size(); ++i) {
-    DedupControlInputs(optimized_graph_->mutable_node(i));
-  }
-}
-
 Status DependencyOptimizer::OptimizeDependencies() {
-  SetVector<int> nodes_to_simplify;
-  std::set<int> nodes_to_delete;
-  for (int i = 0; i < optimized_graph_->node_size(); ++i) {
-    const NodeDef& node = optimized_graph_->node(i);
-    if (IsNoOp(node) || IsIdentity(node) || IsIdentityN(node) ||
-        IsConstant(node) || SafeToConvertToNoOp(node)) {
-      nodes_to_simplify.PushBack(i);
+  SetVector<NodeDef*> nodes_to_simplify;
+  std::set<string> nodes_to_delete;
+  for (int i = 0; i < graph_view_->graph()->node_size(); ++i) {
+    NodeDef* node = graph_view_->graph()->mutable_node(i);
+    if (IsNoOp(*node) || IsIdentity(*node) || IsIdentityN(*node) ||
+        IsConstant(*node) || SafeToConvertToNoOp(*node)) {
+      nodes_to_simplify.PushBack(node);
     }
   }
   while (!nodes_to_simplify.Empty()) {
-    int node_to_simplify = nodes_to_simplify.PopBack();
+    NodeDef* node_to_simplify = nodes_to_simplify.PopBack();
     // Discard nodes that were marked for deletion already.
-    while (nodes_to_delete.find(node_to_simplify) != nodes_to_delete.end()) {
+    while (nodes_to_delete.find(node_to_simplify->name()) !=
+           nodes_to_delete.end()) {
       node_to_simplify = nodes_to_simplify.PopBack();
     }
     OptimizeNode(node_to_simplify, &nodes_to_simplify, &nodes_to_delete);
@@ -468,17 +415,17 @@ Status DependencyOptimizer::OptimizeDependencies() {
 
   if (fetch_nodes_known_) {
     VLOG(1) << "Deleted " << nodes_to_delete.size() << " out of "
-            << optimized_graph_->node_size() << " nodes.";
-    EraseNodesFromGraph(nodes_to_delete, optimized_graph_);
-    node_map_.reset(new NodeMap(optimized_graph_));
-    BuildNodeToIdx();
+            << graph_view_->graph()->node_size() << " nodes.";
+    graph_view_->DeleteNodes(nodes_to_delete);
   }
   return Status::OK();
 }
 
 Status DependencyOptimizer::TransitiveReduction() {
   // PRECONDITION: optimized_graph_ must be sorted topologically.
-  const int num_nodes = optimized_graph_->node_size();
+  GraphDef* graph = graph_view_->graph();
+  auto node_to_idx = BuildNodeToIdx(*graph);
+  const int num_nodes = graph->node_size();
   // Set up a compressed version of the graph to save a constant factor in the
   // expensive algorithm below. Also cache the set of control outputs and the
   // highest index of a target of any control output from each node.
@@ -487,20 +434,20 @@ Status DependencyOptimizer::TransitiveReduction() {
   std::vector<gtl::InlinedVector<std::pair<int, int>, 2>> control_outputs(
       num_nodes);
   for (int node_idx = 0; node_idx < num_nodes; ++node_idx) {
-    const NodeDef& node = optimized_graph_->node(node_idx);
+    const NodeDef& node = graph->node(node_idx);
     if (ModifiesFrameInfo(node) || !HasOpDef(node)) {
       // Ignore function nodes and nodes that modify frame info.
       continue;
     }
     for (int input_slot = 0; input_slot < node.input_size(); ++input_slot) {
       const string& input = node.input(input_slot);
-      const NodeDef* input_node = node_map_->GetNode(input);
+      const NodeDef* input_node = graph_view_->GetNode(NodeName(input));
       if (ModifiesFrameInfo(*input_node) || IsMerge(*input_node)) {
         // Ignore edges from nodes that modify frame info and from Merge nodes,
         // because we cannot know which of it's input paths executes.
         continue;
       }
-      const int input_node_idx = node_to_idx_[input_node];
+      const int input_node_idx = node_to_idx[input_node];
       inputs[node_idx].push_back(input_node_idx);
       if (IsControlInput(input)) {
         ++num_controls;
@@ -566,16 +513,12 @@ Status DependencyOptimizer::TransitiveReduction() {
 
   for (const auto& it : control_edges_to_remove) {
     const int target = it.first;
-    NodeDef* target_node = optimized_graph_->mutable_node(target);
+    const NodeDef& target_node = graph->node(target);
+    const string target_node_name = target_node.name();
     for (const InputSlotAndSource& slot_and_source : it.second) {
       const int input_slot = slot_and_source.first;
-      const int source = slot_and_source.second;
-      const NodeDef& source_node = optimized_graph_->node(source);
-      CHECK_LT(input_slot, target_node->input_size());
-      target_node->mutable_input()->SwapElements(input_slot,
-                                                 target_node->input_size() - 1);
-      node_map_->RemoveOutput(source_node.name(), target_node->name());
-      target_node->mutable_input()->RemoveLast();
+      const TensorId tensor_id = ParseTensorName(target_node.input(input_slot));
+      graph_view_->RemoveFanin(target_node_name, tensor_id);
       ++num_controls_removed;
     }
   }
@@ -584,26 +527,17 @@ Status DependencyOptimizer::TransitiveReduction() {
   return Status::OK();
 }
 
-void DependencyOptimizer::BuildNodeToIdx() {
-  // Set up &node -> index map.
-  node_to_idx_.clear();
-  for (int i = 0; i < optimized_graph_->node_size(); ++i) {
-    const NodeDef& node = optimized_graph_->node(i);
-    node_to_idx_[&node] = i;
-  }
-}
-
 // Suppose there are cross-device control inputs to node C from multiple nodes
 // that are located on another device, e.g., we have control edges:
 // A->C, B->C
 // where A and B are on device X and C is on device Y.
 // We can reduce cross-device communication by introducing an intermediate
 // NoOp node C' on device X and rewriting the control edges to:
-// A->C', B->C', C' -> C
+// A->C', B->C', C'->C
 void DependencyOptimizer::GroupCrossDeviceControlEdges() {
-  const int num_nodes = optimized_graph_->node_size();
+  const int num_nodes = graph_view_->graph()->node_size();
   for (int i = 0; i < num_nodes; ++i) {
-    NodeDef* node = optimized_graph_->mutable_node(i);
+    NodeDef* node = graph_view_->graph()->mutable_node(i);
     if (node->device().empty()) continue;
 
     // Creates new noop nodes for devices on which multiple control inputs are
@@ -614,66 +548,50 @@ void DependencyOptimizer::GroupCrossDeviceControlEdges() {
     // that device.
     std::map<string, NodeDef*> noops;
     int num_noops = 0;
-    for (int j = 0; j < node->input_size(); ++j) {
-      if (IsControlInput(node->input(j))) {
-        const NodeDef* input = node_map_->GetNode(node->input(j));
-        if (input != nullptr && !input->device().empty() &&
-            input->device() != node->device()) {
-          auto emplace_result = noops.emplace(input->device(), nullptr);
-          if (!emplace_result.second &&
-              emplace_result.first->second == nullptr) {
-            // This is the second cross-device control input from the same
-            // device. Creates an intermediate noop node on that device.
-            string group_name;
-            NodeDef* noop;
-            // Creates a fresh node name; there may be conflicting names from
-            // a previous iteration of the optimizer.
-            do {
-              group_name = AddPrefixToNodeName(
-                  node->name(),
-                  strings::StrCat("GroupCrossDeviceControlEdges_", num_noops));
-              noop = node_map_->GetNode(group_name);
-              ++num_noops;
-            } while (noop != nullptr);
-            noop = optimized_graph_->add_node();
-            noop->set_name(group_name);
-            noop->set_device(input->device());
-            noop->set_op("NoOp");
-            node_map_->AddNode(noop->name(), noop);
-            emplace_result.first->second = noop;
-          }
+    auto controlling_fanins = graph_view_->GetFanin(
+        MutableGraphView::InputPort(node, Graph::kControlSlot));
+    for (const auto& controlling_fanin : controlling_fanins) {
+      const NodeDef* fanin_node = controlling_fanin.node;
+      if (!fanin_node->device().empty() &&
+          fanin_node->device() != node->device()) {
+        auto emplace_result = noops.emplace(fanin_node->device(), nullptr);
+        if (!emplace_result.second && emplace_result.first->second == nullptr) {
+          // This is the second cross-device control input from the same
+          // device. Creates an intermediate noop node on that device.
+          string group_name;
+          NodeDef* noop;
+          // Creates a fresh node name; there may be conflicting names from
+          // a previous iteration of the optimizer.
+          do {
+            group_name = AddPrefixToNodeName(
+                node->name(),
+                strings::StrCat("GroupCrossDeviceControlEdges_", num_noops));
+            noop = graph_view_->GetNode(group_name);
+            ++num_noops;
+          } while (noop != nullptr);
+          NodeDef new_node;
+          new_node.set_name(group_name);
+          new_node.set_device(fanin_node->device());
+          new_node.set_op("NoOp");
+          emplace_result.first->second =
+              graph_view_->AddNode(std::move(new_node));
         }
       }
     }
 
     // Reroute existing control edges to go via the newly introduced NoOp nodes.
-    int pos = 0;
-    while (pos < node->input_size()) {
-      const string& input_name = node->input(pos);
-      if (IsControlInput(input_name)) {
-        NodeDef* input = node_map_->GetNode(input_name);
-        if (input == nullptr) {
-          ++pos;
-        } else {
-          auto it = noops.find(input->device());
-          if (it == noops.end() || it->second == nullptr) {
-            ++pos;
-          } else {
-            node->mutable_input()->SwapElements(pos, node->input_size() - 1);
-            node->mutable_input()->RemoveLast();
-            it->second->add_input(AsControlDependency(*input));
-            node_map_->UpdateOutput(input_name, node->name(),
-                                    it->second->name());
-          }
-        }
-      } else {
-        ++pos;
+    for (const auto& controlling_fanin : controlling_fanins) {
+      auto it = noops.find(controlling_fanin.node->device());
+      if (it != noops.end() && it->second != nullptr) {
+        TensorId tensor_id(controlling_fanin.node->name(), Graph::kControlSlot);
+        graph_view_->RemoveFanin(node->name(), tensor_id);
+        graph_view_->AddFanin(it->second->name(), tensor_id);
       }
     }
     for (const auto& entry : noops) {
       if (entry.second) {
-        node->add_input(AsControlDependency(*entry.second));
-        node_map_->AddOutput(entry.second->name(), node->name());
+        graph_view_->AddFanin(node->name(),
+                              {entry.second->name(), Graph::kControlSlot});
       }
     }
   }
@@ -681,22 +599,17 @@ void DependencyOptimizer::GroupCrossDeviceControlEdges() {
 
 Status DependencyOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
                                      GraphDef* optimized_graph) {
-  optimized_graph_ = optimized_graph;
-  *optimized_graph_ = item.graph;
+  *optimized_graph = item.graph;
   nodes_to_preserve_ = item.NodesToPreserve();
   fetch_nodes_known_ = !item.fetch.empty();
-  CleanControlInputs();
+  graph_view_.reset(new MutableGraphView(optimized_graph));
 
   const int num_iterations = 2;
   for (int iteration = 0; iteration < num_iterations; ++iteration) {
     GRAPPLER_RETURN_IF_DEADLINE_EXCEEDED();
     Status topo_sort_status;
     // Perform topological sort to prepare the graph for transitive reduction.
-    topo_sort_status = TopologicalSort(optimized_graph_);
-    // Set up index-based graph datastructures to speed up analysis steps below.
-    node_map_.reset(new NodeMap(optimized_graph_));
-    BuildNodeToIdx();
-
+    topo_sort_status = TopologicalSort(optimized_graph);
     if (topo_sort_status.ok()) {
       // Remove redundant control dependencies.
       TF_RETURN_IF_ERROR(TransitiveReduction());
@@ -709,9 +622,6 @@ Status DependencyOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
     // nodes.
     TF_RETURN_IF_ERROR(OptimizeDependencies());
 
-    // Dedup control inputs.
-    CleanControlInputs();
-
     GroupCrossDeviceControlEdges();
   }
 
diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer.h b/tensorflow/core/grappler/optimizers/dependency_optimizer.h
index 7b032673fb..eb7fc89080 100644
--- a/tensorflow/core/grappler/optimizers/dependency_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/dependency_optimizer.h
@@ -17,6 +17,8 @@ limitations under the License.
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DEPENDENCY_OPTIMIZER_H_
 
 #include <unordered_set>
+
+#include "tensorflow/core/grappler/mutable_graph_view.h"
 #include "tensorflow/core/grappler/optimizers/graph_optimizer.h"
 #include "tensorflow/core/grappler/utils.h"
 #include "tensorflow/core/protobuf/rewriter_config.pb.h"
@@ -46,24 +48,22 @@ class DependencyOptimizer : public GraphOptimizer {
   // Returns true if bypassing node does not increase the number of edges or
   // number of edges crossing a device boundary.
   bool BypassingNodeIsBeneficial(
-      const NodeDef& node, const std::vector<NodeDef*>& input_nodes,
-      const std::vector<NodeDef*>& output_nodes) const;
-  int NumEdgesIfBypassed(const NodeDef& node,
-                         const std::vector<NodeDef*>& output_nodes) const;
+      const NodeDef& node,
+      const absl::flat_hash_set<MutableGraphView::OutputPort>& fanins,
+      const absl::flat_hash_set<MutableGraphView::Edge>& fanout_edges) const;
+  int NumEdgesIfBypassed(
+      const NodeDef& node, int num_fanins,
+      const absl::flat_hash_set<MutableGraphView::Edge>& fanout_edges) const;
   // Returns true if node is not an Identity node or if it is an Identity
   // that is safe to remove.
   bool SafeToRemoveIdentity(const NodeDef& node) const;
   // Returns true if it is safe to convert node to NoOp.
   bool SafeToConvertToNoOp(const NodeDef& node) const;
-  // Removes all duplicate control dependencies.
-  void CleanControlInputs();
-  // Builds a map from the &optimized_graph_->node(i) to i.
-  void BuildNodeToIdx();
   // Tries to optimize the node with the given index, possibly additional
   // optimizations by inserting nodes in nodes_to_simplify, and pruning nodes by
   // inserting them in nodes_to_delete.
-  void OptimizeNode(int node_idx, SetVector<int>* nodes_to_simplify,
-                    std::set<int>* nodes_to_delete);
+  void OptimizeNode(NodeDef* node, SetVector<NodeDef*>* nodes_to_simplify,
+                    std::set<string>* nodes_to_delete);
   // Eliminates redundant control dependencies by computing the transitive
   // reduction of the graph.
   Status TransitiveReduction();
@@ -76,9 +76,7 @@ class DependencyOptimizer : public GraphOptimizer {
   RewriterConfig::Toggle opt_level_;
   bool fetch_nodes_known_;
   std::unordered_set<string> nodes_to_preserve_;
-  std::unique_ptr<NodeMap> node_map_;
-  std::unordered_map<const NodeDef*, int> node_to_idx_;
-  GraphDef* optimized_graph_;  // Not owned.
+  std::unique_ptr<MutableGraphView> graph_view_;
 };
 
 }  // end namespace grappler
diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc b/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
index 8d70d9d5c7..673fc987a8 100644
--- a/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
@@ -91,7 +91,7 @@ TEST_F(DependencyOptimizerTest, DependenciesDrivenByConstants) {
   // The 'z' node should have been optimized away leaving only 5 nodes.
   EXPECT_EQ(5, output.node_size());
 
-  for (const NodeDef& node : item.graph.node()) {
+  for (const NodeDef& node : output.node()) {
     if (node.name() == "id1" || node.name() == "id2") {
       EXPECT_EQ(1, node.input_size());
       EXPECT_EQ("add", node.input(0));
@@ -125,8 +125,8 @@ TEST_F(DependencyOptimizerTest, ChangeToNoop) {
 
   EXPECT_EQ(item.graph.node_size(), output.node_size());
   int found = 0;
-  for (int i = 0; i < item.graph.node_size(); ++i) {
-    const NodeDef& node = item.graph.node(i);
+  for (int i = 0; i < output.node_size(); ++i) {
+    const NodeDef& node = output.node(i);
     // "add" should get turned into a NoOp and removed.
     EXPECT_NE("add", node.name());
     if (node.name() == "id1") {
@@ -164,7 +164,6 @@ TEST_F(DependencyOptimizerTest, ChangeToNoop_RepeatedInput) {
   item.graph.Swap(&output);
   status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
-  LOG(INFO) << output.DebugString();
 
   EXPECT_EQ(item.graph.node_size(), output.node_size());
   int found = 0;
@@ -748,7 +747,7 @@ TEST_F(DependencyOptimizerTest, Identity_DeviceCrossing_ConsumerOnSameDevice) {
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
-  LOG(INFO) << output.DebugString();
+
   EXPECT_EQ(3, output.node_size());
   for (const auto& node : output.node()) {
     EXPECT_NE("x_on_2", node.name());
-- 
GitLab


From fed73306b650d0b5c99689e8b142a610fd8a3488 Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Mon, 7 Jan 2019 16:32:46 -0800
Subject: [PATCH 0303/2345] Update unidirectional rnn test case.

PiperOrigin-RevId: 228257798
---
 tensorflow/lite/kernels/unidirectional_sequence_rnn_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/kernels/unidirectional_sequence_rnn_test.cc b/tensorflow/lite/kernels/unidirectional_sequence_rnn_test.cc
index 494c449589..de1f7818bd 100644
--- a/tensorflow/lite/kernels/unidirectional_sequence_rnn_test.cc
+++ b/tensorflow/lite/kernels/unidirectional_sequence_rnn_test.cc
@@ -255,7 +255,7 @@ class HybridUnidirectionalRNNOpModel : public UnidirectionalRNNOpModel {
     tensor_type_ = tensor_type;
   }
 
-  void SetWeights(int weights_idx, std::vector<float> f) {
+  void SetWeights(int weights_idx, const std::vector<float>& f) {
     if (tensor_type_ == TensorType_UINT8) {
       SymmetricQuantizeAndPopulate(weights_idx, f);
     } else {
-- 
GitLab


From b9762fed18234b2d87efc9786384e467008fc674 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Mon, 7 Jan 2019 16:50:53 -0800
Subject: [PATCH 0304/2345] Add benchmarks for pack_lhs

PiperOrigin-RevId: 228260001
---
 tensorflow/core/kernels/BUILD                 |   1 +
 .../eigen_spatial_convolutions_test.cc        | 226 +++++++++++++++++-
 2 files changed, 214 insertions(+), 13 deletions(-)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 136902e249..701bda7ef6 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -2511,6 +2511,7 @@ tf_cc_tests(
         ":eigen_helpers",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
+        "@com_google_absl//absl/strings",
     ],
 )
 
diff --git a/tensorflow/core/kernels/eigen_spatial_convolutions_test.cc b/tensorflow/core/kernels/eigen_spatial_convolutions_test.cc
index 22f71d6260..03002adec4 100644
--- a/tensorflow/core/kernels/eigen_spatial_convolutions_test.cc
+++ b/tensorflow/core/kernels/eigen_spatial_convolutions_test.cc
@@ -14,6 +14,8 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/kernels/eigen_spatial_convolutions.h"
+
+#include "absl/strings/str_cat.h"
 #include "tensorflow/core/kernels/eigen_cuboid_convolution.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/platform/test_benchmark.h"
@@ -1540,22 +1542,187 @@ static void PackRhsHelper(int iters,
     pack_rhs(packed.data() + packed_offset, sub_mapper, depth, cols);
   }
   tensorflow::testing::StopTiming();
+  tensorflow::testing::SetLabel(
+      absl::StrCat("patch: ", patch_rows, "x", patch_cols, " D", patch_depth,
+                   "; num_patches=", num_patches, " patch_size=", patch_size,
+                   " num_inputs=", num_inputs));
+}
+
+static void PackLhsHelper(int iters,
+                          /* Input dimensions: */
+                          int input_depth,
+                          /* Filter (kernel) dimensions: */
+                          int filter_count, int filter_cols, int filter_rows,
+                          /* Block dimensions: */
+                          Index block_rows, Index block_cols) {
+  // Set random seed for benchmark repeatability.
+  srand(12345);
+
+  eigen_assert(block_rows <= filter_count);
+  eigen_assert(block_cols <= input_depth * filter_rows * filter_cols);
+
+  tensorflow::testing::UseRealTime();
+  tensorflow::testing::StopTiming();
+
+  using Dimensions = Eigen::DSizes<Eigen::Index, 4>;
+
+  // Default Eigen::Tensor layout is column major, so we configure dimensions
+  // starting from the inner most (`filter count` aka `kernel filers`).
+  Dimensions filter_dims(filter_count, filter_rows, filter_cols, input_depth);
+
+  static const int packet_size = Eigen::internal::packet_traits<float>::size;
+
+  // We are going to reshape filter into 2D tensor.
+  using NewDimension = Eigen::DSizes<Index, 2>;
+
+  // Contraction dimensions.
+  using nocontract_t = Eigen::array<Eigen::Index, 1>;
+  using contract_t = Eigen::array<Eigen::Index, 1>;
+
+  // Input to the ReshapeOp. It is the tensorflow TTypes<float>::Tensor
+  // with ColMajor layout, instead of RowMajor. But that doesn't make any
+  // difference, because TensorContraction swaps LHS with RHS for row major
+  // inputs, and contraction mapper always works with column major data.
+  using ArgType = TensorMap<Tensor<float, 4>, Eigen::Aligned>;
+
+  using Evaluator =
+      TensorEvaluator<const TensorReshapingOp<NewDimension, ArgType>,
+                      Eigen::DefaultDevice>;
+
+  using InputMapper = Eigen::internal::TensorContractionInputMapper<
+      float, Index, Eigen::internal::Lhs, Evaluator,  //
+      nocontract_t, contract_t,                       //
+      packet_size,                                    //
+      /*inner_dim_contiguous*/ true,                  //
+      /*inner_dim_reordered*/ false,                  //
+      /*Alignment*/ 0>;
+
+  using SubMapper = Eigen::internal::TensorContractionSubMapper<
+      float, Index, Eigen::internal::Lhs, Evaluator,  //
+      nocontract_t, contract_t,                       //
+      packet_size,                                    //
+      /*inner_dim_contiguous*/ true,                  //
+      /*inner_dim_reordered*/ false,                  //
+      /*Alignment*/ 0>;
+
+#if defined(TENSORFLOW_USE_MKLDNN_CONTRACTION_KERNEL)
+  using PackLhsImpl = Eigen::internal::mkldnn_gemm_pack<float, Eigen::Index,
+                                                        SubMapper, ColMajor>;
+#else
+  using PackLhsImpl =
+      Eigen::internal::gemm_pack_lhs<float, Eigen::Index, SubMapper,      //
+                                     Traits::mr,                          //
+                                     Traits::LhsProgress,                 //
+                                     typename Traits::LhsPacket4Packing,  //
+                                     ColMajor>;
+#endif
+
+  Eigen::DefaultDevice device;
 
-  std::ostringstream stringStream;
-  stringStream << "patch: " << patch_rows << "x" << patch_cols << " D"
-               << patch_depth << "; num_patches=" << num_patches
-               << " patch_size=" << patch_size << " num_inputs=" << num_inputs;
-  tensorflow::testing::SetLabel(stringStream.str());
+  // We will reshape kernel into 2D tensor.
+  NewDimension reshape_dims;
+  reshape_dims[0] = filter_count;
+  reshape_dims[1] = input_depth * filter_rows * filter_cols;
+
+  // We are going to contract along the 'in_depth * filter_rows * filter_cols`.
+  nocontract_t nocontract_dim = {0};
+  contract_t contract_dim = {1};
+
+  // These values computed using the algorithm in TensorContraction.h, with
+  // 'nocontract_dim' and 'contract_dim' values specified above.
+  nocontract_t nocontract_strides = {1};
+  contract_t contract_strides = {filter_count};
+  nocontract_t i_strides = {1};
+  contract_t k_strides = {1};
+
+  // We use tensor of the same dimensions to store packed data.
+  Tensor<float, 4> packed(filter_dims);
+
+  // We generate multiple filter tensors, around 512mb in total size to measure
+  // realistic workload when input data in not in L1-L3 cache.
+  size_t input_bytes = filter_dims.TotalSize() * sizeof(float);
+  size_t mem_size_bytes = 1024 * 1024 * 512;
+  size_t num_filters =
+      std::max(static_cast<size_t>(1), mem_size_bytes / input_bytes);
+
+  std::vector<Tensor<float, 4>> filters;
+  std::vector<Evaluator> evaluators;
+  std::vector<InputMapper> input_mappers;
+
+  for (int i = 0; i < num_filters; ++i) {
+    filters.emplace_back(filter_dims);
+    filters[i].setRandom();
+
+    ArgType tensor_map(filters[i].data(), filter_dims);
+
+    const auto reshape_op =
+        TensorReshapingOp<NewDimension, ArgType>(tensor_map, reshape_dims);
+
+    evaluators.emplace_back(reshape_op, device);
+
+    input_mappers.emplace_back(evaluators[i], nocontract_strides, i_strides,
+                               contract_strides, k_strides);
+  }
+
+  PackLhsImpl pack_lhs;
+
+  const Index packed_total_size = filter_dims.TotalSize();
+
+  // Round up row/col/memory offsets to make them multiple of packet size.
+  const auto round_up = [](const Index idx) {
+    return (idx / packet_size) * packet_size;
+  };
+
+  // Block rows is in the [0, filter_count) range.
+  // Block cols is in the [0, filter_rows * filter_cols * input_depth) range.
+
+  const Index max_row = filter_count;
+  const Index max_col = filter_rows * filter_cols * input_depth;
+
+  tensorflow::testing::StartTiming();
+  for (int i = 0; i < iters; ++i) {
+    int filter_idx =
+        num_filters == 1 ? 1 : internal::random<int>(0, num_filters - 1);
+
+    Index row_offset = round_up(internal::random<Index>(0, max_row - 10));
+    Index col_offset = round_up(internal::random<Index>(0, max_col - 10));
+
+    Index rows = std::min(block_rows, max_row - row_offset);
+    Index cols = std::min(block_cols, max_col - col_offset);
+
+    // Write packed data to random memory location to emulate cold caches.
+    Index packed_offset = round_up(
+        internal::random<Index>(0, packed_total_size - rows * cols - 1));
+
+    SubMapper sub_mapper =
+        input_mappers[filter_idx].getSubMapper(row_offset, col_offset);
+
+    // NOTE: Eigen gemm_pack_lhs accepts contraction depth (k-th dimension) as a
+    // first argument (aka block cols). MKL-DNN pack is generic for lhs and rhs
+    // and accepts block rows and cols in the same order for lhs and rhs.
+#if defined(TENSORFLOW_USE_MKLDNN_CONTRACTION_KERNEL)
+    pack_lhs(packed.data() + packed_offset, sub_mapper, rows, cols);
+#else
+    pack_lhs(packed.data() + packed_offset, sub_mapper, cols, rows);
+#endif
+  }
+  tensorflow::testing::StopTiming();
+  tensorflow::testing::SetLabel(absl::StrCat(
+      "filter: count=", filter_count, " dims=", filter_rows, "x", filter_cols,
+      "; input: depth=", input_depth, "; num_filers=", num_filters));
 }
 
 // -------------------------------------------------------------------------- //
-// Macro argumentnames:
+// Pack RHS
+//
+// Macro argument names:
 //    N: batch size
 //    H: height
 //    W: width
 //    C: input channels
 //   FC: filter channles
 //   FH: filter height
+//   FW: filter width
 //   SH: stride in height dimensions
 //   SW: stride in width dimensions
 //   BR: block rows
@@ -1563,16 +1730,16 @@ static void PackRhsHelper(int iters,
 
 #define BM_CONCAT(a, b) a##b
 
-#define BM_NAME(prefix, N, H, W, C, FC, FH, FW, SH, SW, BR, BC)           \
+#define BM_RHS_NAME(prefix, N, H, W, C, FC, FH, FW, SH, SW, BR, BC)       \
   BM_CONCAT(BM_##prefix##_##N##_##H##x##W##_IC##C##_FC##FC##_##FH##x##FW, \
             _s##SH##x##SW##_B##BR##x##BC)
 
-#define BM_PackRhs(N, H, W, C, FC, FH, FW, SH, SW, BR, BC)         \
-  static void BM_NAME(PackRhs, N, H, W, C, FC, FH, FW, SH, SW, BR, \
-                      BC)(int iters) {                             \
-    PackRhsHelper(iters, N, H, W, C, FC, FH, FW, SH, SW, BR, BC);  \
-  }                                                                \
-  BENCHMARK(BM_NAME(PackRhs, N, H, W, C, FC, FH, FW, SH, SW, BR, BC))
+#define BM_PackRhs(N, H, W, C, FC, FH, FW, SH, SW, BR, BC)             \
+  static void BM_RHS_NAME(PackRhs, N, H, W, C, FC, FH, FW, SH, SW, BR, \
+                          BC)(int iters) {                             \
+    PackRhsHelper(iters, N, H, W, C, FC, FH, FW, SH, SW, BR, BC);      \
+  }                                                                    \
+  BENCHMARK(BM_RHS_NAME(PackRhs, N, H, W, C, FC, FH, FW, SH, SW, BR, BC))
 
 // Number of input channel (input depth) it equal to the number of patch
 // channels (patch depth).
@@ -1645,4 +1812,37 @@ BM_PackRhs(/*batch*/ 32,        //
            /*filter*/ 3, 3,     //
            /*stride*/ 2, 2,     //
            /*block*/ 36, 432);
+
+// -------------------------------------------------------------------------- //
+// Pack LHS
+//
+// Macro argument names:
+//    C: input channels
+//   FC: filter channels
+//   FH: filter height
+//   FW: filter width
+//   BR: block rows
+//   BC: block cols
+
+#define BM_LHS_NAME(prefix, C, FC, FH, FW, BR, BC) \
+  BM_CONCAT(BM_##prefix##_##C##_FC##FC##_##FH##x##FW, _B##BR##x##BC)
+
+#define BM_PackLhs(C, FC, FH, FW, BR, BC)                              \
+  static void BM_LHS_NAME(PackLhs, C, FC, FH, FW, BR, BC)(int iters) { \
+    PackLhsHelper(iters, C, FC, FH, FW, BR, BC);                       \
+  }                                                                    \
+  BENCHMARK(BM_LHS_NAME(PackLhs, C, FC, FH, FW, BR, BC))
+
+// Number of input channel (input depth) it equal to the number of patch
+// channels (patch depth).
+
+BM_PackLhs(/*input channels*/ 128,    //
+           /*filter channels*/ 1024,  //
+           /*filter dims*/ 3, 3,      //
+           /*block*/ 256, 56);
+
+BM_PackLhs(/*input channels*/ 128,    //
+           /*filter channels*/ 1024,  //
+           /*filter dims*/ 3, 3,      //
+           /*block*/ 56, 256);
 }  // namespace Eigen
-- 
GitLab


From a253b9eab582307315567ba36d2b25033ad29b8b Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Mon, 7 Jan 2019 16:54:22 -0800
Subject: [PATCH 0305/2345] Change tf_upgrade_v2.py script to wrap labels
 argument with tf.stop_gradients in tf.softmax_cross_entropy_with_logits call.

PiperOrigin-RevId: 228260462
---
 tensorflow/tools/compatibility/BUILD          |  1 +
 tensorflow/tools/compatibility/reorders_v2.py |  1 +
 .../compatibility/testdata/test_file_v1_12.py |  9 ++++
 .../tools/compatibility/tf_upgrade_v2.py      | 45 +++++++++++++++++--
 .../tools/compatibility/tf_upgrade_v2_test.py | 28 +++++++-----
 5 files changed, 69 insertions(+), 15 deletions(-)

diff --git a/tensorflow/tools/compatibility/BUILD b/tensorflow/tools/compatibility/BUILD
index 452e6db941..31dbc02963 100644
--- a/tensorflow/tools/compatibility/BUILD
+++ b/tensorflow/tools/compatibility/BUILD
@@ -69,6 +69,7 @@ py_library(
         ":ast_edits",
         ":renames_v2",
         ":reorders_v2",
+        "@six_archive//:six",
     ],
 )
 
diff --git a/tensorflow/tools/compatibility/reorders_v2.py b/tensorflow/tools/compatibility/reorders_v2.py
index 3f05aea6ca..5c11388516 100644
--- a/tensorflow/tools/compatibility/reorders_v2.py
+++ b/tensorflow/tools/compatibility/reorders_v2.py
@@ -65,6 +65,7 @@ reorders = {
     'tf.nn.moments': ['x', 'axes', 'shift', 'name', 'keep_dims'],
     'tf.nn.pool': ['input', 'window_shape', 'pooling_type', 'padding', 'dilation_rate', 'strides', 'name', 'data_format'],
     'tf.nn.separable_conv2d': ['input', 'depthwise_filter', 'pointwise_filter', 'strides', 'padding', 'rate', 'name', 'data_format'],
+    'tf.nn.softmax_cross_entropy_with_logits': ['_sentinel', 'labels', 'logits', 'dim', 'name'],
     'tf.nn.space_to_batch': ['input', 'paddings', 'block_size', 'name'],
     'tf.nn.space_to_depth': ['input', 'block_size', 'name', 'data_format'],
     'tf.nn.weighted_moments': ['x', 'axes', 'frequency_weights', 'name', 'keep_dims'],
diff --git a/tensorflow/tools/compatibility/testdata/test_file_v1_12.py b/tensorflow/tools/compatibility/testdata/test_file_v1_12.py
index 5ce4dd49ad..2663762aa7 100644
--- a/tensorflow/tools/compatibility/testdata/test_file_v1_12.py
+++ b/tensorflow/tools/compatibility/testdata/test_file_v1_12.py
@@ -70,6 +70,15 @@ class TestUpgrade(test_util.TensorFlowTestCase):
         [0],
         tf.argmin([[1, 3, 2]], name='abc', dimension=1))
 
+  @test_util.run_v1_only("b/120545219")
+  def testSoftmaxCrossEntropyWithLogits(self):
+    out = tf.nn.softmax_cross_entropy_with_logits(
+        logits=[0.1, 0.8], labels=[0, 1])
+    self.assertAllClose(out, 0.40318608)
+    out = tf.nn.softmax_cross_entropy_with_logits_v2(
+        logits=[0.1, 0.8], labels=[0, 1])
+    self.assertAllClose(out, 0.40318608)
+
 
 if __name__ == "__main__":
   test_lib.main()
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 9d9c5878f7..2dbbe27598 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import ast
 
 import pasta
+import six
 
 from tensorflow.tools.compatibility import ast_edits
 from tensorflow.tools.compatibility import renames_v2
@@ -94,6 +95,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.convert_to_tensor": {
             "preferred_dtype": "dtype_hint"
         },
+        "tf.nn.softmax_cross_entropy_with_logits": {
+            "dim": "axis",
+            "_sentinel": None,
+        },
         "tf.nn.softmax_cross_entropy_with_logits_v2": {
             "dim": "axis"
         },
@@ -682,6 +687,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.norm",
         "tf.reverse_sequence",
         "tf.sparse_split",
+        # tf.nn.softmax_cross_entropy_with_logits *must* be called with
+        # keyword arguments. Add keyword arguments in rare case when they
+        # are not specified.
+        "tf.nn.softmax_cross_entropy_with_logits",
     }
 
     # Functions that were reordered should be changed to the new keyword args
@@ -714,6 +723,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.to_float": self._cast_transformer,
         "tf.to_int32": self._cast_transformer,
         "tf.to_int64": self._cast_transformer,
+        "tf.nn.softmax_cross_entropy_with_logits":
+            self._softmax_cross_entropy_with_logits_transformer,
     }
 
     decay_function_comment = (
@@ -950,10 +961,6 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "'deterministic' arguments. Now it takes a single 'seed' arg. If "
             "'seed' is zero, the execution is random and deterministic "
             "otherwise",
-        "tf.nn.softmax_cross_entropy_with_logits":
-            "tf.nn.softmax_cross_entropy_with_logits behavior has changed. "
-            "'labels' needs to be wrapped with tf.stop_gradient to keep the "
-            "old behavior. Also, 'dim' argument has been renamed to 'axis'.",
         "tf.test.assert_equal_graph_def":
             "tf.assert_equal_graph_def no longer takes 'checkpoint_v2' "
             "argument. 'checkpoint_v2' now defaults to True.",
@@ -1228,6 +1235,36 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
                                                                     dtype_str)))
     return node
 
+  @staticmethod
+  def _softmax_cross_entropy_with_logits_transformer(
+      parent, node, full_name, name, logs, errors):
+    def _wrap_label(parent, old_value):
+      """Wrap labels with tf.stop_gradient."""
+      if six.PY3:
+        new_value = ast.Call(
+            ast.Name(id="tf.stop_gradient", ctx=ast.Load()),
+            [old_value], [])
+      else:
+        new_value = ast.Call(
+            ast.Name(id="tf.stop_gradient", ctx=ast.Load()),
+            [old_value], [], None, None)
+
+      # This copies the prefix and suffix on old_value to new_value.
+      pasta.ast_utils.replace_child(parent, old_value, new_value)
+      ast.copy_location(new_value, old_value)
+
+    # Check if we have a labels keyword arg
+    for karg in node.keywords:
+      if karg.arg == "labels":
+        logs.append((node.lineno, node.col_offset,
+                     "Changing labels arg of "
+                     "tf.nn.softmax_cross_entropy_with_logits to "
+                     "tf.stop_gradient(labels). Please check this "
+                     "transformation.\n"))
+        _wrap_label(karg, karg.value)
+        return node
+    return node
+
   @staticmethod
   def _batch_gather_transformer(parent, node, full_name, name, logs, errors):
     # Check if the call already has a batch_dims argument
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index d8bc0ba75a..80d86d7a2b 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -684,26 +684,32 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
     self.assertEqual(new_text, expected_text)
 
   def testSoftMaxCrossEntropyWithLogitsV2(self):
-    text = "tf.nn.softmax_cross_entropy_with_logits_v2(labels, logits, dim=2)"
+    text = (
+        "tf.nn.softmax_cross_entropy_with_logits_v2("
+        "labels=labels, logits=logits, dim=2)")
     expected_text = (
-        "tf.nn.softmax_cross_entropy_with_logits(labels, logits, axis=2)")
+        "tf.nn.softmax_cross_entropy_with_logits("
+        "labels=labels, logits=logits, axis=2)")
     _, unused_report, errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, expected_text)
 
     self.assertFalse(errors)
 
   def testSoftMaxCrossEntropyWithLogits(self):
-    text = "tf.nn.softmax_cross_entropy_with_logits(labels, logits, dim=2)"
+    text = ("tf.nn.softmax_cross_entropy_with_logits("
+            "labels=labels, logits=logits, dim=2)")
     expected_text = (
-        "tf.nn.softmax_cross_entropy_with_logits(labels, logits, dim=2)")
-    _, report, errors, new_text = self._upgrade(text)
+        "tf.nn.softmax_cross_entropy_with_logits("
+        "labels=tf.stop_gradient(labels), logits=logits, axis=2)")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, expected_text)
-    self.assertIn(
-        "tf.nn.softmax_cross_entropy_with_logits requires manual check",
-        errors[0])
-    self.assertIn(
-        "tf.nn.softmax_cross_entropy_with_logits behavior has changed. ",
-        report)
+
+    text = ("tf.nn.softmax_cross_entropy_with_logits("
+            "labels=foo(bar))")
+    expected_text = ("tf.nn.softmax_cross_entropy_with_logits("
+                     "labels=tf.stop_gradient(foo(bar)))")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(expected_text, new_text)
 
   def testSparseMatmul(self):
     text = ("tf.sparse_matmul(a, b, c, d, e, f, g)\n")
-- 
GitLab


From 09713e439363d763ca7c12d0c279b8d55d5b6053 Mon Sep 17 00:00:00 2001
From: Davide Libenzi <dlibenzi@google.com>
Date: Mon, 7 Jan 2019 17:10:18 -0800
Subject: [PATCH 0306/2345] We should be using on host shape as the device one
 can have tuples in place of complex or S64 types.

PiperOrigin-RevId: 228262394
---
 tensorflow/compiler/xrt/tests/raw_api_test.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/compiler/xrt/tests/raw_api_test.cc b/tensorflow/compiler/xrt/tests/raw_api_test.cc
index c8479cb778..be0c4b9392 100644
--- a/tensorflow/compiler/xrt/tests/raw_api_test.cc
+++ b/tensorflow/compiler/xrt/tests/raw_api_test.cc
@@ -956,6 +956,7 @@ TEST(RawApiTest, CompileAndExecuteWithS64Argument) {
   xrt::XRTExecutionConfig e;
   e.set_release_input_handles(true);
   e.set_release_compilation_handle(true);
+  e.set_return_exploded_tuple(true);
 
   Scope root = Scope::NewRootScope().WithDevice(DeviceFromFlag());
   auto e_config =
-- 
GitLab


From fbbecc81be3b1eab0c15020ce3b0566e60e25c7f Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Mon, 7 Jan 2019 17:48:53 -0800
Subject: [PATCH 0307/2345] Dynamic Padder

- Change HloGetDimensionSizeRewriter to use DynamicDimensionInference,
  when trying to get the size of a dynamic dimension, GetDimensionSize
  is replaced by the hlo instruction the represents the dynamic size
  instead.

- Implements Dynamic Padder, which uses DynamicDimensionInference to
  analyze the graph and inserts certain operations to make sure the
  result is the same as if there is no padding in the bounded shapes.

PiperOrigin-RevId: 228266743
---
 tensorflow/compiler/xla/service/BUILD         |  41 +++++
 .../compiler/xla/service/dynamic_padder.cc    | 161 ++++++++++++++++++
 .../compiler/xla/service/dynamic_padder.h     |  44 +++++
 .../xla/service/dynamic_padder_test.cc        | 152 +++++++++++++++++
 .../hlo_get_dimension_size_rewriter.cc        |  26 ++-
 .../service/hlo_get_dimension_size_rewriter.h |   4 +-
 6 files changed, 421 insertions(+), 7 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/dynamic_padder.cc
 create mode 100644 tensorflow/compiler/xla/service/dynamic_padder.h
 create mode 100644 tensorflow/compiler/xla/service/dynamic_padder_test.cc

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 9bc6218f75..ec63ae1ac6 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -1938,6 +1938,46 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "dynamic_padder",
+    srcs = ["dynamic_padder.cc"],
+    hdrs = ["dynamic_padder.h"],
+    deps = [
+        ":dynamic_dimension_inference",
+        ":hlo_dce",
+        ":hlo_pass",
+        "//tensorflow/compiler/xla:literal",
+        "//tensorflow/compiler/xla:literal_util",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:status_macros",
+        "//tensorflow/compiler/xla:util",
+        "//tensorflow/core:lib",
+        "@com_google_absl//absl/algorithm:container",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/container:flat_hash_set",
+    ],
+)
+
+tf_cc_test(
+    name = "dynamic_padder_test",
+    srcs = ["dynamic_padder_test.cc"],
+    deps = [
+        ":dynamic_padder",
+        "//tensorflow/compiler/xla:debug_options_flags",
+        "//tensorflow/compiler/xla:literal",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:test",
+        "//tensorflow/compiler/xla:test_helpers",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/client:xla_builder",
+        "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/compiler/xla/service:hlo_matchers",
+        "//tensorflow/compiler/xla/service:hlo_runner",
+        "//tensorflow/compiler/xla/tests:hlo_test_base",
+        "//tensorflow/core:test",
+    ],
+)
+
 tf_cc_test(
     name = "dynamic_dimension_inference_test",
     srcs = ["dynamic_dimension_inference_test.cc"],
@@ -2981,6 +3021,7 @@ cc_library(
     srcs = ["hlo_get_dimension_size_rewriter.cc"],
     hdrs = ["hlo_get_dimension_size_rewriter.h"],
     deps = [
+        ":dynamic_dimension_inference",
         ":hlo",
         ":hlo_pass",
         ":shape_inference",
diff --git a/tensorflow/compiler/xla/service/dynamic_padder.cc b/tensorflow/compiler/xla/service/dynamic_padder.cc
new file mode 100644
index 0000000000..4db280f817
--- /dev/null
+++ b/tensorflow/compiler/xla/service/dynamic_padder.cc
@@ -0,0 +1,161 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/compiler/xla/service/dynamic_padder.h"
+
+#include <algorithm>
+#include <vector>
+
+#include "absl/algorithm/container.h"
+#include "absl/container/flat_hash_map.h"
+
+#include "absl/container/flat_hash_set.h"
+#include "tensorflow/compiler/xla/literal.h"
+#include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/compiler/xla/service/dynamic_dimension_inference.h"
+#include "tensorflow/compiler/xla/service/hlo_dce.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/status_macros.h"
+#include "tensorflow/compiler/xla/util.h"
+
+#include "tensorflow/core/lib/core/errors.h"
+
+namespace xla {
+
+namespace {
+
+// ChooseIdentityValue looks at the instruction and returns a identity value
+// which, when padded, doesn't change the result of the instruction.
+//
+// nullopt is returned if padding doesn't need to be reset.
+StatusOr<HloInstruction*> ChooseIdentityValue(HloInstruction* inst) {
+  HloComputation* comp = inst->parent();
+  // Padding on elementwise operation doesn't affect the result of the effective
+  // data.
+  if (inst->IsElementwise()) {
+    return nullptr;
+  }
+
+  switch (inst->opcode()) {
+    case HloOpcode::kReduce:
+    case HloOpcode::kReduceWindow: {
+      // Because of the way we do reduce, we already require the `init` operand
+      // of hlo reduce instruction to be identity value. Here we reuse the
+      // operand.
+      return inst->mutable_operand(1);
+    }
+
+    case HloOpcode::kConvolution:
+    case HloOpcode::kDot: {
+      // Use 0 as padding value for convolution and dot.
+      PrimitiveType ptype = inst->shape().element_type();
+      return comp->AddInstruction(
+          HloInstruction::CreateConstant(LiteralUtil::Zero(ptype)));
+    }
+
+    case HloOpcode::kPad: {
+      return inst->mutable_operand(1);
+    }
+    case HloOpcode::kParameter:
+    case HloOpcode::kGetDimensionSize:
+    case HloOpcode::kReshape:
+    case HloOpcode::kTuple:
+    case HloOpcode::kAllReduce:
+    case HloOpcode::kBroadcast:
+      return nullptr;
+    default:
+      return UnimplementedStrCat("Unimplimented padding for instruction: ",
+                                 inst->ToString());
+  }
+}
+
+}  // namespace
+
+StatusOr<bool> DynamicPadder::Run(HloModule* module) {
+  bool changed = false;
+  VLOG(2) << "Pre DynamicPadder HLO:";
+  XLA_VLOG_LINES(2, module->ToString());
+  TF_ASSIGN_OR_RETURN(DynamicDimensionInference dynamic_dimension_inference,
+                      DynamicDimensionInference::Run(module));
+
+  for (HloComputation* computation : module->computations()) {
+    for (HloInstruction* inst : computation->instructions()) {
+      for (int64 operand_num = 0; operand_num < inst->operand_count();
+           ++operand_num) {
+        HloInstruction* operand = inst->mutable_operand(operand_num);
+        if (!operand->shape().IsArray()) {
+          continue;
+        }
+        for (int64 dim = 0; dim < operand->shape().rank(); ++dim) {
+          HloInstruction* dynamic_size =
+              dynamic_dimension_inference.GetDynamicSize(operand, {}, dim);
+          if (dynamic_size == nullptr) {
+            continue;
+          }
+          VLOG(1) << "Has dynamic dimension of operand" << operand_num << " @"
+                  << dim;
+          TF_ASSIGN_OR_RETURN(HloInstruction * identity_value,
+                              ChooseIdentityValue(inst));
+          if (identity_value == nullptr) {
+            continue;
+          }
+
+          // For each dimension, first generates a mask representing the
+          // effective area of data and padded area of data using iota and
+          // dynamic_size. For example, given a dimension of 7 elements and 5
+          // effective elements:
+          //
+          // iota = [0, 1, 2, 3, 4, 5, 6]
+          // broadcast_dynamic_size = [5, 5, 5, 5, 5, 5, 5]
+          // mask = lt(iota, broadcast_dynamic_size) = [t, t, t, t, t, f, f]
+          //
+          // Once the mask is generated, the input data is then padded using the
+          // mask and pad value.
+          //
+          const Shape mask_shape =
+              ShapeUtil::ChangeElementType(operand->shape(), xla::U32);
+          const Shape pred_shape =
+              ShapeUtil::ChangeElementType(operand->shape(), xla::PRED);
+          HloInstruction* iota = computation->AddInstruction(
+              HloInstruction::CreateIota(mask_shape, dim));
+
+          HloInstruction* broadcasted_effective_size =
+              computation->AddInstruction(HloInstruction::CreateBroadcast(
+                  mask_shape, dynamic_size, {}));
+          HloInstruction* pred = computation->AddInstruction(
+              HloInstruction::CreateBinary(pred_shape, HloOpcode::kLt, iota,
+                                           broadcasted_effective_size));
+
+          HloInstruction* broadcasted_identity_value =
+              computation->AddInstruction(HloInstruction::CreateBroadcast(
+                  operand->shape(), identity_value, {}));
+          HloInstruction* padded =
+              computation->AddInstruction(HloInstruction::CreateTernary(
+                  operand->shape(), HloOpcode::kSelect, pred, operand,
+                  broadcasted_identity_value));
+          TF_RETURN_IF_ERROR(inst->ReplaceOperandWith(operand_num, padded));
+          operand = inst->mutable_operand(operand_num);
+          changed = true;
+        }
+      }
+    }
+  }
+  HloDCE dce;
+  TF_ASSIGN_OR_RETURN(changed, dce.Run(module));
+  VLOG(2) << "Post DynamicPadder HLO:";
+  XLA_VLOG_LINES(2, module->ToString());
+  return changed;
+}
+
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/dynamic_padder.h b/tensorflow/compiler/xla/service/dynamic_padder.h
new file mode 100644
index 0000000000..509269f7f5
--- /dev/null
+++ b/tensorflow/compiler/xla/service/dynamic_padder.h
@@ -0,0 +1,44 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_DYNAMIC_PADDER_H_
+#define TENSORFLOW_COMPILER_XLA_SERVICE_DYNAMIC_PADDER_H_
+
+#include "tensorflow/compiler/xla/service/dynamic_dimension_inference.h"
+#include "tensorflow/compiler/xla/service/hlo_pass_interface.h"
+
+namespace xla {
+
+// With bounded shapes, only part of the shape contains effective data and the
+// rest contains padded data, whose value can be anything depending on the
+// source of the data. When a bounded shape is directly consumed by an
+// instruction that collapses dimensions (reduce for example), the padding data
+// would affect result of the instruction.
+//
+// DynamicPadder uses DynamicDimensionInference to detect bounded shapes in a
+// hlo module, it then inserts certain instructions to reset the padding into an
+// identity value so that in doesn't affect the result of subsequent
+// instruction. For example, it'd reset the padding to 0 before a bounded shape
+// is consumed by a reduce-sum.
+class DynamicPadder : public HloModulePass {
+ public:
+  absl::string_view name() const override { return "dynamic_padder"; }
+
+  StatusOr<bool> Run(HloModule* module) override;
+};
+
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_DYNAMIC_PADDER_H_
diff --git a/tensorflow/compiler/xla/service/dynamic_padder_test.cc b/tensorflow/compiler/xla/service/dynamic_padder_test.cc
new file mode 100644
index 0000000000..55a11286e4
--- /dev/null
+++ b/tensorflow/compiler/xla/service/dynamic_padder_test.cc
@@ -0,0 +1,152 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/dynamic_padder.h"
+
+#include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/compiler/xla/literal.h"
+#include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_matchers.h"
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/service/hlo_opcode.h"
+#include "tensorflow/compiler/xla/service/hlo_runner.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/test.h"
+#include "tensorflow/compiler/xla/test_helpers.h"
+#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/test_benchmark.h"
+
+namespace op = xla::testing::opcode_matchers;
+
+namespace xla {
+namespace {
+
+class DynamicPadderTest : public HloTestBase {
+ protected:
+  DynamicPadderTest() : HloTestBase() { module_ = CreateNewVerifiedModule(); }
+
+  StatusOr<bool> RunPadder() {
+    hlo_graph_dumper::MaybeDumpHloModule(*module_, "Before padder");
+
+    DynamicPadder padder;
+
+    return padder.Run(module_.get());
+  }
+
+  void ExpectPadded(const HloInstruction* inst) {
+    EXPECT_THAT(inst,
+                op::Select(op::Lt(op::Iota(), op::Broadcast(op::Parameter())),
+                           ::testing::_, op::Broadcast()));
+  }
+
+  HloComputation* GetScalarAddComputation() {
+    auto embedded_builder = HloComputation::Builder("add");
+    auto lhs = embedded_builder.AddInstruction(HloInstruction::CreateParameter(
+        0, ShapeUtil::MakeShape(F32, {}), "lhs"));
+    auto rhs = embedded_builder.AddInstruction(HloInstruction::CreateParameter(
+        1, ShapeUtil::MakeShape(F32, {}), "rhs"));
+    embedded_builder.AddInstruction(
+        HloInstruction::CreateBinary(lhs->shape(), HloOpcode::kAdd, lhs, rhs));
+    return module_->AddEmbeddedComputation(embedded_builder.Build());
+  }
+
+  std::unique_ptr<HloModule> module_;
+  const Shape scalar_shape_ = ShapeUtil::MakeShape(U32, {});
+};
+
+TEST_F(DynamicPadderTest, ReduceTest) {
+  auto builder = HloComputation::Builder(TestName());
+  auto input_shape = ShapeUtil::MakeShape(F32, {1, 2, 2});
+  auto reduce_shape = ShapeUtil::MakeShape(F32, {2});
+
+  auto data_param = builder.AddInstruction(
+      HloInstruction::CreateParameter(0, input_shape, "data_param"));
+  builder.AddInstruction(
+      HloInstruction::CreateParameter(1, scalar_shape_, "size_param"));
+
+  auto negate = builder.AddInstruction(
+      HloInstruction::CreateUnary(input_shape, HloOpcode::kNegate, data_param));
+
+  auto init = builder.AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<float>(0.0)));
+
+  auto reduce = builder.AddInstruction(HloInstruction::CreateReduce(
+      reduce_shape, negate, init, {0, 2}, GetScalarAddComputation()));
+
+  module_->AddEntryComputation(builder.Build());
+
+  // Set up dynamic parameter binding.
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{1, {}},
+      DynamicParameterBinding::DynamicDimension{0, {}, 1}));
+
+  TF_ASSERT_OK(RunPadder().status());
+
+  ExpectPadded(reduce->operand(0));
+}
+
+TEST_F(DynamicPadderTest, ConvolutionTest) {
+  auto builder = HloComputation::Builder(TestName());
+  constexpr int xdim = 3;
+  constexpr int ydim = 2;
+  constexpr int zdim = 1;
+  auto xy_shape = ShapeUtil::MakeShape(F32, {xdim, ydim});
+  auto yz_shape = ShapeUtil::MakeShape(F32, {ydim, zdim});
+  auto zx_shape = ShapeUtil::MakeShape(F32, {zdim, xdim});
+
+  auto* a_param = builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/0, xy_shape, "A"));
+  auto* b_param = builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/1, yz_shape, "B"));
+  builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/2, scalar_shape_, "size_param"));
+
+  auto dnums = XlaBuilder::CreateDefaultConvDimensionNumbers(0);
+
+  dnums.set_kernel_input_feature_dimension(0);
+  dnums.set_kernel_output_feature_dimension(1);
+  dnums.set_input_batch_dimension(0);
+  dnums.set_output_batch_dimension(1);
+  dnums.set_output_feature_dimension(0);
+
+  Window window;
+
+  auto* conv = builder.AddInstruction(HloInstruction::CreateConvolve(
+      zx_shape, a_param, b_param, /*feature_group_count=*/1,
+      /*batch_group_count=*/1, window, dnums,
+      HloTestBase::DefaultPrecisionConfig(2)));
+
+  module_->AddEntryComputation(builder.Build());
+
+  // Set up dynamic parameter binding for non-contracting dimension.
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{2, {}},
+      DynamicParameterBinding::DynamicDimension{0, {}, 0}));
+
+  // Set up binding for contracting dimensions.
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{2, {}},
+      DynamicParameterBinding::DynamicDimension{0, {}, 1}));
+
+  TF_ASSERT_OK(RunPadder().status());
+
+  ExpectPadded(conv->operand(0));
+}
+
+}  // namespace
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.cc b/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.cc
index c919dbd82d..862b202971 100644
--- a/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.cc
+++ b/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.cc
@@ -17,6 +17,7 @@ limitations under the License.
 
 #include "absl/algorithm/container.h"
 #include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/compiler/xla/service/dynamic_dimension_inference.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
 #include "tensorflow/compiler/xla/service/shape_inference.h"
@@ -25,7 +26,9 @@ namespace xla {
 
 namespace {
 
-StatusOr<bool> ReplaceGetSize(HloInstruction* instr) {
+StatusOr<bool> ReplaceGetSize(
+    HloInstruction* instr,
+    const DynamicDimensionInference* dynamic_dimension_inference) {
   if (instr->opcode() != HloOpcode::kGetDimensionSize) {
     return false;
   }
@@ -36,10 +39,18 @@ StatusOr<bool> ReplaceGetSize(HloInstruction* instr) {
                           instr->operand(0)->shape(), instr->dimension()));
   TF_RET_CHECK(ShapeUtil::Equal(instr->shape(), legal_shape));
   TF_RET_CHECK(ShapeUtil::HasPrimitiveType(instr->shape(), U32));
-  uint32 size = instr->operand(0)->shape().dimensions(instr->dimension());
-  HloInstruction* new_instr = computation->AddInstruction(
-      HloInstruction::CreateConstant(LiteralUtil::CreateR0<uint32>(size)));
-  TF_RETURN_IF_ERROR(instr->ReplaceAllUsesWith(new_instr));
+  HloInstruction* operand = instr->mutable_operand(0);
+  int64 dim = instr->dimension();
+  HloInstruction* dynamic_size =
+      dynamic_dimension_inference->GetDynamicSize(operand, {}, dim);
+  if (dynamic_size != nullptr) {
+    TF_RETURN_IF_ERROR(instr->ReplaceAllUsesWith(dynamic_size));
+  } else {
+    uint32 size = instr->operand(0)->shape().dimensions(dim);
+    HloInstruction* new_instr = computation->AddInstruction(
+        HloInstruction::CreateConstant(LiteralUtil::CreateR0<uint32>(size)));
+    TF_RETURN_IF_ERROR(instr->ReplaceAllUsesWith(new_instr));
+  }
   return true;
 }
 
@@ -48,10 +59,13 @@ StatusOr<bool> ReplaceGetSize(HloInstruction* instr) {
 StatusOr<bool> HloGetDimensionSizeRewriter::Run(HloModule* module) {
   bool changed = false;
   HloProto proto;
+  TF_ASSIGN_OR_RETURN(DynamicDimensionInference inference,
+                      DynamicDimensionInference::Run(module));
   *proto.mutable_hlo_module() = module->ToProto();
   for (auto* computation : module->computations()) {
     for (auto instruction : computation->instructions()) {
-      TF_ASSIGN_OR_RETURN(bool replaced, ReplaceGetSize(instruction));
+      TF_ASSIGN_OR_RETURN(bool replaced,
+                          ReplaceGetSize(instruction, &inference));
       changed = changed || replaced;
     }
   }
diff --git a/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.h b/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.h
index 30f44c23a8..9aa79fe66b 100644
--- a/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.h
+++ b/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.h
@@ -21,7 +21,9 @@ limitations under the License.
 
 namespace xla {
 
-// Pass to replace a kGetDimensionSize instruction with a constant instruction.
+// Pass to replace a kGetDimensionSize instruction with a hlo instruction
+// representing the dynamic size if the dimension is dynamic, otherwise a
+// constant instruction representing the static size.
 class HloGetDimensionSizeRewriter : public HloModulePass {
  public:
   absl::string_view name() const override {
-- 
GitLab


From 5ed2c498eddc0b95ae43d6f4c39e29011cd97694 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Mon, 7 Jan 2019 18:49:39 -0800
Subject: [PATCH 0308/2345] [XLA] Add testcase for ptxas bug.

This ptxas bug caused the wrong output on this testcase.

We never received a clear explanation of what was going on from nvidia, so I
can't say more than "it used to not work, now it does work."

PiperOrigin-RevId: 228271902
---
 tensorflow/compiler/xla/tests/BUILD           | 15 ++++
 .../compiler/xla/tests/ptxas_bug_120501638.cc | 82 +++++++++++++++++++
 2 files changed, 97 insertions(+)
 create mode 100644 tensorflow/compiler/xla/tests/ptxas_bug_120501638.cc

diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 9433264917..106b3fd6c9 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -2118,3 +2118,18 @@ tf_cc_test(
         "@com_google_absl//absl/synchronization",
     ],
 )
+
+xla_test(
+    name = "ptxas_bug_120501638",
+    srcs = ["ptxas_bug_120501638.cc"],
+    tags = [
+        # Disabled in OSS until nvidia publicly releases a fixed ptxas.
+        "no_oss",
+    ],
+    deps = [
+        ":hlo_test_base",
+        ":xla_internal_test_main",  # fixdeps: keep
+        "//tensorflow/compiler/xla:debug_options_flags",
+        "//tensorflow/compiler/xla:test",
+    ],
+)
diff --git a/tensorflow/compiler/xla/tests/ptxas_bug_120501638.cc b/tensorflow/compiler/xla/tests/ptxas_bug_120501638.cc
new file mode 100644
index 0000000000..0e5d7db97e
--- /dev/null
+++ b/tensorflow/compiler/xla/tests/ptxas_bug_120501638.cc
@@ -0,0 +1,82 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/debug_options_flags.h"
+#include "tensorflow/compiler/xla/test.h"
+#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+#include "tensorflow/compiler/xla/tests/test_macros.h"
+
+namespace xla {
+namespace {
+
+class PtxasBugTest : public HloTestBase {};
+
+// Checks for a bug in ptxas, tracked as Google bug 120501638, and nvidia bug
+// 2459377.  We never received an explanation of what exactly was going wrong
+// here in ptxas.  Known-bad in ptxas 10.0.145, known-good in ptxas 10.0.249.
+TEST_F(PtxasBugTest, DoIt) {
+  const char* const kModuleStr = R"(
+HloModule test
+
+add_F32.14 {
+  lhs.15 = f32[] parameter(0)
+  rhs.16 = f32[] parameter(1)
+  ROOT add.17 = f32[] add(lhs.15, rhs.16)
+}
+
+ENTRY testcase {
+  arg0.1 = f32[2,5,2]{2,1,0} parameter(0)
+  reshape.2 = f32[2,5,2]{2,1,0} reshape(arg0.1)
+  constant.3 = f32[] constant(0)
+  pad.4 = f32[2,6,2]{2,1,0} pad(reshape.2, constant.3), padding=0_0x0_1x0_0
+  reshape.5 = f32[2,3,2,2]{3,2,1,0} reshape(pad.4)
+  transpose.6 = f32[2,2,3,2]{3,0,2,1} transpose(reshape.5), dimensions={2,0,1,3}
+  reshape.7 = f32[4,3,2]{2,1,0} reshape(transpose.6)
+  reshape.8 = f32[4,1,3,2]{3,2,1,0} reshape(reshape.7)
+  transpose.9 = f32[4,2,1,3]{1,3,2,0} transpose(reshape.8), dimensions={0,3,1,2}
+  convert.10 = f32[4,2,1,3]{1,3,2,0} convert(transpose.9)
+  constant.12 = f32[] constant(0)
+  pad.13 = f32[4,2,1,3]{3,2,1,0} pad(convert.10, constant.12), padding=0_0x0_0x0_0x0_0
+  constant.11 = f32[] constant(0)
+  reduce-window.18 = f32[4,2,1,3]{3,2,1,0} reduce-window(pad.13, constant.11),
+    window={size=1x1x1x1}, to_apply=add_F32.14
+  constant.19 = f32[] constant(1)
+  broadcast.20 = f32[4,2,1,3]{3,2,1,0} broadcast(constant.19), dimensions={}
+  divide.21 = f32[4,2,1,3]{3,2,1,0} divide(reduce-window.18, broadcast.20)
+  convert.22 = f32[4,2,1,3]{3,2,1,0} convert(divide.21)
+  transpose.23 = f32[4,1,3,2]{2,1,3,0} transpose(convert.22), dimensions={0,2,3,1}
+  reshape.24 = f32[4,3,2]{2,1,0} reshape(transpose.23)
+  reshape.25 = f32[2,2,3,2]{3,2,1,0} reshape(reshape.24)
+  transpose.26 = f32[2,3,2,2]{3,1,0,2} transpose(reshape.25), dimensions={1,2,0,3}
+  reshape.27 = f32[2,6,2]{2,1,0} reshape(transpose.26)
+  slice.28 = f32[2,5,2]{2,1,0} slice(reshape.27), slice={[0:2], [0:5], [0:2]}
+  reshape.29 = f32[2,5,2]{2,1,0} reshape(slice.28)
+  tuple.30 = (f32[2,5,2]{2,1,0}) tuple(reshape.29)
+  ROOT get-tuple-element.31 = f32[2,5,2]{2,1,0} get-tuple-element(tuple.30), index=0
+})";
+
+  // Create a module with the true-default flags, not the default-for-testing
+  // flags.  In particular, true-default flags enable unrolling, whereas for
+  // testing we disable unrolling, and this bug doesn't trigger without
+  // unrolling.
+  HloModuleConfig config;
+  config.set_debug_options(DefaultDebugOptionsIgnoringFlags());
+  TF_ASSERT_OK_AND_ASSIGN(auto module,
+                          ParseAndReturnVerifiedModule(kModuleStr, config));
+  EXPECT_TRUE(RunAndCompare(std::move(module), ErrorSpec{0.01, 0.01}));
+}
+
+}  // anonymous namespace
+}  // namespace xla
-- 
GitLab


From 9b884d8392706a82af674adbbff862ca2de593a6 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Mon, 7 Jan 2019 19:02:54 -0800
Subject: [PATCH 0309/2345] [XLA] Don't constant-fold instructions which
 transitively contain kRng or kAfterAll.

Constant-folding avoids folding kRng/kAfterAll instructions, but it also needs
to avoid folding instructions which *may contain* kRng/kAfterAll, because e.g.
folding a call that contains an rng is morally equivalent to folding an rng.

PiperOrigin-RevId: 228272774
---
 .../xla/service/hlo_constant_folding.cc       | 66 +++++++++++++++----
 .../xla/service/hlo_constant_folding_test.cc  | 46 +++++++++++++
 2 files changed, 98 insertions(+), 14 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_constant_folding.cc b/tensorflow/compiler/xla/service/hlo_constant_folding.cc
index 799cc9fd91..e7ed858e8c 100644
--- a/tensorflow/compiler/xla/service/hlo_constant_folding.cc
+++ b/tensorflow/compiler/xla/service/hlo_constant_folding.cc
@@ -35,6 +35,34 @@ limitations under the License.
 
 namespace xla {
 
+// Checks whether instr is or transitively contains an instruction that we
+// shouldn't fold.
+//
+// Specifically, we don't fold kRng or kAfterAll instructions:
+//
+//  - kRng is already marked as side-effecting and so is skipped elsewhere, but
+//    we check for it here.  Even kRng weren't side-effecting and took an
+//    explicit seed, we *still* wouldn't want to constant-fold it, because the
+//    evaluator's handling of rng is not guaranteed to be identical to any
+//    particular backend's rng.
+//
+//  - kAfterAll needs to be skipped because a kAfterAll op with no args can
+//    currently materialize a token "out of thin air".  TODO(b/110532604):
+//    Remove this check once AfterAll requires at least one operand, in which
+//    case constant folding will be impossible.
+static bool IsOrContainsIllegalInstr(const HloInstruction* instr) {
+  if (instr->opcode() == HloOpcode::kAfterAll ||
+      instr->opcode() == HloOpcode::kRng) {
+    return true;
+  }
+  for (const HloComputation* c : instr->called_computations()) {
+    if (absl::c_any_of(c->instructions(), IsOrContainsIllegalInstr)) {
+      return true;
+    }
+  }
+  return false;
+}
+
 StatusOr<bool> HloConstantFolding::Run(HloModule* module) {
   // Limit the constant folding to 0 iterations to skip folding loops. This
   // retains the behavior from before while loop support in HloEvaluator and may
@@ -52,26 +80,24 @@ StatusOr<bool> HloConstantFolding::Run(HloModule* module) {
           computation->root_instruction() != instruction) {
         continue;
       }
-      // Skip Constant, Parameter, Tuple, AfterAll, Rng operations.
-      // Tuple constants are not directly supported by any backends, hence
-      // folding Tuple is not useful and would in fact be expanded back into
-      // kTuple by Algebraic Simplifier.
-      // TODO(b/110532604): Enable AfterAll once AfterAll requires at least one
-      // operand in which case constant folding will be impossible and this
-      // special case is not necessary.
-      if (instruction->opcode() == HloOpcode::kParameter ||
-          instruction->opcode() == HloOpcode::kConstant ||
-          instruction->opcode() == HloOpcode::kTuple ||
-          instruction->opcode() == HloOpcode::kAfterAll ||
-          instruction->opcode() == HloOpcode::kRng) {
-        continue;
-      }
 
       // Skip instructions with non-constant operands.
       if (!hlo_query::AllOperandsAreConstants(*instruction)) {
         continue;
       }
 
+      // Don't fold Constant, Parameter, and Tuple instructions.  Tuple
+      // constants are not directly supported by any backends, hence folding
+      // Tuple is not useful and would in fact be expanded back into kTuple by
+      // Algebraic Simplifier.
+      //
+      // (We do allow folding subcomputations that contain these instructions.)
+      if (instruction->opcode() == HloOpcode::kParameter ||
+          instruction->opcode() == HloOpcode::kConstant ||
+          instruction->opcode() == HloOpcode::kTuple) {
+        continue;
+      }
+
       // Broadcasts dramatically increase the size of constants, which is often
       // detrimental to performance and memory capacity, so do not fold
       // broadcasts.
@@ -80,6 +106,18 @@ StatusOr<bool> HloConstantFolding::Run(HloModule* module) {
         continue;
       }
 
+      // Check for instructions that we can't fold even if they appear inside of
+      // a subcomputation (e.g. a kCall).
+      if (IsOrContainsIllegalInstr(instruction)) {
+        continue;
+      }
+
+      // Don't constant-fold side-effecting instructions or instructions which
+      // contain side-effecting instructions.
+      if (instruction->HasSideEffect()) {
+        continue;
+      }
+
       // Don't constant fold unless it's a net positive or the output is small.
       if (instruction->shape().IsArray()) {
         int64 elements_in_removed_operands = 0;
diff --git a/tensorflow/compiler/xla/service/hlo_constant_folding_test.cc b/tensorflow/compiler/xla/service/hlo_constant_folding_test.cc
index 92b748d813..4bdc980c9a 100644
--- a/tensorflow/compiler/xla/service/hlo_constant_folding_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_constant_folding_test.cc
@@ -268,5 +268,51 @@ TEST_F(HloConstantFoldingTest, DoesNotFoldLargePad) {
               GmockMatch(m::Pad(m::Constant(), m::Constant())));
 }
 
+TEST_F(HloConstantFoldingTest, DontFoldSubcomputationContainingAfterAll) {
+  const char* const kModuleStr = R"(
+  HloModule test
+
+  Fn {
+    tok = token[] after-all()
+    ROOT root = f32[10] iota(), iota_dimension=0
+  }
+
+  ENTRY entry {
+    ROOT call = f32[10] call(), to_apply=Fn
+  })";
+  TF_ASSERT_OK_AND_ASSIGN(auto module,
+                          ParseAndReturnVerifiedModule(kModuleStr));
+  HloConstantFolding constant_folding;
+  TF_ASSERT_OK_AND_ASSIGN(bool result,
+                          RunHloPass(&constant_folding, module.get()));
+  EXPECT_FALSE(result);
+}
+
+TEST_F(HloConstantFoldingTest,
+       DontFoldSubcomputationTransitivelyContainingRng) {
+  const char* const kModuleStr = R"(
+  HloModule test
+
+  InnerFn {
+    c0 = f32[] constant(0)
+    c1 = f32[] constant(1)
+    ROOT rng = f32[10] rng(c0, c1), distribution=rng_uniform
+  }
+
+  Fn {
+    ROOT fusion = f32[10] fusion(), kind=kLoop, calls=InnerFn
+  }
+
+  ENTRY entry {
+    ROOT call = f32[10] call(), to_apply=Fn
+  })";
+  TF_ASSERT_OK_AND_ASSIGN(auto module,
+                          ParseAndReturnVerifiedModule(kModuleStr));
+  HloConstantFolding constant_folding;
+  TF_ASSERT_OK_AND_ASSIGN(bool result,
+                          RunHloPass(&constant_folding, module.get()));
+  EXPECT_FALSE(result);
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 5265604a31dcad7e37a326b1f7d5f1ff006968fb Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 19:17:05 -0800
Subject: [PATCH 0310/2345] [convergence_tools]: Moving the norm, nan, and max
 computations inside of TPU.

PiperOrigin-RevId: 228273700
---
 .../contrib/tpu/python/tpu/tensor_tracer.py   | 124 ++++++++++++------
 1 file changed, 83 insertions(+), 41 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py b/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py
index 66689cde2f..c7e29860e5 100644
--- a/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py
+++ b/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py
@@ -377,10 +377,7 @@ class TensorTracer(object):
       return True
     # Reasons for not including following op types:
     #    Assign: cause incorrect result with CPU tracing.
-    #    others: compilation problems.
-    # TODO(deveci): Check if 'Pack', 'Shape', 'Reshape', 'ArgMin', 'ArgMax'
-    #               are still unsafe now that we have handled int64 tensors.
-    if op.type in ['Assign', 'Pack', 'Shape', 'Reshape', 'ArgMin', 'ArgMax']:
+    if op.type in ['Assign']:
       return True
     return False
 
@@ -471,7 +468,7 @@ class TensorTracer(object):
                 temporarily_marked_ops, sorted_ops)
       # pylint: disable=protected-access
       for ctrl_output_op in op._control_outputs:
-      # pylint: enable=protected-access
+        # pylint: enable=protected-access
         visit(ctrl_output_op, cycle, permanently_marked_ops,
               temporarily_marked_ops, sorted_ops)
       temporarily_marked_ops.remove(op)
@@ -737,6 +734,59 @@ class TensorTracer(object):
       self._write_report('%d "%s"\n'%(i, l[i].name))
     self._write_report('%s %s\n'%(_MARKER_SECTION_END, _SECTION_NAME_GRAPH))
 
+  def _preprocess_traced_tensor(self, tensor):
+    """Computes NAN/Norm/Max on TPUs before sending to CPU.
+
+    Args:
+      tensor: The tensor to be traced.
+    Returns:
+      A tensor that should be input to the trace_function.
+    Raises:
+      RuntimeError: If the trace mode is invalid.
+    """
+
+    def _detect_nan_inf(tensor):
+      """Trace function for detecting any NaN/Inf in the tensor."""
+
+      if tensor.dtype.is_floating:
+        output_tensor = math_ops.reduce_any(
+            gen_math_ops.logical_or(
+                gen_math_ops.is_nan(tensor), gen_math_ops.is_inf(tensor)))
+      else:
+        output_tensor = constant_op.constant(False)
+      # The shape has to be 1. Set it if it does not have the information.
+      output_tensor = array_ops.reshape(output_tensor, [1])
+      return output_tensor
+
+    def _show_norm(tensor):
+      tensor = math_ops.cast(tensor, dtypes.float32)
+      output_tensor = linalg_ops.norm(tensor)
+      # The shape has to be 1. Set it if it does not have the information.
+      output_tensor = array_ops.reshape(output_tensor, [1])
+      return output_tensor
+
+    def _show_max_abs(tensor):
+      tensor = math_ops.cast(tensor, dtypes.float32)
+      output_tensor = math_ops.reduce_max(math_ops.abs(tensor))
+      zero = constant_op.constant(0, dtypes.float32)
+      output_tensor = gen_math_ops.maximum(zero, output_tensor)
+      # The shape has to be 1. Set it if it does not have the information.
+      output_tensor = array_ops.reshape(output_tensor, [1])
+      return output_tensor
+
+    if self._trace_mode == _TRACE_MODE_NAN_INF:
+      return _detect_nan_inf(tensor)
+    if self._trace_mode == _TRACE_MODE_PART_TENSOR:
+      return tensor
+    if self._trace_mode == _TRACE_MODE_FULL_TENSOR:
+      return tensor
+    if self._trace_mode == _TRACE_MODE_NORM:
+      return _show_norm(tensor)
+    if self._trace_mode == _TRACE_MODE_MAX_ABS:
+      return _show_max_abs(tensor)
+    raise RuntimeError(
+        'Tensor trace fun for %s is not yet implemented' % self._trace_mode)
+
   def _make_tensor_trace_fun(self, tensor_name):
     """Makes the tensor tracing function called by outside compilation.
 
@@ -787,29 +837,6 @@ class TensorTracer(object):
       with ops.control_dependencies([print_op]):
         return array_ops.identity(tensor).op
 
-    def _detect_nan_inf(tensor):
-      """Trace function for detecting any NaN/Inf in the tensor."""
-
-      if tensor.dtype.is_floating:
-        output_tensor = math_ops.reduce_any(
-            gen_math_ops.logical_or(gen_math_ops.is_nan(tensor),
-                                    gen_math_ops.is_inf(tensor)))
-      else:
-        output_tensor = constant_op.constant(False)
-
-      return _print_tensor(tensor_name, -1, tensor, output_tensor)
-
-    def _show_norm(tensor):
-      tensor = math_ops.cast(tensor, dtypes.float64)
-      output_tensor = linalg_ops.norm(tensor)
-      return _print_tensor(tensor_name, -1, tensor, output_tensor)
-
-    def _show_max_abs(tensor):
-      output_tensor = math_ops.cast(math_ops.reduce_max(math_ops.abs(tensor)),
-                                    dtypes.float64)
-      zero = constant_op.constant(0, dtypes.float64)
-      output_tensor = gen_math_ops.maximum(zero, output_tensor)
-      return _print_tensor(tensor_name, -1, tensor, output_tensor)
 
     def _show_part_tensor(tensor):
       """Trace function for printing part of the tensor."""
@@ -822,16 +849,17 @@ class TensorTracer(object):
 
       return _print_tensor(tensor_name, -1, tensor, tensor)
 
-    if self._trace_mode == _TRACE_MODE_NAN_INF:
-      return _detect_nan_inf
     if self._trace_mode == _TRACE_MODE_PART_TENSOR:
       return _show_part_tensor
-    if self._trace_mode == _TRACE_MODE_FULL_TENSOR:
+    # The input tensor has a shape of "[1]" for _TRACE_MODE_NAN_INF,
+    # _TRACE_MODE_NORM, and _TRACE_MODE_MAX_ABS, as related computations are
+    # performed within TPUs and only their results are transferred to CPU.
+    # Simply, print the full tensor for these trace modes.
+    if self._trace_mode in [
+        _TRACE_MODE_NAN_INF, _TRACE_MODE_NORM, _TRACE_MODE_FULL_TENSOR,
+        _TRACE_MODE_MAX_ABS
+    ]:
       return _show_full_tensor
-    if self._trace_mode == _TRACE_MODE_NORM:
-      return _show_norm
-    if self._trace_mode == _TRACE_MODE_MAX_ABS:
-      return _show_max_abs
 
     raise RuntimeError('Tensor trace fun for %s is not yet implemented'
                        %self._trace_mode)
@@ -889,9 +917,18 @@ class TensorTracer(object):
           op_id, _REASON_USER_EXCLUDED)
       return True
     if not out_tensor.get_shape().is_fully_defined():
-      self._instrument_records[out_tensor.name] = TensorTracer.reason(
-          op_id, _REASON_DYNAMIC_SHAPE)
-      return True
+      # If trace mode is nan-inf, norm or max, then the tensor will be reduced
+      # to a scalar before the outside compilation call.
+      if self._trace_mode in [
+          _TRACE_MODE_NAN_INF, _TRACE_MODE_NORM, _TRACE_MODE_MAX_ABS
+      ]:
+        self._instrument_records[out_tensor.name] = TensorTracer.reason(
+            op_id, _REASON_TENSOR_GET_TRACED)
+        return False
+      else:
+        self._instrument_records[out_tensor.name] = TensorTracer.reason(
+            op_id, _REASON_DYNAMIC_SHAPE)
+        return True
     rank = len(out_tensor.shape)
     if rank < 1:
       # scalar
@@ -1000,11 +1037,15 @@ class TensorTracer(object):
         if self._skip_tensor(op_id, out_tensor, user_included,
                              user_excluded):
           continue
+        # Create the list of consumers before calling _preprocess_traced_tensor.
+        # Otherwise, adding control input below, will introduce a cycle in the
+        # graph.
         consumers = out_tensor.consumers()
         tensor_name = out_tensor.name
-        out_tensor = _cast_unsupported_dtypes(out_tensor)
+        processed_out_tensor = self._preprocess_traced_tensor(out_tensor)
+        processed_out_tensor = _cast_unsupported_dtypes(processed_out_tensor)
         trace_op = tpu.outside_compilation(
-            self._make_tensor_trace_fun(tensor_name), out_tensor)
+            self._make_tensor_trace_fun(tensor_name), processed_out_tensor)
         if consumers:
           for consumer_op in consumers:
             # pylint: disable=protected-access
@@ -1050,8 +1091,9 @@ class TensorTracer(object):
         if self._skip_tensor(op_id, out_tensor, user_included,
                              user_excluded):
           continue
+        processed_out_tensor = self._preprocess_traced_tensor(out_tensor)
         trace_fun = self._make_tensor_trace_fun(out_tensor.name)
-        trace_call = (trace_fun, [out_tensor])
+        trace_call = (trace_fun, [processed_out_tensor])
         trace_call_key = 'tensor_tracing_cpu-%s:%d'%(op.name, i)
         tracing_calls[trace_call_key] = trace_call
     self._post_tracing(succeed, sorted_or_cycle)
-- 
GitLab


From d95400ecd815c493e4241a96bea1c483387d0059 Mon Sep 17 00:00:00 2001
From: Pooya Davoodi <pdavoodi@nvidia.com>
Date: Mon, 7 Jan 2019 19:42:00 -0800
Subject: [PATCH 0311/2345] TFTRT: use six.text_type and six.binary_type

---
 .../contrib/tensorrt/python/trt_convert.py    | 24 +++++--------------
 1 file changed, 6 insertions(+), 18 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/python/trt_convert.py b/tensorflow/contrib/tensorrt/python/trt_convert.py
index f5d624ef64..ac39a570d8 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert.py
@@ -48,30 +48,18 @@ from tensorflow.python.training import saver
 def _to_bytes(s):
     """Returns encoded of s if s is a sequence of chars otherwise returns s.
     """
-    if _six.PY2:
-        if isinstance(s, unicode):
-            return s.encode("utf-8", errors="surrogateescape")
-        else:
-            return s
+    if isinstance(s, _six.text_type):
+        return s.encode("utf-8", errors="surrogateescape")
     else:
-        if isinstance(s, str):
-            return s.encode("utf-8", errors="surrogateescape")
-        else:
-            return s
+        return s
 
 def _to_string(s):
     """Returns decoded of s if s is a sequence of bytes otherwise returns s.
     """
-    if _six.PY2:
-        if isinstance(s, str):
-            return s.decode("utf-8")
-        else:
-            return s
+    if isinstance(s, _six.binary_type):
+        return s.decode("utf-8")
     else:
-        if isinstance(s, bytes):
-            return s.decode("utf-8")
-        else:
-            return s
+        return s
 
 class TrtPrecisionMode(object):
   FP32 = "FP32"
-- 
GitLab


From 8c3f0f594c8323c8f5bd004200e738ffd4a5a64a Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Tue, 8 Jan 2019 09:28:38 +0530
Subject: [PATCH 0312/2345] Update ceil.cc

removed space as part of clang-format
---
 tensorflow/lite/kernels/ceil.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/kernels/ceil.cc b/tensorflow/lite/kernels/ceil.cc
index e0ea061f25..6bb763255b 100644
--- a/tensorflow/lite/kernels/ceil.cc
+++ b/tensorflow/lite/kernels/ceil.cc
@@ -42,7 +42,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
 
   optimized_ops::Ceil(GetTensorShape(input), GetTensorData<float>(input),
-                       GetTensorShape(output), GetTensorData<float>(output));
+                      GetTensorShape(output), GetTensorData<float>(output));
 
   return kTfLiteOk;
 }
-- 
GitLab


From 95b18ddcbf0527e8da892e5a665e4d2547b47d67 Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Tue, 8 Jan 2019 09:31:36 +0530
Subject: [PATCH 0313/2345] Update ceil_test.cc

updated using clang-format
---
 tensorflow/lite/kernels/ceil_test.cc | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/tensorflow/lite/kernels/ceil_test.cc b/tensorflow/lite/kernels/ceil_test.cc
index e120105082..74d2bfb483 100644
--- a/tensorflow/lite/kernels/ceil_test.cc
+++ b/tensorflow/lite/kernels/ceil_test.cc
@@ -55,18 +55,11 @@ TEST(CeilOpTest, SingleDim) {
 
 TEST(CeilOpTest, MultiDims) {
   CeilOpModel model({2, 1, 1, 5}, TensorType_FLOAT32);
-  model.PopulateTensor<float>(model.input(), {
-                                                 0.0001,
-                                                 8.0001,
-                                                 0.9999,
-                                                 9.9999,
-                                                 0.5,
-                                                 -0.0001,
-                                                 -8.0001,
-                                                 -0.9999,
-                                                 -9.9999,
-                                                 -0.5,
-                                             });
+  model.PopulateTensor<float>(
+      model.input(), {
+                         0.0001, 8.0001, 0.9999, 9.9999, 0.5, -0.0001, -8.0001,
+                         -0.9999, -9.9999, -0.5,
+                     });
   model.Invoke();
   EXPECT_THAT(model.GetOutput(),
               ElementsAreArray({1, 9, 1, 10, 1, 0, -8, 0, -9, 0}));
-- 
GitLab


From 4cff2f464cf195fedd8165a11319e8379746456f Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 7 Jan 2019 19:59:50 -0800
Subject: [PATCH 0314/2345] Organize the dot op emitter a bit better; NFC

This CL pays down some technical debt around dot_op_emitter's organization.  It

 - Extracts out a GetDotImplementationStrategy that decides the lowering
   strategy for a dot operation.  Earlier this logic was scattered through the
   lowering code.

 - Removes calls to PotentiallyImplementedAsEigenDot.  This is an implementation
   detail for dot_op_emitter and exposing it like this has aged poorly in the
   face of the various tiled LLVM IR lowerings we now have.  Instead, we expose
   two more specific hooks: DotOperandsAndResultMustHaveRowMajorLayout and
   DotImplementationCanHandleTranspose.

 - Renames xla_enable_experimental_llvm_ir_gemm to
   xla_force_enable_experimental_llvm_ir_gemm since its behavior is that it
   enables the tiled LLVM IR GEMM even when it does not look profitable.

Note: In parallel_task_assignment the call to PotentiallyImplementedAsEigenDot
was redundant since we already check for non-loop fusions and dots.
PiperOrigin-RevId: 228276282
---
 tensorflow/compiler/xla/service/cpu/BUILD     |   5 +-
 .../compiler/xla/service/cpu/cpu_compiler.cc  |   3 +-
 .../cpu/cpu_eigen_tensor_alignment_test.cc    |  16 +-
 .../xla/service/cpu/cpu_layout_assignment.cc  |  64 ++--
 .../compiler/xla/service/cpu/cpu_options.cc   |   8 +-
 .../compiler/xla/service/cpu/cpu_options.h    |   2 +-
 .../xla/service/cpu/dot_op_emitter.cc         | 301 ++++++++++--------
 .../compiler/xla/service/cpu/dot_op_emitter.h |  40 ++-
 .../xla/service/cpu/dot_op_emitter_internal.h |  88 +++++
 .../service/cpu/parallel_task_assignment.cc   |   2 -
 10 files changed, 318 insertions(+), 211 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h

diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index 86c5f71096..1eb42cf5ad 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -382,7 +382,10 @@ cc_library(
 cc_library(
     name = "dot_op_emitter",
     srcs = ["dot_op_emitter.cc"],
-    hdrs = ["dot_op_emitter.h"],
+    hdrs = [
+        "dot_op_emitter.h",
+        "dot_op_emitter_internal.h",
+    ],
     deps = [
         ":cpu_options",
         ":cpu_runtime",
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
index 91c64e45b9..a6d92ce10e 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
@@ -304,7 +304,8 @@ Status CpuCompiler::RunHloPassesThroughLayoutAssn(
   pipeline.AddPass<TransposeFolding>(
       [&](const HloInstruction& dot,
           const TransposeFolding::OperandIndices& candidate_operands) {
-        return PotentiallyImplementedAsEigenDot(dot, *target_machine_features)
+        return DotImplementationCanHandleTranspose(dot,
+                                                   *target_machine_features)
                    ? candidate_operands
                    : TransposeFolding::OperandIndices{};
       },
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc b/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc
index 8727c72b6e..823bdf259c 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/compiler/xla/service/cpu/dot_op_emitter.h"
+#include "tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h"
 #include "tensorflow/compiler/xla/service/cpu/ir_emission_utils.h"
 #include "tensorflow/compiler/xla/service/cpu/target_machine_features_fake.h"
 #include "tensorflow/compiler/xla/service/hlo_parser.h"
@@ -23,6 +23,10 @@ namespace xla {
 namespace cpu {
 namespace {
 
+using internal::DotImplementationStrategy;
+using internal::DotInfo;
+using internal::GetDotImplementationStrategy;
+
 // Test that we don't call into Eigen with tensors too small to be aligned
 // reliably.
 
@@ -47,16 +51,18 @@ ENTRY DotOperation {
   TargetMachineFeaturesWithFakeAlignmentLogic target_machine_with_no_alignment(
       [](int64 size) { return 1; });
 
-  EXPECT_FALSE(
-      PotentiallyImplementedAsEigenDot(*dot, target_machine_with_no_alignment));
+  EXPECT_EQ(GetDotImplementationStrategy(HloModuleConfig{}, DotInfo(*dot),
+                                         target_machine_with_no_alignment),
+            DotImplementationStrategy::kNaiveLlvmIr);
 
   TargetMachineFeaturesWithFakeAlignmentLogic
       target_machine_with_full_alignment([](int64 size) {
         return TargetMachineFeatures::kEigenExpectedTensorAlignment;
       });
 
-  EXPECT_TRUE(PotentiallyImplementedAsEigenDot(
-      *dot, target_machine_with_full_alignment));
+  EXPECT_NE(GetDotImplementationStrategy(HloModuleConfig{}, DotInfo(*dot),
+                                         target_machine_with_full_alignment),
+            DotImplementationStrategy::kNaiveLlvmIr);
 }
 
 TEST_F(CpuEigenTensorAlignmentTest, EigenConvAlignment) {
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_layout_assignment.cc b/tensorflow/compiler/xla/service/cpu/cpu_layout_assignment.cc
index ff11409482..95b8025f87 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_layout_assignment.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_layout_assignment.cc
@@ -93,60 +93,38 @@ static Shape ColMajorShape(const Shape& old_shape) {
   return new_shape;
 }
 
+static bool OperandsAndResultMustHaveRowMajorLayout(
+    const HloInstruction& instr,
+    const TargetMachineFeatures& target_machine_features) {
+  if (instr.opcode() == HloOpcode::kConvolution) {
+    return PotentiallyImplementedAsEigenConvolution(instr,
+                                                    target_machine_features);
+  } else if (instr.opcode() == HloOpcode::kDot) {
+    return DotOperandsAndResultMustHaveRowMajorLayout(instr,
+                                                      target_machine_features);
+  }
+  return false;
+}
+
 Status CpuLayoutAssignment::AddBackendConstraints(
     LayoutConstraints* constraints) {
   ShouldMakeOperandColMajorCache cache;
 
   const HloComputation* computation = constraints->computation();
   for (auto* instruction : computation->instructions()) {
-    if (instruction->opcode() == HloOpcode::kConvolution &&
-        PotentiallyImplementedAsEigenConvolution(*instruction,
-                                                 target_machine_features_)) {
-      const HloInstruction* convolution = instruction;
-      const HloInstruction* lhs_instruction = convolution->operand(0);
-      const HloInstruction* rhs_instruction = convolution->operand(1);
-
-      // In order to implement `convolution` with Eigen convolution, the layouts
-      // of the input, filter, and output need to be row-major.
-      //
-      // These constraints are not hard constraints. Ideally, we should decide
-      // which layouts to choose according to some cost model.
-      Shape output_shape(RowMajorShape(convolution->shape()));
-      Shape input_shape(RowMajorShape(lhs_instruction->shape()));
-      Shape filter_shape(RowMajorShape(rhs_instruction->shape()));
-
-      // Set layouts of the instructions' shapes.
-      TF_RETURN_IF_ERROR(
-          constraints->SetOperandLayout(input_shape, convolution, 0));
-      TF_RETURN_IF_ERROR(
-          constraints->SetOperandLayout(filter_shape, convolution, 1));
-      TF_RETURN_IF_ERROR(
-          constraints->SetInstructionLayout(output_shape, convolution));
+    if (OperandsAndResultMustHaveRowMajorLayout(*instruction,
+                                                target_machine_features_)) {
+      TF_RETURN_IF_ERROR(constraints->SetInstructionLayout(
+          RowMajorShape(instruction->shape()), instruction));
+      for (int i = 0; i < instruction->operand_count(); i++) {
+        TF_RETURN_IF_ERROR(constraints->SetOperandLayout(
+            RowMajorShape(instruction->operand(i)->shape()), instruction, i));
+      }
     } else if (optional<int64> op_idx =
                    ShouldMakeOperandColumnMajor(&cache, *instruction)) {
       const HloInstruction* op = instruction->operand(*op_idx);
       TF_RETURN_IF_ERROR(constraints->SetOperandLayout(
           ColMajorShape(op->shape()), instruction, *op_idx));
-    } else if (PotentiallyImplementedAsEigenDot(*instruction,
-                                                target_machine_features_)) {
-      const HloInstruction* dot = instruction;
-      // In order to implement `dot` with Eigen dot, the layouts of the lhs,
-      // rhs, and output need to be row-major.
-      //
-      // These constraints are not hard constraints. Ideally, we should decide
-      // which layouts to choose according to some cost model.
-      Shape output_shape(RowMajorShape(dot->shape()));
-
-      const HloInstruction* lhs_instruction = dot->operand(0);
-      Shape lhs_shape(RowMajorShape(lhs_instruction->shape()));
-      TF_RETURN_IF_ERROR(constraints->SetOperandLayout(lhs_shape, dot, 0));
-
-      const HloInstruction* rhs_instruction = dot->operand(1);
-      Shape rhs_shape(RowMajorShape(rhs_instruction->shape()));
-      TF_RETURN_IF_ERROR(constraints->SetOperandLayout(rhs_shape, dot, 1));
-
-      // Set layouts of the instructions' shapes.
-      TF_RETURN_IF_ERROR(constraints->SetInstructionLayout(output_shape, dot));
     } else {
       for (int64 operand_no = 0; operand_no < instruction->operand_count();
            ++operand_no) {
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_options.cc b/tensorflow/compiler/xla/service/cpu/cpu_options.cc
index 92debb83e3..ff654c83d6 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_options.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_options.cc
@@ -23,8 +23,8 @@ namespace {
 
 const char* const kXlaOptimizeForSizeCpuOption = "xla_cpu_optimize_for_size";
 const char* const kLlvmIrDotTilingFactor = "xla_llvm_dot_tiling_factor";
-const char* const kXlaEnableExperimentalLlvmIrGemm =
-    "xla_enable_experimental_llvm_ir_gemm";
+const char* const kXlaForceEnableExperimentalLlvmIrGemm =
+    "xla_force_enable_experimental_llvm_ir_gemm";
 const char* const kLlvmIrGemmTileSize = "xla_llvm_ir_gemm_tile_size";
 
 }  // namespace
@@ -57,10 +57,10 @@ absl::optional<int64> LlvmIrGemvTilingFactor(const HloModuleConfig& config) {
   return absl::nullopt;
 }
 
-bool EnableExperimentalLlvmIrGemm(const HloModuleConfig& config) {
+bool ForceEnableExperimentalLlvmIrGemm(const HloModuleConfig& config) {
   const auto& extra_options_map =
       config.debug_options().xla_backend_extra_options();
-  return extra_options_map.count(kXlaEnableExperimentalLlvmIrGemm) > 0;
+  return extra_options_map.count(kXlaForceEnableExperimentalLlvmIrGemm) > 0;
 }
 
 static absl::string_view RemoveSuffix(absl::string_view str,
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_options.h b/tensorflow/compiler/xla/service/cpu/cpu_options.h
index 47c7eb13b6..99e6702d14 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_options.h
+++ b/tensorflow/compiler/xla/service/cpu/cpu_options.h
@@ -26,7 +26,7 @@ namespace options {
 
 bool OptimizeForSizeRequested(const HloModuleConfig& config);
 bool VectorizedReduceDisabled(const HloModuleConfig& config);
-bool EnableExperimentalLlvmIrGemm(const HloModuleConfig& config);
+bool ForceEnableExperimentalLlvmIrGemm(const HloModuleConfig& config);
 absl::optional<int64> LlvmIrGemvTilingFactor(const HloModuleConfig& config);
 absl::optional<std::tuple<int64, int64, int64>> LlvmIrGemmTileSize(
     const HloModuleConfig& config);
diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
index bdd688c2b9..70d8fee265 100644
--- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/compiler/xla/service/cpu/dot_op_emitter.h"
+#include "tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h"
 
 #include <memory>
 #include <vector>
@@ -42,6 +43,18 @@ namespace xla {
 using llvm_ir::SetToFirstInsertPoint;
 
 namespace cpu {
+
+using internal::DotImplementationStrategy;
+using internal::DotInfo;
+using internal::GetDotImplementationStrategy;
+
+namespace {
+// Returns true if we should call into multi-threaded Eigen routines.
+bool ShouldUseMultiThreadedEigen(const HloModuleConfig& config) {
+  return config.debug_options().xla_cpu_multi_thread_eigen();
+}
+}  // namespace
+
 DotOpEmitter::DotOpEmitter(const HloInstruction& dot,
                            const llvm_ir::IrArray& target_array,
                            const llvm_ir::IrArray& lhs_array,
@@ -76,43 +89,9 @@ DotOpEmitter::DotOpEmitter(const HloInstruction& dot,
   return dot_emitter.Emit();
 }
 
-bool DotOpEmitter::EmitSmallGemmIfProfitable(
-    const DotOpEmitter::MatMultDims& mat_mult_dims) {
-  if (ShouldUseMultiThreadedEigen()) {
-    return false;
-  }
-
-  if (!EnableExperimentalLlvmIrGemm()) {
-    // TODO(sanjoy):  We should make these numbers micro-arch specific.
-    bool small_gemm = mat_mult_dims.k <= 128 &&
-                      ((mat_mult_dims.m <= 32 && mat_mult_dims.n <= 128) ||
-                       (mat_mult_dims.m <= 128 && mat_mult_dims.n <= 32));
-    if (!small_gemm) {
-      return false;
-    }
-  }
-
-  if (mat_mult_dims.lhs_non_canonical || mat_mult_dims.rhs_non_canonical) {
-    return false;
-  }
-
+void DotOpEmitter::EmitTiledLlvmIrGemm() {
   PrimitiveType primitive_type = dot_.shape().element_type();
-
-  switch (primitive_type) {
-    default:
-      return false;
-
-    case F32:
-    case F64:
-    case S32:
-    case S64:
-      break;
-  }
-
-  if (!(mat_mult_dims.lhs_column_major == mat_mult_dims.rhs_column_major &&
-        mat_mult_dims.rhs_column_major == mat_mult_dims.target_column_major)) {
-    return false;
-  }
+  MatMultDims mat_mult_dims = GetMatMultDims();
 
   llvm::Value* lhs = lhs_array_.GetBasePointer();
   llvm::Value* rhs = rhs_array_.GetBasePointer();
@@ -154,21 +133,13 @@ bool DotOpEmitter::EmitSmallGemmIfProfitable(
       /*rhs=*/rhs, /*result=*/target, b_,
       /*enable_fast_math=*/enable_fast_math,
       /*optimize_for_size=*/optimize_for_size);
-
-  return true;
 }
 
-bool DotOpEmitter::EmitLlvmIrDotIfProfitable() {
-  if (dot_.shape().dimensions_size() != 2) {
-    return false;
-  }
-
+void DotOpEmitter::EmitTiledLlvmIrGemv() {
   PrimitiveType primitive_type = dot_.shape().element_type();
 
-  if (!primitive_util::IsFloatingPointType(primitive_type) &&
-      !primitive_util::IsIntegralType(primitive_type)) {
-    return false;
-  }
+  CHECK(primitive_util::IsFloatingPointType(primitive_type) ||
+        primitive_util::IsIntegralType(primitive_type));
 
   MatMultDims mat_mult_dims = GetMatMultDims();
   bool is_column_major_matrix_vector = false;
@@ -209,9 +180,7 @@ bool DotOpEmitter::EmitLlvmIrDotIfProfitable() {
     }
   }
 
-  if (!is_column_major_matrix_vector && !is_row_major_matrix_vector) {
-    return EmitSmallGemmIfProfitable(mat_mult_dims);
-  }
+  CHECK(is_column_major_matrix_vector || is_row_major_matrix_vector);
 
   int64 tiling_factor = GetGemvTilingFactor();
   CHECK_GT(tiling_factor, 0);
@@ -264,8 +233,6 @@ bool DotOpEmitter::EmitLlvmIrDotIfProfitable() {
         /*enable_fast_math=*/enable_fast_math,
         /*optimize_for_size=*/optimize_for_size);
   }
-
-  return true;
 }
 
 Status DotOpEmitter::Emit() {
@@ -306,27 +273,41 @@ Status DotOpEmitter::Emit() {
     return EmitScalarDot();
   }
 
-  if (EmitLlvmIrDotIfProfitable()) {
-    return Status::OK();
+  switch (GetDotImplementationStrategy(hlo_module_config_, DotInfo(dot_),
+                                       target_machine_features_)) {
+    case DotImplementationStrategy::kNaiveLlvmIr:
+      EmitNaiveLlvmIrGemm();
+      return Status::OK();
+
+    case DotImplementationStrategy::kTiledLlvmIrGemv:
+      EmitTiledLlvmIrGemv();
+      return Status::OK();
+
+    case DotImplementationStrategy::kTiledLlvmIrGemm:
+      EmitTiledLlvmIrGemm();
+      return Status::OK();
+
+    case DotImplementationStrategy::kEigen:
+      return EmitCallToRuntime();
   }
+}
 
+void DotOpEmitter::EmitNaiveLlvmIrGemm() {
   CHECK_EQ(addend_array_, nullptr);
 
-  if (PotentiallyImplementedAsEigenDot(dot_, target_machine_features_)) {
-    return EmitCallToRuntime();
-  }
+  const Shape& lhs_shape = lhs_array_.GetShape();
+  const Shape& rhs_shape = rhs_array_.GetShape();
+  const DotDimensionNumbers& dim_nums = dot_.dot_dimension_numbers();
 
   // Reduce along dimension 0 of the LHS and 1 of the RHS. Vectors are a special
   // case where the reduction dimension is 0 for both LHS and RHS. This results
   // in a vector dot product producing a scalar.
-  int64 lhs_reduction_dimension =
-      dot_.dot_dimension_numbers().lhs_contracting_dimensions(0);
-  int64 rhs_reduction_dimension =
-      dot_.dot_dimension_numbers().rhs_contracting_dimensions(0);
+  int64 lhs_reduction_dimension = dim_nums.lhs_contracting_dimensions(0);
+  int64 rhs_reduction_dimension = dim_nums.rhs_contracting_dimensions(0);
 
   // Verify the reduction dimension in the two operands are the same size.
-  TF_RET_CHECK(lhs_shape.dimensions(lhs_reduction_dimension) ==
-               rhs_shape.dimensions(rhs_reduction_dimension));
+  CHECK_EQ(lhs_shape.dimensions(lhs_reduction_dimension),
+           rhs_shape.dimensions(rhs_reduction_dimension));
 
   bool lhs_reduction_along_minor_dimension =
       lhs_reduction_dimension == LayoutUtil::Minor(lhs_shape.layout(), 0);
@@ -441,8 +422,6 @@ Status DotOpEmitter::Emit() {
   // Set the IR builder insert point to the exit basic block of the outer most
   // loop.
   b_->SetInsertPoint(loop_nest.GetOuterLoopExitBasicBlock());
-
-  return Status::OK();
 }
 
 Status DotOpEmitter::EmitScalarDot() {
@@ -489,7 +468,7 @@ Status DotOpEmitter::EmitCallToRuntime() {
   // The two transpose_... parameters are actually booleans, but we use int32
   // to avoid target-dependent calling convention details.
 
-  bool multi_threaded = ShouldUseMultiThreadedEigen();
+  bool multi_threaded = ShouldUseMultiThreadedEigen(hlo_module_config_);
   bool use_mkl_dnn = hlo_module_config_.debug_options().xla_cpu_use_mkl_dnn();
   PrimitiveType type = target_array_.GetShape().element_type();
   llvm::Type* float_type;
@@ -600,12 +579,52 @@ DotOpEmitter::MatMultDims DotOpEmitter::GetMatMultDims() const {
       LayoutUtil::Minor(target_array_.GetShape().layout(), 0) == 0};
 }
 
+// For vector-matrix dot products, it is always profitable to make the Rhs
+// column major.
+absl::optional<int64> ProfitableToMakeDotOperandColumnMajor(
+    const HloInstruction& hlo) {
+  if (hlo.opcode() == HloOpcode::kDot && hlo.shape().dimensions_size() == 2 &&
+      hlo.shape().dimensions(0) == 1) {
+    if (hlo.dot_dimension_numbers().rhs_contracting_dimensions(0) == 0) {
+      return 1;
+    }
+    return {};
+  }
+
+  if (hlo.opcode() == HloOpcode::kFusion &&
+      hlo.fusion_kind() == HloInstruction::FusionKind::kOutput) {
+    auto* fusion_root =
+        hlo.fused_instructions_computation()->root_instruction();
+    if (fusion_root->opcode() != HloOpcode::kAdd) {
+      return {};
+    }
+
+    for (auto* fusion_root_op : fusion_root->operands()) {
+      if (fusion_root_op->opcode() != HloOpcode::kDot) {
+        continue;
+      }
+      if (auto operand_num =
+              ProfitableToMakeDotOperandColumnMajor(*fusion_root_op)) {
+        auto* operand = fusion_root_op->operand(*operand_num);
+        if (operand->opcode() == HloOpcode::kParameter &&
+            operand->user_count() == 1) {
+          return operand->parameter_number();
+        }
+      }
+    }
+  }
+
+  return {};
+}
+
+namespace internal {
+namespace {
 // Return whether the given shape is rank 2.
-static bool IsRank2(const Shape& shape) { return shape.rank() == 2; }
+bool IsRank2(const Shape& shape) { return shape.rank() == 2; }
 
 // In a gemm operation where output = lhs * rhs, check whether the given shapes
 // are valid for the operation.
-static bool AreValidGemmShapes(
+bool AreAlignedGemmShapes(
     const Shape& lhs_shape, const Shape& rhs_shape, const Shape& output_shape,
     const TargetMachineFeatures& target_machine_features) {
   // The inputs and the output must
@@ -634,88 +653,106 @@ static bool AreValidGemmShapes(
   return true;
 }
 
-bool PotentiallyImplementedAsEigenDot(
-    const HloInstruction& hlo,
-    const TargetMachineFeatures& target_machine_features) {
-  // For certain types of Dot, we can call Eigen
-  if (hlo.opcode() == HloOpcode::kDot) {
-    const Shape& lhs_shape = hlo.operand(0)->shape();
-    const Shape& rhs_shape = hlo.operand(1)->shape();
+bool IsAlignedGemm(const DotInfo& dot_info,
+                   const TargetMachineFeatures& target_machine_features) {
+  if (ShapeUtil::IsZeroElementArray(dot_info.lhs_shape) ||
+      ShapeUtil::IsZeroElementArray(dot_info.rhs_shape)) {
+    return false;
+  }
 
-    if (ShapeUtil::IsZeroElementArray(lhs_shape) ||
-        ShapeUtil::IsZeroElementArray(rhs_shape)) {
-      return false;
-    }
+  return AreAlignedGemmShapes(dot_info.lhs_shape, dot_info.rhs_shape,
+                              dot_info.result_shape, target_machine_features);
+}
 
-    if (ProfitableToImplementDotInTiledLlvmIr(hlo)) {
-      return false;
-    }
+bool CanEmitTiledLlvmIrGemm(
+    const HloModuleConfig& config, const DotInfo& dot_info,
+    const TargetMachineFeatures& target_machine_features) {
+  CHECK(IsAlignedGemm(dot_info, target_machine_features));
 
-    // If gemm can accept the operand shapes, use it rather than a custom
-    // kernel.
-    if (AreValidGemmShapes(lhs_shape, rhs_shape, hlo.shape(),
-                           target_machine_features)) {
-      const DotDimensionNumbers& dim_numbers = hlo.dot_dimension_numbers();
-      // The size of the reduction dimension should match. The shape inference
-      // guarantees this invariant, so the check here is for programming
-      // errors.
-      CHECK_EQ(lhs_shape.dimensions(dim_numbers.lhs_contracting_dimensions(0)),
-               rhs_shape.dimensions(dim_numbers.rhs_contracting_dimensions(0)));
-      return true;
-    }
+  if (ShouldUseMultiThreadedEigen(config)) {
+    return false;
   }
 
-  return false;
-}
+  int m = dot_info.result_shape.dimensions(0);
+  int k = dot_info.lhs_shape.dimensions(
+      dot_info.dim_nums.lhs_contracting_dimensions(0));
+  int n = dot_info.result_shape.dimensions(1);
 
-// For vector-matrix dot products, it is always profitable to make the Rhs
-// column major.
-absl::optional<int64> ProfitableToMakeDotOperandColumnMajor(
-    const HloInstruction& hlo) {
-  if (hlo.opcode() == HloOpcode::kDot && hlo.shape().dimensions_size() == 2 &&
-      hlo.shape().dimensions(0) == 1) {
-    if (hlo.dot_dimension_numbers().rhs_contracting_dimensions(0) == 0) {
-      return 1;
+  if (!options::ForceEnableExperimentalLlvmIrGemm(config)) {
+    // TODO(sanjoy):  We should make these numbers micro-arch specific.
+    bool small_gemm =
+        k <= 128 && ((m <= 32 && n <= 128) || (m <= 128 && n <= 32));
+    if (!small_gemm) {
+      return false;
     }
-    return {};
   }
 
-  if (hlo.opcode() == HloOpcode::kFusion &&
-      hlo.fusion_kind() == HloInstruction::FusionKind::kOutput) {
-    auto* fusion_root =
-        hlo.fused_instructions_computation()->root_instruction();
-    if (fusion_root->opcode() != HloOpcode::kAdd) {
-      return {};
-    }
+  bool lhs_non_canonical = dot_info.dim_nums.lhs_contracting_dimensions(0) == 0;
+  bool rhs_non_canonical = dot_info.dim_nums.rhs_contracting_dimensions(0) == 1;
 
-    for (auto* fusion_root_op : fusion_root->operands()) {
-      if (fusion_root_op->opcode() != HloOpcode::kDot) {
-        continue;
-      }
-      if (auto operand_num =
-              ProfitableToMakeDotOperandColumnMajor(*fusion_root_op)) {
-        auto* operand = fusion_root_op->operand(*operand_num);
-        if (operand->opcode() == HloOpcode::kParameter &&
-            operand->user_count() == 1) {
-          return operand->parameter_number();
-        }
-      }
-    }
+  if (lhs_non_canonical || rhs_non_canonical) {
+    return false;
   }
 
-  return {};
+  if (dot_info.result_shape.element_type() == F16) {
+    // TODO(sanjoy): This is probably easy to fix, but I want to keep the CL
+    // adding this comment NFC.
+    return false;
+  }
+
+  return true;
 }
+}  // namespace
 
-bool ProfitableToImplementDotInTiledLlvmIr(const HloInstruction& dot) {
+DotImplementationStrategy GetDotImplementationStrategy(
+    const HloModuleConfig& config, const DotInfo& dot_info,
+    const TargetMachineFeatures& target_machine_features) {
+  PrimitiveType element_type = dot_info.result_shape.element_type();
   // Any Matrix-Vector product of floating point or integral type, or
   // a transpose-dot fusion of the same can be lowered to a tiled LLVM
   // IR implementation.
-  const Shape& shape = dot.shape();
-  return shape.dimensions_size() == 2 &&
-         (shape.dimensions(0) == 1 || shape.dimensions(1) == 1) &&
-         (primitive_util::IsFloatingPointType(shape.element_type()) ||
-          primitive_util::IsIntegralType(shape.element_type()));
+  if (dot_info.result_shape.dimensions_size() == 2 &&
+      (dot_info.result_shape.dimensions(0) == 1 ||
+       dot_info.result_shape.dimensions(1) == 1) &&
+      (primitive_util::IsFloatingPointType(element_type) ||
+       primitive_util::IsIntegralType(element_type))) {
+    return DotImplementationStrategy::kTiledLlvmIrGemv;
+  }
+
+  if (IsAlignedGemm(dot_info, target_machine_features)) {
+    return CanEmitTiledLlvmIrGemm(config, dot_info, target_machine_features)
+               ? DotImplementationStrategy::kTiledLlvmIrGemm
+               : DotImplementationStrategy::kEigen;
+  }
+
+  return DotImplementationStrategy::kNaiveLlvmIr;
 }
+}  // namespace internal
 
+bool DotImplementationCanHandleTranspose(
+    const HloInstruction& dot_instr,
+    const TargetMachineFeatures& target_machine_features) {
+  DotImplementationStrategy impl_strategy =
+      GetDotImplementationStrategy(dot_instr.parent()->parent()->config(),
+                                   DotInfo(dot_instr), target_machine_features);
+
+  // TODO(sanjoy): This is not quite right, it should be `impl_strategy ==
+  // kEigen || impl_strategy == kTiledLlvmIrGemv || impl_strategy ==
+  // kNaiveLlvmIr` but I'll fix this in a later CL in the interest of keeping
+  // the CL adding this comment NFC.
+  return impl_strategy == DotImplementationStrategy::kTiledLlvmIrGemm ||
+         impl_strategy == DotImplementationStrategy::kEigen;
+}
+
+bool DotOperandsAndResultMustHaveRowMajorLayout(
+    const HloInstruction& dot_instr,
+    const TargetMachineFeatures& target_machine_features) {
+  DotImplementationStrategy impl_strategy =
+      GetDotImplementationStrategy(dot_instr.parent()->parent()->config(),
+                                   DotInfo(dot_instr), target_machine_features);
+
+  return impl_strategy == DotImplementationStrategy::kTiledLlvmIrGemm ||
+         impl_strategy == DotImplementationStrategy::kEigen;
+}
 }  // namespace cpu
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.h b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.h
index 4c2041b556..eaa226a4c2 100644
--- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.h
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.h
@@ -30,9 +30,16 @@ limitations under the License.
 
 namespace xla {
 namespace cpu {
+// Returns true if the two operands and the output of `dot_instr` must have row
+// major layout.
+bool DotOperandsAndResultMustHaveRowMajorLayout(
+    const HloInstruction& dot_instr,
+    const TargetMachineFeatures& target_machine_features);
 
-bool PotentiallyImplementedAsEigenDot(
-    const HloInstruction& hlo,
+// Returns true our lowering strategy for `dot_instr` can fold in transposes to
+// the either of the inputs.
+bool DotImplementationCanHandleTranspose(
+    const HloInstruction& dot_instr,
     const TargetMachineFeatures& target_machine_features);
 
 // Returns the index for an operand to `hlo` that should ideally be column
@@ -41,10 +48,6 @@ bool PotentiallyImplementedAsEigenDot(
 absl::optional<int64> ProfitableToMakeDotOperandColumnMajor(
     const HloInstruction& hlo);
 
-// Returns true to indicate that we can generate a tiled LLVM IR implementation
-// for |dot|.
-bool ProfitableToImplementDotInTiledLlvmIr(const HloInstruction& dot);
-
 // Helper class for emitting LLVM IR to perform the dot operation.
 class DotOpEmitter {
  public:
@@ -81,10 +84,6 @@ class DotOpEmitter {
   // LHS and RHS) and store the results in the target.
   Status EmitScalarDot();
 
-  // Emit an LLVM IR implementation of the dot operation if we can.  Returns
-  // true if an LLVM IR implementation was emitted.
-  bool EmitLlvmIrDotIfProfitable();
-
   // Emits a call to the CPU runtime to perform the matrix multiply.
   Status EmitCallToRuntime();
 
@@ -121,7 +120,15 @@ class DotOpEmitter {
   // of rank 2 as well).
   MatMultDims GetMatMultDims() const;
 
-  bool EmitSmallGemmIfProfitable(const MatMultDims& mat_mult_dims);
+  // Lowers the dot operation as a tiled Matrix*Vector loop.
+  void EmitTiledLlvmIrGemv();
+
+  // Lowers the dot operation as a tiled Matrix*Matrix loop.
+  void EmitTiledLlvmIrGemm();
+
+  // Lowers the dot operation as a naive nested loop that computes the result
+  // one element at a time.
+  void EmitNaiveLlvmIrGemm();
 
   // When doing a tiled GEMV in LLVM IR, a "tile" consists of this many vector
   // registers.
@@ -142,17 +149,6 @@ class DotOpEmitter {
         .value_or(kDefaultTileSize);
   }
 
-  // Returns true if we should use an experimental implementation of GEMM
-  // (general matrix matrix multiplication) if possible.
-  bool EnableExperimentalLlvmIrGemm() const {
-    return options::EnableExperimentalLlvmIrGemm(hlo_module_config_);
-  }
-
-  // Returns true if we should call into multi-threaded Eigen routines.
-  bool ShouldUseMultiThreadedEigen() {
-    return hlo_module_config_.debug_options().xla_cpu_multi_thread_eigen();
-  }
-
   const HloInstruction& dot_;
   const llvm_ir::IrArray& target_array_;
   const llvm_ir::IrArray& lhs_array_;
diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h b/tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h
new file mode 100644
index 0000000000..cc28918ed6
--- /dev/null
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h
@@ -0,0 +1,88 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_CPU_DOT_OP_EMITTER_INTERNAL_H_
+#define TENSORFLOW_COMPILER_XLA_SERVICE_CPU_DOT_OP_EMITTER_INTERNAL_H_
+
+#include "tensorflow/compiler/xla/service/cpu/target_machine_features.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+
+// -----------------------------------------------------------------------------
+// INTERNAL HEADER.
+//
+// This file exposes internal implementation details from dot_op_emitter.cc for
+// unit tests.  Please do not depend on this!
+//
+// -----------------------------------------------------------------------------
+
+namespace xla {
+namespace cpu {
+namespace internal {
+
+// Represents a dot operation.  We use this in lieu of an `HloInstruction`
+// because we want to be able to create this for the "inner" dot operation in a
+// batch dot, for which there is no separate HLO instruction.
+struct DotInfo {
+  Shape lhs_shape;
+  Shape rhs_shape;
+  Shape result_shape;
+  DotDimensionNumbers dim_nums;
+
+  explicit DotInfo(const HloInstruction& instr) {
+    CHECK_EQ(instr.opcode(), HloOpcode::kDot);
+    lhs_shape = instr.operand(0)->shape();
+    rhs_shape = instr.operand(1)->shape();
+    result_shape = instr.shape();
+    dim_nums = instr.dot_dimension_numbers();
+  }
+};
+
+// Dictates how a dot operation is implemented.
+enum class DotImplementationStrategy {
+  // The dot operation is lowered into LLVM IR that implements a naive nested
+  // loop that computes the result one element at a time.  This is our
+  // "fallback"; we don't really want this to kick in for any non-trival dot
+  // operation.
+  kNaiveLlvmIr,
+
+  // The dot operation is lowered into LLVM IR that implements a tiled
+  // Matrix*Vector operation.  This strategy also allows fusing in a bias add
+  // into the dot.  The matrix can be row major or column major, both are
+  // supported.
+  kTiledLlvmIrGemv,
+
+  // The dot operation is lowered into LLVM IR that implemetns a tiled
+  // Matrix*Matrix operation.  No fusions are supported.  The two inputs
+  // and the output have to be row major.
+  kTiledLlvmIrGemm,
+
+  // The dot operation is lowered into a call into an Eigen routine.  No fusions
+  // are supported today.  The two inputs and the output have to be row major.
+  // However, we do allow transposing either the LHS or the RHS as part of the
+  // GEMM -- we expose this flexibility as flexibility in the contraction
+  // dimensions, but we can also see this as flexibility in the input layouts.
+  kEigen,
+};
+
+// Returns the implementation strategy for a dot with the configuration
+// `dot_info`.
+DotImplementationStrategy GetDotImplementationStrategy(
+    const HloModuleConfig& config, const DotInfo& dot_info,
+    const TargetMachineFeatures& target_machine_features);
+}  // namespace internal
+}  // namespace cpu
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_CPU_DOT_OP_EMITTER_INTERNAL_H_
diff --git a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc
index 3b423f6391..6121d1ca9a 100644
--- a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc
+++ b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc
@@ -146,8 +146,6 @@ int64 ParallelTaskAssignment::GetTargetParallelTaskCount(
       (opcode == HloOpcode::kConvolution &&
        PotentiallyImplementedAsEigenConvolution(*instruction,
                                                 target_machine_features_)) ||
-      PotentiallyImplementedAsEigenDot(*instruction,
-                                       target_machine_features_) ||
       (opcode == HloOpcode::kFusion &&
        instruction->fusion_kind() != HloInstruction::FusionKind::kLoop) ||
       instruction->shape().IsTuple()) {
-- 
GitLab


From 8c91712adfb586da675aeaa59730fd4c6e8c5ded Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Tue, 8 Jan 2019 10:02:21 +0530
Subject: [PATCH 0315/2345] LOG the result in case of error.

---
 tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc b/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc
index 77737a76b6..2879b00f55 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc
@@ -46,7 +46,7 @@ bool CheckMask(se::StreamExecutor* exec, void* ptr, int64* mask) {
 
   Status result = exec->SynchronousMemcpyD2H(gpu_ptr, MASK_BYTES, tmp);
   if (!result.ok()) {
-    LOG(FATAL) << "Could not copy debug mask";
+    LOG(FATAL) << "Could not copy debug mask, " << result.error_message();
   }
 
   bool ok = true;
@@ -66,7 +66,7 @@ void InitMask(se::StreamExecutor* exec, void* ptr, int64* mask) {
   se::DeviceMemory<int64> gpu_ptr{se::DeviceMemoryBase{ptr, MASK_BYTES}};
   Status result = exec->SynchronousMemcpyH2D(mask, MASK_BYTES, &gpu_ptr);
   if (!result.ok()) {
-    LOG(FATAL) << "Could not copy debug mask";
+    LOG(FATAL) << "Could not copy debug mask, " << result.error_message();
   }
 }
 
@@ -176,7 +176,7 @@ void* GPUNanResetAllocator::AllocateRaw(size_t alignment, size_t num_bytes) {
   Status result = stream_exec_->SynchronousMemcpyH2D(&nans[0], req_size,
                                                      &nan_ptr);
   if (!result.ok()) {
-    LOG(ERROR) << "Could not initialize to NaNs";
+    LOG(ERROR) << "Could not initialize to NaNs, " << result.error_message();
   }
 
   return allocated_ptr;
@@ -192,7 +192,7 @@ void GPUNanResetAllocator::DeallocateRaw(void* ptr) {
     Status result = stream_exec_->SynchronousMemcpyH2D(&nans[0], req_size,
                                                        &nan_ptr);
     if (!result.ok()) {
-      LOG(ERROR) << "Could not initialize to NaNs";
+      LOG(ERROR) << "Could not initialize to NaNs, " << result.error_message();
     }
   }
 
-- 
GitLab


From 8808014ef840f98aafd93f1e54a069601a184586 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Tue, 8 Jan 2019 04:50:04 +0000
Subject: [PATCH 0316/2345] Update protobuf to 3.6.1 in install_proto3.sh

This fix updates protobuf to 3.6.1 in install_proto3.sh.

The protobuf library in workspace.bzl has been updated to 3.6.1 for
some time, though in CI build protoc still uses 3.6.0.

When build a custom op (in docker image `tensorflow/tensorflow:custom-op`)
and using grpc's `grpc_defs()` (`grpc/bazel/grpc_deps.bzl`):
https://github.com/grpc/grpc/blob/ac6795a57e05523b8fa220bc5cef26abb876aae5/bazel/grpc_deps.bzl#L5-L7
The issue arise because grpc's dependency is 3.6.1 (and an immediate earlier version is 3.5)
so there is no match to fit grpc+protobuf no matter which version of grpc is used.

This may not a problem for tensorflow itself as tensorflow does not call `grpc_defs()`. Tensorflow
links to protobuf 3.6.1 explicitly. Still, it would be good to match the
version of installed protoc (3.6.0) with protobuf library (3.6.1).

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/tools/ci_build/install/install_proto3.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/install/install_proto3.sh b/tensorflow/tools/ci_build/install/install_proto3.sh
index 821d50baff..3cb1008567 100755
--- a/tensorflow/tools/ci_build/install/install_proto3.sh
+++ b/tensorflow/tools/ci_build/install/install_proto3.sh
@@ -17,7 +17,7 @@
 # Install protobuf3.
 
 # Select protobuf version.
-PROTOBUF_VERSION="3.6.0"
+PROTOBUF_VERSION="3.6.1"
 protobuf_ver_flat=$(echo $PROTOBUF_VERSION | sed 's/\.//g' | sed 's/^0*//g')
 local_protobuf_ver=$(protoc --version)
 local_protobuf_ver_flat=$(echo $local_protobuf_ver | sed 's/\.//g' | sed 's/^0*//g')
-- 
GitLab


From e7b6486adb822bb83a9ff669dc4d2d477d911b3c Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Tue, 8 Jan 2019 04:57:32 +0000
Subject: [PATCH 0317/2345] Update install_pip_packages.sh to protobuf 3.6.1

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/tools/ci_build/install/install_pip_packages.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/ci_build/install/install_pip_packages.sh b/tensorflow/tools/ci_build/install/install_pip_packages.sh
index eeadabaa73..5ae840cfa0 100755
--- a/tensorflow/tools/ci_build/install/install_pip_packages.sh
+++ b/tensorflow/tools/ci_build/install/install_pip_packages.sh
@@ -60,8 +60,8 @@ pip2 install --upgrade markdown==2.6.8
 pip3 install --upgrade markdown==2.6.8
 
 # Install protobuf.
-pip2 install --upgrade protobuf==3.6.0
-pip3 install --upgrade protobuf==3.6.0
+pip2 install --upgrade protobuf==3.6.1
+pip3 install --upgrade protobuf==3.6.1
 
 # Remove obsolete version of six, which can sometimes confuse virtualenv.
 rm -rf /usr/lib/python3/dist-packages/six*
-- 
GitLab


From bb447c4db4c1c5990f476b7b5fdf1d3c5b3afacc Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Tue, 8 Jan 2019 04:58:19 +0000
Subject: [PATCH 0318/2345] Update protobuf to 3.6.1

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .../tools/ci_build/install/install_python3.5_pip_packages.sh    | 2 +-
 .../tools/ci_build/install/install_python3.6_pip_packages.sh    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh b/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh
index 0654fb81b3..a58f49af28 100755
--- a/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh
+++ b/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh
@@ -52,7 +52,7 @@ pip3.5 install --upgrade absl-py
 pip3.5 install --upgrade six==1.10.0
 
 # Install protobuf.
-pip3.5 install --upgrade protobuf==3.6.0
+pip3.5 install --upgrade protobuf==3.6.1
 
 # Remove obsolete version of six, which can sometimes confuse virtualenv.
 rm -rf /usr/lib/python3/dist-packages/six*
diff --git a/tensorflow/tools/ci_build/install/install_python3.6_pip_packages.sh b/tensorflow/tools/ci_build/install/install_python3.6_pip_packages.sh
index 3edf4571fb..b1c2a0ab00 100755
--- a/tensorflow/tools/ci_build/install/install_python3.6_pip_packages.sh
+++ b/tensorflow/tools/ci_build/install/install_python3.6_pip_packages.sh
@@ -64,7 +64,7 @@ pip3 install --upgrade absl-py
 pip3 install --upgrade six==1.10.0
 
 # Install protobuf.
-pip3 install --upgrade protobuf==3.6.0
+pip3 install --upgrade protobuf==3.6.1
 
 # Remove obsolete version of six, which can sometimes confuse virtualenv.
 rm -rf /usr/lib/python3/dist-packages/six*
-- 
GitLab


From 246780be77509fa4bea195d05fa73faa99467c58 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 7 Jan 2019 21:32:32 -0800
Subject: [PATCH 0319/2345] Hide the DotOpEmitter class and make it use a
 DotInfo instead of a HloInstr; NFC

 - Hiding it in the .cc is good encapsulation.
 - Making it depend on a DotInfo as opposed to an HLO instruction will make
   implementing batch dot (in terms of non-batch dot) easier.

PiperOrigin-RevId: 228281578
---
 .../xla/service/cpu/dot_op_emitter.cc         | 159 ++++++++++++++----
 .../compiler/xla/service/cpu/dot_op_emitter.h | 130 ++------------
 .../compiler/xla/service/cpu/ir_emitter.cc    |  16 +-
 3 files changed, 156 insertions(+), 149 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
index 70d8fee265..e8a84ebe6b 100644
--- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
@@ -53,9 +53,107 @@ namespace {
 bool ShouldUseMultiThreadedEigen(const HloModuleConfig& config) {
   return config.debug_options().xla_cpu_multi_thread_eigen();
 }
+
+// Helper class for emitting LLVM IR to perform the dot operation.
+class DotOpEmitter {
+ public:
+  explicit DotOpEmitter(DotInfo dot_info, string dot_hlo_name,
+                        const llvm_ir::IrArray& target_array,
+                        const llvm_ir::IrArray& lhs_array,
+                        const llvm_ir::IrArray& rhs_array,
+                        const llvm_ir::IrArray* addend_array,
+                        llvm::Value* executable_run_options_value,
+                        llvm::IRBuilder<>* b,
+                        const HloModuleConfig& hlo_module_config,
+                        const TargetMachineFeatures& target_machine_features);
+
+  // Emits the IR to perform the dot operation.
+  Status Emit();
+
+ private:
+  // Emits instructions to perform a scalar dot product (a multiply of the
+  // LHS and RHS) and store the results in the target.
+  Status EmitScalarDot();
+
+  // Emits a call to the CPU runtime to perform the matrix multiply.
+  Status EmitCallToRuntime();
+
+  // Represents the dimensions of a matrix-matrix multiply operation.
+  struct MatMultDims {
+    // The number of rows in the LHS.
+    int64 m;
+
+    // The number of columns in the LHS, which is also must be equal to the
+    // number of rows in the RHS.
+    int64 k;
+
+    // The number of columns on the RHS.
+    int64 n;
+
+    // True if the LHS matrix is column major.
+    bool lhs_column_major;
+
+    // True if the LHS contraction dimension is not 1.
+    bool lhs_non_canonical;
+
+    // True if the RHS matrix is column major.
+    bool rhs_column_major;
+
+    // True if the RHS contraction dimension is not 0.
+    bool rhs_non_canonical;
+
+    // True if the result matrix is column major.
+    bool target_column_major;
+  };
+
+  // Get the MatMultDims instance for the dot product this DotOpEmitter
+  // represents.  Precondition: the dot is of rank 2 (and thus its operands are
+  // of rank 2 as well).
+  MatMultDims GetMatMultDims() const;
+
+  // Lowers the dot operation as a tiled Matrix*Vector loop.
+  void EmitTiledLlvmIrGemv();
+
+  // Lowers the dot operation as a tiled Matrix*Matrix loop.
+  void EmitTiledLlvmIrGemm();
+
+  // Lowers the dot operation as a naive nested loop that computes the result
+  // one element at a time.
+  void EmitNaiveLlvmIrGemm();
+
+  // When doing a tiled GEMV in LLVM IR, a "tile" consists of this many vector
+  // registers.
+  int64 GetGemvTilingFactor() const {
+    const int64 kDefaultTilingFactor = 8;
+    return options::LlvmIrGemvTilingFactor(hlo_module_config_)
+        .value_or(kDefaultTilingFactor);
+  }
+
+  std::tuple<int64, int64, int64> GetGemmTileSize() const {
+    // Tuned for broadwell - Intel(R) Xeon(R) CPU E5-2690 v4 @ 2.60GHz
+    //
+    // TODO(b/80093688): Tune for other architectures and centralize this
+    // information in one place.
+    const std::tuple<int64, int64, int64> kDefaultTileSize =
+        std::tuple<int64, int64, int64>(11, 9, 1);
+    return options::LlvmIrGemmTileSize(hlo_module_config_)
+        .value_or(kDefaultTileSize);
+  }
+
+  DotInfo dot_info_;
+  string dot_hlo_name_;
+  const llvm_ir::IrArray& target_array_;
+  const llvm_ir::IrArray& lhs_array_;
+  const llvm_ir::IrArray& rhs_array_;
+  const llvm_ir::IrArray* addend_array_;
+  llvm::Value* executable_run_options_value_;
+  llvm::IRBuilder<>* b_;
+  const HloModuleConfig& hlo_module_config_;
+  const TargetMachineFeatures& target_machine_features_;
+};
 }  // namespace
 
-DotOpEmitter::DotOpEmitter(const HloInstruction& dot,
+DotOpEmitter::DotOpEmitter(DotInfo dot_info, string dot_hlo_name,
                            const llvm_ir::IrArray& target_array,
                            const llvm_ir::IrArray& lhs_array,
                            const llvm_ir::IrArray& rhs_array,
@@ -64,7 +162,8 @@ DotOpEmitter::DotOpEmitter(const HloInstruction& dot,
                            llvm::IRBuilder<>* b,
                            const HloModuleConfig& hlo_module_config,
                            const TargetMachineFeatures& target_machine_features)
-    : dot_(dot),
+    : dot_info_(std::move(dot_info)),
+      dot_hlo_name_(std::move(dot_hlo_name)),
       target_array_(target_array),
       lhs_array_(lhs_array),
       rhs_array_(rhs_array),
@@ -74,23 +173,8 @@ DotOpEmitter::DotOpEmitter(const HloInstruction& dot,
       hlo_module_config_(hlo_module_config),
       target_machine_features_(target_machine_features) {}
 
-/* static */ Status DotOpEmitter::EmitDotOperation(
-    const HloInstruction& dot, const llvm_ir::IrArray& target_array,
-    const llvm_ir::IrArray& lhs_array, const llvm_ir::IrArray& rhs_array,
-    const llvm_ir::IrArray* addend_array,
-    llvm::Value* executable_run_options_value, llvm::IRBuilder<>* b,
-    const HloModuleConfig& hlo_module_config,
-    const TargetMachineFeatures& target_machine_features) {
-  PrimitiveType type = target_array.GetShape().element_type();
-  TF_RET_CHECK(F16 == type || F32 == type || F64 == type || C64 == type);
-  DotOpEmitter dot_emitter(dot, target_array, lhs_array, rhs_array,
-                           addend_array, executable_run_options_value, b,
-                           hlo_module_config, target_machine_features);
-  return dot_emitter.Emit();
-}
-
 void DotOpEmitter::EmitTiledLlvmIrGemm() {
-  PrimitiveType primitive_type = dot_.shape().element_type();
+  PrimitiveType primitive_type = dot_info_.result_shape.element_type();
   MatMultDims mat_mult_dims = GetMatMultDims();
 
   llvm::Value* lhs = lhs_array_.GetBasePointer();
@@ -136,7 +220,7 @@ void DotOpEmitter::EmitTiledLlvmIrGemm() {
 }
 
 void DotOpEmitter::EmitTiledLlvmIrGemv() {
-  PrimitiveType primitive_type = dot_.shape().element_type();
+  PrimitiveType primitive_type = dot_info_.result_shape.element_type();
 
   CHECK(primitive_util::IsFloatingPointType(primitive_type) ||
         primitive_util::IsIntegralType(primitive_type));
@@ -258,11 +342,6 @@ Status DotOpEmitter::Emit() {
   // which performs the sum-of-products (the reduction loop) before storing
   // the result in the output buffer.
 
-  // This routine assumes that the dot operation is not in a parallelized
-  // enclosing computation.
-  CHECK(
-      dot_.parent()->root_instruction()->outer_dimension_partitions().empty());
-
   const Shape& lhs_shape = lhs_array_.GetShape();
   const Shape& rhs_shape = rhs_array_.GetShape();
 
@@ -273,7 +352,7 @@ Status DotOpEmitter::Emit() {
     return EmitScalarDot();
   }
 
-  switch (GetDotImplementationStrategy(hlo_module_config_, DotInfo(dot_),
+  switch (GetDotImplementationStrategy(hlo_module_config_, dot_info_,
                                        target_machine_features_)) {
     case DotImplementationStrategy::kNaiveLlvmIr:
       EmitNaiveLlvmIrGemm();
@@ -297,7 +376,7 @@ void DotOpEmitter::EmitNaiveLlvmIrGemm() {
 
   const Shape& lhs_shape = lhs_array_.GetShape();
   const Shape& rhs_shape = rhs_array_.GetShape();
-  const DotDimensionNumbers& dim_nums = dot_.dot_dimension_numbers();
+  const DotDimensionNumbers& dim_nums = dot_info_.dim_nums;
 
   // Reduce along dimension 0 of the LHS and 1 of the RHS. Vectors are a special
   // case where the reduction dimension is 0 for both LHS and RHS. This results
@@ -317,7 +396,7 @@ void DotOpEmitter::EmitNaiveLlvmIrGemm() {
   // Create loop nests which loop through the LHS operand dimensions and the RHS
   // operand dimensions. The reduction dimension of the LHS and RHS are handled
   // in a separate innermost loop which performs the sum of products.
-  llvm_ir::ForLoopNest loop_nest(llvm_ir::IrName(&dot_), b_);
+  llvm_ir::ForLoopNest loop_nest(llvm_ir::IrName(dot_hlo_name_), b_);
   llvm_ir::IrArray::Index lhs_index = loop_nest.EmitOperandArrayLoopNest(
       lhs_array_, /*dimension_to_skip=*/lhs_reduction_dimension, "lhs");
   llvm_ir::IrArray::Index rhs_index = loop_nest.EmitOperandArrayLoopNest(
@@ -561,11 +640,11 @@ Status DotOpEmitter::EmitCallToRuntime() {
 }
 
 DotOpEmitter::MatMultDims DotOpEmitter::GetMatMultDims() const {
-  CHECK_EQ(dot_.shape().dimensions_size(), 2);
+  CHECK_EQ(dot_info_.result_shape.dimensions_size(), 2);
 
   const Shape& lhs_shape = lhs_array_.GetShape();
   const Shape& rhs_shape = rhs_array_.GetShape();
-  const DotDimensionNumbers& dim_nums = dot_.dot_dimension_numbers();
+  const DotDimensionNumbers& dim_nums = dot_info_.dim_nums;
 
   return {
       /*m=*/lhs_shape.dimensions(1 - dim_nums.lhs_contracting_dimensions(0)),
@@ -754,5 +833,27 @@ bool DotOperandsAndResultMustHaveRowMajorLayout(
   return impl_strategy == DotImplementationStrategy::kTiledLlvmIrGemm ||
          impl_strategy == DotImplementationStrategy::kEigen;
 }
+
+Status EmitDotOperation(const HloInstruction& dot,
+                        const llvm_ir::IrArray& target_array,
+                        const llvm_ir::IrArray& lhs_array,
+                        const llvm_ir::IrArray& rhs_array,
+                        const llvm_ir::IrArray* addend_array,
+                        llvm::Value* executable_run_options_value,
+                        llvm::IRBuilder<>* b,
+                        const HloModuleConfig& hlo_module_config,
+                        const TargetMachineFeatures& target_machine_features) {
+  // This routine assumes that the dot operation is not in a parallelized
+  // enclosing computation.
+  CHECK(dot.parent()->root_instruction()->outer_dimension_partitions().empty());
+
+  PrimitiveType type = target_array.GetShape().element_type();
+  TF_RET_CHECK(F16 == type || F32 == type || F64 == type || C64 == type);
+  DotOpEmitter dot_emitter(DotInfo(dot), dot.name(), target_array, lhs_array,
+                           rhs_array, addend_array,
+                           executable_run_options_value, b, hlo_module_config,
+                           target_machine_features);
+  return dot_emitter.Emit();
+}
 }  // namespace cpu
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.h b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.h
index eaa226a4c2..105bd3005c 100644
--- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.h
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.h
@@ -48,118 +48,24 @@ bool DotImplementationCanHandleTranspose(
 absl::optional<int64> ProfitableToMakeDotOperandColumnMajor(
     const HloInstruction& hlo);
 
-// Helper class for emitting LLVM IR to perform the dot operation.
-class DotOpEmitter {
- public:
-  // Emit LLVM IR to perform the dot operation on lhs_array and rhs_array and
-  // place the result in target_array. IR is emitted at current insert point of
-  // the builder. Upon completion of the method, the insert point is set to the
-  // end of all instructions emitted for this operation.
-  //
-  // If `addend_array` is not nullptr then it must be an array of the same
-  // dimensions as the result, and the result is computed as `addend_array` +
-  // dot(`lhs_array`, `rhs_array`).  A non-null `addend_array` is only supported
-  // for Matrix-vector products.
-  static Status EmitDotOperation(
-      const HloInstruction& dot, const llvm_ir::IrArray& target_array,
-      const llvm_ir::IrArray& lhs_array, const llvm_ir::IrArray& rhs_array,
-      const llvm_ir::IrArray* addend_array,
-      llvm::Value* executable_run_options_value, llvm::IRBuilder<>* b,
-      const HloModuleConfig& hlo_module_config,
-      const TargetMachineFeatures& target_machine_features);
-
- private:
-  DotOpEmitter(const HloInstruction& dot, const llvm_ir::IrArray& target_array,
-               const llvm_ir::IrArray& lhs_array,
-               const llvm_ir::IrArray& rhs_array,
-               const llvm_ir::IrArray* addend_array,
-               llvm::Value* executable_run_options_value, llvm::IRBuilder<>* b,
-               const HloModuleConfig& hlo_module_config,
-               const TargetMachineFeatures& target_machine_features);
-
-  // Emits the IR to perform the dot operation.
-  Status Emit();
-
-  // Emits instructions to perform a scalar dot product (a multiply of the
-  // LHS and RHS) and store the results in the target.
-  Status EmitScalarDot();
-
-  // Emits a call to the CPU runtime to perform the matrix multiply.
-  Status EmitCallToRuntime();
-
-  // Represents the dimensions of a matrix-matrix multiply operation.
-  struct MatMultDims {
-    // The number of rows in the LHS.
-    int64 m;
-
-    // The number of columns in the LHS, which is also must be equal to the
-    // number of rows in the RHS.
-    int64 k;
-
-    // The number of columns on the RHS.
-    int64 n;
-
-    // True if the LHS matrix is column major.
-    bool lhs_column_major;
-
-    // True if the LHS contraction dimension is not 1.
-    bool lhs_non_canonical;
-
-    // True if the RHS matrix is column major.
-    bool rhs_column_major;
-
-    // True if the RHS contraction dimension is not 0.
-    bool rhs_non_canonical;
-
-    // True if the result matrix is column major.
-    bool target_column_major;
-  };
-
-  // Get the MatMultDims instance for the dot product this DotOpEmitter
-  // represents.  Precondition: the dot is of rank 2 (and thus its operands are
-  // of rank 2 as well).
-  MatMultDims GetMatMultDims() const;
-
-  // Lowers the dot operation as a tiled Matrix*Vector loop.
-  void EmitTiledLlvmIrGemv();
-
-  // Lowers the dot operation as a tiled Matrix*Matrix loop.
-  void EmitTiledLlvmIrGemm();
-
-  // Lowers the dot operation as a naive nested loop that computes the result
-  // one element at a time.
-  void EmitNaiveLlvmIrGemm();
-
-  // When doing a tiled GEMV in LLVM IR, a "tile" consists of this many vector
-  // registers.
-  int64 GetGemvTilingFactor() const {
-    const int64 kDefaultTilingFactor = 8;
-    return options::LlvmIrGemvTilingFactor(hlo_module_config_)
-        .value_or(kDefaultTilingFactor);
-  }
-
-  std::tuple<int64, int64, int64> GetGemmTileSize() const {
-    // Tuned for broadwell - Intel(R) Xeon(R) CPU E5-2690 v4 @ 2.60GHz
-    //
-    // TODO(b/80093688): Tune for other architectures and centralize this
-    // information in one place.
-    const std::tuple<int64, int64, int64> kDefaultTileSize =
-        std::tuple<int64, int64, int64>(11, 9, 1);
-    return options::LlvmIrGemmTileSize(hlo_module_config_)
-        .value_or(kDefaultTileSize);
-  }
-
-  const HloInstruction& dot_;
-  const llvm_ir::IrArray& target_array_;
-  const llvm_ir::IrArray& lhs_array_;
-  const llvm_ir::IrArray& rhs_array_;
-  const llvm_ir::IrArray* addend_array_;
-  llvm::Value* executable_run_options_value_;
-  llvm::IRBuilder<>* b_;
-  const HloModuleConfig& hlo_module_config_;
-  const TargetMachineFeatures& target_machine_features_;
-};
-
+// Emit LLVM IR to perform the dot operation on lhs_array and rhs_array and
+// place the result in target_array. IR is emitted at current insert point of
+// the builder. Upon completion of the method, the insert point is set to the
+// end of all instructions emitted for this operation.
+//
+// If `addend_array` is not nullptr then it must be an array of the same
+// dimensions as the result, and the result is computed as `addend_array` +
+// dot(`lhs_array`, `rhs_array`).  A non-null `addend_array` is only supported
+// for Matrix-vector products.
+Status EmitDotOperation(const HloInstruction& dot,
+                        const llvm_ir::IrArray& target_array,
+                        const llvm_ir::IrArray& lhs_array,
+                        const llvm_ir::IrArray& rhs_array,
+                        const llvm_ir::IrArray* addend_array,
+                        llvm::Value* executable_run_options_value,
+                        llvm::IRBuilder<>* b,
+                        const HloModuleConfig& hlo_module_config,
+                        const TargetMachineFeatures& target_machine_features);
 }  // namespace cpu
 }  // namespace xla
 
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index b352848824..91e3693354 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -970,10 +970,10 @@ Status IrEmitter::HandleDot(HloInstruction* dot) {
           << llvm_ir::DumpToString(*target_array.GetBasePointer());
 
   // Dot operation is complicated so we delegate to a helper class.
-  return DotOpEmitter::EmitDotOperation(
-      *dot, target_array, lhs_array, rhs_array, /*addend_array=*/nullptr,
-      GetExecutableRunOptionsArgument(), &b_, hlo_module_config_,
-      target_machine_features_);
+  return EmitDotOperation(*dot, target_array, lhs_array, rhs_array,
+                          /*addend_array=*/nullptr,
+                          GetExecutableRunOptionsArgument(), &b_,
+                          hlo_module_config_, target_machine_features_);
 }
 
 StatusOr<llvm::Value*> IrEmitter::EmitTargetElementLoopBodyForConvolution(
@@ -2203,10 +2203,10 @@ Status IrEmitter::HandleFusion(HloInstruction* fusion) {
     llvm_ir::IrArray addend_array(
         GetIrArrayFor(fusion->operand(addend_param_number)));
 
-    TF_RETURN_IF_ERROR(DotOpEmitter::EmitDotOperation(
-        *dot, target_array, lhs_array, rhs_array, &addend_array,
-        GetExecutableRunOptionsArgument(), &b_, hlo_module_config_,
-        target_machine_features_));
+    TF_RETURN_IF_ERROR(
+        EmitDotOperation(*dot, target_array, lhs_array, rhs_array,
+                         &addend_array, GetExecutableRunOptionsArgument(), &b_,
+                         hlo_module_config_, target_machine_features_));
     return Status::OK();
   } else {
     return Unimplemented("Fusion kind not implemented on CPU");
-- 
GitLab


From 2ca0bf7cf4e16cbed6affd7dd844fea4ed3a62df Mon Sep 17 00:00:00 2001
From: Andiry Xu <andiryxu@google.com>
Date: Mon, 7 Jan 2019 22:40:38 -0800
Subject: [PATCH 0320/2345] This CL checks StepStats, records the output slot
 for each Switch Op execution, and adds the output slot vector to GraphDef
 Switch node. When VirtualScheduler schedules the Switch Op, it will take the
 actual output branch.

PiperOrigin-RevId: 228285499
---
 .../core/grappler/costs/virtual_scheduler.cc  | 113 +++-
 .../core/grappler/costs/virtual_scheduler.h   |  12 +
 .../grappler/costs/virtual_scheduler_test.cc  | 516 ++++++++++++++++++
 3 files changed, 623 insertions(+), 18 deletions(-)

diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.cc b/tensorflow/core/grappler/costs/virtual_scheduler.cc
index 7071c679dc..bb8425a5a1 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler.cc
+++ b/tensorflow/core/grappler/costs/virtual_scheduler.cc
@@ -15,8 +15,6 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/costs/virtual_scheduler.h"
 
-#include <math.h>
-
 #include "tensorflow/core/framework/allocation_description.pb.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
@@ -38,6 +36,12 @@ namespace tensorflow {
 namespace grappler {
 namespace {
 
+// Optional attribute name for Switch op as a vector of int that tells
+// which branch the Switch output is taken on every round of execution.
+// We use this side information, if provided, for scheduling ops after Switch
+// correctly (e.g., While loop).
+constexpr char kOutputSlots[] = "_output_slot_vector";
+
 Costs CombineCosts(const Costs& left, const Costs& right) {
   CHECK_NE(left.max_memory, kMemoryUnknown);
   CHECK_NE(left.max_per_op_buffers, kMemoryUnknown);
@@ -402,6 +406,8 @@ Status VirtualScheduler::Init() {
     name_to_node[node->name()] = node;
   }
 
+  // Traverse the graph to check if the graph is annotated with Switch outputs.
+  // Also record _Send nodes.
   // TODO(dyoon): Instead of identifying _Send node here manually, add _Send
   // to _Recv as control dependency when creating GrapplerItem.
   std::unordered_map<string, const NodeDef*> name_to_send;
@@ -410,6 +416,11 @@ Status VirtualScheduler::Init() {
       const auto& attr = node.attr();
       name_to_send[attr.at("tensor_name").s()] = &node;
     }
+
+    if (IsSwitch(node)) {
+      const auto& attr = node.attr();
+      if (attr.count(kOutputSlots) > 0) switch_outputs_annotated_ = true;
+    }
   }
 
   // To reuse _Recv ops.
@@ -771,6 +782,82 @@ Costs& VirtualScheduler::FindOrCreateZero(const string& op_name,
   return it->second;
 }
 
+// Check Switch outputs in updated MetaGraphDef, add corresponding nodes to
+// ready queue.
+// Fallback to add all outputs if fail to find the actual output.
+bool VirtualScheduler::AddSwitchOutputsToReadyQueue(
+    const NodeDef* node, int curr_iter, const Costs::Duration& curr_time) {
+  if (node->attr().count(kOutputSlots) == 0) return false;
+
+  auto& node_state = node_map_[node];
+  const auto& slot_vector = node->attr().at(kOutputSlots);
+  if (slot_vector.list().i_size() <= curr_iter) {
+    // Sometimes we encounter infinite loop. Fall back to add all outputs.
+    return false;
+  }
+
+  int slot = slot_vector.list().i(curr_iter);
+  for (const auto& port_num_output_pair : node_state.outputs) {
+    if (port_num_output_pair.first != slot) continue;
+
+    for (auto* output_node : port_num_output_pair.second) {
+      auto& output_state = node_map_[output_node];
+      output_state.num_inputs_ready++;
+      // Execute a node as soon as all its inputs are ready. Merge nodes
+      // are special since they run as soon as one of their inputs becomes
+      // available.
+      if (output_state.num_inputs_ready == output_state.inputs.size() ||
+          IsMerge(*output_node)) {
+        // This output node is now ready.
+        output_state.time_ready = curr_time;
+        ready_nodes_->AddNode(output_node);
+        VLOG(3) << "Node " << node->name() << " iter " << curr_iter << "/"
+                << slot_vector.list().i_size() << " Add Switch output " << slot
+                << ": " << output_node->name();
+      }
+    }
+    return true;
+  }
+
+  return false;
+}
+
+void VirtualScheduler::AddOutputNodesToReadyQueue(
+    const NodeDef* node, const Costs::Duration& curr_time) {
+  auto& node_state = node_map_[node];
+  int curr_iter = node_state.num_executed_times;
+  ++node_state.num_executed_times;
+
+  if (switch_outputs_annotated_) {
+    // If the graph is annotated with StepStats, reset num_inputs_ready so we
+    // can schedule the node multiple times.
+    node_state.num_inputs_ready = 0;
+
+    // For Switch node, get output branch from updated MetaGraphDef.
+    if (IsSwitch(*node) &&
+        AddSwitchOutputsToReadyQueue(node, curr_iter, curr_time))
+      return;
+  }
+
+  // Increment num_inputs_ready of the output nodes and maybe add to ready
+  // nodes.
+  for (const auto& port_num_output_pair : node_state.outputs) {
+    for (auto* output_node : port_num_output_pair.second) {
+      auto& output_state = node_map_[output_node];
+      output_state.num_inputs_ready++;
+      // Execute a node as soon as all its inputs are ready. Merge nodes are
+      // special since they run as soon as one of their inputs becomes
+      // available.
+      if (output_state.num_inputs_ready == output_state.inputs.size() ||
+          IsMerge(*output_node)) {
+        // This output node is now ready.
+        output_state.time_ready = curr_time;
+        ready_nodes_->AddNode(output_node);
+      }
+    }
+  }
+}
+
 bool VirtualScheduler::MarkCurrNodeExecuted(const Costs& node_costs) {
   // Update graph_costs_ and per-op costs.
   graph_costs_ = CombineCosts(graph_costs_, node_costs);
@@ -798,6 +885,10 @@ bool VirtualScheduler::MarkCurrNodeExecuted(const Costs& node_costs) {
   // Node is scheduled when the device is available AND all the inputs are
   // ready; hence, time_scheduled is time_ready if time_ready > device curr
   // time.
+  // TODO(andiryxu): Current node_state result only records the last execution.
+  // With annotated MetaGraph we can schedule a node for multiple times.
+  // Refine NodeState structure accordingly, e.g. record time_scheduled in a
+  // vector.
   node_state.time_scheduled =
       std::max(device.GetCurrTime(), node_state.time_ready);
   // Override device curr time with the time_scheduled.
@@ -831,22 +922,8 @@ bool VirtualScheduler::MarkCurrNodeExecuted(const Costs& node_costs) {
           << ", scheduled: " << node_state.time_scheduled.count()
           << ", finished: " << node_state.time_finished.count();
 
-  // Increment num_inputs_ready of the output nodes and maybe add to ready nodes
-  for (const auto& port_num_output_pair : node_state.outputs) {
-    for (auto* output_node : port_num_output_pair.second) {
-      auto& output_state = node_map_[output_node];
-      output_state.num_inputs_ready++;
-      // Execute a node as soon as all its inputs are ready. Merge nodes are
-      // special since they run as soon as one of their inputs becomes
-      // available.
-      if (output_state.num_inputs_ready == output_state.inputs.size() ||
-          IsMerge(*output_node)) {
-        // This output node is now ready.
-        output_state.time_ready = curr_time;
-        ready_nodes_->AddNode(output_node);
-      }
-    }
-  }
+  // Check outputs, add ready nodes to queue.
+  AddOutputNodesToReadyQueue(node, curr_time);
 
   // Increment num_outputs_executed of the input nodes and maybe update memory.
   for (const auto& input_port : node_state.inputs) {
diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.h b/tensorflow/core/grappler/costs/virtual_scheduler.h
index f500e4ab51..ffc5e9187c 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler.h
+++ b/tensorflow/core/grappler/costs/virtual_scheduler.h
@@ -70,11 +70,15 @@ struct NodeState {
   // Each output port uses up memory space from time_scheduled to its
   // time_no_references.
 
+  // How many times this node has been executed, e.g. in a while loop.
+  int num_executed_times;
+
   NodeState() {
     num_inputs_ready = 0;
     time_ready = Costs::Duration::max();
     time_scheduled = Costs::Duration::max();
     time_finished = Costs::Duration::max();
+    num_executed_times = 0;
     // Note that num_outputs_executed and time_no_references are not initialized
     // here, since we don't know the size (i.e., # outputs for this node).
   }
@@ -330,6 +334,10 @@ class VirtualScheduler {
                           std::map<string, Costs>* op_cost);
   float Round2(const float x) const;
   bool IsPersistentNode(const NodeDef* node) const;
+  bool AddSwitchOutputsToReadyQueue(const NodeDef* node, int curr_iter,
+                                    const Costs::Duration& curr_time);
+  void AddOutputNodesToReadyQueue(const NodeDef* node,
+                                  const Costs::Duration& curr_time);
 
   // Scheduler states:
   ReadyNodeManager* ready_nodes_;  // Not owned.
@@ -360,6 +368,10 @@ class VirtualScheduler {
   bool initialized_;
   bool track_mem_usage_snapshot_;
 
+  // Whether the input graph includes Switch nodes annotated with output slots
+  // information.
+  bool switch_outputs_annotated_ = false;
+
   VirtualPlacer placer_;  // owned.
 };
 
diff --git a/tensorflow/core/grappler/costs/virtual_scheduler_test.cc b/tensorflow/core/grappler/costs/virtual_scheduler_test.cc
index c5651715f4..5e044b2037 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler_test.cc
+++ b/tensorflow/core/grappler/costs/virtual_scheduler_test.cc
@@ -869,6 +869,439 @@ versions {
     grappler_item_->fetch = {"while/Exit", "while/Exit_1"};
   }
 
+  // A simple while loop strengthened with Switch outputs.
+  void CreateGrapplerItemWithLoopSwitchOutputs() {
+    // Test graph produced in python using:
+    /*
+      with tf.Graph().as_default():
+      i0 = tf.constant(0)
+      m0 = tf.ones([2, 2])
+      c = lambda i, m: i < 10
+      b = lambda i, m: [i+1, tf.concat([m, m], axis=0)]
+      r = tf.while_loop(
+      c, b, loop_vars=[i0, m0],
+      shape_invariants=[i0.get_shape(), tf.TensorShape([None, 2])])
+      with open('/tmp/graph.pbtxt', 'w') as f:
+      f.write(str(tf.get_default_graph().as_graph_def()))
+    */
+    const string gdef_ascii = R"EOF(
+node {
+  name: "Const"
+  op: "Const"
+  attr {
+    key: "dtype"
+    value {
+      type: DT_INT32
+    }
+  }
+  attr {
+    key: "value"
+    value {
+      tensor {
+        dtype: DT_INT32
+        tensor_shape {
+        }
+        int_val: 0
+      }
+    }
+  }
+}
+node {
+  name: "ones"
+  op: "Const"
+  attr {
+    key: "dtype"
+    value {
+      type: DT_FLOAT
+    }
+  }
+  attr {
+    key: "value"
+    value {
+      tensor {
+        dtype: DT_FLOAT
+        tensor_shape {
+          dim {
+            size: 2
+          }
+          dim {
+            size: 2
+          }
+        }
+        float_val: 1.0
+      }
+    }
+  }
+}
+node {
+  name: "while/Enter"
+  op: "Enter"
+  input: "Const"
+  attr {
+    key: "T"
+    value {
+      type: DT_INT32
+    }
+  }
+  attr {
+    key: "frame_name"
+    value {
+      s: "while/while/"
+    }
+  }
+  attr {
+    key: "is_constant"
+    value {
+      b: false
+    }
+  }
+  attr {
+    key: "parallel_iterations"
+    value {
+      i: 10
+    }
+  }
+}
+node {
+  name: "while/Enter_1"
+  op: "Enter"
+  input: "ones"
+  attr {
+    key: "T"
+    value {
+      type: DT_FLOAT
+    }
+  }
+  attr {
+    key: "frame_name"
+    value {
+      s: "while/while/"
+    }
+  }
+  attr {
+    key: "is_constant"
+    value {
+      b: false
+    }
+  }
+  attr {
+    key: "parallel_iterations"
+    value {
+      i: 10
+    }
+  }
+}
+node {
+  name: "while/Merge"
+  op: "Merge"
+  input: "while/Enter"
+  input: "while/NextIteration"
+  attr {
+    key: "N"
+    value {
+      i: 2
+    }
+  }
+  attr {
+    key: "T"
+    value {
+      type: DT_INT32
+    }
+  }
+}
+node {
+  name: "while/Merge_1"
+  op: "Merge"
+  input: "while/Enter_1"
+  input: "while/NextIteration_1"
+  attr {
+    key: "N"
+    value {
+      i: 2
+    }
+  }
+  attr {
+    key: "T"
+    value {
+      type: DT_FLOAT
+    }
+  }
+}
+node {
+  name: "while/Less/y"
+  op: "Const"
+  input: "^while/Merge"
+  attr {
+    key: "dtype"
+    value {
+      type: DT_INT32
+    }
+  }
+  attr {
+    key: "value"
+    value {
+      tensor {
+        dtype: DT_INT32
+        tensor_shape {
+        }
+        int_val: 10
+      }
+    }
+  }
+}
+node {
+  name: "while/Less"
+  op: "Less"
+  input: "while/Merge"
+  input: "while/Less/y"
+  attr {
+    key: "T"
+    value {
+      type: DT_INT32
+    }
+  }
+}
+node {
+  name: "while/LoopCond"
+  op: "LoopCond"
+  input: "while/Less"
+}
+node {
+  name: "while/Switch"
+  op: "Switch"
+  input: "while/Merge"
+  input: "while/LoopCond"
+  attr {
+    key: "T"
+    value {
+      type: DT_INT32
+    }
+  }
+  attr {
+    key: "_class"
+    value {
+      list {
+        s: "loc:@while/Merge"
+      }
+    }
+  }
+  attr {
+    key: "_output_slot_vector"
+    value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+        i: 0
+      }
+    }
+  }
+}
+node {
+  name: "while/Switch_1"
+  op: "Switch"
+  input: "while/Merge_1"
+  input: "while/LoopCond"
+  attr {
+    key: "T"
+    value {
+      type: DT_FLOAT
+    }
+  }
+  attr {
+    key: "_class"
+    value {
+      list {
+        s: "loc:@while/Merge_1"
+      }
+    }
+  }
+  attr {
+    key: "_output_slot_vector"
+    value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+        i: 0
+      }
+    }
+  }
+}
+node {
+  name: "while/Identity"
+  op: "Identity"
+  input: "while/Switch:1"
+  attr {
+    key: "T"
+    value {
+      type: DT_INT32
+    }
+  }
+}
+node {
+  name: "while/Identity_1"
+  op: "Identity"
+  input: "while/Switch_1:1"
+  attr {
+    key: "T"
+    value {
+      type: DT_FLOAT
+    }
+  }
+}
+node {
+  name: "while/add/y"
+  op: "Const"
+  input: "^while/Identity"
+  attr {
+    key: "dtype"
+    value {
+      type: DT_INT32
+    }
+  }
+  attr {
+    key: "value"
+    value {
+      tensor {
+        dtype: DT_INT32
+        tensor_shape {
+        }
+        int_val: 1
+      }
+    }
+  }
+}
+node {
+  name: "while/add"
+  op: "Add"
+  input: "while/Identity"
+  input: "while/add/y"
+  attr {
+    key: "T"
+    value {
+      type: DT_INT32
+    }
+  }
+}
+node {
+  name: "while/concat/axis"
+  op: "Const"
+  input: "^while/Identity"
+  attr {
+    key: "dtype"
+    value {
+      type: DT_INT32
+    }
+  }
+  attr {
+    key: "value"
+    value {
+      tensor {
+        dtype: DT_INT32
+        tensor_shape {
+        }
+        int_val: 0
+      }
+    }
+  }
+}
+node {
+  name: "while/concat"
+  op: "ConcatV2"
+  input: "while/Identity_1"
+  input: "while/Identity_1"
+  input: "while/concat/axis"
+  attr {
+    key: "N"
+    value {
+      i: 2
+    }
+  }
+  attr {
+    key: "T"
+    value {
+      type: DT_FLOAT
+    }
+  }
+  attr {
+    key: "Tidx"
+    value {
+      type: DT_INT32
+    }
+  }
+}
+node {
+  name: "while/NextIteration"
+  op: "NextIteration"
+  input: "while/add"
+  attr {
+    key: "T"
+    value {
+      type: DT_INT32
+    }
+  }
+}
+node {
+  name: "while/NextIteration_1"
+  op: "NextIteration"
+  input: "while/concat"
+  attr {
+    key: "T"
+    value {
+      type: DT_FLOAT
+    }
+  }
+}
+node {
+  name: "while/Exit"
+  op: "Exit"
+  input: "while/Switch"
+  attr {
+    key: "T"
+    value {
+      type: DT_INT32
+    }
+  }
+}
+node {
+  name: "while/Exit_1"
+  op: "Exit"
+  input: "while/Switch_1"
+  attr {
+    key: "T"
+    value {
+      type: DT_FLOAT
+    }
+  }
+}
+versions {
+  producer: 21
+}
+  )EOF";
+
+    grappler_item_.reset(new GrapplerItem);
+    CHECK(protobuf::TextFormat::ParseFromString(gdef_ascii,
+                                                &grappler_item_->graph));
+    grappler_item_->id = "test_graph";
+    grappler_item_->fetch = {"while/Exit", "while/Exit_1"};
+  }
+
+  // Create a FusedBatchNorm op that has multiple output ports.
   void CreateGrapplerItemWithInterDeviceTransfers() {
     tensorflow::Scope s = tensorflow::Scope::NewRootScope().WithDevice(kCPU0);
 
@@ -1942,6 +2375,89 @@ TEST_F(VirtualSchedulerTest, WhileLoop) {
   ValidateDependencyChain(start_times, {"while/Switch_1", "while/Exit_1"});
 }
 
+TEST_F(VirtualSchedulerTest, WhileLoopWithSwitchOutputs) {
+  // Init.
+  CreateGrapplerItemWithLoopSwitchOutputs();
+  InitScheduler();
+
+  // Runs the scheduler.
+  RunScheduler("");
+
+  RunMetadata metadata;
+  scheduler_->Summary(&metadata);
+
+  // Nodes in topological order:
+  // * const, ones
+  // * while/Enter, while/Enter_1
+  // * while/Merge, while/Merge_1
+  // * while/Less/y
+  // * while/Less
+  // * while/LoopCond
+  // * while/Switch, while/Switch_1
+  // * while/Identity, while/Identity_1, while/Exit, while/Exit_1
+  // * while/add/y, while/concat/axis
+  // * while/add, while/concat
+  // * while/NextIteration, while/NextIteration_1
+
+  int num_next_iteration = 0;
+  int num_next_iteration_1 = 0;
+  int num_exit = 0;
+  int num_exit_1 = 0;
+  int64 next_iter_start_micro;
+  int64 next_iter_1_start_micro;
+  int64 exit_start_micro;
+  int64 exit_1_start_micro;
+
+  std::unordered_map<string, int64> start_times;
+  for (const auto& device_step_stats : metadata.step_stats().dev_stats()) {
+    for (const auto& stats : device_step_stats.node_stats()) {
+      start_times[stats.node_name()] = stats.all_start_micros();
+      if (stats.node_name() == "while/NextIteration") {
+        ++num_next_iteration;
+        next_iter_start_micro = stats.all_start_micros();
+      } else if (stats.node_name() == "while/NextIteration_1") {
+        ++num_next_iteration_1;
+        next_iter_1_start_micro = stats.all_start_micros();
+      } else if (stats.node_name() == "while/Exit") {
+        ++num_exit;
+        exit_start_micro = stats.all_start_micros();
+      } else if (stats.node_name() == "while/Exit_1") {
+        ++num_exit_1;
+        exit_1_start_micro = stats.all_start_micros();
+      }
+    }
+  }
+
+  // Makes sure we run the loop body for ten times.
+  EXPECT_EQ(10, num_next_iteration);
+  EXPECT_EQ(10, num_next_iteration_1);
+  EXPECT_EQ(1, num_exit);
+  EXPECT_EQ(1, num_exit_1);
+
+  // Start times of while/NextIteration and while/NextIteration_1 should be
+  // different, so should be those of while/Exit and while/Exit_1.
+  EXPECT_NE(next_iter_start_micro, next_iter_1_start_micro);
+  EXPECT_NE(exit_start_micro, exit_1_start_micro);
+
+  // Checks dependency among the nodes; no matter what scheduling mechanism we
+  // use, the scheduled ops should follow these dependency chains.
+  // We have to break the loop into two parts, identified by Switch outputs.
+  ValidateDependencyChain(
+      start_times,
+      {"Const", "while/Enter", "while/Merge", "while/Less/y", "while/Less",
+       "while/LoopCond", "while/Switch", "while/Exit"});
+  ValidateDependencyChain(start_times, {"while/Identity", "while/add/y",
+                                        "while/add", "while/NextIteration"});
+  ValidateDependencyChain(
+      start_times, {"ones", "while/Enter_1", "while/Merge_1", "while/Switch_1",
+                    "while/Exit_1"});
+  ValidateDependencyChain(start_times, {"while/Identity_1", "while/concat",
+                                        "while/NextIteration_1"});
+  ValidateDependencyChain(
+      start_times, {"while/Identity", "while/concat/axis", "while/concat"});
+  ValidateDependencyChain(start_times, {"while/Identity", "while/add"});
+}
+
 TEST_F(VirtualSchedulerTest, InterDeviceTransfer) {
   // Init.
   CreateGrapplerItemWithInterDeviceTransfers();
-- 
GitLab


From 34d2f298f78d009b5ad2732e436cdae640509a33 Mon Sep 17 00:00:00 2001
From: "Li, Guizi" <guizi.li@intel.com>
Date: Tue, 8 Jan 2019 16:11:08 +0800
Subject: [PATCH 0321/2345] update based on review comments

---
 tensorflow/core/graph/mkl_layout_pass.cc      | 100 ++++----
 tensorflow/core/graph/mkl_layout_pass_test.cc | 160 +++++++------
 tensorflow/core/kernels/mkl_conv_ops.cc       |  10 +-
 tensorflow/core/kernels/mkl_fused_ops_test.cc | 213 +++++++-----------
 4 files changed, 212 insertions(+), 271 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 23de237df4..dc799e7819 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -538,10 +538,10 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     minfo_.push_back({csinfo_.conv2d_grad_filter, csinfo_.bias_add_grad,
                       csinfo_.conv2d_grad_filter_with_bias,
                       GetConv2DBackpropFilterOrBiasAddGrad});
-    minfo_.push_back(
-        {csinfo_.pad, csinfo_.conv2d, csinfo_.pad_with_conv2d, GetPadOrConv2D});
     // Merge Pad and Conv2d, only if the pad op is "Pad"
     // Doesn't merge if pad op is "PadV2" or "MirrorPad"
+    minfo_.push_back(
+        {csinfo_.pad, csinfo_.conv2d, csinfo_.pad_with_conv2d, GetPadOrConv2D});
 
     minfo_.push_back({csinfo_.pad, csinfo_.fused_conv2d,
                       csinfo_.pad_with_fused_conv2d, GetPadOrFusedConv2D});
@@ -930,8 +930,8 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
 
   // Find Pad or _FusedConv2D node that can be merged with input node 'm'.
   // If input 'm' is Pad, then check if there exists _FusedConv2D node that can
-  // be  merged with 'm'. If input 'm' is _FusedConv2D, then check if there
-  // exists Pad  node that can be merged with 'm'.
+  // be merged with 'm'. If input 'm' is _FusedConv2D, then check if there
+  // exists Pad node that can be merged with 'm'.
   static Node* GetPadOrFusedConv2D(const Node* m) {
     DCHECK(m);
     Node* n = nullptr;
@@ -950,7 +950,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     } else {
       DCHECK_EQ(m->type_string(), csinfo_.fused_conv2d);
       // If m is _FusedConv2D, Go over all input edges
-      // and search for Pad  Node.
+      // and search for Pad node.
       for (const Edge* e : m->in_edges()) {
         if (!e->IsControlEdge() && e->src()->type_string() == csinfo_.pad) {
           n = e->src();
@@ -959,8 +959,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
         }
       }
     }
-    // Check if only VALID type of padding is used
-    // or not.
+    // Check if only VALID type of padding is used or not.
     if (n != nullptr) {
       string padding;
       TF_CHECK_OK(GetNodeAttr(conv_node->def(), "padding", &padding));
@@ -968,8 +967,9 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
         // Then do not merge.
         n = nullptr;
         VLOG(1) << "MklLayoutRewritePass: Could match Pad and _FusedConv2D "
-                << "node for merging. Only VALID type of padding in conv op "
-                << "can be merged with Pad op Input node: " << m->DebugString();
+                << "nodes but cannot merge them. Only conv ops with padding "
+                << "type VALID can be merged with Pad op Input node: "
+                << m->DebugString();
       }
     } else {
       VLOG(1) << "MklLayoutRewritePass: Could not find matching "
@@ -2090,7 +2090,6 @@ void MklLayoutRewritePass::CopyAttrsPadWithFusedConv2D(const Node* orig_node,
                                                        NodeBuilder* nb,
                                                        bool change_format) {
   DataType Tpaddings;
-  bool is_filter_const;
 
   CopyAttrsFusedConv2D(orig_node, nb, change_format);
 
@@ -2099,11 +2098,10 @@ void MklLayoutRewritePass::CopyAttrsPadWithFusedConv2D(const Node* orig_node,
   // Check if filter is a constant.
   Node* filter_node = nullptr;
   orig_node->input_node(1, &filter_node);
-  is_filter_const = filter_node->IsConstant();
 
   // Add attributes to new node.
   nb->Attr("Tpaddings", Tpaddings);
-  nb->Attr("is_filter_const", is_filter_const);
+  nb->Attr("is_filter_const", filter_node->IsConstant());
 }
 
 // Used with MergePadWithConv2D
@@ -2141,7 +2139,7 @@ void MklLayoutRewritePass::CopyAttrsFromPadAndConv2D(const Node* orig_node1,
 }
 
 void MklLayoutRewritePass::CopyAttrsFromPadAndFusedConv2D(
-    const Node* orig_node1, const Node* orig_node2, NodeBuilder* nb,
+    const Node* fused_conv2d, const Node* pad, NodeBuilder* nb,
     bool change_format) {
   DataType T;
   int num_args;
@@ -2154,15 +2152,15 @@ void MklLayoutRewritePass::CopyAttrsFromPadAndFusedConv2D(
   DataType Tpaddings;
 
   // Get all attributes from old node.
-  TF_CHECK_OK(GetNodeAttr(orig_node1->def(), "T", &T));
-  TF_CHECK_OK(GetNodeAttr(orig_node1->def(), "num_args", &num_args));
-  TF_CHECK_OK(GetNodeAttr(orig_node1->def(), "strides", &strides));
-  TF_CHECK_OK(GetNodeAttr(orig_node1->def(), "padding", &padding));
-  TF_CHECK_OK(GetNodeAttr(orig_node1->def(), "data_format", &data_format));
-  TF_CHECK_OK(GetNodeAttr(orig_node1->def(), "dilations", &dilations));
-  TF_CHECK_OK(GetNodeAttr(orig_node1->def(), "fused_ops", &fused_ops));
-  TF_CHECK_OK(GetNodeAttr(orig_node1->def(), "epsilon", &epsilon));
-  TF_CHECK_OK(GetNodeAttr(orig_node2->def(), "Tpaddings", &Tpaddings));
+  TF_CHECK_OK(GetNodeAttr(fused_conv2d->def(), "T", &T));
+  TF_CHECK_OK(GetNodeAttr(fused_conv2d->def(), "num_args", &num_args));
+  TF_CHECK_OK(GetNodeAttr(fused_conv2d->def(), "strides", &strides));
+  TF_CHECK_OK(GetNodeAttr(fused_conv2d->def(), "padding", &padding));
+  TF_CHECK_OK(GetNodeAttr(fused_conv2d->def(), "data_format", &data_format));
+  TF_CHECK_OK(GetNodeAttr(fused_conv2d->def(), "dilations", &dilations));
+  TF_CHECK_OK(GetNodeAttr(fused_conv2d->def(), "fused_ops", &fused_ops));
+  TF_CHECK_OK(GetNodeAttr(fused_conv2d->def(), "epsilon", &epsilon));
+  TF_CHECK_OK(GetNodeAttr(pad->def(), "Tpaddings", &Tpaddings));
 
   // Add attributes to new node.
   nb->Attr("T", T);
@@ -2701,12 +2699,12 @@ Status MklLayoutRewritePass::MergeConv2DWithBiasAdd(std::unique_ptr<Graph>* g,
 
 Status MklLayoutRewritePass::MergePadWithConv2D(std::unique_ptr<Graph>* g,
                                                 Node* m, Node* n) {
-  DCHECK(((m->type_string() == csinfo_.pad &&
-           (n->type_string() == csinfo_.conv2d ||
-            n->type_string() == csinfo_.fused_conv2d))) ||
-         ((n->type_string() == csinfo_.pad &&
-           (m->type_string() == csinfo_.conv2d ||
-            m->type_string() == csinfo_.fused_conv2d))));
+  DCHECK((m->type_string() == csinfo_.pad &&
+          (n->type_string() == csinfo_.conv2d ||
+           n->type_string() == csinfo_.fused_conv2d)) ||
+         (n->type_string() == csinfo_.pad &&
+          (m->type_string() == csinfo_.conv2d ||
+           m->type_string() == csinfo_.fused_conv2d)));
 
   bool is_fused_conv2d = n->type_string() == csinfo_.fused_conv2d ||
                          m->type_string() == csinfo_.fused_conv2d;
@@ -2726,10 +2724,8 @@ Status MklLayoutRewritePass::MergePadWithConv2D(std::unique_ptr<Graph>* g,
   TF_CHECK_OK(GetNodeAttr(succ->def(), "padding", &padding));
   TF_CHECK_OK(GetNodeAttr(succ->def(), "strides", &strides));
   TF_CHECK_OK(GetNodeAttr(succ->def(), "dilations", &dilations));
-  // Data format for pad is not available and not necessary, thus
-  // dont need to match data format for Pad
-  // Check if the data types and devices of both succ and pred are the same.
-  // Assert is not used,  because it can be too strict.
+  // Check if the devices of both succ and pred are the same.
+  // Assert is not used because it can be too strict.
   // Don't need to check for data formats because it is not available in Pad.
   if (T_pred != T_succ ||
       pred->assigned_device_name() != succ->assigned_device_name() ||
@@ -2782,41 +2778,32 @@ Status MklLayoutRewritePass::MergePadWithConv2D(std::unique_ptr<Graph>* g,
     }
   }
 
-  if (is_fused_conv2d) {
-    DCHECK_EQ(ConvDataInputEdges, 3);
-  } else {
-    DCHECK_EQ(ConvDataInputEdges, 2);
-  }
+  DCHECK_EQ(ConvDataInputEdges, is_fused_conv2d ? 3 : 2);
 
   // We will use the node name of Conv2D as the name of new node
   // Build new node. We use same name as original node, but change the op
   // name.
 
-  auto fused_op_name = csinfo_.pad_with_conv2d;
-  if (is_fused_conv2d) {
-    fused_op_name = csinfo_.pad_with_fused_conv2d;
-  }
-  NodeBuilder nb(succ->name(), fused_op_name);
+  NodeBuilder nb(succ->name(), is_fused_conv2d ? csinfo_.pad_with_fused_conv2d
+                                               : csinfo_.pad_with_conv2d);
   nb.Input(pred_in[0].first, pred_in[0].second);  // In1 (input data)  of Pad
   // pred_in[1] will be 2nd Tensorflow tensor for Conv2D.
   nb.Input(succ_in[1].first, succ_in[1].second);  // In2 (filter) of conv2d
   // In1 of Conv2D is same as output of Pad.
   // Thus, only need to add In2 of Conv2D
 
-  // FusedConv2D has one additional input, args
   if (is_fused_conv2d) {
+    // FusedConv2D has one additional input, args
     std::vector<NodeBuilder::NodeOut> args;
     args.emplace_back(succ_in[2].first, succ_in[2].second);
     nb.Input(gtl::ArraySlice<NodeBuilder::NodeOut>{
-        args});  // In3 (args) of FusedConv2D
-  }
-  nb.Input(pred_in[1].first, pred_in[1].second);  // In2 (paddings) of Pad
-
-  if (is_fused_conv2d) {
+        args});                                     // In3 (args) of FusedConv2D
+    nb.Input(pred_in[1].first, pred_in[1].second);  // In2 (paddings) of Pad
     // Copy attributes from Pad and FusedConv2D to PadWithFusedConv2D.
     CopyAttrsFromPadAndFusedConv2D(const_cast<const Node*>(succ),
                                    const_cast<const Node*>(pred), &nb);
   } else {
+    nb.Input(pred_in[1].first, pred_in[1].second);  // In2 (paddings) of Pad
     // Copy attributes from Pad and conv2D to PadWithConv2D.
     CopyAttrsFromPadAndConv2D(const_cast<const Node*>(succ),
                               const_cast<const Node*>(pred), &nb);
@@ -3018,17 +3005,12 @@ Status MklLayoutRewritePass::MergeNode(std::unique_ptr<Graph>* g, Node* m,
         m->type_string() == csinfo_.conv2d))) {
     return this->MergeConv2DWithBiasAdd(g, m, n);
   }
-  if (((m->type_string() == csinfo_.pad &&
-        n->type_string() == csinfo_.conv2d)) ||
-      ((n->type_string() == csinfo_.pad &&
-        m->type_string() == csinfo_.conv2d))) {
-    return this->MergePadWithConv2D(g, m, n);
-  }
-
-  if (((m->type_string() == csinfo_.pad &&
-        n->type_string() == csinfo_.fused_conv2d && FusedConv2DRewrite(n))) ||
-      ((n->type_string() == csinfo_.pad &&
-        m->type_string() == csinfo_.fused_conv2d && FusedConv2DRewrite(m)))) {
+  if ((m->type_string() == csinfo_.pad &&
+       (n->type_string() == csinfo_.conv2d ||
+        (n->type_string() == csinfo_.fused_conv2d && FusedConv2DRewrite(n)))) ||
+      (n->type_string() == csinfo_.pad &&
+       (m->type_string() == csinfo_.conv2d ||
+        (m->type_string() == csinfo_.fused_conv2d && FusedConv2DRewrite(m))))) {
     return this->MergePadWithConv2D(g, m, n);
   }
 
diff --git a/tensorflow/core/graph/mkl_layout_pass_test.cc b/tensorflow/core/graph/mkl_layout_pass_test.cc
index 5544e2bc36..3ac25cdf02 100644
--- a/tensorflow/core/graph/mkl_layout_pass_test.cc
+++ b/tensorflow/core/graph/mkl_layout_pass_test.cc
@@ -1245,9 +1245,9 @@ TEST_F(MklLayoutPassTest, NodeRewrite_FusedConv2D_Negative2) {
 
 // Merge test for PadWithFusedConv2D Op with BiasAdd fusion
 // padding is VALID type
-// A = input(image), B = input(paddings), C= Pad = input of conv2D,
-// D=input(filter), E = input(bias), F = _FusedConv2D
-// G = Zeta C=Pad(A,B); F=_FusedConv2D(C,D,E); G=Zeta(F,E)
+// A = input(image), B = input(paddings), C = Pad(A, B) = input of conv2D,
+// D = input(filter), E = input(bias), F = _FusedConv2D(C, D, E)
+// G = Zeta(F, E)
 // After layout pass
 // _MklPadWithFusedConv2D(A, D, E, B, DMT/_0, DMT/_1, DMT/_2, DMT/_3)
 TEST_F(MklLayoutPassTest, NodeMerge_PadWithFusedConv2D_Positive1) {
@@ -1283,9 +1283,9 @@ TEST_F(MklLayoutPassTest, NodeMerge_PadWithFusedConv2D_Positive1) {
 
 // Merge test for PadWithFusedConv2D Op with BiasAdd+Relu fusion
 // padding is VALID type
-// A = input(image), B = input(paddings), C= Pad = input of conv2D,
-// D=input(filter), E = input(bias), F = _FusedConv2D(With relu)
-// G = Zeta C=Pad(A,B); F=_FusedConv2D(C,D,E); G=Zeta(F,E)
+// A = input(image), B = input(paddings), C = Pad(A, B) = input of conv2D,
+// D = input(filter), E = input(bias), F = _FusedConv2D(C, D, E) (With relu)
+// G = Zeta(F, E)
 // After layout pass
 // _MklPadWithFusedConv2D(A, D, E, B, DMT/_0, DMT/_1, DMT/_2, DMT/_3)
 TEST_F(MklLayoutPassTest, NodeMerge_PadWithFusedConv2D_Positive2) {
@@ -1322,9 +1322,9 @@ TEST_F(MklLayoutPassTest, NodeMerge_PadWithFusedConv2D_Positive2) {
 
 // Merge test for PadWithFusedConv2D Op with unsupported fusion
 // padding is VALID type
-// A = input(image), B = input(paddings), C= Pad = input of conv2D,
-// D=input(filter), E = input(bias), F = _FusedConv2D(With Unsupported)
-// G = Zeta C=Pad(A,B); F=_FusedConv2D(C,D,E); G=Zeta(F,E)
+// A = input(image), B = input(paddings), C = Pad(A, B) = input of conv2D,
+// D = input(filter), E = input(bias),
+// F = _FusedConv2D(C, D, E) (With Unsupported), G = Zeta(F, E)
 // After layout pass - No merging
 TEST_F(MklLayoutPassTest, NodeMerge_PadWithFusedConv2D_Negative1) {
   InitGraph(
@@ -1355,9 +1355,9 @@ TEST_F(MklLayoutPassTest, NodeMerge_PadWithFusedConv2D_Negative1) {
 
 // Merge test for PadWithFusedConv2D Op with BiasAdd fusion
 // padding is SAME type
-// A = input(image), B = input(paddings), C= Pad = input of conv2D,
-// D=input(filter), E = input(bias), F = _FusedConv2D
-// G = Zeta C=Pad(A,B); F=_FusedConv2D(C,D,E); G=Zeta(F,E)
+// A = input(image), B = input(paddings), C = Pad(A, B) = input of conv2D,
+// D = input(filter), E = input(bias), F = _FusedConv2D(C,D,E)
+// G = Zeta(F,E)
 // After layout pass - No merging
 TEST_F(MklLayoutPassTest, NodeMerge_PadWithFusedConv2D_Negative2) {
   InitGraph(
@@ -1391,9 +1391,9 @@ TEST_F(MklLayoutPassTest, NodeMerge_PadWithFusedConv2D_Negative2) {
 
 // Merge test for PadWithFusedConv2D Op with BiasAdd+Relu fusion
 // padding is SAME type
-// A = input(image), B = input(paddings), C= Pad = input of conv2D,
-// D=input(filter), E = input(bias), F = _FusedConv2D(With relu)
-// G = Zeta C=Pad(A,B); F=_FusedConv2D(C,D,E); G=Zeta(F,E)
+// A = input(image), B = input(paddings), C = Pad(A, B) = input of conv2D,
+// D = input(filter), E = input(bias), F = _FusedConv2D(C,D,E)(With relu)
+// G = Zeta(F,E)
 // After layout pass - No merging
 TEST_F(MklLayoutPassTest, NodeMerge_PadWithFusedConv2D_Negative3) {
   InitGraph(
@@ -1426,24 +1426,25 @@ TEST_F(MklLayoutPassTest, NodeMerge_PadWithFusedConv2D_Negative3) {
             "_2->F:5;E->F:2;E->G:1;F->G");
 }
 
-// Test if input control edges do not duplicate after merge.
-// If both the merging ops have input control edge from a common op
+// Tests that there are no duplicate input control edges after merge.
+// If both the merging ops have input control edges from a common op
 // then, the merged op will have only one control edge from that
-// common op.
+// common op. This test only add additional input control edge check
+// based on the previous test NodeMerge_PadWithFusedConv2D_Positive1
 // padding is VALID type
-// A = input(image), A1 = input, B = input(paddings),
-// C= Pad = input of conv2D,
-// D=input(filter), F=input(bias), E = _FusedConv2D, Z = Zeta
-// C=Pad(A,B); E=_FusedConv2D(C,D,F); Z=Zeta(E,Y)
-// A1:control->C:control
-// A1:control->E:control
+// A = input(image), X = input, B = input(paddings),
+// C = Pad(A, B) = input of conv2D,
+// D = input(filter), E = input(bias), F = _FusedConv2D(C, D, E)
+// G = Zeta(F, E)
+// X:control->C:control
+// X:control->F:control
 // After layout pass:
 // _MklPadWithFusedConv2D(A, D, B, F, DMT/_0, DMT/_1, DMT/_2, DMT/_3)
-// A1:control->E:control (only one control edge)
+// X:control->E:control (only one control edge)
 TEST_F(MklLayoutPassTest, Input_ControlEdge_PadWithFusedConv2D_Positive) {
   DCHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
   InitGraph(
-      "node { name: 'A1' op: 'Input'}"
+      "node { name: 'X' op: 'Input'}"
       "node { name: 'A' op: 'Input'}"
       "node { name: 'B' op: 'Int32Input'}"
       "node { name: 'C' op: 'Pad'"
@@ -1451,8 +1452,8 @@ TEST_F(MklLayoutPassTest, Input_ControlEdge_PadWithFusedConv2D_Positive) {
       " attr { key: 'Tpaddings'        value { type: DT_INT32 } }"
       " input: ['A', 'B']}"
       "node { name: 'D' op: 'Input'}"
-      "node { name: 'F' op: 'Input'}"
-      "node { name: 'E' op: '_FusedConv2D'"
+      "node { name: 'E' op: 'Input'}"
+      "node { name: 'F' op: '_FusedConv2D'"
       " attr { key: 'T'                value { type: DT_FLOAT } }"
       " attr { key: 'num_args'         value { i: 1 } }"
       " attr { key: 'data_format'      value { s: 'NCHW' } }"
@@ -1461,45 +1462,45 @@ TEST_F(MklLayoutPassTest, Input_ControlEdge_PadWithFusedConv2D_Positive) {
       " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
       " attr { key: 'fused_ops'        value { list: {s: 'BiasAdd'} } }"
       " attr { key: 'epsilon'          value { f: 0.001 }}"
-      " input: ['C', 'D', 'F']}"
-      "node { name: 'Y' op: 'Input'}"
-      "node { name: 'Z' op: 'Zeta'"
+      " input: ['C', 'D', 'E']}"
+      "node { name: 'G' op: 'Zeta'"
       " attr {key: 'T'                 value { type: DT_FLOAT } }"
-      " input: ['E', 'Y']}");
-  Node* a1 = FindNode("A1");
+      " input: ['F', 'E']}");
+  Node* x = FindNode("X");
   Node* c = FindNode("C");
-  Node* e = FindNode("E");
-  const Edge* edge = graph_.AddControlEdge(a1, c);
-  const Edge* edge_1 = graph_.AddControlEdge(a1, e);
+  Node* f = FindNode("F");
+  const Edge* edge = graph_.AddControlEdge(x, c);
+  const Edge* edge_1 = graph_.AddControlEdge(x, f);
   ASSERT_NE(edge, nullptr);
   ASSERT_NE(edge_1, nullptr);
   EXPECT_EQ(DoMklLayoutOptimizationPass(),
-            "A(Input);A1(Input);B(Int32Input);D(Input);DMT/_0(Const);DMT/"
-            "_1(Const);DMT/_2(Const);DMT/"
-            "_3(Const);E(_MklPadWithFusedConv2D);F(Input);Y(Input);Z(Zeta)|A->"
-            "E;A1:control->E:control;A:control->DMT/_0:control;A:control->DMT/"
-            "_1:control;A:control->DMT/_2:control;A:control->DMT/"
-            "_3:control;B->E:3;D->E:1;DMT/_0->E:4;DMT/_1->E:5;DMT/_2->E:6;DMT/"
-            "_3->E:7;E->Z;F->E:2;Y->Z:1");
+            "A(Input);B(Int32Input);D(Input);DMT/_0(Const);DMT/_1(Const);"
+            "DMT/_2(Const);DMT/_3(Const);E(Input);F(_MklPadWithFusedConv2D);"
+            "G(Zeta);X(Input)|A->F;A:control->DMT/_0:control;"
+            "A:control->DMT/_1:control;A:control->DMT/_2:control;"
+            "A:control->DMT/_3:control;B->F:3;D->F:1;DMT/_0->F:4;"
+            "DMT/_1->F:5;DMT/_2->F:6;DMT/_3->F:7;E->F:2;E->G:1;F->G;"
+            "X:control->F:control");
 }
 
-// Test if output control edges does not duplicate after merge.
+// ts that there are no duplicate output control edges after merge.
 // If both the merging ops have output control edge to a common op,
 // then after merge, the merged op will have only one control edge
-// to that commom op.
+// to that commom op. This test only add additional output control edge check
+// based on the previous test NodeMerge_PadWithFusedConv2D_Positive1
 // padding is VALID type
-// A = input(image), B = input(paddings), C= Pad = input of conv2D,
-// D=input(filter), F=input(bias), E = _FusedConv2D, Z = Zeta
-// C=Pad(A,B); E=_FusedConv2D(C,D,F); Z=Zeta(E,Y)
-// C:control->A1:control
-// E:control->A1:control
+// A = input(image), B = input(paddings), C = Pad(A, B) = input of conv2D,
+// D = input(filter), E = input(bias), F = _FusedConv2D(C, D, E)
+// G = Zeta(F, E), X = input
+// C:control->X:control
+// F:control->X:control
 // After layout pass:
 // _MklPadWithFusedConv2D(A, D, B, F, DMT/_0, DMT/_1, DMT/_2, DMT/_2)
-// E:control->A1:control (only one control edge)
+// F:control->X:control (only one control edge)
 TEST_F(MklLayoutPassTest, Output_ControlEdge_PadWithFusedConv2D_Positive) {
   DCHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
   InitGraph(
-      "node { name: 'A1' op: 'Input'}"
+      "node { name: 'X' op: 'Input'}"
       "node { name: 'A' op: 'Input'}"
       "node { name: 'B' op: 'Int32Input'}"
       "node { name: 'C' op: 'Pad'"
@@ -1507,8 +1508,8 @@ TEST_F(MklLayoutPassTest, Output_ControlEdge_PadWithFusedConv2D_Positive) {
       " attr { key: 'Tpaddings'        value { type: DT_INT32 } }"
       " input: ['A', 'B']}"
       "node { name: 'D' op: 'Input'}"
-      "node { name: 'F' op: 'Input'}"
-      "node { name: 'E' op: '_FusedConv2D'"
+      "node { name: 'E' op: 'Input'}"
+      "node { name: 'F' op: '_FusedConv2D'"
       " attr { key: 'T'                value { type: DT_FLOAT } }"
       " attr { key: 'num_args'         value { i: 1 } }"
       " attr { key: 'data_format'      value { s: 'NCHW' } }"
@@ -1517,34 +1518,31 @@ TEST_F(MklLayoutPassTest, Output_ControlEdge_PadWithFusedConv2D_Positive) {
       " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
       " attr { key: 'fused_ops'        value { list: {s: 'BiasAdd'} } }"
       " attr { key: 'epsilon'          value { f: 0.001 }}"
-      " input: ['C', 'D', 'F']}"
-      "node { name: 'Y' op: 'Input'}"
-      "node { name: 'Z' op: 'Zeta'"
+      " input: ['C', 'D', 'E']}"
+      "node { name: 'G' op: 'Zeta'"
       " attr {key: 'T'                 value { type: DT_FLOAT } }"
-      " input: ['E', 'Y']}");
-  Node* a1 = FindNode("A1");
+      " input: ['F', 'E']}");
+  Node* x = FindNode("X");
   Node* c = FindNode("C");
-  Node* e = FindNode("E");
-  const Edge* edge = graph_.AddControlEdge(c, a1);
-  const Edge* edge_1 = graph_.AddControlEdge(e, a1);
+  Node* f = FindNode("F");
+  const Edge* edge = graph_.AddControlEdge(c, x);
+  const Edge* edge_1 = graph_.AddControlEdge(f, x);
   ASSERT_NE(edge, nullptr);
   ASSERT_NE(edge_1, nullptr);
   EXPECT_EQ(DoMklLayoutOptimizationPass(),
-            "A(Input);A1(Input);B(Int32Input);D(Input);DMT/_0(Const);DMT/"
-            "_1(Const);DMT/_2(Const);DMT/"
-            "_3(Const);E(_MklPadWithFusedConv2D);F(Input);Y(Input);Z(Zeta)|A->"
-            "E;A:control->DMT/_0:control;A:control->DMT/"
+            "A(Input);B(Int32Input);D(Input);DMT/_0(Const);DMT/_1(Const);DMT/"
+            "_2(Const);DMT/_3(Const);E(Input);F(_MklPadWithFusedConv2D);"
+            "G(Zeta);X(Input)|A->F;A:control->DMT/_0:control;A:control->DMT/"
             "_1:control;A:control->DMT/_2:control;A:control->DMT/"
-            "_3:control;B->E:3;D->E:1;DMT/_0->E:4;DMT/_1->E:5;DMT/_2->E:6;DMT/"
-            "_3->E:7;E->Z;E:control->A1:control;F->E:2;Y->Z:1");
+            "_3:control;B->F:3;D->F:1;DMT/_0->F:4;DMT/_1->F:5;DMT/_2->F:6;DMT/"
+            "_3->F:7;E->F:2;E->G:1;F->G;F:control->X:control");
 }
 
 // Pad + _FusedConv2D with padding is VALID,
 // Input node pointing to both Pad and _FusedConv2D
-// Output of both Pad and _FusedConv2D feeds one node (Z as Output2)
-// A = input(as image), B = input(as paddings), C= Pad
-// F = input(as bias), E = _FusedConv2D, Z = Output2
-// C=Pad(A,B); E=_FusedConv2D(C,A,F); Z=Output(C,E)
+// Output of both Pad and _FusedConv2D feeds one node (G as Output2)
+// A = input(as image), B = input(as paddings), C = Pad(A, B)
+// E = input(as bias), F = _FusedConv2D(C, A, E), G=Output(C, F)
 // After layout pass - No merging, since Pad and _FusedConv2D both
 // feed to the same node (Z)
 TEST_F(MklLayoutPassTest, NodeMerge_PadWithFusedConv2D_Common_InOutput) {
@@ -1556,8 +1554,8 @@ TEST_F(MklLayoutPassTest, NodeMerge_PadWithFusedConv2D_Common_InOutput) {
       " attr { key: 'T'                value { type: DT_FLOAT } }"
       " attr { key: 'Tpaddings'        value { type: DT_INT32 } }"
       " input: ['A', 'B']}"
-      "node { name: 'F' op: 'Input'}"
-      "node { name: 'E' op: '_FusedConv2D'"
+      "node { name: 'E' op: 'Input'}"
+      "node { name: 'F' op: '_FusedConv2D'"
       " attr { key: 'T'                value { type: DT_FLOAT } }"
       " attr { key: 'num_args'         value { i: 1 } }"
       " attr { key: 'data_format'      value { s: 'NCHW' } }"
@@ -1566,15 +1564,15 @@ TEST_F(MklLayoutPassTest, NodeMerge_PadWithFusedConv2D_Common_InOutput) {
       " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
       " attr { key: 'fused_ops'        value { list: {s: 'BiasAdd'} } }"
       " attr { key: 'epsilon'          value { f: 0.001 }}"
-      " input: ['C', 'A', 'F']}"
-      "node { name: 'Z' op: 'Output2'"
-      " input: ['C', 'E']}");
+      " input: ['C', 'A', 'E']}"
+      "node { name: 'G' op: 'Output2'"
+      " input: ['C', 'F']}");
   EXPECT_EQ(DoMklLayoutOptimizationPass(),
             "A(Input);B(Int32Input);C(Pad);DMT/_0(Const);DMT/_1(Const);DMT/"
-            "_2(Const);E(_MklFusedConv2D);F(Input);Z(Output2)|A->C;A->E:1;B->C:"
-            "1;C->E;C->Z;C:control->DMT/_0:control;C:control->DMT/"
-            "_1:control;C:control->DMT/_2:control;DMT/_0->E:3;DMT/_1->E:4;DMT/"
-            "_2->E:5;E->Z:1;F->E:2");
+            "_2(Const);E(Input);F(_MklFusedConv2D);G(Output2)|A->C;A->F:1;B->C:"
+            "1;C->F;C->G;C:control->DMT/_0:control;C:control->DMT/"
+            "_1:control;C:control->DMT/_2:control;DMT/_0->F:3;DMT/_1->F:4;DMT/"
+            "_2->F:5;E->F:2;F->G:1");
 }
 
 TEST_F(MklLayoutPassTest, NodeRewrite_Conv2DGradFilter_Positive) {
diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc
index ead2f1e09e..f75a0e5768 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_ops.cc
@@ -1126,7 +1126,7 @@ class MklConvOp : public OpKernel {
 
   void PadWithConvFusion(OpKernelContext* context, memory::dims& padding_left,
                          memory::dims& padding_right) {
-    const Tensor& paddings_tf = MklGetInput(context, input_index_pad);
+    const Tensor& paddings_tf = MklGetInput(context, input_index_pad_);
     OP_REQUIRES(context, paddings_tf.dims() == 2,
                 errors::InvalidArgument("paddings must be 2-dimensional: ",
                                         paddings_tf.shape().DebugString()));
@@ -1168,7 +1168,7 @@ class MklConvOp : public OpKernel {
   void set_fuse_pad(bool fuse_pad) {
     fuse_pad_ = fuse_pad;
     // In PadwithFusedConv OP, pad is the fourth index.
-    input_index_pad = 3;
+    input_index_pad_ = 3;
   }
 
   // This method is for the base class MklConvOp, which handles the
@@ -1245,7 +1245,7 @@ class MklConvOp : public OpKernel {
   bool fuse_relu_ = false;
   bool fuse_pad_ = pad_enabled;
 
-  int input_index_pad = 2;
+  int input_index_pad_ = 2;
 
   const int kInputIndex_Src = 0, kInputIndex_Filter = 1, kInputIndex_Bias = 2;
   const int kOutputIndex_Dst = 0, kOutputIndex_Filter = 1;
@@ -1972,6 +1972,8 @@ TF_CALL_float(REGISTER_MKL_CPU_2D);
 
 TF_CALL_float(REGISTER_MKL_CPU_2D_DEPTHWISE);
 
+// Note we are registering _MklFusedConv2D.
+// We check the fused_ops attributes to decide if bias is enabled or not.
 #define REGISTER_MKL_CPU_2D_FUSED(T)                                \
   REGISTER_KERNEL_BUILDER(                                          \
       Name("_MklFusedConv2D")                                       \
@@ -1999,8 +2001,6 @@ TF_CALL_float(REGISTER_MKL_CPU_2D_DEPTHWISE);
                               .TypeConstraint<int32>("Tpaddings")   \
                               .Label(mkl_op_registry::kMklOpLabel), \
                           MklDummyOp<CPUDevice, T>);
-// Note we are registering _MklFusedConv2D.
-// We check the fused_ops attributes to decide if bias is enabled or not.
 
 TF_CALL_float(REGISTER_MKL_CPU_2D_FUSED);
 
diff --git a/tensorflow/core/kernels/mkl_fused_ops_test.cc b/tensorflow/core/kernels/mkl_fused_ops_test.cc
index f1243c5b68..f2527a675d 100644
--- a/tensorflow/core/kernels/mkl_fused_ops_test.cc
+++ b/tensorflow/core/kernels/mkl_fused_ops_test.cc
@@ -38,8 +38,12 @@ namespace tensorflow {
 static const uint8 dummy_tensor[] = {0, 0, 0, 0, 0, 0, 0, 0};
 static const TensorShape dummy_shape({8});
 
+using BiasAddGraphRunner =
+    std::function<void(const Tensor& input_data, const Tensor& filter_data,
+                       const Tensor& bias_data, Tensor* out)>;
+
 template <typename T>
-class ConvMklToTF : public OpsTestBase {
+class CommonTestUtilities : public OpsTestBase {
  public:
   void PerformConversion(DataType dtype, const Tensor& tensor,
                          const Tensor& mkl_meta_tensor, Tensor* output) {
@@ -61,8 +65,8 @@ class ConvMklToTF : public OpsTestBase {
 
   // Runs a Tensorflow graph defined by the root scope, and fetches the result
   // of 'fetch' node into the output Tensor.
-  void RunAndFetch(const tensorflow::Scope& root, const string& fetch,
-                   Tensor* output) {
+  static void RunAndFetch(const tensorflow::Scope& root, const string& fetch,
+                          Tensor* output) {
     tensorflow::GraphDef graph;
     TF_ASSERT_OK(root.ToGraphDef(&graph));
 
@@ -84,6 +88,35 @@ class ConvMklToTF : public OpsTestBase {
     test::ExpectTensorNear<T>(expected, output, 1e-5);
   }
   void TestBody() {}
+
+  static void VerifyBiasAddTensorsClose(int depth, int image_width,
+                                        int image_height, int image_batch_count,
+                                        int filter_size, int filter_count,
+                                        const BiasAddGraphRunner& run_default,
+                                        const BiasAddGraphRunner& run_fused) {
+    DataType dtype = DataTypeToEnum<T>::v();
+
+    Tensor image(dtype, {image_batch_count, image_height, image_width, depth});
+    image.flat<T>() = image.flat<T>().setRandom();
+
+    Tensor filter(dtype, {filter_size, filter_size, depth, filter_count});
+    filter.flat<T>() = filter.flat<T>().setRandom();
+
+    const int bias_size = filter_count;
+    Tensor bias(dtype, {bias_size});
+    bias.flat<T>() = bias.flat<T>().setRandom();
+
+    Tensor conv_2d;
+    Tensor fused_conv_2d;
+
+    run_default(image, filter, bias, &conv_2d);
+    run_fused(image, filter, bias, &fused_conv_2d);
+
+    ASSERT_EQ(conv_2d.dtype(), fused_conv_2d.dtype());
+    ASSERT_EQ(conv_2d.shape(), fused_conv_2d.shape());
+
+    test::ExpectClose(conv_2d, fused_conv_2d);
+  }
 };
 
 // Testing MKL's fused convolution ops
@@ -96,10 +129,6 @@ class MklFusedConv2DOpTest : public OpsTestBase {
   static constexpr int kImageHeight = 32;
   static constexpr int kImageBatchCount = 8;
 
-  using BiasAddGraphRunner =
-      std::function<void(const Tensor& input_data, const Tensor& filter_data,
-                         const Tensor& bias_data, Tensor* out)>;
-
   void RunConv2DWithBias(const Tensor& input_data, const Tensor& filter_data,
                          const Tensor& bias_data, Tensor* output,
                          int stride = 1) {
@@ -115,8 +144,7 @@ class MklFusedConv2DOpTest : public OpsTestBase {
         root.WithOpName("with_bias"), conv,
         ops::Const(root.WithOpName("bias"), Input::Initializer(bias_data)));
 
-    ConvMklToTF<T> conv_comp;
-    conv_comp.RunAndFetch(root, "with_bias", output);
+    CommonTestUtilities<T>::RunAndFetch(root, "with_bias", output);
   }
 
   void RunConv2DWithBiasAndRelu(const Tensor& input_data,
@@ -137,8 +165,7 @@ class MklFusedConv2DOpTest : public OpsTestBase {
 
     auto with_relu = ops::Relu(root.WithOpName("with_relu"), with_bias);
 
-    ConvMklToTF<T> conv_comp;
-    conv_comp.RunAndFetch(root, "with_relu", output);
+    CommonTestUtilities<T>::RunAndFetch(root, "with_relu", output);
   }
 
   void RunMklFusedConv2DOp(const Tensor& image, const Tensor& filter,
@@ -149,6 +176,7 @@ class MklFusedConv2DOpTest : public OpsTestBase {
     int num_args = static_cast<int>(args.size());
 
     TF_EXPECT_OK(NodeDefBuilder("fused_conv_op", "_MklFusedConv2D")
+                     .Attr("T", dtype)
                      .Input(FakeInput(dtype))
                      .Input(FakeInput(dtype))
                      .Attr("num_args", num_args)
@@ -156,7 +184,6 @@ class MklFusedConv2DOpTest : public OpsTestBase {
                      .Input(FakeInput(DT_UINT8))
                      .Input(FakeInput(DT_UINT8))
                      .Input(FakeInput(num_args, DT_UINT8))
-                     .Attr("T", dtype)
                      .Attr("strides", {1, stride, stride, 1})
                      .Attr("padding", "SAME")
                      .Attr("fused_ops", fused_ops)
@@ -180,40 +207,11 @@ class MklFusedConv2DOpTest : public OpsTestBase {
     // Index 2 will need to be changed if the number of outputs produced
     // by MklConv2D change.
     const Tensor& output_meta_tensor = *GetOutput(2);
-    ConvMklToTF<T> conv_comp;
-    conv_comp.PerformConversion(dtype, output_tensor, output_meta_tensor,
+    CommonTestUtilities<T> test_util;
+    test_util.PerformConversion(dtype, output_tensor, output_meta_tensor,
                                 output);
   }
 
-  void VerifyBiasAddTensorsNear(int depth, int image_width, int image_height,
-                                int image_batch_count, int filter_size,
-                                int filter_count,
-                                const BiasAddGraphRunner& run_default,
-                                const BiasAddGraphRunner& run_fused) {
-    DataType dtype = DataTypeToEnum<T>::v();
-
-    Tensor image(dtype, {image_batch_count, image_height, image_width, depth});
-    image.flat<T>() = image.flat<T>().setRandom();
-
-    Tensor filter(dtype, {filter_size, filter_size, depth, filter_count});
-    filter.flat<T>() = filter.flat<T>().setRandom();
-
-    const int bias_size = filter_count;
-    Tensor bias(dtype, {bias_size});
-    bias.flat<T>() = bias.flat<T>().setRandom();
-
-    Tensor conv_2d;
-    Tensor fused_conv_2d;
-
-    run_default(image, filter, bias, &conv_2d);
-    run_fused(image, filter, bias, &fused_conv_2d);
-
-    ASSERT_EQ(conv_2d.dtype(), fused_conv_2d.dtype());
-    ASSERT_EQ(conv_2d.shape(), fused_conv_2d.shape());
-
-    test::ExpectClose(conv_2d, fused_conv_2d);
-  }
-
   // Verifies that computing Conv2D+BiasAdd in a graph is identical to
   // FusedConv2D.
   void VerifyConv2DWithBias(int filter_size, int filter_count,
@@ -233,9 +231,9 @@ class MklFusedConv2DOpTest : public OpsTestBase {
                           out);
     };
 
-    VerifyBiasAddTensorsNear(depth, image_width, image_height,
-                             image_batch_count, filter_size, filter_count,
-                             run_default, run_fused);
+    CommonTestUtilities<T>::VerifyBiasAddTensorsClose(
+        depth, image_width, image_height, image_batch_count, filter_size,
+        filter_count, run_default, run_fused);
   }
 
   // Verifies that computing Conv2D+BiasAdd+Relu in a graph is identical to
@@ -258,9 +256,9 @@ class MklFusedConv2DOpTest : public OpsTestBase {
                           {"BiasAdd", "Relu"}, out);
     };
 
-    VerifyBiasAddTensorsNear(depth, image_width, image_height,
-                             image_batch_count, filter_size, filter_count,
-                             run_default, run_fused);
+    CommonTestUtilities<T>::VerifyBiasAddTensorsClose(
+        depth, image_width, image_height, image_batch_count, filter_size,
+        filter_count, run_default, run_fused);
   }
 };
 
@@ -343,8 +341,8 @@ class FusedPadConvOpTest : public OpsTestBase {
     // Compare output to expected results
     const Tensor& first = *GetOutput(0);
     const Tensor& second = *GetOutput(2);
-    ConvMklToTF<T> conv_comp;
-    conv_comp.ConvertAndCompare(dtype, first, second, expected);
+    CommonTestUtilities<T> test_util;
+    test_util.ConvertAndCompare(dtype, first, second, expected);
   }
 };
 
@@ -405,7 +403,6 @@ TEST_F(FusedPadConvOpTest, PaddingConvTestNchw) {
 }
 
 // Testing fusion of pad and fusedconv2d
-
 template <typename T>
 class MklPadWithFusedConv2DOpTest : public OpsTestBase {
  protected:
@@ -414,45 +411,12 @@ class MklPadWithFusedConv2DOpTest : public OpsTestBase {
   static constexpr int kImageHeight = 28;
   static constexpr int kImageBatchCount = 8;
 
-  // 0: top pad, 1 bottom pad, 2 left pad, 3 right pad
-  int padding_list[4];
-
-  using BiasAddGraphRunner =
-      std::function<void(const Tensor& input_data, const Tensor& filter_data,
-                         const Tensor& bias_data, Tensor* out)>;
-
-  void VerifyBiasAddTensorsNear(int depth, int image_width, int image_height,
-                                int image_batch_count, int filter_size,
-                                int filter_count,
-                                const BiasAddGraphRunner& run_default,
-                                const BiasAddGraphRunner& run_fused) {
-    DataType dtype = DataTypeToEnum<T>::v();
-
-    Tensor image(dtype, {image_batch_count, image_height, image_width, depth});
-    image.flat<T>() = image.flat<T>().setRandom();
-
-    Tensor filter(dtype, {filter_size, filter_size, depth, filter_count});
-    filter.flat<T>() = filter.flat<T>().setRandom();
-
-    const int bias_size = filter_count;
-    Tensor bias(dtype, {bias_size});
-    bias.flat<T>() = bias.flat<T>().setRandom();
-
-    Tensor conv_2d;
-    Tensor fused_conv_2d;
-
-    run_default(image, filter, bias, &conv_2d);
-    run_fused(image, filter, bias, &fused_conv_2d);
-
-    ASSERT_EQ(conv_2d.dtype(), fused_conv_2d.dtype());
-    ASSERT_EQ(conv_2d.shape(), fused_conv_2d.shape());
-
-    test::ExpectClose(conv_2d, fused_conv_2d);
-  }
+  // 0: top pad, 1: bottom pad, 2: left pad, 3: right pad
+  int padding_list_[4];
 
   // Verifies that computing Pad+Conv2D+BiasAdd in a graph is identical to
   // FusedConv2D.
-  void VerifyConv2DWithBiasAndPad(int filter_size, int filter_count,
+  void VerifyPadAndConv2DWithBias(int filter_size, int filter_count,
                                   int depth = kDepth,
                                   int image_width = kImageWidth,
                                   int image_height = kImageHeight,
@@ -460,7 +424,7 @@ class MklPadWithFusedConv2DOpTest : public OpsTestBase {
     const BiasAddGraphRunner run_default = [this](
         const Tensor& input_data, const Tensor& filter_data,
         const Tensor& bias_data, Tensor* out) {
-      RunMklFusedConv2DWithPadAndBias(input_data, filter_data, bias_data, out);
+      RunMklPadWithFusedConv2DAndBias(input_data, filter_data, bias_data, out);
     };
 
     const BiasAddGraphRunner run_fused = [this](
@@ -470,21 +434,21 @@ class MklPadWithFusedConv2DOpTest : public OpsTestBase {
                                  {"BiasAdd"}, out);
     };
 
-    VerifyBiasAddTensorsNear(depth, image_width, image_height,
-                             image_batch_count, filter_size, filter_count,
-                             run_default, run_fused);
+    CommonTestUtilities<T>::VerifyBiasAddTensorsClose(
+        depth, image_width, image_height, image_batch_count, filter_size,
+        filter_count, run_default, run_fused);
   }
 
   // Verifies that computing Pad+Conv2D+BiasAdd+Relu in a graph is identical to
   // FusedConv2D.
-  void VerifyConv2DWithBiasReluAndPad(
+  void VerifyPadAndConv2DWithBiasRelu(
       int filter_size, int filter_count, int depth = kDepth,
       int image_width = kImageWidth, int image_height = kImageHeight,
       int image_batch_count = kImageBatchCount) {
     const BiasAddGraphRunner run_default = [this](
         const Tensor& input_data, const Tensor& filter_data,
         const Tensor& bias_data, Tensor* out) {
-      RunMklFusedConv2DWithPadAndBiasRelu(input_data, filter_data, bias_data,
+      RunMklPadWithFusedConv2DAndBiasRelu(input_data, filter_data, bias_data,
                                           out);
     };
 
@@ -495,21 +459,21 @@ class MklPadWithFusedConv2DOpTest : public OpsTestBase {
                                  {"BiasAdd", "Relu"}, out);
     };
 
-    VerifyBiasAddTensorsNear(depth, image_width, image_height,
-                             image_batch_count, filter_size, filter_count,
-                             run_default, run_fused);
+    CommonTestUtilities<T>::VerifyBiasAddTensorsClose(
+        depth, image_width, image_height, image_batch_count, filter_size,
+        filter_count, run_default, run_fused);
   }
 
-  void RunMklFusedConv2DWithPadAndBias(const Tensor& input_data,
+  void RunMklPadWithFusedConv2DAndBias(const Tensor& input_data,
                                        const Tensor& filter_data,
                                        const Tensor& bias_data, Tensor* output,
                                        int stride = 1) {
     auto root = tensorflow::Scope::NewRootScope();
 
-    // As FusedConv2D only support NHWC format, so here use NHWC format.
+    // FusedConv2D only supports NHWC format so we use NHWC here.
     auto padding = ops::Const(root.WithOpName("padding"),
-                              {0, 0, padding_list[0], padding_list[1],
-                               padding_list[2], padding_list[3], 0, 0},
+                              {0, 0, padding_list_[0], padding_list_[1],
+                               padding_list_[2], padding_list_[3], 0, 0},
                               {4, 2});
     auto pad = ops::Pad(
         root.WithOpName("pad"),
@@ -525,20 +489,19 @@ class MklPadWithFusedConv2DOpTest : public OpsTestBase {
         root.WithOpName("with_bias"), conv,
         ops::Const(root.WithOpName("bias"), Input::Initializer(bias_data)));
 
-    ConvMklToTF<T> conv_comp;
-    conv_comp.RunAndFetch(root, "with_bias", output);
+    CommonTestUtilities<T>::RunAndFetch(root, "with_bias", output);
   }
 
-  void RunMklFusedConv2DWithPadAndBiasRelu(const Tensor& input_data,
+  void RunMklPadWithFusedConv2DAndBiasRelu(const Tensor& input_data,
                                            const Tensor& filter_data,
                                            const Tensor& bias_data,
                                            Tensor* output, int stride = 1) {
     auto root = tensorflow::Scope::NewRootScope();
 
-    // As FusedConv2D only support NHWC format, so here use NHWC format.
+    // FusedConv2D only supports NHWC format so we use NHWC here.
     auto padding = ops::Const(root.WithOpName("padding"),
-                              {0, 0, padding_list[0], padding_list[1],
-                               padding_list[2], padding_list[3], 0, 0},
+                              {0, 0, padding_list_[0], padding_list_[1],
+                               padding_list_[2], padding_list_[3], 0, 0},
                               {4, 2});
     auto pad = ops::Pad(
         root.WithOpName("pad"),
@@ -556,8 +519,7 @@ class MklPadWithFusedConv2DOpTest : public OpsTestBase {
 
     auto with_relu = ops::Relu(root.WithOpName("with_relu"), with_bias);
 
-    ConvMklToTF<T> conv_comp;
-    conv_comp.RunAndFetch(root, "with_relu", output);
+    CommonTestUtilities<T>::RunAndFetch(root, "with_relu", output);
   }
 
   void RunMklFusedConv2DWithPadOp(const Tensor& image, const Tensor& filter,
@@ -567,10 +529,12 @@ class MklPadWithFusedConv2DOpTest : public OpsTestBase {
     DataType dtype = DataTypeToEnum<T>::v();
     const int num_args = static_cast<int>(args.size());
     Tensor padding(DT_INT32, {4, 2});
-    test::FillValues<int32>(&padding, {0, 0, padding_list[0], padding_list[1],
-                                       padding_list[2], padding_list[3], 0, 0});
+    test::FillValues<int32>(
+        &padding, {0, 0, padding_list_[0], padding_list_[1], padding_list_[2],
+                   padding_list_[3], 0, 0});
 
     TF_EXPECT_OK(NodeDefBuilder("pad_fused_conv_op", "_MklPadWithFusedConv2D")
+                     .Attr("T", dtype)
                      .Input(FakeInput(dtype))
                      .Input(FakeInput(dtype))
                      .Attr("num_args", num_args)
@@ -580,7 +544,6 @@ class MklPadWithFusedConv2DOpTest : public OpsTestBase {
                      .Input(FakeInput(DT_UINT8))
                      .Input(FakeInput(num_args, DT_UINT8))
                      .Input(FakeInput(DT_UINT8))
-                     .Attr("T", dtype)
                      .Attr("strides", {1, stride, stride, 1})
                      .Attr("padding", "VALID")
                      .Attr("fused_ops", fused_ops)
@@ -594,10 +557,8 @@ class MklPadWithFusedConv2DOpTest : public OpsTestBase {
     for (const Tensor& arg : args)
       AddInputFromArray<T>(arg.shape(), arg.flat<T>());
     AddInputFromArray<int32>(padding.shape(), padding.flat<int32>());
-    AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
-    AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
-    AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
-    for (const Tensor& arg : args)
+    // Add MKL meta input for input, filter, pad and agrs.
+    for (int i = 0; i < args.size() + 3; i++)
       AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
     TF_ASSERT_OK(RunOpKernel());
 
@@ -606,17 +567,17 @@ class MklPadWithFusedConv2DOpTest : public OpsTestBase {
     // Index 2 will need to be changed if the number of outputs produced
     // by MklConv2D change.
     const Tensor& output_meta_tensor = *GetOutput(2);
-    ConvMklToTF<T> conv_comp;
-    conv_comp.PerformConversion(dtype, output_tensor, output_meta_tensor,
+    CommonTestUtilities<T> test_util;
+    test_util.PerformConversion(dtype, output_tensor, output_meta_tensor,
                                 output);
   }
 
  public:
   void SetPaddingList(int top, int bottom, int left, int right) {
-    padding_list[0] = top;
-    padding_list[1] = bottom;
-    padding_list[2] = left;
-    padding_list[3] = right;
+    padding_list_[0] = top;
+    padding_list_[1] = bottom;
+    padding_list_[2] = left;
+    padding_list_[3] = right;
   }
 };
 
@@ -626,28 +587,28 @@ TYPED_TEST_P(MklPadWithFusedConv2DOpTest, WithBiasAndRoundPad) {
   const int filter_size = 1;
   const int filter_count = 12;
   this->SetPaddingList(2, 2, 1, 1);
-  this->VerifyConv2DWithBiasAndPad(filter_size, filter_count);
+  this->VerifyPadAndConv2DWithBias(filter_size, filter_count);
 }
 
 TYPED_TEST_P(MklPadWithFusedConv2DOpTest, WithBiasAndPartialPad) {
   const int filter_size = 1;
   const int filter_count = 12;
   this->SetPaddingList(4, 0, 2, 0);
-  this->VerifyConv2DWithBiasAndPad(filter_size, filter_count);
+  this->VerifyPadAndConv2DWithBias(filter_size, filter_count);
 }
 
 TYPED_TEST_P(MklPadWithFusedConv2DOpTest, WithBiasReluAndRoundPad) {
   const int filter_size = 1;
   const int filter_count = 12;
   this->SetPaddingList(2, 2, 1, 1);
-  this->VerifyConv2DWithBiasReluAndPad(filter_size, filter_count);
+  this->VerifyPadAndConv2DWithBiasRelu(filter_size, filter_count);
 }
 
 TYPED_TEST_P(MklPadWithFusedConv2DOpTest, WithBiasReluAndPartialPad) {
   const int filter_size = 1;
   const int filter_count = 12;
   this->SetPaddingList(4, 0, 2, 0);
-  this->VerifyConv2DWithBiasReluAndPad(filter_size, filter_count);
+  this->VerifyPadAndConv2DWithBiasRelu(filter_size, filter_count);
 }
 
 REGISTER_TYPED_TEST_CASE_P(MklPadWithFusedConv2DOpTest,  //
-- 
GitLab


From 94c604bdd72717b0c18a1648927529b27aeea320 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 00:15:37 -0800
Subject: [PATCH 0322/2345] Documentation fix for the unsorted_segment_max op
 RELNOTES: n/a

PiperOrigin-RevId: 228290885
---
 .../core/api_def/base_api/api_def_UnsortedSegmentMax.pbtxt     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentMax.pbtxt b/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentMax.pbtxt
index 7a60e4387a..ed4a2bd558 100644
--- a/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentMax.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentMax.pbtxt
@@ -3,7 +3,8 @@ op {
   in_arg {
     name: "segment_ids"
     description: <<END
-A tensor whose shape is a prefix of `data.shape`.END
+A tensor whose shape is a prefix of `data.shape`.
+END
   }
   out_arg {
     name: "output"
-- 
GitLab


From b504d86ca21f9ce57700d017ab8f6d411ca5afde Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 01:03:21 -0800
Subject: [PATCH 0323/2345] compat: Update forward compatibility horizon to
 2019-01-08

PiperOrigin-RevId: 228294335
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index dbad494e40..5008741d55 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 7)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 8)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 84cf1da2b9552f0af564519e8b200ac8859a17b4 Mon Sep 17 00:00:00 2001
From: James Keeling <jtkeeling@google.com>
Date: Tue, 8 Jan 2019 01:34:31 -0800
Subject: [PATCH 0324/2345] Split tile functor GPU code into multiple files

This file was a bottleneck during compilation, often taking many minutes to compile. In local testing this change reduces the wall-clock build time for the tile functor GPU kernels from 217s to 71s.

PiperOrigin-RevId: 228296771
---
 tensorflow/core/kernels/BUILD                 | 11 ++++++-
 ...e_functor_gpu.cu.cc => tile_functor_gpu.h} | 28 +++--------------
 .../core/kernels/tile_functor_gpu_bool.cu.cc  | 31 +++++++++++++++++++
 .../kernels/tile_functor_gpu_complex128.cu.cc | 31 +++++++++++++++++++
 .../kernels/tile_functor_gpu_complex64.cu.cc  | 31 +++++++++++++++++++
 .../kernels/tile_functor_gpu_double.cu.cc     | 31 +++++++++++++++++++
 .../core/kernels/tile_functor_gpu_float.cu.cc | 31 +++++++++++++++++++
 .../core/kernels/tile_functor_gpu_half.cu.cc  | 31 +++++++++++++++++++
 .../core/kernels/tile_functor_gpu_int16.cu.cc | 31 +++++++++++++++++++
 .../core/kernels/tile_functor_gpu_int32.cu.cc | 31 +++++++++++++++++++
 .../core/kernels/tile_functor_gpu_int64.cu.cc | 31 +++++++++++++++++++
 11 files changed, 294 insertions(+), 24 deletions(-)
 rename tensorflow/core/kernels/{tile_functor_gpu.cu.cc => tile_functor_gpu.h} (85%)
 create mode 100644 tensorflow/core/kernels/tile_functor_gpu_bool.cu.cc
 create mode 100644 tensorflow/core/kernels/tile_functor_gpu_complex128.cu.cc
 create mode 100644 tensorflow/core/kernels/tile_functor_gpu_complex64.cu.cc
 create mode 100644 tensorflow/core/kernels/tile_functor_gpu_double.cu.cc
 create mode 100644 tensorflow/core/kernels/tile_functor_gpu_float.cu.cc
 create mode 100644 tensorflow/core/kernels/tile_functor_gpu_half.cu.cc
 create mode 100644 tensorflow/core/kernels/tile_functor_gpu_int16.cu.cc
 create mode 100644 tensorflow/core/kernels/tile_functor_gpu_int32.cu.cc
 create mode 100644 tensorflow/core/kernels/tile_functor_gpu_int64.cu.cc

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 701bda7ef6..fee30fa6c3 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -1012,7 +1012,16 @@ tf_kernel_library(
     hdrs = ["tile_functor.h"],
     gpu_srcs = [
         "tile_functor.h",
-        "tile_functor_gpu.cu.cc",
+        "tile_functor_gpu.h",
+        "tile_functor_gpu_bool.cu.cc",
+        "tile_functor_gpu_complex64.cu.cc",
+        "tile_functor_gpu_complex128.cu.cc",
+        "tile_functor_gpu_double.cu.cc",
+        "tile_functor_gpu_float.cu.cc",
+        "tile_functor_gpu_half.cu.cc",
+        "tile_functor_gpu_int16.cu.cc",
+        "tile_functor_gpu_int32.cu.cc",
+        "tile_functor_gpu_int64.cu.cc",
     ],
     prefix = "tile_ops",
     deps = ARRAY_DEPS,
diff --git a/tensorflow/core/kernels/tile_functor_gpu.cu.cc b/tensorflow/core/kernels/tile_functor_gpu.h
similarity index 85%
rename from tensorflow/core/kernels/tile_functor_gpu.cu.cc
rename to tensorflow/core/kernels/tile_functor_gpu.h
index 84a5060fc3..0de32e730e 100644
--- a/tensorflow/core/kernels/tile_functor_gpu.cu.cc
+++ b/tensorflow/core/kernels/tile_functor_gpu.h
@@ -13,6 +13,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#ifndef TENSORFLOW_CORE_KERNELS_TILE_FUNCTOR_GPU_H_
+#define TENSORFLOW_CORE_KERNELS_TILE_FUNCTOR_GPU_H_
+
 #if GOOGLE_CUDA
 
 #define EIGEN_USE_GPU
@@ -80,28 +83,7 @@ void TileSimple(const Device& d, Tensor* out, const Tensor& in) {
 }
 
 }  // end namespace internal
-
-namespace functor {
-
-typedef Eigen::GpuDevice GPUDevice;
-
-// Register functors used for Tile functor.
-#define DEFINE_TYPE(T)                       \
-  template struct Tile<GPUDevice, T, int32>; \
-  template struct Tile<GPUDevice, T, int64>;
-
-TF_CALL_bool(DEFINE_TYPE);
-TF_CALL_int16(DEFINE_TYPE);
-TF_CALL_int32(DEFINE_TYPE);
-TF_CALL_int64(DEFINE_TYPE);
-TF_CALL_float(DEFINE_TYPE);
-TF_CALL_double(DEFINE_TYPE);
-TF_CALL_half(DEFINE_TYPE);
-TF_CALL_complex64(DEFINE_TYPE);
-TF_CALL_complex128(DEFINE_TYPE);
-
-#undef DEFINE_TYPE
-
-}  // end namespace functor
 }  // namespace tensorflow
 #endif  // GOOGLE_CUDA
+
+#endif  // TENSORFLOW_CORE_KERNELS_TILE_FUNCTOR_GPU_H_
diff --git a/tensorflow/core/kernels/tile_functor_gpu_bool.cu.cc b/tensorflow/core/kernels/tile_functor_gpu_bool.cu.cc
new file mode 100644
index 0000000000..c7a814c7a2
--- /dev/null
+++ b/tensorflow/core/kernels/tile_functor_gpu_bool.cu.cc
@@ -0,0 +1,31 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/tile_functor.h"
+#include "tensorflow/core/kernels/tile_functor_gpu.h"
+
+namespace tensorflow {
+namespace functor {
+using Eigen::GpuDevice;
+
+template struct Tile<GpuDevice, bool, int32>;
+template struct Tile<GpuDevice, bool, int64>;
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/tile_functor_gpu_complex128.cu.cc b/tensorflow/core/kernels/tile_functor_gpu_complex128.cu.cc
new file mode 100644
index 0000000000..4dfa4bac1b
--- /dev/null
+++ b/tensorflow/core/kernels/tile_functor_gpu_complex128.cu.cc
@@ -0,0 +1,31 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/tile_functor.h"
+#include "tensorflow/core/kernels/tile_functor_gpu.h"
+
+namespace tensorflow {
+namespace functor {
+using Eigen::GpuDevice;
+
+template struct Tile<GpuDevice, complex128, int32>;
+template struct Tile<GpuDevice, complex128, int64>;
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/tile_functor_gpu_complex64.cu.cc b/tensorflow/core/kernels/tile_functor_gpu_complex64.cu.cc
new file mode 100644
index 0000000000..525ede938f
--- /dev/null
+++ b/tensorflow/core/kernels/tile_functor_gpu_complex64.cu.cc
@@ -0,0 +1,31 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/tile_functor.h"
+#include "tensorflow/core/kernels/tile_functor_gpu.h"
+
+namespace tensorflow {
+namespace functor {
+using Eigen::GpuDevice;
+
+template struct Tile<GpuDevice, complex64, int32>;
+template struct Tile<GpuDevice, complex64, int64>;
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/tile_functor_gpu_double.cu.cc b/tensorflow/core/kernels/tile_functor_gpu_double.cu.cc
new file mode 100644
index 0000000000..25e024083e
--- /dev/null
+++ b/tensorflow/core/kernels/tile_functor_gpu_double.cu.cc
@@ -0,0 +1,31 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/tile_functor.h"
+#include "tensorflow/core/kernels/tile_functor_gpu.h"
+
+namespace tensorflow {
+namespace functor {
+using Eigen::GpuDevice;
+
+template struct Tile<GpuDevice, double, int32>;
+template struct Tile<GpuDevice, double, int64>;
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/tile_functor_gpu_float.cu.cc b/tensorflow/core/kernels/tile_functor_gpu_float.cu.cc
new file mode 100644
index 0000000000..f0f31370e4
--- /dev/null
+++ b/tensorflow/core/kernels/tile_functor_gpu_float.cu.cc
@@ -0,0 +1,31 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/tile_functor.h"
+#include "tensorflow/core/kernels/tile_functor_gpu.h"
+
+namespace tensorflow {
+namespace functor {
+using Eigen::GpuDevice;
+
+template struct Tile<GpuDevice, float, int32>;
+template struct Tile<GpuDevice, float, int64>;
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/tile_functor_gpu_half.cu.cc b/tensorflow/core/kernels/tile_functor_gpu_half.cu.cc
new file mode 100644
index 0000000000..6c3810a0bc
--- /dev/null
+++ b/tensorflow/core/kernels/tile_functor_gpu_half.cu.cc
@@ -0,0 +1,31 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/tile_functor.h"
+#include "tensorflow/core/kernels/tile_functor_gpu.h"
+
+namespace tensorflow {
+namespace functor {
+using Eigen::GpuDevice;
+
+template struct Tile<GpuDevice, Eigen::half, int32>;
+template struct Tile<GpuDevice, Eigen::half, int64>;
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/tile_functor_gpu_int16.cu.cc b/tensorflow/core/kernels/tile_functor_gpu_int16.cu.cc
new file mode 100644
index 0000000000..2280dcbc82
--- /dev/null
+++ b/tensorflow/core/kernels/tile_functor_gpu_int16.cu.cc
@@ -0,0 +1,31 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/tile_functor.h"
+#include "tensorflow/core/kernels/tile_functor_gpu.h"
+
+namespace tensorflow {
+namespace functor {
+using Eigen::GpuDevice;
+
+template struct Tile<GpuDevice, int16, int32>;
+template struct Tile<GpuDevice, int16, int64>;
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/tile_functor_gpu_int32.cu.cc b/tensorflow/core/kernels/tile_functor_gpu_int32.cu.cc
new file mode 100644
index 0000000000..b05403bada
--- /dev/null
+++ b/tensorflow/core/kernels/tile_functor_gpu_int32.cu.cc
@@ -0,0 +1,31 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/tile_functor.h"
+#include "tensorflow/core/kernels/tile_functor_gpu.h"
+
+namespace tensorflow {
+namespace functor {
+using Eigen::GpuDevice;
+
+template struct Tile<GpuDevice, int32, int32>;
+template struct Tile<GpuDevice, int32, int64>;
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/tile_functor_gpu_int64.cu.cc b/tensorflow/core/kernels/tile_functor_gpu_int64.cu.cc
new file mode 100644
index 0000000000..2d83c6b3a1
--- /dev/null
+++ b/tensorflow/core/kernels/tile_functor_gpu_int64.cu.cc
@@ -0,0 +1,31 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/tile_functor.h"
+#include "tensorflow/core/kernels/tile_functor_gpu.h"
+
+namespace tensorflow {
+namespace functor {
+using Eigen::GpuDevice;
+
+template struct Tile<GpuDevice, int64, int32>;
+template struct Tile<GpuDevice, int64, int64>;
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
-- 
GitLab


From f09a6224bac04fd8b45d72b5babee08212dcf836 Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Tue, 8 Jan 2019 02:23:41 -0800
Subject: [PATCH 0325/2345] Make DebugString() in ResourceBase constant

PiperOrigin-RevId: 228300469
---
 .../compiler/jit/xla_compilation_cache.cc      |  2 +-
 .../compiler/jit/xla_compilation_cache.h       |  2 +-
 .../compiler/tf2xla/xla_compiler_test.cc       |  2 +-
 tensorflow/compiler/tf2xla/xla_context.cc      |  2 +-
 tensorflow/compiler/tf2xla/xla_context.h       |  2 +-
 .../compiler/xrt/xrt_compilation_cache.cc      |  4 +++-
 .../compiler/xrt/xrt_compilation_cache.h       |  2 +-
 tensorflow/compiler/xrt/xrt_state.h            |  2 +-
 .../contrib/bigtable/kernels/bigtable_lib.h    |  4 ++--
 .../kernels/stats_accumulator_ops.cc           |  2 +-
 .../decision_tree_ensemble_resource.h          |  2 +-
 .../resources/quantile_stream_resource.h       |  2 +-
 .../kernels/v4/decision-tree-resource.h        |  2 +-
 .../kernels/v4/fertile-stats-resource.h        |  2 +-
 .../contrib/tensorrt/resources/trt_resources.h |  2 +-
 tensorflow/core/framework/lookup_interface.h   |  2 +-
 tensorflow/core/framework/queue_interface.h    |  4 ++--
 tensorflow/core/framework/reader_interface.h   |  2 +-
 tensorflow/core/framework/resource_mgr.h       |  2 +-
 tensorflow/core/framework/resource_mgr_test.cc |  8 ++++----
 .../core/framework/resource_op_kernel_test.cc  |  2 +-
 tensorflow/core/framework/resource_var.h       |  2 +-
 tensorflow/core/framework/stats_aggregator.h   |  2 +-
 tensorflow/core/kernels/barrier_ops.cc         |  2 +-
 tensorflow/core/kernels/batch_kernels.cc       |  6 +++---
 .../quantiles/quantile_stream_resource.h       | 18 +++++++++---------
 .../core/kernels/boosted_trees/resources.cc    |  2 +-
 .../core/kernels/boosted_trees/resources.h     |  2 +-
 .../kernels/conditional_accumulator_base.h     |  2 +-
 tensorflow/core/kernels/conv_ops.h             |  2 +-
 .../core/kernels/data/cache_dataset_ops.cc     |  4 +++-
 .../data/experimental/indexed_dataset_op.cc    |  2 +-
 .../data/experimental/threadpool_dataset_op.cc |  2 +-
 tensorflow/core/kernels/data/iterator_ops.cc   |  2 +-
 .../kernels/data/multi_device_iterator_ops.cc  |  2 +-
 .../core/kernels/data/shuffle_dataset_op.cc    |  2 +-
 tensorflow/core/kernels/fifo_queue.h           |  2 +-
 tensorflow/core/kernels/map_stage_op.cc        |  2 +-
 tensorflow/core/kernels/meta_support.cc        |  2 +-
 tensorflow/core/kernels/mutex_ops.cc           |  4 +++-
 tensorflow/core/kernels/priority_queue.h       |  2 +-
 .../core/kernels/random_shuffle_queue_op.cc    |  2 +-
 .../core/kernels/sparse_tensors_map_ops.cc     |  2 +-
 tensorflow/core/kernels/stack.cc               |  2 +-
 tensorflow/core/kernels/stage_op.cc            |  4 ++--
 tensorflow/core/kernels/summary_op.cc          |  4 +++-
 tensorflow/core/kernels/tensor_array.h         |  4 ++--
 .../core/kernels/tensor_forest/resources.h     |  2 +-
 tensorflow/core/kernels/variable_ops.cc        |  4 ++--
 tensorflow/core/summary/summary_db_writer.cc   |  2 +-
 tensorflow/core/summary/summary_file_writer.cc |  2 +-
 tensorflow/python/framework/test_ops.cc        |  2 +-
 52 files changed, 78 insertions(+), 70 deletions(-)

diff --git a/tensorflow/compiler/jit/xla_compilation_cache.cc b/tensorflow/compiler/jit/xla_compilation_cache.cc
index d5a1ab0850..bff4cc57ee 100644
--- a/tensorflow/compiler/jit/xla_compilation_cache.cc
+++ b/tensorflow/compiler/jit/xla_compilation_cache.cc
@@ -62,7 +62,7 @@ XlaCompilationCache::~XlaCompilationCache() {
   // about?
 }
 
-string XlaCompilationCache::DebugString() {
+string XlaCompilationCache::DebugString() const {
   return "XLA JIT compilation cache";
 }
 
diff --git a/tensorflow/compiler/jit/xla_compilation_cache.h b/tensorflow/compiler/jit/xla_compilation_cache.h
index 846d0c963d..02aa8f8839 100644
--- a/tensorflow/compiler/jit/xla_compilation_cache.h
+++ b/tensorflow/compiler/jit/xla_compilation_cache.h
@@ -88,7 +88,7 @@ class XlaCompilationCache : public ResourceBase {
   xla::LocalClient* client() const { return client_; }
   const DeviceType& device_type() const { return device_type_; }
 
-  string DebugString() override;
+  string DebugString() const override;
 
   // Describes the types, shapes and any compile-time constant arguments
   // to a kernel. Key that uniquely identifies a compilation output.
diff --git a/tensorflow/compiler/tf2xla/xla_compiler_test.cc b/tensorflow/compiler/tf2xla/xla_compiler_test.cc
index 32a4301840..492010f731 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler_test.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler_test.cc
@@ -82,7 +82,7 @@ namespace {
 // compiled kernels.
 class DummyResourceForTest : public ResourceBase {
  public:
-  string DebugString() override { return "dummy"; }
+  string DebugString() const override { return "dummy"; }
   void Increment() { ++value_; }
   int Get() { return value_; }
 
diff --git a/tensorflow/compiler/tf2xla/xla_context.cc b/tensorflow/compiler/tf2xla/xla_context.cc
index a69af70503..6139bf3cea 100644
--- a/tensorflow/compiler/tf2xla/xla_context.cc
+++ b/tensorflow/compiler/tf2xla/xla_context.cc
@@ -61,7 +61,7 @@ void XlaContext::set_args(std::vector<XlaExpression> args) {
 XlaContext::XlaContext(XlaCompiler* compiler, xla::XlaBuilder* builder)
     : compiler_(compiler), builder_(builder) {}
 
-string XlaContext::DebugString() { return "XLA JIT context"; }
+string XlaContext::DebugString() const { return "XLA JIT context"; }
 
 void XlaContext::SetRetval(int index, const XlaExpression& expression) {
   if (retvals_.size() <= index) {
diff --git a/tensorflow/compiler/tf2xla/xla_context.h b/tensorflow/compiler/tf2xla/xla_context.h
index 0767d1faac..eb4ad3fe6a 100644
--- a/tensorflow/compiler/tf2xla/xla_context.h
+++ b/tensorflow/compiler/tf2xla/xla_context.h
@@ -47,7 +47,7 @@ class XlaContext : public ResourceBase {
   XlaContext(XlaCompiler* compiler, xla::XlaBuilder* builder);
 
   // Virtual method defined by ResourceBase.
-  string DebugString() override;
+  string DebugString() const override;
 
   XlaCompiler* compiler() const { return compiler_; }
 
diff --git a/tensorflow/compiler/xrt/xrt_compilation_cache.cc b/tensorflow/compiler/xrt/xrt_compilation_cache.cc
index d1405eae46..8bf0f28d22 100644
--- a/tensorflow/compiler/xrt/xrt_compilation_cache.cc
+++ b/tensorflow/compiler/xrt/xrt_compilation_cache.cc
@@ -273,6 +273,8 @@ Status XRTCompilationCache::Lookup(
   return Status::OK();
 }
 
-string XRTCompilationCache::DebugString() { return "XRTCompilationCache"; }
+string XRTCompilationCache::DebugString() const {
+  return "XRTCompilationCache";
+}
 
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/xrt/xrt_compilation_cache.h b/tensorflow/compiler/xrt/xrt_compilation_cache.h
index c43d0fc478..7398e847d8 100644
--- a/tensorflow/compiler/xrt/xrt_compilation_cache.h
+++ b/tensorflow/compiler/xrt/xrt_compilation_cache.h
@@ -118,7 +118,7 @@ class XRTCompilationCache : public ResourceBase {
   // EntryRef holding the program is returned in entry.
   Status Lookup(int64 uid, std::unique_ptr<XRTCompilationCacheEntryRef>* entry);
 
-  string DebugString() override;
+  string DebugString() const override;
 
  private:
   // An entry in the compilation cache. The entry is deleted once it has been
diff --git a/tensorflow/compiler/xrt/xrt_state.h b/tensorflow/compiler/xrt/xrt_state.h
index 3e3d502412..4aac37737e 100644
--- a/tensorflow/compiler/xrt/xrt_state.h
+++ b/tensorflow/compiler/xrt/xrt_state.h
@@ -172,7 +172,7 @@ class XRTTupleAllocation : public ResourceBase {
   // ownership of the device memory is transferred to the result.
   xla::ShapeTree<xla::MaybeOwningDeviceMemory> ToDeviceMemoryTree(bool release);
 
-  string DebugString() override { return "XLA allocation handle"; }
+  string DebugString() const override { return "XLA allocation handle"; }
 
  private:
   // Creates a new handle with (tuple) shape.
diff --git a/tensorflow/contrib/bigtable/kernels/bigtable_lib.h b/tensorflow/contrib/bigtable/kernels/bigtable_lib.h
index 4652021fec..e3b4535bac 100644
--- a/tensorflow/contrib/bigtable/kernels/bigtable_lib.h
+++ b/tensorflow/contrib/bigtable/kernels/bigtable_lib.h
@@ -42,7 +42,7 @@ class BigtableClientResource : public ResourceBase {
     return client_;
   }
 
-  string DebugString() override {
+  string DebugString() const override {
     return strings::StrCat("BigtableClientResource(project_id: ", project_id_,
                            ", instance_id: ", instance_id_, ")");
   }
@@ -67,7 +67,7 @@ class BigtableTableResource : public ResourceBase {
 
   ::google::cloud::bigtable::noex::Table& table() { return table_; }
 
-  string DebugString() override {
+  string DebugString() const override {
     return strings::StrCat(
         "BigtableTableResource(client: ", client_->DebugString(),
         ", table: ", table_name_, ")");
diff --git a/tensorflow/contrib/boosted_trees/kernels/stats_accumulator_ops.cc b/tensorflow/contrib/boosted_trees/kernels/stats_accumulator_ops.cc
index e446c411a8..6faf696301 100644
--- a/tensorflow/contrib/boosted_trees/kernels/stats_accumulator_ops.cc
+++ b/tensorflow/contrib/boosted_trees/kernels/stats_accumulator_ops.cc
@@ -96,7 +96,7 @@ class StatsAccumulatorResource : public boosted_trees::StampedResource {
              TensorShapeUtils::IsScalar(hessian_shape));
   }
 
-  string DebugString() override {
+  string DebugString() const override {
     return strings::StrCat("StatsAccumulatorResource[size=", values_.size(),
                            "]");
   }
diff --git a/tensorflow/contrib/boosted_trees/resources/decision_tree_ensemble_resource.h b/tensorflow/contrib/boosted_trees/resources/decision_tree_ensemble_resource.h
index 94aeb2c7bb..0fe57c0a4e 100644
--- a/tensorflow/contrib/boosted_trees/resources/decision_tree_ensemble_resource.h
+++ b/tensorflow/contrib/boosted_trees/resources/decision_tree_ensemble_resource.h
@@ -34,7 +34,7 @@ class DecisionTreeEnsembleResource : public StampedResource {
             protobuf::Arena::CreateMessage<
                 boosted_trees::trees::DecisionTreeEnsembleConfig>(&arena_)) {}
 
-  string DebugString() override {
+  string DebugString() const override {
     return strings::StrCat("GTFlowDecisionTreeEnsemble[size=",
                            decision_tree_ensemble_->trees_size(), "]");
   }
diff --git a/tensorflow/contrib/boosted_trees/resources/quantile_stream_resource.h b/tensorflow/contrib/boosted_trees/resources/quantile_stream_resource.h
index fdaaae7f47..574e3065e7 100644
--- a/tensorflow/contrib/boosted_trees/resources/quantile_stream_resource.h
+++ b/tensorflow/contrib/boosted_trees/resources/quantile_stream_resource.h
@@ -43,7 +43,7 @@ class QuantileStreamResource : public StampedResource {
     set_stamp(stamp_token);
   }
 
-  string DebugString() override { return "QuantileStreamResource"; }
+  string DebugString() const override { return "QuantileStreamResource"; }
 
   tensorflow::mutex* mutex() { return &mu_; }
 
diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/decision-tree-resource.h b/tensorflow/contrib/tensor_forest/kernels/v4/decision-tree-resource.h
index d3edb43733..3100a5a0e5 100644
--- a/tensorflow/contrib/tensor_forest/kernels/v4/decision-tree-resource.h
+++ b/tensorflow/contrib/tensor_forest/kernels/v4/decision-tree-resource.h
@@ -32,7 +32,7 @@ class DecisionTreeResource : public ResourceBase {
   // Constructor.
   explicit DecisionTreeResource(const TensorForestParams& params);
 
-  string DebugString() override {
+  string DebugString() const override {
     return strings::StrCat("DecisionTree[size=",
                            decision_tree_->decision_tree().nodes_size(), "]");
   }
diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/fertile-stats-resource.h b/tensorflow/contrib/tensor_forest/kernels/v4/fertile-stats-resource.h
index eea0be27ca..44f2b3f473 100644
--- a/tensorflow/contrib/tensor_forest/kernels/v4/fertile-stats-resource.h
+++ b/tensorflow/contrib/tensor_forest/kernels/v4/fertile-stats-resource.h
@@ -40,7 +40,7 @@ class FertileStatsResource : public ResourceBase {
     model_op_ = LeafModelOperatorFactory::CreateLeafModelOperator(params_);
   }
 
-  string DebugString() override { return "FertileStats"; }
+  string DebugString() const override { return "FertileStats"; }
 
   void ExtractFromProto(const FertileStats& stats);
 
diff --git a/tensorflow/contrib/tensorrt/resources/trt_resources.h b/tensorflow/contrib/tensorrt/resources/trt_resources.h
index aac9e5c7bd..8d877b392f 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_resources.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_resources.h
@@ -48,7 +48,7 @@ class TRTCalibrationResource : public tensorflow::ResourceBase {
     allocator_.reset();
   }
 
-  string DebugString() override {
+  string DebugString() const override {
     std::stringstream oss;
     using std::dec;
     using std::endl;
diff --git a/tensorflow/core/framework/lookup_interface.h b/tensorflow/core/framework/lookup_interface.h
index d33945fd1b..7e5dbe5632 100644
--- a/tensorflow/core/framework/lookup_interface.h
+++ b/tensorflow/core/framework/lookup_interface.h
@@ -131,7 +131,7 @@ class LookupInterface : public ResourceBase {
   // - the default_value tensor shape matches the table's value shape.
   Status CheckFindArguments(const Tensor& keys, const Tensor& default_value);
 
-  string DebugString() override {
+  string DebugString() const override {
     return strings::StrCat("A lookup table of size: ", size());
   }
 
diff --git a/tensorflow/core/framework/queue_interface.h b/tensorflow/core/framework/queue_interface.h
index 4ca4416c5a..9395cce164 100644
--- a/tensorflow/core/framework/queue_interface.h
+++ b/tensorflow/core/framework/queue_interface.h
@@ -85,11 +85,11 @@ class QueueInterface : public ResourceBase {
   virtual Status MatchesNodeDef(const NodeDef& node_def) = 0;
 
   // Returns the number of elements in the queue.
-  virtual int32 size() = 0;
+  virtual int32 size() const = 0;
 
   virtual const DataTypeVector& component_dtypes() const = 0;
 
-  string DebugString() override {
+  string DebugString() const override {
     return strings::StrCat("A Queue of size: ", size());
   }
 
diff --git a/tensorflow/core/framework/reader_interface.h b/tensorflow/core/framework/reader_interface.h
index f894acbe1d..e47644cb8f 100644
--- a/tensorflow/core/framework/reader_interface.h
+++ b/tensorflow/core/framework/reader_interface.h
@@ -76,7 +76,7 @@ class ReaderInterface : public ResourceBase {
   // Note: Must Reset on error.
   virtual Status RestoreState(const string& state) = 0;
 
-  string DebugString() override { return "a reader"; }
+  string DebugString() const override { return "a reader"; }
 
  protected:
   virtual ~ReaderInterface() {}
diff --git a/tensorflow/core/framework/resource_mgr.h b/tensorflow/core/framework/resource_mgr.h
index 8a7c25da92..9c381e7d6b 100644
--- a/tensorflow/core/framework/resource_mgr.h
+++ b/tensorflow/core/framework/resource_mgr.h
@@ -77,7 +77,7 @@ namespace tensorflow {
 class ResourceBase : public core::RefCounted {
  public:
   // Returns a debug string for *this.
-  virtual string DebugString() = 0;
+  virtual string DebugString() const = 0;
 
   // Returns memory used by this resource.
   virtual int64 MemoryUsed() const { return 0; }
diff --git a/tensorflow/core/framework/resource_mgr_test.cc b/tensorflow/core/framework/resource_mgr_test.cc
index 7c7f0af0ce..1c785736e6 100644
--- a/tensorflow/core/framework/resource_mgr_test.cc
+++ b/tensorflow/core/framework/resource_mgr_test.cc
@@ -32,7 +32,7 @@ class Resource : public ResourceBase {
   explicit Resource(const string& label) : label_(label) {}
   ~Resource() override {}
 
-  string DebugString() override { return strings::StrCat("R/", label_); }
+  string DebugString() const override { return strings::StrCat("R/", label_); }
 
  private:
   string label_;
@@ -43,7 +43,7 @@ class Other : public ResourceBase {
   explicit Other(const string& label) : label_(label) {}
   ~Other() override {}
 
-  string DebugString() override { return strings::StrCat("O/", label_); }
+  string DebugString() const override { return strings::StrCat("O/", label_); }
 
  private:
   string label_;
@@ -245,7 +245,7 @@ class StubDevice : public DeviceBase {
 // Empty stub resource for testing resource handles.
 class StubResource : public ResourceBase {
  public:
-  string DebugString() override { return ""; }
+  string DebugString() const override { return ""; }
   int value_{0};
 };
 
@@ -305,7 +305,7 @@ TEST(ResourceHandleTest, DifferentDevice) {
 // Other stub resource to test type-checking of resource handles.
 class OtherStubResource : public ResourceBase {
  public:
-  string DebugString() override { return ""; }
+  string DebugString() const override { return ""; }
 };
 
 TEST(ResourceHandleTest, DifferentType) {
diff --git a/tensorflow/core/framework/resource_op_kernel_test.cc b/tensorflow/core/framework/resource_op_kernel_test.cc
index c1e503dc57..7a2a87045b 100644
--- a/tensorflow/core/framework/resource_op_kernel_test.cc
+++ b/tensorflow/core/framework/resource_op_kernel_test.cc
@@ -46,7 +46,7 @@ class StubDevice : public DeviceBase {
 // Stub resource for testing resource op kernel.
 class StubResource : public ResourceBase {
  public:
-  string DebugString() override { return ""; }
+  string DebugString() const override { return ""; }
   int code;
 };
 
diff --git a/tensorflow/core/framework/resource_var.h b/tensorflow/core/framework/resource_var.h
index f5de5dba88..9387b6c23c 100644
--- a/tensorflow/core/framework/resource_var.h
+++ b/tensorflow/core/framework/resource_var.h
@@ -67,7 +67,7 @@ class Var : public ResourceBase {
   mutex* mu() { return &mu_; }
   Tensor* tensor() { return &tensor_; }
 
-  string DebugString() override {
+  string DebugString() const override {
     return strings::StrCat(DataTypeString(tensor_.dtype()), "/",
                            tensor_.shape().DebugString());
   }
diff --git a/tensorflow/core/framework/stats_aggregator.h b/tensorflow/core/framework/stats_aggregator.h
index af53ed0a3c..7c960840d7 100644
--- a/tensorflow/core/framework/stats_aggregator.h
+++ b/tensorflow/core/framework/stats_aggregator.h
@@ -83,7 +83,7 @@ class StatsAggregatorResource : public ResourceBase {
     return stats_aggregator_;
   }
 
-  string DebugString() { return "StatsAggregatorResource"; }
+  string DebugString() const override { return "StatsAggregatorResource"; }
 
  private:
   const std::shared_ptr<StatsAggregator> stats_aggregator_;
diff --git a/tensorflow/core/kernels/barrier_ops.cc b/tensorflow/core/kernels/barrier_ops.cc
index aa91235822..d5bd36b4ce 100644
--- a/tensorflow/core/kernels/barrier_ops.cc
+++ b/tensorflow/core/kernels/barrier_ops.cc
@@ -300,7 +300,7 @@ class Barrier : public ResourceBase {
     ready_queue_->Unref();
   }
 
-  string DebugString() override { return "A barrier"; }
+  string DebugString() const override { return "A barrier"; }
 
  protected:
   template <typename T>
diff --git a/tensorflow/core/kernels/batch_kernels.cc b/tensorflow/core/kernels/batch_kernels.cc
index 35ddda0ec0..5ba461aa9d 100644
--- a/tensorflow/core/kernels/batch_kernels.cc
+++ b/tensorflow/core/kernels/batch_kernels.cc
@@ -233,7 +233,7 @@ class BatchResource : public ResourceBase {
     return Status::OK();
   }
 
-  string DebugString() final { return "BatchResource"; }
+  string DebugString() const final { return "BatchResource"; }
 
   // Ingests data from one invocation of the batch op. The data is enqueued to
   // be combined with others into a batch, asynchronously.
@@ -878,7 +878,7 @@ class UnbatchResource : public ResourceBase {
     timeout_enforcer_ = nullptr;
   }
 
-  string DebugString() final { return "UnbatchResource"; }
+  string DebugString() const final { return "UnbatchResource"; }
 
   Status Compute(OpKernelContext* context, AsyncOpKernel::DoneCallback done) {
     const Tensor& data_t = context->input(0);
@@ -1094,7 +1094,7 @@ class UnbatchGradResource : public ResourceBase {
  public:
   UnbatchGradResource() {}
 
-  string DebugString() final { return "UnbatchGradResource"; }
+  string DebugString() const final { return "UnbatchGradResource"; }
 
   // Flushes the information for one batch, given its context and done
   // callback. Clears all information about it from the available_tensors_.
diff --git a/tensorflow/core/kernels/boosted_trees/quantiles/quantile_stream_resource.h b/tensorflow/core/kernels/boosted_trees/quantiles/quantile_stream_resource.h
index 1c31724272..965bf2c924 100644
--- a/tensorflow/core/kernels/boosted_trees/quantiles/quantile_stream_resource.h
+++ b/tensorflow/core/kernels/boosted_trees/quantiles/quantile_stream_resource.h
@@ -37,15 +37,15 @@ class BoostedTreesQuantileStreamResource : public ResourceBase {
         epsilon_(epsilon),
         num_streams_(num_streams),
         max_elements_(max_elements) {
-          streams_.reserve(num_streams_);
-          boundaries_.reserve(num_streams_);
-          for (int64 idx = 0; idx < num_streams; ++idx) {
-            streams_.push_back(QuantileStream(epsilon, max_elements));
-            boundaries_.push_back(std::vector<float>());
-          }
-        }
-
-  string DebugString() override { return "QuantileStreamResource"; }
+    streams_.reserve(num_streams_);
+    boundaries_.reserve(num_streams_);
+    for (int64 idx = 0; idx < num_streams; ++idx) {
+      streams_.push_back(QuantileStream(epsilon, max_elements));
+      boundaries_.push_back(std::vector<float>());
+    }
+  }
+
+  string DebugString() const override { return "QuantileStreamResource"; }
 
   tensorflow::mutex* mutex() { return &mu_; }
 
diff --git a/tensorflow/core/kernels/boosted_trees/resources.cc b/tensorflow/core/kernels/boosted_trees/resources.cc
index 2798722536..42df484881 100644
--- a/tensorflow/core/kernels/boosted_trees/resources.cc
+++ b/tensorflow/core/kernels/boosted_trees/resources.cc
@@ -31,7 +31,7 @@ BoostedTreesEnsembleResource::BoostedTreesEnsembleResource()
           protobuf::Arena::CreateMessage<boosted_trees::TreeEnsemble>(
               &arena_)) {}
 
-string BoostedTreesEnsembleResource::DebugString() {
+string BoostedTreesEnsembleResource::DebugString() const {
   return strings::StrCat("TreeEnsemble[size=", tree_ensemble_->trees_size(),
                          "]");
 }
diff --git a/tensorflow/core/kernels/boosted_trees/resources.h b/tensorflow/core/kernels/boosted_trees/resources.h
index f961ed3814..3c7b2df9b0 100644
--- a/tensorflow/core/kernels/boosted_trees/resources.h
+++ b/tensorflow/core/kernels/boosted_trees/resources.h
@@ -48,7 +48,7 @@ class BoostedTreesEnsembleResource : public StampedResource {
  public:
   BoostedTreesEnsembleResource();
 
-  string DebugString() override;
+  string DebugString() const override;
 
   bool InitFromSerialized(const string& serialized, const int64 stamp_token);
 
diff --git a/tensorflow/core/kernels/conditional_accumulator_base.h b/tensorflow/core/kernels/conditional_accumulator_base.h
index 4a5ec6f0fb..2618ffbb09 100644
--- a/tensorflow/core/kernels/conditional_accumulator_base.h
+++ b/tensorflow/core/kernels/conditional_accumulator_base.h
@@ -68,7 +68,7 @@ class ConditionalAccumulatorBase : public ResourceBase {
 
   const DataType& dtype() const { return dtype_; }
 
-  string DebugString() override { return "A conditional accumulator"; }
+  string DebugString() const override { return "A conditional accumulator"; }
 
   // SetGlobalStep is a modifier method for current_global_step.
   // It returns an InvalidArgument error if the new_global_step is less than
diff --git a/tensorflow/core/kernels/conv_ops.h b/tensorflow/core/kernels/conv_ops.h
index 7ec878e0b2..7ccbaf4bf2 100644
--- a/tensorflow/core/kernels/conv_ops.h
+++ b/tensorflow/core/kernels/conv_ops.h
@@ -63,7 +63,7 @@ struct Im2ColBufferResource : public ResourceBase {
   // the buffer memory held by this resource.
   mutex mu;
   T* data;
-  string DebugString() { return "Im2ColBufferResource"; }
+  string DebugString() const { return "Im2ColBufferResource"; }
 };
 
 // Convolution parameters specified by Op attributes.
diff --git a/tensorflow/core/kernels/data/cache_dataset_ops.cc b/tensorflow/core/kernels/data/cache_dataset_ops.cc
index f00b38e732..535f49cff8 100644
--- a/tensorflow/core/kernels/data/cache_dataset_ops.cc
+++ b/tensorflow/core/kernels/data/cache_dataset_ops.cc
@@ -614,7 +614,9 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
      public:
       MemoryCache() = default;
 
-      string DebugString() override { return "CacheDataset::MemoryCache"; }
+      string DebugString() const override {
+        return "CacheDataset::MemoryCache";
+      }
 
       // Marks the cache as completed.
       void Complete() {
diff --git a/tensorflow/core/kernels/data/experimental/indexed_dataset_op.cc b/tensorflow/core/kernels/data/experimental/indexed_dataset_op.cc
index a07eaebdf9..83eeed7892 100644
--- a/tensorflow/core/kernels/data/experimental/indexed_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/indexed_dataset_op.cc
@@ -106,7 +106,7 @@ class MaterializedDatasetResource : public ResourceBase {
       const std::vector<PartialTensorShape>& output_shapes)
       : output_dtypes_(output_dtypes), output_shapes_(output_shapes) {}
 
-  string DebugString() override {
+  string DebugString() const override {
     return "Materialized IndexedDataset resource";
   }
 
diff --git a/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc b/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
index 8ae45ed5c9..fab3cab7da 100644
--- a/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
@@ -51,7 +51,7 @@ class ThreadPoolResource : public ResourceBase {
 
   int32 NumThreads() { return thread_pool_.NumThreads(); }
 
-  string DebugString() override { return "ThreadPoolResource"; }
+  string DebugString() const override { return "ThreadPoolResource"; }
 
  private:
   thread::ThreadPool thread_pool_;
diff --git a/tensorflow/core/kernels/data/iterator_ops.cc b/tensorflow/core/kernels/data/iterator_ops.cc
index 9f5881563b..81e26d35c0 100644
--- a/tensorflow/core/kernels/data/iterator_ops.cc
+++ b/tensorflow/core/kernels/data/iterator_ops.cc
@@ -231,7 +231,7 @@ class IteratorResource : public ResourceBase {
     return Status::OK();
   }
 
-  string DebugString() override { return "Iterator resource"; }
+  string DebugString() const override { return "Iterator resource"; }
 
   const DataTypeVector& output_dtypes() const { return output_dtypes_; }
 
diff --git a/tensorflow/core/kernels/data/multi_device_iterator_ops.cc b/tensorflow/core/kernels/data/multi_device_iterator_ops.cc
index ba2125a66e..05528e0ee0 100644
--- a/tensorflow/core/kernels/data/multi_device_iterator_ops.cc
+++ b/tensorflow/core/kernels/data/multi_device_iterator_ops.cc
@@ -59,7 +59,7 @@ class MultiDeviceIterator : public ResourceBase {
     DCHECK(lib_ != nullptr);
   }
 
-  string DebugString() override {
+  string DebugString() const override {
     return strings::StrCat("MultiDeviceIterator for ", devices_.size(),
                            " devices");
   }
diff --git a/tensorflow/core/kernels/data/shuffle_dataset_op.cc b/tensorflow/core/kernels/data/shuffle_dataset_op.cc
index db0cc6fa4d..4c380c1fa2 100644
--- a/tensorflow/core/kernels/data/shuffle_dataset_op.cc
+++ b/tensorflow/core/kernels/data/shuffle_dataset_op.cc
@@ -412,7 +412,7 @@ class ShuffleDatasetOp : public ShuffleDatasetOpBase {
             parent_generator_(seed, seed2),
             generator_(&parent_generator_) {}
 
-      string DebugString() override {
+      string DebugString() const override {
         return "ReshufflingDataset::RandomSeedGenerator";
       }
 
diff --git a/tensorflow/core/kernels/fifo_queue.h b/tensorflow/core/kernels/fifo_queue.h
index 697ee81c39..4d3a7c1971 100644
--- a/tensorflow/core/kernels/fifo_queue.h
+++ b/tensorflow/core/kernels/fifo_queue.h
@@ -49,7 +49,7 @@ class FIFOQueue : public TypedQueue<std::deque<PersistentTensor> > {
                       CallbackWithTuple callback) override;
   Status MatchesNodeDef(const NodeDef& node_def) override;
 
-  int32 size() override {
+  int32 size() const override {
     mutex_lock lock(mu_);
     return queues_[0].size();
   }
diff --git a/tensorflow/core/kernels/map_stage_op.cc b/tensorflow/core/kernels/map_stage_op.cc
index dd89597369..27a8696e54 100644
--- a/tensorflow/core/kernels/map_stage_op.cc
+++ b/tensorflow/core/kernels/map_stage_op.cc
@@ -480,7 +480,7 @@ class StagingMap : public ResourceBase {
     return map_.size();
   }
 
-  string DebugString() override { return "StagingMap"; }
+  string DebugString() const override { return "StagingMap"; }
 };
 
 template <bool Ordered>
diff --git a/tensorflow/core/kernels/meta_support.cc b/tensorflow/core/kernels/meta_support.cc
index 39e60c9fce..44f2997e18 100644
--- a/tensorflow/core/kernels/meta_support.cc
+++ b/tensorflow/core/kernels/meta_support.cc
@@ -54,7 +54,7 @@ class Scratch : public ResourceBase {
 
   uint8_t* buffer() { return scratch_32_aligned_; }
 
-  string DebugString() { return "MetaGemmScratchResource"; }
+  string DebugString() const override { return "MetaGemmScratchResource"; }
 
  private:
   std::unique_ptr<uint8_t> scratch_;
diff --git a/tensorflow/core/kernels/mutex_ops.cc b/tensorflow/core/kernels/mutex_ops.cc
index ddb7a606c1..1603a2aa86 100644
--- a/tensorflow/core/kernels/mutex_ops.cc
+++ b/tensorflow/core/kernels/mutex_ops.cc
@@ -45,7 +45,9 @@ class Mutex : public ResourceBase {
     VLOG(2) << "Creating mutex with name " << name << ": " << this;
   }
 
-  string DebugString() override { return strings::StrCat("Mutex ", name_); }
+  string DebugString() const override {
+    return strings::StrCat("Mutex ", name_);
+  }
 
   class LockReleaser {
    public:
diff --git a/tensorflow/core/kernels/priority_queue.h b/tensorflow/core/kernels/priority_queue.h
index 8e69b5b699..a719c518c3 100644
--- a/tensorflow/core/kernels/priority_queue.h
+++ b/tensorflow/core/kernels/priority_queue.h
@@ -68,7 +68,7 @@ class PriorityQueue
   Status MatchesPriorityNodeDefTypes(const NodeDef& node_def) const;
   Status MatchesPriorityNodeDefShapes(const NodeDef& node_def) const;
 
-  int32 size() override {
+  int32 size() const override {
     mutex_lock lock(mu_);
     return queues_[0].size();
   }
diff --git a/tensorflow/core/kernels/random_shuffle_queue_op.cc b/tensorflow/core/kernels/random_shuffle_queue_op.cc
index 31e8ce944f..02b9b022fd 100644
--- a/tensorflow/core/kernels/random_shuffle_queue_op.cc
+++ b/tensorflow/core/kernels/random_shuffle_queue_op.cc
@@ -59,7 +59,7 @@ class RandomShuffleQueue : public TypedQueue<std::vector<PersistentTensor> > {
                       CallbackWithTuple callback) override;
   Status MatchesNodeDef(const NodeDef& node_def) override;
 
-  int32 size() override {
+  int32 size() const override {
     mutex_lock lock(mu_);
     return queues_[0].size();
   }
diff --git a/tensorflow/core/kernels/sparse_tensors_map_ops.cc b/tensorflow/core/kernels/sparse_tensors_map_ops.cc
index 74fa3a15f0..939638b370 100644
--- a/tensorflow/core/kernels/sparse_tensors_map_ops.cc
+++ b/tensorflow/core/kernels/sparse_tensors_map_ops.cc
@@ -43,7 +43,7 @@ class SparseTensorsMap : public ResourceBase {
  public:
   explicit SparseTensorsMap(const string& name) : name_(name), counter_(0) {}
 
-  string DebugString() override { return "A SparseTensorsMap"; }
+  string DebugString() const override { return "A SparseTensorsMap"; }
 
   typedef struct {
     PersistentTensor indices;
diff --git a/tensorflow/core/kernels/stack.cc b/tensorflow/core/kernels/stack.cc
index 5c70a2d62d..2af6b4b814 100644
--- a/tensorflow/core/kernels/stack.cc
+++ b/tensorflow/core/kernels/stack.cc
@@ -96,7 +96,7 @@ class Stack : public ResourceBase {
 
   DataType ElemType() { return elem_type_; }
 
-  string DebugString() override {
+  string DebugString() const override {
     mutex_lock l(mu_);
     return strings::StrCat("Stack[", stack_name_, "]");
   }
diff --git a/tensorflow/core/kernels/stage_op.cc b/tensorflow/core/kernels/stage_op.cc
index c91bdc43cf..65174e163c 100644
--- a/tensorflow/core/kernels/stage_op.cc
+++ b/tensorflow/core/kernels/stage_op.cc
@@ -132,7 +132,7 @@ class Buffer : public ResourceBase {
     notify_inserters_if_bounded(&lock);
   }
 
-  string DebugString() override {
+  string DebugString() const override {
     std::unique_lock<std::mutex> lock(mu_);
     return strings::StrCat("Staging size: ", buf_.size());
   }
@@ -170,7 +170,7 @@ class Buffer : public ResourceBase {
   std::size_t capacity_;
   std::size_t memory_limit_;
   std::size_t current_bytes_;
-  std::mutex mu_;
+  mutable std::mutex mu_;
   std::condition_variable non_empty_cond_var_;
   std::condition_variable full_cond_var_;
   std::deque<Tuple> buf_;
diff --git a/tensorflow/core/kernels/summary_op.cc b/tensorflow/core/kernels/summary_op.cc
index 1f4e3418f4..1053aa7d53 100644
--- a/tensorflow/core/kernels/summary_op.cc
+++ b/tensorflow/core/kernels/summary_op.cc
@@ -124,7 +124,9 @@ TF_CALL_REAL_NUMBER_TYPES(REGISTER)
 struct HistogramResource : public ResourceBase {
   histogram::ThreadSafeHistogram histogram;
 
-  string DebugString() override { return "A histogram summary. Stats ..."; }
+  string DebugString() const override {
+    return "A histogram summary. Stats ...";
+  }
 };
 
 class SummaryMergeOp : public OpKernel {
diff --git a/tensorflow/core/kernels/tensor_array.h b/tensorflow/core/kernels/tensor_array.h
index 384a63e945..507ab459ca 100644
--- a/tensorflow/core/kernels/tensor_array.h
+++ b/tensorflow/core/kernels/tensor_array.h
@@ -261,7 +261,7 @@ class TensorArray : public ResourceBase {
     return Status::OK();
   }
 
-  string DebugString() override {
+  string DebugString() const override {
     mutex_lock l(mu_);
     CHECK(!closed_);
     return strings::StrCat("TensorArray[", tensors_.size(), "]");
@@ -376,7 +376,7 @@ class TensorArray : public ResourceBase {
   const DataType dtype_;
   Tensor handle_;
 
-  mutex mu_;
+  mutable mutex mu_;
 
   // Marks that the tensor_array_ has been cleared.
   bool closed_ GUARDED_BY(mu_);
diff --git a/tensorflow/core/kernels/tensor_forest/resources.h b/tensorflow/core/kernels/tensor_forest/resources.h
index da258e5017..f0a78f9726 100644
--- a/tensorflow/core/kernels/tensor_forest/resources.h
+++ b/tensorflow/core/kernels/tensor_forest/resources.h
@@ -34,7 +34,7 @@ class TensorForestTreeResource : public ResourceBase {
  public:
   TensorForestTreeResource();
 
-  string DebugString() override {
+  string DebugString() const override {
     return strings::StrCat("TensorForestTree[size=", get_size(), "]");
   }
 
diff --git a/tensorflow/core/kernels/variable_ops.cc b/tensorflow/core/kernels/variable_ops.cc
index eadea18f76..00994bbe8e 100644
--- a/tensorflow/core/kernels/variable_ops.cc
+++ b/tensorflow/core/kernels/variable_ops.cc
@@ -35,7 +35,7 @@ class LegacyVar : public ResourceBase {
   mutex* mu() { return &mu_; }
   Tensor* tensor() { return &tensor_; }
 
-  string DebugString() override {
+  string DebugString() const override {
     return strings::StrCat(DataTypeString(tensor_.dtype()), "/",
                            tensor_.shape().DebugString());
   }
@@ -116,7 +116,7 @@ class TemporaryVariableOp : public OpKernel {
     mutex mu;
     Tensor val;
     string name;
-    string DebugString() override { return name; }
+    string DebugString() const override { return name; }
     ~TmpVar() override { VLOG(3) << "TmpVar " << name << " deleted"; }
   };
 
diff --git a/tensorflow/core/summary/summary_db_writer.cc b/tensorflow/core/summary/summary_db_writer.cc
index 7a5d796821..b203d439cc 100644
--- a/tensorflow/core/summary/summary_db_writer.cc
+++ b/tensorflow/core/summary/summary_db_writer.cc
@@ -972,7 +972,7 @@ class SummaryDbWriter : public SummaryWriterInterface {
     return MigrateEvent(std::move(e));
   }
 
-  string DebugString() override { return "SummaryDbWriter"; }
+  string DebugString() const override { return "SummaryDbWriter"; }
 
  private:
   Status Write(int64 step, const Tensor& t, const string& tag,
diff --git a/tensorflow/core/summary/summary_file_writer.cc b/tensorflow/core/summary/summary_file_writer.cc
index 593ccdd684..711a7d3d10 100644
--- a/tensorflow/core/summary/summary_file_writer.cc
+++ b/tensorflow/core/summary/summary_file_writer.cc
@@ -148,7 +148,7 @@ class SummaryFileWriter : public SummaryWriterInterface {
     return Status::OK();
   }
 
-  string DebugString() override { return "SummaryFileWriter"; }
+  string DebugString() const override { return "SummaryFileWriter"; }
 
  private:
   double GetWallTime() {
diff --git a/tensorflow/python/framework/test_ops.cc b/tensorflow/python/framework/test_ops.cc
index 99e184a8ac..1d0145f61c 100644
--- a/tensorflow/python/framework/test_ops.cc
+++ b/tensorflow/python/framework/test_ops.cc
@@ -157,7 +157,7 @@ REGISTER_KERNEL_BUILDER(Name("Old").Device(DEVICE_CPU), OldOp);
 // Stubbed-out resource to test resource handle ops.
 class StubResource : public ResourceBase {
  public:
-  string DebugString() override { return ""; }
+  string DebugString() const override { return ""; }
 };
 
 REGISTER_RESOURCE_HANDLE_KERNEL(StubResource);
-- 
GitLab


From 8624a703ebd914e9d91bb7992570b52946fad970 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Tue, 8 Jan 2019 02:47:46 -0800
Subject: [PATCH 0326/2345] Try to make resize bilinear test more
 deterministic.

PiperOrigin-RevId: 228301953
---
 tensorflow/lite/kernels/internal/resize_bilinear_test.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/lite/kernels/internal/resize_bilinear_test.cc b/tensorflow/lite/kernels/internal/resize_bilinear_test.cc
index 1c5ac1992f..4a19b69a7c 100644
--- a/tensorflow/lite/kernels/internal/resize_bilinear_test.cc
+++ b/tensorflow/lite/kernels/internal/resize_bilinear_test.cc
@@ -76,6 +76,7 @@ void TestOneResizeBilinear(int batch, int depth, int input_width,
 }
 
 TEST(ResizeBilinear, TestResizeBilinear8Bit) {
+  RandomEngine().seed(38291);
   const int kTestsToRun = 100 * 1000;
   for (int i = 0; i < kTestsToRun; i++) {
     const int batch = ExponentialRandomPositiveInt(0.9f, 3, 20);
@@ -91,6 +92,7 @@ TEST(ResizeBilinear, TestResizeBilinear8Bit) {
 }
 
 TEST(ResizeBilinear2x2, TestResizeBilinear8Bit) {
+  RandomEngine().seed(38291);
   const int kTestsToRun = 100 * 1000;
   for (int i = 0; i < kTestsToRun; i++) {
     const int batch = ExponentialRandomPositiveInt(0.9f, 3, 20);
@@ -106,6 +108,7 @@ TEST(ResizeBilinear2x2, TestResizeBilinear8Bit) {
 }
 
 TEST(ResizeBilinear, TestResizeBilinear) {
+  RandomEngine().seed(38291);
   const int kTestsToRun = 100 * 1000;
   for (int i = 0; i < kTestsToRun; i++) {
     const int batch = ExponentialRandomPositiveInt(0.9f, 3, 20);
@@ -121,6 +124,7 @@ TEST(ResizeBilinear, TestResizeBilinear) {
 }
 
 TEST(ResizeBilinear2x2, TestResizeBilinear) {
+  RandomEngine().seed(38291);
   const int kTestsToRun = 100 * 1000;
   for (int i = 0; i < kTestsToRun; i++) {
     const int batch = ExponentialRandomPositiveInt(0.9f, 3, 20);
-- 
GitLab


From 07c6aa1a40f4515506d53262d7da2e8663c049cb Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Tue, 8 Jan 2019 02:48:31 -0800
Subject: [PATCH 0327/2345] Use int32_t rather than int32 for compatibility.

PiperOrigin-RevId: 228302002
---
 tensorflow/lite/kernels/sparse_output_fully_connected_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/kernels/sparse_output_fully_connected_test.cc b/tensorflow/lite/kernels/sparse_output_fully_connected_test.cc
index 8152ae6685..7d5fec192c 100644
--- a/tensorflow/lite/kernels/sparse_output_fully_connected_test.cc
+++ b/tensorflow/lite/kernels/sparse_output_fully_connected_test.cc
@@ -62,7 +62,7 @@ class BaseSparseOutputFullyConnectedOpModel : public SingleOpModel {
     PopulateTensor(input_, data);
   }
 
-  void SetLookup(const std::vector<int32>& f) { PopulateTensor(lookup_, f); }
+  void SetLookup(const std::vector<int32_t>& f) { PopulateTensor(lookup_, f); }
 
   void SetBias(const std::vector<float>& f) { PopulateTensor(bias_, f); }
 
-- 
GitLab


From 81492c074aa134d8756b361557ec2dd9aad3cdc1 Mon Sep 17 00:00:00 2001
From: Chris Jones <cjfj@google.com>
Date: Tue, 8 Jan 2019 02:50:54 -0800
Subject: [PATCH 0328/2345] Add DistStrat `ReplicaContext.all_reduce`.

PiperOrigin-RevId: 228302155
---
 .../collective_all_reduce_strategy_test.py    |  44 ++++-
 .../python/mirrored_strategy_multigpu_test.py |  47 ++++-
 .../python/one_device_strategy_test.py        |  22 ++-
 .../python/parameter_server_strategy_test.py  |  38 +++-
 .../distribute/python/strategy_test_lib.py    | 162 ++++++++++++++++++
 .../python/distribute/distribute_lib.py       |  54 ++++++
 ...nsorflow.distribute.-replica-context.pbtxt |   4 +
 ...nsorflow.distribute.-replica-context.pbtxt |   4 +
 8 files changed, 363 insertions(+), 12 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
index 0fb672dded..4c8c01a216 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
@@ -397,9 +397,11 @@ class DistributedCollectiveAllReduceStrategyTestWithChief(
         self._test_complex_model, self._cluster_spec, num_gpus=num_gpus)
 
 
-class LocalCollectiveAllReduceStrategy(CollectiveAllReduceStrategyTestBase,
-                                       strategy_test_lib.DistributionTestBase,
-                                       parameterized.TestCase):
+class LocalCollectiveAllReduceStrategy(
+    CollectiveAllReduceStrategyTestBase,
+    strategy_test_lib.DistributionTestBase,
+    strategy_test_lib.TwoDeviceDistributionTestBase,
+    parameterized.TestCase):
 
   def testMinimizeLossGraph(self, num_gpus=2):
     # Collective ops doesn't support strategy with one device.
@@ -428,6 +430,42 @@ class LocalCollectiveAllReduceStrategy(CollectiveAllReduceStrategyTestBase,
     self._test_input_fn_iterator(None, None, num_gpus,
                                  input_fn, expected_values)
 
+  def testAllReduceSum(self):
+    if context.num_gpus() < 2: self.skipTest('Not enough GPUs')
+    distribution, target, config = self._get_test_object(None, None, num_gpus=2)
+    with self.cached_session(config=config, target=target):
+      self._test_all_reduce_sum(distribution)
+
+  def testAllReduceSumGradients(self):
+    if context.num_gpus() < 2: self.skipTest('Not enough GPUs')
+    distribution, target, config = self._get_test_object(None, None, num_gpus=2)
+    with self.cached_session(config=config, target=target):
+      self._test_all_reduce_sum_gradients(distribution)
+
+  def testAllReduceSumGradientTape(self):
+    if context.num_gpus() < 2: self.skipTest('Not enough GPUs')
+    distribution, target, config = self._get_test_object(None, None, num_gpus=2)
+    with self.cached_session(config=config, target=target):
+      self._test_all_reduce_sum_gradient_tape(distribution)
+
+  def testAllReduceMean(self):
+    if context.num_gpus() < 2: self.skipTest('Not enough GPUs')
+    distribution, target, config = self._get_test_object(None, None, num_gpus=2)
+    with self.cached_session(config=config, target=target):
+      self._test_all_reduce_mean(distribution)
+
+  def testAllReduceMeanGradients(self):
+    if context.num_gpus() < 2: self.skipTest('Not enough GPUs')
+    distribution, target, config = self._get_test_object(None, None, num_gpus=2)
+    with self.cached_session(config=config, target=target):
+      self._test_all_reduce_mean_gradients(distribution)
+
+  def testAllReduceMeanGradientTape(self):
+    if context.num_gpus() < 2: self.skipTest('Not enough GPUs')
+    distribution, target, config = self._get_test_object(None, None, num_gpus=2)
+    with self.cached_session(config=config, target=target):
+      self._test_all_reduce_mean_gradient_tape(distribution)
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index 9cbc34412d..9821828b2d 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -66,8 +66,10 @@ GPU_TEST = "test_gpu" in sys.argv[0]
         combinations.core_mirrored_strategy_with_gpu_and_cpu,
         combinations.core_mirrored_strategy_with_two_gpus],
     mode=["graph", "eager"]))
-class MirroredTwoDeviceDistributionTest(strategy_test_lib.DistributionTestBase,
-                                        parameterized.TestCase):
+class MirroredTwoDeviceDistributionTest(
+    strategy_test_lib.DistributionTestBase,
+    strategy_test_lib.TwoDeviceDistributionTestBase,
+    parameterized.TestCase):
 
   def testMinimizeLoss(self, distribution):
     if context.executing_eagerly():
@@ -117,6 +119,24 @@ class MirroredTwoDeviceDistributionTest(strategy_test_lib.DistributionTestBase,
   def testGlobalStepUpdate(self, distribution):
     self._test_global_step_update(distribution)
 
+  def testAllReduceSum(self, distribution):
+    self._test_all_reduce_sum(distribution)
+
+  def testAllReduceSumGradients(self, distribution):
+    self._test_all_reduce_sum_gradients(distribution)
+
+  def testAllReduceSumGradientTape(self, distribution):
+    self._test_all_reduce_sum_gradient_tape(distribution)
+
+  def testAllReduceMean(self, distribution):
+    self._test_all_reduce_mean(distribution)
+
+  def testAllReduceMeanGradients(self, distribution):
+    self._test_all_reduce_mean_gradients(distribution)
+
+  def testAllReduceMeanGradientTape(self, distribution):
+    self._test_all_reduce_mean_gradient_tape(distribution)
+
 
 def one_device_combinations():
   return combinations.combine(
@@ -128,25 +148,42 @@ def one_device_combinations():
       mode=["graph", "eager"])
 
 
+@combinations.generate(one_device_combinations())
 class MirroredOneDeviceDistributionTest(
     strategy_test_lib.DistributionTestBase,
+    strategy_test_lib.OneDeviceDistributionTestBase,
     parameterized.TestCase):
 
-  @combinations.generate(one_device_combinations())
   def testMinimizeLoss(self, distribution):
     if context.executing_eagerly():
       self._test_minimize_loss_eager(distribution)
     else:
       self._test_minimize_loss_graph(distribution)
 
-  @combinations.generate(one_device_combinations())
   def testReplicaId(self, distribution):
     self._test_replica_id(distribution)
 
-  @combinations.generate(one_device_combinations())
   def testCallAndMergeExceptions(self, distribution):
     self._test_call_and_merge_exceptions(distribution)
 
+  def testAllReduceSum(self, distribution):
+    self._test_all_reduce_sum(distribution)
+
+  def testAllReduceSumGradients(self, distribution):
+    self._test_all_reduce_sum_gradients(distribution)
+
+  def testAllReduceSumGradientTape(self, distribution):
+    self._test_all_reduce_sum_gradient_tape(distribution)
+
+  def testAllReduceMean(self, distribution):
+    self._test_all_reduce_mean(distribution)
+
+  def testAllReduceMeanGradients(self, distribution):
+    self._test_all_reduce_mean_gradients(distribution)
+
+  def testAllReduceMeanGradientTape(self, distribution):
+    self._test_all_reduce_mean_gradient_tape(distribution)
+
 
 class MirroredStrategyVariableCreatorStackTest(
     test.TestCase, parameterized.TestCase):
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy_test.py b/tensorflow/contrib/distribute/python/one_device_strategy_test.py
index d46cd6f529..2403dc8f12 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy_test.py
@@ -25,7 +25,9 @@ from tensorflow.python.eager import test
 from tensorflow.python.framework import test_util
 
 
-class OneDeviceStrategyTest(strategy_test_lib.DistributionTestBase):
+class OneDeviceStrategyTest(
+    strategy_test_lib.DistributionTestBase,
+    strategy_test_lib.OneDeviceDistributionTestBase):
 
   def _get_distribution_strategy(self):
     return one_device_strategy.OneDeviceStrategy("/device:CPU:0")
@@ -57,6 +59,24 @@ class OneDeviceStrategyTest(strategy_test_lib.DistributionTestBase):
     self._test_input_fn_iterator(
         iterator, d.extended.worker_devices, expected_values)
 
+  def testAllReduceSum(self):
+    self._test_all_reduce_sum(self._get_distribution_strategy())
+
+  def testAllReduceSumGradients(self):
+    self._test_all_reduce_sum_gradients(self._get_distribution_strategy())
+
+  def testAllReduceSumGradientTape(self):
+    self._test_all_reduce_sum_gradient_tape(self._get_distribution_strategy())
+
+  def testAllReduceMean(self):
+    self._test_all_reduce_mean(self._get_distribution_strategy())
+
+  def testAllReduceMeanGradients(self):
+    self._test_all_reduce_mean_gradients(self._get_distribution_strategy())
+
+  def testAllReduceMeanGradientTape(self):
+    self._test_all_reduce_mean_gradient_tape(self._get_distribution_strategy())
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
index d8c6c50db2..7836687e7d 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
@@ -603,9 +603,11 @@ class ParameterServerStrategyTestBase(
         self.assertEqual(expected_value, computed_value)
 
 
-class ParameterServerStrategyTest(ParameterServerStrategyTestBase,
-                                  strategy_test_lib.DistributionTestBase,
-                                  parameterized.TestCase):
+class ParameterServerStrategyTest(
+    ParameterServerStrategyTestBase,
+    strategy_test_lib.DistributionTestBase,
+    strategy_test_lib.TwoDeviceDistributionTestBase,
+    parameterized.TestCase):
 
   @classmethod
   def setUpClass(cls):
@@ -782,6 +784,36 @@ class ParameterServerStrategyTest(ParameterServerStrategyTestBase,
     # Verify isolate_session_state
     self.assertTrue(new_config.isolate_session_state)
 
+  def testAllReduceSum(self):
+    distribution = parameter_server_strategy.ParameterServerStrategy(
+        num_gpus_per_worker=2)
+    self._test_all_reduce_sum(distribution)
+
+  def testAllReduceSumGradients(self):
+    distribution = parameter_server_strategy.ParameterServerStrategy(
+        num_gpus_per_worker=2)
+    self._test_all_reduce_sum_gradients(distribution)
+
+  def testAllReduceSumGradientTape(self):
+    distribution = parameter_server_strategy.ParameterServerStrategy(
+        num_gpus_per_worker=2)
+    self._test_all_reduce_sum_gradient_tape(distribution)
+
+  def testAllReduceMean(self):
+    distribution = parameter_server_strategy.ParameterServerStrategy(
+        num_gpus_per_worker=2)
+    self._test_all_reduce_mean(distribution)
+
+  def testAllReduceMeanGradients(self):
+    distribution = parameter_server_strategy.ParameterServerStrategy(
+        num_gpus_per_worker=2)
+    self._test_all_reduce_mean_gradients(distribution)
+
+  def testAllReduceMeanGradientTape(self):
+    distribution = parameter_server_strategy.ParameterServerStrategy(
+        num_gpus_per_worker=2)
+    self._test_all_reduce_mean_gradient_tape(distribution)
+
 
 class ParameterServerStrategyWithChiefTest(ParameterServerStrategyTestBase,
                                            parameterized.TestCase):
diff --git a/tensorflow/contrib/distribute/python/strategy_test_lib.py b/tensorflow/contrib/distribute/python/strategy_test_lib.py
index 6e5280e356..4fbd630cf7 100644
--- a/tensorflow/contrib/distribute/python/strategy_test_lib.py
+++ b/tensorflow/contrib/distribute/python/strategy_test_lib.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.core.protobuf import config_pb2
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import distribution_strategy_context as ds_context
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.distribute import values
@@ -31,6 +32,7 @@ from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.layers import core
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
@@ -292,3 +294,163 @@ class DistributionTestBase(test.TestCase):
       global_step_tensors = strategy.unwrap(value)
       global_step_values = self.evaluate(global_step_tensors)
       self.assertEqual((1,) * len(global_step_tensors), global_step_values)
+
+
+class OneDeviceDistributionTestBase(test.TestCase):
+  """Some tests that should work with any one-device DistributionStrategy."""
+
+  def _test_all_reduce_sum(self, strategy):
+    self._test_collective_comms(
+        strategy, _all_sum, inputs=(4., [42., 43.]), expected=(4., [42., 43.]))
+
+  def _test_all_reduce_sum_gradients(self, strategy):
+    self._test_collective_comms_gradients(
+        strategy, _all_sum, inputs=[4.], expected_grads=[4.])
+
+  def _test_all_reduce_sum_gradient_tape(self, strategy):
+    self._test_collective_comms_gradient_tape(
+        strategy, _all_sum, inputs=[4.], expected_grads=[4.])
+
+  def _test_all_reduce_mean(self, strategy):
+    self._test_collective_comms(
+        strategy, _all_mean, inputs=(2., [21., 22.]), expected=(2., [21., 22.]))
+
+  def _test_all_reduce_mean_gradients(self, strategy):
+    self._test_collective_comms_gradients(
+        strategy, _all_mean, inputs=[5.], expected_grads=[5.])
+
+  def _test_all_reduce_mean_gradient_tape(self, strategy):
+    self._test_collective_comms_gradient_tape(
+        strategy, _all_mean, inputs=[5.], expected_grads=[5.])
+
+  def _test_collective_comms(self, strategy, comm_fn, inputs, expected):
+    inputs = strategy.make_input_fn_iterator(
+        lambda _: dataset_ops.Dataset.from_tensors(inputs))
+
+    self.evaluate(inputs.initialize())
+    outputs = self.evaluate(
+        list(map(strategy.unwrap, strategy.experimental_run(comm_fn, inputs))))
+    self.assertAllEqual([expected[0]], outputs[0])
+    self.assertAllEqual([expected[1]], outputs[1])
+
+  def _test_collective_comms_gradients(
+      self, strategy, comm_fn, inputs, expected_grads):
+    if context.executing_eagerly():
+      self.skipTest("`tf.gradients` is not supported with eager execution.")
+
+    def step(c):
+      x = constant_op.constant(42.)
+      y = comm_fn(x) * c
+      return gradients_impl.gradients(y, [x])[0]
+
+    inputs = strategy.make_input_fn_iterator(
+        lambda _: dataset_ops.Dataset.from_tensors(inputs))
+
+    self.evaluate(inputs.initialize())
+    self.assertAllEqual(
+        expected_grads,
+        self.evaluate(strategy.unwrap(strategy.experimental_run(step, inputs))))
+
+  def _test_collective_comms_gradient_tape(
+      self, strategy, comm_fn, inputs, expected_grads):
+    def step(c):
+      x = constant_op.constant(42.)
+      with backprop.GradientTape() as tape:
+        tape.watch(x)
+        y = comm_fn(x) * c
+      return tape.gradient(y, x)
+
+    inputs = strategy.make_input_fn_iterator(
+        lambda _: dataset_ops.Dataset.from_tensors(inputs))
+
+    self.evaluate(inputs.initialize())
+    self.assertAllEqual(
+        expected_grads,
+        self.evaluate(strategy.unwrap(strategy.experimental_run(step, inputs))))
+
+
+class TwoDeviceDistributionTestBase(test.TestCase):
+  """Some tests that should work with any two-device DistributionStrategy."""
+
+  def _test_all_reduce_sum(self, strategy):
+    self._test_collective_comms(
+        strategy, _all_sum,
+        inputs=([1., 3.], [[39., 2.], [3., 41.]]),
+        expected=(4., [42., 43.]))
+
+  def _test_all_reduce_sum_gradients(self, strategy):
+    self._test_collective_comms_gradients(
+        strategy, _all_sum, inputs=[1., 3.], expected_grads=[4., 4.])
+
+  def _test_all_reduce_sum_gradient_tape(self, strategy):
+    self._test_collective_comms_gradient_tape(
+        strategy, _all_sum, inputs=[1., 3.], expected_grads=[4., 4.])
+
+  def _test_all_reduce_mean(self, strategy):
+    self._test_collective_comms(
+        strategy, _all_mean,
+        inputs=([1., 3.], [[39., 2.], [3., 41.]]),
+        expected=(2., [21., 21.5]))
+
+  def _test_all_reduce_mean_gradients(self, strategy):
+    self._test_collective_comms_gradients(
+        strategy, _all_mean, inputs=[1., 3.], expected_grads=[2., 2.])
+
+  def _test_all_reduce_mean_gradient_tape(self, strategy):
+    self._test_collective_comms_gradient_tape(
+        strategy, _all_mean, inputs=[1., 3.], expected_grads=[2., 2.])
+
+  def _test_collective_comms(self, strategy, comm_fn, inputs, expected):
+    inputs = strategy.make_input_fn_iterator(
+        lambda _: dataset_ops.Dataset.from_tensor_slices(inputs))
+
+    self.evaluate(inputs.initialize())
+    outputs = self.evaluate(
+        list(map(strategy.unwrap, strategy.experimental_run(comm_fn, inputs))))
+    self.assertAllEqual([expected[0], expected[0]], outputs[0])
+    self.assertAllEqual([expected[1], expected[1]], outputs[1])
+
+  def _test_collective_comms_gradients(
+      self, strategy, comm_fn, inputs, expected_grads):
+    if context.executing_eagerly():
+      self.skipTest("`tf.gradients` is not supported with eager execution.")
+
+    def step(c):
+      x = constant_op.constant(42.)
+      y = comm_fn(x) * c
+      return gradients_impl.gradients(y, [x])[0]
+
+    inputs = strategy.make_input_fn_iterator(
+        lambda _: dataset_ops.Dataset.from_tensor_slices(inputs))
+
+    self.evaluate(inputs.initialize())
+    self.assertAllEqual(
+        expected_grads,
+        self.evaluate(strategy.unwrap(strategy.experimental_run(step, inputs))))
+
+  def _test_collective_comms_gradient_tape(
+      self, strategy, comm_fn, inputs, expected_grads):
+    def step(c):
+      x = constant_op.constant(42.)
+      with backprop.GradientTape() as tape:
+        tape.watch(x)
+        y = comm_fn(x) * c
+      return tape.gradient(y, x)
+
+    inputs = strategy.make_input_fn_iterator(
+        lambda _: dataset_ops.Dataset.from_tensor_slices(inputs))
+
+    self.evaluate(inputs.initialize())
+    self.assertAllEqual(
+        expected_grads,
+        self.evaluate(strategy.unwrap(strategy.experimental_run(step, inputs))))
+
+
+def _all_sum(value):
+  ctx = ds_context.get_replica_context()
+  return ctx.all_reduce(reduce_util.ReduceOp.SUM, value)
+
+
+def _all_mean(value):
+  ctx = ds_context.get_replica_context()
+  return ctx.all_reduce(reduce_util.ReduceOp.MEAN, value)
diff --git a/tensorflow/python/distribute/distribute_lib.py b/tensorflow/python/distribute/distribute_lib.py
index 76cbdd53d9..4ad8cc00b8 100644
--- a/tensorflow/python/distribute/distribute_lib.py
+++ b/tensorflow/python/distribute/distribute_lib.py
@@ -33,6 +33,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import custom_gradient
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops.losses import losses_impl
@@ -1554,6 +1555,50 @@ class ReplicaContext(object):
     require_replica_context(self)
     return (device_util.current(),)
 
+  def all_reduce(self, reduce_op, value):
+    """All-reduces the given `Tensor` nest across replicas.
+
+    If `all_reduce` is called in any replica, it must be called in all replicas.
+    The nested structure and `Tensor` shapes must be identical in all replicas.
+
+    IMPORTANT: The ordering of communications must be identical in all replicas.
+
+    Example with two replicas:
+      Replica 0 `value`: {'a': 1, 'b': [40,  1]}
+      Replica 1 `value`: {'a': 3, 'b': [ 2, 98]}
+
+      If `reduce_op` == `SUM`:
+        Result (on all replicas): {'a': 4, 'b': [42, 99]}
+
+      If `reduce_op` == `MEAN`:
+        Result (on all replicas): {'a': 2, 'b': [21, 49.5]}
+
+    Args:
+      reduce_op: Reduction type, an instance of `tf.distribute.ReduceOp` enum.
+      value: The nested structure of `Tensor`s to all-reduced.
+        The structure must be compatible with `tf.nest`.
+
+    Returns:
+       A `Tensor` nest with the reduced `value`s from each replica.
+    """
+    def batch_all_reduce(strategy, *value_flat):
+      return strategy.extended.batch_reduce_to(
+          reduce_op, [(v, _batch_reduce_destination(v)) for v in value_flat])
+
+    if reduce_op in [reduce_util.ReduceOp.SUM, reduce_util.ReduceOp.MEAN]:
+      # TODO(cjfj): Work out why `batch_reduce` doesn't return the correct grad.
+      @custom_gradient.custom_gradient
+      def grad_wrapper(*xs):
+        ys = self.merge_call(batch_all_reduce, args=xs)
+        # The gradient of an all-sum is itself an all-sum (all-mean, likewise).
+        return ys, lambda *dy_s: self.all_reduce(reduce_op, dy_s)
+      return nest.pack_sequence_as(value, grad_wrapper(*nest.flatten(value)))
+    else:
+      # TODO(cjfj): Implement gradients for other reductions.
+      reduced = nest.pack_sequence_as(
+          value, self.merge_call(batch_all_reduce, args=nest.flatten(value)))
+      return nest.map_structure(array_ops.prevent_gradient, reduced)
+
   # TODO(josh11b): Implement `start_all_reduce(method, t)` for efficient
   # all-reduce. It would return a function returning the result of reducing `t`
   # across all replicas. The caller would wait to call this function until they
@@ -1564,6 +1609,15 @@ class ReplicaContext(object):
   #   to that point that the first result is needed. Most likely this can be
   #   implemented in terms of `merge_call()` and `batch_reduce_to()`.
 
+
+def _batch_reduce_destination(x):
+  """Returns the destinations for batch all-reduce."""
+  if isinstance(x, ops.Tensor):  # One device strategies.
+    return x.device
+  else:
+    return x
+
+
 # ------------------------------------------------------------------------------
 
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-replica-context.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-replica-context.pbtxt
index df707e8920..22f8160c96 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-replica-context.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-replica-context.pbtxt
@@ -26,6 +26,10 @@ tf_class {
     name: "__init__"
     argspec: "args=[\'self\', \'strategy\', \'replica_id_in_sync_group\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "all_reduce"
+    argspec: "args=[\'self\', \'reduce_op\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "merge_call"
     argspec: "args=[\'self\', \'merge_fn\', \'args\', \'kwargs\'], varargs=None, keywords=None, defaults=[\'()\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-replica-context.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-replica-context.pbtxt
index df707e8920..22f8160c96 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-replica-context.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-replica-context.pbtxt
@@ -26,6 +26,10 @@ tf_class {
     name: "__init__"
     argspec: "args=[\'self\', \'strategy\', \'replica_id_in_sync_group\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "all_reduce"
+    argspec: "args=[\'self\', \'reduce_op\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "merge_call"
     argspec: "args=[\'self\', \'merge_fn\', \'args\', \'kwargs\'], varargs=None, keywords=None, defaults=[\'()\', \'None\'], "
-- 
GitLab


From 0ef4b190443a2f7b3b2330f053df5503af6b0191 Mon Sep 17 00:00:00 2001
From: James Keeling <jtkeeling@google.com>
Date: Tue, 8 Jan 2019 02:53:00 -0800
Subject: [PATCH 0329/2345] Split topk GPU code into multiple files

This file was a bottleneck during compilation, often taking many minutes to compile. In local testing this change reduces the wall-clock build time for the topk GPU kernels from 155s to 48s.

PiperOrigin-RevId: 228302319
---
 tensorflow/core/kernels/BUILD                 | 16 ++++++++++-
 .../{topk_op_gpu.cu.cc => topk_op_gpu.h}      | 12 +++-----
 .../core/kernels/topk_op_gpu_double.cu.cc     | 28 +++++++++++++++++++
 .../core/kernels/topk_op_gpu_float.cu.cc      | 28 +++++++++++++++++++
 .../core/kernels/topk_op_gpu_half.cu.cc       | 28 +++++++++++++++++++
 .../core/kernels/topk_op_gpu_int16.cu.cc      | 28 +++++++++++++++++++
 .../core/kernels/topk_op_gpu_int32.cu.cc      | 28 +++++++++++++++++++
 .../core/kernels/topk_op_gpu_int64.cu.cc      | 28 +++++++++++++++++++
 .../core/kernels/topk_op_gpu_int8.cu.cc       | 28 +++++++++++++++++++
 .../core/kernels/topk_op_gpu_uint16.cu.cc     | 28 +++++++++++++++++++
 .../core/kernels/topk_op_gpu_uint8.cu.cc      | 28 +++++++++++++++++++
 11 files changed, 271 insertions(+), 9 deletions(-)
 rename tensorflow/core/kernels/{topk_op_gpu.cu.cc => topk_op_gpu.h} (98%)
 create mode 100644 tensorflow/core/kernels/topk_op_gpu_double.cu.cc
 create mode 100644 tensorflow/core/kernels/topk_op_gpu_float.cu.cc
 create mode 100644 tensorflow/core/kernels/topk_op_gpu_half.cu.cc
 create mode 100644 tensorflow/core/kernels/topk_op_gpu_int16.cu.cc
 create mode 100644 tensorflow/core/kernels/topk_op_gpu_int32.cu.cc
 create mode 100644 tensorflow/core/kernels/topk_op_gpu_int64.cu.cc
 create mode 100644 tensorflow/core/kernels/topk_op_gpu_int8.cu.cc
 create mode 100644 tensorflow/core/kernels/topk_op_gpu_uint16.cu.cc
 create mode 100644 tensorflow/core/kernels/topk_op_gpu_uint8.cu.cc

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index fee30fa6c3..3df63d34e3 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -3862,7 +3862,21 @@ tf_kernel_library(
 
 tf_kernel_library(
     name = "topk_op",
-    prefix = "topk_op",
+    srcs = ["topk_op.cc"],
+    hdrs = ["topk_op.h"],
+    gpu_srcs = [
+        "topk_op.h",
+        "topk_op_gpu.h",
+        "topk_op_gpu_double.cu.cc",
+        "topk_op_gpu_float.cu.cc",
+        "topk_op_gpu_half.cu.cc",
+        "topk_op_gpu_int64.cu.cc",
+        "topk_op_gpu_int32.cu.cc",
+        "topk_op_gpu_int16.cu.cc",
+        "topk_op_gpu_uint16.cu.cc",
+        "topk_op_gpu_int8.cu.cc",
+        "topk_op_gpu_uint8.cu.cc",
+    ],
     deps = NN_DEPS + if_cuda(["@cub_archive//:cub"]),
 )
 
diff --git a/tensorflow/core/kernels/topk_op_gpu.cu.cc b/tensorflow/core/kernels/topk_op_gpu.h
similarity index 98%
rename from tensorflow/core/kernels/topk_op_gpu.cu.cc
rename to tensorflow/core/kernels/topk_op_gpu.h
index 2fbe1fe7cb..6f3bec20f6 100644
--- a/tensorflow/core/kernels/topk_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/topk_op_gpu.h
@@ -12,6 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
+#ifndef TENSORFLOW_CORE_KERNELS_TOPK_OP_GPU_H_
+#define TENSORFLOW_CORE_KERNELS_TOPK_OP_GPU_H_
 
 #if GOOGLE_CUDA
 
@@ -561,14 +563,8 @@ struct TopKFunctor<GPUDevice, T> {
 };
 
 }  // end namespace functor
-
-#define INSTANTIATE_TEMPLATE(type) \
-  template struct functor::TopKFunctor<GPUDevice, type>;
-
-TF_CALL_GPU_NUMBER_TYPES(INSTANTIATE_TEMPLATE);
-TF_CALL_INTEGRAL_TYPES(INSTANTIATE_TEMPLATE);
-#undef INSTANTIATE_TEMPLATE
-
 }  // namespace tensorflow
 
 #endif  // GOOGLE_CUDA
+
+#endif  // TENSORFLOW_CORE_KERNELS_TOPK_OP_GPU_H_
diff --git a/tensorflow/core/kernels/topk_op_gpu_double.cu.cc b/tensorflow/core/kernels/topk_op_gpu_double.cu.cc
new file mode 100644
index 0000000000..8a5a7e71b1
--- /dev/null
+++ b/tensorflow/core/kernels/topk_op_gpu_double.cu.cc
@@ -0,0 +1,28 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/topk_op.h"
+#include "tensorflow/core/kernels/topk_op_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+
+template struct functor::TopKFunctor<GPUDevice, double>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/topk_op_gpu_float.cu.cc b/tensorflow/core/kernels/topk_op_gpu_float.cu.cc
new file mode 100644
index 0000000000..0b69396bb1
--- /dev/null
+++ b/tensorflow/core/kernels/topk_op_gpu_float.cu.cc
@@ -0,0 +1,28 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/topk_op.h"
+#include "tensorflow/core/kernels/topk_op_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+
+template struct functor::TopKFunctor<GPUDevice, float>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/topk_op_gpu_half.cu.cc b/tensorflow/core/kernels/topk_op_gpu_half.cu.cc
new file mode 100644
index 0000000000..e53586aeca
--- /dev/null
+++ b/tensorflow/core/kernels/topk_op_gpu_half.cu.cc
@@ -0,0 +1,28 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/topk_op.h"
+#include "tensorflow/core/kernels/topk_op_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+
+template struct functor::TopKFunctor<GPUDevice, Eigen::half>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/topk_op_gpu_int16.cu.cc b/tensorflow/core/kernels/topk_op_gpu_int16.cu.cc
new file mode 100644
index 0000000000..5bd310523c
--- /dev/null
+++ b/tensorflow/core/kernels/topk_op_gpu_int16.cu.cc
@@ -0,0 +1,28 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/topk_op.h"
+#include "tensorflow/core/kernels/topk_op_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+
+template struct functor::TopKFunctor<GPUDevice, int16>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/topk_op_gpu_int32.cu.cc b/tensorflow/core/kernels/topk_op_gpu_int32.cu.cc
new file mode 100644
index 0000000000..55b393a0c0
--- /dev/null
+++ b/tensorflow/core/kernels/topk_op_gpu_int32.cu.cc
@@ -0,0 +1,28 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/topk_op.h"
+#include "tensorflow/core/kernels/topk_op_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+
+template struct functor::TopKFunctor<GPUDevice, int32>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/topk_op_gpu_int64.cu.cc b/tensorflow/core/kernels/topk_op_gpu_int64.cu.cc
new file mode 100644
index 0000000000..3e4a775056
--- /dev/null
+++ b/tensorflow/core/kernels/topk_op_gpu_int64.cu.cc
@@ -0,0 +1,28 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/topk_op.h"
+#include "tensorflow/core/kernels/topk_op_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+
+template struct functor::TopKFunctor<GPUDevice, int64>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/topk_op_gpu_int8.cu.cc b/tensorflow/core/kernels/topk_op_gpu_int8.cu.cc
new file mode 100644
index 0000000000..ac73cd170b
--- /dev/null
+++ b/tensorflow/core/kernels/topk_op_gpu_int8.cu.cc
@@ -0,0 +1,28 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/topk_op.h"
+#include "tensorflow/core/kernels/topk_op_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+
+template struct functor::TopKFunctor<GPUDevice, int8>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/topk_op_gpu_uint16.cu.cc b/tensorflow/core/kernels/topk_op_gpu_uint16.cu.cc
new file mode 100644
index 0000000000..8d5f8ceb06
--- /dev/null
+++ b/tensorflow/core/kernels/topk_op_gpu_uint16.cu.cc
@@ -0,0 +1,28 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/topk_op.h"
+#include "tensorflow/core/kernels/topk_op_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+
+template struct functor::TopKFunctor<GPUDevice, uint16>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/topk_op_gpu_uint8.cu.cc b/tensorflow/core/kernels/topk_op_gpu_uint8.cu.cc
new file mode 100644
index 0000000000..fc1a8a2c8c
--- /dev/null
+++ b/tensorflow/core/kernels/topk_op_gpu_uint8.cu.cc
@@ -0,0 +1,28 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/topk_op.h"
+#include "tensorflow/core/kernels/topk_op_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+
+template struct functor::TopKFunctor<GPUDevice, uint8>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
-- 
GitLab


From 6c74d3a5893ff6e3ba35b471841ccc349848587d Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Tue, 8 Jan 2019 17:01:58 +0530
Subject: [PATCH 0330/2345] Ensure variables are initialized

'ReadVarint32FromArray' & 'ReadVarint64FromArray' doesn't ensure output variable 'value' is filled always. Since 'unused_ok' is not checked for true/false initialize the temp variables, so that random values dont propogate.
---
 tensorflow/core/util/proto/decode.h | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/tensorflow/core/util/proto/decode.h b/tensorflow/core/util/proto/decode.h
index 8dde14dffc..188830cc1f 100644
--- a/tensorflow/core/util/proto/decode.h
+++ b/tensorflow/core/util/proto/decode.h
@@ -91,7 +91,7 @@ inline const uint8* ReadVarint64FromArray(const uint8* buffer, bool* ok,
 // the 64 bit version instead of copying the code.
 inline const uint8* ReadVarint32FromArray(const uint8* buffer, bool* ok,
                                           uint32* value) {
-  uint64 tmp;
+  uint64 tmp = 0;
   const uint8* buf = ReadVarint64FromArray(buffer, ok, &tmp);
   *value = tmp & 0xffffffff;
   return buf;
@@ -106,7 +106,7 @@ const uint8* ReadFromArray(const uint8* buf, TensorType* value);
 template <>
 inline const uint8* ReadFromArray<int64, WireFormatLite::TYPE_INT32>(
     const uint8* buf, int64* value) {
-  uint32 temp;
+  uint32 temp = 0;
   bool unused_ok;  // The Counting pass would have failed if this were corrupt.
   buf = ReadVarint32FromArray(buf, &unused_ok, &temp);
   *value = static_cast<int64>(temp);
@@ -116,7 +116,7 @@ inline const uint8* ReadFromArray<int64, WireFormatLite::TYPE_INT32>(
 template <>
 inline const uint8* ReadFromArray<int32, WireFormatLite::TYPE_INT32>(
     const uint8* buf, int32* value) {
-  uint32 temp;
+  uint32 temp = 0;
   bool unused_ok;  // The Counting pass would have failed if this were corrupt.
   buf = ReadVarint32FromArray(buf, &unused_ok, &temp);
   *value = static_cast<int32>(temp);
@@ -126,7 +126,7 @@ inline const uint8* ReadFromArray<int32, WireFormatLite::TYPE_INT32>(
 template <>
 inline const uint8* ReadFromArray<int64, WireFormatLite::TYPE_INT64>(
     const uint8* buf, int64* value) {
-  uint64 temp;
+  uint64 temp = 0;
   bool unused_ok;  // The Counting pass would have failed if this were corrupt.
   buf = ReadVarint64FromArray(buf, &unused_ok, &temp);
   *value = WrapUnsignedAsSigned64(temp);
@@ -136,7 +136,7 @@ inline const uint8* ReadFromArray<int64, WireFormatLite::TYPE_INT64>(
 template <>
 inline const uint8* ReadFromArray<uint64, WireFormatLite::TYPE_UINT32>(
     const uint8* buf, uint64* value) {
-  uint32 temp;
+  uint32 temp = 0;
   bool unused_ok;  // The Counting pass would have failed if this were corrupt.
   buf = ReadVarint32FromArray(buf, &unused_ok, &temp);
   *value = temp;
@@ -160,7 +160,7 @@ inline const uint8* ReadFromArray<uint64, WireFormatLite::TYPE_UINT64>(
 template <>
 inline const uint8* ReadFromArray<int64, WireFormatLite::TYPE_SINT32>(
     const uint8* buf, int64* value) {
-  uint64 temp;
+  uint64 temp = 0;
   bool unused_ok;  // The Counting pass would have failed if this were corrupt.
   buf = ReadVarint64FromArray(buf, &unused_ok, &temp);
   *value = WireFormatLite::ZigZagDecode32(temp);
@@ -170,7 +170,7 @@ inline const uint8* ReadFromArray<int64, WireFormatLite::TYPE_SINT32>(
 template <>
 inline const uint8* ReadFromArray<int32, WireFormatLite::TYPE_SINT32>(
     const uint8* buf, int32* value) {
-  uint32 temp;
+  uint32 temp = 0;
   bool unused_ok;  // The Counting pass would have failed if this were corrupt.
   buf = ReadVarint32FromArray(buf, &unused_ok, &temp);
   *value = WireFormatLite::ZigZagDecode32(temp);
@@ -180,7 +180,7 @@ inline const uint8* ReadFromArray<int32, WireFormatLite::TYPE_SINT32>(
 template <>
 inline const uint8* ReadFromArray<int64, WireFormatLite::TYPE_SINT64>(
     const uint8* buf, int64* value) {
-  uint64 temp;
+  uint64 temp = 0;
   bool unused_ok;  // The Counting pass would have failed if this were corrupt.
   buf = ReadVarint64FromArray(buf, &unused_ok, &temp);
   *value = WireFormatLite::ZigZagDecode64(temp);
@@ -280,7 +280,7 @@ inline const uint8* ReadFromArray<double, WireFormatLite::TYPE_DOUBLE>(
 template <>
 inline const uint8* ReadFromArray<bool, WireFormatLite::TYPE_BOOL>(
     const uint8* buf, bool* value) {
-  uint64 temp;
+  uint64 temp = 0;
   bool unused_ok;  // The Counting pass would have failed if this were corrupt.
   buf = ReadVarint64FromArray(buf, &unused_ok, &temp);
   *value = temp != 0;
@@ -290,7 +290,7 @@ inline const uint8* ReadFromArray<bool, WireFormatLite::TYPE_BOOL>(
 template <>
 inline const uint8* ReadFromArray<int, WireFormatLite::TYPE_ENUM>(
     const uint8* buf, int* value) {
-  uint32 temp;
+  uint32 temp = 0;
   bool unused_ok;  // The Counting pass would have failed if this were corrupt.
   buf = ReadVarint32FromArray(buf, &unused_ok, &temp);
   *value = static_cast<int>(temp);
-- 
GitLab


From 3436665db2438dd339a0edddd0d0a3d49b5b2424 Mon Sep 17 00:00:00 2001
From: James Keeling <jtkeeling@google.com>
Date: Tue, 8 Jan 2019 03:28:39 -0800
Subject: [PATCH 0331/2345] Split scan ops GPU code into multiple files

This file was a bottleneck during compilation, often taking many minutes to compile. In local testing this change reduces the wall-clock build time for the scan ops GPU kernels from 107s to 96s.

PiperOrigin-RevId: 228304727
---
 tensorflow/core/kernels/BUILD                 | 10 +++++-
 .../{scan_ops_gpu.cu.cc => scan_ops_gpu.h}    | 16 +++-------
 .../core/kernels/scan_ops_gpu_double.cu.cc    | 31 +++++++++++++++++++
 .../core/kernels/scan_ops_gpu_float.cu.cc     | 31 +++++++++++++++++++
 .../core/kernels/scan_ops_gpu_half.cu.cc      | 31 +++++++++++++++++++
 5 files changed, 107 insertions(+), 12 deletions(-)
 rename tensorflow/core/kernels/{scan_ops_gpu.cu.cc => scan_ops_gpu.h} (97%)
 create mode 100644 tensorflow/core/kernels/scan_ops_gpu_double.cu.cc
 create mode 100644 tensorflow/core/kernels/scan_ops_gpu_float.cu.cc
 create mode 100644 tensorflow/core/kernels/scan_ops_gpu_half.cu.cc

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 3df63d34e3..b934c64bfc 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -3279,7 +3279,15 @@ tf_kernel_library(
 
 tf_kernel_library(
     name = "scan_ops",
-    prefix = "scan_ops",
+    srcs = ["scan_ops.cc"],
+    hdrs = ["scan_ops.h"],
+    gpu_srcs = [
+        "scan_ops.h",
+        "scan_ops_gpu.h",
+        "scan_ops_gpu_double.cu.cc",
+        "scan_ops_gpu_float.cu.cc",
+        "scan_ops_gpu_half.cu.cc",
+    ],
     deps = MATH_DEPS + if_cuda(["@cub_archive//:cub"]),
 )
 
diff --git a/tensorflow/core/kernels/scan_ops_gpu.cu.cc b/tensorflow/core/kernels/scan_ops_gpu.h
similarity index 97%
rename from tensorflow/core/kernels/scan_ops_gpu.cu.cc
rename to tensorflow/core/kernels/scan_ops_gpu.h
index ed66c02dc5..976b221540 100644
--- a/tensorflow/core/kernels/scan_ops_gpu.cu.cc
+++ b/tensorflow/core/kernels/scan_ops_gpu.h
@@ -13,6 +13,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#ifndef TENSORFLOW_CORE_KERNELS_SCAN_OPS_GPU_H_
+#define TENSORFLOW_CORE_KERNELS_SCAN_OPS_GPU_H_
+
 #if GOOGLE_CUDA
 
 #define EIGEN_USE_GPU
@@ -290,17 +293,8 @@ struct Scan<GPUDevice, Eigen::internal::ProdReducer<T>, T> {
 };
 
 }  // namespace functor
-
-#define DEFINE(REDUCER, T) template struct functor::Scan<GPUDevice, REDUCER, T>;
-
-#define DEFINE_FOR_ALL_REDUCERS(T)           \
-  DEFINE(Eigen::internal::SumReducer<T>, T); \
-  DEFINE(Eigen::internal::ProdReducer<T>, T);
-
-TF_CALL_GPU_NUMBER_TYPES(DEFINE_FOR_ALL_REDUCERS);
-#undef DEFINE_FOR_ALL_REDUCERS
-#undef DEFINE
-
 }  // end namespace tensorflow
 
 #endif  // GOOGLE_CUDA
+
+#endif  // TENSORFLOW_CORE_KERNELS_SCAN_OPS_GPU_H_
diff --git a/tensorflow/core/kernels/scan_ops_gpu_double.cu.cc b/tensorflow/core/kernels/scan_ops_gpu_double.cu.cc
new file mode 100644
index 0000000000..adce37e473
--- /dev/null
+++ b/tensorflow/core/kernels/scan_ops_gpu_double.cu.cc
@@ -0,0 +1,31 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/scan_ops.h"
+#include "tensorflow/core/kernels/scan_ops_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+template struct functor::Scan<GpuDevice, Eigen::internal::SumReducer<double>,
+                              double>;
+template struct functor::Scan<GpuDevice, Eigen::internal::ProdReducer<double>,
+                              double>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/scan_ops_gpu_float.cu.cc b/tensorflow/core/kernels/scan_ops_gpu_float.cu.cc
new file mode 100644
index 0000000000..b72415822d
--- /dev/null
+++ b/tensorflow/core/kernels/scan_ops_gpu_float.cu.cc
@@ -0,0 +1,31 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/scan_ops.h"
+#include "tensorflow/core/kernels/scan_ops_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+template struct functor::Scan<GpuDevice, Eigen::internal::SumReducer<float>,
+                              float>;
+template struct functor::Scan<GpuDevice, Eigen::internal::ProdReducer<float>,
+                              float>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/scan_ops_gpu_half.cu.cc b/tensorflow/core/kernels/scan_ops_gpu_half.cu.cc
new file mode 100644
index 0000000000..f9fb528be9
--- /dev/null
+++ b/tensorflow/core/kernels/scan_ops_gpu_half.cu.cc
@@ -0,0 +1,31 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/scan_ops.h"
+#include "tensorflow/core/kernels/scan_ops_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+template struct functor::Scan<
+    GpuDevice, Eigen::internal::SumReducer<Eigen::half>, Eigen::half>;
+template struct functor::Scan<
+    GpuDevice, Eigen::internal::ProdReducer<Eigen::half>, Eigen::half>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
-- 
GitLab


From 137c142341c9f10a1e105403fbfb1123a1802eb9 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Tue, 8 Jan 2019 03:57:48 -0800
Subject: [PATCH 0332/2345] Make toco_port trivial initialize-able for test
 cases.

Problem is toco_port indirects a way of initializing google.
One impl has a static way of finding out if it is initialized (internal).
One impl has a non-static way to find out.

In the non-static way, test cases won't work since we initialize the tensorflow
google emulation elsewhere (gtest override)

PiperOrigin-RevId: 228306710
---
 tensorflow/lite/toco/BUILD                     | 2 ++
 tensorflow/lite/toco/import_tensorflow_test.cc | 2 ++
 tensorflow/lite/toco/toco_convert_test.cc      | 2 ++
 tensorflow/lite/toco/toco_port.cc              | 7 +++++++
 tensorflow/lite/toco/toco_port.h               | 4 ++++
 tensorflow/lite/toco/toco_port_test.cc         | 1 +
 tensorflow/lite/toco/tooling_util_test.cc      | 2 ++
 7 files changed, 20 insertions(+)

diff --git a/tensorflow/lite/toco/BUILD b/tensorflow/lite/toco/BUILD
index d77234c80a..40bceedd6a 100644
--- a/tensorflow/lite/toco/BUILD
+++ b/tensorflow/lite/toco/BUILD
@@ -342,6 +342,7 @@ tf_cc_test(
     name = "import_tensorflow_test",
     srcs = ["import_tensorflow_test.cc"],
     deps = [
+        ":toco_port",
         ":toco_tooling",
         "//tensorflow/core:framework",
         "//tensorflow/core:graph",
@@ -386,6 +387,7 @@ tf_cc_test(
     srcs = ["tooling_util_test.cc"],
     deps = [
         ":model",
+        ":toco_port",
         ":tooling_util",
         "//tensorflow/core:lib",
         "//tensorflow/lite/testing:util",
diff --git a/tensorflow/lite/toco/import_tensorflow_test.cc b/tensorflow/lite/toco/import_tensorflow_test.cc
index 70382f546b..de7f4cdb7e 100644
--- a/tensorflow/lite/toco/import_tensorflow_test.cc
+++ b/tensorflow/lite/toco/import_tensorflow_test.cc
@@ -13,6 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 #include "tensorflow/lite/toco/import_tensorflow.h"
+#include "tensorflow/lite/toco/toco_port.h"
 
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
@@ -569,5 +570,6 @@ TEST(ImportTest, UnsupportedOpWithMultipleOutputs) {
 int main(int argc, char** argv) {
   ::tflite::LogToStderr();
   ::testing::InitGoogleTest(&argc, argv);
+  ::toco::port::InitGoogleWasDoneElsewhere();
   return RUN_ALL_TESTS();
 }
diff --git a/tensorflow/lite/toco/toco_convert_test.cc b/tensorflow/lite/toco/toco_convert_test.cc
index 3730c53ae1..739b924607 100644
--- a/tensorflow/lite/toco/toco_convert_test.cc
+++ b/tensorflow/lite/toco/toco_convert_test.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
 #include "tensorflow/lite/testing/util.h"
+#include "tensorflow/lite/toco/toco_port.h"
 
 namespace toco {
 namespace {
@@ -176,5 +177,6 @@ TEST(TocoTest, TransientStringTensors) {
 int main(int argc, char** argv) {
   ::tflite::LogToStderr();
   ::testing::InitGoogleTest(&argc, argv);
+  ::toco::port::InitGoogleWasDoneElsewhere();
   return RUN_ALL_TESTS();
 }
diff --git a/tensorflow/lite/toco/toco_port.cc b/tensorflow/lite/toco/toco_port.cc
index fb8c1b8337..b222032e61 100644
--- a/tensorflow/lite/toco/toco_port.cc
+++ b/tensorflow/lite/toco/toco_port.cc
@@ -57,6 +57,11 @@ void InitGoogle(const char* usage, int* argc, char*** argv, bool remove_flags) {
   ::InitGoogle(usage, argc, argv, remove_flags);
 }
 
+void InitGoogleWasDoneElsewhere() {
+  // Nothing need be done since ::CheckInitGoogleIsDone() is aware of other
+  // possible initialization entry points.
+}
+
 void CheckInitGoogleIsDone(const char* message) {
   ::CheckInitGoogleIsDone(message);
 }
@@ -152,6 +157,8 @@ constexpr int kFileWriteFlags = O_CREAT | O_WRONLY;
 
 static bool port_initialized = false;
 
+void InitGoogleWasDoneElsewhere() { port_initialized = true; }
+
 void InitGoogle(const char* usage, int* argc, char*** argv, bool remove_flags) {
   if (!port_initialized) {
 #if defined(PLATFORM_GOOGLE)
diff --git a/tensorflow/lite/toco/toco_port.h b/tensorflow/lite/toco/toco_port.h
index 2f39e3d6d5..231612ecd4 100644
--- a/tensorflow/lite/toco/toco_port.h
+++ b/tensorflow/lite/toco/toco_port.h
@@ -55,6 +55,10 @@ double round(double x);
 namespace toco {
 namespace port {
 
+// Things like tests use other initialization routines that need control
+// of flags. However, for testing we still want to use toco_port.h facilities.
+// This function sets initialized flag trivially.
+void InitGoogleWasDoneElsewhere();
 void InitGoogle(const char* usage, int* argc, char*** argv, bool remove_flags);
 void CheckInitGoogleIsDone(const char* message);
 
diff --git a/tensorflow/lite/toco/toco_port_test.cc b/tensorflow/lite/toco/toco_port_test.cc
index d80d423ed7..997da58b8f 100644
--- a/tensorflow/lite/toco/toco_port_test.cc
+++ b/tensorflow/lite/toco/toco_port_test.cc
@@ -61,5 +61,6 @@ TEST(TocoPortTest, JoinPath) {
 int main(int argc, char** argv) {
   ::tflite::LogToStderr();
   ::testing::InitGoogleTest(&argc, argv);
+  ::toco::port::InitGoogleWasDoneElsewhere();
   return RUN_ALL_TESTS();
 }
diff --git a/tensorflow/lite/toco/tooling_util_test.cc b/tensorflow/lite/toco/tooling_util_test.cc
index e44b94b712..faa6fe412e 100644
--- a/tensorflow/lite/toco/tooling_util_test.cc
+++ b/tensorflow/lite/toco/tooling_util_test.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/lite/testing/util.h"
 #include "tensorflow/lite/toco/model.h"
+#include "tensorflow/lite/toco/toco_port.h"
 #include "tensorflow/lite/toco/tooling_util.h"
 
 namespace toco {
@@ -208,5 +209,6 @@ TEST(FusedActivationTest, DefaultsToUnfused) {
 int main(int argc, char** argv) {
   ::tflite::LogToStderr();
   ::testing::InitGoogleTest(&argc, argv);
+  ::toco::port::InitGoogleWasDoneElsewhere();
   return RUN_ALL_TESTS();
 }
-- 
GitLab


From c503c6254608a622b06340f1f7ddd8f31565ab5e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 04:37:51 -0800
Subject: [PATCH 0333/2345] Small clean up in in saved_model/load_test.py

Temporary `function` in checkpointable under test is not needed in most examples
as `cycle` is passing signatures explicitly.

PiperOrigin-RevId: 228309220
---
 tensorflow/python/saved_model/load_test.py | 34 +++++++++-------------
 1 file changed, 14 insertions(+), 20 deletions(-)

diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index 9eac3e6555..a1c81d3f14 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -42,9 +42,6 @@ class LoadTest(test.TestCase):
 
   def test_structure_import(self):
     root = tracking.Checkpointable()
-    root.f = def_function.function(
-        lambda x: 2. * x,
-        input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
     root.dep_one = tracking.Checkpointable()
     root.dep_two = tracking.Checkpointable()
     root.dep_two.dep = tracking.Checkpointable()
@@ -52,19 +49,25 @@ class LoadTest(test.TestCase):
     imported = self.cycle(root)
     self.assertIs(imported.dep_three, imported.dep_two.dep)
     self.assertIsNot(imported.dep_one, imported.dep_two)
-    self.assertEqual(4., imported.f(constant_op.constant(2.)).numpy())
 
   def test_variables(self):
     root = tracking.Checkpointable()
     root.v1 = variables.Variable(1.)
     root.v2 = variables.Variable(2.)
-    root.f = def_function.function(
-        lambda x: root.v2 * x,
-        input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
     imported = self.cycle(root)
     self.assertEquals(imported.v1.numpy(), 1.0)
     self.assertEquals(imported.v2.numpy(), 2.0)
+
+  def test_capture_variables(self):
+    root = tracking.Checkpointable()
+    root.weights = variables.Variable(2.)
+    root.f = def_function.function(
+        lambda x: root.weights * x,
+        input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
+    imported = self.cycle(root)
     self.assertEqual(4., imported.f(constant_op.constant(2.)).numpy())
+    imported.weights.assign(4.0)
+    self.assertEqual(8., imported.f(constant_op.constant(2.)).numpy())
 
   def _make_asset(self, contents):
     filename = tempfile.mktemp(prefix=self.get_temp_dir())
@@ -72,19 +75,16 @@ class LoadTest(test.TestCase):
       f.write(contents)
     return filename
 
-  def test_assets_import(self):
+  def test_assets(self):
     file1 = self._make_asset("contents 1")
     file2 = self._make_asset("contents 2")
 
     root = tracking.Checkpointable()
-    root.f = def_function.function(
-        lambda x: 2. * x,
-        input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
     root.asset1 = tracking.TrackableAsset(file1)
     root.asset2 = tracking.TrackableAsset(file2)
 
     save_dir = os.path.join(self.get_temp_dir(), "save_dir")
-    save.save(root, save_dir)
+    save.save(root, save_dir, signatures={})
 
     file_io.delete_file(file1)
     file_io.delete_file(file2)
@@ -110,18 +110,12 @@ class LoadTest(test.TestCase):
     with open(imported_output, "r") as f:
       self.assertEquals("contents", f.read())
 
-  def test_assets_dedup(self):
+  def test_dedup_assets(self):
     vocab = self._make_asset("contents")
     root = tracking.Checkpointable()
-    root.f = def_function.function(
-        lambda x: 2. * x,
-        input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
-
     root.asset1 = tracking.TrackableAsset(vocab)
     root.asset2 = tracking.TrackableAsset(vocab)
-
     imported = self.cycle(root)
-
     self.assertEqual(imported.asset1.asset_path.numpy(),
                      imported.asset2.asset_path.numpy())
 
@@ -154,7 +148,7 @@ class LoadTest(test.TestCase):
     imported = self.cycle(root)
     self.assertEqual(4., imported.f(constant_op.constant(2.0)).numpy())
 
-  def test_nested_func(self):
+  def test_nested_functions(self):
     f = def_function.function(
         lambda x: x*2.0,
         input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
-- 
GitLab


From 114f0d81404e7f9a93297f11ebc2fbceee8dbf7c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 05:20:40 -0800
Subject: [PATCH 0334/2345] Add fine-tune capabilities to object based saved
 model save/load.

- Keep "trainable" bit of variables.
- Wire variables in restored FuncGraphs so they are automaticaly
  captured by GradientTapes.

PiperOrigin-RevId: 228311974
---
 tensorflow/python/eager/function.py           |  2 +-
 tensorflow/python/saved_model/load.py         |  8 +++++-
 tensorflow/python/saved_model/load_test.py    | 25 +++++++++++++++++--
 tensorflow/python/saved_model/save.py         |  1 +
 .../saved_model/saved_object_graph.proto      |  3 ++-
 5 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 58d1f6b886..05fccfbcd9 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -105,7 +105,7 @@ def _parse_func_attrs(attributes):
       attrs[key] = attr_value_pb2.AttrValue(i=value)
     elif isinstance(value, float):
       attrs[key] = attr_value_pb2.AttrValue(f=value)
-    elif isinstance(value, (str, bytes)):
+    elif isinstance(value, (str, bytes, six.text_type)):
       attrs[key] = attr_value_pb2.AttrValue(s=compat.as_bytes(value))
     else:
       raise ValueError("Unsupported attribute type for %s with type %s" %
diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
index 3e85fade50..1ee1b69b03 100644
--- a/tensorflow/python/saved_model/load.py
+++ b/tensorflow/python/saved_model/load.py
@@ -58,6 +58,11 @@ class _Loader(object):
           bound_inputs = [
               self._get_tensor_from_node(node_id)
               for node_id in monomorphic_function.bound_inputs]
+          bound_variables = [
+              self._nodes[node_id]
+              for node_id in monomorphic_function.bound_inputs
+              if self._proto.nodes[node_id].WhichOneof("kind") == "variable"
+          ]
           if name in seen_functions:
             if self._functions[name]._captured_inputs != bound_inputs:  # pylint: disable=protected-access
               raise NotImplementedError(
@@ -69,6 +74,7 @@ class _Loader(object):
             # concrete function, note that we did not modify the FuncGraph
             # itself.
             self._functions[name]._captured_inputs = bound_inputs  # pylint: disable=protected-access
+            self._functions[name]._func_graph.variables = bound_variables  # pylint: disable=protected-access
 
   def _get_tensor_from_node(self, node_id):
     obj = self._nodes[node_id]
@@ -123,7 +129,7 @@ class _Loader(object):
   def _recreate_variable(self, proto):
     # TODO(andresp): Can we use the checkpointed value as initializer?
     dummy_value = init_ops.Zeros(dtype=proto.dtype)(shape=proto.shape)
-    return variables.Variable(dummy_value)
+    return variables.Variable(dummy_value, trainable=proto.trainable)
 
 
 def _load_saved_object_graph_proto(filename):
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index a1c81d3f14..35e384dbfc 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import os
 import tempfile
 
+from tensorflow.python.eager import backprop
 from tensorflow.python.eager import def_function
 from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
@@ -52,11 +53,13 @@ class LoadTest(test.TestCase):
 
   def test_variables(self):
     root = tracking.Checkpointable()
-    root.v1 = variables.Variable(1.)
-    root.v2 = variables.Variable(2.)
+    root.v1 = variables.Variable(1., trainable=True)
+    root.v2 = variables.Variable(2., trainable=False)
     imported = self.cycle(root)
     self.assertEquals(imported.v1.numpy(), 1.0)
+    self.assertTrue(imported.v1.trainable)
     self.assertEquals(imported.v2.numpy(), 2.0)
+    self.assertFalse(imported.v2.trainable)
 
   def test_capture_variables(self):
     root = tracking.Checkpointable()
@@ -248,6 +251,24 @@ class LoadTest(test.TestCase):
     self.cycle(m)
     self.assertEquals(4.0, m.f(constant_op.constant(2.0)).numpy())
 
+  def test_basic_backprop(self):
+    weight = variables.Variable(1., trainable=True)
+    bias = variables.Variable(0., trainable=True)
+    g = def_function.function(
+        lambda x: x*weight + bias,
+        input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
+
+    root = tracking.Checkpointable()
+    root.weight = weight
+    root.bias = bias
+    root.g = g
+    imported = self.cycle(root)
+    with backprop.GradientTape(watch_accessed_variables=True) as t:
+      x = constant_op.constant([3.5])
+      loss = imported.g(x)
+      grad = t.gradient(loss, [imported.weight, imported.bias])
+      self.assertAllClose(grad, [3.5, 1.0])
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index a76b370565..d79b5fe3a6 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -610,6 +610,7 @@ def _write_object_proto(obj, proto, asset_file_def_index, node_ids):
     proto.asset.asset_file_def_index = asset_file_def_index[obj]
   elif resource_variable_ops.is_resource_variable(obj):
     proto.variable.SetInParent()
+    proto.variable.trainable = obj.trainable
     proto.variable.dtype = obj.dtype.as_datatype_enum
     proto.variable.shape.CopyFrom(obj.shape.as_proto())
   elif isinstance(obj, def_function.PolymorphicFunction):
diff --git a/tensorflow/python/saved_model/saved_object_graph.proto b/tensorflow/python/saved_model/saved_object_graph.proto
index f46927d6e8..1e2514b7f7 100644
--- a/tensorflow/python/saved_model/saved_object_graph.proto
+++ b/tensorflow/python/saved_model/saved_object_graph.proto
@@ -104,6 +104,7 @@ message SavedMonomorphicFunction {
 message SavedVariable {
   DataType dtype = 1;
   TensorShapeProto shape = 2;
+  bool trainable = 3;
 
-  // TODO(andresp): Add "trainable" and save_slice_info_def.
+  // TODO(andresp): Add save_slice_info_def?
 }
-- 
GitLab


From 0ad1d56cd974c1feef19ebf81b202b9bd4066247 Mon Sep 17 00:00:00 2001
From: Grzegorz Pawelczak <grzegorzp@graphcore.ai>
Date: Tue, 8 Jan 2019 13:44:32 +0000
Subject: [PATCH 0335/2345] Make RemoveUnusedOperandFromSort a seperate pass

---
 tensorflow/compiler/xla/service/BUILD         |  32 +++++
 .../xla/service/algebraic_simplifier.cc       |  75 -----------
 .../xla/service/algebraic_simplifier_test.cc  |  68 ----------
 .../compiler/xla/service/sort_simplifier.cc   | 125 ++++++++++++++++++
 .../compiler/xla/service/sort_simplifier.h    |  35 +++++
 .../xla/service/sort_simplifier_test.cc       | 102 ++++++++++++++
 6 files changed, 294 insertions(+), 143 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/sort_simplifier.cc
 create mode 100644 tensorflow/compiler/xla/service/sort_simplifier.h
 create mode 100644 tensorflow/compiler/xla/service/sort_simplifier_test.cc

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index ec63ae1ac6..7be180c624 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -3462,6 +3462,38 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "sort_simplifier",
+    srcs = ["sort_simplifier.cc"],
+    hdrs = ["sort_simplifier.h"],
+    deps = [
+        ":hlo",
+        ":hlo_pass",
+        ":while_util",
+        "//tensorflow/compiler/xla:statusor",
+        "//tensorflow/compiler/xla:util",
+        "//tensorflow/core:lib",
+        "@com_google_absl//absl/algorithm:container",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/container:inlined_vector",
+    ],
+)
+
+tf_cc_test(
+    name = "sort_simplifier_test",
+    srcs = ["sort_simplifier_test.cc"],
+    deps = [
+        ":hlo_matchers",
+        ":sort_simplifier",
+        ":pattern_matcher",
+        ":pattern_matcher_gmock",
+        "//tensorflow/compiler/xla:test",
+        "//tensorflow/compiler/xla/service:hlo_parser",
+        "//tensorflow/compiler/xla/tests:hlo_test_base",
+        "//tensorflow/core:test",
+    ],
+)
+
 cc_library(
     name = "tuple_util",
     srcs = ["tuple_util.cc"],
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index 5ac746c9f3..b76f2febc7 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -371,11 +371,6 @@ class AlgebraicSimplifierVisitor : public DfsHloVisitorWithDefault {
   // Tries to convert slice(reshape(X)) into reshape(slice(X))
   StatusOr<bool> TryToReorderSliceAndReshape(HloInstruction* slice);
 
-  // If the sort instruction has a tuple shape then looks for unused output
-  // values and removes them from the sort instruction. Returns true if the
-  // graph have been modified.
-  StatusOr<bool> RemoveUnusedOperandFromSort(HloInstruction* sort);
-
   // Current HloComputation instance the AlgebraicSimplifierVisitor is
   // traversing.
   HloComputation* computation_;
@@ -2819,69 +2814,6 @@ Status AlgebraicSimplifierVisitor::HandleSelect(HloInstruction* select) {
   return Status::OK();
 }
 
-StatusOr<bool> AlgebraicSimplifierVisitor::RemoveUnusedOperandFromSort(
-    HloInstruction* sort) {
-  if (!sort->shape().IsTuple()) {
-    return false;
-  }
-
-  if (sort->parent()->root_instruction() == sort) {
-    // Can't analyse users of the root instruction.
-    return false;
-  }
-
-  // Index 0 is the sorting key used by the sort HLO itself.
-  absl::flat_hash_set<int64> used_indices{0};
-  for (const HloInstruction* user : sort->users()) {
-    if (user->opcode() != HloOpcode::kGetTupleElement) {
-      // Can't analyse users other then get-tuple-element.
-      return false;
-    }
-    used_indices.insert(user->tuple_index());
-  }
-
-  if (used_indices.size() == sort->operand_count()) {
-    // All operands are used.
-    return false;
-  }
-
-  std::vector<HloInstruction*> operands{sort->mutable_operand(0)};
-  std::vector<Shape> new_shapes{sort->operand(0)->shape()};
-  for (int64 i = 1; i < sort->operand_count(); ++i) {
-    if (used_indices.count(i)) {
-      operands.push_back(sort->mutable_operand(i));
-      new_shapes.push_back(sort->operand(i)->shape());
-    }
-  }
-  Shape new_sort_shape = new_shapes.size() == 1
-                             ? new_shapes[0]
-                             : ShapeUtil::MakeTupleShape(new_shapes);
-  HloInstruction* new_sort = computation_->AddInstruction(
-      sort->CloneWithNewOperands(new_sort_shape, operands));
-
-  // Map from original get-tuple-element tuple index to new HLO instruction
-  absl::flat_hash_map<int64, HloInstruction*> result_map;
-  if (new_sort->shape().IsTuple()) {
-    // Old sort key maps to new sort key.
-    int64 new_index = 0;
-    for (int64 i = 0; i < sort->operand_count(); ++i) {
-      if (used_indices.count(i)) {
-        result_map[i] =
-            computation_->AddInstruction(HloInstruction::CreateGetTupleElement(
-                new_shapes[new_index], new_sort, new_index));
-        ++new_index;
-      }
-    }
-  } else {
-    result_map[0] = new_sort;
-  }
-  for (HloInstruction* user : sort->users()) {
-    TF_RETURN_IF_ERROR(
-        user->ReplaceAllUsesWith(result_map.at(user->tuple_index())));
-  }
-  return true;
-}
-
 Status AlgebraicSimplifierVisitor::HandleSort(HloInstruction* sort) {
   auto operand = sort->mutable_operand(0);
   int64 dimension_to_sort = sort->dimensions(0);
@@ -2895,13 +2827,6 @@ Status AlgebraicSimplifierVisitor::HandleSort(HloInstruction* sort) {
         sort, HloInstruction::CreateTuple(sort->operands()));
   }
 
-  // Remove the unused values from a key-value sort.
-  TF_ASSIGN_OR_RETURN(bool removed_operand, RemoveUnusedOperandFromSort(sort));
-  if (removed_operand) {
-    changed_ = true;
-    return Status::OK();
-  }
-
   if (!options_.enable_permutation_sort_replacement()) {
     return Status::OK();
   }
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index e39321f1f3..a2bbc4861f 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -2709,74 +2709,6 @@ TEST_F(AlgebraicSimplifierTest, DontReplacePermutationSortWrongDimensions) {
   EXPECT_FALSE(simplifier.Run(module.get()).ValueOrDie());
 }
 
-TEST_F(AlgebraicSimplifierTest, RemoveUnusedSortOperandArrayResult) {
-  const char* hlo_string = R"(
-   HloModule permutation_sort
-
-    ENTRY sort_computation {
-      keys = f32[64,8732]{1,0} parameter(0)
-      values = s32[64,8732]{1,0} parameter(1)
-      sort = (f32[64,8732]{1,0}, s32[64,8732]{1,0}) sort(keys, values),
-        dimensions={1}
-      ROOT gte = f32[64,8732]{1,0} get-tuple-element(sort), index=0
-    })";
-  TF_ASSERT_OK_AND_ASSIGN(auto module,
-                          ParseAndReturnVerifiedModule(hlo_string));
-
-  AlgebraicSimplifierOptions options(bitcasting_callback());
-  AlgebraicSimplifier simplifier(options);
-  EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
-  auto root = module->entry_computation()->root_instruction();
-  EXPECT_THAT(root, GmockMatch(m::Sort(m::Parameter(0))));
-}
-
-TEST_F(AlgebraicSimplifierTest, RemoveUnusedSortOperandTuple) {
-  const char* hlo_string = R"(
-   HloModule permutation_sort
-
-    ENTRY sort_computation {
-      keys = f32[64,87] parameter(0)
-      values.0 = s32[64,87] parameter(1)
-      values.1 = u32[64,87] parameter(2)
-      sort = (f32[64,87], s32[64,87], u32[64,87]) sort(
-          keys, values.0, values.1),
-        dimensions={1}
-      gte.0 = f32[64,87] get-tuple-element(sort), index=0
-      gte.1 = u32[64,87] get-tuple-element(sort), index=2
-      ROOT tuple = (f32[64,87], u32[64,87]) tuple(gte.0, gte.1)
-    })";
-  TF_ASSERT_OK_AND_ASSIGN(auto module,
-                          ParseAndReturnVerifiedModule(hlo_string));
-
-  AlgebraicSimplifierOptions options(bitcasting_callback());
-  AlgebraicSimplifier simplifier(options);
-  EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
-  auto root = module->entry_computation()->root_instruction();
-  EXPECT_THAT(
-      root,
-      GmockMatch(m::Tuple(
-          m::GetTupleElement(m::Sort(m::Parameter(0), m::Parameter(2)), 0),
-          m::GetTupleElement(m::Sort(m::Parameter(0), m::Parameter(2)), 1))));
-}
-
-TEST_F(AlgebraicSimplifierTest, DontRemoveUnusedSortKey) {
-  const char* hlo_string = R"(
-   HloModule permutation_sort
-
-    ENTRY sort_computation {
-      keys = f32[64,8732]{1,0} parameter(0)
-      values = s32[64,8732]{1,0} parameter(1)
-      sort = (f32[64,8732]{1,0}, s32[64,8732]{1,0}) sort(keys, values), dimensions={1}
-      ROOT gte = s32[64,8732]{1,0} get-tuple-element(sort), index=1
-    })";
-  TF_ASSERT_OK_AND_ASSIGN(auto module,
-                          ParseAndReturnVerifiedModule(hlo_string));
-
-  AlgebraicSimplifierOptions options(bitcasting_callback());
-  AlgebraicSimplifier simplifier(options);
-  EXPECT_FALSE(simplifier.Run(module.get()).ValueOrDie());
-}
-
 TEST_F(AlgebraicSimplifierTest, ReplaceEffectiveScalarKeyValueSortWithTuple) {
   auto builder = HloComputation::Builder(TestName());
 
diff --git a/tensorflow/compiler/xla/service/sort_simplifier.cc b/tensorflow/compiler/xla/service/sort_simplifier.cc
new file mode 100644
index 0000000000..3280672574
--- /dev/null
+++ b/tensorflow/compiler/xla/service/sort_simplifier.cc
@@ -0,0 +1,125 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/sort_simplifier.h"
+#include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/statusor.h"
+
+#include "absl/container/flat_hash_map.h"
+#include "absl/container/flat_hash_set.h"
+
+namespace xla {
+namespace {
+
+// If the sort instruction has a tuple shape then looks for unused output
+// values and removes them from the sort instruction. Returns true if the
+// graph have been modified.
+StatusOr<bool> RemoveUnusedOperandFromSort(HloInstruction* sort) {
+  if (!sort->shape().IsTuple()) {
+    return false;
+  }
+
+  if (sort->parent()->root_instruction() == sort) {
+    // Can't analyse users of the root instruction.
+    return false;
+  }
+
+  // Index 0 is the sorting key used by the sort HLO itself.
+  absl::flat_hash_set<int64> used_indices{0};
+  for (const HloInstruction* user : sort->users()) {
+    if (user->opcode() != HloOpcode::kGetTupleElement) {
+      // Can't analyse users other then get-tuple-element.
+      return false;
+    }
+    used_indices.insert(user->tuple_index());
+  }
+
+  if (used_indices.size() == sort->operand_count()) {
+    // All operands are used.
+    return false;
+  }
+
+  std::vector<HloInstruction*> operands{sort->mutable_operand(0)};
+  std::vector<Shape> new_shapes{sort->operand(0)->shape()};
+  for (int64 i = 1; i < sort->operand_count(); ++i) {
+    if (used_indices.count(i)) {
+      operands.push_back(sort->mutable_operand(i));
+      new_shapes.push_back(sort->operand(i)->shape());
+    }
+  }
+  HloComputation* computation = sort->parent();
+
+  Shape new_sort_shape = new_shapes.size() == 1
+                             ? new_shapes[0]
+                             : ShapeUtil::MakeTupleShape(new_shapes);
+  HloInstruction* new_sort = computation->AddInstruction(
+      sort->CloneWithNewOperands(new_sort_shape, operands));
+
+  // Map from original get-tuple-element tuple index to new HLO instruction
+  absl::flat_hash_map<int64, HloInstruction*> result_map;
+  if (new_sort->shape().IsTuple()) {
+    // Old sort key maps to new sort key.
+    int64 new_index = 0;
+    for (int64 i = 0; i < sort->operand_count(); ++i) {
+      if (used_indices.count(i)) {
+        result_map[i] =
+            computation->AddInstruction(HloInstruction::CreateGetTupleElement(
+                new_shapes[new_index], new_sort, new_index));
+        ++new_index;
+      }
+    }
+  } else {
+    result_map[0] = new_sort;
+  }
+  for (HloInstruction* user : sort->users()) {
+    TF_RETURN_IF_ERROR(
+        user->ReplaceAllUsesWith(result_map.at(user->tuple_index())));
+    TF_RETURN_IF_ERROR(computation->RemoveInstructionAndUnusedOperands(user));
+  }
+  return true;
+}
+
+} // namespace
+
+StatusOr<bool> SortSimplifier::Run(HloModule* module) {
+  VLOG(2) << "HLO module before SortSimplifier:";
+  XLA_VLOG_LINES(2, module->ToString());
+
+  bool changed = false;
+  std::vector<HloInstruction*> sort_instrs;
+  for (auto* comp : module->MakeNonfusionComputations()) {
+    absl::c_copy_if(comp->instructions(), std::back_inserter(sort_instrs),
+                    [](const HloInstruction* instr) {
+                      return instr->opcode() == HloOpcode::kSort;
+                    });
+  }
+
+  for (HloInstruction* sort_instr : sort_instrs) {
+    TF_ASSIGN_OR_RETURN(bool result,
+                        RemoveUnusedOperandFromSort(sort_instr));
+    changed |= result;
+  }
+
+  if (changed) {
+    VLOG(2) << "HLO module after SortSimplifier:";
+    XLA_VLOG_LINES(2, module->ToString());
+  } else {
+    VLOG(2) << "HLO module unchanged after SortSimplifier";
+  }
+
+  return changed;
+}
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/sort_simplifier.h b/tensorflow/compiler/xla/service/sort_simplifier.h
new file mode 100644
index 0000000000..8c6f313aa0
--- /dev/null
+++ b/tensorflow/compiler/xla/service/sort_simplifier.h
@@ -0,0 +1,35 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_SORT_SIMPLIFIER_H_
+#define TENSORFLOW_COMPILER_XLA_SERVICE_SORT_SIMPLIFIER_H_
+
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/service/hlo_pass_interface.h"
+#include "tensorflow/compiler/xla/statusor.h"
+
+namespace xla {
+
+// HLO pass which removes unused operands from sort, where an unused operand is
+// defined as an operand at some index 'x' at which the output is not used.
+class SortSimplifier : public HloModulePass {
+ public:
+  absl::string_view name() const override { return "simplify-sorts"; }
+  StatusOr<bool> Run(HloModule* module) override;
+};
+
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_SORT_SIMPLIFIER_H_
diff --git a/tensorflow/compiler/xla/service/sort_simplifier_test.cc b/tensorflow/compiler/xla/service/sort_simplifier_test.cc
new file mode 100644
index 0000000000..5a81e537de
--- /dev/null
+++ b/tensorflow/compiler/xla/service/sort_simplifier_test.cc
@@ -0,0 +1,102 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/sort_simplifier.h"
+
+#include "tensorflow/compiler/xla/service/hlo_matchers.h"
+#include "tensorflow/compiler/xla/service/hlo_parser.h"
+#include "tensorflow/compiler/xla/service/pattern_matcher.h"
+#include "tensorflow/compiler/xla/service/pattern_matcher_gmock.h"
+#include "tensorflow/compiler/xla/test.h"
+#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+
+namespace xla {
+namespace {
+
+namespace m = match;
+
+using SortSimplifierTest = HloTestBase;
+
+TEST_F(SortSimplifierTest, RemoveUnusedSortOperandArrayResult) {
+  const char* hlo_string = R"(
+   HloModule permutation_sort
+
+    ENTRY sort_computation {
+      keys = f32[64,8732]{1,0} parameter(0)
+      values = s32[64,8732]{1,0} parameter(1)
+      sort = (f32[64,8732]{1,0}, s32[64,8732]{1,0}) sort(keys, values),
+        dimensions={1}
+      ROOT gte = f32[64,8732]{1,0} get-tuple-element(sort), index=0
+    })";
+  TF_ASSERT_OK_AND_ASSIGN(auto module,
+                          ParseAndReturnVerifiedModule(hlo_string));
+
+  SortSimplifier simplifier;
+  uint64 num_executions = 0;
+  do {
+    num_executions++;
+  } while (simplifier.Run(module.get()).ValueOrDie());
+  EXPECT_TRUE(num_executions == 2);
+  auto root = module->entry_computation()->root_instruction();
+  EXPECT_THAT(root, GmockMatch(m::Sort(m::Parameter(0))));
+}
+
+TEST_F(SortSimplifierTest, RemoveUnusedSortOperandTuple) {
+  const char* hlo_string = R"(
+   HloModule permutation_sort
+
+    ENTRY sort_computation {
+      keys = f32[64,87] parameter(0)
+      values.0 = s32[64,87] parameter(1)
+      values.1 = u32[64,87] parameter(2)
+      sort = (f32[64,87], s32[64,87], u32[64,87]) sort(
+          keys, values.0, values.1),
+        dimensions={1}
+      gte.0 = f32[64,87] get-tuple-element(sort), index=0
+      gte.1 = u32[64,87] get-tuple-element(sort), index=2
+      ROOT tuple = (f32[64,87], u32[64,87]) tuple(gte.0, gte.1)
+    })";
+  TF_ASSERT_OK_AND_ASSIGN(auto module,
+                          ParseAndReturnVerifiedModule(hlo_string));
+
+  SortSimplifier simplifier;
+  EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
+  auto root = module->entry_computation()->root_instruction();
+  EXPECT_THAT(
+      root,
+      GmockMatch(m::Tuple(
+          m::GetTupleElement(m::Sort(m::Parameter(0), m::Parameter(2)), 0),
+          m::GetTupleElement(m::Sort(m::Parameter(0), m::Parameter(2)), 1))));
+}
+
+TEST_F(SortSimplifierTest, DontRemoveUnusedSortKey) {
+  const char* hlo_string = R"(
+   HloModule permutation_sort
+
+    ENTRY sort_computation {
+      keys = f32[64,8732]{1,0} parameter(0)
+      values = s32[64,8732]{1,0} parameter(1)
+      sort = (f32[64,8732]{1,0}, s32[64,8732]{1,0}) sort(keys, values), dimensions={1}
+      ROOT gte = s32[64,8732]{1,0} get-tuple-element(sort), index=1
+    })";
+  TF_ASSERT_OK_AND_ASSIGN(auto module,
+                          ParseAndReturnVerifiedModule(hlo_string));
+
+  SortSimplifier simplifier;
+  EXPECT_FALSE(simplifier.Run(module.get()).ValueOrDie());
+}
+}  // namespace
+}  // namespace xla
-- 
GitLab


From 0f9478c3c2a71f26fd26083b2e2ed76fb69d1030 Mon Sep 17 00:00:00 2001
From: ktaebum <phya.ktaebum@gmail.com>
Date: Tue, 8 Jan 2019 23:19:38 +0900
Subject: [PATCH 0336/2345] resolved incompatible dimension reshape before feed
 into final dense layer

---
 tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py
index 15776c694e..9b5a2c947b 100644
--- a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py
+++ b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py
@@ -128,7 +128,7 @@ class PTBModel(tf.keras.Model):
 
     self.linear = layers.Dense(
         vocab_size, kernel_initializer=tf.random_uniform_initializer(-0.1, 0.1))
-    self._output_shape = [-1, embedding_dim]
+    self._output_shape = [-1, hidden_dim]
 
   def call(self, input_seq, training):
     """Run the forward pass of PTBModel.
-- 
GitLab


From 7fa90cddf07c6aac532306dac15e02216a83b9fa Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Tue, 8 Jan 2019 07:29:42 -0800
Subject: [PATCH 0337/2345] [eager]: Disable benchmark that is currently timing
 out.

This timeout seems to have started after
https://github.com/tensorflow/tensorflow/commit/0445684a64d1bea8490a99eb9ce278176133df75

PiperOrigin-RevId: 228326334
---
 tensorflow/python/eager/benchmarks_test.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/python/eager/benchmarks_test.py b/tensorflow/python/eager/benchmarks_test.py
index 31a7efca82..9191c8e689 100644
--- a/tensorflow/python/eager/benchmarks_test.py
+++ b/tensorflow/python/eager/benchmarks_test.py
@@ -867,6 +867,10 @@ class MicroBenchmarks(test.Benchmark):
     self._run(scan, 100)
 
   def benchmarkScanDefun(self):
+    if context.num_gpus():
+      # TODO(b/122081934): Re-enable this after figuring out why this became
+      # really slow with control flow V2
+      return
     elems = math_ops.range(1600)
 
     @function.defun
-- 
GitLab


From 107b7c4de6727ad02a0b4fc24aa1ac17bd697776 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Tue, 8 Jan 2019 11:37:37 -0500
Subject: [PATCH 0338/2345] Fixes for rnn_keras_estimator.ipynb

* wrapping `range(batch_size)` into a `list` call so that it's convertible to `Tensor` in Python 3, where `range` alone returns just a generator
* specifying a `dtype` argument to `LSTMBlockCell`'s constructor

These should fix the breakages in #24701
---
 .../autograph/examples/notebooks/rnn_keras_estimator.ipynb  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/autograph/examples/notebooks/rnn_keras_estimator.ipynb b/tensorflow/contrib/autograph/examples/notebooks/rnn_keras_estimator.ipynb
index 44532cb078..831c613f2c 100644
--- a/tensorflow/contrib/autograph/examples/notebooks/rnn_keras_estimator.ipynb
+++ b/tensorflow/contrib/autograph/examples/notebooks/rnn_keras_estimator.ipynb
@@ -186,8 +186,8 @@
         "\n",
         "  def __init__(self):\n",
         "    super(RnnColorbot, self).__init__()\n",
-        "    self.lower_cell = tf.contrib.rnn.LSTMBlockCell(256)\n",
-        "    self.upper_cell = tf.contrib.rnn.LSTMBlockCell(128)\n",
+        "    self.lower_cell = tf.contrib.rnn.LSTMBlockCell(256, dtype=tf.float32)\n",
+        "    self.upper_cell = tf.contrib.rnn.LSTMBlockCell(128, dtype=tf.float32)\n",
         "    self.relu_layer = tf.layers.Dense(3, activation=tf.nn.relu)\n",
         "\n",
         "  def _rnn_layer(self, chars, cell, batch_size, training):\n",
@@ -241,7 +241,7 @@
         "    seq = self._rnn_layer(seq, self.upper_cell, batch_size, training)\n",
         "\n",
         "    # Grab just the end-of-sequence from each output.\n",
-        "    indices = (length - 1, range(batch_size))\n",
+        "    indices = (length - 1, list(range(batch_size)))\n",
         "    indices = tf.stack(indices, 1)\n",
         "    sequence_ends = tf.gather_nd(seq, indices)\n",
         "    return self.relu_layer(sequence_ends)\n",
-- 
GitLab


From 02d4333dfc5a9fe2c3929ba6288e9a679725da85 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Tue, 8 Jan 2019 09:18:09 -0800
Subject: [PATCH 0339/2345] Give bytes constructor the appropriate utf-8
 argument.

PiperOrigin-RevId: 228342070
---
 tensorflow/tools/git/gen_git_source.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/git/gen_git_source.py b/tensorflow/tools/git/gen_git_source.py
index 0c37508650..c2449da923 100755
--- a/tensorflow/tools/git/gen_git_source.py
+++ b/tensorflow/tools/git/gen_git_source.py
@@ -29,8 +29,8 @@ from __future__ import print_function
 import argparse
 import json
 import os
-import subprocess
 import shutil
+import subprocess
 
 
 def parse_branch_ref(filename):
@@ -175,7 +175,7 @@ def get_git_version(git_base_path, git_tag_override):
       # two "-" are those inserted by the git describe command.
       abbrev_commit = split_val[-1]
       val = version_separator.join(
-          [bytes(git_tag_override), b"0", abbrev_commit])
+          [bytes(git_tag_override, "utf-8"), b"0", abbrev_commit])
     return val if val else unknown_label
   except (subprocess.CalledProcessError, OSError):
     return unknown_label
-- 
GitLab


From 3ba26026545be700c4aed9bc609b25fad9c1531e Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Tue, 8 Jan 2019 09:25:59 -0800
Subject: [PATCH 0340/2345] Removed ResourceVariable._cached_shape_as_list

_shape_as_list is no longer on the hotpath of EagerLinearRegressionBenchmark,
the change does not seem to affect throughput estimates. Note that both
estimates have high variance.

Before:

    entry {
      name: "EagerLinearRegressionBenchmark.eager_train_cpu"
      iters: 2000
      wall_time: 1.24479007721
      extras {
        key: "examples_per_sec"
        value {
          double_value: 102828.583183
        }
      }
    }

After:

    entry {
      name: "EagerLinearRegressionBenchmark.eager_train_cpu"
      iters: 2000
      wall_time: 1.25307798386
      extras {
        key: "examples_per_sec"
        value {
          double_value: 102148.470924
        }
      }
    }

PiperOrigin-RevId: 228343127
---
 tensorflow/python/ops/resource_variable_ops.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 1d98fb2c89..d27e38c735 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -504,7 +504,6 @@ class ResourceVariable(variables.VariableV1):
       # all in graph mode.
       self._handle_deleter = EagerResourceDeleter(
           handle=self._handle, handle_device=self._handle.device)
-    self._cached_shape_as_list = None
 
   def _init_from_proto(self, variable_def, import_scope=None):
     """Initializes from `VariableDef` proto."""
@@ -562,7 +561,6 @@ class ResourceVariable(variables.VariableV1):
     self._caching_device = None
     self._dtype = dtypes.as_dtype(self._handle.op.get_attr("dtype"))
     self._constraint = None
-    self._cached_shape_as_list = None
 
   @contextlib.contextmanager
   def _assign_dependencies(self):
@@ -632,12 +630,9 @@ class ResourceVariable(variables.VariableV1):
     return self._distribute_strategy
 
   def _shape_as_list(self):
-    if self._cached_shape_as_list:
-      return self._cached_shape_as_list
     if self.shape.ndims is None:
       return None
-    self._cached_shape_as_list = [dim.value for dim in self.shape.dims]
-    return self._cached_shape_as_list
+    return [dim.value for dim in self.shape.dims]
 
   def _shape_tuple(self):
     shape = self._shape_as_list()
-- 
GitLab


From aa5e17a2dc94ba47d41a74021d6701a8c10de4a5 Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Tue, 8 Jan 2019 09:37:02 -0800
Subject: [PATCH 0341/2345] Fix linkage for several lite/testing tests

Use the tf_cc_test rule for targets that have TensorFlow deps.

PiperOrigin-RevId: 228344897
---
 tensorflow/lite/testing/BUILD | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/testing/BUILD b/tensorflow/lite/testing/BUILD
index fa25cfaa69..19c950b4f8 100644
--- a/tensorflow/lite/testing/BUILD
+++ b/tensorflow/lite/testing/BUILD
@@ -257,7 +257,7 @@ cc_library(
     ],
 )
 
-cc_test(
+tf_cc_test(
     name = "tf_driver_test",
     size = "small",
     srcs = ["tf_driver_test.cc"],
@@ -286,7 +286,7 @@ cc_library(
     ],
 )
 
-cc_test(
+tf_cc_test(
     name = "generate_testspec_test",
     size = "small",
     srcs = ["generate_testspec_test.cc"],
-- 
GitLab


From f0fba8b59b3e3fa6b068c0c2715104153983c554 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Tue, 8 Jan 2019 09:54:11 -0800
Subject: [PATCH 0342/2345] Use correct _t integer types to fix mac build.

PiperOrigin-RevId: 228347592
---
 tensorflow/lite/kernels/dequantize_test.cc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/kernels/dequantize_test.cc b/tensorflow/lite/kernels/dequantize_test.cc
index 6343745eef..be7caa3189 100644
--- a/tensorflow/lite/kernels/dequantize_test.cc
+++ b/tensorflow/lite/kernels/dequantize_test.cc
@@ -12,6 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
+#include <cstdint>
+
 #include <gtest/gtest.h>
 #include "tensorflow/lite/interpreter.h"
 #include "tensorflow/lite/kernels/internal/types.h"
@@ -59,7 +61,7 @@ TEST(DequantizeOpTest, UINT8) {
   // [-63.5, 64] -> scale=0.5 zero_point=127 for UINT8
   DequantizeOpModel m(TensorType_UINT8, {2, 5}, 0.5, 127);
 
-  m.SetInput<uint8>({0, 1, 2, 3, 4, 251, 252, 253, 254, 255});
+  m.SetInput<uint8_t>({0, 1, 2, 3, 4, 251, 252, 253, 254, 255});
   m.Invoke();
   EXPECT_THAT(m.GetOutput(),
               ElementsAreArray(ArrayFloatNear(
@@ -70,7 +72,7 @@ TEST(DequantizeOpTest, INT8) {
   // [-63.5, 64] -> scale=0.5, zero_point=1 for INT8
   DequantizeOpModel m(TensorType_INT8, {2, 5}, 0.5, -1);
 
-  m.SetInput<int8>({-128, -127, -126, -125, -124, 123, 124, 125, 126, 127});
+  m.SetInput<int8_t>({-128, -127, -126, -125, -124, 123, 124, 125, 126, 127});
   m.Invoke();
   EXPECT_THAT(m.GetOutput(),
               ElementsAreArray(ArrayFloatNear(
-- 
GitLab


From 0ad4e9ea2b41ad1d9c8c59844fd814b01820bab2 Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Tue, 8 Jan 2019 09:57:31 -0800
Subject: [PATCH 0343/2345] Update unidirectional lstm test case.

PiperOrigin-RevId: 228348125
---
 tensorflow/lite/kernels/unidirectional_sequence_lstm_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/kernels/unidirectional_sequence_lstm_test.cc b/tensorflow/lite/kernels/unidirectional_sequence_lstm_test.cc
index 274f907076..bc35d90773 100644
--- a/tensorflow/lite/kernels/unidirectional_sequence_lstm_test.cc
+++ b/tensorflow/lite/kernels/unidirectional_sequence_lstm_test.cc
@@ -252,7 +252,7 @@ class HybridUnidirectionalLSTMOpModel : public UnidirectionalLSTMOpModel {
     tensor_type_ = tensor_type;
   }
 
-  void SetWeights(int weights_idx, std::vector<float> f) {
+  void SetWeights(int weights_idx, const std::vector<float>& f) {
     if (tensor_type_ == TensorType_UINT8) {
       SymmetricQuantizeAndPopulate(weights_idx, f);
     } else {
-- 
GitLab


From 011a8094c182101a65fb84e98a60a152e0f58f0d Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Tue, 8 Jan 2019 10:06:30 -0800
Subject: [PATCH 0344/2345] Aligned ResourceVariable.__reduce__ with
 ResourceVariable.__deepcopy__

PiperOrigin-RevId: 228350078
---
 .../kernel_tests/resource_variable_ops_test.py     | 13 ++++++++++---
 tensorflow/python/ops/resource_variable_ops.py     | 14 ++++++++++++--
 2 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index 67fa350317..2594055119 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -286,12 +286,19 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
       tmp_dir = self.get_temp_dir()
       fname = os.path.join(tmp_dir, "var.pickle")
       with open(fname, "wb") as f:
-        v = resource_variable_ops.ResourceVariable(10.0)
+        v = resource_variable_ops.ResourceVariable(
+            10.0,
+            dtype=dtypes.float16,
+            name="v")
         pickle.dump(v, f)
 
       with open(fname, "rb") as f:
-        v = pickle.load(f)
-        self.assertAllEqual(v.numpy(), 10.0)
+        new_v = pickle.load(f)
+        self.assertEqual(new_v.name, v.name)
+        self.assertEqual(new_v.shape, v.shape)
+        self.assertEqual(new_v.dtype, v.dtype)
+        self.assertEqual(new_v.trainable, v.trainable)
+        self.assertAllEqual(new_v.numpy(), v.numpy())
 
   @test_util.run_in_graph_and_eager_modes
   def testScatterDiv(self):
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index d27e38c735..2d46e21074 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -20,6 +20,7 @@ from __future__ import division
 from __future__ import print_function
 
 import contextlib
+import functools
 
 from tensorflow.core.framework import attr_value_pb2
 from tensorflow.core.framework import variable_pb2
@@ -595,7 +596,8 @@ class ResourceVariable(variables.VariableV1):
         trainable=self._trainable,
         constraint=self._constraint,
         dtype=self._dtype,
-        name=self._shared_name + "_copy")
+        name=self._shared_name + "_copy",
+        distribute_strategy=self.distribute_strategy)
     memo[self._unique_id] = copied_variable
     return copied_variable
 
@@ -938,7 +940,15 @@ class ResourceVariable(variables.VariableV1):
     return assign_op
 
   def __reduce__(self):
-    return (ResourceVariable, (self.numpy(),))
+    # The implementation mirrors that of __deepcopy__.
+    return functools.partial(
+        ResourceVariable,
+        initial_value=self.numpy(),
+        trainable=self.trainable,
+        name=self._shared_name,
+        dtype=self.dtype,
+        constraint=self.constraint,
+        distribute_strategy=self.distribute_strategy), ()
 
   def scatter_sub(self, sparse_delta, use_locking=False, name=None):
     """Subtracts `IndexedSlices` from this variable.
-- 
GitLab


From 35641fa70182fb9ee4c7e015b4ed38d0148cf340 Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Tue, 8 Jan 2019 10:15:50 -0800
Subject: [PATCH 0345/2345] Make dataset instances passed to `fit`/etc
 propagate to the `model_iteration` loop. This is a necessary step in order to
 start resetting a dataset of known cardinality as part of the loop.

PiperOrigin-RevId: 228351978
---
 tensorflow/python/keras/engine/training.py    | 185 ++++++++----------
 .../python/keras/engine/training_arrays.py    |  23 ++-
 .../keras/engine/training_dataset_test.py     |  55 +++---
 .../python/keras/engine/training_utils.py     |  67 ++++++-
 4 files changed, 192 insertions(+), 138 deletions(-)

diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index bd06751428..1eda8cf797 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -19,14 +19,12 @@ from __future__ import division
 from __future__ import print_function
 
 import collections
-import weakref
 import numpy as np
 
 from tensorflow.python import tf2
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.eager import context
-from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import tensor_util
@@ -122,10 +120,6 @@ class Model(Network):
 
   def __init__(self, *args, **kwargs):
     super(Model, self).__init__(*args, **kwargs)
-    # Create a cache for iterator get_next op.
-    self._iterator_get_next = weakref.WeakKeyDictionary()
-    # Create a cache for dataset - uninitialized iterators
-    self._dataset_iterator_cache = weakref.WeakKeyDictionary()
     # initializing _distribution_strategy here since it is possible to call
     # predict on a model without compiling it.
     self._distribution_strategy = None
@@ -1247,7 +1241,8 @@ class Model(Network):
                                 'compiled with DistributionStrategy.')
     # Validate and standardize user data.
     x, y, sample_weights = self._standardize_user_data(
-        x, y, sample_weight=sample_weight, class_weight=class_weight)
+        x, y, sample_weight=sample_weight, class_weight=class_weight,
+        extract_tensors_from_dataset=True)
 
     if self.run_eagerly:
       outputs = training_eager.train_on_batch(
@@ -1316,7 +1311,7 @@ class Model(Network):
                                 'compiled with DistributionStrategy.')
     # Validate and standardize user data.
     x, y, sample_weights = self._standardize_user_data(
-        x, y, sample_weight=sample_weight)
+        x, y, sample_weight=sample_weight, extract_tensors_from_dataset=True)
 
     if self.run_eagerly:
       outputs = training_eager.test_on_batch(
@@ -1359,7 +1354,8 @@ class Model(Network):
       raise NotImplementedError('`predict_on_batch` is not supported for '
                                 'models compiled with DistributionStrategy.')
     # Validate and standardize user data.
-    inputs, _, _ = self._standardize_user_data(x)
+    inputs, _, _ = self._standardize_user_data(
+        x, extract_tensors_from_dataset=True)
     if self.run_eagerly:
       if (isinstance(inputs, iterator_ops.EagerIterator) or
           (isinstance(inputs, dataset_ops.DatasetV2))):
@@ -2103,13 +2099,6 @@ class Model(Network):
       self._make_predict_function()
       return self.predict_function
 
-  def _get_iterator_get_next_tensors(self, iterator):
-    get_next_op = self._iterator_get_next.get(iterator, None)
-    if get_next_op is None:
-      get_next_op = iterator.get_next()
-      self._iterator_get_next[iterator] = get_next_op
-    return get_next_op
-
   def _distribution_standardize_user_data(self,
                                           x,
                                           y=None,
@@ -2231,7 +2220,8 @@ class Model(Network):
                              steps_name='steps',
                              steps=None,
                              validation_split=0,
-                             shuffle=False):
+                             shuffle=False,
+                             extract_tensors_from_dataset=False):
     """Runs validation checks on input and target data passed by the user.
 
     Also standardizes the data to lists of arrays, in order.
@@ -2275,6 +2265,10 @@ class Model(Network):
       validation_split: Float between 0 and 1.
         Fraction of the training data to be used as validation data.
       shuffle: Boolean whether to shuffle the training data before each epoch.
+      extract_tensors_from_dataset: Boolean. When `x` is a dataset instance,
+        this indicates whether to extract actual tensors from the dataset or
+        instead output the dataset instance itself.
+        Set to True when calling from `train_on_batch`/etc.
 
     Returns:
       A tuple of 3: inputs (arrays or dicts, depending on whether `x` was a dict
@@ -2287,60 +2281,30 @@ class Model(Network):
       ValueError: In case of invalid user-provided data.
       RuntimeError: If the model was never compiled.
     """
-    if isinstance(x, dataset_ops.DatasetV2):
-      if context.executing_eagerly():
-        x = iter(x)
-      else:
-        if x in self._dataset_iterator_cache:
-          x = self._dataset_iterator_cache[x]
-        else:
-          iterator = dataset_ops.make_initializable_iterator(x)
-          self._dataset_iterator_cache[x] = iterator
-          x = iterator
-        K.get_session().run(x.initializer)
+    if isinstance(x, (dataset_ops.DatasetV2, dataset_ops.DatasetV1)):
+      # Graph mode dataset. We'll pass the dataset as-is (unless
+      # `extract_tensors_from_dataset` is True, in which case we extract
+      # the tensors from the dataset and we output them.
+      training_utils.validate_dataset_input(x, y, sample_weight,
+                                            validation_split)
+      is_dataset = True
+      if extract_tensors_from_dataset:
+        # We do this for `train_on_batch`/etc.
+        x, y, sample_weight = training_utils.extract_tensors_from_dataset(x)
+    elif isinstance(x, iterator_ops.Iterator):
+      # Graph mode iterator. We extract the symbolic tensors.
+      training_utils.validate_dataset_input(x, y, sample_weight,
+                                            validation_split)
+      iterator = x
+      x, y, sample_weight = training_utils.unpack_iterator_input(iterator)
+      is_dataset = True
+    else:
+      is_dataset = False
 
     # Validates `steps` argument based on x's type.
     if check_steps:
       training_utils.check_steps_argument(x, steps, steps_name)
 
-    is_x_eager_iterator = isinstance(x, iterator_ops.EagerIterator)
-    is_x_iterator = isinstance(x, iterator_ops.Iterator)
-
-    # Validate user inputs when data is given as a dataset or dataset iterator.
-    if is_x_iterator or is_x_eager_iterator:
-      training_utils.validate_dataset_input(x, y, sample_weight,
-                                            validation_split)
-
-    # For eager iterators, when we have to process multiple batches of samples,
-    # we will standardize the data when we actually loop over iterator and get
-    # the batches. For now, we just return the iterator as is.
-    if is_x_eager_iterator:
-      return x, y, sample_weight
-
-    # If input data is a dataset iterator in graph mode or if it is an eager
-    # iterator and only one batch of samples is required, we fetch the data
-    # tensors from the iterator and then standardize them.
-    if is_x_iterator:
-      try:
-        next_element = self._get_iterator_get_next_tensors(x)
-      except errors.OutOfRangeError:
-        raise RuntimeError('Your dataset iterator ran out of data; '
-                           'Make sure that your dataset can generate '
-                           'required number of samples.')
-
-      if isinstance(next_element, (list, tuple)):
-        if len(next_element) not in [2, 3]:
-          raise ValueError(
-              'Please provide model inputs as a list or tuple of 2  or 3'
-              'elements: (input, target) or (input, target, sample_weights)'
-              'Received %s' % next_element)
-        if len(next_element) == 2:
-          x, y = next_element
-        else:
-          x, y, sample_weight = next_element
-      else:
-        x = next_element
-
     # First, we build/compile the model on the fly if necessary.
     all_inputs = []
     is_build_called = False
@@ -2349,40 +2313,51 @@ class Model(Network):
     # rather than list inputs (e.g. FeatureColumn-based models).
     dict_inputs = False
     if not self.inputs:
-      # We need to use `x` to set the model inputs.
-      # We type-check that `x` and `y` are either single arrays
+      # We need to use `x_input` to set the model inputs.
+
+      # If input data is a dataset iterator in graph mode or if it is an eager
+      # iterator and only one batch of samples is required, we fetch the data
+      # tensors from the iterator and then standardize them.
+      if isinstance(x, (dataset_ops.DatasetV2, dataset_ops.DatasetV1)):
+        x_input, y_input, _ = training_utils.extract_tensors_from_dataset(x)
+      else:
+        x_input = x
+        y_input = y
+      # We type-check that `x_input` and `y_input` are either single arrays
       # or lists of arrays.
-      if isinstance(x, (list, tuple)):
+      if isinstance(x_input, (list, tuple)):
         if not all(isinstance(v, np.ndarray) or
-                   tensor_util.is_tensor(v) for v in x):
+                   tensor_util.is_tensor(v) for v in x_input):
           raise ValueError('Please provide as model inputs either a single '
                            'array or a list of arrays. You passed: x=' + str(x))
-        all_inputs += list(x)
-      elif isinstance(x, dict):
+        all_inputs += list(x_input)
+      elif isinstance(x_input, dict):
         dict_inputs = True
-        keys = sorted(x.keys())
-        all_inputs = [x[k] for k in keys]
+        keys = sorted(x_input.keys())
+        all_inputs = [x_input[k] for k in keys]
       else:
-        if not isinstance(x, np.ndarray) and not tensor_util.is_tensor(x):
+        if (not isinstance(x_input, np.ndarray) and
+            not tensor_util.is_tensor(x_input)):
           raise ValueError('Please provide as model inputs either a single '
                            'array or a list of arrays. You passed: x=' + str(x))
-        all_inputs.append(x)
+        all_inputs.append(x_input)
 
       # Build the model using the retrieved inputs (value or symbolic).
       # If values or generated from a dataset, then in symbolic-mode
       # placeholders will be created to match the value shapes.
       is_build_called = True
-      if is_x_iterator:
-        cast_inputs = nest.map_structure(lambda v: v.shape, x)
-      elif training_utils.has_tensors(x):
-        cast_inputs = training_utils.cast_if_floating_dtype(x)
+      if is_dataset:
+        cast_inputs = nest.map_structure(lambda v: v.shape, x_input)
+      elif training_utils.has_tensors(x_input):
+        cast_inputs = training_utils.cast_if_floating_dtype(x_input)
       else:
-        cast_inputs = x
+        cast_inputs = x_input
       self._set_inputs(cast_inputs)
     else:
+      y_input = y
       dict_inputs = isinstance(self.inputs, dict)
 
-    if y is not None:
+    if y_input is not None:
       if not self.optimizer:
         raise RuntimeError('You must compile a model before '
                            'training/testing. '
@@ -2390,23 +2365,24 @@ class Model(Network):
       if not self._is_compiled:
         # On-the-fly compilation of the model.
         # We need to use `y` to set the model targets.
-        if training_utils.has_tensors(y):
-          y = training_utils.cast_if_floating_dtype(y)
-        if isinstance(y, (list, tuple)):
+        if training_utils.has_tensors(y_input):
+          y_input = training_utils.cast_if_floating_dtype(y_input)
+        if isinstance(y_input, (list, tuple)):
           if not all(isinstance(v, np.ndarray) or
-                     tensor_util.is_tensor(v) for v in y):
+                     tensor_util.is_tensor(v) for v in y_input):
             raise ValueError('Please provide as model targets either a single '
                              'array or a list of arrays. '
                              'You passed: y=' + str(y))
-          all_inputs += list(y)
-        elif isinstance(y, dict):
-          raise ValueError('Please do not pass a dictionary as model targets.')
+          all_inputs += list(y_input)
+        elif isinstance(y_input, dict):
+          raise ValueError('You cannot pass a dictionary as model targets.')
         else:
-          if not isinstance(y, np.ndarray) and not tensor_util.is_tensor(y):
+          if (not isinstance(y_input, np.ndarray) and
+              not tensor_util.is_tensor(y_input)):
             raise ValueError('Please provide as model targets either a single '
                              'array or a list of arrays. '
                              'You passed: y=' + str(y))
-          all_inputs.append(y)
+          all_inputs.append(y_input)
 
         # Typecheck that all inputs are *either* value *or* symbolic.
         # TODO(fchollet): this check could be removed in Eager mode?
@@ -2416,13 +2392,13 @@ class Model(Network):
                              'TensorFlow tensors. '
                              'You passed: x=' + str(x) + '; y=' + str(y))
 
-        if self.run_eagerly or is_x_iterator:
+        if is_dataset or context.executing_eagerly():
           target_tensors = None
         else:
           # Handle target tensors if any passed.
-          if not isinstance(y, (list, tuple)):
-            y = [y]
-          target_tensors = [v for v in y if _is_symbolic_tensor(v)]
+          if not isinstance(y_input, (list, tuple)):
+            y_input = [y_input]
+          target_tensors = [v for v in y_input if _is_symbolic_tensor(v)]
         is_compile_called = True
         self.compile(
             optimizer=self.optimizer,
@@ -2440,7 +2416,7 @@ class Model(Network):
     # Note: in this case, `any` and `all` are equivalent since we disallow
     # mixed symbolic/value inputs.
     if (not self.run_eagerly and is_build_called and is_compile_called and
-        not is_x_iterator and any(_is_symbolic_tensor(v) for v in all_inputs)):
+        not is_dataset  and any(_is_symbolic_tensor(v) for v in all_inputs)):
       return [], [], []
 
     # What follows is input validation and standardization to list format,
@@ -2462,12 +2438,14 @@ class Model(Network):
       feed_input_shapes = self._feed_input_shapes
 
     # Standardize the inputs.
-    x = training_utils.standardize_input_data(
-        x,
-        feed_input_names,
-        feed_input_shapes,
-        check_batch_axis=False,  # Don't enforce the batch size.
-        exception_prefix='input')
+    if not isinstance(x, (dataset_ops.DatasetV2, dataset_ops.DatasetV1)):
+      # TODO(fchollet): run static checks with dataset output shape(s).
+      x = training_utils.standardize_input_data(
+          x,
+          feed_input_names,
+          feed_input_shapes,
+          check_batch_axis=False,  # Don't enforce the batch size.
+          exception_prefix='input')
 
     if y is not None:
       if not self._is_graph_network:
@@ -2541,7 +2519,8 @@ class Model(Network):
                          str(x[0].shape[0]) + ' samples')
 
     # If dictionary inputs were provided, we return a dictionary as well.
-    if dict_inputs:
+    if dict_inputs and not isinstance(x, (dataset_ops.DatasetV2,
+                                          dataset_ops.DatasetV1)):
       x = dict(zip(feed_input_names, x))
     return x, y, sample_weights
 
diff --git a/tensorflow/python/keras/engine/training_arrays.py b/tensorflow/python/keras/engine/training_arrays.py
index 9887fe34a8..ab7d455bfa 100644
--- a/tensorflow/python/keras/engine/training_arrays.py
+++ b/tensorflow/python/keras/engine/training_arrays.py
@@ -23,6 +23,7 @@ import functools
 
 import numpy as np
 
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.eager import context
 from tensorflow.python.framework import errors
 from tensorflow.python.keras import backend as K
@@ -146,15 +147,15 @@ def model_iteration(model,
 
   Arguments:
       model: Keras Model instance.
-      inputs: Either a list of arrays or a dictionary.
-      targets: List of target arrays.
+      inputs: Either a list or dictionary of arrays, or a dataset instance.
+      targets: List/dictionary of input arrays.
       sample_weights: Optional list of sample weight arrays.
       batch_size: Integer batch size or None if unknown.
       epochs: Number of times to iterate over the data
       verbose: Verbosity mode, 0, 1 or 2
       callbacks: List of callbacks to be called during training
-      val_inputs: List of input arrays.
-      val_targets: List of target arrays.
+      val_inputs: Either a list or dictionary of arrays, or a dataset instance.
+      val_targets: List/dictionary of target arrays.
       val_sample_weights: Optional list of sample weight arrays.
       shuffle: Whether to shuffle the data at the beginning of each epoch
         concatenation of list the display names of the outputs of `f` and the
@@ -186,6 +187,20 @@ def model_iteration(model,
   if 'steps' in kwargs:
     steps_per_epoch = kwargs['steps']
 
+  # In case we are passed datasets, we extract symbolic tensors from them.
+  if isinstance(inputs, (dataset_ops.DatasetV2, dataset_ops.DatasetV1)):
+    inputs, targets, sample_weights = model._standardize_user_data(
+        inputs,
+        steps_name='steps_per_epoch',
+        steps=steps_per_epoch,
+        extract_tensors_from_dataset=True)
+  if isinstance(val_inputs, (dataset_ops.DatasetV2, dataset_ops.DatasetV1)):
+    val_inputs, val_targets, val_sample_weights = model._standardize_user_data(
+        val_inputs,
+        steps_name='validation_steps',
+        steps=validation_steps,
+        extract_tensors_from_dataset=True)
+
   _validate_arguments(steps_per_epoch, validation_steps, kwargs)
   if mode == ModeKeys.TRAIN:
     _print_train_info(inputs, val_inputs, steps_per_epoch, verbose)
diff --git a/tensorflow/python/keras/engine/training_dataset_test.py b/tensorflow/python/keras/engine/training_dataset_test.py
index 646ce31909..40d42c13f0 100644
--- a/tensorflow/python/keras/engine/training_dataset_test.py
+++ b/tensorflow/python/keras/engine/training_dataset_test.py
@@ -25,7 +25,6 @@ import numpy as np
 from tensorflow.python import keras
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.eager import context
-from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util as tf_test_util
 from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import metrics as metrics_module
@@ -99,29 +98,6 @@ class TestTrainingWithDatasetIterators(keras_parameterized.TestCase):
                                  'the `steps` argument'):
       model.predict(iterator, verbose=0)
 
-  @keras_parameterized.run_with_all_model_types
-  @keras_parameterized.run_all_keras_modes
-  def test_get_next_op_created_once(self):
-    model = testing_utils.get_small_mlp(1, 4, input_dim=3)
-    optimizer = RMSPropOptimizer(learning_rate=0.001)
-    loss = 'mse'
-    metrics = ['mae']
-    model.compile(optimizer, loss, metrics=metrics,
-                  run_eagerly=testing_utils.should_run_eagerly())
-
-    inputs = np.zeros((10, 3), np.float32)
-    targets = np.zeros((10, 4), np.float32)
-    dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
-    dataset = dataset.repeat(100)
-    dataset = dataset.batch(10)
-    iterator = dataset_ops.make_one_shot_iterator(dataset)
-
-    model.fit(iterator, epochs=1, steps_per_epoch=2, verbose=1)
-    # Finalize graph to make sure we are not appending another iterator
-    # get_next op in the graph.
-    ops.get_default_graph().finalize()
-    model.fit(iterator, epochs=1, steps_per_epoch=2, verbose=1)
-
   @keras_parameterized.run_with_all_model_types
   @keras_parameterized.run_all_keras_modes
   def test_iterators_running_out_of_data(self):
@@ -172,9 +148,6 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
     # Call fit with validation data
     model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0,
               validation_data=dataset, validation_steps=2)
-    # Finalize the graph to make sure new ops aren't added when calling on the
-    # same dataset
-    ops.get_default_graph().finalize()
     model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0,
               validation_data=dataset, validation_steps=2)
 
@@ -292,6 +265,34 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
 
       model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1)
 
+  @keras_parameterized.run_all_keras_modes
+  def test_dataset_fit_correctness(self):
+
+    class SumLayer(keras.layers.Layer):
+
+      def build(self, _):
+        self.w = self.add_weight('w', ())
+
+      def call(self, inputs):
+        return keras.backend.sum(inputs) + self.w * 0
+
+    model = keras.Sequential([SumLayer(input_shape=(2,))])
+    model.compile(RMSPropOptimizer(learning_rate=0.001),
+                  loss='mae',
+                  run_eagerly=testing_utils.should_run_eagerly())
+
+    inputs = np.zeros((40, 2), dtype=np.float32)
+    inputs[10:20, :] = 2
+    inputs[20:30, :] = 1
+    inputs[30:, :] = 4
+    targets = np.zeros((40, 1), dtype=np.float32)
+    dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
+    dataset = dataset.batch(10)
+    history = model.fit(dataset,
+                        epochs=2, steps_per_epoch=2, verbose=1, shuffle=False)
+    self.assertListEqual(history.history['loss'],
+                         [inputs[:20].sum() / 2, inputs[20:].sum() / 2])
+
   @tf_test_util.run_deprecated_v1
   def test_dataset_input_shape_validation(self):
     with self.cached_session():
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index 5edf1de1b0..949f1e400d 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -31,6 +31,7 @@ from tensorflow.python.data.ops import readers
 from tensorflow.python.eager import context
 from tensorflow.python.eager import def_function
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import tensor_spec
@@ -1081,10 +1082,10 @@ def is_feature_layer(layer):
 
 
 def is_eager_dataset_or_iterator(data):
-  if context.executing_eagerly():
-    if isinstance(data, (dataset_ops.DatasetV2, iterator_ops.EagerIterator)):
-      return True
-  return False
+  return context.executing_eagerly() and isinstance(
+      data, (dataset_ops.DatasetV1,
+             dataset_ops.DatasetV2,
+             iterator_ops.EagerIterator))
 
 
 # pylint: disable=protected-access
@@ -1199,6 +1200,64 @@ def assert_not_shuffled(dataset):
     raise ValueError('Could not assert that dataset is not shuffled.')
 
 
+def is_dataset_or_iterator(data):
+  return isinstance(data, (dataset_ops.DatasetV1,
+                           dataset_ops.DatasetV2,
+                           iterator_ops.EagerIterator,
+                           iterator_ops.Iterator))
+
+
+def extract_tensors_from_dataset(dataset):
+  """Extract a tuple of tensors `inputs, targets, sample_weight` from a dataset.
+
+  Works only for graph mode.
+
+  Arguments:
+    dataset: Dataset instance.
+
+  Returns:
+    Tuple of tensors `x, y, weights`. `y` and `weights` entry may be None.
+  """
+  iterator = dataset_ops.make_initializable_iterator(dataset)
+  K.get_session().run(iterator.initializer)
+  inputs, targets, sample_weight = unpack_iterator_input(iterator)
+  return inputs, targets, sample_weight
+
+
+def unpack_iterator_input(iterator):
+  """Convert a dataset iterator to a tuple of tensors `x, y, sample_weights`.
+
+  Arguments:
+    iterator: Instance of a dataset iterator.
+
+  Returns:
+    Tuple of tensors `x, y, weights`. `y` and `weights` entry may be None.
+  """
+  try:
+    next_element = iterator.get_next()
+  except errors.OutOfRangeError:
+    raise RuntimeError('Your dataset iterator ran out of data; '
+                       'Make sure that your dataset can generate '
+                       'required number of samples.')
+
+  if isinstance(next_element, (list, tuple)):
+    if len(next_element) not in [2, 3]:
+      raise ValueError(
+          'Please provide model inputs as a list or tuple of 2 or 3 '
+          'elements: (input, target) or (input, target, sample_weights) '
+          'Received %s' % next_element)
+    if len(next_element) == 2:
+      x, y = next_element
+      weights = None
+    else:
+      x, y, weights = next_element
+  else:
+    x = next_element
+    y = None
+    weights = None
+  return x, y, weights
+
+
 class ModelInputs(object):
   """Encapsulates model inputs.
 
-- 
GitLab


From e5319d26bb5b3fed5e3c9c865193db41f75cf958 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Tue, 8 Jan 2019 10:17:23 -0800
Subject: [PATCH 0346/2345] Add mean relative error v2 metric.

PiperOrigin-RevId: 228352265
---
 tensorflow/python/keras/metrics.py      | 80 +++++++++++++++++++++++++
 tensorflow/python/keras/metrics_test.py | 56 +++++++++++++++++
 2 files changed, 136 insertions(+)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 9a2e057780..2c2700ffa3 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -51,6 +51,7 @@ from tensorflow.python.keras.utils.generic_utils import serialize_keras_object
 from tensorflow.python.keras.utils.generic_utils import to_list
 from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import confusion_matrix
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
@@ -346,6 +347,85 @@ class Mean(Metric):
     return math_ops.div_no_nan(self.total, self.count)
 
 
+class MeanRelativeError(Mean):
+  """Computes the mean relative error by normalizing with the given values.
+
+  This metric creates two local variables, `total` and `count` that are used to
+  compute the mean relative absolute error. This average is weighted by
+  `sample_weight`, and it is ultimately returned as `mean_relative_error`:
+  an idempotent operation that simply divides `total` by `count`.
+
+  If `sample_weight` is `None`, weights default to 1.
+  Use `sample_weight` of 0 to mask values.
+
+  Usage:
+
+  ```python
+  m = tf.keras.metrics.MeanRelativeError(normalizer=[1, 3, 2, 3])
+  m.update_state([1, 3, 2, 3], [2, 4, 6, 8])
+
+  # metric = mean(|y_pred - y_true| / normalizer)
+  #        = mean([1, 1, 4, 5] / [1, 3, 2, 3]) = mean([1, 1/3, 2, 5/3])
+  #        = 5/4 = 1.25
+  print('Final result: ', m.result().numpy())  # Final result: 1.25
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile(
+    'sgd',
+    loss='mse',
+    metrics=[tf.keras.metrics.MeanRelativeError(normalizer=[1, 3])])
+  ```
+  """
+
+  def __init__(self, normalizer, name=None, dtype=None):
+    """Creates a `MeanRelativeError` instance.
+
+    Args:
+      normalizer: The normalizer values with same shape as predictions.
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+    """
+    super(MeanRelativeError, self).__init__(name=name, dtype=dtype)
+    normalizer = math_ops.cast(normalizer, self._dtype)
+    self.normalizer = normalizer
+
+  def update_state(self, y_true, y_pred, sample_weight=None):
+    """Accumulates metric statistics.
+
+    Args:
+      y_true: The ground truth values.
+      y_pred: The predicted values.
+      sample_weight: Optional weighting of each example. Defaults to 1. Can be a
+        `Tensor` whose rank is either 0, or the same rank as `y_true`, and must
+        be broadcastable to `y_true`.
+
+    Returns:
+      Update op.
+    """
+    y_true = math_ops.cast(y_true, self._dtype)
+    y_pred = math_ops.cast(y_pred, self._dtype)
+    y_pred, y_true, sample_weight = squeeze_or_expand_dimensions(
+        y_pred, y_true, sample_weight)
+
+    y_pred, self.normalizer = confusion_matrix.remove_squeezable_dimensions(
+        y_pred, self.normalizer)
+    y_pred.shape.assert_is_compatible_with(y_pred.shape)
+    relative_errors = math_ops.div_no_nan(
+        math_ops.abs(y_true - y_pred), self.normalizer)
+
+    return super(MeanRelativeError, self).update_state(
+        relative_errors, sample_weight=sample_weight)
+
+  def get_config(self):
+    config = {'normalizer': self.normalizer}
+    base_config = super(MeanRelativeError, self).get_config()
+    return dict(list(base_config.items()) + list(config.items()))
+
+
 class MeanMetricWrapper(Mean):
   """Wraps a stateless metric function with the Mean metric."""
 
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index a02b95bb46..4f8406f0e3 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -1603,6 +1603,62 @@ class KullbackLeiblerDivergenceTest(test.TestCase):
     self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class MeanRelativeErrorTest(test.TestCase):
+
+  def test_config(self):
+    normalizer = constant_op.constant([1, 3], dtype=dtypes.float32)
+    mre_obj = metrics.MeanRelativeError(normalizer=normalizer, name='mre')
+    self.assertEqual(mre_obj.name, 'mre')
+    self.assertArrayNear(self.evaluate(mre_obj.normalizer), [1, 3], 1e-1)
+
+    mre_obj2 = metrics.MeanRelativeError.from_config(mre_obj.get_config())
+    self.assertEqual(mre_obj2.name, 'mre')
+    self.assertArrayNear(self.evaluate(mre_obj2.normalizer), [1, 3], 1e-1)
+
+  def test_unweighted(self):
+    np_y_pred = np.asarray([2, 4, 6, 8], dtype=np.float32)
+    np_y_true = np.asarray([1, 3, 2, 3], dtype=np.float32)
+    expected_error = np.mean(
+        np.divide(np.absolute(np_y_pred - np_y_true), np_y_true))
+
+    y_pred = constant_op.constant(np_y_pred, shape=(1, 4), dtype=dtypes.float32)
+    y_true = constant_op.constant(np_y_true, shape=(1, 4))
+
+    mre_obj = metrics.MeanRelativeError(normalizer=y_true)
+    self.evaluate(variables.variables_initializer(mre_obj.variables))
+
+    result = mre_obj(y_true, y_pred)
+    self.assertAllClose(self.evaluate(result), expected_error, atol=1e-3)
+
+  def test_weighted(self):
+    np_y_pred = np.asarray([2, 4, 6, 8], dtype=np.float32)
+    np_y_true = np.asarray([1, 3, 2, 3], dtype=np.float32)
+    sample_weight = np.asarray([0.2, 0.3, 0.5, 0], dtype=np.float32)
+    rel_errors = np.divide(np.absolute(np_y_pred - np_y_true), np_y_true)
+    expected_error = np.sum(rel_errors * sample_weight)
+
+    y_pred = constant_op.constant(np_y_pred, dtype=dtypes.float32)
+    y_true = constant_op.constant(np_y_true)
+
+    mre_obj = metrics.MeanRelativeError(normalizer=y_true)
+    self.evaluate(variables.variables_initializer(mre_obj.variables))
+
+    result = mre_obj(
+        y_true, y_pred, sample_weight=constant_op.constant(sample_weight))
+    self.assertAllClose(self.evaluate(result), expected_error, atol=1e-3)
+
+  def test_zero_normalizer(self):
+    y_pred = constant_op.constant([2, 4], dtype=dtypes.float32)
+    y_true = constant_op.constant([1, 3])
+
+    mre_obj = metrics.MeanRelativeError(normalizer=array_ops.zeros_like(y_true))
+    self.evaluate(variables.variables_initializer(mre_obj.variables))
+
+    result = mre_obj(y_true, y_pred)
+    self.assertEqual(self.evaluate(result), 0)
+
+
 def _get_model(compile_metrics):
   model_layers = [
       layers.Dense(3, activation='relu', kernel_initializer='ones'),
-- 
GitLab


From dff89fe60f5bbf1aa6c124fe295c9af3d9e3a868 Mon Sep 17 00:00:00 2001
From: Tong Shen <endlessroad@google.com>
Date: Tue, 8 Jan 2019 10:18:15 -0800
Subject: [PATCH 0347/2345] Support outside compilation in function call.

PiperOrigin-RevId: 228352466
---
 tensorflow/compiler/jit/BUILD                 |   1 +
 .../jit/encapsulate_subgraphs_pass_test.cc    |  98 ++++++--
 .../jit/extract_outside_compilation_pass.cc   | 196 +++++++++++++---
 .../jit/extract_outside_compilation_pass.h    |   4 +-
 .../extract_outside_compilation_pass_test.cc  | 216 +++++++++++++++++-
 tensorflow/compiler/tf2xla/graph_compiler.cc  |  28 ++-
 .../compiler/tf2xla/side_effect_util.cc       |   4 +
 7 files changed, 475 insertions(+), 72 deletions(-)

diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index b9a87ba296..ba9fa93654 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -634,6 +634,7 @@ tf_cc_test(
         "//tensorflow/core:framework_internal",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:session_options",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
diff --git a/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc b/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc
index 8617beec00..1f8ec09e19 100644
--- a/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc
+++ b/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc
@@ -25,6 +25,8 @@ limitations under the License.
 #include "tensorflow/compiler/jit/encapsulate_util.h"
 #include "tensorflow/compiler/jit/extract_outside_compilation_pass.h"
 #include "tensorflow/compiler/tf2xla/side_effect_util.h"
+#include "tensorflow/core/common_runtime/device_factory.h"
+#include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/framework/function_testlib.h"
 #include "tensorflow/core/framework/graph_to_functiondef.h"
 #include "tensorflow/core/graph/graph_constructor.h"
@@ -32,6 +34,8 @@ limitations under the License.
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/public/session_options.h"
+#include "tensorflow/core/public/version.h"
 #include "tensorflow/core/util/equal_graph_def.h"
 
 namespace tensorflow {
@@ -513,6 +517,18 @@ Status Encapsulate(GraphDef* graphdef, FunctionDefLibrary* library,
   s = PerformStaticShapeInferenceBeforeEncapsulation(graph.get());
   if (!s.ok()) return s;
 
+  // Create FunctionLibraryRuntime.
+  SessionOptions session_options;
+  std::vector<std::unique_ptr<Device>> devices;
+  TF_CHECK_OK(DeviceFactory::AddDevices(
+      session_options, "/job:localhost/replica:0/task:0", &devices));
+  OptimizerOptions opts;
+  auto device_mgr = absl::make_unique<DeviceMgr>(std::move(devices));
+  auto pflr = absl::make_unique<ProcessFunctionLibraryRuntime>(
+      device_mgr.get(), Env::Default(), TF_GRAPH_DEF_VERSION, lib_def.get(),
+      opts, /*default_thread_pool=*/nullptr, /*cluster_flr=*/nullptr);
+  auto flr = pflr->GetFLR("/job:localhost/replica:0/task:0/cpu:0");
+
   std::unique_ptr<Graph> graph_out;
   s = EncapsulateSubgraphsInFunctions(
       "_encapsulate", /*outside_compilation_attribute=*/"", *graph,
@@ -538,7 +554,7 @@ Status Encapsulate(GraphDef* graphdef, FunctionDefLibrary* library,
                                     std::map<string, int>{}});
   }
   s = ExtractOutsideCompilation("_encapsulate", "_outside", clusters,
-                                graph_out.get(), lib_def.get());
+                                graph_out.get(), flr, lib_def.get());
   if (!s.ok()) return s;
 
   GraphDef graphdef_out;
@@ -941,7 +957,9 @@ TEST(EncapsulateSubgraphsTest, OneFunctionOneOutside) {
             {"key", "host_compute_channel_F1_O1"},
             {"shape_inference_graph", shape_inference_graph},
             {"shapes", absl::Span<const DataType>({})},
-            {"_outside_compilation_subgraph", "O1"}},
+            {"_outside_compilation_subgraph", "O1"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}},
            {"c"}},
       },
       {{"f_0_retval_retval", "F:o:0"}});
@@ -1101,7 +1119,9 @@ TEST(EncapsulateSubgraphsTest, OneFunctionTwoOutside) {
             {"key", "host_compute_channel_F1_O2"},
             {"shape_inference_graph", shape_inference_graph2},
             {"shapes", absl::Span<const DataType>({})},
-            {"_outside_compilation_subgraph", "O2"}},
+            {"_outside_compilation_subgraph", "O2"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}},
            {"F"}},
           {{"outside_compilation_O1_host_compute"},
            "XlaHostCompute",
@@ -1112,7 +1132,9 @@ TEST(EncapsulateSubgraphsTest, OneFunctionTwoOutside) {
             {"key", "host_compute_channel_F1_O1"},
             {"shape_inference_graph", shape_inference_graph1},
             {"shapes", absl::Span<const DataType>({})},
-            {"_outside_compilation_subgraph", "O1"}},
+            {"_outside_compilation_subgraph", "O1"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}},
            {"D"}},
       },
       {{"g_0_retval_retval", "outside_compilation_O2_host_compute:outputs:0"},
@@ -1244,7 +1266,9 @@ TEST(EncapsulateSubgraphsTest, TwoFunctionsTwoOutside) {
             {"shape_inference_graph", NameAttrList()},
             {"shapes",
              absl::Span<const TensorShapeProto>({shape_proto_expected})},
-            {"_outside_compilation_subgraph", "O1"}},
+            {"_outside_compilation_subgraph", "O1"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}},
            {"D"}},
       },
       {{"e_0_retval_retval", "outside_compilation_O1_host_compute:outputs:0"},
@@ -1269,7 +1293,9 @@ TEST(EncapsulateSubgraphsTest, TwoFunctionsTwoOutside) {
             {"shape_inference_graph", NameAttrList()},
             {"shapes",
              absl::Span<const TensorShapeProto>({shape_proto_expected})},
-            {"_outside_compilation_subgraph", "O1"}}},
+            {"_outside_compilation_subgraph", "O1"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}}},
       },
       {{"g_0_retval_retval", "G:o:0"}, {"i_0_retval_retval", "I:o:0"}});
 
@@ -1397,7 +1423,9 @@ TEST(EncapsulateSubgraphsTest, TwoFunctionsTwoOutsideDependencyFromOutside) {
             {"shape_inference_graph", NameAttrList()},
             {"shapes",
              absl::Span<const TensorShapeProto>({shape_proto_expected})},
-            {"_outside_compilation_subgraph", "O1"}},
+            {"_outside_compilation_subgraph", "O1"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}},
            {"D"}},
       },
       {{"f_0_retval_retval", "F:o:0"}});
@@ -1419,7 +1447,9 @@ TEST(EncapsulateSubgraphsTest, TwoFunctionsTwoOutsideDependencyFromOutside) {
             {"shape_inference_graph", NameAttrList()},
             {"shapes",
              absl::Span<const TensorShapeProto>({shape_proto_expected})},
-            {"_outside_compilation_subgraph", "O1"}}},
+            {"_outside_compilation_subgraph", "O1"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}}},
       },
       {{"i_0_retval_retval", "I:o:0"}});
 
@@ -1527,7 +1557,9 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationNoInputs) {
             {"shape_inference_graph", NameAttrList()},
             {"shapes",
              absl::Span<const TensorShapeProto>({shape_proto_expected})},
-            {"_outside_compilation_subgraph", "O1"}}},
+            {"_outside_compilation_subgraph", "O1"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}}},
       },
       {{"f_0_retval_retval", "F:o:0"}});
 
@@ -1615,7 +1647,9 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationControlInput) {
             {"shape_inference_graph", NameAttrList()},
             {"shapes",
              absl::Span<const TensorShapeProto>({shape_proto_expected})},
-            {"_outside_compilation_subgraph", "O1"}},
+            {"_outside_compilation_subgraph", "O1"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}},
            {"D"}},
       },
       {{"f_0_retval_retval", "F:o:0"}});
@@ -1716,7 +1750,9 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationNoOutputs) {
             {"key", "host_compute_channel_F1_O1"},
             {"shape_inference_graph", shape_inference_graph},
             {"shapes", absl::Span<const TensorShapeProto>({})},
-            {"_outside_compilation_subgraph", "O1"}}},
+            {"_outside_compilation_subgraph", "O1"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}}},
       },
       {{"e_0_retval_retval", "outside_compilation_O1_host_compute:outputs:0"},
        {"f_0_retval_retval", "F:o:0"}});
@@ -1821,7 +1857,9 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationControlOutput) {
             {"key", "host_compute_channel_F1_O1"},
             {"shape_inference_graph", shape_inference_graph},
             {"shapes", absl::Span<const TensorShapeProto>({})},
-            {"_outside_compilation_subgraph", "O1"}}},
+            {"_outside_compilation_subgraph", "O1"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}}},
       },
       {{"e_0_retval_retval", "outside_compilation_O1_host_compute:outputs:0"},
        {"f_0_retval_retval", "F:o:0"}});
@@ -1949,7 +1987,9 @@ TEST(EncapsulateSubgraphsTest,
             {"key", "host_compute_channel_F1_O1"},
             {"shape_inference_graph", shape_inference_graph1},
             {"shapes", absl::Span<const TensorShapeProto>({})},
-            {"_outside_compilation_subgraph", "O1"}}},
+            {"_outside_compilation_subgraph", "O1"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}}},
           {{"outside_compilation_O2_host_compute"},
            "XlaHostCompute",
            {"F:o:0"},
@@ -1959,7 +1999,9 @@ TEST(EncapsulateSubgraphsTest,
             {"key", "host_compute_channel_F1_O2"},
             {"shape_inference_graph", shape_inference_graph2},
             {"shapes", absl::Span<const TensorShapeProto>({})},
-            {"_outside_compilation_subgraph", "O2"}}},
+            {"_outside_compilation_subgraph", "O2"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}}},
       },
       {{"e_0_retval_retval", "outside_compilation_O1_host_compute:outputs:0"},
        {"h_0_retval_retval", "H:o:0"}});
@@ -2082,7 +2124,9 @@ TEST(EncapsulateSubgraphsTest,
             {"key", "host_compute_channel_F1_O2"},
             {"shape_inference_graph", NameAttrList()},
             {"shapes", absl::Span<const TensorShapeProto>({})},
-            {"_outside_compilation_subgraph", "O2"}}},
+            {"_outside_compilation_subgraph", "O2"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}}},
           {{"outside_compilation_O1_host_compute"},
            "XlaHostCompute",
            {"D:o:0"},
@@ -2092,7 +2136,9 @@ TEST(EncapsulateSubgraphsTest,
             {"key", "host_compute_channel_F1_O1"},
             {"shape_inference_graph", shape_inference_graph},
             {"shapes", absl::Span<const TensorShapeProto>({})},
-            {"_outside_compilation_subgraph", "O1"}}},
+            {"_outside_compilation_subgraph", "O1"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}}},
       },
       {{"e_0_retval_retval", "outside_compilation_O1_host_compute:outputs:0"},
        {"h_0_retval_retval", "H:o:0"}});
@@ -2214,7 +2260,9 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationClusterDependency) {
          {"key", "host_compute_channel_F1_O1"},
          {"shape_inference_graph", shape_inference_graph},
          {"shapes", absl::Span<const TensorShapeProto>({})},
-         {"_outside_compilation_subgraph", "O1"}}},
+         {"_outside_compilation_subgraph", "O1"},
+         {"_xla_token_input_nodes",
+          absl::Span<const string>({"_xla_token_arg_node"})}}},
        {{"outside_compilation_O2_host_compute"},
         "XlaHostCompute",
         {"D:o:0"},
@@ -2224,7 +2272,9 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationClusterDependency) {
          {"key", "host_compute_channel_F1_O2"},
          {"shape_inference_graph", NameAttrList()},
          {"shapes", absl::Span<const TensorShapeProto>({})},
-         {"_outside_compilation_subgraph", "O2"}},
+         {"_outside_compilation_subgraph", "O2"},
+         {"_xla_token_input_nodes",
+          absl::Span<const string>({"_xla_token_arg_node"})}},
         {}},
        {{"outside_compilation_O3_host_compute"},
         "XlaHostCompute",
@@ -2235,7 +2285,9 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationClusterDependency) {
          {"key", "host_compute_channel_F1_O3"},
          {"shape_inference_graph", NameAttrList()},
          {"shapes", absl::Span<const TensorShapeProto>({})},
-         {"_outside_compilation_subgraph", "O3"}},
+         {"_outside_compilation_subgraph", "O3"},
+         {"_xla_token_input_nodes",
+          absl::Span<const string>({"_xla_token_arg_node"})}},
         {}}},
       {{"e_0_retval_retval", "outside_compilation_O1_host_compute:outputs:0"},
        {"h_0_retval_retval", "H:o:0"}});
@@ -2354,7 +2406,9 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationNoInputsOrOutputs) {
             {"key", "host_compute_channel_F1_O1"},
             {"shape_inference_graph", shape_inference_graph},
             {"shapes", absl::Span<const TensorShapeProto>({})},
-            {"_outside_compilation_subgraph", "O1"}}},
+            {"_outside_compilation_subgraph", "O1"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}}},
       },
       {{"e_0_retval_retval", "outside_compilation_O1_host_compute:outputs:0"},
        {"f_0_retval_retval", "F:o:0"}});
@@ -2465,7 +2519,9 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationShapeInference) {
             {"key", "host_compute_channel_F1_O1"},
             {"shape_inference_graph", shape_inference_graph},
             {"shapes", absl::Span<const DataType>({})},
-            {"_outside_compilation_subgraph", "O1"}},
+            {"_outside_compilation_subgraph", "O1"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}},
            {"c"}},
       },
       {{"f_0_retval_retval", "F:o:0"}});
diff --git a/tensorflow/compiler/jit/extract_outside_compilation_pass.cc b/tensorflow/compiler/jit/extract_outside_compilation_pass.cc
index 8b01768c49..2a770c527b 100644
--- a/tensorflow/compiler/jit/extract_outside_compilation_pass.cc
+++ b/tensorflow/compiler/jit/extract_outside_compilation_pass.cc
@@ -30,6 +30,7 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_shape.pb.h"
 #include "tensorflow/core/graph/algorithm.h"
 #include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/gtl/cleanup.h"
 
 namespace tensorflow {
 
@@ -308,6 +309,10 @@ xla::StatusOr<NodeDef> BuildXlaHostComputeNodeDef(
     host_compute_builder.Attr("tpu_core", core);
   }
 
+  // Set input tokens.
+  host_compute_builder.Attr(kXlaTokenInputNodesAttrName,
+                            std::vector<string>{kXlaTokenArgNodeName});
+
   // Populate inputs.
   std::vector<DataType> input_dtypes;
   TF_RETURN_IF_ERROR(GetNodeAttr(call_node->attrs(), "Tinputs", &input_dtypes));
@@ -398,8 +403,8 @@ Status ReplaceOrRemoveOutsideCompilationCallNode(
 }
 
 // Resets "device_ordinal" attr to placeholder value for related nodes
-// (XlaRecvAtHost nodes; XlaSendFromHost nodes; If nodes containing
-// XlaRecvAtHost/XlaSendFromHost).
+// (XlaRecvAtHost nodes; XlaSendFromHost nodes; If/While/FuncCall nodes
+// containing XlaRecvAtHost/XlaSendFromHost).
 Status ResetDeviceOrdinalToPlaceholderValue(Graph* g) {
   AttrValue device_ordinal_value;
   device_ordinal_value.set_placeholder("device_ordinal");
@@ -429,6 +434,10 @@ Status ResetDeviceOrdinalToPlaceholderValue(Graph* g) {
         n->ClearAttr(attr_name);
         n->AddAttr(attr_name, branch_func);
       }
+    } else if (HasNodeAttr(n->def(), "device_ordinal")) {
+      // Function call node containing outside compilation.
+      n->ClearAttr("device_ordinal");
+      n->AddAttr("device_ordinal", device_ordinal_value);
     } else {
       return errors::Internal("Unknown node marked with ",
                               kXlaHasHostTransferAttrName, ": ",
@@ -1217,20 +1226,129 @@ Status BuildHostGraphForWhileNode(
   return Status::OK();
 }
 
+// Builds host graph for func call nodes.
+Status BuildHostGraphForFuncCallNode(const string& func_call_node_name,
+                                     const string& xla_cluster_name,
+                                     const string& func_call_host_func_name,
+                                     const string& host_graph_func_name,
+                                     FunctionLibraryDefinition* fld) {
+  Graph host_graph(fld);
+  AttrValue device_ordinal_value;
+  device_ordinal_value.set_placeholder("device_ordinal");
+
+  // Step 1: add key placeholder node.
+  TF_ASSIGN_OR_RETURN(
+      Node * key_placeholder,
+      AddHostComputeKeyPlaceholder(xla_cluster_name, &host_graph));
+
+  // Step 2: rewrite `host_func_name`, replace key placeholder with an _Arg
+  // node.
+  TF_RETURN_IF_ERROR(ReplaceKeyPlaceholderWithArgNode(
+      xla_cluster_name, func_call_host_func_name, fld));
+
+  // Step 3: build a function call node with `host_func_name`, with
+  // `key_placeholder` as input.
+  NodeDefBuilder call_builder(absl::StrCat("oc_call_", func_call_node_name),
+                              func_call_host_func_name, fld);
+  call_builder.Input(key_placeholder->name(), 0, DT_STRING);
+  call_builder.Attr("device_ordinal", device_ordinal_value);
+  call_builder.Attr(kXlaHasHostTransferAttrName, true);
+  NodeDef call_def;
+  TF_RETURN_IF_ERROR(call_builder.Finalize(&call_def));
+  Status s;
+  Node* call_node = host_graph.AddNode(call_def, &s);
+  TF_RETURN_IF_ERROR(s);
+  host_graph.AddEdge(key_placeholder, 0, call_node, 0);
+
+  // Convert `host_graph` to function, and add a "device_ordinal" attr.
+  FunctionDef oc_host_graph_fdef;
+  TF_RETURN_IF_ERROR(GraphToFunctionDef(host_graph, host_graph_func_name,
+                                        &oc_host_graph_fdef));
+  if (fld->Find(host_graph_func_name)) {
+    TF_RETURN_IF_ERROR(
+        fld->ReplaceFunction(host_graph_func_name, oc_host_graph_fdef));
+  } else {
+    TF_RETURN_IF_ERROR(fld->AddFunctionDef(oc_host_graph_fdef));
+  }
+
+  return Status::OK();
+}
+
 Status ExtractOutsideCompilationForNodesWithAssociatedFunctions(
     Graph* g, const string& xla_cluster_attr_name,
     const string& outside_compilation_attr_name, const string& xla_cluster_name,
-    const std::map<string, int>& host_compute_core,
+    const std::map<string, int>& host_compute_core, FunctionLibraryRuntime* flr,
     FunctionLibraryDefinition* fld, std::vector<string>* host_graphs,
     std::vector<string>* shape_inference_graphs,
     bool* has_outside_compilation) {
-  std::vector<Node*> if_nodes, while_nodes;
+  std::vector<Node*> if_nodes, while_nodes, func_call_nodes;
   for (Node* n : g->nodes()) {
     if (n->type_string() == "If") {
       if_nodes.push_back(n);
     } else if (n->type_string() == "While") {
       while_nodes.push_back(n);
+    } else if (fld->Contains(n->type_string())) {
+      func_call_nodes.push_back(n);
+    } else if (n->type_string() == FunctionLibraryDefinition::kGradientOp) {
+      // Only gradient for user-defined function should be considered as
+      // function call node.
+      NameAttrList original_func;
+      TF_RETURN_IF_ERROR(GetNodeAttr(
+          n->def(), FunctionLibraryDefinition::kFuncAttr, &original_func));
+      if (fld->Contains(original_func.name())) {
+        func_call_nodes.push_back(n);
+      }
+    }
+  }
+
+  for (Node* n : func_call_nodes) {
+    // Extract outside compilation for the function call.
+    bool func_has_outside_compilation = false;
+    NameAttrList func;
+    func.set_name(n->type_string());
+    typedef protobuf::Map<string, AttrValue> AttrMap;
+    *func.mutable_attr() = AttrMap(n->attrs().begin(), n->attrs().end());
+    string new_func_name = absl::StrCat(n->name(), "_oc");
+    string host_func_name = absl::StrCat("oc_func_call_host_", n->name());
+    TF_RETURN_IF_ERROR(ExtractOutsideCompilationForFunction(
+        xla_cluster_attr_name, outside_compilation_attr_name, xla_cluster_name,
+        func, new_func_name, host_func_name, host_compute_core, flr, fld,
+        shape_inference_graphs, &func_has_outside_compilation));
+
+    // If the function call does not have outside compilation, nothing to do.
+    if (!func_has_outside_compilation) {
+      continue;
     }
+
+    *has_outside_compilation = true;
+
+    // Change `n` to call the new function directly.
+    NodeDefBuilder replace_builder(n->name(), new_func_name, fld);
+    for (const Edge* e : n->in_edges()) {
+      if (e->IsControlEdge()) {
+        continue;
+      }
+      replace_builder.Input(e->src()->name(), e->src_output(),
+                            e->src()->output_type(e->src_output()));
+    }
+    for (const auto& attr : n->attrs()) {
+      replace_builder.Attr(attr.first, attr.second);
+    }
+    NodeDef replace_def;
+    TF_RETURN_IF_ERROR(replace_builder.Finalize(&replace_def));
+    TF_ASSIGN_OR_RETURN(Node * replace, ReplaceNode(g, n, replace_def));
+    replace->AddAttr(kXlaTokenInputNodesAttrName,
+                     std::vector<string>{kXlaTokenArgNodeName});
+
+    // Build host side graph for the function call.
+    string oc_host_graph_name =
+        absl::StrCat("oc_func_host_graph_", replace->name());
+    TF_RETURN_IF_ERROR(
+        BuildHostGraphForFuncCallNode(replace->name(), xla_cluster_name,
+                                      host_func_name, oc_host_graph_name, fld));
+
+    // Record the host graph.
+    host_graphs->push_back(oc_host_graph_name);
   }
 
   for (Node* n : if_nodes) {
@@ -1251,12 +1369,12 @@ Status ExtractOutsideCompilationForNodesWithAssociatedFunctions(
     TF_RETURN_IF_ERROR(ExtractOutsideCompilationForFunction(
         xla_cluster_attr_name, outside_compilation_attr_name, xla_cluster_name,
         then_branch, then_branch_xla_func_name, then_branch_host_func_name,
-        host_compute_core, fld, shape_inference_graphs,
+        host_compute_core, flr, fld, shape_inference_graphs,
         &then_branch_has_outside_compilation));
     TF_RETURN_IF_ERROR(ExtractOutsideCompilationForFunction(
         xla_cluster_attr_name, outside_compilation_attr_name, xla_cluster_name,
         else_branch, else_branch_xla_func_name, else_branch_host_func_name,
-        host_compute_core, fld, shape_inference_graphs,
+        host_compute_core, flr, fld, shape_inference_graphs,
         &else_branch_has_outside_compilation));
 
     // If then/else branch do not have outside compilation, nothing to do.
@@ -1316,12 +1434,12 @@ Status ExtractOutsideCompilationForNodesWithAssociatedFunctions(
            body_xla_func_name = absl::StrCat(body.name(), "_oc");
     TF_RETURN_IF_ERROR(ExtractOutsideCompilationForFunction(
         xla_cluster_attr_name, outside_compilation_attr_name, xla_cluster_name,
-        cond, cond_xla_func_name, cond_host_func_name, host_compute_core, fld,
-        shape_inference_graphs, &cond_has_outside_compilation));
+        cond, cond_xla_func_name, cond_host_func_name, host_compute_core, flr,
+        fld, shape_inference_graphs, &cond_has_outside_compilation));
     TF_RETURN_IF_ERROR(ExtractOutsideCompilationForFunction(
         xla_cluster_attr_name, outside_compilation_attr_name, xla_cluster_name,
-        body, body_xla_func_name, body_host_func_name, host_compute_core, fld,
-        shape_inference_graphs, &body_has_outside_compilation));
+        body, body_xla_func_name, body_host_func_name, host_compute_core, flr,
+        fld, shape_inference_graphs, &body_has_outside_compilation));
 
     // If cond/body do not have outside compilation, nothing to do.
     if (!cond_has_outside_compilation && !body_has_outside_compilation) {
@@ -1469,17 +1587,27 @@ Status ExtractOutsideCompilationForFunction(
     const string& outside_compilation_attr_name, const string& xla_cluster_name,
     const NameAttrList& func_name_attrs, const string& new_func_name,
     const string& host_graph_func_name,
-    const std::map<string, int>& host_compute_core,
+    const std::map<string, int>& host_compute_core, FunctionLibraryRuntime* flr,
     FunctionLibraryDefinition* fld, std::vector<string>* shape_inference_graphs,
     bool* has_outside_compilation) {
+  // Convert the function to graph.
   const string& func_name = func_name_attrs.name();
-  const FunctionDef* fdef = fld->Find(func_name);
-  if (!fdef) {
-    return errors::Internal("Cannot find function ", func_name);
-  }
+  FunctionLibraryRuntime::Handle handle;
+  TF_RETURN_IF_ERROR(
+      flr->Instantiate(func_name, AttrSlice(&func_name_attrs.attr()), &handle));
+  Status ret_status = Status::OK();
+  auto cleanup_handle = gtl::MakeCleanup([&]() {
+    auto s = flr->ReleaseHandle(handle);
+    if (!s.ok()) {
+      ret_status.Update(s);
+    }
+  });
+  const FunctionBody* fbody = flr->GetFunctionBody(handle);
+
+  // Check if we have outside compilation nodes.
   *has_outside_compilation = false;
-  for (auto& node_def : fdef->node_def()) {
-    if (HasNodeAttr(node_def, outside_compilation_attr_name)) {
+  for (Node* n : fbody->graph->nodes()) {
+    if (HasNodeAttr(n->def(), outside_compilation_attr_name)) {
       *has_outside_compilation = true;
       break;
     }
@@ -1487,16 +1615,6 @@ Status ExtractOutsideCompilationForFunction(
   // We cannot early return here, because we might have outside compilation in
   // If/While function body.
 
-  // Convert the function to graph.
-  FunctionBody* fbody = nullptr;
-  TF_RETURN_IF_ERROR(FunctionDefToBodyHelper(
-      *fld->Find(func_name), AttrSlice(&func_name_attrs.attr()), fld,
-      [&](const string& op, const OpDef** sig) {
-        return fld->LookUpOpDef(op, sig);
-      },
-      &fbody));
-  std::unique_ptr<FunctionBody> fbody_deleter(fbody);
-
   // Preprocess edges between different outside compilations. They will be
   // restored in `ConstructHostGraph()`.
   TF_RETURN_IF_ERROR(PreprocessEdgesBetweenOutsideCompilations(
@@ -1553,16 +1671,11 @@ Status ExtractOutsideCompilationForFunction(
     TF_RETURN_IF_ERROR(ReplaceOrRemoveOutsideCompilationCallNode(
         graph_out.get(), n, host_compute_core));
   }
-  if (VLOG_IS_ON(4)) {
-    dump_graph::DumpGraphToFile(
-        absl::StrCat("extract_outside_compilation_for_func_after_", func_name),
-        *graph_out, fld);
-  }
 
   // Handle nodes with associated functions.
   TF_RETURN_IF_ERROR(ExtractOutsideCompilationForNodesWithAssociatedFunctions(
       graph_out.get(), xla_cluster_attr_name, outside_compilation_attr_name,
-      xla_cluster_name, host_compute_core, fld,
+      xla_cluster_name, host_compute_core, flr, fld,
       &outside_compilation_host_graphs, shape_inference_graphs,
       has_outside_compilation));
 
@@ -1580,20 +1693,31 @@ Status ExtractOutsideCompilationForFunction(
   FunctionDef updated_fdef;
   TF_RETURN_IF_ERROR(
       GraphToFunctionDef(*graph_out, new_func_name, &updated_fdef));
+  const FunctionDef* original_fdef = fld->Find(func_name);
+  if (original_fdef) {
+    for (const auto& attr : original_fdef->attr()) {
+      (*updated_fdef.mutable_attr())[attr.first] = attr.second;
+    }
+  }
   if (fld->Find(new_func_name)) {
     TF_RETURN_IF_ERROR(fld->ReplaceFunction(new_func_name, updated_fdef));
   } else {
     TF_RETURN_IF_ERROR(fld->AddFunctionDef(updated_fdef));
   }
+  if (VLOG_IS_ON(4)) {
+    dump_graph::DumpGraphToFile(
+        absl::StrCat("extract_outside_compilation_for_func_after_", func_name),
+        *graph_out, fld);
+  }
 
-  return Status::OK();
+  return ret_status;
 }
 
 Status ExtractOutsideCompilation(
     const string& xla_cluster_attr_name,
     const string& outside_compilation_attr_name,
     const std::unordered_map<string, XlaClusterInfo>& clusters, Graph* g,
-    FunctionLibraryDefinition* fld) {
+    FunctionLibraryRuntime* flr, FunctionLibraryDefinition* fld) {
   if (VLOG_IS_ON(4)) {
     dump_graph::DumpGraphToFile("extract_outside_compilation_before", *g, fld);
   }
@@ -1610,7 +1734,7 @@ Status ExtractOutsideCompilation(
     TF_RETURN_IF_ERROR(ExtractOutsideCompilationForFunction(
         xla_cluster_attr_name, outside_compilation_attr_name, xla_cluster_name,
         func_name_attrs, func_name_attrs.name(), host_graph_func_name,
-        host_compute_core, fld, &shape_inference_graphs,
+        host_compute_core, flr, fld, &shape_inference_graphs,
         &has_outside_compilation));
     TF_RETURN_IF_ERROR(
         ExpandHostGraphIntoMainGraph(g, fld, host_graph_func_name, n));
diff --git a/tensorflow/compiler/jit/extract_outside_compilation_pass.h b/tensorflow/compiler/jit/extract_outside_compilation_pass.h
index e07e7c5dd0..d64cc2a103 100644
--- a/tensorflow/compiler/jit/extract_outside_compilation_pass.h
+++ b/tensorflow/compiler/jit/extract_outside_compilation_pass.h
@@ -89,7 +89,7 @@ Status ExtractOutsideCompilationForFunction(
     const string& outside_compilation_attr_name, const string& xla_cluster_name,
     const NameAttrList& func_name_attrs, const string& new_func_name,
     const string& host_graph_func_name,
-    const std::map<string, int>& host_compute_core,
+    const std::map<string, int>& host_compute_core, FunctionLibraryRuntime* flr,
     FunctionLibraryDefinition* fld, std::vector<string>* shape_inference_graphs,
     bool* has_outside_compilation);
 
@@ -101,7 +101,7 @@ Status ExtractOutsideCompilation(
     const string& xla_cluster_attr_name,
     const string& outside_compilation_attr_name,
     const std::unordered_map<string, XlaClusterInfo>& clusters, Graph* g,
-    FunctionLibraryDefinition* fld);
+    FunctionLibraryRuntime* flr, FunctionLibraryDefinition* fld);
 
 }  // namespace tensorflow
 
diff --git a/tensorflow/compiler/jit/extract_outside_compilation_pass_test.cc b/tensorflow/compiler/jit/extract_outside_compilation_pass_test.cc
index e9a89e34e0..7c3a24feff 100644
--- a/tensorflow/compiler/jit/extract_outside_compilation_pass_test.cc
+++ b/tensorflow/compiler/jit/extract_outside_compilation_pass_test.cc
@@ -23,6 +23,7 @@ limitations under the License.
 #include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/compiler/jit/encapsulate_util.h"
 #include "tensorflow/compiler/xla/test.h"
+#include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/framework/common_shape_fns.h"
 #include "tensorflow/core/framework/function.h"
@@ -31,6 +32,8 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_shape.pb.h"
 #include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/public/session_options.h"
+#include "tensorflow/core/public/version.h"
 
 namespace tensorflow {
 
@@ -222,7 +225,42 @@ TEST(RewriteOutsideCompilationSubgraphFnTest, ShapesInferred) {
   EXPECT_EQ(shapes[0].dim_size(), 1);
 }
 
-TEST(ExtractOutsideCompilationForFunctionTest, Basic) {
+class ExtractOutsideCompilationForFunctionTest : public ::testing::Test {
+ public:
+  void SetUp() override {
+    SessionOptions session_options;
+    std::vector<std::unique_ptr<Device>> devices;
+    TF_CHECK_OK(DeviceFactory::AddDevices(
+        session_options, "/job:localhost/replica:0/task:0", &devices));
+    device_mgr_ = absl::make_unique<DeviceMgr>(std::move(devices));
+  }
+
+  Status ExtractOutsideCompilationTest(
+      const string &xla_cluster_attr_name,
+      const string &outside_compilation_attr_name,
+      const string &xla_cluster_name, const NameAttrList &func_name_attrs,
+      const string &new_func_name, const string &host_graph_func_name,
+      const std::map<string, int> &host_compute_core,
+      FunctionLibraryDefinition *fld,
+      std::vector<string> *shape_inference_graphs,
+      bool *has_outside_compilation) {
+    OptimizerOptions opts;
+    pflr_ = absl::make_unique<ProcessFunctionLibraryRuntime>(
+        device_mgr_.get(), Env::Default(), TF_GRAPH_DEF_VERSION, fld, opts,
+        /*default_thread_pool=*/nullptr, /*cluster_flr=*/nullptr);
+    auto flr = pflr_->GetFLR("/job:localhost/replica:0/task:0/cpu:0");
+    return ExtractOutsideCompilationForFunction(
+        xla_cluster_attr_name, outside_compilation_attr_name, xla_cluster_name,
+        func_name_attrs, new_func_name, host_graph_func_name, host_compute_core,
+        flr, fld, shape_inference_graphs, has_outside_compilation);
+  }
+
+ private:
+  std::unique_ptr<DeviceMgr> device_mgr_;
+  std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_;
+};
+
+TEST_F(ExtractOutsideCompilationForFunctionTest, Basic) {
   // Build the XLA computation func.
   // "const0"
   // "identity0" = "const0" (outside compilation cluster "0")
@@ -256,7 +294,7 @@ TEST(ExtractOutsideCompilationForFunctionTest, Basic) {
   NameAttrList name_attrs;
   name_attrs.set_name("cluster");
   *name_attrs.mutable_attr() = attrs;
-  TF_CHECK_OK(ExtractOutsideCompilationForFunction(
+  TF_CHECK_OK(ExtractOutsideCompilationTest(
       "_xla", "_oc", "cluster", name_attrs, "cluster_rewritten", "host_graph",
       host_compute_core, &fld, &shape_inference_graphs,
       &has_outside_compilation));
@@ -362,7 +400,7 @@ TEST(ExtractOutsideCompilationForFunctionTest, Basic) {
   }
 }
 
-TEST(ExtractOutsideCompilationForFunctionTest, NoHostGraph) {
+TEST_F(ExtractOutsideCompilationForFunctionTest, NoHostGraph) {
   // Build the XLA computation func.
   // "const0"
   FunctionDefLibrary fdl;
@@ -384,7 +422,7 @@ TEST(ExtractOutsideCompilationForFunctionTest, NoHostGraph) {
   NameAttrList name_attrs;
   name_attrs.set_name("cluster");
   *name_attrs.mutable_attr() = attrs;
-  TF_CHECK_OK(ExtractOutsideCompilationForFunction(
+  TF_CHECK_OK(ExtractOutsideCompilationTest(
       "_xla", "_oc", "cluster", name_attrs, "cluster_rewritten", "host_graph",
       host_compute_core, &fld, &shape_inference_graphs,
       &has_outside_compilation));
@@ -406,7 +444,7 @@ TEST(ExtractOutsideCompilationForFunctionTest, NoHostGraph) {
   EXPECT_EQ(host_graph->num_nodes(), 2);
 }
 
-TEST(ExtractOutsideCompilationForFunctionTest, XlaHostComputeRemoved) {
+TEST_F(ExtractOutsideCompilationForFunctionTest, XlaHostComputeRemoved) {
   // Build the XLA computation func.
   // "const0"
   // "const1" (outside compilation cluster "0")
@@ -432,7 +470,7 @@ TEST(ExtractOutsideCompilationForFunctionTest, XlaHostComputeRemoved) {
   NameAttrList name_attrs;
   name_attrs.set_name("cluster");
   *name_attrs.mutable_attr() = attrs;
-  TF_CHECK_OK(ExtractOutsideCompilationForFunction(
+  TF_CHECK_OK(ExtractOutsideCompilationTest(
       "_xla", "_oc", "cluster", name_attrs, "cluster_rewritten", "host_graph",
       host_compute_core, &fld, &shape_inference_graphs,
       &has_outside_compilation));
@@ -489,7 +527,7 @@ REGISTER_OP("XlaRecvFromHost")
     .Attr("key: string")
     .SetIsStateful();
 
-TEST(ExtractOutsideCompilationForFunctionTest, OutsideCompilationInIf) {
+TEST_F(ExtractOutsideCompilationForFunctionTest, OutsideCompilationInIf) {
   // Build the XLA computation func.
   // "const0" (bool)
   // "const1" (int32)
@@ -555,7 +593,7 @@ TEST(ExtractOutsideCompilationForFunctionTest, OutsideCompilationInIf) {
   NameAttrList name_attrs;
   name_attrs.set_name("cluster");
   *name_attrs.mutable_attr() = attrs;
-  TF_CHECK_OK(ExtractOutsideCompilationForFunction(
+  TF_CHECK_OK(ExtractOutsideCompilationTest(
       "_xla", "_oc", "cluster", name_attrs, "cluster_rewritten", "host_graph",
       host_compute_core, &fld, &shape_inference_graphs,
       &has_outside_compilation));
@@ -651,7 +689,7 @@ TEST(ExtractOutsideCompilationForFunctionTest, OutsideCompilationInIf) {
   }
 }
 
-TEST(ExtractOutsideCompilationForFunctionTest, OutsideCompilationInWhile) {
+TEST_F(ExtractOutsideCompilationForFunctionTest, OutsideCompilationInWhile) {
   // Build the XLA computation func.
   // "const0" (bool)
   // "while0" (input = "const0", cond = "cond_fn", body = "body_fn")
@@ -714,7 +752,7 @@ TEST(ExtractOutsideCompilationForFunctionTest, OutsideCompilationInWhile) {
   NameAttrList name_attrs;
   name_attrs.set_name("cluster");
   *name_attrs.mutable_attr() = attrs;
-  TF_CHECK_OK(ExtractOutsideCompilationForFunction(
+  TF_CHECK_OK(ExtractOutsideCompilationTest(
       "_xla", "_oc", "cluster", name_attrs, "cluster_rewritten", "host_graph",
       host_compute_core, &fld, &shape_inference_graphs,
       &has_outside_compilation));
@@ -782,4 +820,162 @@ TEST(ExtractOutsideCompilationForFunctionTest, OutsideCompilationInWhile) {
   }
 }
 
+TEST_F(ExtractOutsideCompilationForFunctionTest, OutsideCompilationInFunction) {
+  // Build the XLA computation func.
+  // "const0" (int32)
+  // "fn" (input = "const0")
+  FunctionDefLibrary fdl;
+  {
+    tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+    Output arg = ops::_Arg(s.WithOpName("arg"), DT_INT32, 0);
+    Output identity = ops::Identity(s.WithOpName("identity"), arg);
+    ops::_Retval retval(s.WithOpName("retval"), identity, 0);
+    std::unique_ptr<Graph> g(new Graph(OpRegistry::Global()));
+    TF_CHECK_OK(s.ToGraph(g.get()));
+    auto node_name_image = g->BuildNodeNameIndex();
+    node_name_image["identity"]->AddAttr("_oc", "0");
+    PartialTensorShape shape({2});
+    node_name_image["identity"]->AddAttr(
+        kXlaInferredShapesAttrName, std::vector<PartialTensorShape>{shape});
+
+    FunctionDef *true_fn_fdef = fdl.add_function();
+    TF_CHECK_OK(GraphToFunctionDef(*g, "fn", true_fn_fdef));
+  }
+  FunctionLibraryDefinition fld(OpRegistry::Global(), fdl);
+  {
+    std::unique_ptr<Graph> g(new Graph(&fld));
+
+    tensorflow::TensorProto tensor_proto;
+    tensor_proto.set_dtype(tensorflow::DT_INT32);
+    tensorflow::TensorShapeProto shape;
+    shape.add_dim()->set_size(2);
+    *tensor_proto.mutable_tensor_shape() = shape;
+    for (int i = 0; i < 2; ++i) {
+      tensor_proto.add_int_val(1);
+    }
+    NodeDef const_def;
+    TF_CHECK_OK(NodeDefBuilder("const", "Const")
+                    .Attr("dtype", DT_INT32)
+                    .Attr("value", tensor_proto)
+                    .Finalize(&const_def));
+    Status s;
+    Node *const_node = g->AddNode(const_def, &s);
+    TF_CHECK_OK(s);
+
+    NodeDef fn_def;
+    TF_CHECK_OK(NodeDefBuilder("fn", "fn", &fld)
+                    .Input("const", 0, DT_INT32)
+                    .Finalize(&fn_def));
+    Node *fn_node = g->AddNode(fn_def, &s);
+    TF_CHECK_OK(s);
+    g->AddEdge(const_node, 0, fn_node, 0);
+
+    NodeDef ret_def;
+    TF_CHECK_OK(NodeDefBuilder("ret", "_Retval")
+                    .Attr("index", 0)
+                    .Attr("T", DT_INT32)
+                    .Input("fn", 0, DT_INT32)
+                    .Finalize(&ret_def));
+    Node *ret_node = g->AddNode(ret_def, &s);
+    TF_CHECK_OK(s);
+    g->AddEdge(fn_node, 0, ret_node, 0);
+
+    FunctionDef *xla_fdef = fdl.add_function();
+    TF_CHECK_OK(GraphToFunctionDef(*g, "cluster", xla_fdef));
+    TF_CHECK_OK(fld.AddFunctionDef(*xla_fdef));
+  }
+
+  protobuf::Map<string, tensorflow::AttrValue> attrs;
+  std::map<string, int> host_compute_core;
+  std::vector<string> shape_inference_graphs;
+  bool has_outside_compilation;
+  NameAttrList name_attrs;
+  name_attrs.set_name("cluster");
+  *name_attrs.mutable_attr() = attrs;
+  TF_CHECK_OK(ExtractOutsideCompilationTest(
+      "_xla", "_oc", "cluster", name_attrs, "cluster_rewritten", "host_graph",
+      host_compute_core, &fld, &shape_inference_graphs,
+      &has_outside_compilation));
+
+  // Check host graph.
+  {
+    FunctionBody *host_fbody = nullptr;
+    AttrValue device_ordinal_temp_value;
+    device_ordinal_temp_value.set_i(0);
+    protobuf::Map<string, AttrValue> host_func_attrs;
+    host_func_attrs["device_ordinal"] = device_ordinal_temp_value;
+    TF_CHECK_OK(FunctionDefToBodyHelper(
+        *fld.Find("host_graph"), AttrSlice(&host_func_attrs), &fld,
+        [&](const string &op, const OpDef **sig) {
+          return fld.LookUpOpDef(op, sig);
+        },
+        &host_fbody));
+    std::unique_ptr<FunctionBody> host_fbody_deleter(host_fbody);
+    Graph *host_graph = host_fbody->graph;
+    auto node_name_index = host_graph->BuildNodeNameIndex();
+
+    // Verify we have call node for outside compilation in `fn`.
+    Node *call_node = node_name_index["oc_call_fn"];
+    EXPECT_NE(call_node, nullptr);
+
+    FunctionBody *call_fbody = nullptr;
+    TF_CHECK_OK(FunctionDefToBodyHelper(
+        *fld.Find("oc_func_call_host_fn"), AttrSlice(&host_func_attrs), &fld,
+        [&](const string &op, const OpDef **sig) {
+          return fld.LookUpOpDef(op, sig);
+        },
+        &call_fbody));
+    std::unique_ptr<FunctionBody> call_fbody_deleter(call_fbody);
+
+    // Verify we have _XlaRecvAtHost and _XlaSendFromHost nodes.
+    bool has_recv = false, has_send = false;
+    for (Node *n : call_fbody->graph->nodes()) {
+      if (n->type_string() == "_XlaRecvAtHost") {
+        has_recv = true;
+      } else if (n->type_string() == "_XlaSendFromHost") {
+        has_send = true;
+      }
+    }
+    EXPECT_TRUE(has_recv);
+    EXPECT_TRUE(has_send);
+  }
+
+  // Check XLA graph.
+  {
+    FunctionBody *xla_fbody = nullptr;
+    TF_CHECK_OK(FunctionDefToBodyHelper(
+        *fld.Find("cluster_rewritten"), AttrSlice(), &fld,
+        [&](const string &op, const OpDef **sig) {
+          return fld.LookUpOpDef(op, sig);
+        },
+        &xla_fbody));
+    std::unique_ptr<FunctionBody> xla_fbody_deleter(xla_fbody);
+    Graph *xla_graph = xla_fbody->graph;
+    auto node_name_index = xla_graph->BuildNodeNameIndex();
+
+    // Check that we have call node.
+    Node *fn_node = node_name_index["fn"];
+    EXPECT_NE(fn_node, nullptr);
+    EXPECT_EQ(fn_node->type_string(), "fn_oc");
+
+    FunctionBody *call_fbody = nullptr;
+    TF_CHECK_OK(FunctionDefToBodyHelper(
+        *fld.Find("fn_oc"), AttrSlice(), &fld,
+        [&](const string &op, const OpDef **sig) {
+          return fld.LookUpOpDef(op, sig);
+        },
+        &call_fbody));
+    std::unique_ptr<FunctionBody> call_fbody_deleter(call_fbody);
+
+    // Verify we have XlaHostCompute nodes.
+    bool has_hc = false;
+    for (Node *n : call_fbody->graph->nodes()) {
+      if (n->type_string() == "XlaHostCompute") {
+        has_hc = true;
+      }
+    }
+    EXPECT_TRUE(has_hc);
+  }
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/tf2xla/graph_compiler.cc b/tensorflow/compiler/tf2xla/graph_compiler.cc
index efb7574972..0c2bb02239 100644
--- a/tensorflow/compiler/tf2xla/graph_compiler.cc
+++ b/tensorflow/compiler/tf2xla/graph_compiler.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/dump_graph.h"
 #include "tensorflow/compiler/tf2xla/literal_util.h"
 #include "tensorflow/compiler/tf2xla/shape_util.h"
+#include "tensorflow/compiler/tf2xla/side_effect_util.h"
 #include "tensorflow/compiler/tf2xla/type_util.h"
 #include "tensorflow/compiler/tf2xla/xla_compiler.h"
 #include "tensorflow/compiler/tf2xla/xla_context.h"
@@ -191,6 +192,9 @@ Status GraphCompiler::CompileFunctionalNode(Node* n,
   // into the functions.
   XlaOpKernelContext xla_op_context(op_context);
 
+  XlaContext& context = XlaContext::Get(op_context);
+  auto* b = context.builder();
+
   XlaCompiler* compiler = xla_op_context.compiler();
 
   NameAttrList func;
@@ -219,8 +223,12 @@ Status GraphCompiler::CompileFunctionalNode(Node* n,
   TF_RETURN_IF_ERROR(
       PrepareArguments(&xla_op_context, graph.get(), expressions, &arguments));
 
+  bool add_token_input_output =
+      HasNodeAttr(n->def(), kXlaTokenInputNodesAttrName);
+
   XlaCompiler::CompileOptions compile_options;
   compile_options.is_entry_computation = false;
+  compile_options.add_token_input_output = add_token_input_output;
   XlaCompiler::CompilationResult result;
   TF_RETURN_IF_ERROR(
       compiler->CompileFunction(compile_options, func, arguments, &result));
@@ -234,9 +242,19 @@ Status GraphCompiler::CompileFunctionalNode(Node* n,
     }
     handles.push_back(expressions[i]->handle());
   }
-
-  XlaContext& context = XlaContext::Get(op_context);
-  auto* b = context.builder();
+  if (add_token_input_output) {
+    std::vector<string> token_input_nodes;
+    TF_RETURN_IF_ERROR(
+        GetNodeAttr(n->def(), kXlaTokenInputNodesAttrName, &token_input_nodes));
+    std::vector<xla::XlaOp> token_inputs;
+    for (const string& node_name : token_input_nodes) {
+      auto token_or = compiler->GetNodeToken(node_name);
+      TF_RETURN_IF_ERROR(token_or.status());
+      token_inputs.push_back(token_or.ConsumeValueOrDie());
+    }
+    xla::XlaOp token_input = xla::AfterAll(b, token_inputs);
+    handles.push_back(token_input);
+  }
 
   auto output_handle = xla::Call(b, *result.computation, handles);
   // The output handle of `Call` computation is a tuple type. Unzip it so
@@ -251,6 +269,10 @@ Status GraphCompiler::CompileFunctionalNode(Node* n,
       ++computation_output;
     }
   }
+  if (add_token_input_output) {
+    TF_RETURN_IF_ERROR(compiler->SetNodeToken(
+        n->name(), xla::GetTupleElement(output_handle, computation_output)));
+  }
   return b->first_error();
 }
 
diff --git a/tensorflow/compiler/tf2xla/side_effect_util.cc b/tensorflow/compiler/tf2xla/side_effect_util.cc
index a9144cfc4f..412f31adbb 100644
--- a/tensorflow/compiler/tf2xla/side_effect_util.cc
+++ b/tensorflow/compiler/tf2xla/side_effect_util.cc
@@ -58,6 +58,10 @@ Status SetDeviceOrdinalAttributeForNode(Node* node, int device_ordinal) {
       node->ClearAttr(attr_name);
       node->AddAttr(attr_name, branch_func);
     }
+  } else if (HasNodeAttr(node->def(), "device_ordinal")) {
+    // Function call node containing outside compilation.
+    node->ClearAttr("device_ordinal");
+    node->AddAttr("device_ordinal", device_ordinal);
   } else {
     return errors::Internal("Unknown node type to set 'device_ordinal': ",
                             node->DebugString());
-- 
GitLab


From 1b2e900f3cadc1a61d0c0a65c239e53afba7a7ec Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Tue, 8 Jan 2019 10:19:40 -0800
Subject: [PATCH 0348/2345] [XLA] Start moving over the DynamicSlice /
 DynamicUpdateSlice API

* Updated operation semantics document for new syntax.
* Added XlaBuilder APIs.
* Marked some of the old APIs as deprecated.
* Moved over a couple of slice tests to the new API.

Other uses in XLA tests and in TF2XLA will be moved over incrementally.

PiperOrigin-RevId: 228352804
---
 tensorflow/compiler/xla/client/xla_builder.cc | 62 +++++++++++++++++++
 tensorflow/compiler/xla/client/xla_builder.h  | 30 +++++++--
 .../compiler/xla/g3doc/operation_semantics.md | 59 +++++++++---------
 .../compiler/xla/service/hlo_instruction.h    |  2 +
 4 files changed, 121 insertions(+), 32 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc
index 433033cae6..328abbc7b9 100644
--- a/tensorflow/compiler/xla/client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_builder.cc
@@ -705,6 +705,34 @@ XlaOp XlaBuilder::DynamicSlice(const XlaOp& operand, const XlaOp& start_indices,
   });
 }
 
+XlaOp XlaBuilder::DynamicSlice(const XlaOp& operand,
+                               absl::Span<const XlaOp> start_indices,
+                               absl::Span<const int64> slice_sizes) {
+  return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    HloInstructionProto instr;
+
+    TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand));
+    std::vector<const Shape*> start_indices_shape_ptrs;
+    TF_ASSIGN_OR_RETURN(const auto& start_indices_shapes,
+                        GetOperandShapes(start_indices));
+    absl::c_transform(start_indices_shapes,
+                      std::back_inserter(start_indices_shape_ptrs),
+                      [](const Shape& shape) { return &shape; });
+    TF_ASSIGN_OR_RETURN(Shape shape,
+                        ShapeInference::InferDynamicSliceShape(
+                            operand_shape, start_indices_shapes, slice_sizes));
+    *instr.mutable_shape() = shape.ToProto();
+
+    for (int64 size : slice_sizes) {
+      instr.add_dynamic_slice_sizes(size);
+    }
+
+    std::vector<XlaOp> operands = {operand};
+    operands.insert(operands.end(), start_indices.begin(), start_indices.end());
+    return AddInstruction(std::move(instr), HloOpcode::kDynamicSlice, operands);
+  });
+}
+
 XlaOp XlaBuilder::DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
                                      const XlaOp& start_indices) {
   return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
@@ -724,6 +752,31 @@ XlaOp XlaBuilder::DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
   });
 }
 
+XlaOp XlaBuilder::DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
+                                     absl::Span<const XlaOp> start_indices) {
+  return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    HloInstructionProto instr;
+
+    TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand));
+    TF_ASSIGN_OR_RETURN(const Shape& update_shape, GetShape(update));
+    std::vector<const Shape*> start_indices_shape_ptrs;
+    TF_ASSIGN_OR_RETURN(const auto& start_indices_shapes,
+                        GetOperandShapes(start_indices));
+    absl::c_transform(start_indices_shapes,
+                      std::back_inserter(start_indices_shape_ptrs),
+                      [](const Shape& shape) { return &shape; });
+    TF_ASSIGN_OR_RETURN(Shape shape,
+                        ShapeInference::InferDynamicUpdateSliceShape(
+                            operand_shape, update_shape, start_indices_shapes));
+    *instr.mutable_shape() = shape.ToProto();
+
+    std::vector<XlaOp> operands = {operand, update};
+    operands.insert(operands.end(), start_indices.begin(), start_indices.end());
+    return AddInstruction(std::move(instr), HloOpcode::kDynamicUpdateSlice,
+                          operands);
+  });
+}
+
 XlaOp XlaBuilder::ConcatInDim(absl::Span<const XlaOp> operands,
                               int64 dimension) {
   return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
@@ -2751,12 +2804,21 @@ XlaOp DynamicSlice(const XlaOp& operand, const XlaOp& start_indices,
                    absl::Span<const int64> slice_sizes) {
   return operand.builder()->DynamicSlice(operand, start_indices, slice_sizes);
 }
+XlaOp DynamicSlice(const XlaOp& operand, absl::Span<const XlaOp> start_indices,
+                   absl::Span<const int64> slice_sizes) {
+  return operand.builder()->DynamicSlice(operand, start_indices, slice_sizes);
+}
 
 XlaOp DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
                          const XlaOp& start_indices) {
   return operand.builder()->DynamicUpdateSlice(operand, update, start_indices);
 }
 
+XlaOp DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
+                         absl::Span<const XlaOp> start_indices) {
+  return operand.builder()->DynamicUpdateSlice(operand, update, start_indices);
+}
+
 XlaOp ConcatInDim(XlaBuilder* builder, absl::Span<const XlaOp> operands,
                   int64 dimension) {
   return builder->ConcatInDim(operands, dimension);
diff --git a/tensorflow/compiler/xla/client/xla_builder.h b/tensorflow/compiler/xla/client/xla_builder.h
index ebef8e0879..8908d172fa 100644
--- a/tensorflow/compiler/xla/client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_builder.h
@@ -359,11 +359,18 @@ class XlaBuilder {
   XlaOp SliceInDim(const XlaOp& operand, int64 start_index, int64 limit_index,
                    int64 stride, int64 dimno);
 
+  ABSL_DEPRECATED("Use span-of-indices form instead")
   XlaOp DynamicSlice(const XlaOp& operand, const XlaOp& start_indices,
                      absl::Span<const int64> slice_sizes);
+  XlaOp DynamicSlice(const XlaOp& operand,
+                     absl::Span<const XlaOp> start_indices,
+                     absl::Span<const int64> slice_sizes);
 
+  ABSL_DEPRECATED("Use span-of-indices form instead")
   XlaOp DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
                            const XlaOp& start_indices);
+  XlaOp DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
+                           absl::Span<const XlaOp> start_indices);
 
   XlaOp ConcatInDim(absl::Span<const XlaOp> operands, int64 dimension);
 
@@ -875,9 +882,14 @@ class XlaBuilder {
 
   friend XlaOp DynamicSlice(const XlaOp& operand, const XlaOp& start_indices,
                             absl::Span<const int64> slice_sizes);
+  friend XlaOp DynamicSlice(const XlaOp& operand,
+                            absl::Span<const XlaOp> start_indices,
+                            absl::Span<const int64> slice_sizes);
 
   friend XlaOp DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
                                   const XlaOp& start_indices);
+  friend XlaOp DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
+                                  absl::Span<const XlaOp> start_indices);
 
   friend XlaOp ConcatInDim(XlaBuilder* builder,
                            absl::Span<const XlaOp> operands, int64 dimension);
@@ -1319,10 +1331,15 @@ XlaOp SliceInDim(const XlaOp& operand, int64 start_index, int64 limit_index,
 // The size of the slice in each dimension is passed in 'slice_sizes',
 // which specify the end point of exclusive slice intervals in each
 // dimension [start, start + size).
-// The shape of 'start_indices' must be rank == 1, with dimension size
-// equal to the rank of the 'operand'.
+// The shape of each element of 'start_indices' must be scalar, with the span
+// size equal to the rank of the 'operand'. All elements of 'start_indices' must
+// have the same shape.
 // Slice index calculations are computed modulo input dimension sizes to
 // prevent dynamic start indices from generating out-of-bound array accesses.
+XlaOp DynamicSlice(const XlaOp& operand, absl::Span<const XlaOp> start_indices,
+                   absl::Span<const int64> slice_sizes);
+
+ABSL_DEPRECATED("Use span-of-indices form instead")
 XlaOp DynamicSlice(const XlaOp& operand, const XlaOp& start_indices,
                    absl::Span<const int64> slice_sizes);
 
@@ -1338,10 +1355,15 @@ XlaOp DynamicSlice(const XlaOp& operand, const XlaOp& start_indices,
 //   [4 5 6]  => DynamicUpdateslice(data, update, start)   => [4 10 11]
 //   [7 8 9]                                                  [7 8  9 ]
 //
-// The shape of 'start_indices' must be rank == 1, with dimension size
-// equal to the rank of the 'operand'.
+// The shape of each element of 'start_indices' must be scalar, with the span
+// size equal to the rank of the 'operand'. All elements of 'start_indices' must
+// have the same shape.
 // Slice index calculations are computed modulo update dimension sizes to
 // prevent dynamic start indices from generating out-of-bound array accesses.
+XlaOp DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
+                         absl::Span<const XlaOp> start_indices);
+
+ABSL_DEPRECATED("Use span-of-indices form instead")
 XlaOp DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
                          const XlaOp& start_indices);
 
diff --git a/tensorflow/compiler/xla/g3doc/operation_semantics.md b/tensorflow/compiler/xla/g3doc/operation_semantics.md
index 78d4617017..7377ed729b 100644
--- a/tensorflow/compiler/xla/g3doc/operation_semantics.md
+++ b/tensorflow/compiler/xla/g3doc/operation_semantics.md
@@ -944,21 +944,21 @@ dimension: [start, start + size). The shape of `start_indices` must be rank ==
 
 <b> `DynamicSlice(operand, start_indices, size_indices)` </b>
 
-| Arguments       | Type                | Semantics                           |
-| --------------- | ------------------- | ----------------------------------- |
-| `operand`       | `XlaOp`             | N dimensional array of type T       |
-| `start_indices` | `XlaOp`             | Rank 1 array of N integers          |
-:                 :                     : containing the starting indices of  :
-:                 :                     : the slice for each dimension. Value :
-:                 :                     : must be greater than or equal to    :
-:                 :                     : zero.                               :
-| `size_indices`  | `ArraySlice<int64>` | List of N integers containing the   |
-:                 :                     : slice size for each dimension. Each :
-:                 :                     : value must be strictly greater than :
-:                 :                     : zero, and start + size must be less :
-:                 :                     : than or equal to the size of the    :
-:                 :                     : dimension to avoid wrapping modulo  :
-:                 :                     : dimension size.                     :
+| Arguments       | Type                  | Semantics                          |
+| --------------- | --------------------- | ---------------------------------- |
+| `operand`       | `XlaOp`               | N dimensional array of type T      |
+| `start_indices` | sequence of N `XlaOp` | List of N scalar integers          |
+:                 :                       : containing the starting indices of :
+:                 :                       : the slice for each dimension.      :
+:                 :                       : Value must be greater than or      :
+:                 :                       : equal to zero.                     :
+| `size_indices`  | `ArraySlice<int64>`   | List of N integers containing the  |
+:                 :                       : slice size for each dimension.     :
+:                 :                       : Each value must be strictly        :
+:                 :                       : greater than zero, and start +     :
+:                 :                       : size must be less than or equal to :
+:                 :                       : the size of the dimension to avoid :
+:                 :                       : wrapping modulo dimension size.    :
 
 The effective slice indices are computed by applying the following
 transformation for each index `i` in `[1, N)` before performing the slice:
@@ -1009,19 +1009,22 @@ the rank of `operand`.
 
 <b> `DynamicUpdateSlice(operand, update, start_indices)` </b>
 
-| Arguments       | Type    | Semantics                                        |
-| --------------- | ------- | ------------------------------------------------ |
-| `operand`       | `XlaOp` | N dimensional array of type T                    |
-| `update`        | `XlaOp` | N dimensional array of type T containing the     |
-:                 :         : slice update. Each dimension of update shape     :
-:                 :         : must be strictly greater than zero, and start +  :
-:                 :         : update must be less than or equal to the operand :
-:                 :         : size for each dimension to avoid generating      :
-:                 :         : out-of-bounds update indices.                    :
-| `start_indices` | `XlaOp` | Rank 1 array of N integers containing the        |
-:                 :         : starting indices of the slice for each           :
-:                 :         : dimension. Value must be greater than or equal   :
-:                 :         : to zero.                                         :
+| Arguments       | Type                  | Semantics                          |
+| --------------- | --------------------- | ---------------------------------- |
+| `operand`       | `XlaOp`               | N dimensional array of type T      |
+| `update`        | `XlaOp`               | N dimensional array of type T      |
+:                 :                       : containing the slice update. Each  :
+:                 :                       : dimension of update shape must be  :
+:                 :                       : strictly greater than zero, and    :
+:                 :                       : start + update must be less than   :
+:                 :                       : or equal to the operand size for   :
+:                 :                       : each dimension to avoid generating :
+:                 :                       : out-of-bounds update indices.      :
+| `start_indices` | sequence of N `XlaOp` | List of N scalar integers          |
+:                 :                       : containing the starting indices of :
+:                 :                       : the slice for each dimension.      :
+:                 :                       : Value must be greater than or      :
+:                 :                       : equal to zero.                     :
 
 The effective slice indices are computed by applying the following
 transformation for each index `i` in `[1, N)` before performing the slice:
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h
index 2827eed0df..789ace6a26 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.h
+++ b/tensorflow/compiler/xla/service/hlo_instruction.h
@@ -556,6 +556,7 @@ class HloInstruction {
   // Creates a slice instruction, where the first operand is sliced by
   // start indices specified in the second operand, and by size specified in
   // 'slice_sizes'.
+  ABSL_DEPRECATED("Use span-of-indices form instead")
   static std::unique_ptr<HloInstruction> CreateDynamicSlice(
       const Shape& shape, HloInstruction* operand,
       HloInstruction* start_indices, absl::Span<const int64> slice_sizes);
@@ -567,6 +568,7 @@ class HloInstruction {
 
   // Creates a dynamic update slice instruction, which updates a slice
   // of 'operand' with 'update' and 'start_indices'.
+  ABSL_DEPRECATED("Use span-of-indices form instead")
   static std::unique_ptr<HloInstruction> CreateDynamicUpdateSlice(
       const Shape& shape, HloInstruction* operand, HloInstruction* update,
       HloInstruction* start_indices);
-- 
GitLab


From 3b6f11930a3c6fe91c43fc92aeb303e8cefffc5d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 10:22:53 -0800
Subject: [PATCH 0349/2345] Automated rollback of commit
 ec569e692435fb68191553fd7ffa5b633fdf1ef2

PiperOrigin-RevId: 228353495
---
 tensorflow/core/grappler/optimizers/BUILD     |   4 -
 .../optimizers/dependency_optimizer.cc        | 456 +++++++++++-------
 .../optimizers/dependency_optimizer.h         |  24 +-
 .../optimizers/dependency_optimizer_test.cc   |   9 +-
 4 files changed, 291 insertions(+), 202 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 5a328a91c5..7e29cee86a 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -303,18 +303,14 @@ cc_library(
         ":constant_folding",
         ":graph_optimizer",
         "//tensorflow/core:framework",
-        "//tensorflow/core:graph",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/grappler:grappler_item",
-        "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/costs:graph_properties",
         "//tensorflow/core/grappler/utils:topological_sort",
-        "@com_google_absl//absl/container:flat_hash_map",
-        "@com_google_absl//absl/container:flat_hash_set",
     ],
 )
 
diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer.cc b/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
index 9bceb6b906..7fee3ae9d5 100644
--- a/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
@@ -18,14 +18,10 @@ limitations under the License.
 #include <unordered_map>
 #include <unordered_set>
 
-#include "absl/container/flat_hash_map.h"
-#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/op.h"
-#include "tensorflow/core/graph/tensor_id.h"
 #include "tensorflow/core/grappler/costs/graph_properties.h"
 #include "tensorflow/core/grappler/grappler_item.h"
-#include "tensorflow/core/grappler/mutable_graph_view.h"
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/grappler/optimizers/constant_folding.h"
 #include "tensorflow/core/grappler/utils.h"
@@ -42,15 +38,20 @@ namespace grappler {
 
 namespace {
 
-// Builds a map from the &graph->node(i) to i.
-absl::flat_hash_map<const NodeDef*, int> BuildNodeToIdx(const GraphDef& graph) {
-  // Set up &node -> index map.
-  absl::flat_hash_map<const NodeDef*, int> node_to_idx;
-  for (int i = 0; i < graph.node_size(); ++i) {
-    const NodeDef& node = graph.node(i);
-    node_to_idx[&node] = i;
+bool RemoveInput(NodeDef* node, const string& input, NodeMap* node_map) {
+  bool removed_input = false;
+  int pos = 0;
+  while (pos < node->input_size()) {
+    if (node->input(pos) == input) {
+      node->mutable_input()->SwapElements(pos, node->input_size() - 1);
+      node->mutable_input()->RemoveLast();
+      node_map->RemoveOutput(NodeName(input), node->name());
+      removed_input = true;
+    } else {
+      ++pos;
+    }
   }
-  return node_to_idx;
+  return removed_input;
 }
 
 }  // namespace
@@ -67,10 +68,7 @@ bool DependencyOptimizer::SafeToRemoveIdentity(const NodeDef& node) const {
     // The output values of this node may be needed.
     return false;
   }
-
-  MutableGraphView::OutputPort port = graph_view_->GetRegularFanin(
-      MutableGraphView::InputPort(const_cast<NodeDef*>(&node), 0));
-  NodeDef* input = port.node;
+  const NodeDef* input = node_map_->GetNode(NodeName(node.input(0)));
   CHECK(input != nullptr) << "node = " << node.name()
                           << " input = " << node.input(0);
   // Don't remove Identity nodes corresponding to Variable reads or following
@@ -81,23 +79,20 @@ bool DependencyOptimizer::SafeToRemoveIdentity(const NodeDef& node) const {
     // Don't turn Identity nodes following Switch into NoOp or remove them
     // if it requires anchoring a control dependencies the Switch node, which
     // is not valid.
-    MutableGraphView::OutputPort control_port(const_cast<NodeDef*>(&node),
-                                              Graph::kControlSlot);
-    if (!graph_view_->GetFanout(control_port).empty()) {
+    if (str_util::StartsWith(node.name(), kConstantFoldingCtrl)) {
+      // TODO(rmlarsen): Try to remove this artificial contraint.
       return false;
     }
   }
-  bool node_has_multiple_inputs =
-      graph_view_->NumFanins(node, /*include_controlling_nodes=*/true) > 1;
-  for (auto consumer :
-       graph_view_->GetFanouts(node, /*include_controlled_nodes=*/true)) {
-    if (node_has_multiple_inputs && IsMerge(*consumer.node)) {
+  for (auto consumer : node_map_->GetOutputs(node.name())) {
+    if (node.input_size() > 1 && IsMerge(*consumer)) {
       return false;
     }
     if (IsSwitch(*input)) {
-      if (graph_view_->HasFanin(*consumer.node,
-                                {node.name(), Graph::kControlSlot})) {
-        return false;
+      for (const string& consumer_input : consumer->input()) {
+        if (consumer_input == AsControlDependency(node.name())) {
+          return false;
+        }
       }
     }
   }
@@ -131,7 +126,7 @@ bool DependencyOptimizer::SafeToConvertToNoOp(const NodeDef& node) const {
   if (!SafeToRemoveIdentity(node)) {
     return false;
   }
-  if (graph_view_->NumFanouts(node, /*include_controlled_nodes=*/false) > 0) {
+  if (NumNonControlOutputs(node, *node_map_) > 0) {
     // The output values of this node may be needed.
     return false;
   }
@@ -139,56 +134,61 @@ bool DependencyOptimizer::SafeToConvertToNoOp(const NodeDef& node) const {
 }
 
 int DependencyOptimizer::NumEdgesIfBypassed(
-    const NodeDef& node, int num_fanins,
-    const absl::flat_hash_set<MutableGraphView::Edge>& fanout_edges) const {
+    const NodeDef& node, const std::vector<NodeDef*>& output_nodes) const {
   const bool is_multi_input_identity_n =
       IsIdentityN(node) && !IsIdentityNSingleInput(node);
-  const int num_fanouts = fanout_edges.size();
+  const int num_outputs = output_nodes.size();
+  const int num_inputs = node.input_size();
 
   if (is_multi_input_identity_n) {
     // multi-input identity_n with input/output control dependencies will likely
     // increase number of edges after optimization.
     int num_edges_if_bypassed(0);
-    int num_non_controlling_fanins =
-        graph_view_->NumFanins(node, /*include_controlling_nodes=*/false);
-    num_edges_if_bypassed += num_non_controlling_fanins;
-    num_edges_if_bypassed +=
-        (num_fanins - num_non_controlling_fanins) * num_fanouts;
-
-    for (const auto& fanout : fanout_edges) {
-      if (fanout.dst.port_id == Graph::kControlSlot) {
-        num_edges_if_bypassed += num_fanins;
+    for (string input_node_name : node.input()) {
+      if (IsControlInput(input_node_name)) {
+        num_edges_if_bypassed += num_outputs;
       } else {
         ++num_edges_if_bypassed;
       }
     }
+
+    for (auto consumer : output_nodes) {
+      for (int j = 0; j < consumer->input_size(); ++j) {
+        const TensorId consumer_input = ParseTensorName(consumer->input(j));
+        if (consumer_input.node() == node.name()) {
+          if (IsControlInput(consumer_input)) {
+            num_edges_if_bypassed += num_inputs;
+          } else {
+            ++num_edges_if_bypassed;
+          }
+        }
+      }
+    }
     return num_edges_if_bypassed;
   } else {
-    return num_fanins * num_fanouts;
+    return num_inputs * num_outputs;
   }
 }
 
 bool DependencyOptimizer::BypassingNodeIsBeneficial(
-    const NodeDef& node,
-    const absl::flat_hash_set<MutableGraphView::OutputPort>& fanins,
-    const absl::flat_hash_set<MutableGraphView::Edge>& fanout_edges) const {
+    const NodeDef& node, const std::vector<NodeDef*>& input_nodes,
+    const std::vector<NodeDef*>& output_nodes) const {
   const bool is_identity = IsIdentity(node) || IsIdentityNSingleInput(node);
   const bool is_multi_input_identity_n =
       IsIdentityN(node) && !IsIdentityNSingleInput(node);
-  const int num_outputs = fanout_edges.size();
-  const int num_inputs = fanins.size();
+  const int num_outputs = output_nodes.size();
+  const int num_inputs = node.input_size();
 
-  if (NumEdgesIfBypassed(node, num_inputs, fanout_edges) >
-      num_inputs + num_outputs) {
+  if (NumEdgesIfBypassed(node, output_nodes) > num_inputs + num_outputs) {
     return false;
   }
 
   // Make sure that we don't increase the number of edges that cross
   // device boundaries.
   if ((num_inputs == 1 && num_outputs > 1 &&
-       fanins.begin()->node->device() != node.device()) ||
+       input_nodes[0]->device() != node.device()) ||
       (num_inputs > 1 && num_outputs == 1 &&
-       fanout_edges.begin()->dst.node->device() != node.device())) {
+       output_nodes[0]->device() != node.device())) {
     return false;
   }
 
@@ -197,12 +197,12 @@ bool DependencyOptimizer::BypassingNodeIsBeneficial(
   // cost before and after.
   const string& node_dev = node.device();
   int num_cross_in = 0;
-  for (const auto& fanin : fanins) {
-    num_cross_in += static_cast<int>(fanin.node->device() != node_dev);
+  for (NodeDef* input_node : input_nodes) {
+    num_cross_in += static_cast<int>(input_node->device() != node_dev);
   }
   int num_cross_out = 0;
-  for (const auto& fanout : fanout_edges) {
-    num_cross_out += static_cast<int>(fanout.dst.node->device() != node_dev);
+  for (NodeDef* output_node : output_nodes) {
+    num_cross_out += static_cast<int>(output_node->device() != node_dev);
   }
 
   if ((is_identity || is_multi_input_identity_n) && num_cross_in > 0 &&
@@ -216,10 +216,10 @@ bool DependencyOptimizer::BypassingNodeIsBeneficial(
   // Make sure we do not increase the number of device crossings.
   const int num_cross_before = num_cross_in + num_cross_out;
   int num_cross_after = 0;
-  for (const auto& fanin : fanins) {
-    for (const auto& fanout : fanout_edges) {
+  for (NodeDef* input_node : input_nodes) {
+    for (NodeDef* output_node : output_nodes) {
       num_cross_after +=
-          static_cast<int>(fanin.node->device() != fanout.dst.node->device());
+          static_cast<int>(input_node->device() != output_node->device());
     }
   }
   if (num_cross_after > num_cross_before) {
@@ -228,32 +228,45 @@ bool DependencyOptimizer::BypassingNodeIsBeneficial(
   return true;
 }
 
-void DependencyOptimizer::OptimizeNode(NodeDef* node,
-                                       SetVector<NodeDef*>* nodes_to_simplify,
-                                       std::set<string>* nodes_to_delete) {
-  const string node_name = node->name();
+void DependencyOptimizer::OptimizeNode(int node_idx,
+                                       SetVector<int>* nodes_to_simplify,
+                                       std::set<int>* nodes_to_delete) {
+  NodeDef* node = optimized_graph_->mutable_node(node_idx);
   const bool is_noop = IsNoOp(*node);
   const bool is_identity = IsIdentity(*node) || IsIdentityNSingleInput(*node);
   const bool is_multi_input_identity =
       IsIdentityN(*node) && !IsIdentityNSingleInput(*node);
+  const string node_name = node->name();
   // Constant nodes with no input control dependency are always executed early,
   // so we can prune all their output control dependencies.
-  if (IsConstant(*node) &&
-      graph_view_->NumFanins(*node, /*include_controlling_nodes=*/true) == 0) {
-    for (const auto& fanout :
-         graph_view_->GetFanouts(*node, /*include_controlled_nodes=*/true)) {
-      if (graph_view_->RemoveFanin(fanout.node->name(),
-                                   {node_name, Graph::kControlSlot})) {
-        nodes_to_simplify->PushBack(fanout.node);
+  if (IsConstant(*node) && node->input_size() == 0) {
+    const std::set<NodeDef*> output_nodes = node_map_->GetOutputs(node_name);
+    for (NodeDef* fanout : output_nodes) {
+      bool optimize_fanout = false;
+      bool data_connection = false;
+      for (int i = fanout->input_size() - 1; i >= 0; --i) {
+        const TensorId input_tensor = ParseTensorName(fanout->input(i));
+        if (input_tensor.node() == node_name) {
+          if (input_tensor.index() < 0) {
+            fanout->mutable_input()->SwapElements(i, fanout->input_size() - 1);
+            fanout->mutable_input()->RemoveLast();
+            optimize_fanout = true;
+          } else {
+            data_connection = true;
+          }
+        }
+      }
+      if (optimize_fanout) {
+        nodes_to_simplify->PushBack(node_to_idx_[fanout]);
+        if (!data_connection) {
+          node_map_->RemoveOutput(node_name, fanout->name());
+        }
       }
     }
-
-    if (graph_view_->NumFanouts(*node, /*include_controlled_nodes=*/true) ==
-            0 &&
-        fetch_nodes_known_ &&
+    if (node_map_->GetOutputs(node_name).empty() && fetch_nodes_known_ &&
         nodes_to_preserve_.find(node_name) == nodes_to_preserve_.end()) {
       // Mark the node for deletion.
-      nodes_to_delete->insert(node->name());
+      nodes_to_delete->insert(node_to_idx_[node]);
     }
     return;
   }
@@ -264,23 +277,33 @@ void DependencyOptimizer::OptimizeNode(NodeDef* node,
             << ") with NoOp.";
     // The outputs of this node are not consumed. Replace its inputs with
     // control dependencies and replace the op itself with the NoOp op.
-    int num_non_controlling_fanins =
-        graph_view_->NumFanins(*node, /*include_controlling_nodes=*/false);
-    if (num_non_controlling_fanins > 0) {
-      absl::flat_hash_set<string> non_controlling_fanins(
-          node->input().begin(),
-          node->input().begin() + num_non_controlling_fanins);
-      graph_view_->RemoveAllFanins(node_name,
-                                   /*keep_controlling_fanins=*/true);
-      for (const string& fanin : non_controlling_fanins) {
-        TensorId tensor_id = ParseTensorName(fanin);
-        graph_view_->AddControllingFanin(node_name, tensor_id);
-        nodes_to_simplify->PushBack(graph_view_->GetNode(tensor_id.node()));
+    std::unordered_set<string> ctrl_inputs;
+    int pos = 0;
+    while (pos < node->input_size()) {
+      const string old_input = node->input(pos);
+      if (IsControlInput(old_input)) {
+        if (!ctrl_inputs.insert(old_input).second) {
+          // We found a duplicate control input. Remove it.
+          node->mutable_input()->SwapElements(pos, node->input_size() - 1);
+          node->mutable_input()->RemoveLast();
+        } else {
+          ++pos;
+        }
+        continue;
       }
+      // Replace a normal input with a control input.
+      const string ctrl_input = ConstantFolding::AddControlDependency(
+          old_input, optimized_graph_, node_map_.get());
+      ctrl_inputs.insert(ctrl_input);
+      node->set_input(pos, ctrl_input);
+      node_map_->UpdateInput(node_name, old_input, ctrl_input);
+      const NodeDef* old_input_node = node_map_->GetNode(old_input);
+      nodes_to_simplify->PushBack(node_to_idx_[old_input_node]);
+      ++pos;
     }
     node->set_op("NoOp");
     node->clear_attr();
-    nodes_to_simplify->PushBack(node);
+    nodes_to_simplify->PushBack(node_to_idx_[node]);
     return;
   }
 
@@ -334,80 +357,110 @@ void DependencyOptimizer::OptimizeNode(NodeDef* node,
 
   if (is_noop || ((is_identity || is_multi_input_identity) &&
                   SafeToRemoveIdentity(*node))) {
-    auto fanins =
-        graph_view_->GetFanins(*node, /*include_controlling_nodes=*/true);
-    auto fanout_edges =
-        graph_view_->GetFanoutEdges(*node, /*include_controlled_edges=*/true);
+    const auto& output_node_set = node_map_->GetOutputs(node_name);
+    const std::vector<NodeDef*> output_nodes(output_node_set.begin(),
+                                             output_node_set.end());
+    const int num_inputs = node->input_size();
+    std::vector<NodeDef*> input_nodes;
+    for (int i = 0; i < num_inputs; ++i) {
+      NodeDef* input_node = node_map_->GetNode(node->input(i));
+      if (input_node == nullptr) {
+        LOG(ERROR) << "Invalid input " << node->input(i);
+        return;
+      }
+      input_nodes.push_back(input_node);
+    }
 
-    if (!BypassingNodeIsBeneficial(*node, fanins, fanout_edges)) {
+    if (!BypassingNodeIsBeneficial(*node, input_nodes, output_nodes)) {
       return;
     }
 
-    int num_non_controlling_fanins =
-        graph_view_->NumFanins(*node, /*include_controlling_nodes=*/false);
     VLOG(1) << "***** Rerouting input around\n" << node->DebugString();
     // Now remove the node and re-wire its inputs to its outputs.
-    for (auto fanout_edge : fanout_edges) {
+    for (auto consumer : output_nodes) {
       bool updated_consumer = false;
-      NodeDef* consumer = fanout_edge.dst.node;
       VLOG(1) << "consumer before:\n" << consumer->DebugString();
-      if (fanout_edge.src.port_id == Graph::kControlSlot) {
-        updated_consumer = graph_view_->RemoveFanin(
-            consumer->name(), {node_name, Graph::kControlSlot});
-        if (updated_consumer) {
-          // Add all non controlling fanins of node as controlling fanins to
-          // consumer.
-          for (int i = 0; i < num_non_controlling_fanins; ++i) {
-            TensorId input = ParseTensorName(node->input(i));
-            if (graph_view_->AddControllingFanin(consumer->name(), input)) {
-              nodes_to_simplify->PushBack(graph_view_->GetNode(input.node()));
+      for (int i = 0; i < num_inputs; ++i) {
+        const NodeDef* input = input_nodes[i];
+        // Forward dependency from input to consumer if it doesn't already
+        // depend on it.
+        if ((is_identity && i == 0) ||
+            (is_multi_input_identity && !IsControlInput(node->input(i)))) {
+          // Replace regular input from Identity node.
+          string new_input;
+          const string& input_to_forward = node->input(i);
+          CHECK(!IsControlInput(input_to_forward));
+          for (int j = 0; j < consumer->input_size(); ++j) {
+            const TensorId old_input = ParseTensorName(consumer->input(j));
+            if (old_input.node() == node_name) {
+              if (old_input.index() == i) {
+                // Regular input
+                new_input = input_to_forward;
+                node_map_->UpdateInput(consumer->name(), old_input.ToString(),
+                                       new_input);
+                consumer->set_input(j, new_input);
+              } else if (old_input.index() == -1) {
+                // Control dependency
+                new_input = AsControlDependency(NodeName(input_to_forward));
+                node_map_->UpdateInput(consumer->name(), old_input.ToString(),
+                                       new_input);
+                consumer->set_input(j, new_input);
+              }
             }
           }
+          updated_consumer = true;
+        } else {
+          // Forward dependency from input to consumer if it doesn't already
+          // depend on it.
+          if (node_map_->GetOutputs(input->name()).count(consumer) == 0) {
+            consumer->add_input(AsControlDependency(input->name()));
+            node_map_->AddOutput(input->name(), consumer->name());
+            nodes_to_simplify->PushBack(node_to_idx_[input]);
+            updated_consumer = true;
+          }
         }
-      } else {
-        updated_consumer = graph_view_->UpdateFanin(
-            consumer->name(), {node_name, fanout_edge.src.port_id},
-            ParseTensorName(node->input(fanout_edge.src.port_id)));
       }
+      // Remove dependency on node from consumer.
+      updated_consumer |= RemoveInput(consumer, AsControlDependency(node_name),
+                                      node_map_.get());
       if (updated_consumer) {
-        // Forward all controlling fanins of node to consumer.
-        for (int i = num_non_controlling_fanins; i < node->input_size(); ++i) {
-          TensorId input = ParseTensorName(node->input(i));
-          if (graph_view_->AddFanin(consumer->name(), input)) {
-            nodes_to_simplify->PushBack(graph_view_->GetNode(input.node()));
-          }
-        }
-        nodes_to_simplify->PushBack(consumer);
+        nodes_to_simplify->PushBack(node_to_idx_[consumer]);
       }
       VLOG(1) << "consumer after:\n" << consumer->DebugString();
     }
+    node_map_->RemoveOutputs(node_name);
     if (fetch_nodes_known_ &&
         nodes_to_preserve_.find(node_name) == nodes_to_preserve_.end()) {
       // Mark the node for deletion.
-      nodes_to_delete->insert(node_name);
+      nodes_to_delete->insert(node_idx);
 
       // Disconnect the node from its inputs to enable further optimizations.
-      graph_view_->RemoveAllFanins(node_name,
-                                   /*keep_controlling_fanins=*/false);
+      node_map_->RemoveInputs(node_name);
+      node->clear_input();
     }
   }
 }
 
+void DependencyOptimizer::CleanControlInputs() {
+  for (int i = 0; i < optimized_graph_->node_size(); ++i) {
+    DedupControlInputs(optimized_graph_->mutable_node(i));
+  }
+}
+
 Status DependencyOptimizer::OptimizeDependencies() {
-  SetVector<NodeDef*> nodes_to_simplify;
-  std::set<string> nodes_to_delete;
-  for (int i = 0; i < graph_view_->graph()->node_size(); ++i) {
-    NodeDef* node = graph_view_->graph()->mutable_node(i);
-    if (IsNoOp(*node) || IsIdentity(*node) || IsIdentityN(*node) ||
-        IsConstant(*node) || SafeToConvertToNoOp(*node)) {
-      nodes_to_simplify.PushBack(node);
+  SetVector<int> nodes_to_simplify;
+  std::set<int> nodes_to_delete;
+  for (int i = 0; i < optimized_graph_->node_size(); ++i) {
+    const NodeDef& node = optimized_graph_->node(i);
+    if (IsNoOp(node) || IsIdentity(node) || IsIdentityN(node) ||
+        IsConstant(node) || SafeToConvertToNoOp(node)) {
+      nodes_to_simplify.PushBack(i);
     }
   }
   while (!nodes_to_simplify.Empty()) {
-    NodeDef* node_to_simplify = nodes_to_simplify.PopBack();
+    int node_to_simplify = nodes_to_simplify.PopBack();
     // Discard nodes that were marked for deletion already.
-    while (nodes_to_delete.find(node_to_simplify->name()) !=
-           nodes_to_delete.end()) {
+    while (nodes_to_delete.find(node_to_simplify) != nodes_to_delete.end()) {
       node_to_simplify = nodes_to_simplify.PopBack();
     }
     OptimizeNode(node_to_simplify, &nodes_to_simplify, &nodes_to_delete);
@@ -415,17 +468,17 @@ Status DependencyOptimizer::OptimizeDependencies() {
 
   if (fetch_nodes_known_) {
     VLOG(1) << "Deleted " << nodes_to_delete.size() << " out of "
-            << graph_view_->graph()->node_size() << " nodes.";
-    graph_view_->DeleteNodes(nodes_to_delete);
+            << optimized_graph_->node_size() << " nodes.";
+    EraseNodesFromGraph(nodes_to_delete, optimized_graph_);
+    node_map_.reset(new NodeMap(optimized_graph_));
+    BuildNodeToIdx();
   }
   return Status::OK();
 }
 
 Status DependencyOptimizer::TransitiveReduction() {
   // PRECONDITION: optimized_graph_ must be sorted topologically.
-  GraphDef* graph = graph_view_->graph();
-  auto node_to_idx = BuildNodeToIdx(*graph);
-  const int num_nodes = graph->node_size();
+  const int num_nodes = optimized_graph_->node_size();
   // Set up a compressed version of the graph to save a constant factor in the
   // expensive algorithm below. Also cache the set of control outputs and the
   // highest index of a target of any control output from each node.
@@ -434,20 +487,20 @@ Status DependencyOptimizer::TransitiveReduction() {
   std::vector<gtl::InlinedVector<std::pair<int, int>, 2>> control_outputs(
       num_nodes);
   for (int node_idx = 0; node_idx < num_nodes; ++node_idx) {
-    const NodeDef& node = graph->node(node_idx);
+    const NodeDef& node = optimized_graph_->node(node_idx);
     if (ModifiesFrameInfo(node) || !HasOpDef(node)) {
       // Ignore function nodes and nodes that modify frame info.
       continue;
     }
     for (int input_slot = 0; input_slot < node.input_size(); ++input_slot) {
       const string& input = node.input(input_slot);
-      const NodeDef* input_node = graph_view_->GetNode(NodeName(input));
+      const NodeDef* input_node = node_map_->GetNode(input);
       if (ModifiesFrameInfo(*input_node) || IsMerge(*input_node)) {
         // Ignore edges from nodes that modify frame info and from Merge nodes,
         // because we cannot know which of it's input paths executes.
         continue;
       }
-      const int input_node_idx = node_to_idx[input_node];
+      const int input_node_idx = node_to_idx_[input_node];
       inputs[node_idx].push_back(input_node_idx);
       if (IsControlInput(input)) {
         ++num_controls;
@@ -513,12 +566,16 @@ Status DependencyOptimizer::TransitiveReduction() {
 
   for (const auto& it : control_edges_to_remove) {
     const int target = it.first;
-    const NodeDef& target_node = graph->node(target);
-    const string target_node_name = target_node.name();
+    NodeDef* target_node = optimized_graph_->mutable_node(target);
     for (const InputSlotAndSource& slot_and_source : it.second) {
       const int input_slot = slot_and_source.first;
-      const TensorId tensor_id = ParseTensorName(target_node.input(input_slot));
-      graph_view_->RemoveFanin(target_node_name, tensor_id);
+      const int source = slot_and_source.second;
+      const NodeDef& source_node = optimized_graph_->node(source);
+      CHECK_LT(input_slot, target_node->input_size());
+      target_node->mutable_input()->SwapElements(input_slot,
+                                                 target_node->input_size() - 1);
+      node_map_->RemoveOutput(source_node.name(), target_node->name());
+      target_node->mutable_input()->RemoveLast();
       ++num_controls_removed;
     }
   }
@@ -527,17 +584,26 @@ Status DependencyOptimizer::TransitiveReduction() {
   return Status::OK();
 }
 
+void DependencyOptimizer::BuildNodeToIdx() {
+  // Set up &node -> index map.
+  node_to_idx_.clear();
+  for (int i = 0; i < optimized_graph_->node_size(); ++i) {
+    const NodeDef& node = optimized_graph_->node(i);
+    node_to_idx_[&node] = i;
+  }
+}
+
 // Suppose there are cross-device control inputs to node C from multiple nodes
 // that are located on another device, e.g., we have control edges:
 // A->C, B->C
 // where A and B are on device X and C is on device Y.
 // We can reduce cross-device communication by introducing an intermediate
 // NoOp node C' on device X and rewriting the control edges to:
-// A->C', B->C', C'->C
+// A->C', B->C', C' -> C
 void DependencyOptimizer::GroupCrossDeviceControlEdges() {
-  const int num_nodes = graph_view_->graph()->node_size();
+  const int num_nodes = optimized_graph_->node_size();
   for (int i = 0; i < num_nodes; ++i) {
-    NodeDef* node = graph_view_->graph()->mutable_node(i);
+    NodeDef* node = optimized_graph_->mutable_node(i);
     if (node->device().empty()) continue;
 
     // Creates new noop nodes for devices on which multiple control inputs are
@@ -548,50 +614,66 @@ void DependencyOptimizer::GroupCrossDeviceControlEdges() {
     // that device.
     std::map<string, NodeDef*> noops;
     int num_noops = 0;
-    auto controlling_fanins = graph_view_->GetFanin(
-        MutableGraphView::InputPort(node, Graph::kControlSlot));
-    for (const auto& controlling_fanin : controlling_fanins) {
-      const NodeDef* fanin_node = controlling_fanin.node;
-      if (!fanin_node->device().empty() &&
-          fanin_node->device() != node->device()) {
-        auto emplace_result = noops.emplace(fanin_node->device(), nullptr);
-        if (!emplace_result.second && emplace_result.first->second == nullptr) {
-          // This is the second cross-device control input from the same
-          // device. Creates an intermediate noop node on that device.
-          string group_name;
-          NodeDef* noop;
-          // Creates a fresh node name; there may be conflicting names from
-          // a previous iteration of the optimizer.
-          do {
-            group_name = AddPrefixToNodeName(
-                node->name(),
-                strings::StrCat("GroupCrossDeviceControlEdges_", num_noops));
-            noop = graph_view_->GetNode(group_name);
-            ++num_noops;
-          } while (noop != nullptr);
-          NodeDef new_node;
-          new_node.set_name(group_name);
-          new_node.set_device(fanin_node->device());
-          new_node.set_op("NoOp");
-          emplace_result.first->second =
-              graph_view_->AddNode(std::move(new_node));
+    for (int j = 0; j < node->input_size(); ++j) {
+      if (IsControlInput(node->input(j))) {
+        const NodeDef* input = node_map_->GetNode(node->input(j));
+        if (input != nullptr && !input->device().empty() &&
+            input->device() != node->device()) {
+          auto emplace_result = noops.emplace(input->device(), nullptr);
+          if (!emplace_result.second &&
+              emplace_result.first->second == nullptr) {
+            // This is the second cross-device control input from the same
+            // device. Creates an intermediate noop node on that device.
+            string group_name;
+            NodeDef* noop;
+            // Creates a fresh node name; there may be conflicting names from
+            // a previous iteration of the optimizer.
+            do {
+              group_name = AddPrefixToNodeName(
+                  node->name(),
+                  strings::StrCat("GroupCrossDeviceControlEdges_", num_noops));
+              noop = node_map_->GetNode(group_name);
+              ++num_noops;
+            } while (noop != nullptr);
+            noop = optimized_graph_->add_node();
+            noop->set_name(group_name);
+            noop->set_device(input->device());
+            noop->set_op("NoOp");
+            node_map_->AddNode(noop->name(), noop);
+            emplace_result.first->second = noop;
+          }
         }
       }
     }
 
     // Reroute existing control edges to go via the newly introduced NoOp nodes.
-    for (const auto& controlling_fanin : controlling_fanins) {
-      auto it = noops.find(controlling_fanin.node->device());
-      if (it != noops.end() && it->second != nullptr) {
-        TensorId tensor_id(controlling_fanin.node->name(), Graph::kControlSlot);
-        graph_view_->RemoveFanin(node->name(), tensor_id);
-        graph_view_->AddFanin(it->second->name(), tensor_id);
+    int pos = 0;
+    while (pos < node->input_size()) {
+      const string& input_name = node->input(pos);
+      if (IsControlInput(input_name)) {
+        NodeDef* input = node_map_->GetNode(input_name);
+        if (input == nullptr) {
+          ++pos;
+        } else {
+          auto it = noops.find(input->device());
+          if (it == noops.end() || it->second == nullptr) {
+            ++pos;
+          } else {
+            node->mutable_input()->SwapElements(pos, node->input_size() - 1);
+            node->mutable_input()->RemoveLast();
+            it->second->add_input(AsControlDependency(*input));
+            node_map_->UpdateOutput(input_name, node->name(),
+                                    it->second->name());
+          }
+        }
+      } else {
+        ++pos;
       }
     }
     for (const auto& entry : noops) {
       if (entry.second) {
-        graph_view_->AddFanin(node->name(),
-                              {entry.second->name(), Graph::kControlSlot});
+        node->add_input(AsControlDependency(*entry.second));
+        node_map_->AddOutput(entry.second->name(), node->name());
       }
     }
   }
@@ -599,17 +681,22 @@ void DependencyOptimizer::GroupCrossDeviceControlEdges() {
 
 Status DependencyOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
                                      GraphDef* optimized_graph) {
-  *optimized_graph = item.graph;
+  optimized_graph_ = optimized_graph;
+  *optimized_graph_ = item.graph;
   nodes_to_preserve_ = item.NodesToPreserve();
   fetch_nodes_known_ = !item.fetch.empty();
-  graph_view_.reset(new MutableGraphView(optimized_graph));
+  CleanControlInputs();
 
   const int num_iterations = 2;
   for (int iteration = 0; iteration < num_iterations; ++iteration) {
     GRAPPLER_RETURN_IF_DEADLINE_EXCEEDED();
     Status topo_sort_status;
     // Perform topological sort to prepare the graph for transitive reduction.
-    topo_sort_status = TopologicalSort(optimized_graph);
+    topo_sort_status = TopologicalSort(optimized_graph_);
+    // Set up index-based graph datastructures to speed up analysis steps below.
+    node_map_.reset(new NodeMap(optimized_graph_));
+    BuildNodeToIdx();
+
     if (topo_sort_status.ok()) {
       // Remove redundant control dependencies.
       TF_RETURN_IF_ERROR(TransitiveReduction());
@@ -622,6 +709,9 @@ Status DependencyOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
     // nodes.
     TF_RETURN_IF_ERROR(OptimizeDependencies());
 
+    // Dedup control inputs.
+    CleanControlInputs();
+
     GroupCrossDeviceControlEdges();
   }
 
diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer.h b/tensorflow/core/grappler/optimizers/dependency_optimizer.h
index eb7fc89080..7b032673fb 100644
--- a/tensorflow/core/grappler/optimizers/dependency_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/dependency_optimizer.h
@@ -17,8 +17,6 @@ limitations under the License.
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DEPENDENCY_OPTIMIZER_H_
 
 #include <unordered_set>
-
-#include "tensorflow/core/grappler/mutable_graph_view.h"
 #include "tensorflow/core/grappler/optimizers/graph_optimizer.h"
 #include "tensorflow/core/grappler/utils.h"
 #include "tensorflow/core/protobuf/rewriter_config.pb.h"
@@ -48,22 +46,24 @@ class DependencyOptimizer : public GraphOptimizer {
   // Returns true if bypassing node does not increase the number of edges or
   // number of edges crossing a device boundary.
   bool BypassingNodeIsBeneficial(
-      const NodeDef& node,
-      const absl::flat_hash_set<MutableGraphView::OutputPort>& fanins,
-      const absl::flat_hash_set<MutableGraphView::Edge>& fanout_edges) const;
-  int NumEdgesIfBypassed(
-      const NodeDef& node, int num_fanins,
-      const absl::flat_hash_set<MutableGraphView::Edge>& fanout_edges) const;
+      const NodeDef& node, const std::vector<NodeDef*>& input_nodes,
+      const std::vector<NodeDef*>& output_nodes) const;
+  int NumEdgesIfBypassed(const NodeDef& node,
+                         const std::vector<NodeDef*>& output_nodes) const;
   // Returns true if node is not an Identity node or if it is an Identity
   // that is safe to remove.
   bool SafeToRemoveIdentity(const NodeDef& node) const;
   // Returns true if it is safe to convert node to NoOp.
   bool SafeToConvertToNoOp(const NodeDef& node) const;
+  // Removes all duplicate control dependencies.
+  void CleanControlInputs();
+  // Builds a map from the &optimized_graph_->node(i) to i.
+  void BuildNodeToIdx();
   // Tries to optimize the node with the given index, possibly additional
   // optimizations by inserting nodes in nodes_to_simplify, and pruning nodes by
   // inserting them in nodes_to_delete.
-  void OptimizeNode(NodeDef* node, SetVector<NodeDef*>* nodes_to_simplify,
-                    std::set<string>* nodes_to_delete);
+  void OptimizeNode(int node_idx, SetVector<int>* nodes_to_simplify,
+                    std::set<int>* nodes_to_delete);
   // Eliminates redundant control dependencies by computing the transitive
   // reduction of the graph.
   Status TransitiveReduction();
@@ -76,7 +76,9 @@ class DependencyOptimizer : public GraphOptimizer {
   RewriterConfig::Toggle opt_level_;
   bool fetch_nodes_known_;
   std::unordered_set<string> nodes_to_preserve_;
-  std::unique_ptr<MutableGraphView> graph_view_;
+  std::unique_ptr<NodeMap> node_map_;
+  std::unordered_map<const NodeDef*, int> node_to_idx_;
+  GraphDef* optimized_graph_;  // Not owned.
 };
 
 }  // end namespace grappler
diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc b/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
index 673fc987a8..8d70d9d5c7 100644
--- a/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
@@ -91,7 +91,7 @@ TEST_F(DependencyOptimizerTest, DependenciesDrivenByConstants) {
   // The 'z' node should have been optimized away leaving only 5 nodes.
   EXPECT_EQ(5, output.node_size());
 
-  for (const NodeDef& node : output.node()) {
+  for (const NodeDef& node : item.graph.node()) {
     if (node.name() == "id1" || node.name() == "id2") {
       EXPECT_EQ(1, node.input_size());
       EXPECT_EQ("add", node.input(0));
@@ -125,8 +125,8 @@ TEST_F(DependencyOptimizerTest, ChangeToNoop) {
 
   EXPECT_EQ(item.graph.node_size(), output.node_size());
   int found = 0;
-  for (int i = 0; i < output.node_size(); ++i) {
-    const NodeDef& node = output.node(i);
+  for (int i = 0; i < item.graph.node_size(); ++i) {
+    const NodeDef& node = item.graph.node(i);
     // "add" should get turned into a NoOp and removed.
     EXPECT_NE("add", node.name());
     if (node.name() == "id1") {
@@ -164,6 +164,7 @@ TEST_F(DependencyOptimizerTest, ChangeToNoop_RepeatedInput) {
   item.graph.Swap(&output);
   status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
+  LOG(INFO) << output.DebugString();
 
   EXPECT_EQ(item.graph.node_size(), output.node_size());
   int found = 0;
@@ -747,7 +748,7 @@ TEST_F(DependencyOptimizerTest, Identity_DeviceCrossing_ConsumerOnSameDevice) {
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
-
+  LOG(INFO) << output.DebugString();
   EXPECT_EQ(3, output.node_size());
   for (const auto& node : output.node()) {
     EXPECT_NE("x_on_2", node.name());
-- 
GitLab


From 93f760a1d83404cadf2ce1bc8cd8268c47e2f4f8 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Tue, 8 Jan 2019 10:41:24 -0800
Subject: [PATCH 0350/2345] [XLA] Fix VLOG(0) usage in
 dynamic_dimension_inference that caused a lot of LOG spam.

PiperOrigin-RevId: 228357053
---
 tensorflow/compiler/xla/service/dynamic_dimension_inference.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/dynamic_dimension_inference.cc b/tensorflow/compiler/xla/service/dynamic_dimension_inference.cc
index f2b7e9a186..2b158d7a6e 100644
--- a/tensorflow/compiler/xla/service/dynamic_dimension_inference.cc
+++ b/tensorflow/compiler/xla/service/dynamic_dimension_inference.cc
@@ -433,7 +433,7 @@ Status DynamicDimensionInferenceVisitor::ForEachOperandDynamicDimension(
 /* static */
 StatusOr<DynamicDimensionInference> DynamicDimensionInference::Run(
     HloModule* module) {
-  VLOG(0) << "Param Config " << module->dynamic_parameter_binding().ToString();
+  VLOG(2) << "Param Config " << module->dynamic_parameter_binding().ToString();
   DynamicDimensionInference inference(module);
   TF_RETURN_IF_ERROR(inference.AnalyzeDynamicDimensions());
   return inference;
-- 
GitLab


From 2ce3e3758d0fc1eea31b3e16c94321198c121953 Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Tue, 8 Jan 2019 10:47:29 -0800
Subject: [PATCH 0351/2345] Fix for Raspberry Pi build, removing default
 dependency on Intel MKL

PiperOrigin-RevId: 228358313
---
 tensorflow/tools/ci_build/pi/build_raspberry_pi.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/tools/ci_build/pi/build_raspberry_pi.sh b/tensorflow/tools/ci_build/pi/build_raspberry_pi.sh
index 864278c647..987f0769b2 100755
--- a/tensorflow/tools/ci_build/pi/build_raspberry_pi.sh
+++ b/tensorflow/tools/ci_build/pi/build_raspberry_pi.sh
@@ -107,6 +107,7 @@ bazel build -c opt ${PI_COPTS} \
   --copt=-funsafe-math-optimizations --copt=-ftree-vectorize \
   --copt=-fomit-frame-pointer --cpu=armeabi \
   --crosstool_top=@local_config_arm_compiler//:toolchain \
+  --define tensorflow_mkldnn_contraction_kernel=0 \
   --verbose_failures \
   //tensorflow:libtensorflow.so \
   //tensorflow:libtensorflow_framework.so \
-- 
GitLab


From 570767085d89516d145e2960ecc6edb23aa1456e Mon Sep 17 00:00:00 2001
From: "srinivasan.narayanamoorthy" <srinivasan.narayanamoorthy@intel.com>
Date: Tue, 8 Jan 2019 11:04:30 -0800
Subject: [PATCH 0352/2345] Mkl small size allocator fix for reducing lock
 overhead in critical path

---
 .../core/common_runtime/mkl_cpu_allocator.h   | 78 +++++++++++--------
 1 file changed, 45 insertions(+), 33 deletions(-)

diff --git a/tensorflow/core/common_runtime/mkl_cpu_allocator.h b/tensorflow/core/common_runtime/mkl_cpu_allocator.h
index 429b19599b..640300e523 100644
--- a/tensorflow/core/common_runtime/mkl_cpu_allocator.h
+++ b/tensorflow/core/common_runtime/mkl_cpu_allocator.h
@@ -39,6 +39,8 @@ typedef unsigned int uint;
 
 namespace tensorflow {
 
+static bool mkl_small_allocator_collect_stats = false;
+
 class MklSubAllocator : public BasicCPUAllocator {
  public:
   MklSubAllocator() : BasicCPUAllocator(port::kNUMANoAffinity, {}, {}) {}
@@ -62,15 +64,8 @@ class MklSmallSizeAllocator : public Allocator {
   inline string Name() override { return name_; }
 
   void* AllocateRaw(size_t alignment, size_t num_bytes) override {
-    void* ptr = sub_allocator_->Alloc(alignment, num_bytes);
-    if (ptr != nullptr) {
-      std::pair<void*, size_t> map_val(ptr, num_bytes);
-      mutex_lock l(mutex_);
-      // Check that insertion in the hash map was successful.
-      CHECK(map_.insert(map_val).second);
-      // Increment statistics for small-size allocations.
-      IncrementStats(num_bytes);
-    }
+    void* ptr = port::AlignedMalloc(num_bytes, alignment);
+    if (mkl_small_allocator_collect_stats) IncrementStats(num_bytes);
     return ptr;
   }
 
@@ -80,23 +75,11 @@ class MklSmallSizeAllocator : public Allocator {
       return;
     }
 
-    mutex_lock l(mutex_);
-    auto map_iter = map_.find(ptr);
-    if (map_iter != map_.end()) {
-      // Call free visitors.
-      size_t dealloc_bytes = map_iter->second;
-      sub_allocator_->Free(ptr, dealloc_bytes);
-      DecrementStats(dealloc_bytes);
-      map_.erase(map_iter);
-    } else {
-      LOG(ERROR) << "tried to deallocate invalid pointer";
-      return;
+    if (mkl_small_allocator_collect_stats) {
+      const size_t alloc_size = port::MallocExtension_GetAllocatedSize(ptr);
+      DecrementStats(alloc_size);
     }
-  }
-
-  inline bool IsSmallSizeAllocation(const void* ptr) const {
-    mutex_lock l(mutex_);
-    return map_.find(ptr) != map_.end();
+    port::AlignedFree(ptr);
   }
 
   void GetStats(AllocatorStats* stats) override {
@@ -135,10 +118,6 @@ class MklSmallSizeAllocator : public Allocator {
   // Allocator name
   string name_;
 
-  // Hash map to keep track of "small" allocations
-  // We do not use BFC allocator for small allocations.
-  std::unordered_map<const void*, size_t> map_ GUARDED_BY(mutex_);
-
   // Allocator stats for small allocs
   AllocatorStats stats_ GUARDED_BY(mutex_);
 };
@@ -215,23 +194,50 @@ class MklCPUAllocator : public Allocator {
   }
 
   inline string Name() override { return kName; }
+  inline bool IsSmallSizeAllocation(const void* ptr) const {
+    mutex_lock l(mutex_);
+    return large_allocations_map_.find(ptr) == large_allocations_map_.end();
+  }
+  // AddLargeAllocMap and RemoveLargeAllocMap are always called with a lock held
+  inline void AddLargeAllocMap(void* ptr, size_t num_bytes) {
+    if (ptr != nullptr) {
+      std::pair<void*, size_t> map_val(ptr, num_bytes);
+      large_allocations_map_.insert(map_val).second;
+    }
+  }
+  inline void RemoveLargeAllocMap(void* ptr) {
+    auto map_iter = large_allocations_map_.find(ptr);
+    if (map_iter != large_allocations_map_.end()) {
+      large_allocations_map_.erase(map_iter);
+    } else {
+      LOG(ERROR) << "tried to deallocate invalid pointer";
+    }
+    return;
+  }
 
   inline void* AllocateRaw(size_t alignment, size_t num_bytes) override {
     // If the allocation size is less than threshold, call small allocator,
     // otherwise call large-size allocator (BFC). We found that BFC allocator
     // does not deliver good performance for small allocations when
     // inter_op_parallelism_threads is high.
-    return (num_bytes < kSmallAllocationsThreshold)
-               ? small_size_allocator_->AllocateRaw(alignment, num_bytes)
-               : large_size_allocator_->AllocateRaw(alignment, num_bytes);
+    if (num_bytes < kSmallAllocationsThreshold) {
+      return small_size_allocator_->AllocateRaw(alignment, num_bytes);
+    } else {
+      mutex_lock l(mutex_);
+      void* ptr = large_size_allocator_->AllocateRaw(alignment, num_bytes);
+      AddLargeAllocMap(ptr, num_bytes);
+      return ptr;
+    }
   }
 
   inline void DeallocateRaw(void* ptr) override {
     // Check if ptr is for "small" allocation. If it is, then call Free
     // directly. Otherwise, call BFC to handle free.
-    if (small_size_allocator_->IsSmallSizeAllocation(ptr)) {
+    if (IsSmallSizeAllocation(ptr)) {
       small_size_allocator_->DeallocateRaw(ptr);
     } else {
+      mutex_lock l(mutex_);
+      RemoveLargeAllocMap(ptr);
       large_size_allocator_->DeallocateRaw(ptr);
     }
   }
@@ -299,6 +305,12 @@ class MklCPUAllocator : public Allocator {
   MklSmallSizeAllocator* small_size_allocator_;  // owned by this class.
 
   SubAllocator* sub_allocator_;  // not owned by this class
+  mutable mutex mutex_;
+
+  // Hash map to keep track of "BFC" allocations
+  // We do not use BFC allocator for small allocations.
+  std::unordered_map<const void*, size_t> large_allocations_map_
+      GUARDED_BY(mutex_);
 
   // Size in bytes that defines the upper-bound for "small" allocations.
   // Any allocation below this threshold is "small" allocation.
-- 
GitLab


From dd355d3b10a73474fb34972ec0ec708868bd0a24 Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Tue, 8 Jan 2019 11:13:29 -0800
Subject: [PATCH 0353/2345] Fill zeros for DT_INVALID tensors in
 TensorListStack.

PiperOrigin-RevId: 228363806
---
 tensorflow/core/kernels/BUILD                 |  1 +
 tensorflow/core/kernels/fill_functor.cc       |  5 ++
 tensorflow/core/kernels/list_kernels.h        | 71 ++++++++++++++-----
 .../python/kernel_tests/list_ops_test.py      | 41 +++++++++++
 .../kernel_tests/tensor_array_ops_test.py     | 13 ++--
 5 files changed, 106 insertions(+), 25 deletions(-)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index b934c64bfc..aaea09bdc4 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -2256,6 +2256,7 @@ tf_kernel_library(
     ],
     deps = [
         ":concat_lib",
+        ":fill_functor",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:list_ops_op_lib",
diff --git a/tensorflow/core/kernels/fill_functor.cc b/tensorflow/core/kernels/fill_functor.cc
index 7090417dfd..9c4c0487f0 100644
--- a/tensorflow/core/kernels/fill_functor.cc
+++ b/tensorflow/core/kernels/fill_functor.cc
@@ -51,6 +51,11 @@ DEFINE_SETZERO_CPU(uint16);
 DEFINE_SETZERO_CPU(int16);
 DEFINE_SETZERO_CPU(int32);
 DEFINE_SETZERO_CPU(int64);
+DEFINE_SETZERO_CPU(quint8);
+DEFINE_SETZERO_CPU(qint8);
+DEFINE_SETZERO_CPU(quint16);
+DEFINE_SETZERO_CPU(qint16);
+DEFINE_SETZERO_CPU(qint32);
 DEFINE_SETZERO_CPU(complex64);
 DEFINE_SETZERO_CPU(complex128);
 DEFINE_SETZERO_CPU(Variant);
diff --git a/tensorflow/core/kernels/list_kernels.h b/tensorflow/core/kernels/list_kernels.h
index 559e14b03c..fd1be80f11 100644
--- a/tensorflow/core/kernels/list_kernels.h
+++ b/tensorflow/core/kernels/list_kernels.h
@@ -28,6 +28,7 @@ limitations under the License.
 #include "tensorflow/core/framework/variant.h"
 #include "tensorflow/core/framework/variant_op_registry.h"
 #include "tensorflow/core/kernels/concat_lib.h"
+#include "tensorflow/core/kernels/fill_functor.h"
 #include "tensorflow/core/lib/core/coding.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/gtl/array_slice.h"
@@ -65,6 +66,15 @@ struct TensorList {
 
 Status TensorShapeFromTensor(const Tensor& t, PartialTensorShape* out);
 
+// Allocates a Tensor of requested shape and dtype and fills it with zeros.
+template <typename Device, typename T>
+void BuildZerosTensor(OpKernelContext* c, DataType dtype,
+                      const TensorShape& shape, Tensor* zeros) {
+  OP_REQUIRES_OK(c, c->allocate_temp(dtype, shape, zeros));
+  functor::SetZeroFunctor<Device, T> f;
+  f(c->eigen_device<Device>(), zeros->flat<T>());
+}
+
 template <typename Device, typename T>
 class TensorListStack : public OpKernel {
  public:
@@ -94,7 +104,7 @@ class TensorListStack : public OpKernel {
         !tensor_list->tensors.empty() ||
             tensor_list->element_shape.IsFullyDefined(),
         errors::InvalidArgument("Tried to stack elements of a empty ",
-                                "list with non-fully-defined shape: ",
+                                "list with non-fully-defined element_shape: ",
                                 tensor_list->element_shape.DebugString()));
     if (num_elements_ != -1) {
       OP_REQUIRES(c, tensor_list->tensors.size() == num_elements_,
@@ -106,34 +116,59 @@ class TensorListStack : public OpKernel {
     // Compute the shape of the output tensor.
     // If `element_shape` is fully-defined it gets used. It is assumed that all
     // element tensors have the same shape.
-    // If `element_shape` is not fully-defined the shape of the first element
-    // tensor is used and it is checked that all other tensors have the same
-    // shape.
-    TensorShape resulting_shape;
-    if (!tensor_list->element_shape.AsTensorShape(&resulting_shape)) {
-      const Tensor& t = tensor_list->tensors[0];
-      resulting_shape = t.shape();
-      for (int i = 1; i < tensor_list->tensors.size(); ++i) {
+    // If `element_shape` is not fully-defined the shape of the first
+    // initialized element tensor is used and it is checked that all other
+    // initialized tensors have the same shape. An error is thrown if the list
+    // only contains DT_INVALID type tensors.
+    TensorShape resulting_element_shape;
+    if (!tensor_list->element_shape.AsTensorShape(&resulting_element_shape)) {
+      bool resulting_element_shape_initialized = false;
+      for (int i = 0; i < tensor_list->tensors.size(); ++i) {
         const Tensor& t = tensor_list->tensors[i];
-        OP_REQUIRES(c, t.shape() == resulting_shape,
-                    errors::InvalidArgument(
-                        "Tried to stack tensors with unequal shapes: ",
-                        resulting_shape.DebugString(), " vs ",
-                        t.shape().DebugString()));
+        if (!resulting_element_shape_initialized) {
+          if (t.dtype() == DT_INVALID) {
+            continue;
+          }
+          resulting_element_shape = t.shape();
+          resulting_element_shape_initialized = true;
+          continue;
+        }
+        OP_REQUIRES(
+            c, t.dtype() == DT_INVALID || t.shape() == resulting_element_shape,
+            errors::InvalidArgument(
+                "Tried to stack tensors with unequal shapes: ",
+                resulting_element_shape.DebugString(), " vs ",
+                t.shape().DebugString()));
       }
+      OP_REQUIRES(
+          c, resulting_element_shape_initialized,
+          errors::InvalidArgument("Tried to stack list which only contains ",
+                                  "uninitialized tensors and has a ",
+                                  "non-fully-defined element_shape: ",
+                                  tensor_list->element_shape.DebugString()));
     }
-    resulting_shape.InsertDim(0, tensor_list->tensors.size());
+    TensorShape output_tensor_shape = resulting_element_shape;
+    output_tensor_shape.InsertDim(0, tensor_list->tensors.size());
     Tensor* output;
-    OP_REQUIRES_OK(c, c->allocate_output(0, resulting_shape, &output));
+    OP_REQUIRES_OK(c, c->allocate_output(0, output_tensor_shape, &output));
     if (output->NumElements() == 0) {
       return;
     }
 
     ConstMatrixVector inputs_flat;
     inputs_flat.reserve(tensor_list->tensors.size());
+    Tensor zeros;
+    BuildZerosTensor<Device, T>(c, element_dtype_, resulting_element_shape,
+                                &zeros);
     for (const auto& t : tensor_list->tensors) {
-      inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
-          t.shaped<T, 2>({1, t.NumElements()})));
+      if (t.dtype() != DT_INVALID) {
+        inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
+            t.shaped<T, 2>({1, t.NumElements()})));
+      } else {
+        inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
+            const_cast<const Tensor&>(zeros).shaped<T, 2>(
+                {1, zeros.NumElements()})));
+      }
     }
     auto output_flat = output->shaped<T, 2>({1, output->NumElements()});
 
diff --git a/tensorflow/python/kernel_tests/list_ops_test.py b/tensorflow/python/kernel_tests/list_ops_test.py
index fc0e270667..ec6906f20c 100644
--- a/tensorflow/python/kernel_tests/list_ops_test.py
+++ b/tensorflow/python/kernel_tests/list_ops_test.py
@@ -189,6 +189,47 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
       self.evaluate(t)
 
+  def _testStackWithUninitializedTensors(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=[], num_elements=3)
+    t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
+    self.assertAllEqual(self.evaluate(t), [0., 0., 0.])
+
+  def testStackWithUninitializedTensors(self):
+    self._testStackWithUninitializedTensors()
+
+  def testStackWithUninitializedTensorsGpu(self):
+    if not context.num_gpus():
+      return
+    with context.device("gpu:0"):
+      self._testStackWithUninitializedTensors()
+
+  def _testStackWithUninitializedTensorsInferShape(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
+    l = list_ops.tensor_list_set_item(l, 1, [1., 2.])
+    t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
+    self.assertAllEqual(self.evaluate(t), [[0., 0.], [1., 2.], [0., 0.]])
+
+  def testStackWithUninitializedTensorsInferShape(self):
+    self._testStackWithUninitializedTensorsInferShape()
+
+  def testStackWithUninitializedTensorsInferShapeGpu(self):
+    if not context.num_gpus():
+      return
+    with context.device("gpu:0"):
+      self._testStackWithUninitializedTensorsInferShape()
+
+  def testStackReservedListWithNoElementsAndPartialElementShapeFails(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
+    with self.assertRaisesRegexp(errors.InvalidArgumentError,
+                                 "Tried to stack list which only contains "
+                                 "uninitialized tensors and has a "
+                                 "non-fully-defined element_shape: <unknown>"):
+      t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
+      self.evaluate(t)
+
   @parameterized.named_parameters(("NoMaxNumElements", None),
                                   ("WithMaxNumElements", 2))
   def testGatherGrad(self, max_num_elements):
diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
index 3f1bd60d1a..e8af998a70 100644
--- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py
+++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
@@ -185,8 +185,8 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual([[0.0, 0.0], [4.0, 5.0], [0.0, 0.0]],
                           self.evaluate(ta.write(1, [[4.0, 5.0]]).concat()))
 
-  @test_util.disable_control_flow_v2("b/118890905")
-  @test_util.run_v1_only("b/118890905")
+  @test_util.disable_control_flow_v2("b/122324791")
+  @test_util.run_v1_only("b/122324791")
   def testTensorArrayReadOrPackNotAllValuesAvailableFillsZeros(self):
     self._testTensorArrayReadOrPackNotAllValuesAvailableFillsZeros()
 
@@ -202,8 +202,8 @@ class TensorArrayTest(test.TestCase):
     self.assertAllEqual([[0.0, 0.0], [4.0, 5.0], [0.0, 0.0]],
                         self.evaluate(ta.write(1, [[4.0, 5.0]]).concat()))
 
-  @test_util.disable_control_flow_v2("b/118890905")
-  @test_util.run_v1_only("b/118890905")
+  @test_util.disable_control_flow_v2("b/122324791")
+  @test_util.run_v1_only("b/122324791")
   def testTensorArrayReadOrPackNotAllValuesAvailableInferShapeFillsZeros(self):
     self._testTensorArrayReadOrPackNotAllValuesAvailableInferShapeFillsZeros()
 
@@ -1321,8 +1321,8 @@ class TensorArrayTest(test.TestCase):
     with self.cached_session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
           dtype=dtypes.float32, size=0, dynamic_size=False, infer_shape=False)
-      v2_msg = ("Tried to stack elements of a empty list with "
-                "non-fully-defined shape")
+      v2_msg = ("Tried to stack elements of a empty list with non-fully-defined"
+                " element_shape")
       v1_msg = (
           "TensorArray has size zero, but element shape <unknown> is not "
           "fully defined. Currently only static shapes are supported when "
@@ -1385,7 +1385,6 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual([10.0, -10.0], read_vals[1])
       self.assertAllEqual([[2.0, 3.0], [4.0, 5.0]], grad_vals[0])
 
-  @test_util.disable_control_flow_v2("b/118890905")
   @test_util.run_v1_only("b/118890905")
   def testTensorArrayWriteGatherAndGradients(self):
     with self.session(use_gpu=True) as session:
-- 
GitLab


From 9c091ca7134fc51f49ea930aac7a48bbdf84e7dc Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Tue, 8 Jan 2019 11:15:25 -0800
Subject: [PATCH 0354/2345] Automated rollback of commit
 acfc65f0110e272abc8bd1abce867f2f534ba263

PiperOrigin-RevId: 228364189
---
 tensorflow/compiler/tf2xla/kernels/shape_op.cc | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/shape_op.cc b/tensorflow/compiler/tf2xla/kernels/shape_op.cc
index 85b0367f73..12830816ec 100644
--- a/tensorflow/compiler/tf2xla/kernels/shape_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/shape_op.cc
@@ -20,7 +20,6 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_helpers.h"
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
-#include "tensorflow/compiler/xla/client/lib/constants.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
 #include "tensorflow/core/framework/tensor_shape.h"
@@ -92,20 +91,14 @@ class SizeOp : public XlaOpKernel {
 
   void Compile(XlaOpKernelContext* ctx) override {
     const TensorShape input_shape = ctx->InputShape(0);
-    OP_REQUIRES(ctx,
-                FastBoundsCheck(input_shape.num_elements(),
-                                std::numeric_limits<int32>::max()),
+    const int64 size = input_shape.num_elements();
+    OP_REQUIRES(ctx, FastBoundsCheck(size, std::numeric_limits<int32>::max()),
                 errors::InvalidArgument("Size does not work for tensors > "
                                         "int32 max."));
     Tensor size_constant(DT_INT32, TensorShape({}));
-    const int rank = input_shape.dims();
-    xla::XlaBuilder* builder = ctx->builder();
-    auto size = xla::One(builder, xla::U32);
-    for (int64 i = 0; i < rank; ++i) {
-      size = xla::Mul(size, xla::GetDimensionSize(ctx->Input(0), i));
-    }
-    size = xla::ConvertElementType(size, xla::S32);
-    ctx->SetOutput(0, size);
+    size_constant.scalar<int32>()() = static_cast<int32>(size);
+
+    ctx->SetConstantOutput(0, size_constant);
   }
 };
 
-- 
GitLab


From bdeda4c087a3515578e6287c647bc00fe5f35491 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Tue, 8 Jan 2019 11:27:47 -0800
Subject: [PATCH 0355/2345] Internal Cleanup.

PiperOrigin-RevId: 228366686
---
 tensorflow/python/keras/BUILD              |   2 +-
 tensorflow/python/keras/optimizers_test.py | 157 +++++++++++----------
 2 files changed, 81 insertions(+), 78 deletions(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 3ffff61c2b..9f8da126bf 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -377,7 +377,7 @@ tf_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:training",
     ],
-    shard_count = 2,
+    shard_count = 4,
     tags = ["notsan"],
 )
 
diff --git a/tensorflow/python/keras/optimizers_test.py b/tensorflow/python/keras/optimizers_test.py
index 18a20567ce..de24bf0536 100644
--- a/tensorflow/python/keras/optimizers_test.py
+++ b/tensorflow/python/keras/optimizers_test.py
@@ -44,116 +44,119 @@ def _get_model(input_dim, num_hidden, output_dim):
   return model
 
 
-def _test_optimizer(optimizer, target=0.75):
-  np.random.seed(1337)
-  (x_train, y_train), _ = testing_utils.get_test_data(train_samples=1000,
-                                                      test_samples=200,
-                                                      input_shape=(10,),
-                                                      num_classes=2)
-  y_train = keras.utils.to_categorical(y_train)
-  model = _get_model(x_train.shape[1], 20, y_train.shape[1])
-  model.compile(loss='categorical_crossentropy',
-                optimizer=optimizer,
-                metrics=['accuracy'])
-  np.testing.assert_equal(keras.backend.get_value(model.optimizer.iterations),
-                          0)
-  history = model.fit(x_train, y_train, epochs=2, batch_size=16, verbose=0)
-  np.testing.assert_equal(keras.backend.get_value(model.optimizer.iterations),
-                          126)  # 63 steps per epoch
-  assert history.history['acc'][-1] >= target
-  config = keras.optimizers.serialize(optimizer)
-  optim = keras.optimizers.deserialize(config)
-  new_config = keras.optimizers.serialize(optim)
-  new_config['class_name'] = new_config['class_name'].lower()
-  new_config['config'].pop('name', None)
-  if 'amsgrad' not in config['config']:
-    new_config['config'].pop('amsgrad', None)
-  if 'decay' in new_config['config'] and 'schedule_decay' in config['config']:
-    new_config['config']['schedule_decay'] = new_config['config'].pop('decay')
-  if 'momentum' not in config['config']:
-    new_config['config'].pop('momentum', None)
-  if 'centered' not in config['config']:
-    new_config['config'].pop('centered', None)
-  assert config == new_config
-
-  # Test constraints.
-  model = keras.models.Sequential()
-  dense = keras.layers.Dense(10,
-                             input_shape=(x_train.shape[1],),
-                             kernel_constraint=lambda x: 0. * x + 1.,
-                             bias_constraint=lambda x: 0. * x + 2.,
-                             activation='relu')
-  model.add(dense)
-  model.add(keras.layers.Dense(y_train.shape[1], activation='softmax'))
-  model.compile(loss='categorical_crossentropy',
-                optimizer=optimizer,
-                metrics=['accuracy'])
-  np.testing.assert_equal(keras.backend.get_value(model.optimizer.iterations),
-                          126)  # Using same optimizer from before
-  model.train_on_batch(x_train[:10], y_train[:10])
-  np.testing.assert_equal(keras.backend.get_value(model.optimizer.iterations),
-                          127)
-  kernel, bias = dense.get_weights()
-  np.testing.assert_allclose(kernel, 1., atol=1e-3)
-  np.testing.assert_allclose(bias, 2., atol=1e-3)
-
-
 class KerasOptimizersTest(test.TestCase):
 
+  def _test_optimizer(self, optimizer, target=0.75):
+    np.random.seed(1337)
+    (x_train, y_train), _ = testing_utils.get_test_data(
+        train_samples=1000, test_samples=200, input_shape=(10,), num_classes=2)
+    y_train = keras.utils.to_categorical(y_train)
+    model = _get_model(x_train.shape[1], 20, y_train.shape[1])
+    model.compile(
+        loss='categorical_crossentropy',
+        optimizer=optimizer,
+        metrics=['accuracy'])
+    np.testing.assert_equal(
+        keras.backend.get_value(model.optimizer.iterations), 0)
+    history = model.fit(x_train, y_train, epochs=2, batch_size=16, verbose=0)
+    np.testing.assert_equal(
+        keras.backend.get_value(model.optimizer.iterations),
+        126)  # 63 steps per epoch
+    assert history.history['acc'][-1] >= target
+    config = keras.optimizers.serialize(optimizer)
+    optim = keras.optimizers.deserialize(config)
+    new_config = keras.optimizers.serialize(optim)
+    new_config['class_name'] = new_config['class_name'].lower()
+    new_config['config'].pop('name', None)
+    if 'amsgrad' not in config['config']:
+      new_config['config'].pop('amsgrad', None)
+    if 'decay' in new_config['config'] and 'schedule_decay' in config['config']:
+      new_config['config']['schedule_decay'] = new_config['config'].pop('decay')
+    if 'momentum' not in config['config']:
+      new_config['config'].pop('momentum', None)
+    if 'centered' not in config['config']:
+      new_config['config'].pop('centered', None)
+    self.assertDictEqual(config, new_config)
+
+    # Test constraints.
+    model = keras.models.Sequential()
+    dense = keras.layers.Dense(
+        10,
+        input_shape=(x_train.shape[1],),
+        kernel_constraint=lambda x: 0. * x + 1.,
+        bias_constraint=lambda x: 0. * x + 2.,
+        activation='relu')
+    model.add(dense)
+    model.add(keras.layers.Dense(y_train.shape[1], activation='softmax'))
+    model.compile(
+        loss='categorical_crossentropy',
+        optimizer=optimizer,
+        metrics=['accuracy'])
+    np.testing.assert_equal(
+        keras.backend.get_value(model.optimizer.iterations),
+        126)  # Using same optimizer from before
+    model.train_on_batch(x_train[:10], y_train[:10])
+    np.testing.assert_equal(
+        keras.backend.get_value(model.optimizer.iterations), 127)
+    kernel, bias = dense.get_weights()
+    np.testing.assert_allclose(kernel, 1., atol=1e-3)
+    np.testing.assert_allclose(bias, 2., atol=1e-3)
+
   def test_sgd(self):
     with self.cached_session():
-      _test_optimizer(keras.optimizers.SGD(lr=0.01,
-                                           momentum=0.9,
-                                           nesterov=True))
+      self._test_optimizer(keras.optimizers.SGD())
+
+  def test_momentum(self):
+    with self.cached_session():
+      self._test_optimizer(
+          keras.optimizers.SGD(lr=0.01, momentum=0.9, nesterov=True))
 
   def test_rmsprop(self):
     with self.cached_session():
-      _test_optimizer(keras.optimizers.RMSprop())
-      _test_optimizer(keras.optimizers.RMSprop(decay=1e-3))
+      self._test_optimizer(keras.optimizers.RMSprop())
+      self._test_optimizer(keras.optimizers.RMSprop(decay=1e-3))
 
   def test_adagrad(self):
     with self.cached_session():
-      _test_optimizer(keras.optimizers.Adagrad())
-      _test_optimizer(keras.optimizers.Adagrad(decay=1e-3))
+      self._test_optimizer(keras.optimizers.Adagrad())
+      self._test_optimizer(keras.optimizers.Adagrad(decay=1e-3))
 
   def test_adadelta(self):
     with self.cached_session():
-      _test_optimizer(keras.optimizers.Adadelta(), target=0.6)
+      self._test_optimizer(keras.optimizers.Adadelta(), target=0.6)
       # Accuracy seems dependent on the initialization. Even adding tf.Print
       # nodes in the graph seemed to affect the initialization seed, and hence
       # the accuracy.
-      _test_optimizer(keras.optimizers.Adadelta(decay=1e-3), target=0.4)
+      self._test_optimizer(keras.optimizers.Adadelta(decay=1e-3), target=0.4)
 
   def test_adam(self):
     with self.cached_session():
-      _test_optimizer(keras.optimizers.Adam())
+      self._test_optimizer(keras.optimizers.Adam())
       # Accuracy seems dependent on the seed initialization.
       # TODO(b/121051441): fix test flakiness.
-      _test_optimizer(keras.optimizers.Adam(decay=1e-3), target=0.73)
-      _test_optimizer(keras.optimizers.Adam(amsgrad=True))
+      self._test_optimizer(keras.optimizers.Adam(decay=1e-3), target=0.73)
+      self._test_optimizer(keras.optimizers.Adam(amsgrad=True))
 
   def test_adamax(self):
     with self.cached_session():
-      _test_optimizer(keras.optimizers.Adamax())
-      _test_optimizer(keras.optimizers.Adamax(decay=1e-3))
+      self._test_optimizer(keras.optimizers.Adamax())
+      self._test_optimizer(keras.optimizers.Adamax(decay=1e-3))
 
   def test_nadam(self):
     with self.cached_session():
-      _test_optimizer(keras.optimizers.Nadam())
+      self._test_optimizer(keras.optimizers.Nadam())
 
   def test_clipnorm(self):
     with self.cached_session():
-      _test_optimizer(keras.optimizers.SGD(lr=0.01,
-                                           momentum=0.9,
-                                           clipnorm=0.5))
+      self._test_optimizer(
+          keras.optimizers.SGD(lr=0.01, momentum=0.9, clipnorm=0.5))
 
   def test_clipvalue(self):
     with self.cached_session():
-      _test_optimizer(keras.optimizers.SGD(lr=0.01,
-                                           momentum=0.9,
-                                           clipvalue=0.5))
+      self._test_optimizer(
+          keras.optimizers.SGD(lr=0.01, momentum=0.9, clipvalue=0.5))
 
-  def test_tfoptimizer(self):
+  def test_tf_optimizer(self):
     optimizer = keras.optimizers.TFOptimizer(AdamOptimizer(0.01))
     model = keras.models.Sequential()
     model.add(keras.layers.Dense(
@@ -188,7 +191,7 @@ class KerasOptimizersTest(test.TestCase):
     self.assertIs(optimizer_weak(), None)
 
   @test_util.run_in_graph_and_eager_modes
-  def test_tfoptimizer_iterations(self):
+  def test_tf_optimizer_iterations(self):
     with self.cached_session():
       optimizer = keras.optimizers.TFOptimizer(AdamOptimizer(0.01))
       model = keras.models.Sequential()
-- 
GitLab


From 35279e5c61d3ff5882c31d3d291256321ba27267 Mon Sep 17 00:00:00 2001
From: Doe Hyun Yoon <dyoon@google.com>
Date: Tue, 8 Jan 2019 11:37:17 -0800
Subject: [PATCH 0356/2345] Replace the references to PredictCosts() with
 PredictCostsAndReturnRunMetadata().

PiperOrigin-RevId: 228368607
---
 .../core/grappler/costs/analytical_cost_estimator_test.cc | 5 +++--
 tensorflow/python/grappler/cost_analyzer.cc               | 8 ++++++--
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/grappler/costs/analytical_cost_estimator_test.cc b/tensorflow/core/grappler/costs/analytical_cost_estimator_test.cc
index a9a1abfa98..8a56344378 100644
--- a/tensorflow/core/grappler/costs/analytical_cost_estimator_test.cc
+++ b/tensorflow/core/grappler/costs/analytical_cost_estimator_test.cc
@@ -98,9 +98,10 @@ TEST_F(AnalyticalCostEstimatorTest, SimpleTest) {
   AnalyticalCostEstimator estimator(cluster_.get(), true);
   TF_ASSERT_OK(estimator.Initialize(item));
 
-  CostGraphDef cost_graph;
+  RunMetadata run_metadata;
   Costs summary;
-  TF_ASSERT_OK(estimator.PredictCosts(item.graph, &cost_graph, &summary));
+  TF_ASSERT_OK(estimator.PredictCostsAndReturnRunMetadata(
+      item.graph, &run_metadata, &summary));
 
   EXPECT_EQ(Costs::NanoSeconds(9151), summary.execution_time);
   // Note there are totally 17 nodes (RandomUniform creates 2 nodes), but
diff --git a/tensorflow/python/grappler/cost_analyzer.cc b/tensorflow/python/grappler/cost_analyzer.cc
index b474e19894..bb8c6d5b85 100644
--- a/tensorflow/python/grappler/cost_analyzer.cc
+++ b/tensorflow/python/grappler/cost_analyzer.cc
@@ -42,9 +42,13 @@ Status CostAnalyzer::GenerateReport(std::ostream& os, bool per_node_report,
 void CostAnalyzer::PredictCosts(CostEstimator* cost_estimator,
                                 CostGraphDef* cost_graph, int64* total_time) {
   TF_CHECK_OK(cost_estimator->Initialize(*item_));
+  RunMetadata run_metadata;
   Costs costs;
-  const Status status =
-      cost_estimator->PredictCosts(item_->graph, cost_graph, &costs);
+  const Status status = cost_estimator->PredictCostsAndReturnRunMetadata(
+      item_->graph, &run_metadata, &costs);
+  if (cost_graph) {
+    cost_graph->Swap(run_metadata.mutable_cost_graph());
+  }
   *total_time = costs.execution_time.count();
   if (!status.ok()) {
     LOG(ERROR) << "Could not estimate the cost for item " << item_->id << ": "
-- 
GitLab


From a849894f02e41fdeb08f86aafad51a2f02b454b1 Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Tue, 8 Jan 2019 11:59:53 -0800
Subject: [PATCH 0357/2345] Reduce the number of tests inside unit-test and
 shard it to speed it up.

PiperOrigin-RevId: 228373086
---
 tensorflow/lite/kernels/internal/BUILD                      | 2 ++
 .../lite/kernels/internal/logsoftmax_quantized_test.cc      | 6 +++---
 tensorflow/lite/kernels/internal/softmax_quantized_test.cc  | 6 +++---
 3 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/tensorflow/lite/kernels/internal/BUILD b/tensorflow/lite/kernels/internal/BUILD
index 885f650dd5..74a11a72ef 100644
--- a/tensorflow/lite/kernels/internal/BUILD
+++ b/tensorflow/lite/kernels/internal/BUILD
@@ -641,6 +641,7 @@ cc_test(
     srcs = [
         "softmax_quantized_test.cc",
     ],
+    shard_count = 3,
     deps = [
         ":optimized_base",
         ":quantization_util",
@@ -657,6 +658,7 @@ cc_test(
     srcs = [
         "logsoftmax_quantized_test.cc",
     ],
+    shard_count = 3,
     tags = [
         # TODO(b/122242739): Reenable after fixing the flakiness?
         "nomac",
diff --git a/tensorflow/lite/kernels/internal/logsoftmax_quantized_test.cc b/tensorflow/lite/kernels/internal/logsoftmax_quantized_test.cc
index 889a726f3a..945300dad1 100644
--- a/tensorflow/lite/kernels/internal/logsoftmax_quantized_test.cc
+++ b/tensorflow/lite/kernels/internal/logsoftmax_quantized_test.cc
@@ -225,7 +225,7 @@ bool TryOneSkyscraperLogSoftmax(bool small_depth) {
 }
 
 TEST(TestQuantizedLogSoftmax, UniformLogSoftmaxTests) {
-  const int kTestsToRun = 1000;
+  const int kTestsToRun = 100;
   for (int i = 0; i < kTestsToRun; i++) {
     while (!TryOneUniformLogSoftmax()) {
     }
@@ -233,7 +233,7 @@ TEST(TestQuantizedLogSoftmax, UniformLogSoftmaxTests) {
 }
 
 TEST(TestQuantizedLogSoftmax, SkyscraperLogSoftmaxTests) {
-  const int kTestsToRun = 1000;
+  const int kTestsToRun = 100;
   for (int i = 0; i < kTestsToRun; i++) {
     while (!TryOneSkyscraperLogSoftmax(false)) {
     }
@@ -241,7 +241,7 @@ TEST(TestQuantizedLogSoftmax, SkyscraperLogSoftmaxTests) {
 }
 
 TEST(TestQuantizedLogSoftmax, SmallSkyscraperLogSoftmaxTests) {
-  const int kTestsToRun = 1000;
+  const int kTestsToRun = 100;
   for (int i = 0; i < kTestsToRun; i++) {
     while (!TryOneSkyscraperLogSoftmax(true)) {
     }
diff --git a/tensorflow/lite/kernels/internal/softmax_quantized_test.cc b/tensorflow/lite/kernels/internal/softmax_quantized_test.cc
index 743ce0355c..8ac62d9af7 100644
--- a/tensorflow/lite/kernels/internal/softmax_quantized_test.cc
+++ b/tensorflow/lite/kernels/internal/softmax_quantized_test.cc
@@ -210,7 +210,7 @@ bool TryOneSkyscraperSoftmax(bool small_depth) {
 }
 
 TEST(TestQuantizedSoftmax, UniformSoftmaxTests) {
-  const int kTestsToRun = 1000;
+  const int kTestsToRun = 100;
   for (int i = 0; i < kTestsToRun; i++) {
     while (!TryOneUniformSoftmax()) {
     }
@@ -218,7 +218,7 @@ TEST(TestQuantizedSoftmax, UniformSoftmaxTests) {
 }
 
 TEST(TestQuantizedSoftmax, SkyscraperSoftmaxTests) {
-  const int kTestsToRun = 1000;
+  const int kTestsToRun = 100;
   for (int i = 0; i < kTestsToRun; i++) {
     while (!TryOneSkyscraperSoftmax(false)) {
     }
@@ -226,7 +226,7 @@ TEST(TestQuantizedSoftmax, SkyscraperSoftmaxTests) {
 }
 
 TEST(TestQuantizedSoftmax, SmallSkyscraperSoftmaxTests) {
-  const int kTestsToRun = 1000;
+  const int kTestsToRun = 100;
   for (int i = 0; i < kTestsToRun; i++) {
     while (!TryOneSkyscraperSoftmax(true)) {
     }
-- 
GitLab


From 68727289bca144c9d1a8f618d81474e4a7322256 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Tue, 8 Jan 2019 12:01:56 -0800
Subject: [PATCH 0358/2345] [XLA] Move ArgMin/ArgMax out of TF/XLA and into XLA
 client library. NFC intended.

PiperOrigin-RevId: 228373518
---
 .../compiler/tf2xla/kernels/categorical_op.cc |  4 +-
 .../compiler/tf2xla/kernels/index_ops.cc      |  5 +-
 .../compiler/tf2xla/kernels/index_ops_cpu.cc  |  4 +-
 tensorflow/compiler/tf2xla/xla_helpers.cc     | 67 -------------------
 tensorflow/compiler/tf2xla/xla_helpers.h      | 10 ---
 tensorflow/compiler/xla/client/lib/BUILD      | 15 +++++
 .../compiler/xla/client/lib/arithmetic.cc     | 60 +++++++++++++++++
 .../compiler/xla/client/lib/arithmetic.h      |  8 +++
 .../xla/client/lib/arithmetic_test.cc         | 67 +++++++++++++++++++
 9 files changed, 156 insertions(+), 84 deletions(-)
 create mode 100644 tensorflow/compiler/xla/client/lib/arithmetic_test.cc

diff --git a/tensorflow/compiler/tf2xla/kernels/categorical_op.cc b/tensorflow/compiler/tf2xla/kernels/categorical_op.cc
index 7199b9b6fe..c2b4c28d15 100644
--- a/tensorflow/compiler/tf2xla/kernels/categorical_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/categorical_op.cc
@@ -99,8 +99,8 @@ class CategoricalOp : public XlaOpKernel {
     xla::PrimitiveType xla_output_type;
     OP_REQUIRES_OK(ctx,
                    DataTypeToPrimitiveType(output_type(0), &xla_output_type));
-    xla::XlaOp argmax = XlaHelpers::ArgMax(softmax_entries, xla_output_type,
-                                           /*axis=*/class_dimension);
+    xla::XlaOp argmax = xla::ArgMax(softmax_entries, xla_output_type,
+                                    /*axis=*/class_dimension);
     if (num_samples == 1) {
       argmax = xla::Reshape(argmax, {batch_size, 1});
     }
diff --git a/tensorflow/compiler/tf2xla/kernels/index_ops.cc b/tensorflow/compiler/tf2xla/kernels/index_ops.cc
index 843b6bb4e6..978e9480ea 100644
--- a/tensorflow/compiler/tf2xla/kernels/index_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/index_ops.cc
@@ -18,7 +18,6 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/kernels/index_ops.h"
 
 #include "tensorflow/compiler/tf2xla/type_util.h"
-#include "tensorflow/compiler/tf2xla/xla_helpers.h"
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/lib/arithmetic.h"
@@ -66,9 +65,9 @@ void XlaArgMinMaxOp::Compile(XlaOpKernelContext* ctx) {
   xla::XlaOp input = ctx->Input(0);
   xla::XlaOp output;
   if (is_min_) {
-    output = XlaHelpers::ArgMin(input, index_xla_type, axis);
+    output = xla::ArgMin(input, index_xla_type, axis);
   } else {
-    output = XlaHelpers::ArgMax(input, index_xla_type, axis);
+    output = xla::ArgMax(input, index_xla_type, axis);
   }
 
   ctx->SetOutput(0, output);
diff --git a/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc b/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc
index 3e7e8eae6e..30b993045c 100644
--- a/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc
+++ b/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc
@@ -16,9 +16,9 @@ limitations under the License.
 // Native XLA implementations of indexing ops.
 
 #include "tensorflow/compiler/tf2xla/type_util.h"
-#include "tensorflow/compiler/tf2xla/xla_helpers.h"
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
+#include "tensorflow/compiler/xla/client/lib/arithmetic.h"
 #include "tensorflow/compiler/xla/literal_util.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
 #include "tensorflow/core/framework/op_kernel.h"
@@ -74,7 +74,7 @@ class ArgMaxCustomCallOp : public XlaOpKernel {
     // shape isn't supported.
     if (!ctx->compiler()->options().allow_cpu_custom_calls ||
         (input_dims != 1 && input_dims != 2)) {
-      xla::XlaOp output = XlaHelpers::ArgMax(ctx->Input(0), output_type, axis);
+      xla::XlaOp output = xla::ArgMax(ctx->Input(0), output_type, axis);
       ctx->SetOutput(0, output);
       return;
     }
diff --git a/tensorflow/compiler/tf2xla/xla_helpers.cc b/tensorflow/compiler/tf2xla/xla_helpers.cc
index 00035d24b7..04a5d93406 100644
--- a/tensorflow/compiler/tf2xla/xla_helpers.cc
+++ b/tensorflow/compiler/tf2xla/xla_helpers.cc
@@ -34,63 +34,6 @@ limitations under the License.
 
 namespace tensorflow {
 
-namespace {
-
-xla::XlaOp ArgMinMax(xla::XlaOp input, xla::PrimitiveType output_type, int axis,
-                     bool is_min) {
-  xla::XlaBuilder* builder = input.builder();
-  return builder->ReportErrorOrReturn([&]() -> xla::StatusOr<xla::XlaOp> {
-    TF_ASSIGN_OR_RETURN(xla::Shape input_shape, builder->GetShape(input));
-    xla::XlaOp init_value;
-    xla::XlaComputation reducer;
-    if (is_min) {
-      init_value = xla::MaxValue(builder, input_shape.element_type());
-      reducer =
-          xla::CreateScalarMinComputation(input_shape.element_type(), builder);
-    } else {
-      init_value = xla::MinValue(builder, input_shape.element_type());
-      reducer =
-          xla::CreateScalarMaxComputation(input_shape.element_type(), builder);
-    }
-
-    xla::XlaOp input_max = xla::Reduce(input, init_value, reducer,
-                                       /*dimensions_to_reduce=*/{axis});
-    std::vector<int64> broadcast_dims(input_shape.rank() - 1);
-    std::iota(broadcast_dims.begin(), broadcast_dims.begin() + axis, 0);
-    std::iota(broadcast_dims.begin() + axis, broadcast_dims.end(), axis + 1);
-    // Compute a mask that has 1s for elements equal to the maximum.
-    xla::XlaOp partial_mask = xla::ConvertElementType(
-        xla::Eq(input, input_max, broadcast_dims), output_type);
-
-    // In order to make identity elements for a bitwise And, we:
-    //   Left shift the 1 to the leftmost bit, yielding 0x10...0
-    //   Arithmetic right shift the 1 back to the rightmost bit, yielding
-    //   0xFF...F
-    int32 bits_in_type =
-        xla::ShapeUtil::ByteSizeOfPrimitiveType(output_type) * 8 - 1;
-    xla::XlaOp shift_amount =
-        xla::ConstantR0WithType(builder, output_type, bits_in_type);
-    xla::XlaOp full_mask = xla::ShiftRightArithmetic(
-        xla::ShiftLeft(partial_mask, shift_amount), shift_amount);
-
-    // And with the vector [0, 1, 2, ...] to convert each 0xFF...F into its
-    // index.
-
-    const int64 axis_size = xla::ShapeUtil::GetDimension(input_shape, axis);
-    xla::XlaOp iota = xla::Iota(builder, output_type, axis_size);
-    xla::XlaOp product =
-        xla::And(full_mask, iota, /*broadcast_dimensions=*/{axis});
-
-    // If there are multiple maximum elements, choose the one with the highest
-    // index.
-    return xla::Reduce(product, xla::MinValue(builder, output_type),
-                       xla::CreateScalarMaxComputation(output_type, builder),
-                       /*dimensions_to_reduce=*/{axis});
-  });
-}
-
-}  // namespace
-
 xla::XlaOp XlaHelpers::Zero(xla::XlaBuilder* b, DataType data_type) {
   xla::PrimitiveType type;
   TF_CHECK_OK(DataTypeToPrimitiveType(data_type, &type));
@@ -148,16 +91,6 @@ static Tensor MakeLinspaceTensor(const TensorShape& shape, int64 depth) {
   return linspace;
 }
 
-xla::XlaOp XlaHelpers::ArgMax(xla::XlaOp input, xla::PrimitiveType output_type,
-                              int axis) {
-  return ArgMinMax(input, output_type, axis, /*is_min=*/false);
-}
-
-xla::XlaOp XlaHelpers::ArgMin(xla::XlaOp input, xla::PrimitiveType output_type,
-                              int axis) {
-  return ArgMinMax(input, output_type, axis, /*is_min=*/true);
-}
-
 Status XlaHelpers::OneHot(xla::XlaBuilder* builder, int64 depth, int axis,
                           DataType index_type, const TensorShape& indices_shape,
                           const xla::XlaOp& indices, const xla::XlaOp& on_value,
diff --git a/tensorflow/compiler/tf2xla/xla_helpers.h b/tensorflow/compiler/tf2xla/xla_helpers.h
index 4858dfee55..490923526b 100644
--- a/tensorflow/compiler/tf2xla/xla_helpers.h
+++ b/tensorflow/compiler/tf2xla/xla_helpers.h
@@ -53,16 +53,6 @@ class XlaHelpers {
                                absl::Span<const int64> shape,
                                xla::Literal* output);
 
-  // Returns the argmax of `input` along `axis`. `output_type` is the type to
-  // use for the output.
-  static xla::XlaOp ArgMax(xla::XlaOp input, xla::PrimitiveType output_type,
-                           int axis);
-
-  // Returns the argmin of `input` along `axis`. `output_type` is the type to
-  // use for the output.
-  static xla::XlaOp ArgMin(xla::XlaOp input, xla::PrimitiveType output_type,
-                           int axis);
-
   // Converts `indices` into a one-hot representation. `depth` is the size
   // of the new axis to add. `axis` is the position at which to add the new
   // axis. `indices_shape` is the shape of `indices`. `on_value` and
diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD
index d863ebfea1..df1ee330f1 100644
--- a/tensorflow/compiler/xla/client/lib/BUILD
+++ b/tensorflow/compiler/xla/client/lib/BUILD
@@ -34,6 +34,21 @@ cc_library(
     ],
 )
 
+xla_test(
+    name = "arithmetic_test",
+    srcs = ["arithmetic_test.cc"],
+    deps = [
+        ":arithmetic",
+        "//tensorflow/compiler/xla:literal_util",
+        "//tensorflow/compiler/xla:test",
+        "//tensorflow/compiler/xla:types",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/client:xla_builder",
+        "//tensorflow/compiler/xla/tests:client_library_test_base",
+        "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+    ],
+)
+
 cc_library(
     name = "cholesky",
     srcs = ["cholesky.cc"],
diff --git a/tensorflow/compiler/xla/client/lib/arithmetic.cc b/tensorflow/compiler/xla/client/lib/arithmetic.cc
index 33ff3971d7..3b875135af 100644
--- a/tensorflow/compiler/xla/client/lib/arithmetic.cc
+++ b/tensorflow/compiler/xla/client/lib/arithmetic.cc
@@ -123,4 +123,64 @@ XlaOp Any(XlaOp predicates) {
   });
 }
 
+namespace {
+
+XlaOp ArgMinMax(XlaOp input, PrimitiveType output_type, int axis, bool is_min) {
+  XlaBuilder* builder = input.builder();
+  return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    TF_ASSIGN_OR_RETURN(Shape input_shape, builder->GetShape(input));
+    XlaOp init_value;
+    XlaComputation reducer;
+    if (is_min) {
+      init_value = MaxValue(builder, input_shape.element_type());
+      reducer = CreateScalarMinComputation(input_shape.element_type(), builder);
+    } else {
+      init_value = MinValue(builder, input_shape.element_type());
+      reducer = CreateScalarMaxComputation(input_shape.element_type(), builder);
+    }
+
+    XlaOp input_max = Reduce(input, init_value, reducer,
+                             /*dimensions_to_reduce=*/{axis});
+    std::vector<int64> broadcast_dims(input_shape.rank() - 1);
+    std::iota(broadcast_dims.begin(), broadcast_dims.begin() + axis, 0);
+    std::iota(broadcast_dims.begin() + axis, broadcast_dims.end(), axis + 1);
+    // Compute a mask that has 1s for elements equal to the maximum.
+    XlaOp partial_mask =
+        ConvertElementType(Eq(input, input_max, broadcast_dims), output_type);
+
+    // In order to make identity elements for a bitwise And, we:
+    //   Left shift the 1 to the leftmost bit, yielding 0x10...0
+    //   Arithmetic right shift the 1 back to the rightmost bit, yielding
+    //   0xFF...F
+    int32 bits_in_type =
+        ShapeUtil::ByteSizeOfPrimitiveType(output_type) * 8 - 1;
+    XlaOp shift_amount = ConstantR0WithType(builder, output_type, bits_in_type);
+    XlaOp full_mask = ShiftRightArithmetic(
+        ShiftLeft(partial_mask, shift_amount), shift_amount);
+
+    // And with the vector [0, 1, 2, ...] to convert each 0xFF...F into its
+    // index.
+
+    const int64 axis_size = ShapeUtil::GetDimension(input_shape, axis);
+    XlaOp iota = Iota(builder, output_type, axis_size);
+    XlaOp product = And(full_mask, iota, /*broadcast_dimensions=*/{axis});
+
+    // If there are multiple maximum elements, choose the one with the highest
+    // index.
+    return Reduce(product, MinValue(builder, output_type),
+                  CreateScalarMaxComputation(output_type, builder),
+                  /*dimensions_to_reduce=*/{axis});
+  });
+}
+
+}  // namespace
+
+XlaOp ArgMax(XlaOp input, PrimitiveType output_type, int axis) {
+  return ArgMinMax(input, output_type, axis, /*is_min=*/false);
+}
+
+XlaOp ArgMin(XlaOp input, PrimitiveType output_type, int axis) {
+  return ArgMinMax(input, output_type, axis, /*is_min=*/true);
+}
+
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/client/lib/arithmetic.h b/tensorflow/compiler/xla/client/lib/arithmetic.h
index 632e8cc8bc..d4a7812c44 100644
--- a/tensorflow/compiler/xla/client/lib/arithmetic.h
+++ b/tensorflow/compiler/xla/client/lib/arithmetic.h
@@ -57,6 +57,14 @@ XlaComputation CreateScalarOrComputation(PrimitiveType type,
 // Note: if predicates is zero-sized, Any() vacuously returns false.
 XlaOp Any(XlaOp predicates);
 
+// Returns the argmax of `input` along `axis`. `output_type` is the type to
+// use for the output.
+XlaOp ArgMax(XlaOp input, PrimitiveType output_type, int axis);
+
+// Returns the argmin of `input` along `axis`. `output_type` is the type to
+// use for the output.
+XlaOp ArgMin(XlaOp input, PrimitiveType output_type, int axis);
+
 }  // namespace xla
 
 #endif  // TENSORFLOW_COMPILER_XLA_CLIENT_LIB_ARITHMETIC_H_
diff --git a/tensorflow/compiler/xla/client/lib/arithmetic_test.cc b/tensorflow/compiler/xla/client/lib/arithmetic_test.cc
new file mode 100644
index 0000000000..a13839f9db
--- /dev/null
+++ b/tensorflow/compiler/xla/client/lib/arithmetic_test.cc
@@ -0,0 +1,67 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/client/lib/arithmetic.h"
+#include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/compiler/xla/test.h"
+#include "tensorflow/compiler/xla/tests/client_library_test_base.h"
+#include "tensorflow/compiler/xla/tests/test_macros.h"
+#include "tensorflow/compiler/xla/types.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+
+namespace xla {
+namespace {
+
+using ArithmeticTest = ClientLibraryTestBase;
+
+XLA_TEST_F(ArithmeticTest, ArgMinR2Axis0) {
+  XlaBuilder builder(TestName());
+  auto x = ConstantR2<int32>(&builder, {{1, 7, 4}, {6, 3, 5}, {8, 3, 3}});
+  ArgMin(x, S32, /*axis=*/0);
+
+  std::vector<int32> expected = {0, 2, 2};
+  ComputeAndCompareR1<int32>(&builder, expected, {});
+}
+
+XLA_TEST_F(ArithmeticTest, ArgMinR2Axis1) {
+  XlaBuilder builder(TestName());
+  auto x = ConstantR2<int32>(&builder, {{1, 7, 4}, {6, 3, 5}, {8, 3, 3}});
+  ArgMin(x, S32, /*axis=*/1);
+
+  std::vector<int32> expected = {0, 1, 2};
+  ComputeAndCompareR1<int32>(&builder, expected, {});
+}
+
+XLA_TEST_F(ArithmeticTest, ArgMaxR2Axis0) {
+  XlaBuilder builder(TestName());
+  auto x = ConstantR2<int32>(&builder, {{1, 7, 4}, {6, 3, 5}, {8, 3, 3}});
+  ArgMax(x, S32, /*axis=*/0);
+
+  std::vector<int32> expected = {2, 0, 1};
+  ComputeAndCompareR1<int32>(&builder, expected, {});
+}
+
+XLA_TEST_F(ArithmeticTest, ArgMaxR2Axis1) {
+  XlaBuilder builder(TestName());
+  auto x = ConstantR2<int32>(&builder, {{1, 7, 4}, {6, 3, 5}, {8, 3, 3}});
+  ArgMax(x, S32, /*axis=*/1);
+
+  std::vector<int32> expected = {1, 0, 0};
+  ComputeAndCompareR1<int32>(&builder, expected, {});
+}
+
+}  // namespace
+}  // namespace xla
-- 
GitLab


From 989c8b40a9b3b7bd45aedef43315675b0d4baf8d Mon Sep 17 00:00:00 2001
From: Adam Roberts <adarob@google.com>
Date: Tue, 8 Jan 2019 12:03:30 -0800
Subject: [PATCH 0359/2345] Disable writing examples/sec and global_step/sec
 summaries on TPU if save_summary_steps is Falsey.

PiperOrigin-RevId: 228373932
---
 tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 98babbb543..2317300041 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -2654,7 +2654,7 @@ class TPUEstimator(estimator_lib.Estimator):
         if self._log_every_n_steps is not None:
           examples_hook = ExamplesPerSecondHook(
               ctx.global_batch_size,
-              output_dir=self.model_dir,
+              output_dir=self.model_dir if config.save_summary_steps else None,
               every_n_steps=self._log_every_n_steps)
 
         if ctx.is_running_on_cpu(is_export_mode=is_export_mode):
-- 
GitLab


From 67d8a0330b5bacc0bb45675b1e3f211bd930ec7f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 12:18:59 -0800
Subject: [PATCH 0360/2345] Fixing flaky test in quantile regression

PiperOrigin-RevId: 228376536
---
 .../estimator_batch/estimator_test.py            | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py b/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py
index ee052ac603..47d910d42a 100644
--- a/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py
+++ b/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py
@@ -487,8 +487,8 @@ class BoostedTreeEstimatorTest(test_util.TensorFlowTestCase):
     self.assertTrue(frac_below_upper_0 <= 0.98)
     self.assertTrue(frac_below_upper_1 >= 0.92)
     self.assertTrue(frac_below_upper_1 <= 0.98)
-    self.assertTrue(frac_both_below_upper >= 0.92)
-    self.assertTrue(frac_both_below_upper <= 0.98)
+    self.assertTrue(frac_both_below_upper >= 0.91)
+    self.assertTrue(frac_both_below_upper <= 0.99)
 
     train_input_fn, test_input_fn, _ = _quantile_regression_input_fns(
         two_dimension=True)
@@ -516,8 +516,8 @@ class BoostedTreeEstimatorTest(test_util.TensorFlowTestCase):
     self.assertTrue(frac_above_lower_0 <= 0.98)
     self.assertTrue(frac_above_lower_1 >= 0.92)
     self.assertTrue(frac_above_lower_1 <= 0.98)
-    self.assertTrue(frac_both_above_lower >= 0.92)
-    self.assertTrue(frac_both_above_lower <= 0.98)
+    self.assertTrue(frac_both_above_lower >= 0.91)
+    self.assertTrue(frac_both_above_lower <= 0.99)
 
 
 class CoreGradientBoostedDecisionTreeEstimators(test_util.TensorFlowTestCase):
@@ -806,8 +806,8 @@ class CoreGradientBoostedDecisionTreeEstimators(test_util.TensorFlowTestCase):
     self.assertTrue(frac_below_upper_0 <= 0.98)
     self.assertTrue(frac_below_upper_1 >= 0.92)
     self.assertTrue(frac_below_upper_1 <= 0.98)
-    self.assertTrue(frac_both_below_upper >= 0.92)
-    self.assertTrue(frac_both_below_upper <= 0.98)
+    self.assertTrue(frac_both_below_upper >= 0.91)
+    self.assertTrue(frac_both_below_upper <= 0.99)
 
     train_input_fn, test_input_fn, _ = _quantile_regression_input_fns(
         two_dimension=True)
@@ -835,8 +835,8 @@ class CoreGradientBoostedDecisionTreeEstimators(test_util.TensorFlowTestCase):
     self.assertTrue(frac_above_lower_0 <= 0.98)
     self.assertTrue(frac_above_lower_1 >= 0.92)
     self.assertTrue(frac_above_lower_1 <= 0.98)
-    self.assertTrue(frac_both_above_lower >= 0.92)
-    self.assertTrue(frac_both_above_lower <= 0.98)
+    self.assertTrue(frac_both_above_lower >= 0.91)
+    self.assertTrue(frac_both_above_lower <= 0.99)
 
 
 if __name__ == "__main__":
-- 
GitLab


From a3d639a5a46df8f3523bd514cade1faaf602b66d Mon Sep 17 00:00:00 2001
From: Blake Hechtman <blakehechtman@google.com>
Date: Tue, 8 Jan 2019 12:22:54 -0800
Subject: [PATCH 0361/2345] [XLA] Canonicalize dot dimension numbers on CPU and
 GPU backends cases that previously failed shape inference.

PiperOrigin-RevId: 228377214
---
 tensorflow/compiler/xla/service/BUILD         |   3 +-
 .../compiler/xla/service/dot_decomposer.cc    | 174 +++++++++++++++++-
 tensorflow/compiler/xla/service/gpu/BUILD     |   1 +
 .../xla/service/gpu/nvptx_compiler.cc         |   2 +
 tensorflow/compiler/xla/tests/BUILD           |   2 +
 .../compiler/xla/tests/dot_operation_test.cc  |  34 ++++
 6 files changed, 208 insertions(+), 8 deletions(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index ec63ae1ac6..728adb67a0 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -1873,8 +1873,9 @@ cc_library(
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:status_macros",
         "//tensorflow/compiler/xla:types",
-        "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/core:lib",
+        "@com_google_absl//absl/algorithm:container",
+        "@com_google_absl//absl/strings",
     ],
 )
 
diff --git a/tensorflow/compiler/xla/service/dot_decomposer.cc b/tensorflow/compiler/xla/service/dot_decomposer.cc
index b2ba261790..855424067d 100644
--- a/tensorflow/compiler/xla/service/dot_decomposer.cc
+++ b/tensorflow/compiler/xla/service/dot_decomposer.cc
@@ -15,6 +15,8 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/service/dot_decomposer.h"
 
+#include "absl/algorithm/container.h"
+#include "absl/strings/str_join.h"
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
@@ -156,29 +158,187 @@ Status DecomposeBatchDot(HloInstruction* dot) {
   return computation->ReplaceInstruction(dot, new_dot);
 }
 
+// Convert a dot into a canonical form where non-contracting and contracting
+// dimensions are reshaped together and batch dimensions are the most major
+// dimensions. The requires transposing and reshapes the lhs and rhs and
+// reshaping the output batch to the original shape.
+Status CanonicalizeDot(HloInstruction* original_dot) {
+  auto computation = original_dot->parent();
+  const auto& original_dnums = original_dot->dot_dimension_numbers();
+  const int64 num_batch_dims = original_dnums.lhs_batch_dimensions_size();
+  const int64 num_contracting_dims =
+      original_dnums.lhs_contracting_dimensions_size();
+
+  const auto& lhs_shape = original_dot->operand(0)->shape();
+  const int64 lhs_rank = lhs_shape.rank();
+  const int64 num_lhs_non_contracting_dims =
+      lhs_rank - num_batch_dims - num_contracting_dims;
+
+  std::vector<int64> lhs_non_contracting_dims;
+  lhs_non_contracting_dims.reserve(num_lhs_non_contracting_dims);
+  int64 lhs_contracting_size = 1;
+  int64 lhs_non_contracting_size = 1;
+  std::vector<int64> batch_dim_sizes;
+  batch_dim_sizes.reserve(num_batch_dims);
+  for (int64 i = 0; i < lhs_rank; ++i) {
+    if (absl::c_linear_search(original_dnums.lhs_contracting_dimensions(), i)) {
+      lhs_contracting_size *= lhs_shape.dimensions(i);
+    } else if (absl::c_linear_search(original_dnums.lhs_batch_dimensions(),
+                                     i)) {
+      batch_dim_sizes.push_back(lhs_shape.dimensions(i));
+    } else {
+      lhs_non_contracting_dims.push_back(i);
+      lhs_non_contracting_size *= lhs_shape.dimensions(i);
+    }
+  }
+  // The canonical form of the lhs is
+  // [BatchDims, NonContractingDims, ContractingsDims]
+  std::vector<int64> lhs_transpose;
+  lhs_transpose.reserve(lhs_rank);
+  lhs_transpose.insert(lhs_transpose.end(),
+                       original_dnums.lhs_batch_dimensions().begin(),
+                       original_dnums.lhs_batch_dimensions().end());
+  lhs_transpose.insert(lhs_transpose.end(), lhs_non_contracting_dims.begin(),
+                       lhs_non_contracting_dims.end());
+  lhs_transpose.insert(lhs_transpose.end(),
+                       original_dnums.lhs_contracting_dimensions().begin(),
+                       original_dnums.lhs_contracting_dimensions().end());
+  HloInstruction* transposed_lhs =
+      computation->AddInstruction(HloInstruction::CreateTranspose(
+          ShapeUtil::PermuteDimensions(InversePermutation(lhs_transpose),
+                                       lhs_shape),
+          original_dot->mutable_operand(0), lhs_transpose));
+  std::vector<int64> lhs_reshape_dims = batch_dim_sizes;
+  lhs_reshape_dims.push_back(lhs_non_contracting_size);
+  lhs_reshape_dims.push_back(lhs_contracting_size);
+  // Reshape the contracting and non-contracting dimensions together.
+  HloInstruction* reshaped_lhs =
+      computation->AddInstruction(HloInstruction::CreateReshape(
+          ShapeUtil::MakeShape(lhs_shape.element_type(), lhs_reshape_dims),
+          transposed_lhs));
+
+  const auto& rhs_shape = original_dot->operand(1)->shape();
+  const int64 rhs_rank = rhs_shape.rank();
+  const int64 num_rhs_non_contracting_dims =
+      rhs_rank - num_batch_dims - num_contracting_dims;
+  std::vector<int64> rhs_non_contracting_dims;
+  rhs_non_contracting_dims.reserve(num_rhs_non_contracting_dims);
+  int64 rhs_non_contracting_size = 1;
+  int64 rhs_contracting_size = 1;
+  for (int64 i = 0; i < rhs_rank; ++i) {
+    if (absl::c_linear_search(original_dnums.rhs_contracting_dimensions(), i)) {
+      rhs_contracting_size *= rhs_shape.dimensions(i);
+    } else if (!absl::c_linear_search(original_dnums.rhs_batch_dimensions(),
+                                      i)) {
+      rhs_non_contracting_dims.push_back(i);
+      rhs_non_contracting_size *= rhs_shape.dimensions(i);
+    }
+  }
+
+  // The canonical form of the rhs is
+  // [BatchDims, ContractingsDims, NonContractingDims]
+  std::vector<int64> rhs_transpose;
+  rhs_transpose.reserve(rhs_rank);
+  rhs_transpose.insert(rhs_transpose.end(),
+                       original_dnums.rhs_batch_dimensions().begin(),
+                       original_dnums.rhs_batch_dimensions().end());
+  rhs_transpose.insert(rhs_transpose.end(),
+                       original_dnums.rhs_contracting_dimensions().begin(),
+                       original_dnums.rhs_contracting_dimensions().end());
+  rhs_transpose.insert(rhs_transpose.end(), rhs_non_contracting_dims.begin(),
+                       rhs_non_contracting_dims.end());
+  HloInstruction* transposed_rhs =
+      computation->AddInstruction(HloInstruction::CreateTranspose(
+          ShapeUtil::PermuteDimensions(InversePermutation(rhs_transpose),
+                                       rhs_shape),
+          original_dot->mutable_operand(1), rhs_transpose));
+
+  std::vector<int64> rhs_reshape_dims = batch_dim_sizes;
+  rhs_reshape_dims.push_back(rhs_contracting_size);
+  rhs_reshape_dims.push_back(rhs_non_contracting_size);
+  // Reshape the contracting and non-contracting dimensions together.
+  HloInstruction* reshaped_rhs =
+      computation->AddInstruction(HloInstruction::CreateReshape(
+          ShapeUtil::MakeShape(rhs_shape.element_type(), rhs_reshape_dims),
+          transposed_rhs));
+
+  std::vector<int64> dot_dims = batch_dim_sizes;
+  dot_dims.push_back(lhs_non_contracting_size);
+  dot_dims.push_back(rhs_non_contracting_size);
+
+  DotDimensionNumbers dot_dnums;
+  for (int64 i = 0; i < num_batch_dims; ++i) {
+    dot_dnums.add_lhs_batch_dimensions(i);
+    dot_dnums.add_rhs_batch_dimensions(i);
+  }
+  dot_dnums.add_lhs_contracting_dimensions(num_batch_dims + 1);
+  dot_dnums.add_rhs_contracting_dimensions(num_batch_dims);
+
+  HloInstruction* dot = computation->AddInstruction(HloInstruction::CreateDot(
+      ShapeUtil::MakeShape(original_dot->shape().element_type(), dot_dims),
+      reshaped_lhs, reshaped_rhs, dot_dnums, original_dot->precision_config()));
+
+  return computation->ReplaceInstruction(
+      original_dot, computation->AddInstruction(HloInstruction::CreateReshape(
+                        original_dot->shape(), dot)));
+}
+
 }  // namespace
 
 StatusOr<bool> DotDecomposer::Run(HloModule* module) {
   XLA_VLOG_LINES(2, "DotDecomposer ENTRY\n" + module->ToString());
-  // Gather all batch Dot operations.
-  std::vector<HloInstruction*> batch_dots;
+  // Gather all Non-canonical Dot operations.
+  std::vector<HloInstruction*> non_canonical_dots;
   for (auto* computation : module->MakeNonfusionComputations()) {
     for (auto* instruction : computation->instructions()) {
       if (instruction->opcode() != HloOpcode::kDot) {
         continue;
       }
       const DotDimensionNumbers& dnums = instruction->dot_dimension_numbers();
-      if (dnums.lhs_batch_dimensions_size() > 0 && decompose_batch_dot_) {
-        batch_dots.push_back(instruction);
+      // A dot it not canonical if there are more than one contracting
+      // dimension.
+      if (dnums.lhs_contracting_dimensions_size() > 1) {
+        non_canonical_dots.push_back(instruction);
+        continue;
+      }
+      if (dnums.lhs_batch_dimensions().empty()) {
+        continue;
+      }
+      std::vector<int64> canonical_batch_dims(
+          dnums.lhs_batch_dimensions_size());
+      absl::c_iota(canonical_batch_dims, 0);
+      if (!absl::c_equal(dnums.lhs_batch_dimensions(), canonical_batch_dims) ||
+          !absl::c_equal(dnums.rhs_batch_dimensions(), canonical_batch_dims)) {
+        non_canonical_dots.push_back(instruction);
       }
     }
   }
-  // Decompose each batch Dot in 'batch_dots'.
   bool changed = false;
-  for (auto* dot : batch_dots) {
-    TF_RETURN_IF_ERROR(DecomposeBatchDot(dot));
+  for (auto* dot : non_canonical_dots) {
+    TF_RETURN_IF_ERROR(CanonicalizeDot(dot));
     changed = true;
   }
+
+  if (decompose_batch_dot_) {
+    std::vector<HloInstruction*> batch_dots;
+    for (auto* computation : module->MakeNonfusionComputations()) {
+      for (auto* instruction : computation->instructions()) {
+        if (instruction->opcode() != HloOpcode::kDot) {
+          continue;
+        }
+        const DotDimensionNumbers& dnums = instruction->dot_dimension_numbers();
+        if (!dnums.lhs_batch_dimensions().empty()) {
+          batch_dots.push_back(instruction);
+        }
+      }
+    }
+    // Decompose each batch Dot in 'batch_dots'.
+
+    for (auto* dot : batch_dots) {
+      TF_RETURN_IF_ERROR(DecomposeBatchDot(dot));
+      changed = true;
+    }
+  }
   XLA_VLOG_LINES(2, "DotDecompose EXIT\n" + module->ToString());
   return changed;
 }
diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD
index 5e81281bae..30a8bda606 100644
--- a/tensorflow/compiler/xla/service/gpu/BUILD
+++ b/tensorflow/compiler/xla/service/gpu/BUILD
@@ -699,6 +699,7 @@ cc_library(
         "//tensorflow/compiler/xla/service:call_inliner",
         "//tensorflow/compiler/xla/service:conditional_simplifier",
         "//tensorflow/compiler/xla/service:convolution_group_converter",
+        "//tensorflow/compiler/xla/service:dot_decomposer",
         "//tensorflow/compiler/xla/service:dynamic_index_splitter",
         "//tensorflow/compiler/xla/service:executable",
         "//tensorflow/compiler/xla/service:flatten_call_graph",
diff --git a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
index 9be7609508..d1522280ba 100644
--- a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
+++ b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
@@ -37,6 +37,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/call_inliner.h"
 #include "tensorflow/compiler/xla/service/conditional_simplifier.h"
 #include "tensorflow/compiler/xla/service/convolution_group_converter.h"
+#include "tensorflow/compiler/xla/service/dot_decomposer.h"
 #include "tensorflow/compiler/xla/service/dynamic_index_splitter.h"
 #include "tensorflow/compiler/xla/service/flatten_call_graph.h"
 #include "tensorflow/compiler/xla/service/gpu/cudnn_batchnorm_rewriter.h"
@@ -165,6 +166,7 @@ Status OptimizeHloModule(HloModule* hlo_module, se::StreamExecutor* stream_exec,
       // We need a cost model for GPUs. Currently, do nothing.
       return false;
     };
+    pipeline.AddPass<DotDecomposer>(false);
     pipeline.AddPass<ConvolutionGroupConverter>(
         cost_model,
         /*convert_batch_groups_only=*/true);
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 106b3fd6c9..0fd0fc108a 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -711,6 +711,7 @@ xla_test(
         "//tensorflow/compiler/xla/client:local_client",
         "//tensorflow/compiler/xla/client:xla_builder",
         "//tensorflow/compiler/xla/tests:client_library_test_base",
+        "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
         "//tensorflow/compiler/xla/tests:test_utils",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
@@ -768,6 +769,7 @@ xla_test(
         "//tensorflow/compiler/xla/client:local_client",
         "//tensorflow/compiler/xla/client:xla_builder",
         "//tensorflow/compiler/xla/tests:client_library_test_base",
+        "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
         "//tensorflow/compiler/xla/tests:test_utils",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
diff --git a/tensorflow/compiler/xla/tests/dot_operation_test.cc b/tensorflow/compiler/xla/tests/dot_operation_test.cc
index c5d8b663f4..2e02968ac5 100644
--- a/tensorflow/compiler/xla/tests/dot_operation_test.cc
+++ b/tensorflow/compiler/xla/tests/dot_operation_test.cc
@@ -25,6 +25,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/reference_util.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/tests/client_library_test_base.h"
+#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
 #include "tensorflow/compiler/xla/tests/literal_test_util.h"
 #include "tensorflow/compiler/xla/tests/test_macros.h"
 #include "tensorflow/compiler/xla/tests/test_utils.h"
@@ -1147,5 +1148,38 @@ XLA_TEST_F(DotOperationTest, DotRank2AndRank2NonDefaultContractionDims) {
 
   ComputeAndCompareR2<float>(&builder, expected, {}, error_spec_);
 }
+
+class DotOperationTextTest : public HloTestBase {};
+
+XLA_TEST_F(DotOperationTextTest, DotReorderedDotDims) {
+  absl::string_view hlo_string =
+      R"(
+HloModule ComplexDotMultipleNonContracting
+
+ENTRY %test {
+  %lhs = f32[7,17,10,13]{3,2,1,0} parameter(0)
+  %rhs = f32[7,9,10,13,6]{4,3,2,1,0} parameter(1)
+  ROOT %dot = f32[10,7,17,9,6]{4,3,2,1,0} dot(%lhs, %rhs), lhs_batch_dims={2,0}, rhs_batch_dims={2,0}, lhs_contracting_dims={3}, rhs_contracting_dims={3}
+}
+)";
+
+  EXPECT_TRUE(RunAndCompare(hlo_string, ErrorSpec{1e-3, 1e-3}));
+}
+
+XLA_TEST_F(DotOperationTextTest, DotReorderedDotDimsAndMultipleContracting) {
+  absl::string_view hlo_string =
+      R"(
+HloModule ComplexDotMultipleNonContracting
+
+ENTRY %test {
+  %lhs = f32[7,5,17,10,13]{4,3,2,1,0} parameter(0)
+  %rhs = f32[7,9,10,13,6,5]{5,4,3,2,1,0} parameter(1)
+  ROOT %dot = f32[10,7,17,9,6]{4,3,2,1,0} dot(%lhs, %rhs), lhs_batch_dims={3,0}, rhs_batch_dims={2,0}, lhs_contracting_dims={1,4}, rhs_contracting_dims={5,3}
+}
+)";
+
+  EXPECT_TRUE(RunAndCompare(hlo_string, ErrorSpec{1e-3, 1e-3}));
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 495b3eeef0386b1b89b7aa9df42f2cf438de6ebc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 12:55:56 -0800
Subject: [PATCH 0362/2345] Enable fast GPU code path for solving many small
 linear systems in matrix_solve.

PiperOrigin-RevId: 228382682
---
 tensorflow/core/kernels/cuda_solvers.cc    | 12 +++++-----
 tensorflow/core/kernels/cuda_solvers.h     |  5 ++--
 tensorflow/core/kernels/matrix_solve_op.cc | 28 +++++++++++++---------
 3 files changed, 26 insertions(+), 19 deletions(-)

diff --git a/tensorflow/core/kernels/cuda_solvers.cc b/tensorflow/core/kernels/cuda_solvers.cc
index a59baaa96f..39d0a998fd 100644
--- a/tensorflow/core/kernels/cuda_solvers.cc
+++ b/tensorflow/core/kernels/cuda_solvers.cc
@@ -692,8 +692,8 @@ static inline Status GetrsBatchedImpl(
     SolverFnT solver, CudaSolver* cuda_solver, OpKernelContext* context,
     cublasHandle_t cublas_handle, cublasOperation_t trans, int n, int nrhs,
     const Scalar* const host_a_dev_ptrs[], int lda, const int* dev_pivots,
-    const Scalar* const host_b_dev_ptrs[], int ldb,
-    DeviceLapackInfo* dev_lapack_info, int batch_size) {
+    const Scalar* const host_b_dev_ptrs[], int ldb, int* host_lapack_info,
+    int batch_size) {
   mutex_lock lock(handle_map_mutex);
   using CudaScalar = typename CUDAComplexT<Scalar>::type;
   ScratchSpace<uint8> dev_a_dev_ptrs =
@@ -714,7 +714,7 @@ static inline Status GetrsBatchedImpl(
       cublas_handle, trans, n, nrhs,
       reinterpret_cast<const CudaScalar* const*>(dev_a_dev_ptrs.data()), lda,
       dev_pivots, reinterpret_cast<CudaScalar**>(dev_b_dev_ptrs.mutable_data()),
-      ldb, dev_lapack_info->mutable_data(), batch_size));
+      ldb, host_lapack_info, batch_size));
   return Status::OK();
 }
 
@@ -723,13 +723,13 @@ static inline Status GetrsBatchedImpl(
   Status CudaSolver::GetrsBatched(                                             \
       cublasOperation_t trans, int n, int nrhs,                                \
       const Scalar* const host_a_dev_ptrs[], int lda, const int* dev_pivots,   \
-      const Scalar* const host_b_dev_ptrs[], int ldb,                          \
-      DeviceLapackInfo* dev_lapack_info, int batch_size) {                     \
+      const Scalar* const host_b_dev_ptrs[], int ldb, int* host_lapack_info,   \
+      int batch_size) {                                                        \
     return GetrsBatchedImpl(reinterpret_cast<getrs_##type_prefix*>(            \
                                 BLAS_SOLVER_FN(getrsBatched, type_prefix)),    \
                             this, context_, cublas_handle_, trans, n, nrhs,    \
                             host_a_dev_ptrs, lda, dev_pivots, host_b_dev_ptrs, \
-                            ldb, dev_lapack_info, batch_size);                 \
+                            ldb, host_lapack_info, batch_size);                \
   }
 
 TF_CALL_LAPACK_TYPES(GETRS_BATCHED_INSTANCE);
diff --git a/tensorflow/core/kernels/cuda_solvers.h b/tensorflow/core/kernels/cuda_solvers.h
index 2c30d036df..1fc344731c 100644
--- a/tensorflow/core/kernels/cuda_solvers.h
+++ b/tensorflow/core/kernels/cuda_solvers.h
@@ -235,13 +235,14 @@ class CudaSolver {
                       int batch_size) TF_MUST_USE_RESULT;
 
   // Batched linear solver using LU factorization from getrfBatched.
-  // See:
+  // Notice that lapack_info is returned on the host, as opposed to
+  // most of the other functions that return it on the device. See:
   // http://docs.nvidia.com/cuda/cublas/index.html#cublas-lt-t-gt-getrsbatched
   template <typename Scalar>
   Status GetrsBatched(cublasOperation_t trans, int n, int nrhs,
                       const Scalar* const dev_Aarray[], int lda,
                       const int* devIpiv, const Scalar* const dev_Barray[],
-                      int ldb, DeviceLapackInfo* dev_lapack_info,
+                      int ldb, int* host_lapack_info,
                       int batch_size) TF_MUST_USE_RESULT;
 
   // Computes matrix inverses for a batch of small matrices. Uses the outputs
diff --git a/tensorflow/core/kernels/matrix_solve_op.cc b/tensorflow/core/kernels/matrix_solve_op.cc
index 169f3dae76..f3919a16aa 100644
--- a/tensorflow/core/kernels/matrix_solve_op.cc
+++ b/tensorflow/core/kernels/matrix_solve_op.cc
@@ -214,9 +214,12 @@ class MatrixSolveOpGpu : public AsyncOpKernel {
     auto input_copy_ptrs = solver->GetScratchSpace<uint8>(
         sizeof(Scalar*) * batch_size, "input_copt_ptrs",
         /* on_host */ true);
-    if (n / batch_size <= 128) {
-      // For small matrices or large batch sizes, we use the batched
-      // interface from cuBlas.
+    const int kMaxMatrixSizeToBatchSizeRatio = 128;
+    const bool use_batched_solver =
+        n <= kMaxMatrixSizeToBatchSizeRatio * batch_size;
+    if (use_batched_solver) {
+      // For small matrices or large batch sizes, we use the batched interface
+      // from cuBlas.
       const Scalar** input_copy_ptrs_base =
           reinterpret_cast<const Scalar**>(input_copy_ptrs.mutable_data());
       for (int batch = 0; batch < batch_size; ++batch) {
@@ -230,8 +233,8 @@ class MatrixSolveOpGpu : public AsyncOpKernel {
                                &dev_info.back(), batch_size),
           done);
     } else {
-      // For small batch sizes we use the non-batched interface from cuSolver,
-      // which is much faster for large matrices.
+      // For small batch sizes or large matrices, we use the non-batched
+      // interface from cuSolver, which is much faster for large matrices.
       dev_info.push_back(solver->GetDeviceLapackInfo(batch_size, "getrf"));
       for (int batch = 0; batch < batch_size; ++batch) {
         OP_REQUIRES_OK_ASYNC(
@@ -279,11 +282,7 @@ class MatrixSolveOpGpu : public AsyncOpKernel {
         /* on_host */ true);
     auto transposed_rhs_reshaped =
         transposed_rhs.template flat_inner_dims<Scalar, 3>();
-    // TODO(rmlarsen): Enable the following branch when I figure
-    // out why it causes a segfault.
-    if (false && n / batch_size <= 128) {
-      dev_info.push_back(
-          solver->GetDeviceLapackInfo(batch_size, "GetrsBatched"));
+    if (use_batched_solver) {
       const Scalar** input_copy_ptrs_base =
           reinterpret_cast<const Scalar**>(input_copy_ptr_array.mutable_data());
       const Scalar** transposed_rhs_ptrs_base =
@@ -293,13 +292,20 @@ class MatrixSolveOpGpu : public AsyncOpKernel {
         input_copy_ptrs_base[batch] = &input_copy_reshaped(batch, 0, 0);
         transposed_rhs_ptrs_base[batch] = &transposed_rhs_reshaped(batch, 0, 0);
       }
+      int host_info = 0;
       OP_REQUIRES_OK_ASYNC(
           context,
           solver->GetrsBatched(adjoint_ ? CUBLAS_OP_C : CUBLAS_OP_T, n, nrhs,
                                input_copy_ptrs_base, n, pivots_mat.data(),
-                               transposed_rhs_ptrs_base, n, &dev_info.back(),
+                               transposed_rhs_ptrs_base, n, &host_info,
                                batch_size),
           done);
+      OP_REQUIRES_ASYNC(
+          context, host_info == 0,
+          errors::InvalidArgument("The ", -host_info,
+                                  "'th argument to cublas*getrsBatched had "
+                                  "an illegal value."),
+          done);
     } else {
       dev_info.push_back(solver->GetDeviceLapackInfo(batch_size, "getrs"));
       for (int batch = 0; batch < batch_size; ++batch) {
-- 
GitLab


From de86e2fff13c5b2c41297ef923c2d60e5a7297e8 Mon Sep 17 00:00:00 2001
From: Thomas Deegan <tadeegan@gmail.com>
Date: Tue, 8 Jan 2019 13:29:33 -0800
Subject: [PATCH 0363/2345] Fix broken image

The image tag in this markdown file seems to be broken when viewed on github.
---
 tensorflow/core/profiler/g3doc/profile_memory.md | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tensorflow/core/profiler/g3doc/profile_memory.md b/tensorflow/core/profiler/g3doc/profile_memory.md
index 6eda5abdd9..a2c7cc6289 100644
--- a/tensorflow/core/profiler/g3doc/profile_memory.md
+++ b/tensorflow/core/profiler/g3doc/profile_memory.md
@@ -14,9 +14,7 @@ Open a Chrome browser, enter URL chrome://tracing and load the timeline file.
 ******************************************************
 ```
 
-<left>
 ![Timeline](graph_timeline.png)
-</left>
 
 
 ```python
@@ -77,4 +75,4 @@ _TFProfRoot (--/74148.60MB)
                   seq2seq_attention_model.py:320:_add_train_op:tf.summary.scalar... (0B/64B)
                 seq2seq_attention_model.py:360:build_graph:self._add_seq2seq() (0B/25216.74MB)
                   seq2seq_attention_model.py:192:_add_seq2seq:sequence_length=a... (0B/21542.55MB)
-```
\ No newline at end of file
+```
-- 
GitLab


From 01dd371b8441aa96536732b804f1c5853fb8af6e Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Tue, 8 Jan 2019 13:29:19 -0800
Subject: [PATCH 0364/2345] remove JackOptions block.

PiperOrigin-RevId: 228388690
---
 tensorflow/lite/g3doc/demo_ios.md             |  45 ++++++++++--------
 .../g3doc/images/ios/bundle_identifier.png    | Bin 0 -> 64902 bytes
 2 files changed, 26 insertions(+), 19 deletions(-)
 create mode 100644 tensorflow/lite/g3doc/images/ios/bundle_identifier.png

diff --git a/tensorflow/lite/g3doc/demo_ios.md b/tensorflow/lite/g3doc/demo_ios.md
index fbf1dd6392..f4b481dc61 100644
--- a/tensorflow/lite/g3doc/demo_ios.md
+++ b/tensorflow/lite/g3doc/demo_ios.md
@@ -7,22 +7,23 @@ instructions walk you through building and running the demo on an iOS device.
 
 ## Prerequisites
 
-* You must have [Xcode](https://developer.apple.com/xcode/) installed and have a
-  valid Apple Developer ID, and have an iOS device set up and linked to your
-  developer account with all of the appropriate certificates. For these
-  instructions, we assume that you have already been able to build and deploy an
-  app to an iOS device with your current developer environment.
+*   You must have [Xcode](https://developer.apple.com/xcode/) installed and have
+    a valid Apple Developer ID, and have an iOS device set up and linked to your
+    developer account with all of the appropriate certificates. For these
+    instructions, we assume that you have already been able to build and deploy
+    an app to an iOS device with your current developer environment.
 
-* The demo app requires a camera and must be executed on a real iOS device. You
-  can build it and run with the iPhone Simulator but it won't have any camera
-  information to classify.
+*   The demo app requires a camera and must be executed on a real iOS device.
+    You can build it and run with the iPhone Simulator but it won't have any
+    camera information to classify.
 
-* You don't need to build the entire TensorFlow library to run the demo, but you
-  will need to clone the TensorFlow repository if you haven't already:
+*   You don't need to build the entire TensorFlow library to run the demo, but
+    you will need to clone the TensorFlow repository if you haven't already:
 
         git clone https://github.com/tensorflow/tensorflow
+        cd tensorflow
 
-* You'll also need the Xcode command-line tools:
+*   You'll also need the Xcode command-line tools:
 
         xcode-select --install
 
@@ -31,35 +32,41 @@ instructions walk you through building and running the demo on an iOS device.
 
 ## Building the iOS Demo App
 
-1. Install CocoaPods if you don't have it:
+1.  Install CocoaPods if you don't have it:
 
         sudo gem install cocoapods
 
-2. Download the model files used by the demo app (this is done from inside the
-   cloned directory):
+2.  Download the model files used by the demo app (this is done from inside the
+    cloned directory):
 
         sh tensorflow/lite/examples/ios/download_models.sh
 
-3. Install the pod to generate the workspace file:
+3.  Install the pod to generate the workspace file:
 
         cd tensorflow/lite/examples/ios/camera
         pod install
 
     If you have installed this pod before and that command doesn't work, try
 
-        pod update
+        pod repo update
 
-    At the end of this step you should have a file called 
+    At the end of this step you should have a file called
     `tflite_camera_example.xcworkspace`.
 
-4. Open the project in Xcode by typing this on the command line:
+4.  Open the project in Xcode by typing this on the command line:
 
         open tflite_camera_example.xcworkspace
 
     This launches Xcode if it isn't open already and opens the
     `tflite_camera_example` project.
 
-5. Build and run the app in Xcode.
+5.  Under `Project navigator -> tflite_camera_example -> Targets ->
+    tflite_camera_example -> General` change the bundle identifier by
+    pre-pending your name:
+
+    ![pre-pend your name to the bundle identifier](images/ios/bundle_identifier.png)
+
+6.  Build and run the app in Xcode.
 
     Note that as mentioned earlier, you must already have a device set up and
     linked to your Apple Developer account in order to deploy the app on a
diff --git a/tensorflow/lite/g3doc/images/ios/bundle_identifier.png b/tensorflow/lite/g3doc/images/ios/bundle_identifier.png
new file mode 100644
index 0000000000000000000000000000000000000000..398763916b353e61f236392e2b8898aad2aafe8e
GIT binary patch
literal 64902
zcmeAS@N?(olHy`uVBq!ia0y~yU_Qmbz|h6P#=yW(bbs?v1_mDaOlRkS%;aPS29M6E
z)7c|}Pl`1>pQfUsvc-XANy|N#m!Vl1j4VsI%$$TgG!#`>1g$r%I1m-EF~K$5H7A=f
zWn-d#cfxyqZjDpBA~r1iq{g{m_5JGi)z5!FyJw$%ZjR;iJ3A*^GpsmXaO6==2y=r>
z;64>IlL?lJmaU#uECLP;A`7^EEMit#D5$9kIY{pMAAIj_!?l2Qw@%moz4Jfbb@K7)
zMT`zf3d=t(xg&Feo$<ik9Z8;c3@vgi%x6#2Z}VvhWXk2_5!keIj`5T?iqB0{-L-Na
zB}YyOW(-(ZWHE_{;mVUjL+%}t7y6@EQ`iOlHJa}5+?=lc;DbVan~j#by7;DDrYWzt
zy#JH(=WzP14gP%U$up+z)NoPt&=5^8UiN?f(;xRIP1cp3b(qI&m4lF&_W8)T?Q8CE
zSOzQ37hdY>>U^!K#`i?<jng~56}Y`6w){UZbyLDw24;pSryjGoI+!)&Xr1colDYDG
z8MpI|35jnnRwUY4$61O!^(;ze|GT)gCbQs<K(R{K&L?K6o7S{6IL?=5srfeH{4yoS
znn^}}Th=<vkYJ7E{bZXN7IS35+0ROh+F2*ul)9%L=b3NGc1cxnqnGp<8+WTI;?GnC
zHZ5keG>R9HUnbM-`>;r4<4dP=PrTJ#_>P`@%+9L)`B2x-MJ)$^Z1gIuOL(*3Z&+d3
znFY4j&Yb6)`r21<yBGJPwqxI(9X`73j5zzFKQk&)b9R)esoS_oGUstCF><pos!cdJ
zjjhO0O<>zMlP?G2Pt6lJz$i3<@yCI>0Amv+&x^`U*DV5A{xuz8zx1%Mu`A9_qv4nS
z;i#uq<(6nyaf>K#au9zaBfz-nVgAgdDH#>pTt2X5wb-9M*Saz7u-g0Ip2tJvF9|kx
zIvz1d;<(6m{djF*zKu}a-^AiWwgu-G?$~!zco)ap7WbCdEqsUGt~``<l7FJ{L5mBF
z$vl#0^xpn{Q{esLswn@F+6DD5q|+b#Iv`ixXw3Sct{^Xb#b3Lyn+3=2XIZ}Je&gVF
zQu)Ncl-Kgg0h>dMWxpw{zw&)+z_F=jF}l|Z&i7Yt*tc4q|DJN)mLp<G>>D=U;Cy3q
zt;du>t&Au8h+d-m6PrC|^E9p<yO6W7t76jv`G@|W3)!Ae3HN;!@chEt50xAEpM2*2
z{8XP?JKQE#>5RY=W9A2WXVW*i-nqDVh96&Ff%w$-e^0G_%ImywxpEKp{gTS#j30EG
zPxa(m@!e_WD`#}5|L&XqO-k?gQU;mK$EQ?Sy=3+FpZFhm<>Bl_>^%?erScjVG36bQ
z{?VKuz_-ZJ#)B<5VJ>UIT(1Kr2Eu2UmoAXeVbwkuwLvh4(fUB%26>Z)v;w9%jN*>c
zJNVum+IopIt|9IN-xp;LC#MV!*^ACSg4{>=3Y}~O;#DL%U6LlIsED3)aOvQg*?GqO
z)I_fo-A$4!UF+QWl+`DE_OLFJuIwmLUO&-aqb(>f_lm$)m*WfMG8m*9k1d{jk;jBz
z*U9_B(HB`;*zUGoTdaPe`$g#%=Gw+@i44I<C1!A@^;ah7=ty}Vyky8}CaryV)dr~?
z{%b7rzl5$~etppHk;|Mu<HG74afd&!%;|pqaPvp58r^-&>yP_C4FA|(q5p5HgR+Fc
z8;&>@vqUDvW`o`UuN6u;$}&oG+TQ!AKa`#5ks`3R<K2h5-fC$==>pmvuRE(dk9WR3
za{P#$Q0Ef&P0BwfEb^2IvRgT2OWT(2lAe;*E1tg=ELxSc=Ft+NMUxgO1!;z@47$0p
zDCB9^so<$~o7iqPy>abR`+VZ^N$(TUCw-rse&SbT?>afj_2a@L!A6>&HCAg@Yb@9N
zu8|)S8tNNr8+v>-SLpLq>%#6Xuk!o4@Rj>2v8&TgE%IA*YDM?T+bfT++`ckD!(^7~
z+4<9)PA}J8cJ-3qCAkZBD=uDjTfOmWMON{ZRaakKUAJOyi2Ca273?eg1NDQ}2iY&E
zi`?tpye4tw!AlP^&Rkn@&0<Yq@M8bT>t3#qTsw1d<ief7pZ7d7)}1tMY1jra|J5$p
z-Pb(V9xv(1b4$Ot^UKdKYrkCn;+MN<+f=h?>)_on)!WNfm#y!c*4OQ~$Zw(F)^mb$
z`PhzaNZBBHvq!b(Z;y13zcP<IOV5M;2$Q7q9oowGUiV%3dj09^X)oVh-Mj9$Xa7p^
zHPe@F_gU}wexKdC{MGhx^|Su5{k{ClR(Y#?RsWMd9qFC*Ph^>|9Jr=%Eurn8ShF`<
zZJS%$!nUNtrfurH)jbk!|J-J}Y4`L=l}cakt?g|+rrpyoX`lYefO*EvGgr?9pNSJ@
zo@^jI^JLoOu*r47+Lu`_CtcpV?E6`bG`X~wY1(I^&QzW4J$roSd6V_V`;#5Feb^un
zv1XG+_Q!jdP8ppnYD{f@YOEcry)ATN=-jKe*|WCJFP&G~`_)u>QE$8Nw6j&S&7!Yd
zyBd}dom+jAZ)@>p;ah9BXUn@vJhk*PS(SFhw|Z4=Lv3U2+`oq}_FSB}xZB;lr_S<N
zjnAnWsmbU1e!hC}YQozMS0AohJS}-g^2X%JkF(XI)z<}Yy=<C3tGMH{%yYNr;d*6y
zHPd<Zw#U3&_c%s;$F?1Hg~jEQcf2ipU1nbHUmE|s?y~c3<ztHvJ)Wa}(D*`eU;5_f
z?)v8IYu5jbtFOLYonHC=OP$%ZES>r{*EaUo`qWz3&91eonYnM}zS%X_e|-K})S1@x
z|2g}o^55-0^Vx2&q_Dl=s*rpkpuuS)@?+l9qqB}$$+fVyvQ1^(%9hQl&a$4jP9RBY
zl3<YdCFxgER?1&0BouX&aw2c|Zd06ndR^-Gw%zULx$H$9dpo)#q(5q0G7A#AX~HKf
zZkP9SM|GfB;Gz|683Jef;^b=Wk7=ptKkNT^<fGx^)yK<^S{*yrV=L|6KU-?M@w?cX
zgg=vh?B=}2xv6EE%hw~Gom)HKPi~r|HQ7q7XQn~+p`$5(Qxc1kxinP`Q=`4^A9>KJ
ztzPC=G&jWR<*}62CtD|<_rExQlI_zIAzxR{S;-#6xzcl`b+CQ$B1_Xb`%KDi?b;Hj
z*S)*zchmFp(Fb2Y`~5-tr~l>JOWK#`r~XLMIsZcJ2j?mlTc>FSYd?ghMLaup=6mGD
z$lIIQHcj62{n?5qOP<MyWp-|ISzgpz#9e&*XxXEnLic&$_fGAJ^K40(Ei%z>n%T4L
z!~Ji(jk0g*Z)#`V$9k1jewnJDb-;>94Xr&Ucgub)dg<To)KxdT@%)2x88b7Ditn9i
z-^`SjJFoYQ-|V|a$8-En&N;VpVR8AH#b-{>`kubgxiIzP?Vp~XXM6E@KbvOrPxZ_6
zT|rxeJYSw=y~k~)cSM{^e4AcT+?KS9*Nx(CwidO2TJ`C?=$$U^{Z{{W{kq<G^ksMS
z^c}i4H%!?PvgOM$qy4GYTBh3FTJrkx+xOJ57Q5c;+OEBPolX4Hpsj_XsZ(=8kH0!`
zwKx0Bb)DGHm1`5;etmV|)!x@U*L0)4*V*R(oyz-9`dg04orrxOFRMLV7Cqb9{QC8X
zYazLcd73vaZ8a@?ee=Fe-Rir`_OGqEJ^k&Xx3bnr<#Se*u3vjMb3?JGY?S)y*Y`{_
z!*9P^aJRSm&G(#pr*A~><9{>xZS}wESMSZgxxdM`d{CHh-m$y+HQz=4saKW1UVdFI
z&B80&d$RX!`(gg=CY9#0zmmVm-=6I|Yjyt%naH9kMO&V{IIwY1a_#Y>$E*EP=2YA<
z+WPd}Nlx|kbEWE6eJwfivN+x8+#DO}f^Ro0=NF%SzW1E`z6XU1A1=Cm^q2R#>3n}*
z{(8PJZ2Prer{7JNzHM`R)~(~)<o7-*e|qHVg4OBkHm;A|BUifjuiK&CXmj<uZ+0Br
z{q;@Rx3}TtT<^ZzpIhBuu>Q@yV}Dn%v+-yBOZ~iluWX*IyBwSCo-Z2jF5W$U=eYhn
zkNGw4bY3nh4Yf6`i~4=$Wn^)=NBQl~o#(CNJL125pYm_%&+EIUU(@IR_xAtyrS~}Z
z&Rwsvo-HnX&&BG^f7hPAJ#F8NeeCax?tR^_|9ipr#N+P${pt3qwN<z5-x|OC{7T;}
ze%3y>n!V-|j13f?d02G$GxC_3*vd9szs<Nn^>BW>8H0xK)O6!kg;SrGG##i<div8>
zF^?hVJG(-n9>c^B(y7cjViQ(=(EHEQ|DAos=FbJvBAL53Prf8BIxm7lo{jCE1d~G+
z(}V7mga!3~4;+}j*5Jl%2AT80^EjU>**E7&DKFo4Zu$fFpXM3&3bkHv1?^^FU{FZ*
z2=ZlMs8VBKXlP+z_{G4$(C~tRq11qZ;Z*_ygVhWM2JwP9y8>+(7#P@+yxm<G{)6Cx
zy~`&vFfecyctjR6Fo>ywFr#OX$_EAp2KEw9Usv`=JR;0SIsvx3wHX*17(87ZLn>~)
z*;_tC<oMPdf6Kl<vDs`naoU{49ZXXkgcP*hwbf5g(mj21-@b?6Z(iT`ZENk^W~Nh{
zR090m8dO=7I#ebWx1B94da>?(|4qrVo3RlpEZrgRKkJ++2@egQ_0Fo+Z|>qZHbP=G
zX}jWDELg*TzuEdiWBDaVMfLDU&u^HVeBSzLrGMGp8)5M->6v}umzJ+TmY(a@KC?7d
z{l=9O9iN=mh+R%!Gsi_~qKAr5r^}_k0ru5`cb#qC$ZehAu}g9O%j1gmkwTqd`APcn
zYKJe_Ckw_#dzAb8e(L11<n;ad{JPD_=e<Rj)ctm4EG(&ex5Vp}#^e_^KUXe%yOr13
zUEu<+taJF2JJZ4DD^09<Xb^h++I)vor{{$9Syuhh*><$BUt{jFUpm)q%r0NdQLnT;
zd9$;KOU3tT3;V7K%NGZ`+jH85Tkbz>)G9DT#IZQoIB;^;51ac<o*)N<+?~#FxsmN{
zi_(40=?5QGw7WikzH7t2cOf3<{*~7p44rd@ap~n2_4Nhsl4|XE(`Uzgs^rK~-xV=C
zyN+w>>Y_i*Ug2Gq6IoW5sLsDs7_@Zr%+)19e(Bd=X4%X5F4tt+dslN$9Dms6muqJC
zLBm1lWT$~@?_>Qb{yGYBf0cL|6D4%rFCQ0W4GGO&yioYK?w^Y5d9O=WzpY%#{nNkZ
zY>8#=*OK`cwSI3-`8kKn^6Z>I$DPJ=U#AOC>8}%(J@;0a`;wP9>(g!QQPZ`pbk`{#
zFOi)xd+nRTUyFR3FQr|ZIX!sQvV~bvaNEUuAMaZ8c2mQO1_K^GGjri%2c|?W=k8Dm
z-XS=p$|hpXH=j?7e_CmNcA53W$MX1O>DzPdH&)F&zs}RHbKjI2zLKrW`?I&)`D$}Y
zQ|GnulF(ZmTht@v6<^F+SsCSGxAwg)EL<lUA2@lm%(8F6%H@-eoYC6*Yf6Ch)xd@+
ziX5(w-mF~d%6Tn#8Owz;-`>pHlkK$W#gkj=Czo^lzM{JGSIX%LCCYumYgRf(eh%sG
zkbVFE{f}EOn`3XWeXQg-J^xv312{nmb-ENyOL~>#t7m0=`OS+4kJ+)&K54T`w6mor
zM{W-3&Qg16^svZHqnxWO-2H-1pWoGkvYaQo7A;tnx!lo-L&Wu<0gp(iE9)1r;`e@*
zqE4TdKPVQQ<iEttzE`8GYtaPL?e=-@CDn?rgTpo%s}_fM<tks1oBY}0(5H>^*51of
zzAJQZx&G5f&ed|)=KWih*YUMJ<xA=*|GjYvG_iWvNvmb-{ude$aOK*p)zkdfeox%S
zv1YC3`op0CJ{r2*g{&T2jzNswu?yY#txF#szQX1lk?VJ>`cmG`DXSU;ZcYd?b?4jg
zZt>O~pQ<nH-!tK@HNRoem4a`1G0v`&&+WXtJYn;!OH98FzXsiUllqTG=$U8QTeVGZ
zp0_R1kh!yU>ZxTc%I<tem&|jhvg-S^?e-<Fyf-yAQy<r4U;XBP`PunTO^zIkE*fl%
z5bD0$csT9sJg;L)EsG6JHuFwDd@s!G+&>OicXt0Pl8BVkl<c!!G%G9n=GN?RZQX~4
zP0LqZ+MXX@R9w8-DKpGERpUil)4J^(5==R2qLWXaVV}ww7HXoLr1kTp0&|U4+3dx+
zH@lo{gi>xka-G5>dHlgAqemAN@@<W$>;9f?@xN>J3fnESTlIAp`B&zb<^P+Z+aA<2
z-)LgjT<IHyD<)<>_dlF=ZPTphJ(Cv;XWELodM;;gG1%l(^=(H+>7J=>et}aK|D7;V
znP>6T`AZf*O?zG7Q+L?l(($Oi@-Iq@YHKgwx^pe6Y(Z#jUtRufvx|DZugwa4WT%7$
zg{Z|o`_Z>mJ7WL-nU2ov$=hb#no{Yc`}w)AwBqOBwl7xm+{FFPo%-35Ic4(s>qYAp
zt8`6FIotkzQ_<hNDjvDxsp9^J^MVd|Y$|){G;RHT$Go2&>*m$k`E}p=mwIXW6}k3D
z0zKW6r@d!=uuW^n8ugihpi&1Es6S;av^2J_*>*O~*t-1Pl0}a^q!l?22sABPq{PN2
zbtN&ejUl!@MN2m)Q!6GUM`>C1o@xi>RniHclQx?N3uiAlljIzDbulYP^Qz|Rh%cvB
z^K{P8<l2$3%I?6sx$KfQsbY_h_bn)UU)7=*V9Fh=x3^2MP}bP<ntt@5Prg&XXz4F6
zKQDAfZR)RAT3OkzH{6I7nI^K_`DTeygV~jkz~Jcqqt6yBSiv#-Y?A;R@7AkVu3lR_
zGgm}J^znx1&bHZ{ZHp3?WMpwYJgt2oGkB$4P~*Z#4K2OAllq&UM!nVJ^6uI&%lGA<
zgB*uE1i4%_UtJUy)?B5n)p<ndN71JfXSqw1ax^5|Ce2cI&){<V@S|8m<Bam9D;^p`
z$`huydj@B&S*@sQ!em)@=xpzmkg&-@T$Qb^6?^r*q%^htViO5nxN;xk?%Upe2fTAr
zRWDX_?Rk`BlgEDW7@N?p#)%ylugDyFdRo!me)-Ww;aj>4bTd}0%)5TgSSoq@)1_{&
z_q~pZuxs4BCg<EXp%iJ>H_w-v+O4}VYvt5SuS#}p`<T)4EWx;Xd;Y@{Hzt13<r2tR
zlXm)=tO?ufjjfC3e5gI<X~kLn@v%sK{5+@N-T$JT7)zMn%}sgK_fh@1nIk-MZ49=?
z{ozvlP_?&5)_Phi_kGowK54>S2hz-p%icuP#O1RyO*mknuvA-R(#ucBo~-`zS?f)9
zTTuI{J~jcTg-Z;khm?vq`!8CltEJ+qD9QWrVUzia)%i(xo33w^GF&t@dgZ#6>U*Bd
zpP{Rz{QGD0w#ZdEe={_t`&(w3tm1ziwqE^)#N(;z^PaW+ey>_op{;NGS!rWS<Gf3f
zHl=Fev9{8?m*(CHcWu<op7!a~=Fc6=4tS`&Fw9Xhzf&gbzHM#dz03L&gwldOYS^~>
zKUux~{)u`0_D_B;k3X1GKB4RQ1(~=%a@xFGzq%*|w#Ao7UXQb#9s7Fv$5Wd>-@5Yf
z)$4tS-hGw-^k=eu<ISeq)6a*$+tJy7zjmsx^!G%gY>BR^>+h7QC%f_HTtD9?m;O?E
z{e~a&-!IP8vJ%tV@ou$<Mbeq-Pp;>7fAnMf{^`S!_y2v~cN)*MS;nt#^M}LqYof@)
z$ES23|N8yzXx^32-{b$z+p%!rD@~q-fiYnYxuvET6FLs<TfF1xzvzf>-@MnveB=!4
z=Cqxt&^gs|`@MrL#clF;l~*6QHq9>Q?lo=e7w<2hOTQC-JIW+y_4IcRyZ=<qj_6Ky
z_Ubgtdb{mVkyL14C)?{AyOQ;5yZ*@B**bSs8^^3!+dM21Mb`5D-uPLN_ZX}DEEYTV
ziZw5}IST(=7e2USf#IZo|4(-*Doio(c*(Tf^=Q_wDZ<}GE{a_Ce9>f+{=L3%;cKgh
z1ro8>3wOrMIQuO4zPzumiV)|gPoHkhUA=0Rl34e}&FTJGSy>w*bS&ojmA$>?8W}0+
zw|ukjuLJ`RS69~BUtbQUy!!X|c4&*hk)(}lOeY>q`WbWVQOQIfHBS{Ghv?O-H}8#&
zb>whVns^{-<CfqVTaFp<@X6cxC{0`tzCJD^J#xo^<mYQ{<{V4fIL9E~eShWp>$lZ<
z*S!u24V^f7a`V+&_d>KLdI)f}8qD<BarfQhZ^hvmvCH+D)3e?@`e?B-go|PF$tkw=
zi$4C_bm{J~mla1I7ruJ+N<%{<qgdZ~YpLzrI*BRMr*BU8w~XGNmz%ul_HuvuQ>RZ`
z-<i*oxpu;$X5YYq84I?1C)Nv_s4tlAy;O_kqKL){6<*PcQFB#U(q>Q2+`K;J^Rm4>
zyMIWfot@+3{4}8~@9yyn-Ro1&9SyeF^LO!=uOc2UhgoLNdTCiABdnI6_OgvDZ^!ez
zKV}t~|GX`{+lpTEZoauUPjda<KW5DCeCc;C?7R6|tIKJ}&OX-9&yr8RKCQbw&iU-T
z-aS8GIhD!J=@RlcEIH*m$2il5-AwQK*2?acGm1H@j9hnEo>C3J>s8q~S-t*mw$Q$f
zcfKC3j{N+sdrjQdwo{&qE51LSUO0{C>9lE9d;Wd8%_C>ar8`wgsLSZ@uh%ndDtROX
z*UWR;S9Ja8lxLPRzklEV;pD5*9tq0{*ZO5$Ls#C~-F!5E-{}wC>mS|mRQ~Z`YW;>E
zORw+v-p+535un-qZ10I{PS$}B(;hwfsPg)I(5-Tf={~cVE=tY&cUn2;=C1yWf9IH7
zGdC)@wKT>mnQ6W4FYz+@6_5M!YbRfL=zFIAneVso5aIr|%oUtPZ?Et7`)^md=hut-
z72ltlOPK74Kh4eE>ak&7d3@IWYmLh7#@j;c%lZ8dNk^$KJDa=pbnM57yI-?wSCz>A
z{=sx0r|?19%p03?rLQ`ly0I&lJuF`Dor|#Sp4V1|6)*K7I3y16F7~}+am&To<-xYC
zGeXKITwQYT{`BQPPPeb*JMu@L{cK_LbTiY>o~}JDdM8RVVhoQQUXgTLa`k2QYUMY%
z9-n$U8`rt=Sv#AUm>A^Wv)Pt&(@AUUBKLl|nDV=&%WF37D0vz5?d|R3f4|?qpXazc
z>#CNPw)W1?=d5R%=gXz0ruxjcoBMoz{k++EyE;F8Dl*Hz=i}+g+33*l_xJbDuU4=B
zbT+?Exc_+ZvfUr1T)c4MKz{x2=#p6X(9kn!#=*<|Qg3a^OrOiZ`r#nE{Dy*uhrU=T
z${c%HvPgqVJif*-c)6eL<*UDAR<2+F{N?ib&%WKx&);s5d-m1U)s<f^y3e$$wMtD*
zO}}d)Z=G)Hn|y0arq6u4zdLd*znRBJMV(5W9(zsAS3~5HfBi4v@_Uu)PoF-`J@@wB
z-r3LRRre{o_azw3oMBO@6dqH^TDE)d<!rabfoA#l&TL3Lydi4s+kM;DUA}3?H^0r;
zu<T>SoSntbH>IDS_lA4(#l`N$Csn5l^t%20JpcchwcGD`{r~s(e989`>0O5{G`kL6
zSm?a-&8E`@7BX|=K0i3vd}globooCI*mK`iAK!EC+#W+y<E$$hpu(ZJ*qG@<g-wr~
ztyK5XODiqcu4@hs+_9s*>-Zu89}W}aCsCqFvmUJ4C&g>%80(|B>hh_a>0<r6wfwe=
zZ$8;_`0e4xg%f)!3{&^-w!Gw)<e9xjC(PP!IrG`1qthR&M{S>e*!|Uq)uDU-f4lAC
z+<Qd0aMfz5fVQn?<#X07+@yF+H9Xeiyj<!O5zzp<9GkHAX14Z&2Wzz}|9`!nk+tHO
zfzj`3R=;>bWj9{s2Mu@r#B}}UTavzh-j!+J)878_dX=qZQM5#EebCWKVWlfp<t>O{
zT)w}hSZ%Iz+`jo0CN7?tjMdlHr)1kW=EfSYo406g%t8&Wt?l20`&w4s+1`AV*L}tM
zY$d5R%9BK5Dz0vQ^XsPn%?;-sEPG$u|ENa(?6Rak`|hs!d40Fb;zXs7DsHEL%zVSg
z)+gy3xjyd6G?B|0;V<uQ=Re!WvdUFaqEC@Ugir3+jE{>yTIKrK{f~+JbNqM9;fZ3K
z(^{V$yHyf>HThOmPJKDI6(>vGo+qb1id<G#l$1AL`NAQ&W)G9g@8Gq2`?t!N+`iKz
zajV-jd;6xMr(WMeOfTmjsc71L?NY;w9dm?B+?U_I@oLWXWgANR&&k_)6yAUGcy>s+
znoGUp1>S>`v!!na&wp_3qFs@!6nA;+&$E}T{@rh_obr6h{XNHmn=(!vNDSMU;q`6C
zvSt6xS$61TNk1>1*y4DKVY;hhuxLxe4n2dF0=`^t(pV?267c3+<jK4{^60fiC;OJN
zJZ!nRR`%NRAkAmz=6Z*Qik9Ck757^{Sv<aG;{89*%>R5@Zh!Rk*GWEVob88a_$*tq
zVDW3Kl_6ZdmpfdPB6P%Zwk_6IwK^r-eYC?xNoBI<qKg`3yKR4m)%^PMa!HV8NT{j|
zpK$llg57s(-e$)u3vgVxapT11^LD3?ipQUsq8a?+`g-~D`?cnJ)3t5GoCG>uE|khH
zzdSS6hC`8~{qT|?%{zJJ+vESZDG8Rnxp9!0-=^XG^W@9Rd~JR{nask(=+-CmbLB0`
z{^Ohf&b`QyvSrWAfBnZFXUu9_mU-lHp^sXyS^vqDR~vfHSfn4@ugsYG;=;lwPoD+`
z2OpOI|HECb=7VEVQIUp@jthsQfdtEer$r05<tU!s;u<LOrp&t2g=zKGu0<LulRcdl
z3WTlxx}$sRI|mb~)VH^`-l=%p8~JwAs#PvZf;nc=VXH-BuSaf7V%=N){n@6|dZ)8)
zTEBk(&f~S^>+ipBZb)SAxBq96nwt9M{r&kFCS7OKPTw=1bW+7^_S(buYhLg3UK+F{
zNORWg*_)%*@@$OgIWAv6$3fx1_kG`OSFKuA@#o`lO&y&le?FhjzueB(u55i?TT83r
z$HVrYAN%Wl`sHkI)y}u8wK|z1<a>G3W6}QOhCIyEPhWL=`PFsC;`F39lTNDK-j<vE
z=ElYzN#nM|?fjn~b?YZ>jhbnaDReMlLWq{DiqJuWo;RPFFI^K~7{HNHecphlIl-V~
z)vDZMCk=YuSZvjMo>Qkcr?XJXw#tN^UoPd}pPw1Cj+tJMnOvJT!{_DOg&!+qW}j_a
z_d0Xtc2~DkJ&x_G!xdRtTU&V~t)_@-hb@>;myseT@b-}IR-OgxoGsWkTD0giEwgyD
zPFRCWWaok_%)4J5$a*wm@5DzLQ*NBuciksLTCs203!mAlmwqpg_BNii>fuTKB@5Q@
z9M)L4;k*Oe7u}-<eT9C@R%mbp?V6{_b@xK2yOYk3<Cou9$|RrJ(it&Fi~U=O?Ahx|
zwvvYwls!3>->!^ybaZtLHNEq*^~v@t3l}WqJ8R9C=Eacke%aMMpPqdFB5ghK(LCD~
z78`b7-lo;C>gEYeJw^Qoipy7YeA+1X>5p8H@4_!%Tm5uoToX2AJ&fM7If3=@GT-G2
z`UlqO-rn@~%sXQ<moR579%XHh^xfAjUft1|dPz{}@#Q}m@hfKCN{sG*`+oBz;aU0H
zy*|ub(xu0BTD{=I)m<Eo3zqoTe|vx9-0{`dAAJxt+>oz-?M=+xRrh2ow*Kq6G?B^V
z_=AAOKZ0h5%<rh}^}Ht2_v!J=ojfPJ^|kV)ULMU3?p%K9@3ysS=l1lTH95-jDOSq7
z??ccV!_S|dr5)mI?n>x7mC!ZwZ|oVl7jBHx-tc~Ycg}z_)<EO_sjr7^C79$Q?>1as
zRJ1_xWbEUbBSB5i%}p*#T>W?}Bv0yPs>3`}Vb?X$oAdA6ndRL%;a~Sj_}90$+z0NK
z-#;szzsFHWOnIWmhc}zg7d<$@XyTiEZjPnTT&t<IzrSrvILLG-&pdit&cURO1&4Xf
z4GJDOtXjRg^4-qox1I)uPQ7(2>e;!ur{CS(ee(SI=d0uYYPq_)u8H2>_v=^Hx;>w~
z6#hCHOZ9%b<gH(LGj;mLnx942*2Q|)*4n0Rp1En0(Y?y&vU&S{roAbv4lADNp(0_D
zA#gpeT9>cAx&H6#c%7&%9U)q-|Ns5n`LIoT)1lJxqj}qn3m!Op`uthaBtsy2TTbHb
zZMi)%mY<f~-kE#b?AiJG{>jP8cJuqQ*X?X$WM<n?@i8fArAYC!GY1WL;_7~;{`&Ir
z@P&oWl}{$R7g)`GbASJQ>-T#mhiDz;*5Bh08!KD4fBx%ryZacK*#d%t4|nVDJMr)D
z?~S+L#uOeE?UAwE<QvV!#igO4v0%|6rtP=SJ~-H{siC1D*8TC#=JTIE9+yv+V0(P5
z_wlOL>kg$Dfr3v=Kd$HY+q4G<8n0ZxUi^CPb|J3Tif=d5EsLITgsq->T7UnXjmgL7
zy?kh;w`$cQ4X)qc-abBSe*Z|?=8dJV!)om0zuzh5um1KX@#G}c89r+FYCij}+xIK$
z+nbw*_f&qq)f-v9?CTs|UES2Ludb?0_MA0)cIBIm$9H_a7QK0z0=Kx{lWn*2lK=kt
zdgbcXtv}B)=iJ#bQ9Q1KG4}e)+uQX&R_vL}>SZIhe9@vN&&g^Rm;1~A`ts5_F)^`B
z`q;O(w`bbd+xhzXK6(DU_%N^egYUm9KOPmoxvNyW-}akCczmrXx0p`C>uYOme!W<H
zr}Ft+8Jmg;w{Au0?f+90SNT-*&_e@JQPGgFFr(ySJZWcVv3_WswRm~;8--k}JInp$
z+4$woJUG~FS@3{CzW$G)u$qqp$H7ghrz<`l70=Acc`_?|UE-%FCyyi<K081E{=0dJ
z5^SHJpHDyB#;ehFXo_a=jxU$IJ(mXU`Soh`n)v;4u7MWc?-b9pELK~!YTnoLt(R^6
zoZEO3PfSn@2@Bg)`a0~<M+^OZKa>u)@g_exF|ox-abCqE&UL%r?K11@7w%-y-~Y#G
ze$6M&tk+GhtL<GwC9O&_z9kx#ytwd1dII0|J-SKL7#B*IxLuapApTx#&+gL+2lWnS
z-)4~y=g4w;Y2wcMbla@pvnj{2f>zwxJ^R<OWhJ#O6TfZa(Uq3#&C$?EToV4(Y<F);
zGB5jg*#i$h1&7Hfh;?16{}8~%%%}D3?ZJ$zH@8fA`t^Bm*n@|Qx8F;<{cY}=s6S%+
z|NpN$_-yWP&a37V;=|7E=zMITp0wmqXU@JarAIHmf3)I4q6AxByk6?jx<?B&nA!N3
zYCKp_bYE`MqY5|cwHhw7q%Q_b9ai^QvS&?0MRM}$3EOjD&%VBx-*0#4l!Zn)M+$P3
z+@3aS1${Jlf38~Y=>6#*pZ6tP^w!^y{$FsGl(gZ_VoB>%qq_D^4~<JkCv;0QLd(9z
zHlNrtMPS7$ucym+b?WOScW=*2IJrpG;?w8C-7>n5lOBA08Wbh-V#C_wAML-gKC#d*
z-gT$!;?{pBEi{)mD`e~LpZewf;REkZrY7u<dvQ`kbuW8>NI`1+s}hkXYfqP2Fmf%>
zs>$?SwfoS&HWkL*ZY#~#_Gb1;n;a`y_O0xab%L6RdBu;#Y`>mG{J3{TMka0QbGuBX
zh1_Q0M{fqsez0s}oSy81q<hEGvJPvOU01L8_E;^AzvSatPsXxq$12OyIqnLqv64`g
zG2c<z)34M&vzFJ>P5kzchhJi9%6g}pE!~=uet3ecO~r&cb7Z=@x;7nWYGl}cuS%Ph
zl~u;7q+`w;nSFoX<{Q`kDw$XP&a&yC!oq-#?f2`XXP-Ux_xpYS%uLPV=jT+#x(lD2
zm{?Znv`}Exs!g@O%h-4%4lwiEIG9L1Z55AWcyMB(@|&BRm;2tGbW&xuS+0^?zj5v@
z6E<F{BjNi#i5}~bbPf&{j^38zc>U`d&Sz^PH&;ApWWTYqSe;v3Z^_D)o$2!`+q(63
zb+F4-FocG#T)EQna%`PbN$m1<>-uipin_Ko+P$>YbZ^zyr1SG^WAoB;b9EmU%y5^h
zY>C@j_3?r;|HZ}b{Bo5~1d}#OwDZX({r~sZ;g{U@^z(A7R<HJ48nh;AYgd5Ak=*Tf
z+m1he_`d$Xb!uv=&6f+#BI4qopUut}>ONZWuvI)GFYnpy`~S=yAMbzOsXlMPzJ2?=
zb8|8>6r_5KzTYh`KCQcbL&im=vbVR6o||h8Dp87`oq2dIIzRPzpX{2LokEH%AP+5C
zq-4FGPtN9sANM8MK#|sG&(i*VU0;83b-4b!{r~Hl4l3mD{VJAxyzl9p;&X|YmUy0-
zXFEH?<kVq)`#X7OjH0%@^H^5<`T6<hC)MXa`M&?Z?cd+um&fm~JMy^DEboqojU4-e
zojYgd-rlzGb=B_sHOFlJUv!sWTL1swo%;WOXPiyrlefE*C-}Z>_e`@~sq;3UdG`N#
zsvo%_fl;o%ebXkRrh^LoHlH|Ze}8*weg7w4+3vaZe;&)vIGgtD%uMI?aeFuEe@#0(
z%XNL+-W?y0Nx!_gSsfIl@9*v9P`t4vQ`oxf&4Xjo`A;T!>m~mC^YhBJYnzgf_vxRJ
zu@z2ce_8FPHT6_h8Na+;&-K@5qheHqIN$Af%-6;%ea!g0jkAWxqKz9j-nC8s{_bwh
z-Ca|^->dGwoxguBs8jvwRo0`A7T(iz5`TVrdZ+YyEU06?=kK@MId^w`T~h;U?wxWG
z&dSJGux?$StaaIj)YD>B-`>pZyRkW$y|lE{;aB9|%Fk&^6Fs)cgoalB|NA|1XOZgm
zyJfcndy96z?Q(kXvBDz#PX&Wbo_}}24YknO=Xde7t=q6aaKp0&j=^tMEC`?6;IDN_
zGggiDXvWn4EB18WeVTY<$3&gzPjc^{{+wY|YVxm%=}noo6NgWj^}PoE(BKWb?^Nt%
zVs%`9(D`KKrb*`WDyK>B|5>WH^C4SY)vHRm-&g-DuzZoU?X=lHyTayPn8Eg)JD&WS
zUGef|dgZU}`#s-Zlq=VBX8(44hE*?XzV4%m|8F*3eW|j10mI~Fi5smyJo^=BD(Wh_
zgSqaFo;63~(P?(iKRq}4yi2;`=ga3af3N<3Fk_{2;(^l#!+y3V2ACDF?ESU=+1H)*
zN7KraidYMF&QY1{*^?Zn@<e31@#j;PIyIkT|Lx1-+?AB{XzBFez3JS)`OFMIZ>!eO
z-xB!!xBbSdqgHLbvggm`p8xzNzrOe8Lxq5lhO>DGQ#L&~&zZ1&+1V-?KYqJd*H96Y
z$*l1T?7K~K)ABz@L~j02I&Ht4YvGTbe~z90XpwtkZk~-?K?PGx&E2ni{(Rxiy*=yk
z+n9S+^Ax`A)Vt)L)L-OYYA0H1ooE?*QRnbS)4N<|yScaDb})^UFsb25yz1T0=)7^J
zsf?bv=5*$*m*1VpSa+VO^QDejCWp_erd3O1tM=|RJ<-)1m~~iFPx?t(%*wfUOF7$~
zL`Uwk@5(zc?UlvB9eu9B5}`SZ{SP$VU-$9aHi2Hl=Ch%NPMpomy`HO_tlAoULkrcu
zmps_G>WtizW%YT#%I){H&-I;J7$(~z)FW+j{IcJkb8Ug=XGpGEb?f0zMrO7I3AWeQ
z*DqhSs_WOUUu%AbWoBxM#}qI&Iy9Wt+ubAo|Htu=&``sI2M#5%`P1Dw4sJ+1Jj15a
zNUXc{wBBwVZI%_QZY9bZNm;JH-hKI{in+OY_M^EcHy1v3b6P0C&M&uQ)he#T2B16!
zGIi<Fr8OT8vLAU|n09W?&1bC+<u0M2uU@_Sarb>4`+-Jg_8p(kSxeg2*{t99D{K1g
z$e^G@-`?KtbWw__dbyN^i81Z$tfTGnbtfh&yWjd+o~_$({Bh#9H#fh$y*)km_BKJT
z))}_dW|o$gcdA~mbx{J9oFKo&@2^`EvvX2{fk$epYCFHYngGX~4Re3~{5hxk-Oe}n
z_Re0mOl{Ihm9JmFhJ=P*++Ci(%x7lPvdl9x44Y4#@@f&7GIi?4l#@dLex9#4N;<-!
z<};%p%(6&ZR~J;)9en-ODDRHNw|95Dk0u>_Tb8{2cH!>3Ik&g1y=?gUUA38U7vJ*B
zCmWgBZ){9vZ*6UT`z-QV_3QI9--8??*KeGCO{b`+Xoh96+WWoV<94s@TD7VO)RHfG
z5fHXE>geqJeJA%+ey(`2uzf>>4#=^J&TSi#j&kuFPB=YH*F|aKn!}6s?6FxBrd_w+
z{<m%M`jdb7bi}w9yY((Ae}C`C-@1EMuXSh5nssAK=4J7`zIo?128*=1C>{Ls^YhOa
zi~CosT<LhdG;jA?vlf9HZ_AeX%yjCPvz=j>%yxR3uCh>PNyYgUYu7IIpKlkTBPOO3
zabSjF@{T{BPH(*Zc1`4Fwd(#WrE?>6jy*d&`{niZ_anc@SO`C6$voYgKEHMvE4SE&
z>hF2W{O8ZR>X%-pFWI-ZZ{6!PvAe|%8{F7cs=e%So~cOFzNb$e+zV@6clWn3n(<~i
zGx;{kXy=N3?0xtpZkMB_z;)|AN~=zu>dDSay>w=^!EKovvSxYzuaz4_zFiVzrCRG=
z@VuG1V5e-4<$IyFr95k-M83)$`uqQ1yvk%Yz4hY74=vvP-d;3oR?`e!Gp3z8-~6gD
z)ec|R_@+*YIp0xUZ)UAv{O^zc8Cg6@M$sRiK1(p2J;UI!5nJC}AEkwlzf>GB*swdM
zV~*w>;{qAI?aZ!?tM`BNH7UNeZ}0sV#XHkKRK(<|?>v0$<JNlZ<vTSyP9`0mK3Q*j
z%@vV%aus}f+l3DsaEN?+vaNlee{r(0+GgI_cfE9k9^HLDf2o~>-7a%SL&LILJZiBu
zlFaV@Dnh-9#k)ST|4~_-`0oG7f;)Els}>bp=dW||TCwAf-n-W~=X+Rmoqc*}iRKI)
zGaa?5>udkB-<YFU;{5Q>>GjUy30DsUU6}lP-~ZY-de*zAFFWzpddBT{J2)qHB$roz
z{JlqF*SQy;(+z#hH_Qw<)js#|^?z&gcW;|_@k79Izw2d%Q;pSkX)w-LIJvTS^&Qj1
z$dZU@9_Oc?G?uiBdr)%Apt8E)-%ncx1D?5(;deVOoVjuK)yH`0Ez<kXe^}&rsZXeX
z<}Ht|wiS*W-Jfncn3wjY{Nr!_*B-B$mMCt>;7ap#6mJq*-`Q9h_eM=LRPnWJbm)O4
z%jO!Yl{Vg8vS=mOE=7ywSN!<rONs=Ds_s5+IZfzKrK|15B?&1LMZ1^H3BC1y(ITZ+
zuU=(jWu1Cw=({xN&E4J87cEk{SMiv4ntuGbA0HnZ&OVzf*ORu{a`*PrDMkxdWnH`T
z_AtNw8-xC%Nhj?Bbj7&a5=$)OYSO<SOfl-bo97;Sb&Y4-vEXmc{I*9nrJj!1UuV01
z|39nv|G%zFnytBQS$iT$g3Z#>vgUR4{X<z>Kb?;MC$#?h?AVyHt#j}HIA+dpi{pr-
zy1`5jnVJ7YoED^S^W~gasx)mT_sp7^g=H2pEX|HB0v2=6z4>o`w?vqgl{Gf&IcW6z
zQK!0uZIwyfdUI_b4#is>+B0{(-gR#0^LgFJA18i&b=BtIkH?^4tm^lBw|n1<-_0Dh
z+Vu9kb1ln#>gBD|?nY|Q4CII{K5P2q`E%wE%kBSmvT}<RoYvjG#dG!RUxn9=CHJ$h
zeLnkUPTq9&nLd}UTsd-LqOzoImC3_`8S#G}iSPM*&N`;(q-unYn2(xpTVg;^(4huK
z<_)*sx+qQ5+wL8D_07Km*T79rYbQQ?F;l{t!S}L;zJB_R4T%;qe80ZFZomJ2`O1}?
zzLz)s&D$7p!GzZ}Fk*8WZ}IbUxy9#C^t}JR|9@@Rnuvqe@An)Am0#uif5-A1KJiX4
zT%^@ofA5rq&h1HCqt5SoerTqY`s%Bve9iA3v3x$~@csY)%5%)5^`>jTe!S?bvC34h
zt(Di(=V{L@bv?fRQFLU&GGFVdZyL|+mo;@YU$EUj(M`UWyDB^8LiE$7xS967+|}7<
zUM#+JdwSe~(zcsf>Jz57pML8S8hR%#zWZ75f|)uM(JhYMcW0KZ_OCM9xwnpKr{xSE
zDJPCQ_ZXJiRtE+|G~8SwsG*beVum4)tnI9Yi`|WLzDTg~yy`x=<k6=enlEooZd$~o
zqqbHxY;)(FKTchzygpRisi=wbnbmbO$@{uvXw<sbhaYZOW?LTp_Ga-+t0`Nq9J~~y
zzgczivyAW7F>+kqr%#+cfAneD<FCJ8Sm#JMAMZ&#@!}$nB-pY^^Pe+^F#Tkm^6iyt
zUY^3<gtA>9SpTfhZGQXZ$A>E06<tl{6PyG4jsDch=k<&4zS^{E=7(Q@BhD<F_Imf`
z+R86;B)99gJLJjzyJO>3{Y~Y;_ep|N*7Pm@dp|OIx9IW>R&(<O!){qk`tS4Al2iCa
z*-Mc%W$bd>x!0<0@qFeV7&0f1U$6b|iiG=5_q|L@u9_AsXAu0v?DHLU-z#mUAqnNN
zP0D9A-9&F|l*~L)I_t&T?FQ)!W`Am6-MlQ}(v&xRTps;hj=x+}{@ZYyZ9eouNP0R?
zfnY&Kgiq%2*fxu*_k5~6+ro~|ym|lV9LKypW>cq3tgM(*A%1Y?vORm|?Ac=zSMiYb
z)$7-pd3n#~mfw45zW*mLFE4M-y*)D(o!biDZoPhKv3viUrr1D{Z}0A&-c|aVqy6xj
zn4Lz_RY{kZ`ObZIb4Q`_?d|#M5<H&{vdag=#Ppngdg|Xgqb+p|%#ja2oS3K_dC+R|
z^7{Y(ZseG`^~;?-Djx5nAaEdS>!mw)&P-DEmawn00gcS7P41MnE}QdG&t7rWs!5Y4
zJI}YPonf5Lw|?I*t=`^V9tnd5r{CV$mMg8Qsw&ZT@YdGsP8TImrRK3;=3~`f8H<93
zTeqV4WGoKMv#q{mao{A#EdovbcE36bEY4KF-+SEnyiN1bq=otS_wBfwr?>A%(y<=N
z$)Yo?7k~YccY7P}^wUSbUXMTj;^N}V)8g9>URdaSb9cG^>aewsK0G|E6SYMn``q@d
zt6GO18f?E)#BH2@F6D3=ugl_##<nrXmwvT*mr*DGd+&-BD-IiK>*&~A_O;Yo8Pav^
zn46Tev}Mf?19pD7Bac5OY>m2T(z`YqG~RV%M`7}{otHNLO^{%7b#+yl><JnLw2(>r
z`s%9etX4A<lMk2Y|C_>Z|7SvghKr7v_xj?yd0pMzCr_VVT>k#vnOUaV(oTAxjr7I3
zQ+;#$|K6)wV<ly*cm&kkc(+m5X+fscVz=I;UteC%DL!ZU<L|$++WS^}7iq}eGZt}8
z1ofJ3Z|k)ze)i$&`nsvj?EIH5T$r#o-fzC$+?SV^CvS`>u#hn^Gn+T{e5<jcU)Q1s
zpMRb>fByNa)$7yl?kdf>w?}gI)vmK?$-lq7ZE;fEmV0~JUtZ3M_pGvwX1=(;-@a`3
z+?B!0%O0G0ZPBe<=+ks<+Wf9X0{%;GR2j5*RsBy+Kec?9TdPZq|8cc9FQgW&RnRly
z+Z-j7V!b|g7hm3kwSA&fzAf3XCVcm=lzUrJzdcQpR-M8(XZqItVYgp*nQ{AEeApGd
z#d^xeWpj5<JDHRGV#UU{Q|9N^b>!+_$&Byt3)`z+_hY5c{9;p0<~6e}#Z_OrXuA9N
zt6g(<e!4i_!9B2V_TI-kPUU{)+$9nIGh}Asc9q<>UoBos@ITFnSbnDR;=$J4*H|Y7
zcidf5bl$Q~?%#^W{}Ca*ch?AYN!O%qn0=7d%!GBF^t;4+%hK-M5L1~Y6k~X!*ORqB
z^l)C@#d{iDb-UY^Y4M!iYOt|=?*00Repm0=rS~n*^)GWcurM*J(*F78l`$8tpS=89
zy0X4sax&LXJz=f~8&*`T$=W`Tjjz3V_xpXbL4y+i|D6ASrfU73hi%dt+S-r*d_MpD
z=kxi`KRi7A<8R%mQ>Ow#MHN|g?%cU1Vq+64x7dX{ch2yd-*L#x%UfRgrp)^9udk2a
zZohwS7T^8i^R_Eibsc_q;B6W6%fIb>?aZ!$AAbLh<UKsYFuCIG*6W+%CL4FJUg+Fz
zlz52c--mYll*19A;me4%VW7M}N!5G9&73FCo*89cT4H?n>2cS~CQOYE-&j9v$-Eq~
zuf|e4eBF~p-FmNztkwPIbX<S^6x6j}Iz2AwU=yo~5NF<=k8U<{{FBvulWuNG{quYO
zf9t)~-<K^|ps;!^9}81tZ@PlO1JGFc=Vxb=4>qwLNitlw_ghqqp7`~cVo)vjbpOBC
z`#ang_sCjryL{t><U{+*>rPHq-<)^Xs_xt7`4i{Q_fJez6zgtnWM(gVIyL;$r_=h6
zKUSPLfByWj<DsEbLG#z!@7K*PusD-CJ$Bk$<GT~DC1-Ao`0%sl=FyDTKi?JY+;p3>
zsHn(FX7joko9Am9CI8!f|1V$nQ75CBOO`BYu`GV};A6#(4~MvCT9@l}i|ZeI{yFt*
zO#k1rx8Iiisj~--(SwF!FD`Ng1yja_1!Yn{?%avd+wp+u*}1vh#~&+7^-h{T{rH=k
zn?L^k8&iDN^vryF``1%Gd^)Y~9vZqQPQRI*|JlQK`DfDif3PiGy7cCIe!CwEdegP7
z@9&Ar50k$3EAGnn{QHm*lI5$k?B=g`&VM(vcxF}VJNK?d2AP*suC0l@{A}Jt!5elz
z9x&gl|6d!lGUQs0wY~lP+TY(kUY`F?M3IG+mDOjK31|1wqHDI|yAE6MN*FYJe0==#
zzu)hTQ%(qg-23m>_5GWwzk|w~mseMZhn;@^{db^9>rCVHP5JljK;gacOl--9y-#kR
zY&_?p(7MGkJ-1f*>)D552e+yE-~L*8T6}85b<cJ8+ZSaW-|6NqwCd8!o#&MnZGGMH
zhVf~OrC-dm<*PilFIa4DJ^#O}zjNZv!;gZWzSfxXabx<FGc(t{`D~jgwAB0MrgM+{
zOFH?^Uf3O`KUwL;>&uqu0i0pQx23PVxA*w8`l#l=w@SuLKW^^!tgBeq$-aD+^<EdX
z^Za@SpU<wmTkn76xuZ#D!46ATmBfH2h7(^K$kx78Twu5AvcT1kH%*J)NOesr`;&T6
zBu9<a*`YY*cxIsCYA#LZ0O4z$D!iZOs@(|_b#@5$Ii4G8xKekXY^h_aLF+OtmN#0S
z%y+`3Mi|LPoyrZhcV6wsFWIj3v0~1nM~~h-I%2?+m6g?TH0jB+XOn!?KK`s(6Tg2R
zXkhYv{r}%b9v8~+v0GYN&VBZ=!sf{1!jjnI20X7`zyA39?@f#DnE?uV)4LN4I!>mX
z%JPoY6IYwunPTJ_8!OA#&fI?3QA6a=<BtngY3*7U92+|~)*xMs{bR+PRjag0VwW#i
zu;6xk_Qle<%a*Ct*u{sf$jQrNyHwroziqA)hm@35Mpo9L*I(1b4Q3V@&s$?GW!`<%
zskpd!_5{tDMpDw!AFKA>+{q}w!?tqe%7b~^Cr_V#eWhpNve>xw;=bO|Ua4D@XI3U}
zjp{s_v?gxvtWTecY~=V)PuEWeW#{Mh|J2KN&;9kQ3gnXtn<vknmF?QT&-y}@PHC*V
zQ0Ie>6`SPl8ZXRrS{PtfbGGx?8tX-`t-gN!8acUc^UXO|OB|zw!$Lz_1)Ao}krC>2
z3EN@&%{+hmZ9%7i=xA>fsk8U&=lG@X-kB=FcJ=Djkg%|%?YA|xwX==izkdCC^Sr~+
z=Wk}1csV;WH##uPK6~w1^|2U}VD-&6d(NgEHt2b?S3{_?X5q6JFXgNiS<aq4ds86u
zcwgr0>N>S60mm+eKeL;%Gd)5_Y)zQ<!hnufud;N+xPSiq85#UNLdUJN)O5nP?g^EO
zrDk@Z+~T@8FlnQN-F*IrTeoh#x%dCfdpC}Y&u-4ycq8DMo|Qr$*Zku}YxsWtkXbGI
z_KIWK>8yQocm1)rZ};?}-etD{yM+^K{%F@N+f`!h__L?LYToKtQ6GC}+1qcwzwz2V
zMQ_fZDNi-#%Dvd7pzpzb>aU-I#{VBm#&XiJU5j;CQ%}B3d3|l|o{z_*K?AhMO}9_=
z@M{auI5Pj=7xh=KUl;C-2?+};x?6hPF!$CL<5<&9mI=inN<U3ztXXy~zo)OXXv)>j
zxgv{Bv-eMb!^3eZ_e|jKS4BUrTyy3+HAl9o$a~Vcx6($>?OHd~-1!o#<M7gnZ|P<I
z<G$-Pk8wWzxok$lwFF^j?$A4Ct)~)>M@#rfwLF@?C|+ffzvuqCzdPRV`@QS$?23~&
zTTefI1S-LHm%mpN>I?`Ab1N+TICI@Jmr1VY0z02PG30&!)bes^hIs3;(wzHxsdl|z
zr_`MMafK=7x{=m=v)9ixBj!15GAo%Iv18|D`4+A}$+2DA&0c337Zlboe@}>6P?Y0R
z+WF-3!4#v?*L%w+Z5CgC`Q(R(htHRos!Z~mSRA5cY3UR8dFK7*1?3WH&I@L45qp2}
z_{o}C*RK_6{#SimEHY(kvR%2<idQ}v(bjhtt(_3W^3ku3GjFxi#&E~rg{3RLoD<Nk
zlzH~$KWG6oXxE5m`GcN34v{5WgHL7%tL>FbOnLNHQ)1QRj=W=!3X(V9wAO!May|T$
z>f_+*CcBNNUti<f;TkSH?_8O-tgKy~&dO}}SvyiDlwArv%W($TOwLm?mzHTQO<vmR
zlq$MvwMuQk#S5nTTNSxW&lVaKZK(D8eR9bM?UzRCFBNvzvMu#qX*G9ptjI0@nLBPy
z{SbEi!KNl}u02o#CvBc{SwUlac>C74_tJ+Llv}S}<}SY2lf&fPzi^jw#O%c_!k&vJ
z8kp_goV#Rha!6sO{;jvkBIRO_R`lLkxJoy1U7snVy%?*j?W^+<633pa<!E<xEh<^+
zx123B)RCj7FXwLbqc@$?W?YtK{c&zt@RZA8&oBGAzkR*Qq4LwD^yS41CNJ{a*-@2a
z^QG^QpEqoSMZAIEW45PXUhP^OaJFj_pXgGpCI3SVX68oZ>dEj~yH?6=32E`LxRGfS
zZ##9tvTf4}bV_?RC~%mW23)_<bMfS(xPu9&UoG9U_l5H#%|e-YH!e4=a|>&CgO;9H
ze}AJs%O|zf%{A0N@{EIP;EEf&=ax!un<O-Uf1976=C-_u<Jb01eS2$x2G_-9UiWH#
zrk2U9aeU#Km3_&OL0emU<+`=XOQlz=^4hU+5`XwQ<w+i;-0sI_?krt(>eZIoSTUL1
zdWrK2_~z&(hL(Jags+)yDv~leb^P6dH!5o;T`*j9l|xx}N|fPEgPAomeAFIol<?EO
zVs=$H*z}F0@tYo@;Lw%UTV~Gx`^@m%%t~AHvopJtctl(k<<>m?c>lkC=sCte4_<G7
zxH0>sO;?iulXFAUxp`~lH-FkNvs3u|!8}7}?X@cfG8yefv!C^f&r3Miwf07inb1d#
zr^?HVe*1<$G**xNb#K3hgiWWS?dJ=;%mD#c!&0|!{ER%fYboE%V=|$qv{uYIbnolw
z$f$+J2_F>J%xb#0s>f%SS*eKo)jG+w<<GC13apI_(x~csDzD?DIp@fNPf9KKuAMW#
z7IV5?Ugv0xjmZAO8T0=?d#}<Wxp~<H4=J9R1$O4Kw(SL1<DTu`G$rmY+k<V<f^8Sw
zobN`?jlRyya>(LW(9&GF#lg?>B;RH)ixHh`sy$6``xE2BHv$<KPo-VjdTm~MYo@EC
zp!@o}yXsc0x)idt{pWfe>sxk-2@<iAlf#}zJ&ScOR$RQ+KYo2GpUlOV_8AIky?pT|
zTc_RV4`tZ?Pt*&Rz$UHxsLA@&e>>0n7|Ep#^ArNN%T45IT%-`>cxGqv#~*D85^Qnn
z*=KKR?J7`iyE@HGIIr|;h>>6ISwXu$37W>sdKVu^b`BKzmLGloRR8t|(otQC4lg{j
zR;|v+;7cz5t)X+`@S2%3Z615Isa^W5{W^QKp@{IqXKYED8ai6XUY7-I{GEI+a#Le>
z{+`({7ym9yc<p|s_L**2zs%_uO)B~Oe_8!~9M|<X`QwhcFFG~a`$7admw$LJHevc>
zR%KV`SX(QphtJ&3l!>dk9(?!lVc879Hpkjn*|gp4BFu>|Os<3!>{uzeT)+L$9~qHL
zheAxQxVWZ@YWj$Xh&?P_I(MGx<G@u~+MLHHfwn80{;2$g<?Gj84K5Q?_K#0RCqCX7
z7#{1H>&&tKcw?SKhU|*Ftp*=NzC4|`{(krVCF~2EvpdvG*6n|t_e-qm=H7?O^J+i(
zdR2c(k14!0$L>nR@pHQl&v=v9)&0qErPuP(Yr&jSucR;fXz_mis*%R-dbIV1kCX=2
z#RO&1saIdDYF+e5DW>eT;WSn*C;y<>P{(AAw%gO49pf52f@S4(s@f;4w%=Plb;A4a
z3l;^(7Iyr4X5i=Fu;`GDk5<bp>CR)pD#?kD4z=7)JeacaVd?I`s5L@DhgvQw#k9|s
zx)EA0arM!X9XkZNJw2~)ZT`loDBz}L{a!}*^SLD*X2k~|eF~g0-+9H#d8b}Q<>={q
zDwjUm;CcK_p1@oc%c+MpISBY4aeT3SLkMJ(pHSzL>JOSMPlfdl-(GI9FFwA}!SLor
zvDhj9@>DwBRyW?5w>;o?J@3_<(wlE~osaQeylTRwQk|^o)rH+Rz3wi2yXNMm={G00
zB^+^H6SJ|aWl`ca5dlxTm>*X9`!=#nzU!tl>CvCl>z{YcoP2X{nWpw?p1j>@V(V>w
zx?hv!NH%xWkXW+fzstVD`)5@a2bMS=e7PmLe3yjfOQn7BcjPypJ@|XQ{lR&!thDae
zF!o2Lo~t}u@ptA<oyh&%n_Zn}*u)fnmi3XDa9rE}-^9?}QU<e4j`_}U`uooJN#Cv+
zv-M{?Ey>g8|9^kJgyHKlrdgfN(e{t{_uqB2ka4^o6Z7}#ZOi&U0<ZYrKKZz_fAjKX
zlDxA1ahBWfS6bULOX=JGHSo5#Y&g4S(xo{u^~d+}oV@(=@&7;aUpX6-3X7hu{T?*y
z)_G<BbIMD!Z^-gFEi@4BUcN=%)>Tn5@6lTE`A_G(e!n;*&|qe+$n<F!SDQ?qWF!B2
zn*QSpjm<rJAB8imPAJ#^=dQAGvRGC8vUI!A^QMm`B>~T^-|d<k6U<!y=jnUS)i1?a
z7Y5AW6puga8^5!o;Aoy){Tt4*&@0Srs(;TtTxY#*`GIVEEuC(a)Awpy!~gB%=|1bE
z!8JX;p8MLew`=b2X<xE(((<`-?{;VM$?I3nt<2xspSD@3U(#p)7p--BQjAWE?>Te2
zJ1o&`rlyoppHVQgXq6oQol@a4r9Gd{9ByGcytAZde#U1<x!<?tJ6sMO5wdz7S7Um;
z_9>H7=<dot8<USc>AU?R`pv`U^7yoSv*lBs-(?NIbKuR%=Kgc6zKN%fh8O)iVI)`m
zL-1QDd!fafj4Fxv|An#lnQGshUw1kFWqQF2UX{t7W$$iuuJVY^>2$dyr^vE0glqE2
zl2hCEgZJbWSy{F&yT0h@{qz4Ed_wm6zj`oP!)?xuK8<gQ|9i8_7Hcv2hI*7e(eUD!
zb5H5l^W719YgC{2%DTL`z-RV$UGk$Hf+6axZbqGXD`Kj?p6!vm*8VjAU$g4!YXuj#
zd{jwJY)X_>WO=wu?aaKmJLT_tS-!7*n(x2wSl?lB<3;~H+`h*j`&|Fue%h3;i|#i#
z&M{6@YUbv)ct7d%%?*X}%zArgT8j51eqU7{@#SLPy1kFHd)~Z#{xx0x>C@@=QXZJ+
zE6L;>n|Qgq`rV#Bxu11n=j{(&`kKvgZpML$b7xx!gzVV0XU4B6$7JWGnb+?hu>AX}
z;`Pqs6>qneOPKCZ_ZHuG@}67oZ@E8makDDE=+{5)WtA`ZJyqDS(rd@ld8c2@&3?Gk
z{ok=$SLa{&|H#qA)Z}9Jr=>CWziVv7=DOE^*4nn|qRoe^H?O?=(_#PjX{FWIGdd+s
zXM2xKwpzi}HgVC)jT5Wd4^51fz7Z{(@@PtoNlDO017-F2(`P!FE1t{ue>&K{+M@o^
z2ftT)rbwsrEm^p4;=N$~$H#64NA50=Tdx~=a_MFD4X^H+TMBF|_+MV%Vp%d_|Npm|
z|Gqv9$eU5i_ku~JHDyKTQ}1a3HgA~n_SP89zP?T+@rb1DQ=4-)e;6u0cX+oa(QM<c
zyL<lJQgS=HW7f^f?i>I8HJ|bQmo|^I<%ARyAD<gPU(Nk)v?ZROFSnv!^#6~B!==R<
z3%Y!x`3nx}h9`!;?r@cQQTdK%-JSxb`8u`T;`b}(1}<jbobJi~wz^#N`&|>^yBBVq
z+07reF7elle=W14&$im=bH~;6G%IR<uYX<_v(oC{x9RqiJU+g(=#%=nxodmc=9zW2
zzY|U8U*sqj>Rd8=l84HT{q^>?)!&XJZJhJ+6(m_+n)5_!yY}hVlcSr;SIqAVYmz%0
zFn7hq!wZfdd}H@3EUF}?FUiRG-yP09KVR*?vF|;{*6i>FqF+)y{H3SfcGgJKw*CF(
z;o*L>54YC&UXihAG5G57zA{wAXy2OFtmGRL3ctLLbULx{vhjun7p>0A_q%t?{oxZf
zt2uOS?d_Wz(m1{E#h;7iubW+5^<zV%Z-oux9zX8t^7qR(+&56`mF_#fs_<gZ=ZNpy
z%U5LcHD@K?IKj#D@;Y}Shg+}o@kLiZ->7|IU~QhWY|DR+Dv=*6FY;FL%OBnHaq&lM
zpBRhBHP`;V)m@d9Wl^!C>EYvaqc!#VAw0|a<&Rn4{=Rs%_@(70^LewUZ8Dtts36yP
zZ<VA1OW5*m&dKVRS8wUK<s5BsX+pxaQt?Z>x6Igdpu|o8_lq6(Rz<&G-rJT{@oM7!
z2isPeTr-~#Z`b!yxGf=M{c+J3XQy~3Yv_D2OnVZrO~=|rN$A;ug~~kg^@X#o{udu-
zRN7c~BPQ>dpGEtnMM^=7+dscMe%iy$NychRf5n+2UEjqPNfZWEHQk!U73AcYEm<-7
zs$b=MIi0BAcK+^K8t=^~bUL#?EStIEcJ2#5amM}avC-cqZV)dx_~oR?v+5Z(Z<(gm
zZJ1Wu{+i*1Z3aiO8{e^&jj0zjxN^IHq@0#Jb<_1y<e?LdwimQle`NVpaPjA5O}$T#
zXVo#Dp0+OW`#fhEtB{i)J{sTZ_4K*^BDC(z9^>yUiUO;qCLTU!yd+*fXrUFKtlgoG
zkJAm367oX=kNte?zIoR_<BG8N3H-&g4gXF#s-+R38@5JFQ^Q+j(l?E+re&ECYr{Z1
zq0T33j$HZrV1s(xs`_0=_dJMr@AK?W@kQyd<d*LVTKlA~zRI2&Q?e^PSc0u}Z~KM)
zDXiW1s^-2}IJal-leC(HybCm1r29U6N$#I`;^py{hs=S>Td$au-TSqrST!Jej&|tV
z`)3!nsVr8Tcl?Lu^T6r?_WeI^)^9F69Om|2;-XT)_9x+?C4W~u*0f!!EW+yQJAK`j
zE7uKJTKBRA#U&))e6;k+`*kUIrcQrou=MQJ(%AE}Bwp|8Q;?nKadOJdOYK!&?iYPz
zZtk0QeBKQA{~LuRFRY%is(H7-+V2~*7P(6Jg@>4bnI_bu*Ipvzl(6;p48zw(RnK-9
zM#@f^HuYhtwMB$#fUnsZuUkK7EX&MfwBFmgJ$7zj^tp+4`5cV_QG438xfVS9_$2V!
zVv(Z;8ai5wmz-C)a!qm7nX<IFYU{mi`KiCRz1{Qw7<Z0!!1?Cw=TDu}ZT%v_b@#x9
z%WqAhWO_9vjq6xj6b(WOesN1!uHk=uU4QW;wo~o>is#dpFH(9OpcC76ftQClB-GJy
zf%?~Jl^2ZER=Mq1*tA@(wkPG?#mey8C2wMkH_hk0n=$k8qNAEI*H5<TT0c`Si*oXW
zpH&bq&?hYCCG+|E&zZrpVWGuekKKCjc;%4xhOc|rY-<>7i(a(+Y)MSmR6Wz?ql{hL
zFMmF@J3aEY0zF#}%<6Gpk==E<{z-)!?|b(hI~!e2d2oBIwGl~Yd15xNKeV+=R{rz!
z`+w(^--|vuv5SlE&28sonLn16Gfa)ugk^5*?cViFqW1Uqg=x>(#Ogl?{{Eixc&)kk
zBOMLp>u--3L{;XAF?C-)e<RR2;_tugknR{az84esEnBoH|8SPrzU^~rtG9jrHv4_j
zY2Aei_Ve%UJ#}M4mf&jpo5!~Khixvn%2xI5UGppsA^(|@_K8A&Pk+kD%CNC=I~I8U
zmY=_#?WeUmRgZJGy}fz7>$~`-Uz2S$RxxShwyr2ET){f)<go@vXGzm45oflq#~*J=
zZ4J8Uwd3@c)oG=dmWTJ3mA<>UnxAj=wQc?a5^WBftzO(*ZoaU<_^0hJqXm1WbTPAY
z9A&w9OxfmtfBlBC--(YNR>jTzE_Bt<+Bp4uT1KT|=+&!BcvyEIb$U9{S!T_eqw$y6
z+vCpOey{iJK;vJXy6?HGg85E)7jL>QuAAJHmG))F#*n<&!?SXy_Q)DD6?dE5kJ;XS
z{eIo_y}!S2d~tXCN}q_A*SD+hwa$uvZx=De;_KN$wjP=DFCPBwNO=~!dS@HM*;fm~
z&l}mr?VQkWck|e@QtgO6HKv>sx0b&6w0`Ye0iEqa@G*ecWw&qnW}OQPS>6BQU-*T?
z?iUt^MtnZrFzLSDS_RqioAM_dgH3+*NgVG`)IK3;H}g@$WUcn*ISmJ9M<2T4sT}d;
z*yc6S8OI-+OrHMSnKehxJg&cQ)v5}nx;J{#6Rw%<`fImlmD~Tq;2GO$`SbS}OlM6q
zUKujUZt*gs+CvFCe8P*5^*)%|n*Y(SZ}$2ai{8_EHb-{r+aFaDeKXH#-TtRWfAqB9
zHORN0zUEbSQNkq4<XmO`<@_54pKADc6gODR`J$=+@bS|8jZfC5h3uZ6oj<ep{L6$>
zJ(e=7Y9^`w|5oh~aG~0ySZ`KOzgye4{~hUaKW)pFvv(`A&p+L`Z01LHrW?9yEd~bG
zHD~ty%6evJ+JFD=`}H3x_D=csw)t|R>Qt{~TleZ}ba8dHnF}09yLKT&x^BOGsAsnQ
z=7;m6=G6RT-nS*ZNYUW-n;(1gLe8GqDSPGTCdubXqNZzKO%prxw~l{RTH-6$VvB+w
zo^pR5=U)(M{UgV`bA`$LiURrUY3q-K8MIui{XJQJ!^d-{{WpH#6`pnU>>Ta$$KP%-
zuKA%6X8+yj{GKm-{CA2h&10%1_4j<!`~Uf3(uFr?c0Q{*#r^k9@s5wL%dYg4?r%60
z*Q}vmP;lVN{6<@D10LqHDN9$ZnRB7h_s{+RddkbRV~UQNe$hR({Kul3GHWuf?z3Aw
zv-qOOAGy|*lifD#xVq=``Ok~8+@H^KXh=#s)pPTWoym0df1j*#^!VTHNh+$%n^NYt
z*{)Uob@1C?QA(h3w@w$OpG}M2SjgC9#$H|-;^(^9Fz2N6m!fxv=h(BlxJj>6_@d#q
zs57P3DrZXC`Q-X7_J+lrWwY1%T2^T}8i*WzT9$Z8<j_KaRX5uXr>V*D3i~&neyTLJ
zConv`KftHr4qp)4jyZloA2~Q>=kQri3=lYDHph(n&;`!crU==*HEF^tU3y-v(rS9B
z5w?7GQ&ZELhUs3%GPAgnHd+_%mXTP=Qd(MiWU^IWzx@2;Hy>TK?6?$Ux5V^r^!0t`
z5BvX}yuR*ryN^^%QzD1ZMh?zcg&@vZ&Vq0B^cm+{OFy<0<PMI$-=G-s$8x9cQ_e*X
z6?l6ceZ;K&W=k%dwc+-emRZd=V)AsB`RHVFm|eRp()FmIa>J}pp^F-)#q}N}6ib*2
zUyj&nl=q5<O~lgh=A0P@8GNn<JMYdg*?QNaMTs*;Pkq*^30_A!j(mOdz3)}d3(uyr
z3v=#%3291D>AU^((2|uZixRV5zY<)<Ip3ar`Q4_8UYG939C}rfxP6vkkmsw{uRD%S
zwzw1bCyt-%<p!U#zP=KxSl1u#UB)5xDx1^wQG~2s5mTLss{n7ihZL9I^vy^ASRQ@2
zH~Po#c)7$6t>%S~-*_fSy!_@mc}A#Xaj@{JqkevV48h&ocE<jGGp&T@oeTTtbBXFw
ze5uc)s(eI}9x9afo}HmncE``VdwHolyV1M!$yHO!<af2Ld(ARiO7i0RA5-22UJK^T
z%6{#!b;i-vvpDzLH=NCKld$Ujy4B^8<oYx&zUkuguUBq)<#Zt+`sl;1tP&BXpROX}
zPZpFu$Th!o$Hiclin=+wvf!e13uj&nyr@$qZdsTA%QZmD@90A35Ob5{(jA(b+DBh+
z*t#<R^^Ties~&Pcd9Y!Yzvq4rm2~Y+m!`YB%Y)<M^uB9=3b!X~o&<BA`XJfzG&$*#
z%VNVfCl61N;10}fmAw++xMY#WDo*9N(}`b|7Rhp)dbz~wnoYz>j><^)*ew}$qJ6Q}
z%g@f4`exq~v&w!=aS@TPD>T;_M7^J37ghGUD&<poh<Rw}%?(d{<?283{_@!~#d_V$
z*28O-oet@n<scu;cJuSgy&E6R_$4PjN&B*1;hDM**S<1e5H>8!z3*47Q}|^^)}l53
z_FYWx-q=0sJM}6<#;8^)Ds!&&>f+a*Zq?+kOg`qe)7Sdw#eeTttndHk9)3*avcJ|=
zcC$~r_n&e7`t4zM{O-9)`!~nl@{uo3^|_RrVgJ5wZuW8M{pTz#J;N<u3g`Z`VR?GW
zHhbs)h4stVS#}-GQJU#xn089%-TwD-r|t;WeQf?6xnZMR`&}Qcc}f8$6QbYm?e|{J
z?`%A2P0jPre?M+aezTrGIh2PdTyOimGV$y6scL4yucuo!)!Y55xA}BTukP!M^_=Ve
zvfKZ=f4no{=$ch8s=5wdUjIPc{*T(W5ZTZ3s#=!Mt8d?trL^?*l>S*q%J+Zu{p8Ae
z|IfYpz*u>esa^{;7R>aSF}H5t*ZKP&O)`ACd`Y~@q|KtxW@gUuE1`|wr$pqGCD^Xg
zSfE|@XxcxHy_0LY*m)Y0I2XMB=J)zi)C`Mn#jEp6G>W%)6|V`n^zHY?Lxs25WTVTh
zAAXzhiRbWwnkypvuIKaYbKooMzLoaBD81CGYF}-vOWNhvg};_>{<Shqo4LLJjY4d6
zdi-(WJGZ`u&t89gb+P(=>lcU5eX6-tlKZ^7ar=Tf>~eqP__p(JKYj1yomanRC|J!i
zo_qbfi{H0f-wMO)7asRj-8JdOXT_;K!GX6Ad{`O(@YBn~H#R-1omcr!a!zB=zWU!G
zQdwFbzRDc5HIH*#lcoQ^;+baM=imE{YmVdx@UMHazy9NXmruV6|9_WTbVKOb%ANfm
zcE|sf@pqIA_mjN!Eb3!|vFTOQB|&VR8OFQ^1vcmHpDeg^=Zh+?{6zmZYR%u{EBo&M
z`zp^fvr2V)e6?Wv_H~ag?zcU6zVO3p`8x;aZoltbyklaezx}bk1(Scj-!n6&zqwJ>
za`}E{Rp|W0@+o?&o1@k*^k4dN>&=v9nkmbSg8B@;_+9372sM4fXB_wY<V@4D(!2G`
z{>a|E8z+;O-|bM*rggDvUDf`7O39mUXy|A$`7Js7w&Ai;#<KcO^<7t%-<nqA_RWa<
zLJp6NX>WJBU&=2_XV7U1x4*wr5t`4#HRZeffivD|396oJip=Y4Q>D+I_09QnJ&Es#
z&z~pN_K#+HW-i>}zHj#Lca<IWf3ppz`$=`J-5L~``BCQnCAZmU4_?U>Hz+NT+@%)~
zDykfH`Zu?D!b{6(@_#-=^XPp%bR=-c+_JE@eG6~Z#F%7@9cj*q{%FR||D^Ea()fi(
z(rxd`s>5nA8H2fN9#nhztczZlDfQ1m&#wNu*SQ&I=VhH+QKNNa=H7ivoc*F4&ZXu2
z{9i0+cDs6k{k>GXQc=I%8`dmH5a2PrbRp#Yz25m7)?0a<Ob}0-%+=-ec4G6IsE>l3
zZFhAdcCtmEOxRP_t6aL(!mF+D+yc9sm%M)SBu4D7`F7n*$e%0cnxE8S4HxILHh*T%
z|GewGr`L5!%dbM~EXoy1*|R0h>a6y1?R~jL&3f~?{j<AEWjEivbIDg%?$hp*-g?kc
z(~~O?vMi7DuG#eJ`FZajwli0GsVtv!StY0I&Hh~dZ!`64ty)uAw}iA@pZ7yF``(7j
z0m1z99AoUN#r!rmty+;2x3?$$&269Bvx-SolJ@_8#(p_bx{UX(@S_#-4wsnTTs(2*
zcxHU}e1>{IzMqzILA(0yzjjgGtLq|>`(9wxEvsYYzs<no5K0q0_DRh7^&w!<<nwnY
z<i*)`K2gckzPW7w+>G^~Jw+?O+*tYVD~Cad@!{h>$;Vn8XXx^=@yee)@JT4hxBUWd
z^To^h508J3Js({Ega2NP;Nm4F{2|<`jW$YCxg)n`itk;<%l_a&A8*c`z@rP+|4_Vl
zU=siJ;0x6p%+=3>#CazMT%D%ziD~uaqhHJG=ALBs+I01za_N(AXVNeJ@_l1@<-mn`
zbr+I#D!#H_d)HZT+;)4=lhT?vW=&{1usl2`%JFCLy1BfE9j@Pg=N~4*8X9`}V}<IZ
z%~P*6J<BYgR@&8av#e?351D&LoFzIuvi8ld*U!H4?$aTIf*p6hXvm-7HNSJ>k?8-E
zo6qk*w8D|!;O!yFGu4%h^*^7U?+m}^HF0xi`n-xsQfJSb>+KO({L`TR$5DU7>2nX4
zZg$`N;PBOms=kAdw_Q|F5aZkYU~2fy=vS`(DjF{=8@SKUKf)@0_St;<zsCE2{I5IL
zdp*wM?L1$bk1v8ne0Iom|6Cy+uh_F?=DwQGY7p;yYF&6bZoU7B<vJebB|#en(vD^X
zw<Jom2w92kmY#jK?v%9N^(~VwyRqN9XK$albG7mA)l)5O_lr!v>E@Vdmls~09?`Yx
z;dyO`@4FZ!41U$*?Ra8)?`Mi?^tZUfI}(L|ey{&8xWE31r`*ra>kU?GzWEol@Bd<X
zqvAUbXWV*b=kICSZ~K$$|M%UZGi+Gz6o0-dV&M~FahIhgZ`vN?`AfB4tQ0wXZhzOw
z=YDCQ-n{-%7d%6Ke&y8o|3$o-&KJsW<?O5dZ~Mf;N4GTi<FhA$6V5mqd)@}EE1c+|
zGD)B7)VFm<0__4huAkp9t?&5b8&@oBxtMP*uySfxw|vFAm5a9-c3qV{kXBxH^6+|>
zPpuEOoQ#?KHv8S5>r;9c-`W3U%27~T<(2fT4ZHX3c$1sFj%(rNI|1zmmLIBOG#pH>
z_!L`o-4;KPXxc1bVs<4cG`2fWgXI&~iSu6<Z!<aJp}NARXvVPsk#dFpMNKh2T~113
zi#BSyyk2`kY&P@!>&>egE=&#ymszMNaxi7%<oJ-5<9WHUmQ81SkMK-wINw~>R-@~o
za<})qb>Aj|rVNu?=cLw}&RUkZ{kHYj#zpFnuUxs7ym9x3-*<A~wJC`vzdW@4rhH++
z&o$hIxBbrUeRtwp>6!X$PG_~VPyTQ=FU@A|bWwVFY|f8!^-3>48+$ExlF;aCI+|sx
z7PhwakIjY|vpLl#9Z^cm>FaBK8my?Gy{fB;r&r~r+8i_YsF~bzZ(jGAb<!y=F0TE#
zi0fhL6Zyqw-hE0auZ&VZB!1$s_=_DpS5p@5aL@1Rx6IqP^ThU@R%@p(nyIPr*3)r0
zr^uZ+`*+5-s)bHY=Ra%^vA4?f_O@K+3}v45YeLNId?}}<XlBg1mMD4k_x0lY&Hv9W
z_`Lq1z1%ML>eX}Olq8cszCOPpA)fK_a{u2`nu@woyYHu5eRepe>~8qW@CQ#nY}>l$
z(<A$gZDNJ@S{!S3{Mc&t98_Uls<rqx_kYQnxkm%nUq8O@>(+=h7BXcvamK&-uM2QW
zTFhA&{7x|%G&jwE^Zt^edsjGb_;g+I^WNuj;-=n2k8^q^zKi4c*X7*a*8A+)v$Cd5
z#~&|@-(NSE>*R!M8sXvLK{xKqIXAs)bN2BvtyfMW;?EYW&|0-9?S7<Lh<wuLX>VO7
zY3=>ZzhsZQpqtU5d9l@oma!_$yfSN6EM8Gw&o*_s;414w*6YtceyV!=<ZJ6We~OP^
zd_U)N<qP(mQ~0I*cf1Rj{rO*U+l7c|&!CNgard~pvkEfq=Sd|u#XWtxW6d+;yKjX$
zT|O;cRP4A@dd*y4HDR^MofSH5r<VoYzTw~#y3u;h+tn*IZnn={q;b>y)Xetf27T`u
z&v89<O5g~%aC7t5<S;oi<HARlH*_<D&*j7^TJ@})Z8?AEJb?u(URI@ilBuoGepz6=
z_O&WY8_zzimpzx|PDdzB^f)Ki)zu|omLs7ZzAoo;Pgv;G#KUcYAt5D)j6_srKluu>
zamUwAn*aY@;UUfwx_WN;c1_PTTmnP+a({RH|9&}sL#DrJn(u<DQ$N1(|C_s#`OKg1
z-}j$gIHhZvPr&_vcIA43|L>>Ut8e*ddM|d9oHAqmpP%)M?Y{({)3rSm|L>Fex?Sqi
zZ6}uA{9OO*=y}e&Mds}GRrC7!4(9*+CeF6w@Sj)h_migIYPSFT@%oNWOBU!%^iWwY
zY%1(+@|*pxlSkW^!{^?XO`bg2*|X5lwyW51Qs5W9GmC{MDqJ<)dL<-uL-KFGEa~PW
zN{-*&-8%VhFMEetaAxB*kHFZ%j@;YZ3LkigY94mbNK-q%W76)tS<R<c1;j`D#?~_b
z{{AlQ_qn?!?a6N+Et9I6H0xE?tL*F>TZ`G-wX!|cl4b7gpDt@%ep#ezk%mB4-jZ$G
z`nsBb-`JSVf6${RAz9XXd;aqS6SYbf&o0@vq*}@9=VVX`X4$pKpyWjW7Z=y2rcDA(
zO-)Q~e6qJvUmLpya$J*ra#Vf(LbIf&)2jC$y(-?bY~3=$zi<3oD*m<}|M_9Eb4=}T
z^Pd-;zbCA^dFamaXMxvcl&@X!h`n-#H@<qpYkfKYv+w*P_Pr5yn_F!s>^6OI<lLCT
z({JaPuasX|(X?w<l~dr{!smYH=k5RV?Dmc?lRSAO1I|tSYaO|}P(JkcwvB1m6RY?n
z`~n^FK1}dXnRMB=y7yCNT+4h7m29`)*=Dm(o;|tHaE{VMuSJo4R#yT;d!1)b|NMJ-
z-0__4lY3kYb{1)Tid?Z~m8wLa;m1qaKVQwsTvPL1^x58>(D~OUg7dHM_J4OPikY_M
zT0|h{Rrx3V`~NU2CLZ!o^GsS+e%@)7mUi?fpGOCOuYbOnb^V7|)vGt&mT^%^Eb6LG
zGCXwUnz@kotR3IJtuH+Iqmg5$c~p_noWR`8A;sP*lioERf1LQ`#YLT%9TQxX8jmJ{
zI2AVY)`>6DXpvfdBV2gN{}#)dugkJGy7O%+yJ&xNfwPU!lLPZ?IfT~4?(C4A<#^@7
zh1at?js$VA96sjz<d(MnV~4yc;`d|xl;oCfsS$WuTeUHz*mm-{xi0IK<fF?|)?5hT
z`YltuZ_9)yQ;IolzKhK^TX(Q@$;6Jl<V#ad-q^a%+#NKWvb-%%bcw~x3!X=Ujx6$g
zv{8iNW5%Qklg>pN8ME3vgRMohf={aW`Wh}fyKCZmyC0@LyzCiaTtB|?|L?U7ZrhM{
zHR{!DCkdygmsaOLbGQG)Xng+V;ZLF297lsdvx*M|oH!2o`p&RjdguFv?>}$pP7mH#
zt7_eUWm~0n?hDOXucTiZXKC60xmYe4l9BZKpIYAD_loa!{xs^#i9g)<icRREfMMeE
zW7Y4TXPcjYcwn2qhK$Wr`TswvdGxkzD7$MV6y!Dy)Bt=Ed~thz{HfEY1)T!o;{5LK
ztMy;Uz}S`L6fE-SqQu7~enx`I(>&HYi?}+5hU!dXczoPH?bw-_ck15ny;F16b;nz^
z7KZ7kn>?9agD*!PG?M$ytGxH@D&f}~lh1#bFqyQG**kK3mgak%*v{1@2EuJq=fBjO
z2FgyV%hm23Om}TeiFtmsJ$ypP(hqDhA+lL1A8rImtvVbcm-Xb)Yqv|Q>*LDw3#?lX
z$i6ta<6*OAiHe4<-hsR29JZ&uIKRI=+f@46le2eT?XM3H-K8sXud8M8$vDlZT5>vA
z#Ax>%^og(~>n>lj-f-YmiDA(liK;Ie3*9{pOa0Vqe}7SV*YjZGK4$|lcMiw?f6Wp$
z3-mv(Qem~U<dHL+vFTD$)pXI#`wU&Y#3M{H1GOzZ&&>#nJQvdGa;er}_SqgO)2?UF
z(%gEbT5sO@x_6J0gv6l((i&+I>z(e!Y@QMS_k4YFzf8i}dr~&FtYXWjGd8z<y5-(&
z5hB0nNb-g+@&Aupe9yFOesEi&goLEMhSFPM)wlI7N7u~kWSu?Z`<LJ!@Ah!FGr3)R
z?|J+4yxV!{Zw@G_`AeREWdE;OYhlqv-}{Sn;%CKrs|cN(xga`YN~^$(wNeY0X|yTs
z_;4z5fksYbb;oCY`KFtTHol7A?HA#9t#g{9z!r-eMm=kGmMmG=v|PTrC)Zs{Q%iHH
zg8A}b%f~C6^#Vj$qmE=Oy*m5Rv|G#NJ9Pa|ta+?|QETF*Upq1?4Y)P)J?yND_&2*K
zO$-PKIM5?$tf8UtqDby+&f*0dI%3vKr!Sar``!NjLT0YWeHDz>t67!Z`PJ?3_h%Iv
zJU_O3=d(%05ns=X|2!N&f5FKU4>xt|sT_Z+qQ8Ga!}(^e8E^f#y1PE^yxVZBGC0kk
zFu+oJdi{xCYuh*MiV+bLeN-0B7<~KlZ9Yb=$LhXn_mf#yPv@Mhp72^s+Y~gy`RSsq
z$tm3(d&*++wB`2Coiwj*(L@%x{mo55@@IJ`K7J;;<*S13EsjN*il&nM!T0ARoMC+|
zX=^fn`n7%TN~!Ei*QUSjY0!z9zEq<{s_(-s>-WY9UqXIWOr5s<(3X>vcgDF1<W20o
zEpz_a!ndU=lY-m6%+Q_avCk!AmYbj7InSpj-<x)|%U4Tv-|du?u&NZRmob@_pwf8z
zwX?FP&PsuT*%$Zx{dv*9{;X{1Ru$e4tVNtZ?krgT<EL_Q;99;j4(BI6|9Ina@<z+c
zQ4bU^>pkOFRuk}EQczWM!=!m<fB(UUpX7?;jz9exw7Fh7UpLCB*23}Tfs;ORg_F{r
zfQAMvQ%kfRBW1KJw!OJ2=%)Vc+S&~_WLb)8SIlBs)ph*+@rRof88%;Sw7=eNzJAKe
z$!(io^rlB{ztDF)Q%(PTJJ*evyghQVp`jmH{@jQ)b6BuKD{sI2?wc+)5vOlG)h-m_
zTN>Xt+d-k>ZW0q?*>dUq1xvP`n<ZI)p#N^tv4>wCTwnY0@hZuBHU5`2jE5W5Sfx6j
zOi|=ntSWQo;%v97{)N}hRqu6C`uS<mOr0lBw_kG5*4j6xre^Jvx7XaxhwYv%xA)F*
zF((Cm``=eV&F}>Ks3nUYeP#c!?bM|C`LDnHY3rZ%@ssxZpbP0N<@c-AZHr{mcg<E2
z>U`qez3FejhaN4H#TPa7io2NR&p%%O*Dh~;Z}z*nC%Efl{Y)l3SjN8W!>h@wIn9pk
z_`Cb^%B&oPDJF^Am&?1%JhfcN&}pMe-gIu?#sH0uq-_tkM0*}x+>+#JlV`3Zx_sL@
zxrH-6?2ui;Q(!rF$s<7_D+}2&rG*MJaxHSjS{4d(v2L`wGegv|xR_)0<<_e#2cHR;
zO_yHnsVv1g(L^LhPu<C1kgILO?mH5l5}eIu$E{B(Pk%q@eCy;n5wV`@URxbp=H~WA
z|6tb3u=lqF+77l?ueuQ{fAmVyIt`X1!B%^1)sB7FJMeM>w}0moP+xIF_4m9}r%!K;
z&`IBAmCn1cElMj}y5J|%lc#JKo}9MrbWyTQF5z~(ndx_N_mYNXZ5*psJ^B>EX?j$E
zX^x!s!~mwl1|NROzOjvQ-QgOL7ya?ey@x$NOKv%rx(VO?HRqDw$|;6x&%N=^n{6(9
zYGLS3rM0WG%I9pmZXXr)`?2iPi5`B^r#xlzn%BKHF*h~ZB+Yf|q|2P==Ajpt>|D_;
zt>JJ=;EMaB7U|bBVw0XtT4-BnF*kkJDxs6&Qm~Q2=RNC-L|KD0=T<27U7g7%BH?|+
z(E8G=1uj~RH`g&peRP=<;>CXS%E5j2wmrL<+j@T6?{@3YvvjS_-2PS_Znyp4y{ns4
zCS5)yUc$dQQ6j=8Z5FTe>)Dee?Y~8Y7;~=f{CkpPVS$7qi{JFuKH=e@np&vS<&v-E
z<EBkFa~5ti^f;<+ZFuA(GnaN(*Ws6k9#38<q%uiIvhlmi`&hrF33qy1a~iq2G~X`V
zw{(@KqQ}YwGei%qn)~V%n^!6G{Ofn09=j4QpK`Qa{-Nf(^Bs(=Id=jNz6fl-xjBo|
zdX|aa-U_R*mjw^D1n;f4ZlA@f+{QQMpB!X$<s74PI~J?~jaSLp)xRseoSM>5QlYj}
zYLdsMlB)&}L$W6B-e}k&z}3dM>)xZ?-VJA$BpJ<&jGFpjahKW@mnp@c58p9ZCNh8b
z-n7Vm^Uv$c?=Pr0!qfKUL~}^XbGg6D-e%^-pJP?8guZ!n-e338^}nZYe*CC)&o<}b
zpYC&u%Z|6Nv6V<a_VanN_2csE`s};ys@hqrS)1E<j_Aqrf{Kob9)5y%lQgf3KHM(7
z{<P{OeICikU7PRU{&=ob{F2iqC!?v?G_p~SrRG@OUiwg=xu0d`t@DkGw(*8URcDBQ
zo?kg>;YTO0zmGewzDs=l?5)q7O8(&Ee2afxs1iCUvdf_G;MMI$MvQ!_r>fuYJzOl$
zuKC`2!arMX!)z|ztye>?UP&yh(A3pDl=MnRW7UF<>)KAOiaJv!uBv+UhK`y*)3IMt
zt5&_fu%^t*%WFgQS(QbOQilpfY&~n8jV)P^?pRy!<bxp3<lak@CY98=w3m9gG+!}4
z9^lgSEMd;3I9tO`6?Z;)H4$H-6Z#7jj^AIoxFB)K3cH#ayAoLP{p61R+<C-_*Mv>@
ziRhEWOCqygWqT$lnLkjM%X;!*qShp*#S2!j>_6T&(JLrEynmVIm1|d%zdi|=@_5;&
z6M~00n-@(A+T$nva2eYpHi?rOUwIwxG}dqx|JB{6KIuhO+2b>wU$#TevG;sl!tMCe
zrR00-PM4x<7g*PB`Z4ACF77r)O<AFnCC<m6pqveT$fqu@VO56k$`GN=N2|6pq_k|0
zn|u7$+o?I#kNhnj6}ntCvb(iux~^}3Vi>QUtgu6%LuhE(y1MS~d|EY6?<m^*|JMFy
zP3Nc8_y6?%fB*k)!<B_o|Ga-M|Iovk=V-vr7q4#bczw+BhR<8}#OvzaKd-)*TdK%b
zWpnI8=Jd~V*WdN|yY2grpZW62leR2$mws7(@amrj+2(;;Gd1mhyq$bQmQTpHcSh{a
zFFG5FKNQ%0<>ig3`dnM{_Selfk^5)F|9^PVtaQioXQwT8WNP03|1^1pnz3KF|G&5P
zatTLQ*FT^7z5MZJO&8s~cR$bdTh>RM=d-dr`Dpt5rx%yUDK2ZuI{on5?|r8Y;zj>|
zJ*~du)x!G6O>t>;|K#nL-#=<zboI!+KhK{3KYQ&JN1F5T+2%9qf3JT(Ve+a6`{n;A
zew{TpMaeN@qrr-Nel0$Em56Zvk2`Mu`Qu)`De0cD&E^c%vh8ydG<@zndARTWpN?*~
zZ@$(^KJKh3;q19@*X#!kT_{cT=yTs*aWzlE>SW5Mv+-I}4;l1qPD<39I486t?0@;U
za<RqRGp6KrIUQJY;r7xE`#EPlzHoZ!Bh%Fpp}dMFi#%BqWgbf(eDh#ac3#q>3r2an
zp5KnCKiV2o_qEqYBJ3D%dHrnbb+bAxg_lMKdK_soF*6OaDX`fdV>fqU^7*Ig_W#6M
zw@LWh|76+5`26GZ{ePyuzi->Fdfxu5diAPX*DZsN7x$lY)t-G|{r~s;e~xc|zbNbR
z?>Ck~z6Lo3?^LgUT>byI>HXdR-L~o1%s*_J*b_2o*DH<bcM9zP|GxjfaciA@>6eG%
z`+wTMn_=4decHMutG3O0KCf<5v(=S^B|%rNAKmff*Y6)s_3JzuZ?`;qb+G*Z-~B)C
z-~ZE;7}v@#zW!~2)fM?$=Y;*wrQbTs|Ka-nf32t0zCBraeBRM0t2S=+X`c?=mwUFJ
z|NmU6iN{qht4-!Izm$Ao=bty}`ybW+`@CDDH#%bPUjvT0zh*_T&8z0z|M%~Iqm4(-
z9D7?@^IU!Z1NHkm=PJwp?cO`}tI8qAZy_!-YF}^vaW}vI<dK@W>hr1^mxVl2TsDEn
z{L`CY|FV^^81p+Kd0pf6E<Y`~Lu;gR_XNIfb5vQFym9x&TW`{L%@R5(-Yv{|IC42d
z`mROl{p*@pcVIv7*4IUGY55(|yngZNhnwzC@4lRP@X5zVn<78_E-tvGdu98%rAxN=
z&p500$4;EDUBTYmVN;&<ZZml|jrNMKOMf0|{;Cp`$amGuXkUHe?ANW*Sr$bGM?%YE
z+|D1Wsd^HB=s(}wma|V)Gj!80Z)5%Q_~vpClh8>$%LSC)?!A9KS>Sn}q}xnmzw?Z>
zg*_79o{n<PZaivR^Gv@^6@7E7hGlN@>kBRXjq9%q=*_X3*VMqPEV{gYev)0ye_cDL
z4bKZ~YpbMWr4+A(y;-az@#w>~t!LKXadQr3e)vnkx3G0}{+{lu+Hy~JmYdf7J$^sv
z(Pjbj=|PjWx0c8KV|VhuAkf*U=)f^YPJZ&TdESYIlFX;70^XMzE{NwWJf6U%!|8NU
z!-?aV->fsN!tRf*JlLeYKINvecLvw-&41-HS0rA4c32|zMLuMzW0G*;E!L;W*YB?k
z;hG(}&^<BE&AnM*!E4P`*{@dQu5p!M%;A&#9-=gHj>O@H0vRDsjz|{Y!W*&b0U{bj
zjw{*At~N%p9==s+`SQoB<2CjNrP&mt)+9vT$&EHOKDzgp!Ge0jU5N%6Pb%8JWa@9x
z>}pbCsq)I#<mj2wIFIA=o#guDlhZVc*X`nPzjJU`X{r0`i&@MY9Zp^g=LlfY(1`F+
z>*`y1c!M3QOL-2H>shhFg4bUEeorr!EIjBs$9Ri+sP*>Cku27(jFV3~&;L|6<%xyi
zsow?XO3YpBtXD1*@f3B@@=P{hJFv=p-s6jv@&y-Fi(gEAobvlu|LreJFMWzs)V0~3
zb@Af2r91`<x_@f~+>R~(8K|>t-B)N;aI({4?Xv5SW}Ig-XlY!e5wRvrXWEzfSF?`h
zl~0&I-MMfc`|N2-Vx03g{L!mC93~Rt>wMYI_p+#~Yi=RSl2ux(s>D`3P?;3AW#+;3
znUdR_x);a<XY@1~h={#MKS5BSYw6{(FWpyt!a}Yd>g&Co$#?W}tYY7>1aaN_my70}
zIOei_=btNbt5UANbB?gt`9(Iy#npF$opW2D%!Un_uhW8LV=hQ)EO5UcGp{;JF1qY}
z`qf9%f1Y!;PkOvhODgO2%Zuqe#>;w?C9@V73OPG6CMHO%eY{CJH)*qUTch#>_xp~r
zR~9Z6Z7ba3d+}M-B2RU3zugW0R2ln1!WLgH{QqRyw}lt7FNj>1G%I4679l*lT;9>&
zt>WRyc2BXs=j&dpefo2_KGnZem^YNyg4uRw!Z9AEZ!A|!d2ddN&fjR*av<Y!+n3kD
zp`np~|FQ4cm0|h2JU{K;XMI79Q%B!#QUKk$W5N7Wc6;c(Nvh!XP7T+&KXR#sS6-J!
zUGVvzIjikz*5QX6g6BMUHCA#t<Y*OK_yV*h<*Ulq$48RFL>?_<o2FOQZvJoQlmeTh
z>76D^HMpi$Z{J*Y*i5KX!draL$xo%X1&?cn1cpl9^SZQFaIUPtT-6tGFRUc2-mGp8
z-!5U|sxURkLv`UYNxPK|w*qcGoW8C+rtRd*^gmOIBftF$K2u$+8X@yU<m-`C^X2EG
z-#mHt>!JzkY-|4c=h+?3FBI0?(*EP_|9`DrD|%<Y`)74-RcX!l#@|2d_t#EYvb;s6
zpE>sW=K7T(Tt{C$T~;lppucajeEo07x}Vc4zgU*DHJ_?B%01Q^@~&oTe$D6GH9KA!
zJhkmI$kY;RpS-83L^dF##&YKl+n2doX{#sSVX`+X=QF=|b06Q%IB~Jv&3l;amx;ND
zAN_mWAhT!o{eR!T|G4@8?9qp36|X;cVgC}c%6D#U>)#)@=9+*0xcl>t|F7epU;Oz#
z=+<WO((16yCGH*3`TJX~_f>J%#u(oFbKR>g-a@!P@aT8>KWzQ`H}%Z1+gotMM`Kp&
zojo;go^2_deE6yIgYOF%R;4Mb>+`p()~;J{;LxL={huB<+b4wIIea~Tzu<PZuyQ@S
z`Qn=opYAO$y%Pp4ZYI0ltkbBtHEpkKtcF<cqfIZq=$uoP>)&pfF2S}iprgXZ&q8LI
z<*LUY#I_h84hxxj>TdRr%XhZ#c-xqNY5j@$SJ!BCO}hFcLu$|eSC?)6Ueh|}KlkX?
z@R*L{iHuXGRjkh4D!F#u1kn{!OKgOA6a!4B@;|pfcZsXHdVdw``G5be&-gw6wCwfx
zCeDeX9jzYOu3sW1q`u$pe(-@O&&!$Zc{1)RrhDwt+_g?PCod)CztEa##_ygj;O&f<
z(jLxoQ}T4;QATFn&wi_VIZLam-yD(C`kPaF$M|CJQf7OOl`cKI+SGFR_~oYe1{4dc
z?dINnv}K;x1&OYQpQ|oB7EJPF-hR7PiRJ9sUYBIH#u)oFU)4Jy{<+S=YO}ll{E6XP
z8+mTF{2GOot1_k2oR3G_O?sUk+aQ&@WZ6E>vyT+??Zvx2T_rSMWoLIBDXpoqSDJe0
z;kO34<(iWYJTG0$c=n*0$Y$p4cb&8Zt*iV3Z(UK;H)rqmw4HVAv0L`WmQL4ME7xWp
z$tiwV7}3${RP*84>o-|E3s=3|C!A*eJg!Rg`IP5@@xi{q8eOL!tExXxm&{d@xcc#T
zZH89Mtl1tMqWh2g7AM>K9^}yL=ils^u*qUW=7l>qTJF9+oOG$9b(W_DS6;vL7ZKlH
z-yO_yalg;{M_zpMWH<jkUlWNVZ@C3N><nyh(ecsIy!5wU;m(x{>()-X?5{L2;rqM0
zQ>IT(-WYMNq*Miz@hvm`Zfkw7xU^%D#+%#Q`(M4vGVwi}wmH#kHYZ!NL|fzSw`yYD
zjz+B>M-6$ZuicTnl>H#Y$j{kXS@W>RadF;b0iJ>J_gfANFIb^4-J{lYe$G9&#QTXc
zB?l{H!h{cb1O`Vs))q@FS9b_Zlv7)pTRVB9fbT<#G~a;u`yD4=En1-=znU#Ufk$@p
z;lrZa@AHbutFfGnXzA+mO44kevxkW(PJFBTw~yaeufJ^nZs`qsvHsJAyN*XI{XDez
z^@ZE3AG}&ymSBEIh%?Gzp0{IvNnGE_q{RykO#A!p+pkUYO((a?#8-V>e^VyH=%fg1
z|MmXk)mvnin>SpRK2i65>bHbLJEh!K^B(=?peW{h@Xg1|hacxhCtr2(s}r<59u}~9
zVzOnFegE@}zdqBpJ?~rV?4HSx!^gjr&;Q%*t4T=@ZWM|f5pv%hWO`BgI4fv|mE(!*
z+5FOq4cm6D-5GWD<m>YXU*})v+GFY7e=|mIlj$X&A4NqBjoWwhByAUtS8#CjyImHM
zb8hk9{!59`kW$#gZ<@cV#{3?xsFz7qvHpohGdD!&MCxuhn&imQcsA{Dicz8jTiWiK
z5k{RyZ{6eezf}|c_`$NLfBv1kZg~6ci;of$m$bgl+sz#OHZ81vQ}dLc2LATRPxtL!
zmw5Wq_L<+W{mwAc{y5{{7w`3tK6RfzdPk{!^{QhB&uag;Y@DB1dZY9BA{QmibF+(e
zub-JUEzU-ZtIOll!A08xJ{>E(>dIXCva>yM-&+nb--fH_b+_L=^X>cnH2+fBsrmaF
zjxQ;=x+ZglQ`S_~bpE$@)l2m+aYfCo4HYz5n!4O=*`ebVLU*LYUOc^EfB9RtfAjtw
zJ%vY8(|4`j)snT%FyY7w;~4H%&G36iSe%bunyWtF1vFMMNnb==S7ZK~x!+DMIX}aj
zTX93w+M5-pW-XiOwUjl}H`wsYWA5oyMxM#ZGuGEU6+0&zcBH%i-%S4Swa=u@&!_!4
z=FefpUUXeIJkjLc!>8B946}}1Ju}a)>+Add2lrf$PyF!VAkXWGvrDH;_Byh1<*Fxt
zR=-d9c4gw4m>)vx_kPq$KE~soX&4zI&~jKYq<qq`S<HL(muVa4@0jk%%oVXS$Fj5K
zQ04TvbL!IHj2>K?J=3&yipV6@blJPMXEtSRGDx3rYT>J{#d0gGy;rZ8@~k?*_ma+*
z-N{=mH`IMg0_VMn9=m+L-%6OVa>~h;n>lWQB0Pr`ggkG%XU*~wSK7kBRPLv*)+6T@
z*w{U>`Pb)z4^_h2xp&`nJM~gY{Xz2eYkyVVu*MkF8nynEQ}f^5uvC6c;{AtJZQoB=
zS1Brfc=rA2p_h|2PklG|`iJ{x!BaIYsip|)3+pXyEx+h-efT$9KXBt_d*-&jgn<1)
zTQYdfR&(|r??0ZZzSP?>Qps{^#w@q{n`(Yddph@n*~H4DrME&)fy#@C9x9Xa&U;o`
z{=8%2d0b>pVbju8T4lRsW3NZI+6u<5a<{7Ka*%i#G3TXGKi58|C3AB*@1Dq7!F$-F
zy*smT(_)Rak0zyE)=ZGl{b`-iC2+%*^-kS;=9tQpC0ksS^=-bD-;3MC$E^C6$Nl9y
zwlzX0BYPHYIl^xDPrgg5H3d?s33;}!kYvqGuP>-NquZ+Ju{20!^2wYNQ(TlBEo9Eg
z2|xO=H6wS|-CNbA$6l9OeOozEEUZ;w`4$<pRqLhfwWo_L_aw{y|8d;jB1YaRG;~&W
z`NwO!yKnBU=jXm(Ih*aUr)Qt|V%4(DUHvQ?TwB?<eYmtb|Dn#i<8y9`2fw+xV%<7-
z!<mM&A3S&$d&Run?{@-d?It+s3Uxj?GUvjVUnOtcML3FziWi3Jq`C%vsIZY}oA|Zh
z?f2iAJxi?CEIyfYSpLMpH7hmB*c)TyS1nTzh?t{ilgAty__3-eK*mpi?cX80nEJCa
zaW%Dyh_%zd#D3oV^pAZ_%*M8blhc#JMb`M~x0|njv|V0TNloX>zF%%tPk(0=?D|+$
z65@LB?Z?K`U6-e~n_fL5T>slvZE~0OcKPS?t=V=TK6|22xgzyp^dI8~k6zup{M0V>
zq`27L%?d|_I-hViF3U{c#RZP^nmmo$e#tUV`#M|~Tb$MW^-ag=dTCZxmcdM)8*j^$
zCUUF{;R4@XuwvDzb$>-$CcXT;_F^z+(?SWoe(5hP3p1FupKe{IrIM8RNV8z)oFj)6
zd5*SRp4g|pDB(gtyt{_b@mB>Cg+DCTY)Y|l(~vp-`18i%qp$xI-mBX8_igpYY)_N!
z$#-PzXK{VZXxYK1w|s7fMb8DD9dE)WedH)D<yGPR@QbM;rmRI^$)v(rvzj8__}%gQ
zz5Jz7=aZgGQ!P%lC0@94MWr>h^9gs?vdoIT_skYbMJqpsEVa#f-edh)=w|&T{%f;l
zWwkK0ANI45aSRlR(Yt=Jy-+=AZ`!5@-=i--yS|Ed|IultN4MRGjqscuZ5{Kj#8A>W
zH{_4yPnM#oi`8y1`G{H9t(J8S+-P-IMC{qak3Tg(+%kw-y7|W3o15n9nNP4w-(;ns
zQ?_7B%ZigR=To*#oSmK>%-;E=yKU8~3Y&T7<0iN$z1&u!rUFhzpIjgGaG&a8(u`FX
z>YU`EcJi#Y!bA^&?xO`eZ8aJ$KUKP-WU{<O*7fC%Te&iIoo4EN@3y~lusF+c;<!w%
zU_t3AHvcEHYmYrFnt74;@r^UG)92n5_HBRar4nDVByt%nmF7I{;XakNx%JpF1$Vof
zodUXXOK)H75nu`pEwqTOsCjoe%KdE4(c50mx{!rgkaAfsJtp9q32(dt|9boF-Y=*6
z+*o$GM+DlH08gc!J}4<Z<>495{owQi5^_<RDC50dV2Y~H$?1}g8&*zy7Qo`M-{SO}
zkK3(cSZd-zcxFkx?l{6*R8+ii#--o$zlO}Yb*n0<>Q;HfqO6L)b#u(JS3hF?YAPzT
zZq2cPGf%`t0w+9o51mr>lhNd?mtWuXf?(T+6<i{$tv#WgO{X6pf4yz7x^Re`rHik#
zvDohJOPh>r__D=s>p(s7@*HSlz@VgL@7l>aOGAB9J6%4px~WvYd};sriT8h}goGDt
zvz9H?FnF>@`{z&o{f(9(2mjZ6R}7Qu`gC~yUwPx>Us)IYI-n+3@k0HI`Ei5UXO{$N
zvh~e7Q~SNPH}R=zf=cAibJ~9%UR1ZdAaJm$@oe}3-|hDgmrjrMc-!cE=l=!&71>=$
zMzaOm`t<WH?Ek*}yXV{U{|OJ<v|1)QnTwc&G9Nu8y1lObciyZ6d)(`e2Y<frY(BG7
zebN_c(^mPq-_c*Rx#zr;n+lpi7WMV@WodTIoU1X>!>>btY5VQo>#uDSZyD<$hJKvv
zU$!|~K!$!&_9S*b;qG2!Sog{Ghj;tN^v_<;_I(f&*5BI~Il1k@$@h(Sd3KxWrr%V*
zy^>Gz&+EOSbIjN2i#hloTe%`P<;a%PJClBSe9MVDoa}GkU%>j~ZM}`l*<<`af9luy
z>eMUT|M9Q<#Puc{)9-?HpS$A&?_>)w%W`UENlTigifukG+qm#`MwtE2ot1OykIM4c
zt)JO{uWq9GwD(VL-F}~*Q7N~VjrY-szOR4&-n{<O^k>$-!pEv_V*cCJ|N8xT&!=x1
z`>NlIp158u)ag?6i-WCshL4&I-|?C2x?GfAK66nLY&!VhrlQ93Ak8;vt#YlEDNMSY
zjJxJ7+O@3j*0HD-!2{0<=Dl3yr84O}Yp>(tW5yXGZ*&4G{`K-pnEa~QS9J5pmrTL*
zo9w3xzQ~<k!*tO7wEcsCx1nE7mNGDzNHe?nB_HeIR8j9w`+jU{j%A3mja>T6{r`?0
zoH6tN```8{=UNUje0dvr#yemC_$;GK<uYg8{s&&3X<W_Kw$8cR)cf6*ZidE9`P}c<
z+8?Rhte5twL5N|Se!REN!r-VW?)~%jN?3jHYvY;UaWlMNnRPH|Not8kSJUB#2OfW%
z^KzAk%B0N=2?jTomZZl7h#hhW6wwLGmOg*UDdNO==I)&{1C%^XgpQ?^7b&ftb*=B9
zRbPM2vlk~+Uie(~4-IWG*d=m#XKDSsm39FlqOOdOZ&+MQ=K!C<l4r7iA4kWu2anf2
zcUe(oFTd{Afg7LSJ^ZA&;n)6uhfekGeo=g{z`@ZlNaoJI#{MwvH#fIT+bYd(nDMRk
zM##a5yQFed4@=lBFjo>yK78xymA1(|@|JV<#0a~|Yzdii^!(qc%Gq%TFO=4(Dpx+p
zHlO&|C|z4sK=<M6OHaN_9hu>1Tlx6o{f$*0vw5V=<`k>>B_3NaF=)09B&U921r>xG
zm(CSgeW=OZ7*S~PreaUpH~DpnLS_XPv2))!28y&MB<r^Ghfi`+Ro~yy6l0$i7GBFX
zJM)E!shNQHEBDu{S5?$9u1R}qa?Qoj-zDPzM|F{}@@^CFmohn;zRfrpaHitUeBagj
zEq=ky${Ss@E;H`96L-h&b|GtmYv77IyXPt^HYYl>B?b9C%+TUmwP?;1hIJfs*Gk(n
zrZX~Ei<hpRZW!;ft$t6JZu=}v4%5O7ehQBir`2z0TD^?lJ<!4Qkxun<OEdl1#}_)c
zYe?7dmESEfHRFD{;SDD<&xK^)KcA28-<;vhx^-(}-j3(?J032Q2?&*GWcc}$RicW^
z|3JH^?LTSD`oj9UZ>R5X%D9+hmV5WuuC3mYAFkbgv+ghF{r!)-@(PltLW|iNv0Zno
z^ul7j4<Da>a<&cM<hyMrQ<UVoA8uNC#mr0~IP{I|j6`j&H+Q!*sn4&Qc<LF8nDCMi
z>*+O5t@P%ZiA}Crvu&sA>|2j6n#HzF60O?wl1EuPMgFY50nfy}1+V{PHhMb-MpvI|
zjaboly#BX+&RYKm8)gYQB@|Zvd}YgTuqvF<O<OnWnUqt_HY@qsg$pK>sm$@4&SAdk
z#nnomU0bI-YLHfR2}{=g_y5~DYrig~>Ie7V&71T18PDlS#X<oF%4s5jhUtdsdmr5W
zt2g8Q6PG|kZMF~x2SLL|4z^8`E>ALR=d`Ze_51troR*`;a~GK3=Rf@JTuJEOx~q4C
z*T-G0{(VxA``YI_Q`lFSmp}jZTJK<*K=jijd&zIplmC|`H?qb3xL%vH!uzGdOpk=q
zI!~q;A2VL#AMO%$s(gy80BhNrxMP#JW^R(Zd(mr)-@1|)-Ov6BeE;0AefsLXeF7{^
zU#own+%)fzFlJkSwPVq$xv%%EWej+v6TOE+F`LEK`*T3*(+$ykKg?E?O#65#)kJkt
z&DWPY*J}7ULlfUlpC24&%eh)%frp#ltP<nreP<r{m*4e^czxvYw}u6Z*Pp7!Z0*qB
zclXqhi?uuc`S%wdxx{+s)A4#`sk0p`PAr-?@$tFG(~8|{@;7R2%YATr?$sW<w@l}D
z2AsdwtGzS%lj)k+-F*RXc;5UfahPe6d8wv8#C6%ib+ZyI+Ez7bWENEJiST7(IP|r;
zvi4XG+q@dHzq?cZA2>C6P3%^tH9x<+p7uL<$&u>;{!F@Z)AyFj2<3hf?l@Tyqj%)+
zkBip>ws!m5|COnZ<7fMRCh2S2ULM(7{kidVr_$yhS*b4D>{u)@|3LnfQ*WhjJPnty
zSzw;|b-Cpoi#I$^A8ac!^Q~k{-#<^{jM|zoZLx0F-FN#=K3Tj-uV;~jwaZ~Kmd0hj
zZ1?;*<n40i?#`dHHWx^Ri0M~!Z;!9+oYTbAHYM#sK)7>mt@5OE9${r>YI~<eSajHn
zI0p#*_@g*Sa%MzHKxkyI(#w<QP99ounMrHkFR83oQWquHK1oSmblGd=r-K|TJewA3
zSpH<q@Cb;Fbk&%3;NiDP4+|HbGCCO0vQXV8<;{e3CpLk?s9M&<?21sZX~i7AfJs-b
zUR`K<tKbIDk5}3CEoW{@m@w@-<`&}Vz2&68#;hZ6&mPLmpP4ct*iEBBDL8hkczm$L
z>ch@yZQL6p*Na+T+gufFX_(uww%R|fAX9ABs!LD43kHW=aS@qs-*WV!+Tq780Z*H^
ztv=z&bdbZjfBuOZ3zz<Qt^G6Mp3X9RJKo20%MCVUn|T>+oUfA?^sj7b?#tB<>a~hY
zOdmajMMbBou*%ByH@n0fdswhws)fUJx3wqE`DV}jeXd4krz_8^i>#Xi-Bn&J-n5D>
z^2)@ylY5WeD2!%cTB&-yQ{s2oRAHXuAJbAFZ<zHXF8k(X84=N?hnED_9yOb$7k&JN
zvEK7}ug@J-IJl}j{^|GNv<)Rs*^W(XT*P%_j%3Mu<EGE&Ej#W16pL-X7k|q4x$Nn{
z<8tTEL~>U?I#@5Tc81F|)g#jD_w}WmO1d#2uOrJ+W$~KWT@$81whlVDXU)wyCS^Q%
zlleTpZe6uPcGtG7!(9LVZ9o2^_RY>Y^}hn&-2K5Sx{t-#=(Azg<@;YUPfxJl#K4gM
z-P6S}<kFI#iAS14|73o0vAgl$c%In=AC(i@(kD!}-#ND9;a(2&Fr(~evoorD{|UAg
zHJQiPb1pnBX@52#?6J+)_4Wqev+d8@828Fm@-gh2Wj!~2nz_N#h5H|y$jF`FpK|5=
zqVLZp8b4t9C7SM_bmX+Z?V;nZn`25+1;2_$&-ix7$*X$G#>%rrV$)t*{%c8Iad*oU
z{c?B9pDoFT%hqm-Z_zz{@A=-M)_s33w-wDg-0z&Hv3l+Qr3T_plxE&q6tlhJ+m<L@
zx$V;N!ILVK8vWMim-n2wcQ*UR<H<8>-tPSJW>W8}Lz%*Q2NL3&;<Pu^-Zfgb-(qsd
z8dXXDV4DvT>vk2`T$P??`1I(`j9xdPhXzvJvvb<}R=u)nTBzaPGy6d6a(VZkyUtf8
z9*Z_<o4jJvsR_w`Pyc*OzPY<jGI-wX-c?tVwxt*U3>Dv0_0UIb{moWK?MYM4%&C5P
zmFL7&r7o#E^K?Fa{v7;r0}G>PK*X<)PHDO~Pi-~7*R!q8*5Bqo-|~2ynK^~>m*amJ
z8qGgksm_!5tvfg}T+mra=!lb|&OM=ZJAQc;@iMJ6UCFd+(u2*dn=7jSEkCiSLu;nR
z+d179bt&=zNo#&(v9{Ls9S~XnaJA@bi3_Iey)zzsZrvQQ;g57rRi}rETg--ndrQNO
z7R{a>sqgo6_PJZ8Tpf$p^3(WSTUj+&Sw)^zRKCdnzPaXS)~{(55!(vH^53eT?sWf_
z`z~F8^~Sz$#q;j3*FR_d*`>upcD?;~lhfw!j;5F`yzH&AH0Rx(zgbSZ!;StPHIFWh
zG<p%C)+c-7Po?~&3jv2;>+f$Vc)9h<S&vU|zC6Dfc9TKl$uz#FU8m2fUe0@ypyQo1
z>8bbq)X!QIJro+0*Nb0|E4SR~SA1*pAK?p<XH%Z0ByL_lk8!HC*+i4wmd{!qpE!St
zA>l;f@r&Q@x=zXeJ1r%$e$ytS4YPvJHyb9a7d=tW{#kW0XqBJcq&;@-AKLHVKRYEo
z?!ioTIY;GF5qEyxDZ5@ObJ2MAu1{W@jOV<G`K)<<U3Al`Q#Xv+404WiggM{LDV;EN
zYNzM*Z#&+Ue(8+aR4Tpx{Ot4d!*x{V`2>|*Ss7zdD6{K}+EbIvOMe`$q%hSh9I7<`
z+sfVl);syDUR=!^#=M>1tk$eQ^I*lMeKwzdh}gt>F>;t*3*48k-<7gt!wP}fHyt)C
zoVQ}pq9e{%uKX;lC_Zuan*x`Xr)B4jIc>pHg+t!m_4RdT*nHSK&{3~2p1GHE>Kh;a
zNemk&^h?E0S+j;`OVDX&U&ROcDK`&o-<khPVA<cVX=kD)Mrb(1?p<G68eUtf`de%D
zL>{rOE-pjqk|T%eUNHr-Si0rT?eDq6B;(c?`*imNQ?;({E`yh*3szmZ9<pKkBDTFp
zJp%(7+1~Qku2^;D-Mv%N`g<JS&zSD?okPj=_K&i=W^MhuXDx33uDs*7_9Rb9*Q-T;
zZcaBeU{g~(w8-`S$4|2u?(QmG?q0|j<+)(vwyv&MRbSGqe^%~%!LQc8>e1@e%NFh8
zTYJ(gE|1S`uUN>`W4p3eWxbmCMeWn4PX>}(jwmxqH3hP$I@UTfl!xh=*}l2Go8L8A
z#onHOvESvGe-4t59x{CW$I&EpT)rnLAg0I7H(+DA{y`6>ayPM=_nNDJhFUsa-M9A&
zyVH%kGoPeYcCt!ma=4v5amq!T^T(^~j)lsH->L}Zq}<=9CA7*VEKg2vHm9oO@^$MN
zJw;ZmUVl8{^MuD6gCaNfFI(Z{S7*o9Jy9)QIndR$w|!+kPf}^>q`4QBX7^l@Vs`oa
z@0Z`?XW|bgNS|U?;NV!)A=4|E|7G#U8G>6i4OKmzCb>*GQuo`Fb?UKKR%d3milpoe
z5RnXJ-Fx(OX8PRLhC{Q~n_c*SVzPu~jX<dM^pAI>y;s;gW${q?_<Ql3(r-_{oWE%B
z^`Se5g?rKKyWyL&9=_i5|EiXs%_9EK&(Ao{x9_M{lQ}<c>->+8qCda1&QK_`?A)(;
z>g4nI0~ejE7Yi$QT$9n;^(v&x)+%o9KNan;H4FXk8C*%3a`gS#l=riy^oY3|rq8$S
zYdqYacJq05#ow!v8BM!CPT@^-=(pLnPVaxAQT@(Ilij|kKFL?mvEEVlDP>KO?B#hC
zlUU{J&K*_f_gd*IlCkE-sX064jVkU`%Ght|Isa_`u{}3Gzqr3##CuWK*4bsoKhCT^
z!L{!3S5@tmvgWg7+S@92be@`JF#T?UEi>2JJ%8^^J~QvLob7u1{t53D-p-qp6&@-g
z;_4a}k~2@H>WfBFcCu;c?(IhD_u6^vRxZ|I{x#Lie+I`9=iW23ZFf)JC()31-pKCv
zvZmtobA0YIX>Xoi-}Uq<uSS=T!7Lkx=?8_67hJokzVY|WpD`szMR|12CLCPtzsY*Z
z<6XDkJKo;XzI<LyXTSaXvz5zvSa+5R+O3jtPcJF@`Q)^K)OO>qC#OelPL=<8`1FL!
zZJ+wp_Ftc9W4+g2En6$4{^f<q72j?e-k9_5TJjuA+aJ#-pI_|3mXpTp?_VvmShaMI
z?PhI$pUSQRo|rUV4y9&)o95$fJxg{ilamzOZa$f3JF_E0eqH{&Iscs6ZQd<fa`<Uh
zWTj%klbzEq{@?O>!|q!t7ov@KzBqQ@xXekl_n?4O^UO7Ga&pxE*4v#l$-K1WC5z``
z)%^=v*2V62y1uSN|K##dYq)0^S93b&NgVzh{I6d)zk15Qa~kD$D%h=GF`qaoF!S!Q
zk3SZdZ~k%9y)x%p`;510HH`{}y_4<jK4_%#=;vMUIyEPJ*{26rreCy*VaZ+;Q+a;A
zgu>fL{Q9*AFQ5NATQ5#ne*Y(l^EL0DpHzQ$JX`Sg)N2=4Ul+Kx=dX@FkHo1`VfU?h
z;X<wx&&Hp5{OrE1L+~L7qY(f5m-U`6ydQsl-+J+akMHG<rfd|*UE!X7dXdTbQ{U!>
zUb%L)@Lg~Hlk)3xjxT%qdB^X2>^DNiodQ&j#n*9{$C<xmuPnb>(eZP0Tykus>FxLm
znP{7qsIQjV+utRAKJ!w-gljop(K5CBsuR!7-*?(w)Sxaz(pX$?=^njP&iuB={{MZ;
zuTrl2{Gg_P@YDXqr?igGE7iOnQ<Ap)Q%>YpM*ct9R=j}|9tT#*m#US=)pM4<wtw{g
zf6c4_CwC*MB{^pz|GaDSUJ><Iu-x{OQq|;{5e5zCra3lVKQPbsstMaHiRyzXc{?7M
zY+L)Iq5Z$@-e#W3`jY40naUSFuRY)7*M2qb#QT3G#`E`E_Lke-?s8QZx_ooTx4!$u
zG5;qqPgPO>@nEg^%zvM(kGKYG<<r~!kMCDh_RjAI(`OW%-Yaw6^?3Zh58=D!RwxR)
zi+y|$|Nq#sR{~P1jkdeR>wlcSTk(YJU&-UUCR|ldDta`ks!FsI-p{+sA|<6MVbS;d
zUkPvb{hB^W`I^R)YgBKDsI4nB`1$R=VOhcUu!$^;+iLzY^1m%#$9}ndmdSpW_LC;Q
z$5YpyOb`y0uRV6meO}6KW8(m!PQK%1S#2CnpC|cRT~X~);!xUI$(phFtImwyeDl}G
z_y<jkc6yUkDr&SO-~SVL*NM|7mn_NF=lc9v<8@SzMDOF(>(idUtNOz`Rj2B`?X{e{
z&N^~S7nw0MT#Y@m;$icNFIF2PWJI2)Z74jNerE3e3ALq+E}^Z@YWJG$ZtmgHmz0#;
zVO!=Rc;;P<+8pb7vD$X2=?4y2rq5}KT(0=s{gjSNvWk#r$i)K-9=c!NzNSOZ_E+(f
z1#wrdg$ej-Y1VI2xDc?jyDdOC)Y|;Ry=v~8(c%hHD-;DfHx(b2DhvP3aqaCgErBJ=
zlDO~Z-M+p{eP{H3#vZ9V?Qizz$&}y8NaEG(Hw$|H_LvY?$)`2JjDb1RmcDd|{IgJ_
zX~HCPku~YZ=l%VqB)D|ZvPRq8-zEiLYZq*;nmjlB&`0&2gJ~BRuTt7s*~<PRXoATN
z88ee8UA^hjb7v)-pI4N0k>Rv`<dO9Fb*h@jSFT>GvXbj}Y;)Bw9-Ah)Snd~5*V->J
z@qe7*Gi!dETe`yG*GH2)DvQ6WX~*TWRry9Auy&kMu*0~(GACn;)t)$gvBm42I<Mg~
zpE>iLnSO`DgM##cYm3=!tEWtSS-8dR{B#qmJ@<GvZQ2wSFBvB;wQ}u#UNP-^Jv=3|
z`W(Njtey9!NvZ1FE7$FMu9?b~zOzj1=aw}~WS1E#w|!r-E6etxi%@+s@5UvEwyhWc
zn3~#SZ}&;$@^af_4MEbr8#y*QC`W&5vH1UN|Ho(9`o%HpPEFDDf5@#KJo#)BLl%$g
z)uOL&zZ(}H<5_24t?*e~=fKt7_E*-Y-n{iz;QQu;kK#|hzUsey>Ee8)(@gGjs|?NK
zN`?G=e{!!byRzjNE91;#4s$j>k2$Hme*dZF?{UX>?R*~?C+py+vSMYA!N*(GGe5q0
zJmXb)fO^T}pV#j_`1ZN~`QFy~0fC_>x76FW?R>9!W}fx*_IJA`{JgyU)6r#n?LAtX
z&K-Iwte@tw-R9iRPho1l7Bh|2<IYTYSLw6k6W=<kbeX7yxjK^FESX<^o_uTD#qDo9
zYg66gABW}NDclx#di42x_xoQnmibjrvR-E~xBq@c=hWL;{29iQs~%5XKHv3(>^bHC
z^G?mr+dF4fY4^=t>t{K*q+RiR{_(^V_npO`cIN!~;r#6y>x9XiuU;$l%r0T>|MNJm
zr1W;?f)4g&K2I+d9{%xUa%xCexZ&fY;(SvQt4+O(Je&1T?I>RV_#pc~k8PpN8QPDw
z+5emBF28qv9N*0IcK;@wbDDbX{mld1`umQ~_^kf<$rg2s#~n3sbN(6qTI4v#U`@mI
zWws};9A6%|I<4T8p|Yo$Y0#Bwk^24r1?7I6o~u#%I&xo~zJ}+k?AI>}pPclmd?VQQ
zBs*g7(s{T4e_{B3ua0wfc<rHYr>Y(;WU$tDnp1T$*QfSNVq5wNoBxNhe+X^qa~JmI
z-Sg)V_kHjCnV0?q$<)cEin6R_dw*i#4z};fclYuK7d*4ziM;n-L;1m}GYT=EO`lFJ
zO_}m^(JBG%tDj#!KYjA;t=am|&L+54HYvOFAI_bAW6$1oTPwUT=!OOcX7c$gUD9O1
znf0o;S7qYNnZ9v#YF(4O9X<W4L@PV<@3&02wS|l0&Fwu+>G3l>Z%z=LBKkC7Z+i6X
z%=H(IQ@&W7le?__^k7i%>4hPplRio`DNOJ=)^Q|RgLC3f<M_m@@1FEelMHqBJEePb
zWuT~PYQeV4rVT1HPn<fZdO1-^UA<eb`OJ>5M(@`|v>to4py)T-sw*mHbB~-_A8uHc
z5U3S8`HoPG&eZQwp;7xftn;@oO>`8Qt!@zVXX@`88#d0_BiMW~`-xwQ;lxDapyk`<
zZV$8BpkF1PzM*ini;0w~qBcj9i-n9^n8=lij`I68ODF&Hf4P2Z5~IVRw9k{`C$uln
zNJ)J@Ir<XwM#~f7w^h{_U46B%a_0mU&$)g~OFSp5f%2;u>(;y9w>7MCDsz;KyqI43
zVtxIx`1y5n3O;3uRLbZ~6m^p=w@H_&W)b7{+4V$8%yvUVTY}10*)^*aBwk!yub+SJ
zXT$MpdTIBU`Kt8To;>mT;6`Sz$UhI&Wo8<8e6?P5r}p{_zdf2-s}?-|qn)4l^Z4W+
zm;YYgu_1ob&(-lumQ?$Pt*!3P?Dto^wPAXO3Zs|SyO!lk<dfZbm)?+CnEl=;$Jsq_
z&TQX;6AY8)>n<yMpJ%q_UZa-C$4{Hj3w^vU)X#P!^5%()dq3AtxR+C=p4QB_{BrZn
zCGMLn4PTd7%*{LFX8ZMRjNaPlKP6Ff__W;?JGEVR-czSM+dNc>bN9@z$6mWM^B#GW
zSQh$Q;;QX(=W5?x(>JR)j6Zf6Yo0vmbk?f3{;PEF!6fws?@#^W<*xktRJ`+tR&XCr
zsB1u|%+0%!woB|;pJ^P8?b`4~&c(4z%(r%za?M9m-q7DS_N?S;{F}1k(?hqT6K{75
z_t`b+^mD38F03vzJ8fS(eZ9y`S<&7p(@N|b<9C-!haP08He37P>ndN9u%8^}KRz61
zIk)qL*1la2tL{apd^~VP^NBUDi)rhV3bRl5Zm(aYqGZzc_`)3BBPqoOMPDX9c_JDV
z72&e>u2=sirle^Md4E^4C<%syrXDyEIL9DMK#X@z;kB1K^^b)A_+ES_%6Z{R#OY_g
zQ#<ZEL$aNk+|xb}He0LZvdLj=i~PEk)c1d`2n{`<{n~w__k*v~;}Y*alMQ)u_v9Y)
z^beoTu$q{-ez<BrX>R<niHF%0j~#AV{PEc)+2V(*&fdHKY=3|V>*lk*(`(fKCG@Pb
z{XOZm*{1Ar%OdY9Pt)}En;cXgDC=cQn$@t}{j~RobnV*Bm0fd|B;{L5o9BV<mdd%c
zYwofcJ{z{pF~2v*y+7G+;d=47>SR6Mg9iWF<kdIddJ`$$`T6T7*P5Ma%&)_B(?4E+
zee#s%fd!&b%xXJ}y9;NjF3Na$d%eD!>8W*lOMk2?+gRppx_nMOtNFbGTk|^=#^Q4e
zO>SRbpOhcV5IE_^wklS6-Jg@@MqfAn|MzIbCZi2Ac{yG=-n!bMQ8a1RDchX8+on#9
zuRXR!eOchd4aP-37>tU#3*M-E|D1R0*_`vbwGS7n+%I*AS7GJ7TQ&V+#6RBr^Oe(|
z&aXXlrc(ZrNN~VYm3_b3{7P3|l?+Hq+?J=Wz9>smOLLi~h{ECp+NrUhR@~V;d)W%F
zBjE>Muity@ly7zL=I2?nC$8Q0&9Ula=`2u#{U_&^*%Nr*+&#BR-+V&o1^&IuRFj*1
z;_hFx`T1z`oa#e@SMvULNw1rCX8HYhjzX?eTJ9Ltdk6U~(8%3<rAb35SL1p9_PeJ}
z%`Khs^^NcLJ2on=ju&QIew>ngW8b&zXM4W!^=+!-(X;k=pdWhh$%&`$_TP1W@Y=lM
z%hH=iLO2yK9uemMJ6HT(q;J`DqpI(k&Wp463oX}GZd<kO<Wi*uu0;w9uD^H@c5AL(
z?Jt9jBN^}h-JUOb_3?2Qh2Yt9MVyZB|9|-UmQ`9Sa+4NKHW7IyesODdc-0r_Ggr-h
z(ju;}@!RF=pz!ohMjoGQ*o0)qG@HNqY4tyN=iPDIHBD^MMUS^nGRztMt^1ue`?0i~
zOVv1=cTZUF!X)<8@8;e=k2_z^?0Eg+(psMN*Xu4Mb}BS1*%%mcW$L<EZwZr67HaY3
znwEaTofVTTR@$CvbYN*%TAcpj%IVD)Z`zb->mFToxiloTTe+nyS#j&){|>q3xu=Sl
zLi6h1o&9yK#=dsB=V$Li38_<BS-bx>z50G4fAuP@v~WK0*g8qQt<zTiubEJP=exrE
zHTRCb6jnD};V#}LQ>kKVJ-zULE&rw@Z(d8unHegAb1$}>e(W%XMZh)3PhG8>QS`zq
zUkh749y9ahJ{ID+kHzG6{+GA;dg->tG##TiX<ODlbq#S<+`UD9@9v$IslP;)?X}=u
zd$RX(lG_B68*kIX>P%*91~2<?WB>oN!RKwxuX|N>t#0p>35t_9AD@3VQuv%o+|I_W
zw_i(`rJd=YeDk%-r4EHx7rMDN$JKPz+yCWycBpYq%^i;|lh}5jZuL`sc;?k1%g6p{
zpErH~dHnwWgKzk!O^lt<zP^ZGEuYi5s@!q?^~KkvUl>+s>@3r`AKcr}aCT+-&qdoS
zKRm1t7Tj*|aZPZcgz1V!+dh|>YRcC9y_h3cTf%kT?!S~^vf1@J1-iD}cPxy&`6j4p
z+w)J3>%NzCG;i|)V{41Ab2{0-v>wk|wBftp9Q%JRWozP2{c)W?d1ide(Iu`XGbF37
zykIWX?7Csg`sMwsle_cx9=kp}TcY^U%P;3HeLj=i|KY?G^_5(jPoAd=a9`V)bW{Az
z$_$3ULrLWchSE{fuSCBs^ErFE^)jTglkZYezY)A|YOl2Uhnwo$J6`^JePc>N!8ggp
z6Q?}$5mj3A=Yj09vp*XC9iM+PG~CZg|9a3}35Q84cGd4|<K}JdSSua&=}fTxgo*Pz
zZ%^ZLTglW}FyYoLgT=?@9eknuD<FRFY!4^5zy;4IB%kcff3|S?^P(dc9~VejNh+D$
zcH|1MdSQ{bw@T=Bc)o&2c;F<?SF2xZytVB5sNSb0w_>@XNT%)$CFgF%t71QlOh4v)
zOu4+dp0RpWO^Lk46sPr#q1j;*{|K($s?y8nsk|}wFxwyBi`Ty0%D<OA_f6jP%!3JA
zFGlR_&%Ir~aC_y8xM`lb&K;ZnHEVp4y0N8FxJdotMxAv!yTq(#{ydajv%6INwB3x8
zz2^7My?Q&<<l4313#AO|vc^J6j7$?VPVwcLKis!pCTsKJV~@_fh-sT<vgqxp_P<|U
zgxUJ6-?3DQSFc>T@@ax{M!uuD(OM?ot`iFsoek7x-`KKONcG;4?DM|LGcBi_R_94S
z_GM+x>Z}C|m&xs2-SF7oigSJB!gUAlSJ`T;x|H>!+-K^_ZF!v{s-0K8#f;M*3YF#k
zZMz;<&t6?;=C_*BH8en!b+vKxySw}5@4LL@=TeJ3)8bZZ9b9Y8$T9VOX6+o-?eC6E
znatW$aN(pwXzQO%jdyCoIYUjio><j!_u;X}WeQvdZ(anyyq3>$aYxhDG}g#t8HUZZ
zD%+>=9L&0?$YG@xV<qXB?ck?&^1bn-Xz%u<%`YbG;(xSlw|B-D_KPoicHGr;zI>tB
zrMS|zYf-}IXT@K8w_kkWEt_tZ5$MY6wz$tb|EKN7mODD@yEMFRyfJ%pZt_c0ukO1^
zOve-6{fpIHbg$*+g{hb94%cruvwbSx;U%7vCa|*gN*w?3xJsx?rC^6mgm0C|%@a#j
z=R2iuQdpe$!caZ<R=&l-0-nh?yDn#{3-T5oKd^XXbnn^C)3n0!InD2vv0E)}Y|R#W
zQuhA5srWo!C81+E#Sb>!o^ZJLMAwbV-CY|)*>!i$J{u;TwwTW~FtGJp>coH-wNEU(
zQbp%32)a;fu>J1crOxp$wo1MFma}}5;u+27F%I*tA3Sxm_~s7%PPf(RKQ1m0xb?k*
zxn$*37rkBo%+B!~H;8^SgC|OF|G%(TucQN9dHI%GtHy7#uK#g*{>9n48mo@1e5}5(
z%St5D?Mk%8>!#HczNkHUZ@3ZMz|6d~#r?{Q!W(P%*nRz9@jX&@lg`Dh?-iEat2RFD
zW!&*wt047h(v_Dd^z@dL{W6ZJdT8Czb!Ne1{bx7+#YX&Sbn@%#vpjEi<nmO}8=Jj<
z?BDNmEbmHEd6mE#wl$GQ=h%I8nOF5e?w*<TiIWBsqh~aBbX^QSutNFw<|Rq8$K`An
z7;6^p*y9lvy?J>+(w)}u!qZ)%HQF~X&dTYLt+)FA_jTs6`Crv`?zAk?*K}Atae`0S
zrXt%66Q&NKb3Ht3{Dc{-7Ee7BW6tKzcKu!<|Mx%d@_nT2<xeI&I$^A@ays?*!CzOR
z=JZ%(vH0)*Xk={5r)RU(e%_rLowGN2OstgW@rX}SN!VanXc;{3!KAvwCu@HT%hmoc
z{P*p_{2d=&y{`B$-@o8CulGb1-oO3U%<J#A%<<V4f1-WG_Tq!4<{!?t%RinMo$ut5
zbm0c)mzQBqf0j(1@$HY#y8Zu>@5qU(R$6ALa-PgNID76S#=s*FKPGH4e|;q3;OwV|
z622R~T$Nq<Vg6;S6B8bq1g9HqpUPfT&6E72q5tRe`|l1t`jD`t;^Ph76RWzE%43?|
zo_TrBG(7IWWc_`|o_~&aUTs|bto3@p?p#*y?Pi5P-)=WB2~YigZNI_$!}=CqZwUV=
z_!{zV-^(hSICDwW))~A}E}ja9CtsW;dMKy#!LcR}ErVspUmsm_J!pNCy7T$=eBbA1
z+t_;6+2rlY6f)zLSmQT`Pjc?flS@{seZH{YYF_rT!$m9d&zt-|%rW~L8~@9yiykUX
zk&~Bf(U1<A>au{-Qp)x693?HuJ_Sjhj9f7RmTl9*g}ky1q<ZIGyDn5K_-+2pMTg}s
zPdo3JaW8uDHtS`o8aEo}Z)Mq(wR-zw!`#v(uQKmn`}g+!|GIReKN>S#87+kpjurbS
z>e^1*e2g#a?*2Rb&ipefw=j3ixXZqGf1Ru2!^4rrHzW)SkG7^KPso_78Mf%){~Pb>
zq%0CL1D03a^Pl&(Uy!A7SzE1*d9GA+ma>J9lvGn-pLMd*a#>p@&j9{Kr^}4Co+;ZN
z<9ycp)omAnsQ#BLKKIT4du?Ls%!C8(|MPafC~dzU6Fti$laK$=M~mCr^Ot*1*LzX&
z@~YW?&d`Y;O|M)O>0GXMNmW;G`|i@ElUAMhQQIrAhL0zT_n`yxRMS!pDdw9uZ%TZX
zirbNHBySqE<G+5n(6`?kcJHmY%f}M(r_Mg3=f>G8p^Y3<r-o`Y@%10~vssw1edfo|
z_fHO5=cTSMRyl7|CGPf;LG$D9niY%W+RtA3m1prleNITBr7TBQ)2vlmvle;U$mwS(
zHQr2g_c)f3(e<dv>H>$BUA*$5drfDTs0(smmgss|W--Bq!|Y06=>3js8LHOC%zMLr
z_5ZrPJzqX-wdm}%XIJ<gS!81J%V0+C4IQ>7MtiAd*2!_*b9x?fA1k=A_QXbmiA%c9
zW;q6gGW~N?GGv{8yfsXS<CpY`><^1gXPi89CgFPce8caTQ&n6HP4xm(WK@}FlpN#i
z2$VPdT4!RVe2+)JYHiCZl@+Vks>)0*w3s`=Cu!47fhQLK0(jc*wyCUOVX9-9E4ez*
z-AL@9eBW|A;g<@{*B?8spXzZ;`G>Ra@g>I$Cr+5Zt4dyKFYD|`|23K?j=w*X7M&~*
z=+hf4(mT;`xlckxq}W%}t3PHd&B!g=lPBKk(U1^v?4{RN*Wd&0F%t`JtYwjY{eksR
z#2da6sR<=&JyMMO&-Y(^=CFQh2MhC&P{!G3yR_6l{ARilW0q$3DSeu0)Q;!Rv^z~f
zXEE%UyOMKy#ga~sNmKp>TH9VX$*l8ZYGgQ=FyYRf|9jRxo*;edVg`$>A=jq6YuTcf
zn@BtF)_&c&_;`ZBzYT5wt_Or3dMME3HviP)@5aL2EuJfw(k5?A=2`u}X!reA1{Wts
zEv?lZ$5M0r%C*!}s&1Y;-=%Y!<&WL3lCwt^Twbz0ZMo;<Wz0<q0{zD|_4WOwPwkh`
zdT;YlA^z8zJJsh?c<eu9>&vZqblyDv;3Qk6{}oRR`(6cCtnh!qesSi)Klc~az1Q3J
z=2o-QZI#_8r~1C%q$iVSHdm&9)63cU89m2tAIn<Q_Gr_+CtH>|{>fjw;la|(HQ%Cl
zeErp3|6%J)wX^5Xu0ItpEp=wY`gsK_IG@_<M2or#PQKZ6_MxAvq~GVcf5P|9VgG3H
z|N8B}a?kJFb6&G#?T-1fk<&h!UYVRJY`rAV3slR9r9`U9@LgUTy?x8VKiq=6?H$@o
zVcjp;)e{4rs$3QQeIP;g@?)zN=N}b3oo+Ja@}GjFi3Xgm2Xf4o3AZe<eDX)Y`R9c7
zv!|$Vwx904`|@Yp@p6NfOEv1R>Mwiz;dB16|JJ+j7SEk;{`=3B<z3=$TC2azbMi9>
z5m!(9?)pR?HRCq<$gl|}DNq0J+H>M<VCd8Wi<%WjJ&Fxx&DB2W@F2i@k{a*Bf*;lO
z_OeCJl9!I<&obHXR_9y(=3n)v%S${z-&g;;ciyw<hqAT`@vw2SHMd7iG0D6n^K8Yc
zuB%yp<BrRTwH1Lnfq%Ap9DDp@wN}Bir^;O~b)_>eE%E#ueK>D>^7h;OTO~3tE&177
z81W=9Ok}2u(#3~UeboLxo(Gy>Uh-3q#Zf`9dt>!Sr&-_LItuW0yT^e{gfxr~&vCZP
znvy!xWeKO|&5uiu{b~7AdSubW`S(0#nPmP`4`iHsv*+5oC1p329$n<HH}mRM;P9B#
zqS2+mv2uQcLf6{;%YQx<WKo>@O~n^#=(F@E6ZH4`sZEwBWnztti#p`IQtL#~q*=GB
zUe%PY=;CgEIh`wczMAi>`&^!p%RWx_H<%V45WHP9`+8gokKE6_r@9v1(DOI{@k%RV
zYmw}<GfGt=)i?HRZi}<^zPb0C_o>+`Zt|8q&2CNtAtI`K<$q0g5MX)r?cIV^o=dB@
z>v=duy$6L2C~3@ZcYenf7|QA>aN*_G7bPpF$E`n}VKVVy>Ey>-7fu&)IkfQo)b*hm
z-@bG#s<^=_l*t)&?O*8%kC)HePu6ObDsW`3kDVporob`f{jKIT8Xg-hc%`KOofp3l
z68-t^^|=ZQRA&8nbJ#LxWrl;0>x9K;=Kd7Q-@ku~gG$r-GV%3i{&Cc+`29S#Rp;+7
zX`wEMtKUR*qBiyHP};&~on5W=_j&f-zZ;#Fzwdr~FMoG?)co@Q8*AFjZ%-A!b?-Oh
z`Sa6$Z~A)U@4J`ZzW+El?=1TqW*G(N-_Ko-KV~Ug8-1p|+g^3$vU5M%Us%I~Jwc|Z
z-$-k1^Xk9sX;yRPCZFs$yDs(mzNkZrtdnmRU0|G}z`;^m`sQ}HtB_#1o7qe`b)|N-
z<@xQGLsI_ib6do@(>E^g$h(4t|6Pr*aG&^Su=@HM)!otZo90gYFTfk2`r`XSj;Ne#
z^PgP}50$K6+@E1{a}k41n046Zht}U0b2m9$J$E+5MDxqbtFmX$zcdwMadGr`-~NBC
zB8SUiMy0RO8~Ej1dJns<3|sQ>>Tv_T{{oLs3J0?lq_5E8ow_^1AYJTklH8^>e>;Dc
zZLU4-$0k>*=lpBR#gMJ@T#PywdG%Oi+Soo$P&oGK>3pMn=Z>aBdLqv&E@Zj<;oW-W
z+V2aJYg5uw7hL6->-US3iAC!5ij~<e*#;A*PVJ0a%@}(0euR50WPMtg8r$4y6<av#
zHTo8)eE98bu}3oED|CI@sdI-GOmtWvEO{j4X9CX^7T<*yH!F97)})2{TFg4|@Y|!p
zTMI8QU(S#<Y1gS*lQyLY**v2?dXIeT{w%b}>zD4;5cqI)a)9X$nX*NS_E$|yRYEw|
zn+xy0+A+uHLG~BLPNrTvd#lxxd6u`{TBW7L$(kP~W;S_Vzybf5z`)6dHUHFf_{`(B
zFW$L-l1A69PoYikt6zcQcYjfcq^YFx$`$*6R>z2%oLJNsAHJ{CB$F>+g|XyP`TG;r
zU)nyEFSwX-A;aXzqCg$CiB|E-fq_SEoHWXDY?ymr@A=W`=|Q3$i!VC_nG{`mDJ*GS
zWgxY2ak+Eo)GN)QFJAo2G?a?&51w)@F)8|InYZ?L?fFJKSX$Eh{k&^0E~}dN|M}mX
zoQ~v6*@2;=bB~^54UawFu*hdh+0XjuTLD++s}<jFX=+f|a#D!@=Z}JOvwL<}^vg`&
zTA8`>MwQR+Ur7h6=FR@Z@b%fdJ9T$eZ}23oj(MYg_+QbR`*}JS4sie9)z4e1DXSvP
znKQF>PSLT7wy!U{JiLGQZQ@vTpzrGGh#<w_Ym2R;I?Lv*{yS}_ne@Ibmustc-g?d9
zs1UJ#y8pD=!ySgnAEJ6SR^8fpj(^9!mNNU$5ZBzPnsZP83{c!+`!>Ym%G$(=^{Q_)
z?mO4rJ6U10C*(<)`j$V=HFwz`X}-Q$CA-8b()a2_p3g5Av97rvdAcuBKe^|4;Op=W
zWxwUNnVoe}*6Mlr)IlIf+xXP=+Ee0F3+_A!dRD)y>r<P?<@vQ!XXo#lm2x#}*?xn`
zQuQJt69d)f9<^L9+w3T0HDi90!iMa_&u#u5lKzpba6Gw0x{PW1{BpK^Z%<c#+G*aY
z^7X_<23Pl<r>{PrTafqpy3LO}Iy`R59}^hXp7iWH?vOf_zh2?M3}dyzdn=D`O8*-)
zuX5vJp6BxlZKlrN|EzKOJ(mj^1~ZFh)*NnK6Zb_*zGkQN>^Oy0rp51`PL{l>{Jml6
zlcq@zKJ!2ObnFd(2dCDedna!`k9VB8cAlpH{-Za%)s2<nPj9KWb-XxnxnWs?+<Bkb
z$G-jIh;lvlNGtV|@AStW*9_jS=YKevM@6_$scD*CtL2rj&`VD~PrmtFB5!vY(`xZO
zXXm758_f9H;*|JkQtRjSJEmQ%-L<7p+BGoY`L9&=IoavI{3ffNT`#}%ne>yX=Y{%?
zc>J68d{=0zRkpp^lQ_2^g9&xBmU@2voN&QuV(^kP%0&kApC;yNbT8_-nsqd3W5V{^
zhBIqIuCxY<B)6yTXi~TkvcG+Ld_`x+PHXe{VlltwQ;%5lKYr8JpBNl)V8SIHsaIJm
zazCC*Nl(6%-Pzjw!`YBUweh_Cp_vmqU9UfzO2k4`&GkO1q6^d3`?MWz)jG}MEZ8>B
zVB$vST}r397A-mWw)Db{0B@hG4+^auJcGi++plFDPBeb7di`HPqx9qp96zL_166e|
zs@X5h5ZZ3ey?k%#Bu78Vg$!LjrB~x>d=uC3@z0dhJCe3H>`lb^SH4e=Y*F0y&UWU(
zrI9OEc}>3W?h_a&99Z*DK&*W>PrtC^RQ_A*l+-y{lzD9Hl#<)0JzBs1Npt1q8UKEm
z^pxK_cwpU4jgv>}Htq0?-OaV?)*oMu3!d>E4GuP!TwGmMJssPRKUNT&UT7hcp~`n*
zX`uF|;>%Ib_8H5&=Lt4fl^o2uBFOn`#hO))Vx=k1Q<4o|SxfC(<&*5Jyh%au)9(Nm
z+nYDzoO(;I{C2ju6Srqw$*it;F-FZO=7m`-d*^xBGzG>qX|xK=a(W_arg&`jiC57c
zM~mf^Yd=VteYce8sR|IiP(49ad_mK(6C2g%9&6*B{i5tr>Kvn0@)O%+%yRFzY!}vl
zye07OhquwQrB-n;bw&s`F7Qg5{%=m@AA@K49ELw{d4G7Z*v9g8ZpqQZKR*9>yy8b0
z=lSz9eCIRPN=8hYI4dS|ow8(J%DdV5N@@HL-gz#6bbo7K`2Dy8?`8#h2(}(v?4Nez
zgW{FExO12Fb(dzn`Pfk;6kpp@yl&?t#hXI^_x*RQnRoHdzum7RcYi2-wl?DMN9Er}
z<>%Hgr@S!FU@(h3_PoFF*->%Bw19G+AfAg;4vSu&@7NiBKHJ~!<f^CljrO?PF})2t
z#G%w1YF$x%jj3&JrQvbjSEnw0Pc!;`cITgu{NaX02ZHX^el0B%%g6|bjj_oQ@ceI9
zw43kQmTvcio~s|8Y)Su-<}T{~boCEush381O6juJGa@hlKYQJ}=1$%7w<o4%S3a6q
zelxn-rn5k}I!h_0_O|Jl^XA6CXRbUm>#yG5U9r=v+P1zZ`1bum`7D#ny001z&bhqX
zWalSMKHTt2+C+gh#L*!@)Twal1Zz*HteIDQC##+HSCwdPE6A$3tR4KOWbZLW^C?%a
z{kK|vbocVBSs!=Y{V{9O5n;}fs@mPjF5mLv4$I9y@+ABG()8&$A)<b>3mDmc3w?=d
zeEWKu;oqY(T~c&UR?K+Rc)Z|h>*JO8HuPxfpL5MNnZ(h4G2%{&bxBC*bwM6i%cMIj
zdNzM8PF=4($8CIXk>MPdsiAE$#ahq%<xk)HdV0pEJ2@e1#Wvf{KD~a|gsa@?n<~!g
z@kC8rlJe}1_xd!4bz*^69(+rgIDdY>&T^Z`Neec7dE9y?YSYy0{5_Jnf2-zI{FC+D
zUtx6fk=?TWACgn-+CTeA9C{<ZeA7CmHWihQR;P$PsxgIMr#_Kl3CfdT4E6kQ^HAg-
z+yB8Sfi6>h=9Hatm9gK%uVENgyS3?%oqIyp^$%CON_k$)YIgJa@bgflPQ*W6x4kYZ
zQp$P}7Vj@<OI|#CdX4z<Prm7IrC418o6U{e7ad}n(!|77yvOFt_TQ2FYD{arZFHh7
zy0?Fu{Pc+OZ?o^l9ZWg#zb((dzb^iKZl#jblD_Nnt7kox{+?ts`@&1rNkL}i_v%a+
z-qoJ@@6qhFyL2Q=bcB!drTp8&pP8}Z;n(T$PCLFt{Yq1*Ec^92XLSyPNRfAY(b3fB
zlQPYh^`tsFbhgd-`)GgUM(Y;O#R6PMH8nI9L*iTaYqlJlcGmQIy#IF@aj#pO|D0Cf
zXvr>L9JbCeY@z0+-gk=^uZfpwikNg`TI!0fo?wp6r!pMAzjS{fr1>J!$41Ka?2GoT
zDHbN{*KLT%oxu|&QzjnP&FOon&2#zD#PX!=vp#I!a(mYyXV=BX4>rs+kP5NU6F+(E
z?VDRs%~~NJAO7a<XsuCs!V${r*fPl{O0I^9X`+MeRnf#_einU)-teTnmw5NiOknRx
zr^)LkthJtIkpC`6=wOr(r{t_~-H75{23zv^i+T);ytoX%zVNQg6PqZvRO9}-qw&$F
z*R8#(v6TH(g7eX7w|fH*vrT$=WpanRrrwE!tJCLi;+txFweC~#RKF122dnRT`v!Ns
zdi%O^e;NC~Ukr)$GnVZ*;JMsC<?gKgM(>R$?*9L!x@ZoY(kvaLIkzJ=709icbU?m$
z`8h^|h&4P?RxNw>#CUBg{t)R~qIpg|kLTgFW1L5mR%m^ZwCI?~T%CN8OUC+}OW2-&
zq6Z7sRI0MOF!6%?|4IJKTs7ZW@%Dz2-`GN`IF!1UEWUSV?gqW)`<H6<RvEop#yjJ&
z=?bUUvvxb{hRrhB&lxoNjn*cnr@dTD^8{<v!#*mSPq{t0<MfKITf6>*=3fx#R$;gx
zz13q<i%O?P=U=w`Ca>gD*=J{*xHiRTboH!iy7%bmmd)p!91_olOTVap;@Kl(*RiVi
zbj7<~)sT5bOe&Lp{0%&3^}FcT^Nbn4J}9%yQQrCW-Rj7#A4>22&k|LayV(A-_}PN<
z@<(ii`_j+fI$!Z|?awc=CIY+X9Jw#@^vP%a2QL=xz1S(}lFcU`TdH+bPIkRt{UlNC
zYZuh>C9;f8J^G%usodAgt$Xo|&FAHgA9(1#`ORn3J^x<um>j%YajMN=fxBFE`1Jo4
zGxx`w^i=*O^vPUcnrw`HrBT?<zD$3clTG~g&mT=@UwTE2%Oor(_1qnPNt<^zeezMq
z(m%&1-+NQ}<$Z9MYT(gZzPlxBo`t5cUjO)O($p(`(!PDp&Ng>kQhjHf&f77Mu{G>o
z-8;{;^OgPcuFrQrQ9hx}-*f9~Ev~LbO4?dx{(QDoA{)1EoVDt1^i1RT&TR9q1Wsf8
z>ot421vhQhiT>n{2+jP&Zm`p9=fAGaH$Qxg-t*~Qmfr54X}c^%m-)>-wu*Q6i+RuU
zv?o8?-xnt|(L`wL)~gk}e}25a%H`FaDPq<WXN4=z$Wg26e{=fV`m)48nU0ATWu7W}
zTjPr-8##quxwiDq+vy+WOCtBqsw!J*b9UwSEt6c*^6ZWtoq3|mX}a8BP2CSAMvY2J
z3zXD@qs;?CMIs$NU1pen4&8T4?(q#__oNQ#i2e7jubXpWkF7)QM47F3>ueb>cuMX0
zeKH{If#SAe@9au}6?eAHQlD4XE$7~|B!7F%t5fAQ&-dATCM&LA(4o^VxO727-QhD8
zZ`md}&RpV<#p<cirm$ko&k8+b@tU(ME*_uo+F(=i-?nQXb54tix`r~CraEY!y{#7|
zqS-JjWY+W(+!|h!LLwUb4xZdH$>rGhOW-Q)tOl3M?89?UH1)|wKK{6~P4Po?WV5b<
zX5z1$dX8de(<;-h@U>21zU@&POfu{KuF&#bHM1jy-BERx#gBr&9}K3jdK^hfv)enT
z!ld(V>Y}Rk{qveXXRP52+<N`^jn2QCub#Cwvpy4Qc~-km_O<WQipD#8n9AP7oNL<o
zUV!h)gegz2)pmx3D?0!BrgB<DU0+=|`+A(nv*&`ZqRy~qUwZ^uuec>-(~G^|JKAcG
zJPOpxV!pw+diRXL!|W4ZnoaC@sd?_~gW00<Kl&P-+_#ItwN=aH{p+naWm8fr0!3IC
zpY79Vx;giBOY!l~-}nC)Hr%bge|C*f+T6A|dziNU=a?zJF3wqLqKhamk5rvW?ddLo
zt2TWySGw0{Yj>!zZ*gnSU2(_Nc}3@{xzZKu*U8k!He@I`C<rYrI}KW}sImGs??wev
z(dZv#x9y7BMHa5oy8DYa;mQNM8()Nu%bL&1kN-U>L4ymtXpz}!E>EcETm2l6iI*EU
z{csKpVytC#I#^ZqW~ICOme<`bQ@UocEs3o@srmYFhvVLjuT=Z&6M0_8|CI_AJ$Y6;
z<#%7IY>B3a)EYi%skLI$Pxt!yZb<jH+_+-e)A>410ZA%PElRgeRS9k6TQ((QLHc^e
z5?w8YxS$T}&5d!XOH(8tKiQevQoGQ(HvI2-!O5&ly@4Wc?ToxkbmnR&Z+UWAqV>uq
z8wY_CS~?5vT@zioL%jO+wMT~CFI;o8|F3FTaZ$0usx0ST-}-bpD@Kvf%<pe9E3cVI
zm#tXA@xs?kYIS*FT%2R%y5>deB;Vw5Ossd3ocJnNCvxY+N%vO%USAuV);@cVllteh
zz24jYnmFyg{_)qGrE}`fPW>VM_sJ9fq$>~h)ojc#YxL+z`)rh|u>IWqEw=wdOt+Tb
z&nOSParN}NzbP+nl?O!sOk3p3n8Ycp+}3ws_4m(lxwb!9S1!ACdiA15XS5U!uev1S
zf0XyewpZL{VI~*X*SCDXUpIHjc6s67hyGkozp(g+7}HnAp1znqiBgru+|w(+vqov>
z<{v4W(Gd`A<?d^HeE-i+*>*p2Sev`n?y=5S<ja>Vo7|`6>KJ$+r9AM)%6-4(UwIq*
z&NBJ$8p6_(G-;BMxZ2`>=0Z2T9X#4DT$c!Zbs+bu(W=?9CTVrvai2{x>nd3mO%#ZK
z$0hj8K7X=}4Nq#Q>+0zlpWH8Gn97*%<2ROzKGin0dS=OCje9k}xcB|3h~B*LPGlux
zcpBToS?e{|uTfsCBOclsv0|RTeY{`Z^?Nmj`E1&!gk>zNB=UA2dG@ySZOAnnhv1;*
zyI0<?RPbw>^(#i%GbJr;^7Q$gi=G4pT$&u`8u#e7`O-adUZ*Z^%>T?auQpLF)!8i|
zY>M;OdAYgE_PU*)aPhH4wa@x1m$E}uHPr({J&q-FG&vYaDhu^5RB?HBzNR&F_hkF1
zv=C8C&C9Q{gQiS5cyrbEEBu~D6Q|DgUUt~U&ri&jZD!%?)O(Q|xq7cUKK;fUveGlh
zs&i`m`lK%hlvo%=cB<}((0adh7024fiB{R$uUcN2^NZE-X#CE8Ug^^(r41$ay1jD|
z6P7enE`7VlKkDOweJ^6?96YvG=wil{mNg0-LJtchA_D^+E8eU9Jk{@4eC~GtEv|N)
zv(I|hIm$IHnlN>)f1W4*k!PWNZ=D}=zAfQf@@vsf`%5ol>Jvj)earNZL^RaAXOq(6
z*-`C(XxdHHt?WBx*%*JhCtZ1Wr|Vj1*M+6(i%woDo3m<>>8%&`83yam&h5RP`20(<
zcVXI-B%>vAGbdeZxSZ)M#9ET3ZkEmD7<%%p8AJa+@3@L*qIr8?TlrZ(XPQ@=$M*g2
zQ*M{Jjd9BTM-vSv_^4U)l)B8FdRZ;*1o!#9XO!#zYd0-ee168p63r8;MNc^5>VL=n
z`!xOhhl&`HClwbvm=@h@Ii2i2iABI=jh{MOw_L!*SG_xI%u2-6m;22)vibbpTaIkY
zuevNg*l{}9eM->z{xgr_cwX$C@_d<X!_jxg5=tL!k_=3dbj`cHxJM`XJLonR<BHsK
z)iqx~*6>tuAD>%j*IRw<V`*ha=k$1=`8+G`e3Y246MxuH+3n-aW#JVc*4H0=T=@9R
z`NIn?D|j*aP3LY7a|%6}uq{!%XXW!LOO8G)O1$9tBebA;i%xaD+I!P0$z`d<A6=Ix
zUG6^n%YTufl||_xm3h_w4ix|G?U{3ViD$6@i{pU|lcbG*w!D#P7v&TTUF_)Uy7=Ov
ztv4KIf74j*`T267qesnpWqy9=os!*mJC3e8{>)@j^a~ZCzU1OrC(aZLWbnxUDG6=e
zv#CyG_r1qor^g*ExX5xxtua;p+~d1TXYYJ<Oz+1l>67Q=k4!ww{_*Le?VRq9&!%2w
zwvozJju3SXG`M}I<Yak}V9&*^>G4ieqa!vYG2dnH_mI6TWEd{c!BTT1RrhdPzk>7Y
z=a1dwuRXgZV(EGGUXfYel%R_nU3nvSRtnrXE1iFC@o~fD3SoH<wwgaHx}3WFQGSMs
zb^QnaX*-W@RuOvs`f0bs4R!h5F~=XRzkd1aip_=@t=BYezq^w(C#_Q>>esi|r=NwM
zzPZKRY5SkpHFtx~YCexYz9aE2L&Vk>y7S_9bnPuYuIMFnJ8b=f#j5c}3yz%o^Ibgo
z<g2E$ISv8*m)5ztOq_fr<Xz%xGu?f^yu{8{w|BM7?L5prVd3h-8|Ujte{Ye{>^duF
z#Jc&cv**6<Q_Q~}aqRl4_xP=U{NuNl?=LMsaBNSd{3Vgki%xmgeC3_{DrmLwf+L4=
z?#B7;+idNsdNd~{)YZ)~JSJ#{tGv#^d6(yQtZ%C1wp&{AAdp96%Nudt-v|DA@Sm=)
z6mj|EJJrHRiMN-z@cWBymCm`}TF>b`Sbd{cJF;J<Tfx#swnsDcLeZC#&y5nUSd_SN
zN9~>WS#LjU??DfvnMdZ{o_l+}&atdZ3G!xrha9z}CRuFSU3C2YdsV6Mf*U*;v43~`
zo#!M~<EYZix&FB8(_K^de7>4B=`15>Xz0PP3zLKUBoF46|DRVC&k&e><Z<D`s~tSj
zwtbuI4tE?8%_+0aGU_~-<FJAM;kiHmBF;*MI38}{)^oog<*Alhw2S}gxkZc-zELKz
zzl(p{x~?^<i1FJpsbk~qfOlu)8?LT#4HQ}PE@s`%hkQ~sj3!mWV$FRO7H8D+_nlkk
zutRsj4!83U43i~mzMP!$Mb^zg=4i^shb1>W-J0(dhv&a3lb*yd`TdUW7OkZ1Q9B;L
zOJZ8ck=D)IyF@9I_k}4__`OPTwr){N&clAT?TxD%+<X@mJ}tUBw>)uM^o{900avmG
z!eeUqHU@BPzT0>?&C&1F6khw8i<We;&D?Y8+&R;^a$3HSNuke&E!Xfo{X3m+TgC4y
z|0gK>hlZY6C!@FPk5OFR&(dZ8J|DdDz$5w5m(Ix&R~B=>n&@s+{I2~@`F*b(Gi}H!
zx`P>qo)#*2F?A@JEf=1Aw1F$*tJ)kncDC8GgVpAo{;a?Epz`<s%#H#Y(p4ssWyYmQ
ztLWA~nEEo<U{>x6pS(F@#!X8O=9zE)wD@LD#46TM*Yv`olctkDu8sFPG2x`^<MVDS
zezqN-U)jC&c7MW+8HRfdpEt|vZNA>QM9}fdLUGZwy+1?_7O>=BbnvswQ;E`Sy8F~=
z`NV>>(uZ?Csv8&I`(3lSL~+rQ=QD!UAFRI9d@}mbgnO|UUVm+H=s)Ou@^6yu)9O`%
z0vjy9>coFB>u&OWu}0EoR*4qhVz$sLW%t&pb=}Q-_PRgq^V@W(Crf(F_j6yrQ=n$c
zeWzp9TZYJEJ=WZlAxr4I7eSWL+5EqBhOhSF##tM_*fo7VZvNq7y7$ekH~rG;1i$Wn
z)WK%#+%I)j`n}Gp72bhI-f1qVzwUFd_O<DnYp+2o>E5iYXiS%@X7Lm_q4DC}$4bc~
zO;6WEd}D1p6UU>oExCB-t&%*0hhHp$|0;IeknuYcc6i~wx&M|6urOK*s@@Vw^e$m(
z-RtKgd6K=dfP=}RsCz}sUheHXdN*8bJDK)ALB~7FRxUqFw0qU6h}>=07(ym3IhuKW
z;VP|3_gc<f*wi0+AgefGX|CkPldlfs6mL6~boSK3{|TI1@BT|)e78A_VeM7Fd1o(1
znTb019eR24$W6bx&``%qkFQAHdbM>~k=*K4U2eC(-TY^?Wsgek@lUH(Ei#WhX?8PX
zX65DACR4t=YgV6@6YZnu@x;PMgTvxZ+?;vares7li%#E|yXi#E*6lCPKRoiRWZTLe
zKWAL2UU~nkn0DK>&zBR5tpr(Fz1_@aswtN(m)?8(YGUWMZFB!w+h2FCEnNm#I-gnB
z3t6sWpd{3pa8d1KkwxXdhyELH@%rre#c^-OwjK|Qb4y;k8$Lg}p7YnDn5y3rt7>{p
zR81s$k1_kZ32?C2hFra(v|6dGUAq5v;|-aJWg8dH%ZNFcwYf<rc}s+2=Vg)3<gF1C
zSTeG>lrnf{^j1yU$l-gD#cVaJr${E3$)y+ki!MqQ?hbBCTl7<6_0c93GxvC2*32(U
z`;4WxtXRZ0InX`$fWOYfs6#7OX)V%7+<d#k-C5;g()JmSQ#pV9>8KJ;YkuZ_Ql?AI
zFmsa4#|yO)RXxdn3T-)Eg)Cy^)~MNb=?G_WEi#V0kZHnk!sOa@VV>g#GuK+oEfcyZ
zFtMfUZdT@Jm6cq+jSQewY3uEdX3d|Nt!FdaMWyHFvjeHy72BmN-yEzjxwlSf`>O@2
z<(qG29nQJ3^;xE90N(^xAw@Ax;mM&oX~v5bWG;wTt!>+7D;@dc<n$?Dq&y%?(jve3
zPO$&`Pi&7^=dWLXOEVp{J6)80Sq>YX5wlino>rma5?jl>YQ@2aCCi_9t8uW*x-eT;
zqD08|;NNwVJ(yN+U2E^MmT|IUxkmol|5BTDF5d`fzrIGPv)!G=&r#rm>9x&ePt&J-
zm1?^B%whRuMbDu2Irmqsn&hL_dHk`<qQGryBKizt{Qqr8{`l<Kf2E)6Z$E#Tr+y<*
zy1+|--)CmQhCh$~>pt43;puuH<I3{negC`yL|P+s+_vZ4z3{Sb_W481fgjDyF1+C1
zmgc$PMoh&VvzGZU58d7VRJx2kq>b;*(e11Evt`@;-SxC?{ic&HQ+DlQ&|A+w*X_-R
z$l$|q)7IZUEa=X2aMAyjb|S5!+|NJdE<Um|eSTM;MtuFg8Ts3<6}<60zB%tBQ_G!y
zbBwAE^zWPR%=shjrxROEvAV*BonO+r+<gxGnjKXo>-P77*p;>N3RC7a6dc;mH+$*z
zi_5I+BDZwhS~A(#&z5iY!w1)Wa*y)a{do{9!@Aev-;2bIOZ*?_vzwm06{=l#!C1k!
zYgt0W(b<2maNKO@3RO2)x0`M9w*$%i-2I!lB+go?-L>Ef6uBsE(z8EDV-=U$>UqX`
zlC!w#Qm-xiwQupfw0tM=*_*$<7wd0Q@Y=C=Cc8|3%8M6MH*WoW>P3&m*JIbJbj;Ml
zcCT&@U-s8<rPaZVZHXqgUka>q58Y@PGxyY*%~rRb2)ew{6W2Z1RM&Hcuem{n^TDYT
z2VQM@eDcJhyL+aGg@&>O9?2><nI0Y}SXgCLz?nNwaqr1X&u(VAWqg~~)zozL;W6!5
z1-on{mWc6B;#ySlp^SB-u<VMpZbh%wGp7cIUXVQd;`Z0f^xB2ykt=WSotmnB{h>i-
z28WyFyPi%9-Kr4Bw;|KD5)2{@XRKMPS-!>1*z)JJYX>q{ty1!klF>_FE&e9qy65qK
z7B1#j{wu6nm1HpMrR@WaFIBQqy~~`}ef`$4s6)l`MX8n7FFqBK56R!t{wTQ>b+6eU
z5^dkTr>p41cA*zM$FBXh_nmcE%=BDv|Gm?bjqQ%S-J+;2B=Pp=os#eKXBJ#qFh6xt
z`r$`6_v}?Yc;H{^p0CHe1B#A)ms1JP`Q4?RKlhxn|KHj7SkLjzzxpojeEjk~hr9Rw
z{;%PDGkkBeWPbg*AD_>u-Cmh`_mh9c@|g3IkNs2c@9N+2?wP&uj~RtBhE*2x_gJ%^
zW#Co+&*E%(c46wp)t7gCKUr@1wCVN4+pXNwl<S_ItD4)CxZH3>wwcQ8ia!UdEAyAm
zf1Q8c{>gpS{hC);^7s8Sa=L!`a@(i+iOzW%+)OHyjW$;B?6njvH<RjLXu6bb=f@P8
zwSf#ONgpLnF6mK_iFxy0s-^5-oqO}L$A;{6WeKwEQZBk|FTQ90Kk)tb|G$e<!a`i;
z&fKiGOUiMvyWc7HYZp9cXC3#lu3fNlRo5D>9*aU7=McuV3rv=0dVX}iZ9Bv63iH%k
zj`RNZ2ZoAv8@Frc@0#3oN^4P}<m}77Wy{z-1NLuz>vub7|L*_)E|*SvHr;*w^<)Ey
zeZl{m6ik-SoNOS$b0f#>#98OiI&DqYy2VSr|6Tqwbisvy>nA2GKL2?A_n4kLA*V0b
z@9BQ2zvm3+aUK^gbAe#fikPwh(N|fk1bNG3(s)g;Zhi4ZD0FJ*g6%;uAuWN^rG#0f
zvo!7pH(q?FsHh$=B}8D>k~uCDZZ`>BLpe7g_xQ;}DK<X7rLvtVt9*3sxcod<G-*;}
zipsrNi)9twtX^L8Jub!5KQU0yY%`;(<og&u-kgY?Q&#rd`o{Ig$yB?@pWE>#&F$EM
z6=olw9+zKOC|>kYRXiXl;s9@boj^~Vi^EjmkT+V(kKJ#%JlokZPff|F)by&#1%Vao
zk3RV@Y5L@ke|xh+7p~hTWwLC_y2fK|QHwS?OuO!^a>JqWr<_p6No}n|xu%NhkFSK7
zKRESa(RHsQ%8pr5hf-2g1wNV!bRF$%WpZ&5{wWz&<6Eer>KE!O8y2qCS{a$)!Wilo
zu(RCUl$Y@gpMERr&KD|;3C^|;E1DvL8RA2hEm*;!_$>YUyeuC-e%GItYo7-H;&@nc
z>twjKub9)0uQDZ1#6=%%T4r;$Gh;^V)2A%ToJK{TPMj=JUSU+a)l`L5w{cfi%1MXB
zM7zdYwISD8KRkIddGf<oH~Llu#D_PpG8J04{+PJmq`Bc8*E5%`tdoDQAL;3MKY7QC
zIl>#IqK`e%3VkYF@$Tbs36Zmljue|beC4|)c4y;bwL2$0-FYonm|F^RZ_ZA)JebjR
zW=_qfy?QK+k1iTim%UXI=zDnfoU-}lX07GxoO@LQVuQO6&h)Hno4a>+nPAV!C*q>z
z7u+i=Eqx7V&e`K+ba&g8(wNX@4bCgqexEp@e=PfI(8lKn>Z~PXU+XGe&Rw`}S(k=Y
zk<lHFqDiZ2MKf~qZ1{{HI@s<md#}9C<B6Ea!@?Pzt$(kuF5)Qaxn(6Uxs%nRhiUhz
za>Z;P5pmJPOW7P|SA;`Drc}(BWhKv-Z?^MhoC~8+k<iAUb#@=(PC5M9Ic?J){^Aq7
zscXa9?~6a_-WnauxNaYFporm2-5WC3-beoaZ_w0mGwJVoUW2b{pBd$z$1l43-iftG
zOgeVAX?wMO&-Bgz#4Kb?e><yd1~q=*J8tYb`PudJ{~s+`*u}QoeEvGsqa0p*wce+0
zgefna81l^4bn00z&hP);IUR{;sQ-6A_C|Z`xl;9%qR%beE6itmu439c_3E7ByYF|T
ze^Qa!zV_*~+4<>b`?Pe*#Klhfzq?S#9^wCb`uWH2s^t^HV>uirO4R?<y;FHtrAJn0
za>z3_M-h*<te?;2-#vKBAC_PeU9i*VM>1>2BA3JKc<<D{zFV_1ReaToOW8+m+SJ^?
z8{vF=N97?^*(Y-<{>tz8_9fBg>j#svY4xY)*VU`<s<-bgJuUMr*T3K)=iiDqOSebH
zi#~mK|9_{u{7-iIJ>O)q4o1A&_eA4f<-<a`IiHvk16@s03r@d(pMI%usm-tF(v|71
ztWuF_kEWIvKHvIy|NezKC(cxTV%+y3_45tq^29J_pP#qyvKaL}>ML&lxO{hB`qO)C
zUyc+$<9)P{{lDY7kfZYJDqCjD)z8@1s<*B7r`~qI=wqAx@1GYxZ+Ggi{yE3R7Xzv;
zsjQQne0pttEzjbUO#)wnC!2iQz2E-SPxUWP=Y}u)bl^((1*<tM*^6V!e($|gaFBD)
z|1<l4Jn#8A<E@@l4R6ZSuSchR2@Vc*t-W(VUcPEhasSJUtNJIbNZM*Oqi9l_<Fr0U
z&DUk?p7WRAKN|jY>kE%ZdU8jc`|tRhPd}Ah|KC=B?{8JL$$X*jpRL<(d*)^H=ZY`-
z=AYhe|7x+sg16N17oU{Ff-L#atzTc*w))goS=Dme<X$h-IkEVEIk$X1PlCWAmfEj3
zirGKjxWazp=e^fwqCaRFzRmi#=K9C;?e*@<LfJnGtl#^?Wd7Q_=VsOaW%btS+LW$1
zf6cuUr;f2-Tv7P1;IZqAZ?8|d?C3i#zi0aM=XH%YSGXEf?0Eg2zxY!_j@<v-v3s^W
zh`!@CvE$yk_ws+t-P4Zm$UJoPikF(uM-4}wFYm9m+FF-ih=}(782J3-m&4`B$B$q6
z{qF$x=V{H0CInuXU2pS=<L+9!yg3De)svY#_fFj{U&HgZ&-G98Det$x-uC<KdvJs=
z(=ocE{Z+-EfA(c7zW;NVTkvZ2^~#iE&wVOiS(Z#az4PU{`r^CH{sk$4lWynjYAqG@
z&6vuv?v0~Q<tIiV>HE(YRsU-}nWT6tqW|11*UWiSWvy4QT)Fz`9qIok<kwmDf1m$T
zJotG{%gww)&-sO}Tm3xD++TF;!=q~<>lesW=>L5uA5(B@!k1OO^E-ZAUi~A}V^YiE
zr>fTLJMJ`1dS8?JT=Zc<<wTX2%gz7TAE-C|^{M{D9r;xc1HDUfTmvt;eX18^Wb9qy
zxY%?5rS)I5|5|<6T3^%fUsT2L;3HQ3!rN=N7anz;{A2&j&yt0Q3L{LE4C@<~If!`F
zhX1UsV|r+@YR-aXo7ye~rhoX<bA89>MUyLjzikaMH#2<wbNP)&n?HS7o&Whw^8a?B
zUV*Naxy|v84H3-!|9{_~@q6B(H#>_fU##PvX<w@qpL6T>%AQ3Uo4!4c+_NV`_`A&?
zlg;t<(;EFcGYon(R4RVoRKFo%Wcar?a*xSdMXm6&&%f@BZ`6`kcpja<*WF|OQQ!T$
z8}(w@i|-!}w#Z8`oH(Js?U)-EyUgjc)90LApnTk@^wj<r(JvNWyzXHzX~Du8+2DVF
zb(Y^L(Jy|!?{Mq!X`g;9zI-EEUh%?(4%N-_{`+m3v(<Y4Ha3UqHn;O7ycM6naj#Ls
z{vzx6`j5KKZC}#<efHn5&FT1K|NV!w{rz3zH?|$0Ki^enrb@(X<(qqYECjoQZa;7j
z3sGuxbBg$JcmMon^UGtKJv{=p>K~Xn{qE81{BsF8D;vK*JF6trzj#H|x_y7`?#(c4
zetll*NHzC;<p~oy*`zo*#s2+xA#783e{Ih1Kgy^2tmd2Fn{%)**0<)%KP9cQn#;@j
z%kMqhRQUHpXwI_7Y~?ez&!2m2RjpF+{L0qwW%g~hK?mEl%{IS#eAC9hp11t&f0q2c
zAGOq{NLGG{zr2F;$j9rZNoQZ2-|{^E@wd-?Pbc$-75&q#Pd{edpUC<4Y4iH+@0L4@
z#$UB#64b5#ef{ztlfQ;fW?l8E{wV9_zh^@5aoMvU(|w=K|G$H^e`jU#@^gn+!(+~G
zY(6!k=DPikUym;De9`sVvdlyB`n^*5?XhKMDoG}$teII^i&kp3Fs=EUVRkuv?UQZw
zK2O)0rxn!rTg&UlG}wzcKG%5Iczj;MvF!7PH*`N8+_O|gWah3_QorVw@U1^L=io-w
zs<1WdAMaav{^O0>?VGAk)@%#A^6~7W@``uOU;q4!zg+qJ?erIFp+>pCvRjfkU#-r5
zS^UUykJV#Uzc-VeZ8YCr&*%Q&9x|y@;^5yWvEN_LVSY9D*B2kRITjO>G*8s>-#K3U
zf0oy|6_YLVGQ<!4WAkF4cjEH-eP_7)?K^I4=sJIG>SC42S3=xoRGi&=rs7$l&sXae
z!n;1+$zEUl`{(h`?+*E|=snWOUYm1M`q7`+|BuaHzyHj}om#=mY+REqL{owVT%}H3
zFWWI)&uZJ0<MOqW_VR9Y{_FAQYVir%Qw0+qm}OpCVwoeT+PUV95091aEydYtzO(o^
znjCE8mhai~C!qUN_u^Gd>w03UzP{^~IqBMI_UYvs$vNex1UnXWEa;d!g|Yka(H+75
zYWl{L1Jz~D3eVr`yhroEvAw@7vK}q%d2^pv<LRo+$2Tolq?^~Ez+rMVbYtREK9+?u
zrq8dP*0FzDg~@?)r`-D9&8>H+X7V(v`K!B^t^dK9$L<Li53q*E96Qdhqa>M^boiUE
z$<M@lul1W{R44Xq-d!Wkx%GWxk>1Sat`_af&3^ex*4EOwyA}Ga=8E-S?mId~L0y~K
zQ#yBA^pdLcVr8bkZtSemJpV4v`|ds~rmK#xm=!NXN*ez%Y5Vf(*WzCbk1x15W#+u}
zkKwN+Pk0^bo7eMZ;%R05_s>og>m^pK7h2HLeNOW1XYK2kMP#oSu4`3LF!6U^(RHd}
zqI=Ps6^b)lZY*C_v5V))lw|h_FHMW?1Ugq_7xRQBF9_ei@64RfjKS9-#})M64K9ol
znY!<=yHQC(dD8AhCFf72T)F6c;+%G(!}KJ(kJmSCuQA>&{a3U0O_cYZb$vmLw>A45
znf0UWyIkL#3Q6T(Q<!2aUp);MD6P8p?>5gaeu;xCkIOZ>GEdgq*HFH$e&VWK3NLhe
ztRAuK5`QKve`Wfbd8ItEH>M|Lw@f-D^D3K(J!VSRxnnnq^&fWJi)|3nv#eBk|Mq4}
z#y*DGce|2`CqLX){vpC@j$DoybDY0g*k;DHhkMV?_;owa^y9_Z=RZGs<GNx|$GJC8
zmfio~=o`Ow@$&-{d+hZ}b??>w4|bWP6*hU=y}R5$KHR+?xn@<4*7}9%Gbi23dR|cY
z$5VY${Eb$-qp>RrBVJx!J!8gE|80713Nj1+tuDP{$`<+QV9d1_3(N$TZ_Lv7y{IDA
z-hKJle8=mDVjLER@87p)XZ-%orbFrP_Z<7Y+3tnH`)TX<+b+1*cFOjF#)Ea$?-LGX
zK36QUZ!`Z_D(S}Aaz{@jZC%>4;>!+wLBc%^YZi2I&6`x$V_T$g^cqKOb@_3#FCJaR
zn>qPp9g<F}yDZ`qaZSCl@p!SD_x`vd!EU)2hh&3WS@xx6`af4QUtvG|WcQc$Cy_s*
zA8s(10@^<D+DX7cIdj62FUp|(2kf8r@A~8&X!=FksO_}=ZV^U{uP+YHsrzQS=Efd&
zM-QPcQw3(JP}hmKA60NLb-WGQz}Yn8lNh7dn%wAFbEa6|{l4h>3k|Kee)H{GPjvbG
zf0!(B<#7UI9gB;j!?fE;JsMs{q7#A{Sa>BPS--rz&Ym(;&i41krRM`!mcC@LjOOuj
z4c(t?f9~MrFPUHcPE~zbrg6CI&Sv&xztRcs%@(du_|xLr@0M@3`1qIPkL{apI;%1t
zo8vL>I?t*uncT~}WhMp+GB&zwYAmpM+88{^Xw|J{<$((x7nM6^{0*MO;^L_ivU&3U
zdv)3e?;M;{dMM+}L9bP>ZfEaR{hzsN)wEyHS2;7j@_}qMINY_QWZ{kH&b@hg)?OKX
zZ-qU##TIYsnH3_yDyX{hg3>mbU<YL-58XrCm#BzXZCS<Slpeceg-Uek(nV@dukFj{
zJi2((f4zojy7^AgmQ%0k`o;d@c@(o)JXBypiP|R{vwearQ&Kj4jS5L+brrm^@mF%*
zuFr{m7GHh(r0z`JD}C+l_Mi=_(O$cA<#eleu?mEqC`iAw++Ti8)QgLI+56_Ki`e4y
z`YXG@s+%Y8=k1=F5cj*z`(*5;bb;qmHgBJ7-4qwfzvBA5&?Z-I(TyBU3KKfmIoh2Y
zTKm&&PBve@pZe#>XN|KPHhuf7f9oB`!esj`^JZ!(tjN7SW734^-=>=C0Z;m(-oJ^H
zs_mLv^ZlWq>fh^U6yvLS7ibuDx-2@ZJ=5pkaof|Gmmu3;Oxn4bI9F+J`|$SI*){Ro
zXPMutYqvV;)OWk{<cid%pO)5atrGUOpLhIu{CfYj%U-L@Yg?t!!KPh%s_RF|HQ|~c
zN29{@Vh>Et-{<X->34Hk)1nm-dv1Pt6MtZymi%3IzNHtk6Ryq*RpF0VG5wa&(iK8`
z-M^$C4xj(=)SUhsyAoCX{cC6K`XwN>xYe$<>QVL$=P=LmUsFXlReqJqYVuTRWPTbF
zywjM+%J^1>Zua$jl}meScBTr4Wq&wZ<|^u{xOoSArC&zRDIQ5X9_Jm6SEb7oJCs(O
z^1t&~ukg{0$v3|Jv_2ENZSG`03$DQIh4KA*HD>8j^0gbMZH=!vA^g6^Yx=y}!l<)H
z4x8@z{!TsT{<e9Mo6{e29}|c&DlmDyd#~KpZTlG-D|U45dS&3=XWM+y{ma6ARpt|S
z%rScMqGHdc_<{ge-<QhQIe7zTE<D_x=*g0~W~Ji<msMrC&la}-KbyDz)PlIBI`h7=
zyszKamA~&t%W3u41E1foU3&k8VouCQOa1ddC%m0~{llBb{ttCEm5%2+hCcmP8Ikdt
zZ|l$b3|=$Lm)Rf6{w$`<Ewt#)k!oZ4^oyUau8H|&cYg10OUu~m<O^F;RRtSk@9RDM
z_gg+KJy)RjT!Y)<r?W#}XYjZ=FihPhD&Ttex6!*P!4^NCOqQ(vb8&}{Tb2CGNxxgo
z#W&@iw)*#enfA}mQzU=X@E%FEUOvCZs<zAM#@_WzJD={d_b9k3c2{dv+dO5q1xJ`P
zzR9%lzH|S3@9diuYuu(B;Y)q=P+_Yn`?uL$4vCTT|2{Fj__*UA`<30BceO2h_4d!>
zea9^xt-5Wo&f{>r{0ujPi3{rDO~0)P^0Uvjk$h$$y7lXpI|p*FPrPOI@xnhT<_Rhj
z&)=(_KmEB*((AUVHlMF_r#wl2v`sR4yY#j@-n?($Ej0y2@Bi7t)9lsT@5rj0HaeLh
z9JO(AppG`z!W}*5GF<Zw{++mW8Je&&>o{Fv<jhy6?>V~kwVjvbgg&0UeV_H%%FZtS
zxQ6qI-p<LZ^=5g*)ybV(XIr@IOox>1%$bohN>-+Bn{>^?ZJD&^CF6gKPOiC8AiX`V
zR#D8KZ>H;AjT0q`97<)=*$<{M%$=P5eoy=T|Nq{jY>R%aFxM?zO<P;KSLH(OnVB)x
znWp{!zlOzanW1s!^rGrayAG>!`(8wu<^E(gk1J!83S1EVKiOixg>>NE@`gDxXXdF0
z+E^}~V`4I0^ZQ@M@@t!S%!#{mPJk()D1GuwO{LBTg$D)I2i6^*dAt4o-7?$y`9^cU
zr}-RCDJptc`aQu|x*#<zq3EAUenegW!7tidHpiJyXiTkaIB-)u_VhQ|??u12PG<-(
zOOctZFVk=Ae7;5d+_Qf#zkj?KZB|sYN2Em8LAf*HjozLaQF~WfZEjs7!$0epLnhm5
z+2yvKwyS4cK5siwU4Bpd_OpCNMrWRgr7CbFY}_4O%CKn3@!2ywJ?CFqeDmEQ<yj|n
z-kg1(zkA+(-<ju{BrkusvBsug4r2>LuW9I>&j*#<diPJ^6puf?EqDL(b+7fDg<MXt
z$)ERh|GI4TYX0pp+d5y??{9v|w4=D{!|~ti4kurIeCEV~Hy*u9OVzSB|54%)^2=v4
zo6e~!miUsLdt&tr$@)7CEOR**D;{+I?z3#s!p=zt)r-$A+01aFFn!V1B}~i1-rqPg
z=O(wst25l1dd0@SekA`qF5ExqeQELEpU$9hs4S}ry{~cZa}TZ!&w2Qg{oSL=zylL+
zH}B53XxtRjXn6SEvw5@ijyybj@X-mi%emGM>I`qkRIn6#ZI`mWx5mduYT~`*W5stZ
zXV3WdTe`wY`QZuu$9?vGhZY2%$OykTE%eMwVgDy9|Nio|Inj3^WN)<9mFnyA=TlD}
zn)##0_0Q9`=n3EW9{#d^m;O-vxk1*SihI94{onE7&E^xQ&#G>&))Z?No4f38sfve%
zvC#&rpzC|iWvFUj?{-sOv}(fS#mVP3^M5KyRsHrx=-+$&$s47&-k6ievU=g+y;~N1
zjoY<5*5#9ly;$Pqdk+-vCQ5W4HRv+r+7$CPP2O?Z<SSRM1wZdzuziQYj5=q}CzqG}
z6jeTLbg-fS-`%2%uP^aG1Z~Fju#rpu{q1eZ%S)_t{k~t;{u8=D=!5Aa{x*-F+gN<W
zto7QrTKaZ(t=~887MpxnyL5A0^9s|1tDYv;%oECXUC1_75jK=!5{&V)mI_q3^kng-
zU3|S4+m7d{=&inyYsOS358kcqpn4;~J+xLiYVp)JXL!Ec>btijMaj)MnPd6w#xNnz
zV@e?*VM>b+&VHyU=9^G{<3b11#scBrTB`zFd1v0d8PQWEV%fNU-^T|<Z8au8oGk80
zW?YqcxTEQAp0XgXV2@hit+yRbUG@*$FGy^h`DMi_r`S-{-bR72_U;}YbD;;H&6QrA
zx#2VS{dykN2eY4YOMW;zx8cCEOS3iQ3>jCenubl8Tv)c||E$~jdzuBs|F`bev);Eg
zN9U4DYO1KGNs8%w=?0npP3spmwP-#1TGgSN62m9`al!l*N)HN_e0!{xBq%j|(T+KN
zX|b#v>LD^^)+ZxcPMy*cacn=HtUfQG;70C|rhV6}LM$DVYn7*^*|z<)OUim>9Q8$z
zC)xD%$94a#A8m6z;p%?T;K}+On?7CeV%q2-EGim$!s&Q%fRN!TEp5)DlP&H>?wBja
z5_<6YXT?{+zkH4yGBG<E^od&wwrTzF^$QGA!ezNrUa#N7a{9yS9e>}{{ctig+Elc0
zHgAL1iTCQ7FE-rvQArY%s!5E=Q;+KPP!e0XY#rm|qm75tj=axyN#C^9kEg`V>mApu
zMMir0o9{R*SvhgltEv(;Yrf5LlDuM@`Qmr~Ub^?+mA~r(Ed>{EN|Jxr(sg=v=k2`4
zw7=VBoQvm|y)l<;zgriOT=~Dsw#<0nv8qW;V%rb%b@@2w%1%1jb~tOn#&vB~@xo=x
z+2^`BCU00EKFNe-@!T)Y#qzC>Qg-C*DL7wRb0y>9zx{VSZ@fSJ?%ShMYk`1|zyGd1
z-|+YS(lb2eR<>6hGq~6y_T2REoxr;yqLk_Ju55$SqE5LYXG^Kxglj4<gC>+Z%)8~G
z(sN#Xf#y1)saqv_(*6a{-_FnTE!J`>KkHmp9qW)!or^kFX|eMjxf0j@d;j`X3SBGp
zYh8azJ`r!(Hv6Z*vXZQu!pk(g+V9tA#kO~Cd!73~<~~#ItA7u7`AMGwZAte3lP`5D
zvrhBh<b^Ri-)8IWdllAWdHBco#q|p>eAX`6rgCV7{GFy<UzuAj2@ClB5YMQeX}dg2
z=fL0kzsleKJvaLKMwxG`+Q0U|%IBT!KQ5&|yjQK3>~VXqJ-7Xl0J|@XW2S}#wD>8-
zzb>irQByYCJuUWk7H7MNPugim<rMBFhYR7y_tl1~2o|QNOnzCYRP^GK=+=45lpcRK
z74qG}eNVCLhU_}#wA%l>IZ!sd`t_)`Hr(&mdbKg~mfWc#qdxl!EhkqbKfR>tac>=u
zbU@n&VV-D5kF!1RZcpxV$td4>^5iL()q?e3R1O{N`SU+yS=RG3tyv<!zP<B{{h6lK
z#8q2W>aP69!1C7hugRzW+pKUoq`2rJi`8Pm$qRGC>;$=myIdx2Vbf82TtBA<ysO<L
zv(C4IyUOtI&+rX9_gd(_Y;dy|aZG=DZ1=ye6~6hkHd{D)_I|PIo9O!G<;Giz4=+fc
z`YpJ2<>JD#ReO}%7Z^&Bv=y|lKEHUg`PSGcho&vy)!px`wZQGS2~X38`Rb+nKWXwW
zYEa;kRr=^U`&H$?`QJBhRP6Iy?;09fB4#ZfQ)<iHWfXX;azA7z=;jN0v*NVkJYMan
zIB~qa_+HA9qm@r~tXZ+@)+@%)P=P5d^5<(?eHqowV=Os0hdBtinyvpg<#~Gj$)(|O
zM{em)U#L4(!f1iDR7i&I{USNp+jsuYiON`W>txULxF@sUhDz{QC0$hc+b?yx^6~K`
z^Q^$og*U737PdO1KcB{>qqsTgXHe2Eo<4~?+#*ZA-k7p(N}6QQr)?D*SOXJ|ze?3v
zIpO8a-Rjoi`pF;dW^RI9A@Xd?``=-bEQ|~1s_$94?z~pYuJ3Oe<Akifyw`7Bpce7*
zCwtAtG;3Ddwt`zXE=rg(X%vRlZR<$Q-u6KA<@uY-Wtt<^<vw*T0iDz4Ir-V~MM?sF
zWoIN`e@kmvC6vP@oAt5o%1@3?fnO;?yi6xkK+|0&Mt_{{-)K@$P**ZhRceot{ePQ-
zNr`J=!h_}4r?LrjxE(sncD~;3*9PM$&9_h8G<hmH=WkV#!R!rDYa?{T-jrFZP44g7
zWzd!MX2N?v0hU>>zJK_{tuJU;{`^_?xkH%&k$WEK-m816TExXA5j4xuIhJ?p*Io}R
z<;a^iXMBlDdZ_R(fXC7E!RPjZw3MWaj-jchT}#xh)wvh@d1vJZ@|u?RzUT=n+GKn&
zb7{!W1ePl*2Lq<e`NL$QvSQsDB@y2T1=S}%><CDmnV@m#g0PxVnMZg^q~jEm9eeqt
zq_a9#O4<0aYk0nT_3DCDu0>AUmAYl}I>lyey$$O(3Wi<+-CEM6R#?5c;8wJn<dZ3l
zJPS^Qgugv{UF6uS4-@7-UASe*1lIL6r{2w){Bq`dH($4ipI5JY^yPdiDz^MQN!#(c
zLvz=v6IZrq&!76xt*<@fi+jPYm<*SxZM&*W+@#IA8w)P@TQKr<w}H;+Jo}DgdR(z_
z>Q6nhy1g@=r=K}AcjD<5;dTXg=QnFtsK|If`jYaf^tNEgRJHy65kYE_+z+$69(_uC
zSb8fXYfbvecJ7Q#C5v}xi1%OZIRD&P;D9{GqYuv>Y|2=<@{YH&Ld(r(O4g^nPc2`+
zL@C#)_=xY)r}JJddG9)l$xdmbN@qrggw>}aql;VeG6DlaX8wA`_05!Z<E}j^J7n)a
zpZ7=7<7~x|MTty}4A$oP3z7=zccocpe!99L@v>M=BSTvN%Tm>D=UYL3ppy(t_B**u
zl3<En5oMacU&VTVo{Oqmoil4$-~zQ731`LZH%+m)lC;$G^K19e%%HRPvwu{d;C<Ss
z`2Aqagzl8A=vNmXZM~eewc<`*jj!jUP8O@Z`|cVlRhzF*?DtRlwDtL=&E96KxR(3w
zU}&!8tXg~ETj}W;{~q}$ZJL&1<MZ)A@{U)BK8EB+9GfEke37Y?XUpLXoxDx5R;yOM
z=4e|aW0=PzKCe#Z>1(^^t&;f{CHxkh(sJ&zocj5Af9m^RsXmo?W~;)wZ_YdWG(7gi
zt5)O4J!#@>YdwODir*~_H&zJm*2pTT-gM&pdG~3e0xT+yi7$_H&)k}#ef?gA@vUE9
zZ9e|7o>TWRWz*4GpZbT#Pt@=|pKH)^Q{OK&)|JQ8Lx8(=LxuN_N3XO`nr?Y|C3q^!
zeBC)mZ-2IT6>|+adE}+Aexl3ygDH~Y8?)YuRf+Yf1Z?lm-}WZ?#LVSMmo?A){t(OZ
zUf`T{s)F}7{rY#RUru&?Jo2S{M#*1a5s6t9Z~nDwN)%d6pJ&_V|9tPkXC0G^{okBk
zm0w<b$a1oz=?e7~TJIkFyH4PFU?7m2^<b{#`HdA{&9+!^h9*CL!>yP4<$%z%T7&M&
z?(ig&SsgA;FW%gYG5sl(bzLv*>n~Xqlh8d$8;kFC{+;pfMe>#7p)7M}8mq;&ElktR
zKl7jE<N|fEPzmFAI>-8@@6M`T|8|$xQ@=a=z7@}Vf8qOb!^}U1I?)!hBjxKF!xpP<
zzRr=iqeQ7Jc@<Nj$j!}(to`rr`l-k`U9;ZtV^y|f)U@B9-@N`euVayiO2qkNy4IQ}
z%QmIo7F#8K*dr*q?;N!Jy56GEw6Kz8b>O2ZA^V$yYEQIpu{YL|GUapNVczN_=6TSm
zD$Uw=)_jlGKg_32KD~hTr2_M#!|zurEt&T7)J2JpZuU~$k{d0x=BjvlOzN0)L*HT2
zf=dyx?UtJ-H0g+UDJ@>LV0(q-+cld70wO+NW&0&xDWu%azbNam(Y)$p&Z*1iFm$Ta
z)lb=cZr_}LmQu^?{%|lp_sHA(J1b1LX72a>QAa!7za;*fWBKAUd*S(($2T{BGmfjt
zwyS05J))=n(cIPY=O1p1s*L=SB$kV_4!efW_1frueD&q|DM!0ZznqkAo7J=F$KIK5
zVkb|!_(<c~o6W}!3w~VG`CY@ie!ms7Z-M!|O^Q4(CLX_b^{KtzGyU+QOC9@K{_wV>
zvS0U~*Au!f`IfN%rU!@5@BHz^QFCi2Q?F?BOylp)Z2i{#kK3nxy!q$)4C`#YzyB+2
zn}3z0{(Lo+#b4ssIo@oJ4n4MS@0eCOX^6ONQp;Fzqs-xF<>qIBRexD{y5{>@uQr&m
z`o_L;{)4`=Gjm>?-}ctJ@ayYlsjq_9qh16~<6kw&NI}(hv7ZfZU_<HrGtUmM-&TAs
z_4<YVH%{J>Ja6#(?awc|yFu3@B*YwVy&mU0P49?szwhJjskbfP-$;)9d^T#<nY%mR
z+?g!cwELs!FHI@SzG8n{v02|fp19mCQE=$y9HTUb)%J6b&n`~CT<wtG!hA@Y3$g*O
zdjG!p?)x40db-T&Smma-<BwKV*~#OJ#I4sn={Qn4$NXcEm#xxe`RX41eGz95Y^rfd
zsngtIB^=h>COg|a@Ur&&-r3t0lvivx_E*NO@Evs2?R{@tQ%t*9q>)g5?f329mrfHY
zIkAW>GN+Jl<^GeOeI`GfzU*kAc4xp75zC02TZfMX<dt9Mp1@-k`1NAC$=e&fhUVSt
z!sqm_xVla5_Df}_jGOFU?Yn|5&Q&yD-ymx}eRIM)>lS8Hl~l{EcCw<YS5Hu^t)7(H
zHF^5v&aL1h<~d4rpEa9if4myLxwzamN-Ofqy=s@kc1a&KM7J`&X@DLvUozX{$QP04
zX&=sZo&TXWGbKK@VbP&0J*^5HH{S3W8ST;NYPy)_E^_eAxzs~DAOHM)|9|I<yOZzx
z+Hr37bM}j4yj|aNhC6({;zW-{U2JT;_Ote-x^`sAx%0g`dVQ;?%Vg%De<_X&78Z%~
zE>v4pmiJ)quA)DsjEfo>_H~>&u+TZ;%aPkTw|91%X75|Nq1wLaZdiz^OIH4d8<{^l
z>^+>4mt`07Rd3I8xng>0?RkSFU(=jh8>eQ)PZHegB{22V73KK5`?XnJ9Mhin-A$SL
zE%C^NeW0yg^9xK?&#USG<&d>MFtC5#mzRNEGX-M3xhEuh_M~yEEoQu#bMWbgNBKb)
zXX^6ETTXhkC&x_ZZ`TgP$;roiRJ0EtJ8|*&r<;>Rr@YTTuy=lar~nIMTi9=xDYd3`
zE(><J>$81*-|G2BVr_JwNLSUjw1oL`H@<pSmFuU*D}Cdx-@7t{H|*C}y`-b3dTTyD
z6@RmWFW^?b#KBzc44?X&kTtgPib@ak-Z=ja{GTvi-_fLUh3c>OiYo*zRvJq3u6S9R
zka_7(j>HyC<(+4g<-VM=^5J<}wn8Dpw8EY5sOb***Y9d)G#MXHJn^Z{U6O<iN^6fs
zH|4GL(t4lx>ie?={%7T!da)iI-r#UL?R@R*@DFdejWwl>nl>$xS@k_9+xV?%Ra=kT
z@tet2OF;)#dRlg_ax#*07GRllQet8SkE)_k{WI+~x90x2Xta3M3D2FX)~2f#mBi%T
z)|sY)a&)*)#acF<Ywtv=x9U{o3z|OtI;Y~J0*_R{t^A3CR{HZ_7F}8#ZMfPl)_t*C
z;)fTPSHx_d;BWU&<mN=@z-8RI7y9ik9%<vXei=TcQDfDLqMr43e???06+AVHqPes!
zH5YRw?S2q*)=ELNXU)?ox0=#8bN_Q^X<eAbuxibM6DvbGyB00bD0$?Yx7tx#?#B1M
z-9{1HHJ4_D1_rWj^<DRDQ`O!nS>mFBQv|NY|Ks*^Ro%WsdiS=Rw6oW>4R3CKd2efn
zX!QPnM^j;orS)7ki<h5I-f*IT!`Sd&(h}b%CYg0{DvTvU_h<fn`oVY#<H-f?&$lou
zyxOAisoY<5)hegBNPDk4+!c?XN=oS2NN@K{xat16GXG`Tx?Kyo|FOwivNBs1{ZQEV
zr$B6C#G<b&FCSdL?fxpU&``tMcS<x10&j{e&OX1dW%afVPhR(ak+5mvlgo2d5c)DX
z;0uqu9n<=2=hf}(w@sV+q08s#z1W;v8j9LnO)C_Q&H0<ED;}@?`Nig0NXx3Pd+dt!
z%;T!08r|eAS)1!NoYBqiJQBI_kMq7)!41Cal23N=me_vM;fdO~v!ly;J<D`gXA#}n
z-(OQtztX?4!~Be3Q^&nGNgo>Id}<TTByHU;ZZ}zf_+M0LXyJ3Q@~Ao%rJr}J1S>1~
zt{!vJdl}lc^P%*;>VL9sa|;d2y5)N9`V(ure-^xxPO;?uqVZ`{?pdR_s$9F{zPtMN
zrN++K{zTU9-y3bVvcgaIcFQN7JaatRSbjqGPF1z33LO$9RolYfCFhsgs%`Z(UA5}b
z$=6MhGY?nG*PMCj&YyHtoqI*-nUb@g1NK+%w+a>AeAan7(|5+&`^6t7@Ezwmu_kI?
zi|<vg<@0Mwj?Z3KS$kaZ<NZ}yZnN9vmL7exBEmIqy3wY)*S@@33vM@jUhU4(BlN*L
z?WWIUwX^?u+Ltc8)A(0er?hF+su{7CbILEO%$xH~=KJ5D{V%pKafY5hDXf>`&@ZIa
z%>L<-^!uXkxB55S%#qk?6)^j<Mdp!dH^Mk3aZI`~EzVPU(dHc%FTWU9yg4cU<FW00
z#T6HdpLFiXP0LymQW@&8H1(9uJ5R@FN^;dN^d)r5N;WmPi8zV~PLbfgU(>eqbIFQT
zP4`}bMi!UUef|BWTVluOo{t=%t#|&fpZ=rhvA{20=|kM>&;EXBl)dzdzbE%ch4X$p
z=O<V=xt(P2v#DZ{Qk<f-xXE~P%Vj^Q!`;U5$uD;sJ6<S!6e*r?B|AXy<-P6ley7>L
zytypC{py7|O`Z-DR73)g&b;|_!Bf_7Gu54Xt8!I$$eOiW(z_v3*0p=eLgCPmwVlG}
z?fOqGpZDmb`}^Y=7X*7%Uetd%DPz2dzfAMg#=~X?pJ!_BQRS4#xfxUS)<Q;2w@FK3
z>t;7rM;~8KPphgD?c|GJ!YdxRa$oO}Z&TsuI6HBHY^3U(%2ONnd^k3{=HtQJ8jBi(
z)#IEzr!`v{K3yxu;5bQug>~)ClQ)ddC1m~HCqMO7bz_<&3nSx9-__GMW;CDTJT7}R
zYTwc0Ym^rKn<SU&=#asALoMs&wcYZig|e!9s}@~fa;7ZM%l$KSZ~|11FEjoppVQRz
zbZ=ya7Xt%>YKdz^NlIc#s#S7PYGO$$gOP!ug|4BIu8~2Ap`n$jft87gu7QP>fkDXU
zg`y}La`RI%(<*UmP+cdvl7WFi17t&Ha<Wx&enDkXW_m^mgENe!;8;+Qld9mEm#n9w
z0J2%%ht-jRfk7OmCndEwxhS)sBr`vc!6mb}ASba>Tzcvb1_lNOPgg&ebxsLQ054{@
Ae*gdg

literal 0
HcmV?d00001

-- 
GitLab


From 9a9f67ec10c4ad5e695df0f7d1c63a69f763fd56 Mon Sep 17 00:00:00 2001
From: Grzegorz Pawelczak <grzegorzp@graphcore.ai>
Date: Tue, 8 Jan 2019 21:37:08 +0000
Subject: [PATCH 0365/2345] Add SortSimplifier pass to compiler pipelines

---
 tensorflow/compiler/xla/service/cpu/BUILD             | 1 +
 tensorflow/compiler/xla/service/cpu/cpu_compiler.cc   | 2 ++
 tensorflow/compiler/xla/service/gpu/BUILD             | 1 +
 tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc | 2 ++
 tensorflow/compiler/xla/service/sort_simplifier.cc    | 7 ++++---
 5 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index 1eb42cf5ad..e325b75260 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -133,6 +133,7 @@ cc_library(
         "//tensorflow/compiler/xla/service:llvm_compiler",
         "//tensorflow/compiler/xla/service:reduce_precision_insertion",
         "//tensorflow/compiler/xla/service:reshape_mover",
+        "//tensorflow/compiler/xla/service:sort_simplifier",
         "//tensorflow/compiler/xla/service:transpose_folding",
         "//tensorflow/compiler/xla/service:tuple_simplifier",
         "//tensorflow/compiler/xla/service:while_loop_constant_sinking",
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
index a6d92ce10e..60e39506bf 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
@@ -93,6 +93,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/reduce_precision_insertion.h"
 #include "tensorflow/compiler/xla/service/reshape_mover.h"
 #include "tensorflow/compiler/xla/service/scatter_expander.h"
+#include "tensorflow/compiler/xla/service/sort_simplifier.h"
 #include "tensorflow/compiler/xla/service/transpose_folding.h"
 #include "tensorflow/compiler/xla/service/tuple_simplifier.h"
 #include "tensorflow/compiler/xla/service/while_loop_constant_sinking.h"
@@ -285,6 +286,7 @@ Status CpuCompiler::RunHloPassesThroughLayoutAssn(
         [](const Shape&, const Shape&) { return false; });
     options.set_enable_dot_strength_reduction(false);
     pass.AddPass<AlgebraicSimplifier>(options);
+    pass.AddPass<SortSimplifier>();
     pass.AddPass<HloDCE>();
 
     // BatchNormExpander can create zero-sized ops, so zero-sized HLO
diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD
index 5e81281bae..175f2aafc8 100644
--- a/tensorflow/compiler/xla/service/gpu/BUILD
+++ b/tensorflow/compiler/xla/service/gpu/BUILD
@@ -717,6 +717,7 @@ cc_library(
         "//tensorflow/compiler/xla/service:llvm_compiler",
         "//tensorflow/compiler/xla/service:reduce_precision_insertion",
         "//tensorflow/compiler/xla/service:reshape_mover",
+        "//tensorflow/compiler/xla/service:sort_simplifier",
         "//tensorflow/compiler/xla/service:transpose_folding",
         "//tensorflow/compiler/xla/service:tuple_simplifier",
         "//tensorflow/compiler/xla/service:while_loop_constant_sinking",
diff --git a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
index 9be7609508..e1bf0d0547 100644
--- a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
+++ b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
@@ -79,6 +79,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h"
 #include "tensorflow/compiler/xla/service/reduce_precision_insertion.h"
 #include "tensorflow/compiler/xla/service/reshape_mover.h"
+#include "tensorflow/compiler/xla/service/sort_simplifier.h"
 #include "tensorflow/compiler/xla/service/transpose_folding.h"
 #include "tensorflow/compiler/xla/service/tuple_simplifier.h"
 #include "tensorflow/compiler/xla/service/while_loop_constant_sinking.h"
@@ -200,6 +201,7 @@ Status OptimizeHloModule(HloModule* hlo_module, se::StreamExecutor* stream_exec,
           [](const Shape&, const Shape&) { return false; });
       options.set_enable_permutation_sort_replacement(true);
       pass.AddPass<AlgebraicSimplifier>(options);
+      pass.AddPass<SortSimplifier>();
       pass.AddPass<TupleSimplifier>();
       pass.AddPass<WhileLoopConstantSinking>();
       pass.AddPass<WhileLoopSimplifier>();
diff --git a/tensorflow/compiler/xla/service/sort_simplifier.cc b/tensorflow/compiler/xla/service/sort_simplifier.cc
index 3280672574..ee91762253 100644
--- a/tensorflow/compiler/xla/service/sort_simplifier.cc
+++ b/tensorflow/compiler/xla/service/sort_simplifier.cc
@@ -26,13 +26,15 @@ namespace {
 
 // If the sort instruction has a tuple shape then looks for unused output
 // values and removes them from the sort instruction. Returns true if the
-// graph have been modified.
+// graph has been modified.
 StatusOr<bool> RemoveUnusedOperandFromSort(HloInstruction* sort) {
   if (!sort->shape().IsTuple()) {
     return false;
   }
 
-  if (sort->parent()->root_instruction() == sort) {
+  HloComputation* computation = sort->parent();
+
+  if (computation->root_instruction() == sort) {
     // Can't analyse users of the root instruction.
     return false;
   }
@@ -60,7 +62,6 @@ StatusOr<bool> RemoveUnusedOperandFromSort(HloInstruction* sort) {
       new_shapes.push_back(sort->operand(i)->shape());
     }
   }
-  HloComputation* computation = sort->parent();
 
   Shape new_sort_shape = new_shapes.size() == 1
                              ? new_shapes[0]
-- 
GitLab


From 530d1b267f124fdb7dd6ce65b94423b8801152cd Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Tue, 8 Jan 2019 13:29:30 -0800
Subject: [PATCH 0366/2345] Add fine-grained BUILD files for stream executor.

PiperOrigin-RevId: 228388744
---
 tensorflow/compiler/xla/BUILD                 |   2 +-
 tensorflow/compiler/xla/service/cpu/BUILD     |   1 +
 tensorflow/compiler/xla/service/gpu/BUILD     |   1 +
 .../compiler/xla/service/interpreter/BUILD    |   2 +
 tensorflow/compiler/xrt/kernels/BUILD         |   2 +-
 tensorflow/core/kernels/BUILD                 |   3 +
 .../core/platform/default/build_config/BUILD  |  39 +-
 tensorflow/opensource_only.files              |   2 +-
 tensorflow/stream_executor/BUILD              | 703 +++++++++++++++---
 tensorflow/stream_executor/build_defs.bzl     |  11 +
 tensorflow/stream_executor/cuda/BUILD         | 365 +++++++++
 tensorflow/stream_executor/host/BUILD         | 108 +++
 tensorflow/stream_executor/lib/BUILD          |  62 ++
 tensorflow/stream_executor/platform/BUILD     |  47 ++
 .../stream_executor/platform/default/BUILD    |  25 +
 15 files changed, 1277 insertions(+), 96 deletions(-)
 create mode 100644 tensorflow/stream_executor/build_defs.bzl
 create mode 100644 tensorflow/stream_executor/cuda/BUILD
 create mode 100644 tensorflow/stream_executor/host/BUILD
 create mode 100644 tensorflow/stream_executor/lib/BUILD
 create mode 100644 tensorflow/stream_executor/platform/BUILD
 create mode 100644 tensorflow/stream_executor/platform/default/BUILD

diff --git a/tensorflow/compiler/xla/BUILD b/tensorflow/compiler/xla/BUILD
index 45623c2718..aa43fc8dab 100644
--- a/tensorflow/compiler/xla/BUILD
+++ b/tensorflow/compiler/xla/BUILD
@@ -152,7 +152,7 @@ cc_library(
         ":status",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/stream_executor",
+        "//tensorflow/stream_executor/lib",
     ],
 )
 
diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index 1eb42cf5ad..de62aa60aa 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -241,6 +241,7 @@ cc_library(
         "//tensorflow/compiler/xla/service:tuple_points_to_analysis",
         "//tensorflow/core:lib",
         "//tensorflow/core:stream_executor_no_cuda",
+        "//tensorflow/stream_executor/host:host_stream",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/strings:str_format",
         "@com_google_absl//absl/types:span",
diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD
index 30a8bda606..2be5d918ee 100644
--- a/tensorflow/compiler/xla/service/gpu/BUILD
+++ b/tensorflow/compiler/xla/service/gpu/BUILD
@@ -731,6 +731,7 @@ cc_library(
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:regexp_internal",
         "//tensorflow/core:stream_executor_no_cuda",
+        "//tensorflow/stream_executor/cuda:cuda_diagnostics",
         "@com_google_absl//absl/container:node_hash_map",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
diff --git a/tensorflow/compiler/xla/service/interpreter/BUILD b/tensorflow/compiler/xla/service/interpreter/BUILD
index 2e1d2cfbda..9b4cafa4b4 100644
--- a/tensorflow/compiler/xla/service/interpreter/BUILD
+++ b/tensorflow/compiler/xla/service/interpreter/BUILD
@@ -119,6 +119,8 @@ cc_library(
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/core:lib",
         "//tensorflow/core:stream_executor_headers_lib",
+        "//tensorflow/stream_executor/host:host_stream",
+        "//tensorflow/stream_executor/host:host_timer",
         "@com_google_absl//absl/types:span",
     ],
 )
diff --git a/tensorflow/compiler/xrt/kernels/BUILD b/tensorflow/compiler/xrt/kernels/BUILD
index 67f475846e..78d093ec4a 100644
--- a/tensorflow/compiler/xrt/kernels/BUILD
+++ b/tensorflow/compiler/xrt/kernels/BUILD
@@ -62,7 +62,7 @@ cc_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
-        "//tensorflow/stream_executor:stream_executor_headers_lib",
+        "//tensorflow/stream_executor:stream_executor_headers",
         "@com_google_absl//absl/strings",
     ],
     alwayslink = 1,
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index aaea09bdc4..5794ba520e 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -2313,6 +2313,7 @@ tf_kernel_library(
         "//tensorflow/core/grappler/clusters:virtual_cluster",
         "//tensorflow/core/grappler/optimizers:meta_optimizer",
         "//tensorflow/core/grappler/utils:functions",
+        "//tensorflow/stream_executor:stream",
     ],
 )
 
@@ -3819,6 +3820,8 @@ tf_kernel_library(
     deps = NN_DEPS + if_cuda([
         ":reduction_ops",
         "@cub_archive//:cub",
+        "//tensorflow/core:stream_executor",
+        "//tensorflow/stream_executor/cuda:cuda_stream",
     ]),
 )
 
diff --git a/tensorflow/core/platform/default/build_config/BUILD b/tensorflow/core/platform/default/build_config/BUILD
index da1f66dc67..f5a194428c 100644
--- a/tensorflow/core/platform/default/build_config/BUILD
+++ b/tensorflow/core/platform/default/build_config/BUILD
@@ -32,15 +32,25 @@ cc_library(
 
 tf_cuda_library(
     name = "stream_executor",
-    deps = [
-        "//tensorflow/stream_executor",
+    cuda_deps = [
+        "//tensorflow/stream_executor/cuda:cuda_activation",
     ] + select({
-        "//tensorflow:using_cuda_clang": ["//tensorflow/stream_executor:cuda_platform"],
-        "//tensorflow:using_cuda_nvcc": ["//tensorflow/stream_executor:cuda_platform"],
+        "//tensorflow:using_cuda_clang": ["//tensorflow/stream_executor/cuda:all_runtime"],
+        "//tensorflow:using_cuda_nvcc": ["//tensorflow/stream_executor/cuda:all_runtime"],
         "//tensorflow:using_cuda_clang_with_dynamic_build": [],
         "//tensorflow:using_cuda_nvcc_with_dynamic_build": [],
         "//conditions:default": [],
-    }) + select({
+    }),
+    deps = [
+        "//tensorflow/stream_executor",
+        "//tensorflow/stream_executor:dnn",
+        "//tensorflow/stream_executor:event",
+        "//tensorflow/stream_executor:multi_platform_manager",
+        "//tensorflow/stream_executor:scratch_allocator",
+        "//tensorflow/stream_executor/cuda:cuda_platform_id",
+        "//tensorflow/stream_executor/host:host_platform_id",
+        "//tensorflow/stream_executor/platform:dso_loader",
+    ] + select({
         "@local_config_cuda//cuda:darwin": ["IOKit"],
         "//conditions:default": [],
     }),
@@ -49,9 +59,10 @@ tf_cuda_library(
 cc_library(
     name = "stream_executor_cuda",
     deps = [
-        "//tensorflow/stream_executor",
+        ":stream_executor_no_cuda",
+        ":cuda",
     ] + if_static(
-        ["//tensorflow/stream_executor:cuda_platform"],
+        ["//tensorflow/stream_executor/cuda:all_runtime"],
     ) + select({
         "@local_config_cuda//cuda:darwin": ["IOKit"],
         "//conditions:default": [],
@@ -62,23 +73,31 @@ cc_library(
     name = "stream_executor_no_cuda",
     deps = [
         "//tensorflow/stream_executor",
+        "//tensorflow/stream_executor:dnn",
+        "//tensorflow/stream_executor:event",
+        "//tensorflow/stream_executor:multi_platform_manager",
+        "//tensorflow/stream_executor:scratch_allocator",
+        "//tensorflow/stream_executor/cuda:cuda_platform_id",
+        "//tensorflow/stream_executor/host:host_platform",
+        "//tensorflow/stream_executor/host:host_platform_id",
+        "//tensorflow/stream_executor/platform:dso_loader",
     ],
 )
 
 # Dummy stream executor cuda plugins.
 cc_library(
     name = "cublas_plugin",
-    srcs = [],
+    deps = if_static(["//tensorflow/stream_executor/cuda:cublas_plugin"]),
 )
 
 cc_library(
     name = "cufft_plugin",
-    srcs = [],
+    deps = if_static(["//tensorflow/stream_executor/cuda:cufft_plugin"]),
 )
 
 cc_library(
     name = "cudnn_plugin",
-    srcs = [],
+    deps = if_static(["//tensorflow/stream_executor/cuda:cudnn_plugin"]),
 )
 
 # OSX framework for device driver access
diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index e00d063cfe..bade64dcf8 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -241,7 +241,7 @@ tensorflow/third_party/tflite_ovic_testdata.BUILD
 tensorflow/third_party/libxsmm.BUILD
 tensorflow/third_party/zlib.BUILD
 tensorflow/third_party/eigen.BUILD
-tensorflow/stream_executor/BUILD
+tensorflow/stream_executor/build_defs.bzl
 tensorflow/api_template_v1.__init__.py
 tensorflow/compat_template_v1.__init__.py
 tensorflow/api_template.__init__.py
diff --git a/tensorflow/stream_executor/BUILD b/tensorflow/stream_executor/BUILD
index 3c45bf314a..7cf7e75a55 100644
--- a/tensorflow/stream_executor/BUILD
+++ b/tensorflow/stream_executor/BUILD
@@ -1,126 +1,663 @@
-licenses(["restricted"])
+# GPU executor library for data-parallel kernel launches and cross-platform
+# HPC-library APIs.
+#
+# Throughout this file, all targets are built with the standard crosstool and
+# do not link against restricted binary blobs.
 
-load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda_is_configured")
+licenses(["notice"])  # Apache 2.0
+
+load("//tensorflow:tensorflow.bzl", "tf_cc_test")
 load("//tensorflow/core:platform/default/build_config.bzl", "tf_proto_library")
 load("//tensorflow/core:platform/default/build_config_root.bzl", "if_static")
-load("//tensorflow:tensorflow.bzl", "cc_header_only_library")
+load("//tensorflow/stream_executor:build_defs.bzl", "stream_executor_friends")
+
+package_group(
+    name = "friends",
+    packages = stream_executor_friends(),
+)
+
+package(
+    default_visibility = [":friends"],
+)
+
+# Filegroup used to collect source files for the dependency check.
+filegroup(
+    name = "c_srcs",
+    data = glob([
+        "**/*.cc",
+        "**/*.h",
+    ]),
+)
+
+cc_library(
+    name = "launch_dim",
+    hdrs = [
+        "gpu_launch_dim.h",
+        "launch_dim.h",
+    ],
+    deps = [
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "device_description",
+    srcs = ["device_description.cc"],
+    hdrs = ["device_description.h"],
+    deps = [
+        ":launch_dim",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/base:core_headers",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "event",
+    srcs = [
+        "blas.h",
+        "device_description.h",
+        "device_options.h",
+        "dnn.h",
+        "event.cc",
+        "fft.h",
+        "kernel_cache_config.h",
+        "launch_dim.h",
+        "plugin.h",
+        "plugin_registry.h",
+        "rng.h",
+        "shared_memory_config.h",
+        "stream_executor_pimpl.h",
+        "temporary_device_memory.h",
+        "temporary_memory_manager.h",
+        "trace_listener.h",
+    ],
+    hdrs = [
+        "device_memory.h",
+        "event.h",
+        "kernel.h",
+        "kernel_spec.h",
+        "platform.h",
+        "stream.h",
+        "stream_executor_internal.h",
+    ],
+    deps = [
+        ":dnn_proto_cc",
+        ":host_or_device_scalar",
+        ":stream_executor_headers",
+        "//tensorflow/core:lib",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/base:core_headers",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:optional",
+        "@com_google_absl//absl/types:span",
+    ],
+)
+
+cc_library(
+    name = "kernel",
+    srcs = [
+        "dnn.h",
+        "fft.h",
+        "kernel.cc",
+        "plugin.h",
+        "rng.h",
+        "stream.h",
+        "stream_executor_pimpl.h",
+        "temporary_device_memory.h",
+        "temporary_memory_manager.h",
+    ],
+    hdrs = [
+        "blas.h",
+        "device_description.h",
+        "device_options.h",
+        "event.h",
+        "kernel.h",
+        "kernel_spec.h",
+        "launch_dim.h",
+        "multi_platform_manager.h",
+        "platform.h",
+        "plugin_registry.h",
+        "shared_memory_config.h",
+        "stream_executor.h",
+        "stream_executor_internal.h",
+        "timer.h",
+        "trace_listener.h",
+    ],
+    deps = [
+        ":device_memory",
+        ":dnn_proto_cc",
+        ":host_or_device_scalar",
+        ":kernel_cache_config",
+        ":stream_executor_headers",
+        "//tensorflow/core:lib",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/base:core_headers",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:optional",
+        "@com_google_absl//absl/types:span",
+    ],
+)
 
-STREAM_EXECUTOR_HEADERS = glob([
-    "*.h",
-    "cuda/*.h",
-    "host/*.h",
-    "lib/*.h",
-    "lib/gtl/*.h",
-    "platform/**/*.h",
-])
+cc_library(
+    name = "kernel_spec",
+    srcs = ["kernel_spec.cc"],
+    hdrs = ["kernel_spec.h"],
+    deps = [
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "kernel_cache_config",
+    hdrs = ["kernel_cache_config.h"],
+)
+
+cc_library(
+    name = "module_spec",
+    hdrs = ["module_spec.h"],
+    deps = [
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+    ],
+)
+
+cc_library(
+    name = "shared_memory_config",
+    hdrs = ["shared_memory_config.h"],
+)
+
+cc_library(
+    name = "stream_header",
+    hdrs = [
+        "blas.h",
+        "device_memory.h",
+        "dnn.h",
+        "event.h",
+        "fft.h",
+        "gpu_launch_dim.h",
+        "kernel.h",
+        "kernel_cache_config.h",
+        "launch_dim.h",
+        "stream.h",
+        "temporary_device_memory.h",
+        "temporary_memory_manager.h",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":dnn_proto_cc",
+        ":host_or_device_scalar",
+        ":stream_executor_headers",
+        "//tensorflow/core:lib",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:optional",
+        "@com_google_absl//absl/types:span",
+    ],
+)
+
+# It implements :stream_header
+cc_library(
+    name = "stream",
+    srcs = [
+        "host_buffer.h",
+        "stream.cc",
+    ],
+    hdrs = ["stream.h"],
+    deps = [
+        ":blas",
+        ":device_memory",
+        ":dnn",
+        ":event",
+        ":fft",
+        ":host_or_device_scalar",
+        ":kernel",
+        ":launch_dim",
+        ":platform",
+        ":rng",
+        ":stream_executor_headers",
+        ":stream_executor_internal",
+        ":stream_executor_pimpl",
+        ":temporary_memory_manager",
+        "//tensorflow/core:lib",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "//third_party/eigen3",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "timer",
+    srcs = [
+        "device_description.h",
+        "kernel_cache_config.h",
+        "timer.cc",
+    ],
+    hdrs = [
+        "blas.h",
+        "kernel.h",
+        "stream.h",
+        "stream_executor.h",
+        "timer.h",
+    ],
+    deps = [
+        ":host_or_device_scalar",
+        ":platform",
+        ":stream_executor_headers",
+        ":stream_executor_internal",
+        ":stream_executor_pimpl_header",
+        "//tensorflow/core:lib",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/base:core_headers",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "platform",
+    srcs = ["platform.cc"],
+    hdrs = ["platform.h"],
+    deps = [
+        ":plugin",
+        ":stream_executor_headers",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "rng",
+    srcs = ["rng.cc"],
+    hdrs = ["rng.h"],
+    deps = ["//tensorflow/stream_executor/platform"],
+)
+
+cc_library(
+    name = "temporary_device_memory",
+    srcs = [
+        "event.h",
+        "temporary_device_memory.cc",
+        "temporary_memory_manager.h",
+    ],
+    hdrs = ["temporary_device_memory.h"],
+    deps = [
+        ":device_memory",
+        ":stream_header",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+    ],
+)
+
+cc_library(
+    name = "temporary_memory_manager",
+    srcs = ["temporary_memory_manager.cc"],
+    hdrs = ["temporary_memory_manager.h"],
+    deps = [
+        ":device_memory",
+        ":stream_executor_pimpl_header",
+        ":stream_header",
+        ":temporary_device_memory",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "fft",
+    hdrs = ["fft.h"],
+    deps = [
+        "//tensorflow/stream_executor/platform",
+    ],
+)
+
+cc_library(
+    name = "blas",
+    srcs = ["blas.cc"],
+    hdrs = ["blas.h"],
+    deps = [
+        ":host_or_device_scalar",
+        ":stream_executor_headers",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "device_memory",
+    hdrs = ["device_memory.h"],
+    deps = ["//tensorflow/stream_executor/platform"],
+)
+
+cc_library(
+    name = "host_or_device_scalar",
+    hdrs = ["host_or_device_scalar.h"],
+    deps = [
+        ":device_memory",
+        "//tensorflow/stream_executor/platform",
+    ],
+)
+
+cc_library(
+    name = "device_options",
+    hdrs = ["device_options.h"],
+    deps = [
+        "//tensorflow/stream_executor/platform",
+    ],
+)
+
+cc_library(
+    name = "executor_cache",
+    srcs = [
+        "device_description.h",
+        "device_memory.h",
+        "device_options.h",
+        "event.h",
+        "executor_cache.cc",
+        "launch_dim.h",
+        "plugin.h",
+        "plugin_registry.h",
+        "rng.h",
+        "stream_executor_pimpl.h",
+        "temporary_device_memory.h",
+        "temporary_memory_manager.h",
+    ],
+    hdrs = [
+        "blas.h",
+        "dnn.h",
+        "executor_cache.h",
+        "fft.h",
+        "kernel.h",
+        "kernel_cache_config.h",
+        "kernel_spec.h",
+        "platform.h",
+        "shared_memory_config.h",
+        "stream.h",
+        "stream_executor_internal.h",
+        "trace_listener.h",
+    ],
+    deps = [
+        ":dnn_proto_cc",
+        ":host_or_device_scalar",
+        ":stream_executor_headers",
+        "//tensorflow/core:lib",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/base:core_headers",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:optional",
+        "@com_google_absl//absl/types:span",
+    ],
+)
+
+cc_library(
+    name = "multi_platform_manager",
+    srcs = ["multi_platform_manager.cc"],
+    hdrs = ["multi_platform_manager.h"],
+    deps = [
+        ":platform",
+        ":stream_executor_headers",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/base:core_headers",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/synchronization",
+    ],
+)
+
+cc_library(
+    name = "plugin",
+    srcs = ["plugin.cc"],
+    hdrs = ["plugin.h"],
+)
+
+cc_library(
+    name = "plugin_registry",
+    srcs = ["plugin_registry.cc"],
+    hdrs = ["plugin_registry.h"],
+    deps = [
+        ":blas",
+        ":dnn",
+        ":fft",
+        ":multi_platform_manager",
+        ":platform",
+        ":plugin",
+        ":stream_executor_headers",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/base:core_headers",
+    ],
+)
+
+cc_library(
+    name = "scratch_allocator",
+    srcs = ["scratch_allocator.cc"],
+    hdrs = ["scratch_allocator.h"],
+    deps = [
+        ":device_memory",
+        ":stream_header",
+        ":temporary_device_memory",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+    ],
+)
+
+cc_library(
+    name = "host_buffer",
+    hdrs = ["host_buffer.h"],
+    deps = [":dnn"],
+)
 
 tf_proto_library(
     name = "dnn_proto",
     srcs = ["dnn.proto"],
     cc_api_version = 2,
     default_header = True,
+    provide_cc_alias = True,
 )
 
 tf_proto_library(
     name = "logging_proto",
     srcs = ["logging.proto"],
     cc_api_version = 2,
-    default_header = True,
     protodeps = [":dnn_proto"],
+    provide_cc_alias = True,
+    visibility = [":friends"],
 )
 
 cc_library(
-    name = "stream_executor_impl",
-    srcs = glob(
-        [
-            "*.cc",
-            "host/*.cc",
-            "cuda/cuda_platform_id.cc",
-            "lib/*.cc",
-            "platform/default/*.cc",
-        ],
-        exclude = [
-            "**/*_test.cc",
-        ],
-    ),
-    hdrs = STREAM_EXECUTOR_HEADERS,
-    linkopts = select({
-        "//tensorflow:freebsd": [],
-        "//tensorflow:windows": [],
-        "//conditions:default": ["-ldl"],
-    }),
-    visibility = ["//visibility:public"],
+    name = "dnn",
+    srcs = ["dnn.cc"],
+    hdrs = ["dnn.h"],
     deps = [
-        ":dnn_proto_cc_impl",
-        ":logging_proto_cc_impl",
+        ":device_memory",
+        ":dnn_proto_cc",
+        ":stream_executor_headers",
         "//tensorflow/core:lib",
-        "//tensorflow/core:logger",
-        "//tensorflow/core:ptr_util",
-        "@com_google_absl//absl/container:flat_hash_map",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
         "@com_google_absl//absl/strings",
-        "@com_google_absl//absl/strings:str_format",
-        "@com_google_absl//absl/synchronization",
-        "@local_config_cuda//cuda:cuda_headers",
+        "@com_google_absl//absl/types:optional",
+        "@com_google_absl//absl/types:span",
     ],
-    alwayslink = 1,
 )
 
 cc_library(
-    name = "stream_executor",
-    hdrs = STREAM_EXECUTOR_HEADERS,
+    name = "stream_executor_internal",
+    srcs = [
+        "dnn.h",
+        "stream_executor_internal.cc",
+    ],
+    hdrs = [
+        "shared_memory_config.h",
+        "stream_executor_internal.h",
+    ],
+    deps = [
+        ":device_description",
+        ":device_memory",
+        ":device_options",
+        ":dnn_proto_cc",
+        ":kernel",
+        ":kernel_cache_config",
+        ":kernel_spec",
+        ":launch_dim",
+        ":plugin_registry",
+        ":stream_executor_headers",
+        "//tensorflow/core:lib",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/types:optional",
+        "@com_google_absl//absl/types:span",
+    ],
+)
+
+cc_library(
+    name = "stream_executor_pimpl_header",
+    hdrs = [
+        "device_description.h",
+        "dnn.h",
+        "kernel.h",
+        "kernel_cache_config.h",
+        "shared_memory_config.h",
+        "stream_executor_pimpl.h",
+    ],
     visibility = ["//visibility:public"],
     deps = [
         ":dnn_proto_cc",
-        ":logging_proto_cc",
+        ":platform",
+        ":stream_executor_headers",
+        ":stream_executor_internal",
         "//tensorflow/core:lib",
-        "//tensorflow/core:ptr_util",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/base:core_headers",
         "@com_google_absl//absl/strings",
-        "@local_config_cuda//cuda:cuda_headers",
-    ] + if_static([":stream_executor_impl"]),
+        "@com_google_absl//absl/types:optional",
+        "@com_google_absl//absl/types:span",
+    ],
 )
 
-cc_header_only_library(
-    name = "stream_executor_headers_lib",
-    visibility = ["//visibility:public"],
+# It implements :stream_executor_pimpl_header
+cc_library(
+    name = "stream_executor_pimpl",
+    srcs = ["stream_executor_pimpl.cc"],
+    hdrs = ["stream_executor_pimpl.h"],
     deps = [
-        ":stream_executor",
+        ":blas",
+        ":executor_cache",
+        ":fft",
+        ":kernel",
+        ":platform",
+        ":rng",
+        ":stream_executor_headers",
+        ":stream_header",
+        ":timer",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/base:core_headers",
+        "@com_google_absl//absl/strings",
     ],
 )
 
+# The stream_executor_headers target does not prescribe an implementation.
+#
+# TODO(b/25131218) this is OBSOLETE/DEPRECATED -- get rid of this target altogether
 cc_library(
-    name = "cuda_platform",
-    srcs = if_cuda_is_configured(
-        glob(
-            [
-                "cuda/*.cc",
-            ],
-            exclude = [
-                "cuda/*_test.cc",
-                "cuda/cuda_platform_id.cc",
-            ],
-        ),
-    ),
-    copts = select({
-        "//tensorflow:windows": ["/DNOGDI"],
-        "//conditions:default": [],
-    }),
-    data = [
-        "@local_config_cuda//cuda:cudnn",
-    ],
-    linkopts = select({
-        "//tensorflow:freebsd": [],
-        "//tensorflow:windows": [],
-        "//conditions:default": ["-ldl"],
-    }),
+    name = "stream_executor_headers",
+    hdrs = [
+        "blas.h",
+        "device_description.h",
+        "device_memory.h",
+        "device_options.h",
+        "dnn.h",
+        "event.h",
+        "executor_cache.h",
+        "fft.h",
+        "gpu_launch_dim.h",
+        "kernel.h",
+        "kernel_cache_config.h",
+        "kernel_spec.h",
+        "launch_dim.h",
+        "module_spec.h",
+        "multi_platform_manager.h",
+        "platform.h",
+        "plugin.h",
+        "plugin_registry.h",
+        "rng.h",
+        "shared_memory_config.h",
+        "stream.h",
+        "stream_executor.h",
+        "stream_executor_internal.h",
+        "stream_executor_pimpl.h",
+        "temporary_device_memory.h",
+        "temporary_memory_manager.h",
+        "timer.h",
+        "trace_listener.h",
+    ],
     visibility = ["//visibility:public"],
     deps = [
-        ":stream_executor",
+        ":dnn_proto_cc",
+        ":host_or_device_scalar",
         "//tensorflow/core:lib",
-        "//tensorflow/core/kernels:ops_util",
-        "@local_config_cuda//cuda:cuda_headers",
-    ] + if_cuda_is_configured([
-        "//tensorflow/core:cuda",
-        "@local_config_cuda//cuda:cuda_driver",
-    ]),
-    alwayslink = 1,
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/base:core_headers",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:optional",
+        "@com_google_absl//absl/types:span",
+    ],
+)
+
+cc_library(
+    name = "stream_executor",
+    hdrs = ["stream_executor.h"],
+    deps = [":stream_executor_headers"] + if_static([":stream_executor_impl"]),
+)
+
+cc_library(
+    name = "stream_executor_impl",
+    deps = [
+        ":device_description",
+        ":device_memory",
+        ":dnn_proto_cc",
+        ":dnn_proto_cc_impl",
+        ":event",
+        ":kernel",
+        ":launch_dim",
+        ":multi_platform_manager",
+        ":platform",
+        ":stream",
+        ":stream_executor_headers",
+        ":stream_executor_pimpl",
+        ":timer",
+    ],
+)
+
+tf_cc_test(
+    name = "stream_test",
+    size = "small",
+    srcs = ["stream_test.cc"],
+    deps = [
+        ":stream_executor",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/stream_executor/host:host_platform",
+    ],
+)
+
+alias(
+    name = "cuda_platform",
+    actual = "//tensorflow/stream_executor/cuda:all_runtime",
 )
diff --git a/tensorflow/stream_executor/build_defs.bzl b/tensorflow/stream_executor/build_defs.bzl
new file mode 100644
index 0000000000..a7ddf5a0d7
--- /dev/null
+++ b/tensorflow/stream_executor/build_defs.bzl
@@ -0,0 +1,11 @@
+def stream_executor_friends():
+    return ["//tensorflow/..."]
+
+def tf_additional_cuda_platform_deps():
+  return []
+
+def tf_additional_cuda_driver_deps():
+  return ["@local_config_cuda//cuda:cuda_driver"]
+
+def tf_additional_cudnn_plugin_deps():
+  return []
diff --git a/tensorflow/stream_executor/cuda/BUILD b/tensorflow/stream_executor/cuda/BUILD
new file mode 100644
index 0000000000..58c9480884
--- /dev/null
+++ b/tensorflow/stream_executor/cuda/BUILD
@@ -0,0 +1,365 @@
+# Description:
+#   CUDA-platform specific StreamExecutor support code.
+
+licenses(["notice"])  # Apache 2.0
+
+load("//tensorflow:tensorflow.bzl", "tf_cc_test")
+load(
+    "//tensorflow/stream_executor:build_defs.bzl",
+    "stream_executor_friends",
+    "tf_additional_cuda_driver_deps",
+    "tf_additional_cuda_platform_deps",
+    "tf_additional_cudnn_plugin_deps",
+)
+load("//tensorflow:tensorflow.bzl", "tf_copts")
+load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda_is_configured")
+load("//tensorflow/core:platform/default/build_config_root.bzl", "if_static")
+
+package_group(
+    name = "friends",
+    packages = stream_executor_friends(),
+)
+
+package(
+    default_visibility = [":friends"],
+)
+
+# Filegroup used to collect source files for the dependency check.
+filegroup(
+    name = "c_srcs",
+    data = glob([
+        "**/*.cc",
+        "**/*.h",
+    ]),
+)
+
+cc_library(
+    name = "cuda_platform_id",
+    srcs = ["cuda_platform_id.cc"],
+    hdrs = ["cuda_platform_id.h"],
+    deps = ["//tensorflow/stream_executor:platform"],
+)
+
+cc_library(
+    name = "cuda_platform",
+    srcs = ["cuda_platform.cc"],
+    hdrs = ["cuda_platform.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":cuda_driver",
+        ":cuda_gpu_executor",
+        ":cuda_platform_id",
+        "//tensorflow/stream_executor",  # buildcleaner: keep
+        "//tensorflow/stream_executor:executor_cache",
+        "//tensorflow/stream_executor:multi_platform_manager",
+        "//tensorflow/stream_executor:stream_executor_pimpl_header",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+    ] + tf_additional_cuda_platform_deps(),
+    alwayslink = True,  # Registers itself with the MultiPlatformManager.
+)
+
+cc_library(
+    name = "cuda_diagnostics",
+    srcs = ["cuda_diagnostics.cc"],
+    hdrs = ["cuda_diagnostics.h"],
+    deps = [
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/container:inlined_vector",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "cuda_driver",
+    srcs = ["cuda_driver.cc"],
+    hdrs = ["cuda_driver.h"],
+    deps = [
+        ":cuda_diagnostics",
+        "@com_google_absl//absl/base",
+        "@com_google_absl//absl/container:inlined_vector",
+        "@com_google_absl//absl/strings",
+        "@local_config_cuda//cuda:cuda_headers",
+        "//tensorflow/stream_executor:device_options",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+    ] + tf_additional_cuda_driver_deps(),
+)
+
+# The activation library is tightly coupled to the executor library.
+# TODO(leary) split up cuda_gpu_executor.cc so that this can stand alone.
+cc_library(
+    name = "cuda_activation_header",
+    hdrs = ["cuda_activation.h"],
+    visibility = ["//visibility:public"],
+    deps = ["//tensorflow/stream_executor/platform"],
+)
+
+cc_library(
+    name = "cuda_activation",
+    srcs = ["cuda_activation.cc"],
+    hdrs = ["cuda_activation.h"],
+    deps = [
+        ":cuda_driver",
+        "//tensorflow/stream_executor",
+        "//tensorflow/stream_executor:stream_executor_internal",
+        "//tensorflow/stream_executor/platform",
+        "@local_config_cuda//cuda:cuda_headers",
+    ],
+)
+
+cc_library(
+    name = "cuda_gpu_executor_header",
+    textual_hdrs = ["cuda_gpu_executor.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":cuda_kernel",
+        "//tensorflow/stream_executor:event",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+    ],
+)
+
+cc_library(
+    name = "cublas_plugin",
+    srcs = ["cuda_blas.cc"],
+    hdrs = ["cuda_blas.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":cuda_activation",
+        ":cuda_gpu_executor",
+        ":cuda_helpers",
+        ":cuda_platform_id",
+        ":cuda_stream",
+        ":cuda_timer",
+        "@com_google_absl//absl/strings",
+        "//third_party/eigen3",
+        "@local_config_cuda//cuda:cuda_headers",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/stream_executor",
+        "//tensorflow/stream_executor:event",
+        "//tensorflow/stream_executor:host_or_device_scalar",
+        "//tensorflow/stream_executor:plugin_registry",
+        "//tensorflow/stream_executor:scratch_allocator",
+        "//tensorflow/stream_executor:timer",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "//tensorflow/stream_executor/platform:dso_loader",
+    ] + if_static(
+        [
+            "@local_config_cuda//cuda:cublas",
+            "@local_config_cuda//cuda:cudart_static",
+        ],
+        ["@local_config_cuda//cuda:cudart"],
+    ),
+    alwayslink = True,
+)
+
+cc_library(
+    name = "cufft_plugin",
+    srcs = ["cuda_fft.cc"],
+    hdrs = ["cuda_fft.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":cuda_activation_header",
+        ":cuda_gpu_executor_header",
+        ":cuda_helpers",
+        ":cuda_platform_id",
+        ":cuda_stream",
+        "@local_config_cuda//cuda:cuda_headers",
+        "//tensorflow/stream_executor:event",
+        "//tensorflow/stream_executor:fft",
+        "//tensorflow/stream_executor:plugin_registry",
+        "//tensorflow/stream_executor:scratch_allocator",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "//tensorflow/stream_executor/platform:dso_loader",
+    ] + if_static(["@local_config_cuda//cuda:cufft"]),
+    alwayslink = True,
+)
+
+cc_library(
+    name = "cudnn_plugin",
+    srcs = ["cuda_dnn.cc"],
+    hdrs = ["cuda_dnn.h"],
+    copts = [
+        # STREAM_EXECUTOR_CUDNN_WRAP would fail on Clang with the default
+        # setting of template depth 256
+        "-ftemplate-depth-512",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":cuda_activation",
+        ":cuda_diagnostics",
+        ":cuda_driver",
+        ":cuda_gpu_executor",
+        ":cuda_platform_id",
+        ":cuda_stream",
+        ":cuda_timer",
+        ":cudnn_version",
+        "@com_google_absl//absl/strings",
+        "//third_party/eigen3",
+        "@local_config_cuda//cuda:cuda_headers",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core:logger",
+        "//tensorflow/stream_executor:dnn",
+        "//tensorflow/stream_executor:event",
+        "//tensorflow/stream_executor:logging_proto_cc",
+        "//tensorflow/stream_executor:plugin_registry",
+        "//tensorflow/stream_executor:scratch_allocator",
+        "//tensorflow/stream_executor:stream_executor_pimpl_header",
+        "//tensorflow/stream_executor:temporary_device_memory",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "//tensorflow/stream_executor/platform:dso_loader",
+    ] + tf_additional_cudnn_plugin_deps() + if_static(
+        [
+            "@local_config_cuda//cuda:cudnn",
+            "@local_config_cuda//cuda:cudart_static",
+        ],
+        ["@local_config_cuda//cuda:cudart"],
+    ),
+    alwayslink = True,
+)
+
+cc_library(
+    name = "curand_plugin",
+    srcs = ["cuda_rng.cc"],
+    hdrs = ["cuda_rng.h"],
+    deps = [
+        ":cuda_activation",
+        ":cuda_gpu_executor",
+        ":cuda_helpers",
+        ":cuda_platform_id",
+        ":cuda_stream",
+        "@local_config_cuda//cuda:cuda_headers",
+        "//tensorflow/stream_executor:event",
+        "//tensorflow/stream_executor:plugin_registry",
+        "//tensorflow/stream_executor:rng",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "//tensorflow/stream_executor/platform:dso_loader",
+    ] + if_static(["@local_config_cuda//cuda:curand"]),
+    alwayslink = True,
+)
+
+cc_library(
+    name = "cuda_kernel",
+    hdrs = ["cuda_kernel.h"],
+    deps = [
+        ":cuda_driver",
+        "//tensorflow/stream_executor:event",
+        "//tensorflow/stream_executor:stream_executor_pimpl_header",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@local_config_cuda//cuda:cuda_headers",
+    ],
+)
+
+# TODO(leary) we likely need to canonicalize/eliminate this.
+cc_library(
+    name = "cuda_helpers",
+    textual_hdrs = ["cuda_helpers.h"],
+)
+
+cc_library(
+    name = "cuda_event",
+    srcs = ["cuda_event.cc"],
+    hdrs = ["cuda_event.h"],
+    deps = [
+        ":cuda_driver",
+        ":cuda_gpu_executor_header",
+        ":cuda_stream",
+        "//tensorflow/stream_executor:stream_executor_headers",
+        "//tensorflow/stream_executor/lib",
+    ],
+)
+
+cc_library(
+    name = "cuda_stream",
+    srcs = ["cuda_stream.cc"],
+    hdrs = ["cuda_stream.h"],
+    deps = [
+        ":cuda_driver",
+        ":cuda_gpu_executor_header",
+        "//tensorflow/stream_executor:stream_executor_headers",
+        "//tensorflow/stream_executor:stream_header",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+    ],
+)
+
+cc_library(
+    name = "cuda_timer",
+    srcs = ["cuda_timer.cc"],
+    hdrs = ["cuda_timer.h"],
+    deps = [
+        ":cuda_driver",
+        ":cuda_gpu_executor_header",
+        ":cuda_stream",
+        "//tensorflow/stream_executor:stream_executor_headers",
+        "//tensorflow/stream_executor/lib",
+    ],
+)
+
+# It implements :cuda_gpu_executor_header
+cc_library(
+    name = "cuda_gpu_executor",
+    srcs = ["cuda_gpu_executor.cc"],
+    hdrs = ["cuda_gpu_executor.h"],
+    deps = [
+        ":cuda_activation",
+        ":cuda_diagnostics",
+        ":cuda_driver",
+        ":cuda_event",
+        ":cuda_kernel",
+        ":cuda_platform_id",
+        ":cuda_stream",
+        ":cuda_timer",
+        "//tensorflow/stream_executor:event",
+        "//tensorflow/stream_executor:plugin_registry",
+        "//tensorflow/stream_executor:stream_executor_internal",
+        "//tensorflow/stream_executor:stream_executor_pimpl_header",
+        "//tensorflow/stream_executor:timer",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/strings",
+    ],
+    alwayslink = True,
+)
+
+cc_library(
+    name = "cudnn_version",
+    srcs = ["cudnn_version.cc"],
+    hdrs = ["cudnn_version.h"],
+    deps = [
+        "//tensorflow/core:lib",
+    ],
+)
+
+tf_cc_test(
+    name = "cudnn_version_test",
+    srcs = ["cudnn_version_test.cc"],
+    deps = [
+        ":cudnn_version",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
+cc_library(
+    name = "all_runtime",
+    copts = tf_copts(),
+    visibility = ["//visibility:public"],
+    deps = if_cuda_is_configured([
+        ":cudnn_plugin",
+        ":cufft_plugin",
+        ":cublas_plugin",
+        ":curand_plugin",
+        ":cuda_driver",
+        ":cuda_platform",
+    ]),
+    alwayslink = 1,
+)
diff --git a/tensorflow/stream_executor/host/BUILD b/tensorflow/stream_executor/host/BUILD
new file mode 100644
index 0000000000..59472b14c1
--- /dev/null
+++ b/tensorflow/stream_executor/host/BUILD
@@ -0,0 +1,108 @@
+# Description:
+#   Host-platform specific StreamExecutor support code.
+
+licenses(["notice"])  # Apache 2.0
+
+load("//tensorflow/stream_executor:build_defs.bzl", "stream_executor_friends")
+
+package_group(
+    name = "friends",
+    packages = stream_executor_friends(),
+)
+
+package(default_visibility = [":friends"])
+
+# Filegroup used to collect source files for the dependency check.
+filegroup(
+    name = "c_srcs",
+    data = glob([
+        "**/*.cc",
+        "**/*.h",
+    ]),
+)
+
+cc_library(
+    name = "host_platform_id",
+    srcs = [
+        "host_platform_id.cc",
+    ],
+    hdrs = [
+        "host_platform_id.h",
+    ],
+    deps = [
+        "//tensorflow/stream_executor:platform",
+    ],
+)
+
+cc_library(
+    name = "host_platform",
+    srcs = [
+        "host_platform.cc",
+    ],
+    hdrs = [
+        "host_platform.h",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":host_gpu_executor",
+        ":host_platform_id",
+        "//tensorflow/stream_executor:executor_cache",
+        "//tensorflow/stream_executor:multi_platform_manager",
+        "//tensorflow/stream_executor:stream_executor_headers",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+    ],
+    alwayslink = True,  # Registers itself with the MultiPlatformManager.
+)
+
+cc_library(
+    name = "host_stream",
+    srcs = [
+        "host_stream.cc",
+    ],
+    hdrs = [
+        "host_stream.h",
+    ],
+    deps = [
+        "//tensorflow/stream_executor:kernel",
+        "//tensorflow/stream_executor/lib",
+    ],
+)
+
+cc_library(
+    name = "host_timer",
+    srcs = [
+        "host_timer.cc",
+    ],
+    hdrs = [
+        "host_timer.h",
+    ],
+    deps = [
+        "//tensorflow/stream_executor:stream_executor_internal",
+        "//tensorflow/stream_executor:timer",
+        "//tensorflow/stream_executor/platform",
+    ],
+)
+
+# TODO(22689637): Rename this target.
+cc_library(
+    name = "host_gpu_executor",
+    srcs = [
+        "host_gpu_executor.cc",
+    ],
+    hdrs = [
+        "host_gpu_executor.h",
+    ],
+    deps = [
+        ":host_platform_id",
+        ":host_stream",
+        ":host_timer",
+        "//tensorflow/core:lib",
+        "//tensorflow/stream_executor:kernel",
+        "//tensorflow/stream_executor:rng",
+        "//tensorflow/stream_executor:stream_executor_internal",
+        "//tensorflow/stream_executor:timer",
+        "//tensorflow/stream_executor/lib",
+    ],
+    alwayslink = True,
+)
diff --git a/tensorflow/stream_executor/lib/BUILD b/tensorflow/stream_executor/lib/BUILD
new file mode 100644
index 0000000000..133ff2b161
--- /dev/null
+++ b/tensorflow/stream_executor/lib/BUILD
@@ -0,0 +1,62 @@
+licenses(["notice"])  # Apache 2.0
+
+load("//tensorflow:tensorflow.bzl", "tf_cc_test")
+load("//tensorflow/stream_executor:build_defs.bzl", "stream_executor_friends")
+
+package_group(
+    name = "friends",
+    packages = stream_executor_friends(),
+)
+
+package(default_visibility = [":friends"])
+
+filegroup(
+    name = "c_srcs",
+    data = glob([
+        "**/*.cc",
+        "**/*.h",
+    ]),
+)
+
+cc_library(
+    name = "lib",
+    srcs = glob(
+        [
+            "**/*.cc",
+        ],
+        exclude = [
+            "**/*test*",
+        ],
+    ),
+    hdrs = glob(["**/*.h"]),
+    deps = [
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:ptr_util",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:span",
+    ],
+)
+
+tf_cc_test(
+    name = "statusor_test",
+    size = "small",
+    srcs = ["statusor_test.cc"],
+    deps = [
+        ":lib",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
+cc_library(
+    name = "utility_headers",
+    hdrs = [
+        "ptr_util.h",
+    ],
+    deps = [
+        "//tensorflow/core:ptr_util",
+    ],
+)
diff --git a/tensorflow/stream_executor/platform/BUILD b/tensorflow/stream_executor/platform/BUILD
new file mode 100644
index 0000000000..702b2cdfe0
--- /dev/null
+++ b/tensorflow/stream_executor/platform/BUILD
@@ -0,0 +1,47 @@
+licenses(["notice"])  # Apache 2.0
+
+load("//tensorflow/stream_executor:build_defs.bzl", "stream_executor_friends")
+load("//tensorflow/core:platform/default/build_config.bzl", "tf_platform_hdrs")
+
+package_group(
+    name = "friends",
+    packages = stream_executor_friends(),
+)
+
+package(
+    default_visibility = [":friends"],
+)
+
+cc_library(
+    name = "platform",
+    textual_hdrs = [
+        "logging.h",
+        "mutex.h",
+        "platform.h",
+        "port.h",
+        "thread_annotations.h",
+        "initialize.h",
+    ],
+    deps = [
+        "//tensorflow/core:lib",
+        "//tensorflow/stream_executor/platform/default:platform",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "dso_loader",
+    hdrs = ["dso_loader.h"],
+    deps = [
+        ":platform",
+        "//tensorflow/stream_executor/platform/default:dso_loader",
+    ],
+)
+
+filegroup(
+    name = "c_srcs",
+    data = glob([
+        "**/*.cc",
+        "**/*.h",
+    ]),
+)
diff --git a/tensorflow/stream_executor/platform/default/BUILD b/tensorflow/stream_executor/platform/default/BUILD
new file mode 100644
index 0000000000..f1ae7d86ff
--- /dev/null
+++ b/tensorflow/stream_executor/platform/default/BUILD
@@ -0,0 +1,25 @@
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//tensorflow/stream_executor:__subpackages__"])
+
+cc_library(
+    name = "platform",
+    textual_hdrs = [
+        "initialize.h",
+        "mutex.h",
+    ],
+    deps = ["//tensorflow/core:lib"],
+)
+
+cc_library(
+    name = "dso_loader",
+    srcs = ["dso_loader.cc"],
+    hdrs = ["dso_loader.h"],
+    deps = [
+        "//tensorflow/stream_executor:platform",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/strings",
+        "@local_config_cuda//cuda:cuda_headers",
+    ],
+)
-- 
GitLab


From 60bcd1c1fd2cf412b217d24e7672bc3699458a43 Mon Sep 17 00:00:00 2001
From: Duncan Riach <duncan@nvidia.com>
Date: Tue, 11 Dec 2018 16:56:09 -0800
Subject: [PATCH 0367/2345] Add cuDNN deterministic env variable (only for
 convolution)

---
 tensorflow/stream_executor/cuda/cuda_dnn.cc | 40 +++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index a34aa9354d..e91caf3bdb 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -794,6 +794,19 @@ bool BatchnormSpatialPersistentEnabled() {
   return is_enabled;
 }
 
+// A helper function to decide whether to enable deterministic functionality.
+// TODO(pr/24355): Support all cuDNN functionality (currently only convolution).
+bool Deterministic() {
+  static bool is_enabled = [] {
+    bool is_enabled = false;
+    TF_CHECK_OK(
+        tensorflow::ReadBoolFromEnvVar("TF_CUDNN_DETERMINISTIC",
+                                       /*default_val=*/false, &is_enabled));
+    return is_enabled;
+  }();
+  return is_enabled;
+}
+
 // Turns a ConvolutionDescriptor structure into a cudnn convolution handle
 // within a scope.
 class CudnnConvolutionDescriptor {
@@ -3056,10 +3069,19 @@ bool CudnnSupport::GetConvolveAlgorithms(
     algo_types.push_back(CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED);
   }
 
+  if (Deterministic()) {
+    algo_types.clear();
+    algo_types.push_back(CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM);
+  }
+
   out_algorithms->clear();
   for (auto i : algo_types) {
     out_algorithms->push_back({i, /*use_tensor_ops=*/false});
     if (cc_major >= 7 && CUDNN_VERSION >= 7000 && TensorOpMathEnabled()) {
+      if (Deterministic()) {
+        // For determinism: always use tensor op math, if it's available
+        out_algorithms->pop_back();
+      }
       out_algorithms->push_back({i, /*use_tensor_ops=*/true});
     }
   }
@@ -3104,10 +3126,19 @@ bool CudnnSupport::GetConvolveBackwardDataAlgorithms(
     algo_types.push_back(CUDNN_CONVOLUTION_BWD_DATA_ALGO_WINOGRAD_NONFUSED);
   }
 
+  if (Deterministic()) {
+    algo_types.clear();
+    algo_types.push_back(CUDNN_CONVOLUTION_BWD_DATA_ALGO_1);
+  }
+
   out_algorithms->clear();
   for (auto i : algo_types) {
     out_algorithms->push_back({i, /*use_tensor_ops=*/false});
     if (cc_major >= 7 && CUDNN_VERSION >= 7000 && TensorOpMathEnabled()) {
+      if (Deterministic()) {
+        // For determinism: always use tensor op math, if it's available
+        out_algorithms->pop_back();
+      }
       out_algorithms->push_back({i, /*use_tensor_ops=*/true});
     }
   }
@@ -3135,10 +3166,19 @@ bool CudnnSupport::GetConvolveBackwardFilterAlgorithms(
     algo_types.push_back(CUDNN_CONVOLUTION_BWD_FILTER_ALGO_WINOGRAD_NONFUSED);
   }
 
+  if (Deterministic()) {
+    algo_types.clear();
+    algo_types.push_back(CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1);
+  }
+
   out_algorithms->clear();
   for (auto i : algo_types) {
     out_algorithms->push_back({i, /*use_tensor_ops=*/false});
     if (cc_major >= 7 && CUDNN_VERSION >= 7000 && TensorOpMathEnabled()) {
+      if (Deterministic()) {
+        // For determinism: always use tensor op math, if it's available
+        out_algorithms->pop_back();
+      }
       out_algorithms->push_back({i, /*use_tensor_ops=*/true});
     }
   }
-- 
GitLab


From df17a7db51000ccf7c6b062e37b5c3acb2cd286b Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Tue, 8 Jan 2019 13:42:38 -0800
Subject: [PATCH 0368/2345] MacOS support for simple speech example

PiperOrigin-RevId: 228391393
---
 .../micro/examples/micro_speech/BUILD         |  24 ---
 .../micro/examples/micro_speech/Makefile.inc  |  27 +---
 .../examples/micro_speech/audio_provider.cc   |   6 +
 .../examples/micro_speech/audio_provider.h    |  10 ++
 .../micro_speech/audio_provider_test.cc       |  26 ++++
 .../micro/examples/micro_speech/main.cc       |   4 +-
 .../examples/micro_speech/model_settings.h    |   1 +
 .../examples/micro_speech/osx/Makefile.inc    |   6 +
 .../micro_speech/osx/audio_provider.cc        | 139 ++++++++++++++++++
 .../micro/examples/micro_speech/timer.cc      |  22 ---
 .../micro/examples/micro_speech/timer.h       |  31 ----
 .../micro/examples/micro_speech/timer_test.cc |  49 ------
 .../experimental/micro/tools/make/Makefile    |  35 ++++-
 .../micro/tools/make/targets/osx_makefile.inc |  10 ++
 14 files changed, 241 insertions(+), 149 deletions(-)
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/osx/Makefile.inc
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/osx/audio_provider.cc
 delete mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/timer.cc
 delete mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/timer.h
 delete mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc
 create mode 100644 tensorflow/lite/experimental/micro/tools/make/targets/osx_makefile.inc

diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD b/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD
index 70eeac1458..702b893e6f 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD
@@ -196,29 +196,6 @@ tflite_micro_cc_test(
     ],
 )
 
-cc_library(
-    name = "timer",
-    srcs = [
-        "timer.cc",
-    ],
-    hdrs = [
-        "timer.h",
-    ],
-)
-
-tflite_micro_cc_test(
-    name = "timer_test",
-    srcs = [
-        "timer_test.cc",
-    ],
-    deps = [
-        ":timer",
-        "//tensorflow/lite/c:c_api_internal",
-        "//tensorflow/lite/experimental/micro:micro_framework",
-        "//tensorflow/lite/experimental/micro/testing:micro_test",
-    ],
-)
-
 cc_library(
     name = "recognize_commands",
     srcs = [
@@ -259,7 +236,6 @@ cc_binary(
         ":model_settings",
         ":preprocessor_reference",
         ":recognize_commands",
-        ":timer",
         ":tiny_conv_model_data",
         "//tensorflow/lite:schema_fbs_version",
         "//tensorflow/lite/experimental/micro:micro_framework",
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/Makefile.inc b/tensorflow/lite/experimental/micro/examples/micro_speech/Makefile.inc
index 5623eabdde..6f7563c02f 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/Makefile.inc
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/Makefile.inc
@@ -5,6 +5,7 @@ tensorflow/lite/experimental/micro/examples/micro_speech/micro_speech_test.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/tiny_conv_model_data.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/no_features_data.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/yes_features_data.cc
+MICRO_SPEECH_TEST_SRCS := $(call specialize,$(MICRO_SPEECH_TEST_SRCS))
 ALL_SRCS += $(MICRO_SPEECH_TEST_SRCS)
 MICRO_SPEECH_TEST_OBJS := $(addprefix $(OBJDIR), \
 $(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(MICRO_SPEECH_TEST_SRCS))))
@@ -32,6 +33,7 @@ tensorflow/lite/experimental/micro/examples/micro_speech/yes_power_spectrum_data
 PREPROCESSOR_REFERENCE_TEST_SRCS = \
 $(PREPROCESSOR_TEST_SHARED_SRCS) \
 tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc
+PREPROCESSOR_REFERENCE_TEST_SRCS := $(call specialize,$(PREPROCESSOR_REFERENCE_TEST_SRCS))
 ALL_SRCS += $(PREPROCESSOR_REFERENCE_TEST_SRCS)
 PREPROCESSOR_REFERENCE_TEST_OBJS := $(addprefix $(OBJDIR), \
 $(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(PREPROCESSOR_REFERENCE_TEST_SRCS))))
@@ -51,6 +53,7 @@ test_preprocessor_reference: $(PREPROCESSOR_REFERENCE_TEST_BINARY)
 PREPROCESSOR_FIXED_TEST_SRCS = \
 $(PREPROCESSOR_TEST_SHARED_SRCS) \
 tensorflow/lite/experimental/micro/examples/micro_speech/fixed_point/preprocessor.cc
+PREPROCESSOR_FIXED_TEST_SRCS := $(call specialize,$(PREPROCESSOR_FIXED_TEST_SRCS))
 ALL_SRCS += $(PREPROCESSOR_FIXED_TEST_SRCS)
 PREPROCESSOR_FIXED_TEST_OBJS := $(addprefix $(OBJDIR), \
 $(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(PREPROCESSOR_FIXED_TEST_SRCS))))
@@ -71,6 +74,7 @@ AUDIO_PROVIDER_TEST_SRCS := \
 tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider_test.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc
+AUDIO_PROVIDER_TEST_SRCS := $(call specialize,$(AUDIO_PROVIDER_TEST_SRCS))
 ALL_SRCS += $(AUDIO_PROVIDER_TEST_SRCS)
 AUDIO_PROVIDER_TEST_OBJS := $(addprefix $(OBJDIR), \
 $(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(AUDIO_PROVIDER_TEST_SRCS))))
@@ -93,6 +97,7 @@ tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.cc
+FEATURE_PROVIDER_TEST_SRCS := $(call specialize,$(FEATURE_PROVIDER_TEST_SRCS))
 ALL_SRCS += $(FEATURE_PROVIDER_TEST_SRCS)
 FEATURE_PROVIDER_TEST_OBJS := $(addprefix $(OBJDIR), \
 $(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(FEATURE_PROVIDER_TEST_SRCS))))
@@ -108,30 +113,12 @@ feature_provider_test_bin: $(FEATURE_PROVIDER_TEST_BINARY).bin
 test_feature_provider: $(FEATURE_PROVIDER_TEST_BINARY)
 	$(TEST_SCRIPT) $(FEATURE_PROVIDER_TEST_BINARY) '~~~ALL TESTS PASSED~~~'
 
-# Tests the timer module.
-TIMER_TEST_SRCS := \
-tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc \
-tensorflow/lite/experimental/micro/examples/micro_speech/timer.cc
-ALL_SRCS += $(TIMER_TEST_SRCS)
-TIMER_TEST_OBJS := $(addprefix $(OBJDIR), \
-$(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(TIMER_TEST_SRCS))))
-TIMER_TEST_BINARY := $(BINDIR)timer_test
-ALL_BINARIES += $(TIMER_TEST_BINARY)
-$(TIMER_TEST_BINARY): $(TIMER_TEST_OBJS) $(MICROLITE_LIB_PATH)
-	@mkdir -p $(dir $@)
-	$(CXX) $(CXXFLAGS) $(INCLUDES) \
-	-o $(TIMER_TEST_BINARY) $(TIMER_TEST_OBJS) \
-	$(LIBFLAGS) $(MICROLITE_LIB_PATH) $(LDFLAGS) $(MICROLITE_LIBS)
-timer_test: $(TIMER_TEST_BINARY)
-timer_test_bin: $(TIMER_TEST_BINARY).bin
-test_timer: $(TIMER_TEST_BINARY)
-	$(TEST_SCRIPT) $(TIMER_TEST_BINARY) '~~~ALL TESTS PASSED~~~'
-
 # Tests the feature provider module.
 RECOGNIZE_COMMANDS_TEST_SRCS := \
 tensorflow/lite/experimental/micro/examples/micro_speech/recognize_commands_test.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/recognize_commands.cc
+RECOGNIZE_COMMANDS_TEST_SRCS := $(call specialize,$(RECOGNIZE_COMMANDS_TEST_SRCS))
 ALL_SRCS += $(RECOGNIZE_COMMANDS_TEST_SRCS)
 RECOGNIZE_COMMANDS_TEST_OBJS := $(addprefix $(OBJDIR), \
 $(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(RECOGNIZE_COMMANDS_TEST_SRCS))))
@@ -153,12 +140,12 @@ tensorflow/lite/experimental/micro/examples/micro_speech/main.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.cc \
-tensorflow/lite/experimental/micro/examples/micro_speech/timer.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/no_features_data.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/yes_features_data.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/tiny_conv_model_data.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/recognize_commands.cc
+MICRO_SPEECH_SRCS := $(call specialize,$(MICRO_SPEECH_SRCS))
 ALL_SRCS += $(MICRO_SPEECH_SRCS)
 MICRO_SPEECH_OBJS := $(addprefix $(OBJDIR), \
 $(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(MICRO_SPEECH_SRCS))))
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc
index c0365d5690..52db18e686 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc
@@ -19,6 +19,7 @@ limitations under the License.
 
 namespace {
 int16_t g_dummy_audio_data[kMaxAudioSampleSize];
+int32_t g_latest_audio_timestamp = 0;
 }  // namespace
 
 TfLiteStatus GetAudioSamples(tflite::ErrorReporter* error_reporter,
@@ -31,3 +32,8 @@ TfLiteStatus GetAudioSamples(tflite::ErrorReporter* error_reporter,
   *audio_samples = g_dummy_audio_data;
   return kTfLiteOk;
 }
+
+int32_t LatestAudioTimestamp() {
+  g_latest_audio_timestamp += 100;
+  return g_latest_audio_timestamp;
+}
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h
index 7e2442a5e8..b690673641 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h
@@ -33,4 +33,14 @@ TfLiteStatus GetAudioSamples(tflite::ErrorReporter* error_reporter,
                              int start_ms, int duration_ms,
                              int* audio_samples_size, int16_t** audio_samples);
 
+// Returns the time that audio data was last captured in milliseconds. There's
+// no contract about what time zero represents, the accuracy, or the granularity
+// of the result. Subsequent calls will generally not return a lower value, but
+// even that's not guaranteed if there's an overflow  wraparound.
+// The reference implementation of this function just returns a constantly
+// incrementing value for each call, since it would need a non-portable platform
+// call to access time information. For real applications, you'll need to write
+// your own platform-specific implementation.
+int32_t LatestAudioTimestamp();
+
 #endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_AUDIO_PROVIDER_H_
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider_test.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider_test.cc
index 5f7c7605f0..85fbbb80a6 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider_test.cc
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider_test.cc
@@ -14,6 +14,9 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h"
+
+#include <limits>
+
 #include "tensorflow/lite/c/c_api_internal.h"
 #include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
 #include "tensorflow/lite/experimental/micro/micro_error_reporter.h"
@@ -41,4 +44,27 @@ TF_LITE_MICRO_TEST(TestAudioProvider) {
   }
 }
 
+TF_LITE_MICRO_TEST(TestTimer) {
+  // Make sure that the technically-undefined overflow behavior we rely on below
+  // works on this platform. It's still not guaranteed, but at least this is a
+  // sanity check.  Turn off when running with ASan, as it will complain about
+  // the following undefined behavior.
+#ifndef ADDRESS_SANITIZER
+  int32_t overflow_value = std::numeric_limits<int32_t>::max();
+  overflow_value += 1;
+  TF_LITE_MICRO_EXPECT_EQ(std::numeric_limits<int32_t>::min(), overflow_value);
+#endif
+
+  const int32_t first_time = LatestAudioTimestamp();
+  const int32_t second_time = LatestAudioTimestamp();
+
+  // It's possible that the timer may have wrapped around from +BIG_NUM to
+  // -BIG_NUM between the first and second calls, since we're storing
+  // milliseconds in a 32-bit integer. It's not reasonable that the call itself
+  // would have taken more than 2^31 milliseconds though, so look at the
+  // difference and rely on integer overflow to ensure it's accurate.
+  const int32_t time_delta = (second_time - first_time);
+  TF_LITE_MICRO_EXPECT_LE(0, time_delta);
+}
+
 TF_LITE_MICRO_TESTS_END
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/main.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/main.cc
index 515f82fcbc..3a9a5a4df1 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/main.cc
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/main.cc
@@ -13,10 +13,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h"
 #include "tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h"
 #include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
 #include "tensorflow/lite/experimental/micro/examples/micro_speech/recognize_commands.h"
-#include "tensorflow/lite/experimental/micro/examples/micro_speech/timer.h"
 #include "tensorflow/lite/experimental/micro/examples/micro_speech/tiny_conv_model_data.h"
 #include "tensorflow/lite/experimental/micro/kernels/all_ops_resolver.h"
 #include "tensorflow/lite/experimental/micro/micro_error_reporter.h"
@@ -76,7 +76,7 @@ int main(int argc, char* argv[]) {
   // Keep reading and analysing audio data in an infinite loop.
   while (true) {
     // Fetch the spectrogram for the current time.
-    const int32_t current_time = TimeInMilliseconds();
+    const int32_t current_time = LatestAudioTimestamp();
     int how_many_new_slices = 0;
     TfLiteStatus feature_status = feature_provider.PopulateFeatureData(
         error_reporter, previous_time, current_time, &how_many_new_slices);
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h b/tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h
index 1d8f3123a5..f48252d14d 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h
@@ -23,6 +23,7 @@ limitations under the License.
 // frequency information. This has to be a power of two, and since we're dealing
 // with 30ms of 16KHz inputs, which means 480 samples, this is the next value.
 constexpr int kMaxAudioSampleSize = 512;
+constexpr int kAudioSampleFrequency = 16000;
 
 // All of these values are derived from the values used during model training,
 // if you change your model you'll need to update these constants.
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/osx/Makefile.inc b/tensorflow/lite/experimental/micro/examples/micro_speech/osx/Makefile.inc
new file mode 100644
index 0000000000..89d107cfe0
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/osx/Makefile.inc
@@ -0,0 +1,6 @@
+#Settings for Mac OS platforms.
+ifeq ($(TARGET), osx)
+  MICROLITE_LIBS += \
+    -framework Foundation \
+    -framework AudioToolbox
+endif
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/osx/audio_provider.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/osx/audio_provider.cc
new file mode 100644
index 0000000000..892757e799
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/osx/audio_provider.cc
@@ -0,0 +1,139 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h"
+
+#include <AudioToolbox/AudioToolbox.h>
+
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
+
+namespace {
+
+constexpr int kNumberRecordBuffers = 3;
+bool g_is_audio_initialized = false;
+constexpr int kAudioCaptureBufferSize = kAudioSampleFrequency * 0.5;
+int16_t g_audio_capture_buffer[kAudioCaptureBufferSize];
+int16_t g_audio_output_buffer[kMaxAudioSampleSize];
+int32_t g_latest_audio_timestamp = 0;
+
+// Checks for MacOS errors, prints information and returns a TF Lite version.
+#define RETURN_IF_OS_ERROR(error, error_reporter)                       \
+  do {                                                                  \
+    if (error != noErr) {                                               \
+      error_reporter->Report("Error: %s:%d (%d)\n", __FILE__, __LINE__, \
+                             error);                                    \
+      return kTfLiteError;                                              \
+    }                                                                   \
+  } while (0);
+
+// Called when an audio input buffer has been filled.
+void OnAudioBufferFilledCallback(
+    void* user_data, AudioQueueRef queue, AudioQueueBufferRef buffer,
+    const AudioTimeStamp* start_time, UInt32 num_packets,
+    const AudioStreamPacketDescription* packet_description) {
+  const int sample_size = buffer->mAudioDataByteSize / sizeof(float);
+  const int64_t sample_offset = start_time->mSampleTime;
+  const int32_t time_in_ms =
+      (sample_offset + sample_size) / (kAudioSampleFrequency / 1000);
+  const float* float_samples = static_cast<const float*>(buffer->mAudioData);
+  for (int i = 0; i < sample_size; ++i) {
+    const int capture_index = (sample_offset + i) % kAudioCaptureBufferSize;
+    g_audio_capture_buffer[capture_index] = float_samples[i] * ((1 << 15) - 1);
+  }
+  // This is how we let the outside world know that new audio data has arrived.
+  g_latest_audio_timestamp = time_in_ms;
+  AudioQueueEnqueueBuffer(queue, buffer, 0, nullptr);
+}
+
+// Set up everything we need to capture audio samples from the default recording
+// device on MacOS.
+TfLiteStatus InitAudioRecording(tflite::ErrorReporter* error_reporter) {
+  // Set up the format of the audio - single channel, 32-bit float at 16KHz.
+  AudioStreamBasicDescription recordFormat = {0};
+  recordFormat.mSampleRate = kAudioSampleFrequency;
+  recordFormat.mFormatID = kAudioFormatLinearPCM;
+  recordFormat.mFormatFlags =
+      kAudioFormatFlagIsFloat | kAudioFormatFlagIsPacked;
+  recordFormat.mBitsPerChannel = 8 * sizeof(float);
+  recordFormat.mChannelsPerFrame = 1;
+  recordFormat.mBytesPerFrame = sizeof(float) * recordFormat.mChannelsPerFrame;
+  recordFormat.mFramesPerPacket = 1;
+  recordFormat.mBytesPerPacket =
+      recordFormat.mBytesPerFrame * recordFormat.mFramesPerPacket;
+  recordFormat.mReserved = 0;
+
+  UInt32 propSize = sizeof(recordFormat);
+  RETURN_IF_OS_ERROR(AudioFormatGetProperty(kAudioFormatProperty_FormatInfo, 0,
+                                            NULL, &propSize, &recordFormat),
+                     error_reporter);
+
+  // Create a recording queue.
+  AudioQueueRef queue;
+  RETURN_IF_OS_ERROR(
+      AudioQueueNewInput(&recordFormat, OnAudioBufferFilledCallback,
+                         error_reporter, nullptr, nullptr, 0, &queue),
+      error_reporter);
+
+  // Set up the buffers we'll need.
+  int buffer_bytes = 512 * sizeof(float);
+  for (int i = 0; i < kNumberRecordBuffers; ++i) {
+    AudioQueueBufferRef buffer;
+    RETURN_IF_OS_ERROR(AudioQueueAllocateBuffer(queue, buffer_bytes, &buffer),
+                       error_reporter);
+    RETURN_IF_OS_ERROR(AudioQueueEnqueueBuffer(queue, buffer, 0, nullptr),
+                       error_reporter);
+  }
+
+  // Start capturing audio.
+  RETURN_IF_OS_ERROR(AudioQueueStart(queue, nullptr), error_reporter);
+
+  return kTfLiteOk;
+}
+
+}  // namespace
+
+TfLiteStatus GetAudioSamples(tflite::ErrorReporter* error_reporter,
+                             int start_ms, int duration_ms,
+                             int* audio_samples_size, int16_t** audio_samples) {
+  if (!g_is_audio_initialized) {
+    TfLiteStatus init_status = InitAudioRecording(error_reporter);
+    if (init_status != kTfLiteOk) {
+      return init_status;
+    }
+    for (int i = 0; i < kMaxAudioSampleSize; ++i) {
+      g_audio_output_buffer[i] = 0;
+    }
+    g_is_audio_initialized = true;
+  }
+  // This should only be called when the main thread notices that the latest
+  // audio sample data timestamp has changed, so that there's new data in the
+  // capture ring buffer. The ring buffer will eventually wrap around and
+  // overwrite the data, but the assumption is that the main thread is checking
+  // often enough and the buffer is large enough that this call will be made
+  // before that happens.
+  const int start_offset = start_ms * (kAudioSampleFrequency / 1000);
+  const int duration_sample_count =
+      duration_ms * (kAudioSampleFrequency / 1000);
+  for (int i = 0; i < duration_sample_count; ++i) {
+    const int capture_index = (start_offset + i) % kAudioCaptureBufferSize;
+    g_audio_output_buffer[i] = g_audio_capture_buffer[capture_index];
+  }
+
+  *audio_samples_size = kMaxAudioSampleSize;
+  *audio_samples = g_audio_output_buffer;
+  return kTfLiteOk;
+}
+
+int32_t LatestAudioTimestamp() { return g_latest_audio_timestamp; }
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/timer.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/timer.cc
deleted file mode 100644
index 6c96a61ab5..0000000000
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/timer.cc
+++ /dev/null
@@ -1,22 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/lite/experimental/micro/examples/micro_speech/timer.h"
-
-int32_t TimeInMilliseconds() {
-  static int current_time = 0;
-  current_time += 100;
-  return current_time;
-}
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/timer.h b/tensorflow/lite/experimental/micro/examples/micro_speech/timer.h
deleted file mode 100644
index 162952844a..0000000000
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/timer.h
+++ /dev/null
@@ -1,31 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_TIMER_H_
-#define TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_TIMER_H_
-
-#include <cstdint>
-
-// Returns the time in milliseconds. There's no contract about what time zero
-// represents, the accuracy, or the granularity of the result. Subsequent calls
-// will generally not return a lower value, but even that's not guaranteed if
-// there's an overflow  wraparound.
-// The reference implementation of this function just returns a constantly
-// incrementing value for each call, since it would need a non-portable platform
-// call to access time information. For real applications, you'll need to write
-// your own platform-specific implementation.
-int32_t TimeInMilliseconds();
-
-#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_TIMER_H_
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc
deleted file mode 100644
index 0487a12b25..0000000000
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc
+++ /dev/null
@@ -1,49 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/lite/experimental/micro/examples/micro_speech/timer.h"
-
-#include <limits>
-
-#include "tensorflow/lite/c/c_api_internal.h"
-#include "tensorflow/lite/experimental/micro/micro_error_reporter.h"
-#include "tensorflow/lite/experimental/micro/testing/micro_test.h"
-
-TF_LITE_MICRO_TESTS_BEGIN
-
-TF_LITE_MICRO_TEST(TestTimer) {
-  // Make sure that the technically-undefined overflow behavior we rely on below
-  // works on this platform. It's still not guaranteed, but at least this is a
-  // sanity check.  Turn off when running with ASan, as it will complain about
-  // the following undefined behavior.
-#ifndef ADDRESS_SANITIZER
-  int32_t overflow_value = std::numeric_limits<int32_t>::max();
-  overflow_value += 1;
-  TF_LITE_MICRO_EXPECT_EQ(std::numeric_limits<int32_t>::min(), overflow_value);
-#endif
-
-  const int32_t first_time = TimeInMilliseconds();
-  const int32_t second_time = TimeInMilliseconds();
-
-  // It's possible that the timer may have wrapped around from +BIG_NUM to
-  // -BIG_NUM between the first and second calls, since we're storing
-  // milliseconds in a 32-bit integer. It's not reasonable that the call itself
-  // would have taken more than 2^31 milliseconds though, so look at the
-  // difference and rely on integer overflow to ensure it's accurate.
-  const int32_t time_delta = (second_time - first_time);
-  TF_LITE_MICRO_EXPECT_LE(0, time_delta);
-}
-
-TF_LITE_MICRO_TESTS_END
diff --git a/tensorflow/lite/experimental/micro/tools/make/Makefile b/tensorflow/lite/experimental/micro/tools/make/Makefile
index 6a7c875888..f9a2dffae2 100644
--- a/tensorflow/lite/experimental/micro/tools/make/Makefile
+++ b/tensorflow/lite/experimental/micro/tools/make/Makefile
@@ -21,6 +21,38 @@ HOST_ARCH := $(shell if [[ $(shell uname -m) =~ i[345678]86 ]]; then echo x86_32
 TARGET := $(HOST_OS)
 TARGET_ARCH := $(HOST_ARCH)
 
+# Look for platform or target-specific implementation files to replace reference
+# implementations with, given a tag. These are expected to occur in subfolders
+# of a directory where a reference implementation exists, and have the same
+# interface and header file. For example,
+# tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc
+# defines a module for supplying audio data, but since no platform or OS can be
+# presumed, it just always returns zeroes for its samples. The MacOS-specific
+# tensorflow/lite/experimental/micro/examples/micro_speech/osx/audio_provider.cc
+# has an implementation that relies on CoreAudio, and there are equivalent
+# versions for other operating systems. 
+# All lists of source files are put through this substitution process with the
+# tags of their target OS and architecture, so that implementations can be added
+# by simply placing them in the file tree, with no changes to the build files
+# needed.
+# One confusing thing about this implementation is that we're using wildcard to
+# act as a 'does file exist?' function, rather than expanding an expression.
+# Wildcard will return an empty string if given a plain file path with no actual
+# wildcards, if the file doesn't exist, so taking the first word of the list
+# between that and the reference path will pick the specialized one if it's
+# available.
+substitute_specialized_implementation = \
+  $(firstword $(wildcard $(dir $(1))$(2)/$(notdir $(1))) $(wildcard $(1)))
+substitute_specialized_implementations = \
+  $(foreach source,$(1),$(call substitute_specialized_implementation,$(source),$(2)))
+# Here we're first looking for specialized implementations in ref_dir/$(TARGET)
+# and then ref_dir/$(TARGET_ARCH), before falling back to ref_dir's
+# implementation.
+# The argument to this function should be a list of space-separated file paths,
+# with any wildcards already expanded.
+specialize = \
+  $(call substitute_specialized_implementations,$(call substitute_specialized_implementations,$(1),$(TARGET)),$(TARGET_ARCH))
+
 INCLUDES := \
 -I. \
 -I$(MAKEFILE_DIR)/../../../../../ \
@@ -67,6 +99,7 @@ tensorflow/lite/core/api/op_resolver.cc \
 tensorflow/lite/kernels/kernel_util.cc \
 tensorflow/lite/kernels/internal/quantization_util.cc
 MICROLITE_CC_SRCS := $(filter-out $(MICROLITE_TEST_SRCS), $(MICROLITE_CC_BASE_SRCS))
+MICROLITE_CC_SRCS := $(call specialize,$(MICROLITE_CC_SRCS))
 
 # These target-specific makefiles should modify or replace options like
 # CXXFLAGS or LIBS to work for a specific targetted architecture. All logic
@@ -115,7 +148,7 @@ $(OBJDIR)%.o: %.S
 	$(CC) $(CCFLAGS) $(INCLUDES) -c $< -o $@
 
 # The target that's compiled if there's no command-line arguments.
-all: $(MICROLITE_LIB_PATH) $(MICRO_SPEECH_TEST_BINARY) $(PREPROCESSOR_TEST_BINARY)
+all: $(MICROLITE_LIB_PATH)
 
 microlite: $(MICROLITE_LIB_PATH)
 
diff --git a/tensorflow/lite/experimental/micro/tools/make/targets/osx_makefile.inc b/tensorflow/lite/experimental/micro/tools/make/targets/osx_makefile.inc
new file mode 100644
index 0000000000..0e8aad1993
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/tools/make/targets/osx_makefile.inc
@@ -0,0 +1,10 @@
+#Settings for Mac OS platforms.
+ifeq ($(TARGET), osx)
+
+  PLATFORM_FLAGS = \
+    -DTF_LITE_DISABLE_X86_NEON
+
+  CXXFLAGS += $(PLATFORM_FLAGS)
+  CCFLAGS += $(PLATFORM_FLAGS)
+
+endif
\ No newline at end of file
-- 
GitLab


From f9b6cccd90939bf9516d0ae0e3bbdc4a7c4f4a54 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Tue, 8 Jan 2019 13:50:00 -0800
Subject: [PATCH 0369/2345] [tf.data] Update `map` documentation -- calling out
 the fact that setting `num_parallel_calls` results in asynchronous
 computation.

PiperOrigin-RevId: 228392788
---
 tensorflow/python/data/ops/dataset_ops.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index 61210f3a7f..45e732be0d 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -970,8 +970,8 @@ class DatasetV2(object):
         shapes and types defined by `self.output_shapes` and
        `self.output_types`) to another nested structure of tensors.
       num_parallel_calls: (Optional.) A `tf.int32` scalar `tf.Tensor`,
-        representing the number elements to process in parallel. If not
-        specified, elements will be processed sequentially. If the value
+        representing the number elements to process asynchronously in parallel.
+        If not specified, elements will be processed sequentially. If the value
         `tf.data.experimental.AUTOTUNE` is used, then the number of parallel
         calls is set dynamically based on available CPU.
 
-- 
GitLab


From 4aaaee6112c40791e5dbc9730c2a06274ff35b3f Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 8 Jan 2019 13:50:17 -0800
Subject: [PATCH 0370/2345] [TF:XLA] Bump open source llvm revision to r350570

PiperOrigin-RevId: 228392842
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 834ec28447..ae26ba6e68 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -500,11 +500,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "20c0ebd54f433d68652a54e328d70b0ec7cf37ce734645a0259cf904d14c2c53",
-        strip_prefix = "llvm-0e2aef924e02d00a4d3454d63c1921a243d316c6",
+        sha256 = "83a4f199742f3d6892994dd6dc46d6a53019aedaa28590b460c120f3dfc7bc47",
+        strip_prefix = "llvm-671f057a2cf137914be6c786daf8000469adebab",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/0e2aef924e02d00a4d3454d63c1921a243d316c6.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/0e2aef924e02d00a4d3454d63c1921a243d316c6.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/671f057a2cf137914be6c786daf8000469adebab.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/671f057a2cf137914be6c786daf8000469adebab.tar.gz",
         ],
     )
 
-- 
GitLab


From 731ae198b0d16e2b51299c6fd5050a428bc18152 Mon Sep 17 00:00:00 2001
From: Duncan Riach <duncan@nvidia.com>
Date: Tue, 8 Jan 2019 14:13:09 -0800
Subject: [PATCH 0371/2345] Renaming and factoring

---
 tensorflow/stream_executor/cuda/cuda_dnn.cc | 78 +++++++++++----------
 1 file changed, 41 insertions(+), 37 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index e91caf3bdb..16d644629e 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -796,12 +796,12 @@ bool BatchnormSpatialPersistentEnabled() {
 
 // A helper function to decide whether to enable deterministic functionality.
 // TODO(pr/24355): Support all cuDNN functionality (currently only convolution).
-bool Deterministic() {
+bool RequireDeterminism() {
   static bool is_enabled = [] {
     bool is_enabled = false;
-    TF_CHECK_OK(
-        tensorflow::ReadBoolFromEnvVar("TF_CUDNN_DETERMINISTIC",
-                                       /*default_val=*/false, &is_enabled));
+    TF_CHECK_OK(tensorflow::ReadBoolFromEnvVar("TF_CUDNN_DETERMINISTIC",
+                                               /*default_val=*/false,
+                                               &is_enabled));
     return is_enabled;
   }();
   return is_enabled;
@@ -3049,9 +3049,22 @@ port::Status CudnnSupport::DoFusedConvolveImpl(
   return port::Status::OK();
 }
 
+inline bool TensorOpMathAvailable(int cc_major) {
+  return cc_major >= 7 && CUDNN_VERSION >= 7000 && TensorOpMathEnabled();
+}
+
 bool CudnnSupport::GetConvolveAlgorithms(
     bool with_winograd_nonfused, int cc_major, int cc_minor,
     std::vector<dnn::AlgorithmDesc>* out_algorithms) {
+  bool tensor_op_math_available = TensorOpMathAvailable(cc_major);
+  out_algorithms->clear();
+
+  if (RequireDeterminism()) {
+    out_algorithms->push_back({CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM,
+                               tensor_op_math_available});
+    return true;
+  }
+
   std::vector<dnn::AlgorithmDesc::Index> algo_types = {
     // clang-format off
     CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM,
@@ -3069,22 +3082,13 @@ bool CudnnSupport::GetConvolveAlgorithms(
     algo_types.push_back(CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED);
   }
 
-  if (Deterministic()) {
-    algo_types.clear();
-    algo_types.push_back(CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM);
-  }
-
-  out_algorithms->clear();
   for (auto i : algo_types) {
     out_algorithms->push_back({i, /*use_tensor_ops=*/false});
-    if (cc_major >= 7 && CUDNN_VERSION >= 7000 && TensorOpMathEnabled()) {
-      if (Deterministic()) {
-        // For determinism: always use tensor op math, if it's available
-        out_algorithms->pop_back();
-      }
+    if (tensor_op_math_available) {
       out_algorithms->push_back({i, /*use_tensor_ops=*/true});
     }
   }
+
   return true;
 }
 
@@ -3113,6 +3117,15 @@ bool CudnnSupport::GetRnnAlgorithms(
 bool CudnnSupport::GetConvolveBackwardDataAlgorithms(
     bool with_winograd_nonfused, int cc_major, int cc_minor,
     std::vector<dnn::AlgorithmDesc>* out_algorithms) {
+  bool tensor_op_math_available = TensorOpMathAvailable(cc_major);
+  out_algorithms->clear();
+
+  if (RequireDeterminism()) {
+    out_algorithms->push_back(
+        {CUDNN_CONVOLUTION_BWD_DATA_ALGO_1, tensor_op_math_available});
+    return true;
+  }
+
   std::vector<dnn::AlgorithmDesc::Index> algo_types = {
       // clang-format off
     CUDNN_CONVOLUTION_BWD_DATA_ALGO_0,
@@ -3126,28 +3139,28 @@ bool CudnnSupport::GetConvolveBackwardDataAlgorithms(
     algo_types.push_back(CUDNN_CONVOLUTION_BWD_DATA_ALGO_WINOGRAD_NONFUSED);
   }
 
-  if (Deterministic()) {
-    algo_types.clear();
-    algo_types.push_back(CUDNN_CONVOLUTION_BWD_DATA_ALGO_1);
-  }
-
-  out_algorithms->clear();
   for (auto i : algo_types) {
     out_algorithms->push_back({i, /*use_tensor_ops=*/false});
-    if (cc_major >= 7 && CUDNN_VERSION >= 7000 && TensorOpMathEnabled()) {
-      if (Deterministic()) {
-        // For determinism: always use tensor op math, if it's available
-        out_algorithms->pop_back();
-      }
+    if (tensor_op_math_available) {
       out_algorithms->push_back({i, /*use_tensor_ops=*/true});
     }
   }
+
   return true;
 }
 
 bool CudnnSupport::GetConvolveBackwardFilterAlgorithms(
     bool with_winograd_nonfused, int cc_major, int cc_minor,
     std::vector<dnn::AlgorithmDesc>* out_algorithms) {
+  bool tensor_op_math_available = TensorOpMathAvailable(cc_major);
+  out_algorithms->clear();
+
+  if (RequireDeterminism()) {
+    out_algorithms->push_back(
+        {CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1, tensor_op_math_available});
+    return true;
+  }
+
   std::vector<dnn::AlgorithmDesc::Index> algo_types = {
       // clang-format off
       CUDNN_CONVOLUTION_BWD_FILTER_ALGO_0,
@@ -3166,22 +3179,13 @@ bool CudnnSupport::GetConvolveBackwardFilterAlgorithms(
     algo_types.push_back(CUDNN_CONVOLUTION_BWD_FILTER_ALGO_WINOGRAD_NONFUSED);
   }
 
-  if (Deterministic()) {
-    algo_types.clear();
-    algo_types.push_back(CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1);
-  }
-
-  out_algorithms->clear();
   for (auto i : algo_types) {
     out_algorithms->push_back({i, /*use_tensor_ops=*/false});
-    if (cc_major >= 7 && CUDNN_VERSION >= 7000 && TensorOpMathEnabled()) {
-      if (Deterministic()) {
-        // For determinism: always use tensor op math, if it's available
-        out_algorithms->pop_back();
-      }
+    if (tensor_op_math_available) {
       out_algorithms->push_back({i, /*use_tensor_ops=*/true});
     }
   }
+
   return true;
 }
 
-- 
GitLab


From c52d92f10021f8caa00ea271755d47a9c86259bd Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Tue, 8 Jan 2019 14:06:17 -0800
Subject: [PATCH 0372/2345] Internal change

PiperOrigin-RevId: 228396077
---
 tensorflow/lite/experimental/swift/BUILD      | 101 -----
 tensorflow/lite/experimental/swift/LICENSE    | 202 ----------
 tensorflow/lite/experimental/swift/README.md  |  54 ---
 .../swift/Sources/Interpreter.swift           | 265 --------------
 .../swift/Sources/InterpreterError.swift      |  99 -----
 .../swift/Sources/InterpreterOptions.swift    |  29 --
 .../experimental/swift/Sources/Model.swift    |  40 --
 .../Sources/QuantizationParameters.swift      |  38 --
 .../experimental/swift/Sources/Tensor.swift   | 138 -------
 .../Configs/TensorFlowLite.tulsigen           |  57 ---
 .../project.tulsiconf                         |  14 -
 .../project.pbxproj                           | 345 ------------------
 .../TensorFlowLiteApp/AppDelegate.swift       |  24 --
 .../Array+TensorFlowLite.swift                |  22 --
 .../AppIcon.appiconset/Contents.json          |  98 -----
 .../Assets.xcassets/Contents.json             |   6 -
 .../Base.lproj/LaunchScreen.storyboard        |  44 ---
 .../Base.lproj/Main.storyboard                |  95 -----
 .../Data+TensorFlowLite.swift                 |  13 -
 .../TensorFlowLiteApp/Info.plist              |  46 ---
 .../TensorFlowLiteApp/ViewController.swift    | 299 ---------------
 .../swift/Tests/InterpreterOptionsTests.swift |  54 ---
 .../swift/Tests/InterpreterTests.swift        | 315 ----------------
 .../experimental/swift/Tests/ModelTests.swift |  59 ---
 .../Tests/QuantizationParametersTests.swift   |  43 ---
 .../swift/Tests/TensorTests.swift             |  83 -----
 26 files changed, 2583 deletions(-)
 delete mode 100644 tensorflow/lite/experimental/swift/BUILD
 delete mode 100644 tensorflow/lite/experimental/swift/LICENSE
 delete mode 100644 tensorflow/lite/experimental/swift/README.md
 delete mode 100644 tensorflow/lite/experimental/swift/Sources/Interpreter.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Sources/InterpreterError.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Sources/Model.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Sources/Tensor.swift
 delete mode 100644 tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
 delete mode 100644 tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Tests/ModelTests.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Tests/TensorTests.swift

diff --git a/tensorflow/lite/experimental/swift/BUILD b/tensorflow/lite/experimental/swift/BUILD
deleted file mode 100644
index 53bcb0ecbd..0000000000
--- a/tensorflow/lite/experimental/swift/BUILD
+++ /dev/null
@@ -1,101 +0,0 @@
-# TensorFlow Lite for Swift.
-
-package(default_visibility = ["//visibility:private"])
-
-licenses(["notice"])  # Apache 2.0
-
-exports_files(["LICENSE"])
-
-load("@build_bazel_rules_apple//apple:ios.bzl", "ios_application", "ios_unit_test")
-load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library")
-
-MINIMUM_OS_VERSION = "9.0"
-
-SWIFT_COPTS = [
-    "-wmo",
-]
-
-swift_library(
-    name = "TensorFlowLite",
-    srcs = glob(["Sources/*.swift"]),
-    copts = SWIFT_COPTS,
-    module_name = "TensorFlowLite",
-    tags = ["manual"],
-    deps = [
-        "//tensorflow/lite/experimental/c:c_api",
-    ],
-)
-
-ios_unit_test(
-    name = "TensorFlowLiteTests",
-    size = "small",
-    minimum_os_version = MINIMUM_OS_VERSION,
-    tags = [
-        "manual",
-        # DISABLED: Following sanitizer tests are not supported by iOS test targets.
-        "noasan",
-        "nomsan",
-        "notsan",
-    ],
-    deps = [":TensorFlowLiteTestsLib"],
-)
-
-swift_library(
-    name = "TensorFlowLiteTestsLib",
-    testonly = 1,
-    srcs = glob(["Tests/*.swift"]),
-    copts = SWIFT_COPTS,
-    tags = ["manual"],
-    deps = [
-        ":TensorFlowLite",
-        ":TestResources",
-    ],
-)
-
-objc_library(
-    name = "TestResources",
-    resources = [
-        "//tensorflow/lite:testdata/add.bin",
-        "//tensorflow/lite:testdata/add_quantized.bin",
-        "//tensorflow/lite:testdata/multi_add.bin",
-    ],
-    tags = ["manual"],
-)
-
-ios_application(
-    name = "TensorFlowLiteApp",
-    app_icons = glob(["TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/**"]),
-    bundle_id = "com.tensorflow.lite.swift.TensorFlowLite",
-    families = [
-        "ipad",
-        "iphone",
-    ],
-    infoplists = ["TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist"],
-    launch_storyboard = "TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard",
-    minimum_os_version = MINIMUM_OS_VERSION,
-    sdk_frameworks = [
-        "CoreGraphics",
-    ],
-    tags = ["manual"],
-    deps = [":TensorFlowLiteAppLib"],
-)
-
-swift_library(
-    name = "TensorFlowLiteAppLib",
-    srcs = glob(["TestApps/TensorFlowLiteApp/TensorFlowLiteApp/*.swift"]),
-    module_name = "TensorFlowLiteAppLib",
-    tags = ["manual"],
-    deps = [
-        ":TensorFlowLite",
-        ":TensorFlowLiteAppResources",
-    ],
-)
-
-objc_library(
-    name = "TensorFlowLiteAppResources",
-    storyboards = glob([
-        "TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/*.storyboard",
-    ]),
-    tags = ["manual"],
-    deps = [":TestResources"],
-)
diff --git a/tensorflow/lite/experimental/swift/LICENSE b/tensorflow/lite/experimental/swift/LICENSE
deleted file mode 100644
index d645695673..0000000000
--- a/tensorflow/lite/experimental/swift/LICENSE
+++ /dev/null
@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
diff --git a/tensorflow/lite/experimental/swift/README.md b/tensorflow/lite/experimental/swift/README.md
deleted file mode 100644
index deb16c1e91..0000000000
--- a/tensorflow/lite/experimental/swift/README.md
+++ /dev/null
@@ -1,54 +0,0 @@
-# TensorFlow Lite for Swift
-
-[TensorFlow Lite](https://www.tensorflow.org/lite/) is TensorFlow's lightweight
-solution for Swift developers. It enables low-latency inference of on-device
-machine learning models with a small binary size and fast performance supporting
-hardware acceleration.
-
-## Getting Started
-
-### Bazel
-
-In your `BUILD` file, add the `TensorFlowLite` dependency:
-
-```python
-swift_library(
-  # ...
-  deps = [
-      "//tensorflow/lite/swift:TensorFlowLite",
-  ],
-)
-```
-
-In your Swift files, import the module:
-
-```swift
-import TensorFlowLite
-```
-
-### Tulsi
-
-Open the `TensorFlowLite.tulsiproj` using the [TulsiApp](https://github.com/bazelbuild/tulsi) or by
-running the [`generate_xcodeproj.sh`](https://github.com/bazelbuild/tulsi/blob/master/src/tools/generate_xcodeproj.sh)
-script:
-
-```shell
-generate_xcodeproj.sh --genconfig tensorflow/lite/swift/TensorFlowLite.tulsiproj:TensorFlowLite --outputfolder ~/path/to/generated/TensorFlowLite.xcodeproj
-```
-
-### CocoaPods
-
-Add the following to your `Podfile`:
-
-```ruby
-use_frameworks!
-pod 'TensorFlowLiteSwift'
-```
-
-Then, run `pod install`.
-
-In your Swift files, import the module:
-
-```swift
-import TensorFlowLite
-```
diff --git a/tensorflow/lite/experimental/swift/Sources/Interpreter.swift b/tensorflow/lite/experimental/swift/Sources/Interpreter.swift
deleted file mode 100644
index a14b5966b1..0000000000
--- a/tensorflow/lite/experimental/swift/Sources/Interpreter.swift
+++ /dev/null
@@ -1,265 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import Foundation
-import TensorFlowLiteCAPI
-
-/// A TensorFlow Lite interpreter that performs inference from a given model.
-public final class Interpreter {
-
-  /// The `TFL_Interpreter` C pointer type represented as an `UnsafePointer<TFL_Interpreter>`.
-  private typealias CInterpreter = OpaquePointer
-
-  /// Total number of input tensors associated with the model.
-  public var inputTensorCount: Int {
-    return Int(TFL_InterpreterGetInputTensorCount(cInterpreter))
-  }
-
-  /// Total number of output tensors associated with the model.
-  public var outputTensorCount: Int {
-    return Int(TFL_InterpreterGetOutputTensorCount(cInterpreter))
-  }
-
-  /// The underlying `TFL_Interpreter` C pointer.
-  private var cInterpreter: CInterpreter?
-
-  /// Creates a new model interpreter instance.
-  ///
-  /// - Parameters:
-  ///   - modelPath: Local file path to a TensorFlow Lite model.
-  ///   - options: Custom configurations for the interpreter. The default is `nil` indicating that
-  ///       interpreter will determine the configuration options.
-  /// - Throws: An error if the model could not be loaded or the interpreter could not be created.
-  public init(modelPath: String, options: InterpreterOptions? = nil) throws {
-    guard let model = Model(filePath: modelPath) else { throw InterpreterError.failedToLoadModel }
-
-    let cInterpreterOptions: OpaquePointer? = try options.map { options in
-      guard let cOptions = TFL_NewInterpreterOptions() else {
-        throw InterpreterError.failedToCreateInterpreter
-      }
-      if let threadCount = options.threadCount, threadCount > 0 {
-        TFL_InterpreterOptionsSetNumThreads(cOptions, Int32(threadCount))
-      }
-      if options.isErrorLoggingEnabled {
-        TFL_InterpreterOptionsSetErrorReporter(
-          cOptions,
-          { (_, format, arguments) in
-            guard let cFormat = format,
-                  let message = String(cFormat: cFormat, arguments: arguments)
-            else {
-              return
-            }
-            print(String(describing: InterpreterError.tensorFlowLiteError(message)))
-          },
-          nil
-        )
-      }
-      return cOptions
-    }
-    defer { TFL_DeleteInterpreterOptions(cInterpreterOptions) }
-
-    guard let cInterpreter = TFL_NewInterpreter(model.cModel, cInterpreterOptions) else {
-      throw InterpreterError.failedToCreateInterpreter
-    }
-    self.cInterpreter = cInterpreter
-  }
-
-  deinit {
-    TFL_DeleteInterpreter(cInterpreter)
-  }
-
-  /// Invokes the interpreter to perform inference from the loaded graph.
-  ///
-  /// - Throws: An error if the model was not ready because tensors were not allocated.
-  public func invoke() throws {
-    guard TFL_InterpreterInvoke(cInterpreter) == kTfLiteOk else {
-      // TODO(b/117510052): Determine which error to throw.
-      throw InterpreterError.allocateTensorsRequired
-    }
-  }
-
-  /// Returns the input tensor at the given index.
-  ///
-  /// - Parameters:
-  ///   - index: The index for the input tensor.
-  /// - Throws: An error if the index is invalid or the tensors have not been allocated.
-  /// - Returns: The input tensor at the given index.
-  public func input(at index: Int) throws -> Tensor {
-    let maxIndex = inputTensorCount - 1
-    guard case 0...maxIndex = index else {
-      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
-    }
-    guard let cTensor = TFL_InterpreterGetInputTensor(cInterpreter, Int32(index)),
-          let bytes = TFL_TensorData(cTensor),
-          let nameCString = TFL_TensorName(cTensor)
-    else {
-      throw InterpreterError.allocateTensorsRequired
-    }
-    guard let dataType = TensorDataType(type: TFL_TensorType(cTensor)) else {
-      throw InterpreterError.invalidTensorDataType
-    }
-
-    let name = String(cString: nameCString)
-    let rank = TFL_TensorNumDims(cTensor)
-    let dimensions = (0..<rank).map { Int(TFL_TensorDim(cTensor, $0)) }
-    let shape = TensorShape(dimensions)
-    let byteCount = TFL_TensorByteSize(cTensor)
-    let data = Data(bytes: bytes, count: byteCount)
-    let cQuantizationParams = TFL_TensorQuantizationParams(cTensor)
-    let scale = cQuantizationParams.scale
-    let zeroPoint = Int(cQuantizationParams.zero_point)
-    var quantizationParameters: QuantizationParameters? = nil
-    if scale != 0.0 {
-      // TODO(b/117510052): Update this check once the TfLiteQuantizationParams struct has a mode.
-      quantizationParameters = QuantizationParameters(scale: scale, zeroPoint: zeroPoint)
-    }
-    let tensor = Tensor(
-      name: name,
-      dataType: dataType,
-      shape: shape,
-      data: data,
-      quantizationParameters: quantizationParameters
-    )
-    return tensor
-  }
-
-  /// Returns the output tensor at the given index.
-  ///
-  /// - Parameters:
-  ///   - index: The index for the output tensor.
-  /// - Throws: An error if the index is invalid, tensors haven't been allocated, or interpreter
-  ///     hasn't been invoked for models that dynamically compute output tensors based on the values
-  ///     of its input tensors.
-  /// - Returns: The output tensor at the given index.
-  public func output(at index: Int) throws -> Tensor {
-    let maxIndex = outputTensorCount - 1
-    guard case 0...maxIndex = index else {
-      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
-    }
-    guard let cTensor = TFL_InterpreterGetOutputTensor(cInterpreter, Int32(index)),
-          let bytes = TFL_TensorData(cTensor),
-          let nameCString = TFL_TensorName(cTensor)
-    else {
-      // TODO(b/117510052): Determine which error to throw.
-      throw InterpreterError.invokeInterpreterRequired
-    }
-    guard let dataType = TensorDataType(type: TFL_TensorType(cTensor)) else {
-      throw InterpreterError.invalidTensorDataType
-    }
-
-    let name = String(cString: nameCString)
-    let rank = TFL_TensorNumDims(cTensor)
-    let dimensions = (0..<rank).map { Int(TFL_TensorDim(cTensor, $0)) }
-    let shape = TensorShape(dimensions)
-    let byteCount = TFL_TensorByteSize(cTensor)
-    let data = Data(bytes: bytes, count: byteCount)
-    let cQuantizationParams = TFL_TensorQuantizationParams(cTensor)
-    let scale = cQuantizationParams.scale
-    let zeroPoint = Int(cQuantizationParams.zero_point)
-    var quantizationParameters: QuantizationParameters? = nil
-    if scale != 0.0 {
-      // TODO(b/117510052): Update this check once the TfLiteQuantizationParams struct has a mode.
-      quantizationParameters = QuantizationParameters(scale: scale, zeroPoint: zeroPoint)
-    }
-    let tensor = Tensor(
-      name: name,
-      dataType: dataType,
-      shape: shape,
-      data: data,
-      quantizationParameters: quantizationParameters
-    )
-    return tensor
-  }
-
-  /// Resizes the input tensor at the given index to the specified tensor shape.
-  ///
-  /// - Note: After resizing an input tensor, the client **must** explicitly call
-  ///     `allocateTensors()` before attempting to access the resized tensor data or invoking the
-  ///     interpreter to perform inference.
-  /// - Parameters:
-  ///   - index: The index for the input tensor.
-  ///   - shape: The shape that the input tensor should be resized to.
-  /// - Throws: An error if the input tensor at the given index could not be resized.
-  public func resizeInput(at index: Int, to shape: TensorShape) throws {
-    let maxIndex = inputTensorCount - 1
-    guard case 0...maxIndex = index else {
-      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
-    }
-    guard TFL_InterpreterResizeInputTensor(
-            cInterpreter,
-            Int32(index),
-            shape.int32Dimensions,
-            Int32(shape.rank)
-          ) == kTfLiteOk
-    else {
-      throw InterpreterError.failedToResizeInputTensor(index: index)
-    }
-  }
-
-  /// Copies the given data to the input tensor at the given index.
-  ///
-  /// - Parameters:
-  ///   - data: The data to be copied to the input tensor's data buffer.
-  ///   - index: The index for the input tensor.
-  /// - Throws: An error if the `data.count` does not match the input tensor's `data.count` or if
-  ///     the given index is invalid.
-  /// - Returns: The input tensor with the copied data.
-  @discardableResult
-  public func copy(_ data: Data, toInputAt index: Int) throws -> Tensor {
-    let maxIndex = inputTensorCount - 1
-    guard case 0...maxIndex = index else {
-      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
-    }
-    guard let cTensor = TFL_InterpreterGetInputTensor(cInterpreter, Int32(index)) else {
-      throw InterpreterError.allocateTensorsRequired
-    }
-
-    let byteCount = TFL_TensorByteSize(cTensor)
-    guard data.count == byteCount else {
-      throw InterpreterError.invalidTensorDataCount(provided: data.count, required: byteCount)
-    }
-
-    let status = data.withUnsafeBytes { TFL_TensorCopyFromBuffer(cTensor, $0, data.count) }
-    guard status == kTfLiteOk else { throw InterpreterError.failedToCopyDataToInputTensor }
-    return try input(at: index)
-  }
-
-  /// Allocates memory for all input tensors based on their `TensorShape`s.
-  ///
-  /// - Note: This is a relatively expensive operation and should only be called after creating the
-  ///     interpreter and/or resizing any input tensors.
-  /// - Throws: An error if memory could not be allocated for the input tensors.
-  public func allocateTensors() throws {
-    guard TFL_InterpreterAllocateTensors(cInterpreter) == kTfLiteOk else {
-      throw InterpreterError.failedToAllocateTensors
-    }
-  }
-}
-
-// MARK: - Extensions
-
-extension String {
-  /// Returns a new `String` initialized by using the given format C array as a template into which
-  /// the remaining argument values are substituted according to the user’s default locale.
-  ///
-  /// - Note: Returns `nil` if a new `String` could not be constructed from the given values.
-  /// - Parameters:
-  ///   - cFormat: The format C array as a template for substituting values.
-  ///   - arguments: A C pointer to a `va_list` of arguments to substitute into `cFormat`.
-  init?(cFormat: UnsafePointer<CChar>, arguments: CVaListPointer) {
-    var buffer: UnsafeMutablePointer<CChar>?
-    guard vasprintf(&buffer, cFormat, arguments) != 0, let cString = buffer else { return nil }
-    self.init(validatingUTF8: cString)
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/Sources/InterpreterError.swift b/tensorflow/lite/experimental/swift/Sources/InterpreterError.swift
deleted file mode 100644
index 5de58b997a..0000000000
--- a/tensorflow/lite/experimental/swift/Sources/InterpreterError.swift
+++ /dev/null
@@ -1,99 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import Foundation
-
-/// TensorFlow Lite interpreter errors.
-public enum InterpreterError: Error {
-  case invalidTensorIndex(index: Int, maxIndex: Int)
-  case invalidTensorDataCount(provided: Int, required: Int)
-  case invalidTensorDataType
-  case failedToLoadModel
-  case failedToCreateInterpreter
-  case failedToResizeInputTensor(index: Int)
-  case failedToCopyDataToInputTensor
-  case failedToAllocateTensors
-  case allocateTensorsRequired
-  case invokeInterpreterRequired
-  case tensorFlowLiteError(String)
-}
-
-// MARK: - Extensions
-
-extension InterpreterError: LocalizedError {
-  /// Localized description of the interpreter error.
-  public var errorDescription: String? {
-    switch self {
-    case .invalidTensorIndex(let index, let maxIndex):
-      return "Invalid tensor index \(index), max index is \(maxIndex)."
-    case .invalidTensorDataCount(let providedCount, let requiredCount):
-      return "Provided data count \(providedCount) must match the required count \(requiredCount)."
-    case .invalidTensorDataType:
-      return "Tensor data type is unsupported or could not be determined because of a model error."
-    case .failedToLoadModel:
-      return "Failed to load the given model."
-    case .failedToCreateInterpreter:
-      return "Failed to create the interpreter."
-    case .failedToResizeInputTensor(let index):
-      return "Failed to resize input tesnor at index \(index)."
-    case .failedToCopyDataToInputTensor:
-      return "Failed to copy data to input tensor."
-    case .failedToAllocateTensors:
-      return "Failed to allocate memory for input tensors."
-    case .allocateTensorsRequired:
-      return "Must call allocateTensors()."
-    case .invokeInterpreterRequired:
-      return "Must call invoke()."
-    case .tensorFlowLiteError(let message):
-      return "TensorFlow Lite Error: \(message)"
-    }
-  }
-}
-
-extension InterpreterError: CustomStringConvertible {
-  /// Textual representation of the TensorFlow Lite interpreter error.
-  public var description: String {
-    return errorDescription ?? "Unknown error."
-  }
-}
-
-#if swift(>=4.2)
-extension InterpreterError: Equatable {}
-#else
-extension InterpreterError: Equatable {
-  public static func == (lhs: InterpreterError, rhs: InterpreterError) -> Bool {
-    switch (lhs, rhs) {
-    case (.invalidTensorDataType, .invalidTensorDataType),
-         (.failedToLoadModel, .failedToLoadModel),
-         (.failedToCreateInterpreter, .failedToCreateInterpreter),
-         (.failedToAllocateTensors, .failedToAllocateTensors),
-         (.allocateTensorsRequired, .allocateTensorsRequired),
-         (.invokeInterpreterRequired, .invokeInterpreterRequired):
-      return true
-    case (.invalidTensorIndex(let lhsIndex, let lhsMaxIndex),
-          .invalidTensorIndex(let rhsIndex, let rhsMaxIndex)):
-      return lhsIndex == rhsIndex && lhsMaxIndex == rhsMaxIndex
-    case (.invalidTensorDataCount(let lhsProvidedCount, let lhsRequiredCount),
-          .invalidTensorDataCount(let rhsProvidedCount, let rhsRequiredCount)):
-      return lhsProvidedCount == rhsProvidedCount && lhsRequiredCount == rhsRequiredCount
-    case (.failedToResizeInputTensor(let lhsIndex), .failedToResizeInputTensor(let rhsIndex)):
-      return lhsIndex == rhsIndex
-    case (.tensorFlowLiteError(let lhsMessage), .tensorFlowLiteError(let rhsMessage)):
-      return lhsMessage == rhsMessage
-    default:
-      return false
-    }
-  }
-}
-#endif  // swift(>=4.2)
diff --git a/tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift b/tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift
deleted file mode 100644
index 2365fd7ade..0000000000
--- a/tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift
+++ /dev/null
@@ -1,29 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import Foundation
-
-/// Custom configuration options for a TensorFlow Lite interpreter.
-public struct InterpreterOptions: Equatable {
-
-  /// Maximum number of CPU threads that the interpreter should run on. Default is `nil` which
-  /// indicates that the `Interpreter` will decide the number of threads to use.
-  public var threadCount: Int? = nil
-
-  /// Whether error logging to the console is enabled. The default is `false`.
-  public var isErrorLoggingEnabled = false
-
-  /// Creates a new instance of interpreter options.
-  public init() {}
-}
diff --git a/tensorflow/lite/experimental/swift/Sources/Model.swift b/tensorflow/lite/experimental/swift/Sources/Model.swift
deleted file mode 100644
index e8c49ff1ae..0000000000
--- a/tensorflow/lite/experimental/swift/Sources/Model.swift
+++ /dev/null
@@ -1,40 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import Foundation
-import TensorFlowLiteCAPI
-
-/// A TensorFlow Lite model used by the 'Interpreter` to perform inference.
-final class Model {
-
-  /// The `TFL_Model` C pointer type represented as an `UnsafePointer<TFL_Model>`.
-  typealias CModel = OpaquePointer
-
-  /// The underlying `TFL_Model` C pointer.
-  let cModel: CModel?
-
-  /// Creates a new model instance.
-  ///
-  /// - Precondition: Initialization can fail if the given `filePath` is invalid.
-  /// - Parameters:
-  ///   - filePath: Local file path to a TensorFlow Lite model.
-  init?(filePath: String) {
-    guard !filePath.isEmpty, let cModel = TFL_NewModelFromFile(filePath) else { return nil }
-    self.cModel = cModel
-  }
-
-  deinit {
-    TFL_DeleteModel(cModel)
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift b/tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift
deleted file mode 100644
index f367875644..0000000000
--- a/tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift
+++ /dev/null
@@ -1,38 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import Foundation
-
-/// Parameters that determine the mapping of quantized values to real values. Quantized values can
-/// be mapped to float values using the following conversion:
-/// `realValue = scale * (quantizedValue - zeroPoint)`.
-public struct QuantizationParameters {
-
-  /// Difference between real values corresponding to consecutive quantized values differing by 1.
-  /// For example, the range of quantized values for `UInt8` data type is [0, 255].
-  public let scale: Float
-
-  /// Quantized value that corresponds to the real 0 value.
-  public let zeroPoint: Int
-
-  /// Creates a new quantization parameters instance.
-  ///
-  /// - Parameters:
-  ///   - scale: Scale value for asymmetric quantization.
-  ///   - zeroPoint: Zero point for asymmetric quantization.
-  init(scale: Float, zeroPoint: Int) {
-    self.scale = scale
-    self.zeroPoint = zeroPoint
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/Sources/Tensor.swift b/tensorflow/lite/experimental/swift/Sources/Tensor.swift
deleted file mode 100644
index b738d87549..0000000000
--- a/tensorflow/lite/experimental/swift/Sources/Tensor.swift
+++ /dev/null
@@ -1,138 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import Foundation
-import TensorFlowLiteCAPI
-
-/// An input or output tensor in a TensorFlow Lite graph.
-public struct Tensor {
-
-  /// Name of the tensor.
-  public let name: String
-
-  /// Data type of the tensor.
-  public let dataType: TensorDataType
-
-  /// Shape of the tensor.
-  public let shape: TensorShape
-
-  /// Data in the input or output tensor.
-  public let data: Data
-
-  /// Quantization parameters for the tensor if using a quantized model.
-  public let quantizationParameters: QuantizationParameters?
-
-  /// Creates a new input or output tensor instance.
-  ///
-  /// - Parameters:
-  ///   - name: Name of the tensor.
-  ///   - dataType: Data type of the tensor.
-  ///   - data: Data in the input tensor.
-  ///   - quantizationParameters Quantization parameters for the tensor if using a quantized model.
-  ///       The default is `nil`.
-  init(
-    name: String,
-    dataType: TensorDataType,
-    shape: TensorShape,
-    data: Data,
-    quantizationParameters: QuantizationParameters? = nil
-  ) {
-    self.name = name
-    self.dataType = dataType
-    self.shape = shape
-    self.data = data
-    self.quantizationParameters = quantizationParameters
-  }
-}
-
-/// Supported TensorFlow Lite tensor data types.
-public enum TensorDataType: Equatable {
-  /// 32-bit single precision floating point tensor data type.
-  case float32
-  /// 8-bit unsigned integer tensor data type.
-  case uInt8
-  /// 16-bit signed integer tensor data type.
-  case int16
-  /// 32-bit signed integer tensor data type.
-  case int32
-  /// 64-bit signed integer tensor data type.
-  case int64
-  /// Boolean tensor data type.
-  case bool
-
-  /// Creates a new tensor data type from the given `TFL_Type` or `nil` if the data type is
-  /// unsupported or could not be determined because there was an error.
-  ///
-  /// - Parameter type: A data type supported by a tensor.
-  init?(type: TFL_Type) {
-    switch type {
-    case kTfLiteFloat32:
-      self = .float32
-    case kTfLiteUInt8:
-      self = .uInt8
-    case kTfLiteInt16:
-      self = .int16
-    case kTfLiteInt32:
-      self = .int32
-    case kTfLiteInt64:
-      self = .int64
-    case kTfLiteBool:
-      self = .bool
-    case kTfLiteNoType:
-      fallthrough
-    default:
-      return nil
-    }
-  }
-}
-
-/// The shape of a TensorFlow Lite tensor.
-public struct TensorShape {
-
-  /// The number of dimensions of the tensor.
-  public let rank: Int
-
-  /// Array of dimensions for the tensor.
-  public let dimensions: [Int]
-
-  /// Array of `Int32` dimensions for the tensor.
-  var int32Dimensions: [Int32] { return dimensions.map(Int32.init) }
-
-  /// Creates a new tensor shape instance with the given array of dimensions.
-  ///
-  /// - Parameters:
-  ///   - dimensions: Dimensions for the tensor.
-  public init(_ dimensions: [Int]) {
-    self.rank = dimensions.count
-    self.dimensions = dimensions
-  }
-
-  /// Creates a new tensor shape instance with the given elements representing the dimensions.
-  ///
-  /// - Parameters:
-  ///   - elements: Dimensions for the tensor.
-  public init(_ elements: Int...) {
-    self.init(elements)
-  }
-}
-
-extension TensorShape: ExpressibleByArrayLiteral {
-  /// Creates a new tensor shape instance with the given array literal representing the dimensions.
-  ///
-  /// - Parameters:
-  ///   - arrayLiteral: Dimensions for the tensor.
-  public init(arrayLiteral: Int...) {
-    self.init(arrayLiteral)
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
deleted file mode 100644
index 16bc6cbfe8..0000000000
--- a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
+++ /dev/null
@@ -1,57 +0,0 @@
-{
-  "sourceFilters" : [
-    "tensorflow/lite/experimental/c",
-    "tensorflow/lite/experimental/swift",
-    "tensorflow/lite/experimental/swift/Sources",
-    "tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp",
-    "tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj",
-    "tensorflow/lite/experimental/swift/Tests",
-  ],
-  "buildTargets" : [
-    "//tensorflow/lite/experimental/swift:TensorFlowLite",
-    "//tensorflow/lite/experimental/swift:TensorFlowLiteApp",
-    "//tensorflow/lite/experimental/swift:TensorFlowLiteTests",
-  ],
-  "projectName" : "TensorFlowLite",
-  "optionSet" : {
-    "LaunchActionPreActionScript" : {
-      "p" : "$(inherited)"
-    },
-    "BazelBuildStartupOptionsRelease" : {
-      "p" : "$(inherited)"
-    },
-    "BazelBuildOptionsRelease" : {
-      "p" : "$(inherited)"
-    },
-    "BazelBuildOptionsDebug" : {
-      "p" : "$(inherited)"
-    },
-    "EnvironmentVariables" : {
-      "p" : "$(inherited)"
-    },
-    "BuildActionPreActionScript" : {
-      "p" : "$(inherited)"
-    },
-    "CommandlineArguments" : {
-      "p" : "$(inherited)"
-    },
-    "TestActionPreActionScript" : {
-      "p" : "$(inherited)"
-    },
-    "BazelBuildStartupOptionsDebug" : {
-      "p" : "$(inherited)"
-    },
-    "BuildActionPostActionScript" : {
-      "p" : "$(inherited)"
-    },
-    "TestActionPostActionScript" : {
-      "p" : "$(inherited)"
-    },
-    "LaunchActionPostActionScript" : {
-      "p" : "$(inherited)"
-    }
-  },
-  "additionalFilePaths" : [
-    "tensorflow/lite/experimental/swift/BUILD"
-  ]
-}
diff --git a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
deleted file mode 100644
index 82ac8aa381..0000000000
--- a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
+++ /dev/null
@@ -1,14 +0,0 @@
-{
-  "configDefaults" : {
-    "optionSet" : {
-      "ProjectPrioritizesSwift" : {
-        "p" : "YES"
-      }
-    }
-  },
-  "projectName" : "TensorFlowLite",
-  "packages" : [
-    "tensorflow/lite/experimental/swift"
-  ],
-  "workspaceRoot" : "../../../../.."
-}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj
deleted file mode 100644
index fbbf9a1de2..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj
+++ /dev/null
@@ -1,345 +0,0 @@
-// !$*UTF8*$!
-{
-	archiveVersion = 1;
-	classes = {
-	};
-	objectVersion = 50;
-	objects = {
-
-/* Begin PBXBuildFile section */
-		4A7304B421500B8400C90B21 /* Data+TensorFlowLite.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4A7304B321500B8300C90B21 /* Data+TensorFlowLite.swift */; };
-		4AA72B732146ED64006C3AEF /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4AA72B722146ED64006C3AEF /* AppDelegate.swift */; };
-		4AA72B752146ED64006C3AEF /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4AA72B742146ED64006C3AEF /* ViewController.swift */; };
-		4AA72B782146ED64006C3AEF /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 4AA72B762146ED64006C3AEF /* Main.storyboard */; };
-		4AA72B7A2146ED66006C3AEF /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 4AA72B792146ED66006C3AEF /* Assets.xcassets */; };
-		4AA72B7D2146ED66006C3AEF /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 4AA72B7B2146ED66006C3AEF /* LaunchScreen.storyboard */; };
-		4ADDE0CE2176600E00FF07A2 /* Array+TensorFlowLite.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4ADDE0CD2176600900FF07A2 /* Array+TensorFlowLite.swift */; };
-/* End PBXBuildFile section */
-
-/* Begin PBXFileReference section */
-		4A7304B321500B8300C90B21 /* Data+TensorFlowLite.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "Data+TensorFlowLite.swift"; sourceTree = "<group>"; };
-		4AA72B6F2146ED64006C3AEF /* TensorFlowLiteApp.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = TensorFlowLiteApp.app; sourceTree = BUILT_PRODUCTS_DIR; };
-		4AA72B722146ED64006C3AEF /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = "<group>"; };
-		4AA72B742146ED64006C3AEF /* ViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewController.swift; sourceTree = "<group>"; };
-		4AA72B772146ED64006C3AEF /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = "<group>"; };
-		4AA72B792146ED66006C3AEF /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = "<group>"; };
-		4AA72B7C2146ED66006C3AEF /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = "<group>"; };
-		4AA72B7E2146ED66006C3AEF /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
-		4ADDE0CD2176600900FF07A2 /* Array+TensorFlowLite.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "Array+TensorFlowLite.swift"; sourceTree = "<group>"; };
-/* End PBXFileReference section */
-
-/* Begin PBXFrameworksBuildPhase section */
-		4AA72B6C2146ED64006C3AEF /* Frameworks */ = {
-			isa = PBXFrameworksBuildPhase;
-			buildActionMask = 2147483647;
-			files = (
-			);
-			runOnlyForDeploymentPostprocessing = 0;
-		};
-/* End PBXFrameworksBuildPhase section */
-
-/* Begin PBXGroup section */
-		4AA72B662146ED64006C3AEF = {
-			isa = PBXGroup;
-			children = (
-				4AA72B712146ED64006C3AEF /* TensorFlowLiteApp */,
-				4AA72B702146ED64006C3AEF /* Products */,
-			);
-			sourceTree = "<group>";
-		};
-		4AA72B702146ED64006C3AEF /* Products */ = {
-			isa = PBXGroup;
-			children = (
-				4AA72B6F2146ED64006C3AEF /* TensorFlowLiteApp.app */,
-			);
-			name = Products;
-			sourceTree = "<group>";
-		};
-		4AA72B712146ED64006C3AEF /* TensorFlowLiteApp */ = {
-			isa = PBXGroup;
-			children = (
-				4AA72B722146ED64006C3AEF /* AppDelegate.swift */,
-				4ADDE0CD2176600900FF07A2 /* Array+TensorFlowLite.swift */,
-				4A7304B321500B8300C90B21 /* Data+TensorFlowLite.swift */,
-				4AA72B742146ED64006C3AEF /* ViewController.swift */,
-				4AA72B762146ED64006C3AEF /* Main.storyboard */,
-				4AA72B792146ED66006C3AEF /* Assets.xcassets */,
-				4AA72B7B2146ED66006C3AEF /* LaunchScreen.storyboard */,
-				4AA72B7E2146ED66006C3AEF /* Info.plist */,
-			);
-			path = TensorFlowLiteApp;
-			sourceTree = "<group>";
-		};
-/* End PBXGroup section */
-
-/* Begin PBXNativeTarget section */
-		4AA72B6E2146ED64006C3AEF /* TensorFlowLiteApp */ = {
-			isa = PBXNativeTarget;
-			buildConfigurationList = 4AA72B812146ED66006C3AEF /* Build configuration list for PBXNativeTarget "TensorFlowLiteApp" */;
-			buildPhases = (
-				4AA72B6B2146ED64006C3AEF /* Sources */,
-				4AA72B6C2146ED64006C3AEF /* Frameworks */,
-				4AA72B6D2146ED64006C3AEF /* Resources */,
-			);
-			buildRules = (
-			);
-			dependencies = (
-			);
-			name = TensorFlowLiteApp;
-			productName = TensorFlowLiteApp;
-			productReference = 4AA72B6F2146ED64006C3AEF /* TensorFlowLiteApp.app */;
-			productType = "com.apple.product-type.application";
-		};
-/* End PBXNativeTarget section */
-
-/* Begin PBXProject section */
-		4AA72B672146ED64006C3AEF /* Project object */ = {
-			isa = PBXProject;
-			attributes = {
-				LastSwiftUpdateCheck = 0940;
-				LastUpgradeCheck = 0940;
-				ORGANIZATIONNAME = Google;
-				TargetAttributes = {
-					4AA72B6E2146ED64006C3AEF = {
-						CreatedOnToolsVersion = 9.4.1;
-					};
-				};
-			};
-			buildConfigurationList = 4AA72B6A2146ED64006C3AEF /* Build configuration list for PBXProject "TensorFlowLiteApp" */;
-			compatibilityVersion = "Xcode 9.3";
-			developmentRegion = en;
-			hasScannedForEncodings = 0;
-			knownRegions = (
-				en,
-				Base,
-			);
-			mainGroup = 4AA72B662146ED64006C3AEF;
-			productRefGroup = 4AA72B702146ED64006C3AEF /* Products */;
-			projectDirPath = "";
-			projectRoot = "";
-			targets = (
-				4AA72B6E2146ED64006C3AEF /* TensorFlowLiteApp */,
-			);
-		};
-/* End PBXProject section */
-
-/* Begin PBXResourcesBuildPhase section */
-		4AA72B6D2146ED64006C3AEF /* Resources */ = {
-			isa = PBXResourcesBuildPhase;
-			buildActionMask = 2147483647;
-			files = (
-				4AA72B7D2146ED66006C3AEF /* LaunchScreen.storyboard in Resources */,
-				4AA72B7A2146ED66006C3AEF /* Assets.xcassets in Resources */,
-				4AA72B782146ED64006C3AEF /* Main.storyboard in Resources */,
-			);
-			runOnlyForDeploymentPostprocessing = 0;
-		};
-/* End PBXResourcesBuildPhase section */
-
-/* Begin PBXSourcesBuildPhase section */
-		4AA72B6B2146ED64006C3AEF /* Sources */ = {
-			isa = PBXSourcesBuildPhase;
-			buildActionMask = 2147483647;
-			files = (
-				4AA72B732146ED64006C3AEF /* AppDelegate.swift in Sources */,
-				4ADDE0CE2176600E00FF07A2 /* Array+TensorFlowLite.swift in Sources */,
-				4A7304B421500B8400C90B21 /* Data+TensorFlowLite.swift in Sources */,
-				4AA72B752146ED64006C3AEF /* ViewController.swift in Sources */,
-			);
-			runOnlyForDeploymentPostprocessing = 0;
-		};
-/* End PBXSourcesBuildPhase section */
-
-/* Begin PBXVariantGroup section */
-		4AA72B762146ED64006C3AEF /* Main.storyboard */ = {
-			isa = PBXVariantGroup;
-			children = (
-				4AA72B772146ED64006C3AEF /* Base */,
-			);
-			name = Main.storyboard;
-			sourceTree = "<group>";
-		};
-		4AA72B7B2146ED66006C3AEF /* LaunchScreen.storyboard */ = {
-			isa = PBXVariantGroup;
-			children = (
-				4AA72B7C2146ED66006C3AEF /* Base */,
-			);
-			name = LaunchScreen.storyboard;
-			sourceTree = "<group>";
-		};
-/* End PBXVariantGroup section */
-
-/* Begin XCBuildConfiguration section */
-		4AA72B7F2146ED66006C3AEF /* Debug */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				ALWAYS_SEARCH_USER_PATHS = NO;
-				CLANG_ANALYZER_NONNULL = YES;
-				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
-				CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
-				CLANG_CXX_LIBRARY = "libc++";
-				CLANG_ENABLE_MODULES = YES;
-				CLANG_ENABLE_OBJC_ARC = YES;
-				CLANG_ENABLE_OBJC_WEAK = YES;
-				CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
-				CLANG_WARN_BOOL_CONVERSION = YES;
-				CLANG_WARN_COMMA = YES;
-				CLANG_WARN_CONSTANT_CONVERSION = YES;
-				CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
-				CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
-				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
-				CLANG_WARN_EMPTY_BODY = YES;
-				CLANG_WARN_ENUM_CONVERSION = YES;
-				CLANG_WARN_INFINITE_RECURSION = YES;
-				CLANG_WARN_INT_CONVERSION = YES;
-				CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
-				CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
-				CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
-				CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
-				CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
-				CLANG_WARN_STRICT_PROTOTYPES = YES;
-				CLANG_WARN_SUSPICIOUS_MOVE = YES;
-				CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
-				CLANG_WARN_UNREACHABLE_CODE = YES;
-				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
-				CODE_SIGN_IDENTITY = "iPhone Developer";
-				COPY_PHASE_STRIP = NO;
-				DEBUG_INFORMATION_FORMAT = dwarf;
-				ENABLE_STRICT_OBJC_MSGSEND = YES;
-				ENABLE_TESTABILITY = YES;
-				GCC_C_LANGUAGE_STANDARD = gnu11;
-				GCC_DYNAMIC_NO_PIC = NO;
-				GCC_NO_COMMON_BLOCKS = YES;
-				GCC_OPTIMIZATION_LEVEL = 0;
-				GCC_PREPROCESSOR_DEFINITIONS = (
-					"DEBUG=1",
-					"$(inherited)",
-				);
-				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
-				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
-				GCC_WARN_UNDECLARED_SELECTOR = YES;
-				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
-				GCC_WARN_UNUSED_FUNCTION = YES;
-				GCC_WARN_UNUSED_VARIABLE = YES;
-				IPHONEOS_DEPLOYMENT_TARGET = 11.4;
-				MTL_ENABLE_DEBUG_INFO = YES;
-				ONLY_ACTIVE_ARCH = YES;
-				SDKROOT = iphoneos;
-				SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;
-				SWIFT_OPTIMIZATION_LEVEL = "-Onone";
-			};
-			name = Debug;
-		};
-		4AA72B802146ED66006C3AEF /* Release */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				ALWAYS_SEARCH_USER_PATHS = NO;
-				CLANG_ANALYZER_NONNULL = YES;
-				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
-				CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
-				CLANG_CXX_LIBRARY = "libc++";
-				CLANG_ENABLE_MODULES = YES;
-				CLANG_ENABLE_OBJC_ARC = YES;
-				CLANG_ENABLE_OBJC_WEAK = YES;
-				CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
-				CLANG_WARN_BOOL_CONVERSION = YES;
-				CLANG_WARN_COMMA = YES;
-				CLANG_WARN_CONSTANT_CONVERSION = YES;
-				CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
-				CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
-				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
-				CLANG_WARN_EMPTY_BODY = YES;
-				CLANG_WARN_ENUM_CONVERSION = YES;
-				CLANG_WARN_INFINITE_RECURSION = YES;
-				CLANG_WARN_INT_CONVERSION = YES;
-				CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
-				CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
-				CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
-				CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
-				CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
-				CLANG_WARN_STRICT_PROTOTYPES = YES;
-				CLANG_WARN_SUSPICIOUS_MOVE = YES;
-				CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
-				CLANG_WARN_UNREACHABLE_CODE = YES;
-				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
-				CODE_SIGN_IDENTITY = "iPhone Developer";
-				COPY_PHASE_STRIP = NO;
-				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
-				ENABLE_NS_ASSERTIONS = NO;
-				ENABLE_STRICT_OBJC_MSGSEND = YES;
-				GCC_C_LANGUAGE_STANDARD = gnu11;
-				GCC_NO_COMMON_BLOCKS = YES;
-				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
-				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
-				GCC_WARN_UNDECLARED_SELECTOR = YES;
-				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
-				GCC_WARN_UNUSED_FUNCTION = YES;
-				GCC_WARN_UNUSED_VARIABLE = YES;
-				IPHONEOS_DEPLOYMENT_TARGET = 11.4;
-				MTL_ENABLE_DEBUG_INFO = NO;
-				SDKROOT = iphoneos;
-				SWIFT_COMPILATION_MODE = wholemodule;
-				SWIFT_OPTIMIZATION_LEVEL = "-O";
-				VALIDATE_PRODUCT = YES;
-			};
-			name = Release;
-		};
-		4AA72B822146ED66006C3AEF /* Debug */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
-				CODE_SIGN_STYLE = Automatic;
-				INFOPLIST_FILE = TensorFlowLiteApp/Info.plist;
-				LD_RUNPATH_SEARCH_PATHS = (
-					"$(inherited)",
-					"@executable_path/Frameworks",
-				);
-				PRODUCT_BUNDLE_IDENTIFIER = com.tensorflow.lite.swift.TensorFlowLite;
-				PRODUCT_NAME = "$(TARGET_NAME)";
-				SWIFT_VERSION = 4.0;
-				TARGETED_DEVICE_FAMILY = "1,2";
-			};
-			name = Debug;
-		};
-		4AA72B832146ED66006C3AEF /* Release */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
-				CODE_SIGN_STYLE = Automatic;
-				INFOPLIST_FILE = TensorFlowLiteApp/Info.plist;
-				LD_RUNPATH_SEARCH_PATHS = (
-					"$(inherited)",
-					"@executable_path/Frameworks",
-				);
-				PRODUCT_BUNDLE_IDENTIFIER = com.tensorflow.lite.swift.TensorFlowLite;
-				PRODUCT_NAME = "$(TARGET_NAME)";
-				SWIFT_VERSION = 4.0;
-				TARGETED_DEVICE_FAMILY = "1,2";
-			};
-			name = Release;
-		};
-/* End XCBuildConfiguration section */
-
-/* Begin XCConfigurationList section */
-		4AA72B6A2146ED64006C3AEF /* Build configuration list for PBXProject "TensorFlowLiteApp" */ = {
-			isa = XCConfigurationList;
-			buildConfigurations = (
-				4AA72B7F2146ED66006C3AEF /* Debug */,
-				4AA72B802146ED66006C3AEF /* Release */,
-			);
-			defaultConfigurationIsVisible = 0;
-			defaultConfigurationName = Release;
-		};
-		4AA72B812146ED66006C3AEF /* Build configuration list for PBXNativeTarget "TensorFlowLiteApp" */ = {
-			isa = XCConfigurationList;
-			buildConfigurations = (
-				4AA72B822146ED66006C3AEF /* Debug */,
-				4AA72B832146ED66006C3AEF /* Release */,
-			);
-			defaultConfigurationIsVisible = 0;
-			defaultConfigurationName = Release;
-		};
-/* End XCConfigurationList section */
-	};
-	rootObject = 4AA72B672146ED64006C3AEF /* Project object */;
-}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift
deleted file mode 100644
index ffa90a06ad..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift
+++ /dev/null
@@ -1,24 +0,0 @@
-import UIKit
-
-@UIApplicationMain
-
-final class AppDelegate: UIResponder, UIApplicationDelegate {
-
-  /// The main window of the app.
-  var window: UIWindow?
-
-  func application(
-    _ application: UIApplication,
-    didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? = nil
-  ) -> Bool {
-    return true
-  }
-}
-
-// MARK: - Extensions
-
-#if !swift(>=4.2)
-extension UIApplication {
-  typealias LaunchOptionsKey = UIApplicationLaunchOptionsKey
-}
-#endif  // !swift(>=4.2)
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift
deleted file mode 100644
index 56df1ce659..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift
+++ /dev/null
@@ -1,22 +0,0 @@
-import Foundation
-
-extension Array {
-  /// Creates a new array from the bytes of the given unsafe data.
-  ///
-  /// - Warning: The array's `Element` type must be trivial in that it can be copied bit for bit
-  ///     with no indirection or reference-counting operations; otherwise, copying the raw bytes in
-  ///     the `unsafeData`'s buffer to a new array returns an unsafe copy.
-  /// - Note: Returns `nil` if `unsafeData.count` is not a multiple of
-  ///     `MemoryLayout<Element>.stride`.
-  /// - Parameter unsafeData: The data containing the bytes to turn into an array.
-  init?(unsafeData: Data) {
-    guard unsafeData.count % MemoryLayout<Element>.stride == 0 else { return nil }
-    let elements = unsafeData.withUnsafeBytes {
-      UnsafeBufferPointer<Element>(
-        start: $0,
-        count: unsafeData.count / MemoryLayout<Element>.stride
-      )
-    }
-    self.init(elements)
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json
deleted file mode 100644
index d8db8d65fd..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json
+++ /dev/null
@@ -1,98 +0,0 @@
-{
-  "images" : [
-    {
-      "idiom" : "iphone",
-      "size" : "20x20",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "iphone",
-      "size" : "20x20",
-      "scale" : "3x"
-    },
-    {
-      "idiom" : "iphone",
-      "size" : "29x29",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "iphone",
-      "size" : "29x29",
-      "scale" : "3x"
-    },
-    {
-      "idiom" : "iphone",
-      "size" : "40x40",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "iphone",
-      "size" : "40x40",
-      "scale" : "3x"
-    },
-    {
-      "idiom" : "iphone",
-      "size" : "60x60",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "iphone",
-      "size" : "60x60",
-      "scale" : "3x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "20x20",
-      "scale" : "1x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "20x20",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "29x29",
-      "scale" : "1x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "29x29",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "40x40",
-      "scale" : "1x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "40x40",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "76x76",
-      "scale" : "1x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "76x76",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "83.5x83.5",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "ios-marketing",
-      "size" : "1024x1024",
-      "scale" : "1x"
-    }
-  ],
-  "info" : {
-    "version" : 1,
-    "author" : "xcode"
-  }
-}
\ No newline at end of file
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json
deleted file mode 100644
index da4a164c91..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
-  "info" : {
-    "version" : 1,
-    "author" : "xcode"
-  }
-}
\ No newline at end of file
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard
deleted file mode 100644
index a07a1321be..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard
+++ /dev/null
@@ -1,44 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="14109" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" launchScreen="YES" useTraitCollections="YES" colorMatched="YES" initialViewController="01J-lp-oVM">
-    <device id="retina4_7" orientation="portrait">
-        <adaptation id="fullscreen"/>
-    </device>
-    <dependencies>
-        <deployment identifier="iOS"/>
-        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="14088"/>
-        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
-    </dependencies>
-    <scenes>
-        <!--View Controller-->
-        <scene sceneID="EHf-IW-A2E">
-            <objects>
-                <viewController id="01J-lp-oVM" sceneMemberID="viewController">
-                    <layoutGuides>
-                        <viewControllerLayoutGuide type="top" id="Llm-lL-Icb"/>
-                        <viewControllerLayoutGuide type="bottom" id="xb3-aO-Qok"/>
-                    </layoutGuides>
-                    <view key="view" contentMode="scaleToFill" id="Ze5-6b-2t3">
-                        <rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
-                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
-                        <subviews>
-                            <label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" text="TensorFlowLite" textAlignment="center" lineBreakMode="tailTruncation" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="3Gq-PV-hia">
-                                <rect key="frame" x="16" y="315" width="343" height="38.5"/>
-                                <fontDescription key="fontDescription" type="boldSystem" pointSize="32"/>
-                                <nil key="textColor"/>
-                                <nil key="highlightedColor"/>
-                            </label>
-                        </subviews>
-                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
-                        <constraints>
-                            <constraint firstItem="3Gq-PV-hia" firstAttribute="leading" secondItem="Ze5-6b-2t3" secondAttribute="leading" constant="16" id="aXL-9T-5Pf"/>
-                            <constraint firstItem="3Gq-PV-hia" firstAttribute="centerY" secondItem="Ze5-6b-2t3" secondAttribute="centerY" id="cDf-Go-1FR"/>
-                            <constraint firstAttribute="trailing" secondItem="3Gq-PV-hia" secondAttribute="trailing" constant="16" id="fB9-BX-A3B"/>
-                        </constraints>
-                    </view>
-                </viewController>
-                <placeholder placeholderIdentifier="IBFirstResponder" id="iYj-Kq-Ea1" userLabel="First Responder" sceneMemberID="firstResponder"/>
-            </objects>
-            <point key="canvasLocation" x="52" y="374.66266866566718"/>
-        </scene>
-    </scenes>
-</document>
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
deleted file mode 100644
index 10cae6e855..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
+++ /dev/null
@@ -1,95 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="14460.31" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" colorMatched="YES" initialViewController="BYZ-38-t0r">
-    <device id="retina4_7" orientation="portrait">
-        <adaptation id="fullscreen"/>
-    </device>
-    <dependencies>
-        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="14460.20"/>
-        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
-    </dependencies>
-    <scenes>
-        <!--View Controller-->
-        <scene sceneID="tne-QT-ifu">
-            <objects>
-                <viewController storyboardIdentifier="viewController" useStoryboardIdentifierAsRestorationIdentifier="YES" id="BYZ-38-t0r" customClass="ViewController" customModule="TensorFlowLiteAppLib" sceneMemberID="viewController">
-                    <layoutGuides>
-                        <viewControllerLayoutGuide type="top" id="y3c-jy-aDJ"/>
-                        <viewControllerLayoutGuide type="bottom" id="wfy-db-euE"/>
-                    </layoutGuides>
-                    <view key="view" contentMode="scaleToFill" id="8bC-Xf-vdC">
-                        <rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
-                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
-                        <subviews>
-                            <textView clipsSubviews="YES" multipleTouchEnabled="YES" contentMode="scaleToFill" editable="NO" selectable="NO" translatesAutoresizingMaskIntoConstraints="NO" id="7Mj-sL-hrd">
-                                <rect key="frame" x="0.0" y="367" width="375" height="300"/>
-                                <color key="backgroundColor" red="0.0" green="0.47843137250000001" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
-                                <constraints>
-                                    <constraint firstAttribute="height" constant="300" id="YUb-MC-D5w"/>
-                                </constraints>
-                                <color key="textColor" cocoaTouchSystemColor="tableCellGroupedBackgroundColor"/>
-                                <fontDescription key="fontDescription" type="system" pointSize="14"/>
-                                <textInputTraits key="textInputTraits" autocapitalizationType="sentences"/>
-                            </textView>
-                            <toolbar opaque="NO" clearsContextBeforeDrawing="NO" contentMode="scaleToFill" translatesAutoresizingMaskIntoConstraints="NO" id="Qwg-EP-bd6" userLabel="Bottom Toolbar">
-                                <rect key="frame" x="0.0" y="323" width="375" height="44"/>
-                                <constraints>
-                                    <constraint firstAttribute="height" constant="44" id="jhT-Q0-E9N"/>
-                                </constraints>
-                                <items>
-                                    <barButtonItem style="plain" systemItem="flexibleSpace" id="P3q-uA-YUa"/>
-                                    <barButtonItem title="Invoke Interpreter" id="A4J-Mg-nmd" userLabel="Invoke Button">
-                                        <connections>
-                                            <action selector="invokeInterpreter:" destination="BYZ-38-t0r" id="lZU-x7-PsJ"/>
-                                        </connections>
-                                    </barButtonItem>
-                                    <barButtonItem style="plain" systemItem="flexibleSpace" id="Qad-Pa-ySg"/>
-                                </items>
-                            </toolbar>
-                            <toolbar opaque="NO" clearsContextBeforeDrawing="NO" contentMode="scaleToFill" translatesAutoresizingMaskIntoConstraints="NO" id="Gkb-TR-PCB" userLabel="Top Toolbar">
-                                <rect key="frame" x="0.0" y="28" width="375" height="44"/>
-                                <constraints>
-                                    <constraint firstAttribute="height" constant="44" id="hSD-2q-fUE"/>
-                                </constraints>
-                                <items>
-                                    <barButtonItem style="plain" id="LKw-TX-bbH">
-                                        <segmentedControl key="customView" opaque="NO" contentMode="scaleToFill" contentHorizontalAlignment="left" contentVerticalAlignment="top" segmentControlStyle="bar" selectedSegmentIndex="0" id="rhA-nW-xzT">
-                                            <rect key="frame" x="16" y="7" width="343" height="30"/>
-                                            <autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
-                                            <segments>
-                                                <segment title="Add"/>
-                                                <segment title="AddQuantized"/>
-                                                <segment title="MultiAdd"/>
-                                            </segments>
-                                            <connections>
-                                                <action selector="modelChanged:" destination="BYZ-38-t0r" eventType="valueChanged" id="YnG-Ov-B5D"/>
-                                            </connections>
-                                        </segmentedControl>
-                                    </barButtonItem>
-                                </items>
-                            </toolbar>
-                        </subviews>
-                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
-                        <constraints>
-                            <constraint firstAttribute="trailing" secondItem="Gkb-TR-PCB" secondAttribute="trailing" id="4Cr-Sf-I7n"/>
-                            <constraint firstItem="7Mj-sL-hrd" firstAttribute="bottom" secondItem="wfy-db-euE" secondAttribute="top" id="6ot-zD-sze"/>
-                            <constraint firstItem="7Mj-sL-hrd" firstAttribute="top" secondItem="Qwg-EP-bd6" secondAttribute="bottom" id="ELA-C6-NiG"/>
-                            <constraint firstAttribute="trailing" secondItem="7Mj-sL-hrd" secondAttribute="trailing" id="HDO-xr-mBl"/>
-                            <constraint firstItem="Gkb-TR-PCB" firstAttribute="leading" secondItem="8bC-Xf-vdC" secondAttribute="leading" id="Kmo-6K-gS4"/>
-                            <constraint firstItem="Qwg-EP-bd6" firstAttribute="leading" secondItem="8bC-Xf-vdC" secondAttribute="leading" id="hGu-lm-fMG"/>
-                            <constraint firstAttribute="trailing" secondItem="Qwg-EP-bd6" secondAttribute="trailing" id="iXR-LK-nTO"/>
-                            <constraint firstItem="7Mj-sL-hrd" firstAttribute="leading" secondItem="8bC-Xf-vdC" secondAttribute="leading" id="nr7-jW-ZYf"/>
-                            <constraint firstItem="Gkb-TR-PCB" firstAttribute="top" secondItem="y3c-jy-aDJ" secondAttribute="bottom" constant="8" id="uCF-VW-rR0"/>
-                        </constraints>
-                    </view>
-                    <connections>
-                        <outlet property="invokeButton" destination="A4J-Mg-nmd" id="UxZ-Ft-E45"/>
-                        <outlet property="modelControl" destination="rhA-nW-xzT" id="KKf-TT-BQ2"/>
-                        <outlet property="resultsTextView" destination="7Mj-sL-hrd" id="T4I-z4-tYA"/>
-                    </connections>
-                </viewController>
-                <placeholder placeholderIdentifier="IBFirstResponder" id="dkx-z0-nzr" sceneMemberID="firstResponder"/>
-            </objects>
-            <point key="canvasLocation" x="125.59999999999999" y="133.5832083958021"/>
-        </scene>
-    </scenes>
-</document>
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift
deleted file mode 100644
index bc8a70c848..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift
+++ /dev/null
@@ -1,13 +0,0 @@
-import Foundation
-
-extension Data {
-  /// Creates a new buffer by copying the buffer pointer of the given array.
-  ///
-  /// - Warning: The given array's element type `T` must be trivial in that it can be copied bit
-  ///     for bit with no indirection or reference-counting operations; otherwise, reinterpreting
-  ///     data from the resulting buffer has undefined behavior.
-  /// - Parameter array: An array with elements of type `T`.
-  init<T>(copyingBufferOf array: [T]) {
-    self = array.withUnsafeBufferPointer(Data.init)
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist
deleted file mode 100644
index 3ca3875f04..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist
+++ /dev/null
@@ -1,46 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
-<plist version="1.0">
-<dict>
-	<key>CFBundleDevelopmentRegion</key>
-	<string>en</string>
-	<key>CFBundleExecutable</key>
-	<string>$(EXECUTABLE_NAME)</string>
-	<key>CFBundleIdentifier</key>
-	<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
-	<key>CFBundleInfoDictionaryVersion</key>
-	<string>6.0</string>
-	<key>CFBundleName</key>
-	<string>$(PRODUCT_NAME)</string>
-	<key>CFBundlePackageType</key>
-	<string>APPL</string>
-	<key>CFBundleShortVersionString</key>
-	<string>1.0</string>
-	<key>CFBundleVersion</key>
-	<string>0.0.1</string>
-	<key>LSRequiresIPhoneOS</key>
-	<true/>
-	<key>NSCameraUsageDescription</key>
-	<string>NSCameraUsageDescription</string>
-	<key>NSPhotoLibraryUsageDescription</key>
-	<string>Select a photo to detect objects in.</string>
-	<key>UILaunchStoryboardName</key>
-	<string>LaunchScreen</string>
-	<key>UIMainStoryboardFile</key>
-	<string>Main</string>
-	<key>UIRequiredDeviceCapabilities</key>
-	<array>
-		<string>armv7</string>
-	</array>
-	<key>UISupportedInterfaceOrientations</key>
-	<array>
-		<string>UIInterfaceOrientationPortrait</string>
-		<string>UIInterfaceOrientationPortraitUpsideDown</string>
-	</array>
-	<key>UISupportedInterfaceOrientations~ipad</key>
-	<array>
-		<string>UIInterfaceOrientationPortrait</string>
-		<string>UIInterfaceOrientationPortraitUpsideDown</string>
-	</array>
-</dict>
-</plist>
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift
deleted file mode 100644
index 73c74fd19c..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift
+++ /dev/null
@@ -1,299 +0,0 @@
-import TensorFlowLite
-import UIKit
-
-class ViewController: UIViewController {
-
-  // MARK: - Properties
-
-  /// TensorFlowLite interpreter object for performing inference from a given model.
-  private var interpreter: Interpreter?
-
-  /// Serial dispatch queue for managing `Interpreter` calls.
-  private let interpreterQueue = DispatchQueue(
-    label: Constant.dispatchQueueLabel,
-    qos: .userInitiated
-  )
-
-  /// The currently selected model.
-  private var currentModel: Model {
-    guard let currentModel = Model(rawValue: modelControl.selectedSegmentIndex) else {
-      preconditionFailure("Invalid model for selected segment index.")
-    }
-    return currentModel
-  }
-
-  /// A description of the current model.
-  private var modelDescription: String {
-    guard let interpreter = interpreter else { return "" }
-    let inputCount = interpreter.inputTensorCount
-    let outputCount = interpreter.outputTensorCount
-    let inputTensors = (0..<inputCount).map { index in
-      var tensorInfo = "  Input \(index + 1): "
-      do {
-        let tensor = try interpreter.input(at: index)
-        tensorInfo += "\(tensor)"
-      } catch let error {
-        tensorInfo += "\(error.localizedDescription)"
-      }
-      return tensorInfo
-    }.joined(separator: "\n")
-    let outputTensors = (0..<outputCount).map { index in
-      var tensorInfo = "  Output \(index + 1): "
-      do {
-        let tensor = try interpreter.output(at: index)
-        tensorInfo += "\(tensor)"
-      } catch let error {
-        tensorInfo += "\(error.localizedDescription)"
-      }
-      return tensorInfo
-    }.joined(separator: "\n")
-    return "Model Description:\n" +
-             "  Input Tensor Count = \(inputCount)\n\(inputTensors)\n\n" +
-             "  Output Tensor Count = \(outputCount)\n\(outputTensors)"
-  }
-
-  // MARK: - IBOutlets
-
-  /// A segmented control for changing models. See the `Model` enum for available models.
-  @IBOutlet private var modelControl: UISegmentedControl!
-
-  @IBOutlet private var resultsTextView: UITextView!
-  @IBOutlet private var invokeButton: UIBarButtonItem!
-
-  // MARK: - UIViewController
-
-  override func viewDidLoad() {
-    super.viewDidLoad()
-
-    invokeButton.isEnabled = false
-    loadModel()
-  }
-
-  // MARK: - IBActions
-
-  @IBAction func modelChanged(_ sender: Any) {
-    invokeButton.isEnabled = false
-    updateResultsText("Switched to the \(currentModel.description) model.")
-    loadModel()
-  }
-
-  @IBAction func invokeInterpreter(_ sender: Any) {
-    switch currentModel {
-    case .add:
-      invokeAdd()
-    case .addQuantized:
-      invokeAddQuantized()
-    case .multiAdd:
-      invokeMultiAdd()
-    }
-  }
-
-  // MARK: - Private
-
-  private func loadModel() {
-    let fileInfo = currentModel.fileInfo
-    guard let modelPath = Bundle.main.path(forResource: fileInfo.name, ofType: fileInfo.extension)
-    else {
-      updateResultsText("Failed to load the \(currentModel.description) model.")
-      return
-    }
-    setUpInterpreter(withModelPath: modelPath)
-  }
-
-  private func setUpInterpreter(withModelPath modelPath: String) {
-    interpreterQueue.async {
-      do {
-        var options = InterpreterOptions()
-        options.isErrorLoggingEnabled = true
-        self.interpreter = try Interpreter(modelPath: modelPath, options: options)
-      } catch let error {
-        self.updateResultsText(
-          "Failed to create the interpreter with error: \(error.localizedDescription)"
-        )
-        return
-      }
-      safeDispatchOnMain { self.invokeButton.isEnabled = true }
-    }
-  }
-
-  private func invokeAdd() {
-    interpreterQueue.async {
-      guard let interpreter = self.interpreter else {
-        self.updateResultsText(Constant.nilInterpreterErrorMessage)
-        return
-      }
-      do {
-        try interpreter.resizeInput(at: 0, to: [2])
-        try interpreter.allocateTensors()
-        let input: [Float32] = [1, 3]
-        let resultsText = self.modelDescription + "\n\n" +
-          "Performing 2 add operations on input \(input.description) equals: "
-        self.updateResultsText(resultsText)
-        let data = Data(copyingBufferOf: input)
-        try interpreter.copy(data, toInputAt: 0)
-        try interpreter.invoke()
-        let outputTensor = try interpreter.output(at: 0)
-        let results: () -> String = {
-          guard let results = [Float32](unsafeData: outputTensor.data) else { return "No results." }
-          return resultsText + results.description
-        }
-        self.updateResultsText(results())
-      } catch let error {
-        self.updateResultsText(
-          "Failed to invoke the interpreter with error: \(error.localizedDescription)"
-        )
-        return
-      }
-    }
-  }
-
-  private func invokeAddQuantized() {
-    interpreterQueue.async {
-      guard let interpreter = self.interpreter else {
-        self.updateResultsText(Constant.nilInterpreterErrorMessage)
-        return
-      }
-      do {
-        try interpreter.resizeInput(at: 0, to: [2])
-        try interpreter.allocateTensors()
-        let input: [UInt8] = [1, 3]
-        let resultsText = self.modelDescription + "\n\n" +
-          "Performing 2 add operations on quantized input \(input.description) equals: "
-        self.updateResultsText(resultsText)
-        let data = Data(input)
-        try interpreter.copy(data, toInputAt: 0)
-        try interpreter.invoke()
-        let outputTensor = try interpreter.output(at: 0)
-        let results: () -> String = {
-          guard let quantizationParameters = outputTensor.quantizationParameters else {
-            return "No results."
-          }
-          let quantizedResults = [UInt8](outputTensor.data)
-          let dequantizedResults = quantizedResults.map {
-            quantizationParameters.scale * Float(Int($0) - quantizationParameters.zeroPoint)
-          }
-          return resultsText + quantizedResults.description +
-                   ", dequantized results: " + dequantizedResults.description
-        }
-        self.updateResultsText(results())
-      } catch let error {
-        self.updateResultsText(
-          "Failed to invoke the interpreter with error: \(error.localizedDescription)"
-        )
-        return
-      }
-    }
-  }
-
-  private func invokeMultiAdd() {
-    interpreterQueue.async {
-      guard let interpreter = self.interpreter else {
-        self.updateResultsText(Constant.nilInterpreterErrorMessage)
-        return
-      }
-      do {
-        let shape = TensorShape(2)
-        try (0..<interpreter.inputTensorCount).forEach { index in
-          try interpreter.resizeInput(at: index, to: shape)
-        }
-        try interpreter.allocateTensors()
-        let inputs = try (0..<interpreter.inputTensorCount).map { index -> [Float32] in
-          let input = [Float32(index + 1), Float32(index + 2)]
-          let data = Data(copyingBufferOf: input)
-          try interpreter.copy(data, toInputAt: index)
-          return input
-        }
-        let resultsText = self.modelDescription + "\n\n" +
-          "Performing 3 add operations on inputs \(inputs.description) equals: "
-        self.updateResultsText(resultsText)
-        try interpreter.invoke()
-        let results = try (0..<interpreter.outputTensorCount).map { index -> [Float32] in
-          let tensor = try interpreter.output(at: index)
-          return [Float32](unsafeData: tensor.data) ?? []
-        }
-        self.updateResultsText(resultsText + results.description)
-      } catch let error {
-        self.updateResultsText(
-          "Failed to invoke the interpreter with error: \(error.localizedDescription)"
-        )
-        return
-      }
-    }
-  }
-
-  private func updateResultsText(_ text: String? = nil) {
-    safeDispatchOnMain { self.resultsTextView.text = text }
-  }
-}
-
-// MARK: - Constants
-
-private enum Constant {
-  static let dispatchQueueLabel = "TensorFlowLiteInterpreterQueue"
-  static let nilInterpreterErrorMessage =
-    "Failed to invoke the interpreter because the interpreter was nil."
-}
-
-/// Models that can be loaded by the TensorFlow Lite `Interpreter`.
-private enum Model: Int, CustomStringConvertible {
-  /// A float model that performs two add operations on one input tensor and returns the result in
-  /// one output tensor.
-  case add = 0
-  /// A quantized model that performs two add operations on one input tensor and returns the result
-  /// in one output tensor.
-  case addQuantized = 1
-  /// A float model that performs three add operations on four input tensors and returns the results
-  /// in 2 output tensors.
-  case multiAdd = 2
-
-  var fileInfo: (name: String, extension: String) {
-    switch self {
-    case .add:
-      return Add.fileInfo
-    case .addQuantized:
-      return AddQuantized.fileInfo
-    case .multiAdd:
-      return MultiAdd.fileInfo
-    }
-  }
-
-  // MARK: - CustomStringConvertible
-
-  var description: String {
-    switch self {
-    case .add:
-      return Add.name
-    case .addQuantized:
-      return AddQuantized.name
-    case .multiAdd:
-      return MultiAdd.name
-    }
-  }
-}
-
-/// Values for the `Add` model.
-private enum Add {
-  static let name = "Add"
-  static let fileInfo = (name: "add", extension: "bin")
-}
-
-/// Values for the `AddQuantized` model.
-private enum AddQuantized {
-  static let name = "AddQuantized"
-  static let fileInfo = (name: "add_quantized", extension: "bin")
-}
-
-/// Values for the `MultiAdd` model.
-private enum MultiAdd {
-  static let name = "MultiAdd"
-  static let fileInfo = (name: "multi_add", extension: "bin")
-}
-
-// MARK: - Fileprivate
-
-/// Safely dispatches the given block on the main queue. If the current thread is `main`, the block
-/// is executed synchronously; otherwise, the block is executed asynchronously on the main thread.
-fileprivate func safeDispatchOnMain(_ block: @escaping () -> Void) {
-  if Thread.isMainThread { block(); return }
-  DispatchQueue.main.async { block() }
-}
diff --git a/tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift b/tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift
deleted file mode 100644
index 54b4f59b28..0000000000
--- a/tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift
+++ /dev/null
@@ -1,54 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-@testable import TensorFlowLite
-import XCTest
-
-class InterpreterOptionsTests: XCTestCase {
-
-  func testInterpreterOptions_InitWithDefaultValues() {
-    let options = InterpreterOptions()
-    XCTAssertNil(options.threadCount)
-    XCTAssertFalse(options.isErrorLoggingEnabled)
-  }
-
-  func testInterpreterOptions_InitWithCustomValues() {
-    var options = InterpreterOptions()
-    options.threadCount = 2
-    XCTAssertEqual(options.threadCount, 2)
-    options.isErrorLoggingEnabled = true
-    XCTAssertTrue(options.isErrorLoggingEnabled)
-  }
-
-  func testInterpreterOptions_Equatable() {
-    var options1 = InterpreterOptions()
-    var options2 = InterpreterOptions()
-    XCTAssertEqual(options1, options2)
-
-    options1.threadCount = 2
-    options2.threadCount = 2
-    XCTAssertEqual(options1, options2)
-
-    options2.threadCount = 3
-    XCTAssertNotEqual(options1, options2)
-    options2.threadCount = 2
-
-    options1.isErrorLoggingEnabled = true
-    options2.isErrorLoggingEnabled = true
-    XCTAssertEqual(options1, options2)
-
-    options2.isErrorLoggingEnabled = false
-    XCTAssertNotEqual(options1, options2)
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift b/tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift
deleted file mode 100644
index e98da5f951..0000000000
--- a/tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift
+++ /dev/null
@@ -1,315 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-@testable import TensorFlowLite
-import XCTest
-
-class InterpreterTests: XCTestCase {
-
-  var interpreter: Interpreter!
-
-  override func setUp() {
-    super.setUp()
-
-    interpreter = try! Interpreter(modelPath: AddModel.path)
-  }
-
-  override func tearDown() {
-    interpreter = nil
-
-    super.tearDown()
-  }
-
-  func testInterpreter_InitWithModelPath() {
-    XCTAssertNoThrow(try Interpreter(modelPath: AddModel.path))
-  }
-
-  func testInterpreter_Init_ThrowsFailedToLoadModel() {
-    XCTAssertThrowsError(try Interpreter(modelPath: "/invalid/path")) { error in
-      self.assertEqualErrors(actual: error, expected: .failedToLoadModel)
-    }
-  }
-
-  func testInterpreter_InitWithModelPathAndOptions() {
-    var options = InterpreterOptions()
-    options.threadCount = 2
-    XCTAssertNoThrow(try Interpreter(modelPath: AddModel.path, options: options))
-  }
-
-  func testInterpreter_InputTensorCount() {
-    XCTAssertEqual(interpreter.inputTensorCount, AddModel.inputTensorCount)
-  }
-
-  func testInterpreter_OutputTensorCount() {
-    XCTAssertEqual(interpreter.outputTensorCount, AddModel.outputTensorCount)
-  }
-
-  func testInterpreter_Invoke() throws {
-    try interpreter.allocateTensors()
-    XCTAssertNoThrow(try interpreter.invoke())
-  }
-
-  func testInterpreter_Invoke_ThrowsAllocateTensorsRequired_ModelNotReady() {
-    XCTAssertThrowsError(try interpreter.invoke()) { error in
-      self.assertEqualErrors(actual: error, expected: .allocateTensorsRequired)
-    }
-  }
-
-  func testInterpreter_InputTensorAtIndex() throws {
-    try setUpAddModelInputTensor()
-    let inputTensor = try interpreter.input(at: AddModel.validIndex)
-    XCTAssertEqual(inputTensor, AddModel.inputTensor)
-  }
-
-  func testInterpreter_InputTensorAtIndex_QuantizedModel() throws {
-    interpreter = try Interpreter(modelPath: AddQuantizedModel.path)
-    try setUpAddQuantizedModelInputTensor()
-    let inputTensor = try interpreter.input(at: AddQuantizedModel.inputOutputIndex)
-    XCTAssertEqual(inputTensor, AddQuantizedModel.inputTensor)
-  }
-
-  func testInterpreter_InputTensorAtIndex_ThrowsInvalidIndex() throws {
-    try interpreter.allocateTensors()
-    XCTAssertThrowsError(try interpreter.input(at: AddModel.invalidIndex)) { error in
-      let maxIndex = AddModel.inputTensorCount - 1
-      self.assertEqualErrors(
-        actual: error,
-        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
-      )
-    }
-  }
-
-  func testInterpreter_InputTensorAtIndex_ThrowsAllocateTensorsRequired() {
-    XCTAssertThrowsError(try interpreter.input(at: AddModel.validIndex)) { error in
-      self.assertEqualErrors(actual: error, expected: .allocateTensorsRequired)
-    }
-  }
-
-  func testInterpreter_OutputTensorAtIndex() throws {
-    try setUpAddModelInputTensor()
-    try interpreter.invoke()
-    let outputTensor = try interpreter.output(at: AddModel.validIndex)
-    XCTAssertEqual(outputTensor, AddModel.outputTensor)
-    let expectedResults = [Float32](unsafeData: outputTensor.data)
-    XCTAssertEqual(expectedResults, AddModel.results)
-  }
-
-  func testInterpreter_OutputTensorAtIndex_QuantizedModel() throws {
-    interpreter = try Interpreter(modelPath: AddQuantizedModel.path)
-    try setUpAddQuantizedModelInputTensor()
-    try interpreter.invoke()
-    let outputTensor = try interpreter.output(at: AddQuantizedModel.inputOutputIndex)
-    XCTAssertEqual(outputTensor, AddQuantizedModel.outputTensor)
-    let expectedResults = [UInt8](outputTensor.data)
-    XCTAssertEqual(expectedResults, AddQuantizedModel.results)
-  }
-
-  func testInterpreter_OutputTensorAtIndex_ThrowsInvalidIndex() throws {
-    try interpreter.allocateTensors()
-    try interpreter.invoke()
-    XCTAssertThrowsError(try interpreter.output(at: AddModel.invalidIndex)) { error in
-      let maxIndex = AddModel.outputTensorCount - 1
-      self.assertEqualErrors(
-        actual: error,
-        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
-      )
-    }
-  }
-
-  func testInterpreter_OutputTensorAtIndex_ThrowsInvokeInterpreterRequired() {
-    XCTAssertThrowsError(try interpreter.output(at: AddModel.validIndex)) { error in
-      self.assertEqualErrors(actual: error, expected: .invokeInterpreterRequired)
-    }
-  }
-
-  func testInterpreter_ResizeInputTensorAtIndexToShape() {
-    XCTAssertNoThrow(try interpreter.resizeInput(at: AddModel.validIndex, to: [2, 2, 3]))
-    XCTAssertNoThrow(try interpreter.allocateTensors())
-  }
-
-  func testInterpreter_ResizeInputTensorAtIndexToShape_ThrowsInvalidIndex() {
-    XCTAssertThrowsError(try interpreter.resizeInput(
-      at: AddModel.invalidIndex,
-      to: [2, 2, 3]
-    )) { error in
-      let maxIndex = AddModel.inputTensorCount - 1
-      self.assertEqualErrors(
-        actual: error,
-        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
-      )
-    }
-  }
-
-  func testInterpreter_CopyDataToInputTensorAtIndex() throws {
-    try interpreter.resizeInput(at: AddModel.validIndex, to: AddModel.shape)
-    try interpreter.allocateTensors()
-    let inputTensor = try interpreter.copy(AddModel.inputData, toInputAt: AddModel.validIndex)
-    XCTAssertEqual(inputTensor.data, AddModel.inputData)
-  }
-
-  func testInterpreter_CopyDataToInputTensorAtIndex_ThrowsInvalidIndex() {
-    XCTAssertThrowsError(try interpreter.copy(
-      AddModel.inputData,
-      toInputAt: AddModel.invalidIndex
-    )) { error in
-      let maxIndex = AddModel.inputTensorCount - 1
-      self.assertEqualErrors(
-        actual: error,
-        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
-      )
-    }
-  }
-
-  func testInterpreter_CopyDataToInputTensorAtIndex_ThrowsInvalidDataCount() throws {
-    try interpreter.resizeInput(at: AddModel.validIndex, to: AddModel.shape)
-    try interpreter.allocateTensors()
-    let invalidData = Data(count: AddModel.dataCount - 1)
-    XCTAssertThrowsError(try interpreter.copy(
-      invalidData,
-      toInputAt: AddModel.validIndex
-    )) { error in
-      self.assertEqualErrors(
-        actual: error,
-        expected: .invalidTensorDataCount(provided: invalidData.count, required: AddModel.dataCount)
-      )
-    }
-  }
-
-  func testInterpreter_AllocateTensors() {
-    XCTAssertNoThrow(try interpreter.allocateTensors())
-  }
-
-  // MARK: - Private
-
-  private func setUpAddModelInputTensor() throws {
-    precondition(interpreter != nil)
-    try interpreter.resizeInput(at: AddModel.validIndex, to: AddModel.shape)
-    try interpreter.allocateTensors()
-    try interpreter.copy(AddModel.inputData, toInputAt: AddModel.validIndex)
-  }
-
-  private func setUpAddQuantizedModelInputTensor() throws {
-    precondition(interpreter != nil)
-    try interpreter.resizeInput(at: AddQuantizedModel.inputOutputIndex, to: AddQuantizedModel.shape)
-    try interpreter.allocateTensors()
-    try interpreter.copy(AddQuantizedModel.inputData, toInputAt: AddQuantizedModel.inputOutputIndex)
-  }
-
-  private func assertEqualErrors(actual: Error, expected: InterpreterError) {
-    guard let actual = actual as? InterpreterError else {
-      XCTFail("Actual error should be of type InterpreterError.")
-      return
-    }
-    XCTAssertEqual(actual, expected)
-  }
-}
-
-// MARK: - Constants
-
-/// Values for the `add.bin` model.
-private enum AddModel {
-  static let info = (name: "add", extension: "bin")
-  static let inputTensorCount = 1
-  static let outputTensorCount = 1
-  static let invalidIndex = 1
-  static let validIndex = 0
-  static let shape: TensorShape = [2]
-  static let dataCount = inputData.count
-  static let inputData = Data(copyingBufferOf: [Float32(1.0), Float32(3.0)])
-  static let outputData = Data(copyingBufferOf: [Float32(3.0), Float32(9.0)])
-  static let results = [Float32(3.0), Float32(9.0)]
-
-  static let inputTensor = Tensor(
-    name: "input",
-    dataType: .float32,
-    shape: shape,
-    data: inputData
-  )
-  static let outputTensor = Tensor(
-    name: "output",
-    dataType: .float32,
-    shape: shape,
-    data: outputData
-  )
-
-  static var path: String = {
-    let bundle = Bundle(for: InterpreterTests.self)
-    guard let path = bundle.path(forResource: info.name, ofType: info.extension) else { return "" }
-    return path
-  }()
-}
-
-/// Values for the `add_quantized.bin` model.
-private enum AddQuantizedModel {
-  static let info = (name: "add_quantized", extension: "bin")
-  static let inputOutputIndex = 0
-  static let shape: TensorShape = [2]
-  static let inputData = Data([1, 3])
-  static let outputData = Data([3, 9])
-  static let quantizationParameters = QuantizationParameters(scale: 0.003922, zeroPoint: 0)
-  static let results: [UInt8] = [3, 9]
-
-  static let inputTensor = Tensor(
-    name: "input",
-    dataType: .uInt8,
-    shape: shape,
-    data: inputData,
-    quantizationParameters: quantizationParameters
-  )
-  static let outputTensor = Tensor(
-    name: "output",
-    dataType: .uInt8,
-    shape: shape,
-    data: outputData,
-    quantizationParameters: quantizationParameters
-  )
-
-  static var path: String = {
-    let bundle = Bundle(for: InterpreterTests.self)
-    guard let path = bundle.path(forResource: info.name, ofType: info.extension) else { return "" }
-    return path
-  }()
-}
-
-// MARK: - Extensions
-
-extension Array {
-  /// Creates a new array from the bytes of the given unsafe data.
-  ///
-  /// - Note: Returns `nil` if `unsafeData.count` is not a multiple of
-  ///     `MemoryLayout<Element>.stride`.
-  /// - Parameter unsafeData: The data containing the bytes to turn into an array.
-  init?(unsafeData: Data) {
-    guard unsafeData.count % MemoryLayout<Element>.stride == 0 else { return nil }
-    let elements = unsafeData.withUnsafeBytes {
-      UnsafeBufferPointer<Element>(
-        start: $0,
-        count: unsafeData.count / MemoryLayout<Element>.stride
-      )
-    }
-    self.init(elements)
-  }
-}
-
-extension Data {
-  /// Creates a new buffer by copying the buffer pointer of the given array.
-  ///
-  /// - Warning: The given array's element type `T` must be trivial in that it can be copied bit
-  ///     for bit with no indirection or reference-counting operations; otherwise, reinterpreting
-  ///     data from the resulting buffer has undefined behavior.
-  /// - Parameter array: An array with elements of type `T`.
-  init<T>(copyingBufferOf array: [T]) {
-    self = array.withUnsafeBufferPointer(Data.init)
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/Tests/ModelTests.swift b/tensorflow/lite/experimental/swift/Tests/ModelTests.swift
deleted file mode 100644
index 025db18906..0000000000
--- a/tensorflow/lite/experimental/swift/Tests/ModelTests.swift
+++ /dev/null
@@ -1,59 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-@testable import TensorFlowLite
-import XCTest
-
-class ModelTests: XCTestCase {
-
-  var modelPath: String!
-
-  override func setUp() {
-    super.setUp()
-
-    let bundle = Bundle(for: type(of: self))
-    guard let modelPath = bundle.path(
-            forResource: Constant.modelInfo.name,
-            ofType: Constant.modelInfo.extension)
-    else {
-      XCTFail("Failed to get the model file path.")
-      return
-    }
-    self.modelPath = modelPath
-  }
-
-  override func tearDown() {
-    modelPath = nil
-
-    super.tearDown()
-  }
-
-  func testModel_InitWithFilePath() {
-    XCTAssertNotNil(Model(filePath: modelPath))
-  }
-
-  func testModel_InitWithEmptyFilePath_FailsInitialization() {
-    XCTAssertNil(Model(filePath: ""))
-  }
-
-  func testModel_InitWithInvalidFilePath_FailsInitialization() {
-    XCTAssertNil(Model(filePath: "invalid/path"))
-  }
-}
-
-// MARK: - Constants
-
-private enum Constant {
-  static let modelInfo = (name: "add", extension: "bin")
-}
diff --git a/tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift b/tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift
deleted file mode 100644
index 65648c2698..0000000000
--- a/tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift
+++ /dev/null
@@ -1,43 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-@testable import TensorFlowLite
-import XCTest
-
-class QuantizationParametersTests: XCTestCase {
-
-  func testQuantizationParameters_InitWithCustomValues() {
-    let parameters = QuantizationParameters(scale: 0.5, zeroPoint: 1)
-    XCTAssertEqual(parameters.scale, 0.5)
-    XCTAssertEqual(parameters.zeroPoint, 1)
-  }
-
-  func testQuantizationParameters_Equatable() {
-    let parameters1 = QuantizationParameters(scale: 0.5, zeroPoint: 1)
-    let parameters2 = QuantizationParameters(scale: 0.5, zeroPoint: 1)
-    XCTAssertEqual(parameters1, parameters2)
-
-    let parameters3 = QuantizationParameters(scale: 0.4, zeroPoint: 1)
-    XCTAssertNotEqual(parameters1, parameters3)
-    XCTAssertNotEqual(parameters2, parameters3)
-  }
-}
-
-// MARK: - Extensions
-
-extension QuantizationParameters: Equatable {
-  public static func == (lhs: QuantizationParameters, rhs: QuantizationParameters) -> Bool {
-    return lhs.scale == rhs.scale && lhs.zeroPoint == rhs.zeroPoint
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/Tests/TensorTests.swift b/tensorflow/lite/experimental/swift/Tests/TensorTests.swift
deleted file mode 100644
index 4540043a16..0000000000
--- a/tensorflow/lite/experimental/swift/Tests/TensorTests.swift
+++ /dev/null
@@ -1,83 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-@testable import TensorFlowLite
-import XCTest
-
-class TensorTests: XCTestCase {
-
-  // MARK: - Tensor
-
-  func testTensor_Init() {
-    let name = "InputTensor"
-    let dataType: TensorDataType = .uInt8
-    let shape = TensorShape(Constant.dimensions)
-    guard let data = name.data(using: .utf8) else { XCTFail("Data should not be nil."); return }
-    let quantizationParameters = QuantizationParameters(scale: 0.5, zeroPoint: 1)
-    let inputTensor = Tensor(
-      name: name,
-      dataType: dataType,
-      shape: shape,
-      data: data,
-      quantizationParameters: quantizationParameters
-    )
-    XCTAssertEqual(inputTensor.name, name)
-    XCTAssertEqual(inputTensor.dataType, dataType)
-    XCTAssertEqual(inputTensor.shape, shape)
-    XCTAssertEqual(inputTensor.data, data)
-    XCTAssertEqual(inputTensor.quantizationParameters, quantizationParameters)
-  }
-
-  // MARK: - TensorShape
-
-  func testTensorShape_InitWithArray() {
-    let shape = TensorShape(Constant.dimensions)
-    XCTAssertEqual(shape.rank, Constant.dimensions.count)
-    XCTAssertEqual(shape.dimensions, Constant.dimensions)
-  }
-
-  func testTensorShape_InitWithElements() {
-    let shape = TensorShape(2, 2, 3)
-    XCTAssertEqual(shape.rank, Constant.dimensions.count)
-    XCTAssertEqual(shape.dimensions, Constant.dimensions)
-  }
-
-  func testTensorShape_InitWithArrayLiteral() {
-    let shape: TensorShape = [2, 2, 3]
-    XCTAssertEqual(shape.rank, Constant.dimensions.count)
-    XCTAssertEqual(shape.dimensions, Constant.dimensions)
-  }
-}
-
-// MARK: - Constants
-
-private enum Constant {
-  /// Array of 2 arrays of 2 arrays of 3 numbers: [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]].
-  static let dimensions = [2, 2, 3]
-}
-
-// MARK: - Extensions
-
-extension TensorShape: Equatable {
-  public static func == (lhs: TensorShape, rhs: TensorShape) -> Bool {
-    return lhs.rank == rhs.rank && lhs.dimensions == rhs.dimensions
-  }
-}
-
-extension Tensor: Equatable {
-  public static func == (lhs: Tensor, rhs: Tensor) -> Bool {
-    return lhs.name == rhs.name && lhs.dataType == rhs.dataType && lhs.shape == rhs.shape &&
-           lhs.data == rhs.data && lhs.quantizationParameters == rhs.quantizationParameters
-  }
-}
-- 
GitLab


From fd54d400ba60e57daf431afe209133226e03acfc Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Tue, 8 Jan 2019 14:10:55 -0800
Subject: [PATCH 0373/2345] Unify RMSProp w/o momentum.

PiperOrigin-RevId: 228396933
---
 tensorflow/python/keras/optimizer_v2/BUILD    |   1 +
 .../keras/optimizer_v2/optimizer_v2_test.py   |   2 +-
 .../python/keras/optimizer_v2/rmsprop.py      | 155 ++++++++++++------
 .../python/keras/optimizer_v2/rmsprop_test.py | 106 ++++++++----
 4 files changed, 181 insertions(+), 83 deletions(-)

diff --git a/tensorflow/python/keras/optimizer_v2/BUILD b/tensorflow/python/keras/optimizer_v2/BUILD
index 964cf5fcba..2b990c9334 100644
--- a/tensorflow/python/keras/optimizer_v2/BUILD
+++ b/tensorflow/python/keras/optimizer_v2/BUILD
@@ -217,4 +217,5 @@ cuda_py_test(
         "//tensorflow/python:state_ops",
         "//tensorflow/python:variables",
     ],
+    shard_count = 2,
 )
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
index 58c8b1f343..b41e4aa064 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
@@ -507,7 +507,7 @@ class OptimizersCompatibilityTest(test.TestCase, parameterized.TestCase):
       ('adadelta', 'adadelta', True), ('adagrad', 'adagrad', True),
       ('adam', 'adam', True), ('adamax', 'adamax', True),
       ('nadam', 'nadam', True), ('momentum', 'momentum', True),
-      ('sgd', 'sgd', False))
+      ('rmsprop', 'rmsprop', True), ('sgd', 'sgd', False))
   def testOptimizersCompatibility(self, opt_str, test_weights):
     np.random.seed(1331)
     with self.cached_session():
diff --git a/tensorflow/python/keras/optimizer_v2/rmsprop.py b/tensorflow/python/keras/optimizer_v2/rmsprop.py
index 3f153c4a41..83b81cbe5d 100644
--- a/tensorflow/python/keras/optimizer_v2/rmsprop.py
+++ b/tensorflow/python/keras/optimizer_v2/rmsprop.py
@@ -17,8 +17,14 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import numpy as np
+
 from tensorflow.python.framework import ops
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import state_ops
 from tensorflow.python.training import training_ops
 from tensorflow.python.util.tf_export import keras_export
 
@@ -114,77 +120,122 @@ class RMSprop(optimizer_v2.OptimizerV2):
   def _create_slots(self, var_list):
     for var in var_list:
       self.add_slot(var, "rms")
-      self.add_slot(var, "momentum")
-      if self.centered:
+    if self._momentum:
+      for var in var_list:
+        self.add_slot(var, "momentum")
+    if self.centered:
+      for var in var_list:
         self.add_slot(var, "mg")
 
   def _resource_apply_dense(self, grad, var):
     var_dtype = var.dtype.base_dtype
     lr_t = self._decayed_lr(var_dtype)
     rms = self.get_slot(var, "rms")
-    mom = self.get_slot(var, "momentum")
     rho = self._get_hyper("rho", var_dtype)
     momentum = self._get_hyper("momentum", var_dtype)
     epsilon = self._get_hyper("epsilon", var_dtype)
-    if self.centered:
-      mg = self.get_slot(var, "mg")
-      return training_ops.resource_apply_centered_rms_prop(
-          var.handle,
-          mg.handle,
-          rms.handle,
-          mom.handle,
-          lr_t,
-          rho,
-          momentum,
-          epsilon,
-          grad,
-          use_locking=self._use_locking)
+    if self._momentum:
+      mom = self.get_slot(var, "momentum")
+      if self.centered:
+        mg = self.get_slot(var, "mg")
+        return training_ops.resource_apply_centered_rms_prop(
+            var.handle,
+            mg.handle,
+            rms.handle,
+            mom.handle,
+            lr_t,
+            rho,
+            momentum,
+            epsilon,
+            grad,
+            use_locking=self._use_locking)
+      else:
+        return training_ops.resource_apply_rms_prop(
+            var.handle,
+            rms.handle,
+            mom.handle,
+            lr_t,
+            rho,
+            momentum,
+            epsilon,
+            grad,
+            use_locking=self._use_locking)
     else:
-      return training_ops.resource_apply_rms_prop(
-          var.handle,
-          rms.handle,
-          mom.handle,
-          lr_t,
-          rho,
-          momentum,
-          epsilon,
-          grad,
-          use_locking=self._use_locking)
+      rms_t = rho * rms + (1. - rho) * math_ops.square(grad)
+      rms_t = state_ops.assign(rms, rms_t, use_locking=self._use_locking)
+      denom_t = rms_t
+      if self.centered:
+        mg = self.get_slot(var, "mg")
+        mg_t = rho * mg + (1. - rho) * grad
+        mg_t = state_ops.assign(mg, mg_t, use_locking=self._use_locking)
+        denom_t = rms_t - math_ops.square(mg_t)
+      var_t = var - lr_t * grad / (math_ops.sqrt(denom_t) + epsilon)
+      return state_ops.assign(var, var_t, use_locking=self._use_locking).op
 
   def _resource_apply_sparse(self, grad, var, indices):
     var_dtype = var.dtype.base_dtype
     lr_t = self._decayed_lr(var_dtype)
     rms = self.get_slot(var, "rms")
-    mom = self.get_slot(var, "momentum")
     rho = self._get_hyper("rho", var_dtype)
     momentum = self._get_hyper("momentum", var_dtype)
     epsilon = self._get_hyper("epsilon", var_dtype)
-    if self.centered:
-      mg = self.get_slot(var, "mg")
-      return training_ops.resource_sparse_apply_centered_rms_prop(
-          var.handle,
-          mg.handle,
-          rms.handle,
-          mom.handle,
-          lr_t,
-          rho,
-          momentum,
-          epsilon,
-          grad,
-          indices,
-          use_locking=self._use_locking)
+    if self._momentum:
+      mom = self.get_slot(var, "momentum")
+      if self.centered:
+        mg = self.get_slot(var, "mg")
+        return training_ops.resource_sparse_apply_centered_rms_prop(
+            var.handle,
+            mg.handle,
+            rms.handle,
+            mom.handle,
+            lr_t,
+            rho,
+            momentum,
+            epsilon,
+            grad,
+            indices,
+            use_locking=self._use_locking)
+      else:
+        return training_ops.resource_sparse_apply_rms_prop(
+            var.handle,
+            rms.handle,
+            mom.handle,
+            lr_t,
+            rho,
+            momentum,
+            epsilon,
+            grad,
+            indices,
+            use_locking=self._use_locking)
     else:
-      return training_ops.resource_sparse_apply_rms_prop(
-          var.handle,
-          rms.handle,
-          mom.handle,
-          lr_t,
-          rho,
-          momentum,
-          epsilon,
-          grad,
-          indices,
-          use_locking=self._use_locking)
+      rms_scaled_g_values = (grad * grad) * (1. - rho)
+      rms_t = state_ops.assign(rms, rms * rho, use_locking=self._use_locking)
+      with ops.control_dependencies([rms_t]):
+        rms_t = self._resource_scatter_add(rms, indices, rms_scaled_g_values)
+        rms_slice = array_ops.gather(rms_t, indices)
+      denom_slice = rms_slice
+      if self.centered:
+        mg = self.get_slot(var, "mg")
+        mg_scaled_g_values = grad * (1. - rho)
+        mg_t = state_ops.assign(mg, mg * rho, use_locking=self._use_locking)
+        with ops.control_dependencies([mg_t]):
+          mg_t = self._resource_scatter_add(mg, indices, mg_scaled_g_values)
+          mg_slice = array_ops.gather(mg_t, indices)
+          denom_slice = rms_slice - math_ops.square(mg_slice)
+      var_update = self._resource_scatter_add(
+          var, indices, -lr_t * grad / (math_ops.sqrt(denom_slice) + epsilon))
+      if self.centered:
+        return control_flow_ops.group(*[var_update, rms_t, mg_t])
+      return control_flow_ops.group(*[var_update, rms_t])
+
+  def set_weights(self, weights):
+    params = self.weights
+    # Override set_weights for backward compatibility of Keras V1 optimizer
+    # since it does not include iteration at head of the weight list. Set
+    # iteration to 0.
+    if len(params) == len(weights) + 1:
+      weights = [np.array(0)] + weights
+    super(RMSprop, self).set_weights(weights)
 
   def get_config(self):
     config = super(RMSprop, self).get_config()
diff --git a/tensorflow/python/keras/optimizer_v2/rmsprop_test.py b/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
index 4d61cfbbc5..67662aae20 100644
--- a/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
+++ b/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
@@ -58,14 +58,18 @@ class RMSpropOptimizerTest(test.TestCase):
   def _rmsprop_update_numpy(self, var, g, mg, rms, mom, lr, rho, momentum,
                             epsilon, centered):
     rms_t = rms * rho + (1 - rho) * g * g
-    denom_t = rms_t + epsilon
     if centered:
       mg_t = mg * rho + (1 - rho) * g
-      denom_t -= mg_t * mg_t
+      denom_t = rms_t - mg_t * mg_t
     else:
       mg_t = mg
-    mom_t = momentum * mom + lr * g / np.sqrt(denom_t, dtype=denom_t.dtype)
-    var_t = var - mom_t
+      denom_t = rms_t
+    if momentum > 0.:
+      mom_t = momentum * mom + lr * g / (np.sqrt(denom_t + epsilon))
+      var_t = var - mom_t
+    else:
+      mom_t = mom
+      var_t = var - lr * g / (np.sqrt(denom_t) + epsilon)
     return var_t, mg_t, rms_t, mom_t
 
   def _sparse_rmsprop_update_numpy(self, var, gindexs, gvalues, mg, rms, mom,
@@ -78,12 +82,18 @@ class RMSpropOptimizerTest(test.TestCase):
       gindex = gindexs[i]
       gvalue = gvalues[i]
       rms_t[gindex] = rms[gindex] * rho + (1 - rho) * gvalue * gvalue
-      denom_t = rms_t[gindex] + epsilon
       if centered:
         mg_t[gindex] = mg_t[gindex] * rho + (1 - rho) * gvalue
-        denom_t -= mg_t[gindex] * mg_t[gindex]
-      mom_t[gindex] = momentum * mom[gindex] + lr * gvalue / np.sqrt(denom_t)
-      var_t[gindex] = var[gindex] - mom_t[gindex]
+        denom_t = rms_t[gindex] - mg_t[gindex] * mg_t[gindex]
+      else:
+        denom_t = rms_t[gindex]
+      if momentum > 0.:
+        mom_t[gindex] = momentum * mom[gindex] + lr * gvalue / np.sqrt(denom_t +
+                                                                       epsilon)
+        var_t[gindex] = var[gindex] - mom_t[gindex]
+      else:
+        mom_t[gindex] = mom[gindex]
+        var_t[gindex] = var[gindex] - lr * gvalue / (np.sqrt(denom_t) + epsilon)
     return var_t, mg_t, rms_t, mom_t
 
   @test_util.run_deprecated_v1
@@ -117,14 +127,17 @@ class RMSpropOptimizerTest(test.TestCase):
           mg0 = None
           mg1 = None
 
+        if momentum > 0.:
+          mom0 = opt.get_slot(var0, "momentum")
+          mom1 = opt.get_slot(var1, "momentum")
+        else:
+          mom0 = None
+          mom1 = None
+
         rms0 = opt.get_slot(var0, "rms")
         self.assertTrue(rms0 is not None)
         rms1 = opt.get_slot(var1, "rms")
         self.assertTrue(rms1 is not None)
-        mom0 = opt.get_slot(var0, "momentum")
-        self.assertTrue(mom0 is not None)
-        mom1 = opt.get_slot(var1, "momentum")
-        self.assertTrue(mom1 is not None)
 
         mg0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
         mg1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
@@ -137,8 +150,8 @@ class RMSpropOptimizerTest(test.TestCase):
         self.assertAllClose([1.0, 2.0], self.evaluate(var0))
         self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
-        # Run 4 steps of RMSprop
-        for _ in range(1, 5):
+        # Run 3 steps of RMSprop
+        for _ in range(1, 4):
           self.evaluate(update)
 
           var0_np, mg0_np, rms0_np, mom0_np = self._rmsprop_update_numpy(
@@ -152,10 +165,11 @@ class RMSpropOptimizerTest(test.TestCase):
           if centered:
             self.assertAllCloseAccordingToType(mg0_np, self.evaluate(mg0))
             self.assertAllCloseAccordingToType(mg1_np, self.evaluate(mg1))
+          if momentum > 0.:
+            self.assertAllCloseAccordingToType(mom0_np, self.evaluate(mom0))
+            self.assertAllCloseAccordingToType(mom1_np, self.evaluate(mom1))
           self.assertAllCloseAccordingToType(rms0_np, self.evaluate(rms0))
           self.assertAllCloseAccordingToType(rms1_np, self.evaluate(rms1))
-          self.assertAllCloseAccordingToType(mom0_np, self.evaluate(mom0))
-          self.assertAllCloseAccordingToType(mom1_np, self.evaluate(mom1))
           self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
           self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
@@ -191,10 +205,12 @@ class RMSpropOptimizerTest(test.TestCase):
     self.assertTrue(rms0 is not None)
     rms1 = opt.get_slot(var1, "rms")
     self.assertTrue(rms1 is not None)
-    mom0 = opt.get_slot(var0, "momentum")
-    self.assertTrue(mom0 is not None)
-    mom1 = opt.get_slot(var1, "momentum")
-    self.assertTrue(mom1 is not None)
+    if momentum > 0.:
+      mom0 = opt.get_slot(var0, "momentum")
+      mom1 = opt.get_slot(var1, "momentum")
+    else:
+      mom0 = None
+      mom1 = None
 
     mg0_np = np.array([0.0, 0.0])
     mg1_np = np.array([0.0, 0.0])
@@ -222,8 +238,9 @@ class RMSpropOptimizerTest(test.TestCase):
       # Validate updated params
       self.assertAllCloseAccordingToType(rms0_np, self.evaluate(rms0))
       self.assertAllCloseAccordingToType(rms1_np, self.evaluate(rms1))
-      self.assertAllCloseAccordingToType(mom0_np, self.evaluate(mom0))
-      self.assertAllCloseAccordingToType(mom1_np, self.evaluate(mom1))
+      if momentum > 0.:
+        self.assertAllCloseAccordingToType(mom0_np, self.evaluate(mom0))
+        self.assertAllCloseAccordingToType(mom1_np, self.evaluate(mom1))
       self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
       self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
@@ -325,10 +342,12 @@ class RMSpropOptimizerTest(test.TestCase):
         self.assertTrue(rms0 is not None)
         rms1 = opt.get_slot(var1, "rms")
         self.assertTrue(rms1 is not None)
-        mom0 = opt.get_slot(var0, "momentum")
-        self.assertTrue(mom0 is not None)
-        mom1 = opt.get_slot(var1, "momentum")
-        self.assertTrue(mom1 is not None)
+        if momentum > 0.:
+          mom0 = opt.get_slot(var0, "momentum")
+          mom1 = opt.get_slot(var1, "momentum")
+        else:
+          mom0 = None
+          mom1 = None
 
         mg0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
         mg1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
@@ -341,8 +360,8 @@ class RMSpropOptimizerTest(test.TestCase):
         self.assertAllClose([1.0, 2.0], self.evaluate(var0))
         self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
-        # Run 4 steps of RMSprop
-        for _ in range(1, 5):
+        # Run 3 steps of RMSprop
+        for _ in range(1, 4):
           self.evaluate(update)
 
           var0_np, mg0_np, rms0_np, mom0_np = self._sparse_rmsprop_update_numpy(
@@ -358,8 +377,9 @@ class RMSpropOptimizerTest(test.TestCase):
             self.assertAllCloseAccordingToType(mg1_np, self.evaluate(mg1))
           self.assertAllCloseAccordingToType(rms0_np, self.evaluate(rms0))
           self.assertAllCloseAccordingToType(rms1_np, self.evaluate(rms1))
-          self.assertAllCloseAccordingToType(mom0_np, self.evaluate(mom0))
-          self.assertAllCloseAccordingToType(mom1_np, self.evaluate(mom1))
+          if momentum > 0.:
+            self.assertAllCloseAccordingToType(mom0_np, self.evaluate(mom0))
+            self.assertAllCloseAccordingToType(mom1_np, self.evaluate(mom1))
           self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
           self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
@@ -420,6 +440,32 @@ class RMSpropOptimizerTest(test.TestCase):
     opt_3 = rmsprop.RMSprop(learning_rate=0.1)
     self.assertEqual(opt_3.lr, 0.1)
 
+  def testSlotsUniqueEager(self):
+    with context.eager_mode():
+      v1 = variables.Variable(1.)
+      v2 = variables.Variable(1.)
+
+      opt = rmsprop.RMSprop(1., momentum=0., centered=False)
+      opt.minimize(lambda: v1 + v2, var_list=[v1, v2])
+      # There should be iteration, and one unique slot variable for v1 and v2.
+      self.assertEqual(3, len(set(opt.variables())))
+      self.assertEqual(
+          self.evaluate(opt.variables()[0]), self.evaluate(opt.iterations))
+
+      opt = rmsprop.RMSprop(learning_rate=1., momentum=0.2, centered=False)
+      opt.minimize(lambda: v1 + v2, var_list=[v1, v2])
+      # There should be iteration, and two unique slot variables for v1 and v2.
+      self.assertEqual(5, len(set(opt.variables())))
+      self.assertEqual(
+          self.evaluate(opt.variables()[0]), self.evaluate(opt.iterations))
+
+      opt = rmsprop.RMSprop(learning_rate=1., momentum=0.2, centered=True)
+      opt.minimize(lambda: v1 + v2, var_list=[v1, v2])
+      # There should be iteration, and three unique slot variables for v1 and v2
+      self.assertEqual(7, len(set(opt.variables())))
+      self.assertEqual(
+          self.evaluate(opt.variables()[0]), self.evaluate(opt.iterations))
+
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 0ce305b6ce8560945be3a92eae1bcddd60af5e10 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Tue, 8 Jan 2019 14:53:40 -0800
Subject: [PATCH 0374/2345] [XLA:Python] Add Gather and Scatter to XLA Python
 bindings.

Refactor handling of repeated fields in *DimensionNumbers SWIG rules.

PiperOrigin-RevId: 228404715
---
 .../xla/python/local_computation_builder.cc   |  16 +
 .../xla/python/local_computation_builder.h    |   9 +
 .../xla/python/local_computation_builder.i    | 281 +++++++-----------
 tensorflow/compiler/xla/python/xla_client.py  |  12 +
 .../compiler/xla/python/xla_client_test.py    |  40 +++
 5 files changed, 183 insertions(+), 175 deletions(-)

diff --git a/tensorflow/compiler/xla/python/local_computation_builder.cc b/tensorflow/compiler/xla/python/local_computation_builder.cc
index b0f0e9e570..c153603105 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.cc
+++ b/tensorflow/compiler/xla/python/local_computation_builder.cc
@@ -927,6 +927,22 @@ LocalOp LocalComputationBuilder::TriangularSolve(const LocalOp& a,
                               conjugate_a);
 }
 
+LocalOp LocalComputationBuilder::Gather(
+    const LocalOp& input, const LocalOp& start_indices,
+    const GatherDimensionNumbers& dimension_numbers,
+    absl::Span<const int64> slice_sizes) {
+  return xla::Gather(input.op(), start_indices.op(), dimension_numbers,
+                     slice_sizes);
+}
+
+LocalOp LocalComputationBuilder::Scatter(
+    const LocalOp& input, const LocalOp& scatter_indices,
+    const LocalOp& updates, const LocalComputation& update_computation,
+    const ScatterDimensionNumbers& dimension_numbers) {
+  return xla::Scatter(input.op(), scatter_indices.op(), updates.op(),
+                      update_computation.computation(), dimension_numbers);
+}
+
 StatusOr<LocalComputation*> LocalComputationBuilder::BuildConstantSubGraph(
     const LocalOp& operand) {
   TF_ASSIGN_OR_RETURN(XlaComputation computation,
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.h b/tensorflow/compiler/xla/python/local_computation_builder.h
index 5e83415921..98759cf984 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.h
+++ b/tensorflow/compiler/xla/python/local_computation_builder.h
@@ -418,6 +418,15 @@ class LocalComputationBuilder {
   LocalOp TriangularSolve(const LocalOp& a, const LocalOp& b, bool left_side,
                           bool lower, bool transpose_a, bool conjugate_a);
 
+  LocalOp Gather(const LocalOp& input, const LocalOp& start_indices,
+                 const GatherDimensionNumbers& dimension_numbers,
+                 absl::Span<const int64> slice_sizes);
+
+  LocalOp Scatter(const LocalOp& input, const LocalOp& scatter_indices,
+                  const LocalOp& updates,
+                  const LocalComputation& update_computation,
+                  const ScatterDimensionNumbers& dimension_numbers);
+
   StatusOr<LocalComputation*> BuildConstantSubGraph(const LocalOp& operand);
 
 #define _FORWARD(method_name, return_sig, args_sig) \
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.i b/tensorflow/compiler/xla/python/local_computation_builder.i
index bf5d667c6a..cc96d84724 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.i
+++ b/tensorflow/compiler/xla/python/local_computation_builder.i
@@ -34,6 +34,8 @@ limitations under the License.
 //  PaddingConfig proto                <-  corresponding Python proto
 //  ConvolutionDimensionNumbers proto  <-  corresponding Python proto
 //  DotDimensionNumbers proto          <-  corresponding Python proto
+//  GatherDimensionNumbers proto       <-  corresponding Python proto
+//  ScatterDimensionNumbers proto      <-  corresponding Python proto
 //
 // Arrows indicate whether a conversion only ever occurs in one
 // direction, or whether it is maintained bidirectionally.
@@ -167,8 +169,41 @@ bool HandleStringAttribute(PyObject* o,
   return true;  // Handled string attribute, ok!
 }
 
+bool HandleRepeatedInt64Attribute(
+    PyObject* o, const char* attr_name,
+    tensorflow::protobuf::RepeatedField<int64>* field) {
+  PyObject* seq = PyObject_GetAttrString(o, attr_name);
+  if (!seq) {
+    return false;
+  }
+
+  int length = PySequence_Size(seq);
+  if (length == -1) {
+    Py_DECREF(seq);
+    return false;
+  }
+
+  for (int i = 0; i < length; ++i) {
+    PyObject* item = PySequence_GetItem(seq, i);
+    if (!item) {
+      Py_DECREF(seq);
+      return false;
+    }
+    const int64 dimension = numpy::PyIntOrPyLongToLong(item);
+    if (dimension == -1 && PyErr_Occurred()) {
+      Py_DECREF(item);
+      Py_DECREF(seq);
+      return false;
+    }
+    *field->Add() = dimension;
+    Py_DECREF(item);
+  }
+  Py_DECREF(seq);
+  return true;
 }
-}
+
+}  // namespace swig
+}  // namespace xla
 %}
 
 // Required to use PyArray_* functions.
@@ -657,128 +692,27 @@ tensorflow::ImportNumpy();
 
 %typemap(in) const DotDimensionNumbers&
     (DotDimensionNumbers dimension_numbers) {
-  int length;
-
-  /* lhs_contracting_dimensions */
-  PyObject* lhs_contracting_dimensions = PyObject_GetAttrString(
-      $input, "lhs_contracting_dimensions");
-  if (!lhs_contracting_dimensions) {
-    SWIG_fail;
-  }
-
-  length = PySequence_Size(lhs_contracting_dimensions);
-  if (length == -1) {
-    Py_DECREF(lhs_contracting_dimensions);
-    SWIG_fail;
-  }
-
-  for (int i = 0; i < length; ++i) {
-    PyObject* item = PySequence_GetItem(lhs_contracting_dimensions, i);
-    if (!item) {
-      Py_DECREF(lhs_contracting_dimensions);
-      SWIG_fail;
-    }
-    const int64 dimension = numpy::PyIntOrPyLongToLong(item);
-    if (dimension == -1 && PyErr_Occurred()) {
-      Py_DECREF(item);
-      Py_DECREF(lhs_contracting_dimensions);
-      SWIG_fail;
-    }
-    dimension_numbers.add_lhs_contracting_dimensions(dimension);
-    Py_DECREF(item);
-  }
-  Py_DECREF(lhs_contracting_dimensions);
-
-  /* rhs_contracting_dimensions */
-  PyObject* rhs_contracting_dimensions = PyObject_GetAttrString(
-      $input, "rhs_contracting_dimensions");
-  if (!lhs_contracting_dimensions) {
+  if (!HandleRepeatedInt64Attribute(
+        $input, "lhs_contracting_dimensions",
+        dimension_numbers.mutable_lhs_contracting_dimensions())) {
     SWIG_fail;
   }
-
-  length = PySequence_Size(rhs_contracting_dimensions);
-  if (length == -1) {
-    Py_DECREF(rhs_contracting_dimensions);
+  if (!HandleRepeatedInt64Attribute(
+        $input, "rhs_contracting_dimensions",
+        dimension_numbers.mutable_rhs_contracting_dimensions())) {
     SWIG_fail;
   }
-
-  for (int i = 0; i < length; ++i) {
-    PyObject* item = PySequence_GetItem(rhs_contracting_dimensions, i);
-    if (!item) {
-      Py_DECREF(rhs_contracting_dimensions);
-      SWIG_fail;
-    }
-    const int64 dimension = numpy::PyIntOrPyLongToLong(item);
-    if (dimension == -1 && PyErr_Occurred()) {
-      Py_DECREF(item);
-      Py_DECREF(rhs_contracting_dimensions);
-      SWIG_fail;
-    }
-    dimension_numbers.add_rhs_contracting_dimensions(dimension);
-    Py_DECREF(item);
-  }
-  Py_DECREF(rhs_contracting_dimensions);
-
-  /* lhs_batch_dimensions */
-  PyObject* lhs_batch_dimensions = PyObject_GetAttrString(
-      $input, "lhs_batch_dimensions");
-  if (!lhs_batch_dimensions) {
+  if (!HandleRepeatedInt64Attribute(
+        $input, "lhs_batch_dimensions",
+        dimension_numbers.mutable_lhs_batch_dimensions())) {
     SWIG_fail;
   }
-
-  length = PySequence_Size(lhs_batch_dimensions);
-  if (length == -1) {
-    Py_DECREF(lhs_batch_dimensions);
-    SWIG_fail;
-  }
-
-  for (int i = 0; i < length; ++i) {
-    PyObject* item = PySequence_GetItem(lhs_batch_dimensions, i);
-    if (!item) {
-      Py_DECREF(lhs_batch_dimensions);
-      SWIG_fail;
-    }
-    const int64 dimension = numpy::PyIntOrPyLongToLong(item);
-    if (dimension == -1 && PyErr_Occurred()) {
-      Py_DECREF(item);
-      Py_DECREF(lhs_batch_dimensions);
-      SWIG_fail;
-    }
-    dimension_numbers.add_lhs_batch_dimensions(dimension);
-    Py_DECREF(item);
-  }
-  Py_DECREF(lhs_batch_dimensions);
-
-  /* rhs_batch_dimensions */
-  PyObject* rhs_batch_dimensions = PyObject_GetAttrString(
-      $input, "rhs_batch_dimensions");
-  if (!rhs_batch_dimensions) {
-    SWIG_fail;
-  }
-
-  length = PySequence_Size(rhs_batch_dimensions);
-  if (length == -1) {
-    Py_DECREF(rhs_batch_dimensions);
+  if (!HandleRepeatedInt64Attribute(
+        $input, "rhs_batch_dimensions",
+        dimension_numbers.mutable_rhs_batch_dimensions())) {
     SWIG_fail;
   }
 
-  for (int i = 0; i < length; ++i) {
-    PyObject* item = PySequence_GetItem(rhs_batch_dimensions, i);
-    if (!item) {
-      Py_DECREF(rhs_batch_dimensions);
-      SWIG_fail;
-    }
-    const int64 dimension = numpy::PyIntOrPyLongToLong(item);
-    if (dimension == -1 && PyErr_Occurred()) {
-      Py_DECREF(item);
-      Py_DECREF(rhs_batch_dimensions);
-      SWIG_fail;
-    }
-    dimension_numbers.add_rhs_batch_dimensions(dimension);
-    Py_DECREF(item);
-  }
-  Py_DECREF(rhs_batch_dimensions);
-
   $1 = &dimension_numbers;
 }
 
@@ -861,85 +795,80 @@ tensorflow::ImportNumpy();
   dimension_numbers.set_kernel_input_feature_dimension(value);
 
   PyObject* o;
-  int length;
 
-  o = PyObject_GetAttrString($input, "input_spatial_dimensions");
-  if (!o) {
+  if (!HandleRepeatedInt64Attribute(
+        $input, "input_spatial_dimensions",
+        dimension_numbers.mutable_input_spatial_dimensions())) {
     SWIG_fail;
   }
-  length = PySequence_Size(o);
-  if (length == -1) {
-    Py_DECREF(o);
+  if (!HandleRepeatedInt64Attribute(
+        $input, "kernel_spatial_dimensions",
+        dimension_numbers.mutable_kernel_spatial_dimensions())) {
     SWIG_fail;
   }
-  for (int i = 0; i < length; ++i) {
-    PyObject* item = PySequence_GetItem(o, i);
-    if (!item) {
-      Py_DECREF(o);
-      SWIG_fail;
-    }
-    const int64 dimension = numpy::PyIntOrPyLongToLong(item);
-    if (dimension == -1 && PyErr_Occurred()) {
-      Py_DECREF(item);
-      Py_DECREF(o);
-      SWIG_fail;
-    }
-    dimension_numbers.add_input_spatial_dimensions(dimension);
-    Py_DECREF(item);
+  if (!HandleRepeatedInt64Attribute(
+        $input, "output_spatial_dimensions",
+        dimension_numbers.mutable_output_spatial_dimensions())) {
+    SWIG_fail;
   }
-  Py_DECREF(o);
 
-  o = PyObject_GetAttrString($input, "kernel_spatial_dimensions");
-  if (!o) {
+  $1 = &dimension_numbers;
+}
+
+// GatherDimensionNumbers
+
+%typemap(in) const GatherDimensionNumbers&
+    (GatherDimensionNumbers dimension_numbers) {
+  if (!HandleRepeatedInt64Attribute(
+        $input, "offset_dims",
+        dimension_numbers.mutable_offset_dims())) {
     SWIG_fail;
   }
-  length = PySequence_Size(o);
-  if (length == -1) {
-    Py_DECREF(o);
+  if (!HandleRepeatedInt64Attribute(
+        $input, "collapsed_slice_dims",
+        dimension_numbers.mutable_collapsed_slice_dims())) {
     SWIG_fail;
   }
-  for (int i = 0; i < length; ++i) {
-    PyObject* item = PySequence_GetItem(o, i);
-    if (!item) {
-      Py_DECREF(o);
-      SWIG_fail;
-    }
-    const int64 dimension = numpy::PyIntOrPyLongToLong(item);
-    if (dimension == -1 && PyErr_Occurred()) {
-      Py_DECREF(item);
-      Py_DECREF(o);
-      SWIG_fail;
-    }
-    dimension_numbers.add_kernel_spatial_dimensions(dimension);
-    Py_DECREF(item);
+  if (!HandleRepeatedInt64Attribute(
+        $input, "start_index_map",
+        dimension_numbers.mutable_start_index_map())) {
+    SWIG_fail;
   }
-  Py_DECREF(o);
 
-  o = PyObject_GetAttrString($input, "output_spatial_dimensions");
-  if (!o) {
+  int64 value;
+  if (!GetIntAttr($input, "index_vector_dim", &value)) {
     SWIG_fail;
   }
-  length = PySequence_Size(o);
-  if (length == -1) {
-    Py_DECREF(o);
+  dimension_numbers.set_index_vector_dim(value);
+
+  $1 = &dimension_numbers;
+}
+
+// ScatterDimensionNumbers
+
+%typemap(in) const ScatterDimensionNumbers&
+    (ScatterDimensionNumbers dimension_numbers) {
+  if (!HandleRepeatedInt64Attribute(
+        $input, "update_window_dims",
+        dimension_numbers.mutable_update_window_dims())) {
     SWIG_fail;
   }
-  for (int i = 0; i < length; ++i) {
-    PyObject* item = PySequence_GetItem(o, i);
-    if (!item) {
-      Py_DECREF(o);
-      SWIG_fail;
-    }
-    const int64 dimension = numpy::PyIntOrPyLongToLong(item);
-    if (dimension == -1 && PyErr_Occurred()) {
-      Py_DECREF(item);
-      Py_DECREF(o);
-      SWIG_fail;
-    }
-    dimension_numbers.add_output_spatial_dimensions(dimension);
-    Py_DECREF(item);
+  if (!HandleRepeatedInt64Attribute(
+        $input, "inserted_window_dims",
+        dimension_numbers.mutable_inserted_window_dims())) {
+    SWIG_fail;
+  }
+  if (!HandleRepeatedInt64Attribute(
+        $input, "scatter_dims_to_operand_dims",
+        dimension_numbers.mutable_scatter_dims_to_operand_dims())) {
+    SWIG_fail;
+  }
+
+  int64 value;
+  if (!GetIntAttr($input, "index_vector_dim", &value)) {
+    SWIG_fail;
   }
-  Py_DECREF(o);
+  dimension_numbers.set_index_vector_dim(value);
 
   $1 = &dimension_numbers;
 }
@@ -1151,6 +1080,8 @@ tensorflow::ImportNumpy();
 %unignore xla::swig::LocalComputationBuilder::QR;
 %unignore xla::swig::LocalComputationBuilder::TriangularSolve;
 %unignore xla::swig::LocalComputationBuilder::CustomCall;
+%unignore xla::swig::LocalComputationBuilder::Gather;
+%unignore xla::swig::LocalComputationBuilder::Scatter;
 %unignore xla::swig::DeleteLocalComputation;
 %unignore xla::swig::DestructureLocalShapedBufferTuple;
 %unignore xla::swig::DestructureXrtAllocationTuple;
diff --git a/tensorflow/compiler/xla/python/xla_client.py b/tensorflow/compiler/xla/python/xla_client.py
index 378bbdcb17..4e71121c09 100644
--- a/tensorflow/compiler/xla/python/xla_client.py
+++ b/tensorflow/compiler/xla/python/xla_client.py
@@ -1477,6 +1477,18 @@ class ComputationBuilder(object):
     return self._client.TriangularSolve(
         a, b, left_side, lower, transpose_a, conjugate_a)
 
+  def Gather(self, a, start_indices, dimension_numbers, slice_sizes):
+    """Enqueues a Gather operation onto the computation."""
+    return self._client.Gather(a, start_indices, dimension_numbers,
+                               slice_sizes)
+
+  def Scatter(self, a, scatter_indices, updates, update_computation,
+              dimension_numbers):
+    """Enqueues a Scatter operation onto the computation."""
+    return self._client.Scatter(
+        a, scatter_indices, updates, update_computation.computation,
+        dimension_numbers,)
+
 
 def _forward_methods_to_local_builder():
   """Forward remaining ComputationBuilder methods to the C API.
diff --git a/tensorflow/compiler/xla/python/xla_client_test.py b/tensorflow/compiler/xla/python/xla_client_test.py
index 002a20e60a..874e087eb6 100644
--- a/tensorflow/compiler/xla/python/xla_client_test.py
+++ b/tensorflow/compiler/xla/python/xla_client_test.py
@@ -1129,6 +1129,21 @@ class SingleOpTest(LocalComputationTest):
     self.assertFalse(c.IsConstant(non_const_expr))
     # self.assertTrue(c.IsConstant(c.Sub(c.Add(x, a), x)))  # TODO(b/77245564)
 
+  def testGather(self):
+    a = np.arange(9).astype(np.int32).reshape((3, 3))
+    indices = np.array([[[0, 2], [2, 1]], [[1, 2], [2, 0]]], dtype=np.int32)
+    dnums = xla_client.xla_data_pb2.GatherDimensionNumbers()
+    dnums.offset_dims.append(1)
+    dnums.offset_dims.append(2)
+    dnums.start_index_map.append(0)
+    dnums.start_index_map.append(1)
+    dnums.index_vector_dim = 2
+    c = self._NewComputation()
+    c.Gather(c.Constant(a), c.Constant(indices), dnums, slice_sizes=[1, 1])
+    g = self._Execute(c, ())
+    expected = np.array([[[[2, 7]]], [[[5, 6]]]], dtype=np.int32)
+    np.testing.assert_allclose(g, expected, rtol=1e-4)
+
 
 class EmbeddedComputationsTest(LocalComputationTest):
   """Tests for XLA graphs with embedded computations (such as maps)."""
@@ -1186,6 +1201,14 @@ class EmbeddedComputationsTest(LocalComputationTest):
     c.Mul(c.ParameterFromNumpy(NumpyArrayF64(0)), c.ConstantF64Scalar(2.0))
     return c.Build()
 
+  def _CreateBinaryAddS32Computation(self):
+    """Computation (s32, s32) -> s32 that adds its two parameters."""
+    c = self._NewComputation("add_param0_by_param1")
+    c.Add(
+        c.ParameterFromNumpy(NumpyArrayS32(0)),
+        c.ParameterFromNumpy(NumpyArrayS32(0)))
+    return c.Build()
+
   def _CreateBinaryAddF32Computation(self):
     """Computation (f32, f32) -> f32 that adds its two parameters."""
     c = self._NewComputation("add_param0_by_param1")
@@ -1568,6 +1591,23 @@ class EmbeddedComputationsTest(LocalComputationTest):
       execution.join()
       self.assertEqual(want, got)
 
+  def testScatter(self):
+    a = np.arange(9).astype(np.int32).reshape((3, 3))
+    scatter_indices = np.array([0, 2], dtype=np.int32)
+    updates = np.array([[10, 20, 30], [70, 80, 90]], dtype=np.int32)
+
+    dnums = xla_client.xla_data_pb2.ScatterDimensionNumbers()
+    dnums.update_window_dims.append(1)
+    dnums.inserted_window_dims.append(0)
+    dnums.scatter_dims_to_operand_dims.append(0)
+    dnums.index_vector_dim = 1
+
+    c = self._NewComputation()
+    c.Scatter(c.Constant(a), c.Constant(scatter_indices), c.Constant(updates),
+              self._CreateBinaryAddS32Computation(), dnums)
+    expected = np.array([[10, 21, 32], [3, 4, 5], [76, 87, 98]], dtype=np.int32)
+    self._ExecuteAndCompareClose(c, expected=expected)
+
 
 class ErrorTest(LocalComputationTest):
 
-- 
GitLab


From 1ad09edf03a1c4b96eff12a7d6c765a430603b7d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 14:55:09 -0800
Subject: [PATCH 0375/2345] Use nsync, version 1.20.2, which has a bazel BUILD
 bug fix for aarch64 systems. It requires bazel version 0.16.0 or higher, but
 TensorFlow already relies on higher versions than that.

PiperOrigin-RevId: 228404987
---
 tensorflow/contrib/cmake/external/nsync.cmake | 2 +-
 tensorflow/workspace.bzl                      | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/cmake/external/nsync.cmake b/tensorflow/contrib/cmake/external/nsync.cmake
index 479609458c..b15143bfc1 100644
--- a/tensorflow/contrib/cmake/external/nsync.cmake
+++ b/tensorflow/contrib/cmake/external/nsync.cmake
@@ -16,7 +16,7 @@ include (ExternalProject)
 
 set(nsync_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/nsync/public)
 set(nsync_URL https://github.com/google/nsync)
-set(nsync_TAG 1.20.1)
+set(nsync_TAG 1.20.2)
 set(nsync_BUILD ${CMAKE_CURRENT_BINARY_DIR}/nsync/src/nsync)
 set(nsync_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/nsync/install)
 
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index ae26ba6e68..5b0f6d4c84 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -395,12 +395,12 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
 
     tf_http_archive(
         name = "nsync",
-        sha256 = "692f9b30e219f71a6371b98edd39cef3cbda35ac3abc4cd99ce19db430a5591a",
-        strip_prefix = "nsync-1.20.1",
+        sha256 = "704be7f58afa47b99476bbac7aafd1a9db4357cef519db361716f13538547ffd",
+        strip_prefix = "nsync-1.20.2",
         system_build_file = clean_dep("//third_party/systemlibs:nsync.BUILD"),
         urls = [
-            "https://mirror.bazel.build/github.com/google/nsync/archive/1.20.1.tar.gz",
-            "https://github.com/google/nsync/archive/1.20.1.tar.gz",
+            "https://mirror.bazel.build/github.com/google/nsync/archive/1.20.2.tar.gz",
+            "https://github.com/google/nsync/archive/1.20.2.tar.gz",
         ],
     )
 
-- 
GitLab


From 9f0ab258f7d715171f878080e6ed5c880aea004c Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Tue, 8 Jan 2019 15:17:41 -0800
Subject: [PATCH 0376/2345] Prevent OOM in decoding GIF images.

PiperOrigin-RevId: 228409275
---
 tensorflow/core/lib/gif/gif_io.cc | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/lib/gif/gif_io.cc b/tensorflow/core/lib/gif/gif_io.cc
index 9a5215320f..3fad8c8b14 100644
--- a/tensorflow/core/lib/gif/gif_io.cc
+++ b/tensorflow/core/lib/gif/gif_io.cc
@@ -82,9 +82,20 @@ uint8* Decode(const void* srcdata, int datasize,
     return nullptr;
   }
 
+  // Don't request more memory than needed for each frame, preventing OOM
+  int max_frame_width = 0;
+  int max_frame_height = 0;
+  for (int k = 0; k < gif_file->ImageCount; k++) {
+    SavedImage* si = &gif_file->SavedImages[k];
+    if (max_frame_height < si->ImageDesc.Height)
+      max_frame_height = si->ImageDesc.Height;
+    if (max_frame_width < si->ImageDesc.Width)
+      max_frame_width = si->ImageDesc.Width;
+  }
+
   const int num_frames = gif_file->ImageCount;
-  const int width = gif_file->SWidth;
-  const int height = gif_file->SHeight;
+  const int width = max_frame_width;    // gif_file->SWidth;
+  const int height = max_frame_height;  // gif_file->SHeight;
   const int channel = 3;
 
   uint8* const dstdata = allocate_output(num_frames, width, height, channel);
-- 
GitLab


From bef89aba9c73880ed0db8109e2d029b8e65cb367 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 15:17:50 -0800
Subject: [PATCH 0377/2345] Improves Flex performance.   - Avoid recreating the
 EagerOperation every time   - Avoid buffer_map churn by keeping the inputs
 and outputs along with the EagerOperation in OpNode.

PiperOrigin-RevId: 228409298
---
 tensorflow/lite/delegates/flex/kernel.cc      | 179 +++++++++++++-----
 tensorflow/lite/delegates/flex/kernel_test.cc |   7 +-
 2 files changed, 138 insertions(+), 48 deletions(-)

diff --git a/tensorflow/lite/delegates/flex/kernel.cc b/tensorflow/lite/delegates/flex/kernel.cc
index 84745ba9cb..d2853cf9f8 100644
--- a/tensorflow/lite/delegates/flex/kernel.cc
+++ b/tensorflow/lite/delegates/flex/kernel.cc
@@ -51,6 +51,15 @@ namespace tflite {
 namespace flex {
 namespace kernel {
 
+struct OpNode;
+
+// Represents the origin of a given tensor as a reference to the output
+// of an upstream node.
+struct TensorSource {
+  OpNode* node;
+  int node_output_index;
+};
+
 // A list of inputs of a given node of the TensorFlow/Eager graph.
 class OpInputs {
  public:
@@ -65,8 +74,26 @@ class OpInputs {
 
   int TfLiteIndex(int i) const { return inputs_[i]; }
 
+  // Given a map relating tensors to the node that originates them, populate a
+  // list of sources for the tensors in this class.
+  void InitializeTensorSources(
+      const std::map<int, TensorSource>& tflite_tensor_sources) {
+    sources_.clear();
+    for (int i : inputs_) {
+      auto it = tflite_tensor_sources.find(i);
+      if (it == tflite_tensor_sources.end()) {
+        sources_.push_back({nullptr, 0});
+      } else {
+        sources_.push_back(it->second);
+      }
+    }
+  }
+
+  TensorSource GetTensorSource(int i) const { return sources_[i]; }
+
  private:
   std::vector<int> inputs_;
+  std::vector<TensorSource> sources_;
 };
 
 // A list of outputs of a given node of the TensorFlow/Eager graph, along with
@@ -81,6 +108,33 @@ class OpOutputs {
   }
   ~OpOutputs() { ResetTensorHandles(); }
 
+  // Stores information about which of the tensors in this class are also
+  // outputs of the sugbraph.
+  void InitializeGraphOutputs(const std::set<int>& subgraph_outputs) {
+    subgraph_outputs_.clear();
+    for (int i : outputs_) {
+      subgraph_outputs_.push_back(subgraph_outputs.count(i) > 0);
+    }
+  }
+
+  // Returns true if the tensor given by index 'i' is an output of the entire
+  // subgraph.
+  bool IsSubgraphOutput(int i) const { return subgraph_outputs_[i]; }
+
+  // Returns a handle to a given tensor and, optionally, remove it from the
+  // internal vector.
+  tensorflow::TensorHandle* GetHandle(int i, bool remove) {
+    auto* handle = vector_[i];
+    if (!remove) {
+      handle->Ref();
+    } else {
+      // Don't increase the ref-count. Instead, simply take it out of the
+      // vector.
+      vector_[i] = nullptr;
+    }
+    return handle;
+  }
+
   int Size() const { return outputs_.size(); }
 
   int TfLiteIndex(int i) const { return outputs_[i]; }
@@ -102,6 +156,7 @@ class OpOutputs {
 
  private:
   std::vector<int> outputs_;
+  std::vector<bool> subgraph_outputs_;
   tensorflow::gtl::InlinedVector<tensorflow::TensorHandle*, 2> vector_;
 };
 
@@ -111,7 +166,9 @@ class OpNode {
  public:
   OpNode(const TfLiteIntArray* inputs, const TfLiteIntArray* outputs)
       : inputs_(inputs), outputs_(outputs) {}
-  ~OpNode() {}
+  ~OpNode() {
+    if (op_) ClearEagerInputs();
+  }
 
   const string& name() const { return name_; }
   void set_name(const string& name) { name_ = name; }
@@ -130,6 +187,8 @@ class OpNode {
   int NumInputs() const { return inputs_.Size(); }
   int NumOutputs() const { return outputs_.Size(); }
 
+  tensorflow::EagerOperation* op() { return op_.get(); }
+
   tensorflow::Status InitializeNodeDef(const void* custom_initial_data,
                                        int custom_initial_data_size) {
     if (!custom_initial_data) {
@@ -162,10 +221,8 @@ class OpNode {
 
   // Build thew new EagerOperation. In case of error, the returned 'op' is
   // guaranteed to be 'nullptr'.
-  tensorflow::Status BuildEagerOp(
-      tensorflow::EagerContext* eager_context,
-      std::unique_ptr<tensorflow::EagerOperation>* op) {
-    op->reset();
+  tensorflow::Status BuildEagerOp(tensorflow::EagerContext* eager_context) {
+    op_.reset();
 
     const tensorflow::AttrTypeMap* attr_types;
     bool is_function = false;
@@ -179,33 +236,47 @@ class OpNode {
           "')");
     }
 
-    op->reset(new tensorflow::EagerOperation(eager_context, name_.c_str(),
+    op_.reset(new tensorflow::EagerOperation(eager_context, name_.c_str(),
                                              /*is_function=*/false,
                                              attr_types));
+
+    op_->MutableAttrs()->NumInputs(inputs_.Size());
     for (const auto& attr : nodedef_.attr()) {
-      (*op)->MutableAttrs()->Set(attr.first, attr.second);
+      op_->MutableAttrs()->Set(attr.first, attr.second);
     }
 
     return tensorflow::Status::OK();
   }
 
-  tensorflow::Status BuildEagerInputs(BufferMap* buffer_map,
-                                      tensorflow::EagerOperation* op) {
+  void ClearEagerInputs() {
+    for (tensorflow::TensorHandle* h : *op_->MutableInputs()) {
+      if (h) h->Unref();
+    }
+    op_->MutableInputs()->clear();
+  }
+
+  tensorflow::Status BuildEagerInputs(const BufferMap* buffer_map) {
     for (int i = 0; i < inputs_.Size(); ++i) {
       int input_index = inputs_.TfLiteIndex(i);
-      if (!buffer_map->HasTensor(input_index)) {
-        return tensorflow::errors::Internal(
-            "Cannot read from invalid tensor index ", input_index);
-      }
-      auto* handle = new tensorflow::TensorHandle(
-          buffer_map->GetTensor(input_index), nullptr, nullptr, nullptr);
-      op->AddInput(handle);
-      handle->Unref();
-
-      if (buffer_map->IsForwardable(input_index)) {
-        // Take it out of the map, so Eager/TF can reuse the buffer for an
-        // output tensor of the op.
-        buffer_map->RemoveTensor(input_index);
+      TensorSource s = inputs_.GetTensorSource(i);
+      if (!s.node) {
+        // This input is not produced by this Eager subgraph (it could be a TF
+        // Lite native buffer, or could be produced by a separater subgraph). We
+        // need to fetch it from the delegate's buffer_map.
+        if (!buffer_map->HasTensor(input_index)) {
+          return tensorflow::errors::Internal(
+              "Cannot read from invalid tensor index ", input_index);
+        }
+        auto* handle = new tensorflow::TensorHandle(
+            buffer_map->GetTensor(input_index), nullptr, nullptr, nullptr);
+        op_->MutableInputs()->push_back(handle);
+      } else {
+        // If this is a forwardable tensor, we will remove it from the previous
+        // op's list, giving TF the opportunity to reuse its buffer.
+        bool unref_handle = buffer_map->IsForwardable(input_index);
+        auto* handle =
+            s.node->outputs_.GetHandle(s.node_output_index, unref_handle);
+        op_->MutableInputs()->push_back(handle);
       }
     }
     return tensorflow::Status::OK();
@@ -214,11 +285,12 @@ class OpNode {
   tensorflow::Status PersistEagerOutputs(BufferMap* buffer_map) {
     auto* handles = outputs_.GetTensorHandles();
     for (int i = 0; i < outputs_.Size(); ++i) {
-      const tensorflow::Tensor* tensor = nullptr;
-      TF_RETURN_IF_ERROR(handles->at(i)->Tensor(&tensor));
-      buffer_map->SetFromTensorFlow(outputs_.TfLiteIndex(i), *tensor);
+      if (outputs_.IsSubgraphOutput(i)) {
+        const tensorflow::Tensor* tensor = nullptr;
+        TF_RETURN_IF_ERROR(handles->at(i)->Tensor(&tensor));
+        buffer_map->SetFromTensorFlow(outputs_.TfLiteIndex(i), *tensor);
+      }
     }
-    outputs_.ResetTensorHandles();
     return tensorflow::Status::OK();
   }
 
@@ -236,19 +308,23 @@ class OpNode {
   OpInputs inputs_;
   // List of outputs, as TF Lite tensor indices.
   OpOutputs outputs_;
+
+  std::unique_ptr<tensorflow::EagerOperation> op_;
 };
 
 // Executes the TensorFlow op given by 'op_name', with the attributes specified
 // in 'nodedef'. Inputs and outputs are given as indices into the 'buffer_map'.
-tensorflow::Status ExecuteFlexOp(tensorflow::EagerContext* eager_context,
-                                 BufferMap* buffer_map, OpNode* node_data) {
-  std::unique_ptr<tensorflow::EagerOperation> op;
-  TF_RETURN_IF_ERROR(node_data->BuildEagerOp(eager_context, &op));
-  TF_RETURN_IF_ERROR(node_data->BuildEagerInputs(buffer_map, op.get()));
+tensorflow::Status ExecuteFlexOp(TfLiteContext* context, BufferMap* buffer_map,
+                                 OpNode* node_data) {
+  TF_RETURN_WITH_CONTEXT_IF_ERROR(node_data->BuildEagerInputs(buffer_map),
+                                  " (while executing '", node_data->name(),
+                                  "' via Eager)");
 
+  node_data->mutable_outputs()->ResetTensorHandles();
   int num_retvals = node_data->NumOutputs();
   TF_RETURN_WITH_CONTEXT_IF_ERROR(
-      EagerExecute(op.get(), node_data->mutable_outputs()->GetTensorHandles(),
+      EagerExecute(node_data->op(),
+                   node_data->mutable_outputs()->GetTensorHandles(),
                    &num_retvals),
       " (while executing '", node_data->name(), "' via Eager)");
 
@@ -259,6 +335,8 @@ tensorflow::Status ExecuteFlexOp(tensorflow::EagerContext* eager_context,
 
   TF_RETURN_IF_ERROR(node_data->PersistEagerOutputs(buffer_map));
 
+  node_data->ClearEagerInputs();
+
   return tensorflow::Status::OK();
 }
 
@@ -286,8 +364,10 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) {
                             ->GetBufferMap(context);
 
   CHECK(params->output_tensors);
+  std::set<int> output_set;
   for (auto tensor_index : TfLiteIntArrayView(params->output_tensors)) {
     op_data->subgraph_outputs.push_back(tensor_index);
+    output_set.insert(tensor_index);
   }
 
   CHECK(params->input_tensors);
@@ -313,6 +393,8 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) {
     status = node_data.InitializeNodeDef(node->custom_initial_data,
                                          node->custom_initial_data_size);
     if (!status.ok()) break;
+    status = node_data.BuildEagerOp(op_data->eager_context);
+    if (!status.ok()) break;
   }
 
   if (ConvertStatus(context, status) != kTfLiteOk) {
@@ -322,6 +404,26 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) {
     return op_data;
   }
 
+  // Given a TfLite tensor index, return the OpNode that produces it,
+  // along with it index into that OpNodes list of outputs.
+  std::map<int, TensorSource> tflite_tensor_sources;
+
+  // Find out how each tensor is produced. This does not account for
+  // tensors that are not produce by eager ops.
+  for (auto& node_data : op_data->nodes) {
+    node_data->mutable_outputs()->InitializeGraphOutputs(output_set);
+    for (int i = 0; i < node_data->outputs().Size(); ++i) {
+      int output_index = node_data->outputs().TfLiteIndex(i);
+      tflite_tensor_sources[output_index] = TensorSource{node_data.get(), i};
+    }
+  }
+
+  // For each node, resolve the inputs, so we can keep pointers to the nodes
+  // that produces them.
+  for (auto& node_data : op_data->nodes) {
+    node_data->mutable_inputs()->InitializeTensorSources(tflite_tensor_sources);
+  }
+
   return op_data;
 }
 
@@ -370,6 +472,7 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
                            node_data->name().c_str());
       return kTfLiteError;
     }
+    TF_LITE_ENSURE(context, node_data->op());
 
     for (int i = 0; i < node_data->inputs().Size(); ++i) {
       ++tensor_ref_count[node_data->inputs().TfLiteIndex(i)];
@@ -392,7 +495,6 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
 TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   auto* op_data = reinterpret_cast<OpData*>(node->user_data);
   BufferMap* buffer_map = op_data->buffer_map;
-  tensorflow::EagerContext* eager_context = op_data->eager_context;
 
   // Insert a tensor in the buffer map for all inputs that are not constant.
   // Constants were handled in Prepare() already.
@@ -414,7 +516,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
         reinterpret_cast<profiling::Profiler*>(context->profiler),
         node_data->name().c_str(), node_data->index());
 
-    auto status = ExecuteFlexOp(eager_context, buffer_map, node_data.get());
+    auto status = ExecuteFlexOp(context, buffer_map, node_data.get());
     TF_LITE_ENSURE_OK(context, ConvertStatus(context, status));
   }
 
@@ -433,15 +535,6 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
     tensor->data_is_stale = true;
   }
 
-  // We don't need to keep track of internal TF tensors any longer, so take
-  // them out of the buffer_map, but make sure we keep all the ones we might
-  // need for other subgraphs, or as final output of inference.
-  const auto& outputs = op_data->subgraph_outputs;
-  std::set<int> keep(outputs.begin(), outputs.end());
-  const auto& inputs = op_data->subgraph_inputs;
-  keep.insert(inputs.begin(), inputs.end());
-  buffer_map->RemoveTensorsNotInSet(keep);
-
   return kTfLiteOk;
 }
 
diff --git a/tensorflow/lite/delegates/flex/kernel_test.cc b/tensorflow/lite/delegates/flex/kernel_test.cc
index cef8017e6e..5b3a6d1647 100644
--- a/tensorflow/lite/delegates/flex/kernel_test.cc
+++ b/tensorflow/lite/delegates/flex/kernel_test.cc
@@ -144,12 +144,9 @@ TEST_F(KernelTest, BadTensorFlowOp) {
     return GenericPrepare(context, delegate, {0});
   });
 
-  SetShape(0, {2, 2, 1});
-  SetValues(0, {1.1f, 2.2f, 3.3f, 4.4f});
-
-  ASSERT_FALSE(Invoke());
+  ASSERT_NE(interpreter_->AllocateTensors(), kTfLiteOk);
   ASSERT_THAT(error_reporter().error_messages(),
-              ContainsRegex("while processing attributes of 'NonExistentOp'"));
+              ContainsRegex("Op type not registered 'NonExistentOp'"));
 }
 
 TEST_F(KernelTest, BadNumberOfOutputs) {
-- 
GitLab


From 2c6316172f2c17929d34aa64df45f08210f08a83 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Tue, 8 Jan 2019 15:29:53 -0800
Subject: [PATCH 0378/2345] Add a new experimental initialize method only to
 TPU strategy

PiperOrigin-RevId: 228411389
---
 .../contrib/distribute/python/tpu_strategy.py     |  8 ++++++++
 tensorflow/python/training/session_manager.py     | 15 +++++++++------
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 9cb2e461ab..f498f58b25 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -455,6 +455,14 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
     with _TPUReplicaContext(self._container_strategy()):
       return fn(*args, **kwargs)
 
+  def _experimental_initialize_system(self):
+    """Experimental method added to be used by Estimator.
+
+    This is a private method only to be used by Estimator. Other frameworks
+    should directly be calling `tf.contrib.distribute.initialize_tpu_system`
+    """
+    initialize_tpu_system(self._tpu_cluster_resolver)
+
   def _initialize(self):
     if context.executing_eagerly():
       # TODO(priyag): Add appopriate call here when eager is supported for TPUs.
diff --git a/tensorflow/python/training/session_manager.py b/tensorflow/python/training/session_manager.py
index 1ec9639177..cd22e7e7de 100644
--- a/tensorflow/python/training/session_manager.py
+++ b/tensorflow/python/training/session_manager.py
@@ -182,13 +182,16 @@ class SessionManager(object):
         set.
     """
     self._target = master
+
+    # This is required to so that we initialize the TPU device before
+    # restoring from checkpoint since we'll be placing variables on the device
+    # and TPUInitialize wipes out the memory of the device.
+    strategy = distribution_strategy_context.get_distribution_strategy()
+    if strategy and hasattr(strategy.extended,
+                            "_experimental_initialize_system"):
+      strategy.extended._experimental_initialize_system()  # pylint: disable=protected-access
+
     sess = session.Session(self._target, graph=self._graph, config=config)
-    # TODO(jhseu): Delete once tpu.initialize_system() goes away.
-    initialize_ops = (
-        distribution_strategy_context.get_distribution_strategy().initialize()
-    )
-    if initialize_ops:
-      sess.run(initialize_ops)
     if checkpoint_dir and checkpoint_filename_with_path:
       raise ValueError("Can not provide both checkpoint_dir and "
                        "checkpoint_filename_with_path.")
-- 
GitLab


From d4985a621d8e71b128d34fec2a0260109bb4fb7d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 15:32:32 -0800
Subject: [PATCH 0379/2345] This CL isolates collections between instances of
 wrap_function calls.

PiperOrigin-RevId: 228411886
---
 tensorflow/python/eager/wrap_function.py      |  3 +-
 tensorflow/python/eager/wrap_function_test.py | 39 +++++++++++++++++++
 tensorflow/python/framework/func_graph.py     | 34 ++++++++++------
 tensorflow/python/ops/cond_v2.py              |  7 ++--
 tensorflow/python/ops/while_v2.py             | 12 +++---
 5 files changed, 74 insertions(+), 21 deletions(-)

diff --git a/tensorflow/python/eager/wrap_function.py b/tensorflow/python/eager/wrap_function.py
index 2c65e97871..6f978feb3c 100644
--- a/tensorflow/python/eager/wrap_function.py
+++ b/tensorflow/python/eager/wrap_function.py
@@ -197,6 +197,7 @@ def wrap_function(fn, signature, name=None):
           name,
           holder,
           args=None, kwargs=None, signature=signature,
-          add_control_dependencies=False),
+          add_control_dependencies=False,
+          collections={}),
       variable_holder=holder,
       signature=signature)
diff --git a/tensorflow/python/eager/wrap_function_test.py b/tensorflow/python/eager/wrap_function_test.py
index 17e204b4d8..a6e1931fcd 100644
--- a/tensorflow/python/eager/wrap_function_test.py
+++ b/tensorflow/python/eager/wrap_function_test.py
@@ -104,6 +104,45 @@ class WrapFunctionTest(test.TestCase):
         fetches=f_wrapped.graph.get_tensor_by_name('fetch:0'))
     self.assertAllEqual(6.0, pruned())
 
+  def testCollectionsIsolation(self):
+
+    v1 = variables.Variable(2.)
+    v2_holder = []
+    def f():
+      v2 = variables.Variable(3.)
+      v2_holder.append(v2)
+      ops.add_to_collection(ops.GraphKeys.LOSSES, v2 * constant_op.constant(3.))
+      return array_ops.identity(v1 * v2 * constant_op.constant(1.), 'fetch')
+
+    f_wrapped = wrap_function.wrap_function(f, [])
+    self.assertAllEqual(6.0, f_wrapped())
+    self.assertEqual(
+        len(f_wrapped.graph.get_collection(ops.GraphKeys.LOSSES)), 1)
+    f_var_collection = f_wrapped.graph.get_collection(
+        ops.GraphKeys.TRAINABLE_VARIABLES)
+    self.assertEqual(len(f_var_collection), 1)
+    self.assertIs(f_var_collection[0], v2_holder[0])
+
+    v3_holder = []
+    def g():
+      v3 = variables.Variable(4.)
+      v3_holder.append(v3)
+      ops.add_to_collection(ops.GraphKeys.LOSSES, v3 * constant_op.constant(3.))
+      return array_ops.identity(v1 * v3 * constant_op.constant(1.), 'fetch')
+
+    g_wrapped = wrap_function.wrap_function(g, [])
+    self.assertAllEqual(8.0, g_wrapped())
+    self.assertEqual(
+        len(g_wrapped.graph.get_collection(ops.GraphKeys.LOSSES)), 1)
+    g_var_collection = g_wrapped.graph.get_collection(
+        ops.GraphKeys.TRAINABLE_VARIABLES)
+    self.assertEqual(len(g_var_collection), 1)
+    self.assertIs(g_var_collection[0], v3_holder[0])
+
+    # Both have only one value, and their values aren't equal. So no sharing.
+    self.assertNotEqual(g_wrapped.graph.get_collection(ops.GraphKeys.LOSSES),
+                        f_wrapped.graph.get_collection(ops.GraphKeys.LOSSES))
+
   def testGradientsOfPrune(self):
 
     v1 = variables.Variable(2.)
diff --git a/tensorflow/python/framework/func_graph.py b/tensorflow/python/framework/func_graph.py
index 9528a24b46..9603c1536d 100644
--- a/tensorflow/python/framework/func_graph.py
+++ b/tensorflow/python/framework/func_graph.py
@@ -18,7 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import collections
+import collections as py_collections
 import weakref
 
 from tensorflow.core.framework import attr_value_pb2
@@ -80,7 +80,7 @@ class FuncGraph(ops.Graph):
     seed: The graph-level random seed.
   """
 
-  def __init__(self, name, read_only_collections=True):
+  def __init__(self, name, collections=None):
     """Construct a new FuncGraph.
 
     The graph will inherit its graph key, collections, seed, and distribution
@@ -88,8 +88,13 @@ class FuncGraph(ops.Graph):
 
     Args:
       name: the name of the function.
-      read_only_collections: whether to not write function graph collections
-        back to default graph. Defaults to True.
+      collections: a dictionary of collections this FuncGraph should start
+        with. If not specified (None), the FuncGraph will read (but not write
+        to) the outer graph's collections that are not whitelisted, and both
+        read and write to the outer graph's collections that are whitelisted.
+        The current whitelisted collections are the global variables, the
+        local variables, and the trainable variables.
+        Defaults to None.
     """
     super(FuncGraph, self).__init__()
 
@@ -97,10 +102,9 @@ class FuncGraph(ops.Graph):
     self.inputs = []
     self.outputs = []
     self.structured_outputs = None
-    self._read_only_collections = read_only_collections
     self._weak_variables = []
     self.outer_graph = ops.get_default_graph()
-    self.captures = collections.OrderedDict()
+    self.captures = py_collections.OrderedDict()
 
     self._building_function = True
     # Map from resource tensor name to last op (in program order) which uses
@@ -122,9 +126,7 @@ class FuncGraph(ops.Graph):
       # specialization (currently used in cond_v2), here and in the cache key.
       self._colocation_stack = graph._colocation_stack.copy()  # pylint: disable=protected-access
 
-    if not self._read_only_collections:
-      self._collections = graph._collections  # pylint: disable=protected-access
-    else:
+    if collections is None:
       for collection_name in graph.get_all_collection_keys():
         if collection_name not in WHITELIST_COLLECTIONS:
           self._collections[collection_name] = graph.get_collection(
@@ -132,6 +134,8 @@ class FuncGraph(ops.Graph):
       for collection_name in WHITELIST_COLLECTIONS:
         self._collections[collection_name] = graph.get_collection_ref(
             collection_name)
+    else:
+      self._collections = collections
 
   def as_default(self):
     outer_cm = super(FuncGraph, self).as_default()
@@ -338,7 +342,8 @@ def func_graph_from_py_func(name,
                             autograph=False,
                             add_control_dependencies=True,
                             arg_names=None,
-                            op_return_value=None):
+                            op_return_value=None,
+                            collections=None):
   """Returns a `FuncGraph` generated from `python_func`.
 
   Args:
@@ -365,6 +370,13 @@ def func_graph_from_py_func(name,
     op_return_value: Optional. A Tensor. If set and `python_func` returns
       Operations, those return values will be replaced with this value. If not
       set, returning an Operation triggers an error.
+    collections: a dictionary of collections this FuncGraph should start
+      with. If not specified (None), the FuncGraph will read (but not write to)
+      the outer graph's collections that are not whitelisted, and both
+      read and write to the outer graph's collections that are whitelisted.
+      The current whitelisted collections are the global variables, the
+      local variables, and the trainable variables.
+      Defaults to None.
 
   Returns:
     A FuncGraph.
@@ -376,7 +388,7 @@ def func_graph_from_py_func(name,
   if op_return_value is not None:
     assert isinstance(op_return_value, ops.Tensor), op_return_value
   if func_graph is None:
-    func_graph = FuncGraph(name)
+    func_graph = FuncGraph(name, collections=collections)
   assert isinstance(func_graph, FuncGraph)
   if add_control_dependencies:
     control_manager = AutomaticControlDependencies
diff --git a/tensorflow/python/ops/cond_v2.py b/tensorflow/python/ops/cond_v2.py
index 7d09e32e24..5dca8e501b 100644
--- a/tensorflow/python/ops/cond_v2.py
+++ b/tensorflow/python/ops/cond_v2.py
@@ -68,14 +68,14 @@ def cond_v2(pred, true_fn, false_fn, name="cond"):
         true_name,
         true_fn, [], {},
         func_graph=util.CondBranchFuncGraph(
-            true_name, read_only_collections=False),
+            true_name, collections=ops.get_default_graph()._collections),  # pylint: disable=protected-access
         add_control_dependencies=add_control_dependencies,
         op_return_value=pred)
     false_graph = func_graph_module.func_graph_from_py_func(
         false_name,
         false_fn, [], {},
         func_graph=util.CondBranchFuncGraph(
-            false_name, read_only_collections=False),
+            false_name, collections=ops.get_default_graph()._collections),  # pylint: disable=protected-access
         add_control_dependencies=add_control_dependencies,
         op_return_value=pred)
 
@@ -554,7 +554,8 @@ class _CondGradFuncGraph(util.CondBranchFuncGraph):
   """
 
   def __init__(self, name, forward_graph):
-    super(_CondGradFuncGraph, self).__init__(name, read_only_collections=False)
+    super(_CondGradFuncGraph, self).__init__(
+        name, collections=ops.get_default_graph()._collections)  # pylint: disable=protected-access
     self.if_op_needs_rewrite = False
     self._forward_graph = forward_graph
     # Maps from forward intermediate tensor -> the unwrapped captured
diff --git a/tensorflow/python/ops/while_v2.py b/tensorflow/python/ops/while_v2.py
index f9fea0e5bd..f5a51bb1bc 100644
--- a/tensorflow/python/ops/while_v2.py
+++ b/tensorflow/python/ops/while_v2.py
@@ -115,15 +115,15 @@ def while_loop(cond,
             loop_counter < maximum_iterations,
             cond(*_pack_sequence_as(orig_loop_vars, args)))
 
-    # NOTE(skyewm): we set read_only_collections=False for compatibility with
-    # TPUEstimator.
+    # NOTE(skyewm): we set collections to the outer graph's collections for
+    # compatibility with TPUEstimator.
     cond_graph = func_graph_module.func_graph_from_py_func(
         cond_name,
         wrapped_cond,
         loop_vars, {},
         signature=_build_signature(loop_vars, shape_invariants),
-        func_graph=util.WhileCondFuncGraph(cond_name,
-                                           read_only_collections=False),
+        func_graph=util.WhileCondFuncGraph(
+            cond_name, collections=ops.get_default_graph()._collections),  # pylint: disable=protected-access
         add_control_dependencies=add_control_dependencies)
 
     # Add external_captures of cond to the list of loop vars.
@@ -174,8 +174,8 @@ def while_loop(cond,
         wrapped_body,
         loop_vars, {},
         signature=_build_signature(loop_vars, shape_invariants),
-        func_graph=util.WhileBodyFuncGraph(body_name,
-                                           read_only_collections=False),
+        func_graph=util.WhileBodyFuncGraph(
+            body_name, collections=ops.get_default_graph()._collections),  # pylint: disable=protected-access
         add_control_dependencies=add_control_dependencies)
     # Add external captures of body to the list of loop vars.
     # Note that external tensors will be treated as loop invariants, i.e.,
-- 
GitLab


From f71e77471c140bac2f4dcb9a003139fb5c657302 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 8 Jan 2019 15:35:49 -0800
Subject: [PATCH 0380/2345] Don't check for alignment before calling into a
 tiled dot (Eigen or LLVM)

The alignment restrictions on Eigen was lifted in cr/226091175.  This CL lifts
the alignment restriction on the tiled LLVM IR GEMM.

PiperOrigin-RevId: 228412524
---
 tensorflow/compiler/xla/service/cpu/BUILD     |   2 -
 .../cpu/cpu_eigen_tensor_alignment_test.cc    |  38 ------
 .../xla/service/cpu/dot_op_emitter.cc         | 117 ++++++++++++------
 .../xla/service/cpu/dot_op_emitter_internal.h |  88 -------------
 4 files changed, 77 insertions(+), 168 deletions(-)
 delete mode 100644 tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h

diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index de62aa60aa..a197bdddc8 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -385,7 +385,6 @@ cc_library(
     srcs = ["dot_op_emitter.cc"],
     hdrs = [
         "dot_op_emitter.h",
-        "dot_op_emitter_internal.h",
     ],
     deps = [
         ":cpu_options",
@@ -1029,7 +1028,6 @@ tf_cc_test(
     size = "small",
     srcs = ["cpu_eigen_tensor_alignment_test.cc"],
     deps = [
-        ":dot_op_emitter",
         ":ir_emission_utils",
         ":target_machine_features_fake",
         "//tensorflow/compiler/xla:test",
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc b/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc
index 823bdf259c..485769a373 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc
@@ -13,7 +13,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h"
 #include "tensorflow/compiler/xla/service/cpu/ir_emission_utils.h"
 #include "tensorflow/compiler/xla/service/cpu/target_machine_features_fake.h"
 #include "tensorflow/compiler/xla/service/hlo_parser.h"
@@ -23,48 +22,11 @@ namespace xla {
 namespace cpu {
 namespace {
 
-using internal::DotImplementationStrategy;
-using internal::DotInfo;
-using internal::GetDotImplementationStrategy;
-
 // Test that we don't call into Eigen with tensors too small to be aligned
 // reliably.
 
 class CpuEigenTensorAlignmentTest : public ::testing::Test {};
 
-TEST_F(CpuEigenTensorAlignmentTest, EigenDotAlignment) {
-  string hlo_string = R"(
-HloModule DotOperation
-
-ENTRY DotOperation {
-  arg0 = f32[5,256] parameter(0)
-  arg1 = f32[256,1024] parameter(1)
-  ROOT dot = f32[5,1024] dot(arg0, arg1), lhs_contracting_dims={1}, rhs_contracting_dims={0}
-}
-)";
-
-  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
-                          ParseHloString(hlo_string));
-
-  HloInstruction* dot = module->entry_computation()->root_instruction();
-
-  TargetMachineFeaturesWithFakeAlignmentLogic target_machine_with_no_alignment(
-      [](int64 size) { return 1; });
-
-  EXPECT_EQ(GetDotImplementationStrategy(HloModuleConfig{}, DotInfo(*dot),
-                                         target_machine_with_no_alignment),
-            DotImplementationStrategy::kNaiveLlvmIr);
-
-  TargetMachineFeaturesWithFakeAlignmentLogic
-      target_machine_with_full_alignment([](int64 size) {
-        return TargetMachineFeatures::kEigenExpectedTensorAlignment;
-      });
-
-  EXPECT_NE(GetDotImplementationStrategy(HloModuleConfig{}, DotInfo(*dot),
-                                         target_machine_with_full_alignment),
-            DotImplementationStrategy::kNaiveLlvmIr);
-}
-
 TEST_F(CpuEigenTensorAlignmentTest, EigenConvAlignment) {
   string hlo_string = R"(
 HloModule ConvOperation
diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
index e8a84ebe6b..b018e0cd46 100644
--- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
@@ -14,7 +14,6 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/compiler/xla/service/cpu/dot_op_emitter.h"
-#include "tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h"
 
 #include <memory>
 #include <vector>
@@ -43,17 +42,63 @@ namespace xla {
 using llvm_ir::SetToFirstInsertPoint;
 
 namespace cpu {
-
-using internal::DotImplementationStrategy;
-using internal::DotInfo;
-using internal::GetDotImplementationStrategy;
-
 namespace {
 // Returns true if we should call into multi-threaded Eigen routines.
 bool ShouldUseMultiThreadedEigen(const HloModuleConfig& config) {
   return config.debug_options().xla_cpu_multi_thread_eigen();
 }
 
+// Represents a dot operation.  We use this in lieu of an `HloInstruction`
+// because we want to be able to create this for the "inner" dot operation in a
+// batch dot, for which there is no separate HLO instruction.
+struct DotInfo {
+  Shape lhs_shape;
+  Shape rhs_shape;
+  Shape result_shape;
+  DotDimensionNumbers dim_nums;
+
+  explicit DotInfo(const HloInstruction& instr) {
+    CHECK_EQ(instr.opcode(), HloOpcode::kDot);
+    lhs_shape = instr.operand(0)->shape();
+    rhs_shape = instr.operand(1)->shape();
+    result_shape = instr.shape();
+    dim_nums = instr.dot_dimension_numbers();
+  }
+};
+
+// Dictates how a dot operation is implemented.
+enum class DotImplementationStrategy {
+  // The dot operation is lowered into LLVM IR that implements a naive nested
+  // loop that computes the result one element at a time.  This is our
+  // "fallback"; we don't really want this to kick in for any non-trival dot
+  // operation.
+  kNaiveLlvmIr,
+
+  // The dot operation is lowered into LLVM IR that implements a tiled
+  // Matrix*Vector operation.  This strategy also allows fusing in a bias add
+  // into the dot.  The matrix can be row major or column major, both are
+  // supported.
+  kTiledLlvmIrGemv,
+
+  // The dot operation is lowered into LLVM IR that implemetns a tiled
+  // Matrix*Matrix operation.  No fusions are supported.  The two inputs
+  // and the output have to be row major.
+  kTiledLlvmIrGemm,
+
+  // The dot operation is lowered into a call into an Eigen routine.  No fusions
+  // are supported today.  The two inputs and the output have to be row major.
+  // However, we do allow transposing either the LHS or the RHS as part of the
+  // GEMM -- we expose this flexibility as flexibility in the contraction
+  // dimensions, but we can also see this as flexibility in the input layouts.
+  kEigen,
+};
+
+// Returns the implementation strategy for a dot with the configuration
+// `dot_info`.
+DotImplementationStrategy GetDotImplementationStrategy(
+    const HloModuleConfig& config, const DotInfo& dot_info,
+    const TargetMachineFeatures& target_machine_features);
+
 // Helper class for emitting LLVM IR to perform the dot operation.
 class DotOpEmitter {
  public:
@@ -190,9 +235,8 @@ void DotOpEmitter::EmitTiledLlvmIrGemm() {
   }
 
   int64 size_bytes = m * n * ShapeUtil::ByteSizeOfPrimitiveType(primitive_type);
-  b_->CreateMemSet(
-      target, b_->getInt8(0), size_bytes,
-      target_machine_features_.minimum_alignment_for_allocation(size_bytes));
+  b_->CreateMemSet(target, b_->getInt8(0), /*Size=*/size_bytes,
+                   /*Align=*/1);
 
   int64 max_target_vector_width =
       target_machine_features_.vector_register_num_elements(
@@ -696,40 +740,34 @@ absl::optional<int64> ProfitableToMakeDotOperandColumnMajor(
   return {};
 }
 
-namespace internal {
 namespace {
 // Return whether the given shape is rank 2.
 bool IsRank2(const Shape& shape) { return shape.rank() == 2; }
 
+bool IsSimpleLayout(const Layout& layout) {
+  return layout.tiles().empty() && layout.format() == DENSE;
+}
+
 // In a gemm operation where output = lhs * rhs, check whether the given shapes
 // are valid for the operation.
-bool AreAlignedGemmShapes(
-    const Shape& lhs_shape, const Shape& rhs_shape, const Shape& output_shape,
-    const TargetMachineFeatures& target_machine_features) {
-  // The inputs and the output must
-  // 1) be matrices with no padding, and
-  // 2) have an allowed element type.
-  PrimitiveType output_primitive_type = output_shape.element_type();
-  if (!(output_primitive_type == F64 || output_primitive_type == F32 ||
-        output_primitive_type == F16)) {
-    return false;
-  }
-
-  if (!(IsRank2(lhs_shape) && IsRank2(rhs_shape) && IsRank2(output_shape))) {
-    return false;
-  }
-
-  auto is_aligned = [&](const Shape& shape) {
-    return GetMinimumAlignmentForArray(shape, target_machine_features) >=
-           TargetMachineFeatures::kEigenExpectedTensorAlignment;
-  };
-
-  if (!is_aligned(lhs_shape) || !is_aligned(rhs_shape) ||
-      !is_aligned(output_shape)) {
-    return false;
+bool AreGemmShapes(const Shape& lhs_shape, const Shape& rhs_shape,
+                   const Shape& output_shape,
+                   const TargetMachineFeatures& target_machine_features) {
+  CHECK(!lhs_shape.has_layout() || IsSimpleLayout(lhs_shape.layout()))
+      << lhs_shape.DebugString();
+  CHECK(!rhs_shape.has_layout() || IsSimpleLayout(rhs_shape.layout()))
+      << rhs_shape.DebugString();
+  CHECK(!output_shape.has_layout() || IsSimpleLayout(output_shape.layout()))
+      << output_shape.DebugString();
+
+  switch (output_shape.element_type()) {
+    case F64:
+    case F32:
+    case F16:
+      return IsRank2(lhs_shape) && IsRank2(rhs_shape) && IsRank2(output_shape);
+    default:
+      return false;
   }
-
-  return true;
 }
 
 bool IsAlignedGemm(const DotInfo& dot_info,
@@ -739,8 +777,8 @@ bool IsAlignedGemm(const DotInfo& dot_info,
     return false;
   }
 
-  return AreAlignedGemmShapes(dot_info.lhs_shape, dot_info.rhs_shape,
-                              dot_info.result_shape, target_machine_features);
+  return AreGemmShapes(dot_info.lhs_shape, dot_info.rhs_shape,
+                       dot_info.result_shape, target_machine_features);
 }
 
 bool CanEmitTiledLlvmIrGemm(
@@ -781,7 +819,6 @@ bool CanEmitTiledLlvmIrGemm(
 
   return true;
 }
-}  // namespace
 
 DotImplementationStrategy GetDotImplementationStrategy(
     const HloModuleConfig& config, const DotInfo& dot_info,
@@ -806,7 +843,7 @@ DotImplementationStrategy GetDotImplementationStrategy(
 
   return DotImplementationStrategy::kNaiveLlvmIr;
 }
-}  // namespace internal
+}  // namespace
 
 bool DotImplementationCanHandleTranspose(
     const HloInstruction& dot_instr,
diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h b/tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h
deleted file mode 100644
index cc28918ed6..0000000000
--- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h
+++ /dev/null
@@ -1,88 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_CPU_DOT_OP_EMITTER_INTERNAL_H_
-#define TENSORFLOW_COMPILER_XLA_SERVICE_CPU_DOT_OP_EMITTER_INTERNAL_H_
-
-#include "tensorflow/compiler/xla/service/cpu/target_machine_features.h"
-#include "tensorflow/compiler/xla/service/hlo_instruction.h"
-#include "tensorflow/compiler/xla/service/hlo_module.h"
-
-// -----------------------------------------------------------------------------
-// INTERNAL HEADER.
-//
-// This file exposes internal implementation details from dot_op_emitter.cc for
-// unit tests.  Please do not depend on this!
-//
-// -----------------------------------------------------------------------------
-
-namespace xla {
-namespace cpu {
-namespace internal {
-
-// Represents a dot operation.  We use this in lieu of an `HloInstruction`
-// because we want to be able to create this for the "inner" dot operation in a
-// batch dot, for which there is no separate HLO instruction.
-struct DotInfo {
-  Shape lhs_shape;
-  Shape rhs_shape;
-  Shape result_shape;
-  DotDimensionNumbers dim_nums;
-
-  explicit DotInfo(const HloInstruction& instr) {
-    CHECK_EQ(instr.opcode(), HloOpcode::kDot);
-    lhs_shape = instr.operand(0)->shape();
-    rhs_shape = instr.operand(1)->shape();
-    result_shape = instr.shape();
-    dim_nums = instr.dot_dimension_numbers();
-  }
-};
-
-// Dictates how a dot operation is implemented.
-enum class DotImplementationStrategy {
-  // The dot operation is lowered into LLVM IR that implements a naive nested
-  // loop that computes the result one element at a time.  This is our
-  // "fallback"; we don't really want this to kick in for any non-trival dot
-  // operation.
-  kNaiveLlvmIr,
-
-  // The dot operation is lowered into LLVM IR that implements a tiled
-  // Matrix*Vector operation.  This strategy also allows fusing in a bias add
-  // into the dot.  The matrix can be row major or column major, both are
-  // supported.
-  kTiledLlvmIrGemv,
-
-  // The dot operation is lowered into LLVM IR that implemetns a tiled
-  // Matrix*Matrix operation.  No fusions are supported.  The two inputs
-  // and the output have to be row major.
-  kTiledLlvmIrGemm,
-
-  // The dot operation is lowered into a call into an Eigen routine.  No fusions
-  // are supported today.  The two inputs and the output have to be row major.
-  // However, we do allow transposing either the LHS or the RHS as part of the
-  // GEMM -- we expose this flexibility as flexibility in the contraction
-  // dimensions, but we can also see this as flexibility in the input layouts.
-  kEigen,
-};
-
-// Returns the implementation strategy for a dot with the configuration
-// `dot_info`.
-DotImplementationStrategy GetDotImplementationStrategy(
-    const HloModuleConfig& config, const DotInfo& dot_info,
-    const TargetMachineFeatures& target_machine_features);
-}  // namespace internal
-}  // namespace cpu
-}  // namespace xla
-
-#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_CPU_DOT_OP_EMITTER_INTERNAL_H_
-- 
GitLab


From cb0ecda8555f827c4fa4d9c160c68ea028bdc305 Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Tue, 8 Jan 2019 15:41:12 -0800
Subject: [PATCH 0381/2345] Unbreak windows builds.

cuda_deps seems not to work well with select() on windows.

PiperOrigin-RevId: 228413463
---
 .../core/platform/default/build_config/BUILD     | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/tensorflow/core/platform/default/build_config/BUILD b/tensorflow/core/platform/default/build_config/BUILD
index f5a194428c..c0c1022c16 100644
--- a/tensorflow/core/platform/default/build_config/BUILD
+++ b/tensorflow/core/platform/default/build_config/BUILD
@@ -32,15 +32,7 @@ cc_library(
 
 tf_cuda_library(
     name = "stream_executor",
-    cuda_deps = [
-        "//tensorflow/stream_executor/cuda:cuda_activation",
-    ] + select({
-        "//tensorflow:using_cuda_clang": ["//tensorflow/stream_executor/cuda:all_runtime"],
-        "//tensorflow:using_cuda_nvcc": ["//tensorflow/stream_executor/cuda:all_runtime"],
-        "//tensorflow:using_cuda_clang_with_dynamic_build": [],
-        "//tensorflow:using_cuda_nvcc_with_dynamic_build": [],
-        "//conditions:default": [],
-    }),
+    cuda_deps = ["//tensorflow/stream_executor/cuda:cuda_activation"],
     deps = [
         "//tensorflow/stream_executor",
         "//tensorflow/stream_executor:dnn",
@@ -53,6 +45,12 @@ tf_cuda_library(
     ] + select({
         "@local_config_cuda//cuda:darwin": ["IOKit"],
         "//conditions:default": [],
+    }) + select({
+        "//tensorflow:using_cuda_clang": ["//tensorflow/stream_executor/cuda:all_runtime"],
+        "//tensorflow:using_cuda_nvcc": ["//tensorflow/stream_executor/cuda:all_runtime"],
+        "//tensorflow:using_cuda_clang_with_dynamic_build": [],
+        "//tensorflow:using_cuda_nvcc_with_dynamic_build": [],
+        "//conditions:default": [],
     }),
 )
 
-- 
GitLab


From 29579a2e548800994f1acdf37fc2f8bf42b043ee Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Tue, 8 Jan 2019 15:50:48 -0800
Subject: [PATCH 0382/2345] Fix tf_driver_test

Use the proper path for the tested model.

PiperOrigin-RevId: 228415121
---
 tensorflow/lite/testing/tf_driver_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/testing/tf_driver_test.cc b/tensorflow/lite/testing/tf_driver_test.cc
index 363d162d56..e79704d616 100644
--- a/tensorflow/lite/testing/tf_driver_test.cc
+++ b/tensorflow/lite/testing/tf_driver_test.cc
@@ -93,7 +93,7 @@ TEST(TfDriverTest, SimpleTest) {
                    {"1,8,8,3", "1,8,8,3", "1,8,8,3", "1,8,8,3"}, {"x", "y"}));
 
   runner->LoadModel(
-      "third_party/tensorflow/lite/testdata/multi_add.pb");
+      "tensorflow/lite/testdata/multi_add.pb");
   EXPECT_TRUE(runner->IsValid()) << runner->GetErrorMessage();
 
   ASSERT_THAT(runner->GetInputs(), ElementsAre(0, 1, 2, 3));
-- 
GitLab


From 0b53b291f31d9ac9d1bb9cffbc037fe978f593fd Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne <reedwm@google.com>
Date: Tue, 8 Jan 2019 15:57:52 -0800
Subject: [PATCH 0383/2345] Automated rollback of changelist 205730535

PiperOrigin-RevId: 228416180
---
 tensorflow/core/BUILD                         | 11 ++++
 .../core/common_runtime/gpu/gpu_device.cc     | 51 ++++++++++++++++++-
 .../core/common_runtime/gpu/gpu_device.h      | 11 ++++
 .../gpu/gpu_device_kernel_check.cu.cc         | 37 ++++++++++++++
 .../gpu/gpu_device_kernel_check.h             | 32 ++++++++++++
 5 files changed, 141 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.cu.cc
 create mode 100644 tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index e6af9211b5..6a842a432f 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -87,6 +87,7 @@ load(
     "tf_gen_op_libs",
     "tf_generate_proto_text_sources",
     "tf_genrule_cmd_append_to_srcs",
+    "tf_gpu_kernel_library",
     "tf_opts_nortti_if_android",
     "transitive_hdrs",
 )
@@ -3162,6 +3163,15 @@ cc_library(
     ],
 )
 
+tf_gpu_kernel_library(
+    name = "gpu_device_kernel_check",
+    srcs = ["common_runtime/gpu/gpu_device_kernel_check.cu.cc"],
+    hdrs = ["common_runtime/gpu/gpu_device_kernel_check.h"],
+    deps = [
+        "//tensorflow/core:stream_executor",
+    ],
+)
+
 GPU_RUNTIME_HEADERS = [
     "common_runtime/gpu/cuda_host_allocator.h",
     "common_runtime/gpu/gpu_bfc_allocator.h",
@@ -3200,6 +3210,7 @@ tf_cuda_library(
         ":core_cpu_lib",
         ":framework",
         ":framework_internal",
+        ":gpu_device_kernel_check",
         ":gpu_id_impl",
         ":gpu_init_impl",
         ":gpu_lib",
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.cc b/tensorflow/core/common_runtime/gpu/gpu_device.cc
index 010fdff4e9..891b6e0e2a 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.cc
@@ -31,6 +31,7 @@ limitations under the License.
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/common_runtime/device_factory.h"
+#include "tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_event_mgr.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_id.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_id_manager.h"
@@ -388,7 +389,7 @@ Status BaseGPUDevice::Init(const SessionOptions& options) {
     }
   }
 
-  return Status::OK();
+  return CheckGPU();
 }
 
 bool BaseGPUDevice::RequiresRecordingAccessedTensors() const {
@@ -907,6 +908,54 @@ Allocator* BaseGPUDevice::GetScopedAllocator(AllocatorAttributes attr,
   return gpu_allocator_;
 }
 
+Status BaseGPUDevice::CheckGPU() {
+  se::Stream* stream = tensorflow_gpu_device_info()->stream;
+  TF_RETURN_IF_ERROR(stream->BlockHostUntilDone());
+  Tensor device_tensor(gpu_allocator_, DT_FLOAT, {});
+  if (!device_tensor.IsInitialized()) {
+    return errors::ResourceExhausted("Failed to allocate ", sizeof(float),
+                                     " bytes on the GPU for initialization "
+                                     "checks");
+  }
+  float* val_dev = device_tensor.scalar<float>().data();
+  const cudaStream_t cu_stream = *reinterpret_cast<const cudaStream_t*>(
+      stream->implementation()->GpuStreamMemberHack());
+  {
+    se::cuda::ScopedActivateExecutorContext scoped_activation{stream->parent()};
+    run_test_kernel(val_dev, cu_stream);
+    // We have to use the CUDA runtime function cudaPeekAtLastError here,
+    // because 'stream' does not provide a way to check if a kernel launch
+    // succeeds. Calling 'stream->BlockHostUntilDone()', which internally calls
+    // 'cuCtxSynchronize()', does not catch all kernel launch errors.
+    cudaError_t cuda_error = cudaPeekAtLastError();
+    if (cuda_error == cudaSuccess) {
+      cuda_error = cudaDeviceSynchronize();
+    }
+    TF_RETURN_IF_ERROR(CudaErrorToStatus(cuda_error, *stream));
+  }
+
+  float val_host = 0.;
+  stream->ThenMemcpy(&val_host, se::DeviceMemoryBase(val_dev, sizeof(float)),
+                     sizeof(float));
+  TF_RETURN_IF_ERROR(stream->BlockHostUntilDone());
+  if (val_host != 12345.) {
+    return errors::Internal(
+        "GPU kernel for initialization returned wrong value: ", val_host);
+  }
+  return Status::OK();
+}
+
+Status BaseGPUDevice::CudaErrorToStatus(cudaError_t cuda_error,
+                                        const se::Stream& stream) {
+  if (cuda_error != cudaSuccess) {
+    return errors::Internal(
+        "Failed to run GPU kernel for the initialization check. Received "
+        "error ",
+        cudaGetErrorName(cuda_error), " after running GPU kernel.");
+  }
+  return Status::OK();
+}
+
 const int BaseGPUDeviceFactory::InterconnectMap::kSameDeviceStrength = 1000;
 const int BaseGPUDeviceFactory::InterconnectMap::kStreamExecutorStrength = 1;
 
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.h b/tensorflow/core/common_runtime/gpu/gpu_device.h
index d002d02c51..86622dad49 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.h
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.h
@@ -26,6 +26,7 @@ limitations under the License.
 #include <vector>
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "cuda/include/cuda_runtime_api.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_event_mgr.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_id.h"
@@ -121,6 +122,12 @@ class BaseGPUDevice : public LocalDevice {
   se::StreamExecutor* executor_;  // not owned
   std::unique_ptr<ScopedAllocatorMgr> scoped_allocator_mgr_;
 
+  // Returns a Status corresponding to a cudaError_t. The CUDA error must have
+  // been obtained from a CUDA kernel launch used to check if the GPU is
+  // initialized properly.
+  virtual Status CudaErrorToStatus(cudaError_t cuda_error,
+                                   const se::Stream& stream);
+
  private:
   struct StreamGroup {
     se::Stream* compute = nullptr;
@@ -161,6 +168,10 @@ class BaseGPUDevice : public LocalDevice {
   Status MaybeCopyTensorToGPU(const AllocatorAttributes& alloc_attrs,
                               const Tensor& from, Tensor* to,
                               StatusCallback done);
+
+  // Checks that the GPU is capable of doing work, by running a test kernel on
+  // it.
+  Status CheckGPU();
 };
 
 class BaseGPUDeviceFactory : public DeviceFactory {
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.cu.cc b/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.cu.cc
new file mode 100644
index 0000000000..017565195b
--- /dev/null
+++ b/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.cu.cc
@@ -0,0 +1,37 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#include "tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h"
+#include "tensorflow/stream_executor/cuda/cuda_activation.h"
+
+namespace {
+__global__ void test_kernel(float* val) {
+  if (blockIdx.x == 0 && threadIdx.x == 0) {
+    (*val) = 12345.;
+  }
+}
+}  // namespace
+
+namespace tensorflow {
+
+void run_test_kernel(float* val, cudaStream_t cu_stream) {
+  test_kernel<<<1, 1, 0, cu_stream>>>(val);
+}
+
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h b/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h
new file mode 100644
index 0000000000..064fb7a49f
--- /dev/null
+++ b/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h
@@ -0,0 +1,32 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_DEVICE_KERNEL_CHECK_H_
+#define TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_DEVICE_KERNEL_CHECK_H_
+
+#if GOOGLE_CUDA
+
+#include "tensorflow/core/platform/stream_executor.h"
+
+namespace tensorflow {
+
+// Runs a GPU kernel to test that it functions correctly. Sets 'val' to 12345.
+void run_test_kernel(float* val, cudaStream_t cu_stream);
+
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
+
+#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_DEVICE_KERNEL_CHECK_H_
-- 
GitLab


From 03ec3f7b11572b70ef7f33f2a9af9bebbb6ce473 Mon Sep 17 00:00:00 2001
From: Toby Boyd <tobyboyd@google.com>
Date: Tue, 8 Jan 2019 16:24:00 -0800
Subject: [PATCH 0384/2345] Installs TensorRT

---
 tensorflow/tools/ci_build/Dockerfile.gpu | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tensorflow/tools/ci_build/Dockerfile.gpu b/tensorflow/tools/ci_build/Dockerfile.gpu
index ff1fefe892..f5a28ff163 100644
--- a/tensorflow/tools/ci_build/Dockerfile.gpu
+++ b/tensorflow/tools/ci_build/Dockerfile.gpu
@@ -7,6 +7,12 @@ LABEL maintainer="Jan Prach <jendap@google.com>"
 RUN cp -P /usr/include/cudnn.h /usr/local/cuda/include
 RUN cp -P /usr/lib/x86_64-linux-gnu/libcudnn* /usr/local/cuda/lib64
 
+# Installs TensorRT, which is not included in NVIDIA Docker containers.
+RUN apt-get update \
+        && apt-get install nvinfer-runtime-trt-repo-ubuntu1604-5.0.2-ga-cuda10.0 \
+        && apt-get update \
+        && apt-get install -y --no-install-recommends libnvinfer-dev=5.0.2-1+cuda10.0
+
 # Copy and run the install scripts.
 COPY install/*.sh /install/
 ARG DEBIAN_FRONTEND=noninteractive
-- 
GitLab


From f3b8bf9aca77b6264d791425593bd8cd02aa8843 Mon Sep 17 00:00:00 2001
From: Doe Hyun Yoon <dyoon@google.com>
Date: Tue, 8 Jan 2019 16:23:50 -0800
Subject: [PATCH 0385/2345] Add aggressive_shape_inference option to
 VirtualScheduler, and enable it in analytical_cost_estimator, but not in
 VirtualCluster::Run().

PiperOrigin-RevId: 228420860
---
 .../core/grappler/clusters/virtual_cluster.cc |  6 ++++-
 .../costs/analytical_cost_estimator.cc        |  6 +++--
 .../costs/analytical_cost_estimator.h         |  3 ++-
 .../core/grappler/costs/virtual_scheduler.cc  | 27 +++----------------
 .../core/grappler/costs/virtual_scheduler.h   | 12 +++------
 .../grappler/costs/virtual_scheduler_test.cc  | 10 ++++---
 6 files changed, 25 insertions(+), 39 deletions(-)

diff --git a/tensorflow/core/grappler/clusters/virtual_cluster.cc b/tensorflow/core/grappler/clusters/virtual_cluster.cc
index e80d6b54d9..118f74e8b0 100644
--- a/tensorflow/core/grappler/clusters/virtual_cluster.cc
+++ b/tensorflow/core/grappler/clusters/virtual_cluster.cc
@@ -73,7 +73,11 @@ Status VirtualCluster::Run(const GraphDef& graph,
   item.graph = graph;
   item.feed = feed;
   item.fetch = fetch;
-  VirtualScheduler scheduler(true, this, node_manager_.get());
+  // Note that we do not use aggressive shape inference to preserve unknown
+  // shapes from the input graph.
+  VirtualScheduler scheduler(/*use_static_shapes=*/true,
+                             /*use_aggressive_shape_inference=*/false, this,
+                             node_manager_.get());
   TF_RETURN_IF_ERROR(scheduler.Init(&item));
 
   if (metadata) {
diff --git a/tensorflow/core/grappler/costs/analytical_cost_estimator.cc b/tensorflow/core/grappler/costs/analytical_cost_estimator.cc
index eac6a0de3f..09cddad8ba 100644
--- a/tensorflow/core/grappler/costs/analytical_cost_estimator.cc
+++ b/tensorflow/core/grappler/costs/analytical_cost_estimator.cc
@@ -113,8 +113,10 @@ AnalyticalCostEstimator::AnalyticalCostEstimator(
       node_estimator_(std::move(node_estimator)),
       node_manager_(std::move(node_manager)),
       use_static_shapes_(use_static_shapes) {
-  scheduler_ = absl::make_unique<VirtualScheduler>(use_static_shapes_, cluster_,
-                                                   node_manager_.get());
+  // Use aggressive static shape inference to minimize unknown shapes.
+  scheduler_ = absl::make_unique<VirtualScheduler>(
+      use_static_shapes_,
+      /*use_aggressive_shape_inference=*/true, cluster_, node_manager_.get());
 }
 
 Status AnalyticalCostEstimator::Initialize(const GrapplerItem& item) {
diff --git a/tensorflow/core/grappler/costs/analytical_cost_estimator.h b/tensorflow/core/grappler/costs/analytical_cost_estimator.h
index ade61f4051..6275c62198 100644
--- a/tensorflow/core/grappler/costs/analytical_cost_estimator.h
+++ b/tensorflow/core/grappler/costs/analytical_cost_estimator.h
@@ -34,7 +34,8 @@ class Cluster;
 struct GrapplerItem;
 
 // Estimate the cost of running a Grappler item based on the theoretical
-// performance of the hardware that will run the model.
+// performance of the hardware that will run the model. Note that this
+// internally uses aggressive shape inference with static shape inference.
 class AnalyticalCostEstimator : public CostEstimator {
  public:
   // Does not take ownership of cluster.
diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.cc b/tensorflow/core/grappler/costs/virtual_scheduler.cc
index bb8425a5a1..0aac0348b5 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler.cc
+++ b/tensorflow/core/grappler/costs/virtual_scheduler.cc
@@ -310,29 +310,15 @@ ReadyNodeManager* VirtualScheduler::ReadyNodeManagerFactory(
   LOG(FATAL) << "Not a valid ready node manager: " << ready_node_manager;
 }
 
-VirtualScheduler::VirtualScheduler(const GrapplerItem* grappler_item,
-                                   const bool use_static_shapes,
-                                   Cluster* cluster,
-                                   ReadyNodeManager* ready_nodes)
-    : ready_nodes_(ready_nodes),
-      graph_costs_(Costs::ZeroCosts()),
-      graph_properties_(new GraphProperties(*grappler_item)),
-      cluster_(cluster),
-      grappler_item_(grappler_item),
-      use_static_shapes_(use_static_shapes),
-      placer_(cluster) {
-  graph_costs_.num_ops_total = 0;
-  initialized_ = false;
-  track_mem_usage_snapshot_ = VLOG_IS_ON(1);
-}
-
 VirtualScheduler::VirtualScheduler(const bool use_static_shapes,
+                                   const bool use_aggressive_shape_inference,
                                    Cluster* cluster,
                                    ReadyNodeManager* ready_nodes)
     : ready_nodes_(ready_nodes),
       graph_costs_(Costs::ZeroCosts()),
       cluster_(cluster),
       use_static_shapes_(use_static_shapes),
+      use_aggressive_shape_inference_(use_aggressive_shape_inference),
       placer_(cluster) {
   graph_costs_.num_ops_total = 0;
   initialized_ = false;
@@ -343,12 +329,6 @@ Status VirtualScheduler::Init(const GrapplerItem* item) {
   grappler_item_ = item;
   graph_properties_ = absl::make_unique<GraphProperties>(*item);
 
-  return Init();
-}
-
-// TODO(pcma): Merge with Init(const GrapplerItem* item) when this
-// deprecated API is deleted
-Status VirtualScheduler::Init() {
   initialized_ = false;
 
   // Clear all internal states so that the VirtualScheduler is reusable for
@@ -372,7 +352,8 @@ Status VirtualScheduler::Init() {
 
   // Construct graph properties.
   if (use_static_shapes_) {
-    TF_RETURN_IF_ERROR(graph_properties_->InferStatically(true));
+    TF_RETURN_IF_ERROR(graph_properties_->InferStatically(
+        true, use_aggressive_shape_inference_));
   } else {
     TF_RETURN_IF_ERROR(graph_properties_->InferDynamically(cluster_));
   }
diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.h b/tensorflow/core/grappler/costs/virtual_scheduler.h
index ffc5e9187c..d96371bcab 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler.h
+++ b/tensorflow/core/grappler/costs/virtual_scheduler.h
@@ -260,16 +260,9 @@ std::unique_ptr<ReadyNodeManager> ReadyNodeManagerFactory(
 // dependencies, device, etc.
 class VirtualScheduler {
  public:
-  // TODO(pcma): Modify power_analyzer.cc to use new API's.
-  // DEPRECATED
-  VirtualScheduler(const GrapplerItem* grappler_item,
-                   const bool use_static_shapes, Cluster* cluster,
-                   ReadyNodeManager* ready_nodes);
-  // DEPRECATED
-  Status Init();
-
   // Does not take ownership of cluster or ready_nodes.
-  VirtualScheduler(bool use_static_shapes, Cluster* cluster,
+  VirtualScheduler(const bool use_static_shapes,
+                   const bool use_aggressive_shape_inference, Cluster* cluster,
                    ReadyNodeManager* ready_nodes);
   // Initializes the scheduler for the specific grappler item.
   // Should be called immediately after the c'tor or when the scheduler will be
@@ -367,6 +360,7 @@ class VirtualScheduler {
   bool use_static_shapes_;
   bool initialized_;
   bool track_mem_usage_snapshot_;
+  const bool use_aggressive_shape_inference_;
 
   // Whether the input graph includes Switch nodes annotated with output slots
   // information.
diff --git a/tensorflow/core/grappler/costs/virtual_scheduler_test.cc b/tensorflow/core/grappler/costs/virtual_scheduler_test.cc
index 5e044b2037..128cb986f1 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler_test.cc
+++ b/tensorflow/core/grappler/costs/virtual_scheduler_test.cc
@@ -30,8 +30,11 @@ namespace grappler {
 // Class for testing virtual scheduler.
 class TestVirtualScheduler : public VirtualScheduler {
  public:
-  TestVirtualScheduler(const bool use_static_shapes, Cluster* cluster)
-      : VirtualScheduler(use_static_shapes, cluster, &ready_node_manager_) {
+  TestVirtualScheduler(const bool use_static_shapes,
+                       const bool use_aggressive_shape_inference,
+                       Cluster* cluster)
+      : VirtualScheduler(use_static_shapes, use_aggressive_shape_inference,
+                         cluster, &ready_node_manager_) {
     enable_mem_usage_tracking();
   }
 
@@ -68,7 +71,8 @@ class VirtualSchedulerTest : public ::testing::Test {
     devices[kCPU1] = cpu_device;
     cluster_ = absl::make_unique<VirtualCluster>(devices);
     scheduler_ = absl::make_unique<TestVirtualScheduler>(
-        /* use_static_shapes = */ true, cluster_.get());
+        /*use_static_shapes=*/true,
+        /*use_aggressive_shape_inference=*/true, cluster_.get());
   }
 
   NodeDef node1_, node2_, node3_, node4_, node5_, node6_;
-- 
GitLab


From 46137ebafbb56857622403115ba99051d896351d Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Tue, 8 Jan 2019 16:25:48 -0800
Subject: [PATCH 0386/2345] Remove disabled test (should live with pasta).

PiperOrigin-RevId: 228421179
---
 .../tools/compatibility/ast_edits_test.py     | 23 -------------------
 1 file changed, 23 deletions(-)

diff --git a/tensorflow/tools/compatibility/ast_edits_test.py b/tensorflow/tools/compatibility/ast_edits_test.py
index b2e9f67638..4accd8fe29 100644
--- a/tensorflow/tools/compatibility/ast_edits_test.py
+++ b/tensorflow/tools/compatibility/ast_edits_test.py
@@ -39,8 +39,6 @@ following new APIs:
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
-import ast
-import pasta
 import six
 from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test as test_lib
@@ -418,26 +416,5 @@ class TestAstEdits(test_util.TensorFlowTestCase):
       self.assertNotIn("not good", report)
 
 
-class ManualEditsTest(test_util.TensorFlowTestCase):
-
-  def disabled_test_arg_order(self):
-    """Tests that generated arg order is sane."""
-    text = "f(a)"
-    t = pasta.parse(text)
-    node = pasta.ast_utils.find_nodes_by_type(t, (ast.Call,))[0]
-    arg = ast.keyword(arg="b", value=ast.Num(n=0))
-    node.keywords.append(arg)
-
-    # This is only needed in Python3, and I think it's a bug (but maybe in ast).
-    arg.value.lineno = 0
-    arg.value.col_offset = 0
-
-    # pasta.dump should never put kwargs before args, even if the col_offset is
-    # messed up.
-    # This fails if run with python3, but works find for python2.
-    # In python3, the dump yields "f(b=0, a)".
-    self.assertEqual(pasta.dump(t), "f(a, b=0)")
-
-
 if __name__ == "__main__":
   test_lib.main()
-- 
GitLab


From 5439aec55231818d345767b5038410bf647844e2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 16:36:37 -0800
Subject: [PATCH 0387/2345] Internal change

PiperOrigin-RevId: 228422956
---
 .../g3doc/images/convert/sample_after.png     | Bin 185267 -> 166567 bytes
 .../g3doc/images/convert/sample_before.png    | Bin 155610 -> 136956 bytes
 2 files changed, 0 insertions(+), 0 deletions(-)

diff --git a/tensorflow/lite/g3doc/images/convert/sample_after.png b/tensorflow/lite/g3doc/images/convert/sample_after.png
index 6c451f97903f7f70a9f28dee8abf6daeb7ec5693..db09d0a6ca70695205833acfd2bd8ac6682cb065 100644
GIT binary patch
literal 166567
zcmeAS@N?(olHy`uVBq!ia0y~yU~Xh!U`yj*VqjpHxzUV`fq};))7d$|C9}97C$W-&
z!J~6(czH<hck%ywCbqN~2n$RQ{3hfolESg6eSv~>OpA(xLdy@2fWqi(?WpY*1y=$^
zH{P;h(9~LVE%M0P(mw&+T|wPZ-yeK;XWI6yyl(&J-Ou-Yf4B4g`M-D0?|nYE{2pu1
zq$65Wn@rfcre>*Gs>Mv1IaT-39acuMiVY2WBZ5|Od3$*=G|YQ`|Ls#z0fvH$-fMz1
z|5g5rKkK5UJ(a;ls39hE<q`>nZ;V^!yBuA?7^co}yfbU6RDv|KfYHpzn+&%(8w}!K
z|7I3=mi3o~!AZ^3k}2UiLsiYj87J9!L>XLCC)JoH>|kiv9;6k{)Yi-3q{5n{neao-
zVgI8?LYfIr*beA^*SGmu|M<+?ry|{-t7~)h85s6@RIc<=ocE``?{R(fMyJcchXon-
zH+1^4DX?_Zx~hIW$?$`rFleT=kzj+SL(C*^#<}(T9fW5uJG_6|6t@&+19p#nAu^Ku
z2c#5ar*uu^+*RQ*&&G<c!O&sZsjG!6Z^Z<inxgXh$4SpkmsU)Xi$B9)<<q)ML)Yi{
z&V3GI*AL$eh?_8L;Z@IV(@tJ?XEI<pF?*@2=0~eDi$67WOth77n>>f@0Ox`U$6eI3
z4W|SbX8Om6BsALOEji~OAim^SR`JS0p~4U@2JQx}DO1zbzPy-nN@?+y+Q7}Jr%dM7
zWv%p*oww$2exT;0l8RM<%Q`&S9xw#xE=|?6-5Hyu7HVEErBxcVDok2eM}*)1Qq}_7
z2@M=BPAt4GCtNwX6!TxMdEMesF2rjn@x+-;KX!?_GINqafK!jil*i>7iq4t&aTZ4s
z6r3fe{J0=}_1@kI?1mCk9NU6K9@<HWD9dr3Sis0EIc4wFOCO~rPW%syZ4%N5U}F1J
zf6T?r*04;(dXntvzO>z!!%8@0UDPiqmT1U2{l4(8#G!wkf5o-hYn9hxucd}bzy4M6
zGv}G{Q>7jUX_H+mPfYlI{auFmt@dwA55G*XYT;|>Fk8O%fh+&Azb_4{BxGHFUt+Ak
zyz1rKAD4AXV`B34PQE?N`*c_Bx&s#uU)Vm~mF?;Mw1rov-nL%bac}bZ-yJ_r|5>8p
z_~*IDtL!ZO(k3gXxr?M$T6L^A9AaBK&C28I-=}wf@bY}BUajiPrrNcj;g9^+BfDnt
zq-cCy_wn6_y7ku0QD&aJ41%hwPWIUBchKbuUMXS}I49`R{`n`4S1+)iFmtNzrMjqj
zvzr+lX3YO5GW!AxO9+F@zvVM-2wD`#3+(wLQe4CKL7bUAT2*S9kMv^323ENqg%;JE
zhaBb=2ifa5oHQ00$aH$RiXC%oViIXS=;Cu!=giR>onu^U4zlGOE34_ZUE!?U_aJ?a
zqqg2-wp$naj=wnMy+@g&$yq{x>CmJN%5ymV54C?#ck0kkYV=Us)FGoZKR`$-N$Vr4
z;rs}%HHv0JSC8}`VSL0<*k$1#qyA1n_K5K#UC)U%K_0gz=sjV$G@nQL@yU=PotgfA
z3a3w6K5;4%-Pw7^^V%f$6UI+cH}%%|>|5aQGWgjP+o!HSg9SsBruse&<T0JQYJt{D
z-z$2n8m$~-m)u(6_lo!IGS)2}-xf8$h_LFNc0u^X%P&k-ynE-aTiSn7{H1k?_}_`C
z2R5ZuZeofoT;mgFm#vhow|dgnsaxll&MBRF^<34-w9RL4u6v{SCZbGi_XOM4y$yT&
z_RhY0#P0mQ<8=og7U)>co0H(D<bEcw_*qd{aaj>xv0lL~%k^Kse0}q!=F6h5o4!t|
z+Wb}PYuO9gOS>0*cX%&*ea7oEyie4Mmfkse$Mufx9kzFtdDh=$&$m23#r<sQC$7rP
z6{{<DR~)Z2uRQ<t>sQ$?%wODJp8lHtYW=1Ej{leZll^O66TFYzzRA9A{X_jH`H#&%
zuKzq=T0&xnM2l34^a@@vsSs%y(;c}VZUrRVxNmSSAj{(LhEp4^eQ0_pth{dG=D-;j
zEfyCo_FBBx?Rk$?@6Ka-#{`chJ$`vC^cZ{3da3;ef-`tbJWVvsA`><pelq2xb8z3v
z?#wAGuaxw?OuM{z*>{#H2Lc5YCvfZ&$Pttg-@<*<DLiq`h5(EBAA2{r-bhW$GP|L3
zH)m4h)SHf*&5BRmIcjq*CUM`1n*N`npSRcM{A&3+@$1a%>EZLaYvUrnY~|QGAu=T5
z&9)smAFl<aNTx=n>^!pR+^1=xxla>KPh~B-TDAId*xcII+U8$gzrw!Uz5G2q`_iq$
zRyV7<zNWq2d`&+mCpRf4#B6nL->s5cvu?ND61%0h-8SWRqWT8zjgxQdZnw_)?vbL#
zC9LVP+U2-Q?6+&*Hhz=(-TO`V+k2T9*-J8M)^EyoJUvnTrTk2B%)1+JZahtSZgMDd
zRdDa+_2**J>z^|{pL$MxPW;XtyHa-4JPmsO>B7}RrsuZa*?R2jve}Kd4YqUU&Ay|z
zbMxK5cf0Rg-?_bNg4{kiR=H5Q=Qf`H_WnDcTb=*>H{vhTFU{Y%ugb4>zZZQL`YiNu
z>GJ)R|6BfT{(b#xz0rY8hi?t4%zivSUc7z1_QSnLwaE)!9(c%{&YvUk<IM5H?vH|>
z7H3X4xZs7OG22!CDytjC8*)DUI(WT?b)Rz0{!h8z{ys22(*N=H)9Z#uGCj^s*y_CQ
z7x&-Ef5d;x=Sg5~VL!u>!BZjpLVN|^9pNAD6T$-mFZ3;FcaCj8)s`n&rGG;)LS~QQ
z4qX?KNy49UJ#HKPPJEO&yX`J-DbLilJl<M?XL4Z@+l=3M?kKkq-e><O)Ft7_mX2i}
zT1IM{ls2iJGSSk_^1bplqy5F77qechlej-a)6jS0nj1fEsBR2@obyPf@M>Y~;dY++
zW`81V?Cn(ReE-~K>1R<}n8;$8^rItHIj^roZlz4*^o}Dtj-+;OpA<7$P35mim#tQ2
z$ki3UPclzlt~TF8a7mB<&556kOl?X7U-g=7H}Q=s-WB+3#+O%L!X+n4dK=#|Vo&iq
zZ8JG_s_OYw=bxo4PtQMFXK0o6rR(C`<eKAWWQ$YRyuRU??BBatYPm#t!Jdm1AAjt%
zTr;O<j;!VTqn(d@ABpcQ+?idxTSaiv)JexL1-vZEY}bj{7V=F&-y%Qr-r+wF{-pkt
zWj)NL#clOe@RRAM<*gfArMcooRmF0(UxlS;l!?C<eJ&y&FiWd@%5kq8Z?&m&!Yfxj
zTXjo~#V^8eVRvWxwrN63lf7SGb=vAx8}@kF&!FPvI!o59dh_+jm(FVSnLa^&R@OI@
zKkYfme`@jR{j;6?C;DX0%d#)McE$Sgp=G&q{Cv*Oy=QZ{@Yo#5x5mk7dGn9X4mEi_
z{pP*R)6f0;+})!u_3Xx(nU>G*O*f3ccE)th=`(wOg<c4k-nVT3wEDY7`4{$F{cIQJ
zq3w76`@DF||9_cIHeJ5Y^C#`9`CabMV%u~+MQ^Dytln7kajNR!X*uy(o3_-v+?%RA
z?W&%*?)r#-&$~Wq`-SqYyR%`^_Mpw1azhWT`gPTBwfOqF{mT;0K3(-;)!pp1*Uv@&
z&YD|txAp77*X6kzZW`1dzMcQ{e%x(|%^S-mpI&zA?3=Txv)|`?-Y<G&{r~-{?NPaN
z_kTTX_H^5}w<mA+hw`sncQ0{oav*Qy;*~FNSf-`F{yy>k_4l7{Z7tmX^W9JT2kdA0
z`|Hm9{CH~K58JcVf4{%G^?XnM+q#4Qx}p}973GG<I>tuVhW-7?e3;+FoW=Cb|CRs!
zZ%e(EUd=a4Hm0PbJmcKLcZZ+0$D6F2W!3*ge&=<YeLs&~JhOOuU$51*BAb6dPR?Ik
zZ&#)C=E%dz2i2whS?5_5-1=W~S8~7c*7Vx*@BSbCKK-w5o$l7>ZF5%V^Je5)*nj!6
zko&Nn%ub2QkI%ger=N?FtNi(A>K5(m>-u)yD)X&+`>^!r*X8@C{r&as*qPaDW{Ym`
z$q&7^ZQtIP@$dfMy8G;U*$2NTb2pa<zq_|XedqTd+b`X}xBdLi{DS?T8t*p0YPaK;
zwq9f%CEr_j<zK|tAF<Ex-1n`E|9#}`$$j=t|1EA$y?OQDl*(Iwa}E}tet!F(%%Akn
z-Alx`$BXT+`M&DW)~B!E=Kr}rr(*WUZwGgGzPEfIUR&`0!Uw~DkI%k)^e%7e3_)fF
z4Th9Bxeqzg2@_a3q8RUGyv<`{P}uTYWx-$epXc|L)iX2vV7%?rxMik;lkkRpoCWhZ
zKg@1WVOY6Pft~l&-jc9c<|o`wA5CYt_WIqs=VlDc8GjroDc>J`{qEmHZiiU*S!x;O
zP7b@9>#P|VqHfQBQ2g6G<6fcGyH?jp3=9mM1s;*b3=CqbAk63)r1F7*fm0(hB%&n3
z*T*V3KUXg?B|j-uuOhdA0R(L9D+&^mvr|hHl2X$%^K6yg@7}MZkeOnu6mIHk;9KCF
znvv;IRg@ZB<rk7%m7Q#vqGWHUU{hgLkz0_PT9T+xk(-lOY*k^a1UA<yuh<GCtgoa1
zR#K8}tK<l>P$9xMK*2e`C{@8!&rCPj(8N^1+)~fb%-F=zQb)naz|cb9z(U{9Sl7_Z
z%EZ9R$VdSSl<X92ic-?7f?V97b`_<h*(zm}loVL$>z9|8>y;bp<rk&v8(Lag>Khp8
z8yV>qrKIT=SLT%@R_NvxE5l51Ni9w;$}A|!%+FH*nV6WAUs__Tqy#m#BDcWT7jAG~
zF*Kl(^K<np5)1SV^$hfLb3y(Bd$_p7wX6ic>f(^ff>iyW)Z+ZoqU2Q9vedj1Wte5|
zp1uKa-5^h-XXX}wy-|@{0JkJ18Lp<Fv?vEGpOUPfnv<HFnpaY+Z>VPoS6h)=09TCF
z@i41B3rdnrDsl^4D-iC_EGS6^n_gU!qK~9N-^jqgLf61Z*T@oXQ9({=F|tji#i<}6
zu=bpkG<}c<K!F9bFSQ~Q5nm`S1Id8H6{fTxBfliSI3pj2PvFvUO$ENbRxbI;rJ&&O
zbb+P8l>Fq(6f3xDt7MC$q_h;nG+mQ414~`=WRnzKOJnmyUBgs^WK+{bi?pPa6p#}@
zrg`QSmn7yTr-DqY$Su&z%uKN|GBGnZGcd8xO*A($(=|ynG1Rq4GdI;Wu{2LLO-eIJ
zGfXmu>Gv<nOwY_q%t3Y)$efhS6suHIV*`uilqB8MM9UOi6N_YX-9+<L3*F>2GfTr{
zLqk&&LrWzEP|~n+4Dhs7GSV}EaszS_OVaX-a&47-6O$GEgQ3D9nYpRKC5fPzH#9Z2
zurM|?wKO#~H#M;|fvN~gEh^5;&jXodXrO1Nq>z$q<(6NRn^<D2l$o2Do~mDvmkt&M
z8ILAfky~KpT$Gwvl3x^(pPyr^1ag{!k)ELe*tZb5RF}k(R9ht@14Bz)14CT{ix5LI
zD<exQV@q8FQ!4|gQs4ZP%(O~uN{tPzOpH;Krj{h8B$gx+R18W!Rsoq6sW}lYnYpQX
zP>)(z7$_+q<RX#e3@t$JK_m?4{G9wE-~5zRTP0&91%$Y#vvWXEep+Tus;!a>WGn_$
zh=5B7E04_LlKi4dXt5NWS_oz3WGa9Q1*^nlP!VIRRGe6rnxX_1OwLHmOHWO)Rnmuv
zWMWBa#))QW29|~<x`}3Hrn)967RkCvCP{|6DJEt{mL_S*sfkHuF#W~(X(i=}MX8SI
zsd*)~O75At1>m?*(14^7O_;*+j8sqpFfg>xH8jvQFbgp-w=%S}GBwpTFb74DK2aXP
z=3L@31|(GAu0TpAC~*%_hUP0s-4OsQ7ZjYGL3M>C*pHccpuz&~O{5S8`A8pB#6w~S
zR3!QOB1Ifh31{V#pPUGaFK|LnO@W4mQ(`*U5d12!$p&ZSm*){-3QQfc8IDCI<@rU~
z{zd*tS*gh-cuc`miEK_#YI<gINl~RIKJTC?0+rxao}dCDxU`@kCo{DOYI$&KAvhqg
zDnJN=LoXk<CWJg#k8^%OWl?5&Mu~rU9(JoRRDkvQWG1KP6=SytNe&$MNY%Zaf(=+G
z$i)rB1h-5;trR;2MDr&#FU3}=NXgz#^}QH715?RUPZ!6KiaBrYmPZ6Xzs>&P{Yg&`
zryE?0n7B5%+*z@MU8{rn`@LIPvTOHkyR~qweRj0_+HHMVukUfobTGXOT>5B5hj7de
zN0&y2YbPuJd)!#Cz(L~s^H-&&k9%g%P=9WBKE*_emqW2dz)5S}_Wb){D}z$MyttTs
zeVu8Ej*GvB00)mOGf38h^HQoc&*6Yu!DSqhZH6U0Edov)imDpMw-;_%{${~$MkkOV
zLAP4k_!>YmOHyXeo_&9J`Fg*(R$*ab`~Uq~z3{Li!_x&IeJdy2+gJPh!NKM;%l+rs
zFbc@a|G!`VKXsqR<dZQmF<$m&Obrh!?_Dqi8Pp=O`PSBK`>HQ5_Rel?Yy0)<*EA;^
z74cX6UL1-o6S!8m^-8tgJah2i!X?UzKNrbN0qGIFaAKnJ{k@Eb4Zgm<zCL<;-peD-
zMGIka4UEh!hst}aA6C}g*pRqXjQ?b7Q2@yKAr7IdLMrp!&mU}NpYCU%qJPSfQ?W%L
z$dFgkXvu=@5^b4RSB0*P+UjL2H%S)ilNnXt-|0^D_*C01^@z6*YL1W5OdSOUh5Oqh
zrpa0qBxoLY-n1BM`?U)Toj=#^Eq;D3baj~S(*%#QDa}Go9EwX0efs>_*xda5Z!6Oy
z(lSufH}v%Mu<=Mtc<s6|;?C~!eCHPDO#yshpLglJfAJ%3Z`I6Mv!qlcCUrusPLZ`L
z*|2qMs)X9-spV&Gx_I$IjXJ@?bnwB!=E`~3ZB0#0zkdI2p5ws+v-`B2)xP@w|B^P|
zXgDY!3-$Iv)9h<!PM-AqzGK17Sst1&AH+XP@sYJIGs#hLanE?c0FL&yH#asW|NHY(
z=D4KQgrsF5T8uMh&b+!h{QQNDBKvohzGh=(O_eVX{bo2bwgeQUPHcQKJJQd~xhH2H
z&DOe_u~p~rl=Uua4c;wX%Ix9cp%JfbA84xqN_R!vnOCQ*E!Y|5YWh6w%)90KJhHA6
zL-)*<0Y#WV(6vBtXb3oYNkjapxMa)V5}JMC0wX9*E@3jv%(%67S<U&xwF{Xw<&31a
zbrsGFzr3|mzp}D(&V;r1uPv#K4m@SQ-fXVkMByot(73tb;nuVERc^9Kz3Im<#m>ir
zcv@tS-{xO2|9y0nUhLl==WaL8elk5?<nT-h>F%Ayy4lgM<`o)lJX&yOQ}3-upui6D
znZY*gY!1VWFE36@SWQb>X!+%H^Tp!(`vP9g%MxC<H)gZm$0M$;lY$O-9o;H@vu59x
zTR*-XEl{(wGu1Kp7{jx^I*i9%=d}jAYKwrA$47xO{s|ix4%{#?FI}l&$yWdT?ak_4
z?YEBX&V1glnjiJ6=GKe!hlabvm*yUyRVj8@ezj%2O=awkUyu1i{6eO@X9vY9Pq(4r
z>j%r~Ig~&D`^zisXLxGW<<!kqhFaV8z5E_A7#K{tR;oYG_+$Uo%%jCiHue94<PnQ2
z@+oO&vxMdyIOTVusq}q@&h5!+=dZ2O*(`W%%c-hU>X}J#ObM}fCI52!?`#pdJA2La
z{wwRGn5RDb`1H$Z{!R1UZ)iE3aX+d2B<O#c)5Y0a{vNlutM~P`*Q*=cE?IA9Pw!fs
zzQcx*>(zGCvhC?tGG|Ta7Zv|^`u4rQeUqOx`-j?in{GB?S6KCy?G6jWF}FKAmwFug
zkd+aC$8Ud)_4_)e>t;dx(JM>?mTLXCUgx=jhxh%@1(1Ay^uQK(Z?>3Evm)<2*uQpe
zuVh3-Psa3ZR@nvH7kth5o!DzU|C8r+IsfPbhGt^(&zWsbWBl+byKu5tM)uFp-$he4
zMm}r%YvQjoul~FB^9y&b?BAT}%&(k0t@`4Lk`=qJ@GzK^o|Ri)9sTyU;IU~*a#{yE
z7=I`-3)TH_`e%_`Vi$b6=-YO&bCK*%?L7nSDpp<Gz;yO+%TZ}=tCi1W>kX2VQj!-{
zujuMzO1P4~EBF1goNc}pPrI*Yi>amaMI}jv%(TCGM(?$Wgq@$L?6;ba$GD7o9RdTQ
zZ9s8pH*4km*4aJ&eSV!6m$k)Rz4}Jd^tIZlZJlN{7n`3RFF7f;e$V%cTTFZm^7DUO
zRWdzLtRGjmH6Z&{x_SPTD>|x~fA_F2-d|Cw%h+(?-0o+843qBP3)6iqExFm|z^z^0
zEt$JEbEt%cKJPK#CQ^L<yq%QktA{UkYI@2X4%<5KPh^_(u9N2Xqbz1;3hN#1mHPDH
zdb3nRqOa<?cWUqS+{BL698^F1H@zs*bLEEL&&76Kk$m~mT&|=2Z^-8IV#Z7N*ZhBX
zaD^g&{oU1U5*ycEeQy7{x|?fOc>L$ynR918J9%?QptD@hKbLYPW&ilyJ1hM{IHBcl
zzpC6FSB@ofEmyD>WR_<cF&G?*nzq-rD5pF#_R-uFhMpaLmEOgFyT8uf*zoGnzY@!Q
ztK0?-5Bn`2&c2L(@HlvyUSI*kfg9$D+v>j;oQmU1V31oieSUBC7Ml;bPg+~{2Ck7{
znDDEM{lI<C`e|GYVOPI~PBPo8bN39xlKIw$7<cS@{Pd&+gH1^1=XTqFc~d<Ps!u&t
z)bl=FN9Jgp?h@O!sQ+8Eiq9?kvSEMly|S$@U$Hn;etW;<@^{7$*O}+9u{g`jvm{J7
zjf>&tmOvrHigzbZzYi;9_;F>*();@=4>Bgm?hbnyv{&N4<tf&zZuJ%crxV8$@950F
z`e5q!xc56b8EXE#exPwVk(WW@_ZOiz`)BT5XleRkzSjLRVFtB*wej&c7#yBm?{Tau
zIuXe+YqpgXv%$NM_U(GRq}&ZxOK0@25-Q*PMf;*wYW+%y7#|O-2Jh<lcqR$qhs`XV
z-JO5#Z)Qli8<3;N#NfC1eTJrJs>-Ban$xbB%YBfQm5}h1-l=}yxbnQG{{FK(^R|9n
zWq4Kl^Gdac4{tv>dvbX`=kGuM93MRVIe(kK_TRh3RCVRvnW_oTw2D{czR6>4*e$>E
z)SU+Z+Ttghj?Q?-&=bAI*Jjsm=l_hSR3R1Lr$ClirX@)qHf;a(irr0##bnO~$M0gv
z&ob{_vU%wgCe^;%QIj+F?Apl2<l>oM-71reKUY1gs5APyQEcm*R-K9!mz4Ea8%yo2
z?42dprO+Z3<goCp^_32P-+xa(r!cI$yGHz+>Bh4!a!)?H7jS8X>bLvM2kzG$V{~$m
zozL=n*L&;Cun#qtmM&l2XBc{B!lVV21<|uNhD=(rF4t~8_xxE^%IDVfGBdB-QYx-o
zST}o4XGmX;UfRKHUlJPk)W@*8S%RxgJEr8Mds`V9;!d8j`5G+pQRevT<t9nKyTAMH
z+Mt+SZD;!PitlH}uaXfnN<Vv4zfoqJ)X&cBu&XYw?C8p=`HMfFK0YIg^+>?!N3$iT
z<z2J+u2uNeX5Zs2tMYCh`+FodXvx><lFkoq99@+9VK#Su>s;GbhV%Bmwtr>J-dksy
z-8s$vYWMYf|7WILu<<|JWq4or$8jmuBgYuD*GO#sB`yc4sef|t*`8e!Y4`8W;`L$A
zD)t_1xn*o+&~RYi+Rnert{ji#{WNQWbJxDFsyat}*}_XVKR4~4)c;ESTM7f4+a$KU
z{KfgtEM~3l_gs5@<?*=Z!VH%Wl$|vXGTg#BeO}GZ7G@u-XLlOvMgAYY^@{tLcZBlk
z&egtbs@*zgE0*lJyUi<f#$>s@v%YfuUU#IkG33@2!>=Fr=BzJds8}(De{#vy1*})g
zvmU?So6S?_>XJMO)X4etFs1nxSHqQH_oDrUwb4aF3<eXTzI;FbQA~cSh$Mf*#uBrQ
z!rIFHPoiWzUj3fAv2<IO<FdxL*B`ImR(95$!Ld4ir3Awzzjqg&mhD-mobk;?-}hwp
z0rlhi_^c<3RX_f^P`BT`c2>!kf9&&3rf>1)|H__uMep}(rQce+zMF1LzdV_d;aK0M
zH`*SC-;|OWF6{HvIMb;m1WmFFbACl$-TcI~|6bdj^MA$W|9Vz1<=w&!6Z6+GKI=O0
zcvqIaoTjE?`q%YpElbs2gvqVS`h5BPF}*YfmUVYm{r%Y661w-^Hg)6V({}EY4g{^T
z_bwG@Fv!e{Im~k6n*5ZNp<kz;3^KfQ{!PX1zvr)O|Et*jydbRU*S*P10;wVOML)yC
zvijRzv|Zwsxt*pPbZSSW$vQ>{VOc%@ikUKn^1DQru<@R~6L@9+YQY1_s+qwY3<g(Q
zcKy2Kb-{QFq>b{)G3VBeGcWa7DyF(6oK<*VWxxKhWdo~3$ku<~HZR|0*;(?4U(Nna
zc4+h&`~6d{`kY^`_SP!hY^g1?!J}JO)7D=sn$wWVsBvzcf8Rx}-Da14EwolV)0*U{
z%g(SiCcaxM)B57M;uRMz`AmOnl(c<T{o4bZC+rA)x$ETf-s&@jZzdLM^<CBqzNFL?
zmuvOw-DD%a+%;BLFTOo$Wiq)kZmRq9NT<&Js*T^a9B%o><gvPDYryUw?ms`j3cG)1
zO4mX4`;|K@<QcAhi@#T~X2S+y?$}+o{if`X+1vltYHL-wIKwCB`cuLc3^}{Sl<rQb
z>3$_z*CBNH^VE`74JQu8mM5kK4-VK?f4lSMPFw0dOOLjdp}A+79<SMW@6PP2lQ;L*
zPd^)z8+_j8w_MGq-s8c~)1QU<yw2QepF45W*Z*;;C2Us(OC-XJQ_>UNbk)x9^|bqO
z{q^7Q-sS7oJWHKgd-}`f)$wa=d5;}QdiQ?X)>6?W0mtWg{J6g4*M?_22bBMw{q-~7
z@4k8bj=QJK{@bgUet*+4p_+AW$c2ZiqmpMmx#e11egAR%r<v}a4@{Tb(unVjyw<*G
zSM8&t`t|8cC-wB+{GzWbeJkC}`SYo8&)d`X+1ASUbRPX9I<3r-?fmjL|K5e(<9cTE
z>`u?kzh7LPoSd$%3cbI(@TMcAoZ2K}#{RdWr*p^L<*!Oy>^eD;mA<S|7hvw%{_4f;
z){`MD>v&t2@ralAdOs-jcTE#IySCoa@`2frf4Ae-e0q2~-n=mMZHtj)@)3uWoRsHz
zx6ZKgJ3V@q_jkkOXY!rRvo9QD+B0eQUwQTO`P|3cUcA&0XbwNJ`g7Pyu~^Hqjw^NN
zfm-Hm2bOF3yNi68_UzaGpZh}@-rmh#HOK$!=ZnU?Iq6@L7J5bfiLL4pE6-2QQ)Azd
z)4o#eZL|NZdX>WRkiz$On)<KD^#A`MJhj|JtrKLN=yYe>jS@UPKEL~gPHl=4+p%rq
zoXy7$O{(7gzu{ri>MJYcAbr&EJG@IbO%LIjSRMX5G<N3}k+qY=XUyEFQ?Vk5Z*P5E
zRge=*rMUdpq;Hp=v~%v-_<haBl_xIwEb#RFt^elblEC%fVhjo@rb=9G26rNzy7$lc
zarmf<!-Ig?j6c4(<T`aNOjd&SB#JH^vo$H<P~qNsR2Y=*CR~D6795(n5G5@GA*5zf
z!T=f+&@BE}^&{edL0eyw$SH6~?@&9m^&I);?OR5j?%I?W4jzIM(qLfKwJ5P-o;hS7
zLQ`9W%eACrOW11D5{090US3{a`}>=$RmltS_?pBQ5nb_XqqeRJU7hwqqpR%|xKlsL
zRcT^HWu>H5*Q4_P|7sbot`5(?w`b?VWJQKvPbH9h6qoix`qTnJw;+8V4#lOtqe7rw
zAeBSH7dnpM)HUC0sh5YxiJD)Jj&d_-YHIq=w>w)Cs5LbzGE!2(g2}+<ytEY?IPoTh
zt&iI~+dO|?`R3r|ep~bJ?-O~XxD(nZ(NfqL5i{fLnKNg+tb``P#&H;XrA%Mll{Vmc
z{OZ-M9fglCb+$iof|NsA3mlu-I9cAGDz2C(Z<Z6W@~AU>z-YqSwYlwlviJ6$ouKIK
zr7AXwA5!TC8SY&?YjWa@Wxlg_?btCT%|^u<Ch_(4b$=D1r@zIJ#&TwyIeT_@`TKeA
zqj?VVNSUmd@>uaEWT;{SSN4kw3xDnd4T?#bW=$zFfSZ)~^~FVJwaHIw_b#|IWr`Hk
z{6x>Cm-f~EepVHzBJ}nB{rd_ZC-^{e@Diod){K+q&!4|^=~Blg#hH+V>*0EBskgYc
z_Ubq51DtCj9iV<TS{N{+mSazd*4wMAr!ziQw1S$^bUN<i&E@{`$;rvgY|bT6Ggz<h
zD$TaGw!WJjHGgxDfRt53k>~2GcGcf{o>w}v2PKBCjSAJ8DrLnLw69R+`=6i1TFX_B
zA3NrzBPMOd6;x(h{jKM*g|qv_rSAQ5tHal`J$U%=<ByNY&h3+zdQH`ux;bvzLirbq
zs(yW0xqp9sNXV3h%qvfvIdk^=``gzyCc8VcJDplx{r%m~($`|r!pScjv^K4}cCEds
zIeJS*;tL0lnTvx~{`&S-T1Y7Ag~KU!Ztm_48#X*UJ6qZcRHkKIytAY5@b&ZWUrtuP
z`s%yb>|0V+bGkY&YsCHA-u^Ijwt2psO~r<VhZUcmU+O)5*U#VktG=Gv=2)*BDeBxF
zY?vAHY5MhV9W7_(-Q8XN<wfF)0uRYW7c)SESo!z&P1B7w%e!;q!ow#MX3WS)OAEWe
z^@&Y!`swZY_w$mHihg~0`RnUzX{$Lwu9g{sb1aLOrLT|ETv^m#r=4xK@XiVrRr}p_
zrOEZFpSt7g|CS!>k!<5@pQ7&GC-d_9{{2BmO(pi6JGOcG!ow?1+_TyB<NmXQv%W03
zx;ni6(@AxZN=}>e{W6wCudl8B`|IoN9fgY>91i>4)3>pyd2pa{+O%ou=jWA{mYSAW
zEN$N>)4DbN{5-wbT}w1~s4XrJW{SFWChJQ6$D4*Tx4xbpUuRkSt7KQOQtOt)!)^0y
ztFsjHto|veE>JzYDC~2Z@ZlrRzRj=u^^)KIPr{1=kC_IUmzJ!J-d^_lTJGIlrBfQj
zg08QNHP5~E<<(VfdHMP7{c@64b1pf5UC{dJ_xt_+^X=}QII`>Gi@j;4(o$A)PC3n;
zS5kWUfjPf2C?G38r%C;FxR-lxPo@2z56xE&?fZDcL0H<{xji^?MNoWP<bC^}kymyU
zJ`P(Srz>(P;QFc8r}g*8Y)bL`{q1dYQ<I8N=fcC9>DzK|e|x!n{<W3C&)?kKe06oW
zd5Oi+#-a<~A5_*oJ2MkB4qX`fd|7hylCZ5jvVl(ztrWdp-q*ih@Xor}-OTKK5fKp*
zR!Tu)g)-)umzM1R_v`eQUFnkz8<UJ*t&Uw~a_UO<Z8w#Bwh!OG-(USbZ>CS$i;fP)
zKkw`R&$cel>lV{(YHogM{UPz+zu)gOFE6XCr~pN{*VL~2)8`glC=Xm2ve>=<-p1ti
zoz9!p7MFMSX#W#m#PewDt=k8U<>P<sj@w(caN)w4!g~Z4O`bN*tm4Ci2M-dopY!#$
z<qLY9;gJnYUGnsbIn#2R{hKbIKK=T`!^79EU2E&>n6m1A-S6CEJ(9fAW;R7nI(F{d
zx$v;&@o9RoR@L9$sQJ#CV^ewQ%$lbEvjX{|7Cd?K<lf%u`8Ji8ZXMo#aKhI$9m`gm
zEMZrleyru}+Q7|e5g99%FJHd!uwv@b)#2-_zP$MOe*gblW>QZq6P7mUIv<YJUO3~z
zPJTb#(vl)8DY>}iezMN(&R!?u>;IZ29%3;!-~QplhgG4grL5)@y+1BrAG0CB@%6Q}
zhYugt{<de~l;&seZ*ES%yQ{Rjt7}uX;kLX3>w6kZOI%pauY0#=WBI0)oo^mAGQYjO
z-Mqx&$Ok!_iU+S>OA8AJ@4ow7@9r~0p5zxD9FO$=)c$@D9=bZrRz~R3nn+`i1MY0g
zz3tW`5x6>RZSwKH&`EJUOB0@*nW-JV?#-p8-US5(ta99rmKk<=cXwUAdj0#!$H&jb
zrdj{2Nqte^G405V$Je*@iF{b48@-K3#^S<-hXT&Kik^1GZ_nf8<jk`bzmjEo@xsFi
zIv+0{475Jp*?VJe^|v3te@k1*1T}qofB*je`u|y1SIJtJtqIeW(hr=o%xC7N#KUb3
z4Gnv%zlU8}Soh}6&SG9!=bJ_E?pV%0H#dH-*P83~p=+;|Bx*<ZthNX(eEUhss^a;!
zUNJGTd-v|O^>s+NMMTVq-=3$r@+EIu-in(gGZr3Jv^<e>M`!LWyWQLBe_!i0&Ayg%
zX-VfoWhGT58=F1;^X;Zho3?P_!Yf}F-T3+Q=iS}q`WhM<W@g*`=i7;Hn%iD<L8hsx
ziHV76N|uS##=G0<8WtW_{3$RyboH~FudmKs-2UyYho|SuTU)bRb!I-1HOswaQt=^S
zt;4O9Gxt{I&E=KlJK28a(79cC+keb<Zz{_d<zTr|x$Kc;^|zdUIol;cnI0Y;?d|Og
z53g*HXnT2a@$s*(uQxX}&9kq+w>h14&W9OA7j(44)<_tq@n~<%GRoUOH8|V3-I+^e
zQdCI(mH%fCbE`Xlzqhk^dHDJ`9dFq(?QT$!^ZC5JxzXx}Ur#y~Tdzvc-rbWDnk)UK
zR<>G8r{eqO^z(oI{>?o0iR<yDrQZAY?PF+aZeBTW_Y)45rntRTv3n{E4Gat{EhE3r
z3|F}#FTzkU@BgpY>&11WQeuv-Jb9z`SZJ2CmC~oWIaan=$9J^MR+P0Wxv{_A{w!P2
z!_2L>G=rD@`T2bQvTa{y_3(<!zWAcO$YHV1&-~LLu5Vmk_3h2h;^*hOY)@>T)X~?M
zcYj}PRMf3~wZCs|%}#$&ptWzymMtJh>?lY)H^=hxv$Mt}7Dqda0#;mowK4fPC?($6
zm-+bO?%ouW^)5RPW^dijSM&49{nB~!<}F;f&@}s+Nr}snB}<ks=a;oA`Ty^4Sm`ZO
zsg1pAjJoby=2%soe<J?>ZPvX#l^z}(XYU5DJ9^~EmfYKBhK7aL*TvS=)wS_0-{j%v
z_wD=l{Z(IIEpqMN_iNQ7o~i5JSY~*6dEMVv+b?S!wsl?nVMXZ(<z4Rj_3!5YFg5#g
zd{$Z6!^FfZx+|Tk{{AZMm$TK2*^zL2!~W+D_N-3I&h9Q<ZO?zsxF9b#@890O&BYZF
zGbT@-yii$5UP(`Ho$qWjP-(xXvUt8-Esw0<(@8UCym<Y(`t`N7p{v7czuio4<6B-4
z@=E5*&(F{A?ke3~_BQJ5v&FVm|J^1xOpEu|&)+X;arDd?lOGQbZp*zbWhE0dN6xP1
z#q;Oo&(F=hmAib$kMqkFGoFUu`DwbnIXyBuTHdl|N9N^aZG7z_XS>DqZ*9w!mX_X~
zc6Qdo!|l>mb56bZ`T2S5?y|l0|NlKZJKMcSBJo8*m*dm3j`{cZ-MxDKdtpq>+;jJ@
zy!^9u;bBGj*H`Bz2i|t$T5xY$ZuI|ue|cs3H08qA$9;W$P1nxu-rD2mw?_FJl~_DU
z-rHkWQTx22GeTlVx&GEC)0Ef8@6Wrw?ryKNdEM`~+c|~RB&_C~($>>kSN#0k`T6$u
zH>dmG-&e~kd+zwEC5uHsBAY^2tNYKZ`V#HPnQ=7W>e|iMm$yqQcQ-P#>&5T8bK&8H
zq=GLmE^d2!Yp<72mPHD)>H<+`)AMn@$KQr$yV_QLxv`^g@xsH3t&`@=xwAF<y0m%T
znc3$3LIH=13JNT0eiU3?70S*pSM%hApmY1@jatckyXyb{^Ru(lEJ}Smarc{9p)I#^
z7A2=$UvV?teQD?%yZ?Va&ooXCyJhvSEAjQUwg3M8)9TALGFuVF{HijnFv4nEmYJbX
zN>uHc8HS&qpEobDcw&3$(xqMH@9TbkdOFiM{oD-0!xtV3SXf)Lv$M~hG2_eI+uy&u
zyxhk3d~wLGZ9hU+hxyJla#dM<tKv#tarLaLYnXyg#>d_@d69c(Y2x9w+#4GlRfHt1
zl!8*Y#dLl=o1L%n%v7qM(`4(?!ZgtdMn8W}{rWd3?#>L0!be@A+8JH<H+e1%`ugUk
zv5ig5r6r!-;`(MK7AmKoo}OOH{ZP)nZcpXsXKj4XnM2pfe7T*!|L@c3@ja_8%;bbs
zR(mlUYwP&UmD^hDb7-ydT<dZ^9v&0p#g#Ruc-Myg{`q|V`k0-UwnW`MmdvCoW^zBQ
zb!ni=|Lv(aURS3xF)@|BxuH1Ki&s|gq~qd?JTevq_xIH<cI%DWS)|I#u9JOkuC=&;
zfWlNS|M_;Y`|E0*+dnI6dGoCbTW$aI$z<QzW;1V_=Gmn6Pdsz$g5bnSN3(a%{{8>`
z%cH%#QYIX{yldm;J~P&57vVa(A@OiiQ<KizTPAWlV>K3*vTIqabrap2eyKaVde7dy
z{IXUdOT9#$-Ce$HiF$itqH?LTjd{)uhwPZCzV7{UvIYqaOP4;qxj8*HHkMb`Z(?Yd
zjESk~*82bd5)%_c@9s8}*)`F05-7mE-pNK+yuWe&8z0}p#Kd>+-nH?y?>YVY`ugf`
zZz})&`I*K0>ze$IC<PH|E2SW*ih2E(HoqU<6xGwyd-(9-*6izMB`z!sYCbbQT#v6W
zeSIzW`MJ4H3ol%FIDrW?3;yH7L({A)7Ul0^)<$jRk@dS6@<-;&)6>&I`M>IG7Vmn&
zufP8JO>PJ?^ViGWEBa;U&Yd?GK6r3#ZM3wN%%uH0c38y6-@m;*|9Y0~$*srFUwIXN
z)8^On^VaSAHg4R=X<M|U@bNKFY4`inY5nOsk(U-aw=Y<ruyMvaw$r=b-rf#sF8lfU
zJwG>BJHEG0)P{X;_4mB+`1|hs@>W036utYjcHv>gwvVsQUCh4i#wD3|yiaz^Hh0OA
zx3^69=iQC7sekv%et+5Sm2G_O630%jsJ&iZuQgTTEjQPzOG~>K9$vZU-}n9h&(5{p
z?%FMO@ZdosNmJ=*{Pq9;e&3RL*~Z4^+#Ji`zF%8Hp2*1w3JTuSjXq}E_4&nbcGU$_
zBSW|A#m#T|ptQN}@2|BnJ1<>$I6=m5j>W}gxwn7(2<L5Evz%p%(dugp{3mbx9U}Ak
z{rml?r^TGxgFSU4H>JG0vy)fa?9TRldDR6^PpHqY`E=Re{_dX2&EC`X^6&36EwN}l
zdTy@u^$UhmI-1$}zrDDq9J?^nq9*#Q+g~18K0|}f%F@|Yw()oQtOa(&t_=D0<3~Vt
ze&mMQ-{1cI{aw6g#m5)1*GhLwTFE3O1bMfwdbjVRIp6!6J0o?(F7MjCH0SIr)A{xP
zcJAK&`{CjCwQJV!$nt4Q<>%+md;a~+&EjWgW-2<jP3f-W)K25um3DU4L}hm)w`ab`
z-)%0+O9yo#6pXfQ$-Scg^5D+L?ceY0uix+9FK42<Fi|&Z%ZttD?Tn39UMu<I$ld<Q
zYQL;zT#|71ZMNFD<8y3^3ZAbHU;pRNpBH5tOLA^+%e}LsaBKE;K3?9nVQXiBO7TUp
zyUR|VJlWXTD6StTW0DbYQ{E?x@ptpNna1h={`?GDearV4%c(V>CXHsaMd+=qZx_ei
z<$Hf=sdqxcf|V;*E>vFCvnA^7mzS6S|9H$VEd5<AnMrk88K||bD|Pht>e=mTckQn)
z_n-gv_I7U-A!qlHgKzKe-(Ua#-_=#2)@5%Zc1lP47R@wDJ=Ml5UG?>qsDwnsvC9{@
zrZ#7p%<{MYTXMLKH_cSqRC4q26Gx6+5S+NF<9d<#H{WWpojU^;ySewtY}~rlw8Uj<
zchE{1yP6;O_Ezi4Mw?3Qj8)kRY82>P$_lw&?q4A(dSzAU>IV-JvZVAZ6B83{tG{j8
zuz`c0U%$WDXHv|TsJA-`AD6wqXPa_j!rJKV)1K~()%N4lntuBC_xJKrQc*|O7B5bm
zohcOXDersVzoPd)nA)SZ9y)yJ&Ye3vvVN%(r%VwM5GeTe=4RH{>g)1f#C@5(?lnID
zxx0N|?U(af=H}ZsY%o|gUz;=i{Jgifwyt(j;^gJkU7r1R!X6W;vl|i*^T}G7)cyGZ
z>Uy0i%UG#knNd@-XaD~HPft(JHj#aH@jE|b^fK-Fo{}F`iaxKcy{w~i#+gqx>*l7V
z3lFdSA!lFr2HbSeZ4VQ^Q>Bw|>cd@2O>4P%v4!SOc4uwdmbh>sue6y-iG}E?rQXx6
zN?$Fxn&s@|G|MzQ?L~n{VUX6;zrVlR7d|?&CG+w$z1XZ51)stNi!RKVF=O4nef;t^
z1+RWA-TYq0x!w8H<d*2fz}xPf5p~bc&1Ga{Y~x$5w7c%_uJ!Zpe@;Ii^XHjm*^fgP
z9!~i5<mlD|FKmT-mp=OY``gbSKP0T?a4A_??TX!9_VVS+-{0P5U+-gkALfvmnYp*>
z>#l-_hvrxoKRY`+{YAm2z;~||_2Ty2*c7_jonQXy+s#vi0)j%WmDcXHvs{wu`|I1=
z-Bn++oZEvF=h{?mDxGbnAFp;r{*Srpf+Z_*q`zewsRgV|`}60|w%prkFAB7pX3Usz
zZm#w6Wy|vJ@0%-hq3hI*8x|TGD+(VUv;X(w@uQ>Nyt00Fr@r6uQB+j?cX#*lrK_cQ
zZpSSC=m7GuYG)<y>nhv0yIgnom(Pw`yDVy2()R4@>-N?DUR9a6IM6!Brb@y}CP>e7
zZB}UM_uhWHE!D@<l9Q98%D*QcIeN78#f67QyTxal=fAty+?C?0Cf3c&#N_1cT>j=p
z;M%CIo31@xB(weK|G(ew3#<Eyc<D;_-pLH?Rufl|cbzI#pL)3A$@2fbQl?pNZ*5Jj
zvFk2MIXOu+zV7GJ8)vqGM+!2IF#pu(TB06q_Qkt*zisKOE2*caO?}F^KlK`@owp^k
z`2D@T`~UsQzOf+@G#b(-t{+$O^3v0Pe}B(5%ayV$TC(u)$)LF63-Wb867TJ)3_2a<
zdyMVWnphrLzLgvnQ~%uB7P$VM%-7GKZ*NZT@9gYcsH}CrV%~hi<YSkXdZ%7n=X;#Z
zQ|B$Otl&zHD=+i^-L_GAdGKf5-(R_RcTHW!r#tD$kt5&U-oC%1F!|-BrSt24tz39m
zaq07qkB{5g*`1$ne}8Xv`Tn}p)Q=}VO$X)sPGR+R(c90hacfpf{w_81LJ{AR8yS{o
z&(_A<?B24~bFx}Hzx=YJPIfbbR?3)WU0D~qTPLg9TyE=RHAdB=9;*Z6?tguG`t|SI
zTQZA3JaF{({#7RGRa8{8+;6U4{JuYr`|bbT|NnPB$PKz?X4|r_uUj9#->%?+!}h$?
zq}(S()0$5`{nW_JE@xM>qbqxP+u>~-M?->LyW(?x{Q0decD<~Gho8H+aADLprRvYm
zd>tGP%rs8VdG&RTeTA;}qFGu&XDZfyJGU_Z(Yf{OqPOQoM9kQ0dv()(_o%2@>({^E
zS^PZh?5wYUzu%X(l1Y*<&%5*C>sMcYe|}!x-240XE<CImZkgfa)U<#9|E1p3SD8!3
ztiR92sBKi@vgA<JpZ9IIfB!#!>C~qJ507p!T^?D#%1=*D3ftT7mp0G4QC<J&Ubdhw
zlb6@I=F9a*WM&5Mb?d#L>@H=Jk?^A6l-|;yo2$dufB5jhbF$jhvmY-mxe~Ll=H~tT
z@&`UXK0e#r{6oa7qRGwoc9*YTz5e~*>hChvKII>0XTK=$I6A%Hv&WS_LnXzs_qVQY
zPWN|i54Ll0cmMtCr)~AGD@V)MB_Ds;#@8NFbN>JB;^gz`$Bs>EX6Gw;bK~NLhf`do
zdeaUxFoMcb)2u6}rfQ#lGs$wvl{wbsdI|~-N)zYU)!y1*p8s@lmy(gurQNBZQg=7|
zA1_V~P1$R5cW3q4uMm~^{_gJY_51%Bl~_DI@bdEV=<S)G{~Yx`@0hK@sJe7<)ndlp
z<8MM&Jbe9ncky#S=l0-d8C!2%T<rcHQr&2Wty#b_g(2heva{#rT0bqSe12~3`+Iw{
z4|tq%<g5De;o&z~t37^Rc_umC&ZqrB3ExF0%QWB2&?hI)KJgICE61;SyUX9dyR|iY
zR#Uc7ReA5RW?osol`g9+$}`L!b~r?(o|vErD!&&V){M6M{bqCS?QK7Q|6U)xy>Hnj
zrJ1`*UT&)V{OtPr`0`g*UOqhB-o~|fO}+B<_3`&thwJkm@jWiT<>QiX0YSG?uIcHj
z$(yde_~5vFy^ML@9Jj@WB`&EKH>I9t<CQwnp}W`4Ht1-p+E4YRXSY=S6`L<RIdXU1
z*Q?=|m;3AM*PijZ)gx*A@7L?~eX`bhcXv&FW}~+A%d4x|XJ?uE&$qjJ_GZciiBl!@
z7b-tL`}^TAzYzCtwPa@BoI4;t@4-5FcZzG?^ZmbGtycA(CJ|nmXJzJ*6?*!D;KWnj
z=%bSxBXVAJ@MONbv-AJ&`~N`=Z#f$e`^Y5<+1J<oeblZ0>e|}sXJ=-v3SB+z)6LTv
ze5VTUEb*ND@7HVn%dh779%Jb`n*%CE-fWrk%<aLH=iy})m6ft~H8*bEx^+QNa-P>x
zDch<qeX`b|sIkdfequ-0LS?0?vdhC_?tk^H`Es!A?JZf0f&+&R9l9VW*_Kzk$*ork
zr2ozxP$$vo?f3ij|G(YNmp0F<`Sj#u(7$a{j>?$m>gIlUaPZ*4gJI`nrFw5@iOiO=
zQgZr|<n;ghgDd%cetr)V6VIJH*T#1ov?TZT?(+A+%l#s&f0#>bnkVYXD=VnTa3Fi@
z_U3fEeILsAR(*AHc7FTj4UcSKX7uK?vqw6G@7}$8_|PG*{b}7(-0lB<xqROK|D9E#
zs~5ZXOG!&lzg2lUhVR#_tE=DL-5tND;^MZ83g;QMGlc?zuG#cOl>a_3JuGxhpN!od
z@Bm)2#QAx)pFe*FRk^X<g{915#!S~P2ntR*+#Gy5$*Fz?($J))ZS?j$S<9l7x3{+T
z%h^Wlttti8XT9t9|Ev1>>FHeSa=Vfj0V}^IYerkXXlCc%_3PKMV@Iy!{cr8Q|35>F
zGsx`deU6>`iq}l^*ie36?rzHdsi)4)Hn08rXP!;nn>(d_!OQh|W%(3;R+p7N@7+7u
zH|5wtcK&Its)=3)_tpOX_5FQ*c6RljABF7f>>!J0t_)uOimi#AjcwXFg>6Ptn&bA>
z?3}&7-pQ${`g|+jS~UqP8K-+s&n~Y2f7@h6@V?m1X=is7KIW12dkGpkJ0G`q(&S*9
zUu##~ELpMeu;P<gp=7_9+g45oR&UF>X;kpQ!MQzn)1gC$s=mF+%*>ROlk@B5nasc7
zYS!_7`Tg0~*BxwT&%eEGt%%y^r03`6Mqhfa(X(NL!T!9vK|z<Lc_QO4XM=_gpV+F1
zm4&|krGCdsXiGKsVFOvK5`z+#rHTdy1_}xeOWhU=-p+l^4tDF|9bL(y*Y)%lHJ9hz
z&PYsr`1Wn?iw>_-H#ev6E`OhwlOrP}6tq1r7UX-?w!~fK@8x7=Wkp23yt%n~o^AEC
zlH!t?CmhXlZ<%CVP>55=HA>45-LBWh*FM3?a8l^T+#C9u-$HIIybFr+@88$o2zJnq
z-&gbI#zv>ismmVoMObY$DzRu0>AJBd%Jh80k=@6enzt=qvZUngt*wu~e(LJ!`SSIv
zC<n{Ig9m%1&CN<ImdFTmurM+*PS=a&;^C?J{H%BW{P@m!=RV|q1SQNjH#e(_mH8ff
zW4bcT?ftzqzBAv*mP`(+s5~fb9JDO$Tb+p1KhLU#=aTXdm*uL2+zrd^)P5lEsv>bS
zD&^>>^mD7%ElGc;n(<h1)x>3yMH98JWOjwF^LKmh=@s_JNHh88Isd~)+TLcLXNkM^
z<p0DjmcP>6XCv(%P5RZgE5b=HO)B+NeYD%2KHHm%-TPyAmrZ@vaVY5U=Ra#AH;d`V
z<-~0Dn{4O^9*11?C(u%tV`^~et=Fq(v+b=cDg5x@;B2#8BO@cD5|^XTetmuY_V#vv
zfB*aY>+5fC%k}<M;<eM}x09^U%r~*jLQ}mezm?jy2+iCP@L2rm`uQ=I#c_A`gnyQr
z^k@5nN&jr?{!P8LCi3|<vqxw4&bzxy<mZ(;+6!FI9-6bX;_0sx1_q~@I;)=)dhJyD
zEmmpA`|j&uZ~iTDzH>kQ@7b|rg}3>nt3mzKeAjwah-5@XML9Se`1$#H`uTZ(ANSi&
zyVJ?Dvw-P%VeoRlzn{<B`{uqjm6X;n)84H7V@4pO(fOR<KWfLd?Cs+&@UMMveB!a2
z&zU^CgZ->N&5M5;rmH&N?85)(BX+^eKjUx9+;N$E$AuxQ@Rs)S9f6FK`nLVbU-W6E
zn8x4i=y(2X3?j2`&KB`EnpgTlLGr<b(3nZ`8ut9x>!sADy?IgQ7Pn2u;@zKA-<&%2
zf?S1H7b92-1jSE1naV5B<vY9QW^wZSC(i9<ptkFWhlfRTj6}9Z*`}SDk$ZdFOzR3$
zxvi%?!qWa*X)R}6Sj07PW9a3N?2F5KZtpC9-Yu^G?fw0D!?d$Kot-c5>@3d8dbKWg
zx0fO#>y!59r89&ll{R0>zSubb?`x3_g@?4(pIiN7jegB@`7%A}5A!U!=FPkP!a8vO
zgIC*k*K(h749;)w`@MfY15cEc(em!xBfnf9*WX@Rz51!SHSfgcXa7<dIBL`WfBya8
z%p~#Z2hZz1^QG*&{d)Ocd-ff#<%&=G_r1Mq@OXRdn%`$OSI<53$@<{!)z$5Nzvl}2
zTGejN?EJ%eQ$kvrosI3+@86)N=#?ua&(6$LEm3MVm0h-c`S;h?-=DYtzh%|sbvKV(
zFq+fA^q~IkEUVP=U#okbr=2@g#+vi$R=||X#LFt#V(a~D-?#D}X#F3eF1M#%(4*C<
z{;ap}L&kr9La#2=_-mV{8Gmh!Z)tDjpY_qf-;bot{qt+)mWwy_au}TYmh|6TeqrkA
z+k%;r;${b@>^r8-=C!xvdYbNfjR!7v=bRW$@|&1<{ms{~V+wm>S<svOJ9Fa1iT8A)
z@9nR@e?d@m_LC<m_4WUAZ*L1-`}+CUlM9XJG&G&cj0^GGoa}nZ?eO+n)&2IRuR>;<
z<<7IK&3e&c7ZwvUr@#OF>hSeuxwoFYejTm#e9G5IiORS0Z4XQSmG%Bm&OTpc!^_**
z$Bl#ozBhh87t>vr&&0JLZS%YA=lkmeHU(rYst!2o#nxf_$1Okl?^<pNhNXK{rUb3a
zmY(o6QbKY^Ldmn&_N=adrt31YJ}~>R{PptfB`GWnbvNqtlo@@weiv6A4y!Y+yUp6L
z(LifK*W}$be<T{_71yrydUXE8;WKC6JUG}aCnvW#?QGQ6tWeImM-419l9G~^`_1K(
zu}HYRp?<Pqqm$=8&-$B_W*v#`yq;#0ynE}J@70G&*3UOxxbhWm_=z-4yV%`7msmZm
zoqhE9s%i56_sf-X52o@t_{ePk-t_9~?{8XVX6M|x+r5mQr}JB`>o~W=<mS~A&yy2d
zdN>Y9fBlp_Q(oc!qn%Y>_kD4H|EESm^L64uhCjV8jG}f<>P(cFp7+0)KceRSy}e@h
z?oHQ=1(iMzetv$QdwZL0*_(*7&lXGf--y#oTe!SvR_CeAxE_aq?car-ue<N=?!NuQ
zw|$kLmn~HORiGcY=f#zkmmeSR4__ZAD-f`XnI-b`uDR-4FW5f2`uf}7&r18(K1nbC
zGwtV*b?e*{XZIhycF%TKY5Kj%b3Eq!@n>Ow%f}~weTx>uW~pzyw@Y%q@7voo_fg%3
z@Y`=sO8Vces`-|zUVAh*<jv`tvI|S>A69-masRj3h7%PhS5*By_%OZCC;#8Me~Mkd
z6QlDR3t!&a^yQlKwoRM3`1sZZFF*I><Ye>wdoe=G4IKD>eRy~{Y;BaQi_4WP<E8%|
zY-LK^`EvT?33IM~3EMj_aIcN@3@Z-lwEeF&z502V72H+QJ6^<|wB_cml#lDVR?X5-
zR4QB6FL>o<w#$#R|5ugt>&$JfJU;hZmXZ+D_4O+@x9|PlWTg~0`_s89MRERxnhaaN
z{n-4dW6u-IEq}IL+@QOD@}%2a^W~+avR)L}oqBtFdvry$yQ}Nfmw$5i?%O|oapa4Y
zbC^Q>?jK!wxu=Rb%)a{HqR`jZ($CK`EwPYSGBmt+{P^>Gd#g`R)0H+m!(e<e;^yrX
zyMNU$7hbP4yI`~T!`Fp5yq8yfH&;lxz0*K?vsvFdxxSnE%g<z$>)dtCHQ}GmdM0i5
zjI?gYYq_7yce(H0Z~epOyqx*r@5dVc*Ezg?v)d%&@}jn>Q>UKX{b}R&i_e`abZ;;6
z7oHvT)+sYP8?^EwJ>5L*%#6L&-^G4^^tNXI`+oobJ<;3tNDDu{_E~(poc+}GA3NRL
zcn)Zn`bTW*Pk-mXK0jFSfcyPQcgCbBtIdq_oa)QB#rFRBU&hPc!NSPv`d6CE+M45&
znRbQM#<x26<=58onjHwfazA*+cb9p7YxXlUAGp`Xo1yXQz22GSt)WM@l&AkT7h&DO
z{nqG|sdDOn^N=V}1_|rnV}AE9wg0&Ns`{6dfSE%0*~uMm1A-a)xO*=y>bPaOnPJlW
zC-+&`=4|@-=4QzsM&2XvX_cQ2|NngUKOMGqo=xSei}k5<&us<`T?be$E&3H_I-4)k
zCTqKm@7<ln&nNBJG;PI`Cr{SJ?!LFDviRShpYv-zbvh;moW87(agB$E;qBYhh35C4
z-Fk58;8p>X+5c~w@A~$!MsxD~`+F-tzq`A;T;9en?AhkVi5ECxE}dO*Ixx^&-llr)
z_pldVAMfs;e1HCiQ;+5E?~xQO{rv3lk1KoJcl_27ZTRzb^6bFCz;gNL!W`58{CIbE
z#flZP%jHiSHwV~7n(w_IHt+GmD(&!7Pk;SY$%{=l6=!~M_4T)Bs}!%<+}ZqM#(~<X
z=;KUmQKB8)R&^2%-{+n6-hcJRy?b^lDo1YJsyZ=2(Y;Sb^Ir4xET`HpFD}~E{$8dX
z!rPv+;@SfL%z3+RXzU2~mxxanYbXv@-u+^YU1-nB3%A`G%wKI-zGr!~-R^7OMSQb#
z&)u728Fsd{O{>iAwBaVlH2tW1ekrp)mh5P{y-WRXgw&ec8*yJ<d$TxwT`MFemZ_~g
z;LVzR>KxyMYbVP>W#%mSr|a?a&?MRWQBSgzO~f20FDT5?`LO9~{JiQd5B}YlQm2~y
zTTfT@uvAK|`L8X{mHn5j?|HaiPHFAZ-RC-QbULRpB>nfCEx(ra$KQ*yBzs#KR$Vt;
zRjz2eIXdF+F(qyV&9K-jeMvo2ax)Yeen@ZcE@F;li@26|GbicB)USV!^+<kxa&q#`
z&FRusix%D7o_~LWqO*z3?TRPsR=s;#yv{^xd5GWr<*QcrUtiw7K2OS^;K70W_5W*M
zU0K=Iw<6%urAvSQ{#_rl^V8Ss@t|4Jg2ofAveka?9vS|6?aLsx)>7_6_5C&Pi?Y6K
z3pGEg9JZx&+WB+!FXDo3ygF(eCUfTBr4v7;_rG{MwU(*<$n3<;ziQ<Vz2=^HA^p-n
z*ZEh)j89$nSJIc^z4zPQW67`6Z=buRZ_R$r`}yIt%D>`mDO=e~=KcD;htqD;lXiE$
zwpRK7pIw$Ldp9NiShwGqh$@rEI>!C~>@Ha)|Ni}a+xJ;Nimx%U%1)E}v9fz{!S!@T
zZ+$Js0>SQG+ag#V+`cQ`RyBiZ*}wUIF%=)n8M4h=__J+|Hvdk0v~+QG<=X|Ev0Zl-
z1{eOHuugH`%+J5g^%&2cJ9qHl!MeJ-+TY)Jd3kk@g&y}>FIRGUnl5}=agW*phKM<~
z>o1jaOy>A)nOfTr8`ZVsz*p^6?LVhltY7v>a_!Ne58{2xdgCTrrN;l#W>C7ee)-3Q
z*)q|$th;<H|5Y}bJzm!(Yv=HYCv8UmlBJ6;gs{ALmB6KxB+fte+QiKBGQQG#-S*i}
zV@}A?{k3boV8bob)?-Q;3=6kjI6bXWVs+h;>>X1!INy4upC)VN(QYcBW*>Sse*Ukk
zc1OAS7%s1wT++h3zpVYweRCUz6+6G~e){?Q(%e0Z|CjEl3;e32+%PZF)W0V`R`J06
z-7!yIPwrm5ukn41qw#|&iE3tfx@xk<Ti2SWpPSPutX?#SaoOXa2HN54-dtZVuW$7E
zm&%eoY6}(=c|F|{aORDa>FkP-sG~1GpSPcHS-k9Ex>k+;(x98m{pZ*J`<edf$;skp
zXABEWimtr96MelfxcJZS%l(@_?fLCqKJUp^d!CHczAb;g`9zpsOF!1tJa_rJU*=X;
zR=;koV_nRcGtai_RNIL+w{_>+{_gPp?Z2$&O6ZQPzV)wci^GkcxKIE8e*0E$XQitQ
zJuR%;mu1Q6ec+1_xs`c#V*R4U`mfIw<@^+Xe9fiFfB%+(r=1?X+%uxJ_s#scyXx4X
z*30Y_%ok+hw|sn}e3$Xjp}M!GAD_qF|5N)f>dVv1*2i{tEwDA+d;LZC`rhjQm7n?_
z8ZqQ_XS$#JU;XdQOa}A6wwa>u<sM6jF^F$`@#E|nz9*lIZ(nz~u&{i!()xwJzV+=>
z+~>&Kd^R;?ecWDVcD@dmMRK-PF?%XD?%<2|vV76R%I(%CqdDbfg)hg}BCXb+%3eJT
zT_x|%aR246dYq>8ckcX-x35k(ot5o)xU_PgO3xJ8cdASuY`iu;+b8CbWn*p}v67MD
z*!tBf)xxts^;Uh0`f2W%DZD+s=fhMEcJq{JZzt4m(*3*FSmN6^(QsBdH;rreuQUDm
z_qfYlhJj;kt8dWG%<Q1Wq5@0-#nIbOJeLdqw?X+YCp$yhGaiOxK^7-^S&!>lE$IqX
zR*c)M6?DDoxXA|rwI!bmPew63`0?AbCTGt#&ua=SZ$I`lZhzO?Q&K<ClXu~AmWsuk
zweKzZY;Ns*E&b!vCy(3Z0<&+L&6datUH*5LY4)#gZ)gAeS8sLr_p{mg%Y0@kZC!OY
zCuzsdT4Cq*&w?xF1X=3JJl5J8?PHbo?!nfIkB=T6ZeJau<?J4!(CcPg@!`RT4<9Zr
zcGr*Ea^k|nhWgb)UNLQc*(<B9(vRE!(ii)l^8Ne4HQ&$h^*Q<0Exz(jK5FxqMP3i=
zUhd|6o5FD5+H1qV=RbaBh>0+I_Mv%A%(dz#>&2U<)f*N*YwNhXYvPYtk3aLfuB`s^
zq5MMMUD^BQOfQf1*>C%Fr$@Iq>h^5rM^(0Vv)^ek75qM*eBSrfxxIB?kNZuM_>j}>
z*{(Fb|K+{fJN)~7%0wA={MKP*xaX$-ugLhd-RIqjx@RLgR$r0-f7pcKL9$<bpK$lt
zEUQws12bks%shE!&6-WVQ=)7iOlREXZl7<tbhq!-J$p=^w>a6<ENEY|B35AMihx?@
znwLEt`W}<je3_V;E&D!y{o32rr8O;iS_R*#t69r^W_~(ret$;T=M2M_J65iB_ng0R
ziD$!Jaid@1(H!hI865r|TeRxH<BU+93ELO9*UBzrW%wYzXtS2vmh)Gxmn?pnl2d(o
zO|Sg@fY-Npe~P^8um7UDqiyfM+kf|Ean){)&CP!~YopQK`mJAfbG?dUmW(aFG%vgF
zWq!a<^DKqrYZCtA3q=@CpF4Y_cHZ`V(`S~Nnyp@`x<66oxKZ}JlpK3@HizwMN(|c<
zZ|X17nDXiNf$4JtlCGuXEXX~~$j&bZ+J>E!v}mES)`Y0Fa;8~VnpnBdyq#?_za%8=
z>d6a&wK`hM8Kz!#5|j-8`0L4p(z|)Lx97jVwl=z`=+p1-nx#=|%dV^l1dU}IBpmqk
z^fa$*-#oUA3)Oqp9baB_x~t>5O~a8B%F&g1pVVW|e$waobYej*gVoo}^toS^Zf{z@
zFFE_i&oy;N=T>dGRzK7CmEX;uTi-oA`uf*A)A!MQDi6=>o6JxrYgV-8+>C49?^iK?
zn3cP+Q_5lfo3byjL!#Gc7arSIrel;=?zN}%q00KFdHPZd@7J&SzTxPu*s{3VDPN^0
z#&0ZqbtgXDd5ztce}{~D8>AnyGt^w(?K#oCTxyr1-R^5^qu<}z8LT(G{N0_O=jK|=
z*B_X)_Ijv>h;uvN-R=4Fmj=&1ro5e7RW#8rfuUxP+N0iWy{!xhQCm!+>K*Rh4(2FO
zWL?t`bl>iO%>P5qzL(jY8dgSoH&p-35h!>6ylU$Tt%RIYVXeRRYVstX3Q1)Au(ABg
zdZVPkYy}}Eow$_u(s2%!E0UEN7StS$?Ro4Xd7sZzfoZ|Tebsq#c5|&iJKeq;#$a%E
zk4yHW9oI}*AME%L?JD<lLH>isr&5{Y%b3DK>Ob5UD!yHRUgvw>O(PqfGZ!yjyx{nB
z+N4RB*2nK(?A{+%`u<kI3$<%2nsmOpJT-|A@%x<QddW?B+ub|%kB@Zn=HA}M3tBv~
zH^(>9w(!xBx3{;KzrR-tnqJHDO<3kN`M>nTk2j}DKS(x{+i|U`=k&w<VaAIJZKH2I
z+u!`?<J1#1S?;ZTU)6SZh20CCze_^*{dI%KPuIL((pi1`A<yoq&(GbdUeJ<$uXKNr
zcWtS0Tl0dO=KK5BEZpxXZCEzPV{MpoZ`RGrkC~3W+265~b-~e&$mFQoE3YybBprzm
zypqMRNbCL1*OPYe&%I`p^3wi6au{3u+YgyvYrFe6@2GDrH?7Qnu=9n~Bi46!d2a~J
zk9~e_?)IFUMmaYO<mKlVKR<VMUF`4VzObhHORq7OrPTZ{UnX-?bY<q{qZ|!4!YtNl
z2D`T`iSTt1<QEG6P{C@u^;;`bsA|Ku0{f3rwFj$8g14&L-PgT$nQOtb3WhbaqRSsT
z-&*o5F5ve!L2kx{TR)s$$HcAhvhmre>hjah_YZvEvd8R0>T!df)&FZYR-UzP-}f=^
zrqP!Z6OFbm_}ne7zpM21xBK<~uV$ESikqMMG=p#1p#?>=^tF~(8ZvE*yPqobSt`)I
zKI`FISvj$1@80FTSnwt7%#4rU@7I^Vxlwp}nyypgOb?e$iP3zLy}I#?9Y1RC@BHN}
z78k-7b>+3JX3f4>7e?b~Yq9dHdt#>kw^ho1c`dL-SVmd%j_;-k7jvIKy1H;@=swn0
zwqO4_Z4`0X)LJ~-OqaX#*c_=Za!)sXsp0u?d-b<B>%-FZ*UOq*yZ-C%@289MJKh_v
z=uMP<wE6YT&CfyWckBLC#7g}#G|y2jwf_3z;^MHiQFHC<_l3Ro<ZWBCK7muCVao4U
z8GpqiE?o0w$?%V>dH?53kNesh`+|Rgf2#G0GK+E*pYIT3VhC3E@8wHA$+kn}+x+N^
zqtDiU$$x+HTJ`-W*)@B9ZOSuXVVT$G)bRhi*Tm&(&t*I~JXiYZjz<0IQETn9ElM7*
zKC)!n+gq*e?b22<d#%ggZTax+-~0Xl=hUQCev6JfC=?K2smn0+a?_QMGb7jkU#0Fp
zZ(GjQSM%%t8J1Wud^pT+|L5E7{CoHA?Jj>W)_wZX)Q}e!-W`7z#IQ%L;L(fs64&<E
z|1K|mc5!Ljay_@_vzg7Z!>+%X`#m8yJ52k@@(FAX(v_bb-&HQ2@aM_1=0o3lb`^hd
zyLV2qKmXpD^kpmN%+S(Z*7t$oKyiEe`8Ynu_IJ5m_l|pB7vL|+$^Fsx;|&|X+@Is}
z^>_By$8Qh1&Mv>)rRYNT^>tf6eA`$2yl?gYn(g!UPg(EjaNqGrTE66i^8eGnu3lTS
z&@!T+{ND+QzuT{F^y`a`NYB3e{QUk?w&{^-jcZvLgo@+-+%vr=#t?AZ`;z7Hy^9jc
z>@(&2%hM!o_MhA|xwa_gc#L${@h#JKcz?QWvHZa4n|f&{XFXW*y=7Lo8?$`*syXH#
zo(8S0d#{%KeWgX>_jh-_GtPD@?JRicbUJLU+;Q7)FMhWdPtWI-_1n0x$m{8s<SSpK
zu2o!FC$jU>@#E?D_tm!XKD!=YKi51zZn0Z$Q&Uq<@96+_$zCp>u=`e#(<lC){_{xn
ztqZ%}OOz=W=G>0uaVR!=<5%=8-|y4m$yp3$+gPuB|L9rnw~DdC;`t2q>mEPmSwB~O
z{I6!knYu+wbQf3mT1_qg_u~!w*{T(L_rCdl*mBo{J$tt;zP;f6yfmGY(I0Lt$XU&}
zcr)M4{r=9Im7Y80zmMU1xRvR9rtfJt=bEzQ<9&aBKA)d|ch}XW-qW||e^&X`8vMud
z#mC3TZ*R$+Jz>I)gcHARy*+ijC%g5qMYy2Dk(6^6gv8gLdok_If9aW5mDEn^)Jcjr
z{D|ny^WM02Mf|lbS9|)q^CqT#otV=rd~<zt+*#L9#)3QN=GfjaYPVzPY1McgQ)4AQ
zHE^$C&4w+TRT(0pHx;)@Ur+pb!T9Dnt4cjTw;;9yGkdgnz0*JT=e8O#B<$KUrRKG6
z(7pinWe!J-7!(987zH^0+%2B||3}2vxjluw8?SSlpHfUWcpDSJkSV&TQD0npx!GB@
zHm=mGZoN`pKcBbXU*9fN{NO;N-&`xz<x9+^c3xihQ-j$p=nkma_ctQ+?yid3DhZ{}
zb&%y`0f}qk_WoMEe&4F-?e7jYv!6YC`bq}Z)4L~5-+6j{ChMUk^XBKBu#}j7@vd=o
zag+DS9r?zmd=?(j{q4`lko)A>gY22n`}US)$(}b{^Iu~Bno~}Tb53jQ`}6%o*gWAK
z{;$`+&s_E2D}LwHohoU!xbHp@+%;E4ueR+v>w)ZTtUgiS4I-HxX8tKSy+)5aS$f^e
zaQ)T#ivr($dH3?Q2t#R9`BmGyrp5_pPyUu+(0D)p_Pweb{<{C)xBoJ7TkOMrU~b(6
zmIm)#)#l$H%@JB|R%|rw-JebSuUk)k+9OevvL)uq$-fL%Q<t{=c(d=vqwf6uf4}Yj
z|L^s+wX<X5t{C<@Y<#}p_V)bmpFan$nyHq|oGblJ`(CrO$mM-IC+=opus)z(Y%7$z
z;hoonH8-bj^q9L<)%ry6|7{z4FWh%4pL_YA)VdHRhRd7#uiO_BFckjn^}2&$XPWO0
zDTbC`O^@${$UIQkFwH17%4?a47}KARhkvbHGB<nUu8V9*#f`EI2kv!lO=D~@y|{nV
z(;T<|S%TIz3r^npxalbK($bU)xurK|vIt9Wo5sS>qx)rTMbF9wt2h{1!g8!NgC1RZ
zF7&n3@|)bGslqAQ>L=Pv1)3h4$%HXhEUFJTsF?8Oy5KS9y`R?IJ;eTMdhatn_x-Ei
zUb@w`^|GkH;Ny9Qk7i8VG%u8eA@7c+d2yClr{~sPpkC<OXCLpD-}jwo6S-RVlf#7m
z?Nwi2?W_Hr6)C*lpD*(ISKYr1UOZESwU!?}d}m$Z=JSV^hV|{LjP<I2bzyDv_PTF3
z)BXJX4jw!hx++BT>E`{Djo#gVa8`ZFo~o;T6Qm_yvqgTJVaL~Su|f0RuS+bcpAI+A
z{(r&NuJm2Zne4y5{-Wz2*8AnLKDeQO`1+hZhZ*zb=ghPHBhGL?f96xAv;_|zSIQkY
z$6K9e=N+Q;^!xeT?~?zmS!A!RlUjTF*gd<hy{A7jJd(G)`qXY^p}GFOs|+5s|AP1!
za&}*ge{t+h@qfM77gq(|-&Z?(+pCQRE{2I$`+d(y=zUB(;wjo-yglj5+P9xt-2E*W
z%KPGK7d>e;z5ODshiOAi*7~-uzrTF^^2+pMdgJdUmCDkLE23_5GOWM3t>WQ)Zo8Kz
z8HcahWb4fQ)jNl)LD2WrqmS!$bEH1Ho2?$ou*E-K{e<zpoUY%;uis_5aCdjv!+)7P
z4|c!b_xsQ1^ZGG6F02mMUuv}f>biTKn#UGx&A7NIa&wyJnN{X;o3C%#m0l=&?SX{m
z@8cdTGyAUpSn&2!vnIohcy*8Y^F5-47PDShwf4(po3|O6{!=IOtz^7$MWF7UXU_Ds
z6O|PGzWx>rea5Btc6Rsg>MoznTi0sd&)6m#Hpi3kK#kdn9j&h4X4Smid);U5oY=MN
z=X*vMd1pUX`Lk{E5hY^*KWWK@F_LF3cSxVj&Y$v^qrGoc%`T5ky^F5q-JMjX(<^z^
z`O?O;Yx0x+IPLwoA?SKy&&HJLd{d%c+?$a7@5a&->y>9F9Ars8&3=mWA#16T@EOU}
z81?IKEVA;x{`!(W?@{GiSBC8szEww8$DdphxM%OK%7{yUH}9?fes5px?vP*a7M?l!
z5)?*1pU>}In`JI1t+G05`IU3gmq4YLZkg84)vKl1_tn-^9_bK#eQm99r&p%srv(cZ
zFfy}+tdIM9V`DNOpX<C!dssB{&S#k!6hBmA5Gc7(oLCtv$^7qR6WiU_a(~PZ1TsB%
z*%@DJe}6{y`*Zi-#F+U^m)2O{niFM~U72!B%PXh*Vg32Jm+r*b=h(cv&#WDOp7*3#
z)4ro8EdS58x!`w<;gm49d)+?O*VV^2)IRH}H_SiV`sVpBO%0_3S6x2-c+{kCcegsc
z{<wO<>+MgDpWj^(;<>l@!tVGmb``f9`L_?gt1p+YH+t-!-14{S|J>;4Ibs{XZvV(#
zmYR3-^Z#i+@=voDYnSEcpUpMS&|lB0@XFFU=U>9?npIZ9)2_X*DgL)^KQouqqiGwS
z7VmFWFG|VJ>CRmE`q=DyHy+x)P<JzV`Q@r->Sj5U%U4zggVxz>PCvh3!2%`T+1m4D
zLRMe(n{W5G-~OM$#=>kPHIJms?;nq^S+T}fl0BI1!0DDZ!tT#EAG|G+n{#n_&$%Z#
z$_C<0Qvb?q{xgO?QCrqCTdzkYH)oyXc2}R^Y_<pb_Szp#C+8{~_@CUIyzrd%hjk}c
zOxl{xWmj=@!POP(-p~7Cq_rXS^t5(9*-e|YKFUk>x*6B~`2kwsel_Px${9Ij;o0Bc
z2S)~4*50qw_O^T)l6iewe_g5d8t>_Pjg5`VmMwdK|M}yVwzhAtuAaVh$;ig$&e5Yw
zm-wd#oO*ISUsQ^t)ku4F&8|?brzi8Daxn-+iW=Xn{#7g1<6Sp*#RUc@Pf_j7{1#m5
z{(e_l6XTBhoOxEIZ}(9*At^cdfz!KF*FSpA?2WwGKDqGIj~&bNTf|J?<aXIC4%WP!
zwe|7w{`*_AuRl20ygm1J*pHWoyA;_&HAD`#^YioYY$?gPn(?G;;+0Q-e+xJ<%zOU6
zt>*iSJ6`6X6|Wnjwq{Mevok0-n4L#L0KSSgaxs(jC1IyckGI}ATYGuFTVB?c6&EKe
zyXWL`&-l5?Vvc3;vFq2vwWhYUwN+Jx&3Wb$^f1!|v`K}XoxNY${N4Ng|Mi|EeCa>y
zIC;{fs;{p~-`&{>S#}$Kbar#+gh}6C+&N|4zHe<>wsh4KkHv1iUXMid<M(~}{(XPZ
zQ?K4-hi!iSQ%pF0N5I?Z)Q;-oVt1^+bvv((dYG7)mzSqn*mdH>3BS2kjG&DnmzVi^
zKN4w~A~V&?G~qx)^6|dfk4ME%Pt(;tqbEPL*{=HAn&Ri@w&&d~$}qjS_PD-JWYnI<
z2}Ms_Lju0M;|%J&xbVRPK3OY~_Xp)|DlVKn{`_GxyWiY>MenS3SMd9JJpG7R&f71O
zd2dhUN4Y=8dZlx3Z}au{w=a0$FxxD*>-&S2DScnQe2LpzHP^nr4zfk6`u#Wgsiy0q
zw!XTyHX3B)`>hj%0-Wq-<Tmf~J;ru2IC%fRU$2wwe@s&Kmb0sPaIKh8bwNuF&+as3
z>E1gtzx?|8{eFG?8}|QSug6cGG-+GjUC?s7{Y!!}6A~6IUi{eK{_m7oQ4e}f&9u`h
zyyDBAdVP(OubxLoVua<^sbPh`HfUbE=UaYemT$Y*?}?l%bJpD5Ep>9$G#5)>e}DE1
zJByzu*&8V?J$dHLn%dvrqSl5jt@`hED5TPp19To1(n)j|KSUg!>?wJA&woCtf1VfB
z-)iejOi0O|D0+In<@#Np3e#7<*U8D#_3(aFc4ys;(@O;wIX|5;;nD%kr(C}4uB&X~
zpD+8RcDeVj?>~NqXKhZKd;3<+ojrEz<zx2${Id4Svb^ByMqAf~&UCX}@vt;QTk^q-
z^sCb>o-R>l)s6eK5qSwk(e!VN1ipNqc!g<Ryzl+Mn1g~rYJrxjOP;9a%$!$!QK9YY
z{o1<3*vN&?X6vlFa(m(9fA7^9ny0m$dHKoE*hHQ6tJKDx>pl_nU1?XfY_ht|_e}Wk
z<zn`wf7O!>J;OFV>;Jk^wLFn|LBNaiQ)f*oPSX=i4xeJG>??aCieZ7Ru<umuB~6Qq
zygW{Bmiu`8X+4wp>uZ(lk3Sv$_W1gGU9E*IA`Jnn;+s}Pt!tfSv2>3{_x$X6Zn}?6
zdp69EcV5M!%e0DJS9y!0xk+zSH^Y*>4?jiU+-U)t@^0Ro{3F@$=Cr@8#<eTXyxPCT
z^7<CmcQb>%%M}^UO26}o;&k!!m<b9~q4amx@(PL<%j;`NHg+$MKB=dvQ~1eV@Xa?a
zwY<L2&3o7I%)9sN=a1z1W*an~6h6|EV|`$E<+x`|(95D$m#+&_+te1XTD)Y}-X{f?
z)2sy+pS9cX(atxiP3S~-K%i1a?s2AyqUhK^pQi`}nSldVa;5=8g&nW_WzMAm5z_AA
z{Kutka+k#1_>}Lbq0`!`|HM<hnV<Q^yXy-=>Lj%IR<Y}@U!%)<Hj8V)!-Egs3G127
zSLF8bPIvU3so=Y^xB7_d^P2}QcW(81bWVlQVfD007xQ!)w5Bm-2g`BUIr1`GGiypS
zW17h3A6KMr>f!a!SU-6gUyiOI!`uCat!w)gmuy;C<mKTRtY%;TMTOHz(Ix1%)VH*s
zJi0-4)dlZEyWEsEu8CURHr1kh(T`srV|VVHymzL+fy>3xf75eVK74qZovO>IvZ?3n
z)bnySRg>3EoE5fp<3z>V(zBKadh4%}xVC@m?8=jWXNn3vUGQ{bf>->;>7{)^Q$9Fs
zHHi7D^>}6P<P1OYjhWkcC*3bNKY5GO$_GDfP2+uUteAEoq@`UjGkww)zITuOrC;pl
zye2p4s2s!cnR~-tUUt5;QH{ILpIteX@x#H}Pa8FVF~5@f;lNwd@}&QirlP8@X3p0~
zlG=P}^_frp^u~YrcFM;^wEz772lj?7=XbHcwoEEY-u`OU^_6}1?)_=re6adIM`uX=
z&cAnFu!*fKFZ<#5Cv^MGWh?KRGv?gA`TRSNnEmb_bKn0Eh`YTf_;a_q9#e*8^6UG_
zQZKh%F+C#9`*~^Gk41}Qw%if?lfR}=SMZY16?=Bg`3!q&m+YMzneD!KK9j-21}=u=
zQ|DePnX%S?b@}z)@D}6UC3j^bjJdC!X=1I&_TIR0!bMY~3=4*i-OGPz_Ws_VC#3n|
z!>XjWy62oY8kYwipS8xw(NxicH6erl@jBC)Ie+i7=)Q?y;7>Naa=_g~F2s*hGn4c6
zv0r+7SG=+*z3MM~>*RsE7F$X#T$eUKFDE;B%ft7EXBiZ}UgQ5K>>Q9D_(81ReRB5h
zebV0^H$KqSZf4}T)$^uG>CwHVyA7il+??1NI*adreptHivuypO>*pfMD;6woos^vF
zantnoW`Side?6`@-!7Q-NO}41o?wa8W&WQ3S>zcFt}a(JJt8jpm0dS|*N=|uPKLf{
z=_NHx3^(d+{>Cg4Yq)Xk1-Ce3gPv}$595lR6C`vSY~82$TQCHut#DgEv3_6Rzmwa(
z$mrH(*_kz7|GMkqlFa!_K3w#ynS4u!A*ICnc45p)mLB781}i6CJW$^4c1viI^weM{
zuUmhev!}&{aNbd`HD5TFhv7z*?Xv%t`{Ww9y^qP%Enq14|6-d!oOX`kir&SX4Q<WK
ze9J@`Kg`>uXqEN#O#F5Bw>R!iJUNYHW0JA9aV)3M?5I;IX<=zbOYE$~SF)uzadJCk
zT+TW5X4?Oyuc}vN#ksM*lzL>F=~6Q>kS}VVLEP2|hD?2v@J)MnX(u<Vn=Q@luw=LR
z;{_M?ukWo~%&M?0VXkpmWxoA;^OZIX8`oaEB=@LF>BGxAD<i(%?NQ$LNrcN0lz6*A
ziPu+7atpu1a=W){HpitzfBWn5UE@!U4#U=2><p35B%D?sdpn!4qi4_5IUD;8xh7q#
z(KlSl<WQaUWya3`HXbYwZWSInue<v7y&mg&6E+*uBlQxe98U|oX{}H^_5ACU-*eWn
z9o@l|&E8RByxQ}In5694dFr1TCakNCdr=&_YxTw0|2o-S`=qpIUyRxDc!$<wsa5$8
zs(nC?b9_2wg3$7c8danDH!rsEeJwGayIjxhG$ZePK8NB@cWQU$xp7}Rc94zXbwKd6
z^5;z)Z~q><%DP~$u*Jgk-yiqCuhz=Gv((X)VJ7cK;T4MingjTv&WT)ToL;|JyGKWv
zah=VD*r_wt#U3$YNRYa*fnnZ#?L$j7zRs+=mzjD@X0OlRW0QIhtNJpeWIwR>2f3#m
z6j*B;PSt|OJ{k1G3mw+Ftz~qme)WBAx+R0hPBB5Ygr4|`L2lQoZ{1Q=j(RoE<&W+0
z^{#(y1NA4aRotg+dZb?9l;de$_VSDTxvjfaPg^Ias{d44Ppo72+F!3%u{GR?+ETRU
z<EM{C{*sg8SA@su99nwf$P~{#Yv=8-d>iC+EY9fR62_#+H0F3C(+`4>bQtG#<Jp4}
zzwVx2=l0ApU7gJ~W5-wS=-;dj9Zx6E>X~c0I{&t%;!E4~HBz_tWYt-VwoE?VTbus3
zw&wSxcan46|C{GLvi~R*>%3=$yy=m8g;S2FxexC6_UP8LKee%$lQ-n*Y~Fu5WGX|0
zQK)_6wywvFHhWgu2rk}P=)Ti5;@?lkzc)JGm{jVO&0JsdPUY{~)gAuV*Kv5wuXOp?
z#mum9<pL{-TITz)Sr3#|#I=^UOxbb7+tK*njTaYPVPkSfb0*8r3t`@{aqA^pajsN`
zG@bb0*E#I>ugq@pcll$qEcRYo#bO)dOX+(jRb^MK&g8cY$-a>K@au`*`~6G|cF!)Z
z7T?wOqx0@c#ed9!d{JQ@M(-DWipiH)`QWtUr6`9tv1d0%?>c6)EH-=b8QvJ_t{g^(
z>L+ug8hSTh6ww#7$a@|6_1WfCSJST^pZ9SmKNrJ_#QA{*Ex)cE&brFN@XC6#navSC
zr&wLTnmNzvmxugH$l@sFt(jQ9vavtJ3RDDE-ntSZ$8cDJTXpAa@%eGq7Ik0NJH5B8
zb+6%LxOL*}!Rto%YpdU0v6`10@!zn{k3r;n|Bs%3GfU^+T)()yyHD1-EazcH@}$od
zzYdi?z2^OwTPp5)QNim#rH1SL@9ct&%5GhL{{Ds9UD0aZKYxz<2{F|CF^zSui7wCT
z`0rq1pCPHxw3X9(#l$+1=8e5CPV@Op5>is=bw2wlM>aZ)gW=YS-?v_Co+{?ck(O>O
z^j$1}J4<}d)hh*JzTd?dcgQZZnZ(Ahwq$eA<~MIRBU^nBf7D#Rv({ZSJ?KG9_f#f^
zBhQ$uA8;O6{vn@HGvTG$zl(QH+zb2g$do_KVDapW{_Y{`8SK@?8jL0upUHS&J&(yr
zOAr*zOn#E(niUUSRT<)>H|HPUm$r57!m#e1-mCeK|JiQIe%*PS!RmL_wQs9_KYvnh
z7hU=9hW_E}S2$$uU)242@a4_J^^70>PK(q_Vmq+6zK`iao_^NTof@Bc)~!1$Rr=Pn
z%q8Di^7FUAH?i~B1^&NSDN~xl;G;9=z#Nvcvk^JAXA_U+&lYFRW&LpaLj2vs`!kc2
z9}2yXvtHS>p!w!yZNmq>ze<ay20H&?UnFzo!yYN`WhUpo@hb{e?2>%Cd2UR#Dl5aa
zci#1{mEXP5+46nDnlB6tE4Ci@*W7mJy1z={gDtDR2Y9aA+p3eCsVj3k{8Gq4vn?0I
zB{ojY_{UOqecBe~%v&AoG5PLd32$;|-tc~@-s3VqYU4CMxBY$>*eCyS`g@%5>x7SP
z8jm(`I%zS0Ld)Xl+sLzQX=^i&9M6d}NH4CFF3X9KPPkpSt!DP&WvQE|>CeBFZ^_V*
zYgqI5j^^37U%!`r{l#<e(1)1c_x4Ixe$Pv}pnLVaDN}mh&Xv`MkJfwMd;OdBul04;
zn7yy}Fus2Frsdm}uKOR(E_?j1bW4BCjpcv#O`OYADs8;xf7jp3^5<XhCG^Y}H$75s
zaLSSQ{Uz6<FS=cCHQz6e+4$Vzan9ix+zYRKxcJd>)`wr)1sN+ULN}VaMPHsYQIfG?
zW6#>xDyx!@9%DT@{aLu_CVQK|n|crUHMOwh=2$Vj`@_m0neKVfZ8^h%hr%zf_7r@V
zGyO2}R4b?;^?S7ML)Dc-N^cbMZv4H>d)Hlcf8FW}4jNM;BcHx_Rr$6+qCxUC-}lwp
zmA}L0&0EK|z@u}2S=swTUl)E8FLOPAYB|^DxcN8#Z~OmcEt6VkPRZ)?^<pJojU6=o
zr=1p+;y9bT=E#Fb_53*#^|q`HUh>dah5eM{X?;60yZQH$*YO^`D;we4l=aS)x8cT;
zGvyY~4EHbWpSbJ6>PwF+-ah_5_vxqFU8@9xDnL0rGs#$6db7>?eS5bh-`F#^ba|e{
z<wJ}Ok=Ml5ea$ZBzMd~4Hiz-#nw(F4(L$%s9NBbc@dmSReP;5K6-(WdCuvFXSBo`V
zh_X1O`P*La@}<}>?(&90SJ*Qbi8=A~f<of0=M_zc?f-2`K7PENqpv;xuI<;-_LY-P
z&cC*lb1F~oD@W~Fv-@}6Gi7Z!u)KRC|8aHgiVZz8TNoGAEdHq+k@|vxQxW9M!ZO|{
z#{VB?9oTr6iFMuCKe5@$x*soe3N!3+|5sE|<L%7up~1ek;9Xqsx(8dDXG?`nHAs*&
zmSp~LGJgkmc!+h<ipstTMvWjBo!#?r#pB)CIT3q{Uw77TEm!8h+spF!Qt^>B|Cs;I
zwz`z#mCE#Vf#`*$8DE!Xn7Z`LWnq{UR3Rnp?GXv;<4g)WyZH3=`jVfsf=`J`IcQAm
z%;c$_>13I4bm_DCD|ls@6`$N)F-IuqQp83dHM0@{6?y21g)LLeAP0*%d1Nl$_CrT)
zSyS288>@~nOqh1ezV@uE?c=8jzs$w&?OL@jaM#<^d44&QS1~c<Pu@|Vv}OIh?R&eK
zZ||M=;gsgnnO~IVpN8&&&^xFa+Vxn(XZkXye;a>A)YRp}_e2C`{`a03<o0ySky9_N
zyCXKrw4BOJJ)r7(*P|%=L5NfDeXpG{lb^dT3{cWrxNhE@KR5SUi|gBcmb!5Fo%6Gz
zAiEuZ<Yi`CMb&!j`ugkO*V+E}y>effm9Cx9yh4MWl|fj#tjgU}w%aCNzt4Zt;Y&P!
z)?WI3G-b(~>w8u6(jpIj$y{$MJa3<!-u#*C7}IJRmh`S&_vT}N>nWwVr;kjMy?gG&
z>1nLT=k8zA`L|<(_ZN}3v!%Xf=A;~aw6M)uZ2j5YjoHkB{l}CVx_3B)Z{zAdm3eaN
z<?mt}lPo!=UOx5U<;zlM`};dyoKUU*eaHKv82itM`M1tnE<EvE_Sn+k@89Zbcm3Q_
zpgrwN_{TSY_=DA2?_6bQSa7q??(W;JvKYO0pE|$V*KYm3?%C<L&t=x?bjdjd1*{i5
zU2(SM>+)G*wa<$hivqUfYf4^r_-|>`7y54Zik*uetkt>s?riU#>`l|s7`80^QLXu-
zaK@EBRtq~{?qoq`hBXUf&0ns5o5}t;a(Dm!LU+|CJ$te@F?0znPw<NGoL=gd&Z9d&
ze&#=qqXk^z<rn{N756rtm&G+>>#^gJ911@hpSfN1cRa<awcMd|%4H|0SG^Q8OOBc`
zNSDQYnR{~`OO{${?Y|vYUjEydxp$d+Q~QFO{?X4(CN8=6>;0j=N^L)xF3Z9U-P=h~
zFXA7suK%)A^{UNsof~g<{yW~<p2xUJDb+jwz1-}(@BjOy>D1(_{F!j_gQ@@GX&<HB
zZk%7AcytX5>sR)#S@S9rFU#30%`c9rRc)QV`lH3X=vs}x%P(v3G8kOd_0BYH4a+@R
zrZew;pii9k{4H^I&pRIV{<|LY;JVwC_j=-a#kce750p07e~j5%75{6o(4IW;!<r&m
z{|?!|lwS8NxBBEEk4S!Qwg>qx!B4%8x;)+T<Wy#27c_-VsCs_n((WvV?c4HpOq-KG
z+riMFJnQD_$r22QBX;kyytUixg;l|!wrusZQX%FC|6;%WeRd+rf;T(u{jCMr-@nOk
zTD|4ghgf&M^?yGlpOY{DsKM}kp`D*-x$SbVxy?pFWg*Fb_ny5f+@)J7bulS#^|kvC
z8IIjrQ?ifICtF->edzr;m&$)u$t?f5|FGBJ_f-eWOQ*>P%{}{m?}EOL>&vD*sJU^p
zpzHW|)7gu6MxT72B~$WIU=~wAcEA1a55g0Tx=&?xPP=?a1d>j@8g}of-~Ml*Sy6FB
z=k@Z>HBv_TH4(2B<MpoC?)h<k?UN*{)}6hF+25ol#qV!*_m%d(+;35Q>EwgM47+}t
zt~gLUE9}*_A9}gHU*ZLJrbYYw+duF5|G?gY$!Y&*pbld^zrp0D^&-q&Ky+=TQqryK
zf)U-UAJ3bnKdNL9Fg&MbY1Zfai19|W(JntVFOICPACH$8>s(%5yKLuLf$C?M*SBT9
z*V`b*eJN<ian+yaZ(QjtH4$C1H1ly>=Hkg+ulG$W+;-;UpS4WSB<mL|nSSr-eE%-=
z%DI`W6aFl5`E^pTS^H^GR_@6e>9>Ra=bj0xh}Y0cu+&}95ooFEdO$rca^KTs=Q~bL
zs=wZH=c>&A%=3r$&D*+JJz$Pe<h;GcTW*~3;c=KZXU*=DNk=#3Y(Akq-*(4$Em!4<
z5n*dX53dq%?rSm4Do<xo+!I#)@}9S9zFHcC#^m|-Rg+j23Mj42<*nXa^Q%N_zHP-9
zHCZRcC1TsYnWf#le_3SZdmq2|c`+0A*M0kPRJ+wp!*J`0ml4j>vhO2}ru_RI&N0=J
zJyj~DM53}HHuOP5nCW(7r<q0W<xhouUTS~abS_XxNN$B#+v+&R*je4RuQ+ngN;Is`
zK6Ncm;nWUhq4Hf*?7kbN2YZ{`KCY2{KIQBi*)H`2rd?9iFWVwDOybu~eBRlzQGn5M
zcZ8)y_r8xOU)6>>#oTe7zg=*4+VA4aLRa>)YIZ*sI>hrbIYxc;#Yxs*_^&VBI`Q3c
z$InrJzsfzWZkKq?uaYVq#li6J*xUXrAC=Tl&FR5!4eTQq=UiX#I{nhLJC|<V;7%{U
z7{7bne2>}N)ZX7H(OUlAMquh?uLsXI${O~3HlOcnvR+Vllfj(pEblCTpA46MV`Ls+
zV|8)W_3)Ygy=?!AzWttOq;dYQW!<r9vl(3G)^|o+3+B;~oB#h{eG<p~((^Zqk32b+
zcQtf<aO~e-=IbZcGt@k~yxWGq;p?hs=Q9kRiL1rWM-&?THdB6{|7X$m@HC}a6`yb1
zag3BTXWXEBEceIL&1z<<h6}WsKb@S$S}gvqvthx^?r%Xn6^*wy95@@l`^2WGq7vSZ
zZij2M3!H7VN~8C9B+RaKUSD;+ZTb3@>sK&x{4>9H<Mb?F?|}1hldY<JU;T^VYuLK@
zqCcY>Ly8zvbZxLg@czaVFL=MluW>6s9)8>Vj$g}<h^sdeTK~xfoIQJQVFeSzpRU-K
z>lq5}{CeB9$NojnN{4d8T9MWJcUbO_IQv=9`=I+g59z&XrD+=;g>sgxFUrmM5?j4d
zb2YEElKft^(!;)d4I8&+INeUl6ua<mhuzndV<&Dtv3|Ap;;PS`Q4HIaFC7>Eb%^c2
z(a`$`%Ju$#Jh5bs_S>RU`nM;3J=^nFJ@=%pZo`k?uT%qN8lIb9p8h0U-23U5yDR2|
z)y!(IekL#cRpR!Ap2D|M517~vOcPzWesiu;;j6O0N7~X77&hE9K3{h~n2jOneq?r=
z@tOm*F&`$)OJZ2Lz3#(&i6*P=Cgnwo7jHky$sn*N_356Qo5gjy1D>z4K3z3U=+pCX
z!v~BdFJ9<5AKJCi&NxiwjZNv*{TDA@ym-A#SAsX;^P7voW-qF{_wCuT^6!C6^_gYb
zeJ(EU?(x=}(=84c{a+`2^XF;p_1^EgSJ`}Vp5I!nw8uWSR5$DJz89al*(x&L=SXjQ
zm$Oa!#*+G`euf#ZE^lBmKeE{9`GOu6$<X_^X7}H_FTSd~f5$YA2RpVpo>FIM_<Q{F
zdDSe&j@;gT1%hw*{pVN3)K9FDSUO{acJ0l=qg=J>wa22E7;dG{YA%#v=(xea{`V|z
zva$iI(7P4qo|JdKx)Y<q_Q7p$RQ*%op6q;<2S0s}o|4dQ_>n$q|2omnM|&r3n!Kl%
z!S0jCuRaU812g|$+4XwM($I}lJA8kO=rb^+opTGe@~{dMRIv}x+Yz><FaPbGue>!T
zm$o?>d05_+S#^asw|t%csT_sX`G*;9F{gk2Ul(LqYp%6i^xkRi_HUnVt#8{R%IstM
zzR*^MVaxXEt@mnX*BT#NcJuoE?@R|=b*%pT#A}{YKX&ioR^??p8<H6x{JWT|xF_sJ
zQh>C(Fas~U@9Z~POb={JolCBCT{GSEe8DI6?{dfOt7S^x9k{bEcmKWFYz<%M{k*%u
z?K}V4$f`iqg85(Wwv-t$B)tAQ<KOciRSbGMlEpt7nNGFW_eC-#9Gbt``o?UAgtf2E
zS$$j1Es^2h8(p&^!+3>gqnF%z)&n6Irv?T0v#n#e63A~nJKRKnaZ?HJ$9bY<E_?i*
zo^{-uxGHK-z4ZDhorW#qv$Su<-7R&k4=gNBo@n~a;rDyf+KW}QFM8U4epGDXR6jA}
zLd5+eSC>xo;Q9Ba{^y@%#%729ozq!xS9HylTI<|7ty?1){w&S8t^MR^&GBcC6t^%e
zcy`E0OZSJ!*>9^~wfmb|pWn0DXjR6R)g^OZMKn&IEl_k;{-Nt`k6`Bb)S#-TQ~aOn
z?YY3zFmuClRR)8gZA;fG==#Y{d>^@T&OYI(m%S8JYYwwrt4`}@6JuZ~eD1&h`reHd
zr)S2ToHIMWJ~s5lyA7*%h0D#c6wu7Lzc|wOZZKrLkHH=~-gn0Ml83Zq?SlH{>+ZJs
z{mFYD%X@pz@19oa^Xp_Yc`H*c*fGrgwPo|UTeEU+eiy#G_`~OoD>sUN`xM1tRgxE?
zTg<&Is=M>=QidyviRIn952P8CeV2ao>e@zs?R18KeRV&A_!=fkrrxtSHrHN?dEX)P
zBa4H+vtM=K^k8gQnW%bTb9v}SgCNJ=KWQ<ZJI~ubOgMgM2Uqs=sp&?GnqDjXP&r$l
z|3gfy=SJ9#vk|2#z8B6-JenO^SIlK(yvy^5!xPS<6HWelpPduI5Rje!d~@x_k1NhG
zd2~PDRIX`V8(6t(hrdX9wCFpvH(A;Y6AJueugK;6VzcX$`k|%AJoWN})vwmwJ;HE#
z)y$0a9k({`J0bFC=INi$;zIoHH=oLEUB06}@1d(Ii@Ni7?~s*R2TpiYI)0z_>%uSY
z;N7Js;wm%mB^=@CKcQ=`RA(<@eC4$3AFVIH6HcnHex3HE?aGIj3?}O5?@s4mrJu|I
zAHV#4CPJj-&h4f^2BDcdr|K-8zZ|><)@${Jh4)gzFP_@{Dp-7Ra_hyPe>HQI=lgH?
zUY_9}TR$b{{$iD5%o};x>qJciJnDA5|Gxj(pSc%fO;=Cexlggq$u+;jJ#`)X)5}}+
zjh9u6N$>w3wzwf@ZvEol^EUfDv^HpeG5ZP2TN~D8eood8Ue*5$;qT#^P~}zdcm9^C
zH-FE6xo;BFlnX`~R#|=Q2{~FTq9&zZwEXT8ADD92;mzd#SGrg)nSZcd%C?W|bih*+
zRX^di3=GEZiM9)hV-=s=Ir!(={3S)d;zInit@Mh{yD_Yb%YS}$-u8rpO|0C{&hl3-
zc`d|cmzMqE>YUq~&sOs@AIN5MUcdM15!W@3c2?R-r|*kh9hl8<VcxIW)$87SpW51C
zz>xf>=y?jy>-Q0-kDWTUtgY8KjERfE!+GDCoVO7SdtyQ_yj`=iH+12@7yU~8H}>Ax
z9NI5`Zbly8|Mcr~8s5B5H9Jzi<^}J^6=y6p4jfl|wZG*XZ_lf}`Ps3PGGbb{o_-(u
zRC?WqQs%<#ru`NSE8^U37ETn9Dc3o#(avbwV||M&&wYu0=ehMuHZZxLu8Gr2OUpTI
zBQEVfJ1JL6VB=(|^m)z(OO{{UZFWv`aU*|uUI24EYubvUQ|&L82Z?`qsZcik!9(5B
zdx{nze)qdiWgeYgmHl_x>_-z<H(g_QP4D31;<DQ7HoeVl&7BhM^xfM!qi?@lr^F!L
zC&R;MsZlWh*;|d1-}3fv*c#8!5n;FT<c-M9zqZX4o^-GH*s6_Zi|^+?ogr_cp3HDy
zW%tF(=NlL<tci46yZT$mm&I4+<aaSRTsU*{n$G#Uqe_{lZ<IZLo?9gU^L6Wi{8?8F
z&Ts9Qv`#VZ%KAs<M-~@7UvQ&3YU%sshc-3O7LRy-cj?+6#o>3Z*G^Q;nJptOx@;8-
z!<3b-S;?2$?mlEV(DQYRn9cshr$TKn+;zXTcbS5jl5O?zdPm-is{)0D|8n<CuMRsf
z^Yr)5u9f+%R>>EqH}qNR<*mKtwf~KBq29d73<p-oB}GrOFIIUwONOE4np^yJ-8bb%
ziyQl`U3qInXDvMS^6TkY;S3hXZu+XQl`CnL_vdOYZ*_dS<<Rc>l6sXN@2kyEezgi-
zdi2V_V+;#yw?6&ycTRrPFJDPEhpp)s=S&s1oB3$^>UV(}Qy6D#ta%%+{O8ZIb)0sB
zA5YIHtCW7kt%6u1+V^zUB<OJTs;5uaR%`ybfBR;+Z;cdtgP=vksRp4J8&-65M^prI
zCKSB?wrjt~lIS<#|DP{?A@xYTZ2b@3k1L*7YRq4BbGG5cuG-|2*L^?#>F#C`X6|XP
z?8xaYd^2rbP)p`@tJQo5wk|v&#<0cx+)73bBetK4k0(v3yt;B$@uUOX(?cZx9@!#Z
z)aRCUKg{UB%+?s;iERx23*YU3$J<-?Uel~<@!h}k-O4Au;#E;Ldgk$Y!_~jmMqg#^
zzIf06{Hf}T{nmFnX?>5D-tDnj+R!2SK>6pD=GvOjuZ^Gio=Fxzy%lihqT2+q<x%p>
zW5w3Z-+F5A+#Rv=VvX%?$V}GXR-DMVB#=4YHGN0Xskt==UPfKL{K#bIvb8G`o|@cO
zTIT$8%AW2=-)6lp=f1OiYpLnU&vLi3p5)wJ{2;&F{^ZZAt8Qv7uvTU`u;JfaV>jiO
z{FVPZ&cB!+`F-YJtE%lUZfs0GF4kwZlW$$<(oYN$$0BCbPtwS{S<l6gp={0`f4b)D
z+pWQ`EBX@u9{*wKIp@}kS4sLhcS~%w{e3MrmNGZY`!D@SDUD%I@A(<At6tn(5wPUH
zsF6^bXr*u0zO-+ZHA)NxoYxn<WBT=P!JpIGyV`zidndYIQEqCLj}XIy7avOsCMD)M
zH57iIv~KO`*u6{l&0CP#Gjms;l0N%FfsVaX?EV|E7d)4_VP?kkc$S;}wySy;t5-hM
zne|SWsleK4rF8d&xcKJ>cUWxfc*bC|<yw4c%qtGT`B&2xNLSpx8n&Qv?MXYSm9^d{
zCHNH?PP81he){YBL;aZ0&maGEiZ5EXjj7?|OU=(6k0&i<GXEPHa65X}#))a))b5^H
z{60PR@U53p4lX^rHRq+c96FxOXZxVadJ}_=Y|lQKmHHmD%wOHt7t8gV*5sG?c){;4
zTW0sKC{4d3eVVs8jj`;Bf@ji%M{d_t=V!iOq<`R9C~JclgL10vY7T}wJA#|KdnXwk
z3V8aZV9BoO%Gzx8n$Nd*iLRUTEci&tiaC?Y1MTjAJ=`G>r@ba~Nr=+?cjwi%$x0sd
zj60pZ?QHDrU*9=zzA{qPV_sI-u4DV&Zk>N@WS`sf>i;sV2kypQIaM6^tQ%=qzVqA5
zs~hC?wHQC#c_{q+xaF4O)4Mx@z-uo5J>NX_dH-Ld8sFXnA7$?bp{%=PI31`RHt)5S
zVo-I~k1zJ$_5NOQomTefy}VdlNLSUle?|vn85H(cyPD5j@h5f9o^`eNY<|biKR4I=
z*O%`<FHb*l;$myi-{V`-7Tv$hX#4IB=f$&`t2$@cm!4tTnfY4cgWF%Lou{<woNXA|
zp4;!KP5JfZ<>_g<+2-0&Nty9IhEwfkOTB5?HpxnC=l_$J?`81K4Ust~)p3C5gJ5}?
zSemHG)^9&Fk52m?)$lWCLSc7~yGK{3o0n^BHrJ;8Mt>`=@o!t%oKb21@vCrVWN=*f
zjrikU+%j<r*U#KrwL8Obag~SFwKF=!f4a>#Y)vrXdb@V_s|^{YB@5dgaXsrXS~oen
zp>0<5UOU_3Z!;NVRqx7fUbmjJa8;{2SKNwa8q0UQ&)c$M@4u>_7FxCESkE2v{E{qQ
zc#ZvZ;p*$l+!?;k?n<wC`tHrQ4_Uu{nLONcO{pPzkJ{ctSJuyoy|crw^zrNK>&4$E
z9`BLdT=@8yU-9FXNuTyUo9%e&W%^;?s1399%~PZ87bQ)-+|+gBPWK$%sh9uE`L)sS
z^Raj7=jZ9(JIp6%Q*nJ=Z2rAHf}LF!&rT;i*?nx=<)=*y2X?ICxIMY{&+osro2$>>
zoL|cxc)s#^-OJfC+p^}#Wk0cgyZnJSW1r>zM=K}JyO;dQcIk!mbN_Xw*K9X`-l6Q+
zZYw_R`dRhN;5hD^TXL>_@_6xCX4}yZ`=9$|&eNS5ax6an|LynM{HbSmnSJ`bbuxR^
z>4q!T?0bLL{_gv6U-9FmZ{HTzpZof#r}X&oEi2dx1Yhry+5ct5yQ7Ww|Jd$ZT2r+C
zrQrj`jLX}f{akST7Uye!-?Nv0Zk3s7Q~Bvwue7kR@Y*%^j1P1dUC=Soe4KGIWTK&w
z*aLy@P1+BX4qDtcc;=*GT2u1J{cAziOO=jY>(6dK{-xS5XxTi?WGzRhCNs0SAGaKI
zXPay<_&D=yN8@zQTk0?0-`{g(#tg1K*+#2U-p))HT|Qaz>4Mh4Q%|oP33|Yp>vD<x
zo!`?fkERBna_r{+{-!X=w0d^T)EU>y&dZzSKkq4R|9y4po?2~x|8VAjB}d||cb7l>
zwcT28&HDH)R!*s(UT^2(ZE>18HS{0nlAw8i!qzdgY|;wb6!fxfrBB`8u*NelmrOai
zURYG>vsUPn^N;Q=;m&=1WA@bN_ZGCCchs(qt||LeVrRA2z<x?ij$y2;-ktjYyYyMv
zCMrkSMa6R-@sUh<qxkT1*{f43DKpczi+tw38M6G@;^H_PnV6n^dD@p&aQ$P6a9i=n
z?0u-qlmn+)W(EasIi9z_-Z<#vlp;%C-%HP|^=lQMM*DT|7YMXeeJT`v_@3Hz@BH;|
zCCu{E@2-?Swxxg}VVbGg-K}-Gc@KZ}vX(H^vTvL@_3ZBU9o?yEW}CkEHBE2V{iF9}
z<&hgZ4(MLGbSa8ww&@zzvrOAH6x24(4COUum~hkj**_O8lT(h`3dx+G-^pz0TB$H!
z(W!Oug9>HA<S*+E=JW(883?9EOLqxO^3@bk(ptV`PL)t8*MYb8{IOrkbMxlkELmJr
z_WjWNzjo7oB7?6d7>jW-{_LFG9}~;P^Wfdx-Ph{kr=<Bjm3_KElxKg^_4ciA#l!y8
ze*Ixua#iWc;i*oFi;BEXY3^oQ+fZNO``|`iBh&S?f9tkRoL2GT2q(im<$6n-eM=uY
z3I9I#>vs9Hhv(u21HpI0PB1!%e_kc5>t&agC}ajCC@kdji%(zWWA{CqQq;;O;FVID
z9;YPA0=cqIGdW_T&$RMmfk6vHUX_Axg>&j~46w~;4Z0uuB6zZI<;qC$*p|fi*UXyo
zzPkQl{yMFHhIW@gQob_u;HaP@PmgT+y&))u$Hk|bKX7%{pZgDAH7(m=9+9)8sN(N~
zv_(57cTautcXk~^?c+K<r%snM(_E76)$7aNpF7%peh*v8cH<in|2Fj%UFd3gdDktd
zopsL58P=KUT!tFkLcLGkVEc9RT2lNajVLEO9%C(sOBciHWrOXm)_u5>E+5d<-PkeL
z-YIMOY{|KgbpitTvN~>fsDAphKF+!9>kRpi-sdM}H2e4Rp5FQ{P%V6W-`m~4%U)l+
zd3Bk@rTC+pKim7v6e$X4W@t*Y{#CMlu^lVxf^9#zRAcKy9|&`oS9(;eva1W8{Qa1L
zywWkhUk3{XgEoGAzJ3*->&xz-iI>mMX<fZy=?;qw|M;-I7v}n%*5<l*PGIB2Q%WC$
zeuT{nmXE4e+!r9ebdgcvYf;(Shy6V_=iYJuCnQss@_X0P$?O&P_ZEM8e}CV*+b6RR
z?_4flujFZZUX$DG{Hs8528G3|_LVP?*`o9?oPFurf9@Nr$`hxj^>scjb6LT?ahtqx
z?TVZx?b>LLlkuR_ahwFhGS91pKh!GH)MLx~_rmmQr}}%}dAx$sD|Mcvo9VMN*rmPR
zw_3GK>~h%MKPCNdWMXBqZ<uFXe7afwn%w;r>*ZsOGQUl|F28r56nuQYdX87arY<g3
zQ`c&XsNf}A*Uoghv$*tyLZ(LJ9IFPwPkMh=mxm~dDda!=)esb;<02XR=-x$%%oR60
zT9<e3HhA8$?~luxJ0;$6Hrz?iix-`7U$v(F>K-;-$p?%Kw@$23&+mTf^306yXwjUm
zUad}E_ahdwQ*47ZeyA8+-Te2Dw#lE#A`Bb0r<Ba~oTvVAm+OQ#*&BS<uqb`_xPGUx
z<@L_AB~5u-XI$RCbIxg1L*2Qorm0OPOU>R1iqFkItIp1#JNM=+BSwpVAzR$SS6*JV
zG-Zo4_kyma2X0MiZ}JaFm%Z$E=)~nM+vk5~_`gIzbX7}@fpD<n?x;0}KYpaVJ!_G>
z&dWX9+i~+Ori_Y$9lu+;&;HC!S<;kO$`W{Smb%iiTG_^}5d!~Xiq+<^CqRbeC!Vqo
ze6BHX=F@k3(@&ne8D!7(Fn86y6DeXTtxoX~Tv{>jHIAAIxvhN|JbCL*E;|N)^=poc
zZDo$%zP-VB6}OAmzbm=E6IZ8M?qRf<d*j!%CKJ{A_Y>8uRMQ12lZ6d+vpKU)v1Ikt
zOP+T<@|@$$Ew>A2+!QW7IO+XvDVwKe&B=97+)gd`R?Jp>VURlKW!L`Zg|=J2fB9?q
zuXNMBXH!lVrEWWWG%YQO;nS0pyKkknr=`wz4f*-z;NEaesm>#tvert4wa$v2Qkwao
z%hsInPVEOKULB*p&o6wfTAVy|bKZWAE{>b|=g-B-Djcl>p1IjKW~aXXzH(t|0e53j
zK=a0xs<YlEPx)!`u<y}?Ad6*=*Y<mSVt)GkVOB8vMgF@-oERQ7Z#&T(SytAtrBCOz
zMYAwyG(P#@mhU2EtJGJeDl#0uyLYbK>`Eo|i*xjnF6S)wt2=D#e~D{pQ+v2*w6Ia!
z)`aIg=I3mR`k&qTrTdcqb%*^~j+3{XH{D=<V6yqT);%kZ`gdxEtW&t+bP^df;#a82
z1c*L<?(AW9<_JUQ_S|pTobr`cwNG>J`tUVr)*g+!X1UIUlRt#}aOnC5y|YK{N?W$2
z@f^5rnwo7ckUE96aJl!(V*Tv)pC_eKckhU?pS<nKv}?!WEOvMQt19~Ia%+;r+ui3U
zS;uHvZ=AnnRbKSYXQ$V8xoBD^FSz3Le_z?(Ut4Op!xLJrt)6F)uet8M=Bz7GbM|_F
zi!c?JT6N=RkNL)w<J-k{-s0$Wy`YjYM~Zt-&R5<T+o*#29IySK{`H%?zgf(@WZq2q
z(rlZ3D`ymkH8a&E-r7E;CqqGm$;(t<%#h*NQLTMe>sBWwxqY2bvgTyd<I4v<A{lBv
zlvO5p?R%~|HR9FU+|_wrc9G^EgD=<rw-e#4`(Dm|{I~!5c^~&Zy4V@Eci*WycQPL6
zr|*BS+5G9w{Kx4zJ2@S${G8sAYJ0Td(Z`?if7IJPL>BM)B4fC7V|3}&*CFBi|DDYi
z2|0b<cCYl}>L)LQgdHB8O)h5td;et4@x<t!+aFG+`)*>>Rc&1xXOn#N;2U>I%lgOS
zd-rBA1U(Fl-gEHC@l|zw7w)U8+6UB?dKLd_N^+gDuk^vkrC-h;zRD2v=t{TVO2#Aq
zCf5a=)sxd&t9*H$<t($R-kN$H_5(MfzLdr*vzK4&xaPK1hlOFmIjPVCGd30P>hL`n
zZ#&Vf$~VhHhW)^qrqW`Os~bf-HXVI@$05dj(caqN(&xPV3`*PTtpo4aUiEhu*|lq4
zd${Q4?WgN`L+luhmu@~H^5^4&E1a{R1UHGkWY{q&%Sv(Tya%%bjq;<-H2uClopt5n
z>&x+s442pJJtED(@?SdpP3qZYsw>jhd&=%r3k}y>BimcYWx%AcS>VR3m#b7}{^E6U
z%ir-o$#j?0)LU_n%9QGE_Fb9$@t3ym8~aS}yiCR;_rvu&A2HbMpSjN4;ePG3m>(tY
zPH&ZLH*I1)u)O&4wI%QBS6K@t>pI>$wDq5s{Dr6~+eNBPMbBsGPHWm<V3PY~))md)
zm*N=(HrCnpvfpNKcst7}S<s~6Lez`Y(>6V!_Fm5z9S-qjMjz>7<9wZYdfDEGaVaKk
zx2iq%eG_@|n>+b&ipjFZ0FHk?1+QAZetPe}njylv`}j2m26q3&eWsDAO<7-}ex_#!
zAD*AZxJpXtXNR?D$)xM&UPsCcec$-lmf^#z(u31C=e<+$ySr}wjSD<H4A-ik-F#nt
zioxNHb@thUHqjmZ+;NwJc@Rr?3tUP{zHGB(WN6yHW{!FFx29CVS20ub+#>yh-?!9k
zd&<VZVRdD@ab;y@@bAXUenxM@WI8+g`u3HjoiAv6`Y+ae)92}udtHUE^?seX@lnF_
zm8<{t_VxAc+m?MWgJE`h|GZ6#n^f#qb+r`*IPa?V{oBFJu=&**V}=KpSf}PCH0<4M
zuzB6)xr+rJd~?mN{@t~jiC<ad_4{nby8nf(dF2bYZun+<^@r&v+rlV@6Jf!ZmoA;C
z$zUojw3v0n^NyhM8c>i({rt6K&5=Om>wh+C<{pf^s=xcvyOoRw&bS4Ho;|MiVb`9B
z$;)ad9dcLy*rWMcfr;U6#<DBvvz!Z~HVU4tck^;#m)|&>WtRD?o1issD|Qx3&W!O5
zF8L^UU5HU;yP3yN7KRP~D}L>C(FHBH71wL}`|xFO|Lg1*@@|@HZ@l8mVm3ZnwQte6
zC?<y4&Qo{ovV8t)(V3#~`)!O{*v}Om_<ep-{*(10zdlW!=^NL1^?u4K#wVIF>3Z6q
zrzN?a;`=VdSRuDGqBx#g!F1ki$u(CP7#?2_Rq}kaY^`T+m;;By+1$HI^Q&Wi9_}lN
z^)xru&pP--*?)=et*QUMhW(ruQEMLh{aMoD+duyt^5`-CCB1~f;f}2yE5i(_FeV1y
ztj!MBOc<^Ew5K){3NBWh>Tqabe%a=G#<wlJ7))N9Mty&^>g@4N<+l@ZX4U?!)n|w~
zvgq^qFHbxCd-fjAcK4A?IL>t7@8a7|6>B#fU0~Ly#vsKv%j{+-%Yo`wZ7)`cuFZaO
z!ZCQheT|s>{~vi*_FesK^gQQR(cwA29=YS5mhs@cW4|TsrR&rKwU^3;Z=GalSbIJB
z*Zjw?8Fy?*IrEU+^}R$~WFTKw#I4Ev77P<M*zxU(FU~z9t;-O>@z;m3!BEs}^&yqJ
zf%hBsb*|mEz2t1f%y;{veswo4Shw|^<GWKHe=V5xK5MP8-*>}l{@1jwy=!(&3Ur^i
zX!V?lMq)o%&uUkwG8okT?K!?{14Ca-u<O0Wir!09IIOHTG05EuNROMqHz(}h_mJYR
ztBg!qKdw5;<Warw<h@6Gv>&g2^XwSYhAZ(^TmHvgj^A}p?X|XD>Ftj%@3e|XF&GGc
z+rq-&c52%e(IrP$Dl;qyNY}Kp^x9Y}3qHSPlP~xB+GYEu@i0u-_|}*4Me)Hc&u1lP
z{%YP6$SiLVIAxZ#z~cV}PJP<%Ui@>Jc;K+f{2Ot{i`R?P+>$=~QiNeey!(W`em{J*
z_#JYxw`^(OF^{9hs$46qcKW(=D$BQD?_2w5=gaxuFMaKpXi@rW<8G0LXFvX2ww!Ef
ze*fRDRTu6i{%ZWpe@>=DWXt=DYnHuR8FiMqLFcpH$yJ5F(-@SL^HrZ)<{ycVFME4s
zW%T`p#bIyQ9vCP6dZzrpCof9Z^xL7hIBv)GbG@rl^Uj218*bb$(qN_XW?c+ofqCEY
zZEt>ez6<>EGh)J_h5r_?mHq0f>M+)3yS$-MpfKxL(2H2t^5g>*=Avx#E^W!z6Kdw)
zzb4V=rj`za-M7A9p~qTX*EV`^Y^Y#V%V*+pShCx~^M=^63%vzZ^Uf^hny{{RUa@!L
zwbUv7o=>D^ZafiIKe>6<?pZ6|Mz}JBU064Hi>vb4Z2?D5rOSVJ6qqD`Y|6$&#t+-&
zQcKS~3f__0Zo$wqWv$|=^z~DI-x203)9G2gp)B(3{+<;NVhn$*+Ldi?cwkGG+AYa-
z-`40GvDQD^B*maCD)LOn-|k0$&GKL0eqI-esaTPD%Ynn7G@DakZs0e2JIQksw|rWj
zY+=qXmLQYM>Tz;!UsTqfXQzFo7-DC2SDT8>d{*=QQta6R^DvHxYa+F#VHJN@6`d@6
zW#m^BIPdO9>4swU=HA5;q6sNUB|kUR{kS;CR=hTS+v#6Hv&|(84=ER&?(MkGf0XBI
zk>Gx=zjhIyPQKb@^w)Mqn?L&k%gT5AY65B{nKzj6nK4AP&)Rh7dF#ak@%_Hp>pVjp
zl9KZBrdbxfeA)Zr&!ZISpEXT;_d8c>X+28|NwJ&j)q;Hdiu|*m59K{moxUz$JEorb
zulg{v!8*-PY`ZGDjoM`+tT|UVy?k-zQLmZ_>xCEUQ!7e>xBgo5Z-1$H$m;W}!sZ9b
zo-p<Zdwz<!c6Rc0y(CEn1KyknhV$<;@0nVCtqtGgTYXu7@5T9nIjS<&SDw6XQ{8w&
za^;sV@zux9nf;ev&6KzJ>(+*ZnNE7ulZ2LchA}kET)aW<!p?O9Meh=74CgPEII24J
z%&A?LrwenM^wgi4vWBd<WzAVHn{BjaqlfVFS^NDb*6s*=W&Cx|<x@Qe&%hS<ciATe
z^<~Dnt+8Gruw&OgyR8-XnpnBTbY_@W$MtmYo9ef-^zc)e&83_0&rZFL<YNe0=i0(3
z@p$_AuPbBk-U>?#Gc(P)5iwIV?Zuv*y`j$@Z~0?xxOr<b<A)+qQ*JS#rwiJ|i+syv
zg-bH@UcEYyB@|*mH-NFBWA2U6815aiOY>Vrz$cgN<w!osCUQYp_uaNv>sL!LBxPSb
zwei^L<EL#>J?a+5{S<s1$-Ke%Y51E->pKF{;)?F<Ro&Id(ITS0(C+{0w-bJA9h|}S
z%HxZ9+Og1x-G9D??R&^|SL;~2>uc|%`Fu&`oO1V0MQ!@D_*{_9bme1<!}g`lsTYYi
zI+@|uFZ{{U`)}JL`Q09?x_+Dx&b-{e;@#2@Hz$`02djPDR`uxF7H>{T`9>d$%m$16
zUXhhs@>=9O66gGn*lV2;A5zk`V8Mc|lcO0M%!+<blzFW<W5?FH@3fdUJUnsm!Gva}
z@0s`dCgr^UhI6Ze-Scy1_8E3tmM@ZD`mDy2g`p`;Z<|r<_GiK3i$AYDm3eJVp50}O
ziGKq4vMLI<o9?>E{I2(C=t1|mpyNj};ushnztL!!#l3&(<LmpkO<z*n*Y~&S=HDf?
zHk)D>7RP-Kk>P*v(D!Ml#Mxf!Q)>@?I?u21d7JdJ70*w+Uo~?lW3tv9>yT&4hn8l)
zTABUiVc{<GFU*1*ntoC*v@Yq)b=|vZ``4};N1c1!Eg$TAr=*a0J0k6V?@BF3hSV&<
zW!XFGB0qC69JkordE|kg%gg;AYj%BGd~VX-X(4m|b1baMk~EI%?73)Pv}W>Rvuih_
z!teYD->El+Rp$9&i`UN<J=)^G>3YGd{*xu^^qCk~w`;p`H(Xcv9lEzz{bhcvYt61Q
z*MMJVl(}O0l@_jCV8y}6ppmpq-%9O`$)cIQ@20V>k7F+7@6*5c+2DBjjZ68b3>a3~
zc0XIQ=fg=6`_*%-xmt}FjHj*to82=1{uR@M>PmHzm)Ci0x81sDPbvSQ|9j{D{d#11
zA%jF?^U~D0+wLA^Fi}7JP9<Ic_ep;LRg5=oL^W1fiSeoOrZ#@_iE&f(zLwMbnDOn+
zp72R>FWVhIyo{N$?`O0D#vvqG3|V*mgdG3&s9ZgEGox!dhs^e`YpOm+UVXLsRWj?A
zTj48b{Y*`mvnoaU!mL2S(6zG|9BR|IwYG#eXTDsvpC!+J(FM~aG3R&_r=mxfcXt1(
zvU9xecuYAaLh`{Qg%|^`DA7CnvhQq+-gU}*S?udI9}U*=x^81wU|ZzJ>oC!<!^~Lm
z^qUtkroX4me&A>Kpd|kB7K^hp*T}`!B%Wh35Zjfvc;_X%i4NwECK#q3H$D3zBFf?6
zC$qD$U5}GZcfHg<vTadaq@=;)$4V>{{y5aCRPRoh*K^q;>ItXzrmsxmKXz|qkc$s}
zS=U;0RPCZ@rQqUhtFNCDHrLO|5(VuZIa>(cJ+i`B<K)A02YVIwl-)W}Q{<9(cG|9q
zo_o{^PItQBypJ|@rtwbxqvT)X*LIuZ-QSDJT9>`Kap>YPmYo?!``X=prgsW0H`}%9
z8n?H3?0GN4V85KZ3=5wBZ`;vv6zRAWYs7IW_Ua7zkK@z-7{33lpk;W<@pQ}LiCM9!
z=f8UYO8b6zx~UYygY&T`cM0WL&c0Zokkrm^5@{W`tzyBJKA!W7MXV<?aM&0fGqy}A
znV0=kkA-2{T-$>w$;axmkDs3QZR!?dR)*cLel1)x&GgILo}Q2e@AvB_-SIuR{KKxP
zr*-Zgv1gMw|6$^`4T1~>dAgD@&5Q|2hi`c@eArmH^rD)F)@<41M!RY@FwOV7G*#GV
z-)psvsXw{hr-nDYo<8;Yqpb`**;0RRglRCx%yGTE*Oej9U8D48WOaG$x+6O*UcR3!
z^-5b%a{8R9-R)mhC+;uNJwBuP(43?9I<JOGZBk>34czt5!M;7@blx#<cFBARt+h-2
z9cS)SPKh^9{j3>tEoWZPn!lzo*H=$^dDHz>K4-^_&f2A=E)6l8)na%L9BEst%E<7z
z=ck@Tuw$qo1IOAMWtN}bFm*<_ec#`y+d6Mq&t6{Z^c@>zySIq1Vx3;PGyPq>L~C8I
zRN145$Ek}`g7<t4FHO&Xd5m)tn^Hq+dWrF+t<AqIH!@7=Y<(~5sI_VN^l#s0moqRd
zIP<<XvGlsxF8AGOXKcS(MCa~5l6t@-pW)r(UnzZVmv(+XAINa$t?OEw&0pSHr`>y1
z>;J@t!SA0<iuZ)P$HC9f9o=(oyWqj1D_v919&iB9q9so<eK5&?&A+;bu1i-uf1}TQ
zhcV=%)W7FX!WjDg8r7UVX)W{N8_IT@3HSFdnynG?&uHE>Hf@2!pXKeg{`m6p@^szk
zYk6T?j$GO&k-TcoGCuRntR*w|#!bBFb*{!VEKEPvwzA#m_xJ0^Jb&b>ce<waDlhNt
z30T+jEN703eYs%z{yDFvMJD;QO1Fs_#5pazStPE!Y3&&n)9t^N%r}&N`x*LU!P_E7
z@63GnznaPVOV9T|dfsoe?X2OHxax;3NxnQ!7sx+-^XORVD(U$Bb$cuLt<%oT=#@4v
zn<cG2Wyb#c*G4CgxLNFOKVnu~uq&$iPsFcjYi{PwWVCy9?%bg@Pd%^Pdc0gD`dv2T
zv(1jjbagiAg?Mf}<*{-_wy)#2@MZGnPg&SY=Ur;LTC>x0V*1y`<#SjU|B@)*a<b67
z{;$uiNzr@i7Ai0_7&-3!F4tDseQDCp+22j3Cz%L(*1Z3p{{GzKq+7o&8tTvMoV~JT
zy2_Iv&B#qD_cIrUXie(Mxn39h`t0MV^{?ZsukY*5S?l6y?YfQ2b482T($MP5I?sP5
zJ9Rjnow@S;@6!15!Ix(i+hv`b`Tozje8b-rMSH6bW-mYd=aUW_1F!C}MS*L|w%#lJ
zd};N6?p5blE|^XDnO*U=I7~ty%J$Zj*wvpz=6zhj@%xlPSO0$-)?Y@&Vdh(J{pb30
zllk4V>KcX>pSd<`O}vx#^`8S%;uT)K(;Zjid)XL7Ep7J499qi$_wb%ThJZ<jZ$Cai
z^_$d}Uv6{j_VO%RyjZ$?dtYDQym|B9-`ksP^m<n7!pfCiyOVPM?&rV1X_?%P@ajL+
zp<U7Ozi(yBG9FOdZ*4X=TgvojWTNd^@%&vv*_S_FF5kZC_gyxh^`AR>F4nv1OqNaC
z_-ONS-)pfqY&Z59zPWSfxO&;Y?e-7OwYx7{XS?k{qQhFX=y!_`1n<84?OwgTGVk8~
zr!Rix{=Id&es$f?!!3)Id{*zz`<H#asLDw{V#9&EclRz>;1C-6^>%a;ua@SM(2Tnd
z70X{dR$gn8@m9O_W9i!L<*OZolNdgnOyBucXP-^snl($!{$6cfWvqE<n-+^Z8-qii
z^ZQGkD;IO|#+$F-%eGcl`qQVDt>P#0mO5++Q;1+#f4^p5h50YWh~~QJYSCG3It4#d
zlJe$;>pF!u2PD-bH?#fx<n!m`;w?c6TSPhXRA)xG2yJo+H&YR-wPMbQRljat#>$ZY
zxn-vBmPZ1UFN7*eRa7!EFf5ua#wcGe?P9LmbX5JZ-M^xH|BrdKcJl7N<zcbLQMKWL
zubboRZ&LHW{;zuSb^GUd@oBRAv+lRX#>B)$o@8!Vb?J`#MZ0J}k@@>;7g<05y=z&)
zY45Jns~IF@Ohf(d&ON(Bwe@IdO@e{Ns#UtnPpol2oE`9Hxq{}Yu9kAUS{4298R6fz
zO}MpT;)c1s5z}s!)Wv=MRb9j2@%P6zEd`?!leZb|Ve1b+aY8h$=3_#f=FIRe?-C`$
zlNv&4HCs(s7>vHEd!6UsXCqwyNn@+O@TN&Ve->R>^ZCiD@^D@^ojEG$C;q*9;>B=a
zfpGr3u)n&qw41;0{qJD(akAdC`L*1=U&|d=i0uli?maK3RabKF=X&ngI%~_l`^^|M
z>SSi!+xc6+Nmu=j<o)>f8!Wn{=H&MmrfaUsdZ6(`zU{B$Y~#b<N*EHhB>B7y`OWXk
zW0LT0W2&Rv`=Z+I&jp;$HptBW@vT0K>qlI}lIXNUr3@8i1`p!g3fH#p%%8M=z061H
z>Vt3e;%81}3go!$ID7LFo`XNOnK0bS2;O;-Y15J}7ORCP)EF9m_w9eq#TV?l;->QV
z%10A8uKd~h+K}Uc*vap64X;hVwsXDf^Y{ge6cm)07}$LNet)&5F@)Wv^nlbBOYgcP
zchv=~zD*2F7j{q=v1DP0{Nz_)z3*4esq?<l9Rf}qo;<3`*9-L5PHxwm)44<2XP^9p
z8hH-e)tY(2GCIFXLKa)eu9CjC;A~8elD^qQcArJE(hLh$Y*B3H5pd#A)VzG_LdjFP
z%Tt;1Dn;cR4n8wU;q`M-UE>(dCU-MrafYB1hvG@c^g@d@zs#M57=ABQZUJd4JY06+
z*pZ2<AyVI@r=F;ObmYiFqq`xQp;r$rVsboSYq!u*I_bofRS(|pXrJpI__^`9rjzgb
zXMGiaRX)zE{JHqVh8@#cKYd!Y)~e*?^>}%%`!>CM=Pymkxx7F3?|C=gHq#7cB{AP;
zFU&K_5;r}Ze7Ku)^1j5b9Co$H5I$Lp5=-W3R|SmL&tYN6o3^1ZbcNugu!?y~SDu`j
z7cZ5tf1P49UrgD<7hkNxv=;R!eEzZi+U}%YwLaaid^M+ZYr|bmWom6=NO<sHFM;>g
zjty+NJ~oUD!VT&*(~kM;3TbUk<9qAD&hXFr-&S>|WiktIH?Cal#n7;%^xT}(<L?f%
zqz6u&J%7^aSWCB(hwZ7Cq<-f1dTvRcF!SqL_U3@g2S0DAtC+XQ?a_vUvn%$+ip*RZ
zUaoIb+|Ryc-)lad7ppw>oUM3PUXz`&sX;mTzuyrLfxWqh)>U79{g=<@h0dEm%RlD>
z9JQ0B=dy>#Gu%A!T7h%N)0!~u_dB~xGm;;3ZkxNJhKYfJYnREz>gdwW1L_mm84A|l
zD%rAT^SYGM!>S@LYNv2ABwor^WO(yo)rNOIr#*zNEKX*ceOoxqqA)~0Y)xg0_Gx9W
zhn2t1IzO!Jk~;b+`1<m%f$R(n@3<?i82B45ug#yYr*mk7`}~Q|ny-J~bZ(Kw<K?F&
z?L8MJP}03l_69@C+ie1WpIg4N@=9lqsdhAb$8P&>s%rD*^wzWQ7T#I0d#cf-(uq^c
zHsxq|uB`j??{<Ifxva0HoV$-RD?B+O{pR+q?MqKHF$gT0ld|-}e?j|wQ_if<dcxc*
zanLNu+47!&%alW(zuf*{<J!A$R>F<h*F_dE1gy?}(XKDKzi0l*%7|+V;ub#&oHkX&
zlgo3{A~hB7s~g%Evbq19K9Tp;zRLwK4_5yd4_ta^!j`@bkJ{cY=<O)sUwcJ=_48A{
z)428|@BGQ*6#n#a^Ga701_4mP+Ho&nQ+F1dbIEI$>(98?FPmIj@>15)#g%pD2d@t|
zUA*6JeX%z*(?_A0n>BF0-nBK$ea)vRKbrT^__fbrm0KGfn}atu&78z_k&ofW{eA52
zJXe-xZs3%gucIyKR=dy4gkiG8jmB%!oF8+Z*uJyL`u+4SIg<&tzVrDQvb45@ajAq%
ztejM~TzD(Tr1E)2oWZqL-@7Mo|Hf7^PwA!7di!&qzcie`Uh>~beEH?-%`;V1Ro}}e
zXliX@VA%1rHHMLyq2XNMlMbz|hkBmW>?%vwO?!7*+;VBnwkD_Ox8LTRUuEIwd93@)
zA}zb-wU->9Po6(*Pvy+WV=pf3ti8Ma3hSLuS{nLKyY^}8>gls98d%(y(?6<vvTkl9
z3lEz?s_E8S+I3~gu>t?oY_<1q={lI8oxSa9&|1r)Yrf^(sqYv*yvkMF&djS)5HM{=
z0`ILEMweJ5zvjEL9s4+aM?sI>JoYDVFPu7@>BKOrd+!=GS%wEU|E{tTb^Ff8ld9&h
zGeO|O?Y;N!F}6Gm*Rr|Cz#zc&<*L~xtF3;T$8GNU+2?67Flb~L|Cc$<#Gw86Rph_Y
zHD^|?)&G3`0l)nFj*H2biy0X__7^{${#vT*;$G?Ee+koa*1D&CV7b07G`8YDpLvZ!
zwHH4dbHi!nNFOGKH^HmJT0U>(jo4g$WzUT$$xYiOnK*u|?hm~hE2(^CippD?x#{ZH
z<7YA`Y~6E8U*xJF>-k`nRuP_RjVrqay`8P=pHFaPd~fe}@z*<U8;eD@(hFXEF<G5_
z(uN_T{@J3mYkLjmzqR{bd8Y7l`egNQ?AJ{G372#~UikmZ<rS3|m)*5VxBWi9zj}G^
zji>uhZdokE@L_Wvk5N1WL)+<d6~6C(rcT<FF3#q2E9`UchhIFK85h~y+wNboa(9L<
zm)(}!oBK|0m7n!%`|tOUdtF;Iew{7efBvsI>xsI*U*6xX=3`hFT@w}|HO1CG->!Uj
z#_!<$k@J`qlt;=wetYA@T;HWE_qvy}?0oH08PoaVZ$r<nh&l^j4u+Ja4-VAIF+5QX
zZ4>-bCo6es(RW$Cy|V3RP6$1@ZsKvlE&3#HGLyl}+jCM%Ee@@?(3RPKT7M}QgTp$}
z;-#^^^7*1iXYJQ_o_THEI+>T}SO5Q}W&f!mezMAnn%?lA-?~K~mhRms`8`or`phKO
zPh}ZLw;WF8)(?Be%rM#RnAf>LW(J4Flh1sS<6)Z+7=G&69F6$u#ZnAWFE^!SIR5H>
z<?9%_==7P@?4hgAPtE`S^pUjmq?|{V{+v%|vH3j`Kf6o+M8&*C&$><XKA)8p=4E8y
z*n9Z8e^17FHim?)v$y6MGVFM_+ic4J7e_v>wfkPlaDdxlgMG$<LsPY5e;G11h{Vjz
z+@>4Mz%W5>&)uqb_SgPfOMTWbe0S_;<lLDGd-ffvy0zCyh+);ey@Im(KNF+o3a>jU
z?bFe@Q@SR{l7S)j$ioTI<;h8l*8RW!^Y7nw|EIF^4XbYMJFT`~cE{`)GiLlaQT*A2
z!D4UKtwWQ)GcM4wJ(2XzUeo09VL^WXIUn{u{>{%YC+g?^=$$o{vAbtX5LV`tV$iX!
z`}*$gj1!N;bT>1wu5n!Hyz9QGsgKZK_Bt6ci%r6^4i#_v6wLappNA}cake4;(EYL<
zHNFl{KM6nWG)?C}o5fJO$I@H&!PU8|gfpEOJ_S`=&1Pg^SS*?>A<Urg*HZuE+zHXD
zd+N9EdS@;qH#5p**|aM?HYfCzjJr-8xnuRrMJ45>-Z3AB6Mid%_6kpb%Jpx@p{L9Y
z0?B7?ykf}NIjeZ<4iN^0zpV>I7#c1FEoJQYIjiiNzwi%NVw&d6h%Zxb_i=a%7|vOq
z@<KUIvtQ@b<DWKvPe~})_y-mrld`QfV_=y1=$-uUhZm2`P-sfDFU;6|?U=A%#m}xT
zGv@aFi-f+NEiH_!RLWYsREJ^0pUuw~8!xmdy#48NNwfXG61f!~?9-yk139@<PWA36
zVVGcUaMR-Gs{1QAZ;SP;+x=j1v52T;mX>Mm_Elb8GOn(np+~LqCoN}{xmUV=R+_>8
z^OlD#7#O~+dpu2M&Q(!{y!SKezNU)q|02u4aOI2ggzJV|R5Bcv?K!8<Zsi>q8hGXU
zoTUp6J=9pGwe5zG$or$g%T0R!ub7j1gmIeM<+z2nwu^8??%K%KJ3n0Z95aK0gz4Tz
ztM7B(>z}Vs-crziYm)?n)-0Z6RVju)Mh@Q!<S+Vney{b)h>dc*BF!+bV0XIfUe6o6
z9=GG#7xFMT-1GZ$W1`=Ww(l!iuDT00ILohNIJmRvk5*fZ=zlj&28VLlSt)0CHeUEH
ze7I6d`gm{e<7=~7&IezKG2GIgC#SyKXu+p%>w^1bdv+Idtnk`dp>Q(c;ih;Ob?*84
zTDNYM8iuT`oOI}i{Xc723w72L)tr*7Z;p5bah%`gH?uHSl7Zpf-QDW9-W_Vb?(J)R
z;pN(#yZ^SX)%QHy=05fQ_uU)}i*wqy{`#R%yWsBnf*eDJmLp4M^&1vQrcSt<H)n<9
zRs~(->1#eej`5yf#grYuHF4j@t8Rx=89E*^&)lbIW9!_;bfEco{2bP;b{!LU{t>sC
zW+yD+RpM*KFk$8S%AfD==T+1s7%{w<z|7*1Y?e@xA74KC=iBf<cD51>F4Or;?z{Z^
z6kNk#@b>tocSqOk*A{RJVmKe%GD|A^$d{=C8~9qTpKrH)pO+jkM?%<xYx}Iy|22nS
z^*@(gc<t`S`Dcz*-+9t9MO@3DalwC)?Jt{ef2?3&2wrw3fvKOJ!N>HUdirz!)u%f4
z#~=5wZ7aLZI(O#p$(7gNtA<%xue<Y@nIW_MFaP-{VUKNJqCRJDU|SmW%lKZmqmh+Q
zz?%~?414l+@d$7n3VKz!@!FXUXW8v!B^cIEUS(T!bn<q-m#*`d<lgW6_?u&!m2lUk
z)&Kj1|K*)NR&Js4_JfV<ij^l{&))g<clv4DcT-En>o@<5&${6_>)<0s+5LBa`A?px
z&d~7S#EBCpm>fDcy_=xOe00J$?d{^CrnfID&UBv^&dqSj=-u0B^XF%5=N-N^*>LgA
z*=`oes=rj<@O$Zgx}<#ZxaO?05!)sj1%8_Jf62V%S00FQdNMFLJ@1~z@6?*N__|Ad
z<e?4DbF0<jpL6Y&-e@*!)76<z{&sY!I$qWH^8C}$Jd4%FUob0+OYE~aBg0Ai*KbP=
zALJa}A*metx>ksR;mg}}1#boh8~)m$Pe=WYCr#p#uHF<Kc)a2DccIrZ_pWTu7r(do
zGxM=?7RCRk$A-UKQ~F}{iNtI7wb!{GlA4m#|G2`Ef#HJN?1c;|J~exbKMKCeVQ852
zX;yT_eu>S#b~C+ZZ@U$D?b?Qk>C(IB2NesIU-|ij*L(Za7pgALrVFawoyouu%5+{t
zkzvB0UB9n=D14W#k_!&km6Eg1s=Qz*c(UHK(PciTv(?q_J(I#G+AzFW6KLGM{=?08
z|JUtXvDfwE;?K$Ie`khXUE%!thHQJ(CWZ!i$v=CRe4Cg3DI??5eY^d_3=)f%@x0dH
zXE=~>W#@xdpYHjK<+3lRg<M_m?(Ct>{P*(15Am<gE<12**Z0IMCI*i6cOIzi5#IYN
zQ?GB6nid1Y+K?C7F;7ZfJlU!t#~`rj3$Nas!^RfcziVE%{xf{8{@%*vY>zP$L&Ker
zeHjM&-jg%`7hcOt+9Joy;1&0&NvlEEievHxRtAP-=c>1oT#bR3KQAuLmSQNmzLfvK
z?|KiWfVz2nQYY+enu`;Rc9%03KdS6n!oT>OzWvp-R~wfuTA!lk^J-PZ^CcegqDR_f
z)LB^$SAJT-XVrYxg}>@wVzGshF9X8?-`$mY8$K^<^uAvDT;RZ2n|+_YOB#z+9)DV|
zt@`;d$6kM@uqPeQ>R0_#%zn_)%D`~w_ww~NzcvJZT>SdDC!^!`8Fp^F_cXCEB!>AZ
zGBE6^oiruJSgn(d;efj9*+u^G;tUQCPd_o&EsZr=<2bu(57XI&>1S6*tkW&tWcG3E
zO>w#VtHt*2iC6vn+Q#z9txebXQ%iys*rZPW$veZ+kiLCKpv{Mly8E`XJ<AIx*qe24
zKI<6P&U}+IbkWRpw_h7r6>x}6Hh6JtO>tcS1A~GOk4wJSkpp(}{=2`;eX;fh14BvW
z#XE}*y;Y3SV_-;3)y`hXbU@|Vt&^_|7*G9dcdlRvP_^oO@nxs+ld|1KVtf+>-};p|
ztobd|Cp+ON1H*zv=Wk^wZ_nA1^FQpzCufn#4WFk-+-*IWBvjPJvsv%Hh4Kd**C20&
zir*@aide)M94hYKw%^tgU4C-uwfjkq>1qr*n@rNKZgsq|zfWCxzR0xV&2q8xOmnRn
z*4UcwmX(#rZSPp-#?-KPf24Hj_N{Nza%{iL&pP;L|FUl^4V$jKh-G~E{he!$m2Bv9
z-Vz3e2WrRe2)6Qcd;b2A5m|L<Z;Bnml+^vAw&7=QJblpQ$RO}-{pp7%o-;K(nr<Gp
z`wurmw(XrS5j=e|j0}g;*^iwmh~A%ZDC2pkrUTP~XIHy-eU11rBj6Pa3qyt8?Xnj~
zt|f+*@pDJNC`zd)%oLiIoci7Jq~z+_A73w>IbQT__on?p99)~dJQ)&84u#uu{!Ka&
z_wL9{V}~|HfqQ)`Zp_hOXxL=FCv;)x`+N4Cj_M8yd+Jwzs^6Y1r==s(b?=WB7fb0m
z{#9$$AKSGjU)#29>Xu%`+qHb@eP(B`hUG2R^e*8{SZVZsp7Q=R+-ptdZhz(#nr6xA
zv#Ysz&u^Y{4PjgMe0g;>ebqb$h6(P<S!vtZ)O=@py19nj(-mN1xX{@p^RGAP;oRwq
z5?rc%qq1_I)@R;uln(SXIu~rTJMEFbk?Qhy#!TmfRgSKjxAW^8)2d5dzuzx!`0y+H
z*8)4Yd!D7b8yQyp`kTVh_P1l+tJ1$B3<<~cC%1ilpZspmtJZyY_i1lelUl{dz)<_*
zk?`>_zg0Iod^{O1Wxeb(IR7*E1p`CQSH3C-h65dSb6@|N6FIGHXY_Wv%Vxj3_u2lL
z?Ymv>^7{Rn`-=;{F?6#o`cU5Y=idHk^Z!qb7=B%-tUjL?c}2Wq@BW+8rLWCyWI2Y|
zFBf(!SNj&goX{It<zTXK<%75d@nP2whc|sc^NNe%(39V4-ZECjTh9EO#lfJkXJ7vF
zm;dK-ZO^zrVdwQ(icM>(&#hvM+>~<i-QC^oJ5@J@-ZD>GCf4qG-`Y9M?D^-XCY+JK
z=B?bs#h|Lc^MCgf=`BhP{&uUgld@uYX9+RC`je?GI%^%1$`bDV|4zQ&y%jtT$xty{
z{mh>~hovw2b~D(`jM)6g<f3nP!pD<4zON44etp5#Waqef24ADSW48xfyj}Hj%f#wG
zKMEheEBp23<-2$97GGO#e8a1E_w#8>InM_#>Nh*6E2MVq=dwSGKiK3dT{nOA<#non
ztnA$#yGu(-zI^#o^ZlLd<dbXaUq9>me`@XYucwzTw~zPR{Ay-&e7i$X#@>_Zk1f_V
zHEAVmxcX|ALtH>yP(U0jLyoHHQUCD7=hL2uO)-qhUTKyk?~~`5SSTtl!PB?uRr*og
z?FQQ=?Cj3zp3!|~Y$5gDTwDK)?mDJQ)86ypGpE;ai|G`Um+Nb21ca_lkCEi-STgUW
zfh23gZT@|?M9xoWERvJ_{%rMpv7qApw*n@HtDITA{{5}*r?gc+d+ir%^ZBFlJpI0t
z+qw;m-lfVag|j}{s6M_VU2mym<3I6==k7AK2_o-|-+$Z6t99x}$*x6hZI7;-EhrW-
zwJ7~RW$B8AU#~v@YcOrmtLv9GgqWJ_`=?tFVjA`Cm&f}=UN7EWzcZ%FDKD-b>R}3U
zpUtPb$M}AQaMr@PQw0}E%v;W3x^-30jIXczLNmTdT#EkoCw`t0-+$h#zp7G|nq9p&
zH`V)P@ff9O8E__C`fADG(AHkOe^2r8Kt-444=GPF-!i<4`yi`$B?Vl8?%h$R_28au
zL$x3y!<K#OVtv0p#aZrO60+r5YFJsAr9{H##LbP#lU4{$K2p8k&gAD0zpK-^SMW%k
zx|e=b?5j}X3UDq?&*6Tj(jp)VHu5D;`73S~28M;(U!M6^w`#us$tj{6*FRtJa#5h8
ze?KTQY!AJ6{>Xe`Q`Q{zrEYmr`R5YjtXY{>ozQRNW?(qt62A0EU6s}+>HWtgoL)$S
z&D^-PeZEgkL|hze4&U;151n$|b$hyhf1TVe5wXie_(%Japl9LkJPZs^LLbeRbh>i*
zv&cd<x90`>N}J~%%_?&~F39lb$=(XH*}M!l`D>*#d7N~^3#)Qc9cR6mlNGb5_L)jd
z))!mlE0;v}b*$1mpWpR4BqZ(buF};PzxPbA2n`82a^wg@Lr;&-Ug`f={w#R;@};s{
zU(L*E*9wmW?KON|S7ASC!BnQe-^F6vyUzUna&7h=w!7<WA4(rzU%cpSBO8ycsFd)D
zKiir9?^;$eEqr0`#l_keIpek(*mchTF*Wl0zS=2miax&$PVmIt5U;&{WSelG&cdM8
zt#eN=I4h-F{68x-!qvIo>BJ=+tIJm-7?_MKeG8VaxVCzV=a$vAf1gjDdC9&t@!sVJ
zEye46tSUb~I%=4F%;ep<`S$g8MORk5G0*>UEa!zqQsLuwds2VB5#=l0GPUkCPu7~#
zu5;agz6lBoi8}7H^u_TfHlKXI%u-fAzE4WRqU6PeV|RZxv-7v{O0zLEHZ~qRcC2^z
z=Vj^-rqyqHrQ3Y=XGxi#fXKeCT1}I8mtU$cd;Vl!Pc?_^?~;(eAES5Ji#rFz#eID!
z&DQ+!+rz0JCdFO$E#3E6<IbTc&-J^tejW(OSigL6Xm(jZUS;Ll`wyq@chU@qjkQfY
z)bioOho#=r-`&`__}<(4S8;o`ZTmL&_I6zzok#25uhBaBE^GgEx9O+;{b*tOo3DQJ
zmiW8ya;<Cjt19y+gfH}$R!d&v$=;}-);Isx%i^b|60eVMygHMCX_CLX1cO-Xp4@wP
zZq4>JNscZ{VPKf<zG=o0jpR>`Ay4c5?JM8^{1zycQ6VpRJI?4-NySm|mfG4&7u4=?
z=AI3)JbPa5G3Oq8U5EXJQyl-^Fs`b7^WONB#GFm5{&TH3$IhTI&F7KC)m@o~&ptf&
zar(Q^b4KsyR#g>l>{{iO;BcUUky)zt|F2)a%HQ9SG*0umw6*lkBc0mSx-7ix%Kt<z
zU)=Qg_O)NKTrBnStN#?7t9!ilZp8HN^|G$c;c2g5+{)#DGPU^WzFw=ZwcTH}vhqJ&
zy*exRRtz_nk&#i>m%ra%U48xP>gs3Do|&1MU0)ab`rh8^#qx_dr>dTpeefYsRfQp<
z?%lI3a;FxpXH}h<Xs15??UyCee+1uNe%b!)g~*cQu5)f|Iix;w|DQW&`&ON@-_$+7
z@1xnT2QMny?Wzl|?wrkQ?{)2NF?agE*+H+xQ~&N?{CW=iCWcj))=5PFfB$&dmVM9j
z=f61jv!-R5%Hz*JgO~Ym^7GF(&A#^S?QQ8#Q|7<rU$t;y;p1bx?CkBU!_S9L-)ma*
zEn&lx`l9(F51dUIxKI2RoccSsMMllRH`F;g<G9(s4KbT0toik&vPI`_e)KjY2iLH#
ze;TqFZvBsW`XzMcYW0If{_8RVGcRA<G)?v;!y)ZB(S5ELZhE9$WtbUvI7Pfb`}=j1
z#w*2B+AbI;=*ga&^!Ee*uZ1hQq*K-J{x-1JcVIM<3$3r1{`XrT8*j?_cLy05@<ct;
zg%^phNQ$1hHSNhF0Xc@sBb#{`ILvCFs@yCq5%BoAq-W*A|4Ul)Uw$~PuXym_T<h|@
zMT@uO-~V@QZS=Kk*DhXkU*%Q2%iqc?ecOel52aZB9heyY%jWM@-`>H=ppcdQ=Uevf
ziF)Gqt-SKGzBXOI`qW0kZi?;Q35@dVQeLP}P?;&S;nml*%xr9I2M->+b?cUI$%2K>
z?Qy$GGVkmt3=0cexNu=rRh4haf@8-OOgNYv%CD!EU+(+F#K;hPr6pXX=EM5b-_EYl
zf~Sn^FKh^(BeU&RU8YLp=U>wk|2RZm<1l3CVk(pU|3=a2*>}VD-fye=CyDMoUvp29
z;e~ltNTAD-6+(OeykD>I{&RmX<L=2v7rTp<SS@jv*%DF0(V!grxU|LT;hUSAnVFc>
z{O8sD_>kC?abf;S!CB50GJX5^|35R+n0u0P5`X_%z3Shee=qTJn<l_;s`A6XccL>V
zpS3R8_~~o7-V?LMk*63G1Z21cbnmO*5jyphxxuJ-p`a`S2W#70Jz0iL!aMf4GBBL%
zU!~05!r*d0aLqbiYcJ=J(8=5LL+8~TSYLm6*<DV<dmWrM2ajGjHCb?Ky37C8@IT_;
zKh}Pl?Y=Pbdjf<1gQuZp&qIE$Is8=drKx|&n-enH><YhLe^{_4<cHs>`?0M|b&Jj)
z+VVknZ`Idb)zAGdi!M+)b?Vf@g$sq%{m%IAktwgSS-)xH^z&JjtGrugzK@@vbl<L7
z>Sa;j`9PP%dCwjyz3+8C@my`r{Exq`H6CncKYrwhNi3sQQd-)xmoFnDA~G^FXU>~<
zZ^`kj8kgH;QC$orR&)RD`!gx?@?Ap}tqjlozjm|L@y2f3+MRykYVzrNrAXx?8g(VP
z;YRu2{LW8Uut57gYrF5ttrAbS_fL9j`d?%3|3BS^{Qhrl78`!-`yJ(ek$>G9+lhx%
zUH6+a>~gzpW-l|zY5)I!yO%FtzIE%@mzS4k^6tBO&0_l#37%sclaF6p7aRRa?cuw+
z%b)g4al5I@YWKgu<=35OcTb8=yLR=4+lfo=Tb439xQf~T``diT{7-r)1H=B3^i|9=
z_)<<WF>vJD|Ju>Lz{QS%VQ$_c_toDuc&AJ}W<OV>BI;9czK#8U;auire8OHvKNB;X
zq&1riF4nI+^rubM^>)KT&Xfv=r&nh&b=ruBRBm?IqN=D~8k@Y&FCg~gh2&4OBlwMt
zH?LT+;@aBi?uQF>#9kk2<?a^Kt@{4z+u!DAp{^G5f1UsNi1lah^|HsYdQv6xlC;Yg
zKL{5+K6gX)><8)T>EdEy%N8wCYHPlA>(;K_yVFxsBO@YG($l*icJ#jd?C&;5&iPmN
z%2lgQ{jb=lcz<7Y{o!p=T3q&mub(GRahJL8ctO{P{YCcH{QDnmb+6u9ertPQ&^McB
z7A01@a_7xkw4!!$@|z969;|O>Xi(A4`o8q<%4sTHYhreOI?^eurluAb7uWNCrsZnG
z+g0W{H!iG;-MzE;`91rLYnQJE?{wbpc4sF`*8RqHJPZu^Mp@$hDOrshqNAk>UNnnz
ztOIo<Y!ZXk-kVSv+2q9#$xvHswd6|hhm+a@v$p>83JpDw+HmQ&dr-HXB<IJe=5kKo
zFKee4ifZh=zh%{%&z=0Q?|2?g68Lt^h+)BviCgRWRzI4n@T>0X&n>&y_g<|O`Ma$o
zPwxHZnFsXMKehAx|H!{yXvHp@_l>)q3*G))SsC0fYke%GYrVK>uiNkU`~TnCn*AMg
z;hS~FrR-hXoupU3DpTWqRg-J}>nYRfGt0uh>b*Yb`TNp#e{r3N4e96SwVA*DV<Rp;
z{nMvUJBy$9Nt^3gSVY9e#^$t(Dd%}|Zojs@M2vgKomrck&jtR;S$AQ5NZS4FldU*U
zoj#pi`fAbU2%dlKwHri4BN+mwifvU}#P#6S5y$Y%x$e8pmTr~GUGu~CtkO<Re|DXi
zuQz%A)E@6?UTk=Oqj=|*%*)I6R)6P{GI?=nsrP;B?l9%==0Cn(kH5Vo)A-^+#dkA8
zzn@e$<<XTY|1adaf9(rak7a4KLGEg)9LKgVWfZV2k$jcIz>wFt;oA)6O;c(PGs~)-
z$h=V6`2Foecjn(4=a$;<yYzWpar_D+_A4cFs`>1pa(?T#-<&ifM|=6_jpqbe?g-7`
zn<lt7=tt|GsrT;uS#SHbI!U7>>6)Cb%5oob^QSy#AHV(n*luP1f2D;Fr!I`QS+O>v
zSY+Om^1mXQ=l!b-F0EVi;OYN=-}l%5``JE;M>}jy#`Se`SFe8k_i%ExpXKUR#&ae~
z=yp};iLcl=<(Z1DMP_{HpR~`<&gSLiJ>oY=KR0L7rcF;zPrrTRM$F!-P)|?KKa#T|
zI{!a1jDB}DWCBa^vzx|u%Y37>q;6bE{Z^kY)|=;Gd6~nyQnvC!)zaVX`LX5m-hNK{
zzv!<a14GNbeGfh>Z|RVBzgBm{VTR1l_m@u8XDXNd%rSVo=GuY!+~tL<`)fZY6kd#G
zP~fn9Ik)8h{{R1eKc8RkqWo%~?aA!9QvaLp?kIe`&fQH*vq-MeBK?l-%@@C?%g8U=
zx9Z0i<|xtcwWsA}UDnq%KG?j?VN+0uALoL558W3&oXWkv;C}rD;a_pjW&74VV|K8t
zKmYI(o5%tNg*z3CbUyt(adXmhoB2mx&0tbwu-P4KxtftRgE@Ih>gV(KR&{wi+FSSC
zaB7V9GcNIO=WDv&-)LiF2vF_$S(ByyGI)+){r?G(sSj8G{rx>Sw4uvqp3To&TeEZ2
zj?CY1zOp6wkD2Ai7`=eS8a}%vt1Ec++gv|<xOoy&NXpGksjsiCt^V<0;oP~guB~6<
z|Jhy7I@MWyT#QGkHhFJpQvbST#c7gE2X+-dyvM8MAf2v0_n2<IM4ii)IE{lF$_<UB
zo_<*^C*ZUC>coEiz6(jr8-D41`eapJbNW{5wA%aT&pFoXZ<Mb4ntJp5-^KP*KdXo_
zNGP|LC3#-{@twa<-hN-!)m3`2yINXW{@ggeL~ZBJ3;XN;TW`-V|FQJ&<?im_W^=Qn
zic6Xo?CX=|%|5ALeb{=I!Y$#m_j<P8n{}39;qv&<8r$=~kF4wtwPaveutIxt0Za7~
zskKj3%ReuEpAogjoGnt5$M?znzGv$`ubEM{NXhE;-*c8d_kaIQi1%}vW*0G;&EZ~O
zPQY57lb`0b9+BIsQlYwU-@eb!&o8&)>ya{jl~%H)t*z?N&zbu9Zx&klbzk?G&i!Tb
zJ++t~`$TPL@tD0;SC1ZTy=CzF`ug}?B`@b%m&?e=?1;(x>p#Kh3*%?AGobO>rxV;g
zKAem^J})Bw)65%>cmFzWe#lQ^Eqkt=1v`VszV*kKK6t$P`7)z7d#?FseXU~-UE9}Y
z`zzYY=YG+rgT31A>Lu$kx5w;!RC4co%3qhzQ%CP^Z@h5(>+y)p7`9h`etzEn|6lcT
zzqyAFAFkiA+atKN;Ns=WyNiF9y}!5jveI_-s`KJpOEykl%hI{`tgj9`L&nvKHye4+
zzYAfl{k}ZuVWad_28NPZ8||1+{j7G{SDo;){Kx~&e)bT#O8b&<=f~{#?bT;*oWkf8
z7bfpF{hg?w@%w5Uo3|?tB)Yn|+_-fsNOpPK#EBR0T9@<i@B}qov)R4rQ+n7MhZT|)
zwXY8>F1{SKa`nBQJ~LC(rB-|l2l`~K)6UP6m6Uw>=+U9)=KISpneG3T!@}_J^Jeeq
zZxf={#$_fuob-$0pS$m3kQjsC`nYY{lXidmBD8n?7T;B+GJEdrPW!Oiz*2MhA9nrO
z8`HP{oZ$3&|9{zqs<q{h^yBy0RDa9S)m_`s(P5TbSEEobT(&9i?yj%B(&oo{BtNUY
zd((V*GRxJoxm)?vu3WG1TfcvQZZo^s1Z$^N_tsBS;;8(h?0o(5{LuC%S8NR5AMG#I
z`MpoV@cXJqUJMKoSzF5P*BiXeGfJCfQgQtE>))qSJ3lGq6i<48sO;^ndwrdfCK&=(
z3+)!JT`Mar47#@aOEkOiIbZQlv*V+?T?F1gWvVrwq%xoV&(A-n{#%v5t9kzI?L6D+
zpt&y=H?#9wSzE{Nttx$XW~NhO@$6FXxcj$iJ$-k+H_1D~e>UO1hV{dq|Bp|2G_f(T
z6hG_e-&}XC_1>}4$6*W%55K0buYXcjb5nSJ=I*43;O*;wPn<P7Y2m_!!OQ(Z!^8VK
zI$m5@c=-IARhjI+PM$s;zq2Uy)|O1x^M~KbZ(E`Fd*xGQ)w8=#&GNS|n!m4fL$;mG
z#7b7v)lV+IoWJ+4z@`!ghC0ieA3DuX#5g*ho&H60aeaZx*RyA4{eHZ*YC~O){C(-i
zRYv>_4gc=u&;JzFW6yrt!Sa8;zvez}e=b)4KXvtj-`=;2xvvtw>gejKs-tt}+_`se
zA9oiX&#j#Leudu^p3a{|jr+PM#Q*(qXw7x=@4PY=25M^0rpMPUw7RICVKQs??%zK?
zKCZ2)sr&tQ`?K=>hYomze>wV+RXzUiiyN2c-{)Mo>^3V0gF@wT{@B`&Z+j(ke|OAs
zs{C+wKG%<(-|J^@Y0Uf@{zN``$veG2YooWz$jVyE_+@0gIPaf*_sL{m1Bsr=>i++J
zKA*1|$^Tx?K==Im=kK*#CuO~|blUz_eZSk`Z>HbpZobd8=@cVFM3%|jC}{yM6J0e1
zhtkds4|@+QyKS&DQC;=ol+VQ<|Bm?l`V%L=%I#LlYWbNTUQ{QG6np+iT9Ln(gZG%3
zw%O{g{cnAye&J&g3kkRRP;hG3^Sq=aAz9hK#^yS+q-LM}_U&8Tj)KICr=vHv9}C%g
zdXZINvGTLitXURyp`Ld*ch7ORGcz-L`0!y?7O(6}9X}qn!<R2#K6B>G-MfGL?f)&X
z@>?vFBKLcF+6jx3_PZb5$X!)7Nv`M>LuByYs&#*U$35JhUE8+DhBrfb*Y@B0!ooK_
zI5_E|v3$=z<3I7c<+Hym4Ty=Evv;p;^Fa<izIPWFA3y&-z(;uA-@o7QxAVz9+ULE$
z{=ymy+b|pU+^n4|Yc6WYul=^w+FEE=mI*_HP+bM9<liQKh676^Pu1R8R2w}jWA0vc
zf2m)Iy~~c4TulGKy}I0`BP>GEtv|K?{oUN_g*v`glA<9WSI96lT=Ysyo&LU>@sC4~
zViga=3XR`={O?+rYG><4Z|mCsHYD_|=iD^k3DI|bv&5{{>z(hnO`H1v%dGD1(^prA
zi;C2xHKi6VxR_D>{oUO?mBlY_FaORrYsxc|0w>)o3Xk^0ENFW?E#<zw$ubSys(X7X
z_wC!aY15{!_U%eWMn>ZLaWY0JCmNaA|NVNczv{7x*ulBVr~8-wDY8&W`fL%&P_gTK
z`1~jD_wK&SVXb)o*q?o03Z~z5&yTss#-OmrzWVUd{MuLAetX3Zs%B(y-QT*Ty7JUC
z!OM%>*LM9lzwsKAchigi)8=?<{abh7(exQJGLn-IKmN$X#PsWH_8iU!N5A*krJtMA
zEv|owC3U0w@u#-DVz&#WFF(&`JoI8Z6T^hdX*|{c1$<{G9$;grkU6c*-cfx*o0VY!
z@7vEGdyil8*mB6^{^oNtr@pFKyzsAO+fjyxPqVM~2d~Obi%c@ElmGS8VLpGhZq3ds
z)#=LDJb0sAtG;%YJbX7nD{#4ebK;uB`T0M3=Ux#$yhZ+XecGCnRr0}tQaul)Oc)*>
zU2)#Jr0%Uoi_YJ8^~lnO<NQB!d0v>^l`y+2A;$Kl)ohdVm5%ez!lp!svaWw8YohJf
zUs_yjUHfZ`Mf$@3o_TqBa&mHiKHa(DoXmB3(aJvMsrx?N%6oG9sJUO+6^GRP`S)xS
zf@XbAxxCDmoloY+jsr`UOqx8In}_Gk<>lw6=|-2myW?3pr>*)v<J0b!%H8snXK&5V
z*i`m7L6`TmD7%ZWO74u3FQ-lKNSv7^U2fEK{=X0Jy_4a|3?g%X-p^n8Wn=wWnFZG}
z*JNx`ulsR}`MNP9gI64DRVAz2F|lh_*Wb;tu8=>ma(DgTr=Kn2?YX{RUB@VVZbB2U
zakZ$rdZ&uII{W+x<M%(To8Ra7i*CMhB}7|$^`=cm_V)W99+uAa%G|h3D>6FT-OWu)
z>x7-n+ZmPbr&#d+HIB->aJ$pW*od>)UwSi>^6Fv+h8O3`jAR%RE?t|sL4x6r)4heD
z8Q!T)pFL|BGEz^>Tj<MhU_srRf2UX&Dy?5l&g+Y}=X#iR_*$Y<oY40oO@;$6v+tBI
zS}w7v{{1J<*{42y|2lutFZKFJ-rQ%C7=0KXzEqH7n6R^Ee#pg>F7vHA?|z-k^fKFe
zr8=9U_x9IEue?8DBCoppc$=w1tHh1o$^+BRG8t*D_|z@5<=tX|O<#H^r<yV{Ecnf^
z?Q}up?SnbD59YK#+;nVKNyYK~O1#~3rl;8LslTvglY;M+H68*%W*=vFcY6l}OyF^r
zs6V8qez5<QMbPur%M+F3^!62sTyb#KPktvaby)jsS$TQ<)-2Jhg;HLDfrct7OR}%8
z%f7b8vgirN)k3M3=m~!`Hc2tGM7CCLb(<Kzw=?qXC;fv3WwUSaGH`6&mu=?H#4xG6
z=HoT(*7-F%9@+KRs{GQjex3VP-lRr7zMJvD-r_&&b-(Wj{vUm-)tTWy!%J;70TTv`
zuebBVCv(qZe_Qzbwf^?|%Xr`ab<YS2tMn}Ua8Pygx8pN(tT$vFd2Y_Mze*(U*}=-P
z<DQHRe_p2Kr#@FmN>AUO^E`E0clX3aPxb|;q@+YdMLm1|Jbp((;+q>Adv~Ne{n<Nv
z^5n^9&zz0lUv)Ke%j|V4UFQ63(WyNj63MFY_uSNOt~K8?vSXf>*h%{y7M8IJtXzJr
zIJ`gowT2Lb3Iij9YvfgHrd8SXOw#xIZI_BSGae9UXy#K8aB~fr>+$=-s*k;k1y09W
z+;x#(e(LA#b0M~jN%vT8{>zfTU%E+H>wkIlg}waQGe17>&0Q}SvTncn1%nAUf7Pu$
z-P3Z)bUUlm+!@lo*=tv*3dmT6RkA;LZ7pb~a;$sX8pW*ndt|3Q`EbX1$H%Gj-+Yh`
z)IEO6_sYLl(ytz`=#urf|8uG1>AE?2IV(;)I>$d@LEz_~=MM{=^IIzXr}+7~n>TL;
z%054CTl1sf%#Drt#YJC^|Gk`ef@#%b53$o5HwkZj8Nct_zK^%WYVS%+@BY=##Bhtv
zsGfIvqt&`h4u%JBlm8!|w|`S4k9~Z6adGjg$2HsM%$c*$xqaEPWpb89M{eGn`O1PJ
z^!%M)RxCn?_vF8S`|0G~mp1*b$=h6NjBc~-{r@QYc=}44{W&@D!3=-OUfi0a_;%H$
z!_Ojh1a%8p7)+Xf9<=*>n?dBrJZZbPpG9q_$@0&?x4$&hx%|^riQEN$7TxYJZJf$*
zU`v$jzx9i?3Y5$AV?tKDzwGgqmjA!=+hIG#hGqFXlkWyyKEGVy@3FuKJD&&r?p2Kc
z{Wtg1?&QmnabHSr7T;(5_PW9BQFpwWX;D#;kFW3LOPBoSTA7x<x^nEUC9A0h!;>dZ
zdStD?Jv`hVRlB@w>s{Xq8vB?(eU;j>@XMALRb32B9_IyBuI*%3GyEei)y|hHni5mB
zSvLAYT=TN^E+;N5+%>Cas;<x4HUBm3y@awatl3fQxsO{^RFr95jHFQt$JN5RYp<`b
zKYRA<eEa%)i6@%QO`E-Bo#XbGHEY>&6%1qaHn9C|`!eJ3`E@(CW?g+16%g?L-rkjc
zpPMgTzTD0uIce(D(qCUPUoDsUm&JQ4hT~N6eUWLOwJU9w?b$SInw#)F^XV(3wk%k1
z;Le>tFE1}==aZQMx<CH?{8=*tO-)S~FIx2G?OSJO|M2P0BrNAHI(_2me-pzW7w3g)
z=Cf$#@7u}Rzh~m5KfiNRpKuw^y2=uBUs1OvI_t&#x3{+Q?<+fd;lhMh7QG9c+xZL)
z4Idxt-CbK;AL{8>-tKI2tpCi?20P`p)2@M^XP@3<I6-Bm(3WqZSL9-C_fB~5pikC1
zXzq*Snc3Ox?d|-sRwcK#WEK?_O}aEoyLR(hbDp`2FaFyt?)={{XW5dNG;c4joSd9_
zcD1`oUte3M;m~;KdUsQkQ$oUnGc%2CZETYIqrT00rg}It^#A2q9L`_571rdYY~9St
zBdKJ(@z!(S;@Qn>D-GVNUk+vbQ8FvTpk1X$NujlA`M=Y~@8`X^xA*tf?CV>$Y+2!V
zB_kv>^zh-sbLY&_i~Y1#@AO;KuR6_tmAjeDjeV>b8H#dKugw<xd6mUk!9w`#^p?n+
zr=~A1?=p69*p?Msyg4q*?Xaos!iMQDZ0(;f4FCV{FEbNUT}7$m$%V_8i;Ibw)%++>
zS6}WuU5{7RYRV>!noUd$E!VH-*cnzQRL8OUS6_JXPHM-BL*)V@%UbUHv8fp=KQm!q
zm=gTkanqtjht8dQcd(f~``VhwjY+P;vaS2-{{B+CRr>e%vdg)-xvN*NKFqYYcz#Ua
zW%J1sgBPlBI;!(*$(Xt{MU!7dG=nQg>96Yh%HqB8dUrb}?wxU~%Y<*ThMwwatyc4-
zRV)8`b5>04KezLl%@--lq7)Ag595mD-#n8~mJ}46IDfvM@8j8esWpii2NS%1*#!Ig
zhR(b;i_vwdO0#e9B=s3n4@s|YnRG$fWvA)kn0~$Eiv+Je*==NGzxtQYCKr~)GV`N<
zJ(!Xl7qnpN)KCY9hDkxjKQ)#vTxeMJ<%Mzjxr(Z)J_*A`xwp4nd7|($W?9mDMdvS5
zDt<;<Rvuk&{q%~dI(^fA^IiRUetoO@w6JMhp}cK8Z;X=;J((D5vf>wS(at-blhs^Z
zU3KI3RGgls>szjn^{ct=*?~r8ZgIVoqAS%qt2DPxzjn`Z1?!_wkt4hPwfy$mSAI$<
zFE5vr^xWBgNd(kau>b#O^4crgzVpQYycWBnuQ|ZE(7Wi7|E=k1>gVR!Mr%Dw_!W2Y
z;>Bmrp6Q5bYiO(pOyhIcm7IURyr_tahi8sy_O*+P-DiJ^5DIK&I&idGT-|@(8TFTL
ze#@`&`mF404*09Z9e;f8hSewhWB1kkytXzv`p|^A!OQ*5&NA&zIQ4x_mEY3G39B9}
zgzVpR=lMtVH>>ZZ{`q=6KKt65g;svSoSdAJ=6Q2AZ7TBcIB|acf{rMig#i}D&wTXt
z*PooMKHsvq%`N-ZvIx(Wj*Jhkt&M)XW=&VO_pTdnGt=KVmCCg2TdMN<BJ+37tErW{
z%ijL_`~CjPK4rUu2M?Bg`xY7#_ubPyx?J8LtnkIvx9!W-xp%6%_sh+-tF^jXIP2O{
z@9AnjGcN2ZeZ67B20mG<09kg=rLElJ^K2?F-MV$_`t|7tGd?Lg3xzZ@9oU?H{>+)P
z#+NRa`h>llu_I8Hec>+AB`x+=##7zG=35r0Jv!1Ea$Ioxj~_n_laD`p`t)UV?mL4F
zy{m-+tD^HBO>N;ejx;KXm!E%ZZbWp_T-)lhQ&Tke|M}#dk&!XmEO$j;^OBy~vv2RK
z{q5u9!^FfCvGtiizSX4AW+sQSvTrjClZAwYZhf1bW>Kg9G`L;q%gi-DxIOoAi#)Lm
z*Nc}kdvkiLUv$fK{rFikXD?s2<d4xC^Sv9Tqn+$oRAMjX%h#4ww(i<hb>ZsjW5<q#
z92cCg?A~`~rm=f`{QgM3i7vj+-|zqb@B96Fc3vqFDJd;=^<^)ORM)YwHcXl%R9*dh
z!-frEvB$m@Sb3(O3YyEYP%3kxb#SwJ#@1YEA(=Z{v%^<C4p_Nh!GTMcKE1oUdzPE<
zZeN>*;C7{x6N_^Ho_@W%q@<*&xw*8YWJRBHo^{!qh<!CXCr%6u2ng6blaax6<=x%o
z|9?K8zqjgZS5J?~^K3Vv>;Q%bA1f{|_qVV9wr0|{Lj7=mNxr%n&LYaKhtjA2Ke3~D
zOZ4`<>+50{=kSSm21Q1`eDR{<=cm-rxeslZ&q>@cyWp)ri<Fbb1KW?WGi2)*=HK7f
z*x0x>D$x9`$<k}<<M$^XZu|H5cl^$x*299ApS<$$0&UFxe!t$_#H6RID=0Wv`1!(>
zbL78By9Bp~g@w7rd_CvO6>{A?{gh^~EZ5c;rnsw%KVRM?A0uH`Q<0I8(OsMTyC>t-
zh3)zG@9}&+TQ9aKG2>*yNwC_QyqkwFPcEL%5)l=}#>S?ltsOkK!${e!M?+it^S5vN
z>i_?1;S@e%u!F^OW%2WKXXo3;ySqRC_4T!slvI-4u9j;H7#svzR<2w*Yt}8%{h#yY
zf37N%Iu>+giIqrJXN73%NA<Vd!Ak1Bv!1+u-QCNz^<u`B!pFx9C3>>MZSLEB3x91R
z^*&0$tHAsIG5O7v-+#SZ6S?`^JX`H~f})eooH?VTu73Rbb#q0<gXhkzTd982@zRO=
z_v?@KNao(&wzu%H+wQWrN8}xx-XFdG?Dgx{Z{FxANZQWvU;TP^fGqn~ovE!aKdQey
z^*LyoPUNQK<9sqTpH0>M=Yev0rq0oM!RJ^1F`DqoVnwRvY2UhnOZzry)ZX1yDs+2;
zQk~Pnf>&2ork|fT*C6rG$;s*)d0e-)wzODSSgcsF!o}S^*mU_Ckt+)r1Y~7%Q*&d3
zqS7+9ug{%-GUT|UuItI!$NBwk6?@-*cVlDn%S%f`j!!JPzAkq2=IzhJxT`GWuD?ig
zEtR<vw5PiE`$J9lx~sbnxAUK$W!nA9Vp8k*=hhY$F?%X5UcFlS|KHyqno^y2_EcUz
zcFZj-?Ax(k>ASnjmut0^Tv@;{Vak-0w7hv!XRgfNw(RnXry<83&&E4uJy~oV%;7k-
zz5VzS&&ezLlx35WlaEJ5JbC_HIwJq7oW<-{7At<9t`++6;}DNp^M{Y$wr$z6#Hvqa
z)w*@twr%6%<C|w+|8HCF?G=5^E5yac(-RXJ83Y9dYkz-RYS4KFWbf3ere$w#9PgLk
z{@Q+X_t`S0eC-8llmA|Lz43ZXW#65tr>DF3NH~_tWKNnkEiEnYpOWd;#~+UFO1_r7
zn>))&vfSTh_ZFW}->+E-3H$5*uIN+Vsim!b_RN{QdwYH!m#<$Fy}j?1MTb{sXJ_)U
zo|S9X@bK|1tLe(l)OvhtTW<B|XJ-YK-BRw^{`=6=-4O0ss&vI>@1A|XKeonO7Hyhq
zRhH8@nfu=ozqwX-`#N9V-v0iy^lst3&p23ZRtYbBrm6a4{c`c?e=dETZoNG}nekWd
zrcIl;#dI!Qy?S?7>FYjO>lJ;?8j~kYN=i))4GSy#_{jAXcff>oQX&i@qN2^s&9btx
zpC>Ur?u-1a#kJC}WtQkgl?9sDuSrTt?Xs{|S5f)$?yj|k#f(=mLCZo{hh1MEzkSP=
zGj$In>x-1;M|+hjU0K7y`set)&v9<=UR+*&e(vq<rKP1S{Zw9ldwcu&n>S~k7V+}(
zUSAhm{o_O8Dq%&gCCisbN5MK!XJ#0N9A})cOG<<xB_-wDT<h&QH#e29e}C6HBP?=;
zU#Zd+AC8}Q+@CL7dR#`%s^rDm=<R&6R!^QkKmPcmj*iZ?wbAM#Tr2rBe4jjd^5^g0
z`oCYVAG&hlhI8`HrTHMwaN2Eub5OJW&5q*a<9&MzAGiHTH<_ZR;;9$E@6Y!8b(gPR
z?d|E|;pe}8^=j;{l9w}$(^vGh9nn%(KYr<wk+$~fyLa!NJJ<Kh!gR?>`xoh;A@r1!
zLerdH$6c-wx&3W{72l-1J*O76FQ3~Lcg$yw#l^#iA0K|wq#w6uhC$+?!|nXOzP{(@
zTDQN}uV2qAZMG(Ue_WXB?&~G-mAqNY{yUs?E4_L9hp@2sJ)2o}wZA4RyBECrn*X?H
zXYewgkKew@nPgmedV0F6_k$(Nm-9;+sc37z{_^tjwr$%&(p??}1Oz1H<^B8e@-j0!
z-<jFw{jV(KJqvlc7~<mMf|vOS3K_ZS$z@e7232>Vp@C_CJa=!p^SoP3_tw_z>*<?c
zgocLd#_ag;<KyGEx3+%%^hwG>Kz;uoyW{=x^NrK_<}ST*`D;^A`@fLmj+a*1=$rmq
zQk(j4wQ2S>P-z1y09spH&CJZq^Y6_uNNj3vXP<rc--pFlu3S-cZW9p^si>;*I$;}m
zrNqn2OGigX&Zc5R>FaBv+F=2*&xJ3Q>+dsTXK$Y{LBP_?_FnCy+3eyVX9i`KJ=)u>
zmb&x&_Po2l?$`gXwd^&Xeb%n*&5e_j)w#L2+W6)7Re#UZO7hw`(>OhDU(HOu(8A+u
zis!Qgf!Y)0FTcN+H+sC!+rH+9f!KntUf$l)76lK^&NiQ}7yIhz)6m@vtPHWScQ<S>
zaCLQ6S6{wjg~lo;CBqdfR&3b3x%$I{gR@MtRn{`FX6Cbtdiwa7$QwWN4VyOa=dV8H
z>7W+N<U``-a+Nz87JRz0GI+B3_04}AotzG}a0&+n1*!SWDERZE(ACxTV@r<QyZig?
zuN_*nVZnt1Y1O|OjaT+HFY&y0Ih*tOGDFD+PhV!Hww{*1^7!7~>aeh|pFe-@E`R^;
z?e_cowngUTyjc;rxWy?kAwgm5WS2#1hn^Ox`^*p!6x^6{a?)Ju@*vsed}kN3u{NAJ
zbH*_Fm`&xUC4UdZ^RC(U=*q$sibZliRjSTTot*qTewRsBNb|3vDO0ALnPYkR_;GiC
z|MSO>FL&pE)A2_(C^#52V6kjzsm#WZd>+4`4k?w#^L}sc_6`XNWnyMx<B@PEmGLb9
z`RVD&$?B6;D&O7N`SJ1b`s&BC&GTo?oVhV#&4L9AduEv$*Vx@Zc~aBB;KTL!dRy>7
z&Z}!aSzF8A-nzT1H2cPeM9J;_c2?G*2Uah#5}Z`Be$h9TIrCR8Tv%spwX2a;gF!<}
ztE#&C{jII9udWVfX5+CaeHAjfEhnv{wDkR*eYrU~e?I*>cRa4RmC5dE=Y%ERyf?pG
zd0+mP@7DI@qzw#fukWk<os*kuDdU%tlCqaogMmlV=*gv}-lC$S*REYVdGe&Myg+AX
zfW{ot?68D{1(PRVUg+GuqVIF)*(I_eEvKLQ&9%CE`n31Y>5O}0^bJx&SNOH42zpLB
z&-`zC-RlL7jgAEc8@6rx#{cHYx3{<X`1oedoN1hX?#}l7`{{SrUNE0BeLARq+gla7
z_Opg^IzOBDyjLzBLY%CBmUQpm-)Vb!b@=)P3l>~k8y!5?W6?yFok>T#WUb3yynM;Z
z%36E<k<@g(*jWaNPOq=6J$&epRmqDBC5jVo=)~;!@Z{uVGcz+gJG)6XwJw`iIWjh=
zoCGZnytjAu<yFP=CF^pgehX1|n&f)ox!t!ppTB)Pda#*2erJ)Z>(k@*E1o=kYH4lV
z-qu$3{@&fomp6M)*Xy0Jx$^PnpT*D4SQb8NSu`#1z2rTkPZ4no{5e*gmY-XCwq7nP
zFYn#s$IANs@*(?{E?RWx>eZ{qk1ziw<8%M%o12@1m-)QBu+TXoB4V~#?kpu0A!&2-
z?X|zZ&73tWZfDWc!#2&6I``H6ty*PlqSmBvtVeS3f&~H<FKf;v+ne4M)M2fub~de;
z`d)5_^%Hykjhi=r{{B5YAmGQ$y@eJs>FMcg%?}?xE`D^R^V-_z@9*yJ-nhst-$d%{
zv14u;B5iC{duwYVWABHwE%qwaP|;j<`tzgye*LP-(^s#4{qytlrcIkd)F)*~N=l}s
zr3nfO3ak6=DS3IRzjx}Ji;LT*O`F!<?p{!^Vej6(bLPY>e(v&UW8vdtbFIty<ZLVq
z3<5r#pZn|@->-wM+~Ujp=6YSa>86){rNlizmOXI&tJrJJlfNy^pRX6QW5YSi8FdjQ
zC0jOcE_QP}w#0LCpNwVG<EPT^PE1t({OoM<iB(@pzJ^>oHgPA%sn78?7K!13KeDc_
z3Y~41yU40<$?n?U+t#kl&C7cS8j4sK>n(8P{F~h8=jNWCrfY0u#KpxG6cm)TW`ajf
z+1p!NtG>Rvx3}8e&CTnmb>QZ;j*JRMMwin5{P^_g)H-4QZ7(e+g4z{JE&b)@%n5)m
zl<3{}|IndBpaODN$;#m6e$UU(e}8>_{9^e<J9ia5?K-@~)5h-0jG9eTA4aTtte_>D
ze#Ew*{6VP5oBR9Y7ljp1^jYpdf1Yi%*<3&S(pOhnIEAHB=PznK`|Im#aRGq?4-2GH
z=X==n$=U9z_?YDEe0ar*74k3o9Xp?u>F?XtCu`j&ZT{}j(r*2{`AZF-gw!20oT?j=
z81<!Dxa`oX^5uSWXW7@+1<N>vg<ZRNF)%jv?)B^6e|&tLV^)1(fusFRwTn(KUca6_
zYsQaXRr_~Uhq*1>8ninuMBVAiiE6P=52i3P@3?pGUhp!Xg;sqo!d$KOe?A_Uum7`f
z=~7WCDXXe4FOJ>%+##rZ>eQ*4ni^0koOp=EQ|??-mUa2NJ58+IB_$>6;`gsxxX`h5
zPFvQww+oacB_$W725n_OdsY6szMt%^OHWjOs8*ewn0h*9SH;C6N0zMQ|Ni2lvb+27
z^z-v(&YnHrt~RP`9qSb@PftT*^ZKu!QkMs6mz)g?mSyjZe)jjWzrJ2rP!JCfhlB}e
z`d1~&)3Y<<-tk6e_P9M28#U8-W|-&STj<<=tWUQ3-=9jg$v0p8cJDoX|Ni}>M_s+W
zUq3rL`;zHq$It5>85gWrapKmkU(e3YUc0;hz1>TWUd4-tj|3=J)fWH%+z`6f)O0;7
z>(@6ogKgM#_4VU-6fE4md-t+s%i7!9CAjWCd-(SDb}=!rj~_oq9n)*u_W$CG8w{6&
z_Hg|D?)oF`EuU=D`dDKFg9TQ7F4Z3&wQk*78WeO1v^q-0@{-Hqd7hKzSe3q-X`KG%
z?c2L|?@Dj{Zj)7V@ZiD8YQ8~%fjcWdFPk?{?rPya|L%&!l`B@Xw6^-rFnIX(v3KF|
z<Da$e2hQa%wFJ#>uc?yTu&Vg^x$?aGV$qCT3^IJrpPZZwx+z0n-`w2%|NZ~}jz52#
z^5$zcWMclz*|YbL%&V-p6&Mf;Y5_j+Soomx{UY&)vwM4b-rd>P%lcxwtDD=m_xJzb
z+?*c0J<qoERmf%E-Od6nyLaziyLRpR_3xWlxucw-r%vhW>nkfNnlybnKR5T}T}&cT
z9@Z;50yJXwRD8U*xB96mPu2Mk$=chOS_x)pIn{klUcWP6xQ3lirsC--(N$IH%a<*)
zva&jM^eC^4#fLjPi}(Ne#I0($gnRyZ&<fv`!OKN%e|uwlH`-0k+Snwz=~b8e^JP!%
zxy=j>KYsrF`{d3`v-UQ#^Xo-z@dyk1_UGs4Y15`9StON~Zr!@Iw4mU_p32YL^6%@l
zDKC02BrUC7rRw71^5w1e)X!Ze&ka52D{>^gxv{aRs3`oe{vO{C!B=Gu1|8VGJ*^`?
zROY>%=Kq^pKR!MlzprNIp`V%|p`putXNLs_K79Q6@w<2HR<F*^$nXf2$aGwM@mRn7
z`RAWOV|&8uSG=5k^2Ea{3(p3WR@VLDyuD~khPb%6zP|p+`6nhg)!5ZPJk%;}mIJzV
zsG{P>gM-aSi({<j=H=(hi;6yd{rdFdk8V8@it`>XYODD3<KvFP$E#MaZf<H?vYoq>
zdzGUr8^7F|<;$NxuhskAI6bI3IWjc!)!yz`M|JCj%dhORFwVQTr*gU9+({EBE;Qqx
z>t}9m{{Hs%_fxgQ@7%pxTvX)NFV|c9=e23p6%AkCv#0Bx7t8%xQ6hCH?9w7D!7Lw9
zR{iAu9_`bfMa9Lp_t)Fc^;>Q=@rjm>&X#T4zJ2>92Of9qnBhLhVCET4VYQaFwqHMf
zY>3c#xwdrH%qdf*Y)n2rZQ8WBeKk7;(_IDji<Ri_+qPkYLA3SmuC7&ieC?*aZ&vm-
zOS#{?y&N=t*0tFz_tuuIt6Hlb*E~8q+gwjiZ{oy>cXyS3{_^Dr+r_pI-@lialsw_g
zv{l=^{Z*1*smv9NE@OF{g83)3&)?sie*RD^_sTxyi}UPivvPCi&YERa{_f7+>hD3O
z!9j_M4?jFSEG#Ts{^o{Z`MVe|qXqnzE?@Tb@!7O_^WMFC-@bXXqVIEX)R7rW^k>eP
zai^y*@2Z#X!#_RY7k{tlYnF1ncslw!fA7PoPj~Fz?H$F)#Sj%0m6(`V|Nrmz%gg<H
zdwct3ES*Y~G!v4Og#`r{YMigDxMleHU*@XE3*09Bw=sFwp?XmJ<HwH&nZDd^YHDKR
zk@)cH>gw9x-(+nnCLCI66&f0P_|Tz}($eX=(a#<}bPVNTVYqqorkI#mWMt&KckeF6
zH8^$WvWr&!{k3)H&Yz#3pSR|jcfazEy~W}kU#fopxRiW8{-(~d`Rmv3-05OI<L<7~
z(<e`A+S=Ydc<|wgiOM?^;`Yw7ug^=%oA>Y6)^jhWS}ps(*oyC0VdTd2$$#pCLqa%q
z6s4Y+pcry|V#e+5`Q^pM!C_&bNH)z9xmw7;U}R*}#v?gt=FG}hS5~G4{PkG;=g%Kd
zbGY``mj?%%&n|Y{<$8SqLqJ4CMrx|4u<+$w?Xl(UhfUay-eteYUm;uk;^)+<YooUA
z5)-+yyF7o@<16b|tjKtFW~R7)+?yL4AOHOP{P*|w{-sKu(<e_hE_-vMonJod>)cyc
z%gh}QziYnq-1Wyb@rAQ*ZOxY#lDV_gTYS}Hk9DtJy|OHRwqy71@|TyMetLR(*REY5
z>c2EhO-oNqP-JCg71xWAuoeryCc663qa<$b?sk6pyxZGye|~xzay(N;_e<O#_EoD^
zJ$(3Z+qP}_!hat5-T1V+Yo%YyK4#BJC&R<-_rF=TdUf^pbFwQn92gy3U5}nWpa1{g
z-<vmYI=AuM+?49g{a+(O-`Dr-iWM5c!I#ay``cUCMh3(>mMVRb1r?+BABiuVUG?Qf
z;rDa0Iu834m6elUTv+(--8)eCxcGUWi|7J|1*=y}%gL=<vEs#>H#Lus@veHjML|?l
zw5+U*U*0ZecUkUcO$Wx?j;<|E4;Q(1e|vv_{e@c_zVpP3@g17V=ESIfvAXV@^ZI{(
zf3vf&tXRD|c<zhcswyfzvrJyz+?<}4mbTn~eqNX9_gTVn{pQ8beE9kKmwww@?fZGs
z!!3)f1Xs;(2%lMyS9S05g22UYv3n{e9@=?HCEmrQW%1(VxVU+<XWzE}|3`V_YT=_t
zkM_z~25E?}v9Nr3eZBvc#aa33r?+n3-rv&`vnl1|xw+OW`!3H=`J$?qW+Ih)e_!qY
zf4}oz-d?hs-N&nV(h9$pR|nT?b=q4hK3_O}>(<hglqDu&3m7gecJJS@V~3=1T1!id
zgrsC^TU*dvk41(v&s<*azdmN?CJW_%_ZT;KHBApucl!DuROE=<)>G9|Po6vx5fN!&
zePNz+Z_mv$XL?RQ{q^%_?!7&cN!$SppplWQtHWEJ9{%|FSl+hErF2eO-}&d=&q98F
zy&gY1l9B5M-!Dmi{(ed0Gy$K=#hU4%)``<AIgjd|?6gk|pY=KA+nby0d@>nqley>F
z*VnzfvvcXvrP^U@ZfwaEZe>nbdg$NZ-}S%Wnyb0*oxXZ{Q1#0f4Hub0MfP;rXNFJw
zpLDcKG<sXkLC5*=!9hVrxwp1-i|b2CN&Wiy^X2Q;!E;wQIypJ5S+hn&RMgqYX^HCQ
zD}r3DmzVp0|M;=6sOZy&4-=mFtJ$+<UpU{|)^_XG&CsyW@^XtyyV`ZH7G8LMs%ZZ4
z`5V?(P5*d3zJ6`w=C)TBy=uC;XD?p-`0wxUzrVkO2A2E!_^x)U+(OK2oLheNtMAIb
zX1nl7Utcu-Zh5t7&)wbS`cYdtURg}i*3>+C?%cbdpP%#b@u~aGc~D>xqW+8T-@kv~
zzI~IDl1fZW+#y!DSlP4a;i1-d@7_H=-v9mEx4db&!e`em`&YSa*|Lj)fv>W(`p@~-
z*S=P`T6jUdEAH;T`V(`)e?2?WDSUP%18aldTq{ry_2;Lj-{0LeHZroR`C)LiQ)SlL
zwYe1)JDA$PKDe@E&5g|=#~tUN@aOBdO>!){Us+X^#Q2MM-n@B#e|@d~_~>Y#thJ4;
z?b6i@tPIl9voA0AfBxi2L{wB-dU~+z@|}gn#l_|2=BB253m!Uc)&b?4{m-th4uAIS
z*~yb9_w&TANdIVl&vg>Z&dzWly)!8%FVxo7%=)-RK~PZe@-pAkCr)hGvc;zC&5UKs
z)UI}_T*sUmVPr@-I@kYG^a_RZJBxmP+K_m7MW6D*A2t8FM72R(zg?x-`uh4S|GCQA
zR+W5vbMx1)Uk48!TpRV!<KQt+eC6KUl=|VpK}AKyqN1XZ<C$6!&2^!Ujt6%XK7My+
z=i*G0jkk|5Ta;>Sy_7FsS@x5^wDjwO)#1Ipy({}JZ|Ump7T1fZcy>mzwDhZUJKxRQ
zkBc_NPn|p0Hto!emX?-nf0s%3-ERzT57O36s?IubY}?ygtns_Afu}J%R&39|Z)IWO
z;_AvPWpd*7?PzcBSLsPNw`O0zwA8!$%L_pfktJ)^_>{`5++O_roN4y8FYoSJ8yRh?
z?0<5&`eXdkWy{0_1rw8#Cx1R+{qG4&T`1?(&M1vZD#1<HY(K{@yu9-AUF-5$wTxU0
zUS3{ua&q_X-Ba=On_-an>dH!0)gPZ$M}s<Lix-2YsY>5Japq5FTtDfRi>`>Kh~3^T
zI;JVf+w<<eySKBM^~Lo&yGmD=zrPn47`XrMx7%M|U!U#B$i?72P3Pu@L}w?brnPHx
zBO@iR7B0CxQQ7_4n#kmg3<)8jOMAEj4wPM};$&%R<rbH-D)DgKoc&&MpUzRU5cRCc
z7N&c`^I325+x-6aR$4;h#-+S3De39w=U5tRXm~_N-@bdd_Q{EfA?lL?Y~=EPeR-LF
zeqQJ$vv*HlX&#<<YUi#c(;LF47Sy@L1wHui@bKTie+yY}t-rZ7d;09z(vwfx6g}zK
zyxI6_q14STQSEK{_w8za6xiF_o6Ytum01Z&3O6^Wzkc!J&YsH6swGDjt&iV7?-h?w
zuiN5YY4g65DRH~s{QUM&)pYMgqfbA+sj1cEy=)Gi{CN_Sj;<~<GjsF<T{%`(*7Wpr
zUtiz)`u~rQ_lt{*_e&Tq+F9oJU$VsN?zL;%7A;azI-zc^84!Daj^=Wo(iPL1!UHAh
zFYPlhExNSCv#zc#$b5sFO;=Z!jI8YUH#ZkAUMwsml$3RgN4?m<*tq)lH&A2#@$vrL
z+}zm**Hk8zy}Yz^rg8eV+}mbaTB~wzZwomt$ezQVy8<*1I>&Od+TshvuPUeZHLvKq
z{HM_SzMVy4{G`7X*VaU~^T}pCoWA+Wl`ERT%kJzfUY>n@UHZ8>FK=#EzuKvCWu9%d
zn0}nj;moVqv$iMgo)>aF^Pd*?<;VWFraoV)6T7SA_qVqeHJkmm=ie_YDe>|5FE1*x
zs`;^D&mNnrowK++C#_qzPEJm)tgP(ayLY;OcKwZQYHpU5k-2i^%A&=Kzkd0$qEA`v
zhWwU=F|o1COiVLO;_iI>m@lj!EW3QB_G#aJ<p-9i{Qmgs>+60w+b(gV*x0+X&GYl_
z>@c*n+!?>W?*E_9{SFEXtfr_$ZONFpeY^RGuH^oRC+h8B{U^P4lwVk4qo4Zx++6Fj
zHwT!$c&m7V8WuM;Bvw{dy7$RgR)5Pe)^lKNQ0SPf?tkm%&7VJi>PBo}klrb!VP$pe
z<VjCQM@1c-HBUtrG`wAKUt3QvZ%wOfT->}c`Sfj0pWm-`<YZj`%J+^W<Mt;X9v<GD
zc6O0fUrJ|xe|ceH<*zR<7dSTi&9z#(dbM|{lBWo44r$rdbgu5jZl!acb;ig3Q@hse
z7B=6mw(8On&yeF2U4H)jd3l*HHxG}Afx(AQpI+VDYkjrz$dX;VZXG-J?AF$7H8r)`
zUtbnl^)1oX*0wHu#4^_pv~c<McKxe`N4j0^zdYJ4zDxIIGMDO;Cr_A|m{$A?_Fi%7
z)TyK0;=(M7>gwCm&&w?g@F-RC{PN`F<g&N7ga10^JrkKaMM+tCB}nau4+R%zeAf(K
zHf8G6Kw0-YT3T9d{PKQZwL`K_oIH7VM`80TdnwipvR~Xo!o&Ahd}OLG^>z35Ud`I)
z@JsLd`uO>V$;VDl*LUxcD13CJ6TBkD<55Ux=-!&2K~d)}-`<{oewJzW*;%H2eSIP6
znJdcP-ZK5YGr5jo{`vULX=kskjeh#%$+!3S?Mq*UtP&1vRyq0PNy;13D+0RTbosLb
zT9$-HM&A7N^t7w93@E{uzl&KVyii_TOe`%ctLnjl#>&sn0_U>+-LYlMmXnj!@9!=@
zf8)lFKR-X;h-O=uf56a_!&AGm@~3k<Uu#RtiWMu)%r-xN=8TTAa`USIj;%52=jXk>
zy88Ox-`~U6L<kBCH?wLu-ETA1aJqE%$`uns!@^@dk_#6uyt}K^y6%s~)lPw|!c(VC
zE%TfE>d_-5b#-wWnLVYi!_+2wmTI)9$X;8(;c4sYdi3$}es(stQ>Ra#pJjUb(xp#l
zXPbw#JFQsS(cAm>>gwt3?d`3tu1-!)o9EQaEnL=Ms^N6DrM30wWOe^#J~NBHzPh?D
zc6U`pMZjE+g=u{%C)Y-A=aVwgu(hqdzAl!Lkx@6-HnF9S?b-qkQ~6V;PA&JJzpvz_
zQ1{VUR;8<YdwI<|l&-iG6&0O1d-m>~J0&G0@9yo@o@LnkDuBaNc#WfARvu^>kWJ+y
z@C^1-z1UqcHWdQP_dBd8oNHUXE&KYqg$oxNr=RQT?mqmqSP*n}t<n{ql`B`ywXgs8
z>FMdNuCC9|&&yktytusFe<fdQK=Fjhla1^CREUd<Tg`p<_;K-@8-|vamaCi<Ri<uW
zb6s(B)22;cUS3f<3P58q)!*Lito!@x)vGL!f}<{H=l}cnkB^Vf)6?_YH_%wFM^_?m
zS3vQy1q(KmzrPo~J+JiLoyx;)ytdWf_EdgWySh+dlAZi4dr6+Q{{H^u%a`xkWd#z;
zi)dE5@`qQ#K*7$g?(wnS%*@QgZM?Is%kv5g4M7T|<mBYq+S>m8`!`w5x9ZaqPP2NY
zD;gjNx+WOSTphmN&%;B)Ea%3<!|lAhynp}vS;5yjVae%p=eBLzHqWY5Yx2o2A3yFa
zc*q3O7Xi}8`tr`s;*^vp7Z<y0YiYfC`!+W>xA2o%%f4fW4=>KXzAk)SOkqjMk$dww
zOszlxAIZ@)!!-Na<>mhBesd~bUQ(U2hjXERZN|ST6DNNB{Mowx-=0~sq^>R$$jSp5
zuX^dql`o$@9XdH%;Mb*<!OLCF&Y!IAU-#)r=i0TpptLv-r0VIfoKMS2UtfFp#CL@%
zXdZP(;p1h#v(>Wx*_6MN0lBpVq|9}-ZRO?o`oF0uDJ_%D7f$oM@<&oa;>D9EQ>IL5
zIXj=j)O3xbpp(3Me$~%Us($xcqGqjGqZ9a9efHUXB`+`C+?*a78VZW+&+VofP8<yC
z`A)i5Pfk|fUHV!~LgK}nH-Da-6qc5jUL~w}B`ATnOW+p=NQrw`*tFKxqdk(wH*&V^
z-~WH6ar%az!it55q(l_IaL(Doxs^rTf8Lv0TeAZKCUkdu2L(;qy44iqx55o<t{hh%
zZaz7ijZa2GT-?1>qeaRwEp6F~pTT~9?>;>}Ehi_ZrKJVRSgK&-LKmH!Ev6qQBQ1S<
zef<8PpP!41iU!JZEtFRg;Q06Zy}h&Z;ROp8Xa+B{sr+>0)Tu6T0T9uw)Uwaj(J}G*
zy4a5&KQ^=RT2*{luy5bLlatjyUbC5GRsR0o+uPgUKRVhS8XC&X#N;>MuJ`PGjzI40
zfR=#2RbO5Rii$pc_s&n?NROm(8;|6t7Z;URIV)aKaCLPpetu5YBqJax>CvgF+LI?u
z($Upj$=50Xa_toZ7RMdCcG=bct2x-j3bN$#^7D&ay9=ZORUX>d*zoZ1@X6cRq@0-0
z&M*J(_xt^zhH1MK2O~Ia^-i5S_50h~-Sz+L?(eJp{_gJT_3Ph1eCSxJ(emoUmzS3(
zD!cp5wVJBpSzi8K+B{D}N-9W}OYw`~wFMjt-(OoDzJA7x8Og`{I9Zxj1~2E6Hq$XQ
zocJn$<LZhvYht!$O`SOLVz0D$+WC2Z-`q3~4-a4Etk_})iuV`p0xX8b&(1t~@&vTa
zbyw-?@bz)t)Ae+9b(dO+XT4-%W(KW`dcXg_l$6xHU8S!-JUndi*~V!SXLdl#h5q2c
zK+B2`59V5zSABc4GyA$;ZSCLU=jQ@txwZyKN=lZMl|4Szd;0Y0+dGTXV`A<cXk-Qj
z*Dozl%Cujde}7;3`+K$}F9fEaE-fpQGtIgJDS)B`?B?%JKF;Ute7IX&|JLo>_SN5V
zPEJxSv6}lThGSv=K`9Z%D;yanx8_(D&$F+$D}Hw7_4W1Al9GQv9+yu|O$~YObT;0{
z$LIaMz0>vM*DYLF`1Mt&y!`xaxwk>V7Ge*wzt=4`_V102$%cl8cXyZH-<BIaT`zXe
z-n}dNT4$|oYHEtyoW?6@^yKH~=b)0r$4ADxOy}xC0Vm#8Qw^sT6L}>J9{l_J`}eoE
z+S=O11qGmn-M>GUR~H6scAE60Q&^pcho?{8-md7$iI$d@UTO1Fr%naU<yQP63(5zk
zc2d3D5)L*!KR<u>u3c@dtzTbUOuo7*R8&+H<ao!Vq(>VP52u}*<GK9u`8k%ta&qg|
zulFz2Xt_{+807g76BCm=ckeD<ym)uX%TMq3|1W!a>FD|M{jUN%x~5K_{{Q1~`9&`?
zpP!%q|IJO~^z?L4gx3jxQp+;aiVqKV?AUR4clrM6@8|B_yLaRWi~1IK`v;endbhQ;
z-QJdKY-F^n`1!eie}7M%Iu#sYg`iwDOJ=U$|G(ewgAO6dy|tyIqhp?JwO+)A1sgU@
zcopMeWmEs}&ZSG2u3h_fcXxU9_jh~C-^bnGR|_^E5@f&;m*C*fPfkw${_gJU>+A0~
zGP8gA^5xsRyQiOjUU{xn<@)aO_tSJDFYT@VzB+upm8GR^^|zddhgwaf_7y8$k$Et)
zM?gqBIr;I788eL2&z(4RN-t{5iuCjI5)%_c+MR^l6$E6gN<94i^S{5lYg_%Tr?V5(
zKAdfqyW(7{K$Z!M=cJHy2ac^SPoF-WZJvK`Uu|`0=+v!Szn+_G4O&>nr)sxx<HnUM
zSIY0J{qy6a{r^A4$;WttgD>;*uUffs=8PEvERL=YTCQ$xdU1O!6crcx&Nc%zU{|ls
z=KiCqaOF)RZ<oNLORK}zhpmZt`1rB1vhwDNkB_dd4%gPu2#{TMa_ZEnrXS|doH66Z
z*6i!c&M8}hhHP_gZt51--xRTCsrU4CQCq#@;^w^y;8>{EXsY4VQDvTY=f~gg_vLM?
zmaJU)^Su3kpV?+>Z*H&7%*;$pPrtr9GBs6o_E|A!xl>c8PR+fwCG+~a*ciR*=g#H*
z{`OYZx@?J+h|>zr#QK01fv;`+@^bR>^)D_c+Re95KGyT^@9+EjYG=O+@Xfxrr}Ar_
zap|io(*hbMsl0sm?%vMg=Vxb|zkdDN-Nog{uU|=wT>+ax<%NgRg^Vqh#m|;3S)voS
z=f}gt?al1`Wo2dOPMun06&_qzRFpL(cS)>|kI(;qf0z5uKlk_}pPY?^yu5wU6Aw@#
zJi_??sB(+Sxy|Y4r|Co%y}f11*Z%m~vu#O7yV7pT+*lvK|Hh3Q>tkE4UcFjeRP>@`
z*Nos!-?X%6^7VfT!P81R6Am_kD#tmWIksxt;Oi2YWWmP9=IH3i$jF#`dt2=8vfSx<
zu{AX{EBLPdm@{Y2zKdK2nU|J4i)si93(LE)!7)Dm|DT_qlaKd-VtHNc?hOpCE0%&v
z6qSWKy1K%`!rSxj$AyHL6h1mKQQ6(D>PyF~fZmfAFMgDEUO($~?+<QqJqZa3(4v><
z=-;odf(p#5*RL<P5^?Ha0;O^l@yRD+w&g??6@6M0xp~o|MZdnhG%kO4=g*&-RmG-!
zzP@Lze%lGLEL^+RHsgZANB69t!oou5cD}hbl|`POC)dU977`LV)+g&-s?j2l3Mwo-
zt$2BR7rXZ#J9Z4TN&dsb!&OyPyUX4_`dAV2Jd35&tZb6X%`;Mt_xIIWCm-vn`}^ze
zzS`f1PZ)!mv3e~6Qww~AoIGY#R8{@^cwGMF%a>JERngmWHg4Ve_t-HTIelGS-%0sj
z_f>!Qb9R1wy2?sXabflMcT-%lVy<4jy1VS{tu2|snVBz7OjIs^fA8*McYaVrsxWP9
zR%%&ta*5~UdA8NxzTdA;Pfx#n^JcGvp^}~5zijFMzh19rV`DqsCo7zO=>4KKYjP43
z1AhxPDeNkHdy9uFV3LY!tbO>pn3e0+y?gyyT2Qc&Rm158GpJ||0xhBgrFKuxnWouc
zB_&%nZrr$f_3P=zopu!;9$Z-&%mCUn-_XFIHOcHm`u4l3|J`aF9S<&WY`$~n&c2Pl
zB_&@P7@5`l=fwmBe0abAzg_LGE9cJn<!}l*1%VUbk_acKL!X|Wc6N4t`sB$p-RNh}
zo*jEw;9@UjS@h)V>+6gS3LIkHrGI~Y{rC5G@XW8;TH4ytTeCvr;{H9HxMhn;XRPXq
zJ$q`NoS69i{eAWZc7C}pZ*G3x%sbD%{@t@@&)&a3pJMd<+}zz|Z=*^}OSzhqT2$n~
z;T6#@XY1tT#Kg>OUGic>;bS*j+r5vE_j9)%I&i=tkdcc)#j|K<3^#Z8lP4)&hp+jD
zg?)2gAJ(Ms>-&5CjjS5|$2V`?Dk{2w;X`C_Mqb{#KR-Y3E`9y%{rmj<{Qpl*PPVQ7
zmcZC0@Jk9DjOPy>Qc_fWc%YHF#;*R_n#jFXUqRE<Z&#XoPSVlQF)=niee&c&gXh<4
zcKr04v}E&U<E@GN+Eh-i3|`*W)>c=cqN1Xrq{PIarKR=n-|zg`*t^Z_{NLW*{(fz3
zw4I$@RaMoMR#y&B#y#Tw0$FxmUS6P?rQ+v)cJt$(vw@1>uCAbf0D<BS`Hw&UTpPVz
zPe*6T#f<!Wdou6tGQHaQYD31wMSH8i*L_QPe{b*e^Yi!T-QBfjjZSRr-LtdJZ{NNh
zEX1#<QVPlfrrN2gsr&zY@=i>AcwH}Vv3oxs7uPCV%haeSsp+Rh!w>AzjozlCq2Uo4
z`t{k_*;mvQIh@?xpKm^C{r}(Z_oms`YTjzTdi5&%`ntK+<$8WUI|Z^VKtXi&)UB=A
zpi=g2?wq-EZEb8^%wLzkzqdE#q|io}$x_|o`uDbGUvJOvm$5WTKR3r}y9$SstE;M|
zWu&jKZ_G;pr?V{0uNXKKuVmcYQ@J_!_O>c>>*{Z7rcJwcZEZB8g0ZnMgWGD&?31TV
zD);3Jad}F;U48m^cdvU|UR2br_Wat~+UMuydJB}AM|4=-s;a6wH{U+KM^ez~EF-9{
zewKW8mT72cDDTap+uL&A-r5?yC1c`?+c#xRw%%B?tVhlCT-wvRA67<2MW3FWWcbi)
zoyOw0CVv0Dt5?4sUK7EgctyZMPfqcQkBErK9E-w5jncv_8!J90<>$Xw>s7h>=<(yV
zYu5az`L}&{&P0`&)2DAwKF+t1)tSSTA5@Ob;;gLv$;vHO^74|a+p>SVUoF~o>gG*L
zW#z>$7kkdXawVj=_;XiR7h9iX#P+;cM@Prt<xU)?>J|DR8#hF(sr~(pvB5#1;^QMw
zKUhduc)m^Lryn05KY8**KRsK?bJFq0m6er&f?<n7LqliInDONKbMrr%pFVv$di1D;
z5vSsn0<h!%e5}~><jIp;p)b3|^;?`Cw)5IlODEI^OiVQFe<CX4<?Wqq|KsKi6}8DL
zmoLA*)O)&K>@EojiH?8aetv#$5(S-tct8z%?;}r(=G)c&dUA5|+1cjmYHCTmzYboQ
z;XdDX(SijD85t3~AHBJ<Qh2W4auqqJ6*mu5f8<y=t>b9Y*Ecser=Oq4FK6T7<Fmx7
zFD21H;`6h!zXJ-hva*8aawuNm0oD6XFO;Pi#l*y_zr9&##o@GKCa9IxA~kW+nwt?z
z%cNL@8^RBs_0{j+d;DzWag+HgPkC0oE}#9Xf<sY76O_Iex-DF?ts?gK!K(T3p|OqO
z5gx&zs+V4X;^6{2nDoz1Uj6?)4=W!7L*%@fFCRx1JZ_w@cws`;@?67~rK`9YZ1PNx
zDhfJzOnM;ml|%7MamL{<<<~?O?Ah|c{qnQDygSb6>xw;FB2t}G`Q}S{-8u$_7hP*V
zG<6F&$v;pPRBWm1y7lTN7Y_r&rXL4|x8IHTX`TP!)%L|x8Ct5M)dWPIICCg=wtRmi
z==4H;%gig=c29b7y2<;vy}pBMOJlyA|LZHfEnDT1C1z=r<~x5{wQ$#-KXnS<bKlB0
zt(m!4=ey&>-sflTJ)Xj2+S#?;wJ!XJslUl8TjS*h%8lV(O-*vE_sTFZM4qgk_2=Y1
z0VfY7aG2WnW!{P1Vqtr-cB1Tre_PLX)t+0b{_H~o3&Vo1m+$@C?N)uX-@a^a>_mYn
zdp<T?xDa;xXo%--+sUklc$_P}YkT`lyZJMt`|HedJuMer{r7$U)ew6-)Bk_u*Z-Mk
z#BNzK-*myViNV@&tTx$C>eZb%OwB+AnUlTWI`7%D*8iA#KciMeE%uBsZ`X3(%l3QS
z7VlNk|9fn4=e3Q(v2nI1SQ&Q8CGXCSWN6vb6j!ar?ZFWBJ5EZrbJC9=lbrMz1Y~62
z9=+y}ws%hQSG8vCTe%a@gDhrZ(60!-)EmQ~v3sXHcR<U8kONX8iY;|A&L(1<4gak+
zE}Qvzc1YN{fCR>q=Y3+gHM6TY6#ldKF247HvEgE-&+op#e)U}<)0i0@ij0Hni+%o;
z$9~#l`?Q9EAxvBJqTjuo&*Hrs!{su6KU7+`WUYr&-L?9Iy?^vwI2JAgH7Ys&_RP{-
zP@=l_*ZSox^JXoSV)$_R-L!h0i0;knk9F+}lCu!v=$-X<-;SSp?i>sbX~jDk0#+Y7
z`|U(^Br8Kp)cmkm2?mB-wXLsCJhR^$c&oumLa{{!R9Y#1dE7G9)Kp{PGRO7hH+DZ`
zund#s=XQGE6=fxvyW7a#=2XJ9jW4TrI`T6zIJh0YC&ut3r|o2e!uv-v+1TAV6hVVT
z0#5ep{#5#%4d<zx$(9rU-E?}5Zq`c{w#|*=sRv8fA1Y#Cm?^h7m2>Ge<}D{5wcKGZ
z?LY0#p~%?+8p8Oy<5jxmqFwz7Te8iQ6ZYQjKBjAUt88uYx`2bXMe>;WeU?3OS<cAd
zn6WMP-Lf-{lV)T`G%B?$;kdSdL-C97Jel)#k*dFf?v-#o&))vHG^cc1-h(NwQ>HO6
z2+Z+Y6`OVTlV?dnF#`jO>(2XD(;iHIujSVk!Ze{~SJD1=CMPOi*FJCmFUX;&QU}Sd
zmAhZZs4_4J^pyQ-->#KWR~I{VbNTC?m2E}tx>70odp(&M7$kkCOUj!iSocq5X3#im
z^^7|p@UzDJ?>FmS$UODmC(icFMo&$Gf#J{c?OtAmX;m!~Ob%MU5_Ed;JSDznQ_!;0
zY~p;}8LQZSRsTIT|Jc)e)e*70Yx>+?{q1=w`ZDF@<|jw<mM(R&*k`u<b>Z1>I%(<J
ze~+eW?mmAqVy9$8Ia`C%|Gll2PEqpOm3#~g&cX9pS;ReFT<JZ_#GtU{i>PU)s-=AP
zS*H6LZ{IzBp>Se%)wEgl)=nIMH9!^Ti{*Pi)L1=T_UlvX+p5=w5+#1q>tuI-ss6wD
z)S=hY<KEhT-&dq_F6LYH*HiYlzvZnrs3>}Wa57`VlV|Dk?oAT@<a6xbd$~ThMQ=H-
z#@*fHzf^wr{jzB@UwHW%+<N)8wEpAmE#>m3E~txhc^%ujNcG9RpL;HfD7L7`fFkRP
z#ECV3a=kx0c4lTAxR&8Mefsi~4ErC1-t4uWe_vUBbJ72~^S!sEe!Y}>qG7`9;GYk6
zzEHoqZ*_95{r$qY5BJu~Z?z~sK2OO=%w1WrWeFeH$d0Z`&+<x7-WwCnJUW(cS|al2
zI_G33@A@irP?TDzaVmmtP3n{eDQ*#P^0)&kwLxN!K!eL1iY)>{=AdxlP;8m716+D>
zD7GBoeV*pTvGADV>)hMB)Zf14ye}VbGC5<l(cO791<&p_AF`^~OE>j)jqaQ;`}x{?
zGX{gI*{i<2aY|?U9~*Xa?}t;<--o2>D9O#y{B!EEpp!=tXspP|zB|og-Q6vnQnI0*
z_qq=W*TpF8ub21oGOoXD_VRp^L*du6yRrkK_J0e`6=68#l)qo~*0!5p{G)BJ_S?@j
z=#7pz6cnnXx4+X=!^z_l*s1o7*Zix08})uGGQH1LVSj<+)VBEC2RHs7ej4#(YN>hV
zVK)ti2?0~8fA*dF@3-Y@FzX7vJ?~~etZq&-SoJrhIyKalVS%Q0Z9NMp6yET62{_p|
z>a40NeEoBOyLWxkNoB{#vcB{CrhM3vpTEpx4LiexeEI#SWv?!qF2%rcS^3hp`H!Eb
z6t1sca<1^Kk0v8S<Xb*_IabA%Bdpm0EdswdQ@q~&vt7zR?{i(!$+EZ)A7#!z=-P8C
zKiB2;@{5t23|{YBxUIKM6twkNzt^?!+^fS+?dLHmwjALCbu|Nm7nZ&{lXAN5M_9xR
z|5Gb3-(+BL5G$CUcJ1DbAK%us^#&v`I%GQDa8~%@QvN41*L|zJ_&mW6Z~ihax@R+Q
z{pzKz%s)OB268B>h=DS8KyYAD$>o<zKR&$ORXyGN_KfX$=b0E79MA1@ooRkade>@x
z^GC%k4oAWZVpAd*6fWN?ojvE>&nb7GySS!*j4zgRFce$Bp?I<#RFke)De~9YxGueg
zfq}t8<J65ZoyR{)g&K4!zm+V?J<J`j;A`;vH7pFOAI)1B9G30jtoZb>ovT^>yH#;e
z_8kWX1|0`MrywR!5M?<Xy1hFnZx15_Q^>9Lk!$xKl3DQL>HNH>_l@g?a}RL1UhMs`
zy4<BdBqn0!lix26>v2rmHSM+5;<?$WuXir(7jT*cF(xYY_jd7V3=Ai}zF2;sQR)BP
zR?GX>BUYwdYW`86{BLg;_v*)Y_@1Uusi|nWxo`c$pUhlb6^ZVk9zcj<OMtL)@FhN$
zu488>Px$=pO!NEdU%#b4rY@5HxOFM(zNlA!3YLc-{XKIbhuiLRx5JD#nu;i@d<12q
zh0Da{l~)OIUU>6kKj;0gSNP2zeZF_UBc0bgS+i!1(a-I+la}S_)ZCZzS|O=fek9x4
z$>Y+)QZB`oS#vIRvCjD(zq#5v>)WqBOV6ylBjPI8?sTZ%%KG~&di&S?b(<!Bh`Y7^
z|J3Ve0<{+3IKM*BNx3B;7&KtpGV8_x)m^n)TOuENE>C7;a9H;Cy|i@I?l~_4i<SS~
z`#MYdV@lY`(?w3VAANQTlbH5-OT?Gd(!jdpRCkVrYE9r8XHVCx|Kc?tg`@2EvM%P(
zsQr8YUYEA?y_5pRh6g2=|K9zxnfK(YS!q6#!m|%9X-sx8fAm@TeO*P*>`(@W{M|X<
z)eraIE0<PXxZ=w<_09K{TPCo8T~;Ty__{>ji#TiVdBMB&)h=#3`z+N<M`M5Gng>f6
z81ChK7Axg>$}{u!!MudGx}m?eiPvf_zWep?VJ)>b7KX?VYhFA({%M=ldJzVWu=aE6
zcW&>FFgCrtiOuhl<&SnDZiWSae(fw@YT>B?3P&qXQ0<p-_>13~fU5g9Cns#Ve7CCe
z$v(fbpKGUFcw3UQe3cSILrnk1O#+dg8C&BmCx&j;-Fx*(=%!h_=gD?Vn7#9GfzNaP
z-BaY>WZB%<YLSzEYNF3qy)1?RIr*^NE7sI~6Izgc^>m1n_x7!4*RRdB`@*~S%xC)_
zzkE`J|IZe5S|JE2)Ep(I8tJZjCqGd$(|GMAyGj-9$KN?6<8vYkYa-u#v?^ct<=K;y
zOqFq!FW&mg`E_~Nechy^;KFghL`Lnl=Ug@go!a2C$9&uBm0Fgtf(mJsja{dm|D4)i
zGs8!d(?hWKq<7aN)lPE}Ck|6Ka1jMApFx?&i9=BZG}i<w;1vtO4R--24w8%d>z+;=
ziXTp%yPkA9V6KxAhvzC`g_bBbP^F+dYuYq1hKls(!Llw*D^{%vl4Vr9@<&jmbAF|u
zlWurv>DRx%zyGYU+iIAUoIG7WeqY7MM;{*_51!rPwwPDKpy6ankTkR6l{bmj8cr)*
zCr_LxX;WcvRkPvvV@8MP=jOKa%lnmzwFn5Y%S;A)qU8uL6+Jd;6ps3y?JA>y(+bs7
z=g*%%ckbVZx2LD;Gb(6nzyA36xV&6QU|{C;b+W7v1)NsA6jbpHn#-uz5;d)>t1E16
zRA@-ZmEEaDMVsc$n<t^9*fL?p!@U(P0z&4Ik&#UbQkzm&hpnAu#pA>g2rid}%$pRR
zotqo|KF!R`?9{1KzVhsfEgGO{W{$uECr_>n+~&xksM-40QP8PFYOZy;pRe!VTdp$>
zKYa4^>B&>4x-3PVI6B!u<4*#nbLP(7e~~L8JG+}z!)b*iIJvL*_%`}&g|eLDl|5j0
zEb7^~v9PA*&-Ckwp`ll|=ifJyuxK@QT5<DGSwM@xqMpf<gXi)xPFC|h#3Z74We+G>
zas)m*JKOy2t*yU5S;@%C+NPWk@B(?PBEee2i6hX>+Z(iJYOA~=$D-KXWzFpT%d~_*
zjsj&>7uEFi=eO=pQhE90$&~}4Qv{qmsvzzN3JeT<eQj-}&AoPaIlG#SloXdll@<XZ
z<{Wlc4n?h%D^~PK8oPZlbrgu*RdNvI7$I<JBJ$fnbj1rdo2`igP98QN;%uBa0@o~B
z<P;mbche>zhM?eJ(1^cZi-40a-}CvCEtnb?tXR?UN+hU3fy30)bXBmzN!Lj!iHQrh
z)SQtsOmYbbn6PTqs!5ZA>gw#S7D~B9MoOw1x^nzA$YGzVGjrxlW%ph$U(bbB9VHSL
z1rN4nUl$P-UCGy^;^*epwsh&!$H)6u@?Dy<HTSleg2ICI^YcR39XUKz*Ep(r<mJ7)
zu+Vw(<jK*JQ6~zb0%hGVg^KOdsSz|yIWghEg9OmD`(p)8(7=gd^07UYpO2k7<rNz{
z_mu^2XjGJzzP`ThW}~T7r^fCo={%Ygy9czjqve$ahvLg-(-liJw6vrQlU$sfn)Y3N
z#@GBjP}bebX7`(Ea>XakoOyF<s<x4lQOI#eAw?sjqCY=APMtc{y<cvwMWNEw!daVN
zym;X`S#9d{>GN$WFC9JF3Nlr*`SBxFr^v{gGr;Ewl=y}xGu|rr6*hb8FF#&R&Y32e
zLi0h#N-nzB$}N88>{-SG_wUPdUsz^V^5VjV4F<fty_&(xa&B&d8fL2TRZCm@c&~K%
z`+Ix)4vUvd?QMExaj8Y2W8Jp@nNq$xN?u;tl6kqsN%3l@z|^ugHw-I3rIeO_y>Q{e
z!^5CeLaqiTCVT4s?pnC;;i;+G^}pYm+pr5d@wR_|q<QS|$7B8S^<Q6I{l;AXHfi(R
zR~A8@8V~HP9#8silJV`$&31nIWmbHP7(6F=PuIJ<w|aYHW8?n)^}Ea7uJF6ENkm-S
z+QQ<_p32MT&b@ncb90W_>{l)vikb~?7la6Lu=x4;g|Cmxy?p%Sfhy^%g-b+x&lwci
zEnQ{(=)^?j)6?~rTk#2*>rH?C;>C{Y@9!L&*;-p$6B)JYyu7@o>&I`~y!rFz&&Gy^
z2X8(VaN>20-(P2YRT8wyFa696&^$+Po|&zhtYXHG$_K{<P267RU31MeS8m?Jm~f;+
zFht!c>C?Nr*52NySBJ0flefQj`EoF5;>bl;r1{{6ty^_9H9vm+`t|$w?c2761qCg#
z5^?fal*sEf6SQ2<y38l`{*@C?O?vx2+x*v&id|M({<`VOhY1Y&tgNk$&1|48E<4Yh
zITN_p?dY*%=jK{}e{itbM{V*e7hNewfo02<adC0Y1A}GDmaUE6z6@l1QDS{S)|!}|
zMW?6fYRzr?Ht(HtdeZ%sZo#qh<g0Hi()s>#p>zAQGc%n_Wv;Xc2@4<Zk(@kr>ej-?
z$Ci3e50YiCee(A0-`DH+Z@T#gw5LzXv<sBnI9p9MSV4ozRbR7Ib06Co|CQ9)Q+ngX
z4pqzV{4>MF_I~^Dp`fg6nH67_ka^yn4coS@Td~5T=EsJ`i<LoZc=PV;c=+N)L}Vmr
zIsM7W>cO(?iYlEO_!sT?`~7}BXsYhQn_Jbp^u-qVHZEOpGyR(Av_;cauYP^BTO71y
zV*d2$(`U|<^qeHFA9sW)#E4hk&c?(fBsTW$$&;D4&14EX1y(U%TaZ-x=f}sR-QvYj
zFTc(#?bKZDQ#vQjJ1lhfd7b*UHMf>{PHt*$2AOwxUF>c(-&spmt&$QFYWiNO8o531
z?z;H>c?k(0j?35U#O;w-&gaTe%l7_g_xV<C@i}wm+&FdX@3XjM&CEdA<v+F0*2z?-
z$jn{C#mOmYRibgV@WOc|&!V57o?4f`dvj;!=KcHkZ`)>ewKL!^Xz`(pOj1&kn%|rk
z4<8;(+PK26MPQOhM03~4soLQ=xw(>a!fKz#Ob&kSQ95VZA`4rycRP5a+d6tW49v`y
zf%Yt1y?S-sx^<w%{RRmQ+qZ9DwMy%1XMl6z&O4IEX)(J>GGAO+I9)$JEG%r9m7tS{
z(u2Jfg&h3+^9>T2=B~YzzRX^1!J{Qha_?OJ>Xa&a{>qs*A0HnNIqqn`V#kgfckcW-
z)XE(e7S`C<7}4M*F4q0@;X}jxdwX8IcyX|qJv}8QU@nKEir@ye+@kGycQ<X?ruK5v
z-t-?Qx0@}n^6OrFD|5xf&od`Y+H_G(O;b}d<hbK~U2W~`6B86cE7)wSzwIo3ULfm{
z>9Y9Zw*32`gGtm?Re2>080-a<S{U9RHQ)N<#}7ee_pmU|s`GnR+*xRUwJ>Vk_Ri_T
z=XPz&z5eam+1WSx1G-dHRcFqgot~DKcVh#icv3E#jjiqTGc%R7wO_w~-~W21;*~>*
z))~THUS39~#@g#2Ncq~fvubROD&K52r$E72Ir++pKub%@kmHWLO1nyDPo7--<wc;5
z!|Zm?$!efY04kow#>W2s{;M7<w5(_`%?S4N_Lj~~3i`6{#)+R>X1(KB^;jX}cKW7+
zKV<de_81r%YHDaKu;N?Q-rwKf*~vNo{P)kFi;IdDS@C6=D7*LZaB>C)2N#!=+}T;I
zezj1*Y05#VEuJZ9X?-%LTI%Y{cORY@7JDo}mi_7lIRo$fGx7fOYz)oK^))pYTJil_
zWn29Xa&iU>JA1#J?W|W8D<*z@aZ%aGC@4O@{^q7s@9BC+^%6O@`b0FF?Ynd5&c(&<
z%lFsiX6*OBd}7`!ix<MO?tkuOUg_Gg!-AW;yQ_<9)#C-{A)ABm*Z+?V4>vb5+61cp
z)SY~tqNA;CY-%1HXxv@)c9u<LQE4e?nUK??Hq#7cZ7nS$LqktLziZp?T{?H8^_9g6
zzFldnx3Zt#UHW?3+O@jVPcOCN`?X@%u3JZsrvCo+_V2H+-2D9SFDzuf+8H1m6d1_K
z!2#Oe{N;;@v9Y$E9yo=r*}#_TlbV{Ee5|MP&yS7kZqDLseg-OgrZ!$pTe~mV*~v*p
zR`zF&-PJ+?1_w{ilQ%XdpPy@;o|#$s^;IaST_C(r?ReVe^Yd)2tG~UGHqYacv$4?D
z?gmv19}h}xxfl`_78MnBZ+H3n#~+T)b$cig=Xw3=H+$d3oS^;CX1P-9Qhay5dGltW
zbGw_1%b7WrpJ97^osyguUj%KAO+4JD?mv%5!r;Nl$?75MP8^+_*B10H-nHx2y?b_A
zTC4sZi2r&l50nVx9T%4tTiiKy;nAC$n~R?(EjFDx&#u-g;Xp%6OUu^m>+f!EZueUr
zJeQ;M_NS+(K@mE6GU&Ko(D49|6<UrkW?yhq5a5_IXHHX7)0STr(h>QKt@yGuq~<0*
zd9rSm^`jRT7oVP{yV%N)yY%z3vyRPdsi~>5b~QK7p6z{Q(P8B0=T}!($JBV>%9Sg(
zZ_j>Z!Ld-y9a57mk@lQ)dAa}of`?A8-#y-KEu)qZam`z}=~%|yv&%Hzradv0XE@Lr
zedNsf^XHEpV`I>Z*^!W)?OiJ4ve9p@)y$bQfBybG->TFrGP3eO10yIDOcixz)LlE&
z%I!bjZf#z|wYwiB91mA_Tx!=-zFxg+`=V(V+jyn7<=^)!og<d=|KDF>HJ^Z>ATB;W
zGh^e$R(z8<+}zwIPMo-F*RF1H{kVNKmVUBb0mf~wkNy`iH{ZTCdi$ypsaw^%_~)Jk
zmwkJ>ntr$IoGO|xYf~YxyWPxmlFI+@`~T<O+~n%+{`}L^)1W<*S34)9XliK8uqb?V
zcXzqE|GYQv-sveTJC`c81Ux=0_2oF|IN9uLYXUEsy)(W%|KOzMR(!uA?@XHWyZv~G
z$eu4BJ`@xl+!;HYtN-|H!{lQ#jnjL2duwZJuU@^n(yv7&Fg11Q>ebl^2@g(8RMwB(
zwPeK#50KHyho!!-gX*ttas9B2%V#Uf6oSKMzOq<R7$UapzP-(|T_(4WA7|gKZq|C~
zQc!jE?+X_$Y)n2r&#pGg&#&*5i$@VBOVd*C>2lU(YpTD$yS^^=@+HttM5h&oiM+p*
zL5r5Muj%+{@7-72oMN$dMPIX2aAlm=n?!EK?JrJE)qeKuS;+Csu%e<O&|%XrE-scd
zPJ452uk`Z-C*F;vudne+n{COzZ)ah#!+W}3$Z<ywQ>itM`){MxXAhr$N?T!KZTRhl
z#jXuT78VsZHYA1|&r}2Lp3BSo_v`Cx`~QE6udWIOoz(H<%uM6Y&(DVk1nek!dTN5A
z^NPM^fh?Jb=DPW(r|aLodDGBR>R55plPgw1vh1c)ANB0mk$g=@&Be{-%iG)2Us(j{
ztEnA3b!yd)9To5I?OnHS9mr616O$`9Z_Ygbe7<e<wR`vGfzq#AL$Psf`-Tl0I)&9Q
z-PGEf{xg~J)*j8R?=N4zy1DDLxv_cupC5)-3zy8^zyJS>i;GvUTD51-o(&AH_Q$SV
zF|n|S@bt`lbfok5x3`O}_`pLOw*o-b*S3v{pz14T%JmD&t@yHvR?Tgb*f~d2Q&T!O
zY0{)gPNg!Lr5_$RT3cIJR{qSszOGl&xb2mNN7vU^SGA3dg52GoZ@*txTwHv3b_Iv0
z^%_TM6+64SHLa}u{$FN2TYF<0`_;k;GgiEwT{gc)$zH$y$%?@B@%xuq^?ixlnsxQh
z&(GE+F9ZYyFE00&U-ejF<@eM2`&-)EzrVd*{`%V5;N^Z#KURS9*k6k`{89UtELpPf
zaNAyOz1OB^y?x7Bt`<(%k$N<~;*qJlyZiHJXJ^lvH7n%!L_Wxpg5`d5U)|lkeg6FT
z)nOL3ELkQ~r%nak7R1G61l}$j91yU;O3>+rdxCX_ZQPy;!>lha7L^}$J0};>u+1m8
z+FGjT{>Mj0yDu(w7Z(u;kag#pGIeUNwE4R$D}$dsdv<+Y>|rJmQ@zN@NPqwHpxtY&
zt-P{UCf3%V*7!pCM$;GDq4nGM+`R9e?$M<OS1Lv8y=C0|<yfANmzR;Tu{NlB>vO68
z{q60B#KTR^&9Skud-v?Ikm0-98DQ+P{4(fNhN`cx-23G~i~4SEP6q`>t@#?q{jc8M
z-oDta_mqeK`=tHho7-it7784-;)}nMc}4!?$B$xSVxR?k$0r7yJb99tji=(ngM+I=
zSM$kO97r%&;ny;2^Z&o^>)H8aPMkU=Ws(u#<HG~0mG84>U)UuoDmru4tYu4<yok=v
zOz-h-S7u;{-E42oGq3*jwY47~A1~e+v+A+O9P9FTJdm~G_xIJ-|NE(awJ@Oj!K+uV
z-o8Eiw8*_zs`TY0RZ#n3AM>>Zr5d`rrskkTx^dmjTaen#Q=oQr{q1k1yu7?~q204S
zmYp>}Kb@Ees%LF%ZI}Dc2kr80nlXDeJ3G6!me#GCH)Smf8bCRs?U2-$;;B=of{wAQ
z|6g}p(A%mv%B@sJrI07LJX^~5%lGf)rKO@=ts%!Ja=7=&Y}~r_X@SM{b+MZ_Y*=8$
zmsMEx{hh3!;Khx}$62|>9(Zlw@Z?_OXdf6Gn+x83z0|f~+r1fILykL6>R51bm!(Wr
zYDVR|J3Dt4Z=5M}ZB3+c`Z<~0+_xPa9LdSas~#^9TNS+AZ&7B(|G(ek!^6YpScxc}
zZ2#O`ry{_i20p%Q_0{7$U$^{aaSWKt;i;!MU;4e>N(0lPlao}f%ik@r;xqkG^Y6{g
z&HnRjZmtg3x3{<F<>g%^yl~pOMT?S7P0_r*F4oxC`19AVQ>RZ~4obMIH^^>b^PFUy
zcILy^ueo>cZd?5~PRHTm_KOm0;)2g>{x?dvzApClwY7___*hS!KD~C$8ozlqno%dD
zxdT)<pFDZ8XV0ExzO%)Igp6`-Y?w1gW|gpF%f2qic_JPytlVNU_H{9L58OF%bFSNQ
z&eVXv86|12^UPRJ3Mw+jefspNnT>am)x;l9o;{m2YnGe4`|ce(=2(?>olOe{9iwL=
zb@tLFA%;DB_tw_d>gwoRxpHL%U#q|>=55VP3_Eu2l(ne{2u+_c@pGosT+dP&6;p|v
zqjsD1PGw81iP-q)!b0be<ARf~T)D!?%y#7H(VaVY%G%XPs580>y*O|G|IYUO_^PU3
z2O62*-`VM0D$^1myTM)~=gyA8x3{)>-8xpA_RBcpMkPnBdEU$0zuV7fbH&ZGtF1aU
zMKk1hX4Ln0cQ<d@qNAe&+8lmnCg`l73)@*(S$$_3wW@gf`}=2SXM@tM-Nj4U7eIR?
zBR9D$^fiBcv(YCqb=6~q%+Sy&h3_X+mS%3-zJ2@L*v+w_+6D#%kB)Szc!KJ(lHJ0J
zoZ`~b+4=eL5fK#+4m4U8KMRm$SNsy#y-&RU&5ez!-qQ-AO5Yj377|<VB7H;Xt?x5S
zXPyi!FDcnm_Ezd@;jAMYlaH@iyEZj7wYjNj1B2_OLpN?%C@C#+Zs#k0elB)zl_@B@
z%5S|>cH#5$^VZhZo2PF5eRl7$pi5b=4sQ)oh>+j+aPrKly!Rv)teB#ce<Sc#_Jy-m
zMyuwBRxi2!H~nqi0g1cQ;(kr5w+Re*9l1rZ&1WswzrN}wtNB-#*!@1<l`^gF%=w9`
zAz9@Xe;#dDU+=g*+yBS0kFWpemR?W&lI^-$LugL@wI}~~U7Dl5(Ou`|p;HeYJV-p;
zX8QHvk<7opzJiW4xUjH!|Ni~pGb>&l)74+Uw?uUI;f3|t*?E<3r>>L8;d>oDyFMUp
zhNQgc;aBN*{a<S<Z)9;TKb-ujva8m&Eo+0F&Fiz5Z~Tt*I%f9ww%HSn-q)M&^Dr<N
zSj_29pVtr_7h2t|UnkvkD14)S>#<s^tADM9L{yL1X+AtD#rJ0Mtp)cN?Rfg^*{z#5
zZ7V(`+}e`q-Y<95?+k}$lUS8p+`gL1Q&Ti`SkKyR*6Uu#aP`HJJ6Tb?F9=o5S;6uD
z6wj%<zc)_*y6SJa;IFx>cf>@UR|MbP7WjG16?2;gtL%WJKLXqMWF4l@`?N=?%YDbK
z4enXj_9Pc<Zee(|-jOq0+Ryyl^8V0|;qzCWs`-DnL-0-KE&2P`yyA+)Td(|i_ayYi
z<+hU&*_Rv@IyxUOzqYadR?EdVd<-}Fwa%Nn{_)q$D0^#r*Zkd#t*uK_Q(Rm?>tX-R
z@a;SQ{QO+&?>|01wypm5<@4w4yP(x-`<DHA6|a1arS4Bd(%!58<+<ms&zVzx;Kmb%
zpC4KCH@!RbuJi7`+J-f=-#shd-n}pGY3jeNQd{2K+&a_SobkYt*v4!h)Aw_3SY<C?
z{=%L4e|ck<)R)iqc^C}b*6;f||8cvq{?-5TzxV&K|8u2k@_BnRHjJzNJU6am7P0?O
zIP0hU!8!Gh9p|h46#rww%iH_lL4vrrczOBvIdf#><>mR>ga2O^aPnQLdas+Gn|roJ
zq0-#7m#U}jpL0t;$s#FFD*x6)qxV<%^DDL(GVi+<=Fs#0+d>9a`TMKpl<oM(B*Ktq
zzoxGu^8cr&Q%#RbM;>5iFnZ=#>fq!XJt5+>zuHB=ZE7dK8E>sO{$Q~xDQYc~2m@2I
z*7Eo@LVN4&b_ajl$FNj)%jDb76|!|!Eo@B*jXfTCVOco?gMgcA*US6Muk7>p4?nM^
zcmK?tKOY|-e`J|)dF6@~683c<o_oKXtc;A#K~@chf}HJl>Pt@FZe3xybKRdx%g@@g
z#q7C$2Q)}o{1N@CvGE?02*a&!TW!9T|CQyE++^x39lg4UA)+mL*<9N#Ui=&k0>xJk
z>x$o%esCji=lk82x7v@J|GZln0ov1C+wR++U(utI!LZ@oowMIrGXg&xRCQ}UrgQrx
zCqv7oGvRvCO*=jqFl>m*Uw2F7e~;UthXs4~?sYZ3!7pw0=I!n6A?i*mmSz{4f{yN6
zvu3U5uAKLh;!jP#9Ibq@`lHRe-|M4ap8m2fTbeg^Szkt;1|!1+%}n04b53|YeCZKa
zdsM6P#QH<8!>%f>Kc&vVFkzD3hn=x=?;l7ukxJ?mPWOA>zkct5537!RS52{AYjy7#
z2irlGhVngo`|Z}8_j{J|nJ;!t&EK=EKiQ-V7#jZj34~shI4JG5^|FBS^v$ZOsf&Gm
zCQY7vwEV%bY15~hCmrcHaKJ%bT|G0hsBPVqqbYUu7q6?l|8X?B)GshjuJ+&KOJQ$!
zNtk{4X1MxlLt(ORXO-UUH-2J*;eYn5YdP~~F&{%$)Z5;+9+|NFo26nyZ;3u$_}uGW
z{c8KXd&dGzdAD<}ES(zv=akQ+6yLw+*R_B8-JWu*q@1UdE9#Q}v)k@mcArH!Y&SQ`
zN~u>(PBf5c=aY@vS(I8+^XEV#b8t}53cr?s?rWQ}F8uiMW22Z@4ddGDCh|wO*&eC=
zp|wehf#C-egVo(#{<80@zt!G5@8`wNz_f39y6DA*KGD>rhgNs~v3A`mul)ISj#G}2
zvfGB<8v*}3OMU0<)Bbh0p-V6MfVlt0cV|-=Z2F#Gef7~3dPQE{YOm$IdG72DXVUku
zm95CtdG$-pR&D;7^ZF8|RbO6TUw^dv!!c01w2_(p-v0XfpP!z(_sgXkEls-@bM>2i
zc#hRB?FApYd4EK14`;iw%P=<fKW|Hvf8L2>CB1!fd9APgvsn7$w8zJPOh4C`-Cnco
zd)SJpjmBjM|GUIj{;jn9-JPz|`eO;h&ay`jZu!47T(W=Ks#UB0Y`>@3C7jIUsknE4
zZa(kJ7k9R%ot?F}`a7t#@ZdmW`TKiIt@xaD!?{;EPM$Q$=j@qN+B<Ax&uf>zaQ9?m
zW0>x4^n!t5zl33_{a?+62VX4qDfs;3y!SrCfF6kz-rHXu+hDMO|MIG8$H09I4W63~
zll%;lW@KEp+x^%5#3db`7hmrSrwAS9eIeX4!;Gz`EPhXQ*=v{W7t|OQ>}9)qti$qP
z!@_?@`EF%j%-v<EUXpy}oMnbW^(;GChASZ%YehQ3-hZF-yXX6&Wv-L+j6YnyG}}}D
z;xsm<gIh%uYxr4ntJu{TINsMi7nW{yGhe+<m_H>otxw7}>XjJJp8Eg)4m2`fzH|vP
z0>ZZHagBUjCF_M>-&mO#TF#fndu#p6X})p#c*C{t@8|v(zAL|;`}^6`PnIls0y>HQ
z&Lpet6B17}EYQ$Y)KgS^`E>RneS0p^G?nxzSK>EJd!DAEs;Z)*n)+l~?auZm=BW$;
zaWiI4nJ{6(oP&Qq{cM`l^JGa&OUsfcJ^k}v@0fXf`i=RDtSTYm(~D1^&YpMtntAc-
z<4g>b%I`eiESB_7`~BKPljft&)^+bEg*UZ)Sj~`-d3F7{b&Dot9_x{uH*cP(sHn1g
z9}fq|0#Ia4DXGc%Td5znN5ZtK<l^o}^YosFl(q$nHnV;B6u5Plp|>o<44&C~TV(z1
zr@5Q`O_5;uo?KaU_|?DNCGU^Tv7ezD{d%IL&B3kNq2l*yf;Jc3GXBn&thML=wWVGK
z$95a7X_C0tYq&~#bF&dIOVg9|sinOSRwi0aczfq@GTZHkPyTkC`m5<wDf(;8!87Ni
zcr4XEeEu=F>SB*+w%xIw*ZMb#R~gAMFmSN`%94(p8&$e4SuTIM{PtT7UlSjMXghy1
zb7o+y|7sZc-v3jOQoM=8YG3i0>$YcCaWgo4kMl`wSQK(k_=nvh>5mHcq(5%W{8}VZ
zaC&}R6zIgV6)Q9{BMm^u?+B~=iEywyKR0*sxy-$%C6(8T&5-houH<6aCdaq^=lOS!
zyEp25eAB3MZAz_k>FMz6rX}kYG;{tgSvv8_apB4L>YtsgfByAaLjA6r|8MSozhT)q
zeZB|xXV0GCz`)8X%4n|iC?j@<=$(!oxp$e}?}oizwYqQB)m5dRg};a0;p#G)v!J8v
z5mSr?U&sVO50AWe@B3R8HZ5XGoOAy2bIm(<_I&5y{QkuDy@m1G<8y*Dmvhc_lmGjC
zdsmX1;BO_14{tcnUtQibHTYYM)SSTU|7^}Iw|@RTrb=ru&&9Mm^`G6XJcZ{~$=g4E
zy!)VjL}>DxB=d>)pG-}^yw2$Bn|kK6m(JDKWcp7JtWP)5$k_Vn#-eRgTYlFU$5#3>
zHgNx19=o`n;f#&IjqB$)3|^TeUAdI~pYg+uSC`knJn-e=?Dy;Ee|>Vb)@|G0XN+?{
z9alGP{lONxzplvf(#l|Uef{<GY^!(e+Ld=_N8yhT4|nx-Oh{TD@uk_W{@<Rhi<8gH
zo)J}RHuGx6`o|eU7W36_{p*S<b$xlay~pl#NF2k?tnewb&aPsK-#@RSD0rjYj{}=e
zHs76mOwO~C-Qh@Cf#Hv%e-<0B)#~iA_;)qBH0nM3?fuh_91JsAurjW75)(r}nr*ZD
z;!y3sbz56GFRtX?uJX6g<=-sk111Gm<P-Q9L{D?`x&$yf87KbTbmp?hQ)A!tYg_(q
z-6_D}@@CV%U!hB0ai>pa-+j8zt%!l=jLYTSuU&1uVxO#?8~f$UvYKmhH$ORdeTh^q
zU$-=?I?ZKGckM6j$(ixa93Q4T?OU3_#n9w7O{Zb`#K`qGr`WHNX$yV5<H5bX)o*WY
z_4YWi%YCf_WVAZ=NYcjX)59P7I5098UCFo{YFrTf)cX0K&pp?FKi>TNPvzG;alT#(
z-Ti9$>+<t&Rxh%@oV2z)AR;2-MQih4$!WWHRVK{VG*7bkinxAe*3;OvPd{vAHkDnc
zb^mMZ@^}6XZ{Ez>A6Bu(Msy9!0`)WN=8Fc{#y)!R>wKnr`TNs}iHg6c^RN}<ooz{c
zckn^*I{nz^;%ip1J^Ob{a`OLRzqnl`LHg_rTKDU>{mgDJbD0}?FZIg(*tDEoLQ-Ny
zb?ayMa=dErjAe`OHhr<{V0~%5Cd0OiF7LI&*T?L55Ec7-=9VeW&l7pL8@PWR{~q^?
zq3C9-?K^=juWlXr6<vSnBmdUcAGgF>*SjxHJlwW#-@dT0FlD!%8P?_N)*V({azpc%
z<%`+o`7(3o%yW0W`86VX+qQS+8w9%w%?xsjCT-l%zV(~l)g2S(2?|tdhqBe!74%$8
z|9xGYv7$8k-M<K41<khERZa_;7?wF(dp--=@u<0IT6@m1hfe~lgSLF`RXJpqG<Ejw
zHwjau4v4qSG*UZccQvx$j=g%Y^3)3H#ZmHWR!%hD`nJWh@Q%y(B?gu)3{&o`o4cuU
zU)Rp+;J5;YgqSAA3tRW@F!^BOvG3_dsR#YPPR%c5P%shR#vuQ5+Vv-sR;7IHcCpjZ
z`&=9QcD9-0y>08}J&RAtP~0+|K|W-8@dgWrv^KHK)*rvEBEFbx_R>8bUN)!d@v&aN
zxmK#`uUB}Rr9V4A-#+Jt0XH{ye0)56`}yNf#iVL#Rs{sDeShTGto-!%d)L$o7U=Vx
zu{>zylN;)};#ZB?>_;!#6#p|x>`V;P)^=uCli8d3+$@Bdx%9w>RCVL&hh*9G_-*#2
z9a^z6x|uH{?a}HQ@p*gJzOgC4{BSRORDCYnrMr7qmw&wTWbWqKQBxU?m|FigJL$&q
z;9y(vdY<evyNin(7ppMY+|KxN@`0)IoVo31V{MMPu_*jWRbn{Ot~+~E^VYe)()S-b
zrgLOz*3PqYIT&h={G6b%@y3ptr)CTi&d$NfD>tspT-F+NdZ*sIr9WT2pUXI_e&VaM
zUvJhG75=|hcInOJXKM<#`)6~1WUX}jBObCk?Ckyfpp7@5pPvUENA@&hkG`TxxyVNu
z6Ft3kxwp4X6cpSR`%g1||NM1_FFq0u4No#rwOQt<?0ZP_MaWdv<CU$e7;EZ^x2#gq
z;bf>uSZ3OGMStqTS@TRaZ(q`Q{N#E0RQ)aDkHX@wnWZHz-N{{>>do{!%c+(@$2xz-
zm0A!g+1G5QYNowd(sk8dZ`J)lb1$of{9#vJ5PR1%dhYrc3*OJW=EVB;uYbT{p7^qN
zUaQQmPT4f;&7Q~Sl^FQ*>OU`8r1s58u_UtN{F<!Y_Sz=W#XmniT^GAsOC*x#)n&`R
z$8LQxmgVp6+}fJ``}gng_4C$yyFFnQ4cNVUcG{8UNzZRBh>$z*JK=q-jd@yyb@GGH
zG0K`o7bb^0Us&aPw?}ow45MDRuUFU?*p#ZpO#Aq)@Xei&==<ht49o79oSlCCUB>RO
zy!(Xi-sX1y!>IG#ykNz#3Cpw(Pi3gM`tHPcEr;8Mw|6xsyKa0Yn;vj&lIyKke(&G2
z|N5Nu?w~#Mmh<wjF67UArujS1?(`4oiC1qgE1ErTmR!=)le&%ip$zYS$uJu%etrJ_
zo?SHyb&sTnE%TXaoPKUg=H;}{&(5w}x6WuLAGdmoh<@w(s!7G2ot<xQZ{J`1-0x~x
z@tWBpYjT_T+cl!gMVJL<b=4VqeO<4xDdpq0xL3V7=hmlA{kpvM!17g9TU!}|7dJ8V
zZ?aH)weVGJaA4+v10A|^U5kFN)OyCm(8YgetA^i{Z?6KfcBO@8{XJv8b<Ml*9>L%)
z)&uT#A^W^{3u}c6a59Rm-#nLT(b44v;&H3zp5|g$=2$ziTIBkcJq$9>i~mQfi)M0v
zJYO2saehtJ^ZlVKH*HSe{`KYM<CiaQcF4UvY0jKKr}g)P_9&IUzE=JHovD%0uV2Mn
z%*(Ev-_O6{ckc7p4}!P*TFqpG&c3dfmb_>%-7R&qjs35@h_LpXyZjD6{3(Bhv3#ED
zl~r@<Y#y6tX58LmfB4;_o!tHNG!p**+!sIJtEybTsd>S{o~O*ZTjW2o@yMzLX-y4S
z<n!!{#ca#p>y_UzXP6}1ny_q|mns8~gh2?)QlW+wSHgFFQrT9kZf3XjlhKckQuVxT
zQi1PI*gt34wmAA$&F5Ruvz+!ExVF<k?2)mtiOH8QUu2Ba&b+(3J9=x@)2B~U)4px>
zKhrWnK5WPTf*MCJuddCTiwg=0mWEB%vi7@HYBArdC#Y4W<FzDn!Yq?(+Sy7>SC%*L
z+nFUJ`hDvYpOtmG?wOnm&Rtx##;cG0fY#2ljVtd<@i45|^8I}J_n`7)DIYBs&p&$b
zj1BYDx4S!A8!q@*ES<YRMc}*3)Yz<fbH6zK5sejpWW4$Hu1Vg$c4co49B5=Ne|t++
zK%_(K$nNs@pvyVGzPcJ;_cQhJvEJ4U-eocg(*La_tp76ze7Qf-duv@>mG8yp^YxzY
z+c;I|S^R3_&1zSpyB{1koV32QA&W!uY~|Hg+p{ueJ!Rj#zNSQ|>no$mlr^{3L_U^@
z{g}LUH{YslpS)Opu)pS)(+xg4IsCiit$(+R^>P*W70*v=-6Al5riaR}`~Ux?Cnj#p
zxoO1C&Mq%MUn_Le?UZhTN!DCPVhye@UAC<3{k^}Bj&@)3bqjlK;D7yL%84lkpMK|O
zO}2ULRaNS(r06yC?z_1&4a!6}{h9jleA2#2J!+?~SjKn9Wt^&ui^|SCf9;H8oe~Sf
zvQz1@+j}@)N4<T$-rUYj@{{9aJ?=N!oQ(fo^{)RPW4BQEh`LCJQ;n6I!iM)-51u(!
z`TpMC#csWqN-Y>Z{P>uBR;Y8bn(wC%A2gykH5fS7M(Y$@V?VH(uUG1i`r)!yH#dC>
zpSPHgqx#!PBTpp;0h1L`-?Ufro9f3udp&mnLqiI~s|VHocCqGMocix<E9Y-E`|#n-
z-Sf+L+U_sc@1DBI?%0$ple`95hIyZ7nP&5{terEv_U-XcmcEK|8F#k`h(EFiEszLZ
z9cEkk$z`(zxEk-!ZnD%kUbz3?uh%axFV}tbb@jVVA=e+KDE-Np=X8eo>EWIyUHgT;
zEcMa0aH<Yp8S-@Ty4S1HW~u!wpTACbhF?G!6T^FB^_B5&Wm0Y$`pGX<HY+{iJj*L*
z|DKYBWog>irf>PV!rQ}Ob$gxp?w((JwSG(RwWj<ER9WG7B(*xtOd?m&ZqEXdN0&Jx
z66fx_?G>86H}mo`&|LragDovBCV6*$TwLsadOGM3U&X^UuWP0-c}?Y7=JKoj(l%z}
zx^sNOzptIByXD*Z>sDrjUATOT-9I}kE(YZdW+(F^W9{B8+%MD+d$s@C$*aP3sSQu}
zA2zcIP0<TD@>b?fYQTZ#uaBqanLPIX+8p}7vetw}-L546xLv*5>C3myl*uM+^IU&W
z()Zh&u;V{w*Z=;=zkdFst4y~$zj9S7{u4gj&VPS*d45vTqGcNEUl(v_p6ZI?JN5a`
z1Vv{)dApcpVcTcfUK3k!dsSp*qr?QMCGAJ#xvr{3uZnoOug9m`^Y-nQ@MWr5h7+~U
zx-b_;2J%ntF`m1MlTpPcuqyr4htqTJUCr0f;%#{RE%q@(hV&GUtW=c_pOZaz9Sa|>
zE7O_j8Tzs5nbTR7CEI#cjb&ZBuAg$AslL2^n#-$~#x-2!)@#K#|JVPO>%(y1Ov~K6
z_ZM)Tp1*HFOMvyi#XgdZ4N;+y?dRF#kGW{cFI&4*xKcfR^F$+7<qZN^TptfUWL+UJ
z-%sJnYU_l4&!fk*!`IC*PVZZH2y`pb>1n!;A3rXCfA8<@{QWPrdTbVO@bf)%O1gYs
zpv}}eW7(YMm1Z|~`u(bmNxJpi%r=)xb>$tO;<N{+-j*AE>-6|?d#h-}>G=I&!s&l(
zFAArrH0Q6YdjH6>@$Hd$w<a#TW&WY}_si%cwf)z0<Q7&=S`xMWpDDwKRd?4dHn;Z;
zU<^>Pxof%h?WGe|?`*TudL~`^ZL+t!&zB`&X{Pnr?Z21r+Vwko+m}$AeOs1i?sx62
zpC9dgYK4d<!;a7Xj~EIHA3mI)cTRKVot&~;*X9OSUHhN3X^z-ocS+sy+y6Bfjw=Or
z{BP;*KD{z{`H`f;>3Xqd`S;d<Vs=wk6rWYvnHh7f%cpJ2jekAe*XN)_K0AZbjF}q_
z*9kFdgh#JWzqQZtp`zE>-j~lpbB(k_4RfVmhh8Xi<dV87yzH&YlDKz^X9k?zQF8fx
z3RB07z`I{pcQDL6b=RGf!RamUx-8p`tG6ETnV)$(!swg2;(^~)nbT^-Z%n=P$4BDz
zmaR|MYHd!_Vc5{d<gH-nRrzx7w&Pb$`EANuawU1mmiSyni@eq0S`5y`+nQJyc)YS+
zf9;vLU<(IB%d#Bq+BbV<nDsgx`Sw2d#Y45&Ytj3=MAqk>{(iL4|B2~U_sOpwv94pA
zJok(A%-Vn1kN8_N>lEgM-<W<(V#dtP(`MaX(Xr9@bn5LTqK|6#7g{c_Is8iUkI9dQ
z!k@di#dIRJ<wX8t-%$IzEImDaqDRT6C!XQq*Z=S<oYz>iZr+WD%O6aA{e9l9+1v6b
zcs}<$%<$r?)a!(GwXZdW=E>dP81=dIz*29)jkEY|)taxZ%VuDE)$nh!nTyJb1FAoA
z=WPBm%jt<xcGqfMh7~^~=CN<ce|@l8|Gu`!rsLDUS?v7F?x@}Wwb}G$#GlP43g)p3
zeAbEj%&DdM>fY7@(S{}A`6&zswn^AH8mG_Sy=ztR<Xv}nm#@FR<nyKAPZM8@|H(O1
z@cupP&*e-z3hbStbEogE{no_%;lrD=;#_eN`);yrpEb+<TF>jC<zd_NzJ1-@^D8>R
zy6WG7T?}5~Yj_&M-^Oh#IXvlMU+uPge{w9@7V|M5XH0NeeEZhaA72ihvA_KN;lh6_
z8LnL1n14B(ZG!!`yEXC+o1Kr!{LAAKXJ==RkH7!v>FM1&cb=VZAAkL!K#+-{&HTkH
zwOS8dSs6TCH##k9=~4e?{!NSW?-uIuU(H`}lGRwnUDK^vo6#h5o&6#s`_$brn;-o7
zwD8yb+^UoE_mr*YaWFg+JM^(H>9o($BijR4{;yoVO#kz}O$&D{-+MwMSTy#d<JQ~W
zb&7(|rn=T|7ft6=df2JtqLr2VFOaL^xznfp-+wyJRb6{J{Y#I^p0K>nL3aWU-3rUE
zJMFX8VTJ!Z-_k7YlO`8E_GYJldb{b+%)UvJUh)>adZ2K0*Ot(wyS80^ALaQw;>p)3
zt%{M~!nmgOm>5(pl6X1!t^n)7H;!zfc0X0MclJg<$-TQIG-A)LnMRMJ9N)0}F88Uc
zzkAp6<JrGoUtfQo?*C+wYj@h&S*e+sm0w<5lrqhFa=Y~NoIeM*u<i9vy>*|XDU6-@
z`jrVr58`!JzKwbOBlNnL5W~f~)3^KYe>=JP_L&>oSIxI&a6L0uSNmUdxsk@-g^k5F
z$G((#yl3gzRh4u1Smp92<G7i<Z0G)E`{a9TGHd*q#wWAiIN4jj?XB7Ew_j_%A2@S8
zs>I^N)nftw{@j|xscdt2tNcxttJQyYC)7mm4)WyhxBfC?=EbHB$KO3ZH+#<AOWX63
zT5SDd=7tB(k=c^ndVA8}m+8Cy{CmH|@$FgD_iKOr{ygjJ$0t`AT=X`d)A@7MrhDDF
z?b_K}cWy8EujX4dw{^Sq@$!gQ-vfW0GMiOVwR!f%yL<U+#S9l+e4gK@SM_(n%eeh@
zclTC*zi<JxWI6xFhQ_l+zXhC5eAlwFe9^?p{p`%l;6u9{Yquw_xp#fLv;LklJrdc<
zYzJm<e<dE7n(<Qoi25q!{oC~~T<BQ2E>p!%cIGEr-4IjpDFRGd{~US#d)nST6_cQO
z@b>ojtea<I0+c>h>&31&igx3F*uUdM{=Cyu?u&HhH<_1t)cvnG-Y3h<#&hEJ=|UOv
zvNtz+rOj{MzAbNCwIya}{#=E)9*q>vk4BaXQ>LBTbKv)Lbq6Kg)HzQjEf1+4**ASP
z!@?y~R(>qsa_FScz3f98fu|zQ)P0IL^Sk1}g*!I4SB7-mkZ8Sf$D&<$rCpx2*01Ib
zOIoM?G++NBWQU5H+PAm2ub)4E{^-%v`}=Cy*es)eH3&>%c2!}I-I{fEiRa{=(^prS
zKU(6kf62PTiYm*W6rNwdv)seCqVmIogYWKsE|6b8Wy+NJ`oCXqY)saT-gaku{``#}
z!<SVEJDE)Jdb%QTG01`qiHC!i`%TsR8@aJd;8QHunU0h48!|61^PH^ac{Zo6%-V@l
zLo-H2E$nNXtgxr||4*m&m;28@w|MoQ`v3PXUi^4tWAfgruU1uGG*lO~JelL6^6A5e
z88(%lwq{?KGSB<-<70BHyNicl;EUgm-{0Q8zCAy_)3dd0v0r!Z%55H8OOISHU;qF7
z%+Td?EQ^=9c8l%2{p0iV^S8HT3d_maB_3*V?G_VVaa4&z)8)?HyXyY)a?;Yuetmg4
zzwVbOs1$i2`b%cZo~2J`Z<n9t!*j)a?Sj`496#$mp3jeoy}qmR^D}Y1m=kxW9q*Oi
zUiS7DE4SE-8rL8vuHDJU`@X)ty*>GO-<A>$Mf-IZu0KjOnpyaB*OZ1yht{=RJwItu
zm{H_A{nhiQOkqj5u_1BiJtM`Vy{lJezr3_mSeX0Oq5KvRtyVjBRq*DaR=%eTzzmiB
zfqY&ZpgpdiZZDrLC@d>rI4v^wm5BQD)XHtAs!X%5?OgW3t>KXqmx2&aj+RmCPww!b
z{F>`UUmWyIKgDK+ikg`Ss~j(^mKS-^JJDm|G0l{stLg5oLMtuKe>?Z^(Yjx8mCH@?
zE-c&mbGyFs{Kz+xt-mjN{Pk@1mCc${y$=dHo#<aA6Y_n{>JGbm493k}kMHk$s>i0i
zC+^v`Ppmz&S1*Z-JR3W?UN3}U!#cg9HF`_|jluUhe*7_;t-SNniE_a&C6huJ;_mv~
zoZTO;denEx^+$4>`|FA}oQ*jjx_8s<7oXf!Z7poye0OE|wZ_JD_uGD--96TKQV(dw
znHG6F?7hR)C*V{karIHD{wv2<3FSAhe!q2gL8yGl?mfYN_k3z69^j5#789A6FaC1R
zv7J}1AG%*VLrP@rTm35~n@>Hx#bU1cG1@3rFnMO?{2%Xgp0rCwZWd*5s8#H-a`cp2
zbuTwD;>J<czi&<Mzxq1wy|~o0a!w6~h5O2+&loL}&$m0HnbM_m?tGutuWwp$M>(oX
zRyfaH&@zFk<(DHj$G?CXs{*}k8CG)M*{Q)7IOqA+E5E)TNq*hU5VA)1=hKNf@jiMn
znjaX&mpzg>m9DbYaN^4B1Cd65c5;;V%}!MNI(61PuX(bo{x!N4IVMLdGF)YU=TuY|
zH%-J}ZEei1#Ot;Ux>+}7K93ZPa}L@$`F`iC_X!bugPxsL6jfI<t$kXS=kj)vWsdr*
zTagUgYUgToXI3*gc`!QtUBK#ktmA@@`L@@JT@zS*w<{$_>s_ylRGVM>g`@LHpX-X^
zeI}`^!zNh?ov?dX`RAHjQS*Y8s@xBbGN=BFn=TT5XiK8;%AW=gT1?lcB!{0=DqbgD
zns%1QerKn_^Pl;1{&VUxoRiIZUU<%jL-BswFGucO?eDLC?TKVqmKx!}yI_`SG;h{=
zjV)V5XK>D3U~rZD>`J9&siKWDBNl{wQ0x!Ux0`$Z2uQY*^<?Lr&ui-5ew!M3+2Wnk
z9d~x#UKU-g4QvmRg|2*Gbmct*gRhh~TX;I-6eGbIk{>60&Yv2e(UKVxnxL0BMaxdp
zvG(ES#<kz3MrKdFrQNmND7{`!T|o0k1N*A-6N^|QH~l)OE%zXP$=OPyO=;R43@g?@
z%YUw|Xt~6r;_Dk%nHHyxFQ=m-B-0JIdVKtJa&dX5R_Bo=C9K?+&!<@&wd0>Ybwy69
z4vRo-X7uutw;60|g;%doaJbc?;rixu`g;2nEx(qYp5-0svE<U1u#k(}!{fYDZ}|DH
zo_=C?y_#Z64f`oaZmVqJ+@Eh^b?&d;xg#(&F_Uv~N2qC*;u-~326yeF8TqNVk4>uE
zYFsz_Zq%oPb(*g)85zG`wMf!0>F;?d_Jp)^Wh+%zMLTAw9J;*V33uVklQPDqLLUW~
zeU_cbZ#Tb)X^~*2xY6dxI-8j-u5Nnl)xJh*Wvaf^-7kw@^Mw{I_dor8?-O3%i0#Xi
zq>C3ea;)0E>Xbd_tLazEzLe!1S$l6@5i>(ViteA?i;r*woT%K`!nmMfuGOcARnL5U
z73D%Y)L8@X%<Mg_w%&H$&kF}yZ>yE8oj<SR)cp9Fw==J3+C3JYa=O9LK2(3Y)a;Ox
z2OH~)KDIo66@2iNX0;ygx>%!^5AS_jJT2Srw%xf(>jT$jls-+^ZoT{2Ze2^Y4Vf2&
z%>Nv=<dAy9dnd?y)!*F<HIKahn(g0O;pE}yY?*Oz^2!6bxvQ_QS)ykAIJvjwZElN`
z18awGtLT$GTs;C8E+<EAz4ShOu5X85Y^|E<Jr|3e-Ai;%gJwMh^lwWtzs{=qEc=xq
zH!Lepan3Erpr|D~nJ%CAOwvy^nQ9w)Y$>ZTn`Fpz1wF0OiRG`X1$Wo1sLM3qjq-BZ
zwnWG4S>1b$i|q66O{u=~+>2}GHrX%s`6{^v+VT}<NB?fTbz#oj>%RN9bi93#)S$d+
zvR8<Xz6nD}!v&uR`9GIERFn1AB%E<L^zT+o#p6X>3CFfglxX~Jt9bjTP@CAHR6QZa
z>GvjUWu%;Bs*qZ_$jL@C;a;BHt-7aMub4f)rZ08ViR<0FlT&M#<=-~UE^xVX&DU9q
z!Qs;1opashI!&$jd-Epbq(RC5`G3V^tTz_l<z#3|W4`^9)pnPh+(Nbm>M0BoVQ=e>
zE<ZK@?cG0>U!R<f+xymA%VOI8>KnNy4ciN6+qjvr?RahdUt*7r>E-!fcbrz=!`M^2
z${7;yj~%&Xt_57U;dJ3rn8niFb-!l$`maBI{nzQc+m<ZJus8o;z5TS>JP*#Q8N1%E
z+vQ&>CiDIL$txbJ_l~W&#-8o<*eBOoQ9<Hb5dVSI)kVJBGtPKl{qJ=7^_M@Z_HKH(
zJJ>Gr!se-yckE(W#eXXD-pcn38aur&&RQq6Y)?yn&FSp6Ln|j;)ff7?!Q$)lUV~pb
z|9>eltj{}rSAI34%F%*ge~l1^10@;i6OR-M-<^Eui2zeT(eAQS&wgq>d*@qukg=hV
zlZ_$yNK`k2!%QXyPyJB!#aEJSbIbb{x<)Dfvpd_N&U*Us<agTZmvZH}|F_)KD#CDM
z)?D@6g4LV~AMPv0`9EQhn}4^qE1X;UYi8I<0f)JE!pCoPFj(~5mMW`$eC260-+S8&
zvrc|dpCu|iTS}UN;fR}B(8`S~H-5R<o50XBZ`q9l*WQ1PX125XJ9kA@@$U`G1oHij
zA8{(O@;zO^+IJ(w<kHau5AV*slH3YEnaiK8$%*4&XYl&!t;(R{zWf)Hr~b}f#+CU#
zt-lx|%I*cM{0%}cYuIzumq$uo+cj$g6L-V@I&I~^$*1jPr~Lf*h0F1^mrld-a$~h$
zGS=&Nh_2mL8C+;(c>9F0c~Yorgb>4z13unrE914Cg&7!vwk;K8$Tge!$E(yY+UL2w
zM&p)!332DN85&+*IGm7SD%Mb*k@Z12x7qhit5bmDJQ0UY4-UICSd?WaPAi;{d#85C
zGwu1>6B(v>$uTT>xmm!sKI<lfL&1l?EnA*Tq*Qa}{=9c$k4VFT`rqg8ml{rER+)NS
zM>1d2Q~OA~r8MI_u8%DfnmV&Svidg_{mfi?gJZ(yz(UE(wI)Fp4qq-#L>}b+9g=k}
zXr*-6r4p4@@+tGJ>zWLc&O|c&`I|0z_{tBLysMtMpQcaV9jUu|jr~0qyU?qvby;Re
ziL!8bd$(@Z7S`H1`~B6m-kb-xqAXsSuUdVpHfrCyrOym=75`a^KJs)_uCG(J=+-^g
za?SMkjgD)r+RW=(PyV{O`iBUE)_?E*)Y>dh$p+KyX6y%|rcbZ?_arCcN^UIgZ>0nB
z5ua{1)<@~yGP!znXX&lxTTXvGw2lZnajCpw*DaW2zvA$tD~f@yCd}{`zM!LNDaWw-
zi5dH{yge)a=5K!1?GW1a?3wZb^PI2CFMY`{nwDDp|EZwp_gvu{+pCww{9oW3dUV;L
zg|)oPuP07iDNx_a^>N9AlgqO2?O3DA-7tM=kwfb)Uq%Pn5BJw~%P^?4vCU;PSaR5W
zNzWl8-!i$ln+~532FQMRzQND_?2YeY4$GWsvLS9e<;Xqn{i{s^vo?j=Ti>`e&-C)2
zwn#_S$#b?ZeJi&<<8P)$&%^~;(>A}}eDp@>iBG1-?7L3}8f9f3S$|hQT6&4c`(s;6
zC-7|b&_1FLN)<Yu51((F(|6&@1?F|FsW(0EFL2~dKCq#x-&jtVfn}}a-_I8^FYhv+
zU#oRA_0z9~yx!?wm#ir?nz_$+_F6@||9x*4SuV2KSn%Z)cTALG+?8ffx?t)my0EAz
z^|(cC$dc_VmE|slOuEK+=*sr99vV+G<2&r+x);v9w5};l*<tdkMS5;C<-|6g{w2k;
zNwB8s-*FT6X5}sYp{qV!G+mW7Iq6^QCGr23G7JW8hbn8A`YFZmhxQdu|LKx>D@DgA
zGFQ=V4%f#e4xP8JC!hMYLg?(=qYNSc(&f}r<f`X>xDwBKV9%PB&#D&&a3y?_>i^IC
z;zHt_-<<mw9CePWiR-xjJID3c7iIo$JO7rw5j^trR{yWxyUzFhGjn2ixPAVq2@5oj
z^cO92vQb>put;Xh0fA*-Lu*!UJ2PckP1U*9-M4l+sD&Qop2fYC;lP$$otZBcg7$>1
z;|R-?(+iI_I^gtM#5#B4+?8Jo{g=0jayR%tT$gFB$$Iy<1kZ|NUX5AT4yiVO@ou&j
zTIKg%LpRusVL@2vwSzA%yCo@wp0{8Sm@fMD#EYiXIk4f+cl-amnZIWFd4`6r@TSa|
z*xB3OeVo>wc`03Z;W7bv8Nb$wN$V{tZ?&uMJ<oPoJ5Dyzs#J9Suf3sfkJ%n)YOty}
z-g)86@yE{0W}jsamq}e(XvFX{Ie+=V`pw&dtGAS8TmfA%*sC>Dy6(=RZ*MCf|C4^p
zD7wD#{+;@7at?1Me~6fP@#@^^rt^=eh}ru`?sKSqHhcCgr<&quaH`apH915<LW=X(
z)vN#Xc+RxWJrf#dINvXpJz%ZY>XnyXPEE<~no>TMVS>ccJzwIkteL{lFm-Oq+_`0U
z0}Aehtq+Og+<$#hSt={T<0nd6yB`UCVZFL_?!*-@7VKZUF6(jV)@29u^b|L+#4%K#
z*OdBx)tu=7*BqIzb7f4Ie{M_{HD^%y^s&J%At-jwf|hxqkq6IiKKcIZjehAs{qKP~
z?={4l3tujqU!-zWyHoP%qIg%96Cv;HycpV^<*;%xNK8pt?Ig)*;Kp?J&{C)1ZLM~C
zT#O5{f=}vi)chUysqNXCr7R5dqns9K9{K&OVv(q#eb-}=P0x>;ufJ9!Uhwh7MWJ3V
zX$Q`UjUGSW<bAj&YOQ?0IA{6%RZ}!n-|kO5^Ys3;Sj+7@_U}Hbxc%DI$@AvSzglnD
zU}?QPRM?n7F4m5_q56Zr@bCId`bo)`7v$~Mb7n8uI>qSYd8NaZat>!UXGpH{V3e4)
zhUb|44g1QOH(g$D)h_Fiy*sP=!}<N}Sw64tI`F*9%g|_O^UdDByU2)XPaMCyw5BMi
zni4tuIr52BtB0qsaHHp0Hs!q~vM0V9uV3@sKYFh}pBiZFtlmU<WvF)TS-zPs=iT?4
zDZ4_~pZ}FkW~|-gi!Ikr<;HbM-(s_?oKpXnG1c2hYv-@J9h25BNl%~hQ*Y_L{l+(~
zCEK1Iikz#OdinHkudVNQPMI5cE8P2f*{@jbOzECI^FHlcwIuMQj+EhcfxdE)=(Jtj
z`+B|pp11zWnvlDC`o3Q)pJ}R}UuLy+YKm0es(V}A@2%>0j8q8L43d4Z+#r8teCyHG
zvo(#^iLR`>!0r9!t=8<;mX;@Zo1NQuZf?uHeR{gS_ml;9rS{*Sy};Km?|<!AlZmGv
zd3VPyVOYRnyS4V}7Sn?l_GRC2`)hty`t}8_wVRDKZ`@1tnAXcNH|>9JVaBU-)i2#8
z68jH_KMxf)F7MZQV_ZG|*@K?9`_@k4-zNW-+fM1P&-DA@=Ou2vi1_}@`*5&(9;d{2
z-NoLXejzIY4qm(V?a|ThU3@JQocfC{EINFlV`bhEhK~MV_hmJ|G!~z~Vv^LOQt>-X
zcIKfQvz2vLue;_SF_YuW*M$Ke=g-ec)!T4o+qbE1h@q+TWgE{68Y?gB5maWIw>M-z
z^Q+Lb(=QbN2#WuVT`L+frCH3itK~+;PA;c~SFU_nyl~z^cCl}>_ve2~TQ6s-^6}%Q
zDI%&2l}SD?T+c6!eq{W9`%liu9G^WhZA;G>zqMQM(J}R8sMf+QQ&vu4-7({NpnOnE
z>gou|%NZB$xLIl_?rhEa=;|J%BtL0JZq&);9}+cR_J2w=w9NWkIp<epj2ZvY74x6#
zY}nrU=k^=s-+f|Ea^frB`3EiO?Daj{Yi}(jzia#bKeeXEa%C@_kZLGACTX@+tH^tM
zj&R!l?U7ko|8DM4-MDe3#pTe5;?-Wy?8VF#Tb`KCn(V@%`19@2w+syYW^?v@v$4CJ
zc=Ys#hL>xPc^>Oo;mEBe=+tHPie0s3!k@Ir-JAt8!)2vzeVlM$nWOsbE|0qUOXv9;
z4j(ugm)Bb=tDxBO#PXCQr=rUKsVj_6d~Y~<t@n4?&)Cp*_W3;gY{6$!4{VvY*ZS&e
zm(`krPF<`Zr7rBJZfu>qZ`a!rKgG$b85X)|Eqm$QG9jt8D8Py1qHLJX)UN$?Qi?4i
zmL}?*0!|!?El<o}vFrLQ`PFmsxirhEpWAot_F2}E8Pl4ZbvEa7;J(vW+xFWEIi8>2
zq8efMVpmzPhLc6)o5kt=kAy-QzV!NdP1lP)c4tQGn{QXyr+s}r^I5*F^|#83Bd0AG
z+@6JnYDLzpwV&)=@$<){r5~);&DGgod&^q<_m^M7rGKlYett85MTJli_taORMt>*%
zJ|r&v@KIj;{@DwzY|ox^f4`=a2cwH+#=^uD-Kd*^bI$7b`2Se2@3+EL+b_q(&WUWS
z|2i?`!kV75g{li5%UoHuZN+DQr3P-UB|K9ax_fVI&b)hR?sGd!y$zXn7nglH=)AGI
znQ!*9+t;RaiT|GdXWxT_V_|MhuWZfM%)EHTY0}f1;@i*s&pm3$AYgLi^_`QSqEEe#
zIn-S|FIs*5jq|ts_P&Vx`T43_y@a^vqwT2|<>x(IrCF^PYNR{Uz4yrdFK?{h*t?nh
z`g69*UPNDUiN^w&Ef+j`f_Cj(eP`E8b7OIa0M!+I@u6D0ET*eDUQbKy_1@Gg`=x%}
zqCcNEJYpAe|9Ulg<)^5stG$vdDl69={Wo{cDlX7W;oYmHDlX3#>E*sQ$=dnsb)on6
z((KUHQ(QFq>r1a@zy6{6o4>6&(faA*`J2O~>q;ZV*NNOP$p|$%`2LnN!>jgow^AR=
zg4E1jolgmgQ`7#2z4kn&f0N&1zv{Ui#?sIH<I2N6^9Bd$U$5qWeE9s?S08uVufJBa
zTK&&F6}$A_0>ypL8Frk>&i_@rQOK!FOo~5h$A{zZSD3^vt$VOXt~mYhl+}Gpc`TLu
zSr|Iby5xPc|6m>ZDK7EUgzv9nI=-|U{#q03$@buVTcCPI;dix+$!d(LH~gY&*7cXH
z%hwa#tfOtZwd9Tc?@tR;uO9gDs>A%_&n=B%TkgLreRxVxoA0j8p3<+=oE`0~t8Q^H
zewZ_T_w~bT<{gZ>abs&n(D&Q=OT*V))3+1fqi%ZT_r?jEkG`7za{v9?r<1+rnJd-T
zq+PEyxO&TU-3R6CN-ZLiQ<}w$CVkPH;`3Lrrsq}36tluLAs^kXy<9SRQzC=Um|uO<
z;>Iu~>;81B-sTr`Y?6+CFrB48U2g9E51W(9J739O|7XonbL2GtrW#I<veN5uCp&x}
z-wwULC!9yOZr<+1T2ljt39C-M%PdNLwLRWDYImeEcW`j<;XS{ib2CC$PmHZB7GAt3
zPsQ%dDb2XEX0N_&w!C_uLy_U?qi!D6>2KTKgo(X5zl=%n$Bc+~hfeqV85@h67>IX-
zRD5i!UhEX3H2<gEKSl!+L5r#Bt~yC^EA@`l)Nl%3-qEyRPsoBNKg<ndE*$T_6`r;>
zG;6)azPQkBlcsq3J<GR$8dq~^p=MH4nBa^~RxeihPaa=SF4kSOi9slCh0^|q#~r#B
z9KUzl|6>+|()6Q#Q#buf3Awj2J3Gx?%P6j;t0nw=+!05O7LWaFTBm+gf2=RM@nQeH
zsoOVCEZX$Ef8NoPp=#62!j=g!bhWU!E)$Mixp4}sMPyUYWw{I0mDWqX{d&qb^LFy7
zE7C7C=Dq#?lR@m}=MU+HMN^L~snMMJVs{;b<-wyi^<S*~^bCzOV^7`ra=HHP_0-80
z^}h}abT16Zu%5lue&%}fMZF)ktrurVpWaq!FmI|rkVYV1l!?Jr@66qvLXV8KUiJuD
zpAOyMW%77W{5`F3nJ^b72ICc*my}kD$;@YRo6mY+{tad8DKpBlv~HPA+p>P{KQ6)T
zdv)i_uGo8eOOB3YK(q#nh*z9W62q5{DgPg?HZo4%eMRW%Dg_-)rj!`r4bw8i-}7#L
z-}z*2=G<Ai(%!#PP5Sb5e<tjoQYv`sfn@DAKTpO3_iH~K{Vk>WD`nxo<)-nzKekqG
z+c|Z^r7L%LAGzwy#PH}`PK;UQ+mC1Z`D1qEI$i&jRM5YEo@L#yp5u=323N)M3b_B|
z9$UF3uqSxiZ@zWmpWbwtUACQkdH#+U1rt{HF0`@Q;JLi>ma=-&v(~)7I!!It^(`3K
zswCIHTB80uy+m@lotIJnk#_b!wzZE|e&{LpnqBz+to+L<(ktT*ZS0ht7I%5s!k&y9
z?_$4c|NbM#Bj?a7`2Vfc`ze=m!vFnPvP|Iqf-Jowt{j>|E5zb6miE>dn+Ww3O}%1#
z&4(!<p2xG=;fUVr)$y*Y_7#{_ZrJ@MLxugo0UwjSyX6?NtM^`e8hP~gm4kAd?%PbN
zS#*`b?p4PWuG`ne)Q+rbl3Qi6U19aDDnBI#gY|37muw5*_;;vt$;9ZC;&aQH3|4E&
zWi@hz9cr7}qH22AHt1T=Df|9s_cCvbTI`hABEaadI_%fqEN<aN!Qy$IzaDq|F=cPa
ze6Yg)Kuq<=EVElqf0nMu`Cab0aY5DZ3-Q01jHc8)G&ar--M!6{`@!313;iZvm&-TK
z`+If6h7B7o{PkvIzG0aC;l|p73_)>mA1z*RGt?Ye6L|UF*ZCVtUOr(<VEAL5bDG=l
zPUSt*enVfA-g7nrFS3$39Fp!n`_rb#;I_{8ZPZDF?$_#jcZjWHy0D_o^=;$vdwhDy
ze`Z!``^WHcb8~;ZeS3f6yYJ1%+N;do+Fg6S@ipK4*NKUVajV&xEw-mMRUhcSlW{NS
zz_gzEZqw_luCBXTwq&+!!_Mj&{%1CftdHcgxIb>yUGcbCV3Dh3Mq+A$?yS3!o3bX9
z<i<0%Z_;5nvD5gT=JCr_1-rJ1PiLFQc5=H-81tXN_@Bmn3@vGg#FYN5ku;k%ZT-8i
zp&AU${p$|6$N5Xv2kbL+_?JIHjZ@%86eoiR&z8w?kA)c(UI|X;VCZRG`Jk#J>B+Y#
zA`A^$j$6;fn*B~WHAQ!mL2hsSgGUFjtjx`vzcP3EUi+;Zu8KX%bgL;;_wQumZ%DMh
zx<=vetC#)$yVmcSrnN!#@ADI4K@17mW%ITiT&vb_puWArHt^b;Z6B_E+;5`pfA7WC
z%3B8M&lOMG9%FuR^<rU>T*KiB_vcD7Hawko{ZhK6Eoc1+j^AH%QfGcGW%grj7U%!7
zeskFthdci?jTw7l_Aba>D#UQ)+15H~+h4!y`67dCVsF;l&B)T;llDJHkul-!hGw<-
ze07teU$5H8xBstcPa@--8#^vPKNDu&zdKf}^4qG|XVFp*^X6XzdBa9gMYdK&cgvKX
z)XKyv`y;IRH>dUQ;$i)Inu{T%<U;xNbw`db$#vIVk=WMmX~R$$xp(SZ$$sOm-I_5D
zON?wjCzx4oH(cxewb|$PrcOqNBgYEfKXGE-`Km^V;mRtd6E>;S1+&^)m)1SFWaRNL
zUjFE^yd=ehplMsfIleG%c(Z3=&{<ZKt2GR1sS%nXCqp{a=RdqExbdURKBKv=Vl_WF
z7>?)Zo-yLv)VrI%{kRK*UAU5U=%(q~j17s_rr8NgEuC`~`SxyE>zz~WTVk>Glh>-}
z>n?Qm*-INwaF>Y@ePWVcvr{LzT7FN$uR6CI$@f{@&(8Mw_v6y30#%l^QKlO61=|Dl
z)Du3IpR4|qwa|IK+snD#ZQts1<Er$W-rhKB_kCS(-j3%@XKm9~$-nzoYviK0dD-q;
zb<xZr9qKJ9twk3OY5fd+^`iKdsq<G;<G|O~#1yrXujwWqnUci#VIu$3_m`@Qrf7Av
zFmzu}Sl7P#S9p4BgJ<PMdFR&hJgt9U|5jYmn{BB7FV|!`#}fZa->yL4o9mb^Y+JbF
zwEv=<<%jBQivLdxiiu@eC>VWy4&#C!2V%P)xn$})s46fm2zt%@)U>stkLx3gz{}~E
zY7V;FRI6ua#Xa&?KRy3G$GId`{e`7x4b2vnK0MWsow%v)){~y+lS8!>W)~IR<=eaQ
z6W5i4^ZxbMm#zKuFYJJ3NMZS^nOApqy8r*BEAi))a(#&R?Y}P~UxYH`H1B%)x#;X4
z@$b%zTe5QJSH4(Yy4cTI`st@up4H_y%}&;TIJKGKS6SZIwaUJxtrd-HjX|B)nHS?*
zoVr=3+}ZSb<>%;Ej0Sq^x9>1CS2obSAChUUJpZouUCGH$KTrL9{_*)~NrnstTk9s}
zYz|y|Pj&yq&E3(VVtyH_nz~LgA+HVzd~L~_w0_;2MNb$64}2`*V~9}QZfWp(Y1XbR
zu8$W(I@B9(tZ~b>{c(~}MRgfN{29l_Cu{hnr*BOWT=yh(Wz0D@1<upvaSt?RzE!t3
ze1Asd`o!>WJG!mTug(8e>2#7ccb?5{|9vkGJl!_)c&An+LrqQAJiQ1lk=M*SKD=mU
z`gY$&_@DI6Yb&|tux>~`b1hEMcwZ`~1!T1*I3c(_cI4i^f_>#~<*nOy7H$2i9&-9e
zD;HzK%A~I^*UvCaj9tyi(>vok|CGgLD)IHR%cf7^a{n$=Uwc3{V0B-GW3*?cp!}QZ
zHYcsW1}e?}+;7zNJ?re5>V=xWD<jt}waw07qIu-;!&t2&!VE#T*>p|f*T$9^ealYI
zK4!um!+pAF=><04tS3A8{cj~buwdBpYR&fZormkXIBlit=RI$I_vdi#)2TbF&+WdI
zI#Fiz;pTYtCzbL0K7BYFK7ZM+n4BD^ytwA-hab1MKl6^;`kTN1Kz_x#Wdi(L^1NCr
zoKCoFu<L#~_v@7Nr95+cORrrucN8Nx?pt!!J-C-;bJdIT&B?bdUaikie-wNrn}4~_
z*J;_WzB(y89xK?MV_h)AY^w`{!t-zLD;Phl(qGTLqBbm(lR-sg>Du7Ko08|)%~Hz=
zcDGrwa!DfZL_vl=?gdUZj0{&^nuQpjcl(<#_4(E7H|Ku4@qa=1jx9SkOEjGNf97!i
zr2Q?+|9LSyOWnUveO-v{@_#(1Lu$AD_#S!P)%o@n-M=R}`Th0YZ@-fMY~JU+S<}7j
zPwcm}uRCP7Gp%E@mBoW|v%j>rT-f&@bmtXoEuojp4?=hUFaPBD?D54q?dWYc=U01b
zh=19yDJLj;xAxDpl56Y*^=Avte^?@8XZi2p*Wf)-3Dz6sK<Qh4vCNhUCuf*WDU^PE
zy<*vtyE9quR=>Q?@<Zg^H}<o(5f}c8|20!%NJu+$?p=OB>r?;u0IgufZDLu=ejIaI
z`H$<z8Xdhvt>sHQ+Mcq^Iulm>eoDrv0wuT0YXk51UHu<*W9zHVQ%nuNj-*~~-NNTL
zdDSD<v#mi43AfCqohcE}jMHYl#>4<BF<<h$P1l!P^0R2`?R<9UMfSFh-JfsUtUSGA
zznrQ;Z*?ClL+OV;;e{{zkGHesJhN<jYqG>SmBDGE97CI-hkW;xZ}pKm&qQxZp3eIm
z`-8)jVfoBlUHf%*?mySQHLRU|BDF4gaZ7d7!-w0?`{f(IE4{0=HM-<d&g>`uZXe1v
zyd_?9X0rxE0DFr~?f+i;rY#Ki>w-hFF2)s4eO&)-PQ8Lg*uh3`4%?l}{{F~)Y__B3
z@(SOx=UE!&eYmVuUGsu*dud|sGo6gA#CLy=SDk&{xc^tqubPj`*GN6g`z_D<zyp*5
z(>2(2eRz6VnCma8wXr>IStr)>ZQ7D2VwKuk%&i1}yVrYi3#UCldgY7q-)iZq9iG#_
zUR$qpboK2|wS@wV2{%v5el@AJmetzKAo6~vhrv^oS&sQlQKDP*`{$p|ey^eG#m#VH
z>+ho*7Pm>Qky4R;y&!h#?p4kIwoSbBF0=U&&#CU`rsB7x4SQd|@Li$L^x?s#M@>SV
zfv3DPqc26@^t{$PrK)Gjx7E5V$8*0ve&fx_aHn>LzjXa%r-en&HIIk)urt^_TJQL_
z+9$r$?8&>&)Ai@K-~Vj=;NDul?vL8%&)uj?i!nYO^!0bX#ATKPJCFXo`&Yim;m_pH
zn}6xE8Kghol9j~vKrW7x>A-5`L-Lu~zx5Z#Gd5WLIJWos%=OWCH@Y($B&_<H_48uR
zk`>*vt$OcsI8-L}b$5H`uYGs-x1Uw3*5cd$k4(w>^n8gK!va2uS68yXHY&55oqO};
zx7*pv)4%72mLJ+T=e_o-3VFF#^SK;kWq*fFySV@Kl&lQ}jjNx%X8wFP>dNZm_~oAa
zU%oSC`)9r)@TQD8zr)85yUbGSMgOXOX8triJnd`z8IIq+uT$6WeDp@Omf^#xv%>!^
z1y9$m75tfe|Lk{03+s?S(<(R=k1`$p+`9Ww<fS7n2|=3=dg#ZpPkO#4_p8a}ek1RS
zMH259&*g~OzcA&EzvR_*v+GV)TORar_dMC7$5~o1At_dZeSv54$&On(6AX_lJ=|LK
z)$nP8dGndSw*L92?uSphqCK-v#nQ{rbV~d)rHlUApAS{G?&#YQsL9yy<oc<dcB?Fz
zzuM1TbYvy-`O|lmFPZmv-d?JC`Revc&3mEZ=kv9gcBH=3Ji0;t%ORoa-KWwN6%}{h
zeltPQd0X!7Z=ar?7E;}3v~YPt@R2+A8?M&AJXbtlrd-ORdQEhH*0PuSb*?W~zFoa+
zEvMh@U6XUK)L-7Sk#{W*W9$7(jgQOI_U!njbN9Mo+35#M?Ihj*S(@z*p1S3|#r>DP
z%TJ`UuPr($!E<VFM9}q0@%dkms~|=}Z+-2*J?ZZ)b{5{Y@2Tc*xmLZqQJ?U>?`-`h
z^Qq5L;?Ccm``B9N>*+hT+}kc5xjg&srM9yL3*E1OpZ#>+zZVBAqt(CUP57+o^?v>P
zOv@kd{rUXvy|G>rDEag7<FD0cuE<Dk|M=xZ+O$tAiuae!n)0t%^hTfEW6NDJ-+I3o
zeJOsixkIqmuD-Zr;!F0m0SlYfu6=v7Tf9`3L$Ouv=>pgI&|*hU#RZS#k9?S4y4N7|
z>`Wmi4WX-vosqVd(LH-vnMLK7-Bp{naeKlMEjyLA^e;EBGJoxDXi@Ky{HGOdtTx*l
zdB8QJk3;5^@k-8D{@=Fht7)*G<@8hR?(d&|Qbf~B)#caHv$~r+pNVj8{8%K~;<R#Q
zrr)z$sl_{6!t)c-694Q?mK5C-G1Et{JyEkbd83Y*n7xqCHjBtN)ib%AGC6}Y<IYd`
zFE&R<4Y~lYGHkZDsNa-kp-Ish6Dlr>OMY&ByXxnO8ShVj`>~<_(h>iCtPBg~QoQU`
ze&-lzPUQD|VsE#|?z_V7%bbi1VLGO!)1NH}_RY!3`uDhc+2&(Ub?Tn{Oj?<FEK4{j
zIGDks>Wyvj!tISa9k-qL^7GTYTk>mu?Wv~)oGM{iS^omFX8)f4LvPKMJ(GHWUO%tg
zBH}%zd6JE@vlZvbt^kdyy0h~)uAKH(vL@w7YtiIo9~1T$cmA%~!J^pmN8yyCr{3gQ
z2D1WJSG6&DKlprdbBjo^&}%dIL+?bsc?&q5;9n#&<;gO&v#QE(w&-guT$%CEBhYC|
z(w=g$+v-A29EwXU7K0RMi$4{-Zm;)tW!K|3N}w^(8p)>%RHpCsUASa2XpA)8G5d-z
zNPNN{H_MDd)g-CgT+6oz*Y09BQ)~%7v1vho#-_;EjorIn6~DT&vfHq0jr@PdjsEXk
zkDS_H7guG`)zTimch9=IZ7VaE1q;Qw&9N$Y)!MX5tVLk%!l;^$%1QrUDX&yWWoUTp
z<@D3<x}adt1dVe#G6p{%?A4JMzG3p>=B16!$NhM7y_U-RR25X+c;C(NC9*QB>^9d<
z+4HOam;NlRf8OfY9lLMWpFMw`PTiU8KA$U`F~cS~e!1IIdu!Li!qb25hsiKAI5zJ+
zH&=)I(yVWwG2vg1-HM=bM}FZ7hJeraFYI>m*r<?a|J@<t#7ob$TW{Vx(0Y5%-o5<t
zH})mu<d;@_X!P}NFU#NgCgfNims9OqHHXws3H(dGU#~6uIOo%oX$K^(hWm?2hDm*S
zzG|MC6T{6Gr8Sq<+<*15mVqJYRf|H)3xTH#j`mOb&K2CY=l`K+o-HEPNvE&%$(qK8
zZQHzc)22-uS6|P)GUs*NYpDjt4YkKENps$E+P0)feZO!!|C|q!VfxpPZ#$v)CW4uv
z=Eyo_^Ziw3*)iwV#XYu~7JouFl3RMWnD47+VKprRpYCd~>n?N2l>XvbBE_c2IxT3W
zuVHYg<Y5QZYr4yQvbh;vJz&=ty2Q8n+0>jjT9HqFZJWMo#eti3x1wAx?0Wk>)lzcb
zO0U1SB#WMg`zy9guyJ2HL#aO#ytXUl<SX{|6%F+hKV3X5&)_WF7+e{4MCsYoSBKPZ
zAMc)VOqAv6QYO{o`#T>t+0|XyXOz7*sM_`8*HxRQF7YWcs{gI|{n2CAmuq!y%=cwg
z`e3;6PV1ZRed<d#TrfXVCH(Ei^M{*+rmpQV{u}r-MSt<4`>Ll}ojgx&da7%;?uvF&
zyj}HQN7XC(vDacwIVnx8n|CPA{wbs5>WzncuH{)VmNq>9d3>{V=*ds7Hr)_EA9rMx
z!p1d0QU3O)-<qeMKRo~F$xxxmpO%I!lKEfEsi?9(fG=u_WF+_hcN&IFB1LgJi3}QX
z%lmAmt>jh_&6W1Pb=|LCvg7u>Y3KQ_@L6Zyj4)cTEr3H~%ASZL3@-#V{~r!dg)9RL
zb&LpNIk7`6LHF{zwR;zSvz@r-vnqpu+Z&zmzgjc%cJJR4y3S%v$C=0Vriwdti}$`*
z3tFg`uqsqTmvuo$t)NWBH{|76LB(-bmf8xt>Gm<)-;nq<`??&H#Fi_c`O~8r8k91w
z*KAik&SJ(J^{B1hg59xsZ)0?pr~NtE$?x9B)%zROd|n~w7H&S5hpqm>`Nw&4)u#S7
z+2gXmxBK{d+kdU~4woi>Hs00q`FSP_pW+gRg;6zv$9i&=4Vm6oD`h?@yLm(SlL*6|
z<F*rn5`W)1)DjVT!Fg(0s3eg?Shkw*A*|i@tbTfIx-JbJvDd=)cZJkG%=wg1e)FOS
zUqj=}jH>dze||j|o@6c3S;!UX<mB|`b2X!P+qP)Q7)FO)#^l$txfyTXG_7CzA)-<%
zg44l&g|E`nz<YDu{>mz@c>k_E`}8zt$M%108)|>`oHo>A@GR%wt!w$rAn)?q!_n7e
z_HE3K+O4yk-IZ15+w-fRW3wEStgoEqV_1KuM(N4qQk@j_=a#+k)prt)e9}%#OiWbV
zEza|6$=oGa2D22uHVZkG^cP)F>NZST>&mdehp}PF^i`7svo_vSceS$nRCVe{^QT_M
zPxZ|vkxLF-^I~vFT3~BzC&;i!uCSBgK*S8Lb?q8%dHxrtd<o@TmHKYeP4(bqITKq?
znw%HdVX-0hcDFaPfAf=Qxz&vSR;>Rc$<E+poXEZF??k7k_Y>#7GR?ZQ#Iw6^@9}rL
zma)hyN^Y#XAEIu^;9-6KZb`-a#%pKX>%Bue<rx@)k~O&xZ0MS6tjr+)@y%SG1DmA}
zGDz6gGB8B%zn;(g<3<1R6ATsAMaLMgC1!2*5@O)p#dA$2cK_o!I+s1)udnv|?vv(n
zSghxL0uRF@tGjp9Zl=`rAALLDNcg~stK7H3v=|+7;$CmRxVZk$aVHK%`wmc$O#jTq
zkYM%2Kq9J_^?~dxsXs3syKZ}W=a|6TiTs(DWqo)U6wKPH)t^tcUvW;I$ziQl(;3+r
zJp!|uYG1Lx%e%$;|7&0d->fO3*Yvd+0xmk<uU!%HIiB@xg1IB_=9queg(fXOw)V!`
z+D8Yz^=dEcmyO`%Vfgvq?fUEo6Du3P`?qzxn!fez`;6CD_HXW;!ua9e$8HI`$0^F6
zgBY0pzqPNwel&t_LWtbvls!wIEI!}<apCi8pYJUB7~8|hFzHku_nfbcD{92&)lb^D
zXqM82VqJ#zk86!ZUwrk^HFleJN<KbHa$U=U(A8{`*{z%0mfipVSfOP?jMHPsZavlV
z&wF>3YBEMRa=ZU^D-GT4fBO2~KR;y5lH>l|&d?Jy@cM10>1LX`o6F&L+ao>u&YlUq
zK9@CqNL*E3x?>l|A)%Z|F$Vdc0YCM0Uxuzy5Mt7aYi7(S64EsB3SE;Fdh)`mNwIs=
z-u~i$xx2FKwO4)Xsq)b2OQ$s!TczwgVjcZz`ODR|j5hnK&fLxYwz8i+De6it<}#?S
z$9XmM-o-LLnDn2s`L>VNg7g*QkCxtT&HQ#S`|a{=35+Ke>&**d$cVaawEMyQ);QJ!
zW-FrOW<7OLb>g@vdCIYS)7yEPZnZqGrl&A;__jL#mNMh?iWHi4wyt^p##K6VE_ek+
zwd`1NrB(42=UtVXX^S$a$S5#{1Z)&wI{$9zyPlsL>zNgIO{>gR>(FgK7gczvK8NK;
z$oa_+C*Q8OnyBG>>2r4W^S(Q!zBjASUDNiP<hu6P()2}#z-y)~*Zp3S@F9TxkEKq+
zywf`KkM3U*&JmK|WHx^acTP?c^R%=57wk?SJI40#A%n>7ZEZVD^D|B{#B4G<^k(|{
z@J|dseq5Qe?=7dI%KiY|eHxoh9$kMoeJyX0$}-uFpcPXmE*58B71Ynr*3^B}{CARi
z<#Ic(C&m9NT?LmvgACSi_kad#F0SqqN%G<3mVTyP+x6c0yH$zIF6|wUP6;~}SD713
z>zZV2d+KD$FSFcjZ9o24Z|i>6@#)B+i>vjQEblC@?Q=B<4bQhJ_;KRx29tJv`EL@J
zSs1PytlsmUslhASpJU(mU#CB=INThs9`LbIAZSB?SwM;*`qHSpf}YmQW${(pYL$Iv
zM=3E(DE_T{g`q-p-@ZVF8JC0@JUpbMqQIjt%ugYsFsHvx-NSj(zUI&Ef_dx*4A>6!
z+s}N-X}Uhw^}#(+hL%3&s1PF-hJ^;7e?K^AxBY&fZPSjJ-+}e<-)~;L@5Z6<=UedG
z6y?ez^XkuW%Cegn{QpuM|5tCXY5Wg8>mO52XP9Md{eS!XqbZK84k}3u4ze-@&wYP6
zJ7uWru3S+*MPQNjlxDG5tvJqQg{5cL&$$0sm;dsaloN9py=I)sG|u2X&1tf<>};;*
zB1_*RA*>8nwkLnxUv@C+t4k`QL+ztIkG`?{N@>0L-(4py%wS-gnEu^-#i38j^)>ci
z;80}koo8^l#^&ACb*_(<4loOR&B#~2R??AmKg9f9_9vUEIy2c9^wr67GgK6RkK0k^
zH;JL)!i68N&Yhms6P)z%^OUFDM)B_+e#~87@%2cfuHF3)u@U<=rlkq3x7u>~y`;+3
zUA(Iq8^ZJAwwMOY_Sd>+SACFi$Fq8Cu@zSqO-_@W`a7R{{nED<@}G6M7_KlX`rrS5
zp}$SQsgB|40@vvR46Dqh{z*E>U|^E2tIF`>gof#XDVO~YiC$T?qL%khOkB0RZeSFH
z!@8XdH@;c1kZm=a=W(kSnZHt2S-r@Y)_j|{VsD3I<;OkGoT`<V70T}A{OZae7ZVl}
zko5d$$lJGmnEM-_{@p)!g8ZSAN$+HCmV15f{VsUkTTwAw|L2=qKm4~_y`A<;lfmGS
zoV5GaD|^x(&3DZTIx{2vq7j3D7q0}vEB4pcJaf(RYy6kj8JlLlo)r1`ZFc|O<pvMr
z{(OGln83<VQ10x^Xp$3kTcTs%yPL`X3>g}fRv&12nlmeN-!1#fxC5V;xG^uw{<hx!
z`}+^~E{Dt7?#hn(t0(u+YuPN&!9A^e_dcH;RqvO@%<!sVq2Jyxd5fZi$7lTZ=U*||
zv?qV{F5zz{H|-Jvts7~3&i!3s;UqORrpdE--Kt(%O=xmgiLf}aKu6)siytBEzfRs&
z-f-cp!sSQ06;;9i_Dxz6cd|$MWMJ3Ig@261JGZ@`92{@4Uf{Z*xt1!!gg;T1*WFFi
z7yMfuJjJ5)LFLZsTOwSaPl31e<s4gl(6OsDG33wbYptr^%BG%ma&ju#epAu8E$8;O
zw@*(`SM{25;_kF=jhhoc>Rv9-{2#OU+~3_L@2)M2EY)OyY^M91_iJ&_hT{)cf7Uu0
zKmUn|&u0CJ3#$5gPrGaM{k6OISg*ls@^$|eRku^O=>O>WlD_lLzN%vZ?t71`zbc<O
zu`a--;IHY~KjplYN4Bup+RaSb|DqtE`f#=PvAz#H*Xv$nn;h`nTU=}qT>4;1qwKA1
zD>6g7T9q=c+vj<I%bu^CVX#MkcV>7%>G6Ne)#W+=ChyG@+ubrLClqrH)A=(Un#o5x
z8q@W!zTa0@l}5N9&ZY(0Fjv{`1@6vos-BYKW|?tuQAclQdqCEzyJ~TQ(!%S7yv`bz
z^@gh)IrT(qYCzPh<C`5Ko!0p1wO_b!=Sqa2^lGh-k(0S1*Z8PSKfGbe&W#ab(%iw9
z&#(Ub<h2I7s>=2Yw|rdh-*)+z`{;Y^r3Z)OKHaS6W^l4SF{S22`ti%I$1aFnT~Z-5
zd)Dmq|GEqtH?F*4F<np1)NJaqt*U0GXOpf5ZM+gPXI_byZ|2l0r;c?OD*`?xuh`<p
zq1ZAZ-`!H<r(BS*n{jBw`G5&;`E*-AUD%roqHKPq`OfloyO9}h=@fkN(8NV88b>Sg
zqYA;xfGk}tHGZar>zz_wSHH)<XH~)Z9A(fIhIWf(rquLr*PK!U>MuX}eX$|{)Dae#
zbbp2XmIVgJNt+`4o)&BUbM=ZlJ}G!>POIP3n@7x7o!z<d)1IlUx6cGAFijE$ZFxCA
z-6-`7>)f0hK}ijd56c%j7G4qe-p9T?;6&&unK>F$JdU#Fe=Un#@qgz8_4mBIzGt<r
zWZYO`l@xz(zno0$qJ7zmKke%46>!?e_;dkl)7G{3{!FWix91Po6V_uB=8*Q^Aa=Ir
zstpdIW{(m-@#iw>3;DcUmmhDG+>`ly+N;a%lfEBKnR98$!ju>OhrV22om~5+*KcmU
z(O2p4yXr2_enxk*Uf*<w^K)_&TQ&3cE0c`k?Tzmj=$rAs-}I?!Zk((z|GZu6J^lzB
zm!4uO#++f3EdO*t%Y;L%Ssz)w??hic`nqef<I4A2G-dxk*!IX{f|5qi)I*F6;S;Vs
z_VMKY^mn`30;OX?_J5_*yyTgGPx3hw9eTkz^_G3ig@v1!WZjbVmGXA`qawn@WjX&|
zeQHcxzMu8Q@9zFx+P2PY47WEtpCc;$?TzJz#DfZF3vV-S{w{N4bM5C@nR&Sk440U`
zNzUE9W6kqZ*(*QWTt6SXtG03Rz7q#dtV#W@mB8@i(=_)vPScuo<_b8~xiJ2gj^1Ot
zcb<{H%j9Gu(buo0iZLG88}RgF3WHb2jZo*WU-w^euzPGS=a(1gC?LId*YB^Bjg#Sf
z!Po-M#+{GrI`LYQ+w}jphAa20UYTpDHgv2_ZaLw?(4@DuHK2&SZ%M^b#)MyGQ?_hQ
zW0VLNDm4kHQh)w!e~(D%n!_RW{f)EMHifkHo3QI@UtgJ<&b=UR@ipJodT)$AZ+*vf
z#w+MY<K2@yw^ydIavms&eir=q;p<DGUT?}Q{&t+ddwyA$@Z?$himc5Sm7R>qohIO9
z2ij1^!o0%2J*8CHQ@(%M6TSV<k5;j+Fq$!C#W${C)}ytzH%D!qbY-H$v~T%qBU^pz
z7DdeXnE%|rwbmr;{H?GEu|4lSeVMo#?mwPe^m1<+cfhd)4|=aFv&JMftSYWQ{$;P$
z(^DaD!&ge`?>V)0&O$y71LyUO3Yx#>*{^^0GbQ7Y$n0BJw;4~Lw&sS_^wo@k>DSJD
zm^_=~<yu#clhIfBqZC|hJ#3ZI-q*a{@;Us8lW>UXNe}0zo`1Mi86swK9C^Ta;^{3<
zZ`dZW_2E<3s-#Yu{PVW|PVxC7XI*_xYcUybbXfVoS#G6~ebm`mTa6Q(<W|Jh7^=9%
zU)ac0{Et!nbj^~k2W2x}xkf!UTpuEywB>XAER$(G>&`K0q$(L0KT>?&FXwIdt8li^
zSq__}zZ{hqLiEpT-P0;RR#S5Bqe1BAIlT*1pCn&;xBT0_snh<Rifk)u-=1$Crmm*4
zL|#Z*x|`8Ku{*TDi6LR>u^``NV!v(v$Y$pzD|S_WdQrN_D=8)9-Yu_Rg<|U6mH*zv
z&f#*JeCb-)vo=Zo0#1LqeFAkw2F8AhlbxrnSlg$5>~u<iv&P>?J(@>PtDm=9;-Jw|
zcXCtMv-xkm&!3EQbKUPfg}Zgqq}0>VGqSpWUYx$Ibk-i;1DDM=c5yL@oXCu9nXss_
z=mJxdz1Xj>ox2S6E#sN@)>oyAJQ3Y^%AQ~A9^=%vU)yeQG8%BbdsTdIS<{9k91{wC
z4@52N6?qL=!lS?S{<d75m*LirSud<rm47pRlb#drffG?*(%lUiroUR5cRXk!>$0VL
zFKk@4!pW1-Lhp~Gm&%203vaCK+OWou`GaZCjjpY(Znq{M?YkcQthM0Uwnu`Q;U>_1
zZaVyx&rGIYOn1K`&ba6OOA+7B|7-G3-8`?CpS$Meq32UsGaGa?MR?X`rA&UEs<v%q
zdVNvV=9lW1t(;kTug$x4;glECk+%|W9to~vG049@anr@?=PLES^MrL-^~)X1k1Y3E
zXE`^~I@j)ZxpT21!-;wC=G-sNyyeVrtLJzAKM(a}h6jJ9ZI#{c&F0uFJAK=qoH?!w
zm0upTYqvYS<^Brhwr#~1juaadc=xV3*0^tfd0yQ7`+qA-e+h28xM6K~{qLJGi+-BD
zx-^e-v6y>9{*4X4wr2A)@<hG**}ggV(&4>t-F2_e*<3s4b6LY$!`G({#urs?&{S+;
zSP)gSP*`}T)z$FXBCC>u-Q_|u)Y|6nUr}*{@qyp{(CaHzbXKlB=Naj(`QK)i_xu-+
zpCnyyVqsVt_44~_U7rrW*d3}b`;|Xkmtk1excDDKF@wRyv<iV~XR53#vZF6HEUOar
z;$m<(nVq-x(xj$Mx2!dSS8wxp&s?q@wDje!l@alGoLW}6uk7CQbXN4Q%d<a4R;Qfi
z*ECXJur<eZHT$dSYz(DGZtMS*(V8mE&=6DG%GNw-kJ7t6PcH4>`-Y)m<ukbrOQ%j{
zVK_1^uUMT)V#~LZ;?Nh<&i=1p(`MB0Ul;%UtwGtZFaLO587j_YN9<13w{ZAc(%Jml
ze!1-3z54B~t#9IuBo6E}6>i(f&dl&6|8OY70=~LGQPW&S8W!fB*A8Dd@8b+6hNiH0
zcGK6#?JQD!U_X1!vp4&9?%SLCzVn|?_ttxkiHV5|#reLaN4$IWtTA`losOrBCl>Fq
z?)_!$6mb9U`)kv5+nf0Q@+SQLA~`)dq%CxJ{KuP@W(hLP*mLvstE<}oX2)tUs<u24
zJmttee}+e@2NMH_uj~F98T>}i4sq!SDTP)yiz;tlp7F^hv?*)m?woHYr)Wl}B;MLR
z`@qr0RG*gGRhu8|sP|juruEUxZL3|x<fTSi-?lI;lG8eDw?ee2^uyIFI@uos`GfA>
zNo8Mr#o<`T)x~z2k1u_CynOpp<?qYae44s`@5-0=Q_rY0OzN*$J@>{sudw|dPYz$Y
z{zGJ=x0{q-o~GYxt5kPh(-#>NzWO&@OrJeXl%Zz9{-$ebN|l~(Oj1}FK*x8*`rAvD
z9i1)s?ZwqQn|%Jxi{_KLq>{UB@ALQ_d&@%p|GT_BCgQ;w{)X_k_o_c0#d%Jiz6Haj
zu;gcDc`t5M-`W2*XVc61sY{O?yA-5$%e+04GkN94jVo6cin$gYz0B+UkJX^)@S3o&
zHin4HQkT!=u3%$()sT4FPm3Y+5j#UenM`A%sCBv<gWaQ>){+q$H?G|HELYvkXrfoI
zZ?7h+YRePRRY%3${``AzrOL00;qs!D8TUO^8_c%0IDI%+K7~PIVc8|8SL}MZ$E^OZ
z?@tOg-zZ?9dHR+%3v?shvMIV^nbsj^PHhNUk>d6B(_hP%34cnaRtUf4_Nq0_&2X$~
z+NSVh>eJa%UNzr4*_O1-O*QbZ*Q`<p)>zrEd;HT3g&tfldo;=6(BzQs8d1yUJ<(lQ
z`dM}E*X8qlIAeDH^?Uzo>k^(Pg6`MuZ<e**xk!yYWTlDc>{Q;}TPpqBVweBh{`krX
zjt|%K?wj#^VBfIk{sC@YABKkgg}O1U4dHM5YW}w`-MPVMub%g|lAl>BmXV?Mh?DP(
zUb4>wpL{p}&CE46rD`={>AN>=`zxCpA6a&-WbwZGKT2)qKh-Wd_X@O_UFwu0_tF_2
zPY*Avx~giRu|9H+$AvXQ+%r%8e%xt&;)?&F<16Gek6v{ZklOh5j3h%B^jJBe045~{
zt*v$wN?kul8!XD7cl<!R%SVQD?gh`Lx^my@I@)k!&RQ;)x|(@g_r~qs#8eY-?z^T!
z_SeaaeD7qPD48>dyY#@lV*c!%w~n>PS{_~fKQsLGWnp9A)#+YWe=S{juPM9h!-U(*
z9%^w&or$o!;ACB2d*DK%($XMK)l&{rch|Qpe&3c_I%Dl?6PbDc9c2riM>^^5p1*AU
zH<k}SUKy5ezq!2Plqsj;E=S7@#iR1Omd<&pz{K)aYSZ%foLU<y>f(PsJ+x&KvkSxV
z)0PYgQ|~-Gpz~b%WyG1}ryn&E^|Qrf^#A`*bmo3I-}kFRu<jlI(27a^Jgc|QzV_k$
z((5AX&*UVZTPc6K!`ArLrQK_izMbcSNY43&LY|Ur4qJb%m74V`CvmILvt={)`&QNE
zzP&SX_d3n$^Ea!GdTv?~YJbXwQ}Jj2v1YNbhYL@o%P`Ccxi<Sw1jh>IyyqERJ09(|
z5@b*?NnKs{EyKHdTN>ky>KEsPeP^9X*>~Rf*`ZWDp^Y(HbEexE%~lth#NK1oD+kJX
zb`=j!oV<H-Zu>T~zpo}UFnoO-9>x^Fe!p7i|I1{^v1Pj(vsN-6nx|S@<H5qv#1?v5
zYCr$o*B1(3tb5R<?ay-d<=q1jT8k=cTLk7VklFGfX#S4XObmkA@zX;0g#D~rYI1hU
zGzJH!ps05C(|v`%H12N!@4HL-{q*15^R?Gsd92d6^Hxe@SYcQ2a;IAI5xv)H7pJeD
zmODND-n5fzmmk!ru9BR*Cv;Wa?nEB$2lb!R=O?}TniKXj`@BcRtMx1i$;)DAe7>J3
zJx|Mse|OJq%N?f|G=%SzPH+9iH%q@}o!yC=KeZ(y3~mXzO1`{zb~80xD0JxdYGY6k
zNwfU!-+yg|ddm~TU6tY$H=k!+XAYjf)VAvVx$rcdE34GCxEWpvPY)G;m*sD^QY-F4
zj0P*ivfSj8KI{`8ZT%FWBvJcq{lB2^3opI=y01k3(z>&nf>*qDcb?qJ`}t+!a`DUF
z*<q%lS;y<AXeu)Vtk$}{N9lLAZ6NpOGpSyV`>T{1`1h^zT`gH#Rv7>H(t)|rtBSU*
zS-tnd)wmlcr^&uvb$^YBu%vIQ%cs9Bi{|8%1>`L_{(jc_FIC@G&b+EMZ|2_p?>5hT
zE|Iu@pKMg=>v(2{^|#(yReiq9BFf-;=<?jlxAc0~JifPBt>K0D?KVDHtHgi*^!41Q
zX`btE3sBu(caX6`j=@9A{Ibt3eo)EcRCQ|hi|yIk4Ab}5tyK#<IjNH2)r;#J85+#X
z-?wHzSK(xEIGVl7^}T&Hzrr`qAKsE{4|;Q)6gH9D@<4E5U1^|cZo10lT}$unZMbq`
zjX7^f^}R6Pf1VtD*556<ZO$y%QxR8u@3UBXNTkV{eH;gxPj_*9DS(c~S`n$q$e?X~
z(#krH)s3U!@wTd^OdD1ytx-s2xN)mn$9YH2HZP$YQ!mxDC9XNm^&x_H!P2#A>8Yzi
z(*%z(u1H>HyF5x%F)5@*NuK`&v)a|8++Hstr;47qI)S6%*Q30Pr~Xb4Ui_6cFZZi<
z$A>@LVvOI;y_hckXVv<f*OpCdesX!Hg8UTqeTAC?oGuk^n(_H~fA+p_v1K3JvR7M)
zsc!o7AaqsWaznG#*IKoYTVEGXFFmyL=>D9oH{RdckmdDc=Emmf%riEpx2T?;CTp|s
z*13s-4|c9zZDk&rnw%^wC&!oY<QS{Hw6VjY`_oLXuhT4e8}&JUPIcOL8)1>e@FoTa
z*@Abc9!srVozbV}diU~~P5xD1-Po!%ec0B<man{fBB9jbzo)cKc;5;&-*2}~%3qqc
zo=CkW5+3cKXqv#VLVnGAgD;Whhu!DZ-IBYky7}0DS<SoB7MqPE>J$DyNlg=8dwt!F
zkNw|Exi!Cb3ON<Ye{B}i>9O0Lw&LN9WmPZDErql9Ml$m*PY=$k`!$PM;JZty`RVq?
zuChs*wM&&{zqHJ;x}aa_Ib)mm;lsv9?XC7!Hw!XxuAjPk`*IddMg_C5wJ$ID)Xgtt
z%8HA8Vt8pm-mR`Ly|<;-<f;E!+ka)}9y9i}S3~NTPF|6p-dtC9>%8UDEblO@PKMG6
zX=~IaR&Q!L@mjN8X-ndaDdN4abYmYZiVr&{YpFO}c7^S|l~ZpR{Z?F6o~K=3A!e_+
zOiu2{W<TZQC*>Mv<yT%^*!XtVwb>i=XNOP8{MuI4v%!7!uBDr^Kin^@jWStvec$~v
z?pJrOw=LW`Jyw(9kdg1Vs{D7Ei))oz+S|ETeUGSo@MlNbtiSR+TG!TbF*Jo;J)O2!
zBK^|i$KSqvWz)A@_VZfpoYl2&6&dwY9i`nYN~DB&<Qy*iS2NkP-Ck^-%9e~x7xT~0
zILCMA>esuou54X@Ipxo-&CJg$Vxs<ZYch0L|JfI}d4Xp6+ba)W%zCGOZcW*0P6t`J
zJ841r(koqe|J=*qAiJULXrJWmTI<N#*Dqw8I8F+-ov%vW;k0;3XXk+>Y0VSkcJaBG
z-?Do7eYKWFt;06`;CO|_=hw-t$Y)Wp2|V*5{N^mffRFr1KhD2j)8^(=s?6q4Gw0Xj
zbz#QBj-kqT`huRjwxy@)ZJ6@lZS-5$vpy_kOC<#v`s#nZy6n-_k88uC#>Cya;`*zW
zU#@%g>B)Cl_O>g3U+jRK=JoNFUbl^S$+n9BaeNDxgotsUDCXSw{;5{V*A+_7o>nJz
z8ZFy->+XsPsz**gX$XjV^|?)Sqv1yZFW;@Vk7y?)KMHkPS$S*S$4xh8R9NzO&y-HT
z`80YD`XFD{$JUu0rRz@BEtJr`((H4xeb1S1$!<ZQZGT@E=RdxZ|9Q=^zcy}v|2)o~
zyS&Ou^M>5~d!=3B>vQwx-{m~%sWr{i%-sCkv%+gPOm-hX@=Q!@r@$liUa#J>OK-9&
zo0*!Q{}=7ydp5IP#mOUXvCNhWiFaaDdA&H3?ibGyj27bHJGW25x0sXTC*!B*;m^)~
z^)Oq1_f7iWy`R6o{`znIPS<bs#$R8Y*Hmn&Vg9u^@`Qm~PV=TU5*p!Q91b7NO7Es`
zp14G<Cyi<CyoCpZPNcaBDn8McW6iOzd@EIO_PvTi#)i~2kDnhc3UykPw*K^s@QX$_
zg1##%wmcCy>#3FU!{yzz>fB2!y#m(S{dwu;?mNr-+Aa^1<{d4|7PTt#hL`Iwbe|Ir
z;t0*Wxt&phAz_v7t*w7P_x3INd+l1O%%Yj1CI9P%rEcCnZ}a29=Q0k(mYTj_j@+_!
z&FP=`YLB^eh)421-BT1OSNQj__0m-Z{?^Ja;VRd3x3}G$bn?jrkIi%RL+%ykc%1mq
z&&T<)UrVv&3CG`^I#W_KBYleeznisA@^dQU)}C6o@_?S$<tv3w96#k+_ld1vzI^(Z
zXZwsS^S8ySsq8;-&vx4J{TD8uJ~_|!zT0G#%1Y78Ur%$KbPZo0SNr{)Z0!dHmGBLp
zpPjv2%46@MB2+ooMoL;RV4EL*<Ai^DHR{I7$3o6XEdQ(iY~mcV%P034-Cj6rgJ0#g
zMU3~3XMO+3%pjPodA6d<u)Ae_j=0w*)&Byuf9uc9QF^%l|G(9Ni{0Lt#jOld_4hwt
z`}^C>S+lOKkGC(E)~~Q=)p1>T=D_BvL;d#I#T)&b-yfZ}+2Chm|AQ`dMMZg)RNpWA
z_xJAMYxb@XEqfhz<nr?M_tYe(7+pTU=*7{!Jp8-M?H^@cS*_xBe~rtu-^bF6r~22O
zng4%E_oB(_{@-3*_4fDA|NHCfRPFF<r4}tBRjqZsRSd<?&O~lVa6H+?_q9zf@$07F
z#$KC_N^F$={VM(3{rPOF=ea|Tf4%v#Zqlon+mAiFB3=`;&E+G5@yh*M{FVtVeY7>{
ziDBx#qcKyKs2yd=f1^<?zBBdoG)3pOCr4H1*;HOy7rXn!i4*th|JVNe^ONWE)VJrj
zH!k_@EK`-O#+m7JvpBQh?W9@jceaQ>x7qF&w{iZ~xm6d76Uw^N>jV*-9?$63M+;ZV
zv58le)b5Xb#~@|4rta-6XAY@9k9V$`?0Za&_0DhW)BhxPyq*=5aCw2(kFOjb_>`xu
zm$-k>!aY22&YXKWzfSg?Rgv<)w>{HioAz=sUO73r$!fkvrlwy%eagD9pb>QTyoLtX
zW5?&`=g*%pV@KB2RkN*s{j;gC|Jqh(EE>dU_#*+d_bO{8M*x$6&Wq*wCz)jCuW4)S
zU}OlpuxRnW-L@x>3;+E8)}QHs$Bmb@Uvwi2wurqf4_tPnwW4I|-tL&aRi$#O3mCLQ
zSBXeSym<3wPtwt@>H6_`cXkA-Xf0Z;${|_Y#<*&o&Azp{s?L4P5oc2t!L~~liv<V0
zx8A1G#cyxE`uO{Qd$zePQ2)~xVrPG-qNb>Df8=k8ZI|bDduK};e_S}v_xIV^w~{xU
z$j*;Gdu;Zn4%W55DnGq4GcH^EDqOtdw*9`hP5*r}b;V<6zEb)3EA)u0IBRLl5vPA&
z>X~<~<qr2MuIKPPXq10%PwMGuq3dFPo|$PJx;pGCD9H%AE~(m7^z_uj!|li2ezm`o
zTk>p$%AdLO+*dQ6m@<LUcUHG<<en-G`NxXVdtJ?LNB?u2oyuipBAn?TtfU$Ex3S88
z|8%d370KS;k6bSo*NrmCyR*Y)Z`jLAOaK1;8ygb?IxKH(^!6i1j@ay-(`WcpSGR|8
zTI$U$D*o$Z*2dT9RrqK9`1?|5b^Nmbb@}@Jn)U*b_u40lY%4m($e?`ff86OAi~)=M
z{QDpJ9&q=WS)w4uQ1JZ$n}3|{2HC&q`*$%gOukpYZXKK3#^giyZoe-o5<TlbZR)J4
zThE;R>&mS#)yp*LNJr}FX;;_B+iPfa2n9G51qz;YWX!p<V`KjPefjUJ+hrwAuC|Ze
zHus6gvAZpqQN6A?uX-4!rZN5dta0^y9M{$5%O}1+t{I${_geV>pJP3e#%br~%;XO@
zF)*+wdUB#eQ2Ec_zqhw$e}8p#b<K_y=W2chhQRl2zW={C9lmc<o<sZm;fk7XE9}>&
z-SEqOHQm!bV!LxS!=2{(rg<k6`z?N)w`Az6l{tIKFX7Y`$rl=IAMW3CaOGPU<fEdh
z`t{At;_vV7&b2D#;@~haHvavqm_yMrOeE{06jNiu?{9C7X1=*$=pW`EwfT5ftb0g@
z;jgAoO8371ap!)pI)0VhiKE|yVp&6%tO#AXz3gj$by4M=9fkG(|CaBIot_$5UjF^l
z)6?hYT5m6XeeHOk?CQ9^Rr_N1E(lUl(J-n#-E}lTW6RxtYp(p5Jk@dY%FQ*=%8`#!
zj3Spg>aK~<30b;b(ah{$#*{#_#1l<_`<W7?|HQ=23ACL)WqbBnjz5)K&rdxe_Eo;>
zQx*en_O#PqTvqxvi+%VUIM3>HMc8wxpl99Tnqv3%R9;@}-ap$spHJE>=kc*#r>LJ(
zC)^2JaxcVbVZr}@f9G14+m+|-4)Bk<o4RqGveud(B6*8`Ki8gMr0UABV9~<)_HE(C
z+rRTx|Nn4jf}-=k-|zST;ZHa>$8y@VY07Rr1w}=lzI-VuC^+$7f@@Msvaq$z-eqSM
zBrk5byZcS}x1X`b8&@9d_+Qpre(Ipa=a`HqGyk{gUEaR5-a6yxyQ=<cz47I&3^TH_
z&%en{{kXNip=<m1zrl<q8CkzCzEL{7iC4Pg&w5R!KZoU}cCauwYDb^$_uIH8mi<Gc
z)q`i7TcpZV1NOMOxs|=V^z`QD^y25|bl<E#E)bL;IyY)hTZE2X*_(*TYQFPg<K$m%
z4!HhMM6+k&fgaX|^odhi>g?w|KWyWEyhZl?)z#OR`OYpeb*+7IVWGO;oB$1xbLY;f
z`^~xF$-~-qvqAT|%`uaUHJ6{xkK3BPP4;CPd%`=p`^}bK4#mqSo|?P-&Zocg)5-(f
zZY-|fZPi!X9?q;>H1lcw^Ap#F0~k(;oDDh|wAcTa_!k~tNrnkc(=(rMeqDdZ^t2aG
zu|V=h0U4P$FJAok^YgRsY_mWnFOPegS(X|M%*@RC`s?S{|NFV|{w8%ss~0I+t8STy
zbTZZi<n=6gxb6P?e-}@#&wEo`R(5Z1b@{Ot$=DY+Ha`CO`FU?oPu<_I*L`Q3y)>A_
z)KL24?RrpG`^d4mcIn%8Ro;2K>BQ#nw?^@`Y46OGbS|fyY@5hXadqKb_wzaPOtpe;
z{5`P8&Qi8rBW2$vtGnW65y}h<`ugkpWUbZI)#a_r&YV5#d;Ot6&=k>wG9nJ0LaH@2
zHE-U$QSv+A$@S?%sf8<Jvfi7>sjOFi`)j<=`*!!x-n9KYcI_;EeQn;nd8~=0OMZQQ
z&CbRawl1df(UH#L=jX1LTI>qV+wq)<;Z^(Y`uTs>pGdt{5*|I*Bxu>s39RRSF_fK|
zwdYd2VfyhUS&tU&o|nzIIra23yV_q85)v#dEG6&mM1tINMD(nTh{MXDrRU~ahcES-
z%I>c2e*ebu*9BbGX+`dx5**(qSF6|m&6%{+ecykZy&HA~E_OTKFF)Ob>(rffTA{00
z*x0VEkFWpv>8Y|?Pk>hU`=IEytGjb%UkUqMdjH(oec3O6#ALeMDt+}o%9x3vCnI8Y
zeiU2$(fY5vv%03NpBv<(8?hnb&W=J+tq=|#o}3#S7J}-^7uuUFH5!VZp8EOg*R<_h
z_LP6vZk~9&RMt<?>_q7OxF+lAldlJg{;Rg#z2)7-o35p=u2|Oo+Ojb5pj1VATAG-E
zzye9R3!Br=FIl1z>%PQzTj`e7_U|M%e|`C#ulnCdw;4g9*$s>bw9WIsRDIz+f5lvj
zpRGaX>txBdD?L4J8nVpN(o}U#+Jo%+dM7VuVhBFFC}rdAXU<E5{(ifi?>$XtW%&BI
zfPjFlSyxMB1r?7v1)g(c1f4f>{`&SUkuR;Q+xJyhgk66q;<eXGdW)WRg}{_g`4XxN
z?!P>`^)0{uxixWnf8E@i4l?L<!6t>NUj4GxW;r)D6hA+=HUIv<v$M^2^?8Wz5SOdZ
zZh!Y-)!q6EmRGALYRrH8x-z$INByru*&8kI1<o|xTz-vNpysEe)3e8(7q2%>WjGbC
z6~8X3lp%oKrsCs8P5nG}W`^K>J}VzhZ9bK<DQ<7o+Z!7fJGb*aKR@5x-28f}ON)rO
zs|`DALUQus>UVQCZcKc!?%&3K{a03VoQ~b}yuWjgf{{k5XNKGS&I3+c>L#?5ai3fm
zzMb!$wb4SSqh0&<?PIO%&7PKOYh&~5+}!Hd*VbNJ8}07m^1^CS%`w)$`CaZVuI2Lk
zRp)+Xyc2JH$M{e5?v~<(g8$A=<=nsfy2Qqpe?F!%EZ~#-czvP_{~<B+0y&lj?tOx5
zPn?LEyhuK)>~2Y3RAA7~eQA%bY~4N6TysJzdpyH}>zil$7E0Ag2@7A|S6eMFKmTAe
z`+SQ+C#|WktdyL(CN7%a%(NnO^|bRdjfFOTxb`jn(c*Qlt>!o_PA&Faze*=%)q<2A
zx42FJTVB8T`@HL#?~bx7Uw`0golzZ^$nG{xJN(@J^XqGVe%i(N{PnZ5v$tnnUiSR_
zd}}MKxZP#G*B?IFwMDes?zG7xXZ{^)xwDPa9fP@xpW9!{7W4SE`usBS`R0n3@;0wj
z|5Fm#EMq<?SHj`WI?2aMH~T)fTTWSZCvfwP$*0QC{c7;PeL}Cti1(JYG1I)yR+18P
z%X)T~ePLjjc>0l;n9zcl<v;ZpYW{!o%b(J}Z{J~m#)NkbZZl{9K4ia*qvFepiHjFM
zK0Dhy{p_r*rLV)HTIWru3DYSGVBnQ9`CwJ*bz((c&g+>!w0y2V6k*bp_6AKLI%vIl
zzvBMdl`fy-_1SHeKO8jget%}_oND=p<<;jEf`f(a<L`S-)p}_)XHHRZv31>_3NJ6O
z<9)LJbFG%XE-0G(sJ3E7y%s}h$(!g%H}i#mo=%PYD{g$K?sC;l<^(U(8GD!TGsNwz
zd}OblWG`jDvZBH^<nz_lFZ$hdp6r?U<`9F3<Lu}9dzVT4xc`3E+%B26KNlwpGTgAs
zn-l)y_na0+&#fQ-_cCm#{WIyb{!@lU&1HS}E5&`bPJL{Zwd&2)-91vrHtZ?-Z1dfL
zkKudP={KH&53+kA?(dwG*tag|M{7%qg_V_E^*2ym%`K*L0o0syKdQ>kdSQ`kx3YVm
z!_1Q8V)L7~ABCm=aZ|dswCr0flhM17kUsVmqTz-9%=vF$WNTctFaNe??c4G-pB8(c
zJ9mzek+H3<ZU3K7r=OjjT`KFhu3~p_&E6{A`l~s?E9*AwD_$H`+E%%T=?Q=T^?dU~
z^TR){x_exIFM~qA)%Jv6b!RWjy_s=w)%hyli@*MUwf*+(>#wud*KK}ickb%Fdv&j`
zn*V5X+r-N}L3ZY~Gr3PA8LH2*9ARoWl>gX%mhnM%pSSml&OW^!&*!{&&*O;oXN^~{
zJl~%azQ20=t+#ilaj*Jb@cq`!Q%Y}r&qn#}s>_NxFXfosc75g5v-|vRPrV*geg4Ll
z+?wt$_s*@4TN?2B+qZWY7CHw7U7D!u?$#%>5_CbpGKqQmaVL*FJltOW_0`h!kaY)7
zT>iD}+AhB34hfL0JO{vMwEsAJex8v>mcC=rmZUG9uVk)zRTOF;IqZ6*Xy5a5qW9(+
zC@U-9-k$&e$H&K+mzSNLY3zRe;gcT7fQg|lAKN<5`(O6VcqBB>eXn1j)BO!wT3a)x
zE&b9m<N12~B@RpCZ{0a$q8@##-m5I7=Jd_FPiDs7*1k&rznJm)^xWIqdZo?RrJS6!
z+;6Vdo_v3&u7;#5{HreRt^S^Sd)vv!i=Ay^L^#)%%K9-nR~I~MTVnn_Kj*F915tJV
zIT}~WcdQCs&Bn%dZFPA5%S%fayY&XG42iGzJLu^AJ0f|tmzkNFS=ddL1q?lB*XHEp
z<Y-0KhBI{?`VkzrCF9}@!{jw<*D^CRbAP=(d4ke{4}V#I7BaK(czAh9*;bWwv}ZR5
z#wj%i?dtQm-ZJYt<K?5F_kW41UrCN@Sz5L4j~9F8?Y1h@-UUgl+k_Y{yj(Y#wd2j*
z#{&G}Tb2I3xVU(2^!96|7WSv^-1+n8=jXjuUvKR!e*XIU`ttYpVlSxkWs7Bov!99E
zl<JbKox17LuFhcH&rO?@9Rd&9?t4}KUsgZJ?&6u0NlRZ@Es}X)0Xn4n_O{&LFPG2X
zvv=?5DL<F>MsO<bTyODg0YlyIx7(L2S+X*Cd0R!vYWqh^0zTD7FA@>e`gbw<TnBT*
zrlpb74hmllch4}m_N>Y17te9UU#7=5)f6a*K2q$j7JPKkRkM0``TMx^^z_*F$<|AP
zUVeOh+`UI)W9jQ_-`?Jqx2xIVpua#v>!n577yqg%hgXk(e-|}aduP#%e^XC;uX@q(
zd*b`bjS+7?{a$h<B=hw9o7YX^Rn^t`B@7;Xd3pKmt*zE2F9IBIt2a$}wDLwT``Mgx
zb1b>VbTlFt2JTICkF;fuZFlC{er1XIb+a;`y6+jMr|CMk@mwsm*qQ(B%}s7`J<y%r
z8<UT(3S0U$(R1O)Z*OnQ*;ci*wD8DSTsVB#Sw-ljm68)j0MGH}G_(4De>^9v)mD^!
zebyI$RjYN~Va3)NMyh&87Ds>D?Vf#YjpigyjRsb3u^qd2ySuttT3W{L_j~<j(xHbR
zA0O}Q>r+%zG)_BnV_&Vc>Vg&#nWMYpLz0hl2+lUkHClafqu>8nrfH|jp3Gh;(y1sH
z`)L0CAHU!4Pd_tb;p>7=*FHWz{`<?z!v<&0&9$~Jd$Zzo0msit(&l+3@9+J+ySqI5
z`Z`(Lsx9jdE4I`uiTK`}h1M_3^e*|7Yms{DY;NtN`S)iSCc91acVH}edFkl=`}YqV
zSWy1{Ui7w{i=`GVDX-7WG=6(~yT6BrMac^RkV_q=G&?B><lNtP_v+QJr7ND_RTf^6
zJmblnsHs)i>HBx6Pe0x({r&s>`qx%-Y92j!Q1JAWXm<AN!pFySV|D~we<(1iRo1%f
z%d4y2I%2<n{w#fc&G-640jEt4TPt*1ogO}VRP^ml<m+YoZ|>5$SZc9E;K9|H7w;Sm
z%l&yIjZ!{8JIfQiC(Nb(^E2P$eX{xY_E=_ISa5Z9_~p3UOCCh*srY!Pm3!^lwddzp
z2EQoNR9^C9fsBcBQBl$EvbTEi`(kc>>|gaZ?WL7W(1MRP@?`~AR!je``u=Wj`TMxW
z1s2mpCQX`D_3cgNvokaKWUb!Z*?IXugV=Nht7C<Cj&urd&$$`2+)p+?|9#QZQ=o+4
zu}y>huZNS9)9Y(%^Y82kTzmMB*IKm~Rx&{acdbAD?p{6Hy8K<uj}H$Y*<21=8Nw%T
zSM&SZ+pn*#T9>?-FmvY2b;?Rd!;6YO9cX0UyLa#HEt$nXJ|rqDE62LKaQvL~=tkvd
z)?>$x+1c5Zm6ZuDf9}Fs2{PNEwfbyTVQKN}Yis%CY$E0e2QVm2%(%TR*Eszg4<~0~
zNlDG09~(htUjf}vRr>15+qZZ3*VpUo>w7tHYW~n*{~NM0XlYGt?dp(~pB}t;@?yf?
zU3~2#7X9aRYpW7_o0|`J3ahIK-C_}95Yvs?Qup_ly8pZkv7{@d7E5ka{`#_V^5o!x
zf)7_#2A99N5qQR4QYa`QK-VriFfh<}mRYUj-%Vxrb5pZ+zOs@D3b5I|<=w-RGfQ7z
z`}+O6e&nVl>key1^vT)&dcA%>7faJz>+-S}7aXrYobc$;v$M1B?k=zY{q1f2|G(yW
zcPwJtojFcwzG6Q+YwOmn`{&1JZoX9hjko&ygEi|8E3zK?oPAqv-ki+r?AMPUJ9EBx
zzN_r*tJkl!#k#A%zuQ~;yDYZdxvO<^`uT17_wDTL?p?k-St!7XOZ=4M=FqiKruU1V
z_w@2Qmw)fCtqHvTP(Zf2^!0(i%EH=VtDd}m?QNXD>GBfK$=c!TBBG-D<ZO5S(3#;?
zl6rkztZnr-kxrNA=jOh?w|6%<i%G9KTJKs=P>}8J-YWF9O;$TG7Idr0-Hx5rS|#5b
zHl?19*;%tQ>1fw3zU2$5zrG4R-Y4tq?A*>LyDRIe*0j`8SwYRFX?n3%_5W(LwO3zW
z?tlB{&6id(PAAM4%Y1pQ7qjESzS`=(iC5$cFRh5tjcs@Sbo)2Q{@J&G&zNGeujHfB
z+VZ*1ot>R`cbD6jzl%BECtLpJhNE};q?jdPtJQsHJ$d*raCO+)%gg<vBO+c{DLI|^
zz98z)^{p8fAHBG^IN+p{S^4C>y%$O?RN{}?^6uHXc>ThV|95U~p3?Aa&culmr%w<6
z|L^bKy?d?8-u!rX*E-hSrK<ySnZ(t@hZmQ=zBXMib`i^z35U9iF34ow-}iT4?eCD2
z@11R8f|C}P$nu?3|EQmL@7A^GHR=|{ze?`zDg|}Fl!ZFW-rO+Ey|v}__4U?PR(6%2
zT&@dFbka97yLR*D&nG7*Loem(?d>g<6;yokJCJWy@ZH_z_xIKAwz0E1a@cj^hp6ih
zC)iB?@$24M-Mg%N9zE$4=Huhr#kV{lNh0m!B-QL|YbyWz*a&Iif`a+TJlpDLPo9)~
zeRWkcc-fnKd!zmR{jUoOI@NJJU2uOv{r`W`W;q^xwNj5B&eB?ZD=AQ|`umGx&(^)V
z(sH<+e|z5DsOt|!@<10xeEHJT-=BYf-`|^?w_K9^DYrfEZr0shr7kW<R)wyfrW<|C
zpl6-3Qp+EWQ;z%3t_WN_O*eX)$ma*gp5<v(?pb$OvC>KI`=hq@)j{9N&d;-*YgKxw
z)WWqft!|Qs%BzcukFSf}T_Ve;nNm|*dwYAnytw%G`v3npg+Zl@lgGP+=Vxaxe=Wf^
zQ_XkQ2b=fBpB(N?)=qu>MPS1g&6tUkW_?ks2!1~0#`5TGIWMoTpTF*~rq!CYYhz<$
z`(>@SRe#T07;s}vq;agf3s?Kz>hF9K1`ax6)z#IP)192UgkG`#m8<{z_4@n$|NlL_
zd3CmRUFd3&U3~3Nq*9A=PApq@^1a;J$jxaV9vs}ow|vFQz{PF>0ReAsZGC=zzJ1-F
zioVAsvVxi~rs>DOd-g1Bna|AA)AhG!Tyz4J@<qaH`G5U&>y!Dp|Nr0fYut`o{VnH=
z0QJ-Dy_S0L+<YSR|IxEYN4uB%&tLbtpleM_OUt_0-M@bSwzjs;zrQawK0f}s;KU}6
zwPF7AY<|}N{~dpJmZ|@IyWGsom9INE6nFOga@>EvjaT~D*Vo^_?JK|0IB`$Z+Sdgq
z_#aNuG~fQUufDMO*_oMB8lGt;CqKTt-2eOA+uyIO44!LMTJ`_m-gU}KtbH+h@2{`F
z|Nr0j{nK=#<M!3e6bf+aQaR<gf4+mljcvKn*Vo0yw^!Y>HEOJy!mYnm>+7|1b%p-x
zdgSfvE-rH2#kc&$<;$0EZ_U0gU;n4@?JZMv_2ud3=e@L23TilV^r)q^wY+`Z9{>4v
z>F4HX1}<s=C6XtcryT1$0yOSy&yU|-_BM{~eMLo4+q%Pw_nSdOe|)v_?^fTu6LbBc
zh&a#TKcCOr|Nr|vJ|!guRPWr~we)p|N8^V-Ka17XmoHebz;m+N+q=8Nb;Mp<DLHw>
zE&RA(r-G&B&f4GKrt8J7dh^5mYT4!zS-ziaHs{-AC4Rno_Rp^N*PEN0-4z!2r~Z8V
z;9&FL-`}razI=OgdjI6flh++q+*R`X+uOXmyY|-o{dK?o|KGpg@0ZH*DXQdOlr<NZ
zkdT;TRk|u}Z&m-_X||jZOJ5h9FyB#M@@e(#-`3^t_7p$&^KvL%6uv$#{rtSt$Zgf%
z-&KEmbFtLIC7p?h>DJcl@BjY(uKoQDH1JmFV1D;bVopxYvon&`*1J0{7cXWqe`zH%
zN%>*i=D3)<Z(h8p`0&7SV#l@0t24!Pqhj{e?992j>2N!L{I;B#>kcbgZK(hMZ(Zze
zBOXvR+tvPhVI||_QMVwfCa>!6uhPwFXXR|GLN;GN_UUWg)l!Qm-g0xJKfSn<#LnJc
z{QR8f(xBLOX8~E+y9=G$H*DA-sO<LV_xt@<bYpAQL~Km@_U5LTm>4K&hOV9lD)^rW
zpK>gY3<(LjoOX6myvy}>yVv^YhR3!$Z~FG-(X9``_1D%$U%!8Uf6B>8yZG8Ojvhb0
zd)Kac_VxFU9!<TyEmx=$RNn1b8n$}5-`rh^huN&HcZ164$VhPG;@`q~Z82%Nxo@Ap
z&Q8lKOTWLNod4V_E15~H&#!N5o&W3dojWy8PE4HAAf}<}HRZ#{k1sDTKi?^=K4<RS
z<mBX1S;3ReT&+n58W^R`ax(7i`MK14x(1`_k{t`{F4zc)iCx>{+I?inl=*EHv%>P$
zy)N)*3*XLn=cfIkQ>VNXr)!IJZP;KC^>5dLhn01wrf6!1t=X_~qh;Bf8C$n*U8k(H
zQ(sSSo&S8h>H6{SuCBg*ZmxAEubNYr;w$#E!r#7qvr9kcapX|+?fTCCUF%*KbcCI`
z8=T1QHdQ<P-JzeKXPf8u_4mu0<;+;utTaDK;u+|^!;ML<6%`er6n%f++SeT(det8v
zxz0Ar)r;R}Q}^cwG=+S0vCN3eys)5g`}FAn2kzdurTp=#Y;3#p2fIVxzuh~y)NAIi
z-#<gwL~Ja6ey*9F|J%opH+K{+UgxYFIm0CL(!Sc?Ny*8#w`M<o|9<`J0*`f^+}zn$
zS6yvp=Qo)xa^Z=wQ<v_lqtA8a<>#N9Yi;~!60cj8_t{l1t>!$^es^-Jbo4H1ucObI
znO<F4x!7{Xgb4}xppv=#_qVs8(GD}GNlP0V7{bG^KY5Z;Uj9A4?x*YZhZE#W|NZ$H
zU-wg0RTUKLwZFfCW~c;W7x6EsTphc+?DzNg&x4X+m8GAd=>JF0&c5Ayw5rK?x&LzY
z1q?mCz2^D%=1iGll5#@emqD1p)~u_qK0Q61dTNTKS<a0vQEgCmR%GXwD=95~T6ypG
z?b$*BP8`<x_iSR12OPP&B5-lh(^Kr<^Xkj*@Ag@n=d^L*I%Tamq4@i|A3oFyIK9_z
zZq~ImH`V*YRt7yiJKMaSPgW~((~{=qW<MpB__Vydd%MfuA8zNDw=7c8(_7~|+l=S%
z3oDIJXF*dDg^!O-)eJs%{rdL8$H#W@wF@k&4(UFx{_fqo#csVuChOzsmrS3xWy`w5
zntCbk<zma<A6R-s&bn+(@N&P%`O1^8u8aMBb#?gmyt|t=Zan#%!_#k1{r`LC&gtpt
z`K=5AUCw@UQ|mN7U6+b^^UZQ^?J9kplbd^fj-@lVtbmir3bFbAZhbO4Ykq!OGNawx
ztUUPS<5F3_jS8^N=JxXU_udEvFa)hEd3|kd(axF+3mj#RKTU5@X$N&?XUv$vD{c1W
z)z#NGHYV%o=@k|h?&9lu(&4mF!lK~8uF}`D%yO%qo)YzP;8c9MLTrBhER#&81J9o{
z?obaub?v2MLT2+Pn<=xtsQsuvr&)M@Ti)GYS0|>6YK8px@UVIL^5u<<2cLhfS`=OV
z{ax&a1jp@pcc<ya-rAh*e>VTr>C>lg-n@C^M#S}t6ZL<7c<9_OXM1jr<>kHA-;<B`
zNlHqB3js!_tv~*lRepLRsvR~XEcff@2h2>0YrCeaPdT~q`rWz0_wPjSE_*w}FuCZH
z$j+F!?Rj^X`_Bhmn0D`;-Fjse%P*gvdRJFhPxKHG6H7}=TV}bV=I1B-e?Jz->@0Hc
zlUexoDEFEdFJ7#R-F<I&dH$IhhO5KZFEgFNp*VkmJ=f2-_v`=58Ya2qq&U2I=a}WU
z^mRd(!c?V?$ELmO%zvjFx5uKLQ-h(?WmE0%vZ-FC85b0$cGc;#9^H_5_|c<BckbT3
zEF8cPyv*n3=JfNs%irJHmRtSrPo->|`zFwwU)b8HySvNtLCv)5>z1-i5%^^JYxe?X
zX|tRc*Vo^_{rg_|xBdrLvtrwWmlT|uGW&Mb@%=k2i=Ul&d3kyL<zi{mtS`sq>+kF+
zOuo3twNgaxr`o%B@0R<|mov!-xV|pdzVgNO_<GyoXFQ-B+04%WO2J~%-fnUIZMnD2
z)YQ_RpPPGmneXLNi<SvYU0EM@Eeu*&_4QThmluKEY|PnZ@xIw-pKxu}`ueQT?)~Av
zYCbbIWL;hLK10#ebZgz;U;J`599&$#em<XHZ0+!g7gVb(_md3{zHD9oZi+^r(&^&7
z`};uENcj3VP<<(^=5yjXhvzgtS*s_{o-JFxy!^+9hud;*zXH`y2VMUz*rh1K_5J<*
z`|DzNPd{iaH*e0u*9BcnacA$|v;I=M<Nk$(&f)7~e%AKBc=<B-{yy0Q3mlt+f`f}q
z9X|11$@==>VDse3leg#HofXT(wV>?nEk{R3)2u5qEQ_CASsDCNc<Z^fvAfG29B5p&
zY}u@J0iceR<^>J*tCF_1dzIb$OtP=(y!zC3!kD{6*6-o5&wu7J?>~CS_TqB?`M$Hw
z{(9{H<Zu7?%k%m5ZLO_m=USVKE34d3`T6PTbiG(9>#{YICtsd^e#^FP`)Ym`U0&vU
zeQmUP!2^d@r-}B0ll(u}*w_C0l6iUA%^NpBGdydf!$IwBE~`~XUq?7OH7#D8?BU_D
zm9IWD%QW!%Ly@rWeLhu%rQO}#rx!Z6yDXBA{o=JW$kM{%&aTqemzH{OsSNtZ&CR_!
zet+H3F46pZdsePrKY#M_6Hh<=`}<qos$@mv=Ct46-|I(iN|8J6VK-^kEUTIy1^)i$
zkM&B=w<uf`vNC8_Uxz>tM*!cd>Wv#W>PBtZQT=`0+Sf&|j~y<R^?N8>_v-v|@#|KW
zmUFF2y>!maeY)=5y}i-v<MwiiYTemYs@<L`a8lH3>88ZPZRPLo?5z1&wAih;=%35h
zDBHwCEnBm%-@17dH2hNY^HZg`%1IqNyMNQ;>vk4AY+Akg^}W5-FaJ7moK#$O^!2=z
zD_`y`e!ePX?XL$nrhvQQ%O_Np<ej|Soh@yiSM%n^#yegPj91n~eqO)-9~VFWe9Pjr
zyse%^kJj)12fD=J%gf9AH5pkI1URm(iQHWB@>2Epcc9AQ-QC@lB8n&9JUtB>_uO0X
z@KFB#zhy5jD1r(yOLx#UW0<X$U<;eQ8`SP(huz&(%FM*H&!G6}?-f^HrJbF{%F4>h
z!m=gfqEflEdgQLEuUVIu`DR~TRr>bU*7tJ38>_#+JJ`%FEg>OeQLv!$^Ru1OiYMRv
z`}=#{x^)*XUhEdrUF4tLG9lm9GJ{b_Q1IfK%Fo;1--|Xax@>In`<HOn_9r*g_1C{&
z6}b3VE4R4UbiKLa(>eJ0*GF&9+gbF~$=R8gm-q7{A(dD!FR#11O6S{Fe|sDqI@h{<
zUG(<6S&m(mUv4QF&CI#H%-7S?bGCWDoJqz6Q1eX0$wt%T-JPAuNm;92i@PoiTv_z=
z#fuj<e|s~oNy?gLh0Kd<{a|CCcgKRkd%E7)sn_!E?TOr$6Dhmjq1$R@NY$?|FJpez
z3W8fTx&@COKUP*&E|qmvJYe9{D`onr>W}34&wYJ;jEszSc6M9tCEok^`Exd?Oj+i8
ze!l(u=-*aqs;aH+?U(28*b}@l$#oW!KgZ9!Da}HkME&R4tc=~gt+Vr_xY-*;dnG%&
zePL^(Hg4RwaN)wxRUx2}Hvy+l%xAmRpJ;1^uG&)b^V9S5^Zn;o1YX}L;w!A?Q}OT5
z&reTJYci^~OnB7%%aQZsL_S%oIhMuGetdkar>D28Z>fi6)t8Lzd3UW!UOagFcJ1p3
z(0TUqr$9=@_2X{bzWw_3>-&4FuTOc_`*hOw{QK9|L<TPnYHe+e1Pv0%IB_WcY|r}G
z@}y(w(x>(R|9=1V^|fL0u@_d;79~G9F%i@o+ny(@x}ZfM=uUvWhT{DLSFe8k@$oS`
zzucRfo0qRk*8BqM-e+7`5VyBV)}p}S`bGgKj+6Ny|MTA4QyIKD&9^P_-Mzipd7$Pe
z*T(1P=U-nH>Kz|%ucOm*?Ym;jgrttFk1aLLMyaRPL~nn0cXzq6d*78(omPqSpPk$J
zdi(pofB3L~Wr~1Pm-;D3&YxzrzrMV@yZiga#qQ;AZ!LWt!L_>d^|iUy<#{(YB%V8W
zZdYHcfKx|afbP5r`QQHi{XNSxTT7&CclrBEDYNoL1qB1Q=fxHmf4;pvUm8^Vf<j=I
z<KG1>HJgK$dhOi3yZ*<AgP^JRjUq-e$9ZKe3?L1<DGfqS9)+M=749FtdbRYyfyR6H
z?#ao??dn@<5V@-)Ga%r?kxt?0Z8<B!QSd_O>Lbpde)4uT8@7XnRd*CW_iN=6Es^z8
z6p)g-wIT8Fs?gPKEiE1@LZD(=(PjDy`6(0LhipnYd3LtBx|$lFtko2|)3u>lS+5!x
znXhDh?US_zZEzBBveZz#zrObOH)Xe;51&6rZ_k^%E?M)%vokY`pP!o>xjF6X>TrK>
zEL^l*byWD1^Q1|WvaYTY6&4njl8V}z6&l+f>=3)NXldo=XZHX96elDs2++7vYSAL#
zRMP*;k@KfwTU%T7_B>xtPf3%E2~lgWm0GY)^ih+xC|EFW-aVKz7CBmKDB7<IT^;uL
zSnuwlr>DRTRBuO6ZEaKa<;DB`|JMaBZd;eE*dh=V0E&&636mxTZO^-#etzEF9fgO_
zo%6f?P(;Db&hG8)?fF+%O`Shq{;&b4ZRzCU2r_)9vVp;d$jxacPYgGIdAd69{=V3a
zNv`0~TEeg}%0_YL;=;$rf|vR1to!@x(b4WwS#L+fv@<haym&EHJ3NTVi$n2=@D+Y9
zk9C0>B41x!<>u$#U-mW%JTBGOS6Nl{?bX%Q#m~=y)}NH>v<NtH$$+wruX)~`j4LZ9
z&Yc@uP_SX%ym{*mUz};4fA7Za+n;UZ#l*t4<wQapaRBUym(S<d&$BFk_V)Jn|G!@A
z#~xqgeYl;!{`<ST$9knfIagqk>MM3tmFb#OPr3C<-Q8FF`_xozUTL!{r508jQcq7y
zJlyv6?d|gK@9u(=t_Pzl=;{uA9i26Cd#k2uhwrQWoK|>S`tQrj%eQ7<H#0Kg;^sCs
zG=!8MTpFv63Qzj|=Iz_(XJ>zZaIo3_|DWLN8=q*;d*03?Icdg>9jT|qK=naO4dd5l
zp-Ji%ec89RWEK>B`1$$yY}4#zuM0k%Tot-n&3Beb)|C~!(q=EL#GE)3S*4yXXn8WV
zonQXnm&^WQVqyyyE(}^3a{b|xoxi`o|NrOn`KPC+>#O~b104d?1S&wLzPh^l|KIQT
z`{itBS(UC@mwd8VYiibm1C1+JuDrcHe>o@zp9l}+^YW0JWsvC9Ev9RhbEDwh9n09`
zleYi+_xt_p>+Ahz8ZFJdybKZ!iVj;VoIYK9e7yhlm6gd!NkU>`X*oG-UKf0F@$m3?
ze5^M&H@CUD`DoHcP&ELm>g_cYD_0jiJ#~7TZuP%EKaX?@m&$t2n{fImGdmy4hUDXX
z;DC4{0P@<-z{PG`v#;m9zP5I`-`q>3Iy>K=xBtJV{QbQ<J3)i)tHIe=L@K0PUBw<W
z<8bfZy#ognbac*KxdIA?S={1!Tk`JiN<7>)T`x8%G7L2E#i3~e%Eo^ZGBbB(U0oHv
zKCV~RI_&z!C$p9;S#q$M-CJeS+uPfN!AWy=0H2qKoup00hVJgu_x4uv%UUgY9Z__A
ziRa{bHkF%7UIs}^OUE8};!s>N0hG{wKL7gqx@q>clt?ojoilep<3xW1^!4@A)6><}
zmv2r#Z^X0sbpeNBi--~E7{m47-roNHdj0;g*Vo>DczAeM-_zfF>;K=och62$_362}
z)?r~`ki7QL85GP{_4og|v^spfvU?wBxb*tQn*A3SyYH|2TlMwTRZ!W-@zV~Jm+Gp&
zzni;vZ*5jqRz$>%)>hYFAJfmz+qrwU{~U{r52n0SR#M{O<vl%Bd-~6Wi!V;8wg@bW
z2J6#~-L+-+?&_GBIkRU+|8nH#@9z@T){EQo<9_}BtBE-?Pd@!Lzy9CNLx&FCy=(jT
z!RhJx$9p7`^YZ5XR8VXY__S7oUDbs>CN`FxN8-WTw{IUl{P^kV={{L&K6$$s<IT$-
zywRzDd1>jAB`OSy-Fjc$*!cMLbbZDhJ9i#Ec5K(~-Lq%Td}-z2#G%+KarF`B#a@1S
zyBoJ}OG`=ZtNUwZXJ_}<;;_N{dwaDxUf=MaZT5D4{XftD+fq(WvaR}ZV^8Jh1C7kv
z^Y5=ad;k=@j92)*I`Wn*TXyT#ExVc@28xP`p9|+b|NG}piD+nA`nfqL_q+(IsrmEl
z>}*~ho@ZxgFW<cYq)%{4v(O^udGqF(W?lL4^(*U#O5H^YQywQLB{?}e8y{cx)5_9v
zXW`>xadC0GxO)YhJOUjoH57kpnVEgN9$)|V>gw>An3!qm20X>z-rTI|dsRH^kA=6P
zq2al6=MJ~?`}_EuIX?kpk19C&tx8^O$iHuwa`o@ZgP;FAKHfjywtCwB!rE8dl^fKG
z`1$)+g|6OL@li>>S*b-Js05T4_Z5|uv3<CDR<|^+&%XHCneg+Sot>4xzvbGoICCg2
zd9hGt%7hxz@^^QVkM~X02z+#UdF?`xdHixV6$>IeUvi(Cu3!K0(b4&JzclNeK!zRx
z8Cv)H($dp!Z*P}2&->F_d~TlY>cgd%YiepXReM}+e*NlI)$ea_)6UL%njFv~;M65@
z%8~OTqqg>HZgD-I`F655uROiKx4QiKxw$p{-mkfv9b26q&N9vJle3Ky2s2P^5eTXX
z<n!vt<7ho}v|D_>UG1)rl|hm77OYlW=(W^nH>=m`(A8oxGB#yzZq)E~3OIRObhgw`
z+;wGN?eE*$^S5VRef7D$#8kyZLqo&T((<#Qum9Txj?JJJ;&MOPxh}38ic2~c%1jZE
z)ec`bXZ`y3dn!Lmw4IEZ&d<RSpf&ZU{Q2jt6}7dX#hrP#xA|&KeO7M+@`eh?8@7!O
z2IcQ!;^X6si$8y9K6&z_qod<b`JVoM|7RgTmd~%-Rr)&Y|G&SV?Lp^?xJo=-&>|wT
zHhTNM;^%(5%idn<e^PJx?{GW+!Gi~F&Szh`z$vW8!pa)DI!t%|VTBffpeKe_G5=d9
zyxCLtH)>(P3oEX>5B{IOSId6n&;LS(`r3Ev*6o_owOGU{^|Px;<W1L(HBR5;ZvM%M
zuYSMxyI5CGO6!NMlTyq*R+_BawQKkJ_x;}9ny>A8pFf`e{_gX8^Upsoz5e~v8(Ta1
zX?OQje%_RNdW!%3{q^flZQi|mcfYiGUq?x4Y52OBo4ZQ0L+9;vFf=g<2@ik1i;JN_
ziCa>HYtG%><@_=h6JBdA*zx+#&d)QA(|4?6VpwoOs34ZXq3r#=y~>M@{o3_Rwd*UF
z#LlLv8`n%&e~Ht0Qb^^FkgQcpPd<KHXnt`~?)mn9M@Pq)a7KoJDT>WbjHY)NZQ5D3
z@!Wy~3#BI-#!P2m2so1HE5yLSz_3EfNMlgYvl?DAG1M_HK5MLb^vUwnpeZ&+|MI7Q
z4xVvIv~t$ZjOik({XNgGXRW%sXJyo%xBAuV|If|Z@>6Z;KI_s+N*|52e=b%R3KiLw
zvozROZfi~dqkcIF1_mzilmLw*EZ_3f{H}h!9Q5y<&xbpU>vzwauOwx#wuJZYo~Vx!
zjZIUR+D_bG^*7uram&|fIhv8v=0EP{%B(rH`}sxoYo5+G?pt-2t=`nlp7H8KXUEjN
zor+FutPC3GCO9tC@Zjb3srVPAcll!GU7uJFouwXmao%02S9`UyuNyu7+&T4cT-`5j
zp=rGS30c#l?3TJ@+6ihZ&9AroK4D9%%aLVEIC@rf{9Czvq570o+dsd&-0vUjQT!GC
zzR|a1<}0D=NfkR8&;E3A`NWd{z2!UStF*0KUjJXbF4Sr9s;Lk6f8Cci^V`>R%kNwG
zX6*T%>iT(_lC9!N?tAB%85lx9$!V|4`ls{x|GK*z=`f8p3szT}q`vdLZ%0a}#x9rY
z$-I9*o5roLdUJXG_WM&_B>A~?tzbC$m3wA<&+YKr1?Nk?9{I3iZn^i387u!kje2t3
zU-f&A-X*1JLX&O$Ra`fDUe`bJ?#i{fJA?Lj@j1Uz{mtvW!#I4}_lZRlKLi#1Qk@yu
z{iUUTLk<Ij#*z(coLr`i-toy(7^ie-T>j(Lo4?WcUs8_6>B~QE3%(0qTE|`b=lws0
zl=;VO`8;d>%(*e)ii_x!N&jAQ8rj*JDVDuf-T7%_Sf%ge`+s{6a$M?*arm#rz`zjd
zXs;oXWYq5*Z=${Y*;So~KLWLc=lq*;f5Fscg_3PSDT@4)<@TMv6SUSfz4T@9$+_i!
zqSl(Mo|Y)2tRfd~<RAUyYSyaAojj9g&G@9{Fwdv!!S|WRr&K;On|1Yc(5jvwp(Xw6
zPbqEYv&fFl*t$;5wzzHWRj<`ql1Fb_cSfF{$C|v*`h->CR}oG@L(!t_sh=fXlN>Go
zelg4OnbE}aqe@CLdeWgMiXV?;d_LPFskA7i^1|;s?8k$PYdvo}ZLZX9oMts!V&aox
z*Ljs+g_Om%Z3`1#shQ-ol<}d<&Fd|i9TLUTfnPuMNS%zUQ2*JUUN<#<^-8x?;dxQ3
z9=(44<h1#dNdMK>7H!`3W!{s#rmELrhbF!1T5ouAU*!3JhvyqHFo>|6SP(E{qrPz8
z{xh4WK5DF<^C__PTdLaE>z6$1m(JGt@^1P6wNq!88@#v~boq#jf{JT~Y45i*E77e}
zba||{{jf+%;}SG{G|5=}Md-!Er<z<r&p&rAow8yk^CM>w7Z;!0=6d&+66rFtV%V1c
zsIHCL=YB9__4dLGxk;+4B4%l*Ufi;9#X3#FjaPo1UgCIir|8`qj{7rhTd(iCG9hN#
zyC-#SS&y<F{qg^m_VO=t{L*cyIX#!QFZm_U9a}eRqg;Z~8bjwJL6@d`HVf^0QX>~v
z+MT~&Eh+V)WMPO&%Dx-NbpC4IUhIFT@#)3s@dARED<*v0aE(c+a?zCaswzr9t@|JB
zWH;i|S}No$_VV_OWwvi`Xuf>5?t|>~&W<B%Pn~(OdZ~JfW~;|s#iVL$*K?BUoPi&*
zwqDA;8xwNY$JZlgPOqPQa&P|od8ID}H|G_-IUKPlrvB7^yKV*shJQ>wN1ZnYF53I?
z&%aq#9}Rau+g~jE{i&ylAZO3oQ-7@+By`VesWB;Tbg_ACrK)A**)Okt)T6WG;;NXe
zifARF63M0EuI_S0Z~q?ncd)t5WxuD&q~)qcan6U=UM}q^O<8E({QG;|b1Rq1g0HjQ
z_B`iP2vq*NuI!emx3NmPM{LM-6Vt$(>yM|K7`ZKx+!~bitn^Cu)zH<U?l-p8-;Dbj
z;dN>&^QZRokeDd3S0BA@8hpKZuwU9nXuJQP3hlM(7ALbSUwpdka^%-K{@q`ubqM*d
z_c(DWRXcfV<m8KC?=6<yX{b82_S1pco|CquMI~SB$~_YlU)|pG=5ONBn<AEmOIE)A
zWPf*7O_I)$b!&f45!#zq@qy#|_t)!Bbb3siw6n(TxYPY(CZ;w9-~MT*9=VqBYK3Z2
zhv4*CrV-x$8vjc^JOBUpKKA*s<0mBRR$f}{X5f;wb@r4Msms(t<PJuxm^|gruhOn5
z9UTjWs*?X-6*08`rOC|T;Ml~&b=0-<qhFSb=kw`xd;c}v==gC}d-78kmmW{s-V4Es
z>zq|Oj|gnMbfLqu`B1n%@B9s(TIxzdQ?xu?W|{VOcBFK=xa2s!5i6f|W$JzTi$DHJ
zIVC+^IRF24Pd}F&r$4VG;}1V|Ibxohoggb<cS!Zq(v}W?mrvsC&hC8wAJwW~d%Z#>
zSYCa9Qt;#dzS`!Qt3EEtbUXLf)c9+DoqoUZHbWoVI;WrSSf)yAR2wd}_Sd@K=eBp*
z{beax+1JfdZ}J<aGa1RxSt#_&SZr$Su9b)8y?r`IYWdrD?*+I02#MnEzPD7WBl6~K
zf9`FkRhCqg{rG%z`zdq9tczWzt_yQ@dYn6UR9NtLLi(4}Pu@R!d`dO$wbSx*-S-TB
zFFqcvl(b@rTk|HDs+y-wOI!4H1UKqWwo?-jd@22@a|!!%_I~R%XWn<sc%Jx|y)Mc?
zJLkro7K_xIOTYXw`+d+qGPuQZ1p@;s$FWBXD;}8YB(8FGiIROjHGi?pRMAgAU0isk
zURtqg@|SqeROg?&tvgToIey#h<K?t=i?-6q{c6IQAAR5UroXI-+p1}L>G$i$T_-Pc
zC23`ZUsYH7dHc6yZEen>>)ia_d>I}tQN6R=V(rvw|4!676XDfm_ri0HQr-JqTXtRg
zK4EX?jhD_!rCy$ir$V%E?%EiYcl!9}@?#PIGkUFl^@L3oy0*)(JjwInbO}w-X`(?d
zt!}C9`60kM_5G6IX`XxE|GnE1Ww|(i(aNc1I|II5UOORE+W7p5knQVEt1CICKF+P4
zBF`5;BW!gQtFhWIUhg9lmi_zEyXWxjeQ|YD3SQ3I^>X$Z^#tMgddm{^w<+<-Q~6n{
zs$33wzE?6mvqklLo{-?wb;?&ll^6A7TV}4(l-IChUo!RVzMebplxN*&QVcTQsu%dH
z-TrC&#+^0G7#J30omdcHGh+tFk0U`|m;ScD)V>%QsihpC%(yfyF5v&)UnNRPUb}wx
z<X)Jg7_|D*mOIPXIPbOZUlZr<qLQ~@UT{Q8{+;RHvNfIv?~f_96P!Kc_$AGouiHEK
zh3~W7JxO9^?8U8a=VZ&5el9)c(jVPr#b@&<dwZ66@pFmDFV{4+Rg~IHyi4+A_dlEz
zW-i?p^ytp|6nj2Zw?2tevp2b}+0(KwQ8v&wZce@1^tBh3hzOokjz7-sW|TQUOv^iq
zM^L*vK`8HN`|cN0pBn#}RHYYO5@p|iXM@2n$7`9ae|kzkb{%gtdR~8t@37htT{&rY
zmuIqG`g2#uhN<tZEt^^IbG~NM65TB=N2VBso_@T%hlh(HAV8tn>Enbc*XGWAv)S-k
z{5<F8JJ;uFZ~VT!Mee%J^D7gU_Y`hp-YRYK(;_X;D{s1FlJk_iD~^aiN#~jIaCNHf
z-LpG(%~#o}<8kV`Wb?VB?zdh~d-nP4tW}o{x8~_yYMxW|_L$AH<~-Tjt{qb+9M)uE
z5{~=x>FCGT)23WdZ1Z~IA2G|Opr~rTw)S*E!O1d0o0FsaT5INRYuI12%P&(=@Z;%g
z8o%E!Z2A0JssH}Tr)$$IA0G+Pu&iA)&owjbpK8|Wnr%;i1c}+tneN~+<&0MS5+h4(
zg=g&csSFGn1_tsXUEFOQilx1`#BRnL<S)JZZ2p{YCw5i;?E3V1wY|?h3GrFmPRzTw
z(ICdi=<hezQ}>R)*HpUq=Tz65mgvCM#d{Z5?K^iu?a0qPDoSl$FTcsJNnf~_<0FUV
zk0~7<8BtY1^}I@*=a0%vx83M+r0&P2g!P+>+VXR@q$O>O7wbt}`u=~*)i;-7eM6e^
z{}^UX%AV04|4aJ%JBw8*e{J%8ypCMa+g;i@&o$HaN%Mrp<VWxJ{8)KgQ}dAWi|=)(
z7#J29omddCCu5uR!cePVbEV+jZ?+s<I%~%N40%hziQgWus!g^II<j`ff8O64zB;Wc
zTH3u&(Ed}}B%>)=GrP_h)h?6}Hq}*9I(pqhDe2|;($i1h_qeEh{t~p#`O>!BrEZ@r
z`VVi?HV^KQ2)e#1^OV>9<5hbW%6IIL*mhh0?Tsqlo@L)p{83U~5pZp7@-y8m+h^0?
z-f~@><^H(-dRFI*DaosyV~)GBYjr<dJ$uUOj9E_)pO~@qK=wVqU=Nom6}q`^18@0;
z)&F@|TmAUcdd^e4`#%M<F)&<pVryNbqWL(n=$hOXkM%7cmV$;MUV92APMgUm`0<SO
zlt}Z~=J`UkA1aRR`}Ah-G@C$ArKHKJdsc;=s&j9Bzi)%G|K+$e<wx(d!pl~_{`O|_
zzUfilx4m-FIePrfR|TcUg^YrNCpU>6%c$L5@~8ay*()o094r|_{t2h6C<XmE%64qY
zRLRdTUdvi)<{jPV^ZNKy)|k^p2bb^|eT$yRHUH}T_jNwjbywG={<@;9)Y)HuYuc`J
zsqMFoU5<3PIzN-XJ^SSS*dBh*M9<*ezN)D!H+uh@s*(R`O67@I@9@93wMv&&rKZ+v
z&DMJQa;fhbQHfcCf|=%5j4r92%CyT+d$rekh498Ns#UkI?EytEIO7E>UJ85vXw^zr
zmm?F>P99v>bV=1HE>$?G?_Lm_wp+x;GiTO{mdUR8u)i+KN@ed(?RQg~R$h%;;p%ck
zKY5<nk(hl^x0Y)3o-y$@<~b?pn0ojnbA`*1jOi<~om{&58;urCaqiyV=GEzukQQH7
z5#Hu;p{!@&UP(dEbNSO;HXVuRSS2AK=o}^G`)Zl^8RPt=_qR;?B)Naj(y+6eEcY4L
zZFb8#b@Z+(@6<${_r`U~h5J{_T=4&3vE<IIf`9hYI+iYp==WJYQ6hKY<jrBXmWMlC
zKia8(DQw@Q9jSevR{6Uuxth0WN4@m3lkdLh&bp>2`dx0tj`X!Rj`d7!uZyd5*tySp
z^=p;6zfwCpR^Iel;i{vmYR{*#tZA0VPQ6uI(jIT&zLFhg9A+mlBP7)0(k~a&S$yVk
zo3}o4GtafP6Et+{F1pP(eg5l7aqADP$)5SpFYD=Pj_NZ8S<#&xD$C1NJs;Ko2|oAn
zF)IUupMlQrVBvKP|90KnoUR|S;lZ3z+v427y6=MJ#*z2_{(jlXxjfx%-bsnPXImvg
z*2QNZ-0o)@_RsX)n{6+Q`&neuXINbI%0G7L`ii^!U)Z)hOTYE$s=nv?g%-vAF|U5_
zeptNkZf(2Z>m9zAR!=UpFAe*p`s~508?W;{-z03>Q~vB$#LM6P&dZJ0b@xQg3JTkF
z;Id+Of2vLT7Vm$%9nV{wHu~`+`Ty@nAHTV$pXs@17O{PA%ZJnx7j|ZDoVZfq-YysK
zqVLVOqs(egFH==otMhxKh`eQHMAV$8Hzo2UmwsH!W%kbV!;QmM#>-!YUOfFJ)21V0
zn~|&2lHBAic~hhW`{x&~xWT+Sd7YHRTM5-OR~FBFQ5ynUy7lr*{{NUgZrXeE%~yKW
zzLl-?zkOln!cS>_@#TMie?4k{|Nnx>pVx03ex0PZKfvzn+vPs?ix0nLKg1=<%*6C5
zkBfm}KZ~RY*PQs13w|EY;Lf`CfU|Q_a`0c~`d|O++;moCo)FI3FrPWd`B8Dz)hX&K
zW|m4<<5j<FoIfrp`PuE&8}65@cYM*^6XnYqFTcDd{{Q)Td-k->+Myz-X~%LxX|sH9
z_`N25y^|Y)ZXP~vf0g;9@p@&AgD0BjpHPl`8f+@dd$YT`$n%)I^vZw751FyA2-ThW
z`TR%o`co$sFfi21uRF_flxMa=b=iCKvi=FS-9L_o|8J|AGm&?f`q^EsAK#?u89Ua0
zdU8-dxT&}{RPayG#NVs!7jRTfn)G$$m)&MP0lmHzGe0S8KUMdBIS<IdXU$9wieH*l
zV;C3$BovyN92Op&qQCOb+@so=pAK*R#J~`6AyG$;fs4IYP@8vc&y@bk)U~JHZ}sA}
z5!>~{WP2zh!-4gVY^)836e~X;;a5MEa&wBt!xdr-3_oH|EMN%8&``23j%8wC_#-)k
zp9?hVv#O1UzO>qdT?`F#zE*$O(&f?pT5G|Ks@XF_dS6R1Fm#!v1ZWh@GR+QKHN`Q@
z>y`Gjhr6aZc`gi4QBz&HdgaR1tBYg36PK=De0g7O^}5vz3>v2-rlsy&$H?$crM~`u
z)Yh!C^K7f{?Wv5p&cwhl!T1b2E69hT&TOMJXsCqTa1bttWnlONTG{sZ`~CWt{>jP7
zTeoiAyJyd#Lx(Pzzk2cF%pA+$`1twVOpFW+o021@J>A8|@IyB$D$3K-v(jvL)z?*P
z*RJJy#mK;LLipGtMy|bnbFH4<Y&-wlJnv2f>rn;<h9~Ss8V(DeKYBDNT{=BIotKwa
zRD_v<VS@aL1q`imzrVfpo~HBjF=z@q&{LOzf#GKp4_Cm8l`B_TT3Y^H;J7kqY1G!N
zrJJf47#Kda@Nfl){Mj*kbNcytOpJO>jG(^FwWG`;tX!>!Zrrf2v9URx|K;uN?Hd(1
z85p=kzBDt5u-eFhHiX@04`@p~HC;bHH}|Y(C<6n-PDeIY4W%NPzVyg#{`2j44zJiJ
z$H34ae;~rf;bLN1+P7zCXXnJfTJAsp+N#j*)sq+)7>Ziz0vZ+_zIoHKm?2eSnOm<E
z#{ve100zZoCK1`2ZI7oie~b}jW?=ZhEYZ!b5%lW%`u{I3F8=cV>+S9NzP`RYbvYRr
z9#p?zXVnP$_2uPao&`LIO|q_VFp4lRaH*vPG$<_%UmtfW^n=c6D{JfGSUv^@hjRvE
z`~g=?^6&jQ)+_BIA}1HO%tvzFX;64G=G|xi5N%^?3tCeVps_<wr8z`PRY|VjJolE#
zy6lsO?%b*Q^yK7@Yz+nmfnSb{b;oy=zCPA3->)C|@8<2>^44WO-rn7>ofsGzl20sP
z*x!_TQ!{wknuv{yyryck^UIr=nLXRJs7cS%wDi;zP4_;Tja#;S`S3v_c9+R@QAP%a
z=@T3o>!Q58PBkzx^T}GJWM)==dg95<%)Cxp%jDUanVXZ2cAc84U0qojx-Mqs>mmk*
z2ICV880^*c^v+$p`0>@%)n#R6zrVe`eC5iMU5lL3USC@)tnQ~1x@yYu<>}3AygT&R
z85njn@^F0+wyXUGnx9ziKc7#=qTuW-(;B<^ue-RGZcabHYuBz%KmSZt_urO(e;(Kp
z<{fq#4t1<-Y*CSswza=Xo}HQb`ucizhQFzNe0*|pa<5*!lC>_ovpJpr`cno52KEV#
zjGvA|mT@K~CeAj?on=>Rb^WPG)wOHa!q>-{nwZEKBpf)@%Dofhpb*~Vj|`Q?<>l|M
zt`_e<etUnt{jt9P3;gHXm4bJX)gS2)ls3zG@bN>qA%jC<<NY5Dl{Yq}b}wGMc-yvb
zN4v!Z9scV$J0CthUEe(WTF&QZXYcMR_4e|5v`dMBAwV!Cph147W%08PL1nFw6%XFN
zjon{YyY94RQ%On5ox68;m%p!jc4lVS+9+3d_u^PV28I<(#~v{jsZBoF#v{2Y{rtT8
ze?OmJSs9$PamMSeE~9Cw%l+r?tNUBU&CRW^zn)7}s{o|w52(hloZz8ycX#>zijPUN
z&2stVY&3*AcdT0x5wXAS@0QHVr>5yve|&V*s`OO>C?)=JJ+Xk{rmKs~jhi=ZYkpif
zdNlR%v0kRehSyzP6NT0NcqENnG(_gv)$R&i9Tqd4nSr66`&lzn%Ew<{v$L~bxAV*U
z%`{4VeQhnr0<CL%tINyF%i9ufZO^ZNaX}H3o*VYFN{T2P{`>p;+O=!%@2}rq_V$)#
z@w0+hKf|kQA~&C&u0LHb_SVgtl|Mf{g&5Cz>=EOoimECrOG{0WuD#XY%ii3W_`0ZT
z_k{}=&ds%EXJhM=HqX1dDs)GQ27`lObNp)t7Z>aDcO?Y{0$i<fa&jCCSQ|HP{5VxR
ze9hXm*Vo0~ULCH#GUQaLD+9v^b;V|;3x9roX6Kj7Nl7XB`YKdgTYKH<Mw6<ls^{nD
zzrVBd^T)@>LHk9&z7Aif&BV}P&zAf#V9m*=pC&51=iT16w(RY#<$iOGOiZ5aQaa(c
zFu-S)N#^ToYfZDR?5Oy-XyZmhP$oJc-@wClG(0FMXnow?bMtI(FLLcZ+|K{{#fySi
z!6#WqyF`nhozV<mH)s0v@LnlXaF(bOH_{LZ4h#(Z_U+sEcXwZ3Tzq_6?(HjAuIyOH
z@u}d&g@uC3ZXGU*Hm9FA%f8kF$`TK@C)jC-xWB)@fBUv=`^w+n`|<H{`P*Bq3{e_i
zUS0+zm!h3}d}o^-@0WkRONN2rhwh050dH)ozMMF9>ebiR*H2H=jowk9c-SB&oaN`$
zRiWDY`ueuEd*|6!uZ!N^2e#pQqK;hG^wZPzt*xw<Em;EEL3r`v#Mea+!uQnw-&g$n
z-1_+a^Gq@?ZAv};A)JN5A+e<{K*Nns#v&m-{rL0G=jYjWGelK{u8FvK`}XZimx_*d
ziPrx9=IiVG6qGN1WS&?M5Tg^jYfJw9eMuXCe0qA?cedHnT{1s(+1S{`^<rLJSa|qg
zGyB^&Zyr5<ymK7~14A93k%mYuS8LLl8HSga`L2%JyQ{ySf8Cp4lZ>n^Ioqm|2L~F%
z*TrPs+VXO_|NI?bElkgvodl=Pne*n+qoPkwJf)>)FLv*rrV}~ob=L#_=q(u+54Cb@
zhpc$;^l55h;=xi&28IdxCl&-uIeO%X1$dd}>g?<5tjphtaJ9ySv(z7*XIp)&Pj>a{
z)u6?{+1c6PbiM}^Qfnqmn4so6>&~vy>?0k5zrVe8X3#wp6C2wvXFF@kl$7l3*YDn)
z16yF85}+|_o^5qnYU<ZVN4r4>{2c8PwUPT5uK&oY<VC^9N3OZKx$f@Ab)&a|GwKt@
z{|3&2PaZ!`PEHmU6x>+++|S(HeBEhAQSk18?d7dpqHaABhQ-f(!0wShSQDU;)hBCx
zZF_!vY3bL)?fkcI-71RZTM^~r(z1B*<Hhd%F|o1R^Y7~kaDc<9sG%-EBk0<i$jh74
z{cCIgdQaC=Y!LHL+IZvi>D9io&8ok>d3j~!WEZ7}ptw81Xrv(`sB_wHmdVMpXV0EF
zbLRH#-F|beV!~PG2zI)t`^_mREZkY~G3oQOv&u?JU>i0$v$ZZ#P*pv8^JeAKQ&V}R
z&GuA%&1&Tm-La0tLQQ~UTi#u()KgR1`Q@XcqxaYSo%OnlfkA=$4122+XnThDbiM2A
zVmEK!et&oQ`en=1u8TS{f>vsqo0~H;GyD7ddwY9>{ciGtz4g%B$H)84%*?K@i<LIX
zxbX0ByP$*OY(?ib8M_(_Ma6}#-D30R%>x%XnG+lr3RHi5)H-$Q)y?VW)6UP^n|IgB
zZvOdF*8qXiS65CRKJ2U^Vr60R=GNA3P_}5`kQCuse|vkrdET8LkB|39Z_Al!RjPIU
zsY7F0YHI1%SE0(viyfQUbR#x2fYdX5c3fCsRr{;t{k^?EYyKT<X1~2H_jIXcfJM!(
zFDtXJuRA}_R@$oM#WLU77Yg|p4k&(Uc6zuu^YXI1ySrY#cu`SVso3E3Pf|)MZdb`l
z*KV=f+j5IbN_-%-OT#13tl>;a<Fp<r(^FTjT)BOFcii5pnCZ-|J$-%WW*VngeR)y%
z?aj?;y3uZ~u7$CJ3=W2^8!I$SQjBgsIN1FC-CfXH;NazcQ}tqJz3y7DAt|!#*_oNy
z*VY&s8}BZ88FaKu6qNTC+;C)TJ*23pxUl;BySTWxx3{+5-e14pL*>yfC5<m;d3P*I
zUxjRpsCjs(wf^7Fc96%rKvTIacJ=>q($l}+umAt`^>xrWc;M<~L5D@@t1Al}o3C8G
z>Oa@&>M~zxP~>U2O>kTY-c-7~?Crh1)#kakwnT1DTls8BP^ZhZmn+*mzAP5K6137~
z+RB{{&MIfFH)VU4yr?&?Y2}rblFEE?$4RdE{i^rVLu5~1^F4a#%U-*E6$SP_+Z~^~
zJ^fTX&0hPqy0eQ5(?pLM|6KH@s-)+Ayt`bnzwZBoxr-eCdRf-HeVVbHf8Oz>Sz+7P
ztzSPW^|g8aJ<$437p0qftG9P|cWe1^GBCKj@NZ%QIq%!IZ-$13udlC{w<uV!%1`^_
zocl@B+>b5FY)^gH{YLBgC5xiq=4-PfDt~>xR@}FyaMjPp!Yy}iUC#T#_4Kji@mo))
zISMoB^Uc~Yp`&Wuw;5NLy}28Enr*$+s+yBkzXbJ+GQRn@b+D{WyBBv`yEgAg_IKC2
z&vq~0|7ydBBa6<3PVIkMxG>!B&Ac72H1@xmGFx5i<&%=88}91Ac%n|P=-u;g#lQGU
z_WPjH@agI4yGveP+Ma)Z!v=$xY%vB0(+zrZT<bnRKM&dmt~61`vdE>CaY~0v?DTu}
zMZb4&PrfEt{;odVxj0km_Y<!whPAhn|MjZ>t7HpM(n`<V|M-4lrI~O>mbUzqi{gTu
ztGAci?nrHuG;~?==;wx@&X$fT^Us`AUH9^~_`F@eErJ<TmAd7)=f~c^Kk1?1#W~K~
z_px4(SbsLJRBHSD9j`P54OOoezkD>seos|vht=8K<^Rp^x#?wHS@H1k<KVSXTQe^&
z11&IH7_j2Q69$H(B1Re_0oG-2UfkKadD*gOt=!^o-o6cuXNcb(^?G9Ol>VDn<}S}~
ze}2WQ=<|nFU(}CXUG=>DM?}P}RmtqtYW?~P+*GaK7ccLdyW3<%ou<+yL!*_+mp4Vm
zRsStKnSb6iqe|o5?cnY{Ww9H}O7Ga!{<UnEP4Qf)qy1+4wT)#b^Jib-&HlQp<Z)B9
z^H=%#Ctn}nPC9cdD*T7>(&y1Ti=M8F*?DP+=j5-iuYZ4WQQ6Wm@+d0<gNXTw1pxwQ
z=USJan4oy<*s*)}?(s^SJz2$5D_r>N@!Qqs-RrMAh?S+^`su@bS@XW{)IW@obyA;?
zE_-&>x7SEDdTy8Q)w)MI`~APw`97cdr%Ki6s=_OStX-cjZu~#@@0YuBjMM%0D3xss
zo_uEam2H_ROV+Pn|9A0y%Z>l`$*pO+zarb#ko|It*EF5VcXxKKi`_lXrgD?-Y%@bc
z!&U1U85aC-U~6r1)tk;QX_RtrPi5(=D?7{H&YC$>vi;~1FQLx#Oivy`O{VKHJRN7R
zJvew%_Q>Wx<yT9d{;2R<B7YyWnx2)FwbkXw(;H5)3;aIKXjsla-;TH9yJ6J7*ya}#
z7EIW&hk3P%l8VBEmzNtFCUmsC?-%%D*w~?RaUrAsd^=vQiFNz;+vuriJa~C|IisWE
zq%TTK7FyrkC7bY6XXz2IB|@IJ>ZHVSmz#+Q3C`a2*=3tt(#_wGPp|v1sxLol)@i5J
zF(>E8eN|EV?Z;a@M{5<6)53!L`)ZZldUlk)4x6mz>*VBQp~b?m;KkutDjE(#!ot<x
z-c<hkQyCq7+qs=@szzYb>n@e1X>;ASu6$>et1qZ~yK3v_sN+35`#Jxyo3nEK&dZ2$
zHJV(sW1maNfvG!f)nk&4)x<wZ^WNH9;bObd?OIw~@b0tUeq8gpko9C{s<IZBWM;{W
zzt+>w&y9FqXkU71Ve~r5$G7fJ|5^9{&pU5hqqhF&{z&Wk2<^Z-zq!-*>c!rDQyU+a
z`{mKWtb4-Y@&C%EX->cX&3pf;|F@&tW!m&lStzVOn)SUk*N4^Jy(I3fx~kI8yQ#U-
zy2^_rkH7n}GOlSw%iRkPz5iYly4ZJj`kdKcyyx$e{QBS%XV}H!h}rY!U6}dj+nyqY
zdz((3fA@?1W5pg(?XW4@;d!8LM_8D+e%zYS)ne^m85l%DU$D0}Jvu#IUpIQ2P5r-`
z^>Luye<%Z!hM=J5im2Cmr#feETD^IFd&-kN+o$)(B#Rc$k$N5U==<;5x((Ya`#)Zv
zktVHo=j-?H^+k_&a37w2|Mt!GUzhw}w3+4FSL5_JgKWE$68qHl)o)%c(+lo9Hu1@)
zUzMxN)_mJB-^=Ay)16<Dohk39N0i2uT`YUudi&o${(ike)kWH`kMrKDntx*nySI7O
z{pm9WKPMgj(wQBst{402S&FO6rq!pv#r$4i<FMg6SNp?mnPvIkYajjp@ay5hXIV==
z6c!hJ?Y{PE)6ZL0;(~%7rF-{3s6PMwSnoNpOnY(W)qTwpc7iA0>+O~jnE$WxuEmzA
z6}r1j^!{(2U-IVkmc}2q-0u9(ee>+uw&LgKIyyL{rKRQN<-yfe(*(zb4i(?u#XdgP
z`}fbEJ9qAsy}7Y)*|J-j{X3JBgIT1~YL7*0&p&5XW1TxAO0FvBo8`W%Q^b}rMZPNE
z?Rh_6Zo{OGw%mC)54p_l`0?xNvbdW$ZakVbY`XUID>d${y5D8J{93=XUF!F{$)~3N
z*tGipy!u(U3eSBDZtvfECY$xr=fki0Hhk**oo;G)QB~;L3yVW%V|qK7%;SG9iOqet
z*_}JbG*0kzzR9OQf4?%ndCPt_(&ki5Z%32eF=4@@zx3yC>5nfx^?%OV(jO`&D@-fS
zRVXQayZW{LpwmLXonC7@J1$<kc3tr0i`VJ90#+qHKk9eNz5Cm5jr$Ydd~S}schYIP
z+*Q||?a$_%oFC^S-4Xt@>xcY{tE;cSyu4hL%XN3z+df(AW1t4FE3c78!TbIH|83s1
zsp{{quU(?r|NefDZ~J-k$EWg>9a-g)Yvz2^yu1Hlo{6_h;CFqQgr|$!wp>;GRkteb
z&}(b;2-c5LTh>LZ)hY)syZGVVESD*-9*U}*e8IK#?G2UOn)uzfUMOFl!vA;P(pVlZ
z?fzFv`E&NUxoFS%%^M<hA*D1xOK{_+=JktHTry?1>)yT{CZc5@v3Aq3nqOt5%U6h(
zuiMB~`C!w&%-{T5isIJYd3s_ozvSd#Rm(ZMgI>QoCHLsc=iTpWrU+jA?ETKgWUu$$
z`TKU&9F*udf9%?|Z<m&O|N8aIw)$Jl<}_Z=*n^1Xo&HKk!}6DxRNdW=U%7JS@9*#B
zZ*NU~T~u^T;>zp`+kbWZJ>OZo_4r@U$&6|9Zf`TlYPK@fnw|W6!}EWZxzDcWnzs0@
zQEffHnZIzW*3$g*GYnS6Ze7WJ*#5w&{5_TGOJ^OL(>KZW(Da7zqDPT_@fXi)tbM=Z
z{Q<|Ul`B_H@k%v1{y65^oA&)xjxN2w{A4d2u`r$5u_FI|W7d>eUM^W|m1|lqt+<uG
zXY27-w~y?*Rq-ycdtux1>9G$(WpCbzu|9ujb0FW5&#P{Uw>17Z^XkgV$8X=>-C3M&
zZod7(g$qwlPZ!^^e<A||*UF}^ujRUC|NZ@)U&bOpMQE;7>8WGK+FlpcHRRvDJ@uQv
z+M?ar{`Xgf=C7Q;<+&02;rTXQ>nzq6rtO>O>TUXInz2-6X@eH0M5^^>HeIP#&u8=&
zy*hBFhGC(a;L)jTl9H3>t~X8#-R)h!`Rcy)9lUo+{`ky1r5@pWzVWTs^S$wAnam$$
zt8a-+X#8<TD`Z8%*H>5H-QB%DX6L8J$NQ_ly$KBe|1i{#kwHP;K#YG*<erMc{dIp&
zP0=iVdTMHutV3r9%hTDq8?LJ;g_RZUc>P~uU+nj1Q$O8sfB5*n;_cK)S2{W>riGq*
zR`_+{vZtq>+~xFgap`nfvP1rZ-tW@I8hR!VVnhD*IqK`%|NnO=P?3?*GDcHN``Chu
zGxlBW?3h}8ZqCkm6}oS7KmCf>w_K(}J?rKY2Hqp-X$c7rUcSuC&#(XS;o-Npw~rrv
zS~F9Zf#HOGO2C5#r-c%>RbP4}ji-F~SQEF`>bmGtpJ`4#dB2PUXV(c|njg{oK3MmA
zs=26Cgr<DVF7X+a)gciEf`)rORqec+HMzI4Qg>>{s&LJzm78}d^m9(rX3wqDKfC9S
zUE2QD&h_)(ua}?ybAcqkXk+%Zs(mN_B)yq@Shnh)zRu6vhQ)i$HTG|uoPA7jclp}I
zpD#OK-KZ~Lv4Jb@>c_9#HymoxX3d(_&Mz-#l5t^c_Vrn2xxYR>b_b1+Xxy3L=y-L~
z<jK3s-|s7Yd~BKT?6`e3H&2}Cc->W`%)9G<z&``4zlXeUncmW#K1VqDSysII-nlma
zqSKdGMYUX7k+L|aGX3kl^um`H4etFukf-#@Wc~J;2Ffc|avL3b@Vr`_Z$nYs)sJ^C
zr_EaCdj5ahlKVo&hBhI7UvwR_W|Wq$baPqrCgJ%+>-)z~c6dmAuls++%sc-2&0A~T
zj@3wiee+RI$7)`OmU-stnyM17=WDOuJJs2-BtKs^>(QPIRfpz8DY>cnDk=T^T5jl~
z($C3ts;=0t*UN14zPr1MgTAxb^PIk)bz)M+Q~&jI6_su?SqjXToPR!kd*0nQH#g6>
ztIY!U-?_m7ab$v{<I<$0q^PY~rN6#py1ToVmzT%J-re*4VVlX1dpm3I1(}szPV9ev
z|DE)b>lbqc1Xo)3ZWCJG>XCWp_vYW%H06Ka+4WT`YA0Xp`u}ac(~e9ybo^-U=hYUH
zCm-+GekSgfdgyDD;-KfNTwILIno>JooOt(`QGW4~pp&<w+oO2ny!VJ%o9)!P9Py_8
zx}%P_i=h0WDW9!`o0$JJrMOJ_@N#oYN5}m$rE`kT?)!6ZR$uRHD=VvO`F9Fex_{c>
z%l+!5!KW1syRQdzyg1e!p85Frq!auLi<GJ=e+I6YB06RBO`9EUpQe6ImU(sM)vc{*
z%<T`mD?;7Ry$_$>;T5{Z#bwQ#2cMSCp3bxJ?atp9%fClxUEFtHLMiEr+qRYe7R#?N
z(Ry)mtB6uii>v#xcu9qyZ?DeP?>KR9Rgzu3*_~NdrLU&xM%&f>xv``0@$K#Tpiy#%
z#G@N>_`a7j{_t+&lidY6Fk!KqZtSit57z1X@y*G)d06@K-`<YvkABSkdewzz<3jJO
z+u_#L>o>1t6Kt(qpJ86+zin@{zx}3TJJkd|mFK;U+FJJb=kAyKUuW-44&h(@#qYp_
z>UYxH&xFRc#;pw8ANkqstAUuIOVInJe~-PZpZ8Db@O!K5f**g@mK)8_ez<b?+_PIJ
zJO7#1BVDDWkt=F_e$nkytjV`S%snb9udTY>)1%C>OV%cP|J}NRUu$PeIBqZg{wz#H
z?Vjz1tPhhGztpw%jd*!&|J;b1OC^6Nt@AxEyr!w+iq+XKzqU`eXkT~k#MP})?0>3D
zb{oY#S19^_<D2*1%}2uP7Jn2k=hOT6XG7@u)}C7@UiZ6+eW<%|MlW{PmA%!V)YZ|!
zVfTNzzAGyO!-)k8g63M4TGjvC<1*=Z(%VHYTAkffRfIWR*k<Z~_*ct(NkH<&lN(C$
zPYN#vo-|TwY-`+iPG81Vud_R7+QRGw%1_d(b(OTHc|_}+E7hE8V&$YI^!iD$V(7JG
z_V!xQDJDjO|G$XN-EnofzkGT5_a#eIczJvGL^Crugt3;hGKhpSIez@L-%sO`V0YaX
zF1zz=KM!BOc-~g=S^9tZDF?+nkM7j-3KDwNU2j*vO!oGbm_OdTi+I8_m>7P@&LDAa
zGi=YgNz5PIJyeBmmP@lT@h<$JAG<n6?)%5LXQLfUt!_(lODc77ecJt7Ytxd<X0@xQ
z|GZL6ei!@o@1#}9H+Q|%j|+J@{bceyy9&AKt0M32+GnTtj*lTAMzNXck51?FFV8mb
znOt1aYPfl8{=I$K?P0w-{{q%czIVs8{)kq&lG41ViPhH0ih`at%X#CarX1p!qi-`O
z?~k{kp5li`3plwuS3J+^kIdEjGc)P4`nQXncFos}vXs}`uUdcVGZTYDW7EcphWV<m
zcD+0Jc5}8)r=rrMNIjd9Z--VouTJaVm=a$-|Gr^r!1lK0vYJW%Cl+&R-L_ai`&w}2
zy)X8XTYD>AT=oP-*$X=?otfvadckM6W}xI>8$G-K%TLP3?R9dQ`)r%GOZhVvh6a1S
zzY7@lJ8hq|wR!D_edR`nr<iRwFcmaXd$#p<S@E6s*XmYo52?#P`=+*Z8~=tsZ$E$N
zJ+r!-HNT@T?UqJS#GA;+hr@Fpd;RPOb&Dg*`1k+4FYt8j1QV^BmGd|MaJnV7Y-!!@
z6Z0qjxd!b6uio-ml!1Xu>Ps`zAF*k^OTL_3_~+7Ib=^?AFT4LLz2l#&WNE5;TWp_4
zPTbu)9_sh&olo6c|I%i1{CbyVdBOW`T-md0<-{)alw-esS8d>`TfFY;ksr(w3=1L-
z+cPuNPdcz-k?rBn38G!>`F7rsvWmjrKZaUJ&XQi)%Cfg}VrKZdIJ?{r(Q@LS)J)VO
z)h=9E9dOfSiY<?qQ<L2hmD46)PD!4)Qe-jN#Oj4}QR-(M#UQ=TiqDERGflpDO%sxS
zx%{l>)V6nXuDoT?eje<##>LA;De#u|MMX!UyIyLMo7b4UyWD<3LWsk|ck`NwoqpG5
zuDz(}82`iVncJ;NFD~#eylBL0%UwKqjqeiq`?Wu2sa)0x^wJR%%<M3lmId#h)<3`Y
zje#M6MWLDLk7rIugurPb!S#MSGwyHp_Iq#tKsLiPW!l%{PgmPc>Rj=4j&SyzRr~gq
zo=E($sON5lZrE=9oa9xjYo>j8_p7Ac@Op%+U;2bL7MmR<@BX~{`}{}f@;f#c-$&2r
z{(f(3#hX=|Bl{|o5~Yg|UKE@g{@b(Dqpm#iw%h5Bj8ET$r!Ia!$9m(r$91okIB$>1
zp8jW>f6w)4Ple6roUPec_y3*b^`(sp@BX%Kj+?$N@7q58)n-*66HMQCuGZI<=7{p$
zxo2B^rjpX6+AqK3cOR5nvuI(ivwgn3(x!P83BMb5WghpctB$i2$d7y)ZWqJApy6a7
z#{c2A(Z4C(-*4Fb6y4}@Wkb@BNruM!73{MFi_3pEm3m%FJ0fuLc*~Q7^`d--#VvjA
zzx#dU|HH3`o6lbLc=-6)gID5bg#r)V<_??d7MuKPy<Pe3`}hAZ{GK?yP4e#oPW`>d
zzQt^;`fsU}bZTY&>x3Rlt8Gtjgr--Yx%c<N{rmq<x{9~?bIs?pHLFz$oPPHDwVfRu
z9rrW-u->oUXRenOAG0y*cvk=Kl`c#4686bj7d<}^CTu<@*Ea34ZGC<He|7iiJA^Lo
zo4Iar`F(Y>mj^HP-us@Kc4W^DgW5;8u5@}&-(?voIPuM^O}5wVH17X9IaAqY#;@Ib
z<ljeaf4%YhyJ<ZASB&lE-k6@r{BdjVhwJ(b4u$PJTpwnuEX;ROvTL*1`_%6G#+O@!
zJ6(=!pS3CD-kZz){o7M2)_17RTl#y~*W-Kp*T!$HTe-I8=aSk#tAhVM&slp|eWByp
zgT=>$1vjo=XV0V7WoebkCdeDT-QlLLmdJG}!=jX5C%5Rz_}N?67KKGCEy_QAr}=z-
z)PuP(YwzFs|8;|ikl_6tmqoOdl0+TjRn`Qo2woSIt(3H7>(^2jt6#_NK6`6X`Xj<y
z?ez8kZ-1^`{Wqv0)cs$6gt4HY;C?sv?Pr^_1uo8=%DiWh?{mBAEeFCLi*EemrhNW>
zy#0ZDbr<FwU3}(hhsKL!8^ORs?a|+ZJ6)J~?F2cGOk12DW$))PUst=n$}m?!PWurf
z1H)cHhW+k^_t&<^{xYs}{{8XpoLx7~FDNQG?rfWub^Xthwafk1c(^>u(!Cwly=(Vh
zsnrV(q}H<knxdklWN&Z(bf3FitwiL{*RK|YJldL(7^(MohU&i4IX!91)D5SX&sA8o
zPF8x!`Q2+CT4ZZJK0L#8onDfWl$VC_TAlMM-~H#RDcxRo)+;^dNwhWJLfOZiFDAXu
zwhj({dTrX<Ty3tY?N_z3&n#bXWsBrGsatF=OFk6J8kB$T+UV=UdnTGMS|o2qnE}rc
zW(Ef}(2RTb&6TUeRA;6r@lIT~Pvi41nR(mHCa*30v0B{a%8K__7=3$!gF8EZeUn|`
zwsNJ*lC%%+PO2(BioT|+erjLxI-x`I`zHyncU{Xh^V-jjp0}|vb&qUS+W(Z>fBk*7
z>65A3%+<L(USB_co^G)`HNflVKfS<H*VNv!hgxrZyz|l1lZkWvqIe?>$_u8OZM}T2
zqex+1_*FUPj|>bBAh&fsygf5a;5GMWX-QTmU!^YX$fL)O{rVjqFPGfe(d+xOQ~M6L
zgsxK1j}{k??3r0l>i6de72jL4_|3}{)t{Q3`KEQpy<WC-{(diT@$TErxANzkKEDvU
z{8s8E_D9y=wj6Y@VPFt=*39&0na*|gv)iXnpI=}9^}>XPh6qc+iApc(Dr9EgPd*uu
zT2u4mQSrCwDi;ec{gHFCG<pB{x&FzQq28yne(snxDMsmHmU-##tIn(6?)NpD``qSF
zS&L4t!n_r!!k|!Q5H!+YP#1x8V)m$hJNb~$_WyU)w@-iDoi}1y+HmRG(bJnsUA2}c
za<Q<vxRm4^TPHrr=VYkdt|P0SPMjXUdHZ><*aPZHmke^V^i`B@ny#N7`6&O+@9dQ>
zDtf;Uzw<6tQu3;Jc&Xw~wdK^6chg>O{#~8+U0o?DzV?SwXYJ1e&7mJw^{wx3tYBbJ
zP)Z4Ch+pdAnb6shxP@isu~l0brvLoqGIjQrP&*5&rJt{5=Rb9ExwL&dpJ`!ShHau>
z`?hbVye)ENKdz5=iTa;)q&-1NsrCHf?}4Xuv$A*uPp;qpLnGDm@S4XbUb!4O<@fE4
z`Stg)Q|4&Ad%W4yZ!RBKTj|=^NgXMdRFo#g*NZ#5e44Rf!FiX=*VF%bzqS>PDwp?l
zS+ZmQ`#Y1rWF^o4@h0Ud&;2}y-(Hfz<ySWEf4JdAg_2U^fr!Gp-&6N|YT&KviA`AT
z=%O;&es5V@M~<NYKLf*kP}O9!)8<X%btZdF!Hq@!`S%qv{U)m>-?!U;P^&}Z%hA;@
z@_AXF=1dW{QCftpOJJDs$A0@Y7Z<MijceyzleMnRInA@1GkVI~@GqHx<#z-*kFwbC
zDk+5t%7`f{E@G>Fu5o;ysJ=hnn=5<#&d%z+{q5ezRgZpcW}e!8#~ZzK5qPBi`l;-f
z?lm02eEXzL3U1o0G8BxBe6#86EpN-fH@jaO$BGGGeDgQ>Y)g-g)`NFjMU-}Zd=z@i
zg~{_tdw`|j-aUUbLQYNUQC)PjHtdi1L?4$=D;B6<59+wSDC?$vc=<jBrRvpP^>5r6
z7#Lb*4sZH7uZ-=(ZL88(B_AIhy>TOA9rsV!p!GX_)#shq)z%K_ei-Sd@qdfTF#lNa
zOf7j^j7(axkBiHs-Rxn!aYZJ7m--2Z?fROp?)Q4ulzX-tbRLJ!dpg1H&s+9)Q8#vd
zd3|wV;L7vLyZl4q*y2wnKJ0XzyZY4Pd28q2sc(<JHTUVy-k5Fr5_d0nSg#Fy)4kKo
z)_((+QqXIY(rM>*-ehOK)PI0u*W6v^*L_Y1c+RhQ6Kh=h_iz63=jKH>?aE7>h2OSJ
zPhY$1?1??64{rW;`z`zO`)4`@ORS$;ziI0Jn;Itf<<)JseRqEu|IeQABJ*mo;$pK;
zzZOq)T`7MBa{h^@r>B(E{(T}m3=3{Jv9V6o5|aFKeuDR-MJgv<mVi19XX>_BO!8W!
zI_XKuB{PB79I6*DE?GbSa>=<hDuov(rOb3tIJIzNjF6zH_Yz&P=R0?JNQukN)>%BK
ze=>iN=Ny+wI?9dfBH}$%rn|7TLpmAR&-WHTY~<l$_+Y)kj-f%iW@2B&lxi7~t}m@G
zfBG+(zyF`}#J;+9?N7RoJgQ;yTG&^2&+)7K@9vV8nXevnA6fS7(R~X!1_lvFQhOEf
zm6flXnN~E!D0!aS=c{l*faB-=*g5~q&wu`Sws=27T)Q4`OZ}Gv3=9nQVn!MV?yK$J
zx$Jz0?D^uFvx^qi-v0L1>gINnQzi4y$6I;6IQM_&TD$*E9X~>s-%x61VsJRvwy~mN
zfA9U0KNok4Z`+zu;o{PBbIx3M{TPew<)@9NPkW{R*XG)J`P<j39=z?pUpnQV%-m&v
zmz^!zq@=WL$*=iuV_6v(emtJw$Qa^yd&$0B?^30tiZs*LVdjR0Yx9%MbN~Ley=)~c
zsQIoxs!Hu$x!-~7hU??*Zz{WKAEe8`aNvF$57z=Iqo?%|W*=`mD?KuQaWDL)z1Hrs
z*Uc?|&lWFPwoKOX-)wd!28JJ}4dg`{qHEUPiJN%5bq{B+=QK`}FIpjHf~k`6)9Y%!
zKg^!wv)M&+V}#2RMWM9HNDr0%-(QSOg4C34lZ%CGd(W)*T4dkWS`X6aHAN>qOXXD2
zgBJ_T<4=W}U!UNzr&3r?IAe-NYKqdeb78Yol%5=(E+NVJ#cS&Sf1wNv4K|9+Oa+>j
zkG8U}R#0k-kMvhgj=Th&4#_y4z42CUqI7U9&L-%~+?ki2)EE2z7u3GAz31j5%QMSt
z1Sgu5{GYVbyZ^rGyrnZA9n?;$6g#K>-Pc%nX$y<Z$DK8WFTW+bm{i?*!Ru|d<o!E?
z+V4#1r?*VduH0W=_kF3H<mPbbyv<zupDP&{94@wRtYDa$B%$jVb#00}%g-|#=dZeR
zFUyVnaenN~gK^q2ZTdws_TBy7`Ty|icJ|q;CmuR}_TW|hS*l(3mc=dX!S&O>KaPD}
zX8Y~l-ANB8R!y^43qJq%cKwkqKj*Zy@uqhtNhsYisCr&)Dd>6r{L5Wme#EX`ID4MW
z-P&9C%R7ThkDhMcTYhiK>BY7E@*!*iO1CzoIIrHfc$x2ug*8=W9~)e!+Y1Puyr;kM
z)t)zt-d^M0{`arET<jw+m+))-(p$d%yZ`qJ1H*z7j%=&}nqlvDCjLGboBf=BMdbBs
zg<CcWm%P}|KFuoBH(T)a{+dg6byqe27jL`vaPox(*B)3t?(Db`eqEj|7)#r3LaHTu
zvRUo>t4AMKZOeJR=h*`lrKD8#_*x$|uZb=i9UX=jo(l_3T@iPCPDjxEebYbiR-cQk
zE-FiZ`mUHSHr~3l<@AA@+xwS&Jt{Z5=#}tAoRb`!M@2#y7=B1fbh8KO9@F`^^ZBZ5
z{i)|o+VXbUZHs&+IWtiIT}-m5$c{Dd9p+DRae4Cm`SO~VCl>PyM@=(VP-^Xe-LP|Z
zcu#IpR>Hla8;c4KpGg<E@$F6A{+>@8H?&0jR4|J3S`xdpR{!&j2kUxQ3kZ5HzF$90
zKv4Nqk*<1Mde~*B_@5Em+Ya{~WM_Y0aqiUoeZ3s@RllSJ*A~1hKOEHG(ed}Zz6WTC
z^1<sBCyyolv14Rt_;YB#B!kw`4`<TW$1k(6y_|SWQfbmVoBx-VX4?MknYrNM`Z$5W
z#ruNUO$_~fRh3@dE?*$EbeYh^S&cpkm+Atx3R}<9j+-ZQ?d6iI6IhGc%QJ(mUw?h8
zs-heew>a+biZ0v3!QZwCeSH08M$Ibrz&pP$XKz1ytIlk1ZCa-G_x-Vt_odC&+w#~_
z^WFY>ahVTZem19HYX6d4sTu#qx?n<7R+iS%AjwG&zjHD$Jdyu*mLWiMns3ka4O@5l
z#TT8=zV%Zj?A=SNEsL&qPJC%QlL;|e|5$L%vG)<0a}5gr%PQ^KT^siM?dE4PyIzIt
z_R&*6W9Fu}vpVPUgP+xvV&{JA-4(kRbFH|sT27Sv%4InY$)`ef;8}qG_hRQVFf6!n
zxF&!hR9HZ;)9Re{y}1tsCn^<8{`;r<(wf}3xVnFTRbM+ZF)eyp)akPN?v=OU{A;(R
zU)s0-^PKkkeG_W0|EV_o&2A=bQ=fmePW#=QIdSJb!YA~rXeyX0DaCc)QoGgZ0v^aW
zt6se0LPL4rPta)!3cN-d4DKzW%PZFe+4)$SW!@|Ne@0)6>H3<lHIpm7VMFCw2bO;P
z@x4eZc=y>$>EFMfdFStPWZ$n3s+Sw8rg7J$`F!=^+qe6Qzn9D2T_)jXfx*f9vggM_
z2I-4b_s1reJY8`1{nDBG%jGxUzq)Y$`Q>}=889$7u%!fmi;;udH4UVmMpv(sURP67
z7PsD(f4@NBQ?=Zw(S3d|_k6t-Z7DeM%iFcizH`K*{AUDvyPpL&<!l56Ii<fy>^>0I
z+~F}zkn5?r-O(E9#_6s#0e)<u;vZ5!W?Fs?b`cSrX!GghWram=K09`LOf&RUfA**`
z*5yiarM&RO%JAt~E=NxBmC5b=Tsx<yV@cur@BX=)zm(2hdpGOPJ9lOVhNGND8Vy`0
z4$q6Z`&?I5DePV4L$3yxd16ARZ};7>vC!SS_Qm~25B4ThM$Z1C`}(-Wm6cbwW}ju_
zJ1qWjRo|TZ>(?1&>%H6jXn#kF(j<3Q+nv*#_K1aQzY_9PTBOw3G4*bR;G6|EhmK!w
zJjFi$kR7+l<!h3;viIt4?a7RNedC;!oP*1z-T(U}1IkalJDzz|tLB&AP00_2URpm(
zs!ndWTBn&cH?1!#xDh<cEotbYvVZ%s<j1@E`ue^~m&d%CXp?ifsmf}GA{PULePiOy
z`+HqY9sZTRy7KbT(e8~K@B3=#oKjhS+-&~SxpOLo4{x`Ay8VjJiZi+M%lNj$zn#9Q
z#&hlEN4vXo<z*vGD^6bg@@=l#nv&P|{m!UQs=xInrF`a8_Ps$%<g@<H{J+(}_s*M)
z?S>krwbMJT{iR}Gn}2k0Q%`?w$zFZ*t>m6xn?7aszr4*J?d@i{W3SS*3f*m0I+~^F
zh1-ALoUZY8#*WNC4QVg8J9_bbj5f-9(YavD%jGuRdD`MVxxsfIY`0r@=p<6>OsaA_
z8$-hy(DXxBD%)SzBF~s9Dl<DwN_a~)-Q%0rv1F1(r%I$7Q|!il2FvdjEBAKy`KnD^
zx6fn7HxsSMW!Va1A74J*_`h`a`*WbCr0wb+6|0#Z&M6C&Pt|29RmPNrZ0hk?khoy|
z=f9_qvKVPF9C&|V>r4hNjeQ>OI;Xyxy>Xh@XYf6|-eJXJ{c9&bUD0@*7~vXX^s)5F
zqdWcQ&R6j;Ff`0>`?-K&!d!;>IZO-(;z129uTL|6xF{)2s_FZYx6+1z;edSWJ3a;l
zuR7DKo1$;&+KPU6FW->#qwILi^f2MXt=0?-e?(6#U~pIxnE9{y_vg)d!W@DQj0_L*
z9obkJlpb;KkFMfpVEFJo!48z28CFPv7RR^8Xn9V1x_qI7bI1cLDOLY<v6VmcyrQG`
zm$YW_N=F8KIWErsuKlIii#sc1u5V>za1aEI%Dfheo44_mnYz91T6MXbb_<@}U-w-7
zgmXDxVZ`4BeZlQN9?R)3`52!Zv38S{lhUSnbu5x13=IFwQUV(67aV^Pxoz6jk}HpF
zulLwJnC@kB+H$tr)eDJs?JV<d?R*n8A!f<B($2^e`Fij7-O^8ex6=3SS<$VNXZ?S_
zX-`S>xvBD%`xL{zwEL@0J<+mg{;sLQ3=AU7puT(dr8xQPzm|4%c>Hpk?LWm!WmRV6
z{#g@N#_g^<UFQ4j-(!Jeg}srgEV9-!PJDQq6>>o#X_wyql#fPxLQ?NGhl$_c@X5YP
z?WN^*wfSuEl2N;#$lK~LFo;w|OlJ9D9VFqny)2V)N{5jzuW!A^iW{jDUKBd*OY2?b
z<`i+jH0<D}&P-RXudA=G(-WJz=0?!l^Yb)rWdALBpJMCiIGcl;f#Hs`y~cs{m$$vB
zD49RiVtv}xUvHxSd+O@xy}Qk}u&-}Ihlf_@v{S4uE;SSTJZ4^5<lMCH!|B=5D_!?g
z3(M(W)(Bh_Qz;y{O+>w)yHZ$cvW{@Zi_jHwo?hM2|K3TMMf8kskzCMpZ`G43ozu1E
zA3g4+vP7ZrMf>Zekik|LCdSNZI@zE8_C9(#ssCVN{G-XNOuds*PfNaBKG*WiHM!lL
z)g3SXKF`Qxl@mSS8p*(LKsF_y;eOzvs<%RwW<p)3>mySy|ELMNRXp!0TmM;IACr!x
zZ#&Cw2~P2HDdwM}C;hDa>5(qs@4;8C`eaVMyY*t?nw)bV-j>D1T?v@;Ew{G%^@PJ8
zTqWI~Z;g1nZ$s}&J;9TQ)|uy-cXh1tw<*YRxcgq)lfx$IYH{T);nwp9wuPOl>brk4
zRCoCWDba$|yz`vbr+oKUQc|j0?6_OJe}($B06X4&r5SfV99`d$d*i~P^Y2v!zs6T3
zZK}Q6GQIsaZ~VfX&E-9R<CN;Q&fIsr`tQS&%NQ5}G7@#<99Uaqr{9Ts88Ww{!|?X2
z&8u5G*#swEsQ57Fqsx{0!$+5GQr}&sXI*GNwIk?!{;i6GSzoqf>hWIf=$Kl~m$lm_
z#Lq?K&DKMSKa|>E-mabh!+uX)#fF_ynl*Q1V*W0QRPy@wE3k6kg5&wcdmisDyZUb5
zzVCa=e3X<PZM{+T@|$h^eQCSwv&FmApFd}>p8Z{-zBhB*!$Yy4F;w%u-3B#3_s;zr
z<^RjSa@PayvcuwgADP~G^|#{sy2<7KdP<A-US4*;Z$f5x;g`KzR;wOXUesxA`~L3d
zz1ubmGifj|EOBOI4d6{&&FBnjepcU+Uy}dz``!Z_yDUvj^)4^?y)IdO>cX{I{Xfm;
z<w?AIWR!com-X8$P_-tvPtt4Sztie(ABWHVWS^cbYsbl(rKzj~89P;ayK1fYm$z3g
zZ2SEwaBUG^P0hr<z7r1mYu|^(hr2IXym<Zk$%hqoe);>1QM)ta#<Q8aK3h6F*2m|*
zntC?v$b>`b7e0LsQi}W&uB7BO+kHXo&Rw>LmA_qW*B5%kUHYYT%ar$fTIP8yS-ifP
zgNuPd64ZNh3E1~?Qr|3>6430o(6kPXpI2Y{GKRXBtXXlU!Dyz@7Rm6{>*j9@?$isu
zHcQairOw}ZQQV!K$F!a_p8mz>ll*j!*@n6&7uhWZKYEwX-BodY*UsECrS$=xb<^Bc
z_v|Zf3KdxI`g&LDC3B-xDc<+%JY81G#W@OvZO#c5zrAX0jFr~zz{U64+&1L+u?Iid
zBFMndWqO94H9#`4oAL6CKQci-WcGd96!v@-cnM)r<n5@N(#ukoxExy$w!ZBZ_n#$>
z{_%S+R^wP?VzkJ9$20XeJL~;s|16E>T_~LG`S9}_=a{vx{`*9iH+9^*SA5^9Vj6q*
z8cqg=pRGJx3#@K|7Wl<%x#C}yckZR(+~xcCZJHTgx@YRjWBlchR%U~?TVJ_))wEM%
zU5co*boSIMvzcQz);(L!saPldjK})H^jpc_^6rO8aYtJycQP?J)FtT1IZXAs(JSK(
znw|cby!^bp&l`&kOY7!Y3kpt5J-libkKo1Ar_Vp$S8b}Ipz!N_Tx^^tX#I!cYc*3P
zrFp4SH(3Sx^F^0RR6M9VdiD9cbMF#YDhGN0;Z|i}aG2P_!?hrYjrSH`(hrv{x&L(d
zm(`We_Nsg<`2Am$mf+0puT#!LS6bXrf(%lMPn+{N=1hFZ!p5ne4lb>3oVne-l~q{q
zu=aX&ubsseqJo?`zs-!VswoL^Y$>vuVzn+1G#uy1#`>d~#nSG~sSWr0I8V91x?6qi
z!pVn=JZ8Gu?$|PUsxi~^d)q{{%rjTtsldL(p>4@R>*c-D76orUw#V0nEWNdT_w{s5
zBg2@=zv`t*O1)lYdC#(z#5jshpO^P|FaI00MSh>FW^<-aa+%g4lKAkVi_4dvl@0;7
zW~s;tPV5OS>)!W6z}4mA`zAZDe{SKr3=AS(Cl)Z&cf|brx$}EuzNX;BL(g~oZnnrg
zH}Uj>iFY-MO_ijw?>o#0Zr1Lue!K6RO_UOEby3-d*IY`oYBN?C&S!a=W5&O3Vb12d
z9_P7HvAeJ3JTRycT>t9C&%dvP0uQ~Oz50Co*Uz8hcD#4%yUwnvq_lGX``FZp;WhV{
z<@<8*ec+xX7<g#;c_AgGz5k@mqJ*5e<i)q2UH>RWrT_Ciqw1COL)KnC<2iS=5T{EN
zkDw-#`Br(0(swuBt(}v;M`>Eev@<f%o4>tC@5xh5>Ui<u$kXUD-Rx^8KNgm}@~>UQ
zz!1;^ni%Fh*%<u#-{n=$pZuP3Ewv)&U&FDluVwf<&ad3M&(*oq?ZR5uyFr$Fy<D~w
z`J7MdeYL%K-p6By3J>qR?RS+IG}%=0XPxZjs%550@q3lFU2d5g-q&aHL|8C$%Blr>
zCIol4J+qAKHGcT^<}B0KyY&jg?e1DFD?WRRxyH`xU7lQ4s<~I>pS!t_rLL}dDK|yI
zAZyRj-xpT&g?*}?y~}J}^k%=u8>RKW&))m^+M~E%E-r_@37cu^XJ~!@Y#|z!{MvGT
zX>+y!Xbe7kyWBIS#oIp5?dTA(lMZ`3d*Am}b5Hh%$1^fG7#`YS8GoMt!TJ|hR$e~Z
zEuNg5yi>0wc+>ab|AiIiiSgfVXR5LLZGH3bpq7l-yod`IHMF#5vPiL6#yZ65b#_lu
z5uVt#uF>z>?hW3Ll@<xHr1pII|3JoPrHWqIBp($4$rtJSi)}p@D<^p>T=L)TyT`;z
zDNIeEbN^k_I3ZcV**>?}lIF_nuHiNg55NBSc)z-;YG#H)G&92=lbG3@4p;BR**rL?
zWwvGY%;@GLERyrrD~r8;a;-uCl3;gHUFZsBvs0z}<5kZ5Epc(G-DvTBxymK+$NxMS
z7(Uph1Tchp@O_+Xo$kJ9$2MQvvxh(TulgQw`-;Z*{P|Y<W`IuDdpO^UA++N0=I?vv
z%TJvDy;5o3({&fO-S3}z`q87Q-*@r+7ExQc>V+K(LxU1e_9TXog+-t3rX3LwG)&{>
zUz4$7{@+PHiT9*m-Qo*a$5pj4H)ioH({vgBKmRNk7#Lice}M+<7T0{f`>$VhQpzQP
z+U;xFJ66nj`=Xt{J>{m#f$PEy3|%^Zl-U;;)l7Wh&iuwHabmO;9|OY#?w+H}3=9kn
zs}{<OJkX!yz4W9}<T75Si5k)A%fH+`d^{!Ue#rmp`+GjU_|AV%Q|ik4DVG)Y|4F{^
zc|ymIkf6)!1D5XUbmY^0WM^A1C^LQ5s%0zl!#3}#)7!hza-OT!#FVOPCf<nx-`m(3
z8vZ!2vHs|0>i2!W^GndL;JT}t=Kf!Pmo4?J{NVgPmRo&3$0T2?lG7pkpXkfP%ZXP#
zJ@T$|b>FAeemg4{1#Qc&IJJ15?)*FN8{)Oj-`P}jp*j3W6mRUtteoFIXP@|=%PE?-
zb83f_*_IEVt37Q_r>bAOG4u7a=Q^M@_XRWNUF=J*{Bdqw)S@@1*D^3L>;tvL#W`DM
zxrD5}f6F#FD)Yosmzq@b#GDILXWmtdNfteo_NpoVM6O}n?C1V>Dk6WjXs(>Mm)Yp&
z)h4lX>JhH5U$S4bxUSk|X|?TaQ|{y_wbeiUGMB!2T~l4R@$(sFp8M}EoL#!LWX273
z^;s6@=UzG+diHa{>@B~u{=LwVn)+s*&Bn){HE&dJzI#L0Md^<dBSS;Yp_+gI&9=l8
zqsWR1i|ef`LM$&ADJj{_KmUK)gCDa}WBTGg+%bG<-nS|*V)gZ<(&BfdRz3gm>GR{+
z{-Bj~6aN>l^LBBmvi-OHT#v-deTD7w>lBr|KE&Jpl@R2;YqD?}pR>!Z(p~Q*F3q?R
zKKV|c>+{z-r?wvvpKko8*E~o=Q>5$q`gmc%!wc?<Gdu`aXm0W<eRpSON7S|)NfD7F
zrMg;ElumDRyuBoB@vEmts#NP9FIaTTtheXO*RNZPe~GE@x7zoHOYr8mExLbmrGuCG
z*+(l1uC=Xk_xkef)6DEOO%HEx%?nVg-M{PYZNZ5)1vl@OpRAn!boQJ4^(!WOE&coJ
zYxS2GfxFAz7ML<KG{iUaaA}!lTv+h-_V)I+wryEgU)|eV9W(v##+XI3P4u`sL$pp-
zD4oAPQP|GL_MleBip1VCwo{afPp5&_?tTqZ+FAdWNARQW>#4$NU!G)6S$z+*F+}zL
z_5I(Lb?sRnzrXC&m6u;$UcS4lw79$+w1KhV5SOG#NbZLR2h-2b^Y!xDw0ZOA4<8=v
zn!}`VuD0g%vHK>O``$jyzF*ioTVwvNyggcHk1Hz0?Nv~Ev~@++oYc9K3g23RrlkJI
z?X4@+ZSByR9dh=P)9xH+Z&RgDJD<saZ@=dsC?qC!ZCB~*+uQT|<!pc5*qHo+g_Yq)
z>WKvh7^|zR<>lokdR$o@zCLz$Sz)Z7WALu`di!M;OKH!y&?#OOyZG#egI7B<&MjPh
z{P?z0{;xcJeS+*RoY@uBxqji?Zyxi6TwPq0zUI8{xUzMtq>_^0LoM%{HueAZxOR)_
z$L)#OlydUd*VmOcfBvyB{3!h)%XKH?#)gM;t;@ASRv4t6nep@U^Ws=P$HUvD4D$sJ
zt1l{!{uck~mTkrH;`UbcIa$+B7N4(~-?*jjj>D(pexW9J_Ec{6ooyy4DERyP`}tO-
zTHwXWRVNl4kTf>lY+e5D3gq%vWw)LQC%bN+nkH{7Q<TIgs2L|W$DnX&)bg2^t-_z~
zTJq@im6gGKe0)Va?_9oo`RLJB@YXJ-lz<08GmTQ$#qYnjE_QeA?{8mUTy$1)d>+k~
z%iV69mpi5ZcS+8|&yTM~p1iTdChCNy;riw$`9Fg%g079O`}^zV%a?a|6fX9iZ5A_~
znIXVMp_%FUmaSXQ&$qu1zTFXYuCu$FTTyHvW6)MV(WlL|i_7-^pXm{?XvxG+jzQP<
zJ~^rB;?fgoxiQW%sl#K^-(7*4oSZs>GiOd0`x|x9mE-g#<Fqp~Y$`X+oO$!^?(+XX
zpUZ>SX<AKiY^-K#OgKN!);R6VkN5lk=j7(z-&cFO)Rnc_rsVh{*;DOn-aL3BTmEVP
z>KkX8Iy@c<OUz0wt$uO&w~LF<PNSsBv%bA@DfwTPFingzvF>|t=ZZO*JCpXQX{xHG
zJ~+^reSO{5O`AS_{kk?{qY`L8u!!b~1pxs%F*^z#AM1U8Z|~-<TWf!OP`v)Mz^|_S
z_7d6kT6ue)>|L^0O>l37{S%h}C%f`%E3P=~TypaL&*eU|RUN<Q3rwu~zV4aR!lkwK
zN|WyY*Y`QKY`X_X^~ycXD{3cq)xYQ!@hU7VjNM&kTlQu~b93|d?b{n08NrsDo>&mj
z&?95{>B-5-g@uK0Z*Twp>C>ZKbN2f!m{!K@Q|kE5?)VM$<81j~@@=afg_Sq{ed0Uq
zh`_}?^KLuuw|2W#znOoI^sTe+U0lM&V?FM-@>Xv+AMNE*{LAc-@YIbt>E}O0?0o*y
zI`RIj6MxUiC@XFC>z*x{`Q)*V>*eMC^KWg-ef{{camkAd!OQ(Jv$Bdn8xRFsc(_=-
zudj<;AGcRaglq1cIiR)BrIz(mzdUVLKjl9EkX?G6;?KqJeWoAnIZ(MO*jFiuM<-qP
zrKd~It8*VupL0#0^+>aqtEx!1wL>KF;zS{{&yG4WZTv|;a$cR=8U6l5r^mm{>pI)R
zkAz9@TDf?onc&2>n>V&PcV@i5p}O5(O*rtz-M`7-?bVOve4W8JbG_5A=FK9u`tkd6
zZf<(IIsN>z^YizYz7Dh7Kf~XXf#FB=1jmI3GA}JT`S$kq-ku(}9*K=9CxzCX{;9o4
zPG(i)=6#?4D4$v$eY|8{<mPAZs^@oV{Cs+oEw?=`S#-^t2REY^x=fL&Nj<n_@u_Kz
zzn5))vGb*{VBnK~8^69bf4_E*`f{a$;NxyvU-8_UX!zxvd0F)X!>!d@cfW224WOpY
zHCz2RsLJR5zVw{UUqV99m2G)`PBN|Q_2FA)1#hBu-+drz)14P>`+d8C*ZzXqxOB~^
zclY+*UKP5!U*7)Rot?$k)<lAH*c3yZ=p)sp3nu*f@iDobPj;SNtrg_9S@BEnR!l$s
z?|SaO>enybuUTAwvZ`%I$Ak~-J(nzt^vYSHf9hrOl;f=|?dnS>Ei2PhR{k6RV}Wqo
zye+TH+ADPX?nlP^9(l6YxpB!JJLdl5)^&e&%$+L>E^P{%_RMA|IC$d3i)UwN$L=b*
zxwAOEqT<KBz14=g@2|h;@GUuFpL6iKq2b?^r~A{Ef6PtFez-OHIG?$>d2MYiVo$He
zuL+I|A6&Y4ac9NHMXOi8et5Y3?c2Bhb1W9FUw^(d(B56mS80*9ciLMqUj5~&-l3~+
z>`LwZbLYd}fC=vVcU7r(T)E;@Gu60Uwe?F__~&P5qqpTmF7=waXV0EhAuA7+S~4&k
zWjgj~;ezDIvQJM=YKN~gF*h$SFV~OXXLDWjZ_9@2chBWqSFR5B^6CoFy1Me(uK3$h
ztwmvuev_PQw#)6(zalzm*DkB29KEx@Y3024Q=8l?W%}wsBQxmI`41liy6f*G^Dr>f
z^B8G}@MT|HQ&?QA9lh<%%HZX%uB<#<Y8fD+qN1{6$BvkNH9MW#`HuI?i+6S%U%H%u
zfr~9AK*I#Ok9%Q2#g7jUr|Cu)#qx>d2?`4K_4W13*-9BC9N3n7n}x~8hlwFTGiEj?
zSFfCHRYhf`A<yNd-qV%c`wC+DM3VCF?a92k>FHwk{<L#*c4lAKV+mkjxa!Q-dT5Dj
zx7hal`(oWko7s4ywq}W5f9kOC%ZG=~(b2abJa}++wz)oZFLRe>&r#=x!a_oz%jX&z
z8U&TybfUNQ96vliXhOWWe%zbe+t)iSv?zMw@%-Fe(9s$TejKvD7w}`oj2U-!m7YF!
zEba5Nvwwemo$R$VCY+`9@I2e<sxL1VE?oHV?b}`kE`|fTDFGT&zI^!t+C{lGdi&+&
z{^hT(ynKCqeM~qD>)l1J-EO^7sW~}sUcb)H%{>c_IoV^6777Gwh&(>ldwWBo^VY1Z
zckbRjJyrWSXao1jr=LDPKK}mJ*42J<t;F?W3`{^rH-M&GS`X#l+>{!;+|Sg=$g21m
z=n|YAdhA@&mMmFP_xD$2Rn@k<yT7ii3|94;@&L5uUBQX1b<x8Ejm*q!JOZ6A?)`GP
zx3*|rf9l})@#*R5@9yrtzrX(e&6}0q-`&m1%8ChRVPMDv9W}@#E<T-GT<^rwB6jxn
z;N^aWv3w#N@9*!=zq;z`-{0TW{pane|6j)vz|g?NDk;*{acyn%{XLbz{{HLZ_t(uf
z%YC#<=E04}$NO`0bMNn|++6ndR_f_#hCGKsCt6HkJ@#m!N9F&2wY$sSHZ?UJ>y_SK
z@X$%6|Iy1=S6A2m`r_&7X<6`~Ve({QP=0jayTH%o>IFK6p|*DInl;zf#ab6UI51_3
z$U1GNP|!8bix(@;Kfk}`rxB9|gF|C050`6iV`Jm??c1B14{uC9zN_r5l(_iuQr87m
z+F@%Xj8Z&qZc5F+zwhpvNMq1`xQ3WS9l0*GoiTPrPfq0C-gb3W=<6>pFYi!EnYcVa
zL(O;AmeSYPq)fA7_EcQFabw2oE(V4N?jSb_&p&^ESLy3NKR?g6t-iJ?_4IW8_@Y?B
zKdvWFp48LRi{74>du`3lx3{+oIxsSD*?nnt65O7BeceLm_E#@n%&{mubm~;s>!Jte
zyE{8yUR>P1em$uD_U-L$brCMm=5>ZoZFK<}kJ$KRD&F0(Tp5y;pI`s)PvyGPjrKYw
zCLl-c+qduWv0m$<CmbN3CpPkMxyHBi$=1}=Y}vZC`un@R$;bIX%8%W@zu&c6tg*2X
z<iyQsyn07j83H;IbmY3audk2ao_+lt=;n)MJ~NHe`K~`b@P1d_->UU-dnZkr^zF^f
z%gg=2`vx24H}h~E^>A@%*}L~|Gdq7wOpICH9S!hN40RVfh1J>k<<=}*SorJ9%doXk
zh2W!FOnQzw8!Cx)*Vfd4j)-`0pz+<^-QFrf>$IEVH`e|A_3!U*X$grtdn$z)MHm)L
zaI)7Bu{N5S^Wwt7-R1B9J)d9iH`hv4tb6A=j!)gGH(y>}zCM2cy+el%oj$$Vsg(<y
zUiWkST@dhwgQY2UciGuFmY**!cHg#bTmHR04|ge@m_B8SNO<`5udlD0=iRw+{kk~F
zngC~e4UzchZ8?&rSyT4y`v)r0uCG5|YWYFAu&~g!`rDqu$8OX0V)<mPmaJTP5|m~?
zL|@?NI(q%X!^8e_t)@<xaA9BV?{0DZqFBBsuP!h5&%Lz;bU8$apt9dQn@$EU@LFl7
zjn{S-r|0MA^YVg<wl_C7x4$k@U~g<})Q#Hm;rsXa9R&}U`OdDe`E!klL4(UcjNbuN
znO|Rb7j&`9_x=CR&M<6#U8K;we*OB%lY?t&{+yd@{rmg-^PmFc!*PXXCIP|PUtd-_
zxAXbWvyrqYcreQ}`$vu4by3EhyLa<T7&s^h%&{n3RQvlIs4iuw7joOm(9yxe%bS01
z&&^$>uVZ(YX};fgxYYIN-D_*3=gys5T3Wh3et#dx{a+NCnU;X=#$CRA`R2`=moHCm
z=aXHuXwi;!D}Kzft^U@+DXbN~uIBkU*_cfX3=2G**jP_2Shj3i{r`U_C#!=>m220&
z9d7532@iR*WZ5!4Nu!jfr>0JuHf`U&eN(kU!DkHp&^C}4QDA?4ZLPF@-JS>?yV6%7
zcXyTU)LXv8!^5MXtZZM=)2_XH|87h^zGcf6@Npdf#8Ltp=E&OD-8p*HH8%F|g@w*L
zG<R~<{QGnB;>C|2A0KaK<Gr=3RC`)0BZEk;L^u11nUg19etLTP&zgOU7e9V@xP7PI
z@{a1ize-O})0MZcv#I{}M%KDap@E4(r1Hc9hMk~WIN#phetL?gam9xRQ?<j3V*MsA
zol`3<B(y2__O{B;&%g<L1=F!djGsW4MeVEo{rvoVerdCuqg|pq{LjyS`ts$?_3`#f
zR#jgzmix^GACwcq@_!c7lY=*I{P_0v_Sspcw|AAU?(X(3Dl)n*Dk{6X?Cq<Yo0m_U
zc5Q3+bpZ!PhAz>RfQCIUU%dhyp5fjv_wL=hxmKlJot+PNEn4&dbRnd1n$OQqPit#y
zrA@PrfYQT)AdqMEr={M#bLZN+*yycUq2=Y}$;rv<w6)eeDf)S1WAgVmH$m4H$LuIr
z=rhv@oOeLy@_i@{TN7be`6<QO`LJ>NxhrcTC%-P@nt$ZTk$ttl<Mvc+%)Px$-GAPd
zYuBEEYLo{IMj8kFLB;Os)vK2+dv<ZL`)t$fW2Kg@C#UPj%gD;g+tu9Io_~L?b@{55
zD?vwhFcdI<2A$6}->z0KX2*d|InU3{^`B>RvedHm*zWT8Ik~y=b~P5cx3<*&{-y|u
zi1ZiitUp}+=h<Z5-&b2v@#8=vGw94Rz2ytMy}WL1%Z&~Rx$^7lYxiEMqad#zFa)h<
z{r>Q9d)eDtdux79dhNu;zA&KT#RWxoclUMNj0_C#Kx3g|D?@&Lc^Ta5bg)!W^4^}x
z$8X-)6g)Vv(7C-R)>}+eD?}rDo6hx61_p;l1<>(<hkP|eqN1XTVqL|QCU$i6^!65B
z^krbs*swv36LjQX2rp;`m4RWDCNUg*50o-8eAs<$UF`4s_5YXJ_jYzFE;3F(x2OKU
zU2Srwkm|Fuv(pn27X11I8d5vHLFaTSFT;oEEnBusnl$NCPTAL2q0!ONqE|p?_o??B
zWfrN5-BogOb8lOl+hjH0e|3nn`z)=kpWj>-zCO-rVF1z*fY;Z>TG!V8mEZpI^78(y
zF3=Hn{3!tqPW}o4UtU~%%+BjIU9Wd0=s>*%3ZQfQ_EvxYcX0LIs;`ee|J2fEV_;D5
zj}Ye%5ZR!Uos<M>s>$rXSpM$L#!Z`oYD2*X`q^tZ1RgN(d4F&3e*OhKheK8bFfxM9
z){_FAt*3N6)LrNFv-GPr3=9oji8^u%UToN~q2lYS&~5r#_wTO<9j~Ru$-toC{(_xV
z!)nglxyGO)1GBEJ0aYywtO7<F4i_ipT7Z|x2PA+D@W{Nd;9&j#ztjKit^58i_Gp)=
z<ysB~1_gGI2|<_IA4|4v+_ue)ft8`5iBnQ!fyc|MtHlo+w1C!wYuuRN$k-*~?0opy
z+1c9i)|Qrq(L4+c4tXpw;tT;As_N?TI|>v{O{czgDhgN`q$)2jzcAp%<Hv_fbuCxN
z?k>B!Ds<;MCI*HbEvB9f`;$O@k+3xnc6A*(b?A=Gho|14Lo^xmKm)<yDr#y%7kRq9
zmR`BKbKTMjGe77UXlZr5)?#3Q9Q=1~ozKckPKwt<MT~woA9mWYj*)@EqTSRJl-(9&
zflfnZU>G=>X&pC%hMC013+uMJY3h3Udewfnd3i4R+8$@m?6@~^{NL33kL2Hp-XEL0
zPbYGoz~qX@bDlihy+6o+f#CpX=y3tpo@33Yc=L20EYE*&g0EwGAZO#R*+2HmeVT7M
z<=6z*HnU})W=PD7b8Yk3op<=f)4$1o>t9JQFfhm*j<7j!U%gDje9hJd|4%cf3%*XP
znPKw%^xAM0orP*DlhRKu{dvGdE;%Fll-fQXS$m@=6%2xcA6eOwggSpNow+{<v{og)
zfrsnEbxvz@aRI@T``69Uul(UB_|vHF_Z$A#`X@R(6vSrzIdVN<x2vquBzF5yfs^O;
zIqudPeL8UKh4iOO+SjJ#1-^Yhg`a`p!}CNPxd-cm^`nD+He5V$<AmLf??0nt7f)MS
z_sGGA)mK<hP|*JE?Afo1m8N=RpNY!V_41hdZ)OoA1H(VNlz@i)eN&eS2@1xSZ-_p0
zW$H^~^I*TYY1`lE8|XPIZhYB`elN;pF`bWRS8qDGbjsnkJ+t)om$<t1KXH8D>MS-j
zQ$bB>>!zudPoIi>jZ;(vtqfkESf#)7Uz1mKhS?g`3zIzG?^@on!R1SC_b#O`6Q7*x
ze)F_Ti%a@?puT2OuyAm^{ls7Um6ei?Fd6^#>768Mc;vG8=_@%A3t~PUx6@-_(BPAp
z_V=dhP6mg?ZoRj*WL{pI#_P@9T4^R$GAH#>NbJuUzC67#TOB@W`Td-t8Qav`<DY%}
zirJ6yDSaFLp4A5b*=whJe{sSQw$;~Py-2m+q@>ojeEL#b@A<+^!lpi_S9AXCy0AO`
z@adQ420y<Aow~03IVg)yFFASEJ-eUh-uXxCEw#TYWfY&2d`#fv(xQD6{9QEr`&-xA
z`R-{;-M%A}>+*Raxl*-={oc+`O_*GN+?}v8TePxHz{Tat>Gy>l4z{08Pyb`e&d{)k
z^Rpx4KdYJT)lN~*RNPKS*c4~)e5riZ#l=OEamw`z1+PnFlVu}coPF?b@BN*8KR>BV
z>bR1<&3K-5!;KD&?S%)q%T+u(JD$AX>zsC$J9eW=&gDlF)`m}!UEO(acT@V=-#h+u
z`bS4AJP{T=f5U5kW8kL$bJz3V+vvix@-=UK#Ia*0svq8o^WPWI6r6Zr*Xn5+xrV~)
zk3KOnOLNgQeBH**U&?9uuczb6>oV=g_~{~b4-dB;X<5emefh7so%VaJ%wqgpIG<my
zJyFiU%D@od#K!vL_l@Nflf~7Kq{P^H?#;SfG`s9&`A_xQ?u?pxUd4@{E}Z{=TSZ;T
zOtJ23&-#Z&N{?o5GtEf$$Z7d=HM@0AM~83kR95z7cGXJURr=L`LoKHrJhpEtJNvSj
zo3e(*Zb?dlg4+Lu3^(6j{`7UyweuZSb63~|9{QP@*nRz&e6)O>Swn|}mS$F<=T9Mj
z|JfZKPd;^2C{2p}K3!1o<5k~Nf`S`4<I~@IC@Lv!{dJf3n9e2b)Ba!W#VY@ti~n_l
zfnfm$XiW&`x6CIM)BUH*o!rM7*k)3?+fUPahozuku;cDkc7iAG-}66tL-pb7*7j5F
zQN6RCJ(hX7nThApKkdJBWF}p6ocT+&u_L0(|5N3fJ4*FmqawK%PusTm&)L~iTz1-g
z+Nt8==#q6-JLNy?!X00K{xDWb`mwY`YR0BQ{lnAepL0)SU|_hZ*vz!T@6*9l#pYwR
zm7g?gyVEW|{FSQ}_>%3FrLT5s{+;QkIy-(mUG@J;s>|gM#$vL|+G^C4lo&H>cK2lX
zcx^lR;LYXQMCBMKwY67D<|c0XzF}kV<M;KBC(h}2)y==Qbk?UQlirB0Vgij^u>N@M
zbzSh<`dJ^R^mfFXD!w_h`3vtYUn$|hbJwS*JExqIoKbTyKdxck!kX5Ix^Bm(+hoPI
zuQ|#V#}!#<cDzHA^Vh1H1^2d;|B49$EzN7<;rdYi@!v*cDeK_lI;YEZpMF`?zrE?Q
zltieYfaY$M;@9s(T=oPy_AjpL|M`@COPKhnWd1+;B_H!{KA$4Pw5)8k(b}FV{E}Ls
zRlg?ej~8cP5OD;p2blh-aKfA)Re#yjZ|m6>S?h+W9n)6ISJM<fmp!w~XnE){7vJs<
zkB_EC=Y==!uk5XITXXqJPsfcT2i8x&FC?hCG;M#8^y1i}U8}9)<u<40*2oA7o?nuh
z`TNxV(l`bNjcF4c8UJn4pPyRvXyz83%lj*zoz3FwowC_9RX$GZM#qlb`JEk#rFZAm
z8r>?%U+s4!sOC%@zmk%dSGD83mkzTsAOCo@Y-*>+THo}AozK3Xt=M(y^~uv3mbMXE
z>*vgui!-e})zq@<)c;lGD}JsGHq*Vk%F_3uvDgWw^L2ZRzn}i5ed?*-BDuxJcmIK!
z7x7@n<QH3Q&1siDFTFhS@|5sfPPa?Tc5a*6(J{5%ZH=A3i)Oo6J?~VZi3iT?d-Gu(
z&%_09C7i2`l_t%(uhW^**^z$N6Lf^X`Q|HH@k(!hD9)c67<_iq^oM&&kLm5*a@P0f
zsShW_xm;Xaj&y8YHC<V$ZA%O9<veZuEt`YVd`*L&mRwb7?dj-vC@r0PJ8jQT2g~14
zr%F~oHQ4>=UaVEwVcxfABYu6V{Aj4OXu0)%n@%;Qwo}t04Sg$j1~Y)>Hftv_)E)J|
ztL|BTQE~64K+pL<qK-wrD^cOD7uuM#^2_bJ7M+m8?2{+`Tdt`6SE#t4^v&1pM?y5V
zRwZ)o)tT)2^kKR4@^zsCQ)g^i8PdB&S>3jJZQyFJUp9Wzf|B~HyXV|moq8)!@aL1)
z?!teqB){KFc|5PBCvs7S!HE=~?5CCyKd-NE=l>kVe^vLix>DpCmj$!Mj3?*iao#VS
zd;QSa6o<n-K~ERTv-|li-1E@RlL4})@!4`Cjs1?=U!VN;{8y;5YDL`gE{7$4Qzl7O
z=$oq8wO{Lr2)H~&9dyooXyy8EH4m<5sa#O|nZ5tZ@6-R4+}4EMpK{6nY{%7CvyDy)
zYW`fO;ty&YIk<pEt`EGQXuo5Y*~H*ac1OS0%FS)fX?YWQ#DD&yo3@qr)_!X2^{5q?
z_!<;^_AH+r85tND9Ij3n<TRuu`mw@BY17V!>Q5f?@0_nZRohEaq9B%)fuVp$QshDS
zm)F<d-`!nav~$jDr=kfHCKSBCXB!-R*=wp+VXXJLTif%`mvS;N958H4ja-+_&|rV^
z{{8)7YokC1c$??TUEj*U!0?3UaL3?iYJ#G3Ae*|B+Rt-(Q_OV+hW)7z4m6tQ-@9_<
zN{F4Wrza;r|MjaN5Sr)d<t5a8l$~F0O~gi}^iU=Sh7|B|>n~ruI(GEv)0=Out&R4b
zWfJIU%fP@;c__lhVd3@l@%!b?6C{@9+}w2S+O<_H*+D%q@Zs(6@9nL>(~x@e-rnlZ
z6+iZYZt`l}SkbU(LDJE#g9i`V|F8lb<m}kYwn_(d@;UnjeWn1BFE1~vZ)Z4V;Pdp<
zRL~IZ0te7R>|AvQGUqp~*}QqPRyU|;2s%u@t7DdFHX94elOMf&JUm-AYzU|Y^+|R*
z+G{unPJCQobJ(Et^)**;_liwYWI@M^%gfJymj-W5Wd$3+m!18(`un@o-$CsyEfUhw
z#i2qB3=Vx4__+c~Ql$IRC6;AgUWRmryKVKiGc}4;puNN3A?<*Yj-!vl)<!M07v*5_
z^YfFr2I~DvgSwNdpHD3*dwc8U>(|8*;C>spJLm51URqlE^s!9%DQ3tOeGIN$eSLn%
z`(&r~TbI8(lWPd-sS1KF{8T))=}i+Wcb}~FvH*=8>r|a{a&!b&u3jAt8vXDV6B84=
zy**z)bd|{URt5%!?*+914e?WE&9W+ZaDbVeFC#1K)vH%Uv3_jRQdNaI+uGV30vi6E
z@bUEQ?CV<>w6tsb^zfx#Q^5ma7HvFSAG9Amek?01yFO-T&~m@Ig34}7{13Ls@2L9v
zYN2zx*L1zNFJDIP2c0#}$iScn8oAl>;lqb-Z*SXIep*ub`C0AnZ)a;f{eEn0=a<jB
zz0Ef^_V1IEli%Lj`WSR#^aCCvjRWT^e|}2cUG~<l=EsGd#pyb~ezH0`I=;TXe!ACE
z>+*MRuCJE|O)W5(U#MsPApGvm&dWOrA5WMd5FQ@Rav?rBFV8ObmWizF+<$+6>ql+r
z04*M9kbc3=dSd<N^z+}|+zehA&?8~E2y}Mv^xz#wj~oG=d}U%%a%Dx}>uYNd*RU}(
zJYtg+QHbA`d)qAO2**SZle{|?I<NmJPntC8+WL5T35gq9v#&e1^JQjafX_i+!Dpo5
za4+rbtgr9)|L;1QWNyCQDD_l9tlve&$tTO++<5r>`Sl%zkBgt5dw6aU1H(~vBaH+7
zCsH;obZ&olZ}0A;qg{`W_g}ww5i~;BmU?rsdp~IGFzf27xVX5dr>8#$4c!H>rUW#^
zc&Z4gt1ssk*SoSNviQo1K&OQhUUzlr=<Bav?A|XWB^9@$fU!82k)ewjG}P+q=637m
zP1}kO3p^*Qty{MaeB6DPg}wcLS?jWltgLyK#mBB)n+6J@2k$|Hzz1iW=il0v`}^D5
z+kLXuR#sL;v3`XgA06%Hl{Pz(vT1Mi_iyj+&IYU3KlC+#VbkvV|NFLX{kpsSeN0S@
zfJ5lYsI6I-m-z~Jy43#q61lTT6%<nk;z1W4=;+1lxUjdn{AtjVs;{pKWBm$6L`An&
zeSOt0Z@+8j&b2W+CnZXN)=M<l^Mj6Xe|L9xJCEd|g$oVy?(EPCT~!e4_fd7C$F;T5
z-(Ou_J<qoK-QC^ds}C|TbjhUzG?@JV_xt^t$jv-FJZe5O5;8JQlv*B*&B}VUE%&yZ
zUCoU>mBrx0v0@=Ma7v0aJSqD5;^N}%IX9D%k{&&K7PcwHbDcJml2WTv;fo83`T6f>
znP!8AP-Au$#enA4z>}%vo}Qk2tG}0(m#+_7d+Wu;#ipjFJJ)e+`tad{Mdha@8#jJD
z+AW@aWd-<H?+@t;%}yJ|L`AP|Og_H5{JmM$m5d7u9KpvCE3og|w{Lg(d%eI#M{aIT
zpKX?V3N#-2fnA~5>EqXbe}9*~y~WDPx;6j)y^|+7U$5sjs;~dwD{a0>r~1W(g?V>(
zfrqmH=%oZ`ER(aZyR*=_{rr4;P|!dYxK#fB@bK`ftE(SBetdmxw0YH+4DfW>eb60=
z7pH26?<#vcYt9^-loJBrQ>+{RboKPaY|Xm*{QUgoJ~KC^oD?dFWo8hG10CTmV^eYA
z>ebNL*xD~IE{3iSd%8>JhvmnQAG5EotNs0LZFl$Sb+NmzfKQHRV5nm?(h$)<JxzDE
zRq3jE^XAQ-JsWgN{nG>MCwMK5-j-AO>51oTvs}M<Hj#^5IKj~|zm13MXsgr0g5u)n
z?Rj%2PQ2K{Dg5Q@*PZJ)>gGBvys;mAlJo0pYpWj~YTcB2x(IyvfJ{n&hS-uI&^U9y
zynS9?o?Y3S2wh#>b=pn#2kwHvUZ+;BxVShmF){EQian#GNY}pg@%ybTEI@0O9Glsk
z+jtK099s0`Sg-W&Z*R3DH$D0J`T5$lYe5sq44=3qMY?uexNrd!6)Y@UvaYTwe}B(L
zt{-%cd-bO$C!d|2U1@Xg;$nAM85wX1`3W?i3W}(2Z*RA^wwAuXclX@6zSl($@+*IT
zdwXkZwyF?FQ#+q5II<_0|GTrZc;`CC2kUp#{jIvaEqC$a#h|MKva?T@T7D3J`SRt}
z)#2|?PF8pCmz#R}>5oEY29cT<?5&Gl{`mM<PfyQAF8|7kK+~)%4|d4}wC>ok<Lc`0
z`X3(>9~@|m-kvAQ!2(XWJDu2Cl`=qO?D2m6$W2dvd<0$Le!kRlfz`>ACuf`Ge)|4>
z`{vEoWp8GH57%a3I3a1IA)={s+Ha;&>d`LI)YH@E8YDWc3|hKl9Y=_--t_nP_TIjJ
zeY;~b+q`-6mU>SIPcDE@XqVO0)MRGku_$>F@buKw<Gs?$*RDNV>Uv=2WOe_lii(hR
zF*i@2US0I`6gWn1DmFU_3W|$AfAON?>#M7~N?(I6osW&3`?{+k`QE*I?EG?Q7kO-Q
zU~66U;PP_+x3{;~|N4^o`Ptdu-`;k2H#A+U|NnP;?rpP1hk{2(I{9R+3P6*XDGqF{
zPWlE08)A2t9Xxn2c6Zs^D=USsKXo|x>DSlSTeGjLsjKJT-&cE}ff4MY%!1kgjXO29
zwb5I%L?tC(zIt^Fd{B5JlY4yp|M~U*D!;tAI8oW1g~@Sq+F9_lR>uW?uGU$*cUP~E
z+Z!=W)-cKC?X9gl_1L-At&iWoE^e=tib~3l4-Y}Nd3^o}ijaUiip@@mTT@R@3ta4$
znVsD)ZGP^`l_{@_7J%kRqW0I-CL|<0KR1_O-fj(a3iGIxk%owuar!yX1W<&|JcGoh
z2%TrUWHe+_jBcKrYyJN2?)2Q;+JAp4Yinykc}ip71jmIFCZ0@jZsWPRIlX`7%9lF|
zAD5Jr>{!RaB|I&)U)K5=`2K;1PoIV^bYcbTSvJ9Op@FA|M?q21u8NOIYooTh^~q?0
zC$zbidQaDrwJchab90lpUd)Q<?Ruc1UBvywf`AMc7Z>Pt9><O?tN;HmB8+MO;q&M7
zkN3&Wnl)>=|NOkGt3<)dtxqfnu-H@gcUSK1ZSU{x1ugWuzAkpB9(z~QfddD4rOlQs
zS@P!Q=HoYSf^T~10-xvp?*9J#^mK7q*|(1#U0USIz3z15B4rbklCQ6>8X6jUPuH7k
zl*$D*Uirj=fD4u2yF5;&Y)U-b_V@F7`&E{it3p<t_!8o)@#o@Q`{3ZET|Q?kvz)fp
z+|>!0CGgem!`20-?5BD^D=M#cJ@oqddf_Ltre;OFSh4i|*RPLvU%xnO>g(`}r_V?J
zTaz=NM^Nx%i__LG>Rln3S<l4o*GhG@bZnoPqk7rMvfiz$<-ztu&!VqIY0GYD?R=V5
zQu1Y0=xV#VKNe+gZd_d*{@86j1H-CD9xm2f=jK|s^GYvUxze)i%?+*4RV!X?x%Asz
zY)-|LS^xIbz2~%jHSOfm!aq|pyY)6pZd^U>vBY$@WA=8AXHH9dmsM)Wm&;i%FSJ}M
zGh4O&-ptMacmyMN`n-$s{(klAXKS1HTJc8362?0`<<ft0Z>l$PIkNs{sbBo+FIN-i
zzmttFewFxZ_18JazUL&Ys9EFD_38WW!V91E&aZg<rqH)|k6rqs-PxbF#iaaLo359&
z_;2_<CqH#xpP21=u>k=SCQrWn@9*!8*LWEmSU-r{i^P0>eqP?DV#B<7_rAQm94g_^
z>5*~s_5Gv8$4wJ2?Nj(KTP-ow`|^>5|6drgQm)osd$hEYEkNng>X7n|$+`2+{&<y{
z=n)?6u_P^K`OOuLbEh`?op`eI`}`?dQ?>lFZ@#|&aj$V+`OU+*_ZImuPU+y0{u*JG
z?NXZ5vEud$%lk|jhc2&XUv@sWEMVrstt>nLyb9d&`Mdg}dYSCiIt@Y-r%k(dq*GYh
zB%|QZkB?@#w`%_W3T;(qXV3s8b%jZ%pB`@Goo$@n*V%cpN79&wmp3$?+5hs2wAou8
zoIbVQzCv<#u=Y#u`14=X+1!r(l(h@HyfSEgUhFAlk5!$o|K%^beWUYo^o(f=N@pjr
zJ=M*MFj)2P)-tz0=GpV|FKu7mRvKNtexlhAIhlEvk1z7*R1CYYCBFI7%fGv`RD0Jh
zJ3s5v!c7m_)4OV_w0mZ3+IrUe@4{Q3y?uRie|&hjHfrmx%Fk(_v&5a9Lt_{i915Fx
zxLEyyf+j6q{CHdL?N#CH?Q(BznWXAn6xw-0;W5(>J8^l{n(Gc9zh0f_uKChGa_1lQ
zzIp49*~k{&EP6C)=DrnDOINM_wkV|Q1o!Gb<7HAavjnbX-j4kF;@-yg`2Dw)zs$U^
zu5`RDZDzS<e&Et2+qQW;cHd}iX!62v<DOH~L`yApd=TqDUR+l8?%v*N2Zx5KQ>X6V
zzyDP>1A~U$1jmI5on2j7H#a@KxjB99+O@~~<<GzM-79)YBuOh&sjWjqUf+h-`^>cm
zXPxDfv@Ca8Jh|sT`<JPbjgF3ujg5_#p5}2erhN?}oQVhDH#Hwt+NV~t{ZYHnqC<-(
z?Ylo`u8E$Oj*f+ej*gb!ufKwa_Iq`D%n}e4laY~;oBHeTzpf=KK5TgK;K7CuD|%Ah
zS6;MZzjh~0)a8>(=Bkic+vZo@oHbXAyQ5=j{Ql`{+am-k-><PhzoNO-VtQ-X&LEcG
z_G*iE{SSHeVR>G9x_QNi2OmBZNK4OteSLj#I0u7+;P=CYh74D$zP|%iA8~uDu7Vow
zcXu22WIeZerZ6*paop{wW6^x~6&8KIcmDPFt!HNZ*5LiPS41o0Rn*p=FBN{)>McL#
z$CN8H7)|^8an%-yYnww|=juGvvHSA!;Z=3H`s`z?FCJT4p1o0OshZ2)+P{A$-aqP|
z|K!`^XC~V7KkrgiQp$X?_Hy?5)F`cx&AI#Et<B<?tTg}M_b)F$uReeO#ObYXuR4AB
zv*5XLDs$agzstXFEh+z8SNC#v*7WLQj5A-=s#{zY`TYJ}T;;>wo(_@q=XK}R_H|hG
z&%M554%@^u+1uBi--}UEnlF8RR>X}Z{`Y(8U%z)>Gd1_@{CO8<O8xY(6o_~Ivw8Qw
z$X;plYgez{-BFl)ds{BQjKzZ|Pmbi;GBCIb{c>EmpyJDmiCedReSLlX?=LSOKY0=Y
zKEi$W8L6uo3+C+5{BAEOG1c(Xzn?efG9<}L&&u4SdH=n{?|09dN`EuY$XXZtOgDB{
z$;UN?z7q2uZ9V_Ywf(+&%1()+OCKMfFA<h~^P1Y)!(V?sHFbZdHu1{;OE0<Y=Jnq@
z(dqG_d10pOlbe>S&)Kdt%s4e?srP@`%ey57f5zY1e)qR-;H%l2r_bL%QCg@q)~e`3
z)17tY`McdhJ3QXizjHev{*-Z_&F6@%r}xe~{9tGI^ncdbCEU(-8ySCx=hjYM5LbWT
zOr39w%aPTW5>MUvS)6`$RsM>FcUHSht<sf{6V!~G&&}ceFRa((!gnphtI~h|w46KQ
zpUM1DO?Bsf=CEEV(`CN1mn~fiy5^<y^)*nF-2uE5`OwqT(`TDzi;0Sw*8KQzZEbXM
zte=`^(Y&ac$I8-9r5}H{$4h!yn9F_Zj~(GlgG&wL|NZ@XHb{AW&PN5M+l;ZRBr^mA
z{coR0JjURxl*GKP(&n6vNZitupZ9$HdwcsUlbnAMPrYh({(tFy@Ke1Ed%wKtwyL<-
z)12N{+)Wexro3yGbcWzc$?Lgy4^HrRNu2&Sxy9}7G`&}G&ToYU*L%(G<o;#8K6?Ak
z<n?B4HA-Clo@z>7Tpuk8CZ}K7=q+sQE4MOp_Lh=24lYO9_E(56UAAmx{4zB`Ln*0S
zvz&Z$-};DqU0r*9^~6sXXGZH!dG8_k{o9exA&tI^i|lJlm0dqfKJF6oeg3IcF@pb@
zpPrsR-!R#2rcvs#UTO2nPbo3S_!u6XcVc6G_xAq&`{&MquDqIKaq-fnNw14G1q+0(
zxbI~7IJoTfrPnOeIv(A(+j&;&)|NF(J(v8@o1;=+_tss<@M?D0m9sZi1s~lu(Ms^7
zye;o}ljP0E?4M;e`es}y+tU2$;p__;Pn-8V3pvJqk6%Hl-*@)*r8BD^PM*`@QE08?
z=r={IJp0NzMZf6N>(;G4zc@d<yo<kD)?v+~$3MUPeV${pKBxTo;pty07v>c4ajx;{
zZOgPh(q9u-KUFyJ#O8LFD?1&J&i{8dD1q&fv3BUHD?O6NTH)*VBpu}v7oR?5ij37B
zMurF16YMk!a!yWCy}aE2`t|GO#l_m;>twD!ed2U`%ii9P@1hqTX}kYJ|Hhuko9g$%
zCF(v`g_acO$O(&0x||l*Dz&ZTz0yWT#dslKqfKGQs+4(^yVZE!o2};fBQw+MSeB)%
zDtBhh4Ruz-sWNkBzgpUxaN8vR-1O5=Rn+JDxSz{CTleO<fzqm^%PVK@+_$N-*SC0I
z(M7v>>;9#`v)p>$o{_6&6O^v|xu5@Pj<=n~>+=#@S4M>?pFgzOQ@X?a&vFHxBk9Z5
zuC=ZF^h7s$n~YV-i_g!`FZY{!>Egwc)#eNgC%BC?3fvY3^hlfM9d6@&dTMHMaq;Er
z*PriNB>!Nhd7j^PKB0@Nx%uN}h1h&|OS`(e?&N%%u4Q>~kI&dlOf0pt4WAJ8Pi*Sv
zOIBW|XN9d^Gxza|+Ha<xZhl~W<uXM@D>*T78tAO{g>}Ks=NEud@O#?}nvdPrgno{)
zpMP;5?~&+8@Hy>Qu3mNTm)l$a|KIBH^=7%ZzC1eGJtt;^7{h{&gAq0b%9fTp-TURF
z&2lzu*bwloxhOVJu7XYM?*Bk#rJx^gZx!`aluC!4e*13j{SAzX@sl>^JrWi)oO<fT
zv%;?nSN+;mS>7Zh7|12KG5@gA{@ovfgF+&Xe4lP|_U7{R#QX6UT3R|f?b@!Xi({Ae
z_8JOKeDv|-<G3Rs|BpX<&VG3+^G6rcy2SXY4?b4>{r&y@t*x&wFZW-&W(^ZFGiU+n
zf(y=Utb9j~9NCh2`Pq|`lc!CaCamUj;nJzUtDNUdI^AKkY0-vhy|urDH}mz@Z+Nq8
zO6D!zY4i5|oKd-3<rlZ0V6fMe*K5sG=QBNV3Gs_j`W(h+-@ks<@on0Ik?VfWEXaO+
z>TIK-{f*X>CNI|hsCfA3@%fqYzC8Oj=ltC`Bma0x(cWz~=l292TWihQ`)d02KQFnZ
zRO+gJ&xn{8*xtpOqhL4NZ>|-ugn@#I$(F*$$M)^p2kJfr>`-iOTGiUpVv>33$+fl7
zYooWz+12cr6=nDPcZmMi%d^Yh>~8sTFo*YX)G5{HN2cV4ZLg`ab}foZS3m6E9gvk&
zoO(KZp1Jd+OW~hYZ1<PW?RZt19b<NRw%7;bWk-(et1Frv`k}|l#AkNqtg-{;3twJu
z^zZBq3#e)e(Y}3PnT+7XN8euA+6j7cDS7#J?I}$;x$^^8vu^L@{f}R6UD`cOMKDmZ
zJL=T(a{l$7_<pQWQ1bfw=TiB-W5#J^eQHjVP6u^($O(M>|LvWP-l|pqnSVULmnh+O
zJTW+>(9^-t!MRx5G&uOqSySEXMzyDkR>t>-r9~}U_H@?H=)9TF_BD8`>2-WsGJz}2
zXciZb!fxfm2508pmv8RsXzJoQ{rs<g@pt`iPWHv;Y@fei@YvPTa)#;q#N8$@iql`7
zd#7t7|Fitm+@yEjTP}TlVbAzu>dC$zOt-e>TIb!_v1ZMhK3VH`x3;c+!QIjELUD%U
z%8<w~$2}DvA0239zF+hCtdS9CMq<xNL&1fCZku0z-*DoF^=r13vr1}38+=du-Fvm?
zRPV3Z>HY7w*Yy2*9@KR1QqhX4)FT}J90u<HcRVa;ifdZZZ?z}4UoHIF7XR;Qc?=2%
zCp|mV*LHmA>mE1fM@uIghreroV4al`?JaeAxr)jm9tC67vqG)Bnj59ovN+72mdL>H
z@n>1%wCTc2XFPbnymnW@!6wbi`)&&-zA-3zQ0(P+>F80Hr%XE}=1;hOJMV;#-&BSp
z^Y>3nYI*U%cvG$J>50tQ&ugtJ|226AKU;fKbw~ND`^)qivaZH5IB?zF{`~vh)oFTG
zzuq%v>u5@E=Vehi5%>9fMablBfzu0YwZt2ytUA3z`facI;=Ut{93RT`e${(RUf5Ci
z`2WA}`-^to*|ccQ*3*hgTI*T-s@#`mURdBb+dLn1d*AA?wQFKeAC`P<?i(pGGrv}u
zW8dRt{NDo@Jnq_7t~>62Ilg2g>jbgA`QL6Ov#$I0{_n#J)3X^3((YMEZTTPIa^|jc
z?(e==S2q2-Yxn2L4CDR(H=Z@(-rHI}+pgAndr=sJf!o{2ZSP#(?ET$!_V-r*`(;0h
z{-yKIvH0s6Zk^nf_Wf7r!&`fn=<L`oacTeN{qCmDhi<O?Z2o+=bN9yo;sq(^PE~uG
zck0^xTQ$G?PJyLK#?^Ph<^DOh-mYDjE4^AbKJe?u+skyh+9ey-WX?ay9JP}B=c_91
zice>D{Cn(XT74s7ZnXNfv-gCj&aR*S;%@1s-Ro~1ZL8f`v0R}#_tY7`)cvww)*U%=
zWc~hsR_5m4zrDR}W+rpT)8)$zPzw9{=BBZQMa9!oQ?KbQ`69tvwf7)LN7rw)iuw;M
zCq%VU7p9AJFS-7*M@ZpRsMb^AJFZ4AN~al3(OMd#Q=k*PoLwaAt8;shWa-SETw6bV
zj9)W>XXeSw$W2;(=UyG3cm3Y3Y5MVXzrJMdjmuB}exo=Lbja^MVcY1U)}VD2S2TVo
z`D@4puB@tX(mD5)Yw54V)DEuFK*qpzLH0Mh_bzgJ`{<OUd1r@~>^^=k7Z(?orM;3<
zIyyQ!yrhja1O){Jr<$b&C@Co^Y2}_-0Fn>6HN&y9qoZTRmcl%>MaR=j?nGK>$_Gw4
z{q+0`@3~9-{dbjUE-bsukZ#HFLde9BbNdw2S3fIm*uT^*H4HSgntbW<`RAPKp;bwf
z-u;?tvG?S)#dWr#`&oqrL&MH2aOpoSu=D5ZwSoL=A~+oizH(>W*b&(<e|t3NQ>(}K
z-OK-ol&~?doabj^nxo75XIEAKHI|KE>-WEW`p3mk&-&k=Q@qxUTckdD&Ud@&z3j%L
zle@m{SJnQ`z3KY0Gi$<UomW*_WYj7-<wc?9EdJRK=a(ej{Vu)k0MEhV2fx?br)PKU
z7b`CMeQ-rb%Zd(xi@RShDUU1FXmqoix!V8rL$gm?cl$GzmA_wJAhghw(`oDRsIZe6
z1}8)-gU_DM`D}M%wO3`S(#uz^+zhk6<i_-cJyTIyBy^m|>x;@1y|1%>-HhopatheJ
z^J>Yo!#B73Eq->^FXu;@f5<u!;XPkn<_4~>u=`r<!XDnAl@$9RWZ&E_@%umW-#*c?
zR-gXpxJB}%oBCoYHI|$ArfRC}^gi%he)s*RxwE~uF4y~1D3L$UT9EzV*23nX+!|G-
zMVs34lLc*Mi~D}QJH2qxucz;tb>&T`Kl-Zrg=O){kc9pf0lTYqSc%j}R=iC6u;+eD
zRoPEHfvbA!<n&Z8hW)nKl=Q!|CGtzwzd2E1yZqIZ7Hw*MyTC=&vt*4RgQ8H{ORuCh
zK`+ZMFF7R(N=gNSg(l_Aev?&Y@w~Lo``45uS8iLX&CI>}O79&1msQe1LX+~J&&XMA
zwQ|X!rbU6%Cro<%s4Bi=3eQZ>lP@hdtc?m?S{Cqh@8m2W-kVz#uIO|cyySIb;Bvhu
z80*!vu&ev?-^gn#{QW)m^3LR0bm!j1$jbk-*Y29aGWX2ItB#2mmZ~hjXLd8Jw77KN
zf05*>pi{YvJ@dlP#;s*|(pBxeb;)aSB}NvLk5#jlzU}N-!IZ$Jsuj5Fipvbfj`Nmx
z_x*W3&)(VJ<!JTx_rdnH`2{V44vY5*w{0o$-cwR^Z&vL0bwV}mbtVVyFYQV_J)z{|
z_4n7^4#(VJ?$Im^*PqkNVO1Db^4_qt=C*aBpL}V_d8^YemhS#O$Kb&I(6U#{-#44?
zWLPr8cJs1#j2DuxR7}5ZDERQVvdOY{$@^=T`c2(^b-wkU{Kp43|D1ol?7)lzY4eQ!
zo-Iy4KeyXTW~pIiiQJb}zE_Sd_&iNG<)5uOf9_&729KAs&TbBKJM`x1^QE1a7*^c>
z`!{g?mA%2g&vJOX@7aEO^52(2f`P3EdAz=CX{kQj@vGy0{Y55*J*$?#kK7&4y@S!v
zy)5SA=VIq6VjiYXROa6|&p*6z*<QD=RV(-TUhBOduFHGTZT<Z1HD9K1#%Ud#@a6iI
z_w|hrEWa#X_)jmiXQp{meXP`~ldt>CRp!L+|FPoQ90rGiUAGUcu-=;YtxqF)^}gh)
zEnlX1uiR}vTRN*RZvThx{^?(<3ZG2LI=J~-tCY+8b^pzuEY(_e`#FCO_r8S}v(7T)
zyesXz&lPEC9WnX!{mLf?`glbrBq=wG=UbKi+5VI>GWlL!-twEZ8eD1S3hy4Qmu9`6
z*UbL-)c+>G<1_T$Sc=`<(4JrFAipowcS%70m-NMZt{7<sZ~ixXs@u)I>?xU_tsZ4t
zZ&04sv0~HX-wRy)HLXoGEsbrT{#t5$Tdd~m>z6VNi^|V8{{7`M!}OkblUT!ppBFbI
zKDLv$uQ~C-ao?qVyvJR?%dOwOZoP<N_|?4o+qD^H2<yf9GL|In&tbcec*DVa^NN_*
z-&_6WzO(arc&p^ln@-_hW^4)P&Sh)(_Wh}KpK@{W+YR@0ZJ#o@EMI23m+`=wy{Qc`
z>q=I=<agMxIapKFYie+J-FrU?>vQ5x|KyboKNW0H3om$oWp}bOD?`Utrd@qG@!SWV
z)}B6pf7QMBw`czS`y|TY%eyZhuOGh6%ChCP^@_=}c^sUg%orpl>1X;h%=OV@61W;S
z_xH8`*9*R<hQ0h1KaqiHRz;zUjiBI0*(onx&RD-D-sJncX)i=CEOk9C#yHQ5VbS90
z({-XIzrMe^nwKG~{Jz_}?MZzuhkJhP^$w5OQqq`hB6WIe;`K?^*UxZWo2BsMSC!?x
z-OKMb&(!@dJ!2QQ;zNd6rYp<8Tc@r~t7UtvlvP#zWOn7tytJ(w_P%dxTmA4q^MmVq
z7XG^Dx6$$QH;YYj2X@X_A|8_!@$HxCx8jr4`_}HA%(2!p(Q|W=wJ1aJmrbI#@5|lU
z+30P${kiqNiSH(q8<!ms77VmIv%tmw_1Vt(m7#(~jMA+^OHa(_b8Vd)@Zd?$5etTU
zxmEiv9XM%pZ?Es{GNTzj3@NLQynfEm5mzePZJ)hd@$;;(rU%ixtEN_4i~2FHjkZ5_
zKEY?_Dou-Dat$wHc2A!DX2P|SK;i5POa10q@)FDIzrQ*5*7i{Uky(!qhu6!$ySuOc
z{<pKzS5I2cxGwnqT8r#Q7Z;YafJMty*{&393m2`9RGHbU%TRlDmYv;Kd7D+XybRCQ
z++n}YR9nHgfi<De@9c?{Z<C^*yyTx8{3FlW+k3g?ZVT1Uue*DyPVXpu+<BTIsV`@4
zaZ2|7jq|szNUmS6yl*PM^`p)XjvDzXFEXFbm)~+@%k#%_b8Jmi_4K@%7&eDZUZ`IZ
zP?NZ1&yNaKyUh<2oBiT%Oj^{sY035OSrN{O?zVgHbQ{i|lwzPE^<C!c&l?LF%j3-M
zetSHt-pIF7_G80*i1`U@s++hp?JDY1lK$C7UO)Uy=iZ{tD(0W3zIgqeX{pAO(m3x+
zwFg5slzizn@;?1&<yrT~pPsvGmVTdpza?`s*Svar>6si1pQLzapJ1G^=-;J#;`N&+
zFZN})@@iH8kHSw}CP#GVZnN`j+ST{}LH>TZ%iDDsW(ZHpm)x`^Tq#lP?ryu|eHBVd
zOb2+pzARChoZ8W!pk4i9ePCRJ@Tar>A(xj=e0q4L@8^FVep46@%&2(z<cq5H>+;vX
zo9Y{+86Nb{FUvCe9Jl+-o|!xh&t9xHFP&z)h=;-FVqvD6L)>Q1=TQs_t@?4>bk$x?
zV|j5opGE)IyCpXkM$Xk@@Y!j};B(Q_ieZwTqN4B#?Uz|OYzz+X?rt(>n6X~QzWQ(8
z*2wFNozDmzX(-dbvgh}8*MIWImA-Df7c^~giJ{x<+?@;4PapVKSHxi8d42JJmIq*e
z>}r1BEM#lv>9xD8u3b%>VabUXr`KDVmrS;w;PTbeTyLf-gXTw0tCPN}Oby_M``Y<^
zN0=-&l-*-Ic=ha@1*|OJ{r=v*(%KPsS4x-Piof_v*Uau)-!osYxSX@!IP=dwRpW2J
zzyB;+QUB*F3xk3B^3=55R<|p+y_7B}2vj-q?8k?x_uHS_#udnZ^t3*$@Ez9vU*6UH
ztv~o%v)Xp!=x3MC`*<2JSjMy>Z2i|0=cd<LRs1~kPX2$voDZwk&sKc-DBrt`@x}Kz
z{%}i`*chJ>y@u|OoK-tC86}J+2dgrCe7=6^mbLG77!RzI|F*OK)@RGDYnOQMkekYV
z@cP}r@@F104Gj6u(?9=j?&vrre=tYyTXwpO|JwBP^9~+7s2#p;OPQko;flQ9;nM#1
z?c3Zx$ePzy{Og=B>HPaSyA^f*Yc}Tlzk858v+yLWxz2C;_FLHXZM|13HLq1zmb~A)
zxGDYjwMP?$4d2zRjo!}lAuRpRo`hzz`?<6EW_)lnHuo%ld_dpZ`CU)0_}-{~sT(&>
zAJluVQu}9HX7`~)Hihg<tNo`MM>{`$ch@P$@Z;*bbwv!vUQayUY%aT}D7{nd;_9y}
z>gMj9^OlD*efhcjbN)|)btSqjJ9X{mUHf?FlBKvpWZHgq2E&9SNA7(s`uD5oqo3oq
zTT^Dv)e5qH@#*Pl|M_-nufKl#=1rH~?2e9i$9cS#u5wDbFiBDJi&e8=L(Iw`yBi%@
zCNFr}?!;)US~az(*^hndsuwS3@l?${sL{D-(S*sW&ddDxHLko;Vi%YEYPC1e>EU-x
zw#AP51`?_rUR%BRJlf7Y*`gCE>Yn~}@}0G3lGs$0s-)g@t8d~9To|G)C&&JjYwDl8
zq~H}@*XvdUtdgtp4qT|0$1+7b^ixt#Q~i&xTuW`@WkDT*Rdq*xy10aJ7WxR`Z}}TG
zgA@d2CbOw3DJd<wbZqC$j(L+(3?#VU3A^s}(VO-r+~(wjc_Pxn+IudX365K2UT4<h
z8NPmD+w$(N_>#_!6#}4g-1%4af1%TdX7d+)=F0fu^jMtvkIcNHtTIj0l+$ewIxH9#
zRrBxPG|zGAl+_m%cbm0kFMXH1za%hpW0|dsONf|Zvyd(C#>DCT`?LO?UtfI9<uK2|
zt@l?j#`SHRdGh0P|NmlAqARbu$lZDP^W2k(yfQiSA8DRy%dNh3^WVP5B|NKcOnF-7
zT6?Hf_p|w(*aMf{((V>@bgX!zb^8At!KwUQ&#U&;nK3BrxV9?#^v#<~=lMIVPmJ$>
z`BjNw#`-^dp0D|rDW{+nYy0Zz>g0H#qju4Y?(F-z!*l+g+jnoZ3J3;9f|~wcmhAaI
ztN6^NyK@(%cbn}KV3@H;PgQ!}FUe(0{My|j=eF*(mASfVdF0E<9UWS%-;N5~T5oDE
zJAm4b?)tr-N2>ItrsQ^Cqe|22>Q;`OkM=z_YgpoceOcNppB1m3JX^Et|J#6*6XwmC
zHgm4><eOnz%S!_8e%D*2r|PI)Tvj?ERZVqi`s|#yZ)8@B*Zf+Xt<87x&yDE<f`O@r
zdAz=?kuVlkPGnQ)FkSV8yQ_1$-?`|`ukUSpVEtt=<3GJ=8?BF>UT7Wm)=qgx<*!Hg
zHP*5m{M`I9IqSaXq^78jy(Kv}XVsRk=d1ZWna}R3?)=I6bMI_On=ba@Q_<y&EeyB#
z=3P)eaCU|FPNCwLPCxDy^m<-f63pHr)3w8RT0D=$-<J;hTh~Zz1$8wgz8w{wDs%9f
zntry|OiuCb=YAgD^zi=f&d+n>w?6s&``Ox?SGQbWk-2%_+>b}u*Y7%+&=+T)ztb(9
zZG!CY=d13f?_qX7JI^5PMoa#&FIg-%p8nT0oqKm<&e?s--u~v1zjRxese##E-ggqi
zff+@4?`FkD9sE4^e0rap?Zefb`3x5SUa$Y3GHX*_R$tp2&g(N6W+?H=cCER;GkpEE
zI3tD$O5ta2O;-|)l@=5X<V<2y)v^qFa!}d5|K0tKHJ8%wpZS0D634B?1HbdPl$^G9
ztWUn*_IOcs|MlqW;aw9}XKjCfTa@7pr}+A13?BQIF)G}M`1Qf{yx3fuiDmcSEN;qv
z|G(y&+w`z~y>W6@+pHNp9(rb4GWh(u&A?#2)QQdE;Uhr?#%C>0qS<~|>ALmLPN=lJ
zx;ON{ej5M#8@o!^UyT2fuf)!vY4Ya%7Nu-uB_%D#H{I%5mQAbe^o%sL?rloFocQF*
zfwrIbkM9u`7v?p7^j$Bu@ZzPtYcFQ#v>#fhTAm$p*KaLTLFN5^>l9_{y;lNM*I%p*
zNMBy__lME`>9ek%mR8W;9vp3#lJ=Zw+cv(u+&_0N&z-eHi@{;~lvy+Pn@v6`lr#5k
z_R3FXnzO%4h)HdX;*QxlvF5bdw~Jdpa%&3;PE|+?STsp_$2Uo{h+oC0_uN0naDun<
z!nSXfeyVqD=S`GgKd{^Pe7KawOdo~|%Y3aRebg9?rg9dndi}c1K(`?4yZ`cUKlo&>
z{yoZY|Bt^x`hDq2{qhpt27$_SBfr3}G3OH>GpZb&W4Y(k_T9g^4QBg%Hw)PsU3v4(
zg(q3-)zwyZtpJZ~cnKJ51fDc}o+2HvJMz7w;q&_56OAG}b4q(pK8`QmQ5=(&%<T}r
zby>gv-$U;g<bAkzWTE}}`vt6NGZt@O({u0Imh(@{qgHa;-wsK*Jz<gMhTPH>$F8h;
zKO?j5)7*Id$_Syo&Qor3GO$=n&xS;VWzH=HEl(u|jagr$uQM?*6#rSc)B2ofgkM~>
z-MfRDuY(^wDiKm?c6|5BKJD0(?N;Vn-<Z$6`|)$*{%&LS7i(W920vfJu{S9B#qrC(
zw|oiFO0hhucl+|?v)fkO0u9p$7;9WKR#<=W^?qj^`C|XvJ9WSIu|8e;<H6sFoj<>)
z#PZMHT;V-S@Z<8Zbw7;-i`tLL-k<UDW%b&-DGXb#zdu``e}nVqy3_kbmKi=v5nz}Z
zFD)-_%`m~XrtDJnCQt(|=4xp7n%<6%6)c9$LQ`c9hLt@J_j>o{|H%)1)6S=F*S9PC
zD%LQ|u2-kpFL%~=_mV}aUQ3U7t1s8T8M#0D{w=N(Z>G$;P_NS&{O&`{ck|!k6-_0E
zU-Kp}nNoTA($zd=VaAgalGzlNrbV<JPd|1{?OEv&`!Co3T)SGVn8um(ditML;*rrG
zPo`8pyy13ie_}_+iULqmSkv?6b^R7MNk)gRKiaF-CMTD-YA~&M_xsj~Z`13ngt!y#
zZ;~{7Uw7%w<@A)_QPXVMubs12*s*J-@Ro)L_g2==-o@avVe+HzY;(W8xOgd;PxITa
zQieFYty2wRE~hdXcu(JQQg%-48-^un&v*~`d!KDsJ^7H?`tSd5i<-~AtvYv~u}%H=
z{Ooeo8@+!YnVx>u8r$7(S##sc>a@H^JLgY-%X;B`^S5(K4+30rkEZqPud<C#Ec5@|
z?(O0tIyJ$p_*<ryt!?!;9zMRfokdHxa%*a~{a^dLD*MV`*{ewhj;v?f{mgg86v0hN
zejC)TZ&J**%J?O{?|&QH+FP}+-o5nQz4z+Y+~lns44KpU&i&Kmde&0o-u^W!T)X7p
z_Vz8`-4sge%bT3PT<Tr;U2I|gmG6hm=bNwjpO(5MHC64qm@<3$8Qa};cPB<L{JZPM
zR~5Op^wc6I=P4}T#SXKV*Z#X_<k5DTfBjmEyjWYy<eTf5tatDKeN+41x1)Pqe{QV#
z`RVd<|JPrutgWBrC@ngE4Ah`GlzKs31>T@JCEruww)b4Z6qUe*LALj*-nzvulUw98
zb-T}=!09WeyR=3vZ7{66JjHdQfw`<fqC+tI+;bd(>w+}p_f!XN4U|g@P+GM7;Jyka
zZ2?iQtxi8bJT6-5^{?Y_|BisA)9h=68H$R07Oe<bEnoGUJ=<92@7Bua@!MXt_ZOZz
zd2;2o1>x_*_<Gj3xP;i8S>Q6YTVb+j?WMmiGj&3RS<Sb%&!4nl?~4md+1)LL1Op2V
zn}tHTCdK7P?lLyjy~?otxa2dwotyu+<nOOlOBEChOg+lu74mVZtNfh3-!?ogzB92j
zxPL+Bea`$XH7>i=1O+c{1}m`qRW{FX@!#6ra{JxqL32tdS5f7`gPtxfE@4LmSzKIP
zTtaT$;Pd)oqO$m)hG?yn!YR?zymh{ZzrNrO&wusi&8o2CZy!FDHS5{_%ko)u>9uRV
zd-ioV7Z*_(V~vZGPhXq&v*_TpM{^?g|7KqtxAO0r*Ofd$StnM#Z(e^TQ*uJBmD0Hx
zbEl?;-O$cuxbpt1B53T^Hv7y17u5x+&+Y2wz4MxBSM>i@Krd%g`u&M>vo$A9T=myD
z#eYgz&<&1j>kRE*zOUtOGTr%CtuN<c*1EZ$zve$*nl<li)<5Zq92#ERf=*@_oD{8*
zoC;~ge%Z3?fBGryud5gtdOFLL5}4j(RbMNfet5B;?fqk4x7YrvjhY!He0@T}FBiTe
z4N>Q|G0fth_vT-g+tyj9F6jUFG7*1$wkgTjEoa8OZ?Dp2UxjG?icwNp<#Xy5cb3aj
zZ@nX8y`Pyh8mbxJ#Z42=HpyQ9<9dSD(^GPa)~0$=X)}sHbxm1q^QvNnXz1$S(>}EL
zo_x;#PbxL!<d5$>f`Xz)$^sUtr#}}ncxS6$8n}Gt{y+CVl+27<zkl;#x0O?v8!j$5
zJvD_vL2C+^chKdfDlbybt>fEtX33H-Ig{7Uots>m?%F7#{xWz{Q2z57DceF<@4izW
zGJoEbS-Pv=y^8ta<uU1ELW|<<jGL#frAsHYt*CfXp$Q&)R8BEEvHFI!{f-NJQat?q
zPM<k9@np{GTh>-9KFym`_VUIZ-s21^sV{o2vQ9bZ80q4YVhe5szYKid=<|(X=dDQb
zFITs8%-NHBu=t3&^8E5`ta<YB|6)6?u`IY>E&TTK`WTM;((4RQ&MJA;Fu_@{(o-<$
zzzcu&s+FImG==r&&e@)JeeZ7Wea`k$58el{HcY82ytQUNV?{}Q{T=O6`86um|E^5$
z{{Ad?#^UfZzb*xx-|;fphKqact{vRB#d152?KS6gc^`g!&&7|HTi2!<|GadPH@hs8
zQD5J@*NuC@+XtMsOm{YIP27C_&E`49HE-`cZg}zW*Oo$k+iOw_U0gyMLG=Su<z&a;
z)NS78^ZJ?}{5m(~B5w#2!-=`KzQ2{)l#+k@;pEyg)6Lu8T+C2qSk~d(&T{|U-aYy5
zZ@I6|P;gy0BlPS3`(KL0Cf3y5-@|)+);FoEub2KZi!`r`-)iC_y|?b(r;EXA6)Rh0
zVs|~Q<h(wKXT|S?2b+uY4RfQq-|YBsQ~SHqhuZW{4ZFAwW;jhfv?=k})yKxu-?l!m
zel*#9Uir4kB_FHPb@+NYBdv^I?$tazFPQV#l;Yy+Z<-$bJeR&$YDH>W%jV$c`F8(a
zujjQnZo1-T&aLHo(P?XY+IIflJ%7uZ*sB_yJPTZ0O2mI3>+ov7vp<%d!9ZQcy87$8
zd(zYYH{Dp|>g*0_pgrJbY^YxSFaC_&y^W=>;}-9Kao>fHAt>X<{mcvZZ)|_K<Hfzl
zU$1?gs`O;lt+&6+w%+;t=&A@q*}J;Gw@to&%Kn(9zW)&O{1pLOqM@N>)7Gv!rNy;0
zXlanxC-KIQ&)%Q@#`wbE)7R&7-!gP)eUF>pTf&^s7q?OW632`$+Zh?kyjpo04joly
zU^>HjayyT}(^IRwypvb<1}zQH2=#IcP*U2}{&s=OQkRl%l2?v=yWTP9wBzyoEnn8G
zW$5Xwm+Se;@#b05$p*E_s#`2~Uk`d3yCmmcazw_FEZu^qZl82--^iFDxWD{mSNF8S
zXHvb#isEi;eUjI^s(aC{%Jb&4#6vqndK`b<>@<96;QZlh{kebl;<lEzPunK#|9pnh
zIn_P)jvPL?ws(H@u3+JCR)!aof{f)q9auEq-Nj`pkFiGJ%8B;(*NH1_J0w&4Y0tOb
zw7kfT|6LCKSysQ6>G@3GpKKRbXKnwq>E^<dSL6JBf{rqN$h-V`Dx2T5;9FZV)tgVU
zI4m=onz7$4>Q3*WvhLEqRu8_^%~+hTl797D#k%JkJ~B*7IwuqNGg{@;6_yYemyl(k
zgezxl{(oMa_4kaT^LfAH6E??jit}~8i`^TOBVRdDW3w;=N12(K**1nRq1nHFT+yFe
z#l3J#_|y9)OEUO!qGeYZZhd8)Hfw&(^XuL-Tsk{eSp2Zl2+R!HcVQ(rgU7<<e*Yy4
zm6m(%eje%?oBZ(Q%e0fLZhbEN#oo8<$kCVcT>H0MTIpH-{ugV+AYhYyOFKW7f3{V+
z>x-#U&i3=}Enm7kNb$7h8#xyj7k*H`BlPMk)4EUrMwy%cqL&1$Z|(OE3{O#apUtpi
z)#|8i(5Bz{6T9Th(mzf!NmdlM-}vh59$oo!nU6ld-#>R24};U!Rb6*xOm<{Qsmra2
zdBfnKroF}WdRy@oVZn>cKk}IaId^^MR<-$3etO^86?xPA&2-~G1&D35`n`7bbEh*x
zN3QAr3|;^7@`+8KuPm#qcTx9ZNbuQHQuJ%m+^@U8u3gK}vr=*Yy7-Mru|KXx_wQKl
zqo%#)#GIZG&ISc$hMnHc>+*~krZ@N9zi|I;+;#W47F-K5_fL;~I!W<3kAv6Q4Ltql
zdwPA(yzUTZbZA-{Y4UCvUxS{e;@tzF`gBUW<dhXH9zkg_Uv935VK8u?Z&&S_uVYgb
z`kC*=6E$W5i2(n&8=T^0x^`l+tBoZtub&^eH%GsrPJT<dn4;B>o}BsS4KEMes+{jP
zZ-2t2DV5n}obTrT+P;!o!TMuR!4cM8wXElz`+50wuT(rP>REc?^Yr&ymrDgD>299<
z>ar0J!vee${tYWOh%(H0%wM}Z>GHQy?){OQT)7!;{Jnk4bMp0j4x-|Y3<X(__Z@cs
zUibI!pX^<Cj{js<Ra(^5lRT-zs|nmJ`q}<?&aHiau9SU|Hs9OYewR5ty!7)H$4A|~
zufwVz1k9_=7LN{-6uJ}LBXQ?zrTFO>pQOWji`_%HqqhDQVVEJj#o=;ZerfLQ2wkRz
zwYP5aUSK+3y;Iamdb#4uH&@=9-@p4O`RyKimh*y>+i%agx#QIX`)P9x-LLuDHZ=+_
z-~Zy)gSY38%kkc-*LZZ{PiDK0`|WSCS68OivND{Ae_{40=E$jx8LGZ1JF2_4+sn<X
zF8jMnd1KmJ(f@hMv7&rV|EDSLZ#^heYrwJh|L2?B=lbPpU0qzJb|jb;3usTBH_x`3
zjh%gc%+5=<S|g^G6klF`|NfhQaR;B8yl1(&j9HB*<JE%8l8UFe=Q_k0y(nc<X4o0D
zbWgo#*<!oT+Z;ozS9opcKb&#lHLsgP$chlw@OABh(^qbn*|}S@lk4l;(8s*Hd}{SX
zLa&~3nq6~xGiXhmU2W9*>$h*({xEIs=;%677O?2jv<f{Fp=W}<+<x3u;eLIdB2%-z
zI)^V37xmh@=)>ZorMDKo5Pc)xAG2_Z{gc9m7;vMLYwE1m%7TIyn`bz7dNmt9ou7N#
zsp!hq+E+K*a``uBnqE2X_-(^v)|<}!f`WpfJZBcTELHsY{;{AG?}E-Q2A1;|=l<b(
znRxxRZg%XlWlN<3esIrqad8o4K7LeqYTGQ{_-`!?3{9^Oe82xCNOaC5P2FD)mfv5q
zgUPsIhw|%XezUgx+_LS<YZWCWE!iA)RW074=g*JVSYJ*sa9UT95W&x20c(NYQ1*9m
z5fuj&qJpAb?xh<}KNZb;VIe4Z(RqXZJk)u8&~TIUq>C9-KCQo@sU^Z{b*C3HMt3^c
zM@#?Zq;+>ps;{k%yrH|~OUsQXJbO8fLQBK>(?b>0UtCUpJn5_2<TGJUqfR@2onX$y
zRoiuOLOpLq;JOK)t91nhMcF~!QW5!ml^@@IE;o1mE*E@!`(ESQyQdVHTBTe+R<QO;
z?VIU(tF-(J>OJQdDgL=}Y(em%>i(6Fv&Ck($6dO6<M-)H0ll6DpG&UBU1YlV_P*4m
zrG@)0FUhW#`}J|VJwrw1&P<aij(Pch62F?Vr#`YdI-{h_Yu{9R#fmzETA$o=nyxM`
zM?upGJSTH<7%R5Onk{ReA2W&J;w7II{yR3L{%d<-{rdE6o0A+BcXuAPHkaEYT(J6v
z&4!M&M}6z3w;X<R|MlO6YhN?%3VxmT-eOas+q#@6etYle^rR>6<m<VcB|aRy;<$CH
z)3vW3UV3l)yD4Gko5>shRc7zDf2CP8y&-ZllZ)WVi~G;VR!eM1R@M*{6czq|azT*l
z^wYj-lie0?EK^*1YGvDGCIQ#LJ)gPNPrZ%qpZ(?J?z{6Jtt)DeT5I?B(f-)IT>sT1
zrRy|g(hlwoet&IR=*!Txaq`dDp2)?Em;DuEkjQ&;zl6)7UGQ#}HY-Eut1S{&_t~A{
zls2DpP4u<qG<nYuYfHPM{6TuDsi{Yg9N}mxcwet2C}_*~@2%(3SBLrSL$szAKR?&o
z)3c>acj@sQBhed+3)a1T`*1~=?b%cBx~%N(@o9yI?ryq#m$||4<Ck~b41c13u6+M4
z<a+K~b@mPRI>$Iv=GhDL8XswJoA<5Uf2H&XCI-#A^=g{SUo5|)ICK7^=Nj#oE?r7H
zJ8SBsNuNF*m%n}MR@818rA0-Zl2g1we}8-H?Buj6boIBx{Pry^EfKqAw5vTzj9<M^
zHjv`|ZnnfpUsJ>SXO+bn2Ak}g&u_0S_?*RYK%`LbWWo-qzv>JZKU*HJX`YZHYg?np
zU?D#FON)J|tGKxM_p|2rj~qRE7PPzZ&dzIk?Hw!jJo>#L<W|&{3_&q5F$swsd3Seh
z*|Np1_E*I2MVr)qpZRsO+GOW6UdL<i{%$O763^YB^Z4b(x*OH8f4b(Jwo^9bV_=Cd
zU;cYm)ZDD=tmh8P)aojFN>AlJaQgV)ea0eL`udSu)Xc43wf%b<<$u{PS}}H4N#@5#
zM=dQaUteD@Z<NvjUO0992A@}0!2Y_w8#ZiM<~#e{&CSo3&#yDGy$~9^Gc?=y*XzK!
z(h+8}`07^nwb!?QlVkYR`!^@XkV%8#WWr0&_Lx2E`D3TtzH50nqw;ZZV1dn)m_&63
zfj2Dw-Ub{1t<ieF|9_mw(FF?>*g(DLBlW2#tg^1H0qwOqaNxk><Nf+^du}8cMC@K<
z<hthm#{a9EI3B!_KYo6`{kB&gIhPrQqpob!|8y$Jp0U5K_u`z`llMOAoO-|i|Gu1?
zM(OF#laKe6zPfVq)>8q&snur|q?l?=%{o8N_HY}ob@{tHb1aLEjE&dE2F^^npLmFY
z;n|N1ywcxJ&ieN9>Y1~*R6~Q(HdmgWrn~>or|#|B%OfHpCad|bi4_zKbUnc1bxmdZ
z>Feuaf4^KlKW=YT>9sYHo}M!7#r=o2zMPTQt1~~Iuj1dvxU(h)BaTQ46?N_M{Mcq6
zTCj8H&edUSZ|$$IKRr$N@S#JmuC7jh+ttyb#gP`UY2Qra^nV`?^Lu)F8mFK8^ZkDP
z+E_oyM{&oGetP)hy!&K1b)j3Kg&S%N?%98fGkV6<@ZeQO{MA4K28kl>Jw4ebDYF@O
z>|D9?!Nj=r@f#8xBO)Ts&$I29v3&IAO%B)+zO(=>;bVQWyK`@wH6JucKR4&|^Yh}h
zj;RVu<!*`=JuiN-UU7PN{mNQja|Q$V<I7)te(%3GF;e!x$$hhe-!e!u%<}&DeQu)1
z<qRK&IawB=pZ3cd85yOWpC>CTYg_f@g{*a%#?(0~4IL{~LO<1~uHbojdHMN=huh!2
zc_XG5BOxV~^|ok@->#Y`(<ih@y|4N8bAj46hDUsBxcpD(Jv&+?u77&MGVP6$Kg*=I
z%-YX*V@u7&P`&Rtm&6)Mp5EV5^Y`1$m7LQ=U6^aMN@vdxTmARh?EJhtI}B@oeR+C%
z`soz^O|w8{1(UJHRK>N?+wCepEopB*{_5)L?{97<zb%@#bN9((ZebtH4PXCy8Ook-
z^2&HYWAiT0e!+PH3_0;%+`SG~KZ(^4i8+44Zb7{;gLs+io!u&by)vc<I^4f@PbuO3
zPbUTzkBRB}%kNq`U;0zY)v)HJf_(d*$Is-yP3h89Q%k$HCNeWKb9MOoc~+&XK-p*2
zj2VtAb1MJ+eC|Kj>gtNX#j@6AYvT9EP15>sKqC3i)z{PfGgnG}m-YDne7ovyg>COH
zZW3cyu_sRc4p+m?l9I#oIAZtN3oX`rSQVnjpcy*7@c);vJm2{0P=T|{7T^BALx*vK
zk<+%E3rupdZf1oWnHeh2>K(Rbu-KRP-TtK|!=)sr>vKC=A6S`eDEt@cA^&kom%5VD
zquuZKO;*YL{_bx6{=eJ6&fLJVdFEz@pqZ0r&iwi7^?G^Bq9vY_)sm0(Xzuqe%J$EX
z-0-)j{yl&Dz8LNQ|DwbF&qZuoec<HX`+whv=Kr44vN`_D{hXV^KfXQTv+FxHg**G%
zleLps917l>RQ}24+fkZy_l<4f&Y$lUs><hndAFLce0%Ro^=~;himmvXw9jxdM17UI
zy0-N1pXT04$$oheJH5Gg?BtAE8(EP5@!N!VymQ|Bq{h!#&T#YA|Jt1i+AQ}AFSl1u
zdA`s`QBhIaET^ETh>MFW>)xK7V85K0;kdHr)X9^QhDk@3`Oe<6XV0;I`Sn37*Ti0&
zToJ*h>-+1>lpY;rmz}5X1uoTC^V<2+a+P!m)e~<+`uJ9<zmQoeAzAYK+uQ2z@4kNd
zVpH&-Vf*&&V1pm*h_jg*xNX}uY4g02^78-xe!ovYJIl22(UDhIS4Zz&QRKO@ENqp0
zW!s99vdUGC_dgZcr5sN&S{u9j+mTLT{e3@@0s<y<cXw~qR#IB^V}|3(H$p5Acfa5F
z`NM}FFBXGN%C<~A)M8coDq{DF<H@JaG8QC$&v(y_FAkdFv*}gczO7xGA5`yDcqz0v
zY0HF@LHhF<7*eJm-(JqkcDUdEU&X&aKlNgF<=ox1HS6lCYwJ2XRs<!)+e~E!EqFD{
znc<`M`$04Ry!wBY+0#EuTuM4OXYaDj8y)lV)?Lh4^XTFIWr1tCjD;@D_;mi^<nSk<
zPd04a=(w(T@_Fy4Z<n7wb0%g-!NaSo!`*u%HWoegl9sO1RaeqdJ+mOhuBD~r%e%YQ
z+S;$*Zoi**b=B0Zrx$i97#bG-`}5P&({q0Pzn?vl#%p5v1OuNQ;_-UgwsgCRon767
z1C6n}N;Z~RhU|;oTUC0niPdUu-pNU-@ArP62M+F%8ICJ$b{0N%OGtQdv|HS|^p#1*
z1qH_yBEidoJ3+<wrKR4ijyJYsUVgoPf8N`ojt(v5v;eL2OP4NHeS0G*C3S00<>$b~
zZYQ6LrnT@-o%Hy4|N7Y7*Y4e`dw*}Q{{BCkz_u)z;kfe8$#duAj8aZ)%e`GwQ)Bn%
z!(pw^RS~<Dw9L!i-`iXK{G3(kD;bM|1&N2-z%}!yB(|kRo|Be*e0==>hr|4Ga&p4z
zekDako61~6u6a({Qup`QV)y=i9}aOJJa{lKFAowK$!trD-rUaLKi8&m)8fUC-Q{ba
zJbak=wrIthQzuVGZp(>GNl7{0C%ZlW{<<*xs%v#hN>$HhIIgU@y&>`NiHXX&xw-a#
zJ~+$E%V$q_PMzqtJbFjLLbt`0pU;}t|NEJ~&)LT%M9iQ$XlMD03k&zv{!U9z-~apF
z?z6Max0hLlY*SHJKR?@CUs3VlWPiJ#-)`qyPG=Gflsw4eCF<zn;&Odm?Bq$4*!boC
z+{#`bxm!j{dtL1AZ3zdP&YU^(^Yin5dHZ{yO7x=bnFS$^F?*{_4GcbfdU`r`SBa!S
z0>kXx8d_ReCnu>+QpvrsA@M*1BPg6Y-W}%g5_R|T^7{VnuK#?yzn8rAdu1#qW!HzN
zsr%3S^K$w85Us20Vt3!#nw<`czZD8;0jmPORQ;9C-?K6Oyj*<z{f)`TwWfMyhr5Ic
zh=^<{eI3Tn-~adb_us#NNAH$VQqp2E)(ACGoBZ<c_xtf1lU(24+N$n1=f#5u32(br
zeDRpHCG+yK!|nXPfBt-Xe?RElAW&3vaH-EW%bm3~TS3cnkp}2o5Rs#?yUXTU79YER
ze}B~0tZRB*Ud<OXVm2nZW@Nn3Ucaa4#RWxiaq;YM(16C#vVc_^!q&&xrk#;k8f2Mw
z$09#JKYP0K`LM9CYl~dF|NVSEpOKjjRLn07h}f;9q*Nu67O?8Xx3{;=6Amy;K53G6
zW`>&Ytccw*RbfHF!Ozdm7MGW|&%3kZ=jZ3$i!wkp^$LzN3qlUIwzrqRzh`S|T6$uF
z;`)8RR&71KaFV#Rv~~TzKU=nJu`GU8^Xa5IIH<c?C8ucZn=xa?fddD$!`FRzadC0}
z{e9Q;+PyB_um8Wd;$za{i$6Y}x8MKuS~NsMx8xM9)+I}p<lWsh*Rpt-?`*Tf?flbE
zrd(Us;pI?K@nco!>NjuRl)bre@ax8oj#J`k0jqTK?(e(1zyAN<-`~x%uDtmB{r>YA
zQ$M|XaIks*zhA3Yug)$gC~$CS0OhMi#}D&(P4&pjd$;@jzTcnE+h1QDp8w~^$E>Zl
zZi!x;Y%u$5^tK$(aq<2A=Np;XUtL?94f6MkjfTxZON~=cP08Q)^Vrd&sgI6ynr2_y
zQfB$3qNunye0^MQ<iyhV_xAev`GKQy9w-60fp(}{m#y(yI?Eu@X>ZlnHL-l6)8p31
z@2U8BzyAN--R19nXPXu3hAA!j1oHOExz^=;ayB<|%=Ul3SKTXZzOBqMWXqN<TUfcp
z?(8UhZ2$k~d4Bsp2O3jdT%Jl9YlOzB`_FrGak0Cb+p!N158vLFoBg)vMc|{ykDs5P
zKY!9BqvT^f=k5RB0q2m*No-5K-oD@epO1%UPu<^Nuh;Ltw>n%ud%AP$?wX&U7P)qB
z+Ppb@ZIo%!5svKX&Mq#h4U$u|tk;I!Ug|x)`1!f5+1K-4Tv!<MY~vcs;%8^NM75t3
z{oIy&dsoTJByc84F=!T2G&bIxe}7*)zdRoY$B*;%|B|z_v)^`UoR_cvv$62;u~%1D
zv$L|AW?wr4a#LVuBAY4`!{Un;Nk=;3>wX?ReLDNkkB?eYPu&s~+?ID&N<`$z)2F2m
z4lwHLgB<GOqS`4rMWg>_&ODRMOLuk_$HvCm|9Y_)T+J;ybvni9@0ZK|>i+X`uC0j-
z2ngtvG6gk@1*aOM1uXil6SD(!4nx+~szWWDF%8GHHf`QKT_;jVRP^fF=<T}E+d|ex
zUA-kLC>VG$nN3ye#rFGk(ncvA8ePf9`@Vkuyt&MBs<)flu_jjTb+Nm@owI(g6TR)s
zv17+TS}qzN<napWSnS^a?cLqz<(GS9EI&P+9>1;3a;j@m(xW|<pY!hR`FT`4zUJ%I
zaBw=Gr<fM7=uqsAf`#7G^$H6M|Nj2|{@U8vTTie2VIp<*P%HQ5O`E3a#adN;$pFX3
zc`;*+i_S-n9b;o-b8~kue{-Yo!-Io!tiIgr?CFWwmNQeMYqoj*vm{0rm#K`#8W$HY
zS-$-HtE;bne0+R%mTC3(yX8-dc5c;P)zBwvy)Eyq6+3%-?eA|}v#y>3$3!M*MR(J|
z1PP;*6W7*8OPgdE<lO-saucz85m)N-b8~l>zh4)<{oTQ4c4?E01aO2;F=!SN^_{eY
zQ&>$WZqJRocmFQ#w<`iyH=>Oyp4r#e+1CB3aBy%aDA=%i^=eS_V$r3;JYFF&#m~;n
zG|2>Q8$Z%1oPBjw>f54I=FJB;<lNi@%CW)A{cdl|T^+V|7FbWvQ68@^5@vaKERv3J
z?2P&MuwC9P=f;IwPp5=$&$}D7qu}9G?eI1p$xUfzrS_$IyM%Cp7BtNRl`sDD?QY(@
zx!A2Y>*Au;t*15XCrz64?EL)sGiF$1URn~lIgPU^0hGT_aLsV+l#5PDNx8l*R#-@g
zjaTZ*&f@g!>A~~<{C>ZmTTEw1>1$9SYWL#-GdMoNK<j-}{{8)a|G<F*_5XgV8yFNk
zIMA4LM!Sz&Tu&!*6Uzb65iVa}gKLlzvt~GU&I|DJ^6Hnf^_y!Iy4-IrIMvSCbS}fh
z&DC|~)mL|Smv7I#eGL>H6Q_cFV7WKWzUaw`v$M^kcTf0q@b>NM>+53o?b~-vucf17
xO~2$6$SBnH{c4c4V>~2oJHc3dM#zBw{5S6?y<4VsViD+AJ5N_Xmvv4FO#tx&pu7M8

literal 185267
zcmeAS@N?(olHy`uVBq!ia0y~yU~Xh!U`yj*V_;wqySy-vfkD!>DkP#LD6w3jpeR2r
zGbdG{q_QAYA+;hije()!*4kO=lVmI>wf=uBQq7{#Zo#+Y=Dmde{r8>2f=!pVZ{D^h
zFU`=%n5p1ni~Ekx_Wx_|)_*)+bzUzzcj@#;KPOkeEBxU7dA|Mp>kmHv{#~r`<4ft6
z<&XP<_e}or!e0Jo&6(++D|wqAfBVaAS}!}_UcS7vwyyK-YnS-^Kc$D3XuH?_{1x-t
zDzIz*<A<;BudA=ke<Hu_^Vj;w`B&TiZao$MZ?5*i8^?YIm3&+>!KPk3R&`_3l}i^4
z|9xBeL2>>o>q|dx2mPyk6Z1Rj)B1zIzunf~`1$ecieHu@@3gP1`8{2tR+j(U?;CqL
z{_a@x;qNK?y@{40{~ynnPbz<(_kG^^cW!zvt9GpS{kwJQcWL=KEk=9Sue8znFTMZz
z_sc&sa|$PJ^fgnrKiw*)G;yzDyR6%ehkLGzig};;c;r-ekM^?iHA|(cWArs6?H?ZV
zlG>*1Zm7FynuW~%wW@nwe=FZBDiwaO(f`<ilT7!2-0uH<_u&82=eG;rU2Vm2aJ&B;
z4eo2Ap_g@>XI{Qv)78@QTQs=(W3BzW_FW(CRm>(jRww4=6#V#Ud34K%z1uFGE`C(}
zZ1d;&*EmY+=Qaow<|Z2|PGpRC+7!^V-td}0N3ybg^VH*09;+}P)Oq(nZ>ML8Q?Kr;
zTQ$quzTeFFSS*pDC^(I4?WHRkD%$Cb>$38$T-cCub7thmth2j%Cwb|Gd98GnFwI)E
zGfH>swXkYg-K}@lObv_9-rFlz&2(tZ^tjmkz26OYym)Y>IiKIo@K}%WS)1Hxk?FH*
z`&@sn*^@IhG(5VtSGN4tmj8OYcik<!|G33jb8gL>DZ&2Me|x;w-YxmOH9Bwa+i%wU
zQgv&#zV(mf-k<Vr{${a#o0j!F{l4>;ZMMw1uj_+Z4t8p$r|3L$IJVlz!O{IG&y91I
zTE(YkzMr^kgU7WdO-pNy<P?MX0&m;$cmAHYv;NHN&41*d>i%1O#&YSq|H{RFD}SH5
zbN}<V*=yc@lu+HvvXWagc)s3&W9@D^f7s>zzkIEk8EL!m`_4-~(#w`UkD9&s+{1kC
z;)@!O-yH8--WMBrK}pW~W2@mQ6N#nktPfll5PoshHgfjeNT025LQfpaY_$-)U-5iS
zt#H`s+UYjOVqeD153`;swk+hkZr8H4cPkH1e<Swe_QB%?))70xUMvv4$+=ZE*rWLN
zOe3G~E?4996u--<-H+coNzhK{dFMNyZQE|G3p>WQ`ixxk$y*EU|6aS4%d}2Ca7y7+
z>F+&X=lL<eo%K%5J@sAD<lURwjq`QfE~M`&J$stZ%+V)Ve_x@(Z0VJa0p~iyZbhk{
zxx#TguVv$!*gvJdEOyuVjHf#tJn+DHKCkGOJ*VEyi`u7pW68<hY2C^5f}^+lewJX3
zUw_}c+Wge3iRT((U(8~>YifVenc+}q=FXs-TT*YQ%=C&$Zu6;ie-s(LVX@YNr3_||
zHU;0E+Z!OGy0|9IVrzN5_nq%7xA)%cn#tB<e7)$HRb`HK%x$yR(hK9ZFPd3*(@wr#
zygJ0HR$Xk_{Y_U+F5S=18lV)qQT^^kTh5Fn+0SH8`OW>pzJEv3i-{t#`K4!eSD*M=
zHqH6a`*rOeN2<N%ZV~>kVb&tIdvj>gj9_iQM)TQ^e_0>CGv~<s+mAKnQcm{%`mU}M
zXm^oob#Hb{cy{g+vzv2Ub)LP^%5c84F#XlHYXxq%(%0=4n10dde@6R+*tN&GTxE}2
zyq)xDtMVqr=Gv0@rLJD{J9}4s(A%#dp~Y3QzI0kZutdR9QT?gRaa%Y;w!}&0Gq)~(
zd*a#l&u^q}ytvkPiS^c#o8GfjWJ@|HFL;0BTVnE3JykZPHQT?q27h&YzTw#0<-3==
zDiy3ac9rQwDMye*&E;@0V}6Gc3+CSK$|;jyPI5YZ-r9!!{F5bbp6?1ixW=LQw&dSS
z@_Xl>opelxA@aRx!(o@1!L6KS@(U^_&tfbvvr>{<WOcm6>2cDu9TuJE9JCk;&uaX+
zb?ipWf{XQ{Pj03&Ojcj_I{a47T>tAu<uli^9XAzU;rM%s<dt=`suTK+S4L&T@Xunr
zb|pRk!QBQa9;+ztBh!=8m@jZX;Ag#%&YpW+#D4y+TUB>=X2)M`;azxX&Qjm4yQXeF
zs>>k8>+^3n`^H;^ji*?y_v~X)*X-sBWN1-e{Q3Q|o40dx&u=>ER}t>{yUp&9MYGb^
z1qKP%?^q<9&soYf)kws!e~G_O3FpFWcbDkvpVs()4Qbsu@5b6ntA)p|U$|dx$z=A6
zNu7Hu<1I#JgAjI>L#(HJt2}co92h^8%=zqa+$G~)NMr3az9&oqddhsK_QrPCUSG89
ztC#81!i3LuN7n3pyVUKg>6`9F4nG7JSx01vl&P^hHaacPKf|cv*y>R4w4KrHD)U>0
zlkXgub(n=F<)jws-9OD({N{o;NB9ShX|pao*D00ZTCa3)H)Fi9(BEr}*OF=k?^g3Q
zo}6?y^i0~O*%Qx2^A<jgsARd6bXX@R$w_Gb33f(-mj^nv@*`Q+@^A3xdcQ>_Kt72x
zYW<u8lVUQ~Z;w27Ztm)i0<OP?2MbHG87K5^JNZEI>Z})e1r2>}nuUA+?p*QoS=dsc
z_KWgQmPy)j9F=?_{Jf(w=>7t6U(er%ud=Vp%lVjbzw%4KuWu6-9x8m4&Yk-#bAiqV
z<{+zBUcF{{jpm$!TWn3)`xOhdEIuDfP^g&p?T(23h6g+*?-><3vzJ6nbPG#7ZTY}x
zZhvlKy2544l_D=@uba;$@rHlTN`;%dOt~iYEs}a+J8jXt9|xE_E^lBK4PAO`%86VP
zUa`(!%!k;{Fbc?Sb$AkM9<j^5j)(v48fO`XopKiy4K}F9zE=3cvF8K3RC|g*N=%vT
zFN5%`yK+~!A|Lg%9-0!8$U5PrVKdK>J&Hn7Pdvgld1S~L?=1<7nvt}eXJrJ-g2i8x
z9Bj{orilMk`hV%?g5`HJpSZoS?&*HQbSUyC>&oLEbGhF0Z!!~lWpzaIo6!OLF2ygL
zf{k-0ty4bcv@-6?D__N~;8%BchB6*|eSFr7wKvp7-?3z?<S+!h6}ToBz_Rn5!_u%#
z{>4`ex}99+F`jI?*LQD1#TVTfMQT4}yP9PJo~|t28hu-?lDmF$=mfji)CY^i4=~u=
z(YSQ7bk(;5+!?|@wHLZLTdbNK^Y-NOXl|h?9eS@i#6L}c^6c7^8ab0DwikkaJsB$Y
zvtNXBNY1=JZS||97kcRq41oulG#{HYPF&h~K!!_T#Vv-Z%Xeo*7Z^QHy!up5B&uoM
zgE|!r$L|aVZ}>Uo6VJ75NIthkG-lIMKgYXXN4Gi!Z^>rf*-+!M+G%UZ%D@RJg<@N#
zI!+0%+Pyfuf#b_`-TEKTUL9fm<#OAbKR~rk&xBh=Klh^k+{b*g7InT^D|CO|r$v1Z
zTnAT`2;RQaYpe4_;+?kti-dDsGowzt{{5hG(;}9N;|&4-6t?O2tA3MSoS?4|{gQ=`
zrPrrGPG!#Ku)W>O9OM~__PkY?P`q$C?*q@NKZCwc+Zm89-@=!{@$@0X3ulcI=EWQ@
zgL5}+t*~9tz$YMkwRZ`xn(5uYZf!30#)@xmN!&Qql<+%)+swn@xsQSVTJ|DI--pV&
zwSALs@Jw+F75>>FYJ0&{x5rBFMnc#7%+HZq*G}enGyU?3X&N%k8u!&^JHJ*F`LJki
zM$svQiGr847#2GVPHAJGX*$*RV}!s9HrAI*lxO-z2AW)d{<7f-=S-zf%&S~<rYOj0
zO;2$7%)L^Y!*S{-#WD{u7uky|S1^`2K4E$yW@}~b|3odNJ%P!1r-{#-jhnU~`pkRw
zVA+I)rOB_%Y`p(m^p@N%)U{^rg6b@@pv({lZn@(Zr&KXVzMUb|VmN<s@`Xz-T8F2+
znYS-zy3>9IH_!HWNB2+HD4%j+Aw$6SFE%$099LL>F|a_DbMKe$d(GnK#`c_v)UdUw
znDo=NpSLMx#;jK5=7ba_<84u|Bu^Zh^UzS>#)AY`xhujMs)ypuS}*<(o8#nlHDva#
zJASVmUsar#WfEe4M{7=_Y1099WiOL03$jm5VrYA-e8?++{ml_J&qEtdCRVeFzU~z|
z;NAG=fKq09`IJ9xQVVO6HijxNNX&NPUB0Ox?S|t9wmDX_ekooq$k?kSSRCuY{c6V9
z@7f=yYGtjA{kBtj#=Vfz2VW;0-^ggj&G7JRoXm8#_l5d3|CF~pv9?gvXgqT)r0FK3
z?JVgN8u2>US>3Zjl;85CJ)6sQH=gUBShhi-qGr@m3k9)#1xC+~UhXO|oSN}+vAk!6
z`?{6!E~b*tTyH4-b*nvY64LMZ=Z>p}0jJo0foq;eFSC?r++-;;Tz~TW_WL~FW0)NR
zbNsbW{JC&`f92FAjx(Az#4>rz>F($ae8V|W@!$7_SF&1N-K4Yb7j(=@+a?vIpyA+R
zW62|Z+u>5+`YkFa?ydW@=iv3J3Db0HJ0=%P*WI^OT`|Wk_sERB8y0fSRLD|k+;DMc
zj_f-QjV@JAEn&%ZoVu<5-0mGxTC-&RTz7{^Q_}^{Yb2}~m|k}>J36lHbO@K$xZf4g
zTy|Pnm`BjZJ78gsR&J8TN7rh(oxNSM@3j<5Dw^1p7sQ$FWlOklEArO~`^_^_^ct>;
z?74m0yu*1_#HD7Lw1nx`4w*68PW)5WdwxT8fPKa-u{Cn45gXOE<lD1_>YHCQOs}2P
zl36u**}~f@_0_fx>Bs)*yPVtUw87D8W#qnTyP1!e8Pu>#E>%mIq;-p%b%IOM$*iE3
z;P(w3E23^KJYr!I)wPLR!y%%^f?M`y+nf1YbQk4!v~ApEc0PIA%&Ub{O!Gv07d&j9
zrNM8pMl|!O$K##{7w&!8bNs@}fF{njHupaVd{{6sQ!X-_`NE29C5FEI8EU?nbzIUn
zn)kG6&%bk+Pef>g%IT>*hHC@w&lXc)TAZiy!*$Jix7LZXx}GrGv%M8gvO2YOJLBnw
z%YHWuR3v@(F#Jt(TYgT%LgkY5ALRot{svjiwY_CmQI+p|c<nQ$R)^#6GJ0W4B3f*p
z9Dme0rQ`gxyGte;#!S$z=GimB<!;BK6~_)7nXvVh#%>YSa}&g_sIat6kkc-4F}NbQ
zL|gUg{WC{PgwNE~vUd0v>F9XRR6cy$?Q?1GCV^9t--DafpP86nm~_sxVE;Dfh)E0d
z6VzAq_h+5tJI%CEVMg+1oeCL$#(6rOH#757gij`1P8GMBx$9+QQJ>0V|Gzcmm)E(i
zVb;AmCGbvZ+ZjR68=()PKYQ=%ob^m-i{bxcYcID&vi-m2_W#mi$7iaRQ$(ZuH*@~a
zs?mMLG4V>DMNN#RWk+|JlE;)kDZ#bR=5inK_m1wCn07^$`;~^@jrM0#bdGWdUumej
zsvxYr=;NaXYsbhd(;eSF+j*pE;=0!Iy}Nfgx2?D%&Nju%VLS8L&=Q}v#O|JT*Vb%1
zaKmuHOZm$(KX1BD&Hm&k92mm4WkCSjZuiY>28?qJpG@&*)Stj46=r|`pjFA?1zi^%
z|EapE?|Bit_x(AU_stnjmIcCXndh8$2Su=TT`~|f->mV&c9Po~RjJwu>0YkepQX7r
z?b0h_ke|?Cp6%1ybAy%js&SOqgGHAv{`$M;Bx{t(caw*)VLjJxN;yw%oftcXaY8Yt
zruJQl(%Y9@E&Y4>8f8w&{MBZ#uMwFlls9eefny2NZ^Rp&KBuq4(lPCb!vg*yfyU-1
z20RX`9_ME+>~Ys}Vf-0b>^4icM_Y2=Dx;ldn~o)!yw&8;-@sPpl=*BXE5q!(J?(lH
zFDBe){BY&xg!#hdEgJKs!X7a_joLlEqN)4C#3^<h3?C%+9N>Dnpl#Pu(WvLEwkvO3
zIeFe+!*;`f&`c+p0}+*}4C1;yyB4l0cvRBw`D{6Ngm!qc^J6#FHw!hZT23Vj9%0Mi
zI>x`|)?EjQMaJfhc?X`aS35QRh2JIqDPo2Dc5N2Wmpmst(JUY$wET|>D~H;{Ide@j
zMEXB}%RT&m`;z4yY(|EPOzn9rN(G7%EY?ivN)uMBeaA9CcAvb7!r2f-heDIP3!bJ}
zX9`=1ocA+N<C=DQN;&(Q%k2}G%&G!*_I>wS`@*y0k?8*qj?O_fis#swKVH#%;p2Me
z&VgkMK5}0Y$^Vn%AHQSHGY<QysX9!57HbKdR89#L*yg#JqyO$Vsl|)0e(WeoVKVpG
zq}{lj!7+N%A{obL57>%cDeN~<$?2(SQvNw9yn<Qa*7n#I{u!p3d-HnZ8=iKzOxexV
zC&7K>hWNK^w-TSAQ2X}2v^Sxb(?0Do{jj{U$;X53y5+)UQCl1*+%Ng-72JG+`LgrY
zpX+u%<;dtr?NVmDxV3GytN(N{E7<^LfnRN<yIQx-cInDqY%(Rk<J!cY!h;GC8ae9!
z6V@Ki_A`1sHKgU<&$ohF>sr|kg`0WtP3hOGT+(ynkNo=;sxdnpB_%sDr8Wc~Y`k!}
z<Ffb`&r%UThw98Uw(bil!OUNE_5VKM(P>j&;Fh-Fc$d6%-)X5^9vdFc4D?i;`Yyug
zF1MclmAx<7mc9Hr<C&?H59{Q+HPeN+H*Q$)rLgwXBf%xpE0ZcqB95$kti8BRVv$MT
z8ZT|hDNMikdDnRzD`vFcqBEf{W9IukC$xH3eDVD8Lg)LEy?K6*WU^b5**rg6ir!(6
zZCLI+dDAZM(*<i9(gcH?7-qK5z3B5m-tFi^UcqyzZ%j@1ax1?Q?3&RbeyKn2@LR<b
z{0G)n3(mN(@ONhF+y&A#hntq2PTZ}q+1ym{YD*>OQ33V0kCuMbaChDl)W-Xe@wr#k
z#98IDyNcto_!AZ^ZH(I1W$&2zxpA5F#kj*?xMt=?iOuDC_}J*9^Na5C=&2sAcN-0S
z6z16U<|yANjn?(DYm|Az%)8*wx>+l4nb|I5%9DMO>M{9A5MLPEiwm{e<U~r1EZqyQ
z$?fflTVeBH-QVU*|6We@ihp34I_asp%(v|49$^vlCT6v`TZQiJSRfdts$(2~GCk`9
z`^?2ji#b<6ozz~`<QR8jO^4vs<u-G)|8$8?-0{7sO+%LXVE}ilTKBBilQ>SO7aKAZ
z3%&J`K6pH&EV97-*>W9$bssI)*+1ALEu=WP_F=t9xNOjw6RA6@_MJ)eI2R>r<(6?b
zVuzT9&9+5u|1#2(e#><<tY}=t*d4I0hT(}rp{j*$6~lyS*{AH=xU_GXifZqAzdOO`
zO{v!x#%kjUOsefgwl_ScwU?}s3Ym0GAv5dJ)(SPj3ob4R&l+8r89p>^t?3bRx$q|b
z<-_^$oC*pHJ}OqtQh4<(W?$%`2Oj@s>Cb3gdQWk>h20ws`Fkwdww`n5Eo@oYeusJ0
zv`?3KmMeB0`^vK8upa9kW*x(Y9_0zUGYecT7pnza-NwCBjzK*Bfp@@^-8V`lv+v3u
zTe946>OCh`9?AL4E|2B(nlio4D_lG-kuuT3iE(CHpOmHO^6ncvOwwQMk5rY;z9ela
z`tgB9kKA;zoC?1wUycN^Of-1g`^dw8UR3$QMOpV&RQ`30|MSaA;H7wU&T&Px2}yQu
z-K-``Z8=vxOZy?eRM~0oa>h6Zm7XHU=pVeHEeWx^b{BVa*#EOq)Zcw9B=7Rw6iI(y
z{*@9bj~k1eZVRe=tX^<d>$sq;@5Y>nmU4f6r@4L=;fkM5{`5R@Naw+QbH(Flvui$9
zRy5^J(%dgKCGPG=b~Tlo+ur?OoH42TyS?f4t6%2Db}xE(hVPAX5Z|Md;Y=JJ^KK-t
zF!ja%m)hU^M>p=BJ-2~UOnKIii%yp7=gqxv=K7qHJ@c$TEDU+jQQo1h<@fkoN{Pg|
zidWwPFA8fEc=1b3o{^X5-!;>iaZ~kf+Z*eag+%z-bzX8-miBR&`jXAyR>>WK#mk(z
zB^~c7zAt$!`GPaJIr@mt@=c{5mmU0*vt3d6LG;SY6MtXkk=|K7!S`{D?M_FgK8vl=
zml9uxnO)rzoSo-el`G;BbLlhJ*S>wnKObD?uJcFUsiFKnpGbY$rGU69FXdj%gRXlQ
z^)+{G&b;KFo)LCRx-i2->G8UYw+k=$wRrn$b(~zu&K)7(+RD+W>iPAExvb5r&X$Ir
z>=O&p%Y|kgy}l^&MEbF3zpp5>7lvgjlvJ?_EO)d1eL#dEWS_>*vZw=BKG(f|Vd%@2
zHkWnhX|H8+osMVS9<<(BBdWA#L3O}~Dv2tGuC<8_%lIRhbp+=WoYS+=bT%ngb1Z4R
ze>T?khHFOE-=2wGX}>LGc8JvKU*J&@^iP_b8^rBj#@SQ-+wjr5{(zrX${)Av&8_U`
zjyX_O()Ye<&XV8LWTfiq<gNt#e`zRFG5IOm2AQ)ZOD9d_6}fpK*l2z=U+Co?=4CRU
z+*_4pWQ3TiSJ!zcUB0kW=2N?bd<n0GYo~0CMP1z<oAglC4|6t#o_IMgQ08~a2M=wQ
zN~i0+iUwsFG5rTD0<^EL`5swv+m-*gM5{1s(1QZ!$0wd%m}VC9u;lqprM`=kIdac@
zpD8UH?E3d;$>lEI%y~S#35<5<nOT$^zt(zYe1Go0KJ%WcsignWhSQGxFL{zYmp|Ur
zuf|Z3mo@EOz9sV>PTQ#olNU=na>$kHKc8Xwu#{hB^N*@L>3X{la{~FR0~fZi*ml2p
zT~+Yh&E>Jm>fmR;UDhw$*e{oBa@{6IqP8br=yX<#X_H`uKoMKii!bHkZ=D#vU;MC7
zi;ZUm&-uBL{7*{m+H6~Od`H>+fG3CkcK$YHE^#hTD!c0IVH;_2vR^Ln#IBW>zWkXj
z$9~CH*>APF+(i}#nbj5N7m4>?nBQ6Sf7!B+PP=#iXBFe<7h2!{E}VgZM?KToIl$A|
zSs@_E-_6s<m4TsR&eZAb5y2<L8lF#6QBm3Az_O&}p3BS7tPDn$C0u4sLLM55DlCH5
zn^qi%3fP$78t$5t&6u(=QNKIkJwLa`sa+8p7JgFWT(J6n_514Qzn|T+Pd_)u^7);e
zldTz694|QXC?|xuK_+mYikZm-OGV38&ngxH2L_P^+&&gDD=ie%)Px))cl{5(cemkM
zz`9$fYyaN)AMZN(`1B%1ha`pNAD7&bIl<0&;O>qjPdkPdxfSNKr|Gx(v;;Eca`Fgl
z+BwH~${WS!CaUgQIggSfCj>JFEG)8^#KUmqNueS44#^AsQLHKKg8mv!cX)12*M9Ip
zp}x&VOI=-j(=OAL*IVBIN%?a){niG5KK0}oQ+H~(sCsCKrWY^!KmX~E`;#W?O3ymX
zW46jcNKE^DWZd>OcQ`DA73T{tb#-;V)>PwrBKXGXo!$!E-V$5>ADFr+;Vc6)!<18x
zSzH~=8gjHw^>xWy`Mr$W`No9Aw-+lC?X2T0#h!W=CA0rs+**@aa7Un6rEBLCv(!y%
zS{fYZOS9B`n{a-al4H#zBfl+c9cD<dM)H2L%?yh<vf%7zB}VP66K+b~Q;+k^w`9Ad
zs<_ch`izac)fDk(ssfu9vsoI&3&<~%>GpkCB(m|P)43<!>MndoPd;X6)&6{_>*u1D
zgFiNU71kxZS@1Wku<XnN+iPde^G$v2tGL~Z`%&AmZ_f@NU3NyC{n4Kp6{$Hp%GA_t
z+$5RvxRn^WSs2wO9Gu2h<ftaF?VHJ$1M#Qk2^?S)n!xzuKwW^biIV3<<)-Tv0WANT
zj<8>PSlHMVXQ$EdOaE}x)2nhzw5zy9ls7quKamk&-1IPiX3~_5ift|*Sh8B|&z@`D
zn08q0{cq3XA@Y|5n>!tk7$k9AWV?R6wlLpDDDH1!@gduS^9y(EyD7Yj<86z3%j*`t
zLvL3eN;=6u(fFXn1;%6^$uoLyf4?d4esNWl|48kE`WMpa4}KkxD{nMreNb1B7rx@J
zUD(ZnWB0QxUv$55a674d;$O;ZdF6o3p~bS_l-6JQzBS<3RI?b}YX#@~D>v+0Ezf^X
zxo*o5u_X2ln{RNwvANb`%Ai)plYK-l(fx_d9<zBG*N$Dt+1ORFX@UGh|IdYN&!>d@
zz6yAL;q8aY4g60&bANuS&#fJ96RUJa;E6HwgS@lpn_TZ)Ts*^%udhIS>ifT^);{HR
z-nd-3hx>j><#EOjI?bng@~!ypH1m}+I@EvnP5&mP_j@UWOy=WLDy&|zdiziO54`el
z_9FJ42lrBWjf<G_4oLrKP7vT*<Y?o;7Mw7bwP3E-0TTn^Gt5gD$mp<YAB@@{n8Rp&
zAa8@bNkduz(;P-|N9i4W?+$If#2MER_kr(=GKZ5>28ZlL=N>`sBYcHUHUjY~5}htd
z6H`<~Pdd1C@XYKy<9=$QSBmZ?$(62k?tIGX6Fz%b7fDxklqj#C=&#Wh6qtKOV5`gV
z1#%e-QjNzJPrk@w!msP(ec|YftSxMJTdyrvztH`nbPIEB<F`bH;G+^VIMezo6LfT>
zyboS7<TR7kKD=s!R1W_&mib>o*D$|6X!povPM>jM^^Um1A6VveKYzISBUg>?KIZku
z{U3&ZY_HJ&H`PH|Lf{QY9E({ZlVY<$Z-Cbdr5t4$r8#Zy{nQ`IPV`6-SljXL!(DH+
zG@*0>?T*)-)t$#X-yS)B#7?MliTftypA!~&$^_Z1oU)~DOLs|6N$VBQUketkN?P-1
ziO`}+i<E*i!&U~}Tv-(IwChyx)VfV<H=EwL_Nje7arvb8iRhEQPfkDaE3$W;oaFj(
z;gMh?&CeREHLEq2Ykt?r4+#zR4YdtDzM3oa`Konccb8ZBeO>s<eU;eNX{Q$XEjqQL
zd*$tw$5(D&nV(@YOZDvhX-=n?>n^)`$?uZf1-lg&uez<?c(o#{_{yrQudc3Ju{T70
zb@U4M75;(xLF<F;7t}@Wb#Gpixbono2N`Fst+-~frZ9N1|KxQqS4gg%xj1s+&fw2`
zo*C;-nzl4-gP8wnm+bCqo@<Yn^yImvU)=fS=a;o#E`RaMU9@egS+sTV?wIQBWvk29
z_f6~T_FLq)&~NKG!MS{F$2O#Fki6NW+Vi(Zy2oFc$DO6;L4Slv()kW;<$JIDu6(`z
z^!2ot@2>7$_uI37rTCiZOSk*1cYMFkZe9Lr`?&g9|JeRs{$;DY)xE0!NuQ4N&iW^^
z%vTOvQ@EDUc2KO@o2|CZt!-gj(qYp!b>8Y83AcZ4Gu^a%`lL#wFZb5=wjR^&>6f%m
ze`UZt<K~&GXM)ef2{TVN5T1E5ZF1P;x?t_gESHlm?_Ku&tVWt#+RHTUGf`)%&i0-?
zKJ&cEdgJ}cj@v$L5Qtc_$s+sXy-TNzP8Kz$Ha|7ij@8~4Ix%$aRom=YTj!U~EA9Pi
zD!r(;-FMpAs@Z1ISFT+RONh>`zR9<>c(d@WwcE4hT_v7cdYP<ByW(5Ds<xrFv3Bm?
z!xwulPF&pWZr)R8d923g)Qr^RbA3NwJ$N<Y?S`ul*Daowyd!yI^5n<a>e1@!g125a
zO`lcV@mc1%+w*X}GQFDVyn5SXUaor_BfewXj=I9)^2s~i7QQYsFZVBve_nUl`L^=0
z#fKixQ9o#Wp|~%7^K*B7^Yu0B|Hjo<->y!teE+4+>{^yi{hMnW`)hq_E$n93TGh<l
zw{qX?8tXqke=O=u>-ztk{Zskx_MiD|H&{~G-f&e&z7Wvhv=R9+@9EK5N3G;qSX<eq
zvTkL|W>sfd&s!&uBsEDeNc@uYD=91GFBKAsI!ZZ_H+;7#&OW^^^?Tdy_VZl!qK>^C
z-4W6sH7=P23Eec|lNGni`?;e!P%LoKina`avwd-Lwf4uf)byYAe?0Qh@bT*7<wvcK
zo$IlccJH4pwcYq#Y)!(SNk4XTUgO-<GR@`d5zo%8o$n_%P12fdrPedkAp6kKl)ov7
zMaf*6s)ni2UiXhY=+ss(^DCMgV)gP^%IcG?lh6BKoIlC->4}i9E9b0a58_<uxzak=
zzIc(P>70EgWw&;1iPP)e-SxZa`T6LBub=(?p#9VTa_uGU%kxuzr0AS~A@+lF6^pIY
zw1TxCLenCi9Xs<q@?zxe&1{<{Z~Fdh#giq^WW+K%H@Pe?>Mi0fzJ0XpQBa}#yzqOc
z_QZL%q|6qX=r_&mS@z-nH{M3sH}yBQv+iTP$|}E1)z3O$MWlw-9+SIezZSjp?{?~{
zo85T+!MTi?nMTF;&a`i4O3R(sd&Y0}U8CbUekbRg+qtl~{LJDrr)Pan-{@SJ`tkNp
z&(E{Hc)XuYv-zj`W%{n5twEkI&$8a*Hq$#I&LzH0uPAOy+QsWeaW`9w+CQ!ObYApM
z7x#Xvf4hENZ#??4J9_#K-J2Vx><HQN<(Sd_RBJ6$?QSi3efjNs>R5|iZ+30hUcSyI
z{%O$GLebQzIibg2ow(YYedfAO?B~k032(o?I`C@mYo2SmQQzxq^Z!oe{U`k`$K+1L
zzK@sH9xjWX?QDMidc?JmT*W-i8<)147QViD-==Q$-DUgN*4&={cF|i|>!k8It4i0e
zy_>nA*i$x2ef8^mrkUZl-z~V?Tm9yH&b`w&qWAH?nf$i;U-hf^=HJ}k<Xb){OgQh@
z-Ta#GBLCE@%3m+Pu9jxumF+#*`?mcs|8|o~bJ<_XU*vDk_MNr5|AkCs(UhVsPhK3@
zxG1^y_|fClekpS*?ig)-`tBsB`ue$2^{c*?9C=xs?sRUBjda1c8<z8n&pzLKPJZ8m
z!i5hP-9Gxud);)tzb}71Ul_Li+ON~^rc2+pxjpOF@on;ZAC*5na&^J#^mQB8$L^6U
z-TT+=P;a!k`rS7>j_&^YrtI6>@N%wqU+&MX?k`yXX5X>DtJvB2v;L)iUcXm1Pu5+I
z&34ZhjdvIC9=~&3f1bzuns+)c7nO$En$|`AKJzlNxZI=s_UF#?*6|(jU%pTIxAf=r
zUDL1W^Z$GM|NGK=oO|c4S6R;%7ry6W_2$28Pv4%lZ^l0M_eJ-<?$`gl;Cte6_x}EL
zd)3;iTlQ~_Uw(e2Zx%mmpIgmd^9jZV3eP+&y8Ib=%uH-$8?N7GT%dY5zuk;MLwIVs
zajU|q&r6yP)F(at>8qH>kn^2gAyJQE;s@ze<{YsJD?jM{XX*dWzGCy|0%?)VU7IIg
z5*M8p!6DDac29!IA&cojcS^#7`o9MbOkZno<2HlL`QUk+PnGPO^Q4rQZ#y^rf%{MM
zjC+Mzt@|g`F);8S%?ybsiSYHYO3u&KOH9d6O4X~#Enolv8~cia#N_PM5{0DH^vpb4
zrT4q{D=B2A*eZpa`WpBaIHzW0dQ=sq23ProBv)l8Tc#-4+i}@cSXJZ}<ffJ+Dpcg=
z<P}?0*eZd|waP2D0txFYDS(xfWZNo5_y#CA=NF|anCO}48R)uJWR@8z*>Ne@6s4qD
z1-ZCEjVMY<vsKC{DJihh*Do(G*DE*H%P&gTH?*|0)Hg8FH!{*KN=ef#uFNY*tkBIX
zRt6aXF~cRbIJqdZpd>RtPXT0RVp4u-iLH_nmx6)<)bPxLl4RG461W8*KG^u;k`#T<
zf|6vDirfO%iV}Sz0|N_P10!7{OMPVh6}bhzzHr@n#n4bp&d=4aNG#Ad)HBe}%|+2s
zT;f`Wun((_;*iRMRQ;gT;{4L0<W$$P)Vva9WJ?QjN{f;0EG<q234z_6lai)ikqfaI
zS&zG?Zvb2mC>YZ-a|^&aK&p{drX<7F6_gg`fYqcV>!;?V=BDPA6zd!68KQUyv}6yi
z1I0fe8E_CF8()!IfL%2#yugaV;cDfQpIi#E)YHXQ3FKa@l>Fq(6e}<@Eyc*d(!xAN
zH_gyIN!KLN(o{FeJjGBq#XQ9%$;dR>(jYkz$tcgf;*!L?<W!JR6}bg^nVBh8#wJN7
zCWa<Sx~VA!sk$cSi7C2?#+IqN2B{{7Nk*xrmPQtaNJjV<Wu|B5CFUTz3S?ADW{Op+
zsj-1Ya!QhJYNBO|u8Bpmxo)C)s)cTHnwh0xvZ0}=iJ>LfC{R3DIR<#zDjDe+AVdOk
z5=+wZi*jw1d=rxu{DToPA(^?U!6k{HP%|_&wy-cZHMKM~H8(Y}G(ji|OD!tS%+CWE
zYG|Nm29`)kwsOla%1tb>Rm#jwOi$G>$V&%HfTG1JAhRMhC&DE&H#HBc)4~Gc9)xrx
zlC+_P0YthYx4_D|C^fMpzbGU>KgU)H<Rb+mJwpR<##68XWg*v!5-Xql<U~-`11nBV
zu><EiaNc%GOxFi<ZS?Ue#xN;3BfmTktPzsG@tA{93OB{EsH8l<DBHitKPf9UxdgvC
z*c8J}3QA4SEG{Xk^d#UZG<7zRMD3ATT#{c@iDZ9pY9WLN@m5YIIH(j9z(s{sVlpTc
zZIy}>%TiOo5vTx@O3p~kOHWO)Rf1+nn0O|(#GYuDW?*S(qMK-DW~ytFVv($yWRhg4
zn_^;SWNDI?oSK*fNjGrQi}TY;$`gxH9n(|uN^F(fGjj{T@vNW$Nu!#mjw#Pb1tmZO
zLknF)16>2N5Cd~7LrW`DQ(XgdD+2>1NR%ts=!1%4n3rupg_W<b6>^~1fC@7!Pf(f-
zE-fg?$xJPR1P?gz1g924OtR6(p$<tuIPK*V)Q+qSqT4yYpt2}4J)^|GJP)t+SQJ7`
z@X1V0%`3)hH?k5)upre~c3jY)3UYC?<Fe5QSDT<J6cQGoDvXvGngnTSg@VGUB_xIK
zXmE`N7fB&NlH$?SH5yzbg#by4M^hKof{P2$H&4w=u~jNpvbUQh$t}Xbz`&N|?e4<x
z9|RZdT|SwCfq}EYBeIx*K};2d89jqkJ}@vau$OrHy0SmwV&fAwi}6?&%D|w&;OXKR
zQgQ3e-SU{wuaA#^y#92PIY&qJ`@P@899#+&))hsknf-|}+xP4AnP1y-BeM>hY=7h=
zA~SRGnkzo8#^2*CW-^uk`F9{B#K%`y@BQ<~7at3j7ry%~$N#+c{PUW=gDFOlf?zPa
z=;x<YE)flf|Ns8}e6@PLNJmU^etP<H5f>21ITQc;CtEY)+t1$Hgfa{yrZjbcK<~~%
z`}p5~ij-?r6_h|=QvVHYCa`(8A3uG1w2_%zBX}9lzJ2@l{Qvix%Qb|_6>Qqkr>Cb|
zmb?hqcDqmBp6|fT&FMe?e!su@?(=8Q+;$c{6^XG+*$}mMmTU+}wMxHs<R%sOKAD}%
zpC37POh`s%&4*8tbA>fP7A7%siR;yzKd%|LXGc>orwGXIiF;P$-``jMH=_BV!`7^;
zD?(OILXxWf{_e)@`*)|vhpnD!SNluDWU2#5li<nKTp}6;{?EUEeRDH-;YyY+kOw{D
zuB{GVzrVKUV8VjJ$H%5<1UfY$q;?jk&zU>-{SOIVR#B)a(hr|KbE~NMAuFG3BGuW`
z6B41-2y&-V5_i@2ceTfV-`<wnJ%9fErXYx01)~Logqo&L5AXY)t+8SS#MT=E?(Xip
z%HP-B{yk^ToELX?ZgzqzpLAJg;ascILx&GLGeTX`p__SglWJpQqXbXeZmsr3Lg0{5
zs;pGMf39zyZFSPtsP%WBJ$&exk+I^CAk?AJC7*Q`3JMBZSXydI1VS9M(e>-AtBXrs
zUb39#1QimLE`Iz}A!=*ZQU?}>YFBV9S6+Q{bF-v<olV;2$izEtRaI6D^XAP9SsCQo
z_+&xQQZJpD9UrE3mgG!5SoQekBCzi(JyKFsK0iOde9G$XwjECoq-;zuk@)0$aA!Ds
z&}9Y>503-q&+EtUNtXpXWTK7!vO?J&Ez2VLV{h&q-y^*I^VRBW;%Z={l%N5s1oJ0V
z1;{Y`KOZz)ikQDkRcz6BI`!vi#1uEN?F>DvF~vKKc~Ac|jtKtu_oz`=c3X9KnuW^W
z6FW0sf3bu{-yQ4c$JrlUH()rv`L@OWcT<=c-$e5+i};>ZUej}Qj@z@1_B;2KNNGm<
z+wW;LWO-m)remjCyY6nvz3uVxmuxQ-1b~w7#BA2ZR(4;R7z+4bcR6-4Bt(4^YyZ{%
z^wY~Pub1j4TBobkpZ)p1{^_ZsC2xB4ET*pUJg`|<!(zwL!_gA2BPTsxWU;?c{@VV$
zOYTb`$*0Kvjdjc8oISVLnO{8)^xJ*<`swTQW9}6^sN%HQKmE%5u*nv=Ov~rJKYw!S
z?{#;dU&>LK7b+iDwru{j2|vDi8GT&|P0M%8o2}>RYRCK+bl;<-H05-!;N2h1hQDW~
zzTA9I_2Z(SQDu5lwd|e*RrQ<+sa5;#{2Y=67(!k*F5fR0e5mk)Zr5JH9?|HBugvCO
znKGMAW*2Kh{E_d=f{Fw$?=z14yQ1c3cI(Nik1Wqxmb|{ZGEX|DOYDdP+p^?n&)Zcm
zuFd^&?4d<u`Gil!TzU)7ZY<y7^6Jsm8UL$#v(6Ui1@l<9`{+zRSCD%3SN8Y2XVyg-
z$7qWko-niP<(b5FIlpX=Kb%~?ImdGPrj&Jae2!f{&dtz1Nwnc^`0Zo{zg+2u57G|I
z;+%H5+T->v_2bhwY2O!_eyVXoW>ZhfrTX}0O~+=tP)La<r_jH_O~PQZ#ZT2Uw)>Cl
zU!%>o=VRR-%axNiowv!nzQq3MuPeJ&GUq+N@i{-)jOD@KBN;Y-Hy@iC6}+vu<i_m?
zua<xht#v!9?)_MkH>2<WnU(+RdUjUS+Bn^Q6u8)^Z=Ktj<=Kxe?pD$Zm(O4I<mATn
z`oSNr9xwb_c*u+GK?mc8Z?F6gWuKo|^rr2n53lM6$-i3nQf)WuKcAp-iGS{a6r-ec
zE006f_y4`!H1nOBL$h@s<Gbs1XKrj~W;m{=%<(2h{_)wlhJQ`O)%H6->YaQ-VpVX$
zpUV|uXZM}4G3)60Vd1y;``WaeS_TstaFKmPndLoKUWD*t&s*t=mSwlME>7+Hdh2n)
z<-@w#_n%%^zIwf=Q1-9JHB1}{wx@rJ-&YmhFw4LA+0pV16)gcJG4?SwlXv_$Yr*%m
zzOVSUFvAA@TN9KXvp*2IBYEHLYC(x<5^GPRXyG>hz1w8U&h7irE7`NUU-((Dw8f?M
z^6kI-RbMWWY`)0<fBF1VW-*ulPF=&4;d$fG#k#!~U;YL8G#q@<SbVc<N4QS=;_cek
zc1LWxZIr&g?VIepqrcZb@|Mcn&G+!ibKkC#b^WrLF~6Semw9!)`|x{x1z`gV)<?hJ
zL|;EIyJfZi!jtPCL|zfT{E;{E^qu+>TT=_yNcmjc$h>9UDo7IkQ|xDZJCT3My7*b4
zZ?>ILwlsWg*Idh4V->#dt7r7SXMg5L@*7!9J-B+}^{>BIT|LZ^@@LnKEh5@-=gW^X
zdPINYn||#t<F(ByzCRkY!;cq8bzMDX@Or<D+&%{5$mjPWPd+#k+-&hFQ)Ie%zEa{r
zz8=5uc>A9*VU}C|dEPqtWX<gw?pXU<Vi!2)b49vezGSxUgw}HF^y0L)7gww5Hh+Hn
zyWq|K`H$<wPd|H9_S$A8+xsAktGNx{5&3`a|FOBPs&eVkK9!e60*sDJ89nZQP!+!}
zd*;_&>%|uTZd$ErU=**JFViaf;9HIToVf?G?9R>nUsty^Kz*{w!%)4s=b|K%S1NFA
z(Ox3@P5u(3fI4}fNA$zWaEa_+jCyDGSMkmMsw2Lr{K3WLRt)QMtrE9aeAjgroV>Vh
z$$`~xH!|O8-d&V<IfUVk)2l><-Xk-36T;>DgCfPF%6Y5jK6?Als_9wF$MrKkJ|5n=
zNIi4^(a96#8oPyOd_OvYZ9^)f_%;T%@;Cgd4`R;Eof2=BVta8vn_pabfAB2d>*v1~
z7w&kt;=$L`Rw?hSTDFFMT+<ny6{hmB;rCtr+t*ItytmtMgI??Wm9yWP1s_V(bN5#-
zyeogByi&G$Qy-Vw(Z5B(jq@BY9Tlti#W3$_p>~|m^{&Y05wrO7)E*x@9yNLAmL%<a
z%&*dQ)0@-g#GCrRUhB00)RH*ycVo7AU$=0CX50Jj^@#=B*nJ}2cE`W(oEQ2r>&elh
zzBjuCs>LTw5onpdUvPKO*TT>Lr%n0%^~0)1tjn)WsyV}?+Lf4p#_HF!ZI#+H&3hJY
znPp^n!@9vX<q=cN!AX~S^sl7|{r|4h$?){{<NxO(?l9+Dl%K2kz1;G84omlDy|)Q>
zi-kU46`D}_O-W>R3IFM7fA>b~=RG{~e6^v-hn6KQKJDKUJa<0)9>#UXvLxH+<nF#l
zH@8@@#S~;-;&WeaZBz3%?b7dS$4_z`Q<`e%)F=F~1L7}_b3E@VSW{)+ZeAu`xVq}^
zCbqE6KVMBax~A>v#f949>*Zda4m(qKqc{9}mc)l+n`={p`nMkB^oS9j`*p^a*W$+e
zKOJ__JAQxRJ_)ZsXFp9l@O$dSA0Z!(XW1S9>G^oWwko;(T;{$@ib}Y{gOm%3S$7A$
z{g5YecB$|5jrq~0uV2jh%5w6xpW^{ro!h==()lLtEdO2Wo^LgE&fm3nUmm=cJmb>A
zlcEN<?*H5fZ7xh&{7ClW7xN_B-4m7d{q}tQ_sT!0D8-wnA$DWh4)NL3N=x1!GZNn1
zqaC~YVGhgkTecrp2k@+Z$<evmFIC%uO{Q>NT;u7FyYv6rKE3@Q?h(_;)%tf9GK8rY
zeA3=w(WsipeLVgLgYkoVugaFb<VcVI-Q|?ef9&Rqj`J5hcYOCaa`3pYwr9+|pFg)M
zPA}h^vwB<V{<{_Bp@$l{^-8@a*1S1z*3>9F@T9`u53c{L1b&pe|I$h4zjHTd=_F-e
z9%WDi=;yTP{<Is6C+*kl_{&^&PUYjHuSSn$1*>Ks+S=Sb`+S=1vE@4NT<3b+-_vyF
z_MSQ3AA)!MlFrjD*XKXB-t(vR`yJakjy=r^_KLOs9p$-c)*AD&Rj-3CZ=7{t){gny
zg|9vD?(gP4z3*|ve)SzYUOixr-FDmaZqbil+1GRGPXFaQ%Jt`~htF^G31y{9axvZy
z!`>~Mm88+u{Q9a;g{GtSI!K~8QOoh7EnifA%`)YfKFf;V$0H{1+){IVtNhz+w)<tv
z&S(c_ualj-X_8COnYP#EYDXr{N!S;q@|UCHvu@1kRNnhrwC~Srn!l%Mvtcv0@$+<T
zYxDS~gXfMv{Z%BQHT|poyRU`1HD7Pln9mc=&}h8>f8Md9oR_YhfA(pO{*Dhf)nn%E
zx2X=_Gd-2LV8_E3^M9T@{#a*q(z>$-S>dZB<;=ybomux}i?SUM**|GYi?PbTnh&pv
zEFN85T>kBlgka0#t2z39-8+9tv2OhT`ols$fj#O{d|Kd;-@ss|n&9Jd?PsB++M7ST
z6B73RP1c`tv}S9v<cGumuP(RvcwLp}Us0K6XZ!9)(U&-1uhhNS-V-%>r$xE)yu;Bc
zyn9QuJP%%+ym7|a8?kenKA&pK_qdaO&~C-Ot=U4iM0NU`rBmYO<{X_=^!LW1;>`<1
z*4~n?mOr2U{J7*Ct$>tMITn(KFP^`=F80S=(;S|qhw>t1HcNU}39Yo(e{yF_X!+ip
zwnx4nb7P;~dM(j?{qWjXW<ihmp9U-_FZi{(RHS|rYuS!~2S>{{?D@{7xt#0a?^7$*
zoU9P~|Kv<{H^Ye;U+wim|M9$E<acBGYljCHM9%p{wHUsT<SIR(wO8}>K82^Ro~Pv8
zV`$%>X7E*S$;qU5jR&)~Mu|3qO1CGj6DLkQGv9vx-Q3wpCU>5eD7FcS%=u%s?q=Fz
zlj6)vm$#q!_tGp&h%t0e-dpp|-4@@@{K#2)_`;J)FT0BkOC<k4yu<sP`QOoXPZhep
z<aY1<c+9D+^T^WR9UCg7*PZuSk`Q}8dfxPHi9*ZwN3pY&?>S!A<L`Mr`+MrU|MObn
z?`U3G{_kow_p-v5vp>c}l?$vdPkSg`@!V(9^IkQ@X9w=<&5k`Su4%a7TW<W}E$M>C
z|4sj_D*fiww?zs26Xw`%_WwU~<MV32|L;$~IeJ#0;dzJnhn0akw<ow}|FYCOH|N09
zQufV768|ObT=VDpnPlFpx14@C@UBeH&At7V>VA_aOlX)te?BDTcbuJipnY5V+rHq>
zt5VkdmV7NMmmI&9tIY4f)YB=)-am{hKlb+2^#k_~H~f52b|$>P>*mz;jm^uAm)X}H
zoWm6I!gXV~b)t{`nVH|0Cw|lJ+B;jC&7Ij|$D@cXN^)OM9b72U*Ejd$foX61PRH(H
z`o?%K!T0!{=ik>nJ!80>;mzu4ht6@X2|qV!=H-jW*cI%jIq<TJf_ev)0&7k#S^xWI
zzI^hnpQr7AnNHh#dvm<c-&eWU^*Hla_&qq`Cvoz@qw4>vx${}Sy*v4Bjz~j)+T%p8
z+6AY}jz6{gr7r3I-t6?|8D;aXT78|6kt6{!RWLee50{@`r0|*wmoJC!y<Nr>uus!2
zS?%%L<Y%8Ash-l7zcdljw~j6<5r6q|Zb2badzYigZyS!{ETL&7+GX?RN%Tfuf94%k
zmM`8eetV%Zq_4c~$gbwgi`6GoSTyc`reV6P>cNA@opUuiuiSrZVYo*A`Lq0>u%A=<
zAkC~Bvo`N9`nNUSlK+ON{!G*2j9ZmDoSvuLU}Ft4pT0<JRk)d9u>zD(JD}B8M+dZj
zLR|r9BEU5KiQ=~|2heE7r(K%u{fX%Y8XP;*Kl{xI19x}@Pil9GYO@`1|J|>_von9)
zx^+8t?Rqx*d-jH?wPCeyK!wjF@z|Y3tv`NL=!kV6+R-bl?ss8pc6inIcV~;w+b(w~
z3RvXA$t9*UBmdrdNWgf^d-~>$&)r?6qAq-VayA?e3!Pe5#P8q7B<LV~7!njT2?Cg*
zFq$R-K1dArSeLk-0~D(+pZ4|i@CXSBZMs|b>q{oXq)C%33Lm*_`yCS#^Wp2)(1M>6
z5QU}3y|ATTqGj*z)%u_R`LJC+<Km*$L#NYWeJ3^xOUt{Ti*0OeMdv`%(-YUfzrOB%
zeO!u%?bLLA|FuzDMG#W9RbLLAd-tyF{n{|?+FxG+GoWJ_lfqfq*e-2KJ-xfGW_{dV
zky$XqQp<}=&L<cIEccT=HC2200ccK9(SLnytu!AWUwKVWTT>TQb?=8ypB6c{g9HyI
zEO76a<MLJw0DC#d@%_EM?<?xN#dI}dc5FZz(GnADn?3t>?Q|FDAksveBS(+6u8ZA$
z@0%SvJG+vSlE@^e@i!*T(fbb?0_*SZXMzuMU0N4wT~k||zcbd^(G_Cd5w@GRZ{Ig!
zUeVpnJ=dx<$O77)OX|+Edv$%Ow|Go!tY{B3UEElt<~u83OUA`)`{yO|7Cf>4)}`=$
zQ-sbOn@XY8S63P2bj+VGucxQS<@CvP^UX6WgO_iq-E7Fn$Osw@<JhRF?mzEHj@k2@
zPx-mIlP@jtWC&RoGc);k-_x7lgBCb2%ATxo;0gQ7dTy%U91F*c3=QkDHvtl6$1b1l
zKH4pIaaE{xtK!c~jm+#EnfK@Y{rydoNBZ&e_vbgg$dEG0SYV)X{`83x3$m}TJ96ZR
zMo-M4+qY-$*kSSN`ug<-AxydF*TwGU;NyIF<l|$JHP<B7`yE=7l+P^Fp7XQTjxC$*
z@^b&>>F4Khxqh0e8@=s<oynBq=jQ|?Y?<7*GjSbtn_(1WvANcc?;G2*^RwL}B4#*E
z+_T$mwvUUO+op_*N>Zj-E^DK<e){s;vi_gVym|91EG(Yf{61mQq(f^WHy4zbr{CRG
z8nQafS9_J$g0~U->uhDMOO-S&TTk6p3tF<xt7)yL&CbcW_uVGUtC;oue*OH3%I;bs
zC&k0oL^y8m;J9RbCglB3Q^rthEfMMBmnYW$vD>S@Y)iM#ER#+}=Qa(Ilf`Sov}=EV
zOFh;jS@QZ?Z~p$jV%FBaiziC2iP<^n#0ig@nwlFM5}BW#o*u68-7~vH&#3-i&8@B3
z?yp3RxBfmK%(x|RUhC?M4_02?@pJm~R`IxmBOQWP`d&HruB;4xd1Irp+ME@}6)U+G
z^nTI{oL}YqxZso4F=2n3gW2o%E(=gOFKLz|VVrhmLGABvHGjWe*J{}55fC77d71Cy
zJ3EWFWL|DlcJI@e@_UJL<IW?;<?GLM2r3tpd|w{+pS414<-&lRn6z82+k9W2Tzc4E
zSW@!jmzS4K7dG8kQuz4T$6ML!A3lohiDF({z`tTaKu(O?C&{|sYE4HbTT7T^2uxP<
zJ#;Uw_@d{#?}xbcKOAJ2zp%)aTTCyeWAo<CPA6N_&doU~>~G^Z(<l`b0<B!4K^j_(
zmEBjfQm?EC1dX5X_%(IuxdW{`eoPB6_}OClnalb)&$Bn{GG>|QpPQuW{o&iUFoPuN
z*V}KN*^qd6hEXb)T<fZ!Y6lU&c~-|Q_Zuu#-(36b@tpg=Z5J+FcxJXazqz?NtMblk
zuh;KCcVlC6=7j}}7cX9niZ9-&zyHsq=kx35otmn>x$bWjm&gUnXIhPwE^cmYe}8{Z
z|Muo)h3I<sxBrR@3$Aj_dB2ZiL&ObX+pgr<`<8f3)jD$QSlaGo{T^{sX3g?CKhL(a
zukYCQ9Pcc<Z?fOc&Ery=zj21~O;OW1!TZlH%)7fwCw`wyeSJNv^3HSZ@^vRpOjKSO
zxR^~lY|Vkwr@a?GVH8&LN%->O;+1u=v$tknfAn(Yeq*i1oiRyCN^`Bt)85|N`Xiig
zY1R6C1FILe3V+^QxK;RuxcRh0u3ZA+(%x5Bg^G4P@qB)6uCk6!Pki0a)&sL_SKO*q
zDsy9Jo!tMnTlVxqd9{F5C7EGsV?1XXrHXVtkqTbs^YEnl{3TAUTx()>3fbG+w@zx`
z^YNJUp1<F2n`B;UnW`QB%Bo4Lap(1ulhrft@3XZod$S;;xiN&RUcPy5;6V=c?Pn9O
zH!c5IP@ep9lIoU>i%ttY<`uoTpxD&hd~<ugyy|Rw@15Ts8%XeQxmoOb>HSToR?yzw
ze%kRP0qf&zTb26noSA7{`RC(t&A>%0tHak{+nHK<WkulP85V^~Z*Of?wz7&U)>*;w
z%&z{=M|Q`92W9NbdN#dv3*L3Mo6B{R<^Fu$+v#79o||>B{C=%_O3D(ai8c?n->*|%
zQd=z~B(&nf$6PNHxm5uwvp0Xd_-2MNhq}8m_p|frCMF>ZPbhD$_?V=mq@)qCL1A_H
z`eiPhXHQO;&MR*>XJznmAvw8oO|0A|l2`9vn6v%=qF!n9ODhVK{pMJBPJI`@g5gxI
zsK)iSUH)qhJb2>VFK7GX#o~UEt|wK6g@q>B_vU;S*9%^JzC_ev=h}u<vMY@%G}Y$3
zxUy0>zW(o4r-|;5*1tac^VcsSVd2G08O(M+p9m`{D@Sfh;jI1r?P$_QjVaa!eX`bN
zYmzE{i0Q?6R8&~JoEh->V<bx$ceutRW%ct-B5Q7S7T2#!y1D44imIxhh{%$}(n;FB
zv(0)z^B^0G7k+DC{Slh6IzYu;@1E<&^;hO?-W|T7;2zJLH*ZY7#QqDFHp@}a(cuvn
z7f(t`S`o8zlhZ~<w&unS8w|?c-8p%vmD{rPm58kMt65ttpDlg3fd78_{Muui&)an`
zo^5aX`~2b68M>`X<*LSCYp!i&>Hh;t|1-^UrTY5%xZH|*&F|MtzPQ-^@tfEwx0uBq
z#BG@5G;#N&c@?wnFDp}i8Dity#`CaSf8T+OD?dMeD#*RLsrB00=;VzN9T#`5+_p2<
z(|>vN_PjN*yQc{%yM1`rF0Umr^+5gft65K<PLF?fxBULmtqz%4{4p)rjgz?C-gR0U
z_sVW&pSZ!gZJKWMrizbAzOlOfbB-n@{`l~)=HpTE3YM^i-wtTZu`ylPbVoY4N@#!m
zcM0)t{~eC^$<8!R?@RkOxn0f9Zl0KK)Pz~HPAzin&bYT{r_;v3l`B`4yt>lqJKM}K
z?To~+Ug_llAx!^eXU?3-BV#c^(m3rw=o-GQzs?`N=o`#;y0&g%!p0BBCy879JAFN5
zZPe5oHzK-3G#4d2ndsr|y}11SJ)O8c68DYHuKfAKCzd(j-G6S@VzrN-=Rf=MzIbK)
zemg@$!&!^s<#yEm{&sOqr181=_UG^HEav3o4Ad~wGc=rd;X=T++}p=?zu$MdTR%&W
zZ|Z?|9Zk*7b+NlQRejC+qtzO)_<BfB-><abe><D+??3-<YJR|yjKHvCM_<~o{k$H#
zqW~1aX1TXq5_&mSmHlV3uI3G#R(~vO>n8ngrhMyv&oD|oy~#+g@6u9lZHY>SJ~>;j
z%gcN%3m>t7s$=izdR(gWE$aW-96WgN&&T8PHUEA-xA}S{n9Ei5!$XVGS0Zn3Z%_aI
z?XAel`}0rDwZCG@@U+otW2xBg#chvYzI6O@X{kx}HJ^ni_CNaY@Ni2@%ZKOl>w`Ak
zEI8HLcRj`RQ$qQ*`dMe@y_o*t`}g#tpPpV>8SK9BgniMkFPXDUvyc7z`+KHMrBT1_
zw;4_++kZcEbaa%lshIHc^779=9`|dDOg+G!eSV%T3oCo^n-_tnw$!#7<;I6;d<vNJ
zyV%q^?c32lpR{vxJY8H`oF=ws-QJelvL*lg&CTg61-^-?&f*CMsqBnDb<VQ6uxyih
z)<GuT+mnBnJv_t;s(04K?*8%NFn{Hzlj@>gcBNll1m4<~dwON?@)`E^cCow5&VHZy
z!m5$~gune?m&M0=E$jZUOx<0(j7fukYJBUqE44|MZ0`9=%E~{#T=p03vN2g5wsukE
z=Vx=Q>nBR>UABthlym5YEcI7BQ9jF#f+A_(-*3^aO5b%14JR&GpiuVq*3lh>k3$$f
z?|gK$dvndtqCbEBytuhpT{~<|0OysgkLPvc_RP4Jduzg!Nq%eh6>CQMx6gd~@Uu?T
z<SPHQhn6~I&a3?v>DDXN%J=4Uvsv;n9w%q#lLpc&Z3`4xSHF&ts{B#oWxDL%1q&G)
zF2jTa49EKA*9SaVw{+REOADRZx8>bEwIXn_gk{kZC&5$GbiGSTOy0eFSMv7O)Y8}2
zp1!s*k@+J3>(A%&l6Ex~e0g<OUYYZ+PueQl^=GwF(7BDdQT%%Hcb(?j)y}Xg)#Bsh
z<8u4Aar^zc+2L`Otm*3yU6B%Kbzh*SG->Bc_Fu5I0QKQjk2LFkFO&9MUT#?OBEatV
z8{=5-fOmh)4V9bwWUV(9J@snil|EMee(&-CmDy{fw+l&0aUGatnhomwc`djzJ3PL2
z>i@s*>vxE0xUOa7F8qAiC+$<l4CBi;46_+H>(?yEy}ixF)%D@am72ew{)^BN1BH>A
z-<%HjgI*vfZQicusnp;0(o1Q7{q=Tk_G`j&a_8RM++0{#7-&%RdRLN>prqu<MXud1
zuCJFDR`<K|B(x}6+B{Fi#zsb5T>RSFX!osIS3{VdzE4d{GfFuju*`RMTlmGOD^dpX
z?&kHZ%KcNWCZ&CU^uOTI_577_d#yHZ+$f^8q`Um|-kn8HxoUrXNql&y^~aPvueJL&
zZz%s3Wbi3t{|tjv0sDH1<=f9MPdwbF6TPj6iIq!3V&}fT2%Rr)Z-2j)k{P_*@8~(d
zbX(7JVQZsYzrVZt@#DwD!)?4%^yA}H1B#~2nd9T)!ctgR_#>2Ok=y%xL#thy5+_ah
z*3`8#ufD%1;^U(=J>83wj&^amzB|>*E&k%hMrHfIUxZm#moJTaqVnd=T#=q9Tw$xH
z)}D4Rt~i`NMLWE2s&;tLrW8+BWd@s{PbT}!G;00!_O@~QIhi+a-->qqS*z+j?Zb`a
z{=gjtiDr3sJUWF`Lzte1rJtL#u=e-2oZH)a50%O<_qyI+%Ik1)YG3}$gYw+-kDfj~
zIrZX2lgvvl3s1<(+Ez_@@F3yao14y^YKJxCzfEfk^juf+<hcHi;+xtmEGz=MlUTQA
zT@BLsd52qn&jEJ%8i&)<bR%~ZDEj&R`sDTQ-kQkG5=JQ;OTDM3JwDd^GfiZw*Mrzw
zyUX=K4GYVf8JBY77Uk>?7wvkI_55GV(Qh+rH=56wX<Ka;yDKBiedpe{x3>#l+Go4m
zf9|517ZqM+^==D2)|I6lS1LPK%_N@gE~XpfvAgW8h?m{hU$562m%WLQGRbf#DKW{*
zlbWmg{oQH({ZD>ApTE55>8UmG`|aX+%&#ZB-2LJ6dHc&70+svZEWMtto4<m=Yu<zK
zg|emXM-oauEQ#Kp=QGD5aN!BvegFSe&oawCc60Od7583Ci#nVX<_df7^tQ44gn=|C
zFYnXc@An03{Jg*}UvuEb#$-_2vRh0yh;hnqvy>A8_i8@-mc75{ySwb|j+&pF&hAft
zd}Cws@mGt_-@LTco0FSc`E6IHRR7r}0cZ2LTtD%ouXkbzd3j=|ZJ|!*EZ3?ktK3^#
zE*jV~ZfZN=;^LC9%_?B^|Cm48Tf;RbrSJG5a(sP~L3+7v`oB3ZEi5fTbr<Vof1i0a
zGt=kSPU{lY-c<9msHV17wCj&Due8~btgWEl&b9UN{_ErR=KPuzt<_k+x$))g?fPu)
z`mcCmTJmo`<8u8Z!yYcrlwJ1t(Zb#}k#_T&+4+UU#Fk~}J}rK~@3)@6?N^b$cdVes
z)xD#ywM9-A&$;uLbIrY0?%&roCB2k6H`m(T!=q#0i%o^b>E}`|EO4Bn6FKR|jR-C=
zofQF38Z++gvFz;Zba8e4_<H^RON-t4XU|RXc+T=H;mi!f9(j8?@%xjm$O*L0u8(Y0
zs+?u1JoWA7Uxj(+y+FORkPs0$IXO@pKSHNR!m!ET?kCIlhAxo$&HjO`%9XQTo=V;3
z`~K|FX6cugmU3@SJA3Hn&B%i{798ys7ZwzBJlMn<xwoox->kEN|7;Jp^QXVM0&Z%o
zSfMdnr%9{P{J)~%%!)^y>YyTXo@-W6^?@3hyUZdzH&|95J+sdBd~wm|lB{cMPIifE
zbMWwl82ogps<P_0{U*UJrgLC<NOIP<xvkCJ{Y`?i6AV)~6yKXX=jO%<$9koOWo6eY
zHkqmE>Gk!=T0eUG_Ux&t+Mn;1-hC2(XI_L3kCaJA<>zM?SBLATotcqXe0^>R=N?Bd
zFD^kr!6V0x1ubxj&iXcUT{c6PuhoWE^)eBk&5n9cNl1;65!DI_&@g-V=xBFKd;8|Z
z!)#n(uYI=41()zTY_tqpJ$3KtxTh!j=LauzVtw@JQOS}MZaFtLG%i@6FlX-E&feao
zQCqXPT;;STpLCgTR~xY{N3xlnKkd{MO*7LgF{&R{ZB9QgBqep~+S+K%nC1Lach@z|
z4HTCWy<J-Kx#VBO7n^(c;*RJ<yDfI>73u1=DtjXVZKH_f|4|V=bnjTg@{<dXmdW1x
zAX%OK&->s3het`EOw#6b^6<8tn@+E<t=$-*bL0rLWAeZ2%a;-*C8Zz#e!s8$_w)Ho
zgTyAU1-hU<4XCuRE9tnjHf}-A2Z^;`C+VyF5j@@(`M1CzYNl~I-@}IwxsD!tb7$w|
z2NMMR=Gu7foKQF=>*pM&i8T+ueA9UJod4+6qn)7flV!frt&`Y~9zTA0Pi66?OP3<H
zWC)&}oju9!XE~_iT^X!ymV4{S48vpz^Sn8}d)C+d5B;!y|35EK`=aOv$JAXu0gO{>
zj_waKC~em{oK&(r`}#VcStgzfZ%o*lef`ps%;0l#JQqZr*AHb-jptl*vG*49B9HSX
z0zyKE&df9x^^)71f8Xxbwp{O`BBRY|XM-50xF5QDb!scOc+l!F-Fp>}dGA%fx1C)P
z|MQoSprN5*OW6yDtSGjtzkQY$to2y?a?*Q?YSCl+y(FIh>q!0BG&3bF?Ui(~v_bj1
zm|bOWy_Q{_H|uYjaDb)HLXUlqA4My@j%xaNdWLzm)w*@-K&AS@Cyeg>a!XTBPn%;`
zJL~1;<-%gia%b3B`T6;^w6_aOOP_vobF*f|hJ<<Y)8^#2H`o1qI-N(>N~Bs`KI_}u
z=5o$hW$#bhd!B6mW46`0Q@w6^{QkO{uUEs(s&i$czaM0m|8R&~|G^VUkG21}Kj^IC
z6j7<)UG&jnzwcYOPd9CCKA$nJ`Et?S%>43vUCY{ECcd-Ho*rmq23205^>QL7O<$+&
zE_-WH@j>D8GT+7d_xEWvR-VjWzxUXV!p8+4H67nJ2JZMd{c@Bp(+uOF8+M!JpZ|+-
z_3+^M{QUg!yLV#`J`oICJ$0h8yHVmHmOFn=2d6)9np4}QDIus}{8nhc|NEmyo$W8K
zjW$m^Gh<;k=QF40=jShX?G}4;Yil>Bu-b<s!u}c}Cqw&Wt=(pu<${{mpP!vw9KAhH
zt8wStlPO6rE-VBM?No@Z^LyLKT~MOGVr%3RHnGi}cf033elkrGG?MW2^z`5j3;xZ0
zQugxY%P+65_wOuz{wQ{n;?`ehFPbvA=H1&j^U_Vr@0{n}#Q0oZ<{P=IMAOdhSNWu9
z6*aX>yGpY`EkEh}JqPRmeU@)k+Ii{U-`|{ET#pX5a_dBHYWeo|_G*@AVVatnpkcR~
zpG90#@_l>l?kB!qqAa6infD;9@A-r8?*li5Xoej-8n|vw-;C#B)AeGNY;5LyeSKXx
zV5#!es=C&Tj_wvNdB?Lhet5nq+~U_|6|K-!E-^7ON4s8j{9HV3)-11r0t0FDyeCVi
z$36Q0@BM#~t|wW$%ibzkSj^b6#Uv>?IrHM8R<DG-$rC0xWMpVWZ_8O2@OECv%Wslp
zZtQFKD%C%}p&xv6^AGzUKYnenQEOx4mC~3J^X$jR$1N=_8?vwKb(zXI9rb(vN~%Gr
z{NaUN{x`NK@w9S@tCbz)S{1&2U4Y!gFJDTmN?#p0HC0>EEN8~qGy62Q<-fkRwxz9&
zX+zpssaMy}TNn4-sNu9Jc)$QEA#4g7f^^*j^itMbi<nqFVM8^mbsGPHgL2Q#&h<Xt
zCoAfuCnzuPU*N%^?muTi%!dfCtx^jET;3gAdH?z5?0>;Kc3EvMIVl8Mj~MVTRc&(T
zySuwV;}yTYzCM1WQ`k^~hs*6#Pp`DOkepoK=FP^hudO|89ItWn%k`tP&GVNfE_Rc#
zshH69@cn9rt~>1iPrY4fT%jpu9d1?r?#vX;U=c694S9F1W|`!kT6sBr#jWH`TnkRr
zYJ_aI-pZ!)z)OvVogLJ8YyG6ZDMIJV`}^}DwbsG(`JgUMrQhXczM!H`Q&V$G?(J!N
ztG};evGY1TU4Oaj>1i=LW_X;nyC1mcqQKgvO5acJcBy^E+k5!rY6*Y4pG(eNtvK@R
z>}>ad0D+a^XRf@n??1G)N=szo?~nyK-mh$TGFeY&`ts^(_xJnt@eBWW&#QW+3F@?E
zWN3iKPV{aoR)*~^dppCf)=FG2W<lxeYch5<5vm^oe>|I=4;o<s4~uFthMLy+KAB(i
zS>L^RdHBP~9ai~!KC(5l@g7Ps(wtF$(_OySrC-k0z~X1X_fAm}jq7d;JN`J!wXT8<
zxE>XcPdU;d_~i0wjsMwheKIGfXa;9oT_tLqcIHB9rt10ES5^v7R`-8)q*M61p5nyW
zH}6<BK0em_c<c4J!#8(JY-M+QC>P+Ez4QcgikE-d?=#0O_kXW;a(13+o-db|m&f%?
zadXt#JwG0G*L*n0jy&F}#2<h$@>*C_6lB1E<HpA1%D-Q)XWrPL=sjKU>V1KYn(?(?
zMbF!Qmzk;+n)LVA*B?KBu4J+EN!vU#@o<}A>M0Q$uF$358Z_qEFmF}svSAV1+!-5q
z*z)V`oLAS@p8oXo^vCbt!wvX7k;Xfxub5xT>u{Zk%k4?x8~>EJ#OHSu=k&&VeSUT}
za#ITDy?gh#o-zJ>(9B=)uvL6T<Yu+g)Af&o1~5C${`~Q{zw+zV@Rgyf#j>xjJ3ICD
zPTwg9u3QPJ`u4`Nq{L)jYIM-&kCHd6D>HaSDo=bny6@aVdAlVcFE4@m8#y;OaXn?+
zUGvjuu66mdcKNyk`-?QLR@XLPJm`9`!^S17yVdsMVvB7ry;p^-bjr!maqSihHTZww
z$;ru*RwWvu+F>esdV1I86a_26t=)T7uXUx&^UkaaU0qOG8p>HS0n{LD=ac>M^?E$#
znw!n5?y7_G`~OAB_Vs7ysp>z7J}_PM^z>lRz~rZfIX0D(CQJ}0dw0k4B3ttck$-Y*
z){C12e@>VA^-%fS(S2?Ym0UZzdzWTiUB&g(?(7W1=6$ulL0uqFpKGC0t4P<4J=5ds
zX8POzow6<W_NJVhn_^7&*!WLB;M~q9C?a#_U+r&=n~y9{weDHBvei|tar5q^8<yz`
zeSBp)GwlE0Y7Kt+@9n+4vmZ<l=##g0yScz{O5WclC&5Y<zr5Lp*RlFPe)r7l%cZ5@
zG1ohbUAx5u1qB@w5)`tptyx&{@e!Bnr{45=mCtT1nHm@vxHC;?iA^-q_Po1Zzb^!U
zhDL9El<hLF3(B5(Lb>GS3H?8IYt^59ISp#EJ2tavPWinh@i3cJ*(;Z%B%!IhYuzSV
z?G^1hp<GgT|9j5-j}~Ruc=r8%XRV~HEP5)otGjz~^6|bOKYnZ|dg=w5X$w$^-x#68
zBWE*XXYun(o74T%&dr&rt=FC;V_o*<#-1IHoSFCMEqP+H>ie%YrPALSyuy`7=Irf$
z6s|d?#Xf99!M$<`9=51$IV%(11%`x(Jbd^tCB^CN@pQom-I#?PHc9JPy#g21So<A!
zH%@=Hy`sAM@`glb*3af<IX4{I`D6=9OI6j>+PtP}X^NaI=V4=(wJK4tvzyn-Enf8L
ziKkWBn;$3K^`Gq&vdFz<QuXCU<JEJmpkd1Jg>$dzO^i8oaV<lm>8`@U|NKXEXCFT~
zS)G%Y_bO|{`%BB`*PS}h$h@&!(_yZo_Xo|p5eqw>2>H!p)r>Hm^SL@)cyEE2r>7?i
zE9*;>DKa_t_RM_nAmPoMHywR_%fi-1ak+ldwYIiCbolViO{v_0fq^l*N;)S@nBX4m
z_xV!61@uAF3uXHPKYceS_`GtF_~bQ9?woCZk+k5%*~8twVSUT*{w@;XHV_KAJg>)A
zvZnk?ei$#i^)Iu+1J^TB|N2Y*nO64r^^J>LHZivb`__C|ZR!8?-}J|xH}CJC-OV_E
z_nDiv2RyDPu`Sv*>)jtos~duWr{(k)Po2L#Ws~-0vuo*)AyNzRZ|hZcbb59cKYz9>
za$}M3>Z_-`_4gjBe!sVUb-ZL2e?&)i<D^^rGJh|78?85GW>sVE<*I_0C)R&?b8~X$
z<z)|Fyja0|lNXeSy2W%oyuFh@K00cWeeKOe{opDuu4S|Hipz>0uUmG@%U}Iy$>q&&
z=N1<4`rf%KNKfYZ_OJF2L-q@Q<+i^6^Wd}YKbxMZpG&O@TAypoG$*fS?&LokQ;pU4
zyb`*%z++vkc;58iR~ZV6Pw#6rTE4vC=`;4{6E^=ml(<c1Q^0n=m)7-d&${2-*wlT*
zM_kK%Rr7o0dC|J2rc*zBD5(4OQr)dz?yPkFo{uM{9ktWX@i~)beBS0WPkhZsR=LK|
z6}ARk>i*HQLQgsDFEC$quVn3&BY$7o?0)wCU)HUUZt>xH^LAK$&9FKgcIR*WonyUo
zDi{6wdNuL?NuLMjtUZrvTU^-Xbj@dt>KU=R-v@p67o<nA#{CzcR(FVL#^;cC3^9g=
zRkuX@?iU1~o_4N8e_rkDi4_`i)5GJ=-@JR=<Nve1E4(I6H_!M-`@Y$KSZK=ncMcJ{
z><7-ApKX+z*1N6z(}HgeKi@=sFgZQ#ET|o=6TdI!a^wBHb|d4-jLd8X#m{{9bX{K>
zwtuI^uA4!Apr}fVzHQDP9{-9b?o{8+<{8H6e9Qgkr+s{Mbl1JRp})VqU7UM+n~J)+
zarwJDF@X$<txq3E8L6rL-YzxwUhJXhg{l9Z?c}V9+par5dh`G2{NJ}^>|WJ<-+Xl5
zjUT!Po$v4e_IdT^rvKYm)ZTZdUZ}svvfy}&#IY^hPoC%BOa0JZyxw-%MeX9ELe4$4
z?-;)2x_y3p+;(T7@Yed1(IwK`Req;`dB${)YkkFDVHLUbuT3j^{uJmP>sowc!vd3e
z<q=J%Z;M^1zHnSZrfp+h^j!CO9jjMs2L=YVw6z)4{i(3Av5_#%x}qF8EymtWTUWO?
z@9wUTN5$hGL@%@a+IH+#1S89aPyc#S+Rl4FzQen{YTDN2KkwUYuHkLZvG!QzBK*E~
za*e;W2gB^T^ACPIFRQLPH&y$3`Jzda*ECF>zb$n`PW#JS+wFE1em`;l@C4oOKYwr5
ze|qlg$}jsf**O)`T&0-BVjr&ey*Y1waQ+R)P01DE;db36CvQ12ZHU<U$^7)CWo!Ok
zovSOnJ11xDHe;&*(;}m#Q8wqFot>R|V}&3)zuXnMOTE@NJ3B$%DSqYy8qY63VI-~H
z0FJlWx6KzfbDPS~cHG$33|>xgYm273-<%WQZ*N)b-hXMCuk^XO*2lMGUgqHAJE!XX
z%x!wP+2+5;f7+z&_?PPbrGEOoPx%k032m$`-NqDQSS0g$`|<n1j4T>K>zU(w_iy*)
zc^i9Y{oi`Wc=H3DrxgFrw0s$QDsjiJb4ByNOqr6i(|zw<``dr>_5RGfpRd)xAgXjf
z<@v7n8L1~$sjokKtHPOakH;Nx<rNn#S|)~jZ#a1SV}9S0SvPm3nKH;8XL~%``_soS
zUs7IPS_)cXFg>oS^Su547}XDHpiWQe>uaFy!~rQm&;ZKb&CAc`*-X}-_4DnTr^oD#
zx8#>Bzk1uDJ?rNB&KZR#YvP^r)?|t;TYpyNrQE#YvU;}p>pYV$eqXs~uW2^(&i5;3
zvd%Wz^!CkDHO5rg%Jtig&EqcFq@5Lglz(HkW|YPAGrZr}WEk{PKfNsT-X?qZ@^Yh@
zTQqu=%V+LCc+?{`pE)5lyk?p2y31_MjYV(cmnZ$Gma!;kVB?kElzLihtNnv3D}zn4
zuk~DA9Ui=-u0G4w?({N!ruA=UZ_78^Ub}ho;-KgJ8q#0tj~qXKc|oFc+53BQ^S5U%
z_n*HE)L6AH>+zhdCMY-Wx7t>ts?bGmH?Up5{^rd3$S;pyTH7eM@@}?2Q|5iqYwG)a
zkA;)%c@N*Ze6ZmA?tAB!I{&n*QvAFq+-Qa0HSv~tt=)l(Gqq<)7d_YeGqw5Rm0Yv#
zBhsg1Zv^kT`0H2TdD-9RpZsB3X3`hE*5d2m8SAd@&EIi&qSMu?O+|mX+^&@Gx!tOm
zcT7G(!#!ZJ&`vG&`1trmix+DKFXQpI|7&u2xqmq4pVJQyw`bnjVQByVkFn}3zU5Kp
z*KgRo{jcG7&d!_d8ltNnc}$g&dmVUZo|Jig?<f9aR?_d13qD-?n>Ovu<~t>E9aD?L
zixPM6xw&mEY&kP~|G{lWf2a3k+y9)hReibTeXFZw2jp)2jDDv3zVhHQpE|p2#g}^g
z{8Luv^s_rGYc*BRt=_A5?*(Vpx1W97;=-Bt7O`&Jxbf#MhIcnNw}WCM`8eOHEwxj%
zV(rg~i#J~X=6l;bI%L;Zk5wh}kDLr%o_S{pr?gqliSpZ9K0iNyd6};?pNz!;P{V(|
z-OloBW%rX9Z`S=QI%K5x<chmr&ZDQP=DewA4DSBdH^2M3xR+&$mjBbB*8E$4DuXsn
zb$-Wvy<|rJA@}4pZwoJf+I8{Y%kXu7P2TVKww!v{`%T==hT4dq```KIRrbE}l@7RZ
zT|EB#?$!F!AJ;1#%Q|>t0rytsqWh8sjg5_<5vDB}fowby2bx&9LzwFS|1Fm?%Q|we
z_V<blAL~Ons*UfT+4uCtjIU=ie{HS*@3t#s@89EBw)Qb6sQ&X8uHPSNcwxnwi62&d
zYY8*CwCBUz&n7b&*{u&{TL=Ump11Aj+aj5wfUR35KC}D&&8}P2OepSZW5=p*mU`Y-
zFS$8&r{4|NIWS|Zj_BKyg&XEHH;Re&x?Cwx*t5viHUEa4R7}sFd$*syD&`gWdg9l6
z{q*lCMGBvPxtaedbJvjR*`kxgo@Tu9x6c0`oAct#UYD!L?f+b9x=m?8misH7xF$~Z
zLx&Fi_;x%0b1EZfdNA|K4#VU9^A{8yaXGc6R`dRz^4$AdYBz_vzWTB#Y=7mFC25zo
z<=(EZO@9~Y<>RBGq0w=7ce!!hpNcjf$w{lHn;WZ&zIhe+{#Ak3>3RJfcWZxc>%Y6{
zi}r@q$A9q0?=<`K$?uY||Hks~dEegMJ$=vlxQEaF!{0946Feh+b8oV?b=jL|7fy0p
z@)_<g$k9J8^nR_wvN^WL-rcpnv9a3x_&K%bKXz67r`?@i^w45=`J05}_mZFNxbe)T
z@9o{QeX`a!_I{VpG}kNmRk%;?Vy|SFahTDLIg|XnPJIn8S|@PC`|D=4rj2QJXPa)z
zHk!(&{oB2I<LVzaXT2={ANId>rqA>=<BJO$?RJ*RByLPP+R@p0@c#YzaeJ#aCLQgn
ze_Q^1SJa-Yt6IBC-g@1d%9HhFme;{ov7he8JrXsxmoXQp`P!dx)2dIT_k0~+`@zHW
z+8E0UG$-#64qFv=;^)&<p`{5C!s>Z)8D<8TeRAH2KAW?rw25VV5?{@&JBqJo-&rlU
zJ1OnhACs`<P4x{E{jSQah&&|6&u;OjSnIX^zrTIW2M?ckQ6PTm&X(g>jTI;Ji!Up_
zb^OEwHNGkQ8;)OP^x;(Vy1r>~-h;2QM>nvg+0C4GEZ^K_f2?x)uaNWmtjaSJQ$K&t
z^tqG%^l6;ll!6`mHXCm#zsB_Dzv0x6AH@!wdQ>5-oSnMYZJ}3R$jvF!-r7vd+A8yb
zP4C_LKgXYMmx$@P@?jhEnm4zSPnoaY=6}-dinP@6{QD)z2O1b<?Ca)ST<rdQ_V@i6
z7nQ2MzG^+0wIuBR&L5#U%Rxo#t$jB+_J7J<@%u>8Hs9BNDJd%Q^?wY{+kEcv-T(Z`
zl@KRq=bvx4->>-jbh@Nrl8ao#E0H(RrB7a#O?~<QlL^DUxqI3^^zTmlZ=_*4Z|3~>
z$6kIspAlx(da-<t&nu<E$@h|<q<iP@T>kz?e&xF2yU}T{Z1-FDP2crb-tgM%+hV`X
z4oCcX7d)lJXhPjXi`nLDm8uS^AIUlQq%~6d<^0@=6u<oU2KRfrAIzQcCYkA$-Dktc
z^KzP_A0>wA^d>)@rT?RDbEntg|Hl*B-%DC1UgbS)*cB@O{pkO;=kj+fy8WE*pZd4^
zO_WjM$=O@9Q_tU7erV>0?68-{|NrPY^4?>8aJ6#wi`bjY9~8cHF5L6^^>Kz8(^A#i
zSzpeU&AazgcXdR<)(ekMmi_<pXRkJU<hSbq7phW=ze+`J%ZXgHcrhm@=R(j@yko~M
zEpqK{|5sgnb>e~d`+oObxDa4;#x09KW`;etLW7{$pDBfZ|A?kn@0xn<U*Oj4Ck@;i
zK5ysYyPvo_WP6{m@@$KTO;deW2fehnocXYOo!^BA2`Wn#YuhsCeLm}=$!>Ed%uh99
zUA^*?oF^OKottvhVZPZp6RoLRn_0s5NB+0wbG!HYt&^lV_v`Sbm+FhSy4U+fJ$Uv0
z!0Fh{ZNI1K6qT}bZuq`^dESD?!@rF!KV1(nN>|=HEB3x(*3C<;c30ne+Dtt7BY#q5
z;YCru*^E3_vfmzh%ow(B*4#t-)7RPAUp{00IY?r^#?<`(d-i?2`!kzud49yTzMFjk
z5*+JN_#<xa_&Wdnl_}cwwJWY1TYRl%Yt?S{kOc=fT&-XUOFT2f5H!5J`DV`N_tlS%
zbe6on#=E*%y|n$vlk-d~K*`VS+~y45SG=*U-i?=Re!tn=BW2oUQ=+=AdeXdke$(}0
ze|$VHU-|3ha!KPfpL>x%L{?nc_Ic~)&pz4z`b&>*n!c=g^|KAvvL0@I*ZWcFE1&+f
z1J{{v8+_WlbW82C;=0a$b-Oz2;`M%QMztmNZT@#_PH(o8wVfsUG|C{Q;(oTfs&BU4
z^3N-Fs!Tex%0K@3;q__)n=M}(>|6Xdwx`2b_SxIEhkK-N&pN4Hr(#$4dh64Ozb}h<
zX6=z&lbKVQu5>@&ERc`6zUCX3vi-S~C)V!`7F8c!efp!`{dKlQZ&Uk^?e$;2F#qDq
zsro-Nt8JpMpG_;MWZ&@TK!@MopI@7_=kAX<t=^w{H6ywCdF02-`hT|6**+9qSAREb
zL83(SI=?f?|9-|^d*gTN+rl;TO(ow%UD{drvHp)>{J+otZ(4uev0~M~{{8RoNTxA3
zIyx4)yB6FR{m%Mbclw!mch0K&=6>9`Y2KD!H<#)9FUy^_a%E=NTc6@m*jj+B*6O^O
z+?|<PdI~!q^l@|XoG~bV{_o$8!#oNKT1u}f_s8u^c~|vr@AJBM->c7kuj#iI*%G(&
zk-enT+gn?IKAj$KAi)C~RJ^;p+}_6T!IzhpLCYhn-aE|diEOXGYG$+L#eqrdHoR^6
z&3tde%7b^EQYX9=(rNa6I6YZw-mg8;$?YZ=*Ub3Lu5-q6ovglT;KQdLtBsF%Es7Su
z%&X{eE!%9_8Pm_Nx=fDgot$*_{F3LLzl4mZ&-(ej{N<ffA+fc)r8Jr6^lUPCzhiT0
z)#TP-KW1<F1<M0h9pWwy>fC6tdu1+HxA@G;;>lCZr!6Z!wsUR1r!SAHy=}A8){>|1
z*Pl|j$2MK)ZH-)Rv84Ietd~2dteZ69&7S#Pr#`Ao{o%r%zUirU!3IrlwY?kI<ktNT
zS~O>#mD+<@b4xYL|8?AM+v&WN&$cD>&I@Um%Nt*RSbwe0viRAB{q^;iE?<_v@w)cw
z)$o}HiA?kUg=SclF{^5u3H?(0;``z@b65CFEfe|g(yH2~51aYz5>8A|Oi4+x3AmoI
z{dVP}PW2lb5}B3VdLI1!et-Y^Q|sf-JLGdcJSiTvW14U5uEVl@E9X3Hzpgy-aQw2`
z4RY0XJA3=q&pLd3xABG6-p5-FbLU2>1@FG@vQJ`dZ`Hl87d@`cIq@!~jsN$ASnagW
ze{M{lDgU)uJ9}mA<=bnl5*A;Vj^CX0l7F4gOxA}NE=B5uiK^+&@><=c%BZ}x+V=1f
zsaauif0LA-dtZ5Sw)uC(>cySOD^8p}zEP{nOu72ola=Q8lJ^wf-cjslxI^wRyS|<a
zU-iuEY}*fnvn9XW)l>JF_4!$wWAE!#6HI;u?aNa4o#{2@!2Rpe@%8n4HoMNZKeYHg
zZ^_@P@{3{N&U5}gl3SLyEkWj!#nwIVOE_j6YAk&h5b*!^k~p)hD;-ZyPhYIT6}!6(
zG?~P|P54ED#SDW)ruF-NY1yS_U#z*KR@0RAaYHgw-L0t)<+y{r&MTyMm)&h+JH@a(
z*{GvDZ??4P>X#Rp*h9*WrsSV0o3nYNy>aQ}?LOA?-~66umr~rCzEYEGbJ-5j4Kc3Y
zrE~YUu9_zKDMa<oWXr{SV*hKMewUlYmeJ;`mihc`PKoJ(l`)%B_uZ2!b@~z|F+=Bb
z+O%Xp?sEPaoEqBc{2x!K?>usBjjhN%Bj!2DWxLFaxLz<uH>&M8*WYd$qNQFnS-;m!
zSAE4lhq}LC!>vkQ9GI#dF1I3k%gr1fdAm6;FE0<?l%zbX_c!l*qtMoUoVCKgzBMF&
z(DI#Jzg}rUTKP1cNT=`b?t1$9)hVny`>?>E=0`!@zn|%Bd@?8A+}!-$-(76Q_3|y*
zYvz|;T<#KkwJLYL%lg!fs|^0#kNt2qocoN+`Up;k<8`^4Tm74Sy8pfVI8nMRO<wl)
z#&GTLyQ3GW3Gb^vx;5{+V|l}#@W%Bogl>oBF+Mna@UVnw)|7AgDcW{@myft^t#|(;
zYMtUKe|2HB`zi_5>o=RUGL$AoZqs%z+vr-lK&E#7!kNcqOuIfDl;6?!@$bIio5zix
zDXWB9+dg~MxOMHU<LioV-L>7~I9vY1)qTA?9-Cx-57fx;i`trX^!xq#{+l_=f|vUp
zId<%xt@WF@xD`i}K+9@Vo}QZeBQ)3daH98x*+)`V>rGa;=y%1GeXaLPhR$cYHZ#6=
z_wFnG(s1b2-Ji!buCAT2zkmPhC3BB`{nFra_LS8R_4m2LNpEBFmP)3_*%_s7DAazo
z>xpt;)HR;DKO&^3Zx*?m#@-xf9H;k0rlM9@b%of9*u`IG9eT&&YVG%|(9yd*uRP$w
zk>9H}w=YVcP<DQ{nW2HY<HD)NiHF*h%jX#!6xCX4Q?oG0e8#y=jl2A7dRL|h)aT9i
zt>_lZiP%5y!Gj0C|J@1-3X(8LVA%inn{?`-T><}3=1;WU)3N~^&l41+Z88nMSo8h2
z={!68IjDzqZEf^%<^DVOi)H&|t*5DaPrI-=-T&FSx!r!tcjvEn)w<j9pJ(k+<8Hpj
zsvg$;Z<b#@A#knctLmhOnpNifyE+xCXYIXMSuXra`)XPZFZ2APH*f5+X8YXmEAsBk
zoZHhlvJVS*rfpoc;q83kh?T{MEe_Rq?mhD`nd`-@Pp{e!Ydp`=mO8a!InRw8vki_n
z=jgrl-847OqKu{CnX|g_rcl<LbEjpkN<`RlZZPmH<-V7Z#`XB5c+`epk7XCgJ>Gnx
zc1!&)!LWV%1!Hem+`6h!c>KJnu}Fycwa@!FSkHPc_nRBBB}1^8onK8!sp;+Q?ZHdE
zrm`RB+*9>cYj5@UwCCsMPFZ_>{;vM|PQQGY33F2~mPM6cJ2pw8`HRifh()s)Cp~)W
za`)jz;YoQ9oAZkc4l<?sG(0x5ds9~KeQV`mNgb~<;nTR|bX>K&bAo@Qg+FY${b$Q*
zx9fFoUosS9^vf=0atiz1*sPkpBQ|DXP`doNfR)nowy6XzK7L!mdzyOL)zvNAGY;<B
znw@!L!^7HYU(oDW_4jvw{(iqdCEKd!*55QckzfaoTH#-6Uv!UOzIO7;JA2{alSMK=
z-pbAbt=ITlb?DHccc)*^G|3c-um5W*YgLkPagpnbufKl(;XJwJ$90Z~FG37~^Vg(Q
z-*r24BRBQJ?^=zH-SeiskBVQLu6wR6>zu;<Wx6-}FYvM5SfemAZ}GtZ{hd|3&a<{G
zpIKiQ75u@jes5rQnwMVa+-qy2MNY-8zg}*kDF6P6gqFMg|B3TYhg}vr)bncAV)L(M
z1?^4Wq^|th$H{uu^ZmWOpmTa8Y^%1M7Swsewxs$}P4TNMno-+wPM(`<z4A_$%-4UW
z{uT|D6N>J*_1yfs<Ccjb1K+kc`=*{OSIr7N_dam-950<62dAW!-MN->vMN1t)zgbj
z`ae(FNEKCw2EEsx9Nu2P+sbHbexjAE>BVF5Gd;yF1gb}P&-n2Cp7s0>H)OR~H@3f7
z)RexkXm$FE|4*3o;yoAn-rkmTW5dIJd-`tNh?v#h{`vd;`e4%}qf)jrVtflOm@mxd
zx4fzpx9nT;{NiV4BBQqCoa~deez!Zj;&HF}kNf}sxmQ$J*#G&!?Dn`har;Z{txwm^
zo3(iHt?9;PN3uPN_~o+0H(dDZb3FU?yBRgHtC#s^KZ-u!w!rM^=_2cLbKV1jXCDgh
z^sjXMB*`~_$KFKojr(Pl);B!6+|FP8RqWGl&!w|JzX`~FF@4pV#8XplF39#jf34LZ
zM#)%2<i&+mziMO6=T|)9OglU4=*P#$J3Bfae0qAiwd#*jIn(#|_x(SAC`d|9UU~6x
zu2c2??-$N6MhNg2St_xF)cvsjef(zC!E@_bCHBbIp84y>7QTDq)^$}4=SurduQXq;
z<#_I(Wq6U{iZ|Ood<xG@S~U53@A3IY2PW5T<;u``ZZluU*zS>=_X@XKW##CZ9L7&K
z=j-YG*KT*5lA~_)^4$FO)*)Vrubt(zrb?&2{<ieuim53YRaM{KoIKRZ{eJbQ_qVoo
zi|IxeSv@q_`s@6ns^|Z^+O@v;zL?F>@X43$a={Oypfh1gN=gwsi=Ng7f8)3RGa>Wx
zvWp8GnZwpZG)}+%SNc}oc?QkfCzSK8_<#Fv%RLldF=-a7r(Y7+GGG6*v77TF{d0=W
zRn0$hQuo$xF^Q7NcABQ05zn`6U3K3gc}4ra2fdy0wqIG**IJ&QbYziWt;D{sH3#_~
zP5it3@e3cj$l~OfzhD1)zqET{dM@h9n(bfIPHd5}Q2PB+?zMaEee3V+>yE!GY-`%*
z@VMyL?>KX}Ua6y?)!Y-6-Cx|<X?%Ts{C@kn2YhFnUEJj>trxq?X>a}6#a_4BtN+c3
zc3)JLDJ!9}Db3WsW8+SvGmWR_U%Yv^$tSW^tDA9xM1J|(bRHvv1Ny&rDmiVLuKcs5
zcWdTPX<Oe}nU7NgME0FeUE_Bxg-apu)P%h@S-TE3>a0EQJnzGj$(wB$>x_ciD*}DP
zBDbHNw%>4yrr5k&runBY-Fg##_e6B=7r(a?U)(u#r2a5(g#L|)c}#3KE9d)ItzwcW
z`NDBJHbnSw?vXUlC!+Nq%!JgQ1by44c<bjSmlYxNeuU@xUf%fnLiP02A0Hm>`TcJ9
zch{ZuKc9+QmA&yO^0MM>tMzY9JACx82&lz5q1R~P+{{m3qztB{9B5#y`F1n?=)N=Y
zbw5@0_kIyF&A#^JdVKxTl#{<L%bTa}c(Fg=->zA@J)2$L=B?wI<)PT~BGSIP)Zg>$
ztKQVe<RwRwh3CWtnQ$wAe_k+Q+o#@D3MXwPZm!MZTduczdg(`hUCCa(H>`5Sy{;wu
z4_EicoY_<=C2n+?GhObOh|_*WQ&Znf^~V=K-uG}xaZ_dXCr>focV~Kj{5m5UFgs`Q
zL7SrTDd*y@Mm=7&_0^-OZEv2s&)ocU$LFuxi)wxgMe)|Pi~X-`-gQr?Z1uF@$e({%
zJQ=)%3Jb-Hzt3O2EiKEvocmgLzo}+|lSHlExn+kMb?uoie6-_ZJ$v%=dHeJA|9{8-
z`}aG4SNZ$Cn>l(@9*fNs-jQ-rXjkd$wySEE#!P$y@y#-9LA%rB&WXj&Y!s~0+96dZ
z>yjqLe6noX+~W7557doU^zClFp&j7g?{ISK{dVa!=f3O~+WWim(qVoBkDj;hJx!St
zwi%~4pZyw_n&tWRn48?%CCUeajm^|dt~j-stu6F7h}pa8jh9D)&dSA2pWk<drZUaR
z-xPCbZ_P#d$W4q=DQz}!O*<Fw{rR9Ux9I6-cAc8}EUHYYTGpipS5L9#@%nTz@L1)_
zs`{DR=1rKCko;S;{B-TnC^b9j$7k;t-n;eWRM(+Drn3Sok8PdXr_NUS>0yArT>hfv
zKJR>sbow{$bV>E~Ie6$K6H}^IO4ck#B^~cDr<1|U{4(E8wH8_3p*5dh_0+X1&xCZM
zyRXMp_kMhQ{BeOrS8nD0KcBpReShEYtoQH?$Bj@MmxcLG_gQuZZE|v+Ww)8p|9?Vw
z*_TIF(;qxK+Fki<X8MY#ty&W&PMo3_`)gkrgYdtel!s~BtKJq*TW8mDGc@$?3)7F9
zXVs-F8b3IcKb~y*Q)R#2oTtC`mi{j5i!I&0XiA<@i~h0gED4XRoo8%T6=U5|_o#ds
zPbGiDb#Y&{BN~mdsuzE**+2ika{Kw%gIT7<HNS<F-0GQjeLM5FrsZ_y&N{7m7SF$R
zE<d+RDmzG+Z?m;u@SH23#pj+*z13H7Z|d8SpT%l>qs$i6|Nm!_ce7h^wQkSPKe|#s
zy)?r4gtxk0?#p%W+j-dL>~xE#C3?yAw~tG%?%G$AeLX~+Ve|aA&VP6PeG$H#v-a4&
zZoM3t%Wb9?q%2p3lv*y7Kl7vM<^Ojl7X18u=kvz2!|i#8x13WrY0!6WE7!Fh344W(
z*QMW-O<!Mha?aDc8)ecjn@4G6od2N3KP7L^w_DjZ-|rNImZ{m*{z}*y_5RQ4HDZmQ
zKwI~G=GjP2ey?mI^HbQa|B7Hd<EDC>bl%TL)Ti@*?NUuk&zN$iJ=w-`zh!TKU#9$<
zX}LC)R{b9qZP^@lY;94S!j#LOW89TRi(1TLyiPoryj60}$xUlwjg|Ya8tzc@EZbXh
z{Y}mGV~4hhoPAK0HbMNX_tQ7q%0HYB?abQA<7T%*x3|B~f@969=lf-Sr<{t)IqFz4
zW$O7P5v|rrgSB&y^B#}=8qFp+*Vl5>1Ivg#8oj-lLWiqog7y~d_1Kq?d{9bzrgN-{
z{mg#(?<LEoUgtRyd0OX*t%%$4FNSeH=58=bJAC%o=@&Ds?sRAEe7bqNW{su(+~U$6
zpX^iy5xW_a^`cI3H{30nTg%Npx#SQ3uWxTpgSIPv`Jxk+TJv`6^@{Iz%Rhcv?{qj&
z`-5JvuZ;yLak^aHuPOQL^qc4p7T5j%AHA^9`RB9Q`34d^N0-%ThpkC?aG=rV(+TCA
zJ3A(NP1U-)-~4vbl#{;l>04(lKInXXo_zV)Yfko8jeXW%pFC^-zmtm(My1;C{^`8S
zZo9&D=d!A=)oFa|9v2yWU;l5~%*pA(y|>p)o6Z05wYmC*HH)ktryiQW|HIAc`McB_
z&oBA8VdbCuKaS6hN=yCRlW4PD<M(WplYeJe{bIWO?9Hx)Ht*MTJh~{Qx?#?}vtORi
zf41DBT%}R3gh77qSDCrlQ`Jnrg&$&1kzb|olKsQo^|n8EJ<r&dveSK19mB->eV(Gp
zQi0w7kH2_%dehtMzwPA>3oF8zjGw;zC3r8#wqyRB;|5>m9s0F>e!=?vjovje+lAZj
z#~r)ge@`d!lB$^P^<K%@`6YEvp6PFRH~ZobnJBLx3(6<(JpNI8e+T<Abp!45cE5E%
z%OlI*-&+&0@loF-<GZ2i7akTE6h3lEJ3Gs@^FV-9Z%pIK^-Tf)s$ClTT~y`IHtej>
zeC~3nGQZmPsAG-S!)Z&a7W7T3I<T(DByg#L)i&=R9xZ9pEKfSt<Qdyat4OhXG@Ocm
zo)`C(nW=P&&&kIT+hgoxA3jwznk1WJt$ysZNQC<vNtr*1I^z5~poN@kqPB9?|F716
zTef@d#l`NQ-)_JED0&}buiL*4_fCh82U@@#=CFh*Jtp7uB@aGb{QBjcoyMo9>mNUU
z{Q11;*>`u9etCJB-QM0Fw6Mj)qoX2%^^n)6tFI&8EVy3%`2GCv4=h$&&U~G=*8Sp>
zSgH8;hkO*4O{n?GC1$<AJn+{2>4z+KCHXz`N}K6D?bvmV=i3B3!`CsZa`1fF`%`xJ
ztd6VJ0`bX-Q}idcY`%VBx|aLV148^Myu}tWJ#x0QQcq9&ctqI$!`<@xcfXnFmY?O^
z5~0H*Y1H!c^z_H#cP6nNo|f?8i|{W60ggP$(9q84zf#M$)w^Y6XuQ6@etFT;Q{UIv
ztX#Pgv`?<|^|e5&EyBEP`Yzu-i1B+V2;AAV&r*bM{hSjwU*v=K<7Qu9r>ddxV&2sB
z+uL$W-rf@R_xA_wpP6f=ROPLp;dkS0SsRa}Q%Z^oXw`pw{ok*-syFv&`y~Vg34!*>
z{QmZK%G%r0cQx-53U=UNX|jHH<=gp=*st-Y*jP8_y^WGG$q0Cox})Nwl5N$Og!lLM
zPRX{J5%o8H;pDH69)i?An%{f*XZqvnkX11=d0$@_mXeBkk_uY3lYG37VNK-bwqw20
z-R#N&BF;ueMiLeU3ZUhvKcCM}e|Bc(_j{Z+<?m$XT9rOJF;O`r%W~R<xc7=LOCDu%
zvNSn8Jk%(9<c|TH|77;nO3In{_tj3E`};_za7j<cgD0KBA#Z;OUES>?$nT;cAR?8%
zNW3WJDfbku&`G7)*DtM!+^l5%eW7!E#^q(aFJHd=^Xatymb|-BHUf<cBqq<B=QrQ3
z_QuX)_2A`xskgRdZr@VBusVHy{Xa|hKADAC$ES0=Hg>jXU}{_tz`5%Be7D7634Xp$
zr-sLAUMjBta?yQ`O<~i8dzV|8H3T@g=5qPAUrf01T_tw!sn_fG$1QpO@55n!PcN^e
zkB^QXVie(c$82M38@aE>^5DUPdp@7DUb1Z2y-WXE-z`i%7aO>6<qTzS1%W%Ve^zfh
zxVhx2(`@bMVPDtQJXSyIEWp81-rBH1ucBhQW>KxT|8*ml(pOhJZCDx~*d4rneR|p3
zTMr8?PMkQgW4qr12L%BR7N$lJ)un%kQv}9gX=>+~c*(C}`|_6fIMajciq!tfiJn=I
z9HGtpZ_}x5b0y#G<$k&%I@W(PgKVZJmv_I`A|nnKr~57YI9Zwwxo=F`cB42p_0u=E
zed?)RkIwsA@67qUG;_Hb%e_{Yf8qWzPd?myUBPy3Nn!7$MRFYK0ReL|?jJjH;K7}z
z>beI{?by1lI%u8OLN=>5DTWnmEN<++8h0Z7!pDe~&Qk||*1psc4Rury0JUNj1VpT+
zN0@AsHa3gSk5E2o`y=r=Pga$;Yuu_Ui(gl~vVYHD@Z#}RpSLkLKEGGnxafR9+N!c+
zXKr`RC|FXVc*v1;&AkcdIxI8))astjQ9N0ecGi1uI-5|#(MNjzSzda^GbL2CD(?OJ
zSgUn1qv&};bWet=I0v{O;b19mW@tP2t6TQ8y#mYf&FZZmj!p8Fx%2;@Z6O2WmwBo&
z_0zw8-&e?dwCiTo@8jF&+aB<_+P6gBVf(K|;v04-Rxz>q$*i$RS?;L4Hr49wtV40P
z_v!{F)v#?2T%oWs$L86IjbTZTCg1z<f9|m>fBSA*)pZ-J{dIf2W{$V8sRPXScWk}d
zwmt1kliRcPcGnW_g?<?iY<2rrX7QW3w-wD4YGIpxa^p3xCr85O$nTxF{PombYagq>
z`N!L^L0x%8))W_3jkJz=4vo^))2?OgtPpv*zM?h0&+^1fjor1?!e4z8!t(aN+P-Q*
z?30;`&CWb}$qkF)-97G{XC;HKnryiL`qBv&w(ASR3U*4T7$*MBiMyL~(7%3n)665&
zPM9)$c@wsN$BA_c+>6u$0y0>Byv&G-TP_*Rb!2W}$9dzMjV|4<tWH(G_SkIYVX;(T
z^6_n@t4{q;*Ua(0RJT1RG0A0xn)=eeB`iNyBn0k!l5>-v<H|*ofcArY>3Ui*TbzTs
zW}OxIv{EJbCc}dV=acz9eqdp0ykOqw5U^&W+WhBNW~y9xeei*2z|O88&t{%EdTOWe
z{{qXSeJ7u|&ph??gZZ;}U9q;Z?K@LKwSHyg9lZIL`-Nd9BZInr+|j5=cD4LFyQ3}3
zABkMQ^tR&H6h(sz(gqLbKK=1+Gy9K<a<)l#n!^^V9z87Vx8T7+m*Xw}+-9-d(a@Xj
zmtv6Y%r<?q4)>a@gsgOBJIigp`<4~hoou`peEes9&WhBpCM(t)KhB`O_|lDOPZk8t
zn7I7)&egdNy2A2Tm_)3DpG9@PNL}PyDJe3wM8zhRVd@Fhv}wzCoS!PG)ygqd2$aiM
zm>f5%b$*SnJFvvResAL04co;()ZAA~`O2!hy!?|}|Hd_8o6VWN%2~~8Ti8FL&~)m9
z4+U3*>ji#Z)(LaJT~T<l%<YP{Yu1~fU44HqY(2mI{I@634(lvi=i1vfe!P<x_u6XK
z)m6=Dva>f<f6ntQSJQp=dLnPk@6@)YDxP;*PL8d@8WjP&Oa4ngbxmbn61(Z+J1*_?
zhJAwbzO2;~7I<^ey8qCOcq4`hH7Uy*mu$)}wwf<fY_MUsx~=xfAJZ=VYxHWEFQ~uQ
zB~7I1uB6!15B_ZLmuUW3zMIqk!4o^~;(m=Omzvl0vCE!19ig9Ukk(qZPrbVG;?g}W
zy6!V)TRSiHJ9gW3#T0NJ5a8(2oc<#%B;4-aT*u|<_Rr@U=6-a0cwtq)<d;)SVt$eb
zH!k^EW4mePw#rw6XBn4By?57J<X*t}O~1PLVE$jHCyUN^uw2-h7BT&-%iqoI7U^3W
z+T)6^c!j(_e~$NR#oE^y+g9+Y)oyM({(8!xRVOMYAKIjOenV8&nLWPomvsXAZ4GC2
z3i>C0ypUcTRqnEm^})@KUrHidZqz+wX|i5W+!Dxb9VGJ1idUs!zWLJ4f4kIljw$fZ
z<G;8n>baO)6!%Ww#I3;<burmbdoFWqiaV8XM8^2MuEFQ`?=S6CY&x|}sNS=*_2TWz
zGIKlwJ3WIkLyl@1M;^PVb2sHES4FC&OJ`A})6%|<HI+<_4liaqC}_-+i8pQUFHc{<
zEqy#Y`PQPRAEvClo2?YzvyS(`x_-A!&stn>|J(jPVy)E$b9tB73A(@2|2pa$>}#7Q
zq5bFP&m%HFCQY}WbG&P*{RPKU54ldb)%>lE-Ies^<Kf%4#p~nqJ#3GD%aDH1dv8hR
zB)bio!ppWicpvhe!TtA9<~1jHSl23VYT{B4_`slaamgk%rcCiSZ_aKki=6hQdf_=k
zy|y#Y6j*iEZRP8WEtU#dnzAb0^4Qv*nfs=#+SRkEzglu-E`w=oV)mY1i|;2hBHh=Q
zHg*SSHaaML@l_BIiH1*3aGrR#_Tr-d`(yW;z3E{;kXEdA@YA_j%%|tKz1=eJ(2Ez!
zD(RM8*L_b<P+f8*r0Ehv-`XCDpK8{AF^mh`+70*Iek6E3jnlgRvBWA4R#CqN4;oD;
z+LTT)ezMo@>5fNPjNi3%gBcA{&h;lx%AA*a;mpsJ<rzz@E$U}Ky(yetE*V<)ly{HG
zso9=qzv$eZGEr4_k9EkaEr^U+c(e#KtbRUy@%oV4R}XJ5E$2x)urAAx?VIjx%e=5y
zzT^beTq~}fL6v**x7+;cDyd$z?3vUty>y@VWj+5=RMYoLU)mWL`P}VYzy(7kkr0c<
zCidxn*r%?FIekSh@({}#+liUSJW5{XlyFUS+3Vn|cJ6=i?>6sOmeQB&t{+-#bUtwV
z!cJCaACn@zlTun+gIl~fS(p~)Gl7~g_3iug9nZDNU!Bv)6q0wrEBfh;Q|jIS3*06w
z-(a6*%b>jbZfCNZ@V`Sbc2c(vZB=>odSyv=+;n;Vm@9M6>nvO#>!PuufQgk~Zt5YP
zJr~{HXU#m9!+D<jwP|aG!ov)<__vq88L>(H$eAL(*}XdWxb!6b8$WWszm?Wgtx;Jg
zv(>OHQL0)m)v|B<yh3Kv<};#NQ+1q{!iww|&&0onJT8gq${256uCG7KY;C80fKmy=
z>*sdsEi*PBznF0)MeTe3;VV~Pewp{ha`CLGJDZI(XPk{Infu3d_Ou$G^E!_?|L1>*
zC{wu=yKq4pBQ)?f-LkoQ&SUB%t>ea`Csi)r(tlnil<MQBta;Tou()l}-`Y<brk*|i
zMy6})CW}jFiudmj)_-zj-JP1s<$d%1tnX$ty?b!E_NB_owXLVmeXWw0?N`|8y!4R^
zq5x74|F$OK(i+w)uh#hfOSr#j-i;K~<6$S~+*#DB!ov1*_q11Eqht;(xmo-(E5m%3
zt#$i}vw?feH|5S-xInJbhiiUr<txFqEj5jkxu&eWK2wr?apZ?z1-WypdV;6x?{IQF
z{!L`zf<v4lcg$yO+IHhf;uMd0R&g><JX00y&Ye4^Y&PBT%C@QQ+j6t7m<zBpG;h5<
zS1I&vGp8@l0u7yihmReyH~R63Ut0S7%AeEL#vIiNT2$ViIaPOc=)+xCQ<*{y!Y=KO
zdLu0S=Fk0WJmoh}zl>cNr2on>9JUtVVxdxq7p%1*qRJOEx7Tgblh%7vs{L=?{JY<%
z>PY*G!^<z#zbHNzyEsx}r~U1+>)-RAN{2d5sy5gZ9m=3{{JN0gqu3`Krb}K>obyc9
z>cT?j;wEPACq-)8zrC?7y14hox?P6?`sZAlv_HmH@VMn_7uMtN-}IcFGvkWK<m2-n
zukDNKo-u33eNnDkYCbJZV&z9{8kr0n95`n>^4^IQ*u%uT=SG~;NiH@8-^Ne<CqlR<
z&7FGeVB=#xzGV?N<2;v@F5p}LNp8>8V+X(3znQW+FCl41*`CFLVb`p#E=hf+a%|I?
zDTX&$Pp(_BBqlWC=j_&8&49g|deWDjnQ|qN(Smt}rq*>v{{#A0PDHEPu8>&r_)w^5
zitOZF)gR79<uX0b3pH)r92dxOPDpF3rmIaEs2m2jKwLD!gub~LC2u#|6kaNm{PfaP
zleIz&X3}i>`Z?Eye-vLe63e&aa#+5ubj775YVkey4;s&Zn*tix3eK}!Y~3=c3VCSj
z=7q24`En<fJ$><e%bL%1-;S-8^3A-klz(yCx1~+nk9-aOcx2_Q;8_)~bkbI3+P12C
z1&Vg+9@yk5FYv)h)$Z?or+e`mB^tiZ`!it!@A^v`7g~x``~F57Ib4^!zqxwrngz^O
zL5HJTKxKEpo6JiWj<mE{u4?jhpBvD>r`*W;#%mLkXX|{PSG7#^yZSX`P3oh~QRi}V
z_{?&*Zg{c7dhZAQ{plCJdj`Lo8aV9(_XXos`#X<`&HT0W4pY(>^AF!7W~O#uikSUO
z;Z=*%{RGJgD<o6|_p8r6E5|nP+w_mCpB!vYzLV;|FCqDwzS%$1sOU2Flak<40aR=~
zn6^PCwdnHk$hB!R^BirD9eNoYIpchu?5l9blv_5&v)^mgo<4JO^US|*7}Bh-pFZ1e
z@ct~P$?6w*C9Cefk2gqmUU$5^yIW09Z<eULQN{K<MR9Q@H@51%T3y(3Uiv_n{e6`q
zKR9BS@jjWo@_uu&_WXbRaevE%ZC6=Zh$<C_#h%>IcRKdBR}sJ6-%6$B>R(wBQnsr>
zN|xV^w!3wEZq%x~2mCCY+$wMG8NxqBV<r3QSXH%;#{^a!kIG*AT>X1wM9mq;N_)=y
z<+hKX>IQFK)tL3qh*SGa%^d&iZ|9{i{nQR{XBCdw^+dr=)*<n^UW3k4k3xaRHs@xY
zeL3Z8!;X4zQ3INEaZu1u@<}{nQPCi|y_P}wYqj0kKl2WI?yHjRj+OrY{C)jnj{SeL
zx9BXGyf5-_!z{IR@#d4~t!rHq9VB$~|8f5_Gq(42?VcuRZ2A7)u3LYXeMsNEd}m&x
z&{F1)SKHJ+o$0EzVU9R^a>iM)H!n0LU-S#M9`T=c*(TOMV5`xincV6D9|{%<&7bl8
z#&oH#y<gU-B=4VcaQ5!4JXIe(Bew@k@LPT6{qx4P3}t6ty09<LUno;AD}HPuw3YTQ
zyywR+_D8di-+KLfx>3cg?^kNd?38DRGaWF$eav)4v}@wAJ*+RdA23$@W`8!*<JgP0
z-50H2A53M^W)s$syHwlk!K18Ulw~<t{sr?Fp0%?!zw+F|VQjE^zM$UCAE8Uk#Uah&
z#svrFSZ)w)SRB75>HEvqme-;z#kWrlzh&Q@`RDQ#k;~>bUpnSFI8H45vBl~BCo2bc
zmvnt4k*O)&T$@_f8#_<?I%{s^M9D{K&??iRfIDQxb)g%LS@GvqygPmIYKB+8RH(vR
zuGMDwYzj3us@F)~b-JU|Sir>U>VBtAd%hOmyo@)Fl9QK6y5AF8Fsa5j@%lZUX>!7f
zzR(7mz#T2s*Zk?1`Jc}C|Ma`gy+7)GJ2JwKx;BeAy<a%F^36v%6F#wEbB-2ajUDW7
zIyY`qKgPz^$#qVp%2RXIi8OTqQ0J3{>7Z=Twy^ZLnOA3qi)g2H%U($SWY(24m+2u3
zR~M(*RAaM3X>bN=bWqSJNm0Ic&#h3NrBX@gFuZ@y1adT_;}7cpe^5|5<k;OHWM_3N
zY?|idrGNdrgDtB=A0Dt2H5TeOy<D6!zt{PwLCp%k(3x(jTeda5_T;kev)oa`{kuJ6
z(#y!`^Y_+<ZCfgSXfMx{m+^r~KjXhuo1J_1ZmQ#=;$+)bnbOk2pKqR7@OWcemDee)
z!0N8_y@H_G@iO~Y(Tt`ZFJ?b*boYF8V4;fB>(HlhEFn>6)814{zN->;du%K5aQO_&
z@BTWw%jYjXd0<|M#uQDpHFJKv&gaPU*--y*hYIp|0`Ei?cZ<gF7b0ieIOwu9sB(`+
zm)9i8KQ)fFL3`}VCM`dsUF+NOD^0(8e;MDlT#xj)rdjWg*L_*67R8?0HRV+1-mmB0
zEuDX+W1E}_=Oo9#8$YMd{jsQSUin6zw^G#?)-PmVnqrX|eq+Xq6UDcszVdv@%3u&#
zRL05esqn@3MfAn22^Y){HlMqBfB(ai&&59cyJh=jMTXX+&9fsvXIU@H+xA4+S+V5J
z&Az<9yM!;=d|79;YFe>%W%415u;Z3@gJOfHeS7WMYHhXB=}`RancvTO{}D-lepK}~
z=QsUOj-|dyuJae333=NybNkd|i?q1aKg3;F*V;Mp>z(c2OZH5iy8G>ssE|K@Q>x59
zE9}|RGhe@<`O(*vmp@)h@YgyqOL6)$2L0yC5tB-{ynT1_scPn*S(C2+(`rvLzMy@}
zl&QR+)=KS*`9&24?<bp_j|Lt2{y1-{qRN>A^|vHHp4oJ!ZnDlN!K+)Vrq&!fr}%Ld
zCuk5vJBe55ms-UJ&!oG{?=u?Izs%dU@6NUk<Ls)-`PbyE<nEu~u2mE2<=VAu&*loP
zUZ0gRKdZ&Pm)D+p7JB@~o-Dnh&*Bl^uV%>Y^lmkoy7q9(OYW7%?{(AGSTR-HpZK}J
zHcvU2?N7Vzs@;YsoZ~Houiri|m#{3${SyCz=-P|i6`%DdEVz()NoRHAlZ4Ar#=+({
zZ_HklwC&(amsb7orJv8empu2buG9U`lBD2&({g)4COoWJ)B3UaY|rfUv{zBw*G0;=
zzPxs8!h6Fq+0Y;FG7=+%w8YQd{!x@_wfx;o)ynUu5>5u(+^RhH=kGVHx8gocSsCgz
zyZTK4cnqjk_*d8#DU~n29~MlO+C8)X@s?lZDIaUTsjvRNL`HS>wgvv~Yj^ujVp<_6
z<#SZ(>x)l6B%Zy$e<tO*)zkAc_v|zH^-k2~+tqWoH^)AdIk#EIdD`04vv<U~_padF
zUC*rW<|M}|Db>K4t-=voJmwi4mfm&S<6G{w4I2Fy7A3nI&k$eDAjG)u=+aGITev1Z
z%wk)gSK9AwIXB_CefFAn5?`6NSo{?Avfq5le|L_Hs+Gsi^v^o?_ZrQy%S%t}IW+M}
zQ)^u7l}nGA6N)#yoKtZhJgOtQmW{Jk*ye`Dn}0vqzx%tJa9_Xw+b%QISNXPrqh&+=
zUW>zDi@*0NtK83BZn7b0q0DlZqq|Jjht6}bpYNoXC={)9o#DdfXj?=5Bu0UU3tD}i
zo5}51we{w?^5dWW|9E?6;=v06=cVOb8s_&cds><h?%24)nCs)qosy|myTUIQDjYi}
z+3d1A%ZjPTO6dIUo6}yHoZ}I?>V0QNe$Aw9`xecA(xtwA(c(Rhsh8hNn&-T?NPoWK
zsE2IuJ+`pj25<jlYH}@qpZGQ_CRFW)yL;5E=9bfZNt51{dCzkx-7h^oZo9)to$se7
z&HDW(kyW<v!nR#%*(cMx>ds7X2$1=zd%%BDO5fj@_mO;WcP9w`nkKHcFrSI>)zrOb
z;!0+8EPQPE*RuU;VVT+9&&uk*uP^lSTko`SQqDO`xf>np*QW)}oILqt&h3f4_bSr8
zpCvuL(^<b`PSWv|Pm6t?7X7%Nv`M^nr`DG#C2PWDEx!FZ?7F&Z{pRndXVuILPM_qk
zp1oP!@u}9M+0P~Hw4eLUc@(v4+s^cr3R-Nx?roU5cLMWoZmH^oeN}y%#r2XmzBc?h
zWl>Fm<(^{edpDVD_E<A}zH?lCVbUTaZuJc-Ca##Ss`2~D!JgUavo>qi$fv5Rnq8i8
z$#CKRiE51IT?dy6OP;y0Kz4Jnv9jRRFDyT^78;w@rmVVRyo1-Ir{vD$bF=F;S;EW=
zR1YctJ~-*EPh{3Z9-;oSkBdChirBm7ia($6S!274X8M$nbA_@~S4mfxD*e9EU+w5I
zqw#r^^v9Ehzc1<C{BfV_b*zbw%D=1UyR&XCIw2DFHsHmbQ-`0X++KgH;;H*<-Bg7V
z=_5v8y<aj+b#{OIs8pDv_V?+75AyDroJd?Rb7$HycXt_<uq(z(H{bqbxx=zo{Mn4p
zd+b1ycP#wo?dEz<XN1fRpD*csvgmk(Z8Gz!T1)$XAM@j~;uWitFRwhi<;~}yZK-no
z`;W`7|9(tb`}Tf`vrAmt84XIR`d7@{YsfV1vUTO+;}K!2Z%SXR;+OS5e8K(1j#*y4
zdHomH+&L?AZS#?pQOpzWEKz&fBQ)X2FMa!jOUw7)+wd1U%64<oi5Sa06V2IE<>#|2
znAdmxd~fT#*x~ZtYKd#cI<x+%8Q1kN#RfacR9m!l>6I9q7T@R8QT~l%@|l2u{)4fu
z8X1g&4;Qp{I=Tm33%l6U^!D5DI_c`%TaSP24~#S6oZz`TCHd5x5Vtd)PoGzuRpR9M
z_F%5NY5KX{Q_f{7WXiwT^_8b0CisW-J-@WwAyd`dZb>N^+BFAG6uVxO({-7VCE#aC
z<gRZQ!$h9xZhZ2Sae7Tj%B{TRkM4#AEfwHj<|^^gMYXC}O8ME6Pen^wPrKSIS?pHJ
zAD5ke#Jy}o-P)Og#X&RcBea)?H#8dNzDp};Xx{#8=_Xb0YrCIBNZTJx+3>k*+nbCK
ztHt@39hf*)KIEHvYu<vc@4gCO&Mq)2&iwXWude+W1Cxr}_Kz1C-~M>czx;Eco$Yk+
z)PP*k8}*+b-}XvW^)A}C-lbQsU#MxBbGKdW@vAbkXRj)I{@~e9lhQop_Wf^47`C>p
zS~vN*fy<<et+uXD+-EU+z((-g1cikUmj}zUDimEz_nB?f`qFsuhNsuuC+7xot=?bJ
znVj+7>o;FwMR8}oo2$Xg%keM2ym|gdQrOw&Ox-kJdBy{WG?ix<+_x?J@MvCs=Jy}m
zITLSvdKtA==lAx%Cj>m*etwp2E3KNiH~jt8-5IG@rT@<Om@M+pzy91m=FgFmn<u|y
z&VJbOx39kTGyBckuO{6P+$q+OuKcPyAmAa3nC+xao!BdUeDim0H58kx+!JD!aqh+)
z(Xi!{+3fSZ(szG$&gS{_;HQABeZI%@o4$G4Z=Nu6Cr?{yTs*neKeqXUD$^Xd*^lSU
zjTfG?zQ^&|MXz^ZLYe{X`~{k3&AC4%%;wILG!fjL_V{%2yJ>%y->S6e)#09RIbD6;
z=FeCEE!sRu#eZhuL%9a8FSkmjS@m}0*58hCTRi{&-RZ6Q;ZZvt{JFh-7rXGS*)Ia3
z+&^(@U;cjo=7Ra#65j=APBisb_;Pk_KKJ#=lRnQP_pVcnUJ-EDfwSPlY5AY~>md=C
zr}r-EsQ;?w>Hn^>DJx9neZDeo)(MNgb18QpzBb=CyV1Au=~p`jjllPtKhN1M9=%t{
zf6c3e7cGrZW=Riz<lo;IRk|SGuH)Lls5MoPamQ<C<^H{?N#d*VT3uv0*TFdbpQZ1k
zotI^9{Nv95mVa2=e(BCR=dZJwKlriA5>(#B-mr+1zI<(Ev(eUS-zVJ{w=&)UFJ|A+
z@Kq(g=MP7NVZ6`6TGzEYo9cbl^8Yv$*U3g#zG@DQcz^Qpi34wPN+x;zww`MxpLKOL
zW5e46Cv%Ed-qM##a81+5kew(RV$t~hUdke$mC?8Nd8V$NwLEsiX7l5>QmgN>xo51{
zHrLv)vDyCn?$+m3(;R<9o}FSEnAf~8&bd6s|2Z35&x$`E%kRk-Y6hH7zM6XA;3kjN
zub*<SS?8|3?bJM(-_d7WuQ!PVdHZWJ%s%u>XYGlsuDPnebLQA&KmVO8=^7x!K0m$B
zd{U_i=hfz$=hpC+rx|nwXf|HhzUQm2Z}^SUgFEkp`{&z#ufDA0;U3l0yW)?RI;dCp
zKvBJw_w#e*_WeHVt7bk*TXOmG`rX@pBwXAUBfIFry8fM2%=Hob&u{&6WLtd4wXKW8
zFMrBAyUFr?{QEUOm;cGUu}+iW;>Ajpib5CTk00eq-dviieth3C|H85-X|^*>k{RpI
z$htA4Eec=5vLJc?HkOpzk;`2F*YB>&y&>tJ^+&HRNOnQC|8lm}&HMXPXB&Sr)ctXH
zTa|6$v+9e|FI*$GtFI8e)mW#yI=g#KxMgKvaQeyH%F+4phtA!%Xw!0X%oj9N+hDM<
zS#-{0H<eRAUq6UB7<F3JU-s#zlV0<RVjlJ|iKcwe(hXLeU9Fy|;h?->R?74D*;;44
zJtdmIlzcyxkUOhy+rg-9d-gm(I%&q0yDfqp_i_?*XI1-NUUg3Rwv3GL&hqX1rWLOV
zids~3c*l(U<<UR)a-5!2VA9OHuxb}eX0Fcke;e<9jF1gqF-6H~DT~wnBITm_y?S?h
z8(dB=W7~H>VX<E<v)ziE->0XoZw4=ZeDLsCQH`~6S+wK!)>8Y~Cp^wF*nHUVuE@Gz
z^|OtyZ@$}d{-$^MCOPN!Yf4oY6&Kk>9^I%uTOeuAnO)bF4gdc3&PtQqoff{8CjvYq
zEawuN&pCl}jmV0f*;7wFpY?_1!C%qY9Qvw>j6U~NKYl-FwCg;0Ncr`geL=a$t{Z%a
zoYT8@cXG$|<4=msQ>0w}NZ0*hk%_jf(3-hxYnRo<E}y^0<CBg)cKGBOR`*FXc>g)0
zbB4Ds<dti@kPzgm^6r>-z{$n8yuk3Z&VIM`(b>Hx?kk7yoc6Qsw9t0zkn*kCEuQYT
zt*mNRy;{le=C}3xYu6w4R|OqsVJ}~@^U%UgdzP=8m9nS(`@LluZOzmB`;3k*(95xZ
z{LYkPMY_tNg7o9XtzL6Q7X~pc%-?A>iL)$e&ePAl*Mg;Agaq@awZ;nzg6C~dFt@&t
z7iS1qe6n<z@62OgUS6JQS1Xmj-tMWCct=&w(TTr{N+V`{m0gpY6n}2l-|HLIT|Vwy
zWw5sVa87hs-UYAUhc{037fbKU_D-z7QpMm_cKUAT#3ynr;e1Ez4JH&dHr|wVP|y4s
zw^-lt+>DT`a}J)J_1=r^mw5B@B(+!V{3|YR63@^1ocOR!R;Rdc{u!@p0T=kd!#dwD
z_$dAB`h9g(#QEl1+c%UilU}y9<$3SdSw**X*K@kRol{h|Uw&m;iVw4`2AA|_j~NCV
zri;rqmB{pZJzqSdCdK)h;G1wBxyJQAe{FasSv9(fC?!mgynbQAThrg+`Te);=2(Yt
zCGmGcvi7O%cE|3XxVnDtL<8YDHc6+ABRfHBnVA?@y3g4D!r)lq{LM-CxRR2RV*c~3
zi8$DG|M?D?gDcMOd+MS0aH6WyZG(%J>+=8Iitmj1tG#Q3I0wJk(;ImVpdoYF>~9uQ
zy|d0d`IkFmm(|>RlkL(&Hs6_DnKwh-Kzz@f|M_=in=R+R1r4*WF21yJ>Wtu(JNLdg
zcYU|!)#jS_2c12WF0R<tGw+*j;$gd#>z~3iL)NMMib>Hk7P(^J02(R$Aouz5d&ze*
z3L~F&rkTH=@$SyC6UC+vDxO7WPs*%qKByRYY^Cb8E7qLT3pdz>6-tVfGL_8fKYL>9
z*RAO->b9p}cl3MhPrR^y-JHhr?Xxu&?l^s2Md$a{s7<F%#`4*0(|o7(H=|v8?Gn(U
zwT<&TU%3Xbw|zGfQdyl7J^NVQbK|S*6(79L-mboTXJHUmG-yKbMf5zGKd0ZFDbWeE
zz0x4qdX~LN{<RLMjC5cq__KX}r`yx7`+AG=S7_XwlP>?c$kMmF&VJtMu)31j3%;_Y
z{rO=RY=8Uce)SUN?f*>n#&Q3cV^Xab&->PGd-`*Mot9nS-L{=I7WuMlMpNeH;McNS
zAd8pFHD^wa-r~0DfI_{NIcNFWn;)KEl;$gcn$uMFc9+)=OYYDqkB{qc21ZrreQ>#7
zw(fV5fyG>#>%CrutxbJzH9@nf75#z1Z~jbC-CJ?AO)V~Sj{8=V7ZSf$h&N|Ta(rs)
zb<fdjUYxu7jqYmGSJE6sR)yKyvu{VntaZB8a(btvdTP<k8(G)q@dRzMXx&}p$#HpI
zZ)fAxRD;`{Ud`^WU&Q{ey7v9S<a=&5>nwjf%-_Ab@YTdBwYIdG*O!G&I9F<0sxA6x
z^~`SH@3-%pg<IA%_txCG+iW$j!7(YJrE_`q$vO9?2UdK}cskK}xm&HWEK}(c)f5B$
zg9p#>zFTQ|=$2IG?CB3uGasI|bdS6(@Vr7nP)k+dSDA2a@Qos;!q&oNsyB+yR_J_r
zd(8wi%+q+ZXo}_f{|$O?SZ`g?n0VaHNVna$@Nd_j_xy8CK9$wH|0%G_YnGlz5x-Tc
zC&y!}yC1i@^i+sjzDZ>(;aFbT&R6o{!kh;O?wot8aQLQ%nrGaL&tDJqY^lg;D&C$U
z=JxLVwr{`wt*u$C<x#}%R<^opWmb0h`KRmp_diWATW;8Tw);8rh2Q`Gxpr(We_NF_
z&C`hMpz;;zJNFs+Ys+}IF5mIn=)8JdVb){SZ}J_N|COx^(|=dm+O&;Rc;SNWe|V1Q
zYSh%-p2FAFd2nHsgVLH#_2O);H`YmyAE>fVn(lHof!SB<<wfR?)t4%N>$3*jz&`!(
zZIfK@rd>kfTZ*^(c(;U1Rdfqp`in8|vC5KJazdMr3#6o7dM&i%aNX59XQvrux;{N?
zp01I)SJhT>X6>vMZ*HYK`-a@CIrzXwNI~%8?b%VBTqoyQIWgbK<q17x8+qZtxheU&
z>#ls*?&Pq*r}fMVu4b1R{BuqQ<(XbQw@R3+wt4PMR_)3Uj|{dSxUv4`kCg9omm2Jk
zlw{IQwNq#3zJ4p_=`YQ=#DY@E36f?fj!YF5<4ZO6QN7i=gLOlSq^yPVH&y?cR!g`3
zuFwb*)SvF~;<oRXT;DCdb_b2zLcTpQ>TSNT{mzN#=9Yb&auI8uM}9jNkbA1_lU&5w
zC$>Mfy9mgt$VP{-ulj3!Z_e#5|6@;H>@;pQ_|RGqR=;QRhULFfy*VbIw71^Ed3NgJ
z_m?-F`+ZY7GRtJC<8-?osaw;z)Y#L%uYInYo<C#Nn#9lNman*CCXy_d>$h|^&%^GI
z))mh-&&{dH=O{j<n_ki$5aj>o+!V$4j30h2I_bCkT>r!Qb;rKxC3lqNq@FmU|KsK3
z<(wb=l0RS1H%Q(t>ik}F&yNg=nQ`HF?qyuRFD`uc%9@BretC8qtMBx^`ng$Lv0HfE
z_Y(UK%jDgKWeNI8^)YW}iC>iaYIeSFPT$StQ>TfRIJjq3bO&6B4yy|d<u?77^yo<E
znK_n|w@pqq_!(NWB0n#|a;9OfPl)!^>Z3lp)6%bsac2u{t~X$JlXw{tJ3qD?HmAZo
zp#rqvwphko^Pbn1<&AnzOU-rW@6g<B_$2t`fj95m`11Tt+cj=-e0Df+p1HqW!Zzd7
z!es}Q4fjm%<a_!w$L&gMwm^v2p}fKZm2<~_Nk|)RpOff+{K~_)kkh)sExr#=SH8R%
zw4eV?(xUW&vc>O$oY;)7rQP|aa*S2;@J9pwd->1JPWj1uq`%T@J+O-H-m9E5|8zuK
z^`B=wU3dG&i6v?~`9E3i)Sp|KS+#7d5d)X}8BcNDr;$G%H@AgUAL(vp=NG#PS{8S*
ziIw}sk2I#LV9$m5M?>z<lMHHlwrQ^T{;etX?_2lXJ#g0K#o4=DjZXKw=B&*RT^xDX
z;NOqO{Yuu)L6@s+N<F=!?5z}A^Fzk{htsC@X1y-jv?biO#sB*M3g>U<A8RkakpAkt
zxy-=})l#?bzn*_qaLfF=9?k3YKlnY7ljA-2^r&C&GQS@uYP^^A&p7q}XYG+}`=^s$
zxq{d6p1XSROv$=$@P)d2rb8F%>a;&j^R+p#=YQ>?^wpdE7a70#y0fR=MkZeYyyW)J
zc8BZn0_)yHI_(yG|7=&@CG*)5H}~i)5cqnwyXW5z^Xt2g$N!ymQTI&Gd*i@jw$imP
z<LlJ5{j4R7^R>^LX@9=(>gbP;k0UoEFov&-Ie35n1JBwP;Ru$BYA=@Go>RU}`!J1>
zRY*_6+)zSg6}#7zE6rEe2~Cdpk*pdOa<%Zm4(0odN48&C^guK4w)>&uEYtNYJN&1w
z5IzyBS-np}<rRyEL}Rxn-?=6C_m?hj>F@jSNl5Y}-?q8Jk5gNOHNLEMc;7p}&&s01
zOKVRu7ifFh9EXEv11-vbaiwarJ+rdQ{m-9oTyg8W&#yFI=B!ye>hAhf?J{PuQ=Ry?
z{?Cz~%IlH4vgZ~5>bQLrvNHCTgiHPZKe48(EoZ(?<6dxl_m=(3o*re${uH_{`oQib
z_A@yH*PaaDnY1x@Q%UBT=R!NCty5nS<ys^Eh_P||YnHBp3yU)Hy}~;^_h03o<Mk)j
zOs`C~T;$cqE7I$NRn;!$e({xf;;K7&*7J(VIikVSEXt1@y1iZaQA%*<$)gKah&(!|
zoy&8>n5!T@m9w_((4qM{E;rxB-u3h?-0*QlMLUbr{k)fZJ&T@LTh=7Lj-Hs+eZjnO
zfrN7Kx}=14+3ViNpL&paKlt;_pFKHooDBDKI?b~Kws#%sspQTx`nO&|;^SBTB?~HT
zd++UIsk-ka7GV53eb2F1o7ML0$=g>N6(x0+-9N41^QY})qWk`_{JOVe&79(y<wp~k
zT$k?7Z4uVc2-@Ql)HwIRB|EE6XATreM~H~>H@CPqUQ+aY;cMk3!7#fbkEwA%MK;)3
z6~)gEgfaBhZS1;hn{{DL(zOgB?dOlmKAvoNZs0q^`D~K?)kZmk&6TG$vs&)#SyNb8
z$Wl@r7Xq5}k@E|&i%r|NT-C!SFZqj1;ztgaCU*a5&}z)1v$?)TF<A$^JuP{))^eAW
z@5{i&TuWjTuRc4=(86EcZ@M`84tsd=lGjhxzqE?934h4K1)2~7tqX06(Tuc>U6=`5
z3s7${yH4ZyAN~F5AD`J59&z2Qr6;vSpo)L)ndJWO3ko;Pc;=um$Dz>yq?f7jLF2-l
zwdeL2tu|h-5q3t*dcpB0xgwM2aJL9+z}B4FTsYyjXK(K#JC1jnR)yR!dzl&^JaqiR
z!qwHXE3r!H)afNLTR!`qd7J)+hvUk{z3Ujiww$%xxWz$1KtxWIxpd3A)CDoK=JuCq
zw^qG5bm`#kHpfM=y3D$#SJmu2SazB7MfcyY94tp$R;%+BFHH$``s3?bUljf#^IcxU
z$rKm8R}~>g&+M#Ey1Dnaj4#`_B~Ba9w`t9BXnbHc@x|?1E~b(z^on)NdULnWII=F4
zTX^xSm$9N|{}!ESQWxOh@?YF$pS8$&!fl~V^GX9|1YZ$p<GW}3-2d6zh4VsA%#n`?
z)Hr%}&Vi&FTPDYik8I+<{uDo5wb%Db^<K?a3Ky>Y_}aq%BY)rRD;#efcF&7_d2h9_
zDbxFTi?b}ZPc~QUNiLmpS@WuCYUciVAp#ROerFNn)6x*S6nm`k)YRJsB6@STZhW~x
z_P4IIxsa&M(ZAvxEI$`+DvoDc&9}q!S>N6LF<YMdm40~l@WJ=}Mmr`%s<n45<tT|&
zsF2vPDV@1Hl=b$i$~RtH#DD(uZuzFDmL|S=laH;S*&ACW^EIg_&E<{j5-bWgM*sL#
zzy8AhWZ`G|_SM%<C22*sb6yrV+8OTqTqysdwCWs(gUxvfwL6u1<;x;|d}=S@yL(Kq
zPgO&wchkf=?`lt<noF|{O<hdIIatiE&zN79ddGz2)3>e9&y{{yml*srx1MeG`8!>&
zqkGo7D`=R_xVf!h+vlm4#~w}n{Vve5CgbMb*{{P}9|$y^4qX!W`JBz?V=-pB1>Z&L
zO!By@*RLvHF0n~pH2cb8@AGAgvN+TOK5UpU>rsMl&)gMDtPNJ@ZQQv6vXWKIIh2LT
zkx_Q`>1o#XUU9dAC9EpU4O1k=LnbtAcvNT0ba3ync^kS{&c3@y=<ChxGKQ<aFVl&v
zPf2+{QRjAwfIRP-rGE=tiXWW$x;tvxq{gW(Pc_&&;%>ByHoJTkx+xy1qvh#gGwJc`
z{Ee&UcwYRs?uGi!M>o%_ubeTrK3qrr@`?C%#;HeEbKO37SEFds^1gRdRqO0m?f7<j
z(=@TGrBl9No0wKQp}<Q~ZGWEq^1`c&cKKB1%W7yvC%;)2_Gg-|PY=gcomrRate9#-
zHmr!*lxiQp!1K_YXAfRfiUwFt)?3x)5Fp_7<3&bv<T384M^f0^g=KbrTf_=pZ_8dL
zD$-%SSYZC?YPHH&FXDWUH%eHi^9a3N^^(Kjm4{7!di0`KHq%^A%KVND3O#ot@5+`n
zze=>9p71QSFF(}udSxE($yKdl-O<*0%WPRM9(0)8a(3OMS#M5+dNaNXnYuEie1n?f
z(xe4Gi``2k)@Vid-+I5tJh(CYT=A8!mnWu${(L@V?hFyGb+6ev#ipA2Zr9Wj7ZzP4
zl)q<MgM+LubE(9ef@g~kxmv3%U9t0ybmN16$>(3~T#$XfWqHkyFRwica-J@~-*3Ft
zeXGp-Ia5<Mn;yUCR#R&_)lgl1x6_r>#pd^pt(h6KW{&ko{^`MQX86=iSsBAu|9EX$
z)kMyf;mmb5fBwIjTV5e@^?Jkon>WwqZSquLR5IPdz^M7@B9n(cQxJ<tpktSYMub2}
zNXU^kC41?k8omw_)Gk%O5a8@+^zhDDIE$fCWp>W|_x*F<ESsF*p!B_Vo{fK5*{QiR
ztIy9Xez)^EL&2vrqATK!F8@4Vs4TlB>Vsb7G%v@0mGe$cs+n0VJ}Jif^&ZXWeM0|!
zy<uC~AusN>;6c3dicNhVW%q2#@wQ1!>vcUn;aB8t*H4n%>vJ|e6kqeWd0o}k%Nl>I
z?jOmDOsmn)UnRYH-v0VUz5oAaDe`rim2TZqAK?CUl2n0(MWxN$9cOY^E_=M!uj<dH
zt%g6_nooQBZMi3PPR?kvg5l-qs;T#+A54C_>+Wo;8MgLwH(&j_XTnRlQ&$WPcJe*g
zU|hFs=Inz}c5kx|@NbwQ`h{)!r1YIhsXx3{zmQ>a+~?8rB2{5cPMNB+VwStr<-YWk
zw^t2T_0Kz&TBN1inZBlG-$~J`n=4n&<GZ?b%Gc=(t+(ooLSMZ$I>~JNmVb%y!v&Mm
zcS%%g%zjsxk}eh8XUVy0NotLH|FS|ArC!aA=a{xDU$_1`P4eRP*_R~SXT9l(O?WnI
z`<WSa_okG*f0^>_&6b+f^PfD_d*!b@(`0{r_3FnUZ`+&G4fwZO-q>*PjQ(_y8s?O?
ztycn#J-Yk(-gK6&GQDzP#~Sq&%d3sq@Bf=}PX7M~S^jrVeea%mze9D7eb%$Rv1L12
zWAZQGIpAPAar)6^3$E`>$+>S~oS(CG_j<z>yBo_yQwlVeC_mKQ{p;eZR<&u%J0_}W
zJX<37`1wnBLmT^ULH!>efA?p1I3=Fky;|a0uE|p)>&@p}FD!a49m)6o)pyC=_ue-O
za_>&(Nj_6Ey>Z#=ACeQ}cMF}FasK_&<~-$y)hTDE-AjJpA$3iw{cDw$>24dp%b&K-
zx?NuIF=fs1lOc<l7_Rr_B_5i*ezWfFpHI`y8$RDG{&T;MXUe1-KmMj1^NF4M_|5q`
zo9W^5**z!z7v(D+a(o{w^k_|uROacV7kf9}J=a&8woL9@3+EjB3FowD$NXM;<o1t8
z4X^+2uzk5^(@CGYTeDxesau;bH?+8OB)?EEeyYEu?Bib*f-<X;6TJ1OFJeeIJ*&Xs
z)8tU*fQJDtQq6~hBc<P4wby&jj0!Cc(O>O2eeGYxuU|vVEiCJ<oa&TbRUWDSxguNk
z<MK5%o9?}dv#<CzVdh6}-<q9w4za4Zm6|+gKXmQtN}m5GkF2=*Y9F6#2~Xn#F?-2;
zej}l<Fg4A4r*3=*VLR^^-mB56mdn$!@^`JpmG)z)DVKzDd-(lR3o2Gc+f7;d>ZtnO
z7(b0<&)zJCUFQPV@8er9&G)Zo^W=x(>n}z4XU{!!y*udUv<HE1kG}2TacF3{Y_?3M
zZ0`S07fZG?MqDU+(l+T`!kaTb*S|DPdiCx4q2Qly7hS&l&WP>P6x+@GnqRV3o(|?S
z%UhD}<86CS<IU|ilbRQYtvDa5etBs-|KH{->4A=>42-#wy1l2p=JakZ)Qg-N&o*;;
z)L-ZS>l~NmMb6wJ&9^RvD|U6r@dG<{&8m4?6=160cIEma*Tm<qLfX!oef-Bay?Cup
z$GnA4V}G+=|GFraRnmLW)Pu&pdn}Bn`o8)sQq0oypXuZ6`qbHvQj+?s5;#&{7=-Uw
znl$_9tLDg9|F*1HOCiH{wH+x7Wxg%TV%Q~<*uO{g9D}Eu@GqIw%U4Lc1^4gdaai^F
z=qrz;q@*7|_r@`-nP=bI`S`f;%hw(`JI{WvNWD9)DoI!WTzL4mtXo0r7wOl_|L8RP
z8<#x$%B+>IW1sim+c|}CQqSY>kv6vrbCzvw?mQ!PtI;qx{Nl%_FaO91JNx*UbgT2S
zCEWXQuxFOAs#U826YF;_^~Je!f**7LIeoHa#clpKn<q__Ul7zW?abtwwX3rj*|n4A
z*nI1lcj>@Q597`L){L$9x6Qiy{K~Y8K~JYlO?;>}?bzy5L1yVS%b&&R_3uA>@K(fi
zJMZh!cdl<&iISAbODd}gV?LX=>GNCvOrKk+Pd6XUoD;N_?Y{!=(p;{>&MxbJcjrBO
z;?(8jGpXr`y6~E)DQ2O;-g{HhefJ3OoxUPiS>TELZN923$pdwVkNx=iif7i@gYSHf
z1+dz`xZ>!aD*K?o;Bz>)_?7(n+q2HS%U)OeHFa;r*0#3$W=lVR3Omp9=jP$GHhsV6
zuh*t6liQSTxQXFe?uH45o+3g1(_QVZa!+DV*Dl}5;*of3qI~#Sxu277R{T~z_t;?b
z%3}d}o6?^<de18g_%u!O!VT3adh%w%>#o`R`WAe6mepOQ*Qqy`S7K7kvGkru-p3m)
zBMj~zX;~{~7cu4CwAueu|E<&s?&$J(y!7bHp6G<6%sJ_W$r-`6amPPTd(AfA=i%m8
z-Cg!i1Mbgs`RQ<d^S}LCuN-Gy^VPeRb&%;t$3iuYZz`Lfy;VJtnE$CDboz&^OT`<0
z-+uXd+Y^q;X-%^X-u<1oE>Z9&YqfsmlkOtjKH;v(jf>|kSY%b=m;Z0()-K`P&FYmq
zR%wgvJ2-j0;ihHY^IG=Ww9hykoEd4iTyRF!Po3#PLKgL(47B&npRu&%$PLT)f&TS<
zlFrK*3~G*5+zKn28_&1qY;<&505j*D+m(5rq;w9vG<{gJ^?sMy7u9>R_oAkrHsUXn
zKiiYY&$J=`l)$yQhTZEFdmIk#ZJeB&ekDGk*{40><X+ovZNDDJ1$m2`>Un><d}+xP
zmrI<|^$$56R|!4y{SmPE_xeq5UdC-bT`{HWPJy~oK}y&<o9ox7f00-|v(q}`X}{0A
zM^kvN@7_K|(?whS>ajH|HggNb{`zIGrEN9i>&0tb<_bxsur&Rt)4LT~+B<!7{mo4p
z2HNEjKRz#->)fuB<1sN~4Oin9?Qb%Po!!Q~ccNb1+R3ms{mp;5w}CZzYSx>jB6p-c
z{1zLy{de|D!3T!RW=+33K`dW6NcM4S?(w<m%VpMt#I8Q2(^`9zYn`?J%?`gXdx_Lr
zKX$&;Zr*abY}=B`bG3)39bUJ~His|lSk=70x7MclrP%1rZL_MYynp^H3w!uku1xbc
zZvuJ`R9!VqnV-LR;;J3JAJ*@BT^aDaYWwZSIdep+7A=@jY+S{CS(JTo{7X+amfZ`Z
zs_m+-h<?bu|I5(2$gm^J_S24Pvx1Xu{%RMEt;2<uW;%aY6SX;fWm%tna+<x50`F}T
zMg^@`5+^IS=znV8Sdw>t-n!a(&-SkLW_V#xVDkCe`^199>>j<hj~gB3oVBf_=0)S0
zW3QOcN_^b$+Ab&Cyy#Mj(yQyD3|rpcon;jHZCd8N=at`=`&K`8dA_4vAe-rZaoL~5
z$14RSZH0JWWh>rm6Ir0TCZw(2L21pzX-{vz-IBS=x&P3z<}wxgnL)39_3jWhTbFjp
zDyQs1CZps!g{PSeOfQT?q?&z<?eB1A=&5T5aCwv$S#IysP7hs^wBEAf#Qm)oYq+02
zyI`vR)+^KD+J#+v&Ckkh`uv&A__AK-KLvp&txwbim-PKie!giPe_)VI&E8`V&5oUV
z{mIV%R3^K)Oh6#p{2(LI%`E>tBaK3y9ew3&W>tQbSx9%*iyx_zdv|>;`LS@r=LNpG
z?aMB_-SI)+=6coCPv4Kev(8=~SgWfLxO)>@(v&m5uFgF;@p8po!Pg7IWF;r8hzeuY
zJQu_7>$}1DoBDLc`zu$@o5x#fd99jR#v(J_L03{&Ai(jswDVRO%ap`ZVH@?L1MY0x
zkbUt#!;QVUrgrVBYZ|7n7iYg-Icfi%?Irg8dv032HqZ;4-^l!0!1TYq(*8*2H<x=?
zF5G{r?R$7~#lPw-X<I*q-H$%i-QNE8*_M}{uOw9+)HH60TI;cgADH3Jz4uq`$!XVT
zCHb2+ADk|v`sd^v?`MzV4S(L5f5usMUVG~OhGOgX<o+THt@jFfxgVN))~rmv<tcsp
z`|*UTW$RujedkD0`(Wqpx#{!rUCYibZ|XBx5IjTAyQs<N>Xs>Pi()%?%-`gAFus{r
za`<^))X%!<&-1U@ef|G7EG+I+{oK3q={67J=32iDZBz}szcjb#=ajt(k&n0*^8DSR
zGnd)i=Itf+EXiM6^dC!|SmeyXQu$EsvykZNXa6rx++TjtIJs3vbcKi`AG6*)zn)ib
zKdxW()Nw&_uI>z(HC{Wa-|mn;qHg*ApKGyTYhfqf_u|d>lU8jEGS!Xe`}%Zdo7J0-
zVRmteYSr_mU-33md$%Fs<P(M!*DoBn!^rl}zsJn6P;6?wyHR$pr0>~TG576ec1~6^
z%)F$c);HfHhu1%6a_T`DzGo|rt6%=IF+E24?7lg?7tJ>wcQ=YHG}!2VSf@IyZpG5q
z^3TimMQ-r^dUuI{>`uW3#aO8)E2Xxsi*0rI@6dE=y1ItJYQ_iwFBaCcpL3?YzxQuS
zXmW(;WKqFl)gFg~wnuJd%-(IiC3n^0;tf5v7f-1Ru5u4kH(s=P-l+p056?3&So`a6
ztS!I)HW7u(uf?)+AN5E7bd3GJSbuTytpNFb(rHW92~JvnW5%}qFOIsp+PALk_VwxA
zk`n0SbeQMe5>}>0h2^Q!6GK8d&&4KB_+t`cULoVMPIB9xjduzPst-DHNhjMBtuBwY
zUC3rpeP){K<@AZ$ANOVH3zceRf7<)JnDboqBJDZ;XLieem3r}W#pK3IR@ZkOy7+qS
z9IwD4i6{H&gs%79mN3-sW$v6Qw<hqkxBrXJa}Ek{yes{8Ys;hv&6fQA@rTxH^9c(J
zKmPao>-G4VmYZ6$_PA~=_#1E8_kHcvu8XgB?X!FxzOm}9mdV2Pf?-!D<nG^>Gv}uk
z!^Hc0kNoGY?cV-t%cH$pUw)A=p1aW3`}y4yIi*aWkL;W~qj=@cnuTqV8M6WwuerFl
zul)DF88x!seP>-c7GENm$inqb_EG=syPT{}(yugoovtuSZ`@e6ZpA52k5JtkGC5Y$
zJ|ExjmCP(#TU568W9{CTvF`hp3Z;f_E}CbXrKoi6T)euDPDJjbsZ*!f9Nt`1vFLN)
zIivU5KQm{3*fNpN%0eLb@zVm~*^gxNRCSay8yyr**i9%ne~YE%#fzD?$Nbfee`d<F
zu^xWOqkBN%bAf|j^71XKPDcdg+bh4H*qmi+)9&J-D_bSwBJ+)>O!aQ$m$xVEl2@F!
zXY`(DoHlct`_U@9khSxQ-x)3GIhM;3=d1hR@J{AqSsrekyzQ4|x6S`tB_#WL<*9`?
z4<?nyE=)QfwnFN~jeE2ImtI@A#CqnhY^(fr=DVkhX8bjYy!4!#<tSUR4)^{WP22uD
z%wl;e*L`6}y8ZY6(dzn7bNW6?IZO{+chlkZCH;#0TXys0ueKhDIk)uct~;v9`(~fy
zXWo6`>C=KRcb88$CtUk~Vq(zF!WqB+`|g)N|MBnE?vLvP*K?{VEU;QDS1MY+A^-M?
z&L?j^natdLJH~JKf=%mo3uIqx5!Bd`Ua-m7ZnB8iw#^E0jC0<c+G6{1j{44~N$(zP
zuwD{$xZ&Tul8~uuj2N?c?IRNp`&~b<J$25p1q<WUnhiCMbO-j9J}B6bFuk{O_nMmh
zr;kmGZkaeuYwgKJXF|27>Rygn7TL`%SMWAk-(}fM$AfJypQGfCW<9LhY`uj?P*6+Y
zN!O%z3H43O*%Icf*Gfb<ALF>fam^@mg89<I4NA8TPD?iXek13uQMT8FS-d4vLZ6=d
zb$8w0GkM&OCzB4O#NCg+Wxagq=C?hEZPPA4di&<gCLLGJ5^IM=OZ4QOy_OfMUzqeK
zWnT7@)0rGk6{`>ZDVpBQSy)!?mRcJrTg$vjYL#r(j&sv$jvf28rAe|n?%sc!%Ns3c
zue#vtJFocPgsXq8C*1xkVU)7MuSorCR*d_(!2cK0SZg<2vvr;=kpKMf+a%is3o43}
zuEt!`@{6lB%UAY^UbxfRu_r11+@FqO%Wo?$EE4bc()+1+^6rZnnu<NuJ@exH+YioH
zYTxI1zbw%t`+8s5+gX98C(QZRhb;T}bn1S|v=!erRGSsu54SH==}w&Pbfn3W;meN?
znprQtn$*`!S>@+um|t2lt@v)wy>mK88&%4aJ*H=^%jLXoY%6dvzeP~vNBB*d)Z*%&
z{W2d~YSvk2gxl`m+Wr5;t`Ft^^j06=y?V0bh4a&A9`5mZ_gqV^{^y37@snciuJK#u
zVtjDsHQ}usq3-*2jO%vj7aRXQE0gwQwH;g3YGXdp)0IYNSf_k%Dfhf5r6pn8<-S5*
zLzBzg_}kj0&lktp^}KxGyIJA-R_7?G7xDL&YW!q78Dg^IouS@6O{->y3GCh5Qn(gQ
zGcCz@e|F2meS2<Sb%=~iyr0Hh&*07%bz`kmkDchAA1kJ=zT(p>AiMZ*OxMwm9^JQB
zWD0BTe(KztCFk$cY$UfcP<eXfrqaBfOA8dN(vzAFZCe~<I<tpU|7Q5(X@>6SD!dX*
z-oH8=?*Gf+d1ik2PtMZn`HgPJOecMwW}EuN=E;J|`sR^$BD<@<d|MXX;61JP#be)6
z*@R8P>#k*g<(;&0ji39klvDF`er9T{-R3T>arxHn5AVa(z4I#V7VVt2t4?TYQ;gr<
zE~6!}8;xErIK0;GNtyk#p7p6S1)XbuR85!|m>8F5tFLlq_Rhn-SFZ2;CVBa4ch7>Y
zdarhGkJ)|KAhGem{l||QQ@_mJ5%f3ZWV*lZ(=59;|F%?|Vc9nSyxH0O6$gda?@W8_
zH2?KnRYf1R<==A3em3>hJ9h6XT{-*ZdjI)~K7Z<1#jjY+=ewMG?cD#XwQs)N-QBr&
zUVMflL-R7f;#95Dyz~PvB#)@wJ~+|m$oKc3f7WsDl6x$9TU(?dsF&TqnYZ}vr<)9i
zw$DAlc*kaQ^G9Yo<_CX&<_9|KGphOhn;2+(;mCbXwTqSiR8?F9yS6xedvS5`&tI?C
zXZ{W2X>0I$#3`;EC-=ZxS>O6``fTIYM6W%DS57p)G;`D2^rYh}->PL#=YC`JN}K0c
zY|+`Z#X$F%V}Pmb{IW~W3%;n#-}vE_n)Iak(=7e(E2%c$Jojg^!G^kRn{UYI+<d<v
z{`8xZ$3^R6Gn_8pnSFd!m6TYy!rXp-?HZdC?VD208@x)1Z<+ZaB9TQ*yv#Sx<7<(f
zjAhZ2b+N@KCMcenYd!s&@S}@TUU#OSziaMs`o}xAQ)%b+WEO8K^|t!F`O4=h6^okZ
z^(jV$Cq6#&JI~zs{}$m16(2q2XH7k`>VeA9$c|6fF79knJO1rr^k%7QeZCJ%XXd28
z@lRX#?4NOXL)n1^_w2TCpL!{7yUDm$^SB6GsriL<P3{>HLb*EDtmj)^dU%%=e3>$r
zd1IN<WD&X7b3(t(u}xi^R2n<4NI+r5o@wE?Uay?td(*pT`=_&2+Tsll0`%3koz#0j
zXU)x=rOxWk+isevb3a?2s-+mn(0af>?Tm8Ot7(^>OjvoGKXS&F`|D2q-ShO9wISE!
z>XfYR!za(lihbezViJ6QcG=Nm>p6v(U+v!1n`U`jRQmq9EqA`VJ^0qY>hhLEjooE$
zcV%CHbAP{mx48bZW4+QZN~}IA2XmZR9)5Yp^V^ZzUe2F;?%$i6_GdTlxw=F@d+MBB
z$JF<`U0S^?UF>{aAveG6C#m^s>>Spw+WEqBW#AQF=}Gy!x5%mq86KN`?e4u3Uw`+>
z|2cHs|H<67mnRjp-2DH$Ei+qbb58xYzcYK@h)>JicG>C6t@%HGt$p8dYKmxFo>_yu
zde5Rv)%YO)HK&3mUwGT_%5dV{`CkRI8}lbmRCYHie;2bZcDGwX!U6*(*6j=Lux702
zo_u1-n%k#YGFMD__oRQyo_=1=O94*YH$?l?HAJqg|LhW;FULF0viiiCrU-RWCY7F5
zeUFo;tc>+_|LPF2_rj4+Qx4q^UaG(=$~$4k3l%-v+qFMEZ62sEWIq>M^lYI6SBs!V
z&w-fZVGV5x`5SJCCJBA|Ejf{cbCSkF-4pv<)wjQMu|D;~ljXN(#3j+BNg^uwo$R~<
zlb48ieG_>i#Za$qwKL(jc-%TaB^8HVx-;i}T=RO`bcNH!Wx1Dci}I;y?A$+HMBPx+
zX&a|<z=EoP9VP#y_(Ql(l`i%AGO52Z?U8=Z%&mR<w({QDlc{7I%e&FvQ=#g*k=^{-
z$*#ZCRzFqFlke7?Ia|$m!O;jUf#iMO={hR$`#W#Gi`pz9q2aT;`YMOW#2rli!I~#M
zQ}k{N3(rxx_vprs+b5#JH#g36zv{C@IE24ACMb+iV{Z72%ePN&{<dVo9RGqMyZ-k=
z!6HJzA|D^IHr<wt_VhcaE}G)sx}(L~{M@HW(dCO|Z>F|QS|>0$uHt6!jO;sdLW}2p
zS=jgR+EW>ZEt`%kO-{2rcj8#i@8##c6xELJdUt2r-Jkyy1w`UgPK9%w`Z{~|#qNr<
zNBURlD*2peu`)lHulhF4_f=@&ho5hDW^_ATzApWj(@4x@{c^eg2NWi<Zjn&9#=`Z_
z^i@dF&PiKRxA3f4c5>kn6aK3obMtLgZbeL0n>QUa=g21C9P(g)P@R*k07n<MIA1>d
zgC9MnZ|v)hZ*LdVn|tTnj&1tM5<i)?ew|d2&=ezR+}Kp#yZE`RO_(Ed#)}Il+xhrf
zQ#LbgYT18Kb&HPqo0{EGDLuj*EKN<1SG8&haQvGcvgSV1jpn5Ia~IZ~zIY|aY5S~D
zg||}KN=ghI)!&S-@n-AEO>>ZFY^v^`d{Jzf)toQ+Ph2ybpBVk?5n3>z=J3IN_G@&y
zN;}w9A@i&Yf)*WA*rum>e$M;)M8_lVE&Sa!K1&htFnwY8L09hm^<KA7?@o4A0k;JY
z?mczr>18%FyWp8Mlg-0RGe-!zMX>32JNv7Wq&wT^N<80?V04>Nmu+#}6l*aK<$!=Q
z8#Mp>iP=EreVZP2tvQ}?rTe*phDnm@4(MF4gMz>v_l}tdl6o#iD?Ko~_n=<+PSB&y
zu(&I{R}!{adG30aaJJ4mXhK{2>lYgH)6!4XZJZaNAn?iegn6Sw!ETEUGtS)LUK=ZI
z@T$8yVOi&k+-Qr6XHxt2HBMO>Gke+Dtb<>@Wvm5`9u?|+d38<G3#CIGB7da44r%*K
zt_*!qweD1|c-;Q<3nyzVF3dh<)v6*KVs`3B%d3|kH#jXi8Jd5p=prYFswdpO>Lv4j
z^lsQEFY;^0v*fym#}!MvH@>ZIj6DDS$!ccaibdI6-q+<0Pbz9wIoiJI-?|HzpXL8o
z>TyV9;rgeW!{@+$V~&h{uDaey&rbG|EvsIAvb&jo<K~Cy%yau)JW^$^GRLNhcm2M!
zJF%80<l4cHb6@&cEi)*aFp+JOuxjw$U%Qvdy;RZ464hJM{ZB)H<EPq_*$yA#m)>9T
zJ>&iovwyopuGQLhvHs3oka>vxviqk6`;Xjz-v9aa8vPq(ml`jwwz$8tId-!A^VNUD
z)c#n_J)Ui~a{j%xa&v2q%R9uc@~?d?!gXx-H5qQT-Ete>1WGhE#h5HTx%u)>@4qj^
z7ZqxIUQ1y=ofI-PWB*h?**%qNODgo4r~XNinmUC+{oA5bUtwzj+|TdavPjeW(u>I8
z!#^+k{jz=Qd~^48DL4CEl@I4l?sb=S?SngBKIM)7ZOJ<;w){GINHbn?`stG%ch!1V
ztew6hc)#FVxy+L-ZMHij=H#1wRq?L#bDp&OV?rvk!o})OYFTr4IK6zO`ol7ZdHIyx
zPrEj4s4DT+<G+1~`>S_XU&N7)BL+t$@|jhRN8RNTno!E?yRG1h?(f`&7JtKi&%H!b
z_Jp#kKK?hQL|uFF+dmT*6<n?Pe(Tb<wT*0S2AwN(8XZoUE8b^m`mYeCk{d5qDBa-j
zO|)sIeb(v^H;Uc<`D_)xZSA+WVoGk8u&d|iJG1R3SbcphkfP%`wQSe!Ytw%8-v3zE
zGLvDgMopo*fN|qdvDTflFRz>$^&vh(_2|J%%wp@ZRGpYM)rn7k5|yn~>1~tIz!Uqp
zXp;I#r@L832g}r7KN6K{JrQE+wPdNLjNR)KTlY9G{bclCe1&7X)1gv!Ykf`2Zz~0L
zf{(jDdQfOB{waG;l)J$Rjh*IGIVP;&NYKyNG0Ek#0Iwq(=j>QT)0;E1XB(}%zGqV#
zmm9BH`=R@sd`iMLzqe)NB^2mR56}`_u&`%7Q(CJ_;4>#sTA!*oH|u1?rwO%A`#4#?
z+wI%BYjZ`Mgk!74vTv6^|9H2l@5z%VKG9!)tj>P9VaATU!WCRQdM@67@hvmfzJAi`
zA_LRYuRSb^yR<dSc<SmxIM%fonp>Zkc;;uYxQEg0i&9K0Vr-w+ot%EIKrz(&>8z=r
z_#c);7|c94C8;pqYiH)67U3<&nACS|REd9fr&Z-bmg95Zo1dnBIIlb9a^j<)Kd$SR
z&A6DwJ4GkD@l)eGvA8?7)AN#_F%^7!|2je|`#^$={u{?c7On?>GTEPVuZ&o0o00y7
zL7BHFa{K$+bNqf}sxL@%h|fQ@s`*XDl$l2ubiZn4Yp!JW;fqoTHoW}tZ^5S0QmN1S
zD_3>QKl3B*UdVa1++XUqe%xO#_UOv_x`S5t-oBc|W1AHzx$V-d^wZ7Rm4+La>UAHw
zym0kIzO3K%u^CCBb|)2Hfu|$)zDoUWaOGLTVK(FaDM8b2nV1`F*e9l&uu=0xsov2w
z`EzEPZ@b0GclEPV?fDCu!N(6gv1&Mc?#Ik$e|jSu9SY)4Y#0Ci<HDLHEE-edlO9c1
zX{izB=#s6wvhscS3gs`32V?%;KfEPk^2zGOnTM`yHO`1W$)IAGU%;O>ucR_mO((+f
zX7}%#o8EX&bG><?w7ye(%1xi>*E4@jf0B3R)*nsZW}D>CU*5HTk@=W;tvmbHr>m#@
zUxaLBVffZ<o$>L62g9qL$ODrU`Jdi=cjfVZfp1Sz&a*u)Yb)a`-d_K5-_0xMu6Ow)
zJ^nsf(zrCVFMr<O3cdAq*JkiU8`OPwij(~)xlzL;H>&&ZZ+`}Z7oSu2e9X{TsV?^M
zM2G&}<Nv>WTKtFQ#Ma}R%QOG)-Y)mizK?@_@B4rsS&f~|C%b-sJ}0Uc>d2&e_uv<e
z=?Pvov(AZmF3SuJb!0L)6=I?_Xa9=_tIl5Twfgd5;a0v2nA4W|`ykVnVczUNs&~!b
zw&d8;ujiV72#49lB|U0)F7w}geRf5DXQzJnl8vVFtSbwZZIh#3eRNB_|ICD~eYsJV
zt5ta6*6rT>^1oK+?OTu)$Fp+1YxkjZS6A}P&wcb)R`2l^@Kl}RzbRAwugyLw@IIY?
zrsVz$N2Z_Fb#fHqXsKaV@!FOCRb|zbv;UWeoiJ~Fu)k&AyUAt7PdlwvajbCs@$Kd+
zMvfQSMsJ$0$GDxp!n<XYyK(h3ah>4Q$+>4smaXMazOti`C*zfnUstm1`O<l3SWPXI
z>K;%3BJ)wx_Wr+#doLQw+}YTXXj=N<&bb%!ns3(MRb@VFuD3I~^wasvKc}7l$Xh#S
zZ*_h8<1-ucnvH5M%hlcAJ$>%&{R`8Ud9=<^G0M(hKjstbdVKow&(%IWhK3h)=9x$|
zR6b@m&hneG+V|t@mu8QGlT3R#`+rQnxp}(q&O51R{+YbX{ibr`Bk#+@_k1^Js_#sB
zJ|pt=kH6pd=FGn8EYWa?L*!4eYv#N6VcVN7t*N|b683i0$%e_i6Z!aLwix_bUb5XO
z!GD{v<((td*_R}(Z@cuIygaq4<s`=RpXG8cy<Ew)-ZPmV`QPZ+`S3_mnb6nWruD3c
zmkCT?A~5}pXF!3e=fj#wPCiceN)M{!&t&Y472i{H<4qV_u&~UYowpXX?e;EtnX|wo
z;loFcowHXx60JYw*tIMEr$g^*-OeuSkbAI6L8V>O92A~NFW6vt%dN{-QS;)lP08Cm
z)+m8$*ZIaQ);yJtqMjSi#8xVt-fmcZT{-jjCzsV-HWz*rrs+q%Gplg=?VIxa;mRHf
zgNDM*<)1DtUT0Z%^q+BeS!}~>LmxS7e#<Yu_iC3}{xf>veYIG6n#RUA782{F%4C0L
zB=FDLRzK&Yvh|aEhm&r9{`OAc`f#R(ZQ6(9FW&nnEU^FSnm8{`_+0q!wHZ$FkM2HP
zd-e4Rn;W$z#%;D72li<?N0cAyv3=^+cAdB0wCw8kDp|RyCJT-fy<5eZV*9W0-#gQL
zRr?uw-0GSBwsJ;H=_t7Hd&-o{Ws7{Pq$C{U!`$QTwND4fc)31S;Qi*J9UbbwW69Sw
zyxZ;-q<CFfV30bwRlfT5MJC47ttNU8g_1%K?cvOS5*BK7v><y$EL&L|Q;y%hm^oo9
z%A~m}=TBXgt+{erX}r(1ude63%@$4mYVwJ_$0+wtLUL2{>NH)uG;8Y+)s=Cjym5<8
zbFet=6Ls1rYmxErfMfTBe5QkcS0%7-PYNo}KepC09W=(YBtH3-q^iO3O-a83o!omD
zdbPdTaQiNs|Iu5e93}IQ_8-6Zr>!!(UAX(uzr)XWrgbFFe$?_f-k{(`nNd;NyZig^
z*Iv<YKlia}`Kc2{l2!Y^O77*p`0mj1Erx<$Uvn|s`kv=IbIml>v%hWljbqtfE!$gY
zb4hgy<6GyQF&lSRmwmkTl3PXNr{ye{wxmQBt}cPrr|fG?G9I&fY{^`g{8LTZ^i{@3
zW<J@I&O9l_teTu<emB2dxzxC=WzD*+fgRR~kF(U~9!X=Ld#&`3Xw2)BcWhs^p4#8s
z`+m(_x360pT;86Nns)q#<nPI^?^&B(y}WrLpZ&#17AD2}paPzy)ye<1;^Ty~><{jo
zTcoSL;=1;f*@o4Tt1W&mD&BjCe{<SY|DcssU-Dlab^on9$42J#%7$bH6X_pf&QW1U
zmRW6K<CE#)o4q=0)jE-HbClkmORzgHAGoci=OMdfWbpAVlg+lh??3p*r~X*&Zk~qA
z``4?0&MOIRnW*05a4=T&-;Um3-90PTG5=p>b!bDNxeGTRdvp@Fox)w$xo;Du$Ig9q
zCs6ydaMwX&k5iy%&#c_Kw(*jcN7wdGA05@#e>3Aqg*TW35^AzbpRHXs|I^Pp?$!1Y
z2T$+4r<Qk`2Rv@SbwTlesXaBZ#%Yr-a_j4dS7a2O6p*Xi@zb_&{?FqjKZCa|;!)4l
zP}Am9`1+@AS)IWg*5A2bs_XfxttEaed24(~V$bOg)5h<nT>D*b--%rq9?s0`<)t!z
zWBSUop#eIcepmnN2HNuTZxvCvy!zPwi*Hq4g<N~`Tj!jr#megr=iTS>Ou44%;=07J
zYVXuw&8`pAK&g`Rr<wo<SEu~lxT>S3TyO0+mz?Cf_GgaU_l0kbT5H7SsQEg744eLK
z-i<%vCoA8S|9cU*?*37!`H~CQ-2Q(2$?TeCo|A9gSR^iZtyF`1(W(zuUwPbnf1Ra%
zai7V>NYI$}ZW+sr$6debcP8l8X}tNn*x={_6|dFj)}%~UXITZETi6bsTX_CXw6e47
ztH;4rSBqC!DW;m_slLow<#N0#>zdT&=A*B28k5f3yWP|4(&T1w+6Q)b(ZZ0&=WNt>
zJ~_lMd_Q^Zm5$@`>%uyWRf@}Id*@x{`6S|4D))a@{<XQMHhw?*!nLIKQ>@;s_d!PL
z&Y@AZueAOp+}}1)_@vjoxBObKPw#X6&+Kr!PGRlpU%SdKW_>*QTqtf)#2(P#$FZ0f
zqI>=7?54b2kYXJ@iL0(;ubb7WRnH^$9zOI?x@&hf?~C`o?xAs0IVV)+H}GFgZRNF&
zJ>>9d+RuW9i60h%20pp#>-*+#&9=U}`&X>foV#;(`p1X4-)~s+yL`pNpgEJy--wvc
z*4C5uLdEGyz4Y~o#aHe-z3yCE_ki)b!76bjVYQ@HpzKm08pAK2mGUc??ce*dcmAMi
zR?c<y`{Q$_Jia+|eKpUl<MW?5)~`GezPl_zc(Vxmq;Hb*vf^jH(zV{bKl)Fa{-N!2
z!(5wMFYgw=^l7Gl#=l?e>r_mpai5X;uI1dm@6e^@aPG2ltMxVSbbGZWOSSIoewq32
zheqfV@KCqe;*#B~{}!3Im2e)L@@&%LbKT)P*1h}uJ>K1NrADN$@a{+RWNpgcNikeM
zKj+CL&x&O&f*L0p*UxXO<F9mE<IgJV&+5H>&B~&b{a0hFpS|(4{@KYnpQ}o3%cY>}
z{);M^_2WV>PnN%*oVc9#+51gZdrn!+Wc*t*)AG!TCw1!{otn-+`>yL`?a=E%H#HVs
z_iVX#^=Y)af%D3j5?Z_e=ik#<U9u{~cUjM`t#4)rmR~qIeYQ@-mE<bO05hl5#pAPI
zSsh4GubJ#!Xq<jREHdp#fc!qWl$4Me)1&#?S}KjpcTBX?xD&FS^~$Bsf%@{tN<J)$
zlr&rd3U(E%E7ulX_x$lA`(?~-g(62#xX(NO@WK22N)e(9)Bcs_rN2DItP=gn+{erG
z^|2GP^eR7{T0C#j%bgpaZvJ=X<~Q-{zG8ZA>yvIIgkSvfU3{N|tK=3@xw7f83_spI
zooBM*`s{n9(wB>mEYEO?PkOX@-J%aG3wQs>Ive)wegA{WmDZnmuezA3B!6?;^*;CX
zy~~mNe0%Lr{hO?(Iiu=l{`a3=rg8^*ChA`K#;;Iwn^We%Sz~Jn`v=D1zn|}S<=rc`
zIBM0g${MTKwtYui!sV79HhwR#n(dCi)PJth*Zq$=LYlI6e2cY7GrE8NU(+?)Umu>A
z&VJE;h(jdh*~NADd}?pCA6se?8TVc}w%12GWk+k{(<ez?Ocpmwe_WHBrd7(*Q+w;g
z2Lt^reCofIF3ni7Ox=HP%Hz3b)ACZRdc&q2c&L))Z<YW4b4}sjU6<DeER8LG60L5q
zqfNZHUHWCrLF43zb$yLH5BKu$Uo%*-onOB6(r(t(b^*taAFh=Qj&}Ci)B8Pa_RfSL
z2ldPA=j#`@OL1y1Ry4eK>IZ9DY43zX{ett9PQEaiaz*n^kH_=x*{_mPBbDBE&6U;k
zQ#x;PhbwyNCaFh!OFo~?+_)kV)Y*HuY4@}FPH(3EdD=7~pUKhiK+T-(a#<ej>h~42
zg4rIZ3b$Wa6v#YH&iCZ=2M$@LU!5*}Tl&^{X@2EOO?`XM*J-^KPiB@|gs$cf4uAi!
z(`u>F9*Zqixn{=4ZC_eCbgGKA%B}ZLJpE|8#_}#9=a;9h1n{oh=9~TOwTZXw-mYDx
z(uNh6yuNSyFugi(`#*tg@kWA{k691x>Mgln{e*x0r8Rl6Ve@=Sx4u03{>RVH;TcUw
z%Jsi5b-cT4%NxT<bA9Khef#AxZ$&)Uh09OBzI->UqwjC_&##Y~Th`w@v3*tIYa2<|
z;}`#~$kg$F-s!)C_tMJ$vyb1l&s`d=+Wey1wPlHh*F-knemUD&>-YbgbuIsB0>=uK
zc?E?I>Ywih<nLJ@0qW#!o?LOd=Jy)&I)_h^-1+&Ng`?b<b|3m!!zJ0ylBuy!_5R;e
zEEyahn-70H$5gT6pJnPFm8QQjOdsF7sEZesYIt7JeZOn}3>MYXW_x2FCN!m=2;sUE
zaFL<2)9=wG*YI7l-^{u*;nmX`i=rndVs}5Apy)iqx?JzA`jN$^Qv~!KPi(LGHaSB?
zC(*M=XmU|`Ug(MK|M_m)ukyRxw(!lZt*_U`Zm$3TPbY5A3~BSc1(#nQ@z(HqlU}(x
z(f`~YyM9$?xl?x!<Xr0C+BJFJe4WRO-#y(~c9tiKdFdz3EAI?ci_fN%N!F-(JzRf&
z=W<oc#_W|*avJg#<z8+Hia&q(o+-PK)yu*rRebk()gj*#pY~?Yj?GrlxNNxWHRm0Z
z=KPfkf4(l#etPW0MCF;*<$BlG#VQ+Yc%a$t5@f(QH>&y}`y1<$*P3N(#b@MjvFqQx
zRW2~SAbWjb(xk~pmh3KH->J#fskuFCx06rb@6)-`pt0Y$62(S`6We2+oH$tVblttx
zptj-OlP`I>BUW3?{N-^s`rYhlS?<&Ra&mpz|8(D`i}Mwd|NVaXw^`$Tz4`mF{`v%g
z?HQttj*hH1*p3}o5Z(5dJC8q+<z72W=;e=b(fTQ?Cx5;!vWormgq&nYM-j1&a^VV~
zku(7gmdYf%{nI(+&#36Gu;c$f-@IE*@7U`5=3BJl8MfZ`Uh#|xv_3#U_tlFJmpvCu
zUYW`gaAViiN$EEx8#)xJH+~8d7K#z*4}_U!X|eUM`n5Or`QNA~@h-?b^d|7_XP)C*
z)R#|Ku<qQhil0hqPDkHNf6%8abofsD8{4el%_nz?EK#1A_DucB!ho$t23y+?&Yj?}
zT21B0w$%~(3IZJ9reV{Yko?akzE@gwJWPM29Gq^#duFfBk)s_eZLZXHSU)^vX!`J!
zwRe9vyXKC?9WxL7p3rjHtgZd_iWlmQktdZa8&(CpH4?5?5I>k2exz9h+~8qhax~*P
zm^(xGv%1=3NglVJpD8ITFVu;!^V`kuZqo}XX*6JBE$`sgWDPD6y4>od0&_}}a<Yk-
z&Whmm;+gua<@1&J*-a&4zFO=3-|1r$+vrf>zC*IU;Y;@NyPuxrO)T82-Ci%O@2DVf
zM{wQzTb)-+zOFmHoyR%l{5;!7svImwTefU5`S$KEcf<7Q;S2k8KUp<?{P=N8?)5(7
z<^I82-@d-`;?u#}zeT5hc&2=RA904Ob7u=^F89kym5qNE*xsL2dDeOB6^;uFoFBfr
z&lmPA_`r2@2j}ThMye^Cv#*|%cYk~2&Z2w4j<*9VRIE%dEt<1^qw2yH^Ghzyzx_g2
za6<07JBzLwtZ5bxe0}NWq_XO_6ZV*vtkw+n;<pg-n^z_<b4sqnD;2G@|9`h@#|UiQ
z<zcgJ;$d#~g_A|Y)<if?*Nc63Ypeep3&-8x-tMe<DP+2<#^dU?^_}8DXHMkCIL8^c
zvrg77k5YSnz3)z4@}!_CyMxL@1HQy`#`&F@^zg?IiHPlGx@)7h?#jMySMow&ZS;1w
zgtxc0n&jR2VORBuPe;x8TTb*=HszgD?j0`ttHXZIBJ<9QD8cLZHWxg5Fh5|cU?_99
zS;YDmx8D@B?oO+Vi_)$CzjM~wlf06bR{nas{czFT$?E4OOvz(kD?c?WWO>atrAD#I
znzw&nN-}u$K1laSHS_ALrylp)pS!j;y7KSW>obi~y%zTAe=4}};6Z}Y_qU6qx97>+
z`DcIT<n+o7@5?iu{LWvv_lf%FkC*RR)-5>iw#-oR)=W>$36ghibstMjEwH#zeZeng
z-LaX|IPdaKxwf;1eYeg+)?KmN4J%iw9yjzaG5e4adR=?l4UsQao!@#z`QG|GpPhUz
zX3FHIt&htxE}!VW$GcZU^Kj(uW?j}5ia*}Zi#}X^{#St6_Vqb@)0W+hvyL(`HQ2Jy
z?DT=``x5i^Wu!im(hd)8&HpmlO)8~ZX=BVrgPNtizh`p)u77l+>pBMqPr}<-rY)_l
zcdx#_zACi)zOwSmOG~-k&1)K-K6B0b`a{N3xhO|e+^Y1$vyEIg?@d2^@VN2i%_nv@
z^*p!Mot%HQZ-IdRz84ejeqOzyYGLP7%}24~=I7r1>6`reSn#$OOS3ic{>^N>?_PbC
zHp@A&tMqlnlZoyxuC5lBHp^KMx3}v1@x%VMCp$LOy?de~*%P~C_Q9{-RW7cdShwlS
zK0M>Ut{Y$N<k-`z0!>tZ$Q+UHN%*Fozv0c2xf{95gH1Q2T%7ZhZ|l$ZvBobxJo>(~
z?)$kzcemf|>)x-Qa;12_;Ow`tUuu54+f^*HtU6FrW!hW4Z?11{_Mt_!n^&e!dDpF%
zz5TYNMS()}_Po>^8xm`NKAqlbcU(47SVBbPNT00ri|gy<UtiNbH0zf+@4T7)VehN!
zJxi=37;U)%1z-AfmnL0dH0xhzxT?SLSgMhxiXUrn9^2CGXO8dXox*ZSFRC)LSM<V%
zZwm5Rtgj^gtMtU?1om=FU-UU6RI*@Gm8aC{$qUk$_3v?hUX+-f?-q4w=Cf~M;(KIU
zm%6R!yMFt{$8&!ox>kr?pKDXT*KOAwUWW%YTTHu)!Y`C<$;f-q`fC+aM%V=2N!1L$
z!aqFvdyZ>;&L&1KdmX>hz2(hkcBt=N`C`-Q_t}rk>QB8~J3n}*m#A6p9gnlKO#e?U
z3t1J?X<Pm6!sc}UvbVn&i%*HZvi+&N>0HBud&i#@Xo`K4HdE8tHetyxb>CK*^W|Y`
z;%j6%3&Y&zZBE+cI6I83?Mk7i$n1)<N4Z=BEp|SWJ@<B|Uq*(8w0WM2g~gBfOz+iU
zYk$1me*eY2z1Fr>UlKk)J9}kC;N$)Og|-MePGV{KaCOzrl2xx>|2jL>_}SD+H!PBs
zl9%OvJJB#H&eiC3$>%3x$G+Fj&FxD6Q=Qp;Xu{6o+%}bKHKy9v^HtV|zYN;{BzNoM
zN9$JFERve$dwaXOX43525;0$IetxjK$G^HYX~nhP7ccTo7#(pxv*5P&edc4o)h>2k
ze)*(TJTBqvEK{4WSAtvG+m$_A-y4^osm(Q!iriVGdfxWC%ymDv0);)=*RJe;AFWvb
zZ&7x^?3@&?%Yi$JZXWxteaW)CJN|8)?3XDl1r{@Q9-DQM_eag<qo%TPLC06Hty5I}
znfX%q(XaoX61h)Lj;WsCvu)GcS;7-CF2$X1-hQ#~P|`N7H@7FBo8$aVseW1cyE{Vt
zrWYE|9_%{%@=T{ky5q6?*G%?VzSz-QmhmDop4*Jwd)gACKkCAV5BcVFeV=oB|Ej)q
zb``%md4q!I+}Uy7)N1)*zcZTx=iK{c$t!L4<ok&e^=BE*EJ!&yDd*mvnd<W@np$Rr
z%FOvHvF(e>!n0z^Yfk#<rATGBPSplQ=pI?gT>EJ+MD({>oHF0tdbqo-DDQj;=*G%>
zd#nGOG8;6Xn`^zf;$zaf`2FV=_uF~Z)cpDT*=Y$^h|YWd6DKy!IT`SAQOw?dIk$Gs
zyXd>O>&v7~-<f$L_RK3aJ}&n0?46xv_j~Rg2v^VD^I`wdoT&ZtuC5N>{ogE4(&Ity
z*2hNPalZt2-t=B`<G1?e{eP#48csY@yMKRm|JC>B)vhhtai9MLOQ=Tq{gb`n@wHQr
zc8gyIUDJAYcKb84#S`CU@JtR{o%-y|%sv1A{Z@&$I`jUrRianVUVGKrb?d}iQ-9~U
zs9AWPOrPfdX<Fv$HvOfsTRyu#lb_&S*0(yiEcDe&4c?Nhb-{a&`+ajwxBd7_B6Q(|
zRI6(jL{`aePyI7*o3Z|}gBzbL3J-X9WoNT<X~=cQPpsC_e%3bxUnh7Ui=T7(?M;0j
z>n7)|Q(O~Y&nmg5T6ypPp|3CB?0Z%sJ>Sl#$;V<%<{!)dntHM`{GXqjdvj;8I={__
z1GX2<_Qsvp-~Z>3_WC`CdN=>>nDvq4nMz>Vo{G~?ntGpG>rdv^zkkH0<m2JIeJ2mz
zto`x(!}>2TFSqMPZxav@DDdC1w*37)(88H+as5kcqs{f=_tnVnopHxZYUTd|<5TQx
zLNB7XtlFHvw8~_4aN`1o{Yv{LT3<~Q7Tml{;pE*HB4OR1rfLOl%?kY<|I^`c?pEcm
zN8-`~d*#*ld+^q;{e9!u-($v)e~Rg?_4GWPqkrJ_FMi{^I~Hl@=lLG*la1Vz!g;Gc
z!0XAnGjr;GJZukH6ERUx*-gN7u`xeyl+VdizDM;P7*~cwswF4=y;SMW=WO(D*5qBH
zCJ9qAAF6*$ungV3(b95O?%SwGuIw)x%mY?W<Jrh+zM$r>%Bi&1hH1tTTLP3Wf7_w5
zbOKkGi-+F+YV)%JN|zg-UAdE8C)u^-tWd4=Zky15p<jKZ*Qvj4YmN;}6Pp{B7tOgk
z)bZG)jMG~)zOIt6=HLENNY++5ELf@i*|*2fe%e=jyO}=IvRKXP?%u82A{Jl#aMt{O
zz{Vukdo`bZzny>Kaahs&qjtFeo&!mJ59KDFJhbC?e$u|eiO$8pCVl#D=+-B5a{K+d
z>f87JSRZX*WHy-T(<5Qn^zQEN&4rKMyr%2v#>LHx@bKIA<b&DEuL&C~p0wnAIlnb;
z!C$_4*7h=L!pm}A`=;!bTYI<Z`IV|aSC8mCyKG&le~4$z<4&>H7p~+Sk5*>8xY#-Q
z&;LecYsGI8iHq+o%uG19;h0+NuEYJ+?|;jEJE=Y&bjQ`jMXua+zu#`JkBNG+uJyd#
z|39CnXovTy|N8p)!5O(L)meOwrMoz11~O!YPOv?7^ynP7+ZR9cnO<yk`75BiG<5C3
zqO&uV4I3vdDmck@we*3oig44~11IBWnVWv_@wg^9uj0_5l@79rsWzLteqUQIEEhlH
zre){ys^0;&-F5F1&dRaRjjGxy{QAcF+Y;NaOiET;r*ZpDxbJjj#cqGkneJASnNu^8
zZ><;mx2v~&M`gCv?9V^+Zrs1jH{1D3Im6|vmt|&MZr_rfUh(_w_Mcxa`@etrZQsXZ
z(x6o2*vuv<dT(kw+drR#uV#;2xn|b5r*2wMF{$XOM*FibM{7O{O;+=Lw08TwUFo;2
zYJL<11qVx7m+9=S`kHiiSLqauK&L%NVm#BHX2<@h*?&A@a>@CRpPP2oe0}oz-4f%i
z@ju#F*Oxfd_s{u#XPe{~8FTZ~+Z_M@p6RxH*~e2RS3gY3GAMnlbZgr58y^q<j{H_S
z@6NIG_4$vDeWUk9eN&v9HEq}0<>708SEN1rc=xQz^&%tBPma~|!z}(ko1OpT%Vqz&
zoO8IMnvLw=t-8Ik`un?_``dbj{<$rBZh!da{pTDr0~w5_|MfV1G&F@ZJeL`^7GRsk
zOj+l52j3SwF)Lm=;a$P)$X2g^n<}%d7KcTD=ul8Uz5O=F{j}1`4L)W2c+EvGF_g@G
zzWC<KoYPaj>iul#O*IVrwrb&|<I0yM!#tQjl--(Bb@SZ$>N74fTMOU0UHkn(bMfO|
zMz*e7{83xySn-_o`?jp}U)|?F@-DUaXMTB|KGQ7smTf*m+J`H_{u>hyv+-E{NG^+C
z=h7Xu(n;-p<V+onx)}StvzZ^pZkR1J!@mBXZ7xHaLE0G!yB`mj|NZ&w|Le<3=crjf
z7M_}G#4~5_{F%zpJGOuM^N916{f*9thhH&#ICyBSj_wx5iuCibYkx=lyYq|Lx8Upn
z&s+O=UfgN;=5O!Mt5RZnZ1dX_T(_o~GgQsmv#CXuJN@I&-^mqQJk{RsEw*0#drRWo
znOBtRH--PRVrbYFtG(*w(%<XS&(1RKF27gVers#?=FH2>l!}ks`Ff){?d&YUo%!-p
zSBE^i(RID!_K%4Zq+L4~3w*q|-7t54)yZT1?Qh?`yS#n#JWgBQjKWPn!i%SEHEeyl
zTr_RnivPdcD-2hyZ7f@AqB4J@f#$Do%TI7!PE);m@|lG7=d#Ej358P4>B~31PFj{N
zy>-v6Eic60dEPV9J)OnST6>!-|H;4W9ya?rneFYmQoPfCetNp+%O&rltNy;dy*>T=
zx>y$0sn0$vdTz1#akS#(7N32KjUoeTe*FHx^>4;qY5kiAca^@DFi3FNnEAVfQ@G;q
z*Xx}<Jxk*E*G1@veRTRPaU@Z{?nLpVY=iax>g;BlolCrNX8((yCynN<_<z8*GVd+3
z?hKRU9p3z#^wa<4OV2c|V=264r!(_|N~+OY>4<$=rLW8%U#xTfQFBhQYz^P@kF(~z
z+*z1^MtP%#{7TDdqVeyUSF*+Ly(#*)_S#}G-KZn0*YES%nsxQYuF~v@Z7Kh?8DE%Y
zT~R2yZ};#0|9}0LZ|_q{d+BW<Tc&8;#mW9HvLi5f&a{=Q7H?wgbDtIYzkhDC-?H}s
zo8O$2lDd9rSK2GHoB#Hj-JWwMYVtY$nMPk&Sw-hA`xcTO-gfYbaoC~9Q-AM@kL{Rs
ze*WiQkJGOR3ko{EzqhyhX{|}dg@*ge%ACBss%3wxW#Vo5TmxGk2ynbh2DLW796C5f
zQ})EW*xfI#tPI|`yInJQ*@r{i`Wtd@o3ZgoG~Ctuvh9xM@z1AYgk{bN+kgHwL-R#d
zPkPMy3)R2w@Nbbi<let4DB`TNecSqNkrnm#GP3R@JzE=cb(&AatCN3T+Eh-_Ubg7>
zS?dp<Q>JP9Rn`5G%sqWSe&d4GVQU|?ipMRe`T5Dk#U)@}%*8aPzt_VQ?LQn~esg1E
zvtI13f|n1@%gp+&!nfM<L|@E>z>byqI}5mN^Yvdg9Gc>rcIobX*_<xDSv+$;9$%ZY
z>yc&nqm_%I<UgOkwWoal%;uBp{(oI+5~9;22HNUc9LalUgJI$2zp?C(u1xckiM8wM
z64l<2|2{5iOGe<D_v#iFGxk=0zqBdU`_cC^4?Y}Hw0L`{STE(%+({>!szqG`EjB)j
z^;=UndzNXo(9V2$e%mhr8~4taG*0_)#`ye(yt`KOs^3}8@e7=#VqTN|HRWAK|HA(z
z-?zDjg#7cHcl$(g{yzunlc`c3yPju6?{8pz{j`YZ*9-3J|IfXZu@5&<T)47cH~B+&
z^+o&Ns`4V8g0VF>_EZ|5xBIQL)O)&`l2X&DsoLT{P5)gD3E8)A-<r6+vko@17r#9s
zw&>9AL|<QlOUlybmu%%k=f`Z}Y`*QictVA5;dR9+Pw$9`a^{yRoz4r|ad*jiucqeg
znOo*G%692xKYqtO@55=MuU|#3uKb@ld;9z?JE!idU$x@rwCkVpy!8!k{;c@;SDI~=
zh3ucNuUF5x^(k_i<d*+TjEsyqH#fD~)&BbMeWIw$oU78&zjzfdwD{~(Hi|5G|8PsR
z<)_mI=ibiq`uyzd&E4htp{qhPuYF%~F#|N_*l+ty;`B7#$4{rni%sOK6u6~u<nRAI
z6N@cg@A!URcPiJyZLgjL<!t^@cXuU|{OYwAPS{FkJf3gO{)RJf&z~plFRFY}ZqJT=
zx7y$WsMoc>zP`szdA51}zT~YF|9`DH618@kW%08Q?{>eRRP)ilZvH*a%U6S)s!|@U
zxUDgXW95X;Q?IUk!s)-~&-Ui-RmT?UFI80e&G7wlK-|{aH7EV@%HM6xjjo+`b3=oG
zJ{z0p!LzSY;<8qpzT`UpRiMdJ;eVO4^H-geD?0rlTtiWMNoo;4bJga#C%Mmj&sB41
zH|=%)Jpb9Fe>+Rw+?W`<yG*ol#o_(E2NM!rURv5CW!kmGxZ2`Ft@s&pJ+6gnDMb@c
zryShddUwv5gEwn;2<`2O+gr7<^mUlmRIRV`cbKmYJN@Wr_scswjf<b3dwSOVzKFif
zasE53X?3qomae=}|EzcB?{`0okLUmDeA#t;QH;&I<Zto%D<-Z#dOe(Z*0w9Ff1H1&
zHZ{9l;_gFBzS4^6Yv+H}+!T4`(Sd`_>@RO`*N<8|tykJSXn);b>*rT?hUfiy+;9Kv
z_xt_Fi<IZBkes_yvfhD1aB7RiZtFy^pI$ewFZOJi5)_}jY3|xguHAks3x8Shmey=~
z7rU~_O*e|C{%>jZI%VHVS>~+Fjq%%!R1dw%b8lbQRxhJwxnlRcO`8s#`<m&p`qqSP
zJ>rkE{%>9vC6ZgWqwVz5`O~U?$N8W1Pb<o2zL#b_x!&lxhPmOJLiuCYul{-dd&||5
zsm#XJ1z&HlvGGV8*j4)a!{^Vx{qL=ITl{eM`+dommUu4tQ`_^`*lJSIQ+*aEHSwuF
zVt1k=Cg}@*tref-_x{mQ@%WUVpPp7!SN~uB?&9gw-J;rIA3hxBH?I9vqUJY8BW}-e
zFXd0|8_)Etb^5W4L#?!Y@9*UMw&yoZUitma&Z+yqe|q)$;g!0F?qzFcy}9+G=TR$v
zxX2mHQsX{eK`|9y%|AW=6`Gf8PCI{_FD?9cdB=}s_Ki2+)n7MX99i+@S^cKxM~)o1
zusU2n$4ol=+M0!#m#w3N|IWK)*3K_~u1i$=f<=(!hvagh{@gwCsqT6fH%pIwZS~(U
z?b^)vgbmXIOwB%?V3byS@L;}K(3R`AR|ST#KE24c>h#T&^uoHOGBTCrGs}0sxcDgL
zDx0=@(%N~K_}FyUZ#!k1*=2e8S7Ei>^K~~J{C6wxW_}DyJ8u26YVyt4TZ&t@+MTWy
ze|Lv9zj*nws(bI&JYOjl`QDl9(j_$yW{b#;m&>BxPS}%bSAA>tr2`Y~|Igq1yKmK_
zsNWX<Y(+G=!~}M`wXV2P-K!cXuz|6Qql;B2yT|m&*<Rga895cbdm9xNnYcQ#2)>C8
zaS>b-C>V0UZDQ3udy{9gW<)G1YIweR+Uqz!znMSJ&awP_=HBLVxzJauFK?g7Rx6#E
z5E(zWu(tF?%7Tigdy<cOroH0zv)O;)@vB!|%a<=d<;(P6amTJ*kDfktjXP%RxAv9i
zt*y;P>y{k)_w)Ckk9{Zei{6N=QvRDKb3EK;_XXqhb24FTA{uv9eU$kY8Xi7<!UO?O
zpB&U_d~~$i`+(TL`Ty?M8~ff~T)c0GPXE&1Uv8GH$!fWJMO1%T{0!3q-R=M1%{Bf1
z;%kVP^IzM2-(OFcl-uy~|GsBkyIAe5wjW;5ceMMZ#;0$7=|}fHfB)wPo8rEGEiQAF
z-ufS3oIE?cNJ!Psd*2h`ujMg)Z=`o@eSKan)o$~g*`GBYDv9gc{hmHyg2Ror%$~Ua
z6MkLoj=%k2s&@E`ySvR_zI@5T#&&7x>Gt{A5l?5yfSQLPAtJx3tmb^(^88z8@6mq~
zf-ZW@eYO97%CeOU{%K3OI4vrg7hf56{Nl1Jo8R{@lQQ0wz3|}X&MV=ub!AKo&zw5`
z{LyCP_O4~OdRF{=!*Fq7uUYPCO$JqW{y$&swp`b5Z(3Npb>(W#@A_$*AK#4Rows}Q
z&%b`lyO!Omx%^;#__5u8`~yR;eOSMHZlJHP+P2FqTpxI9!%{TgKYdvL^!wr-D?jF~
znJZ34@B2Pmf3uUj?CfW=uB=|$`SZDz&fC?lyFYQ=GQX74y45U7!%|?@(bd@p>f`uY
z`>ZyzzYO^K^M3E`OyeEheAi^RZhW_*H&$FWDm1LUz4A>>K+X5B$G4v^DoW+w6tsSC
zhHS8g*SR_N&FuVYCMHvYq*z_@Hz-=F1)Q4|dg*n4r2OL<f7DJZh8%Bx?QXGimHJNq
zNgqCZc<}rEe*JH+*Q{UfUr}MfD{Yo?YfI*u$j!?l*5{XHF+E>2@2i@YhtK1G^W6^C
zR6Xag=b6{-bXf2@WAD7$Zr&*4+<OPok2l@<>N8{iXMI+Y6yf?;ce?ja5_<pdYtsem
zecxYCliXyv?Vo#{<;330_39TNRBFfVei;1nf6as$z47&D*R_N^a-RO^*W>;jb+$cv
zy8BYkA9XR3Qy1J;q@;Jc|3d92u4CnE|M}gVSUY>_;@pV;d#qp8%jCSwO>UoicXPd&
znVC=A9_F<Nn=@C%?>Q%(cXwCo)TyGHnwmK`H?eYaclUqxULJDMx=L6`$iUeA`F3gZ
z6`xzRkJ)zJI~cNX@wp&%*)IV?J1if&GJe```eVz&T~}(aYAEmgx#re{qZ{q_-sk_j
zW##JsZ<aJZ53iR~dA32r;<<i!!>%nC(>#iM)74deE?CI-M7%)Zlcw}K(LGO#G-ep>
zFzKz^Up4vTo4Re>8WMNo+qbdl&!3}mH05Kx`{&F%LER_U#@#(t{Mbj#*#Aj#$bON}
zyZ;ojz6tXYyWICIImCVPP1`MT`y?&w=ZC(Y_w>`R?|K!lQ};#rcKfl_+g_}iGpF0V
z!t7#X{ogg_k?z{uv20gr{&L>=$vji1OS|HwO2MfMmrO3TojH<oj79uzpiA&0<&KYK
zPO%e@KRY{n<Hn7e8`Znh&(B-Byp^?l^5m7*U#Dl4N@=HGknfz@%Ez3w>i^sCFMao&
zm;X28PuXe4fX#K9b#H>W{(bxIR^w5Ap1rW{_4JJFOaTdjhMhYrJEQ(Td6HuH`;GCs
zn4O1anPwN1l;}ihfA}J_ZtYu+z>;I{U#!>XT4;agcy^6{UFV<I8s}f-s&A;?%l-6m
z`Gr&8`irlw4}Y7Mviz0JUde+Qr#t67>A#%2^;a^-XL)B)g|AoAcJciFIknrg@^4(*
z_O(YYmsh>Nu3s!_bmQ5rshNuB{qjtz{~x%w^IM3*>=UYwCH^n3=5;?a^X$+6KJ_m@
zGhBHUxiRCIO4XjCb%)F&`8w`fao)c9aQ)qoVy6C!-!B{K|MTnTUoiJAPr&P6=C}R6
z?gtGaKe)9uJLk?0L3Q=z{o%_sLMtMcKY5a(qoZR{{Y@t{+WO*!Do(Dgi!QHy^wssj
zZo`$U`a10<@S9Km-1&F;_p2cno(h|Kt#+<jHtDi?>VwAXD-)Nv?cK!XwbI$IZ<4ep
zYs^WlEqhl)h%|OCH!n6!kooniisk(s&*gq|z5djFyPdy3F)z=rrp9KgRqxZ_)t0KO
zLPJ8k4=k$knWnVi!yTJVv%IZiS4M19+U@L|d2x~J$uke1`C65}a!E>3nm2ErYxcV9
z*Totiwf^CGch^%{S-<;EjPI!_#tT*jJ}f=2bILU#D@*Iyvu8(+9^II9l*_GO?r)%d
z{L=K;H8DGl*!g4*T)Q^Sw%X=o<s_fr_TClKFX=YL&-(s7Z2J2r#XrwU+gXO6{CB!Y
z`Q+^#rRy7_je<{B7w5ibTKDE>=PT1Jfo;|1%a$z@;tE(DrrRfD>9js>uSHdcrN6xT
z<r{gTSyD??<|w&|_MPNUx%aN$Z)Hf4Wr>wJpU%-6M_N8?Fvz;TE;i}&ipb4s)!*Me
z-F)6I`NoFC7q4E4ofQk@y}AAW#H!4mtSMnLk6yf{Gi(3MetG+4`S<r7IeOHv{@<Ve
zI0muK*jRu6{s|KV=FFY@@zbYASE96M@qb<#^0LeIk;5mQ1lHPLuL6uU<$Hs;@{T`y
zQ1AZis$l(k_t`-vSDI}~w~9CWh-}@+I)(M;5mw)a-#+r1<lUKZcXxU4)~wK*JN_?r
zYUO%!cegk@JNveLeg*Zva%bo6xp+ZO`(cg3tn<%bU0xg3H8YJXs7NH^$@9%G6XGV$
zUcb3#qSaxkY=?c5x<dcGuv(=#`)9NEiSw%0@_A0C*Jj?{rhD+<K?y0TX+L6oe0(}O
zIu>|O*E`>zef#H|&F4ev4}PAK!X)x0`PLrZnydTRcFx>+<;@4y-rg;#?cs($GpFCI
zHeoa|-C26%^UuYSB2%*yXU?2?<Hn7GXJ;gr`OZFeWo0n;@_F+ui=X*S-Tz|2y1uPj
zO)V`gU0Qw=zCYy0^J-?G*Y-VL`rDmi9sjP%O0f<6q$2+1>y-<a!wNoE>3zJU9{lI;
zJm>AtW?i`wzIe;BCU^fSOVUL*{P*yicr*V~@n(zdl^Q40YhPYlEB)}{!=k1y<*uE}
z)XK`rmMmFPU|`cQWrsnJkM6|_*CrR(xjc;Z>-CcQ@SszOH`nykw7>;3Ll69uW)<0b
z?%GqW%!Ir}ee0%4b;+pmU!J-|<=F<!g)i262Tq#)x5Zfc?aq3`mV0}vg=J)XZf+`i
z_^jN`&8_C+QSqExTRPW9Z@;uPJA75>>SedM*Pc<14xayPu2o`?+4Lz#Qr_7!O@8LP
za)mOp(aKqKrc{)@?|tfdO_)pT%JSgf(@NgjcjWQ*1bkgPb?)4=2O60z3mz~CtNA>b
z9$%-qXR~kUKJ^cwVQV4|?kIeGK_}Pa^TK^o)?Rel$Lq9W!ksfAi)`6Gy?Exy#w*p*
z-p;<%%TxTg`Nz-a?LW`||A+nUrmGo-N)1L+r93p8R$St|^EhSE-Q3DaBB#USYeUy8
zZ}_pDPtK-e=T6J0sHm2<wz><R>svLrK7IOBP*9+tqQW9BE`DuYZ1jrL!B4sz8gg%M
zGt9W4@M+2`lhO}IHq`Oo+}skt7@4`hu&SANre^pIqg1ZX&(BXk=y&hZv0mxI!a~P|
zk!@*yvo+&)FK$t4;quro?*D}?dY;{$qy6^(PHf4%tTN>~2S5Mv@bz(^vE2_J3he%V
z3FdU2;=#(wYGG~t`1R}43mltoY{?YP%4)rx7pc<i&~Uh&|MKecd^y`HALiLhL-I`{
zz8NTTU3`3`c+STc{(iq|@7%fb<Js(dl_}*SVq$H*YjyoM7iZcO=uVaTJ~^O8!-YRj
z=INT}3S67yH<!JQYU7vJTexT8+uPeO@2{`d3SH&m;=-~u>*}Ncmrl@pz@58yUtV6$
z&&bG_ks<PLgO2Mq4wfAyFNM;cJ-f53a`LXb`<LgHhu=KiqQSCq`uC!|$--HuWXy7J
z9cf@>R+&<-X=OEQo|xF3{q^&!_C60>eOQCjwZo}Ezf9dQdd@|yrG5?j|GUpVvD4z?
zw{K}DCMec?I;sBV=H_;-&{ZB9i;fhOmZsj^lv?us-rNTd61HsF5|XsDt3;4v!jvg4
z=jYjWc6Kg|nw>b~TXf8lXF?*2R(SW^J9M;R<BO@9XV1x6zqqq=bIYfBG5hOejf;!d
z#q2wyK6n4IBT3sz){8z;-c?zmJH5>3=!FXtF2wH6xVma;%cs6;Ya$Pyn5eumeEmEg
zF*UEA`E|cKcNRatw9uLT+nbxs>(;&UDGq<q?NDKMy>0DUT|q&?kemcZVfOMXrll$8
zg&#5R7TY>QJ8|=hsI^iL6BA8xZUi`0o(~ER-nePgqaPn1hvZo`xG{&Fh-q@FVVPcj
z{>d*p^H<l_dPhV^l)k<;HQ?V;*KRSx)Kem}%yPXlGFJ4g((2l(>OJkl{QrO2pPij8
zEGoLR=y&x-4JO8-;$lg=8jE}P?oC;Hv+J4j_MlZ;Vp4@3IR}V|6l5OXe#dItG`C)<
zjn&`tuF5m*;W&Nj)S-L#=E+)@1uV=HpT&E#rES|j7Ud)C0Tbq*Zfz)3am`;?`ubW+
zOABbig!7p#XdY#A+Sx_9x3|5yvvcyLrQY5ei*7`2&y&4h^O<+qvSl(B1q{s0%u}O}
zDH-}dkY0P@^G=q{n>Kaad3t<rx1z)~WB1_gbKZ11?Q#FP)8F#BanBOt=ogoka#w$Q
zGf{X6|C{eSS>E5<dwQWQ<B}Mz<0r0g@vb!z>JxY$7*}SMZp&Wy-pL>=EDYqJb#vyp
z@9~e_U8ZVlD{F6W?>EoJ^Z&oUp-B^X6z@$xJKNki?To~$Yim#MtNm?YWE6Dv*2*0s
zjSO6^O>(wX2?+@Y&N;}xo5#JtY3{j<m*LZFgF3z|m>c(Ilt*9Eb2lt_;K0Ph^hE2O
z`=Vvb+}6kKy>Q_|!uNA)@7T(>99$mn(a)}UZ}J=U^3J&H)7$sU9zS}NH9tRps^o!%
zI}DADl`So2HZ(9KB_(axxKZ=V!%wXh@=Z-m5{5}ES6785|Nr+lWPMz0>FZ^WF3Em5
z&&0%3P*S22ztU0I-Jj=m>2%xRj(SDyw`v#PnWfy>f96Q1aO9SZi9A6;9(V5E{rTy%
ze&FRx%V$pJe`=qpzmVgj{*wFN4|;a%M)KzwB_HGIy7;wlYG^<}!{o`r_wL>M^8UVj
zd3m`ei|a%&Ik~>Mb7h~Np8ov7!DcOc{j`m`OpJVTHXiToYAgPz$Og|&4av6j_!gje
z@8DzW=a!H2yuVG}trNAS!*jBlr$%ee_jh-1+_*Jq?qq-A-(Mw6U+FI7_{b;suWZg{
z>9;4Ve@(c$I{fjYM@kE8R4OYgZ`{1OG55Av8=vf{Pft%jeY8kp5}&M<%l&<|7cX8+
z{QvK-N#31^r_*;vh&M8b>&G2Ab7#(x+uIGzX1mQjzx>2Jts~JMdxehf^mPt+Vt@4X
z>F!2mb}tRBM^{z`M?^$Tnl#DjuifXBH~(pKx^`?Yc<;pVe0z^Wuyf=7`SM|5VM3?e
zr%j&hJm0RCgPS|~%nU=Fhz$--g081tUS7U&%N7;|Bcn;z)<%nWKGpl?s^E~6q$DOL
zHpjkx-mBcXnHS0xxn3429LZLl`}OIb_Zp|BTuXi4CtLFNR;W{;;q`T~htJ%ZvpRfT
z!0yv5J=N!?XuovY(-?i+ydd*;ik3u3LbwM<?yW5o1GqfH!o<?j(qybkI`a4bohE6V
z=Ap63qocQXY5MtjDXFP~Vq#%So_XmBP7T<ab=9Keg+T1?vZvoZew}mIWX7L6IyZ}3
zf3RQwuKV!MG=0srtDWcG+M3#@BO1EgPqt6a(&_cJwLACKb8g(Hta;-6)P$BF?9&g|
zspQ>uv0M3<^LU@!RHh9RUcPu?Q1&LGr?+>eeZ5_`nC_&2mRIubdhyH3-``uZeEH_w
z+uM{v8K%up5^#u*pTA*)!JD_>s{QioUlV4}RIA<~Q2*(-OvCi@^9Qb6nR2kN4>UHk
z$d&uhp+hPPq9GfTT)(}&-9B}ysAu{%p|^God)}xE^$64lO_+bYa{tAc{wSBz({wG%
zzGxgfc1*}eF?w5$WM5xjN_u*7R@SL6FE2;#t=h_>9CGdKZ1atqHYus9w(3T2yRbR^
zywY|))m8o}D}$DT#*`<k`#*b_uWqsNQqP(#vp1=8fBelE6leC_jQ{boiC1gB-AtFU
ztC<nHI!sVP!XqzFZ`!nJXJ!~SH#ISFdLMo9>Xnd)$dT#sb)L7TYWS_)r11M>ET>z?
z^G};DTW-GCx8(BdjT<&3{Qvh?%ep??b76?qOrun;`1-%5$BrFyadq8T{yq*Af+m?l
zv(0iRO=vmg>F0Os)Tyo`M_g=dY(Tku((6E_cbqj-RlTPf6h3m<@<r=n%?$frS7)7w
z)AibNXAS$DuO$x-epTj?FksNvUqAiXiCtA+wf_G8{`}3&&0k(zY(6{NJUK(7_{z#f
zk(<*Di=X+FoVvc`=Ld#U617?jJA`!A|LR?oX8Z8toMjuo{JB}C*(pt*oTkm4dv=1N
z^M}u$7dyA}y}7ZGnVFgSL8P|#a=*DAK0Ze-Ulz{J&gS%4(Q@TV$eFWeJG;A+k9LXn
z$l2~XvCg|wmx-~awpP+8g@d&#$VB!{Yval1o65U|UcPL;?Xvs!ggX6sA0Hoo{OsAc
zQ$J_Tv#UL|qwsOW{yN*MtHY09zdrrj+tfa*&@FX;tK9lzI(LOd&f>l6v@<GKRU}X;
z_0E2$ExUCiGtFn1<x1KA`?0ts=)|Q<mtNf7uHVilyQu#EzlzGrg=^RTy;Qs|W~UR#
z!M?t!udl6j-L!kIMx2~|UC+{`s_EzEB>wyJ(`T;LR*&`mopMT-&z<Z0@S&h&>($P0
z5(nlQN2l<MtDc*@B*Jga8^2$BE1sR7f4=_z@B91bZU~Q#o?ZI-+QWD6{IasNVt17!
zo}8pAw1}&ytSs$#pX`dyQ!c;$Rq)=aqeaPUPP@5(QPEV%oBa8ks=j7vGK3!3nteUv
z{yy7wKG~#8OFV!4`t@%q1B-!~nVOOkQ*CXnjEsy$!GnetbuQnuG&M6bvpJT<ZJC#s
ziT-2~@T%~B@IEOy+3@|%$=02nGiID$QntNw)Anv5#ka5O&Q<cZ{n0VYyK`cJWAnWq
zTMZ2jzr4Rc|MYZyP=o8*`uP5R`}Vc0O7fd$)7iRIHFIU=mG`?RNWGl{3fDXRw|&n%
zOU%e>cz<o}mv?ugoitO_{pKtH<>n<zHl&;s+V}6*>+AcTU0E4i^Y7<#5fKpqA)%m@
zX9>bb-OI|%{{H^1Ze%p6Q&@dd#m7fU8{>jl?8<+C%dPtM=430k_==1FykC8vr(ew3
zm8AP~j{U}ci!WTgs+RG)%<li6x4WI~>;6cjr>Fn<``6IW@ZkOX^VNKx{EG@LetJsu
z%$c)0>)uLj-R*a1$?`J?RlBxFtK00@lB@Jody2Y>#f-AIw?K8s@jh7|DHD#RK_{P`
zo&ECq`uo!Rzg<}9JY(j}g-e&ZW@N1RbFQObO<$j%ot@pu$?3<Z)B2H{(|k8Zny)OH
zw>oaGRXe}@vf}6G?pW=bR(#Lz$#c7@2`(E;9v=uws+nVGWc1|1!RGfN42L$a4FB`z
zPeF0<<72(jTe7eB^-7z29nd;<<Hn3+_x?+(Lbcbuk3Lg*KTzb!b5Das5xa!!+IJkt
zUlF-l&Ane{B2xgXPV_dO?Cfl>>3Y5)Ay=YB8yMzVl_vfE_BJIg?b338c_ASom5`4I
zMMXp&JbL8h;laVj$G0YG>#B&g^S7}4^DHYf^Yioj@#|O6yUArH^1E1l=YA6E(GZ*B
zrTXz;hQo}lu(bgHkDooPy)u5kU1DP5l{Jx{_wM<Aak2Zx&6^*8dwbi$+In-&%}wF=
zo-1<syScHIm6g4@wUxWs-+uYZ#o|&?sVWkIHhwogecrL-t@OJ&w_H9yJIl$-yEJI2
zmynO*|7SjPt){wmizOx{Ib~#Ml)k=J`~3UueYMtqe}7*dwA2eUiMcvVmou`);PJ8E
z!*}n_-B<hj(fjxDZ`?$tDSgQ2i<+$Fn{;xL>XdaqG<Drs7RBysV^Qw<1}=NI_dR*{
zGZ!>*G}pR3Xy2dTdk+-;|5qy`D|>TuI)AUUdD`o1YfUmPI2;OBtgNi;k+Gb_D{ZzQ
zeygUg+mFCiJ##d=7wOuoHy7V?_UydhymP1JzkmOxXex_aa0REOsX4dv9ld}5zkaa8
zAL)Dd?(s+(wM1>r5)>44oT?S7lJoK2&Rx4U<=ix?`tsu7xw+Qyvy4{n2mp7iZ(gZi
z)g#JP_3cgPym@kpw`)D@<U1M*!?%2S;^HadyEBsC@^PZOSB-_4nHdW!>%!&Bzu&W8
zUh?vig_YH!g$o<^@3)sW%UKY#)N4z|MJKbSu7Kd+#rgO5iHL}7P(Lz#_w(|Ei1qAC
zy}~CHxJ;Zfxp4lAuYW5F{u+Q>eEc}OtgI~OGJ(6hO1X`Vjg^#@B@Gf7Zg0=8&u4Bo
z&AimIDs(kyIv^rKV%xTDFJ8U!dhuxQOxtR+ZgKr($;bOzT3SSIKknSZvhUb5-ROd%
zBBzz>d@moIZD8<DYhj1d)0c-`&h5DU=}Df<<72(Qk83L2Ti(ph|LDbw6KkWlOIQ>r
zWM5k|@uG9+nOUaU61G()QdUnh%}TVNW}RA5y(r_P$IeqTdCgLm-35&@r=Oc+DdeSS
zUG+ud+O=zUc9-`{8mAf5{wfjSI$HgI&e3jh<H}Ddke-J|>gj1yKOEiv?Afzp{Puqw
zy2W%my1KNkJ=YCl+1I#Yg~qRMuif3;#CX})Eb-dRDAc7PG*2b|K~Hu4JiA?o_f&o^
zC@uZFV*dPT)7my~Hom<*f4SdWs~<mpTv+BSebYcI<@vd}H+W?{^5je;pZK{3b91_i
zaQ5fTeb&=1yL0ABo%nroE-rTW)@VI*`SRrT`+jkimX@03-SPPU@2~xf=b%L)GmX<P
zE%TLbYHrqy-L)l$<>R?)>tav0a0(lknm#?y$ozik?@c0&M-Tk`{2Wv;FR?B+c==O#
z>+U+`g&b3-mAzGXzDFkTS=xhl@6J6q*vy?9IZZeE*c8oR4GoQib8{?xW*R+hj*%35
z|Ln)d#}QGH9~Is*e*LYx%<c9=p{{_;@A|F6n=3VDWwHMF@ng@|jLDDwYH4U3xO#PJ
zE4O&iVmIEVrlyR0dw#nAe4`n>Y(vIHrCnujy%G`<Hts3+uBn)9p8xE${{AI-cX!Fy
zRBUj$lda|YAbLmXYca^cVBqG%8n3?2(k|xYnmR>B*kjFmkCH9k8*6Wy?J9pC_u_QP
zyE~SkmRre7p=ZyYRa8|arKG4xd%MQ$tC2J=F3yYD?>G1S@}!(^%`HkHm)=gEcO-YV
z!_7^<T3FaGFLY+-Tq<B&_vgn^n=H_pmD}5L<!eK(U0m$GIqRyHj*iZUj~^X_gM-a_
zDv})?9eLzzW~>TbUG(C@LLuW<YeX9JUmk2`2dz|eZs!YJS!gfCAJaUmEv8gx)#L5X
zr#t!o&GVY7z1iAiO4`|3E6;37O-V7xyJKNvW3wV=r%_y7oL1n1uJm{J_I58_s(SI_
z#gJT6iQw1&7cE<-c#vaiVd?ppb?@f7pN-neaeMFVUteE)YqajPT^G01f4*JJo(e&A
zb@gj&B7-$_3%!;KsjI7-<=$#pyH;0DPEP*mz4E{>t*xz~4&ABg`sWuqw;LE5Y8t-Q
zyFX!j$iA9NB_*X7_xIbEpU;2(<yYO4G}|c&E(@#8*dKg9!d`!V`TV+58yCBWu3~6O
zU);WH_wL7U-}<__u|0eC?9cuGf8{GHjS6>atq55u^!xk!<9F`Nv0iRgc<u7MKXZ6E
z-9-L<T&pmjoAL3^UFCAVv(28~xVSioL2T2C6&km;<(^*ZJ$;6Ky`8jqp3ataGq_rr
z`1$$QtX<2=!IALh#zqh6_KLZnZ2jPY!%U;pj*bqEKvo40?+4b}8X5_QS~&mwdcA&%
z;VS=b!N!c~tJMy69J(q0xya(Zb=Ph~#x42x=gqY)U-gIgEa<|dd-vvDUhaQ+Uv0Hn
z{=GBm^J|o(gIS+EefsdxqeI7fr7y0()+2WKuSTfbu6JFBCeELHa_8B@9y>+czQ4Q6
z$;0zUA^1**`SzlxUdQ_7`}_L%?Ck6!A|qEGDSG<k$%W<q@`nx`0+ln|;(AkhRpVY>
zSsASV?e>Wi9#Px%Di<f8wrPAHWveX`5fQOq_3G2>7U<sT<y_JPEsSfP2!C7N-|<bt
z<J8i-UtVADm$fbv=~V314q0(vL*n5RXU-hCe_x)TpTDQK*U;2-YH!quD_5?77Uox0
zR)WS%^~==RzMj!os$I<4)n5}df1=YDwrsf=>k|(SHh=v3b*d!;_vi0w`u&BEj~QlP
z)5*HN?(Dt2)h};sT>NUzl~t>gv$Ib(GP84Va449YPhS_i`&65-zGeMCo4r+EmDJVS
zFI>1FR2{!@3(G!sP!%L?#?#uS=i6Qpyf81F)3rwAX8${Zy$7eMm+Ko<eaV=n8y%+d
zzWK)1Z1KZwypR9={oNyDDP(SLZh0|SaCO+)q&GJ<X1oYl@Y5jv)U^!Fg&qA$#$T;2
zO6Q#L+1s{ywRZ9Ib5jFbxdWr3q?p-wKz-D_ySpAff4+Q^t=~MG%0tiR*PmM%ynKdf
zwpj2opNYI04e8AbzP`BF+|Dn*srb3ynX_lTbHsvJ_NfaB3P#+%F=6J<DQ;z}SKhf2
z<|HT|^6Rd}BJ&%cz62^2KRfdfG|71Bb@B6avTXcvJ|Q6@y;7!0udl7Wq_sk+si|qh
z_U-E19eUN8ZC2bk5aT2$e`WFQbc@G^4R?A(rs>6kx&kUwoJ7RM-RIlY?kIZ7wb-rq
z(e?OxS7+y~?7d#z-iy7b>m50EY*WTXCAVHFRq0JFyDLAd-Q8WTuBpjsZ*QNIljBt)
zXJl}u`qJL&az0rbkE2~|M}G7?ICd~qsOv|9T#WshlcBaJef^IeKD_vZcmqS|>M&I$
zr6$k_+>RX<QCqW)g1SM;s~h&!{uYvy?5vZ_Jy&(B`1g|KElPGXru?v-6PMho^xg2^
zzki^Nci}?7t*zO|d!@}Iw&h4_hp!9y$s{oEZjpeXVB^M(hLhF(kDWg4ZI;G4ZOW9E
zrAt*iJ3A{XDyIBk5?G@8Vejv^+wU8go2%Q|&5M#f{vmVy{H4FUD%QW7cmLeawegGA
zF7-@saM<|fRKbrAi83-WJ1Ree2Hn?0Zf@I@divBc{gjU%KhBsryZH0=`zwAQEx+>i
zW4%yU#d;zC%3{M0*<ZeX<+!(Vvfo@QPmR`$hlg5s?AjHyF2?fDpFcnTd_LcO$Lh|m
z(r&-GR)xR6<(lQ)(NO=d8sO!{Wn2ACMNhA9s&=^7>J1`|@_tcKQhTevKRYo|IV3B|
zIeGWh6`4~eB^=qm|3pEG=tt}Qi<6Faz2V~p4JsZji<P&jXaH5l+j19|y}bphjJM@R
z-{j<qEGjYr4T#uQPg}KiZbrslmeyIjSd{;5U0waWxb(cts_QEpD<UL#rA#ISaC!Rq
z@hQ9a9XWisnT=O!L-KLHp5ET>H@{+b7Pb1#wVF6<)~RW_(Hk~wn9v*JWK;Ep!`s{Y
z+&tUUO|0Bc{$2R5c-PUsflE|tL;3r-HEY-IlomemlK0vZ+gg=_9p%eHv^;NoW&UU#
zV`ygf?8nCBqq`Vd(u<4O*w{AJ{4AO@Y0`@;D~0#}`P6;pZ^#L)@O5Vznb}`lT+CkX
zS5<VZSm3ASOrf5N?~kJ$?ZlbX3rf$+%reP5^fCGP5(XWY?Rj?}Z9Z>z`0!z7Zf@>A
zS?jPR+zt%M$;sc|-R<7D&#tep@6G-F@n$8Q6K2lr?Cs@EKR++^*O!+;4F5W}u+%xQ
zva<Hb+s}J?divt<lT*Z$nJ#EWhc*f3Z<l}iT<EoV+mp@vw&dQP_TWJR4==BG2%Djy
zA&-p31n=p3iD_wV%a^N9)e4<7cd@|JXU`tJdDAm}x_D~#7Es~fHFcfF!H(^d=FfIL
z^X%cBe(tBA&)cWp-j?f{)HVB&mXVQAb#?W-`}_M>uhzEz|EGAOOyRHhbL?to<=x#C
z7!cr4P+$NWfa;C&Nli~r&d&A@2@w%i_e=Ts=%~=C`5xX6+Ihl#ds9zOGc11>Gw;sc
z6CXwIR!*u3Y!b}ZdH*tb(cRyZYpzYVtNkS)DjNET%b`KdcUHl#FPUyV5{?xW7QVC1
zQqRw`-PA0i<m>BeV{7~K-R}1lCEBxuwa=XJ*{gT3!+igQ2{TWv+`4zCym(i0GxNfQ
z3xgQM7FB+JmYA02R#s-#*w}bxw)y%9(FO+H=xr(2*Tss6h+J47Z+~vCwYu~^i+R>h
zpYD~&{c~D>{}iSV`5xX6+#fxCdbC4OIUr+NrP=fZ*_}6I|7?D9{G?0%hKHV?Soi&U
zr43#=v+GXv)5nhwuLxXR@#zU?@N&P@b8{?Rla}myl=Jh`)0)3uuV+j%Tk<miR01n5
z?5LkIv#L3xT-qpjol)tlkh-5w#U)+VPOz!_Gb8bE8)yL2$JdvWhi6IL-m0Tja}OUr
z{PON@^IyMywY0a3|Ng4$$;HJbVOL}E@83TQ3yTL&pN78COY;znV0n9UGkdnQ^rm^w
zF5JlDojNPRTjSBxOYe``&u0GcP`B>SNB3pEv(*;<>Ce8tE-^p<{K?7cpsB}u_wHR;
z8y()H>2iBpZt{f%j!!hNX1;pSnYdR@sH>v%;>3zRW4XU4J@=oVr0OkUU$>_v$U`q~
zPe*Sr@5hfHL1V6ohuOTQ>%}Tvv(?bxh>ni#>FMd{=y)*M-)^GpqK;s#sXfx>eEj_U
zzrMb1pFe+ojLJXuAgOv6+o~@Q9yBsbNKJV@=X^n(^`0On!TOI6js^OyGgAM4Qr0Zz
zh63vY?JFyT+Yh(%8>gNU@tb4eczKyGcmOZBW6j#Npmmc~OSf8VE>!4TzDHH$qqrt#
zwdK}hLaE=Jx8>c{VogxIwX0M+_tut&udc2JjVrH>-ac&xlfb;Q^{-yO^eiee3JVKU
zQB^(Z7kFgnu3e8FK6LDtv%Rq?wOgcNy`pYo>w{yx(lchyo;+c~0rmO4hmU+xf61oI
zutR+J<Qcl>GNuNpAGjaiwy}<R;>3wO(q=u;+w&4rQ;+T{eZ3-P=O&h5553r3Ei-po
zZqK{7B=7tCmACJPItenU%=`8*>4C+UQ&ZAFqrC6#L^^4*R904ky3VTJ(*jlosn*oi
zdVkp#vAaw+zV@r=*H>4Sb#!{VL{l49FM4rYzW&Md__~Flu3LNi;?=9Q7kUV;m=zQp
z{POB*@#B56i;FHZnn-PMdM2dvFi$S_@KV|PNf{X)FV@9w$-3(0RO#F=XPb0qM`6yb
zEu7BI&S7h#TGy^!%M$DXD&1~wRyQ;>49PZ@xm?$sqIXc}UDOk&buH(YMSXqM^7fwZ
z;Wl2?DNY67-^IRp^Twj!0fVen$$|6d`&WggUMVs+H&^qW<uX~#mxGsgY0S=|O`Am&
zyT$d>Zf(ijl5w$V#R?6{gjc+t<rkci5)vFzQdDYxe^dQANB-iwzbC7IMKozPPVrKm
z?wnqrnW0ts`pUx}8<T?=_AT9%div4Rr>*nn%X=-I^zZNQ$4{Odd3^lSTrHl^RUr=_
z9Bc;71FOs{c)n)3^3R=-LOz~0`$R<qw$~bkA4<QxB2c+c-dc?{LD912hk<U?mIntK
znL!!z&F$^}TnC=3`OP`8CUSE}XXn9Nw`P6$Qo^~kBVuzJ@8xB_hp$|j^5H{)Cd2)v
zEi7W2{`{#q)+>E{Rp{yqtM7cc8PoJDRAq+HA_e>D-2ak2tu{}+wl-Q=NJuF*@ZZuU
zOH}5}oy*C~yEN_Wtcr??3u~jzIhP4MdG^ex=10Mvy?ZsIH;bNa&s!Rj>CCxo$)9~s
z?NpDYvwYWYyRg(-ys4>4BYN8!mSB!MckUFFmbwN9Pd3fI_Ta?}jfHy-U0E3%xuZZa
zc2|j_fq_77ZZ4;5Yh`uy=d0oIf!p(9`DCqBwm0oj{W{Y){oE7@uKroGr1n;QbqWri
zeAjwg(f2=PJU;stgh*NjYQ6ikX?{W4V}*C|JfQLA2@?cfHhq$N@$%)v7cV@Xo|?+Z
z#g+8;*H@dWFCC!9s_!HD-|TL&vf0_$M#jdSJYS#RJ1W5QdgkJl+P^C|?q`@V({0Oc
z-OTVspmBl5#zxhw3xc5^9~@+U^Y(4zt`g1C*Vj~SZ05+>raZZ~KRGS!*xlXboE#hv
z{`~wL5gmQ{ijP*-?{9A{tgVCBMw#--SU7CWx~ei|yNYh3R?6dJy%yHii^IJ7)BNnF
ztiS8%q`7R<{5wz2utm$M&pm$hXlv-|u#@~}*$bYY5^dv=Jhbt7>;<#jTP~Z^&Z^9C
z+EDT`NX=)4!^%sdnRaLR|GwHJ+qHVlmf4_j=feL+pkb$nhgyw(L@HWTebM;#?c1F_
zmBQ@o>|yKUd}kV^s?4|^pf#08)=ETQUq2@&=fj5&6Yi&r3X6%g&7CXD#>OUNQz0NM
zEX?^-_JnPtR?3SD3t8CN7q6Ul`sLTZJEV6zJ88CEdaqiL<XSGLJ~#c@nVD;1cZby_
z*a;*jC-cc#9eHzev!{p00^iwYfBt+v|K#~|ZwvXGKR!PG`1y159rrm=$(~+^;y9N|
zRLzk%+&T5`$sP4Gk9Lc1&bX+g$-up^19VyKT<h{@|Nj176S<i!KR;iS;r;}tg$M55
zm6ev3W@2UrZDG0;x5x^#;-I;ixvHwl#l^*g@kjo*(oD_}+vWcAj~%)+>G${d!CRLq
z->SNFeCLi6=W9>Ry*&HG`MNW&uC6XDESwl%H|fQT3{V3;CPqfqswCm@v0e#jX>Wtp
z56t&=`^(GOdoIs9t(Y;@Y(hX+=bkw{@8*1R@}1ede7QO^JKvN5yG>K3h_vxayLotU
ze0_Z#G^&5BSK8a)y@I-W`@Y)Wo2tL(?J9rocY2zx>J+DG^X8pv<CP8w4+o7QUt1gP
zt?|DtXvLD%;p??}eSM}h{LxPPUs75+_29vSn<k4lty!b<w1rc63RkPs%GIk^$LuOu
zxoN=yho+w=^kQ}_SQEFnO@_bgq5a=YsZ#?313^0WR)5!89kzBA%M|A}p1@_xmwRjc
z4@yg0w#j?C-lw}^8;e)`{r&y)xu0?#-V3_oTX$7`T~*?&k+v#qtrqhEkxjL~zZoVS
z;aC^H-|zam*qa*?nYU(NpO#>^Xu1FVqQAesCe_z~#=7+M^n7NU@m5#=PTC*4I()sJ
z^OpJ)P>Da=toP^V=ZjabKK<$G>B6F-NeTDjlQ}s#Ig6wl=QxX62eEJ|dM)|#rDU1U
z%uh)hT3ETo46?3haJuaY7Zel(P2-4ahZ$6T$>4O^6MlVNET{sXV^P=?wl?bF<HyQ(
z@3ac!yRfwhI!*CNJvAlq>Z;I|_V(a?HI;7ta=kNWN^Z})JE@>WK<oVN+q0jYon8Ft
ziRUuExu+&7yDwo7aq<w<Xc5R?<lZlL<k&Hzq$3>P-rRIv?A9Bxziuy!a!Y{c%a<=L
zEG-QyJ|vWsmU41)C;$8NGb1a@%fg96lc`nUk?+gbuQ%8It#Wd5(g<G0qo=1A5gA$d
z{G2SOJI6vJzqwYfpP!xm`03N3OP40im?1IqPb`O`>jD9%Kho2tO^ex4(5UF#wjukv
zo|IWm$MosbS(IBeLcEqv`uX`eXbdUu-k#3cv!xFnJQ%VjBJhwP#J&H+b{0MT@a0R&
z?{9B2GBY1Oc><aOUK$#}5vVa~@?=AE^W#^qPTf`dy6Df3LN%Wm4NeOu6toC{Li)(>
zE8Fwq^<s83?A>c?UG`>yV>8>Dn4Lk(as-`fOzz#gXOeScf^GG;2XEe-d3kwx=Cw65
zS(IBQC<SmR+KDhTGuPDA+}N1R{_EGT9!cZ2^LD>`cI>d=bm#bZP(@w+^7?ptHh#ID
zUAwIQ{rk5jdOKfiY%Hfc#KSc_pib1Iquo8y=6#FZ`!B7DG)_A=$J5WRuSL0~Vtrly
zUv4h0L+8%*b&KmKWoK(2<WO9~A>!20@9Xd1A7B4>>fv_&&o7tHH_E=IGs`sl*h1%a
z6&b}N`@ij9TQ4Og_2c*N&84ryczAhLWfWUP8l5--XWZJF?Osx1vN`?yvs+uUOWxg?
z8MxT(<BN;RAQ$`X*|W#M)KpbNqoa|Tz3AZ~*37@ckN{rg5*$2PzV1ikj2RME<?lcx
zgQ2OZs)0ekAwj1-8<#9m`Ss;x^W@3Gvu4c#*_(P=459bcWq*5BEv>Gt+1Dq{oY}c{
zt?spJ*No=22t3-lXV0FD8ygg@%ikTjcQ5W8%sSUrr-cWO9&O#R!veIs;oIBWsaBkd
zNACZ+e{FxzQZGRXiH`n$el|8Xp*{hpE(gUH4KJ<mbv^6XulG?`Z21u_ZJt+i@t1kV
z1qD#VZr!>S7G+(V!hO{ZlaKMt^mFPEdUI~B^_N#yyAuz$?alamdr#%&E&n(nVf^UP
z`OBVvemw3s$-Nb_Y_h;2mS20n>Z_=*u(Px4#qH5pTMtR@i+23F|59aL{C>U5o*aQq
zDr#!`GX7q^b}jAqx3?A5)xis$SYh!kk`D5-YX9!t*1b}uN-8QW?(Xh-F*_XY@2gdn
zfX2|I6D2}UBK4h}oe@z{pzOD0>sC;Ge0iDgrC;ihbmyf7a{Z&zckj+^<CPYYk~;O^
zVDrw3k4m+_zD!JjB<q%nuO&inZ*PD8<>h6eJ^|M=Z*Omp-}3L}t5=(nkMr%>vqvL-
z-<}p|LOyc3ZoXgAp%%_LR;68Lxwk-@Fy(A26drOceDvw_=b!KE|97ukskw0BLeTsO
zx47Pg-R1c(Z*}ZnzI=K6?Ag*6FJ3HpaiMXlc6iYGINLjS?=J0LA)xi@$H&K8^6t*!
z6jlpZ=)~I8-2Cy&mnqlmA^ugo7c*JS7u2MblatdB;R1F3mix`MD1SGnMY*M`;==>R
z#csWeoZI=196R>uZu$MfkB?l}tXZQ1P0@<?<~c1)cyVFjk>kgW%icuv$=idf5lQ>H
zJ!f<^TogZl`n0I-?=R4%FwnThw>Oc`&du#!w{9KW>-&S(#aMy{;X!?)ySq&H|NEu=
z_0`qGr%!ty5`3iG-Q5j3e#OGla$$(p#}6Ms^Y=4m&4QJ7PJ80d&NhF3Vxsbz=<R)f
z{?yEwGv~#foyHqCZWQVhkdFup6T4sk-xk!F{r~T8&DX2pF}q4S7dp4Y4XCl;YGq1K
zPZtsrk}%B@0ZlwsetzaN%cN7?f1ZbfQw@Wnq9Uja+FATODJ$#L!^7>M&XCV9JxF3W
zavW5&hpme_xiR^8#<ew)d3kwPR)_bWoo()I;j~A6d*0nc$Bwnd?X3b$n~CYiflIOb
z`}V>@z~y1a-Cd@~`{mQGt_n>_Pk;R4#fj(h>-}P4=Cmjuu|9SBbn?YTt}|xM+LU|S
zEGIX2XUR(;VRb*3*w|PmAxP=4XafJ8tSr#f+m9a=bLP&yxjkPVG#Gt5f4{1&Z7h?}
zBV8HGq82gTD1+2fB0W7lpux+3|Ne!n4)cW-;Z9p6q@*_G-L(QOAAWqS7u2@ll`=VS
z{=EMo!AGV+!NG-vg$sk1`yDxc{PSUcdjV<b?$*{;m@itgvahd`RrjBla&uGalI6>j
z)6?5$&XoN4@uN_mK>eZ+El@rE_4W1izrVhgyuQ{uV}?Z9`FXv4eSIt{kbra%w4AYe
zrD55dh)0hfOInp^2nq_CWL|1{zyJTdGiQ7_JvbE1%%+{2YyJ4m8&H8h+blQf{k^?c
zR)_n;{IDp)$jIo>;ls_%?EIG&I5IywH`lwFjhBOmXG(!f=hTT44{pC-_dDs`S;ORG
z1urkD%E-#Xie|;G_C8tbqF-M!r|Co<I^52`aqHHl)!*M4UEHHlxGn#F-^!Jm++sQl
zGA}Qau`cU*dU`r2-NN(WBiGp2x%vBkw*B}~0cvTbq^O*oWt#lv#>OQ}mwGxZis<g|
zKYr>IS9W%`oL$X{_51&w>J(PbxVvjBBmk8JoH`z@EqZ!t$L`(5kB)HO+LC#=OH|vy
z(D32)_<B`+eSapQqo)qH^Ut&>R02)9USAh$VPOIC#IauK_F1#$eB*%R=)k5eSy#K1
z-TO9VUREn9DUq<RvpGA<R9Q*M>5!mnTwR^r>aewkjvQ%G^`3TNO{8(3jAhgI?dGAY
zLMAq}2t2CSnBuiGC`rJnW0I<}a&zV9XB!J1I?c1I^_p*28?mRt5VT&g;NHcR!OM3P
zJw4^!Ki9r~-u(K1GX<61E-ZHMpZ;%lV<Y3;-R15J13W7J2Z08q)O-}w)Y`N{SAF<=
z-rl(6MZn20NKMM|G2J}xPQt4zD}Vg{9lR<;Q_W|_0e<^G4fE#BYxyj8>GI|KnwxLj
zh>$YN(O}%S`1SSmpqa{TxwqS<O%nqh7x4T2{^dS1L8}KL8R60VH~06?2gO72^K*ea
zi&E?U{Y-D;l~&W$%wL#wb(M;$>fWo_n>KB_aP8W(5BC<nm}i{MmzS3ZTB($reE3i+
z_sZDa+gg-c1o9o;af&!`fDX6n=<7?nx+?U^)2E<jCumyd;$nBzIp!u+UtVml+BbRf
zWRZq^&y^vM9zHw>noQbL`FTb7dbx=cC$g}xcra>!D+D8}%cZcj0E#UYujN@xP2Kur
zPCh+7eWpPo6KG#Z^6@?kE2~K}W@s?(ZEJ2m%wAjPS^ngNAjrlK4-cO_dp7mysi`jR
z?w3KmT6Ye`dy`7oMV&Z4f<}Up9v$g?b8qkLOG~{Uzj}4*%F5uSUth^@$-LYayxi~O
zuK8ik&d!r2PYw(ZpMD_@wC14Z_uK71e*9Rla^=Yz8<S@mCb#)~(`ym<-?W8=OR=S5
z>94P^g{7pt3JMHB%Nc)td1;byfuX*>p7VLWg@wfjwd-N*d@>rW53*BIQ$dq1udc3c
z@9*aa4Jg~!%bA;-YcjM#+6Dshlcr7;-MDe1PRx!8?EG>D2?rQ<mA*c<v-r8n9Q%#i
zx7RN|Sh7U>h5hYKsobC<;@O#*SJp&Mj@@0hG5fmStE;QkA3_tiq8*#BudkZ_JRc_~
zCOthp8H<93X}ZxySywcIf`U{QJl6_e_ox5+3QZBNtZQpL+xcWw)`VNs{xX?g`%SW$
zjrY;V$H#Z<+$kt5+zctZz)jE=4IVc)w>7c5rv)$fySUI9)cx%3?BvYL%X{+dnU}+#
zV(s}sE2o4V2;WitT~FDq2h?Gdv90>Dl(Eai)KpbLv8CdCN`i!9i@^Jw*Vop9f)><~
z6&4l-4K=P=yY~Kztvh$_jM-hb_D1Q)>K`8z-Q3*jqeYG%IdWk~Ve-w&hB<R&?%cUE
z!#JJKf1b_4=<Rtrk(*l7d}n!BIB|Sr^z>dJ;M7rWR`9?<FLu`vfBU~KK|w-hW@aUC
zZ%uvi;>GoS-$2vKpo%tn`?`o!(4>NnuI~R&JDe6SNIcvY#PDg2>zD7p|Lyzz&YFjp
zmy?4-!N@3x%Rv!R^tP<JeDUH#`~N@rr|ZWbJ9@MgRDP}s)o$aH{k8ow_dLtuwq9v-
zCWE3U9!<^7-JxphUww*<+1<2k*)q@)>6qBq`!9a7^UE!X-Cf2bWzuowN(iXvjfs&l
zGc)^?D&*AB@3N~Il&sb#B_)AY%vXPV;}{-3{p4iz$FE*_y}YzE-e!69_PjT@x4*y1
zKhddi+BC7>-`+0Hy}j+IQuCTMI$B|CJlx!ldH2WfF6+IuHCudk?%atJ9k=J*wW$2`
zr2Emn$jxc%Yw~JpY}Tz?2U_j1a^=Ytfs1F@)mpW+wS8JD=+x2g6UEdj;It>)cb18w
zjm?}169lrZt~&bW=4L}f!^FS8zCu<}eOkoOlCGO_^XAQnsHjO3CIsjhUATC$^5s2`
zRwu{X+j6@@8E4F%-Mw||*KQ_(iqz}}d#k@o7^m@onv{^SSiP7X2QDr3UdkW>X`(ub
zs85?V?Z}ZM8<LKKw;2=`F0B0gtVh!LnSSw0&8c3OE?qJJFDTv_vF88(vcDl(Q!T8l
z#AfHN4$&$sDoQ#(&sN`7PEO7*B}Ii{=FFLYzFhW~ke1$D^wcYAd*0cHhuc4X|9<?)
z5tn@mP8@+9ORQH2ICb#9`}p|ynVH7zN4v!*PoMt1|9tMXHInRnG7kR!>pj*N$HmD_
z)e2S8)#WucHGOwyr}I*;sa;3E$=KJ~?Em-cwf6t$ZZTbu$Bfg?C>R<FCMPF9J3G65
z?b@|9f-TUDt?F4-Wi`L%6K7^-=AL)UpLgr;Q!p|4qF;O@c6V9tn>TNiHlLfU?ho1(
z$t9{)@Z!S4Ma4x60s^+@-3>o?@%v?ed&AsYTdwbGU${{5@^b(5D=PwxuFn^Ml;w+5
zX4%zRZB9R5_xQQBmDQuCProjD{bG`;_Y8}|MJzuj`pht3G&kQqT`qca8n3c@UrjD!
z<i46pAt50H1A_?)tpZN0nI0@o9Ex^ZtjphRNI2NEy{Z>f)2$9m{rBf5!<~J#(hd4N
zrvgKlhfIw8-o&JumDQ!Rb?=_M`;TvJ&DOUS6A(DCB5-klhRFXLlYamHI~R0>+?g4M
zn?7?xs#-;_MVXhEef;wB^7gvBO|0B2;`UlSKHgvd@kaPot>DmTZ>PTvdxYQb{Zbwh
zBGSypTNU_ZZ=ch`glA`F^2l0!S-R4(MZjqVo1|<AhvE_Tsk3ICYU7o@_i`&&YZIri
z+JooM)g8q^BR^H&-}%PH$>mqEcLuF|^6&3&ov1AyOt927#RN3U@aD!wlZ*=uK}+wg
zlYai*wSDnoWzaZ-f&C`!@O24!d46eWYVVC&4)jPGd-(cJ1qIJhkMaP;76JJw)24}?
zo~FyJU~fNvb@+NvzcVR0d1t{xrswD9t1Bu#%wOIYpf&Z$vuBT9zt&EybMc>V2WlEE
zS+eBMYHo&pIonB$Ni70SM^C(1;KZTG6`Pl*$H~cAugSMQZm*Z4<3aE9cW*7)cSra~
zW2s$HpHWqnP35O2>(%+zOGvl{2MY@e3rm?~I5e~Iu880NulxVit5>Hw!0L`iXH(MC
zg~i2>w{nZmFwd9UxN&0;OC8I6S<COIj(`2~#lY0`ulL6A<$khPu3lBOP;Ajq0;hkD
zsY{nGb={nH7Stq9KQ~9w%q-05XM{S3lvIzL?XRUt>x-VA66%4bHK&e8ZoZ7urcKj{
z-4$|30Ge+*9<9@{N)wy1Hm}7=_{3S2-}iit{N^fH|5Tjy>|BUO?Zn-U?y6df;QC#$
zMZk6E591jI%~G<yaep6Mh!kDE6tFsg@u+DjXjlf4rkzA0L)WY{FK4M*I{)v=Gds@a
z{$IP~oxQVTrt+7n8v&O$2}g-)F-({}S9vC<VvEM6luZ?iEdr{4rf)F*D)v8+;aS}6
z(>oIv@08xtzel}a!(-nW<@NCv??1=~y?e^MU|oD*9Vp<yLr6{{u59)-t@l3H8J==a
z-|=8qOSoV3`ksf!u4kWQ5^1}7!A0>Q$c{<i;OdyP`jo>eQ@!7h&R==^IzE=+-#UdB
z>*wu(%LPUL{A9nng*{a|+;Qe?W=qMGIXsHL^A5(}b2@%TZtff7C2#xPH@oVenib|Y
zv2r?lqjh}EwzkaRe>Sh<4z86bny=w_o@K4ndG&<ww29Kw9U5~y;UTtu`OQ^^OXvNs
zPyH4;sVRToH`D$<=EcTlGh?QlmkY>tKHGXfA;kIrQ+2+-U1w$l2&JuGaNx?US<Te}
z+tPSVwk~)uO*3`PtHhA#??*3ZfBV_weOPVM!WCY<`E`dD96Fsp{n2M#bEd}^-Xypu
zPcN&t*;lA#-lTu>n1~a{$72w03wdvMoNE8?#ObQVVq2TKZ}0g3`^18*xhtPr@-CM8
z_2k~2^E(-9Eld*(%+KAa{`SvTC*#ZMh=3`M=buC}u3)*6WpaGQwx;73FWUH>W^)MK
zrDv-Cg)vp>y8E|h=hnUvpa1i~|42rT3DRcm-M{DWt^c*~X%GV!tkSIrmfGLX#{A^3
z*l+jO_oTJ{-{ttS#bIXoZ0*V7w)KCNgX-qFe43j+*V^>emnCj*nditG%-F&cz;No#
z?U<jp=C$lJ_K?^+rRVuyhpk%E>ObBq7i0POTVl!6U6K30G{<gRX6F^;@h5*qjte}z
z^WXJbGh96#R=Zo?wUv`?8Qb}LPj3|(#c3(XPu%kQO85R#Ti>&BEw;D&`btqw%|5?8
zL~Y{VJs00E6*77G|K{%ZA2d$w?R|XzY@XCY)d>v?J@qfm=A0|QbZNE23MQ^?;Q}Ju
zVlMsdO0H9s5*B{H=Oq5}s|TmYR5isZ9!^4&?w;LOpX0H(bzV{Zy3^j9b{kv&o_+J?
z&h&Ye^IpbS7N7nXwsX~kNUMK*ed1ORB5$yszIea0;=~0lC8b3I8cIqtqhd;uI82^=
zI`qT9w(i+i#qQ60=Wkngdul=J`6=&JWhUEY7WrI_Zdvy){?Gn7AF^&sHR$NRe$X;i
zyE{FDwX37UX+cNFiNhU#v{gI}PS3jFc}DoB;MTdmI}Wv<{5Y|$VD0SZg|<d^Y}4`^
zRx7es)^H{5E}U?IS!}I?pkN?}h@fDl#Kz-(DkdDJGd@k;zx=(kZ~bm%$=_4=RmZz-
ziL-M&vsh=rTNcIoIotH-E9(Db<5OBL-I5>QthAt`LxZKOqr+y6a_~-(`unmD*V<wW
zs{<xqShQBts)i%a++0XdPF~?|*MmT-qbKUxoOT@fZk^1_?R)oA#QPeJSJSt;Fq?Hx
zoOAW`LkEuO&);9!B`he|>L4gM(Y>(pi_W6_=VrlmUVc)3+xC?IuRL?9OhEMi?a6am
z_ZB`7n!bI0U;VMpJ+W14)oZ<<hi~X-k~nx&le=-n5p~^XyU*Q!G41m?_QzpDMK9Re
ze$*xExNjGfTlnyytILy%nHwqu1yA15(|vZr-EWHAC06tFf)5(5CkwkfY<Bfuxcgt(
zOl%H!xI+2YRViolj)>LuJpRijC@9DtY_I-g>B=X%4|<Y2pG|pFK3O<9?ctRgfu&JX
z7#{q(R2akGd3d2$@up+(k|s}eJB`_rHaJe)wBty*LGlMZ&u@G9TH00kE3b?Fb39md
z)*$KWsZ}$h(xJ)VQ-t5yry1ARK9RF%_%pRT*+{ZbJ9h4&rK{B|15K`*>py%ipROty
z@{diNZIAmK_xY377cp+S`o`tytho#uj>bGav?Z}+jcr@-?e&`XH#!{(Ve9vrGWWmq
z^JQ-o<KH*5$R{^z+8;gs_teVrY85_3#)1^S>xX7Pd|5Ez!rZjA`mm&sv#mQu_M=_+
z$;RZFK|3lclg`W%Wmn%2^LzH}{qI%Y$z5)(`z^Mxt;Ql;tL-G`qa>#eyAR=qypzQF
z(%v81z5ngh1%)dkc1XwkcvpQWr~0a54=jyDnj5gcKP2<#gQ8K||D*1&SxrPAHC-26
zI8|9m$-lv>O%M#g=@G;Nr2-I*Q()qAS7(=wj*cV6EAGs)5tZHX=kuZ6{hU8m@cb;O
zU#q)~+4#rz17CmdGx7d>Z-@Aq#;2Fc^z?5f{@=LnjzE9cQbVhzr}w+>+}Yds!ubEA
zd!PRs$jmtSmHkYxi;Ih5fQySt<`3=5J%4O(Jr2FWJo~4}_4+s$o^9#|&t;yx_^QYH
zH~!d77W1U-_v~4$LY6yhpO(U$x9;}ARA;-mAJ6QcXuq5Ir}S-=K(Vb%djFd{9d}lN
zGCEhIl9JL&@A|8n7DWtkteaHq(oF3qu5QrFWe;BU`1HxTjK^pABj#9rJ;8Q^L%HOp
z-yQpZmUaxsifTVQozwp#+a6RW^5V+0IdV7OUVd_B;a3eMCI0W9mkV`tcq~ihetd0Z
z?w_x3Rg{wEy;-x?Twie019rb0i@C2di>_$q$FDiv{=@uAzqr=ZP3^~vr7izH);Xgn
z-g(nz?&R8C#SdL97}l|JXWBxG6OUI;4>+Cb|IF)Pu20-%b2D;w^D-eL^NtnT7gzpq
zUDp@R`8#xP$6V<S?LSO@+vgiEtNF<5#OALQlpWyWBG)*p8B{E2%oo*cT-=?#>G-2{
z@kghrn!8QE-S|n$&|UwI;abJ5?*o_1-JHXyv}4=9`UH)67j&*F&X;GMmHXgNSF)#A
z0{4HJs_vwF_bd)fetW{r#YKe4)y2hVvy3cz;Fd|@n~pCk(^r1f>h?tQfz1DosT1V?
z^>go^U%lnK$1M>P6+WKdon0@RCL1>hSMKUp@ZNZHj)bwc;)lCGR`Ij%SjYehI@hy~
zt}ZTe3UBQ9-muS7zV>5(ZcAa^5s`ELcYAm@w8}ru_<gNkhM{nU-@6~n7Cii2zrEe+
zHs`;{LY14RK^bqU<o*YD&wMZA31JX{7F){|^Nwfu{rShwz;;LT!_5S*x(gf}kIyV`
z?x^WeRF$cDHSup*@P$xoVY&WUp9K$c|MxLVZ!XT;v%B7@HE{mpS+9;QQ&Lg_rxX6S
z&-#^j=DnXXS?B5M$!B>FP3wDDb>eYs|LKSEa?GByn$!eM(+z&J`968|^FvC%wb%-`
znKvJwoaSr)6jq>=-0&-?`s0;Xu9BtPa@v^B=9zwBQt*Y@+h;c|E2&bg-+zDR|K3^d
zA(IyB@GSc?MZ!?RJysCzm6ZQwdCv`)D>eTV?mqro(eIMT7q3an7Mv?(E;u*M=1ydp
zkNS?=7g|<NWN3>KJrNmU{~Ts-$dexra_>y~^EPOwa;3IyRr+mCU&r;bemPuOC)&Ti
zejoJh^!4|16!OfM$IE-C9NN+5m$E+2Q5zP=^DOTF6IuPt_s&sX7XRC`1rB}MYyRcp
ztQ4tJ>SnBW_gZe$T=7=vVY-r`NKVoB2FdO7CM-4=`q#s|dsV42H1Jl6PyONldT*WK
z>fk@Q_B<{HWp6ANP2+z!|MR!Dg>2h5iBzw@7^}YdXq3?Z+&(`+EopTV*1fwj!#6EY
z(u-s6c_MA_?A*EdNq5{clzTd4#kX*|LQ93MK2xP0m+kIdUL1BO=w5RAiR0Y=WgYm^
zJMLJuW~aaL{5W6mv+z6qlGRtIJLT5ApK)$)RAY<%T&{gn-ei{lH~Q%7*e_V7R{5)E
zuTbLr)?!7kKYz4YcWpUfm~S_y);gf%Q2Ub(`NeicpLPEAUdU)vQu-vYzgnxYqa!7X
zuT<mTXZ9x^MU&?roH9S@@brXk%hMnA-U}saxJ}et{NY4iy`A^Z@Pbs~i){H^HCN=G
zw`lfOl>K74zGtzeS)FoZRjvEXW2%$(#$KCscV*OOYx8UUkB<0@pNO@ZZhPWwe7*bn
z1sji_W6&>rea4gL-q916Kb3yk;#1Y1U=lvf@4=fM->80Q3{TQ3jJCL#eEM&Evh~52
zac3=Sk3SZ-{2yn1?tb#76)Tx<SbwcEK6lW5|DQu!*Q6<3f4k%J%LlzO|KIYMhp$oP
zE6@pMzpk{ikl~TizlYD>TVMQY53PPY&GQ6ymI+S$&N=^~wx7zMOul<fmJy-Cf)kG~
zuLk9R7ic-)0xhONG`L^@F+k-uh$bu`*XXG#AUN?$%E6_pD|TC~6z~t6<7+AaZ4RB=
zvv)7!jQW?Wx<2w$R8*|sP*RedaxiW4%bNKe9ZM=VZru3h=4STpqm%NcRldKs7qlp{
z?(eUQt3p?oY`(+C$M@sMj|VSatl;%`QDMF`)j?3uQ+&>xIV&PJr|pVL+kBJ9p;Jh8
zMfCP{VT(FCmT<n3a059SlE6SL!p<i{1@U1sbPd16G1*H&N$Jy%b#Z%7E%ly0$+qm-
z8A%4cxIHHpI=4@<eg5v9-@iXULoEaaE1&V0%|5%#sH3C9X4i@p8aB4Jon2j3D_(-8
z`93~68tNb@Sh?(qOo)ri6T_Xmb|o1|$W>YS`}>RfLsQ@->E?r=`BnL>S<`f*CmE;v
zA*2=`IB;P3uPQ;oiR!N3!7XG(Rj(-zo}8R)^J4kcvuAs?!`D4{^=j3k&W;nt%e5Po
zl$88Eo}ZiRd3NX3Yipym8DO<=)%06x^Rl?Kl$6d-hSaH!p{v7E|Nr~@eUH+MhYt@v
zJw4rUrb)>9jt(37T|yd4N}pI*+1T!F_snSLlNDibb$PNAoB}7RznEYDPqF^rpYJ?P
z^?yFHZ_T<Iw9v)n$<B+Q1Ugau#O>R&tx8|r`MBd?f<pbjKUda72D=JDtF@jb#m~=`
z+oWy2$-%>OB*9>XgObvxn3a|*Iyz1Wi;0Q3&9kXAn0@x7-2eOk|GBqviH0Z$3RbQw
zQ4Mf$dBRy-UcS8W@iESNX&%th&b6S*7hJ_jhF;ppkZ;Z^Dk$j54GC+_j;=1H@^^P!
z+})WAK0Wd564PCEsH5XV@$$y5C+pUPX(uKoYPm=y7)YF$sy&@cQL^i7+T&keU#EQe
zJmKrBtI1zqU9G67n4s*=w=Qn)tcCXZ=jK?NvMuWHsIu61WvYYc%R{Z)SH6FK{G?M@
z%T4Ox>({4enP#7uXS=&g@#VdrpPy&m+M?OcCmUpFG^6I{C(v<5T48HeB!L>SUR8op
zQeJOwZMCfXV=*sv)`KU{RxYwhzj*z8!QU%lSyxxN1_TI1Zb}IZRJm(a@<O0j+Wgu4
z`hTDS(I@BTS|6@|*3rSSDedg6hwW{k<@MgvbPnFU30iga`2G9!ym6oc;L@%Q8w_4u
zU48suGrQ)^Kl?>i{an^H$xSBOtfTn0x{!dt0m#Chpg@(oH@9R8uZ`NOWMwt$<YaZw
z_83vs=AV!I?O$A22wFk<=hJEZl9!iUT?GX_o3B`}VEAUd1LUR$2b)E*Eg~2VdO<c+
zgugy^Yist)3k#XWx?No-mVnk9n`B+#Xjtk!U5g<tU~BgEw7<W;R#aAA+?pNU#w)#S
z5jZrWL}X=sH>aJwvA5d1_4T*K9?kqAQ%sH}7<>qNz46J~3w@wjxW(@Lj~+b=k?<4;
zjn9LQlzMk(C+NV9(%08Q1C2HqrJmYQ`8n;>6wSl#^0h~Hzu&j|@6-;DDC=z@#TU+;
z%`v;YDpWgcZImnH{p*@{i<WdvYC9UK`^R!IcX;2rXT9e45-u(A)aqIiX*Rp}`~CX)
zhRMeQR)uIz(~I>2W%L<7kB;?9@A>s=b<W*gQ$foQil6z!#KeS12u?I{2xl@jTz%Cm
zAwgj^=vZ8>#wl5bPOhG><0BS#a?Pzjyg0(P?$=9oHa?jZi#%qD2n#zO?~|1@PUA7n
zx{~ns*H^BFnLd3omZ04jxwp0?UR>n*<M;3E7mA=vQ#mikOnPtC*QA3@tQAvpJ#TKa
zPx&$Lwb7aB*Pi$+&D(q;@<l2rNcR2t<SpuYX-fV7ztbBU7@nP-?Y=r}ZN$bT)_3pT
zY5Ax<dGqGXmdwi)&*zrk*p@48nsw#Dot?#6J)lm%4gdQ+pZWIx{U*KB+iXeA@|6-Y
zJz8E?NhL*#KV^SD`7U;zqI27ayXE&+YD`iCEq{vInzb?YwAin2Z%==DdAYExEG*H;
zu=yZpr_zsaxAQH_-bmQl*~!>c1hfhYR?hpKc2{`D&zdh6-Fc+Vc)lxbUHRu0Q%~P6
zcjdiZy?ajG^Yq;z`Ab|_-EWEid^@c!8*|VKgTlwhV)j%_T<F~X>EG}7t2HK_`SbHL
z=<1om!a~pn_8&iguG9dv(mrje{e0FOv=>i?b#>6v>p?!}B15N?=&9MX*a}Mv%9c%g
zd3pKeoyF;*uJd-@um3++OgBnERP^W)&&e-tY+T%>^!-klRoNSlk`fb85X`kI4GMH|
znP+Iqe%9q!pX})e2b)h$`>v|Be_H^fL36>mdDFG_PJcY-?rhNE_Z1(LL|x~tto-~;
zP(YyJY+CZiM@P9DisDOOUrW8atMtjaxw8)(aB%AhSe^oQYY1pDb=}`z51&QO(b_xx
z!HZeJt|uL0Y`KD#eh>0lw-c1*9336Cx@?3OyZ0}1Zs)70sMt{UHp=w<<Pgp1Z922f
z^OuFJ466C@u>H&H>;B1{T^%;gS$wlNLGu^~4miXVi7lzRZ))-ULs+2EiVpAg8^42%
zd$vV`rtWTS&0ekXDda|u85^I>i4}p1H3Ap0cu&_`nsBg5tH)-_h7AUBwO>QG<=^-F
z{q615i4(wygY)!s{pF>vug$n)v+`OxZ)A~ZSfJ4k>F3`z<!5=v%;CwowPWI{(ABFn
zKDit{b_}$K>Dif?&HerSpP!#Ue)zC+>qK`UVPWU<^K3y^CxA+Uo10RfoLPBIOG(Lp
z@~q~x1kfJi*xh9kv3FKp%3jHopf+)FLa0{ReCIE^#=Oi-79}qxbSe42Q#sr#ZIX4R
z!?O6<g*}zUVe4X6GHCQ4y?Jxy%gf6@KbxKZ>9~A-z~(gHbMx)j9|Dyvrw-rRmMeWW
zddq>dlBGrGT~p2md%2!;vFW<9%;~vgOZ)vLjr;A@{pJJ&KG_(%r(z-~T~~fiyI1j;
zH|^}K*6!}M857kD3?4l=aAKnJ&tI?CbMo;WJAT|<eS?e3JS9{1vkl+g-}m49>ucw}
z)TKr1k3P})vZ-s5XRO_xo@V3SllANEw&mVkRlV)WL{@fo(0XsZm>mZuC^}166fEdc
z@;`cKXEEpqsdekt>BR1uQuXx}C`J3vSjUoffO%^hBcqGk(SU`wbyyGh==t9io04fh
zkNf%g$IqX;dw6toDe=E86W%*vf`FR;yfq9O{-N!BvY^!mX0v-i1;DwvtHI^#slA{L
zF>L?-wfwDa=9;^F;_3)nQP&gUONEz4)gE6Ny!^xW@6T^-&DQE#a;u4z`@{F|$CEaK
zw&gN0Glz;)zIt_i{rOv4vrkUd7MHayOS!TlPz&6sSW@!v5bLYs@h>&Mzw)n`mbYe+
zM^lsCUd?+yv_Z>b7Q6Lw@biZUK55+d|6esIS>3sNmy?TYkz+HPR+o+O)6>(_FE8_*
zq7ylZk(uqozu)f{TStIeM5=Ka85+FrBeu<ftp$)Wd2r|E<|&!KnJcxoga$t0Ykj@f
zU2s4D4bT#vStg2FT3)UbcW=6xQ}f{<`=0OjsztQJdVFV_1#V1o6?L1pvx${^Q{rJZ
z&<P*5RVH(-N|nsa!dgKcg)?^!9XbT^|BsK4r{oy+{0ZH?LZit4C=*-ohG(;LZg1-a
zEu0q?UcAV|Xye9>B`+_vPE>Xe+FMm>mUF{lp;K#!L}lKcyLTfuCb8;9ZAo}{XD15_
zOTpt~yrOO{E{X=lyDC4cfvU<23!O!{fAlz+{#vE+$zkcN|Kjhn?t->a$Xb^jN!qwV
zgUO+y!eafNPh6m5r7|xsTNAalD`scWN`p^#cJA0=Q2fjXlql=||1HnDw&vu~Zt+kF
zLBZAoXH>1MW-arby{Y<p9@q7^?t=DTuc@#6j#+U|@p*S?srQw2v9o)n%`dGE*9WaE
zTD^KTmvZLwhNA}?FD`aBPB|d}TItf?&+pzR6PO5!4CD42HzGibE2im2YdrXEbu?ab
z(V`R1h87k9->!OAOv^lUW~TAW+u`98jW;iL?|=08@!^Bb>_7i}KL7LI@ArmMy;?m>
zJek?~Qa(L72|BH6bDHnAoSTcl2{SDxZ1vOw2OPrI$Mu#^JqNBcC%XUsB<&l*@kg{j
z>FW%eN+V`Ao`683g=?d?gSL)?8sJMjCrg-QOz2X|j7vyR=oZsGl(qHJhD7IQ=jX47
znCoLUyLX;#^`?S{PM@Y^AN9^Xo@gKuB2oE8w&CPVue)iJH(xjnI?`Kz|DPb&lP+Sq
zQ7!xT+ox@w$;vHuVQIJ3v`JqUgI3ta?iMRAFSjarap1<r<WSIX#(|#abIb1~-rAD+
zL{HRVXJ4>^#fqKNZ0r^Xi)rmQoB!sXsDx1phq8NLNT5-peEpw^Wp8gyRCecUYHH%(
z;0Op*xw|r8A=7NL+(i)^lR&$C8=2X&bQ}dIh8Wz~vfW3VjaTZ(hlhs?^@C=gTCaQH
z#fwFYP6(&=?K(EE(|3ntRS&1I+6%jbJg3q&fBbk{-Z<xmL6?~BsZFV;D?T0-50&t|
zzAkpR+wX60Z*I*NpKDbL+J<`c<VjC(TKCz%@87R%yWek&i$vCVO^x$5v`h~Te6q5G
zlRxqNl4CBh)`!2mynNNhU@>d$=9%+stBsP6@vI74d+Jat_sO}|+qsk_ok~hf?(Qy6
ze|l=_mG$xSRlTQ8u&pkGB=x_bW&D4CeO12Aox1YZF)x|e;#C@-O8A$oo1c7s$+07D
zF`yRAmb|-Bt|u9kl$0`VZ_|B!ZSCW!;c=kwOgTBpwRIxT(X6eY+8(s4>eN*2<Fib&
zKYaTZ22P5X`&R}pzqGqN|Bg*}@Y?rTbMDxLid3p8t)G+deu>(Vqb(b2eiliYW-VE?
zBpI|EG4IX{`~QC?gFKRUMuL}7QS#P@hljyu{BVgVXlQg?TI#(T+<?fuKgY5d)E`kz
zcU<*7wz-&Z?#U_X+K(<Je)G#~Gt`^4V1dG`D=QZ-TH;*)_p5lXl<A{4H#gV(dbu2Q
zCZASU3YT%(nFnjP-&2a-mIFGUA!=(Ds37NR%)GuS^>oJNWxRcPaUpLv&(hh+wOZp-
z%pHsU{k6eS58iOhHp^QQwKYr2%W6~Y@3L9u`R972%_Yq9WbW=R4-Zrc2DNbKT9+S7
z+6XEFZf(m2mFWu<l`hTs`uh6ieYMp+dhbHs?G0fJn73MnbydK|-!XYdBRDdybU3&3
zt<-pV<kC{_%nJ(`x98nWy0@p2t6`F+Y0ixa3l=C8Jvrg{``g=}58LHKz)5>c>g{d0
zTQY76&AvGGl&$#WtE-oEDV?9|J~1lfg;Dc|53@}^e1CqVQ&`JO$~^H9%dIV$&ezsN
zMs7~y?UgnU3skwhDrBV)C~0S0T(sx+yWKoeCLW;HiPHkF31*-rXW_F<4(1pJtUO+`
z>ACj$MIN&z?oM$&Rr+h@>hN_z_s{olPCvg&BlC*-{F+12`FmX#I<?-|QK%fcDkLya
z<#P4MqvD|Dk)VZEKR!Gxd3}v{^UXDfIy{ywt!BF1@Zdp0*7cQ#_nYShtvqg1B)WR}
zB9B=mh5u#x&b)TMxXAV9u2SvF&(A^wFL87qb$Wkq@6Wf}?}H9q0F@t?gKfFGRYYZF
zeXp;Jjoe#h`tRRA(5Aeco10b|C{23#!s|hC*{ds>f22$l-rScCd0`XF7;tGz%~b24
zRaeVF>yg=br3&8OG8I+j-BI~j?d&Ym$J_7MEe>4lwkC438)Lu|y_nco(DkRb)!$P7
z{`v|khP|ijtu#<tB%pEr9O$^A_s`v5JPkNiI-@5&v-hZ4X1wG4dm{Cz8V3&^WMN?u
zkds@t$i{uTe*7}OxmGN!tQYrImxGp?%$_})OSy>OZSlmy$HzeX9sd3K>|aw;V^Q{I
z2A8rDSL3G~zB_;Z{IRHiC$sc+pK|_BGp(*A(yg!m$-DlsO934|mwBmW#R?74r<N9V
ze=I<)c8h`rf4iS8A0Ho2HkuhCF)`%FkH`Iz_H{Pm`f*F#dZlvi?~8Q>cTK7e9XbSB
z8V@>mDWyB{$@*KMe&6zl1ItBL{S>>sEq8I~>aZh6j;zo)f8pF*>y?3v*`A-9`?y`c
z?!je$`_MoaxuEaw?#{HYw=*;}{PFYW#*~vnqOL9?OixdCcXTWW{8N#Al*w*+K<_@S
zt|if`v+KS0W!(j33DDj?P_23Xz^2sGE8_Rt1qKG5nQJ}$dR(=yt6-%IWT8>Vo12?2
zFY}c?)-MmL8Jrf_SYEt%(Iop?k8Sn0fE{u5T|an1WwQ3=DxOO1E$J^WEd?E?xnqZg
z=u^#KFPF~;EjrH2%LDCQSM!~<LPM#4Wzo}9Id^tUJk-kl`P=RM&nMOAuh0O+V!ytb
z*|crBw>PDp7UTN0&++52_*EK{6eVP2G=9Hwu{mv$dTmYQm$$1|-}XP9KEKv&zFn<G
z#0CXv^SmWlS669uoe&OM;xVyhqv`&?-=cSwzhAeJ#Z~38-2Tit2bQ1D+6r1iCZ-qD
zQ9kwDo-6O%dDxn{lqJ39sJ8vxU-sbFNng;pS%Hh)R%-Zn98G%IYkm)OO6<a*rCvHw
zTR8Of^+nyDfCfYsWnW(h@=ZengP2~7$F(()p%Q{814>jsXl}ll^X1LW$*tVtla|M1
z$7NajF$NetJ9h-q$^5y9f3bW2vu^!;0fF-xPMq)n6;=BC|2Q3P<K^Vy3Stnc?Anxa
z^3a~j&!A1`+1J-C^_y$e($ca5oW@sv|8_gyIQN!G(ex~roqdxHEhIukPHyP&i(GLh
z+3x0E?GIPO;|+_Rc!)kVY-wu?+MXACZB3-}biLS!4GE3M7(^<?ej6>G2&z#*Nt2nK
z@4@TWr$0VE9x5Ri$T5Xo+9X3jSk33a;tW}>l6h^-7Xw`<@|=G%X|C2@P(dalU-N-c
zSlurqQ10N#$?6|JeoVZ-uNJfl&vUX`CXd6!^f#CN?Vp0u<(-|yKYsn%RQ%je)YZj^
z<*x9LDLk@PQ_kD}p9Ak*?waplG<`<L={GZPbgalu2c7E{ySpqjFevl;ySpa2x2Av=
z;ijFDsQUV9>7ouB_toL+mt|jHcO%CPwBoN@OgBUX<mxBKdL*6y|NDFLn^WYfeH(?`
zug}ryO7Yr$=-lS&B;_P!J;VBcHMh28E?#7F;M7#@%=`On&1UyzUS4)_OJ=aBs|xRp
z-R1hZx3@k0_4PGq`@LQ5uN4|flk!$tz7RG`Jtgw%`}_Xc>71*+#4Tdbm^rJvnsL>a
zxVFX7c~)g_PRugR4w=IK#y)CGM&s($+N;CYhcSqFI)m!_W4+RvAu9xGe}7B;@Zg|U
zSI3g*T|z%t&d#%)4O&?~X|uxB-|G%qwtp{<S394>HfOS3!p1_yUxLe74=Lri*c~bo
z`CudzaM9(>#)xI}cmG!lUpJ+b?Prfm%r=9s8%&SZ$gBuGCiq^1`~KT)%WmdA%-g#A
zt&_{A^Y5Q!9k6}6Htu_bf8^etEt&}l5A9CZUU%M^q_(QsN<cI%_1nhh@yk0VD8Ko;
zQuEl3jHafh6~W8<$|N*F7nXu{eK@spZ7g{i)FrMzFVQaACq-r26m^a}cRWJPBMl8*
zqJn&Kiw>Us`Ox}i*Pgn@ztP(s$LwXjx;OUS|D<e*-DWw@S6NzL{geIp>6{X~{{<!Q
z<@BDd@H##H^U~G3j*APJZ+}v)k?o_hJ!sz#xzcTM|7Y(_f3zZM>50GB7n=4oSN99#
zl`ZYb7qv0+*q^a*;X=^5VrF)}3ya<PK^?T;rLHP>FUGy*jNFzZ8N0jesAY8AsxNWV
z)R-283%a+N`$)Pzn(?n#L|V?^U+&IVdORyqb|v3(-MlD5Y~ruf6%St@l71@uy!ey6
zalw9pj0K^(;$2dWNp60BM30JXZhWU3yJEiiM-HWDNiq|{oEJ2{)4i)vz1>eHKT_h)
zZCyQ;$`x_8;<v<o>PkfF)Y|?>&)zX<(=O?kfyX;nRrV%q-Bdfjb@KO`diGnU1q%fV
zUOJd;zU<(->1l1}sr9-k_os-NWZs(MD{UUMHp=v^#fE;y8{6|ib&=Q6F44#>8G@Cc
zo}6@;Z!jq|{_5k+Q%|1H+9I)U%6}8hUk#b-4fd_Ff1G=A|LcG8XA*sWZ(14G)E!;F
zH1PhBd0F-|ufJ$i{P1j}>6fbi(hOETaVOJxKX3Z{{NWclQ}c(W`{d7i9#;JDu9V%t
zRN~CGjk2D*x~+D8vETAHEAvz7m$OX&E^i7+TE@Nqz%SPQ1_@VUdY0Nf>1TC4Q8!<8
zM_~eE<KAOQ1vTCG`9n?`L~ofsUEI;p5!8zV?M$06LE!5hZKaoevzqM~UteFpy!Q7u
z&=AC(YR$-^y))FNX@A)=Q*Lse(f=i;H?{KLd|q&QBU{u|Unz!MnOkDM{hs{d42)%e
z^r!Wv>+qZUZn6AWyME@||M%+@rirraa;9&+9P_8xWWs&Z!kzBtJR-|2wy1W=X)-As
zICb^no^3zGw)0=@d)wnB#2{O-+*how;_6j72F6D@cXq`;(8^d6P?K@L$l`5?mV2XC
z$Q!%q(S~A7Ql^hCEOee(cVAFc)OB^(+8I`*TD(#w3K|+3&8!X#=285&56{2-?DEkG
zJKjhBd%4MaA9s06hy8VR?uMmtf4g++<?2>2a7<Vq6}+tGZ^8HcnK8oFpHDr0$Yv3|
ztdKv=?@!AkiAVO4w}gcyXRdk2{r|S|&CmCBZk_xe#NyB}X?@+{h4+^S-%qgD`H^((
znhK*q&jQZBD+3lAGCy+Opt$@p^SUQdH@7F+GMwF?)8g)Sd)3`LGwkc@4lVdp`s&KV
zPft%<DF}i3tVy@G<(lMQWP&#|Q)WdU3;lieh|15~o0kYregC;?eNvxDsn4u!Idd%j
zW+WaGlRdu0H!WzsY_3m?$k9Ey+h<tl&r<%_KhGjP^<ZDbx@9NV9&q^YVE${0$G_63
zoe#y@CokFh=iYa_6r&kD7EFJ#)9%bKJo!qn=;WCrM=#DXU7nX&<9(BP&(nDA=tY;p
zmfCb(S>!Z*CS*XUewkeP>r+#;K|7SDYKIF+NSpw5fo7RL{-%_3CQ$1C_Bg*go6ZJD
z?s;3sT;Trc_?y6!awp?=omp(ES@P}3@|)Twrc3^*pI_=BY^E1~WY5#=m!YcW3^(_3
z2P_P$eI;?}ng8>(mrt*Zsup|bc{p~nx#X+q6<0q$IQg4J)Z0Wn$865--le`jmwexB
z{n2S>-<rFdX8pT5H)yv3%UKJqwnr>K#7<4Pz2#&IXgdRFO!dwE{qphg@qTkG7BVXf
z26CKv@$LQn^V+_%5;j@|tUS)N>ACiNGx0CedT-p`|1s=9#uC@5)$*QO*G@{Yab9fu
z<b%ZbA_=)o7eAiuun3dgw!!$#xrU0bVYjsN_g3kJ6f}Cgo_pFlH$3N1{JWz*Kkg*U
z>1tUgG4Z`MU%un|`D0D8X=QE41QUJlhVM$N`+aIx!I?iE-}CKqcY6N{-gj9)W^Wbm
zel}i<>TfyAZiCv&T=DVo-`?Cje4vqercI?$U!P>XOkh7#+{u6QY^qqVzn|)^_wK=B
zQO#LK$D-u^p4H{IUMsl%kg7)T4eO)t^sV);`8U5l^~pRritkv2Wpvlm(#KP`+xz{<
zzPJ4Jj*|XJmi4g*ICE<5?RzHsStI^+>K<F04<}~rI@~7RzJ2p0vG$KouZ4U&b)$S9
z=%ifGrq<2r=O4X(eR^H&?ve%-C9X!v(1HR3US_6_p00&S-)}zA39<FnU(eZLc-!FU
z|4sWN4o*xe=gx^!TlK_5p!n#a_=hWB8lPy$H&oU=Ju4&Vl!t8o;lKUCYZ?|a>uOtE
zTGzz+zlpCrHz>8Bbb+Vl<HM%gi%bHxPnlRIDw7#`%H?0R%<`$aNlQE(e;0{;FuFB;
z>6Fep-)H19<}2%-wz{-#&C9;?;fJrc%Bj9}J^S=sL%JFBJkNXPUuMh<es-k!)U<u&
zo|pUFUOg>lSUuHL{gGU@ZMt^Gk_pNVGP~u9<NPOw-+Z>`3aI5Cb6~wAvy+pP(b;{;
z=jYk}`1K34kISuJ4s>j;(b;r!ceh4o(^W^!m-kqkwV56?fA{Ow-qXt(oK5%1eJynQ
ze7d^(;e6xcoP2iYK0ZEfDKAxbKiqoeoC8vB9}d>a*hrk2S-<Z7=FXpAKOeW0|5w0h
zWc0=6;6Zjv37IoLFU<Zv=l#d}H|JgB_B(ty*eqjHXBWA;H)YMkIregA4mPuM+SS?p
z^jtoJ+vn~5X_98zW=T5*C7n}J9?gmIVl!B=>+ipU*_B${@5)%OeKP;tzW(^a5c@3g
zvdS~2xlbRog)*4r#{J#*oi|Mc)W3RnXQzv+YvBI6+ORbd3z<~~JwM-ckh=sL=*+&*
zu%SLbVB={Wo33lV;+g-?%-{HA-K!ja>pr!pX=M*~FFx7qbo!Owj0W54vneZ!|DRnP
z6X||$)ytmjQ}WK6r<!Kx89twH?YtzK(f!!fLY3%A$9LV?WRzTI@_I4P72WJZ?^WHo
zo^EMh^!WIq_m>&H7PaluE76-@QyG1I>&EnC6Z?4`%By!zyk{z_`7l+fTRCD~ghtTG
z##NQV=bKHcZ=U+T-*xl<GJ&&cDFxo)UCTTc$*!LvxmBa>M$oyFR(n6{JU-N6)Nu9C
z1I_L4|8Cy+WP|F$$#ccMtUq4rJCe4J;i;D8Bw-HgOuuhG7Vo$#GR5SEh>7ZZFWHU9
zT#8Cg{st|5*NxsbK|6dM&))YDTeC#p-rjyZYpYTAwKYB#4@y*){W@E@J5jQ#=8H7L
zVfEYvPj8r=^#5_k=FFLo2QA9~Hr_g_Bv~{6+>&*y>z|+Id%yYQw3_&|nw{eNYksRu
z-5c}H$XB5GR^7Q%Dwk(S?KwK>Vg9E@n;U1m7QeAMS@yJe&G}QdX>8|idG&u<De+GI
zQ~t90t9^;ulg%SP)R*hTf6Z)&{&9azoY;|Dyuvqj9CvzIzHjOF$8Kyrann;Q?NaXE
z$g5DB`R~iM$Sp5^&lj=&TGbQxi<g&s_Ke?Y-fr)=wcEWu_mE|ejZB08v$qNxe9RQ2
zeeb<Ez2R;$<BU(IqpYsKnfAtRZ+7+djR(Zs+wFMe|Nq+?y*#pTw`RuHq?u{2^DZsn
zTphM{QS9!rEjc$q#{z%)^?Lp51)eUKa?B>Q7ySEE`Reyi!79rsMY3v-BX<S<iu_Ta
zoqb+qo}FI!MZ=p{^_y2uWcw-eU|Or>^G62vmUCuY)jG$s;$@GvS=pNIMaP%S-m}@_
zR^9n|izgmCyZOS!*XLexiIl0#zItq_c424wp4gs}%=b<Hn(n)<m26^nVEFvk+F|F~
zINNZm>E?oUhSS#9?>oufoiCWgY7lVt{M2<HPk&6>_k)3BhK+B?+3W}JC#Rjex9`!*
zq$R;`e+WFyNO4)9?RxjojZJ!8a+8%Jh3;SU7V~$v%X)ldXYj_F%~l(or|;*#lk{PY
zhnrrH=e37N7v9~dGN-)U<kFs;OY7%0x-Io>=w16;r~hwho#j5w7x&e5GdF3zkiFgP
zuiv%bXv%{J2~12(9Q^#}H~j7c?QIm(jRGA%uqt%5Q6huSiN-14@~0Nm&r3c0{7;?T
zPxHTDc3I|ZI{MM|x73pd_tK5^-v900wf<Z3<b{0pYwn$Vc)0y%ara(ksn_dPmVA~!
z@$Ypx+lj(v_RYI0V&m$UM;{9HJl*zj%dY~xhc27)?g+T~N}fGs`|b1(KefZn_S2u{
z|9gFT(!=aOGqzMoxAVVG&Z}9j^ZNK+_w}CD2gJ_aOSIoq*x&j{!s6HA<2(}CX$LkW
ze#w6?ykF+<&(h!|;g`jZ%9$6x+i&?%VRZYy&6$1o(=YFCQ9hEptiD_B&Axf9VZvq~
zYsI(!_;9=b>h1E4wWr0Rruo?1opyS;<laZt&o;i3IoI}}BSky)^cv3x0sUd7Tz22I
z_I(Ub3cm2^-ptoA#a~}t1&v;@a)~?ub<Vr>_g!e5+@Zm;tQ@qPY`r7%x%t-aGq>4K
z*j_WcWa}Tb^oZ@sp6Z4-eYfb&s3>^+dz0|ZhROfCRfTq3kJV=jSB_nEYyRGn4`P%3
zZl7OvpJ(f*ho5Hrkh9@6;Vam=xHI+4{xmlK&L@vu1Ha4_oqOYAcQ<E!I`>|6SEK(i
ze{TK^xznCeV19AolG+<B54DplGuyNmiXD4zsNS;XQ>xD$p3Rr?FZuG_@7n(-cbidi
zl}dbG$HVTM7h^ikAG=w6<MqW=6aRL-yl<WV@$*ZI1IH%uZnu#BZ#8*&r_~LM%A?aR
zB(GG}>f;tP<7;1P^P(r&dD1*5@6Zyj72aX_!l|(h^}1_1EB1Vyebw4C_`>3AC7WZ#
z*8Q#*`S^Rn-{0RaFYy#!v}lphoqdm2uiv*Qe0^L+b@gV@>`hIL(c8K!Wy)fcy$hLs
z|5UU2cJobc=UKzG;xoSgEb6h74_d#0^YmUTm#&|)IoH>+JIv>7UKxJi-pam<i%puZ
z`EM{+?7i)?UQqj8M$*g6xhmFTy^r_qlmB%3TqWn;X;Ef1XS<ryX4M+bblv=1^~;o*
zQtoN>=XqvNWxQ;^IIU5p!C2q4dh$QjH@{CM3tubN%+B0rQ_`@oPH)bEwvSJ@M{G=&
zS#O%_wO)VU)3-ma%l%icOD}kApv>lXIO3JF-P<?bWy%)MwcOw9|2iuEnPvZb>-NpX
zWuHVpZh!xAQ`%AG-O@L0Z3?`%@z*v^yT8r<$7}n`=IU3D7FFuc&)w5nJUv|VaazS0
z)10RdW*u3!KyGG%w0_(ki`izmi;|9Zf%fcwd3RU3`)E@~hf{);h?tn$;Wl2&%110q
zqj=QgubzAPnL+2!p>G#<v^0dgO+DV?yU^3HdPbD!ee0)t`}T5)m?q{+q$ay}DRMme
zXZZQ~+iuG|w^wQ2c5i<wKWkZ?vW-Lge!h2&_<N@9{Si4U|JkH=&wg_@u-9$ho0rFy
z{P@4`lV{I{d6jwhqi0kU2$!Bb^s8k};^X+wZx4%C>*kpY*IqnsnmBKA)vmx3Gk#pv
zd^_{kuJ!ljX5VmLHQP|JW`B_7KZcJF9+y7~TD<0@T)}0>;0sf_SZll%M&)nLy=@k?
zHOtk<$EWQ^e#F|aUteAx7LTuS{Q2oAXx)mc_cWEwn;D))fA!mOT|ey?|C^ruN#(ic
zv(j#?+4HxG;pXG`YIAe{cL|NUd@`>d#vA_JAk}80e|h`dy$g3$rV1Qap8P)e&#bFH
zId!wPn!G>rQ{&;zyX`{jAE$eMRo``J+R@nz5;u2B+g5+u^8b63ubce#_c1xsX3p7@
z)~NDU`jz>&$FAwlhju^h-9M?m`sO;>y9d>xWv(S3epyvL&$jx}<w}=BHqT@%X8)->
z_WO-T*N^SqJD9TLJ{nv9+Zg`+MAYve4f5Mpok{M0KF@P@)3Pb*{&G!CO&`90Pk(h~
zC8#1gal*sJ<r3Fj;Tc?@z5ZSD*VjZGcrLQ$qIzuoJlEf6uGAmB=h<-b@Wj5GNjF1I
zO#I`vLr7+Jf&ZPoQ?v{I7qeUc+ZM2N52r!Et_jR@c74x2|2OqvmT&duD>^rOo*erV
zFn{Ta>9vtb|F&-WJxBDoO7xbQKUUA2W)TxCCw=$1$MFeg9rPC4%FO$7`pi`o_lPv^
zZ{H1^I9A#x=XGz}q^i=ltWx&P!`-WtC!dR4DS2P<%&o=!w>!O)f-ii2xkNL=OLg{N
zcJCaO*Vq1ffhMOnBrvK4|GMky>gsxNW3u}+{dm8>zrN12tv0*4DfKen*Kgl-!c*_P
zk7_etF3)(Ivo7|Rx@o-jxmEMkk_z6qexCmMUiq(XcfR}2Bb7y^_waEabGnnpH<x#L
z%*2%Ct0(QX%3gV^#86^p;^s|HuSh0obbmfvGh=o*r(GgzdsdCjCh2>e&ll{vw0XL6
z+Gfc!6Vy|e)H>uZS5)s~`cQP;aZi}B{j(o;ldrF{Ej*HQKicY<($(uTV!Q-zoS!~*
z@%F~IEcSZ#4tvex5<Y(TYB8m%dzY$QeaEERzK*h9re{lh9fMEJGS%i5(|PcA`~5V~
z{_NG^{h_PF9=?3JQnmq9)|Y}tpL1_bL5x0o{9pKOPHXMMb0H`B5*D<r7p}Z3+h}p^
zP0gRQmgk4$lrGG8%(0VoXKaJTZRh>dZ)VLuma)yK)cwt5%?YX>Umd>rAYxhjz2^?s
zOuuf9-Ei1v@wIa^Hkhl+N<2t-FYv7AY2(4)jqK8%vzqe!-F+Q{wJv9wT)Uw$O>eII
z_Po0@43pXT_+G9*c<Act@XTv#BzKp+U6g#h&u5N>VK5Kl$AX?8Wv>`7+cP9Qzi~82
zH+*K|?&KE+dsy{tp4};~uyt{ES!}KP+WL^2HrsR^{{Lxe&WAz|KMH*vc=vIgdPh)w
z*Rdx#_Ybx_*Ku1Lwa<UatEbVW%YVFhJLkZ$jUPWpPFlWVuk(y~y%m2lCfyL(64@R#
zbCYI<S6^QrXy<N4b+vKQ5ssRgnuy(Hx(<!~_h4%QT)V|W3tgV<3@X0x8+3n+?`*R_
zACJq6thw0!sCU25_W7!nI-3o@J0vBXio4O-WvTdQ&P-nZQpIx)Go8NQZpoR$>szbP
z)74Y`<lqBeIk$ILKdi`$RMxkB@^QQ7zt5t3kB9m<@18jAsQkOv$=s%!-^WKE3SXa-
zSlw>0``c{!>c46xti1m9zK)ZxUOm{j>8<*M$Yq+F)6awM`g{4#c#d7IRpOcr|Nj04
z&0w+~sIEQFbuBR?xcb7zNU=-Xk1v<#@y!U|e!}_99kb#eZ|=4wcC6p&Y})nI_pbu$
z*3VBXcdTyM{ym<du$sfJ_`9M$$9{(oGqd|cmhZnN`!Ft6Z7x6GvtQ10z8gNNnpwVk
z+Q#d5c6A+WKHlhTJL{U(ryts1%7OzH)SXSbbzm~+z6j6~v-ERwJSCJvK&ugWH$|;2
zd3B`|bY5TVQe)NG{NBgiquQ*)N?sTm*Uy~HzU{4bu(;c|IUY8TG-AG->ppqZMpo!q
z<=1D;<_GtDwA~T!Eh!=YtZCZC6SsNa*<8>IJUH`u)~*1pt%u{5R-6}Kz3ajJr{_}T
z?p&LCEp^g%&8<t4D#hP<2R<&~K7Clu+Dg@o)pnlU>?XC<>n^&q9*92~y2oy^y8pAk
z-|zdIpWc^$->#;%*0S`Kh_ZX%lh^C_A4@P$_;`O}guxS!UmeoCyH(@&+x=zIoZEHI
zlPlEsO-%aRb;;#6^=}{T{2|QET3COfVevMDf45Tp=S|Zrd)~3oZ`qamn%`sBbN-Q?
zyeZUK;&Szt1;3wpubkZ(+%=~<hbKAw?#*@oj?Ld(x%BYzm_4c9QJ2`?>a4uD&;8T(
z?CZ0*ZpznN@BMiF-j8Jq{;qhmw#`j*@4sKK&&;!(-O<s}vTPw}L04wS*|f)7ug8JT
zpLwEreb)3_|0b;Ce`535@&V6|R6`Z+TJHZoJSx*(r#)*?<rllz)4ASMozr5Wx-RG5
z6szZ@OV(C@dMPEc{r`{l`G%XVj)tzB;<z&F>yJdOHDS-*Y_p&FAx@8P%IQfp_bg>6
zw{O*T7kb}TsQ>KDg<H#7{1T*o?fTH!uWYE8CgwI_Rm00CSqH>&pC(MwE{nC=^+^7|
zr0w0_nro`uuDxd0atxa_H~N;&v-!3@cxRGZ&AY<a;V(Rm>s4R!{(rL2{e^(q(M{GB
zvxR0gy|c_aD`BlRSy*)UxkG<l<3XbY&*#_Adp6xtL!$$9LfPYEylLm=oSdD%Z{^%f
zBSV?{oB3bwdwr6v_QbEC$1QRuh9b+|FLRwV|K2g%^K!%?``E54UNiEJzqoH9GS8&K
zbwlAXeFhWxOP!C-wTUd$t30)9&*^)Tz0ND^ws`LlH#Ge5*1pQ}<m$y7JMYf@@b=~o
zk>#b+FMZp)G*wOa{;X5?KQf;=-tzzK7e`*nvQ12rZ|!s0wA*0LJGm)(aogW-=(@(^
zQIYemU3cb|mnDC$+qYegP7mDt+5E@8|Ey2%@I)lNX}%g!Z|^M8VA}Kexc>dpe|oN2
zA7{otnz^y@?W$#)oFy*T9$GfFdG<7h2g{c&_<jYl)U0#!W@FG2pI28_YIT81bJ6_$
zf6Xp0_g61p={DuHg=Dj{mG0!lwp+feO0f^9TDV8}?H*zNw!Zhq>O8##3E@c!CB-He
zE?n2Ni@0GvdHU)ZAAEjT_f5=BezaTM{KB5C+P-%f-I;4|?+Cgg*s;YRbFJmW9S^^4
z4BDu5Zi~)~hUKemm+TEbc~7P2@T%L)E35<5raL>m(4LWE@SVxEr{b9WN{P)@6XZKA
z($ijVJ)+&gdTPgqWO?&tUkW$#%jIwVbXu$W^7S=wZd^y@zo(p>d86jagtLuzKF&Y;
zBymZfw$FcMvyGcs-IZhG&YqfbFR!QKs$IpOrOn%C2(xYDklrp`zjJy?TE=M|o{R^l
zPS5*(D=kOICdbbrBdXM*^lYl_-kvA>YtKB(SP(L4!pr$fG+(rJ^nA88u}WPS)L)r$
zJ{+`)(<tdkhso|Wpe2^Rv(1iPyf{%*JM6-yRPVPQ!7K-gcTY3UP3AHCT#>fk#wq2{
zPp<M=U*D&mcX@mN;BoCjc~Rki=BJm{&j0ZJ+1hB!!oOjMO77PGxM+7S{!ZGiFY^y{
z7H~%#zG?BXxb1D<!Hqu?*tY42*0Y^*TX9xS<oO??`48&X2?`6Je7SaW#)WDfiz~^e
ze#mh?e0uEWWSRc@+7~ywIn}fC=5eQeFZ_M);j^WuKlV$ny!?1?%RRrfyN_L1<K_xl
z?=V&Bti-lYVztNkmll;VU&}uD=1$6e^<2v;rHwPwpNO72_I|?usPYDGLo=VBS%wS0
zdvngOzq4C+MZh}KwWq_Dy59f)AmEkmzE@#_4R`d#JUJFBxA{KUzy9~dhtrww?45r%
z$K>6&b+!MW{}AIn{lw#L)Sj|A_K&CAzI#-<PUUUs*K3((K3mOMb6<U4WcKs-hmRi<
z54Z6?IX%7q-d^w#t1C5>KAn-wF`Mn1>ha}ayFBP<Hk+<%3xoQ9O321udYis7>TlTz
zFTGbWJo#(4u}_rR8e6%zqqlMYT`!3jwJHALMfa5t=*$v5x#`D(hfi}K{r@xP<orc1
zuT3r9*b^%|JLkZcU03I3hF_bQqZyTR>`!*$vsKYKoeb-h`u;3m_*r(ZXzTpW#QC$8
zzwUj<6V|@X<>TEz>ASOc2H3}cD!%zpF|96E^l+WAh|bd^;ibV_HY)XrzD~-#aDnx@
zNdNg6az_vDQM3D4teAf3X4K;wn~mG#C-d}P)H*OVtW`L^q$uAc{{7~A-|w8R|DAI1
zuMbb{HM<p@>zL=4o_D&qJGU)OTeyCo>*0HOBBqCyGGDCOb9>gqeUqQ9$bK{T%9M`N
z?vnO(HM4df`gYd*zCp!@gip`UgU*fCh~2fNY>DSxt78kY)^EBr?NLj+v+c9@66yKB
zGIt-SKW)DB%cj^ZIyv`m=;z6JF8=@f>$=Tf{#WoNTTS2k-r0%&<L4C{|LjOg-*mm?
zcMVtm{#TRVG+o^p>;1Fv(R9w{1NXlbZ9gyZ>D%g$zc)#k@x4Cw-`9T9pW~Nq%sOAy
z{^9Bp+44N*?y^gpT*C6^-alJ^zq*<AxNx+F#gv@W<<9^6+fs`PKd(>yvE{GI&l=y{
znLF2LPM-ZOEqIcYq)%?-zSrJyC4EWHdjh}wdw(b8vT6N)>tE&hE!W@1p6^{;6S=y%
z_6E;^jXx8b_Pv|&<B#k*gI#m%55}KQ{q%OhnZ5DWfB#Dv=RBS}@zReX`)fXLeYYuZ
z{`m3(FWd3$KmHzw-X%1HOHok~G>HjXMRh&CelDl5T7robc*ecvS{tA2DbNXQeN9(a
z+^W`eQ#Th?6}8zcsWIcM9E*N@=i?^J()D`<1&^OpuXw7JZN{vcf8JWBZ}LU|w0%eS
z$ec)I(lP66z4h)^&E|&b%s*ovoSJ%N@A~D_nJty(oiPg2Oj79=ma$KLy2LU$FWFFV
z`c31FhJn}pKx+tgmA(CSXZp#kt)I?WzX#nhcWRpM=}D^I6%Sj*w>VeC-S$YaFjvZF
zRJ&8OR7T@(+<I2=?J+6k)sxuog&kTS|9Bd!!OjV%m971g-d#IUu<mZ$Gnt(y-sSAM
z-OaJLjQK>@^y|&mhdX!5dG*W|zd3jQkx#8f?;h=+QB)OtKzp^>iIA?24g~{)37~b9
zdn${cotf!8O(!x0v=E^t3^t|z&&hJto{dc4DSZ#ABJDReyVXtA%>KwfJ3rrlwps4H
z+nYd#RILnF2kkJpyQ_4jaXR14&FR;@geJD9nazHAd%J$|voi~;zP?%$vvbm&oyDsS
zls<*Load16>B-42@9)c-pBD;ww>N?@K!vxcP~Tx{oOfZ#Y2V|0vXPt9&K5`ZWnW+S
z@cHxOp!Hw(_t|<+(+Ok{ndtuP#fuYDG=n7!5*T)uzdv^U`t-}o{lP}3{?cwNes%_Q
zZnOAzr;zu%*`B}2I;T2mnrn+p;(LEz!(*<(A|ftxtxD(Jte$UQFIQh*-zRT>uKNAn
z<L38k7Aq>aJn5Wik}1T@#&h8EWzeC}0V{)4!OMB(x7GaoT>SIY{wpuGGl%y(-BNP7
z)F}P@`{vije}UFQ=|*p}+5Els``vQTax@96l8jwa`vY7q^)L0F4w`<`UcYD2Mi$p6
zD^Gqo>!_@(d~)9UMNA7ktR^jXNR25HQ!q06G;`nkv$MY$z6Pzin5v-U?|OQg?#a2<
z=7NHPT5XWS1v)x9IzR^;fN1b(105Y5C+^M&VLEuxp>BG$y=>WOw<pC?Eqf{)m3`xl
zHr5yOL@#1=XWw<T<jCQS<kR0GzPa0<w6Zx`X|MRSYvzh~FC9W=Ouc!0&8;$5xdT$d
znesdSuG`=6|NOm~?(TtW{{No&a^@M8$t*$&NeYhJ1d=(lthKBc<;1x>{3jAPLAy&w
zYsG>^F)pluA0M&BGdczc2)c$%lx=nBh}*%an8+)j;Ly<GnPznP@_zd&qsfYi_n+3D
zJ8Ajy6$q@F_bX)Y*J+}&U3a=kTA%xsnr-=tYkA%4-yNG`{zX4|^G`mSe~ya9tFl+e
zrcOSkz|o}8!6EYJNSL|tqR0!8k9UOjD!YeVTVfG8qjah4DXABKCrqu@7Arkx@@?jv
zs<`Z#nUSY?97L}kme;;JeZ9=Ogj>_umHW4B3SzbRk<Mtm<Q3N;%bOcN*zV%_z4A)&
z+WWK49`8REKe=Y^t!*=w$5*j8<Ryz6tU3F%%&%;Y<=lA>lF$0iKdHS^JJeBt#gXGh
zD3`hI%-dX*Sxs}l9=EBxSbpg27gyFZ|2I$gyHiT-=z+DOz9nxnk}Do-+ZL_cm|CrP
zlKqcytgG-uzV`Trd8c->7-pFTSXvmo*tE2^m7_cK!tPGFu;nLp7AVZ&-)hybqMOUR
zc$fN*LruH-5`He*lC8$T=yP!4zW@21Tc-9bu-a&?Z&&-h@odeSorW`ZeQ=PS$tuF)
zDB$;7c9N1nSIc$&%si>8bQbR7&}nI_Cag57c2@lQD(XeJ|9sWmx+hQBm*1K>v-WAx
z=DgWnv!a&<E_FCKbMN8_{sAj<E=0_Fxbwsn-aS8N^q%<t%s6NAbED<8PcvllKXH{z
zz4|jKnf2kH&IOgn&l%KeGx+BFD|qePX1OUW`}n8ItrlJyTUSl9f4gRKrT)Fc2k&`T
zs;zwEx^(`|hv5v5<g;B(yB$|GC~!0hG@N1K`se5QvH7x<hjj4VCC{?#1V!s#PhD>%
zm@cu%<L*Ku^TUUaz7x2<XXjL_unnuH+rQPm8S;MRn*CG$-j2DR_>bkG)$x!sPxgMO
zGWZ(Su}aF_>FSHi_S;kbed2VUZrx!JyC%r?Tvz2Uwa0JeJ)9<P*XL+bnDcM5hO5FH
z{oDPOE2P(zt1IOPGSqHcleWgm*GA_|OWH>_tr<Kj%Xi+4FIjlgVe>0Dt%#l-bEH;U
z-+iNh^<t0X{(DI(_Wr5&BeT+zOj}Pp)>_>^-9}t0ZAqqRtANR-V%wavUka{Nuk%dS
z_P=^*{=%4+iCl?N7AAc=&Z*}z_hw6UZjvh4Y3ld1Dx$XS>%`8hB1w1U#L}{kxM<#~
zT`jmdIQgmBc76S&OS8{^-|%(;TgDsxXEW4v!WMQ(9yt8#b-}bCfq(BTiupH{|1UI{
zc|gYJ=nIKM=U<#T>Ur(mT$!`Q$y?n5_Ia61OfPVlyjfqidS=#<me=xonD_fGyt_b?
z;f>{-=FMG#Ys|Xz%Acid{C<J+&a#A4;XQ|zHC#>O7N~m1JX>=2LV@N>ebrf0Cr(#~
z<T$-~H#qitUc4yN>;8W8g$=VhKZP-$lQHL=J!{3I*jty3j(Og9)w^Y~?Ndcig6Zo!
zKFX<;jPpf>jW?Q};mJ)^{Gs!Gy};57E4R(GUOj`u`0%_ZHDU7KSvppx%Wt~0;?c{w
zy1i3ES<f6kV|?^ekmA8pGgur-ji059_nw<n_SVI?_PyrJ{c(Czk9NzRIdh4lH{eBW
z|I8g4!is4H@`b@~^_0JA{Yv+>HVKh5-`Zeb^XS}>-TYCU%yam7yF>MppD%R!bw>9~
zY<LP+*Ni5vL!r+s_V>siUf^G^SGer>2~D0q?zUaoj>q?H-E;L_q*z8{v#IR-ig~-U
z?s_Gj(_Cei)v~Rc_j6i*vSU~G!TbM?1$(AiZmng@o&Wbh`PnP~-p=IG*uMYZt@qZS
z_rFwrVVk9QY{xH?$NXVSRwt}E-%~nyW$x$t`Q1K_&2stL%N8^#a5Qx|JY=#s8)9T;
z*HL|1s&<i<`Wcto7Q$)=cq5d2Lkcr?7$z_^HSw6_%bj_=Riubt#OU30CqK6t5?4-o
zxgK@+JMExN;iYq1ayr&Z9_g6oxGUmw*Wncz!s~4kmhQQ!|3NN4RrO-pf)h@sWGA>Q
zJJ07?I8C%KUF_4jMNWOQMD1RP`c(2<$xl-)En<BCWc8je3#KL{oS)C9_HSv>wUh;C
znGdG>+}<x2lzewRtIv(6iY8u(70(k+%@&n=A;DySYv%i=+gdX@K2Gag+0<E;@#f~k
zLjtLbymm0I;rjpHPPl$eAv4>rt=oj!j%}(eTbAmWv8B^ccCAFH?t`0BRlbhj-xPFf
zu26_E?K4;z9(^oNV@c`%^Oei}gL78o%(S1D=e+F0S84x=g@3g7y@`zPuY08*%Mdj8
z`*YvLu~!4kYQJ0c@3);^dD$*muHj=`)t&uwp3If@{La_HI<sV<Sk&XhYkDisz0R%P
z?DW|n<C4=ZwP{CJYGp(}yHI^4M(XSTNu9M)%v#dR_nrw(=JNloXn*egXEDYL`$J_7
zSw&eK1-x9Zv2ZP2)FPEBle;8xPv=2?cJ>^_jp>V<{lhI@*zRm#y6@3+Y|F~BxEbLN
zpT0Sz<$b)cb{f;FQ@=lR-+b7ld^69^zwoC^Y)})Y!KW?eZ=0iK_uV}BM#udB_LV!%
zt~hqw_|2c!(tqbIzr1H^YgAal{DP0i7CGI0a%t(O+l>A1>!z-evhK?eS<6=Ry!yuc
z<rB;96d&8ddywnyUcN%UyfTBt>GSTb%$XYg%yMt0WW|Fz?Q4&<^nEN`XC01K)0_J8
z@*AG`XHV8jvcCDwb^3!*Rr`5?Wdg57cj~?G^-tY7bN<VNLQmxP^7rbcas8TJ!7AFa
z`sli!!V<d|3MALW$GyKSa`%Gc!#+O!Pp<E+86v(X`is3Qd-1XUZyW0@_Ta;LccMeT
z|BGCHo7+Zyx%n5jzx$PPKUc*4OfV`=JF}?u;g<OcN1ZBCEEMlKdq=RCMO#dKX!h;j
z#6|VyO<ivUMPAwdlKHxoneo2v_lIZCXd1ElimZochYc>XUS2kAnRj-LO4x<tlRo>e
z`15J?+=naU6d!$>@bqftPj;?IwmGMQB=>KcaCoBH-IAa{iF>DyvAn<E;kIw3t=64I
z0@0UUL|w%zrT<H76v%wLbp7^D@0G`NS5>8-+Qwq#J9GD`llA;Netz~>@BV!5@15n#
zWrOd!M6TzEEx)ymXP%Du?4rHwX02(7yA*Vk{x6ul^k<!5^`Yo6*OeK3yy+7=-UvCZ
zpY_XF=T?ZvDZl@FXQ{k0UYEqB#St!Styx^hvLN;OvJj4tY?Hbl@%t95W|zO1?=t=S
z>{RV{(b|)C&w6Rxs;OB&F<NKpgyLrh^Mj<sLeFVi?KV#OB~=u1AgcGdJMV%`TW4s#
zxD|iWK{z<TQGn&)nhwhq0aiQp>>}PA5@NG3G3sZx^4DDCz<NgY@M`y$kGanYYe^f0
zo;jK|YxN7E?d@!#XYCw!>b`%pDT~9n;_lw;KU>WUWc#Kq|ManD|AJi?^a9_`*xWyP
z&F#m_@+NT_6jX~UFUdUmyEFax(ofF~`E^f8IvQTul4`*FzU1@SB=-5KKUwo8YQC^D
z1f`SqWcm6fJI?)iELqKd{`kzFI(Ltr;?2Av?|)?N%i^MUqWd?VzIS%FpTDA(nb&`n
zBVmWiCf@Aax0%&+rYD=!W;fg8YrfC^d~W*s4*|c$jwb!|b^P6-xk7=XsX#16EkI-H
zA7QymebpXkzf71FyNkhQ_r8lqCp-$8xn<pikJ*3fCwKjef4<D&<8_zQN7JtLp4Zu2
zbaZv}Qse&kS<BC@+{}HW;*h?K?nJdJTd`9Utf~%*n6t)$irRwG{mtpN(tfV*8G6(7
zU07G$sqC*iwRzHipZLdfzn)JH{-n?FrRKq-)|oG-srk${p7zVtIdb0AJ^xCq-$_|b
zR(73#C~4UO-6J9)0xXUKf*LUf(=$14imVKH%=3Jfs#b``s>ZH~s|3ZXh5t|amb-Cj
z`W=C^C2k*<+H^LyXr*WCNmXywj}ZvtyZqyewe@^8YfU>rsrx6q*1t$vwqPeX@pK2j
z^q#rL;&A%9okh;KBGVtfzP^8Mfx?zB@w%B70u4|6e-)Ll53l_C@VCUa1r2@8cFV(?
zKZ(y(znZ6RvCcGpsoulGb#rSTuB%jjX`N@PRTmskEDkBjTv)iShxlmO++Ve1Z$^wP
zx9$6lLF;CmSS2p6@j&M2KlABHJKlVL@bcL1ZG1{uGhT|6MY(J~G~>8ah0(c)Uti}~
zzMaBTzhu=_@3cj3tox5&S-kw*cJpu*A77sOGj7_(Id`5c`p;tRZ4??`>Ef<c7kr@v
zoJ<ZErj<#ac|J3B$NIf_^B=^0<i1_(AHR6c>3O2NI$c7Jdc?0dxkS2#?^lY_|L;eb
zZ7$R^eV(TNIqv+jbQ4xytHaAa96h}K?3=U2#pO*;!*1FA){~xI<NMv>@9Zi2D*4Nb
zYL_iIzfByRKO9{a6zrS9quO%nreVOnsH#6#qyOlIDP=`e{I;9CojGNJZ?g7!Q=`W=
zPaZ86*?wN~^jgobc^%)DPkdnf)+Au}6v=%rB-U$&HJ)O-uQO+2`|j2cF21bN0mj#k
ztWa;(>_62%+lFy-+x9z)mdEdIb6ULN(&cma9;U7Tep5&EY)L0a&A%|yuUCr7cSlT#
z-*rR#;zsUo(>6yRZf!ia^^eo1+pc_<x~}B6oJ{AMHA!#Tf~Eq^6@9@+uZF&yIEVdb
z;TP?iH9IR;RtojXzg*s49rUB>-m-=j>gH=xCM`<}xOMS!V-|x&neL*i!S5Fx+3z%6
ztT*GvB39iflO<Up8Jgyw>o5QN@p^;(jkQ0|1pL2M&C@XF?#m}L4oBF(Ex(^I!Sq)i
zTm6S^|92g(|K;QOz8ex=4`uGwX)-Y?n4CHBxct{@XOAhJWr2BOY1!#}mZkBwDm`Ad
zU=vsQjM$uy>txTIF8O%d(D?hb%dCDrO;gVc{rSzYXjRSD4FYFbPxH^*xqnfhc&Kcy
zO3JDyY|<-358AsGMlViScIRc<pQ`@+!)4x*%2ONv9_U%>CuJ|@BhiqwB=mGb)Yc>F
z>#qHMb;Ug6{ACMH7G}Y%I(uT(I%aga{rh7&KiSbsGqd_fLg!?Dxh)9?9gd$nDDo%b
zz@cEad2c0rN{tiBjT1ZnJy;mX6}sn;#ga8nyfJSa%-wZweZC;F@`8!PwTkZD$85v?
z%?w_UZy~=l;O465J2&)POy}CA<Ns=fLd=|B!G*WJfBd|paNmNd*>#VW8}Vw&GwGda
znGyTu?xaIK{mUjcwmD`VkG*<2;M}#j`t$zJeQp<h`qKAZ#)`k#tG2H=y7KPoip9UH
z8TMGd4b7_Z)i3|IF8|WHyz2RCo$uLGl=arxmEXJH8R!3}`IH&6`uBGVv6uI+zgf+n
zuzZ(YfLPSDEQtul8uJVu(YTJMm;3iEi@z_n;`Y7gha<Ot&bt09BH-pGPw{2E?>ry9
zxb<^a_ys-P&dr^WLb8l!*0N0VV#X=G9S0-|*}wkQ*^_x%VO5mC`|X@sUt{W)P2Sh-
zRn?k#&t}H*tf_P6-rl~mA@TJewO;?S4VM-LCG5H5thY?3<Wsx;%}b9mWvm!QPfHb8
z+x(gO&S|BTi_?vlbEWz3`<_k@v}<vjS+TfzYS+=V{;_h?bafM=mhGSQ|M9<se}Px)
z*UL(CTZA3<I=m^}&syu&k%nnvTJx{%w3-xoqFzjNlYB=nKi3WajT*f__lsRx)jRdU
z<wMKsUoWWKE}+HKcr8Khnr^w!Rh=2HxMcopZdfRtqV}OAtlvvzVH9gnaPazDw$<O>
zoSdQ=eB#upMLsi)J_+S9@Tu2(uD<tw=Ys4rviBcv|87;aR(wuQl)=>g_`fY8f9?w|
zO)TFhbFZJx@Mzjkzoipytau)I*=(!SvE3K#Kb+dKw$c9BGQBgA_H$;5F-O<Qcy3QL
zu6^F-7wh)<&-={V<&qhajMqzRGG&~rO;g?-{%&ZscY@vI@Js)NuE}v-XAfN!qG)F~
z&ouj*M<Pd)f>MCXgI>>!8I4hKAIm=NXqRP8U(s`r?Hz})Kw7J^yUiO7g9#F>nK#A5
zT$7c0Tv(egubunx)(j2hskPrN|8LQ5k?i<tclx2S+_&9&{*s{_#s%Ip7`Zo?pFSM8
zAkVZlx=EpS`6<5XSCj=C4{bf_nRm`pCa?Bxd`ikm(Yo1by-z+08cUjFD#qq*FZ#4(
zZWGhqk5->H-Q1F9wpe|B%CTK1@4Zz%*JG<bdomw~uDI)tr(bGr6z;e?Bj;(y7R?5)
zZ~J`wL}sjrux%@d6wpu#5MX&I(_y)y;Hc%)<r8O>=P521%~&ET(!YG-wAzQ?Z<pUW
zyzlYzj>&Hm@4MQZo@mpnTsgh|&XO%9!R064%vXr9kKpL<QToG^?w>d7;A7>qqU!(a
zU)(ehYxvM`qVUQ--_$QpG-sT9JpH<uWGS1wk=P++-Q`c66}6UW1%8^u;waGa*+u3s
zN0S1_A=ORwpS>8i)f&#{{p~le?3!oTFRwO}_x)dgrtgT|mJh0W*BKfKYk(9__jtNQ
z?ZEqtck(a3KH!&l{OYpaJF^q^$NoG$&6siJv8~v*Je^imNS_sCiS%5f(s?O2*JSLI
zvY-6&^!3M{{)u%}ToPHAK6ZvzF3Glj6*hIPG>fAEi(n(DXZB(H!)YyMJd)Sj6jurF
zKFm8~!!4ang}XuHPM{ET6!>>dLoCOvdrC8??Q&PCFA<~{G#Dh1r<9@=up=YqAe+*;
zfaW=Ty~^R)`PQ2ok6jgg*J)~y+S$F>+>3EmuT9bPPNOcjr3D#su(bf6+{B|(^c^0u
zxomyDq4<4X-}9Y}9DLmiYJPUz5fbfdntS&3_Q0KSu}bTmXP=l;`!Xfv<?oX%u15p*
zW|=&<uU|fC_SSCG{Y!(NFDwh$$0KKdb&hS$owp0)?q*#-(Q@VFyDR^luI>uyHHn?Q
z<?7>{dplZV!VNfk*PpLa2B+#BnVj9G#@`%ITWU>kd&kKiy-n08@v`WR&*#>Bc%K)x
zN+fsJnpIPt9usUV^h~q5lgbt{>3S!VNS?vR>5~s`kFPuSBGcR3Hoqs^ar%;)eg75P
zB~KQ2Y>9Qhtz_~2^W7EGD}Mi8+gQ2oYOB7T<SE;W*EV;}G@b1J;f{OpUh%8xZokVy
z_OZP7H!ZjKPOm?@pnsopXML;b=GAix?GI^xF+1rKd+tejTw2+#*$40cbGv$C({+`%
zlk3kt+qnFH^T9jQL+h_iZBv7IWZsF>B^RrkX5X9JywlukyH?rrT{Ab|Yt=q-LiVcf
zoyb7NJ!_mEyB)j}d-v$;;yKeJ7gR?4zwacowRweT9z%cIV>YeBVWwZ6P2YX{tk}w+
zL%c4}@`E$JzhU3^_}}8?7j{<Wo{IFf@RLjVduY9c!7J;1{_M?{-dX)#Xn8{X|9a*5
zOLrR_ySlvcX3Vtw8|&%=xBWk`nP1Ex`)x*^{o)MMz`ys7ONytY9$4|7;kK&H{ldDR
zNi5BFzpsiXX-JoZrZf64(|_8lr@ztZ^N&MKK5f}kwt2lWsts89_`^0AuV7&o#~n#0
zW;r|*y71#?ect`Rhn!PecZbyZ7JvMBhpFn{3x)I5=YkIUygza}=5CA>_vM%+a-Z)M
zsHSs-d1z!am~2~eQfEQI;{uDH-t!quG~UkGlcHRE>$>u(G{-=ZoQYF2Z`Mle|8LMX
zOX7l!q>SClIP;(O8~#Y!d6$1Y<8J@q`JD9<=0DjdO|LntT)1n><twc|5j+0sXll3|
zSG}<1<lO0s=8kuwe$Us5+}j~8%@RBLQ$SPx!r$fH5f+6Xw))@R*d4#@KWp2pJNI{V
znV5ky+@Y^sS4A$Eb4}^EnJ1S1i#6)%?R95<Y}*;J>6-MCJ6UQoIG776Bc`c-_1NK9
zkn-W;e!Gvdeg869c+VwRMJ~J6T=VPU<OQh{N)Oe|wP~39^>TVb?K5w-JMpK#pXBP@
zH+%cOA9H3^q_5<<`&^GJ{9)>e{@x|k(n7M5o+&T3{<P%lUFmzc#mmLDCHC_c|7B}a
z8=pC5-k7<2@sR*e0hWheO7W2oyB-B?Ek4%S8MedvQio*k8=0wBBURV06i8iCH$`x>
z^f&o1%QtP0E2Wl~>-sY+6m*LU4KmQ4P<!F*t2aTP63i~_=$hzdrmPqGxL})Ck>*FC
z)tX0k+3K9RvLf1f;i|I^b^DGzv(@_TJ^O&MbVk@+57utM>A}jS%XeR?@Xda4_xzo|
zd1W4^f_VaEy0cC(t<Rjl^Ktc-RaM^-r+cma=g58dx`@z?ZF6?A-`xG*Vunuw-?F3<
zo`*}<t}K2wVc%zssX~u7tStNX_Ri_T_Sqe$3Xix%?ah4p$mXrlQL#Q>#t`eB-xfuB
zuto=n%?_UY<4?1jYF_G&rFyneQ4-tk6uL3))Q{c2-*l1C(>u?yzleow`u_IGg+lgg
z8#qo)?^ZJUb7}ixi`{QZC(VvtG-YGNftMlP(|)rqeOtfo>}fHk3;U9rSMN~^5@2z3
zS)dT(#&bpMx%}O8wvrj!GC6`P9xqH-b(?AZujhi|@@EUHj!tY;w~ojR^kfWO%)oq*
ztKrz5{I`E5guglDlyUin*X<>buHJH*eU`6n?dNjmS*F`QxlG~|eo&Gbapu>H`suQK
zadFR<IZbg|ek(02+%WJO&yCPo3j%gV*zJClcymw9GSS&bvoGH`_1Lq~cU7@*@06#k
zHPar}mpN~6n!Cx(A^Eh~_7LHRYzNOR6YZ{bXa0Y`_W8FZnX67|-}gBlwEBeLiNZQb
z`!>h<|6-J~`X>fU{OGTW2)McRpLku|A8z6O@oa6gdj2dje?3q7mGA<F=H+^iSMEyv
z`E~I%o|$vyCtR~Go>Zx!Cvv>8)FktJlYLE`(P8u2(zY|tiZb@=WjHl4-Tt`p?rMo!
zE9(#3+LnFr<d>;?te>!oEn%-SkNW<fU-{RTT}_8Hr|;CaIq-tFPBFv4{;c_BVaeNX
zRfF~@{r{H!@9-DKwL3ICU)?zRM=AKiFViJEIhzz>5|7A)?5Gim{w6Gy_T*;!nk&5e
zMNAJ8eX}-lq_rCMvj*RNmiTr3PaD&N=lrer$Zp_DYqMFizQ5C#XOXKa>+^|D^JZl{
z-`Tl5qR8QW$J7}Kn`dUsPEj&H?)#~1dw!XMnr27XJ;&cGvK+to7$rI-f87)OWoFLv
zv!S~p!&{}qs$89?N(l3=s$x}W@LjmcH~Bu})Wv%Bmou*#wk*46adtNQ=ZZ<4cbHpy
z=G1geuP-U^xayh%8rKwjdOq!}#eU5f`7>u-k$!mWKyAxg?LTYvu2k72cW@bcdb(&b
zxX-w<TC~~j@62m|?bB<cq%t#S|H!)%%uw_9Sn!wMTC5FCYqsuUV7Xva@%U!A<*izU
z#qRuz_FYf;e0aI~=J>4_5`VC%+1)=<e*Ajc+Q$ob1Td`GxHj^ut3Si9nq4#6SGCQv
zT)q6;=7ZYnq^|8gE59dOu-D4&JePCqv2(xYZrNfIH2wU7>gLtf!WBiAbKX_%kmoM9
zJ#yXKY|^x8#|{-%-PHY?q97dhZhuqx`UAp~nNRP${JVcriM#Z#FMbRU*5*(D`1dH6
z!-FSf`!zFyw)FUOgQv9eB%G4Y^1M!X`dB@;=2rwquU6IDCK;{$I$95&CpCZlpOL-L
zW`p|8gHK}R93C%E<@;7ux@cun=d!dNyE9}H4gXps&lFY{2vTB~vNjVlcu>v$INa)o
z!Of?upM5l}3f<oJ!}jvrFXxof-i7St%i(cxm%2IQ)spj9n7F!r?c14m%x8mfP90;j
z`fbn6zi0a&cjIc~5WbtGxnR{E`~4GS^(-d6efD|EqTkXUULVS{W*z(O>v&XqMZw_*
z8V?%Ej~!pI<#C_ZHtt<D*Z*+#+zse3+^4_(duG>JZK3xsJ}AY1*~i>aD*r9zVV-z-
ziQLKh`hVNs-*S8`ZuIi%^341D)a89-7Hls&&)8@E<&Vj$pF(eWtEB%Yd~16&pSx^%
z{_%vWIqtK<?W+HU=_#=CdiVdoa4`K@)tvWi`<Hc}D9pdSEK8C>Cva=!y8QdA!@r%g
z?Rjg*#AEt1<DtR3Z|e^C<bG)I7g_#h-=*1)HB74B-M7iTus{CuG5_R~`tcX5ABkD*
zHJ<H0Id5mnmQXYOZ?l`a!UG%yTD1FlwsTJBpWb+Cst=>`!PVk7XT~ILJMFQztE&C&
zHn)R5A97h!4=noe_)Z+-w)vOVZw=u|%3)^~t}ooQ)FJgLpZLq=`gaRg^yr)2t~m5=
ze|}52;j@(0GNm&(3U@?46>Z$QTH5li{``Ya;*SSaIO(cgGqH<#^yKupUviy-qM|}`
z-I^FhS|9Gs*~gbLai*q2YqUttj+5fS0T1&Vo}E;9zJ%-f+dYT<PR)^L%(%HzQ`Apb
zyjT98xX+cdX4BW)b=<wG(_&Au2HR5E@7MZ29^K~RGVKrJ=3}2<aQ8+Sl<Br^UcUKd
z>$E$vc?|As_v`2QtTtyacztu;^9Zhd{-1o!E4hu?w|%kfDSY#*cjHSv1GD9of4v%a
z^6jn)divylm|^tigsdyQZAa5jItl;$zT~3L*$)>s1wRTsadPdS>~FVyUVW09_xApW
zO9}?%HO-3rN8=wcG|y<_f+X~aPyAw+%2&4gSFr{^;eK~c#W;nn>Y{Orq{j2u>)Vre
zcP~EL_BJ-pU^W9QUt&V?uG#Olio_O#8htL(7ykeI^2zqoULVgaXQ^VhEpnLuQ2X8V
zcOlzm9}F)2cy(E0t6%v{%jNfX1?`#lt|W3rDqHXjr@I=lZ!G;+<eUik`bXTBKY^{F
znuASCbY|14BrdHx2{W2CpJ%g3zOl`EK5vE5^P4B{AD7omzsGe{1a+YEJGYeY>EmzP
zif>BZ`{@uav0KbW^Gw{v?%ndM&Tag^E`{;x%nPsjHpk37?*G58PsHTdzaK`Xy8jN&
z-aGr+>-i18%7b1?<(2LKzuuR@B!6N=(%tx+!)0HxD!zUaidrok92aviIQi>dUisG-
z{!F^ABsQ~2>hW#RL<xAdsLh!%Mk_f?)M(L6yX|6enjPPzR`bM$`@G!wcXLklb}=K}
z-#MMTBsOxz<)?l8#>jhR-|16REX*G-v;Of|X_I8|*&_#@g`JxHup&)SIoOuxP0mv%
znI*eJ*6#{7mAU609HTY)+T6Rjf9$(_9QivmR}?fSt~~Hox9PxZ+s`3!i~-N~=%2TL
zeX~F-EKK2UYnr~x(?i#_W@wo{dpj+xO}E@2|A#(L-3tEf{c{_39*(}!Z8Nzc*mIsq
zN8)MOJ;txgZ_U}4c3~<1l?-+6oztvkXXzYdSaWZuyn1KEy28+o^});+TpwINUw?G*
zO6Q$FWE=i0TOhmZE@=E#fg_wVQdlFV%xJslr?Y(Z(_da&lE8M&WSY^hgauVTFGDW~
zEc_7QtGqCgtuMFu<D2V0b8LdvNT(TJ{2s`?jr&aWTPv}9dC5!nL@K9N+Ue_QrKHa*
zySP~KT(Vr3%#>M5qUTahT+s<sG+c8+L}XjOU~s@gx5I8Vf0ylB?0@`>`g5jrdQpdW
z7dcC%-!|P)_xHAbQQAqj%W{89FFyM>)z$l%VfxO=jr}Hei!*MSyiKj@k3AiIf8C>n
zEymnY{#9J<MeF~b*_(ZCzD$U>c#x&#&Tw_%S9NOptS^?Y*t!4V)6Um=c6*hA0~Q>$
z1&sxR#us)h{8>7)`ASmlq}@?bFGEg++|K>FW(%KYr0)F}LO#z~TCeFg|9nwr9M#zF
z|6Jt5DW&%FHCIH^cw(LxpIcbPyf(Yq_LpVIzR!8L7wkM=F0Wm0^>lC9@}0}OcJ_pe
z%*zkGl$;O)9tytnpltoim8&)d`8c*pPq3O>{ZnRQ#kSfrZ@#vlt310-^!{dpyT@wZ
zmTEo&4dHrotcv+oVcYK(y64Bv1x2ON7w;}eSg~>P_qeL&-e+-J_HC|NS-AcD|Ayo4
z`VY6}hkwin+SyQi>tQI5goUvC#a(g!T-yK3Efo4V>!Zu#A7(mjw!ERf<6r*2Z9>oa
zb>zJ_U%OxUG<AX3r~kZZNyl=1A~wwll>Jik_v^|1Gv?O4S`=invv%=ykuR*Ty1?UM
z9S#qV9Vm1YT=wzg8zw%{)mi-ZQfFiv!WL}$+47A0{lD`GQSL9?MU_O_O8FP=TC}lZ
zD|fhJM3S$_&#$M?m3)3bHQiw67O53sKFzK7vcAl$5G%d3##PNbtvUbNy11rA)@7Fd
zS=UX21K&#;y)QRk-F^0z>y-^3zSXL|=ULil*1LAkwG%y`CvYdPxp(K&QIU`xft<c6
zH&&?Ewm+`2KmL6s`?&?;eVc18b(g&QpI`P&D&vob{rm;Ls_rdwP~4EII@v_D`BL9J
z{*cSA(<03^%_W?e#Pojr-#WSY*T1K2r%L253ZAt-aQW3%vk<+bK8D3VmzPKyu@)Z}
zG~qj)`uf`1d;3c+ng2gE;o4uzlJwF^v$ic>7XHPrB2Ptn)8Aa%>ilKjE5j!3dVWJy
zEN3IT&e1&!vo3GjSM%r3FXf&YQ+@h^Sxu+=ezsruwn6jD|3!~g*PhkZ%5P8zez|R~
zy7}2f-_tX<)P2r<*V(t|@9Fu!*;xPGee3-#zTlSD?C?J<4&IR21woBF3V#n8DBHgK
zw0hlyXkQzV+z;OqW3=Au6#YuSd4@~BeETJnLkG5)wlY4vySvS3<Mv*|*so_F^XHWr
zeBN<=W!EBu7Y7x(j(wJ2v8-Ha&he14ga=`(4~EapzW;3frSkAk<z}a*TtD1>?fpUx
zo7>tg{qwh6Tpu`LJ$KmRK!f04H!61AxU^t`mss2VbS1+x4v(!IdDbkq|Ge$~+%;v6
zam%{z_x(&~NfugDbM5%CBX(xBpDRM2u70Z3`SgoT&dqgRPX+TLR+UPeIWN!qq3>(T
zhxh8YFQuM1{%6*%*4oI(H{SP8eTo!*5wl#A`Pjcx7gp}S@c-VaDXXu~)!*~`jBf3E
zYvHgQcUHf5aYv=^-e-PrHgkQI?B*3Y@w3<5KQ!6#m*4(}nb#Vd?F@e&zgDaIDSUOR
zc<c;`tWO!6C-Eg3U9a)n&G`7M|CK%FFK=vAj@@1M^`GsT!~J_tFs-`0{cWtoDbt5v
z#iM^8iCZ7mZ<Kef|BTdsskd=b@4Y`aAD>nH{J_j~(_&lcbIC7rq89S!Rq{P4I3i~e
zoTU*ye`3D<6YF`Vi_CprUsK%ZCoCs#)Ofr4PrJHw*q)Z{uTp-y?KiNmcu>$O(YI#n
zo{N8Nd6pfo{Cd$PeqUSft7ZGuudBpW)XQ1EVqNm~#?eilyEHR?yKVP;EX3ltqf5fX
zclXM-mrl94B=qcP{#7gHmAiu>>2Y(-xp+VI^6iruz5mQ(cx}4j())D|#`Eo`KR0$Z
z{S@V`oyyGCX1hei`lpWXdry<hoToEYSUF1GOsGC_bjMD;d9P%Hl9tq;pU>A9x;1Ef
zPC`Z9vdQ6QnLRHvzMlWHQZ8(LQPRrfg$u)%Je4o{cumMW{~L$i-N>F-iFdWwg^g^E
z&K3d>aZd_Tnjb7?Wf^t)P)GHdhEp>*S&f7%BqMCKCbndWdieV8&HDc4=4SQaWj>19
z+TBK}r|w*N`u@zn?#OuGgfCl`E}fTuHCf_sRr$iSC7KQbh!JLcTd%|KYApB2^Cmel
zs-JfMo^W8xfv`)tUwq`w%sdmkcsfhle*5W(Z&!)uFVK;`G~-}jqDuW8!vfm{yXFU$
zZ%f}*_N`H(`1to9Z};}yW7JhYdHMeG?<?N_5$yT;^LjE@yr2KFu+#6py!fT?{q9tc
zC2yVfE|y8W)xa*X_fF)ao=YEC#r$8zo_P|Rdf9*e=O5C-Opm@+eAZ?wTU&qeTvA`|
zpJ{3Sd!t|Nbeufx;^v3C7Tf;&?A2TU^U&}9c%S*z+K*4foGN}B$5S!=*pGdGjCSj9
zpXYw7zfEs>;P!HZ8NWBZI=|0IIHE@@bd`X-{P|g?*-H#qKogxxAKrWRYAC1rG(;yy
zZ!2rNSi)?&h-Zpnqw#s?sC%lR)k2GTA6)Y|eU0gmPG0Yo9A4)Qhb-1}oVoip<pr1d
zM^T>VCzM;-Bv*(o<Gwcexa}Ja4KDLbD<(A_is4_qImLEC$ASI)t1h2AIaz(<?YSa5
zViNe4)r5SpWSKbm&YZ~>cM^<lcJyqCy=U`O?umhD2jBjl>@#!9ir$pTKt{HY`uXuq
zw|)P-&cfRA_?sPuiQ2-QXN#Dx%T}AsWcN$ZlJ@lpSz)3E8q$q%O%dHzaQ5cJx0d}8
zQr4UFr&nC^&{jO+#r@fAlEQ}FCPC+&_sl(Y+2ibGFV629N@nlLe)zsUX``TI>h1%P
zGTaX*22S6z|BmEmkvi>*)2#!$dp*1h52h<$`h8(}neL;l6T3M&xh%`Ae*P4hvcFw%
zefi^S-8aJ=FP`3g#Qa&-vxh}{)uTm1A1_)W$O&%#w!9XIJfzNX=w_XVZ`!xqh(^VA
z&vq(jTc3!i)z;s)sxj$D^i%L~vY-Zsxyd#TW>;;wW~bheHsh*&!YmJ;9h15}OaAh|
z^2^WRs&stcZT{rbv_Er((%GIj$!g}8Pd2#fX)1sffX83{%#GN6%<<T<AfF|79$jz+
zji`g8nWNsjPo(34Fk}q--@FrPA3=lQpwVs)B_`H~RtIwy6z=If_<gpf{!&E_r_2?8
zZ_0j4{+nYl_j$p`)pF^7C$Z{8Y-r1Qe)W9axp!+*o>=HT{8)T*U!5ZNb$N^VH=d-W
zZb-Ps^v(L{BhZS5G}kkZt*=A=X#Uwfd#aY%`&%}lOLg@6yiS7F0#1&1vpcxp{q}jL
z(+t<y^_yg$Wai(t*fC~jL-F!$y3bxtzsI=X_6=5^?BJ(b&;Bd?zy9+7$Jq9XD*LZb
z<i7d)?!>Lrw*UH?>bOg~;h1y!rFDr*ji+6nbyB(F-Qu11m{y+e`Sx~2&O|8{j-~=*
zCF90JrCse`|LdkyuPa}2Yw^PgPJ%28S;TJbGwt4T<>;Opht_9Q^~^c6$hCWhakf_e
z_t#zOwylxzeao&NJ}a>~A=-g!*@J-l|F^r`T9@^%KR4!3iSJR<dZX8y?uwVJwcMiR
zzwe)6|B;uEcc!1;+4<_b-MJl#clR7OlVlZx1pIYLz24SLy`cWeB|0pQ7W}#qGsSe}
zzsQDpcc`nWwe4uxRZ{oO*G%tE&#xELScBX9p7qD<*IcqS%6i)Q*@xFudROdvch$sZ
z`{rM-{lC<2<yrRs$8*-$wJrC(obDdGva-gm$WPHquW_00yD7VrAk!i1^3QbruU{?F
zuE3Fg@aZS+-A?5XtQ@j)jM?_za<l)iWJ7L!UV}pycoAjBl+`oh%b!`b>3+X$p;B))
zb^jajU4G|%AF>`P+rhPT5ojtViB~Bmo5y5bqROeP1M}V|-ge}@a3emz=ozPi$sv!`
zUR-IcoA`G{&W-hIKbf<jP*izpj$6#CV-IttJ_&lH8W!A9S{*TCW#INUr_kAre`<Qq
zR)>6>zDd2tXUn0x6Wz{aq&#1-=A6)SuBHNIK^du{Yo9HDU)nf*rJK)>O#9wdbJv_<
zmN{FTU=*nty;11m(iy#0#hY$azhKD=Hh8M-mKb~>YxCT|h`bmB!Je}(zn4hNue0R0
z(Vn?COZ@PyQ2T7Vx0b(e?zU$)57P?Q_Yc_cFlXb<>HDTX<}<(6;Wedr_EJ!&aVRmh
zX1hnM=`5GuBE)7PD<!u2-o!#VttIF0Ty|T{z0fyVJ9lzdcqDhOnYF~WbNl<2MJ8+e
ztIRZD^qbyxbj|XcsqM|vr|(<|S^9V@)Mtt*bJ<iijS#(S^Ivzhr+)r4bK29XD-*f$
zZA5&&Uj9|{-{#Sy#g=|$CsvEJyq<8GH@0s>(mkjA#p^t$ellBMaNGaXuK1be)iT^c
zF+Bgina}gmdc9lg^Y(nnA8)Et7F=B5sB4yUgYUrP#p$1me*aMV>Hg`9;1@Yl@#L7=
zSs&T8FNNOyd1}|mE8C_jUGn>T)O>#a(O>I5ZGQVmyxu>@pG#_U-2a*P-o`z-7BpY8
z|LNoV$p@C(&kyIEb8Bzc!nwaTI|*;D0%hYj`zJLOOf|T$e(O{NlWAwIUFWYX`*pTB
zfMLNjqr+*&7tKz(<n_&d@;0w)&yM3NtQSvNZ;%bu?U?7O=995zUU`?@<hOfvdWlaK
z&5C#-${U*GW3u9vXaj>;>nXF|TK15>U4PCl3Oz0`YZfDDMM~4kyw4kcU9XupE8+Qa
z*<CXuJy?yM<@hFrYjRvyJh$hu#oL`vo;}-;|Juv#>#q45HFsS<e|!E5_q`gz3D2+l
z|2yk0e0-CF>wgB8nSYwjPTbj1c7C6C+eSIzY5%96oLsN)v%Q>ghE<Bj{%YBsb8Z>6
z-&_3b(u)7nbLM>(oqy(Mv*!AH=WotgpR#VwVXm`>JNmXBl=5GHkCP!|o#z>E-Lw3g
z=Kee!ZTR5w{LC+B*w^@aE9>e`&A(T-sf2L{)^S7TnVg^{+Mv}rUQBJ8N2-1+>8*eH
ze5H>0XJgxlCNJaMOE)%dY+$@^EwTU7!S<7S@uGeG7gujIJyL)7q)oni_HwUz;U#YH
z@w=`yzIV~b@7#{NabX!SoGv|6YevaN4UR*lN8A0kFU~uxW|Dbn3j6MF$-mgdGbAJC
z|C}>({gzL8M-vtwj*d4<Kc*8U;q^~u*Jr<q|C<H#F85kFuU5ZsFM&n3d9V9d{$uy-
zgTAcy3D{u1&uOpw{Kfmq8k&NN%Ugarr!O?AYOP-VYW3XL<yEK3zkOamJ<0xUSF&`v
zaY~sT+vm%BY#%TlV%!k0yf{Cu$NH~w31~Er&0#)dG|#pyBzNBH+u>U6pqfZ2V24=-
z-&yO~=?@z96u+!mS-wFmS4$-|T~pfHI^@{$Ux)ogisuPDjD0qF^4g*+3n$heI;*vw
zV@>Sc6EBNP*@Q(TCy31pxEbV=QMcpBOKb7Ki!OV+s)Tp04Kxt0x>-{nDpD`JYj4vY
zf5#p7Ui`mkt~1NLd*x&MB#-@aSH4uSmHvs3f5kT^B5CoLOJUq>;XTW%Pgg{hzIT{&
zeaYt=>;9%>H(qaZefgVj-R}+7%FEU(y0fmiT2Xg}K`YvuFKh0dEmO{$P8DY_`P5~(
zAuRjk?YE_ZFXPN6h3@Xo?|Pha_l-YehRK4XnadVAx`gq?l)kv}RXX`{Y(kFT=U<J#
z*|ZWT23dltXV5T+a*S5J+~*66<_L-?`$n@}{@`bRU#ILtWSHj1sSfWKxbwX*u1P$4
z$S-xbq)q7bf?TPH7bVjzAJnXz^K7Hz>9Wn2Qx{H?^hp)b(%NNlXXD+Y7n5#XdCIog
zbN@d9HitP!I1YVv*?D`FaIoo=8LK$2?EPN1>w8U)XS00L8!q*Csq&XIOH!}x{dpkL
zJU#r>-J=r<H#6SYTBj#m#h>h``TF&u_c@NfX_Fd{tqDAD`1<d&J)fUy+g#b_s=3lC
z()h75LtV6C|Aez_T#XAZC<KXH)oyI7-QT!vy~$Pkz&@XAIv390ogDIW&iQ@im#b|5
ze|^7X(piP6;3kZqMvT4B+*pT*?E<f+UbWVo<{`XSj(e$R5#wUBmEu#=d`yLzXGd%1
zw>_TO+T-)pV8iCGcajVO<~K<eG)G^QK6!6#tIz3a<!T(}T+8zX8<!gGz4xQzr}nRj
zjcyO${9MAH6}{{7w4^z=wm16Ue|7i^uimbGKSi^nBP*Bxe9V}1p~PEw*;z^N|LyTA
zth_ebm49DO<$3>u&HLVO^`pJ4ciJb<G`ZhssMj}VYO=({hyLn%dVH5Im#(z<s~uE*
z-@Vy`dBLRj)$2ar2)MU9w5@EDK<Fw^8!Lb-iBBuWSgSp==GR<d>i{3vT7bF!ljq(m
zt+x8uyXH&Rz7Joru5bKMCawMStd!>I&y|INiq@aEbj!Ugko#Wo{5wxbzvkYR5=tU+
zt2EDg2;{Nwq%P_-p7a0Ajg7v>8DA3gmo2|@>_+9uJvX=7Jv7_wKS6Zit+y=4w?12a
zMYZO}`pNO#XQe&w7p`V*xiL+(_@u#=ggZYX&-^Sj&dki*x8Jh(nZd`OYE>@&T=(@p
zNv+=fcN&{!H2>W4f3`0*Wlo7|vx4HsWkJE-l4FN_cu*GxcJK1qB|QB=+%?PKC8fFR
zi&@%FmEXHqdiHJn|MGd3i}zfAXR`gquEV!qMP|wdZTh+AqUwxuoLzeY9Ct8Y@RL)#
zyL;aG1df9tr#J#9tXyGh9r@h!w#n9-re&edUlS&;Jo4*cu5tC}J9p+L?3}vo{Jzu2
zRO2t(`uW?`ewN!mS=^oN{=WSp!Kc+%$cO&=5VF@Vw)Vfi^k>`ZD+j98`lAm{G`5+4
z_xD!g>g~r4PCj(%zbs!z@}c)G$8;~ZG%0|azN;QxD@a^WxOS1&ijQxV*fU-C3HV(P
z4P(CAWqE*m@6{K^A9Y@xi0EEtD7_-^%0jOIj(rU|SJ!1PTXe5F>7JJQZ2ydX%P#$$
ze{f6Rg?=p;4W8F>PNlBNAG4-|%9(Q<MsuGDdD))$QvPf5!@2j(%QL;_91VEX{P}pi
z>!Oprw(<S1Eekqi@?SjEjjnat^g{iLh}5}XnfreqnFm?iSJr8h`@fQl+k{{LOGejQ
zo}Xr+S#OF37VoT2xb<mo!dlPg%k!@vj+tEXe0Ax+Lc`~R-`-vCH(PVVhc94jruBT~
zIh^NvQ=GTeYv@$xFAG;#r&4;}aHR&+?^=&!S=JZ!Ppx-*J$LE_jn-3Pu>$W-d|_bA
z`&xO;IOw-oed*rallET@Tee5;oyIFsO|J9bxE?-M&;61u^2C^Z?M00lnzcDLGOy0N
zd8l$9es4TcB5gt8Tm`mkm#01h578XlV%qv|rpsoh@9P%&yva@W39tS9qj|RM85_Oz
z!2;`awZ6Ui|MBq|)_oIP?aw{iekJ7E|7|TVAKzbIJomTc-aD@+-?-BH@x0k*&;66D
z%YU@Qe!IsUXPAEI+1;{Zo8K18z4)qinBB%{_3`-6oF@bGc@zv+%16&tge<nxj9Y(P
zE7~lVX~98T&lYBOvvbQ%ckFaM^>&T?@#x3zzRK2`XIN-|pS$Rxu=9%dqGjIpf5c<k
zYtH!F`#!%~HFw60cZUKlcg*@@=PN8cvv!Bs`u#U<seh~5S~aC`b6v`{AJ;R!U#f_|
zC2w<Wjy3x^osF`s3LH%zz8_zZdCPZi!h#$ZmSY{(EAFQ+ywbM1^X7>Y>dq@TW-z7w
zwOJGWF2RYd=Z);4%z5W;eqP?39J=<3uC&Cw`7fVL^_5Dtiu!9@zpDM}>NydwA~GWU
z#JhgB$L!g9<m_zQChu~w#Z?j8PaSMYIw!Sg)y#^uOQ&w~O4u_cMScF#PTQ+nlfJHS
z5n2_hU~2kx)!ohM=jZv%G-_S5M(5NNO=bOuyAGV|XP#x4v-9yBL+P>?e#`mhX4dBG
zzW13k`RU?ngM?et4#>L&i$7Z<?{{^wbmhjYm)Cr-YBap|>X5j+O>^-(Z=M+|;xAWS
zc+RtC<IKL=>5bC_C-@zoe#~%7xOY?8*~M|o7cxHAv9(Kgp7{N|l8oe@sQun6<vi}s
zoa}S^!rGo!-+4A){%c&#JO4#*`Yn|p9@7^ellbK_E>E#eI<@eDO69NLU*x{dwHHjT
z{r7doq3M?=ue`d++IE-SiP~~ip-kb)-(L4(9m8lfVBX#$xq+d2vyIH&En2=wTNv+1
zW!{W(m~nLN#93<-Il?5Ki#G1uJ(nl;q1wKty0hDI-u5hhWV)zliecfoilz>Shwob^
z&v`RBVUwU_CO1d%>>L-@TTG9hJ%g^STQ_6JrsvNR$}J9vc!i``oSP$aCsj%FYnh+5
zWtlY}d&{(Lr8`OL>g`GAl{z>?SRSUiZ0itMXX*aQr0+@Wx{VyYTXf#q9NNRH!0|?}
z_sW{$`}OYI+~cMx+$o9Z=;^ZRH?Xy_x%B8Z!#}S??*1cTZ`7upIAQ$0JY%C^rl#JH
z9G6pdpdP!q?IyK=9XkcH{XhjWWX;8&y&GL6G(Jg7h-59<B+?0LNpUn42rE8YaLCtn
z^6xb^lOL@Aw#>s=`S*;IXG@F{CgyyFs8Q(P5NX*jntk2k#ElG%%0p{CAA-7j(6ZtE
ziHS^tHJ}<yf#cBG?lqjC)-0%j;<%%8!c2#Q?yT?QYQ<Zklj{%m{@k<mvG0~uOD2@x
zIJ-o9$IEJ;Er<3_?VZv1C#HAm?K|mrxGxIiEQy`tWXZiCYsvc{Pdh#1>4(m?zbbiD
zuv`C!`q`|_1&{M<^U^POEnV`?&MhNDMRe62zvi~B3hLea&6X7`@UfN4ity9|`%Wof
zhe^`oW{HbBFPhdD%N1Qpf7kmW;OGJScUyyE8M8D}a?YoJeOkw}dE!BS_Q?@u1-wb^
zX>wx!XPiIt=D-VIsfd@Q(=`_aZd=3NFRPJq<j)~vE_cTtzZ28H75i3JH-A2)XM8RF
z7^wFonV0Z&_k7R)ua4a4Ev@)+&}Lu!2k~=zgk`Qft&mv9f7w*dD%fR#0!LGUCuj>t
z(_{BLYi_<ssVh=+uRY|Ed^=6uUd`I`-os@FtP7XjJMjCBs7Up;<mp-rXW6va*}Kjh
zyzAtlldbT*WA>A`>ylD4pEUA?Wr>J4^scpeEZX4L?)LF#{|7I}J(c?-wU!kG3yQ4b
zwp@BpdC$*_;x)hD>g=od?Ro#_5B0W>Z`b6{w0bEV_xA2B>8#xc-OWO;^;EWOl;)DO
z`(1ob%697ZXtrcN&c#lf+gJ7-{5>UH+uHAq-^%VL1&(_66tw`3g!9Lk+Lo`fQ!Ktc
z*Kx}CSN&^e$NZVY|A1-pq2qg-IQJ#a4@v$rr)E}|;iTdNXQz34oj%arf3xFHzTIK7
zN`@6PC0rKqmgVRqqy$#3(|NMfy(u%|97kHKRpJI^fAQA2lYAZo&HS3o$ld=s)9%}X
zslSs;4{!UhDou{3aHB^2s}%*G7noGFiV9Z?ytFNAfA?kX>X@~Bo}3>UuH@cms?O?J
zayh#_?+v@%)N^a{KN)Jg-Lbe!)Zjwp8KJCPdzYWDQhzMFm{gV(p8e$P$vxulA11tC
zYBM<`Olz?M$D#W?GdV?SoY-u1{A|717yodQo_^~6aT6U!^L00`uC(FjzI;bTe*cz|
zS?1e=j&prWaxHf+ZT)lCZ$HBquJf#!s@E-!9-F<Q<JhL3u1o>_zuzums<VFfaqoNq
z*Q}KVpC@d;<lFvEx0XrZz(cJs@*-^8mS&y3-JGt<YW~4^zSV}QqO;h2PajX`OXyTl
z)NIs<j?Bw0{HnubRR6PO;b~cy8BY!?b2NP@cS&Pn{mWh-t!N)5@~@on`kJWM2SgZ6
zm6%!=Jf3o_ZvD$-pB6#xUCF}jhu^!XZJgz_=Jw%3KE1n4MONM7dGb6v=icI9>v#|3
zEQl;s7S(Z!@$0=Z_npwktYca#EfOAgpQZi(>2*IrnLo^ML)PrBzh6GQax(+%e0_88
zCi~qr=e}8IWnQ{*DtPM`zo4BQ%A$u%3Nrqfn9rZ@wR8T~jJ1<aAKqs3_kPq=kvZ-$
zKBx60|35b|{~mWF%GYtn#*H68G)c;A47K_1?Ib+IxK!)mJe#88?)e)8-o<*iU-s(@
z{`&8O=8Z$S`!=qfQ~OdTG^w-OZ1Ub}_4ZG9A`Tx~AtK$pRzx=C{JrD%79TH6R$5e}
zV;H!wQSeGkS^oY0m0OCJFSgofePG`6?R(EyXCCik_c6<v_E|8=Z&TU(r!I^$d<0^)
zDH#SnOy4mdZLDZwo$_Uj-LN`~*jXI~Tv)j3%}t)=?K1GMxf!W-+Nr5xfA)gF6~@Pp
zzP@j5$ra+wvnahO?ktbr)UUry9<7?8k<Io-%Qs*}OirVBxQ)*2>F2lT)JM!d7AEJ<
zwPw}a*_lgp^dD^Isa>X{bM4G){#UYl`?hkuwB7z8=Td)Kr&`p##ARNQ$}6;9-sZZ&
zxFs`TVQ8ta=waT5V{a3=wyVzM)V{Vb=cVb)-rnsWd^XmEdTcOKV+^>v$M9mHQ-@TO
z=)9HbY$nrm^*+p8@Nd_ehWPUnF5f6v{QmEbkd;dhopsJQ^R^}Kf~eS|S87J}C-(Bp
zjL)&V5Ha6hyvFgfgm%H4_48kyI8su}lKJC{{K|?1X7BX+18+pcpYhdJy3Ib5&Mot=
zZ1tApOY7fXQaE+{+S}?GW{(8j#Jia<?D}6XadElmt~WuEcFTB~m)$-XF@Jsi>@#;C
zrf-{Nu{Bq%cUSL6_KL7eum8F2`%}L^&uIJnRK*7y*Lq9dd#f6jF|VOu>BGx4U+)#)
zkxl*Dn)>gXF7y1_ejXDklNCqXcyBXS6n)8w%acF9@~v^k?>FrGegyyCp02gv&Of&7
z{r{g|jqV4n29F5@t#MP?R4(%Al+peNMbAYp9oJ@H3|m{cDfp_jYxnkw6~R}v85nc8
zrshih=3Bw&7aUc}vv8%k-{)rsZRVvYeRoh)R-gaqsl2RAP^n`0{!{~hmzN)v4*d<i
z8FJ0z>E<VvX7|nre`k5gByeD9;FY<Xr<zu=E~x9j<(byxx8hHE`|L%Aziz*Z%*bjx
zGq-im<`WN%?Qefx-fX^a>*s^~=Y*Ixl<k>WXxpT@{`-d$7DaVWqrZmb%gM@JJ5}Z2
zXt7-2jlnB(yBd>&w_CD}5}&55XJFBp`tt8J?p-f_mKc8DX>M3~>Gv7Gi}nude$M}V
zbCKH@X`_~RykGh!EYo}P^7ru#@vL{MC9b|uycd6G5@&avx~}fj_inbV2QK-uefrWO
zl5Zy?<N2L$Rb|kP(i3%M2c`dWWM5zRe0pc|-uVZg`9yrOetmsi>eo~M1<J+pm$#ey
zEseY`9+8-FYI*l0PjBz!%dg(Q%nJy(?!U<W^}#<AA8$Q&BF1HR`hC^I5&fK}|DS$x
z(tg69!@GCgkT#lffAigzYZWuLdaGSNw4lD4%{S`AudmsWhAS6yIDi-XHU-3R?DF_J
zD}JWZ*+0gI>dy(JF6v_1X?^F6N8zRC|2LQ4|7d!*Ugmhbxp}ZsN0)7O_r+Pdp^L(t
zgO#tW*c-JqYHgtF&y`xCQC(fWYggDh31lr%O6xgx&wApCxbNTHl~(vV9sB*WqQd6M
zz1hX*E9XzHKL5OC|MTB-g(rrX9Bn&dR2C(1e{JW2|7Q&T9DXM|t#`GgVT$qcR8Hk1
zM?YSkmEm{nK=12~f8I$?4p}nQJ15oVe?HH@?T<UVdX9ge|3bZt-L~Flmfqz<5mj~{
zSN*JwofDSRv*3EI>)V^o6HDS{4z`?~=yOYc_LqH@YlORwPd0zo`$SS&?)*_}wLkCY
zUq657ME~8lKVH?Y{v9RU`{l)y&bFEJ=Bz$=%;RfJT0`gE=*1?kEhmh{no6z8ynn2J
z$+9{8UG0O&|FN$PXP6&;ySL`ank#c=dQVdowNZ`kN%+0c^Gn9<X^qC)jW4M#oZhTi
z#>oAD--(H>`;G3491s7zZ*kQ>od(fVsam)Fk2qN$f7FcQJgT(q$}7trx6nD67hk8p
z60P@|z3ie`b#?Tn^xfYzA5J(b7hPQ~BlAYy{(a(=$YmveBpcoZw*_d|ANnmJ>t%Q#
zo9n27!yfaw$JZ%xb?hzv{qCvW8{ywOZtabf5^t8A;aBi3O+sSrhVl*1-yNSE`Yh+h
zty_;1)(Zu=sMrRU2Y9lniT!p9-q@#_upq6JTWqbT#uYn_>^@7i8h-UZIR^X9?>T;X
zyW_XpQ<IOMk7+D3-*`yo#I(~L!BZj+y;$s~eAxPC(RTACQ;+Ul{m8ONYNP734Kpf^
z>GIXT`&u_8$FpAa<DcSq@tY4ORL8EJSLxO{qhi`R{^RFPIjlS+7NS*CF6UruSGFzX
z@2unH(Yc}LyVf|In3=I`?)8?BO6UI^4SQf)my>e%fT`e{f2n-!(IH+ltPMV~oUXU`
zI;p>J(S^IGw9L#_J~$b^>Ps#2#gu1{^VN*XbIbh_lFl!5zEPRanh-O)ENA_kimj)&
z^xJl>ty^}3b<UozraxuQo{apcTsP<E#p1Q)Z;JEMU!1&KG2_v}HDdB1Cw?9%ZkrSO
z^5UJg^)`n(BVT(SnBX?$T<D~2=Tvu2S|1qn`qA3U;yd%IE<N|&!>1uADB7shU-mbJ
z=YQDtB`u~Yr>e`Xn2obU%$y>0UdtQ4I9acsKYQKrBCTUxLShN?ZhE*LZ<#!0#qGj3
zmh$aKYu&SX5*P)f`*{VXDMhCqc(jz=$>8Dihy%ISg6+wI+-}#l9^%?iWD+D|@w4N;
z)`Jw!y)kJ8)@_2)$t4Y)Hsab@r7|4ZZH{})x%d8Cx+dGR>2LYQYa-|Uzn-%Db^V|5
zqZNwPk#;MSPM)w0WceX`NKrxTjFxdmjP)lK-YY*f=UPPz2n#bCPMCZn{M*&4mmU9w
z=a~jA^b7wmgX7~p89Q;Oo0swpKZNr$`2F8JDKmCo@>);Vc^w^H4il$qlueWENGver
zKG0JY+LL3E<>sgyV<GnWOsr(`Bcav1Ercg_`qm$tVUhNx&rkm7r<W38bIp&hZ7@uf
zmj5WV+vZtK^j)DtNk8u=l`@M~3YMw9GF)(qzvrg#(N~G5Up}?!m>d^upnY?CR_ogN
zg_>!eldnk|MJHr?{HWjPwROd#3BQ&fKl$jzhYQII#lIP3MwCB$-?{MUg~&InlFzP)
z$_?S^>PRuzr5-TJ_rO7kbE;x_H@hM}UA2DZU~{j1UVX&r711B>?5tjTNO9rOH-Y!c
zje?FT<vuO>apvx;YX|E6U3vSS_2lOChNkEwKX<p=G&R6wo@<)jpFQRiF8CI|ob+@}
zq=4UIzpaH8jY~sUg{Ul)uGpjRexAktm#d<<W!1B5tM6Z_$p3S`)$UQ}5q|g0r$o-C
zo?qL{z4F=y;m@_)A5Uk=w{2U|vv)14RI!(fi|c|Dr{{ET_~nsQuEnkFx3kA3wm0>&
z@%J>74{k}JB0^R7Lo*jooL4Hh?n~G7?TeC2v%RABKgm+Re1&`C`5iLZt6RicZrm>9
z7pZJJ<95t!<1sa(X&Z}`-tUe#a-9Bbro^Kq_K_3Qr|u21wUQS4Rr25Pk@YnxtxJ2{
zCO(gF;z-}FR?{GQzA*pX-uc%44|A^6zhBIq6xZIL@xJ!y)ZF-kURNjm+y8O?V}9HE
z*Oz{_oEO{7l(M-u?fapR3^T0jz0Mx$Y`!{U_U9#rt8`2w_kfbe$p`Zmp18T&+G?e#
zz=IbL)8$_+$Z_Q^x~r>Jer(g<6MOHU4ZG4c@%>D7zQDW%@2s!!tV?sea#dp5T~7(m
z&%d{A^Ld?e`aswDvP+Xp-p|`Anfl(EyUV94`ZDwFn)>T!&ENU;X4I|sFFiMhv)(VY
zTdH#Xgo$CIx+`90RJ`zC=X<neCsXbBd#n3+&D`9!`^|8A^r11LXu8RxS1Em~CtJ!-
zGx^lIn%8ubv_++7>^7ChM%y>koV$O**ZRBudr_0K(Lu-l-P~Px@7C^#lFKZ3p5<oT
zT_EOrYns=Ol#bYj*UybDq$hny6cjC*R>}Oz=vhkTGE;446`vjVRtBs8>WMhhD}Hld
zd)_sU=dX?);a_rOdZhTui6%xb-^D*Guz2$C&=<bwXXlv~eEhyvBTdvhqT<l!kJHx)
z&G@k@+r3X{f{B68BTdIOnk!2`+}PPN?d$<##u&cKDNB`<UOux96+EFF>9ABtOvF1Y
z)i~gHt#@CQd$G|Y&;1hp#}`KIt2!<7=+OtOzdshK2FGc)?N2}S*g<Aycf^IAL5~bx
zTQ9yL$2_M$@whH`R*Juc;mP=0vraDcJ(A>MQSkWDjIDYP;|~7GpA~FSxMlYwKd~pD
zyQ?3W2qw6_e3RyRJ4|7^q{_P2zg2rZ`^B}BSpK&N&;MlkZtuPuv!4~%id~i!KG?uJ
zVQq`3^5%$1_R}RNeoV+^3)!^j<WdQ-_dU-h^-9LRx)pGNNl&KBN9yC{gyfVI?b%0|
zBu<5Gem3!N*qy$_(g}RNCucl5v~F|m)m2O8oA<|_+IQ=x=6rQ-|6Ok;?|s$&MaQo3
zB%|S%vXw1Yh2vhE{t`}m8rhRQb&<=QFHgGpI43XT|GQcGXOCM^=?mwLm7I%PkNq`n
zbbIf&pyJ~r78VwU3$?FK9PxbkenaZB{5Lm+S*yQ&xTmgXQh&=t(B<L&)9N3SuSl<t
zU9K7W`|Of*=kxNuQmrOmPMIDmI5aJ1%Y2iWvR`*SmVWPhyDs>{kEPN_a}M7$s+CCo
z=8|}^g5z(jH&ft^G_M~$J@)^sd9CIy?!2pMv+Yi{<DWv0{5p%w(}(UhYc@5mJ@9D#
zu}O>b8P$5{pZ0(KebVtT-DV}FmyJ<OPZimumG_r!dffeL8vh)FRPBV56UxeXYP_Z&
z+040lr*Mhdw}pDW|3g=7G5s5T`_kT7s?Q2PzPTrKuK(Znb!NM(T{e8Z^f!FIFxP<z
zs}28HsWZ(#a%{_i13#8Lym~m_b|*uLPEup_cb||Q`9VL*U!^^|UL{s@?ULB^J>R<y
zY`U8JV`(4jTItk9zAvk<`xH&z{A{|Fn1hX7`TLKD9sb3n&AB6eaFg)&YnIV*+*MmP
zq^2ZG7r7qWSW_x-|Lwl-f7-XMIq{w6yQ!R=$YiI*3%Zu%{nz_B?NjL7pQmceFC{O#
zk-BAd_}s2qG25r_4>byf>KM*2UmVtIKJQnY{r4=-v#Uf`-cK`F?q!mC>DAmFQ_t}=
zRGur3+)>ANw)_5X9S#nT18ZE2>moPx#+iP)yZ4Ce!lKS{quEy%xvWz^zfruZO4QF!
z^yP=gr<NRF_O<Bf${6=O`t1*mmr7r?d()Z!Xp5O${g;KSCJSpbI~kt;tQWlfkLP)}
zs~x|em(SW0&A9y2`%BfA)Nb*XrXPRdFZ8}~%c6tR_p6`nl-QoGTOX@+c+1s!e{QKR
z{Ay%(U^@S=tqXSK%Dw6>@YeEB@11{of#vL<2K%@2xd~4EQnWAMXr|E@mXyfNsfTr<
zCAG@qmoHYFd1Y;~^Xx@wIT1p~7A<J!J`;cY<^)L&OW~sn)^40~Cb1zx`1tzSa@9uZ
z({DO-p5b(jE(rLe9IafOV!Svb&7#Ea-&Aq$pR#99r1P5R-HF~VbMuVG-!{d>j5Z75
z^z}#Vmx_kYJN`|6u12g1dtF`K`Zu=LWp7Trxw&~~%}=AqO(_@sJh+q28$DRtZ}ItQ
z%(Rj<A3l7Zcw91ICu_Fv#~PDU%@cb+6}%2pnGrqt??&OR#|4Edv=_eqksL36^y}&E
z2InUUw})S;5)&13Du_@?=*ul*m;1XdkR^_vtvFgnCP%(P=h_`H_x5Vm?1%@YP09!L
zb}N4ivzijOw275FXj6*kzkk2;TNRZwo$flSRz^w`YG&0eTi5+;nt)jHq4%FIuXk>_
zXZW!E`N{RW1x3#HUcVCi>Co2-J)5N3Z7Y8-JCR*lb-3o*tVc6L?!R2Qo|nJ(XvUV~
z7cc(PTyK)}{=tdK@u{uJ^K90aM6b<xTezk6)~@J>4_^G>>2F-yth~*t#l<CrrOW1w
zgLrthR=T<JhcKDKnY}5jHMR2>zFu@pLSSNwSY@L4;r5-v?;`r7@^5b_IFh6j?ek=>
z&G`?d=_}W}2nq%YYJB=4WUzVSshvF$j5~`}W&8ro7tQ=LgE28@v*bIzoc5)2C4WB9
zWZu!yVY55<UU9_cbp8$Py4ehyPo4R5=;y}f<44-=Jgj;YD>!{!j8e~}tiM8n6HEM=
z51;ThyxaNsz}g?b{Z=IX{JQ>y`@U~?zpt5j<%U(}2FYS}?tj<s_Qgv_X?O^FiskRS
zKkJkV->(#nV=I0>Uh(L|i%dnm+?g({&-*-?*U5Ox6r_H*^q4Ei#pROeR+dwzr#1R~
z<VauF+nDpL;3Mbvw{L%0PZF+{FEkR%4(d^wq_c*5&v~8y%q^|1O<Q&Sr6#Mc|1AFF
zXKC#b^Tqq5T*1wf;8xce^D<q|NegQIa8X$*xp8alHov)HVN&0MTwILi@!Xfaa_Q2f
z3)#mzIQK7I?5NVqvAc5Ci3i;gQBnIYecu~xuqJxBW~z%y>YvZ&?eFZV3>MdLQE`=Q
z=a;v$QD#p_Ogy;DRGY)*q3^m+XAO+6x94R|SKiFXuFd@I``^t*jz4OXFItEiURp52
z^66(O%_*FDtg|@k6OK!tNdNu7=uNMH(CV7t-6w8Ozwx`ByD(7EXO6|hpru}WH+|QO
z*<m0jCkMKql%1VjD|FQo*-vL4b#QenXsNZ$`2BBY&$Jo#kBU-a9z@pkvrV%0-<Gm3
zO7f_ry?tKvbEVn7pY(o~t}l)`A^L}3=IN1}n`3qgdM$fnxxujh&%Yfq56)jqR=24B
z$F-16x+vx3mq)J!#a$y@t}y>LJC<_fQj9adO}}aOwFl3hwJlw$y8r(_Yu7H3Kn<ly
zN=AQDJ~BAB^QAsHF)>9qq_=DCwB#*kmMAN4mD@10*C#jkaJ~1}&2|rY>?QgS9y@WM
zf30TX|M}JYk00IKXV^6Jxo<;bU2d-C|M~Zl0>c--*k|OXxIF3N!s{RB-kPvsw)#xV
zVl`RovYMx_3!a^ke06p8@spF)e||cxFKL*>!q)6){XEmD|Ki4|?j?>9?_T|X&Sy2{
zbXB}!v7ml`WQ}Q^;ju~gKWulmelt0h^^wK<?X5Ex?>nZf@NMDu+LfyNzm@hqe$aJ4
z(>O(nA#~N0g!A+253Vt?uk?9v_)(->&DXVud;ac{{rvq%Lf9Uoe6}}FS=3gQN+17m
zk5_KvJOAi@jcYt>4jwQS<o|QcI$k%a%3WRd_xTfd@6N6J`|II@2M2bQzFrZtbCZ+c
zM3KeSOpSudZUxWJ$*TFxIPh=h+RELBleTg!3)?8b8<BXZ+?!>x|B>Wu#v=?f3^(=i
z{_Js>)A^y|M9Rd9qCAtuY0lDl|0K_?vR3;SAu}r=BuZNHhS9_FM<)4+tHgu_Tju{h
z-g4LG<of;penoLPcx3S2*;{R{r>EB=Z$Iz!bbaH}S0Oq&Ca2y`-k-~3A;sg6Ah4Pd
zwiY0;F4A=NuGY6g|DN@fyXSEpDn7-rZ`Pid_gv?%nI<v$;o4^AJrbARcyF8au2g)3
ziGSZA+tM|kUz#q?G)@s@d))r#=XZ%yx1>4jUO0X=JN$E5B)iTv-6eT-uUo|3=Ov`9
zHLIBM=x4%(8nrDe8CW*%D35jfcX9r$iDGYlE%Uq7x-C7LUt6m0)53=u$}aPs_WRE^
zuvB1+_Llv2KI_Mahdr{^)0AAh9=v?%d2x}eXjjJ)jjxwIzMR(Ie_?BOIFo<g(+5r8
zjvQYZpO97lFzAq{X9Q0R<NJN-u6D{XKg}o2dzV;aXWaAtntPG;yL%7sEoeKs_kL&N
z1?!J@!XC%JTK{+PyAAni!S7;h53LAX{Nn2B>)EmM&F@t-uZ`Y*X-8qQm~NEIWHsL%
z#n1c7u531Vq8cw(eW2rF#-8;Fn^ekg`rp`5Yx{SVO8(b`Ehjy1AAKG9{GR<xb2ZU5
z#VhRh^F%Q@Y`?#MNlub}htAX4kMDhrHJH1ge!nV<+P*(0E@@WGcx0&k)AIhdl@oqs
zm_^n6);?+=V7i^3oBwXjyRQaPy%N0%Th2PJt8tTke*V`@F=_AZd3PnO%XFq{g&sP7
z+?{nn#|n;`tIPc6dcD53c4uLZ?LY5b$zA>78Y^!yKbK@%C2s7LVA8xNd9s_<-U7R6
z`qy0rf1kKNaf^bUTGQXn(Lyze^380#Ngo~@T$!}jDD~8lPft(3ytY<4dVAi|xV=>|
zJBwVc-^@H$BmZ@Q?$fLg)&6E?zOs|Qucl1juM_J1{HS5{vy^)iuJ1p$+<a}qYE8N3
zkIr^0rhIOEu4S@f{s;4tKk7B-T%4Y8PW$(SwV~KMz39$=&RLsn-p&!e^<Pczj&Dd;
zy86O}3r+Ix&0*yh`|z+`{?XgFzI`&5B0U{@gkJGg)w|}T9h+lW%)!l_{M3uTXWF(Y
zncr47c{1~ur9Ihdu(9&?Vd1OC<NKagED`m)c1m+?XMgXr1An%$%)RNkK5p-ihwbt~
z*GoATPt3f$Y@&I7TpO>n+uK`PBX<_5s(MYiaL!vWu%*T2Z@GR#Dr;ci_g}@*xo_ny
z?Gtji^nQ1W8yz~t@!P+?c=1%xXBz|WL~jxO_ATuE&U1-B@?s}$*mEj>^7U5-^JZPE
z^bWqAa4am?M!J0NlC-7uA3fe&i(KmSuAS@Fqp8`4ru80K^YzJ>32sw1nc6(AI`Mf|
z*;}czvrLo!|NC3=?#|4W!ON4@{nT`Iaea`@bL7sPqut_{R|G2W+4C$RFVnm8*kVy1
zt@N&r#qYNqKO2&vkp1Xw)W%)!_20g^{KJAz_tlxT^?P63*r+V5?&q`hwN=TBfKN|P
zgKjKY5xksFPEIalb(pWU;my_!oAQ~Dz1bPQMpDOWbHYC1{agMRWnP#WbXU!2f4S_M
zpp%=<=gvLk&YW)lA>#K(?iCeTMtjxYd(~`C^7$0wvDW5%;a06zY{$2$#r#+mv-ini
z=3O_QF7is{-uABZQ1xp2<(uaCRNPr3rux~|=8yNjqNk0$0`JfMD^{GMHP^L0hRdSz
zlZt%p7s2P}<|g0RkeKqp@s{fo)4=iz?i)63N_u)~>Ym^4cF(!}(^}p9pZ~%oiEU@n
zo_Opjs(-g+)d7=0hFLy8s%y@b{tw-;bEjeLuP-6?OzF%0=0@zTG7St2+_7WFhN`bw
zZ%?F5Z}v}rcX03X-Jbn5UnWaGJ#}Si_4G>&Jim*Lz31;{@VS{Po_S?H^X&f%j5v>P
zIJieL_x04{J4?g65)Ul;D6mZ~?q3b7f`boBdHRm6TWhYE#(KYbBlJHn=b%Vg{ujU5
z?pxBN*1j$2SZWX-A3taA+?@pvnJzE)4`<X+(qamp?K9J8W8Ph>dv(8a{qiMGOTO>q
zQtkb<pv(1;=k2!=%jR9EotJrVqiB3#>V2h{n3#wS35;&Zw|30joOrk`=k~VVGc%1Z
zuL{*(=0AVk+?j8BLVmxgwvra=>%1ELeB;GpvDiEPA1-R}-#PWvXX<IQ^FQ_M{a(&1
zn5bMP`8lk@DA|MGZ<cxL$q&=^T$r2ijBQ?}(_#Zz%k{T+mA(dDJOA+EL(t*ada=7S
zL<9o`XGo=<oYdMYZ636%B-1ZP;#8%4dF!NO5;A8q<F0TV*<4XN<y`ZnhKADng&*fj
zURf6_&ClO&V*b@>sn^sSd#lY|yF?b|-`~e0ZRTTbdh_YB)yF*;9{rC!Fg^ZnkLzPa
zo)X=etESKWytl8XWFuq3y9w)mq-KjaKmWFP#?)ssecq>?o8x(Un(oOdn!@+)-80F$
z;<2!!qwByGo$7CIDk>{KKAW9iRC6>u=GJzPg-hlNO;|4#mCX7huQR1~hmhx6zubSF
zn{U@I&APg3P1IJd?AKc?XPuv&zwhL`ySrD0tQ2~Dtaowr_PirUGV~@rNeO?Xoc4Y7
z>@UTy-L_83Z2C4OW}B{j{b%;AyqxiS*}BhNshY|8I@#Ce{P8FgL92qe=84MgmsW@C
zbBXDk02vsxw95%JFq-N3?#|Aedn%19KR<o^;?nsY&-QyQT*CM^Hn(6?+hOa&GuPxT
zSb1-!>}1QWnIcb5PftJG#(U%D(zm}S7d<)AxP5zh(DZB1&dwH=l3FD@x5sPRe)IXa
z?#$b4AwB)q^xPA1Q-V6bv&~9MGbp{gQapZ!Z`vm{J#8JUZxd7FGcpb}m>unUuHb&n
z#`C<3O;YtEC1Lk)GZAI~Fm2fceN8j7Y3J>J_kgZ^?-JF1^!Ty!#-yVnJsm4Je7r&4
zn09t{rSfrUnYbIBv;XG>X{?+Pz1Zi<xzuBY^Ev9ig)`2$6?@0Hz5Y7SGM7%FoLgHu
z*F<h!*8O(V)~%^ePfaa(cSllRU;owh_2;){U*B?gvhy9+JM}xRyWSVs-LU>{`SM?>
z^Y2@<-}n8lwc#l9|Ksx<E_dsmnZc|+y~XoYZMmHKll|x0<C6OGow!Tx9jZJTt5f)D
zQbm4WJWu71_a7$YCkf7&KIdjA%f6ls8w~E;xiiDMT#r}EB;oe9Tnj5JEeTL^6%*Bu
zn{zfU?#Augg#~rdO#c7Z1_UWiw=&!D#PsnJe-?YWnh!<WBjvd3^7DNIk50*ovD`3a
z@#)_I8v+!~^6$-gr{--Y)w^T&ZsVLA27EFW4sUO5wJdzpQugO`Xy%VU7v;YTx6asB
z$Zc=twd4G|8~L#xIXHZ`o;HiU?K)q7dElzsCEM0Y&5oGx^U~YjPxUMIZxWpvdrN)I
z*7tULt~37yGygk%?MNkK>?-j?yF8XIam@cFv-8vaO|1;~-!ep)<vSJLUH;?Hy7>K|
zE4*&*C{&iUDoObI>gtVKw;<k|HdQxzTJOAh5z#Y?6V9o#%)h@nAZX(E`<sugT_JcV
z=_4n5xccWM^?FZrXWqEMb^7+lIUyk{AE&&0VfANviF*0FJ2`hZweG3>yzF+atX+-8
z<>mg*54CbzSX*ySKQAZC&|h*tguUM~pU-Ybi{9kFKNDVfta@H}<mv1Clb6N+*!Q$V
z&o1wz*fh`o+fB^X9&ehz@0`GqKaP7XGILIJoV$7=Zb{;$5|NFw+du98epEc_`JAb*
zA14_c7TMlsx?A5i;LN+ojVuh^#}==gd0gtm*>+_q<_$mFzw+mFzq}OpEYEX>a^a2T
zp@#b$%9xh>%{_GU=FGppzZ<8V5J)>aYpLf0R~OfWs=%E^sn^!VcJEZ2R=at2@$;_z
zPM*B153H}1=*<0*nw{+zoi05qWV_$5=V50jotg3B?s`V6pyp`xZ3QpRf3vViNk0AT
ztcYHW#ohek>3XrC3tI&R1x<2pOxT)z{guOy{uR-TGCp${Z2tu*m>-hMx!bj8Pq$*O
zd$K|PVFq(`2A-SRpYQhT3U|$%A^GKRrfDPp-Opl2_gC|){VjQV>G0j6_>!Cc(diwB
z(vNW*d(*k-?h$>>)Y$<cVXuFfo)u>AoOyY*_-m#-+wWG%u|4;McdP0w{@(v)j|}Gl
z)%lfQ=9RPX9R0C0{P2>Uyxuien$NK*FPkCBGk@D`i-sNAXa1~{S2qY;b^E=ER?hc7
zuN8m%{Q2<Bn=?C$pU<!=)lzov3(-(oq-3ynW#neH)6@0aX9RpZo8feWxAcA0TAdyf
zz3)8X(T5*JRNq^&CvNkC&ATlPwwso1GpoqG<#KY8s_~g^yLaq(@b+!*+_|#zYd&!r
zr=7XL$asMF+1GU*N54luoOq+v@ZXR3Caa>ReVTiBCzH#+-CbT^cWt~X(yv*@xVlVb
zORDR<?@3P%9QE|JvMU$Z8ujnoUB3S9-~M$@4tBlOv%`GX=PhyTEG#VzU(au8dUkZ$
z-!G+Y2GjUgI8KijdZ+yV^5xkFjvUH5eP#ZnI{wR?`!avo?SH%H*O}Kf;c?IPBg~Tx
zjy~Fcac`XJEHh2_zj}`iv@iB#zJ3;{eZF>n@3ERMw|7rtJnQ}Nb9DHr>A6k8KX?7v
zq&`cmd71I<^0VwwjPuiNjW&IKef{zRM`kBy=g56EmUnlThifQJS|#bDHu+pf{Gs^z
zzpb-pOHZ3w9K7>4TaFBu#*`^G3AsIgZk=m8GvmkW^9EseD_^Q!+P_!V@0viJX4m=a
zE2bJOZH=*Mm|vZ&(^Hu9?bD*Dtyz*r?UU90gZ9_e`pvgH+rY>Sx_;r;y$2^A^Q^ej
zw6$%P&fUM!LD@4@7Vi+yo3r1`h3CeNy^iy2_-s}_PCN0HJ7%veqnO#AhcnpOZ$EkQ
zO!C;Ld+$Hro#tUDv897Ech05SH=yg$!q>;8oSijwYxecD8|O^~J)^G({g^b%Eceu#
zo0~g%_O;BfPG01+PHRed*`dW8mu6kaK7PY|wPxBfjcaS-<}UP~)+MHU3RH1LY*b1<
z*0a#PU+&1UV`)9T*L8z54`=xQJ9JbtZjZ&!`9Ta!H>{s6ulV>#M1PL`Os)ex+l$?M
zrCiU?v(*e(z_7dgeHf#L(l4!Z9(I4f1Uor9|NQg${Kw+*rA7Uv5h<tEB?P_b@U_U`
zt$C8=+^AhRBU)<53;EvTPt)_StcyN6$!MnHwG6kDM}D5@m3pn<+IrWfcm8hekAVl*
zL~LxjwKY5ahmVY`?9FYt(s_A#XJ(uCuMS_oDdAw#7t2Wx?slGiy!Yq3<1<UD)$I)*
z8j4hG=S!7c`;~ptvZIB+=IhGr&OD^_%;yq=apJ|772S^?y|}yEd~^EwXKT0LOZxlk
z>ytAxgWH=rR&boju>1YS807Y6v-3X`=Q}OU%Qo1)E!;q}K&kSTo6Jn%XvX5biAORT
zj5nUYy!xgCqq}j@Es2iqX7BLF22#F1-i3u<m?9)uB541;zk_r4!9%Uwg@uKI8l5@0
zxshA5MDOk@O}@4!^2+-7`TPI<(*Ao^Gi{>i^Lg_>Pg|-gaQ0>L@s2}{N~T{;r|{X$
z*je&OLOp-xhE1CmeVqQcqi@^O+Us8`*XLJMSj?~cr73NevmkD7l}_|F9&2lBSvI9f
z%U;YokYZNyBEYRr#`9WJ=*k<FtZt`$xLl34ytt74X6CI4A7(F~0U0C-(dhqpxBULc
z{r~^=JGF9cEPCqIC3dxEn$pUXz0&3x*VahZ)zxjuzRp)(Uaq5~Bhu5c#B!Iq!OUI>
z!=}2wzY2ffv41oFcGr&^8m&r_VLmPbb?<yR_E&b@ulw!G`XIPR#&S~T<z<DSyPdDB
zJbZ8Oto9I<troSvOwOG<_v7bJ!;A|G&(6$T%oN~qNoOm|vI%*2cNy0HDk++h<?(ad
zXG7yvrzfZAYz!>_Q2f8+0rziE?@YoZV}etq8|W(O)YH=>c-Wquo$Y?OjaRhmMD~{a
z`*xr-e|EMx=%SSQ_5XG{2~ITe)os*F0bTc*cXyWb@oSIwFR@<sqoP$w(k;e^<#+ma
zl^=)f*m$LuFnzGB{c_Pg=kBhlJ~NFjE^_5gJ2z)#fJ>aPzrX*h>+AhJJUC8I(+vhU
z-JfzEoHf%h{hUlpZ0yde3(d->e=l$gKgsbZd9kIzdS1u$GEjT4?#Dy+)=x~ULRTMK
z7rR>{U;)G4s;`gg|NoxvBv^U$*}1vhvu8_(g@xtZ+|;V(J4-`E&{O=1tVt{A0?!?V
z$!58?TpC}^58U*zpUd@<Zg7nZyZDypzb0NSd2?f;q;Xn;K}OJr>4k4^nM#@EocQ<m
zcjo<lwxA^S?Afb0y-Alq7t8(s_uZaLM5AH(a&=YjY2a!hU|wuRg~iubSCgNgn=4ZI
zZ`Pu(Z5kq;^{fj{w*0>&7jtuahnQ~EhJuGqXU<%?sbJaC)^=%6WwDK|ZRUjqjGv#K
zRSs6`wb7X*^ytx}7q_?T8>gRJQuFiEn)v<m?(8hq1~tAk&g-kFwA|X7y*dBBU6-kZ
z<HM+0k*+1$?$hf}9<5Kiup&_T->=vDy|bpDNlHq3aciqK=r*3Dqg^`D+j?Xyi!>xA
zrijSN`Ry)y8?nF6R=(x~WBuQ+;xmoY!wi%ry?bHwS34;`|NOCDY0cPOChBH9Pxtii
zcA8kCoN!k_@1Joq$mWulL81+s2M!%-y16;sxZ*>?pFe*J%F3R7d3l-3O=aetJ(ZLB
z?f*;wRRh~{qpz)vUd|NY62fxo^yzuF*0as?pPe<o|Kvy4`W;WU2Q2iMRema)Cw=>W
zx2;)ML7m8J*RFA?vi3@ucHOW4Ki4?@oXXiqH6{J$Z*FeBvND)GK0bcm&u7xl&dpuj
z9jP>F-Ad~h*`Q0#@2!oSdwRNF@Vc)#Mbk<{H72Rey=^$N{yNVxr&cad(pwX`S!+so
z%F!;-Em>E&jE#*!3A~w&SETC%_l^Da_P(>to*wI!295K^?XA)j5%m0iMd*i=h(<ue
z+URW`-1^Kd_N*{A@6Qj?nDi{>hVg8W*(oO{<=oj3$ofG0+sEVbn=3!3ty!}s<KiOL
z;%8?9HI&Y0US8%4DvlmJNU*W731R5!=yLFkJlZ9iym`9#w{K-KF&CyCxLmI-;%U52
zCgRY>r?r|PI}B!<=U-z`U4L@-`+cX^#qK^aU0)uQ=bojARPud(etvo7=VuWS5f7d|
zb-ldISG21mMdB6TuUS8T{R+B$bLI+(6*nqXBe_C?G$yTzwcj+YncGe^eBGQkw{}i;
zYUR?Lay@oR{7xs(HOmzr6y{o$CLL<wl<0M1RsJNAe5@z&NQdB)^YiEPN}C1jD#>hB
zQqp32diB=6TI;W`udAO+@LighePYh;HBJ*lUS8|eeKY^^q@2jNH$Q&AKg-5abZ7B%
zKF}q3r%!jc@k$$&zl%Ams_{ww<HwI*USH=gFE0mOdDX@%ePXhDf0N)u9{0PB!Hu&_
zvo|Fj<?7%)obM)lC~r=H#-vv)bMKyQ+_`q+=B)>_FRO_*tn_(yW@hB}JlWja+fx7j
z`U+}J&Nk22l9=csC@2Vu(em>0tn2H}_Q_g9g1K@psFSqFwY!6Nf9w3++JcSGMZ1<r
z9TfjwoKVKSM(6B@I~y9gM71=ggq%2ZsA*^MbEAX<4By_}?cTiEIQ`t5K!ZsSS=rbg
zJ$iI#mT7jz)m5U>W;qA$-HSU2T82>h7gQk5u`KRDGy+5_-@c2nJ@s#<uc5K)#<Z_l
zt3p>V3kWi7K6s$t{@;oG{eO?Wyu7@ytSs&SzrS2=Mr(?np9@^>C;RW`bNOH2-=A+{
z<=(JmONfEeBBg+!{%yIpjZ#mE9AZBI==nRZk{P>-Preo|`z-Q1t;VOOgaedow`N}t
z)5ts`9$(Yg-p+pT;6YGQ^PO!X+M8m-&L^{=?Cq^N7KKfm!fFO(Zz6cunwvm<=Pre>
zFE1`;Pd?tK`sG(4sQH?cvcYL$xSr7qrQdgAJ2`hBoL~R%WV?LbihxTqb~7K1h={nb
zCerxX`T6SuR1V+RQ)vuJ3K|+6pz*Qg{_?F#peAwY>-+odo!j|TpL|mMcD6`a&njl&
z32kB7s@}cEc1n-Fv8J7w(KvhdY$wmfSyxsZ1f}cA>il<im#3eZVaVlrX>TjH_@ynG
z!98xuv(0iZrOyBjLTq{b`~Cjsr}g(g`L?tEhW&QYj~fHIT!UhKDirE2+pGO~rdjv<
zt+`dniv<Cfb_6Z;dieOU^Uk8D5{5}Ed#k^z>FN0$^w?#RcgJE^$xEkmb1ZM}ELJzo
zz80dPr1XDbCDY}GcXxMxJ}zHBVR4A_sqJ(7XI_k1=+V{DyQ}&Bmt!pPch0}KxLDGn
zV1bjTyOWdCj~|cwZ|*A9*3;8FGt-z|T3VV_`K41$jZJ**SJC=EAK6n+O@Ur*=zp-I
z!=@>U>2iZ+@Ujcbe5LbbqWk|(dN<?IW6>^^-8*J*d^?}#_~GbZ9yyx`)&#4R{QUE6
zywVwWcbR5iU-$Gdzx|VIYooc`F6}vS!sFJK%)|2ae-8eBzyJKXxz?c!U7*o~$JQs$
z^xWK>{`uo^`Qon&*xjdncjVnMX%3>MytunO->p|_X}~2-Z*T8s=jWfl9$$ZUo^3T~
zK=S#yxxogL(pXtpA3l0?XjSOyjGLQO=gpg!a&nUE!48lGzt5aK8@VMz@awCqk2k%K
z`4?;KvCw0a!O`P|8Hwt7tU1eqm;3q5FmPPBq?oO_aem#e&Q4+VqEAmeLHV$enO&r3
ziKVz+%mTl;RzH6HxUfFnUQ92hW9rnYO@e}zw{|wX<MfF>bolVf`2BX)Wp5UAe7-KS
z*4$;`iEi-*HQ|5WyNXl)Cq-?`IXS=nU+2u3>v&UDyr=6e&Aq)XA}Z?B^ZE5b%l%|q
zl`;=4bZ*bMw1hK!ecaRM^Xs3T*540`-}4^n)tNrad}c=MF4Gm&4tw%t+QvB_!&dk1
z*^#Wh@6oA$M@_P>bX0zRwvypY!<{>KZfwaEc3V7gneS|n*;*o=U-#_YyE1&e-1fY?
zNf#HnYB9LFgs?2D|Mu-$MO77asP5R$yO%|~Y_>O)I`8;#o@dV7xskg{G{wZkxKx{0
zMQ&EJE`O&IzAmQZ?X9UBlaFhOcy1Nbk2`Z*zP{&R0_d{2X}Zx`B7zg2FL}2h^TyTH
z;VT0dvn}_VyQt@O$fvNJr2$W7&iV9fVsPBzm0@e8;^N{!gA^hUFD>;p$+*B!Uthng
z^!2gx_W##hOY{sq)-Qj){C=(bJe$fJ8xomQPfrWg0F~ax^FBX6AH3I%S1)#t%g$??
zPext~U3fxx?%!%5#_at|k*Y@#PazqZp4F?hZ*R+8T=@9dnR&L-+1c5w%IiaGYHaS;
zeC7>a=CiQm<)xe(8yf%o{H!e@I8l6w_nyp{Mkdx9n<|YhWr_<b0@L&4?k@>=vQh8N
zOy2{y@(W8!mQ;U#r^WD8F>+H%;^Skzp!?ykt`2|v=#f)G!h)uW*VVMNy8P|`PSFfr
z_TgUj`;E!R`9vF(l=MAy8(+5U-*5l^>Y9^tFL2y@^`g;v;&+iQn+K;FRb}Gt^#1wz
zSy)ukGbBXh-@kve0wU#3rWjq^UtfQ$U*7-Yqobhd1y(MR0E16ntgNgTH>G-)y}Q#n
zd$#oFXJ;>q2q-C?bO6~fd9tvyt?kBbM$Zq-@#nDHAHL9I(rfo?kM=KN-&y=T?f16a
zGcydEA3R8~Du1`8;K@WDHfH<(e~cd=?^jn)Xt=dCyZFTgMONkYXO4D@|9sHQe`9B{
z`rhjAY4B8Ysy=jG%*?N^uO;S*9p4vuc1Q8k4Nii2b8nnk*eZKs-rdHv(c1!61Sm?G
zX1TQU$==ves2sf9FVy2x#NS_EK{qtBu(1WLkF(vDcXw5Q%F&;{e+RD))9sVDKj&}%
z_sEe>VN;KZ9UewO<rhja9v|y9$-2VP92%xo9^aqam&B#o%w@H?yViJiQhN4Lv;6#1
z)Ai?niSh8@@bmLyVP!3RaY2zwTyM>RCmZ+d-CI~%s%m31C-d^Mk0;gVYlwIX-rkn`
zc(T8pqLLD*y?1$6X?ClU(xfvbk-D<0!q>}ni|ecDDGAt@hbu9i*Pg<o@#9h5xAU2e
zQ<qB4lQy#nU~oEAGub%pjDm^Dls;MOMXwz{scg-<n)LkK+$U#dPChr++PLmdMXS=u
zyqA}j?%1^}$!KQ8{<^(f4xqZ~lg%uX%tL5>ZJsCRt}cEv|E9=d%~__a!-KbGU3K5L
zF7NKHFE1`O=ic6CSop|=Ro(x|lP4|x{rs%lVo!Fz-?zB_|Gy*0k84YKiV3UxEpcw=
z+w<|5^q;@q?_XIN?7pz0V~Id1yYKU5zO&u><!m<;>o_ib_j<#YTIYpJiWANKH<m{+
zKT=`>B`Ep29}g$T%Zs_~EPA@5=qVSdrmgt+h$~R;^|iIpQ}kkIEm)wCl$2CZTAKRo
z%*<Kd0dX@<rX(F`VDy=1Gjo!v_oJ6DS5C}ubqSgg#dLY+ySuxM^X^zUt^a>|pQLX`
zFHe}p%ZQW<u7B>>E2+Iv;}+Me$ztS4$j&~!CUSE}PtTIt-`|cLJ*w$Zxdt@O{QCNO
z<J40kckbMY*jc37D{Zc(q2clHL~(al*P__nWgshGU0uC7`??+|N3Cxt{PE%8VNvH_
z-`?$h&$OZNtWeXrH%j3@(`HEOiuhOw%U1E;w~y)IusXgba`U{`akH9Oxr=^(%eAqw
zIWa|3*fi@(z=BT`TDirIik^6Y;%0_na>l(qm6r-P=iRl6iHX^;eS3JoC!f6g`+PU2
zo&E6T%aY8?%htr~oOEWUF{r!<cev|lJYoI5UtJ#`AAcOMMJ&HOqG{&NXVG5zNr#wk
zTAyvTzTRh)c}b<}+nb+vPJ=w`)XFu-wtCv$>hF)9JXw;Udh+S%>4il_hYlTLs;{pv
zdU9gompeNOAOHCEdOhgo%+u5Lm*?Ev#N{*Z#iga*HGe)Hm$9vyGS9a9)9dy7v$!|}
zCx&~>-ywP8y#0Tl?Rj@OY+jx{$ojUkzdKapr9;jY*>7iaPR-4Bm$a?=!Y$gsaP8W)
zGc%3bZ`_EuwLRZIXH6gf&gI3=&#|zw8s^^G!p`N;@bB;M&(G)A7kz%_`|8@-)7x@y
zi_EaOHc{DqMd)fV_kOvj$K~r&E-mqd)F577vy4)?a&K*Ucx>ZWnYcT=lij{~Uw)LN
zbi;n*)W`J<3<?aME{-9)Sr3YT2X(I4_+)+ron9TaRV%*!uj%h^Z<DXA2;_=rXklDt
zmUF}5_qVq{e?0D&|2ZpcZPde;FF6}luh#aSt`}PHX~O>h|7L@3J^%Zw^x4_j#~T=#
zMY=jpWG{HPKr$dFM<*{YZ%xF;CY!0Z1DCyAFTumcI{Da<<8GVl^Stv|a~Ar~xAU1{
zpcpw#T2|I~sn=ACx<3|nc6NLIe!I<O&1_xz3N*2D{NTaH!|nW&RlTRF@B4RQp|eTG
zg$C{LbsrA%+g|{+4K)}*Jx{&ZT?v_)C*$k?y58MYDhhJNqG<<sPTT=cvIeh@v(@{3
z?&*EgW)Ux`<gWh57Y_N|;kx0p+;8rWr_<vn<=&h&Z5pV_)$r}@ZDCp2wFit}wYL4J
zuqt`sP*GvAIqhs!79&T%x){rvni`GJRU*yo{Lg0R?{mB^m3nE3CupRFoxOd@H3#s5
ztMxOME>#7k61QHdIrlAFC(lzB^@`GD@Kxi}|Jn1}J?!$PRPSZ}^XI+M-I>4ltJwXz
z-@5<){R7RfEldx1e5^P5c%N*|$D`usO$*CGHQ&x+^{cDH(?2~qxh4C0--;C)tg{^L
z*R%1-oM>QV)`;6<vAg^|s2j@V=JLcexcmZ-o{9=fdV2bwKYui?J*eio{%UnV(9XrD
ze;0n7Khq@d&WzO4(-IRW8eOhEa_m^p+9*>#8H)pbveqlY*3LR~-r>*Rzo415RxZ(`
z|Ns8pxN*bgyOENza^~%Ax~He<E_Umc(uvzM1JvKW!=q$nHEUh$?ne(F9<+YH=kTkm
zt543e-Q5HZhn0>A2@0m!*Pfi1s63@a*1fTOm)4XJt%SGR=lrb}Dt&b&@$Idx5s^#Z
zn8>^L$t+Ad+9jeBF@c$#Z%=@tLZ-iu4^QZ-kcS^09)5FY=j2tPtM`4_^yu;9mzS6G
zPge6?RPyo?D0S_r{H*n)M9;!vhQHm<mbbUJKYsP<)S*`HuRj%)CUvc_Uh&|;gNEtT
z#q-5<Pko&FcPD5}V2RW6J#|ms+@D<Wd~SKf&Z5*V@dk#}Q&S%Pe!m}7;~s2g&sI+n
zxL^O@mYq)~;lY7M{ZIy`88c^oyq&+_Fzt*)pPa4N+NiA-MNc}M{Eq9z@AJ94tMul+
zTI>0>-z1?!O_NToPz{(g$G(2vE7Pokmyc9WZJp<PZqqt0)u|S{|C%#h<6Y>~%2iQa
zZGV|T_N1zv-8?DNtOt)DH*eo=Zkm1V3ZLh3P{(am_<Fyan^JFXPWRtbW*)a8fl=0~
zL?L*Yk51&Kmhbnf{Tr7XAMcYreD&&7@9BDmSywb-ZG=F}{xT1C3ahX9c(3}-j>_&o
zp}P|@r#mm)llHJNeaD`1yQN-JlYW1D3#y~m*T^z4F|7z#$h5Qg`LhQHn{V8_nR$GP
zv!I}0$cli*;^*f!mcNgiGiT1EzqWh!?8&&a#PibMr1g(0rcDtP6Fc_h<z-G@-qd3~
zl3Vic$1Mc4?5AuwbpQVRS?~Az2P7;w<e|zXu6d|$|9>_Ac{b~FW!3!Vc--Gt3+f{V
zFY~$R=h<HM{oULvrJrZ#@AKUF-Ll{T1Fy8%lUG+)bJ@<_xOHpl%}uFS*2l*uLX1|k
zu$VDf-CwKY$_Mu>gMd{bE3eo2_nu5Sbn~WUety1L-klYd`!3DYjo$X*AiI3P-m21V
zxwqS9MxMH7Bqt^1<>1ipOu#3dOH^x#paZDI-Q`hL^z4k}sj1rT+j4GN)cp9Mz1KW2
z$giNlAocV#RRe<w&FuW2p3TnJ5)l-fSmLv|x$A_ncFYcg$H#gfA8O_P@^-bQ>g8^`
z+Fu0^53wHWl?IJQxu`G)crWO%+0Mo*_2A{plc4U;ogEvQJP(JhiEx~(=9_V00psiI
z>%$e4Kuu^@7rC;Jk6h=>nX_Tz#)l^+DqC1sXiPDAajaK*ie_-zjT;ev{{D@BeZW~s
zN$Jz871k>{Y_`|@`~+%<tc%<0b$*_$sE5?mz18J;cXk|na&j`L2iK9Jki{u7QT*Af
zSEp8mu8!DMqIq_<dALI6i90)sGw<)S&CAQviQMGU*Z~Sx5y8q`+1J+vF7uJ(VPkgh
zlR0?tV&K9hjG+Ei&dp6XZrphA{{4EzzpgGWMiQm$q7%iZ8K?WH`OiCZcDDJ;dwX{?
zc{2N`3D31ERkE;<C@(JuwFZ>5m|R`v`JbMqn|W~&t9IC$gnxg2a;Z$6ZBuC!7Z(Sb
zYZez5XH`;Cn#8!$dPT>cEz#TaKu%b+Xb~uHu}OL^^_yei*vchZP*U>b!b0bib8{jW
zc64;4IDq^%eVT1`*)reR$KKxF{_@h&ZYIyxM~@z5Twf<EC@6Slo^3Sif)3DNir~cQ
zk6*p=f{sbPd-sk@WvZlc8jqcw-Iuqwr+>d+9}fzJl!RA&T1qFMeS3S`!qQSuNa)bR
z!|j?88x}ZuvhvB<cuds_)ri`nQCC+N5gn}!N)0?dx{XScayF%%J=G&=yrcLz-(0Iw
zr|UD~_P=<M0h&m8^!V{i>+*G=#wI9FDxF+i_xD%H>ubHb(c26vKc%pKn#C(?HRW(S
zKWL6rD{RdQ2NxF?m%EOxDnDD>+Jd&_M6Qe9f9_y2yJpmu6;74BWp8gCt$x4v`0f1t
zXW!l3ZD?*DuA!u)v}l@xpl4lKS=pIcrrjMK9H6E|P0b#s%A<!49RdXisD7Jk9S#nV
z6&xZHr-ZH!)2;gY>gc-I-4gb7dzdPZ?)&#E+i$MbQGWYB4l9F}ih!;IR|zZ+5cD)e
zDhl%N+j&jbTN@D92wG6Gzpggx>Z+sDbfXO<c(`0$TtF+9PpsZp{XH-1%8G@*FU3v#
zz5Jy`$qRv2F3}*6OFuDyQjf^jbJp)K?60r?_U`WK7Z(?cdT_nIxmkU>UhJdk@pXZW
zAeH?~K%OcL3=9le6EU%&f#KVmo6fCVq9R^gA3l6IaQ1AkTd&l{s;^m~1O!TIPrlyV
zo-ZG|I&5jl%S%6g{YuKrTp2Lw;@0fz1w}=NQj8wHfA7C8W~WG3M@NUvq+QKjC$v9)
zyPXee1o6pQ9r=F0e!at_$m{E3cUFE@1ND!<&hQ60qcSxqDQQL2R;_kE*`(|1VnbF2
zxh_;`o@tzZX=ibI)b_lyO|0DD5CaA7mBaV%&+ih|4qD<N`0f4u{;gZLI#qI}pP#oh
zW@k~#xjB;c_4S)lPcI8_adCOlc||tFB~DXaUA>K8{@lmM$A5mioiBRA+hC>-Xtuz*
z{GCDZGaqnv`s8y(NJD9TOiYZ-)z#t0uV0^@dwW~pVmIE_PfTh)GaUBU{e5w9F?;Yb
zpFmK#vPdbwC1`#9|G(3>->d4jsr*#%;elgLZm#I5VvDjj68Zc8npJ*&7Fq`?RARuc
z=oV7-0=29PUR}`yje@g&l1WNV&b+=(_Tt5hXJ#6M79xU6i-}u6&9^ISA}4DGFMIIx
z=~9Pz7y0e~9Jsf)`sMxo_E%SjgG;tg5Mx(gUhW^fucmTS>gl%q`|ZDd`^NQ@^Ut3@
zpz2p6a+AuwefuIJBQ-??1qCbhKpFe`DvyauJ+ju*xW)A@Y|oEp{lqeH;zUq0t6^vH
z^HmHWm7<MGlgz%pxY%6v^_5}vHJxkgVpluJJ-(g4|LKd1i+QBYdO(A(0w6OLLHT;+
zp`+d6g+)b5!OMJp{Qmv;!Gi?>?>5)|F3Y;U&UdO-=#7oZ>~4z#L8Z$H?m+Ja9ZR@=
zetMdGsD(3TXHl#Ez8_AFchpTxrtryHCFJIwonu)H8V3d!FP`$ChT`L6y^%W#6hYO$
zLg%lquaD0(PM0uF<2l+b4hf7PP#6bIT^+tYXqk^>kDKzpfB!73tkPcCTw5F6-rda&
zY7$*u<}1O&CaVI9lps)auk>0MvvY<`rIC5wodxdwa*|fv(cAMb?krB<lydUWn#j!v
zXUw<z`9v7hQ`=W#$t|XHVuNMPzn{-z_Et@eum3ys&(F`V4+sbfPE1+&Zh_Jyt-E_F
zh1J#77cE}=@%wjm|9Lk2HudlBSb}<}_xIIa&Y6BuSlurnJ>C7?ot^Skc^4PCZppgZ
zrRF>9!}a+3mknT(7K5_r)O!_=c|qC&0t8}rl~g{kEH5wT64g5L;^N|;KcCOP{N$Uf
zo7<;vxAP~f`^zo!o&D`V@pS$8W$EYVSy)-weVGF)X}TN)J-5fi#{T?r**|iBo$b4K
z?_&1V%xvS8zO*&_`jX#|_w3no>B+XYH#Rn3xDdcl_2q@*;Wpl#H9w6QVt17&8W>E-
zyu56pvOAwF2dF+-0}ku0x3}j%e)6Pc?p)dM^>L~e7C#R18#Xu2HqT$S^W;-@9*G0~
z_J6+|NRHf`wls2c8fX-z?CmYX^m8(@3?Nh9fFmHPq{Kwkd)kxx_5W2>RGut+?yDvo
zy)DPH`1aG6FMnRhn1Ar#L6BCTnMQ^+KMLN~2!flxN|N2}?d+h2*5_wuJz4KBUb0nh
z;q~?L&mSCY=FdHUcmCnShbuu1(CvA1`Iw-3UdAURD6Eazdg$!g-USO5sPDU$V6Y+c
z@-maHwdLKnmWy`j>FI$+`>e~~y;uNB+@NewXPR-L!L?hAu|NOs?$aqog}=UJo||hu
zJ#E|j$gXR<Z@+!_&hO$P*PRs~7xD4HESlWi&3(Z6`x#r$s8>R-UcUSpvwnZwUn}#x
zI~N)iA*~XXS}C&}3GMK8D&Es{&i}DKf9a@^sYTtNic28^=ausB?EwvM?D={v+R6Y_
zYq>7ySP~Cf`Pt4V`)F?Yy~<_R^KWhG++y(f?6F?y`M+lE)?Kh}-8vbok`5`;tQVjZ
zBnWc;zo+Nt&wqP+yK%vT1MFRk-9g1~JHNc&*V|uozBA97J^S<R{QVn~j&_-t{|9By
z6&xahm733<J?oLPoy99{Ho>;~oBBSpt@p(xgoQs}vE=O*)oxqAUO&F(V{6$yNb_c5
z52)q^Ssk@C%l4b;%D?*@<mKfL9X>2y^(T)z{r9)GH}}`uKRrGDI-j^8XlO`D2{bnM
z`Po?>c{`crgC7FE2ju7HCnY5Xz5QjsPJ2afFYo7PXOq9Zxq0K(tt|I$XiH-Wzpk-y
z8?W@SXJ=;@mXy>y*!kf@K}}6f(A!=4cXoVSc<Afe?e`uXXk^}!dppcJ98zO;*!Y3g
zsMY*1`1<N<^@ERBZmE9x_VzZY4l|lLiR++U?XL%rg??L{U0qzBOf{Tw>ClW<N-s}b
zT^-)p)fFUg@<@u2Vd^Q7x?eAsFFAg0p6%>c*NUH>n)>DCW%dT0$W0$UpSS;fkX=55
zU2XB5oyEzSnJX0_b}y0JoOX6m@Nz#cCl%HmyLN%5%2*YZls<_(>vDK<)w1{*55s$_
zl!X;(ybP1>JbL7NWR_2aWN^Iy_YE^F*15R`T1@6mOJjSxCdKUcom3_!CXp6!P)xj%
z8PI92nYUaiKxJpne!(RIE-o(fCbG2&{<N?DE*l>D_juu-(w*XZ1%jfY;>*%3jHUSs
zrbG*W*9kx86+P>pOlsE5sf%M$XGKoaK0A?Flszi)osMa_-kcQ*C#F`u|NFP_qf%E_
z8tbn*JGuSG_4S_5`E5JD+TznQ>poC<(%}SZ`*n15Xs~pF`BW7k-S89p4=)kwNMUA^
zzti>PXu<N|N8^u_{VV!r_qFC|!RKAOKbuWk-+12Gv|#@qZt;CTKidDi)8qQ--jd#r
zH`dQNocMS8tt8F1onf*yr*G`ANMkGe8`25(n}^sxN8Xp-6U|cJSJ>aaC~sj{6SvK-
z-21xY6Yc4r`Mwn#PT2I=W9KrycE11Wj0WG`gHMDWs5pK0P3Lq^t6%DYo7E;sJlyg9
z?T3}ljr_YGJ$;?iuKZMgk)tB0aiSRDvZ?#Y<lmjYKO1ZliLZHW*V`^BqmuZy()Obv
zpAD1OgJ3&>#ey=LeL3~}3Kz$#o%<kBd7MYV-7bB?#`*szbDu72=)WnMGij26Xcfz8
zVgE-nZ~L5DDH~(;q&xnr1J8l_AXayoi_F$S(VtVf%4^xuPv;(K)mMBMAtTrL^kd~K
zPMbZ&4<fBPn%>)=T&mf;W6R4kws|K!_1f1T{O6;qv-50%_rZno%7TJ|tqy`G<<_<I
zKD;yM{+xJ;KOa|pHIZBSP^s>u`H{Kh|ElDtE>?84t9^K`y+19qFE-}aQ~e?n{g*tA
z;@^}07Cz;jeYEbk1Zz6avB#$(la}YJxqm%y<=3C`9D6;bg?Ei3_8dGY`Y2pS;oqL-
zAM)Ws#&_H2INKe2FBs2LF=6#*?<2WVey41GFS2jYvtwvHcuoM+faSN*<x!rbJENeW
z^TldLfri&=rz+!wf9kDQdbWny+`s?#a+Z_VU6dEUT$X6~?K{W1KmXJ%lsxuL5|w&4
z^<A;gcT3Ah9W!@H-M+ehul%9aC+v%(&1PJ>cYT4phM?d?_CMy{f+s%(IXy2pxSHi;
z`3C<x-#;t-?<`p_seSFBwt_v|B#+J;j(_?Vo4<J;w^>rIbN7ZktFDbTvU|LqA1_c9
z%6@T6`q{*tQ`a|!cm9!>w0=TKhW<8|4f8F^4i`9YpLF%3R`dbMCzBp|{Hc}Id~Pf;
z<wY>p`(3FXe;%^SJbnEj*2?~y^6VXx9If+xESH|RX*)yvF<SzwfymysdfRiRAC(VI
z-2X+=_?Jl8r-U<aXa4*8Blw1rp0#)rCyxhjB<Br2xzk53oPKgU|8X$a`t(!DpS~R^
zRGsvLH+y63{slb;<%{no+|hk^W3m6F9p7~(d_3e(b|7Gr)wE6gi3RObY9AhXZ&`h~
z?ARyM4?2M?=HFA^Og}eq!?WdE4pbdB-OT#>nT5@(YX|3?X8yf2L#XcTg1IyI@PT?K
ziUCg!KC^JR^PT-&hWPJqsz>7fF3NEJ^u3-j|9<xN4A)a9&(5i=X4*FC{G{cvx_i1r
z4yLgEK50~8#&cx*jP2R4ZF;kdc3l^k6vXVlPEq;9-wu7gXrVHjImH5=j~-1}7t`m_
z+ue5j#OHt7KiGaO%a1w!PUS7<<1;fPZ}8exT*>x2m*p7I^^|$ij-Wk8(~fsr`_J7K
z^Wy&8mfLd+t}gp=wL+-+#&#9=J83s3`)ujXiq*LEu(Ra8-yO4V@6N3!<M#&qlGvVZ
zr+!@Ln82N>O8TL`bK51(xBk1^xw~MtLe#b^rsj_%f)-tJck({}{lj{>8UKnhZqK_B
zncNYz`Fy2S=x*T;L9sp!Z@rY<@}F<;zkM;lk7xeWf4stq?&tYGtd6sJyY-lF$N!(&
z=jwJI`?<AV_S62WobT_X$*3^Oohz=|$NRi}Z=>m<zUZG;PuBeAnP$|uWA;J8XxaPU
z7%Sp}gWQjAbMgNpZ)lxlR4Tjdg?g-u%aizp4;LtX-z~$-S@E8APmR9t->cO!Z$Gh#
z|Fht4&{XI+^uC<6**vb_f_VXR0rv`rz9hCs_5bJoc;S+M;e(#zz3H#5zi0pB+A06>
zsMyAN$#>%(xO|`I^0$<A!_u!=J42T}xN`6Z+mA^_pVIyIe6Vrf{iS5@_T!6rIVA3h
z>V$sCzQ{2@>8C-$;Rg0;$Ga!jJk6RvS+F_%7h@0G_dPqNuW4}qdAv|Rp0W6^xW<P&
zlPx;-*RGFJyPX<uJ8Sa7_DfCo-rbz@&t!`B<zIzW$x-~-H8&*u`Xb-(*zWgxbmf&Y
zLrkR2HB)zEU#`B@isv6RyFb3Jap$Y&_43@`^=kKKZR`J@E&O4<oX4k!1xi)+=Ra=r
z_T1FSs9#i8GkZ?I)WHC*|9Yi{lU%MGnQnP;HTQ9~wWZ%~ELFKXqrTSrVfbE)lj+}u
zH@}^*y8QZPH}5^VYz99jf1fK?@!Txg`m^!z_=?8l*tTNJTWtqtOs`r0<q*RT`><Y_
zr*j!ERNi4Yv+tvdjGa$V(yQzz3g2J9f1GNV^3Dp>Z)$ZA+{rGv!C(4L<Bz9{cYJwl
zSKv}qdFQywk~AIXiRvG1c6M~=*=~$0JSSePGEZewefhaQ>7zHz)hEx`wN^7<k717-
zug6)Fm6PAe6dv)3Y+~lW?tApob5@(P4>rtwa_9EsxeJd79)DVxU-)v(=Z0gCzMd4`
zGkx=3tD=7mqU+Wz+?j8tUcLWJ>j&Q>EoTo`#h#zHF@l%ro}A0Wf6SaO>^q!fr5-(<
zy|XH1SL|1=Nq_z_h0pa-)!J3{MrgTy$<-sDPfBGLzu~d_VG}0P{@CK8{hPgp4HKF7
zZ?{`;V8uz1xrV)ma#t;rE%w(~ZWl9UTj9#*C!;UQyPt}mBWc(xqJ5+M{L?$Dl-CvH
ztc+xrs}kBQna<E(%J;V<XTIN+#+KP<IJYM9&OULn@x9R3mjQn5a-O--dN$f(3`NO%
zcGdaZVNG9ACZq7LWX@;P{Y?^$M{d3^eAK&h-NS|7Uf5fHbuvHKQ_&u*&#SKY?Um(o
z)$--bjZ0;#B2A(LU0j~rUjWJqU*}h9O<ugC*kgJ5<MPTC_pUOQr}Z4$5`W3o`?t{l
z=yh9W6qE>uultr{d2Zq)j<ok%+;wm0sWRM?=x!JI(5L_0Zl=$tp9Ok<<9b>&tR=c0
z1>drqf37S=`sJ_1GKDcwJDX!-wXJW5E@czHen$P{hfEF8j^>I*Y3mu<=bpar<6}KF
zn}6ab@rjqs8;&whnq=@q-+A4x(Ad2}=B46qyKV}%rnB1f-0Pp9Y@vQ}cG7`;Q?;K9
z8;Tn^@8kL*QdrE_Si6FAA1jYaKKK5Ue|)9c_A6CXF1NM5P?)AH*lf2uaot?^50B4z
z<Y+k_5I2t5YCh|--=)PF3T-R)7d&_)`$hVXWM1+92bX5eJ+d~g-~HqMyo|=jrxNwf
z+?;Z4k+)&ye}yOKc;_hmdr@h_T)QFf=gptJHKGR3OZ@LXoaOFQv2*vo+jV*Qw=Avs
z3hHjL>HN*Is^;xxpR63Ow*IiJ;jf*df`Wk@A`^FqE#X*pWoOmL+ht3l6E07j)N}9&
zpPI+Mx}|}S*XHwnoL&BS#gCrP;(y$;0xF7JA02vLlo*qh@p*m6ab2dFkC*4Fl&$Nz
z$#LvZsv%RK_RRQiYEM7#t^2d<+pO?wMNjweAKf)&_t#U?f4<uj8}4MDwElvKdC!k4
znGRo;{S0_jS;IXe_;9zl<*chuKP_MVF)vh>+4G2A<m`_hcOLo~r~k+-f$QJXBCB|v
z2*%Tk^Nzgpynj1WZ_@RdJI~2K`}FU#M2x(r5#OD;w%ar(N<1<v`kCWiByNyo@Jy-v
zIOC6Ne<y#Q-=~sQ&DNaU`!bw;>;HZ8r||hN`J4QpL;2g`>vKPQdfR#L3q1T$^TF)A
z|2(nvXW!g&s7dnOb2i~bw9I#(_;#*FB_$=UMx~<fk1kie7c#4HJ0o}d?Dn%qvSzJ|
zzq8%<;gj!iB6-1*+gCfTC^N6o3VZw|`>^LDvwp<_mmO8&hO^%Nn8?buxSB2Z*UTHo
z&)+d9`O;SVeoHL-^unJt9G0IY4cV?8X#Tll`>L6G+?6(AqUMJSi^4uMpXfbucMGqY
zkL#TsHqV^({Qq>w>|5#Va$m;PY5V`BuV<FrnB#eJf`H?f>G5|y&C%nPYAigHQ1jWc
zdF4h2ORhck?K_?dTmJvM>~_dy+r!z4*B{h9(VNQO{+3C6`JVR0Ve+4*ecIdXugQ4)
z>S^7hZHJ$%`J1~QG>P%^c&6Wi6Se0UBt9JYyfH{JKVTD|-Pd_LHYeMK@ZA1XDD(e*
z`jL4(zSVcPi?G_in!C$t+2yNpTW4={|5utH*4lIX#L+_I4fpj9+{@f8Q+OoLVnWpE
zJx^*DvhVc%nbgmrn)FTY`LSn@p1l6gG_Rrb@TzD%zsGZu%ja+_?<<_U{h)-y!jFHA
z_9#ko+0|U$f34CqzjwDP`>}@0F>mg-1}}`heD%`ukJ?YFw4Tex8T1DxymAM5>-W#M
zE+y5yH++8^eO%EUedSY5O4i?ZZ%aY#N>JQ!$MIG@oO{0DvHqXtstMbpPM(of*~)x;
z`Eib}KkE%$@}4}ptzTI4%4^bky$=t=IUfI<-?@Jl@80fY&OPtKk~NyDUtLyzGyUB1
zw#D@e%U|f-Vx6!_>C9Gv9{;yiJCc>nO7t|d>;2w-{c_;<#YYxG{er^3!k^s|+SwHx
zpYL1!`i`ik$&R<ZGtY!fVXBqB|6y14?e7{F7Fo_)DlPosAIlzHwn@+Wl^30sc6q!1
zR++M#D_8mt$z`7QJlEgOWUE-pH*@bMyWEH3?p=@7e&5&eS$6eOndZBDaqXayYypi=
z30?mGQ=QmqO`jj`uzvq%?OlhsKkju6<qL{E=0~4QKWSfRFz=YkqO|(?|0=fi{C;*v
z=iJd`8BWa|Q~8R%F1cC%cUtiMw_dj|2ztJGwtBT+)r8HlzrQdpE3LI*V4vRkPfdg&
z_@GU=a(LVGqz~Wr<o>?&euDhupSvCui@z<J_aH3koBQ&)abNuOayK>BDjTw2s&Wec
zwod)!v|g`+g=cr(H1BzQ=h@}V_wl<}f8Ms1mrT67LH+I5<JJKWo(eqw5YlWvZQgr9
z0ilk!FSPXBRURJTo?pek`<&sk#osr^?cDXb)c3==&f})53#3j>EBI<+HA%Se^jtIh
zD<KA7f4107{iye(*fcz?>-gvM%dUN53XJ#J+x!~T853c0ec~o2Z!@d>p!U|~M;pX<
zR+Su_qVN9etxTcI`P(m~PNf@&EB~LcQsi%znCty_J=^tF{{0c%*#1W1lzGXyR_30c
z-phC$YgYK&+Wde^<MQl(u(bfJkEWh~P&04-N)}f3^3r=QJa$hi_H-EEm~+pcYl6vh
zXT7;4b_VA}B%B(jM<+1&e2rh6&|w@CF}=R;>mFvFr5kJ}seg^x^Ks3$ie1H}+L7%0
zrTBNp8QFz!zIW%ov&rSy^3@OON}ryLVU4}{wC`3a@9Rlt*DYLDxUe|Jv^V;nG~XQl
z_I}R2d5#jz{Ljo}Ka|z_S)RDLk=L!eKYOxvaNABfU&aS_AJqTYAHaWDK7M)iUF{vp
zNB7Ni-6(l3o!u%&-O@ZwEd2VHk9im6?=PQYYG@OsUTS*1La6ROf7_xR(d+b5CiAaf
zmC4`rT}ya^*srcB$MSq%OMY4N(TMT!@}M`H_x*Tcwk*zhsm7n9F>+G(4DYU-#S}f^
z|5LLa_EY4GZ*SKKx&N~=QoP=E&)1@5kTT`DqzqrXbE?44avoW$DT2yw8+3kndi{!c
z_fFy8S`DLRUwxl<9@UjQ^i%#4zh}wnn{yA%eVw#xnfUR<-7aq~>394}R9|*1KX8ff
z&o4hGddK{!vfjD-n(y_Wj~@t4-*#;B?oIaB%ehtMZ=boDD17PJQprCb-`<{oN3?56
z{SoI!tc%k7m7g6~esF(^Qu4h04{KyPk6ZkGl)w{fw;*8cBguK$r~Vyvsj>Lq_F(Gk
zzy2p>3mzLSnErhK+13lnGqbPrmV3*8@|okt_nA>CPG+8&)};WE|10Gm=H5N?ai8#|
zXXhll9@+m?*12accj{p3x$pIwx*5Admp_;i{$bCulYKYrEk6E=scGj?`L4Wk{p!zU
zO78VOlSDtP^jp}u{sG?($;<oQ&#WnQS%2Djp?>_qnppmOQ$JKLnUde0zwdSSuAbBL
zN>9Y<tbKU0{jlZ!qwP(yTFcLU?|FVH{C#oeePQ#xVzS%3kDQt7@nu%+%zb?S`_k)g
z_U+6t{PyndYU$pNjt-{<Cu+r*-v3&Dm;LhMrv=8$oQ5jzZ=S#Z@1gmR<f<(Sd1?MK
z&%aHQDf}sRTlDx@p1u~3N#AA8DwQ9w&Yr$7Mbt*-w}*|$Y`c`{>l<HQdarW#&7c2y
z#`0RNyZ0Asto!>TF6dnPAG<vMsfS+7=WbS>I7N2<=ki}y?|6kBm~yz`?GgDKL5EjG
zpXA!{yK2&=d~UPsU-lj<?`NK0dH(yOIIHVx*RQXCWEH|8A}F|1xyq&GMDrm*_fzk5
zzyFvqamky$J>|FmzBb*txy+I;PtmaX`rF<0`_9H3*njhkFuR9U;qSL^)cF$Z3-j+?
zuKUn0eePu4hkl;(@&^*ijtiDIEP4C?Ah-%BcPVCab$RlB^U@V;klI|S$llmy%IUBD
z9ntHQX2x2CxVX6dOP1o(Qc@~Ev-$dQ5gx54zizE{5a6#BcKP)0=bS$U8@W`Jloo-8
zD!xx{Isdz`#_^x@{=%cH9zjRVPW(Ml_is*@f|AlC-b_E}a2n|&UnC5?;WRu&S#7Gr
zll#{zf9uI@tXM6e^!>y{?T=FeC0$%xM3`JR&00RcZq}a4&lBcMIiuL_$a?b7n=<S0
zM~^=ilpjBQ*qQa@qoQBGE^SD37IkuQQ4!|hYd^fusH0=a_Vn{|r>5(lzp&7`^84NL
zP|%3_&issl^(-|;EOCb902}@%*+;9f!{!BO<v{-azti4roiRhArly92oBQ#VmBBCf
zJ%9cB^aMrc51&5=e{^wiQMqksA;Z^xN<oR=C_g{{(c{POGv=4Rz81K<ELYUu#l>Zl
zEHAt0ME1DJYQFC>suy4MxVkDd)B-fg!671evd^mQO@;gI@)s8v{r&w#{UFK3Wz*s|
zUg=5N;p=?z@9(Rv7WIXRYnPXo-`bKnd1dbBXJ@_VT9t;Fz>GR)bbI&pb+OhyYPxH5
zl$4ZiP7h-j6=Zx<X3faR2wFcFeOIjh=TmWAE@hDHtSTl~7Y1{4^Lh3Ee$HldXg+wr
zncvo>pkPCnDm2G_Z;s~WEVYo?l7C;W`QQSl&W;X`ZOa}mP*S?i!Nc=p{r-QaHn<mD
zUl%(?KR(V?NKkNMj<0Q_lG61T7Zx7oWxSAJ09uj8>Hu}&&ig4QDqmg&1PBTWPW;{2
z*4)(*^L*v<d7zcBFRNJkWh|XOJ~|pIBPb|%^4k?*4W**F-`?Kd&9vZP!UC_UT3ii~
z)|XPz{;GF7pU13ju#({e?R;3&(a~|D*7>lbtBcWxGsfpXT=utTEco^&^3(J4>miZ{
zmk4#7*t;$1D3_RCOvl~4^V^<Jm@$LHLSD%}<HCZRzm-#TA}9U-`@VjL&#tGSnUX3d
z*C%qbEee%te|>S>K5dPvib~7R&(9yfeY-XZT>88+Hx>N(TZWIFnT;pm&5eyVpH3*B
znQJ}0p@Bj4nf<gmbIvp{GXHqD`~8ZLl|l;_E_`x&x_>JyAgVn*JwcOXmc`EkRtBkx
z>BV@=v#AW3p_6iBLn3HeSxD7OKupZ7Q%E%gIwUgDT3lM%`}8zj&>o;(Dbq){^Y<S;
zecHSAWE1Gro&9xxPs}#g*A82=An9nA7HCGnX+ej_v$>YVZEd{Lo3gL#ftJeu`1sgR
zs#nWLY+K38pj{;|n=WRgJUrBDl5-=#6*Lij;Du$QQc?J~_xJq+0t9xKy-hkXLD6TH
zi6^6hbjGbMnhO^$tf;8Cu%j?J?aYjYps)pH*E_e?@Bim@bycWk=_`@R>i%gT9vsy2
zG82-KIkT(uwMN7Sh1lI?sZUN!1i9-3v&&&e*C&0T<zdg~Rr8sfn}c>kK5V&pW2SL>
z#=Sk3)!*JM3|}8t^XujE9Xns<H!CSAeYgIbt*<oGEcKK~)V7??{(k<|VQYgJ&O8Gh
zC^5@4yW-c&<v;&?KA(AYmFUNhAGLbG)o=#~kEBsc+}^6dEg6A_4j-<3wQ_mLl-~{O
z*Xw&v(^**h`r4llhxxCp3~pbsLPONe#pQ_|D0LTkYieqKd2@5Jq;cAZBf|b0GA=6p
z`TI9?#_yiP4--B-IOsFWq%&x#m!O!K+r>q$p%ReM{mKauYr{Y*EmnoDUJ<@tZgcwi
zV@VrV7<}L8Iav*~Bb7@;;{Zr5U|~}iG%52pojBnkWtw%QPuBXyjg89lYrjdVP4;Z9
zWYCG&&@j)oI>~Hy<emz{(pOgk16^EP-YL%#)=)CHsQF=V@ZiBWH#RmiGP4<!z6xPI
z!0z$-+S<rXDV%K0jcvTrM(O8dL?I)}F7Lu`Z_8D-u$Ym2yw5Q45K9}c^s+@Z4fE=L
zWnNno>AbJz=ZgyqnUjz8K<p3(Rmysxq~tr>?8e(N&_W9-^Sm{SY##W{v+>-Vb{4c<
zFn`ZSw)r)mI_IfsC@D=k`!c>rkTK!cmzO5Fw?sDI^vTFr!L)!uKtSNYt*zOde0*t#
zS~zXK-zip6Q(I*KDPJbd-TVDsHE7K`Xp3;(U8|<1rVx?BlXI=jL5ou(H>cJ7|N9-Z
zrOSJ|-b#?Bn^W4FyKD~Rm`U%g{{F06f8T?R$K^oV14MKp0$L~X+<aRG+DCR}Ww2)W
zI+@jBYn2of9H6;w;u{4!yLr#&Rrf9Ro*uM0%@?#ucK!PGT*})`%ictQ&SQFbsFjnO
zdvVOpB2YpFH49Qw#r5M<R8(4YqqkkymK(kA$0P1LckhNuoZMnr{OrQ&aD5#yZtv-O
zPhT#d9|nq`O`K;rMJf--SeNxYc#yCy=cdzqyV@VGR<9Rfc-pr$|NcDJZn4D8H$my+
ze*OQwuyhc4_S{@+PA;xTA08eCO&U*Bc26>p2$9&SUHa;ZqK;0_ot?#(*Tq_^`OR4Y
z%5^EsQhZvU8X`8Q@#;ozQ_<Aye0g~}Xw|N$>$}e6<9!#`L>lwS+xh(e_gB)kYRkVJ
zpf*zLO7(yzj0V})bfiqPS}ta!yu7rOtD(qWR6DHa!-oQ`&{aonY)pQ6Wu@@RlP6(W
z=#RKtuaxU@zqud2e@{O<%k;{MKxfy9)q4K(Y(R_J)92StySX_XbV`(7{Ju4Zz-43C
ziTy|Owtv2ry&jb8p*tS8Oxgcu@t!?<W|-&8f%d7ty|wkqioj;$^m8jfQ5)IwjHT;N
zyO59&C?9Z(>n*AJ`s&PV^ZuBfMJo+Htp{x@k+3XMaqSjU)zRq*TI#hDI@IvVzoV~j
zS?1+sId^tU6w{3|sQp!PCT%m9^8a0+4E*c$`pL%We$({h{XlgaXw?L3hQ*2#wJJ(V
zO<7l0Z7h8323lvf{eInQ295tS^<sA!<lnP-^!V}0*xhE{({#X@0@SySySca8+|kkT
z&F$^|-|yGY-<W*-(X(apg+KFcf4|ubI{ZgoULLexYl>zttl0j!8?;9K>#M7sJv}ME
zzP$YM_I9}IM4iP+N4r|u+k>}c1cL0Ipy&+JF!BF{XDnSQ#h@*&!OQ(BDk==h-^FBI
zU$=IV$21X1Nzc>MbU_>VVt17!-rABm!$%EVXvHs^=P)rR=g*Iipan?w_V!`x<JK~0
zM9$j3-##%h@yhD(eo5oB3rjqO*MzMGRsNtR_4daF7BBAax3B&E4YY(56#pSB798@J
zmcB6pbQqMZby>>QRiQSY&ltP7xxun*kuj*|dU2t#-~Qi>Q&Y7;dmg&PbXOhn*p_HC
zbB0x^7N|Nr-Y3h#&R+cELn6d|i!2+JZhGY8=$xHxp8ogO*Or!+4`<EqgO+6-^4MmY
zd&}h4x3}K&Y$_u*Cb3S{3Jn6~q<@L8I7LpX`OUZU{rc)EXvGa^tK_>oJGmMrz5V*;
z=H#8l&xORr-OtW4jog^j3d>2qzr9t~(CE0kyWF_sMSz;$oQ{hbA&F1+fxPeJ>})vu
ztV!;zDeL$B@&Z+OfgB>01%E%C)(=@5HTCjxf6z`6Ha?jZi%!(4YiM+U+Hora6nB-s
zKi46s4347u4p0;&c8lvj`}utS@eC8A>Tfx(uCHIe=tS+?Gc%31WL#{zySx1Jqi%hp
z%1<d@tGYWn?j(b}vPjK;p3myAwI9BIRn^h)VGNMtVC5D|`TFWAXqlS+z8_7~bfbmj
z<<~<q(VhGEs$T2HRX!E%64&>usIc(2`?;h`iJvDvKECe9L-t$Sa!+4c>b){_wV1em
zTu36QdJyDg7d`0|6chwnW(V4@=5O~?rCUr_DSls#R*%gD89sLBHlD;kKR&Xsv2CjQ
zn$;z$y$oibJZNp|-Cd@MiHRl|7aC?7rzdTU2$86KQ2Xs>I**KnKz)6E*t(dL!u~d}
zsy`+F`MJ4^0~fpf`F1-Wbb_4cWVJ-2nIRGrL%PLu7gc|MS5Z*`sy|;{UA-JsaA~k~
zrTDk=$u4qiW&`a?acpM$c+UEL%FJ!5QEQjPe4n-asME4*Sw22TW)<DNr{=Ho%sT$%
zMXAzB_v?O%7C&7UqW7fob#Ly)oXGFm;*)AuhHPj1_bj`=cvkK0?S)$DuCqcl=cKOh
z?0GdW%0Wl0_}69MEt3lV9y4FLS@)I1?fzRUe_T1d{U1;6#W%P9sc$?PH-)#3YroDW
zi|6O<|NHDNd%GcOZOz}W*Gt~q2s8&RAltoQ9oxkJ7pCb(b8vDhT3OBVoUC?nZS;1X
zD^}Xt_g0^@UidOREyt>gQ{F_YaCYr^<DE&dYSuX~<JMZgu|BrF(&VzP#nwVL-Z;6}
zzkaR#$(tw5^!VHQMZae&GiI3UO(}mRn(#S%@ym}VZLXUgY~FwRkfOokUCc9H@jRT)
zdA{(i!o$Pfb_=dndta(|Qx^Z=xcO4%OucU^+2S?6`~9CK>ex2S7q{touNx(>xtQ_o
zZvO)h?#-4jx0{-(6};Gu7gV`_es&gAqA&NGd#Tvi)y3sqQy9DG&wicweRHC=W+iTo
z`uT46`%MuHEE_H)T`~Lj<7u^()TLcb)AsJ2GI{$=<7fR_Uos`U-1h$CW%=rN8VpP#
z>qHJsf4{%P{r8MLH8KC^-apSYL-Y0q?#;{hxGJ<=F+N=-wDwv_Y5b4C=_U8&cV-yx
zj~8cP^x7(W@|}Eo(CG(z=JL<p{Bah8gzt^#wJQU*D*g>}-k>=Bcy3iu?wg41Vhw7C
zr5{W``YL2yj3ub(oN1IQVUi)Re*ZtKvfZ)fkk;_THE&Pr?_UzRIjy3y5_GtTlate?
zoakQ5(*fJdMXz7)d-77}-_tj`-!^>ue8>ONUZH?xZ|?GHnmkut@o{sDgi-zKPc=IP
zAD$L7tiPGH%YLdj!+j^$6#;w2{G~TN)75`?!bt1d4CgmnAFeCZce~>E`s<z2A9v33
zPY^oq+0I=2oO4T!&UMrK9~LD1d>--lve&P$Zzpn+Ic9I==V`pcuQyBeb?A<Q#B1wf
zy|-pvonewGlzV%ds-mK!xfr;mIB`wpj}H$)4a|lH2GIWBn4Lw_@;Ao|{x6C5m)<^S
z#q6gOgJ(-@`Lk<bkEdq&|7V)zyVeW-f267zz2`!yZQMSQ)lcVr?V4TG^YLt3LXFY!
z(zjbbywPrsjXzsot(l%z)UkK9cmW5y^X-t`4<-6T*E=cN+yALCD&O6I^RxCsKI>yz
zx)qn~U!RjWF;!a}v}4A#TTCZr$Aqe{uLNaf*GfZL&mPwjH{X15bF=#UeZTcYwL%j9
z{`$(o#+KH-O+9*jL$s>aQoiL|-pij~wbxKGZ+Gp>=?}s>zuQZ`k*cdK?wn&+HDPZx
zf6<2rj!H_&`8&6^pJdLt_~!1`=GfieioTSHY@3}UEm}CC<Ayc&gli`_zP-D9blu+S
z&W@hlFAMqSmDQb>DxI`<ciG0*2`ruFe;*`wTI`q}ySr@T_XL(}+0t_=CcUhUHawkp
z*ip#vn#-DNTa+uOtvY|;)VbCKnUj|zCj|26oHE=ezgKMgrP>vNcRs(5TlywyJ5$31
zwW;}*`3Bksv(MK2e!Kn4tE=6hW#Vh2OqG<BN;V4$3hw09<x&2mzoY0W*V|iLlYf4C
zdSzYgY|G+j6O`Tia!T03e74T|;I(&CiR2evhQ-U*Exy?@?d9_oN8cuXd$M<ib*)#g
z`x>5y&${{J@9duT@a?^ny~=-U^pdB(tNEL#Kkv);GYRtz*tSV|y)gYf<JFgE?HBiI
zq?t{SD4mo$``U(_gDy_nRWH|{bKA8<TCQ^QW$R;SzDF5rnf#q_XxDn*2YaW#O6asb
zc4}$jnz-+l4{rx)e16sbrGS6={T-&)w<I%UtCgtTH~lSt)1rNue*J`hd+%f!<(>3+
zC2DTCKHT_NZfTp{?PnLStT&Qs*#0bkQSD6`2R^C2Ka3V{^J92$dVToig;CE-K7Y~6
zi$BcNKDSK&>hIs5zYFzk`12~@f>(b^xAx_;XL~m$9}n75kl4l}=@bwk06H+I=I5tf
zA3Hia?!3A0Kh@#M{L+VqSV7C~KRrEN`EKX)h%Fg`X50<yWISf>Ive}>bI>*3r^h?K
zEPQ!l*87O}8;sZZh87h)UDj^3{jc=A%-ii>+m`<<d7Ad^&CSE|_LkxcHs;Rsd%o-M
zPu*9?7%VeS2!yfOIiC-=-d&{{e9d>+&xXn)NvUU-l>dBky09hBo9)2ilpTL7m&!a|
zydbS7E_C)v?fS&?^JLF>$4*IoyTts3U8%IiqnmcGj1O&JD$vnbJD+da{g#wpy;83C
z%^X+0l{nk;^>}Q%t)SVoW%~6Fmk(b4@wMOf&5uGoTd!Tswam4%%M!Dvn4h(eKYyfe
z+LaHt+o$f&dw6|*-J-~qTV5;s2PE%Wu6HK+w#}^4YW?NE1L|fOi{8n)wC9y`{cq(y
z%L!&8Up6}3e4XD?`}<q!uP-lM+}wipR+XCN+*mNXAKY&}vH$&xi;F=`642Sd+w<f1
z{rTj5=kDFjPj^}*O){utyR^@L!|Sc(&;Cs^let_r>8JDbL>?)Jvyaq*Cgm3`IFej*
z?`twc+^-rA({79K&nCa7CO)`zey+2}Tj>{eX+L?Lb(*dEQ!g*_-1G4h!@heq_qS$c
zp4#<2a-a6hV~3Y*%y?IG-D~E|kI(OPo+-NV!u+%BD$$diGc>R75U=&NvS0pp{`#jq
z{*rC)@15+pHBaVH<3_LiY5B8Vu1^ry#s7Okc+hs+3M<nEN8cp+{LeR>CU@YrY|}^6
zCQJXyDxTMqqs}=0+3}bqZL@x>{5tN2gU8wJ<GJ_M)kR$LeWUHMng8~VeZLZ(v4zJ5
zoy&bb_XEor+1o+al77sdzo9pF^`*9^U#W+)zwCTIFZtu6qc`4`Ntt9E`1A8~sjv`u
zYW?STP)h`~sr$!=#AAK3r+>fS-+wUS#I@&Bm~5`?Y0q94|LBbW%{jM{%L{LNrCH|e
zxwP4C-^$x*D(^n1iCMgP5H@*N|8xKEw$A^w_T0I}-SFgs0bkVR?`+rdVsvwVtn`<R
z`<pl6b#-+4a@!q;tS_gR%$qN^o;@L_{anfGYiED%Uw2ia{KCv6S<TIsHHNGnufMLj
zy20$|8Mf=YML7zVZCjnYdq=+TYuU1#-|m-RzB1fhs=VT`#C5}cw<~S_*0{Ys8~wB3
z?zMGtH=aC=-}rLN=3Vvc%IxF|3O=j<ynVj-e4DeQanSCv+_3d=z4PbGg9@lWKR)h~
z?Cj`>aW7_a{l~aEeEqW2)6-Pc)s2gv`9y8WD7-&sw&MB8dv31O*_-x{v(NcV#3%Nv
zyVy28>eiIFv-|QAug2(i^{>v`ZvU}v;_=sCT3Ce->}$MxJX!PqhpqSh?dvZrFX6S4
z>N!(%<HyVOi&Z{pJhQf(xr6(SZ3^>tFCMwuGo9A?-SR7W{m!qb$VkQc-)rT!xnb6}
z)fw^(p;o+0vbQo!3a)mwE`8NzeNu&a$8*KYr>|8^TUI+i!Rh#4ratGIZ8_?u@j53S
zUCb`jzB@bJ>(`&R=fA!R2-to}@n)XJi?3QCD?poE&(1PkoPU4c9Gl8XhRMfLe%nLa
ztto+^W1^zB=SAp<fey+3`s(WY&Bi7|e`>^M{Iq8<^N%=l?32GyR<_bxH}j<r?%P!P
z{kN9r$T;J>GU}Z`-r)x?YoF(*8SSXzW!Ap7WYhh%PEDqt+fA*jjgyOQoxZ+0_<hN%
ze$)EK1BLsGIM{!y*PrMA>1`q2^75L(-6WH8^<0gN$7NSHKMlC>T2wb`O6lus7k8Cr
z+x`2I>^I-;Y>%YzhYuev_<Fgx$W=)3X%+F;{&?69YAXNv`FV5M+gra47?!ooX`h$o
z(qa6<B1r60(e&+x64Doc&1^TW7x>NgP3HUyL!Om)3-hK{EOB@y9~7{-;GgwtR-aPV
z1N(aSx*OL$Vcn>Hv--=U*sCuO%(CBk(^+5X+dI`$(|#sPehPWG)h~6f>Grh=Cnk74
z2%9|N761Rg{GdVD1z}s=-_3q)0_w;|Y;2k^LEzcBx!#A{csqN0ul9O@rY%C5Chk9<
zxBc_0)$2c9i_Q;(Hpy;oN_{{3v$rRc^SSK!jREUF&wja$^VQ{fMl&PkEXg>kaJ_#0
zy-oky-pAXN9+IrOBd4_esTq5*OL4;9;x(x^-9O#?kjo$ZEd5mNoxjC>s&V_3GEIzk
zn>=mhO*bl^@Y-tTo8QNc6OYE5=ZJki9y2fZyV-Ifnd{TOnDvC@URM2kduyH}>;3)q
zM+~1G_;zLCg4&8z7yg<$965cuduQ?UOM9!!udR>w@0BvOsQtAiFA6e&RIab6*m!Mi
z^yi!D^FjNOKrQ2iPOa}28~Z$Yuub+xnak4Z7tQ`V+t!Ji7_wYaUE;KBZ_f0`>wJ^f
z{r^(4V^{Wm-^cqB%inB0_}b{<=6!4nPVPQ0oM9Ft{_4{5HO0|$x-agS`p!A~dd!h6
z55C{tSA0FEwso_5?#sio(iL>ArWHOrIpwCwYumZWEBR;m_t+jg)A&`QbO%##^i<`$
zFDm<cnlG)?c>ZdE>Z+^dd(HA4jF+vmpR|j^N6Fpn+`_2#i7fjqdp?OVYj%D-SM|4I
zN(Mu;2|NG1(;t4!;}3s6_g3b4|2Ow{C8>$G7~WONxpiafKX!(!`|8e)`5ouoK&Nr1
zytuIN&8@B73l=DRdwYBN?CIc@*(c_9_w_CFn`;HyEj>*)dQ-|tp`vGJD&tuX2uDx*
zm|LOL_vGXm-oL`DvpJIUq`tp>cg7_*PW6Pn?grIka>*~Z3&+Oi-`FwHIgoSR-?$%N
zjGq_(dn5hg#`j{rqstm}r+v_LmfUJ8_+8EU#r-u~ia(36PvlDv`19`d`<vCjlsY@Z
z`f}zt&y^ECuvRVJ@7*IVeZ%AY#RjFnPdr{8WBl`x-<#Cej_C)wt&8=NjlVr)fSwER
zwN%-^yz=nYTPt%<S4f4|&Y7kg{VC+?u^ukp2TM;XS6sT@@b*!h*?NDb2g_^QFSS)h
zoiDaAjggDkJb7{_pHJ)R&z1c}>psu_5*4g9CEu~zUHj6Pr~C!lcV~aMvgdQSm&Cu@
z>BWq|`JV&N-`euK>N#)!@q$y1htI9IEZ$z7|7z;lJ6oq!d{n4kvh8l2R@dk2>vqj5
zXeqrsw^2i7#f{p(pREcWI4tv-dFbWk<twAMYVH5?sXH%8N$Hb=kp+03sHi-y=A$cU
zQ>ga(J)ooh->tvOF8s}SLr}Dn?91>cN7ky{&!6=~=GRM$lkWYk0ckJAW{H-63z*9#
z@7;VoJ*DEruIBy8HSO+m52_TI$JZJ)o}ZTQx83*o!uU^Kw-=PihN*AUdAM2ga4di0
z{#9Pj%PYQ5-Nq8Z<Dm3#vDN2aXY4QEf6}8{vHW=Cz1h27wtee~*)lUtwbrBK#wqvw
zJtvx)GhdhPnwoX=xN_X`xSKXdWFx!+R&T!~fBj0=O>gHJVwak~UcYi8A?xTS-NTw`
z`wl&eoPXi*;>wrbvOhgL{-t&y<G=9TACkTq?y0hVo@~tUUCo)J`v1-?@9#gbd3@@6
z>igfHXOx+06o1Z5-*<09;6rmaujHwAYi8YRmZ?9dy5g<}&%3=p3)A2K-lYDa3RIbu
zzP|S4%*@FrPI&aVEzV1W<mZ`}mv{<K(~s}FniXpJE@+YIG>L4cghP{_|6*TyKPURb
zmL~_V7wOkmZWoj;o!BCB;EAW4ef+zl7jv&g=*8t0pR@Tft5-7D<;hWr1MKzxZ|k>B
zmJsz^bW}e6^WKbj6?MPZLsM1@xc;e?oAG*G?eA}(xr!e@Dr{_Rzcy<qDe;TgDmguQ
z|4(4%m*g+yGj#X;h^~46b6fejj`j2HzyEr1?97&%83*s*pY(Iq&!5Tr(%*hwUg7zl
z{oSt1i<Mklo;WVE1g&s`41z0x0BArCM1xknfoTkZPw^LTeN+0h9W+XE=Z#wOrz1K4
z^yQA_Du4STSa+{t0@qn7!2~mgf{zcv%;c^7*F6$l8D{q1wq|$6wl%+}a-ZB6vMQRn
zzF6$ff*j<b4|`kXSQi%;#Q>Kl=OQBwe@Ncr^xyGQt<52;-NyN_*!iI4UYh@I)=W8Z
zK7OzJdjDw_`%6|Ixii6yWk&FAs~4$mvOb3rUL80k+swJc-BBa&zt*n(mO>3>kDV%)
z$6eI<?s@c0jKlAD!Uj9OKRZ6x|K%l7LBUFgOp6sAe|*KFeD0jn_dXuX*{ogBJ7@Eh
zsol>$oV;*-#S;skAD{FkFYFbay1V|Q<Zb(#a>A;s$`c+fC|ma}_V;zY`3+ZY&x?-R
z^WTlZXG_%O{i~<<zuy1(cjTrA&r_0*@7l^>B6*L&K6Zzak`lkpSC+05wi7=;Oqg}e
zY_qNnW3!^Xs<-I!j++h>C-n#J5A{y_mV08w2bY}aW5!=@bNRfEdMTgPInhb=^rQS=
z+%w9Z6LNn&<T@q0zs3K+;d0p)p+23(L9aLe`4sJ66CD2;JdEH~$SCh6dVcm-y9F)E
zEoX1Bw>i9G7r)A?t)%pcA=rOG$DLxn<D!>*g-o73{`qflOnlzoRQZknOP+c9ow87h
z`lA`SO{KNw{ipl{&EpdHb2yHv7bon|tNA6a?|Ji4>xr)5w@Kgrc&#|xx6zAx-k0n5
z|GT)j$ff-9b#VDNced}sq!x)g9Hlol=Kt?5?&PaE%a-IX|44^bGxEgTj^=v7xsRMX
zH}OA|3drnMeYAeEi~5p;PiK5D3b}4bnXze&i;Rt5i4)hno!pbvkN$nu>)G?@!ABOa
zDSHez+*~v9Pw5{W&WcIa;c<Vse=a?n=woy8%fGcVo^D7~>6FM;Vi5DbrE`2mM{`86
zozFxbr?jLul6~`g*d}w{*{cz-f@7bg&xD>XixaHLie6%$4oa0ydaS<K<Kr3Mm6KeJ
zX86xM@pAF;og(v_!u$68_#MsQuk%vu@y3nWcexmxW`&e}je|~Op0Gbw_)&yimXG0%
zmWbM`>q~34RH$t}^j-MMXW1tI{=}FS#f39>9Tqd#=bH(tQ!Y=p_o!FcKC36%Xj0YD
z;$3lF*1wdSXYV^!)&5sZ*Fz=geOZ`(^tl}~Pa8~=J8+)cDqgrlPx#-}_C5bjUXs3_
zf6(LAR1;{2nav*7VgBzM?a;ZxS0~zF;)UFkU*>O~sMr5=*E5BSExwY@`~9ciyRhoq
z`|}TapNSeCl}mnl$sla~?>QfGHK$a5b*|dcemVcntS!=Ia~apnyfCw7W&H9v_fFmB
ztrB>(A$Vt5>bACgR)rn+bwFzX-}=IP+%J+u1O+P(fOCxTrN5mojTX-T-|uYUobWU$
z{pX9Vif6d)`zx@S`AZ)>@*{yMSbUnL?DPYj8#bEM{$!rTH}BZ``ZtNk`-@)&UXpzy
z{o?i(?&!sHj(1sanzTS|{f2uVL>^E3duX0*^$kJ0>3vf#e`h%+%klVYS+b;gu_Qy=
zA-U^cnB0ute~sGn?_|&|zQ40%tG{h{-XB_GB5`nEqxo&WXFOuF>i5m-l$~2U<(O7&
z#Lw4ab$>HO?N%TAbaMJ*j=R%%8rn}5oVA{xUbWro%kL7+()ihDrrB2C*!-OTS?v|p
zhWz%Dhga8cZ&aT5cCnpuO*B`d#EtJS6Mp%6EW5uV@Jp`n6Q0`FZpO)f3&VZI`ChuN
zc&NFL>;K&KYIR57Ka3AP{eVsG<^O%+#R|C^FOploY_|uGunA7AK6mQmex(i@*@NqP
zy(R8#Vrv$QDQsK5fU)Moy8yBO|7^GN<ef`+5!R_Htp2g~M~zyoQD@-R+@6>Jb&p+Q
zuGsBtu+Polxu$=AV(pyva+ycRW!2|AT>C$oVUI~+%j)tAZ{p|0?X6y(J<)&VGM#&S
zXSBY*vj3LLbm7zL#!5DgMsw9al->3^r#Sr>!z`YHHvRU$3gT<{&ZNb2*~eU$RI07{
z@OkTl*WwPV%lD^Tyz}!ML)l#A)60$WvW4=NR)7BH?Z2F_*feNeVZ7b@K)wcF`EpgA
zRThWK->10b`tv<J5V6$!^mgAx*&ZBs+iOo~tIl2c?ZL~_D_4HAklwD!Eao%soJIDh
z^+#4aH~qAIcZ+kTS=m}IX|7eFj-UzQdLgli)#vy1?#q>4T75FYvc+!gtqWR978P?W
zTWK&=d*|l5Y^756Z;K@w>Ls(T>K{IO_VOHO>+k)`g17U=$VTieeI0iF<G$OKbMC3?
z8NN#}+HrMB+R{TW@9Q7;ebjgG^)B&6k)N+JF+`cHn~~R-uyfjbgHszAofAS&Me$sJ
zen+|@U7&5dZWNEhTQUFLZH3FWO?WWP_MvzBhS<Na6QxA;-~Gy4+qP9R&1^?sjKTp9
z_Q{u@c~$+Iz<yv~qh4(L=btyOuMD%{SIypgggu>m+1}o^CTJ<~N7_w4$~{1jnX_TJ
zs(RhD*V9c34<A21;o-h?{<q8vyp}hXtQB+ER6Ip>u2t#bxhazv?|d%!(tq`wMd-^y
zu4ta}>lb!x{!@2?EzeXl{>v`aU>(Mmk~bCKT|GJ;Y`>zoNb+8Tyw-7T)w@66Bp69=
zdw1}bqAts%DMhn)mkUaj=^t*=|9_+|K~di{(j(x!AkX`YbE^86UVQ}V$Vzc(LF)TY
z=M5yJ1NW@3uY0zeJ56o#=IwfB*Of!o%hg<d%Xg;y9CyhPmBQ=i=bKB;v90*V>*Aso
zv~S0A+qFlhM%P`xe)aJ44fkGJN^rjVtSYa*&vXCG*$g*q-pWe(RktS=U%U8h<DVF7
zku8P0UqLKvc*fFo!u~<qbC!EcnX~`xdHI<|`P(~HHB-}@{CSUa);xZ9Pvz)Ct}N?s
zb5B=DXNzp?EHD4D+S$cC=kDZ-96{5)rk=X<aIthw#M*l%5*EFs#m5AT&ej|$x88F`
zf1+llvtwh_RTB}=Qa%wTSC#I~@8!?8h}PfzP<2MWxN_Uw+c5>XJ9>F{UuL%u&RpHR
zEal44veKftC!frTW<OK^TKik%dhW^FCfA6Bbie0|IaTqWr(#A>x>%Lia<`6RiJViv
z4EXK{gd|&TyJs15Wxs>k@0T_~|LybSWmkWDID_|-eOQ0s?(K%jFSu7~{t-B?AJJ*K
z!M0Q*Lw?>R@FFA7R*Pzxn;u=oZ8FUVr7!0iyGJqX`Ztjc)X)0+eGlW=8N2>@yb3t!
z0;)B(@+ouH6-%7I#$RX~mXx*r`%S5t*ZxXW{&C6LY_{R?4cF_h-@QBcHrM{$<B~<p
zHq4oKcCRtEUaGSs_W8pc+bTh6f$A#r$G1%qx4l%2T4MKIU)9jA^#5WP<{OJFD?Z4?
zY+iCEf1RAG@xit6rL0%;r~fUCopJX-Njq=zM!{HVJD**GzvLvvKZIOK4X@zc@c+bm
zcD>#ARwv!QE*W_H^3<C0wf`9#Zb70Iw7}l~X3m^ja>qKZu&=bv(dPL7RiShHj3;8n
zPyFk=YIpoTS=-*RD=j+s;j4AW)fs%M)@q8L+P>iEV$sZBpV)UyeRsBcuKe){%eC^q
zuYJu`efQOt#7BRQo|*qitoPoWN4M&8e$TS`cdR_3GwuDl=5uDZ{0p=$SvqV^TAsM>
z;goO8+%K;^k<_yJr4jnt{`~azF2l@B^XsSPRd(CXk&;L^PP_8F=E`QXxBKpSKKXc9
zro<wM`Gw~73+HV_tv~(L4rPq^o#?)7oqYpuhvhG2XRA7n0~cm8bDBtRh`n?A*;h}7
zKU>}h+zN<zUD-FkMC`?ZS##(3lsI&K-fNd*KkdGf@ErNWN1Oj_wBx*!6}0OB>ownU
zK2}~%t!3Gugc!pESvz@WuA1+x4Ov&Ul9H0-9%!iuneAcwE0(f>r7PNZ-+hx#qw8z$
zR4V1D^!_@#H}UiJWosTSeZ?3)nSXYAty6#hv!&~Q&X&LYy_M~U^yPc?pVwS`xxIL`
z%lU6^%g)~NS$RwL<(+SltDe}pufO&;hJVfeRWDDl)?8R)vaNgf%rE=Je-vd;dR=`?
zOLnH&qfjmJPdnB2zDut7TBhT{yWyQL+o!(wMs4Tr&Hnc+E9>YY-NT$F(hV;EIR3VM
zWLDq!cGh?Eu$L#cJX{~NvxC2oeVWjfS8JvF468FaE<06O$j>Z1`}%ZUZRb>3=ZMD}
z3O`&hy>D!mbcsJd_~R+Ardz#7z;l4#!u%hU?kahC===S8|C$<`dGqFJK^A9pI3<K7
zZa!9Y@ztA`pU%u=kKg=JE%J)jl&fA&Oh2A{Jbpb#Rg3G-AD!|qb=Piatl`w0;WKUa
z`|Qm^m4<8|4&MLoH?7!H#Z0TMMI^8Gk9F_EYY}U%u>=`RynLVEIAOC}(W5J>K0CPr
zkKA_`p9Egz*&W6uFj41h#>b=gCv`4J-S!}TpIJ!chuz)p^t;!~Z?Cmp%v%3{H)law
z&EG%o3q&uRnX^m(*=woXEkA<yEZMK80GgC(d<NYm4_a>q-YyR|8tNk~0)OW3{mqxZ
z>d$?Py5*j;oxHSJPiYxVQQY*$D5W#t+ceu#aZMauT}?(TbqWey9Zd>TG*5`7a&6KK
zaB*b3x%$i|Z|Cr1@B5ET@p5%=h`P7_^`~8vCQqJRY-cG`{k-zrX9fn2Mg|53hKBaO
zN+w1dwl|x0skiL^tCc!w{Yjn09|PZZg{Jqv3qDe(f3<GulruG!3=9nd91Q$s40-?7
zY_0OSS$=#g*R++ZrM70Rs=KRoD$&rU<*vormGWOM-bk(YnAFpHFO)-3fPrC&V+lL!
z2YKUHOU<8W{yuebRZ>CxB%Yt96|Zd8#{6tft1*qrUG?Kd!jYn0`CHMZr%Eqf+g!+>
zX#eMH(C#{^8*A2Ni_iNsaiTsu1A_n)<AaAEPKO35hStO|p3&alHAQ1h%+61%QjS`^
zGOt^{?0fc$sEZqeZQW~^zhIx|`Zshzf#+1cX}$W)P|cew<t*o#=hteTToL4ZyDEw+
z_x1J{ORoM5_@-oNn7O-bp9llP1hztdg@%wrE57{S(q$Vp(cS-b)*N58`#q=r`shXG
zy}m5De)DnvS5a$xUmB%^1x#?Av|i`gZR`43sy7#(fBE2L_okamdJeKq(Z29~+t)SW
z=U$!p>D{$?=9AeEHJ_fD?L6(a-M212*}Y5Vv~r)gF12*A`}xyppF-QtSWeMCp?~!7
zMmN`2PnNFUbaF}0%=!Ore`Ns2)TKE)RwzfkEAg1bb1Z$zMf0Bz432L*6WIJtdD8xq
zt2Z6flU?_!l3B%b+h)OE3(~}O%dJAxmh>oHbM=^XYOif{e7VlVH*4E;F5NqO`Rcyo
zGg3}h*KT!MA69cq_KjcmwG;fNSqIBbuHN#=y6#${<<n4uknXf^!FAqEO5fjKY%^VL
z^6K@^NlUK(-n&lUt!`q@Y~Ek~mp5Fv<oR78%0J`Ug=JD(ZGP_UzgpFM`9k>WQ}Zt~
zGBEt%2F3dgLy1eLpKw-|xys&JaIp0L=RI*>|K8%-xAft=E3XW*F4VS2efb~tu=-op
z^<z6<1s#Yk$_bdyCM!~SRP$w<81IXPPunJOMQ@KzoZp`LeOKl>y=8XW|I1DLarpnr
z+J3`T_e&QXtG^Y$^y#hn`f>AD372_xUEg%D*Cgv;+0?gYj@5Oum#J1CHr*7NXvtT%
zI4s!wbl(0&A6V~KFfCB2HT=XKntMC=@4c*dk>CEh%bF&g^}H^8o3mHvcEIBB){IN1
z*52A(KK)tm8wJlO>wmnZ^R`SDW&x+9CAxmUxoSUt4QcI_Ue)L7rLpAW>v>@Z^8TAg
zFP<gp74`F1T<xE8K9byfSMxiaK6zhbrm<+~^n>}k%*!IA%jZr?^2<7X&c#z~W=$;T
zC6SP*_52#k?;36snUqoYb^gc6c2$d8S6$(_ntp%J<6CobC-TkjojgS$d&=%qqu>R*
zMfIDx#I+xAP24|s!qpkEx@`TI7di6U-j=aRUwmxub?00w-b>diJ$0vlz8|}$c+$7C
ze`7Y!E!%6Gy+G>;*MyX7pTAbE@Mp~Tywup4T{>&FC?msx{SM^;4E^%Xb$4%UcbdP>
z{zc@I+qq3<-`17f-di^P|BYo|{n|?sw%jVawP5+3)5=OpHm_@bzxLc4`ZP0jlFD(m
z(icA#s`b5G%T;N-(qU@2>7$x&xAprJb+2qm2|Hh0+@^5;aku#+)t@T^)=twZ_&Y^v
zYW=mZPJcw_?no$=ZF>4mQ;jF?XXuiI1<xxFeC9P-dfNPzQR@6T-|m;_yxLP2y(w;z
ziQ1OAW>e3-;M=rc<I#k(vQGOiPCQ*JDs$wiG$cl*^1PTD-+ZipTJlK-Q;8pQwc~wb
zmo?|cUHZ>7`962WzYW<%-^Fk5Tiz$yG<C*{sC(OT)ho3FJ10Gv>uDCOclFhuIbwb{
z)sG)^t=jreY3f@mANjIM*Q?!24Sw#qYU=QE{^X;-pH+mF^2W^*^E|oY(b?Zwg~gs1
z)wiVFN#XtT>vzig|JvtQEZ=^{A%4j(?MqW?CnYcUld)g=xBcy;@XW9Bm;EgF{Eu0^
z)~fVdSW?oWDH^F$drzA)FetDvd05XmH+xrN>8gyy|D<*%Z~dj)J@xWFC)r(3FTQGB
z|MFUy^<>MrDw@AmriX4!+G(A){mty)FP?v*tE#V=d0zUqo_*D|>&&JV=lrw6bY9!P
z3H7QqSbNK^zG)ZFT*dWG3upa&95&OuKdd{^`~5?$H<Q2I$obIIFl&EE>QqjX0wV^7
z2KK&cCPtfSG5Z4|L-J2BnU<dCe_>(ZSlYfWsD|sce9Y$e`RWt;@9d8ce^VB@_GRej
zz3abTp0jaPw#TkNyqs^>tqt5(edqA%Syxw1+Ql<>q0`%A8Mhbx`xW2EFq7wd)tq<F
z?AD*sns(aXT=nF};NSi6{y%vyF5mk9N2;Fy1A_wx%LKQXpI6-f<ZtUQ{A~YXMec9+
zw#|BcAYE$vqZhfS*T0{+`j)QUg_);+UtRWRwpHTvO=7Fp1@3=;<%{FnSZz!G*H4!2
z|8jp%?xOuq%6}Z4duv^w;5v(2pQcW?U0q!*{qNad&D}N6rl_2JFg4i!t`X})^||V=
zT=sna_Tc8i##{Lo$152jB`jaF;$ENaC1Kxdcg+f`xp(*TpWlkQrnT2qJuexHW>ri(
zU%h>y+FJW(?{-M8PmX@J!s$i9)avXFUB9ZeZ)^;F$e#aare5f(%6wk#LUsM<PX|)(
zm-vO}zhunnwtBf!{=)H;DR13^GHk@3=FQw3e`Vf%zMXQ>!FFb|H7i#{n_Bx!y7K4o
zwNt5@3*2(&6+aESZJzbxR><6rvaJEf+~;3d^L<tJm5GmLU3pUJCj<$!SvO15&il{W
zdG6|_XIs`^IJNb6>Q?oo8<sm)Wh6~s^1*mnZEEf2$5Y<i)KZJK+ZWC(s(;I)w`)pJ
z^E>4yp)X8%bHfh_f3nN6b=&W<Q!R0OyxkJp*TO36Piak2cbF6!I9-32^9{AXg<8ip
z-`(uar_#F5SUkmRy2-B7OWtk0lJ8+}swKCmk^9^3L-)S@4_@D=dZqN%yCsK9asqCb
zFVFga^K{qtd6y?mT%VQW(zNF_f57=yJ5?n^S00u=^?u&u3;P}$U)7Fx>XkMRJNfCW
zNmk{YxVZ*C|35#Eb7WvB_|N0F{IaKH!|&kzb+v1wwt8J(AHQ7U``J~hd*iPiC@tI`
zzo4gdi$&6uPqY8?*giMe8XVi(>+xaD<JkYox&Jj>_MiI~66<!(``_O^Ups$oS-(<Y
zvC3~t-<&Y_>^Wt!R&PAMNj~2H+hOYGdsmJeEB*3x;gOWv71ghlo<`jF%RldTZ%cUY
z2lFgR+t}D+>ptpa-}|3ks+k^oo~^bd!tdnjO=`OGE6?1v3!1!8lV9xepSPS<o0C<4
z-ruC7XXK=O|HZP__DS!8zi8cXna}fj?W<Pf!c#U3vQ{M*npn9-q@}gv<KwxQKp9_w
zWtm%9_7=V9*u+cb3X}hwJtlI!d%N4|tlr7KYOE}*_pFi++jjTwTlBhclBz<0Sn;Jz
zrwh$kQ@AD_)sUU@zi}2%tXA{NPO*I16>F|(cTIkheNv^q`{VmQw~OI=+UL|xK3VP4
z_|NE*s4x>FLj}VOB_@YiiCcbXpLq1Y|Mg7We@6F@R+S%qBfmo7|Ly5dJrY0r9Zx-a
zQse&ug}<MdOFxybFn`U+z;H+U9VbhJ`-vm|`H(hX=FHge{5vJ~+g>?MnL3Gqq2a&C
zeoy8DhlR=`-cQYYG&9PnZ*JAzvPj9EQWN`{Df2cPoKgoDbPl&3&h~e%zqfLsW^Bz_
z76))TZZJ8fv3aJUEhEE$$F0>&j0_A64>+B?6&l#jtgFbLlk>gC^wU(;tJ`NqZcYCi
z_13B`vaYW2)TG~*Q!6(vD0#Z(-;c`W=UAo$9TqPx>SADEa8hX4qWtVZxz_EJ54QW)
z-?@GN)$2ajpu%sqYgfifXS_Lo^y->#yt)zp+8^b|Tn<gIO8s1ZRw;hn|LrXh2Tdq5
zt*>3+<Ylj3|6V%k<IUHzWG8A=PQCVk)AsA0{f^>F{23pwT#Tweq1Wx7Wf!#Dto}|&
zva-;ZhmSb3q7E!f+Nl4!*|aQG^J>8PKf9dVpH6u?@vYyUHD;T2(#|=8z5JnLk8yId
z)s!=D>crMBu5<J3oTQ@_v}?&cW9g6`5uTwdBOT@UU2sSb`h4)1%%wm1d8ZQAEPh`6
z+skOh#pCv?9vu99>W@#uKF!@;?sLDnZ!!M(YoR~u+}E4J?snN8?c`u!$UA(%fI)5p
zi=>Zl$Yo&$)xTf)4o1(P_36dbb7F7%HZHxT+-JEt{`Fiw%em&0`_9U&ICPXt`^2Xy
zpF%#?rTi|+`daZdUVM3vnwu&s1B2#bRe=M{Ps2CNdv;c5^@8e(^Tlh^v;J7-ySlsI
z-*iQ%{`=mNuI;b9_S~&KXmsjP{>Ax%&%K@~#v0kJWwP8OTKZ?jx%(SVnme;St+AS}
zqxn>!dW%y3ylYo2X8Y`uocep_?d#ok)iKMHeNM$HT%WtU;MU{R-pOmsAK7;2#cV&N
z)t<8H&Qk~Tk5<)xmaIGKr8arWyOlYoM4$XOpPzp9W^j7ROSd^bYVo#Hb(a^hsJ(qS
zQ}=S{^S3W2EfE%9r(w5!`jc$I%^{F<(XP?VC4RcV|Gfdb?ON~oVIe<LH+pjK^?Kd?
zR5;Xf`uZiSXWd-5Z<&nO6qQ<w>dsAbJ1^X7P2N(FzG%<uz02~>8Px8yIrq7&d1K$Q
zb&?B>96hJ+1XTo%-vv+F-$8DK8%u}oOY`>(dbmA3yY9G9{qFmQ?E6A*ipi_%7UTph
zmrAw$y+0v(#=PCmU3>P_ueHnhIPIFX?8#};@o&~-%74A^zp2Ldv|sMb#pb)}mmDry
z<ukLXtHVOWGs`|};je4DOJ+0|hi?8YasHQa)zzy!p2wcEY1BBLb}zSAVqmCvb5U00
zK=Oe#Rm$66OipXP%c-|tEtjL1-NEx&LaF4RW!hhAHhSsb{HBrfDns|qORm<=(`qkH
zPfh(=wZASa_}JC2monoQo-?((mic{c#qzLzP$%Qcg8$dK?r#p?ZFVknI@7&L$<4n^
zUM%R)_VWz7vvu>ly;fU(&sbA_BYVG-uc+KU=he&iJm#D-x$e#?*@q8rZAqNN6~R;g
z`yRLb`3tAd-oH}itXjLVXg6rTvF&WJ6}<;Fe)p}+3V$p6_tSH`xlt>v?z+F&5U%MP
zwLLm9v3pDHMC~)@UR;llUYzr(dfUcB^*q6%&9688`*ub2)<)%+*UPV|GcbWO{&8cD
zu(vnPe!Wl`ae4Z`Id87+eW;ykU>dl2dehXcA;GO@<DYQaw9ewcy6pbGfbV9xSM+lx
zw_WvpeyiuL^x3;LQM>kq%kMW1KIZiPRr7_SPv&0Fd|Yi02Aw;6Ysc#;);B^gZ`k%V
zRBg#hyLXXa&oQ3Ln7Dtx>r5lHtu<_HKTbb-zy5$j{oh>uGe3l9-S{Z%d1<vU`-&nx
z`|Hn`=3lPPR@d6kGby>*YD?1J7qGPe^?Q6iPRcP~xAT6+#QnS9uexvC8hZVKDyUJT
zx;n+=wEcznm;cwKwe@U@_#^tlpM`~ip~ZorU-R;1{mc~G@P^xFWo>oacSRi%DUa}0
zdlSA;?(3B2=TlNQS2Fp3C~g&fanp-!@27Vb;S(l_e2iYA_kW}B>$!Y4`Fp}{`^~*_
z@^Adj3rp6`x&N;!sLazSEpYL7yPzA-<yTcTHtfFjzFPK@+Ord8x+PXB>rbr;Q_9|S
zohLFr<omL9QYw1!Yn{WEUrN<pVy4&{vennt|MjD%$;(BbmfU+7EHfv7fkENv{+$|(
z4aeIfr&@oTJdJmnjI>GlTSw8{2#@RE+p>3FUU<>8YuTS|El&*3Ojx-hYTCz(8#*dv
zf4$w%m-feb(wC&$->+<R+8`9W*W)U3TivriiD#<u?{D`WhA#FBd8?%>qi$*T;``)_
zT2Vh(zy7XSsPghg-_luEQWs3K=BxzQSPHx!v?H9~PnBF4DERSL;s>FxSvTU{ulY?0
zUuYg`w*Gx=-cfJ6x^-g4%}3qs?AA@&`eOOK(4$>X<$g=Cu(00SrlV%$6wgx^x>;H1
zYRv_c*6^=aKb%_F0&1z8f5FG~z;Lo-Y20hwlcoOmn|^w4c{6+Vs_QGenBFI6FFh?1
zWcB@r4*T^t!C&sL5#L$Af87oFj|;8LGTxt@Are<`Ui(F8)cLf}*Pq%Jhi)rVKYMSc
zdvg7ab&hAfPv3vEcJ>?{h!1p6gsf__l8%uzR_B*p@w9DHR+Qm=+Z@Fx^=rF-)rfJN
znlWW3U(K>Y?Y?{4zJ0YV4&8l}-Aed*jmq)qyF((+Kb9|-+41)4>GYW9D-wI=g}He*
zZ9DrnWKs%O=4!9X*ya-_Qa;@go+%`+*3Ax1V*h0?%g(>2xBHfFrMBh;liY>R|MHgq
z&75BUFEDs+nWWFcEYV5P=R;OJtjxW)o7Xh+ui<qCgRbivb&Eq3OLm;vYZ~FI%cl8n
z%d&6P&U3B4UTcb-{_fq2J+<!&a?DHrov+lov-;<=)hp~+W7gK*K6dWH`-|J#&Pzv`
zSI<&e`ENqlsr+pxt1f@t_4rI>Y~lLS#QSx!pLlp4@0)Ua&b(^}?fv4Wsl-~=d8Dr`
z^Mh5XH{R~rkUw`{y!iX7cQ+QWde?`Q{F&SAxlH`^isjZCU)M!@J+of3``hAav$oaE
zUl;MXH0}KNSC<Z&X8u}Wc~>N7ijSpsR?_t6wy*o&Uv)UzZ5FgxY3Y>(zd0&Jt9+er
z?^J8;x7-w&JpJ#Ij*e0(yW-2T&pD@9d%WYl|ITvT)`!O@?UgFd+ONd7lWkq6W%<Fq
zdwa6}ADyoCb{^Lv-}#+ZGP#rAT#WjDRiU@5V|w{t#;WCsik@nbmv%F4czl56>Z>pF
z?B3iB33a}IJ#nYggA<H<Lzmom{p+s4f|_@MfzS8;-MEiSdac)2>Fn=QUCuc}S`i#a
zWG_^!#sypztKAyxxi<E5ds)cONk_LP{O`5iV;R7D`p1cctVO3(js8D&uf4RoRcmiV
zkg4<i>ld4PzaHg}zPhGB?fYuCy<gI1Ec=o@@eAAYMA>P<U+@2N$$q|f*_(IwChwbB
zT)k}iikM{wWS?{PSbp2vC8D{KPkDOPG~ZWO16;dGue5Cr%bkD2H}&<GFEYQsoy>o6
zRLOPAmw+B0%Tupcmlj{y92a@B+$Ui1C)JQ86_sEACH3CoowZ0;erw5n=FG6Q+tn(b
zxomOQx&FxbeD)>&)n9j7->~9)8?x2zZ0LbW&Zcc<pII*H+s$41&$upVZp^+p-3$x|
z4nHWdnmdJYhyB;r*RQV%U0w3`S83PB1vmX(>=l=K@gv&UbboT^6y~&-&!XOytT_0g
zRVqxsT`#_Gvd_L#CGDNJ-d@Xy-F9pzQ@n%P&)NC$#&%X-)02Hp%~Jj_(MM7+zxP^c
z_8gVTIy0G4JwG}>E>(TCD`k_0!G%dVGtZ_31|$T<Y>n%@^wr{|e(;1pXXkinzPzxo
zSxhHl!HpXcKYs~LVr1|aTh_twA;M}~*0yQZvcDFkO{|akp4Q{*ws?P2)1Uu^v5Sw!
zJ5IA+pZ<+Mi0{99`I=|DZ>XPKb6~-O-4SY$OwI3ZG;xBvq#x#Yck^6tZ(jZ)Ex<7J
zPrTJxSu20Lv&Ua9l_@Wr`RQR08#p0MSS54U!1=V2l8S4_T_ex9rJ~=ePTVg$zQy%^
zz?9wQzh@Swr-Uj-?Pp+Mc*n$in4v<)=5tZ@^^BR5QckRYxcT@BpM+DFo#sS-Z}n;^
z`NFf~Q+xG+so%wA7EEAZVEErK&4Hnx_5SksYg_j2PThLUbKkW6k}^kC(@(s<wRLqi
zSGb8@CIh(4SlJOMcKUAVE0fuh--H+#9@Jfs6#)$~&r{gK!t~?(jmM8g(>LEOi`*0u
zl=Zax`p4MaOZTWwHUDrhZr7V|zZcIR^G(&7`JV4H)2-5P7wlZ!T(tx5ah>_QhU@vM
zFW<A>=U?7u@26>H7<lg`)71SQiLA5U*V{i@8+P+wka~Rg&xvP}IUu3tAT#5Bysq2d
z896gIUf6wW&xW~w@;T#|m#q!H@nK(7#ffF*&vF9q9at{s^P}5t?N;vPFW>q<-)V0?
zKWNt4^L0PVYwv8Io93=^YPCbAZSjkA*=fP4yL&G#-<$pSscqTYpLtDA>^_rJrfLQU
z2nw>RuD>L;_k;0OO@V^*zXi+RuusjpVE=RW9<S_o+YCgvN?1%}U|>)baM0SxF!QqK
zq<PuqewXjB)r#L8JL%%(U8R37MDb>P0=53;eg82-&gI3<Rab@EU%l18UNKE#-Y3tU
zp}Oxjw1x6rbz1Ix_1F1p=?hg=Vz2JGchxlKll3a0=W9QduYPsjZC~<fyPy{h`Kboi
zF3vf-IONLx1+S~cRyn<BuiEa(Ub%k8>Q_5!-e=m^e7<^do^5H0iP!RLz9Dn(#d~fl
zU}R@t_}*I0)L4Ho!Qe#N=9hndeinHge`2Qja-B&%Zi{=it0*MzT9aj*zckFJ@;PJu
zqW(?dxhv((*M`hDvMY}%zTeat^`hj&>I>Pw6t=WfXS$jGe^C5%y5mRV`xoxq*V`#}
z{LR%H|30gF-c{SPbkV<c2c!b_tXx*2t8dBwdam8)(x>yjAJ$lWQRC!_1#@m%CjL`x
zKfF-cy|3aQFEeO#sNlMkS&oM1s-UG^e6m(9j_<;obdMgb+u8N%=pjqdDTarc|2cGb
zh1LXGGBJz3C~rTVa3@h$VsF?akxNQ0k$Ha^6|xJrau;s>-#dG^@5f1fYk&H^p35iM
zQyQ{1<XXyXt-#gGR8H0<Y}qE?zkB=N9j(2apT>9g_lNH;d;8_}`u(fY&d%yF;s*8C
zcbLz$F28m?zJBfH<^HR)uCBVZF`3=9lH;9B%_C7+*}v2Ouj%UFt`?oP_5SbY$L9R;
z<-Qvmc-m(E-)#z!3rbWygC2dlYL&J%>-n$uDxSwK{SWpl{v@V$e@<)Y(FePp{FA6_
z(f?{+@b;GJ)%Ef5ppCne)qJ;<ybS6Bj~47GkKI*r@%#Py^*3%rTwNLLetNq8a*h6w
zEz|Zzx;{NU{rjKrhfhyhRj9<|Pg(yU^4i1G^F`|Wqx-D>wEy;sjBIVnmp4+|5m9)Z
z`8eC(q8}aqBIYZ2YQ-+R@Rs?^kJR9qm+nnk!u$HW)-3&Z6RnD$^?>d^T^qgq)z|Cs
z*Z=(doXN@pnt-{pe^=@2tix@*n<8|wZf;t7=+GgJQ_7QFa}O(-EKRRGKZ$Me?3Fh@
zU;Uaj{m`T(2aD@2HQn=U-hOB8k=YyP9e)`lVS4Pv>*-&P%v*5!UAx%)DQ!6)C(fV0
zyYzz6va*U6(Bj@xQ?+0B+yA?8VxqEX-Jgoe%8&c)L0Ra){*YxplDW6H`R*)!9=1Mi
zuSv;^fUZv`Bto|x7brE`vFIv){Mk$WtCz1lJ*76wpl0DL(bK;)${b(5FN@W_e`E2c
z%Fk(`>*Hbr0|RH7W>33U{oWQFut7`@4xXHEzkgfq?PdS}zOR=u&6-m6<%Qzx-i0=D
zy>DLaDwv^ivhaiS-ml-(KdrNO_$wv0$i#oe(Qfh8j?HYFZoiG%nACb}OQv%mc#wlX
z?%JBjt%Z->uC9p;Hcmem^7+}>)UB7keAf6OnzP_#?W*c|+t-C#9zA*g{j{@^)!(&m
zFP*(>`PGjCzB3FQ-`?69e5^-uYvyIO(DiY17lF%nxg?2?pD%ApJ$+?gZS~r?y-{1U
zu3kFYEv|V+Ibg+)&GU{MSNmP=te?2e$Y$!{&lYB_Z*M$~VLo_bLcC?Ok=F8<?|&kL
z=X_FfG&J<{4Eo0uneuRTmBt*u<@f6U*M5C_JN)P8=hwfzz5VqNw|;;Gs0cl9INfNb
zN$szaz5oCH{`GWvyoiv{qU7U!fiv1q)<p%o`FL`%Y|Fn=erntAdocn3n%1%R@mdx9
z`u5Sr|NpkjvP{KV*#UZ@D__t0Q2YEE^Y;FKlErR<*21cumY<)cB+O`6P*ilBYgKw_
zN8#g;Z8?#x?d{tB^K3-eKz+Cm8NBj#G4XXjm-flqubVJI;O(ug-HUAIu&j7H`ToTz
zhbv899xs)O)Ss*Jvj1c3)(-(^YI84|$7`NXFc$oM?(fsk)qHR7sCa(Lyyd}qCT;4v
z+xNDrTi0KlS@AsH=z-(f^7EDv+nEkNaqRBv3tQ~gYf}HO#x&!C!|Q8ngO~Zt)ZhWt
zWDoX#yp_E^^Wq}cuD-sowNYEQfHoQz@%!Afn|Iu9*_7KC*1ey(UN8P#=vK$WA&Tx3
zUVWRp;L^KyGo7Pq)?U`kw|tq{_1Adw>T?%<O<FSFZH>=1-sdytn-<T_es=I+Zv2$M
zplg>*LH)etwXM@~ttaa&aH}zopY$a5zjElQht>VR8?7zN-^~Ht@4_o>wjz3aUeun7
zjhB{syC;H^*#EU|y;3ii&##+wGbe0w+F6t0XFgp;;g3o`Jh%0%|1f`N!ELF+^uwkL
z&bk}TaQYgzS!t4r^Ri1)7MohNH>mnuoxQmJ!qokDu8Y_IobYXjdPVu1or}7|p6A}y
zyRl%mm2^*$n%~v-$=i0am28|JeB*$fmCo{cugx64e%ke%`^(w&_a5)^oRs!OZspqa
z$10xb+s+5QkNmCTT-W<H^!K)(T<_EDCv0kF=U?SDRqNN^@ApkhUIh5gGVyd%o}0@E
zDn|P|xkR-xUSC^#YNB#`)t46wCn~!KO|jnb=d+BcW>C;V=lx%GW?xyouuDX9QKofF
zqQScd-?c&)&%S0YTXbvR#qZHEZ%Yf8U(1TvUpeo)?2Mc7Q=@$*tPXzediGUpkZOG1
z($;r+v-+5YL*5@<q;;>dYf|YS6Smdo&!7Gpclr62+M00CNVscP`1KndET-@OZ(a6%
z-IO=~_RUdTpMCVo_o??c2Cl!RvGiDP*UGf%hM%-<xJ(W^6ZBMOyPrxu|L?}zIrsO)
zhOQ2~dUJESY3ZvFUKxuCix?OR#CiOdPo6)g(f!Svo10g~?%oEv*SPfEoyg?leYHx_
z`h4g9JNUNgg>L<K|JCLTO8tNTxw*T(`m;6Sr1e}Q_NM_0p7PzFX&W_h`8jQ~K(FW1
zBQ1+q>?RycNaE8Ax~bNzy8cwuhTJ{#HcQMqQSV&ex9|1Sr%PG6#a8&uwTg*gVA$cU
z=EY!BWm)v(!n3opzn;=wpYiI-N~`*Rd(zI#I9Th`Z~1>g%qRX`r>FM*J2mzC|Hk*t
z?{&Ys+pS#YGjow$?Jo@jg8=QYH32U#Ew!`+N7G7$fFHlJudWJxbzxyM=(cuH!T<13
zD;Ez>O;T1uce(GeePvG^d^eYbpZj8_TJ=0|tJeAZW%ut)K2!H}L*1=1k5AviQhv<;
zJ}D}-nkm~b<%B@5oNd(4Pfstuy1M%7o6YBoniv=g`d{#I{W(6zZ~3opZ&&-<{d8gF
z61i|`srT0}m;LPpkJqIg5BmGzb?VO_JDyc^_xDf#Uwrk-x?`WuEH3ZfaiYL?*5=@6
z44a=kR+d*iA@gTal$~SgrkgphZf;(lc7ER4pru}!KR-R?;^KPr1l*Bc5;wp0+st2I
zUM|kLxheDGqoZ1Jdn{ZTRgNAxvch+^S=8pVUeIwBtlVNJk`ncJL8YOCltKN!nytCF
z!_qcK7C%21I#nxFgKJ(!U|?X@ogEwN{(cQFeRCslvbz7YlSk(AFfja)_7P;XahqkH
zzwYYl@Y4wfFQ$gaUEF@Z&O7kK5v!^%D^`W9TohmbH<X!;Cm<q1f>8ie@J;v@vMwev
zdV5}IK)?jhHMOAYbQeA7+?;(~ueGJ+#g&!8vn&c1wed>31u`&HIK1HFvRS62tPHwl
z>h1mg@wQc8F1)z7`03N5@69}ZhpmlTx-IwisfPtGZs+e0J>15t$;H6HFHj%I)W0%j
zXHn+qX}YT-HZD^0o3o<q?JZ3^yLpEcd}rI$?wVzuuNNO5zczY%9|H%d-mTb_a%M(g
z_Vsn4pyo)*Ng+_{<odeU#ReXe-FhSz&a<u7vapDF`0$}t_&OO^MurE93#>IJa9POk
z?frO6`q!gw{V#Wl&u6~8wDi<WWA#?YSqa;3e|<JPUrR^F=V-V1^;1)|r`lGRaWOL-
zaCY)mm>~6UVO3@3pC2DVHSxb6kNHE_#YF!8_I9y>$JeCf<g1%fy<1ybS4M5ky0xd$
zm{EYCp}l9CgTmCAw$<CL%HC*nA6-=W`Ps{*)8iItcuduc-L+(y@9eDS=jL+p@?PDN
zdHKoPx4wxC3>7CbI9Y;bn&sZgczSB;rrU3`E-Yvi(~t8BobYOnMWGWYkDr@seRX?&
z{MlKi-H^aIaR2A#+cC#_B(JWEwf3E1(6~t9>W#hC+nt=8O1``Z?3Fh6TkPIH?O?(J
z0|o{gr4n}5ppC-nek;<>&ax_bG2vdtV_sf)yE*pDK1Te0xqSYWU8UKoDk>S@-rUT(
zwnh>Zc!G;n1w4*^dUA5{hYtl)b)&-`AL|WX?l*VJgb5CbA0lQMr}s&j=jklIxWY91
zng9bhtL~7WX_R_tSLy333mlox&bN>6l`;(~DKTk1n4EfMhT*GQTUUc_!&)1?eO1s>
zFA-LThW{LQ9UDy!<!yhx^?KapCnqO|ZpjGLjoj3-@Y~1v#TGJE&*zq#WL{FS{r~54
z)Yh!2P0h`kJPZu~6{DCMO%7$4WL;X~DP>i%qVE6S^3at*uI&7BOALOTKXUZwO5fRL
zvrIB4X@#y@vNn3V5CbDa1<wT`jwOzXiHWbSt)2bn@87Kj51pp!Mo(MxVB)T-ud6^g
z<->;p)BJle#_8v#+{{@93Ou$9PL@jx{N`G{Y?rSKsHxfW>e|}Xsi&s}ir5$}_n#lO
zHS4N~oZLF|dlk<2>wfD(a#2J3+OoH|Zf!_x{`2S0l~tkIzB7$l7d=>bdw2PIt?+eg
znA!PWJnpx@mOj7s*cMd=2Ai}Od|XQk-TPz$&(1O}eRIQb@9%fJrEDq;Tn{>fPS)9)
zb2Dh;#*MS=YIj9#&+~<}@qXyLxVo;)ySwYx&*$^6F7uVH{rPmd(M%r^HvTh*TDh+-
z^%np6>sQvvNvf-&wrYW5)zGy(-~p$Ah{zI9jb8Tl)-2oVZL`dBwN_vCO8nr^BX1w~
z_V)JhWj-^vRDMolQea@{w~S(HRPQ*O7Q8+0u1W4Klf54faf8|cO-)T2Jny<MzYO{R
z?=Pr@EUFU`usVEw*t<JBL2(4tcrqm@B4P#`kHi8{QKP7+XjSz^qxGPzw}(eSJD;pc
z-Jgo7`tflOA3g+4#07DJY`YR3U%M2PpA8HGy2bUwPEJzQ<a*~-QnDp#OU6Rq*=8?6
zCD|Oy;wgHuR$R;s4qA@h3Ljct-P^lcOh0bTj2RMFSA}|mHq>kUa0gva|Mk^XP+Ruv
zi;K>nLtGe`KpnaXQJ~}6USC@~dHwqJptyf|d3mtNpS8^Fd@G8cp1QTWe0>|g{5noy
zwG*ei|MxO5Ot^JHR-}PX&Su3d)9hDIPEP*ypqc;6zu)hJMO1|4<kqE~o%Qlquk<Xd
z(p8{q`5<xRarWF?>*cq$W@n$DXZz~P%E@-W-xy2wx^Xe<#_kG{wJw`-GUd{tR&G%F
z*~P@*pyg&S$1rKho12@nKR-LW3UpEHp+i?TrFtJdb}VRy`kf`7lSQPYRy8y*<lfma
z5mFjBL`_n3W?LJv(P`nrg;5)mT0wQk>FN5xGt{pvaBTi^Ejs_@%=CE|x8>fJa1;ZT
z@(o)~+`b*XIqhuF{e87ti=X>}iuNZbCyPAeoHS#`igmHOUrqM63w(KLsi>UXx|*M#
z0wow2E^)l$Wcje&?*E_Sy+5DL=HlYI^yp}J=pq-+u1Uv^9yy|6ViE!>pYQG}HLd?w
zv($h7yhGrg!-pji8xk6CZOK%YmzO^~)7U+9b=Xpk%qy+j;#YPQCcnM6HyYHVUE(=e
zLWYB(!KCBe0)}@hA~&aj>cJkj#h^peb{0Pmnqu<e#6)FK54`OCy|s?bY?-H~Xo8Xr
zM<c_$kf~asm#(dio@$yMR{HXi>-?HeoUWeE|Mq-0HZlsDtnPnpf}*nq7Xw44#9c?m
zJ6l10=diU=x3=fckE{I}+AC@778mzt+LT$KaD01f>tvO)v(5d#zP`R(gMnd5;h(Q;
z57x1=vuD4)wsuv>%1LUzvzD0U-qO(1>pQe0C^0cH>*Au;OUwP&e|>#j7*ZS_c+14Z
z^yP8C{i@2(&!TpftOOlz1iBP%M#!`O|K9(<vN~Mf_V1U=T>SjkyY=^Zfa+7H3$h{y
zY74FAzPh};A9Tm4Q!5v!%HrbVTV~L2x%=+AdwZ+1e|>pb^{`cZOVLxW)U&guLPEJ=
zw{hAT2~bb!=H~S4<@alsE4g+BOgVn#>}>O^+j65p3m8nZu3T8;+WqADb7&j>z}}h0
z>3yKnA4^_d%DlIya#hGmA=k<?F}q4Ud!@~<t&81#Wm|5v?_8_Ud-v{%urV~K3w&o`
z`f)vKd*0fq+To{OmaIxV+!nPZL(sLdN5;M`Mp04m*4FIlzrMa+KGQfIT<$hq6ykW$
zcyn`lf7aDiOFut9zdCri->KQ=`mR6wj~+Xg_36pUT{S;99qX50KhL&0NCec}X>nlS
zKX-Dndg$gfUs;=q2}w3;XVNx5J2P|fv0mw59!R=+aP;``*W2&cO+Nh4W9#vPFE0XH
z+uBwIFZX+OZEf_be|AOB&#le5xv4OTiGjg$Ql}6@#f39zo4uy%&0W=hFlnR5q^9QP
zna=zS3`;b5c$gU=<A)#$v|<!&1&AX6UJU@6@dELPm2y%5sRk`pWnchxH$XhdT2}@V
zrHHkWpgW0njGU>kD#M2*(&l+{>V7_*o^pC!++HouU+;FmFL`vNGi811i3yI*Z9IXX
zuBsdZgGVW3p$vn~Nl*^BapOjH)p>vWze^@e5a?oOU<i`k)y!1!Xz!;}+TXYATfcri
z=p=%N-~ma7pv4z1T&RxPpL%+l(d@G#eBf1d3Z|<fH!oviW`1Aw_uAU%U=wg$G+a5A
zVzjdI^Rx1;dDC<vcZdr>6mQMGy6S4|+UuKAy+H*l8v{d-t{XcmgQwr};GIQJFWuan
zUb_7BqiwmjL$>8aa)ITem>3-<9ZU#V9k%wL8G{rL+tYJ%XM;*pjz)$FVRteQh%dif
zd46#R69a=y-wQr2hV2>=8y4`%+s#SpesFJZ_18b2&j+h8Fico=QHX<KI|ncCUR{te
zqMAW1Fk`MQxuQB{-2w&%1>J?#8VrA2R|YNpa$0}CO17X)-Jczwo}6?pWMFVu<?gM(
z@aO2-SSMb#X3*`#43OznkJVAzax|L{Ds+O(0L5Kq+_yJ3zkWKcUp>z}`&!Q;28ITc
z!!Dc*HXMb8g;6_;mI?_AUq12?bVRDOS<Zx;IoCk9DxUcsvL?b%R4b%ISd5`TT>>=p
zt^n%g7$zSJ=r|s8kcqYA!2?GwMur2zJ<}W*_>Y{OZLaG1Ui{4RGj?_&PYl0Yd(U3-
z^HVApBd8tDAUDS<(pbCurjMeW^Jc%ffhwsC3=cYwA20x=a0d<+5JfO463$HuEKDGi
z@MM{P!D?O%4>l%pOxo#Z`ZLW>U-g*Q#~*e_^KR>=H{PmRHD~&|^AUQ__y6HY{u#!#
zCep_(dd?fe@Y3ST{F~lc`MrgDM|J(Dxl=cLo6hXnuD4b_{kZfd&1N4v_tNBJMK}K+
zP5)6QyYP+Z=TmiGSM4!+HqmiXe5cW_PleU7_p_gSGcYjtAG;qrh2cQG-ipSeeXqD0
zzO7~#{rdB_)z5S7yS6;tkaA8Tf3wPN&tvr;1=~LDow7G6&N5%_%1iV3;3rLg--Y)W
zrE*QuaWqOZ{16LX9>QRg^Q=vO_ioi2rEe!M-1N>$@A%Q?T_3+H%$3<6WaeqadUDR|
zfBKPiQ@eNDJ-m`%<grO#dfDa6VlN-PDcrT`<T{<?W18MipZ_poU~rh_ULL^k-*SGA
zf2Evs&E);h{!}j0Tcf{DCe-Nue|Cu??O46;*BcFAnof#lgRKSFG<`?nLLt?q?@j&c
zUM`(+q+%bWux)5RD>?r~T}p_<bi1y~_mfk;|5Me8Qm&r*&4@L9g3@~R>h<A`U0&f9
zv!`TC7D^J?|6Bd0o*J)wvQB`<xgBSIZC<?mptZv>oywFG8aMgpPT2XHBUM*a#q-m-
zZ#{eFZS$V)e<HDT|NfBNpHY4%c75d6^mbgT9UQXki`miq_*E?)-=9}Z`+cJDS9s+1
zzY{+-MtwRFvu{VuyiF%&Rs?stf77U2UmYOjJi+HD^Z!q67MAP0;v^?e+W7el<IS~m
zj2Rdn{F9vHw|ue@!;kG%-`=d`6jsadUTSnI$?X2+YTvJHTH%`yc{)CSmKpn}WX78I
z#gp~IB8^T|emtfdzCP*HmwMZ$@^9*Ie-rw)?%K}&<);%lf|hN3{A9l2)AOY<z0<;v
z-M;_+3hPms9h&#|_@BO=Gv)D@vRLc+!Hr9c?r!pHT32i8!s)t8Bm3x{lZ$U}%e4A_
zwg2R_rSGjm97R8$RyZk_KmFpHU55=5zt_x_-0kv7YjIBe*FdY!YkwEUP2X?5^g#Hj
zieK-n&*g_dseb%qNB{d3sYmi$XVsK^yrk+E!N4#t;f^rJ1N-#oeIbSyw?Ena<sg@o
zfv4r~{c8E;`sY#@FRbp_<o`<le?t9Dzx-<}zpX9V;~XA)t2l7dli(%Gn(W_bSS~Kv
zd0yJew5BWRuC47`^=q0%4+Q4Q83(1S)hs#TnQgna&-{Ap%PR*It*o>A_x$;jQp9e$
z{=p_LovF@+)!Sd*I)D7>l#~hmMrHEr)`))AdU&8LHcD<C%lGsdr%i*Zye}yj$4&jM
zm3{QjB%zY8pWj(sUE8Z9{^#AT>q5R?LQXQA6kPauec9Ekmpx;Hgq8PQ?o~}L{ktcd
zKY3d1y2;-q86b;e<>b%ZFWLCry=&*Jnpte~o%A)Iw-~VQ`{cCg=`{1(r?2Z+zFl7W
zu$gn6!1JDqy&K=Zo@H$y`T1JFvR5}%)2F;???3VI-WKDJyK<KEX?*-{yGhnc@%F^*
zc{|zXUhh9y_o_kYnqTmhhVP}HmsRilJ82QymdiW)?XO?``2O{$iL>X`c1D^-rQ1m@
zm;QYG^`GzS1l@V7yr!M{IZ1De<lj5}Z#KE_JhUeFaoqI=bE8x0;eOxQkG{BdK+~&W
z{oB}0D~-NNE$q43<8{Y6UR*rqaSS79h?gN#gJa)Kx94?#LbpY4THyFn<id;C)YqRj
zRXgVW6fL@Ioo>JFvhWNW#jK}#=P!TJ-&fLaU6^ft&Bv4fvA%|$(aOKNAA@>cKQ3|J
zC|y=P?KZ#s$1Pe?dnRv7wXX`{G=IA({*L<UYU_LY&;5Qfy}7Z7`(2n*v*G4H-cP3*
z|5|Xaeosp=bSHsr;r*Vll;`Fpndf(=oW8vGN^IlR&z_N9C!h0em$5Q`8?wi`Q?gj>
zUwAQhmBi%!g;OOe5B@!&^7Ee2_6Jv4EQ8N%H!fY~F73MSbLXYfiuj#rXF>}$f2yx$
z+@>qNP+NUcl(g`-KQHo3S-_1UkGGpdrteHyxPQqqjk`a-=+6E5?t4o2U#%(ARqhsg
zm8UPs`RMuZZKBWrydC`+nQQJuTB@H94Vb#9NbX1A#N>VZlQ(RZ+x4vS&F7%iDc>cS
z!0mt~_qs1|mrhyyo#E&6{eO$5Y)EN2muES3OI}}k>Li=Bx&N<Rdfs96D`LxnpC<CF
zmRpBS?!US2^=A7EFZ-9BEK_{=o_SS#XzJQokEdOj85#Mj{Amg#idN1~{;qg4HRRa0
z*q|EQf4A3qI=*|xYpItP_GZ3v=d$zKr{fPVR-C@)^6u@mZy(vd?f#tTBbLuG>A&~S
z*xUOn9j;%KDc0@x_1XA(1y5bos(SXH5^s0?x@vK^Kz?=jI;~Sd-zEJdzzv)Ur($<0
z%)TpHv}s%Y&u!E9O7B@=|Ng}0xi%W@#$Fe1$2y);kZ0YJF}LRLGSer@*B9u9r|XpK
z_xeU1{5PH7WBQf7SJc;Qr&NaLYZR|5ssAmvzQJBSf4gVlcDJ2JM3_>oD*sk8JYzV&
z@y8XtB5&=qpdVlIzO6H2o1J3)Veh)!+v~jJ*MIflOw9bZtHv@J+&ptwRd(98=B4v4
z?TQo9Ug0Y1_iPfG65sB4D)O-D)uzjJQGv<RJtql0t67r$?&lYupRcu|zq=WE7+rm_
z^i65(+PrJ2!l`>-aAm|jeRcXxo~+3;{k32G(>;FuIl(=z=Xva|rJMMF#vIIzI<?**
zPV&D`rrTzpcR#sOIf8y2vb?KkviD5tyw5qOVwSH_e^U8e_X+0$zVhH@2PM|XOjg&f
zc$D*WP4pFEyUXShOKgk}J&9kwd(-u^`pflR9&-O2tN+g|bo#fL!umL~RN2etjmwo7
z7!Le3u$b$op33kc|LdEZ!NO`j7p8foPC1&hZ~1q&o*9dKZt7`&n-lo5+4$bKU2p%$
zx%Nsf{n5QWW*v{?r}v-xjKt1QeREwW{f}zw-?TOB@@>>4i?l!Acz9~U$8Xl`Z=Y{}
zy`}omSDyEN0neUKwln|sUHOylubfTQ-{yE-_{#b^B{R(XUj5B!j~~g!TF>@wJjy)z
z`Ry0BU2jZ-`hw<8T^(-fe|rAqI^XNh&P@&9n#XnGSya~Uys4TBns<-p{quUhJZDPy
zQ@c}FH~M@^-ZKB%O~2a7{{4mGQEqR-mYj4K*N=VF!pZ<@C;pG+FqyVWYyOV8U3aB*
zBR`#3{$sOIWQ~fd#>WGEs&4mo91hj-Sf*6@G<DjX!a%P()qnPVyC3nzb><(H-xE1*
z>e;oNUZm)8PxZ6c_IHb`XDaG>aem)<AG9vHV7(8ht#kI@XMfM~w~_uDihqvW-Ckt5
zIOVtPtNZT{zqiiY9s8lN$W8O(^A+`RYzzz^CKUREdqM(Cpf%MDAXC6e3)HwHueFG$
zNk~wtp+0313xgbszW()mFHv*X6FE=5FFhW(=kCHYGmQeFD>vS6&A#q;ykCC$-8_Hc
z<HEwii)F3LH1hB5IW=8Be%8-<pd%SiaxokbJ_lI?#4rgouGb}|y9zWWabkj^2nz$l
z63H(&!7W%4dwY1ipeR6oprHgcD3MUwD}}fd#aP0Td%!ulz-q4FDTW80*H#9r=ib{B
z`SQ}z&pO^79v6Ooel98?u%MxV;ryo?HzHVBSzpf1-xoN~rcy%=TxmFD2@41;c=6&z
zb=CQ4y3rvEomi22ms_*1zgu_RRD$Q!G~L;tZWO$O8MZd+?$=#@%Y)ZOZJjh-e7Zja
z!vw2~vLXyBPft#E2c779H%6%Z-JO*|OT7Y37#JpKeGE2Z@W{Hm>#IHJK92?I^D5h5
zow{3heWwR3zr6Cog@7(r28JNhUCm4l(G!(ix77dtXT6O>&Z5A9m0N5{1OtP@)kUfT
z40qmLQN5C2u)?KN2sBjXz`@e6MTF@;U#ue|gF}>Kc>u!>*PTUAw<I6$6NzR~R#q03
zk@1-b>KD3sD=^$qTB^5@t=Un3?-wD^(6Rs%qr%$Ei;Irt3Lm^IDgv(f6neS1xPq>(
z3Z1NLJktj>fPWNHb9r?CvvTk`JIhp5Lc${vlIt!lI@T+F`uOqZ`Ob<^5rO*p`m3wM
z&wt*Nc(_f3m4RW2B&cKMSY$O<$}DHbv-9)y+Ye97>VN#}>+4A>^78Ui^<u3;>(yDg
z#jb!>p`4j%T-?OO!0?Qd<--FNH8s!x3}{^W(o*l`4Gj$;vT19>R@c<lrW!51cmK4I
zkWk6fQ=*`w-lO(bZT<B0^zlwE1_s3m0vr$S?=F1Y1{&?#ntgp$#m7glKK_4mx3VHQ
z_20jb<&&0WDok<OSdut#Tfou=)|)AUj*N~67#&?5k2qdDn0Rr1S6$)#QWjR($UBVY
zE~&cR8Xap+DP3T?bSNNd*99?+Mibx3fBt!RM4K@RGQPiFde-#q$w+&d^H;6U7ted%
z$gZ^~T)lp-s`s=HFPG0Rx|urNu>N1ox_!U0VAnr9dGqwilY+;+=AT}#-+yVLGdmlf
zOvRm;>QA7-FxkoK{%RH$GiDm6U)q`-4%c%g_x`@Q>tc6rs{Wprx9?{f@1i2c-*30y
zpJ9;5<hOh>==}TL?{;~E4pILpeJT8_Qqi@rwNb9K&2n#SPUkPbU%Q>@0qZ))X0{$V
z+gTSb1hDZ+9r4!Ry99Iy_#fVtssS$NPQAIgdFR`0w?9<a*nBv^98-EVRO^&$(*J*d
zd!)_#&YbZ9-BSPK<KwHfpj+rcXH-2=^PhKyncv3Y>8YubhDj}MbM|mp%J8ZA%s8+u
z_x6jsyUpj<{nAuaRD|7L;Nc2709}922PQ#5!5%5ou34tpt2D${Ub++nTEGH2qItGi
z?hNB}zN@Rl!$C)=i!iym)HFWa_xs)B!~FJ7KAqN2e{y1COIzD2198js*H4Sb*Bo5r
z+P&jZmv+RC0!7fp98hgf-oN?q@bH({*ZZG5NdevIu_5vBh6o+4Gpe8~R%$*T6?buU
zoj7OCnsOUg(CLb<Pu^$U-&b4m{@&d9x}UAva&H%XzgwPcG&5v|_k~+qvn${4{k|b;
zE$Eoyl2=zeK_Lw~&|?4OFJDSP8~e}AwSIYVF?;Q=FM)x2h0)vdp1$3Ff7zBTQ$T0)
z|NHxUHRyhX=@Px1dP?eFKA*QQK5Ke?L-ut&(3}Rt5A(vOQ^Sp_zGP^HuR9}Ke&?Wc
z{+@-P$cah1BBY^o+|$o*S^oWfjRzBEnPx9r^yjRIh=@Vn9gA=8?w(!|xVYkRuepH)
z4=kKxvh4nRaDKP<dz_P#lZ0Us3+NJ-Q}?etJlrm6UuR=o_U6I1+j);cQ%4OBkb_W)
zG%FvridTF(sm{sI|9oM)oRXH7m+Q~j#XDns=Gn|FeSPiXLTC20v$I^Ut_p?Sv9fc<
z&bxVXH6I++{N{MPy|q=+G)p8gF;T0B;lam>A73u}|NQs+edd)Fg4|*{3TkR@t%BfF
zVJCJ^Jp6D${r`VwW|?+tuirC?UA{(O_0?5}P82>nJNx*VnZ}?~^g((I4Fh2T-R<e<
z$Y@*jCE@F<t6Q?JcGds;JfDS?b)`X3kaqaGCx`j%mv~LpIy299_P4jU!GR4rmc#JJ
z{r~^mudj>!`F8vL9UqTLzr4PFzFYki&;lf}dzH^+_r&Rg66y9kMcuGF3&c0|N}FF=
z8*N@xT)gw|x7#o8?A+X?l%Dea-Ca;laBk-_tou^|@-{4c>bcyn{Vuz=`unmSJ7!eB
z-#cAQH)@4OQP{V)w~yc0n0)f=*{2uX<)2Osj|<G+1U^%^BgSKy|NL`z%kTRZ6d3T^
z{ZI&xDQxXhsz0&NxxM1`+U+-X7OOL}@jPhew>ywywi<LN2Unxgr|{~xTd!BV*?4?K
z*jlN@ZoP}b)<$V{%@OunKDkp^9bBU9EKW~5H^+0TR%i(HDx9C%6*h8iy;4Vie0<#5
z*Y|8;yWFCxudlRv=6wF0^Xke<o#<^nOO~iinmoC)rw4p<0qA~>nj)*YJTev&K$lf4
zaAa=dm-k!hH8n)y=Vef|MQ%<D+@2Q;x+}@a34Vdj^TUnI>@Tja76+~7tNi@zPWAh}
zRpH*}y5;NtO!T+=*|K4S!LxI7Pn%wkSqw_u8Z2EW=4%=oPhJ<hd(x~~r@mf~_y7Oz
zFX&RFo;l|G|NYViop$cFcp~UNhPb^|@JmM0Z){5CHosS)92pt;<?Ze1@9yqit?_9(
zDAe-zeiduymw$G*{QlE@zuyH1LN4oi5)-v8N7DR$jd9uTxz+D>y5IfqM1T71*{5%8
zOuo52Uq1O*&%>Y3=ZC|U&oE48V`k$ykYQq!bHhMLSU6N<r-F#6sA1k6i?a9k&f0#z
zb9jm8WDYK_AXt%;v-j(@=pGr%NgFm8xb?|+f-VFT*||VOT>SI>|No|gZmro{T@E@v
z8Z>(gJuvsg%nwz2ZGOF2{O7s-e?=psN$qk~9a5%QD-4Ppi+10w`SGxQP0Y?o&(6*^
zuK!n~xBHC|=o$~uA%TAGo}Qe|?EKFz``fGP>GcUJyCrPC86vUs!Pz;MlPf<z6BHLe
zeyEkZv%4D}3L#xxT`w*#=l7nbqo}Fb`TPC;`41i>h`Q-X-LL;|dwIG4a<^WoGxP1|
zKRY`co{N-j-n@CI;&E@ztu38=vQ`EK4;;=iYKSlW`ue(Y<)@UUrY6vxIxnuRoej%F
zPjhc?>jg#X#EFjY@9lkgbMx{pCGo9WwwQo!(dg>#o@tcI)y&Qx1`TeOgr}#betB`R
zxtX2cDC2^{t?l{i7oAX5w6>nTG5L5<K)`{|=k3qWFiZyB`XSirFmeC($jxbmzu#_u
z{I#m`K_h#{r6rwRO4Fw>v-5$=#Wyz(|NVad{O$bxy(d#(5%jN4&2P?$4T*<0+|22b
zFl?&&`f8;{k!O*GjL!^%#@yT63O}DU|9mrje&W_BXc;$gf4^b!u@8^B^@Ril4}z{_
zb8%_uQkuR6RA<%yuf4V=^04Xkn8SB=7O#xkyNgR1d?IGi%m0cs@ArQHbkSWtaDAMu
zTaSceM#c&*<$CV8_0#qD|C!XvEk0?&gae0KxmQMSUkB5cx#{*>P?g2XEe5*P>)+S)
z^@$rJLL`3Pj@*(VIKTd%<-~~-Rn*ldPniO)x;0q3Vxm5M`t;)7UhCJ_)-I0SUB)A2
z;?bOZ;{Gk9I_vXU^O*X-U!m71omi@<tJ|v=y9-qPRDFBn`TpMCP!YkJ&&m7#e!D$o
z%9IZm-Q^3P&n;gCs<OBmmBhEtum3mm-QC^65fLXE7@2px+x0p`WMYw#v9Y9ip3Lv>
z@1I`{j|UZTq0s2m^8;O)6TiQ1%Cu=lB`*SIndMd;oHP6M%Ok~Sj5Z%_`nbic?BN%g
z&08<_9L=$L{JG#3_mBJAOqw3e77`b$c{SCx>{8IB4ZGJKQBxDh(%V0u`OV#_ZVOHO
zRzF@HyW#th&R6gB6~wxCK6<)q&W^s=pDLx_ZhXG6^SpP?-dW54nSZ%)^2W^8!?Hd;
zf2(fvh`YsdadS&rm+4f0f0z2>!^0kFbH2qFSG0G4OX3qpAOHP+KmEmpg%NARK!sdQ
z%^%j1GQYZie=7Igda!E4mskJz3$ugH-oN=x;C^pi=h1szFP9X~uQy&0y7sbN^WWR`
zPp0xeW3Ag?`Q!4I&Xpyna=VPfKYnS+XUwpDAXsI-?o?lDab^C$R|l-VR{x!?n42BD
zp!{g(Yrcf6E3=ogZ(H`q!0~4Gz9(nbUk{B~AGv?xoohC$bS_(R-PrWaecAc=Uc+7N
z{=FIJSEk*!-t?M%-rOAw%`e%G?YSa(n>A#vvU{IF-Jc53^1NSPUmw4+GT5|WQb)%L
z$xGoYl<FrwpI6<d=-d{tHp*14{?A8l9R{X|jKbgCd;Xl^xFvN;PW1Ku825wk|1O-j
zaj6qSPu|_>>hX17dzcg)w%$;A_xAeBo^6J#(^;EVnQ<p9yB2!g=*_&@mQvk1ZQSqc
z3M}V}zOvi&f2*k7{?aR(_mucBuykaVYOlBZ$-w(D_KNP|d3xduZ*!uIdXLYq?0xC(
zT(ES({mKv3H@2p7Gx+{C*|~s8!C|S_)QTSu+cWR&u{2IQv!M9-IT?!r2YwOo$$=BO
zr{8{?_Ts`qo4;Q!Us)GBTYX+d(~1=<xW6kV_AU#2zNz6=ss7{XZ`u7fwoZPl{X9HE
zNG|sEn_VKaoICTUxd<(vf9s5(=b!tB7fby6@Hi%XJr~2Z4F#`MR3;_tFp#fYx96+P
z&o^E&PY=gl;{D>Q^ZM7!iN#&Bs=aRXPhaq@{Nf(JpVNYj6Sqcv{xP}#lCIth6aO^%
z*>$IwoK<|KuIx~FooY6F=l_4dcmDhJI&xo)CFo)iW8+|caj4__4?kSsKi>}2fNJBF
z1~u;v9Xh1<<MTS1xWYxhmU5dV*hk7)ZfHJlJ|!gLS?TWo9$9vek4I&=s2B<+9=Cbm
zwBq6ira5_sH}^l;Xl!tW+wxhBZp_uE&kp2g?~6Hp^44x6^YyCIFD`HTSNMCyWC8yf
zXWgDoxXgXY-KaiUIj*NrcZbcFLv!aH$uKE;KDYeR_Wbyu;NZ-=yG*11=+1!Lc~|79
zp`o!L`FJ0w<&?c{XB+6Qv#nA3UB%xmyblO?ui5W<ZnODJ-$-}A9eKZYElucHdcPs=
zPu=fj-OQDiDXC8<OrGrQIPvk6Z1$N=2a?sblf@+^BcI4_n4iKRx<UL&z=IESE|yw#
zbDcPKN<mq>yGiM}=l9vg2foj|9On?`#wKaWy2CHZhMm<iG9@Kt!qnh~&3`m^az$Fb
z^$n}AZIEui+<G~3+wun|+`fF-{-E;i>4@S*ViD)<uARBpWVlP3Q}AH@m-tydy3&ju
zSyIQ37L=F0zc*LWxees3-DPhhw&l!}GgDGh+OO=PD)93&X#MrN*xj3wkMr#+eeHIC
zU#+18kN8DVkz}tNx9P{W&iHY&pW)13yVlk0+gAMOn^nxOEuJcL<zL>q6)#hcX@1K%
zf9uehkC#1`KDrTd?8j?~s@q><<f>{GSWem5dHz`K(PKroC4{YwSN-R8nQ<^xKOsGe
zOZu8#g7dD~v)gRXZY|ZC>-OvRasT^!j?PJDOU}IVRPR^Z{Enj=`+Z-VZ`4fX)O?wj
zwCm2fQb$dn`TH$5tXmoA@5Z0w{?qL8owwSOzYnhII&Nf=^1%6ThDc)4*Y^AgbK+ig
z$7dE-{9I|taL49P<gFtu3}NeDl`t1y5@*=(<@nhxz8T>UZTC7`Z{HxMzt-CH*W>r^
zcYfRWFeCiJkB;tV?{Y3L;|*OM_H=Ify@lS>^?YWV_169U_41&yl2VaS7N^M1$+<T+
zG=fsbmMtdlc0A_$_4Rf6L{<m+{EgFJ9C_CG{qvpZeM!-dNAu?IoE<)~Yr388>*TM~
z=9I*<m+L<N`7zkjH0I>pe!J7F*Y7)Z+W!~Zf{#mdpWdBr_qhJsV|KkblVrC2Zn2)n
z_1>O}QVkaS{OtMa!Urj*7v+1zS<UBtxleD2<MPY1FK+I?>EEAvZhhyU&li_he!VLG
zV|#tTG~I5y#8cm`8O2|27N0!nof_APs4te6C*Li3e}DeFwU7ROopHE-<<G{u&+_LT
zzrmk&am{ssd6hpy<0?KmuIn~xG@rUMS$$$2|Fc&IlXt9MdsMz|_s18;=RcmA>|azG
z;*nq<`e=6H-2bs|{<ma;Hg}y~9=>gD3{zX1i9P%K_p&GTUUNlRpD78vJHNBD6V$Qx
znyOWJJ9j&%dth4wX?C95f3#b?`0du~pf(JsMZ#--r{H5;&*9#SF_X>izGfGz-I?Vc
zS?=Snn7!{-gW*1PUS*#h{$@XWvgPmle)0W%A@hU%4x>Bb6+PQBs%#B!TyMQH`_z+4
z>HgPAlK21l&6Bn4kY4}(<b~&L>~6ZH`9IruCG%!|lRR`H@i4P;>b$-)q1=%VzZP6(
z-kT{9$7v9({gQ2$>rbbEYmao_Y<DR>y;}dG`9&YIgr4_j4xi^=<Qya<`|FNh`u|;)
zzyC6x-Pm4vd0KMMyMy+(t{u-}*}T?F_IJr=^;a_6-f11k*%q`nO__m5QSrtau{CDW
z)42SftuooYH}t{w{Q6r`E4r6wsK+ne!I5>nYlgjzcT;;${OMgyxAYE0Z}FRJW%Kcf
zFzBi^L1niOCzSgaL{5N)*zA?T%Rv_h*vRqo+y61pjoPx~#rl)XSN8tebzpbhQhQyc
zv-gwAGoG(s^P%d+osEWJ2Z|@{-`ib$Iig#fhc~mN@@kiS+p)_lr>>V?QGHmgsIO92
ze`99zrgzJyW$d!Kq5UDUDlPHB^UL+d?*wkw`LQP4tG}Q6>B-4||81R%w$(dGo#*=W
zCQn1$Q+T$DP@<l7)YVt}++SW?sm_<WbYhqI`FlI}NZ)v#GOsQ@FF}5F+4Jy<`4^*W
zK5Ymx-k2K6{ibc{64?)PZ_his?<Zq^Ufoam`Wf*{VjcTK7rXJ!t9Zou=+UDe=j;C+
zZeV2A(A2!h?CIjNL}stB#wYcPr&Gf#{(L;XGH$QcVz*u;HMM7#c3MQ+=H-NU^VgqZ
z$>+0QcYeXatrP$J`dcpHBby->|8>gj_vddFclq~T@qNRRE!JQ@)%K_R3faOO9$jIf
zxen3I6Gh|rZx?-kxJc}o@lLVlikmjsCUh*lKKJcPu@{Ut`}kyicHGS?aDHKU{O{h2
ze*DkU7zOUHQ@?JuLQdkrme}2VGYTHF8QipZTj+oG;+B_j^LsznGw1VlUw{AXlwaZ>
zqiXK97w2LGB|o+Qk7SN@Ts$kTB-XKix2r5uw<v5aK(3iw<r6_r(|h^+I<LGuz4-dS
zrVAG?tg`6r=+I#4niDP8-wsL!-)`p%3kx4!5x96o+}>ZRWmDaxw!CDp_hnCT|D>O@
zzSgq%l6U6a|5^WDw1{6@m3~3+V#Lf9ZE7AL8!Uh1=&4@Fl8Aio^(bfgE4NoRZ|}}(
zs;MpwS?B5G^tU+Z>#p$a>RJEtQ<x*WuYd5)@V(z)AS?Cx_+sCk>lUmP);qOs!Er7=
zzGny7<x@ai;>b;)?OU4@4mNRD`S}S7PW=4+`kanA!bguDy;Jpi?Vi8iZmX!MutYKG
zpJaEhV5*({>-HU+?4JpbrYHC;dOFqAzh3=b$fi%-{vr!&{-<{=O}lz|{*nh>uR_nS
z5xuaU<-qRAYiiYJ&N~$75H|1cnpacyT6RzNtqPnlwe_!0+J+5BFYk=itv_pSlzhy|
zz+_40A<GxLDz|8BOqd~1`0~zjH|1tF-bLl_?}2(`*VaaF&b@7ByT%1{biAs-&&Qw(
zgP)(9JJYgQO*d-G15kfIXsOrqGhcUlFMPD5P)ut3?v;nEzV6;?x%QxE)|GDlKYKK5
z-xto8*Zb1HA#;Ue>go5qABtwY35j2RYDS0F@mi@AtH17d_ME&wvHVU-eOcYCFKqAg
zZr4ZsHjug8eQC4#OjhaQ{^<7}$F$;Re^C8>TK4>h-II4pXI^-9tLyspS6Qzj{{QgU
z?7pe|*RCeF^2eXL_h`T91YNmyu$g`G%$c6E&2l4l7A=*7-Vc(Ve}10rmfYLZ=GXn|
zl+N4H`1}2S`OP<LIAa%fzxfhdQGQdqq=RSWRSSbAwKk7+zN>-_8AQdWbGxS=JS1~b
zVZ+_NpQmp=*05LTYMwH`W^%3h?xnmFpCvoT9+i8;ez~UkqQv6e7mYsXCEeUumO5>I
z%-)vf)_GltXJtG;m;L*0YO5{r{bJCkKf-5r%l!WF@^ko%$-&bVb+bY%Vy5k%_lb-D
z_pQHKY9@R-(SL77?DIeSb${p!rSjii^J<?*zbb$4oXfq#yS91Sj^NK5m@D2qc_K2m
z;QzA@wl97<+NXSVll?y{=GnO3_%dPQFK&j;=^o)1gfs8&vpw9-|NO*6Wl%@$%q&xH
zetpPTnep^lv%FqkTYGa?srL4JRodlu3Xi*5vOieKlu&nzEo{k-54-j!Up?kCr%|uR
zY*yLsbK>WsO4aPPyxA4oSGy*DpTdXgQ~9?~ZrsguJGT1!-F=n+_N@F<eoLrGM(%@!
zN2JkY%a==M%=-Fl*|R`*V=i~)i>p$zPg*4$e_AwYcde?vZzMxo8;|52JApVER?8>9
z)F<m}-H_*4cdp6j(&k$A_c!V<?7wqjs_ZT{H|wuUR6idx?*II3cD~WGng0#ao4fs*
z^Pd*2X6(1^-Xe1U$h+w8&!&`qE)kpfo_~6^`TJ!DSdVRBVJlxFy5nD~eE-vr7av!A
z-e}nKH$klJ#knuJ;o=Qirb<kTCHLmMpYivecTfDc{g0O&|FH4WlG^ZkfxV4xhRd(t
zdb8>IgopO!{tM?eKfSysTVB)t$F3%|qwVpR{_i!*ywu_@UkkbddA?ok44cYLa!E=`
zpSGub$P!YWxVh~2+wGr^ipLk+ul;^;SLti-*N<YVf99O2Fe^HveCWc2hv)e|mH*XA
z`S#*^XXT5?b$uUr+cw#k%B`>Y^O&>z=8I?b#v6L|Wp;Lb+mQ2*Ni5%E^#;G+cFr&E
ztg?GrJ?+`@wU2kdPiIqWwAb1foByjM(|5;F@rdAOK2IjkH`127a-%8QN-p8qB-!J0
z&#zrP@9C=(0x_&F&&>NJm%aIP+$Cd|?I{lT^-Q`}HW{be`uF~RG~@5KlpFuOAGKd!
zHX&-C+p%djf9~kI%$WNo!f^igwDs#YL~edperav3uv32>>+>6@&!ivwz3eQ5&)F*D
zUGc9so$oL1cydvH;?;5s=UM;nDX6#qcsV(I#$)~lQ9t{A(*1X<Wff<xuBbY+UBbuQ
z;Mvl1({sENDtGSOY4iWj=PA>s75(^-*wkd=HVa&ApU8Z5bF=!%lP7&<8o3^|xV`3O
z$%BGj49e3Ut9jYq+?BBH#)hbKsp)L|e7j|*vm4CXcyrD3C97MGzdTU)WzJf&HEEAK
z!xo)*$Dq^F_U~VUg<iGb(rE!}o)s6?xjuPs-?TEw^pLY|{p{a)mzH?0iQLTAe%R5&
zqvM~mh~UKiqP2fDmFhJ&AMLuCWc&Q}JWJ)UXpQGp@7=nmUbt_azPR;*F-JmLw8pdI
zvbu#XE?NI<1O+GV7v3uj8qxp{(SR@Zz<t99{K}7dQC~qtrTQ75v5QIPd)$oEKF5Eo
zoy>l`(Pmrz`GsYhRSx|v*1A!A$@8>j%hyd+Dn~+W+uJu!o;IoNZmz1CVPn8&)*t*&
zH%{JY%-wE1W5%zq8(wBRfIEnS6OEszJQhFyIkfBblhqPRo<6&p)Xq-!=;?oS{;a#+
zHr?NC{Lj~ambae9K6w&{ebu`8`_IqGF_*ag=|%Tm4tsaU6&3H(mihb^Yq-WNx3K)N
zl=z+fS0>)I|9ox=D<lG|c9lXVH(buOwThMRby&Pdx8~C^ld?yw&N|!wy_S!Wdz141
zok>{T-i|4}mP>LXx(xS8_y7HDb?D!gFb&~}oVk)BDbF6?T-`W<>+e0?tGyH69!pr!
z^pt;2^XZpIax`Z@xGNI#tvGb{Q>H)VpEhKl7Gs$8Ewjnjd>wdjrQ?KrV($w7%|{f<
zmhCto;1v?$d+4lyTx9z(-C3V6Zwa3{IjBorcjo%KRfi{JC;wk!Y$zlq=`5NYoA9-;
z@$0MGa!bqp2|BHEpHnJO^mOg>lji)NzIe-S{55^1@}wX8+ZS89a0Cj5>g}mlxNc|m
zcM5Bp$1IZ-3#G$cT%P3n<hOL3*!f)dVDvv#$7GG%yjSu+OlyC~$<&uUE8Ok>=fdU|
zAxUA+8DDke>*v*~^v`Wv7<a;uKR|h*;KU;?EwK~$uPM*mzfn;C*bcE6xvWdJT1rZv
z%$H4b5VX|qxvHnk$t?Zx(ao8s9mONgKCqZ@*m0GhWZDu1_5Q=}1hS5`2)_t;Sr{-O
z{KP{c^JNcCUFqPk_cZrYXSMt(qgN;#v(w7#yyl`Ar~UjKH3cUgXE*EL+_C2m^ArJ}
zr>D%E4(Fu4H`v7$`RnHFH6N-B9vyr5WiNZ?#wmNBX6T<d7^&IxLAb?d`eO#x*7k!b
zOp9l(x_W5iya%kF)sB2+!KO-wbu8YpJ`tBYCL1$z#fcBXd;DKMa)@JWkIg%y7`j3#
zi}6f^7SFNAipi(l7$(g+#u}v$x`_ZZrFTAAUqyVqzCc{u+{4y&iRbjSe`u!af0V5b
z{(m55lKBs((8qSlk{7hPVBOK*hS2V4;7Xl;PjjDsHn&r|y8r)V6??8tWgAofvOW91
zMJ~C7f1cFEeK86}6E@}W&6kl4SoY_m+qcDmOFFO0?{h4wTB#ql^z6f@%bzh^NIt=#
z#J7B7;#(%OCA)WK-nbkSy*%h=clzZ?l|J!X{_WVDbw6JtcJ}e4qBqHxfBEKkMLK!y
znmt2rdvR<4*X8H=d~33^u1vnjd|R-}<d&?*^X}=(Hwfw%b`~yKzJxj8|FU|+C%1Mz
z)vJ}4?ki*M?Z2>B15z9my_3{3X|cTV;qLO3-}T+A3mYuu^%bN$m@d`*Xq(&F=`zvD
zEZ~KXV*O{EV}8FUrtNLJ@OsuS7TKGJ%ir%ked$fevc9=TYLnfB(v2US`8{>{y&obx
z8+mFn6nd4k9%|ik{`CC)CZ)Y=yjK{f8wacFDBqs_*n0n~XGLop%EIbDaou0CBjawm
zhHcc*q;DcdzdNr-KJ-m}6@CBXkJEd1RDF<GrLzBwy7nB2$GboNXNz%G{hny1KgaXa
zypm0uYZXmaIOU&q)|vZ<>*UL&Ca;WUM6ch^#4TrJu4epjvwWmovXG^FTakaaz@>D3
zHCFvVhii+@&&<p4`2O#2;`8%o;wQINpO&ff+1n(?o6W#?t?nnIH1F3P?>iNbw9fh<
zvefwNQOzAbqPwf*AYERcSVvbEmw)fR_$}!8!`Y>I=JfIx4vdnwuN!_o`}{|ZZ|D1k
z!uKaf=f6CAV!HbUfo6RPsievm>vl{0t5=pVNf6j>{>32wR#DGI+aK<?e?{~rY`eNF
zd4Ar$yVb=vcCBXEu=&+xeTEk$*Vi^)b1crt&3kcY=OM{RrP+p`rUv%BxqM`M_oGc=
zhyV26|LEJ*J>y$>Ve#sS`03usrL(^Fe3#ds?d)eTdu_=2n<vDToSimA>PF;V&8$4V
zw@1coPMbvEYL|D%R8^KwPwIGg=Ir(Ohwq<dEeQ)UXDeUBdZ4dz@{O7mY}@zz5~>wG
zo5FnT$2prne-x}#R4*<(|F)#>Q|tCQe~<nPnSK6SiMc@2$@^a~OGvp-xBL2Y{^mcw
zC&(Gq>AvP#6$lC*7u`msqIZ?$Mx}pVi;77;p0s0Qs&<sx(O<g~q*e=xrz@H6o#M7P
z?~zf%g$yaPzGnfq`1h@wTmAggyC-X9-bT!vH&5=y{Z)5=maMF7d6Hu;A$7@MUDVOX
zt#fqV$@~tRJ6G5HI@5(cnR~hJ_gu}oH~U{s>#UDQWQ<=s{}iYU`B3OR=S|daV}?%M
zbsQy+zsy(UjVfu8U&gdSQ@4M$k{)->tQ#s5If`%2Tm5VKwhc08XFd~%U+{UEQQ@cS
zcP{Ve=2coHQ2O-ioic?rZwy|+F3TvgyJPr@^O2UM25XZ}SkAQM!_x)5J07;&mOFEn
zNnu~C|GM0Oxr>==w`E^v6wbTDpeDUja`qp_&3aeQm1Wtb3+~=*n~>r0R%)tNUD`^&
zpxfKh<5|KPcVD`z{`ZDN-0X9KYGUcTZ_b+6x!E_8<IAqQ%kOPBoTDb29P!&(?cUz(
z%KyDvkCeahp5LdN5wkpJt3&vO70cj``=oBDRv1|kn)7V)?K>h*vV)5uUBAiertdu_
zKcjETBg@ytU$;M~4A!)XaC+PEpg5{)QgEG=P{za0V)I{`O02AZEby+xH~p8tHG{|R
zhuU$!A8O}Geq+&n9kSFZ{K7R(h*j?Y{|E|N9`Bp0{y`@-|D4s!nkLrBNSTlFhiA_{
z=h*)J4c{Y8?kwx?Vsk5Gm&>-lep6O&5|rnA{zzrl*QCdb&KJC5<M&(Ms%G2%q5d%2
zq>y*DM`r&2+i<_@PHL=U;}MopR#8afVCR9#Z)%$plHV1c?oV0sXe(!YlKrgeJ=UIQ
zS4ceQY1_1}U+(-z#)}18v{zje$xw{`d(>!CpKsPYr!@P&N~<^Z&#Cw=%whCXe(kf<
zI~J{6Z+T_zC+(ZZS+6F23a-6t=j^OKKg^eD`qMLy?xruB^t|zcRqu=cuRim<y?ige
zU*;3PrRAi(m#>SO?2?|oT6+cLWd9SEeK|Y&`KBdt9Q(4{fBGhd6pJ*k=2@J*%=YX5
z<>=gB{_NLouU#kZ?wgc;|L3${cYhazE>>Ce=XLG>hufEkPu=StzD57Od`HmDt>@#u
zUHB}2YVNF~b>a!@JTL2Q{WE>TG*034Nz=+X_#aqoXn*qYf8x@O!e{?}{<qWH`_Jv?
ziR>S4oII(?bZT8<_%p}<{%Mb*OF3DtK7DQ6$GLf>&ey5~GhEtNXN6U$$VX3J%*Aib
zBp!5j^VBVG9v#@26kWPBf6;#ab+?_}Ug<NatOEOJ%Hnb+*C#Sh?D>C}=RcGF@A9g`
zY|^gBR)^Mi9IBaQ@Z!Yn!)5jJFE;J9ykQ%7<IiiU!$J~wHj2sro4|SCQ2+5w7X&<~
zn78K~5V$*Y-`iJBf>ovYXO{e&eQ@@vC)1Bz-dJ9}>gMI2yY3u$^Z8z7BWrKf=Pw7`
z-kd%0v3*m`H*ry6NjayS-F7OvQs>?FRD55TdP+L`wBHqhlXv?|IC|bhE^XT%y)>Qm
z^Ro~oo!`}A;d|#^l-={>m-=dbtsDK5XGJt<eZ6J0<8+x_%$8=|&Hqn-zv@;Ldst&m
zW%SIC=X4nIDi^R%O#5(SvxoP`s|By`$;i0;+|^;#eM0}s)vWi=?uu+#@4sWe_M0`^
zp9}u2@Ymz#J(hf4QF*Bd4{ztXwDi0D`_?V+{P=K}yQ_=Kk~in2*I$1fD(Q00@BBPl
zPd~r3Q&TkSA8YNn@#Sl=PkeEcVgL`X<i9ry9xn?&asQ9*djIIsJFjBjOZqNvSsZL@
zadTsm-TSj~v59lj?!I3V{adX?>U7`!<fC`GUP~suym9RWzx`6dxi_Y5TRW+P-}Z!i
zz8|+wg#3+o@MIWQe=R?^eeLw6Pu)J79NU_DVr^^(H@D{WxbH@vXMSAsmEA7*a@?17
zTl&_ArG(zuxKVZSyxpxHo8QYAo9U_l-0b@1NOa-<?IntOHZ1Aqs^&?3KU(-(;@kg8
zKOSz?+^1=LO1J*U%-1)+yvxoxC_H1yz3-7i<^8*6R;lj)^ra==a7JFj8SlAQg(`YH
zC#!8tJ1Zr_#|~Q31j!K{P7hiSJuXQo|K*ly7@Zs8e{IW)qNH*wO<h*+rvYoy)XS`I
z<fNOGw_Hv<{_<0TNtEBdwZ}at-erv7;I*CnFeb+04PT6$-`@Ymy$}7^{g!j{yRBJh
zcBbKYq|3j5$}_Y=m2IvplJ-;#aB)#_xa;Wpq$zDn1n1T{ZWoNAH_SZUF3Y`bbH=|-
z5xIZfJ?s&4I<FNmbK|Wu=@wisUdHSa{cNb7yQM`)P;!;Dhl`6#-~V7YRRKW|2;=}w
zut5qM5ML0dz{LF>%Q^%m+BdQ<ug}~6SfY1^Z(7gu*V-EWlbI`LE=iTzv8qz%nfJD5
zdY{VPww+Gu-0<?ZtsrFV;<E(9cHN40{`wO(Pon}0Jx%tl;tW?;75bKbCiR_W%FF4O
zJ?>3-dV1bPTaP^|&I{U`Vpr<SHJ!Wlrd*!XHxt2aUb}dq4byYt%Hel@<j>3gC_R7w
zH^!9}2L(>*I{y+3*PXHG&)%Fk<7QD!>63o<Pb|t$`uY6DK@$#cyW`IjpLTt-<>dU=
zmw#=E<I)A*>;CUnPUnvAoBeF{$BXT|Jj)k!baec=7oV-9RL|sHVJoHfYk#_tOVH{(
z-9v|!S4^MavVQlx^o;eZYv)&%f4Vs5tHrsWm(O~o_}IR>y0Yq}-GQ^Anbv;{<#v8o
zyITO6PgANl^}pGFPiW7)uKsy9roGMRU}PviRrYCa{iVnq{uOT1Jua;Ct?@B+$_=}l
zr=+A*bnA+2h|51EwYu%hD~>xl#woUY-<i>*9jvS*bDDGCuA-a#0R<ln@+QyJ3SFIK
z*mN_=@?G>gg#(W*9Lf>}^UOom$+_-$%aYGoHAm*eegD8hPPJfzsjI`3&y@Y??*7&!
z^JZCr(qSKwr%wwP-0&^%;JBJK=ZB5|_o{OiFRDt-J0ab)m<KVgJ9ED5Kb>X}=e9I?
zQ^dP(Q+xg<gdJ^dYvZX|A>XUi$5-<x>3COuMgBJ{p1_SJPwJPXR38vH-F%*JY1z&@
zX&j~#cK&r&t20=CNN@MO&m41wvRLZQ-`_VWQ~r#SbVql$lC7<D!?(Zgmy|WRrtfW-
zn|b@@rZT<k%XeqYDqAM(9l2-Wos1R7RNhtp347$^n_giH>Y&%#m9mRYw4bjyV}Zko
z&jC|5PhB7P+vG#x;@Pu>;@)LW*<UG_Y(K+wbDVR<v!~&e4^Oi1)H}s;57ZWS*>zW3
zuG_f&;oh6CJKkHhguC%QT{ltRXiIpq!jm2Cv#bA_`}|3eJ8ShV?Lx!s&adUJu|F;v
zefsm)C6;w|#oy-06zQa&&;Rc|Qv1F4$<Dy-j2r%ZKL2y=iL(=HU+P3#wQqg7uWs2p
z-tNdf8|`ANzAnn0WB;Yn%;@LSN|B0lZ>n4%lNnDgJ-B&dk^HocIS0*ss@`zM$$Sr7
zAi5!Ve|=%}>J^(xWvgeH=UN_nA@^r-ZtWU3gWFk`H~O33e|`@%!SbPX_1%~IdL%_;
zDwZ7G_Qpr&w`JZmCubG&`~O+C-@P%bt@pM;!HjwX(eqKCzD@tZH23j_`lkYG#Nroh
zb~!Skg>jGN6ZxW`t#RJ(r0tA8*d>>|-e>dkl)}3F+tcE{+j`z#_HWDU$b+kLf1iH=
zcJDcMXlM($Uvqo5{Ecq)j}oaf!LjQf+%SL3m%8-lGMgLL)63QKpJuq9{PN|?0jt$z
z&oA#S<aKh&Wq32=d;C5_-esRn!xN0--KO8(9q@O?yQRyHWM>4f&sh>s?2+-{`JzWb
z3Ns#T(72`QR=l7pSn02N+Ij6IFK)ap+|B&nxXu0gA+a9^yt?JeHzpnaox*JW{YkNg
z#pf;S-S3}*rBoxu4c}w`e*E)EH936^^MaC>Y@2SY73@44ka)Fu=f0D7AK(7^<QCi2
z!r$)SbFQyrR=cx<(Z}qL!`EHBok^lM54^f5zp2X4IZw7oI9%P-=AFo%qxZNLH_uw_
zI;lnRf)Z!KmRP^@yWJQjOyAb08k`pL<B!Xpyz-;0+iq>?<9;;zOVB3cqLZ8F{`;!I
z(gn>sQhQWItQVGjp1spH*LZ%u2IF}*wSD!Hi{0-yMC=Up{kLvz-zig1j}`T={<fJt
zXbfcjnQ;d+0+X*5@8_ACw)tMqzO|+cm*2TjduZvorV_DpHS<MZ=0$y%JZ0ROwy*Zz
z-21%EsXa5CY7Sl#5OTRNGp4_zqvJ&70{wZePi*XRbWHZ9TIl|Lf4TCz-m|ayX0reP
z$xYa6`Fe@)3GtNF#P1EqUmlUU*|AW+=h->Ev!@>2{gwYC<KZ^rdEy@ej%m%S$~Dq1
zi#v9peg4w&)e&pNgjLfk&u-ip*qFC?c`-N>g4R*KG>ms!@6OIrryFZKcm4Ggx$U#k
z^NqO8p2=<h0~_^^-BA8j#O{Zx_?bkLo&U_@D*iCua{I|va`N1X<$wNsZWB0cSZyTx
zd3X1gsh?}-t87zNR64T%R_2-X2uH)8EBA#RJ8|JudI{XhOaqo=t?ye5+sz+MbiB@9
z7yth&OWmDPj)qyBz09Y>>&>RU4=(<zTcuaDef^W0<@=V_eYd-^twHmM*RQVoOo``Z
z$}G+AUi{v&{K3trh=p6v25-^7aMbuj3X6?}cg_B4F8-pDITse4@hM`Cb*%qCu`=Vj
z+O#U$gq+i+ou=nrRGpi4GDIw+*{5nvcSpyGl`rQy2>ukVu$Zv1RaEoeGu`uUb-tPY
zTh%T`^z54uwD=(}&$}IS`RhMyWjZ(e2b1G>JE!g&;F0Z!eIFig)G=`pdU{q-+Q6lJ
zS^n-_3_CavZZ6mReErCU#i2TBHYsT-3<A>wq_zFZZr3JPSJ~b^dG5W-j{U9c4Yubj
zx%9V{U2&7uv<I8=R;~BD=iWWR=GqkLizVNB`e#lRKD0qSpF`)iQ8tr(+uzw~g{GQk
z?;dd%e$G^Ix>>fiSiWuUzpry-#jDFxWr}qbcd@#<xI9_H6SqEmvq<+G(D5jf=FB-W
zMKd_0he6%G{*z7o(m4z27aMj4p8u+DWKet9ahhLU?~^Z=4rnb)+y41YF7MC#T|f7j
zSC_V)on5+ra^Bn70<ZGl+}d9&xV~lv)4e&#*LHWu$!2YQbT&M4PHZ8=jM?vGJtJQ2
z_KAPEX2YJs`Sr!er}{11XK~bgDRV+c-pL35t_Sa`)0+RoIcD*@=AC<eHR5|E>kl1F
z;<>0Rd1b$JxLe4ofXTAU3vJV1U064Dk3r!zPW#<2@2T%F+qWw3_~aQTnL^pu)*O6T
zARx5pf{qesNOzGyh2sYc*&B2B8Dws|wPBY3v}rLGZ~S89_}#jBXFu&P=`2aqFKW2h
zbT)0}hBL`q1hO_aWb4oW>71~+PWk0^9iH8HS>24yW`A8W*UkTb6=<!c%aikq-z`uo
zI%pKNrX_04@{P+RSD)?qI!ESekM5Nksna!HUUwvV!<?Mgrq|k@+Y|ru(d~FnKHFZ!
z3@<N@m6sag%(tugNCrV_1($RB%_qMcKfLawOJWt5i|gEDw=7fgc^@B>%gE4&w4DSk
z<!$)bUH{GJ{x1GVrOV!6hx)S*Zap<$Lq)@&hUl664i^7->bN)4{`9{eIQ{=Joc-1F
zWLsFE)&ywdG{$R{>6=5``Xx61nu{;!hPZ%d2wk2qrI^hQWf2q<+?kOX!3b{uP24ZI
zm8I*1<+FoNOCDNYKHHO~BmQ?!rJqsI_6^zd=T4Y#DulCNQO}O+4f~DV+PzNMGU3O=
zFYAITX>j?VaW3xo<@~zRW4EvFf9cp36a8E1+`d+w+V{MZ%C9X94KmC&OL?eecSvx3
z>#DH(i?`LSuh)i`pu3_<U-~BHn1oGk?&Y>ibKND)v+;}Moz|!@i-;L#^}U}@nDq9b
zP|A*u^;ZsWtIVG?C+?K)Vz$dhm%QIjdOmaGP3NgMo?ZEKpRI`T^v_)`{oKoM_qeXt
zgO<;8^v&<uw`IQ${rLZxlKByi9gPB6$FC{h;xR0Jb#a4dZJOObw%>cjKOVIzj`$qR
zDKUA4s2q<&_rJxR``#&pW<^DQ{IdQ@mF@GFDKR-SzUlm4U)|(as`b)ML{M<z<BS9e
zLCfP_2X9PLYh>V%`P`P!YLl|3=b5bTpYt#GWvt(^!=O84&jVe}-DaD6XIL4wL}>*t
ze)w~?ol^a0B|HDZMB|qq^^}y9)WPG*%bgZYlKAH!voL*wZ^>+SiIM{?haax^P_ZfM
z?_bGSzc)+eY64Gn9L?gaX-?HYaWIcd-RO)<V4NI#*IkzOmVmWx7Sn{ED)cuu?Q!Cp
zzQd3;e3{#kkFPEHo&-kZEOJY+m=-*D$MM$7tml%#BNa+S9v|U$JDtf_@)$BWyl*Aj
z*{Q39<g#o}enf9fhLyGb-v0l@>&ta43hx)lhVwGd(48@r9ojnoXA_q6D&l+wcX5Wr
zx1Z-fo@tx3skJTQU0sSncpE#jklBea1_qhmo-U3d?U8mb?RCz6+bVYN(*ccDayL#+
z+gcns``xiCTa4#MoO;BYn#hwZ^xd{jyKrIZ+Jc?8!;QW*M~d?2Wt=$B{_CQ~q3flW
ziW9WY@A)IhU}*lg`{DI#wl6Ei-^ZT4Ec|Sm?uTVYKc6mVes2Hg=E5U$lw5CZ-eyqz
z$0S}m$r8R?OG+kU+WuPA%XTwd%>=~7U8}0h7~cNwU*aAla`fivil@z$jbEqN|FJsF
zn0&^*BJGFia=y*b#(h!U%*?9a!84lfr~aCyW8m`Xyy5qY2cF4$9*vpXSG#NDm$^H)
z$eYY!dC#{m!m9mCPW*lE?`=nQ%uJSeudm=+e&#c;!{eJLo=n&N44T0)cl!6NvS;o$
z<+D}13$L4g<+R$={{CL|^rbgf`p8;${yx^DaxQJbkvfHfFw5Pa3xd8cVboIDrF!1x
z@}x&=w(c=X)sv9=bpGPs71@Rlikz1QtE_L{J?--)m-}Yr_C?a+ZS4;>zE6F}EgUg%
z+m!P7Q;!=B%L{j&ZCITkyuRLmn`h&n&z*`U9HtM?M4k_Ryf5MB^ZAz*$}D)Z%%0CY
zuKeTEQ_Urho-V7FUl{uK)D9arR?wu?N{6{EE>D^`n|iHEKK*@h_TllbyR~X|r5|6P
zkmGq;dsQH$wK=Knhf@+}b2D$rJkd3`w#mz2YxQnF@NwHatq<(GU4CuLc8f{!e)nQd
z#Vdx?EeTtrR^)JV>a5CH+j2DPXxip^e;8liI$kbZyS8jD+y7?m!?m}ib1#0WN-_w1
zkR3W_yP)`kG8sE5>y1%0y0fQsNK7(lFmM0k>}N1*W7xK$EpH7@WF1c0n6z1osjH*I
z!!?UjWFmK)p-toM*=(PGwh5J;i9a_lY~z(}4C`X*BByxzTz}^x;~SP&uJEA+)Iz<{
zJyp#1UDDCE?<d*H-kp3NP_S(M>8JO0?y;`ac0Yc`$KRhnXKPlu-CgPH2Qqk;x}LwC
zQONfEt>xi~zL<-YLJclopYq_|hR!+lCaP*WU1gmYWmepq;!(eB^K;Rll2@BA@v1_1
z=qxlYPHm6d@KF21vQ0(BX4_5sA9*~>&pC2hwl`~Tf%f{}GbXy`Zq1YxKVtrJX-fK&
zN$IB%qq3{F!v|&a&iQ}u_#0X6C)&Dl_6^Q!It=+*)t~wUte+KC-7{rs*t4ZnXQoke
zZ$;^D$P(#}m_JU7q%Io2i+;9b`lh;P=Sy<$b<ff5Hgqdj)>Han@;&K@$fMU<Ssa{7
z8Yi~TU0mcVB=jo&gXxB_t+LzCL{2z&Xy*IPN_Tx<+c{UNT|Be>{QkuTpn2GIpZ6?X
zCtAfCcdYu{yHDO$J7VuMakD=mPM7(@ldQY<9zEv9x~n%W|J<R%Nng1(ZQ3yZz1$SY
zs+tdt^)A8AI;N|mET?SjS{^t1OxN7$h94fLzr3p@w)&_`XPy|O9-ZjzIJL@m$GHd9
zHh({S<uQ|bb<s%i5zn^(pSo4+&!?wkJUtL}{$7e$&KcXhx5|>MD{>D_e^Wkp{_E9&
z`gR{aFZ;6hcx~MdgZuJK*#S3$yl$QPxi4eJjlLF+{|Y8*!ppP7c`v1Wgq9BCrxqXk
zu-kv$sx8u&_Uc^Nva3yE^}{XwWymed6*Dhbe7O1Q{IO59S?lU=h=}jJ9scQ!_5Gmz
z)$;!pOj>pR)@V&uuRnAvw%zY_h1%qoBG<zX9dx>Xk>}slGlIM?#Gd|6J^ttEo5CsY
zFTGTFx;&M^1hlv%jsKj`{pa%zHOt7VZ_zNGe8(X?;J>3SOBDO&Z#Nk?JbzknV5MUc
zwEjOQ-rv4xO~|$7Q-0NbU$!ajub<4iVD`ycA3aOHu`gZc`PE>4(p}xnY9Erb)XJ83
zGfTd%s9%)N^gUYcjmHkbo<vTbcM)bQ&K&AL{^`ey!aYB4NF1v(;C3!a|F!M2FIW7W
z-E*JtMtk`CKfFDE+rvM1xl1;tyGK2^W%ho>jY}_dPVk&be*s!HVlRFDUak2?i377w
z)i2joJpPcezj=zP@yy9_T`ME=^RnbvB{s7BX4+8xJ}GYQAIbRoPh$G}KC9mRmwaa4
zN5lT!e=}@Dlny<JF8}(aQYX@E`t$n|$MdRJzH=576y&^cz(DEK^TM}JHyb|gm)?`|
z_4?$9zovJL{zOI{YnhYs_>{|0Hph2YV*<CIdM}rC@3`=c=l^dC+?}-hclqN}9g8PB
zO6+`e?6*WvDrnsDEq{(>3iH`r_dU0N^X2$juPP=Z<tH+MH%3yt^i(C&+{)LLuX^>h
zbbkMPys;*~<LSGL%^XqYbH9uI75ky{w%xzt75k3TE1P3vZ#Go_%a=I)Oh4gc+tHrt
zw8DhF-)h?~><q2_zwvC$)xUXWxgUI)-h19ZaqGpjJD)!&cwP5@I$7T6W7_*Af!`0h
zL>)5tl6&-$<mv;nPc8ac@cFNQL1@K>cdj5G8&vF#<JDC!ng-g^QB-6Uyv!%i@SN#+
zJG<&Xyx;DAVK}^gZS|>!n``cS{4(V}oLAm>vF23gEn7eKvgPle+HbzvcD3fuEIoI@
z_PK6OhjX4U-Q9Bf<$;GKjBYw+($j?HuC(qw!RW?qHhbwIqyCFdE7t0)vwG`l;dL;`
zQe<`H_Dh{UvrIZov#&iUu;Ae240Hs|!MlPc!5mNB-O|)sv2(ZQ+WWEFmmKN6n4xNN
z<=5`lQo3PIy36Cv=kThYsrt9>YR<-_ZGY}Dp1t98cKJpgooyF_w<SOuaVO%>g?^AW
zPS;zivc-(^Yg6RcuUbt}{3?oPzU;m&<)bDV08QI^YCCUF+pK6S>!F^?oY3)f-?>ip
z_t#z)1U&h_v!GtlZ_$-spz^dx%EW3#M~pyL+{I(&QECl}lcN8ZvKMX>_UGPcUn<zY
z-(u$6e~jsy>VC8=m#gm4zm~Fk0;sLK-x=&9gZB877sBHYy)0?GSv2WjU=6&T&0NY2
zn!f>U4ejm}?0)*is&v*_sc)c$|C2ZW9tnXP+k&m2x)jv@{<6Y)MaQ318$EHo{^v86
zy6H?4F8x|EaeKCTjF@-J+dFmjPd7f_SUi9FjVe?Bk7sAJGh5DlB46_0Sj$>B6~l+y
zCa-^M;gA+)$G2TNcth`rD8n0P((C2)mJ6RZEH#Qg^GnSi(ph-&Xk+<Xai1A`cP4G=
ze0MNd`Bt*7_&%<(rJG8wm0U}!-jR9ZvdayvxwAh!*GM>hOMa$Z{nr0GOgGLmnDaLA
z-SYRFl<xK_YE3)wW^M}eN7?r&2YP+`ZXP@QE?(^OeaZXG&}NiG($3^34(qzslpK5h
z@Kp5q8Rq*|?3k!HjeWkXSMJ$vMQBB~(SVheS>W<Ny|!;2N8Ws7IOgzr8f0A7ZB@M%
zN5g_*BZd`oeZBXk{#sPTxFzU9XY|#Woi(5TSg$$iy*TcFQo`+B4Rf!~-*d>^-sgF1
zkns609#1lq<KG>-`KSDtmVw&G2f^EeZ)ZL5I2JXpHeGH0K1<Ha%7-&V<Z9fWC-!+f
zdOCfwfs~^CSL?X{6`TT#!a%WMsr^gZ=?TBV?6W)W=3TmcIkd-q!LQV-8yVad3nuSe
zKIvl2)NP05A5W~#x4QnnsO4(b*~k8NUfXvB*NacvS!S615j>xq`})`PTN!`*cXu#(
zO|L(+s>gZX(Q~#sKXObN=DvI9m%BQD-eIrOD5E9GwI7}t)ZUVMlU^g|*5js}pt7aP
z^sYhug^9}U9^T%E6AXU5)`F}|SpM?<|9|FTtEXO%ub<m1ZN5q&{p9Pi`2{;w_n3V&
zyvngP^880Z-7B6Q>Sxjx7j-;*vU+Dxq;WMjLvNJ2P}IN87j|v&{j&AHZp73!?h|+S
zoV%Wt>9eRX=PK9kRkKS(8RqRR_B^j`x%sok@yA#HesX_vfB*dH@pUtsnwaG3ek3+E
zH;0M{g60P%UcSCD*?pQ`tk+bn(2C!0w@2<OS=seT-J<kE;ll!pzMu1jr=N~YoO!wU
z*>T--=M&cM*UrjWyXrXW$rU@a84B`mxXinJa^02q8vbiHPOmIGqhN6GYEnjr{)=3%
z)0gdE#VcrQcZ<i@OkA@@=U;vGcRA70xgQGu7#~=mu~12~>!G1X(B=bIt{C0E!lqy8
z?G+tlps%U<z*08*ZPBZbVpAJDg0kWqd=GOk(8ycX(G<08^`lu`8Bz+;0w=gE3~cHj
zJl^#(II4r={q^|bpN8qrjEn2i?aq8R%D-n*`{km03lqbN=`(~m{&?QqT^=44HOohB
za@e|<l}$}dpd*eGvV2zZxt$Moa{IY8n&(=riRyvSXz?R2&Ri9G{%N}5tT%t>@2uXw
z=))ubaQO|3qpa`!$X@c8IagJ7s#gE&7{7I!F0Y=~Jlp-XwD^q72MXrq*|O&~u7CXU
zS?lFhx3WEzrq4j90=u}htc%&X=*5eSpTB+yaWO2o_h`Dq0x6p>7o5Gky@RLg#ZH+p
zVL|HYX)7Q7mweRms%-weSwHNh{>S{z-d-YkH+Y?1|BIXZcX&*S<M=Uq+1)h8DJtGK
zr)E#@4KwGvzQxFF&%1=rTX)v+=IyHL4dh+N_tV70q~Oz9$NP1^*V_GjA{-eRscC39
zk%@z00gDi0fNj*aoRv?Wq|EeDTe@V4hLVy~YHBJ=u=Daw?Z5i27`LqozV-e7k{2&7
z-i_MyxoOuAmX*7+Creq*zW3_Qrc*&`->yzszyDtp=wR`Cdv=13TK)I;w=fq&L%yF2
zLx_BJb+xLBO2(NPhE_#SI-H!GLPA1Z3``ew&g1>GQY$jDT2y$m>$TXnY>(?<F2Vev
zVPRQ!ca?(9dbo4<?$l}1)b#cBn;01ygg8R%7hhZ<svY)X?e=>?X=%$qr@P;Y5o21(
z$vbCv#-#)2uCM$4>CK1b&+dkvsCmsEy+nK0+QWtIQ>IVPJ~>JC*0$W)pk?_>mo8ly
zySt2qnSnu7V8>?A5uvTz;#%hB;l<C+T)eqCopUMMma^Q!w?(Hur9A!Dx$WwPz{}aw
z&zeUrx{|bSmqhQu$g{Id!i=4-&e^}|)9kbCrmNq)GPn@7R_02>_E&uxnwo(T5i>4b
zy7cAG=kwXm&&|!cx5x4z1A_x6OO~61L&LT8@#{BjG6J2T`{Lr_mLm1YPg{-iu9sf_
z{_1q>@_!!=n`Ty)YGghwOno~2PC@;@pFOLm%Upe8=WhETa_fVlWBif39=_Pbl$sJU
z)%o~uS<9dI7+YIgXPM{E16@7QDXgCL_*n0z&B?rB3=A5L4!qKKH9N}Q+*sJgE3IW?
z6I1>Do$3_p%M7Ji&$Hg1+8MuopK0*oJ*%c={?yFu<a;vZ-$HkjZ3Vlx-8eIoyLEqF
z>8W!Tik_P)9~D|k_f}n4U4GTrvN()&zm#XxtogxPC+<=4=dYW&D(~jao1im1&ds-9
z|NVY_Jm|RkHEY*$vN14tDHN33{CqOm%hz{mcX#(Rz1USPEi6tud9T;c+`e(&&&=1K
z`hRat`CVBm_iphwU)d+7H~jhI?#`(_+>}4%yY+GhXXnGd@A8$-PYmtul<Btqa=-uM
zY00;~>$9qq=ao#Kx<rlNPR(CZ<#b|g{i+B1xw4Lx|6Ud6Jw5PKMppIg^UV97etCI$
zmUa0$E-o(6wKSlUvRjzIk^dWXkZtYnZ!3$RpS!iAaPgwWi&HmlusQMGSX=w^@$Bnd
zpSbgF9!;P8OUuY_Q&mv&{EXnz#d~MB^}bW|T(!m{%;%7z=zp!e;=;bVJ-ZI>R^1=b
zYJT(L;?U`JRmT#Rp6W`;{qZd1YLnhHr?*k!i*r2n&i^{S`smfaOuf2KZNEi&2Aw)p
zG562Xtd%pGE>~|YI?sCY!J2TN{(BqZAFj3GzjJ4ybNiQ%$K_8aZPc){i+g;mS6PIC
zp{{-5rcIN0S2sw7goG?fJlvM~{M_76pFUk$;yF1bJ3Cr`^ND-={Su}5UQJvW{MBo{
znfaZ1P2<qbiyr0u+TN`8$Jc+i`NgSo-hcIbaVvZK>|=XZrrXXsIj7twzU9b^o`1z%
z##`R4H)sC6a;0A8`BRl%JEHciOMhzTV60}dc-jA}mp*M(S9Ih2T|bHETu^GC%F5#t
zK1%&~;%u(->C@+zAxCxkg10RRTg!9m+`1_XOUrvd%P#qLRyyuKSN{7-53RHR!rr$Y
z^=j*z_3dB8&MS{a)5_l7S~_Efgr$|0mZ4$b-Cd=RAvtr!+;uTKmF(^9*G6u3%gf8-
z;^CRHcdxDLhH2WGr-C1*Zca&^Qad?qQ~n3(W=$Q{Q@?ixq;;p{WUNo~eY-mK1*__Y
zQ*FG`pkv8xe>`aR^75Lrd$+Z^7z0DzL|#D#)AQG^g{=--8}$0x+OH?o=V!dTvy+RD
zZ`$O^!Vj+`mGPZTeO}Pb9({9VW%sN2nU^BYi(9W;vEo9PsP>lh^K!QTemwT_@d@b`
z)BRDv1kTy^%?B6s+y9Hu)6)YTx(+H;7A#QMxog+8`TAd;h892TJ6gXq<#t_BPQXI0
zZ1vJtH=eas7Jr?&^v9x0{^{P=uAQ>q`ux?}rC;+7|J=06sP@;(<)Awk_Uzp&DlNVG
zSg-VFMn;AQW&7KWWJGrSzP2uQwPEtH6Sr<ft>6D|msQD&0R6b~gD;zZ{fpU{le6U0
zr=po>(@cvhUHnV)4zCKmZuoremA)&+MrKA{W@eiLcjn00UfC@76qJHKd?-jwOFMP`
zynna2{<L1+|HoNCh3%cgpqN?c-2Up*)6-Jsd2_11yl~WLSkf)7uVrEqvUBIoS(e4i
zK<9*LFfar#HST)&zinq~YU<0Q;_*|qZ#Q4KAnx>;Ga(TXGrGFFMI|Lyo}8>M%*DVk
z#bH67W&OWBK5CQy{XAd4s{a2!(1i&+Y-R%Pe~X`-5Y&y@;=$JJn0;-{#SMvvIaom@
zNPxMhh{%hd&*xvix3~K0+Gz7$dHXmwW$&d6{&{$MUhERpzOuwq7<4?;-m0%kBH)7W
zZdg>5R(yQ?+W7r(a&mH@W7efivs?^Te17!!G3a>rox65vX=-|cww^5n7tgQQyN@n9
zaKJ$~W=DXYo*pPYEMBa9aKX8{dy5w@o;qjFn&WcSKGECrLRW{a<p39L0sleAHdK8+
zYkqZ2q_OXOyI4Ivy_P1cr>CZBcXf4LdU&|~>h}Ej+#4Gl`S|#tDSLlXQqq(6@Aadj
zqkZREh04jvwKQ2R+_3|68O5m+CoY`U-@oSR>FLJoVj#aWc>el*|NlBcWw#YuwuA_)
z`2=jwyUW4)earm%f03V`pAYx*>$_L+n70;mencBLs9-zrl~2}c2_rL`hPiq8V)y>A
z{dIpiSidhRetu3<N5{w2)s>5vH#8wZ0pjlh<C!yO8qM^vDt$F&&)&UHA3r|4UVX>%
zCCis<8yN+Ob-NlG8h-ltQ4t)j40Yadar4-CrI!5t{e5-%`FWqdf1lpl%X@IaHE&<v
ztD90!r{w3a-?78u>&wgS0SpWaSeSN*s;a71y<9pS)T-%|wboKla40J)V`+}kHZ%;}
zxN&1qVIk<S`7bAw`vt&Bgkk&QMT?w*m-$?LKEHk)yIh6C!Gj00Zf(&#xFBp}gbwJO
zcTkn)H`glj_O{%t%gcBn73>H1+7AcWtKRSZ{^ZG%B@Z4XnB8<+n`kzBYsJT;+V6ME
ztA4#)E-Eg5++GXpuoZmX)Ag=`%FXoj<(Ze4oxFQ@Za_hJ?7o^vZZVw;Cnu{vef)Uw
zix(ND85bH7AX@aIx8+Ri@9)>r(gNKyQCV3j$n~!4&z~C51s0K!k+aNlXE`}Lr>3U5
z8h|neW5{-9cD{@Y3mi|KKR<o_`t|cFpGi77_U<Zp*yQBw92yX?V8R4}y&sQBLn@^M
zb4z}G$(-q<HrKM)t(l$wT8E%A2dn9$Bb~xs9UY)Unm;`~{q_3(e^ViWyMlA)&Ye;g
z1q=A(Y*s9vSLKy_yl?8(t)>Sb^jQ`>XqcuGxu}g-8q_9ub$z`&B*FtG|NHyCKKsdu
ziJ(Ri=++j^;AI;O*xz-$y}kYQY5o0EHgDd%E$^<DzrVi|xasvEPeWh-dJCuUmYSbM
zTeGkGH8wUD6%;5)gmj-e<+V0?yPv10=cf-J0)m1j-P)Gx4e|P&-2DB2x22t*w^mFy
zO2fh;LN|JwipU?$r%#`P&bR;e{yyjc_pHy)&K8xFs7QdaEkjnfYqyxDf`Y@hZ{N7M
zxPo?<z2#u7_we-WR8mqZd31!6SJo;d``Vg`bLYxJB0b=C&aEw;pk$t!x>U`7-WqrL
z+9?4A(Og_yB4T38K0iMXzAfZXD|ciG2Pj81+|n^M4Fw%al9;&gSfA|bdwZ*ex$->2
z!lpSnJ5QZHeR<^Ow9IE`W<q0R?}H<q!bUTFetkSHZ<=$%fLGF}CE&wszqwXRL01`V
z*%IPCO=sf#`OwfU&_8+Z+_df6xA)0dg6=4KF*Q7H;?$|42OqGp^UGa%KEGaX@x>Ki
zUtd3c`}S-|B<>Iw6&1}q(jj<jV{*HVt?k!a+3QbUyEZMM;H>%mn#G~3!%m(&>Dk%Y
z`RdkIZAcsz=o`;I`|0y%(5*yIpFLZ4;DE!;+qZ>zM5LuxA35T(bLUP?O-;|dygY~}
z9AX0^BUefqr@dI*Z#QZ4=FQ9e=j*Aft2eD&uBD~5CFiEm(q+q5WnW+C<>PbZ@Av!u
zkQlsT`tadHP(j$-+}zaE1UlQ_X=S>Xw>PLn|MI1TSH>a$bP3txW4(~JUc;_etJm+F
zw0`~iHEY(02nc{~LJ{<`4UC9bv0;P3T<dbb_4|IUTC`|UN^-I@)Q-}VCr?UQm8|IM
z?tc3I{rb4QRf1l!<sTopg0j^AU)T44IcNR;MKiyh0;B|bH97fs-^tszqwW8Gne5gl
zv$BzyUC2vz>coi)KYS<v<=$>F-AhM0g*m~MN5iidhg!Ko<y+mK9lyT4U43z}`{wBh
zvc*3>B)Yn~o;-EREBpGos}mHRA(c(TuQ$`<>n=`Ebe=M4(xP>-yR#0pa2iSVvIMig
zxwki3JA7Tp<z>EGQ%{S1eQ~iF5`hgNVPR_V@$pBG9a~lV`x~eb09Q8OH~7vr)3mdT
z%e}qr>bJMIukNd@hU6%Q|A`wTL_|cEbaZfR&AS_QdAa{|P;GOk`TgTAQSB*{CMo&*
z`{&->6?*R6IY`C9kf*1j;$ob3CSY^gSrHkTHD<ZD1Wws5TC>I{Ffj1far=J@|NZ@)
z{rA_`UGMk(hEz}tf3|^A&-wHIZi@qhm-}5k%x~|bvGvYU@9CgxXpGa(ow$BIJig{5
zYq2#WsIWe;_V=&X>rHEZ6il5qE$h#ZkFW0SwO)87Xy?wIT)ezjk9LcjCLiNDJJ&j#
zt+^3WCLVZeS@UB98;`_-Idf!srA)of+kTf>c*Q9F{JhYJh#7jZyOywWi@o@IJzgCW
zfghfGwVh14baJwKs?p3ZkGl0gX$R{TzPVxO>gMLPO6t(>L)`k%ir~lRPoF-SRD4Ky
zdwaY8x^?S{ii(t`>^_rYrfq8*%g4vp)!!ff{@z|lOsfhweAU#{oVb7g{%h-Ey(PA`
z?B2cm(W6IKHYU3tVgR*0KnG&3@L$sbq8T(A!IYN*h-whx08tAd$H<c?HN^p>*CF=n
Y{-@dbB|mq!zXI_+UHx3vIVCg!0O{5<+5i9m

diff --git a/tensorflow/lite/g3doc/images/convert/sample_before.png b/tensorflow/lite/g3doc/images/convert/sample_before.png
index e5317ef295062e79c66430512ef1c45925858ce0..55440d324977f0ff5b795bc80898857918066e96 100644
GIT binary patch
literal 136956
zcmeAS@N?(olHy`uVBq!ia0y~yVCrUIU`yj*VqjqKm#ySyVBj&ybaoDK$t*6&NvvdG
z@aUWxULF$sUHt!^i7hP#!U7WnzX`dDq;PC%U!Wiz)1u;_(DK70pfEaHJ8HW{!IePK
zjkl~AG_@98i#)Qn^iM!{S5SA<_XpqInYMi^uiO85_wzm9-|f7A{_mahd!NrOzsK4$
z>4?_UCKI-<saa~4YB5u0PSt&Mhm}#RVnf5;h@e$m-d<h|4fCGgfBRHafT7@`_nIKh
zf0h5@&$?)7Pi1fsYKX~PxkN(Y8{?MwE=N}|hN&|g@64Jil_1S5U^FxGCc|yc27~z5
zznKM|W&LGga8fh1WJ-9>P*t;W#z}S_Q3jXPNj0VkI~W?a2Wf>fwe>PMsjwz#Cj5|d
z*#GE}kY>UYwgbA~^=*FEKR)yJsYv(d>e^g=28O*Jl`Fjz=l!Ygdt4vA(dlyVVL^ud
z4V}Jh3M?J9uBsnTGW=jD44P?eB-o(o5Hrb}ac=#72jSVv4)32f#Vv)|fZbzXh>Rrv
z0VxI9DP0pecU5@Iv$5i9FmzaU>T2Q2TQNbWrl`FBanf_sr4>`;;?FQx`Lr(6(DgaK
zbDx9Q^}{y<;wH>mc-3>;w3C<JnG9G?%wFoM`O)gk;!jN-6Ky5jCeL9zz`0<;aToP$
z!zsaqnf~!135_;+OV0TRh%Y&oRlKrLs4#?!fxAI#%G5NqFE6H?Qd+#FHgI$5DU-Q%
zSu4F{=dC%MAE-I0q+(UzvJOwS2MhtaOH(y%cgAL^g__q(X_W@83X|5=5#jg0l(oQi
zLIa136AQ1)30F=o#r&6RUblFZ3-KCCJaJ~zk6og!%$#Hp;M5~B<#D-&qH|_`oW+p@
z1!u`AKQ2gLy|;G)yP?Ds$F?AmhjtPo%5q#M7BDhPPT70)(no2D6aNEan}jq1nAkqm
zA9HcDH7paco+Nv^FKzebuo4bg7xfE@B^t6$zc2hNap+&?UvaJWTIIFaYpG$<uYXni
z%z0+~RH?^7+GN+t6BB-4f0rSCtNq*3!!Hx8TKF0|%$Bcx;L5-3?@NO!30arlml*3W
zuX_3R$7P+;n3#OMlW!06KHXKj?!bk^7q(A#WqW!*ZQ<3ax2@N9+?#y<cgK&@f0k%C
z{(0{4DmzQRw8_e8?jos`Rvjx2huD@*v+}t5_vzgqygZ+(SF1X+sdg=B_#^-I$gWvD
zDH>nbeSG(!ZoPGLl$j?lgP`iFlRY;39dx;ZSBe+~&I!7-fBuQ%)eGz=%$%xwsV-{X
z>}Cds8T0>%%)Y?F62jo}Z~2THf))kx0(<_56xXnQ5NBqOR+U=jBfXfhfmN<Yp+z<4
zA%}UzLH0ThCyhl0GMyf-V#getm_(Wny7(N`Idil|=NQ+TgKRm+%4+&;S2!#8JxHJ9
zsIB*y?be09<1Y?*?@{Jxa+VNaIy7m6@*Gb8L+u~bojNp>8a>oDb;u~q4-k?{()!41
zI6uN`jiQ;*)g%2!7$0#Ic3JqxsJ|1CJ!1Sw*K=Y`kjJeFdQTWG&F4{md@`g+XQscO
z!s(NiPn?QGcXr<Kyf(@Ggz=NqO}#Zf`xZF741PAn_NnX7V8IZjslHDGc}(Z7TA;Pk
z_ln-CMk~kICAU`iz2ZH)jCD)Lw?)k_BCL9+T@Zfp@(WWH@7}rVmiAv1e`#GJ{&!;P
zflX<Zo0uXC*Z73lWh-Utt)8@X>el(Cb4q7kJy&%yZS&ci>)t57i6|4>J;AnhZ^Pcc
zy|eEgu{*!-c-_H=1v-}V<|Ozjxt|FvepVD#Tvo(atXFW$a{bpYU*CMG`LgKirms_~
zHh<OnTJ}Qr((VP{9p1}cpYi$(?-RA6rFTx=alNB^hwYtZp7nRx^DWO$aX(x7iK}vR
z#p;UP6~`;hE6;!Z`c?J|^B4D*r@yAZT7T)k<Nqc9WdEAi1n*<FZ?bP&|4{!){$ulx
z>p#zzmXO#X(IS;1y@FRvDnwewbVu%oTLB3-?i-v7$g()R;nap}ADSKtE3cclIdH~B
zi^WBYy%z6vd){N!yYradF~MU=k6#`OJ;vU%UTVLA;0zuUPZLeE$b?OYpG-OF9Nf3E
zJ9Em)D<yp|(=IPw_MK(Qfj|Mp2^{+bas*|>w{YKd3QwG~A;2R3$KDODH&WBG%x>u1
z&6yNA^`_%yv*J^Cj@q1yN!)j$rvInt=k2vQzgoUd{5tb`diZ?q+PKIsTRFB)hzyB%
zvu#Jt$7?|;lBtm?JCAHS_i37F?$bomQ(23yR;_*<Hn+C5w)vOWudpw7FMkiuzI5xb
z)y=A|uW7G0U(?UY$xX@$F<YJ6cdO*qtlMq3#BS+rw@ta7sJ?-F<K)}A+pTlHd!(pw
z32VBnb~)}6`|aAdjo+kx_kPp;_Fg7N_L5AR^_#LCPfrwoDL+#j^X|r*8&6Z7n;gnq
z72JDy{kfR*`sYl~r=C-v6TfrEu9O`$PlKL+x^VT7>A9_UwjTSsY<A;qgYBGov+wBb
z+<f=%-R?WrcW$qmAh%DBRW4NSxs9j4z5mYVR_8zejrhy-OY?W`tMaSe??s=5J_~(Z
zx_p1-|CWE7e_#JvZ*(Bj;ah_$vmeio7jIv${c!J5ZSsPb2Oct~^XEwXICK24`=j8e
z#hDWhE_mT+%yyN(%IZe(hMW(-4qmTe-KSi$|5NU_zYolh^nbkl^t$1ZOpkLDwmPr-
z#r=2kAMqdac@kJ#*w1ie@KgxD5MRM}NBD>Pgz$jC3w;aPonzZiwdF}x>EBR{klACn
zL)S%QlJKWokJ|>n6CWkcZoA7{$}_bskGEFfnOvB}Hsd#*JIXDD_u2mmbxAm~rDK_g
zmXX>frA?}*Otf^fe6PICXn*nN#jF?WB<|19H1yrL=Ejd3svE-}=R8s=yjmE0xSeOd
z*`Ej-dpp%S-#>R*`dQQ#CbC#2{pd(l&g(0YTPYJcz2nG^BdML+C&f%wQ~7JsWvi7L
za&^V;lgyKstIhWiT+-uzbK)l>Q=8JjSG^|NO?;z@cLn~M@#WQ*aLI|1-p03#*i-yY
z+e}WKs(OCa`DZE1)AP^P8Cqq1>ALtfx#sv8+2YhSuWxuJ`}b~^S}svuu;*gM#~*tw
z*UageBWwBoXy+r}N8&pRcV-vwRuP;ub<*)m0WXU(+jSzgg?v-cx5&@DclghPKdC=u
zSr2n*aa%nV{ABuRdF#ejX|8xtRk2*{S79j{W#X?zpNq%`%+l(fa@;G&TW#u`@XA%s
zR^3u#@ry8A*xi}FZJN;1WbfBkowj<_hCN>PGpKmE&XP5&-h4gsrL$UnrcaQcmG#Z!
zPkT=CpIUr+|7<7!i9VV0vg}K*U9rA=Xj$$YKcDk+@7WwKJT^!2t#NW%-u$DpLrq>!
zzj<%-^mG3{clYQ^J-cycrseZ{(+%UVoiUwr`pn*6p%=oX_buB$t^Tf2{)IhPKih?Q
zX#1W2J}=(#|6it)O_%TU{7JiNewX{R*fyO{(Oc>at2Y*XoT_?wT26e{rY$uu_ogaO
zyQ(LyyFTLI^RAEDexZEp?rfN}J!tc$+|Wa-eqHrjExx{P|FVR$Pgi|dbvJwM^>fj`
zv*woEZT-6Nb$RZFn+ElVZ|6U~A9q_~^Tx8tr<a{N`{r!w?DzSe_lq7`|9`(~dsOb+
z{a+88J>9nL?aAByq5Lb?-Amk?9LO8Fc;(9*mTBp)zfZh>{r#s~TMM`UeD~A-0sC40
z{<<?iKc1TR!}e_T-|z2kJ>Qf6w(j7+uBb(2MY-Xzj<M0TVSj%zALchPXEA;Af8~Gw
z+fr|(SM$x1jVb9U&p5a6-QlP0@g^&0S@l1W-+A3;-_K(g&n%wa*K2jH$mZXVlk*qX
z+f^yOIr4DwL3L?=)_GP1xBi#hmE3Q<HNE!yyZ=YOPyef1r@Qre+nm+;ycxL`_Fw)i
z<UXt?vs0q-<8$xA>E~kPDu4c&x<&i?y1t#a%6zNdJ}f=@b@~2je}BC@c4qdP*`nKf
z@<Z=!+qd^+{Ja0R?moL-_QCJT+|A{|@9ynT-}(K=_DlEgZ9l&=zhM8T#=Fh0+U@wI
ztruBG$@kV>`4{o^N9^-E_kHW)e;;{!a-Y4^e~a5wZ(jX3rSjI_oP))upWpr`^C$gt
z_Y(2#@nZXHzOQ<;_37)k`G4-ushIuo+rizP?=9bl*B1Q0@WJrk<FoG`y~~?ALy(z4
zgCQkO?n91r!UR^1D8_pkZ}Zp~6t?_US@4(r=lOkQ^~?-E7;if@Zkg%eB)nlCXTdzq
z53?Io7*;M+VCQ|cw<K(q`3d*aN7EUuy?*!Zxf#Q9#vcbt%J)ZKzxy|l+aZ>HmRd%+
zlf&-jI%@`osN3@&6#q8QxL2t4uGMuC0|NtRfk$L91A~|<2s3&HseE8y;MB+ri71Ki
z^|4CM&(%vz$xlkvtH>>200A5Oih{)C?9>v4q}24xJX@vryZ0+8WTx0Eg`4^s_!c;)
zW@LI)6{QAO`Gq7`WhYyvDB0U7*i=|m<QC+nmLw`v<mTiRTUFR9fz7qbE4Bg&>nkaM
zm6T-LDmj8IREY2mP;kyKN>wn`Gt*5rG%-~$x70H<Gd8ib)KM@pFtpG&u+TR&)-^P<
zGBL0+GE#s7B|8P1qLehNAQv~NT}3Hrwn`Z#B?VUc`sL;2dgaD?`9<mahL)C=`UXb&
zMn<|tDQUXJm3bwJ6}oxF$}kgLQj3#|G7CyF^YauyCMG83mzLNnDM5{`$Sv^og&Uk#
z3=OE{{9OHt!~%UoJp=vRT#&!O9xg6%Ei1vVx;Uh=AXPsowK%`DC^^-&EH$r08D^Qg
zr*8mUH^>v|nYjgEZ&c(Kz%5BhhN~$kEy@AQrzGpA=A`DP=9Lud8|oRt)mG#dz!hV4
zJj`m(f|6vDirfO%3WWPJ3rdo~rWco_=p!l6H!?7=&^0j9HL`?TRFG3zjBHbBaVkg%
ztUV_sO&{a|P+-CAORdO6#21RoKr-NPg()q_$S=t+&dA5%6Sy>7Q-QCql}mndDJVER
zU0`W2B|kYc#R{(4D$UH$)YRP2OxME9(o)yZ!X!=C!o<Q{H_bfBGTAi6$jB@$8RP_z
zX`Xq-C5d^-sUXuTatrh_GgGV#ERxNP4NMGm4Gc}pbWPGy4RtNe3=?(D3`~p+EYr-4
zQVrqy{fjcwGxHL2kX;2bCnYn*D$O$4GSS2|NjK3X(OlOg&A>$0BE`U5*Vr`8(#$B?
z#3<RoKuH0VG^`v0JZ+VX^kCe8oWzo}{Gwc2CEvtk1^-~Es*udw)ZmgtP|O>e8e3Qx
zo0^(h7?~It7@I;>gryc0XXfXD%rZ34GgDGXNw#v!FUn0Uu~o{<O-xVKFUU&=i-L?t
z6RpTCuyQU+O)SYT3dzsUu~h;&O~FXd&;T4-5V=&B#FA87B_jhvOI-s)T?2~{Lo+KQ
zODjVIT?11qkcU9V_~xf%rd487YHVm_gsL>PBrzqiB$1$EQ1Y<~$gD`siEzoxP0fRP
z)WX6*NdX}ji6m!e0dfx_VL0dK<QMtor=;2{87nCu#66vz1B&v~GILUGm0TcWF`z;O
zTtZlRWEPj?7ga)wrQp;;C@UvZ0bD3pB_@N47+a;{#In>BC8%I>Mq*xiYKpCrK1?JN
zOG-00u{1L^u`tp#H%$YjG-C^0%M=p>-Lxe0L~~1%<YY@rbC~|({Irtt#G+Kk^whi(
zTP63*+yZdiC}=>^h$c*7c}6NI0T>us=z=1}EX2Uv%Fxov&_dV1927zNM0o(4bBW6s
zkWhiU0x6lG#63hAny(;rM*yr`P;hnz)fJXtKW65E3JbV5kwO^cBYjX24~ZR6k>u-(
z6mdu;oRv?0av~_czzIDy1sWDkiRoZN@T<fo8=R3}o=1c!Fm=dgI2M(Z=ND!B7x^b;
zr6!l)F$Gg4vN=Ji>6yhPMU|fTyn~_$RDxT1f(nG-(t?7V%+w;N<-w_i;DErY03iqt
zy?or75b|I>&iMtEMVaXtCI01k*sa1)0oLb}nVgzejNKk2IdI$~RrhuZHejJ37dH?S
z+%f^RQtT8E&7ah~6kDYtC40L#wZ|tIm>8`*T^vIy=DfLE8X*!b&G4bTY+FJmqpP5>
zj*FD0X!OSDjRH9bcRWikk9XzKd2{HPo``Dbo^46o9864`Hf%7pzAxY1p%HOF=fl5w
zziq<XW*Wag$M^f4WpUGl(@z@&oX*U&t={IK@Z<jff5%G{JhqzT-TCq4<YZnclLRY)
zV;drM#4Z~uv}kQ*<CVIzqmbFYJ3x`Ewdw4-E1q4x0xWR3oV&Y3p>irGPt_!c+<ULa
zp(vV|miFz@QSN1n9kkpR282B0GD&E2<1qA#kDss5apg+LGG@nBti~lv>ppTQie6mc
z$h^jFv6F#w_$%>#fgr(U%a+Yp7^pa7q4w8qcaD`DYn&IqPi$LvMPE*FiPD)fXOfbW
zXZA6z6gzWa>aU~9Eg>vr&)x1H=80VyZ{y_QI%C$XeU+cx76ur|as(j^F*P-1W^Vre
z{=PA?aNqa$_xa`R{@mJ{?Ze&_;_BwsHgV#_2%R)5fny*iujHzc>g8Jb<m!u4zfTD|
ztztVKv(QPavMtS6mcfa`v8LwF=jZ3|@2jo;{Os(rv$Owyzh8fKRp{a4m!?j3QJQH}
zxhX_T)pOF}hd+M(+H^BVZ~E@6tE)~;Tgy3X*~~sh#gLVEcNQPNcP}nFI$CMs6Z?{q
zl9HZ|js!D<BOEJ3PCfp(@BX~mvrj+&eA-TqEp**`We!Erl;q^#)nRMToH_I7i;sZI
z>aevEyj+HDr=Na%d3pJTV|KN_LbiS3nXypNsV8}3#Ov$p#W`4hMw^(L=H})~@^Bhv
zAL$UBHEY&S`>d?2J~`W22P<YYt-ZscI5i<J@86S?!cX^Ky?RyNzAk3DpKO}5lh4G{
zPpz!1lqP!oT-?{^7ZnwiTFt>5(eB2vlB2%9zO=OT>Hf@F$G*hO+<5bioNbi=3nPPy
zXH`W-$RUU;v<{s;d-l*Fr=RcHc%=-ad6MQVU%tGpjg8^M&l-Jw{bh#}+0rsvCak)#
zA+dRr)Y~Udj+E$hB>s$L{rct0nl)=AdRq-ao>_J1+_`gCuZl9r$jHp-V>RqiIa&Aj
z7vq5w6P0K7wHjI^b2?4+%gK53_xJaieUFac+L~Sd^3u{LPo5~d_Zi5Z)-Nn9Tp75S
z?Q-Ch)(;;(w6wHrvRS-wqankKSFfghJiEZ^z2>*4ZYwTr%()UUX--VId-qF+IvsDV
zt51q%_C1<@L}mG|(ia6Uj(9F()+`lq-FWlOxl*4~pO%>Trh6=@`XhfPE~x5VoloZF
zm7fnjK7RVs64!%PCQeiN+EvnL&Wgy)HkS3+DZb|5{$R^%Tb5<caus;}>UxsZgrM@M
z1O0&kKJF9Q=Q}I62so|cFg9`GP+ZbwKo0l0h%*Nx!-w|stCQC8Gnl9@e5%rYoQ=Wi
z^Tp#wZ+qul(pfa^-JFkEHBD@j5`-BXCfHcLUh9&zAaB)7!}^-CKlyXKvOgJ}&5~)I
zqoSaj``5@*i<jZfJUh0xx=W{DTQ;xi*K5~8+i79>>!%uBHCGo+cl=i@bvdl?(6ls{
zv}tEw{y6+UeCfn>PJAr0*%c=3*)zTJ#fq3OpP6sQ9(%Dns^F@y{STeMsV5J2HI!zw
z2rw`#_lphsWL0|mTui0uJj00_bBmAF&)>22_0%WnEPw5iV+|yvKUcgDHv75s&7E0A
zN?mKRy|^1yuGViic=&6rP-$`;@6qL{-m39Nt4;oBu2}s)>rC2Qljv)#46mH`m^_?o
zqJO-4^>vjxa|xC;*Q}nIZoN}I^(N=h-P2RMc`J^!Eeejdb5-Svsg}%-J1+d^kAs#~
z5LcAW*P;N&qddI=ObiPq&0EI)!#mz)Lgy2O^HcUz%q#wIzvp--149ej?KO(p|2O+D
zU}R|V%r&a#PCQ|lA7QX|qmR^vvsv|jBNs3-=uE!VqA_n$!=vI!Ht**CIa(a~q;tyT
z>BYgD7S2fScG@ql-q~<!*}jn35ox@Umh+#mu5#7c-1cVHtoWuY37iZpZ8?)yF)+9=
zA3o{KYFfh3uuE`)MY=Yt!;LD*uLY@}O&R~Hy*t1s!?0-CYPYlVmrTvSC&IwQz?1d-
zZfw&nna^!&oUElY#7ZqxCo%_wIIa1zr2f)QbC!_HC9AmYceQfmEew|TFg{xULF#mz
z<qx4o0WD@r#?@T;i#8iy&sB)-HGG%v>+R8}7rmoi@oLSgxZ2prwh4<@yE^9`Ydbk}
zlie}l9M$tdte1N(KdVSro$+ezjCBk3mK<nMKJjBq68rh&En&H_m3=L1&k4`{!pzWg
zmX$%lq(n?7=*^5-m)8W|lk8->FSENwsk&mpIz#>6Z^R1cGXC-{NnNj}u5)kc=A!+4
z-_PwnS;z2XIx}zSbe-#$guFCk?KqVfE)*EX7_@K)ZEM{3=i|vMM}CFgHc|ClprPO?
z;Les3Fqid}!}hreD=NF!y;fRr%x|{!T;l`1tV`}~S^8w@Z>hVtCQX=KTI72xvux|y
z%{w}$KNfqVHT(ObKZ;o|w_fqRUmAFN^J2gG=Wf0}$JFrc(A=u6S5wcQSJk<{{^$O+
zp=UNtS~maEynk<JU)r})>8tQ@hxv8YdLNh044j?ysAA(SlinWQ$;R(LmWb(GyK-*f
z)zuS2`kdy?+xBY9u_No-)z|O^&70%5oUtJ|l%3&;WK@-Qx!9i8P4P3ucCFkLueI-<
z<tpCS7dRF!65PsT6CGr@-2NTUG{*4N=NGJ55Y5GK^X=XIcKzv1Qv36GzfLR5x%l}?
z>E|H%dX*ms<R7mNHGSE9{t5rS<V`KVe*G^wyGO*C`9w0G$~>FNrSFw}8CUxqpA}tp
z?ar>)<=gxYs-JJ|pZm-H;WlqBh7*-jw^})HE%2yiX4n!lMRd#Vgc)}dH*a07s`|sn
zZ`PmnEDa0+@}?1U|C*&9UUq){QJK$qccRr*iWU^lQ|XcT@p5j<^)sQnl5_c3EssB6
zl5=<J&kxEB7fN0|KfTe!BUt@+n#}!~C9hZ+V{T4<&cN{D=jB)V=Q_BQqIe&r-8;Ky
zqx$E$Ex9)xca~&4JG3^4L7<+EL2AXE;OG+v@9)zrUw2X?Ve|dk5B77!{srAWDiSB-
zJ@5T&@0BaBu4VMy{p!lI<-W7#Z1||<9c%mik=~2nXCKZojbspEj9FWFnKwLdZPse_
z>e9$0^Gz3Ym;Afy8`^g%&XP-1G2|IfP9?*ViA(ctw>`2hH*#JqDjCGE@`aa(gPJSj
z2?g_KoC~k-DO+Iw{GZ)J?OMlV+h13mxfou&d86KW!2DtDFP~<U*Iz<Uu58}<(ZrGI
z>nE2#5$k^hGGCropY(D6oNj|3|01@|*?cG@>cTd4fnO>N*KhS+d9!Tg^e;OT%2f^p
zTs{`?@tYj$1aZ*}PM@uroc^23%ge4QCqB7)=SK`fz!BR}h6_ScJ8HyM2r{s^vonaq
z-?E83wQH{&Bg3cQ<3}A{Ree#-u`am$|FmGD`X;5!-R7y8=B9H@;}1`8Z`gOwAe({V
z=-Zy%FLO?t$xPcDmV5E}<;d3iQ>M#{GVC<>^L#(=+%BQHR(VOMW~*Jj{ys$AQIXjx
zcrt_D^*Jra<wY2-9KZAaUhvcH){{<oo!eWjSa@<ti|X?$-1nVIx9IHKy^`mYuRU|l
zQ#TG#-(GG9&T@TmEqmVCyNXJ-Ff=?6W^iD*!0@G6S)T97i|?7Mtt;fD#R4r&0x$Ql
zEMQ1^Z{OU#%xJS<&C_>YmGf;PPcLU?*xmSPdCXD%zO(0cee~JAzIN+V-zVqiZ<d}V
zrMphE|K`W1k()pL&r;#%_-0;vHbO-sNcr-$ST+U)sSjsL85vIe@wfRoJwU~EU)it3
zTVId7%A1?rGohRB{Ljg&!#%G^tynqzPv&vXR)@)4wLz&B`TL5^zs;*IYMAptpfz{?
zs?~d5q^B=g@a^yHPhaOLm~0W(xqh`;`n~n)Q(^W@4ShF0?p(h2+?L)y{W`8rA=hkM
z>mD9n%{am2$upVzlh?k{^2n$u=g!nDdhlC%)1D<6Yn+u^_Bo!`R51NyC-vy@Q-@b)
zV^^gu^4#`m<;%-6Jti?S=q_U{%4BF_z9V-}YLac|Vz#vB4YN}B-f7`^a6QyKm9gQ#
zQJzckPc-;F!|OzzUt&15FzlzsN*RV1Th_?S%=S8cK7LYAOr5D#>CZ!JLd2q{3(0i_
z$(?xgnW16X{pWjxg-fF{W^Ha*aZ;@@IZMS$VDoIgtHs-ucw%|X{lzmpG8h&-snq}A
zouqJuH!|&PhNgych=(eJ!wSiw94V%TlJ}mPM^jfXO>0w|`pQ)Qx=Gu%P#5{LeLJQW
zKi#A9=*IF6i>B)i$M;$>9QgfsCTCG!qeI0Tn+xyvR|mInSx)#B%E(|6vn6Y_w_MC(
zo;5pd7)o;mB0g;`YIQtr?3e2~<E?vb{SGz9(~Jx+=IAfHaP$6Mr|+%rFYVvIlItGx
zs-Bn)35=lL#i=Xb&d#2@Lr2+e{|9{zUf%hvp|hkmMMS;Q>pk+&Tr+b{%Wa3vfy`&G
z#co|wVie}{kVjtq!u3ZpuP#_+e%)y8sg<GH-<LYA|L?GFrSg8MkCpB5=Z;#pq)Ok-
zdsnmDZ`Ln)`*ph(njdaffAIOO^s{^7um9ywYMUpuFnd*4yKYw%6T^bU{0CP%)gKhj
zzH;i`i}e{n4{o@wULnKqt9g;9dfcCr=L6Sk%4YkXZJ)3Ahn*qn^8c&GSJfhRF<Eb3
z)2wL2UcFXq@7CS7Hb*klyuEYz`c2>29J4H6Z`r<lQCZrj+po=5ow~HfX>)d#m5IX6
zD3!lac?ruCA1=6gqvierHTT}*YnRWg{Ij!d_uRDZz9-My7rApf9KU!icIW#hd-uKn
z9!~8!E2jGD)puEg4JG#tL>B~TOw)<8S-8$~;oChAPp`Tu!4$7-{zpxf`9j9B+vj`L
z9~8{CxcJdr)l*3GZoq24m{$`$7$)pne08qQr~4v>e;sd`Zt;A4Oh?J|`^DgG#nruT
zeYf<!a)|Y(zdCU0$BMk%b=QtWUB2eJJYD|mvZ(pr?rB$4sO_JfSez`@eRy-eM&LYI
zhq~wQ%(vfP6go+|oGVw++UHvNio=hxHeQ`vET<|`<+}0aA{E<aJ7%^iCrkfwHZ2SN
zpUvX5LgnU6^O;%+(V5x*UokQ{Z)%hK`>1l_=T3pPoM78mM?e4Z7e2JWY$f+6@%R4}
z_SoA^OAV5~bWhWP*`@OEc8|$z6Mi>(oz;5eBbDOIC_4SmSvSYrAY0w{0*5Xnd`^0~
zKiN3`-;bs2^*;N8w?CX1yu9B0N|C$X^v^r)r02%g{q}HbUOI2BhQUOSiTPX$53bS-
zIHJm&=<#ZK(XTb;E2``h9|nf`*3QaLG`VV+dPV1^|NY=p&--_s_SXf^_4DdmdH=<S
zlfnD{33>C|wYzcr?a^qhnizD4ucYhGZP^!B%0FGPx4A7Zzq+HEoq-|VJv_MZe79||
zndtP`ip4KvRhcfl$#`QvLsebzX;GHoOUb`Nzh-~oIl4!)WSi5^vJ0A5)DLTgwW(Z<
z*zK=AGbZ-#HO6~xC%WwSB!HYyU+ioI?kd!(|IYQR*=1#Km}=SAn8q}B(x#O5ebaZl
zb#GJfJil+B`PUec%Z7#wpl(HNd!O(8_p{HHh)j*Tv(^9edP7;3!yEL?N$+qJ*nm1W
zzL!OKi&)S7_`Q9_sS;jMlUT`jl3r8O4jtp^4OkkyIZU^oTRgI5mHxryF{TXx4=qGZ
zY&R!|FI%j^t>U>UV$CwehpA<5%kI^(`ksGvWo?gtlt_#vv*MCPudc2x|N82xRp~1O
zS&pOwGYpe;bajvSNFFXx@HzDS0*9igssM+J(#7-Nokat^yt)=HRJ<H$z+-IU<l&mw
zw(g99Gc%}5dFl{P>`Ya4_2jl=JiP)zflQ4G*1vauogMhJp)}(FNGLL*eccunq0R`M
zYZosjS_!lzwz+Yv)HritYM7T-S7RgN<v;^|n6X<nbVhT+IR?(!aK&P28LDSM&T5t1
zb?Fe#+7?ME#U+ajoWrALESw7J$c2ccz3}2T-Xa8YR@Wh(*v!PXutOjh1kPBfz4pqL
zkTVw=K$<pzG+AhFM^Y#o1-EpLk+Jd47`^%D)2#%a!EF~V6F%Jz(hRcEZ^pvU5Sh}_
zuXE<eEL-f5%4=L=2X@}kq>U9ea%wM6u^E@xnO$a7XbDK{m$U8a?-%cMiHeH)_wOHz
zV@i7Z`8k%sv9WVsUI<uum}hSPg3MVjB?O#aNV7PuSh;fJ&74*z$G|Budecuo{rK_Y
zv17-4+S|P5_V@SCoH?_<pZ{{B0e|#PK_{*BCnqKbtu)cr_Er(<JessKCT_W(tk3zW
zEZ3lJl!=dzm+PPYUwQV~clY<(H-B8PsYST!>0wa7SU!99tn}5Dlm924fByZ|)$W58
z0g;ESARY<*a^C(w2Mg2F{hpJyY~7k_CKr-s;QSn{&y|^t=fwNtx0ZT~Hz~~MWuMC1
zwW!4@ajI763_cfC@Q{v@sp-~6X7))cD_5?xOgO->?6701im7R6c=+_AhwiXIBBG|Y
zma|Eq`)Ka1Es0iQA#NKwcPq@<$f6j+bm`KiMT-}E*VM7@j5+tT$l1xsYugPck4ZT<
zH!YntD{AFD*1ue5F0g`P;qTfD7cMw9v(212Q&dJNR7plghT*{d`}<S6b)S5Ge*Wgo
zo5%ZPvu|ujTse<JRMp!0_tn+mZEbCyo}SQX{A)RD*>v6LXYb$d&%M2ETkdTm*>m%6
z@2%dxdiCpFrLWoe<<6AoOucw&s`hl9$VJPRsZI2-kn#Ka>gvs$Z*we*-Fl^tmSp&V
zQra))`F6EZHWdY7VPO#w5no?joyqrnJ&(NIoom<1K0G+MaN)v#|Nhn3&41aUB_19g
zuB)rNFo1(Ws8dB#v-4<DYFe6+toY##%O#v%__wsTM@L4^oIN{!N5R59%hOehi$8DJ
zu;I)ZA9nWk>H6_TvVKn=J$f{0_U!0_?pd>L?I?Uaqs!eD6!?GTW-SX}7jtuUxW2vp
ze%EfXjW<P?Yx+4mCx3r;_ujpGetv%K{PJm5bG%$)WA`SgGw4mX&bqSVk7l;k+OE&b
zTLrY3U%q_l+{WYS?Y+D7b(m}HsY9nvYwtf={rA^b)q8(={{~!N7dzjs)+<niVMD~4
z%*)FRWd%QP;OiCm6&e^A7#0@x_4W1rH9w17eb0NHon^|+&+qT)Sy)n1^Y@qO<%c0$
zZ*Ol851SRK6E<Bhwx_3OMqfwe4m%U47t`0o?k;<JYU*sW+@kXGbgOBbHkZD>mU?<x
z?#)e0{pZ{H91l9^sUoDStGhB})v{$`qN1sZi3dwu&VZ8Tug>%H?fIq6bPNp(@9(Qs
zQ&aP455D&A_xt^d&TTt(?_M9fd)n!zMzVgcY3b?Z-{0MRc6PSAo12krx7UHNyq18%
zcXxIsr>3Un<kUPpHPvgzJjMg3PHEZL*x1?KyLt2H-QDFr?ZFdE-`)~s5Yve`P@=Gk
z$=Jkcg-2ao-Me@1PEXg5-&<9hy3Q^uJ6qnSVnh7?x^LgUE%%$7WF@0!Tk&B*;9@tI
zmox0^?HEel-1w+}TGA-RqpZyA^7K_K3EJjPFW8NYjoo`BK7M<9`|t1X*Vo0)epwK5
z^zQERc0So#o72ySt_~}-kXd$EGyC7af8AobQ9Fy4YOOb$HhZ>p?XNEvE?iJ{@4Ip1
zhJoz4c+j}miVfA@-$h18OG`+!w6r8t>7BTC&CJH;&(qV>*R5MuSXk)O9$a?m^5yC;
zFE&<xe>Y)*K&an&h6T%(fignE!6v`CR%>H-Z;QF^WC&_(Z_K>Bj8E1|CuYZo`}P0V
zp3%MSdhzO2UM{Xz_xA3dJNNF*&FM4wmh0TvReJi;rA=XLqZAbtzrMPfY&9qT((3T_
z7cN{_?A~AY?vACY>C~46{{*%bDYtw|NlmS+u0DS3*t^@?*YBwjFxIfLs=Bcu@$c{N
z@%w5jU1vqCRp?qIrXRPb^!2r~v(3w2Txfh*P&Ye9FMdzO$Aiu6-{0Q8zN<8Q+2NnL
zpy5(R2Ahfx2X5Vpa&~_F>+9<o^Xglho1H5vek^wHkJ^?qbHaoJB^Ijsii!u%o_+i5
z?CggRA3im-WoDRtHf~43!)>{@+jt~D?f?JRUvIjx?8Wy7K|@qeZ%y2se*W9{@5ep$
z>SL#Bh4#tY-`kk%9vywVTU_5r*3XlRk8j&XQ26*36clXDx;lgJ`T89@ce=W`9654i
z$F5!b>i^GsSy0Ektw=d!Wv9!cz{PI16(17*|NEPkmgdtQ9OvrdQc+dKCux-O@zGHg
zPb1lL@e@5%{{Q)W{^sWN^0&91K6n6fciXbe`uhL%|NoYMes=b0YMaFxXXTJe+qyqD
zjvZ?|{II31Evf3Cw4a|}Vq#+XyE~b8ca;VO30;2pXXo?t^Kak2t*)x->gE=<<O>f2
zi{p=9zvkIg79H!6T<q4XwdKartV2A#0ji=B5-*-TGb?>{<?L+p)hkyT%KGh0Pe>?u
zb!BDP+Nj9r==i-=p!jo1NLa9X_3IN8mA}8ey?pK3vn3Ww51l%trKIGfA#&^1ExWou
zGhP<dH68hRRbW+4QBl#Jy?gs*t;;?=@iaF#Uv_w<1xHgsVd2J2n?8N{a^}pLoSU0e
z9R+;Yzk0VgJ^b+Su>V}E(my{ydBIrL@8zS96?b=+hlhvDTa~O>wrttlxw4l9MLAzz
z;0V<_KhJja=FN*WUOhWIn^)Q_#cB@M3fFG2#Kgqyd3UpJZ_6zzGP?Zm&zz!HrHu{+
zudl6j>y<K;^^=sesrYbpb@=wYySH|izdt|U-dOfre`1^43Za|3%k#U%bZcsAw&mVl
zwr<_I5)09=tE)oK&#`>`;zdSkYU$r!rOOU$3i)1M9k#ZpygYtW%E>lf=^r(Amj$P4
z1_xi3HqQ%L9d`E8B_k!Jrk4eE4ADCUSBVrA7a#ADEPimH@$vEg?b+A+UKVKCH7P`G
z$+);B^YX*P?cFX);o;$-*&qEpJzw74{k@r;zp1(T?c29A`PwB_Jh$cF-{&{i>Tnyc
zaQ9I?Jw2_jW{I*=3%lMQ?H2#`?VHj>4IP~|o|8evUgwUTJMZo)ef{d{YFVq2fR#Jd
z*T?Kk%FN76O)dTT>FF%fY$I8|!&c?*WaQ-PetvptmU~M@P0dL5T=<NIf~y|z$=SSk
z^Twv?%ZmdC92y-0HnnyOzh_<Zh5N&2Yk_ZXjf7^N|MvUm&!1~;r2anqz4%emx%D5u
z+Ws=xe89}}1XH`G`+Vl9f<jxaZvWntzG(lfRS%uNY@Dk7mk9*?)jTp7PSg}$J%2ub
z>HUc;oV*UIc6&D!9ga+Fbe@!-`~QC9r1y8SQ&LLa-LcHh&eqmmJ=;8gTE)-(-fEM5
z=iA-Ae7X4fxw)U8oz;H7c)NGk#EBPwetsSs8@qPx+VgX*kC#|f8aOM5Bzme$y13Z=
z{A~00XJ#5JyY*=F27g!`-Tx&|aBuDLE>3o#73*)me=oH$>G!weQEPu&UHS1(V{+TF
z0^1N4)?$O*tL9FRE%j9>eJ#DwDem9LBmR@x6a7^fOl5XieN-~byA!cKIXqd8k>OSv
z-yGS-U3zPKDlhF`{w*)s_V0zmZ<ua+3hmQ&*uL-I{CV3gR6Nhkx39mv%=hoFuhH9b
zB%43G?5+KM?dZ{`A0HnV6cp5r-j<@pbM;8l#<X*DDjytZEPQ;-cb18#)>I=|!L*DP
ztrg+x<18&KmMmS`+TI==5i#Xnr$^%~HpdH}nAMf{pYheLtISx=!N4LF-yZ4h;<PYe
z%J=34Rk^{FPNy}ys7%rjw!V3!c1n9OFT(}nE7}qJTr@<wK25e)UC5x(8K9!Mp<QGD
zlN|!}{P#Ho85quAtzIW9;5dC>NcZOWyAONbB?pT(q~AVV{OrujD|?w*Q|#`RUtiRn
zdUgLkIoGV~>tZ7!Zrt3QZe9Lv&a`P^Rq9%&9v*Ig`}XbcFE5MV+%T;D8}w5zh|$8z
z%FV@PPxbe8m7kyK$L~uqlQ?sMBh<ms(Xp?uZ{ECl)!*M;y>dn8?qts=Va8$S3!dl4
z*9ZPwVEQggAp6rBI~MOL;hZPAKF+quD||35^{`Hre!bcf#)b=uVe3AaZZ9m@kh$&G
zoyS|2hl_vNB(wANKXcV!Pxr8MQ5#cF8?c`~x?E2EILq%Zo3>@9*)3YXcXD&@|6Ue`
z4%_W#%lG}i)TuX3`1`Tr?d#|5m_F^7Z*-|jC({K3dE0%SPj<<kV*6|!-uL><`w8>^
zaO*CdAXEHrZgstJ+{<r5tK#<lI@&ERZJxJgj``w{^PvHLetpxYU%#`n*nhsAZQY+A
zM%MY3OINS1{_x=7!^7>?mX>dCZf-y50ZO6^J(gbj`1ttwIhLQVtPECm@6(xFd}>pr
zsTKdHK%4mbcMsDWt~3|7%)7hx^Yf5D0-F*a$(`oaRy!L}m{NLqGV}5|Az`tW(+<nq
z*KMeF`)zCdeBZifmo~;8_W6;<dwd7?>@cr6$NVI$liMy&+*uqfwXNu4;^x~rw~8Je
zoOtq_wg0_oOI*Lq6Sb^4bwe{a{q*{^Rx;uFpB9y+EVi7z{JXglBSX&aUe=?%wF?Uw
zB$tb;@MT7C@0hN@dA~l%#s9WZh05u)&EooTd-Cq?nrB<RE^6y4o~S+Yb~PEv$;x8g
zhYugt)z$U>T_W1Hef#&F#m~dmL>Q)=nDFLJj$6_Jp5B0?AHThgc6UF1<j9eZ4i0VY
z)tjsfx9#!Td@hdVW%-M+JI$ZhALBE;tX%Y|PQ9|`XUO9E92N$KC8@I)UYWbXg44tP
za<Ekt^Mq}+?l&*4pI0aHI!k0$TF|ElpN>}DvizQZ$LsRzzwg^_Ze>wnxUtjZ8RG`i
zhJO?0{`$truxFQW*>~0Ir>E+zuU(wrxtUR6|HoOei+^_MWu>Rhum1HQYR7JOV@JWz
zCuf%beiyYRH)!dT^*(}DrLRg}Ut9b5cz^lZTc)<Qd%yT?DSUj)EccerT&vRG-`+||
zNi8j3#C7t{oj=#sM$7OWKXvNWy}i|`@(iFn+!4Io&(gwzN6O^IfktM~9B*Zo;Ryzw
zs`Zje*2b$pKL}pTX2$4Qr)?CySL)}ZU))cdlio_XF&toIV930ew08-cV?pDeyr*$m
z>a)$(-QBZy_wL=*zrW@2Zo0hm)ZtHc{spN;oCou+-o1NwuRDMF_iv%9PBS+hS>O60
zs`A{D)Y-o)btDY#EL@>gRCMNN`P$#JMW^ofZr`}^<I~gAUte9Fo|`KxD;xXhX?jY^
zmlqe64GauaR8k%sXjF}>Q(d@j-M+%d$0|QRD}8h0<Nf;ox{u@PW-Juc`krERbBX6<
zHUD`ve|~)2n0$QN*O#nWoX_WXGWbN5m8(bWQJkE+g(G1zqr-tz<_V?JrBm<oRVpuG
zeDL*a@|=6IS)Uu~mQFb3$snL^x8ISkBzw)uW2$k>(-;}%A7*7Ri%)o&R-JloiMIYF
z4bMx0_bvU8=<qQV++;ZLFpHsK-a<~@*u9@#)N*<xonIFzKhe*FkwHZ$vCBYzVzx<!
z?csUHvn<vvy=iXhp`|@HwEs9;fLS5GR7T{t#eEhs=RQ3>9lk#9?%L?>rLV6|O*omD
zn0R)Usj;CUCnx91@b&lRi+)mfez~G2+Fn8@e3SD_l|V=7Fg6yJA3x`w<8G)b$eTWY
z{{6NE8k`Q9JWF5S-CdrM@nWK~`<=UYPZz~lr)9L9vdzzbukJsurM><3jT=)wu3Y&g
zJ^I-i&dP0Bxff&~%;uYWr#vpcS}4eW-Q9}0VJ$MZN=%+P_wogA_j$roG*?h!L&m+s
ze$UgkFeJ?GExv7V!(8D){+%rw<~~<1o*#F&+AwNU{?&{B=gqN{;_0$({}yxUtzGG!
zyT4EFjWXMu`F-bolh;=+7;j&?T|cg)PD-}snU!eu%eEU?r>DGg<YvD;<2KKpbi3P!
z@7!1<`ZdSXyWKZ4Ui;Fnx9t2mclS3tHg32)FD`a_Cc9{LX7+`(B@eqR%YCe7O;6H)
z_h{$t-S>ACCO<zn_w;oA|GzJZa2-9=%KiJ>+t<&ZZ%;bPr8oWeao4{m1FV9!Pn-B=
z_rW4n>nVNL@9YhJx9sffe9oB9ObWAi&d9(2bJN`2%={`#O{T01TN@R=E++HRlFp?|
zQ$0LR+!y)!nuE>#nPQ8V%js#l>1StMUF6EWj9HUURj^ZK!BSO?Ev^exHTHed65?#s
z75LC~XLZ0yp@|$UO!wZms0p27joM%AqIqLu^6|;){@e2I{xWZ^`SBs~!U9JfvFzi0
zvTkl}ssht~?Cg)-|9VSO$pKe+@ni;ukEbQe_MX4kwe2(i`uExE?szK+d1(aw6zfRt
zzx{1*-KW0&oht2r@6MhrTfL|9{$_V&W@%0ih6x<4LPr0^7O04vWbO>$iTatpz(wWH
zr)S4n*A&iQC0muhM9$&U+TPq_Y&y2WSNHa@F{re!nK}7N;coXhn@2aC%b1r2es6nv
zS9s?CuO)Xkyzxs|Jn6-1c_AUAsxL1R54Y(?ZRt3h_Ww`Ae$b?TK%_!tQ`1Y!`qTcm
zr?E1vO}g~?yXxmUeOqRRlND8Zg6ZexY}~dj@9wUzGmX=|)NBvW=UF<Lx#5wU|LkRE
zetc(j)b4bg*Xvza5-Pj<i(&-#_troLhRAetH@^2@O08}O$$DMiRr<Q4qhrPliSY32
z-{1e=Vf$b@XkcOCCtszBKfpl7uBPJFmQ0;6&bkf%zXe!nUSRmJzd5mAK4xZ)v|;AM
z$hzd~ljOaQud^)(G=F(<ZQx6jzYp8?&ud>B7s$EeN&6*935hutg^QxM=arR}9q*IX
z-r2+r>cu-fjmXPe*W7%#onKx_Nl8XV#_J!;e#<F(acl>&Qm;Bldo0$Qr@X@YyO8Rg
zOA=n!7u;pA{&n@@>hi_S=PP=115=DWUh*te=16*Uq*L8*&J4TSUvF-1_TKiPX6fy=
zJ^`l{AxDoMUAlDX)O$RsKb>Ee{WD(uIr8#>4+|AK|9(2H@BGr{3~QzH%SRRr3`!H2
zKRDdJz!702D>pG{$CW<|OSUpFC}@9VcYfJ&YGKK#E3-eD1xB8@zG_3;&mSNA4vSaD
zC0EDDMW^U*2$J<m&&}PNeSICLu$oTnt}P~~j$4^Ht?=m-&Sziqg`0!p!*8poXV+^b
zw|^F1w3%CdZr$lKHUX*q=bujxySU$e%DI*aj8CrqOLD&U>DUelhAEc<tWM|2F<eM1
zyYA$F_jHj)tY%!znF|~XjgGC)yT5($!(+YDS7X>p{$D&5e)N}79=k%l<ZAg7H5)%y
zESs6gslPSy=AWAZR=mll4}SeGZIPRL;8f?o$jy5F+q|{6E-ZN?byj?P-o2%%hno^t
zUkTW?zh&`S0j=++j%KXR@0kDl#@Aw-qU?ajPW|fpCAUv*7vgjcSRrpwWBrZIVbVOk
zi`Om()fLrqMBaMZb@uL4+0z#<9j~{!rDbNe&3n2YznqOm?~1yM-<@BoC@o~Eh_Zhj
z7SGsr$+qgtiQBh-UtH|IG|&Pxx^JlPo4vGojllj&_DNYY9d9&FH*+sLo3q4qiLHfM
z(Z#vbzCD_IvFl#iuV=Yy<Z@?zU-<f+u53}5{~hnkU74%5i1DxAoOtc$v%Nmg&ve;{
zyu2Tx(0i{w>Bx+WPc>|(m#eHV5c-pMxSx~7e&)Pw0Viz^zTeh8foW|~jWQ^4uVi_;
zWA2LT!z>SycOEEwFoo@+wbdNyO-mj>JQ3I{TD3Rk@2dP+m(T4wwz~O0)9l#0nV%Mw
zne*Suop;av=cPHPx84ccv3=tNPy67?+Vbh9+PhN?1hiCt$Ve$J;i}ugBF4&4K5q}Z
zz>HhPYkNvOA3W{9eNw(U`0k#$CK-!d-#_&{TNRltSXo={UAZ*9<rJvJ5O7o3v?JeW
zg~i9zF1DKOEYs5351gFH{rd;&uiWaJ$zIRdKHMz{RAJa1^?8DEqw<#izIU%`e&tF9
zpH5IX(DzG~7c@Gx=HkrG>zS+@K3~#Mt=hiy^8P5^-L>0U#0398sM}t-o2~54?z+!D
zPkqk)d|dV<{@A@EZS(67Y@01okzbkp@9XV{zu!eoxxa7sQ`xdB`jH^tge+U}CH<fG
zU&aSFeCBIxUiE51@wXg~-9P7iu(SPYl+IT)*RXQUv7PC2&#ld3bT~Sfo42k|Rf2(m
zqwo5}z0VxQ7ps;}?l0rX$iLVXwR6eR!?*9=e&zqtDEZwXw?z3}CE_-Jj_*r<yeamj
zq-e;}#;MW=F9rNQsPXW<^(*fdCr<wbQ?E~-o%6`f%W`K~OvnD?A#X2K`&KZtTEEg(
zzqw8(-!LtsC19q~S1I3H1vx>Dy4qn+{vN)XefRG{rHL$V8umww<>ysi{+e@!Nx715
z!^4Y>sozgN<1YKVT_OK_qv6H7Q<C%LY+tB}cvXGBprpEJ_S+vfj+|W+|9XXfFIP;7
z#yyAccjvj!T4&1A>-J@Rebz;jH!I==Kr;?pnijj%tRFs<{rT}j$GS4pE5`T!mVZh;
z+^KBB{Vpcq4X?j<rNsO6!{q|Pbr%BbQ+?lm?Npc7TRZ>Roh?h3U++9REv=?-ciWA?
z{R%aFpqUgY-(NBx=Fczq9PwbG@%w`{?>;H2ZdszFm2CfF#ecUMoL$zMFHIJvC$P6{
z>took_+L3_{52@DDJ`#d_Rg=VRy<##yWXvcPiW-b`YJD)ExRXkZ^EYKA-B`_E>)E7
zdbgs!;Ny&ovHUu=!d`Ff4Kucct7bl81I??~c|W==tzEO%t>?qnuX80Uv@{Y9M?Rb{
z=DO7W_M_KVS5JSwig`Pim=kCGJf4UrrmtNlss<lof4(i(;_u}|%lk5ue|$N0YQD?w
z#kHH{&wsf!$K~(2m0H{5->x|CSNZ4AzxxX7`)Zf3mA&isa|e_4EAgh%jFt=fix!pn
zeT^$my>{;NWf>obDI$Az{cbug-1g&NWFlvG^f&XnLKYA2mfnAwEWGphweM%6|2JGQ
zx+reH^Y^JY$3s;;51ouvZ!*tIKVS6g>15A$^Q<bqeQ67R(!;rb=lS;A%k?vF>8!Qc
zx^4ewY1ywY!?)+h-@l^mQkif&a`&^S)OSbwe%%KT;H%E`Z`yM`R&wvxRqf8E3=EBd
z=UJEE_qWk!6!?>%z^l5@L(Zcm>BqxQoHP6LC4a_$D_F^<w{7?4q&<Ci#J=UOU_CXJ
z??#oIEC14`lHX;F=k443{KNLyu1bQl<F5$u*xZ(z_pJBFarS=|Ex8weduWB8f4Kkq
zYuT^%EpsmYQ~%Jod~1YCT5#v<Wi3K-_ihC47XYpP_&cdQ(7fU5s;Q|5&D<7G{9K*Y
zY;Yw1(~hH!(bIT0zu5a{<^;b!&TlVMb1$E%afz>1XP5pj&6!!=KXdl%XumtN`6m2)
zFk$D;pY?3E?%P)_({T-Mey}BD3G0M!4<htmw;t#F@io3?=88BUog_(zf2$%sO>x(D
zt9Wa%Dc@)FvVYg(#Y=q-rq9XO`#F2|?C+7K&m2!keO_kkZ}0Q3PQGE!yu-QLzULoa
za{I&*zwfZ;qVMv%3jg~(Iedfv{rvg%6&+U(hP}T%w=(|vhDVv1nG+Yp^eqlj1+U+z
z^St)?!)~3;8?WxmO%{3avX&$A*WE7h+}M&E+i#t@|LFSbtLHAtspt29dR?4yY^vk(
zM&1Q0^Dm@b`4)C{|CtO&RhD&CoBq@UpQwBDc4I-<5$9^%Lye!S0zT-osact&&A9hw
zLZF$|#-{(j<SPtLt-a?T9=huG!RIHQZ)a|P7w0p54VT@Oo#*(MU1w-WdHO_R>aynr
zl8?TAvOjBE^yt>Xy$iB$ZF~M=UfR?bFF&mZwM>4AzPhUWN}K6@{iF+Q_fvG9I?RfV
zXTRjr(KWe*fx&wEF&l<$Y!~D=sWS9hR|jj`9v1#RGiKMlU9a!_dA#_~f5q^5IpLG%
z|5bS=dVTvsF-C@*clyUw`}TiXb+e>6mm%Xq{p-K~_E*JB{<m38E_46H2)nHvUw9Na
z*QeOFpPq7H$7zoHdOq$8A8nW_t@uUzL{zViL*e72)fczlzWM6viT7c97#J8{PB%+#
z@@5hU)wxy{ID2d2(S6f>@9tAMdSkM8(`%iWYdhr_JR)NPR5(6Y`Tqa3=5&#iNcsI`
zPh`~umx+H~@l*Qd{J1?!Qqq^rtC;okv8>Vjce}GME;afp<F<;;7&JXIW6LKAp9#gk
ze(o1}wlvQ3)MD1S)s|m(*KB{pI-8jx#*Cq0ht!1Y5yy+=jGXJfMNHLOJLPiS(b%Y4
z-yOD2?2G#N@nhic@cEVQmp?rYe^<xQ(DTJ*);TdT%VVAM<LxqDL<p?;zxL)g9b0SN
zZ-;g~HRrPT{?~NMEB)%@+Dx-4b7D7bd;0S=yPCDW)E3LeUkB|@@;l$#x=!b_{r^o@
zlvnpH`F~At(e&Sc|A2-W|NIGC_vLoI)|1>_Kc@37F>-3!_Ul{R2cIv0WfoOd|EXb#
zwUpOsUwemRVf^9Mw#KhYdaqutvWmR5{aq4=etcJq*qV8_Li%q_`g7J{))w}I>3QDo
z1;rQ5PH~&J)9>iA_?lbmsvm1f`9@W3zH5CWjFqAAPV2e<GE5EY)?R03I4D{EcK<)$
z7XP;D(jV8Sxb6>`@NMg%ck17_{#?i9y|hinzVL6~o4#W!gMY<Z?=+RHta^JmZ}RWB
z+ddm?{aVidNqlSl_GRdQU+WbSr`I}~vVAYBEiSmWrrT6+{fa}mN#3Xb?($h#`0)P1
zJ2#qs-)LWNZSZ3L=Nm_2Cs>#jeY|%xh5x%<n$-TPo&#I5Z%du~EB@o1js3xpqO!va
z))f~`pV#>+?s>ZEgk4{IBNX-T_y7I4zCM*DcBj0K`vv<Mk7N`>=55KjS#)hpq;Av}
zjoFt2g(iBaOx#>q#VV$`NJVKP3zuGW$dQdAkvZitUP?mD@dub0KIQs+VgEVZteAm8
za>Hut8(01Yai}hAaM$u)?4q>O;Q;fADyhE;s}DFDNnXjw33h+KJzH65qQH;T>*W>4
z`(*v++1%V!`ue?`gtYYSrQXxo`Q@g#h1_4{BxdnbPPp**+Ri!GH{N(xF(p9b*R9V{
ze=Z$gxBc<N501U2j(_ePUpK$v*}=DR&m1~08EZDq6I=hF@#Rc~3t@6I5#2P=i<1=l
zY*Wkwq)PK6{^hEB`7hrm!>3?rc<ANf8Qyl^e%1fJ)cx(nMdkeb_wN02Q{5ikIMplg
z%T~pI!P4c+zvmk18BAFABe{OkM~`_92b38-7$$H%j@zU2!gWqS*aCr1-c@W2M)^5?
zD!<Q8&gZN8_$YPNjuq41KML8m(AM5@xvTtyGo1Gf1Eoy9=5>9Vq?Y?KHTB+>=NH>E
z8CF_XNA}(k<y<uVwR_L4P1&azPS4x7Z=dwKJ!}g$WlG)1VOZcA$!W57;lY0bwHw0s
z&vbNroqOgi{}B~thPnSTT-Vn{H#Mc*j;hSxtEOcpu{8hQ^)u<(RbTH(Y{}oY_UgJ9
zDGgl)5q#j~0;?LM@3ZZ%<~hIObcf(lL7u9rUo&1$m|7}bFXxl<#rpgF&C6vQE}r;$
z+N^Y5*xp%k2hvwqChS|DeCMX+?oGGu&bD~qyw~zw@#-^C;ruD*O*tL~o;Z5iKX}`j
zn?H^vK4NoEY`=E;o14w1Yf4p7ueM6sEerfHPyF8DO~=^ubJKa6qkma^`@j9&%#&*S
z1>?$`el(WfmFtxhTmE~^@ukO|WmnZ{{cnB>YR5ReNPqJF+q&IS)t?IjF3ik``k3=K
zFqK_zn`igVZ#Mh2{g22Wl8Y=eU^{WjgEdNvZN<$UoB~x*@wXH?c;9A~%KqOgQ<cAb
zd)}+;gqwvP3=ONM<-UCp&L4Lyu1az9s-?^f3j(j7?fa~u`g%Llm5fKSb=Ur-99VR&
z`>;siN~Na>JLRT5-6hKcTF<j^zDJ+UY_r70{@cUK`riJt-7sa@IeG40W(Eew+kx}^
z!`XiPZ2w+r&fcK5ciFa%*vGqzPM`W3o}Tx9g^UJ2!?SOFVq#*Yt6o1{?o$>gt+&!q
zfQ3_vW#W{HYyb52asGatu{VCQr1YCBg~4m<{&UCOX*|cW$%$tgsKN=d%q;(IvNh(P
z->2rfHJSQ(Yd(Cnjk==n?ETcwAI(-NK7R0HLSS?3>hSfE%hS$UoxNlJMUdsJ-9#1!
zhR`=vJj?QBf2HJSpI~cv)6#B!W1g|i?*51k2M;gPVz@AK<@HS0gPs{IS6|K6dKb5M
zHV;F|t>3R*xY;i+TYGm$<MMaj3=A#j`@3&PNqn}9VMzGgowH2O!b(r^jX-8ve0;x0
z<$Q))ms);b*E*1330n9v_3-MfSGL!p-n;t?tiH19{BAj|ru@m<Kiob1<Bh!7*N5L;
zde185JzW~DemO69=EmTUkB&`cKlR=+<ksceg+Wo<E~j7Y6ya(xd1tb6=k*(6r}S<v
z;y<5x^IY88+uQRVCgkPS8EM_hf4QHbAY<|Usp6TZm^!t}7}=lY%ib+Zb!b@s{$=U$
zX=Xwns`V^Qbjqe3KcxFQ-S4kSe*bjU+GiOC8t-Io6>C0qkKb?!WmJpn%j}XR^VSz_
z`oK3eWbvY~fA8z9*}v-E_r24wc&)%HhEo+=LNh0JwA}BxzDVQ5>(#n~T&nTA!@Bvt
z7lp3jkPq=!6xqC$#in7+<b8Q759{@9y2#GprE+FTMSR<bDMu@23kAx#MgQ$uGuc>u
zv)h`taoxI;HXr4je(jDyl-;d$9dlKk7Pc%8E!ZOJ(c)ye{z6g4eygUwNgYpY82KHP
zCNX%NO4|^#vTxg=eRnt(8ab-2-TX$v<C{fA<(JE<cg5N}p3C#Rjy6whd#vzS_xbJg
zdG|}SV|9O?P+fcSd3brxy-hjq&Yf|d*yzi8X~hB8_D8%W_up17c^Tz=A>&lj!WO>5
zw-;7vyZ7H^th%wQxcSHQ=WQL4oZPIz%XWG-FOPXxeWkuA;L7_I>lYX8IlJkT&AnxF
zPS)-Jp0j0Y<rK{;jcbmac2|z-pBMb)UhAe}!PzP|^XKjPapvzWAHAte+w(SC-G8b-
zNj)GfWnKgOhs$4sPceeWD;%YGZ`l}KQ1`ySp=$rlZ1MQ``1RY&GM-3no%iCR#=6$%
zC-+vr&R%lO{(AkvM(elk7SRXKN#`(Fy<PbI*OAHm)35m#=iRz=`Sp=Uw`@At6jp7$
z|FPoV<GG95WjkfIC%=%{tlK5Jnr)`$x2QK68y+T|5Bs}0JXVzH$+Z6;&7StO{p=HF
zT<#ZlTe{-zi8D)&|DSf{+OO9-!JY0GK&y$SPFz(c8x^?syv-bgDbqQ=oPK;!{tVCh
z)UA2m`}XoMtbf;{dh&*L_`mER_vYzS%~e%Y&#@exJEi1S$oT_&9J#WL4KBZC=Doc(
zh2=pu!(2{=B1s0-=?pJ!oGFVpU6}jn#L;f`Z&#R3hQ`cUbtcrX_}i?9XG%mwSwOwk
z$cMA}q_0;zIaq5s!8HHf235v?Q~G8dZ(I1tU8w2qs@=PHZ;CZ~n!D(Wx4S_Z?{?j0
z(_>rW3~mOmdb|1Fr3t(Wj0`LP{9b6G!eH|9^Zv)Cv(CsEum8L5`EzHhH@mYpp4ng1
zzg;sv$v@Y7ll;Y1zt*RVd4^ng_o8GLyO5{a2Ogg_v2Wg<&H8g`=Q?@Y%1>c?4ZDAv
zUHkXAAynquQ#X#OvfkzSH*cP0m)i0D_4VzSySc-qw&ZCkY~jtmc<HKk|GnuAD&OAB
z`YQM`N``65;hYsyj_beq&->}sjq+Q!F6Oc@Fy!zxg!mmY=T2`lF?Uz$xo!G<!^G1I
zCO?eM{MmTjP&9FmZ>5v^oH@HQFVFm6_go_GbKPDOz5Kg+Cg0~69JywY=(3(qx%}?W
zsmj^roBuej|JTK8Y~r-y%?4i?HS>aq`+GSBw(jOBn{oB96vL0w==_<NY%?AvNEgp;
z{C+0AOzHcZ-3(pU?S1zdqe^~%`+9P()%N`Ri)QZpxrDiVlKh;VU!1m`%)ftlyWzp~
zb;0NRWx74rC!W7)u;%QK=JP)n=}r!R$8A-1ZQ=bzM|9h@4phu8Qqg^OsBTu(woLzv
zuU(%_<Y;yB{KuhH$}2mqw0Bh$lkIK2#bVo^-M9Ph!@=OddHFK;8#V?7LC)4gdn9*P
zR$cA&(f)C}8`S*Vw8Zky1G7o)3=OZZH0U2*$+Av)n%##V@&8^$#C<NWILprPLtvBJ
z@A(JV=Uq#hIKx+U-E5&#A3xmtx}$%SPWOT8$y5Gs>g@OyxGwkVzWFXHlN{`I8(pr5
z{fJ8a&!w$>@u%*3X8kv|_bPwvyI_9W>N<17ZfkY^`0}N<|NK4uE6UE&sQN&c)b#gN
zd+!JDD7HH={pvnPy|c?08+@v_Bt2nTRJ*&ycXqM*n$OaA-54Sj>pt_TTs3Omz^tF!
z_h!~j5AGD6eRqr+C8ZQYmhIEacb#-UPrSzdoLbfYh0?QLP2^{7dvizK{QO3%lFQnQ
zLLBco%wGTW1Itt;+tn*q*8ceL@Z@B5-&rP!20WE99vRK=oU@`nt6B2>i=W{CJkpI}
zTipN8QSy_tF0e3|Ei`+5U_#b>`E9@V74Ks!G*^4F>D%Y(cJX)ngR8H8yq4}(roE->
z7)R0nBev55zqfH7I9}BF_<$zwqVkf24_}|{uKBrX)~qN^(LV4-%%iWL)>nBN|J>SK
z%|CZ%{*9_HU-nJ&X<t(0<Kx4`#AH?eZcpv+GPc7zstaDuy>Vf|f(5U?zrXZ}IX$nQ
ziGiW{UiPoKGiB%StrXV_&%7RCyuaA)<JMw32Q%LluR~XdZU21cLC0@(>+tix@<e;L
z9G~uQ`s=ux<>~FP*~{Sdru_jLGG;ju0Rb1@-QDf|QX~Sj-P544aJt~W*cp9JM|owf
zu3WshvHt(R>+9q1Z^@jzZe3oa`-Hzg9^C!cZuTX7vPXs9?@Fcqv$xmrZIEBUn(*k(
z>ZyDFubQeET2(2{;2=2nbFNfmRD{VH?OxxSNLS~_r?sTERGEG{`c+(sznHP1j90nA
z!q4#IlzlU1%y@XXeS7}>f9K{}*Vyf!at<^o6Hw_28{%o1@X|vfJu6E}R5WyVS+2VJ
z@)IXc7}-_7IqA^M$t)edK5n_++*ennze<~i$!&e|i`$T)!EJHkW2yOf|L;jYy^A3r
z<oTa7IUlb-`cb+s+I4Z*UANe$+nw)mWHMCL{*_o>KK+)B89T#`RNHb1-^>61{my@X
zZ*TbexV@>Tr(N<s51RGRVrWtj5*D7>*TlGFTIaqWC+~)*i7s%@PfAj9{B!kyf!_4p
zb$@@Ay}eagQPCr1n)U0;OP#RDD+ebn{`MuVa_7&0=_1$T+Ef^(3QGQBVR-gsZhA@T
z%)N@YFWp_7xTh-l=>~If!Qi{sY0B|!bGIfNJioT@_ND3NVtE(l+S}z@rpKLEZNJC=
z{`tlaYrWsaKHOuwIrod_j9sdU&lIjpPClMBr~2B<JFCR}&;8V|R7}n1+A#C|<uAJ}
zcX_`)s;1sAV;Q6)c7Jd6^&>}?Z1Hc`PFDzd$I8xLU0Hc{uJ!fXw`aeM_{YWS;N>E;
zt-+m%K~<z{-@56H3SR!pS#HJGFLQCSwElNxjpOk*%6)tSB7w_ZE!N0-c&OFAPsXzL
z*OskYzup(0F=NJyhYuG9XfQD|_xAMs+jn5Uo9Bu>d`<FB*VfeCFUoB^_j7Ssh4{ta
z@7Y&p1PD!hG5cxgsViHvue<fjy?y$0Yr;XMNjlx@TW_4|4cK{oZS?kzj*hw3<$Tg+
zI@Z6BcN?tQ@nlBA{AFUB!`GQ?FD$rOZEtOtwDA2K-Pg}<MO4OB&gPwYKJ3e@y@nb0
zbD~$rUOdTwHpFzU<V(Gxrsm4z<9+>dw!hx*|L>(@%k1#=)z$T}yRWT_wN6ie&MmGt
z#bai@`u_j>R6vEu_xCP#J%2y^`}woEJwi68+v_~Tfy-K3PKZvhVN`w@sW$oSOyl%7
zH#V~K%f;-eD7?8O{eYE;lW1Fn&bO~$_tyRW_3!WR*=D(?N-XwwdfmOb{jb{Tr{c%2
z{MNX%=l<U0XU~L{rGjrqeY^SCz&@p@>a^k<_wQ{-WhE~wZ+kyiGxWEKyoL6og?q(5
zrrdwwcrfqj7QWiAudZHP?A|YLU-#|J&H9^1{yIN=_^_Fc_tmYfuP3Q`?_j;wmNZG@
z;hTxVhWbK|*SF-J*E^T?G{Ns$`TKjk@^*JFTqyYd?(VN&sk?eK?b_W|G(GtI^Y*sf
z+xzS5r|ZRv=|ntu`SN6m%l_sNafXMtkKeoya5>;l&D^F{pWhwq^~syOb@lbo%oW#<
zsQh`ec`{@7vpk+;^Zcow>>k3#m9Cz!-tWCIKqqoj%Kd${?fmk2H#enf&OLhb=FfkB
zf5*ke<>lp7e|xhset(_j+!y9kwx9dy-Y+-Ts`S;Ho14{rXN7!AKhDz|&}vliAt5^Y
zwq5P7miG4FKYwzvJYAOev$RfQr%cDM*s5<E!oKNx`B(k7xKVrI>0a||?BQqMR0?>Q
ziHqd7h-6G;2*`^|*w-&_UsY9AWHIOezu)ICT-XqD9<&<f(UDHw=xtjvFQ?tylqx4D
zx3p?kmx{Xj^ZE7vI(vHd)co8ub*iYc;e+Cg7Oh!rywdk}7PlXNymQwsyV_q<UKZ2|
z#xHPQlGee%V6rOWfVBVpxtvOsWicz>e<?1!$nI(U`&X6AsoyLYrfEemMb-T|b(f!E
z!o+zVmv1_~QeV2dPue_h$BrF4cJ7SbSF>}=<>GxClaI&kudBVgt28by?&+50PLkC>
zJ}lh4*_h$Z?(+VXUwA;P;wD_Va;5C;Ez{y>XZ+?`y}iGG{>y^8cPDqw{hjCi{qOE~
zr!J>3EOz`Ye!chG>lX<xZ+yLTqJHn0StW~*x)-?>w+g=Bd6&J{%H(b4!@VyJ7k)K*
zQ)hH}{e$NsTt_c0_1?R8@AtR2w<jEI+9b&}_57VXHfm~a!NHd=Ui`TKe_i&^m0s!!
z3JbEYuPc6j?(XLF{<b!@=7|aZhAmo~)cxmud3pKxwQFU6e|=qQUR|(qfr`?^mbFvO
z!_8F%PgXL?HMpoe<g7TQoAiLYVnK{#)X}9&mh?y%IxP&4v8&k;wKa>!O-Mw<B=Jy-
zX7I8fKYmPc`@W?}*VeZ7&W^&)&dzpz`LNdG-y+)GRw!j(T5|I4?sCQjpylNoH~O@H
zw)U9k@cMO-l!|`-owL*bGoKXZs7TrAaXv9I@!Yv{etv$}*Tp8A*)luqult*Ke_!pd
zFE8WvRxMq&%t*S&i<6(fyr{^?)U>y!C!`e;-XB<)4(_Y{t>!=P&c52;huis$W&NHy
zUH-Bvbafk_?5^zV>r}m`O}S}t>(SM9vA?gajlOp6nxCJadET7_E1743B3nydUQ%>!
z%elL&^zE&!GJKnh>*`zA-r)!p>zA{Ab$NOJ=FOjHnPz*H8YM3I^768Je*XJAckUc+
z=fA!pu=!;{-Gf_Ovz^=dzJB;H!#JI9cC2Unx0)Xx4qm=I`S|0v_xHz}E=%~$1zLhC
z<~!T0x3lx)*|WJ<S6y8lzJ3PZ@}};do|3Y%ZQHi(D}3CRwJe<><MOhze}8|Mx2?Le
zE_U~h9Tv+DCmz^Y{5&Qm#;W{X&C^q&q2A{i9P}bKG=O%3q_%)hn`l?)czC$|`kKhi
zg^!Q<xQ8%aG})VXcbBAb+M632li%Ffn0hDucR42u6F)!y!hjcV-^Q*EgRF23l$_|H
za(TJ`{Y|N-Hzprv<CR)6YwoiRQ>7Q4dhze?@7iBq3X6-Qw`Pg<hhKaBJv8Cv%a@DY
zdVl@-`FWzUd)BozCrd2$E2^n2tN;H`H+oyl)~ry=*WdpCPD)O0=as&;z_D4@x-26*
z+gR3bV#u%4vQ8_eL_|g1+g<+t^Yim-qqo<6e#U$GVTi!?{QG5PWqEgYd^|hb{B(E~
z8-uj;?En9M`|F5JU)`Da@5sHq)wj3j+b14sS?u1ws;G)B2XscnqmLDTe|?>N`suuR
z^GaV`$-J>a@v`95d}*_sAJ6C4GchsM)YOFRN!_hsBe%Z%{XM3}gc}<YXU&?Wy7k($
zR{~tEi{1O}%HQ4j`T4nZ*_#t37WSZ_-q12bL&MtN-`-wWc=*%P)1j%XONyR;etv$j
zd%vHjCnp!zs?gP9mmf}P5Z90U^8Wt*vbVQpo8?{tZCjc%Nl;kWxaP-)Z*Om}kKVp+
zPfx1G^!P6VPO2U^_spC#=gzj=+u(Cg{49lqg&)6p6SF>U?}iN<x+czJJaFcWj+ND|
zBS(&$JgKRr)%CLA-?UBd;}A`pPm|^`f`;c94m|(-`}_O(lP5oZ`ZRQX++NUW9<qTy
zH&lLp)+wx>cV|c8rza=1jHEq16+JfvFZcWU`ntSX&Ws41vZA7(v-i%~Pn|UB(Zh!y
zzr4JBdb+-O{=GRb3+nhmOYQgVs{LKIzwYm@vbVcZP6}O}8R^NInwHilZLVi$_wU8U
z#ig&VBwEc0nlfXCMaBgMh8sJJmvi+==-q=f4{Vc;bhL7dPdmOo06Z*u>fXJ7`~Ux|
z{{IgYoR1$rp2_F#dUt>Q{hK#;y7fv43JQu^Luz(sC#PFmG9SNrQ}gf;>$1Z?LyczU
zoSS2re7w)L;6cNyGuwad2G3~lLN*MgrKR=BSYA4F=FF8VQ(kt2oZgmu`&hsH|M&a<
z`^`36JE!j5f4wG!sBJkfU%ar$y0T)&jvdo<qgValuK)fn_WixRN0UAt?G~SHl6k1a
zq8>cvX)h)zd2@gL|9g9@^<#Eyxa4;$CLk>A+K-Qq<M-Fqet8in8u@!gPijg^kA&f)
zx3{<Nt^S^7HBG70Wzk&g@^kZSt1m6_^f~@%J7|wt*OY0~%u-HF=;`U%Tm602n<tl2
zA75PT-ptOQcX?Uw{Q2>!^ILP@-`m?SZ?CuVl9IBrsj2DNvuC|mIf!*%-Ctj?t-X4Z
z)&xh;St~~uW>!5tHTCuN_52(h8Ta?uUVdnxY+z9E@KCF?S&oE7fx_jDAtrWqcDzz1
zGYk@$E;p`F%Gv3}A=<9cG1EAG*Nz=LG8P2~8W=x6I~)8leS)gDiqOv=KWu7$nb_Fe
zIdi7xWr3DL*P>G|MRHOZ)^j_(02MUv?(Y8n;o)JwxmK#Gs-Q*im7kt;c6WETwYim)
zm@Io9v?xHsz*(FXq<*@Dghb4qijB#~`}FkmK!vx}oS^OH@9&+PYkhv6ZT0a!S<$++
z>THQ^dTgK+RwwExQ2gvnVPWCN4<96~N?yFVxmo*3dbXe6xh0;HLECU<o9Bnk`#Nc6
zljJUmidX@s{mVmFhkbu{cX#dYvgvxUlO|7|$+x`4w*KE9*KRQt&wD$I*)JDM255*(
zU6p<~{~%AVz^~lU)nR*!o_d|0raO1;TxVzJnS9HS$XJ!EaBk-dUhW5~5qIsfy8JMN
z>qhVz32;&s))C8oc&Jr3YKw$X%83p^<+O8iBG=icA5Ho=MKgHOqDAZC_s2CQ7hm*G
z)Gjv!#qz(H*=D&(Nl9#cGB@^Ce+Q-e%TJfu*Z=!--u{0?M8uIJM>trRmK|OhW8f^#
zY;5B6V!ec<WbE#;+=qu+laKf5?v<RN(RJ#?i3``S>-Qe=ob=`6M@iGHkW7)ju6D^?
z5*EuPoL0zq`1x(yuwg}o?BxkX@4nal{beZIp>-%<k7Hp`!{LV~o)#@z?4ZhD-6hJr
zp4(}KOv2`yGZt#A<UG}5ixY1Nuso2qIn_w=a^QkVAR|G+`IKQ>QFzFVh24$USrtS0
z612-*TZ_9yK@MCYqrlO0G$~NzXo<p#mPA=d_&;S1Tdb|}=5#F29zBkQpBfK8Oi16@
z#s2!j(F3oW1hnKg@Wm>EVi#oJ&j#joS}Jce!+1brc%K;AV?sTC$iU3mRut}W;}p~!
z#^{~eDrew^Ox^8thzGaSzB@+`@brRg11SXY5O!;q8-CcO&~l2s2E`?7?}!Sey-0dc
z>c}yb|HdhZBONcBfVQ*;Occ|Pt9f}zm9Je`T6*>B)u*3-wy*tF^5TM`y1M!@XXpC7
zkB^S7T)9%kQ_iMhL-h7MHa0dLvFux0GH>0w<<q{*36ugtqE?5kb#iv*&fQb{d)wT(
zvdSM92sut#;<Qj>3#c9g6_zS;S6gzqITWXwRe#Iz^*!4Z{pIUdL)+=)o72udD%#1q
z-6Zdh#N}c|P-fR!>pR;l$ISY_`y>^19*F}9HY;3sB#o9la8Z5r>Q(Zwo<sw=tKc22
zmm+koeRz1-Ql{^dyQ%5cgU#&5(n7zgd8JHNgsqMG*DvDAD`PR?U`2>9Xwk_kY0ILN
zI@4LlA6r^j7)bML3C^20&qmJw=lf@8XTN^^+E}`u3$(5wsI*tgbXDYLHHHZ)H}}`q
zFFWk0did~RXUlV5!NJ9EZfx{vUkpAC=~Uy#$H!NLL9%dpYvA_0*r($9J7eyxjW%C)
zIM5a3nWqiy?e2bleLia0_x4y`-nc;M@zbZQ40m^x1_uWxTZx4p`S(+gL-EVnIdkUZ
z-rhF#qs*Kbz3ow3v$C(R(>*<zsbTWu;L16&u3A&`?(PaznrI;FQP|?7$nfIDi#3s(
z+g?U=End9%@zbZS0xpvC#nW{hIsR^2?A~9+BVn*)TVdq9kW~R27;dCmJ3qO6X3ZQe
zHvXjK)1Q{8KKk_Zw5(N$MBw$7mkxDGhUV8h&g=hv@;#dA{*j}{j?J1mbLz*+ji5b>
zxhB5bO%3M-t{1!9IAvG$eB<k#-2SzpdKXOIMs0RWt>`~>p!ZCUTh?aKdZ!kFAh54F
z6hlD~L^2ng*;6*m*|X}K5qqc1jT|w~t~32N&6_tn&Hr22*b-^}H&OQL79ru<X@5@f
zJZPEhZ~DAt$yN3LKYB#xCrTf#-&O3v;I(K1v-eIL<Ehj3r3L0?b)KDh^sRa9{bL=|
zPIEq8bx*~ULACDlyx`M4tLObVn9lk2wCl#*K4v+oPTo&g_+5_wKilKebbr6$CxN!A
z*MC?}x<8&i|Hs|%BO;*XK3;7~N$IDas_ffnxM%OL!rO(0S1!pLuD@k)++J@&ouq1d
z_QoqW_s_7}Vs-2DndsG=AysYZZK12ru6N_;XHT;U{?Oa|vi$mF{`)M=R(n>d&g5R=
zZr*)lmrb(s#g(}yt+y}Bu~uMX&dXaPvL)ix{&3wX^JBli+%L0l<3^pF=|;AP-`|(G
zGw<FjQ_cpN$e2AUro0Rd23J14Tm9>&|5s%fj;YdD{yrCB`1;ZEtCCLgUstY%=;d+u
z-gEG7i*%m0;K02m&@oJ`_RSm&1}o-n?_XdRJ%2)a*ZzOE7MR^$^X>XvRw)OGNv5Ws
z>&)J)jqUbr*jF}xzOH|yhR~r2$@ex)N-3Qt=U!6g9rmR5<nkPb392?z?%PhQdB(as
zkh$Ugx($tL3=`6}e%9wpyws2!^)<WDb#L>ln^Ro)90EBvUg>`RN&Mo>bbn6yf9kqF
z@`R+`EIWIHZPo|lE+0eQojV`KOC{HpB{w)QGjL44wS<A8VZ&FWR}W7;%VwH8_v`yF
zQ)Cx1EU-)4Hm{ra=jCE!UQioQwZoa=!Xg<41zG7y3`G+;7*2#G_xy^!EHr8HX|Z?l
zGwPo5*8f<vO1EdJXVSin>t0-H^EtFO>-Vy*EB-RfwZGo(JhHsB_fD_s`y*LfSxfjB
z8jd{)-Bo%uW1ijYs=tNDh3`I;{Sjp&#h{>_dgq0V;99#qb8U<?cGuaj6?^x_c485O
zg3g!gzSpm)9-TNnh1WV`g?i1c4Hn_`7tANB?3VQ3FxAFe)W_Iw_T#yt@p|GXf0TAf
zGiEX@@L9^h@`1sjuIT3S_v<#y+ivYYcgpjCoYG0>Dg^8^S@YjSH6Ob)r^LB5?#u6)
zx4Y`u?LGI;YrUJgF7WgAkHvObB|jz!&*n3k_r_0Mb>gvO69m2{R{Cu9eQf!`^6dWw
zv#wh-TBXd7ol~_t@~rFKC)=5V?3oxiYnU3IeVZMArgx%ADPR4mw7tJ}ZQ@aM5ASZf
z)5F_+k}*$5;p;Npr}z9rHg8pUbna;UwF~)q9t;v|XU<n%y*P5NdUd5<;>o(R|8v(S
zO`B>`x5_1C+k3Bzpb2O#Pm_kK?Ukl~%4b(ZoQ!j=y;CF95ydu7cjb)B^5tx&8&+RU
z7dU<B^og@3$vew0y(#RxF3c)>?6X|l<iG=I`+q;Ew|F}-t@OLw{OLVEE$)B6oNZXZ
zsB`Q^blBNDdo7<HGIrWKZ(eAUpVfs4Q=jd(e0!;KV+Y%`y?M@7whvN%J#A;%J+H4i
zmVx2d!{wUNDk25aw?3RK?_9hr)&8QcNaup3ve92`O7HHua$KI%?(+3MN3q5G_a1tG
z`Mv7p2~#)bWZb#v-m-4zF3;Sx&#pf0KfgpTLN!f#Q+GZ4;baC4rYj}S->ul0T7Gt4
zFXzTDuXlE^g{qe?PTp2O!&W!%o}ocrT<(rv%Qv_%?0oWW?UVxr3|>Y5Esq=zwtR2*
zmg8?#!rI28_a#EvtWw$T1~Ht-dtYZ=vs2{CkJ77l@qJ4jXIb6+etFv5NY5gsloA(*
zTf&06{k}7HeYm{oa1q0=VlIXY3_G81Tj5v525L$Osqd3kOa7d3;+M<@M>Qsc4Nn;-
zxJ}!YE8X?}&Pkr$w5!Go9lg348C0Gti#V*h`dEvhD1XkRd$w~|8?gP_`(h~*Lxv>-
z!}3g7h6meUEDN1~YFEAbYfl-~g^rVE6n_u9b}*QWq3rv+m>~D`YkyYhW-~RkMPJ|3
z7J2mTlD*p_cm4YE*f{8iql%@WKgZr@Tbch_e*exHANGN(#L&X`^&#_c3(sj&ct1_K
zQ^8tl9mXhfp>R=z;`NiqQ(W(#*PiaQ@qNOzsp$+VygOfR-uvh{U;XyiW}zGMXLMT5
zXL-4sg`r_bbQ0%=(h`;B&Y}y}U*9eld6l8zO3wRJ?=E+K{k3b&fx?{Z&-3p}cur7c
zozS(;g)t>aq=6?QhJnGUpTWvi@8s(Uw+)^F&-SnBmh^X#+3gpzXl7b&=Cg~}m)0z;
z2lwo}B>ujvI>ps`a#7wd?P=b7)4Z<kV3<<yy=dVk-kmE~&$47>*b;H`qWiKx@{AL%
ze{@%4cj{MGc%mfLz{DWd&Rg<ld8zhP%_+CfO=V~J=`YCXeD|B_CjGbP|1DsM+_vOw
z!rfnGx1K$Deq4m1AnOG)!|}`oxBi{D^Iqv;4d1T<1^4~_(RUAizG3kD@>KKG;?9t>
z-@|9LFojDnFl_oW?Q*l*i`^!MQJWRN=CFx;=Bso3z{xOc-^O=`dzqq(8`%<GOEc|Z
zJ2CM&!-ky`gr>~A5WfBBj8zB3k1ktvyCP(A@XYS@b-{P^??|j!|NL*D&HaBX{^Wj}
zm>nusY8+$B_h?<G<JJUODaDYYD?2LZYrf}Kom=I$AzO0o%oAPx)0tPAm#>`Fsdn?!
z@@4+%uTEAotV{d;jDcZUY2*a$UEduP*@O0LF=#AFVk+74YYPLzwg&bC%Y!d-EQ<RY
z*&Nxj@YodIHa<5lOV)V1M9$@juhxXm=bLw$ckcx0!q=a#ofer8RolkIa78L=E3bLs
zy*SAi(&wZ&@4Ft#eYjNd=46YkgfhMrk!ER+CK)zQI>p#9tv2kRhWDS4A6%<HH%_|D
z;84KH;M$gXRdn&u+t2q#mo+oghItjfSEyO}SnJY$&xu<;#d_~mJ;yk&SD2e&fnL10
zivB#Nh7YG~oleh~EjDe_BOBMOO~&%tyIBK*6dJs<g!h*uKRKoFezmh6uS0=@;#3BP
zg=r@?X0DjPtayQqp^2B_fKkQ))2mJ2+b_jGE#TSWBCa>}o$LSFCrzx}Ge8HWp0la^
z^yJ@FZQ0c?YBpb86TLOl)>1}n_daItk6|J;K6%&N7H?Y_=KZ#3w&yEesf=y!R;4bT
zyfMk^-RA0@chq*}?&f$i>-ClulWTK&7qzZ0<ITOa!?4@qtY!VQyenR(&qXeemN%6Z
zJK^!@aj#67tmEHhR<>3vKhIO0H$kc_>r7Ji$JYg}l48duM{T?j<61LA#gnO}?O6uD
zqiOE6Jln0a*lsc3n$#JXsor07`^!V=YZo(qSejb7Z@zfl<H)YHJZyYVW?6k&EjYjY
zhv#MP{`;On-iw}mIkEBasW-Q#O##iJSqb0qjsN~TUT~Z9YW^Y>OPw-V>22Sh-*M;s
zGP~l?rGKwQ?NnA-U6}VIYvL`YuX(u@QZ;$DW=pddzD_%t9Wj5&{N?#y&ujjY4Y|Q%
zy1_hcqlL}ywn-mmPvK!`xLv$ypR=r3c=y_GYG02?JHF=Im3Nl)U2bH)zH07H7KRUY
z&TCgEe>*4o=*jdY-r=v@63(6G$=m<ZZSJfIGGh0yb?(|4ye2vPOz+2s_0}f;CG{^{
zM>%HedBTpUYU{INvOB9xuVzb5x}_%L7gRQDVVSe>*>BG#%|3XkB<j~qm)mjEqzV?U
zco@w-Dadxuo-bd&dcS@9%dlygcg_?mrxlYpxVUcFCf};M9JRl&=fH$D3*Glu{QH`y
zpLoLTYOu(iAcn+%*{qpY)TLg$`J=l;??n8b-`xl5ML0q7i{otmKXiY%_x#M)N4j3L
z_y7C0B=y~;rQY7|Rc|vQzF+76`BGGGkMqX&5BxWtkrI;J7}7X1h=bYJ%9NYIVFBZ!
zH;a;fyp?tim>uH3c#7)t`FEF?9oZY~cJBMf%fWLz?uh(4yqIyp<-O|94=YQVWPIq>
zU%Lb}{2wq=U}=DU(7u~a%TI9UYW_HVrTj{H^pllK1w1z<hG^OEJ=9wMW4HJ6ijRMG
zf7-8dQR$NLpPRjkRqnOdoudW6^D&f6dw1oUGUwMd{fE1J&hvoJHfqt*{CN8N6r0z@
z4~#-zD=%v)&1eyDnmVQ8`=XY+KJ0-epoTHY2gM$DR&M#{;+2<Ilc#q%aLTS9F&lm2
z;<AnAYaOo+VKXY4v9kx%f7-t`@o-yy(GlgiV^g1crWWmVQM()vtbKZN+S4aIk|ry3
z&PN2jy1IJ0R``^U)n9gNbe)>$VRH5AOui@TEVpj8wY8<z$AQjp3b6xCc>VqLRa`$V
z#Y!L~>D}Gk=0#6V%$Omuj5#py>DI4oiY)<^%L6nxni8zIuC_`_DYgh0i*T`aw`Q*J
zsK^v>;xJ@3F8TQ3l0u6>5?k7fM-P`eaws0*IdkFYVee)Erw+*(3%lE=vw^f3IE(Y^
z^Ez=TrX{rL$;C;v2soWNz!Ph;$Am+%Wk#dqt_rFe=y=%#)WwImr)v>tpap8Xb~$|<
zoNO%1uxiGVpqDE)+HKEsHIO|aFT`<$d3*k*ty@#A1X{F=K*7Y*>oqSx<IR<&>2m_^
z?szQL<Z2*$ZoYt;ZRXu=mnS+`TbI5%Qlc~UZZkXov0mxr>(<SiI(6!-S+~~3?w&Je
zPT1Ng-Pm1IUS7Cx{}4~C&6x`;T2$54c5QvDvi{PZ!2eTScK*G6*(6JAMbP&B#~+lg
zo?cnmn_xA^OV`}oT<KAE@|oG@`3wyTr=FVIhj2jRPxa-^&FR<5`qo(OeO(?ITX%W$
zBtPBiZo%iv|K)$&!`eNgZ)s5Wy*)b@E=)YAJ8RY~<FqpgT5=&xkoeUyOiWDd(_70E
zlRkUPj3XsFs?G@k_lvGPZMds$yVnggfMhZC`h|tgmNI^z!HCi~Hzppe2=F`zP6;bg
zWbEtq6km_~TAjBx&$QuXgqKkG;;hSp&-L8*zI<{|SzW#TWr5bphlksrr)M-NRDF9B
zX^^=16E`>!E)aUK{b2nOjr(fOej$drXG?Tc`_Ep^cq8~l$@%ihtzVb>&!54!+$ihP
zlFr7)#!vU9q@?2F;?5jv_Tqwgy}otX%lc*KBDN$Y8^8CCGS!*Ew|vV6kFw@CM$292
z%*y=o@9#^snsX}S$%%<Ja{LV7<D^tCKU^>gl6)`BpL{aqFFT)Q!7I1T6Q^JK;xmJ9
z`IW1g=JAKK)~s#*ytn%MjJ~CpK7RP1z|o{~(y^KCVR}{T!T=4ANhzTGWp4G$IWjVG
z?d{UOntwNIDh%9X`(GBE>e$#k-Sha?yj-ieT_v3_BebU9+M0cRb$I_G4S#?CWyy}J
zZHqMi><i_Q;qU$D5)?Ek`nF$C>~6Qer~j^4bwBmSk-4iE-Y<V3uz#zzjcvQJY+n{9
z7njyvHa1;dT?UIX&^*v_uf;W{rl#`p^0s?VeJTwWP0%j?$N@h8e5&!rjT_hAD!E`L
zA9Xrc>+(heHS4EGFaNDo>-|2@KA-RM#uY7d=FHiceB905onOLW!IWjfff^!_k&(A<
z-C7!G!LX<9ua$|3iH63Bg9jH*@oSu7WMmY#r{ZAJMkZ$Ft5>eH@yVV_+L(2H-P%7w
zZJ!d`-1e(0DIMAyt^fEM_o~}GNmgQQo|`7kfAjkC+P#;*t(~n^Y5U;j=JeWMUp8*q
z6t%x@FL=QA&(7lK=X&Ms>z<vN8NAFV(P~=Pnah`pzrVYCN<aFPjEV?X`uTZ#Q%(vk
za}L&>`YW0B@4DdSex)Vv1GoPc^8u|1Sd;az^j}iNn(FIuJ9qB%X+J!p_V>4~85a*7
zKc0SjTkcaUTjqw{yQ?!YUMzHOuln-h<Im6O%MP#H@ayaA<ttat1OrVGu2!doZEbGT
z^<roAc|T2%mHNA?cedI8{Cktsn)$LNqm5*{xdH<=O<%6|;^O=DaaPec<r2G&KKl6h
zczbK>+xz?XL(X4GS`zf~&CSjFF*^iAL|R_Yp31f}X5He&%0iu=wuh_^)78}MJUAgl
z_}apY`ZLXvukBHNJ*zVI$d?$7?k6+8a-FzVv|(Fu)%SO@mmf}<E3O}R<M!=t7tmhW
z&?R4Z7-ICwzrMQaJzekZot>ZCc%^6XEoVyFd=s>t^8NY)Pm4Bo|Km?=bF){Reww|C
z=XUv<Sx;A-eifQ!EZe80#IeZlcPkTjel+{@-{0RG%lbuHRD4*ldGqHLfs2`#nN3Yh
zX7DW!DS3bI@2{_~cbC6^cX@e#fJTbdoUUV+FB|_qr|2nU{DnVJ+g!`9tZZBL_Ivr4
zPO#OMPn0oTb~v!~WPkJhd2@^Yyg0ZGbdH*L=0pz_VRgSfRbOBI`}<p0N9&DF(N7hj
zlRc8gPEJmJa<*BisYgpJMA=;fckS9085zlNVMF5K^z-u!WhZJWiR(lhDB<yNRaIBt
zy(MS%IkW5w+1lM_Q?2F{IUL^5c;H6f<v)t1Ss~fi)<|A{IAzcE_3_*D@6S8_c&<g^
zp(9HkomXOK$T8cUe7vu%t*x1z|Js!+TQ+Y_wpzs1bTkRHHsoM4dsbGKaoU*-&`74>
zN~N5qZuX!{6|$q<SGlEM&U?)%=6Cx0>XK}0v#&Ofty9lUT6%hB-_jOY+o~;<pPwCW
z<GsB-fBv#%YL_2QS$S`7b^PwK+|$!^nb~*(!hT=451PrDe}1_})`J6$KF1d=+Avl6
zuR};k$Z|c~p6K(d^NeS_j5wv*Y4GyoKD)hlqVsNUS;)jQb(+`G;AK8L3mzW2x3^lm
z*n541&OEc+TWh1Y_sLp^Nk;wHKYixRpD!*ho^(pe#CLnn%}riYwbcFR=}cr|1f4SW
z>(?*UP^rfT&WjH<H8<O~Rn7d$bs_&k-^++ohC3sEy!nuz9lZCkBm2)9JJ8`!+w<>#
zdwcu(jT;rGr|B*=pC4{n^yI|EmzS68M{mn{a$=&-@t|c+&d%)oaycm}Ax+)I>MANL
zR<E9Z#lW>|(Y?LZ+l!z3m6v}P(wtB@W8p<}CZ=tE^Y?D}c%b=~>DrlnOYZ~+Y!X*L
z_G+_T%-)i)@bKv`3;rEh8NA%BSIQJL3H|t(p=>u-<;O=y9~^Af)z#gecUMb*161+`
zifk=-cu3Z|jD>||MTP!}6DLjxD!YAobF=vBs?eCco$qaf-`&0+z&Y#Ze#`&SJ9f>y
z&gyeswaK+e>dVb}nj1E5EAHy<PPUqJUwQILHQ!k#m7ktGKHi^PrFZ1a866XoDJq`P
z(YMw8=NZWQz3l4m&%e4V)WyZ6qGCta@^pp*@GTJ)-`~Z4es(r~U(Lx9i+b+gMam2e
zJ7erpW?cWsE$D0O|9NY+fvn$AM}d~Ozt=2<e(qvjY`rBrcx{yFGUZ>Y@9*z_e`V$5
z5G~unM=p1FmCodI7cI1yqZhj?BO`-H(&)&60}B?}&9g6k6|z0=E*r1ZljqOF<Jk7C
zUb*t?ySvc~19+s()?{5>HEr57ANMcMH%ygo^iX+qY3=vZA{jUG%}?sy|E#ro%@g<K
z_vX#7`gG&rwYAa4vZwv$SQa0<eY^V8lau@Q?ejSv#Aj3eE$737gM55^v(5AM-d~<1
zt|HdG+-GLd$48*^<)&X==|A75(x~J`Ky>tN-RNyormq)oP0%h+ojz&OqX$<$#pZ22
z*36e!rDq#D`<ub*-Mii2H~!|ANwAuxbn@w^mBGvXW*9s?F;RKOy!yb!ZoPSTcd1T3
z`R2`=dG__^N-R_#-`t%3?c2AND_05%3O+TlWp<dX=6mSSp&c>nrcMogsT&=)YQxms
zLDgShO<i29y{T+_)!b=<EOoKkv01y+b}f5bwqA;tYv$^1<I4|4rcIb|VP|pr!hnib
zS62S|`r265ukz5tf?cJrk3Ig_#wVMVk`i)Hef7<KwZ9)6Y+mj^e_z5uCb|COB^LFq
zhaZ-_yR$QDYnF2xkD}+M%FoZ1ZmFo$JQ()w)a!3=Z_D1@9iDHwyX|hkvgDO6Dl<F#
zzkhvqd+k}Z|6f-bhx|I!$~|x1JYhAT88($gKYwuqYxc|A%h^;Ebc^YN&V!O`zroph
zXri*aUG=w|^Yd(ZLASijnIs}EZk}^vgI4IOH#avQKX9PoWkH>2+6%8&cXk#t+qPYQ
zzwzdR!1hVHH!rqbU#;iOS72`Pz2emMw|?LDtlJvF!adKW>-*8#>g$COak)FUnMR2B
zJz=SzxxO<%*_<czcY*P93tzsxQ!9@J>)g}#`|){4$HKW4zE?Roxs4tz$<7aExUik~
zepG)$PS#u2m+E{Mx3iv~wInTe+EaO>%mr4J=H+j_kItX}&Hc%vGwJO%IZ{$`)<q2;
zt+-qRH>aPsD|m3=-QC@?va&OImoc3<ckbKAkCorv+}xOa{LGm%Gx(mHySTga%UEoP
z-Cd>~wkF}vHXYugoi!gH9cAa2yR$ui{;P8Kz@Vw?KdjKny}hmWfx!D?a_(L>w#*JG
zJx_|PBwkLO8^gutmhyGeBZ-$2ABA5!^iq-`r7L{CWAC43uiQ?*JYw=HcA3&k+piPZ
z?#&PP=&<X{sa^2B^`r&cl8t;ZdwezP3*XGU<IyiFspcE3yu9q`_O<7)X$u%F^q!S{
z@N~cGoOz55XRqfOJvnl?;?~yC-RI}qPp|m7U(e9+;@Pur@9nLA`0(M3dEPB;ZO7Kd
z?sn^ydiv%~&5H|)%MSlkHSt{^u@O|~YKN_f*q#^L3TkDfZ5G#y*^zsDo1$~upZE3u
zC!c=m<Nl?1#zO76H;bR2pRxVWq1Su1d1jU}RDN(|WXNGY*Py?-WQOVE7t9P7zQ+ap
z+%1_Q^tJT!rcaxBSQ$=e7+g^Oq}DJ&bDI2V+sPa^cXsVq&%KwK;mUPY9xk7(lmm<p
z1iqT?SF3P(n|)F@_Ghe)MJVG7ovi=PmJ?FkFYmd1@7NsWp9;L^n5?Ep6)Sz8%ENWe
z{$0ko#O^28z0&#8#Ae@Kke?uR{?M^|mUedgLe70O4qq3uapT6!ySu)goUE>W$)$5u
z!jTTag9i^5KR+jHoYv!_lxQ{Qk&3$d_Wb+*-rU@rn3x#5s|2((<oUB_VuFH&@9)`y
zPX)PeEPKv>!%RMgsj@p`-n}&ab?NNyY0CPAraa#7U94<=#m>2NqUG))%jX|hB^fT>
zZ~OaW<()l+b!;lfz6EZ6ayjzArRSH``<~xnXHfXXfAIxBgM)kAN#FJ7-v+eL6_~pD
z!NKcMIcImxsgJQN(p<QnwcM#_@m7ntFd0)J6_F1f9#ylfUyD7x*Y-Z|$f6xOzFk~<
zH03YMU}3ob?@z4yk-+lQ`MNiJb1j>u3keFuN^e-yynkEfX2yMWwG4-DPVuSI`TK3R
zqo)$bqPpd&g^AL+Z|1IxwY$G1^Ky`NXd+i@(!W1H8yg#Um%lG7E!ExpIZW#;_!v`p
z^SqeP&(8MCT8FKVvt6eAtI>P99<RJz&5;g4<MeYXCr@p1=xb~|`0?@azkmO}ee=dF
z_tur|`SHt?wS=xM)b5@$XHMVT-SPXUCm24;S-C7-wQ$cHKdH5OhcXW{hQ<1=jrsYK
zKVEq4WbOM4x%O7nzq|5xk%ZvtaQ}TpyPk&!S{`ODzIU4Ac-Wo)hf?N0m_3KnfhYg?
z=~~6FyVd@2w9M819@}H$$EYyv{Jn{_uNEI#p?Cbigv;CQDvedj&*y4>IJxesi28l4
zhDqn`l=>zyo>-xsuX!QqyE)4=&gIjZxcnOBl}b*s$yj|d{Q5Y4x{UC2PPe@7n5b<#
z>;6_{&i8)#_;`Qrtu37VAaMLk%qkO8)9-I@o9EmxaCbkx*uCFK_MCZ&(akF>gF(v}
z-`q5={`Th7<atv$nm&B}YFqi~$(fnP?!8h|XU{%e;$q(>xhu%S*EjdwomX1})?Yl}
ztg@YFnV{MY71;`ZwgXw&r{xll*NN)>{qZ$UIVWr4C;p<;?dwmJ7d_Y@RmjQk>!CO6
z36b=9CmRnNvAU)${IlYi5!-|C#P4~h{5qz(*-ikh)A!{1%IToS`aqyJYZ61ia>46M
ztqT87HTl)t<jlnIqDJ#^%@QA`mbJX)9lX{r8{Y&A>K`~Z=djr6^-rhVQPqsvnzr}s
zFP5^`SO0%_cz9{;x(O2mva_?()1N<hkl=hc=>MWNhJ-6C0#ErEBpzZ}8FH$`f`MUu
z{Qk0{qN;y?DpOLHba!)KerRwEd@iPvw)W|3*S>vxeB7t~GxLmv+WW4rkB`1><@F_c
z=GQw%lukd|*(Q;Z(EO-ZCyd8UtVX;s>A!T8Z)F-6!;2Whvt}y2j0_5Tp^OZbxo1@v
z9Aes-8vOq0SqtSbHT3a`uHUbAv%-0oX<XvEZ(lVS7#3KnFlcOxpS)(jPGr6BuW8;Z
zp0<8T?boTy=yJN-@;Z%Ud2-h(CE>hfDX)wxUz!$*FA87pdimjUcmEmNimuKIvTAi&
z7_p<^;e&(C<!^3$1f7Xm%2pN178Vwkn3%}P$$9SFIZ$`nYz8wspGp3`ng<6O9UUE2
zx2_2Jxpd`9P;bt2vf9p_J9BStYJFM2b$I_Ju6O6=T9=)T-I7zAwS8`In!4#V)k+mc
z2l;v(=7w_q6i$YnMGOjAuY?&se3{H-v7}j#p&?k<jANxls;~ts!-Y+P>_IE58FGZ4
zh%hwtFfp7s8<fIuapU?ZZr>G=(>Jf=?w`&5N#9-bd*|r@h6$Uk3%@oN<S#Z`qo^5v
zZQ)gp+4^^{ojY=ZK|$S5?5ppUlDfbawVMsj?zY-r)!sFktTx>!Q(3vS{sGJVld5)X
zMyjeyo!j{o6%`W`6W2y<)w+3z@qx3?2QPbj`~7qKFYSLR|6B3bmih4w_fP3=7e3nB
zztAX;HTUb=y$@!tIM6+H&+Lm=9^L4Eb4L7OV_o;2w7EZa>w8XGv4B^bnThGoA3yi{
zfO%gJ?u@OgtCMb8q{869DL?1-wudDLI}f;|i~CvqHT%8Z@3-=;TesNx<=))izrVTp
z@XFxjUVHw;D++bC@yqYaxVXqT{alZvaoa?Rkjd@s$3cgg=H9-xyF5RwJzS(oa+i{I
z{lA*oWplI3<MMOfN2aN7-TII>+u_=lNB@(Z_jOFH6$utk*64lp=I1#Xh7CoJznism
zNI%-pUvj<f6?gT@@{0?Z{X8GppRf;fS-<a<_b={SMJunWFR8u#`8DH}Ef0G$&b`e!
z!!+s3HpM<y&!;j)M-+OGzO}gS_w3ZH!dJx&=||+U&SmDFVVd;jSa3`2k`mt?*>2pw
zq_o23PPOe_-jL!iyK35gIghY8e>Tpj`?SD8@8O1HC6BH@WjH^<`0CWV$0i+~7AWq$
zT)f=yhn1<$9=&-xN|vY3XH>nx_~Y5x*<oQ}W@cvT=jZ7zU22_mqvUn8e)jKgrQLo1
z!V`*L*}bp4y*>X*-tFuAew8nN>Tx&9ZPSG5_sf>0eqOWp_|>Zqv#(@@%&sf?epP-u
zr?*_2?L>z4JFA!9@ttjcUucJw#{K$DGrNC&o2!3&r}YOFPWA2Y<LnRr4=ii6R{VVQ
z+~b*PZA-tZWZc<Nc)U+`HfY}D<m6PNo9Fp8EWf|MzkS=bWtmnwI%f_YTBPG2t2blT
zthoJkv6iYa6%`hjA3ixK>8-U!*1F7Zrrqv~4S%h+zM0wg^f*gai=cd5YRz6|UK7sT
zhVaOv_a@6MTE_My?SiYV)4zQw%)U+MwDRg11XeU^h%+2$V4i=&#dblw;gxb1#py@n
z<U?b1A7A=;`PQvl^}!;(mrqaAy}hIG@wK(l-qm*<3g0N+zUSR|H7kF{{;gt+3^Vz^
zSyZoK3tPQ-;=f*&%I5njN()<*wr*izQ0TMGyZ!ywN$Y=VlVr9{Q~00A%Id&$L~iD#
z_cLmbTx4WoVDQoq`N$HrXU|*>PS;NkN+QkE9-Mv<+v%afn0m5MG5gN)toD1-E7>R1
zWGBzw$mo6ErF8vbMuwA$+0Rr;@4QP~r}O*>$M3_r-&mgh<&bxpR<<*#xV-Xs&|CAV
z6DJy`otZIZ%9O?K{j+AwnDT7pryoCm>PBu#IXlaA)t&rE*O(mziGO~4e0;orzHPPH
zWB%q1e6fbQal2}6y-^iYKDRQbij5&qFV?`8*&$2p@8mB$3`SP_ZhZMudvc|{_j!iO
zmy!%YsSeMLrp;q)IQq==&35alIXTJi=ggV&=lIi_Utd7ondj%{^K)}+H>cLB3LTtM
zY5Z1XVYR;Ib#F1&{Eo;?iDC|Q&ZS`;_n#DYGjG?CPP<tB(^yfLo#ANjJTu?iIWn^%
zeRc@W|NG#|wt3HdzuW#<qvKc^QC4%r>Hf{=YiGA^{IutjnYE?bjpfs~`Wk;0tbIFI
zCg)GS3WLL&ow@ry|DEnvvT@E1b3415uSfQ7xsrctm9nQuNa>qP?FqVf#N8MfQs&;3
zu-p?oSzoa;m4S61i|3;6^Cd1`-nM18>FZao<SdJpq@SNxR#pbe99N5%E?rvs`daS&
zeYNI!cMg2{qit3CN<>bs?&+zi&|`n4dD32VWxkxG`s}bDCy&oeK9;|nO$x7euNV02
zb8e31=X6!XY4m$4E}l5CBIWRJ!>Co6@5~NG7v#RYST1s*P;q(o*TU87r~d!VuPCkd
zZsVN|Ntb#=-NVdnEelS0qMr|U(`v5Oc7=%hewP)j9@_b>zrTXhPI%9Y>dVvOu5HYG
zy6AA)_Q)+yk2EV!o?AX|kFIC;3tR1?A5%64FFQ9aKDyLm-MuHSZ%?=toB6-bcVXI)
z%^!2cx2hyN#LBmR$HVo{6IX{HIr4a>|M$CFGj6Z(%8XmKzNWUeU&hj??$3|R%gd%-
z`V+Qr;lgRtrhR>Nb$8j@sJT|9UXMjYC!wDvoS-sep|(cTgRBQSflsCVH?MN~c)Pgo
zlF8npr>E}it<Jx(A@SlO*Hnvhp=y(}cYcX-INd+Lvqrc~qoQGXx5K>u3Xz|Fd|K)D
z{{{yGgU4wz=atrhAB`Bov{$p;a*!(hDC2l{=X9Q3Zp-eo>GbVhx4`J0?GC}8i%#CU
z`uf<dPYo<c$I1lXw_|JwFF*8dU&{KJpZUkFq>M7n7@pXl%)gM=Ap7^#i>tdY9!%eJ
zY@0@w-{~haKEIFpZvJ@c-9w7szI}W6@L|8aJ!l({)1mk?R{v$Y7Ck#R*E-=qgYRrJ
z&<w=khkyP^gfMg;on=}4tdW`hSg*8s<|UP7Z6_bjN#Y2V0o@1C&iD7r%gbkHoA>ti
zp8j-2r65Dnb;gc!xo6{TmPvyS0Gp*#sl&k0z{`;HHS0<DYke2FXG{!lrQ8?_Wf&L)
zPcT+2$jxBUnfdMe$zVP9MRn8X*{+%Dzv=SiOH(_;e=WV_S&&+!co1}eX>4W5soO`U
z?3u9df`PpKy}}<#=d*MAPbYW(dZxvBzdq?g#(JZ9JDvUg-{07n{QBD3&0DwL-CJFL
zyifN3zB505{BUe$E8KafqoZS0=<0v9jZ>#BUYzXWa^%Oy$Ft4z>z<qt48FW6Z|V^%
zlcaeIIjcSEx2o>FGVOcl@8kcz+bMFIUaH@6?2McJ-pQSw%&h-sggueAv$DE%>eQ;b
zzrUvI$Ll@*!ox6A{(h&1Wy6BU0rCtpOt!6MXgK?@>5-e06a(fVr#>8#4soW6W+}S)
zJ>7~8U#2#le#XeqzM7FCr*R+ao{wMd&QzF`e!TUUpS}9?jg|}tQlGzHoSxz29lqYC
zDxPW5jm_ED+?NOkF)(zx%uqVx{bbr*8-@jE->kd8?Z@ZO-A!*mr_`31q$|u9WHC%V
zHHDd-uSdobbbgD-*YLxC{N`GvW@q1C=-mF`!2^50^N)^HGiv{3`tygc<!J5a?lbAX
z|Ni;&Crs|F^xkj1J7#>=uBq*}JFBz7S7_6L#7h^SJEXSM*zb7vy|%+{ThY>Z2L}g-
z2PauLc^PKvulx1$=h`pUlN1&*t~jzhJS@*bTc^O|)7rQHtv@{9Jn_Y=SD*osA0Hn2
z`T13Ud-HSlgZ~b${|-1kb@iI_=R|SqMn=v0$=mgwtUn#k#Ld7FyUG8qym{cXdrqON
z76&i)J3UPobX@lA*lW4TCk+o4|GT;QVD-V5Gg5MIpSzcPtbD`8>C5j#Z?5BKU|I8}
zdWzoqi;1tMb<S~IXUxPbcDDM)%R4<{Qpct>zy5DtaO+gVRN=py(_-avzdv~LER^f#
z)>w9BW&O9jd<&EGS??$F-rBz-Z%e#8yZ7;R&)xpLcD374@@(6eZ%4T=`Y&!iAG173
z|E@rEF3P_0>KW3}TNcgJ&3w5*^R2h*tk}grA6+}UdrQs_(+n+Bm5k$Gtv6Y3_nW)x
z-;=j*Kj*)#HWM@0X0~nd;^c~oih_cIfPf40rz(hbAMcemPdx>y^ZaHSJ<Y%N^I7<I
zxtDj9s($y03iGLm{OI1ZKl@16+uQQlTi?A7c>H^@kJPL~Ujqb1&K!KRsw}STs%hhu
zwfRT1Y<9dWUt+uWub<2Hf)}rjbl=!fn7p?ywzranA!D{>K^e=dU0aNvdA^@_Yj^I~
zFV6g%xogf&Z+XvkVeS9z{eSlC-Ftq%{r;q*T?d=l)6=K_DfqeQLB{H?&h_ga+89qe
zdpX5wLcHsWiL={R)pDGV{IsJ$$n=8BmYetaPF+&na`b-5b}@#Ih8b2q3;tS^=kc&I
zNX>G|HM99wG-dMYTPvcvKK@osHS@i3zgpC$@R-5zz{PHVAN^x$eDL=6cF@_$Qdi>_
zsVH5XZW!n#G%<i_&-&(NU3NWdmxiA4YxSDe;i90iFW~76U7?0a0S6{`w;nhzf9~j%
z!pFyU?$}XLUH$)E<N0~EukY`#Z)W5D^Y`!8EnDj94)}M14zJwpr|rfb^QF*`jX_}l
zoW$$DW!7d`|Nro7U*+S=4MI!PgP3feMk`z0IUTz3(X$0nEgV0Z^7Yfo-h6ykdtR=W
zCxmbBWR6y$pbx1N?)JBx7nMj-_upx}>22El4R5bS<(#N{bF)U__x#%1wz`ZAoBn^!
z5#0Ihp2gEQ8AxZlvd=_1+cj>&?Ag|}NSAi}scp1xEzMY>Rac@fl>B>z{>yXnkJs(H
z9ldw99)rN)U8{B9oS1JAkbK<W;Io5S|4W{|mso4KR`T@8iRtIsXNCQ-?0Yk9-Jh>_
zf67L$c>7{?%!io;W~!6prrgz;w?ae3Kfu72nZbX-*U4Z0Jl*i9Tr0%refmpDh7)^4
zoIZI7arJD|E8muX|J|!sR%vHu*j9g=;#0-O(0M28*el6*f8M^<U&h4XaObMRwI?@g
zIi|hvUcZw6B761M*^(k#?j5`H$HB*lLBQ(BDban;%=q}s&W6pep7ZU?tERYV%H?<e
zFH6ySdE!*0_O=|!X9+uhTzM?dH~)88N9?V$vDxo5-c|U;3D47D<NbW`@oblMw-@FZ
z)E7V9aqbV_&&kupUJ1_!AGY54^<Qvs@aON}<2NKYF87;zZCz|M+t=tb7d%R9_p?lE
zRD2oMTly^E-IL8pX7f9GC+6&(x1A--Wr@nPrtpV5y_Hsq_1?66zckzL`M&gxuckkA
z3gwu(@ywYsb$@^T{QWzAXHn{p4-ZwNw&~0cU;J4?{^;kg&*kdaJZvr8U);QFwf|+8
z-|ou?vv1Ccl|I%pvpo0sjHic>-x0pVaA2d%zaJl)?;78FKhyGisp<5-w^C_!w^c3Q
z9a=V3X!%CrW9Mc(EJ=BHXG+_B-uc=#NA#{N%ePOsDSO-O@=hyx90#oHw=%xlTlJNT
zm)ExHOUCnabAy9}y|=tMWZ<kk^=q^02AQoK42Mn#yby_EnlN?ciF1E<v4}}G_%bp4
z)#$sFwPxE^{r6rQeuUoC3uz6AEGjFzcIgrqFK_OJ1&$}D&(mFhY0YMF$6vC$M87}y
zvwEZRnJ*oG?3FJb4?nOec%5wc)lF+<|Gr9en)b!mcIjeMhRoaLQ)K_Xdg;D#o!i37
zk1G6n@oSdPE&hLOt;ybd|MO<VdIwbenDb<}V%ZWN1_!-!t2b}THi^IJ`u5D5Vt@G<
z?(**$VIl7spE7(`Q?);^H*9Uxbp3cUbMx&RHvDK-oVcIQJz6m|@P(PVNuuP117<3{
zj?*S=XA#p@4;1@0^Y)_URclKY1f@>zp7G)NmC%i6rd_|>WSh3a%G}`k!6`nqI!D#=
zrYiaE*;^g2lJQwZ$n)*9+Fk!wJP(|x@;>cG*=B3Gi^tdeXn(h8Gk1Eh^K^~pzt>E-
zx3{|d?JZNe_seyE)jRPRflhO1d-Eqf?D^(pjSrn2g`zX&>+cov?A4dOyrgpT`Uz4j
z2U$PuUKz1sPR^|zWuL#j1D^tH7`w_w=lyq<>3{M%Vx5u}eoKkdK0ouz&7Xg55)O8*
zShVUyCG#Ec<=^GOm&+`iKYzZDn)cGy{1NSLT1^*HS32Ii6vVJ#p{L)=BjQVgTc;h^
zrImbD?7-aFm%1-L%2Zc5^{VurPy~a+Cl2P=qQLsPoFx@06IdEjSDZiK6#8QMkE?Hg
zmzmtzQMh>4>4y8>=RcG-@-i?y+}-}s^~+k_r>O>77L8Gdex@riOrCc4E~i-ji4<vF
zr<oI2W6T&hR$Vm@y|cH<T$16%mhaa)#rU>vc@fUFat;ea;iJ#ePaL)%o0S#ws5DEA
z;RE>aC)R0s+rppu?E0kCJ4r?M<J`|KXS7Ue-u+fATcF3lV5#cqe^T0G>BR+Kd8*}B
zKio0xex3K?O_w6SR^&A7VSLA|dpi3$6T<;3lOR^9wkLDSVi;DuU9;_G+cf5dstOKj
z_bVJPZQLDoz3!FR$^hrdd>oS)Ca}pI+QE9CkAb1c&SSmz$tngR>A3GEAD8RgIvAGg
zsT(Qd>Gyj5H$NVsg)t%XXSS}o;2)Zq@YHRC$(4Q4-(KBjHT$%Uqjl1sb1_Q-|92)&
z`J6L<&Smw|8?N26q!=7JTV{&*zxSTLiT_lb(;40cTKiXgNxvPombX%DtJ#W+d+N%Z
z)xVb(ZB|$O`s7yc^-R{+7nUt9P5-uaBe#XY{$BUnb2x>C*4ads=VU4~S{Kim{XHT6
zsLJWv&xOk8#h<9HDt&!4<K*2Zi{DP$pwHg?=-fiv>BqjzwE9=||K#2%rUw55N4Kq9
z7@Y5odZOU9ybJ%1n%kz8mqhqan9F}JDpTi&!Hjv97fzO4o0l!uc4}P#=y3KP&({e7
z3U9V75C3cac0rQvcDAb$cLLMz$xe3u_q6(um`rY{;ndw1!;Wg~2$(u~^{oWktXr8Y
z*QqHcZch91<Q><U+MtB=WM3u4ZoLYTg~y&ah1M^2>f83K?Z?jLx%bN)6Pu;PYeHs<
zJ~Dsr5Tt9&Vg2FKvlHuP_HJ1c*x1c?RQkof+n+ryZ*5x;sGfaoTdVYL`8`+lK4-LV
ztkW^xE{Sn;lybw9BOJ^5ll6Yyn7;gFetr3tscmh?USD5tYAX51dpbv}P+a4@Z>JLT
zo0z(%{+-~V!m`aFz~}!@C+9aB6FuT8Si(1IObpQbagw89|I)>aCr_OkTDA60M7!H6
zzb~arH4ojK60$tO!MIGw)x<3Mvgeh_`Q?5;HkpL|3oOz5yGrM<YrXr<18!ILCNh24
z81ulQz^nPiRMWlJ)#ugh*itvE`0})Qaq42**H8Ugb1^N!nfLm_;xpSc!q!NoEW44t
zaq+>nZrkXu=i^F4ITlR)S#m$NJ2iDHkDKZ-Z4<t}Cx3&YL^$fqwAKVTE_tAC`G5Cg
z)~KhuTBRB0J1;J3jVm_!{xAL39?k8s!ezgLUvA+(<CEU+b(UZ3MuqYj7SO3tvo08K
zV0Majclh*4{KkaS3`>eQ8xEvx`~8wLckkbI+veT~?(PJgBV8EL?)H>(uIpO+-r9Xy
zJ?mu~rXK0k4>nnFfk`^+>fLQT*CZ}i9Zd0Gx$S(Fw{uxz>grFY7#q}T|CR4?n>;i5
z+1C4C|2UnQa@$9zxiR46A)elU(-vDU-_9F8nWbw{@Fb0GVjF+Y_;Frp+T1sl&N??_
zSpUA+Sn~Gm`oktBB_|E@VuDMnzW$S+o|E8Fwf%wJ&MDG=UfUkBaJesLlRarxX=K=$
zxV~p|;(AZ4b>Bbl`n8GnTezN9Z`-l>!ukcq!s{<{xtGOBEp>Unt>oACcX#qWEtp__
zXrAxZj_cR^3#<Cqp7q=L0emd5nv~SxRj)O_|J!t+jfr8&PD`Q0V+WsxSLuJZl8n23
zr!vGXs=c;cS8VF%IH#w*FRuOj`+Za9JeA1?S(ba^Ki{u3)8>r4ptR%Uysx#=V*BGi
zt$kZp*mv6IO!xh@hYs-c{yVkd-K)H-5$pd6pWm;mZW5WiYsGACjgRu8sY&OgQ;%|W
zPP&$TDmUxs{23D;e>ckSDJg4vWZeDW<4gHzIT?q(l|GSteV{wuPJE%;zssH3Sq_KV
zld_&1S@6xKI(Zx43GO-53$l+)vpM$V&-N8xh(03w%j(<P*XO?yU1YQ2?V8wmugt?V
zpYZMGi}IBYI6UvM^_sJ9Isb1H%$>Ps(rw58GoAN8&Fa@T*v;d%RHZ=n#@mUh$8&4L
z-TeNn{lZhpo%2HD_ij(Qw!K*oE<JHq^*B_{`l9dMukRbDiO*PgalWzX^PQo|s<vlh
z7sX7_^4|C?;9OUtO>RTW`|v#v7*04iPre{{nbY2@;2m$IIR7)bb4(QrBJ87l&HwXH
zalVp%SY5E;T>oK@&dKLH7jN(TS9XD~-QbzCo7msOon9|uvd(ti$y7G!n%mxKR}opW
zRWU#G^!mG|Q-nL%9<wZ}@4b?3l6iQeC+oH~5fi`jhp#XF&fe`JB(YGHp@Cm<`Ii5j
zH4id%#X4_>JldtX(6rpA`cCnzqUUbL%8P2BFfsBksI~9(P-(1-Vw$jJ-^(+{)!uD#
z%L?6?eEW*=KgXiKX^p*mbe#hKd8wxee-HF!U{KIIazA-h*$dm$wrNY2dvE!=@5_&%
zZ~txwb<T1#a9(Vm9@)>b;C<cXbt1F%JT5g#+;%@Z$v>UZd%pjCi+PJ5uDQ04O;bc`
z%a#a+FI$b3PWkPvHJxVQ_o~|Q*5b)g{K7YOntu<AoBwuw>C<IrcL>kkY+ut7bu~gk
zuc*w{>FsR!Xsb`{CW(9fm%a{7j<r_4_5Jb6;~d|vYX&Wd)!ygMyzS5CKS4?l@6GtC
z;kD-*BP&Deu~)Zd6yH#q<FMt5=!;|8o%iZrv@NT8;@qfc{r%+G;x&ACzwEnfzQ6zd
z^~DwTZ#8;Oooj#5Y`0qJ_IBeNI}9^GHH2B{WjPt?oiF^nt3RI0aLqTa{cIN2zwONL
z3#H$u@ma1`f44OA=$6o^?)K|94PMuLj$2lo<|^0qC0|Ryt4y+|h|$>OSF+1rEu94`
zw%@n=)$1Q<v2&GW$inA2XMGtyNUpfJTD|_3XXNR+lwaJP`8<15wp7_NUu(LY)P3>Q
zs~{<c6o&PORO2rhOPD<gVa__7%)0R8yc=_*rKarFXRay<d>{Ds`k_my)lnP1u|A8L
zyfLgNZ2j`%O-JVY&#8Xe7O`O2y~S*&7mEE6pEUdafw&bp=D|js>H@mKd*c2r-|6lb
zW4%MX#P{|$nY)+Q2!$@xVtC3VU9Bt^nJs9m`}G!&{AG=r`;Qg9t>pS-u0|}pe){VA
zU&}1tuF2kG^+SKd&8u_O^s~x8FSz^c<=@HrXLrV&&TXxh=AQY*n6>hBLXTwZ%2{p&
zTSR{)m#hk4R5=~7_rf>%H5G~MCJeRjC0QN1HBzsJ78*{wwCa2wlTNs@#r*v`ChdRX
zy}z9`-uay&>zdL{2iG(Ph7)x=V{dNJ<D8-t#>UV!GkUXl&GrW+fqt<9``M@21z6Y3
znYVw<#8%;sX@=mFzCMSqnPGqUwB+UD%Fl~g&Fq*D%vFES$g=EPjJ9&DP{mK~oD)lK
zGAJ;v$h+q!WpZ)R_fJdG_O995aVeV3d1_*tTm8GM>QW5`YeQe?Eh;{0a(jXaTf-hv
zeFw{&$e-b}?&k?hDW7F6Dyg^o8CRyCFP;5R;NNL(%f?OyhYwE}973M|i3^ERVN{uD
z?>V*dYkaP-vI(cHki&tGhW|g_cfBVSY0;?Rv|sD-ztbfedk(a@VO~J6>m0@f1Z6Ar
znip;Y<uP$7-M=gswy^HsoOe&`@T)gpm9MNk-*rekE`_;fCHrYc4<UwzbBpqpYHTU#
zSh1%u^vIROSGzbjX(Vi2<E<yN=Gqij<_%HN6B!nqXJ<IHqa(-nG$X^5{XO2{3=<ab
zE;xI&W8d5>d!$6aYP3&tZe+jc;PdKSeW~jAJMON|>2=-u>W{b?G@ce09y&Y4r&jH_
zS{{Q(O;vL1`s*En`}XZKX7*=g&9Tt_b9s@Si~Q%8S66@UkX*j_F<0L57JYv<olmn$
z7p<72`cl9D2dka%-*bD8PWf<fXRfrLy?)(t&|p{H?=Hx7b;tJ?F7oQ)WtbUXFXH+(
zrX()u&s`?zTOaw~mCxRuop5~LYxDCtDhdn=^^LqcMHn2E1dk!#ve4W9Q8Yoa$G+kG
z!i)147q84@FMeL~@cZlK7h~5<U|8s<r8noS<vgbeP5EJaUfd4rWp_DVW~0TJHTl5i
zho<$jq*a@Z`Jb$v%kgxR*x{;GGe0xFDyV(Mz~Eu$@yvee$Lxu3*Ip@$W|+WW<f_eW
zq_wO-d2^eh<;~vn%!k%?N4?v6=kod`&tFY?SGps(y-0QCrkmn*R|@AGmbyGGdOEX>
z^SO}ls_(j$zxA@tvXmT<_FA&OVEr*ULl567gYYX2kFWMHtNis>zt^a-inHCW+4qiv
z>)Sc6_P#r{X72OlmNObOG%HHKv-Yd=eEd6k)28_{uPmND?z1(PlaDdJz1sNsnY-so
zKUqDW7^U{>k$QRj%)cL;Bj;39f8WO}&CkJKS^e+TNfl1@^0^%McFeUsy>iA}q?7$w
z&BC@h+SNCC+3v2r`{=gN4&!o{5?|-iuTM5Vf43orIlpA~jeu`&nWVdfpF1_j$8fSS
zFf=_bO<7+#p{TL9kMXT;RG{|sEhQDZtE2yF#-^Q#y`k~OSv;Tp=K{;8b)X2*P7m=(
z-BZ-&&}(o><%iL)LbktW-Rc@IhPfI%aLC%YxSKVrMehRVP6mdzITmUvA~)tTcx^O)
zwp%rWk;8Nm+Xc>@3l@5R%kp6QYRbdM8CahB{)WP-ze^cDUA?0EsQz1#*MzKfb0+?L
zGNJhY(#6~3z80Q)zjOMQx<@}3*2PubU+?J}=rC!Dhsv}66+ur0_J>;CyS#>#<pL+S
zj@5O$qT-5VS%#;p*x&pJ-pnSux!=O-!SUDbm)6_N)}1RPdBbG?)5()Mjs(1U#dW1q
z@LT)sH=J5!r??B>DDG~&w4(a(r*pG^#`|CAJ^hZwOzUslk*5Vud*VY98hg+Dw~3xN
zOMdSGk9o{F6(tEbyPp}eoBsPHq@pCq!Lmo+opZ~U6GoA2kNKy3J0*GJRIgWq#`L<W
ziz@!)eiM3n$=>%ogZ=rwMR)dqRw|c<`8sd9cKv_ERqpS>r=9C9y<#V?_<B}xNvC4u
zQ}3#OQyD%@O|1%xe7`7<VRfkX>87oci?QBtuy1$$#>&lU#^UF4o;`SwVEVAE>(9G$
zC!alg_TRcno?)KM%)3{zHs8GQ^4h6CULUq7>i=JAYjN##)x(X<b$`X<)mDkic0HM$
zpH`T-yZUQ%-KP_$(!_-Gmj8L+EzQ8tk;?G1R^(pHCv$W2_nTjxX`W#E%(r@8-HbJN
z;$A%PvN*9^<inPIYc@=r`{6_1pNa~XGr#%Yzi*#s^=;F_=*Ri{5<h&pRD3V+6svov
zf$en0oTqM2PwtGKRJ^~^!)s1WVNBrs>o3YvZTa_~Hj<8hdz;Ui$#yTlqBr+?&#l|q
z?{+O{i&a}c!#=tAa+tq;8vplS^RNG!5Yni|uz_=HBzv1g@n<iA<8izx{d}2gm~Zy)
zwG}(pRgtmY)uJGdpX=Y&9Y*hW^w(ECySM6AircGQVgB~1x20eG`LbY_Y)xNHowdP>
zcSlQ3u6p47llRU2RXxo$UQ^vB+p+(-s(;Vs8%N1nS?hDL_ud@ORRP^~|Dt?F`1Td+
zIrjYbbds(KULLZ}bL;EZvqFRyJz5~RAxHKMFJsXcA5TMR(8YBxpW7K%WIaD9t{49C
z#rLJpRO+`cdVjHDwIdtD&G*T1g|*UIf(!?qiP_Z#8pYpBpBX7VfnkC5&x_6R^|rph
z&d;h?^4Bft=N0+o&-Fh?gl=-QTk3P+%-Ve8?fZP%1Np$0B43}lRVuRK(XsCDt-H2t
zzPfs%{<{4Ec~{pZvc&KEcls&Ahpn$0I*ZqbeTrQj_D}ou)q4v0(XY;3{BIS{Q)HIH
z>>-k)ry<l*Df_8zYmw?SZ;|zv_OGdMUAp{F(e+Pn{Pp94HW>%|+noLL@V)kP1^FdP
z2F}W%3llxgy;{6)-DPv;RUFXSpo+z-xF(0mHFFmE+5ew@Xiqx(q$%#)d%}uel}|*u
z(e4$8#RPMKX=fQHXq8{jFv*E|y!rS1@|;L<28K<3zsjoamU<X_boJNoDt`9MqP1a~
z{Qjcv%nT_@)Gw86+pyx@=Nz*O^QZ2Nzr8_B_x7G=4k^1_zC1Xw$G6<L%cAW(BST)@
z?S)U?Oxmh@KHlEoX63~NXF22jTvC^Ae)aJN!zs0Qpv1~>^6rAQb&QORjCCNu$msYy
z?<n`onw&57-`?2>tkUCR_)%$mIN{AA-^}3730qf$Y`LVAd9}?ee(9xs?|-if<<<(B
z5^_siOLL(sw`(_7tHxo2+q1v_U8s_ITzbxZd-?o(yT4D!KX>Q%yUOh)Q}52+{rp;0
zm7MnT%cAekY|307CmK>)A<BLK$Am2!CnlvBb1uzSH`6qnI?;LAv%l|(4+bnvHlHb7
zob|1tx-ab5x_QOVtE_nEmmkeMeC8Y9<}&}OlJ_$E=AZv`l)rlZ+{=~^T_WbF<oO4<
z6#Z%KKDlV_z2o;MGB6zbrX2NsZp@yKA!pu6b}Ri@F5dU{+T!x}_nMP$-I9KBQ$FHK
z{p9HL+xptxS<4Hb(lnP(sgq}&Ia_2yrn}74O~s4jgVsf?i@d);)Y*T%<n^0ZiheJj
zxB2&rDZ-XBWd%3c8~KTdmwrllBFyMeb6`)UxYV}nB@5T)yLC3p?BUz1tf>C+)U>(3
zPo|51Z*h5AG+F$(>ATc`*IhMUiL-g-*3@ObpLf02t!jt$_bp;aa$o1B?ot1g`m}WB
zWQp0onvKfq+}8ZxC#zkvxXWJIByTPE#OswG?w+hXxo=^m?$%X*dGpq6TQzglj$5l1
zSy*WBFiiX{S;(zgF)w~q0r!H6Um<~=8HbuCxjs(ar~0Jx+0s|LPwk0%>yf@Slr!kS
z<Bu#!V$rcPKc97E)X@I_XYNFi31{YqxE#E`Ns}RA->z!)Se~WJ!dwnyC1l=OGv_o5
zliUW29JQ0az4bHLjN<i$I-lGsF$fYAFZbc{NHDHXPl&ndY8Moldz8EEZOzj13;XnE
zo2#=_^hT6thF19eU)H7KX*4rQuK(ym14fU$Jh_69Df|Dtp2RfiB%9oH{pEkQ-ddOA
zd2joX$il~O%6xb3Jn!<8{q&N3D>prsmA&D)>D#da*{1MS=k#oorXEhc{KIswR`sXF
zn^vzaSSm3|NrZFFFQeUeZ+`M&m{$?4u)mUj=RDuYOv6@hVU2S4pWkF&F&n(@bm4Ah
zJ!^fk_ln}YXTq0Lj=nqI@@-qux37{*beua?YWc)B-md?wb=N+U_ee_dl$)vdQ@r1L
ze!c8n|Ml3(lwz-0Vnz2=mUDZZUCVKk*HriP-k!gf3W1J!UPVSH>h^kQgybgWt@yX|
zaXfo~5QqCtvGWDf)&5v*&V6FD@AWV3<@&MbDoV?q&y<K&m}gu)f1i!fjbzP>i>q&)
zi*=p6Jx6ZI)9s0`{~4Ea#Ij9QN{sqfUUuCjx6HBS`=bTrR=N+<0=hCc?0?>Vm|Ojw
zQ0vi3?{n{qqIRU$dARKT&|5f3J$ZTlovlf;N}E4TdtV!!>b^qg^|h*XeZ8y=7hm_C
z+hoolaA&3F>Gy`t%9_f|3mI8Y_NI#6-f;ZJsWbgTGtE*NIa_QN?-f&iX_H@UcDY{U
zc97?dWxT5mHfAJGNx3Lc#;q&)T<i?@!`AaxQ`MqRTg}|KV`)5Vh2Qu7)_+l}*PXM!
zdu+E%5wA~7jIWle6bnO9uhp#6YkX6CuBHl{u{*8rzxdw1{%;a<?LU55_3qY#B;%!r
z1>UDU{QD(x-}B!sEbA^mJ?g)`G`eip?V#QZMyi$9pO?4BCe|1(e%tV|nC~aI<F!+j
z1}t+e_m!`Em#-cn`Ph)N?)M|+wBDJ!Pb@pk$(@$uu;*dc;#nL1$h~(~)>Jyo^0E5j
zRHus1A%A>WxIP60sGL-0WLTk8bnD?Ep4e^Q6U-Po8mhjuh2$ykKR&ber^)<_9gGaV
zUsJ5j6y$8g?|G_w@9%mkxv<NnwnkJv@8XiiWl9H4%rlo=J+NWwpQ|Z>*0P}t3`f&q
z&!)<yK0SSs>GGW=hV2QNGcBx6KIi0?kGEQBe)vnaXIzX=)<OZhevzX`zpETkVqh@*
zU!L#BX&)H3*N0(7+PTMPS3d82|JhD||9V0H83%cKmk5UjupH3&`hNyb@xuaE;h;B_
z6XIDEm<v8}{lDRC7(Dl>q1VBXHiijrc5B_J^4#!s)rYA%g0Zizo~z$sEjCHW@+13g
zsr8<sHUHE88opYxfBEf8_wSoAZ{N4V;ZJx!ljf$~g~|*Ix~2zi=5TGQ_`b2?t%>(m
z!_^gGe?G51a3$l)msG~DwVv~ORy^Uma^L?2SGV`ChLs)yn;fn!dHQ|E!igq{Oy1ip
zw2fzJJ7;Z@V2JP$n0#S}=xVtD!}6oLy%C=nj6U_aIzO3r_gO!;AcKK~4Ohd?1>19E
z72f1Nb>E&=Ii)J}GyC~P4ncjf;n!;ybyit6D?UxT^`>LT_0+Dqx;<<W0&m|hzP8A3
zsY~SNlP4MO=>PaL<IbGv*VVtx*0ruU%W=BOXj50foT#U>%3f=|F8|cXz|ip9WJ-Se
z3f=<?S1<`NC|D<DUwXoKcJ|A!y1|pK2YGX=6z;SMnp0?Q5y|43G_8k;p~LeafA=l#
z(3^}5Ut^R+j2Ro|fBYg<95Maadge#G*=Igg@8R!V;t?gqnYQWnoS8zZZgooasgsWE
zY_BP1Xw=*tyTw30<riQ0mY*tu%okV7TL+(+W9nizm7i(dk6+3QYmD8FAKN6Yeu61{
z%ZHAPCmA;<g};B$ckXI?!)Z|i%cI)$qSBGRa~&ovjZZ3`a?^()&6!!ac!$n$rcbAx
z5(~rboLRVg`5lu?CLM+cPoFWlKmGjZ-lOMDZ`UclzZL2&P+yequx5h%_NsTk!y-AK
zKkkg{uJL>O{zlKKUF8}Nw0_!rKX7x~yKlexwEOqS8CM+H6fWESXlFOSp26DBe_HoE
zc^P&+l{IZWf04NS7n6`wsdIPcnw3{9x4f*bHP81|ZCP!)n0588);BGA$JWZkY>75n
zbT00t;qt}YrsXrH-rUn1?w+AJy|e9%uhpJ>|H7$v&uA|GdL>f#|La%p9%;@@lc_d4
z{QZuRR@r53?&D^rPdSYvazl+nFK4Yz>b=6Ts=@1boin$j(`?tJqKxkI8kD-e-~AS8
z|9{)9iJ!7o@1Okr$&RlRmacVQd(*%pvvR-Ycb(Y&yxG?t{Jhw=_+9(`d1dLpzb#+z
zOF`j6%F~IiwXfUWMr=J{{{BSn{%taeJ@-E;=bK&);<pQWb!Eb0^V+=WXJy+{N=&Q8
z1i5)W+lrTQFvZw1&oj`CEn8b~_Llkowf7iKUUXfht$ch-;MtyttE;`T^8d{Z35)*A
z$T(d)O<O#_=A%x<p$Ey=COSIbND6zo_Uu0GlggDghV6&hgc>4?9Q0ln{>cx!DQf=X
zXY$;J<E)lX@5inA-@oA&*HtU~J5F9-EOS41+08$oy=b0zZSCLZ_mwgiEV^I5_VBKy
zT3=_G8PDIn;`fEyy0Z#7ygV;fe$#L}vaXbAHMen2oLR-oeHj-U-l<zfZ)d)rt;3-8
zeBtkFJsa9rM?HzyzE$#dS(0n;x7|!#MMd`wYNV6vZ?F4)?^(ZgBy-G{8y{Q#ysG}T
zHusAE@`*dXx`rlc*RS0s%<cGTQqsnobMil5FITNRbN=V-pJyBIuix~0nWNeAH#>G)
z9p>p3SmpJ`@37iucR8QA3I9G`)x9{?oAb`QriRsXm#^`Ss9IS0c^m&a*Ul?*&GLh<
zKNR3udj2z`iaozdVdTy!>vk=(6wq2F!@$rMxjXC?L#x-bGv1x=k4G_ztY-|n^5Wv3
zWt#(UOU`a?|9+ui&+i>yU9Tz^%iXQGc_#J5Z)Qb9VYco6yA7Ykhupks13CvG)a&Sl
zYpu6xPMmL7QadYIQg!#$(py}6udSY}&aLrQ?QQi9ar+I%_R7=ajU<vx-|V{WdU5gX
zOD~gKwSu=vWp8ZY57?v2F3sTJy6sDJWbO5a;<HTK&VP_9$~(0$cb6c@`sA!Fb9XG!
zy&D)(J!4^DsKS$(#cyxE@?qdqRcO?8W?=A8d6@pgTq{qhs$p#mzw9z0=}(_E)SNdd
zO_(2+n7g>kIpce5hIPu(KHlkarGGUd%~PE3A1=Om!0=txwxow!PFMZj{gR{qeAH^Y
z^4M@m28L-ej49V1r*xlH%=^{VwOD$EF{8r-d3J^GsnL8BQm3wAJXa}L{%77Zmj0`0
z$E0G_)Lgb$8`d;Q`xS<-YT5ixd>(hm4JJktIh$)6?W7qL)IZ*`n=gFUHq2ss;Lql5
zEGvu8{F>Q*PwGn0s_>8y-Zk5tLY+D@7BQCReLlZu*%s586EyEgt15)f(ACOo><-KR
z)*yByY5$?$mLYZmZ|<+#pmb$M^Q?yqtG%|qytF^`ck5hffuglJZ&Ka9n1rx4gfXZ4
z8P(g?F6f<@`Q5)cy>xPQslfvqk4Jqi#~6=x_x8@Q+u+|XH@BCEXPIml$LE{elXh(Q
z%=wa2oH?m=qxiY~Ym*;1-IY#DEqiGDvEt{0T*X91^~q}a7h;2NZ{N3q%k5k9*ZV);
z-BC8qx%2!B4};X%=nbduURzYt)NVeXgQ4N5S>gZnGeODA{OiNpS^Mp;?mu8?vWja~
zmrD}Q{~yww)0$X7v%x_|D!Y?h&V7}f@b&J#P$LGV3n5Ak{;Bnm7h}`Hd2&^ibpOx(
z|5>`2!D9vYm1h&2e&6Fd#}yx+weV;|&B^Pb$wn=WQhUxXvSVNnF#dINwT+$?pZ%@J
z0--jG*7=^0OJHJHee%_eoAcRs$lUF$`|kE}$AL#8;hsBnZ@%Bg!k}QAwD=Uy{+5ku
zHFEP^({euU%G+Fj|FPxK>5}cX@529Fy~DnJS;te0oM#>G{H!tS7VKYaG`pnp-HC&m
z@4Sv*%)Q?E(P?kW$!!g%S^J!7W7YOQ5n=3)&f+Zp_)XJBZ+%5@uesoD_cI|o>-^`R
zU1OiJik*{Tg8a6EfNw9j+j_I!CrBBatm2B)cfH5J5MZ-O_uuB~&k;Wk20nDCW;2g{
zQR!x}VEdnjV@Fo=U)aLutg|S>h=t*+#;pA>4I|USdwQO|d1_|#u$|j!zlhcBnVXK!
zJjrlCMf~TF=%z_`xj1#NwbkWlg)drhG9{%uY>vQ+nVXhxOWt>G*{tiyF}%<BUPzQa
z!MJ5|w1##><?M5@`Ps84&0TuxP0~Tlbj@`S{2olq_Tt(uU$iB{)jw}e>X$dm<LVa2
zcUSGx(K@zPe-+RB)8b7U^LIuEm%e!-|K3OJ96J-kyt#LG8*7Ez8UI-H>B|S!w@%@D
zx8*->iQXP2D0n`Fef`d7%QhyyQtX;^CThLSo}<fUyUV}W%Urd6e`67copSEXJ=OQc
zi__%Pt=8M`TIDb8*|7R<ub<uTW3w**SR5uf=c!wXRfXFGeg`oR1_zc2Qit!?Mys%S
zct7aB8T_NieM_$XZ4d8bJ6S)wxm-LLp?Jb@?wJ`ocRosgc_=*Ievb2>SjQLIoD&`$
zZFH^E_hx%~<W61Drt_?u{Y3tj@9<cpX}p;G^f?E%ORm~}qPEJN(`NNVZ)ZL4Q6U%G
z@-nh(BYU3x)=M5WUq38Ve#pONm+y?XIX>kcmlnnCt;)W(MzDM7i}mK}X~&;vzvWvy
z*R1el+kcK_0vcM+r(NE2J9&><)!7(@eY^f;bS>LlEZ%f8{n4z)9~Z7}J^m+db=i!4
zd#r==AFjLUyz9p8ay5wqO>J}CbaOPV%isQ*SX*_*#)^N%m#SN*7!*`~=}NfPS1r@X
zdVH+6tE<b%XzGuU#4{H<!jwK8((a7XmK07XS>f-Z;<3rz%VWVi)rky~7O*QN8134Y
zaL9ajk_XqO6Q=1c-}lun>Esb+aM*f9(5drc3#a5vA%+E?RP|jKEb+*(Ji&cJ;TqrL
zElEucEDQ;Z8w#eVKDD^Tw{D*IwRN#Z$ItFky%+ND)wj2|Po4xF8(;tV8SmYVBEm=J
z+hje=ILOE_A@q7G6T|nb!VDF1<;HfcTl4Ndi}MvV-C`oKi>=qPWsaYBv6Co+%ONHP
z0S1GcSzqn*vsJ|yIgT<h1YCJ<${V+qw^r%gp5ogp+=X}jZ?FCTulD}F+VwFzKYe)U
z%(htQvPN2l%jV+ktxO4hl1j%<3f=utw87%;r=KDWs*@aM#v0ars(2doIBUTX1;&6t
zt$pl2E4BL`{(X9UTk0E&pdT(R-zx&PE?>D4wEI=jxvk{g9ZMaZo@)_Z`3BvUk&|o^
zU;Y=IS+lVrq4Z^be!hJ|!`|BdT}*k34Np=nwNIQnz31rhD>C1As^{I@l67@wW8YgX
zyKnY|KR=~rXTLr-*V^69Eh{@aJS%?2!oaD&G}ssfrbkV6ecF9~x%ZS%N&o$PU7Vto
z3=Gn>Vl_(Z>n7Cf)@GZ${%FjT17<(V@;qIbYEE95zh7_Co(Il<r>(JKVCZQ(^+B)s
zRY3p8cdZ(+^8&-!9-N-W<72jP<*@^`qN{xT9&Oz^Z{BgC&sxiW{+VHa=jSRX{q<Qa
zjun-abM5QvUS3)nyv#>aYteJZsy|IU3<-?x`<F*vx^}p~NR#2+ZV@9()mT^WcA>n8
z@;8kPEa%VV+^NzIpB8^0<DA~Tl25`7b7U07m>4)%?&z#@`4OXZ`qv(%n1~RK2hX?|
zOm12GW_^6{<LowB>5JQh`#e-;+1LM@q8Z%O+&o<`_EbvPuLwC~lT}t-yb<1OQW+Ms
zw%!T3yuSX<@+8-!Nj9@T{*2~ycwWb)-Cpp<RAc|;rZvwO&3^B>71Sy(n6p*o_4$q~
zTO-a&pEA+?`ad(8d8XYho&=-i-_oolPKnjO%{z6YWY>$mbI#mK{FEL0#4^7n`|**^
z`73_4efrI|*;2*CaPs^sJs%b4XQim^-?ZANwBAXh`{=IH*J5`A|4g%AbNFYlS5?*5
zefNK^TBY~DI!5pN!DjZNqWZ~=KEA&1@9)<a;j%7$_2tRQ$z1(!p9}fr#qWB``O~Cg
z<$1}acAx(l?=@Oee7mvFwpP=~SnX5jT)o|Uf3@or8=4x(mYq+?)6Z)EuJ`HYQSNe~
zb8l|WSL(DDl3Jd7qtz!WGkNFdYcub@R(-VkU;E^ug$rBHY`eH^a-U5aBg2*VY59rU
zH?ha^f35lXY3bUvZ&!z}m$Rw3u(SC2s+S_==T^*Ib^V-VSnLy7M~l}|oD3~Cm(!=*
zSvuj+jePkFv1!sh?^my_a|-TF_`Gt9zGOsj=ZSO688~u(o@f8RVyo8Y<<{vgE-pL3
zS7w}@X}mn>eQ#|)O9EE~lUVy%o#+r|6@BymCoAe}3pRl&Hj6Vi`}P|p`AA>2&l7-*
z2ziycD|k*iDe}4E=xyT(cVg_^znwK%{q=GF@2yV-7*wX8{=G+CjA4R`!OSN8^y$|t
zW}aSsqM4oFtxx9W>hSft(c4a(I`!{Yle|UAiwixH#$virUw+@t4`E~MXD^a5o1nV-
z+y2`Ii}!e2q|STzH#L}-fx+>{m8HDlYqkX}TUe%NVNvnz%}wuVIx}r5i$cG3mp>Ai
z>t?N05~LPUH0A!h^TPLAq?&mZn6KRKzdt83O}fY4YsJm`c}FMy{dsG>Wi;32&+#v=
zt-XDBcX>OnblLB3x$*JwzP@L*UYLA-_i}2oSW%4I98Qi2^93bVh(-s66xV&$xcR&N
z_*zeijP)DKUQ6%Sj@I3B;{EG_e=81FRrX)osjpg9;(vSR>Woj*&NgppGOKuRBfOaV
zNO9Wx%Yl!Zj-L_zw&8i~Eiq2n|I8W{)!CQZE*D9;GA_KMyL_6r691VSGI!R_c6D)i
z^Jl{CW$XK<<n6JuPknnLUvIt9@9*#9HzpnJleKQ+k^I!}+bd<d%DJ7-xs9i@r{~Vb
z<n~#!qFP;kEWa`5k<XvfRd06l8rPM0X8Q28b3VGVwK-j7^`3++_qXpqb=kQ8_~%cb
zy1Kh}SAAWzaABh8_dZFfi<+u)O`aLZ$3Cl%Q&u}X<FcpPZ{gHi&T-Go>>v30ddvx3
zS@-u>`1-imJrx_bZr!?n|NgC8OH;o`T;I{WQLlLU0S%?32fuqlmIrR<kG*-zVE56R
zir&om<(aaZBGWQfbZ_Q+-E*?}JL}o_Uw->8y}R}?Q09k+?0Y4h{As_J^G2E8vazav
z^fL7B)`LgWyxJA-o%Q>64L%O_%r3&h=SgKzp?L19{KuR3HXjM7IP}l{Ypi<A?FpNX
zv!9I;El;lWzk613SM&69Uw^o0*}kz|@3SRUZ#@?)E9+!6UoTJ3owdKqy2W(0miWfT
z-kmXHhGp@yh{(vFKYy0KxUkS^)>2;Y!-ozAMHpN=ymwXi8X+Mew|=?1D}$F;e}AW`
zq4DQeQ&*ehuCzreLWP3iztVT+ALYtBI_+1SN~Ca%N3=<hTf_`+fqI*y1f!X&!`8mK
zv-9(biOTAJb1aIVo!Ou9#Q#_D1BQo|C!)N)JO4hDVzBAo{nPJA)b_mM$7!!}x4TN)
z@7Y*-^V#2%hkoiL{yDi=tdCK()Z&kQ&W6Chdvs4KRl^6b`qD)C!c=BFQ#-pwszBt^
zHW!YZJEDvY7cQ8wGyI$F!Fkif^kLMzHUF0JMw>)wuNGfrbmq+&Y10JrbKUp9=RUrf
zs=vObU(U9uu<+%zwb4F4XYSvv`zII~c~duf+l`wydnF8${`~m3IsN>)N7C}5UMpw6
zev=n^@AbOa-PhJeZ_mBGE%9($Xyx<yiEZm<F01hS$}f@eKuBwO*v_L2D<&<8i{#wt
zU-kXo&+_Tt-`xEC?(S~qHlCfOudn_4``gq+tML8NNvc0oT}_V`&x~Au@7}k&f9`+w
zGrztj|Ie*1?kC?%E$Xj5*ev~t;k}(0g9g)v5b4bh8f;<#OQ*G*J~g$<U-_uRmVyGA
zgrFL0%kQ!48k+R2lcr|c7=L7oeD=xf+57kVOI}_A9jGH=xM<R@4coWxFMofpLr{5L
z{Qh^(pNH?S`|I^;Mcc`fCudp|CWVDvn`>P@%QV|9S>@{YV9{SQdtYBa^IhtHLjCza
z<^3=Eo|Z?JCaf=<7H=G|ZnF(T!!ohrdkR16mZmYXl}x<-Y|)0?Er(7VJJR3R!1FLi
zd;2o?LRGQkqWYb>&LUwJC&CtcX>n@$@y%A5EnBJo@7w;6XtSIfA8u{UK6dQbwQJW5
z4Gl#l9&UePvC8F}yBxz5%~dyrrdAZa4h?75?t2hhG)JOK;{NrAe*&AMAB*lTPyRi(
zMuCChfgSJD+mczQPiQ%P3~=7qr+Dy2iP0qX1LrG5-D@8;O;X$=^ZMoNW5VKlcD}Rx
zy=KN?CWdJ;HTQRXw_OxF^K@mC*RQD`D{Rco&E@6i&o<8wi>uNDUHY;*e0|ybdv|Z$
z`t|4M=c%{C&kBf&rlzJ|UFJLc#fyxp%f<`_)AeG#@>y0)iYR??!Dsoyf1!&-UrB1M
zFkfgrC*IgJbvCo*)0PKYCALUyURE|Kdzqh2y^5C*6T^(s3Gz$NfB3F%qpF~G<D};#
zSFdHM{VK*Yjo;pU)*JDu^~uZ>PKK{fX8w?Ezqwac`P({+>F?w}luvNFclp7@T~dEm
zPd~qM)vABV**t8EtG~a~jov0>nkB*z5fv5oqv!kN&A+$H?WxsOUG5rvI7~Ns>(tY6
z`^(PWi>>xwwOi)H8?EW}#ZOoHhR=WAc;@rree6~~w=ey9+GeR;9<x)XMAd!Ksf#(8
z7yTxm{c)D<@A49(E$>c!|9a!slv6tj9z5P)I+@4*ZOq=Uo?QKRBeut$R$H4hD<<Bg
z{O-C~{<uA!{{CUtU)mMzsr<6->g{j8f5^37Z9lH|=7!9Sg3q~$f4{x`8e+0#f9{>I
z{coPiuK6k*_GUddgOITB?{9B!@2UK}VZ(;1udhNYe&1K-SXBJ{Tv%Ax<KzAHpP%*i
z_WFv7d<t_{@mv?RHS6Z4)PMi}$;ruG`Vf66Yqiw7UteGUm6=;wwQgnp8=rEW{RRgd
z4>oER*@qe(KY7R4@!h4wxC}c6gK5420o8gR_GmFFJlk3PYQIojj^gjP6CbTloVGe?
zwNB&jndje{<eiPnI~=iNxyP^XU1Fa$&1d0xck||r`{n%Ip%X8!&A-1-)}~^^u3f)C
zXI51gzTfU@x}-rpt@s85)4Yb*7}MZYe9H|!`Z?Lv)PCvlVBx(z!z?uY?+!->2Jm29
z@u_$Tod+L}$ndr6%yd>~wVBtjH0GbxqxVw(-+kf|6%g1^@bHlDY%|gNv{gy-?P`mP
zi>rTrdU~*#T{|l@s(0eFX=Y__BKX>c1q1@NuiIi1DVDx7+d^7;c68vxnb)W9?Y!ky
zu2bJOLI3Mkp+J5G?Z>AVpHNv4(;oP(qw3>}2?>r{?|4l5{_tq?UfVnFH`ryqtiADt
zhoM)>bXVbHx7XL!u3fwK@uNpt+ddl2$njg*a{SMnX^pXy0&X6y_BNWSpi%Kt#d7JC
zCr_s2)ZH|_a^H-D;Xv5Gz$(4GdCaM+Yw9ANeU5#6KQ_*hi<?{BXU2oKZ)+bP<Bhsx
z%&=nBs<yT^zZnJxuU#|K(CC;Zu_U;nqGGw<++W|{*MEHE`qJ}!gZ9xZ=Gxl7yEks%
zb>myry#MPe-(Q`&d(-ww9ga+@?`K{4tmC=*7f;$u<*zez8RlIUeYpQ*b+!4&4<C55
zr}@tKu=~9F&tM}XquFM;n>KEgw5j;eBWb+q$3gC=$=AyZN|#!9torcg=4L@<w;8i%
zA3uKla?qCGS?%ra$NOa4c_f7d1s}eByLLrHb$L7+gWY`l@^^Q%!`Fd!1V=<%0G%J#
zwoY`*)~)Z4xn+0Fx3M-0Q(Cv?tNzrE&|msfFI-D-2z<HQxMdp~bHyb8P}#7fC+ey+
z4Kq$WbpEZSZ#(_P_4W7XT9^09+uP;b*x*0kF0`zig8_5}&|>%gna1gIHWd@TeTg}9
z;DCaiU7V-q$y295C&`txJI`M3KmXQ^8zS~Op<I`Z6WZ3<hJ=KybB{5(mS}7FL__PB
zOSwgT+aqT|hTo^%#C6m<KC7^%_C4{oyCD8B-n-xWi&o92som)^GBT@{9ar<6b!SiI
zX3xoLvespL%HPLbEs3)J`16nFWVPHI8x)O<E`^>-<9_n=>GyYccc-0|a(6%e_V)Jc
zhgd!T{rhKI{cVm}?yZZ9-KS1{S#|5XenQ(i*Bd$CPF{G>XnR)p+xI8+TH7zUb_FCc
z{Mgjt;SyJ^pSk2X6NA&di#}{}wt`b_CwbKET{r7ahu_{^&w}4fyKy@(^v}=Ou&`^l
zZ&yD)*2~P!_vPcqm8aHa_fMH3A}uYwFhD|HK0hJBq1@tWaambkcei!vt0|Kw2bYzZ
z-F+A`anGJTfs5VV-ra4krFH7;+1_^rb!^5ZZ=D2=^yNlB^C_+0u-^UO?^X90o`2WO
zG8E^XlFWSZs^h->VhjgP7XP`v$>{d4(`zNw^s7`q)M@_w$8o~Irs}lmWX5gN&YbaE
zbKdkq(9Ua3lO~*8ZWH-l|8DpGwX0Wqd-H(yopyF|a&bvn7O6}>y;QYr)~g#ElgrA=
zCae4NadA!hB)!+l%E~<Nj)9@!#!Z`cmA#d^`!HnJjvW@-+N-O+zB+pB*tWd8v)&cd
zF|J;lwNlx=ujhY$*6V}+UY)hND0@2nerWVm=k1KmbM-6d{6DNLQ~5xYks;%h!!`~E
zhn1$)yQhRkPds!e{rItM7nU6ImtFq%ZM1?wMP+5|jsismg#$Nk%($13#n9+rkbP~<
z>+9>~?d$HGIMMO0;NO9x-Qw-M($C(#+gI@L5Z8LO32Y*+x%c)|9`BQ_{{HUm{r&cL
zAN~=mN%GXax-IuM-?TLA={q(|*(&<_)kRr828YLLI*ZwSPOJF@uKM9#{rTCm;K)Ps
zWd&<@ZU|sfI6ULp^t<ady&aW0Jp{Z@Dg>VR{o%=l1A1w`hn@Pv-YlN+pRfDqrtRC;
zhp)f4B5?7PDN}rWd|vdekgERqDfR!qzoMd|=jU2q-%*&n>+nxzR#w(|^XBPBZK?SF
z?(V(4)w@eyFI%+8sXRdR<=@}m54ZFCdwT9Hc{!;>k4sy;Z(;UU_kOvX-(D<EG}~C{
zuDN3GN>;AoI9tvFDf_Y)x2|fx>|449blQQGiAQ*N`SWvgxAebRy?Dch4WRS?A~&U6
zTH;x9{8Rd-&6|IJd6}G+wypTNUz5VNoSU1<-rmxU-ezHCbxAhr=#HA7o7&rtw{QxJ
z=|-vW`g8rRov|=`_xAkzw@O~0%($^Oa((F4O(DB}o$%g0zoO#CuBqMXeqR<?Py6)o
z=O55vfG#d9Y;4<dZ%=zyu!?tKfJOPcJEy1X%UhSNx#jD$r|$1A_kOv<ZM@pH^Q*qU
zySvc2z3knc%D=z9e){yueqYqBPM1Zh-qYq-6e<}QY$$$yuB83*@q(ohxz|?&F5bQI
z;;W|i^L@KLj~t4RZ}@krJ$?Fo79OrI-zVPs!oyHhR8;-#P2{eUmCen~`#}*M!oPCm
z%5B@W-Mo48`@6fV%U=Hc`1tr-tI|(jzUUP1zI^4%mwS7wnHm=aFZb*0>@;!xzCLbW
zO=foX?^jn>v-8V^90%WD$gpDNO4r!kIV*Vmnb^91R_@=&@ONR)ZbyB8|6MhTd*b)k
zl^p++t?bsbqw@2z{{H^myRG|=FSeFhDJCa(Z>jh6i;Lac+uFi*OK-n;@nTt7nY3As
zMtM}6r>CZYLBYE_JNH(9zqh}>{%ZU8{-);U-KDS1($35XUhcPY_3F#A=i(1V=6#vz
zqBPSa_14DA;k<5bn?j{7$oid~D7Dk>zvqRmtHq<Y<#?9s)Gt`EBIDDOlNT>uY;0`I
zy|rbbwaiLop-wTKh=Ol#ZcdmWP+VNR>+sLy^mB6xi;KB=c;4LG`}_QPA5Tw7qm+)m
zzH{g1TK})zzg6qN<Bxmx?0NL)QB!mC>gz?n-anL(sy&uqQ1<?gS@NY9?7yo-Qq(TW
z`uRF|o|tm^vP5S3`=fW)#l4hdc=GgVZEfxIv$L=7E`NXEfWxlC6ZP`%?Rj}`@9*2&
z^WWd!A1{0P^TB5J<Gs@5&(6$@UzW~r;p)}i&d!S$FK*nlY1NtDW7@|KxAU)EyEZi?
z<=wr#(RVK!Kah~B)i=<1vHxV;+IE?M>@44MovRX>T2*B^ImV?QXI@_D<>gh<{`tF*
zurNEXR7zIXuGG_Fnw4uz_MV$-ZJv2)$<58_=Vur;zl-=M;Tq^YO^1_*$EN;Y&FZkV
z|6bhMoPK`S&YhAH5+_ca`132rYJ$qo2M3w+^WP^PZo4S!ee?iN@2SGCudX(pJ3Q;#
z&4}%98FnSFoaWMM+1cp4D`Fd;#S87N`RV)DZ`g3*@L^>;J39@H7ykFJWeaVL*i-TG
z(S{8h9z1yP?d|Q$vghO<fBu<#tf%t#x3?!JtDBmd{$JF(X3d%%J1jEKNgX?O?AEPY
zCGLTV2F{B=sj8^l_`2nK*87!r9jmU(c`sjd=XyW$oI5dd%+|y^m+Oe0ynem>-JPA;
z*VnBzWS(L_Y2G}$q9-1_yuA$#4X<{5(l9pOT>t-{Zq$|!U%#equHUiOY?4-mywue5
zCr)Vm^DB9n{9jyL?9uTZ_mBJfd)&C6ee=&sQyp98hS{^DU0ja*`ubYkch-^TpI`DV
z_X3Ro1TXjN>+4HDH)mzv%byn)yT84?ef#$9{5(8IzQpuCkdXR&=6JvS?(eT=`<i9E
z+|hk-(o$=g%<qanvrct#vj6>dWk$PU?fL}^68`-7*e_?x#ly2C`+WZ@mq|}HrJinT
zZtj<}&B~3{4Nse6Utf1-hT-jPxza`{9CtS^uy0Pz_+p%um1Vv@-Y`SKk4Hx3nfJ-b
zYnO%`+q+obuPVl(dK267bcTSC5SyAGAD*0?{PxY8K3QwGa*L_2@9r+&Tm8N6&ySCP
ze}7+{S#@t)?(E&Wf4{uE+|V}M-J;EHzuH6(n}4%~%HFm8I}xz0$adG^iEiuH=boxA
zG5B<f_2vEZ>lUe~sPIUcoH%wY?Qk2fY0;xEudc2J9a>{k`RUKk&#NCZ7amRecx7eq
z;zf%N9X?!IQu2b2{V%VK^Wykq1BvR|GQZCf9~0ugH?2u8X;+Tb=zQ{X=FAVzuI$}$
zd#k!FbHl`m7xz|w-w~5{eO+w)z8cSR3sFy{iEAP@9y)yZ@r#R#xy5u=^u4sy*FS&n
zo}H<wsg;$~<>IfcyZsm4zq&K{INR4(G7+|JfwyDgwhG>T=)oFvq`oT0NBQTQ*Mj1v
zCMGZVmap1)GbcSgeUeISY;0*s$(JunlB6%!fBE<K_rYfN{q_I<eR_KO*RNkC?Vru#
z_Ex24XJ;oS7Ctz@n0e!&$X`<#=f(C0Sx-(}m~bH7-962gZ@IPDuTHMw`7$rAFWl;N
zWa{Uuzh<iZ&$zV2Q(P}5Vo$}!>C>kNMC>(>-&yuH>Uf{*?$Xy~FE6R;>gw(~{8M^+
z-rY^xw#_rioV0y=`PW;Sg0=kbofq4GdvK6>_q=bVYnNqjm-jE%*`j%>_Or@?%qtuI
z%{E`Rx^MFWQ9XVA{5Ll?u3QOfPpnzCf5P+g^TqRnZ>)_r-<AB++tJZ6E-sFdk<qs5
z%K~e$5Jm&%#rr4DnssYwwExXzwX;g%qkQl8{bx0Q9GG-rLTY!K|Lj+u<vN!@HFNp<
zdux{*D=sd+zppmBl0P~uAYg`RcG%vkt3|pgA0OK^H#HqPe7JD=WGQB2lV9GJmX>#`
zzn)bsDPez;V6ggcvu=I7pO;I~i;Dk$ch8+O=SAPrGSI2N*Vabs$Lw$@R|s1Eqi|kk
zW@chyV*Pguf0el(>`Z=nOG!yxG+(#vnvM@oi5T~%mwd~+_UxAbW0P}e=Lg{#^CnI?
zapugMOG~-CkM64as<rI76q~Whuj)^qKBfE5f026k)xvyv$2v7o!^_-!Tbk?IJ))vF
zHzvF9N_OO6P_VO;lYbuBoY>~}zvX1guXnd?kJc6(+V_q%@?=SSa3m-h-n{wb^yBZ3
z@*exzsi>B{yOWupzd!Z#G}msii?ZCSM09pBJD47P<jO6TX+Fp5uw#Bk!>T^jyAMMQ
zf{ys}$u3_zqyFJJ-_+F9U56bxK7IatyjObqq)A1;zr9U6JL{q>-zpu8HaF4aH#ZEA
z_sP^$rMO(1yt(9f&_owk`S~(Z3HO{H-nz;?W8OrQiZ3rF?%cU^)-2G;$Og98b7m|I
zYz3Wu|KuunzTO#+-H*M?bwr#08|o(3uRr)L?6@=E=~Iuc#dXyE`}c3#wr$;Fy0`XL
zo8PNg_pxA_bf}83@Z=xX7h`vQtvTwOeXEx9?!!Nm|CIO7W8sV4l+bqlOJ>ROCIJyq
zQQui6nr3F-etmsy`j~O`i|Gc=i@C0C$-KO~aQmAoD;w*F7f+<|Ifk6Bob~6sf83mJ
zclM_re{it*Wgn}eho7IENydbY8w+!C)-*LS-F>(~t4UHSbdOc(t2_H7E$^hhS$m*v
z>Xp_A9U;yipXQvZlG@hPbZBPqaqIGTF`jRgzx?Ktu`sCq_9k?7Sg*8s-@Ax_sfT!a
zwOaS=sfl=T@l^KzOG~|_&GYt@ybO9;+9>dLN6fnR_T!7)`<0ZHSHE<Nei0vXTGFFM
zVAaQWcXnQ07i(>5x^>463!n2_h5Hr?I(h8!P?@wf`+DB(ZMo;?+5Y|geYv%ai}aZb
z9EvKgLBYYXv9X$6yQ;tE9d6@&$;W<?GkvF^lZRG-$kyubdEDIHpd+yNR*7CVK4fU(
z#BovI)b#4vvuihO_;9ROdTsRfW!5q(q6Iv?0!}51G_tO*i+%R&+1~2!eSLk$zVvuK
zkdRVr$@u*0s<yegx!wHw&(F?=t&O@U%Qs2SqRov%Q7S-WYw>eGZ*TAF>S~|!D`G&R
z3s-%6byYh&JbYn5#ph>dr|CvtlI5Ee1d(zMywWFYt)rv!>)TstQPI|yl7Cr1-dxeO
zr{?FSvuAx(gk0U-lk?p6w<Kq@2&^h`T3GP))zxX!rv3f(b$8)owq@yGtikS``uf$?
z)sxkHv+nLHt*P15*T;AF;esq?V-qKi&}ltAJo@_UZ``<%dwZLug~bI~zEx=j)1(z!
zwB|+V+}l&RdE&&t)6;b4+f+8al>Eyo<E-2gz`DEg^RxQ@e?eo7(cAN0zIt^@mT%QM
z(Cw1}ttZc&n`fGR?ds}q^Za{n-n{WCw^&hiI5MwAU=@Q<XUg|?ciGt3X3d)A>FGIR
z-o^cgL4o~u^^YGvI)&A>G&ExNR$V=Cpy6FX0B<A6U%#s5?d#@P6ejJj`)gxsyFPw@
z-%H6*1yHPhG1ut&^!<DO<z>8+Px^RyNm&-D+!b6ni!FU;V5w8!mCw)5Yj$~gdq4hI
zaq{Fzh6T%(EnB(Lvpi$PrkEWC3(Mc%Gcqz_a0r~TX3ZK86(d8#oz>s<dIN21etg*f
z|DSO8(e(54Obrbe8VZGg%9N{~5fL*cPoBJJk&<hmi0jN*v&2M1Uc7j*WZAMyvQn#b
z)<$nn+ZdrVam5pk$tO*WjTdXr`}60|(Qa{hyPArlU80YV^(u2Tt@8-zZQp!nh1JVT
zOSxyC)zi>;@Ufz`wUw2Xb<?Izi`{w`S?h&#&N9giN_(GTX}PoT@v+ca6&25SH#aX2
zT^$yq_x<E#^~cBikALaW5>3kpnY&{3>hc#C7Jd;_o7^e6>zK5C-JU&rYEEwV^70C`
zZ?miZwr1hN#DankGmX=0YHMF#Ss8qkaX}P^vB|2mWBu~$mn;eSl)KoiH)>0UW3tT(
zE*@E{5T%JC^}5Q+#$|7Ata@XssCe+)xx8m*W@cYs*W1x?VE4HdrxM!SuKN7{_t!hm
z^VG?cz4G?+?o|YcUbu7#lrKJB*AZKND$V!O<;%x=BrmTH*H>3x?%pr=_3iELS-)IY
zNTg+ioYm0KD0zQx?~m-RMQyy&ZuvGVvV3M3I0gm^GDw+ZT-a6mx+^zEN38tSmCWz&
z?y9<TadC<1Mx~^rT-lI#_{ozeox<vyB6h433pG7+VZ|@anJ=zgzn*_#0ps0=3syaN
z@<c_bGbJ@uJGpm8clYVHx3{OCoi%m-{P;aGYf?{7TOYrFU;Y1ot3p?M-4AtKq4jXN
zQ^?;hZ*B%FP26xZho7H+SF&TM3wXRn%gCtcNQa>4?N3)$1|L3r*b}s;2{eBB<3r-!
zs;}qf*+v_!?-PjS=vwrmk8z>BS;2z?8#ZjXb0@~s)O1&}<9>m<Ut6BL@cXSidZYe%
zaNWV({PJIBnf*Lo=Fz#NbzR<%quvY~($3DBsvRySCMLFDMO%A!{r|ceD~U-$7Hw{+
zt!-_5QYIeGku%>p)VZmMo!%6&%XiiV*%RkGJPS83^L=ky<x{R;vRhR4ZcKnl%;)*f
zyF^15?t8WW_+fR`$$GJD4E6Q@FTLCuk&>d4mZ73*;JkR#Jex`*pWXMSJhLic4|@Bw
z+q2#J`^!EjEtAJ5wm$juT4L8Ck%>Z<kvBhl_~1ENO;l8LYMrNs$gkhO^&>VcsQmn_
zq_sVO^UQ@6GFh99q&96TaSwcZ(#WOA=tuSxH=)iY_5c4J{r)3DXIfKJlZ(>B&p*Gu
zzJC4MwPWh7ikgj*Qi>t3IgTbVWL#YpdV5>0uaD1{uV16L=iOZsX{@u-DTwvIjG)t2
zzG$~LeUVGDtWNtIB&8HpWH}C372kdR%ACQFQ)T+>s3Xgg=B=}d(Tsg)W44O#L#HZ-
zqGrJTjlL&VwW)bYT8pXlNKPs;@;tg><s6%PN8PtdrDe1T{FOQ*wUog%U#$OhUnLtu
z>%-advQ<nnd<+aLHr$Tf^m^sAIcHWZ=VCCb(*5|C`)ZEG>0W_Fvp4OHQWl$9r)}*S
zQ`BSgc=n`Y48rDrBWqWdzfY0QJbXLavoZ3l?QgAu8A@`RhZBQ#ewX*#b5SWntKIe4
zyr}P=Hu$Vw5cw&8o!yMhQ}@ZUE#zvR?<nY0ayeu=qrmgjn&Jsw4{Z~+&oFJ+v0MFO
z_M_&ViWg3-a^23K`Ee%0&&<<vg!<G3UbU^Vns}1^5#x+M&p$HM2-H+q9-qBTo_Qfx
z8z^){IV`VkYM&*kxpBdVR4s-Br#aR!+{k+6`kqT8Q0)6umy*|#oGJ_xrt{ScIqco1
zXJq#GiBWzr<NTDKEDlgPx@)hGh5*CnnKO-Qb|2X#purG%wu*rvq;ZBjqjy`w`*-I8
zH6_iBC3LG_?=m^dXB`*ns2uVV)SR=8+7@N}>$Cx*N23c5kC`+>!<mzJckI#FeW*=m
zQ*-)t!?pgMM_oQPE6?y@@-W;!A$0$|5093;ncTVm{*+H&u6lh<QAtu`w6u&4=ia2r
zpb+(Kweha$dy3qSO_};A-Yn2*b@*c5#Xl9fH=p14<FGu(U%6L_w`QuZ^80nx?0m(e
zt)46*3@a)s%rBmP`pMz!9^q{NB{ME|E%y8H&W4RuSwK9<fl=`=gMxhkLqNrkl8I~E
z_q^$TG9zYJy+n%^Pv63e?^j-TStoap_v`N&DibeXTNb2j+<%YZ?6NnzM1QKQ*lg=Q
z9-f??{n78;*)Mk|Z#D1WpLK>`-@);r|L%u*ca;M!q?u2tiP-D);i)mpR3YUwO=FRW
z{a<G`T?i^GPh)UMY<y5V>%m`khK6Z-r{>K1o2c{R`VaT#YHjx>nrt$(61#TIe8t^~
zACBqXSNDtkS5%uIc>GSm`MkZgn`e4H)R%kl@>`D236sn74Az~TGmqtj*$h)bhKA~I
zhqdkWjsEc4RVb=_E#T>`<93d1D^Dx`_ts^ZTRelnQd^PZ`%m!XuUvJVOHSP=YFnDf
z((e1um>C4Rcv_h_Gd7xVe4Kse(dn7Fy$}9+8(IG^&hCvie8TeR$(2dpI2pV)b1pWR
zD$>Cyk#fuU(^?jWPv=aMp4*3;y#8k@;wL<FS<#cn%O}lu_u2bw|6@%pkqLX&i*FL?
zI#OA);`7svG~sDdQ(7D|v;uGbJTuK#<>X3{hNk08rKcAy`g>f3)qL}_J1Ptf8B0xT
zMU=Cg|K0y5b<tl$F81dKjSaITPp&xSdFe6BOd*DbBM*)w9$aJQ=5kIsa2X53yd6L4
zb|%GVtvqd<Zk--hc!VR==*)!|+dbC(e8Rv`v(MjyL7L%6;%a-1vMaZCxQn&uoV9v%
zJn^Qa?yFndQjHo_8c(wP%bj0deJ?w2ruyWWE-$3I{nwwcab2~3?%EXxR2_rcK7Tws
zYo${6&eG~FQ&Z|ETwLS4*Xp<9kFDa<4lnxkHhypHmq1QC{eKgsH}A<y&wXgUZ_YNU
z_m8&qyWZW({=##U@Fx9_Chie}P2c6i&Ipu5+`6zY?%EknhBK3Xq_XR;%U`eS@$|I$
z!+=%>hOW!kSDQ{=exo$vN%!~P^H=P?y;*d|Bd1AA4=k}$Thdyy>T+B+!;1Zdo(I|c
zHpMMoWUT!u#JyOgVbkoG?e~l$m6O#(p0?+nu{^qJd-}YbV@u7w|E_*_fAKS+{XReH
zWOV#|7AT6eylC5e=f!-Dp7Owiy{B$%`+S*uUyq~x%@x8-h5@bqmtxv$Jw!M3S{X{s
z)>z@W<My@X@Avs>oj6%pp>@4>x?O73uI27_+x_Ow{Or1XU57`)#-PjF<<EBR<8}~I
zSidc!e(_y)Da-AOKbFeP%XtyDfAzcLrtjX1{h2N}^WaNM&wnTH)#wTQIdjPScJaKA
z4s#-ZggMn|2`!J&)Tu3xNsnNz-s`Vzdr$x4pASv^n;6>UN_$>Elf5yu)UGoiW8J-X
zrSJCoaDB30C-?Q)4P~dZ4*%ZXo1<>y^+zZ7*{1$FaaRTfrhor@TUA8b^v>Mb^X~4+
z6s8nc|9MOd2bS|o^M$;8FRK{BozUj?zv0587pnr8woMAI5BsP5?ya_Kem8@|C$&H(
z2TqL%Q<L}9YAZc8Etg%FK67@pO7CU2xmEururRDz@W7Mpgz2>D=O;0U#9C|i*iWsu
z<ZSSC41CA5eY^1SCBKeOdUE|<#98TJg(W{27@CT=s~nh;zj&VbS0Sm=N$d$LwFP99
zmrnZ{@@vb3xjeeEYNwAdthl~Mgp(m4iGg9}HeW`Eb;b5i#Zvw?@A#ws-sL@)!<^7B
zVNS>FCNpp{uyk`W@Gxlo-ua_g;u8yl;$kj_N0S%UB&_N!P&-p~>e)U0CyA~6PAi%o
z>X`o$|HPx_u_^y~&Q2Bvhb>9cD;OCJy#K8`GUds$-%A&mB`(>tGkNjUS9hGcGOkSG
zbYSX;t8eP_3i;^!BLBAP6Ap&1UE5UuN--=9{-xR#U$gGT=lZ^Q(cAr(BepR<aD8}u
z*{a}ao<}-g_IDe<n*K#kh<TyjVZ|dht9~drx3ONi%^!VD^!}M?Z|`+-E>%D3T^fGQ
zkdcAmtAV=FV#n)tys=03EatBNAKrFqdCn5k%qF(iX_va>^QQ*hv+SB&Znjqb`w8>i
zKYIeTw0r{3?N$;h+U|e4*7e^8*_KtDpdlD%uA>YNopy{20qZL>Yr<{^zWVl-V{Kp5
z?zCbNRuP5?Nel~)s4xVqb-OCNZsz6$r>YdOiyJyM-YTCk4Ze`dz_2{o`-HE?pAU{o
zY-w|M9epd_U#Plssd2I(E5joHN8<SmDN}!KXJ@#Od#pj}-reO2ohPb~P0m~tE3)g$
z;jh<uQd1wU{d}}?y5ey*1}+DsC&CL>*{m>T*kF;vV5w!@t}x4d#`8l8TSWG5%Z$7>
zUE>6!<}!wc391Y#J4FsMF(hPcn{;)m`i%Ix6b2{9?u(Z?7!_<p)KpR<Ki#anen0e4
z{vNXx`S(j+E@@Niy-{R;Piyg=;*Pecx6<AB_vG88H)(x+FtPh{x3qkSv}s_*ny8<k
zS{HOOg~N>AtOqOj8Myd9o3xcPG&D6QGBF0Het4rMt!DU6PAlVy#(6fk%YIsY@3eF+
z4rHy$J2&y7^qquzOuWIO|IU79wVw8LX^<3m?W?u7mahBx>+Y*qugl`v8yv4?_55o1
z_D1ltX;d-Cgz)%n4QGGeS$XNpPA;Db<ujJ5Sblq#Gbu*>eA{X>lOp3MZ8<s>+Kdf<
zPV=iYrO(fs{A8+zVa<;U9YKaE>G@&T69w0Z@rIq9+Ier{@1E5c&Mxk}FKC)8HsMv5
z7QfxnqSUC{uVp^0yZqzp@8C1L%6c2GhSYxPu!<{VIkBp;@1cFI_l79bI~I|tH|jqz
z6zyH6@yfVmr^5Zq!PQ$#Ss5nHzO$X_V>8E^4?n}-pWF9o)>L%{wm%iYCyLHzK3;H4
zENyPp@h~+F$B7&(4qZq+Ss2;1HY(Kd3P-3})1nXmB!3<I^z^hb|GW9O_MChle<<gU
zRM#c%u9f?aXWX3i@#D-Nr(fwhZRXq@GV}6$#VrxLUQK$EziKT9U%;M6)26PFZ*j~p
zOLXA-*~ql&f%)MClRT#Up#7S!l|*LtTBH<;N-kWwGrP6m^N~GMK56g&bac@=L(V&g
z4&DF%Fl$L=@v|o~wkBS?-<|hUDv4&v39FI)m2i+TA;RRYZM^f`NP*h81=pkJPM)?a
z)A8KjJNf4r85oYfWRH5ee~<mS=xX<dX<~|}cKk0od+E=2FQd}W-QJr|g}QkCx?3Nn
zA`tSzzcb+IVMCKu3{Pe+xA#2Hpx{}jrMGp*tW$n}Pk-TI2$^QFGsn>6>(UoL-Cyq9
z5o~0=U&Z#npTs1imy!%Cm_GTcX?(r(jXP)0@n}2sXYMa0|Jr)KS=~9iGAZDKad^s7
zwcUTF7w_Mr-YUqu)yK4AW4KDlz1fx0#l@a8+1^Iw+F#6ZYM;L(y=9f)nF}0$cQ%E$
z<`=8V3*TFRQ0m1g#hnvt6xtTXxqmXy`R{I>xTP>4nR~PTeMW`{_bMzC9Ufhros_WZ
zwr|$Eq|m7cl|noXoRz;kUy;7k+vf7KHKL#DYW03u=j7@#On7Y(6uI)xcFm)5vI`zO
zJRZKp=f3K$%fFZ%5?}0M&pyw^7;;N0By$Z%C?BYKtLc#H$>5;#>rd`y&RyF!oqzUA
z)N!tux9w@m_KX#&s{$p8otYH6>Q`piuFA6(V>s}4yGX+aW;+Ivt1J=xpe{hsb2sC5
zt<8-qldqRZz0S4R{?v;-?O5v{@kE<-tDTago1ZQ^E-hbM&F4MWci+AeW`>68+AKHQ
zjQLil)=bF0)p4?RqL6^!_x_du)&o4f0*j=-yZu_%o`1idDIig^(ZXK8Fe2ik_WHZO
z_Wdimw*9%op-0_|9vz=la9(Zayva|rH{afx_Th5(q)RV;ND8c~n6XgM>C5Cj|5+Fq
zDj&&TXPIK@K4r@HuS;vV6#NVR`#p8-2)KIq_XQ5cmI)?@K*LoWp30zL0C9g`SfRt#
zTxjvOR!{Zn{Cg@>TzmpI^?cg4c#@ROMveU|I^JmrI9&yeD@<!|Z(p`-*%bQ?wuXiu
zKRrDiyxec8^@R8tt5$oLb9|XRajNBvX<tJ=J&rZ3<JX+np(12?ZU4S(PEXEwjX%fT
z1XeNaC}vwVO)qxWmcqx!yr=73l;!xf_0OL_Cr+HG&wqYzZ}oEj`Ek2SUhX$vx_EJN
zO3Ih_ZxupTg2t;>#dvvn{r&az_xG}t)YRmpB&{WyEvr~p?z`S0;KXq?3KRewikU5v
zQi?4CliUWFdt{L{L(9K2%l+qTbe%eXUcA#q-oB27ft8ijY_@NC42NRKK?CQ-uby04
z8O+SaGhymfR)(%cudc3E7vY+1miz0|Q}12Q&K%&LT&Ox|q+;_;rp5(NIJ%EYSrjPr
zDk`=Fs2==%VTIMo=<Vw^ZY+GMylK;><YPSx)umblR{6|W7-(voc!<T-_2{I0Q&Ur3
z9-fRIcEuJgPS7Zeg{5Vu%c4i?Z{}>9JzIM3M+J~!JiV(VQ&XRwn`<pz?<;WR#6;zn
zy)YerL4z=!o}3J`OtY^o@e~G)b1(F2l#~j!|Mu-$(#9EQ)BI-`G`@=va9XhmGz!!|
zXU?3pYu7p`NLZC<+})_q5|DbZDkDTQC^)#CUtUgL-e15ab&LK)500jcD=Pw1P53~Q
z)~~OxKW=E{)ZxbB8d!3?A!}Cfa=%__^Som{l8=w|F1DVw|I4SRrwx;jNk~e*y}Nt*
z;fEJw{nm;}OIw$`2nY?mdg#z0|M_;N85b1n?eCvEm-qeM-IDgUfTxEHO;&j*P0Tnq
z$8zDqg~$73jm^!MThG~leVOm<*VosdpJ90T%9SraK0b~p@>Le={`%_b>Ph*=#>R{c
z;T(%@F}!zP%=P)%+1Y+`t!%5my}7nF`l^+!_~hx+?{Ce%erBfe?y|S9o<Dalx7a$p
zv(xi%8?U!~U{KI3%VM?Ij|w5F2F{DS3ctOHEH3_B_xD$8OUsItD=*6WwMyF6{5aYz
zZk~6?qUy_wLx-Gp9bUQgSf6b4o>(tVYisLWO^)Cpwf|l^V&~`CRzEt@>E0(Z(=0bi
zRhM^h;OlE^^Y83%oU~-;&dQ)5p}P;SxGi?;O){E!G~QUE=U_AY^qTGf-Zn|8zmvaw
zDUp_*-O|$X;>8Q_{HtGTU_ii)-R1A6Xa-xAzXRP5w(Ic9(vs3r?`v<hH8ghY+I2~q
z-zgL{$5Sn?7h_;*`u67L=bggpCg6FrOTAL2ukP$DULCf!sJK{LOKXw!oLi}Pb`%yB
z6)`BNt6yK}%)aZe;|i^WHn;j_m6Ja|Ki|E3H#-Z<nK_o5pPkM5-?49BT}X&X|MATm
zHhB2?Ewi2zmGa}mLlM`F5o-b$yIq~P-#p|r=rFK1cXyZH-BsG%+uPgMw{6?DW!7_6
zEnBhT#q;OpiHBISvtJ)>=f5oLmwR!6W3yTAt&XFQE-rSzEX(Z_$}&Hq>ED^D+Tm_}
zGB4k~vr9iG!?rlA{!sOg4-31yyVtILyDj(jjU2PP4==3Wu%Y1WEYp5@`*lmc&9|+d
zHf4%Q!hr^_toXRNeLI-;O}gd&!auRiZNInDM4i}OSFT?#|Nrl=nVDHhyRzt~&!5$O
zXMOqe^Yi25{o6AxHoYr&AtEPdSNSO=JNxy%+TUSu`mC(1U0q#MuT;3X9c$y2o-=oD
z_}ZwYYu3bg?p65m+rWA8{Yf)tTISp^=yZAX{(b!RyjakBf#2WWM(->F<w$mZIiGSJ
zlZxNp-aa`wdGe%5d-m>K{o8%}ojZ42yTxL6m1Lftrn~iXys+ogC|mE&y5N+QC7qp~
zzrVdbKgY89<xkLxE9=TnPwwn2R`;EyBEt2OZ~3c^qmO!}&9`M=KX>7RftlI1U7)$+
z@bLRPi_;w(4t#ogy87#@ub_fR*$FgK_QhDJGv()}r-$45?{7{&f2fr^`)BX-`5D*O
z#g>$mxVgEt^T~SIyX<1??(8%)HQihA@X*%m>(9>44u3W?{%X|kZ*TMO?z*}%Se=`@
zd#ZN$wN?JH9;+Tm>-}2z=FOY9y;ZUMYJNUDJNxdg(#6&?nadX~5)u(Hsrgah?R|Q~
z27_IPSAt4V8T-0F2O638?%Crv-wrf5xJ!9eg^pPEuP-kzUc6{%Xz1qVc71*Pa%-g!
z>BKg-r`=s$pt0xOWpB^TwGQ83SG()*N;5UTITBJ*ubw_NEqxU-(Zi+OLR4QtVS(>#
zGZx1?d#kVCygBn-fmZlpw_YbFr)hezRz*)v1TJ<<O;5ipE4V5aG+3(96}7i&>*md$
z&&{=-t`phB@9-~p*|KF9FJ4?9xA)bJjgMbnU;mPC`5_f`_43!(az8&id%REf_NLVC
zt}d>-f}s=l)&9P>w>sR{*Ec@C{{6kZJ9deMN}jp!LfX>Ovi8>(%i?Ee&dxU9UH;y$
z+~VrgRjXcIT<p%yCzFwwC@3fAS8nmY>-=;5s4XW>omy4?{vKrc*+T0@tByS^xV9#;
z_}7=r`1t#vt5xI9nma8uoURw^RnFrPb!S)U>gw<B%*@Pwe}BK;R(Z*(<jl;>)Kt^D
zKR>>F`I2{cSLo%^vg!Kq+qQ2{H;_<NRJ{6q`3iy3*VjO|!+NL`6&IgAb?ViPjf;DF
zc<wqboW=8cp&$c;YoN4w-klwVkJrWSuKNDYx7@<iD=+WekB^V-Yk!Fd2`yT?*0<cE
z{=whh-!Cuo?e6WpyCpOD{5;#2eC}N-si|jYo4^0|_O^A|n}ULZ1=cbl$ye|7A6{|f
zc)$Gbudl_Wq@+OeTsse~T)kS}y6nuod-u+qdGqJz=aTl%(JDeG`(&+eZOgsAB5*Ny
z)-N?9GxO)KU+->h1*HmRHlB*#-*iFqKb+}1vv*sUzq@hs=GOfC_s*a9_x0_4SCFOF
zq+nI@A|Wv`(8l`K*6i@idD@_PfCfh9Ns}fuH8stdJGW5jyZbDY%tx<Y{d#n?yYTTb
zLkS*kZtl!~>8@7=&lO$Iir$tJxjBvZ+?CLo{PK1?cJCG!6=h{-H_y1x@UGz0sq^RK
zcN9FlxY&KVemrP#3MduE?Wr)#y|sm3-tNua-PbQ&n)I$9;B4dkA6YB?=i6CYS+TLQ
z#_q3^y<9vMbbU`m#EiLf=dN7&a;f+97ktm9kN3-8zkK=kmzTjcH8#th+h5vK`T4_#
z0)2gbclYC*!fG|YzI49p2;gnpd}nK5P0gOUb7Oaxz2%d$S+Q_o;KttN_igw7zR|k7
zd|pA6)U~Pqe+e4=J*WBqLU#GO<A$@g=B_yM>zUN2<SQSYXN%3xe3Y<Q)4Rq_!(;B{
zb;a}Uo@lcyY-i?wV=*;3eff8O{%ImdTHbJV?fP$FWi>bBN>sFDgO5eE!H++^H_Q{(
ztUs%?lk+)$n@XWh^!1-bobS(Uy6I^p8@)R8$!8tTrVZP+nVFiFmX@A9dv@!_jSFkD
zzFb`$uI$#s!OeYpbNcxsox-9bA}@NE&$@K|y1Z>wNlMC+<;&CeR(<8+<$c-55n650
zwytzr{{4H){pU~B4$r%@W24c$rFSlDnP2?*Pwnq*I`(DzuRMIi@croTZ*K!O7H`W?
z2>&LvY?fbbpv$T4{NEEc`ri4Uy-aKQipAGks)YXkSwFv5r=sF>C&P`mZ>z=M-r1_%
zb@}9`#KQG%vb^7ZE-Tr-J%#(yp>MJ`tkQokX8L&j^`}X-&Yz>Q3jb}pKmX;8jgQaH
zHqXDmFE=~;_rJftPrZ0l-r=(7>gw>>vuBsTxbX1X+uLDbVI{6X#w%8=$ho`A)WV|T
z@v+`*+qRuNd9uX4WtI7v3tKs+pDryc`}X;BcTbOuL4w26(kl+zEg4ka+En!auAJ%|
zcW9?W7mJ(t>1*-rZ<yq!pMEM@zjkS;e8b_S#)i|%-Ta2H`DeaLTyivN(^ctf`4Ous
zC!K!Ebz=Q}E)RyF15Y_0%wA@{{f}DGZ;tcPHVhA5_}wv$`&5#BWp&e|=|=D4-Wwcu
z?ssO0zI`z{D4zS~o}jx+?%Ph!vR|Uw-f(K+(xs)ZuB_~nwVpL&hKq|!NRYn0pr~kS
zN{Wam2$&`x<B1Ur3p{h?%%Vk$s=mF+OikTd@iD2<;lt}}0WJQt7q=e$`}<o>O)V=c
zOHfeIcedHuEBjcO`mR6P`E1hbc0D^r28MetbIZf?X0spoUF22Q^8L&Iupe*ZY;Rr2
z^c9*e#N6;?*5b_QU~O;Z!zZub6H}RU=XJ&fvj-2CKN9QG*RH6}p834#ZnW6UZPL|0
zLmi*py&TLoYlf8M4y)*h$XR*&r?D}dKlkoy%%<9_SNe_x^Y5*d-kbPJ(d&h6cI^ji
zp;u)E5-tYQSnsGXGJHD!W{2&v?f<4v_@UGg<ew>chtuZatBkOf{Whrv5}%)+zrVkJ
ze|PukOG~{~g*;QgeGB#TJ9lGaay#TYy9*ZrWM${Bdb#~ug3-*$>i&J5ot0l+Ol)jC
zI8{6RMIT3~_l$+vt#@~qe}8c?`Q#+k=<Ru_n{Q6NC$Ab}^y1a4b31pviqPV`GP@(#
zs(IJ80KH0E+oao9nvdPS<~Mt@S%vfM4T*7yP74?k{ym)CFRbTpzS}hS=FDY#f0w99
zd^pRSeUJ0^|10eM^KDW$NtfndTjjlKS*+EnO%dBpm(Ta$n3wlfT1@-I&Zj2tze%j`
zj58MRSijEx*Rr2qp5C2V#a(_@E@(dIfh?n(O8bW!Z4GydOs&h^{foo+t31!d&qB<P
zc;EJD{ZOmjxvS)5(dTDpPn<X*XIqt$ni`rFFXH<1*RO4vm)j0MJaYW__xJbr2kl<{
zRLUenKtyCq-rZe=kB{+6o9P%EPkz@CVA?FXt4&)&L&B=$#k;$^K~?CJCn2GCyIhi<
z_AyTndUWB#ZT);XOWTqh_r4uZ+%`S<xO8%rv5zGKgHQ79z?7IX(Si(%y?2Lc9a!_;
z;M&xl-P1X<zLxHnJ^1Oqi`R+%m7Le})L-Y{+K{?PQhEkM!o7{W(hW;FLaH3UtyX9F
zVDm1mhAaE-uCse*X?3-*G6Wc%zp?6fK=G@e8?`@{`?c@(ee-9rzxK@X^GnO5FHMS#
z48D_kWp()bH#awniHW_vwRQELUOr)AVe9gDZyrA`{`BPJ*VoszH8n$n+*fRg+go+?
z_;K@`8y5~7NVvW(Hq<P*DU0FE1=E{{+xdNcd@N+nZOy(OwIw4k^{<g*o6tm;$9~<r
zPQLy4!d&&}wsPgR2o?sJ1V#q$g>uH0MmsB>MJ-LNWthOo@SstG!NGO4ox!WsHS=wb
zCMiT&SnzGwzWx2h#qQUa8SOmU8U7=+Qe076h?^s9`oC}AzA>}ixcz;F_Jc0F_AjlA
z4lePOU8f^&o#%hqaarl9vX3(rtUet1ka#R~=W6lhgAqFl7WVg_zp&6bd|gcBMt&^~
zEv;GB<?DKT&n|FmR#H~(?dj3lQ253B)2C0Tr|a)8c<2-m5YWyiD`!*TQ0}s#>Htq{
z^4r_n&tJP%c5e^p=ECWdf9L%yNj+6P|CI7khB<MImoXgJ(pe_MFs(<&W8KoH3=ErR
z#>E%r9P!p;WzbjZKW$-YqUOR7V6*(MPLlBL#f!xnKJ0Xvc9e}lGSZbHq41T&q0<}Z
z?aQ<&JUILHyv^skZt6%fOh}zNXYsnLGf+2jPZwZf0BzlV_uM7#PCg664ZRn?->$n}
zr0VPFnB=f0QNMZD)PJ*o^{rj|_UvqP{g@pWR)??Wm$wV?nEm|R-0aKC&YqsGpPZcB
zJ-;qg@$9r|b>F{O<>%+;-*?~nD8KK%<<qxsAMed8&tcpD#`MYY6%%ad`FT&SSZdeJ
z5%heEsm;8Y3DafXKHqyM_a0~5&C~xn6x32~Tgl7I=f4jwztmmrYw`O1+qZjlRooaD
zRLmc~xtF_q-n3mU<!Lv!1jk<!3e&N*-RnJFuUFQ3Tl)EVmzVi!Ypj?O7vS3@xhrj!
zLE<6M>bm%SHYq12XuY%g{O)C|F<)MBeATPGWnGc-<+nX;eP4HMewTZvN`Rr}TBZFd
zQS&#J#ot3_KAijP$4{O3KO4T^w45utwKq2Ej=Wj+-1ECu-v0bNc2>eXfo<7)U23lW
z|M2#<IRD%}h54~Iwf%-R1+JT|@9ov;wf<sjowr%q_+e7T8iQ|#ep+{y`?b4II<FV^
zc>C?QSM5_~cCT!FxuKek+f5=vx=e9$*sX|nXMP$@NHfjZzbLwzefG8ZX0^@APT$;*
zbv|0;H?ODWlh(I8pp%%UZtME}{MGgK_5c1<Iy)aOdwUDi^^TwA<?k<VQLte7^5@^)
z-sYFF2-u#NTKnvP(C+&#0!LN^E`Id*adC0+*Dqh3oSn5-Du)<92)55wtE~LFWQhuB
z-6iu+$)E#|IUOQ@zTUmC;kYP6^vutfx8^KqI4-<l%GCOzpqbNHZR85-?rBV5nshQv
z?0VkDqcwUbG&ip<nzUPHv3vi!dwYNX`1tt$qo_$Lpe><US66WgtAYBpT+`S;&6hPS
zj$-Z6YtPFw4^(4N(0{n#YX|Rv7e#NrmF?A+FK4k?*N`Ii|MG|Vt+OoM+m7bu>u98C
z7JJp*Rbdc*5V_&{gU4dq`*@!+FieQi>p!dc(_E2NWil_5p8URlTRAH3=>LC_uVi-X
ze8|qa{5e(*X;=7eciOE!l>Ey)l%?TJM=$sNueZz(E!Z(vc%7JgE`M9iKe0>7VU{sw
zz44b<bNzd}c=2LeTie;D+1rYrpHuUl6;ju};@3gLlCwurjGjJy`m*nl_>3ICmtS}o
za{4~$U-~@#bX{H0%xO3Gd!1)E@aNCH^C`wN1GGdhyjUMzboA)anKNh3wX6O0>gsCH
z+T5<Izax^qKVP$UZE{kQj;`+6GiR=?iBuNqT(u-RWNt#6+lpr=PM@9)S+upxXJ*m2
zH<7WicUQfZ_I~qkPEXI7Gc%1BFIu!Net%w0PR-|MXSH5Sg>uhWD7Z?|$=UhqmoF|#
z7eV{g`DCM>{!Q+@8-MBIMa#lRN19l<+xTRy%HG`g^XHEiAA_gvwe|jv9HH!bdV1=9
zb2e<;xNzzBOR8nsHorFB((lPTB4FRVNwrLw$6~(sn|n378<WMS-{`ChN=gr^QMdjc
zTd7)h=aJCvN2@o-rKZ&Kz2tUn_MB|h6KVHNEBnaOPJ7*rvmdx!f7{E*%y#3}Eh`I)
zmbSJ&N#nL<8WTzm8k#JUw^zQFK6iV6fB*OEt7hE)w$$!<`18;t(cUQu1`^AbEi3uj
zr&{*k&(qU0K0aPY?EkNj5bFmLQWyQ#HMSP77dKN5tCZVw>;4!1`%3$-JZE4i2$p|#
z)i7sC#JNoHahj{s!@@*;b;Dzl#E*NdeZ7Cheo!tyWTaa5b(&A!?c!(ew(LA?eRif@
z;K!Vx_nTi^Yk!&?6SYR#EPJl|;qs41_IPhTD}2ao&GZi4zx?KJm)CwQVfm@^{mb5U
zhnd6gNNij%{b^X!&F2MItkRq+D%O0EEBx12ckk^7k57^BO#gkl8~0aTub4BEZBw(k
zo35FFdWquowOcmUh!!R171}x%cFql&DYW~7@WjGJ7Dp$mzy4yn=!MQk=M^y#xuM3I
zT3Y)mJ}$Ddy*BZGN!QN@d<O#$-(E4La%oBJx#!9;>mJ;GdQF8v_|svV-D&;%4DM~=
zW0+u*CwDUJ|E3qawN__noqBR|^6l;U|9@W%li_=Qb#-`G*V9^uiBb_s-;4kM{T_e3
zPxkKJyDxc}Jvn8Zl`{ha&b1%?!}0#8<NK1RLuVI=Tu%7Ev+C3#iTzLA?`5r8Dp#}Q
z%1TA|_1ig<-*1xr%kN%#-g=ux(|M2RV;^T-^{lw{defZ_F@L_#Z1SJqeAwA_EZND_
z>%6Jx<g=QS)An0`sGk1!m5AB%>z`GZdc9$oA@ekTPG3OiVV>S4;f1w+XZBu{nUS<-
z%VC4{Vdd9Mi|)@7{8kX!E%SPtS@K=KH7^!U{$KJ_FTwG!-(s<CcXT6n*Hw6*R6P85
z?Y7v0H%3bm{)*}~6vZVu?0NV~Yt14@j)nE@+D9i`(OPczT-BT(bcC76!gZN^EnQme
zC-qjI7wl$eNn&I;Uefq?<;k*l_rl*Sv(ma8?%FKi^q<MtWRj4N>!aB`r_Otv3~Dio
zPB>})0&)Ub(Ei-&kYzch*M8Lrb#OBLYJS%K>&*F^+jq1-on<nqt#_4x(~J9wZEiu0
zr%rb)Q91DI(~Z=)SMTo4o*MmKJ9z%(mZ@JYZYC9XMC(6~oUL|4?0%-s%JYm2iGd3n
zejlrUc=4BPd%NrVg-0Cai|1X4b8X(&u3E<NSLj~5+ZW~H$=lZ7;$t}e+^;h;GMtg?
z{G85&t??4JCjxEX&;BK|VBYa#Ztr9y0&5oDwfVo1>w4Gg;#+5>Ze4IXR=g$m&6Bg+
z_T0UE{p7bblT$1!djCBB9O0_my5R`l!uUfxy)TZ3T%Wz|?T;-%?$_4+{o|^>U6((m
z!iI5&RxyKu?V}~9c=jV5dv!14|C-}{o4fq4&8~iZPQ<YNFxv~mhy#x~6E+yJv)s*#
zIQ>v$yU3-CtLwB6229aU|NJR_$_;QoD{zmn+l@nxHy7)Rq`%mtT2_93=`H@uPWf{I
zTU#ZiJa-3$nlxEAy06pJyi<4LVv?xs&cbOIH_!jI{dv*8H;F&KYMF&y^w8JY==?(6
zz<F_2i*3Pg&6^K|!};ZzOk=H&bN1<ed{^Y#tXjq~^#D)rD$%}e&8wF)-wj;g6&#cF
z{QuS?c|~Wu*A*3Z&V6@X$U3YxR!x4vjw5{{+zcVU;y2dM&zWwSca%$Kqq8aNx~Fdc
zwUj^Jit;xP5BK*!zq3x=xn|qkhyQYeUzYdzOIlnwS$b}zb-tIp`cBP6yVG}{&HWL0
zy6Mi8g#7&~_vYIin)XC(&fVQ_uUwu#f6By)%Dda`1JkbT`{~&Ke}VEB=ZK{C3+F+L
zESh>g&b^oX;fb5_v^Sq`Z%aKJvXx!B{Qil*ySE>0Qg^Sa+*2B9wY>k@Y1X{jb9*06
z-*Y4+FQ)y!%D<f(?o9hBx$)ocvNa!ie>LkivScRy=F6G8`ETGse?6fi^KG`2AAY0E
z&+YY9yW`vU!0eBUWqB^j9}u!`spCuEdGY?~YN1E_o_=bv4R?#U#?0{GN_kAxtFKpz
znHbJ}KV9f4G{a}o@yDm9>OBAD_GapK_WW~34+W;>Zr@{jTYYs0zXCgh!>1id)2jZq
z*>gFsYivE9^VQmM^*&24u8NuokN+g7vuxY8__&JJw+9#J|E<Z{%$Usk<%v@}r`O*z
zos1vbR+m4H{=fea$TggxJqjxguQyy=+FkgvB((mHkh|prw+emDy)x6wtDer-dXVoN
zUwYqMMmO>O+R>5^Hy_K{c_-iVL00->^@H4P>vAjpoy*CMWMELZq4nYCUQ_jlKXwFe
z<O_bCbFZc|HT&b?0G9hat@rQP8031ta;QFbYi(p*__nAi#`;_gHG&Q&-J<mM^WK{$
z`@as%7Mp1+ZD?8*I>EpG@Ggdcw2rxvF$bE<&HqbIyP^2Sd&a_x@u%yhWY1?<ZoIc$
zS+s9gRn?JIn@$&+^BTlUU%Hj}>y~P9+19s{e(X!=xm~k;t>8@=RfYt?PoJCI>io_#
z{7Dc06Ctd0YlYLn<nMx~pWUs>Sm+aRYL7(SukVjeb#I<OMWQahR`}WRH@dMx_977(
zEB}33lxDk}rM}8kN9CW>x`e+km1}2B3tqyl&~c1uo>rr383*INcDMfy3mX%;JwG+`
zHyOR&eCp=MN4y&v64Npluqv!xcbY|vA;T(HMrPlG61|l9`<GnYdv^NTHAh!w&+Y#i
zBeR|};KQ$X#gXT)onm3wkn+&{=~A17EhZOJm>O<<apz3Fwc_8i)?d!ge3$yktM2-u
zV{cnO@6oo)$C>7>IwH5@?-eiQk5>=3_qEtxcx9HjZL{HBMa2-;0-oM~Qo-)$1ytN8
z#ee=1HLdQM5~Oc7)BI4w=Ss_Wvmfm@-YC()yGZ}J={C6=t~M!ucj@#tzMD0_KlQiY
zjhtG=BT?O~3KKS@IDVSs-DD)pQ1|_2Z_P%BPcbS#Y8YqEJ{o-e*IvfZgY8d5f~MGf
zK5~NZTyDs|`oG1D{P*szNs35n&(fB0UcCSMj?*0CO*;!dd=q*1{PxzSHSXSb%}#77
z(AX@qUDKV7Awq{C;Uoh`w}1E*>A&%_jT`-|Z<;VOH2g6*e^38cpGi}B{(ZGy=kES;
zJGAh~!s(AMGd@VpV{;WPI4R$9D2egwN{jESGhPNPij{88d(KtCd9b_uSpA=arbR{!
z3elewLkrTLs?9E*=y-YUbpHEOzb#cZvY&o&&RHgnd5?}U%>%hJKooRI731!ccV+}{
zX1`Os#i;D-9P8_sGk@&S_LENARz72E^{O);{EGk7Ow+Hs@%zo3bDZjj9;p;oHz`ee
ztRLee_V$b2zdjZQgKhE`cfEh|$aK5m?sT252lL}Bwng{f+gr`wmzkZvAiX{z@V{@K
z?QJWmUGHmrG?w3~neek#BqcBCz%tpwkW=S+>pXWR?M$m}4&$-in--HEeERMDs&nra
z2m3uqDRVjT?}SBQ%$1XqC;GE6d}RFmEH>lDx`}Pu_DQPp6@9S!($_NQh<0?@JN5ao
zcZ7wep7xECxh}z70jg%SoCBWPh>M3-cXf5?$L$FS3i|bRrD&tbhYFj<8ArqPw_at`
zeDIMapz3q;Cw<kK_A5Gm-QtsfVv)m;06DkZC_Y|<adqR;%*{o5!W<O`9&MPZVs82`
zQR+m^UJsQ?*_RJ`xW?Qu)Kq9Z`)BSSr%yW`P5SB`n(*n#$+x$+%iGue$)9)rVL{y9
zs--I;?&<l9s=q%e_*!wb+MAt5M|a(up8S5BpRM_++rEF#d%scIbA9!$^F|K^9G2>u
zzPYUO<Jj|3W|e;Jw7s^L`9BJ~qmQ>u`)}@IE1$jP)az~Ww<rDPv)A9q%%I@EBT8ng
zt%T8`2kQ39|8EL@PCPWDda?cY+2!{)Ufy=Ku&!$A{<i=7|Nnb^W8>o8yQ@>nTUN2k
zI4|ZpkWxJ5>eZ|JeUJH=Jr92=$xt_OUia;X8)hCo-zx+<p?u1;KW*At|BcLc|LBbh
znJ|0OpR=wfPv&zUy)wOT=KksP7#kv|zKqQJyW)r+>kh$tm;XmU++O`P!+E)>XN;Cz
zL2Kj0i+56sdv$bmZ~OM06*~Ug&)mRZM*SN`1{=<}3)}9ln<kt-TjcrmeD61GvvjQG
zp4g@wRb6zj-}&ddBY6)Vm?j>4-NU1PJne7Bl$dQt&K)?AaN*a7mX2PwTv>nHKQ_+I
zs%0Kt54;RR=5BfQ>dq^E{fXHLj~-<OX<oeI7Ln8*aP^>}iPMYkiJh`eq3p&Wrs(t=
zvEFrOe}DTiUC5f@$EL?euM1gg1SdV;Uw1d{;sH5(y^Y?#H)Z_q_j*&b-)8RAGiQIj
zUHbH>YS}|$f0bube_Ktvp(q7vY&|@Cw>aC@w!ikb-b~^84QoHny0Yo;*8PoFjYL_!
z-w3QSs*soRY(2KNRIy0z{NvdvXV$HLZMSh=73-P{f-b**ukd~G`~2Gl=T*yi&Ce8V
zG1~Ux*#cHg%{_hI=4WzN`@K8&C+&E5i?DQ;_nUvlIy+|Q?0R0i=Y8DwUp}DB_G_)v
zOCjqHiRhZ$I}h(!oL5!FE_PS)@wMX3dd*%F4RlNL>`s)J=|(?sd!{CPZ~O8cNAjLc
zxmb1f7u#A7iH#X|UYcI0JGE}Eb6r4^c$JcopUCoxPbqWv?cM##o@<8rd#A(R*`_I9
z&uVPk-W-1T^KAE%cDpy#KCk;U^W(e;ZF+k{#E+ZWZIbnVBVfDX)~d8skFr*G9k90A
zFgfLI??Wq=_p_@cC$*gKUR$|yr;5Al&(q#}?!-J;F=dYCgCbU+KBMJ(!dGvITE^rF
zx<5c?<7%FW+*Yr{h9;B5TC_Iit$qILMg)Vz-ght7#FZW{TNCwUcKMTS@zEbIZ%q~#
zvb9?N_KWGw`pPv+oo;M;ng5l|qfaj=R?>1=XyAuqx&Dq{9UT^&Jf+;t_M+x<mUQTv
zX)Aer%-m<jnh8(8_3_d2>HqvX?@nLeddK|a%ch|2V=ZsmW=(q<ckG``Kw+os3B=+1
zTaUb(?pVjyvbe|iPKViyg%d?48p=(6uwCCS;o6Ly-8oisUgX^6t69I1f$P$u=AhrU
zpCtae>1G}{dVKMh*K$^h`+v==&6fUpIrGWv&(_yD*XW(uP<ivg(%*|*nb%C4UMIRb
zb(OWwM(3?!>qML{IsNGT^eUC<fJw1jzMrjs&HUqGzibj$uXrz4drav}dgXjkbJhG!
zl`_3@XY;M3x7tovaOrf2;loR<&$L?k*SS7jVV!$#n}hpizm?&p8=}}0Lwp}fNO=k-
z7H0Aph;llK?c=NZJa6u{vVA2lZe6=v`dNmbdGd<)QU&WG4}6o1b3W&L({4k<rMye~
z+G{^PTJWbQW@ga(cej^GH(oX3Son(L_yv!sfb*(OhQbU#B>mSLwwLBER#9hWI50{1
zQl4Dmwk;V8{ZjV&F^eu#_<pakxYE`(EdIXzx0mytuiZ5(HqPR0Q4ybJh4`DDA3VP`
zuV4R5#U){@kFQf*568{NFSmRNkB;J0nil0}{q3{u8i7e=GZs#qlCbs9`qyUS{_^(k
z|EJykVrw3&ZT>RhK;?uZvYfrE7vJx$*>&&Kg}=KN$VqOld;FBYe>(p?F(I)cqs&j+
z?x~@lES$gM)7PuKyRr+MSIuf!d2NxMveMJ9{c-0#dwib;GOXG4Ebkfbsz6<4r>6(9
zR?k`~6&B4{v-0ZCD5I}$y_3w^c@-XT-jFd9nI67zs&^RQ=Zctw!hM?ilC-rquigbp
z$dy~KzE`M>k=roU^3A08#h*R;s!r}_*HoC9BT(@3!`C^VqBLtm4{pBj?Ob}_-EWtF
zzdP4zp`AD9?mA8L?=v-~@2qz(D*n76WMy38Uh)3ZleORX_qxF{h0@X6iBo^~olL2(
z<Np6CK0#Z1`nf$aeMK*0+4t{>>NZ;<)aeo#8Cl{UsPuo)ca!z9j=Ps_F^{ZM5?A<p
z?qOr|^R|{8Q&R(uU9)Fae*ZG1QEizU+q!dNVbhk~UcdC|5qWb11A}>08sZB24?7wg
z4{qhDo4ML+YrMo$<GTk6in<zV%8GV3HXeL<WR3Jc>BlOC$;<O?%KyH()OhUiA9)6b
zslPMc9yqw$*-321*5docQ)hq9Sp8f<Iat}>U)@{V`}5O>d*uHxg!V65@#RZN(*aZA
z7sh8Ua9kDeo~HBi*4EeaY^zm-IPY$B_;s}O^QT``=Vw}^A7y*^@x%B0<mz2(<6i7P
zdpP;y)xUDt$10Y*E_=D+RMJho`?X4n6}z_PN32<NbHM}4Yavz}7^b|=xYc(*{cv5#
zvsKl1CRy6EewMq-P*G#J_-ez);;2(x;A<ZY&h)>Zd;9pYb>ZKyRUTkE!Cxjn*ZyL4
zU;FWIw-0`@URHfSxUB4*lfiGFm|`2dlC2+R+&>+E=h8Z*&d$!eyUWj?KmY#fYVo@p
z6*3zor4~vh9c*GX&%49H&Heh)($mfC{29Hj1@GUqCf<!=WVm?Y-ACvD$L@$9Fqodm
z#BjxQoAL5__CAjp8!C^R<uFWB3V6xN)L>S7=j8I&i;aJDiYxE_ntQzZ|DWBf_u1`?
z)e$=Myhvo@ndiz33`usZ2f~6w7#TD?cF)ksTE3WzVZn_xt9PDRyrza%m$z-5V=|xO
z^2?Xz1gu>Yf3jaxOiV9k$A<Fv_ZGYL?yCJ=2HHH^$KlC+=E4fMlb(CN{pFL{+CFdI
zy<TZ^HQ!lJUc9g<es<>1pPE@rg5qKUh3oBW|Mfjd`|R;!PdA%L_~qKVm!IpT7+Q3M
z8!i~!d7@ySfBRO$lx?y5bJ!!#h0eDYtoylY&Z>*&-)@3j1hL?U^wnc)9!;`4>Ah0I
zocX}c;^$}1oG~&so^Mn6=;6c0;5#c!&Rj5^Y;vaflGEZ!?y@JPavrCDzu&KLQa<<9
z7w+WaeZF2^LL4m1mMu$6Oca&A`u?HB_br|=yI)*6Tq0+Fw*1c_o~vTJCjHR<IFrHQ
z>7G6ntLRrNrsf!{DzE04e_7{t)f1VIGapOw#ZF<g__}Yum#d$f7(+x+%bZne9*>GD
zmsmW`E7-)*bwo#!l|f)~(;c04dvrc3S<64cyb0n@lYVq~r#<6|lNTq8T)up{ombkd
z{$Gu~{r&|D6m)fUqYgP-;xR5co06IN^4YU*FE6X_V*VMf!oX3{^e9C2LE+A&)4opF
zb0=oqf-5JP^uiAWtyr%e+FkrV$iu^<qGHFwg%9t?u3NY6-RsxYUtU}^&A!GXV_{%w
zy47j%@2z3Xd!CilZceE1_g)=W^j~*F-pzaJ87Jp)mc6d2v9a~fwcO|R`sdOQ5s7bF
zKbH%=pTpO$GCy|Dgc-~XUBArU=~T0C-o@aMF=P5pFaLMWKQ>-g=(H~Skd$Y8_@&q;
z;a!saH_M!Nt17>GGxbheZtRNc!)wm;-Q7{~;*hN4fnUk|_Ll2xjDFqRBY3(jf3EfC
z!;hMiXYINDaBp`_*~de3Ta2r}EeD_09{N$gUFBr%?QNhFD`R(;^~>45y0$i2l)YgU
z_nK|1X0|vzTobw3Z@!(ZfIvc0kgj~doXYQCP8o7r>M5UNVL5T0gXc=uUCF81^CfEn
zD}VhwyEx^&yxexR#07hg-CCRfXI;+L3rdS?p5^&o-8i{^&*z9Ue<r+mm>crqALl=z
z`RC`GW`~7@T>1F;_<t+s=>nZDv(0jE?I?Ww<mBY#{`2*Mvvx1nQg~3iJ23L5(9Tb_
z+rM7*U#C=Q0~$gSva@gQd-Czf{gTK_Ei?XZp0rL&SX})1%a<#cF5Q}aT`x8^cGqFY
zAYS7V-?B$XIQjYeEsLM|%(0mGF5;ie%(t)aBrpFsi(BsO{q!Y#@})m6OkK;nS^qX)
zq}8qGF|Q{yHqLcwo?F_-XYx?5w?OO5+iiiLuYnF2-LX1Xy!FBB<XbB){bl~sE3O~6
zW5<pynU~Y<@2eG!xcTST*Vmqt)%I3=OuD@-_u92<qE~f(DbKg7joMT3acA*!v%EVN
zH9rb=9d^9RaIWZj3Nt%jPiLoP&5sS;-Q3epFSVXye<TEUs%N<S94?0QkMoW#-*9d1
z?K`VyR^89NC>^buA8S^cx+!3yYL9*8l|vVsm>CpS$K|H)^q*sqn4SIl;lsiwCnk2)
zegK`gGt)RdY;DxjSFfxR54C`1-Vz?aD3`IVx^m@8h?|?5rY7fQ<6pir7H02AJ3H&{
zuF}<Ye}9FAg_VRq_+xl+y8e$O#)Gde-M@FQ?f^seZ@E%d`TXDK-j=J>-<kCFwC)Wj
z1<;Asiu<oDy{hBM&C&!q2`YBimq$mtr<y-BEO>BW$&w{jrLShpo-J=t!0`P1s$-v@
zpO=-9*_MC5?$wo*cXyYsUJ-KrxBiM(iEZnC-pKiOZm#w9HIbM1*Z;5m{q3Ucx%ZPF
zuMoN^U&*jw(Nbql^EKAr92lC?)6Mcay=-=VUA4Md)cu2=r5}$g&)<1zp3(gFr<OA8
zcpr4)$&`{^&HtI#$L-y;b?e!aCwG>=kK0}LR%@%itLstS=xs53Dn33q*lb(%#ly#E
z&6RgqmW>V@vaYVmySvNM!s5vB<I$i4>jYP&88}}T106S7`+M8|{r_KFT>Mgd@?U2E
zbHCV*a50<Z@0>G3^ZJ_GdF%fvo{sV1^=ZFr9AUA_+h2UIt?-*UX>a|mxArd1y}xDl
zzxgkxCeFRhSMuhi_r#fo6Fp--uKzuOX}RCrSGTslzOgYGv`BKzni$i$Qqt1*_toA$
zdUR>|`+K0%Wp#Ac%&N<t+R?#LT>N>-k|pf?aut=8hBL3`#Ibua$7qJ_{_qq1V61;W
zANNU}IyFcC{?waZk58XUR9I-WN4m>p(a+D%=iArYsi+(|bEapSgpslFa{u{tudl83
zoo)8@-QC&KB(mn6Idf)b@$+NHj=j6F@$kKS^WGH%m>!GSRib&<aKWm>NgIECdmFvn
zZ*JgXx0NeagnXNmbu?*1%FUdcLH9-NJzt%!XN=giNr#PLvD6u!y_KJjUA!pBaBjZ+
z{WX!syAB0}gp?E&85tPNu&b>K2?=TElU;SUE+fcQY2usv`|F>an0R+rDK{tQ#p~BW
ziw*aNHh(t?dDzA;U-$XhS=;JwJ9h2*RB>tZR{vbk(qC!m*$)r5&$p=z3Jsn5uHeAA
zbMHPrKJMn`_V)Jn`}=Bt$J?J?GtJw>19T$u#^mGcVs~#VdmF{d%DPM0>8jkhqU&+4
zuC9~S{r6RVPD@BosJpRVDrsZI!$Ymh{pa)X@on3(Wr_8iO+i6HYJPKO%$>Wp_IKGX
zwR@`<H#IehYKOJ7w5*BSd+X{|(fMn8wYWgW85tS<{rx>WFi_IAYD-&N+r){2cLlE&
zWM*de_4O%DeDUaLw{Fyy1=e$lGBPrHq)fdcBP;*>_^2PhZ_B1llO|7IY^{-5{_2Wm
zdiwJ=Ug>#OrK^&U_sy6W$a*O9-L1SUD+0~)?%dc{3z{h{^v!>|v*4lAa=*E3yiz5<
zzGQ;-z#slOQ&8DWMOD>srcZG2<p&QEb{(D=mw#`Mq)EnwE>Z2KrY1fai-6^Rvb&U>
zOnK8XSgNY33JVK;)HWACKj%N+ZmIR0S#COF`cYdlZfyY_C$l;2>?*NjozT!wdHMMp
zHW=jR=d-i7KR-YJvh2hX&AMBlgYT@%-prUV;lj@1bdagS=ZcgW7$hYn-QC?QD?w*D
zU0)r({+4Kx=h~hg9(DEQfByXOp02kw>uT4#f`2{Rwtd@C_}H%Q&x?2O;vyn4GBZym
zP2DV9oLl_*nr?b}y598aS64tsMJFyaUS(s^<`%zm*Dg0Vx4au09`67D@A2oKp!3qM
zJUu<#y6nx3?fLhMpP$Rj$haVT?tX&N%(%EXemNTtAD=B7Hgxp#EU{Lydivyv&m4=w
zGcycNPt&!o|5tN)neWR!j;jST7G5;x;o*7o__3?f#NJ+CFE6gU4@1ILJoTct<y>0g
zdAv_{wt0Ts))_bdapvZ}-LPSUZuB;vStcj1UNzO$?ta&iAs`@-ke9b^)v8%jrkG@0
zP}p_YaVt|=M#!{9i;@BYE=<u3e)Hyylatd6zU99vzrUNickkc7zd@^b?yn41-*tGU
z0UsY<XJ@CCm6e#77<esd>n#8IcA!P#|Nj0yKiB&C+qb>^rH-L&uNPkAS4BQID#vX0
zyMk5iCr+Fwe0<Dzjz!_YCf1%;A4)dw|M25uGCx26M2{zL-{xjyc$8aAU9n<C#@kz4
z+4*E5Hl>^doh=gl-*MuVL-rP5oZZ~qzQ4PB`uzF#x3<1MJzf7LpZhIOl}VsuWaRDt
zeRz1dTTFM=!S~PD*w_LC10NsjeSLrb|2sR2xw*Jr^sNYr*;8@x&YeGJXPfiO+s(15
zEaLsg?i$4X|JjA}8k(A{?CkyW_WPo@=Ord4mb5GHH8M8-{q1cv2!DT9`tnliy8<m)
zM}c?m-uch7IeGfDv9Ymr?JtwNf}u;w-{0H0W5=0Url*e|SGTa(QTO*(iF-?y=9vpG
zuK)Y}em}Rko=)7J8>dfuYlvKu?YpwG@Uh$RK3VJXcQsE>P2E-c`jV{Q)s?}^`}Xgz
zKRr$N`Ptde-@N(r?QQff<zL|{p6_mLJ$>m?P<;IVkB^VrR()AutrW64u~hk&u9}+K
z_jh-TOG~$I-O9_sk#T7W=UqWh)=rmAb$@@={{F_w!gAy0&5N>pywh|dohGFuB|Um}
zcJ|t}YnS`ZzP3KzepmA^@0TxMa*OE%1O|5Y^{p#A`g(=ip{YqP&gb0TmizS7RM6RV
zx3}xx-FQLA+WI#5v>DLK;4;6Kw$@hP`F6H?dVayd!VGbHt4wQueL3DQe_8f={G{pA
z=bL6<TM@YU1s}`8^Pt^|V&dZF6(16+tAB^C4m)|`#06PC+eM2PgMy*z%ZsO{r(Zv}
z+)GtWt?ccsuYZ1iR#Q_eIo=d-w&nZ6i}J?i=KJga*F{A|eSdfN{+>$ba*NR3MJsxx
z%|VB(UcX*`byetAW83;)UosCgFwQp1J@lm~;ORl0-hXYYR=qkoS-qW4HY+)~_}v}L
zU58g%Sz229`T0%Piv=A51YVWgE+^Cp+K%w;+c!NuJ&?t|;GNS)jvV2UG&*weV&TtE
zPfhEhTW8FeadEM`wzhWs&Z4Dd_bT+ZB^+#;F=Ixzxc;>t^@S}jT9Y%r_}|)|U;qE#
z-{<G&e}8dN+17UMy8<n4*TCrQd9r3XGy41c7cYMN;o;$<NrrbFFDjdueEIP3@WF!z
zXUr2^l><6Bs$tOzUTL$OdwVLszq@Oi|L*(a<Nc41_n$v|mY0(=^U8{eE=n)2uaE!y
z?CjpXdtdUcU1eMOX-W3=b>jMQIoH?8-rcAW^7)~JR47A}LfZLxvQkp7UcTH}@iA$V
z%FDhLhmIUQ8W{~bYU9_jUg=&bQzizsW<e2=l81*{o7wp<$;M9o_2c8?21e#>d3U>B
zN-kX0n4A$Z(XQslgGY}h&7WWY;=)3yg5_nPGRG|U*7p4S-~AtYczJ2Xsivo;xq0V=
z`Y28O^768Ke0=@oWxk-m%mth9>dMOEe}5{gtAF>&TF31yYJFGm@7{9%`E9(?Wq*Er
zv@U-)<@W^8%BQ{6->obyuTDBUNn9sFL9F}w`gs3x1&*sq7Hw`*A06wJKHSDTdGcgs
zj;41J|Bkiu%e(c<?M*z)78!YS{+w+UA0J7Z=f&(STAHRm+p6r%jX9RZj~+b&?PKX<
zT^Pk;Y_dv+m6i4Vy}gUAyY{Q->#twC_V4M-8eOmM@893ockbKU+p86{qHL<ZoVa%F
z+oz|e-Q3(jo_O0NDYdZv(9<Ga9i3<A=31wnk=T{IvO?az&cxh2erM6s3k#iJf@}bd
z3jcrg^73*sGqdz_a|}USkEV6GOiGz=SIfi8Yo2q%p<IXKs@skro`v;Sca^@b{r%0-
z(lRwI?UJmw>n!_vJJ-PM>+6<U3$$ddoV8~U>q4!|>*M$T{{G(D+B$x3)z-8(-$i9)
zYX1EAxMRnTTRTlToGvk>?-X1mA>wLjWo4Fgqu}|uxh2Oxt(UW_$w*Hxe|k#P+1Ytl
zvg3+xha&S@wAdU47#SG_1qJKsK<9V*7Zn$?^GIx{{axmBp5r3-@e3TGOP)M=;y2f7
zu3had&_eZye{7zU_Evp0GBk|bo(E1~w;xJKE$q6oDb+hZK0ZDDdDgD1oOgG2`luD3
zon>0W-Xw5nY7$4N2|If`$l;cju5NB&>qC>>`(!w|xx2f&lMOy>ws!ggIuJ@r&Na}_
z&#(0LwYz((&F^mfvyPo#?#qjd%1TOyKu17D2sklVobC-sRp5B^;9xVTrLn)>{_e&<
z)8@>Pad$sH*Sb8Z%FUzdfrQk;Q<El5y0kI*c<^$+t@-!oy^E-8Q#m=$wtCj=*^l3*
zw=CgG%V^QMxU=}Vn$HXYX=&}VyHl$xD{X6keK|AJxZ<~@;>xD~&PpMjzrMVD{P5w!
zCr?0oXZn`bG&VLqeE6`PU;fyao(Wq(a{yDj7ipxYr}Ogm+E#zNAnP6Z{r&y-H#euJ
zq%7ID&+hI<g_cu?{LgZP>NP3w$yf-;%IfAyT3K6HKRD26XJ<EKp5P=ui#E3vS~m6n
z?%cX%rK5A^)Typ_5ey7Rk_x}SySsMn+VFKTp5;0m3qK{cxvl7m*->!t?p<3mvu%#e
zY$eB;E-&--_VncB=Rbb?Jcp<5pZUTIwJxm=U++0t&GYc8jSUSA#m~<@d}YwGM5b?{
z;HsczcK)!iuw@@T)ym)BdwX~H_pRC2Z*9qR4(H)mc)xM}0>M>5|Ni~kxN&2NEw72T
zcJ_k<jn&`Z9sSZX;mct|6Q|I(Pft$|3ky5mFaQ3=Mr8&OF|lXQpI=`az1`^7Bo7s*
zg#mtkd<;jDHqM<J`~2M8>H6_;lC_F0T4Er}v(C;kwXXQ^V1lCarOTJ&_f!P#ulp;e
z8x;~156bq_r}y{tur$29yj)vTQ#X3snn}N$I6`$~oRwYrOJ866`SWLLO3H@{o65?c
zyUX8i%e%YlP_?0fLB*#hoqP8DInc=b`0?ZScXlpbv?%D|UWFDd=Dvl3i`oM;YX1Hz
zb#!EO4OE)=;y@#_=snl0tXJLQ`V0$JtkBTWS>ribZPTVrqW8F+R!mB0b6c^h^XMbJ
z*j;O4cZVqmh{&&8!Xqmn5MbmP77<bL_g5)1bMyB6`(YLC91Bl1NJ=eSHEsU<`oF)v
z8kx_iu(>zax?F49;=jMYPp{!%V6dD2|I^dcudlBU&(Y^t=mjb}RxMk$EG#6%B=wX?
zWo2cPLKnZ7sOZmMzrqsq`Q`1-oII)dGtAL(VdUnt?fLi13JX^rh!b#{3Mx%ZCQP0T
zDzNYFF7NH_wUpswn4nVm<wf8m7422r_4WTBF`7(1Ic56v`S$hyHm9GzwLL#R@U3!-
zmh`%(ZXJFiu5a(}zkm9)w}QZl6DL|)Tfe@zxY%jd)hk!doILrkz(P$;4OBl~T5@u~
zdAGnSuNeykS1DOqTE^&och}dA+VbL1D>vu{mI=R)Twfm#I?j00rcbA)YRk&VoU(0K
z;E<J_J1MiQWx|$&JiP&-3Z9dWCYc5W|2=8h>a?&!CG^SD|9f{UP1M-m5%2Cf@#ABb
zKi?iNul@aP=kDGAtIIhSo;m>9Ih)1W>2fH=VBL(ZORXoYzh+-QclK=WauJS&@}P1=
z5wvNUr&quUX%91r+zDTl)-1a+|7pZM?zJK5YxB00Z%<M_#^0~dvdZ$$d`ZQZOYfX+
z%Dvz_@AKWwTUVy<S>o@rjp^fLWsZeee8whDm%L7Uy0*JI$WC><pZCf~V-wG;F8RF^
z{ygDuTJh`fTw{SrL0rkZy7Svbwn(hGb8}wlhiyB4tnjmctJ?hVZuFu$VK$M#r*e)Q
zS2--&+&r`lb*@AO@3c$YIbBX@&9hbC&N47?=$_qqc*Tv{OzvlUzwTu|pu%x>llu!k
z8O4x517~HG-iW`yPaB1XKG$8DeVKznWcN2!h6RP{tgHghQ<xc+>{-g-@Oif~$HG}m
z={p4{*|kiI>ThEeXXd`7s-PusW7mqU2H}PYb*I=F7I3kq`u+dM;k1J7fT4*~Q2Pd{
zwFfHxJ^XNU(d*~kd9Jm$iaS}BeOeIs;oDjXhlQ(+g&Pv@%whCV`?&FqAm|`dcaBh=
zGZ#2KXNDc#a(uQx5${x=PzDALS5<LuHipfa{Z;Ee-Bf?yye#l>v-=iyNhSl!yVBEb
zFD*8|YtP`cBI$vSxl_=yn-jM$?yvbfP3y#c8|(kCzh4jb&oY$OoRA;H+3-mE{C#;x
zj;joyl$f_d=M%5P%alKV1zQ5Xwr{?JbwfLei9rCAkZNA3JB2nLOqd{a@<6-)|Gk=u
zo|`0i*c|^PTl*VrF4xrjTW!O!(90=6#O{>%!c`xhot-^r&Yb%E^e-<i#>K_Cxw-wn
z!RIK@*4D<u=C?#qWoZFVZ-8hfXj|N|-v4`L`Pwh93SC|H_SWC(U=GF1Hu?F2PJ?H^
zI&9lKWaqWOudR&CY$_@$8X6i;o;=ae)I53WRNbE+A0Hj<&b_^DskKQ<z|w?^vJ6fy
zw(qI^ZDwk^^~dWoXU;s*pE6}i1~-Rd$jO3f(hFZDzPqzC_0z_E`|2jE`DSpjD~7x@
za8?enwXm@0a9K1dUsOcoSiih~vJJ<=Q*Dw`3tu_L#qIN-Z>M^Hl8R-?ivX#|jvS$8
zXD)DrN<Ti<t3C6DnW5puOP3az%e4e7J;c)+ur)C~J^WAWqD4x6g~}n1K{ey8gpCpB
z&Yfd%>~K+1S6AP4*pVYt4`hV;x^?SBTuWbGVtu_vVAaPBa#9PqrcIi(DgC_M!hi_L
zUUB;*52c9}RaIebzZnv;vaGVMXf!Ekt*mXikbfw>OEDyoh3Vj{tE<c3-Ff-!S=e&F
zxn8f9u2|s_7q@Sc)N&0i6O#}(w`0eTA3tzl!K5ljj=y{|&dMQ%>*DrW6+UvQt^GS=
z#tb%IDNuiZwn^rs`Satq=iNQlBe}WcWe{YjXT`Kbzb|n7U47=vnTLnlckkZ4K6ba6
zhQ^Ah<>?Fx9EZ-H)ipEw_V)Jn+*?~Li=Xk_6;urIPHb~~@jeiASn0cMxwp+SE-2X9
z?F(NYSK=O$xMj<hn%dg(x3^4df0ekqAK$XYWS6qjifadXdN0I#d3e0Iu+TXn;la<(
z&q4Kb<@dJCD@(nn-?$M`TU(o){MdK4*+do3au<%PN}$H6eON%ihK!4g4!85`$L`uP
zO{co<<D;WnGA|bu6@7biGdVNUv)qE?Dwjo@+l%upt*tL#zI^)R$&us7`{itRnJr0Y
z$hfzsvZA8mW6$>X_T$U_=fCLVSU8K{*u-i7>#wh`*Z%%C*S>z={Q2|y`}^D4*zO93
znx>~e-(CLROGU`mb}#r4b;lKLhk1H0tnYMLv^D$stXZ>W&767k$dQQX=*zN#xgjAT
z+w$&8$;j*}eC%dzy?e?O5s-h69pdS|pzq@DzCHW;yIWhc&GYWeFwF)Xae7zq;`uXY
zbhNZqx%bP-T9rg>O6hb^SYWNxlBE!%*>-{d>9c3&W|_XewAB0cwYAa)2@H21dMxhg
z>51H&W?T0MbYFpLbiQJX7U&38t@-=+*UvV~ee(3_<HwH;ZU1{Tv+*`HHC<a94cZ<1
z@Nj!Ns7){EWXhhF(Q?Ym%WKodjW>4|gSORQTOZ%Q+v9K0`nbK19zT{g$w){@*pPi)
zuhYc|6#gt0r+Y7iySln+hphpP95u0W-?@8NlIIV{@AOHNCfV599!>f<&$ha!xA!F<
z`@&be_ZtL$y*bp%{p;5+O-;@6cXuT1>ui=i-@jn_^75CLmKr7>`|{>y@vkp0Sy@?2
z+*<-n+ul2JIJ&vDZQlI($;ruk_wMDFvx!(AXS+*z)iI7H3EQfYnwlCvzrK0%?sW>Q
zgJSL#m$AvJs~(fSeEVi*WOV83)!yFTty{M)wO%CU<m`O-`0?ZC&*vX(V&&uGTOYrF
znYB{OD)}=PR($K~@87;*0|z(v?wX&U4z+SWee&c*--=}$w{7d|>@+ktH#aqn-CefU
zLuC<2i*Sr)Tb8okbnE<kdxX{f*2L^Ay0Ia#q&@ia(q+r4Dl3;RUHbOs=H+Jg6L}Zf
zHzj9;yk=o%fBo|1$De<G{HX9bAG+9cvYNWzoEi4@|0XCp$Hc}K7Zrh=pT%HovdT4X
zPsPQ}>F1|vhqJM<<=x*G3)&Lgku{x>neEQr>hQR@xcK<{_wUbtS0FH{%;I!!7VE-*
z9eH=H^7G$I8mIZpGU+^;6zZhj^kV|^#rHoJ+$no=Bkxpv|K~+a%<rFm=(L(9?6xJn
zzM6OD>>`^3_B#)La6QMS%3l14zpU^)$A07Rg4Y!h5h2C>qUUl${#mX0Vtt%>{nFJd
zHaKoucesP`R*sVX&r^+`{eJ}Hw;WH%HB&s{Ybhy^`ob=(|8uJP*{0*g%kRzD^?m=Z
zGyC6t16?fh;N{Dc4-0B)YQB8=qP3L2WtF{*^Ws_WZ*9$PYirxMapU&9yR)oHvz{(V
zXGoaME4QZN?}2^3^QKSQe~q#GzKc)H?z`q*$0{>!e&J#8`Ff1EOG&Oa+6=s~=%UM9
zUlT*4H21jY2jlB{&Vd&;@aq1K+L-_L#>T~!>-XKtVPXi0iaPb?d+_TUuCskV7$^Px
z{x;&Fw5ncdV%3|K-Ck$6Xo1do-?;JP*6iytvY-(!oBDr$7A;zI>J{h0S3Jfht4?m(
zyxF}^#<J?m3sLPb7IyaR?CkKO_DwPh3<*=~pG-eH?}W5F6Jx`|RqJ9Veej6O)BAp!
zGiPm^o5E=(fr85U{hDT*W?3EVV?Gd;9}vGiWS+=}9Wiky%a=5;Fvv}Jud{7_QnZsv
zh{0iwsCn^=cQKP#63UC>Q}|p?96dh&yjF#rx^d!!%vgpA*$EE~tGj-h+qXXOd%$`9
zVcOJb3m6$Bc#i$}2<lu+o-}FE`t|h>54E0p!@Dqx)7WIyOpUHvyUW+_+gCT&s`Sz0
z$8X=hUA-il^WvuE*S{QG^r6z+=m>+u(>t7}3hwM*WIy%$&z>*V?d*K>q=hzJy>dX|
zMV=Z%gU<7tq5n0Rb=T<l-aczR%QdaF(%;yi#Q#YkLqO2{FK^z&omagb?)x^w=Ie#q
zYuux4*(GO6RaRL=Mn%rj=YP&9vS;tsGsOqr>2~|v?%n%G>v`M%V-}uwZoHIUZ2V(E
zhC$+_e{XevPp%MQ*t17w)vhJWt30nXU3#3}u;)lag|oGC@2P*Oj1GINzVgZ0>?nPG
z?c`*2ekl_TW#z?OR<0{{9pLHBTHxL<7a1M>`}gnpmc?mTSB1*(EkCtOEG_t1EjzE0
z#OJ>IObzXd>^gU=Od0Avt`6~w@!x)x`|tM33Y~qs?kAs>=&z39Vo>@1Bu%DLpg1Xe
z@`uo9{b%Ld^4J-cK7X`wPPE_Wy#9}~UYHg){&@9kvF5CE=VCe1=67cI9dBPJCil7e
z<m0&;{`=Q{_;E5=ZFk`Jw6(kPUfD=UJUjmV+}1n=c80XSYE4g~4tzCdkoVU;E0KBq
zjk~?mv%AGBzd1*3>Xe@w@bJx>H%ZCKrKP33yu5q&?)9AX=c_5lRi+sW1ABu)LQIOE
z`9w$Get5V&erM6q0|y>lzaFx|X<FO7&u{qt{8jvJBy{+jS*b~gD7U@8@8^5=>}@-F
zgc%mB+c?+q>z~A^c@ZZr&wh2M#aCn78<R^;-+mrf+BPX^rrXZ-^B4|%-ZA^5=9<#i
zX5X_`tysA9;04o+Gm1}2qGk!NI#4gHsc64UEoS%JHTyFUPwPzc5oBnX@&Bis+3LjK
zTlqdc<UXgS7Sp>;KJm!UKqfI4_IVO^2X+K4-?3j^U447T#YNiT>sne{BO@a2?5*CO
z_1oP;D)iDM$(9!npFb~udkb_Xy05SAt|W~|&b#g2J$oJF{&k00%IQeohCBJwt7;=x
z9t%-9wZwB;R<an!{<+0$3=I1s95(FwvVFULT=ulDT8jl7K5YuwyhNfyjIWQ`rz<%B
z-R><tMz2rbh&^_0={X%q8>VUVI({ZHo%81DFpt^0_g((~9il5!PfvS(Ztm^P>HU)@
z2WMqz=^84i*cR~gW{KF%&reNtjjx)<${;BwAoXmWo=nrt^o-}n7!<^q7&b6ie2seH
z^=SQ(bS8$WpRFdn$XU6V$?4tcKaY-{*|~AyAI59GPg58eB8pfTyxABcazu4yy>=wt
z-N?=)U}L?x@b%i5$U4Twma|zI6#lU&ERQ=b5ZCwhi|^x3?t3xc=hTXcSTHE4vF?n#
z^XP#{bm{EmgK1m-onQ8Jdvb{w_X_``=6j6g-IkO|U;5Op{ASgK?fLNw13Z!$IKcag
zLL5)LG5%e)ez$Pq&MWDcbo%=aPye<y+JCyQ&CV_54|heYxUA>CG}Cxdv;PI5+k$1S
zlhv2U)@^$FL(7(LV&;|U_GxcyvV^Q3di~8!bbgg0bE8g9rO@5!@f-_7?a8fc=Grau
zzEf8Hb^eYsswt8u`0iEY3T;ikruS*)$5!jAGZOzrJnu{t&J<=s-L94Ew~O1fe8$w9
zdz!=DGeoDSul`yfS!TXG{pET6l)Ot1YpWlvm@-%TZ<?RvtSzb;KYEHoZI1Q|Xo=fx
zWVxy_+bp;0@2{`7wr0QNd-9wmW?jSDH8HDfW<4paJP`QokBRvtCOLnGQ>X0Xez<W}
z*f6eMk~-<Q@ae)E^%DwMZS+oP{y!-ys;{iPxcK?G9qXhOL9O4f`_7(YVDRbNRP^)t
zG`r5Tn$>sDpHC>NoA>E-XV2%?1_Dl@%<r9*TLhdgaexk<S5)Z*9TyMczIRr3QH~2q
zc(9kLOZSBGtJ^)2G3TyH*#~Z74VygQ?!UHi*8K0Y`BuA!{5>2k;Pjv6_yvxO9hWmB
z>}Ss1{r#Eb-^{AEpy-5y({tCHvi25u`+o7YMd$V(?-E#*G-ILQBGKT;!*fz^#q6H*
z<<D8Mi23|eb7EPSZabj0wO5pZLEn6Ocu^2&=hebyL)-Hl&K!TaL7m*iMcWQMTAEWT
zyZ?6m_kd(~fB*Bj;*a*_G#=I8+8Y{n!>Ihe_V#bQ#?}*laK5;~#;h0;_`%M^DQI~G
z_ZsKzax*oA`;NWZmTdE7k^hyQg%|Ai|K7XO`SNW2E%8eX^SgKE%ys#&_ZZW*sh<-#
ztcq*i$J||Xyiff}`2EfKxr^(pdUfux9ln~(WT(snUNgzzIk#y0fm8oy@J^Y)e_=(h
zoxrP8J{mWfykj=?PF>{I|IB=D?=3lxfS0MOK2|PI|GVnOOG(9$hXp*n6Sn;Pp`a?h
z?>3Wn{ev^-UeC3;{;}lp<LB?9&oXLgJiqm89pm#^tU2=^nm)NRrS^8o%lmB1ieIi9
zI4is8i!nT@{X0$k@$Nr!lEh;>&rki~WBa=5XmrBg<%@$Rlzo-T&Uta;->Nl%QLkhe
zG;T6=t1f0>ND7+pS?S#m+q0Z<tIzhB?(*^J$S6<x!D2q=k=u${pabb@K{;yMnhsHi
zJ4)BBd$)*)7@eQfn>^!Z>}&I_TUWdhi+Jg;d2wRn-I{Lw*oXB7-|sxv-5_SY_3xhE
zt8(*?pJvZrJniHi?^$~iZ_Ikx+kaupWn-ro_aEL1Tj91+tvdSevP;nkPutJVJy&q|
zN__iK;mtoX#m(bZ?@w#`dA-;>H~H&Rqe;g_|7We{&1|WTRt(`eeu2YtZBTTA=pMV0
zXMWbr60sRvKP9L4-F;=2_|W|Mv|Gnsolf=XGdfqG9MbzRDz9Y~vupET^%|*V_bs11
zFSlHnaNy6>yR)?}$qGBIC<2X-zj9o>bn9)I>|Ki;I6`G*oRz;!KXGSD!uy}=)~%a2
zZ{F$JCg(p)lTXzfeS3eR|5NnC8`lb78Qph}e01q}vfr=jUAli0Ps8?y{|iiq?8=(&
z9(~S|wg369w^i={en_<hw1T=j_fJ$8-CD8vajNC*-raw<<wicfVv_yy+Uj~+b<xdN
z|0+c2-m!Hnx>H?o{_WCD3ntBr+uO0@`lBL!;U%qatv8gO`{vI2bbp*)o!grK`((9i
z7S|Q|?tH%F_`F>!uJ~~0=PHMUf{$T5vE{C__{lG^n~R&)T1hvYy!hTy^@aV_-}+n(
zr*3|15}Cko;PIzZ^7WtQ$35Tmrev$>+Z*-14|uw2el*706shjqX>+vAch^4+9?+($
zCHurSRo<6VtBld}5BR6bP{gZaljd*u|Ke$T8}|bB4=t|?jD0R}fDTIH_`9*`c{!72
z(W?!9OSNb6xak#TtP4py&TaJkV%T{{w`Feg=J`zvoA&JV;mRw!?=s((+)`@u&469u
zc6q|?LU+4!F?Y7_(400$$1y`|BSVE|Hap)vDIuHsD;qo>{HZ87C(gK{=C2v*YS-`X
z;+*^Xdb#i2n|1lxhP4;pSa)6wz9pq+|E=NAJDG;_)hRb6A2PcLuCfC4miMci{*`8P
z^2rMAr$-`dw?A&(Tr7V7^;xxbk=yNb@8nFle_E37@wRpMZl;GdSG{7Gzu@8p)nw}_
zCSQL1Ir)uyx@jW!;dEZ%%6Tz&TxY+!{Uhi7g8Orh`t9G57Z-QGx;uAe$=w6@9A_HK
zT&@3l!vCmS*y?k6cDK`SZ7o>#?2dEy+}_i+yx&hat#}1WclPSb7#yaBn#E4t{kXMl
z8k_U=1%~YtE@x)?tzcqUUHIHpEa&}c|9|Q4^_Uk$d^c$LpPtS=O>Dyk<`4mqkR$Kj
zy*prY-0#i11MlAXy*c#GAwVSBL4-wBY+Ks%x&sdD<+x`s&cEMYojRlTdt<Ym&))TF
zf1DOX>$dmn`|sGkH9BCjrS2S!hcD+{z7=DZ=5i#Z@W*@OOPU$^vR1d2tx^^=v|P=8
zH#Yjo^Ry?^o~_$g9rU<&wwy`kn_qh;>gV0MyltbjUm^1nbrX{V`oU2*1ut&w?%)6O
z=K8Qb1_F}v>q@V0Syk1q_dQ)__Wln^-BssK&gOZT{`==E|Mgk9jz?aWxTu8xd!?!8
zoWxZB_vmka`NG19h4z(i&bIEaaQU<N=j?SgzdvvMZhlWA?!KgO2h$R5kVTW9>Fj?M
z|NM0Et@nF=K6||FN3yY-oadRi6BUo2^qRX&+4IPCrp>e^o)bg@JUTmg-gi#%YU|iE
zCF{;S`~6>YGx@7E1wUTAemCpy?X(C}-;5WV8<>O5>@?4LJbPokSnblDs<OHD|NmO=
zpJJpmNxgjMskLECe_uZSX20~hKJMtd$zPPue&5HS`SJZ%dHbs3Ti+Y*&gkB~JTCG^
z?eo7s{@;IoHg?TKRt1KbcD5h9Ytr9;eZ5Kd#hsP6mK+M5C7Sk5rZ0bAQPJ)oOYJ<l
z=|xAo56xN-`FVN)|J}Yg-9vGazcnfsc28T{QM$tTK=9MBKu4FD8+%UO?Z`;D9CdQ>
zd7GleKBbe3*S1Z)X|Sjy)BTZr>=%BW>!1FgD=Q9MUjE*)IJW+7&V9SvY^AG2igRo-
zejIgMeCwK&t4Y$|l||p~|KL7%lqElM=O3&886V%6I{zx$#D6z@?fqSv)xRR1EvV{A
zae7@B_FFt`E06C+|JMok&*rkrAJfchC~2E&tPt$^eyXv>+NibHq?TySo}wMHB<ABI
z|BS0y&0b3%b!fc%*}FSYWpeGWIdSWCqOM)NbYLTw+0RNn*3)%WlA57h)8^gXBbB${
z{CY)O&*CYC=l{R9eB_b2&|`hqu~U)^CkuFZKd?u%o^o3Ft13Hdi?QHDr=>vxg5FC!
zJ1RwloQi%fKmW2MtkPR_`_F*cIj4SwtgutLbaXEd=k>Ze-j^blZ%_U4<Lkcf!Ou|7
z!OJ^=QKNWD&i$=V*6%yD(Tn@|77u2ImrUhdj0_ANf>Kfp3=Ta7)1(#VX6C<%C^u{i
z7s%YY(Pdg?S9Qpm7m0_3GUxMu&xpwSzVumZec9V|=N~t(ns54+^W@9bGQD>f8+D`q
z###LK%g*{ZSC>Jd$fAwSBb}>%n(pS<=$%iNY;oOgm-}kRotM|Q7OhqdbWsVN^5&a=
zo%Yl7{iV-eTylN<t?lpqE}1p1dC_P0O81w(`M#Z@;l#p)jF;wbjb7`;lbp6P=%%=y
zc|mS`@`<n4v*z7+v-a2bqQ5@`jF<OycLp!7m};h`_O4s+noWfLd%OG_H9wBM)^B@$
zr7&vK$$6jmZ)0e9v2dZ{=FZ;UzaRY$1q%zhyTsf%%Y9wKGqG4zctgIcaQgJ#^83P>
z=k?!SEV?v{H&|=ed7D*gj~1{0UEyzC{&PY0N4v&X{KXP`Y&L0rZ<@<}Yqi*UE4lB5
zcV0Ba`g;4_*jib)+Vl8Knfv#YiV6!0-4+MF5MelUxHRKL$g0rQ&tAX2y*hloX7I9<
zlS+@43Q0!mReWlwJHCGIJ~x-%8u#a^ovVYlKbmn=MfH-+?A<?#rawL{eJdm?H0#mj
z`s~PiH|lNtr@s8UO7!~cd-WE5eYZA8-b$OEe9mU^PG9A7Ehf1kH+qDcc5R()@aWRr
zOJY8fHz#jn?+{|ze@`jTy6laFWl_rWb8}nym=tVeoSm0FdiG3AMkXgY`SHVt8|(g7
zee|1oe1B?pPuz|K-+%*GOOvDD%SYs&{u>dNICrmrx66`}Tcuw^`fk6jY136wTRugj
zy_olI-@l`?eY*>Cqb@ZmE^4W654XKhT64Vb?D2fzi$`k1Ef=+jo6Dd6Z5e6cqB2!j
zaAo|RUs>H5pw%9Z&20Pj?W_Lw#&dt<Lk1TfUf!9#J9qxvQ~9}1)_Pmn+gr0tv)S0$
zx#zx^9Wte6+U(Fulib+WiF0=Tdh=O*=LXh!rsqzq_M30NjB6#Y%cY!o%lX&6%-Xp*
zy}9W3d{u$WztiW-Z~y1eyJ@MG>6>k*mQO5uyhz*7#`9?k&+m$jQLZlcZz$PW&CPpt
zWo789kd@Kf->r?_ZewfP>c^tMm635`&Ye4V*2V0+v@Q4ct*zPpvessrnw~FQ;yZeF
z9Ek|}bY+3RzpUk9X>ZYT1*N;mi(^)KR20;i25;IH{_e)^>E{JrZoT!w`Z&{)rEF_<
znyy`S>Hj-v=N~h-c9zcG_5A!)?f7*vmNv7?S2+1!d%It8>dcRAch&P+gfH%%5pFYo
zf<|ENR@Ha^^ZtlU+rPj5<)x*~O-*_C_t{o_NSJo?>@x=W1q&4~Px4Ut|L=ExLPA1C
z#g6v&;};jZxB5-#;l8oG;GNdihp)aZw6m+7y~|9dD*MBovvcp{GJn&XnQ1a7YWLHn
zOTBIXrOh_dyWH0%Vs!7to#eCCJM#kmKD7R})>bfXe|LLT(c45tb0#LG$<ynfy_;(#
ze(dw~=OvN<PnF)C;a0pSd#8!!z1%{9<c}|^<{S@?*1J>rxw+w`s*%yA?Cb0HR)3#o
zoPKU!?eC(er&LRn7zz*G3i}XlQTZw5{k^?&=gtLPVx;WuwqxPU%d2>UrZzp@k`=Tf
zV$l=T%^_l9_BO|!Y{}x)wAb79@vZKmN0oJFXPHh`^L=$?<>enAABSp9oL9##ASNaC
zPb_L}*{La--qUm>&2l2%-`iVg7OK(M*!cJN_w>lJpP!zdJ9p0MIHQ2RiOHW>Cnu+>
z@9$(~W%m|5bSf_|U-<Y>?3JvqOFSooCLvN!P072v%aBL8M2R8raB0Sm-6DE1Kd!6{
zc5dUzyt(OV10%E1i7#q14<2l0zkB!Y{r&az@9*t>cX#*Vd5m=&yu9^`zrDT9&d%<w
zB2-@fUDmpcLsoq<H!pAQ%}q~t6h1b~xv?SfFq^ft^+I9B6rPNXAKsCXk-E{_a?a1Q
z-M4Qaue4dsrzf2kI*k;Y4?cMPI(l7<rN96A-R1AY*2P4=aAA<-l#=?FHE-U$?Rj@s
z1uyrTsujA-Z|<tgFI)VkRL1NqI(qu_>pMG(udj=hwyoL{qxVkpEW;7@@-9XJ!Pd66
zZ*OmZ|M&Oz=VxaRFT5w;)!8X&oOb5p<Kwlpwb9#hG)rU}dbo~XU~qBSS@aaN9%_zd
z@xPzX=dX+1?dImjHCJQttv!{S+uPf{mU8j&x!E%R;WRNhkj^V*vZMZg-Mc$Gy{GH#
zt@@fZC-Ijmzr5X=*xlbYrJnxt=Z~0fR7zr^Vu?~i4{!QT#(jz!8Y|Mz&yzWRcE0`n
zqeop|SR^gKn2~mF&d#i>t2QPd|Mv2-yVAk+{}@&>)K6sCckt%TpNCqx@7=q{D`g^K
znA8%rR?KQ*#axTRMIkGL{O8;8^6<R5zFz*7q{7w*?}ZL*H{`jzxBC0<@9+25|G#(l
z?%xLoo1N5!=5};+@X1>7$XFD7e00>h?9GXJw!1HMG8l=Pm>j6zxOHpn{<^=%dZmx|
zN?+fUdiuhJfQ7=HO&2p_c9mq#G)kQ_XU@C3yTf;v<$~hQDI=pnes##oOLupd+uGWm
zo~r%)_3LZ{i55SVCAYWdhp!5Gd7zQ``T6<tjZ(WhJ3B%4sNd0K`@lWVuJ+dM^7kJ;
zd|>4ktEj5tl2zY(^ypFF83u{(@9n+3-2eTZotrNdG8kFPI5Yog`t$R%ymeX5n;RQt
ztxA4;zh9r6l+@xkrLOG#y|?e)t-JpE{r&y@Rcs#Y85s@n*EXe|e()f{*w~n#zyIuP
zb8cC6(2{P@`Vix^GdDISA72%^IwvQm#gFAfxPvqEA3kpG?pAJbIjfS4o10R%W?!%S
z@nPYG&Py$R%V!%VyDjyax+-jK)$?<6i=Lk1oXhb*zp0JwNA0_J@2;<p&%d{)vRBGf
z&2P?%iy19`Q)cPK?3l1;(}}aQ&D;6q^`?5Y`muZ{cXVd{BiQM3X?^_u0|yQ)_n$va
zEA-Wqlarm)y;$?_?|XZ3ar^e|-@m-P%r9p%L)~9)p)lj0-U5AdhgFtkZ*JKC|MU3S
zGqciHA)q|-_z&N_=i75{n<XA<`S$j<{~U{pXU_P%aABy|H!*R@x|CruS<N?ULxSTi
zUDLCLb6k`XCDeM4m6w;t?<!fjdUdu`FF%7lCok`WgHv>)%Rn={r>BWpaWE{=jcI3N
zU=UC?F=1e6kXX2ok%8d|2QM!JgMwQ|1_J|wp_4N+qG@#D0s{j>QcD}vwIo%{6Srb!
z_!BWzEA-vHy{m&(K6>=X$JcjhP-aL7XoAygiiTo|34@1l&_ZU01O69Ze>DUH7ZGXc
z*?TVDyBBvMfMG%#sO^?4VU*Ibb7$o#-MqX!%i?DuTKo)4OifH2j)9h@_dT|&&B)5S
zwK?6N@iT)%l(RE)3J+K7qQt{(Jcm{4PZY+So~GN{%fgV!#mhS(vE=o&vuDoe{5+p}
zYRblq8wJhy8J1|8m^icrF7*=aba8S?OG~pj$~)oZ!i9`RJe@9=mibC|x+ED$NSS7-
zl*ll6L}z3)^epk7uIILRqg$_()>N+-It&xG86<N$NOkr0u8mwdDQ>mrQNw)uiW%na
zW!c=_SELL)7t5L^*8J|98<g$9;31lk(O@DXBlG6nt6d9SE{4bir<<HoU0L}pJN$?A
z<x`>8b8gM)N@hAR|L~k7hD`0{OI-_3hgje9<vz&y>bbo9BPNEA2TI9}%tLdM7!FOi
z`hND-g}YmN+Ab8tt?ZLBy{k~7!XO|j<IK!)a9`p#u{f*$PxY()0*b$N<#?Ux=Vyp_
zbY|vo2)Z&+)#%th#ec4Ewiv#0sTQ_w>*z3?ylaQo+keH=a*p{#-SC{0^wHBZQ9V=d
z^Z%;d@^2X!lv+05VPG=2$752WnR%~>Yj(E#lTCXAh10!+Do<U$kQgJ)RhzR#Qfs?M
zPu%1Dd+QqK>=(b4z5MJNK`raYw=dqE#P#LGy;UmfQ&X>9+Tkmlr1yM-{C!71tCe%t
zDrV&0c(Xcky8ra!t&$efsb;};dJmp<aBtHrpRv{Xg}y99omk&O#ukD4FIR7EW>dPn
zN%!-b52oj(l6}rhE)9|XcWcvz!sNKMHvUT%EWe#3zuI$^%D;!VAM<xcBo*B*U3uj0
zo&Qx|8EsGM&e|UOH6vfER%~a&$KUEwGmD)+$EdZXKR?{Y(_8sxUPsb)ozS|y%>Pa?
zr%3#G&85I|u0Ap&eg2CE4!4QAr!Q!#weFe1<;Atb+2zZx+)t^QZl{DKJ3N%83kgP6
zX>{CplXvavT$AhTkI&ni9d%;Tb*(eAO(z{=KNf2@^UEj4FI!Y=3g5qZTKuB9Y|rjR
z0kvD5R|Ia*mtd$9JXgfbaiHdi&&h*=8y)o@J>xU_GI##f!}s=Vkrebi<I|bc`Q?7y
z%ow*LN7v34n;x@8@9XAp_2es0Gwwfjap_rn$a40>d589umm2M<FwWLD>=XL-zl@`5
zu3x+g!yidda&Y)QZR@{s7H=1qUlP26%X#b4RE$<%4f9f2|7K#%Y6)f4q^Uwmu6EH%
zlU^wC9=X*sGilQ8Icx8jhu<&qJZ~B_z0d5cm%+EEZ>RGF&on+2*Br9!j`jV(Dy7pO
zZGUdRwZ2T?!)msM_@?9x2F8MXVTsF0mg0I>j&4-ryS8hAbD_EaEB%nOQTLRu|61Et
z-svLaKc(VT*fgd%zSrAd9agGHI{W&0pOwgr#q+$kd1_a5_0PMb9T$GTBt-i3w8J{l
zL6?4}8v1)ZeU*B?*mITH<+CAWMwg!_2LD;AS-wL?@ym2R2K(-Zl{W316T+IBo2A3n
z6s&N$xZZx@rs*q`dspq*^k>?!>*YIF@YnN7?%pF*b<p+tslD6I?!FlN$hmjPrgY!Q
zJ68OdvOcx=)(1s?yV#(#V(Y-5AL74jUhXfyAt(27Ps+@z(&m4^P3L6DWITR>!70F9
zs?+0djQo$IX?CwH-H%j=CUvGxk^lDOO8ow5+?_r<8>@dl*nj_RyEQ`t$3rWVh7%%r
zGqz?5#)iy^%+CMwm*vOohZ0f_DNcNQ<cb_#2rwwv{ORK(zI1<BnbEN2>yO+opPnyt
zTK+@dJ$}D#Fw^G$OdWjBp0as3vo$9^`|@(*#`nR6Xa2JDRaH$pnUz1kD|uVa-D7v}
z+KRHXG}yPdv7JabHBHyL{N0^Rsi(PQSr{&ft)ok|LW9c7vz3p*{%C@MP2Hav&np);
z*Mz^jvolm{>IF>(1|xox61D|bvsPbQAzksl@z4>U7drQkNtR60I=WT+`P(;dTFx;s
zOz>Q=@aVY&1Cxw9HaZt?6`!zmQ@XcW=)Ax0%7C56rgr}e3k!2fXJk0RbL6+aoxsG4
z895hs)qc=kS8!Q0X<fR~sp#A(2iMg+&Dts~Bm_24vg4Qh-~BS@KhND&vcK$qZ}91_
zLT1c!-f9G-XU$pU(R+*6M5^~fBm;wyjmegZzqiW|ob3MIwdJPLM6H%{pEh%ueW+c!
zb^ZMnGvn5aTX8cu9BXS6TlrAg?d=!q=q-o+!sboPn~<h5>5%U<yPCr$`8TIMwpjR>
zfgvR!Bg1f(QR%N$z14H9-%ZrgZ9Vttw91+*E06AyRy35gs_g#x(SU(rqO)`Nxu(|c
zXF3rtFJ<nJN!2cyc3)-UsqQWJYMw=TtgrvYHJ6n^VX>2Q_vu?(Umx?E^YTvS{zFQ$
zFGT+1&^Fh-m3RE!;^}RC(uyTw3=KS-yt+!DQ+w@GQa1Kg`{WdH&s7!mm@0NowtW4~
zP5F<OXo5qp;=+Zb!XF<J-^<m`v3_H+Szo7QT8Ogo*+-Xt9@wLrUHkkj*spAp8Ru>K
zEB)j5@9lZlcV^tqeReap<=muGBF`dYt=`TzSh}ut=g#fr;b0F-WMn*y-CMOaaPqW1
z-Rn`$z6n@)&kBfoUi5BD`GpN^X=$s%)<$v4vNI?oABfCrWMyHwvg-DRgvU$Gerj{f
zRpnjzhD+@Aj@8xg|JBvifn6za;eu1uj}M7grAj|to0<DM#QB9z<~f__jf--(Z+&~=
z>z9|8!Ri<;TxhEN_owoxmT6Il-1MtUCtirW)S0?&lHb~E=6Tm%eR_Htq)x!vWQ#%E
z-m1`jNv6@~87=SJ=9sG*8~I#2e8tPITVJK_?yHS{p~JwisI5)x!0qk%r^DBzy}NXB
zp^y2(%^U}|6>R<UwPy9zHPSLNIwfKZ3?C2ez4KDn%xs&}Q>jZeUyH+K6iUPn-mo@*
z@TF69^_3k}FJu1w`PtgX3~DXDC|Me|dUeS9#it*IZulj!a5INab!XTdyNxwVv#zg?
zIWg0C`GrCT1{e19otOK=R<Dg*{?IydVN>aH&bh84GG65yOTOyGZ;T2FnF30xF6_rI
zygUNRC0A^8E*{$``E36mo$}VY^QD}kTGw|JUA%wll-CQ3MP+|}d`v$-PcX2i=Fh=q
z_V@Spo;L957Smm`c5Q1P8$)K(L9Tr-4`i5_W#9aEY_Y}iu-Pg)b=tpfO;%Pu`zSxg
zqWTi39j)^yH8FAGj2Rl!Puu5udU@@t__%2L^yv*N85G_=&?%qwxN=|c^mo_a?JvyQ
zDfjh{FY7Cx-e9q{fx8>s77JQ+SM*94I;l-oU7C=Z`t;>X%@REZ5ABQ>Cg1b#Gpg&~
z)4r6p+GUl1Rd<Ex8vE~4ukDLFysT<(+zTD0tLtKSPm$lGb9-H^btg-GE2EL_pRaz=
zhuwQ_y)56B(LQ(0osbtgi;j7yw9eMy?VMtg4as~Rf$#3@<QCQX^3r)}(9NaZ;?Sn-
z@`akg%h%u9aeU`vxs{iKUg$_Rv`n?y8c}YnvdmdM|Bl>8U)$<$CZ>Nste)4}+PZu9
z?hTa;3cd>$UOv^-+MTA4$e=vVN>jx%_x?Pyb;7*8zd7c*_C#&Xnp)H3s5ia*ZP(Fl
zj7F9wC9@)Sm)*V5X&h_yHasFl*vgyj@G0N!ZMQeRzwuPjxwhq;lGV0t+vMcrT#jtY
zIo>A=N-3G#+r^gen`>SEP2Olz#q?%1vz_&Suln4!;u1G*D>-)c_tyU4obT(_Yh)Y>
z5HdgR|NYzP>mfzqCI!>>?VV|2d*Vmt|9iP1dhfUXpYx<Dbo%aRQ)F(=t5En?zF$RY
z{iQCopDIzdA7$zv?RXX=Bw947v~;3Zr~QlNmeXBQ=cf90Z#Z3^WtW<EW+`{n3A22k
zU7xzY*Li{pkMC=ugRdVtRP$y=;C^{4<5xE~KVRxST}DQROI#0>89bshUWAmv3Y52j
zH${Jaym$XVzU89X7tWgSpElE9G+*w|!ostkPisAyB6M8$_V-d-;fp&3oSFG&emY&K
zq{JI3)Vsu-ZNK{YJ7@A+6_tt#r=`^<T2%Umn{4;2v_AP&`Bkx}i%Y+p#11d6!@Wz2
zJ$-J^>bk@iq;0gMCSui^*=Ob$v;Dia_NGy=^mD)8&vjQO=jE9dIQ+lQr@!OCfddyV
zTnLf5aOKLB=auE<-w(I*FMqTlXyulmrCwaDa^?&If+i(dpgY#@t}DK{?e>|?yZ;<C
zs3|&cx$|^xv+2p&JNu^ebeuTRUaMsz`8daSz4FiPE;V}$XDLrpKGt&5vnE1F@Z!GU
zS+y=MU*61}*zwn7*Id)&%m?!IHuJ(}PO$1MiZnKSU-9*LEmxHBM$Vck^Uf4`{GJj#
zhtpi(<LdsGURy)O^c5FRw^B+v6KVZO?9P*pjMIf@J=Kq%@zT_vW$=`1qoAXe>0h<h
z%CPBvztw$KX6L0{TRZ#vdp_kE6DA1A%g@&iU$<k&pOgHD%|-O%K$}26KVSarNl9U0
z;lqayh1LDu+}Nl*?*Wqlr%B1GomF467OUyXMz7m?=IzJj$(cR+k2L+wGFRU@^JuzF
zvVL)J(en9*q4x7kziX|16E&w+_OSI0Pr3UZvuZATxM@XPOUZR@>sTb0ZMG+IkFBU$
z;I4VUOXs&w^>i}{y(K&|+Vj!;%;R?+x?c}W2{!7Kx)*XXU{2zfFy6`WW;>eBEJ)7L
zeeo-^cjFQ#N1wN9%a*B6zP86CrpqMkq*`{yI%@%c(^IE*q_<VeAOH7>W1d9zwZLQY
z9KRMnUpD31&N<d^Wo`B}&YZcb^!2rRqmU}l3XdIykH5XS3ED(9RXcpvj2RqrUo4N=
zS5tXrhGFruGcyg7-R|xx1?8_D!8%h}pcT(#P{s4-XGV}EYuxpH>RO?0elBNEpHzDP
zBJNVV@ztKaTiiKcp1k|&VzrR?T{Dp{0o(lM&9J<8p|bMO$AZPNj;+y49+qvHxKw{$
z<f@p!hZ=@?Hgihs?qtmMkJCO`pwOahBzJeQTv?dASdL`zvSMD>ppLwMnrE|<yh10;
z-Lq8v(e`|wn9q~c7ulMBb@BOn$w<lbYR#nL>HDj>Zr_bm3zha${r7UaQ)>8zh~u%g
z|0EwgfBt=W^3uIOe*Cz&*j-#V>dKKLPtMJ?Zf55{r}5RtCH3?)+saQ*W*8>3@k)j4
zEP9%KeI00QCcnMSj}<iT7PZ*#-mc;t+3Y2kCiLiaedafx)VswuZ`&N(x4Z_Y?B}gG
zmwI04V->UD#j_53r%5cIBeb$bj{Rk4#%s;B(`GPz^Z9c9?a3(x3A}8dv~;)o8Qs~j
zp1WiD@%I&venfv1`Py_@XkySKhb3o}a-S*jR<GDClyG5_i_3hy!sGi*SXP`|v|!7`
z)n~$TEq>gMK0iaN^~MT;!};?c`JF_m&{rD9&ARZi<nyz$+7TNbe13jjKYpK0>8mR|
zyB4_q{q=Qy{Qi53T)UU~&Nj=x_vhkbcPDkGAJq#N{=f41`FZWIl@k|#+Og&wZ*XBm
zu<G0W%PtEAukPdgA5h-y@$D(!um0!#_ILN{rrPHl*1TqFo1_}6v;D=CvU{3$7ky@n
zu6Fa(Tpu8;oZ9hYb-L+izfAF;ucx(3eLl5cs`-Acv-=UVk5g>-t^6Oys6PLZo&21v
z;O59@S}(6;)jm#Js~5X#N>uVPv3bw8=iGes<Vnew7Z;n^`PW5kR0>}g<JA63S?6@w
z%S);X3JWSeK1w}34YWgTwt0RFAL|EmC+GhY!`H`6T_17wTCoG)ce9n&o8~FoT`Kr%
z>z`s%ch<qhXj+GdvxwA5$qvoer-THP)eRN@UE@~Xcx01{OPOV)u+cmoBef!xP$QQw
z=Nw#?T>sEG(d}~0r~2*<X=;-M=la~8%A?t_<B9ZTiD@2xul?woaZA@nYgX{OKbwwz
z6`kVZqB7a&oTc5I<ocfnKDhWFDLmfXtI4b#I=@Y+TzzIpvc*?+{_?-}uWO!inZ^b#
zuX}uZd;aogJL;c3IXU^=yLbEc?em_db8~Zg|K`n`FLVa=c6Oeepy>SW&Q9gYC%eRS
zLAz9c{0RB0&&6PWs88NL?gfwJRZt6S+UlIBEzawT{x7+;!02Sdu~UnlP1?g6x%qmV
zZ~jazt;dHH))l6v{;gT#qNF)>Zr+`1_GSU5<*7Sua)i@g&i(8;J1%hFouF{{nY){}
zI2wL?vvJ9Vm`!1RMoYaOr7W4GS5&P!+1R@HWs_ZcV&ud>n|7_+wB+}!qk)qiP8D4i
zq-O2C)LXM?hLPpw54W3lPO40Kr}t?UtBQNMx@_-~PpT=ZT_@i@-dQYoGJNY#7vpP@
z*%4oV6g%&>I9;{t^wgkZN8>*IYTcN|%lg@&TtjDS`a6@elj8H{)*syA$18nh^XARo
zcMsNIeS3Smb=jL8m7kwYRCYf%!%*3Op3R|Wp2})!YFAf>=il4&^WNTSc3!C^0UBP@
zbWU!1(;v^tP|x$)`Tw#THzLj#P3^sz{ba3O)B+A4$Ly;AegD5yY&q3(a@~>+5q{s=
z$<3C|ZT$WzwX=Iprn-5~cbjI&cj^C~|Nn2yG+y#>@ujqht2%DqR{o<Nv$sUA{@<U>
zPi*#|-rU?gQQ7_1@87?F{3v;O$yMubv|I-V508v>*_+45`<MI7ytFD*yY1)S_BV_V
z(ibeO4?e18dUz)j>+7=uRwwRjyt#F&JiIpErdnn4$+|t|uPT3kdwZl)*gXH<o!#Z{
z|NZ^FI%L(Cn(~Pj)!*LCFih^qcy?xH=;|<0D?W#`w$~T-zkGMMAx4y)bM7PkrvbZ8
z|Nj2|^5;z%NlA~Ecut-(XU><ZziXqnyY<QZEWc(f7`iIt<(r$EL5DS5T<m^!mg)OD
zJC#e68hp5Uc_o9Z<|dg+@SL8e3o0${?k@lP_wVyFGnId+792F>xx6NF^KA3{Yipyo
zm%Y99_SV+we}8sfSjb?cXtL!-<<zh<S&2qFSDxG=w(N4$3?2<_-5<s2WoOsiRP^<o
zwmn=-Wm&oOX4TTV4@TnUM|aIzcGb7uTGMzzZ&txVpRdAO)6dUKI@*<ce_!qMb8}x`
zUoZaZ)OL%LDVyYMtM2TrHqXDe=iR$^%xpXzeSKn9A`Z(Yt^8+uc8Xr?<$Ko*`N}5!
z4xe1cE-llo?OtyB@lWR@zl!u4{X`M3O9JwOnH68}{WyD9B70k9U0cV!{l;!Ch3D(b
z#SD(kD&4lgef|EwdXndUetvFiYpd=v!=U)t8RvGssTzS!FI<#T)6&ecuB@1Cp0B2=
zYH4NVH`7Sf@MC}aBxVLBC+BpIjaEvFZj}G&RoMGp)K)NBZ@o$-o3M4U+RvgDb4`OU
z*c{DH@o^Nc`u%a;yV~U@llAN8*&jJLXXEb+AJ05!Q(JdUNQtX<mHb9`>5ql-*2T*f
zly(`PG*VJBE<X0~vp##zX4}s%lX7l8$+R>6C%pOlySuC7_tzcoldb;pqVVOVrBbFr
z+4FXsIdkvcz3bP%pP6aAJ@2ko$O_N_DHeKHpRH#6k^NVfzbl2QidQg@FPl@)a!Sa$
z)mol$A0K@b<$bR5e~a9)1COi3!z=6kGk;{Cn(XT&m0Xv8b3*^>j2TZS^sQfe!Cg9c
z(?-MZrtG{y>8gR8PcxSKO^$sxr<E@+@O@u;mAQFg@TZR|8$&-;m#NR1YP@ph<lb)M
zs0fdaAHnQbXY*WQ{rXftPGT<irjz<V4^LflL_tR<=gbVlc7Ay|n~D$1d}lxTv|c@=
z`G42~hsIuM^Lx8Wv%kE!Sp59lT)Wz;$H#hG`B*13K9sO>e$aVhbyt#0@#S-UIWarV
zecyD>K}32{Y+a!6yx3_)wmr{oF8yo!G`(D<@Amv1`gc5xs-(VTit;YYk4+77xYY62
zlRZvqv-qU8m}_=VUVJ>d<fQo5<MSjtJp7-8I!3B-+h+z}n&A6tc9!R5wNt|4U!xYy
z5XgQ0v@`LD$cD)wT3W{bkJ9E}%Behda*6!1pL_nKYe#av`1||&;Wl2-ff}1Pb8~RK
zxV)VI)z9rybs|4~czF2oa({X2vNhAEUq9L{-s;CPA#mYBV=lMZDK%d_w&%3(nm=F9
zr0aRdewl<FGDjj)|0db?Y|vRVWyk6F9d_sMo>gDJ=->>GPn-UHO;8q;wVHIJviF(p
z#4nFq?rnVaME$YkcbheecLdyAT&B;L_E6JwwGph>o}v8cs*Iq!`FyDpPTSVaUU}H~
z?zZwL?{p?-1tqGk{<8dPR{Gglp(_FwzIgE>`}(?GdHZ{Z4mrIDnCW<SmZ^UHzBg~*
z-dz{FyNy>GycmE(;jyE0ItPzn{r6>c;SCQ@?LKdOM&<mKzgMdy!j|y}PJVnk`s9-N
z|NBo)e|z|9s&#hfTl0I}{+B)kRbOlD=!kQZ?DUxa=<THTu6_d@o28YxQ$p5cS?v&r
zUMe6hI>$;oa;fy<i#zh}?s977irrE0@W;o;+~Rs?=GpGPu+T`fE%DZt%;MkQ-WEMQ
z6}l?q<n7zmHa0OY0zlKVR?HrDBG;DBaGCsng9Mvk?OwUI$t~)WLUa!bO<X&vLxNpW
zb^W4)$3MUC@Cpeq+ms$N=gH*PCntwbO%d&xyR~HpfAYQhY!%7qW3xI=te!FD(T>TS
z^OGeyBBsT3own(nbADC}1FxutPVtnF;X(PTt4j)ATu=-Tzb>jBwkCc*=$NU#zP^=)
ztG*rUd;0nL`T6_5mD~}(ef7DyE@)46)7kJX|G!J=J-hhrr?UE69<QB-zm8RjZ_hlx
z_0M5r>*e2HZ~t8PX8sYb{kyV{_53_NO*eX5TK<e_>Wgknm<%$`u)g|J`>h%7x1X=)
z3|l@U&Oq_R@oG<(tUv0Z8X~p7zwvT#Y}mL_Gj7k0{(k?QH}bWX4E6jbTW;*!accIH
zX#$44v$X7OQdB4NWUdMk2wuN((~=&(xmVj>&I}Uu6)QC{<-K@j_afVWlTJq;N{Kb!
zto2RE$fz`1>vh`n2~*D;pBh=@wK{Z@@2nMU;gjCoU(cyrC4K(+Cxe+m!58m3yB)c*
zMsNR<^*cYz%D-@_#3}gludqLxYHx{Meae-Tbv@up@$~1Zn$veHEuW_udN{?!H#KqQ
z$9>Z#zS_B1@`>KN?Ca}7SA}Grn4tLhSg&>Er<9pSsX_TUt?s9%rKMkAJLmHI3Xem}
z*S1YxUhN;h<JX=wuG#)a|L<+vdUCGv%aVjc;?km#+@-H$FS#9jQd$~3$8z<u`*UWQ
z|N6J}d6r%JOt!c#Z_rs|0lOQUn-rB!A4*MGeJx`8`Zo&_OxCYk8@n^((yPyHBKkc0
zxBZ_{S@-)&w8G&_mx>Oxa9&>*TmAW&??R{6XJ=+EzWVR~dToXetY=RDoxNsLO6lub
z$HaM|FBIf!w#v1gYT3B#y<YjlgmW!NKi~iT;WEd~sb7BFt^ZSDz`9oRk?j18Lm~?Q
zKCahU`|<fX|8v6M9(DHio;|-`MMWj$NQWS3*??tH%I|M)FQ+xCeciT9RIU9*3h%Pm
znb#Z~U5?D1cX8YOW7R?DFN%k6{r90jYI$Gh(-1SUldXRXlFu7XYP7!fv^#im>#N|-
zVwsIPE4Q1jT`;p|S8<@WlG0i$KW{NH`Sy!NYnnojube8jsz07bTzq-h3LOz0Nu~AW
z>-Q~r-#S%QcgYliD@Ql?mV7uiHTF(NhfrSm%8JT!YwPW5e*ZWg8?&5e*PdgpQ)^gV
zb}wHYUa>y8*1P_-(FytLA0Hm>D16Ms#H8jo=f*;3_D+`LoDBEd_Dj!niTER4yT<R=
z`l~DUy?XqA{qrY+|2EI-?5O|K7cTtK^JVP${SW{B{jH~`mzS4U{_f7pD=UL%t(~8d
z!gXEBWW&;b4?2q1^SB&wkH7n8)yY+!y+8WX%MP#4%e-|d{diyHyL0De=kHTgnzUMO
zUexXFnVIX2-l@9fbU#u0eop>D%%0n|rv5JT_3~_Nwk*28qc!-~^zzsH6-|r^;`TpV
z_io+aU#8lRLzi7S(ww!Z<$|~HuimIv*2ng`Ocy@C==|EbwRbD_o<49>JU;WpnVrw~
zAJzN4&dYA<-!Dh<Ru^uy`(L)$xAKpVNOk?=-bYtsH!nH6y+cM;7Ie8&+1pz;H>dyq
z{eFMu7d?iV3{q16xYoz-|MumJj7`Oc&Q8x+`zGpj>0ez?S?p$%bX`?x*Y4z4$@jM`
zJtqH&wEI=1`+nu(>v}J)X1`e(x_4RWyHf2wb*0C;Pamu;U-xF_^zRR*R33k%Jx$v&
z-Y@KcdwiDJqoU`Id)av<Q!Px3Z!7N>)73K0xqI(&+1(qqq1%hk25+tM-&>y(_3g#w
z?fvHU*)OJSey1BXX~~s!qIOS<G&)Z#PRuv@m2dNV(u=$QKCZV4`Tve*S$>{eUEKQ6
zMAuvAYTmtkX_<Ft$At?QOlrj&czAhvC+reh8rii)A}nlk*73FB-@mUraec{-sSh8A
z=)Jn$?-``^bKQ?Mc~`wZ_P=jCDpm4I>2LItP=mBq?Gg>$`agP}iFx0yE+{<zPVnO6
z*W!6Llg`aB++Jq4S?61%O6#2B)e@=m4lR|?zO=_C`N=(7{tx?Qf^yF3+`lKcW?sjM
z1wRXvTxMM^uBx5fyqNDUzeVZBIP37M-?<{H3op;x)_eET3l(ODKQ?DH*(RKtWD>F{
zZ+npOuCg^R@1OcUK{oKr^zBui_J6h}pWPR?&^v2twAU{E^OnMb6OS!OT{2br>gwq=
zGp#<|T<xOt__E01BDN&1Yx$RUq!fKU=F+nF*DIygMMdKB_wr*-mzmU^pZZ>`;LYD&
z^Dh5bjcL+44FA$B+Rk;%=qsIad6t@fMXJ%-m7e<LFa7UaQvSGZWz40kTNeK8FaN99
zzv&d~*ZREPwfnp0bfg3&ZJP4xR@~7IrP1MMd|lXN1t(7Nn7TGW@p12D^Lqk<@_!%k
z1aI__^S*S{w%32(B>Cdp?aK;x)_6$@P7FCI;F;>85*l&q`NPu9X>ax{R(<Qd|K?}C
zqbE!`Coyqp3Ql}zJMq%7Z5Fm~FXflcxHaL!r=0zX?I%K9UD*7jg?~PLsBe6H)nRXK
z6~F%>T7RAz?h5hx^ZN<+z5I9c>f*MBK722i$n5aT(K+4aPn)vUT;A6U9+>~m^xJPx
zRiTl-J!nPU2d(4+lbNSxTzL_?%u8^knbBmO<f&OZH1?+&r^VRX=Kt9gZCLYj+hb{-
zjA)O8cdYj%ed+Oz`S3(?w*Ko)Q=+z2-uX1|YPwGJwe|Pw?#Bjqt&oVm&NfRl*6mEZ
z)xK=~*R0jiH}^&D|HvV^KL2j*-+K#1Cp(vX7Spdz3C(=Dz|C51OL=)&e(~v>%gy#@
z{1bZjaO2`|=0DTcSEcOPSLz>e+jsJ(__^`Bd!lc$weLFrwd};>OY`<$J$CxB$BX6C
zKi6b$%Macfx^@b8OtIlX>E}AT&e=!VRQne6{95$>**4vRE@=mm4_mJ03;oK}PICzl
zz2}kZZ*MEOH6tZTUpn{xj0%;B9VgDs+Gz5j-`;M)oYQfKj!Pe&)M1!7y)@+A?G<+C
zjl!ht-aL*yl=jRgPLs8EPjPgr(<_xzsb`Pud2H`19Cl$=r|*NslhXOOPFt6=(qcnk
zt;3E?yZW!0t1mD2_n%?#@We#r%vrCNXFh(kg^Rmeq~E>xm4)d_JA2cwzH`FX`hDDT
z_4~I8by72fSMz#JeI(8mzPe9&>CyFj9(|X!tEsrVtF*4J?&Zfc21$W+&gm&9gY8bL
zoQqU(xpUe@X=aK`aB5fSIv(d+Gtw&08(J=%rQIn!Z^FbGX}Te6r|<~Yzw>xBYx<S7
zot?|O>b-N!_SSupTo>)@Y9{I|Bj~;4`L!J1s%f!^A;*_Xj`sJwJIU>vb|7+x>*Ld9
zYD!6=_J;Bns{?BP?)rbPIrMM&(|YBZT$d*?cd(u-N{^7y+;n=`cB9y<S521pS(mvK
z_pc~&Jv#k$%tk-4lAlY`o+Re(X;M_;y!hFA*16Nd@dCjbvs(-&R^0M#I-lwG=Gug*
zS4>Y9e75yZxwF}J;dw{jNj<sYk@t4CDQUWx|DW>A)NrZRiQLYWK^?suClfXoFf*iB
zWW3<%R?S`V@L6=6mx$P!+rK8MCwW=aY&>^$F302SzRX)!b8-^v{+@lJocHgYa$m>w
z7bo?sPpcO)Z&TT)wPeQqAV-&~MJ|0F`FlkDIoI=)Kkk_7Ez>b2a>?14C5il{;Y>Ui
zK|!!0at9-Wk<y%}=Pvx1yWGU`ZD^jH+W&ptF@L7~-*GPVlW5wp%k!S)`{#r@KYX}i
zPTpeaf|kYo-XVg5$3LnhyKgs5l+XIL_PYMtl~YvbTm1i#efze{;t)TXkKyODwjC7~
zH2gM+EstyEu>!5;GlzH?CKNuDuv%;+py?x_RQGS+lI<=_voxe<)$ctwb@$pe@s(v4
z=Pj|^&KqQHHhWT_ioB|@WJky1JAIdOj>PZG)Oe$1syJz;ihc57jm{30cQ;keJM+Bn
zSb6u*l0}wJPw7X;#0P4{AB)ZiI&uZnq}`u$bJI`bY43j=I~D%_?PZ~e|9fxG{;chu
z$M@@fErUbLg$t5`m$&s@3Uo_Ro*fpPHtUG5#lowTmSjc;&vySc<#miM=hK{G?PXU3
zw`rYeV|yumJw(7T^z`Y|yH3fZE;*rB+Gln|`0|3ak@D)-9DLrkoth&wFXGzr-ji?E
zEnB8vx|xY*_qrE6M%%KZH+!0<Ebo6^oVuk|?Wk|)LSN^bkCtjD#aeIw19RJ#Ldn(n
zZ<p@fQ8u&mph+Y%Gs7j3b4BScf2JvWMhE3pta`um&*RO({9iS%?|ik_=HR(m8?UJ7
z=Y6c)wCXhDt|#^~af{0)d`k3OVw9-SxjyFj6pOrjc4w<K<;DD6N|YV0zujjyT~c-L
z<NHFfcMrBc5-Hb8(%m#=(k}6WA8xkwJ6Xa$E%Wyd7n~TWxBj24W#0*b%36=tOKbI~
zobe7)3Oasd!l_S(ox@HRx9A>~No0`Z?^|fRaQ(HNT~*)bXZydjf3wE#(bbdTD<dzd
z)UJ5$|GQ=1<NHE_fiL*Q_B37j*sHqXO8>Lcw;M~|Ub>d&`|IoL?GIjj?e|W|4AMy!
z-C6Fm{i?eD+MLH5-~FrhTAymbprHFeLdwo~^Yp42!B?t}?e42NCwMn{;pabFCrv+a
zt3Y$Z64uK{&Mu339_Om}Zlmq?y4;%H=)ckJ;u5j>KGOvEEq$`>d*;^tQ$*r)qb{ZG
zkJYZ>Qe`#0nQZhm%yzk#8T+9fPf`tbeS6hud5FPC$;6~)nn0$+-mlW9%^a+L?^oS0
z?LKd}?pM98in(nX;TMyRywHhwWUtql`eRGa)W~VGSoO9PluFu^SKpo$sqyXGG3ler
z#k^NfoyQ28Cp0z|SC3q7^l7{K<YnxMd~bQ4+@0%g_@&-EK{VTH$)huAf74cQx8DB{
zX&LG|<wNiFwD-F{7X0hwX6Mf;-SPeuYwHFcHV3x0wsU8i;v{&dPLiL?ba9eU;G5_C
zSHr{nZF0hPPAI&2QaZnD%IRr)SgzSu>Ifh5`{}*=&AUq{yl!2*=jQ4%f7*dbr+hk3
z9LQWeKjwbv>2>MrSIXr@YJKOkh)wdpJ?|c9IiBOfg~kj0wG%&h9*gLU=<_%x5_0Kr
z#lO|ElkERI43eA>wnJn8)P0lJWxo71V_kUh&Dpc|eY7r#eYEk|{=|Cq+O1_*g@1qH
zv@X=I<XmI^kiX*ZwPm5<>ibJB-TG?SZm-j5x^1R+^}Xf2(~oyNJ>8_XE#`p#gu7ht
zj8~uC(ztVb($e$>9`5YuziJDwZ~L{*OLU6tCj<URlIlAb-uC<3v2N8KPq$4IG&&Z^
zWwre(-oG)V?Lz!z^OO%;*VYA{Qr+<A`!Aj;C37!p^oQh5$U4+__S<>8wdpI*|6Uex
z?A7Db7Nv6@x_jqxaArp{{xP@7oqEzhwA)KK{zck*|0>0aB0l$)nes%<D|oT>qTc>1
z56h>WVpyo5W4E?(>W8G}`&S#X!{6=asQ>uj)3L47S6$QjbadaESC_RW%f}z<+BuK$
zLHmUZGkN5HOxmoGwP{c5qa%!Gla*GsI5qQTEpm~a*z)4~^iv0n!xtqy+mm&E)0~Il
z=4&F0l@u0!JaN+W(@f5}oD87FJ?Un@{j$Eub*UX)y4Pyfsp~3&6HT7Zb=to#q-`@N
z>)+!o>J#ENiReAI-81KcCWFG?2NG736OU~PvfH!f<vwR`m5srZ8aq-uJHE&WPW-xF
zf2yLp?SG!q8{Z_EXO+FRO8WQ8_w+TJU*AM--d|p8cW3F3brb$q*L78wR~`RUclH?j
z>Pn?9zK`<vw`N~o_xD-;tt~k-<72c^*%=BC8J46aC)`}P)@NR}hje<y{P=m(Ra<v2
zs$6vMu%naC<r7Zx_k_4D{eJApQX@Urnh34OdpWi1SI!nm&~VxnH}BKB+_N1$?By!=
zgnntC;@EL{^Q^NbbF1=vP0t=yo;r&+_R5B2#jE~*o_1B&nP1yhp=Ufv%d70S`u?l`
zZ6%LKiJ#H4{QJT+U+I#K)crN{pYIZWeD`c#&iq%kr<OXeH~A67;9vp@e*4_oCh!RO
z@%I%<R-zvgJrBqB+sVz3z5f4FkmLtpp~LTAeZFoKD!5YSwao0|eE%E2Pg&-4KJG}}
z`pP0Gs%B4j_Z36KIj1f;9TYnHUiPlds>>H2?L5<Q_TH-c_5R*%g|Bu*^ljZ3AULu2
zZ~O17rW5<me>nE)nU>O2_thI8=lj>MeA=xcp`@p$Hf>)_;^Y4{8=tTVP7IS>uFB1Q
z^?Tfc-tNnrw75m*En^elIaieK^h)L5$NDw-AG^I%5BM9Ljf@i8v!VJwE4%WRDVwhy
zShsFc@@Y4#-Mh@*-nzQCXZr5w$$!5u)-<cAx}4TsmLu~tFzM}8^M}v3_gb!vG)X=5
zj9Z`i+uDprZ#I2=SFQB+=Th4!zo^ZLUasF~x(Ba2y!K4s5;<kbt5=HO&TN;_oaG}i
zDgM-iUl+eDd-wU9Y?Sf3o!zfD9N4h?^R8(nueZ7T7FMU1l<VD7k2tH|5;yBY{*|En
zhVN8&&8e8?;KI+p_Jw)g!zurs-u;!|@!-ns-MhZe-&k^U597IW=TzNft`rI0PVbqq
zuOxH1QQqWliHVsE2Nx{-pQSnd)Pdm8`i8YuSMr0p7cCF#S@G~<%JZw&wOW0bs(s}a
zy!_$*Oh?D($FJ~8TYn2kxzT#|yfokYtMxB3*0&yS+7$Z2V%LT{>A%jDJuSArzA}2^
z#0gV8KIgo9bb9Wuukr?oQ_CKITCTZ1-TQN&<(@Ti$}=x-kMzvH@37VB;HvH4bJxCq
zZoVzM{hu@Y#~P8y-6lB!t@ZOAQZ6oVbyhMBd^E4>|BWqgE-ufP&;NJrg68q97pKqa
z-EpTf;Ju@>o21{_J0+Hfc75Bk^Ki9!p}vR9zS`G&EH_oOU0Yo~|K8#D>RfqawGZLz
zS28<jIXeHpDx}1_c)J?k-FX&F+L3=1udO?|LpJ@p&MGH=wTkPvip--jGm}+6YCN9P
zdSX}HamSbwGhghvtM_Z_z1=xUcRj^>^cP=Axgh+KCrT+v^x94CjV>-;%VWd+pYGUu
zF8zj5Qc+#0=qa7)*Lx0}p1u70lXdIzqD#Xv?p$7cy<}eNW<kNln>^?3b6M}L;O6JT
zwzsdt<MXGUiPhh24*W0j*l4-Arn)*~-d*KI$FJ+(sZg0~n4I_a&&{lDYZN;7-CO<r
zQ(JIZ=(&TeGWJmij6U}L|Cup6tysM0RCDHDMwX`67e0hf@LJk0Z(sNGQ|h}rJA-Qr
z&R$lIju(3yWa&Tc&{Xjs>%XtOlX?H_g>9`<T~Z7LG(EDE(p7f8eCoJ9?6~=l^VSng
zx1ALGRnhqUH22<^8%*b`+IF%pJHLKW+1v?53pZ9<Pg}kEyGHKfqP6$GZ}5pO|FEmL
zf0c0HqH7K+f2-#7SDbV3Irz(0YPyf!tr;Edj!|w=I|KJtot(Z_dH&u@Ld6wM@74YO
zhMN~Db@_Wdko{D3ZO!!dj*dt1M>BMCg6fOP`yT)K`*&&3A7Pdcu@-G&*2h|8FWxB$
z)4m+3>!W6FI`gX5&-16c)K-4u+k1P4?n>p~59{RKbuGPnCal!-tLEQ@JZIkAJUKUH
z^{$m^SHneo)>>p9K7IQ1?c2L!b{5&$*%=xduC(4^<i4-?@3H$`4?8<HZZuZZdXnJl
zlCno|xvElCMcVQ`xu%bouFZLPi8Xh|3Gu{~m6wZLTy`yS$&Klm=JEZ8#&j(qxjFG?
zr>uE9)%x4Jr^YT-Yr`}r3r`F=c_Z!m_Zb_^f1cWsD(0QUC3<SEF8^VlJ1^zkMgD#`
zR;r|A^y0)1l{jyapMN9;GnZUD`Z9BR-Ri?v+FGWk|K_z`Qgq_%>$I~sD&4(rwbyNa
z9@|w}UG;K?@y@WdAGRs(+L*TX%H}7X7n%K9T#9dXy#4ET|MHS|<=q`G9vp0zlau@Y
z?(S@x%A!~Od=nxUEOb3T=j80xb#d2z?0q2nCZcL_bYa}-$W5$ggJ1Lh=<qna`uqLa
zHM2A0!m_+m;sfp-4=wso@MvmAk=DYT3_q1*b#uduEidx+nB96hh4)uypZJ6jt+^J3
zkN*Dt{<UiF?Af>1L~fomDd@dp{>+)m_1T{$)SbSj^E!IZyMMo&t3aa;X*`#A?>@gt
z@~?H=s(t(GS#Kw8-N|mex2xQG_1R;wGc9kwPd#^O6YJbne|MX{xOY$0xVJ4hd;6X(
zB`c5L&j~)3dh6l@r(@D{ck4#%Pqvp_pC4PVbMn-&f~*|-`7g7l`RZJ+tUK=c_|Ds+
z(pW2x-H9F3H-B!a4etmETErB3TkMMNiWhJ8yqmd3<o~}XCp)&yR(rbNYJZvJ_D#He
zrSo6Eov{DbCCA<}pN#kK-rRdO?emJXF4n*19$c0&_cqoXYstH-ck_Cd^_%-a`fO%v
zpS+5DTzh(<Y0;8rj9;Ffp1wMCwOQ(^DWKaSEQ^obylE-Q&VC|gL(#E(ukBNf-+VrO
zYew;B=ALhUpJq;d!mp?1$8YqkY})n&-eRVI>%&rAsx}#?dwi~p)H>Fp`QhV3ZKWcE
z^h1yK|Cax|x^$gZsKLviDQUX?n+m3XEdLt*<;V<=>t!J_cNX!TS^pv-QKERB@}h0N
zKG|sx)z^Df%@Mobl>hVURO6tkFE1{32r93O*=dw?q@z#PdQ*;;O~8@DBb#DkeAYPr
z`nXa3Y_s0dbe>%sOzaC+_4At+Ms0Om7$lH+HLBHX;+}i0USFRk2h9n~ov=zoXL;yk
zrK6Yq=7gQr5}9MA7PdOzW=)mg^3ZILrIYO6zqVR<{Hdv)*0akT@jWZAdFk5OPdJ)+
z_0|8HTV8AXlvlp$cKmmynVtVwpRBZmghl12CuMJMd8%00F#KSQVG-JWV)?J1E-q7Q
z6ej<Xf0H`lU9Rxu(s`416vW!w$3C`F>n%1ue=o=`y3_HzcPfvdd_|zKddKQ;-$?!4
zEt9>^SmxcFwj`WYL|FSIU!LcUzaMXEiKoxpzWk7im-&xPg(b4CW~!4{JoKH)b5wF!
z%#$eb)Xo+0+D~uU&eE<@O4{>UQ^RY&^*jv+>u1R)ENlaHrssYA)6ONTwI%!dy6o%g
zcxA0hE-rEvR`XdAzyF_n-^oobN1~Se=xyxp-Fa^R^~p}DAAkC3e@lL>WhbJu`qzK<
zKKpk^v_j%@{p0u!A9MS9^_`ZI^N()hKfm?vOwXTOv|L|)`oI0W@xd!~{=L4u|IyXe
z;gi+<?P`8}`0(&>`TKio0}^cc8|F8~CmXN&T6e2LN$;SCOVy(1;U?btuX$Y(89~Do
z@~b4p{VVmVBfFd4?7Zyxdi_g}lV+b2l9PD&50{<~iizamF}l-dw&{qvixgi=fy=SQ
z)mILmdv&5o_n+VX?=vS_9A5Dj<o%f`KGmT=b$D)`w0hzp=z8qhDb1aAmp_!c6l%xr
z`f_t~`tvh0jWaJTd3AMl{LUg(Q}^deWoxW&F5LUPbLr`Id%c>ucgA;CZ>hU{lke?a
zJ()eLd_SB2X5F}PWAd>c&ERD|-`?Dmx2-a%_>geL_`vzYh9zxFpZD1rZ!4HG^^5tm
zv*%>&lX{F2Egt2w7Q1Zu*;kijnK`vdNARh(()nYT<{sa*p&;2w#r|yn?;-`esQ(Y$
z9w4n%n$$Dtxv}CVr7CZis>e%wInykcY?jjS7M++|^Yx)g)p9qke;dm_1{>-r|8*{X
zeQj;*?r+C>r9n%p!8iJ_sH-mzk(hq=Ue=ErArjBkBSk*>o!|bOGiq(w%}uG0Et@<%
zJZB~<AMcN6W{5vzSi)BJW0RkE&hs#@&?W!w9Pd3jch$uYE_?2IxE!0he9Py)?B{bY
z*-kdTKWEyL%j+|}etk2x*6e2#j+5kTmG-s2=NY$tS!eZ}efKqO1#fo#ire^pl84G%
ztJ113F9Mf(P0hNx>g)Uc|J__%IApb$jiq{Lo9CAm7w;~5dg|up^zgM&tyOG4Y%^Z)
zJf7Rwabm(Fh8g|lGMwg75uS2)epQ4^7*11jahclX`O>!h)!K<2vZ3yI-adP?)aK5K
z=(V+8wr|ms>7lbufQEDhH9IWgoL!D=aOs@YfBb@)&rznFuXj#f4_2LLsoMD6QfXDp
z%_XjRRsu&tPn|DYm49h-2HU^FAPo`cHlE5aFDCBYTe~u7>9JmEacODok`+DAZ){9X
zJv}XUciG!%y3ya>-;ZD7A-GVOF(tv`^jt&Gq`tPg@}j1tQ~&yI{hVK?@#^pOFJGm-
z`1k(WFni*PB<IV2KK3-9+L5-}%&2o`K#%VHXuI9Z=REqdc>Al|IijpZr#@+Joo{vb
zXk?iA?v|8I6Q7<q7U-sPf8~~*)4J1zf;aD~$<(-Ru{_7a{p8$&qSKn0Yt8nlSJj6b
z|6h60ZvURm%HA$-rDq5U3T{83?>z5?-^o*_vX1pg*8ct`YhAWx_3GCT4mMYRd!sr1
zbnCgPjCb}_7JqqhapJ^@mzVqJUs*8`v}KdoCim;?OK&q*|9=*Ga|!3mwqM0AM`mm|
zZKc)!-eZrO$gHHNJyX16FZR#d_N%_^8ISz={i>Uc(>Kk!@Y*AA&5Doe{?UJzRDXZ>
z^T!VsHnu)F+g+uvuT@yRsGet2X;k;;hj#e79(ntH+1J;t{PabNLEv0bdd%EeGuFLY
z^6KFUnJ@nfY|ch_i?|p)a^4?v<JIb=>*`j{>(tQt=d9{}^t0om70<r}u8G*#%+8;8
zXGh`JS6AcW;*R&ri(5^6v8VEL+JOef&{ZLY<>mE%eiVLxcQ^1n!%7a`xq-G^&mUR-
zyTf_5z4}PXq=%v@f$uB7&r*{uN?zF<61>o<_4IUo{}~1cj~`#2cX!wJyt_hH6E##+
zRD5O{9X)*b@R>6@Dk>>YPE1r&TWQ9x%b;>N^4<L@hZW<$No~%Zo5dmfB!5ZW?<cQn
z&X*frOiN39`E8lZ@wHJ~vmPDkoHuWtu$s?_`S$g@_HJGA;_I)gtHaOFvE1y|D>Yf&
z|J-Et{tKN9M!GW2|CgPWzHW2ty0zJtWfvUly22xmZV^+vJWES^R?Xf!t&w~$Tb`*+
zj-K&r%2|o+yjk~JS=Z+8skpf4%bIkhq{waQ=jZjy*=hwYa(Q}cDm$Nygrwxmd5kQ_
zKmS=5yZg@FyP(GLn;RQ@rOg-3W4t6H^~*eTWl-wtWm~_SZSU*n5VAVmfAjX{o0fS;
z<jz_izJ5AZWsgzM?WLU4z2>|zKEU6s_t5vjFSq0xwl6O(#!g^ZwrtsA_x`#c9~O3Y
zcJAK&dx~bTkdP3I=cIFg{{GbtTeBnKVAE9Xa5>wmD_5_EzHnif8PE2^J#x#dtE;zX
z-E3OWd-iJJqWKeNDKj4z4t%)i(vRPsll5Ou|EVh5Z!+uI6fqU0N6wFX*@S+7<*QBd
zd9dqPZFQN1O~!%^o|{rm8s*>HGuu4huKwSht5-{pc8Rh%v(5hU<@w#c)!PFXyZ!z3
zRobrR$Jf`_SBI=rDp6|K&noq+e@oWYQ{~T|Ea3H>loZD`_v!K$mW;{aix=rn-!1T;
zGq2gqF2i|qpYCg43H@m=bw58lySwu9GiCQa8M~StDJLgYe|sZn6{uDH^V8Dp+shvx
z>y<Xm5|NR)vnkcP?VxW5!%KGFzsvXR-Q6c|YEe+Docl+5;pRVGJX`OuF8AlUcV$!H
zS)=0TXGE+{U%zwbPM^H}zM7v!@$vV)rfSW#E`N7?z2M##FJ3%6+&*19d|koAL;dph
z?=COz_nBt+yoB|{i3=B2COG`BYyMTizBYS>e~*~e>GP)?PRB~{?s~BHtyYyt#KxSB
z7b5qqj@p`Ko_9yWrsBi?|Nm|;@tnM10Z&2sA#+t#)w8oqSG#lyxh*!du&6lH!uhhs
zU5=rqsm)I|*{%Q0rFDfDel273opT|wPKQ~y{GjyC_T9UtX=LoId>l1#ep0H$veMVr
z_~mR?gs+b~+{U}R{Jor%lvc@#_ydJ^Zf(u(7T2$Pe{ZjL_&U)2I`{6yy$E2aH~ko2
zf5sr~><lsa`5WqkyOU(UZz>92__!wgQ<3;=zT&B$Hdl6^DsIhEXYcspG3)uy_UbC}
zGmGwTE)I8_xyDOv^4Hhb*LzRbGfqGE<<qCE+uL$y8l|@IeNEoDaU(Z)HOudPwZFZl
zYN?jUFen`AQC->n{O32>-M{YF1a7%o)*0fWebszje$@P{<+s;wKd(POQu(Qsz@4V2
zp4&TKT{q7?^J@3o>-FUviZ)gT7v|O1JiBs{$148nZ1eoIb8~(^I@<k&-`UyOR0_0o
zp~d&BZ_vsu#m~=e&Au*X#lcY1*5;S$ws@oG=WE;Nt(Y5?J7q(XrR&F)3-|AKR=)g8
zWnEicQOT5-zgLT&pKD}pE^g(m)zQ_Z6}RWd!De<_8ynD`FG=Qg&dZHDT{dO^PFtRH
zX7S#=6^kRz$*I4pSUFdx(r@E>mGhcm+Ot&x)m=8duc)iq_2lc#&CB~!`>NCy)&2cd
z`sRk<R~`m?{^J)iofcmVSsQ5HbUJp&MW%(DQ*0KBi>cm=u77*#r%&<nOaK1<Zs(O2
z6A)MsqSbnii9wQ0%4)5Oib}|e80C9s_Bza6>6Cub<IwuG-8-LEzx`Cn&d$DY^OC;r
z@9$e%Tj$^1wbiMW%PF0)p@)TcZri>6_3_(kM3ehwRj%$8vhqGAP*`2v^1nq}{JfY(
zfWr%&mB()1zJ26K%Ed*lPV9^gJZ?JMjZ(eB!or@PlhKde_Q-o@<vXswEq%&m6Tkm)
z^6%SO@vbCo)pKEWzd3fbRt5$EFBBLie0<=yaZ74?dildQCwJ`5KD2I~m{s=*S9`0k
zSMm?6eB2INj8ZZ!Wb%Xw7Z$tsi)x1@MCmd7;I%kCS8kqdb=a}hN!PlPi>Bo0mWUZv
zeSdSQIajt%?%w;SPgP5%UAeM7KYnY}-n6q)3!51n+FCZ>d8rt`ujcu=IUiO9v`(F!
zR+^_#B2(DOyZ38C_3^#Cx2JBZ+E?)K(7nCY#_w3}GALM>lvouO6<O6j@_3rHHDYGx
z+GO5^k2_rdy#D=t`<8oK>O!|>U2QoxX_kpp@AmEAA06#pyJihI&PCTbFPAqr-@ds0
zoBh4p*3VwDF5KMF^y}^9YUx?J@o#Q#NKCU{Z;*Fqhmfk5N!Aq)P*^lRv?{5xE`H{-
zc-h*nZMXAgo{oK?<8)~cH*Z96`L!vAcXyWyTX}2kE`03P%*Olf-rnr=boCN3h7$)0
zc;?<)6|!<u>E&C$XGb4*|Fbt-W9!F!rt9WX)=3G^R;TZc1C6ilEPlRd9^(W4g$r2&
z*2nD)JuS|bo3pO3N~J`M@r7`5(>2pq*H>R&(8%0!Zql!1esgD;=if8Sy~QER!eC@G
z=c(A4S*F?Syz=3DUMC9AzT=ecY0s8h^WavS?5xoF#?@^XB7@=+5)!s%U43_F=VWDf
zzJ;3^9NOBecd}mInmzqTOUuV#>78ZPs|?O-mCX5a;N|2d^Q}2Cr;UTJ86Pi<-CZ`<
zELZB4BtsDExgzeB_51fBm4x22jHOEWw{2Y=vg*O7r8A4$UhOP?9u*l`SzXNyavkTK
zr(*XmJXF4xZ~yAj(*+@!iY3#wBt7BbFDofJz52%XYU|URoQ`|eDk>^IJvDWA-QTKv
zdn!R$WG0JLlt|V8+xhoDzCLR8F7@d$r*zL+oAdT}?rvTXdgtQd->F73FBAsJpZ`2n
zD>UokBG>!-YQY-W&lPnan&kR<b(Ixser~o&M(eprzvN^kG7j6@mlXIfTa&I7wJD|Z
zV#b@x%lj`lGAINeG~5y)(R)noZ1nxAMP*rA0}rLmv;HwpM$%Pq`4_3Ld#k@2Jr(lT
zkK6O%!-pLO51SS*UVI@Es?S4ZQtWMW>(peYZ(pADx-ERXr0@CiWg90xUl+eKVr$mb
zz{PHyvg*Di@9))qcyMq<I)g{4MVpv-uUqg|x4RsA<qzGsTh4u2?xe5dvuFFmTT?%O
z`}Xbqy}iX{W%u@0Z(qKA`GuW9mv3#&7T1rf`TFXrPvviE1`cjsU8}6EyK0uc`k3~P
z^_7pP)#>-#ySIJ2yS4JV-u&`{0*`y36Lbn6AG^D+Hd=2w_iWv*E-o%RcJ2h7dQ$Q6
z5vVZn;Yi=<>b^8+u4Sdmw{MR=2M4#D`!riAFTa0}#p^>igXh1R*)hvJKW<0CL(O2L
zSK?p)SABnXcXPV`6=Q}af-=tCAt50lFE@!!?aM6=f1_VAEnc|t{QIL54~n0ks~5E7
zMELxwudk-go_+h}<>hnd&SgD0t!rEUeY?y{OQN^urOpawVo)&4$oN?G^;M|m`A=c*
zYGrJU&oBk#JNwz*yM6ipm5oBGUNyz-Q?<j_#qIrdX{oo6C!1aU=WlPL#l@$C?w8Sv
zjRGA5k(v6SUe$Q7jAc=bO}W*puJ@~>Z?~NL6nxKi_m7P7{>AOz>etWfTv7e~-LhrN
zK0iP2e?DR5y7>KdZ*OgNwq<shb;xkb4cm$j2Ua^xy|Xnv6;$+0`!6J2_4?nlY1QSq
zYs1#ws(=6U$w^^<|MS_`*F8NwUB7nI(sj-3{CrX-FW%kVt)`|13ePh<ymJ}v>?~eh
zv^C1?=xtBYX-?@s&6Jl{7bjnx#=Ba|B;$ke@AI=vUmxidKHe|y|2ko%o|)OTn>Q;z
zKRYX`9VTMM&5%-*@uKJ9x3{;Wx8)g29sMo8I;}TrYu43YC+AxBX$LR!nQvEXRsL?y
zw1q*SOB9|I{hXxg-NMJjFjHPiszp#hAYgOa+1=&u)6ULHymrj4@>9z8yt_=y%xBM@
z1?LTUDJd4?tSc)fOqej&x|~b)$?_Ffv&6-xFZG`O@6Spmh6ek_HnxKqTcg6mufM#!
z+&u42#0wq!YwP3p@7Mu47MoL+h2fuAMuxzpR&MdA?RmBD@7a2LPrS!6{SD~m%;;@A
zTwI&BZ4<NNX85Cf;ex^j(9MO*FRzW+8T3No{?dyXckbL-<~#e^5>MfUn;8zov+(jV
zb@%s|mzBkB&x@U^720y{Q?8wzo&S8hyBm|+FE}zhc;D2<_E6TYW=H-1e~;!cKY4a`
z_SaWecUOO3w|DQ}3y~lNk@k)irLV8)#_sxZV`FkF-zQT@v3hfJ`r>(v3V$C;NVz<G
zdwcu2bLZ;n>g43)oYH@Su8Ry`ANTe1XJIQ2hCjU*E+~l2HqW1DntiQT+I&gS$_tSl
z9iS6?mU>N%-k!HK=BX5e!rcct<{hrP%ib0}I|I6{cww`EV4=+Qt=ZR4PF7z$kI_NH
zqK!?Elb5%&yxhM2-=4*b7jN)<c4lVrj}H$gtNR=ADCY<<{K#Io&@o0#KW<O%?QO-+
z&v8cd_RHJrMQ?j^b#*vsCsPO;1H+uBY=u9c&#&*uczJ256YGP@d$(^_Us(~jBAr1%
ztZ$)XiJV<c#6qXm+2;ArPEFPRd%yPOrKQs5d3y>THn}w2k7so7$jA^_tQov4W>?8e
z(3Na|_UXs(^Vuix?#YuW=kj?OK8VVwHc0SFo9Wot)SRBC`|{5|+v;yJ76l5gBpHq{
z9KWCtWS)Nyv?@3uVZqw9xxe;2{`K|s^YioT-`+BvXfuJAq2AKOq~rR9h0gJNtF~5t
zepXVk{`&g(-Me<lS(lwTdsbKW9v=f}CR8EL)6-K=PmfznXGQe(yr})*mzVi=i|JZb
zetNQ{PMM)$e#_=cg*Y`eHSO?qZ|>|Y4hWdAb7$qPy%*2TwU)Q3*iicV+Lmf=hKBf7
z>!S`=R)wq#+MMPqFF*eY=su2;@9*x0NP=z%<mBXZ5s{U>yTAUwY4$Y@k**L)Nd|>b
zP@LWU{r&yixpS|tiQHWJ`daDhYg4sCU)|c8U2pO1^mP5tPfjMMr*E(P{ES!H?8eQT
zXXjd*hhAc0SRyv(Dci=)n>M|=vNHJbvEE)cWx>1q>*qiJX+KB!%KiKIAy;S=K0any
z{Ors3@BVYwFfgoSPv7ZyWznKVQf4_8`S<oLS)#HwZ1$dy_x4uTTfCX5>^{lAP+d)J
zU(L@;H*Z$n-&b32p~m3w%E`GIbdlq`2M3!wJ3HrZljuEG{QTVChZ~mr&;R!&eQnrp
zP)XxE3v_P)e>*FKhigWLfN6aE{cX9oXU&>bcWcs=DJ4HXJhYcIwz09%(|dQhU8wWO
znKL=>@9nMp{Os@KL`H@naVaSm--Q7=_x9Ype*OB9BTp_L>lWAlcXGzRfB)vE@fsT%
zo}6!A@8IC@_*n1%6d#6$koLAV9?-$};(9R~wr`()`l-v3%P;5IRDQa>xONvuZdTT<
zC7zQH9B|Mv;bc&1X>03Qm~wK`!Gj0q&6~Gu*)q4qAOHOPyx%tDlWRtB@MTtRv7CE*
zB3FhiivyY6{CwfV<wuSm-(U6B>xIaX&wut*ex9ZuKTkh?pR#-3pJN;oCr%XA4%@PI
zYwXsnsS?*V1g#9=<KuH$4n9%ou>Z=ieRqCphZjXga>}-JXv8hPDBWwOnJ53dlI@(t
zQoj>fX=)QY9<f}y{9=Z<e%zCDbE97<Feq5f*vL}XrDj;@+nKa<?b;T;NjI<TueJZV
z$b#WOxJ#h~BLf5Y;2lyAY&tY2=>z+l)i?R`RFXR_)?RcFYHjN1=<b_oYpAJpHEr_L
z?J{Yv=gl>jnDInaIdYofIf3aKXXp8r=lEUO_j|uV$mGUt^LE<vr5d!f9g(}StXA^>
z{q;#PQa87*Te_t}^j^8qclCHqh6C#l7@GWP%&DAubH<Ll+4oqbr}FXr`D~oOZLad}
znyxh~e-<yjz0+mw#xRQ;tG}<U*j!clr0cBIrsVr}iouuTYo+h*o0`?T^^#)sqkW=*
z?s?wb;b-kK`y$uL^BiWsd(m50W!aBXF@{T`Qc`vEkGL)S7A;l(v+r9jli<t4@1*kH
zR23VzL>-yfq0%~c$<8zOZ_nJADJ&Rmq5nD4JI_6G;hOpF!B3}JS}*<6nbL7WZ~YOI
z#>-M27J6?K#XlVoC^Eg4AOGfRbla5Un|o$nIU1Uk?)Sgf|G^fvc_nQd+N=KS3hsVq
zl(O#ApB;y$#>Qnn+%v!Zk4S(0!v3q14}J*!x1M?RnGTOhw*wb>eboK+X*L7HhtCBO
z4ExW&*sn9IW76s=9g3`ilHdGlv%~JyeJiqjxuk4c+k(9h-EY46e<gUvrZyM9tW%%f
zGdZ#QgnyZr93C=VA~}E6j$en;TQ_|^vPM+4L{MsJ^VVNCnj)5YU7MwMY{hmX>*#+s
z7jEBm{=L##d$rxqH(xM(f2PxJj%Ru1V}lKH@`4AgECOy!{g7$CS@&PK+xx}hd4?bV
zxu<$@)=gftZs)UEH!c*^MShaZwQEyXGVb5wvV@;oU1@Wp@!HR`KX65Bepg94k+JQu
z_{WZto=r0j&+-+T_Vn?)*6(xAY?(b{Z<Lwuieo7&ZtiY9z0`MCltqNZ)~HS=$0JAI
z&;9&u$*H=UcK`b=8zQYQecaE(kiurs)^;WR*5bV@zeGN1I{JQoM#?$fv-k4*cYbYK
z{-ZqDdfy$%6A{+4pPkEoIP<VTi9yyMpWZ8RQFpAXr#ih~R@yD`_55n%^&wmTh2F7U
z`rJdSR&HNG^p~TvU1Xfqt|f;(joPIb-o2>D?fcKO635ptOMfyCZ|j%(RrE%$=);+7
zH^01X4R>LC+xvN9ww_SoEo*)alPTUc|H`i3e6lcSna+u=Y1v}#k%wOT8ct7MS>YeO
zea*(#(_G{@(|ov7GK0T&SGc(7_`ko;_x?l8Tis)aJZD<3PW|a5vg!5N9d({_Iz9LO
zDF2=J*+uF<>v=2x;Nu(9cxTP5{poA)|KiRPhKBVI;(0xCd-krHpt?SEhIzJ1FMrSO
z8Y9gsEBZYYoKu2UOx*qLf``mU{j2<spTyM``{#eGJ0_wvVYa&LdPSv6a{E$Lt}SV}
z<1yvQe*g01`ahisO4F7vnGtwz;a+R|`08Ho$~x=qn<uH>{cNDLX!>*0s7X<gncwG|
zPFm~R&6E>nt#oZq8f)fft4ZqVUqBxD`0@IDuZTq}8n(aY7o2!T)m~6j+iZ)*iKF*-
z%GK*zq}nhG3VOGh&(fH;|Nj*anU8^i|G)ItMClygb9I@H<kyrM!K0J?Y9E<>EKl}p
zpJ1}x_2T;doG<^}XJjz4n)9?RBwZ=-q(rIZt?w3sf{#}eoI55c`1`2e$Ir_@=I*{V
zO@CVLw<eAK<t|6EC;3Ocp8UW6XNk~nWw(<8ANTCf-(fE}`T5~%DvvssZ2RWo-yy`?
z=@Mk=`mTdFJ27@+cv#RXdAFInwx3RWkfc8MkHw^r{rPb%FTc!rYFpgb!7+QAv(jWC
z`Cr${-HN(TWrr9n>G=L^eqDThXV1*1KQgbGop<`(dty_~zvtSoz1lj=SMj<;Dcuae
z&*FUHwf)a+Ob70>9ltPv%VKV0N5$W2C8d9@RzAw5FEfJIvn0pgt2=2h)zW#H*CQSq
zLBpc96F=4kTzhG2s(svU{;Ts7)~bABop@)@M0LTyg@1&+g6?w}NOf%eTOco)ub^af
zY{{B8D)TzNyw`lSM{T0WyEq}GMSr}qFSXrKU$jGby976T<(JkIFWW7z#Yjnu$Y=dA
z5fOYD##b-CY^tT{$y+Pe{JR?6ra2+mrJ6H2-z=kX@AAp9T@^vnZRQ12{zyAdn%7a`
zYuUYL*{z1)IkBEQn@(Jr)Y0)|jjFRt+Eeb}Cv$g9|Nq^Wf#HYfn;#z^x1M8=JTU9&
zX>ZRTbJR}#|1a|RqM`fkO>2CV&OX=MS~FinVp;XM*;S(9JEv`46*+77reAN3r@zVd
z+FZnaEhzQYhFOPvH1!ppuIY{rFATO2P5H@gcv@WTRCi?f`t@2*GeZsI53Rd-tn05%
zanvS0S5==W7xylX&1CO%S*G!J$=-^-%ff;)`Ddlfyc;v?{M72YPj`>9gkFuh7HSjv
zb(Qx#+3tv{o{kqsttYdk3Wxb$-&1*HTK<8b>pg3}_}wd7vN!!xz|k$@=U4Ub54xh0
z9krup<&(OaB(YFO@zuvleMApFT+BDiB;%jvz1<1yyV>-FR<dS$b?qwk&GC8KDlGKX
z@@&_Xtvc}y6E#FyPkG$iJHcAq=;NBipoJSNr*)W5UZ#G}eAyp07g>&9&&<nCzg}@I
zcGr!YH<v!Fw_!LjLt>u))|sjb!rI}RT=J)B=jML>te;qH9T;RG{`Kxp2Z`N#)|g5<
z&8z=&BQEO1F^NwPzU$ra;<7e;FIS|Q=9T#7%+Dv~;WH;%J#1I5nfm;7Mg03*wUbMY
z_SHD|*k(%~HT3;_CQHINwL(dAr-=T~hvp0az5V_B+&t~5Eid}_J2Ct?{lLm(iRinh
z&HwK;_E+U~c>OF~qNws|`~7ngpT6GyIKS$n`m3M+Ki9~<f4x_@M_;(3<A;Y{*NQ1~
zTO!)}pUBmI<UYo~pOb;%e)H=K4E1N$?5SP-Xx9pf&-09RPN#`m@iL?cS+ueJNahq*
zYL5}HVqr)zvmoO*NJ;Mp%Qbv_cE#_WcluV)3kCZ$11a7)ftsf!>~Htqb6XvvwJ?~W
zK_c-f&kxPQ!or0M7dkEfq51FMKkKqLpbOtX6CGYlPt7)8e?gPMp~vW0AsffJLx&bc
zY)k?vJU`FY*xdZQMkXV}KKAPab*Lre^s$K{FBlkBIxoK*ySuFS`A^V=yho1Av9F)k
zu_^t0+=c|lg^L*({_wRuuUu%%u%s#X_O`8=mycb#Wc1U0R^p5sHzM5KkF)d3ameyA
zJScx4A!XEgYiqW)NY{-UH~gc0e0-|DyiiO~0d;?0Uzp$-5g9pEJ6!Ln*y4*fmibCA
z{KUZUN72M2MGkcCkGc8(FAKZ7y}!M=$$EyF;e+~x3ll0$GA=l*zpnm2a%s@lH#dWC
z*f22EGx74yZ13#!oNHA&>3@!a#4PiCy&O4^9c^tsORK-XJE^|w^i=Ki)Kt&_dc*y8
zkg!{?l+Af<5w1H&c^M>S=WSzp=<4DEx@mH&n|9=;Cp(Lue|mmiUP5Aq7PHsVAa!;1
zw#1-R6HpgQ(%Zqq(9rPlvEJKzDvP!5m8`v(@#yj6w{PAAWglp1VX3bEUHSQ0?N<|V
z2GE#?3CH0C1D&`%6{n_X`pz=p<ls0lNwxbz=f3XeKV_}Uwq#yDcKLGgwKb8IpPmR>
z2{QZ<Es&LBDfsarF+Dxq-hRKbd*6>=zq)qL>+DFJ5w<q!?!MaH1rHCYdQX$FEJ|^5
zYPztHLE-b^$UKkm@9*!+n`B(roPK_p@9Z$p3Am5{Yzki=*DGy)?*9GyOG`Xsca<DG
zn#tf{osr?8e(~bP<l}vJcNRb2ka!prF;mkXXzQ5g-vdnsg{_`zo*&oBCEC)*%rH~<
zT#@sv%dfxg+qciPOT<vBcYEI5U(e^)8<p4!Pu#ux_sPlXzrVk~zpwW9v$M19>;J{P
zaABChxp1Lk?$4h;Ev>EPt;^or*qB`W?99fE8wITbS7qMY^YiWP?cJi<&mKN}`1rA~
zyu5yi62pm*3l}DYfkp#1r=4Bs+<tD3W%2oWwtanlPU>E@UQ3I9e0UhXK8}~4zr3U*
zW^2~eiy0zTA`B@)JNS54zJGgr`(pS0d%Mfm_xJa2-(K!!zx?myy!-oXb8c)na^y(m
z=Vzg-LOfqsFid#(P(mtba@_j)w$)`5m0Xtuy?lFnd-b<BKc`<Y7M!XP_~=Nda5Fpq
zyE{9JC#(6+v#ZqtIqt@V3lp+H6XIJkE?!>lKi{UZ=-HW>_0JE9`p!1nTl#w1?Ag&T
zFD-57la&$|R|n}>apA%QE+%GX(5j-MCnt6mKYw>?>*|1oPEP7VlbjYxSe3lkntfeK
zS(#g0Z%@t7qL?{s4fh*fU-;m@Xwf3KUa771|Nl)?b}xH-YierbLgCKSbFItU_~qrq
z#X;v8XoargkY#5u5|VK~oPX(MiE;WlouDNhK`ZwZKlcMq4F5lW_Uzjm8<S6#pE1q4
z^5Wy;<4)>K4Lr=ey!(^i-`nfjC1P3r?#|20%a{Al&&$lzEKv%~zOv%t+1cjaQ?;JH
zd}&$n;lYg?5etPG9oQP$+TuaG>Q7D8?w2%9JJ`f(UG_#JV#9(9oknb}PKFs57J$Y@
zs=uGRcoB58fXs1DSuO?_-s=k=rthu%oc83zL{P|>W`#UIH@DF5_x8JctIL0UcxY&7
zxNqOStE<Dst@s!^MEVv!JiaCB?Uu~TU834)d3klu&&lTJ=U@3eUn6eMkBQ3eZvAq1
zSBI}><CU_h`r`4zg+aj$+6q{-E&KYqW4+R^OLp~2o15j|i&+@(=Uc!-bAFw>m0zA+
ztzLEL=;m29|K1w>dG&AW?!W8j?bSQ^e$wSXrzQP%ls_-eond@UKmV^+*{6PQ+pPKg
z`hkYqs@msVyuLjrYI{a~$=6q|Pxi}pZwW8cxmokC?84FRhdz4h?A%Iv4=((A>wYe_
z+B5H{%#xUx0~5bmH-)Co$+6$(^42^tqF~-9zia#dZ|m#pyLa#2`uP3tuB`>#Ens77
zJJsvy`+7l!2`?W=NbP%lZEduucGw(~%uAD0y|?Awo@QVF&tAr0dG%p2)<@y%zRcb6
zxvcAKp!DwE*qucaQ@@HWE&dbly75}&rq}-VRUePKtz49{z5Y|Q=GjjoCr{ts{W0SE
z?#?ac`gbl*I<z@%?vkV4)8>SSYD_l%85?`rdVS2_6>fWtxN>jYoBH}0Z@k2egdcx)
z-I#E7cQW_w)pg%%?I)am%FM>Iqw;fFP0gOgiytp??Jg*kVz6^`K5V|H?(Z(|>3UyZ
zUCq9_>gv&MaqX}*GoqBAYlJj(G>Na@FZ%r<TgQ!t!?P>;RHmM`Nd38Gc9^WV_;SC0
zoA=h1DNPOWx@GaT{a4cF?XT{iz8ZH*E2k*!{LG{ZJv}|Y-zy(m^mSNljhegZ>y}sk
z%KL5hZiwCQouw2c`fBfz_Kyemd_FHU$^PrJXE!$;t`-e6TvCv6>GgfKrB5~U_w((q
z{870#tTmKpDVw$RZqT{eHWdZu=h+_Z7PqhaQ?ZJNn?X`mN^0LX(1GKvTsLpr0F5Sl
zPuCOj%=$MWW9ur%<XSJu^LZ9!{T&@w*v0P5*^|IB@z-O$kISF)|4F|zYiY2xXug@Q
zMW4CA#EbhocX#g&m%RR{_K&kL6KpB??<azj;-?8zWjgH-^sYU=WBQAtN0EnmeJ9U}
zjjdJtv16KM@G_VA*%R56swy&E_ZM<6|Dyl?TD^0|i=&Z$%|Gn=>71U||3D-A=C8Ao
zCtG^OEtD2j?021ReO>dL;UqWzDUT|R-d^>s-evaD<F5Bd$bjqP<Nfcit-bx?;^N@a
z9EKAUE?oGqHU0d&shYuQmzH=wd-iOlz14c-@^zQC&VF@e%G~pokK*Uuo4>qrYLnP}
zJNddQ$NyjRxR>!u*wy~lnZ7rd?d?@Te;4KLF&tbLh8G@RYT&H<`RdBr6CN+Rxv#yl
z{rYcX=FP?a_wDZeS-UW}Z?{QCOqay+U(C%-=l&m_zyH5gWB<M@IY*wJYp&v#GCEa|
z^=#X8RyMyi2gM}!rbMiMqchPYe|9HtsrB^u&WAq>{<oUFwaM96+#v(oCSCJ;bN#Zo
zy;a8P=k`>8&x_*WW02&QlG>+O+CO{$lIzksr_5!~OP9&-`Saz_HRiRur-Vz|#d=(0
zlf56FvSD{uM@Q-EYZG_ZsfPHgtc*Eyang-9Uw4;0U3b6A`PxnH+Dewym6gX%?5&o2
zP;a^9#+=A~du>IyBt30TS=<&rnaj0NW$BLe?;S5be&sfo^GrW8BiefY%(&?eF1PxV
zEv{w6cGd6s{G@j^J73lJbwBo6Ke~6v=Fgt2)rY#SSRVcR|I(+~{0x#O?{6|v=wR_S
ziaT$(Q@dy7sySCyFT7Ix<!Fr1VP~bFiBme3EW3GQg0bSOvU6HWr{~=4j62oQaV13|
zXu{IkvPmy=|L?cqWrOZ2{<QVsj1CFi`Fm!HEKyfd>UBKrp>4h@T>t0QiRa^7>pH$n
zocVeG|7mY+HlEhZ-2HfO?#tU-#LeGk-o7mDKbz4Z>Tv$+3G0&L?Zb=NltSP3)plk$
zynm&ow9?1V?f-@hrKHc|nF%|}er3w|yZE1bGs$%I{Mi?UYk#DEdic_^EAHvr&XWG#
zqLob@DMiLglUC()U%ssOH>`anFQ@+bZ_;Ha7EBVF7~<s;BsuY%!?bUe6?fN2TEEZm
z{Cy$++a7Pjp1ux`)xF(4zYFGXi#}ZD5|SIz{3d=$*r^T=Jyj*I9s57JeY|xp^76lE
zR)%_O8E5BTmBpVAyg2xJz4yHISnFjI-S6gI70TXuTS_Va?7Z#sI;NeSF+Vp(etMMi
zw6>pH)*93#<ZpI9Sju;HO~x|Ly!ZmQ*$a)Up7!iG*kfV0Cgi}qNaNl%qwMWZvR@Ru
zIgt4-@l0cM=&ih2CF}04xRv_MFz2rB^K_RtGi&cw)c$;S`M-$2h1jC~ruE^?qPf%f
zvaVD+{gxH>xL5S;=f-nw4X5t(oSb^f=62z^<MTQ?mRQ=Vhki9#|Ky9MiFj$%_j$(G
zzg|E4?d|5=2dCug!ghIH4B6E!&G5iJ;hO)CTN9PtYieu1zrC%Vz~;9)Ec@i4N1vIG
zN0rW*CL*@#{~Z0+%d4(Nt!zD7|K`ip6H!}dh5WbGD2(b0-QY53nsTe7u)7zxtJJ(F
z-#&l7y(RN<BQyJ@ce)G;&Ib%lIs{h-)P`>Uzy0C!xNnPNMOMk5w>nh!B021!&_=Cp
z`8RwF6RZkk!M6sX9E~5-&c>Lz{!K)=X<PV&FlleD-FH+^PuGjJc-|lHE>e5H{^=%L
z)+*nuzz25=`rDS4#b?Fccw>L>#g`vl>|JZ;UqAHn_wPT;TNLL1d;V+v4|VH#&1?-{
znvydFCLG$F8oc@O>F{Hg<u5NEJZ!fn`*PIX>v8{QhNo!;1Qb47GwX-{>zLZ?^-q?r
z?(mqkbxQe4nf>;inLoDvJ$~l%ZP1*O?81eP5zErTCQY*39JM0H=GFHZA)%FT0}5mW
zj~BY!>hBJ!-I*jh(M-=z+w`x&$LGgO*V)zdU0!un{@dR@C88UTzWSOG{_gTcOL1}G
z$we-Hrv)ckz5A>bF~>?z?cL>?1-Dku{#KGZ`;>Po1H&J46O#^yPbIM_7gRs&(JSfg
z-@bZvws1PX|9Zji^J9HL7Jd7(sWEr?x=7>78&3r0&Y3agb<Ey98`fo>&aB>kzCich
z^PmSm{k_BQ^4l2hTN(X*Yk5aUKj)l#Tb^E$wRWqV@?zVckeHGOd+N@{-i`hJH9PFs
zYM+&z>{ng%{q?g<E05otYo{v~qvmycsSyLXH|Us=cBXgs^8RSM9b0n`Eqh&={_b6F
z{k?S4=<-LMj5Qfornb$^nfC5w)Rf8Zv+PPveN?_X@2h3xqf6`hw5pE3U$sX0-cG-}
zUCCmTr)Yeu+FSo#evhfK{%^fg+lx4?D<2$@dcJK+l+l{0O<T5RTbxzZ4E*ygZ|<XN
z`58fn8*hcbir#kh&mZ>P+g9xPmGkf3<+L+A)-NvBoV|Q7G@n1>>5c~(lTWP`N}JcC
z?Bx~5Yjt+oDF%igYBJ8v7lbN~T~UkP_NTO#FJ{-~?RkmP#l~&Ap+@^_`yL$?dH!>X
zc9`h?-TBck)s}|3$cL>_N)na7wzI3Y#64)o5?ODTYs}2dR^nT|HkM31bnDEd4h_XO
z5gq$n<g}HPHvb9rI?1H`c=L_)i6TYk)+SA<<eHwero-p1`_<d`msf57slNJqe$vD}
zA#Ta$a~`oZl(Z#h2+XS%-2Uk3{vD@h^{XygAHL>M^1QsR2T8J~nq`0P%$91t_9J4w
zYLK8{q2a_=Q>tqsB%M#qF!VXHn?=c3X=P}h=1U$nrN@``pY7AVR`cZDoTD{Wy=!-V
zJNWSOR<nB=cBP8bPBAom0a-LH#7$)OgiR&(df~1vX^}zeBCgioc-d6y&3gW}ki=o(
zqpmiOoa9<BA1_^}^Y-(HU#gqU<NGeIV&2vtTl09X;?YZ?YDLdHLu@oZhpttM-~7IB
zT8GBFgL%GRAMqhy2Y2d6@8a#L`uEG;AAPzf@$=-_%jbpXhDvy+GC1ryY-qyKv3q0k
z!DHFG+^?(aY>@k&pKLzugg|BX;bXyf9?vLkkJ)uq``gv7P`#Y5^?jzf!55bnD0*__
z>l8mdV^>=9$tqm6t?ZVK=7PI3-mx#6n0{ZiJNoIxnwVW#N4@%!-PdR3@3|RS9oc;{
zI=&?DX5OLApFGVU9b5PPq_koA3!!%lZ(m&PdP6%~9()YL^}xL)4|pndZ$B{7Ro@q7
zwe#%*K8vHg3@`b!qYqv`vSHg*@qhV8wbn-YdQEMzNcz9tYHO6KXQ=%3dB5&m3ca<%
z;&8p*l&_hbI%}=gUf=Xf>e@D@h7^l7wu`^_i>|iQ^Z2^<R)|K({TrYEfBSV_R4aJj
za;EG^#s&w}!fx{&1-Fae{qw8}{TBvgY>iTK{ic-n^z_tI3x6HFnxOaW;yc;s{=Efp
z25Vn#KcC0UkRoHz#%6dbXp5KS=IU-S`Og^_cRF7#x_JBF%BQAE5r3O^OGVC>xoP@6
z&*s`ve-DNRlh!u2!lbij&vcy3OwLT4e0u5c7o0ZbKVD6~!9V}Xs>hG6XqH_&cH~H*
zi;LFpH(@<X=j@ANYqlu$=_)nocs%L&k1W%^DeD;+1Y|(7hpSRLUOfD&?K0)^wVn<+
zxu}czcMfU(J-z$ueSw35^0jx5SKr!snmxArJ7>t}BR8bh^}M~XB~x;}k(~0kBLc4X
zZ(Q2)^vL1g({_r#FV;1Bbz{@SbL-am?k+3UfBf{_wUqyVGae~zJ1=2a`|^@s>K(PW
z+(JLN*%*FsTwmz8Mnc#~F5aZF%_!mZ!sAbmueA$MZl3~a%<-x}f7+u}vEu3v-|(z$
zUN?~LI4f3pK3O7jvt;z~=hekm*52#-aItiL{tWHxy}N6>ZG!esDvvvDWwtUt<Kn@#
z|L5N9X}fE;=h=;o?K0NwXD(F9%Er9R>|1{KSWjk5azA&P05gLis7@1|e`G^x^c8(c
z$wQjA%rm3Z>;;e7?sj+SUmw5YqlZ=Uu{*bZo++}pe&Owl_&qnba9m%esw<(ibxFWP
z87{Z1`RhfJU6*8Rc<?Ca_=HN~U;ca-XL?F5UD`YK$l{)2F{Rg4vh0P5%g@PJ#T-D{
zaaYC0P<X)5q^37<#&rF7H8r(avu1J4Zh0=edDqn$d)LG-jr)82$!)W(t(%X*j)Jkd
z@9|`Fcud~;%k0;4W?osH-h4KgFRt)c70<h4S@YlZYsbtmGycxK-r`rm`k4nEmME<3
zoRe7eX_iC#o_VIPpDHRX`Vnp@#Vwust9Iglbqk9bi65ew7#>JS{qt|-67`y<^YYoV
zZK<cl7}Z;+p0wW*GVRmd@V~E16P#Xe4Zk083;jqJl{=PV3c98VXV+$iPW@MYzb3BK
zdDb8MMC<2sUq#*bUfRpCGjD;4uHeLvZl7|K`Om+ZvVC>z+N_nri`nOJbUxc6c>8zn
zzU4pt7Hkn061ub{^Rn-3GhRs}mkk083Q`#vKW2F?y|pRz^t#yHpgXU27Cl`ZzJ6JK
zn>izUjrly4#s3#s-d48w^u{)|BJ`-t`z-Evw{&H%WE^k#_U_!f+&s(vtmkV2Ua?)3
z-<N#<mhi)`*1J}g=hz&zwJH1Z?C|I4>v86B_cuIEUHi>-+V?g0a$a2e;AFBpc&qS3
zDdV=6ckia(?D;3FAA0}({-)&1ivOoo8E(i_4gW6nKALasMCS(P7v=Bo@k$ykaqE>b
z&AI}*_o|gk^pZU@LkDlzxrX<TD(gN!JFD#0qY=F9%oNSwBGAQWi}>wyvp*I#o9X$j
z-n8MzxfZoKVYvY-L|me_hAn&~U!rN7e4%ye&(7dlxyCE6EGDfAdAZYtO;Avhr){cs
z__-N|iz7FuO;+>Wv~{cRa|Q;5D2ukX_em1VYJY#@m$xe^EBp58(=3C;rl__5KAyfL
zw>n_yigcx*h??#5CYSr~%UK(s*#X;y^z|smlAy>X^|tc%b$d!)UUKi3vn_sh=F(E{
z!1D|fg!&dfT;9^&&M$BG=gdsw<l}vyQ!m~2PVMZzVORaMI7lEe<mH8>TLjd%Ot!7~
zaNzjy=SRE6i=UlY8NYvDfFXm!nF|*_aIcHqeQkez{aiou<YPSg`uYnWr+63|8ctUC
zzqhyg`-zFlzrVaxo_w;!kA*?0b@QDM++Nf5YJYxus_5KS0$S45&Zi%@N5Cr3Ey+Ma
zOg}DWQ;O%bRC${UgMtSRSBx3{u$!3t*<_Y`%ckbXg{xP$*8KeR_*k#EhlhYwAX~wc
z6BC1$dd;<|ymbG5{l`Z~BR8dh20uGw&lMehU;Xvf)c*eSCnu{P?~%N`5Ohe`{tvaz
z1qB=W`~5RAUfkJP{QBBj?GhP=69ymrwf^jru`J5DvEku?M&{XOxvK&e9%|(lZ}C&n
z%fG)*)*!)Qu2t!)>+9u3L|Upqx3RRf**|^wu<+v}S9$sQ@9ys2o_+n@&(F`D)P?57
zt&iW3(71c|?x2-5A09aV|Mz!k(3uU~4Ex!5c~>qDT^$CB$nx^_0Sgb!Fid{*_;IVB
z$_(drzMZ>w&$ljr_vhzlZZVw=pdn5{h7^yC3=dPUrA1#}Ty%7Fym;|qUS3}I^>wCF
zy-w|3;f{`uUteFBH_yBC=x8^;yxp58U+TFl85sVUn3yct^ycQ~_0ikkU0CRRdt0t{
z__{kgi=VgnO_5u)Xwk24Z=?6u{axle8?<mSY;BYiXh@Hrmp5}!%E?L3&(D`PPV;$x
zZ}06bnV;X>G+rpYRCKPNdHy|{^78WH;?EN%Ot7o{^<a`Q!xAnNlO+p51GW_(4qU#x
zIcTZZVz=I|?rveLKrQQv4-4Ack1zF}t{1z@B>$d`wzf8C81PwRTiceU0UB;S5{2*Y
z?cK4%!rXlO)z#tYechKE8yibsUz=-^dFj$p@4tWkfDfZrVmOg=;lc#DO=)L$RegQ6
z*u5V#1e$Ym)4zZJoYa5SPMkQg_Scup8ygZ&Pt#>)V#>O*LJ>4{I_JWL33Yew+&MST
z);jS}i(T!nEjc$oot&)hr2fk^YHiuMIhJyAa$#X%ii(Q=|9s{@z3h9_RE7`V9h{x3
zvTtq4EGQ^w<q};Lvhva6$HJhUO^fDl%D-<{UjAL$JTGQnP35~gJEu*X1`0eMzRyL?
z2};V!?)`Fm>;M0Ic({E!=<rsH`=a9F=GEWc+_-TAG<du=D)fZ~!-wq-&dmqj-P>!O
zcV|aON5>VZfVE-Aj~#n<Vxsa^wv@<iph5NB<;O26Fie<t;lhNQKYsqyjox<V{CWQu
z3JeVOEXP@BbT1iX3P%ufF)T_K56gvVJm)_<r89W2n&;hladC0`%$bojHGghz&;R`N
z^zyLP+1J)s=H4=a%=WCjz{n8P?zen*?eA~P`41mD)N$h0t*9L?AkU=lRGj?b(<e}w
zxAW+oJ2fXJC^|i5WccBnk#WMts_c!%)Vji`6;)qfIkmz?%irltKRx-s&P0!>Z8<+f
z6PX!O6hO*jVq)y(zA!U2eS2wX_l8OahJP$lQbwZr`T4C*jxJtHFP%Q^{lbEQVLv}F
z?@X_DK3R#jiC#-j)(cqiF&udB?CiX3l79TYnxCH}dD_m+wdS77%J887ft{2<P)f>^
zE1trZ&5EMOO#(GKGM-*iU1^~vcy&eS?!?E(R$8ku9H>7qCrM$++1cjbzrMR=z1~@?
zFY`&I>G7G%U&Y?pe($L3ty}l^l<SKgXJPn}Z_&oq7<yvj+TeF{oR<G+RC=~*sf>0B
z7sHR;53I<&nl91LP(U;C@4iUsUEU`$IC!5=eYGO+@_x%o^XSd_-_+h21-m}_nVKIq
z*Z2Jt28K+w;};aXzVU9}{^8P3{ra25O|v$Krv%qk|Ek!WWqE6Ar}XWoXQ!sy%#13@
zZj{epIAQXEU#er`k`>C{DLu8$f4XmECSU3NrdGxyD448PS~&IA%=#Pp>3-8JHW~TH
zSQ$Bn-Cnlmcl*xGp=mBIWtB^JZWa<W{C4S3WpLm5CuLJ=cd{}#<P=PkK60YM_`-#y
zODa!m=B$&Qc>0^0?;K6fCnuNo9zIZ2<HGX#fkR^Wq37&3zpXnm;m)f2Jd>{nb{+c0
z6Zq}s<16nsM~G~?xVLX&+UCgu(<Q!y#Hs9`lG>a2JiagLkny)`lhdOj_FN7&+xgAy
z#RcJO`wpL38+rC=a@F#vv}K9;;yzv+UBbVl85kOpGX$3ZSiJt)JX3+0({g8~x2~_e
zCXu`-`C23I&)M3C-Fa&*%fu}!(y~wPliYmYW0%DI`g=Z-(zX6)rRCaqXA127xM^a9
z-Ie#Q&;NZaT;o}n>~j6v!r$|ZEI)g9o><qkwApEI>WkGU-6eJlE#q^{XV<+@q*7iK
zzUh4YSBb7FHin4~&dnZ$|4utu|8`+_G3d&PZ1XeJe6l9R^Sez?$BU<xz0;P2g{3Gd
zD;;Z7RB}5N^ohS@QeOwp`FR5JD?2ZJc=(;yWyy7c<mG2n7oNDQf9%IATXn0R^`|C-
z4s+bv!Fyfh+P0btj&l>@^%xFZzi>gp?U`~~l-1@V`#WY^zc}0L(Ux=eZf=Fr%C)wF
zf~rzNol8FbuYdSl`cw*+Qc{P<shW?Go3kv{ctc+PJ^b%(nu^)a;!Sh5xu|^3N?w%2
z!QdbRGIH1UHDC8DPs;y&>3BwmMeYMfUnR{udsSRqeono%gm<UE=MjOAhBktRK}t%U
za~Ioae{|XN&sH^jlcT@OoW72Z{)bCVwN816@BhDK&Ccm99UHeSw~^f(HhI6)Zb$!Z
z-;{T8e|*ZsP{aIAgxT=XJvD=6M~n1gR&*!(%B(4MY=6`g_jzA*@vki>P4lPC+dA3C
zLZr|k%X}ZBbJ1%~;mo^9XPX0-H6)+AS$oC5a+*iv#>HOiw&^WM*>TG(*Eixi6VJ)D
z|1!n*J=VLvW5e4gXMe3IcPW~!D6Csq?D?bcZMX8Q9erXuS6TF2X!niQ{$dWgK62jw
zZ_UdXUM75gcGmh>&zw+$^VjDG-1NRO(JeGCvd;VQ>)kO|p3e{ROjLh%_wbZxyLzqi
zxT>FbejF|jy}5arXH1Aq^7QG8=KVht-;n+Op8nGt-P`m2AKAyiu#y>+H(z-u3$C31
zntA`f9>XUGzsIH2{QRTN{``uIOU;$@Z}}M%SYpV&%<_yT8&ma%cYm*zPXm=d((fOc
zaDndh0<Ym#tgNVrcz$yGof)&97YPXoCqI5%oOX6rS8~>rwA@?Ue2+3RGSqQsr#qxQ
zeh#`E!N}g;eql2M!yiHN9fYNRq34C1kcw|&$P0$dN4K_S|Ni#=dGDSJj-ZW#tH2YB
ztPBht;c7WA?(Qxxe|hcpHW&A>ETxiZ`Hy?sj%r(-n_;u9{CyngPAZ0pr*e4rCOlkZ
z_;|VK>kY@1(hC=E{xkiJYIOgs8*f}QZhToe8MMHkA!*9Wb<X^f&y4GCF<sAdca1ym
zk}^wme)jj|?Cr9-*B5plOaN6&3>|)InHhWI?(fPy?z`K6cd30^(q#7+7WS^|HWj~o
zRQ}D+<?JoqD_L7F6f!V4#27g{tE;GltPlPFXm7@IJ@(XPku80nxg&OFUfdQvf8|@7
z)1iG`T|!oT3=AC}ZTT<E^Y2a3i#80Iz1icn?;4*a+uQ!KyB!kC+vAZQzjcZGO?Cfy
zpc!I@BTC8UUlzOfdri|i{d7(Aiib9Chqz>)e}B5x?9`>j@tWs9U0)v$x?+r>!-*~L
z#p>|&(>|`scx<VXKQ+-Q-ILYQyZl^qdEC}nrTsFNN}$`e1mfIsGwkj6OKp{YvU<Np
zdv*1~%}b<qFDmWYa`V=z*y&eSU+0+1%D})fO;Nh)=O@uz>oWaYdArvJ^Df-H#Avn6
z((ZCwlawb*J_@UGg4DH4`Ix37X<GH>Qt0gNYWc96Vggp)vjT347QL;j`8eyDzVqR5
zz4$YmB!B#>vg+>ceq0&1HthA+UpH<<C>a?VPCWh8={Xa_4_Th`-Q}NNtuK(desuTq
zjhnZ3Z@aH&zFFML@0L-<WBHu-FD6~jdA}}KO-=H>+}AJLtG~Z{c6RpZr$zIgSI(R6
zp<*@n+{u%g78VgNG8h>4D<7Wo)X&FfP3ZQAFJ~{*i~Va<Yb!i4Br$Hw-9GlfWly!j
zx;r}st^C*?ynMMbM61+lZrplv2_CDtetOf*v#<HQh+$yZubFJ}<@WY`+tZ<SLfOGf
z&W44p6SDHNnz+H}c-m=Uol{eEo__uFvv}d-j@WZ^EW1TCClzs@-Xv>PBA~_3a6tYP
z1MlB2FE3A$OWnF{Dz|y0PNMO>*3b4+)<0tj{g@}0dg|`ZV)qw1N!K^0`%iuKS--%t
z_*qFwiHHF|!-4q%)z0_#RBoP<Uio>_|7UCMqF(4U6@Ga0Fre6N=_*tIc{Z9QVu9Hg
z7ad)m>M_MDb$j03mb0LxHZFHB9Pg7|9kti*omH{ysS;7E?gN{w%u8~1F3jvMlQz#4
zvg)q5dE>?u&ERL3+uz>WDr~^Ra6n!t-+6uX{(Y{q{rqyxZi}zgE)fgdaPIljOHUo{
zJ1$=4H(Rkp?BlDOo0sq0X(?!FX=!O;0ZN_^zIT4fT-`Z&*$=&_$Sr^MjDH_;N*5H*
zeY1K&@7XAmHQOp4+H|_?*`L7ax;E_gty@A9*RRiCAGa5@H08r@m+KeY7H_<6U;ACo
zGJefpy;fJVg^xQXItokg**e#K;u`Dixj`BtPR9l3zPh@4zE!D~mDR0d$J}1%Ffi=*
zkKtW=F=N)Y4B-z?_6EQ94Qkzaq43D*j0>%AUd{}aEB$kV5!CN_{Ac1^>+(K1TPr0c
zrLAl}brQF(Trt_oHpOeHtW}AGechg|TT8QgIT=1UD`mZyF=K|*R_Q%YWx32Fl@e2(
zjtg4Pdp=)I_UL}A;%{Fz@>CuC`t0m%&`lKG-LuVdqdJ9DTlzl7e*E~+IQ^VW!Gi-c
z43n$Aztf%iueARUV}rtxOBeLx_pJ%pDmHty+rOyE5^aK3er<wPiHTp%K3(1?UshHo
zW_4P>qod=)$B&kEe}2p`Otva}Gec2t#r@so`9D591f3}T_3PTu)zdB%GAMi&KDxh$
zk&*G;(!D-irFsF?Mz4NOf1-8YJo@R!r`n;`)!)`!h^*^!TRhVwbJFI`o9E8GJJ-70
zs{Gv?#aA;sx<s{~J%4Uq_~?jT?XN2<gP$L4W(Rko{C?ES_|LbC-RRXj=j!rjpI+&f
zOykj<o&DMAY?x`rouuP^vQFtg8*^`Ov$V3hx3Bj1pP!$F)%|?xbhMIJMsI)DD{T(i
zak9{<bzkl8q*Emf9@b3rpWfS(>ixc+-D+D*_T>G4^7q`a+rA|}QnGLf=XL*mIX||{
z-I(ScK6$p0&l~QyU$XCnuV3|KUB&~Q+G^v>stF%2YgcU#+_~razv4-+Lu22~3NA{i
ze$@Em{+~r1CsuD+<Z0I#=5KR+zQsDr=`O8vQetD51U=&am3dAzFfu*zveKJRb7viw
zkAD;xcGHmi`g#BV)$b+b1uOD{Uw>TO&LeZ-|M%kMGhbg{&o5!1U}#wQ?99xkr=|*9
z`Du0Z^?~+o+}m62-Y0W&xxYMUVNZzq6T>e(GdKT#`~LBr%Kz`L8VkDaGx4!sy#4ec
zF&(}2GalXB^z7;s)=eACtPcObZY(&L+i=39z`BjS{T(KIs`6F#8vd}IxnY0(vb#qk
zqwdbpi@j|7M1tM=+t=K;J5`hxx%2bP;ay$1%6|HhEo)cighnN%`^`GPK6#bM#4P3B
z`_kXq&urekqtx1}^4q3=%<Uad?lq;XkK6m|!a`@zQTk<XDjptcJrj7|Ws0h|SbY3_
z(6InkB`0p){CQ!aGboDBxXoyLR=LlAf8%Dcb%k%s&e%j>w3qafJultap;Bsd<B4t$
zXS5-=v&$Sk*T9n!>D9-~e%^iDdr?NxBFRUhy<c1Zz4g+WH8YhLrN7^|!qYl?SwdIw
zMlE5bMKRZWPO)y|iZU)UQc~J1bII<h$hIJ*N8$39GdsJc|M<|&oP6$<;P$s_?nm-1
z*UhZ*^xt%TKfA%THH&-f<-hdaSFf9Q$!dD`_QlqJ^tR>Pja&c!Z1TsDDO0AHBpzyc
zbhP{ViHVQDy}d1KRdS-nTv9|cDCPgZzroA>WKA*x)<$i8cYi;4^_dNeh3WFES<zeH
zR_JQn*?QY><)u&l?ZHO*hrZ64y2in<^5kjH?HyW`Cx2dV`+jqYn!MKRJzm<o#nR)C
zf8Z;dbliNB+I^Er6S=0DMn+FmT(si&EZ%&HeE(L@DeIos-F-eWb&{a()Nd}&!Y2Jl
zzhzkd<6^1E%_%;M+RQ_ZMA9Yph8})0op+sr@e7G)?p<|J$vP_o1cG;Wu6%iNiK#yC
zbfw<VO(&&hz2-X-+`Gx>-OPvgmS2gl6MT2hXyvT}t&jfuC5?n%C(Zp9fA{WMudi0_
zf4*^*E}Z@xH2BaXVYsN|<t1^wm=klYw<{Q4@_l=2>+10J`;v}!otmm`o`0|A(h^Vb
zAe;z8O2x}s9$9JW=#^P(Z=e5q)aspYYn7Uk5^MS$@fGURa$jgEPP(FhLT2xp|2NJZ
zx#3+sWx~8WYvuN;7hcSDcH~ce{UlTU5%1jT>N5Q1w?spBxJu=@OGo~C^rYZv7ngfy
z;p2x(K0kDOUGpi*vyE9%X5z;ea$h%Ww%U|%dWP}bY>i1yC7=B><Aj4(HNAPFYb&_F
zHu=%4={ACIcdk@VS~Y!|py#S*ljrRIv-fe-GA~6lr6UXcdf&5Y2U~5u^WoHt{ZrCg
zL*}qOc@DbzH0S1~efvPiu-x5{=)7$G^OT^KU*^~U+gbWr?EGgxKfmJV=R^-@@-UpZ
zaB1tp!-o&so|@91)NQpbLVMz}dnHqg)C?T{&(c`my=T(uGd5oJ7N0(!Y>fG)Be>pN
zK=3ZlmF&*KQ<@XKUKCE4*RgHhrL0cPg!3--f6r?MA8Is}II()m9jlWCkN4>*%#+*y
zxbt!T(cmlGe%>x#oy-%By0^^iHhHt~omYe7X(7Rrc5({nx3bNRK0nF7FrM$<;@~^+
zJQ@>4R@u)kGB479`t`%Z!&jufwI4ck=+?H}+iN0?^Yh<tPCq|QH`?s(y(pEOoSbK8
zW)}bX@iF;$pQv`&8_+T*&={f4U#VaJ{{DWqbng<@Xwfz1wx#Rcy*fL3+~tdwOe*_0
zCC*}Ty3f9Q?*BLbvx`lcwrT!$iG8KJ;!jS}n-*Q-W&M6i-o?u1wYG~@l)ioD>zShX
zX#KzSdw=s~Jj#2_@GnMBkyBcPMN%tRP{gJ1N^7*E%!6;G*Fy`h1b$x;6&}sr7nL4c
z+Q2NXSJ3=`!F9sJ1nv{({NBC$%N%kitkL_y&p)5d9=<F07mDBa=bhgE=d*GSi~fAJ
zy6bIbN$uM2#}b;}X)&CrTU&AR`;_^!ly+Nx-eGig>iZp~e-^(gofEnG?42<C5)OX;
z^$!->D@^nd(~rBeCG&FC*H^X`9};eCNWA)f(&y{NB_%e6kB;2hn*IFD%*#u?r{A~{
zu}#~FBXfzC*9&R$ye*l}GjG*REZx0Q)Y?cnsl?aAyCJ(=M3o_6g;g9w4;MqjA_j&_
zMH-?4YiCblaF|iha^Q~t!bX-!EDQ!46<QtQ`<AMT6tJ=|h^$~>m}2xSkZ<i>hu6#9
zmaLm#oWEq-?gj=shNCAK7&wF+{&eXvEIQ7RSZ%vu)5F}4&zeOT8jh>Zc|GlE@GIe~
z!Kw_du|4(A*-RSG-Ck((_T-%(U6x1x&fE3QIj3e@+54-{udEE7YGA{_aM<AYx3|Rw
z1r?v3cve*WSmfGWQc|*SPc-j^wb9$H%in!@b#-;{azD`NJZI0Q?pA7<(55(DFE)y`
zy|6g{=7Mqu+39wo)@#;j?%dJ8Ky3cwG%m%DTKia6)-2gE&Hs#)!WIDmo7hc?j`w|*
z8HHZB8T-N6cGG-g^IvUW9MnuUJTohOvNO7<+FWgY?1W{JAK6TwaF$r`W@bEFl&Ynm
zbEfmQ`BX8!J#J=u;(RMFS=^i!b539O=dGASb6EPt{_MKd;;Q@1!{+K5x$}2jS5MYe
zSMiXV#GHG#BCmI=kN1k#kxRq3N_OhK)Cqs<Q#bp9-nl)T@1mIW9wwbSd9Hb%?}-Zr
zm3tlqum7|=<N7wk75D$0zZv)W**R6uPp7T_h(CPzaNoXt6DLlL-kvwtD79<fzPh7*
z>t9;P_<eqM*1b=r^7ps5o|DxCSuQSe?dJL~BH(m2RCH~`qQ&Vqiqd%ePb6h496HCr
z#XTcD-%Cj78Pf@^1Uvi6tyh#CjxO=G-^<#+;S%f1*%K>URKIV&U4P(IhlloxJN}9}
zg>w%z?cvJ#K4sP{tMqeoqPOSi#_lTl@Zg~7&pF@1y}Z0Yd;OD=9=+fH|K9fe_-&^h
zIWif(h5ucn<x{k?f{kJ06wiM_Hp~uHC#OFOJGo@4eB{sjk0cqitmkP@i_nSNQLu3K
z?AdGAzHMgbZ{v|{`uNB6&D*!k%*@x<$L~)#*yK0Y%C_dmgtuJ+POsKIQ!)7!$8-42
z-QCv@9#pinv{X_`I@rW2YN5Ga(W?BNjG*Af<^J>6#qR$1<8i;GrDaZdi$GBPj0-PT
z-`Q2V+O=Ek_O@K>k{1`=-QBILqjO`ONAN_IpO5?P-Q3-|Tb<O^mxK5JPA%qVa`HGF
zy8ZQqT$>+%Ull$PnV`aWAZ5As`8%Pi^+zuW?s9wL;;&nsn|7D6^7<t`spt0FXLnoI
zrxkxXP<Qfaehf#?3_F+Ed$j*5zr20@(Z8zu#j;kas;S@J+!PcPJUvZU+9cz`_bC$#
z^?w{V`<&^*+b{bk>K)79|6p3-^N<7=u@f_{{g-|l`2FtJYb%<h9o_q7PtMws@2NcZ
z`tr+9_3JX<T~>Ng7T&>WRsZyw_jBLXzv_?6KKazK`1bmz*^(!oK3n_e*On~|3zE*g
z+dV7BXU6t91{*}@YxQwj*7#nn%bMS|zFcsA<hBN>jf?!5@8;X<I209q`t$Si<HwJ$
zuZ_OGzyAMwNiM~ZzaiT#zr6MM$;_IVd8l)@GH+M7rJ10PjB?J=-=F#xG3}b*{DD=W
z`m*ceWDey^=i3+>4l!A2Pp|s8_H~Jn+mg+8Dtp&-Rs2g+nwatL&(E{7Oto_*f6o2L
z!yxv~jycD>>`lb~*BjZN&O3M4KJR0n>}=j$JQeoy7#mDhrLW`h^{RaCski^;j+16>
zOn0k#Um5qmSF8Ul6Y}rFvt-K;(Uo7P<lWr$wast$w58A4PAtovTmHuD)vK$M^!@EO
zCckfsXOio@XXf9$=t!)iE8nvFzxNthTkl@%-k*1COXkN%M`f+cdiwhQonsVqy6Rc#
zoxz|H#K0gTYEYg$^|kU8h9~@AEe|j=7+lnTUGXu_p67vJnN!8rU)wgC$C`LWzM5)!
zj6oso7sqU=LI#DmGmGvO{+#V<>idPmdU27t&s4KZ7nF7G|0{H`n7!}q_KUkp6yJ5l
zEnTw6fM;{{_dIs?_9;`QoH%i!^7Av!R{1qdpM785y3OS9XIlL`@#%B-Xm(kaH!N(p
zyfO9g_Uy>hM~@yonmWII<x0y2zUkbbk_vAwcbdgN%WUIH&FN31qi0OFzwak1E`Iuc
zO^FCA!;}aez5Ow>WB#N)xpJW)F*v`Q(cyUArE4F*Oy)J8*X>n5rS^T)>v%P@ONtHo
zH}fC)@3r01b*gt_;rlz>>mB*5M2oxUEMK1f^82-_FE0$s-rRWe<_(vmnBtO`t8#;*
zm+~+ySSQHNpfj8ETiK&2WqUmOCIs?sz4zttjKaW|mqN0mwDXR6uSh?}D4F~C%<Qkb
zy4&(%-(8ZcNtMgZaeH?Bq~Gct=a?AG4>~nErLB7!!|f+8#6DGE@}HJ<Ov_j-wz8i2
zsL=V2|4^P-N}=J}_2(HA*3A)<4=6OR%wD?W!nv}PmwNU$CP!}UIu)_>$?vC6o_u+6
z(OE%2#<J+h-Mf2}*r%T^t&6kyer$dAx7dn1ISs}Q78RHGiq3wfJE!jFS2fjz(&F-G
zRy}7h=U3Y)f7AFn7sHFyfA7RHFj&Z%&yUS5d4J(jD`PF+U5Tql1QxuB6<K=sUvlh*
z1=C9E13vx=W^~(rtv{;vNOi;anaSx5mkYl?<&|4@r)6_}{?3W|XY7g`>yN#R-1d6e
z+hmEVZ&80mjoD{z|MpG!U);9;%f3I7Q<*a7ri@+I^E1~~PflTYDmHQYeanKHGe<Z@
zf9y@0q;kAZ);i%p!`0Q{)|Qs6EG#^-R$CsmxpQPL$+Bo?OJG>gt6o2E{x=QxO%~1@
z?P3-(X?RV^->-Vw;Q7?|?GMw>&AsOJ`c~k{Nx_pQOPYLm5cy`do>WV{exr$-)%z(X
z7E{f(KUZrzEv(_OPc^pCyYk^qE`})#xwkl@t|d*ApK#7fVW0Z5FYncNlz3~hpYWg3
ze@oAk{m`|R%LhA`t1qqFuVnpY&ZEM3{Ud*C_o`{1-&_6NLgw7l)6=zW<}orzw9ec7
zd`6*Fc$i$ZBjaSLs#WvmFTZ<xsh^RPx$p1erI%m5+`q<d)48WdCd~KOXkcvk*`9ao
zSj?=8f4+T5O_%!e=IXo$dwxG(`eVoSl*f&am;dmWJbh>W-cO6X-|y3H$arC^Z1(L~
z$}_h+-hz|MZ<ojYSTSGyuT6I2t5dfQuiR!__twiSYDeigg<_$G==u}u?Yfe0D}Rzy
zdz{tqal^A?^JgfDAAP(3_7d;QUGL+Tu3!KEcK-gg5gQ+Ucz9UU&cey#ameWwHU=lX
zrIRO#)XR$c+b;GvovM*=C^_We#wm$&%YP*<xvsR+p;eq=Pwej5nr|05a#()#RMVQr
z*0A}x>f*A~af}7=`Y&Qt^lUx7cU)Hvm0VTAz`#+zkV$Sis{zBAzZF~!g24<TcdC~$
z)!w^Tcx&2u(RS`Q{Ud+x?p4$7-?*`GY46&QRbOs@i$ArZ$*<jh_s{$f4=(1GhUDw1
zoKDuCmt%4{mYt!Y<gHCpSC>aHQ-RekODTrTceYhLqNVoIJ!jtDpL_6B(BX_#4_wb&
z{cz-1{hDa|m6r3*Gy8p>c5WF%)Uj_W1q*-tOlI)t+f`}axjXIbd|Rm(TB1q}6ZXp;
z-uU`~n9rw;r_0~7%$KuX`FDBL?v(r1-xpn0V6RK9<(9DyS8DMJJ-zu33&ZCR^NfDn
z<#;lYA^&&clzYp4zkGPWEKph*@UJfa-R=L1QZocU^e{3AEAxGxxYSi*WzWM|lcp{3
zVfQ$@l+mKAs9jrjm5R4$L9672xC{FxZxi&e{FbtA*StsTa#TJ@sJcdMkG|r$p=J^P
z_N#}Y!(>947+x@lGwisPmXZ{)sA9%uzJ|tDO9w4}1_vH320MuvvzXm8KO|PH3URnm
z`Th0nHD7KO)P?PvqggBUBjDG~Qz1o7w>S1r4}W>j^1N~JDX%Ku>pz#zkLv$b`bG5J
zhVIE(^^ZhP<jY#vl^lwQ*(IzPa&y%~nO!AXG5aj_gRk*0Gu(UMesRKKh61ad6Bz=c
z85mkr`Mq{$7^wO%Hbh%H$_F1>y6_s8{@P2wdpzwDla4U1$~m(m+~mWFX>JS*xl+Xo
zXYA6cif-d-P&v4ibG@kV*}J-36C2kupPzW#B_w9O=h>hypIT=GF~0aB@sOY4vU=%7
z#s{CaH0+x-kDXzO@8Wx_PFzSo%EMs8%l+hMRkW~I!zTMOA*cTj{ByFo7I-Y%x}%2S
zLFP*)hVbQO!WCv!uhx}p^qNwY`)bZ+Em=K=f}IQpo-%5LBqT)j2d;@O)`{Z#)MC6p
zf5%*Z4F-lMpB~M5wCVSj*xEKmhv%$&&)#1db%2>cSC64VBk%uRHQPC_L;gj#3aom$
z>bA_Sr7Oeiv-udlywlgP(CZFh`e3p77xxc`im&2J<fqTwtCnx-aAVQl4X)K5)f%y)
zj=ZLiN{WTw<uW-iX}pZP%krz#;=tpxxn8+@mEteVyuW@vA4Bbz2`aK3;i^8TpB<W7
z7Gl=U>`?N0k?G?-XHGiYs9sPuIcCF!<bPIL@)ulBurG2Cx0}lM)$_sb&${m}>F(ls
zZo7PK<(Fj3i!q+9D_+OmnX2_W?!%#(|6K~>uH4+b!vAyJV=M1nyRU~mz4O2Jusy?v
z>$7^(yqE1LnZH%H_T7cD{byz@sW#j9{N~oTrKukezrDsYb8p%r*K4s!_l{cXCi$(2
z_Ff(J&hpBE<DZ-gG^S3Fueo3RqVehVcHI{ji<lk1S(POFUi)}|@rxC2*`3wR1!Fxz
z?%kKT@%z2v?YeiynKwTyKJAeG;Jvw+_39+6pzN@cjGupAeBHT>f#JZ`jIDcuwfFMz
z-WT(I+Quf!-yP)ka-L1(?(*PgcZ&P#{6tH{&7C-E-Abc3KGEKLo!{BrL*k(N^~>9?
zC|~lP;cr`G+_aNnlKu8WW{K%6k*qx~9ygdK|B=7`?w{ZP%%hV&KR>@de*eEGCnryh
zx$-=qXk!H5uV4B53v|TAPdBb#|9R!=!};ZpU%jqRTm0Q^TKCZfpMN?wP6}3;&$8f&
z$cNBPNdhlEYkySvKGQgzPsTz(Mkc1R-GL*s<&|0cKd<+$Ya)K_6q9<N`zO(6df$ho
z7g8s2GcY)uQmId!YASK^dZm7veZ`~KWxp?}{aODa&3D>U!;|Wa9<n(bI|WurUHxEX
z;>4jC!kgfv+#=xgibGgRu|*)r-Smr9iRSdSJsWekv@ra%G+BD|jqSP@7kp3Oog`f3
zBNU_je(SoTPaTV^-#;;~nQ*$hAwTKvnzilihr_>aJ=48=*W>k%w+T<@-=M4Uw)pAu
z4>QezKV4Y*wpsfB=0u0X{`sFH7v0!-d7c07>P?n){8EY`p;_-_W(iE1Z2ckr=n1tq
z(`*(UxFnI{aQu0rS^tl&u!t2-AEJ5RtgkNo-Iw(IV(`D~cT8%1{=c(-*ZFIfZT8Ju
z9}lnVY^=Qe{8e)F)NSjS81m+dB}ZSm`um1iysGl2E3CHC!9D3M0ivsRa^@!aaWO6t
zVr#H0i4)U)Y;w8ngT+!A2dBN)r*W<Eu`GL9W&M&t!0AxX`?u4RI3MobyY|BMOV>=F
zm~39LczyDnJJo+a`8A!`{P8G*h%4iXc=M%I+ml5b_Uwo_cU3R?$LG@dt2fOLEZVpu
z=l1rss~d9OnAY9k&rf?EQdn2_Q&7{AmEqB&?Y<i{3JdE#&i?A~w46^y-gt{8n~Bp3
zt*du7-?7XL;cW1(e!FyQ>g&7}|2QUH3u7#3WII&ea87}d@j#5d{>|V+k<YagK4r!V
z8~twl!gKWRMoa!9CMyp=Wb%m95}O~i<&Dk<qvrupGPYg6UQaM_`uTM6;_@ka*IBh%
zI}$}QcR%mDTluWdR^VjHd1dwAGST(Fe2=eM$6I0kZ`SUIu6y+k-O%mq`sr|F&Y$1D
z^H*P)UA=sL{l)v;o2#79-DhY2`+HaE)id*sxlccH?#15n;~C*<|1Zue^zO`yy}UhV
zs$b;)<?~;@OFdoIX7}>SNkbD8$Lt5Iq`Rx1x90u(IVoYr-N(+48jmmArk6YQW3=j)
zQw|&p&#%~7oDmR|t$)&+;gvP#wjRN6*OQB;GBC(Ah+o_kcw1G|;l}L3H99-CdRBca
zd^s^>*FIhIfAfsyG6ngay{l&>vHX(R<xi#8q!QfY_^198Udh1tvRioVf3N8~s`bA-
zIy!mv@h!1(se6BY_so6k+4SaX_KVi$-R-VF{)FsH4h=i<Omq6(Z)@k-WuDx1l4T{s
zv7f&4S1;07KmXA`n_VS8B>rza)9Z5Ssqbqo_37VicbVxh={cO7Sg7o+mis65(UFD^
z%7@Mxo87xEZ?Hhw@!PB|S|9Q*<aZ_Sp5J*ieEIv5H}CG|zgcg$Y5LKnYmN43Z46~|
z;;0Sb^~-43lJ52Y?W4^nwH{2|`#W>HzV+I(Ha?3OBxVZvoH=<V;^(Q)U6FrV7q78Y
zP&v~W$hfB?bRW|Wk%d#3uWVOiT;Qm@c3WxZi<F~td1hU`zNFEGXJKwmTH598ybo_*
z2=A5Y`C7uc$<<i;@-APg&-n&Vt;0+!D?V*-x968_*wh!wpt66>`Ov^1lj93#a&j`b
z8Ot5*DKgTzzN;&LNj3w6g4g~DD@s<Ub1*E(etfg(yT<Y*(=v7Y_HN$Fr?}$zg(d@*
z3uW;SO0&GC7fS1w&%EZpe@m!aeE6T!+=?Mbv))C%+h@yK^*!Zx@r>l~l$}!wrYPUM
zH?jD!=7HDSLKqwZdupB6vdUH{svqHx-P#`2%+tW_Fo}Vor9pwQL51OkZ1|J)3=9YE
z_61AN2w$^DrT6K*6I(qem%rPT*2VDPZ(pd-zs<~%Arlv`xzm<k9K3AO<&$2P4SWm?
zQ|Bw|?RSk#=h0i}EfC47#1PA{ZlS5Th?XeB-#2qhA4&Z_%fBhZru@;Pyy|&#xZm#k
zpEvnF(}qwsrxmNN-mxsHQY?9=xq*eD;f#<}gXhVGjt(CfRo34QVr1==-RW?#y!b$j
ze1jTK#R4V+10@EB>m3i6YJacfYKXZIyxL44Zf>ix^dyP%|5>NA2dHSt?^=I5l(8Xi
z?I+pQ%g%CceSKA_?(jnQpF9qQVM0@9OkrIAJ4SNGwd5($_4X3;^u>?MMDBT-*SIR?
zwbnwhk0#q$Sa_`}v)<ZC>asIv)CsA)zH~$=MX67BJ~zVxEfxU|uW$M`Irqw5bVlEn
zyn1l?_jP+-n4DmYPE}}G6?1m3^YbYN3J-Lp*8L4)`s!n`^StPBZVThzpBX#+a<_7A
zJ~)v*rg&YU5O3O@R^gH&ZpMl^ob&dCs7hFJUYlv)rgo(H^`*-nePpXQt=)R<z3`R=
z5j#J`@9}^0I<UMy@0R>ZkFKa~Co@j^*T#A>gr56b`+I%B$)DBdclCVE`V)6tOMKJm
zC)|JAx0Dxdc&=hpa=K%~<VTY(*@fqAc9)BhGtRl#e{l7JssFF(HorO<w|(8OYWL6F
zisetrH}`-2k$s@6KG=;{X4{*}=$CgN9sPf1?fkFL->eO3%04mIE#=5O6{}bG7qs_1
z$~C^0QMz9Ce$j*VGjE6S%>8L^bF^1rRh8+M?V=0}{VUd{#C_yQS6;NC<zGm$Hp2x$
zAt%kxTnAkGPv|zfJ}#LNJ&E^a&w?j!Z@yXoBKN3T@9W&Jef(BoCyyoxy!g<lwfkPc
zycj*Ub?08+3!a}9tE#UTR=X|m+HA3iqQcbgo2ul$ZFAz7x(ZZy8Lek<IC<n*jbhmS
zl6|Li6B#%dR%E+&J({|xeEZ5rZIet-u*<HFGhgy6BInajEBTwBn%COhUbyLx;m;eV
zxfG8~npG|PYqo%M^{21i85;%j*Rjw0;liO9(wX%^=2x^q`?=G68)I3VI2O)YuuIiM
zQRR8n$6il{SrdZ~JQrkOFsc33^QP+PoK`h8-iw8rQ~d4zw^|>Xwfpn&T^}n<=E`Wt
z%X%_8Nb~>uKdYe9L|A_H59Odx#j7F7m7U?tVeh?J?=G8b^F!(ILgBskRl6d;{JACn
z`nxRmUgOW}xfkEA3%$<UB5;Z0>N}n7a)JBB?r=_@>zSzI>v1CNPpz|`<b<;O_e>tz
z`h@jqZT=8^dXhlh+MQb&eco<aX7Y57oqFeq=C(X0dD{(T&-%+Bx43H>iL`XjPMB4@
zwfedCG@Z<q??Zmi4|{uW?<@AWKijky^~Z-SxHSFy%^THs&lNLn-s13amp_{mM=1MM
zU3H(5&K(?)30ku8KFkjHw5>EJ-;!ErV)Xxp_EScN87~CYp3X|ybDeo{O?6L@`uY=L
zEz?UQE(kI)L`VuQ{jOY^q5dK7&}!y{>;r$c##iZ0zE>I;SS7Q*ZO#PeOSOilRM}Hn
z8QkXY_g{Hz@>7*|!!7IP@iq8;cI=*hOKr(1@BA0l+h4Y7m<#ea_(fLtWi98Zx~t9%
z8nb`$D5)?r<#c!V(Wl3kA36H-y<M|_)2n5><~}`Fx$F)H1H;m@9NzJ9S!*7J+P~eq
z*5asS*UEH}+G?)fQNrir9h+~-Or7-b69>byi@G=EH%_^}+t$D+_viky2XpOIxT@~z
z-8wqce4A+ejMXzF{q*0QG#3^Sl3c8-upnaVCo$6-oh%J2n^~&+PAg4eaOnB^$a$U7
z3YHQ-&!ef&k4j1|&=<Ip`^2?C>`VSsc>$9{mp9(u6j`MAG467A?_!z${2Q}f+;e~C
zy0$;_T{La#*OL+nbFH$UzWv{u{WaxyqsF1UcSZNzfBVhx^~rd$|LcdAZz?D9b3Wgn
z@aRh4&HekYoqAmy)uub!;CsZK9}RoWuB+*LNS9XJ{o!`(>f8U~9E!dncb{FD|LMGB
zB`1UGB1Q+!4@yqkbYoUBdGt^0y`|?VAgs4`%Nn1`_g@d3I3KQ>qA_o}_Z9bztJo_|
zID0hQMdx@hUQj%Kw0NDS!~C3}1vVe&uz78L(4phRdEl{*)Vj;t#2D4~{J+|I{UgsZ
z-pVuw*5i*p-JO4Q?fw^c;^SDJ^Vi=`OTKO|^!ewL-@f|)_W1qHZ>;C`>yFf(-t*!0
z|7HGe{O{-K&PeII{J?f&d3g2G|8mD-3v4tVop_^Oe^iXI;myza(TCjq=9<;s^!xem
z?fD~Z_q@0Ld2(QP-QQ2XKU*fmPBxq;=#;f+nTm<V$_dZ6OjKSkSr9c_c%flQ9-}W~
zvHd)z1^jN>jZZ3W?Y+<JJjHpf)#4rh&)FvW+<V=Al9eam%wM4sz6-t1FP*0MA~&MU
zrRpc+#VMDKGzD~}_S*HHQ+{6lAkJQQZ#=7h<l&c5TO&Vf{CV(-^UB;<XX~}z+W$N*
za|f(sa8PAtsJN}(e43S^yJ{=Lk8i!REh{WuGc$xwU%1HV2oHx8+r6DZ-w)o~S!~9f
zc#xIB;Ai3CUFygAUX*+{N`JmpspW$El^HE;7UJtBFk5C#a8P%z{QGtC(wSP54H<eS
zwOx{vkk&h89usJDT`8+RXO6kC4r5Kt*`8188(n9v+AO(c+lDWK={Ntgcs^I%ZOqv4
z<42IxD@UEQpHgohFTSn(S(0JyvU~rY|C2bnf-kOKt?KR<rh6uPe-s?wF5l7Bbm{0-
z(fs;TtPEPU!K>FLM$HY&k9orR=#pYrkrBg#7hh~&ifr&x5vr|JZVBKG$Q9n@z#8x^
zdKXXdBCGk1uceFT?U^eg%;v<<z}LGoz$3QOb^mo{=QJ<9PtVLB?={}k!nCe|f#Csj
zz*6z4oI(yt)wb<oLT__V>Rns9eMVtaWP#k#K!zi0Cf;&;d5Ysf$p*#gyyDybuTr&$
zeDLVeq^VP%K7T%a%9JBvyj#yseLQ`x#$V%u3#Ks&eEW7yPd>BiV5<|uItGRZx$}8A
z7kUds#pr7Ers(Q1a8xib#4svW6(x&bT;neNdQtnsL<X;&GA{hn){4dVSQxii*(9f1
zaIgPnx-;hA@Av!9&oVuI`gHeEBLSzZMOAS{m)D+-{lv`A@WzJa@|LI4@29eUUForO
zJOB5bw3}%r4%PCxvxN=?wcq=@-p1YQPR<`cexBs?eQR0QH#+=NyBs^Ga_<?-1CP)C
zXY7%!bTCr3t(dY^OuQp6cFVS<fwj67o81Z)86UXXWE|lm&Lm<tCAY?{|IPBZIt&>L
zp5A9*aG0lPufOuu_NUsBQBii4pHdzl>z!*|E@xBm;m^;{yu7@tcd*Gmj#{PNz5JZ-
zpN0D#oj>dDzQ*D~p3aAg?Jtiqg+Dv;zFy^GVEx~^gh&0%I<u#IJsDxw&Xb(scjBp<
z*513>hv!>J#AR9dlqj>jSRuT8@$-_xuSsFNnJ*vx-Ou>#%iX}_Yd?h?BW=sBTwVMA
z$<nRb501z8e$$b=v8XFvpJ9$gVba4xt-iC(db_$dZQZ)|7Za$+O)EW@aXIh?zsTnG
zK@1KYC-=K?=*dlWVOla3G;zSs<87<7_lf-;SG6jZ;Gk=1i~9uPo~>|Q|Kk4@%g4QU
z?%ZkR7N2KRS@iDC&d0}kmF;ta11mLl&Ux}#v#@UBob|~u`;Hz>pA}cL%Kun<b9ZQr
ztexk{qX(`h*8aT4&M)`p(`kKGRaI%ToDW~Vr0kYynXqfcw)CC%jW^Bzc5b58W7*Ej
z;isQnZV_zfKftW^_HJ89l4kHe*Q@ni=?o`o?|#tQ-qIP`U-2(ZO-=3g_Wb=dKR?~v
zoF2U`r}F2gr>m^ZPCjR5SecxY^C#}fr?2)mCwKif{m8?hvVY4@(UWC1`)hs{Jv!1^
zS68R()>HA}0pl@arxkl<O}gMI8NzuqX1|K|Zug$!KbOktZ=X>3UhU8PufMY=E%+1@
z6QdWuZ%@_NS2K;%zrDY2Z)iC2ZPzO1nLgXf-bV5B^A{I?K62#9$H&KuEli*O@8MXu
zKE!stbN!35C4b6Nb{_Ytx+++@mv7CqnwpxbZ*O*HUS75?b~m4dLBhc%RyVg-SKdZ1
zW?^H~i`#SK?p<3gtyPZAY&p}L1%iTqq+9=rvXRSwc4p?sj~^KxJbSh+_qN$ouHW^4
z%)h_C|Nhq2*T25L{{Hs1xQIy0#~y)SJ+n%e-w4mTJnR1LNz5X}|MtAuCug~R`GFpG
zR{hZ1=TF36dwMSKurjC9lywoZ>kNN>$t~}yJ@)u`fBKmj6KBoZm34L1ySux?uWYy)
z5f*ms(b4Y1hYn5CkGHG(l3}yFSK!wotzV`GAAj`Pl(hO;#QpPErkE^WQF%P|dgzm`
zI;Y4*kKTFTI@0vL*ngfz@K%+8X$PO0m|UH4f5W3YDpS1zH<l-v=iT}7=xBHSpO44y
z>@5EM?d|U;Cns~s?3p#uYw7B+wVytG*syJzpO24>b(zjFW2YC{As6$zn16p$k&&GG
z`$^JOnRnHJ3l>Q*6cjv~cEmHhMf<;%^4=NF+0g}W85m}45sG3BEqwRzx9Bn#yE|{U
z79aPs+yCfxW@TmN(PPKHy}G)(`ujWMv@<78pI-gJIb+vi_kOeNYi~|YR{#F)ZuHiy
zsq5D1-4<1BF>w=B-LifA_odwF^XAW+R9IerRc51;%CQsTcQt;Og>4jC(BZ|#aCYZ@
zJ@t1nfA%}~$=$L}JM+TJilzVfY`fYjCnu)<<Js5N+<bO+_EfK@)5}*gzPq>g_Riwx
z4Gj&=?EG@pWozpG{sPVQUNQ(R-?ldO^tAH4c>Nupk9S|V8d`RN>#*|f=z>M>GS9T8
zGkEmhnOv0T_G0O0+o~@s-Xu#(OXuI)^Yho&*9#Xe)Q#F=QT{F_RX%WTYFb*KtaaIm
z35t)8_wTR%o_Bj&?#*=^ic2Q0`cgHcx%u$l6$M7dDle~?w1qXxvBu}7$lFcUx&O&0
zIm*p$|DHWR;v7PboPBn7cKA(4Y11qbA)!r$kB_-_i~aoZ<H*sYp*I&^ZE;bG+*y=*
zdYW!-PL7bUuzSCpufUJo>Xr-Ut6o`$y}GheShDV}*96J>3W=pt!aiQy-y3eeuzF+B
zD^XR386WoTusogNHMQcAdfmILzXKnx(RqJobNYGrez~7Netda#6|`$?eO%_3U0Lzn
zi8Jc||Gj?p?Aw=@m-EZnMC>eD+Sg}QuJ6o|$sqPl^Xk>Bdutw^;qT2p{_f@4DJy>O
zb(OW|Vvlex(7AfBP3EQUoCiy;l|KyPD*UtGIp@nKrMy^EL&L<?JbKfuOJ7}SX<^CD
z%`Gqg?mOEo^nLf`i4!Nzn>X+F_I&%ICnp{rZeP20Ehy+hWH(GUw~>4QVPWn6&u?Y+
zF5Yco-#S%lU)##wYZ~rC9z_-TZ!gBS&J!!pYiql-?(i#j#dEbgZr|QK^J(kngRcx4
zjg5?M?I?VFcD8x`ogEL~y{r57CUTqhs*0B{UoLj*m9ne(F-0@@-JPA6m-$MA*1?3z
z=iHmRC-1G6wsyCxceMqt^_u*PQ;vGssXm{XbfxTQ`}yL?{Pr)5A6KXMfB)?#V_kCf
z)YF{npZMI}k8e&tAGfEX@K}%J?QOZyd#ko?-fVo^HI!jv#GmK$>;L`ze&7D@m&^C|
zR%;iSO0)=Q9sOi^d*#32Z}0B>so0+%cS*ENTho70!Va}Pp?_K5w!Z)Gv0blx=KKx2
zCEY(we30fFc<XQb?y7ZBR_=En{p`)VyUVub$A#<Hx0k=a_wV1oojZ5tgnxM}(Pmlk
zVZrR#(OzC&B_&_h@Bi2Ju}5GPPw19c&!6AjReJjL^8KOLpO)Tb+Z@~dta56B=F1;P
zn17f3{PAN$?CI&EVq&1<;SV%28>gL_VN<zj+BC7-qJPZ=1qF+rol*2GdVH*RZPeDJ
z-7+l!rOU)Pc^N)_{5UVRO2SMwc>Q6XZP^!9W`5^e)~;t=|7=OxQrDoMOXuzX%gD>`
zFM4|F-rnlXvFE})y}hG1BpjS;T^_bR&bH)5z#l15rxlx4dOI;Yq@|@DKWD2k*=(C_
zxbTjGkKrDi(cyo79Js12E+ZpzW8Kqe(8k4MJ(63qu10N6>+SC5zWuajQiKjDctHob
zy?<|?eoh8dR<4rDnsR|5ASUL{;nn%OBJN)-XK#=G_}U{}n#Wgt{@kPbd&J9r?wL7L
z^7hjj>#jwh<M}c!F50ulrY(`9Dd7ChmlG77udRtpPE0&lrpU2S%4M&R6oUxY(Fcjj
z%1J@ee6~w8?`o}<SUN?Gr7t%@!@m00j(PLsZa)oSXiIeKl~NVp*p_qiP?@g$!G{GK
zq(z-pyjmIS#LRHm;Bwj-D~XD8l@${XoS5{s=t_6cB>D1!fZxx)v44O6@Z*)b*3IjS
zKRr2lb91`8yZh?H*LMgydHAk)e5OI6!&my>4D-jwU(CI0x?**S=!9p>kG*+ex5d9*
z`Tx81W*_HFo@`wC=}GPHZ;3X_EdiCUEZXZHe|UJ<-fz~ef&+QUx4-3FKeX>rPkSLd
zyKeNK(_b=!Z?0Ra^!4TC<%N%r?f?5t`gW^A%Y>@H*M7&F+4;?kjbC4`Pxw18`|PYQ
z>APjjo~pL1vi;jpeLT$G-hNy5N{{MqZ#;c{d3ktJb_=u!1X*X<w(EU;adGjvyH!{7
z>~5}l>z{M|P~Vyti&u(^&#{xdu`bpcqz!Vx2?sClTDjK^N-Y6LFEzBVHEh|kg>UYz
zVjb(-CGiXA-9NU{X61GBXZAM)%PfzaJlXj+N=wN1@?5LZPv5@HGtCyey;Y%Qf>qFQ
z>!Wrlsi~#<pQcSTu~ICMSsS!mbfdPg^W~H2&sWWp+bb^`clMDa!?pGC=MB!BxBq{p
zOp!xz$ug5<zF054>A(48ekptmJ$#UTCsXaV(<@`Pc+HbAxfc^r@bHc%Gc)tH(<>AD
z<?X+H`O?$VvnGDO9oX7gmwGHqH0|cg*F2h0?moSJch#xx_)W2XszF}d^6y_ikD6bu
z`^D*Ncly#G%bFh_uC5MGwoz`mke(6b#LUos_~yRc&tEzZFDfq)eX6n{)^F(q(KMSo
zR~N@`f2Q8t_u=Q&f|sDhc~Q~P{gTFQZ=D33Jl1AjihTF!+uQs0sjt?wHGh}9nt!%T
zS9Il*)R;Ssji1x}|EBtV-BYh+W3#9J|35=R!)42s<y>zPaLV!x7M5bri{16aykJJ}
z_p0FZvl;O}Tz{U=v$M3y`aO-;xt%X{_cSj>vF_N|SkQ8w?fLQBvK=`TOP3$y<YnNO
zvw3mr@NU1^^L3;Do+#73ul@UB=aS2ppP!w%v*4lAw(ONUlhysFd8kx>f0qhM1(z74
z%4C1p=H=xD3YxoKygffY=2V&Pebu+$w)JRpm*w4;5)ha$bLPo1%c)8#Do^h0ES_za
zn*`R$Gt0JJZ|d~v*LThiSG;~~-}yAYmDgU+sOw0akuENFDRcVabuCv`hrhobU(d_W
zZ(sk<=5}f5hChG)Ec2cH?)LWcmoHC#8ztc6VVv<zW>?7CsI7<c+bpjM9hH|_F{%58
z-gEl}nyF3~Dwse1tq$D&`s!+ShR@H>pFe;8{<d7{+e<??fKKuJ`4g0PE?i&D$H~j!
zwD7~7m%_2pw`^{`T`G_YD!u|wdOza(VQLn9XII_P6DL-D@zk5XJOBPZRqtsk>gt=L
z<5+)vzh5tJTeT(Ys@67cM-D~bAYUOV29HU1Hotz~x%ijs`xkoWLcdo`7T;g9V|Dua
zvfguZxP&-Zp4LBk@nT2D#YKY3ZUG`k!zx8Jl$DczeR=u&`}_L$sY)#ZzxKQ;ko{$A
zUH<M@<vFL%X8%<DPn7A(AHT3_&4;u0zx(6&WE?zho-iZt?ykzJs&n)0_jhHlF1Wk9
zJpcB#w~Ji6Z|JcowwU;s`eZQdD0z8lp|azvKbNY?-rHn6IB-5^!n}F!-o2}Pc!)JT
z{Q4@bRgZG-?kc^%uXcOhT_w*+;BY-0a=vAq9G{%cj;yPzs=vR>d~s{Or>AG`tt~rC
zUr%dpW(H@DX9rutvYFZWwrtx5Dwc2RJ+WV`@v2?EP6ATj-dM+~*kU52wk7KH>C@S_
zw!B>E+<t3&{`oT9|7TB4)m|UFyX@g1)+PnGx?R@g@9s>|3{FZ)`ts$=jdf46@9wYP
zpMHMc_xJa=Z`hEqTdYOkQpk)8QVEAzIJKr`U0)ZQBmOBg#c1a4-M^byxmRWX=Wh{M
zB{k!Ml!K0#dG4(*=k5Q0`S>w$_cZ<4mc?mRRlg=GyF)@pT5U_(j@sYb=FgA!@#*R8
z<h;GL=G@iQ;qCnL|2{l)PEJk+=Z&*%oNEoAot-^>!i0i9KMJ>T|6KCu(IYkAS%1D>
zkGHM*0&!Etj0;lE3j-=DD$dL>On!1=V$StXg^S($pMCyW|M#mnsIUzvU68m}D%(g`
zckSQb-^<_Kxq12W<y*IAy)C*|{^`ld+2;9f?(UmoIh;5`g)Q2z8Dw8y$IHpNv*>A8
zRPA{Sb@k<$mzOy@I_BQmk_ZZhRcfpJ^H)nHU0V~WH8tyS8*h&Ir#|`mKZRdjT$DJ^
zp%~Iwz_+$?o_+njO{u3>hp)G?wA>iW&2V6zZM9kMEfGn{laD`w!+jOMPe$*S;^*gb
zZ*OyTaoHf(1uwKq%gWNW3$+NWN;6n!8v6d;-q_gK>H6{Kl6Xyg%U@kNdHQtrw>LL+
zrZoyUT@_N>vh38kb7iHaww0e!PEXUlx$0rR{XY&qK0a>l#|0nS*jfZuCCs=W1v<g>
z^Ru(nA09Mr+g5f}{_n@f$3aW(dU|p$EO5LkFRR$H%GM|2tXa|#j@sJ4XJ?y-e=MBp
zwY0tc_!Q0HV|}vGN88;v7D_P*M{O1r6*bMglybaJ_U^9I)u$foMQl(oGy8UBWpHR^
zK#RbtK!b&*aTysePE1smx2wrWNeMA^=i%pne}8|yk5A9am6=y%1QlDd6ntL1O1QNp
zbLY;T?R>JUnvy$@CWVKC?%?_M`uch<zIFj8Q!%xYQiZ8rvrV(NmAt&<*vuCCZKIn1
zJRPy_|39D4KV>K2#BsIYhw98%4jLlw?(eU^wIy@r%$X*>`jMNK1TXg!HQnK%Vq|9a
z@B96FMg>S|$jHbjYHH!cvC#jZWy#l#IX5?5xNu?8q)9C;ES)ZsX3vh^UH109{I@SJ
zFaQ1ZRhWb2?d|RTot>5i4;q#)fBv1nTfphX=K{Xi+8MKExw*I;Ida59Mab56@9F9K
z;U_jM_nW&&!%IgjerM6r6)QXz285j0z@gY;$N&7oRmZ}@!hShhIlCGSvF`gemIV(E
zD7*L7{D1K5*)#q4eKns0qN8u$zWw{cLT44vy8jPA`dEc`S#|XHmlqW&c}`li!rWg<
zQu68Z=k?D|=*R8JxUyoRk6QI-p84m~CERqxig(_5FC(Maav}Xe<hxf30vEe!YHI4~
z={=qPKSV>Mu(0sH<)>C|@l)$}u3ftpbY}6QMM^#CEdr}@W?Z=1vUTg%ySvN({{7qP
zl=#wE#I?1hrRMntM@L7YP8J4_Nmnjkwk~*ZU|sBP7N&#Wr-6dO`-4t-Y4O`zTmSw2
zJ$=d)jj5;1a&O()Rr-2%`yvgm`1p9Gi5c11)>c;YtV&OvI<<<)+m%D{i=;*Sx;(3r
z7ZYa80JT@_=GXuJmb)|N-&to@S65FJp@jh!xwp2gSfK%SnwLfUx&(1&C$n|YtNEs9
z&fn)zIcLTX=WemBJv{q!TYfBAvgEmbUB<mV(+Wk`{`f1&a`5Jt@At1=yOtx~B;XX<
z@>gMHnjedt@KKu&n@!Sp3%pQL;hy`ZBvtiJ&c*G$7sA}T3(6j7ADy84@6)xZIle_+
z6%`h0CQd7!9cWosA?w?-dv@4?q_`D(CMbRT@8!U;a8=9og@R5Tilr@_y#h`gE0ujR
zS_GUtWD`+WPt3T$q1ZCP>QIXthhobWK{e14izR&qP_5Zrp`3@V6ui@!8^y%XHF33s
zf@O^!gT}l$=k8XnT*P-M(|DU-efose4X2+n7&r*2Tv+7f77^`jZG5<%^=q#4L<jM`
zBC-5>JH`J0+{-cDPOW)2_m}^-Hwrj$)QX&2xVmlPfqPy*UpRfcpMK!ZB!9cJKYuT9
zS+h8HzaOhy^S-^>`kP!5GEb_^t=^eheQw>_R!*aNd5uh8EN}gP>u>n$+qya3kxbs+
z6?H$K-gh)w5ng>JY4gOj_WQ+!rPn_B_dHp#Mc|i)&x=)xlke}J@cVpZXpqrTrl={a
zef9)zcsgZ`2O~p5(EQf{|IaRK>6Bxb!1uM&{CtG<r#GilEf=0H%{*m#vHEvQpPs<7
zB~#};%Hm;l*k9$g)GkofZek2aO{t<!pvvzv`{p~j+Am~SG2MT~k=d-$P8tkr*|z1g
z7iOHeSZTaW^ZZUlMh2g*O&*+DSF67^`_;}XKD_<^m68(1hUB+D%3ApEv9t&{S=?PI
z<l}Yl+1~f|Gll2u<ts^FW4cMu=+CW~S1(V`?e9Ha`Eje~)RsqClYJb0Zhg7>$FqKh
zg{I2SWw-uVYZm`!o31#`c+ZSAI+rd9T~apUe?Ft<z21fA#taOqmfd;`3<n=GGrTA>
zn}6Lf{gRR+_og=;@88Yd_-fwGY4a-rW{4MfSd?w*GwV2~bG|nx>+9<NSI6Qb585y_
z2zoO#<lLRW63}^+!J%wUhTRHDW%Ww8JCB%8f7)!TyWC{@@#nA3Y&#Qe(Z|TDn31#d
zH$%J1%tyy1|7mw5?w?g|HhJmEmd&NyOxuco_g>t7W%<)0zWMwPQ|J76eDx&z)B7!J
zBDtU5P-a$aDd0Q1NliFw_chI1yG@_h*euM=elu-Sz5VRQNlo|NmdI7RNLXe^U%#Fo
z8ZKIGRsJ$c^7<5Si)%9<9=>yX;>LusSC1Sv<b*DFo~R-(p>dM?#;o#(b9~r%8n%6}
zS)IH4Rj%Z6*P#5}_4OOp-_&>;Zf?vN@@cnuQQ3W_1y3&eE9mx4+EcOhvlN2?!!5gR
zU8ZkQ{Kdg)?tAW3F?Cd_Ew&E$?zi`j=o%K?+WiZyWTOg>`WIc-Ki6C9$-&T&vf#s2
z{r}Y)pW7|GACM4xe8E~<6}B|}nOaL*4za1Ss>>TNY+5~gUVoITo?2=3?@7LD)9Qt@
zcry>*I^HPY^om7zmamPtG=spF?-#c(_FA;qOD<h=v&({06PH~KNnzp@F<AMiC26hM
zziX@6Mb<^G+*BquVL`=2K86WZ&T8!I7V4>{JIvGES-tql1y894W(B)lD!G?#Ou6{_
zY0;;*?0+WK@;+U@?qb00iU!@t<17tVUSI2beSPw06Q7?|>NoGs_R^c*IO))SGtHw6
zYG;2kEZMm_aC+OO6)P5Hm#ci26F#=~)qB1NuJ85z7BPARq@FI4RC_1MD`K#7=j7#2
zZL(v^59{R$WvE`z+{vSHWV#%~X9If^!5v{$Ywy(yulf<}z@hlHC4J{=so;0}Ep7A+
zGS^>QzmSo^VRg2(5W|V`ZTdHNbV*t*E4zK{+5Y@faV1k3A6zqIt>97!^kmo(wLoKk
z_aw%KOPlYMJUP9ykcVOa7xkbYC-zyocVA+c*g8Lgo#jzWOz!u@G_6$`Kc8;Dx{vF0
zq;K^9SHaa^wT|A{$+b|7A#_i9XrUA@!-GqG;T0y27cAKpFTVI;_s$Iq?uh0yFesGp
zPFrVWR5#D8G;Zx%>HBe!;sOjjx?zkS5?wX#eIFbs{<ZsFa*Mz!z5>43qdYrm7`#G%
zIn4aorI7Rbob)+GpVIn!DYq`oxhBT2G>M_%f)c|Err%bp)lZb^)P1}i%9K_7aG4KN
z&37{y1_tRy!4txdB}7^Co9^FhV@a6Oy0l+nBL~ANvlDwiKHSF;Q1jAmVgAag<;Lk>
zExWEf7CSm^%ChwVtKZ1GpV}X({+Kh*_VfMag_kacI7sP7X>PfuZ2c?x=l4rl_F>`w
z{Ga_$RN~}_oqJDAIh2V(`M2Z#ecNoFaiw1J>E>UM%fP@8wS<w8VOOrKg8DX<x6S4Y
zC)qQ%2wdWLa%rje&2<Z-ETv`EO#Zg0v+<vM-Llt-oBpWh#>yq8D3@HAe5cOC-dj1w
zw&YC=&+l(no*hsBr@hYjNb{9T>bxaeYOOw(vPNx-Iklp5;&Sc%3uRB*oU|`;oU-J~
z%PV(JY@XT|E`IJaxBlW+S6;q6^6(Y6REFmf-m}Nbo_(^o!}e~1ysp1EV?)?}$*s|5
z4>DdBx1KNRx7XgEe{CB#1A`{(p?hmxcZZ$cH2K7~y(?GmR`q0k@F3Z2w|xKchqtyq
zxp4mI!A7n2bGyF2{c0^C5_A4iS;>{xSEBEq-H`D0ywT!5HRraBo7=@^2)a(+#A`d(
zGNv|2`p4cX&fcwCWn6@ow&pFq#+IAQ$u~d${-odui=O7_=UffFv2K0b+BI*UuAaJa
z<<t8uJwL5y_U|b#P~5*FbNae-=gv+2`BJfEf|R1fv&H-QFPhIX$y@Vs?drp8boSMW
zA6jCU$+GH*c>I3G;|mo7CpbK<JDvXh;l7qmxyjK?3=0->CtaN0e!);xw`b-N_0-9+
zho)rb`>5LetT1iSJMPu)cb4tYlr?j`1^>QY`}t)2&u$i$ke%UQj+!UB1)b`+n$6BD
z^+i-)P_bnd^B%slc52^SwZ8Aqbx^na`Rs7*j{Ha)=9*9&<_3#)uQQt4>-$#yIsDyS
z{Nl<*m(H$;X%JJBQQ%Z-_o{lH@=g3cON+p&%nw#3tDJV_`m&n_K6&=qHzN7MtvPSA
zK%)!~Wu!tCFP@zfX?|0WNwLL*`5d%KKuT+(PJY*;*c0b2KYSz|eehk{^PYs=pk}_8
zgo<d+g$s#4*T>mD$T)D4bES`1<$uR^v$Z$%o~$k@EXljQP4{-FLQBBagXbhoK|95M
ze|sAo6a?;>U2xAh5W=t6BCx6v)=VMAO=v9~YHRICkZ5}=6)@Gy*Z1v(h0T3^e6E3&
zm6gkuEfaBl`SPW!o7<*XDJPDF+YJ_G&w2DHiJ7_i`#cj<)3fK!r7e|f5pa@kP2ZU<
zQ(OCYUF_~@^==v>=VqJtudM<%nbb;rO%e~W*xK$j=g*uaXOzOx{Z*kw;FrL>r|UXX
zjAm}!xG_XaxAw)uhlS72$(DILaVUO~o8x@lg{kqt>C>yf>@O`TQS+H`AkP-mWN+b&
zb?%imU-!jjyVufBU%#gAmT3`idVjcO-NIcZFN1!`zrD5f_lIkFEdow!K!P5VKv#7#
z^vK)K0}TRj)Uw>uTxQ6{#TB+D;^T{pi@T&YY}jC+rL`(}xu1$>*og)f$BN2I-+4AO
zPd^Pi(ZHb?GC9#Hn_;Tg*3{F};_Lrzjoh5(=jSJCu_;FH`RAXarc$nf=K1$Tx{uB_
z%YAiobGj>Uw}6wWu-X=Izkf^S<&u(;KY#w{=;++gdwx18DXFQcNk&$dnTbitH0wy2
zC5PgYl?Ds1hR4L*xw0}?SV*X*wsy8z?x`}%|BYTtxwyHl>;L_EaIiUiO+?`8FkR3%
zzv9_u&b57?K7A6?i>dhW;UVZE@du@kwA6$;{pQ=%et&njm0SGR@86#*Y{1HWn>g3T
zP4P-~a5&H<stvjndv|$$Y;5c{?O$;-XU=SFY&?4OsI+<Bmv7(FK*4!QV#bB6Cue3F
zUtZ>W`TF(mpFgjT-mZ81X~>dxet9-l)|)qPva+z4<lUL^wn)IqgE_%zb?fnd`QN{P
ztE;R3{`qs~jvb(3!37$xUR_=N{M_8npFjV8yZwIF^>t^<Tsah%Tr*f037Sv)`T03J
z8ykp~e7sMp>g3PYokt&CUhePi>Uws*J*c-?Q7EO@V#g{x%hYRW(cv~;$SmK7Z{Mz6
zyqLIq(W=Q)r;18QL<9tED17WDFE0<;jS1>Job3x<?&mw-?(U5nH?Cj5e&^1dw?#Ju
zrKP1MByKElY;I_1cvY1ruh_CrQmrJk`pb)pQ#6C)_EbE4{`~oyH)qN$brxs-dUbVm
z`uTZp@9+P=Ds;7vkI#*Dpw0%XTFKRuXV1>HE`Qf4tiET@9;>o9Cmtr;o@6A{`Q*#X
z%jNIx%(O0l*CT1XWy=<@mK*Jyv7#beM}K{N9lk#9?y6Aj{QUgf+}v&2LGJqT`*^sx
zPMtjq+Tp&x?(YpfcEy&6HqO|Wese4gYk!pp3k%!Z@3$&_wIQ}|SM{GCg$W4@rcS;3
z?CfmQ?6tuy0;?isT$rk%rj~YUiss9gFZb=+x5MZ7^~IULetmtt+<*SN%gfKtGR^k!
z^#x5xwL~;<#%dN771jNGI{m?e1bzMWe6m&#RG;T7O!PQ6&sJJa&My7j9MBefrrjw@
zEfd@hw5&@g+WF_a{r{NVWx21ft=+q4&xc8$%TJv<$H&EW>fE`w{dIfSuh+l*6kP7L
ztn+YmbTm#ovm$tT-}?3XaYgGKK7RZNI*TTH`@3gnXWzYZ2kgaFOll=ZD?U6pI7Ktq
z$k>>jN8-YYz{T%nd?SB6`qis`fH|@GS+30Kweh-V_msX(<kaYWd#mb(<lfEvx?UY!
zdxQfoR4!V7^!De2r;Z=5uT3zltl-|eW5tQx@Be?Xzdp4_-6ZbF)i5pIw^6%pOEO+q
zdC~lFoEL}kZ9m<&l@8)*NA6ZEd2{Ko?~khai&{2Ju=(wntF!;-iltuqe)BDhkKMjq
zeRh^<T3XtDO9dy6m6m(#e(jChS7TZ9<b)ph`Ze8{9Up!_mSm`C`)X?|t=1lEyY+jS
z+~03Mr)1x5K78oXR_Xon=NS%MT${S0YR=bJchfzYrTwLy84F5$c{};*etn&?l2g*_
z!Ofk!wDjV3%~2@O(QZh7n;}?cdi&<pdF-d2m*>Y>R=+5lwdT%#t<*<bPg^akx?go?
zyM>J3@jlt<`tko>U0v<gE44OeXOQ0X`u`g^6qjr>Sh)Z6EYoZ$)2uJAuC9K1dV2U9
zB^8-{(-;_DM3+zIIyEmS?LOaR=7xs<WsdwyW3;ADW1YIMQLv%mNkfoUz3gil@9TcK
zGQA8A;s57`UG`e}<!zXQb|yoV=2Y#|xr+71pH}baj{fgCg_&W$-*m&dFV1}CwA^$_
zUevZe>HhA`tnv&MytB5yzq{M~v(wiyfy`rT)32R<C0X<S_QzQP8ecwto^4&8S5UBF
z?b^3DH>Yc7rL+hHna6D7zK}3;=1gvJy_k(ju1jlVJzTGTm&}@7_9e%|C;NEcFNXDU
z?Zs7Jzgm=UTA;(YwBNZYPxP8m`;q9Iynl{=Uw4$jpzyW$KPCnTi{4qbC%mTBJiKwR
zDNXqJ*7e`ZQg&%P%JERkSblF$WPU^a@~E53{aEeJEy_J>>A(4fpYrpva-&mSe0#sR
ztO=F*7jPqc+Vqz<>+=i_{HczzIIw!^!o9QETjcgneQ|9)C;Jk9J+<xf*I&QA@qS;&
zqUVN|d27#V?W&);a^=e-ox;jWN^1V|=1iItba1h9%Y>=}mS46n&b)Qu!iG!x)_R}Z
zam@F2$#UcK?#sOxyz=BHmKCgDU-9>G(~j=n`B$VZcci~R_|JLmA~X38E#7x__PUXq
zv+De3AE`Yq|3OD)_KpaTdw-|I+wQ4|dU|1LwD|gae1eU~yIxyaW)yp*@4t9C{O9+g
z9cB8FnqGGbELPU8*W0`2SJ1}g`OmMtG(6Cne6c*_eOQ|eL!akt#<L;i>r}cWG&=S#
zxV)=T?_=lm1wMNuZywz1sQOoV^Tv$_cWVeZU6oMV@+;}dlzCgT-J67k=1rI`HTUk;
zwKMfTS}mNK$^Sm;OV$B)h6h)&@ATDuGl@US@G#_ljI;$?o~+e{^Kl{?mAjZ8?p~~Q
zS?<^s!BfGHgpcf!JM8{t?b+uH4ZY2({0$-ull8UFon>fvlg@d5lZA5|PkxZQL$DD;
zMe(Owcb0DDom~6;<L4#6`K`(_znqlZWb*i$ad!G@t{oh`0#2p^^PYyqdo5u262+5U
z{q?}Zt^8+>>1T4ET65iAWh+m|nH@eq`KQ$|<T@)i*zhql^ekj#Xkl|>S$OH0<Hz5N
zFBM59=OjIU6jjx|V@tpi7KVVpRD1OuCaS6)%hT4aTY52iPH@B#hAIEE`|_`Do&ERQ
z<J-xW_GhBMo@CIglAC;a$5cx_lO_SD`z+5dyy{vXDlk`*eZ}1kJGBMwY_pZ-;8gHr
z5Ge3u@RL2A_T};q8BJz}oBr<#Y-6&8MHmER7#h~xFutS5`<1aGMrn5h7lX|t76!fk
z`1KplE>8ZP96#;CsY4yLjW4zMI2mGu7%Y}BFqC}FHeO|%{%(ne<4n=oZ#A1b7BDbO
z@ygt|<&9Hu?RPVV2bZ?be7HOG@+P50eAAf&s=9XjzW((?pGUD}AMehRmqFXIFN7RD
zdi3`8{P(xEX1DhHbQ`UoV<A0JLh<VM+?3qZC80@gZ(VY77b<=BX6uzXcMc`}wfg!-
zX71XG!ut1Jmz5?RzLcx}z5Kf9gi9^nj{ZkovYy0-{`Rz;BO<vm(R|&~qbU+MBKPj8
zSJwNgd3gQf8+Uvslzlz2b9J~={<YV($2$FXRVaSj85D58r=sTXqt78bGe4WjZB0pU
zYIfFdzjrm)dg1K73{S$>&)HKK`tsVr!1?YB42lLOXS}v%UB5Twzgd&3<VHos<(Gq>
zE?Ax&=Kb;dNpZVcE2)rMnNk}gWJE<xlaKYBvSM)J*vJ*G^Rw*b%a@RYno~EjMQKi7
z?VzEv-K{fb71O#2|66}Ae_Etj@T2CQ`tlv`d)G`k-S~g0<9%5Ny`?`6b?xijJ!j6G
zIl9r?mMmKqc27~jsl&BGHfnQW#ryUL?EW<}^Y#DyVSC?u=J)wtZHpSkc@Nz<7Us9_
zcSaepv^l+2aLUQ@Je6(<IgP184_;nh_dFv#J?)2S^`%R1es9mWuSm75neck^B|RzK
zv-ZUcj>q47_W22apZwC>d<EvBHLp216k8Yy_|_^`l<7uu$i;F_jVgU{?b<Pg{Nuku
zt=HTCo@Hz4lpLG-Yjck1b6@e^e!*0mvlE$X%4V-uI=f3u)6n=pg&W7h_pO|3HMg+b
za$#0xR*c`?n61A5*q-J5MoSn>e3><#rrH{0tPEMDv){x_ce<A2^%D{dr^+vCs7^oa
zZN98~z8udEorN#?w=G<%6!=pxQ<azDQ<1GX!xW{7U(OzG0+)Y<Q-xXte$`b>i(Xvd
z$-uBcQ}S@Y&+Gi}Icg{5@J%>c`1Suy+fRRL=dJscw&Y2@eSe9W=#(XH?{0m&vbO!+
z&SHMg^yosjH#4utn!I?l#kl=^w&ks5b9RT+FWLX}a`E{YU*7&c&y$t^*W`&u<Z|A?
zs1N5Jw{#c(JL=t)_h0&6(EnE-ABW4Qo8`rP|M=J^Z}q|M-#b6L`t!?oZToCsU1m{v
zYSYB9n$%BcPw>sPdU(H<L-EUBgN3H;4He)1d{lnh^ZeoQd|P8ltG%1!4o$ouasA)t
zDyO)Qm+iN`?3s7(V$(ynJe!JF57#T7DbMA6wYOyJ+6fz;rseNE{vyJ{tZ<_RzYIg=
zlXv?|e(Sur;$ZFlA?fnko}+eaA6)zU@^Q_pf+<##OK*E9m;F<HZIivATX0M6m-CN&
z6_;(c+LlvddT-0;1&Ynw@APjKm@c0_`@_w@?rObkyRD*EYG|IFqIbl$z(nszuYi*%
zgK$*w35g3=O$`i6zs|gQb<4B~%#oT0Hm;0!zdu2L%A9nG%i)<HgVsIWu<-w7)8$2v
z<0sj^;^JgbIBl0QZ=C~&NQ067v!$i^-nk40`Icp&-V;(?85_Jf57h7N-J_t^U1TiC
z#PGp)PR_}Wq?vuaa>solX7fjFJ(J@g&Cs)UZYcZrbH0BQ&rHzcb6TXKr?$LsHP?I7
zP5~!V;d%X6RLsP_u35Ww?b)oovD_|P3=Af$Ei<D(KU=qM9q1A`wHVIBsWWOkZ+?Ba
z>G!r%vyv|`ocVm`(I$7Vl`niRuiAdQz=0t^VG=U~!^+s%hXX`-4?Niu%<q0b@8Fq_
zvMqwU|3#GPySH!`Dz^l%9*SQ+tI3(k<4ITfd?AOI?kBEqZ+r9h_qt~%Gb`i%?5khw
z=*|7&#PRc73=S$<3~qW$<I68TcpV{p<l*K-Nu`}HQqSCbP|<XSiD6b)SJ%?i+aG-9
zNENTY>mA6%;IT<5^jz4n1ylCE>vHsD+)@!R>65|KEiWzqZ9mvsS?DjEXS+EsD>dM1
zK93VeXz&fo$G1b{p7-!yIRCCCX5U{WW6NH%HQB+}ConANj*qb^wX^j;w>mfM{Fy6f
z4r;tSw30h9{LrDq3H!ItTsU)W-NXIt{jrrbPwRXH6IZSF_nvp8W%<z*35y+D^TOXB
zT{&6t`r&o%ulncQI2qyZUG+5P!c}dj;KxTc<<7B<+M&Jf9oNYeuH?LqX#Q7W@p46v
zJhE2p39G&Qruw?lJaq%MZGCQ&dZpLBmVS0zRcgoWsV&}fe^t#8p8CIR%As?T91CCZ
zTBp}om%f<ukq=ZfS{&MbI`gvEPN9k#xtcdAoVwG4^Q#jt6j>hqy<uWokk-=u7uuXP
zr>^?pr0A|Woyq^l%P<*HKI?RamR06^g4am>T+qw*y!fx4#h;>H<@?oN*!+LAvPm=S
z*<{aI5&!CFqkz+k{RduSovZm^wM9)Z^~k-NDd!%4oZ_X*aQ?)id5fESAG+%+Ouzp+
zwdTgr6;^Zq+URW2-lMWq<M+R_>eJ@Q>J;eawFs;VuxP)gvP;a5?@&+obh%qwvhHtd
zt@ZpdFVwp~<;4Ci;cKn#?5K!Ty|P%RrporsqmQfoU+Cw9rquQyY`LZ*v(SMdVU6;Y
zYim>A-n=gU;>GftlQzj~Jq>z!d&jb;K@&qb84g@FG<sa%sdu4hWen%jZ30ts1Ak4>
zlU-a>uCp}cm!I<L@{c^Z*5BWy1sm$~Dz;p({-Cqn?a?O&hE4O-nHx6R$un$P<g9Ni
zm@V3FFY;}c@tU=-P9AOM-&$wjzh5`oa_er>jKhy_r_0^#tYP!n{c!W%*sSR``}b;F
zENxvK_u=QBkDq^cFFq%`r{eydYi6%s1}mIl`_*4;zFhfSTS`;etFyDevi<m;o%i!&
z(3Fs?dvjU@R(Tfitu3rLa3t2E^4ERd&rE(Ewe^2)-kiV^_^9o$=iBPimrt)w@3O5m
z*8F~QpTTFfytCFnc4>2aPk+beJYD|!nt69lKMZr3Bs1lt`MHT}cmKI?aQFPkBrnCA
zD_Un;I&5ACT1<B3ys5r{fz{`SM=xu&>)qW|@wKdj!)DsIrNu8f6kD{!)V3T8WIDce
zY4qPmI&HVNRTl5IVo*4V*38>qXTQDr|DSEjD%%Rbi#ah&ZOMF-a+L9c&=j-V4|euO
z7+uk^e)8j=aHJL^r_I836E<wze1-W!Y3^&+^rOdYJJu;rS$;&lqtv#o|3$QPK8rKQ
z!d0!Df8X-(@GJ>>d2zA3s?f>l+t)et9NIAN^aaDaq2}LYcTeVH@bdDU;CD7AHg>jo
zd1<L(h{x+KFN~9Y<O?GntU7wI(uAR*ORL^;W?k9a6Y>v(&i`xvo3k^&Dtc<3)2AH{
zSCT|}Hg?YKUcGwt>hA8zGh>#plz#g3^fWU&-<9?8^Fb?|R%}Xes<*ba-7Bsi7ZDYu
z7qvyh#AM2%Rgb!K+G}b{+|L)@m3b%6zhhp^lC|-(1Ls;6tGjZZTJvJ_^T}~{EyeO4
z+(`R%X`{paNg9?h3Lzrw_qH5f^!eSK?zazK9S9dovt8`CkcA;(p7&H9h7`M<hyO0@
z+`RcMN|SMw|C;-LlNc_o+9Reb*4-~-xhe1NE<t6tn0+-nx%ztqoI<&(qie-;b93e8
z<@NQ~E4%m2u_|3v_V(7YWy_|P?|5{jI$b~JtnNw8iLbA2-o9nSR`ImEcEUxznKyS8
z*FN$oD0uPo!tPzU>Lr^`?%e;qXw|e=p^G+!v5Wuh*>GXUZ`K%g?MI0bCq7?cvbDZm
z)ih=D@^i;hy*Z73U%UGH*}8W-AC~K@?o}#U+`sI``7hadS$U32+U8o6fAxBCwrcP3
zEnBYq-5?h=rSkWqryW<)TZQJ$oH})1*wcRp-2+1+B5vHT|9^K?DCj7((zmy!s+`n+
z-zMNRi?cd9bcS*IIiI;!S1(++aQ^)HJ9qZH;$ASdXeU>T(?56lx}4&!<Ifp2o_fj2
z8*SIOx|@AcB6z{ir)3jQFFo+^bLE^l&rh<7OgZh$<>9=WtyJs%(>>~DW@ZKk7mgiM
zQ&2c?^XAS|ecb|1IS2EjLnZR_-@m)No1KMaO~l4UuH9ny?%mt9OMYf~*u3vi(X|&|
zzB+u?d-)z4JN^X`UwBkR3Vt3c|7HC--h9oXB|r7Kr#$_)bHxs~UuB}9ztU4vQ(4*A
z#Kgq%Zf-hy`}S_HzD@zBH-{}tj-Eb!I(kRJ!->l7Ro~vceED)EV|vodMID<L?mWPH
zp)BkAxoZL4Z<|haxGXYGKj-4=svEQ8!u|X6kN>_CFT<f2ayG#!eA<p37WMW2mn>0<
zkH4?xJ8MI1U)w?rEfW(H6_u0=3mmn>)~pC!Ep}T}u_Yk%aLc-w=jZ0mwklmUZ{EE1
z>({rovfh50(KchojK|0OZ{NPn&cb2=T4%%O#IccCc-N(sA*;^LHrJ2ew<mIQnw_0p
zXiZ_%=9H8a7FO1&Q>NtH+OjfyeH>^9f>X|+mUT)$fBx(i)3qvj5%BV8rk?owi`DHO
zDqMVg`|AJyTjbik%x7lPdwwU5M5pkT>i+Y7{QhlU`Dsc0|9|V&t;?CdSS&;<^rh<a
zh0g7I@%!%FxG@8CFq(&Mg41RN6_u1%S5`VL1V#I$Pr|l~v)W#~c(Gu?0=Hf%(C}o=
zjS1l4$=(E~&CW}MK<mM7Zc5F|%d@kytE#HX2@kn?>GEY}X6B_qFPHhwuKM!gAb2#k
z%wXZokbr;<85b8dv-6j|zqhye`8m@kkFpC33kwPgN=m*=QuS8zpBLjCXX?c9H+;s0
zH{Ko|GptIpa&p$JU%!6#Z0XxiLzJd^mHzznlv`YnN8YaH=clJ`Zf>9@`%bU8g`;9)
z_EcQFe}De@=ll`|3!>uYfCA&=<KuJZ&Na`y_2k8i6JVFTGgx?)!!;1JAbVFy=H+F+
zwKX+gzJ9&AZbj0PrAt}a*iJqD)X>n7cX!vuSUx9?Og5j4t-%oyJ5o+gs{i+MdG+^q
zvAaqxzBs+*=+UFg{pMEv`0()3Qg3zNSxXqbrzp2fxOJc<EZEc2GkCdQZ)fMr`}_Cr
z-feyR>7R71sjn_A^%mEUlQGS@a%rhIFE1~6{Lq9)ZA;YdnxC8Y@3;4y#LULCVcWK}
z-E-ELzQ4Ek@$vrMyLRQ>*>UmSy?Nk(S59<VZQZr#-@o7QUAx70?%46=<>liC4m7;&
zvQsxT{d!|#vWRQ$y*)dNpPvIK7bza$C}R;(QP6pN=jYjmg@xr@|Kus+`t;c|z4(29
zj?32{@jl0)_+_raLQ~JItXG$p``=vmXfj7r#<ew(TX(X`Dz@Bcg-(s|-DXv25pa5;
kY(T~g5Z2k2#s8U8mwb))zEZx1fq{X+)78&qol`;+0AhmLRsaA1

literal 155610
zcmeAS@N?(olHy`uVBq!ia0y~yVCrUIU`yj*V_;x7Q2#E1fkE(ORY*ihP-3}4K~a8M
zW=^U?No7H*LTW{38UsVct+g{Vi)1$Pw*LPtV#yTVV8h2bDRie}-+X>s<&&SLd1*~s
zJJEe@o=`;vx9GY3|0)mL{|NrvvC!<Tk&9COs};)r`(Awf`uyvxq*}ZE_o58<-~aoq
z?t$2O-ydaNt=B*OYMXOCd)A)&`|2EC|Ni^+@81RAekpzb-h1!<j~$60)qhLfyZ-fi
zuUFmP`S<?wKmJ?1`QL`Q-|aVE&$?W<{?qdR;Y%fU7w(_rY^OT)MEb|&B4GhlGyloW
zPZreca-Cm(>~my&{BDJ}dne8B`u_8(|JLr$f9rn6b$!q?ef@8x+xzdqziZ#l<NCW}
z(U-4J^6$?O%m4H7Jpb0+<=?B%XP0m0nmXyjy(`DV=B?jR5P0g-pBaape_sC)HNXF$
zdwRFqakINIH4z@lsan;+zSd_;E?36sN_wyPnex;${q)ny_2OrhZ(pABO8#ZqQeNu|
zi(`7$iQYRmKil(s@$t_#-gDL8ADL=dxcP|xpHG5y-=F>e^s#brwd^j<qyJV3Z%Bx}
zX?WIXPw~H*t1~=yy`Qyh`M&e+KN!tFwjUF-x-rkF+|1%})!n46Us~6G`m(biw%hc`
z^$eA}@1+Bj=G~3((JE2!($e+pnzebEi>&qEnQLsO&k~pQSo?JDp{3@no)_+Hwx2!!
z;+L4&(RF)yw_fN7YYBX5r(v@C8e83#G6R$7Ow%pbmSn!p-J<Eb%0w`*uy0n>maO2w
ztLvQ8_4j_ewlw>F?H9{W8UYc~MkYOW$7W1!_sg^W`SPS_@wz>yR;|{J-1SQ;Tk6Bc
ztg}b1?Bbnm{eF|=f8FTA@2cNaUSjlGKCkS_i<jwhDi7Y$-F~m^>8rQpcYfa7`bF+`
zeyq2B>AUx_-tX5{9)DFlx308zd#HWh#5sPmzW4M!o%Lcz6+1)8w|PhBX_fdt-;-XL
z(zYmd*R)yY>FVz#WY?%aExVllexKF%>-$>6{;%Hm)V@@&;`KlIx!m`5l|Qe0dpm#E
zw&#mAo@P$)UwGYQ8u!M+i)ZHUzwqdLxW(~R=XNidekJT=?$fHW$mUGjw`q^3mCasS
z+0>J|E941B_0pj3dA`%;ELaxE_w~KOElux7wzsxz-FvhCcIc^_T$N8f?VSu;PA)3q
z%?ZD`Ip}_C(%Nb1(Rcn|xmj6c<@M2TXX>rYOr5z)&s;8FoF_EtjNtWMyk4t{Cf}Mh
z<@Yi-UCFA|JIyq^RnJag{iJc?u}#evTl2lC+&peK+H>caR6P+cvM`l>?8Y0Dpt-K^
z;`W|n`<}0Tk#Tj)^1FG#;rB}AZe2X;<oBxd{mxUfdXEQtEPXjw?v2gswu`Lu7^NO0
zeBTu@RdelH&(-TEIWl-Rmd4Gj*4^dUDN?v&7Q>Tg^HwKSEcqTJbb%q;`P<s;#qSq~
zO<=B=IeDMLw%}iFLiz7E&o}$Fv*77IKMwAaixwX$zOKp5zB)}q!vFQZR_Tf7uYdTZ
zbg*sGHHq78r+s_ZoclIm@|ruv%X$)5J$?4>b`<ZG^GuhO-e@<+{t5K{?lATF6W?^J
z&1Da>Ive)%E&g7~7j*ML?BnQU<6@0RR(rWzbCNGjY5ID4wwM0D%*}gLzRzmASbWU*
z>*fy(#n<13Up)KdMAl6c?IN2T1H(0cG7==FJ};~leRcKGrgJ`w!9IzNW!LWQp3xl_
z8MtECLAl)u&t|tWeYbgemp8%i^PRwV>(u?fx}UFUPY`|<dh!B)b+T@icUq^Y_y#xM
zWw%dPYrSi7W6inwB>u|RkpEXDwzu(SZHZoUGHt5Ps>?6B{@pfAzuZ}Vy7XDA$Klq6
zd2*!-92%#6J*IZ%?Y1b9{0sq(uQ3w)_HLafn0k(_ZA-6C?G~PvV<&urXMEvdVO*N@
z;nu!QYqQPVmTbO!_D{Uq)n|)huNNKuR=_^*a(3*VP8X90VmWg*vg({!C~Y=xZshsb
z>yG&*7Os_KTAw>_mjFYsglVUtKtL^<TTpkyg#-(p|M&BXxhr06-_*lsy=~u0m706s
zWC9rHh;3~&?U9(W@5hmU0W9x!f4J!!`{7l?ZZG?sV`i7P{rooT1momee4m(~ew}xF
zu^or!tv8J)bE8eYZEZKSr>=FFbHTEOb;g<7j8ix6*jM59cC(Y%9sZ*&o<DwGeeU&f
z)3Xy2vP<+92sO<8cd1F{txE4^-V}B1tH~*;+q#_|?SF8sYp$(_qIcYE&%7H`CNJ0}
z`N$&Vh^eaQ+u7$tU;JLsca57#StBU>c&aP+<!v=nZOZMK7%jj4&~Qx%+vMN6@ynSD
zd1tR1EEl-oHTP6%mruaXn(ue4m7ipOKF0g=u<7b|yv1K67o5}6$SsR|pW%IF?sc)*
z0^b^PyARFb(U|KHxRJ?w$H%=J_|^#RHEPQJTw*dy_r@AVhPFnP6WiW5-6;!+J@B(&
z@xfrLIYkSqOKc85-s84KG~nLV!w1<)9XHNkwV1Hbb%sP`f$U7?)>nL~ZHn90c}#IC
zDG-U_yn1x+2Z5#CmpgPe&kdf(e5-vsSA*D*^6p!15-}HB{w_!<36$zTa!f;f5oaZ%
zu<41G$mCCqfoWz2zp8J(VB#%gUv=PHq}i(ecc++Uh(C?UkY-Ci$8oRURsM=tOnLIt
zdBv+7*Ksn~p5geN=fj>;ad}o<d<4Vfo~Ip>j124lt?8S#sJ+7D$2*4YJVu2f?n`eu
zyx4xkmm`V$kHJiKzxjT8S1)Yt-B?hkKJU)GgsVH9WP{l|w3|-r9@R@;bSGak=vJa|
zvq@Fx8$Z^bDY5GkN*?fh*Z%6LoWLU=f1reW{=<}OH?E)cnP6c4BG-5}PsHvICvVyv
zH_MAZb6TwNe(L+s%>S!i$ljT>?%_$B_=Q}lee8mq9*2^i1-LCbxFmr21+(J57YX50
z_yu@mqW2wtn0Geyu2@vd_R?zBokl{}e}1~=!79$GcVpge2Zue$I&RyUS;bxlZn56_
z^`WovVY^M5ua3RtuJ*XS-+Fe;DX;h2eS-IJhMait{?)WO`<^d2y;4@pbN7esyDwL{
zMa(Mi7d-G}?K)%Q0~S0|Wph@g9(j_R{8c<H_aVm-SIG-HMc2#T`<$0ZZ0$<<`b6|t
ze*6xBt|iZUE-q{rxpTqAZmrx*(^HKm^6Z-(Z&hkEv)C&HOPIgev{|k0oGVN0;&lfa
zqWgm<8YkR~$lNTUbZE)-3xY~U9zl-|eR-oUtYmY>Y}ul-t3K}FZ)6s$JbYICsneN4
zd7G{C9=vl2jQeqPJI~a_uZ#Nia*n>5p)ue2$6*6a4#RoRg{$S)7e_|C`Y94|X2$z8
zCxwE5tq&4S<aim3$_!4YbZnV_cwz1j4&NBR`4u99kMCSKQ`~jleCtx79XTwsU(N|g
zN<Y4CTJ=L|!zdRfwd_SR9S*5CaPOSDi2dNsb|b6S<9{}BEq{=sEurJDz_>(&!*xFI
zhNF*fJ(-vpm@ig2BWsVZz>?cHUoxh?IXCI>?5xxZR;y!0j*YE}@8XzB@&lUQgw75K
zTzxJ?m7Pt<b{~h;?p76#hs=i+6mFZFpKy7}w`87bo#njz^?B!l3{*~AO;?ov{E=}Y
zZ>4^kx`pWZ57&eOd5#9RKH!&-uQO{731Yr=%E#^{ht1;0yB;{aak6{kelqv4*}5|?
z8Jrt60$IXJ)jg*e-E{fx%^rB>_Egab>(b?xoGUg~waC|d?OdR&$Z@AK>GLeLtjilb
z-u66`lq`8yy+GjTk*1vT7Pe+J^(+~ane2NLyjwQ4@y^RLI1<d*vh{`XOO1)j+pP=N
zGiEH^;PI`z=iiqo!G~UM#mz>XB4_wxtsaE@xp8uXqxj2{)$9j6`Zt(VM7X{E;?$sf
zBFcG3a9UhL%OYits)Guas%77Ix^TPOaC{V*Z5U-?CH5+DW*5T@=Es_an-uk%rzNtV
z<S3{rJACO#LCq@$#Rt__3afoLb~4^jVw-gPuttI0vZ!D8%m0|J6~6DFH`nsOp48dL
z<E}fLdMNgK+0G4X9~*2u87g1ww4~WOTGjSW<KnIN*xItf{HL~yCU)N4rXk~VVnr~Y
zms`i4%Up?4KXe>A-Yd3FNDyjqJ)zmdUd6Q7N-@M<a;ad$aoNs?#&bi%W7rRVEU>xw
z)VY8)tEF|`9P#PVb~|J?^L?4_8xm5*Ft_E_DdpGM6%K)aRIlv~mOXd)>M}#ERcCBc
zycRY1c1Bu==H@vpteMgz*$_KpM^~{bm*5=3ma~6-3ZEQI&C?3hQSe?PDEp*v$svEY
zOrdKEdV9EpdyHA~KQ#QFA-VA_LrR55+b83|1r<%sVQM1HZ=H%{A2L>bcG?l}wktZe
zX|I~1$PfSROl}<=Mwjn!$h>r}E}ZW4@9yLN4QH&F6!y&X)>OM!6c@37a_xmsk>w)(
zITHMLdMD(3inQq$OS#kWFsbeKTi)--8On^ZyuKaSolyAEXx*(kl^F-ur?Dk4$n7!x
zSDx%-vg@JZC$T*W4(i{%TW?e`R-REiueN`|tcr<#(Tn7pa&)FAnttDPiod_NnaSR+
z@wIhuTl`Al{|`(Rt_wYq{PBd1<GS#7!8Ix}e6ET1f*NUkPdu+Vxo+^;5zSI>wB?$F
z>FI!346jzbc6n8s7}H}fH(haUhiq$+^pnc!D>EB5dL7s#^#1!-<tcO6)>R}qpIqCX
z!*KZ`yOV%t`~zDy5&iQ&E~-!ZR$#7lZbo2}|2)+mMIRUY5BUOIJ#U`}W-?FgITb4*
z==f>5sGhIQ+TSgQV|w;q7i%n8)p}_Kk6qrb7p_q(|3lXLF}F7!3r<$w_+y^#&+g+5
zZ5tYxk}dwf<`7wSC`M&T<8qxP$D6PG^t|{<rEjIz?MGR%(nmbrn*4Hjp>X1p&$A8J
zd)n@6-dp}j_3#lM6Vd3VU-9OdUOjDMJoOwxTD>h>k~npw>%-Pb^Xgcd7#%y(al#<2
zL@;PU|FNs>Gpv&iwtsWpu~T^BL0;o<L+hWx-j7(ebjqZdInO-E_)>h)$NRBIPS2b1
zuvy)*Zr1iHKOdEeJiESLo?WHf%BSz`+%E9X{>_Ku*XLf$XggwaT}f?*uurnI?wVo`
z>+NE9B+j(Py-1Y4cv5U{Q|##?_tOE|X4`*XoWR8P$Hv}GCnLuD+4nuidUl<^)jzYv
z+VGpOa9dz^-mJ)<l@sSKRq8N^E-Mm~vkE%&_3}lTJ0gDX?F(mrC_K4%P0P1(k2G-~
zg}JK^`<mEsw!N<73XNsGDb5_TEBHKL!0*C%vAY5j5}pcq6fF8_<MMXnjejAgcRbaf
zJFQYVu}V4qziP*PF{Rjw_!kmtO$+&sJ<qXdJCi7&^iOuhk9FdK%4+Oo!Aj-vcTW6y
z;ilaw{%2dq-qh<8XK$$Xiu?9ZjpK=DjZ-<F&bb*omSjtw7jWS?RPW!NEn?60t@xeO
z)xi2BAxn{to&5pKbp=c3R=4cAYV)0Y8Rs@-9)`rpvK+^}zNlw+_)FQv$1+c!%p_Q!
zmKG{>r&s^LVG*S@u_@gFW!d64+B^>gw6uyU+*{pLYu^;G<#zwZFt+L=70Ys*D-6Re
z4J{QE9-ThJw@%SBS>Ld3@{YcJsXKT-Y>^h+STk3Q;a<{}zWM8;RFoO{D!BcW_}q2a
z(gegzwW7s7N~WC7kh~zuBsOhH<<2QR``8Z8`+MiT+4fk!4L_Y;2y6_!^176x@=u4`
zhj7LF_8fn&Yll<_tZeFBS<ondZTq_wN7lRiH*HAJmWa|(XL&l0ZAEcly3t>ec~7cm
ztlx7XclEQv)4UgsJzQKG@Wyh3=X&>LON39%f7iTQ{@Ioz3ldfquKoQp^i<*0?~DJv
zO?TW^v28c+1OHteKjuzlU^uy#-Nopk*`}YGKU^bbXP2wY-Mq)LTYc%%<tNYZ3N!f5
zDEFV=Y_?_Vry9p8GfWvYmb6<Gu3n-cTUXQYE%~4r8%N;Ns6AZwxUQ;MmrZ3{C}<P$
z{0>Xq5~B&rMErM4rX(G5+U%I9eEqrLNuitkn%o^<@0S>y70?PhcV4nk$i`?@-30rM
zPwi)hJLx8Q>ppYnvTki;*cc)kewF{G$cYKRInOtrjeGRrnW4f7{<sZi8;bSqGt%|8
zKK?u_ywdsaziJ+{>?6~*+O!><@aWh}6&t^k9h;@@o$ELC|9Hprf^m;P4bw{H>vDfK
zEZX^@<L&<alk8rE^-b6>A0t$q5q@M#f{cXa|Gf-+v#&&4z4PU!7H`A<qr2MsjSf6k
z{v-Y1{5j?6yr1+L7<kk(ot*<bot+f|g8bb)eOwtBD&|a`&K?nbQmo<mG!+$<Ee<S8
zTJE{L49&`5WLd&x<|O2yp{T+lXuWC0fvA9u39jL;IoXUU8x!@r6W;T4Yn<8@v0>pS
zHO>XA?^nOCe*XK}J^S=?b1a|V**V#oVa4%+Bad=Im>XmQ_o<kfOt4h6Z1t>S5pZA-
zS-|aM5wp@lK}}7_L2}ps;Cpu)t_7^Sb-MQNo&WK!laEg?VsuDSSpIRz9hnpCj0f)S
zNb<B}XpviCK6{#en@>w1Q!Xcuz^0vZjHkR&d~TxZu9fpBIdVcUW5B{9i%C2TSDq9a
za_^A5&>zK`!Y=5q(R7FB=5*}`9~A1_Y_!zX#W(FTO?kcL{hyRShtqFu@aI!co-uW&
zhKs6)hG=^6vj6j+{<uGBvaa;3!#rlI9E8NQ&qu~>Uvr1UGFWlG@KRS-=W9(hz9)ij
zoZjiJ!0j!u<^O@Hn-b14Ff&X!^_a!g!K@)i>r`Kt%$48ExSelINPK&-BGJw|&Qk2D
zXHhcy-^HypnFV(QidDLHJ~2z(w5FxOalSN5&9@2Xmnk{cOfvG@vesdS1ZyPkC)>=h
zm?I0$epX`C&N|_y)IIe$&wNX^OR9<+y`<0BxLZvTf2Jz1X)&9nQM`csGMR4QheaYA
zUpk$8;;rt&cl6|Ac2@1rhq`_)YB~60qgP>F!kYzu!wSpJEU>+H<~-lj*S?C|y|^E>
z9sBm|@X=*w#MvMHnNg9Nv!hH+-NsFlIgeY3k(-54ZNkB6Y(<W00^7csd^r$*YM#IW
zMxhCeKMvFd7@H`0UQ}+nZV|xpujvT;rH6%$U2%3A4ZrjcM?JkNw?w;&TSR%2gZL8}
z0me-a^JgYa$*9=o@_{9*#s2KM){SY0)!zU1JRTx{NwB%o@rXeZ$3?d5$7>7oZG__f
zCKeyEEjYh$$G)4wyExvqxVOA+;XCwp<)NgL{1c52T3ldE=8-(3_xAUj0`C`BMfs1^
zE~tMYo&Mn00lD%<W7Y?C1$p5s{@R7zEI4*Q%ko9{8wa<O$|wG%yp~rE*c@6c`%P*6
zmG4^vj!iX-(Y;o1zQ1zAzSZ*l_mu0l91%-m->~@x=Np@AJ*Et5WjxtO^b*~l*z7Tz
zr*ZArg`ACD6`K~wKlJ}x$o70nxbLfg=NI07sNBH+<TLl@r~2I5;Wn{KX9S)YGe5{X
zo4(2Q&c($u{P_9`#HYUhdur`dUgwR=m3z4FmsB2S{Gii(swdxy?@lvcIio}Uci;4H
zQhL9aGRR~;KBdCyC9Ajp#Q(r64`(l8?|E=9mDjk4Der*vkLCmczD15U9&Eu0b6E@K
zdL1w^5I)1abb*WxtM<XD4T3q0)(7%7$eT2z6)??V6nB)~!T0Xa)=Qjm4RIg%z9@4z
zIc0FjUUcpe<UYbz=wu@huOiXuk~A?zMf9YDO9#))&NJ?(CVHjlZjxN-TIbHEtUlqh
zhjo#3Wk-qf`icG;Z9###R|K}Y9A6-p!64OmZ1LoaJSP0QPTm)ezR23bcDMD~V)YB%
zFG{yC*EW7jWC%VgF@rO$zcN8bN6P!)B|}a#Y3;+SHb~|0Ut^j7C3Fq*>w|WWT;}u{
z7gq0xJN$uVPWSVNn?G{Z=<Z`)f876J_{a7N{eM#(lqCe-aKy2gB{C^C8}tTvtx(EQ
zmQkA1_TEqZq3lGD6oIuJ?>^l1R!b8~7tro_-C5mvyz}jm<45d-I+wU_QvNw%k*7?M
z-O4Fj+O~9;^pvz-@%*)5(W<01kCq56nzTqMNHc6@(9M-aAy2zb1y8Np#CEgkjccFU
z=M$GtdY_0s>HFmL6Tc#R*U3q)9~T}8Hq!j8v0AfQW4Y#cjr@?%P~TA7(BrGQLZ7c%
z7j}1fmEYHeuiRIOU7dDnk>8?IE4o+SUU_`w_Lcb=CbLw}&Y$LVdb#ehtC##P$z8Bp
zaq+6#>Wx<`vWl;)y87zsx)pmv)K^EZU|-=Ms2{XG$bLax<X-pYHHj+^UV4ym=GuyD
z7HbNF7yD0M_i}~g+L?<Z7w!!Hyyuy*?xbl;!#0TduXf4qzUI01cu7y5Tl&SFUw(dB
z`{nW%zuZOJrkX`t2k(xl-d?u4Y<=IfzHYxoehdA!o)etQ$98N($_B}sJ*qu_d!&2(
zm3iD*dLHygm?WL=&{n?py6?)@>rY=#d-?9_-gUn{`&WvunZ9(p&w9uA`|Q@`ueOh?
zpY@OJ@8w^%%3Iy5`k(aaNbjtFBFlW`z%_+y32g_(n!VX-+uYg~wj~`lZBys1?vZf&
z=Qh(#yQfd8RQhsnZEx!_?Vf%~`}9`^%rkDDxq2q}Oq?+DWCP)uC(|Z}O|A>pzRYqt
z>GIxX-_L5K$)&wa(>@b*rs{0(+2b?Mo2)n9pX|8p!v=wfHJdE5Ki<1^%IIWKV`}qL
zW9?Y&ZJ`rG=U%nVp0#y;>Ace3ucp$AdfR=covoT}7JcQ~)v$!<-0GWrTZ=af-&(sp
zTi#XTsil|6s<bP<)vIb7Y8z|k{yluL=i<c0-R|Z+b(Y6!d``_sO+MH6^VNe_6W(sP
z`f%OiX~{d1HzrShoUI<MzAkv{Wz+Oo#T}nzp1VB{*DKSjna-=XJ?7=Q$1&nNw(Y1Z
zEH0nC<89&VGV^l(()j0fmz{4bA6tCr@f`Jo#utkF(l<YM*Ee5Zv;J>fef90?^vd^N
z>ddZX>D0fuwz0p~r`E!5cCA&-%zZ2O&91Tj<MYR&&a|%o&)Gkf|8D=8&vt_)h3ySj
zh2#qX4Ne=8AM>6bopsbou7$OgZ7S<lwro~)mi4@K0!dPn1cSscNxzb^QvOmQp{S#j
z6M4gTo8s)#>r%hB?QTEMWiRU3+tD2%{ZZqRS&-086FymSyS$$}ssqIW7p-W^5IEZx
zCs%8KOiNAwS^vi)9}OR`K3;y*>e#s+TWR<H*;3n$-^JD>{F(G)H|I6ZO)b-0z8>-H
z+}inma?>QO$yRDTGYzs29ZmV0l30|?rKxI|8trxe$b(L8^)kPrxgl0BkEN_W**f{W
z|Hb)}Y@eP8`MPq>O7<Ymm7XiDgYAnKS(?t-XHs@+*OoZF?%iF#o1UMKKKT0C?+@BP
z{V&&E(!M-D^+$@%`4?h8I9IXQI!!BB`yn(f;@Pn?-y<(Z-rmf%Y4WD;&sIEH@=Qi7
zvvZTn@}k}%?&8}=%N_+4y3Y&0cWO_ZXG_X#k%@lO%${W*?tkNLlzmfwQ#<QE)~l@Y
z%T)cW16D+8Xzek%TlQ<wOaE@CuDaQc=O3KQn3-u*eD6&AW~Q{<dA(=+X5TeBp5u3N
z&bgfni_6a}K6853_w<d<g{dEJ|MdJk+l$Bh*)*Gfs$Zt>3fdav`SL95J#I6-BjQ}*
z+w_X!wxnIWZWMR3wW$5ms!!)d?{snRxB9p1*Y(DuFT10s@6f%uVakq>Enkiq?N7DV
zGS%+ZlGm5tzNe10*!5=DcJ1ZsY~r5=Z7mc{othJR{MCu8z1e53>%@MpT$}Lr>#GB=
z_P*x1rW^IW&Nlz=RNjBm-*QauMC|)`S?%Gn=-JNZ*RMxh3&~Z?)4XwMt7+luoA+(%
zR^MH=e{Id}>2DXkm9<VPpR=lT{o1>k8;U(;qtsWwzGs>le*4{myS>$KzUSOKeIt4w
z|C`BgtN&HMdT;*C{Y}2*gTjRKj@`|#`7ZKLy{i26^6P487GBxjlf7@-5A$y~sWg}U
zmHb8i_H5r-tNUNbL>5gc+VbSZfsKojYmXm2UhS7Mr{a#$)~D}Ia;mSND^<VhYsrz9
z#pzDx=GaIVe7j*ezxeF)z31fjJt$oGaMA6fzr5E?=llEe*Yky8+pqmP{cgJSZJXP(
zZXMqyzxPr3(<4_GtWICIaeeF_xzfFV-469eo2%b_v*YOQuW!n}y$vttdiUl2-0J><
z^>6ka`@4#rjX&#O>gV-)W%Fd+<=AZZe9?G!@$T_E$MxrV%&&Q;^KwyXsI6&T)bBGd
zBa6#D%5Q(}JZ~M}5&z};lz&TqUf(tSnm+%(xBtH{y~nwC?s}E=Y;oaxE>>^;yY}?$
zY5QjEV}D<C@9Td3-wVDc9(V8WPq$aCt-59Z*7)V;SNdl0v-Y{w>@}ZYY@qPW!=lTd
zk;lx$R<_~#ZN>$vhx6Ob7&L^ZrW>~^ocg?^=|Fwb)1SVIc?>z<*%cD?7$$y@PG!y!
zo3QeO-hY<<@9Zl!e=d*~$=tPh@+EQ6c@Z4)Y;5-=m>ja09(1Q9EU5o`;K1~?1~+aq
z$ea(J$N5yrzBx}ydHJ?;(;vA1G|#wKsMWfELLCDG|Iy5lh>{3jAFJg2T)o7U{G?R9
zirfMQ5U{bYC`e4sPAySLN=?tqvsHS(d%u!GW{Ry+xT&v!Z-H}aMy5wqQEG6NUr2IQ
zcCuxPlD!?5O@&oOZb5EpNuokUZcbjYRfVk**j%f;Vk?lazLEl1NlCV?QiN}Sf^&XR
zs)C80iJpP3Yei<6k&+#kf=y9MnpKdC8`OxRlr&qVjFOT9D}DX)@^Za$W4-*MbbUih
zOG|wNBYh(y-J+B<-Qvo;lEez#ykcdL5fC$6Qj3#|G7CyF^YauyW+o=(mzLNnDRC(%
zC_oL*EGS8Kttf$80OEs<FD^;Z_bez$HmS%haIGlOH!?7=&^0j9HL}!4)?blZ;Oh(5
zomUJE#pL{4{ffi_eM3D1{oGs>9mOTCWeEGQ>L?DWEJ)Q4N-fSWElN&xElbTSQAW13
zAg8n#+0N49RFDwZ-8m^~`W3klo00Xnd-?{z^?-sgJu|letOKMPS!GHxTwOtFQ4Uy5
zO0s@xPHJvyUP-aOp`Ia%hd|2);5tzJ1Cjv;0kZKGxdqr&!@>)!7#yxvF8Rr&AWJ=6
zY?VOnwMxlP&P=faGm{LA&5aX{Omq#*%nfx-EX~bylZ;HvbS;d{jf@hFlT3}0(vXbu
z%quQQ%u7xM8C8*6pqH7MVr7tQYMf$XXr^moU}&Oil4_EwYhh?%uA5|(WRhrZ0K&#d
zM)((Hrf23Q<{-NYWK>FKidC9rvSp%)X_9WDNus%~Nt%I)u0@K0xvsHknx&ahvWZc$
zfdSYkP&`;U26);k8R;R31mq-^q~#ao+A8@bCM);{Bh-du=B5UhB!WWC(A3z%!r0W*
z)WXQbz`)oPp(rf1s5mn}4`isJfu0#yA|=_%Ex#x?vBXv>GdD3kRlguF9V`Kg7OQ~F
ziqxD4m(1MMJg80!3y6CV(ve8gh89o{ROA*|ITxiSmgE<O<mczuDuH~YV5Dbg0M2*{
zHlQrzT2W%<lb@Uj%6eeMsVR2gTnEnEPKoLIV6KfmKE)U&1!v@!=Ycgs@;4rH5K7^u
zI2M(Z=ND!B7x^b;r6!l)HwT+yxJf~&>6yhPMU|ceJcXvt29l^fGK)*{iz<=q4^AzF
z@F3pG$pi<Lf&#dxuu4n@g`%xeabj6&3OE84U{c8$iFxU%DYi<`3<(p@#FE&}O)Sk!
zO)QLb%}vwHbWPHXEp#nYObm3>lFSp$ElrY>EiKJaO)t(*D=AMbN_9+6%`350a?i{y
z0LQa}1|*GYqAD-XNChQ8149d4Ljzp{vk(JwD?>{wLknF4b1MS_B}kMj*yw|bVwjh0
zK!ugBuN88j*nkQ%D^F0G4K6Jx$jMACf&>pZ@dT$9LQJyJ$Ds~MKRE5>6V#5Z45HgP
zzo4=xGd-ikzdR4G^;i@_Oz_D}PR%REYd5kINU$K)Saw{{pbBzvv*WVS2UnY*DijhH
zpel@(7@7oWX@!Eqs3jzY?`Uw11{X;oK$7Co)HNDhB!vJ;ibqoy)q;x)(Kk=cOR-fd
zSF*R8dZk5!fq{W7$=lt9;Xep2*t>i(0|NtRfk$L91A~|<2s3&HseE8yU|=ut^mS!_
z#Kp!ZEcnw&+Kz!ifx*+oF{I+wo4e&Pp<m^Xf4nYR(0E$lPPB31LL-(dNA?PRKQp!3
zWQmTp@9C+`zn3bVE1I(G*}ZAXS47<V4)sQ9EmGq+)yf^SWBTvkkChfW<)zll>v!*W
zk6U-Y_}urJ_218apY#5?MvvQKNyQd{pVwYoTzoNWYt6r(&o4<Tg|Cj<swHifvmj`x
z*ComG6Qfp!c<FN}y4Z!S3{w5}?rwLDe8(L<U0vR-QM$=LE8AULofP*fv<UpP(b0?8
z!0^*fpi)t1Rfv}4>GLyP@)cW7JdcTu-C6LksiftEaAIJZsHzT^VoSw2E>W!(r-ceu
zR#8^Y9uXo!VyyyBHfw^GdKvB3P|@My>~`W%+!VGk$(6I)>CmDH8#G!3l(LOBBjldG
zdgWD7v7@AA!Um9_V#^6Dol|E(Dpf!>oCrGFCHnLC`~9BY-jmM>2#K|}xGJ^?eBZov
z>(YXUhhp|tg(6w}=GE2JU*6uH9$)`A)XMpZSNXd;3oAZ8Qc+j;PWOlqapL%>bYgF}
z()P(WLGIyDj8o8DsH~#X#lCVWcZ+~i%<i(@&(F^n|NN9%_xr7R*7bE~<?H`AZcIA*
z#LjKvVo4shHhy_OFE1{wsa)FH+DuGL0jsY@L`NIv-ZJ6rKkt!fHv8-)UMG%;^|H2A
zBJ%R`bLPx3kl^`#_kHMcKUqFG8;!XtMQf%^5iybS-DdCa<#p)TF*mDMk{(yYoH}aG
zoj%>YDfP6FxOjPdK!}EjNzRP`D~~4`*REYFd3VS1^!(CycOsh>f&G8}kL!XEqvB^i
zTcY09uD7<Z2v{Aa`%+=z<FD`U%d@k$w;y{yLD6|h@jT|8Ji09cObq7c=I7>EPX3r~
zZEfv0-_G~vr>9GcUHZ~Yq%;E-FjVced%Ts2l}kWaSoyoQU4Og=I6N3w*x8fQ)2Dw-
z-+WWYG4Kgj*4A0y-rimupurHZGUUsfo53*W1TS{$T@kq0?bv+n&{ZmPm5O@n{{BkL
z&*yLG?d4@=<C)+$5ft^0DrcK!haG#9baz+jQfHTS6Dtj;b$4^i$;n;Pge0B(d-v}B
z`1zCZ!Ry!BvPwm~Q@vcTuZv}rFv*xua-vwpb!V%<BgxkG_Rr7f*FT&2IP=;XN$avV
z53a3^UK6!d%YXjvs;{pCH>G$=9`V;OF`07W#0i=0@k_m?G99>i)6%`~?j5UpDW+Ql
zoNSc8YfUx0v+HO!tIX#!hDVn#d61Ahclr6-AEPgtWL<2U`r7=2LLHOZWX`QozN<f_
z7Twu5^K9BXiJ*dw7oY5?wf^QW;(IUWXjkd$qPv#UXSyo3RJes{i7wWfcJS~?Pb+2d
zn>qQ1FK#h;*|D4>#jCYRa{ctF=PH<HZ!wecH-37mc6v1^r*J5?fTPoiLoqRsL$O7`
z37kq5TR;h2z=@!=%RF`e0zsz^@e|A5PZufCW@)&Yav&#U;gyJgFHMineKf!G{>MY}
z6&GoAf1dGu!`8qAw^paM8}_WZl+d@oM{%h^nW*dz<8QV?>umqtv*4TG<7}vXsYgrX
zh=6giQCr4o;ncO(Jc|p8%Y({loWC5~?7x@y^}NX%7w=45A;_rWcKUKwG4pT#Wr?K;
zLgjo%pDsB6^ts5IM?SLzMY4Do9549DQmB1B;fj6Yge==BPIaamw(UOnH8DQ%RP*zI
zG_h8JN35H){!3{cj(4fw-Jz0fXtGq~&d)Nv@_##eOLubVKbKGZ9Qy9e4W6Z53opi~
zX88#Oc8dvY{b#W1x>nM)Mz7TGXJ%hK8`E%boye?7nRn*dXew(kEoqs@YQ1~8b=-CH
z!i*1649i|$_vqZ?<mJX8+Yy`D%Ckjgwvklufwd-m|744=<f+!%&un>eD8ozCaetxQ
zcK-Vgudh!GDo$T;(JnH2nR2_wLBXZ5_vY452Blv82``^TCOv1LT%_0&vRAODP(p8x
zdg0NMcXRe7R!%-7w>3L;%FO*swtSl=KUK3`>+|a_rHGxO&kj~B(Fx|<n!WRPyidM!
z<T4qqzotHH9=ofBlaGk@h@YEWm0=qwT{cN%>W2^0ox|SRx@q|CTge{Gw9IF-vDfue
z8p8R>H3_F&Uq(rmX14nrPs@MFoGW-GwQJe_uui4J_uZm5H7vFNe(w3P8LTrHT-P#+
z<iBsfb<B}1iZAu<Ue>UNMO);*EtR(~vN16`VDyZ8`tR?>i*(zkMn7$D>bbPl=+z;<
z{+!Dx225%`6CW<g;du6J+y1}+p7veq|2|KDaPa!si-}8)s6K!BS!4P_P^5XyvDclQ
zFxBBsPvi<IzNI1N{%6=k!_H-u*!LX|sb<kzZ#}uny?yFt<+MpVjI%8YSI*V^TBpnN
ze9^_apspE)%$@yfI!<5l-?;s9;v$2vTh@2x?Go@kuV^u?_}GjLhur0k`xQ@oYF)kU
z97h`KF<*t;ha5uLVJE-z#=Ly^ZCOpjuJ=iEyfZy+SoJ^olpnZamhsBzWkQyPoUBas
zOB%iguH)fn3i-|&{e1nh2QOAAAAQo(%k_of+gA;vb>`QWEYOSLc>I#r<$lH7MZT*R
z2Q{8#(GpdWpLJd~SbIxZLf$VsSDA0gb!+8rC1oY;`ysM=yVOFL*3;^f1K$1+{=LmN
zaH7kfw*P{gJ$*N_F5a+j<K&J)?-Yf7TwR-Xm=vg$oz%DZmhIViZmq|RjiL<8KXw;r
zrn|rTnd{>BQ?UKKc)$hId9&x|9eH_K=a}2)S4&*)A6r$RcxL+c!pDoWkIoiR{uyPo
zZT^jDJ9%{z5eaia_T?^T%8mxi6!J7aCS;r|{7wGO{j?|7SO5R3aV#or<=U>OIz_{K
zg-28iE;`R|Z1EP|{3k~x#%Fi^o#&25n>AVl)_*v?rC>kXjK2Iw^Id0q9+f(N&bA|d
z?|c)5uixMAyi?Alc_Zm_l;<0%ipAIFt5+SH?lS)$dn@k^9^L~RU-+)$mp=EkVd0V2
zT!Akhn;m&;(I;sj7k{poH@d_ABY&E(#_8=#zU*<G68Axt^^|DEi@K9Kx1~Q??#eHZ
zxHr3^^T+z<lbM#d{?Gln$@Eoc{NKNaH+$8(Jq~%JHt}#<aq*85o|l`Q%bKebm+$kr
zvc~exUu{Y4?b{;W$M+xPcy`3t<=gvnZ?)M>O#gaB{^0FY^N2BTTfyqxVg0i|b@sE2
zckg(P`*2-6%)DcBo}jsToI|7O%Jg%~-ye+4tN**VXLGpe9_eoftlmi!R@W=M>tftm
z&3&mpR59-lQxL<X_fhB6)r8E?IbYT8-dOl0Jji!JSp37P5B^75;*PA0bKS9FT68kA
zhtbWX-%*|`?zJ&J&FT1AU@>uX`Qj~`l|N2bz3}v*nXH#KqcGQ+!kX{FT`YPFGIYg*
z^xqzNV5@QZd_%0+{N6tTOF&JSBdddUh)Fqw-jRL2Y^tYS)s!8z?jE}*Gc8$gzf_{*
za0*k9;>#bBB6?djw_8Z8wx7OmcL+yrsQ8)beo=X=%+)6pJY%~t#j*IeURqGc{Vmdm
zkBO?p8ZQoLeEFzq-CpB{?B?CV#sAE_W}mtLu61+nd9y<8T^Yu=mdQE<a)>6py~#1v
zIa($ruju>1qugCe78Rn3Z$EW@k^XnNpV@tj=v)J76N{3pcB_9PyPS7>?wZEp5OpN-
z62s<$JlD)LHo7uutkKzQmpN1KyXJe}NxLHSPDz`uQ<fAxXS;jBimOboZ`gZWIjuA!
z{OHO3R$j|riAb7?{Z^6^5jnK9fBDn)%>BjBBXqL9t3_zm7U(R0{%+ZiHOG$~t@mJ_
z^es>2=?%3<<+6&u?(%p1+AXD;rgzAXHFM>Hf&zy|(`$KJ7PWIt4t((sGJE{?-g-|h
zpO4!;tiBnW@B203#QJTC)1Rg<u<0}1^fO1~-0$yaPNiljwp4`XtYsA5^?vz+-QS;v
z-sNA-)Hh8aW%9l(Ndx9*8$x5^=XoqV=Ue#ht7?{7f|to+jkjG~28=7i-#gi|zjIab
zmVT!5|7QREPb;5J_4*#?sib@Rn^4F7YQE)qb|SNbcpv}#yh1yC$&VwKB>MHwY+?Fx
z@Bht3ml~Y@emEq#l<U$WenI2p^k05qPLH3>`FD5j&+L;kGSBn&zw`g>ne^?>0++>M
zQ{Lpe{J$(AnY`ZX<~xfWcOHE^^mR$5tifAr=Y#v^Zm3+W`%2y~fknu?eL~o3W7p-p
z2Ws-THE*O@o4u1h%2hq*nD4=xPmQiKxdj$DPI&s9UA$;}Ge4`{%-*y1UYC^kh3l1i
zog%kI@&qKRbsj0TpHPr*IPH+#?DGxEnVl)WBTv6B3<TG^s)o|O3pV}#E`Klky1}AD
zPrE&DOe=J_S|G8seXCQU@zqcNQj900{50Cc^!gj~M)N-hdJo?9^k9kp&ZXjdk8$?(
z4XZOUL!-}&CLQ+JbKPx?r837X#l<;(Y0E?wKizBi@#c2%++#ax*)`R&bGl}B%rlzz
zBSSSSU3k~j_Wyr_HeQUlWfZUa(aWgo)7Qd+;HE_;S)vi4I(>FVlBHj_7k>Qi5V%Cn
z^;uolwR;Ji%xaJK*G|<`(0J3epk!^QWoyr4FYk>k9<8~8YxZa{PuNt#tf`iLW!uFg
zZ+B}pzZ2b9zgsiY{n~_r0{hP0+mbDIOuNml9FU<Ecwm3aQ{V9Gg5sG+R&5Zwd+Tk&
zzFB8)T$q@~61wn<tN*vJ1v=^Nif_VwT5UI8V<_`mP<s5BA?Mv!rFU*x^q;usIAc@x
zWyi*wGwh}b8P|IVZ~Dper}Ldx&bIlz?8md?7E77u-La6o=EU*w%f8vx(|29->biX+
z<u6;S?YG0n0yk6%PVsVo^D0!)@{75Trl7<c?S}zNh4^}JKX?(?U>$6KXx01O8QhkO
z3UxbIy5!1eiyd8+6FEt5*~3d~t*?o`P6+lnc(>J2<zJCVYqjv4+rN?@|5f8_{r7Nw
zW_97uHR@7pdClymdj2w&e16{k$i1Utt;Mc6r&nK;x10R$=xgVPjQNW$W^Gs(Hm&96
z!6l*93-)qeHc(}H_IgQZweZ=kymxH>h%aNBXJIkt$+z?M3=e+1EQ!&wS{Pup_-mPJ
zohye_yu8fo@<S_o`Lk3L9=lvxX1ii_xO4PpUM`dG2TLjh3(d|Jt_u=b>g;&t)g_A`
zVNR)UJ5K!(KX&Nejq4wdF3gh5(VBYfR9JH<EBm8sbC2J<FWvr2H}I})=f4n@8OjFL
z+YbIZmG(~V-)sBx(G^Yeq|dd!_!L$ydBk5qSGRY)n%|SZ*H5MVEcU&BQslE1li4?w
z)_-l9q8rwC7GHPE*b){IzPkJ8t$c|*DG$T&V;gH&F0ST_JoHc0xU4k$j=9;S`1g&$
zp6Ve>S1k&0nOyQp=au=a<oKn0eqXf~1Y}(Z`QlpnT>g#C-3@D>bvm!D5_xe!vi()x
zow%w;3(X$q+zMJ!ra8q++r)0F;ho$52P(eYI6QwP=VgO8wzHnTUL|%|^GUSa(yhu_
zyQ5X+b)6~ObGJm(=lS+~r%Ip4B$~~B+Qz5YQo(jF@2bSY(8mv-d@140=w6`pvgzlc
zpYGx>!bBWnFS1^4{rvrGk_Ojw=chc!xWnh2pMJPF$3JqJ%#j#H-s%hgqnEjdE;CYz
zJuV{qzhmx3_t2<r%|~Hkwc&hIW6$i*)#wVSc;^zdY`^oJqyOKXJ^TO8qZipzBexb#
z=6R%^U?O!fBD#28=HBN97v2QkQl0S4LGg`yA8)JesY_9pl=2!EajbH>7*RUU^Wma;
zA<H`6Q<qu|ZeLmOqr=LDSI+g-<eu&BHEi>~ZQl6unQq_yxYqe*IcG|r$GnRaaq0;7
zm2Z9Nbmj5h>oTo|-QQ;|UZ2oYSGW6%&28`Q9p9H$)!VuUANhIw&P)~`LC%|B?uvT*
zy0=f8!u6{4(u$@Pb6w(nCKpbXo*~cI)o*HchI7eXY2jxYKHIwkE-dEI3J#rPwcm5q
zMZVop`wD(4tn|0KC-+gZ;@*5y@UX&>(-R729?9N#<K`WOtgb7eXX+Okcdb}27L?Io
z|LyDSCdFwjw%(oh$|8Qb9a<#d#KAb@aHp)a!<sKH|3p7(Z$JJ|>+c7q6ImIcZWpQj
zubn+tK;BPJF`I2T^WY3MfB&<-K1KWHMa{aY^Z2jl@o2reU7J8%y9&3Zr>8A_et!Rg
z5RFec5^MoaHa_+?o-KKEQ!md0^(8?o1GJ`w@N-Tr{MGU}_sZ<t;-{A3GtQ>{yz0QA
z=(0^KWW|AZcXw;X@7q(-c4A{2pKRCe-PX?Sd{fU=oO?14G!pXPbM@6l*IzGHcIgun
z6?L7c<hn!|)a(F_r?j*{hB;JpxIVJv-QA^{7zhf9j&M;iVIiR<$}W9s#@*n4_nt!=
zG>p5I)Qu&0RMgdj7dSA!<N*h=iVj!d{uL`UxWx5zWFei@7FWeheBDKyQ6RH8IJ=!Z
zBBp@4M^1+p2?&XS$Av)SJP5lLA=YA7Dj+20swX5XtBX*S;yaI97u+8PS)~efqDO>?
zl6PTYVMKIva&ofrT)1Q2sdq=C`?yHU+S*!3Sa_*2EDT)4;zhhvrFz|%7EjnP!yhu>
zlk)V+%HWvYWxRXi`WG)&K6vn;h6tCCkdTCBkxE%v*(Fb>LrHcE3}&BAd&#EQBEVnk
zs3CIW)~!Wht3AEF52qL<nn)Q+^@4_tf|h!Ds&|Ix-Pn-0W7jUDoEru&d&FJzoVgWS
zPFz~x*zDoysc2^>w>QqesL04|@x)J0PkRT5Shk(f@dA%ab+NFpEV!8QuG)8P*yXj+
z<}YO?CZD(r@mGcGojZ3l;`jZjU4Qe&jSCx--NCYFAVWdxFC3Sz?+IId_Qz(grHjJW
zMqM&=Is}nox>xmD*Jyon&fQ(6T&+%4ZcjMWCUeg9b3fR`y3`mPW)-42xw$Xz?X}jL
z%Eivk4$|;aWMZ<5yZhreZ+e0>3&DA(BV0{EfuYsOv8c$%cb3USK0(X2n5H_*B^uiW
zoSqmN8XC%27PU-|RJ)#Gl9Ze4>)~-i+qP_~muoW{?}n{gr#336K>fL(_V>4oi{1HS
zVq@jCvOg>R{Pl}TKt|?|SO5QacXz9(sB9>E8>JP#PKQ}Z-pj+|!KJ0%BAP)hv(59p
zAgPS$K#rMl<)@UU=H{Q@Zs%v-+LF2NZTI{A|K;rM?bFW8a9kg^H>Kc-absg+&DX2p
zH*VcZGLYD?Y11N?PN5Z7vySyh9)57JdFf=3?_K^C{r;9)_4So&O^ppZpUeW6PNAQ>
z?0<cG>%BH=>x-+a#eaW$t88H*AuTN}sqwRA{(SkqzP=YFRt!9B%;x#`Tvmn%iHobt
zeqlPnB(B(U;-HhW^T!V#4xByPJHP&)<$d;})pzdPu_$=JP*_;_<JYgEkB?kmPN`WG
zx3@|~Rn_*(uC;4*-|zdqu4DqJs5l{iCJp55k~cRdE?A(T<};%qN>{wuX!G{%&wsz)
zpZ?%L<CV41)92Uy^0ZQw->`Xe^^3aD?Cfkdez}@>#>$U>PVBdF;+S~&$gyLWc9-WT
zB_-Y1UvIzEYw9GwpZv^hJUiZR`f_Hbar~=C@*h8K&Ai;UIsJT+(M*N_t*JGiPO4ur
z)bBW>!=>oLSM=jUqK%DB#ow>jZ|*E!&T{m4$mTR(Cuir*uC7IXbFKb-y&iu_vohlI
zv$Mu&f>XUz*G6w&R`Kx>$b>_GjJQiV6<bbRGrwQsoRFYU{OruZ`}gCmo@nYuY-l(+
zS=~7Qo{d%cJ000c?2<feGP1Iso}LG<U7L2eoj-W78}CaAP`NYF{O;cB>2|fh3LYQh
zou(V@Hq$6|N$|XjpyAl}_x67L{{8vv{Cx*+-HNhO?f>}g+p`-Rldr6co$cJt=h?o;
zp-3`{LosgQV)y<>4<9<N3|e|)YxZ=OqtA~VKHPlaLcqG%-N&T!_Z*z69lj*^(+soR
zTMPzycPt=9-b8a&R@R5_-=CkN8T{kd>-DMMURxFy6(t?*5-oXqOH^K7-YoBq#@tVH
zT3cHKmwJhca2@sjpa15@MusgpH$Sy*xxc$SUr0#kiHb6KxMXQ}clYA@|Nmn4)y$Ny
z|1+`Z=_$|UdlHWK$v%F&{r)oNc0QlkX1v<k+ApW1i^#~FY2g&+;Nf|)cKf|k!N;df
znQ~<L{JK;7YJW#;%@U1^i~IR2=Jc6G0#18e)cxihxO{o?p;m5TAt5ElkE$y}SBouN
zxG-i<g`mB?y^@mBlHhqkIXOCSZ-dj1x{8X*c~;jAB0u=}_|op|DBSb&+3cFnXU(5H
z-wCczKTbL`)3{wsH_D*yPsK9-`Fhjm#v7%lr-!YN^W9zc_QSVtX|J!Xy`<@%@&Di7
z9lLi2Z_9~va&lS`u+V9q-Mv0p>rMIh?XInfbY2~{wxh4_-1O(Mpz`ksx0t;A`2&s2
zoc#RhH#enL{^K??w<vnTQTyx5!szXJJ~NG6=k<N&P@CM@DXhLJ;ULqqb924B#dJSi
z{V3h*rsg;2M3<;`#I_vCUTO1l;m7rjyPY^bGPSn0K79GIbMIc;>Tho*E|}r`<JT`C
zDXCK{gO_JqSim@O;>1guH#>TImIN+#`}6H~eoI^1v)v0g7KE*qw6C+dU-w&gZ}s<Q
zpU>ML&oF@$!8!Z>ev6)EmV4^O#l@U_eAC=7JT+kD7E8Iit8`1w%}G&Pvp$~I-|w;9
zCgJn5vlW$<7ngVno8{m0*_d?nQ)#_P)|HNCcK)J2KMMc+`Lm<yYt}xCJn-QA`I&L+
z(?2{o*wWhi@o~TXqgSuA=1w{+Yf~|y?(eUQyUX*{{N{9cEj@MpZ=!+3j~|cwZ|*A9
z_MWD5@cQ*|Hcz#*%`+c7NLc1S|J<FO#h?F7Zc0fD<WQWq`0{f9$4{TOPMs?H``g>e
z3ubtO3I}F(zJf0=0;SCJ&YYcX{y8^(j(xqHyu5tgy*(!vI=8E6q&Y4O5D*YJ(8eqM
z<IU#t5;heEo}QlIHo%|lN4v!@uLx9j>ybEkXJ_%oty`!1opj|A*E@5pS32{?2E}>v
z=DoSM*E+s0(ca#kjaTZ(rqt5~MMaYuKg!$K*jQA2P_UZY7qvC3@Yk1@HkET$6@Ho{
zqu5fR*4f#~BV#dPYxea`iHF&wOtVzxPV&|i;d*s-b$fd|``=$*mG$)c*m$KD1ZaHv
zQuw;W>esior;qhYM{duPJv-Zc`oRw~Or&_<-r9Qj+_}E(_v@tD+1dB(-TV29;VDpN
zuxT!Se(vGhx4kD*9&Nv0x47u(sY{xb4u5}rwJdnR@caAw<(Ze4-Pu*DeO~VM-sh9E
zw?=(?f8T$qR%i-?O>5lVs*Tm(^Tc!_9G3gd-BJC09k?)lKTR*z>+i3xl4dy)pwt3t
z4ro?-TwLUOb5kmJ>8mRT4<Bx}t^W38<zwU7XF++mzP^6nuUFb>XJ@tc_V&*Cksh!y
z$@Sa2yWSg<juw=cKYwtrx$@DDCq|&zhlNX)sI>9Pp8E9kG}w=`Oeg!DTr9%H`t|ko
z<yBu_eR+FZd~aO;{Q2?g_Up)sin=cMo6Et?o&5RP*&jcCoH$t*chGC;qRP+Drs&7d
zv#b3zVa^<%y;Wa}zRs8ko{L}-*NZuD{5bpGxbuz7>?^|7&MNV;yRafq`P=*Z=ilAk
zoq2JQ>$<u(x7Wq)E+{WQe)w?nT<h|nT_u^82lX9Rhppw{<xPElZmvn@r50)PJkX>>
z$DX{SU7~wF9+T#gG-^58Ex!0_R{X7Plk?{!)FlXfZf=UGQ+V{y{K(wL#}_CxH8DjN
zKdIXG)<x$2xho9}*XHzAKTv0LI`;F^!&#pbb~&GT#mZ75b=>mrXZ@qqe@{d)F*nb!
zuebXwp098z;ilW2*1IgMtb!6-wEmh3o1c59RB$b<gnvt6_W$yleSe;=v(b&)V{v<1
z?%~Uqh57mU>wdr89%EV;931@f%VqzU*VoH;i|IZ(t-t@w_pd7P$7bBzS8H8YS65M0
zwJG<u8MuBC2TkVjNFJIVUw3kvZuE=$`~Q32XP+%Jk4GwEzVdzDlvlq`cbeS&|Ji=}
zm-JWV_X}@rG>onBPP^?l^OR`fdAlz*5=>V`zRER}bFv3mey%E1RTQ89{IVv`9Ge0*
z=73Avc}q>%XWCYqiAHT(IOSMT<++nr#aFj>7p}WLA!wFJUvA8)?eX=ePEF=qA(x?D
za<XPly#AL<=bOEjCcU|_5mXSbi`{Kh`6;F7@Mc}T_<f+*wW$4N!p<+Z#Ic#})4{q(
zNAHy>PfkqKiQhM;{C@3pe)~TYrc4n5`z}0YR|#i)e0)+;5(h8uQny~I^*Qk&QPFQi
zxy~GWnqx8LW#qAw;*mQ&_#3YM`**94Z7<(kzyI&KZwXCVz2Z>RTDE4ny$|DFg#@0h
zH?Y0GfyH;ak{<tY+qrym?TWoj6Ac*En636cY&3uU{&t>=hk-3KUs9!O^Zf(*AKVVU
zlAW2xF05C0C5vP3-YzDotgY{lpO`p5$?BuS?ZWiPvNddL?{@FoRD5wwq%oh2#er*U
zqcg9r5;Zh5v@GNK|K+m(j~_oS>?}^dwl3Ctp;POV=D%E5RtC3&%GRf+L_u@1N4v#A
zy$qMU?Nwj1UR_z~yua>m#I6!eK0dznW$ip}Hg>;fo&S^jBk!61o!!}bN#C=k9Cp7P
zxzJ_4VWObF$AyA>b9c<Bocudr@nw;7FLjI)x|q0_ma{#0_OyF_xbMDuWwT~%-Yt3k
z;#^1Xe>D}mm(5<oC3Eyj<>wcVU!ObDyL@M9jqzctsz;%hd;i7Ota$y}^+TL@1MmHB
zx{S%qGru_6-m#B3wKaQT&EJDTXEZe~@OQ!10yx$f<%*j7HtdylzPI|=LP>68z0&`C
zvt?P2ZBr=rF-^P3yS*>}c~`x8jOqHagAScSsz07ik6#hBHOsB=*&G!^!-<T{Y#)w_
z$8X5Itah*Zz3tC{{YmG}om&&Jv1woJ@1X5@v2{NlvV)U=^wK3uKHRH*Z&>ui!z}O4
zi3N_$MJMJhowPvHbG6)I4{Q6$x$5tkF0PSSyg1(4-^f1aP1Ok<yB`PVMh4qI*}QV~
zk6pjmcGYibtS&AIdsk~&Yt{4KKKmTM^8N?wlfGu$Tw_$YLjG3Pk@LkICRd)${eJ)9
z=JT&Mz3SHQSaoMY73-B1vwafo{W1vSt#5Xft~)MJJi%`<-^K5X%YS4X$>`bd_jq{+
z7tj0)JfTw!&-~pYef08r&Ffje`Ifv*y}6I$0pILncNdgjzAiU)2It>G+rQRVl+^F!
zE)QEh)1pwxIPJ^=NV7@TJ!YC+<tLT6xHuJcbzv!~uHCzL*K8=B(8$aV%IVAe=6b!q
zx7X9d1JofE*m?Bn>FFzjm-CtD-FfifU~@%9#gjF%J_bP>(zMw$LRi+`@prjVrnmR9
zbhtQc=S-%nFXu|H{IpSgnyQ7?o!NVOU%p6G;+ar%{fxV=Eu-kxT|CEb*K}_FuktwO
z{ghM-?%O}PMdt|pEPh{?6k}yrdoANsmTutL-EpBN4Y%aZTo>-xSHsre?ap8DX3?wz
z2VQQ`$@kCla;nOn*%Ws2mQQZ(>1h{lTS$M?Uii;;<;vP4If=hiUo5-I^=!rEPtRuO
zZ>s-acWrI7`)sq^j^5td)Za$c-*RS|WFG1eRQ~a9_xlaow@>FU<9qV#*|C3re{bBh
z>CoZB&BE$_;Iey8-s5AvKYsnXwAh`0ntpuW;>F2*w~wa;H6AMY6`vopy=PJA`d*%Q
zalAY;^>3`&sJ-mbp?TbQc4uE$nNrBG=F<N;NqfH6iGJf^y|!Z0o+sRDLc(XJXob&y
z_V3@1-{#9J7kjB3S#~G;#r^GCYdlS*0#-<0I(J|G|F^dvzvue}HuqiWS)00SVdDOY
zGlHLA+WOq$-mBNYPux5deSCZI(n9l?t(NQ4VnL1coyF>+S|OlTt+IRHo-1$f?<{Wr
z_V)JXijPT7PEIfG?cH6oqS(UP+B)t0ytC8e>pV|S)17HoTh$k1pqd!SG4cJ<Wy?N2
zY?n8x`B89fZS-;L_j@KAmxp9~KlxVtqS1)=*2DDyi(cBw=)1G7(4F4D^;(w0(d^J$
z34d%`_FsM<{3rW-)EcX;%jU41nSRN`Fe)kXQ}5?Dp{2_ML)mlIivAIm<!|Vnx>dF8
zRz_idNOx?qUCQ-V37@vvYq%s1&rDgWS`ih!Eju-L=H4RBAhzOD*ICy66lP{SwEbWI
zO!nl!CF$PHj_y<5-EYW@Uuii1XGTC+-E_8f{O22HK1vP@yRal)W_wK2yNh?{&WZYP
z^wg;?&ERDpzFv>Nye&66?aYjiLEEYy9AM14w#IX&Q7Q)~=R&7euJiB3r#uhbcCw`X
z@6Bn+RgtN)d1T^jyAS;QJX2oe$g-YVQ^9SyoQ{r;XQFB?YQOz{IxY9rm6gK3zr9Vq
zx+?U?@88dNSj_{q{(jy&(8z3A{ETPe!i5!;l}}I9njD>%d~e2o!J<5=`)lmD%tR}z
zE5!G`dDPsMVOC>2+xD4>Q`ou0vmaki$uVrIl#jT0_`R{%`S}N?tKUz#o&J^4blrQi
zq_Fi2&-D2o-Sy|&kny`<&G~m}FZJCGbY>r$Wq)o%>e)08>r<!X%O2dxV|ia0pIUL8
z?aG&|V|Ulhe;~QHUw8lV55M*8i=Vs^G|Ru)tXaceJ=u1~_dx5g{`alZXN%wU-(k3G
zx%AR3lOrYXw55KWQ+&2>#dDv}ad$KCtYvMLw|F*V^X{33>59uf$m&GwogZ<Rb)QXC
z+q68_;y*J!egB^R{@z}ls4X3u!OQk+c)MijQp37G6}w7aIt2s>1TXhHdh6D&oq3OU
zPRR|j4Bl3eW%hJNT-Z%D-mQP{t7M*wdG_x5qv;#hYh_(4e!SaNJoDN6ZnnePnzuu?
z<wTafz2&+!>uN!HIjGZhyeGZY(p;oN$f-kog<~^YkCbWGudlD4?@Tt9{PKeL!HX4V
zKb5Xs{B2|U%P&>3E4F5su3CO;o#T4;wPI&4%kx&v6<oVm?fRGhhg!Ko5yUN~Q}Hf0
zet+FuIoqlQAzG0;3KWlai`N&jPpX-hcrW&5vS0FvKk*V_>qCCM7XNu>xA5&VnI^Ko
zg^hRbNVU3e#iJH_Ou~aN-hHiJ^^F3P8Afd`S8n#~ZTV1o<=v*KrtM<Es(;IvoBi4y
zmmUpVeM*9xZ88@xTgk}>E6$pz@l=(}JQnA2;{ONvhe!4aHy1GGy`Aqnx%`=N=IO*+
zQ@5W<yU+DgJ~?mOlil<FzpF3$;k9S?@`4{O@<wijcjQYx%s%>LareqkOyQ5)TUrcK
zP6+(^^73#Cr*OdPtLsZ-z&+b1iCVqi7A~0a^Q-#KIjgT)*@Xq1$=f|EovVvsRg9=e
z{1V;hZ4X|)^n84*_vWV5)7`vXA7W0KNitsgcIn`1tGLqIm64BYzr}f1yiA?lbuH63
zy>-&Z(|ieD_wTRW%zyFw+28Vae`=jJGtIj*1JtH~_t#!&%jSb}f>TG#p#%eioErvh
zywb<~?f)K8pI_4y^wVhHsrh?zg1vqPH`~P*He62JEfM!xLB8*Pmv89)>CxLfWBc9-
z-<q^3jj=uX?2PsARc|a+-x?z)$`GG-bJ?L*?#*?7t8Q(}?cToKJp1~(so)XQjyI~R
zst;ekKK=Feb<2tm3j6l$(=TVAw=w7P))0rA0{vco3$*-R*Znn=wLQq&{MB3kcF4+`
z(G|1xH>Q0z?8~`#sG<Coebu$!qEm8%G(>7X9AsY;zki;;{og4U7rXDDv~l^g{`mq<
z91}mgy0{1k3x7T}Ph|4WoXd;)-7jWkyNb2%<8Ax!JIU4U=CXNt89T$bX)5i_nS8IP
zHk{edwX4}^Dw7mzU}T<Q@y%tMUamh{tbBWu@2;LrzTcbTuRN8O%)M_eo$>xy_4}&}
z-?!BLS@rJZxpKL^2~R_(<$n6|CFSHK)jj|J{q~t<@^Wc@e41FRfYN-AeIIu(WjXcd
zC(GS%t0$SMT9uz}e94jhuQ*#z{RC$vmzkN_`n1F67v=;@y1v?L!|70Q!aSqx+G2w}
zlQ#Lj%dKX*kWwPOY1d=p<x_L3QlH0xhM=D;UJ!VwT~D{$p!!=*&g;FdyWgCeW3$ai
z)&J7|09{qDf6jMU#nyHo-r5o7=b`s;&ndH;(~Fj0e_em!?_`H(>~#(We>BSYKB_$v
zothgoea+b;$F^qPUGT~I_?+3N&GvExS-EpeeE;<6B};x!{l=j5t94Z&rKdjW@=vq*
zX>R)B;>(jorKUD9i&%J$9Fr<2Dm?R}YoDA^=e6B?EX<as_UW$qYh!!<lI6pP54Ysq
zowZ@ZhC6~rn>BBzTzGA;VCVijXO%r}s#%o25@BZNdvaxEuzv0BqVtOcoSqnS-aIFG
zL|EvH>goJxN2fCD?0<1|m8$Z}M=KQ6a&(*3WR^Di8+G?Bxp;D`K%TFSh3R|mb=!Z=
ztJD?JRrNZPSz~l|VrT7X<LoPCS><a^%m{3csMMJ8>Xt~~pB?l5Ik!LQUNGbSN|UHf
zr^2MxJ{E2_TYK%}%x$YL+P(JK#<e#2pX^p}W?=X~&1R#BqFmeW9abg(R|vU_ZA*xL
zzdqyVWkt4d-@{86%R8z*GgvlLT~&W=f>qWP)mKI}PR4c756|B5-(8q}T%v4R?varH
zYF9jbAFk`{xw>R>w%C+&VoKr@!3*F%#H;!%@l3mP)ct%6>z0r&8;>pCVXS5HeX7mL
zm_<ukqK$*X>K%Ihf_Fr2R-V__RUbY%OXSjB$SS#s#w$03+sIs-lpmvVO3wS_<S*BC
zRQ(Tdu}A61om=>EE@za?k?rEQKL@V%a=7NdGl20`(pJu!n`Zp;%a*wGjMpr-ukz_?
z&!elAwLgNo6D=oH&lH;9NcLm?x_{!?CHoKUF<<^TVrT9&RsU0&A~D>*CvS__TUdQs
z@5~%)qsKG%&h(xqVLVlXlmD*Yj9at$#pfq@B};$%BX??3vgxO{S(pC!g*|$n2Z>AJ
zWjBgh{JGc?-kwO5Dbwwe?aa9=)wJuIgp+mo0o6Pe+dp>6KdisySbtZ$z3eKxk4w(&
z<IlKv-<Wf6+Mz6l)zfxu{54%!qVH~!@9Cc=>n{ofKL|JZu=mXO`}@whueE>N{V~*{
zV6W-hE$TmSEN2egTEuC!Un}Bdvm2M{w{17CbDzmm-RHRc@PdnvW^9y-ay9*b@J{ut
zVo=A;-D2Z=m#trp8b_`<vj2Nbx@0pr)>C|+2DS31uYG!$J#N0$*Zcp!OB*!XS4VpF
zf7$=NRknPRZvFPe*B=W-%+;Et{B-A^nZN2*PxL4D$439k%KPLqX9q5N5)u(zV#Pf7
zmh~c|7eCj|?2TzU_k3QBY8_X*xPQXat%oe)e>Mj5)lIn)wy=Kt^_vzCg(ExlL9;$b
zv^l5pPm%NNusIp{*vg52+riySWXsq6IoQ@c_2+{9hn?%c{}WP`l6h~s<l-ZZSnW*{
z-+KKyaEWJ5Wizk3`p2p>(joa5`WauVKYEwtT(tSrCZYeTkks}^{dd;fNI9EnC+=U{
zRPj`7-J@gT!By!$J4_Vk#eCEL`2X5b$6JAmlFn+xa6Rs2TFw+TVd@Y4si0BumI@YK
zQ|Z!$X^ZtPi9V9FIp*~TRAg=!vS+$2)H~z)>3Iu#j~niZ=khz8_9*U%P^{cmq57jo
z7V>0np5qz)E%8^Ot!*361n&#}p5T(OeP>~relRoJh5hvmjEu^WdkRfIr{}tUm)V}!
zWqW*217pdX8~vH5zuRBg=2<?G_p`b|!j)g{`PY{y=iJ_Y=2pqS$Zs9e>E(hyZXEw9
z|K8u?><Px=otcw1PMK_T)vKmdWV!wigW9hQQDr|{^4m-v-s|PtB=r9&cyQ>1p^r?Q
zNmHY7nPJw&GP{qu%)e&m{M#k_*l)l8`-bORXL_iay>L5P*<JbnlpNEtJ%@Q;Z$0xS
z$nvqS)^Q^dFT2AVz1RJ>l}_$RIXvlW&b+^}uL4Ww7q#=q+}_c&c6Qpvz^{_;?H~VM
z|9@YnEMJ_@q|7CAj5p0MQ{DGdX#f9x?Uu66Mz^x8vN+=_j%*AGd*mG<;?(g+?cJFy
zwsSX*PGH^r@@4dyPm8u$$R5jm^osqqAw$$AZT+M}2|)`FHt0#pY`tk;YyR#_T^IZN
zCFy&U0@jP&KQ5nsVAcJLr7LqHTaqp<HeOYz`A=f&2a}~Q48GW&+5i85f8?Kt_170~
zySV7*`3K7uhW)PF5xOxkw)}ANmn-#q?<=g1D_*cXwzJ+gMcGeYUv15N^^?C+t}K&2
z^CLsPYaVl|>OXH4&}hC=`;MXteGTbL7yceRoqAN#WE->J3l2`HbGwDbjq{7-vr1+i
zvYb0J`nb9An#JmIrgvZ5Kia-)UmVN(UX^DzqVC+eQ}91QbE8Ad4X-<Q?)+F>z}{w}
zGwtH8=Dkd-e}xvXBpteT>0+@>_H_1(xsUurROQmoPxfS;W3)G^$Li>nf6Ng(vLv0e
z!_+Iz$M@$vJy?C=-#x*e*|VkRF3(lec~ZY4^!<TllecfG-LJ|S9k$nOYUui%#vxf-
zX4Y{U6+AgU_08YU8tr~<#eZK_Zd)JGxye;GX45ZGC0mVqzgrJoG~9daQSDdBqn9Uu
zh7eB(KC+ar;#vCITIBSjqM6r@w!T)Jyh%@g(+172Tq|zgPtp0@WL1=TQJ^aC$J~RC
zb0fdr)T_$b^e6d!zzPFjwtDx&LNmLRW-CS7yyXer7T(@+ndgn=yg+vUM~_z=xc*AN
zDyME%^2N<%8S*i1?FKhiGd#Ry|6-d>&%s`;1(i)<aoL7Fq2>CQYrh)Sb($LFe)fu*
zy8Y+*{l%~29;QZqoyDH<LShf+>(JLfOOFN>tn;|c=ltS<)^|D3P<}_v$vz>|IQ86*
z^Y56=(EJl{V`q3=%E`Y^UGLfZv@B$0xc7C{BE~D<686n&U|ex4OJUi8x|Vw{yk^Q)
zXWV(#{X5E5=8xg31H2~Mxz2}XK91SMw_wxa)Z1CFm)lHvlKo~!$YlS!I>o(Hcj-Pj
zpXYx)WNY5y9|@twY1+#S9tk=M@=SZ~e|pvRJ<icx*ZwkR=-PhtItf~yc0w>@%Gx02
zOW%qfuChCQ|LxnZl{!ydFU4=1BF?zFI^XkTNn%)HFIRxbnuF$Rb0tl)wNC4WEe~9F
zKy=?jZL{g}=bwa$>eR<(@@7taVH754|J1moH_gNP*wYpD)0L%5HcHG}wzlPV%#IAD
z($gH*=J)=7|9x3}Q=abXxK{A+M`hs5Hrtm=0}PjC8;F<wXR~*{qIrz>ht%rp(dn^q
zE{nIZxMiO{s-&I$`%4Aqf``5L<|p3SHLYo_befH;YSM9)T{`;57FhLxX1fpD*PWfF
zop$C}=^Lvu*#_&4<_5EcjarYtxp8NE?Hld@=Y>x`|F1p0^9D=Oq55ZcHLEx8J8mEx
zT%&V}YnIRSJv=#oUr9^l+&}#3VZOn_<1tFUH~0U)c2d9_6t%x5xopn5rIj>)Dd%6;
zIv%d1e}8_S;pb8^FzApsZ!54g`+4%V(Ot!quS`r<y^rVI`?JE_K62%<nz%ks&~%fF
z-$KpfCl1fn`1w`6WX0<~|Hs!v9x91{{VMVOE61wcGiH>(e)vhrFmd+HPt{!NBKE~g
zqGZ~)y*Vf%cUnJv_QiEsj&D|*CM>u$Uu>(6ao$Nm`^&E$Pi5GWX(P<gdUf}oU+(6s
zUY@&9?!dh)JS1t~jJ8wDb=d#yaTQ;mllI!@V#}K7^Ya$3l-YX!#4)Gbip_zhY`MSM
zCg-+aetLbU#ueL0d8<$M&oz_Jy;G|==j8M7olZpTq`0|%epDaTaQhoHX>KrgQtVo>
zv}A#`_F5;;U%r3L>d(RNzXhz4Ep6@Y=}lh{Kk3N_XP+1=!O-e@tKGNkW0h}pZOM$9
zWWK9cIpJd2z4aLhuh+<Jk(_*FiEevl_&?5Dq0rIk^`Fkw@B8TKZR%;M)_OC??DPb2
zrzh#WQYHykRs>#IA8&vB&E%5{+}E2Ygk%>KUa|XQ%*;RKTxCeLT1fv+r7I~HCin18
zdu6SqCvShKG<n(Sy9`U3cCqY>Tc4VByX7E@)KsnHZ*J?&7hTP2s5mFbsr>I?SkS_h
zQ2kT8ZGPWrbzILZ?Ot^0(Dhbs@uFvEB*k<hKAg#IZEwH4t2F!2p+iN7I8;+&VwtYJ
z+W795Jzwq3Pln?9GcUaSlCfpxx!_N(ZI$JV&aDcl3Mp-P@|oxJJFibCL~E{J=Jlyc
zdhC-CD<glSjWae!CS$Hqn6-M#`o(3_*RHYLGDE9>Mxq~Id}sOhDPLHV|6e+OEa}u0
zHHnD(`}MB3ze||mzcBpT<K=#HlO7ysoMTn`YH5BnI1%wJxs*{@Qdp^>k@n|I;O~~V
zE9Yh0j_T#Q^!!CoiD1l*iizFd|CTp$#J*zAU2VkbdV2oW@2_m7gns-!)5?Bp$B9;M
z@lHQ8yXkhj^&2|GPstRwJZ&?~iTlB*x!S!z=hZ#SJtz0{$$`r2S(6PNOuDVA3^GnC
zUWiXg`+x9Ev{C=|JJtdM0teQ_)-$RJok<Ibu1hp{{kYR)_Qvhu#eYw5S(VA2e0A0M
zV^MMP&qD8)D#dR!mzBLawdd+;$=K7)jofvv|67iKxUizz`uLT;xij`wNiNQ=-0;Ms
zYNndzad1`U^k=(Q@2-HCGbhIsMty&;KX<14oV#1Z_TOvV6ZTbaugGT3Ti@98zkFuv
zGR}CG@@&hm7NMgt4Z8z==^p#G+WE*eL!Hk2W1al>9#yp({-_U}5NNFB@T&5SD3@RB
zpF`(12cD5Op1r@;J|Rq2R#(bD?d8GX3M;J_58gRV-sikh=ohm%+c(93bI<x@g>hAx
zCPd85^Qq6}K788duc4|-+onHW(SE#dom)Oz-g<9w_Vzy4ow`qFfG0j%1pE(Fv@SI+
z6aIJGe@)!pR^?|V=g!7V-nhagc9~uLj|`4mrEO1Mvs*m9*A(RNXKseU@(Das)<yA(
z=vk!A;tacC9r$LemCya1{leXBk7WLeYwX?VRqr48eObk9eZexrIkMtGA2p&6d}986
zJ>yC4(UmKHq+99y&SK`g`|5DpPvH#Hj|`cD7hk-enw<51-um~g(4PG9vey^FlJ3nv
zyI~ve$90!`UY=G@G|NqYyRYL*c}Awy4*7KE=fWZ~UgGuU4690I9RGg!FDPKqe_-Zr
z$)B;OduA<U$g?Y55xAJ`srQaM;%BZ%UUvw+p)H~jlCVCY;IrSMz2yN5dVc-86zH$0
zS7TfneQsgS_47AvHnKjm2=KI2;{<QC6EA+lzG}Ds!!E_&r4tOrzZsb-G?@0UYMe2*
zCT^0-_nn8kzB@_WIo&U$>J_}NB=gkuFX>_Fx~tvSKTrF$o2zR3^68cK+WEf&R(Ghr
zRdtz{vVNER{IA*W{|h(2n7QxJJsI({a>Y{o*BY*iRnP6pJbt5A=daz-8LQs}UOB!!
zsZ`2*Q`*O4_LsBEUs>$eo4L%?D=|4W(>3?=3-4q6)8e{pHV92!&A)!PM#nBGPCreT
ze?M1GNuC562vM@0wMbO*N1wXh|HAcQ2RE%>d~-9uZ6u4=T~WU{?;q!LBfKW<aTAQ3
zF-iN){O^+8m-IH2hf93_v{&<)&VKhL34XChHgLY~G-6y8wC87HF58}9`()P5|9(}T
zS=s&K#tzHg*S{Y<dv>htusg>r6WNBTo11TxDyk(%cQxHR6K^+-)p?eXMZFy(<Lnt5
z%Y;I=n3ep<QmsAYezEwEO<JGK!MFPL2@Shv>Smic-wF)-x1o8({~xR>&)(#vor&zw
ztYPO4d#`?trFGTTBPDfGrP(|w^A}1iJKR|F@265#N#XhLPiHJC=@U@8f4XyNaU5r}
z++AnYN}m}4sWp*OC00CAxyv>wRn@(E;eP8%Kvjr`^Yk^x4EKhdYin?G`Tn!@VBEAK
zWwpJ17L(b;k5_+GOrN4sIq_58%Q@<z=i4S84*d4Ntt|EGi{7kB%H<#S?>#!x=*aIt
zX5qQ#8jfB3xbTw=+g!fI#{}0td-&GYcct27$Ns>@uFMi&*bYocK4rM~)FsPJQc)a=
zpNua1rtR3CvT>EIb{XHpH{rQ6B_^G@A<F*ZQ<Td5t1lf_R^HB(+dkLwck;%)Tf1cY
z4&VG&>fLu{_8IZ+M-i17hF7<0%EZ|;PQA|cxaKLlpXHChpy<~AV|#lSH8jPC`z2qW
zwN>V*&wR<-FA_IT)_0l|=+@`{`=a`fr8&<Q$C`bq`OBmIB{otxKz*^w$tN{2lT_||
z>^OC4t^0=pYaiEB&VP8z)>(QyR!o+Ate;&_a%NXoav0;KNlLm)XT6KMUtegg%DN_g
zwdc>ANBv%l&;B^-vH0qN<gorTufglQ;+ARo_3vzsu+f?4<mz>y^ozOA9plMdPv;!4
z>wMH}zaV`o$18KIBbV+PYfApLy1lD6`RB!FU+NDti@kN_ossNkRPc<~Y|fk`uMP79
z_Bz_Lt(os($t%Msa&*?_4__ku^rf8UhqhOgKfK*jIM@BwmCP>%zR8)rts%3%a?47U
zDJR~_D{+`KMdhUZo4XM^>g=S=|D3vXv|8g`RR6d6^{$r3#YLQ#giHplI}qq!zoML5
zYL?Z;r%!(GS-X?*S@#<2BH8Oo`-SfB&|G!ensG<T7Y6g-t1N7O%Y7=VO+Nm&%Q`zn
zhCS20v&%I2;!=Zq8U<PlSb3J*F}=Jaur#Eh*LKEZhMpx%YKs{)8m?#5FWY;6N_4ZD
z(Z^;!i=>3_{jXmdrZm{h)1R;L;ujmUUA40nXd$Olhs^P1O11aao?0M!+CycY{{9z-
z!#_70if&l?e9d-GyHtmTc^=mF|9iUg0;lIsn3Q?3YgdZ$v#*=dGaeW6yr155dBe;d
z;(^Y;s;-D6m!*2^1a63Wem*jMlaz9Mo#$uO=Qrgp9V+6q+OcBJ{2GOt$J_6GdFg*x
z$Y0u;M<(vCt$Whh41<IYX5WWK71ur=JihhG<oD<2ndaQsFrUj={nFA^ljkgOpK$ep
z6URjU9l!ZiN?*rJ3;(n{`?}tq%7^J?p;<nqYnhfSr5Y};@qIAM-Y`L-=Jo#>l?Gz&
z+NYl;ZtZ3A37oU<!Qn4gnKN!a72YR#NoB9wJSlC3qZyp7Vb{KWKkKqYMK$Sy>xPsw
z{vzwIH}fv!HjtkF`Ff$I=cN1@$~Qk8m3fx)^RcAV#em(}?r+NV9$3$n{(JAe?^>ma
zU*6wna`f)AE6jeHwJUvpS3GzM?vF?8vFQ(|*37)QPD<m{#a+|&7vHL8O86X={H*p$
z-_^&}51&4B{Vtr>QxklY|8(89DT<NhB4?`V{@3y!w%)d=^7F4NCB=^qKXu<Jx~)}K
zJZPrymY&Je`!`pfmh*WYzVTayj%w|i8CREczF%w6-&-eln0flfs<&Y^yZs&<+cPz2
zlFIxJ%YVl9vwY{w{_K1#I8LtJak+E)iJ1S#_<l2a*?5Z`4Semi_FD7I8#mhSZ=dVT
z-oP1~^M<)zeXs6bzPa!2FL%$5@e0_psAu2tf8U*VIdCX`VrKuf?uS_DuOy-Uf}b5r
z-=7ZZj{oo3^7W5Q?XpXj%*XUn4n&-`Uf2GIe|f*S@U`=kQ)-Q}F7u|<Y~f!$k2$hv
zSMjm1EFHUp?ce`BD4n2uxF=())B?*`-l{KbFJ7%Ux*(s+tgk5Qla1}|=YNkrnm>Kh
zy3qJ(Z-37|xmJszAm?JiuX8tCZ*R0W+SzTL9@hJ8?Ml^~l{pOI^(U&H+A>`GzNmZR
z6c+CvAxGyVy;@MHG0R-J=v>UCd-EMHOi;_{R9AW7J?*|uw>Eo3-No)uZ?~YMf<aH8
zfZ~dy&_488QlVC;nEmxhrL98D?sYLI<JhhF=3WVMnR{gB{cQ_#E{aY6e@`iDt=8#p
zrcs<-{->^8oA&zp`r>D2Bv*y5o~Hfnri9?3eO_;0#NHA<+g|$O>(8vVtpdNcMZneq
zNVyg+xYlr9(!jVSV{NnQ1dSZYt$WP>r@!C#c-5-^7Jd8Qr75#%yt6G@($v);{C&Y4
zX7hJbXU%M#HfvVYUzyatD|f1&%Iy>?n7!=Ty)G`TO*`*={mpC=zO=QV@%-oca<S{a
z&--k7zxe&{y8X|q_wAJb@3(jVtjhNb9^Ck7`=-S1e5Yt~M9~(3$=9zX2A^k0`kj|j
zv$y8LMm6TuTc6gunq7?D+QKg$_TWON-VNgkT#Hv<tjqA^JD)q@W#9jEbFDXS-FkMy
z_i2fs#q<9i+`He`rQNA^E7OlY^2t}daQ)@k57%mYWL^qvWMZ9Rnl08VZT{@IeEk`b
z+ch~iEBfMYoL?2X`bPC939-jPX0nwNeubMy)m?7$Jt$r2qH?=zpYz^w+4-|8*H`2{
zvlU2pOq@RX&d%-&6-V3-`K?=CI4`0s#w1{4Lf@3<79tDFQhApv%N3<PXY;Cm^0S2Z
z71zJHhO)Ltd~2UszB4p5EO~jUHFR~@#)N}SJU7{1MuJWOIREDJ#Vx-TW_hU^g*-nu
zcV*aGsodM!Qs3X(duE0qv$(kU>pM4g80%PHoA&hb?WEP$7k`d!(zBZ&|Ll*`ZmqWk
zOk6*<Ets}Zce1+c2fuE)9gKetpQ%27NmL_#-xXs|x5j3tgLhY09AFHXytqMXlJ1SC
zVTFb7T^})@PU5$C{*@tb--dPFU#;r?Sj6rwTUz(`myB(dh_v)<o|}AN=S@uWF`OP@
zpRMCKf1UT-5`{}qA0HjHEO{Z&$|ZW}_;Juun5d^^aS`_N+Zo?hn)}OfiR<~yvss#X
zdA0sA=f^jy4K<&7@-2ON>3yu3zNP=+S*}6S7w@0n)8FS^^{+RFD|q>~X+Nd?mK{4O
zac;fuqP(9?BAR!39#6E5>Gj;X?c=F?rWsLYODE_(-5L=6oJ&`%yY>Iy_w}GPkn8vV
zn>EuoJuM=@XXPFNr<7u@)}{|13N&4$0xU%jn7-6Gt~Y(c%ySME6&75sx9-b%tiEbg
z`YPnr)z$6^2?~3wz9#+n@K9swwTtW}|1UU2`Q7R)Pq$ZEwf#UW>x@MfJO1l1C`Z@#
z9bd;8QWze*&{JV)-p!BiCg0XrdT!7Bi%I7%Utj(EWsFr8--aI^M$`NH5_g>z|MLIA
z$($<Lv&VIxyInV9>D$>B|Hv@OF5Y9qa&4_P!_VJ1tU~^`DAvl}C|V~Nt#<vvn`!SQ
z$_*vI_C!|H8BQp&T&vD^cYpo-dA8M)rc61qG5NSgzygP3^=?r*VmnVx5KxM~y`xas
z(a{k!Z7XM6mGU}j2E$Z4xxKgMP5Eh{mX!49;!W$1Z~n~q;ofg>)B5q3H_gi`l6tc8
za(_(W%U&h0ZC7koe}LJV#V3!<FieivS)^+J|Bvxht<b8~jV1;L0-JB1`SbJh#}6M)
zl&tm353<dTFaK6^__}%4r?=e3@2k(g{m3u8DkyJz@)gU(|4!`-HA~h0O`iGhTGsb>
zcfY*5JA1(b1-E{=UawSH`<*EgQ5+NBva+xUh>0EBQ~CMDrKQ}3g@rfQv3>8)Pf6JM
ziN$}GOXT@gKXk1;@2oXA#>18NY)W87*qdYPvI_-j`#N=_KK)&Fl%+cK$@0VB87nHX
zcy+aU6VG1C-14_bz02n2c3aT0>^5HMqF-M!L96bzONlUkdUEpcySuwrhOd{~w{Kt0
z?QOb~Lw&xU;h%VA=6Ch}q7UD>@9Zp`xAye3#W@eY_B#qXJpWP@&~jzwk1!b#UN&ZB
z_r4|h_xI^UZ|l+D|L0OxladE${>x_Slqp9RI5ywdU9RsvO=n@u&Z3*?&wVGJ$&Q^b
zDY`GIdj6t2-Q`c7{O0ZamneJpzNz8zG;Oi4zfHEza)FyCw_R4>RZyHNZ@1&Zti{g7
z7f<cBo!|D?PgI>XUHtp9!^dvj+?4)tdVJl&^7r??yuUyH!Gj0ejadGcJ2^YQys(h@
z>@3sdJ39(9w%$4ue&omPoVcBRkHrrkTxwA|+x+Numus<NpMHIP4O$OgUS2My6Vb4G
z_3Ep+i(^2Wu2<&d<jgQm=L5Mq^YXGDIonxzcXy@s&kets*x%&1wx+PW{D|#Ko8U8Q
zT=Dit_hsm&?)y7S^ZgX1^qEX8d3MKtMJ7vrt>wsiALVjq$Jx}k3SO2=_4rqut;;>0
zV`iLp$3jd$?o5}c_KL{OX>B>}>(}e6`_EI+(CE0eHG6ZxL#HlL?P)u^X7;AtTOYS~
zR^s6{K^d7d4-Pi3EwctqbRFTgDtps0eY*JCsI5vSCQ~vmFH6jy|MmBYdC?aSWw5v!
z&NBPo;N`C-XS3^cud#IIr9}1aZ@#I?>oR8PGf$1;ezy9^j02vmDVzTtss8@%;@W6)
z6Di*Pf4@n;d-qOqm#v<j-kRv`eKU>IAH9En9<rQ$itz0&6%~~a7v1FxKRj?Odw0k4
zXqV{Ps_Lh+KqqIk1}*g}{PH5O#7g$+>hS0N_Ww>is?KSy<zt-~xbpRLt3A$@>n`P9
zu*o|v-MM?`e8<Dv3qF4^3Q=7ZptHQGLwfr`OUccb<hl!WclX_8RCpC=A?7V(xNOaR
zzi)4Ex6hw1zcu@M+VgXBCFMlsU)`P`|Lxt~(^E8qIk>n^O}qXmYVrO0|FZG%@eF=*
zEE2yug11Qh;f;=tuKV>;-K|H$aj{!(#HJL_X?L%0j()cBhlf%3fAiMZ`aA44FE%{!
z?vk13c6gQOifcLCw*SRlI*ob_=Fac&|0(z8`txHY8Ag}?YaDF~eE&H&fcccEzJI@D
zdVoc&(vv5Db)xk*-#qi<<KveX7qfSZ>!*EraWSm;_N?@&y3x}fAMbzs;)Tc7tgAoX
zZomKJ$T^42X=fKjZcYOgL#^E6j~+ajpt&ttZ3l<qk=HFNH>ZQvP{zf@`OLL4J$x<0
zw_MHk<HxEiets|J*@^TAruXw*IR5(LQa^jsmybOUsg;^#UE6y)$;E}`?(XvAIcA%a
zj&c<}KQ~u%TlLjdp*#`>4GW#yjS3zdD4Aa~d9H2MmyWu>zXavv&UJ}u8{I8katV~s
z*Dv&$X(VG^#?v5WnpN=dQ0tqGkF{jKoo6{ZrTxwPfBWCf{5@qw$+sJ?Ln5s-+FDv3
zynfx?-_O6h{QWr-N5KWYv&}%ehUVA*n<=InB_JuOsa|Hq!OMH}`0@5fN4twZJUDp9
z{GkVEihqw?b#*l;)lX1#UJ<|FPCIPPgheUBV%@DLC#zrHS)2}vs$;#<>7m~bR>!t-
zi9R~e$Q-gNgi~H#{@dHz>g?Zj4QKlF$lK5Jn`>oQ{SCa--$crH*&T7oHZiA;`iEa$
zUfz;*l`B3ze&5e$vw32-hwh2xm$O;m-Y=)3rnV{RDA%WFXJ@N^%l`iEZpphlGY=ea
zaO;!ljM6pt@YDt!GZei&??s8#G@ZyvoZ(k1=Y7%PQarLhU|o!5pS-<aQj*fvtgDYc
zJv|*XMd!tXgUvsF{R#pd#Ps&|^wcJ$hW-2P)&1u^`SSAemDS<>v(58QG5Rw{Z_7D&
z{(S$jUg^!**Vp;Ti-0q4{euS&4t#og`sMZY^8fz*TNAn2t!t8NgpSz1@B9B>T<FZ6
zc7ER3Bb~y=Pd9wFFDx!jetm82mdwj-Ql?oO>i$+){`fyBZhiWT3kz%hd_10WV?*P4
zyWeND(og)CHsaB35y*V-<mBWnSy#JmZOsO4vnsKgE228N>dOnky;WZyeSCa;P0Y?o
ze|~;W@BV97_s1gl_O_*IXJ`4$v$6br=7&GCxL(Wx@9BC|bRsA1EPnpz&6_z(zxh{z
zmVWSttO#fnR`>hx<74uxD=QCQT^+tuW9Eg4%I+V&etmkk{QlGH@%3lT90ftk|L)iQ
z?p?BENluaSuPqxhE-JmczW#gzBXh*ws?urPT@0%(E_Uyq=B(Qy@PDq~^2NTh%_Mr=
zc9p(%dw8f-Gj!Dy5w*!P>}sv#YrhC?PCE-4+l@@)_~?7P;Pf=zjoY^`_nxk2VPyr1
zfJ~Y70>Z-1@9*set?G~7p0_mda9d<qe5ynghvL1a$H)6Y<I_?m83)4SYg@xsPhAwT
z@_4`e<=y4^N=izgJvpr0VpA3|i0j8K$-TWzL@T62(l~8`vOAyVDdXqm*Vo0ajNfk;
zyv*n1r*h{BpyqM~_dK4Lpa#dT($__gkMU}DO|5(R``53aEg6AMPEINsc8gs)g=Fk%
zW-zgGJ$U^1@IvSIO|`x{V%)*Yd={pjo>ues>-95pERCb@n0dE>4rI~YxN+mc+}qn!
zR8#~+L|iT|a@CC7v_#~X*PjoE`D=bWY_IwM_dAcIk&5@ZvVZ3l9hWW&T<j*I9oC~4
zyKBOfDJ}i|{@Y@-43w0b4jgd!_v0~t*}FTP+qai*yIBs-93=%$P6*!GlIgrT?W}}(
zUd*!ClZx6MeSOQ~_t)LoRoZ=Zb$IZOf<(<z*&O`*%bnZ#7|zT#*Z00-|Dp8ly}h&V
z?XAANx4QgTkL2XHbrB~Fwg@;aY1Rr|_2G8@{=&z{ctLTyHhTLK4M|Ya`|<17B=h{Z
zLx&EjXzXjWsr<Ad{k+_tzkfe|{hE4vTW)0Ce558r>@JhbdY?5wiw2B1#l^+V^6q%N
zzP5H{;9|Du?RirdEm>??{OrR)c6o!sM=tyR|Er!^@+IG?Iq2kyz{L@}%XFRF`BHy;
zc&MVTo_?}+Z{1(3udl8?esr{Zj!osHTU)c!Z$H*M3>so!<dd`UxV<em^X4Yi<YPS(
zODpd$x^W}o*Z245r)q~s?5#3A+9f)9(UQg$D>QbMzdyG%`+COBO{vpvZU6D3;@R2R
z?ga$~&FuW=z8#MOm1Zdu+WF-#tqfLQv}jSpmW;q@QRO$H=GfKF>XWqw)tOr|FQ=r=
z-JWo9k?S1GVzt=11WPNcplvxfxBrXtS~_W4?(LwBNv@U$gh2T!IZ{VV&3D$3D=UL%
znr4fAeSLlUq9qf5etvG8c1B{E|9rpS-`*}&n0e<wBeQ1YCKb?GTuDoeCBF+eb@co>
zx-P&V=?KT4KYudr@3Vb-YwP3B=k2H8-tmY1<HwJnEj#xAet<U2c}>w!bUn%BqZYg^
zCiJ+J$JJJWLtH<8{w#cSgtPAd-*PF_tRr`KmoL@$f9U4s^pBrEA3k`n@o+o8cfc8^
zGijS!z}t`~?%la_=f>LKWm0B29rNeQ?=F8oZPA`a*KV<kI|`GVnww`@m+NiYwk=46
zvs;N5oKPYJTb&M`n5g{m^JirPg9+#6S_{j_=)_N|S5{PPJlM?s`F8&PkFVG72kl(Z
z>}qkH*tX=A6Gvie*40%<JB8IjyUcuNn?3#a_jk~gaEtnXHl|ru9F_(Zet8kd*%fJ{
zb871zP#`+lD2K0$d2?%P_bk)wN3UO>{`dFyQjPNw`)Vvf^J6zQB<jTM5U{qk*7Oon
z6OJ+J;{omKHT}ORTuah8jmI?W%7WnKekJek%@tI3OL?t-7P`A<p>z9<-R1ea?v(qd
z*hscb0}XXQ%K!2Eck!Peg}1inpa1=S|M?k)$(x=QmG;ZqKU>^yx2WdlCmH)Xnd<85
zOc~f|cMNiJa&PYKoo$$W?85f^c%v*|*2yPbLPA8=ty}l<-#x9VU5(7_msW@CX9@_3
zxmryWjcE~Z+B5O@_xHj3YAR=zC||lddFz$iTQZf8dZY_oY!z_YBiQTKd@*B+Q%8xa
za13aSwB>}#m8`8Tt`py-NPtsmhmVLDFI%&S6Ks3;9-p>_t&pfxJQ6=CF=Jw2VqjPS
z_+U3B;Q$R0jj3KfYLi8rmt;pwnFo$?MI+9au_7;h)0{!8JUAwHW|&CLe4=z|>KDj%
z;EA0c=RrsPfpT!mkJ}qGY`c~A9$w@FHFWZX+s`|`s9lFCK4GxMBj1Pzyf@rMjl0{3
zM;E%USTW8M&2g|z=O4Ex2DUA9t!j6j$qUt}c=&?S-jg{Q;5ic`!IzODFMZn<f@;^6
z6Df&-Rbp^oYM(UNGGPO_fNA;R4Yn>#o~!$4(nKQ;#g-rH@pV5{t;^mxEWTLq@DOWE
zOw5Lj8y_BQX4i?|CnGN}uQ|!%vzfT0WT)5Cq??;kSy)&eJbTs_wDQU0e*0rLZ$@^h
zfIYr-Md)g=-{0TYKfT{4W9jtt)YPROO1>*Yw9ce`*8X4i^;IaBxSo!y7yG68+ARW)
zwz9CZKYsDz$EC;X)~yo~IU4=(<?{K9u4cto{kdCyf9e8=bGLr{{@s0TnEm$|UP~9n
z?k)?G`50F6>dMQd(uKEg-n_WnUtY5hJm4Jmy>(&8rLEcFCQ@fF9|!FQzp-(#i2EK<
zRqtsk<?rsCw@ovenQ?#LUJ-Xt;l+`CWpn!Z=T+jRR<fb1!=@hU*uE?p)b`o6{Qvv6
zZ+lm-UM=DVQ5dqTB=ghr^ZZMA9JlA*4(n37x9Y%w1IxFcDJ&@gb@_t8o0SyTME?Kx
zcjM;GhtHk!Yg=$MI(&)8#O3w@J#Na3jErzK2?+|EoSYIoY<u?ZU8?YCEjJezlY)W*
zLv(bsriWFqH)o5$AuClC6@&D1GOJ_4{8wL{WL=)O@9($h)obR0s<_>r6-7lxo72y`
zt-qeWv$ngllTpFFPX@FDHZt#j(UTL70RaMBt%tO>`{$j~Z4n5|`ugfB%eHLy<xW4>
z=+BsOR)cY?sNVi>?j<^$?{^fwI@FqTH4fC1X=dYn5x2x)+MWF$j=0$5<g76Z_$&T-
z{k<oLTDhla1cD3%wSW$JMe2yX5<78R-Id$R@ZN#W;}?BAjnYql3$m?`%haEFVc!dj
zH9Oz?tpklvDH3}EAg=QRBj>GopwuET^W*e={sKFrrm*f6yplCV%fS7<^1JK3N8Z^_
zU-o{fhG0YX^tKyTt-^K%GWolLwjN4X*jbxj{QdRUq6bD@yY1h}@9lSEo$+Nx?}95Y
zgZ`yeomiZIY?8#!Z{Y$Lw@LP`Ud}GISg_T~v}r|%hQY*=uWYlLR4?3WElz)6;q%|@
zhKX!b^;)KgFK!}M9r3^I7xCZooxJT&`r<P6=TkCo+>bxIcKx-~zvrbbn_C2wJ{z{5
zRZe;yoAc>ubN2p%{Bu#0G&-`spFeqbY1FKJ)1O{a?8#D}QkM14cTPc81Z(U~s~fkp
zpBzY?a;J5}$+Ze;Z)+Bvo4N7ibpF#{_uh&BsJ%nfZMt^tBZp6~<U9O}9H*b3`-8W5
zhkd@8VDYp`Ez-A+aH~wdbnYMb<7b*SKXe&q*ZeeS`SZm@W0`C8HM94Pv30By8Xi8=
z{W6h%ew+JpxwV^fR)FU!XHI$fq|&G1<-JNDap#U%yJUqI=*E0GudXQ*FkPed(!bXF
zoOll3n?d@uoEi*wxc>gE`R%kKR#al~<azarq7|>Ga@qdRe=F<ouup6M<N5r%wjbHK
zK#Fyp#r98if9F<TEz`bpaf)uM%enfGsq=%K|2&&{M9!x3pQ8*z+wJ|yHrv{M&(>R{
z7yD^_|LW*dA48^Ue2IB0%IdX9bKdK-x2DF*sO=RrNZeJ+c=_mO34J-Ws?Tgmb^m*x
zh~J)hh_Q6%_ki7&`)>BHe4(PFGD)K9-Ok+VyW4v9I=`|LbDDaKO*GE2q9#CgYp-&|
zqmIA#3zVL{V?VIY-@x*(O<?N9Z_41-P-bQT!?YhQQ}jP)Ee&8i_T$Ht|Ld=^_ivlJ
z;YDah%#|ZLl7|<^hHrdnH7&;U)5IEc&e=73N8Zoy+p@@P=d0<5WqvaB&eoom9nJB7
zF^8P*r-l3Hwd5*R>uBaTAD(b>hsegGi;7)ycOBZhGrjQTEe2`z*sOeC9l^g<lLhh*
zuFJOVVGkD!U&HHXI%`2(g+b~I|3JCxnuiNZ0w<Ofu(t*A7Hrskw)tiJq!+gyK7G3{
zes&G>o(uE4kNkUFV<U63`9^eFCewQ9dEd@Hdbs%@Z@>g2p4=GwM)vx`nBs34P3Qbt
z4G*^5@@rLnC9<8(-{t!E=sWj()gDf?F57kZj)dX!t+%f>KY700`EH4b+`a2NOh3u-
z@y_dvetK(izK5_*`TIqurt{~?KROqF{G{z-dnvikA9knSVEcOb9^WIgde0{lzn`!-
zV|d&c>2i-bps|-}!-Cw;KeM+!-F?_C$@mL<ky!CT0nz4TM>)AWU3~uwh3>wk5jgSl
z<Bu8sZON}g<&K`!y>shj!|LtUTn8S{*7q<FIcC_;cX+Ap9l7l+$IqvI_Unvz_f5Xd
z&TjI*8xw*(I24Waxf8@cSXS<jc4$1%C7~z(=pO%T$3TJYd8UT4|7TWzHcN8o4qnZt
zahm;M;BJe+_^%K03hWPkZ%e+vz<$^L@??X?sM@*K_t!pnnCbq>MDy(9!rC3*k1EK;
zcrxh*?Bic2S|qsl0lU(+xz35<Q|5>s_N-tLORD^Fm*3{j#;HpxxHVQk2s(9WapddD
zEuYIpUM+f(5u{Rjxc>RU3ol>zF3^42+55`s;a)qncWX{-Z@;}SE%nOB=bdvtSx!uU
zw{XM6<oU}^eXl!OxcA^<;qMy-ysxyJyAx9@)#@GjhD+qoNo)RN#<M$QY-g%$wSILy
zH+<o7Mq#d;LP^u<e~iT^?Eku3W74Gg_vKtYi+DK?6#a33$5Mah?D^{scKJo+TOT`m
zSR}aAzhMyk?93?qYx47l*K~b2uUAgEb^CkB!EfvDPP?D^-ucTa<{6BKC$h%8{v~?y
zMpE#RX~(Vg6BK8jP;9O+D3;q;-m77dz&ZQq;msc-dc=3w=J4>VtvSFEA?I#m>n^<g
z%$Y5a5v3GH&*M%8D@r3eo_<{OQsNP#xqeB*sl`lsrQQ=uzGz(h&A(VDih=uNM3DQV
zFVTk$79ZNSLd<x=j(p~m$LrKC&f(WF@7O=(QgyoLT+RiW;Svn9r5G(ljRmVeX>3f-
zPr4NCAn$i*Bl8ix+YOQd`~TZTh4K~E{9@gi_00d}$(48CoL}hkS@4~nRnw)TqNiv2
zs(ikFdv3uWovu0eL}vY$oi(|;bEnGQu1wG1btkm@UdQs3KRzJg_(<1ynX<S+GSg|_
zNBTz=e3kTQ>|9%ElWDUjZFkq*Qoevh#fk0*9q+ulJ@vJ{?7=UOa;7n*ZVYt%^!2vp
z|N74io>c+QPc;5}{&z;9YWnL%D~fh9YRS9~cfY$z@RWA7;hqmcue?(4FVMZ;x*|nk
z=8xF+h8I(EjT?ey&q^-VIv~^-`2YDAiPosE=_|VK`dztMVKP;EVd$Rr<b^4V!sAa~
zdFu@704q)2Y;(i=h>J1nF}4R*AGIenyG_Vh&FXVR=SSBJ^}}1Au1bzJQ(YA~X<p2|
z{EDiD`<1qet2|9q)+k!EF+8DXyThwHAxqtOr$yV$FzQq<^o?F$b*kanX7L=&JB2If
zPv0Ty-LX5F%~_`Ef8P#~)jh?LqW$NW?3t&wKKoRC?XBe9PZT=qH{P9feD#jaCXCm^
zN(*29u*_<z-0*sp%%3f<EPr)BURCvR$Fi=u-$bYJ-7Wm8-F)2qa_1eHuT^hf-Bg=o
z<&tyLyw&=5+>hj>Q~LvV?_v&bVCms(h~gGp-Ldbk)O4xZ6Ro@-#g#UG=26*x(dEnZ
z{d@kn<=>b#^?dIA`#JwtpF4Wx|FN}q&RQok7yREPf9H;GP+GoxQ5T1zi(2aHzn`7|
z94mU4XVLByE6|^B=TiAv%q`@JjH%phroI(fyej_>9Gh|O9>W%kee<nuE~yZlq9RoP
zNF+^pMNF`o&Wy>E112yp4>Jv_5_?nHz&q_kyM~DC$<LYYQ@F}y+We01UB#ZZ_~lN0
z?^b{DH&dopZph*8{JnjmL|Mb}&E-i>t--6$Y5Hs3m>Kgg>X3bW3uE{l`?s9+8}Htd
za<q6ME_83TZ6Rku^ut)ofBz?GS}t`zlT`61eI8f+hgXlY(wWtgPZfq{1Ts8Z@#x8{
z8?%?l+du!+QmawmD*3VME!V#{H;wJ9ESn<dZCQMRSw*$s+-3Exz2XOtv3?Q%&!CWY
zom0f`=LI_+#iU1JK3eWqE~d|mJD%mm#u^oJuFE|CbZ1+7Q@)az!$QX5O?FFjZ|_?C
zQK$7L2Uqp;DaV6$-<y)BdQq&rN&0kUi6vw2Y~hc$%Tj)sEtaya`jTN;+9Gi1*BsmG
zX_NQAS@h7_GVWh;Cqq&F9hqLY&Rr3qQ=VlX$&YE?*kaGu+Vwc(^2`<S?~}qesq~f3
zdKhBJ9w^*Csp1_==PjchS8}^*Q}c_iX!zWjm2YreqNO-4J>=&-*OeVAiLcXimS#T;
zxj9{iaqWh)3t5`Ej_$2hS;3tgzedK=E~91cJDG>CCl%&@(ev6WY}UH<i2GFa`wyZt
zu11~N?sVb))#gKt$L}@XcwXlAGDzh1pFE50tGl)S2=HHD@AmpWm-P2T7E`Kxo;+lc
z@wVT8*Hdq6LH2#Q+>co|o`ij<cxN(6e($em>gAKa$QVVvxVb%kTb6$Kx|oLtmi4mi
z*!AaWu}Iw9lb07tY4r+~Ci-7F*Y9alD}DNEw1!ql<-Z+z{p#VrwN`BLTQTSDM(z*p
z&l8%rt&p1RT^W-V?hw98LhE#;+}VG9E!Q)`)6SpU5qh#cXr*KLQjuR5;tiMfNoUXI
zKe#n3aY~f>m2KgjG3BDat#}JozU{4@yh~&K+XwH?TzOEc%ym2MYR7EziA#P@5BO<%
zcuT$==k!Gvwn_H6FE(0X;+3<f#z1z4dr;r0^;f68uQ3xp-v2B4{i~LBE^$7UdkVGY
zar+)@Qqkc)x~(o%Cfmeiwy}K~>)k+0afih_z3(`e&+(pmU}Ih6m;HGvKPR?6InG+J
zE=i&2+P3;VzUMFPp2)(+`ta4Otf{%)i5nwyK26kV5xACqbycX&k0;5?#Pol>k3Vux
zHuT<{2)%jFcdq^;@@_@<^CN#AJ=U-D+#%|8@`n4;7T8(<XCHo@gwE=Bg;TjD^yBKS
zt^coj>vuQp^3l)rzn@k=Dl`e6bVlai-;!-v`qpJ{5^ByXJe#KV>AmwoX_@QccHd2J
zU!5H=O`}!m71J{DJgIdB#YcoBXWxpGQLuX_zx})Y_SD}_E8fT@RejobJ}<q|so?Xk
z%YU~?TJ+9&R@!p@#{Yi(+b@npZfk1`+?3+EIw!j5>>>fDl+=%cGxXf{{Mn((^mw&g
z{m0CS*W-6iYE%7Wb?ijRy?58|q<wt9_he!1)8hMw42|M#{)@l=@00MmeZh>hT+4q}
z%l6N>nzL`FkXdoh{Pto$QLfz)?@DjZSH0B=o+NYPnCLc1@9u$=o~IxGaL%ktIxP&I
zQ*h##7?xOgFd|R?MAUB;ThOY;76I^rBo0OJk#?X{(ZS;t0#2X<<-muggS(puX(P#<
zpc%x7jU7i^@>JEl6z?5adqjoj*qfW;pB!#^FB6%XaQC<mNK?ehpru}griz{KrsqZ7
z-6q-N_Jxg^Z`-!8?YSE{IZIw$P`tf$`XbkEp-hwL)asw>ebj_Gt8Dwq$|67AxM^+u
z_}#m_t+$tNh|m%H=^^IC@lkDC{{3@TRtDePSG!xpc~7aj-<$>M=jX}T*Tr;AWPAde
z>=JN_Fx`B!$7$gN$SyX;mWtAd5G~bB;FTv}Pk;vnK%NHo&q3Z4f*ckP8|(lD1Y8k#
zZ9hm6G?ZYsE`St40}6Wa0{E19khj6-%5y@(7J6bI=;Zk+pz$eWMO!yS=)mRo%sQDe
z2|5-c@aQUZl!#cPpyNj1ZY{)Hu-hU)M=wJC-r}kV@s^O7>#+cD&Xyn9Cr_UI^2&Mw
zGy9=UPygM%ecSN=amk9xO3!4NgDb*(e0X|inLSVIk^L!k&Shap%6_Rkw%&7h^+xQh
z5$u&Ro#Zz0>|CqTF3^nx>FMq{IXV+3P6W-fw6w5*1`2<Ce7w{Ha%A47_75KlvaaX0
ze3Z@i+CH~MX?C55W!03!|K=)f;eB&r%6fGHkt?xM?q<_w%<zb~@p<o?@bz)54hsW3
z8e3Y9>Ue4CAK^YX*V=vQsjC5Ed7c}W#i!q1@WZ9ADOBTrt@dHN-;>U}_fKV;)LU6y
z{rKIxf1mof+}-VdJ`vWO03K%QSfj6^(vnrTwP3ZJPoq@xQiYXo?sf$(yzZB#8{xlw
zLEH=<HBB$Gt2>L+O{C5i_s{fElQPTEI4TH^rHNbD#_pbW>}*uPnx8qJLUPnuD;uvV
zia*_zpZI*?YLl!_jGUaDK~r*8WM5x*zSXtW$?@~Ev+-P@gg9k`hRZag++XD-?@c?5
z=3e{g_t7e7!u|!<EI;Hu(9(0Wd=$Kwk6+Ga!lD%8)KgOqo;~~a<$r4n3jq-kl~$<j
zrja^gc2eIslhy0aSKF37n6iD3$kX|DCLvi~+h({nb?eWYTDzm{?JN=1*)MNyR=1kV
z$1v5)_3SLur5e!Cmuo)gaQoZbh+T>D@4wz@%ss~xG{rn%$31b2kT>1oq49omEEHK!
zx-l^`2Wp6{2+^`Cd7%*hNGbNvBCn;BK-o#@?~!B2f|kc6^UbUGxw%gI;*F<M7X9J=
zxr!}t!NcuWzU80jleJ!=aMNX>Q>#YQ77ew@oq2b6c{(16PB)tQ#NCr^(r=xvoagWE
z?!L0_c1OzeQ&aM^w;%ht{?2I=$JY<MUVocA<J()8U8S$JTs_rShpj!eAud*3U7g|D
zn#jYmOtUZj{hOPcD``=nU|D>MfA1TIwuQ#i4kc}jSi+;Udq=@TrmcUZAH-}kX3gfG
z+_Yrjj*jj>8<&VS|Nj1<`@FlC<Gn>KEiAvjy*>T#aC_$Ub+K)C1a9Y;2}?_NZ{A$o
zcICmTsoHPu@1GyKIxKK&R%oV3WI#xWh`4^7%gPX=`hPW`p}RXfjX^=t%4PfappuFX
z*Tl3#hYn>3U1+K)b(?&AbF;`3-3!O>J%95kjOEJrhsSRkFHwJ>Yf=7AW^L5gMM+1y
zeCAq-T3heViSE{o-gaSseLd(bunP;FmlmcN-rZH2e6WetzkY&Er?#P?AqzV@Xfc9j
zkkv&0qKUeoxx_YKy9FnH8>?#UbG14iGll=wsc#p!O){@F9XaBn=_M7lr^3)Q>q^3d
z1C3{98mmXMyEdr#&e~A_ziyg-{5f{{ngfrI_b;88qL_PY%fX8mC%$-*v1sw)&dyFn
zH8nPNcJ^6jxmv7V?w2-eY})fa{m+5)`TdK1+C?p%u2x|6WKWyi=q=4Q;c`)Pd|A%4
z*Qfq1Ua~~x*Z24S9v&Q^wHQ7#je@46gAR(3kvX$1_cmzuE-o%EXo}8(43kO9?tG6P
zKb~K0Q~vJGoxRoJTvPg%qz9&C%GZC=oZmnDP_^T2RgGoO*}J`*JH<A46tD7Hdde<l
ze_d?XC;iwx6%(~WR|&|=gHGF0(b(4%w)*Mo_50KQ{P<{+cW1`S%gen3H!*CDI?FC!
z^Wl!^&ZxCvX%D<3M3mmvP29%+=FlV4-77C-tCzhOnY3t$ajTQ#!(Yi8^zSpTU$t+J
zO=VEmr?zRj(QWJ3>w}h1{rmIN!otEM@ZT}e^ufKo)sY(#7(r_})YRO%CY>yOeeLbe
zvh_hLRdmvhJm1OV+`UFkPp?mnQFrIB0N0b(>=$0T;T33f<FB#w&TW+%+2Q(&^X~2n
znoz&s`s<$`4)gzfIz2vOQwrzP)2B~4<wx!JxBJ;*Tm5ZA)z>Vour(U4CztrB3G3<U
z9XWQ4DPUEI=DE4n>aCvXphyYR*jYbc>uOek_x$OX-cIuh*dny^6kEWiyGc=7_);!D
zIvO-#f8o<pqI>r4?d<7MF);~gdbEGR<(D(8O10u^KC*s&eLelznVCUTe7u$hU0miX
zJ;h74tgP(I%ggRvN{N9T_3s}Y?GE|0s!cOLxr^P$b>Wny3)ggdFU*?qdRwQXv%68=
zpYWgw;0d{V_wJRvywrNMTm12h7aFcV`J=YwoP2k8_e{%TwYRsnR^PH#HJYv$yDK3%
zx%p(uqT1izf+p}N9KN9BZBg<<pm$cC!#=C|laFs+EOMmlW%9eCvO?`Bla(jWNM3##
z*R$lzGg+$=joaIDlP@lEHOiTFY|^Ajpp*MR$IWG5TQl+AEfXPG*|RS%FR#39cqnOO
zz~(gHdzH^+udR>&f9?C~l9!iu?AmpTaoPrrO_O@1&4q4Qb_nUFJ&f8kPeeu7K<#Kj
zo~yf-*!h#ULQiklxbft+tvkxzO0A9Ft`@(q1~m73ILzfFU)Z{slOG-)o@tQCWLy1h
zO4A>9&`Ed6$NMTODi&PKxcBS#-F>yv*4DeX*GcP5K6&Kp>+3Vk^W{M2w=(M{2BxrX
zPCJ`c@j>EV?tY%<f2J-`nAveN<-wy}`*vu(|8TnR?2a8fQg%y$vUS|vD#Mx|1;4(&
zR_|4kjoDK%u}{`IXrU8p+WC3Cd-v`YdCGhE$_&Hg8P?@`o72vonk0OUPtL~U`@6fD
z_x4y$R`+*XeKqa(W4*`rN|&WR-t@4O{%rc5HFxrY8ItxjAyQu3eof4%_SXvi_VC%+
z**DYf^Rh8Fv-78&n4q{Oc6V4)l9g8AB9`CZ-@7jjdieC|(VLsomujqkd2Ma<l~tkL
z8#Wj?IXiEDGnaQ&=xVo!2#M?K<B#9IJ==S_-pS`%P3kA!l<0M1vMx>j*Y$SK^<wt2
z_af64r37)cHZ9^ko*2DMdtPXbMd_=MY4@K?o98`v{`~k<?QjlG&V!dP2TM6wZQi;y
z_1T%3JhE0(qPAvzyzFo98TjvD8?W?-PoEY!HnaWt^ZERfQ&Y9Q->>iP<~}`5cX9Uh
zbw7Um*ii8B(3`)Jx`!_)8F%*gFVA%F)^l}OX~rFp`F4p$<dMIZrFU#Am3V!9%>uu<
zR*^;W=DD|A`sHjTj8Zt7*?1>4=dprX636>wS4M5svj6kJd0Tw`Z&y!GPmojI-rl}B
z;b2qF&MA*>ZccxBXQy#oTpTE6MBkH%uM%@zCMYQA7&?DejQ^RlyJe?&2b@VNn)<sz
zXs!4M$J5_*B6pd1zny(CBjv`1M9^ZNsoLRQ0b7)odQX3LppjXEhfOPd-I<@CpD)!o
z&+-yfvPnvMMnptxi!I*mGt;Q0rDeg@td$WPmDK&_oG3F5=#{gr+A>Gz(5={gH8ZpJ
zZ4D6RT9C1rQ$ysFwYv7EQq2bMk5xJ|?ns|uTlM>A+OOJOwZF}JrOnmU)Y^EZ%>vd$
z7+PKkb6b2-KtQ0uZ~5Zl=jSRaDnN&qrM{l^=f~rI(22MY54RgveaR@Xirtp44ANZw
z?oP?8E1XlkTDNciUi>;NJ=KUucg?+t%I+CAp45K2!lRb|^jy${90z&Xb2E?XX39P|
z-}8Q1+O2BCyxZG)^X~34OgO;M#v?f?cVC3jX5;j8po5jOuB<rt{eJ!Yw|TEcgoT~Y
z&$G2Ge<#B&uD2xR<fKa)aRr~B`R*!tdC2<xp2LS)xeJSnCvVVjal5rCmHRF8?w~i#
z@&(#=74F{KK5KcOnCldY#^m3^k<uUk+^=2rA|ouI(}^>kUv1-$XWD+sy^|K~irVu@
z>*hA2#@_Qfatk+v8t&b!TVCgzu-4V}c#%}m>oYYwRXT5HR(D<AIkB&38`l@Fj$Jd)
zY<Krz*z#@r!XQ7^BQt-LMH}&(a7A6e5a^lu=<?jQ=W)pe)z%@$g?@gUWO=u*(NJ?%
z@oDCdI#18dn(A#FdcN&sijsxJjHTYwK}W5w3SF(HUaTf0E`Iz#BQwaa+1J)Abm<hj
znY5o@-p=Rux3{2mox<vVXTI(2tpEKp#pKh|>G7L#ZW@W{$IVGyzq!uo*r%4Ovy7j+
z-ztA~aCf8jWm63%8Jl(K&y_!%Sh;B8{BKJyFLQO}m9q?0iS?^14PY^tvGb2aPN@EO
zwa6Z^L-MnB$mKqJ$1D6@XMy)+J|5P!t5y~sJ@iO`S;}1WXVHOgR~B$=c-A7=KgWx&
zP37~2u%jE&%&zDzQ+v+6dv!rz;5AOYyT;338wT9A2wryCs&0Sk!m71ZHg$h2g16iT
zx8~QK?G#q$;N_jVU<Rk9g~f-<{`MDFhwInW)<*8DshoB%zWeB*Bb~yW92^Ocj&y>K
zqiuR*Uu(DWYLBFGn`QB{3oC=w?S8*8?r~e(;;MKl_`?eQA5T7aPT{}RRovLKbH%SK
zbEgz6DZ6~(-OS&v(<}U)PdNpM`Nf#ne$T%nBPvq-(^zI+w^t=^Gv}QcksJ2HD*K}*
zpIBDV;cZfE{O{>^{b!2~ylrh2ykpYf=4g2QIrEA`-+q2y8+o^F*;CUeOgbCYrv1rd
zncCHM^uNth+tXPW?t7*s>qH3o)@;_Bc$%%E(DiLrs_BZyy>HGnq?S(2(p8N<|9s&(
zn|Yx%J4;_r3*CM@c6ZssmoGi%T9roZuiL95IqhWW`+IYHrOl19u4w%F^0GN@{rRHb
z761R$g4W_ZJ3E_`lXGF_<z+<=?mN7_wUv{b`|*)Z;gI!lbEVDmQqC6Wv`0k<FTUup
zeZjpqKfbDnR=sn0{a{|=?Mcq*4}bY8%duX$^5NQsq}hxC7p?`LooU}QrS|*Hh@}hi
zwWGCeM0B`zuI_Cu%@($p`D^>OxiT5Er40Drc|8d<)aB(`y|Y%M@#6A%?>C-~cRB2S
z)l_1#=nkuc7i~Lb$}S0n1U9eb`l5DW<%4(foi$YlT!ibTXH{qzW=^%3<#Xo$Rgc|Q
za;2|aYny*fweZD^#^X<~PWgN(q2ANqhHKW;kfMsmpAVnSdhqkd>I2(%bnQ-`)Omiv
z;{^d48rL$I%$@fg-`vH&yF71c%+8{)t9fVBj6qG3&(F`NA8cX`Ssxec_e|%}-tYI6
zO-!cL{r&au?c1}nOtUqjwq#7Zv0c`>?84UUaM0qP4ngIBl_5n3{<E5!n_pWOd%FC7
zt^3ncQ)e0`v#pKZelF;@hHdwuEEN@%3mZQEeq&oW`_o<X^o)QSrI*th7xU}h5WVnV
zJ!7gMFRO6xpPJ`C8=eQ>ZCmv|;zyCE(&1@Gif!UNyPuiaL_YogX(7w5bN&z7938ht
zMb&W|erT9JNpiu0Y+Kn*@yRo{S$9@ztcZy{{q^z5J0{GYZcWdRZG1RSvR_H+&K=V&
zddnVNx9!xinZ#qJHS4L_=6|y!ecYPD)e28ANcg<jmByrQntRtV(!tqrb7!yMe}OA8
zAyckISX3OF^6>bghWiXQS^Y=XFSD=y{%-56vg-HuZ2$cI%gM{D3To6Pi9LDo;>4q)
z-7haK<-WQqH2LeRt23+WdDxf_xA87^@0Zhw*)ie4g9Og5%nyspE-Y{aomwTR>~>*w
zxW16Eu&NH%#A!;(${)Y2Jvw#9oGE$xWu{(}x|SWfAS`l5{UHX8m9LjBt4=kY&aYnV
z|MSmh8}3yG_YCdk&MEq^OVGs2D_gXo^}hbhg9&MucTPFpovijN^yld?PrC!pWKDHX
zuiROyIMaD~kLE$43+^*27$hb!_BLJm8Ep4}d-}CU)7=lAJbhATU$WceUpwbNo;m$q
z)rAH**0_05R+X!ER-gLtrSyMSdL-{F73JN+YZlLaQny}JHR`PT0@vuUH4zgZ-rk&k
zUQR1?Rm=MI`qJikNABJQrJFV3>*M?nA824?_VDvNSMs~y=_%2gni|lGBzAteBj?Zi
z^W{jbirA=>n3#Cv*s(`XpB|lQoUWq7H8D&+Zt9*@3(MEKc0Mx>SheQys}2>N1#F)~
znZ+j^SiFa~?pxZln5_xFQ-pmP#Dpg6thgKc#Jl!KTFR@fUH)-8rhD%87%_{z`|tWA
zXhra$rw{*n`rKXn{CDliGy77Pt$F)-{p3TFq`1!h*%5vI%G|1D@BE(43Tu3Oxwhij
z&5dr~+-GdmUa)kV^`DwQ^Q*W;ujs8)`ZUMzaNB}-)%1%}Q(pfv-pgKdX35F>DJRTk
zoJ~8{Bk6p+PZqR}Ykuvw$Yp!?_E&s15D*eN^!E1lk6*t&y|p!aX4>uPda+&Gx0}Dc
zy*>Tvs?aBAW(M=Uh<@<m#fd$YpCxQ63_#1ZpPiX0B-Z*vzfaEAD>dJK&K|a)74JM&
z-`ZBAr&Y3Q%7<;=e7xT5t~+%gSK;Cep=F<TFZ0p*`M6QiL~GSDzm|`hi`yiGkDoo?
z=CvX+ig}`%A48U?M6kU1l7Hrg*L*~{q_%Ba+LnCkYq#;NR~OWpKF+*e`RUJ;4=a^+
zOcV<jG5NTCskf%+F?HMSZCp0<XU^IDvf@;o${mw}7fWro*?gDdIzC15h4XWtu<Lm;
z#nTr$ou6Ct_ihavkCao@Jw_cRr6x9BDbT`2(DCS@tHXA^`#txu^h}dnsmsgGKK`<J
zdV2gl%Rk1Kmz_<zwPdCYr+?Mg?iq_LHe{JKNA*36`NO^Sa{TkjMNcD(4f|enFFZ5f
zY}x|<>w8<}d8+u<&N{awa`Oz+Tq%3`eV@OiSZ;l)zRY*_u`i+R7no-xS!_+^yIfya
zSUR=BtY7lX`Rk7^eU-4Vw%(j`)9BW=T<@!^LT_$JY!0nIckKA_%UiR<kM+r(UgkUd
z#pUJwH9wa$A2{Gp_w%XvvokZDPfyb|J}WD$qQkX%Q|arlKd()f1w3B)Wc8)9tG1mz
z9qc}RlGdm2Ef1CbMaBE2cT7)7`dui@S9zu8;cO${ok_XUQM;UP>{D@i7nyPGSf8Qp
zn~Yh953S*?%DZd$>A;c&uG!Ydwr*X%?%I*-uP0^3-fvsvQ0Fguckfn{bIVz#cFE1-
zx+ddwY(?P3z0J(=*Di0Uu2pLLZX$BI;heMY^G#B(me|=lq^mud$2pxTYTwIT|Lk((
z$>!2O9#7*8$!aUzruyFaw`ld{6Ejt~)BX1f7@2teGTF4SgRSJ9R9k5H<9NsBTM0?k
zYf87t&z7mwl;1M9)VlA|p3vPTGgtlsEhyZW<Z5GUdvi~v@y_Dse&7F|doAhkdeO!|
zKOT!5Tlep6>wdRQJB~m9^W*V%i|e_*Q*G)C?bW7ov)b~!+4-ew=Ikulsa0PVd+<ri
za7|k_;i=AXz2#2}R?oZdd~T6@d--HHm2*ED<@s&|?|n6S+Nwvk`#1fucrWyIR*38J
z{bzs4iyM5M{N6(LyVLI8eJA#8y*Q=d{EX9KzrMacKFc&4)Io4=<2jgM5OKG%=+hHV
zP-$~CNl`_m<<ryCr>FZ{X~pfasQ&)0c9&K_Mn=Z6JN!E}x1SIwets_b#fJp1?a@ws
zO`)64)ZM(T^<hKf>A>^z!}9vL<}{w(xV~~%rpS}>h9erM?c>6Kguj3NY{dbg-&OzZ
zo|-DyeXYtd=d}sY*63<I9ah)7?dFQ-hi=6^+1Q(O{M!7&vNFHV-#0|)_{=b9e82zy
zyqlZTgSY41<+=NO3HRxbS(b8Bwnm72lASsysA|Vzf88VIAF>0kC4D{kUL*Ja^S4vJ
zd^CLFt2JlNo8b6l-MF>iH~o82E^<MALD{DIcUtVin&GxF5tdWTARDS)t)1_4<j9f5
zo6DV+|N4BhYBHB>RnnvbC;!c?{_^s1=f%%oYV+UyY*RZt=VtvPb<4)z?E8-If3I^a
z-|EQkR#}D}`=3j@-OD}E-<Udo-@je&LL(mTpAhOV-)vod>~hI;_2c`pJly=wcXV|v
z@}F<#;^G1t``{AMNDzH~Y)j^43G+Oe`!%0^r_G)_TQc|dw%&h#e_!5TU*FW+Y<zX8
z_NC1lm+nq~y#LYT2ZdAK?^E?=b#3_Ja@yNbP{HWhr(ceO2NE8JU3tyEH+*mY6KR&C
zQ?6G$yDxQMQpR6LL5G#s0#}`7QVM<lDDvoZP<Q_2Yx{~14Q{<s8%tk@IXO8Q*=R7m
z`1+0KZw<@Tv%e#^Jbp7-`NXl8$0g1G23O==u$X3hx$VOH(?8G5xzPGacmJappI55v
z&uUG+>$ETT551>t-%z{#k;T?TzO$#NC4YS1KY!ml;lk~QB#bp?%nWvW7`ah;_qwBn
z|9`l#ZClauygK#ey<J>(p|(Hs;!lOF=Z=16@S^*@#Q%lzqINMy)<!R^`TI~w=j7!d
z6`#D`%~A;8x@=33hU8`om9JbK5oY!E6V~;fvXSmLWH|qy=R$GCNvmDv-<1|W|F~<%
zjtxamy=Ix``#n81_2!<+;<mi=huir<ZSaqej=s6Q-M?wk)Ql@D1Q#w`_~zDD?)v)r
zLx&Hm>TrDwKh`hpep63+L%FuzQGraSg#iI;PP$AgVZE^>GkDhCyo?MDQSGoLIX5@u
z+}|h5&(Ht*MxwV-UuyQ{z3sNUkFS5Wi&Mw^`-kF#VOR8?eYq3-{>{vD?cCZ1wT}|)
zX6~4@{`A`Ae=mGhnf&STqs^&`b1b)ZE5;@A$Zei^s8=f3d11}b*ow!I*O$m!-u{ww
z-~b~_q~C{+KjuF9u%<Ph&ot}*#xpVHT~nQFW^Vt-@vA*?%bza_vF+bdUu;O|*{#0$
zQCeVI&)W7?Up(q0?uqe>F3@xNy^&F6{k*da-NkguzkQ57=dh>f-(jm;7wb3th^TzE
z?|a|XEr%VO*)|qE^#ZLoKQ3QCM^U>9G!D!qsui#$Be3lKJ>Q3iTAwl>>Z+)y*iia9
zY*qMrzolMNZ){5KZgExo_h`24+U7MmikIe{GZatzTpvH}LefTq%u6buA+awnFH2gN
zt(kha{anMr7cKGK7bAMIZ<c(}Qao*bY29r3Z^zQt>t88`?5>?@)v{($Q+>LRJAdJ6
z294~%jZ2JI*ks&VYrSl-`i_2XxufwL*L#}Z%r+A&{VVdDFK)@1?eWG-*NgAEQn&Y1
z-TOt%SNIZk3cEkqIz{gMX?JIfGS9P1JpV6FOl0RdS-3L#+ww_YZw1~gnt64<%;OKo
zd|AG(J!r3MR{sA-*t+YP-kfs}t-6&kRV$Q(m-p$JnZ{SvL{7fFJ%9P>-?Mzw+<K*4
zcbC1LVVKPJ_;|nj;)^HRn@rjCmrfE?cH2<$G6+<9g7(1uGxOQh)Ggp-^Z4Ro_mX#a
zX0r3keR#co|0Ml*yVbYdM5T1tgu9roN`@Y9)Bnw~;G3LhMxE+0Zia6w=fBH(Q?z==
zj_*nz8&fy3&-q-%VD8x|_)+uno6-kw-Z6SHpR1pGsmCoqWO3$a{uN)IM4fOiul%C&
zRn`7-_p%E~2FDfzTz~Xe&37K#{*LU2E;e`D(=Dbft^c1@B73{{?kk6~Z-1A|Pp)#B
zzGkuWdxbS0!!B7CKTA0~%hYG45v#twzL-vg!{IjG`?r4I-BsGnCu_CfY8EFqH)!p8
zky1L-)m5R&Iyyc7{{DXa_U+kIQ?)hY?pV3sS&|;uvHi&2hFOKVxsulpO<Nyz`%~c5
z|KGMQ+YtIOHlN*XO5NnkVFw?4`&JVY<M;J?;g1iAT48HEHYOdtv9ox2D4T1;i|Xs0
z=8}xnQe|=+2OjJz-N=4LN_WQh!rzHrD%wH2B#$Xq?~s~o^rj~Dy_AXW>>!84igl+S
z9GaJZ@$1LKvL5?#+*bQ_R!HTZS|Yr4ulCcOUfUf%?~u(`dH1Z2jrV?2;Br6Cc~M`B
z&!2x8$Kd_xx=&QCkZqM-&cC*miTwdf$`-u%YUpz0kEDS%3k%Z&`8z$+*?;HGtLA?c
z)V4r++SYpkO9C_`tV%Szr|T_EKR>VL*URNId@}2$SMf@lJ$U*QbP3gs9firW*p4pw
zAbe()NpIxkWgAVNJkyqXWI4@e_OUIQlOz6qx#q6FyRt@Gn4LH2tNca%)_=B(R~uD)
ztu%1m_<Vid^C@iaWa17l@w^P$&%XKD`sds1p4u+b|95Ch=H(eCxv8&`)eil?adgSc
z*`1XSP2b7?n0S8LvbVn5Vq;<=w&h61?k)rEYis2e_lgivvhLiGnew#K_~)GviS=rq
zx9YBlhOGs7JMsTF*F?i8j{1y8RVAkkRv&V8U9D7a&$1jg*m0V%lDAjNH0e+ar_T%n
z#;v!k*yG;G-#>ApU(9Il@r*jVI{EbV$G&`c@6YTX-Xo~~BQ5LH^s{R`FI)UQ;c<`c
z@Qj}e+xV{9C;nTrvF6|J|2M7AZ_rq0#&du3=F{&sHFQmk*l;8+An-_A*WScV)*o*I
zJ9WPa8D4Z*eC(e0n`3g-{l}VOLoVFV-niYvMU(sX_um_}ec$kwzI%O2bmm;k^j|j5
zZJ*cr*>2-o(=>Uv`r6O`eY+WxXP7P%znsLraq(vJ<=4)qtT}q*UF&I1-oM>PXFD0h
z|LXQtPuUV<HD{0e@__Ag)J3K8p5E@174-jm^jR`nU+(>57eBk-+)~N^%)5SLczD<S
zUuJ1nH(aZztho57h24JJrvrOkLeGf1J&4X@<DYz#^{M44<2_ZCQa{A}Di1vKa0)w{
z>_21T24?9S8=lKt@B4dl>n2}~Rf3oP+J-K4V&#*yI<m}ncEr}KtG9)VtYq)*DqWm^
zf8Uz8y|eb!{#N41|6a3V)AVPz7ih`MweG)ocY2}qRqZ!V4)OKOv!1?L@9IbW>5peV
zeWDe^@a>JWcb1^WbMN<k73t+{hQdELpS-lKxQbbBp8c`XwNquDN(BX0f~JEeug`P7
z^Y*Z~k%&%v(mf`l&{bW#FFE{ZJ9<X)yREk0@ugi+uPwI}h|T=8f3{+Xqod=WpU>xm
zb`oD(8!ap@K7HN0<datJkM=GuDmkS7L#Sx!XZ89uO;IUn(Wd2mM~xrl+-lP8-t53u
zDOF}Ylf~%8ymKDc7@}ByZNEI>VGfh?8{>@@mfTM!9=@6KG-S=1Wx;ve_jRu>UmY27
zPgHPrmEne1R;$uiU!rbjn@FvT-Q6~M@@4h%yfan0w^GE+I_Ep&@>EUHe495fULb0%
z+|F+9Q`gsoufN*l*8FMd_q#nSL^ei#X}GjRnN3vhRDZhN<Gj1O0#^p9HZ?Wf+p&gQ
zTu(qk!egOR>kPA8DLp;C_j$)l#a}*@Js`Kb_1xxdmlsd6y|r%lA&2*AIa{Mv{QNBM
z^X<Dw*}*L?uCp)hu-oar<<G?}M$<0-&YT(T(9Pv5bzSe>rKnTC^Ak&7G`ZYXul3=n
z3T*CvcJkNxjhSAluT!c7j%_zhzx4P)jMwfHitlvZp5E?oFQ2pccJ(dozq7xEI?XIt
zB<=rf^TpVx#&T|EcfT#lAJ_H%o2@o=`KRm~OkH~p7jEL@<OJ=advkkx|5WYpM^B$l
z)q5AOxk*Fu<3Ty2#&u_9B`-#}irc%_@fR%#RJ;ADtoJI<>eY*?xz(R-y=<^~>#Vx3
zp=VydInpr6NGmMh(ZZ%VOG9|pDIDLvsq18K=#{mL(&p`VPWoxb>hdnutX9k^@X^l>
zud~5Uf(|PSD=QafXmgzP<~o{wb#)Ev!VK-lodE)YT}m3J@>2t9W}lgBJ^j>F?aNEO
z#l>`^zT~wkHQaMq+4xxE#qy+gPu{xR62E_ED_7)v`AoeN7t~Il-BDa(>C4;{xi!lu
zPEJn8I&bqPAtSCH$-Bb*>x-s*);hUkZF*%zow)Pts!wkZP0rjR@NQ$#83nih-=yE3
zsx+!OeDnD&6WiR^UnczJJ}#Wbxwv-zN10u#nbrBN>ce|3czzbzy=(ou81G~AUr(rc
zba+eAjsLd))UW@&SL-x2YHixBEt#MhjH<7%3=<ErTypmD7wQ%glJDr68am~4PU??K
zEWh-x7R6{U&J-4nyyLPeb?U}SSEkx+FD9&J(UZMyT-|^1+3bwA!wakQW-2}kU{ZEE
z8=|SD{;#d^qqOi2#mN)pqs2=UTc=*woc1!s+oR@*i@Uq9fWUz*nU`<uELOi?^Z9J`
zs+%5jjM%?Me>+fAXcX;l&!!Y<XVLbp*L(7w$A^Cz^jOK(r+zrJvgG9_;oZ+m6R()Z
zS|rNrum13-ZF9Q+$KMBT>2S#%vX9gb+#Z&==<<oJdw1rtC4Uipd26pWGkaLd(+P?;
z8h?0ZFEh!V`|aUVc?P5WYiBr*9k-huIMXfm;HK%Szw?;-Z&-ZM%guULJ2C0mh3ao#
za&B$W1l`N<_0`oY>*MF2IN{-`<K<CP$5HWA%<!(^$0HHSUv_bd7-{X`XmBty-(a_4
z5qr{N-6N_I9e&GxKe_R2I-7QS-mF5+RcYJyPntWs#DyVFj_Zk)@b1S;H|r{X-1zwM
z<L2&e?%&_uE)H1e^nUv%3*njG*Oo6<FiJhjm^bg&<c@W@8ZWda{Q0_QuRNnszDSwt
z*QM#cd)5lh*(clmaO2c{rQXa+seh~==Jr1LQ!6`jj?og$bAE4c`0eBQIOk4vL;Je3
zGc3Nk7iC}CmU+qk@HNjZ&kJUjcuuse?EBP{w;=6vf3#(NmU;W1sKB7lljhAkdT;&m
z^mli5-rShXE+;29MI-Rh+8a%K^(K|}wH&clTYLPr?(`m=unD_6cO*YCD=@yqbfRN_
z_LB$6?5j))Rmw$XDEGdpXuSBpE!L_hb&83m;&Sh3cNsC2fOQQXPa>PIu<Cvk*|q!l
zk6KTSlz>IoU!M%#XW2V{@`4XeU#?3=EEBw!&3}2%gVTNAuWU-a(LdvMpU0eM{*P|P
z?5~@9cX#>X5Uu<F|27GpaN7JacgFpEfv?*|5)LlhwqXC1Z5up7#FYF_PRiryRC~JZ
zDPPBu+~_<tyK`OD^FCeqb~-m+?DSoY!;_mPZQU&^FhB8i$?v?VyW721ZR0<->uEYu
z)WHj-?r{>jK3~54Vc-lc+BqljaGPQ3DUlwx$Lu@*DY%NMs@&6bQj^TOy<1;UJ$>T7
zuOS(W&nytooPKiKo+LxdBQtKMPMPS=ocNnJ$9`4v0jt`iFM)bZnvs!<f)u>fCdM;`
z#&BKz_*TnS;qY7TXz?!-xr4;F9J#z=)AXbJ*GXSq?(ZJ^vOZr;gz<@&7kjC4zFTy-
zyy11r+SjZ%=UkjVbHCZFraY;aO-s^Evhw~{GU`q^z1^^`i6?coUByn0&Q$@)+f6Q|
zob_k)w%C0r_p!)Y{`?K@E1Y)T(Th;lk?>ixcVp4Sy-D1wL`&V?Dtz!yS@U*Z$c<ys
zr$eP|R!>cQ{@~u5J%0o?|C7_M=6g2BZ`v~7%bRoN7Hqj2t+sjI3U*I73%}M?#v9fM
zMLx{?vFzdw!{cs>r<ZSWzx-z2JY5?>`^UE(x^%oe_9)M}<HB}*)80Ryeyo`JGWBRn
zL$@2l;?9@1zg)>(vn;VeSyf4~ee=}F)n6Js#9AAwMMdv@+w|zY#EoQulk6|{PBE!l
z@pylP_R~`<Bq!cJ;TxFZuwd3}s|&|xTzD3)U_R@noOUdG=a!Q(D@3lcNcv@6HNJi^
zJNd%wqsxs~Bqv=^_cs#UEO=wJ_oF-KUbCC4Z@OFN^<+zJ$JtvFi_Ut*#7<hIR6b9I
zr-tR5`F!nXE4a@{S~pJMcAv3f6^A`*#pP}N_M)q&x~fe2D!gp-@hrz}No=b>vps(Q
z^z52c`{2c)zK?d7KmT^F*eA+(l9H6=#k;Enuk2Evc5j}{?W*ZN>-i2p{G*qaZ{KD2
zCqC@DRAbF?wGAyBlP<=WWZs+q{u|4s<tz0WY}0a=r#-RzC2w5dbmQ2?q>C+i_nLnO
ziQLw|vgqlhhi~$~Dwm%2brX2K`Og7~JEr9sxxC9<j~ekrZJ8aG=Rc{@C~$Wvcc1+l
z=V>9w|9tqPx9seU_Mel~H-|r6P<prLkI1xc(aU!Ei;F>Rk-zSycFM2kOwZoIof51X
z`5e@)*t&Wqqcm?yk=TXPsXRH44jf|Toa4Q9?e5J7-bTC*e#(*6SYwmB`}0cW$me^Q
zc6D#jf9$_Dlsz-j<m--&5l+kg9NAR8Q}cx5wAhbHPY)$@?U1n8^gul9&MBv5ue&WC
z$IO(OrF2DGH~HL#mnLy_4N(_Yue@B!wW;Z<#Eo)>sl2ma><)Ha{pWeaI^KlR^IQ${
zN@cZ*mY<IQqVc=Z=kL#D*DU?~{Qi7CZ*O5;&-32b*3htO%F0d8=HJlBj4{!dU3xhy
z*<qdM<*etKrL)`CPLAxpwxnw3+sk?)xjme{-JhE0UF?xG7Lu0!eQ?*pr6(4(9AS5x
zJt5}G%{O^6XFu$pRP=5pYnRxR<h|*2UK&iDPKKqH*I2X#FRh$f;rp(|XX$k%|J~<f
z_-)$57P#Jbl98P}dBMr@$oeXkpYp!~?rgV}z0SM5>|KQ3KO^BM6BZXmvx)!VJ^f<i
z7M+a5!p=&OS-a+{g)(<a@9vg3aCDx*hwPJQE}i_R68LCR+YX_rQgNZhnI`KKcvZCK
zi<K^7J1hNFzB?o4$oYR=UddvrDwjmVj;;55@FZDV?x^gFP#?c5mCVfBB$os$f1G!r
zxa>Gv*O$+mq=mXGu4QyvzTpT@UUX3CShI$9Q?}S1!4ne#n^&B1*~a8=w?eI_;ouT>
zlZwADjvsqx$~kLO_pH6M9yo1kVO)BJRafz&A-FWVG&`op)VOl-lo?7N*Llud`HSW5
zU2U6!rqk7Wi*uhgvYnWqEL|X6`Yr3vwOgHfSKBWw-`mJztM>S%8vB>bY5W0ee)6w5
z8`$$e&pxpFq3br;_op{5eVuW0r*hC9Hzy}>1A<l5M#SWE(Sb;o%{NaqSEoJEW7@y|
z;@++6?sTP2-J+@Jp185-;hILy<3Unx7O6rngU&WG^@SbNt*zS|dB>$`l|{k3^(<z4
z^O(Aw4(#umbZOB{N9!K_+Ih3Y1ZVEGdirnb$_0yZ(~NDmODvuJGC6CC%l*{;t4^$y
zlTwVc*ME-Q+xbNH6369L0_%>MEZot3z`pw0f{bfZZ9cK*?2eh%x!Nk|%*oEOo3X;I
z^A!&@HOHJ3-mKy5Wb<o=yN>+U$!UwNHvJ2bv#GKA_##nuTFdkY)6d%^KKSf-x^bzu
zrC_@2%6_*K&rX{&KCS#avAoKuEi5s3qpQz1{XdtY|5hjH1?Po5xE>~VNp<3pMJ-2G
zUtG6Q`Rb-q9Y3OvpFi%yan5L$Nx>24WVi3rUY&Tla7nPHq@p{|6YX~nAA_{2Y`fA=
zO;uq(w!>Xa?lj93t=Gk`OHZpVPiIwqC;9MG_PhQCVlTtbH!?-K?D%XX=+iiD>(nWw
z(<ciCulg&zBFv@l-x?JWd$lux;xo*Oq!_ZBstcApm~J6-ewwc0iPQYg0zRwlaAUu`
z>662+lG3EttyMX)(X49(WvsWgOZqwsonB>`q4-wzyV13zMRyK8og($YzGzD5h0U?N
zeWmj~c;DuKk$mhhS6?qcFG1wh=G!~MwsE>H)p>EAg?DPS$)le1*k?cb6}McU&EYyb
z=jWx}%J(Ga3Lky^vRP1MRmdC5r?0@hC@b?RdhIG=sw#24OmbeUQl}QaFJV&dyXhLd
zruC!tBgwmdo@@`yCLE1aXtvRemSkW}U{OzR?6K0`K7aAlKU}A8?6YaKPihPnQ!5Xf
ztobwAS8+l--~Y_Up7Zygvzz<!?8{6odv5Z^PlV~6R)t7*?K;8jrU%dbDCOjN@!-hm
zv&@O9Klx;Lb7@Pyo?SHK^JGm{bI)L|x{!`_t}kp$dG`kE%G+AKyg$|2?@>y2^tUYb
z-KFQ|PT);>vArSqt@q-~uO`fXZ?y8gUC}w|12@%}g|_Po)y!CI>yu&PwZM$=o6Zde
zao!iJH{W+xEp@C9GoQmfbMMoNr^WT#78qxDc9|_M^fi6IBhtHPTFN(HrK5kW0?+)r
zes)6UmRIu|Yt`p<xY`B=RvGQsrE)jfUuad`<V80pMNFAc;cD}_vtjCo<t=@!lDEIO
zC|=mQQ_6qp)QFUtg_}0z<=Xh!Wo11V=<MA1;E@vZnm6~>9DVcF<Lw%wq&%q?JI@}t
z{`E@DT<&N3eh(a+CVlXdo2ealsWI;9(>pu)cWyrA@cq%tIV@i<pRUnfkrC{3tk*;P
z=cZFP_C}tb^!(f}B?a}?aEq<0R%L{G^u65Jx&Iy457DWG&pP@zm*-?JeR}Nq(}fP}
zrY;ltld}J#=BDtkGO@dw_MYo&nRo4hn*W14|JI&(Wpe22OWlhq-j2S3eL>CLC0VMI
z3ZK6ys_NlgTDDflXa2rZ&pKaxRps|z`|x)Bp%1#Nlm7*BpXI7+bngwC^Yg9xJCi?i
z@4q@?yzY9^nlNtWW4jFg%w9jk>V^L)bKL_E92c(PnCn*9o8Z0T(`2DlDjPKvKc3=v
za$bjd^UUnbp5ME_9&kE(D(h70HlvC%rtOV#x^psaxg2fy>Q=+-vPkds%gQ^aI?nYy
zIjxlZ!cA7y%KA~p23Lh%ha-x$MhiLeT18Wip1HDFj3wQ2?<wmI`Jpo2atE*T%KWr?
zoh{?3X8G3pM8qbAd3W6;7h79TIv>hb9-k|Bac*^K#+AOBxz27o?|dltaOzt3Y0J69
ztxQVamuRxu3r{hbyG^it>gyYOYYbDg9L}oE%?t|Yy5_OR?pk}|m6t!XyZPrY+|unJ
z>>r+0bGcQieOke`vmCGf33@GGBo;F}EY0hiB+t6gSu%pBXLiU`e(U=3bE(+x>D9)c
ze>?p9n{8lS6rQ{#p4Ur)*T(F&lx6$0b&H*+p9?bi%{yUDv5MXGaFL0IVgji<te5Gw
z>?!J=@ot9D@>|?{^O#zB^4By>6&63!Ai7qvy5roTZ{BG!pH6GG^_;r-;QZ#3N2WPF
zVUM@`ZLVIxbYf<-OW({3?%U?~uD>}sT46m$8u#>`ZM7WpWRG9iyx#obpSE19JcY;H
zXU^<Ss%-SwTI$&D70gv<mvJZa!a{ZXez~*fnXMlwbGmk#yHwjVrQFka<Mqz_{H%&-
zWsBP1h75_%^BAPKrj#xS+2+%saq`KO6s3HV8_T9%VZLdbl^AigTj;D?;rf(duiYG-
zu@_9cJ?G5>XBS?jf18<@Sc^_y5}C|trCnq$SpIv`?1|rW&a1p-U&3?#y>*@f<HU1~
zT1TYa{8Xy1v~3M6OpUhTes}B0ic2ZK>&uSLmTm9WXwaBY!^hYEX8rZMN*3HZx*FUZ
z`TL*mm)m0_a-vkN%sZIN%fn-jSieQ3>ZxyXzBLo=HkK4EI@e^^!k2w+3TyC;;-}fa
zu5It#WVSSL632<n)6`QwGg+lX_&#s5%h@9~{mDyB)ez65K&MBgH_ERZsCW3NrS&S^
zE9%{}*mF<#`ew*SJ17R7`xkvQsi?znv-Y`ri?*^~@oC5q@4FUZ%q|yEa8jc{<)WPX
zf_iIrqq}BBERy>wUE8*X1;?bZriAFH-u}ToH8=LVc!&Gyy*JcX-|gANpfp$7;=w|(
z59^mNOP(m+Yq0TF+z!pACZ@g8t2=JYJaFx|7}xI=D<<W{he#dxlX7P1T5hXPO`fls
z>e}zv?w@|q*80*yXaA~iJZT5pLeI?F)jDH)#PPX{vtL}?qIuSeC*|?Q+5Ug?`gR`r
z6}kUH;e}n|*H-uMI&?~td&h#>=E#(@hI4LNwXc-c-}i>?j_v-*hM9Aw*Ylrw+Mb%e
zRQhFea<#nc@`z<mPyC7h5&Yf{+@txwb?;2MdkOsktwlSH3tnIGl={16f>`O_nKs5V
zl*$e+FBABoe(9Q*>CNc(Rkb|R`Tm~N+-bg7d`?10sc5zT{YMurm2Sy-s9EZ`f2y{I
z;z!9OUjyTWN~UJ%6&xG|i#DEXl4EqgcxapM$2GMI3Rzc7cnX%ds5~jyW|Tcu?*FW;
zDI8bUE<I8s{i$QYm!P13HB9b38-7mdd?m`d?Y45>+gU3<1~A53UU;niyZde5y!q#)
z1D}emjP*MEUiFP{^5WK2sxEyp{=2iUpY&bgZz}4=vdT1I@&@b1UO~2P&qP1|KGpay
zSN4Lt)X&1(#g{gV8tuALqOhK6N*>p*@NJ)dGYW5&UZJ^iqtWYCl5fSu{%kt+@<LE=
zVkOfpCoZdLAq^Qz<84(>_s!5;wqaA`tEM{l8J0%-?HwnV8D;#N&!aJ&d&~D5ChH8H
z&zCLtJ-<EJ`_4|$D{qVDS9Y@%mHn)!dlMml^jX2H`l(aSsqA|4i2JVPA!RPR@6D@s
zSNGhXu9N(0lSCNH*807&m+q(iJye=A%TD~~f-e2dj1Q%!ZBBh0V%EX@a^5TRlX|MQ
zcTP>rnQ7hm@c0)2vlc%0;z!S3oY)_|O-N1qaMRx27oj&idc#aR-`|^i+xom==2?%f
z44pN%clG;DFW>aivpDSH#Hro35nKOi{Q6KI8Cx#4)jQ;<O&ZI-r|Ulcp7ZBRuE7Mw
z!(9a*{Fs|g&CIa05Z6}mKX2}~A!=>dOB0Vh{vmySKUm_O)|L2~|6UTvol-0oAh~u<
zZA$Gg^H8nVbHpdOGEO`;eZ94CSl(^n#H>oq6RNj_QYP81O4zpH$Bvgb`8P)?wC|N!
z?xW<(-JRgK^E{VeUyJlkJ@(u~tKVlA6{QL7_^MI6*G|doYf{2)F3`~}U*9qJwq;*m
z_wdP+FOL1nzwU6=Fu7l<{H9m+L*;zcuH#SQ>MxafxyLZ>HDlWw>650-G~=v7*Nm-E
zC99sF@0K=7)sbvw3y4x~_-f|teMSBJslPSBSKd8Lxh9l6sW|(vLex6V(+{uycm8}w
zEP~~&ci)oy7ffC1@4p$pzWB;vY2ZGWmg8r3@umJ;d`u~9?~=w%?=EreiI#X%9BFUa
zS$}!Sr;s|{Xora<6IgGrjW*W`UF8xGa3MSQ{gWq0?En91-n`j(zFJjM(ie@`H`gpL
z)?dC?&TKC)>tz;fbK>EdP4PQp>}E`J4W7rmH;h}#cTuUIPmf%0H@8yS7Q<!BpLhS`
zI61>mdE>^78dK$;MJ&n7;N*0eouK-t@u}v*89#1ocP$DRjK9^QvXJA!0}bWnvpwF4
z{6Bf(-H$&-YfhdjED;OJ&Ht@GeYVH56sM1&-%TxF`fAj9#n^8WPYGT5YNnOaq~84c
zJSVm!j+vL*^A3xDxV`?Z#>E^F@uW2^`m%lQCnvG+b1zHTSu{oa>fB_0pO*9Qo7>yj
z?bq0G9@sh6K4{MsAFi^$_tZofSyUxGghKiK*DlO_nYbp>to6V{Yk^Js&wT&)^4OWP
z&HFE#B#68Y?^~F{dHdfUhhK>;BFP2o|C#dg^YhQ@zk6oogpU;>`G<~O{;&3LURdX(
z*i*;bgv6(-^dH@EdpW1^+NT#DKEAV~xOwt#vklL@5?6`VM(Jn^w<ZNyZgcDW;mL9I
zOT!gjUB!v#Zd5s(zh<uU_@J-w;rv-kCjSf+I(^jn;R&uX)oiKePp0eE-Qkq<T<<#X
zn@1G96FT>U>Al6_fjb=K-TvE5n0a({(Z4#G`z3!G`zAYgT)APlrO5H@g4EoCau2nO
zdT)~&_ZTa>8av&IY>MJdzVPt!lu|*B<v-e|&5nB<u;}oI-|uW@`}e5!|L8d38EW}I
zUMBUf(Y_$V%GX~Ko<7dr5+Qo-n}%(&`C5;Qq4V0ZroIWXE1J_~nts#7ecSu^!;(gU
zyPRU%dbEE2IkBw;)OuG@vE!FH8K-1FjaB`pf4=JJ;E9H|>E$^KJodMK<6qv9-n(ta
zZuy65t`1>sMRIu#HtSejN{SkEOr+%-;y<YUx}JY_o9xU?(Fr<E$-B;F{J(nU*48c^
zhdaHGT`ayV+0eZ<Z|C9HvYub2PCmgkO|6?_LZ@B2-`;PLnqn3maoX9&u@bKTpL4lR
zN_#WmO^{uZ$&@wE{7+{6yR_1S-Q(^BAt525ms}I?xOz3D>QB$lxSaFYI!$!vp65kL
zYE^AAH-onP^ZmA<=#wDZ&YD?4SGP>qn5>qR6E$0>bH9tx{?9Ho+twdC?#uUL;|bXV
zMYAP}FSY+Jy)g4jgwmo5>UxI4Umutj?|l64@h4H^V`u+HmfQW>x!S8tQ%DHZ`uMS;
zph#Hq%AJXxU*~;0B6vmVvX!-9j7yb>>LkwHtJY@dcJT1cI5u_0l7}q6*wZC$70Pt|
z;Wd!uewfvhd*=nDuiCKQWLx!_XOZgej1u#5-i5T3-R;_x`qK5)uV-8Jds1u9SN*xX
zfme0s@wL&<XUf~moGr#1b~&o7?LOm!z$>w9uI)&9^ilG}(}dbN8bQW8o*v7ZbMi;(
zLYJ3Y=S2M!Ha;Z3cY(mvvtp+dE00f~r>m^?;P-R6X<t7+oIcHX=iHq5GaCa>eA0ZI
z6sdo6=^o)PE$e5UodR0tmdzEs%qQ{Gn#eV`sv4pV<&RI8e&w2<dH?^DFXq`s{ylT7
zeVXp!s?8sx=BS7*Wp{9TbXVy60jDJz59YXq&%2uP^qsTu^~JiEIwyHJ&1`?5EdBG}
zYhja>FFf1l9c1~Q<96C{cZS(I`=`=854Oi~MTN<oXo}qb;quF>9NCE0;MdoL(>RmA
zZ4%IH&wsxDv|E}_<YC+08$Nx>c751(S2L@BhwRU_^3F~rIZ}7#z1sRZaD5zud)M!W
zfk*o9x0|Fn9k&w64{BN<nJyT!uHeIy{patb7GGhV<}<@@##b)mWBL!Arf`PdYtxbb
zXK&W~YnoQv(;44(@ZG<D{_mT|X6{0pvgE1{^|Z*|IM8PM`~vesZ#Kh>YCf;0XL_nP
z8b$b?&bUx+`2Ea>4}bL3Km&7^;?KmDHmsO&8Z<DsOKwu^spx&0n!*ANulp>|?3*~*
zEVk*x`p3<7Ia*(}s*I}7U*+GH&VOG1@h|Q@wx#9$8w=i*w4Jw~FhgMK>H@|aoB4Im
z-!S`S^!d75@C9EfP0h&X9(&3!EiOH`dzmA%`@Uly7sP~r&Q(-0n`+CiQl@DInoyjq
zWNf`QwJAy=Udwy_mTSM~{0Lc79P~ncuh`nKe$RDBuDX8yl3x`R_1tE^$(J<2r9VEE
zz7ALz$kY*IT{p#gnTm6TwA+OA<#WRCJ#Q)bk}+{{VAqV~b;5PO%rt_gCWQoUaq={p
zai+uT;=c6@MB+{DX4t(Avg3U%qpNKlk{G@>^~_iOIXk{wnY3?<)aK@iv->an&{jW_
zX1plFY4y~_7jLXf^$9V_FnTcQ%`@ZIXFI$u9D5P4Gb6~Q>GGCVFXqe52K&}pPL^YT
z9(3y|Yo>&Mq;=-c#hIx(i@#qs+TI{0IxFROxq6BUZ&{U~-Kux&{Xz8^uVpk3x;j0w
zO)zb?ncZ~1Y03&$hwP+RL9#!yR<HOxb<S<~N0L>iyaI$xpPGjKb5>b>@x+6vRi8q`
zE~;%ma5SRv9-mrMnDmF$3lHhf=GolLlO7RX(g|A&uz6<WY2kor^;Nc={qlh;54x_b
zJeR*d<d!%0QT3<l>u(AaPd;7#WeLwD>40Y^oOV8ETRNrlsl?+&JBof9SuD{yI$@&C
zJ=Fr=I|svWUw?9ZMd3uN1@|@ErrbTJ`qw%z_l=&}+Z!kEW=GH5_>_ZHphMvG#>{l-
zurkT6fZbx-)6b|Z{H>y9R-Sxt$H_Cdi$oSS@Vpk~PTHB5Gxw%wd8&+Ja9z^Py>~>O
zD4bqbv9T!8|6jD-jCIDnYxrk3rQY_L#XIjT|7xcbyDDuGrxkB`Z#-}2;p)ee|5~2(
z&y29<lw>{D^X}yO<9&=5^3KTfeE7`pMP{neD^*ZG`d;9HTdIXI=axQDk5|2aCE&wh
zM&-EplLvX4qTDLYCB?U8&XmyHv|;U}qH}CrVpE0d?HWI_>+o=fOuFyeSYJ2s?d->$
z1)&%29&b!beR5<;9Dm1zbGwd2F9|TM=h~Ynx$4c*%)1M>lo;8ohFGqAvNqvMspMPk
z69Pwm{n@nH{95#P%PUJ|H^!P9ZklGj<=ayCR)=Lz?)ofFDteZ``uwV;$r=lIm;3kj
z@!SrXyKv_9WtrSt-?wKO6x&)&?s*hnp7h{Rt3<(qMfD6iw-3#-JSbA(8@l|%pSED`
zXF~f@81A$1AKZS(>QvXao?nxK+-yCPL32I*ej*1;?w?p!v7_?h5mjZ2Rf>8Cown54
z%rrgd^Zx0_JGu)}yc%?4QjVOOIXf+;Yt_uR>r<Yq$!?mn?fAFVN_-)w^a_j)9k^)y
zZ27gxEsI)Oj#MA~)sda3_UZXaKQVR2i@QxK?`{fSG-1}=>Ah1<<$k|fa?0B@pEX_X
z$f6>Ki@QyBnlKe@zhsiROzoCH{`S^SUYXpYWi78iZks0*-Ew-0g~*|l^UI80#^t~0
zeTX&>`1i$<o~n~qV!>muFV1?!Fez~PYo0o&E327ncCmP3A4^X1VY4@vHvZbec`?*)
z`nJ9bO_l?P?%b_Xdw;>Wa(iPoA5(kK+HhCJxtX2Y4<G*ED)L?CXzAb0b|?BRHY6(A
zTJ!myF{usxXKOHBe%jore&5_X-rbU%7IpT@r!$$=hiyIerg2+vGJTh`J*hTrvf*@_
zjndYIwk}4`|8%V`yq12;Nweo<`_(;n{Q?gQuh(>$!@|gZCdh3ss6AoFpmajE<;Zcp
z`P?!4bE+Lrx*E(>3|F*_+*tHvM`x4j(-n-ZQO71Yh)#DZt=a9xe_}%B+J+PTUuAwU
zG47Yjx+Oa8vg*~U$dF5!(<`hs_UkwVT)(u#&iK?Lr?_%e#YFQ<bE@7Sy}7E2Pj`;%
zvOn=2>ejKzn@xXN>t*h{&>sH%;?|4bEm(RMh3;IVAv0&vlR2-nSA`cR9Bi9hw*IVy
z-ny%Sn_H|G9L(xHd~KtU=ZPoWCF>{j?{Sq{mdPD@;ViFCqD$u48SmuMpQb9S%$T5^
z^LfFnGrz>EE0%34W7^;rVK5_SQb754rRt+%A!kdQRUfJ@>G@Tzc3xFQ<<gUHxyDlp
z6ISi-I2!5prczc}FXF@cwQBMUz6S-pn$Q>_Vv_T+<$W@@8p|uQpik>P;WJ}5%8xZz
zt?%*LDI;<JCSy_$Ph8>>U+Z>}*Rs7+&+R_)dSQR3qkdMw#T#~u{4BLEt==qI{Hk<{
zRCU>E55E3O6`O^p%AC0+QKBVovPd_2>;M0Ci;o|?^kidw(U!o#1t%v=sr;34Z0h|b
z{l!-Mv^nk@w5uNHd2y&t^+%ne+?-o(I|8OFZZ<Po9~8HS-)}?3$-Fayfq(XGwQ#IE
znqid0YQB$i!PO1fAzmEn&z8IXdGVE1@c)X9F>`#{6AP#8+}JT=zSzahE^bGJ{{DPh
zW}vIhzr;UWeOug$<JUHZPS93<8GNnr+*Mo6D+^5Dv8Kx0*wuZDS^MHGDWC4HREaGa
z=4<c7T)I;Ys=&|BotyFY%6@n2t{tZ|Z>*JaTw9&=^6kluYs+W;e{QhmxLU#5oXIZT
znw+4CG3PD5_Rg>8s6RQQlF+-k==P5uQ4ywhMcUhi9Ttlz9PARiy?^u4O(!dFrp_0h
zzo_)wZKYp#q-vPjgLb^tIA&V@Y`UARMc_K-dY`wx-#YA9T=|x7*8c6;i!GTkD$459
zb(VhV5ar4Iq567xf69qJ8DTAY3w9hkFp+ithDTvyMzQfnB%hypazSLFe?h2P7}t}$
ztwu3imTX4(Y5DW}>O9%2^eo#W)|nc&mp?doM}9}>lWd)O9>vEUwl%>f%KzJXmgHG&
z)Tj)vk-BD?e~rOVGxm*{PeEpCI#by14e5~zUYCWt=cYW;^NrRpj#|sv>&BVpu_y9O
z)D|^S{&Q{nr)|AFd6$X*o9Zu7NdZeY{QP;r<C>+A(0sw|aVn2`Je`7OPuQ}~?4Xy6
z)2Yz-vY9O`4sA)il6Mn$oQ+eSuHJKxg@5JI|4yGLhbt$PmL2;4d)o=8G$Xh8iEsbb
z?40#i%h9>*L+RSmkdK<IpT8@yui^I>Hu<Ud+P`1rzq5ST?`_K;?|N9XbdUR-t6rNI
z6mFTLl^pjmYt9bQ)7wvO4i!FkDP``gc>f=PH?B|h-G0_)W})jQjh|ZrckP%E^K1rB
z!p<z$bs=h8C!Q?H<yKYeJoUu9dxfL&zI|IS2bXG^?(mrI11j`{gyuhdKI@e3uEp7#
zJ6_M-TUqkf;8?7+g=jGM-${2vUK-8{cHTZ&KfciKoo&O#t&+macTI{`Rw(xC+A^#3
zVgl!5$0<{p)<u4BQ*d^x{^B0i7StNkSTSd>YuTLr{=d7*3Kwi{3Qo&<ZrxFT^3j*s
zJ0?xrYV7>w{x-cANt0YpK5zQ@Q1V;+S?|{ee*f9`ieoPGsn2GQQu7br@&r$d2?@<N
zmGru@U~8#TxA>VuS~bTHJew`?`E|e@+1EzbuAXVL(0*5WD){#MlT&Io-s!!xv}4nW
z;*iJNulbd%w3}l0mN(^vp`^dL<%>6qF0lCT`y(k6RG+lzV&cj9PhyRF>uQ;QZayBf
zOpIe=<P?v`hwY?~oH_dPS8?G>7Z1=7<7L-?ZkFGgg`$kzSItys9{RU%bHt53uE+m<
zzaaXjlvi|XakquGqS>u3jm-u%EBkI=Qk_)s?Cfj5!v{?{WbEb^TAvGeJF(TU@o~lu
z8wpkL;(;Fz?!D15>VLJ9ca5Zh)b$63A)EP`Ca-2Ouz&F8=_a3;s}qy_{I;BamnHvP
z)LUY~q}z77Tny}$T{&0V|F>*><|1ZQ{&T9)yqPO+eHBq$dhOz-V)vEYzl5J$xz?J~
ze_^uYN_T%frd#HUv-c`~{O~94*|{k9eR_vK>d$)kRr_LmdcsV%iuJZWpPT$HdDgpw
z6Z876%?|U<6qtk@pC%o4XLHAg^^KP7UuGD0JgMh@eEu$%_D`$*2mHAo{rtUfY6Q!a
z^_f$ZQ-m_-g?GKG(GyvFYlhQP_F&cB4XY;T*L8(m(M&Y<)w{CA{(j~C1^&VDUR75g
z6&_$%wR+vgjFV=d8U9>}h^?T>Gdthc!Mn>uy{alMR2Oo*PCv`&E%AE$+(+kD?vE%s
zb714!+dICtvaemMo)Nf_=k|N<Ie%3fR<6FbIqNX%u72$UD>lSTay_Z;<+-1s>+bCH
zJvX_cYG$j5Fd98S-CuOxZp$7&qj~c#v!B2DrUJG`<Ttm$GpptKM?Ka5>nelRsKiaw
z@nd@4|01IJa?|X^N3VJPJAY{EZm~2u!$<Rd+4yxb^Vcx+OgI|JFn8*Sl*o&qZow*_
zwDR=E+Z+A&`xIv~7}s=8?Ogrn<Lc)f`;$+Ve%gIf;o2mlhpvhbCUU;B-|%Vv+OS`y
zL6aZmFl7fxEcU2b#S#B&Q`R<Hdxv@L@%=UPMXp{JS@60^c+1se4Kb55axZ^;URm<z
z=C)IFCw#=1J*#<s_UW4g>a3NU{^cw_V%6*~x2)Z>)lKu#t23Fi-}*gP+o5rGi(Lq3
z?-ka+or@MatUPt%U!iK{rp<P0Y$kmv+6s1(dlUXUdi$*6$=H0aXNAbcV;O-rH&4xP
z^I07FpXZHrAkWeny$9lA-{h~4xW=F7m$LiqloOh+lHf)yV`ahB6C%I&hpv=cGCQ63
z&-Eh{@1H*1u`_wWb1~-l>%pH(^%}GP9}JH_VIaBogeX(O(=&>)!sljvM05?n-GLpm
z{ZH!e<1*9lPtG+uf5a%xYoBiNYw<tpw$A!+xayy;9^0KKJ=sf+NTnXs?C#LHbmBv4
zl~JYU%TTW;ZcC=tM5eyEx8~{mJ6<*+4w|CPvR`^QOg+Va7(4iUdnk5!1N+5{n0b;H
zEq9f$R?n5?v|W49ciw@Wf1@W}&1`vHtbJvP0JrDNjY?ADE17>?cANd^pQVBJjFZ=k
zgM(|QNlxee)3A17LFQ|jG?@;kzwf>N6}>$COx}BA*%5<htuafgzB;wE#2ivg5UI9K
zON&{5G-9n7TTki2)PKi){%2bCPM2R0_9d&W<;8@%a&1;C6#w(QIWM)4EBU{K=ifzQ
z(@IvH{C8^ejNA93cj>I}01a||oEEvo)n6x}Z?mg-?}4w%R(0H_Nlbb3FMT}wra~{1
zwQG90!t&Wn|Nkicb{2{#Im4heF)h_)(ITIiUC9q->81XBc>L>{y=jb_JLgYR+wdUk
zlg2WK0%cJ?!78>j-V1JjV$q2d`LbGT`LcGVcM_8O3>&@E8k-_hWcaQYpX3p27gAZM
z@li-4(<)b4*=yE~y)}l~X7i7%txDhb{reQZv({?a*Vgn%nQ~2j`MB0kUT&t}_t>Qo
zHoMGY>*d_d9gj4KzCL`|L|WvOdF}2?+yD2^DC>XPuO0c}`}!`0Ys+7+Fl|}?SM=~J
z);lL68twL9andoGG4noq(AD$(T1`CA6(J=G6<;E5U0Lt@-=3}UD_`W@++O+S3mx*e
z>tdWPW^Fa9`%^Jz&YS=ZyE`)LK;uLew<XTz>`zk+R%Bf|F;H=~!wF69iTye=HWoc<
zlH9Q2%@RQu$*Z4kD{04z7A^=XYn&{|)^~5C!1n`ZR_L<%ElGT^b_08|(XuE#i_<5R
z58bWLWx6Dl{cx4woVhG&x|cRRS$E)tm=ss%g6-d}PH*(J)KL5wggWmQ93Ykd!w$58
z<I~gApyNAco9EZHxGzwzIV?XVH%Kg6eKYeq)*StLGqRsEbZ}UE{1CNFKIXIJUASLL
z+3B7&4eUEjm{#3k{V0}`Z?jnOT%1zJDW4SW4^LRr4sl5MY&*$1x9jRs^Xn3OzsW4C
z?-z>NWL9ansQm8V-`_KDY*5@?_V&@2mzPhZ$op*qPlCyQNMFHdt8|}b)dd@aP4>JC
zmtT%#)OO=;Xx+K{tIxfK>7VkLugkE+rA>>8eHFHXQ^$erxq9+}HAhq#7CD?gmUT;f
zx?!!W&GVj15+a<P{hd0In?PG|n3$PY1})`!eSQ7^h$!(+C%@kZoVF-_jrMj)zodL(
z^`#r<`S-8if9(8vv-nrZ9mmt|{M9)k{%=yJT5+?5eI}E#`#;aiYD?{Az1~~=+^?pl
zW<}6auIg`Z4&J@{xA|Q^sPA&G!Tp<@V(_Xv0@-UibIWyhq)aI^bj=7*XukDL|9(<Y
zMNZoHdNGG5$E)~i8XMwyP2X~LzqsSlW)cu&)jN-UvE|YTpFh?I57}Ft2n)y)pKVmM
z<G|kCVOl9u3*XBtmtQg37s7CA$r7eO%?YZL-z;TWb>#Tj!=I}q?`~FKecrvk;a0Nc
zCUEcUY16e?eW!iqb2K^J-&cEcTduToJKs{!y*VmN-fPY-DJ*B4uy|p_T%(v7d*&Di
zhAFgFho6brSJcjvw{xS^F~hX4o?l+<+>&ZDncw~4f^O$&{+anVW>xU42w_rdUYoPm
zu8rMLj7LyR=85OZYi%tRUkn3ZG@f}n`(y0C$!jbl=6G3#zWgU8-RoBO;~{(1mlp?5
zP1TmPD$!W<<Kf|j;56?$XVdAQH;*rS@@abL#nakPnfHF2AvpQwnQOZ}PwO~Lb?fWl
zxbk57%~MUjRdOy8mKFDXmc{V(ykB$h{@Ft}blqa^K6!F_;e*v7@3%ja<9fCGw1ddY
zP=C*6A->Lf_R5npHfT2f)?!^8y+ZPCRPVE8jTaNu&e?C9w6<l|PDU@2v=hN@XUi*7
z|A=R+rc0VCuhRT+C!K4`>8CPwH9scX9W&poQyDx*Ge|{SG11&$>1(6=dWMFEb8ISw
zroMXHA*!#h|L@=Ld>%IDsoLSs?(8gn`DurL?u=s>Jf9oH$DF(n8C|f>z9TPgcShh+
z3*XsS68Bv_v)}z&!Q1Xtf39seT-~<y*y+RFV(K;=>#|dSO_@1E|L@Uni<jRps+J3X
z^Ue32>Z+uD22A&hj4k#&H9r>R^CoV`9PYpgi`O)*e4F**ZrA3ri8Fg&)upbqTvWDe
z{-O7;uCBhaBCt98`ntkTPdrzJtejLQr4_dN>*c&I&n4FINwI%R`Zp(qn)(_TB&^qc
zqrSp5ZENVp<WQUA6;<;xdZQF>dq+Lr!xL%5sZko1rykr8Ifqp#d2K_(ZmXBu%~z`1
z3T|8}&U`y`LGW_F^;>H_JUG@yZ(o*typKo5;zM(vV3k;!!<Fodr%g3nFKv-NxJFg$
zR?jihr!{-bP74XY{+qo>TJVs$tMR_d*cqLjU2MK=F_TimW}i)Sx!4k>z5V?2Y&A8n
z_G1s#_G<Sn4SvDYmCbVdlZ38UFL&qc*=H5HUTlqeBGr5M%kRk2<)LZ69)hk!%DmjR
ztMs)|<|P#w8JT-oKfA^CK^NfgNE)>)Sg@e~k>%CwtqU$KI5J1c^!}Rbi)vp<*)Qkm
zRqM@UI-9Xo&7@P<?{(z;Yz@n{|6Y;PzhqzBQuNddbR|vQzn|&5%HPlXCv|C3$;1g2
zySHRN@<<hD+<EF$(@oF6mz~rky&M{+M)%LV=QlgS_hlzXzRCk7dE33YDM^)@u1}UL
zh}>rb-J15}<KrzE7n?3z2zYjWzW+3x$V*4}{jIJMR^BJsv#eBmn@(iYuKPjLcFB4t
zns3TYikA5ocW(P)zl&G&jTfumPdT{m<djll`^%f2_`W(NRoiCh=q#9Z{n59cnJOjo
zj~scHrz36NDjD4uz|hFCQEoP0&FNG5O@>##Iqy!^s$St@uJG<!qFT@G7Oi3??`y{l
zG9T+leEZ|Kzf*f<++H0;#p6E`Lp9>AhuMUdo81+4+g`cr%$Fme{r+|}7MGX%s~Z@6
zn0w{ow%ll~kQELI2@2KU-X#9|@-pSr6wO%sbg#3*hBG#qyM<&G1n$d{Tq&5_D}1@s
zjs45xqPF@8Q|F)FQ=%oZHA=Vk_cv7?ou0e9%YzrWbe8O#56=ImV|FGV3sx>lTdt?M
z(N!@yA(eGv|JsVHVT(DgZhW##<-#T9o10RTzrVZt=gVb($k|O!H8nMMKk8~<Mlzmf
zQZoLc%E_5|>hJFBnb*vw9Nn!pS=q=wnz?<G$ri2Fa9@{?`?qfScD}d9JJau8zw-U7
zKTGb1b{)>XwdkSXHPcW-o8{8Kp1xY;ajx@i$J<?<6Sv-3xK-a)i^WLwJ=-hM35p$G
zf2OAAui3uH!g2@ev3b%zR<5$X*0%FTRKBFzqsZsid9TlW{UYE>&*53mcqAl`{fmDU
zknl*0i-kvZ$LZuV=a#QD*I4`5XZ4vcjxjMZ_iMk)hOdiJG%=a-=jZ4Bmw$1IXdIZR
z>@HzZpdhLpmU6I(HDpD=!_RdSp3c;~y+EdI$F~wrulZ$`%l+npE(!@*7c)~>-S5GR
z7c=Vomw(BeP;oy(SImdgaDI;Ymxsw!eDdB~FJvTMIaTrU(f(wfkAWPAX1fcx1g1PW
zF)?IiP^($)tqWVT!+(8$|3B<h-QT%IF3k?-pV{r$X}NgMPVo&ld+!+EdgE5~v~=l@
zE6x@3?wx;<C_J<L<^leq=DV9xJh@H_-IcyJ)i~N@vYzzE)4A_Ym7M~ORo2^ds2}ij
z_@d_<xBX3q%;K3Mi;kP0Fk9R<;ihap!(HER5oc;j|B6&Emenvzl$mTKe(b2}yq--)
zSH7I{EKAi}bolwQ!19z^smqwp&9QU_-4YeI*UG(5rZW4nNPJ6kGxKCM-$iL>XMyfM
z^P6k+@$dKh%TF&9dC#7{s{F^iS1(K6r=6Q~5_HI!RjHP4)E1Bbx1xXT@z|4J_2tdc
zL#n<XGxmyws+GTN@z^7}?!6Ra;PVyX`NyKQH~Y=ClCiIwbKrmjpS<0fJ(ZtN%(48;
zZhh=Y<=2yM-R6C{W4oiQ`@&D*mtui4n-?FO^xDle$pDlOdtdF^B5zvoK2YWX%ZZaS
zy!qIKPaK@3b@P_v)K|sHW{nf(v-$2@_GNNmTj2MOOShHgsR=Le+5N8m{QKi`!e+|P
zXyq0+DtQs0=0DG8b=X?F6L!!Go1(U69o<*^J7Plu<IPQ}lj8(lwzTuhU)q`-E@xA5
z;NoKUle5j=yKlT7WVYo<b^4zhfqyP+7gm3MRq@-sng0JOw8H|7PQ{;Hu}o{#joji*
zJ34u?)-4NM_2AX@&`_V;w<oTY^=&z+x~IUBi}AmdmfO`Z6}wA!rB}}od2!+c8{hm^
z(Y~WOk3Ri+U(j*qaOZ~$+uylQQ+DV3_U>-?^5yDwc6KtBMNgPzS91BfHJ)(a_VV(j
zP0}+<9@t2$y<g_s+?Da=`wp43CYGDVy59GetULM8@5-umDF@po-_V(mykke0`&prP
zpuyBdf`0J<P7@3h56f+7c=yiq(3<|vgENI<me!vCdFR@!j@>)^=iSdgx@04#!%ka;
z>0x(+{Wzw0t@c^_uQ^6?ZJ0Oc7!%WMF?N2rKM77BId5)GPyhbz?v+)c-174B+j4KK
z)mB_$WapDvkaV<5L^o=Rb30$HiIbp%j8(~tfR{pEd+HfO8~5xiHpox0mf1Y*z(v#Z
z>eXxF;=J}4UQV7nPg<?uwCeKkRME;<R;6<iI}^V1Xa_s`-cB+S)9XoJ9Jip@S9IEG
zw>f(+#nnZMynMg>i1l)r(mgYEr|ULMEuQXk=W@ptEse)F3WWpbf2d6{%h<VN6ECaL
z>l=GN`x~X5`H>fO)ktz{*40G`2b;F!-WF>pdU|T_^}LtG(a#Ml12wAG+o;rk6cW*j
z=y<fs<5-{E(K$1VZ#=j)>wa~ZvuO1ML1py~)=E+$f;#j6TZ>ltE8V|Z$RlGoH`aRk
z#@7p5duHi~=tOtvc-_ptx^;tls@IPYXY>8sll7x^?*3aYw^ib3xvHG;<Lf8q>|W$L
zb2Y<Ku4fA;<jZ~I<xD$sX6Bqf&(+G`#T+v&*S%Hp>Dh%V@iVRR#XkRUh@Zaw`IRr7
zI<}<~qCei$h~LK{X|#KJU^#Es(#SN{ny#l6>%Twv`T04hA^-E!)8xOuzJhi~$J$6Y
zDFiI_5(VvAUhX${N9AX=ef##kJhb_J@!=dZ<Dw@XZM@RQ4z+T>ytMSRd%3L2Yne{|
zQ~PRvXIwcl|4ymf#Iu+GPqsO>UfSZ4gNe`<!^-3HPP{p%Ri<-&lbM*qu?^??Qo1f&
zF5WIEAR)3u@wO>*|0nAu8~>cVZ?K?QxK-{2=dDR0b?Xj!)qR|52wLoEf3f=bg5D>0
zCNaDU`Iz$X&&?B0-!J$6bZu=k=mw})uTIUeEViq7_VRL>a@v<DVXt1H3yZI6S3Xjj
zbZL{yywC3bg5F-&%NGk4iF|#e9sbk%N~*rO8r$ScsvGBRxp-UG*Tl?w-V&#aSzk}Y
z#suuURA{&S%=aUcRJ}oG=LoC$c;vV&ahJ2N^EuupyR+)6R=1dL)pJgBA<sMeYG;3Y
zdwcQ44AA+W@pV57{rWXaoxN)UYHDjOYZCbXzmq<8mO0pd_G7UtEB}>=o#3+BF*`>~
zf_q1eqIdPq*ONP@HOS1CzxTZ5+x~hB%jIT{YyU)S%J^3;+{FOes{a4qUlF|+33++>
zLx&F8z1S%*byf(I*A{M9hEy)b<yUU}6>=5)QW#@fK2tk<-I5>gKRwlUdF7)YIel-;
zba77C-}`R7^*b`9bx~zwKuy`*^BSLJh2`b_pP!pM(<qhe_xJby*PR3vl8^PQiQPTz
z++6G6#csUQbfeX3Ywj`6HqT#H^z;<ywAKCpe!X5S+NGen!($0^m#Fr!kIC&TqZaE4
zZv1{C{QY(Q*`>b}H=KVt=gZN}LLUz}EzB{xz39s6&0SB}mFy1xR{eFY@SJ}B$sK`*
zU)<Swnf)evtJA?2PT>z<zp5%IG}Qh5wbWbr?(^*JIVLYII94(4Rhq6ByC`&ZSVeWU
zu$<hyb#MKGKwB|E>z@)56ciN|eP$Z5s;jHNT(mE>#7g$y!GoZS=FZMG58jq@v;W_W
zB_|fS>^gAw?%c=6`6eDM>GNKFJK6B1{d!>q|Gx%YJJP2)I=u>1`t~%6yWjrsZ<SAv
z_|EBbr~G`hQ{G2IZF%5gHyPWiDW=)iE-Z58mb0(Zu{|^2Ks#*Bg0i=_Ko=Xkc8k@9
zIDO=llap&{Z-4yejnC08(VxHH?_bLOTWn{=N2PuH_H_o^)Nwfl3Uv#;?BbC$+MxeL
z-=_Gq<@Oz%$6C%mKb(Elk!eAQnrp~Yr5THKk1XA4bL~+5{F{FZ)pHE5o0X?ukB`o{
zu)uNG&Gj~wp9(%aaQyV-r1Mg*sTPHgTD}<6C%q`K+V}gN^|4;*<MV8*UwoM8o^mzI
zwD{SX#49TTO)@XFWL{pTxKw|SCFq<*zqwYRYpw17{a_XspYH$chRQ~biR%KirshQK
zp0uoVkEQ8)zFo)5?R*O}<b|fqh)OWo{C!{R9M>FWXO^u!d$=r$&&~Mu>9fC^W>oIx
zu$o8BV(Z__9etcU*+Q&JA$HppSs9rbMyXtnkM}Rnyu9qpEK~2Q8TNwfVs<vIU$4*B
z-1y-`fsKug-OS|A-@mKt==AhTn_pTJX{@BIyp;Q&l#9Fj<!!mq&(6#|+{n!S;>O0u
z>g7F?BBpqhb2J_JvU0M-4egG}#o5Bu|L>@m*7)TqInP=ZpcL@-|C_s|4p+X*AOHBg
zkeyv-!GfgcZ`j_<yK%#Gw_)4eb7fY^|K6OlycpSW)Mf4>Cbp#OI!E$CewjZv>wv8V
z*zxNxqg8S7Ifi$Cf22P?{j>Lb?fyHj?e4$-{&#J@v-0yJFQe=FpSMj_EtZNj-no@`
z*XyXXcFFtJvfkWP*S4zltFD-<D;X2{H9Y=kNRjELlNk>_UE8<f-TP-QFA_gJJw1KR
z_3fFLm&sTbwSdynzS?RfW#y+0eh1C+?s)V{nZCHY+dTW)8qcZMzdt=Qb8<5~|D#v0
zyjr<LZ|tw%zvtusLmQKiuLxMkG}9<mNkyfliIscVx$~)+bCoA;23_X*?*8}138$x8
zJY6|QY5T*wYm4vc$BS`a33NN>`uB55FYl2fs{iWQyep<`;rQK@^=#RPYnE*-5qi0A
zeD3k6ZOgwqWs%*Atx>wn?0iS=-=7b<adcCPr{#=bfwf_$Cn!2w6g*&<ZI-*prBf(v
z<K5k5Zx@xmzE<-3T5sTDx7v$(%$o1I*PJ}kBWXOtJYNoU7IffZx0O}ZsybaM`U(mS
z$Ik9k4!@h3qIJ*Y=-T?VQD$3CE_@KzGI6f;V@rwelHzHZj{mE|QZ|QAipxy6vg(J3
zq?mA$T=Bu*)_1f;9$t^1w)6)7+OMy#gRX^j>zDK0o_F_2f*<3dLx(c1uao`x`FZ-!
zPfu^$zCHVBnY+9D<sF5|CQ`hdoSdMeg|e<{6+JnjXnj=r-TnRj%a^NL&F$NoeZB6}
zayQQikw?mNZL7t;M}74OUnOp-^>td)lBA9P>x%12jdzvj9i1{?ZP|~vphEuaY;*Pa
zeKlw1+s8*uRY~5rZy)Gf*Nw@?A3c4VYIghYi;K#yuC8`p8MJhUeZ3u5Ytu~QbfeT$
zB5CL5curRHO?mFIDgC@$)wegEpfhM~tG3*<5p+Ma$ip!2?k?A>ueNSH5jEGQRBdLw
zzMJufKi@p{*5sbO(wY3?>gx1WH`o36Q2{z%Wnb;@58u8mi`ba-DNS9OlarH2-foU<
z^|uX$kKJNoW6!4D&cCz6ac$JrjJvx`^X>kA2lZLAu6AjKt}@8IWs;MVbEZuG;>(gA
zS?g(MXPZBM`_^}=R_K|m-%r`ZCnlwwpEvih;z5hd^97l~;XS`tG`+Mw{kSA|Zhc<g
z_fxJbwKSL_qN6ugd`vplEA9UG*Vo9MMN9WQJn2$VVe#(WJDaL69m48<GeQGw%HPRM
z)ec{le}7*-t1E-Kxw+pQi^j0kON*YKirG<cQ0M!^4I2#Z+`YTA?yuEyzqv-cL3b)H
zvFPjTV|jPq`D0E&U1{)_kLQ<atUPe{aW=>8XGi7!zctCZVQ@7s{r0w8PA;ybi;G-A
z>2JmiiOSFW7rni`J$P-Dsa4sVj{biB%u7oQXXo<s@hx-fmHPAN^ZEMfQ+My(lgrP~
zZ{wHuySuA2a$C;K$hXR`udiQT_4QTBt1F#rqqjeL@+4*2@vb>-Vw#gKW*k}VUvHLo
z+xy}-yA_vHCN5eM@R0Fxht>YJzt!av?d$6{-*~oTheg%*cfJJ$2KVmW6VZ!_C~ML8
zSbo{C=!wUlKYwm)NMv5>J>6{e+xr`n*+J)%+`c_~mT7j~hyA;Yo^pY@k)S&L-=CjL
zd}o_oy{4h2*4D_(Ui9XM;kEVg`DW!u_V@SbbWPd$I4WX8!9N+c%F-uSE=9T6@6d4d
zO!RSM4&V0m_@iy|>V2|ivlno5`d^6HS7WK{*0Uh*?yidJ>eGB5Tkq_toIJ}kyXf^b
zUB7uYnPrZJS678rR8?(CKHgU^D#G~b>FMJWmEBLwG*%bY3VCp1qVmd3Zt3UdEKEAu
z6%iFx^yf$6X8tY(<!&X>4{@8@-dFQ4c~HLp!HnIDL{wM}UOd{dWutNBoG))~JWDov
z>v`|#p;m5_{ChIy=H_YV=FC)@Z$2mJ&%fXAD=I4uOI`%9@k*t{#h>$0JJu)Ly?wj6
zb@{s`4vkFzGwbT=>Pp_;n!3ofJ1H}BX4OA4NuD`YrCm2Sr-QDCw#-n`>AI5g{@z}b
z+>5jBd|ftqV??s1htaa`+C!}G->(SUrm#9(*FRL=L|)s>Y}!2AY9T49Q?pF7|9xiP
zSur<mefpUhhChD&+En+q>eN*2^sw9V^78(mmTc*3vFL3%iq363^HcWm%iBG<u`zk&
zyPcO*y{DOU-|G|-YmKz3{bgdBRsA4lo&L>J-={RGI2Q_<TWsGY8GGCG@$<~dOBM1#
z;eNQCKmF7cO`}yGJMZi)ZvXc7_GG<StFZNPxn<@WHjpF6CYtBRP1BF>i&{HvuG63F
zqO*%UOueRPIKH@;kaT|SlUe^Yo>T{X5~_S|vV8j0#6SFhRW$awUtbq{b9=u0<z>Ey
zPoM7o@uNcXRQ-nR>v~$@>wG4w`QF%=+<uew=;rkELF-~HZES2leET+Sfrj0z9-Xd`
z=PxfWzq00XgL|9Kll)B#+*7BDJk_qQsnicQ`<`&M>*yzw4t@byS>NyP?pjuV)46x=
z-jmbQ{kuNtOYpEA>z6<O>+9>t?Rm1I+OzI&Jp+oZ;N^a2W*WD@-}l?EjO*k46DJmV
zl*-7;a$0?0cisGA=YzX4wKYpbo@(!^PLjR8?aPt-`Q0o>`BT!<kKfvw{qpW^bI|Ps
zkB{{RP2n$jf6w;SwY8^r7C*PB{In#J?I`Fd;I+}_pw6FTV2zN`CXFE7#TPx!zyJN<
z_ix9pE#DpAPVnsd)Oz>EfBSj!V_gr({}a)P2;ll8=r`Za_w%!}D?tZ&|Npajj@KXQ
z>hJHCR)2p7YU;?>{W$nWj&s%Z*DtTHmv3xr%(=gB?i|bFQ+|qeoRX0uE6bm6+mLW_
z(a$gMZ%zyL&KCOnVUoyGZbQl5n7f7&d3jHdY&<L^^1oR~)vMsq5l$s#<)7bf=W}v%
zPhPa=areW1`+pwi=h=b+O4+^7WcPv}e|{F**w}FJ^Q&7}L^Lg$utDR<e#1}g56}F(
z?{aS4iu0F091far{>*~z=Z}7UOWDZr_{KyBqx5q+u0Q+VynTCfLn8CMdGkWn$H~UW
z$7^~``u+X=<A)CqZcILIQTa(Fbaj~NZYj_i3S6RE0UHt=x8>eWyH@Mn+TyBc7uEjl
z?Z=xB&Mqx|TUn8}TjYuH#iMQo|E9N1(Vx*)TmNPX|62bur%!{9dpp)6dH6shv!{oL
z!E5kwXI@ja0{2#xmc75%yLt2Gn^zxPSm+$GCPGkOUmtV~ebCA&%&s*>MQ0ZoXoE(w
z+;aJi!XzBF72RE`Fh3yXi0`@ed2a33dS@8cuDUsw?a!on_Vs?x&&}oJ=U-m?`&(M%
z>($}wH&uR4+p~A?%DBC|BHyuRUt5#-;6NkjAfTwNStlD6Rdl#Mu54*<4_;f9yQb*6
za`@Rop)9+`6T8!QZg^&WtY6OSe_icTh4>SjQcqjd{rM3dS?1!xvOVwaudMTDw{G2<
zdSZfN%Bhq(^L2YGO771x&Azd(*1DBjd|I=kM})|u;Ob9LI89%hC^$`=T+{nJap4sA
zbG0S9>)$-y<X;}97wxe*?d%eTv_tpr&yU=k_VE3C{i8pc;%s)k|MvFwk6*v)Ho1T2
z{8;{EmTTg)OG~}oKOEdEaz@}Z>wB>mTLXip^cTE;Qnl{aw+G+f&Q{gn`Z&|J;zL8$
z)m00FR^HfMzMchi_fE(8v&W9H?a!UPGI%3H`n2oL(qA3AKIz}FvFL7Z6t)zRa9HX!
zb%}ygjoZS73qQTEQ%nr(Sns$pWK+#cr(Y%6Ctof#?X0Va?E1v-<E9+`w=Id|_L0uy
z_NP^vigufSetr&0Cue3FOWIa#5pk(0)Y5DeFi892u_^tu*M7&Znpd(+GmEE9U9_j!
zQT%+Wy#J<~t`F-?IM~_QHM<1ndxnIF%$qkaW@l0B;dcJX>i%+?T|cx^QUW_v=3190
zJ<ebK<MZ}yHa73oSu30M-f2Fs&-Hn^XWN_muk{@RJI=p&b93_)y;v!CcXvKnD;38;
zj|dT^ohxE@n=w}Z-TBhK<F@;BZ^ubT?Dw91nOd6hGO*EC`dU(Is;aBtBjd=OMXK-K
zz3Y*)oz==Me(G!Iqt%lyUvS!U@aom67Zx83Ssgd~^~CKbV~f{BURQo~FY;XV_M|r_
z9C>6e#Fq&u+D+cE!(z{#Jsg~z4-Ygli)e*-bSZ&O_l<daZtm>I$N2&mm(}X!zVrOH
z>TrO@nS2GiSN|&4<-R_>>FDQ-o12!tk#MR}o30<ftn%}-DLRppCQJ~>$;|~{n<H>C
z_Sct}mNFs^v&-gMKDqm0^_u3m_0G$CkGp*?owqS3JNtCs+Sx0DmUg`naH>(;|K}68
zb30$^r6ryfRaIV&pzDK!grudVYih66ZhiXx-rm_SFE8I*{XMVJxWyw?Nm&_mRZsf)
zd7!?pl53Yopvxu|^BtgDx8iSHT<l(0SO~hPCHZ)tXMo7bSv@*jiY)^19Tzh|1NWe!
z_-MEI<rRU-nq3n%XtW48ad7bSFZZ6VH^WD*Pu|||+M39v8cMDcE?;osP&|^JVl*>i
zM}cDMsVNKd?(TYXcXxQxB9Fk7lt2!}mLEQc4j+DbYpeFgjT>cbDke0sa(e~3bSW!?
zuCY3zo@O)?G@!ZEd-}4xySrG}*}VfqG^at_GylTZ*VjQ>KYskk!pa)Bz=2V-%i|){
zToxZS;m^;{FVDHTsiLC7BS2(j8${dxiy&>6mU44)a<Z_pf*MttT^^=TZ60cqJ1ajw
z3tZ+SDP@x3kdUx|#dYGV6DJl4IB|RwJap*Li(6Z@t>*SUc#y!^sdRPHL|4TYfk*Kk
z9v%;t&#zmQb90kP&W#Dp?EGE<A}gT*C!g{D-rkDJ%D@c?j!w?bCubTj7jcE$jdkYk
z-QDGq76qUI1yH#D`uf^Cu;nH+<f>nMtx7vH!?9CH^~8L8`{(E9PhX&MrVrwe_W~j!
zN50>$_kVU~re(<sfuEnAPF@7PJ8ft6mluN0Z9IwZ@9jM^!?0OQH_9WhB@&djITTqE
zOr#=r6ezlOi>W%d@x<(@2y9v;0P&(@Mux`Ts;`HxU7NPI`g_opj6lsU0Z&NS+^qWb
zMRRY}*F}kk+rGTLJ)N1I&m%wtay?mby3x#tokgmx+~RI4LyA5=a=o=ZU%yrGk?a$b
zEdnP$IKO#yb@dD%wY>ZLe0_X)rt8PE9k_ULB9Ekz$HXJy58uD{zq_k+<CZNoJJ&}=
zMlQVkvY@o|>5q?(HIuB*&9OZE{eFFa-1_6^&+~_ehgTN2PS~K~a_Zsh*WR1c&TiPe
z`EZI+qS4HP;$meht66g_i-qLm^jwAFT@N;~2CfWwQL^j$GVQ5eUQ@NEzPWC%rPXDa
zd~8G6+o-xfAKh(iZ38t#J{{xw$Pad@&(^G~0jsZq4wn)ZcFxJsxp?tnL}cW}h0g4n
z6MkeBJw4^x$i#YYR$gvy@ABp8n_}~u4<2}Qv|BS^0mH<J6X)31&wF=ww>KnQG$&7<
zEd2EJ^!n^wK`W<h(Ac-@-QC^E$;qE@nN(N*PO;wf;`wv;kB^Q<?ys{2-2nLW)6*sX
z^W*M$fU2a4pX?{Refu_k_B7k-vQN*>Y9AGh4_zI$)<CLPzkb`6ElY}@pF5dn>p$Pl
zw^zy(G)M^AQFDJ^ZBgzb0U@zg4_hhItS`4p6R%ynR`UK{ESK^=tMv5rn3$Nje`|tP
zPP*FheXe!+iz_RIOJ7}C7`ZtObnPV@uha=?b#U<^9vvN>b#2YhPwA6Sy1c%&cBz3<
z<co`o*~`kx7*bA7YHecWuG^UU`fC;FQjtTg-0`#JWM$93xVTs|a1l%7=Cr4uo}T{l
z>MHkCuXm5ZsbuAq)yo%ki)cDc(}`T7p%j^MZH=VWTt02>)%tbG-n+}*TGae7aBkyK
z)Y5wO^sM&J&(EJ9>y@5kQ#r|VvfAEK$##K9k57i{DNb5>YioA0iB#a~t6$#S4CYdr
z#9Lim4VqwdaA3&2z3p$(>9^O{%irFfe>}k;!9?oiy}j1y=jJE|FZ22F^XJ~Osu+%s
zFExDBCVQSzaQSp)OJ;CQZEa^q$AOC%1G$v!ZhXtV{o?j&^;zccV*Z89b9F6#|1p{C
z>AA0w&+-kVPR`bszg_;8E#cFXlRk4SCf>MlWBU4yn>Hog+?4w1iKwETv1Fu(=DhxX
zeo)taZnN{M+h%j)`loui?u*$Oba&fJ-l|Ve&YkVt{5(bd9}gcZ`{!P>J~K(X%P%8#
zZ<%2EHa+@(MYG`gyVVR4JByx1JAQ0!Xkg&%2Hk@8=>1&#`gzxG8eM!odrnY~h>BPK
zf`1n$2X&m+=sz~=kJN0Do$m_Sl&^(_fsV-4*Vlja=+V6@UneIf?eKM=X}p@xXU#vg
zaf532@5Ng-smz@^RYZw(<E9IVxti~1R@g7T=&{(Xx1w<C6tAU+u3Zybe9@z(#%88*
zx?1$MoKGrZpj4^YV&KkMWy`?IC9)u5W73XYyMmT@2y%((c+9h@T%y|tN_hfKGmN(`
zRsG1x8N>n6;~^$;^rXoiY02&s#=VR@6J6SlFEo>Vc)l}x|DRVUUZyHeV$D3I+akbZ
zWWMl2n`cVt*5~i`Oq}!Xi){F|7!}`-cOq_|*t!%H2u>V{*EaFIY|t&dBlGy=V&&-!
z9e3@TH}9Xv-mCU9G&5lL6xX>gzfQ9h^R$l^JEbqL<+Q}9NGgg$@yy(2_P(2;Gv3KO
zy6E~hd%w|}h;vhKi#@nCRmoW@VOKf->NhtZTvODmvVSsrAMb{%8W)dBY`CIf=sM?#
z&-&&kv6uOKPdOa9wz6gI)+GmKDc-nq=k&_N%}ee%Eiw3{8o;4=CYDWIhw<;{-TD8|
zm9)+X&icfc+4JZ}k{-{SqbnauD^#<73adZ!_5!<u$oAvrkH74?tTCr+vROLA^m<O#
zDM#XYSsyw6p0Ocu^U`Y&e@+b2Tyy`k%QPF+4dISC5fzWlC;slZ(sb)$=Z@-&GfMb7
z_vE>hzYt>MVUQ?W|8%C9k!ePI>eaRgdkY_xPbHDifUi8{C%l$5bzYFghd%2G$p={?
zQcq0zu~}29wuduhTIc$@>#V-ZwD0il>URCN>q++X+3x#hFFEJ5BmmS+ygTP^q1+V?
zZ`FhgNmrU%qvrmwJ}j)X%wOT%#pm*GH)L5(`I0x&@a0#HI1`~Tt-u5adC6|AVl|<%
zzy0pfdNp5+WZ3?+)T(*+RH<^!`;c<YKV$MX---JbUs_)~c<iBG`EMVW{};cmasD>n
zT4;ZwNdKR8C;v^2nsKh6`gr9fGi|4NlFb(OAL?IBa$~&wWsdJUCqb8E4aqN0tq@~h
zuwA3!n5W~Ji^U1iOkXz6Flam^#C_m{#mD={*4=fRQ|+I#W?kV=kCrQu9U@+CVy>^I
zX^LN)DdO^L&4ZV&%GYjgO61~yaBx@i1V!^#J@L;3JZ|i0m{H+=<>s8uo10qLPm6yp
zI4Sk$WW%SArW>~YJ<p!}WX;ix+|3KmWj*=mKRJ%gmErPK<rkrR5jR%-``hWzYRq>x
zN@ZPqj>)ax+b`a$pYvn)pVE!%XSsjd9HLNnw?OZQ%+VQpwN%Z11?BvdimAKp*Z1f9
z&I|cWiI?|uUY@e-$D>w}C@YbtN8iR+hj!F#EncwUjPo|Rc~7Lmq*=pKBrj=aF8}iQ
z<JXELrfv0q#6=X3gsUreE0xwu7j05WJMOfIFWzD1I!1<Fx6?1Q$2~sy?avb35QcNH
z2bAg^bsoK2uwQhl$b>f=HuN6rFkV%{^~GF%&z$9(A~<)*w>=EJ5`X$+AOm-{?yu7i
z*FV=eFhx;ZY~w?TEma4Y4!`x5$dhEO*r#m$YQ^dsQ&;``)s<b8IP<pD;hpw|buW(1
z$`h#9XS2SY!DG1oymzW}`@c$?CH1LQZ80lkDs$(GO1?3)?Q)uFFMjOw!x;>>=P4Iz
zNxR?K)GM*Lxxzw6<KvaZix|!^TO|noFb>?bdMVe&*J?#+vMWAKIbgKb^qBSW3w9p2
znDy5@cY6JG1=n<kk56=VEqVF;wr+hYpEqZH@;v^wZ4Vl!gn`;UD&fcec2|7zD)9c3
z|0pwK?|XlnZL3>8e>oa4u|dnZ>dW(*$`W&Z$-CVPLz)gUc&Q$dk2|vW==YNb#VphK
z^%k877y7$RzH-CWhRahH-ni+Oq`*9F)^eM9clcvxK1^8?%JX>f@rX2!{+GMt`-S7;
z`9uF+OwlW!8@+gb62HFtpEcIL3dvJ*RPJ;ix0<bNxck`$lXYr0rNj^Ycwy3Cc6`FZ
zTvm}=zis{%3f%hi_{6J;o$bxl45zMdyT9tGgi&>hv$R~luO9ER8%JK2F)w>K<2c)7
zD_gC@rtkEv`-9E-uWa-BvXL!ho0pHje{uC;<4e<v-568m*-d+KTuO#3RXy)`zm5NG
z!Cu`NSzpwWmw87_)##jLv8ZPG=aki5KHcsKrRC=4$Jb7gHEg}uKDX|@$AW){GBqa%
zJTv>Cq$=?2%(iOloY3o)7U_29*Bwoc+@cuPIQ!zOgCf=QZpTJ_wutH8<&vsXv?J+t
zhDy;nlXp5c4F|7Dy`EQFc%q`;x|XeAdA|Q5rOT7+KJdTT_OJ0AgI1_T`LlJ8JwFs4
zXke0-ao1P=eJK9Ky+sCHKg{P<NAJ;|x&Q3(%O~&dF#lBjj_r<X@NMqdpS?S}SX}=w
zCzYRndUC161$%{W(*zH1w)hhMPvP6yZ2jVGYTfRN^L*c&{JXKXvPM^KO~WHw&v+x{
z^YxJj4$l=2yiwOY`82PRpUnXm5&!yE2hR4U2U>_7eRfS~-kkn}OZFdmd+>4Pt|W0Q
z2H7L}zvVZwb$pdP+AJBobNf>xt~D3t{x{BKo%iv_^Y|2XVJA?ES`xsOZu0X|!lhLk
z^RID)UWruV|L=Hbtp!7mxqbqRPn2*(WOCtaF{@@?NwGDm!8bXS>UQ@o+aSDbb-K{(
z))|XB?drwX6elEq7El)0UeC?BFuL<rv0y-wQeVf!*qv8b3qRC$Hjs#mSG{U(X)v)t
z)9z#X+xP7Ti;gVYAbg(rutIfe4#U*Rx?H}TZ$76h`R?6wB<Evg_UDjwt_Mq+XJ~gW
z=l1n<cAa*#h&ds4sUlw`ughF3w<|xZU%Yc|{-;>Dy+^`HFVQEfaZ!L~{J)I`w!6Ci
zJ+->MfI(zh=i9I^(KQ|JoyoVwG`y}J(|V+`(eRR%D~Do`l_jt4nHjudUIunnXC}|w
zJ(($m@rtXg@c;OquO2Qup2G9!pGCo&i*oIkd{5Y2>^u=6-1X*XAQPKI`D4kDO~+j>
zeeHUGYQ?qA{f2MnpPc)*LFsw)dc&jpe@x#s{qu#~Tiy2#PcC^}&$dqRw~2Y?@1El;
z_8*vAtS@v&`(^NQ_EzC(Qm#p}Coj1l8L{d>N%wK%<@@%?U0Rj)^oa0{Q{Tg1|J~?6
z`%9Ct4d)K;Y5A`wZtb2oqjYhKxVNW_*Mm1#n@d(H=>6OA&3^B_b>C}>((R7jeA@pb
zEr9*flE6jPMg11cE@$5QA1Q6V65S!<;wIL5(?CUqZ|%&I^OFvp<xR>vzF6xTqdLF1
z$hoNnaRws2bIlATBqznqH<^;Ab)m5**XTn^3jaG68<&Zu;=6A9AGG4%_EU&o)pzM1
zfx=lgj`thBa-3=RI6hH3Gv&O*$Fw(H-ZAsuN&d<<XViWpJ=Ob9=sueW0rxW>?GNtg
ze?C2F!<nhQ7V`^j><XB<+1Z8o-quL|(phs=zj>~qq8w{xVMd(l$Jq<rcq5XlzOb}D
zlHSyl(cXGUa(82Kf!=J%#cNF^zI47`w83fOmUUfA&g4v7$Po53CqN^9rQ(7OrN@-&
zIdvLZ;%hw(A{K6{Jo1_Od&+WeO{3}w)pNunV>d6}w(VE``tX#aJ7f61n7^N#9KnBS
zmS5qvSnhSQH9oiA?A&m6V%RHPP+wq@yz5H7l*}L&hTFfCJ+3daVYs-u=3>^=C7G5}
zV(hCE3;%>K+H8K%jaO`^_T*)r=9ivaZO}6JTWT61mgPB3P}%eOw6-4$=DM7hKL5Eu
z_@$e!(&Z(r7O6W9?wv1|V4uLVEo;N73!KZ3Jq_6Yx3PrPRAa-z%s(DiX6?Urw(dYd
z&eVk9dX8_$dhdMR-BJ2<U&`Ds#y5A1sTUnCSv2|GyuMq-T#LU6o#&G__#@ac@3_s~
z88>EMOnkvJr)X8cicgbLFBRxCKNl?26uSM&_T<(_<}#aC9(d3aJ1f|RyZ^|h1?-mf
zGkt%_nD5vh>Um7cQfT|;<mgNMYZ{rfWqy|_{W~A65yE=HYOi@_RDC~hLhN$>GFKgs
z4u;vcYENF<n3QzFgiWy0yZhf2yUeMRCf)h`_}2kNo9j(Sch4&NKl{4)y8gvhjqTCz
z7nTJiTspS!u*;hB6?*d39kGo}LHUY7x6)UfOqtZ(<<e!Ac!))AO_baI{G!?~Hd8FQ
zobJrMe(T|$@6uuanj&oWw)~qa?Ij{vw6JC|@7a25qq%`kiw{-&e0$B3i+4s+oR0a%
zt7q?@({k3+shD&_<EF^pz}GnwmBsJy)bH-?Kg5?>#@i;owpHX@O2h|cPUEVNe@@*u
z|J!-#^^b2SGL@2(Q`P_Rxba9In_Rm8l4A5R{yWb<O)US)^Cs^{LDm!fguu2t-1Q1`
zCr^6uU~UHY)uW!pvp@XreRA?-$M*R7NAeEni)aWP^E`huW%hQd^mNu;yVW^o+w`sY
z{VgJ`r|M?e^Y=$2drWL9J1y3R-Sbi0XCxUZ!ensg&*$bQrG}56=C5aUWw^ZfgTJ^4
zqYwYZ_`33b*Wfd&Pyc;hYG%vje`i~G)1&>d>5q3eDK%(X8EV|Ru6Ua(zes0Z@7c-9
z|0P6Rxe|CDR;`yj^5WpOfXp``4`cGa*Cqwj<UcsmwK>`<$?4yLjH+{I1uu7Y7$nR*
z+R9XHK6$zOPkyd0a53L;MeUc>#_dW5cD85#p1c%p_?6{YbIiY|9AbVi4d;DSeKmjH
z|H&m>zb?*O+-dJFr?SR9S#J%uQ@_x1skoZU>zb~Ff+`D+&B?o5j@;OHN6xIZMMBd(
zYtgx9JPoBwzggV)|JS`OcI_vD+P3A_c_Yjuy!ZX<W$5@i^VCYsB{DlCVQT^ER2VY5
zM5iv|0$-Qza*F#;=U&H)um8>Z=F9(SXLMoa53WZZ{`U%N@Bja`B>2KccfKT+l7)#k
zLf<&JO<!{RrT@J{EGjnB+W&J-yyP+8Rk6jOoG<i*-40Q{tbk{09N+!Z%FFFn`G4e>
z#`|BwrDvzlIH7mrS;6ubEV9p+&96QFI@Dq%@0}a928Hts1yvGUuQ_HH?)fe8`p(Tj
zx46frUcC*<hjuCgmH3O&4@&0Ut!G=}n9Bca#q*eqx4%EBn(s1ua7_FElegCozck!^
zVQ<C`*_X$U>g`YNDROulB+&iC`bg{^-K}EaF{6$tCo2BCF8=!b#bUd6Uw3t%kFWQg
z!dcPx?Ma~K1_8|(F`L07ix4XXz!Alv*dp*qcv7~Q>#6_8k`oW!y8XS=VCla1c5age
z_nvH-+G=z)H@YXzJtSt%n*&au8|XXEuTDQdFXhC9#w$N&eC8I{+mL=<?$_7X)3xsJ
zuKW4vB;TKz5mAwwUVU@^XQE|y`VX(0vejb2?5odzN_ZMbbozTtFc0cTaY*#)oxj0p
ziO}xy_i4YrynJ(SZ*-U9zo{!$XfQD`+1>u*KgZ(Ww%pqi=6QG0qutKUvD6G))Us!$
zQ%$PTCXE(>l@s>vwdIvI`;+r|`=(8gPE1suUn=R;@x(E&K+p-~DIu{|0T81FG@1x@
zI7k%I4^(V{^r<-%K}9iWuoav<S_E)PtAK}V;UNYNZ4Sj2=uj<sfZ~$=ym*oj$HYH2
z)!%p~pL7Wc5wV)vx7>d|Q$lujchuTxOq>uCci!4ns@*GXe(cmKE{4e`U6%RG^ziUd
zD1UdSrM1;_BE*@8Vy0?`^KFgl4O?CNHz9Oo#;q+YMLakZkBF;Y*WsFYCgseGiIY^l
z_nF>3b*js6u9asaRNtMiFD^1$TkpQUEcL{M#zn5(UWG6j)9h<JbN#-5O5YmgdwpH(
zQUfQ*V8elf2OG6QSMB)|xiH|uzS?Td9!TQpaIvlWa^T)QxdxlcPZPAm^)$O+5^{2K
zFTPe?$uhOA`ZA#j8dN{Jw{A6k_wHTB)~KkhS*ompP8=UYLD}k`TSbLM?yW6~CMF`S
zP9Kj>&o_|@)DTHIKkx5q)@zxUmz_MZOu%V|Ve82hBTvbeird%LMt}VBCFQ~b$BL?|
zS*g6BQRFKtg;l+#B%GY2nqeY!)9_3LXfs6AR;AoqTaFw%_UX+{<4q|iokBuHKvS0w
z548rVC<ZA?Mv6S*wVK<Pc)0E1<Hv`O^-62T?%Hzm+M<Gohh!`Y8XTM19zJ_^Y?<%u
zCD#{Q)+ek8iP>NGcWWlUoDIVS=?yI&N1x1cRg5eA{LEJ?eBGI)-qS5gUQCE&KPsM+
zlM}Hafw7sL|Jm*Q{i#R0L^my-p&1??zA5eOuWKdM)zxhLayqO+pv1Cr_Wb$spb`1R
z!)-Y?HZZ!oyKA1(zO^Ou@VeODGwf=uWUb3o%*?{NlG?1w-}OwLdNulT_LUWiM+L#U
zC&#T%e{o^qmseN2K}(4K{rz1(b#`P>Qxhw9(T4|)JZzxl3(p=M?G6$#zP!+xeTvu9
zczGo;u`M|_KW)tfr|KnA<?rq+%(}X2j$Q4n2M-cXP1RO+{mH&<+curZO)Yk{zd*}I
z>;8TX*X)u!IbHvLcKP$$w{Bfp>MgF>1IkPuwSWHpEi5ixT=w=>&b>V|OI}{`4E)#1
zCu{ZL_xt_EnU_>{mA&<fiJ2pE)Tg4VDk(RYS7lqcR{Xv_BCepKeu*4gbEACSkH);a
zyEayR%>sqZ^11dw6O~*!cz9IQ)Y^<vPZj+9l&aY!nQ?g;ui9kJsa~$_e6mk|K*Heo
z-s<lw;`Ul8Dk`$Dvj=ZXa@9Ol51Oy;64eHcv4R%#zrD3JXiA8l_cR^Q;>*2p{>S@d
zmnt|dDY`TzdPcAPzaP!t-rl~vHrl+6SGujWwI=-670?p+-DPhJ%FENw&9Pi@HB0kU
zx{$Q==_e;ATb94O6CU|*srU396(5y)rOp3oo!-82W97~N!QgW8|B(z6qr5v7)AZxd
zZAv}8B5La@k*CI8U0sJxpY8@tp=MrI0}VpE{yefR|GwY*dwW3_Me|CT{K$GOC@b4L
zckbQr$ZvOdm;e0ve17J|MXZz6d=DKw=vc-BDsMi%2hB=eTN|BhA_barlQd47vZzMU
zd%9lgqa&SX=G)JorW@_$XxA9MJ@4Y~@_fCxJtyAW+<f+5x4(-E%VPKbwD0fko|$7A
z{LNU=PS}!PJ!ox114B$~tY-K+8SAn)6Bhhvt*olrlzLjMjbFZR`gCzlPR^hy{Y$RD
zo@rNWrKhJSq8~SB!UO?HMfN#!=782ptqxzGcBqB(%F5vO4I2z@CW4B`BlnG^dO<hc
zndRQvQ1LOztyfC*x_X_eo}L~HJA3i>cd@6Y>;J!2+<MgBy66c<US8gh-@iB4|F3iF
zk@$E{eZ`E;MNdx^zPO+WS|u^lD0PNK;i4K#aFoCQ23o3hW#!>{w$&V5T#Kr{z6zSM
zUdMNq$(qQ`ZDzT*0=DHu*3{N6)sTPi^78VM_xI*5^PO!}@W5eP?rpbuHkC^Z-Y*PV
zDPfSnaPZ(k(0V>0Rj+~v2N)qK`iFNyf`Xo&-i{qRKuLJAWpSFOm(eBAeaJO65*7st
ze}8>-UVhm#@E+*WR#1X!bvpR*@$t-yi&{mVGN+!OH<wr1%pmWM#jWl6{_T9S6<=OV
z6mjF=>^?MSrjJ_Pub1jkTQUy*`T1GWx@?Wja<iF5#l^}N7BgO7Umv_ONHy=y4n^0>
zqfbvyUm3U8$~f&z!sBDTXXaQ=Hp#r?k!a`C%*Ja`_Xo7pN<=fLMb&$n!uMh>#l*lF
z3)TJS1+0&=EqimrF(5#|H2d0=MJmOamzSm9-BoImduxhO>ZuL2zsokS_pGV0S?u1w
z%&C>@%G&7Zkfn-R)~D;2T+EnZQ)zU2TW<2tPftTug}hw-bt!npA^zY(=k|ia!i50~
zoqqiOJz3RTOw)_&(sF<KYwP3v-{0FSX;WcPwfCORYf!VGOHB9F&(F^-tG;N&#l;yJ
z{p%FfkDF8b``g7;q1vm$*Pol9=-koUJM|F9$Hpb+7kL!=s0m+Q?w|ha%S)Yz4Gp_W
zUwZ`xZRqIW*j4`i+~)N2pp{rRH>DbPzui~=-|p}4@9HWlE&u<%ufMV`HoB}$|G~q<
z?L88PO^4h0jq~r>yt=+#-&OFDJZN=G;5@rpufuJ;A3uInG%yHgO454p<jIj^z0w~)
zd{_{&GU&~noyMX0>n~=c+}l$rq8--r=H}+&H#ZD9yW%Ho-fVn&x_<h>CRWf=!DGGB
z#jmgFf>ViWR>7ASfiW>LC+1q4?=E|5G+R)kctL;$C{cm#2e1DAjz`{3rupE6SKB^6
zIXQWXc6eXu>uV3+zd!%u<KvARH%^p&puHk&t(2moqKIykh`6};GXME`+on5leC)4Z
z{M2NN2PbGZ&6AUppKh5bS@GwG;oV)O%6fWzUteEe-jyA=c+nyW(=3tK*VZN<YT?|H
ze?RWti<Bp~wq}D?G#~Gm-<)z%=+rb_ZCAlZqDGrECT&eWKkwoaPvL88B0=kNRWxQb
zetCKM@$1*#y;7zH#l@h#U7A(ZX1TXq7CN<7e0w9gaN$BBk;x15_x)^pdwaWa!2^d`
zCYetCa<&!U-pmwn>rgpW`1n}wWTTEJ8YU)Fq)f9OJbH9!Rp@Gu>3Vm?qrc64d^=*t
z#rcoj&o4e1Kc~lRef|1f#U<L>wcm=r95B@RB%-`}@2j^nCFTk3cp3iBJdJ(lt8mrk
zsgKs*&pa&oEU5Ll>8;6MPa0mU`hRH8TK^S+iv{L?bIgnC%ZWHJ>%inW+<edal&o1d
zTQacu{a%~&_gr(!ot8<in-&_m6jeRfGK;wqur1+N_%p%7eIK27&e$x(D!czo@-q=G
zR#EM+BWKU{>c#F75EOL0x+*mIO0x9&`2Bvjx8;8P`Ze|Aqobg?w&>_+%MFt^D!F!D
z*qH3j#KiRC_ICZ5MyZP`K0XSPNc@(P5~$)`{QTU~xV=@N*{heAm)Czxf0$O9?a`gT
zC}LNMny}ftn#bGwBktz%cuzm4?At3g<z&*|dFzw<o}5{ee1B@W$-J<Yg>S#vR$huq
zp1xUNr_XcFy}vZ)r>=hLbD?s}3eD$%(|7#jju2R}u;}I*{jffvsa=~&eu`#QO>tej
zxA*;}_!*`-dg+Vr@5p=oGj-Z;8}<F0pLM@ypQ;tAq@mGqxSjv;yLadQ{{BAs@4nTS
zU;cQxe7;8PE|aIHr$7Jj@NmeA07t%8B1*oqOwP<Q?f&}ux^eBV63~*)nxC6Q+`vme
zI(mARxOR)JiP+fG(7+&Np7-bKHC69@`3yTO>P6L+{_ooI{pZX3AI&5WERss)-e-4f
zWpAg7k<qiK=g*y+$;hiEnxULrQF*$}$oT%(*Pfi|qF!5tBaPE*lV7qaDJx%;ikrX8
zlz~lmqTwy`O^Ii+Pw(BCub2B<PKnK7a(^86uH-+(VvW}q$M1V~@FVjS`D>G%q*zyM
zzPMz$IPa;5kAHeicNYICKV-X4dB5Ao(0lCZ>FL|@?s_ftnp#j=ntFd<?a8wLygWQh
zTsnnV*w}<5Bu*@FY~HYC%M@m7R;|!gB8`oWSJuVO<`UI1NIN63C2B3$*N?KTDnF^L
zTer@_%4$>oeY>LP=l+IXcVB(gpx{h_mo)!}*Xure1}EPzdu}28yoVtvuCi3*cN+hL
z-}2!bJ}mDO$v2zXJyGoo!?C|6h1zY=t(&H3+?)M(`y$J|shqxDV#!};rt3fd?et}>
zbI*J0Nv+22kDpzdX?uNfWkHQ}?W|po!WL=7Mon4h6lwnK-G1H!s{2*H$XZYL{HqnX
z>HhjVyF|~N|66x-iotBf%h!s}Gf1AkzI^AQrjM(eCoyG<SU;0p7JOG}eU7t@)q}6?
zr!y^I?VYLT>-Ev}wo7zG#D~ND_8UOA9B<7&zN7H5fmH9wwEMN6o^Z~!E??$5+pI^<
zR!Ux8zUbK*P4-f*<sn+2^?vSsG7sO}+zi^mW&iJoGnX=GiI*wpdh)2PS%zhABGS&z
zYK>a^@0WI}*MWU6wwJtImiT^2Z;SMSl4AMjVzcMB_wcRX`{mp|%P-NPlPrUO8Rwj;
zs?&Ss{kn0V$^;9gpNDTLoSS>^a6?f03;R0uov*Jg?mEPDZ`T#WwzG2gA79yIz4!N*
zWvgGZIIF!?TN`bvwKQ~L(fk94&MuF%U!Zil*7<^+rORf)_Ln9b#F^RqUG|qtz0RvU
z+`wrXC?<8ZY2p2ek(?`vE><}?3z_LX^Jw9`JooRJ(kySy<@FM}HFb?w7am<1s1&>D
zhX1P2)ouX+0*s7|S5}2~xAV)_-HknxlJxV_Q&1(}>*k!1q0uX4I*Hk8>d8r}+@Q5@
zudb}DsH(cOt2A5Arb3}rP)MxxhtQ(Mi!-mT66F@x121^ZzNRC&b}rxCdyQVc#}hC8
z-tf0T!f}7@;?U<_8DG}N#jkjJ=bf&I)DyQyp|xIKyUGs~yy4hh(%@<u!?lDlHSJaT
zyQYUXIOQIAJga#xvATP|XT!w<mp@#7V_!2<>HqJUGxvHe^pm=-9lO#X%ORD0ne|1R
zUGpp+Rep^SSQsASGuNuwYUkSvsxInlZ^m<7$#KoR9BTGY@YKcJ+b?ge=K3#kQeNi&
z+z+qrPmEvdVfNl@na|7*pFc15nQ63R$Bu#r2O8IJk`vd9S>QEQODAeehi>#X&^q)t
zZ{I#$;CJ-Zm6f1CiQJsFG~?nTornzzk-Y_=N%19C>F4Ggyn0o%*X{7XzrRn;HqYM_
zWv@DIMhTmUrIzuoU8ddDW=qbCmcL5d?y#Quf=VyXstsqCJSjTxbh`TF%PD7h93EZG
zGMrFy-b3p7nS_ZmP2PKFR@rKN3fk|PY-3}yB6PLb{b$=$9=X2qu6X)PxK-3@uUP9U
zsa_kK4;gnP`1_+$uDC6#-pRXugLc>S@XCKPE=Zm~x-;|Lw14ti3si4Z8|Ow%SJ`R1
z^2VV<O@WKuHfCH@0&V&{(kc8iNH_WMqel{UH5Q?(!<JTle#XMW60pdH({jPtmfl`o
zVKtuv_wUa)&At|}ucp#(u9Yez+OJI0i|yLI+gjPJC*j?loe|N|>bpy8>g?x8G|OEG
zSne#(@ay}kMz(;+sK!G3?ekUKHrcdFE#SZD9&xGH;5w68;>|U7mLi@TbzhczURwL#
z`R-<+a}I|>mBUsYxss8YI{k-{Z&jDD+w*;=cgikS|MiUV7vERs^<k=P{@Pcw*OwQa
zYt($0zAVqCRZMq!nNRw)g&ZQD%=4piRrr<vbInkd-o|>MHig+}|JI||*6TLEI1+rJ
zv@6YL+wY0HcUwC;I{x_mJ9xRDtekz_pUCOof4|=!yrUrT(4j*doSX+QU;f;^{qxS^
z8tLHl^J(ASJe|{9@w-#Gye}{H+Z)$2^L9;`u>0ftd7Te68>PiAZ@+1wcWjT~rmYWo
zw8Q53PG4MEXE{T3v-*Y7BhRj{yYSa!i_A@f_LbdY8#BJ;Y?IsjIq9jz+wGHXZcaB&
zKRofs#X@J5<$tFcq`#h@e@3}Y_VMq{U#CiM-#5jnm1|@5_dKiecRp`#ZGCxvzx~CF
z7r`Aa{{`;-a-d4gw))$LyXE&)PnXsd)JQL8Tbty!y)ei2UxUfgR~OIT?Y@=w@}j-X
z!7q`k86r$d&C->>q^z}_(fU^>Gq_ShTg*4%%*<Uscvm;YZkx2_boHcb&dWAsWZ6G+
z>rE){zclAt>;aR-^Y&_AFZ%I8ukP+P{=-XSbF32`rQge+EG{uCJvD7f4YR#XdWmag
zmH2IEA&!stPhRA9zq!qp;qcPfH~r;_4-dHINZjY!z9?^tUEq|7x!sf2R+#dLs|Rg;
zQ!{Umk=}|^QGN2Or!TI%+LYhY?*8WXhMtCGt3KAeMDGKY>)QXE`)cH?u_jRE^<T~6
zXJ_8r**W>q(eB5O9vxchJ>9O>VRv!+GT+&kw&%w`J2&^V{{BBs$9m$`UA&{0hwr~w
zyLX%4gd6-@oZI<Y+S-H!1sxYUwSM^caiaYvrK=N<|LxR2ckEbOx48bJH*e;=+IDt9
zuc8|_AK&`=ip*rCi;IpqF8+5xQc!rY>(d7fzm%u1?(JmkW&6s$emCRNsFU4dwpq0{
z6C)lk2wOJeSNWS86K9!bU)o#!-T!l)6wjHN#_gVy)h;e}=V#-Wo1>GruWSDo>7}!L
znYy$7NSTO-w;yr%|L2RA*0#TrI;%E*{BY;UHjnl1n%SP7u2uOriGOJn%Y*71J<BZ{
zSIC&GkIhR^_-NGT{hIw!QLrRmZqC#a-*t^APDoslV*hu`lqZAN>q<sv&-1T;W2Bv*
zX#ak8w)XNzi8*b|>NbT;axCW#@4c?1mTT{{_)e$tGyZSB=~wnz>EuR#dHTn^T=THi
z_t2aQOWt>HX1n~g&1>;nH(5t-DhD4QXe-Q<lat$jette#J6!K)vwje(WTZ&^oS>CQ
zjvZ4g&6^$dIV^xBAX~=HMnml!$60o^qu+JI|LQj-J+hgi?_PGrOLc{*rj`cN{4Z>i
zlaKcWt_soImV5iyy}i|vmPIL_c6iHYzMc}&nsV{l%TQ76uqP)bDo@dko))y!%dqlO
zN{Q9nKNrP8gPTgGmH+-&!WIv{yv%NIZx7m@wkIs;mX+%6ZGQR@QBg(}9}<|DnJo(*
zFsOP@Q_;}y;L`z*bp|oZ%gb|#YPI++e+=1Mq&0Kh$}Oc{eYtbvrW&XFndRR*b7G=0
zs0N#BU9P63<;7<-QEBB87AKC0D`Iz-fd+Y2g|1e6{k3NE+ROd6(u=gryB@p|d!c{O
z`nJ_p-L5wA+_`!C?p^czyJ+p?cg?q~cJA9DX7uRZ{Vk<kH8-!lJgOrfxBv9v>9;?0
zE!pNbSFI#%{VLtv)wK!H%jIG{--&yf<lT`dFE4*}eSLp_KmS~-(n-v|ODv2wX&gxx
zsyA<O4Nl*jaZxF4|23&6%6l(edUCjS?=>aOPyCBoS@ZJpPS)8jT^sx<|MeHG>gwt)
z5zVH!_0yYumUw^-NSeEpNB33C;;h@AUt+gTcgy?#vH9)PX=)5EZN<+jtGd#+|DGK(
z@kZ2><p-rIor5~XbLaL&woER3njtlHmAm=(8|B+dPnm2pKbpm#_VDe)Eh}w$827&U
zuscG0e`e^0?)@IS_XKC!rfAu2zNZ{w*mT#~yZF}k^ktG?r^N1@c87QK#_Zd6JkynK
zls$^RRq_AC!EMSbgUiIYQf;rX`MFkBS>JXRvRHWI<VB_PmU}?O2FJJK{s|8cxP6hj
z-^cs-;Hsx3OvgoB>vu`DtrT7v_039}kMFswe~WnT+&A~HUOSmyoD{9TTQ|q^euL1m
zDHpGKc3g3adfuUb?%cV)?Ca}h9$$R%(f=j>FQ-k3m-m=`DD&RWN#@bB_RNu!?EEHM
zuA;upr1Q}B_uFk=z3~6OM17+~l2T<}z~d($u2&v;axLmf@$0+0&7Yo{`uoK-OR3(r
z*Vor4r>9SUm8G0Jci+1A`~T1LnyPhiZM6AM0Rc_W(%O$RUH5<g9U<~Ntv~VX-Z@Xy
zJ50mxMC>oJXq6GN*p$Sy|L&=V<R4n+ZiTC@UM5f)s~cfdC)az8E$H)4H@ElQ#~*#G
zFkx;FuuPO@{vL7J?0x06V_Q;^?|<5pqkCy{dnAkN$L~|Jc=_IYq-XUW-k-H`%8hHD
zf9FqIeJ$hP(_Q;-UE>VMDwwc61U$}IrXMv+#`2;dZ>@&qI-9lZVh>(ic~dgGRQ$g2
z<P*P_C>>o@w(RvY`IS?RyX@6<7(`Rn-f0Z{DV{XPsMqXg@1Lj|N!DjSE-$%N?&ANu
z+2erX%B?5-Pp;Y8A>h<8#Wk-$@X_judhf{_PCQSrId+&sm}@!D6gS6NTcoDE3Xh)k
z;vyqwmF|Lrxl7oitFJG({l<R9#r2$gD*vx3L5JD{l9pFkuQ_^u?;&Svq3BsEzTP>R
zOlFb$QWLGMtHR`ucUg9OUQQQNog4aouD!#=XZ8;^zhIxIBkR0&)8w7G9$C7Vb}gCH
zBnF=F=y-DK%O5>PHRTomx^miP6o_4$+vt3-?R`$>B=4^iil?78NOlf?wMF94|Ee9I
zA91Y@kDkS#a^&cu-aRKz1-?EJvyyk;!EVEs*FStY`BUU-luJ$IrE8uXAM;O`c01Lm
zwRLo#nQJ26;CuLYfl9qwPG{=zzS>USDYMNFzWjMx*ViCcRYj%Yr|<W@bHaNCwO{7!
z{-5eCJY8|^iKt_f_Q~Gmd}+)4eMT$Kj4ao;U&J+U7a5s7FZ0S4{1^wa1+=0v#o=X0
z$G>8;i~oCH1~2>b_5ak}=bnV@K5|Vdc2ZRLotxWRO`5|Bc5T-FX!vmjPx(yC4-7Y_
z^FEB3tf>D#cj3c!i(BsfZj0HnE*h<L_4H=v%~)N$VbRt(O=9n-u4~}FucfnL^TxXO
z*>*eDP6qc(I~f}^YTQkq2fF{Cp7?QY?%clremjnrKbfv?0-hN9<Gi9~?beQIi|_5<
zc>mtC!|5g#s~=g^U*PY)#ameQJATncjr+FpTepY_{`a|M#R(dvo%m=)rd7Fi*lY96
z<!{$Nn9(ogTXjKYe@wN;KAFtdDlMQ@*G0=ty8YFwU8z4+ApU6gHb3q7{MT20b0kjW
z)4y%?w0iB@$z5lDPC3G?f6GepOO%3cp0e$XYrEO_ZR(%6XTDZ(=~5|o=H6WX{ZQSq
zFK@l3>(#!zU2dVhEhpjP99fsTL#cNxcXitN&c3wfw81vHy+5zL=bae-A=adEowC{g
z$3<HO!z1jgS;dq89iD8(J8{mp=tFbX*4MqTfAFr=)~D7zR*psQX}jt36nC$}C2J=O
zfF>ONOEmQ9$UCf^w)f<{G9B;mGl|ch)LG1%b?#8J;QTkuI=}wpUwK*CUG}&w@VsTu
z#Msv#e}DaP-9AjVXoB58n}y-R|H^iLEO(CBSt4hz$M5ogqxKS{aSZzuy(p*ce@;!B
z-~Q?L%opa@{JlS%a<zEotG_##OF4ILi-5eBpELL7uYdlq?K*qMx&LVWg>5;@j&#mC
zV<NeIs`TGF^^Z3--tSbtlEAQ3$k~VI<25Oh`pF-h{_QxrYJuX8ERiX?W|O1qzE((x
z^GEFe(V`Z#{*Y$zwfdO!*E*>$EU$Df&Q{y2{5z|&lRdll*ZbymR?@yzRf$}YM|bhB
znldY>*E)F11XH`UTk}N>U*6i>`sv{OMf=n|qt(IFiT@&hO#GV=*qfz#pT}zEN159!
z*-AIftKUSIw;aov?Xg_*^~(E;?5xe+F?46G`H?IpcDwyT<}*2|zOT=gi>X@fuTxpO
zEG6~%#f2%Gy=HB7@|F7cYeG#?mGC@Srxo}0ZH?33xIK+~Ha|S(_qU7(XBWi3us5Ew
zOis*a=GO_L^=B6Ke7-Mx@8gRGll4BFw@s(FHH!srIGo}(eV5Dax`RGZn>*Z&3cvom
z&hP}g{;l~Zwx(-#Z7CH3CGSV_t{>A2G{k;ZPkbq>EF-0^{_f4}FYm0K{$HPQDcCGv
z!#{>)J+jA|lCJJ3u%G1@TfgA-np&GnH`L`~xTM|IWd08iV0onf_aIj&|2I)T?u*Y~
zTr0Ct=9{~)r@3j<wCfq0-zu9u{b^{kM{mR5BPK<f^FL*znd;f}E$b=1Rc|`mX6x?_
z|8AB><#QahcRIZ#cYmP%_OJJshBYny>-marDyU3ojM=PlW}0qf*Yl$lKSlbtE-PnS
z_<wWx{foX=|H~hrpBWY8p?PA8q5@Zx#%v}gBW-5~4HlPQD`0B@0tB9%p60t%Bgi3S
zTFycR?`=9WQ%?%b+;nTxa*>IV^$ofKQW>|F{QP^-D7CYv@A_kz^9N_oo?U%^r)T`n
zJJt8iA9h^``W`1JwW)ivMRRcGsj4*QOA}T$7=BC-50mQ6Idp2DcE=N=>}%=I_By?M
zuXCwv*1~<=Gq$W_j%<8kTx~J)SpW9B!iQEB&su(aQg-6+Lam5NYup;bKiHQvTx&J5
zySj<X{OwuwKG8oZE1b@{v$^KUH+|Dv^G$uFvs3hP$D=uatlqmnTYtxYTbAt4L)&+F
zDNj0-nGzVJsa$E><@Be<Z_dUCffwG&%lDS7ml9sb@%MzV@%D8bw$;C#lQ%P}b?kl}
zS*c~Rq-(9_;r3Xi;=|uk_WnJQx~%k1A6K?`km79P1l4<%yhr9Mo#K1v*06l{+sK}?
zUVl#Rxcxk$Y3>PcF{9M#u8*@{?6(w5Q-62lxz6AF)<48feYg;rzuo@g&h(Y-x2!ss
zcu7T03DNeS+MkhV@F{h2VT<csOXc%bF9q&zk#0OayHviy^~LM7=`9v7=Ll%UHr$Bz
zFaNwCbh^`;S;cdrrhZoU+v7JOKKI{>@Ah3E#h;&(O^rGraqigc!W+{*c)ws}Sm)CA
zxWGj?wf>I3O?pAhUfCeyl+)eHc}&XB->F>6dU&8|y@Y(#Q$y~;qpS*N@9r00yGF{S
z>(pv<!|VH*m}mJv_!Ij&X}?_OW$9%{Hmv!kuHSUa%J!o0m1~|ZiGfcn4WB9}Trb*}
zu|a(Pr=yv##T^>rm1ljw=df6Ce(<_Hf3><tPM^!G4Qnqk?6ZoPJ-<5i(Sb_}PLG^t
z*pw@)J6+%O&PC=iv-s!mqzzhe&z0}3yZ-ju)p@(s>fHAC8m{@ViT7&Om1Sjb)!t|Y
zvHSO1op}>JKk#i<(2?aDZ;#6#dmwi#|F^!~pYusi4Ac65%@c5!c<k=?JYos&^*>){
zSbuO1<M}zmYw0PYcUivsD(v4Y|JxRq_W01s9Ij5~l-owra;=&~Ip*G-eJ{H>@X^*Y
zPxNA>+H>_vtN)ycFYB<_)p+so@}&Q@vXjO0l#O1o#J4^P{5|=(TkhN|*F05T8d%8i
zwVzVZpEP^6bp4-?>@P1b_qVE_wN^|=OxrA7c;=n6|7*^i+Pk(`?sNOSEb}BIyPw&|
zA9UShNqBPJcK6N%<-{lNw^<aN^Sl}KQ~qQ5<u2CrGiu7;R$Ok8v*FkKuB&=(s?&QB
z<>s?}7gKy<!VM(<=l<D{+_@=nU-4JxNf*{eo1dDh?cUBOJMY!cw9PZ^|Njv-H{U*e
zm+bYvGs~SWA1s-(r}3R=+}_s{_e*>}nH9yhO<DN=-nEyH9e9|MGylRh&f}H)G=JIs
z6wBFLd@egFdU^5bs>?Nnl4Ux-K3r7yw(z^O>_$S>n>l_Hcl-NST-xT>`TN?XZK*$2
z{?ZYvd-BueYuC?OySTGS!uoEzLyy${*ke5{-+9^Ni7e}`y}z`STU0x2N$u}%zmyeD
z_PL3<dhCfZ4qjHza`blp&i}JNbI!NXxEHnWX3CRmvieDj&lM@@^lYg9XMOPIiXHCy
zCsMp(_>Q=`3za=OQhS=?@sZBrI~y#8njft#t({e<l{|^NOY`L8TUMzGUA>L}N+d1j
zcHZB%C}XjrOm~rhRl35wqg{d*>>H9&PX}#tF>lhp$uajC==6(PvA66GPBiTk(KKvq
zn!DeFqwB+gZGODo-_k$cahSyy)35kkc+C&zTUMqyMlB0i4{L;(B-Y9a|CPIO?d0Xl
zlaCz^FG_gqVblF%!=DFp1uM3e3RUhaY3%k8`WzM-fA3fS>4LM7lf-hpSIsy%>6X=9
z|J=DQaeBQvT^%Z#S+Bn+WW82dv_V5jsq>-!!nK#v{N`-5dGK-fEh|+&&y3ec_U6v5
z2#8)T$s_+wwvad5#xZ*N=knN@|Ko32xfVTFn^>l#Z~JuXo$a@**8Z8Z)P22eQP|c}
zp}+_cPqTCHBF_fc+l$#g=#O@Jq0=QIe<<_y=iSGA);dp8-&*RFJJ)5Bz}(FmpKSiW
z*dNPa_Vw7Mm-8DhOxeru)_T9!vgc(?bxE%$^nLx}refZiJGZF!MW_GIM^jdYP0wi&
zOj~)vT-k29hnlspKHG&qcjJDPmpy)-y!B^gg({zpY~k*;!P6IATw^VG{N911SAH+F
zwe0E4oqKNSZ}(_*vuzKrX?DK<@~^W@PVUT`?X$1F&H5C}s4kQEQR4KIr`!LEf7Jc@
zpv?2ziR3f0jeeFzFF(I~`yu6e+h=?>2i*VKr5zQ}xL(Wjc(&cLwU;L?<;<OXXWB*`
zZ`-VJ-7^c1Z1a2Fo~Gve@RCJN<CV7)yG`$JRXO`!TEe_e>)ZWq<H|(6s4J&Vgez9g
zu=mTI>oQMt>$!>IdYaqLRo3bJx$+_QQG4Ff$~b1RnEd#SbADVGv_5uQSz+1<k2lYF
zcHB4ZS2YqkBl)g!o~ahs>We+Qe+qt9-FQt=-^nX;Zn1w|Q1tTp=#45juX(O5R<XQ%
zV$q4u|6HeCJg{M5U5&I1kHhLj`-_{y%9{4c`|QuWK4X8em21v%$y?97c}%qm>;9N{
z`TcD*OT8?3<^KF)Io^){E9E8Ga~FM$HjgwY(m8(e#J`)x8kQM$f2E%MeO}XQw5rsD
zE74?Tq4`Ytk2m&5-P)i3y5#!*<8E`G9%0q~9`|0TTQfZDlwFpi+U2)T?_|Apne}^`
z(Cmpdajqvz{G*pIUnP-NF54gmN-k+JUE7}>_@dcz?w#lBhPsftbj_(+QyMa77qTrn
zpP&BjCBMa$m7?kKmTt|Z?U|7l*7kb7r?*$X{!sSRXN^Vv^Eb0T@@$@#{x*H8<<!q2
zZ-0MX?D&)M{|x`N&QB@>qSYr}@f6?DVUxZwT=vU~P$7e$TicoA`X<Z#-Jd<<(tXP-
zFT-}&-VUFtm~n~Wp8OB#kLmi!hUv?WN$-Ccy@tb>KV?Cfra9l5_|wvoc^m3Kn<*KY
zKH9yp_T+5S)3u*d!(Iyr3Rb3>Y|{9o#PoP=oSH<Tv4ltQ{fhfr?}rB*EVs4KlRMn?
z=K6=3>~79{x6Z}+{=513#~SzGuyT3#!x`*5KTlokHaA+FGe+I!($jrEml<q3;I@@d
zP;g?plXADx`I(PXY&LFdDUcAZn{)P9T8Qoxx$8$h|DGGmJLi9{-La<B3ggL*-s$I@
z+alIXte&~^mz1e>?dfSNb1m$Cs^o|2p4E@9JEgC#v?2O`Qse#UGCTMDzxPv3Heqp8
z`-$#^{fCaVPX3tvh4;9Qf7RM0Q+w}Ed3xFPc&NQ#tE(VmI#=1sQ#Zuwd}hgcOrN*k
zFn6o{O{IRr>~B&X{E7c232YKs*ZTKI!71_F^J1@8-zeuzf1+PsT;U*mOphn&;ilvg
z>FC*$c<q<3zApJkI&4|Q%I`aFEm42}^i;w78f7j66&3!iHdn1&<{3ZM-ZS%XXK!M<
zoOSk*iwS<wG7ZKz_uhIq)p+kK^+`S}O8fQ{tXy5{*A%Vmcjn!*_P%d7*NWw+8LJ)0
z+L+|^B7D*Xwc}6f&WElS^A@p~{r{%l@h5$|H8Z#IRSE_ry`EFQcYWld4GD+t=Ip9g
znt1)q4C9@}=~+qAvTIz97QFw%AOFLCn(*J>LZ^PNOS*XOWrfX?E$p|K3fO&{=kWdh
zvVco}9n<Y^c6RMy`@XezQ?&4{7ZHnecGx}IJXvN%X`f2ERHR5HV~?8gyZ)ca={`JN
z2i`8=NGK}*?_Mpk`R3n!b-m^1(|x8oevIts?tXk}srSi}x9d0SeF(0qv7gR)_jF*<
zfgq(u?%mHXZt!H_7Vld)Wx0Cs_n0S-=82Z%M@$MjYLodoy6sb{w_)DrN+l=p$1h)c
ze)O+X5!z9fU^j2xyd1v%$_GX7N}F>h|M{qMtJh%l-22me4`04Kxzb&KW9cyur$^2i
zkL@R2jjnq(yY1-Dieq&QG85Y+Zf)Qc7S~&n|59kn{6E@(g6$HIug^$Y+V$}H^^Q$X
zPd_qEP@R_Y>zsuQpI*$4hJy==J6sb3T~s1}|9ZV1bhgK2b$>NGJG;oWzgtTWzo=a+
z*SOc}a`pp`B=Pl6{-0wF-h6j^jb*aVT&von>w~Qgmz1jOiL7us+H=k#T>ik3rR)(8
z;u|v^5AK+nccv*$v>?V~$FGaF-eIfCXE>!^UdXPOo`2;o`?5<9Ll;XWC?0!uB%0Ui
z!s-O6Bv0qc`)UuK9DO9uRIz`P`PO^Yv$HLjuSZ_kCwkqQ{nF}S^<!^i9B#e;%X#N@
z-}-z}o@3u%+N2mR^xmghcCo}Sa?<_A4U66TKi#W-U-<V|Y1Y+MOZjJNfa=8b+TY(&
zUte1b+BLt_OZ40O`}2LS-8pq{{`E_1qMXECob~eg<mWifTVa3TB=__qjxXLE-TSd<
z&XWf}TMVt%ceNTYp38f%L!ehw<4axjo8#@9ZEe~Xzcve88^@{hzx<Qp>7C!_J>Rw?
z;z8JK-rH}j_udf77t8G~WKfGMJUzic@t4hOzaxQ4`NvxRm!I|z)feZB-R1WzXJ(<+
z15dfljMw+Del$w)3HyDiSK9o?r_=f?V|SbBMsG`bes1oQQ&YX=jRGS?CW>o6dXd)l
z>*vN7FHA)rOMd+DA>sA4wIy$FiOS2%+x`Djye@wKyd&3d9qY|tS4>}bZ)QsWqfJNW
z3Eaz@uQubtu9w>)ZqKck7Lu}(G!jn@xx3|yob{BOiLNVt?Rxz-Qf}9r+p7huum88O
zPs+_dn{?o8!h`kN7q`!!Y}jvn<kkkpkIN@L*ZjIVeEqVLmzQGpR7_NM@4K)vSY5VB
ztkrd5z4A`kd)}vv_J34c<{Q0X!RspwZ_0ifM~BTe%e5$bBN4WGYS2<IK@pKDE!*$-
ze4I7a(rSzRY5uqaLZ$OEW%sGrJf1Dm!Jq1!ap-4jUx#syQOKg4d*MQEe=n)9SWP#{
zD?e&2Ec)*{_w7|jQ`SYZmLK@@>hQG5jN#K;Zhy1fEmd%KhUg>bjx9Gfw!Tk1*lDpN
z^YBg8SSODy(VcrlTyL4M{o7F^8xy}g>Z*xP{5E-QhCrEkq2}_7cLaocFH0}Gy?9DE
zWBBzYvODx*b~tQKJNx4Da(>W4r+0UDrktMUEAJB+A@XzkscE{PP1`%FzOFjN+i<n%
zYuK@kQ5{{o-ka3zZar~g#S!~u<q?Y)ZLnId?9Nyk%mO+B0(3rP#>GYFb7syoN=>@8
zB@=WqGw7`CHeTtXUteCT?EIT=`oeMjs`Qt)_o$_wzJ6${cf6K}&haHX<BfMNHb1Q9
zd+2<=<zjJuwc97ZuHorZt#~U{CV%&A*VNe?{_pGC>aNUCpdB;w+qu#e(N+upPEy=_
zoq77>hri6V#n*adFq&q|PFyc`bia?3RhYKl<&DuNV=r{(+UF}6g$qVme7m%!CQ~do
z_IQ7GdGcJ_Ij-#M9~o!Zt*v{1*8Q)#k=ldefB&+?)_(qb^n0VQ(6!9(jA6d>ydsij
zuUzs=IrYtrje9<yv!0?6*yP;K7q~Xcv{pm)XuxZk%7;nG$)NK#w8PdUe0y_qiq6bm
z?}V!h&vUVD{U=o1-5+6EZI)?2`|zBcgFW}fJa2V#M?8Obe%AY_Non_O?Bt3X+DeU#
zOU^_t%a?GTnNzmSmn-4S%RLp#ZO{C0-Th*7f%RGLRFA2@gMEB>Y^%Se+}@U}6SwEb
z%(kPuj~+d0m~}<Ntxv{td*0m--@hL}dQ|n~)x%jE8&=P()}OXrNHq26le+TPbB?nt
zII_+3yY64kY01ZvHYS?Q7OLZweE;Xg#v3vJ3q}9mwXe%jD?fAcjnlsg69oSJ`Ez2L
zuC%b4&jN==reBk$i@0z9Jy~4Oca{kY8(YxMqEsof9FO&JdtcnyS-fhM<F`le1NJmb
z{CF)X=gy4@|GDjb7f-YPS$J5YD7J3${MmJf=gbYQsD4<!qh?2o*uKibxX?*||BIiM
zUcT&Nt}|zuPG09sZ40q*MYE}LZT}wk+h<%{#M;Uwn)LqO-t$}S7j<`YGcq!oWMAu9
z7rPsDAl0!xS?^XZ(Mch`K53tump%HYe_K;hUgFif6ic@=X^+>|-!r}0^HZ?-!GE<A
zYOLYw<DRZwzc1<QtE)XSmXn^Ho&EZBY><v@+qCJ+5)Ze5&h@PP{OsZ)SMEiN76qL@
zCVe^aSVR0{u9VYZ4STj-W|+hCv-g5QdK>TWzNnwZf%$LB-1e0=8UD3TywDkOXHGI_
z>^I@mno|=5I4@1moGRBAx2r_+>gw?2pkuA(+y8Ghy>xE2x&G>~wMicy9Ndz7ds?5Y
z^{4Om>t8d^?E+<xDO#ai;^N|cvewHMXmE8aoe%!^=jY9h$?TRzPZret{B&|f=*fEV
z^6UGPbHaGeg|3KS&Q~gBvcPYyRLb_a#OGTW>}J%@?cXnVuE9txX=CzEk@H(`%=mpd
zebvg@SF?`Zzds+eT(0!>wY^&cUxG5?n;V9kQcgBapDu2ke$FNR>yjLG|9MOD?(XW5
zHt&0Sdb)ApBNwBT6AKP?99neZbDGgi3Hv%5=Qf^&b$@^P%rsIJ&DpB`tcOi6-gCzc
zRp)JcERx&}axT`biTGK2)x#~{`qU=F{g<}oRi3%F-$F2{dA6U(y2x3{$<GQkE-|`%
z44*MqI8i+4=)O<S=hp}AttwSgQo6CbTwm2|O2x|8kDom|_Tu7V(6LvW)6On(>y?tR
ztqPHUCGvwg^}~aMA*;iBca^>tl9KY8X_Oiw5f~v-d2yCm?x}Ngt!El0v#pKV>czJA
z?Yw&v=Px(3wwQMFe($sCi^U7i#`kvGzS_K@p4+Q(+M=6r{`+zwTW9Io%(!!LwwtYb
z(L#ovoY1+ORsXyU%3?mNCAx5-zwa{3H!*j%b|wdI6WLp@6TffHs?gP!mibDH>Bsf#
z-d#N@a{HsB-J6S_`$cWZa0Jc%Utce8o_FU$yA|t=9fitqad9%%Wj%j?f4{ugoxjJe
zRAIsu*KlX=Ny5Uyht=oT96Hu3J<~Xy&$Ua$)AV-U_NWaJ2k-v9=E_yJMlAZ#-bHI8
z)-1an-hFT1D^cqu5A5R#3k$!zyxeY@eQiU*Lnp84dUKat|NiJ`H)x4+@^L=5UMbhN
zx3+#==+~wfx5vZ5f#Kl6gEjyEe!sFZ*xhwvh_`b0q$fLe?J}zTl=AC4_&k=GcC}Tj
zZk8=RSrZ<>VxY$6%5d2>*VX6Qk#8EB78>(xtHtv3^YvnO9Jsf)dU8zw%ZYim)}No9
zO@4J{B?}A7g*}zUhYlb9deBep(c{OUqkdCQO?mkBdi?Vf6P3BT7HwD|<PmgrPi1l4
z@3-b^{_}ipZ_C{np`+!c_386@`{12Lsk6-UL2J=Z%rLCHD-Akl^Kd(V@R|t2Q&Y7;
zOTp&7ludhb)4JdR!_`%x$rl&7f@XOaI=4^Ki?tGU6A=q#@;r5~>a}iDbF<;>vmt9D
zK$Y|g4b3@c)07ny9+W*UtFQl`vRhYea_7Ilzd;dI_w%VZ=(OpKj1~8usHj<4&5GMw
zb#Z|svsKxf6WelcYsBo>z@<Ft#E-M)_iGj#c?3Os^vLP&udgR3C^D~&+NxCi?2L$Z
zm`|%`<GK0vpn2fJ!ooGt+xyJ#*BHCzgg-koGxO#q(1BAWFD^6&FZT=FS5p}i8~b(H
z{*=?x^^ae_K0Rh<(Z-seMOtBNIu0fTB)Ys?G||=5ENWZM$!lw)CC&3>WUWdrTwSJl
z=GNBi%I|l}Gw<v$)DBzoU}|`rXW{K17v1Fx-`p_laZ_IEH8tsA6KmDu_(_}7&tKY6
zn9Rl}bK=g<;+dAkYLS~#0ux<C#8h{F{_sITMTKSYMGw%yzIA_gbWJ*QGsg^c>gV$L
zbzT7h0=CuPd@d?mOB$!8Tw3DE!pbTrA>na%SE*&?rzK@A;!m#G?$Zui<8ic06f|!?
zRVy?o(M8U&eWB`2&>>RCkGF5hynJa@sCHcK*Q+5{<J46-`T0S|8^`RgoBQ?k_0QL$
z^Ow%bJwMmlT)zH~Vfeb3g|)xG-Puvtyeakcqc?BX$Ue3N6?Oae?Q?N;1%-{>j|a?O
zeMBmmq#~!Jtj@i?E$7yjPC;e23roGllai7`rldT%#w!2y)m6|+_mp#UEWiGZGM9RJ
zaWVVv@9&?#xVU(VW^kMDY_rVg6~~m6lwRD~X}mUi`?C7~|3EA8ugBNNwh9JDOu2I4
z^5w}FE(AP!^k_xQPNT%cL@h6&^zFAnYj|RJm!)1@<oe~^UF)my0TZmt^O~BPBy6ip
zVt1D<&Aq)XW><-)^=V^+>TfxF_Ut(^!%+EnpX}no$H%m~Iu0#bVwRVecV@2j^mli6
zKYsi6?4eff&?zdf*6;tf$ZxLIo7>y_Ykz<H_-^<6RT||hqqpn1_sgl~-`fLn_R8Sp
zi)QE8&V78mKX`MR?=ru+r(RuM&B?=~Qv2(RR?msx3vd6axPl_0SK1tOaCVP`A=BiO
zOBSgJ&Nj<^^tj)CS@H98U*6spS6BZVRy*ZzJOAaC!Rn75J>uZuS(0&ak=7|;&^ha%
zv&5dAnOXDq>-CbCmt23(I#IvGf5H-tqg|rO@9*se#oemV)kUwaXo@~n?e6bS|M1`-
zC{2T^$@TH}e?QhSfjVrrxApeh|C{ma>uX^F0f&;3EnJ^DUSC@aI#F_S`gzc9pIxP|
zm(}`$#<S;NTJC?oiIw}sg@w#lSA{BTXn43*dV-3oy;Wat>?~FX9j)j!RZHvC{saSw
zjQjg+L5n<ZZOxvd7duN-J8Xr)_n)uV?+3LEK#ibFOFTh~@?|WGRv0L8bu0B>Tk0)-
zZC&i?@c3HSemUEY-rmr_xCLvYw+C%Za@7i5b)=0~`ouij-CduiY)UzKs7KORBVvQX
z-s<niu3w)HS~C4>*NwMjYCba@Zf;7|jM-t(&Myx-==D~j%REEVFOx*prktGg=I-w4
z+j4J%YOS>M^L+Q!{0y1#T*J<8-owM~#;K=7Kqne*-@d);6Avh8Z@ew*le0Zle!tee
zl}q%;m&^V?A2jm^rP_kp-Jqqw6BL<mZ_oD+wY74MGl_kmwDQ=EjmeTG83K=w^)8Ov
zTcvfP7_`gi*4AwIm>3!Rzh5TvvDzs^7RCt+2{~O`6Uq4F^LhJ^-@k)aPp{mhmU(&E
z()9E5w&dIlvhDy?-<v_|Cad|b2wu+D%+7!8=uubKN;l9JpV?-)8CO?{GPChqm}@w1
z;$rvypsiV<*VaTjpPZx`Dj-<-u;kpLC0cQDaiApo`T6<J_p09)etQ!s`qaM2LI%{5
z0&RBinQhj4u$f(3!%jJTecV#VX0{$FQ!mFr7ZEYnxG6o7#+!0(8rl8%;0)T4_Uh{D
z&?(DB1O*)zyY)uwud}WG_GV$)*;!hrjD_Uo{XvIq1~2E+joK36=<;Og#l#HHt7UI*
zxn5r8`|;z)!~+eC7S^w4t*;N>9kw>=;k$R|+IXc)>K++i+nyg^_V(7%U8S$L#CM(W
zUOLg$^D3wo*;o4;bhxEy)|Cf`TDi4OC0oew9qW;F-kf%JhDoN7ot+)%T-n)Ymp!sk
z_MWDbcyCXoPQ(TW$3T}jt6rTh6}AO}%5I?5dxnLNTx9)fJG8^sU0CSMuB5CCI$>@7
z-fvPLKYpzFaFBh^$79kgY;3DM{$HG^?7kvutJdplYZtq83Tbt8>?x6o6j6NWCmCP!
zk=1*e&cXBN``7RL#kKqHx<hwVRa9EIM72JAyPdze^mSNV_1mp2`hx1}pw)7>wr0;X
z&7Q{fRsL{AGaIjjaavE)q7Ie0`zK08itN08zy7~(x0vpZy1!QT|9&h!ca#})Gm%@b
z)X^!L!E(1)e%;uZyj@>HD|XkGvL*GJc6Rf=->;v4sFm9|>xzb`i_0XQUY)Kx>fY0I
zKqaG9>8pT^Nv>u&Hzs6d7k+&eI?FWs*rTJ}H}_V5SAH+{=;6bI{Puqw7Q6NCD0w-l
zY)Spf{QLWE<e0Vb%dbo7@R+n{qU%KW8@o!i&(1PEeD$g*c$>-p_xt}Z3tb)7BVpKN
zl6lGIV)4FTue7)2-8}`0y24|^QCl(u_f~yXQc-C+Gt+prg3_W58k6R@xVwM89$&xo
zRIdMQGu@rV&;Ny=FSL-kvOa$Pp32XleQa+X`av79K-+deYYI0e9Sso>1dSU^(bv-I
zGD<xqARyrI`r6vi3GF+Io^r*-#qmg)bnx5%31AfjjR|_%_Q~7(Jv}uwa%+}oTwI*i
ziP|@1*6;WKw{z_lOFhydsMXP-61nSutINFZ?f2_uYX&d7up&@dFK*9@raR01=i6Dx
z@U`*Dt^)ad+2-@EPj-Ux_KofN^85Df`}6Dd`W-uVtk5{n#L6A9r@~NCQBk7T4HV=i
zvTb4$pPzbI@Zj-d=XEhVCG6{JR;@pNu$leiw{K~O+jvEEqe84ZI#epHcyyJ{e-~eQ
z@W7j!n{VzYRKC32-~Z}X(DnyVCX+Vza}0EubpHDeH!;^It`>!lSl;jZt*0HfW<k!)
zO)K+$zb&)gnss$i%E?JQ(q=x^EgdS-QjsDP#X)zA_{=urjgP+{QhCtZ!GXbT@kB-E
zwv2Rvi7Bq)J33Bq2XD`d{r2{@`(n4=8(T7iSM7X|`tQ$AP}@LAsA=-#y2afdpB5dy
zp!CVfQie~;G|T1No0~H&3X{6--~0RfJLrA~v%EVU%a<=NV-WN#`yw3f68A<wq`5J2
zbK1tz*I~>2=FVEQ-Bd}5NnT$5Sik)G1sX~xzgI^qcPo7|YH4i^+?eD#O+VgmXVKH3
zJKt`;Ez7#LrgQrA>F0t)Kv6ewIcQI&RoNSlf&v4}q9-1Kx34XB=kJrZKesA$b;-O%
z9V*$s*n}ree+Ie?D0sP_N!FE)8#iu<Y>ow8UhwqP)HAcq{Xv0e+pE)c!ux4~#fPt7
zRg0gU@tJ31c{Bg~zS`d_A~&mnPNpw-c&Mt`RpsVo|2V<Qo%<tr*+3hhK*uoGM(nBi
zX~ZjS25L5Obtz3M+PB<vlZKM#!IP8KR|YI(a&&Z@q91Q}bM5}KX^;1Qzqi<_m5WEp
z#3O%2hlWm9qDXTQY%M^CO?LM6bxUi1e`8^1pKP3dE@kVx`E|cE6B85H#Ow@uuG!_$
zwJ6_2(9<aE>MGak>tZM0W!2xDem?E%tE;;fxVi+X^y+kZOuD?pQ~1u^yOUinzq76W
z7O=Z4*DU{@&qAlxFV8wXx)#qrpfu?T=&;BK4<4x0TOaVAu6J=mqI2DkhwQAbf}R^q
zK#sq(GI+U0{63p*as6dUN4pq64Yk+T*H5;s{wBW9?&HUgM~)qP^y(FtO3Z~vZ{PYZ
z^_nVSkkD|?0H#0a-=Ci|O|!*(XPbde?f&!excrk76PwNN*L+^O=KPP3k3DCb>CQIG
zWdbc%|MK$k<^A>bdn1)5Ey_#@beXgz@o<}A?XQwUhYxRz&{^UOIvVjCsASxK;p>Ml
zU!I(ps0<Qp<CO-T#QpB>?voP~oA0RzdR_!OJ}M_i2Xqjim6cTcVaJ#l8RPVGH49%C
z{Q8pl@5f_)hP-=wJOcti2Wcw%&N8WLc5z7zba|4x;bso#0<4A3?MYjs^8M>26AQn+
zxp}8n;MuWW>71LJSpEI|&&{=7Zte}L(_G@VyuQAEbLwd^xsqi)N0So2zq_lS6T8fB
zZk2xmYe#42!z(L;b8c>O&7TH}=Z-xY>-T)(GS9!ar26~2CBC!8nhzHI-Z<Ml-zfLi
z7V%AT?EG>nN=lFBWxl+$RJs1&AJ777G2N&aXNACNeLCo<+Qh?c8`ICrb&2cu-OOP-
zkhlHwWq<p`?YHOuwmLJz@bR}5i!NrU<lfo>>L1>(|3A0r>8Y2#LK8*&!=1Z5Ha#e?
zm|<0_Wn1;-K-Shrj~+RJ_F-mSsaUsurfGJVl9H0a?6aWbY31wxOsx6&=_N0?fSf3<
ztEAMlV#SK%=Ps|0w+A(?ve)nZ7IR+m&ZboE+*?}?-n~25v6(G!d){65`h`+FPtMN%
zKF|4?5~ww%biUzk-tn6^XI_sx`NQS&{f=JMb9w5{x3-+mott;;XyM;9p2$nPAAVtB
z{BURLYM+@#t>50>e$6X7QRHx>s=&l03U+pKizf?9dx?8)yKV62iKLRU^4By_HsFqN
z0fEFo5Gf+&3ZjVTs8||KdGFUZ&#r&QLtUe9f2Lipo6&JQ+pmAV9;o6`npCvox>oba
z%~F|X`FBp={J31___`#MrH4PLTy_OVeo#kF?}-W5C7B~nT-b5PUfz7i!#6*zwmH`e
z{0AvvNuTQA64X_)_Wt2*#flg6%?|DjlAV+O_DI?5gL~vB{C=(W;kqS{<)?>7d}aD)
z=_x5SxQV%{9Icr>_2Ja*vMzpHN=1JidCuie*y#7K<v7pH6H*s=+&_q%`~LRq&dlA6
z%52Wp*!{yz!ERAWEnF8r-{e?N%A3EYH?{Y(HAG*n;ykE-e)ke@_H7-fIgQ)rdP~@z
z`8DCCl2Xz0-=V4k6I0Gl=nc<$qEvNPW|968=NOY~uXHP(`sy21NbRh$O*A{s`<i+F
zzYQ;ylom~N74+nc+*qT1$#U+K_m%d=#iyn6-rYEM_2lb~$w&4UdRvt9Nl#d}O5g3k
z${*4eiAOSae0uQm=BAY^z$svo(w?h5cVxYi3eLLz3X&AMyjc2W^pvBV2K(Zg!*4Hp
z`e{?`gsl>lX=l~TA1`VU{9IirXS3HW_iCt%%0xcrZjVpje%GAaI;;M@iMz@5{lA-D
zfA_g_=w<3ngMHKZe&4(-@!;FfPX5_?5W8ks=q1|E{&8M?rk;`#sNPfpiz_LCBb8Ln
zq^_4`^-7aW>ULULCieF)e|Yfm`M0J?Z_DP)ol{^WG4J%J{+K;yTE4slr^uuN3z=P_
zt2$ITLB}mCDLvY$yFYYw*q0X<o0E=q+0F;mdy8DVA3a&pY^F4+C<9c^^?**OQ1_qr
zd|S=tn`ff)_quipsfOeTPE=nqf4>r$&IT`lfcYDo3CZ;SF9{jGcIQ-qiBncNHnV}w
zohW&ENl;MG@z<A^o|Dyh!`H<m-rZFyq8a4kDkNA5N%NbEpPwuI_2uR5|Medq9libE
zZ@%4LE)}Iu5}@eNjB;=|!0ep(@KEc?>H71zRKW4FGV<=O($4Pg-RV)gN?yLWu&}vH
z86pvTYfEP3|G(eg&rI8X`{#$l{Gl@7GD+~qkH`J*Ek$BCrFd>iJ-zHu2e^oB*uMSy
z+xp!*cfJ(Z0_ya1oRB?t?%a=Gzmg0jI=Z_T2QGHg>Vit--rjcfVFBp;h^=P?1%H~`
z?+9fQtPDJK=n!a+`NoYK_5W<mzgrI~fb8w}Pv7)+sn^s+si&t&@UXq*QJR$Vf=^HB
z)1An*Vf+4mi$2yb@4vh3?TJaMr@bTlXPf6wGRwU+{nIVJ_GVS@X%8MfI<&-d@`o>9
zuJkD?oooWN({}#)dOiN-ot?(5+~R51*Tp_LGt>C$wME(2*ZIu1o11lY)x}+<+53Jz
zlfJand-+@!mq{jbH)|-JKYFxVJoDZj%eS|;gU;4CGuPTYl--q~`Jlt@vbP+3e9!Le
zEEdrU=?GmNmYEI?Do<M(K6YVsza<_Mm7biL2^#4DO%NWsqZ+*}2h?Zd6jpoi{yk_A
z>-qWl;fdh7FlZYyJKu+M*6$6{&&hx`Wj8P~$KA4QI_Wr7E0lweZ&|^^L!g5e0vEd_
zZoeHW0cminT3Jce*VkuVS>d=kY;DB;I$Jq8Iju9o1~Yv?6_W34Gf;cYYpT}KcXxNg
zG%0FmaPaf<D=8^)@bWGVT^*Li`&r~(#be&EbupE{H<W;`(Wv^mic1-kk|){BwJdI{
z`ughPa)0?{{`1c*@tk~Oj^$>q&+<Qi{Q}*9^XTzo&<x?ZIhI;opfus}Nl;w;_=Sbe
znxU&imU>SIg*9k%acgCpQObz}GYpd>HY6}AyZ0TreS7wX4F;lapyFL6^Y*@4YgMl)
z37?;x&AGd4>O9-(M~@yYI%J~`GUN8P+$$>ro6GN2wy)p+PipnmRfjrGa4)sq(QzW#
z)y?hF=5+sS>*LRFNIaZzVL@Zpr>55S_RE{o{XrK<9_bYR`0ZO*pv#li3nvzJ*jxe)
zK!mIi$h^Ev)x;#k@uTX=na1kH&(Eda*pOKB=i_mc{Cjh{#r0PifU|`DanMCgF*}VO
zAMa0pd1)!=hMq4kFRwPxk6atIm4la8HT~S2imEE3#6v8P9zPD15Uk8Ak%)462RaJ|
zG%`3vGx){r?fP$TZ$Ez0bHDcFtOegce?FYH8MKY3OI&~5qK*^B%l9)V72UqIH5)Yi
zk(ikH=HA}fe6m&=8fuAha&l|p_s_ewHX1Z8_2=*3&c41dNTQjy_QQvQn%dfz*Valm
zHa3E`yoJZthPHaL@vwnMO-)RubaZgo*w{quDp?7M&`<YozAa;7X4VW_Bf-igqF`Y$
z<JHyGs|`*DJUcUUrb(vI<>mg%^Y8Bi9Ry*Ver^RM5d76mOhD&F2#AOr(ck~)(DeAa
z&ef||b4@-Ix+0)4c6Zssr%zp11}&AaD$yt`EY#}hurZB&pfu@}i>vF!-R1ey^kPpf
zaBR-Fwq|D6B&WG{wX@`Gs|x=8sZ{fyx5oNy$BE#JCl+~VZAv)EWMgB~(b=g8iW$S!
z)6@0ky{GFf^`CDCI<aC~?rkAK!G(vwCCZ)G{r3MBq@A4w>RwtFKLZ`H5m)(CR8(zh
z#)Sop)!*JI=HA*Oq8Bqm(l~8}hSH}aB@$68tlho6ponH+WBYV7eg4I*+2M2M&W-;N
z6~P&6G5Kx%(S~ZlTU%DP{kH9ozG0TmSFN06kfP~*y#G{U)q$Wl6BBjj*)|JX&c1y9
z`saDam7LN8m)2Tnm$N6_-8f^xg4{h9GZpr=8rfXd-MOLKu-GhK!I@2X$w|Mv{ZbQd
zr#!gT$$KWIxJ4rF>0&?M6S+nSy*$Y`-{gwq-e=?A{P3L9B7WfyC&TQ1F66fC(XX3-
zV}HGUU0vObvuSF+vs&8Q+wC-zK6yw*ig+#rbucb2a^>XXOFP~tTl49py1t?Y(+rKf
zOE~kQ(~MH5$iMc#bDRA_S()<O_P*p>X5uyWH3b1I25G09&&`&${L*07wCoLQ#{Hct
zGkw@TiE8&{UKZ1_*?s?@pnJ1rZsCO0Iku-}mwovA{r+dSwN+2dR9g=4u%3QoRBE+8
zWQ|ka<i;5v64HEr&TG89EpfMf0h@k$?5{s5#{1^{T~zV$QO><RlJfHMpgaBM+tu!<
z{JhK#oRTI*#q6t@2^zfuT^rlT%npij(4P89*R^=lgqp5Cket2Y-n_i@*MBTI<J}~U
zAJ&yOY>Mva>DiSXCw#L=wAtib!&HsabKUR6X`fe@-D}9a%*XO%;DdcCX<G$EQ+K9M
zx|_v*Kq&QK-FsyhSJ#O@47PbI*8F?uqV%@xz>(c=e?M$(Y<<mlztZcAr&Em8r5zz<
z3^B#i7G4&=`|+2%?3y?8Hc#1h);qbT^L^Xo$-=T$B?;f(-34`xcN8RETNk^UzaLb}
zElJ^PZw8%Hbf}dZbohyrv-8T}<$gkuyDNBDvo9Uq&snHlz1h?zanD)7jk9+?wDXaT
zkUzfFx;XxZZGW3DQ^bw}o~1UnGj~5vN_uAU`S*I`7xy;sJUu;ak<0t}J5u($1Uy(?
zA(neS(ECuvqr1IHb^m{||Jf&Z{DHq#(4WcjmgO%6lK)5ruL&{y^G7OhTTSF1wlznX
zT;|_)tlz)SN9JySkm{`ur{@di%r%@e;fKJjAN?uS6+AmCFKx_DEbE(hYU7dX)?ZHx
z8P&y3_WAzA>9+mpg0~u*ox2~!>J(3^2#>4%8k%-?*3lV;$uF+0mF|@`SF^E+kyis%
zJD!<*vQ{pwT%t1!li8qCt253%Q`y?Lak^~9)oh>SX5qc!?-?FEI5cxgZg}^cBX>CW
z3MyX7`K}v(PQrV+U;M|{SNwP~?{@AD<UFWTw_bbx+56|$ElobLy~rT>TI}+mRw0Hv
z^%L$2ZfthjTYhPmdwNyPJ@MP?5C7EE(cZT&)Ij3?^3M~Fw&?wwcEk0uW$MNaTl~b0
zOkX-|e|I<I`3>*$2ctEf-e#GzC*qGYgJ_Z3mxVfi9n(ARpLwI3-l(nfUv19+%IEbt
zyn62<&%U(s)yyzbS^53g?c1{-JV@XY(Qp7|D;}OBdD~w%tAgsllXuU~HvjqO^Lfw#
zJZEQ{ryp+Pt*EL}ySR04V{saP&Hjq^87sMOO4m(UARl*pd4^lg#&SNv=zs53EGi#P
zs+eO|U6{W|M%}XFpVYPpZn^nCCY?_?u=c_3KkI#@{Ni?YYVN(w9B=sQyfVsk_xjxv
zEu+)sa8*n?o^!x7-*FF*q~bH3*^l1FSk^Y2we}CFxWeI9uGec=IOCF7@pBoE=P^$u
zqRe%*fB!pkk$L~7z31FTjP3OpqHfxyYo3)BsbAT?K>pa(t5fU#{@R#$n60U)Nh5d}
zPxZGq7u$VY3c@&DCankEWK#J#&B@8>1Zcx9|8~QGM}NOn@Me3TmYSVu^(%c7(?tFq
z5>L-<wQ#$`)5cy`n|kIk&&F--`{H`eb{*Ww_4W0YowX0oEVVzplkv#>UGGibni=fz
zjGfOmWsm3|rT>p=el+V%F$`Sl%g`4ZfBA{>kF=NTj<X*);(XzL-|2Zv?k{d)<=#~F
zH48Lpx!A4OqW0Goy9}k3bKJySCvBWDL*mqQ{rNmiNm2|ka}G|p*}YBoj6~1-&xuQI
z7;YrJ|5IhU|Cbu;Bj=j}vkK-dQhvlUn`f%VY;)!#ZiTOYt<^49=8O|^ZsuD5_|>Ez
zHQ~$;dbk@lck<l)^r%Dj;0D*<8<U=X4^@`jtUSrY@S?@uM6W+va~^9wn5D_>YUwJy
zKXcCZ8y$ALS@-MLwUuzTf6gt_o;ZC`UjD?}(N|xD9&5RLJ9b6P2S(-D=gvLYQT6-J
z$!>P@qFqtJT(fvjCf#2ZzFuzQ#*M!wbasGRqn4ZF)2>eG<hi|f;->pPi?YxAh2;F>
z{kmzt7vtnc<=6EOPA6J#*gQ9}`17ULi_2%2e9vB0|Kx0M-d&6Trv=^g<$Bt09yprx
zwIlOx+r7SZGx%~8j^9w2mByL=Y=^0U{GmUpiY7+ylP+{d%sKAA=*soCGj1%loBMlv
zLSntg<rJ0KyRClgd(QG^zj*rFJpP{DX#!&MhyE;m{?Z~{A=>_x_u5U#1`_wv?`=zb
zViv!kRk<PUsrTjTYwK&KybYUvB7DWgMUvG@Sw;0B+uh}o?@c+`VsS{qzQ}5R`Tq&Y
zi=SWmv-zs$CxI;;Cl;^J-0opymUTq~bk)d(h0ehbu0?isZAvOC-@pF*B3bV#tY_ZM
zm~T*dbFG+8SV6hD{mC;CI$?9(*TwE;?LI5I`Q^*S^R5c9u6s9Ql4leTKmYQerCz(N
zqFhWsBW6=M%fGX4ePNbVR{lTS+i2FA9kb_Eq@DSGKF7qq(4h9kwHbH6PV4g6w4@kw
zu;Pj2CChH5NjwqYI)1V2j5XH{o~-bBt@hO5Z@a2n%eh5fzYD_N@hRLFz8CoP(Kbz%
zdA05N_MfgqD=DpP0}UNW8hp;%%wHgBKT&?2=<oB6f(O7$&!?*dCk6CaNS(<%y?B;$
z#7(<oq4j%ay5_foYK;yROSf{PYRlkPy?6NeyUMsHP7I9r|C1pwFfii%uDcs=XFvV9
zWU+bLfilgO^Y(K*p2U84-sa+#wDUJ-XL-WPXq(;lxIsm*pl6=mn|bbs)fjbREZTzR
z%{z0q*wFB6tKqU-cC~XePP8Ab{qXs7@|#-%O-%MjH)mA$MJ3&^N>)+)sJP{${K;=0
zyji~HpWni3J>$|E6XX6k)(4N~U7fS;@{}gE<}O{Om0jl|MJA@47Zh|lTXz4`<38sp
zCFT!;ZG^8zmmf<09keND?$Mst?>By*%)j%)vFTS{PM>*i@m`O~Pc?jQ_ZX|4`fb*@
zb2X^&WzCa2mYx~4?{0MacJ1_}qm|YPslVqe-SYNIV0nVz#1MDoZly_YBB3Saj3=AK
zzaKkYIH~#h`oh1bQVWg<G4BamVIX1jFpnkpm-YAJZL0SU+!TAvu>H)HipS<P-J*%l
z8hNg0P5t>zG2{KAnV~s?o~59qaq^Dnw{};xnuT}&xfwr7^RS4C+41msh}1of{fS~+
zF&wH1r?$^%e9^9_&z_sUUhV&z!kTM~HCONUPt0lTowM%Rl%_P<CNc0RpvvFU(zo+6
z#mao84`glE66u(Kr0qg?MM#CH`w>YqNv(^j`D}~M|38)X+)O7kcz(}0-ILQaAO5?r
zPOsCu)bzodNqh{m?|L1PSJb&Tdurd@eM?I|o(Xwb)6lCr?b4oPYq&Kh>t==qyBQnY
z;8OgXvHQ2_rlsD;kA~U{KMI-ka<=`0$=e^V_&3qqd{^n?Kl<(xdn*j&(wN2dWsVD)
z&u_cB>~7qh1CpCNV)n8<a-OkYrDeU8=t5Yw{iMRjpK&O{y*}of+~V6EX*?g#AO2Ig
ziz~_{NWJ;+!-5q&N+&O`2wZGY{EWxBoiEgoVFP&W+Pir-;+|VMGXAsJD(m`p&$)=4
zb4TXHA3t%{QsUCZ*ZUl@4ki^J&GC}3-{R()Xu9)p-X_pSo-*^LtGwFU+d)Uy9Xfn?
zYjo$5BGBZA$dQ)ccNgmY;;~<M%a`+ftjL4X9dhS>xmvHd_VIMR-PMd^HP;KDSvOwp
zxEag7z*XhzrK=Z|CKc_hG%9_mQWSgRZ2#rupHDYtNPUq!QoVlnN;4%Te-BXdS(K_5
z(O9-uYlVQ4lD{WtBIC>r<--vIH`=%5-hTA_`SAu%?g)!ozT-OQKV5n8I%l?>z<edU
znoIRk?Rq!=ef+U5)%Zj9_p^`Vuge{6oL`^t=F{W<nZ0W!ChBm^`FG^@&kH{n^9x_d
zW2(QoRll?2(4vkd2G{%6EQnB*uPa=8?nLKhN$V<(-njio8VvW_T~3zReQZm**wddy
zN!gFY*)Q#Rw&VAq&?+07xYXnmk~#f8InOy~8BI3ad#q67*0(FKg@xtL^l08UF1Tfu
z^S|&P_rcSV)9vhUmUJxvl@}geN%rTD@c(`ol+W?lI!t{sr@<`En9bJ3yHir`%XjK-
zcq^u)ym-UC|0UT@w>G;(Ry<h{b*6Be%hrUN+pngqikPohY<D=({=|L5?QSpLxVT0Y
zoSxdhsy{kCIIqU4bZOj7lkZ>i_H11h=n}c6|8B|kIjJwcxR&pkw;_ao{{2($EWI;t
z@7eOp6123UV~Ng1^>X9miO(mey)UY7mOTIe>*9CPFL!RJmgAkxvZ3IigjZ#iQTch-
zH`tbeiye>m$%$Y(uVklVCBYbz7Ij#t#^yiQ{QA)M-?A-sFFTlU!mBF%fY7lEZN3tl
z3m9vo%#wwlsY_hwQRF*OaDL0{HQaxu$lX7m>h!&6ip+*7CywhLt9mby_+_W;7p<Tv
zN=Nr4e82d%L_cT>WB0fFoQJNl_c-3$wKT!-Zcu>BB(c~BN=oN_q@LIJbxNL{=^-!Z
z^b=dbR#Us$AlJLjieK{BhtJxYcPp%x+pRu*Jb7R0pWn78e`cvY(Ax7&F@AgH^Lc^W
z4`)vce0Tca6XT<sHl-Us`ZMig=+rwFubw`9eyg~0R`Y4`YrKoUZ>@Usca>EiH|vbx
zYCUoL&x)7#@3=bucJ@=rKcCD0{Q0*$CoWSZ`#^uq)961F&+pukZo2%n$<hU$+eJhs
zE^!5==KD8gIuk!0bUgm4Ui~0w&G?e(dv}<FHlNtk_nx_F^f=_eWScbRb(P$YQcukI
z;BDrw`fIA8;E&USNqT>dEqr8pvi7>s;{G$s|0c@2{oKcUae2h9i>I<@3PnAVFnV71
z=<uQMn>KxazSU&)x3EJ+MU}-r9`M-Ado=IDxyn6}MjIG6KbJYa+;QXaeeb?mdS~8S
zu!GOB8=MVn)=6$=+K~7Aq&xeH+sBQ1c1Q{_&EV~@|GHpx$JvVVjKdcbCv(Vbm%VQ_
zuYA+XR^1=HeaVln@E5Yb*|?E2@#@$8aylCem2Wwon7O)K?dIYgB416Cr<Cc<;beA~
zXe=(Xt-5x^`9i5}skfsbXm(lTgk`wvva+u!&sT`)+$#RK&VAkHya@SgcTPAq{s=H@
zQL}m0VR2Z^S@qPjvkZ{-n~nOhW1$DGnkssveamv-K5q8xe{9>On{$NcOFMm>6`T3Z
zwKVD6+{fpcwIytR^LI(jk=U9eET*9L$KOdNeEtI_(eN0COBVO}RvgU^d-mc&osNA4
zqxY4{Wu-qRZJ2Rf^nj#Hc<1iyOGkSbM^4--e|vh(>2-zjn_s;6+J3J7;{!Lp@N6%~
z&3umC9zuWm!Z}@@>^BZt_gN=xpVhQ$YgZX>NC=e@HQ7-AwN%Pq@`!uf&J`G~JfDLz
z4)<`(Y54p7(VDw&8w;=V9D<hbvnm=5&9~a?w(qQ1+R`2M=DB3>Qd@?9yxI5azCO%$
z(tJI4+FIZJ3*Pka)eL^7ThOwx_OqgvN#zN~%-^YIdXcA=O|Wo6Y&(nQt-6q1UHi*y
zy=VM_?`z^$PI&t;ekx=G+f3*N)X(qMo+!(cy!toIs3Ozc=t+weoBFT%7w*TNK3*~H
z;hoyg^(Qu7EYAFQGfTEfY@*7^2L%=~TFoAxOc1TV$>+Rou8q(UTVs}fX6BvuyRB|{
zg-rRn?zf(Oc9@QsblPUiho>Gx_loL$KO-vs+-qTZ*TjQq$1{wW%GQ(_NWMQ}y80;7
zx-=iRAI0bM^lm)Zx3r<y@6@zY|L<SF1!*S+z}kr|?d`9d)h3++&2P@rUCg<8H|J+&
zd6S3l@~Zmu^A6?sMeGyo=<HK@bA;1l^0Rq6lah@monNgnGjF-^WBaCACu`EVL7O-q
zHBJ3JtqHslt$VB4?(F^GLEeetXHC;RqqgY&oIf4BQ8LKVRb}cWRz0LSg#YtE(<wqJ
zswEG7wz*pwFowN0RbZ~{p0+G#>7^7SrfZkKX#Hi3^DeuvW5Mm<y<Kk)i7jhQTb}%P
zyZnzGwpu-+pPy}h)uX<+-Q>t$!^^HNK}w*>W}%ctC6m?mNZGV<O3(S2d%rhrYxjox
z>vcnC`8_ad*xWhk?d}Qoi(d!tKlN{`@Xvp*70%t3|Fh$wEq`QEpFow-w|U#lK(jZX
zIg3w+_Bef<HPuk!&WQxG$!il>|D>0w|8wE^pI+Et$nI<Z(q-fBDur9Rfi7;5@1pyr
zo;acLZz6kgpK<n?K$&ej*YN$_U%zk9U8gCcMo&+CO@U0*sc7yhdmF`dZ&%BMboLGI
z8=r4a;PGSie4q8;z$W<*A9W7BX%1)9zO7ca;N*dqe=K$gEp1zW)alUOIHp^?iLcL1
z{eS*mt-E{R-+Y<+^V8m_&1MnuSkmkvEF9&)q~Yu6;NaM`ph?74r=hFPQB!Hr0wq_G
z4FN7X3p9d4SX=_DxD_TY+HxUV$n56bH}C%Z^H{NHk^PdE=cRqmU!;_!eK+3xc=Pvj
zcf5~s^>02{TNTk;WwHsj7T{)Ve7(oUi<c#9_jhlPo9%u}vf6gi7fIdKJdZZ7T%2KD
zA?LPmf80FjrAwCz^00kn6&Ad^S)<5w_uFl^B@B~T`fWaOfUek!-&eEqHM7)DPL=&G
z>*YR2Y%E}9DDQpY&Z*5JWxlZN|MpGmljgnM^YL1c)mdr(&o8X@guivItAE4yY0a#Y
zdmeq2sCu3oa^h3rpODXw6w`dRUVOKClZf8E{sZl`KkK}`-)RZ+sheH;y3g@T?526M
zm3dd)u3!8&{6Ht~^Z0q(zO&7GpU<oAW0$W9h&K`pJiX||)ips&y;RiIg=J-ZudWK6
zX;W#mGb>2#v%xj#-yfg7;LP!fI=6)JyV2*9);}|XwZhy*SiPU$F?^wOJpHtIK*n2@
zUpCfkJ%?^II2H1K@O|ysS{;`de{@<%=b3rcKac1aSH8b~eujCjRPgh)tPhUrPZwtJ
zdlPeZ+XT5Orz5XVGZxxj{e0GEz4N}HwaB1#O&x;DGYpfDO*J*10BYi#*e@(7c<{tT
z@D<E6Eee%*rOi|<EiI?l?0OR!!o(0J9vXem+xGCG@29uV-y5aSda;xDhuso8XZNqm
z<?{YCtaVW0PWN3<Jnv;mc3Ah!+x`r3V!iQa4xEg?xWLWymC5^4)wS;qS*P-x?wfT}
z(_n^E)PpzQKP*38%sBu2@zeVIefCy;&A7CLGyB>aMMJ~Dcs)>MbJ7>IY;jZSY0zTG
zUMbV00}YHRX=!m+=f8ZgPgSDQ!nFU(d+vSDs*OFia=C7DKDSLl(TrWaF80iCH&Lmp
zI&QCiEv?~CsoX0Q`CZp@Th^CZ8`{EOSx0X9$MDbgzOmtAPF-QIpe--His*)P90}tO
z=)XV1q*&Cf+G5%N;(gIu*&CAGzbDO7Ke1uQx!pFp1*P-NyTx=LJ)51M_V`%un%Ldb
zmd~$~S{-`60#p@yy!+!6tz+};Msm)L4UM1$C|k2a_x*e}d-aFOEFQl;`QEEI-&_B4
zxy|L}{Qg_kczjO3_Th_k@XHw!<}cB|_w4F|`K-eBN#XM<-kF47URPzazGidlrGue1
zKR&SZwg0GC_45CWZ%<wJ-M&1<FZAx7D=#9qt?^QNx#0hmeRYu^lr@)~y!3GHj6K_x
zCjYvB^n+fx!2Lx`vG;Fv2=1IZH8b@5?mx>*+1H&@sQaJ*I#(2O67n4V{eLDca_xTQ
zt)f()Fk|N{Hg!SE%YS};2HhxGSXj8{+pTO7tts8|_jxM{f9LLy31f;gw}`P!R;cpy
z{`{z`=Z|}`65F0U8;e5A2IIJ&>8IC=Ez1N=!Laem`5f<)t$f%jUa&Lf*Cx<Z!HKP>
zUVk-8IU#Vb?sx9I+HaA&%HOZcJC=Q=jaT}`&CTjztEYZ^d|X&ueEA}f@{W|-n^L*&
z?k)!{r{DAQ*=!XRl@%J-Ikw+E``}>n&nJ`pC5%!yVt1DvHQ*6-bJ6Q~7b&vS4YY`u
zm0Rq>?(+OwTQZ&3$L%fp*>I1^Ap4q*kdP4Qh;GnT!u+;hCNwPqC;a)l|Nr~#>F0MW
zY2$<m6Ao<2yc{wmCgc0NyIV3Zw>2`ee>%u6|KT9JynwLqV#tigpS{o}cxh*59PAKO
z&bYV7a{u3Nv$-a7|M&|!Dcx~x)K(84pOnA9zFt`y9o`DAs(wBf5*Ci!oW=`UxE!~)
zYR{ifr+?Jgi9XZXnsd`A>)M)={r3M(fR4w%zFuBYQBkX><InyVjN*cfn|ABYuXx01
zmV4{SiHXWrrAkFXCwTw;{XPBOo=VWVLbKdkD>RgfF0M!obYakl+M*F(_fz$7JOA^G
z?($2EpP!SlsR(G@dB}IJ)znp?s~;8Z^zicw3ji-A`<ZW=eXYlLwprks2*bR6Khy5j
z|F6BmSorkjo=Rg-mb<f~@XS2h*@nr-1SBO_9_l#Zx;!sK@Mo~JdES!f?RhLrjG%oN
zH8p#>ie^^Y$bm+z?^VCIW#f@J@VMW;@93iYf0aJnzbNYF@-GCkgjYCrR|)7gR?s5t
z?YGw+icz~?`(5_xs?f*Z@7JdvZsYy(;-WKT`RtQVMm){!a#bC^v&{<s{3sL>5}E;8
z4qD9l_V)IEcDafM_4zfEw%@Dj&b+(~bonr37W2ueiT3q%YX0-iOwkMmoeiIRYYS+V
zlGd4e4GA8jvNsW{!q<Zq*Zp`pJwD^giiwapxuU15f|v8XzP?`F&~PH?7~P~J9DnNe
za}`fq8@qd&Rq3k_SHt5sR(woq<B@dA$XLOptTcU&T!G-k`nTWj*DtUB{_f4q&FvQ#
zyHB1mW5uFB_Dh#8HO#o6u<yqs?lwNzQ_cK#D;9N}NL@P3Rq*F^uW33bx8JWj-N49P
z@t~2t;?2h67FAzXaDCRV{rB^^ii*mFsZ)<G^PMds;tE=%Rn%^lduzg+IX)E?7NM)d
zQeRzJsddU8bZ$K8)cOsHhi91OO1<CvUGCz=i&{OPgzfQ8P*U>b?EHNv=U5g?nCHoW
zV)OH}vs!23Km4rWkv8kGDt#5OzpnP$+GzK+QCmYGli7E)6%`w&YKMb%G)S3b9O%~H
z*Kz!DVB)!tm;LQ0&X{pxmT7jx-YV1ozwiIwa5D#D*9p^t#ElUzE-q%5Hp_Vc3XG_&
zS!d?k$G0+qrhX6e+xtw{iv`U}g0?hOe0-$U(-E`ri^-ObKhHtS2;^$N1cHisP(|S5
zbH+5B`LYQ(gPUXse7|3BufOky(!+-jtt`Ql^emg6o5<Nz7%X=0Klc1{?6mI!94s3m
zbR^meBfrg`FyX+Bjmaw`HZJnx0M#LaJ1>Eztj+ILG)tLgt&mVs68HG7)78-dE@C<$
z>nF(MfVLWwsrAqzkPA*|wq?#*c1l5MXXxIlub_nkdeghNY%w`>=+K6nIWkry9je~b
z0(KUqih8;{(Z3PGBzW@drcIkHs=jEr_sg*=n3{gQaz80KIaEP#;{Jts8G<LTR(*fx
z>*8`mep>jdkj_J`+^ZeHD`JXtv+wLMY;<_ACN(iJvE=nN-=l({;F*{`_wd67m7kwU
zv=wGWDhMp_oo%Mo2M!yKlDD^}e)#Yq_<!u)s?tS^7l%rK3$lrC)EDVadviLrqGHFc
z1>lufF6VY!h}7AVeVwoSXw&`o&)4mK7Zm6MT2SEP_xj38;e!bi*m$KL6zvQV5S+-p
z#BrzVlTDt!zDpD5TBTIlPEV=NuD|Qk>BQRE)$`~4_o**Ge)_XOe8+xgci9Y(BL#n&
zPnta0b4A2Ney!>IRepbFZElIs^KCzsUd<;Ew%tzHZq6!ZU8Qi-vk#P%riVVXnHX|H
zc;;)n@K2oP*5@SFxw|~cT3U@c`V7igE)ZwBfKwKBj>`4RJbFq+uA9wkn_kaSPf+Zr
z|1RgVe75s>{-f(Jb#%lCWkrfi)DF)n=GwNdGyh^mS*di3wLtgc24UX}eemjqrlmEJ
zYa9%|GR&~{tX*pVt|!7Nx#(Gp*`bX`!;gp+Hx?U6KfG~j%Hc;=>Gr9o61AlY6W_m-
zU!$!wNqU8Jl*_t^gIA?Ll&^AfnXMSn?f3Y>4t<L+k5)$*`Dommqr`ms;JaLw#q$+a
zjGC2&#RLoEjh)WK-}rl?WKsO<Zf&&_59?x2?s=HVwQX(F%dJWMn<jpHsL-dU>+*)1
zU!aLU+NbW1^nQh>U!TU9*J&tbYW`rpXu;i*xJ-59>i7On?91<7U-nX=)8pO$hwFqo
zPKav%mHy`UhtXajY%%ZA`;v}FO5HirM4s<@emilUg!`!j|K8M=8$M{bZ?9N7TRO+j
zz`VHPUfKaJH^U^WtBM<{)PCRp-=*W!k#kk<`r+@wX}8ZEyqI73$DIG$j%O|+yhr!f
z8fxyH|M11O#|%sAOvIj7Ucagu0!~VLs}C$Hd{ei9-+j`zL<_D7?`JBuOk3A^nQ^A)
z>rLUe>VI6R?P)v9W!b+!!l}s0|5M0O#t7~(rB~mID#F&Od~N^QGWA^Z%Tj^D^RYKB
znNQeOVmEVl-P04=`N?4~oDN2FP25^2b$ps-!QA}2cO7LF1W$gw!mFzkzNTZ}`lOjx
zr(fx{_N;!a`NzNK(XLe+UGB)tlVv-6xFJO0^?GxWBzC4HzE!PTCNt{q(`3zWf3R?=
zjHIEydgipFg5eDR_4$%^-{<=`SEk}Ex525TjsH&WlK%eMq0?jCvT3e@JClC$MR!)M
z(Z4q5|2eByj7$}iw+pizdY{qH%ds+g>g4_%5@rWp9bvLqcYc2D!q2it`lr75=ggri
z*0}YZut@D9v6<F3Hy&9F)TXEQ+=%X|{T&%2$ac<s-XT%32RAyNpJe*9=K}BN4d3^?
z?&^3_22QMD4?b}AY?!U_=h^HZm)5k1bspMY$d|eL`Q)#`wd-~-)pAv@@A)6WCUAho
zNNkPbeO^;dqo2u}g1#jtsV>Si<U4!QVWIV7O@kk@K9?*%T~c21dzR3*8tYG6DyP}x
zDinXT==&GeyQu$25}Q%sR%x#t+HBP~_=4JW4wb#z<^3I$n%1SyShiB}w9AuC9)5ny
zD&I!W4Bq+YXL!NUq(XU>7g~*bQ{<X{Ul+D0`86r^$dm(z{FvO|Ex2){Oy>3V&M$Wz
z3yvo7o>nQ)_{)ELvsmA;JD;@E@8xFbhVdDF|5Nz)^NtIp_IEe;%Z9G{{BH_>=>pd$
ztH1$Ww39!r-Re8vCbK^u&p%rz%fhfYEpttT;hg31;sx{Ts~dIM|8KC^@cKG;o%jEg
zSiy;R9KP#xow&Nzr+8g<rSd-K%$EDNUByiWCwBYjb}JQ?-u@)|ySjd<RY%7k_b<xe
zrMX0|$z5k8l^*D#w>r1I-=OaJyO{4g_uZ_Y@oK8+>{T5xe@)D{bi|m|?Vlgs&Zlbn
z^@r-s`)AgtO`H)i_jOXT{>gv0?%eKcX<>nuk2|FXZyg63q$IVP^;nl2%=u*gr{8~b
zZg1ngY*KvnxmMU33BBpw>=EFS`k(Il>)wCad~bX>z23s<aPisaTt8}`6k0}X%rP`H
zG_<+^Uh?|M|L5PjS$FStP3h4Ps{Y@Ucrv!!@J)n4OZ02Agq!EPPs}!7&$pwagJV~E
zpk9E1#FDRjzfay{?l5!O`zN~=a0EQs_Wa}VJ5G0ROBFsk!YNxK2&(^f3M|XKwDGd@
znj0kzMY(Fp8=u`{dnqlo-)GiiHz|{>w&Ra49ux;Doa*rJT(bX{q<`o47bNfQ-*7`_
z=fBU>vdo(H-iw%ZuarMBGSZ3>?3*X&|Ez!i*=PP==BM&A`b?kPstCHVAwuJFlFn>?
zi7i=|=Q$`W5OxGtROd{eJYBfQJe93bHZ7s}+0JeIB*LVhM)@aQownHK!bb0P(rO-H
zGylcA?YC@uxcI*6zQ;aJF+bOz)o<5rk$yk-)OYP|A5JXxf3ZXW<VwMYPGNP!2S>v@
zUt0*fr!;2o61z6R;M9HdUj2QMhm6mEyLwU))Pg9=oo!cZWm|23+*dYDKq}UUl`-d>
z$0Mtnb1nV*o*X^s93lW}P)y0bd}n9z%TFI3AF(x<`{9O>)(Ocsi9b8)KWMw{vz;iY
z?7mV#NvY`U1s*-71B+a{cYNQ+u6h5TuoBm?)e6racI@c;(Pu1avV?P@pkT?%OOdYN
zFhBVhbZb|v?yZ1rXI(yttX|dhhc!JU@R3#B!Qk^|$~JdgZ*R*D6%iCXdG!i!ZLhbN
z*P}kQvzn#fBNlnzTh$c95bmRy_v(Wrf9d;^Z*Fc5g*f$2=I3W;ujF5tA=DeMUcZw~
z>x8{P-~+W&+yA$G<1a~jc&JqioN&F~ZAw1QmvuecVc{<3oqGTDM4!Z;2;Rc=^p@YV
zch$ExmAniR1-Bej%R$@tw_n-U`=@iS#*_<wtUv2JcI^{e8okeXvGx0;x3{)JOgwe<
z?T3el?*#oeQ+i|_w4^Fr^oi1m_-AHb_bi)kOPMKJT1K)83Qml1^>>bU4hjz5xiybD
z=eqgQh_`WCCtO9OulG2Fv2A^An(%Xyyk_L4C0*d;Ui7tK_uVVI_Zsf2G1V-st`~h`
zAF!b)nkiE9PSA6+FumX4_MnE6Qqk35^Dm|cUVpu`p|H5`rg!kFliFG*>Q|gNJNHEJ
zMsAP?oZI=p5p^~Abduil@4qw4!nK2p7HL-ZFI%}=r0edjX4!qhccQj4p00~?xwoe>
zbjs<BdwVK#?(dUr`0}MB?d+_hzrMaU<Y~UPHd<Izbg5Ej#~;?iCceR`C+0RCRQSFn
z^Zcr{nNzZVD~heIlaBG)U;9_);mg%OUbg>iX>VVxFx@jHMMY9lvZbYE&Q=>^<H_^u
z|IM6bn!U<HNvYoP?uDIGGG~>%`68fgCc7YJ_DQa;=?8gx<JHba&SUGH_3&dw$b{2N
z^6&4v@wUv!*!cV}_56E#RMgZ~fg8iummQz?;^?3G^$(Ag$tQ2L1gQ<VY<}Z)@6&V7
zbWff-738Sb{QKSR_xz{VglT`jTP`owt=9<31lJe*|5`im<>vYYf0!?CY-?GmFnwa;
z#(-r<BeuzGDe#~DjfII(^r?_`*cu7Y+5g!yFTY$8yxb4mpnkM9{lsbK%KTG2kF3rd
zcy?xSQw(Rk(dyo!wdxzIP6x`pxWRf?t>$Iv>ubFmHe9x!qol0t>ERKO2+H>Bbr;{4
zIs6))nL{UNzwA!qet4{U`KP~Kx+Smw?cgfv+?sp)Tgd7>4-byH*5zT~_V;>SVRgTx
z1KsO|?iZ@{onX>BA$mee$nEFvWgnkz_kQ`;M_%-a(5d<M_Mxl8&f4*XY0Fxb1O&Rc
z=uMEDc$RgU|9rpm_s$0{ysYE2xI4B*S87ZC8wI;N4Dl1br&rx9J9mG#Noj!Am4-76
zhtq#s)XZCZ#k1go&bea`ug>JTD`E5c>)qQsJM7;*z4((+Zl`TaSF3K#-%PgnrSW%e
zUD<QfvHP`d#pVM%GiuDwAMT$SuxrlTb05ueW}KNdckddvIdi9(M@ib>Exz{LR4Jz?
zFE{<w6rX8-?dJz>v{deVZP4r~JWD`Q@?gWi^RHR<x$G)^eQc7dw?@zsj?`0A0>P!_
zdS1}B=F^A!g|8Q~t($R4Jae_0eC^xhfa?Zb(QCKtGVES>a{1vn7f<W$V%r=Q4(!lB
zFE7GacjwXa$WZHZpL6Q(JKFy=YMLOb^3y^^%vdig_R-^<nC+L({}Hu2CC>eHngqA@
z{<6=>Y2WT||MyBg;CN$wsG5D6)Rz1g51uKReR}l#j$>X{$R=Syj!h4hY|rHIcdM;?
z?BllY?u?F(f_ZPQex7ApZ=Q4Gz`MJ<^-u0Qbm)+Q@8p&imS^YYR%hyMFq1BQb>-m0
z3mq(=KE(8csKsUcleM##I|c7-5%{ED?bmwf_v)LQ|Ep$9DoShEBz?s7;J;?Ic1B~{
z6B8LCm=jv0M2@Mbs;{ZtdEoGLOa0q*Ivf$_Z&mwDag^JCa+`6KRP4eB=UNs8vbp!q
z+;Ki)f5;TaSDcfbBL(749DBS~re^<f{uz7zukFuzJ2zxSPJ65!Ll1|xA&bObtKW9p
z`kz(Kh(C95b?Ldi#)lV#WorL6`*rTt{r}UIA6Y%JK9_P~f|^T9`{k|K)3aaQxw|Qq
zyY|<Y#G9K^j~qW9zv*w-ihxFDc0PlwD@$jd)zH=L1zi{Ns9XQiLT7f~tIA4T-9;}!
zXOHZT+OjZbcIOk#u&?`XcB-E^SE3Pj!0mPtTmD-k!=$$%MQjl}?k@NzX?xE!zwFY^
zrm273eY=iC+}Nfyd3qSTBFECK7Nd=2Wv3r5%$2jVNVuSS(eZ_%{N1!)=k6`P>vVsb
zUs7Yo9<fW?t6%Ovw<JE9oplbIq}*hK<&2qiziQtc)X04PGWzY0TDG6ciBFyQ4{Gtr
zNSI^^y50EWqxR<0<|hdgs#xEc^ndsmYGtw_QGLVR^ZMy;ZdJ|`k_Zetb<20(t%{HB
zdkWn|Ro6WhxB0t2t)+eUj_>!Xi~m;cuK!;LTA<z|X$%@c0qtR3xNzalU&i{eyQcj6
z``ftYM?o8}wA*Ag-x5aQiL1Nnp2f9Jnlx$0V$Wi)trFfPC!_!Gju%qgkhpaEV~4G;
zZU!&ix+?Yg11tI67I$-Vj@+u)v8ngx{M|3jm75K-|4v!;>dC#y!S9y+t}OV&`RpyH
zQ1B_orOkEM<T#U__B)@rvi9wQHL543?3}4{Pgv#EFM)R_zI?tqjX~haY{|TNe-E-x
z3|r+cVRPev<1;b&#bOmV`W?4MzdaK_nRP~um1;BR?oHnflS{=C3Vxq0JK*&9weFuC
zU*7+oD*WipjXQN>DSymO%Z^TYw2W{5#}AA5&aka6TW6bo`SRtL*VoHGJv|+?z`gYK
zwU1ve`#=A+fFoeB8}GC8^Zoz-`&;>H<?<EL+d=z#FSPo&h=@Jizp?Uj+AH%6pH8*I
zC*!vrtlwkKe19QB_cs>BTW-6P!m^GY68Cg;I`_g<Pb!<$XVagX|Myloi^cJr65W=t
z`M|c`n+xAOW{uk;aYrD#K;`wH->YMeblvAS&F53px_o|>uid0eiMI`fnbfi`J-l#^
zA&y~rt5SpU{NCzExl3}sZPYp0Z~lDm&GQ*rKKGxhpX8mmH0Ms}MZdc3^CMr@|Nr@V
z>b9bc?*EI*9Or8<I^a7c|3Bwj`4?vnwQ?6<{rT$iPYKH+m8D)&laBYv>csAvGCjU-
z=86?7vfu5#y(Kd^Y+a1!>uYO4hpDWM+M4wG+S*+!K&xJ2Y^=-PEGXN{CbG(U!s|z~
z;wRO%StdT?l8@tgmm$-&^X74>19kICH=ld4()RPs^x0qEJgxip=O6oW(VCTv$8<gz
z6*cx7cK?0SZ@OuB$J*yEma`dW==?u;_ylX*F`Z9_b>$O;6rbPTbn^M?nue)jon_nB
zp8Iv1p<)wvRpF<4``2Fc7KLhDFEc8?e|eg!_q3R=&sSy6s`+x!9dwZ*=!~Vv&1oOc
zS-%h1l5sKnmvzq8sFzn(3VV8bhO7u^lrqh_u(!ILPsU;a_nZz5om2nM-`iWgbLqF-
zC;o3-7q*m5&)FZzXu$s~vhof~)0yarcWW}vbLPn<d_Txg$CDLuCNASh1jo^ov*#2d
z4$j(~=aZ<tkY(n6Lk~BGoO5s1y-BvzofIiInU}q}C071WvUwCI|JDoJlO;I)m}MLu
zJaIN>3p-G2al(*CYAfH;G8Pft%;#&D@tvsbIF@JJp<S|Tcg^+nb#<qim>8d_vj3L|
ziA{gUEOhGT#>ziB9{WwU_CXtRkC&BiE_-`x^|zS0_Vsq0oSdK$c+duUN#isDaq;E)
zA0?ffok4fBy}q{gaNhRdeKnPAJQ5C7RlB;BmBR1w-xZImc*r{S;i<A89}=^!tn!Zd
z$*vLg`r`k(_A>6h@5+Mg4mouOg&q3-`{#!Zj<(?w%+1+$>|Qr{+4bm+5jr8-9ILOT
zZ9L1rUDaY`=pC<a3+dNl>7O3P?btV?w=4Cnipa)`Z`8EyYi6H(x2Wvaw9Su8A~nQY
zv_2{xvEt<tH;MQZEY^DZ#?FmP=BkOlP09Q8;hG}%|JAP*&ZGrg;4%*{4$wHWpuldj
z|77M6%{+BQyOv+(5kH=N_St92u{~s4@XuueNtc=b?0ElV??g4phhdNQ)ZM#XRu<9I
zzxlGRBFpl#3U=2H{hRA3Wmb^z=T)-bsim{$>FdOMIxX@ts!DL5Wxjjzj2SalKeLJ5
zU8W0K*8l3tN}KO@ia`U?kB)Y~=Kf>8@Z-1J`O8CBht+&MDsJ=jO0b0t->Q(6O;@u*
z8(p4sCCr$$;#=3WS+l&7*6w2AHd}Dy?ca4@E_Vh67W|n}dF@8`x`5^;zZ>P}3e4-5
zdbBY9yX^e-Pw5U*4Jp2v>F=)R1so}BT>oF#P>`)+Q~TqcSH7+NJEgXpZDY;J{wX?<
zpStW`-QJ%6eCP9dpc9us`?vh&S|x7IT+mvde7KF5N5Y^X_x3i!loJBCw&ivwN`Obe
zPt9K)yI!%H$voiIxkRmFrfR82UHWw851xA)z47+j@H@Y)YknAjMzMpJ`z@{h{_e}0
zo5IsiFJ&$l{PW>3|Cjgo<r~7|Yp4GD`g*m7Qqj#9^NwoY`SsOPWbNN6T!p+^MaAlU
z^7eH{?;f8vapJ)T2b(p+*U3aiMuJYQcs{S%&+(%YXzIfL&j)7P>TgdTw#$Q#bz2j?
z-48PNWO-cl3hT@7za`D{WI$!k-(O!r*Ty|O+#VnLJv1)1{oaFl)$bC&zPf6Xd8x&%
zSE}%S?e~qz$N5CvK?Uupkk4uC>$mbfT{>;yo=a9e;Zg0ER=4#A{a@5>Vspyt{j`NW
z`|o>d^6c8q{rv4Q@qG(_u&__w)mm76!}afd<Gr_bR_xlcPwe?D!(=w|yCuS)o&A@W
z`=@_;ax(nS-YGj<@8&&!*e;*;_t)1c8iC12e~81@0yMR>uvC40b#$V#dx#9^%p&d6
zlSJdL{o(;t>|1OmpP&49%Gw7@;-3cy{ZCA=DbzVp*xh)TQ_km`?3c~oD?Y4W95ZEV
znCj-}TYd7}o8AXB$ch)Hi7KD+ic!hAD}UwSInZp5kgzc5#F_Q`erXjyJF~Fp=_#!<
zsRD9x=RTjeKmY3L>XT>AEXldK33P~KPY(|?L?+HMo?86MaB`^hW{HhD+#7;cw>-V#
zao13gP5pgbcc+ntXyG}xH*8Ct9)6JQcdOrd;IOy%dAUEkJX##jtQ5%lQ1UrI%Pn>O
zrrq`D7rp1v|9vjPUhU9=kSmsVw^^>|?>EpW3R_^>(QW(JjxlS_oLm0RcR!xi-+y6C
zX0TPsivx%G?axe7_0G7kppmPXwWGHebY2JOs54N4xU|&!^EvDH0vs$MGJ-$-Gitbh
zhHolzTc<Zg_+D2?q0!2HtvvmnGcG6GHsWtAQar*WSgE@t|NEyWdvqRYw_Py4cx>L^
zfISVYHjZA$ZoZuRLgCAV{ygK_7V%9Vos`)l-J4v*Hx!jjZjY>!F7#WlX>y&P{JbW`
zBTt(n<#+zHxx1ixb%2(o@1>0s?q?c3{>*bIyY<qB4O6#lQJ6SU>_Eyv9dr3#xjlJO
zhtAD)eB>4BpSu{+V*Gb`<==HBnwpwl-rt|Uv-tU^FPHs4KWvv55)xV@{Ew^TFt52m
z>M0S>@jUzM{#HEhH3!WnmEW)3-leQ0J~c}0sr)Iq%#~@j?6uLY9}^<AGplQQB3>Ql
zUy}FH=wCX2HP<c8Y)+%pJhkrcES<aTTEbX!zxePxY|oVBnKSd?^y;RB_WS0)BCAz8
zM1>717IjWOv&84uxie2z$2xLN+qN-v<@B?RH@+3C>HnQ`bo=ei%WF?EW!-zVq{?pc
zLHV>xzn#rj-xZ76R`BWW`N)dK#nF3gEbN*^c=9?n1}xt*x8n@Mdi@U{E`DPuyZv(I
z-)VV!%%aml2X<^eZ|8lqOSH4IQ&Cg1^SFF{jN?Zw(C%E&@i!M1I!hWPFqmdvd-8mK
zJ!HF~o;q`W+Jv>ywLP<+Ejykgzngz1i$Ub9C+RPbF;uSK{OG&$+1bt=r@m~?n%qC7
zCM<^~qDrS)d-?N8W|otTKWr7uTG(<mCG}iMP4T{S4-Rt5@lC$VQ&%+aWyto*S9?`N
zwm-W6SY0*!<R#g<4bdywIA^T!<YQ2K#~{#fb^mY0OXrrXOBA{J?(yX02?EoNx864`
z`#t^o>!(|<$1N^-c?ooDx2Sg5heO=@pxNTff3z-dNOYd27wa`$FLuZKeZN6d<tI;`
zgp`>}`MVM})s>3PSN$1T_U4V@UH@~EZ?`n*n;w|K8t10>!7y#({5^?m2>~x`_z&MW
z_e;F_HTRr--wo7GIsLP*wO5-dCmg+Is*<I5nd~XPS^J9bT|Qa6<o)N*&zSgsH;ZLm
z=YRb2QrN4h+ln-9)}JkLWN&s<^PT1L^3u|uFPG1sVV*CSzwf6SE4SDc%{tS1Yu@Iv
zx3@Nyy^R7LF*4Vx^u?v6+|}RS1Qvp3hV*(TuWgKc?$en5t9V-VCgwYB&5noG@ju)s
zy|$%2>cwrto+2^f4FUU$%BoJkKf~AN`euS4!|clke9SG`LiSxf8M~3~<5Q=Km*)iL
zSpME<#gnv1N5Eq5e&6=ygzZ7LTGf$fXXi{1OZeBh`Tl144N;2wn&uu!Xs{FZoA9Jj
z`qT~0Z1c!zHMu2@=VuzXcZq6$0_{+){FGw%??>{fX}Z7V-(5VtD)X_VX_m-hH9eKL
z{PpFVe;j_Zzkl8?p7--}R0~S9=l50h9lUJ*?_s`sn6KKtOWJQf%*y#!XXRY|CE?ey
zE8YJ*zv=w%Nj&Ukn0LqGnD37!k^FD{ORwmD1{r$Tt+py_=Jyw2cCx08Dc>z8r_8<K
zF<&o#^Lu-nyE1(2{x%<3JUu;K+}(p0I<fxx^3pjgYn5ISsN4E~?vF0TEgETzJhQfz
z%#+hQp0}{vTb+;RUCzrTU9!*qJWy3Vy&&psz_Ahz<(%&JiKq9Rxh=ZD<!OLgDev;Q
zHLOi+zEW11K^K@R-+oz{@!=bje%$h@V!AOlvB!^{nJE|4aQLN~>*_Y6B@3I|SKVT4
zZrk3zA!TCw{BN^OvW1T=n3iaAHrwc(xBl9E{)3H<i`n?z`EV?35$#g5(3q1Nu+~wd
zR@zE4nSayC?1a-?AAI%dxvGwKiC$S5+<tFwwQ<1%2T=Fp>#M8!2j|I#<X&$3Ag}iE
z-3Pfdw~HRF+5f9tcggPy=9Tpu=U)4+_UOmtwGoTE12u{jrTyll^IxC3Ec?5bi#3l|
z$eNbNzvT;xKc*SkZpvh}zN6N<+I;b{WX|McJrCR5&T2e5k;iyGc$w9=?`i_2-mzU4
zJT}$g1|RPl^3>HE{wrpF-ZisEI&f0N>-zYgHkF?YGA<~Dt%*3;$jlD9JNj@t|MAyf
zUp1*IiTe~g$3Oh^Y<Hj6-TFg<g?B@L{_A#;cwp{%(q``&(JX<ZH&6e2vsLruv-^71
z#k*POZg_j6b<QQWrHc~o8&2*CXqGm(Ij!>P)%=g%M?G);l)iZJ;{Ph?=xsR*^Y8Bi
zT{sO|)%N@Se)-i`t2*sFXZuIpJLMf+b7S`2Lz}{9J#9I}z}>!e`SB_78&tnNe*XQP
zt?it^17|N@VOaO#L@;B~=P7w#w%eTC@_U{7S>B23j2EXiFodnK{QYUqmI>^qpP&6V
z`MU2caV2Gc{%Il~*fv<L*>U-fg>r1f`5S&wv!vU#h1x~`u!n!T@HYSR5&!o=s+ZW*
z%`Rzfl-;&``G>saUN`kEH%{~0I^C$|?BY3ALC?&rLS+sGd#ek)x2lY8oh~^yd(-oY
z`IV2J{h9u!=GLA{V`g?f6-~`f@%WmF>~a+h-LAQ8N}sNmOr7vQ#Os&5^Fr&-@0%Z;
zydHOAlkxWIL`Ok|>R9uHw78j$ACqou$rS$h@nguUkj~QA*Dh{MK7Lw2gt18?_1%M&
z>bEDDuGY7IcQ!BIxL+sz)jhF2=0Eh#7*~0G*ZJQuE%%o}?kx|YM~`Og`sMIV-YF(0
z&S0W?)IFcGuQ)^A{pMt6X8)NKqW?&%BsJJ?`umyXo;mYA^VzcPJ2_$A_JEbo+27dh
zXq;#p6==9n$+qf_^VeBdSAKh>bLT+kU2lmik1LP$z3_K>>Q}S<FNeKf&D^O6r<PQG
zDy_Tsy77a>F55C;Z}&artGa)Fe{ZM${(!ZG_ND5-^*YY4)RwJTb>OY&=h$hV7v|jD
z1lr5&Gsj}$_4xX^$K|Sh#5F-v@jIuzy}y6{ks~hi>V9Q9IXmmWuKp-%wDQ^-(X~9>
zwf50W2YzKvUU6@!VC#oDeIMP!irlA9e`#K5(r;tb!1~_ykT$>Fp;=qrB#6a4e)6*a
z`o^4N9$sgp4xPxqyfRk1I&5Y!d!K2)&ck{8yemJfd}nj@cjX`MxTllX8g9vp6+e8d
z<=y(4W2?@`A2Y9I+Q0kmnH`@47u~iw^Vr|UPrR7-JkQU@mTS2Of_l&2d+EH^>1O%)
zu-tpHP2Dp-9oBmwVH|(sH1qcVf4{V|hn-J7J~vnM@s2&kb?RzrMg<QX%HH2wtKSJ)
zIdfvF5_9xE>CC@5yf*KBlh5Ast@(KF>y{N0#P`4dQvBlG-`fX*-d+iQb3Js|+^lD2
z*Oo7N^nK^fjp^B57CeVe-#Nu>-oD{&gj!$6htKZ}bARhI<V61zUcK#>gxMQjsaScx
zeK%?%yp9K+NqwLb*OuiL_u<W>-w6%L){`Q(lyIB#*7pkDU7d6`bJ>?_zwJ@Cxzvre
zWNK`lH~;XmC9HpHp4Wzy{EcP!^67KCaYlW)>KZFamnT|_Bx78jOgj0+;i+2eO0zY_
zvQm1B-ga0XVQ@eH*`Vv~m8&Pcb|lXE{%Pv=8N4<B?C;J${{HBOt54Tizqcv9|M=kB
z>XQsj`{!>y-yq);WRhEG9_q)ofAMdRo7MZGrDd+Xy~1An&EeJ^cKtIE-@Y>%Ea(39
z)O`KrP6efK(XBeCToW!V+&DQrs_j$I>43OQ*;CrO&Xp<$-Tu{ASe0eIc=GmKqDlXc
z_xEzIO8L2&Z+*QXHt44D!5g^=Pgc7<{~arFP3=zfZV86E`!D0yuVY%OAsch_Pw36>
znzP)?KJ5*h{p#rF2L0aqm;LW0R80|`U%g}ZwXYV%e9O;fGHiRJc`%pn(-E=bU-ll`
z@bdMCN2#CRT$kMWL_=|&a713glRr`4ie(Dio-b$Ge|hEc-%ED<opa+Wv+caO%!xPo
zkN#ke>yxvw{or45^VgsEY3$RA`h%9Asf;qqH8bd+zpMV!fnTj}R@|Rvr02Z688l+C
z_1Kf<^|e11traPp;WPJb;=^+ap{uPb_pW{X<?`BkZI4fa9Gj*8ln0($aIe>F4|}s!
z&UCK_0-KNXCg1qf+EbG#lifG#&ZU2>v%Y5eXSN^I|9o%Z{SB%tlj4(ZTv__#Q)u#?
zD~&zx%d8I{>s{>LKeOjY4u|<Jo=@R9ZO`ZWeUUYtCT?Z#bmfJ9#_W3i$;)ea4^FN5
z^V6$t=4U?r+>eHz))wZRjc(uQfA{DX$M2H`raSC5E&P(1Vw85}Hj7#PY4g*QXW0I;
z-NqLF>0^(`t$K?YlTN!|Tv4T(WitD)bAHJ}kB=4a%&%WqmlnG3UhT8V0f8oa_RD^~
zeb4dCPWhi1M!{|UiA#4*-5xjJ;ZvZweg6!v-`8F>iA`L+!PCR}dCbjq^A;BK{3<I8
ze6(PC-sa7>gQD(>9-GVh;IgJ<&0OQ&IM`_OF%uoB?1Mi1lO}Pxt(aOUJEz9-=-H&z
zj&c2-rzHEgwYf`7*>Lcf@w}?C)+TG=E;mV;vz3Q_E(r=`TdD5DH+PPo!Kq`rb7g8w
zkDd(=KX|3?|NVu<VtO$r7C1Kl`1N}I`a_%Vdq>R@cTag@nXJ!Nuzcr1(azUQlfw_Z
zK303~f@S?iqrC0nw`cz5-BI{^?hcv%cCRY$W<EG0*Z17Zp!{8on(wS5Z*FcbT_`$n
z|B`C4r;n^GwOj8x#n^ERB!s(ttN)(+QbnxeM7Cn>?&<2!f4;n{*Hd$xS)w(!Vp^)f
z3v-U70{?9B+_^V{x&QqrWfLxQye_s-WtR_M@=}K|G5uQk1=-H*tTAri>ef1hHE_zO
zZ`!+H?(+PtkH1Sk>2I0JZCsh1ZBWgZ@qMc~`<JUH9aEnCY<sco(XaO6@Mu{*C0X%3
zM(c{hx9!rinDOuB^1{r0xq>^bR!T?dWd)RoY%$)FEAhURZ}A5G1KZ=Pf3U8IzU>(4
zJwdbBIC77=$Ca;&TeLEk#;EAbegAOh&bUj(ZNeKFtgls86x6U;8*0QJXcEj)G`!~Y
zsbB6_-Mhv;W?E^#B5OgzaX-GW^q>DUA#;0C{9OULoCgZU-Ti^wHgAqExV2XC$?*ko
zncBa%#aI_Rbc2@Eotk=>>t1tRn7r9WM$MCUcK1@5*rs+zeEhxe*;z%G)L)TXPafY@
z>-deGC#sAu-9d8l$<_X5-m})+mtyd-i4+LF&^1$X!F6}ZH@a^VLT0}So3FF!`4lzn
z6SGPzHeNsf%Va}aga(tv=bJm!@41~ylRrJ_k4)#Lplf-XzDUhvj&gB%k`!RRB_>7f
zvUr7t%<Dx#R|Dj%ryY#mKigvM18-h~w1X$51$c#$6R#|n+>-lXK~|)?^WxeszpSiJ
z+&ur{6E};`hUk~q?L=Ou@!P7U8*ZB|Y;C#sUizOWmU9G;h%qtxO#A=f;p92h%DoZX
zXSFOhfQAUK2MP%a?hMF^oYH-a-@<A0dt;fescP+wS-f*h`VDRt7<k<Hs%TTTc<P4$
z?po`%i4*So^+&JinK|*~hplxvhGAS5yfRO=NIc7#%pjtjy_iAmbl7jr;EQe=3H+vh
ztp4feUd{<PaHvnWt}pSzic*cO$_p|)t@BMw1Kw6?w@<ptT3cGQ`N!dpJyPa<g^Sw^
z>uwqC`F~zlF4A=753ltb4p%=s|3fz7kKTFTyDJVWg3eR;-{biEnBS>ejTzSKO|}S6
zlT5f!9=9hV<ECBDi?xwUQY1rG%buFSm)Mf~@x!-w3_rUqE0%7Yye)Ojn|m{k-sP2T
zaQqwXv?B1LU!=7K*R2lgN9wZ8X>Si#?wKQM&(=H3S~JJ+VAfLq-2rj$W!!G)$*))a
z`7>uG|NgsP&%Zt7$xBFoxJRe!mhZHc+`qJ*KDt(RcH@52Y2`uMpAu%f$oxOAYqd_>
z_xiigj&nQ1znxen`ljH2Xu+3HpN|x}Og**!u%gm+ky7W=t_hzmUY}W0^6t_7%`eRU
z^%Ultr3b1UV%YELb*`ZF?(z92<ou2&Z_*Vmx+UZvIqQk_*Z0NhJ2aC{*R9pQ(%>_B
zSMOm3frKa1<@#>>1QqR?5w804#V*b6MGq%bo_S)hp(d)~?yIJKRguak!_T@M;&uIJ
zqrrAZ#+Ua&*T2UF)|r`kGV3loUvCj#>v=wpk*VST1Fx$4*3;DMRylRfIlbuji@<`X
zmbNkHPD(}`%VHA~+_`N<YM|cADPJx-eq7>w+>h<({)wf(w!S;_=A+2lumfE4&Rll<
zc;;*Qa~9WzyT?tIWaMp2|LMLZ=W=`IWw%KJ;>WLCWiRrZwzlWZAA@A|zNh!RbU|}Q
zi}y#K6kl8=7`{+@_8qypIX|qlo3&qldGh+tp6?3rJg-ju{yhCd-tJXCw@PFhm{&jF
z^!VJiDTNs@2d#^GxY2Bt+ku6T*<z*I4gO@`UzwnBUF`D-uXis#8OiJ1beQVVEH-t~
z&bJ5McYJ&<`Q(40k?>{d4a=VIeY10+eEGNY?)L?+a-7`Hc`v?&m)$gad*jN@Ki|x?
zn{w`sN4Mm{J$s({ox0ljZ1og|2Tz~#r(gIvF>h8Qc;)CiF?GIQFTZb$Zxy)ndwb*3
zTPv5pSiic&`s$gFYy3Tz-rDG#@aqGE>*@O}ZT%kFGX$4EFaGxYj_$+nGX&=Kx0(NO
zdENEV=I^oPTk>b{2VGAqcf3Ev|0rn7_(X5jeRJmaU3IqDV>Nl9v}yR}Kheh{wtvm|
z#`^Y(=vl8F6Ys{>1fRZlrS7X-6;E++)p@O7dwv_~JvcHyZ@t3Y<!6>YnPOh~Yvr~x
zwU1)mrpKM=`<1KtI7oa8sD&6~Bwfz-@w%nuo8A+xvFn1)M?Fis`CH$}e2?DJ=EpPk
zzhW@IrqxiHqJCd`Yvq;-Ilm=32@}@}&oAHAy=NQW5x4a7{BO5B&$*hJu(_S_0e^+%
zr`uk3(ZbH#-Q$*JEv>oocHWUgr;Ic1uk8jcuuTknvR_OtG2jrJbpOm5GaZAN6L=ZQ
znuGjqOv{^Bes^*VV(vD*cV>jvDTS{sCl~5mwY<(Llk{!3vc#T4+}gh{S5^il9X@1X
zP|c^U8#;5t3QOD5sfD*|w@?1~<MEQ3@7&6E(uaEWnwhKXj-L57W7^mI_FH!z6jm=T
zd3RkP=H&7D>r)Qce!P?aZDH)$RbOQ7`+D`aC2q0g_&qB{u70iTdYj0e{`*&gFDMn|
z8u?tlqP!@4!=9h(76*NwU*Hn+_y6Km#j7W?cS--5l^39Q#Q*xs$S-f-EOJQ`P|~$%
zaGq^zvsuFdM@6|sH%vkMKuf>z&A1EaE^oi}@i)&W_d};zXYSc*T3(W^U$@di)4E>S
z%Ia7A-nxf}T0sX<t>6Fem(<T=6`*Y@9X6IFepc~cDuukCKfDuqr|owB;mMzL*XwJY
z(7wBBuCT}59e>XSJ3p<seCdOgQfCQE!SbJlu2w~%OF?6BOFSpfu&v&vw?*mGbd!?4
zm{s>xuju6~Cok{6o}RYpa?-*pEC28RU77WV^Vu7Iq2Nny69c$AH;PTF|8n7bG5?dT
zfrgjQzj~}DkaBFfdPU8&cAwo-f4^$`kig77GrFcH>OrDC+toZ?$HQvzt7Tl=<{2J)
z)H%1tpxV+?d)mg#=W=ELc1_U~o*uCOVQ+oS%B!VSKCTA@B|XdRwmjRlKm#<w60R_r
zcQ=#EH2&0@Z1(%LPZOmRHXnF5NArhKjb(aDT0@+C>W1xGngeq;{a2ssdQPMv((&4o
z!<~EUt}SmgJ>n#IVCMN_wja%c%zj5qdvaaCYLc!^-}khh#UJK9tuaVsy01F-!Mo>i
ziw=H$xT?3~1nVW+?xK@5miL{*LfLh-Zd5FBmQWPl<bGq@+}~Fk?t1P2wlKoH`h}ro
z_r|+(Zk&8qYxnv2*@ry}+XOfgHvM7epZ|pCRPK?e4(`_8nX7g>>@(W>;?w>pt(J}x
z`<F~}-RZ*fYkTRY%9ZWy%PJYU4bQzf`FD+FqM>&Bt!FDXnQ<j-%Go;KNHn*F>07|#
zD`j&mm*+V#a$9fvVcF)m_}aoNeGQ9L-|o43#j&I5(bR);4=q1jb#YU1`mesK`TWOy
z6ITEJUpaeAIs5<a^#{_U|D}AGeZWx6Pujn~I!?hdJ>Gqr&ZE|Ib6?)s$-da`&xFFq
zX_J>NR#qx<ys}ASde8l=r!%)LeDk5f{OtLUlN6_HXi`4txVlGYmDt9Hs}I(l(79xo
zam{Xy)$!&Zh8b7xoSamv7P0nArueCk0osbYYk2%V6)W4xdGahQ{#~GOVoj#xn!ugw
z85qU(mGQ4{+k2*8IPdW0pV_bOeCA1>RlZ=$Yx%gUOJy7EOzlOx+M3TgPF>d@Wb}SM
z&#si-MInyd+}qzRv2IA$*1vPP`H1KJk4{b&tL<uM>Ty4+d>|;R2wK{=vuVa2qn|90
z>9A?WGUu99-mf&ZyItkxeesk|*Nsijo61_HCS(R}FO*<m>iKi&aN+HjI~O2TfyZJl
z6zxpikX|dM`kv!3>yA@q&u%~5Z=n3tqLlk-y92|Tus!cg3fzj0ZC2gUFQ@w8l=G6q
zTuVN??hliB^~1L;VvE?4-1F`nb0od*W-eh{<KmvyzfeN9`q#_C^9OTl^4WIU9%9|G
z;#L`h{u<d+dh1X2@-cHvC^Hd_xfpY!Scmb_r8^ce>HE|7<Xze$F!QtAzb<oK-t|{q
zmv}3icb%B(fA>O>=T76Z!HysGQ&ZWGPF`Q={4FFe>_f^tzOM9j`=cE{atXUVoc&g1
z|4#v1_wz~f#kuZ(RGF(C^`q=g@sZFq9;eUE^;{7*xy4I&#?F@<OVo_FJhknaY1aCD
zby(V?Pt0#3|8V{luaYoM5eTlo<$7uQ`wL2X^E6-gGslVb*Um6&R{wBt!Y!3Ozs^P`
z=QCMw-#sz6t@U}dn$gGKXITm^Bn9Mq-Qp?8Pt7>Z;#Ozb&dD4t82>TSY{{DRl;>I@
zudJBOGiwarC_d7%@!NF!$&q)eJg+|5B)fl6Us$BR@AkaE%;}%!l)o_i;M?|PPpg-$
zeM0<!v&-iSzp%{FdF=L0Nl&luRBif0-ZzggC)P?@mFdW?4o~_pUAlA@X#d8EqXKf-
zi?di{tnw}_-Pm?)r&Ywt|7UX8w(Fg5D*tt9K^FHi{rml=1%2m=`D*q$em)+T`uNz<
zFZq)WH82)i%=jc6TyFBbzW>XUnRTX>pKtey?AXfu_f*S;^cSn|lzSUKk}_YoE%D?#
z@sByrd$&)!w9(zainogY{gjV?>fYH-+$Qv8zIC59FZbd3JKD8k-8LM2dmyp&^Go6H
zlddM4F<aR?oKK&3$b{c9#OCbmtA}msm^zk3zJ500?;bnjcQsR9nDKx9A>4oYXv|?_
z=VuRXH?{mves9OTc*Qk?zW?UCPpaKixx%7($(qrrzhp^AiGYja#AO|<EPAdlnWirb
z5)pWLiRESJsxSAP-^+R`OxQHRciSC)P7aQ3-kz0p^G|G>`T5+LPj4)Z)9?MLvDUTT
zzWj52RZ_fpkL2z36PG>n*!S|@yf)QG>YY*_U%rlsw`^OuZPS+jOKk7%o}TLW{$5Lj
z>D9HMX275Oi=5wnnN{1quK3-DpQj?eDyYre^TPk|zT%rZ9$2p`X#1BNmwavE?22Vx
z-dWoxP2u!mU$U^Q`<#W}zf))5<lfx1#)NzS|4C8I+aBzZXcM(AbXa5iZwl*TOJ2#(
zDm?eAo~7SASDJFmeyZHveaz0!j?asJ{JAKytt}`0*@QBSGQUMPS0COfQaLTY?5NP3
z@2U(u)3)51z06wAOo~U2;l|^n$MQWpPptQ2-+oR$<?y%3BEPCEly6N}d3HyT!SDUO
zL-T^w+W7vq&9aPNbZoY0!S0>2`EU0xzqg<=<lK`74_?KenEHy3-7M!>+AZ@RX(j%f
zUe;Y!nsnmj9M_3YY8qzy%(1JT)h(_+$u#@g^IdOdY}TGxyHR1SR=RTRy;D8+40`3J
z-cP=G&)ej*bbVphUA`FypFC|msH+_l<8^b{>Ys&&J*K5}EO@9Rw?qDO^11iLF=-_Q
zxo__4B%WBI7190e_`eh0eP?9*x87o(;+MTJW3GSX!@R{|(^uwLh4MC7n^hkQe>i`>
z{fnX-2Pe$7;*}2g6L)}n|FO=)W<1MIU0EOd`(Mg~h0oU|9X`<Y@2%zJ{mPSHJ!fCk
z(N{K6wc5ZmE8V*yRV4gy@(1tVHSZ_9E3mk+A(2^a@;`2YiTjt$bNzXnC(g|-|IOJC
z%kJ^yKRSBAE$-w0yjN#GsLc!cQuQp`;rI4hiQl_^+gF936iPd`^=o2Mc7D~Af3^ZY
zSR^EW%xmq;)#X{A{O@m{(MgY+n^H3`FXMfBYU*P5ez}(RcKhwA)oG^=F5~-qLtM`O
zOqR#%e(?$F-m`v)TYMLoSx|eqGrs=V;>gfBx6_i(M9;dD$=edQgJ)&ZM%9>;Le<~-
zww)-*ad?xdE~jRE@wmD1$$&YtQa1|~W?1iMDm0Ux_wM7~CN+QNm&V6FI{t6{em>P_
zs-CFCrwQt>y;F=Oetde)>%wI2|0c2gQ1+XR4{U^6c{yzB_3Zx4x)ji|dCMz<&eLJm
z7sbslExcW%6S?V8m+ZL<^Z64FAL~qwH!O;_Fu&Hn`1^W&<B#oTOIW}8%xluzVUb-`
zdu-NZ^THeJ43|6W_Z_jhUz_yu($Xn9k&~3&`wR*mIMi+gtzWWn-N&|RdGp-c$q{7-
zUB&8^WBQ-ApEZ+N+4JOE?KSNOkD^x0UA_JPt9|F$r1mE%U5Q+D0JavuR$#*IX8*2T
z8%vJLmfdOjbJS+#bM76|k~c1N&zQXN&Cj!)^=ai+{|^hy5}Dznw(rj;?>;$Oub3Da
zM@L7{VX?c*=O1>N^3Lu_@!RAF^;RGLK6(Fhi-)e*^`HE2wU2!-_$9%!rG7vAeg3x|
z-MLpI?rdaw$$j-mVU0v;_m<SfZ&hWi+df|1)SaUD_S}o_t3G{vb66(QVp^h~_3LG4
z9wn_hbfxU=)3-*gl7CKnezN#}Vbj};WoKh~&fPsTZ(G;ym6wy^uebBrwi*2Vt*?2#
zNJN+I%hHzrr?<#mm%QEoe_>97!e8;X3_3F!V%D1}m%qu-Vq-J<I>&lx%=XCL-=xl_
z@hCh$7kEWS_=tAt^w0myb5Bg#to7LHdV1dfat*<VrtRNZFMZ}UUTpoZBYK`r&V1{6
z3;*7}R2I#!-t_gc<4g4}tPE7`nzrKHTl3vPwwFJ@4860j)*7^r^uhwi9x2nV`1-$7
z4;*l)UEg^kzWnNr7`?f%b32y$O>bQAE!yCBTE)ZHLK7~}-Cir(WA#EsVL$s}t;&L$
z*RpLgWs^)VG^@E;$II6^O(<r6bm0DV21C6=kK}G}?&Wz>m0%^?ptFxn&;9S$b(tTT
zUdL8g_BMWq{uu5XzUqPN>L!DlKe2zjc09HcIo@YsP%(4Hr9)>fMTR*Q&8ob9-tYa{
zk9Wh)N>v3b&+Tp$eqUrInEU@M^X9^zha@j8^ZgxS_x{St;F^CwpTD`WvH86H|2bm1
zQ3)n?yI)2>N<ZE?f7Y?QO%uM|yJKQ_JTtUPWA64IWpzFsejoG48zMK()}6sP`)b?G
zBsQ6dwPiZfj-SbWydiPx><u$|EpoyPqUGcQ%5H3sJUc@+?2vNK4BgDi56LoS+J+Gu
zD&{=Dyy)-Ggx@J#(LDUM@8qYYZ%wTFAa{jxe)8Yh!OtVsPE6SJ=i}!2jgOD_UtaDn
z@8skJ+Jt;|w)ye1zJFPiKJi=0g)sfx?R5T_q1MCu^ZMU7i1QyWTD;IaxcP3!9wG1c
z)Qu(;ce~jldFSkAw7j%CvA;O6$ecmXY1^?6UlZB4uC5BQUM$VWZ(bwIw{L#w^wm4h
z)I^?~yL9uSng0)nShh0X%@oc#rmZtg@Alj;Rd>&L&trRcWXFeZCJsyHI!4Rx2{rg<
z|A5cecIGC9HrYZ3wp&-soFiiXq|0r&_`7kN>oG>bJ{M7u`K>mUpA7Qv*?=w}y1p*<
z=eynSU)<ksU;64wrP}H4^9$Yn$Z_so*HKWmJ5RV>vitsF@d^ofj_PN>SDx#5>RBu#
zb@op6sf=wl-ztq8E;`-XGVQoc?m?y->th~-U(b!Q{P$yLeZ;+FwG@+Cy7k5L+%`n~
znfG(szN)WU%<OzAN4rEz-rt-1^Yim$v)Nx=)s+6vsEa!lY@9o@SL1-7?ET#m>~ro5
z9^Ub`@J8cl_T&u9e|+j|*tpoMKNK}s`xZ&x-11NFa@o<Q<rB@lq&WGv7eDgSc`j!;
zUHi#~xqr>)N)!spW~HC>WPk8vI&0Xj+#XhYW!Jo;8$ZkumMN<`=gHCaF7Hd-ai^ap
zY#+*>-}UKVvt#$f>qfgbR<E}`nzQ=SQt!=$kKOkD`;`q^+qX6A>Wz)b$3I8zcq~=C
zeBIN#<xP>s@2bmUdn-S(cG?`>A}L+@eXhbbox7d7ocXsqIygYvbY~i+f^JS*?l*Ul
z_jEn3E~QWFSEdKfn`gnke@^iwmHF03%`GZb7dKwjp4(C>S@0m~{>C4AvuCnD`mt$F
zr`FELyK{>~K0HuWma&jM{Z!j;k#j6h+_Q=^Ofh$Q9{+P)y*z*Cfrm-)A}z5635Wm2
zePdQIJ>tJn_<72Ko{So!nG$i=CkL_KnOXPsd*O%guVxp{-Ddb~jq%akO;KC30zr4L
z`Oo(|Jxy0LYKz9--`~~i|NS|ixA0hx$=iGCg`$V_`E!MCTm5x5o89|<|NnV{%5DMc
z<7}OroK8&9oV;(k%ai}h=DAj`3d`v~+BR{TukFT`V;f&KK7ZG`V}^wI1Eq7E4^j?U
zNh^onme{E`tyr$jes00<gYo9`J}lon*XCBdXzkT=pS1n<$nVJ7UB;{L<#Oy|Vr)&t
zmvaBcc&5C(J>TzD&#@})@|vpk@mBWwj~|cA7r(t_`u2in!{wJL&(6$bVPyraUo*?Q
zbK*#+uqb#FecYO8PUUUi+^ra51IxnV3Lj0s)%*Cj+>S0*=9NNDQA?fFcDzlv%d4KS
z<oRp4CC44!u8P|;{l14!!^C@~XGH3K3U)v4n9s+`VEmv!wZ)0q?SbDJX4~#KHmzqy
zLAt9S{(C1k=VtGO#UHg}cWx5*-E?TuVLq*=CZFPD^>2y#>}+qjne)v4|4;tqesh%+
z6dHn-dR^R}AFrgOwB@jR(CV<YkEVvlC7zn12|7RgXt#Lq0tZG>H<x)@yw0cXpG`jL
za=+-Kr(Suczg5+3iLKwiU)fO+E>rqyW<=fB#(Vq=l5F#f+$2vV6&h7;>%Fj>ZJUTu
zklkwb_^qECPFPf(?NiQOFf(T6-IEV9A0ONkX>nn0l;qt5cjxM#a=caZE}`UOTxPjU
zq^&{X0yk6sW3fdS6gFG`&wO~SQCncj&ED;6{PLK$^B*;MwPtd|9JwuL#OG_>U-^3O
z1{U_k(=CJleDj@aW$N6{_w=&AJ?M~d5#6XMda=7Mtc$g_v9T#(P+<byZ5O^iuBELF
zbO6Y!>+9Dq>NsJ%cxAEB@A^BP8L#H-U!9YxFD2xCo%bW-r3a}`ev8dpGq1Wm_Q29;
z{`W;<`plC5pI))_fAl{4faudlZin_8R~*b=xK5>f_JQTLrRVk+J~<&+_2q@5hX=>r
zs;`gMZohZwinjiVna1fauCJE|t(uPCU$-Ui?ys4CwJJ(XQf<?gzu4P$&!^`6A@-j<
zI(I)*`!81Pl}<Y#dtJUssp0Fo*9#YL{E2rIRM2_#JwZb3>7xQ6o~};Ug-)#zyGk@c
z>lZIA@tmR?J?-l1@YM$AFMfS}{pXv_=T`(R<x=&U^5DzM%X|L+`_0t_8lv$$y;7TR
zk&K?%+RmSU4~dnV?+Pzaf3vMy$aGJI`D({juSHrRD-L{ndmD83qHWce1z~HW-rU~q
z-&)zWXz^l6lMDe>?`cPlA8((m?*Hkic>IH>PnSOGI8nSf@r8?C@HCl}HUC6kehv>e
z43c==Yy3t=^vv21vw!^fp|Yy9*g^(0i?6@`&!h(r5<vH!o||L2_2RrqJ9b!rX1X>c
z9$pc**9x=}B6=I>IEOs~uOfY(r1{N1v(oi#=Z-Drk=r-F(N1GC|Cm&u?5|bT=-)80
z@qOfs8`lnfuu3Zuu21y+e=LqC=aFxJ_@wV!BMrCf6<0(tAFKHuyNJ6BbhTF5+gpXN
zu4s0NYP&r>H5IfXeBr`{RvG{K^`>`Mets6X$b~a(ecahqp{r-u)$Rfx%W`UR=*pnh
zo5sruQl3S-#9GhExh*M>ptAVz!ROpbH&V=vHY^l3R8PM7dA9KXGw(|3?#_L&y6o4v
z?uT6`d;c(=EWh5{yU*{$mS11qeUy~1`B1kgE<8U!AGDOTQ&|1Or%#JqI)y;PgfCuX
z$nN=np^aC1MZ`v>#qRxShgvxQd^)W!q8k+gUOW3Udu#Ufv`60;TiWb3ytW~w>U$FV
z4cU+BE>CnblwQ4vWy|Y-{5!QgQufsm>&=~a7<TLsd3wa}lTEeKJ3kA>VCfH;yUSl*
z+8I14@_ot6ONK>HJifiVd-~_+=b&;Gv|zVvcP#sdc#D!30$*QWU+&z_2deh&?k*2r
z>Lm(3&gAL*%v)PDmD_lqeCX%hDP3n6x&7-CN9n!h|LaUkQ)C1lK8W-2TgQB%ELx<!
zzNT3$|Jt(p{Yxre?|gG}vvKh=pRjc?C%Z(oHRASIaEs|&VEiK?t{=C=cea^H#)XDo
zUtbr$xnX#(=CdykTP%y35?A*p{jHgoXVuy67BE_Cw{691&BYhrxy7A+bL`#Tle^nb
z36@V!Homu{uGgsM^Vbb3_vbBGpa6=#A0Hn}8YZz!)e23zyv*0Cqj9!*{-wRu<)@}<
zyWiVWX<7TrWPZ&j&Uf$LY4w5T(*M8T+$a0ukEb-B|62Dub7!C2xA*;3^T$@X$!Bgp
zsZJMNZaXb`^X%Y_DSCB3>U=h3MPFSNdUJC+f9$T3hmZU1mn9x<ld-F@aFyF#{{CL-
zj}H$;w8MCoUp~1p`S^?b`|B6&Y4CJ$Ni^(yefKqYlxhBfhnc+_j=QMcF1xnwp=;*}
z3#&EV`=8IRH+oZ1Xem2q#`RfAJA)02e@#iTWXUv;sCd10JLs^Wt=ZR?#qF&EW#yP%
zC7P}>>-pN7HG`K0Eb$QRld(M1Cu=QXoW{e*$oR`hc%q8q_cp&pZTzx6At5H+=_U&U
zx?3e|_e>7`trh-lk!RS9jfWhU&T-o${^h|_Z!<PIA3NI~z3JV;+_xrM$I5Wm&71z_
z{jrDd-uboj$!1(#C3?7xcX7_mO`v<fjZ#mk+%|J}ary9e`~69V$!=o0Q7$WkmVSAv
zs^t0JeZSgMzgvlq=ZdYpx#2q#>+JIf&!+fo%FPToKHKV;#Vx%)WrgMJhgVhGeBxqX
zeEfc@e8BQPb6%mi*;D35UFu}Go>>?B-Med#^6{^)uS=TcNQmpj?D&6UsrPiyB@q7e
zY)(GDzT>3J>1nzvBR4PGw*j;!GLC2YWzWOM@;AQknfk|f`QtR7S#yu_w;kPIlUtkr
zYQmPok7mYKJ*ur%k2YU>`1<v5Ya;<MF}I0IuAuT$)q7fqql=3W=iJRRu9z>FE&ch~
z8I}LuY<{|b1g0Bj@?E~yS(^B%e1ARr^y53r*{5HRD|+mgbLno{^XvQv-)`^CiP`Vv
z_>oc8s$_v{w-{HKl2T_+mDuFY88al}*2Z;b_pN!d);@~WbBSK=&pvDSWgD}Pr^P<_
z&~UU{{PNmpb3Qp6(CvI&;0uDzJC&4}h-!tapIXkcGw%O-u1P0iZkkU05UYG+YppTp
zDq%;VJNs&9A2{HUc7C33E0<^pc<XfKwf`UY1?}TH=VN*zZtHTcNggq`?c1}Ze{D{F
z?N;>k)Y2v!&~Yjim6adwmfv3~p`_F~Vc9v)w4mjFXP3o?2mhMd74zkTYIZB1)+uhW
zsuaoT*S@8_UiOu%t3P6M8ZYP|A{7;vqXL4SOG+BOjh?)Je|}nTcBhTW%+uEDD;4xF
zoDF_$vHOkjwY71M2b)+!C$NKVjRYN6b(r7&g`S#{Qs<1eg_AcHKR*|^EXUO-<c;sL
zNBJui&fi>HeB0vR<=uZaB^joko)+pT_wnWO`AJ)&X4+J4>QYpi)R8G+C3$+HGW+bh
zSm&Eup$1hIlNC42>-sduCFa)gkCwU-7XQ^*j~>6VDV3XBT+e6Y_f9dvz=$a>%Ju*L
ztcl+|=~QU^;TP{*f7M(KoszEeYWrfDxnISMm-#Nqy1Gj1gmO|+QpV+FyvfIU6iZ)S
z(dy{%P+DFr7TmC4fkIv6z8ROfZwuFzb#r~<iOiq#@I-&ULERaSS674m-#z~F_V)Co
zqg{df>uN=vT~s(PSaxfkSik?D*IzeZ%^Nj*E93MQHrb>aZr3X;DAH}?e|7fv`|Y8Q
za>?iC*@Es;ac<{(`2PL+rj8C3##fO(j;lge9(p4gY+?0TchS0+F&mek6f2!OA@l3Y
z&z0(|N7X@>CzZawR`dC+IVg>Bbx(@8x-NEh=i~iPKYTeBaoBIAg1<7$UC!qpuKX(b
z@xj)r{Fe`FWz+wE-}k4SpEq~={kmvS{5hHM%ysgeXVZG_?(d)_U%B>NN(*)Tr@C9W
zYl7z2itoKyHGi)Lh(6(Ta&iJ)oXXB8qY%8z2NW(si&ifTUU6-0w4kW0@BaVSmS^0K
zH~5_v5cp5g{Z4G(*^0?*>shrm{zcA};fvay2Ra!I)YzS*!0Ws`LDss)>$(4}f{HBL
zNAK>0IR0Z?ZaeL<z3K9xm3MmNmsd;up9$Kub!BDn<z>Fo;PAX;u;s$m-_Lz!f97tD
z);;><<?^OKp#dc~EbYGAHf@?6X;RPUwm8tWvT3rqf6%%Z%d+?Pe80W93EENL>gu`b
zR{fnu*VX00D~>w#$lcRAA-p3-&+kmW`@1U>L5U4i_@A18WmV8pFO$qmE&ld@Ls$g`
zJr|kq=rZ*D%@bMA_Wb#cH)$4c%|xGY`su!OiMn1nO+R+Qf*;|(uiV&ZG1t%T=M!PD
zLxayRy7RomD)Yt<-EVU@PTK#n<WA47e=i-=!@jj-zwYSXSoGwCU`<U;$Q1P*1rM22
zy{Dy|o~HZc^mKoajfbvm*4Y1^Cok{JY~#s)^d&8JE|Aq*9_;E_P<J<>XW{G{dy}<6
z8$xWWzl8+;JM#Pee*fuuv7ocKCNX~1IVFGO$dM0OU!!7*Q&jW+Uew$tH+@^p>AA}G
zo3f9mF)=aiD19vkx(R5${rm?H65bm0&oD~mN<P+;`25`5T|eAGrROB=)mOcKdR^`K
zr0KZs`v2KUuL~-#i{!nPI=Sxc?30sJPyaps{OoM^`}=Bd?yt9(lasrZC=|21tarfz
z1<)3ViAt^^0)i7$91dUj6d~5#8h?G=k-x{n&aIc8V$^*8pJhv)?73;ZujRP97(lm%
z<lox^x_Q(3H1n<P`To1h-h#a%_*LiBd@gS8#b2~VKc2bXvqsc<*^IJwxyWO!Q)6%J
z%Cu$W5~+BTy)JGq=$>>?d1PQ<aP+-4=n~Nb4UC``>fW`Weeshf#q6z`8g%ffXY3>4
zO)T4ezrEdCSR}h#d|UG2U7;e{VQ>87trHKifEt|Ja&H$Dd+(I}YMyszLBYdAI&phu
zTw3b=dV!#zr<)0nZoOvkGM}8gTjp($Ro=1d{qmH%*P@?qE6V!E{%2Fo$9Oi^j{UE$
z-@ZLNaIxFc7qv^A+xblL@5ML<x~ND7onQ1P_jtej@|;$!l(m*~cpAS>iv6vm_xACN
zGv<*N`DUT3!~WiWZ(aP1=itGEZ*Fbvma!~avg0pksIuhUotdD*;px+*%&wpa3fjcX
z&KI!rRrV6yOEQx!LpNnkzbttAtF7?Nua8|`U0eIR|Gs_AkAiD!A`id1x;psEKLJ5O
z$Akn0<MeY&l8$y+Ie<3qyC(eq_xH@Kzrud$pKg_Ag{(gXx<YdOz1Ia6pj$sc4I<s>
zZ9ay-oo;W-1>N8E=Eg=V1F+5~=VfdvCP<lP6}-5hXq0kd!kgQ>%ic!a+Lr6>;lZ(Q
z-@aRn5OwRnytz3U)G_#c-d_E6S=8DvE^)mxI|?8FE6ra6T8!1T#KLTL?;_W3P}fV!
zBx6F(?VAh1S2)kKDqUr^3p88Q@kcm%Th7C^+wVOB-7)GlRqN)~{MXmkg3eu(v8$P3
zl6fhkxf?X;al-!V$K&$C($c3}IE5#L?oLWd;^5*+`tabONzRP`=_pXxx%}&0u|k81
znR%vFsaEK!5KptWuP!cTPd`5|_1&GFw@!f4jMAt75gU_OWv$D6%qEAvzq@<-t*zO`
zFD@uDF)?jvbae^hxckokiR8-A)nZG%r!VuLZ|AvBx3I8qOWxgC>-YWg0>yZ1;Q2)-
zj7uIL<7HxIj@*_bsp>t=XVY5=g9HZa@^>jWH>KWM0kO;E>>SI<4-dC*PCqZ_qZT|V
z`oFS!A84Bm=$7AUI+0r(U0s4Cc%8d#lF!aIS1*2chDX{==j7VmhaWB|eSM8b#v%Z0
zVbJ+SCyEPRUQ+d%qH*xq+1V$j>&H)ue7}DGzgeQ%VH?WcM)9yUn=Jtcsf{*hdH?eH
zb)W{Zr&0yzzOC@M%GSBo<&%ul{c;@yCyIC-zMypeq`Q1A%Yhw*kA?p3+7+V*I=Axt
zJX=sX9%S)Vr_07&-EYnUr&g{vcXv;p9$#m9aozJBg^y?0)mlYw%Q<+~cfTfhEzOgi
zpq}FGZMn%mKRxX%`l}bWr{e#gQ-6Mb&Tj7PSW@uHNTBlEuF}`XrfP@JFwG8I6tlnV
z?X8O|gVkAG1qB(eY}WYX1hT@a^p${+kkhX(FNM!mfsV2bT<o?n`}#T+OZ6q-^rRvk
zySwb^tE;Qe%&{~MTys8u|6jL@i(Ek$Njo|^-f{r9sy->*-CI5VP%F1_)t8K{tE;{q
zy1qPcv73la#Dqp>_N>Pp9Vcp~(uFD=_w3m-!yu7KH+tKX4-XIj`1NbwhSHxupU=0f
z`(v>={rt5yLBWabFPOw9im&jRs&!>WV6&WU)t&mB+uL$wtV&+Y)GPn@=ckLiJ9EM3
zXTDZtZ%#ZpIoWXb*;^V)N}mLHoxArWY|p>1W?(Sk<YaZ@>TfyA{N{SC3|e|)YxZ^H
z$a_b-#ZOMtmHzkd-<dg<ljZCG2-erv@B8^oI+sUK@Fz1^|H|Oy&p`JVJ2taf%=KgF
zle6tQ{j^GLJL{bdiOhC(b}g-~7Z*4(8>OE+)5a^kMbX9O$wm_%-A_!dt*wFU<80mf
zWKQ1QUB2;V&iDD&MNdu`M?O0>Rr@`^<Nck*?cL(~LJ|@u7P)qBaRi^-qhk2-_I7>m
z>3UDk&9&yYeOF?2b4%uB<H%>fzP^_Kbvtcy;p=O<Qf4_Petv$wH4vg@ql94+%h_3`
zhi~8hz3}+^dwWlJiE7XL^=VzqPN$TVCx_#i^74NC__3kjq0^@H^Xqz*z`3Hc^=3}m
zvokY$Bn+AK_1Cx0yS=OQb#VN>HePADUB=zhr;A$_KYQ}vV6#ca1qW&H5jU<vi}~$-
zH0VZe6A%z+Xm4j{*tl`y9LwToEA^fq{4Bb6@7~NC8x$246`!1$87%DxE}?`rKRVhC
z>SZ=HH-G&0?byqbO_iUYE#K8s{am}Mx*C)|($3B4T)upHm%fsc|B^~E)yVFyu1Bw~
zuJ+8D8>=U-?B191=f}q<Cnh#e)eg6tdY+e;cWKhmt|k8S=XDCJPqMH7rxGgOebfn*
zk%Pgdl#OlZ>FLRz`(*dbu`0d9z01(>h~?ZRwND;RuRr~+Kq#$Ot+}5+FV8bS;NhFI
z1$r?%9Ima2+?w3oVdFk)O7x_yGpx&J#H<lmdU;8h;iMf_zx(gm-w$$enI{jrjS>=i
zN=l%i9neAJT-_iNbd)z<nUnM0Jy6mNcL-(Lb>;Y74vQR%w-3_Hj@v)q;rSyxzR3K|
z&HKEUAD_)zqCZh^`rW>RQwlt!dL;@H`u8+zKK4Gc?$@jhuI~T;8cCHY)fQZ1pO$#M
z?xl*Lcio{f!&mwBeG3a}|J<+Q_20ib=4Xil-&<EDrIYij#8jVFX<E<Im+Y8hKQsFH
zoQkxQO|So~kNKY|{Nwq7IS+rz$ZfRmKPokut#}&4n(n#>zm|M#`DZX$;*a4y3+8+G
z{(o<j+MU64ZH7rj+Q}1hUF`+771=LST0g1q!1W({{(QH;Tl#~cP|g0%p0C!ypt4_O
z=legIcQ@)uyI=Xnd?WA3{bGxEpA_FzKP#+#>FLAz>F9pF{5l`~-*#W(n(P1EKep$l
zq0NtqYc)HbpZxHlN#|xE|C#^o&1V@N%$GHPpCMyr{op|1PnN#P$2ZQu<9TG0tIL}^
z?SC)6vu@^JcF5K6=kI@ikL`Li{h^ZB@2A2?a%UXfTG?*!pj*}A|NHfy&-ZEOe`^>2
zo-jK`yra8=dH3IqZ|ifSb?rWuo|(mOep>nV1H-MNf)kIp-esPYWNBV={w#a5bb>;i
z#I1cF_TTxS>3<;n@ka%*$@Nul+Qo~WiYrUnuj;*dyW?*k^W^P<*Vi`3{L0~Y_AUN#
zyumc<hl~}o%ZnBs=dalIXA+O-W`Sy5w(~lf8`eqrwk+h5%n~^!S;k!c|I3bZy3gI-
z9Dbd+>hkQKgU@$d`7O5Zz~6%(4&_f?8k`$*eO=<2NvwMBADnuycaO=l<U2dhp3P5x
zc=egG&OL#>ISh%ryFPuI{&hqCzj@WoXV-oB;s054rmdnveCPk&mklmG`En@Lum5CW
zTlW8#ts9TZ-)7k;VZ%K!^%KvX9=Di@?mIrcbl^F=UnTkK^p2e(&Eetld9~JZ!H2K?
zNd0&?bF(VXN9&HLAExHlqBvfBf9Je)uT9OmGB3f2M{ev~B-CLu?>qbZ2f0@6FJ2hl
zu@i2tPOz=XJFi%Czcj|B@b!M(9Y2aW+DsZN($DtA7pT2ypDuJQU&3%1gOBU22^+oS
z@_6%3Z=EM!`1Shgh`8SppBI)Mu{@Asb$)7o{iA2s_BJpx$1QnjJ8}AP-=|O2SPhRH
zvD@R+R+I7~g#Y?I&X|`sr*_vqbDWkCm-X^&uWmS_^)}ntoOAbx79>~~$mqo$DtvBZ
zu9mlFf6ta?x6hxR++1?{x!;#xk}diD4N7l4<fk-ttW4&Nt>kGg*SoYg_H}<Q(}9le
z+jn04(|f-B&EW^r<<GB?^a!=%pEKw9*L|`(UVk{G?Jlxi?Co*ZBRLXhcTQ{G=qB}X
z<!?5-{*`OrpZ;Yf1lkhisl~$>@oCASRO83L-D-UT6<zY;Pv7<Sd$i8A*>=kA>2oZ~
z<{s!|juGEDuj9z#H_KCGFSXouW_Mh-W=F?SQQ`j+w~5{^NIIfDo!#%jyEfM&?;?G!
z%u42De53c!_IAO_TJLlIH(MWHeE57#V}+FBw?9vR?AOmzJI1!`%F2zp_c|tjS&}=6
zJDIKiKu3Nr*9@useYa=EckY(|w5a1&t(dv8K-mQ8%hQ-2#962_%oBb6aQbb>D~)$u
zWG%n7o2l~HHUG6TOw@~UKd-jMzGp+fl~C0-fthZ8hi-G6vnYt;IP9u(&Gh%d(vA;b
z)elY)-QU!|er-pGik(!OSf!5sot_6gx6hpwTrJF$Vt1xd|7bj8&Y#EDi`;mRU0q}P
z^0c_p<M73t(Pi8W;b-<bKH2g6@JTi~iKse*#Z|}bAKx-)(w1g$-gbZf9_jyb!M_+D
z@D#BBmOb!<aba=ijyo+TqV_7x^CJ9P9>y-R)Q|u3ayx7-z)e{fo`~NU6y>*Ps9fLE
zK4E8#!A$%8$DST6E-W^0d@W?|#`82T<@1U-#^1`hyZ!~XtG8s!2Yr?HnYHEljw=s~
zFZIfQ{jhTm=i8#Fw+*}n^*On3+v_i{Q&*aFVnuo&OTm$2-)HW8ly}_z!LEj{MLf$6
z>3T%(mS}7E#M1M&?dWxed-EP_INomC%+7Y8Qk!40ZHsc6ll-~=<;-lC9*b2memJnb
zuHaAlvy+|D<s08wcU<N>_GN!O$K$Sen{yxBm7m=G@$|^uwhd>G++i&4+{DSA=Vx=d
z;cd#FDOMA+za@S+-E{n{ZVO*z<KY+GnvV<uIc>|nPh%9YUTD2t?{>lY`xAKH?{&I!
zRa$~)=ZP;rt#?Q=eRydztKq?)ANH5jf}@z%wP@$XFqA(QVk)%!v#C1SdIw*7eWA^n
zhVoU_8-<fs{dn;r*iOGiPT|ud;Vf+>rOv*M@%!ukhOl|AQ|p)a*EAQOxz}d?4=Lx9
zVkueQxZKLkeJs5ayh>;9R-E2x`~J}G96=q|(-U{OWy)Bq6+VthG};#{wYXjWScur3
z*P%z>JnnSY6Yeu{^$1qwDE@axBKA0^{D0kY8?K4VqGWZJHhIPxvS0kX?@;7M=9Nav
z56xJrkk@7~M>Xm}ycKs|-n$vz&yQ~_74G}N)0Y37amlF{-J0%!GS9CUWoMgA3y)X!
zotmJtb@jutJkjgN)^^o@Y>lqCTHE@<)45}39ar+tT^43O&(5ukuGsUFmGR*BX@MNK
zuk}WR#~+P(EV}3Ow1c<bY6O;FN<0}m;|g!w@!K^$+P2dlg>L`ooV%-3hx=bC_v5rr
zhhOkMi`x3Z>vgaDy^ME9QjaY^puI$EK6mqM$>MoNJMQjka8Hoi)3o|==t9PQFV_{;
z9uRmZQ~Gb=wnT5f|1Y=A`5G;ezcs6M|Ni}DIf4_PpLkHTGe)afCH!fZ_~N}69bcNP
z{Qp;f)ta9V--*g3*4tKYJ!o%z%p}3S@?Rr6v%yo(?$1vO=Ut1e%&YvIyrY!qxboyS
z=|8oFcccBg`6CwpzbSqw`P%=Dxm9mw*A`UYZ2J7D)PN<yN51Xwg^rxVuLbQ^SeJh|
z^6^CapUJnb-K?MDDs3LPA;IzO4JD=X6VFA8{N(QIKK}n(c5LGFjqSF-5;VR|Ug%#s
zcaD0$ed164mUZ<KBJ;QW>(9NMd|#&bbJCZ;X4Z!qg8uDDJo13+XyK`!^;g4OT;g~@
zO`-E`{*_F##1zeRWbT(adw%tDyXxYi!WndaQHPE6#b}Pd=f$o%x~Oypp9dd!P11?k
z9z_d1cdAAjS66=gx9;wqIclMsau?rydE{NeqnGpc@IMbW&r8}Eu_d^>!zP_=`st@Z
zk}5m9dU~E*TIzjrSFf~LPQtr8J9}iU%_6s7PTDxZwz}-?6(yzfGpoc@BR_xsyf|Q?
z)5%@E&(F<0eEqul&F#T3KjCe}5!u#Tv_S*ZCL~fdiZac9PgMcWs-)!Pn_IKR^`>_#
zyY~sm$(?(4b~Ynuy}-`m=V23FT%N2n*b$>Q{glF_Eq}gT_TTgO+wCP@Q@g6$Hylh*
zP*Q5D`S}TSWWK1Ui_1K&uR2{Sj;7hyj^vmL3krU`@Hj3muI9r*cC85=9X6&`>47So
zUURKXpPrif`|xAG<&(GHuZw0C0xcq*IAvYz?y?`he%XDA0Nq|c!*FqzGPw7k8G9FU
zd;S0IN0S!D?k>~n0rw6){`~9&Ex-S}|81Fdue3Sn*b2}|+mH7C|M&ZQ`=ZlRwcTG`
zSs5w<IxkdYV#U?f;m4mAC7R7H72VR&VIw@t%KeGt&a$^s=6QE2o}Jc=+hZ}?ELSQ0
z+?<w{7Mt7pJd->B{QR73AORUA@I9sbX$t6`@$LEXY3Jtrj9KT?*V(D4rNvcW|37`-
zninrJn3$L(EQ_8bFMlT}IPr<QZMTwt=iR*YGcyc9bAWBU(nTL0I9^)nJ$=os7ezZ~
zSeNVN@BeFd@Zdqvy0M&_n=&7Fcc>&=r3XII1Pxg<Ha6-+Z|j+=9d49#q$5RKMDgFW
zCg0g+ssI1|eR6j8^hZa#KOYs37m$@*JJ-QQW#%P=Ehqjzd~tDc$-6r<i=Uq}toV?S
zb#2YeMYrm|z6$;I?d|C{UTM&n#oDN?mwxtu1Gnh&Gv8ldULHPg|KDeO-rXM$n)wZ8
z2EB`v1x=(^e}8u)$IPuy#<P`6bj#$94i#akHnE@Vj~+b&^#fVCL=L<x*_40Z&hFQX
z#UipIjG)1aTU)bV-q@)8e$QvVUTO1Ze|~<xl?d*h1@)z#o+c<Eabi{I>WHmbqVe^A
zO*1bq3r&=}u%j^f+xz?e3!PduA~&hnR((;>((-Br$D8NAJ{e1=UMbTTx3+2@ZsScp
z*u)ANU2Uyo^7r@ORsR0mxw+Omi=J}*{PZ;W(UDGzxqhPVE-IXtEW4Hb3mzZi1x>~t
zYUO@;aWQ-Dtu3A(51;*i?BCztFR!hYE`N6?@yUsaA*(_>T?GX_oj@ZK9Xobd?ECdf
zn@dEa;bcnE|9^kCWMB7l{1^{97&!adnuXre^|s{PoCJ!^Z*L++-N1d3myvra3|YCw
zKv!{q?rTXr+&0mwG)wfEX2z8jg1^7LRo2kxxU|&!^49EdC1qvE3HK9^=&Pu-tO{KX
znri=W*8Dzb5!9zACl?>GNln{4b6xE2P1)D=oSdCc&NkP7e7yfUD9$=(oQs?yqN}6R
zGtahqQ^m(5(6tXy+w*!aW`rcpo6OA42Ri&rNm*IavPdO#RY+i<i;IxtR~@e|5ozhu
z8xjv|1T0{fH*cPdMM1;&`}OgTA1%4LxRS1}3jMCHuquAP-E#l=pqwNk<{CGrm0LV$
zfdeDxMvz}$UrQRLaI6ek8k88f;MdpJo6FzFc}>-FEhsRE-BqHfq~rvNu*y0<84HJ}
zr>26ACIiiMzq_;Z$+@}Ft)41p(v0U?m#3YdXM1N?X?M-fPoOhEWcb><l$Aa)y`1Cf
z`E6bN{&V8-H3vUEJ^k|LW_8`@ZC474g$iC=P~27i-mjv<LfNfnLDbePt*(v}+zaNp
zdTz@xlWu0`Uly>?>Bp~MMGp=z-rAD6c)<)85zU|$(Df3l!}aa{d~kksZSCom!OKG=
z1SdW}b7Ot{{yME@74y%}&MtOrW;=51m{IMo5-Ib%o|`#!t4{CORL=Rh?qBO(>;8n3
z(Jef2{Qtl0+Ei>9{477=`LVk@p6d9d-9PU0>g|rB&noRdmCjF1Ss%<<&wrRX^SjO6
z?ogFQw|1xO+c`VZc-g1Bk7kFg4rAzFF1`AuVY!a6`TOS|uC0wR+~f04r>>#7`Sfzb
z^rvcNu`LUnA`L8+EO!)1757JKOt$~aJU=lt@_(Mp>9Bc!GUvU!_4M@gezV+LA8zOG
zzql<odR^RJuYv-DdGqGoy6EaMkNK;P*CtS2)Cya3qJfcF!n#by{_huI%i?D>YNt=d
zERypray3jm#If&B<$vY#ZH9Yu<<)$C^o3lV6<~E)xT)8W_omUYjsLE++S{cY1@C$5
z{>0~Pr~fvllycd`^Dl2KmlER7wQS-wKDagh`MVh#d+z2ebeehQdsyG1r_3`AUW#1J
zo_;WA_nAYS!IC%sMmsHf%U>UI#8d5Tze0Y))0tOp9X{NA-tKqL-s<l{qN1+*YJUEB
zxBGpG-D1!X=85~CLB3iQs-1Oxo$uvkzBhN3YKv-z)jT_Wc8a<<!|`K2$?;F#FaPf*
z6LXMfI`bW^V72;P?aH%EC!O$k{dYezlM$1ckJ<*;sGailJ(8KXK00@?%`ljCd6H%F
zGxurl6pW07ZZ}tID%x1C30u?uOk?)fg-_#R{_CE#&N<ALaMJnrM)?Oec9YINU-W+e
zvtRdm_gwnx%oV|taL(y4zx}ejOn$k&em_s$6P>gE?vy=ZhmF^6-@o7fe%)_f@9BD}
z2b)+ysqO0O@Yf6ipq^UgrpL$oC#(C*U0UXQx{;Y3G{v5mcRv04wwpR7?=Ck7o$LLx
z<CVvu#bMKW?wvkxaX#}k_Bh$2zXIp}$T*O4?C*Emu04h`k|*lDVK~QYq7yhnDD`x}
zmJ{kZJ4DVNbhW==u)*Lm|J{B6cYb$?zvcFy&sNfWk-2Ku)}*sLGrpaXa7ehg=*x@y
z&5Ke`1uVOv{)VgfJ)_OWKkQat{!Vk0JuQ6Yqq|=0ty3TR>>qW>x?WE#-tnw~XZ@9h
z8xQvWyS!b6<(kH{+U;|vtrxFaz-xGfk>B4WCHZ*|>z2HTNzY5oG7=ILPEXTa3|i)t
zdwW{k-l~@l5MS>BPXPN!vhheXtY82BbL<XnvHJ(Kw&Z0e-d!-?@;$@#qhEE-U7Mk`
z^RHah%c)-%zqHWpxae%R`1b@$9{oKNwp;6~O#2~sQa>&C;fI5!C0aXs&;Qa_KK^>^
zij0D0`#G;NQXSv+d;EUJUi0at`j*1Ci(fCeDzff&Ly*LY_~pXw{Q1w6uc!6>6YG8b
z%nWVE#eBVIi!M*UYAE*N=#4zzlNl^NH!FXit~&D1W}fUry_j&b54pFVP4W3HzPDA-
zedfO999O%RRlmEvuk7tD(20T0?R-#Q{+uj(^R4UKQ-w!v{?)j@LSFB8&9prG)6;Hx
zSF8@L)cMAKUVlqr#aAV+5>3ZHk+p73tF|53Sj|5D5ySRVBB!Tw^+>L~yvnri?(ZFb
zQbx}zed1%!xYm_g?I``HAiZt=k_*=lOJ3ZklD64;$J8Z02@DS$Z`rsRI6m3=L+<?L
z-P<=jpVGO|dH=#RKKVyUi-oVfDgU`@zpLi;%(}SU?Q^}SU0PP_`tQ}I1+S~a3;tbQ
z_QX^==*Bj2K~G5^t;Q4k(~ro8&tZ1H#`UG{>SRXwYllVdbg$LlooLUP;AwHnr(%D~
zlPBxdH=6MuVOnx(j^K_CYlg55%xNB?69n%(^7y)BO4{~jMe|ZAvjso)7PA~)FPK++
zw_eKl$lcS^%B0zXQ(s+QS8M;Hb>42T7k|xc4yaG)x36oR7QZcN?n=wmx0nC#=DKmi
zDw*-@t*ws2LY*Bd{=Yo+KGk>pcxw~ZF1sOM^U>uDHP`k@i5+<Ij`19C%sxr^lSN`H
zx@)gjol}e9D1LUma>xF__BCxg+Us`Bjrec+jQ_MS!;yJqj}p83E8fZ3ebnE)WI0EA
z=G8=@>1)dt>E|tR-v0E4?nn6qFOKE!9M;GNv8EgrwVt!kUj9H^ukh)8Nl&&F@Nmk`
zH;YzTIB|dcU73*emJFfQ*Hj<NN4=4h&$AT@GqbAW*p-&R_LfIU>Ex6uG1q@fUsq4R
zu==z~+3vaOho{`Tq9574E%g0|sLn4poWFm(dLomd#=b;E|LfnDL+k&x7f!Ffu*Rv}
zYU}sJpBt}7+O1KtGdy0Q{ypH{JF6-D#_`Gh4~xUfwT~WbJuLZbOK{pLouVtJ53N#t
zT{wp${p3C^li7w1zGth0HlOj}ITm&1md%uG?tMwyi_IUWuY0~&>EaqI$>aM1Kd<<G
z_;%0hnEBuS-cQV{|4{K++w42%k-7~=eG8dahCGugHvg*d;NEnnX9;Y3q<Zp~{^2*j
zA$aW9PwpS3<ubc##Wrfx#&+!rD_B?J%oW9ROVPz;lIGoa{*(4VTAv(Zx=~Y9u3b;u
zoV2xZ{{6$nLbujTykYP--SGci{zF+?6F+@E{6~L!_U5RANeBOIzkavzVh)SU4Vi>(
z#|1jo>z;|5bL`EzX#~2+Y?7*Xi9tt)4fCr=k;-#DY20=Fp6O}b^3UqOIY#_B^Z$C$
zS+UdhG2g2fUg)^VA))&tb8=UQO14!xXbJ*+KZ^^f$>rkW;xbQZ?q-coAvbRrmTOrb
zj-9bFv72FYeOT9YeczXRX8!-U{I1QeM$Y^C3fJ42GbX(jyX$>z@s&ThCg)y%-hM5$
zc7K<Srb+Ka-%s{?)!gKlxgYl}`SWn`59|FM9UhyO&2yc&{FKj!*}WekZRc@LY~1qe
zNvs!J%(BwO*DLBzy{rF|`LgKgu|Kcf*6cWtvaI5E|B0EmKKnC2TBz$T^zKxWlJiQl
zDW|@a<TqBNUQY`9`Ao&ZF%azVf4uoMLAwKOirN0?WNJn}XJ*^;>g2ko$2(YL_}CK;
zXO*h#x#yUYqGEPjd859O(Cm}HzeoR*G`_Uuak~g_MS_Du;;%W?`FnT#i=I_}qs{am
zugI#ujAu{V-dOxn!N_LL33st)9zSkNWj;NaE_2P}ka8}6>&3aoVi$Hrdfwd#o^p2i
zr~mz3QPbQT8RF}1c>BD&*tW>E`;x&1m4m7F&tL3Z`{n(G$%V>Q!HXiaug&RZ%gOn-
zPx$OV>zOB+cN>04(UFs|*ztm0icKu-*TUN$4;Gvbk=koHqqVp#2vkRL-#mk;SuCVB
zwbk`DRn(tiWQcjF$dma+?eVj7OVdv#ZIRvIcVf0;&zlP~7O8e!T6kD`ie_|MP`utz
zu9%qrvcBx+QXikU%`mE}t_fCp^!`TB+6=4Qc)^JxPS16^Y`E>%zP*V(|0?A8&F{hW
zX>ZP*+wk)(=Z!nfp2wnUc0MzXnSSU-;CtIkebElDoA_*=nJ#<1;Od&3>OTK#V#&An
zh(+vZ&5lb=Dir)trK|N#PPqPx+nZ(Ij=M+h%9ps=yPZQiaOUJ=jU~S}S<OzHnv}DB
z*^G^Wo2%;f9b|DaT>iG&Wo!JQ1+O9>%=N1I<rVDqG{)<-+SHj_4AtMV^QYc;x^zm8
z&CJcQajE~)g-@FQ5|;|g^ZK%3s|!>4_jM0y-Y$D$`npe7NlCM{O3Za$Ylb-6`z?FE
z1o>rsjX1Q8nS13_bKQM*Y&}c6CL1dVyK>FE`KGAg^&dyG9DC1&>)VYN)e7?I{_4EM
z>n&^_yNx|lxNTwYGn4P_OO*X&59sC>{^@1Nd?8XaVH#_&u(&kugo1O6W}4Wo6uS1@
zXtm$5K%MAUEe5kEYc0LQ$=RzE>tJ=~6U(HlNe7Ih<Gdt;nD0$_5jp$l{E)?OS8NS=
zeM7)LCui=)hBV$L6Bb>iNlh!#1D{B01}bzYx&EsV-K4GbP`=Dsq9}NIB6FAeo#>mQ
zb<-rT%eXx0u6z>9e|)pVZr<lFEpjFZXvwy;NB#eh_4}{#-G=^y2ew9EY~W8&Dh(4p
za%&^=xy|)^UZ*cS7H?o*tXNfZqg^VvZ*$V$Y@hqVE&uCgJ4%PUsIY?P;D7Kw<8yI2
z{^y}&Isg3o^-F&)S&UNPCvX4yWL;=ivU0kh&^sojon@yEo(v9W=&*fz<Zy~wsDoKd
zVe-$cXJv<S!}!j<JUqj8`-bDk^pB@?tC(Eso8I1NoEZ3EW^?%J3WLgvjV*D<J^Ie~
zFN~U~BOu%xtyg?V-`#pUqyKr0W#4~w_B@@vvEX$@%(t0yZtm{;yxvT8y{q#DB_++*
z^`?TAT{3TGTF-j9JIQkEZu`ZTzdU{1k{Dh6!me)4KEC5G3zq5W``o_OnqijAFniYb
zC22D!O7v`<v9(g8wfE|ej8Nr!>mp|Csk~$HvM4C+_<Eg9ZiZ(vm!FvuDf0T4oH4u1
zlFIkrBaI}Q>}5~SSa!3+GD7nBg~kxA;y!t`DJfU=tv9~idG5UF{^chR+>FWisdo1F
z{KbFn+PZw3>DK?|`j2I=BV^Q@ex#-|_E;|~IJI;3&CmQ3&s+8RIonp~{xv+c=cj7;
zao#17)5}dkk$y-bMsK=z>VzdZPftw+okH!}Ev9<0M$-Oo@5%H0pPBh@uL=9GCE*z3
zF?AP{*}U3nI%$s&zuz1o_iIi2Jt4`?yKmZ${wWVT+jW(#u1+lHn#YGPhv%HQ^k`G+
z=^2K}Y~Is!E*?}+`osfDTdu~lHvaf)xO1lNv}*l|Pt|ABbo^`X3*Y<Xq4D=~=;ng_
zt7pXQbF~F0ig-LnzFqH6F=(dXT%KI)=^j&egV|?q<d_}sImYBTML)dDXWIUHrv=J?
zL_R-1mv*()!Hqw!#dWEu;KXzX+is=vk-M5gHJ%r@yg#;PQOe|Ldv33gF|YsCas5=?
zIm6fY_uo4+!%SE*xo^Ysk~Ljg;8By4?Sk?0@&+ktxmxy@_QlNjUj6m>(};uTj0N}a
zKa-|vbm`ZL>fDxg_eTr|oE(l_kqD|}Na)?7W;5?VqN<IJQDWTD>V8WhJ1e2tk0<)~
z^{8BWb>W#Hcogi&p(m2|2fudSwPrXm$FA<uJK@vy3!LxgOJ3P%<0xDIWEOLbdHLnV
zXA{hF<Gn5_%@;^2ZB(@VCR<yvJy7}76kX?&=fws1?%Xl%cFV6j+2wjac+m;<H(b2Q
zSHH&#L~POMjoQA<W#*<S99btGEjB+`*lX!*pMGLT;hu1{4#sZ2?86-$C*J=LU&*4R
z-*Aw7&g!3e4+3oX&!zo*`RU8tEiJER`s~Qdd)#||`{&2ECZ_P6KKp6e-x(Vhv-vI-
z-)t2v+ID4~*TfwWdDaIGmx?#P<ZV0m&b{yTeu0qR?CZX)I{H(7|G~d&f9|=gBE2m2
zsO5***1A&H6wjA?ul)YeAj;3oXn8g7uP>2)`=+T&W@(?+kJTzPyBvAz+dJRpSKJzF
zthaV_aDbW(hB4)a64@7*_!T<&TN&)XJi#rsh)JOG9S6hm^8Cc}n!?WQb<SsH>{7Va
z_`SK)CsLVgaJ|cH(T@3y8y-zt&M@-~n-8O<MydzzjFr|^+VM}Vf7*Y0XKc{5D);e^
z`m(akv$+_BmVB4XdU2^cL^{+#fYnvRRYY_NM-!{7qu}lb%?(Y9rYJmRoT4Pl=(y{W
zqm%z8&u!;^D=zGMTlq3(zkbWvcT;|!$y;Zep2oMh{ae}(K7GM5=J07bDO*FAMa7gU
z%o2;8r?w$k@&5ULeM*r*H$NN>>Pyc)8~vVVK||A`j*gU|^Na3$;5nRde_!pAWy^%b
z#L6B&an+tWM^wuAQ?vTY-x>LAqQ54E-#Rw&-S>nWuM5k&#nSWV&E4@^pz+VuNRw=t
z$?djY{nqLqKdqTMWzUmI%xnH`k>`KE>g=ZQc=iAHW~mguJ0!VZzM#q0gyryw=BfKS
z_pq)mcJF4IdEVpMh0c;in|9`PSmoW_C3-kuhp6Drpevg-%=g^dnjO3{NL4Fj#epX$
zC+`$Y+fdRnhui#Sw(S}9{_82vF8r3*|B53ri94b4e!`J(=?@PQs}Fg&>DxP=YOBoI
z68>;yq5a1!W^wyJH7xEPM|S4de(QN(SN5B6pVZz7?NQfM?iA0E&HgiCTIEdd0$tCU
z_IdyRX**e&E=_FK`cwWO_)q!qLZ*3#eJYC-_AiP5|8`&UalWXnSx1j_3ctL)ef>QS
zaORvYCMS39$;rt-f4yG+^T*@<mEr5<R)?+KmAC!-iDNT+e$_Ct)IYduzP!MK;cE51
zGoN|q?76vC@%<l}!@vFSpS`m1yRf{tfBXDL5*IZ2mY+U-S$BT=t1ln_d{6M+6;fdQ
z^>ZuBw)4sFcD>d4n`D#Sls&_sQr`ES{?T?@sf)jEez%O^akINH;lG@Z`XPfCKMU;b
zBreX@J1uRvxIEWu%d69mL%&^G9vXY<xcRe?-RYTge*bzKYE$`1rS|u?rG<}=ndIG>
zp&Pv|V0T&WUO^X^bpnTX3GPaC`KNe#(+rbLp})VsFOS-q1-jGs$?56u+wM$nINp1k
zIWX|~{k+@GL6h1)tkC%P>)TyMn>_n-1?mqDzAY5W%YA%sm2ySZ&EFzw4i#5gw|^|#
zJ^6r~-;T?B^j<&w{u>lVpdA%!qqnEs*Z>+3b!Ok&aYA(2JlCB&_x+V%1Ko^zdwagR
zo?hQ9(`=)X7Xf$jzVmL^UuOJg(E^SKDzEaJl^SMBiWtWUzhim*h4Wg(@(&OHI0vwB
z@b~Y^Yku_S=jWACTeXUxpL_cI{r=~FzuylJ5E0xNRUdtIvxfQRW_JEhe?Fi8{P+8P
z<BAUn@Am)y7yUV@<3QrFo<B2eEgCgmmDf#JwR8P{?uc!<x83&F{k^ffTpx6%GH7_v
zQCDfv292VhYvcCLy0^Fb@z*MlUre*Fg#@lkR`;8e@Z-ZnpV?-;;&By?aeJ#C7FdAB
z0J9wqUnueht%24KTeBel{yv*87o0)E=6kEZuQIqUx&Qw88yk~XhO87?mboeSw%M;Q
zFBd~j!6_;&d2ylf?QQUV3t3lJ9qrcN=i%km#Z@dj!#JHUbX7>=!6sJFY5P60*3;hH
z+`QUAN$L8GwuMipmA<&3`0MNI<4vsG5+)e}bFIsdrCt8Me0A>YYiqx}zTW@$_xI1&
zqVq5AsVo+Cb$Md7Xn#-9=~Hv9&4tx`5}uuz3EE$EYisuAoSR0X&$KMd-bnb{eib=8
z%XD$->1jFl_C!KD7*Bo$?<`8K`}5HqWEtpA+~t0AZ){HQ=PGXfQDb+n^11Az$Bzql
z#^l`I)_ZSnb@KM?Z-L;-=XJ#IZ*OnztF`|4@#B?sv9leU*(T1Jv*u8Y+vfE1YIb(>
zRK2HdD0vwKGGbHe>1Bt&^DuWLV)Vp8TWLQ(J1b*fH-}49%b@O0Mbp6rU7!DNsQjE3
zwJqmlxBk8pH#R1N_V#K-Zd$^n434$rQ&Y9Y=huFd+?;;?*=ha#PrlvGPd_<HRqM?D
zk}od;=T$u7jM|ptxjJm^kDt%yXI@&;3DsD<B4niyXzcC60>>|JZcgTtwc1eqJx}ym
ze#PE6w|+U_ni`vl6DNZ9Yl8-;_PBukH#>N?S?-N(xzblxhcC~&y9;#3!jdH_qR)iV
z&dzdudTMIqz8cHZ)AiH8yttThc2;O>;vtjku(bf-*@&GxE$90E`u_gB{{BBsK|w;I
zS|JZE``cgr*)!8D_m<1&XJ<j1THf#f54x~JL_aPDGA6!r>ytAxC;Qv~owBp|`KPbf
z<AWEw@z(u#*dD!2uCufA%d4y1yWj2V1|1Au_}ER#JZ}wTZSa$8VGEsD#q{IOTv+J*
z^6F~wx3{)Fz8+t{H}%{7#~(jDo1GuDvnX{_+SyZmvep%EHXb+d4Q6i#ji`3axwtA+
zdt2UJueZ0hu8iMr_xIP=$6sDvHvekpKi_Wdj~^A+*2RJr?}0WTotUKB&BrD<arT5~
zZDKonKo=EQ6et*{oq6!{`TXTcN4w0g{!>*|1#K$xn{Rja#l^*$7Zx;1#X`EXYe840
ziO1JWJlxJ7|FLxE-8{efcE0cK>|Fn2m(S%T-qZDL{(LxmBS&fmc$jeF>o_MTCi$8V
zjGUaDT5TO2F=el$1O&m&9}pQBAp)XWK{xV&22)AkP@olJ-Nem2r#I<Lb$oJcQ`y@n
zxymPkN=ix|etyqxY)lr>3gOs(`>bsFor9~_?^~tR*>NH@F`UyS&n9mD^sle4$A7rd
z$jr{r(bpH&=;HEZ(?x?VCwNz+ot^dM#Kgz^{qAmVm$v7}i+Z?##xU|qV|SGt%=`cN
zfx}|A-Vhl`;<{S;_SV!16DAbrr_JsydwVO^$e|08#ga}<)2%)jP!bU#!Eg6t0T*Ou
zbkf4;?RlV;qA_~n;1UlqGGbKn?vCVPgAKRerq_#sO%`O}64i29=+ye*_urDYw?bJ3
z1t*^NOFz^dlh)bUxyR^@DpTG6(8&0VX9i!6zu#Ye_1nh6$;<APp0~})5u6y^kX9hL
z^9*QFz3^SRMd=6kgs0Xzo<1bYAy7GWUDLmpDMFo=Ns-@A>qAsmiMg(8xV1C?apRxL
zyi>K>qLSx+GY0qSC>3>Iu<Qm+SCcbqU33|AJM#60#ZQHw?9rB!cKq|3KWXP38QBj~
z^`^78R2~<8vddh~ea7S~OXUlmOg^f$q@yFI@KvP9&Ra?EmBMe&s;K`lq1LwT5#NzV
zd(Iw>_q!x_al)Ost0tQKaO_)ir`J^?@b2wbygJ}Eg!$ds>q<C%6e@mkL{wY;eWh?`
z<;Ip{5|aJ~N*ku@_S>p`y3_sX$w3u{NM+C3|ExhWz9+-9=WRANX?IWgBONSI{cq09
z)161-_B@ckmj6V*@v%^UcSlD|SKC6@b(<dS^8F#d#wBI<bi>cSsvn#4Eq`5leIrO$
z<96eo+NcCsyPlcm`Z5tG4{kS~yf{6U$LYkcCmc@o_FbDRbl4_jKk44#pQ^qp!fem}
z`3LHvdCZuW<o)ZN5Y)PHw%S2v@0;ciY;}~Uy?D4@ShQ3~e!JkK$kYGQ|9gK~@w}tQ
z%roAQKmMeeON;D-8`g(CZq8Y9Oh{1ht4>!;>B(YClRCpXg*9&0Ts0<LC+4VgESsi~
zK7aRyYqe(%GM?X*|Ka+C?N-~j8e1A=usQCT^WwXNdz{>vkMVaR{=JoXVLj#X{PhYQ
zom=f2_r_LFo|7UJWpUzR`@8i2U+x=x5;^xHOd@!;PUYMk>h`A>J}fp!(h{0hZhU-R
z9lJk+lh?7wM&AowOiTx@Mt4aJd=ka==L6&K-QOPnICfa#-0c(p-kfd`>pq;?RDQm(
zNBwQI*Dm%qk(-@XFf%Ae9}C!;XnuZg<K;aJKg$1eg(<yszr2wz@8ph%W7GNEc-GaJ
zdCWO>&vEgG(E83j&Qb{v&sz8!?U;2~I4rsQT;tYCu{hIya6XH%p7=Lj-Fpt_wYW+5
zs%9x?3mcwrOsO%`IWSMz?DNc(KY|&nPt4bQ-PD~VbFN2$Gr|1dk3Ai2Pvow(7u@HU
zsrcjFG-FQXG-dz3p8D)E1(Q1g%AKIn_{7#nwrN6tEB<d5U;e{y2{WV3?6isVxX%5U
zH8cOrp&g43ZcgT2lG@_%+QrSr#OS=)wxd%NRMOofw$v;?$LhSFzri8Rd|~OKL-%&@
z@$FGLAJMSb_n1|W&c=WR+?LX91sNY$YMv-;`+qn~MNrVvTYN`HOlo7YrbJ@u3F9=L
zdn%JJ?73KBeqJr}^ZSU!`xN~5^F5LN%VuZHc6-}RnR%J&X(#4Nv<TRKQT>sl^YZq$
zWD!lrc@y^*{7*|{T4xo@!CxpGd!$t`!)sgL!WmDyQ(`1ON2v4p{hfKh_UK8&8lioQ
z)f)vieXZ@(2~*kD{Q2XXxz1HV;)_0ARnM=NXOELz4@zRtg7wo(lT4vt_PqzBUAbd^
zaX#F5<4z~bVeUs7b6Ov~w->gkdU*Rn*dCQ*C*Rgr{AqgrM4#jLH-SIq(&BR6lGEHC
zWEi>mI9lv#um8=s;nx$*d;By0{J30Yx~=1cE~qOXKHJ(*`N7Wb4{n#dT+ULS^ZWKu
zmhW$W&c7yGkp3q__U__s0)i9wFD({x1q~l3VvZ;ix>NbdvgML7F6$a{)%`cVN|`ph
z=>D~NK1z2b*SUfAC5lY^4{Akp>IHQ0H-|5OY^Q2E_43J+PfpJKylRd9T~WvOKG~}m
zl$5q}9Nwhkohq=iP*p|c!$EfW2@@tPNb2Y~k-D6Gr-7EfxPU7cklzS-1XVSbkRYf4
zw+(r+Wm6xV2Q|oF8mA?&uAQB=U&<^8RMA~IpafdX{$u|vvs@|uviF{SU9kpl7n+}~
zi*&S4wtsb~<6-m5o1X(sw{(E=<G<+lJ(bfPo#U4JJkReq75r}5?fo%R-z6XLUVC%*
ze}?sYjbeE9l$4a}y`JmDEqP!3SK`wv>nZQ;zAxkHVo1En9r*8IWzYX2y}4HQ9rxdd
zAMWnx=s5H!{^QS@JJ&aI3oqAQbm;JBj<aD8mfgEQ;e*5}yBP-_@wV?={BrJ&yLqxK
zV7FFiYHDhlWL=shUvKGmr+IThOX_~rZ?}0k7ndB@zwx$j;jgz}Evmk(;A0a6Rkb$v
zCn~$Ih+53|+qP1p^uCLa`wQKz|J%+k`C<G1QE#rp(o?2CzFv?2x(?LhX?1OwYh50+
z(u<d=a#rx0br&VfK2KZgda#Z8M9@#qRUzw-&N9vZa!x={aALHz&bp|rU0J5DC#<cC
z=ekz!=gu@?c~&@=nY`~|>siL>=VU;41H0s%-7sUq1c%4V%7a!s-TkC*f4uv_$|NuI
z#Ubtb8TVg0+Wjq=!v*nJ_RkN8`(NC6n&?$N%c*ZtZz)TR(>110&F>d-e(f{pQFLwx
z-OQ`BeM9uCWp|j__@4dsR|{NsyS(MO#-eNgxgQ!H)%gFR=2&0$;eCJEfBrbUamKQh
zhhp3(O`5bphkN_&-t*5v4NiuZwl>h!N=)GPQpB{>yKk%|Ian$-MDst>jq`u`d%1v+
z;Oc4r+yBf<+2;Gw>1LB)&F`<%TiV)IX<Tp2yu9q<GGA#YC#MZM+^08fxS3P2H_mIC
z&dF<QqeE2$1$XM?{Yrds<x0pbll|SUufGL<TgbwCwEpC~o+TyASLRqt9eZ0URrKfx
zr|46wfW1|vF?!;0>&54vZ@>Tk`0KAiLPAcjudNML0L9!N^*&i^x39Nfe|-HiXUeM&
ztZ5!QqW;uddib!bM%8<Gg_vrEt~@!<wmM`=wvPWin<-kMulW0CFY|nUZZ6aW>-~Sf
zNl(?feqi!qxufr8yU%V}z4iZmuDga)J`~+u8S}QxEa%38rkG^V0w)`rKm3ZDTsnoc
z`oKwjYU%51flI4&m9%aB4!SX4JvBY>-mCrz-}?lw<QA}Vi|GV7u3NY*_x7Xjzt8`>
z|Fr1h-s*DD;v*5U|EKm;e$KeCLh|SAy{`AN_*b^BZ<te99=PZ(OVrl4N1h9-uTr?~
zu`TE3im<h}So=#~UGZ!MwYHDif4*7&&-peVf54{0DoNhv;jiNyFScd{eLFbQ=HnNJ
zxXRKNuA<&AFD^EpIN`zIHBHA8bOE!7py19md9$LQ?=g9k>bCc{;nMg2XLa0oly!6U
zk>Xzhv+eVJZ|{~eDm#;rV>Y|1NOb4jyrL&37!RxnTwL+xqB~?co=v=g#FLB1=RLS>
z>8)A(|GLHEyx`SwssFxQ&iT7Yc&hi)C#&_<AC*{zPFcMoaIu?C%#I15B^A2SW}s9q
zB6i-x-+y`NZN8j8Uqw{k&+fP@xl(Lz*uuN-Q;(b5o{%h>^0PP%bYNigF1h&HucFNC
ze4ssF;dX1%?<E`R=)CD#e>!jb<(<XpZ9I}rB_&(<Y6JyakG{U%DXgAx=ZjxoWTj@^
z`|2A$?NOhlHtTfXf5vyf>Wgi+_}Sg6=jTUn&yxl9ygfZV_xyUbI%G}6$7Q$nKY5Z8
zSO2#(?cAJ`3mltY+}x~QwmZ*TRcX<Nin%wp=gXH$n;blIea0c*wnT}xs83d#>TgQ)
zt^Ylt_49sZJ-v6+ZahES&VPA*y#2HD^Us4!xU}@NZ|%Ql`uqPJ0^Jbze*gb-D*_j1
zTw2n}S0gxa_7&dm*K-~p>pc>_)-z+~I+0AP7{`xRQ`TP-z7v+uvGtW%%Ew1X&A;wj
zwtTs9&JBY{j~;b&b|xNf<1Kl6>+7=p7nb|XG*VGf`Ebcw|KjR!{aL2j$F}9(F8Ki3
z))W`rEvCE3@9(iEnP0emW_=5J9eX@a`+i%}TKk9Rs&5&UzPd8INPhpHPu`#vf4{!I
zj@(tEslWHjrJJ{J+|in4o`3Gv)@)F(b8GhXXPxTvF8q0@q+~u*I!&VL(vy>4YWDD}
z-s)A4ei$`9b>_yOZ*3wEGctPEzexVU^}qFKr<vppgBF9s55C0luV2d-Q)F5Cc<vp^
z^yfErnZ(_ev8m_$@<zb#X0V)Gw7`RlZi-XRCnukq5|XACfA80oHLsrBb278B+rT9D
zfA0KqlT0+iE**4EeS2f$jHweoM)X>+ul1T)^CQ%1_qk}dr&4;mQusflG`-)H8~*8`
z?;oR&c5OUwyyq{kx-XJH%QXAihbzJU6KBmj^?LpOb8olbU*|9M;K#?upyitR`~OaZ
zjw!vYaTA<4`vC97%@&WoPPDnVTfQ(hJ!abijtM)?pU=MkyW?bD(3TjEQoUVn%Y2>1
zT{q26`TXm@i3XF-&4=vM8(;J9-)J%C#q$d*6@E13*s?Bs_EgqPUpjqxpo&iX+tlg<
z3i&-J-@mri@jU(G@44JMrkg&O>OXH(IlJJotlq8gQXyf^m*Q^8ucmK$?8y^%?QQp&
z&}m$EpBvRYVgK^(@^tB+e!lH|vWtqJpSx3h-nQiBrPkEb(=;{asaJn{(<5ctWm){p
zp!!?RqeqWs*j8@?53p1`jhN=TUj6rnV>1e;ytnH=aoa2YTj`F;x=k~eW`y0e5ufYi
z`zqx;M;YULSH|Os0^->d9-k>wW4DxJsAuR2Xx_8%`8=!IM61K$DJ*C2s{fZblC+4o
zE6$`ya^BBhH5b_)irenb*y8Auwxah$?(*wz{fcfFH@#f`c~3<{k?8rQamtre+8E;3
zaWjZbe!l$ksUmp`J)X60TRb`2k6t?4csedQyu|KQ`TWyYTI4g!@BjT#vrqb!@1$wd
zj$K&j47##%XYq5;EsrJd?nFL6{jW1<sh45$F`l@}r=p<igl3s!9-5{b4H^fOzEW!K
z_T*zl#{G@ltIg$?)O@eJ5gWPgOjit>JKrykl+u7DJ{(iSmV6Wr*q+95?R}l!-8A(t
zF>OoILdE%~8vp#;;(AX)diS+uhBdJQO#xfnG#<ZBVd+@B`_jyUBMYKS)fc+&{W9a>
zUX6lVFYcdm)e)F^)$LOrf3sNxPtLO^X44(BcDc?t-JoL6yx-%v#+wH-Z}I(!YnBPz
zkQQil(93U*gvnm>o(-&-J-45yS0+vPwcT=u_NCKjQZix=J0@(OAAj!tQ|~E?cIO)J
zzD-YjHC1!7vdJtX{xb<zx4gTOf2GHg^M2g-=ieiz9er|g^2_`C?LkK|-rSV><ox{j
z`BMJ_E-&-Vys|=2zV?gY*H>2;`_484Ep`G8T3RS6nNN7VDgEP@mzS^PUTFDRT{Gq7
zr?*qNyJpU5RJd&Z^MJ|D)q8k1T->zc;2lxBbEi8cHkQe)%t$|IV$QtZ^Xc8MbzNa)
zh4*Jx3#<Qr`74p@Ge29W=Bp)6i{^hl-aO+^;ft+tvg)cij&}d`dY7^{yNGiw^qe(+
zrZtnpgq`<yROW3IU*xmORjMTSjY@9&^zGuyK3lVeW?4skn9RRn#-~cH$KO)dJf2-@
zcVIzaD0{+UM@4Dxus^Z>^Bn&sskN8?D2zX7HJeXd^|tSWfDHzrtFG_UGGFsAer?p&
zE-BM2gRCnW$NJ^Zw{QxJi2R?Pzqri0-}ajXXuagL)WA(Cp7*L=ua#a`9_iwe_|dB9
z!vjaH@YP2HH=C&5>N}pcpDXxyt3=MN7mv#qB-|8uHLYBr^2+h@PVut6&$>++x#ORe
z8oW09`5_^|sGoN?Z&~iSAIv|FI`PZ#bRChB`5tREC-hAAqwmgVS1GzI{~5Xa=;>oA
zH%c!xH5~OnSx}PCz~6A%E$+b7yMC`XmGR|#-&C|%-al<p>747KGxnT6?`G~Iy0y5x
zlHKx7UE1HXM@`b(7KG>U#(WHtHj_RY)z|W;TsVHI+U@Q6``7jD2dxL*nCu?4J<oT0
z-ra~j6^5o+S1za@{U5w7=Vry{v*t7H>+L|xV0~s9otb6o4IZ!jQ@$<drc=$;SvR`3
zEzjt_zV4P~qGY4Ni92r@IktaJD78Ml?|WiP+^?VeJ{w*AnXY*2EVumk&(9zK(G%XY
zoO#dg3x{8{#@&m3rS6{d%+=_bE92=r2C>y*%Z;lRwwKp``1J6f0;{v$)|Bqd?7a;q
z_g<BK6ZQP}?R$rQSg?O9vFDz=J}z2(#dn^Y+20obdRrQ~G3ltO{kNdtU`d+_gQusb
zFL!8US`)v2p5I)nmwS$!UgOd!RP+1o_C5c8y$0RzYm#-P<Lqp6(6~EScYXfLOG~e;
ziJgDLs`*6x?~|*y#c3Z^QaYxhzoBC4^pu9+x_y?MVQKtH3dS<;{COT6U*Pt$$0Ys8
z4UVIamhW{mICzGq(y-FUiE~pz@q{Z3H)16lAE!L~RP$17wor46DA&V}L27~v&3@Sn
zxa~R3pf0e%Xnly80ps_L3hIU#x6GzkO`NcPL-R)6xf3dnZa$nRF#UJTiksWdo?pMO
zhj;$|ul?dzcO24>asQj7=Dt1drs50lld|_450&qApZu)cIQdvlcF~>4X-8W)g+a$^
zfwtTq?GgoTCiw87pw=qJvHVUU`|7Z@iz+`qGs(EnAZJ@u@a09IsGG|Zs}w#-qm&b0
zHz<leULE-2oZ%Cl>hGb^=`L$dKG+?bQhsZN1b6ZFUxm*MKjwbg^rOY`{<eir3MDO+
ze7vL_r@iGWIKj{RIQCB5fhYGYtfi+dGyEtZaw>m$4JZ4@%C~nSkEMLvWAg2J@r8X$
zn8SDUPI6okV%8Ed-Em$F+v({)PXym_TzQCZ*7WiT+f?UnbkDVkKkqr?uEKQA(A5?v
zWz#08?>;(ZEu)rbfk3=j3!7`snYq)B-EmnFdeQNyk>U|G_K2b%XLe2$TPdhkC}%(a
z%`O9-X}!f4l<ioi-(8TirdRp+!+Aw&-TS_8xLW=7wylnP;wF(bhojGLTTs8hZpO08
ztJn4>9p#GJo_F@u)zy{n_kMqIb+ve4VBoKi9Z~l~a?P~D*PUVJw>bb^X5N+<u+(en
z>MEIzjuY{_^YTu~%FUdbbw1~apj55$l-@-b3tlZby7BXO#VZQax%uXA`B$hPd(VUU
zj~M%<e+MVavi($2c>Cw870;(z{~x~o@3JqW`LFu=w}~JAKA1Dds&dYT4I8xgt*ib0
z?c<%|^B*7g+k?)>c(?n#9j|#jd)d1?p8x;-y}2=&UD>_w$-L@!pcVV;+V#I{9sGYN
z>AI_<;DNc*KK|G*dCqpiAN{6z=ac>)ef{<C|0bn^v(1mLJ}TQSX;GjcsvWk(W1<o$
zTky$RX=uc$cZq5ryB=SEc17S~(4G%oDU%1#yIn=ZT-OzB*{);v`;Bpzxc<3yvAZL7
z7O66`@$AsrJLA)P)0`U<rfP>5eZO0Nc}HQgT-}dEAz@+Yo`uuzkIUCD@tUf&=l{Ro
zKJ#p57Ct^UQQcoo^qE!qEH!@nKL*lfISM8wQ!Xv_HcmRiu_tbQm$FjP)R%LPdasDQ
zdU7voP6boKEXnO_EsGAgABwWy{qO1C7Uf00a%<(CHX3xBt5s#(QJk}Bxs8q4^ZB|f
za`sG4*P89~$7U+)^xQJh=H*r`iR*kHpPhT~^&^Kpcb@CKySsaO`uy5ypoNMhF9L3D
z&#%9^{pW*SGks#VNs9}by!gYAx;eID@sj4%X&=6xIy@mSaPjBnkh8U&JByjrBdv0^
z;(aZB&U5MqY5m!wSa@D;-ufEx&}F*I4W(zAr7PP1lRIv`e%qr@ajV}ZKE1*pS)<gV
zBgemU)%J()vML)Gq|^N$w8wf%>|Awmact?FMisMje<vONXJu1ae)99QhiZv07?@()
zipv*voQ``PySwb;A#VK*rLV*8RXpa6+M2c0KVNX-Y8Tt?=^JXcwttOl-eYLCZH-s*
ztuTAlQ#&Po1ZZ59{ht@9sop$ilcYuLgubH>UF_Ta{+2ozFY>)5a*l&{xgVQljmd#|
zr!9Er3+4)mvs%~l-z&+TK55b+Zv8zDOTDIQ1TJEEeSQ6YB}YL8|39%?bSIrTF5mxX
z>H2H&Uv4k1xmRl-U;jrhQflkok^mNi#hMdNA73=L`u`8>-IKlU$}cVm6BlIl3cT>E
zb?a`u^z*aY=THCJ#IAld+C(|`P4%AbCUb6GJ@;kLtSRe7j~}1^Zb@NxmW3_*k8h99
zOZ@#KVYc*#e$17ZvNcw3O^zn7Kcm~;C8m4o%F5u%=X1*?Y$^=CzPg%xxQ+MM2GAzw
zJ!O9B|9tIgFD1QAlV2S6i(&F@?pq4R2FF~a(yJ@LjYylT8xJqY(2%iy-!=Uu_hrd_
zj}FZY`s5~cWDf5_w)1yHy?(yvS$6aP?EZiED=g}h_RR2TUUX4BcFM$`zdAgc_lUDZ
zHSdUc+%#{6RQj_<zkQXrPH)mV)+_D)@zK$nyUX=ghp%5&|Nmc0TU*@4ug~<tSQ^$Q
z|3B>S`P}NIfBv7o&M=0YuPeen9ojj;_MOV4N#|rmHg4S6Q1GVggUU|J>XI2dGw%Nm
zJazn*;oO@iCcl34@H2y_r)R}E`#DDWTv7$Q3*&7R`;XTitop7i<-A;ddV0(R(YT&@
z?`@q5Zf8t;=V$qQj`C}<;Kx7RjjL?tR_^?M@%YZoSscstqrMz0y}xm}Yjsz9{LSvx
zSJ&6qf1Bp+JKJn&^!7X%%OV%~2qmTP39+g_viE&C6EH#Y-lwplkCDvMD_@Ew>NUq|
zU*?>`qbSyKCQG?;d%NA|+9|s7PVZlp@%`gt&D-o)HQ`Vp-><N8p{gt1{gQD?MYkm{
zUFm#ZF(GIB%sDqEozvMr*Frqt;!%g`)4#+h9!YJFoP5{E+w8?nwJBVB-6=Uw3L<Na
zb*FaRcHXJn?->*{VTFct!p4GWvpego-*4ebe1A`&(Dm%*(>wOgIQj7Gz14NX@3d77
z4JR)1o&Bh2=M1A%u8$u-hUeau^SAKle|P-SbT@;<d-@?+((F5C{yb;Zw*T_{Mt@z4
z_cu@KR8<>H<`Z9FaeBrU<9V8oe^2CB_`&@B*z~i1xWfC()Azf5*>#df(xPp*y8hDa
zoofo>x|hC*KQ}x7%#o?Kz2}#!8fG1nSX3|Ldj0yc3!8Gih4y(dMB5(ImGHe9VSPKj
z{>Y7`>KwL4t(AebGp=su$p130A(gfA+gFV?`^)oUOrtIx`_Zv*zW(B;50=ePnf&1$
z*RHxxXDWYB=;Pl}u&s=LwOZX_=l*XGvhK)UH1XY4`r7UGw%p3MTd!-x?lMU}*0V7A
zc;BxBLV`P)u5AACyf&2K%9{5}>Sr75nr7f>^&<5Omww%YFMRy+|7M&!7{ljvqp9X!
ztduuDTc@6g)9jeCs&M|qk5@wC=1e^GrP;CIK>qB%YwYEY9AV;_m+?+1H(Ba$`Vzz9
z_z5vb-@J)<m{`qF_IJjy5AwBVZqGHm^PD;BT3Mj(?xPzzGr!Dzkv8|X+v@7rKi}Qu
z<sKa2<BzrXdCI`%nv`=z_pO4F?tv4nFMaOG9!Rv$mwR!xarUyM*Vab2pG^bpzza9k
zNRv^EQ+oXBbHMikv3-9;tC<qsMjz>6%3Trv<&boZe9RZVz5jz7%j2rMp2yEVB%Qi{
z>EX-wADy+i7B^#0&spjJ-t$kLuFT3{YMo`eyf1P0k#H;X`&%k{a^A`<ye_==McUsZ
zoAu8xe>3-c0BiFk-QOE3KL^Qep0?@fcab%ZPiNk(XN{fz?&RfK_6PAltm5BkJI$Xi
zl{`)AMD_Wt>eD%Ia9?`3boYn4uZr6K5fSY52|nBWLe70N-`l!;`-h*micGZT|6MiB
zPD`r`bWqa6!|k7+PLF@3><wz~M;Uxr!g_P(<<soJcWq`qNc|t!%^;?2w)SQ!I|G9P
zgQtsQ$gjwamJZkEsMXnDStI3CGHd?U=shwg@{-gKbWN7@wltr<<><#VucLYbq*;Bt
zO=n*Icy``ouj_>`Wt5er=gdt1wdJ<eZrzWpK7ZK%%ABZ4IVxvwkrQ%|NvH0X6vLXH
zhsPP#{65Pc$Y=Lj&h6gaxP3O2_iQe#G!OSqJw5HM)Y_Zv=jC13hB<Heb9u(rN9NON
z4JZGWTjRRp|DVs?wmI|c=LVfWl0Hu;A!uQuI8%1mru*zSUIlJ+5i=;h_~~Ff$C{s=
z$}W>v-%$Pa>eBa*wFU=H+DiXSJj!=!&ri0+Rbj_omNcw2y80-oQDBp(@j*V%-SvzK
z*Jq`!xtp1Oq+@$Vlm^p$bJgP0=QJ<=uv-<e`$yKnsIaqhT({3FE%@il+PCI+pDpjL
z;5(JM@4{!=R+rU=bauozO11s#Ucez{zy7n~XQ6|qCjK<PliKvXq@i5=*SUS`GDFo9
zCfq(Kzi!^P_z%sF3vcuM&{Y;V=5l+(zb|Yz#WG)teT$g9=Z6Ku8*3?+oC^J0%Zy)p
z*u6<V<J7&aH0DaCfSXC?jjow8dpp|}tTz>ApQhh1alcu!)DDSnFJf=3=6`a$#_Z2+
z#zV%^NBnN&s#)*5-?%$i=Apwho1G1{Gndcab&%ge(K|<P!_O0+(lyV%XJ4$L`+je`
zSNb_Yo5C5wa<AX4FHHO5#=PXWt;DygJ9ifyz4>7Jj^6^E>E9pP2o~D>tKi+W?~4NW
z+qpCD8y~+Pa^P^k@cxC#{dpNpc7hXUPp%UCe`Mx8hx@LH=TGh7FnJu~*6rKrc!;a@
zLxtw#jsWSu@*InooMp-}Sux||bK`bxWr=5{4p|Iu^q948K2izTxUGA4GViHrJc$+E
z0_S3kv#ofO7Uj;_dF$YGzUV&IgNZ@9pWiu|p1z@|d7x<u_x79oGt{Q&K3`B};IO|?
zGsmZ0>8R|Kf`~PDqym`Z)t1-aZ?=BGekk+SQ%;6?uWoh}9C*jk_<&!=VxHhhuIamk
z|G#qmGp$nb%=-KFy^Bk|KS}T74*K`aHUCk~7O{;D<>3<qs@;whx1`*ebWCnubJ3q;
z&Hi<#7`M-#QrlhmY3I!!hmFM&&hsqiJMg+)XCv!|J^2QYR`XVT*kGHevUvLZ11J7W
zd-zdh;*64auby$0mohDQ*kFC~YrAb(YRl~jXL@~4N;Kt3O%G;o?l|G=W!pVHU|S8x
z_h;u%TrA&KRQPFjN!Txj_TGJKV=N3aE}lBtn15;J-W$$ew=E1=DeN`XYQz8MCULQn
z3zE0=T5Gj^irjf-opk(!XXQIOPGvH;H`hI7-n0F};f6N9cYDocIE+e*j{R&qK1p{~
zDBshkix~TE<TM<V?R9!#tUmWrq|?sLu@X=FoaT9S^DnE)xUC_g&gLs^%_-B+q-ODW
z?vqIAC0>sMe!b_LbYJj^c)w$6e8bL@S&vUYJh5vgPt7y?H5=b5KL6o(Cw8ahUFl_)
zRP5OQ9hom-`OBW6#^6zIUiC)t)Y)I2{$>9ABI0-DuP;3b->#*dsoTpZ_%eFu{udlp
zHr7q<pJme4UY!5)d}!k1?fD<7F8FW#I9anKJnq@q7?*_KoN50;=JyG=Z(MSjVOE%Q
zZ=zZHr%Qq||I$>lo3eitTvC$Lo_=VLtX*Bz@9cMfRxrP}uQj)Azjyx1LV-(pD&FkJ
zAKs7XKhW{;%GzzSS9<>vtbXRK=wI4qy23Bz?j-Sd1rId3m(?Ero?Wx1bH=$ZGUY+{
ze)ZiAeZRD1$1)!g!JUyWm))7}3F-=opSvt0mnCra{`tFe%r8A~*z|RK|J?m|V)Sl&
z?SEEcrXj`K?jN<JO-kfoQuwW};rsue_ucvY%*?|p0vFG)tNq1WxQ4|h?~YtY*RBuq
zTI9-w<mR}vTK|99RIuCj#^OZ<|N38l-EmhyuJ5yael^48OeRJf#)4m$JrB7~`FxAl
zNnJ(7pzx7P+POI^{YwNVYCG6=Pd~Y|X|DCzU+&M@;$Cg77ZW+J^L;+MU3ctE@mmwp
zHhTM(o}GEFV3FR$b8{XB{<qkYs@L&B<N5#Wz2`oBsNeZE-DzRxWtKAqi>@qDQZhFM
z9YKM9T*LH<=OXu<IW?{1!rC;Bg$Fv1C^QOmpWpW}=zE{QqtCZy{0`MJE;aq+d*<+|
zmGd{gYH1bbpLh8|&q@EKQK2iNY|ihGyY*4m{=-N1`vFfbCT`I!n)F=zzU$P?)0x3G
zkK}VtFH$U>5&6u%ColTbr?*L`(tdr7b9&z|cY4N8R~MJOtXGkHdU^sTAAe<Kxy^8{
z+~c5|A`N?PD$G@g?suMX_{1K@&3E?93CX|S)$vpQQ(=${|F>5@Pj4{WTv^4*-yi>Z
z{q$*jJp$+4*FV<FTBbgAm%ZD=HS6~CblB_vRxJ(_v$kAave@h0iPO8<*~8B##HA!A
zfmUXSh&{dEc`{XqYsu0ypZ(^Qepa(D=Z0`Ki$)tg%lG4Ta7j7FyX*XP{cCO)FL29q
zv-bojKi*~|-V)Tj=;E&RcfV@)bgVJcdV10`q{2*VdreCsZ_Ju`d#9=>2yigS-Fath
z?#@49%9NJpi>jAaPnz`M#^H0mKhvKrs(tUSDPMiB!Qt84;&XdS?k_SdsVb^`7WzXi
zHOj_m`R()XFY-(%dBox=zE)?#&-(I7vp?4h7pNC1oh){gh&BBvE%z!S(s)}(#~mdT
zo?1!Ocl;B0rmPh{HbLg)o{d+|&%S)=#KZF|N(7p(<sQy_e`<zT!O1xx$JB3EEZ)lf
z+GRqA#AM~ympwRcRPNZse8=%jfpnR|*K+;l^oo_&F1U8bm@z4uDKRtJ&6qQ-@NmJ-
ze{*I&WwE~}!hL%8{KCJxK6)O~>Pp;jI_KDwm*t<Yu87>RY5B*WTU6Z6N;p4u;+s}?
zq5onv*9DCd<K*&+O2+2i%l1hp4jA6Ct<;P?otk=o)A@?5D{@g=H+;PH!o_aZm3jJA
z{s%TS+5D~&Id5|M*cI{lPan*k|M6q9=mx)>B$w}s66Ge(4&3<e@!GU&`>i8Jc`pL&
zBDeIi@p^5@xLD?NYwxK`2g{3UF7CGZ8!tY$>))Z2a_fCno7m=@o3Zif)K9beBVC?^
zEu44s^hxmtU;kHmuyIW*GyJwNAj&mL=IdPTZ}WIxmi#rCx>rJE_gS}nHf2RK9OwO2
z>(%_-B%+g){@O~OohhM;>C10}R?p}^EqBG7UhfdNC1x7ANu&4wnr}^~mz1V{eIGMj
ziRaq7=p%oAUaow*Fp4L7^8D*=-<E_K3Lnln=iU9xFf&Fy?A}+4o40J8)~i=tJo~FN
zJ$~N#hg08Q%64>B+w)DtHom6$IFGQtSmD`O!N)hG)*Q9o+a}hZ|8VU}=N<nSs=r9n
zO>K{yW~Xm@OviM>_j%d=FE^ED+}feCd?WKopO^m_PTJmo_S0QH^-Py$%f8#~+v~d)
z<=(E6NGq<|_scYU`QE1Qs|)`;EBkyncz(blPw#hczX_Eu{qHAKkaD;DP3FaUkNm#o
zfF{dM{ZILyZ#VD9g|lyB>k4MvsmzxsJ;!Gm{b!FcL&TEbMr~`q=YIR}Nk+dmy7iLj
zeqJZuqQr;n-*Ubc^89#zsz>on(@*23#DzhMS5D<_NSr=Bl)YK&*H?|Bhd53j%iS)z
ztaATMo%<hj&vvIP8yz~ZCaCk!#+{ns$4&-mqqf|pJ%0W&KIrScRVjD$tXpzs9=@>5
z_}7~WUtZ3Y-XOZ(rH50%cfPOm%P1>tyZmD=4<(t8`z=!5l#+Lz@!8uCfg2sJoiT`>
z@hem&$IC~pI;Sk>=%W+zc@MM#=4iQvDfIkc)SYp0wcy?RaSn^`eHUtNmshWIK3=Vr
z!4c*1L~K#wi_<UWrRgbMU;4Rn(a~`3%a&3cOS3sATdC&g{55@S+4uS1DK;jtqh4(W
z3)VI7yua(`PA3W3&(2k8hiA+@ZyMuvOg{ZGLsQM33x_wJNj)OIbt3cYH~*jh5{zHP
z^z_u<k6J0OOP=x9*-N-)R=&3lIW;HxrqxGjVSaXtSf#&<!eb@QU-~Q8wA*N(OG?Wr
z26dsk^%dLxzp=inaLB--#)!3p;Z4ffeD?M7_5y$2_8YTJ&HT)C`O}nz_ZK(sO#WlG
zFZzl~GDnq<pXs`e6S|<$n2U|U7dFi7-ZS&R{S+A%oo{Mv2U8!YJZ74{x&8=qz_J!u
zCdX@PPU|!jGA10I`0=;SO!18_F48wRH~k5>PcT{#$zf@banWE3yVoqPuImrW4cH5;
zRR8ktFKls{^Lv`u#JMP~rk8*3D0P{wKXmQl<eMdjm?l~5F+R{Ym!0v$J1(K5@=2=O
z7^c0qb@Pil$k{);H<-=1?5*dYdp%z(Y&=!eesb~}{`i_<w1ey7f;x`J%kodWDfLOZ
zy`@BI>S^wmnm4*EjQ3mqvSi&6bF#+ngLl!@S0}dH*L6Ide`Nh@u}#YztGn8>K~Z?^
z*J`!tzDb){_1K%=@jw0^q9Ks{_(h)n{W)7c`e>^sTv!msvAoZLll?~V&6C@VIkw-F
zyrDd2@p6XtUb(NbZ$3MgWQyk>iEueCu_k`fu{CSOEW2l3IVFGSgq-4pC&>@QckO!m
zODN&(mGU1oTc^*>Zu{i-rQ2qnWzK8=Kb@TO^;I^!W|@Ch&-=Sr>I;iP32!Uj1({nf
zN~9I5>B~AsZWio6o9)_em8uzc<`|#0$JVL$jqlDq^V-q=>M{Rik2dogWZrPI;;ou7
zbG_47^?S)VznPZguW?T}Qt&G5_E~rN@U(k(LMyJxJzTv^{zBOg@i%X8_4zrM2W#i*
zdN%)e-rv+-`TFK}j-2%e?u0Hi2=7ZzFW`;$`8`wU-S?NqN-b)Z+k)2K^_Hosw~(Fx
za*bk!;Kb;j#K>u-wmBXZDPgN@Hw(IFeCdt$ZFl6Gf7GzuF=orbl}{AdFa0~XuU)ET
zSt^g}aTh5W-!{j)hd0*wq<&%K4*%48_{_tFY&&B%yk1gzS*GJY|GASZCm-kJ=Kg(s
z{aoX8zQu06Nx#0lG|$*~X2;)NTP?2VmHgFr@(NlXs}#JO)6!jOvty(1I}`ny*oq&l
zr*}P2_4NEPH?YO}|H)-L3#%2hv}F3NIb#AX)^FZ2W5R?3M>>UPS{5&}yXfk&j^V3L
zoZr4P0nazznDgU&^Bn85kIvjobDylMkUT@b+-~xewRS;zAOFN=YJK-TTp}d*_4t!#
zJpA^bTkPxCUiSPkf8RU*xS~Lfzps6{pPyM~d3f=aFeRn!3UfDC+%<f0<BVrWkwE#D
z=DJYR7wSq%O4s|U#MWC&8t?ZBQv@wdOZ@mNrDfKa?1w6|IazdV7z?b7)-&+3&73;n
z+$3!)o}3Gtoh?%RD=Tu&+3#;yTC`_!o~W!vlVe%#w-2{o^?0w(4lBL1bB?V0g{0SM
z`uXd=t)94k<74Z(gfrQ~H{4mCN{1YsIcaKN|F$RV*{Xue^q-v#6#kKQQET4M1Udd|
zrxrVPb(|1gp7_GM(?ny2Y}i^4^~CkA_aub-Dt>0HY<Ccwyl7&06|)YX`Y+3pSMT$7
z<X%|THdn?W=b}K>)a{0^cZY<@wNDa_y(7x`^YI*;po_=4Oyl^fcu$3uT--fl*56rP
z_fJl^<{$LtW4NUGyU@JyoTq<VHS24q7CvA&{H|`c_5B|$?~i>^Wzo0S|IOOJN)xo+
zI6U5`<>0ibFKhPdpSZT9=Rnz?hzWDsnf7eI<X*CFzee=499_eSY=(W^i(*$eUVOp*
zK5)AEr%T`0%-&}(_n|gZ`hR)3Ss(NFWjubv%3GSfJ0tR(qu&A#v2K1l|Hr$X9qs16
zx81F&>vd~}wPi1-bnPAs!@0o@TY__Id^aaO(%ZMa$;G9O`C&+&{R~CdZJW;*KU=hd
z!BgFD!V}Bu|CY$4vPT{eR;d5G?#eg6#g`|aJ|8FHs_=QIoVn*aW5G+#L6d&>@4P(s
zNT;`ro|00Lrc~R$mLvCT_RRgc`Sl{Fx3}Lsy|Fj3ChV@;B}aXKwThj~BQ?H1F~3pE
z!}WB-!|#zdWZ4onzC2!X?~cp@S7&kEqMIBoH#&G&ek2r0wRc%mu3Xm~IXQU7w3@uX
zsgGq!R=-pEvSNCA7>neMGbeNPny-{}8%$^Ptc&|+c%XKv@vlYma^5HXF@L${3!_9y
z_QRINcFR<qfApB#Nq66*d+F+)>vOcrHKQcV*tV=X*J*h9MbLhh?|g5&kLM-Ky>X=D
zowLEaj5CcJ-%q@jEs|fZd(eEbeR6sMZ*sh(-`!JBBaBO!rinhed3|xl?0Wgy_&C4k
z@2=kb_R-~<gzHQ5HKw!GYEHkAy{-B6NOjH4#e8aV)As#mS^Rz4vTa{2=cm`tR6QU1
z`1eBF7vHan$w_{eY!$aZ*6=s>=citGjj6u!{7OoT<{kZivR=qWg}HG`Pn^5_hV)dc
z_v=4AWarEAS^m0xWlp*Dl(lkB`y$wLyJzV{PB~T|R(&w_(ymP>JB91E$$#1OdlAd_
z%+26A_J}1azh>@KW?-55yLa|;v1-LrOYV5e-+#yFbl`l0=1j+niOCoKKD=@&&!MOJ
z+)hh<4W}GW_U(cR&(hAS*XHhDB${%eT>iqL4IiHguU&a1S7i1bzd30}!Mp4oiq3DZ
z{CCk^a&r>b!uQ`Eu2p5U;;(4Z+NSlT_NVKvEt#%}vAE@#r_(|<mL+P}%0D(=|8%Pu
zyTs4Ea~bdZBtM;!e)HSGG(YAg@1L#Rs9o{k<UJeHFTytS>rdprTeGF6P^VwKd_&4f
zrfHQ`2H!6q7ZCxSUGVY$QSpR#?-VYo%7h$p>y2USHe<7_GV);Pp1&=vWw)nvO~S1e
z@g|GeHw3H{$a-paW5emwtC}5ygqZitdhm3)eazvc!asYCm(AUxknl0sNan7zMfJQ}
zGb@*Eo_BYh!eO;#7qY{wG9^MZOYd3iIQ*n_SJ|{PUXRtcJ}?!ZxL!@!Y;D4&Y+H^s
zzozu@-b<*z1Z_H{b-Rfys}p6PEi~Eq*+jNKGvjq??)B`>uVQ#3c~H;4@lC_zq*SfI
zJy|iwjv9UASaU#%Z<lVt)F-b)xliu>XZ*|dyPia)QT3VRii$7NIlr@%9{NU2@IDLP
z<?+A%{@M?Z4lc>A7Jt}jH?6=?k6FR?E|X4;V}pXc{uhI)eeD;y_rBpXJn>QdwEh10
z<E-_S@z1*3rv>x<oLOt&5%FSYn%u3MN6%TV?6GD^5U88Hwf%PO<2lXWPha7WtUbhZ
zNb~+D&VALJwWrk!dDnR>f?G^a-vmxD-7mYRzV7QU*7jEW$9r7u8@>x24}QS#J|zFl
zcSYWcw{IP<96vi#b-v%38P!#<f*)H7?AUnAPF2-a-UC#qT|ejfpS8aI{H95N&sa*R
zXszze6))kHN;%1sS$g`$!;ps7<4p}G8`2b?99ZCbI6zJ#bCO)P(Tb&$z549enVT|{
zg+Dv0@261ktj3vH*u&R;eY)Y51%85)-G26NJ{x7*-|HLA)ZloZ{ZNP9gJe~+S7K(8
zRrMOBhw|o`X+>|-Y3G+;<~!T$&7GZ}ciq1yv$Mv==*#WR;>%M`&SULK5;+||jqR(m
zgn6xV+WSJG+sl?-UJ(<w;jrd`d6hzI?TvY=Y&=z#Nx4X{DSn$!e9%DU@#{A2&o?&h
z<_X%S`!sm5kZ<<0=U>ii|B(AoX)|HdyoKv`&fIU**v=Gx>G@<)bC=!KHgceY^X~7f
z&Ahm%)ou-FFX!ADU&NJ)ZdZQ%*0X<`>*`%EWS>2JTl)X{y;}B=$Zz#B_4AKD+`tlY
z#pTW{=_^~#$m}ed);0B+{_U?nI?^tCoxLRI?(eT@wkF3yi~H<eqpd~xcOJR1GK!|k
z@}>X%cR<JY^clt<m46q@yesa1UwXIw{LXh@qqF90J?FVxFYTG?-*0mK7uIykPc`Cg
z`Ca|vO*Z?=GG^_pnHf?K57*EC*2tsYyuVB+>bBR@O=4QVtj<}B#4g{~%9q-cu<^sI
zxdKslnbh=ar6LR5e<U1pxP8s8rsBxi+Em>=A2v+XJ+826%|!OiwsYA0^3SHeVdSsf
z^(*kU_=*?KK1Vm#r}SG!Ob9NyrD<a`=kD%u<FYpqpl!R&{B|c+uixiYP*Bhq6aW9h
zqaAfJUjH3A??fv0|L^@jt=-*cq2mhuUbmXF58dBAUue#E>B_#wo`vzBqLxg5|Nl2%
zwe9PlFZKslJUc1Zay_@P`U}S_bGs8Cru*o8t}@Vzaf%IBnBN<D_FdM&KLxLyQ~que
zuzCHn?wnX$maM1ydygu!b<WHGzjh2g&n8#-CU=+gvdJ~))$a%eG|rN(ogwjCCBLoa
z+=buy+8c8JK{uzqy1M%KudlCH#_Zf=x533_onZ6z*I&h2L_hNLEc^KRV?>T(>#_>p
zs2eZ5<n;B<^G{%#-PteqE%oDe%Qeh>OXkN%)z5A;+LwOb@sH^X7AcEal}~*Gj4m{o
z8~>kqMU|O%%BF)KuWUZQLeFIyukO*8Unc+f`FFG8`7K*a*!bnnoSA71x_H6g<|B(*
zFk^MfqT7v!Q;xnm_w0%!d)WVgKl(f7PE=NAo^JYrfBk%O-!iiwzmxyYK7Foi=Ev=`
z3amFBeC3~5wcyFKv$HFIKAnE@%$X;k%LY<UPx~bzF1YhpiNr6iFE3vmUtasCa@DRz
z%hUDJnI<g~8=1uR)N^z(+$?)v=_vTa-BD0M_w)9I8Pi^-Rx9#!g$JLUq&m|ek;!|S
zPU5{im7opGs@~H?)=V{t{LDP><opX4E_AawWnY-j?#l4`K5tk0^tbt@bN}W)<m@`$
zFaLaI`aDG|t6A;xbu-S+HrLkZ>d?>;i>dD4d9tMX=g;;#_w{l%N>u*T<IY^Z*CZqR
z@qZ;Att-4C%l%}h=|mm^Eh_%^^Ev1+M@~*oEg!|nvuB^4VVIn8WrbkssVRzzijB+X
zRdFp2l#f+Xy54;*au3t$YnBmP-!+NPm2>ku9azXU19XApzS?Tg9Pn(j+!q%Yv#;O#
zO-fNwQOl1}TwMJ4g@w+U7ZxzSzP=u`6;i9K;{@-5_0NNzMCypmtN&M7_VyO5Lr92-
zeBBR4z3JXZH5))XL1&xgR(!u(4%+Q<dAUD$zjI?r<LT0CPx!Rw9`1ecZBv4y-fSjc
zo~-xYncPd{%MBCPH9y`yqjF;>>*RwD*Dl()t=0Rr@L5*ugCfhaH#ZI*Xk_;E@kx1d
zV&azU>wfcDSRIOrjGWu~Qg3cb1+7PQm#+<JpY7_x)xh-GGiPJO37>m$Pq^j8(iZjH
zKC5k9^z&wFl1sSsPR`04i+2lCBDv3R-9BOJlSf~7#PrSVy(1O&<$pxl^0!Byp4y!k
zu6un|X!p@>@z3Y1-y39HPyj7fDSLl!t$(xhk-Y83MNd4WOtV~ETv#|cIc@&^cnrC-
z>iKM=m&M<Fs#~_)?d~o+diO!mbN**xt25P#m(LYkJ-5c>`ng9T{B@~kLl_U7p6J7`
zx5sst%yRzq$*<F8e}0+lzcFg<_0pTN)@3Qbzr6+RvMzgj%dp^qgWo)xmHw@-U);{$
zzqIb}uQ&Jh&hC{qH!6DK!NkNAA^}>tZU5@ZN@07QS*H(L{<YYa_{lf$PfG>6bKTFk
z*Q_P|9`wwvO|RE`6)wN@-JJ95)>nUe8y^4RMsokX38fn%bZmaV+5G0_X7=u*PJe%W
zm9($3@mn4|pYJ;8x<=5^eH%A!^qFfl_5HrzeO#hiD-1xDTm8Di$H&&hel>oXegENw
zOM&L$-Vr`qw6jzG9-IAmQTO84cF#ZjY1H|f$oTzE@p<0(`4=t(fL0uyxBq|U(o*l8
zZ@1l^VPC(`PI29)B}-J~>i-m0eS32<*x&YO8?SW6tt~4d>u`Ur3SE6{+1J&Es=6uf
z*DzMtzMH*B@z<T~qr1-^{LOCTS{Kc)C+T<WSj4sUk&cJkc)73ay1F8;xs_YosO(Jy
zXmR^=z1SN&i<jGJ{#!U_j?BHP*Sb>Xc|J?Mrh0gIJh<pCe>CC3Uk}i=F6$ju1}%-)
z{ww{|PqmG&XSWwvzjfUuXtizYMTNtj|K&gP*G}6aa`x?)9kD7k|MS;mp1!%IuC%dk
zUjFZIZ*OkPm5$w2ve3Dm545LRSl#c1@Z<bXReNjxe!cGE>bkN1f1O$Wy))7IdqIa)
zcY$_)|2WUW!m=UdV$8cWp-pe+mmixMY@%<Iaq*B_KgW!{cYQy5sr@S1^s(gZqH@s{
z*7~z!<G(Kndf<6&O^n~kNvhm`tY)801MS3~XIuT@^Jma8o;}j$e$S2LmX*G~c5zdx
zx0=t416wjL|9Cb#zoVlg0DK7Tdb#eShs@)BAN}98J?XUi+YR%rVwQBthn;M9Up4zc
z%A)dgd%4`?#i@}q<mcCG>{>Br?~Lhlk6k-&b#YUjMYq@)y^_UVii(P$t^K0fVINNG
z@4v9Xk@?x#*~|U=y)CM~X!P5B;@DgLeOdheI?y?=YQD2T>7_Mf;f@F2-Wn=vhOg6c
zZoBJzwtb)Cik!Vlfq$+rSK1gUzQ5F)to7&ijM_&G)@tR?XCALM{IvFAYgP2B@b%~R
zRDK36+yrf6ILvRKa%Dx}m36Vv^Hm(m%FO=%zW@KnkH`H#zwiHlRy@9@Fg9<Ko|4k_
zu!WZQZrU%CkFT+sCp&p}anbw!)@=_gE91_n>h`>O=52iY=yJ{-N6$!KG`ls`Y|`ZL
z3hV1Lm1p0u+GwlXzBlAW^P~4$b8k;u8@(NLx29G3J0Bk(p3KZl^SA9C*I%c;y0X&6
z)wS@)hs13;HxJFu-?wsshEkA=$c*EEW7kd94!^X~+x*m4$#c4a{Hs{GR|r?vCA{xR
zpJ9}#6uYZLbJOk0UtcmoE1;J7%sh0Ukr{Lmjh&sHxv%y5$jxoj<El6(pFDD7WAe_w
z-)>+12}+&$dt!H&UEK85mgRbN&!@BOt3<hHEa?rl-1UW5?9ANC%PsADtG|D`Q+$47
z_H{i_Dz_?s7qevh_Y)J9_k6pR4T_7v#crUD4tLo=_o8iVV4bu?C2Gqq_VSZzA$j&(
z&GVF_6=dSRnMqnN+wkYOYsl)bx65w-Q}v!^Q1HM3v`p^B#l<Uwm-|h;weQXECj!r5
zYXRQm{QG_X|EHJB=L-o7ANJPYyX5d$Pzk{r?&ik!y<|`S;+u{k@BeYSZ4T9~YOb)A
z`*yaP@%77Io#;&smzH|p-?Ft}_uZOLC)If*janwF`#<_xWhlYpn)YOE`uTZ1a<;QJ
zrJi2(7&Oo<R#RD7Y4amM{@M}sqU`nSH{5W#vg*3EjOE)moz-)?4)Q%aJJT^CVL_ME
zzgBrrLfp4+A83`HYtrl<H)XfpM_@ya*`H!&+jro~j-OZF*~;FZP_Xq)>7z52N6)R~
zzpy4TbeZ2jneL;D7R@m&e&!Qb^U-x%&do*Y(>gjl9?EKH2z6dhNVrq|{==yQ>Nh`S
zS{qH?zB+Y1^Y4`|kL)wu*H_Ore*5|)SJ(c@vu1hS-&Z@+By$qzJR6aBb3#Sj#OmMA
z&foX&Wc}CY*W~64M(vy#^+W2U#kQ$)gXJIeF4X`0rs1E~3D$RezsG?N)~NsYSzgrH
z#f8hXIPmrYcKMnKmzH`L=WqYL@<JWY(${6d+VQFC-*uH0-+9{oe!{$H@#4@4+%^Ax
zK37pw6A~9+p48Fdu`v9Jn^^tU+}mbRTi-f#w$*B0|8hU;<!Y76)7w?_>Uu4WAMbd*
zZg<E8ZW*hR4!2&ZjU_LGY}o=p`zX10DZBSw`17tz;&61}%DWtQO@7FG{i)PhZdQM2
zlPWjpf>N#`#{T1n@0Q>9UG6ux;$f?Jh=8D=rcl=xTZ2<mwT~Yw)#luopXh%_xFV)9
z>!bFBdj6x1JHFMQUiNjhsEGc<yXE(fzWxeIwJRl*l!8)3UQRq3Zz;pK>;C=jvz+QX
zz8+jTqqh0Vr|Bmj7fmqut9k0irAU){v7=q0!GU^7$NOYK2T_;4z81L9iB;6a#igrf
z_M>;Z!d6ecdB5I$xmYPEH}~~C&iZ-!!It@H$Cs>g3ES}D@+DWUuIZ~LU-o=|Z?B|j
z))Y_#Y3PXE;x(0;`Qc&3yxM1q+iEVgt+`^G_1Au4^)HEN^?T0OxAD%jtd$B|Ew{8n
z^KculvXWBMak*-rqXL2xm7XuN;Hh2cIa&Sk`v2T_emzie(z>%SuAuSEW9yriPwV0;
zH`?p(U&3wr@64;Kt3hqGj~_p_w6=n_7cSZm5du0a#pV8gfvHw|y(8Y9d&QYP*J?x4
z-TQWjTE5?39KJqI>%{yM=gz4a8BN-Lzi#&Lce_D%D+;zAU6l6u+1V{SmkTeQW#j(i
zt^d6*a}`rxe}6Ogv(EZGUu>G$`LEskDE{l)TW>!<zWADttzC*rO0LsBrrrCoVuc2%
z%Dgs>Qzpjd&9ZkK?qRhjHp<68zRW2vAT8}36g25upZ&&yhfdGV&p*GT@No!_prGcZ
zyjK&C#?Q5>oD`O8U34p^bJh30;~7TxO>V#YT4lK*!(g7Wcf_~HPN$~ppP!w-@8qki
zt0OlgG=7`VZBhD41hmre(h|=tIX8nq2QVm|T_NP~@zK%9y-AW+yVEzmUj4h!a#LQ{
z+iyGOzM1(?<me{zjfH=;PEFV6Kd^fJzEdlMm;d;1nE$G!-lF{b`&``JL5<~(LyyiL
zN!x5#fA0I1%$UyH`K2oQ+Dkq1*Potv@|@<yD_V1ppPFB<qN4KT&igZI#*@{2AAP+Z
zpMHN|?JJr6pTAy@-<*5f%&kvmWfFLA?Ac=}MjwA3>92e?ab3>OC9*d&U5zy?jrum1
zCRJPA32olFdD-LFuYb2^g6gFSip~{39<~?mzWYk%z{cd`72j^A&$Oxh<eIIfq!c6q
zTJihi%jHg+@0SuznRFgZ*(f^m?Wd|3mGyGxiw*q^OHO3h+pWFseS2GO=FLs2>-YV7
z<$GWK*4AwI%gcNvjZ!+w1O!2sz5b~-tTWBEa$W24_V(6CjlbXTUz6D{XY19>#v8FE
zLvZ56iRQWSKOQvmgAPd24qta9#VGSR_~P7&@xF7drcP9L-&FYc*z8Xm>p_Rwdrnqc
z6SH$t*40%jok4zHbfWd${{MAKN=iT8*Z;2$&fR|d?Xz>%@0XOnzxU<!b$?mN<zjlz
zo}8R~Wp#M}`hCB;($3DhYjXSVwdnkhm;LQGW?WR_VQaQ(0&V2%iUD1O_wxFB`Hvqz
zvaqp<{d{|OOQ!JK+uPGGFY}$E8yzMKxv}xzsmuQMy{DfhJv!3qT4h~aTny@EfVv)g
ztG)(-_9`D*bmHlWbLT+!LjQSR|9`RfbUoKH>-qKnD*NPYPi@J(eB}jbH}wjm6Zbb}
zUDaB&cyVWc|MHZRlU(zDPt%Y0dw*{)=p5~Z3m0Y;fI>1v^2z(Er&GgkY)<DlPCKI!
zbbIQ=iH>)7mHvFOxPQm<dDT}gfX&;W5w4`K-|yBd1!{f&`tmY3W!q~|Q!#7nr;G0L
zD<`_TxG-$?QV2hE{CNAx$?BI^g=&M2>2Yy!S$KchOyl%TNk_Rr)drgw=r{u9?oa(a
zlE!VV+~S`e_uGrT`Yo_**|MPZakk6+=le}o^S!b`Nl7VPGL-43dR)ar))+nU_j|v`
zExgyKKEGxXs9)n4=;HE(Nh)$m{Tt9Fad~%UJUu-<cyroWr|tjd*Z;G8e5^P5?yk}Z
z9Wm1t;3$pBonQA$6VxNizP3g)r1ABIh0dUDAxpid2kkD)HC@utae}o(B5GZi`n-y!
z=)9e+=J#thUs$vG_~VBY-Q^x06^{=I0Bv+->D4(EI(gD0r<4>G(1n|YkB_m|)!wgs
zE}M3K-q}s5rwydO?)7v59fY%Pr=)S(hf~_?K}R0_`SXYCP1U;_8=J*+qc-H-wbBY(
zv!YEH6bn;cKYiFPuco2V@$vC-@e^N*t>%K9R`>VU#~+XTS04oL;eT3XRs4*n^!2r;
zll|=$=HA}Mn)f%Z`mO1<{QKwDL~gzk0bYz!q^uLWYl>a%uMMTI!<?L)xXx68cHQUQ
z-6be5?;jB{L&^*6w7e;d%xn{;O*<AIUmL1$P2Jjh_K6c7$9g24udj=}>HszC=G(GY
z*Vmulka)P_K_h$Kk6)lWkl(z?+4tv@w`m{fbb?ENou16vcrypI>>p&}lqoI8A3ywh
zJ^uN--S783JALldDJ2t=DeL$DlWG8ULF)h2Bqb+bH2`hVZ*~)VI;;9|uQ}*e?wK=Z
zzPY)%ea;-2OP4Nb1TH#a8vSgJWijZAhkg6@nPgw<`SGJ-na@n8)6;aXD!{dKb8{!3
zon?CD`0>e;Cp#x3D0olP*}3fW9Mf#EyzR5!yveEi^U?j>982f8xOq~}E-p{<FP&Hv
zV=O5t3EKIXb9dL&H*em2t}Fll_q%cKtu5Kx-Wa8xD){r`<L5ZgJuSh@{c65m4WHqo
zW;y|!jP7{sE_=Hu?d+@>K5BRJzE|$&=Hz^MWo7X7l%1d*m2SOKPo6!S_kGvZtfQ~L
zPMSNn_hwEQb7x0K44Z7*wCrVWy;7hnO|42_{dst?zOu3sG!{8ar~Kj~*ZPWUo9EA$
z=ePYLkbZ7XASkT`Moei^Qc`N-l{OO)7H&S9X8nEF?YC(U54B!TnR$7+zq-0QJHzCY
zE(r;sJDU?99qIh>>(?sK5f=YI*1dkVe16@c=<RtaX=#s2tTHbx;nWUaw`bMqg57sn
zSXn1dp4>ctzWmkI;px}bL{8BT_v2MiQYvCK*`#sZ_w%!}o*o_#*6n_GC~NDctgBiw
ze8)p$kNx`k`sB%z2h%niCLCb6wmyFS0u3dlPy87v8z)5XDNszlFBkmc+~M2xFAS`O
zgm!%1c3VSEsOIXk-P!NszRB4fnae93WMi_V;B60^@?*PyGdemrN^=ASH4RFZ)hm9l
zy;GiYG+)hXkI%xW_t(sXmuH75DwqowdtD2iogZ8;&$D%PVriPQSJ}c<CtuXINA9iK
zdT0@-fIW5f>Y@~lw}-q|2(;cgWPDSrsiQ;00^9M>xKF4+k?FFrJ-AfExJA(B1?!*g
z-%NRX_{8IUB-Z{sJ8ye6U*`OO1#kCHG^v~ubT_N{Fl(Bz+ktM*W=@;P<sWZ(J$O9*
z$i^9hJJw$_@_ArcJZH`P^vORjuRE<DIk#H#XM%N2^S_(FIyyQ;M5Q8iE;)Tz8^!m0
zo9mJq8}U;u$)<1Bzdm01B<1(x@P$`OBe~D7>YNiZy)(bU;@;lRmjC{-1gX~@cxk&c
zg6YypWsb%3f6D7CmlhiBd-$-Rfb+QhJU_jsnjdcjczxgbc+>O8clWU_|2cnsKRD%z
zFzvK3(^YzVx^l<2cZc6H`G-lJYuUPFzkT-|<I10tr><$(8P)pvUZ#d5N5je}R-XO!
zwu`K$trBnXab|dC_0?d0?fIS4|1uvCXj>VhP<i@sA@e+8_5$^H$EI$WJ;$|hPQ;d;
zJ;pnyo9^HL=-wu2^9OeOq{|O+UAiCLA^l-x#LqvK;k!i_|9?MSylv^XwLfpawC(xa
zBOJ4?JJ0@~=!Pr%zW$LtZm&?E-nF9cUyXiXw4mU`LoSEaU1xqbc=>q$jvw!X(kFj1
zW8sl;_E43M%jP`u?yt=M6844uB}ZPRADa63*qii2v*pfDOm|QGd)<1<nwjqBRvP?S
zlWl49B6-E-$sZbJKdPxGbMdz?&i44FHg%H!*2Cr}y2KlNg4%UXolh0Z{`ZMFD)8A<
zu_gbqIreRzSr@&&`^@_rCwiyU{l6b!arWem^&4#OiPzhom}Na@OX*jQ?`h&O`)gc2
z<#4kdm_2v0{Fiq-M1>!mI(P2>&ZEXtDi_$rwR3uR%<R$3FVI@5`tYyRqZ?D({&zo|
zBlCwP?vuffYf67MYfAX4?UA&$5Hn^v?3^F8EHtyI|4CA{VZ)ajoHKUYxdl`R$h9*s
zzRI=Z<ez`CerN9dX}_hm-$25B+peV@9U^*Ck$Yxdt(_$_ajC`UqwGucZnPd;Z?rJ5
z-EylYtNzR-|DPuc?Z4foxWueZP?TfWU$L0m&nKAW@a%kMaHj3m6sa4m?~nOge7n`H
z!q?mU;kM@0KRlMp4|uDoC)+=GvgqfNXHDWCUVqVgYB$H@=lA&sjz6)JRy)7z$R98J
zN5?xwJsY@>x5az>%~5@DY-6gqjAfJca{+a`znb>)%yqGCL1y2$mE^h(?ab#rZvR7$
z=c(q;jN&`e>4i0qa_0OH5V_0!|JnA$bN762?RNIOT_@A5-@vKA-16h6|Aj9;`${zB
z9W`Gha&%T$Yl}rt-248!xrf6HCx4ka`y<oN)5i-xOq`gKK7-@M9ogW!>Hh+5KeC;@
z;(TXE$Du_ca?`A)GJMFr*1A9|&uIG^^<sCUb5C>%mL}OhI=DBFU+&HWUzKWqHap*>
z4vDMHmlxkZBpN6kFZ@K-YL@-sBk?Cn8!Rmtwjbv1Qz<h0<Mb@t_~V?nIXCa`J2$c3
z_S{4F`$r!d%)QU3Z@E7~W^>&kXHKW;kYgdMR1%anyp5f)_;&K0{XEa#r%m3S$m+LQ
zRDb52e;G38au%dGuW4B6qoTMrU$y*i)ww@grp@WKHfXu$*Lpr_W6n&UV{!ZmOyOKR
zGvoGH^Ut~a&*aiCK83Fix_OoPN_A=<EDjc1@ak8l{NEtCC)#qW3FDck9DDvg+A1n2
zsF_nD@k{N|-Vd{zj`G}7o>Nh7GH>(xpIdfv3;(Qq=`=Ih{n6X65~{PVEjm+we4<f`
zW!!-nC7v7nPad3NJYzSZ{GrX9pQrzSHQ16adz}645$=fkxkt_|eLnq{ZhGH0g(WwQ
zryQF8#y0bZX+_5+J@fQ7eam`le;wu0DeWERy91K0czlwRcbRvVy<qakqaH?mbMD@<
zDo?cht9yLevjC-Criy#plb)Ft>4(`kGaPN}RZ;n(;QnjB(?`za^Y+s(UuUTJn6kS;
zW>4N_*X-;k4|dLXb#dWh?$x<x(v`T4DerpzN8Oh!Y|g29l}pXtoFy2~_bRU`l(eba
z_n_=)yLB@^+kwe(Q-c=IQ~N%7q0F!IMLbguoZ%`eK5%Qn$(wy4oc9F^RNfqUT{K@l
zV0G9bmWsK{>$yvBM*jT&&nowp{`5KW4#)Us8?dxKE-qqSx%k;*Gu9-#&2!JhAD(~q
z`F6hAAK8=A-yKTO^Pl;j??A;94n3b1zDn+B7lmbFhEuv<Hu!Ho&RJ07c`4ZT<SEAJ
zl(V0&-<@Zl_VVoB$qTsZEFRS#iQQyp7j|I%jrE2C!5ja)khri#YDu2@=M$<~tNS#R
zlv;Z-Z)`|B9MX0u``(+IKjbZ4?j|`-pQo`vP__NP^nM2Jwh3N0R<TanoTWXF+1<d!
z?9hHM*{1i|{qsx3J>%1ZSEVca*g4MV`uX_!QqgM+@ikhr3)`=#E=g6n7QH&*)4OL^
zwz#IeEqzp9ZtWCxudn|2H3@#bM?ZcAP05NEUTn#6zsv4P%6+ew$D<RsNf?IbFf8Bp
zz+#rXNbN4ZB`%M<Bx;)sZu3nrdwf9t+4MgT7j0%Vkd1zBePWhH*PSVQXUr-)w)c;5
z(YIyJTlJ>YZPXU)7A&56Ov0A^<UHB-)lu3^S|O?{Dh)5y`yE?fRm$}KoliiB!OO+-
zl+Bm0s#bqHviz0ok7;HF&)P+rqOTpx;5+n${TiR_hSpyXgLS)Q(j%F&?l^8-Hq}sK
zZNzG`V>9X`=HCk4c=sOL?bz>v5jHcYJX~Cm=l{n!w}0=im-24q_4yZ98ehr2+jeVf
z_Uh#B4v)sf4^?~PrY`*O>$-5i@U|jL{;As4Pv<U3Vf9gb;AK&A^s?0IyIqR4{O`7`
zzC2gY)ZY1|!=B8UKka|8AGmq(<mqSfZ7Xv$?zP<A-}_1OkU==_(YYIb`yZBRW9WJM
z=cCH9C-;R`xJczWo1fZ$w>3C_uMgk89r`Oy-(UImz~1)CiYpRw6%QDp4V-g}9{5_f
zXuqBB``YtHZRquvL7%O~MIOFqcWrnRa))!J_!$$?>#gf!OCCm62yH&M_xN14J7Ge~
zYCQKs?o`|UU$IR|Nr@G7gW}F0MqkeA1$vd4%Q<SQOpd<OSUjtv!(-y&iLMj1r*3V!
zA|AGaPf00gNoESDB?vlN61>EU0EbKRZu&-!h$%%v^_ypH+HiZqq*I5rnjx*LFAnS5
zElMA;%<gq9pKxQ|?9;n^XJ_7f(x-HH+S=9Emt|gF_Uej~QqTg4J#p*9QztaN`S9>C
zsKtJ`oqx3iXt~$SbCDt>9NO86^Po@4kMk_%iCcetwULM8<QX$OK0iDA@%#7VZ_A25
zJ@Gtr=n$x3%*M7U@2(Z-?n0Nuz`QU?Nl6jyus4hLb8~YmtE#%X3JD7C<SUVg;##s`
zfx@jVnV;LQ=bA~c-}9-9O9j+d)-d0_e16@m0|yR#j{96;^X1LW;G=?qf;;^{de+>z
z6XWFUyz%zi=W*FrRtRQaUl&^F;^MMSxL3!k@#*W=-pBi7&;Nb%{{8vO{`PARfzLea
zdQ$zUQ~jK5SJ}>(lK1ywL7@{EA#(EiuF}`V|M$OMw_C5W6>>b<Dc=(*Muq?XerE<9
zQv(`JQ@TD=uVA9~q<QnsbqFfo*p_?SH2&Jl%gZD8*V&feuQgYj{PNa20XAk?>#`;J
z_xH^>n+7^*O+!aZ=hXd&MLR40|NUP1`K&qU<Q~)PYkTxgms-d~ti6`KPiFh=vt6Rv
zC(fNqyR)NkOaA@1UOmttw^nmZ-_Os_jS~(q{Q3L0v#)Pi&dp7(WzmlxKb~n<Yjt&X
zxVo9yG%3@p70lpv>4~Fn?(CfW@9*!;>F4EkmB07fTlF=s>uuT27|>0JtlVNB9yIeC
zB^}}D64zhHrwFn_i9y|eUdo3D2Wvi`HQ$nd|KF}T#t{Oksi}tP=VZ)sZX9@Uu(`9d
z^I+1(70qBf?#!9vr@nFH#vTd7rZ!&bOFN6xnV6XJe(h3GRTUH$cXxAROFrJWG<dn+
zD;-E2PTyJl+$iAyLsL^zM_=EwY0-IsfqI}mHE4ihs#a)6Z|~BuwNYB&!&tp8Zc9GS
zmz0!rV|%{*#fulOtPFN{{VANcefHDS(~ax@)ht@P_~-fhf6kpks-Vfqz=$b}o*wBG
z-t+6#>LW*wUR>zR9<?Q-FnHedilv)2Z=PwK&iDHI`sY`}<5LebFj~zI1dTl}{ZMQ*
zchC2G)jcwnliYfxF0Kq#|MT~6=nU;CbLRNO#K^pS`4Th<r5C$vg$CFYt(VtCZU&9O
zaq{vmt^WRQie|8z>rSt2`S<-|Vq`?M!&J=7!rDKB)=&j<_2=H+c5!{YJ)f+VOG$~z
z*6izHf$Km+*`J=Co@t&hcXwB5@{<!2r|3imfsYDv)tqtZ(k0NL)%WZF+k#S$Yqyw0
zubb#IsnaJ<CZ3yP37TMWY-R%u7}eC&K>gjMsIT8&e!q5l;9|GJ-*2~H2KnO8pO6`^
z-~9Oa*k``o+$~#7?p43H-BtE>73kj9LyPW2fNVH#_ghERdzy-s)vTbUUMn@sZ}iDp
zcXW3zj@en%BX9q2S!E8mFX+W6Bros3zwR$+>h13C^6R{dxh49KgKo1tGsm)c$M<{H
zlD1VQiHV7jJ3~)xojPmQsnh!V&%C+08FWkK+1cjTD?3>u{QUSnKRcWJ_0`oqUoLrr
zP7#S%8wNRE?dhu`3mMS(Tk@G1hIa~&%a**k5%^qHOi5Ey6Xb;{Q$+67evjQ%{(c?A
z3o){nm;0wbIM6snE41s~-QBPE9C6CIzAm=r``z+8MW=Op<n8CJSfL^63X15br*iJ?
znOXPu*TgAPT>9l~R|GCTW*XV|`HN!Rzf|*x17RB#eOJYQc#~W$q+Nf@OU=(|waL=J
zsT1ta9hrP|-q+U=yVd4~oiADbrqVrX?wc7O)n=dmw)A^$<C~cMeZ1mx-^u55{=TM{
z8a_?Ze);4FTD_S8sw^KS>_}y`?th-ZvGUB_-qlG555G*d{+T*^^0n_SiNEj0oO=^_
z?9k-U$_1W3Ojmwi|J?Z8T<ht6bFB=kzGQe!*IUb{ucTx?#V<?SrBB}8Z@yh^#@$_}
zp{qg?KRh^iT}Oi{qDlM0rl)J)L~olD8NcSylIMT!%m2${+B{wN@rwAh-R!Om$qYxE
zW8Zk#6;6zwr4hDLb!t=XK?#S54G)Db$5wjy+$rW!7hC1xHfj40_GxcEF8#H$)Kn;N
zug>Xd%LA-8X^G8|xwD~3{(0@uRmwZX-^ln0)kQVcM?BbiTe8<JYD>n!4-XGp7CvGz
z&ARen?e=?z9u}<l+zsk2{MmVNu{-GKZNE7dAVuqM_ipVCldoa4(Z0FC{BQY9&DPM<
zpPYNvfBhrvYb{&$=G%QHzkrPb2gD}}&M9M`cfWN1hZDOqgKX{hu(GiJF?@IP?G4%e
z{ww9?$Q^im<Xu6W!N$jXp3b|ma|ZuDp972ux~{K}zdJU0OM~soozIt978jU@T5q?q
zo|L&T>2Mn8kmmP_*MENACZGJYk>k~L{j#&!d7E!`^!24(Sm1bOp6%>YQ?-rr@7cU%
zQBo>$o)OyAq_3&jd9az?IO#~o>-yQE(`R$VRt5e%vHeW6#Ch4P(E==UZfulFZvUNO
zQm^*%dC}ClvfRrze<~?FHQ~UiDP2cwuFiZEcVPGAa`ydm>}M`i>)-Q(;c3zLjW2eu
zTynj>O?}o>`}&>pjMvQEK0)P=N6f~iPqs-rc2{t1PCL3$`KI$zBe!qr)jM{dSGaYR
zN3rsBP~DBWz5d6tw%xM&;B?&JS?y2mxIHzt$KCq-whPCYcDd=K9y`UdXxFu)9dpli
zf4}&qlkd6yFQrtAVuN{CR#<D~?yOtN&&~aKX8OFuv$IS=!;OdA`Lojn1Sh`sQ|@kU
zIR04C!r}+p-Tw2w>nx{K`QCcy8FgUV5gCQap_kv7<OekFx$ir3U$NojdQJcN?6)uf
zJSb5$HTwJy&DTFoz3=^ge_hh{Q|PU5uY^lIXCqF9Ezg^~=}F8I>3KJH&9GdTwI%QN
ztiHLcH>$m}Hp*M+X=<;$?AAq_UvE5;tU?|y@BLknruyNYd3ENsg{HDr_u8v}%Y-eR
z`e>e9sll|p{q9!x{Wb_1Ok3jF|7*v!e<$|o=r4JnH1WS|2#>)ZSKc`)Ya-k~vx(-P
z-~-*5QIfS0)VT6$d|JALv!`Q8>%kL;u2x&j+|fG4%vV!3;YVylNm0scwo_#*I=<#y
z*NT6wXT$!j_|#KYXJ4%;ADNR47tJz1Ui?7dr5`urzHQxYo0X#<Yo-<#R9u(a#4L63
z_ru>BiD%!ho4TqsnbBw81KX+IZnq2>g81G%bl5a;o^5sT_9^;u<^LW&yDuN07yO)0
zt1>uZ_hW@>^W0^wuciJxO?Y_I`NG!ItgDPWI!^eyw=aCEZ&}FPc;{O{@HGFQZHBYG
zKVNz6dvNBT-79^5h!{`UV$6Ab?!ySi{SO2lJh_>#va{m6-2JVh+y@g19;P?;bn9Jg
zR=)0?aZ>9_=bx`!Io8WJpPRS%MbUq=l?Qu{=N?a3<@xu-iQ<F$%3n>=_lW;=OVM?o
zy)AU6K{C_r-tteUPVDDvNas5?IqG%UsTcEZDk&8SUpleqkL7Cqm(i077*z9eJyh7v
z_}&h_n8q-flOcXHr$La`9=myecYADozI@A+yV-{3T}xUN?e9pQ`?l@L+hW7*`=%-!
zNHUu+DUtm^QCVGD_tZ~YwKl%peKN;>Ugz=3S*DX8S-dhzVlsH{pKkJ#NkAZkX??a{
z@pZXlos*yFeva6n-@Uo}=*umW6aT;UIl{#|iy`3ozF#cyI;(O-EM71$2pj^P3-By0
zNh8&6ckvVXro19|*0(xJN=3n-G3ZI^+MP>_pKpA8!|vJoy{+qdwF|@BkA<ub<W!!y
zX8!gWb1Mwj#GTo-LC<8><%#aM{geO9zJFoiR{iKnN|hfEO8wY1X~*U-GD-XA`M#cD
zEc3vQt#qBswT*Arblh)$eL(Ws)o+J4<O)~tZ+JC*?Ut-76V0rXX8w7{!EnpgtRqTt
z_m8aovz^;?-*BH#d%sM!V$!A?FMFO{<m3HObBuk>OvY!o<&!e57u@MI{di{H$qx%N
zzsTR3xgdVOB5Szft}xLY(^G#EXK(JyO~1&0`lHI!^q02;-mQ7<f2UFC%j;ktStl2l
zJg;4v<%*!Arxq<*Bw?K9^N?evA3LA;jhyH5+e@w9>@Y~)9QRN4C||p5PuLo<$D+E|
zUluf6JvBA{$eV+S<;hzk^CE16gM%yo|NZXi<8$Slf|Anpxo%=lR~bjGiP4FUT(fPz
z?5#Cnx9y|W9sT9+Z(@?P^(DiWj*dG=>aT|~fqEN|&cej~zU>R2<`-?sf2;rI^rxhQ
zoyP1sj;b2*d$~fr*#E!XS@}jYCu(cgS@lCN|2n9!r|miUY*X#ayF1RLu07qi-}LX(
zq79YDwpYyhw!m@5`dKk9E_qo}kyG}cbeei>`@`LD-KX6;eQ{52u$HLxN%i-APcF~r
z|MQnGgIo1L<rHR3y`CA6+U&p0f>hbt{;QaGoO$`*tlG$a?dgRjhOD5re5>nDCc7WM
zBv(iT@ys@D=$cqA9DDlEzq7|TO#WZtRPd<aPgTA%f7x!?=dJv)-z+v*D=Mgbi7Dsl
z-!oh7s`I(oXC4;Jsa~(1Y#*FFeZu39F-cYY-`4275!X3W{4KrxZsEbr^XI0d-aKEe
z`}a-pf>Pl;K|w*;wrTZOs(qd_&9c7UeZ978{{EjWOJ{!%Wb-g{6Z#y(d`<U8i|T=s
zU8R<P?ibgtkbUNz_Q*=DZpZu;)l4((>$R8NntIZ1+uWV^eO34VGq$<7#%5lR@|nMB
zjCNPGi!1)|hrAGW{_*dp%m=yDLr-?vT+nIy%e{Q=?0IXH&-^Q4YqQ#Fx%CJ8r(6Hz
z_e!&GtU1Or?Y+mMu(i=QHY#&|e#>vPC0pd$n!h!luZpcQ(K=gx|IOn2(zP1trJM4P
zTFts&yM3Q$C#dv1CHmR4;K2jFZ9ionKX@g%#wK7=-1%)v1?Nqs9dp0*cIvkRqvpTw
zYdZcfK7VP^8u8D&Hu}unI@MfW@9>fD%P&{n(9pWg6Joaf!#_9uf2pg3xBEqkC9ue>
z`Ta(j>0NE_ZtY(a-&A)LYWEv%IsJA+k??uDi>*O!>i>?{&AoSSmcPBy_N&}?;?-GZ
zU3+kP;o}I2Z%n#U*AsrOGl;UiFq!?bx#5$&PnVQ+94bo?zU_7G!NC)^rWEhoSiR@@
zJm%W}i>vrReYQi3Vs^&pv#<oLiIA8WJ8!vfL}8AbhaJx`FHsFqk)Es54l}KvzCqai
zMyk2+zX1Jz`U`m8ZHbjJzk68MjG19s@!ktg4eNRDG<;V1cwG5~%sY06naBE#HyJ)r
zThD&rDa)Guv+qsxtJySr#iaE2r!#at&z@4dmh$9Lyrw{MrnKss2f=$xrk<Fep5c-k
z6TCkm;L%*0Ynx54=CGZ6bEclJB(r-~go_JH$+<;;n5Ukarmm#a)N$#j<42j_H|uTl
z45p~9N_19IR$dvroX^yI*2#~vK34fpEPm&5@V=f(;s0Oq{10qBqK@qhFSxb2Aa(=a
zyZc{L;uUO97k*<Eu2b!)xclx;<kGk$pV>nnZhby~(av06;ro@%y(=T(VbhzJ%v2;(
zS^uZ^<oE4;sk}-}B5FQS=RDilkDrY7`BlaJ<3~fftTo>f<_S}$^2M>2@kg9t3o?3>
z@Kya#!h_kBr!A&ERd`)t6MyDj>%k|J=d0TKZ8CbZ-#1^=^K4XF-K`1<bp}h7^~Hg^
zV>B#^4W}5*fBPoyO+!Dc5r4kw#gNW#{(sxqvm<ZptmoI}cL`WOXCAYzlF}#b%Nr68
zx2@!1jdpWm6ISzC;5*xlH6hnVCSvyl)zy3xx4h?=_aJ&jiMIjUfysp(M>mTVnA~h!
z>XLbCI^W`wtpeZlW*uFWYQHJR)JEWP=cR*7SIU&kTED#Rc)xXBO7I^^o<v5L%Ay;x
z`)4VCJgyO@+hzSu>-@q<<H#SMjq2jWUFN61Q!}~1{=5J4%zb~{a}pP-%$dE<tT!*)
z;ohfC9iLA{R{z8|y{~B4CMu)zr%tM~ldmhBLpk-a{~F63?D-o{h47zDd{b>NEK<`o
zm-A>wfa>I{-)7ypzJjNX>B_&GH*bOtjca6P&jL5R&OP{AwO7meVA+=!fw02-4nx0i
z`{CA$W&buWwDn{6(}~b6c<GqO&(|)!`fK_7FBSzmbR%@d>-#U~9Z%ktxOdHGzPUHv
z>a8)8c3aMDEtWH_$@IazS+h?4e!t&8DQQs|o1oy%cO{M8G1YF|VY8nq=|}&39W5!f
zF-o5!-%b0P{mEjn*>CgL%wfH~{&m{Mq~L7Ooa%?^5ul+hlym8yth#8~U37F)UvB!*
zZHpsx#OC;^Cnzq-sazGl+GWGnPy5?bWbYr{bo906RjpZWsejkKHnZ3qbYn}B`9s~=
z!C~i{!~_L@GWY7766My2o}9HUS?6D(_SC$5x68LzP1^aZ*o|-YF~j5cXZU{%no<5v
z>AThX{hz<o)x~X_A$hv|abPicK;YEX_jYFS@d_gKv*j6&DrZ#2mw(7ftoJ(iK4>QM
z?!SM&yePbJa?|tscGBJUic5QE_@;fZ*kC;;#=tN#vg};-)yJ0B+1BT{C(XN@cl_Ot
z*FVxZ*ZHhIm%DOF8)yZBifr4o)t4upyH~6B^MM_MQ}d_(mumTur}%eo{&+ItY*;jV
zT-6HAb$7dG?z-A;eOe|pG%xwr_HG9I8ymCqZeLsXm?7@ntG`NfUrkZ?{6I_XoZXq+
ziH&FC3MMU&d+YY?*H53=J)K5VwI}bXx0moLu0JDeqVYKK?4DMYhwhKMM5nwLbiUs9
zuzueozwJ`~E-p{}Gg1QoDRNhoZ_0Qkv#;*GXwv@em;au8zCt#qX${N0-)Z`1t+p(4
zKK{EPfBxaFy_s+3ZRs&8U7T@ak2UX{<>zg^W<9X7v8)oQwGK&sI%}^*!KU}qf6jQ9
zadz%m>x)}UnA-hkuPBTbNp1L*TV+)BMPQfPJQck>ZqI0O<Kv;d*K(sTZ!9*wcBJ;W
zV0)AYWO0Lnx`cA3OlV_IivF`@yqOV8MHnnIk282!ZfIvd=~ngajqlwc$B-9NM~~_M
zKdjEZJ$Uuq+#3wxebb{%Pbad})F|3JMs-yED6N|6m-w*fWX=3Hhpw}TMKhQ=tFCHc
zXJT|Qd)uL2X>Dzuzi`UggAZe?Pw-{9yl;QMxJ54teCEH-t1eOPw!3-HZ|Cn<RaamB
zVC@3){s*5*Uwn<Qj@j??X!ln4&&f+}cODFx{A=Pw7qc9XloZ*f&u^yedK<@PSEhgS
z^dkw&5;OJl^Wt6<#aGN**v@S4`*`lvvZ%7^2`^5i#q9t6cKzhpI%x(X$7erR-I5~u
z%Ee7cGP2#u)cf&?BC%K7?Cbtm1TXVh=+w$}OAs6?dU^&16P(-m1m)z;O;B_OExO1n
zKWK6NT*j}vRblxjg>zSL%yiSSe{bC!AHcb1-Oe+y)e`D{leV8pK6p=f&EGuJqpQpx
z-c0OoPFJ<`Q%>XAap&0P&tJ~|zg|3P>*wv4Z8k}TJ<puPU!m|m=f`KcBk#`Cuc^HB
zZtINqYrNmyntJow-IQ(e=Z`$LIXla<P^R?n(Z%O|)eiVMS{EByThGqCyzJw*+xeeA
z9OmDga#H9m2e^aV#w%^+ldAFK&!0apE-mE-Z3MWrCG*TI({3SEF99*JvcuOjl?#<W
zi|MNC8c+3cS9rR*EqGf|XwJ#iGj@MapYyQ%!m6gmotx7)mz)eTV=;O9a@$moyI)wn
zg=!1tdK`${V_DkNw=w6XpV`vcE5ptmxR%OoIb()j!c>n_zva@Nt*LzDvoYCV>4u`8
z=cjb$INW;0`)9*<mbNXrHPt(`&byndtnrED={)_T?q5+|%&w+SbLAHoRvb{het+`R
zBdd;jM@+70xNIrrHC@|%!uj66|DKfXj@*_bDPRA`aQpo_>&<Cr7x~UMo8hAd>CZ#P
z&oyeEotx|Z|KDFuK0Y-ot64^=r$la)ezlObh)LKS`TxsS?>*AX5BDC;Vw__!M{JFX
zVd4oHt3CfD5-T>u8Su9s=5x~y>z;U}`L{>;;h%x0d6y-u3egE&@M=r#@3Kda9)Ye&
z+qiKfXuY^>tcy#Y#a*lA6F_?<-rm}(Y-To1P}!~EcJ6l2@FEiv6W5(xQo3O|)ArZB
z6?%WH=v}6P*cvef<!<ID503R*P5W`!dQI5v{EZjtx*oqg$8EKXyVye@l*^-b(&WjV
z-QADByu1urZd~%}il=OhiwhS|uZ|d_I%Mz`bOqnHx3@2EN<F=(`o^W=Hq+13Cbda^
zSo|!uJkavS@_;<AurRUW=jT%Y{`$%zXES5Q3<({vZmTWemfzNm#n1gfL$15a_3!R3
z2i@JK<;4gJ;9Vt}+~Rsq8rkI%j&uk@Z(I{>b=_(A?EHLxP`L8&ECCIN>PCmTYAQT>
z{CK8eGF#~CFjX_NX|qhTS3yQkug^dIH0jTekD#42zrMb{ywrO-Q>ESGvyX0UOa=||
zojKzJ+8@-;CmSLnC>R(qMf&}fmBFAD+n{BOcYgMyZC;nXD{j-KO*3q(&1ReBCY_j|
z2wFY(RUCBloMuJY`+L4VK0Le2-!F^ap6BA`wo2prgws#0%HxvO$L*D{tFc(Pa3N%n
z((B;1f`?AOzQ6B3o2G1KHS5EN0@0^hw>PD7UtaFN+^tvY$gyKZuh(veI=!|1^wUSp
z{B{p|&F?8_X?5wv?h=rYSaIl&vwOdss-7O-?z`u96g~#6?}NJikhZPuTt(+LgVI+a
zpyigU!`7DA-4#7?;zYu?H#c)`Zfd<AUq5$W?eA56;Gt#L<MV8*U)<kse|vlW^6KyJ
zTwGmOYD_=)v}j|>Ng*-aC>PK|V!Ij((3YtlP<(k^+*0u|DJdywL)6+YFD^PiH~!b@
zJKGF&27A`^b!T@JK3);GcUKp9-C59=op<x>{(Nw5<CS*XTlE!`^Bo->wa!Gk_sJ;g
z>G8E6cFf7q$-TWT6f~)o7?@|b+<*Qv(4w=``umr}?k?MsclXyaJAQp-<>tuEX&Xyl
zhqduaAKU-$>-t5%|KtjS!hdHBXys@9|9?l0AHTdd+FZ&cqaZkU|LLcX+U4sKK0Z2X
z^XY^#=;-qY4-!gwKoKnRbFZrRGy!4Z=94Ll-23G~tMY#4-rG^AY@BvR!N_RR#^mEc
zTeCt-S-`4yifq4q_PBignF)%{D}tBveZN<&f3!<fyXf}J88c3l->+@I{Wk5-kB?ij
zu6p(AgQqp7zdke57-Z*p`~P!_o}Loj=`d&7v|}G09<Ka!QoS?|Jix7`b84!Wk59_~
ze}7%G+YKaE#P8p?jzJu3r=XzV#QjMjoEWVllqM3f9KA39KNv{zizlT<FfcGMc)I$z
JtaD0e0suv#pJf05

-- 
GitLab


From 5489e3a299d3b241e6ca2397d792e4f571e3d654 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Tue, 8 Jan 2019 16:45:50 -0800
Subject: [PATCH 0388/2345] [XLA] Use Iota in Cholesky rather than a std::iota
 constant.

NFC intended, just a cleanup noticed in passing.

PiperOrigin-RevId: 228424323
---
 .../compiler/xla/client/lib/cholesky.cc       | 39 +++++++++----------
 1 file changed, 18 insertions(+), 21 deletions(-)

diff --git a/tensorflow/compiler/xla/client/lib/cholesky.cc b/tensorflow/compiler/xla/client/lib/cholesky.cc
index e100d47922..414bd1494c 100644
--- a/tensorflow/compiler/xla/client/lib/cholesky.cc
+++ b/tensorflow/compiler/xla/client/lib/cholesky.cc
@@ -68,29 +68,26 @@ XlaOp CholeskyUnblocked(XlaOp a, PrecisionConfig::Precision precision) {
     auto body_fn =
         [&](XlaOp i, absl::Span<const XlaOp> loop_vars,
             XlaBuilder* body_builder) -> StatusOr<std::vector<XlaOp>> {
-      Shape col_shape;
-      Shape row_shape;
-      for (int64 d : major_dims) {
-        row_shape.add_dimensions(d);
-        col_shape.add_dimensions(d);
-      }
-      row_shape.add_dimensions(1);
-      row_shape.add_dimensions(n);
-      row_shape.set_element_type(a_shape.element_type());
-      auto mask_zeros_row = Zeros(body_builder, row_shape);
-
-      col_shape.add_dimensions(n);
-      col_shape.add_dimensions(1);
-      col_shape.set_element_type(a_shape.element_type());
-      auto mask_zeros_col = Zeros(body_builder, col_shape);
-
-      std::vector<int32> mask_vector(n);
-      std::iota(mask_vector.begin(), mask_vector.end(), 0);
-      auto mask_range = ConstantR1<int32>(body_builder, mask_vector);
+      std::vector<int64> row_shape_dims(major_dims.begin(), major_dims.end());
+      std::vector<int64> col_shape_dims(major_dims.begin(), major_dims.end());
+      row_shape_dims.push_back(1);
+      row_shape_dims.push_back(n);
+      auto mask_zeros_row =
+          Zeros(body_builder,
+                ShapeUtil::MakeShape(a_shape.element_type(), row_shape_dims));
+
+      col_shape_dims.push_back(n);
+      col_shape_dims.push_back(1);
+      auto mask_zeros_col =
+          Zeros(body_builder,
+                ShapeUtil::MakeShape(a_shape.element_type(), col_shape_dims));
+
       auto mask_range_row =
-          Broadcast(Reshape(mask_range, {0}, {1, n}), major_dims);
+          Iota(body_builder, ShapeUtil::MakeShape(S32, row_shape_dims),
+               /*iota_dimension=*/n_dims - 1);
       auto mask_range_col =
-          Broadcast(Reshape(mask_range, {0}, {n, 1}), major_dims);
+          Iota(body_builder, ShapeUtil::MakeShape(S32, col_shape_dims),
+               /*iota_dimension=*/n_dims - 2);
       auto body_a = loop_vars[0];
       auto body_l = loop_vars[1];
 
-- 
GitLab


From 2c7444faf3d23ddb959f887b5589ec53ca431e9d Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Tue, 8 Jan 2019 16:53:15 -0800
Subject: [PATCH 0389/2345] [TF:XLA] Move some bridge implementations over to
 the new DS/DUS API

Move some bridge and xla client library op implementations over to
the new span-of-scalars API for DynamicSlice and DynamicUpdateSlice.

PiperOrigin-RevId: 228425409
---
 .../compiler/tf2xla/kernels/image_ops.cc      | 17 ++++---
 .../compiler/tf2xla/kernels/stack_ops.cc      | 12 ++---
 tensorflow/compiler/xla/client/lib/slicing.cc | 49 ++++++++++---------
 .../xla/client/lib/triangular_solve.cc        | 16 +++---
 4 files changed, 49 insertions(+), 45 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/image_ops.cc b/tensorflow/compiler/tf2xla/kernels/image_ops.cc
index 96ddd42e2a..a31b5a2cfd 100644
--- a/tensorflow/compiler/tf2xla/kernels/image_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/image_ops.cc
@@ -13,6 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include "absl/types/span.h"
 #include "tensorflow/compiler/tf2xla/kernels/gather_op_helpers.h"
 #include "tensorflow/compiler/tf2xla/lib/util.h"
 #include "tensorflow/compiler/tf2xla/xla_helpers.h"
@@ -351,24 +352,26 @@ struct SuppressBodyFn {
     auto num_outputs_so_far = values[1];
     auto iou_mask = values[2];
     auto included_iou = values[3];
-    auto zero_r1 = xla::ConstantR1<int32>(builder, {0});
+    auto zero = xla::ConstantR0<int32>(builder, 0);
     // Determine if current elem is active using a slice.
-    auto row_idx_r1 = xla::Reshape(row_idx, {1});
-    auto active_elem = xla::DynamicSlice(included_iou, row_idx_r1, {1});
+    // TODO(b/118437727): The only reason we need an explicit vector is because
+    // some old GCCs can't deduce the right type for MakeConstSpan, and
+    // providing a single-value initializer list directly uses the wrong
+    // overload. Delete this once the deprecated overload is gone.
+    std::vector<xla::XlaOp> row_idx_vector = {row_idx};
+    auto active_elem = xla::DynamicSlice(included_iou, row_idx_vector, {1});
     active_elem = xla::Reshape(active_elem, {});
     // Increment output count iff current elem is not suppressed.
     num_outputs_so_far = xla::Select(
         active_elem, num_outputs_so_far + xla::ConstantR0<int32>(builder, 1),
         num_outputs_so_far);
     // Slice out the row_idx.
-    auto starts = xla::ConcatInDim(builder, {row_idx_r1, zero_r1}, 0);
-    auto row_iou = xla::DynamicSlice(iou_mask, starts, {1, num_boxes});
+    auto row_iou = xla::DynamicSlice(iou_mask, {row_idx, zero}, {1, num_boxes});
     // Remove the diagonal from consideration. An elem cannot suppress
     // itself.
-    auto update_starts = xla::ConcatInDim(builder, {zero_r1, row_idx_r1}, 0);
     row_iou = xla::DynamicUpdateSlice(
         row_iou, xla::ConstantR2FromArray2D<bool>(builder, {{false}}),
-        update_starts);
+        {zero, row_idx});
     // Create a suppression by inverting polarity.
     row_iou = xla::Reshape(row_iou, {num_boxes});
     auto supp_mask = xla::Not(row_iou);
diff --git a/tensorflow/compiler/tf2xla/kernels/stack_ops.cc b/tensorflow/compiler/tf2xla/kernels/stack_ops.cc
index 02d71a3942..d0c5231e84 100644
--- a/tensorflow/compiler/tf2xla/kernels/stack_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/stack_ops.cc
@@ -146,9 +146,9 @@ class StackPushOp : public XlaOpKernel {
     xla::XlaOp value = ctx->Input(1);
 
     // start_indices of the DynamicUpdateSlice are [index, 0, 0, ..., 0].
-    auto start_indices =
-        xla::Pad(xla::Reshape(index, {1}), xla::ConstantR0<int32>(b, 0),
-                 xla::MakeEdgePaddingConfig({{0, elem_shape.dims()}}));
+    std::vector<xla::XlaOp> start_indices(elem_shape.dims() + 1,
+                                          xla::ConstantR0<int32>(b, 0));
+    start_indices[0] = index;
 
     TensorShape slice_shape = elem_shape;
     slice_shape.InsertDim(0, 1LL);
@@ -202,9 +202,9 @@ class StackPopOp : public XlaOpKernel {
     OP_REQUIRES_OK(ctx, resource->SetValue(xla::Tuple(b, {ta, index})));
 
     // start_indices of the DynamicSlice are [index, 0, 0, ..., 0].
-    auto start_indices =
-        xla::Pad(xla::Reshape(index, {1}), xla::ConstantR0<int32>(b, 0),
-                 xla::MakeEdgePaddingConfig({{0, stack_shape.dims() - 1}}));
+    std::vector<xla::XlaOp> start_indices(stack_shape.dims(),
+                                          xla::ConstantR0<int32>(b, 0));
+    start_indices[0] = index;
 
     auto slice_shape = stack_shape.dim_sizes();
     slice_shape[0] = 1LL;
diff --git a/tensorflow/compiler/xla/client/lib/slicing.cc b/tensorflow/compiler/xla/client/lib/slicing.cc
index 611fffba8d..77145ba7d4 100644
--- a/tensorflow/compiler/xla/client/lib/slicing.cc
+++ b/tensorflow/compiler/xla/client/lib/slicing.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/compiler/xla/client/lib/slicing.h"
+#include "tensorflow/compiler/xla/client/xla_builder.h"
 
 namespace xla {
 
@@ -51,17 +52,17 @@ XlaOp SliceInMinorDims(XlaOp x, absl::Span<const int64> start,
 XlaOp UpdateSlice(XlaOp x, XlaOp update, absl::Span<const int64> start) {
   XlaBuilder* builder = x.builder();
   return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
-    // TODO(phawkins): make int64 work on all backends, remove the int32 cast.
-    std::vector<int32> start_as_int32(start.begin(), start.end());
-    auto start_constant = ConstantR1<int32>(builder, start_as_int32);
     TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x));
     const int64 n_dims = shape.rank();
-    TF_ASSIGN_OR_RETURN(Shape start_constant_shape,
-                        builder->GetShape(start_constant));
-    const int64 start_length =
-        ShapeUtil::GetDimension(start_constant_shape, -1);
-    TF_RET_CHECK(start_length == n_dims);
-    return DynamicUpdateSlice(x, update, start_constant);
+    TF_RET_CHECK(start.size() == n_dims);
+
+    // TODO(phawkins): make int64 work on all backends, remove the int32 cast.
+    std::vector<int32> start_as_int32(start.begin(), start.end());
+    std::vector<XlaOp> start_ops(start.size());
+    for (int i = 0; i < start.size(); ++i) {
+      start_ops[i] = ConstantR0(builder, start_as_int32[i]);
+    }
+    return DynamicUpdateSlice(x, update, start_ops);
   });
 }
 
@@ -90,18 +91,17 @@ std::vector<int64> ConcatVectors(absl::Span<const int64> xs,
   return output;
 }
 
-XlaOp PrependZerosInMajorDims(XlaOp x, absl::Span<const XlaOp> starts) {
+StatusOr<std::vector<XlaOp>> PrependZerosInMajorDims(
+    XlaOp x, absl::Span<const XlaOp> starts) {
   XlaBuilder* builder = x.builder();
-  return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
-    TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x));
-    const int64 n_dims = shape.rank();
-    auto zero = Reshape(ConstantR0<int32>(builder, 0), {1});
-    std::vector<XlaOp> padded_starts(n_dims, zero);
-    for (int i = 0; i < starts.size(); ++i) {
-      padded_starts[n_dims - starts.size() + i] = Reshape(starts[i], {1});
-    }
-    return ConcatInDim(builder, padded_starts, 0);
-  });
+  TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x));
+  const int64 n_dims = shape.rank();
+  auto zero = ConstantR0<int32>(builder, 0);
+  std::vector<XlaOp> padded_starts(n_dims, zero);
+  for (int i = 0; i < starts.size(); ++i) {
+    padded_starts[n_dims - starts.size() + i] = starts[i];
+  }
+  return padded_starts;
 }
 
 }  // namespace
@@ -119,7 +119,7 @@ XlaOp DynamicSliceInMinorDims(XlaOp x, absl::Span<const XlaOp> starts,
                           .subspan(
                               /*pos=*/0,
                               /*len=*/n_dims - sizes.size());
-    auto padded_starts = PrependZerosInMajorDims(x, starts);
+    TF_ASSIGN_OR_RETURN(auto padded_starts, PrependZerosInMajorDims(x, starts));
     auto padded_sizes = ConcatVectors(major_dims, sizes);
     return DynamicSlice(x, padded_starts, padded_sizes);
   });
@@ -127,8 +127,11 @@ XlaOp DynamicSliceInMinorDims(XlaOp x, absl::Span<const XlaOp> starts,
 
 XlaOp DynamicUpdateSliceInMinorDims(XlaOp x, XlaOp update,
                                     absl::Span<const XlaOp> starts) {
-  auto padded_starts = PrependZerosInMajorDims(x, starts);
-  return DynamicUpdateSlice(x, update, padded_starts);
+  XlaBuilder* builder = x.builder();
+  return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    TF_ASSIGN_OR_RETURN(auto padded_starts, PrependZerosInMajorDims(x, starts));
+    return DynamicUpdateSlice(x, update, padded_starts);
+  });
 }
 
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/client/lib/triangular_solve.cc b/tensorflow/compiler/xla/client/lib/triangular_solve.cc
index 6061e64656..c2f31742e9 100644
--- a/tensorflow/compiler/xla/client/lib/triangular_solve.cc
+++ b/tensorflow/compiler/xla/client/lib/triangular_solve.cc
@@ -165,10 +165,10 @@ XlaOp InvertDiagonalBlocks(XlaOp diag_blocks, bool lower, bool transpose_a,
     // The first or last  diagonal element should be set to 1 instead of -1
     // though, since we never update it
     auto pos_one = Reshape(One(builder, shape.element_type()), {1, 1});
-    auto start_index = (lower) ? 0 : block_size - 1;
-    auto output_block = DynamicUpdateSlice(
-        neg_identity, pos_one,
-        /*start_indices=*/ConstantR1<int>(builder, 2, start_index));
+    auto start_index = ConstantR0<int>(builder, (lower) ? 0 : block_size - 1);
+    auto output_block =
+        DynamicUpdateSlice(neg_identity, pos_one,
+                           /*start_indices=*/{start_index, start_index});
 
     // Broadcast diag([1, -1, -1, ...]) to every block
     XlaOp output = Broadcast(output_block,
@@ -211,12 +211,10 @@ XlaOp InvertDiagonalBlocks(XlaOp diag_blocks, bool lower, bool transpose_a,
       auto body_out = GetTupleElement(input_tuple, 1);
       auto body_input = GetTupleElement(input_tuple, 2);
 
-      auto zero = ConstantR1<int32>(bodyb.get(), 1, 0);
+      auto zero = ConstantR0<int32>(bodyb.get(), 0);
       auto j = (lower) ? i : ScalarLike(i, block_size - 1) - i;
-      auto start_indices =
-          ConcatInDim(bodyb.get(), {zero, Reshape(j, {1}), zero}, 0);
       auto input_row =
-          DynamicSlice(body_input, start_indices,
+          DynamicSlice(body_input, {zero, j, zero},
                        /*slice_sizes=*/{num_blocks, 1, block_size});
 
       // We want -L21 L11^{-1}
@@ -230,7 +228,7 @@ XlaOp InvertDiagonalBlocks(XlaOp diag_blocks, bool lower, bool transpose_a,
       precision_proto.add_operand_precision(precision);
       auto update = -DotGeneral(input_row, body_out, dnums, &precision_proto);
 
-      body_out = DynamicUpdateSlice(body_out, update, start_indices);
+      body_out = DynamicUpdateSlice(body_out, update, {zero, j, zero});
 
       auto next_i = i + ScalarLike(i, 1);
       Tuple(bodyb.get(), {next_i, body_out, body_input});
-- 
GitLab


From c842394896de32c4160fc0f5f94dabf06890476d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 16:57:19 -0800
Subject: [PATCH 0390/2345] Support not fully defined batch dimension in the
 inputs to group_norm.

PiperOrigin-RevId: 228425992
---
 .../layers/python/layers/normalization.py      | 18 +++++++++++++-----
 .../layers/python/layers/normalization_test.py |  9 +++++++++
 2 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/layers/python/layers/normalization.py b/tensorflow/contrib/layers/python/layers/normalization.py
index 11033a2e9c..76b03ff514 100644
--- a/tensorflow/contrib/layers/python/layers/normalization.py
+++ b/tensorflow/contrib/layers/python/layers/normalization.py
@@ -186,7 +186,7 @@ def group_norm(inputs,
 
   Args:
     inputs: A Tensor with at least 2 dimensions one which is channels. All
-     shape dimensions must be fully defined.
+     shape dimensions except for batch must be fully defined.
     groups: Integer. Divide the channels into this number of groups over which
       normalization statistics are computed. This number must be commensurate
       with the number of channels in `inputs`.
@@ -249,13 +249,21 @@ def group_norm(inputs,
   """
   # TODO(shlens): Support partially defined shapes for the inputs.
   inputs = ops.convert_to_tensor(inputs)
-  original_shape = inputs.shape
 
   if inputs.shape.ndims is None:
     raise ValueError('Inputs %s has undefined rank.' % inputs.name)
   if channels_axis > (inputs.shape.ndims - 1):
     raise ValueError('Axis is out of bounds.')
 
+  # Use dynamic shape for not fully defined dimensions in the inputs.
+  dyanmic_shape = array_ops.shape(inputs)
+  input_shape_list = []
+  for i, dim in enumerate(inputs.shape):
+    if dim.value is None:
+      input_shape_list.append(dyanmic_shape[i])
+    else:
+      input_shape_list.append(dim)
+
   # Standardize the channels_axis to be positive and identify # of channels.
   if channels_axis < 0:
     channels_axis = inputs.shape.ndims + channels_axis
@@ -289,8 +297,8 @@ def group_norm(inputs,
   # Determine axes before channels. Some examples of common image formats:
   #  'NCHW': before = [N], after = [HW]
   #  'NHWC': before = [NHW], after = []
-  axes_before_channels = inputs.shape.as_list()[:channels_axis]
-  axes_after_channels = inputs.shape.as_list()[channels_axis+1:]
+  axes_before_channels = input_shape_list[:channels_axis]
+  axes_after_channels = input_shape_list[channels_axis+1:]
 
   # Manually broadcast the parameters to conform to the number of groups.
   params_shape_broadcast = ([1] * len(axes_before_channels) +
@@ -369,7 +377,7 @@ def group_norm(inputs,
     outputs = inputs * gain + offset
 
     # Collapse the groups into the channel dimension.
-    outputs = array_ops.reshape(outputs, original_shape)
+    outputs = array_ops.reshape(outputs, input_shape_list)
 
     if activation_fn is not None:
       outputs = activation_fn(outputs)
diff --git a/tensorflow/contrib/layers/python/layers/normalization_test.py b/tensorflow/contrib/layers/python/layers/normalization_test.py
index c8d3c91b10..9a85084b23 100644
--- a/tensorflow/contrib/layers/python/layers/normalization_test.py
+++ b/tensorflow/contrib/layers/python/layers/normalization_test.py
@@ -221,6 +221,15 @@ class GroupNormTest(test.TestCase):
       normalization.group_norm(inputs, channels_axis=-1,
                                reduction_axes=[-3, -2])
 
+  def testParamsShapeNotFullyDefinedBatchAxis(self):
+    height, width, groups = 3, 3, 4
+    inputs = array_ops.placeholder(dtypes.float32,
+                                   shape=(None, height, width, 2*groups))
+    output = normalization.group_norm(inputs, channels_axis=-1,
+                                      reduction_axes=[-3, -2], groups=groups)
+    self.assertListEqual([None, height, width, 2 * groups],
+                         output.shape.as_list())
+
   def testCreateOp(self):
     height, width, groups = 3, 3, 4
     images = random_ops.random_uniform((5, height, width, 2*groups), seed=1)
-- 
GitLab


From dde706e1d3561656f0a6d842a79c50cfbe1ad508 Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Wed, 9 Jan 2019 06:48:03 +0530
Subject: [PATCH 0391/2345] Changed result.error_message() to result

---
 tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc b/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc
index 2879b00f55..f8c67b60c3 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc
@@ -46,7 +46,7 @@ bool CheckMask(se::StreamExecutor* exec, void* ptr, int64* mask) {
 
   Status result = exec->SynchronousMemcpyD2H(gpu_ptr, MASK_BYTES, tmp);
   if (!result.ok()) {
-    LOG(FATAL) << "Could not copy debug mask, " << result.error_message();
+    LOG(FATAL) << "Could not copy debug mask, " << result;
   }
 
   bool ok = true;
@@ -66,7 +66,7 @@ void InitMask(se::StreamExecutor* exec, void* ptr, int64* mask) {
   se::DeviceMemory<int64> gpu_ptr{se::DeviceMemoryBase{ptr, MASK_BYTES}};
   Status result = exec->SynchronousMemcpyH2D(mask, MASK_BYTES, &gpu_ptr);
   if (!result.ok()) {
-    LOG(FATAL) << "Could not copy debug mask, " << result.error_message();
+    LOG(FATAL) << "Could not copy debug mask, " << result;
   }
 }
 
@@ -176,7 +176,7 @@ void* GPUNanResetAllocator::AllocateRaw(size_t alignment, size_t num_bytes) {
   Status result = stream_exec_->SynchronousMemcpyH2D(&nans[0], req_size,
                                                      &nan_ptr);
   if (!result.ok()) {
-    LOG(ERROR) << "Could not initialize to NaNs, " << result.error_message();
+    LOG(ERROR) << "Could not initialize to NaNs, " << result;
   }
 
   return allocated_ptr;
@@ -192,7 +192,7 @@ void GPUNanResetAllocator::DeallocateRaw(void* ptr) {
     Status result = stream_exec_->SynchronousMemcpyH2D(&nans[0], req_size,
                                                        &nan_ptr);
     if (!result.ok()) {
-      LOG(ERROR) << "Could not initialize to NaNs, " << result.error_message();
+      LOG(ERROR) << "Could not initialize to NaNs, " << result;
     }
   }
 
-- 
GitLab


From fa8ed584884fae406e2398718bbcd6e20498c7a2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 17:24:16 -0800
Subject: [PATCH 0392/2345] Mark certain tests in export_output_test as v1 only
 as export_output.as_signature_def does not support v2 due to its dependent
 method saved_model.build_tensor_info not working in eager. Also raise a
 runtime exception inside build_tensor_info when running in eager.

PiperOrigin-RevId: 228429679
---
 .../saved_model/model_utils/export_test.py    | 255 +++++++++---------
 tensorflow/python/saved_model/utils_impl.py   |   6 +
 tensorflow/python/saved_model/utils_test.py   |   7 +
 3 files changed, 140 insertions(+), 128 deletions(-)

diff --git a/tensorflow/python/saved_model/model_utils/export_test.py b/tensorflow/python/saved_model/model_utils/export_test.py
index 776bfff886..ef512150a2 100644
--- a/tensorflow/python/saved_model/model_utils/export_test.py
+++ b/tensorflow/python/saved_model/model_utils/export_test.py
@@ -22,7 +22,6 @@ import os
 import tempfile
 import time
 
-from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -52,110 +51,110 @@ ops.register_tensor_conversion_function(LabeledTensorMock,
 
 class ExportTest(test_util.TensorFlowTestCase):
 
+  @test_util.deprecated_graph_mode_only
   def test_build_all_signature_defs_without_receiver_alternatives(self):
-    with context.graph_mode():
-      receiver_tensor = array_ops.placeholder(dtypes.string)
-      output_1 = constant_op.constant([1.])
-      output_2 = constant_op.constant(["2"])
-      output_3 = constant_op.constant(["3"])
-      export_outputs = {
-          signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
-              export_output.RegressionOutput(value=output_1),
-          "head-2": export_output.ClassificationOutput(classes=output_2),
-          "head-3": export_output.PredictOutput(outputs={
-              "some_output_3": output_3
-          }),
-      }
-
-      signature_defs = export_utils.build_all_signature_defs(
-          receiver_tensor, export_outputs)
-
-      expected_signature_defs = {
-          "serving_default":
-              signature_def_utils.regression_signature_def(receiver_tensor,
-                                                           output_1),
-          "head-2":
-              signature_def_utils.classification_signature_def(receiver_tensor,
-                                                               output_2, None),
-          "head-3":
-              signature_def_utils.predict_signature_def({
-                  "input": receiver_tensor
-              }, {"some_output_3": output_3})
-      }
-
-      self.assertDictEqual(expected_signature_defs, signature_defs)
+    receiver_tensor = array_ops.placeholder(dtypes.string)
+    output_1 = constant_op.constant([1.])
+    output_2 = constant_op.constant(["2"])
+    output_3 = constant_op.constant(["3"])
+    export_outputs = {
+        signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
+            export_output.RegressionOutput(value=output_1),
+        "head-2": export_output.ClassificationOutput(classes=output_2),
+        "head-3": export_output.PredictOutput(outputs={
+            "some_output_3": output_3
+        }),
+    }
+
+    signature_defs = export_utils.build_all_signature_defs(
+        receiver_tensor, export_outputs)
 
+    expected_signature_defs = {
+        "serving_default":
+            signature_def_utils.regression_signature_def(receiver_tensor,
+                                                         output_1),
+        "head-2":
+            signature_def_utils.classification_signature_def(receiver_tensor,
+                                                             output_2, None),
+        "head-3":
+            signature_def_utils.predict_signature_def({
+                "input": receiver_tensor
+            }, {"some_output_3": output_3})
+    }
+
+    self.assertDictEqual(expected_signature_defs, signature_defs)
+
+  @test_util.deprecated_graph_mode_only
   def test_build_all_signature_defs_with_dict_alternatives(self):
-    with context.graph_mode():
-      receiver_tensor = array_ops.placeholder(dtypes.string)
-      receiver_tensors_alternative_1 = {
-          "foo": array_ops.placeholder(dtypes.int64),
-          "bar": array_ops.sparse_placeholder(dtypes.float32)}
-      receiver_tensors_alternatives = {"other": receiver_tensors_alternative_1}
-      output_1 = constant_op.constant([1.])
-      output_2 = constant_op.constant(["2"])
-      output_3 = constant_op.constant(["3"])
-      export_outputs = {
-          signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
-              export_output.RegressionOutput(value=output_1),
-          "head-2": export_output.ClassificationOutput(classes=output_2),
-          "head-3": export_output.PredictOutput(outputs={
-              "some_output_3": output_3
-          }),
-      }
-
-      signature_defs = export_utils.build_all_signature_defs(
-          receiver_tensor, export_outputs, receiver_tensors_alternatives)
-
-      expected_signature_defs = {
-          "serving_default":
-              signature_def_utils.regression_signature_def(
-                  receiver_tensor,
-                  output_1),
-          "head-2":
-              signature_def_utils.classification_signature_def(
-                  receiver_tensor,
-                  output_2, None),
-          "head-3":
-              signature_def_utils.predict_signature_def(
-                  {"input": receiver_tensor},
-                  {"some_output_3": output_3}),
-          "other:head-3":
-              signature_def_utils.predict_signature_def(
-                  receiver_tensors_alternative_1,
-                  {"some_output_3": output_3})
-
-          # Note that the alternatives 'other:serving_default' and
-          # 'other:head-2' are invalid, because regession and classification
-          # signatures must take a single string input.  Here we verify that
-          # these invalid signatures are not included in the export_utils.
-      }
-
-      self.assertDictEqual(expected_signature_defs, signature_defs)
+    receiver_tensor = array_ops.placeholder(dtypes.string)
+    receiver_tensors_alternative_1 = {
+        "foo": array_ops.placeholder(dtypes.int64),
+        "bar": array_ops.sparse_placeholder(dtypes.float32)}
+    receiver_tensors_alternatives = {"other": receiver_tensors_alternative_1}
+    output_1 = constant_op.constant([1.])
+    output_2 = constant_op.constant(["2"])
+    output_3 = constant_op.constant(["3"])
+    export_outputs = {
+        signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
+            export_output.RegressionOutput(value=output_1),
+        "head-2": export_output.ClassificationOutput(classes=output_2),
+        "head-3": export_output.PredictOutput(outputs={
+            "some_output_3": output_3
+        }),
+    }
 
+    signature_defs = export_utils.build_all_signature_defs(
+        receiver_tensor, export_outputs, receiver_tensors_alternatives)
+
+    expected_signature_defs = {
+        "serving_default":
+            signature_def_utils.regression_signature_def(
+                receiver_tensor,
+                output_1),
+        "head-2":
+            signature_def_utils.classification_signature_def(
+                receiver_tensor,
+                output_2, None),
+        "head-3":
+            signature_def_utils.predict_signature_def(
+                {"input": receiver_tensor},
+                {"some_output_3": output_3}),
+        "other:head-3":
+            signature_def_utils.predict_signature_def(
+                receiver_tensors_alternative_1,
+                {"some_output_3": output_3})
+
+        # Note that the alternatives 'other:serving_default' and
+        # 'other:head-2' are invalid, because regession and classification
+        # signatures must take a single string input.  Here we verify that
+        # these invalid signatures are not included in the export_utils.
+    }
+
+    self.assertDictEqual(expected_signature_defs, signature_defs)
+
+  @test_util.deprecated_graph_mode_only
   def test_build_all_signature_defs_with_single_alternatives(self):
-    with context.graph_mode():
-      receiver_tensor = array_ops.placeholder(dtypes.string)
-      receiver_tensors_alternative_1 = array_ops.placeholder(dtypes.int64)
-      receiver_tensors_alternative_2 = array_ops.sparse_placeholder(
-          dtypes.float32)
-      # Note we are passing single Tensors as values of
-      # receiver_tensors_alternatives, where normally that is a dict.
-      # In this case a dict will be created using the default receiver tensor
-      # name "input".
-      receiver_tensors_alternatives = {"other1": receiver_tensors_alternative_1,
-                                       "other2": receiver_tensors_alternative_2}
-      output_1 = constant_op.constant([1.])
-      output_2 = constant_op.constant(["2"])
-      output_3 = constant_op.constant(["3"])
-      export_outputs = {
-          signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
-              export_output.RegressionOutput(value=output_1),
-          "head-2": export_output.ClassificationOutput(classes=output_2),
-          "head-3": export_output.PredictOutput(outputs={
-              "some_output_3": output_3
-          }),
-      }
+    receiver_tensor = array_ops.placeholder(dtypes.string)
+    receiver_tensors_alternative_1 = array_ops.placeholder(dtypes.int64)
+    receiver_tensors_alternative_2 = array_ops.sparse_placeholder(
+        dtypes.float32)
+    # Note we are passing single Tensors as values of
+    # receiver_tensors_alternatives, where normally that is a dict.
+    # In this case a dict will be created using the default receiver tensor
+    # name "input".
+    receiver_tensors_alternatives = {"other1": receiver_tensors_alternative_1,
+                                     "other2": receiver_tensors_alternative_2}
+    output_1 = constant_op.constant([1.])
+    output_2 = constant_op.constant(["2"])
+    output_3 = constant_op.constant(["3"])
+    export_outputs = {
+        signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
+            export_output.RegressionOutput(value=output_1),
+        "head-2": export_output.ClassificationOutput(classes=output_2),
+        "head-3": export_output.PredictOutput(outputs={
+            "some_output_3": output_3
+        }),
+    }
 
     signature_defs = export_utils.build_all_signature_defs(
         receiver_tensor, export_outputs, receiver_tensors_alternatives)
@@ -222,35 +221,35 @@ class ExportTest(test_util.TensorFlowTestCase):
     self.assertTrue(int(time_1) < int(time_2))
     self.assertTrue(int(time_2) < int(time_3))
 
+  @test_util.deprecated_graph_mode_only
   def test_build_all_signature_defs_serving_only(self):
-    with context.graph_mode():
-      receiver_tensor = {"input": array_ops.placeholder(dtypes.string)}
-      output_1 = constant_op.constant([1.])
-      export_outputs = {
-          signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
-              export_output.PredictOutput(outputs=output_1),
-          "train": export_output.TrainOutput(loss=output_1),
-      }
-
-      signature_defs = export_utils.build_all_signature_defs(
-          receiver_tensor, export_outputs)
-
-      expected_signature_defs = {
-          "serving_default": signature_def_utils.predict_signature_def(
-              receiver_tensor, {"output": output_1})
-      }
-
-      self.assertDictEqual(expected_signature_defs, signature_defs)
-
-      signature_defs = export_utils.build_all_signature_defs(
-          receiver_tensor, export_outputs, serving_only=False)
-
-      expected_signature_defs.update({
-          "train": signature_def_utils.supervised_train_signature_def(
-              receiver_tensor, loss={"loss": output_1})
-      })
-
-      self.assertDictEqual(expected_signature_defs, signature_defs)
+    receiver_tensor = {"input": array_ops.placeholder(dtypes.string)}
+    output_1 = constant_op.constant([1.])
+    export_outputs = {
+        signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
+            export_output.PredictOutput(outputs=output_1),
+        "train": export_output.TrainOutput(loss=output_1),
+    }
+
+    signature_defs = export_utils.build_all_signature_defs(
+        receiver_tensor, export_outputs)
+
+    expected_signature_defs = {
+        "serving_default": signature_def_utils.predict_signature_def(
+            receiver_tensor, {"output": output_1})
+    }
+
+    self.assertDictEqual(expected_signature_defs, signature_defs)
+
+    signature_defs = export_utils.build_all_signature_defs(
+        receiver_tensor, export_outputs, serving_only=False)
+
+    expected_signature_defs.update({
+        "train": signature_def_utils.supervised_train_signature_def(
+            receiver_tensor, loss={"loss": output_1})
+    })
+
+    self.assertDictEqual(expected_signature_defs, signature_defs)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/saved_model/utils_impl.py b/tensorflow/python/saved_model/utils_impl.py
index 5caabe59fe..d43cfdf2b3 100644
--- a/tensorflow/python/saved_model/utils_impl.py
+++ b/tensorflow/python/saved_model/utils_impl.py
@@ -22,6 +22,7 @@ import os
 
 from tensorflow.core.framework import types_pb2
 from tensorflow.core.protobuf import meta_graph_pb2
+from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
@@ -53,7 +54,12 @@ def build_tensor_info(tensor):
 
   Returns:
     A TensorInfo protocol buffer constructed based on the supplied argument.
+
+  Raises:
+    RuntimeError: If eager execution is enabled.
   """
+  if context.executing_eagerly():
+    raise RuntimeError("build_tensor_info is not supported in Eager mode.")
   tensor_info = meta_graph_pb2.TensorInfo(
       dtype=dtypes.as_dtype(tensor.dtype).as_datatype_enum,
       tensor_shape=tensor.get_shape().as_proto())
diff --git a/tensorflow/python/saved_model/utils_test.py b/tensorflow/python/saved_model/utils_test.py
index 2afe8abfd6..1e12de91b8 100644
--- a/tensorflow/python/saved_model/utils_test.py
+++ b/tensorflow/python/saved_model/utils_test.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.core.framework import types_pb2
+from tensorflow.python.eager import context
 from tensorflow.python.eager import function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -81,6 +82,12 @@ class UtilsTest(test.TestCase):
     self.assertEqual(42, x_tensor_info.tensor_shape.dim[0].size)
     self.assertEqual(69, x_tensor_info.tensor_shape.dim[1].size)
 
+  def testBuildTensorInfoEager(self):
+    x = constant_op.constant(1, name="x")
+    with context.eager_mode(), self.assertRaisesRegexp(
+        RuntimeError, "build_tensor_info is not supported in Eager mode"):
+      utils.build_tensor_info(x)
+
   @test_util.run_v1_only("b/120545219")
   def testGetTensorFromInfoDense(self):
     expected = array_ops.placeholder(dtypes.float32, 1, name="x")
-- 
GitLab


From f9c3fea70f7304c1ee96e8d293dd57623c06fe19 Mon Sep 17 00:00:00 2001
From: Davide Libenzi <dlibenzi@google.com>
Date: Tue, 8 Jan 2019 17:24:41 -0800
Subject: [PATCH 0393/2345] The HLO input/ouput alias config can be setup by
 the user, expecting a must alias semantics, and opportunistically by the
 compiler. Since the upper layer of the code will perform aliasing actions on
 their own, based on the HLO input/output aliasing configuration, we need a
 way for the upper layer to distringuish whether a given aliasing is coming
 from the user, or the compiler. This CL adds a single field to the alias
 data, to differentiate between user and compiler/system aliases.

PiperOrigin-RevId: 228429740
---
 tensorflow/compiler/xla/client/xla_builder.cc |   5 +-
 .../compiler/xla/service/buffer_assignment.cc |   9 +-
 .../compiler/xla/service/copy_insertion.cc    |  18 +--
 .../xla/service/copy_insertion_test.cc        |  21 ++--
 tensorflow/compiler/xla/service/hlo.proto     |  14 +++
 .../xla/service/hlo_alias_analysis.cc         |   9 +-
 .../xla/service/hlo_alias_analysis_test.cc    |  24 ++--
 .../service/hlo_input_output_alias_config.cc  | 119 ++++++++++--------
 .../service/hlo_input_output_alias_config.h   |  51 ++++++--
 .../hlo_input_output_alias_config_test.cc     |  60 +++++----
 10 files changed, 209 insertions(+), 121 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc
index 328abbc7b9..7a04c0ec69 100644
--- a/tensorflow/compiler/xla/client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_builder.cc
@@ -350,8 +350,9 @@ StatusOr<XlaComputation> XlaBuilder::Build(int64 root_id) {
                              alias.param_number,
                              alias.param_index.ToString().c_str());
     }
-    TF_RETURN_IF_ERROR(config.SetUpAlias(alias.output_index, alias.param_number,
-                                         alias.param_index));
+    TF_RETURN_IF_ERROR(config.SetUpAlias(
+        alias.output_index, alias.param_number, alias.param_index,
+        HloInputOutputAliasConfig::AliasKind::kUserAlias));
   }
   *module->mutable_input_output_alias() = config.ToProto();
   return Status::OK();
diff --git a/tensorflow/compiler/xla/service/buffer_assignment.cc b/tensorflow/compiler/xla/service/buffer_assignment.cc
index c3285f516b..d07615b828 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment.cc
+++ b/tensorflow/compiler/xla/service/buffer_assignment.cc
@@ -1530,15 +1530,16 @@ void BufferAssigner::BuildColocatedBufferSets(
   VLOG(4) << "Input/Output Alias Config: ";
   VLOG(4) << module->input_output_alias_config();
   module->input_output_alias_config().ForEachAlias(
-      [&](const ShapeIndex& output_index, int64 param_number,
-          const ShapeIndex& param_index) {
+      [&](const ShapeIndex& output_index,
+          const HloInputOutputAliasConfig::Alias& alias) {
         std::vector<const LogicalBuffer*> colocated_set;
         AddBufferToColocatedSet(module->entry_computation()->root_instruction(),
                                 output_index, points_to_analysis,
                                 &colocated_set);
         AddBufferToColocatedSet(
-            module->entry_computation()->parameter_instruction(param_number),
-            param_index, points_to_analysis, &colocated_set);
+            module->entry_computation()->parameter_instruction(
+                alias.parameter_number),
+            alias.parameter_index, points_to_analysis, &colocated_set);
         AddSetToColocatedBufferSets(colocated_set, colocated_buffer_sets);
       });
 
diff --git a/tensorflow/compiler/xla/service/copy_insertion.cc b/tensorflow/compiler/xla/service/copy_insertion.cc
index 2b837901f0..5e26a63ceb 100644
--- a/tensorflow/compiler/xla/service/copy_insertion.cc
+++ b/tensorflow/compiler/xla/service/copy_insertion.cc
@@ -349,11 +349,12 @@ Status AddCopiesForAliasedInputOutputs(HloModule* module) {
     ShapeTree<bool> param_indices_to_copy(param->shape());
 
     module->input_output_alias_config().ForEachAlias(
-        [&](const ShapeIndex& output_index, int64 param_number,
-            const ShapeIndex& param_index) {
-          if (param_number == param->parameter_number()) {
+        [&](const ShapeIndex& output_index,
+            const HloInputOutputAliasConfig::Alias& alias) {
+          if (alias.parameter_number == param->parameter_number()) {
             param_has_alias = true;
-            *(param_indices_to_copy.mutable_element(param_index)) = true;
+            *(param_indices_to_copy.mutable_element(alias.parameter_index)) =
+                true;
             *(output_indices_to_copy.mutable_element(output_index)) = true;
           }
         });
@@ -395,13 +396,14 @@ Status AddCopiesForAliasedInputOutputs(HloModule* module) {
 
   // Add control dependencies between the input/output copies.
   TF_RETURN_IF_ERROR(module->input_output_alias_config().ForEachAliasWithStatus(
-      [&](const ShapeIndex& output_index, int64 param_number,
-          const ShapeIndex& input_index) -> Status {
-        if (!copied_parameters[param_number]) {
+      [&](const ShapeIndex& output_index,
+          const HloInputOutputAliasConfig::Alias& alias) -> Status {
+        if (!copied_parameters[alias.parameter_number]) {
           return Status::OK();
         }
         HloInstruction* from =
-            copied_parameters[param_number]->element(input_index);
+            copied_parameters[alias.parameter_number]->element(
+                alias.parameter_index);
         HloInstruction* to = output_copy_tree.element(output_index);
 
         TF_RET_CHECK(from != nullptr);
diff --git a/tensorflow/compiler/xla/service/copy_insertion_test.cc b/tensorflow/compiler/xla/service/copy_insertion_test.cc
index e4e9d7ba05..4d4074943e 100644
--- a/tensorflow/compiler/xla/service/copy_insertion_test.cc
+++ b/tensorflow/compiler/xla/service/copy_insertion_test.cc
@@ -1376,9 +1376,11 @@ TEST_F(CopyInsertionTest, CrossingParameters) {
   builder.AddInstruction(HloInstruction::CreateTuple({gte1, gte0}));
   module->AddEntryComputation(builder.Build());
   ASSERT_IS_OK(module->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{0}));
+      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{0},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
   ASSERT_IS_OK(module->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{1}, /*param_number=*/0, /*param_index=*/{1}));
+      /*output_index=*/{1}, /*param_number=*/0, /*param_index=*/{1},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
   InsertCopies(module.get());
 
   EXPECT_EQ(CountCopies(*module), 4);
@@ -1409,9 +1411,11 @@ TEST_F(CopyInsertionTest, ParametersAliasing) {
   builder.AddInstruction(HloInstruction::CreateTuple({gte0, gte1}));
   module->AddEntryComputation(builder.Build());
   ASSERT_IS_OK(module->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{0}));
+      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{0},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
   ASSERT_IS_OK(module->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{1}, /*param_number=*/0, /*param_index=*/{1}));
+      /*output_index=*/{1}, /*param_number=*/0, /*param_index=*/{1},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
   InsertCopies(module.get());
 
   EXPECT_EQ(CountCopies(*module), 0);
@@ -1475,7 +1479,8 @@ TEST_F(CopyInsertionTest, ParameterWithPartialAliasing) {
   builder.AddInstruction(HloInstruction::CreateTuple({gte0, gte1}));
   module->AddEntryComputation(builder.Build());
   ASSERT_IS_OK(module->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{0}));
+      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{0},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
   InsertCopies(module.get());
 
   EXPECT_THAT(module->entry_computation()->root_instruction(),
@@ -1516,7 +1521,8 @@ TEST_F(CopyInsertionTest, ParameterAndParallelOpsWithPartialAliasing) {
   builder.AddInstruction(HloInstruction::CreateTuple({negate0, negate1}));
   module->AddEntryComputation(builder.Build());
   ASSERT_IS_OK(module->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{0}));
+      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{0},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
   InsertCopies(module.get());
 
   EXPECT_EQ(CountCopies(*module), 0);
@@ -1557,7 +1563,8 @@ TEST_F(CopyInsertionTest, ParameterAndOpsWithPartialAliasing) {
   builder.AddInstruction(HloInstruction::CreateTuple({add, negate1}));
   module->AddEntryComputation(builder.Build());
   ASSERT_IS_OK(module->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{0}));
+      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{0},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
   InsertCopies(module.get());
 
   EXPECT_EQ(CountCopies(*module), 0);
diff --git a/tensorflow/compiler/xla/service/hlo.proto b/tensorflow/compiler/xla/service/hlo.proto
index 9b50f1ca5b..263b42a29d 100644
--- a/tensorflow/compiler/xla/service/hlo.proto
+++ b/tensorflow/compiler/xla/service/hlo.proto
@@ -229,6 +229,18 @@ message HloScheduleProto {
 }
 
 message HloInputOutputAliasProto {
+  enum Kind {
+    // Define a UNDEFINED_ALIAS equal to zero to get around the default-0 proto3
+    // behavior and missing has_*() APIs.
+    UNDEFINED_ALIAS = 0;
+    // An alias setup by the user as must alias. A use setting USER_ALIAS is
+    // expecting the designed output to be dropped over the given input
+    // parameter number+index.
+    USER_ALIAS = 1;
+    // An alias setup by the compiler as part of its optimizations.
+    SYSTEM_ALIAS = 2;
+  }
+
   // The following proto describes a pair of aliased an input
   // (described by parameter number and a ShapeIndex of the parameter)
   // and an output (described by a ShapeIndex of the root
@@ -249,6 +261,8 @@ message HloInputOutputAliasProto {
     int64 parameter_number = 2;
     // ShapeIndex of the parameter instruction.
     repeated int64 parameter_shape_index = 3;
+    // The kind of alias to be setup.
+    Kind kind = 4;
   }
 
   repeated AliasEntryProto entries = 1;
diff --git a/tensorflow/compiler/xla/service/hlo_alias_analysis.cc b/tensorflow/compiler/xla/service/hlo_alias_analysis.cc
index 4c46ea595e..e511f1951c 100644
--- a/tensorflow/compiler/xla/service/hlo_alias_analysis.cc
+++ b/tensorflow/compiler/xla/service/hlo_alias_analysis.cc
@@ -176,13 +176,12 @@ class BufferValueMap {
       const HloValue& value, std::vector<BufferNumber>* aliased_buffers) {
     // Get parameter value from an aliased_input object.
     const auto get_parameter_value =
-        [this](const std::pair<int64, ShapeIndex>& aliased_input)
+        [this](const HloInputOutputAliasConfig::Alias& aliased_input)
         -> const HloValue& {
-      int64 param_number = aliased_input.first;
-      const ShapeIndex& param_index = aliased_input.second;
       return dataflow_.GetUniqueValueAt(
-          module_->entry_computation()->parameter_instruction(param_number),
-          param_index);
+          module_->entry_computation()->parameter_instruction(
+              aliased_input.parameter_number),
+          aliased_input.parameter_index);
     };
 
     // If the value shows up in a root instruction, alias it with parameter
diff --git a/tensorflow/compiler/xla/service/hlo_alias_analysis_test.cc b/tensorflow/compiler/xla/service/hlo_alias_analysis_test.cc
index 7e6150e941..b6dbf07959 100644
--- a/tensorflow/compiler/xla/service/hlo_alias_analysis_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_alias_analysis_test.cc
@@ -238,13 +238,16 @@ TEST_F(HloAliasAnalysisTest, ParametersWithAliasing) {
       builder.AddInstruction(HloInstruction::CreateTuple({negate0, negate1}));
   module_->AddEntryComputation(builder.Build());
   TF_ASSERT_OK(module_->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{0}));
+      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{0},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
   TF_ASSERT_OK(module_->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{1}, /*param_number=*/0, /*param_index=*/{1}));
+      /*output_index=*/{1}, /*param_number=*/0, /*param_index=*/{1},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
 
   // Cannot alias an output twice.
   ASSERT_IS_NOT_OK(module_->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{1}, /*param_number=*/0, /*param_index=*/{0}));
+      /*output_index=*/{1}, /*param_number=*/0, /*param_index=*/{0},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
 
   const HloAliasAnalysis& analysis = RunAnalysis();
 
@@ -279,13 +282,16 @@ TEST_F(HloAliasAnalysisTest, ParametersWithCrossAliasing) {
       builder.AddInstruction(HloInstruction::CreateTuple({gte0, gte1}));
   module_->AddEntryComputation(builder.Build());
   TF_ASSERT_OK(module_->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{1}));
+      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{1},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
   TF_ASSERT_OK(module_->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{1}, /*param_number=*/0, /*param_index=*/{0}));
+      /*output_index=*/{1}, /*param_number=*/0, /*param_index=*/{0},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
 
   // Cannot alias an output twice.
   ASSERT_IS_NOT_OK(module_->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{1}, /*param_number=*/0, /*param_index=*/{1}));
+      /*output_index=*/{1}, /*param_number=*/0, /*param_index=*/{1},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
 
   const HloAliasAnalysis& analysis = RunAnalysis();
 
@@ -365,9 +371,11 @@ TEST_F(HloAliasAnalysisTest, InputOutputAliasingWithWhile) {
       builder.AddInstruction(HloInstruction::CreateTuple({negate_1, negate_2}));
   module_->AddEntryComputation(builder.Build());
   TF_ASSERT_OK(module_->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{0}));
+      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{0},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
   TF_ASSERT_OK(module_->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{1}, /*param_number=*/0, /*param_index=*/{1}));
+      /*output_index=*/{1}, /*param_number=*/0, /*param_index=*/{1},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
 
   const HloAliasAnalysis& analysis = RunAnalysis();
 
diff --git a/tensorflow/compiler/xla/service/hlo_input_output_alias_config.cc b/tensorflow/compiler/xla/service/hlo_input_output_alias_config.cc
index 70d4df5d1c..b01c00121b 100644
--- a/tensorflow/compiler/xla/service/hlo_input_output_alias_config.cc
+++ b/tensorflow/compiler/xla/service/hlo_input_output_alias_config.cc
@@ -20,28 +20,31 @@ namespace xla {
 
 bool HloInputOutputAliasConfig::OutputHasAlias(
     const ShapeIndex& output_index) const {
-  return aliased_output_indices_.count(output_index) > 0;
+  return alias_.element(output_index).has_value();
 }
 
 Status HloInputOutputAliasConfig::SetUpAlias(const ShapeIndex& output_index,
                                              int64 param_number,
-                                             const ShapeIndex& param_index) {
-  TF_RET_CHECK(!OutputHasAlias(output_index))
-      << "Output index " << output_index << " already has an alias setup";
+                                             const ShapeIndex& param_index,
+                                             AliasKind kind) {
+  TF_RET_CHECK(kind == AliasKind::kUserAlias || kind == AliasKind::kSystemAlias)
+      << kind;
   TF_RET_CHECK(ShapeUtil::IndexIsValid(alias_.shape(), output_index))
       << absl::StrCat("Tring to set up alias at ", output_index.ToString(),
                       " which is an invalid index for shape ",
                       ShapeUtil::HumanString(alias_.shape()));
+  TF_RET_CHECK(param_number >= 0) << param_number;
+  TF_RET_CHECK(!OutputHasAlias(output_index))
+      << "Output index " << output_index << " already has an alias setup";
   // Output can't be aliased with multiple parameters.
   TF_RET_CHECK(!alias_.element(output_index)) << absl::StrFormat(
       "Trying to set up output alias for param %lld at %s but failed: output "
       "index %s is already aliased with param %lld at %s",
       param_number, param_index.ToString(), output_index.ToString(),
-      alias_.element(output_index)->first,
-      alias_.element(output_index)->second.ToString());
+      alias_.element(output_index)->parameter_number,
+      alias_.element(output_index)->parameter_index.ToString());
   (*alias_.mutable_element(output_index)) =
-      std::make_pair(param_number, param_index);
-  aliased_output_indices_.insert(output_index);
+      Alias(kind, param_number, param_index);
   VLOG(4) << "Set up alias between output index " << output_index.ToString()
           << " and parameter " << param_index << " at index "
           << param_index.ToString();
@@ -51,15 +54,24 @@ Status HloInputOutputAliasConfig::SetUpAlias(const ShapeIndex& output_index,
 HloInputOutputAliasProto HloInputOutputAliasConfig::ToProto() const {
   HloInputOutputAliasProto result;
   alias_.ForEachElement(
-      [&](const ShapeIndex& index,
-          const absl::optional<std::pair<int64, ShapeIndex>>& data) {
+      [&](const ShapeIndex& index, const absl::optional<Alias>& data) {
         if (data) {
           HloInputOutputAliasProto::AliasEntryProto entry;
+          switch (data->kind) {
+            case AliasKind::kUserAlias:
+              entry.set_kind(HloInputOutputAliasProto::USER_ALIAS);
+              break;
+            case AliasKind::kSystemAlias:
+              entry.set_kind(HloInputOutputAliasProto::SYSTEM_ALIAS);
+              break;
+            default:
+              LOG(FATAL) << "Unknown alias kind " << data->kind;
+          }
           for (int64 i : index) {
             entry.add_output_shape_index(i);
           }
-          entry.set_parameter_number(data->first);
-          for (int64 i : data->second) {
+          entry.set_parameter_number(data->parameter_number);
+          for (int64 i : data->parameter_index) {
             entry.add_parameter_shape_index(i);
           }
           result.add_entries()->Swap(&entry);
@@ -75,14 +87,18 @@ StatusOr<HloInputOutputAliasConfig> HloInputOutputAliasConfig::CreateFromProto(
        proto.entries()) {
     ShapeIndex output_index(entry.output_shape_index().begin(),
                             entry.output_shape_index().end());
-
     int64 param_number = entry.parameter_number();
     ShapeIndex param_index(entry.parameter_shape_index().begin(),
                            entry.parameter_shape_index().end());
+    // Handle backward compatibility with existing protos, which only knew of
+    // system aliases.
+    AliasKind kind = AliasKind::kSystemAlias;
+    if (entry.kind() == HloInputOutputAliasProto::USER_ALIAS) {
+      kind = AliasKind::kUserAlias;
+    }
     TF_RETURN_IF_ERROR(
-        result.SetUpAlias(output_index, param_number, param_index));
+        result.SetUpAlias(output_index, param_number, param_index, kind));
   }
-
   return result;
 }
 
@@ -90,45 +106,44 @@ string HloInputOutputAliasConfig::ToString() const {
   std::vector<string> pieces;
   pieces.push_back("HloInputOutputAliasConfig");
 
-  ForEachAlias([&](const ShapeIndex& output_index, int64 param_number,
-                   const ShapeIndex& param_index) {
+  ForEachAlias([&](const ShapeIndex& output_index, const Alias& alias) {
+    const char* kind = alias.kind == AliasKind::kUserAlias ? "USER" : "SYSTEM";
     pieces.push_back(absl::StrFormat(
-        "  OutputIndex %s is aliased with parameter %lld at %s:",
-        output_index.ToString(), param_number, param_index.ToString()));
+        "  OutputIndex %s is aliased (kind=%s) with parameter %lld at %s:",
+        output_index.ToString(), kind, alias.parameter_number,
+        alias.parameter_index.ToString()));
   });
-
   return absl::StrJoin(pieces, "\n");
 }
 
-bool HloInputOutputAliasConfig::ParameterHasAlias(
+HloInputOutputAliasConfig::AliasKind
+HloInputOutputAliasConfig::ParameterAliasKind(
     int64 param_number, const ShapeIndex& param_index) const {
-  bool output = false;
+  AliasKind kind = AliasKind::kNoAlias;
   alias_.ForEachElement(
-      [&](const xla::ShapeIndex&,
-          absl::optional<std::pair<int64, ShapeIndex>> alias) {
-        if (alias && alias->first == param_number &&
-            alias->second == param_index) {
-          output = true;
+      [&](const xla::ShapeIndex&, absl::optional<Alias> alias) {
+        if (alias && alias->parameter_number == param_number &&
+            alias->parameter_index == param_index) {
+          kind = alias->kind;
         }
       });
-  return output;
+  return kind;
 }
 
 absl::optional<ShapeIndex> HloInputOutputAliasConfig::GetAliasedOutput(
     int64 param_number, const ShapeIndex& param_index) const {
   absl::optional<ShapeIndex> output;
   alias_.ForEachElement(
-      [&](const xla::ShapeIndex& output_index,
-          absl::optional<std::pair<int64, ShapeIndex>> alias) {
-        if (alias && alias->first == param_number &&
-            alias->second == param_index) {
+      [&](const xla::ShapeIndex& output_index, absl::optional<Alias> alias) {
+        if (alias && alias->parameter_number == param_number &&
+            alias->parameter_index == param_index) {
           output = output_index;
         }
       });
   return output;
 }
 
-absl::optional<std::pair<int64, ShapeIndex>>
+absl::optional<HloInputOutputAliasConfig::Alias>
 HloInputOutputAliasConfig::GetAliasedParameter(
     const ShapeIndex& output_index) const {
   CHECK(ShapeUtil::IndexIsValid(alias_.shape(), output_index));
@@ -137,10 +152,9 @@ HloInputOutputAliasConfig::GetAliasedParameter(
 
 void HloInputOutputAliasConfig::ForEachAlias(AliasFn fn) const {
   alias_.ForEachElement(
-      [&](const ShapeIndex& output_index,
-          absl::optional<std::pair<int64, ShapeIndex>> aliased) {
+      [&](const ShapeIndex& output_index, absl::optional<Alias> aliased) {
         if (aliased) {
-          fn(output_index, aliased->first, aliased->second);
+          fn(output_index, *aliased);
         }
       });
 }
@@ -148,10 +162,9 @@ void HloInputOutputAliasConfig::ForEachAlias(AliasFn fn) const {
 Status HloInputOutputAliasConfig::ForEachAliasWithStatus(
     AliasFnWithStatus fn) const {
   return alias_.ForEachElementWithStatus(
-      [&](const ShapeIndex& output_index,
-          absl::optional<std::pair<int64, ShapeIndex>> aliased) {
+      [&](const ShapeIndex& output_index, absl::optional<Alias> aliased) {
         if (aliased) {
-          TF_RETURN_IF_ERROR(fn(output_index, aliased->first, aliased->second));
+          TF_RETURN_IF_ERROR(fn(output_index, *aliased));
         }
         return Status::OK();
       });
@@ -167,20 +180,19 @@ Status HloInputOutputAliasConfig::Verify(
     param_has_seen.emplace_back(param->shape());
   }
   return ForEachAliasWithStatus([&](const ShapeIndex& output_index,
-                                    int64 param_number,
-                                    const ShapeIndex& param_index) -> Status {
+                                    const Alias& alias) -> Status {
     const HloInstruction* root = entry->root_instruction();
 
-    TF_RET_CHECK(0 <= param_number);
-    TF_RET_CHECK(entry->num_parameters() > param_number);
+    TF_RET_CHECK(0 <= alias.parameter_number);
+    TF_RET_CHECK(entry->num_parameters() > alias.parameter_number);
     const Shape& param_shape =
-        entry->parameter_instruction(param_number)->shape();
+        entry->parameter_instruction(alias.parameter_number)->shape();
     const Shape& output_shape = root->shape();
-    TF_RET_CHECK(ShapeUtil::IndexIsValid(param_shape, param_index));
+    TF_RET_CHECK(ShapeUtil::IndexIsValid(param_shape, alias.parameter_index));
     TF_RET_CHECK(ShapeUtil::IndexIsValid(output_shape, output_index));
 
     const Shape& param_subshape =
-        ShapeUtil::GetSubshape(param_shape, param_index);
+        ShapeUtil::GetSubshape(param_shape, alias.parameter_index);
     const Shape& output_subshape =
         ShapeUtil::GetSubshape(output_shape, output_index);
     TF_RET_CHECK(LayoutUtil::IsDenseArray(param_subshape));
@@ -191,19 +203,20 @@ Status HloInputOutputAliasConfig::Verify(
           "Expected aliased input %lld at index %s and output at index %s to "
           "have the same size. Input sub-shape is %s with size %lld, output "
           "sub-shape is %s with size %lld",
-          param_number, param_index.ToString(), output_index.ToString(),
+          alias.parameter_number, alias.parameter_index.ToString(),
+          output_index.ToString(),
           ShapeUtil::HumanStringWithLayout(param_subshape),
           size_func(param_subshape),
           ShapeUtil::HumanStringWithLayout(output_subshape),
           size_func(output_subshape));
     }
 
-    // Check each param_number and param_index pair only show up once. No
-    // input can be aliased with output buffers.
-    TF_RET_CHECK(param_has_seen[param_number].element(param_index) == false);
-
-    *(param_has_seen[param_number].mutable_element(param_index)) = true;
-
+    // Check each alias.parameter_number and alias.parameter_index pair only
+    // show up once. No input can be aliased with output buffers.
+    TF_RET_CHECK(param_has_seen[alias.parameter_number].element(
+                     alias.parameter_index) == false);
+    *(param_has_seen[alias.parameter_number].mutable_element(
+        alias.parameter_index)) = true;
     return Status::OK();
   });
 }
diff --git a/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h b/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h
index 3967743d53..b0b71dece8 100644
--- a/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h
+++ b/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h
@@ -32,6 +32,28 @@ class HloModule;
 // parameter index in the entry computation.
 class HloInputOutputAliasConfig {
  public:
+  // The kind of aliases which can be set. A kUserAlias is one setup at
+  // compilation time by the user, and has to be respected. A kSystemAlias one
+  // might be setup by the compiler, if it decides it is convenient to do so.
+  enum AliasKind {
+    kNoAlias,
+    kUserAlias,
+    kSystemAlias,
+  };
+
+  // Defines the alias information for a given output buffer. A given output
+  // buffer shape index can refer only to one parameter+index.
+  struct Alias {
+    Alias(AliasKind kind, int64 parameter_number, ShapeIndex parameter_index)
+        : kind(kind),
+          parameter_number(parameter_number),
+          parameter_index(std::move(parameter_index)) {}
+
+    AliasKind kind;
+    int64 parameter_number;
+    ShapeIndex parameter_index;
+  };
+
   HloInputOutputAliasConfig() = default;
 
   explicit HloInputOutputAliasConfig(Shape shape) : alias_(shape) {}
@@ -41,12 +63,19 @@ class HloInputOutputAliasConfig {
   // Sets up alias config from `output_index` to `param_index` at
   // `param_number`.
   Status SetUpAlias(const ShapeIndex& output_index, int64 param_number,
-                    const ShapeIndex& param_index);
+                    const ShapeIndex& param_index, AliasKind kind);
+
+  // Returns the kind of alias for the given parameter number and parameter
+  // index. If no alias exists, AliasKind::kNoAlias is returned.
+  AliasKind ParameterAliasKind(int64 param_number,
+                               const ShapeIndex& param_index) const;
 
   // Returns true if the given parameter is aliased with one of the output
   // buffers.
   bool ParameterHasAlias(int64 param_number,
-                         const ShapeIndex& param_index) const;
+                         const ShapeIndex& param_index) const {
+    return ParameterAliasKind(param_number, param_index) != AliasKind::kNoAlias;
+  }
 
   // Checks whether the provided output index has already been aliased.
   bool OutputHasAlias(const ShapeIndex& output_index) const;
@@ -67,19 +96,17 @@ class HloInputOutputAliasConfig {
   // Returns the number of parameter and index of the parameter buffer that the
   // given output buffer index is aliased with. A nullopt is returned if there
   // is no parameter is aliased with the specific output.
-  absl::optional<std::pair<int64, ShapeIndex>> GetAliasedParameter(
+  absl::optional<Alias> GetAliasedParameter(
       const ShapeIndex& output_index) const;
 
   using AliasFn =
-      std::function<void(const ShapeIndex& output_index, int64 param_number,
-                         const ShapeIndex& param_index)>;
+      std::function<void(const ShapeIndex& output_index, const Alias&)>;
 
   // Iterates through each aliased output and input.
   void ForEachAlias(AliasFn fn) const;
 
   using AliasFnWithStatus =
-      std::function<Status(const ShapeIndex& output_index, int64 param_number,
-                           const ShapeIndex& param_index)>;
+      std::function<Status(const ShapeIndex& output_index, const Alias&)>;
 
   // Verifies that the given config is valid for the given module.
   // Specifically, the config's input and output should be in-bound and size of
@@ -94,12 +121,10 @@ class HloInputOutputAliasConfig {
  private:
   // A ShapeTree which indicates the list of buffers that's expected to be
   // aliased. The key on this shape tree represents the output index. The value
-  // is a pair of parameter number and index into the buffer. If the value is
-  // nullopt, it means there is no parameter aliasing for this output.
-  ShapeTree<absl::optional<std::pair<int64, ShapeIndex>>> alias_;
-
-  // The indices of the output which have been aliased.
-  absl::flat_hash_set<ShapeIndex> aliased_output_indices_;
+  // is an Alias data structure which defines the input parameter coordinates.
+  // If the value is nullopt, it means there is no parameter aliasing for this
+  // output.
+  ShapeTree<absl::optional<Alias>> alias_;
 };
 
 std::ostream& operator<<(std::ostream& out,
diff --git a/tensorflow/compiler/xla/service/hlo_input_output_alias_config_test.cc b/tensorflow/compiler/xla/service/hlo_input_output_alias_config_test.cc
index aeb9b0fdc8..a46a107723 100644
--- a/tensorflow/compiler/xla/service/hlo_input_output_alias_config_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_input_output_alias_config_test.cc
@@ -45,11 +45,12 @@ class HloInputOutputAliasConfigTest : public HloTestBase {
     EXPECT_TRUE(aliased_output);
     EXPECT_EQ(aliased_output.value(), output_index);
 
-    absl::optional<std::pair<int64, ShapeIndex>> aliased_param =
+    absl::optional<HloInputOutputAliasConfig::Alias> aliased_param =
         config.GetAliasedParameter(output_index);
 
     EXPECT_TRUE(aliased_param);
-    EXPECT_EQ(aliased_param.value(), std::make_pair(param_number, param_index));
+    EXPECT_EQ(aliased_param->parameter_number, param_number);
+    EXPECT_EQ(aliased_param->parameter_index, param_index);
   }
 
   void expect_not_aliased(const ShapeIndex& output_index, int64 param_number,
@@ -60,11 +61,12 @@ class HloInputOutputAliasConfigTest : public HloTestBase {
 
     EXPECT_FALSE(aliased_output && aliased_output == output_index);
 
-    absl::optional<std::pair<int64, ShapeIndex>> aliased_param =
+    absl::optional<HloInputOutputAliasConfig::Alias> aliased_param =
         config.GetAliasedParameter(output_index);
 
-    EXPECT_FALSE(aliased_param && aliased_param->first == param_number &&
-                 aliased_param->second == param_index);
+    EXPECT_FALSE(aliased_param &&
+                 aliased_param->parameter_number == param_number &&
+                 aliased_param->parameter_index == param_index);
   }
 };
 
@@ -84,8 +86,10 @@ ENTRY main {
   HloInputOutputAliasConfig config(
       module->entry_computation()->root_instruction()->shape());
 
-  TF_ASSERT_OK(config.SetUpAlias(/*output_index=*/{0}, /*param_number=*/1,
-                                 /*param_index=*/{}));
+  TF_ASSERT_OK(config.SetUpAlias(
+      /*output_index=*/{0}, /*param_number=*/1,
+      /*param_index=*/{},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
 
   expect_aliased(/*output_index=*/{0}, /*param_number=*/1,
                  /*param_index=*/{}, config);
@@ -114,11 +118,15 @@ ENTRY main {
   HloInputOutputAliasConfig config(
       module->entry_computation()->root_instruction()->shape());
 
-  TF_ASSERT_OK(config.SetUpAlias(/*output_index=*/{0}, /*param_number=*/0,
-                                 /*param_index=*/{0}));
+  TF_ASSERT_OK(config.SetUpAlias(
+      /*output_index=*/{0}, /*param_number=*/0,
+      /*param_index=*/{0},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
 
-  TF_ASSERT_OK(config.SetUpAlias(/*output_index=*/{1}, /*param_number=*/0,
-                                 /*param_index=*/{1}));
+  TF_ASSERT_OK(config.SetUpAlias(
+      /*output_index=*/{1}, /*param_number=*/0,
+      /*param_index=*/{1},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
 
   expect_aliased(/*output_index=*/{0}, /*param_number=*/0,
                  /*param_index=*/{0}, config);
@@ -149,11 +157,15 @@ ENTRY main {
   HloInputOutputAliasConfig config(
       module->entry_computation()->root_instruction()->shape());
 
-  TF_ASSERT_OK(config.SetUpAlias(/*output_index=*/{0}, /*param_number=*/0,
-                                 /*param_index=*/{}));
+  TF_ASSERT_OK(config.SetUpAlias(
+      /*output_index=*/{0}, /*param_number=*/0,
+      /*param_index=*/{},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
 
-  TF_ASSERT_OK(config.SetUpAlias(/*output_index=*/{1}, /*param_number=*/0,
-                                 /*param_index=*/{}));
+  TF_ASSERT_OK(config.SetUpAlias(
+      /*output_index=*/{1}, /*param_number=*/0,
+      /*param_index=*/{},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
 
   ASSERT_IS_NOT_OK(config.Verify(*module, [](const Shape& shape) {
     return ShapeUtil::ByteSizeOf(shape);
@@ -176,8 +188,10 @@ ENTRY main {
   HloInputOutputAliasConfig config(
       module->entry_computation()->root_instruction()->shape());
 
-  TF_ASSERT_OK(config.SetUpAlias(/*output_index=*/{1}, /*param_number=*/0,
-                                 /*param_index=*/{}));
+  TF_ASSERT_OK(config.SetUpAlias(
+      /*output_index=*/{1}, /*param_number=*/0,
+      /*param_index=*/{},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
 
   ASSERT_IS_NOT_OK(config.Verify(*module, [](const Shape& shape) {
     return ShapeUtil::ByteSizeOf(shape);
@@ -200,11 +214,15 @@ ENTRY main {
   HloInputOutputAliasConfig config(
       module->entry_computation()->root_instruction()->shape());
 
-  TF_ASSERT_OK(config.SetUpAlias(/*output_index=*/{0}, /*param_number=*/0,
-                                 /*param_index=*/{}));
+  TF_ASSERT_OK(config.SetUpAlias(
+      /*output_index=*/{0}, /*param_number=*/0,
+      /*param_index=*/{},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
 
-  ASSERT_IS_NOT_OK(config.SetUpAlias(/*output_index=*/{0}, /*param_number=*/1,
-                                     /*param_index=*/{}));
+  ASSERT_IS_NOT_OK(config.SetUpAlias(
+      /*output_index=*/{0}, /*param_number=*/1,
+      /*param_index=*/{},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
 }
 }  // namespace
 }  // namespace xla
-- 
GitLab


From fb8a14fde29f2dc19331d73795c2ead0e8ef0e8e Mon Sep 17 00:00:00 2001
From: Yifei Feng <yifeif@google.com>
Date: Tue, 8 Jan 2019 17:33:23 -0800
Subject: [PATCH 0394/2345] Remove cuda runtime dependencies from stream
 executor.

PiperOrigin-RevId: 228430943
---
 tensorflow/stream_executor/cuda/BUILD | 16 ++--------------
 1 file changed, 2 insertions(+), 14 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/BUILD b/tensorflow/stream_executor/cuda/BUILD
index 58c9480884..87c8eae416 100644
--- a/tensorflow/stream_executor/cuda/BUILD
+++ b/tensorflow/stream_executor/cuda/BUILD
@@ -146,13 +146,7 @@ cc_library(
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
         "//tensorflow/stream_executor/platform:dso_loader",
-    ] + if_static(
-        [
-            "@local_config_cuda//cuda:cublas",
-            "@local_config_cuda//cuda:cudart_static",
-        ],
-        ["@local_config_cuda//cuda:cudart"],
-    ),
+    ] + if_static(["@local_config_cuda//cuda:cublas"]),
     alwayslink = True,
 )
 
@@ -214,13 +208,7 @@ cc_library(
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
         "//tensorflow/stream_executor/platform:dso_loader",
-    ] + tf_additional_cudnn_plugin_deps() + if_static(
-        [
-            "@local_config_cuda//cuda:cudnn",
-            "@local_config_cuda//cuda:cudart_static",
-        ],
-        ["@local_config_cuda//cuda:cudart"],
-    ),
+    ] + tf_additional_cudnn_plugin_deps() + if_static(["@local_config_cuda//cuda:cudnn"]),
     alwayslink = True,
 )
 
-- 
GitLab


From 9e0e9e255fb9210ed2da50dffba156f1d5617b8b Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Wed, 9 Jan 2019 02:03:37 +0000
Subject: [PATCH 0395/2345] Update docstring for tf.train.init_from_checkpoint

This fix updates docstring for tf.train.init_from_checkpoint,
to match the actual implementation. (thanks sharvil)

This fix fixes 24756.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/training/checkpoint_utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/training/checkpoint_utils.py b/tensorflow/python/training/checkpoint_utils.py
index 74b46179e7..5e18f4b722 100644
--- a/tensorflow/python/training/checkpoint_utils.py
+++ b/tensorflow/python/training/checkpoint_utils.py
@@ -180,8 +180,8 @@ def init_from_checkpoint(ckpt_dir_or_file, assignment_map):
       (in default graph).
 
   Raises:
-    tf.errors.OpError: If missing checkpoints or tensors in checkpoints.
-    ValueError: If missing variables in current graph.
+    ValueError: If missing variables in current graph, or if missing
+      checkpoints or tensors in checkpoints.
   """
   if distribution_strategy_context.get_cross_replica_context():
     _init_from_checkpoint(None, ckpt_dir_or_file, assignment_map)
-- 
GitLab


From c8191270e47ece2bba777c271428e205d34d0827 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 18:41:13 -0800
Subject: [PATCH 0396/2345] Remove Identity nodes where all inputs and outputs
 are on the same device, but the identity is on a different device.

PiperOrigin-RevId: 228438774
---
 .../optimizers/dependency_optimizer.cc        | 17 ++++++------
 .../optimizers/dependency_optimizer_test.cc   | 26 +++++++++++++++++++
 2 files changed, 35 insertions(+), 8 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer.cc b/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
index 7fee3ae9d5..8b81cb2430 100644
--- a/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
@@ -205,14 +205,6 @@ bool DependencyOptimizer::BypassingNodeIsBeneficial(
     num_cross_out += static_cast<int>(output_node->device() != node_dev);
   }
 
-  if ((is_identity || is_multi_input_identity_n) && num_cross_in > 0 &&
-      num_cross_out > 0) {
-    // This identity node follows a device crossing, so it might be
-    // following a _Recv node after partioning. Do not remove such nodes,
-    // unless they only have consumers on the same device as themselves.
-    return false;
-  }
-
   // Make sure we do not increase the number of device crossings.
   const int num_cross_before = num_cross_in + num_cross_out;
   int num_cross_after = 0;
@@ -225,6 +217,15 @@ bool DependencyOptimizer::BypassingNodeIsBeneficial(
   if (num_cross_after > num_cross_before) {
     return false;
   }
+
+  if ((is_identity || is_multi_input_identity_n) && num_cross_in > 0 &&
+      num_cross_out > 0 && num_cross_after > 0) {
+    // This identity node follows a device crossing, so it might be
+    // following a _Recv node after partioning. Do not remove such nodes,
+    // unless they only have consumers on the same device as themselves.
+    return false;
+  }
+
   return true;
 }
 
diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc b/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
index 8d70d9d5c7..5883fcb926 100644
--- a/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
@@ -356,6 +356,32 @@ TEST_F(DependencyOptimizerTest, RemoveIdentityOps_DeviceBoundaries) {
   VerifyGraphsEqual(item.graph, output, __FUNCTION__);
 }
 
+TEST_F(DependencyOptimizerTest, RemoveIdentityOps_IdenticalDevices) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+  Output x = ops::RandomUniform(s.WithOpName("x").WithDevice("/CPU:0"), {1, 2},
+                                DT_FLOAT);
+  auto id_a = ops::Identity(s.WithOpName("id_a").WithDevice("/CPU:1"), x);
+  Output id =
+      ops::Identity(s.WithControlDependencies(id_a).WithDevice("/CPU:0"), id_a);
+
+  GrapplerItem item;
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+  item.fetch.push_back("Identity");
+
+  DependencyOptimizer optimizer;
+  GraphDef output;
+  Status status = optimizer.Optimize(nullptr, item, &output);
+  TF_EXPECT_OK(status);
+
+  EXPECT_EQ(item.graph.node_size() - 1, output.node_size());
+  for (const NodeDef& node : output.node()) {
+    EXPECT_NE(node.name(), "id_a");
+    if (node.name() == "Identity") {
+      EXPECT_EQ(node.input(0), "x");
+    }
+  }
+}
+
 TEST_F(DependencyOptimizerTest, RemoveNoOps_SingleInputOrOutput) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output x = ops::RandomUniform(s.WithOpName("x"), {1, 2}, DT_FLOAT);
-- 
GitLab


From ec81825aaf7e848d9f8ddffdf1e0d20aebe9172c Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne <reedwm@google.com>
Date: Tue, 8 Jan 2019 18:50:27 -0800
Subject: [PATCH 0397/2345] Add GPU explicit padding to tf.nn.conv2d.

Benchmark results:

All benchmark results were run on a Z840 with a Titan V, with internal TensorFlow.

1. Resnet50 Eager results
The internal resnet50 Eager benchmarks were run, to ensure no regressions in Resnet50 in Eager mode that could have occurred due to the extra Python overhead this change adds. The benchmarks run are here: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/resnet50/resnet50_test.py. Each row was run 150 times and the average was taken. Note none of these benchmarks use explicit padding. Numbers represent time, so lower is better.

Benchmark Name                                        After   Before  % diff
apply_async_gpu_batch_64_channels_first               0.0726  0.0726  -0.06%
apply_gpu_batch_64_channels_first                     0.0725  0.0725  -0.06%
apply_with_defun_gpu_batch_64_channels_first          0.0755  0.0756   0.07%
train_async_gpu_batch_16_channels_first               0.0776  0.0778   0.27%
train_async_gpu_batch_32_channels_first               0.1268  0.1271   0.23%
train_dataset_gpu_batch_16_channels_first             0.1085  0.1094   0.77%
train_dataset_gpu_batch_32_channels_first             0.1473  0.1477   0.28%
train_dataset_with_defun_gpu_batch_16_channels_first  0.0800  0.0803   0.37%
train_dataset_with_defun_gpu_batch_32_channels_first  0.1325  0.1326   0.09%
train_gpu_batch_16_channels_first                     0.0812  0.0813   0.18%
train_gpu_batch_32_channels_first                     0.1329  0.1325  -0.32%
train_with_defun_gpu_batch_16_channels_first          0.0789  0.0791   0.26%
train_with_defun_gpu_batch_32_channels_first          0.1325  0.1325  -0.02%

There is minimal impact to Eager performance.

2. tf_cnn_benchmarks
tf_cnn_benchmarks was run internally with the following flags:

--batch_size=128 --model=resnet50

It was run 60 times with and without this change. With this change, tf_cnn_benchmarks had all instances of a tf.pad followed by Conv2D replaced with an explicitly padded Conv2d. It got 330.96 images/sec with this change and 330.80 without, and the difference is likely noise. Therefore, this change does not improve tf_cnn_benchmarks performance.

3. Conv2D benchmarks

The added benchmarks to conv_ops_test.py were run with this change, each 400 times and the average was taken. They were not run without this change. The table groups the 8 benchmarks into 4 pairs, with each pair running two similar benchmarks, one with explicit padding, and one without explicit padding.

Benchmark name                Explicit  Non-explicit  % diff
explicit/manual pad forward   0.001815  0.002006      10.56%
explicit/manual pad backward  0.006261  0.006937      10.79%
eager explicit/same pad       0.039320  0.038403      -2.33%
graph explicit/same pad       0.037039  0.037034      -0.11%

The first two rows show there is theoretical performance gains to using explicit padding over a manual tf.pad followed by the convolution. On Resnet50, we were not able to achieve this performance gain in practice, as tf_cnn_benchmarks saw no improvement. On models that use larger paddings than in Resnet50, the performance gain will less negligible. The last two rows compare explicit padding padding to the equivalent same padding, to see if explicit padding adds any overhead. In Graph mode, there is no overhead, but Eager mode has some overhead with explicit padding over SAME padding.

PiperOrigin-RevId: 228439591
---
 .../tf2xla/kernels/conv_op_helpers.cc         |    9 +-
 .../api_def/base_api/api_def_Conv2D.pbtxt     |    9 +
 .../api_def_Conv2DBackpropFilter.pbtxt        |    9 +
 .../api_def_Conv2DBackpropInput.pbtxt         |    9 +
 tensorflow/core/framework/common_shape_fns.cc |   84 +-
 tensorflow/core/framework/common_shape_fns.h  |   48 +-
 .../core/framework/common_shape_fns_test.cc   |   82 +-
 tensorflow/core/framework/ops_util.cc         |    3 +
 .../large_function_graph.pbtxt                |    7 +
 .../grappler/optimizers/layout_optimizer.cc   |   23 +
 .../optimizers/layout_optimizer_test.cc       |   64 +-
 .../core/kernels/conv_grad_filter_ops.cc      |  134 +-
 .../core/kernels/conv_grad_input_ops.cc       |  146 +-
 tensorflow/core/kernels/conv_grad_ops.cc      |   51 +-
 tensorflow/core/kernels/conv_grad_ops.h       |   20 +-
 tensorflow/core/kernels/conv_grad_ops_3d.cc   |   21 +-
 tensorflow/core/kernels/conv_ops.cc           |  223 +--
 tensorflow/core/kernels/conv_ops.h            |   13 +-
 .../core/kernels/depthwise_conv_grad_op.cc    |    6 +-
 tensorflow/core/kernels/depthwise_conv_op.cc  |    3 +-
 tensorflow/core/ops/nn_ops.cc                 |   11 +-
 tensorflow/core/util/padding.cc               |   43 +
 tensorflow/core/util/padding.h                |   19 +-
 tensorflow/core/util/tensor_format.h          |   27 +-
 .../python/keras/layers/convolutional.py      |    4 +-
 tensorflow/python/keras/utils/conv_utils.py   |    4 +-
 .../python/kernel_tests/conv_ops_test.py      | 1232 +++++++++++++++--
 tensorflow/python/ops/nn_grad.py              |   15 +-
 tensorflow/python/ops/nn_ops.py               |  316 ++++-
 29 files changed, 2218 insertions(+), 417 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc b/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
index 6b049f653c..5b4f863f74 100644
--- a/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
+++ b/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
@@ -212,8 +212,8 @@ Status ConvBackpropComputeDimensionsV2XlaShapes(
       XLAShapeToTensorShape(out_backprop_shape, &out_backprop_tensor_shape));
   return ConvBackpropComputeDimensionsV2(
       label, num_spatial_dims, input_tensor_shape, filter_tensor_shape,
-      out_backprop_tensor_shape, dilations, strides, padding, data_format,
-      dims);
+      out_backprop_tensor_shape, dilations, strides, padding,
+      /*explicit_paddings=*/{}, data_format, dims);
 }
 
 }  // anonymous namespace
@@ -227,6 +227,11 @@ xla::StatusOr<ConvOpAttrs> ConvOpAttrs::Create(int num_spatial_dims,
   TF_RETURN_IF_ERROR(ctx->GetAttr("dilations", &attrs.dilations));
   TF_RETURN_IF_ERROR(ctx->GetAttr("strides", &attrs.strides));
   TF_RETURN_IF_ERROR(ctx->GetAttr("padding", &attrs.padding));
+  // TODO(reedwm): Support explicit padding.
+  if (attrs.padding == EXPLICIT) {
+    return errors::Unimplemented(
+        "XLA does not yet support Conv2D with explicit padding.");
+  }
 
   string data_format;
   TF_RETURN_IF_ERROR(ctx->GetAttr("data_format", &data_format));
diff --git a/tensorflow/core/api_def/base_api/api_def_Conv2D.pbtxt b/tensorflow/core/api_def/base_api/api_def_Conv2D.pbtxt
index 070d6adb97..d0794de4ba 100644
--- a/tensorflow/core/api_def/base_api/api_def_Conv2D.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_Conv2D.pbtxt
@@ -33,6 +33,15 @@ END
     name: "padding"
     description: <<END
 The type of padding algorithm to use.
+END
+  }
+  attr {
+    name: "explicit_paddings"
+    description: <<END
+If `padding` is `"EXPLICIT"`, the list of explicit padding amounts. For the ith
+dimension, the amount of padding inserted before and after the dimension is
+`explicit_paddings[2 * i]` and `explicit_paddings[2 * i + 1]`, respectively. If
+`padding` is not `"EXPLICIT"`, `explicit_paddings` must be empty.
 END
   }
   attr {
diff --git a/tensorflow/core/api_def/base_api/api_def_Conv2DBackpropFilter.pbtxt b/tensorflow/core/api_def/base_api/api_def_Conv2DBackpropFilter.pbtxt
index ff2d9d71db..c8af9ff976 100644
--- a/tensorflow/core/api_def/base_api/api_def_Conv2DBackpropFilter.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_Conv2DBackpropFilter.pbtxt
@@ -41,6 +41,15 @@ END
     name: "padding"
     description: <<END
 The type of padding algorithm to use.
+END
+  }
+  attr {
+    name: "explicit_paddings"
+    description: <<END
+If `padding` is `"EXPLICIT"`, the list of explicit padding amounts. For the ith
+dimension, the amount of padding inserted before and after the dimension is
+`explicit_paddings[2 * i]` and `explicit_paddings[2 * i + 1]`, respectively. If
+`padding` is not `"EXPLICIT"`, `explicit_paddings` must be empty.
 END
   }
   attr {
diff --git a/tensorflow/core/api_def/base_api/api_def_Conv2DBackpropInput.pbtxt b/tensorflow/core/api_def/base_api/api_def_Conv2DBackpropInput.pbtxt
index 2de38b4263..8aaae4aab6 100644
--- a/tensorflow/core/api_def/base_api/api_def_Conv2DBackpropInput.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_Conv2DBackpropInput.pbtxt
@@ -40,6 +40,15 @@ END
     name: "padding"
     description: <<END
 The type of padding algorithm to use.
+END
+  }
+  attr {
+    name: "explicit_paddings"
+    description: <<END
+If `padding` is `"EXPLICIT"`, the list of explicit padding amounts. For the ith
+dimension, the amount of padding inserted before and after the dimension is
+`explicit_paddings[2 * i]` and `explicit_paddings[2 * i + 1]`, respectively. If
+`padding` is not `"EXPLICIT"`, `explicit_paddings` must be empty.
 END
   }
   attr {
diff --git a/tensorflow/core/framework/common_shape_fns.cc b/tensorflow/core/framework/common_shape_fns.cc
index bf2d902af4..3d3711ffc5 100644
--- a/tensorflow/core/framework/common_shape_fns.cc
+++ b/tensorflow/core/framework/common_shape_fns.cc
@@ -37,6 +37,11 @@ Status GetWindowedOutputSizeVerboseV2(int64 input_size, int64 filter_size,
       *output_size = (input_size - effective_filter_size + stride) / stride;
       *padding_before = *padding_after = 0;
       break;
+    case Padding::EXPLICIT:
+      *output_size = (input_size + *padding_before + *padding_after -
+                      effective_filter_size + stride) /
+                     stride;
+      break;
     case Padding::SAME:
       *output_size = (input_size + stride - 1) / stride;
       const int64 padding_needed =
@@ -71,6 +76,11 @@ Status GetWindowedOutputSizeVerbose(int64 input_size, int64 filter_size,
 Status GetWindowedOutputSize(int64 input_size, int64 filter_size, int64 stride,
                              Padding padding_type, int64* output_size,
                              int64* padding_size) {
+  if (padding_type == Padding::EXPLICIT) {
+    return errors::Internal(
+        "GetWindowedOutputSize does not handle EXPLICIT padding; call "
+        "GetWindowedOutputSizeVerbose instead");
+  }
   int64 padding_after_unused;
   return GetWindowedOutputSizeVerbose(input_size, filter_size, stride,
                                       padding_type, output_size, padding_size,
@@ -81,6 +91,11 @@ Status GetWindowedOutputSizeV2(int64 input_size, int64 filter_size,
                                int64 dilation_rate, int64 stride,
                                Padding padding_type, int64* output_size,
                                int64* padding_size) {
+  if (padding_type == Padding::EXPLICIT) {
+    return errors::Internal(
+        "GetWindowedOutputSizeV2 does not handle EXPLICIT padding; call "
+        "GetWindowedOutputSizeVerboseV2 instead");
+  }
   int64 padding_after_unused;
   return GetWindowedOutputSizeVerboseV2(input_size, filter_size, dilation_rate,
                                         stride, padding_type, output_size,
@@ -123,8 +138,8 @@ Status GetWindowedOutputSizeFromDimsV2(
     shape_inference::InferenceContext* c,
     shape_inference::DimensionHandle input_size,
     shape_inference::DimensionOrConstant filter_size, int64 dilation_rate,
-    int64 stride, Padding padding_type,
-    shape_inference::DimensionHandle* output_size) {
+    int64 stride, Padding padding_type, int64 padding_before,
+    int64 padding_after, shape_inference::DimensionHandle* output_size) {
   if (stride <= 0) {
     return errors::InvalidArgument("Stride must be > 0, but got ", stride);
   }
@@ -137,6 +152,11 @@ Status GetWindowedOutputSizeFromDimsV2(
   // See also the parallel implementation in GetWindowedOutputSizeVerbose.
   switch (padding_type) {
     case Padding::VALID:
+      padding_before = padding_after = 0;
+      TF_FALLTHROUGH_INTENDED;
+    case Padding::EXPLICIT:
+      TF_RETURN_IF_ERROR(
+          c->Add(input_size, padding_before + padding_after, &input_size));
       if (dilation_rate > 1) {
         DimensionHandle window_size;
         TF_RETURN_IF_ERROR(
@@ -166,9 +186,18 @@ Status GetWindowedOutputSizeFromDims(
     shape_inference::DimensionHandle input_size,
     shape_inference::DimensionOrConstant filter_size, int64 stride,
     Padding padding_type, shape_inference::DimensionHandle* output_size) {
+  if (padding_type == Padding::EXPLICIT) {
+    return errors::Internal(
+        "GetWindowedOutputSizeFromDims does not handle EXPLICIT padding; call "
+        "GetWindowedOutputSizeFromDimsV2 instead");
+  }
   return GetWindowedOutputSizeFromDimsV2(c, input_size, filter_size,
                                          /*dilation_rate=*/1, stride,
-                                         padding_type, output_size);
+                                         padding_type,
+                                         // Give dummy values of -1 to
+                                         // padding_before and padding_after,
+                                         // since explicit padding is not used.
+                                         -1, -1, output_size);
 }
 
 Status UnchangedShape(shape_inference::InferenceContext* c) {
@@ -371,7 +400,10 @@ Status ShapeFromDimensions(DimensionHandle batch_dim,
   return tensorflow::Status::OK();
 }
 
-Status Conv2DShape(shape_inference::InferenceContext* c) {
+namespace {
+
+Status Conv2DShapeImpl(shape_inference::InferenceContext* c,
+                       bool supports_explicit_padding) {
   string data_format_str, filter_format_str;
   if (!c->GetAttr("data_format", &data_format_str).ok()) {
     data_format_str = "NHWC";
@@ -464,13 +496,30 @@ Status Conv2DShape(shape_inference::InferenceContext* c) {
   Padding padding;
   TF_RETURN_IF_ERROR(c->GetAttr("padding", &padding));
 
+  std::vector<int64> explicit_paddings;
+  if (supports_explicit_padding) {
+    TF_RETURN_IF_ERROR(c->GetAttr("explicit_paddings", &explicit_paddings));
+    TF_RETURN_IF_ERROR(CheckValidPadding(padding, explicit_paddings,
+                                         /*num_dims=*/4, data_format));
+  } else {
+    DCHECK(padding != Padding::EXPLICIT);
+  }
+
   DimensionHandle output_rows, output_cols;
+  int64 pad_rows_before = -1, pad_rows_after = -1;
+  int64 pad_cols_before = -1, pad_cols_after = -1;
+  if (padding == Padding::EXPLICIT) {
+    GetExplicitPaddingForDim(explicit_paddings, data_format, 'H',
+                             &pad_rows_before, &pad_rows_after);
+    GetExplicitPaddingForDim(explicit_paddings, data_format, 'W',
+                             &pad_cols_before, &pad_cols_after);
+  }
   TF_RETURN_IF_ERROR(GetWindowedOutputSizeFromDimsV2(
       c, input_spatial_dims[0], filter_rows_dim, dilation_rows, stride_rows,
-      padding, &output_rows));
+      padding, pad_rows_before, pad_rows_after, &output_rows));
   TF_RETURN_IF_ERROR(GetWindowedOutputSizeFromDimsV2(
       c, input_spatial_dims[1], filter_cols_dim, dilation_cols, stride_cols,
-      padding, &output_cols));
+      padding, pad_cols_before, pad_cols_after, &output_cols));
 
   ShapeHandle output_shape;
   TF_RETURN_IF_ERROR(
@@ -480,6 +529,19 @@ Status Conv2DShape(shape_inference::InferenceContext* c) {
   return Status::OK();
 }
 
+}  // namespace
+
+// Shape function for Conv2D-like operations that support explicit padding.
+Status Conv2DShapeWithExplicitPadding(shape_inference::InferenceContext* c) {
+  return Conv2DShapeImpl(c, true);
+}
+
+// Shape function for Conv2D-like operations that do not support explicit
+// padding.
+Status Conv2DShape(shape_inference::InferenceContext* c) {
+  return Conv2DShapeImpl(c, false);
+}
+
 // TODO(mjanusz): Unify all conv/pooling shape functions.
 Status Conv3DShape(shape_inference::InferenceContext* c) {
   ShapeHandle input_shape;
@@ -551,13 +613,13 @@ Status Conv3DShape(shape_inference::InferenceContext* c) {
 
   TF_RETURN_IF_ERROR(GetWindowedOutputSizeFromDimsV2(
       c, in_planes_dim, filter_planes_dim, dilation_planes, stride_planes,
-      padding, &output_planes));
+      padding, -1, -1, &output_planes));
   TF_RETURN_IF_ERROR(GetWindowedOutputSizeFromDimsV2(
-      c, in_rows_dim, filter_rows_dim, dilation_rows, stride_rows, padding,
-      &output_rows));
+      c, in_rows_dim, filter_rows_dim, dilation_rows, stride_rows, padding, -1,
+      -1, &output_rows));
   TF_RETURN_IF_ERROR(GetWindowedOutputSizeFromDimsV2(
-      c, in_cols_dim, filter_cols_dim, dilation_cols, stride_cols, padding,
-      &output_cols));
+      c, in_cols_dim, filter_cols_dim, dilation_cols, stride_cols, padding, -1,
+      -1, &output_cols));
 
   ShapeHandle output_shape;
   if (data_format == "NCDHW") {
diff --git a/tensorflow/core/framework/common_shape_fns.h b/tensorflow/core/framework/common_shape_fns.h
index 362899b947..14b9688bdc 100644
--- a/tensorflow/core/framework/common_shape_fns.h
+++ b/tensorflow/core/framework/common_shape_fns.h
@@ -38,11 +38,12 @@ namespace tensorflow {
 //
 // Padding (P): the padding we apply to the input tensor along each
 // dimension. This is usually used to make sure that the spatial dimensions
-// do not shrink when we progress with convolutions. Two types of padding are
-// often used:
+// do not shrink when we progress with convolutions. This function supports two
+// types of padding.
 //   SAME: the pad value is computed so that the output will have size H/S.
 //   VALID: no padding is carried out.
-// The padded area is zero-filled.
+// If you want to use EXPLICIT padding, GetWindowedOutputSizeVerbose must be
+// called instead. Note the padded area is zero-filled.
 //
 // The output dimensions for convolution and many other operations, when given
 // all the parameters above, are as follows:
@@ -95,6 +96,9 @@ Status GetWindowedOutputSize(int64 input_size, int64 filter_size, int64 stride,
 //   When the stride is 1, the expression simplifies to
 //     H' = H-K'+1.
 //
+// If you want to use EXPLICIT padding, GetWindowedOutputSizeVerboseV2 must be
+// called instead
+//
 // TODO(b/67112639): Merge V2 versions and the original versions eventually.
 Status GetWindowedOutputSizeV2(int64 input_size, int64 filter_size,
                                int64 dilation_rate, int64 stride,
@@ -102,9 +106,12 @@ Status GetWindowedOutputSizeV2(int64 input_size, int64 filter_size,
                                int64* padding_size);
 
 // Returns the same output dimensions as in GetWindowedOutputSize, but returns
-// verbose padding dimensions (before/after). Any excess padding
-// (caused by an odd padding size value) is added to the 'padding_after'
-// dimension.
+// verbose padding dimensions (before/after), and EXPLICIT padding is supported.
+// When padding_type is EXPLICIT, *padding_before and *padding_after must
+// already point to initialized integers with the padding amounts. Otherwise,
+// *padding_before and *padding_after are set by this function, and any
+// excess padding (caused by an odd padding size value) is added to the
+// 'padding_after' dimension.
 Status GetWindowedOutputSizeVerbose(int64 input_size, int64 filter_size,
                                     int64 stride, Padding padding_type,
                                     int64* output_size, int64* padding_before,
@@ -122,7 +129,8 @@ Status GetWindowedOutputSizeVerboseV2(int64 input_size, int64 filter_size,
 // of the output tensor and padding to be applied to the input tensor at the
 // lower end of every dimension. Use for 3D convolutions, where the input data
 // is padded with zeros, as well as for 3D avg/max pooling, where the input data
-// is padded with invalid values that are not considered for pooling.
+// is padded with invalid values that are not considered for pooling. EXPLICIT
+// padding is not supported.
 Status Get3dOutputSize(const std::array<int64, 3>& input,
                        const std::array<int64, 3>& window,
                        const std::array<int64, 3>& strides,
@@ -140,21 +148,23 @@ Status Get3dOutputSizeV2(const std::array<int64, 3>& input,
 
 namespace shape_inference {
 
-// Like GetWindowedOutputSize, but deals with DimensionHandles.
+// Like GetWindowedOutputSize, but deals with DimensionHandles. Does not support
+// EXPLICIT padding.
 Status GetWindowedOutputSizeFromDims(InferenceContext* c,
                                      DimensionHandle input_size,
                                      DimensionOrConstant filter_size,
                                      int64 stride, Padding padding_type,
                                      DimensionHandle* output_size);
 
-// The V2 version computes the same outputs with arbitrary dilation_rate. For
-// detailed equations, refer to the comments for GetWindowedOutputSizeV2().
-Status GetWindowedOutputSizeFromDimsV2(InferenceContext* c,
-                                       DimensionHandle input_size,
-                                       DimensionOrConstant filter_size,
-                                       int64 dilation_rate, int64 stride,
-                                       Padding padding_type,
-                                       DimensionHandle* output_size);
+// The V2 version computes the same outputs with arbitrary dilation_rate, and
+// supports EXPLICIT padding. For detailed equations, refer to the comments
+// for GetWindowedOutputSizeV2(). The 'padding_before' and 'padding_after'
+// parameters are only used if padding_type == EXPLICIT.
+Status GetWindowedOutputSizeFromDimsV2(
+    InferenceContext* c, DimensionHandle input_size,
+    DimensionOrConstant filter_size, int64 dilation_rate, int64 stride,
+    Padding padding_type, int64 padding_before, int64 padding_after,
+    DimensionHandle* output_size);
 
 // Transfers shape of input(0) to output(0).
 Status UnchangedShape(shape_inference::InferenceContext* c);
@@ -222,7 +232,11 @@ Status BiasAddShape(shape_inference::InferenceContext* c);
 // Shape function for BiasAddGrad-like operations.
 Status BiasAddGradShape(shape_inference::InferenceContext* c);
 
-// Shape function for Conv2D-like operations.
+// Shape function for Conv2D-like operations that support explicit padding.
+Status Conv2DShapeWithExplicitPadding(shape_inference::InferenceContext* c);
+
+// Shape function for Conv2D-like operations that do not support explicit
+// padding.
 Status Conv2DShape(shape_inference::InferenceContext* c);
 
 // Shape function for Conv3D-like operations.
diff --git a/tensorflow/core/framework/common_shape_fns_test.cc b/tensorflow/core/framework/common_shape_fns_test.cc
index 7c395679d3..b94925c04e 100644
--- a/tensorflow/core/framework/common_shape_fns_test.cc
+++ b/tensorflow/core/framework/common_shape_fns_test.cc
@@ -408,12 +408,14 @@ TEST(CommonShapeFnsTest, BiasAddGradShapeTest) {
 TEST(CommonShapeFnsTest, Conv2DShapeTest) {
   ShapeInferenceTestOp op("Conv2D");
   auto set_op = [&op](const std::vector<int32>& strides, const string& padding,
-                      const string& data_format, const string& filter_format) {
+                      const string& data_format, const string& filter_format,
+                      const std::vector<int32>& explicit_paddings = {}) {
     TF_CHECK_OK(NodeDefBuilder("test", "Conv2D")
                     .Input("input", 0, DT_FLOAT)
                     .Input("filter", 0, DT_FLOAT)
                     .Attr("strides", strides)
                     .Attr("padding", padding)
+                    .Attr("explicit_paddings", explicit_paddings)
                     .Attr("data_format", data_format)
                     .Attr("filter_format", filter_format)
                     .Finalize(&op.node_def));
@@ -536,19 +538,73 @@ TEST(CommonShapeFnsTest, Conv2DShapeTest) {
   INFER_OK(op, "[1,?,4,1];[?,?,?,?]", "[d0_0,?,2,d1_3]");
   INFER_OK(op, "[1,4,?,1];[?,?,?,?]", "[d0_0,2,?,d1_3]");
   INFER_OK(op, "[1,4,4,?];[?,?,?,?]", "[d0_0,2,2,d1_3]");
+
+  // Some tests for "EXPLICIT" padding
+
+  // 4x4 input, 1x1 filter, 1x1 stride, [0, 2, 1, 4] padding
+  set_op({{1, 1, 1, 1}}, "EXPLICIT", "NHWC", "HWIO", {0, 0, 0, 2, 1, 4, 0, 0});
+  INFER_OK(op, "[1,4,4,1];[1,1,1,1]", "[d0_0,6,9,d1_3]");
+
+  // 3x3 input, 2x2 filter, 1x1 stride, [1, 0, 1, 2] padding
+  set_op({{1, 1, 1, 1}}, "EXPLICIT", "NHWC", "HWIO", {0, 0, 1, 0, 1, 2, 0, 0});
+  INFER_OK(op, "[1,3,3,1];[2,2,1,1]", "[d0_0,3,5,d1_3]");
+
+  // 4x4 input, 2x2 filter, 2x2 stride, [3, 2, 1, 0] padding
+  set_op({{1, 2, 2, 1}}, "EXPLICIT", "NHWC", "HWIO", {0, 0, 3, 2, 1, 0, 0, 0});
+  INFER_OK(op, "[1,4,4,2];[2,2,2,3]", "[d0_0,4,2,d1_3]");
+
+  // 2x2 input, 2x1 filter, 1x2 stride, [1, 1, 2, 2] padding
+  set_op({{1, 1, 2, 1}}, "EXPLICIT", "NHWC", "HWIO", {0, 0, 1, 1, 2, 2, 0, 0});
+  INFER_OK(op, "[1,2,2,1];[2,1,1,1]", "[d0_0,3,3,d1_3]");
+
+  // Unknown dims in the critical fields lead to partial inference.
+  INFER_OK(op, "[1,4,4,1];[2,1,1,1]", "[d0_0,5,4,d1_3]");
+  INFER_OK(op, "[1,?,4,1];[2,1,1,1]", "[d0_0,?,4,d1_3]");
+  INFER_OK(op, "[1,4,?,1];[2,1,1,1]", "[d0_0,5,?,d1_3]");
+  INFER_OK(op, "[1,4,4,?];[2,1,1,1]", "[d0_0,5,4,d1_3]");
+  INFER_OK(op, "[1,4,4,1];[?,1,1,1]", "[d0_0,?,4,d1_3]");
+  INFER_OK(op, "[1,4,4,1];[2,?,1,1]", "[d0_0,5,?,d1_3]");
+
+  // Explicit padding errors
+  // Negative padding
+  set_op({{1, 1, 1, 1}}, "EXPLICIT", "NHWC", "HWIO", {0, 0, 0, -1, 0, 0, 0, 0});
+  INFER_ERROR("must be nonnegative", op, "[1,2,2,1];[1,1,1,1]");
+
+  // Too little padding (7 explicit paddings instead of 8)
+  set_op({{1, 1, 1, 1}}, "EXPLICIT", "NHWC", "HWIO", {0, 0, 0, 0, 0, 0, 0});
+  INFER_ERROR("must contain 8 values", op, "[1,2,2,1];[1,1,1,1]");
+
+  // Too much padding (9 explicit paddings instead of 8)
+  set_op({{1, 1, 1, 1}}, "EXPLICIT", "NHWC", "HWIO",
+         {0, 0, 0, 0, 0, 0, 0, 0, 0});
+  INFER_ERROR("must contain 8 values", op, "[1,2,2,1];[1,1,1,1]");
+
+  // Padding in batch dimension
+  set_op({{1, 1, 1, 1}}, "EXPLICIT", "NHWC", "HWIO", {1, 0, 0, 0, 0, 0, 0, 0});
+  INFER_ERROR("batch or depth dimensions", op, "[1,2,2,1];[1,1,1,1]");
+
+  // Padding in depth dimension
+  set_op({{1, 1, 1, 1}}, "EXPLICIT", "NHWC", "HWIO", {0, 0, 0, 0, 0, 0, 1, 0});
+  INFER_ERROR("batch or depth dimensions", op, "[1,2,2,1];[1,1,1,1]");
+
+  // Padding explicit_paddings when padding is not EXPLICIT
+  set_op({{1, 1, 1, 1}}, "VALID", "NHWC", "HWIO", {0, 0, 0, 0, 0, 0, 0, 0});
+  INFER_ERROR("must be empty", op, "[1,2,2,1];[1,1,1,1]");
 }
 
 TEST(CommonShapeFnsTest, Conv2DDilatedShapeTest) {
   ShapeInferenceTestOp op("Conv2D");
   auto set_op = [&op](const std::vector<int32>& dilations,
                       const std::vector<int32>& strides, const string& padding,
-                      const string& data_format) {
+                      const string& data_format,
+                      const std::vector<int32>& explicit_paddings = {}) {
     TF_CHECK_OK(NodeDefBuilder("test", "Conv2D")
                     .Input("input", 0, DT_FLOAT)
                     .Input("filter", 0, DT_FLOAT)
                     .Attr("dilations", dilations)
                     .Attr("strides", strides)
                     .Attr("padding", padding)
+                    .Attr("explicit_paddings", explicit_paddings)
                     .Attr("data_format", data_format)
                     .Finalize(&op.node_def));
   };
@@ -628,6 +684,28 @@ TEST(CommonShapeFnsTest, Conv2DDilatedShapeTest) {
   // 4x4 input, 2x2 filter, 2x2 dilations, 1x1 stride
   set_op({{1, 2, 2, 1}}, {{1, 1, 1, 1}}, "SAME", "NHWC");
   INFER_OK(op, "[1,4,4,1];[2,2,1,1]", "[d0_0,d0_1,d0_2,d1_3]");
+
+  // Some tests for "EXPLICIT" padding
+
+  // 4x4 input, 1x1 filter, 2x1 dilations, 1x1 stride, [0, 2, 1, 4] padding
+  set_op({{1, 2, 1, 1}}, {{1, 1, 1, 1}}, "EXPLICIT", "NHWC",
+         {0, 0, 0, 2, 1, 4, 0, 0});
+  INFER_OK(op, "[1,4,4,1];[1,1,1,1]", "[d0_0,6,9,d1_3]");
+
+  // 3x3 input, 2x2 filter, 2x2 dilations, 1x1 stride, [1, 0, 1, 2] padding
+  set_op({{1, 2, 2, 1}}, {{1, 1, 1, 1}}, "EXPLICIT", "NHWC",
+         {0, 0, 1, 0, 1, 2, 0, 0});
+  INFER_OK(op, "[1,3,3,1];[2,2,1,1]", "[d0_0,2,4,d1_3]");
+
+  // 4x4 input, 2x2 filter, 1x2 dilations, 2x2 stride, [3, 2, 1, 0] padding
+  set_op({{1, 1, 2, 1}}, {{1, 2, 2, 1}}, "EXPLICIT", "NHWC",
+         {0, 0, 3, 2, 1, 0, 0, 0});
+  INFER_OK(op, "[1,4,4,1];[2,2,1,1]", "[d0_0,4,2,d1_3]");
+
+  // 4x4 input, 2x2 filter, 2x2 dilations, 1x1 stride, [1, 1, 2, 2] padding
+  set_op({{1, 2, 2, 1}}, {{1, 1, 1, 1}}, "EXPLICIT", "NHWC",
+         {0, 0, 1, 1, 2, 2, 0, 0});
+  INFER_OK(op, "[1,4,4,1];[2,2,1,1]", "[d0_0,4,6,d1_3]");
 }
 
 TEST(CommonShapeFnsTest, Conv3DShapeTest) {
diff --git a/tensorflow/core/framework/ops_util.cc b/tensorflow/core/framework/ops_util.cc
index e8cf014ca0..4e603b9598 100644
--- a/tensorflow/core/framework/ops_util.cc
+++ b/tensorflow/core/framework/ops_util.cc
@@ -30,6 +30,9 @@ Eigen::PaddingType BrainPadding2EigenPadding(Padding padding) {
       return Eigen::PADDING_VALID;
     case Padding::SAME:
       return Eigen::PADDING_SAME;
+    case Padding::EXPLICIT:
+      LOG(FATAL) << "Eigen does not have explicit padding enum "  // Crash OK
+                    "value";
   }
   return Eigen::PADDING_SAME;  // Prevent compiler warning about missing return
 }
diff --git a/tensorflow/core/grappler/costs/graph_properties_testdata/large_function_graph.pbtxt b/tensorflow/core/grappler/costs/graph_properties_testdata/large_function_graph.pbtxt
index 415c347a1d..d4e23e901a 100644
--- a/tensorflow/core/grappler/costs/graph_properties_testdata/large_function_graph.pbtxt
+++ b/tensorflow/core/grappler/costs/graph_properties_testdata/large_function_graph.pbtxt
@@ -511,6 +511,13 @@ library {
           s: "VALID"
         }
       }
+      attr {
+        key: "explicit_paddings"
+        value {
+          list {
+          }
+        }
+      }
       attr {
         key: "strides"
         value {
diff --git a/tensorflow/core/grappler/optimizers/layout_optimizer.cc b/tensorflow/core/grappler/optimizers/layout_optimizer.cc
index 8f25a1c8c1..e9b706a583 100644
--- a/tensorflow/core/grappler/optimizers/layout_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/layout_optimizer.cc
@@ -503,6 +503,7 @@ class NodeProcessor : public GraphProcessor {
       UpdateAttrKSize();
       UpdateAttrStrides();
       UpdateAttrDilations();
+      UpdateAttrExplicitPaddings();
       UpdateAttrShape();
       TF_RETURN_IF_ERROR(AddLayoutTransposeToInputs());
       TF_RETURN_IF_ERROR(AddLayoutTransposeToOutputs());
@@ -753,6 +754,28 @@ class NodeProcessor : public GraphProcessor {
     }
   }
 
+  void UpdateAttrExplicitPaddings() {
+    if (node_->attr().find("explicit_paddings") != node_->attr().end()) {
+      auto list = node_->mutable_attr()->at("explicit_paddings").mutable_list();
+      int size = list->i_size();
+      if (size == 8) {
+        int64 height_before = list->i(2);
+        int64 height_after = list->i(3);
+        int64 width_before = list->i(4);
+        int64 width_after = list->i(5);
+        list->set_i(2, 0);
+        list->set_i(3, 0);
+        list->set_i(4, height_before);
+        list->set_i(5, height_after);
+        list->set_i(6, width_before);
+        list->set_i(7, width_after);
+      } else if (size != 0) {
+        LOG(ERROR) << "Cannot handle explicit_paddings attribute of size "
+                   << size;
+      }
+    }
+  }
+
   void UpdateAttrDataFormat() {
     if (node_->attr().find("data_format") != node_->attr().end()) {
       if (node_->attr().at("data_format").s().compare("NHWC") == 0) {
diff --git a/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc b/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc
index 20e47c1b26..eb2a8e87dd 100644
--- a/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/layout_optimizer.h"
 #include "tensorflow/cc/ops/standard_ops.h"
+#include "tensorflow/core/framework/common_shape_fns.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/tensor_testutil.h"
 #include "tensorflow/core/grappler/clusters/single_machine.h"
@@ -80,8 +81,13 @@ class LayoutOptimizerTest : public GrapplerTest {
     Output filter =
         ops::Const(s->WithOpName("Filter"), Input::Initializer(filter_data));
 
+    ops::Conv2D::Attrs attrs;
+    if (padding == "EXPLICIT") {
+      attrs = attrs.ExplicitPaddings({0, 0, 1, 2, 3, 4, 0, 0});
+    }
+
     Output conv = ops::Conv2D(s->WithOpName("Conv2D").WithDevice(device), input,
-                              filter, {1, stride, stride, 1}, padding);
+                              filter, {1, stride, stride, 1}, padding, attrs);
     return conv;
   }
 
@@ -100,6 +106,28 @@ class LayoutOptimizerTest : public GrapplerTest {
     int input_depth = 3;
     int filter_count = 2;
     int stride = 1;
+    int dilation = dilated ? 2 : 1;
+    int64 padding_top = 1;
+    int64 padding_bottom = 2;
+    int64 padding_left = 3;
+    int64 padding_right = 4;
+    int64 output_height;
+    int64 output_width;
+    Padding padding_enum;
+    if (padding == "SAME") {
+      padding_enum = SAME;
+    } else if (padding == "VALID") {
+      padding_enum = VALID;
+    } else {
+      CHECK_EQ(padding, "EXPLICIT");
+      padding_enum = EXPLICIT;
+    }
+    TF_CHECK_OK(GetWindowedOutputSizeVerboseV2(
+        input_height, filter_size, dilation, stride, padding_enum,
+        &output_height, &padding_top, &padding_bottom));
+    TF_CHECK_OK(GetWindowedOutputSizeVerboseV2(
+        input_width, filter_size, dilation, stride, padding_enum, &output_width,
+        &padding_left, &padding_right));
     TensorShape input_sizes_shape({4});
     Tensor input_data(DT_INT32, input_sizes_shape);
     test::FillValues<int>(&input_data,
@@ -112,8 +140,6 @@ class LayoutOptimizerTest : public GrapplerTest {
     Output filter =
         ops::Variable(s->WithOpName("Filter"), filter_shape, DT_FLOAT);
 
-    int output_height = input_height;
-    int output_width = input_width;
     TensorShape output_shape(
         {batch_size, output_height, output_width, filter_count});
     Tensor output_data(DT_FLOAT, output_shape);
@@ -124,10 +150,21 @@ class LayoutOptimizerTest : public GrapplerTest {
     Output conv_backprop_input;
     Output input_sizes_i =
         ops::Identity(s->WithOpName("InputSizesIdentity"), input_sizes);
-    ops::Conv2DBackpropInput::Attrs attrs;
-    if (dilated) {
-      attrs = attrs.Dilations({1, 2, 2, 1});
+    std::vector<int> dilations{1, dilation, dilation, 1};
+    std::vector<int> explicit_paddings;
+    if (padding == "EXPLICIT") {
+      explicit_paddings = {0,
+                           0,
+                           static_cast<int>(padding_top),
+                           static_cast<int>(padding_bottom),
+                           static_cast<int>(padding_left),
+                           static_cast<int>(padding_right),
+                           0,
+                           0};
     }
+    auto attrs =
+        ops::Conv2DBackpropInput::Attrs().Dilations(dilations).ExplicitPaddings(
+            explicit_paddings);
     if (const_input_size) {
       conv_backprop_input = ops::Conv2DBackpropInput(
           s->WithOpName("Conv2DBackpropInput"), input_sizes, filter, output,
@@ -186,7 +223,7 @@ class LayoutOptimizerTest : public GrapplerTest {
 
 TEST_F(LayoutOptimizerTest, Conv2DBackpropInput) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
-  auto conv = SimpleConv2DBackpropInput(&s, 7, 2, "SAME");
+  auto conv = SimpleConv2DBackpropInput(&s, 7, 2, "EXPLICIT");
   Output fetch = ops::Identity(s.WithOpName("Fetch"), {conv});
   GrapplerItem item;
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
@@ -306,6 +343,19 @@ TEST_F(LayoutOptimizerTest, NotEqualSizeWithValidPadding) {
   EXPECT_TRUE(node_map.GetNode("Conv2D-0-TransposeNHWCToNCHW-LayoutOptimizer"));
 }
 
+TEST_F(LayoutOptimizerTest, ExplicitPadding) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+  auto conv = SimpleConv2D(&s, 4, 2, "EXPLICIT");
+  Output fetch = ops::Identity(s.WithOpName("Fetch"), {conv});
+  GrapplerItem item;
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+  LayoutOptimizer optimizer;
+  GraphDef output;
+  Status status = optimizer.Optimize(virtual_cluster_.get(), item, &output);
+  NodeMap node_map(&output);
+  EXPECT_TRUE(node_map.GetNode("Conv2D-0-TransposeNHWCToNCHW-LayoutOptimizer"));
+}
+
 TEST_F(LayoutOptimizerTest, Pad) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   auto conv = SimpleConv2D(&s, 4, 2, "VALID");
diff --git a/tensorflow/core/kernels/conv_grad_filter_ops.cc b/tensorflow/core/kernels/conv_grad_filter_ops.cc
index 4e3de33e83..0df05ceb02 100644
--- a/tensorflow/core/kernels/conv_grad_filter_ops.cc
+++ b/tensorflow/core/kernels/conv_grad_filter_ops.cc
@@ -102,6 +102,7 @@ struct LaunchConv2DBackpropFilterOp<CPUDevice, T> {
                   const Tensor& out_backprop, const Tensor& input,
                   int row_dilation, int col_dilation, int row_stride,
                   int col_stride, const Padding& padding,
+                  const std::vector<int64>& explicit_paddings,
                   Tensor* filter_backprop, TensorFormat data_format) {
     const CPUDevice& d = ctx->eigen_device<CPUDevice>();
     functor::SpatialConvolutionBackwardFilter<CPUDevice, T>()(
@@ -204,6 +205,15 @@ class Conv2DFastBackpropFilterOp : public OpKernel {
                 errors::InvalidArgument(
                     "Row and column strides should be larger than 0."));
     OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
+    OP_REQUIRES(
+        context, padding_ != Padding::EXPLICIT,
+        errors::Unimplemented("Current CPU implementation does not support "
+                              "EXPLICIT padding yet."));
+    std::vector<int64> explicit_paddings;
+    OP_REQUIRES_OK(context,
+                   context->GetAttr("explicit_paddings", &explicit_paddings));
+    OP_REQUIRES_OK(context, CheckValidPadding(padding_, explicit_paddings,
+                                              /*num_dims=*/4, data_format_));
     OP_REQUIRES_OK(context, context->GetAttr("dilations", &dilations_));
     OP_REQUIRES(context, dilations_.size() == 4,
                 errors::InvalidArgument("Sliding window dilations field must "
@@ -282,7 +292,8 @@ class Conv2DFastBackpropFilterOp : public OpKernel {
     LaunchConv2DBackpropFilterOp<Device, T>()(
         context, false, false, out_backprop, input,
         /*row_dilation=*/1, /*col_dilation=*/1, dims.spatial_dims[0].stride,
-        dims.spatial_dims[1].stride, padding_, filter_backprop, data_format_);
+        dims.spatial_dims[1].stride, padding_, /*explicit_paddings=*/{},
+        filter_backprop, data_format_);
   }
 
  private:
@@ -319,6 +330,15 @@ class Conv2DCustomBackpropFilterOp : public OpKernel {
                 errors::InvalidArgument(
                     "Row and column strides should be larger than 0."));
     OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
+    OP_REQUIRES(
+        context, padding_ != Padding::EXPLICIT,
+        errors::Unimplemented("Current CPU implementation does not support "
+                              "EXPLICIT padding yet."));
+    std::vector<int64> explicit_paddings;
+    OP_REQUIRES_OK(context,
+                   context->GetAttr("explicit_paddings", &explicit_paddings));
+    OP_REQUIRES_OK(context, CheckValidPadding(padding_, explicit_paddings,
+                                              /*num_dims=*/4, data_format_));
     OP_REQUIRES_OK(context, context->GetAttr("dilations", &dilations_));
     OP_REQUIRES(context, dilations_.size() == 4,
                 errors::InvalidArgument("Sliding window dilations field must "
@@ -587,6 +607,10 @@ class Conv2DSlowBackpropFilterOp : public OpKernel {
     use_cudnn_ &= CanUseCudnn();
     cudnn_use_autotune_ = CudnnUseAutotune();
     OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
+    OP_REQUIRES_OK(context,
+                   context->GetAttr("explicit_paddings", &explicit_paddings_));
+    OP_REQUIRES_OK(context, CheckValidPadding(padding_, explicit_paddings_,
+                                              /*num_dims=*/4, data_format_));
   }
 
   void Compute(OpKernelContext* context) override {
@@ -626,13 +650,14 @@ class Conv2DSlowBackpropFilterOp : public OpKernel {
 
     launcher_(context, use_cudnn_, cudnn_use_autotune_, out_backprop, input,
               dilation_rows, dilation_cols, stride_rows, stride_cols, padding_,
-              filter_backprop, data_format_);
+              explicit_paddings_, filter_backprop, data_format_);
   }
 
  private:
   std::vector<int32> dilations_;
   std::vector<int32> strides_;
   Padding padding_;
+  std::vector<int64> explicit_paddings_;
   bool use_cudnn_;
   TensorFormat data_format_;
   LaunchConv2DBackpropFilterOp<Device, T> launcher_;
@@ -646,7 +671,8 @@ void LaunchConv2DBackpropFilterOp<Eigen::GpuDevice, T>::operator()(
     OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
     const Tensor& out_backprop, const Tensor& input, int row_dilation,
     int col_dilation, int row_stride, int col_stride, const Padding& padding,
-    Tensor* filter_backprop, TensorFormat data_format) {
+    const std::vector<int64>& explicit_paddings, Tensor* filter_backprop,
+    TensorFormat data_format) {
   using se::dnn::AlgorithmConfig;
   using se::dnn::AlgorithmDesc;
   using se::dnn::ProfileResult;
@@ -661,35 +687,33 @@ void LaunchConv2DBackpropFilterOp<Eigen::GpuDevice, T>::operator()(
   TensorShape filter_shape = filter_backprop->shape();
 
   ConvBackpropDimensions dims;
-  OP_REQUIRES_OK(ctx, ConvBackpropComputeDimensionsV2(
-                          "Conv2DSlowBackpropFilter", /*num_spatial_dims=*/2,
-                          input.shape(), filter_shape, out_backprop.shape(),
-                          dilations, strides, padding, data_format, &dims));
-
-  // TODO(yangzihao): The padding computations should be done in
-  // GetWindowedOutputSize() functions.
-  const int padding_rows =
-      (padding == VALID)
-          ? 0
-          : std::max<int>(0, (dims.spatial_dims[0].output_size - 1) *
-                                     dims.spatial_dims[0].stride +
-                                 (dims.spatial_dims[0].filter_size - 1) *
-                                     dims.spatial_dims[0].dilation +
-                                 1 - dims.spatial_dims[0].input_size);
-  const int padding_cols =
-      (padding == VALID)
-          ? 0
-          : std::max<int>(0, (dims.spatial_dims[1].output_size - 1) *
-                                     dims.spatial_dims[1].stride +
-                                 (dims.spatial_dims[1].filter_size - 1) *
-                                     dims.spatial_dims[1].dilation +
-                                 1 - dims.spatial_dims[1].input_size);
-
-  // TODO(zhengxq): cuDNN only supports equal padding on both sides, so only
-  // calling it when that is true. Remove this check when (if?) cuDNN starts
-  // supporting different padding.
-  bool rows_odd = (padding_rows % 2 != 0);
-  bool cols_odd = (padding_cols % 2 != 0);
+  OP_REQUIRES_OK(
+      ctx, ConvBackpropComputeDimensionsV2(
+               "Conv2DSlowBackpropFilter", /*num_spatial_dims=*/2,
+               input.shape(), filter_shape, out_backprop.shape(), dilations,
+               strides, padding, explicit_paddings, data_format, &dims));
+
+  int64 padding_top = -1, padding_bottom = -1;
+  int64 padding_left = -1, padding_right = -1;
+  if (padding == EXPLICIT) {
+    GetExplicitPaddingForDim(explicit_paddings, data_format, 'H', &padding_top,
+                             &padding_bottom);
+    GetExplicitPaddingForDim(explicit_paddings, data_format, 'W', &padding_left,
+                             &padding_right);
+  }
+  int64 expected_out_rows, expected_out_cols;
+  // The function is guaranteed to succeed because we checked the output and
+  // padding was valid earlier.
+  TF_CHECK_OK(GetWindowedOutputSizeVerboseV2(
+      dims.spatial_dims[0].input_size, dims.spatial_dims[0].filter_size,
+      row_dilation, row_stride, padding, &expected_out_rows, &padding_top,
+      &padding_bottom));
+  DCHECK_EQ(dims.spatial_dims[0].output_size, expected_out_rows);
+  TF_CHECK_OK(GetWindowedOutputSizeVerboseV2(
+      dims.spatial_dims[1].input_size, dims.spatial_dims[1].filter_size,
+      col_dilation, col_stride, padding, &expected_out_cols, &padding_left,
+      &padding_right));
+  DCHECK_EQ(dims.spatial_dims[1].output_size, expected_out_cols);
 
   auto* stream = ctx->op_device_context()->stream();
   OP_REQUIRES(ctx, stream, errors::Internal("No GPU stream available."));
@@ -711,7 +735,7 @@ void LaunchConv2DBackpropFilterOp<Eigen::GpuDevice, T>::operator()(
       dims.spatial_dims[0].filter_size == 1 &&
       dims.spatial_dims[1].filter_size == 1 && !is_grouped_convolution &&
       dims.spatial_dims[0].stride == 1 && dims.spatial_dims[1].stride == 1 &&
-      data_format == FORMAT_NHWC) {
+      data_format == FORMAT_NHWC && (padding == VALID || padding == SAME)) {
     const uint64 m = dims.in_depth;
     const uint64 k = dims.batch_size * dims.spatial_dims[0].input_size *
                      dims.spatial_dims[1].input_size;
@@ -779,31 +803,43 @@ void LaunchConv2DBackpropFilterOp<Eigen::GpuDevice, T>::operator()(
     return;
   }
 
+  const int64 common_padding_rows = std::min(padding_top, padding_bottom);
+  const int64 common_padding_cols = std::min(padding_left, padding_right);
   Tensor compatible_input;
-  if (rows_odd || cols_odd) {
-    // If a padding dimension is odd, we have one more element on the right
-    // side or the bottom side. This is unsupported in cudnn. Therefore,
-    // we pad that extra element and make it compatible.
+  if (padding_top != padding_bottom || padding_left != padding_right) {
+    // Pad the input in the same way we did during the forward pass, so that
+    // cuDNN receives the same input during the backward pass function as it did
+    // during the forward pass function.
+    const int64 padding_rows_diff = std::abs(padding_bottom - padding_top);
+    const int64 padding_cols_diff = std::abs(padding_right - padding_left);
+    const int64 new_in_rows =
+        dims.spatial_dims[0].input_size + padding_rows_diff;
+    const int64 new_in_cols =
+        dims.spatial_dims[1].input_size + padding_cols_diff;
+    const int64 input_pad_top = padding_top - common_padding_rows;
+    const int64 input_pad_bottom = padding_bottom - common_padding_rows;
+    const int64 input_pad_left = padding_left - common_padding_cols;
+    const int64 input_pad_right = padding_right - common_padding_cols;
     OP_REQUIRES_OK(
         ctx, ctx->allocate_temp(
                  DataTypeToEnum<T>::value,
-                 ShapeFromFormat(data_format, dims.batch_size,
-                                 dims.spatial_dims[0].input_size + rows_odd,
-                                 dims.spatial_dims[1].input_size + cols_odd,
-                                 dims.in_depth),
+                 ShapeFromFormat(data_format, dims.batch_size, new_in_rows,
+                                 new_in_cols, dims.in_depth),
                  &compatible_input));
 
     functor::PadInput<GPUDevice, T, int, 4>()(
         ctx->template eigen_device<GPUDevice>(), To32Bit(input.tensor<T, 4>()),
-        {{0, 0}}, {{rows_odd, cols_odd}},
+        {{static_cast<int>(input_pad_top), static_cast<int>(input_pad_left)}},
+        {{static_cast<int>(input_pad_bottom),
+          static_cast<int>(input_pad_right)}},
         To32Bit(compatible_input.tensor<T, 4>()), data_format);
   } else {
     compatible_input = input;
   }
 
-  CHECK(padding_rows >= 0 && padding_cols >= 0)
-      << "Negative row or col paddings: (" << padding_rows << ", "
-      << padding_cols << ")";
+  CHECK(common_padding_rows >= 0 && common_padding_cols >= 0)  // Crash OK
+      << "Negative row or col paddings: (" << common_padding_rows << ", "
+      << common_padding_cols << ")";
   se::dnn::BatchDescriptor input_desc;
   input_desc.set_count(dims.batch_size)
       .set_height(GetTensorDim(compatible_input, data_format, 'H'))
@@ -826,8 +862,8 @@ void LaunchConv2DBackpropFilterOp<Eigen::GpuDevice, T>::operator()(
       .set_horizontal_dilation_rate(dims.spatial_dims[1].dilation)
       .set_vertical_filter_stride(dims.spatial_dims[0].stride)
       .set_horizontal_filter_stride(dims.spatial_dims[1].stride)
-      .set_zero_padding_height(padding_rows / 2)
-      .set_zero_padding_width(padding_cols / 2)
+      .set_zero_padding_height(common_padding_rows)
+      .set_zero_padding_width(common_padding_cols)
       .set_group_count(dims.in_depth / filter_shape.dim_size(2));
 
   // NOTE(zhengxq):
@@ -922,8 +958,8 @@ void LaunchConv2DBackpropFilterOp<Eigen::GpuDevice, T>::operator()(
         dims.spatial_dims[1].dilation}},   // dilation_cols
       {{dims.spatial_dims[0].stride,       // stride_rows
         dims.spatial_dims[1].stride}},     // stride_cols
-      {{padding_rows,                      // padding_rows
-        padding_cols}},                    // padding_cols
+      {{common_padding_rows,               // padding_rows
+        common_padding_cols}},             // padding_cols
       dtype,                               // tensor datatype
       device_id,                           // device_id
   };
diff --git a/tensorflow/core/kernels/conv_grad_input_ops.cc b/tensorflow/core/kernels/conv_grad_input_ops.cc
index 9f983ed816..74b97b9864 100644
--- a/tensorflow/core/kernels/conv_grad_input_ops.cc
+++ b/tensorflow/core/kernels/conv_grad_input_ops.cc
@@ -106,8 +106,9 @@ struct LaunchConv2DBackpropInputOp<CPUDevice, T> {
   void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
                   const Tensor& out_backprop, const Tensor& filter,
                   int row_dilation, int col_dilation, int row_stride,
-                  int col_stride, const Padding& padding, Tensor* in_backprop,
-                  TensorFormat data_format) {
+                  int col_stride, const Padding& padding,
+                  const std::vector<int64>& explicit_paddings,
+                  Tensor* in_backprop, TensorFormat data_format) {
     const CPUDevice& d = ctx->eigen_device<CPUDevice>();
     functor::SpatialConvolutionBackwardInput<CPUDevice, T>()(
         d, in_backprop->tensor<T, 4>(), filter.tensor<T, 4>(),
@@ -220,6 +221,15 @@ class Conv2DFastBackpropInputOp : public OpKernel {
                 errors::InvalidArgument(
                     "Current Eigen and libxsmm implementations do not "
                     "yet support dilation rates larger than 1."));
+    OP_REQUIRES(
+        context, padding_ != Padding::EXPLICIT,
+        errors::Unimplemented("Current CPU implementation does not support "
+                              "EXPLICIT padding yet."));
+    std::vector<int64> explicit_paddings;
+    OP_REQUIRES_OK(context,
+                   context->GetAttr("explicit_paddings", &explicit_paddings));
+    OP_REQUIRES_OK(context, CheckValidPadding(padding_, explicit_paddings,
+                                              /*num_dims=*/4, data_format_));
   }
 
   void Compute(OpKernelContext* context) override {
@@ -286,7 +296,8 @@ class Conv2DFastBackpropInputOp : public OpKernel {
     LaunchConv2DBackpropInputOp<Device, T>()(
         context, false, false, out_backprop, filter,
         /*row_dilation=*/1, /*col_dilation=*/1, dims.spatial_dims[0].stride,
-        dims.spatial_dims[1].stride, padding_, in_backprop, data_format_);
+        dims.spatial_dims[1].stride, padding_, /*explicit_paddings=*/{},
+        in_backprop, data_format_);
   }
 
  private:
@@ -336,6 +347,15 @@ class Conv2DCustomBackpropInputOp : public OpKernel {
                 errors::InvalidArgument(
                     "Current libxsmm and customized CPU implementations do "
                     "not yet support dilation rates larger than 1."));
+    OP_REQUIRES(
+        context, padding_ != Padding::EXPLICIT,
+        errors::Unimplemented("Current CPU implementation does not support "
+                              "EXPLICIT padding yet."));
+    std::vector<int64> explicit_paddings;
+    OP_REQUIRES_OK(context,
+                   context->GetAttr("explicit_paddings", &explicit_paddings));
+    OP_REQUIRES_OK(context, CheckValidPadding(padding_, explicit_paddings,
+                                              /*num_dims=*/4, data_format_));
   }
 
   void Compute(OpKernelContext* context) override {
@@ -661,6 +681,16 @@ class Conv2DSlowBackpropInputOp : public OpKernel {
     use_cudnn_ &= CanUseCudnn();
     cudnn_use_autotune_ = CudnnUseAutotune();
     OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
+    if (!std::is_same<Device, GPUDevice>::value) {
+      OP_REQUIRES(
+          context, padding_ != Padding::EXPLICIT,
+          errors::Unimplemented("Current CPU implementation does not support "
+                                "EXPLICIT padding yet."));
+    }
+    OP_REQUIRES_OK(context,
+                   context->GetAttr("explicit_paddings", &explicit_paddings_));
+    OP_REQUIRES_OK(context, CheckValidPadding(padding_, explicit_paddings_,
+                                              /*num_dims=*/4, data_format_));
   }
 
   void Compute(OpKernelContext* context) override {
@@ -694,13 +724,14 @@ class Conv2DSlowBackpropInputOp : public OpKernel {
 
     launcher_(context, use_cudnn_, cudnn_use_autotune_, out_backprop, filter,
               dilation_rows, dilation_cols, stride_rows, stride_cols, padding_,
-              in_backprop, data_format_);
+              explicit_paddings_, in_backprop, data_format_);
   }
 
  private:
   std::vector<int32> dilations_;
   std::vector<int32> strides_;
   Padding padding_;
+  std::vector<int64> explicit_paddings_;
   bool use_cudnn_;
   TensorFormat data_format_;
   LaunchConv2DBackpropInputOp<Device, T> launcher_;
@@ -714,7 +745,8 @@ void LaunchConv2DBackpropInputOp<GPUDevice, T>::operator()(
     OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
     const Tensor& out_backprop, const Tensor& filter, int row_dilation,
     int col_dilation, int row_stride, int col_stride, const Padding& padding,
-    Tensor* in_backprop, TensorFormat data_format) {
+    const std::vector<int64>& explicit_paddings, Tensor* in_backprop,
+    TensorFormat data_format) {
   using se::dnn::AlgorithmConfig;
   using se::dnn::AlgorithmDesc;
   using se::dnn::ProfileResult;
@@ -731,35 +763,33 @@ void LaunchConv2DBackpropInputOp<GPUDevice, T>::operator()(
 
   const TensorShape& filter_shape = filter.shape();
   ConvBackpropDimensions dims;
-  OP_REQUIRES_OK(ctx, ConvBackpropComputeDimensionsV2(
-                          "Conv2DSlowBackpropInput", /*num_spatial_dims=*/2,
-                          input_shape, filter_shape, out_backprop.shape(),
-                          dilations, strides, padding, data_format, &dims));
-
-  // TODO(yangzihao): The padding computations should be done in
-  // GetWindowedOutputSize() functions.
-  const int padding_rows =
-      (padding == VALID)
-          ? 0
-          : std::max<int>(0, (dims.spatial_dims[0].output_size - 1) *
-                                     dims.spatial_dims[0].stride +
-                                 (dims.spatial_dims[0].filter_size - 1) *
-                                     dims.spatial_dims[0].dilation +
-                                 1 - dims.spatial_dims[0].input_size);
-  const int padding_cols =
-      (padding == VALID)
-          ? 0
-          : std::max<int>(0, (dims.spatial_dims[1].output_size - 1) *
-                                     dims.spatial_dims[1].stride +
-                                 (dims.spatial_dims[1].filter_size - 1) *
-                                     dims.spatial_dims[1].dilation +
-                                 1 - dims.spatial_dims[1].input_size);
-
-  // TODO(keveman): cuDNN only supports equal padding on both sides, so only
-  // calling it when that is true. Remove this check when (if?) cuDNN starts
-  // supporting different padding.
-  bool rows_odd = (padding_rows % 2 != 0);
-  bool cols_odd = (padding_cols % 2 != 0);
+  OP_REQUIRES_OK(
+      ctx, ConvBackpropComputeDimensionsV2(
+               "Conv2DSlowBackpropInput", /*num_spatial_dims=*/2, input_shape,
+               filter_shape, out_backprop.shape(), dilations, strides, padding,
+               explicit_paddings, data_format, &dims));
+
+  int64 padding_top = -1, padding_bottom = -1;
+  int64 padding_left = -1, padding_right = -1;
+  if (padding == EXPLICIT) {
+    GetExplicitPaddingForDim(explicit_paddings, data_format, 'H', &padding_top,
+                             &padding_bottom);
+    GetExplicitPaddingForDim(explicit_paddings, data_format, 'W', &padding_left,
+                             &padding_right);
+  }
+  int64 expected_out_rows, expected_out_cols;
+  // The function is guaranteed to succeed because we checked the output and
+  // padding was valid earlier.
+  TF_CHECK_OK(GetWindowedOutputSizeVerboseV2(
+      dims.spatial_dims[0].input_size, dims.spatial_dims[0].filter_size,
+      row_dilation, row_stride, padding, &expected_out_rows, &padding_top,
+      &padding_bottom));
+  DCHECK_EQ(dims.spatial_dims[0].output_size, expected_out_rows);
+  TF_CHECK_OK(GetWindowedOutputSizeVerboseV2(
+      dims.spatial_dims[1].input_size, dims.spatial_dims[1].filter_size,
+      col_dilation, col_stride, padding, &expected_out_cols, &padding_left,
+      &padding_right));
+  DCHECK_EQ(dims.spatial_dims[1].output_size, expected_out_cols);
 
   auto* stream = ctx->op_device_context()->stream();
   OP_REQUIRES(ctx, stream, errors::Internal("No GPU stream available."));
@@ -779,7 +809,7 @@ void LaunchConv2DBackpropInputOp<GPUDevice, T>::operator()(
   if (dims.spatial_dims[0].filter_size == 1 &&
       dims.spatial_dims[1].filter_size == 1 && !is_grouped_convolution &&
       dims.spatial_dims[0].stride == 1 && dims.spatial_dims[1].stride == 1 &&
-      data_format == FORMAT_NHWC) {
+      data_format == FORMAT_NHWC && (padding == VALID || padding == SAME)) {
     // 1x1 filter, so call cublas directly.
     const uint64 m = dims.batch_size * dims.spatial_dims[0].input_size *
                      dims.spatial_dims[1].input_size;
@@ -841,22 +871,28 @@ void LaunchConv2DBackpropInputOp<GPUDevice, T>::operator()(
     return;
   }
 
+  const int64 common_padding_rows = std::min(padding_top, padding_bottom);
+  const int64 common_padding_cols = std::min(padding_left, padding_right);
   TensorShape compatible_input_shape;
-  if (rows_odd || cols_odd) {
-    // If a padding dimension is odd, we have one more element on the right
-    // side or the bottom side. This is unsupported in cudnn. Therefore,
-    // we pad that extra element and make it compatible.
+  if (padding_top != padding_bottom || padding_left != padding_right) {
+    // Pad the input in the same way we did during the forward pass, so that
+    // cuDNN receives the same input during the backward pass function as it did
+    // during the forward pass function.
+    const int64 padding_rows_diff = std::abs(padding_bottom - padding_top);
+    const int64 padding_cols_diff = std::abs(padding_right - padding_left);
+    const int64 new_in_rows =
+        dims.spatial_dims[0].input_size + padding_rows_diff;
+    const int64 new_in_cols =
+        dims.spatial_dims[1].input_size + padding_cols_diff;
     compatible_input_shape = ShapeFromFormat(
-        data_format, dims.batch_size,
-        dims.spatial_dims[0].input_size + rows_odd,
-        dims.spatial_dims[1].input_size + cols_odd, dims.in_depth);
+        data_format, dims.batch_size, new_in_rows, new_in_cols, dims.in_depth);
   } else {
     compatible_input_shape = input_shape;
   }
 
-  CHECK(padding_rows >= 0 && padding_cols >= 0)
-      << "Negative row or col paddings: (" << padding_rows << ", "
-      << padding_cols << ")";
+  CHECK(common_padding_rows >= 0 && common_padding_cols >= 0)  // Crash OK
+      << "Negative row or col paddings: (" << common_padding_rows << ", "
+      << common_padding_cols << ")";
   se::dnn::BatchDescriptor input_desc;
   input_desc.set_count(dims.batch_size)
       .set_height(GetTensorDim(compatible_input_shape, data_format, 'H'))
@@ -879,8 +915,8 @@ void LaunchConv2DBackpropInputOp<GPUDevice, T>::operator()(
       .set_horizontal_dilation_rate(dims.spatial_dims[1].dilation)
       .set_vertical_filter_stride(dims.spatial_dims[0].stride)
       .set_horizontal_filter_stride(dims.spatial_dims[1].stride)
-      .set_zero_padding_height(padding_rows / 2)
-      .set_zero_padding_width(padding_cols / 2)
+      .set_zero_padding_height(common_padding_rows)
+      .set_zero_padding_width(common_padding_cols)
       .set_group_count(dims.in_depth / filter_shape.dim_size(2));
 
   // NOTE(keveman):
@@ -971,8 +1007,8 @@ void LaunchConv2DBackpropInputOp<GPUDevice, T>::operator()(
         dims.spatial_dims[1].dilation}},   // dilation_cols
       {{dims.spatial_dims[0].stride,       // stride_rows
         dims.spatial_dims[1].stride}},     // stride_cols
-      {{padding_rows,                      // padding_rows
-        padding_cols}},                    // padding_cols
+      {{common_padding_rows,               // padding_rows
+        common_padding_cols}},             // padding_cols
       dtype,                               // tensor data type
       device_id,                           // device_id
   };
@@ -1041,7 +1077,7 @@ void LaunchConv2DBackpropInputOp<GPUDevice, T>::operator()(
     return;
   }
 
-  if (rows_odd || cols_odd) {
+  if (padding_top != padding_bottom || padding_left != padding_right) {
     Tensor in_backprop_remove_padding;
     OP_REQUIRES_OK(
         ctx, ctx->allocate_temp(
@@ -1053,12 +1089,18 @@ void LaunchConv2DBackpropInputOp<GPUDevice, T>::operator()(
                                  GetTensorDim(input_shape, data_format, 'C')),
                  &in_backprop_remove_padding));
 
-    // Remove the padding for odd rows or cols.
+    // Remove the padding that was added to the input shape above.
+    const int64 input_pad_top = padding_top - common_padding_rows;
+    const int64 input_pad_bottom = padding_bottom - common_padding_rows;
+    const int64 input_pad_left = padding_left - common_padding_cols;
+    const int64 input_pad_right = padding_right - common_padding_cols;
     functor::PadInput<GPUDevice, T, int, 4>()(
         ctx->template eigen_device<GPUDevice>(),
         To32Bit(const_cast<const Tensor&>(pre_transformed_in_backprop)
                     .tensor<T, 4>()),
-        {{0, 0}}, {{-rows_odd, -cols_odd}},
+        {{static_cast<int>(-input_pad_top), static_cast<int>(-input_pad_left)}},
+        {{static_cast<int>(-input_pad_bottom),
+          static_cast<int>(-input_pad_right)}},
         To32Bit(in_backprop_remove_padding.tensor<T, 4>()), FORMAT_NCHW);
 
     pre_transformed_in_backprop = in_backprop_remove_padding;
diff --git a/tensorflow/core/kernels/conv_grad_ops.cc b/tensorflow/core/kernels/conv_grad_ops.cc
index 507720c998..0fd7550830 100644
--- a/tensorflow/core/kernels/conv_grad_ops.cc
+++ b/tensorflow/core/kernels/conv_grad_ops.cc
@@ -52,24 +52,23 @@ int ConvBackpropDimensions::SpatialPadding(const Padding& padding,
                                        1 - input_size(dim)));
 }
 
-// The V2 version computes windowed output size with arbitrary dilation_rate,
-// while the original version only handles the cases where dilation_rates equal
-// to 1.
-Status ConvBackpropExtractAndVerifyDimensionV2(
+namespace {
+
+Status ConvBackpropExtractAndVerifyDimension(
     StringPiece label, const TensorShape& input_shape,
     const TensorShape& filter_shape, const TensorShape& output_shape,
     const gtl::ArraySlice<int32>& dilations, const std::vector<int32>& strides,
-    Padding padding, int spatial_dim, int filter_spatial_dim,
-    ConvBackpropSpatialDimension* dim) {
+    Padding padding, int64 padding_before, int64 padding_after, int spatial_dim,
+    int filter_spatial_dim, ConvBackpropSpatialDimension* dim) {
   dim->input_size = input_shape.dim_size(spatial_dim);
   dim->filter_size = filter_shape.dim_size(filter_spatial_dim);
   dim->output_size = output_shape.dim_size(spatial_dim);
   dim->stride = strides[spatial_dim];
   dim->dilation = dilations[spatial_dim];
-  int64 out_size = 0, pad_size = 0;
-  TF_RETURN_IF_ERROR(GetWindowedOutputSizeV2(dim->input_size, dim->filter_size,
-                                             dim->dilation, dim->stride,
-                                             padding, &out_size, &pad_size));
+  int64 out_size = 0;
+  TF_RETURN_IF_ERROR(GetWindowedOutputSizeVerboseV2(
+      dim->input_size, dim->filter_size, dim->dilation, dim->stride, padding,
+      &out_size, &padding_before, &padding_after));
   if (dim->output_size != out_size) {
     return errors::InvalidArgument(
         label, ": Size of out_backprop doesn't match computed: ", "actual = ",
@@ -79,10 +78,13 @@ Status ConvBackpropExtractAndVerifyDimensionV2(
         " stride: ", dim->stride, " dilation: ", dim->dilation);
   }
 
+  // TODO(reedwm): Correctly handle explicit padding here. The rest of the
+  // fields set on 'dim' are only used in XLA. TensorFlow ops do not yet support
+  // explicit padding for XLA.
   int64 effective_filter_size = (dim->filter_size - 1) * dim->dilation + 1;
   dim->expanded_output_size = (dim->output_size - 1) * dim->stride + 1;
   const auto padded_out_size = dim->input_size + effective_filter_size - 1;
-  dim->pad_before = effective_filter_size - 1 - pad_size;
+  dim->pad_before = effective_filter_size - 1 - padding_before;
   dim->pad_after =
       padded_out_size - dim->expanded_output_size - dim->pad_before;
   VLOG(2) << label << ": expanded_out = " << dim->expanded_output_size
@@ -94,22 +96,14 @@ Status ConvBackpropExtractAndVerifyDimensionV2(
   return Status::OK();
 }
 
-Status ConvBackpropExtractAndVerifyDimension(
-    StringPiece label, const TensorShape& input_shape,
-    const TensorShape& filter_shape, const TensorShape& output_shape,
-    const std::vector<int32>& strides, Padding padding, int spatial_dim,
-    int filter_spatial_dim, ConvBackpropSpatialDimension* dim) {
-  static constexpr std::array<int32, 5> one_dilations = {{1, 1, 1, 1, 1}};
-  return ConvBackpropExtractAndVerifyDimensionV2(
-      label, input_shape, filter_shape, output_shape, one_dilations, strides,
-      padding, spatial_dim, filter_spatial_dim, dim);
-}
+}  // namespace
 
 Status ConvBackpropComputeDimensionsV2(
     StringPiece label, int num_spatial_dims, const TensorShape& input_shape,
     const TensorShape& filter_shape, const TensorShape& out_backprop_shape,
     const gtl::ArraySlice<int32>& dilations, const std::vector<int32>& strides,
-    Padding padding, TensorFormat data_format, ConvBackpropDimensions* dims) {
+    Padding padding, const std::vector<int64>& explicit_paddings,
+    TensorFormat data_format, ConvBackpropDimensions* dims) {
   // The + 2 in the following line is for the batch and feature dimensions.
   const int num_dims = num_spatial_dims + 2;
   if (input_shape.dims() != num_dims) {
@@ -152,9 +146,15 @@ Status ConvBackpropComputeDimensionsV2(
   dims->spatial_dims.resize(num_spatial_dims);
   for (int i = 0; i < num_spatial_dims; ++i) {
     int image_dim = GetTensorSpatialDimIndex(num_dims, data_format, i);
-    TF_RETURN_IF_ERROR(ConvBackpropExtractAndVerifyDimensionV2(
+    int64 padding_before = -1, padding_after = -1;
+    if (padding == EXPLICIT) {
+      padding_before = explicit_paddings[2 * image_dim];
+      padding_after = explicit_paddings[2 * image_dim + 1];
+    }
+    TF_RETURN_IF_ERROR(ConvBackpropExtractAndVerifyDimension(
         label, input_shape, filter_shape, out_backprop_shape, dilations,
-        strides, padding, image_dim, i, &dims->spatial_dims[i]));
+        strides, padding, padding_before, padding_after, image_dim, i,
+        &dims->spatial_dims[i]));
   }
   return Status::OK();
 }
@@ -169,7 +169,8 @@ Status ConvBackpropComputeDimensions(StringPiece label, int num_spatial_dims,
   static constexpr std::array<int32, 5> one_dilations = {{1, 1, 1, 1, 1}};
   return ConvBackpropComputeDimensionsV2(
       label, num_spatial_dims, input_shape, filter_shape, out_backprop_shape,
-      one_dilations, strides, padding, data_format, dims);
+      one_dilations, strides, padding, /*explicit_paddings=*/{}, data_format,
+      dims);
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/conv_grad_ops.h b/tensorflow/core/kernels/conv_grad_ops.h
index 9551959463..c8e8cf28c5 100644
--- a/tensorflow/core/kernels/conv_grad_ops.h
+++ b/tensorflow/core/kernels/conv_grad_ops.h
@@ -176,8 +176,9 @@ struct LaunchConv2DBackpropInputOp {
   void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
                   const Tensor& out_backprop, const Tensor& filter,
                   int row_dilation, int col_dilation, int row_stride,
-                  int col_stride, const Padding& padding, Tensor* in_backprop,
-                  TensorFormat data_format);
+                  int col_stride, const Padding& padding,
+                  const std::vector<int64>& explicit_paddings,
+                  Tensor* in_backprop, TensorFormat data_format);
 };
 
 template <typename Device, typename T>
@@ -186,6 +187,7 @@ struct LaunchConv2DBackpropFilterOp {
                   const Tensor& out_backprop, const Tensor& input,
                   int row_dilation, int col_dilation, int row_stride,
                   int col_stride, const Padding& padding,
+                  const std::vector<int64>& explicit_paddings,
                   Tensor* filter_backprop, TensorFormat data_format);
 };
 
@@ -195,7 +197,8 @@ struct LaunchConv2DBackpropInputOp<Eigen::GpuDevice, T> {
   void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
                   const Tensor& input, const Tensor& filter, int row_dilation,
                   int col_dilation, int row_stride, int col_stride,
-                  const Padding& padding, Tensor* output,
+                  const Padding& padding,
+                  const std::vector<int64>& explicit_paddings, Tensor* output,
                   TensorFormat data_format);
 };
 
@@ -205,6 +208,7 @@ struct LaunchConv2DBackpropFilterOp<Eigen::GpuDevice, T> {
                   const Tensor& out_backprop, const Tensor& input,
                   int row_dilation, int col_dilation, int row_stride,
                   int col_stride, const Padding& padding,
+                  const std::vector<int64>& explicit_paddings,
                   Tensor* filter_backprop, TensorFormat data_format);
 };
 #endif  // GOOGLE_CUDA
@@ -217,6 +221,8 @@ struct ConvBackpropSpatialDimension {
   int64 output_size;
   int64 stride;
   int64 dilation;
+
+  // The following fields are valid only if the padding is not EXPLICIT.
   int64 expanded_output_size;
 
   // Number of padding elements to be added before/after this dimension of
@@ -248,7 +254,7 @@ struct ConvBackpropDimensions {
 
 // Common code between implementations of Conv?DBackpropInput and
 // Conv?DBackpropFilter. Verifies that the dimensions all match, and computes
-// sizes/padding for the spatial dimensions.
+// sizes/padding for the spatial dimensions. Does not support explicit padding.
 Status ConvBackpropComputeDimensions(StringPiece label, int num_spatial_dims,
                                      const TensorShape& input_shape,
                                      const TensorShape& filter_shape,
@@ -257,13 +263,15 @@ Status ConvBackpropComputeDimensions(StringPiece label, int num_spatial_dims,
                                      Padding padding, TensorFormat data_format,
                                      ConvBackpropDimensions* dims);
 
-// The V2 version computes the same outputs with arbitrary dilation rate.
+// The V2 version computes the same outputs with arbitrary dilation rate and
+// supports explicit padding.
 // TODO(b/67112639): Merge V2 versions and the original versions eventually.
 Status ConvBackpropComputeDimensionsV2(
     StringPiece label, int num_spatial_dims, const TensorShape& input_shape,
     const TensorShape& filter_shape, const TensorShape& out_backprop_shape,
     const gtl::ArraySlice<int32>& dilations, const std::vector<int32>& strides,
-    Padding padding, TensorFormat data_format, ConvBackpropDimensions* dims);
+    Padding padding, const std::vector<int64>& explicit_paddings,
+    TensorFormat data_format, ConvBackpropDimensions* dims);
 }  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_KERNELS_CONV_GRAD_OPS_H_
diff --git a/tensorflow/core/kernels/conv_grad_ops_3d.cc b/tensorflow/core/kernels/conv_grad_ops_3d.cc
index 562a9c8aed..ca46da6ba3 100644
--- a/tensorflow/core/kernels/conv_grad_ops_3d.cc
+++ b/tensorflow/core/kernels/conv_grad_ops_3d.cc
@@ -1152,11 +1152,11 @@ class Conv3DBackpropInputOp<GPUDevice, T> : public OpKernel {
     }
 
     ConvBackpropDimensions dims;
-    OP_REQUIRES_OK(context,
-                   ConvBackpropComputeDimensionsV2(
-                       "Conv3DBackpropInputOp", /*num_spatial_dims=*/3,
-                       input_shape, filter_shape, out_backprop_shape, dilation_,
-                       stride_, padding_, data_format_, &dims));
+    OP_REQUIRES_OK(context, ConvBackpropComputeDimensionsV2(
+                                "Conv3DBackpropInputOp", /*num_spatial_dims=*/3,
+                                input_shape, filter_shape, out_backprop_shape,
+                                dilation_, stride_, padding_,
+                                /*explicit_paddings=*/{}, data_format_, &dims));
 
     Tensor* in_backprop;
     OP_REQUIRES_OK(context,
@@ -1537,11 +1537,12 @@ class Conv3DBackpropFilterOp<GPUDevice, T> : public OpKernel {
     }
 
     ConvBackpropDimensions dims;
-    OP_REQUIRES_OK(context,
-                   ConvBackpropComputeDimensionsV2(
-                       "Conv3DBackpropFilterOp", /*num_spatial_dims=*/3,
-                       input_shape, filter_shape, out_backprop_shape, dilation_,
-                       stride_, padding_, data_format_, &dims));
+    OP_REQUIRES_OK(
+        context,
+        ConvBackpropComputeDimensionsV2(
+            "Conv3DBackpropFilterOp", /*num_spatial_dims=*/3, input_shape,
+            filter_shape, out_backprop_shape, dilation_, stride_, padding_,
+            /*explicit_paddings=*/{}, data_format_, &dims));
 
     Tensor* filter_backprop;
     OP_REQUIRES_OK(context,
diff --git a/tensorflow/core/kernels/conv_ops.cc b/tensorflow/core/kernels/conv_ops.cc
index dfba15792d..a8138fd0a7 100644
--- a/tensorflow/core/kernels/conv_ops.cc
+++ b/tensorflow/core/kernels/conv_ops.cc
@@ -122,7 +122,8 @@ struct LaunchConv2DOp<CPUDevice, T> {
   void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
                   const Tensor& input, const Tensor& filter, int row_dilation,
                   int col_dilation, int row_stride, int col_stride,
-                  const Padding& padding, Tensor* output,
+                  const Padding& padding,
+                  const std::vector<int64>& explicit_paddings, Tensor* output,
                   TensorFormat data_format) {
     if (data_format != FORMAT_NHWC) {
       ctx->SetStatus(
@@ -130,6 +131,11 @@ struct LaunchConv2DOp<CPUDevice, T> {
                                 "NHWC tensor format for now."));
       return;
     }
+    // TODO(reedwm): Enable explicit padding on the CPU.
+    OP_REQUIRES(
+        ctx, padding != Padding::EXPLICIT,
+        errors::Unimplemented("Generic conv implementation does not support "
+                              "EXPLICIT padding yet."));
     const int64 in_depth = GetTensorDim(input, data_format, 'C');
     OP_REQUIRES(ctx, in_depth == filter.dim_size(2),
                 errors::Unimplemented("Generic conv implementation does not "
@@ -274,6 +280,10 @@ Status InitConv2DParameters(const OpKernelConstruction* context,
   TF_RETURN_IF_ERROR(context->GetAttr("dilations", &params->dilations));
   TF_RETURN_IF_ERROR(context->GetAttr("strides", &params->strides));
   TF_RETURN_IF_ERROR(context->GetAttr("padding", &params->padding));
+  if (context->HasAttr("explicit_paddings")) {
+    TF_RETURN_IF_ERROR(
+        context->GetAttr("explicit_paddings", &params->explicit_paddings));
+  }
   string data_format_string;
   TF_RETURN_IF_ERROR(context->GetAttr("data_format", &data_format_string));
   TF_REQUIRES(FormatFromString(data_format_string, &params->data_format),
@@ -313,6 +323,10 @@ Status InitConv2DParameters(const OpKernelConstruction* context,
       dilation_h > 0 && dilation_w > 0,
       errors::InvalidArgument("Dilated rates should be larger than 0."));
 
+  TF_RETURN_IF_ERROR(CheckValidPadding(params->padding,
+                                       params->explicit_paddings,
+                                       /*num_dims=*/4, data_format));
+
   return Status::OK();
 }
 
@@ -381,14 +395,22 @@ Status ComputeConv2DDimension(const Conv2DParameters& params,
   const int dilation_cols =
       GetTensorDim(params.dilations, params.data_format, 'W');
 
+  int64 pad_rows_before, pad_rows_after, pad_cols_before, pad_cols_after;
+  if (params.padding == Padding::EXPLICIT) {
+    GetExplicitPaddingForDim(params.explicit_paddings, params.data_format, 'H',
+                             &pad_rows_before, &pad_rows_after);
+    GetExplicitPaddingForDim(params.explicit_paddings, params.data_format, 'W',
+                             &pad_cols_before, &pad_cols_after);
+  }
+
   // Compute windowed output sizes for rows and columns.
-  int64 out_rows = 0, out_cols = 0, pad_rows = 0, pad_cols = 0;
-  TF_RETURN_IF_ERROR(GetWindowedOutputSizeV2(
+  int64 out_rows = 0, out_cols = 0;
+  TF_RETURN_IF_ERROR(GetWindowedOutputSizeVerboseV2(
       input_rows, filter_rows, dilation_rows, stride_rows, params.padding,
-      &out_rows, &pad_rows));
-  TF_RETURN_IF_ERROR(GetWindowedOutputSizeV2(
+      &out_rows, &pad_rows_before, &pad_rows_after));
+  TF_RETURN_IF_ERROR(GetWindowedOutputSizeVerboseV2(
       input_cols, filter_cols, dilation_cols, stride_cols, params.padding,
-      &out_cols, &pad_cols));
+      &out_cols, &pad_cols_before, &pad_cols_after));
 
   dimensions->batch = batch;
   dimensions->input_rows = input_rows;
@@ -404,8 +426,10 @@ Status ComputeConv2DDimension(const Conv2DParameters& params,
   dimensions->dilation_cols = dilation_cols;
   dimensions->out_rows = out_rows;
   dimensions->out_cols = out_cols;
-  dimensions->pad_rows = pad_rows;
-  dimensions->pad_cols = pad_cols;
+  dimensions->pad_rows_before = pad_rows_before;
+  dimensions->pad_rows_after = pad_rows_after;
+  dimensions->pad_cols_before = pad_cols_before;
+  dimensions->pad_cols_after = pad_cols_after;
 
   return Status::OK();
 }
@@ -463,33 +487,35 @@ class Conv2DOp : public BinaryOp<T> {
     }
 
 #ifdef TENSORFLOW_USE_LIBXSMM_CONVOLUTIONS
-    if (LaunchXsmmConvOp<Device, T>::Run(
+    if (params_.padding != EXPLICIT &&
+        LaunchXsmmConvOp<Device, T>::Run(
             context, input, filter, dimensions.batch, dimensions.input_rows,
             dimensions.input_cols, dimensions.in_depth, dimensions.filter_rows,
-            dimensions.filter_cols, dimensions.pad_rows, dimensions.pad_cols,
-            dimensions.out_rows, dimensions.out_cols, dimensions.out_depth,
-            dimensions.dilation_rows, dimensions.dilation_cols,
-            dimensions.stride_rows, dimensions.stride_cols, output,
-            params_.data_format)) {
+            dimensions.filter_cols, dimensions.pad_rows_before,
+            dimensions.pad_cols_before, dimensions.out_rows,
+            dimensions.out_cols, dimensions.out_depth, dimensions.dilation_rows,
+            dimensions.dilation_cols, dimensions.stride_rows,
+            dimensions.stride_cols, output, params_.data_format)) {
       return;
     }
 #endif
 
-    if (LaunchDeepConvOp<Device, T>::Run(
+    if (params_.padding != EXPLICIT &&
+        LaunchDeepConvOp<Device, T>::Run(
             context, input, filter, dimensions.batch, dimensions.input_rows,
             dimensions.input_cols, dimensions.in_depth, dimensions.filter_rows,
-            dimensions.filter_cols, dimensions.pad_rows, dimensions.pad_cols,
-            dimensions.out_rows, dimensions.out_cols, dimensions.out_depth,
-            dimensions.dilation_rows, dimensions.dilation_cols,
-            dimensions.stride_rows, dimensions.stride_cols, output,
-            params_.data_format)) {
+            dimensions.filter_cols, dimensions.pad_rows_before,
+            dimensions.pad_cols_before, dimensions.out_rows,
+            dimensions.out_cols, dimensions.out_depth, dimensions.dilation_rows,
+            dimensions.dilation_cols, dimensions.stride_rows,
+            dimensions.stride_cols, output, params_.data_format)) {
       return;
     }
 
     launcher_(context, use_cudnn_, cudnn_use_autotune_, input, filter,
               dimensions.dilation_rows, dimensions.dilation_cols,
               dimensions.stride_rows, dimensions.stride_cols, params_.padding,
-              output, params_.data_format);
+              params_.explicit_paddings, output, params_.data_format);
   }
 
  private:
@@ -551,7 +577,8 @@ void LaunchConv2DOp<GPUDevice, T>::operator()(
     OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
     const Tensor& input_param, const Tensor& filter, int row_dilation,
     int col_dilation, int row_stride, int col_stride, const Padding& padding,
-    Tensor* output, TensorFormat data_format) {
+    const std::vector<int64>& explicit_paddings, Tensor* output,
+    TensorFormat data_format) {
   using se::dnn::AlgorithmConfig;
   using se::dnn::AlgorithmDesc;
   using se::dnn::ProfileResult;
@@ -580,7 +607,8 @@ void LaunchConv2DOp<GPUDevice, T>::operator()(
   bool is_grouped_convolution = patch_depths != in_depths;
   if (patch_rows == 1 && patch_cols == 1 && !is_grouped_convolution &&
       row_dilation == 1 && col_dilation == 1 && row_stride == 1 &&
-      col_stride == 1 && data_format == FORMAT_NHWC) {
+      col_stride == 1 && data_format == FORMAT_NHWC &&
+      (padding == VALID || padding == SAME)) {
     // 1x1 filter, so call cublas directly.
     const uint64 m = in_batch * in_rows * in_cols;
     const uint64 k = patch_depths;
@@ -634,49 +662,78 @@ void LaunchConv2DOp<GPUDevice, T>::operator()(
     return;
   }
 
-  int padding_rows = 0;
-  int padding_cols = 0;
   const int64 out_batch = GetTensorDim(*output, data_format, 'N');
   const int64 out_rows = GetTensorDim(*output, data_format, 'H');
   const int64 out_cols = GetTensorDim(*output, data_format, 'W');
   const int64 out_depths = GetTensorDim(*output, data_format, 'C');
-  if (padding == SAME) {
-    // Total padding on rows and cols is
-    // Pr = (R' - 1) * S + (Kr - 1) * Dr + 1 - R
-    // Pc = (C' - 1) * S + (Kc - 1) * Dc + 1 - C
-    // where (R', C') are output dimensions, (R, C) are input dimensions, S
-    // is stride, (Dr, Dc) are dilations, (Kr, Kc) are filter dimensions.
-    // We pad Pr/2 on the left and Pr - Pr/2 on the right, Pc/2 on the top
-    // and Pc - Pc/2 on the bottom.  When Pr or Pc is odd, this means
-    // we pad more on the right and bottom than on the top and left.
-    padding_rows =
-        std::max<int>(0, (out_rows - 1) * row_stride +
-                             (patch_rows - 1) * row_dilation + 1 - in_rows);
-    padding_cols =
-        std::max<int>(0, (out_cols - 1) * col_stride +
-                             (patch_cols - 1) * col_dilation + 1 - in_cols);
-    const bool rows_odd = (padding_rows % 2 != 0);
-    const bool cols_odd = (padding_cols % 2 != 0);
-    if (rows_odd || cols_odd) {
-      Tensor transformed_input;
-      int64 new_in_rows = in_rows + rows_odd;
-      int64 new_in_cols = in_cols + cols_odd;
-      OP_REQUIRES_OK(
-          ctx,
-          ctx->allocate_temp(DataTypeToEnum<T>::value,
-                             ShapeFromFormat(data_format, in_batch, new_in_rows,
-                                             new_in_cols, in_depths),
-                             &transformed_input));
-
-      functor::PadInput<GPUDevice, T, int, 4>()(
-          ctx->eigen_device<GPUDevice>(), To32Bit(input_param.tensor<T, 4>()),
-          {{0, 0}}, {{rows_odd, cols_odd}},
-          To32Bit(transformed_input.tensor<T, 4>()), data_format);
-
-      input = transformed_input;
-      in_rows = new_in_rows;
-      in_cols = new_in_cols;
+  int64 padding_top = -1, padding_bottom = -1;
+  int64 padding_left = -1, padding_right = -1;
+  if (padding == EXPLICIT) {
+    GetExplicitPaddingForDim(explicit_paddings, data_format, 'H', &padding_top,
+                             &padding_bottom);
+    GetExplicitPaddingForDim(explicit_paddings, data_format, 'W', &padding_left,
+                             &padding_right);
+  }
+  int64 out_rows_check, out_cols_check;
+  Status status = GetWindowedOutputSizeVerboseV2(
+      in_rows, patch_rows, row_dilation, row_stride, padding, &out_rows_check,
+      &padding_top, &padding_bottom);
+  // The status is guaranteed to be OK because we checked the output and padding
+  // was valid earlier.
+  TF_CHECK_OK(status);
+  DCHECK_EQ(out_rows, out_rows_check);
+  status = GetWindowedOutputSizeVerboseV2(in_cols, patch_cols, col_dilation,
+                                          col_stride, padding, &out_cols_check,
+                                          &padding_left, &padding_right);
+  TF_CHECK_OK(status);
+  DCHECK_EQ(out_cols, out_cols_check);
+
+  const int64 common_padding_rows = std::min(padding_top, padding_bottom);
+  const int64 common_padding_cols = std::min(padding_left, padding_right);
+  if (padding_top != padding_bottom || padding_left != padding_right) {
+    // cuDNN only supports padding the same amount on the left and right sides,
+    // and on the top and bottom sides. So we manually create a new padded
+    // input tensor such that we can pass it to cuDNN.
+
+    // TODO(reedwm): In some cases, we can avoid an allocation even if the two
+    // padding sides are different. For example, if the input is 2x2, the filter
+    // is 1x1, the stride is 2, and the padding is (1, 0, 1, 0), the result is
+    // equivalent to as if the padding is (1, 1, 1, 1). Changing the padding in
+    // such a way would allow us to avoid the allocation.
+    Tensor transformed_input;
+    const int64 padding_rows_diff = std::abs(padding_bottom - padding_top);
+    const int64 padding_cols_diff = std::abs(padding_right - padding_left);
+    const int64 new_in_rows = in_rows + padding_rows_diff;
+    const int64 new_in_cols = in_cols + padding_cols_diff;
+    OP_REQUIRES_OK(ctx, ctx->allocate_temp(
+                            DataTypeToEnum<T>::value,
+                            ShapeFromFormat(data_format, in_batch, new_in_rows,
+                                            new_in_cols, in_depths),
+                            &transformed_input));
+
+    const int64 input_pad_top = padding_top - common_padding_rows;
+    const int64 input_pad_bottom = padding_bottom - common_padding_rows;
+    const int64 input_pad_left = padding_left - common_padding_cols;
+    const int64 input_pad_right = padding_right - common_padding_cols;
+    bool in_bounds =
+        FastBoundsCheck(input_pad_top, std::numeric_limits<int>::max()) &&
+        FastBoundsCheck(input_pad_bottom, std::numeric_limits<int>::max()) &&
+        FastBoundsCheck(input_pad_left, std::numeric_limits<int>::max()) &&
+        FastBoundsCheck(input_pad_right, std::numeric_limits<int>::max());
+    if (!in_bounds) {
+      ctx->SetStatus(errors::InvalidArgument("Padding is too large."));
+      return;
     }
+    functor::PadInput<GPUDevice, T, int, 4>()(
+        ctx->eigen_device<GPUDevice>(), To32Bit(input_param.tensor<T, 4>()),
+        {{static_cast<int>(input_pad_top), static_cast<int>(input_pad_left)}},
+        {{static_cast<int>(input_pad_bottom),
+          static_cast<int>(input_pad_right)}},
+        To32Bit(transformed_input.tensor<T, 4>()), data_format);
+
+    input = transformed_input;
+    in_rows = new_in_rows;
+    in_cols = new_in_cols;
   }
 
   if (data_format == FORMAT_NHWC) {
@@ -698,9 +755,9 @@ void LaunchConv2DOp<GPUDevice, T>::operator()(
     }
   }
 
-  CHECK(padding_rows >= 0 && padding_cols >= 0)
-      << "Negative row or col paddings: (" << padding_rows << ", "
-      << padding_cols << ")";
+  CHECK(common_padding_rows >= 0 && common_padding_cols >= 0)  // Crash OK
+      << "Negative row or col paddings: (" << common_padding_rows << ", "
+      << common_padding_cols << ")";
   se::dnn::BatchDescriptor input_desc;
   input_desc.set_count(in_batch)
       .set_feature_map_count(in_depths)
@@ -723,8 +780,8 @@ void LaunchConv2DOp<GPUDevice, T>::operator()(
       .set_horizontal_dilation_rate(col_dilation)
       .set_vertical_filter_stride(row_stride)
       .set_horizontal_filter_stride(col_stride)
-      .set_zero_padding_height(padding_rows / 2)
-      .set_zero_padding_width(padding_cols / 2)
+      .set_zero_padding_height(common_padding_rows)
+      .set_zero_padding_width(common_padding_cols)
       .set_group_count(in_depths / patch_depths);
 
   Tensor transformed_filter;
@@ -767,23 +824,23 @@ void LaunchConv2DOp<GPUDevice, T>::operator()(
   int device_id = stream->parent()->device_ordinal();
   DataType dtype = input.dtype();
   ConvParameters conv_parameters = {
-      in_batch,          // batch
-      in_depths,         // in_depths
-      {{in_rows,         // in_rows
-        in_cols}},       // in_cols
-      FORMAT_NCHW,       // compute_data_format
-      out_depths,        // out_depths
-      {{patch_rows,      // filter_rows
-        patch_cols,      // filter_cols
-        patch_depths}},  // filter_depths
-      {{row_dilation,    // dilation_rows
-        col_dilation}},  // dilation_cols
-      {{row_stride,      // stride_rows
-        col_stride}},    // stride_cols
-      {{padding_rows,    // padding_rows
-        padding_cols}},  // padding_cols
-      dtype,             // tensor datatype
-      device_id,         // device_id
+      in_batch,                 // batch
+      in_depths,                // in_depths
+      {{in_rows,                // in_rows
+        in_cols}},              // in_cols
+      FORMAT_NCHW,              // compute_data_format
+      out_depths,               // out_depths
+      {{patch_rows,             // filter_rows
+        patch_cols,             // filter_cols
+        patch_depths}},         // filter_depths
+      {{row_dilation,           // dilation_rows
+        col_dilation}},         // dilation_cols
+      {{row_stride,             // stride_rows
+        col_stride}},           // stride_cols
+      {{common_padding_rows,    // padding_rows
+        common_padding_cols}},  // padding_cols
+      dtype,                    // tensor datatype
+      device_id,                // device_id
   };
   AlgorithmConfig algorithm_config;
   if (cudnn_use_autotune &&
diff --git a/tensorflow/core/kernels/conv_ops.h b/tensorflow/core/kernels/conv_ops.h
index 7ccbaf4bf2..105a4b1b82 100644
--- a/tensorflow/core/kernels/conv_ops.h
+++ b/tensorflow/core/kernels/conv_ops.h
@@ -36,7 +36,8 @@ struct LaunchConv2DOp {
   void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
                   const Tensor& input, const Tensor& filter, int row_dilation,
                   int col_dilation, int row_stride, int col_stride,
-                  const Padding& padding, Tensor* output,
+                  const Padding& padding,
+                  const std::vector<int64>& explicit_paddings, Tensor* output,
                   TensorFormat data_format);
 };
 
@@ -46,7 +47,8 @@ struct LaunchConv2DOp<Eigen::GpuDevice, T> {
   void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
                   const Tensor& input, const Tensor& filter, int row_dilation,
                   int col_dilation, int row_stride, int col_stride,
-                  const Padding& padding, Tensor* output,
+                  const Padding& padding,
+                  const std::vector<int64>& explicit_paddings, Tensor* output,
                   TensorFormat data_format);
 };
 #endif  // GOOGLE_CUDA
@@ -72,6 +74,7 @@ struct Conv2DParameters {
   std::vector<int32> strides;
   Padding padding;
   TensorFormat data_format;
+  std::vector<int64> explicit_paddings;
 };
 
 // Convolution dimensions inferred from parameters, input and filter tensors.
@@ -94,8 +97,10 @@ struct Conv2DDimensions {
 
   int64 out_rows;
   int64 out_cols;
-  int64 pad_rows;
-  int64 pad_cols;
+  int64 pad_rows_before;
+  int64 pad_rows_after;
+  int64 pad_cols_before;
+  int64 pad_cols_after;
 };
 
 // Initializes and validates Conv2D parameters configured by OpKernel
diff --git a/tensorflow/core/kernels/depthwise_conv_grad_op.cc b/tensorflow/core/kernels/depthwise_conv_grad_op.cc
index da3bdb475e..c152f2b7e4 100644
--- a/tensorflow/core/kernels/depthwise_conv_grad_op.cc
+++ b/tensorflow/core/kernels/depthwise_conv_grad_op.cc
@@ -633,7 +633,8 @@ class DepthwiseConv2dNativeBackpropInputOp : public OpKernel {
       // conv is supported.
       launcher_(context, use_cudnn_, cudnn_use_autotune_, out_backprop,
                 reshaped_filter, /*row_dilation=*/1, /*col_dilation=*/1,
-                stride_, stride_, padding_, in_backprop, data_format_);
+                stride_, stride_, padding_, /*explicit_paddings=*/{},
+                in_backprop, data_format_);
       return;
     }
 
@@ -1115,7 +1116,8 @@ class DepthwiseConv2dNativeBackpropFilterOp : public OpKernel {
       // conv is supported.
       launcher_(context, use_cudnn_, cudnn_use_autotune_, out_backprop, input,
                 /*row_dilation=*/1, /*col_dilation=*/1, stride_, stride_,
-                padding_, &reshaped_filter, data_format_);
+                padding_, /*explicit_paddings=*/{}, &reshaped_filter,
+                data_format_);
       return;
     }
 
diff --git a/tensorflow/core/kernels/depthwise_conv_op.cc b/tensorflow/core/kernels/depthwise_conv_op.cc
index f0902fdba6..dacd3cfea8 100644
--- a/tensorflow/core/kernels/depthwise_conv_op.cc
+++ b/tensorflow/core/kernels/depthwise_conv_op.cc
@@ -404,7 +404,8 @@ class DepthwiseConv2dNativeOp : public BinaryOp<T> {
       // conv is supported.
       launcher_(context, use_cudnn_, cudnn_use_autotune_, input,
                 reshaped_filter, /*row_dilation=*/1, /*col_dilation=*/1,
-                stride_, stride_, padding_, output, data_format_);
+                stride_, stride_, padding_, /*explicit_paddings=*/{}, output,
+                data_format_);
       return;
     }
 
diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc
index 0e94259f75..0f4f725937 100644
--- a/tensorflow/core/ops/nn_ops.cc
+++ b/tensorflow/core/ops/nn_ops.cc
@@ -269,10 +269,11 @@ REGISTER_OP("Conv2D")
     .Attr("T: {half, bfloat16, float, double}")
     .Attr("strides: list(int)")
     .Attr("use_cudnn_on_gpu: bool = true")
-    .Attr(GetPaddingAttrString())
+    .Attr(GetPaddingAttrStringWithExplicit())
+    .Attr(GetExplicitPaddingsAttrString())
     .Attr(GetConvnetDataFormatAttrString())
     .Attr("dilations: list(int) = [1, 1, 1, 1]")
-    .SetShapeFn(shape_inference::Conv2DShape);
+    .SetShapeFn(shape_inference::Conv2DShapeWithExplicitPadding);
 
 REGISTER_OP("Conv2DBackpropInput")
     .Input("input_sizes: int32")
@@ -282,7 +283,8 @@ REGISTER_OP("Conv2DBackpropInput")
     .Attr("T: {half, bfloat16, float, double}")
     .Attr("strides: list(int)")
     .Attr("use_cudnn_on_gpu: bool = true")
-    .Attr(GetPaddingAttrString())
+    .Attr(GetPaddingAttrStringWithExplicit())
+    .Attr(GetExplicitPaddingsAttrString())
     .Attr(GetConvnetDataFormatAttrString())
     .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn([](InferenceContext* c) {
@@ -304,7 +306,8 @@ REGISTER_OP("Conv2DBackpropFilter")
     .Attr("T: {half, bfloat16, float, double}")
     .Attr("strides: list(int)")
     .Attr("use_cudnn_on_gpu: bool = true")
-    .Attr(GetPaddingAttrString())
+    .Attr(GetPaddingAttrStringWithExplicit())
+    .Attr(GetExplicitPaddingsAttrString())
     .Attr(GetConvnetDataFormatAttrString())
     .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn([](InferenceContext* c) {
diff --git a/tensorflow/core/util/padding.cc b/tensorflow/core/util/padding.cc
index 117de5ee4b..9e7fb8489e 100644
--- a/tensorflow/core/util/padding.cc
+++ b/tensorflow/core/util/padding.cc
@@ -29,12 +29,55 @@ Status GetNodeAttr(const NodeDef& node_def, StringPiece attr_name,
     *value = SAME;
   } else if (str_value == "VALID") {
     *value = VALID;
+  } else if (str_value == "EXPLICIT") {
+    *value = EXPLICIT;
   } else {
     return errors::NotFound(str_value, " is not an allowed padding type");
   }
   return Status::OK();
 }
 
+Status CheckValidPadding(Padding padding_type,
+                         const std::vector<int64>& explicit_paddings,
+                         int num_dims, TensorFormat data_format) {
+  if (padding_type == Padding::EXPLICIT) {
+    if (explicit_paddings.size() != 2 * num_dims) {
+      return errors::InvalidArgument(
+          "explicit_paddings attribute must contain ", 2 * num_dims,
+          " values, but got: ", explicit_paddings.size());
+    }
+    for (int64 padding_value : explicit_paddings) {
+      if (padding_value < 0) {
+        return errors::InvalidArgument(
+            "All elements of explicit_paddings must be nonnegative");
+      }
+    }
+    const int32 batch_index = GetTensorBatchDimIndex(num_dims, data_format);
+    const int32 depth_index = GetTensorFeatureDimIndex(num_dims, data_format);
+    if (explicit_paddings[2 * batch_index] != 0 ||
+        explicit_paddings[2 * batch_index + 1] != 0 ||
+        explicit_paddings[2 * depth_index] != 0 ||
+        explicit_paddings[2 * depth_index + 1] != 0) {
+      return errors::InvalidArgument(
+          "Nonzero explicit padding in the batch or depth dimensions is not "
+          "supported");
+    }
+  } else if (!explicit_paddings.empty()) {
+    return errors::InvalidArgument(
+        "explicit_paddings attribute must be empty if the padding attribute is "
+        "not EXPLICIT");
+  }
+  return Status::OK();
+}
+
 string GetPaddingAttrString() { return "padding: {'SAME', 'VALID'}"; }
 
+string GetPaddingAttrStringWithExplicit() {
+  return "padding: {'SAME', 'VALID', 'EXPLICIT'}";
+}
+
+string GetExplicitPaddingsAttrString() {
+  return "explicit_paddings: list(int) = []";
+}
+
 }  // end namespace tensorflow
diff --git a/tensorflow/core/util/padding.h b/tensorflow/core/util/padding.h
index 76f9b4dd9a..a1dd1c0bd9 100644
--- a/tensorflow/core/util/padding.h
+++ b/tensorflow/core/util/padding.h
@@ -20,8 +20,10 @@ limitations under the License.
 // kernels.
 
 #include <string>
+#include <vector>
 
 #include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/util/tensor_format.h"
 
 namespace tensorflow {
 
@@ -34,16 +36,29 @@ class NodeDef;
 //   VALID: No padding is carried out.
 //   SAME: The pad value is computed so that the output will have the same
 //         dimensions as the input.
+//   EXPLICIT: The user specifies the pad values in the explicit_padding
+//             attribute.
 // The padded area is zero-filled.
 enum Padding {
-  VALID = 1,  // No padding.
-  SAME = 2,   // Input and output layers have the same size.
+  VALID = 1,     // No padding.
+  SAME = 2,      // Input and output layers have the same size.
+  EXPLICIT = 3,  // Padding is explicitly specified
 };
 
+// Returns an error if the padding attributes are invalid.
+Status CheckValidPadding(Padding padding_type,
+                         const std::vector<int64>& explicit_paddings,
+                         int num_dims, TensorFormat data_format);
+
 // Return the string containing the list of valid padding types, that can be
 // used as an Attr() in REGISTER_OP.
 string GetPaddingAttrString();
 
+// Like GetPaddingAttrString(), but also includes EXPLICIT.
+string GetPaddingAttrStringWithExplicit();
+
+string GetExplicitPaddingsAttrString();
+
 // Specialization to parse an attribute directly into a Padding enum.
 Status GetNodeAttr(const NodeDef& node_def, StringPiece attr_name,
                    Padding* value);
diff --git a/tensorflow/core/util/tensor_format.h b/tensorflow/core/util/tensor_format.h
index a296fb447e..643e14e0b5 100644
--- a/tensorflow/core/util/tensor_format.h
+++ b/tensorflow/core/util/tensor_format.h
@@ -408,18 +408,24 @@ inline int32 GetTensorDimIndex(TensorFormat format, char dimension) {
   return GetTensorDimIndex<2>(format, dimension);
 }
 
+inline int32 GetTensorDimIndex(TensorFormat format, char dimension,
+                               int num_total_dims) {
+  int32 index = (GetTensorSpatialDims(num_total_dims, format) == 3)
+                    ? GetTensorDimIndex<3>(format, dimension)
+                    : GetTensorDimIndex<2>(format, dimension);
+  CHECK(index >= 0 && index < num_total_dims)  // Crash OK.
+      << "Invalid index from the dimension: " << index << ", " << format << ", "
+      << dimension;
+  return index;
+}
+
 // Return the element from 'dimension_attributes' that corresponds to the
 // specified 'dimension' according to 'tensor_format'.
 template <typename T>
 T GetTensorDim(gtl::ArraySlice<T> dimension_attributes,
                TensorFormat tensor_format, char dimension) {
   int index =
-      (GetTensorSpatialDims(dimension_attributes.size(), tensor_format) == 3)
-          ? GetTensorDimIndex<3>(tensor_format, dimension)
-          : GetTensorDimIndex<2>(tensor_format, dimension);
-  CHECK(index >= 0 && index < dimension_attributes.size())
-      << "Invalid index from the dimension: " << index << ", " << tensor_format
-      << ", " << dimension;
+      GetTensorDimIndex(tensor_format, dimension, dimension_attributes.size());
   return dimension_attributes[index];
 }
 
@@ -476,6 +482,15 @@ inline int64 GetFilterDim(const Tensor& tensor,
   return GetFilterDim(tensor.shape(), filter_tensor_format, dimension);
 }
 
+inline void GetExplicitPaddingForDim(
+    const std::vector<int64>& explicit_paddings, TensorFormat tensor_format,
+    char dimension, int64* padding_before, int64* padding_after) {
+  int index =
+      GetTensorDimIndex(tensor_format, dimension, explicit_paddings.size() / 2);
+  *padding_before = explicit_paddings[2 * index];
+  *padding_after = explicit_paddings[2 * index + 1];
+}
+
 // Return the string that specifies the data format for convnet operations.
 string GetConvnetDataFormatAttrString();
 string GetConvnet3dDataFormatAttrString();
diff --git a/tensorflow/python/keras/layers/convolutional.py b/tensorflow/python/keras/layers/convolutional.py
index 80ec55f410..30b919cc0a 100644
--- a/tensorflow/python/keras/layers/convolutional.py
+++ b/tensorflow/python/keras/layers/convolutional.py
@@ -180,12 +180,14 @@ class Conv(Layer):
       op_padding = 'valid'
     else:
       op_padding = self.padding
+    if not isinstance(op_padding, (list, tuple)):
+      op_padding = op_padding.upper()
     self._convolution_op = nn_ops.Convolution(
         input_shape,
         filter_shape=self.kernel.get_shape(),
         dilation_rate=self.dilation_rate,
         strides=self.strides,
-        padding=op_padding.upper(),
+        padding=op_padding,
         data_format=conv_utils.convert_data_format(self.data_format,
                                                    self.rank + 2))
     self.built = True
diff --git a/tensorflow/python/keras/utils/conv_utils.py b/tensorflow/python/keras/utils/conv_utils.py
index f486e631e5..ea7427f61a 100644
--- a/tensorflow/python/keras/utils/conv_utils.py
+++ b/tensorflow/python/keras/utils/conv_utils.py
@@ -194,9 +194,11 @@ def normalize_data_format(value):
 
 
 def normalize_padding(value):
+  if isinstance(value, (list, tuple)):
+    return value
   padding = value.lower()
   if padding not in {'valid', 'same', 'causal'}:
-    raise ValueError('The `padding` argument must be one of '
+    raise ValueError('The `padding` argument must be a list/tuple or one of '
                      '"valid", "same" (or "causal", only for `Conv1D). '
                      'Received: ' + str(padding))
   return padding
diff --git a/tensorflow/python/kernel_tests/conv_ops_test.py b/tensorflow/python/kernel_tests/conv_ops_test.py
index 2f6f3bb383..7ff1a61e47 100644
--- a/tensorflow/python/kernel_tests/conv_ops_test.py
+++ b/tensorflow/python/kernel_tests/conv_ops_test.py
@@ -26,13 +26,18 @@ import numpy as np
 
 from six.moves import xrange  # pylint: disable=redefined-builtin
 from tensorflow.contrib import layers
+from tensorflow.core.protobuf import config_pb2
+from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python.client import session as session_lib
+from tensorflow.python.eager import backprop
+from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import nn_impl
@@ -165,6 +170,12 @@ class Conv2DTest(test.TestCase):
       # as we will be using its gradients as reference for fp16 gradients.
       return [dtypes.float32, dtypes.float16, dtypes.float64]
 
+  def _CreateNumpyTensor(self, shape):
+    total_size = 1
+    for s in shape:
+      total_size *= s
+    return np.arange(1, total_size + 1, dtype=np.float32).reshape(shape)
+
   def _SetupValuesForDevice(self, tensor_in_sizes, filter_in_sizes, dilations,
                             strides, padding, data_format, dtype, use_gpu):
     """Verifies the output values of the convolution function.
@@ -183,26 +194,22 @@ class Conv2DTest(test.TestCase):
     Returns:
       Symbolic tensor value that can be used to execute the computation
     """
-    total_size_1 = 1
-    total_size_2 = 1
-    for s in tensor_in_sizes:
-      total_size_1 *= s
-    for s in filter_in_sizes:
-      total_size_2 *= s
-    # Initializes the input tensor with array containing incrementing
-    # numbers from 1.
-    x1 = [f * 1.0 for f in range(1, total_size_1 + 1)]
-    x2 = [f * 1.0 for f in range(1, total_size_2 + 1)]
+    x1 = self._CreateNumpyTensor(tensor_in_sizes)
+    x2 = self._CreateNumpyTensor(filter_in_sizes)
 
     with test_util.device(use_gpu):
       t1 = constant_op.constant(x1, shape=tensor_in_sizes, dtype=dtype)
       t2 = constant_op.constant(x2, shape=filter_in_sizes, dtype=dtype)
       strides = [1] + strides + [1]
       dilations = [1] + dilations + [1]
+      if isinstance(padding, (list, tuple)):
+        padding = [(0, 0)] + padding + [(0, 0)]
       if data_format == "NCHW":
         t1 = test_util.NHWCToNCHW(t1)
         strides = test_util.NHWCToNCHW(strides)
         dilations = test_util.NHWCToNCHW(dilations)
+        if isinstance(padding, (list, tuple)):
+          padding = test_util.NHWCToNCHW(padding)
       conv = nn_ops.conv2d(
           t1,
           t2,
@@ -254,17 +261,8 @@ class Conv2DTest(test.TestCase):
   def _ComputeReferenceDilatedConv(self, tensor_in_sizes, filter_in_sizes,
                                    stride, dilation, padding, data_format,
                                    use_gpu):
-    total_size_1 = 1
-    total_size_2 = 1
-    for s in tensor_in_sizes:
-      total_size_1 *= s
-    for s in filter_in_sizes:
-      total_size_2 *= s
-
-    # Initializes the input tensor with array containing incrementing
-    # numbers from 1.
-    x1 = [f * 1.0 for f in range(1, total_size_1 + 1)]
-    x2 = [f * 1.0 for f in range(1, total_size_2 + 1)]
+    x1 = self._CreateNumpyTensor(tensor_in_sizes)
+    x2 = self._CreateNumpyTensor(filter_in_sizes)
     with test_util.device(use_gpu):
       t1 = constant_op.constant(x1, shape=tensor_in_sizes)
       t2 = constant_op.constant(x2, shape=filter_in_sizes)
@@ -312,16 +310,29 @@ class Conv2DTest(test.TestCase):
       expected_values = self.evaluate(expected_results)
       computed_values = self.evaluate(computed_results)
       for e_value, c_value in zip(expected_values, computed_values):
-        tf_logging.info("expected = ", e_value)
-        tf_logging.info("actual = ", c_value)
+        tf_logging.debug("expected = %s", e_value)
+        tf_logging.debug("actual = %s", c_value)
         self.assertAllClose(
             e_value.flatten(), c_value.flatten(), atol=tolerance, rtol=1e-4)
 
-  def _VerifyValues(self, tensor_in_sizes, filter_in_sizes, strides, padding,
-                    expected):
+  def _VerifyValues(self,
+                    tensor_in_sizes,
+                    filter_in_sizes,
+                    strides,
+                    padding,
+                    expected,
+                    dilations=(1, 1),
+                    gpu_only=False,
+                    test_grappler_layout_optimizer=False,
+                    tol=1e-5,
+                    fp16_tol=1e-3):
+    if gpu_only and not test.is_gpu_available(cuda_only=True):
+      return
     tensors = []
-    dilations = [1, 1]
+    dilations = list(dilations)
     for (data_format, use_gpu) in GetTestConfigs():
+      if gpu_only and not use_gpu:
+        continue
       for dtype in self._DtypesToTest(use_gpu):
         result = self._SetupValuesForDevice(
             tensor_in_sizes,
@@ -332,19 +343,71 @@ class Conv2DTest(test.TestCase):
             data_format,
             dtype,
             use_gpu=use_gpu)
+        if test_grappler_layout_optimizer and data_format == "NHWC" and use_gpu:
+          # Grappler's layout optimizer will not optimize a fetch node, so
+          # this identity allows Grappler to optimize the Conv2D node.
+          result = array_ops.identity(result)
         tensors.append(result)
       values = self.evaluate(tensors)
       for i in range(len(tensors)):
         conv = tensors[i]
         value = values[i]
-        tf_logging.info("expected = ", expected)
-        tf_logging.info("actual = ", value)
-        tol = 1e-5
-        if value.dtype == np.float16:
-          tol = 1e-3
-        self.assertAllClose(expected, np.ravel(value), atol=tol, rtol=tol)
+        tf_logging.debug("expected = %s", expected)
+        tf_logging.debug("actual = %s", value)
+        tol_to_use = fp16_tol if value.dtype == np.float16 else tol
+        self.assertAllClose(expected, np.ravel(value), atol=tol_to_use,
+                            rtol=tol_to_use)
         self.assertShapeEqual(value, conv)
 
+  def _VerifyExplicitPaddings(self,
+                              tensor_in_sizes,
+                              filter_in_sizes,
+                              strides,
+                              padding,
+                              dilations=(1, 1),
+                              test_grappler_layout_optimizer=False,
+                              tol=1e-5,
+                              fp16_tol=1e-3):
+    """Verifies Conv2D with explicit padding generates correct values.
+
+    It does this by comparing with Conv2D without explicit padding. This
+    function assumes Conv2D without explicit padding works correctly.
+
+    Args:
+      tensor_in_sizes: Input tensor dimensions in [batch, input_rows,
+        input_cols, input_depth].
+      filter_in_sizes: Filter tensor dimensions in [kernel_rows, kernel_cols,
+        input_depth, output_depth].
+      strides: [row_stride, col_stride] for the convolution;
+      padding: Explicit padding amounts.
+      dilations: Dilation values
+      test_grappler_layout_optimizer: If True, allow the Grappler layout
+        optimizer to run, which turns NHWC Conv2Ds on the GPU to NCHW Conv2Ds.
+      tol: The absolute and relative tolerance for non-fp16 dtypes.
+      fp16_tol: The absolute and relative tolerance for fp16.
+    """
+    input_tensor = self._CreateNumpyTensor(tensor_in_sizes)
+    filter_tensor = self._CreateNumpyTensor(filter_in_sizes)
+    input_tensor = array_ops.pad(input_tensor, [(0, 0)] + padding + [(0, 0)])
+    dilations = list(dilations)
+    conv2d_result = nn_ops.conv2d(
+        input_tensor,
+        filter_tensor, [1] + list(strides) + [1],
+        "VALID",
+        dilations=[1] + dilations + [1])
+    expected = list(self.evaluate(array_ops.reshape(conv2d_result, [-1])))
+    self._VerifyValues(
+        tensor_in_sizes,
+        filter_in_sizes,
+        strides,
+        padding,
+        expected,
+        dilations,
+        gpu_only=True,
+        test_grappler_layout_optimizer=test_grappler_layout_optimizer,
+        tol=tol,
+        fp16_tol=fp16_tol)
+
   @test_util.run_in_graph_and_eager_modes
   def testConv2D1x1Filter(self):
     expected_output = [
@@ -510,6 +573,126 @@ class Conv2DTest(test.TestCase):
         dilations=[2, 2],
         padding="VALID")
 
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2D0x0Padding(self):
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[1, 2, 3, 3],
+        filter_in_sizes=[2, 2, 3, 3],
+        strides=[1, 1],
+        padding=[[0, 0], [0, 0]])
+
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[3, 4, 3, 2],
+        filter_in_sizes=[1, 1, 2, 1],
+        strides=[2, 2],
+        padding=[[0, 0], [0, 0]])
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2D1x1Padding(self):
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[1, 2, 3, 2],
+        filter_in_sizes=[2, 2, 2, 2],
+        strides=[1, 1],
+        padding=[[1, 1], [1, 1]])
+
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[1, 2, 2, 1],
+        filter_in_sizes=[1, 1, 1, 2],
+        strides=[1, 1],
+        padding=[[1, 1], [1, 1]])
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2D2x2Padding(self):
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[1, 2, 1, 2],
+        filter_in_sizes=[2, 1, 2, 1],
+        strides=[1, 1],
+        padding=[[2, 2], [2, 2]])
+
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[1, 2, 1, 2],
+        filter_in_sizes=[1, 1, 2, 1],
+        strides=[2, 1],
+        padding=[[2, 2], [2, 2]])
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2DOnlyBottomPadding(self):
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[1, 2, 3, 3],
+        filter_in_sizes=[2, 2, 3, 2],
+        strides=[1, 1],
+        padding=[[0, 3], [0, 0]], tol=2e-5)
+
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[2, 2, 4, 3],
+        filter_in_sizes=[1, 2, 3, 2],
+        strides=[2, 2],
+        padding=[[0, 3], [0, 0]])
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2DOnlyTopRightPadding(self):
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[1, 2, 3, 3],
+        filter_in_sizes=[2, 2, 3, 2],
+        strides=[1, 1],
+        padding=[[1, 0], [0, 2]],
+        tol=5e-5)
+
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[1, 2, 4, 2],
+        filter_in_sizes=[2, 2, 2, 2],
+        strides=[1, 3],
+        padding=[[1, 0], [0, 2]])
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2DLotsPadding(self):
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[1, 1, 1, 3],
+        filter_in_sizes=[2, 2, 3, 3],
+        strides=[1, 1],
+        padding=[[3, 4], [4, 2]])
+
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[1, 2, 1, 1],
+        filter_in_sizes=[2, 2, 1, 3],
+        strides=[2, 1],
+        padding=[[3, 4], [4, 2]])
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2DExplicitPaddingWithDilations(self):
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[1, 3, 2, 1],
+        filter_in_sizes=[1, 2, 1, 2],
+        strides=[1, 1],
+        padding=[[1, 0], [0, 1]],
+        dilations=[2, 1])
+
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[1, 2, 3, 2],
+        filter_in_sizes=[3, 2, 2, 1],
+        strides=[1, 1],
+        padding=[[2, 1], [1, 2]],
+        dilations=[2, 3])
+
+  def testConv2DExplicitPaddingWithLayoutOptimizer(self):
+    # Test with Grappler's layout optimizer, to ensure the layout optimizer
+    # handles explicit padding correctly.
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[1, 3, 2, 1],
+        filter_in_sizes=[1, 2, 1, 2],
+        strides=[1, 1],
+        padding=[[1, 0], [0, 1]],
+        dilations=[2, 1],
+        test_grappler_layout_optimizer=True)
+
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[1, 2, 3, 2],
+        filter_in_sizes=[3, 2, 2, 1],
+        strides=[1, 1],
+        padding=[[2, 1], [1, 2]],
+        dilations=[2, 3],
+        test_grappler_layout_optimizer=True)
+
   # TODO(yzhwang): this currently fails.
   # self._VerifyValues(tensor_in_sizes=[1, 8, 8, 1],
   #                   filter_in_sizes=[2, 2, 1, 1],
@@ -517,19 +700,22 @@ class Conv2DTest(test.TestCase):
   #                   expected=[72, 112, 392, 432])
 
   # Testing for backprops
-  def _RunAndVerifyBackpropInput(self, input_sizes, filter_sizes, output_sizes,
-                                 strides, padding, expected, data_format,
-                                 use_gpu, err):
-    total_output_size = 1
-    total_filter_size = 1
-    for s in output_sizes:
-      total_output_size *= s
-    for s in filter_sizes:
-      total_filter_size *= s
-    # Initializes the input tensor with array containing incrementing
-    # numbers from 1.
-    x1 = [f * 1.0 for f in range(1, total_filter_size + 1)]
-    x2 = [f * 1.0 for f in range(1, total_output_size + 1)]
+  def _RunAndVerifyBackpropInput(self,
+                                 input_sizes,
+                                 filter_sizes,
+                                 output_sizes,
+                                 strides,
+                                 padding,
+                                 expected,
+                                 data_format,
+                                 use_gpu,
+                                 err,
+                                 dilations=(1, 1)):
+    if use_gpu and not test.is_gpu_available(cuda_only=True):
+      return
+    x1 = self._CreateNumpyTensor(filter_sizes)
+    x2 = self._CreateNumpyTensor(output_sizes)
+    dilations = list(dilations)
     with test_util.device(use_gpu):
       if data_format == "NCHW":
         input_sizes = test_util.NHWCToNCHW(input_sizes)
@@ -537,18 +723,30 @@ class Conv2DTest(test.TestCase):
       t1 = constant_op.constant(x1, shape=filter_sizes)
       t2 = constant_op.constant(x2, shape=output_sizes)
       strides = [1] + strides + [1]
+      dilations = [1] + dilations + [1]
+      if isinstance(padding, (list, tuple)):
+        padding = [(0, 0)] + padding + [(0, 0)]
       if data_format == "NCHW":
         t2 = test_util.NHWCToNCHW(t2)
         strides = test_util.NHWCToNCHW(strides)
+        dilations = test_util.NHWCToNCHW(dilations)
+        if isinstance(padding, (list, tuple)):
+          padding = test_util.NHWCToNCHW((padding))
       conv = nn_ops.conv2d_backprop_input(
-          t0, t1, t2, strides=strides, padding=padding, data_format=data_format)
+          t0,
+          t1,
+          t2,
+          strides=strides,
+          padding=padding,
+          data_format=data_format,
+          dilations=dilations)
       if data_format == "NCHW":
         conv = test_util.NCHWToNHWC(conv)
       # "values" consists of two tensors for two backprops
       value = self.evaluate(conv)
       self.assertShapeEqual(value, conv)
-    tf_logging.info("expected = ", expected)
-    tf_logging.info("actual = ", value)
+    tf_logging.debug("expected = %s", expected)
+    tf_logging.debug("actual = %s", value)
     self.assertArrayNear(expected, value.flatten(), err)
 
   def _CompareBackpropInput(self, input_sizes, filter_sizes, output_sizes,
@@ -691,41 +889,51 @@ class Conv2DTest(test.TestCase):
           err=1e-5)
 
   # Testing for backprops
-  def _RunAndVerifyBackpropFilter(self, input_sizes, filter_sizes, output_sizes,
-                                  strides, padding, expected, data_format,
-                                  use_gpu):
-    total_input_size = 1
-    total_output_size = 1
-    for s in input_sizes:
-      total_input_size *= s
-    for s in output_sizes:
-      total_output_size *= s
-    # Initializes the input tensor with array containing incrementing
-    # numbers from 1.
-    x0 = [f * 1.0 for f in range(1, total_input_size + 1)]
-    x2 = [f * 1.0 for f in range(1, total_output_size + 1)]
+  def _RunAndVerifyBackpropFilter(self,
+                                  input_sizes,
+                                  filter_sizes,
+                                  output_sizes,
+                                  strides,
+                                  padding,
+                                  expected,
+                                  data_format,
+                                  use_gpu,
+                                  dilations=(1, 1),
+                                  err=1e-5):
+    x0 = self._CreateNumpyTensor(input_sizes)
+    x2 = self._CreateNumpyTensor(output_sizes)
+    dilations = list(dilations)
+    explicit_strides = [1] + strides + [1]
+    new_padding = padding
+    new_dilations = [1] + dilations + [1]
+    if isinstance(new_padding, (list, tuple)):
+      new_padding = [(0, 0)] + new_padding + [(0, 0)]
+    if data_format == "NCHW":
+      explicit_strides = test_util.NHWCToNCHW(explicit_strides)
+      new_dilations = test_util.NHWCToNCHW(new_dilations)
+      if isinstance(padding, (list, tuple)):
+        new_padding = test_util.NHWCToNCHW(new_padding)
     for dtype in self._DtypesToTest(use_gpu=use_gpu):
       with test_util.device(use_gpu):
         t0 = constant_op.constant(x0, shape=input_sizes, dtype=dtype)
         t1 = constant_op.constant(filter_sizes, shape=[len(filter_sizes)])
         t2 = constant_op.constant(x2, shape=output_sizes, dtype=dtype)
-        explicit_strides = [1] + strides + [1]
         if data_format == "NCHW":
           t0 = test_util.NHWCToNCHW(t0)
           t2 = test_util.NHWCToNCHW(t2)
-          explicit_strides = test_util.NHWCToNCHW(explicit_strides)
         conv = nn_ops.conv2d_backprop_filter(
             t0,
             t1,
             t2,
             strides=explicit_strides,
-            padding=padding,
+            padding=new_padding,
+            dilations=new_dilations,
             data_format=data_format)
         value = self.evaluate(conv)
         self.assertShapeEqual(value, conv)
-      tf_logging.info("expected = ", expected)
-      tf_logging.info("actual = ", value)
-      self.assertArrayNear(expected, value.flatten(), 1e-5)
+      tf_logging.debug("expected = %s", expected)
+      tf_logging.debug("actual = %s", value)
+      self.assertArrayNear(expected, value.flatten(), err)
 
   def _CompareBackFilter(self, input_sizes, filter_sizes, output_sizes,
                          conv_strides, padding):
@@ -866,16 +1074,8 @@ class Conv2DTest(test.TestCase):
   def _RunAndVerifyBackpropInputDilation(self, input_sizes, filter_sizes,
                                          output_sizes, strides, dilations,
                                          padding, data_format, use_gpu, err):
-    total_input_size = 1
-    total_filter_size = 1
-    for s in input_sizes:
-      total_input_size *= s
-    for s in filter_sizes:
-      total_filter_size *= s
-    # Initializes the input tensor with array containing incrementing
-    # numbers from 1.
-    x1 = [f * 1.0 for f in range(1, total_input_size + 1)]
-    x2 = [f * 1.0 for f in range(1, total_filter_size + 1)]
+    x1 = self._CreateNumpyTensor(input_sizes)
+    x2 = self._CreateNumpyTensor(filter_sizes)
     default_dilations = (dilations[0] == 1 and dilations[1] == 1)
     if default_dilations or use_gpu:
       with self.cached_session(use_gpu=use_gpu) as sess:
@@ -912,24 +1112,16 @@ class Conv2DTest(test.TestCase):
         value_2 = self.evaluate(conv_2)
         self.assertShapeEqual(value, conv)
         self.assertShapeEqual(value_2, conv_2)
-      tf_logging.info("expected = ", value_2)
-      tf_logging.info("actual = ", value)
+      tf_logging.debug("expected = %s", value_2)
+      tf_logging.debug("actual = %s", value)
       self.assertArrayNear(value_2.flatten(), value.flatten(), err)
 
   # Testing for backprops
   def _RunAndVerifyBackpropFilterDilation(self, input_sizes, filter_sizes,
                                           output_sizes, strides, dilations,
                                           padding, data_format, use_gpu, err):
-    total_input_size = 1
-    total_filter_size = 1
-    for s in input_sizes:
-      total_input_size *= s
-    for s in filter_sizes:
-      total_filter_size *= s
-    # Initializes the input tensor with array containing incrementing
-    # numbers from 1.
-    x1 = [f * 1.0 for f in range(1, total_input_size + 1)]
-    x2 = [f * 1.0 for f in range(1, total_filter_size + 1)]
+    x1 = self._CreateNumpyTensor(input_sizes)
+    x2 = self._CreateNumpyTensor(filter_sizes)
     default_dilations = (dilations[0] == 1 and dilations[1] == 1)
     if default_dilations or use_gpu:
       with self.cached_session(use_gpu=use_gpu) as sess:
@@ -965,8 +1157,8 @@ class Conv2DTest(test.TestCase):
         value_2 = self.evaluate(conv_2)
         self.assertShapeEqual(value, conv)
         self.assertShapeEqual(value_2, conv_2)
-      tf_logging.info("expected = ", value_2)
-      tf_logging.info("actual = ", value)
+      tf_logging.debug("expected = %s", value_2)
+      tf_logging.debug("actual = %s", value)
       self.assertArrayNear(value_2.flatten(), value.flatten(), err)
 
   def testConv2D2x2Depth3ValidBackpropFilterStride1x1Dilation2x1(self):
@@ -1111,20 +1303,347 @@ class Conv2DTest(test.TestCase):
             use_gpu=use_gpu,
             err=1e-5)
 
+  def _RunAndVerifyBackpropInputExplicitPadding(self,
+                                                input_sizes,
+                                                filter_sizes,
+                                                output_sizes,
+                                                strides,
+                                                padding,
+                                                data_format,
+                                                dilations=(1, 1),
+                                                err=2e-5):
+    x1 = self._CreateNumpyTensor(filter_sizes)
+    x2 = self._CreateNumpyTensor(output_sizes)
+    dilations = list(dilations)
+    padded_input_sizes = input_sizes[:]
+    padded_input_sizes[1] += padding[0][0] + padding[0][1]
+    padded_input_sizes[2] += padding[1][0] + padding[1][1]
+    c = nn_ops.conv2d_backprop_input(
+        padded_input_sizes,
+        x1,
+        x2,
+        strides=[1] + strides + [1],
+        padding="VALID",
+        dilations=[1] + dilations + [1])
+    c = c[:, padding[0][0]:(c.shape[1] - padding[0][1]), padding[1][0]:(
+        c.shape[2] - padding[1][1]), :]
+    expected = list(self.evaluate(array_ops.reshape(c, [-1])))
+    self._RunAndVerifyBackpropInput(
+        input_sizes,
+        filter_sizes,
+        output_sizes,
+        strides,
+        padding,
+        expected,
+        data_format,
+        use_gpu=True,
+        err=err,
+        dilations=dilations)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2D2x2Depth1Padding0x0BackpropInput(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self._RunAndVerifyBackpropInputExplicitPadding(
+            input_sizes=[1, 2, 3, 1],
+            filter_sizes=[2, 2, 1, 1],
+            output_sizes=[1, 1, 2, 1],
+            strides=[1, 1],
+            padding=[[0, 0], [0, 0]],
+            data_format=data_format)
+
+        self._RunAndVerifyBackpropInputExplicitPadding(
+            input_sizes=[1, 3, 4, 2],
+            filter_sizes=[2, 2, 2, 3],
+            output_sizes=[1, 1, 2, 3],
+            strides=[2, 2],
+            padding=[[0, 0], [0, 0]],
+            data_format=data_format)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2D2x2Depth1Padding1x1BackpropInput(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self._RunAndVerifyBackpropInputExplicitPadding(
+            input_sizes=[1, 2, 3, 1],
+            filter_sizes=[2, 2, 1, 2],
+            output_sizes=[1, 3, 4, 2],
+            strides=[1, 1],
+            padding=[[1, 1], [1, 1]],
+            data_format=data_format, err=1e-4)
+
+        self._RunAndVerifyBackpropInputExplicitPadding(
+            input_sizes=[1, 2, 3, 2],
+            filter_sizes=[1, 1, 2, 1],
+            output_sizes=[1, 4, 3, 1],
+            strides=[1, 2],
+            padding=[[1, 1], [1, 1]],
+            data_format=data_format)
+
+        self._RunAndVerifyBackpropInputExplicitPadding(
+            input_sizes=[1, 4, 3, 1],
+            filter_sizes=[2, 2, 1, 1],
+            output_sizes=[1, 4, 2, 1],
+            strides=[1, 2],
+            padding=[[1, 1], [1, 1]],
+            data_format=data_format,
+            dilations=[2, 2])
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2D2x2Depth1Padding2x2BackpropInput(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self._RunAndVerifyBackpropInputExplicitPadding(
+            input_sizes=[2, 3, 1, 1],
+            filter_sizes=[2, 1, 1, 1],
+            output_sizes=[2, 2, 5, 1],
+            strides=[3, 1],
+            padding=[[2, 2], [2, 2]],
+            data_format=data_format)
+
+        self._RunAndVerifyBackpropInputExplicitPadding(
+            input_sizes=[1, 3, 6, 1],
+            filter_sizes=[3, 2, 1, 1],
+            output_sizes=[1, 3, 4, 1],
+            strides=[1, 2],
+            padding=[[2, 2], [2, 2]],
+            data_format=data_format,
+            dilations=[2, 3])
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2D2x2Depth1Padding_1_8_4_1_BackpropInput(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self._RunAndVerifyBackpropInputExplicitPadding(
+            input_sizes=[1, 2, 3, 1],
+            filter_sizes=[2, 2, 1, 1],
+            output_sizes=[1, 10, 8, 1],
+            strides=[1, 1],
+            padding=[[1, 8], [4, 2]],
+            data_format=data_format, err=5e-5)
+
+        self._RunAndVerifyBackpropInputExplicitPadding(
+            input_sizes=[1, 5, 3, 1],
+            filter_sizes=[3, 2, 1, 1],
+            output_sizes=[1, 4, 8, 1],
+            strides=[3, 1],
+            padding=[[1, 8], [4, 2]],
+            data_format=data_format)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2D2x2Depth1Padding_5_0_2_2_BackpropInput(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self._RunAndVerifyBackpropInputExplicitPadding(
+            input_sizes=[1, 3, 3, 1],
+            filter_sizes=[2, 1, 1, 1],
+            output_sizes=[1, 7, 7, 1],
+            strides=[1, 1],
+            padding=[[5, 0], [2, 2]],
+            data_format=data_format,
+            err=5e-5)
+
+        self._RunAndVerifyBackpropInputExplicitPadding(
+            input_sizes=[1, 4, 2, 1],
+            filter_sizes=[3, 3, 1, 1],
+            output_sizes=[1, 5, 2, 1],
+            strides=[1, 2],
+            padding=[[5, 0], [2, 2]],
+            data_format=data_format,
+            dilations=[2, 1])
+
+  def _RunAndVerifyBackpropFilterExplicitPadding(self,
+                                                 input_sizes,
+                                                 filter_sizes,
+                                                 output_sizes,
+                                                 strides,
+                                                 padding,
+                                                 data_format,
+                                                 dilations=(1, 1),
+                                                 err=1e-5):
+    x0 = self._CreateNumpyTensor(input_sizes)
+    x2 = self._CreateNumpyTensor(output_sizes)
+    dilations = list(dilations)
+
+    x0 = np.pad(x0, [(0, 0)] + padding + [(0, 0)], "constant")
+    c = nn_ops.conv2d_backprop_filter(
+        x0,
+        filter_sizes,
+        x2,
+        strides=[1] + strides + [1],
+        padding="VALID",
+        dilations=[1] + dilations + [1])
+    expected = list(self.evaluate(array_ops.reshape(c, [-1])))
+    self._RunAndVerifyBackpropFilter(
+        input_sizes,
+        filter_sizes,
+        output_sizes,
+        strides,
+        padding,
+        expected,
+        data_format,
+        use_gpu=True,
+        dilations=dilations,
+        err=err)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2D2x2Depth1Padding0x0BackpropFilter(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self._RunAndVerifyBackpropFilterExplicitPadding(
+            input_sizes=[1, 2, 3, 1],
+            filter_sizes=[2, 2, 1, 1],
+            output_sizes=[1, 1, 2, 1],
+            strides=[1, 1],
+            padding=[[0, 0], [0, 0]],
+            data_format=data_format)
+
+        self._RunAndVerifyBackpropFilterExplicitPadding(
+            input_sizes=[1, 3, 4, 2],
+            filter_sizes=[2, 2, 2, 3],
+            output_sizes=[1, 1, 2, 3],
+            strides=[2, 2],
+            padding=[[0, 0], [0, 0]],
+            data_format=data_format)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2D2x2Depth1Padding1x1BackpropFilter(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self._RunAndVerifyBackpropFilterExplicitPadding(
+            input_sizes=[1, 2, 3, 1],
+            filter_sizes=[2, 2, 1, 2],
+            output_sizes=[1, 3, 4, 2],
+            strides=[1, 1],
+            padding=[[1, 1], [1, 1]],
+            data_format=data_format,
+            err=5e-5)
+
+        self._RunAndVerifyBackpropFilterExplicitPadding(
+            input_sizes=[1, 2, 3, 2],
+            filter_sizes=[1, 1, 2, 1],
+            output_sizes=[1, 4, 3, 1],
+            strides=[1, 2],
+            padding=[[1, 1], [1, 1]],
+            data_format=data_format)
+
+        self._RunAndVerifyBackpropFilterExplicitPadding(
+            input_sizes=[1, 4, 3, 1],
+            filter_sizes=[2, 2, 1, 1],
+            output_sizes=[1, 4, 2, 1],
+            strides=[1, 2],
+            padding=[[1, 1], [1, 1]],
+            data_format=data_format,
+            dilations=[2, 2])
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2D2x2Depth1Padding2x2BackpropFilter(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self._RunAndVerifyBackpropFilterExplicitPadding(
+            input_sizes=[2, 3, 1, 1],
+            filter_sizes=[2, 1, 1, 1],
+            output_sizes=[2, 2, 5, 1],
+            strides=[3, 1],
+            padding=[[2, 2], [2, 2]],
+            data_format=data_format)
+
+        self._RunAndVerifyBackpropFilterExplicitPadding(
+            input_sizes=[1, 3, 6, 1],
+            filter_sizes=[3, 2, 1, 1],
+            output_sizes=[1, 3, 4, 1],
+            strides=[1, 2],
+            padding=[[2, 2], [2, 2]],
+            data_format=data_format,
+            dilations=[2, 3])
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2D2x2Depth1Padding_1_8_4_1_BackpropFilter(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self._RunAndVerifyBackpropFilterExplicitPadding(
+            input_sizes=[1, 2, 3, 1],
+            filter_sizes=[2, 2, 1, 1],
+            output_sizes=[1, 10, 8, 1],
+            strides=[1, 1],
+            padding=[[1, 8], [4, 2]],
+            data_format=data_format,
+            err=1e-4)
+
+        self._RunAndVerifyBackpropFilterExplicitPadding(
+            input_sizes=[1, 5, 3, 1],
+            filter_sizes=[3, 2, 1, 1],
+            output_sizes=[1, 4, 8, 1],
+            strides=[3, 1],
+            padding=[[1, 8], [4, 2]],
+            data_format=data_format)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2D2x2Depth1Padding_5_0_2_2_BackpropFilter(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self._RunAndVerifyBackpropFilterExplicitPadding(
+            input_sizes=[1, 3, 3, 1],
+            filter_sizes=[2, 1, 1, 1],
+            output_sizes=[1, 7, 7, 1],
+            strides=[1, 1],
+            padding=[[5, 0], [2, 2]],
+            data_format=data_format,
+            err=1e-4)
+
+        self._RunAndVerifyBackpropFilterExplicitPadding(
+            input_sizes=[1, 4, 2, 1],
+            filter_sizes=[3, 3, 1, 1],
+            output_sizes=[1, 5, 2, 1],
+            strides=[1, 2],
+            padding=[[5, 0], [2, 2]],
+            data_format=data_format,
+            dilations=[2, 1])
+
   # Gradient checkers
   def ConstructAndTestGradient(self, batch, input_rows, input_cols, filter_rows,
                                filter_cols, in_depth, out_depth, stride_rows,
                                stride_cols, padding, test_input, data_format,
-                               use_gpu):
+                               use_gpu, max_err=0.002):
     input_shape = [batch, input_rows, input_cols, in_depth]
     filter_shape = [filter_rows, filter_cols, in_depth, out_depth]
     # TODO(yangke): re-factor the computation of output shape.
     if padding == "VALID":
       output_rows = (input_rows - filter_rows + stride_rows) // stride_rows
       output_cols = (input_cols - filter_cols + stride_cols) // stride_cols
-    else:
+    elif padding == "SAME":
       output_rows = (input_rows + stride_rows - 1) // stride_rows
       output_cols = (input_cols + stride_cols - 1) // stride_cols
+    else:
+      self.assertIsInstance(padding, (list, tuple))
+      output_rows = (input_rows + padding[1][0] + padding[1][1] - filter_rows +
+                     stride_rows) // stride_rows
+      output_cols = (input_cols + padding[2][0] + padding[2][1] - filter_cols +
+                     stride_cols) // stride_cols
     output_shape = [batch, output_rows, output_cols, out_depth]
     input_size = 1
     for x in input_shape:
@@ -1145,16 +1664,19 @@ class Conv2DTest(test.TestCase):
         filter_tensor = constant_op.constant(
             filter_data, shape=filter_shape, dtype=dtype, name="filter")
         strides = [1, stride_rows, stride_cols, 1]
+        new_padding = padding
         if data_format == "NCHW":
           new_input_tensor = test_util.NHWCToNCHW(input_tensor)
           strides = test_util.NHWCToNCHW(strides)
+          if isinstance(padding, (list, tuple)):
+            new_padding = test_util.NHWCToNCHW(padding)
         else:
           new_input_tensor = input_tensor
         conv = nn_ops.conv2d(
             new_input_tensor,
             filter_tensor,
             strides,
-            padding,
+            new_padding,
             data_format=data_format,
             name="conv")
         if data_format == "NCHW":
@@ -1178,8 +1700,8 @@ class Conv2DTest(test.TestCase):
           # since fp16 numerical gradients are too imprecise.
           err = np.fabs(jacob_t - reference_jacob_t).max()
 
-        tf_logging.info("conv_2d gradient error = ", err)
-        self.assertLess(err, 0.002)
+        tf_logging.debug("conv_2d gradient error = %s", err)
+        self.assertLess(err, max_err)
 
   def testInputGradientValidPaddingStrideOne(self):
     for (data_format, use_gpu) in GetTestConfigs():
@@ -1436,6 +1958,248 @@ class Conv2DTest(test.TestCase):
           data_format=data_format,
           use_gpu=use_gpu)
 
+  def testInputGradient1x1PaddingStrideOne(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self.ConstructAndTestGradient(
+            batch=2,
+            input_rows=5,
+            input_cols=4,
+            filter_rows=3,
+            filter_cols=3,
+            in_depth=2,
+            out_depth=3,
+            stride_rows=1,
+            stride_cols=1,
+            padding=[[0, 0], [1, 1], [1, 1], [0, 0]],
+            test_input=True,
+            data_format=data_format,
+            use_gpu=use_gpu,
+            max_err=0.0025)
+
+  def testFilterGradient1x1PaddingStrideOne(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self.ConstructAndTestGradient(
+            batch=2,
+            input_rows=5,
+            input_cols=4,
+            filter_rows=3,
+            filter_cols=3,
+            in_depth=2,
+            out_depth=3,
+            stride_rows=1,
+            stride_cols=1,
+            padding=[[0, 0], [1, 1], [1, 1], [0, 0]],
+            test_input=False,
+            data_format=data_format,
+            use_gpu=use_gpu)
+
+  def testInputGradient1x1PaddingStrideTwo(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self.ConstructAndTestGradient(
+            batch=2,
+            input_rows=4,
+            input_cols=5,
+            filter_rows=3,
+            filter_cols=3,
+            in_depth=2,
+            out_depth=3,
+            stride_rows=2,
+            stride_cols=2,
+            padding=[[0, 0], [1, 1], [1, 1], [0, 0]],
+            test_input=True,
+            data_format=data_format,
+            use_gpu=use_gpu)
+
+  def testFilterGradient1x1PaddingStrideTwo(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self.ConstructAndTestGradient(
+            batch=2,
+            input_rows=4,
+            input_cols=5,
+            filter_rows=3,
+            filter_cols=3,
+            in_depth=2,
+            out_depth=3,
+            stride_rows=2,
+            stride_cols=2,
+            padding=[[0, 0], [1, 1], [1, 1], [0, 0]],
+            test_input=False,
+            data_format=data_format,
+            use_gpu=use_gpu)
+
+  def testInputGradient2x2PaddingStrideOne(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self.ConstructAndTestGradient(
+            batch=2,
+            input_rows=5,
+            input_cols=4,
+            filter_rows=3,
+            filter_cols=3,
+            in_depth=2,
+            out_depth=3,
+            stride_rows=1,
+            stride_cols=1,
+            padding=[[0, 0], [2, 2], [2, 2], [0, 0]],
+            test_input=True,
+            data_format=data_format,
+            use_gpu=use_gpu)
+
+  def testFilterGradient2x2PaddingStrideOne(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self.ConstructAndTestGradient(
+            batch=2,
+            input_rows=5,
+            input_cols=4,
+            filter_rows=3,
+            filter_cols=3,
+            in_depth=2,
+            out_depth=3,
+            stride_rows=1,
+            stride_cols=1,
+            padding=[[0, 0], [2, 2], [2, 2], [0, 0]],
+            test_input=False,
+            data_format=data_format,
+            use_gpu=use_gpu,
+            max_err=0.003)
+
+  def testInputGradient1_2_3_4PaddingStride3x2(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self.ConstructAndTestGradient(
+            batch=2,
+            input_rows=8,
+            input_cols=5,
+            filter_rows=4,
+            filter_cols=2,
+            in_depth=3,
+            out_depth=2,
+            stride_rows=3,
+            stride_cols=2,
+            padding=[[0, 0], [1, 2], [3, 4], [0, 0]],
+            test_input=True,
+            data_format=data_format,
+            use_gpu=use_gpu)
+
+  def testFilterGradient1_2_3_4PaddingStride3x2(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self.ConstructAndTestGradient(
+            batch=2,
+            input_rows=8,
+            input_cols=5,
+            filter_rows=4,
+            filter_cols=2,
+            in_depth=3,
+            out_depth=2,
+            stride_rows=3,
+            stride_cols=2,
+            padding=[[0, 0], [1, 2], [3, 4], [0, 0]],
+            test_input=False,
+            data_format=data_format,
+            use_gpu=use_gpu)
+
+  def testInputGradient4_3_2_1PaddingStride2x1(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self.ConstructAndTestGradient(
+            batch=3,
+            input_rows=5,
+            input_cols=7,
+            filter_rows=3,
+            filter_cols=2,
+            in_depth=1,
+            out_depth=2,
+            stride_rows=2,
+            stride_cols=1,
+            padding=[[0, 0], [4, 3], [2, 1], [0, 0]],
+            test_input=True,
+            data_format=data_format,
+            use_gpu=use_gpu)
+
+  def testFilterGradient4_3_2_1PaddingStride2x1(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self.ConstructAndTestGradient(
+            batch=3,
+            input_rows=5,
+            input_cols=7,
+            filter_rows=3,
+            filter_cols=2,
+            in_depth=1,
+            out_depth=2,
+            stride_rows=2,
+            stride_cols=1,
+            padding=[[0, 0], [4, 3], [2, 1], [0, 0]],
+            test_input=False,
+            data_format=data_format,
+            use_gpu=use_gpu)
+
+  def testInputGradient0_0_0_5PaddingStride1x2(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self.ConstructAndTestGradient(
+            batch=2,
+            input_rows=6,
+            input_cols=7,
+            filter_rows=3,
+            filter_cols=4,
+            in_depth=3,
+            out_depth=2,
+            stride_rows=1,
+            stride_cols=2,
+            padding=[[0, 0], [0, 0], [0, 5], [0, 0]],
+            test_input=True,
+            data_format=data_format,
+            use_gpu=use_gpu)
+
+  def testFilterGradient0_0_0_5PaddingStride1x2(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self.ConstructAndTestGradient(
+            batch=2,
+            input_rows=6,
+            input_cols=7,
+            filter_rows=3,
+            filter_cols=4,
+            in_depth=3,
+            out_depth=2,
+            stride_rows=1,
+            stride_cols=2,
+            padding=[[0, 0], [0, 0], [0, 5], [0, 0]],
+            test_input=False,
+            data_format=data_format,
+            use_gpu=use_gpu)
+
   def testShapeFunctionEdgeCases(self):
     # All shapes unknown.
     c1 = nn_ops.conv2d(
@@ -1473,6 +2237,55 @@ class Conv2DTest(test.TestCase):
           strides=[1, 1, 1, 1],
           padding="SAME")
 
+    # Negative padding.
+    with self.assertRaises(ValueError):
+      nn_ops.conv2d(
+          array_ops.placeholder(dtypes.float32),
+          array_ops.placeholder(dtypes.float32),
+          strides=[1, 1, 1, 1],
+          padding=[[0, 0], [0, -1], [1, 2], [0, 0]])
+
+    # Nonzero padding in nonspatial dimension.
+    with self.assertRaises(ValueError):
+      nn_ops.conv2d(
+          array_ops.placeholder(dtypes.float32),
+          array_ops.placeholder(dtypes.float32),
+          strides=[1, 1, 1, 1],
+          padding=[[1, 0], [0, 0], [0, 0], [0, 0]])
+
+    # Nonzero NCHW padding in nonspatial dimension.
+    with self.assertRaises(ValueError):
+      nn_ops.conv2d(
+          array_ops.placeholder(dtypes.float32),
+          array_ops.placeholder(dtypes.float32),
+          strides=[1, 1, 1, 1],
+          padding=[[0, 0], [0, 1], [0, 0], [0, 0]],
+          data_format="NCHW")
+
+    # Wrong amount of padding
+    with self.assertRaises(ValueError):
+      nn_ops.conv2d(
+          array_ops.placeholder(dtypes.float32),
+          array_ops.placeholder(dtypes.float32),
+          strides=[1, 1, 1, 1],
+          padding=[[0, 0], [0, 0], [0, 0]])
+
+    # Only specify one padding amount per dimension
+    with self.assertRaises(ValueError):
+      nn_ops.conv2d(
+          array_ops.placeholder(dtypes.float32),
+          array_ops.placeholder(dtypes.float32),
+          strides=[1, 1, 1, 1],
+          padding=[[0], [0], [0], [0]])
+
+    # Explicit padding elements are not lists
+    with self.assertRaises(ValueError):
+      nn_ops.conv2d(
+          array_ops.placeholder(dtypes.float32),
+          array_ops.placeholder(dtypes.float32),
+          strides=[1, 1, 1, 1],
+          padding=[0, 0, 0, 0])
+
   def testOpEdgeCases(self):
     with self.cached_session() as sess:
       # Illegal strides.
@@ -1513,6 +2326,41 @@ class Conv2DTest(test.TestCase):
                 strides=[1, 1, 1, 1],
                 padding="VALID"))
 
+      # Filter larger than input + padding.
+      with self.assertRaisesRegexp(ValueError, "Negative dimension size"):
+        sess.run(
+            nn_ops.conv2d(
+                array_ops.placeholder(dtypes.float32, shape=[32, 20, 20, 3]),
+                array_ops.placeholder(dtypes.float32, shape=[24, 25, 3, 2]),
+                strides=[1, 1, 1, 1],
+                padding=[[0, 0], [2, 2], [2, 2], [0, 0]]))
+
+    if test.is_gpu_available(cuda_only=True):
+      with self.test_session(use_gpu=True):
+        # Negative padding during backprop.
+        with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
+                                     "nonnegative"):
+          sess.run(
+              nn_ops.conv2d_backprop_input([32, 20, 20, 3],
+                                           array_ops.placeholder(
+                                               dtypes.float32,
+                                               shape=[18, 18, 3, 2]),
+                                           array_ops.placeholder(
+                                               dtypes.float32,
+                                               shape=[32, 3, 2, 2]),
+                                           strides=[1, 1, 1, 1],
+                                           padding=[[0, 0], [-1, 0], [0, 0],
+                                                    [0, 0]]))
+        with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
+                                     "nonnegative"):
+          sess.run(
+              nn_ops.conv2d_backprop_filter(
+                  array_ops.placeholder(dtypes.float32, shape=[32, 20, 20, 3]),
+                  [18, 18, 3, 2],
+                  array_ops.placeholder(dtypes.float32, shape=[32, 3, 2, 2]),
+                  strides=[1, 1, 1, 1],
+                  padding=[[0, 0], [-1, 0], [0, 0], [0, 0]]))
+
 
 class DepthwiseConv2DTest(test.TestCase):
 
@@ -1546,7 +2394,7 @@ class DepthwiseConv2DTest(test.TestCase):
       conv = nn_impl.depthwise_conv2d(
           t1, t2, strides=[1, stride, stride, 1], padding=padding)
       value = self.evaluate(conv)
-    tf_logging.info("value = ", value)
+    tf_logging.debug("value = %s", value)
     self.assertArrayNear(expected, np.ravel(value), 1e-5)
     self.assertShapeEqual(value, conv)
 
@@ -1668,7 +2516,7 @@ class SeparableConv2DTest(test.TestCase):
         conv = array_ops.transpose(conv, [0, 2, 3, 1])
 
       value = self.evaluate(conv)
-    tf_logging.info("value = ", value)
+    tf_logging.debug("value = %s", value)
     self.assertArrayNear(expected, np.ravel(value), 1e-3)
     self.assertShapeEqual(value, conv)
 
@@ -1828,6 +2676,194 @@ class Conv2DBenchmark(test.Benchmark):
             name="conv_stack_iter_%d" % iter_index, wall_time=wall_time)
         tf_logging.info("conv_stack_iter_%d: %.4f" % (iter_index, wall_time))
 
+  def _bench_op(self, name, op, burn_iters, num_iters):
+    config = config_pb2.ConfigProto()
+    # Prevent Grappler from optimizing away the entire graph.
+    config.graph_options.rewrite_options.dependency_optimization = (
+        rewriter_config_pb2.RewriterConfig.OFF)
+    with session_lib.Session(config=config) as session:
+      variables.global_variables_initializer().run()
+      self.run_op_benchmark(
+          session, op, burn_iters=burn_iters, min_iters=num_iters, name=name)
+
+  def benchmarkExplicitVsManualPadding(self):
+    """Compare performance of EXPLICIT padding and calling tf.pad.
+
+    A Conv2D op with EXPLICIT padding is benchmarked, and a tf.pad with the same
+    padding followed by an equivalent Conv2D op is benchmarked.
+    """
+    if not test.is_gpu_available():
+      return
+
+    with ops.Graph().as_default():
+      burn_iters = 15
+      num_iters = 300
+      batch_size = 64
+      # The input and filter correspond to the first layer of Resnet50.
+      input = variables.Variable(  # pylint: disable=redefined-builtin
+          random_ops.random_uniform([
+              batch_size,
+              3,
+              224,
+              224
+          ]))
+      filter = variables.Variable(random_ops.random_uniform([7, 7, 3, 64]))  # pylint: disable=redefined-builtin
+      strides = [1, 1, 2, 2]
+      padding = [(0, 0), (0, 0), (3, 3), (3, 3)]
+      output_explicit_pad = nn_ops.conv2d(
+          input, filter, strides, padding=padding, data_format="NCHW")
+      input_padded = array_ops.pad(input, padding)
+      output_manual_pad = nn_ops.conv2d(
+          input_padded, filter, strides, padding="VALID", data_format="NCHW")
+      # Benchmark just the forward pass.
+      self._bench_op("explicit_pad_forward", output_explicit_pad.op, burn_iters,
+                     num_iters)
+      self._bench_op("manual_pad_forward", output_manual_pad.op, burn_iters,
+                     num_iters)
+
+      # Benchmark both the forward and backwards passes.
+      input_grad_explicit_pad, filter_grad_explicit_pad = (
+          gradients_impl.gradients(output_explicit_pad, [input, filter]))
+      self._bench_op(
+          "explicit_pad_backward",
+          control_flow_ops.group(input_grad_explicit_pad,
+                                 filter_grad_explicit_pad), burn_iters,
+          num_iters)
+      input_grad_manual_pad, filter_grad_manual_pad = gradients_impl.gradients(
+          output_manual_pad, [input, filter])
+      self._bench_op(
+          "manual_pad_backward",
+          control_flow_ops.group(input_grad_manual_pad, filter_grad_manual_pad),
+          burn_iters, num_iters)
+
+  def benchmarkExplicitVsSamePaddingGraph(self):
+    """Compare performance of EXPLICIT and SAME padding in graph mode.
+
+    A Conv2D op with SAME padding is benchmarked, and an equivalent Conv2D op
+    with explicit padding is benchmarked, where the padding is the same as in
+    the SAME case. The purpose is to ensure EXPLICIT padding is just as
+    efficient as the SAME case
+    """
+    if not test.is_gpu_available():
+      return
+
+    with ops.Graph().as_default():
+      burn_iters = 15
+      num_convs = 20
+      num_iters = 50
+      batch_size = 64
+      # The input and filter correspond to a middle layer of Resnet50.
+      input = variables.Variable(  # pylint: disable=redefined-builtin
+          random_ops.random_uniform([
+              batch_size,
+              256,
+              14,
+              14
+          ]))
+      filter = variables.Variable(random_ops.random_uniform([3, 3, 256, 256]))  # pylint: disable=redefined-builtin
+      strides = [1, 1, 1, 1]
+      padding = [(0, 0), (0, 0), (1, 1), (1, 1)]
+      output_explicit_pad = input
+      output_same_pad = input
+
+      for _ in range(num_convs):
+        output_explicit_pad = nn_ops.conv2d(
+            output_explicit_pad,
+            filter,
+            strides,
+            padding=padding,
+            data_format="NCHW")
+        output_same_pad = nn_ops.conv2d(
+            output_same_pad,
+            filter,
+            strides,
+            padding="SAME",
+            data_format="NCHW")
+      grad_explicit_pad, = gradients_impl.gradients(output_explicit_pad, filter)
+      grad_same_pad, = gradients_impl.gradients(output_same_pad, filter)
+      self._bench_op("graph_explicit_pad", grad_explicit_pad.op, burn_iters,
+                     num_iters)
+      self._bench_op("graph_same_pad", grad_same_pad.op, burn_iters, num_iters)
+
+  def benchmarkExplicitVsSamePaddingEager(self):
+    """Compare performance of EXPLICIT and SAME padding in eager mode.
+
+    A Conv2D op with SAME padding is benchmarked, and an equivalent Conv2D op
+    with explicit padding is benchmarked, where the padding is the same as in
+    the SAME case. Currently, EXPLICIT padding is slightly slower, due to the
+    fact the Python padding list must be checked and processed before the Conv2D
+    op can run.
+    """
+    # TODO(reedwm): Make EXPLICIT padding as fast as SAME padding.
+    if not test.is_gpu_available():
+      return
+
+    with context.eager_mode():
+      burn_iters = 15
+      num_convs = 20
+      num_iters = 50
+      batch_size = 64
+      # The input and filter correspond to a middle layer of Resnet50.
+      input = variables.Variable(  # pylint: disable=redefined-builtin
+          random_ops.random_uniform([
+              batch_size,
+              256,
+              14,
+              14
+          ]))
+      filter = variables.Variable(random_ops.random_uniform([3, 3, 256, 256]))  # pylint: disable=redefined-builtin
+      strides = [1, 1, 1, 1]
+      padding = [(0, 0), (0, 0), (1, 1), (1, 1)]
+      output_explicit_pad = input
+      output_same_pad = input
+      for _ in range(burn_iters):
+        output_explicit_pad = nn_ops.conv2d(
+            output_explicit_pad,
+            filter,
+            strides,
+            padding=padding,
+            data_format="NCHW")
+        output_same_pad = nn_ops.conv2d(
+            output_same_pad,
+            filter,
+            strides,
+            padding="SAME",
+            data_format="NCHW")
+
+      start = time.time()
+      for _ in range(num_iters):
+        with backprop.GradientTape() as tape:
+          for _ in range(num_convs):
+            output_explicit_pad = nn_ops.conv2d(
+                output_explicit_pad,
+                filter,
+                strides,
+                padding=padding,
+                data_format="NCHW")
+          tape.gradient(output_explicit_pad, filter)
+      end = time.time()
+      self.report_benchmark(
+          name="eager_explicit_pad",
+          wall_time=(end - start) / num_iters,
+          iters=num_iters)
+
+      start = time.time()
+      for _ in range(num_iters):
+        with backprop.GradientTape() as tape:
+          for _ in range(num_convs):
+            output_same_pad = nn_ops.conv2d(
+                output_same_pad,
+                filter,
+                strides,
+                padding="SAME",
+                data_format="NCHW")
+          tape.gradient(output_same_pad, filter)
+      end = time.time()
+      self.report_benchmark(
+          name="eager_same_pad",
+          wall_time=(end - start) / num_iters,
+          iters=num_iters)
+
 
 def GetInceptionFwdTest(input_size, filter_size, stride, padding,
                         gpu_only=False):
diff --git a/tensorflow/python/ops/nn_grad.py b/tensorflow/python/ops/nn_grad.py
index 7131e4abc4..6ca2b2aafe 100644
--- a/tensorflow/python/ops/nn_grad.py
+++ b/tensorflow/python/ops/nn_grad.py
@@ -514,29 +514,40 @@ def _SparseSoftmaxCrossEntropyWithLogitsGrad(op, grad_0, _):
 
 @ops.RegisterGradient("Conv2D")
 def _Conv2DGrad(op, grad):
+  """Gradient function for Conv2D."""
   dilations = op.get_attr("dilations")
   strides = op.get_attr("strides")
   padding = op.get_attr("padding")
+  explicit_paddings = op.get_attr("explicit_paddings")
   use_cudnn_on_gpu = op.get_attr("use_cudnn_on_gpu")
   data_format = op.get_attr("data_format")
   shape_0, shape_1 = array_ops.shape_n([op.inputs[0], op.inputs[1]])
+
+  # We call the gen_nn_ops backprop functions instead of nn_ops backprop
+  # functions for performance reasons in Eager mode. gen_nn_ops functions take a
+  # `explicit_paddings` parameter, but nn_ops functions do not. So if were were
+  # to use the nn_ops functions, we would have to convert `padding` and
+  # `explicit_paddings` into a single `padding` parameter, increasing overhead
+  # in Eager mode.
   return [
-      nn_ops.conv2d_backprop_input(
+      gen_nn_ops.conv2d_backprop_input(
           shape_0,
           op.inputs[1],
           grad,
           dilations=dilations,
           strides=strides,
           padding=padding,
+          explicit_paddings=explicit_paddings,
           use_cudnn_on_gpu=use_cudnn_on_gpu,
           data_format=data_format),
-      nn_ops.conv2d_backprop_filter(
+      gen_nn_ops.conv2d_backprop_filter(
           op.inputs[0],
           shape_1,
           grad,
           dilations=dilations,
           strides=strides,
           padding=padding,
+          explicit_paddings=explicit_paddings,
           use_cudnn_on_gpu=use_cudnn_on_gpu,
           data_format=data_format)
   ]
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 6f2d2c15bd..f71fcef130 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -171,7 +171,7 @@ class _NonAtrousConvolution(object):
         raise ValueError("data_format must be \"NHWC\" or \"NCHW\".")
       self.strides = strides
       self.data_format = data_format
-      self.conv_op = gen_nn_ops.conv2d
+      self.conv_op = conv2d
     elif conv_dims == 3:
       if data_format is None or data_format == "NDHWC":
         strides = [1] + list(strides) + [1]
@@ -1373,6 +1373,44 @@ def atrous_conv2d(value, filters, rate, padding, name=None):
       name=name)
 
 
+def _convert_padding(padding):
+  """Converts Python padding to C++ padding for ops which take EXPLICIT padding.
+
+  Args:
+    padding: the `padding` argument for a Python op which supports EXPLICIT
+      padding.
+
+  Returns:
+    (padding, explicit_paddings) pair, which should be passed as attributes to a
+    C++ op.
+
+  Raises:
+    ValueError: If padding is invalid.
+  """
+  explicit_paddings = []
+  if padding == "EXPLICIT":
+    # Give a better error message if EXPLICIT is passed.
+    raise ValueError('"EXPLICIT" is not a valid value for the padding '
+                     "parameter. To use explicit padding, the padding "
+                     "parameter must be a list.")
+  if isinstance(padding, (list, tuple)):
+    for i, dim_paddings in enumerate(padding):
+      if not isinstance(dim_paddings, (list, tuple)):
+        raise ValueError("When padding is a list, each element of padding must "
+                         "be a list/tuple of size 2. Element with index %d of "
+                         "padding is not a list/tuple" % i)
+      if len(dim_paddings) != 2:
+        raise ValueError("When padding is a list, each element of padding must "
+                         "be a list/tuple of size 2. Element with index %d of "
+                         "padding has size %d" % (i, len(dim_paddings)))
+      explicit_paddings.extend(dim_paddings)
+    if len(padding) != 4:
+      raise ValueError("When padding is a list, it must be of size 4. Got "
+                       "padding of size: %d" % len(padding))
+    padding = "EXPLICIT"
+  return padding, explicit_paddings
+
+
 @tf_export("nn.conv2d", v1=[])
 def conv2d_v2(input,  # pylint: disable=redefined-builtin
               filters,
@@ -1418,8 +1456,13 @@ def conv2d_v2(input,  # pylint: disable=redefined-builtin
       1-D tensor of length 4.  The stride of the sliding window for each
       dimension of `input`. The dimension order is determined by the value of
       `data_format`, see below for details.
-    padding: A `string` from: `"SAME", "VALID"`.
-      The type of padding algorithm to use.
+    padding: Either the `string `"SAME"` or `"VALID"` indicating the type of
+      padding algorithm to use, or a list indicating the explicit paddings at
+      the start and end of each dimension. When explicit padding is used and
+      data_format is `"NHWC"`, this should be in the form `[[0, 0], [pad_top,
+      pad_bottom], [pad_left, pad_right], [0, 0]]`. When explicit padding used
+      and data_format is `"NCHW"`, this should be in the form `[[0, 0], [0, 0],
+      [pad_top, pad_bottom], [pad_left, pad_right]]`.
     data_format: An optional `string` from: `"NHWC", "NCHW"`.
       Defaults to `"NHWC"`.
       Specify the data format of the input and output data. With the
@@ -1441,15 +1484,98 @@ def conv2d_v2(input,  # pylint: disable=redefined-builtin
   # pylint: enable=line-too-long
   if dilations is None:
     dilations = [1, 1, 1, 1]
+  return conv2d(input,  # pylint: disable=redefined-builtin
+                filters,
+                strides,
+                padding,
+                use_cudnn_on_gpu=True,
+                data_format=data_format,
+                dilations=dilations,
+                name=name)
+
+
+@tf_export(v1=["nn.conv2d"])
+def conv2d(  # pylint: disable=redefined-builtin,dangerous-default-value
+    input,
+    filter,
+    strides,
+    padding,
+    use_cudnn_on_gpu=True,
+    data_format="NHWC",
+    dilations=[1, 1, 1, 1],
+    name=None):
+  r"""Computes a 2-D convolution given 4-D `input` and `filter` tensors.
+
+  Given an input tensor of shape `[batch, in_height, in_width, in_channels]`
+  and a filter / kernel tensor of shape
+  `[filter_height, filter_width, in_channels, out_channels]`, this op
+  performs the following:
+
+  1. Flattens the filter to a 2-D matrix with shape
+     `[filter_height * filter_width * in_channels, output_channels]`.
+  2. Extracts image patches from the input tensor to form a *virtual*
+     tensor of shape `[batch, out_height, out_width,
+     filter_height * filter_width * in_channels]`.
+  3. For each patch, right-multiplies the filter matrix and the image patch
+     vector.
+
+  In detail, with the default NHWC format,
+
+      output[b, i, j, k] =
+          sum_{di, dj, q} input[b, strides[1] * i + di, strides[2] * j + dj, q]
+                          * filter[di, dj, q, k]
+
+  Must have `strides[0] = strides[3] = 1`.  For the most common case of the same
+  horizontal and vertices strides, `strides = [1, stride, stride, 1]`.
+
+  Args:
+    input: A `Tensor`. Must be one of the following types:
+      `half`, `bfloat16`, `float32`, `float64`.
+      A 4-D tensor. The dimension order is interpreted according to the value
+      of `data_format`, see below for details.
+    filter: A `Tensor`. Must have the same type as `input`.
+      A 4-D tensor of shape
+      `[filter_height, filter_width, in_channels, out_channels]`
+    strides: A list of `ints`.
+      1-D tensor of length 4.  The stride of the sliding window for each
+      dimension of `input`. The dimension order is determined by the value of
+      `data_format`, see below for details.
+    padding: Either the `string `"SAME"` or `"VALID"` indicating the type of
+      padding algorithm to use, or a list indicating the explicit paddings at
+      the start and end of each dimension. When explicit padding is used and
+      data_format is `"NHWC"`, this should be in the form `[[0, 0], [pad_top,
+      pad_bottom], [pad_left, pad_right], [0, 0]]`. When explicit padding used
+      and data_format is `"NCHW"`, this should be in the form `[[0, 0], [0, 0],
+      [pad_top, pad_bottom], [pad_left, pad_right]]`.
+    use_cudnn_on_gpu: An optional `bool`. Defaults to `True`.
+    data_format: An optional `string` from: `"NHWC", "NCHW"`.
+      Defaults to `"NHWC"`.
+      Specify the data format of the input and output data. With the
+      default format "NHWC", the data is stored in the order of:
+          [batch, height, width, channels].
+      Alternatively, the format could be "NCHW", the data storage order of:
+          [batch, channels, height, width].
+    dilations: An optional list of `ints`. Defaults to `[1, 1, 1, 1]`.
+      1-D tensor of length 4.  The dilation factor for each dimension of
+      `input`. If set to k > 1, there will be k-1 skipped cells between each
+      filter element on that dimension. The dimension order is determined by the
+      value of `data_format`, see above for details. Dilations in the batch and
+      depth dimensions must be 1.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor`. Has the same type as `input`.
+  """
+  padding, explicit_paddings = _convert_padding(padding)
   return gen_nn_ops.conv2d(input,  # pylint: disable=redefined-builtin
-                           filters,
+                           filter,
                            strides,
                            padding,
-                           use_cudnn_on_gpu=True,
+                           use_cudnn_on_gpu=use_cudnn_on_gpu,
+                           explicit_paddings=explicit_paddings,
                            data_format=data_format,
                            dilations=dilations,
                            name=name)
-tf_export(v1=["nn.conv2d"])(gen_nn_ops.conv2d)
 
 
 @tf_export("nn.conv2d_backprop_filter", v1=[])
@@ -1478,8 +1604,13 @@ def conv2d_backprop_filter_v2(input,  # pylint: disable=redefined-builtin
       The stride of the sliding window for each dimension of the input
       of the convolution. Must be in the same order as the dimension specified
       with format.
-    padding: A `string` from: `"SAME", "VALID"`.
-      The type of padding algorithm to use.
+    padding: Either the `string `"SAME"` or `"VALID"` indicating the type of
+      padding algorithm to use, or a list indicating the explicit paddings at
+      the start and end of each dimension. When explicit padding is used and
+      data_format is `"NHWC"`, this should be in the form `[[0, 0], [pad_top,
+      pad_bottom], [pad_left, pad_right], [0, 0]]`. When explicit padding used
+      and data_format is `"NCHW"`, this should be in the form `[[0, 0], [0, 0],
+      [pad_top, pad_bottom], [pad_left, pad_right]]`.
     data_format: An optional `string` from: `"NHWC", "NCHW"`.
       Defaults to `"NHWC"`.
       Specify the data format of the input and output data. With the
@@ -1500,17 +1631,75 @@ def conv2d_backprop_filter_v2(input,  # pylint: disable=redefined-builtin
   """
   if dilations is None:
     dilations = [1, 1, 1, 1]
-  return gen_nn_ops.conv2d_backprop_filter(input,  # pylint: disable=redefined-builtin
-                                           filter_sizes,
-                                           out_backprop,
-                                           strides,
-                                           padding,
-                                           use_cudnn_on_gpu=True,
-                                           data_format=data_format,
-                                           dilations=dilations,
-                                           name=name)
-tf_export(v1=["nn.conv2d_backprop_filter"])(
-    gen_nn_ops.conv2d_backprop_filter)
+  return conv2d_backprop_filter(input,  # pylint: disable=redefined-builtin
+                                filter_sizes,
+                                out_backprop,
+                                strides,
+                                padding,
+                                use_cudnn_on_gpu=True,
+                                data_format=data_format,
+                                dilations=dilations,
+                                name=name)
+
+
+@tf_export(v1=["nn.conv2d_backprop_filter"])
+def conv2d_backprop_filter(  # pylint: disable=redefined-builtin,dangerous-default-value
+    input,
+    filter_sizes,
+    out_backprop,
+    strides,
+    padding,
+    use_cudnn_on_gpu=True,
+    data_format="NHWC",
+    dilations=[1, 1, 1, 1],
+    name=None):
+  r"""Computes the gradients of convolution with respect to the filter.
+
+  Args:
+    input: A `Tensor`. Must be one of the following types:
+      `half`, `bfloat16`, `float32`, `float64`.
+      4-D with shape `[batch, in_height, in_width, in_channels]`.
+    filter_sizes: A `Tensor` of type `int32`.
+      An integer vector representing the tensor shape of `filter`,
+      where `filter` is a 4-D
+      `[filter_height, filter_width, in_channels, out_channels]` tensor.
+    out_backprop: A `Tensor`. Must have the same type as `input`.
+      4-D with shape `[batch, out_height, out_width, out_channels]`.
+      Gradients w.r.t. the output of the convolution.
+    strides: A list of `ints`.
+      The stride of the sliding window for each dimension of the input
+      of the convolution. Must be in the same order as the dimension specified
+      with format.
+    padding: Either the `string `"SAME"` or `"VALID"` indicating the type of
+      padding algorithm to use, or a list indicating the explicit paddings at
+      the start and end of each dimension. When explicit padding is used and
+      data_format is `"NHWC"`, this should be in the form `[[0, 0], [pad_top,
+      pad_bottom], [pad_left, pad_right], [0, 0]]`. When explicit padding used
+      and data_format is `"NCHW"`, this should be in the form `[[0, 0], [0, 0],
+      [pad_top, pad_bottom], [pad_left, pad_right]]`.
+    use_cudnn_on_gpu: An optional `bool`. Defaults to `True`.
+    data_format: An optional `string` from: `"NHWC", "NCHW"`.
+      Defaults to `"NHWC"`.
+      Specify the data format of the input and output data. With the
+      default format "NHWC", the data is stored in the order of:
+          [batch, in_height, in_width, in_channels].
+      Alternatively, the format could be "NCHW", the data storage order of:
+          [batch, in_channels, in_height, in_width].
+    dilations: An optional list of `ints`. Defaults to `[1, 1, 1, 1]`.
+      1-D tensor of length 4.  The dilation factor for each dimension of
+      `input`. If set to k > 1, there will be k-1 skipped cells between each
+      filter element on that dimension. The dimension order is determined by
+      the value of `data_format`, see above for details. Dilations in the batch
+      and depth dimensions must be 1.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor`. Has the same type as `input`.
+  """
+  padding, explicit_paddings = _convert_padding(padding)
+  return gen_nn_ops.conv2d_backprop_filter(
+      input, filter_sizes, out_backprop, strides, padding, use_cudnn_on_gpu,
+      explicit_paddings, data_format, dilations, name)
 
 
 @tf_export("nn.conv2d_backprop_input", v1=[])
@@ -1539,8 +1728,13 @@ def conv2d_backprop_input_v2(input_sizes,
       The stride of the sliding window for each dimension of the input
       of the convolution. Must be in the same order as the dimension specified
       with format.
-    padding: A `string` from: `"SAME", "VALID"`.
-      The type of padding algorithm to use.
+    padding: Either the `string `"SAME"` or `"VALID"` indicating the type of
+      padding algorithm to use, or a list indicating the explicit paddings at
+      the start and end of each dimension. When explicit padding is used and
+      data_format is `"NHWC"`, this should be in the form `[[0, 0], [pad_top,
+      pad_bottom], [pad_left, pad_right], [0, 0]]`. When explicit padding used
+      and data_format is `"NCHW"`, this should be in the form `[[0, 0], [0, 0],
+      [pad_top, pad_bottom], [pad_left, pad_right]]`.
     data_format: An optional `string` from: `"NHWC", "NCHW"`.
       Defaults to `"NHWC"`.
       Specify the data format of the input and output data. With the
@@ -1561,17 +1755,75 @@ def conv2d_backprop_input_v2(input_sizes,
   """
   if dilations is None:
     dilations = [1, 1, 1, 1]
-  return gen_nn_ops.conv2d_backprop_input(input_sizes,
-                                          filters,
-                                          out_backprop,
-                                          strides,
-                                          padding,
-                                          use_cudnn_on_gpu=True,
-                                          data_format=data_format,
-                                          dilations=dilations,
-                                          name=name)
-tf_export(v1=["nn.conv2d_backprop_input"])(
-    gen_nn_ops.conv2d_backprop_input)
+  return conv2d_backprop_input(input_sizes,
+                               filters,
+                               out_backprop,
+                               strides,
+                               padding,
+                               use_cudnn_on_gpu=True,
+                               data_format=data_format,
+                               dilations=dilations,
+                               name=name)
+
+
+@tf_export(v1=["nn.conv2d_backprop_input"])
+def conv2d_backprop_input(  # pylint: disable=redefined-builtin,dangerous-default-value
+    input_sizes,
+    filter,
+    out_backprop,
+    strides,
+    padding,
+    use_cudnn_on_gpu=True,
+    data_format="NHWC",
+    dilations=[1, 1, 1, 1],
+    name=None):
+  r"""Computes the gradients of convolution with respect to the input.
+
+  Args:
+    input_sizes: A `Tensor` of type `int32`.
+      An integer vector representing the shape of `input`,
+      where `input` is a 4-D `[batch, height, width, channels]` tensor.
+    filter: A `Tensor`. Must be one of the following types:
+      `half`, `bfloat16`, `float32`, `float64`.
+      4-D with shape
+      `[filter_height, filter_width, in_channels, out_channels]`.
+    out_backprop: A `Tensor`. Must have the same type as `filter`.
+      4-D with shape `[batch, out_height, out_width, out_channels]`.
+      Gradients w.r.t. the output of the convolution.
+    strides: A list of `ints`.
+      The stride of the sliding window for each dimension of the input
+      of the convolution. Must be in the same order as the dimension specified
+      with format.
+    padding: Either the `string `"SAME"` or `"VALID"` indicating the type of
+      padding algorithm to use, or a list indicating the explicit paddings at
+      the start and end of each dimension. When explicit padding is used and
+      data_format is `"NHWC"`, this should be in the form `[[0, 0], [pad_top,
+      pad_bottom], [pad_left, pad_right], [0, 0]]`. When explicit padding used
+      and data_format is `"NCHW"`, this should be in the form `[[0, 0], [0, 0],
+      [pad_top, pad_bottom], [pad_left, pad_right]]`.
+    use_cudnn_on_gpu: An optional `bool`. Defaults to `True`.
+    data_format: An optional `string` from: `"NHWC", "NCHW"`.
+      Defaults to `"NHWC"`.
+      Specify the data format of the input and output data. With the
+      default format "NHWC", the data is stored in the order of:
+          [batch, in_height, in_width, in_channels].
+      Alternatively, the format could be "NCHW", the data storage order of:
+          [batch, in_channels, in_height, in_width].
+    dilations: An optional list of `ints`. Defaults to `[1, 1, 1, 1]`.
+      1-D tensor of length 4.  The dilation factor for each dimension of
+      `input`. If set to k > 1, there will be k-1 skipped cells between each
+      filter element on that dimension. The dimension order is determined by
+      the value of `data_format`, see above for details. Dilations in the batch
+      and depth dimensions must be 1.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor`. Has the same type as `filter`.
+  """
+  padding, explicit_paddings = _convert_padding(padding)
+  return gen_nn_ops.conv2d_backprop_input(
+      input_sizes, filter, out_backprop, strides, padding, use_cudnn_on_gpu,
+      explicit_paddings, data_format, dilations, name)
 
 
 @tf_export(v1=["nn.conv2d_transpose"])
-- 
GitLab


From e3b8fedc923d03fd5f388daf444b42b22dd909b8 Mon Sep 17 00:00:00 2001
From: Cong Liu <congliu@google.com>
Date: Tue, 8 Jan 2019 18:56:15 -0800
Subject: [PATCH 0398/2345] [XLA] Propagate compiler specific ShapeSizeFunction
 into TargetVerifierMetadata.

PiperOrigin-RevId: 228440043
---
 .../compiler/xla/service/hlo_verifier.h       | 33 ++++++++++++-------
 .../compiler/xla/tests/hlo_test_base.cc       |  6 ++--
 tensorflow/compiler/xla/tests/hlo_test_base.h |  8 +++--
 3 files changed, 31 insertions(+), 16 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_verifier.h b/tensorflow/compiler/xla/service/hlo_verifier.h
index a1a6aba972..479905b317 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier.h
+++ b/tensorflow/compiler/xla/service/hlo_verifier.h
@@ -168,8 +168,13 @@ class ShapeVerifier : public DfsHloVisitor {
 // An interface used to encapsulate target-specific verification quirks.
 class TargetVerifierMetadata {
  public:
+  TargetVerifierMetadata(std::function<int64(const Shape&)> shape_size_function)
+      : shape_size_function_(shape_size_function) {}
+
   // Returns a target-specific shape size.
-  virtual int64 ShapeSize(const Shape& shape) const = 0;
+  int64 ShapeSize(const Shape& shape) const {
+    return shape_size_function_(shape);
+  }
 
   virtual std::unique_ptr<ShapeVerifier> GetVerifier() const = 0;
 
@@ -178,20 +183,23 @@ class TargetVerifierMetadata {
 
   TargetVerifierMetadata(const TargetVerifierMetadata&) = delete;
   TargetVerifierMetadata& operator=(const TargetVerifierMetadata&) = delete;
+
+ private:
+  // Returns a target-specific shape size.
+  std::function<int64(const Shape&)> shape_size_function_;
 };
 
 // The default implementation of TargetVerifierMetadata, used unless the target
 // needs to override it.
 class DefaultVerifierMetadata : public TargetVerifierMetadata {
  public:
-  DefaultVerifierMetadata(bool layout_sensitive, bool allow_mixed_precision)
-      : layout_sensitive_(layout_sensitive),
+  DefaultVerifierMetadata(
+      bool layout_sensitive, bool allow_mixed_precision,
+      std::function<int64(const Shape&)> shape_size_function)
+      : TargetVerifierMetadata(shape_size_function),
+        layout_sensitive_(layout_sensitive),
         allow_mixed_precision_(allow_mixed_precision) {}
 
-  int64 ShapeSize(const Shape& shape) const override {
-    return ShapeUtil::ByteSizeOf(shape);
-  }
-
   // Creates a ShapeVerifier that checks that shapes match inferred
   // expectations. This creates a new verifier every time because ShapeVerifier,
   // being a DfsHloVisitor, is stateful. We want a clean object for each run of
@@ -210,11 +218,14 @@ class DefaultVerifierMetadata : public TargetVerifierMetadata {
 // the module.
 class HloVerifier : public HloModulePass {
  public:
-  explicit HloVerifier(bool layout_sensitive, bool allow_mixed_precision,
-                       std::function<bool(const HloInstruction*)>
-                           instruction_can_change_layout_func = {})
+  explicit HloVerifier(
+      bool layout_sensitive, bool allow_mixed_precision,
+      std::function<bool(const HloInstruction*)>
+          instruction_can_change_layout_func = {},
+      std::function<int64(const Shape&)> shape_size_func =
+          [](const Shape& shape) { return ShapeUtil::ByteSizeOf(shape); })
       : target_metadata_(absl::make_unique<DefaultVerifierMetadata>(
-            layout_sensitive, allow_mixed_precision)),
+            layout_sensitive, allow_mixed_precision, shape_size_func)),
         instruction_can_change_layout_func_(
             std::move(instruction_can_change_layout_func)) {
     CHECK(instruction_can_change_layout_func_ == nullptr || layout_sensitive);
diff --git a/tensorflow/compiler/xla/tests/hlo_test_base.cc b/tensorflow/compiler/xla/tests/hlo_test_base.cc
index d57846e19b..66f72ba8d2 100644
--- a/tensorflow/compiler/xla/tests/hlo_test_base.cc
+++ b/tensorflow/compiler/xla/tests/hlo_test_base.cc
@@ -139,7 +139,8 @@ std::unique_ptr<VerifiedHloModule> HloTestBase::CreateNewVerifiedModule(
     const string& name) {
   return absl::make_unique<VerifiedHloModule>(
       name, GetModuleConfigForTest(), verifier_layout_sensitive_,
-      allow_mixed_precision_in_hlo_verifier_);
+      allow_mixed_precision_in_hlo_verifier_,
+      backend().compiler()->ShapeSizeBytesFunction());
 }
 
 StatusOr<std::unique_ptr<VerifiedHloModule>>
@@ -147,7 +148,8 @@ HloTestBase::ParseAndReturnVerifiedModule(absl::string_view hlo_text,
                                           const HloModuleConfig& config) {
   auto module = absl::make_unique<VerifiedHloModule>(
       TestName(), config, verifier_layout_sensitive_,
-      allow_mixed_precision_in_hlo_verifier_);
+      allow_mixed_precision_in_hlo_verifier_,
+      backend().compiler()->ShapeSizeBytesFunction());
   TF_RETURN_IF_ERROR(ParseHloString(hlo_text, module.get()));
   TF_RETURN_IF_ERROR(module->Verify());
   return std::move(module);
diff --git a/tensorflow/compiler/xla/tests/hlo_test_base.h b/tensorflow/compiler/xla/tests/hlo_test_base.h
index 1d1e7f4372..69a4f96288 100644
--- a/tensorflow/compiler/xla/tests/hlo_test_base.h
+++ b/tensorflow/compiler/xla/tests/hlo_test_base.h
@@ -46,10 +46,12 @@ class VerifiedHloModule : public HloModule {
  public:
   VerifiedHloModule(const string& name, const HloModuleConfig& config,
                     bool verifier_layout_sensitive,
-                    bool allow_mixed_precision_in_hlo_verifier)
+                    bool allow_mixed_precision_in_hlo_verifier,
+                    std::function<int64(const Shape&)> shape_size_function)
       : HloModule(name, config),
-        verifier_(verifier_layout_sensitive,
-                  allow_mixed_precision_in_hlo_verifier) {}
+        verifier_(
+            verifier_layout_sensitive, allow_mixed_precision_in_hlo_verifier,
+            /*instruction_can_change_layout_func=*/{}, shape_size_function) {}
 
   ~VerifiedHloModule() override { VerifyOrAddFailure("in destructor"); }
 
-- 
GitLab


From 6522d752cf3e5d26a507d9c4c517ee7633ae4c57 Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Tue, 8 Jan 2019 19:41:27 -0800
Subject: [PATCH 0399/2345] Removes the hack that mutates an HloInstruction in
 order to auto-tune the convolution for a specific algorithm.

Redemption.

PiperOrigin-RevId: 228443963
---
 .../service/gpu/cudnn_conv_algorithm_picker.cc   | 12 +++++++-----
 .../service/gpu/cudnn_conv_algorithm_picker.h    |  3 ++-
 .../xla/service/gpu/cudnn_conv_runner.cc         | 16 ++++++++++------
 .../compiler/xla/service/gpu/cudnn_conv_runner.h | 12 ++++++++++--
 4 files changed, 29 insertions(+), 14 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.cc b/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.cc
index 6d6780fa1c..057d0d2364 100644
--- a/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.cc
+++ b/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.cc
@@ -146,7 +146,8 @@ tensorflow::mutex_lock LockGpu(const se::StreamExecutor* stream_exec) {
 // trouble, but we may want to revisit this if we ever find a model where
 // caching would speed up compilation a lot.
 StatusOr<CudnnConvAlgorithmPicker::AutotuneResult>
-CudnnConvAlgorithmPicker::PickBestAlgorithm(HloCustomCallInstruction* instr) {
+CudnnConvAlgorithmPicker::PickBestAlgorithm(
+    const HloCustomCallInstruction* instr) {
   // TODO(timshen): for now only check fp16. It can be expanded to other types,
   // with some work on the HLO routines.
   const bool cross_check_enabled =
@@ -249,12 +250,13 @@ CudnnConvAlgorithmPicker::PickBestAlgorithm(HloCustomCallInstruction* instr) {
     VLOG(3) << "Trying algorithm " << AlgorithmToString(alg) << " for "
             << instr->ToString();
 
-    backend_config.set_algorithm(alg.algo_id());
-    backend_config.set_tensor_ops_enabled(alg.tensor_ops_enabled());
-    TF_RETURN_IF_ERROR(instr->set_backend_config(backend_config));
+    // Use assignment insetad of brace-list to make GCC 4.9 happy.
+    RunConvOptions options;
+    options.profile_result = &profile_result;
+    options.algo_override = alg;
     bool launch_ok =
         RunCudnnConv(instr, absl::MakeSpan(operand_buffers), result_buffer,
-                     &scratch_allocator, &stream, &profile_result)
+                     &scratch_allocator, &stream, options)
             .ok();
 
     if (launch_ok && profile_result.is_valid()) {
diff --git a/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.h b/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.h
index 642af787af..4991db0948 100644
--- a/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.h
+++ b/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.h
@@ -56,7 +56,8 @@ class CudnnConvAlgorithmPicker : public HloModulePass {
 
   StatusOr<bool> RunOnComputation(HloComputation* computation);
   StatusOr<bool> RunOnInstruction(HloInstruction* instr);
-  StatusOr<AutotuneResult> PickBestAlgorithm(HloCustomCallInstruction* instr);
+  StatusOr<AutotuneResult> PickBestAlgorithm(
+      const HloCustomCallInstruction* instr);
 
   se::StreamExecutor* stream_exec_;                   // never null
   DeviceMemoryAllocator* allocator_;                  // may be null
diff --git a/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.cc b/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.cc
index 3425e1b494..b628f27f4b 100644
--- a/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.cc
+++ b/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.cc
@@ -395,32 +395,36 @@ Status RunCudnnConv(const HloCustomCallInstruction* conv,
                     absl::Span<se::DeviceMemoryBase> operand_buffers,
                     se::DeviceMemoryBase result_buffer,
                     se::DeviceMemoryBase scratch_buf, se::Stream* stream,
-                    se::dnn::ProfileResult* profile_result) {
+                    RunConvOptions options) {
   ScratchBufAllocator scratch_allocator(scratch_buf);
   return RunCudnnConv(conv, operand_buffers, result_buffer, &scratch_allocator,
-                      stream, profile_result);
+                      stream, options);
 }
 
 Status RunCudnnConv(const HloCustomCallInstruction* conv,
                     absl::Span<se::DeviceMemoryBase> operand_buffers,
                     se::DeviceMemoryBase result_buffer,
                     se::ScratchAllocator* scratch_allocator, se::Stream* stream,
-                    se::dnn::ProfileResult* profile_result) {
+                    RunConvOptions options) {
   TF_ASSIGN_OR_RETURN(CudnnConvParams params,
                       GetCudnnConvParams(conv, operand_buffers, result_buffer));
 
+  if (options.algo_override) {
+    params.algorithm = AlgorithmConfig(*options.algo_override);
+  }
+
   PrimitiveType output_primitive_type =
       conv->shape().tuple_shapes(0).element_type();
   switch (output_primitive_type) {
     case F16:
       return RunCudnnConvImpl<Eigen::half>(params, scratch_allocator, stream,
-                                           profile_result);
+                                           options.profile_result);
     case F32:
       return RunCudnnConvImpl<float>(params, scratch_allocator, stream,
-                                     profile_result);
+                                     options.profile_result);
     case F64:
       return RunCudnnConvImpl<double>(params, scratch_allocator, stream,
-                                      profile_result);
+                                      options.profile_result);
     default:
       LOG(FATAL) << ShapeUtil::HumanString(*params.output_shape);
   }
diff --git a/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.h b/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.h
index edbc75a94a..7cc325bce3 100644
--- a/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.h
+++ b/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.h
@@ -28,6 +28,14 @@ limitations under the License.
 namespace xla {
 namespace gpu {
 
+struct RunConvOptions {
+  // Nullable output-parameter pointer for profiling results.
+  se::dnn::ProfileResult* profile_result = nullptr;
+
+  // Use this algorithm, instead the one from the instrcution.
+  absl::optional<se::dnn::AlgorithmDesc> algo_override;
+};
+
 // This file contains low-level routines for running cudnn convolutions.
 
 // Calls into cudnn to run the specified convolution.
@@ -46,13 +54,13 @@ Status RunCudnnConv(const HloCustomCallInstruction* conv,
                     absl::Span<se::DeviceMemoryBase> operand_buffers,
                     se::DeviceMemoryBase result_buffer,
                     se::DeviceMemoryBase scratch_buf, se::Stream* stream,
-                    se::dnn::ProfileResult* profile_result = nullptr);
+                    RunConvOptions = {});
 
 Status RunCudnnConv(const HloCustomCallInstruction* conv,
                     absl::Span<se::DeviceMemoryBase> operand_buffers,
                     se::DeviceMemoryBase result_buffer,
                     se::ScratchAllocator* scratch_allocator, se::Stream* stream,
-                    se::dnn::ProfileResult* profile_result = nullptr);
+                    RunConvOptions = {});
 
 }  // namespace gpu
 }  // namespace xla
-- 
GitLab


From 9a54ef2b7cd156ce9fb11d143cea61cc84aed3b3 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 8 Jan 2019 19:54:03 -0800
Subject: [PATCH 0400/2345] Automated rollback of commit
 f71e77471c140bac2f4dcb9a003139fb5c657302

PiperOrigin-RevId: 228444974
---
 tensorflow/compiler/xla/service/cpu/BUILD     |   2 +
 .../cpu/cpu_eigen_tensor_alignment_test.cc    |  38 ++++++
 .../xla/service/cpu/dot_op_emitter.cc         | 117 ++++++------------
 .../xla/service/cpu/dot_op_emitter_internal.h |  88 +++++++++++++
 4 files changed, 168 insertions(+), 77 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h

diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index a197bdddc8..de62aa60aa 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -385,6 +385,7 @@ cc_library(
     srcs = ["dot_op_emitter.cc"],
     hdrs = [
         "dot_op_emitter.h",
+        "dot_op_emitter_internal.h",
     ],
     deps = [
         ":cpu_options",
@@ -1028,6 +1029,7 @@ tf_cc_test(
     size = "small",
     srcs = ["cpu_eigen_tensor_alignment_test.cc"],
     deps = [
+        ":dot_op_emitter",
         ":ir_emission_utils",
         ":target_machine_features_fake",
         "//tensorflow/compiler/xla:test",
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc b/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc
index 485769a373..823bdf259c 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc
@@ -13,6 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include "tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h"
 #include "tensorflow/compiler/xla/service/cpu/ir_emission_utils.h"
 #include "tensorflow/compiler/xla/service/cpu/target_machine_features_fake.h"
 #include "tensorflow/compiler/xla/service/hlo_parser.h"
@@ -22,11 +23,48 @@ namespace xla {
 namespace cpu {
 namespace {
 
+using internal::DotImplementationStrategy;
+using internal::DotInfo;
+using internal::GetDotImplementationStrategy;
+
 // Test that we don't call into Eigen with tensors too small to be aligned
 // reliably.
 
 class CpuEigenTensorAlignmentTest : public ::testing::Test {};
 
+TEST_F(CpuEigenTensorAlignmentTest, EigenDotAlignment) {
+  string hlo_string = R"(
+HloModule DotOperation
+
+ENTRY DotOperation {
+  arg0 = f32[5,256] parameter(0)
+  arg1 = f32[256,1024] parameter(1)
+  ROOT dot = f32[5,1024] dot(arg0, arg1), lhs_contracting_dims={1}, rhs_contracting_dims={0}
+}
+)";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseHloString(hlo_string));
+
+  HloInstruction* dot = module->entry_computation()->root_instruction();
+
+  TargetMachineFeaturesWithFakeAlignmentLogic target_machine_with_no_alignment(
+      [](int64 size) { return 1; });
+
+  EXPECT_EQ(GetDotImplementationStrategy(HloModuleConfig{}, DotInfo(*dot),
+                                         target_machine_with_no_alignment),
+            DotImplementationStrategy::kNaiveLlvmIr);
+
+  TargetMachineFeaturesWithFakeAlignmentLogic
+      target_machine_with_full_alignment([](int64 size) {
+        return TargetMachineFeatures::kEigenExpectedTensorAlignment;
+      });
+
+  EXPECT_NE(GetDotImplementationStrategy(HloModuleConfig{}, DotInfo(*dot),
+                                         target_machine_with_full_alignment),
+            DotImplementationStrategy::kNaiveLlvmIr);
+}
+
 TEST_F(CpuEigenTensorAlignmentTest, EigenConvAlignment) {
   string hlo_string = R"(
 HloModule ConvOperation
diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
index b018e0cd46..e8a84ebe6b 100644
--- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/compiler/xla/service/cpu/dot_op_emitter.h"
+#include "tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h"
 
 #include <memory>
 #include <vector>
@@ -42,63 +43,17 @@ namespace xla {
 using llvm_ir::SetToFirstInsertPoint;
 
 namespace cpu {
+
+using internal::DotImplementationStrategy;
+using internal::DotInfo;
+using internal::GetDotImplementationStrategy;
+
 namespace {
 // Returns true if we should call into multi-threaded Eigen routines.
 bool ShouldUseMultiThreadedEigen(const HloModuleConfig& config) {
   return config.debug_options().xla_cpu_multi_thread_eigen();
 }
 
-// Represents a dot operation.  We use this in lieu of an `HloInstruction`
-// because we want to be able to create this for the "inner" dot operation in a
-// batch dot, for which there is no separate HLO instruction.
-struct DotInfo {
-  Shape lhs_shape;
-  Shape rhs_shape;
-  Shape result_shape;
-  DotDimensionNumbers dim_nums;
-
-  explicit DotInfo(const HloInstruction& instr) {
-    CHECK_EQ(instr.opcode(), HloOpcode::kDot);
-    lhs_shape = instr.operand(0)->shape();
-    rhs_shape = instr.operand(1)->shape();
-    result_shape = instr.shape();
-    dim_nums = instr.dot_dimension_numbers();
-  }
-};
-
-// Dictates how a dot operation is implemented.
-enum class DotImplementationStrategy {
-  // The dot operation is lowered into LLVM IR that implements a naive nested
-  // loop that computes the result one element at a time.  This is our
-  // "fallback"; we don't really want this to kick in for any non-trival dot
-  // operation.
-  kNaiveLlvmIr,
-
-  // The dot operation is lowered into LLVM IR that implements a tiled
-  // Matrix*Vector operation.  This strategy also allows fusing in a bias add
-  // into the dot.  The matrix can be row major or column major, both are
-  // supported.
-  kTiledLlvmIrGemv,
-
-  // The dot operation is lowered into LLVM IR that implemetns a tiled
-  // Matrix*Matrix operation.  No fusions are supported.  The two inputs
-  // and the output have to be row major.
-  kTiledLlvmIrGemm,
-
-  // The dot operation is lowered into a call into an Eigen routine.  No fusions
-  // are supported today.  The two inputs and the output have to be row major.
-  // However, we do allow transposing either the LHS or the RHS as part of the
-  // GEMM -- we expose this flexibility as flexibility in the contraction
-  // dimensions, but we can also see this as flexibility in the input layouts.
-  kEigen,
-};
-
-// Returns the implementation strategy for a dot with the configuration
-// `dot_info`.
-DotImplementationStrategy GetDotImplementationStrategy(
-    const HloModuleConfig& config, const DotInfo& dot_info,
-    const TargetMachineFeatures& target_machine_features);
-
 // Helper class for emitting LLVM IR to perform the dot operation.
 class DotOpEmitter {
  public:
@@ -235,8 +190,9 @@ void DotOpEmitter::EmitTiledLlvmIrGemm() {
   }
 
   int64 size_bytes = m * n * ShapeUtil::ByteSizeOfPrimitiveType(primitive_type);
-  b_->CreateMemSet(target, b_->getInt8(0), /*Size=*/size_bytes,
-                   /*Align=*/1);
+  b_->CreateMemSet(
+      target, b_->getInt8(0), size_bytes,
+      target_machine_features_.minimum_alignment_for_allocation(size_bytes));
 
   int64 max_target_vector_width =
       target_machine_features_.vector_register_num_elements(
@@ -740,34 +696,40 @@ absl::optional<int64> ProfitableToMakeDotOperandColumnMajor(
   return {};
 }
 
+namespace internal {
 namespace {
 // Return whether the given shape is rank 2.
 bool IsRank2(const Shape& shape) { return shape.rank() == 2; }
 
-bool IsSimpleLayout(const Layout& layout) {
-  return layout.tiles().empty() && layout.format() == DENSE;
-}
-
 // In a gemm operation where output = lhs * rhs, check whether the given shapes
 // are valid for the operation.
-bool AreGemmShapes(const Shape& lhs_shape, const Shape& rhs_shape,
-                   const Shape& output_shape,
-                   const TargetMachineFeatures& target_machine_features) {
-  CHECK(!lhs_shape.has_layout() || IsSimpleLayout(lhs_shape.layout()))
-      << lhs_shape.DebugString();
-  CHECK(!rhs_shape.has_layout() || IsSimpleLayout(rhs_shape.layout()))
-      << rhs_shape.DebugString();
-  CHECK(!output_shape.has_layout() || IsSimpleLayout(output_shape.layout()))
-      << output_shape.DebugString();
-
-  switch (output_shape.element_type()) {
-    case F64:
-    case F32:
-    case F16:
-      return IsRank2(lhs_shape) && IsRank2(rhs_shape) && IsRank2(output_shape);
-    default:
-      return false;
+bool AreAlignedGemmShapes(
+    const Shape& lhs_shape, const Shape& rhs_shape, const Shape& output_shape,
+    const TargetMachineFeatures& target_machine_features) {
+  // The inputs and the output must
+  // 1) be matrices with no padding, and
+  // 2) have an allowed element type.
+  PrimitiveType output_primitive_type = output_shape.element_type();
+  if (!(output_primitive_type == F64 || output_primitive_type == F32 ||
+        output_primitive_type == F16)) {
+    return false;
+  }
+
+  if (!(IsRank2(lhs_shape) && IsRank2(rhs_shape) && IsRank2(output_shape))) {
+    return false;
+  }
+
+  auto is_aligned = [&](const Shape& shape) {
+    return GetMinimumAlignmentForArray(shape, target_machine_features) >=
+           TargetMachineFeatures::kEigenExpectedTensorAlignment;
+  };
+
+  if (!is_aligned(lhs_shape) || !is_aligned(rhs_shape) ||
+      !is_aligned(output_shape)) {
+    return false;
   }
+
+  return true;
 }
 
 bool IsAlignedGemm(const DotInfo& dot_info,
@@ -777,8 +739,8 @@ bool IsAlignedGemm(const DotInfo& dot_info,
     return false;
   }
 
-  return AreGemmShapes(dot_info.lhs_shape, dot_info.rhs_shape,
-                       dot_info.result_shape, target_machine_features);
+  return AreAlignedGemmShapes(dot_info.lhs_shape, dot_info.rhs_shape,
+                              dot_info.result_shape, target_machine_features);
 }
 
 bool CanEmitTiledLlvmIrGemm(
@@ -819,6 +781,7 @@ bool CanEmitTiledLlvmIrGemm(
 
   return true;
 }
+}  // namespace
 
 DotImplementationStrategy GetDotImplementationStrategy(
     const HloModuleConfig& config, const DotInfo& dot_info,
@@ -843,7 +806,7 @@ DotImplementationStrategy GetDotImplementationStrategy(
 
   return DotImplementationStrategy::kNaiveLlvmIr;
 }
-}  // namespace
+}  // namespace internal
 
 bool DotImplementationCanHandleTranspose(
     const HloInstruction& dot_instr,
diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h b/tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h
new file mode 100644
index 0000000000..cc28918ed6
--- /dev/null
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h
@@ -0,0 +1,88 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_CPU_DOT_OP_EMITTER_INTERNAL_H_
+#define TENSORFLOW_COMPILER_XLA_SERVICE_CPU_DOT_OP_EMITTER_INTERNAL_H_
+
+#include "tensorflow/compiler/xla/service/cpu/target_machine_features.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+
+// -----------------------------------------------------------------------------
+// INTERNAL HEADER.
+//
+// This file exposes internal implementation details from dot_op_emitter.cc for
+// unit tests.  Please do not depend on this!
+//
+// -----------------------------------------------------------------------------
+
+namespace xla {
+namespace cpu {
+namespace internal {
+
+// Represents a dot operation.  We use this in lieu of an `HloInstruction`
+// because we want to be able to create this for the "inner" dot operation in a
+// batch dot, for which there is no separate HLO instruction.
+struct DotInfo {
+  Shape lhs_shape;
+  Shape rhs_shape;
+  Shape result_shape;
+  DotDimensionNumbers dim_nums;
+
+  explicit DotInfo(const HloInstruction& instr) {
+    CHECK_EQ(instr.opcode(), HloOpcode::kDot);
+    lhs_shape = instr.operand(0)->shape();
+    rhs_shape = instr.operand(1)->shape();
+    result_shape = instr.shape();
+    dim_nums = instr.dot_dimension_numbers();
+  }
+};
+
+// Dictates how a dot operation is implemented.
+enum class DotImplementationStrategy {
+  // The dot operation is lowered into LLVM IR that implements a naive nested
+  // loop that computes the result one element at a time.  This is our
+  // "fallback"; we don't really want this to kick in for any non-trival dot
+  // operation.
+  kNaiveLlvmIr,
+
+  // The dot operation is lowered into LLVM IR that implements a tiled
+  // Matrix*Vector operation.  This strategy also allows fusing in a bias add
+  // into the dot.  The matrix can be row major or column major, both are
+  // supported.
+  kTiledLlvmIrGemv,
+
+  // The dot operation is lowered into LLVM IR that implemetns a tiled
+  // Matrix*Matrix operation.  No fusions are supported.  The two inputs
+  // and the output have to be row major.
+  kTiledLlvmIrGemm,
+
+  // The dot operation is lowered into a call into an Eigen routine.  No fusions
+  // are supported today.  The two inputs and the output have to be row major.
+  // However, we do allow transposing either the LHS or the RHS as part of the
+  // GEMM -- we expose this flexibility as flexibility in the contraction
+  // dimensions, but we can also see this as flexibility in the input layouts.
+  kEigen,
+};
+
+// Returns the implementation strategy for a dot with the configuration
+// `dot_info`.
+DotImplementationStrategy GetDotImplementationStrategy(
+    const HloModuleConfig& config, const DotInfo& dot_info,
+    const TargetMachineFeatures& target_machine_features);
+}  // namespace internal
+}  // namespace cpu
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_CPU_DOT_OP_EMITTER_INTERNAL_H_
-- 
GitLab


From 8d4893d6efa971b35db9f4eb654adb1559b7186c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 20:19:22 -0800
Subject: [PATCH 0401/2345] Update ops-related pbtxt files.

PiperOrigin-RevId: 228447178
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 254 ++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 |  27 ++
 2 files changed, 281 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 28d085b2d2..9698673dfe 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -13631,6 +13631,151 @@ op {
     }
   }
 }
+op {
+  name: "Conv2D"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_BFLOAT16
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "use_cudnn_on_gpu"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+        s: "EXPLICIT"
+      }
+    }
+  }
+  attr {
+    name: "explicit_paddings"
+    type: "list(int)"
+    default_value {
+      list {
+      }
+    }
+  }
+  attr {
+    name: "data_format"
+    type: "string"
+    default_value {
+      s: "NHWC"
+    }
+    allowed_values {
+      list {
+        s: "NHWC"
+        s: "NCHW"
+      }
+    }
+  }
+  attr {
+    name: "dilations"
+    type: "list(int)"
+    default_value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+      }
+    }
+  }
+}
+op {
+  name: "Conv2DBackpropFilter"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "filter_sizes"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "out_backprop"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "use_cudnn_on_gpu"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "data_format"
+    type: "string"
+    default_value {
+      s: "NHWC"
+    }
+    allowed_values {
+      list {
+        s: "NHWC"
+        s: "NCHW"
+      }
+    }
+  }
+}
 op {
   name: "Conv2DBackpropFilter"
   input_arg {
@@ -13655,6 +13800,7 @@ op {
     allowed_values {
       list {
         type: DT_HALF
+        type: DT_BFLOAT16
         type: DT_FLOAT
       }
     }
@@ -13693,6 +13839,18 @@ op {
       }
     }
   }
+  attr {
+    name: "dilations"
+    type: "list(int)"
+    default_value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+      }
+    }
+  }
 }
 op {
   name: "Conv2DBackpropFilter"
@@ -13720,6 +13878,7 @@ op {
         type: DT_HALF
         type: DT_BFLOAT16
         type: DT_FLOAT
+        type: DT_DOUBLE
       }
     }
   }
@@ -13818,6 +13977,15 @@ op {
       list {
         s: "SAME"
         s: "VALID"
+        s: "EXPLICIT"
+      }
+    }
+  }
+  attr {
+    name: "explicit_paddings"
+    type: "list(int)"
+    default_value {
+      list {
       }
     }
   }
@@ -14063,6 +14231,92 @@ op {
     }
   }
 }
+op {
+  name: "Conv2DBackpropInput"
+  input_arg {
+    name: "input_sizes"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "out_backprop"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_BFLOAT16
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "use_cudnn_on_gpu"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+        s: "EXPLICIT"
+      }
+    }
+  }
+  attr {
+    name: "explicit_paddings"
+    type: "list(int)"
+    default_value {
+      list {
+      }
+    }
+  }
+  attr {
+    name: "data_format"
+    type: "string"
+    default_value {
+      s: "NHWC"
+    }
+    allowed_values {
+      list {
+        s: "NHWC"
+        s: "NCHW"
+      }
+    }
+  }
+  attr {
+    name: "dilations"
+    type: "list(int)"
+    default_value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+      }
+    }
+  }
+}
 op {
   name: "Conv3D"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index b4767d35d2..4a1b3a6f2d 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -5876,6 +5876,15 @@ op {
       list {
         s: "SAME"
         s: "VALID"
+        s: "EXPLICIT"
+      }
+    }
+  }
+  attr {
+    name: "explicit_paddings"
+    type: "list(int)"
+    default_value {
+      list {
       }
     }
   }
@@ -5953,6 +5962,15 @@ op {
       list {
         s: "SAME"
         s: "VALID"
+        s: "EXPLICIT"
+      }
+    }
+  }
+  attr {
+    name: "explicit_paddings"
+    type: "list(int)"
+    default_value {
+      list {
       }
     }
   }
@@ -6030,6 +6048,15 @@ op {
       list {
         s: "SAME"
         s: "VALID"
+        s: "EXPLICIT"
+      }
+    }
+  }
+  attr {
+    name: "explicit_paddings"
+    type: "list(int)"
+    default_value {
+      list {
       }
     }
   }
-- 
GitLab


From 283bd7dc2914f36daf5d6604ffa25d939967bd7d Mon Sep 17 00:00:00 2001
From: Pariksheet Pinjari <pariksheet.pinjari@huawei.com>
Date: Wed, 9 Jan 2019 10:33:43 +0530
Subject: [PATCH 0402/2345] Update api_def_StaticRegexReplace.pbtxt

Documentation error fixed
---
 .../core/api_def/base_api/api_def_StaticRegexReplace.pbtxt      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/api_def/base_api/api_def_StaticRegexReplace.pbtxt b/tensorflow/core/api_def/base_api/api_def_StaticRegexReplace.pbtxt
index e382bcec81..8bb88f491a 100644
--- a/tensorflow/core/api_def/base_api/api_def_StaticRegexReplace.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_StaticRegexReplace.pbtxt
@@ -14,7 +14,7 @@ op {
   }
   attr {
     name: "rewrite"
-    description: "The rewrite to be applied to the matched expresion."
+    description: "The rewrite to be applied to the matched expression."
   }
   attr {
     name: "replace_global"
-- 
GitLab


From ae2aceda4f6630dbdaa6c9a4da6ba690f34f40de Mon Sep 17 00:00:00 2001
From: Ilango R <ilangokul@gmail.com>
Date: Wed, 9 Jan 2019 13:10:52 +0530
Subject: [PATCH 0403/2345] Fix astor url to pypi.python.org

---
 tensorflow/workspace.bzl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 5b0f6d4c84..a7f3665a31 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -285,7 +285,7 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
         system_build_file = clean_dep("//third_party/systemlibs:astor.BUILD"),
         urls = [
             "https://mirror.bazel.build/pypi.python.org/packages/99/80/f9482277c919d28bebd85813c0a70117214149a96b08981b72b63240b84c/astor-0.7.1.tar.gz",
-            "https://files.pythonhosted.org/packages/99/80/f9482277c919d28bebd85813c0a70117214149a96b08981b72b63240b84c/astor-0.7.1.tar.gz",
+            "https://pypi.python.org/packages/99/80/f9482277c919d28bebd85813c0a70117214149a96b08981b72b63240b84c/astor-0.7.1.tar.gz",
         ],
     )
 
-- 
GitLab


From 746a9750f49d98b975a4cd0ee8c0f353426ffbc0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 01:02:25 -0800
Subject: [PATCH 0404/2345] compat: Update forward compatibility horizon to
 2019-01-09

PiperOrigin-RevId: 228471346
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 5008741d55..dfee972cab 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 8)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 9)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 4d594fec0345b34de40842a0b6e39b7771dad7ec Mon Sep 17 00:00:00 2001
From: Dayananda V <dayanandasiet@gmail.com>
Date: Mon, 7 Jan 2019 15:59:12 +0530
Subject: [PATCH 0405/2345] TF Android Example compile warning fixes 1. The
 minSdk and targetSdk version should not be declared in the android manifest
 file. You can move the version from the manifest to the defaultConfig in the
 build.gradle file. Reference :
 https://developer.android.com/guide/topics/manifest/manifest-intro#uses-sdk
 2. Configuration 'compile' is obsolete and has been replaced with
 'implementation' and 'api'. Reference :
 http://d.android.com/r/tools/update-dependency-configurations.html

---
 tensorflow/lite/examples/android/app/build.gradle             | 4 ++--
 .../lite/examples/android/app/src/main/AndroidManifest.xml    | 4 ----
 tensorflow/lite/java/AndroidManifest.xml                      | 1 -
 3 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/tensorflow/lite/examples/android/app/build.gradle b/tensorflow/lite/examples/android/app/build.gradle
index e5f5c7efd1..53e502978b 100644
--- a/tensorflow/lite/examples/android/app/build.gradle
+++ b/tensorflow/lite/examples/android/app/build.gradle
@@ -45,6 +45,6 @@ project.ext.TMP_DIR   = project.buildDir.toString() + '/downloads'
 apply from: "download-models.gradle"
 
 dependencies {
-    compile fileTree(dir: 'libs', include: ['*.jar'])
-    compile 'org.tensorflow:tensorflow-lite:0.0.0-nightly'
+    implementation fileTree(dir: 'libs', include: ['*.jar'])
+    implementation 'org.tensorflow:tensorflow-lite:0.0.0-nightly'
 }
diff --git a/tensorflow/lite/examples/android/app/src/main/AndroidManifest.xml b/tensorflow/lite/examples/android/app/src/main/AndroidManifest.xml
index bc9574d646..e63c5d267e 100644
--- a/tensorflow/lite/examples/android/app/src/main/AndroidManifest.xml
+++ b/tensorflow/lite/examples/android/app/src/main/AndroidManifest.xml
@@ -24,10 +24,6 @@
     <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE"/>
     <uses-permission android:name="android.permission.RECORD_AUDIO" />
 
-    <uses-sdk
-        android:minSdkVersion="21"
-        android:targetSdkVersion="23" />
-
     <application android:allowBackup="true"
         android:debuggable="true"
         android:label="@string/app_name"
diff --git a/tensorflow/lite/java/AndroidManifest.xml b/tensorflow/lite/java/AndroidManifest.xml
index b91c6d149a..a76a727ec7 100644
--- a/tensorflow/lite/java/AndroidManifest.xml
+++ b/tensorflow/lite/java/AndroidManifest.xml
@@ -3,7 +3,6 @@
     package="org.tensorflow.lite">
 
     <uses-sdk
-        android:minSdkVersion="4"
         android:targetSdkVersion="19" />
 
     <application />
-- 
GitLab


From 1bc98108a3e835221318e019fef1642914dd5720 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Wed, 9 Jan 2019 02:37:03 -0800
Subject: [PATCH 0406/2345] Allow adding extra parameters to a cloned
 computation.

There is currently no way to clone a computation and add additional parameters
to it, except if it is a fusion computation. In a future change this will be
necessary, so we add the possibility to add such parameters via
CloneWithReplacements().

PiperOrigin-RevId: 228483852
---
 .../compiler/xla/service/hlo_computation.cc   | 18 +++++++--
 .../compiler/xla/service/hlo_computation.h    |  4 ++
 .../xla/service/hlo_computation_test.cc       | 38 +++++++++++++++++++
 3 files changed, 56 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc
index 9e15722633..f9b64d12ae 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation.cc
@@ -846,7 +846,7 @@ std::unique_ptr<HloComputation> HloComputation::Clone(
   return CloneWithReplacements(
       /*replacements=*/std::unordered_map<const HloInstruction*,
                                           std::unique_ptr<HloInstruction>>(),
-      context, suffix);
+      /*extra_parameters=*/{}, context, suffix);
 }
 
 std::unique_ptr<HloComputation> HloComputation::CloneWithReplacementPairs(
@@ -855,7 +855,8 @@ std::unique_ptr<HloComputation> HloComputation::CloneWithReplacementPairs(
   std::unordered_map<const HloInstruction*, std::unique_ptr<HloInstruction>>
       replacements;
   replacements.emplace(std::move(r1));
-  return CloneWithReplacements(std::move(replacements), context, suffix);
+  return CloneWithReplacements(std::move(replacements), /*extra_parameters=*/{},
+                               context, suffix);
 }
 
 std::unique_ptr<HloComputation> HloComputation::CloneWithReplacementPairs(
@@ -866,7 +867,8 @@ std::unique_ptr<HloComputation> HloComputation::CloneWithReplacementPairs(
       replacements;
   replacements.emplace(std::move(r1));
   replacements.emplace(std::move(r2));
-  return CloneWithReplacements(std::move(replacements), context, suffix);
+  return CloneWithReplacements(std::move(replacements), /*extra_parameters=*/{},
+                               context, suffix);
 }
 
 std::unique_ptr<HloComputation> HloComputation::CloneWithReplacementPairs(
@@ -879,12 +881,14 @@ std::unique_ptr<HloComputation> HloComputation::CloneWithReplacementPairs(
   replacements.emplace(std::move(r1));
   replacements.emplace(std::move(r2));
   replacements.emplace(std::move(r3));
-  return CloneWithReplacements(std::move(replacements), context, suffix);
+  return CloneWithReplacements(std::move(replacements), /*extra_parameters=*/{},
+                               context, suffix);
 }
 
 std::unique_ptr<HloComputation> HloComputation::CloneWithReplacements(
     std::unordered_map<const HloInstruction*, std::unique_ptr<HloInstruction>>
         replacements,
+    absl::Span<const HloInstruction* const> extra_parameters,
     HloCloneContext* context, const string& suffix) {
   std::unique_ptr<HloCloneContext> context_ptr;
   if (context == nullptr) {
@@ -950,6 +954,12 @@ std::unique_ptr<HloComputation> HloComputation::CloneWithReplacements(
   }
 
   std::vector<std::unique_ptr<HloInstruction>> instructions;
+  // First add the extra parameters to 'instructions'.
+  for (const auto& instr : extra_parameters) {
+    CHECK_EQ(instr->opcode(), HloOpcode::kParameter)
+        << "Only parameter instructions are allowed in 'extra_parameters'";
+    instructions.emplace_back(instr->Clone());
+  }
   for (auto instr : postorder) {
     std::vector<HloInstruction*> new_operands;
     for (auto operand : instr->operands()) {
diff --git a/tensorflow/compiler/xla/service/hlo_computation.h b/tensorflow/compiler/xla/service/hlo_computation.h
index a0ccbc583f..e6a1eb89cf 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.h
+++ b/tensorflow/compiler/xla/service/hlo_computation.h
@@ -323,11 +323,15 @@ class HloComputation {
   // that's not already in the computation, it's cloned and added to the new
   // computation.
   //
+  // 'extra_parameters' allows to specify additional parameters that should be
+  // added to the computation.
+  //
   // All relevant instructions are cloned, *including* unique_ptr in the
   // `replacements` map.
   std::unique_ptr<HloComputation> CloneWithReplacements(
       std::unordered_map<const HloInstruction*, std::unique_ptr<HloInstruction>>
           replacements,
+      absl::Span<const HloInstruction* const> extra_parameters = {},
       HloCloneContext* context = nullptr, const string& suffix = "clone");
 
   // Convenience overloads for CloneWithReplacements.  You want to do
diff --git a/tensorflow/compiler/xla/service/hlo_computation_test.cc b/tensorflow/compiler/xla/service/hlo_computation_test.cc
index 216d56b868..251c7bbec4 100644
--- a/tensorflow/compiler/xla/service/hlo_computation_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation_test.cc
@@ -15,7 +15,10 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
 
+#include <memory>
 #include <set>
+#include <unordered_map>
+#include <vector>
 
 #include "absl/container/flat_hash_set.h"
 #include "tensorflow/compiler/xla/literal.h"
@@ -492,6 +495,41 @@ TEST_F(HloComputationTest, CloneWithControlDependency) {
   EXPECT_THAT(successors, ::testing::ElementsAre(cloned_add));
 }
 
+TEST_F(HloComputationTest, CloneWithReplacements) {
+  auto builder = HloComputation::Builder(TestName());
+  Shape r0s64 = ShapeUtil::MakeShape(S64, {});
+  Shape r0s32 = ShapeUtil::MakeShape(S32, {});
+  Shape r0u32 = ShapeUtil::MakeShape(U32, {});
+  auto param0 = builder.AddInstruction(
+      HloInstruction::CreateParameter(0, r0f32_, "p.0.lhs"));
+  auto param1 = builder.AddInstruction(
+      HloInstruction::CreateParameter(1, r0f32_, "p.0.rhs"));
+  auto param2 =
+      builder.AddInstruction(HloInstruction::CreateParameter(2, r0s64, "p.1"));
+  auto lt = builder.AddInstruction(HloInstruction::CreateBinary(
+      ShapeUtil::MakeShape(PRED, {}), HloOpcode::kLt, param0, param1));
+  auto module = CreateNewVerifiedModule();
+  auto computation =
+      module->AddEntryComputation(builder.Build(/*root_instruction=*/lt));
+  std::unordered_map<const HloInstruction*, std::unique_ptr<HloInstruction>>
+      replacements;
+  replacements.emplace(param2,
+                       HloInstruction::CreateParameter(2, r0s32, "p.1"));
+  auto param3 = HloInstruction::CreateParameter(3, r0u32, "p.2");
+  std::vector<const HloInstruction*> extra_parameters{param3.get()};
+  auto clone = computation->CloneWithReplacements(std::move(replacements),
+                                                  extra_parameters);
+  ASSERT_EQ(clone->num_parameters(), 4);
+  EXPECT_TRUE(
+      ShapeUtil::Equal(clone->parameter_instruction(0)->shape(), r0f32_));
+  EXPECT_TRUE(
+      ShapeUtil::Equal(clone->parameter_instruction(1)->shape(), r0f32_));
+  EXPECT_TRUE(
+      ShapeUtil::Equal(clone->parameter_instruction(2)->shape(), r0s32));
+  EXPECT_TRUE(
+      ShapeUtil::Equal(clone->parameter_instruction(3)->shape(), r0u32));
+}
+
 TEST_F(HloComputationTest, Stringification) {
   const Shape s1 = ShapeUtil::MakeShape(F32, {5, 10});
   const Shape s2 = ShapeUtil::MakeShape(F32, {20, 10});
-- 
GitLab


From 94548dbeca41c3a94287bf9e506f876272904227 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 02:49:25 -0800
Subject: [PATCH 0407/2345] Object based saved model: Wire __call__ attribute
 as a method in each user object class.

PiperOrigin-RevId: 228485119
---
 tensorflow/python/saved_model/load.py      | 20 +++++++++-
 tensorflow/python/saved_model/load_test.py | 43 ++++++++++++++++++++++
 2 files changed, 62 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
index 1ee1b69b03..bbb1485612 100644
--- a/tensorflow/python/saved_model/load.py
+++ b/tensorflow/python/saved_model/load.py
@@ -85,11 +85,17 @@ class _Loader(object):
     raise ValueError("Can't convert node %s to tensor" % (type(obj)))
 
   def _load_all(self):
+    """Load all saved objects and wire their properties."""
     self._nodes = [self._recreate(proto) for proto in self._proto.nodes]
     # After creating the objects, construct the edges between the objects.
     for obj, object_proto in zip(self._nodes, self._proto.nodes):
       for reference in object_proto.children:
         setattr(obj, reference.local_name, self._nodes[reference.node_id])
+        # Note: if an object has an attribute `__call__` add a class method
+        # that allows `obj()` syntax to work. This is done per-instance to
+        # allow `callable` to be used to find out if an object is callable.
+        if reference.local_name == "__call__":
+          setattr(type(obj), "__call__", _call_attribute)
 
   def _restore_checkpoint(self):
     variables_path = saved_model_utils.get_variables_path(self._export_dir)
@@ -113,8 +119,16 @@ class _Loader(object):
     return factory[kind]()
 
   def _recreate_user_object(self, proto):
+    """Instantiates a SavedUserObject."""
     del proto
-    return tracking.Checkpointable()
+
+    # Note: each user object has its own class. This allows to make each one
+    # individually callable by adding a `__call__` method to the classes of
+    # the objects instances that have a `__call__` property.
+    class _UserObject(tracking.Checkpointable):
+      pass
+
+    return _UserObject()
 
   def _recreate_asset(self, proto):
     filename = os.path.join(
@@ -132,6 +146,10 @@ class _Loader(object):
     return variables.Variable(dummy_value, trainable=proto.trainable)
 
 
+def _call_attribute(instance, *args, **kwargs):
+  return instance.__call__(*args, **kwargs)
+
+
 def _load_saved_object_graph_proto(filename):
   with file_io.FileIO(filename, "rb") as f:
     contents = f.read()
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index 35e384dbfc..f7020e4a45 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -269,6 +269,49 @@ class LoadTest(test.TestCase):
       grad = t.gradient(loss, [imported.weight, imported.bias])
       self.assertAllClose(grad, [3.5, 1.0])
 
+  def test_callable(self):
+    class M1(tracking.Checkpointable):
+
+      @def_function.function(
+          input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
+      def __call__(self, x):
+        return x
+
+    root = tracking.Checkpointable()
+    root.m1 = M1()
+    root.m2 = tracking.Checkpointable()
+    root.m2.__call__ = def_function.function(
+        input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])(
+            lambda x: x*3.0)
+    imported = self.cycle(root)
+    x = constant_op.constant(1.0)
+
+    self.assertTrue(callable(imported.m1))
+    self.assertAllEqual(root.m1(x), imported.m1(x))
+
+    # Note: `root.m2` was not callable since `__call__` attribute was set
+    # into the instance and not on the class. But after a serialization cycle
+    # that starts to work.
+    self.assertTrue(callable(imported.m2))
+    self.assertAllEqual(root.m2.__call__(x), imported.m2(x))
+
+    # Verify that user objects without `__call__` attribute are not callable.
+    self.assertFalse(callable(imported))
+
+  def test_chain_callable(self):
+    func = def_function.function(
+        input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])(
+            lambda x: x*3.0)
+    root = tracking.Checkpointable()
+    root.__call__ = tracking.Checkpointable()
+    root.__call__.__call__ = tracking.Checkpointable()
+    root.__call__.__call__.__call__ = func
+
+    imported = self.cycle(root)
+    self.assertTrue(callable(imported))
+    x = constant_op.constant(1.0)
+    self.assertAllEqual(imported(x).numpy(), 3.0)
+
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 2db3c2ed12f7871322448c3265b82e21fa50e6aa Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Wed, 9 Jan 2019 02:50:05 -0800
Subject: [PATCH 0408/2345] Alter cast conversion so returns valid data types
 and allows a positional name arg

PiperOrigin-RevId: 228485190
---
 .../tools/compatibility/tf_upgrade_v2.py      | 11 +++++++++
 .../tools/compatibility/tf_upgrade_v2_test.py | 24 ++++++++++++++++---
 2 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 2dbbe27598..940b57891a 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -1213,10 +1213,21 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
 
     # Find out the dtype to cast to from the function name
     dtype_str = name[3:]
+    # Special cases where the full dtype is not given
+    if dtype_str == "float":
+      dtype_str = "float32"
+    elif dtype_str == "double":
+      dtype_str = "float64"
     new_arg = ast.keyword(arg="dtype",
                           value=ast.Attribute(value=ast.Name(id="tf",
                                                              ctx=ast.Load()),
                                               attr=dtype_str, ctx=ast.Load()))
+    # Ensures a valid transformation when a positional name arg is given
+    if len(node.args) == 2:
+      name_arg = ast.keyword(arg="name",
+                             value=node.args[-1])
+      node.args = node.args[:-1]
+      node.keywords.append(name_arg)
 
     # Python3 ast requires the args for the Attribute, but codegen will mess up
     # the arg order if we just set them to 0.
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 80d86d7a2b..b2bdddf15e 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -861,9 +861,27 @@ tf.print('abc')
     self.assertIn(new_text, [expected_text1, expected_text2])
 
   def testCast(self):
-    for dtype in ["int32", "int64", "float", "double",
-                  "complex64", "complex128", "bfloat16"]:
-      text = "tf.to_%s(x, name='test')" % dtype
+    for (name, dtype) in [("int32", "int32"),
+                          ("int64", "int64"),
+                          ("float", "float32"),
+                          ("double", "float64"),
+                          ("complex64", "complex64"),
+                          ("complex128", "complex128"),
+                          ("bfloat16", "bfloat16")]:
+      text = "tf.to_%s(x, name='test')" % name
+      expected_text = "tf.cast(x, name='test', dtype=tf.%s)" % dtype
+      _, unused_report, unused_errors, new_text = self._upgrade(text)
+      self.assertEqual(expected_text, new_text)
+
+  def testCastPositionalSecondArgument(self):
+    for (name, dtype) in [("int32", "int32"),
+                          ("int64", "int64"),
+                          ("float", "float32"),
+                          ("double", "float64"),
+                          ("complex64", "complex64"),
+                          ("complex128", "complex128"),
+                          ("bfloat16", "bfloat16")]:
+      text = "tf.to_%s(x, 'test')" % name
       expected_text = "tf.cast(x, name='test', dtype=tf.%s)" % dtype
       _, unused_report, unused_errors, new_text = self._upgrade(text)
       self.assertEqual(expected_text, new_text)
-- 
GitLab


From 41b0fb94457c0de1688ed595e5ff2ada068fc93f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 03:00:27 -0800
Subject: [PATCH 0409/2345] Optimize transpose_conv.

PiperOrigin-RevId: 228486143
---
 .../internal/optimized/optimized_ops.h        | 22 ++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
index 8f09fab4ca..ac68757b06 100644
--- a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
+++ b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
@@ -6079,7 +6079,27 @@ inline void TransposeConv(
     const float* filter_data, const RuntimeShape& output_shape,
     float* output_data, const RuntimeShape& im2col_shape, float* im2col_data) {
   gemmlowp::ScopedProfilingLabel label("TransposeConv");
-
+  // The complexity of the reference implementation is input.flat_size() *
+  // filter.flat_size() / in_channel.
+  //
+  // While the complexity of im2col->gemm
+  // implmentation is batch * output_height * output_width *
+  // (filter.flat_size() / out_channel)^2 * out_channel.
+  //
+  // so if input.flat_size() * out_channel^2 is much smaller than
+  // output.flat_size() * filter.size() * in_channel we should fall back to the
+  // reference implementation.
+  //
+  // TODO(b/122331966): optimize the intuitive implementation.
+  const int out_channel = output_shape.Dims(3);
+  const int in_channel = input_shape.Dims(3);
+  if ((input_shape.FlatSize() * out_channel * out_channel * 4) <
+      (filter_shape.FlatSize() * output_shape.FlatSize() * in_channel)) {
+    reference_ops::TransposeConv(params, input_shape, input_data, filter_shape,
+                                 filter_data, output_shape, output_data,
+                                 im2col_shape, im2col_data);
+    return;
+  }
   // Note we could use transposed weights with forward conv for unstrided
   // cases. But we are already getting good performance with this code as-is.
   TFLITE_DCHECK(im2col_data);
-- 
GitLab


From 964e3d0bd8bcfbfc8bcc1de16f32df8190cd7b75 Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Wed, 9 Jan 2019 03:29:29 -0800
Subject: [PATCH 0410/2345] Add conversion for tf.image.resize_* methods

PiperOrigin-RevId: 228489020
---
 tensorflow/tools/compatibility/renames_v2.py  |  5 +-
 .../tools/compatibility/tf_upgrade_v2.py      | 51 +++++++++++++++++++
 .../tools/compatibility/tf_upgrade_v2_test.py | 18 +++++++
 3 files changed, 71 insertions(+), 3 deletions(-)

diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 5ff8a86b91..e9085e94b3 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -141,8 +141,8 @@ renames = {
     'tf.diag': 'tf.linalg.tensor_diag',
     'tf.diag_part': 'tf.linalg.tensor_diag_part',
     'tf.digamma': 'tf.math.digamma',
-    'tf.dimension_at_index': 'tf.compat.v1.dimension_at_index',
-    'tf.dimension_value': 'tf.compat.v1.dimension_value',
+    'tf.dimension_at_index': 'tf.compat.dimension_at_index',
+    'tf.dimension_value': 'tf.compat.dimension_value',
     'tf.disable_eager_execution': 'tf.compat.v1.disable_eager_execution',
     'tf.disable_resource_variables': 'tf.compat.v1.disable_resource_variables',
     'tf.disable_v2_batch_normalization': 'tf.compat.v1.disable_v2_batch_normalization',
@@ -221,7 +221,6 @@ renames = {
     'tf.image.resize_area': 'tf.compat.v1.image.resize_area',
     'tf.image.resize_bicubic': 'tf.compat.v1.image.resize_bicubic',
     'tf.image.resize_bilinear': 'tf.compat.v1.image.resize_bilinear',
-    'tf.image.resize_images': 'tf.compat.v1.image.resize_images',
     'tf.image.resize_nearest_neighbor': 'tf.compat.v1.image.resize_nearest_neighbor',
     'tf.image.transpose_image': 'tf.compat.v1.image.transpose_image',
     'tf.initialize_all_tables': 'tf.compat.v1.initialize_all_tables',
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 940b57891a..e93699fdbf 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -594,6 +594,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.compat.v1.initializers.random_normal",
         "tf.truncated_normal_initializer":
             "tf.compat.v1.initializers.truncated_normal",
+        "tf.image.resize_images":
+            "tf.image.resize",
     }
     # pylint: enable=line-too-long
 
@@ -725,6 +727,11 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.to_int64": self._cast_transformer,
         "tf.nn.softmax_cross_entropy_with_logits":
             self._softmax_cross_entropy_with_logits_transformer,
+        "tf.image.resize_area": self._image_resize_transformer,
+        "tf.image.resize_bicubic": self._image_resize_transformer,
+        "tf.image.resize_bilinear": self._image_resize_transformer,
+        "tf.image.resize_nearest_neighbor": self._image_resize_transformer,
+
     }
 
     decay_function_comment = (
@@ -1292,3 +1299,47 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     logs.append((node.lineno, node.col_offset,
                  "Added keyword argument batch_dims=-1 to tf.batch_gather."))
     return node
+
+  @staticmethod
+  def _image_resize_transformer(parent, node, full_name, name, logs, errors):
+    """Transforms image.resize_* to image.resize(..., method=*, ...)."""
+
+    resize_method = name[7:].upper()
+    new_arg = ast.keyword(arg="method",
+                          value=ast.Attribute(
+                              value=ast.Attribute(
+                                  value=ast.Attribute(
+                                      value=ast.Name(id="tf", ctx=ast.Load()),
+                                      attr="image", ctx=ast.Load()),
+                                  attr="ResizeMethod", ctx=ast.Load()),
+                              attr=resize_method, ctx=ast.Load()))
+
+    # Ensures a valid transformation when a positional name arg is given
+    if len(node.args) == 4:
+      pos_arg = ast.keyword(arg="preserve_aspect_ratio",
+                            value=node.args[-1])
+      node.args = node.args[:-1]
+      node.keywords.append(pos_arg)
+    if len(node.args) == 3:
+      pos_arg = ast.keyword(arg="align_corners",
+                            value=node.args[-1])
+      node.args = node.args[:-1]
+      node.keywords.append(pos_arg)
+
+    # Python3 ast requires the args for the Attribute, but codegen will mess up
+    # the arg order if we just set them to 0.
+    new_arg.value.lineno = node.lineno
+    new_arg.value.col_offset = node.col_offset+100
+
+    node.keywords.append(new_arg)
+    if isinstance(node.func, ast.Attribute):
+      node.func.attr = "resize"
+    else:
+      assert isinstance(node.func, ast.Name)
+      node.func.id = "resize"
+
+    logs.append((node.lineno, node.col_offset,
+                 "Changed %s call to tf.image.resize(..., "
+                 "method=tf.image.ResizeMethod.%s)." % (full_name,
+                                                        resize_method)))
+    return node
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index b2bdddf15e..c1882afd5c 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -886,6 +886,24 @@ tf.print('abc')
       _, unused_report, unused_errors, new_text = self._upgrade(text)
       self.assertEqual(expected_text, new_text)
 
+  def testImageResize(self):
+    for method in ["bilinear", "area", "bicubic", "nearest_neighbor"]:
+      text = "tf.image.resize_%s(i, s)" % method
+      expected_text = ("tf.image.resize(i, s, "
+                       "method=tf.image.ResizeMethod.%s)" % method.upper())
+      _, unused_report, unused_errors, new_text = self._upgrade(text)
+      self.assertEqual(expected_text, new_text)
+
+  def testImageResizeExtraPositionalArgs(self):
+    for method in ["bilinear", "area", "bicubic", "nearest_neighbor"]:
+      text = "tf.image.resize_%s(i, s, a, p)" % method
+      expected_text = ["tf.image.resize(i, s, ", "align_corners=a, ",
+                       "preserve_aspect_ratio=p, ",
+                       "method=tf.image.ResizeMethod.%s)" % method.upper()]
+      _, unused_report, unused_errors, new_text = self._upgrade(text)
+      for s in expected_text:
+        self.assertIn(s, new_text)
+
 
 class TestUpgradeFiles(test_util.TensorFlowTestCase):
 
-- 
GitLab


From 6e6d7aad7300294425933bd053afce55107ccd35 Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Wed, 9 Jan 2019 04:31:58 -0800
Subject: [PATCH 0411/2345] Make tf.cond remove the strict arg and emit a
 warning to say it has been set to true.

Adds additional changes which ensure this warning is emitted if the original argument was positional.

PiperOrigin-RevId: 228495150
---
 tensorflow/tools/compatibility/ast_edits.py   |  8 +++----
 tensorflow/tools/compatibility/reorders_v2.py |  1 +
 .../tools/compatibility/tf_upgrade_v2.py      | 21 ++++++++++++-------
 .../tools/compatibility/tf_upgrade_v2_test.py |  8 +++++++
 4 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/tensorflow/tools/compatibility/ast_edits.py b/tensorflow/tools/compatibility/ast_edits.py
index 859fd95314..2254c223ce 100644
--- a/tensorflow/tools/compatibility/ast_edits.py
+++ b/tensorflow/tools/compatibility/ast_edits.py
@@ -213,7 +213,7 @@ class _PastaEditVisitor(ast.NodeVisitor):
       return False
 
   def _maybe_add_call_warning(self, node, full_name, name):
-    """Print a warning when specific functions are called.
+    """Print a warning when specific functions are called with selected args.
 
     The function _print_warning_for_function matches the full name of the called
     function, e.g., tf.foo.bar(). This function matches the function name that
@@ -241,13 +241,13 @@ class _PastaEditVisitor(ast.NodeVisitor):
                                              full_name, name)
 
     used_args = [kw.arg for kw in node.keywords]
-    for arg, warning in arg_warnings.items():
-      if arg in used_args:
+    for (kwarg, arg), warning in arg_warnings.items():
+      if kwarg in used_args or len(node.args) > arg:
         warned = True
         warning_message = warning.replace("<function name>", full_name or name)
         self.add_error(node.lineno, node.col_offset,
                        "%s called with %s argument requires manual check: %s." %
-                       (full_name or name, arg, warning_message))
+                       (full_name or name, kwarg, warning_message))
 
     return warned
 
diff --git a/tensorflow/tools/compatibility/reorders_v2.py b/tensorflow/tools/compatibility/reorders_v2.py
index 5c11388516..f9b0e3f9d8 100644
--- a/tensorflow/tools/compatibility/reorders_v2.py
+++ b/tensorflow/tools/compatibility/reorders_v2.py
@@ -31,6 +31,7 @@ reorders = {
     'tf.batch_gather': ['params', 'indices', 'name'],
     'tf.batch_to_space': ['input', 'crops', 'block_size', 'name'],
     'tf.boolean_mask': ['tensor', 'mask', 'name', 'axis'],
+    'tf.cond': ['pred', 'true_fn', 'false_fn', 'strict', 'name', 'fn1', 'fn2'],
     'tf.confusion_matrix': ['labels', 'predictions', 'num_classes', 'dtype', 'name', 'weights'],
     'tf.convert_to_tensor': ['value', 'dtype', 'name', 'preferred_dtype'],
     'tf.decode_csv': ['records', 'record_defaults', 'field_delim', 'use_quote_delim', 'name', 'na_value', 'select_cols'],
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index e93699fdbf..2c56cbf36b 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -50,6 +50,11 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "*.compute_gradients": {
             "colocate_gradients_with_ops": None,
         },
+        "tf.cond": {
+            "strict": None,
+            "fn1": "true_fn",
+            "fn2": "false_fn"
+        },
         "tf.argmin": {
             "dimension": "axis",
         },
@@ -620,6 +625,7 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.argmin",
         "tf.batch_gather",
         "tf.batch_to_space",
+        "tf.cond",
         "tf.nn.space_to_batch",
         "tf.boolean_mask",
         "tf.convert_to_tensor",
@@ -847,10 +853,6 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             assert_return_type_comment,
         "tf.assert_rank":
             assert_rank_comment,
-        "tf.cond":
-            "tf.cond no longer takes 'strict'. "
-            "Now 'strict' defaults to True."
-            "fn1/fn2 arguments are replaced by true_fn/false_fn.",
         "tf.debugging.assert_equal":
             assert_return_type_comment,
         "tf.debugging.assert_greater":
@@ -1146,23 +1148,28 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     # Warnings that are emitted only if a specific arg is found.
     self.function_arg_warnings = {
         "tf.gradients": {
-            "colocate_gradients_with_ops":
+            ("colocate_gradients_with_ops", 4):
                 "tf.gradients no longer takes "
                 "'colocate_gradients_with_ops' argument, it behaves as if it "
                 "was set to True.",
         },
         "*.minimize": {
-            "colocate_gradients_with_ops":
+            ("colocate_gradients_with_ops", 5):
                 "Optimizer.minimize no longer takes "
                 "'colocate_gradients_with_ops' argument, it behaves as if it "
                 "was set to True.",
         },
         "*.compute_gradients": {
-            "colocate_gradients_with_ops":
+            ("colocate_gradients_with_ops", 4):
                 "Optimizer.compute_gradients no "
                 "longer takes 'colocate_gradients_with_ops' argument, it "
                 "behaves as if it was set to True.",
         },
+        "tf.cond": {
+            ("strict", 3):
+                "tf.cond no longer takes 'strict' argument, it behaves as "
+                "if was set to True."
+        },
     }
 
     self.symbol_renames = {
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index c1882afd5c..0a85cb39c8 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -904,6 +904,14 @@ tf.print('abc')
       for s in expected_text:
         self.assertIn(s, new_text)
 
+  def testCond(self):
+    text = "tf.cond(a, b, c, True)"
+    expected_text = "tf.cond(pred=a, true_fn=b, false_fn=c)"
+    _, unused_report, errors, new_text = self._upgrade(text)
+    self.assertEqual(expected_text, new_text)
+    self.assertIn("tf.cond", errors[0])
+    self.assertIn("requires manual check", errors[0])
+
 
 class TestUpgradeFiles(test_util.TensorFlowTestCase):
 
-- 
GitLab


From 7e175f8f732806256a739e55e8f71e59c645c130 Mon Sep 17 00:00:00 2001
From: Jia Qingtong <wanywhn@qq.com>
Date: Wed, 9 Jan 2019 21:44:20 +0800
Subject: [PATCH 0412/2345] [typo] Fix a bazel url typo

Signed-off-by: Jia Qingtong <wanywhn@qq.com>
---
 tensorflow/workspace.bzl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 5b0f6d4c84..2ce70557fa 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -141,7 +141,7 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
         sha256 = "9de38f2d162c51599b802f7c36d9f3773980d19ac908c61638f8344d2c10e1ca",
         strip_prefix = "eigen-eigen-88fc23324517",
         urls = [
-            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/88fc23324517..tar.gz",
+            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/88fc23324517.tar.gz",
             "https://bitbucket.org/eigen/eigen/get/88fc23324517.tar.gz",
         ],
     )
-- 
GitLab


From b5fdd811998d495950c4b736879454be39acf163 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 07:08:38 -0800
Subject: [PATCH 0413/2345] Stop using set for signatures comparison. Sets
 internal hashing mechanism makes it impossible to use any kind of structures
 (lists, etc.).

Also do some refactoring.

PiperOrigin-RevId: 228511638
---
 tensorflow/python/eager/def_function.py | 16 +++++++++------
 tensorflow/python/eager/function.py     | 27 ++++++++++++++++++-------
 2 files changed, 30 insertions(+), 13 deletions(-)

diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py
index aa4f20df49..4c22a12c1c 100644
--- a/tensorflow/python/eager/def_function.py
+++ b/tensorflow/python/eager/def_function.py
@@ -446,10 +446,12 @@ class PolymorphicFunction(object):
   @property
   def _cached_input_signatures(self):
     """All input signatures used to call this PolymorphicFunction."""
-    seen = set()
-    # Preserves signature ordering rather than returning a set() so that we
-    # don't need to re-sort signatures later to work around Python 2's set
-    # nondeterminism.
+    seen = list()
+    # We are using a list so that:
+    #  - the returned collection is deterministic, and
+    #  - we can use a custom equality operator (is_same_structure).
+    # This is run only at serialization time on likely very small inputs so we
+    # are not concerned about O(n^2) runtime.
     # pylint: disable=protected-access
     concrete_functions = []
     if self._stateful_fn:
@@ -458,9 +460,11 @@ class PolymorphicFunction(object):
       concrete_functions.extend(self._stateless_fn._function_cache.values())
     for concrete_function in concrete_functions:
       signature = concrete_function._python_call_signature
-      if signature not in seen:
+      equal_to_signature = functools.partial(
+          function_lib.is_same_structure, signature, check_values=True)
+      if not any(equal_to_signature(s) for s in seen):
         yield signature
-        seen.add(signature)
+        seen.append(signature)
     # pylint: enable=protected-access
 
   def get_concrete_function(self, *args, **kwargs):
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 05fccfbcd9..67c633726f 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -74,6 +74,24 @@ CacheKey = collections.namedtuple("CacheKey", [
 ])
 
 
+def is_same_structure(structure1,
+                      structure2,
+                      check_values=False):
+  """Check two structures for equality, optionally of types and of values."""
+  try:
+    nest.assert_same_structure(structure1, structure2)
+  except (ValueError, TypeError):
+    return False
+  if check_values:
+    flattened1 = nest.flatten(structure1)
+    flattened2 = nest.flatten(structure2)
+    # First check the types to avoid AttributeErrors.
+    if any(type(f1) != type(f2) for f1, f2 in zip(flattened1, flattened2)):
+      return False
+    return flattened1 == flattened2
+  return True
+
+
 def _parse_func_attrs(attributes):
   """Convert the keyword arguments into function_def attributes.
 
@@ -919,10 +937,7 @@ class FunctionSpec(object):
     else:
       assert not kwargs
       signature_relevant_inputs = inputs[:len(self.input_signature)]
-      try:
-        nest.assert_same_structure(self.input_signature,
-                                   signature_relevant_inputs)
-      except (ValueError, TypeError):
+      if not is_same_structure(self.input_signature, signature_relevant_inputs):
         raise ValueError("Structure of Python function inputs does not match "
                          "input_signature.")
       signature_inputs_flat = nest.flatten(signature_relevant_inputs)
@@ -1049,9 +1064,7 @@ class PolymorphicFunction(object):
                          "input_signature is provided.")
       if args:
         # If args are provided, they must match the input signature.
-        try:
-          nest.assert_same_structure(self._input_signature, args)
-        except (ValueError, TypeError):
+        if not is_same_structure(self._input_signature, args):
           raise ValueError("Structure of Python function inputs does not match "
                            "input_signature.")
         flat_inputs = nest.flatten(args)
-- 
GitLab


From 433d47e39ce5e1c8c478393e7f5d2917c7df0fa7 Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Wed, 9 Jan 2019 08:00:10 -0800
Subject: [PATCH 0414/2345] Add symmetric quantization support to the accuracy
 tool.

PiperOrigin-RevId: 228517966
---
 .../ilsvrc/imagenet_model_evaluator.cc        | 26 ++++------
 .../accuracy/ilsvrc/imagenet_topk_eval.cc     |  9 +++-
 .../ilsvrc/inception_preprocessing.cc         | 42 +++++++++------
 .../accuracy/ilsvrc/inception_preprocessing.h | 52 ++++++++++++++-----
 .../ilsvrc/inception_preprocessing_test.cc    | 41 ++++++++++++---
 5 files changed, 119 insertions(+), 51 deletions(-)

diff --git a/tensorflow/lite/tools/accuracy/ilsvrc/imagenet_model_evaluator.cc b/tensorflow/lite/tools/accuracy/ilsvrc/imagenet_model_evaluator.cc
index 9a74e221c1..129747fe4d 100644
--- a/tensorflow/lite/tools/accuracy/ilsvrc/imagenet_model_evaluator.cc
+++ b/tensorflow/lite/tools/accuracy/ilsvrc/imagenet_model_evaluator.cc
@@ -22,6 +22,12 @@ limitations under the License.
 
 #include "absl/memory/memory.h"
 #include "tensorflow/cc/framework/scope.h"
+#include "tensorflow/core/lib/core/blocking_counter.h"
+#include "tensorflow/core/lib/core/threadpool.h"
+#include "tensorflow/core/platform/init_main.h"
+#include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/public/session.h"
+#include "tensorflow/core/util/command_line_flags.h"
 #include "tensorflow/lite/tools/accuracy/eval_pipeline.h"
 #include "tensorflow/lite/tools/accuracy/eval_pipeline_builder.h"
 #include "tensorflow/lite/tools/accuracy/file_reader_stage.h"
@@ -29,12 +35,6 @@ limitations under the License.
 #include "tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing.h"
 #include "tensorflow/lite/tools/accuracy/run_tflite_model_stage.h"
 #include "tensorflow/lite/tools/accuracy/utils.h"
-#include "tensorflow/core/lib/core/blocking_counter.h"
-#include "tensorflow/core/lib/core/threadpool.h"
-#include "tensorflow/core/platform/init_main.h"
-#include "tensorflow/core/platform/mutex.h"
-#include "tensorflow/core/public/session.h"
-#include "tensorflow/core/util/command_line_flags.h"
 
 namespace {
 using tensorflow::string;
@@ -185,21 +185,17 @@ Status EvaluateModelForShard(const uint64_t shard_id,
   const TensorShape& input_shape = model_info.input_shapes[0];
   const int image_height = input_shape.dim_size(1);
   const int image_width = input_shape.dim_size(2);
-  const bool is_quantized = (model_info.input_types[0] == DT_UINT8);
 
   RunTFLiteModelStage::Params tfl_model_params;
   tfl_model_params.model_file_path = params.model_file_path;
-  if (is_quantized) {
-    tfl_model_params.input_type = {DT_UINT8};
-    tfl_model_params.output_type = {DT_UINT8};
-  } else {
-    tfl_model_params.input_type = {DT_FLOAT};
-    tfl_model_params.output_type = {DT_FLOAT};
-  }
+
+  tfl_model_params.input_type = {model_info.input_types[0]};
+  tfl_model_params.output_type = {model_info.input_types[0]};
 
   Scope root = Scope::NewRootScope();
   FileReaderStage reader;
-  InceptionPreprocessingStage inc(image_height, image_width, is_quantized);
+  InceptionPreprocessingStage inc(image_height, image_width,
+                                  model_info.input_types[0]);
   RunTFLiteModelStage tfl_model_stage(tfl_model_params);
   EvalPipelineBuilder builder;
 
diff --git a/tensorflow/lite/tools/accuracy/ilsvrc/imagenet_topk_eval.cc b/tensorflow/lite/tools/accuracy/ilsvrc/imagenet_topk_eval.cc
index 2b086cdf70..f5642d52a8 100644
--- a/tensorflow/lite/tools/accuracy/ilsvrc/imagenet_topk_eval.cc
+++ b/tensorflow/lite/tools/accuracy/ilsvrc/imagenet_topk_eval.cc
@@ -67,11 +67,18 @@ Status ImagenetTopKAccuracy::ComputeEval(
     for (size_t i = 0; i < probs.size(); i++) {
       probabilities.push_back(probs(i));
     }
-  } else {
+  } else if (output.dtype() == DT_UINT8) {
     auto probs = output.flat<uint8>();
     for (size_t i = 0; i < probs.size(); i++) {
       probabilities.push_back(probs(i));
     }
+  } else if (output.dtype() == DT_INT8) {
+    auto probs = output.flat<int8>();
+    for (size_t i = 0; i < probs.size(); i++) {
+      probabilities.push_back(probs(i));
+    }
+  } else {
+    return errors::InvalidArgument("Invalid datatype");
   }
 
   CHECK_EQ(kNumCategories, probabilities.size());
diff --git a/tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing.cc b/tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing.cc
index 9a889f0dd8..04b6cb7558 100644
--- a/tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing.cc
+++ b/tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing.cc
@@ -57,23 +57,33 @@ void InceptionPreprocessingStage::AddToGraph(const Scope& scope,
   tensorflow::Output cropped_image;
   CentralCropImage(s, decoded_jpeg, params_.cropping_fraction, &cropped_image);
   auto dims_expander = ops::ExpandDims(s, cropped_image, 0);
-  auto resized_image = ops::ResizeBilinear(
-      s, dims_expander,
-      ops::Const(s.WithOpName("size"), {image_height_, image_width_}));
-  if (is_quantized_) {
-    this->stage_output_ =
-        ops::Cast(s.WithOpName(output_name()), resized_image, DT_UINT8);
-  } else {
-    auto squeezed_image = ops::Squeeze(s, resized_image);
-    auto normalized_image =
-        ops::Div(s,
-                 ops::Sub(s, squeezed_image,
-                          {params_.input_means[0], params_.input_means[1],
-                           params_.input_means[2]}),
-                 {params_.scale});
-    this->stage_output_ =
-        ops::ExpandDims(s.WithOpName(output_name()), normalized_image, {0});
+  auto resized_image =
+      ops::ResizeBilinear(s.WithOpName("resize"), dims_expander,
+                          ops::Const(s, {image_height_, image_width_}));
+
+  ::tensorflow::Output preprocessed_image = resized_image;
+
+  if (!params_.input_means.empty()) {
+    preprocessed_image =
+        ops::Sub(s.WithOpName("sub"), preprocessed_image,
+                 {params_.input_means[0], params_.input_means[1],
+                  params_.input_means[2]});
+  }
+
+  if (std::abs(params_.scale) > 1e-7f) {
+    auto squeezed_image = ops::Squeeze(s, preprocessed_image);
+    preprocessed_image = ops::Div(s, squeezed_image, {params_.scale});
+    preprocessed_image = ops::ExpandDims(s, preprocessed_image, {0});
   }
+
+  // Cast the output from float to output datatype.
+  if (output_datatype_ != DT_FLOAT) {
+    preprocessed_image =
+        ops::Cast(s.WithOpName("cast"), preprocessed_image, output_datatype_);
+  }
+
+  this->stage_output_ =
+      ops::Identity(s.WithOpName(output_name()), preprocessed_image);
 }
 
 }  // namespace metrics
diff --git a/tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing.h b/tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing.h
index 4a1d3ce476..371feb3e76 100644
--- a/tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing.h
+++ b/tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing.h
@@ -13,14 +13,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_LITE_TOOLS_ACCURACY_INCEPTION_PREPROCESSING_H_
-#define TENSORFLOW_LITE_TOOLS_ACCURACY_INCEPTION_PREPROCESSING_H_
+#ifndef TENSORFLOW_LITE_TOOLS_ACCURACY_ILSVRC_INCEPTION_PREPROCESSING_H_
+#define TENSORFLOW_LITE_TOOLS_ACCURACY_ILSVRC_INCEPTION_PREPROCESSING_H_
 
 #include <utility>
 
-#include "tensorflow/lite/tools/accuracy/stage.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/lite/tools/accuracy/stage.h"
 
 namespace tensorflow {
 namespace metrics {
@@ -31,28 +31,53 @@ namespace metrics {
 // shape {1, image_height, image_width, 3}, where 3 is the number of channels.
 class InceptionPreprocessingStage : public Stage {
  public:
+  // Preprocessing params that govern scaling and normalization of channels of
+  // the image.
   struct Params {
+    // Input means are subtracted from each channel.
+    // In case of an empty vector this is skipped.
     std::vector<float> input_means;
+    // Scale is used to divide the input.
+    // A scale of 0 means divison is skipped.
     float scale;
     double cropping_fraction;
   };
 
-  static Params DefaultParams() {
-    return {.input_means = {127.5, 127.5, 127.5},
-            .scale = 127.5,
-            .cropping_fraction = 0.875};
+  // Default preprocessing for inception stage based on |output_type|
+  static Params DefaultParamsForType(DataType output_type) {
+    const float kCroppingFraction = 0.875;
+    Params params = {};
+    params.cropping_fraction = kCroppingFraction;
+    if (output_type == DT_UINT8) {
+    } else if (output_type == DT_INT8) {
+      params.input_means = {128.0, 128.0, 128.0};
+    } else {
+      // Assume floating point preprocessing.
+      params.input_means = {127.5, 127.5, 127.5};
+      params.scale = 127.5;
+    }
+    return params;
+  }
+
+  // Creates a new preprocessing stage object with provided |image_width|
+  // |image_height| as the size of output image.
+  // |output_datatype| is the datatype of output of the stage.
+  InceptionPreprocessingStage(int image_width, int image_height,
+                              DataType output_datatype)
+      : output_datatype_(output_datatype),
+        image_width_(image_width),
+        image_height_(image_height) {
+    params_ = DefaultParamsForType(output_datatype);
   }
 
   // Creates a new preprocessing stage object with provided |image_width|
   // |image_height| as the size of output image.
-  // If |is_quantized| is set to true then |params| is ignored since quantized
-  // images don't go through any preprocessing.
+  // |output_datatype| is the datatype of output of the stage.
   InceptionPreprocessingStage(int image_width, int image_height,
-                              bool is_quantized,
-                              Params params = DefaultParams())
-      : image_width_(image_width),
+                              DataType output_datatype, Params params)
+      : output_datatype_(output_datatype),
+        image_width_(image_width),
         image_height_(image_height),
-        is_quantized_(is_quantized),
         params_(std::move(params)) {}
 
   string name() const override { return "stage_inception_preprocess"; }
@@ -63,6 +88,7 @@ class InceptionPreprocessingStage : public Stage {
   void AddToGraph(const Scope& scope, const Input& input) override;
 
  private:
+  DataType output_datatype_;
   int image_width_;
   int image_height_;
   bool is_quantized_;
diff --git a/tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing_test.cc b/tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing_test.cc
index 5d0e01d7d1..f88847035f 100644
--- a/tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing_test.cc
+++ b/tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing_test.cc
@@ -17,10 +17,10 @@ limitations under the License.
 #include <string>
 
 #include <gtest/gtest.h>
-#include "tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/public/session.h"
 #include "tensorflow/core/util/command_line_flags.h"
+#include "tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing.h"
 
 namespace {
 tensorflow::string* g_test_image_file = nullptr;
@@ -48,7 +48,7 @@ Status GetContents(const string& filename, string* output) {
   }
 }
 
-TEST(InceptionPreprocessingTest, TestImagePreprocessQuantized) {
+TEST(InceptionPreprocessingTest, TestImagePreprocessUInt8Quantized) {
   ASSERT_TRUE(g_test_image_file != nullptr);
   string image_contents;
   string image_path = *g_test_image_file;
@@ -56,8 +56,8 @@ TEST(InceptionPreprocessingTest, TestImagePreprocessQuantized) {
   ASSERT_TRUE(status.ok()) << status.error_message();
   const int width = 224;
   const int height = 224;
-  const bool is_quantized = true;
-  InceptionPreprocessingStage preprocess_stage(width, height, is_quantized);
+  auto params = InceptionPreprocessingStage::DefaultParamsForType(DT_UINT8);
+  InceptionPreprocessingStage preprocess_stage(width, height, DT_UINT8, params);
   Scope scope = Scope::NewRootScope();
   preprocess_stage.AddToGraph(scope, image_contents);
   TF_CHECK_OK(scope.status());
@@ -77,6 +77,35 @@ TEST(InceptionPreprocessingTest, TestImagePreprocessQuantized) {
   EXPECT_TRUE(outputs[0].shape().IsSameSize({1, 224, 224, 3}));
 }
 
+TEST(InceptionPreprocessingTest, TestImagePreprocessInt8Quantized) {
+  ASSERT_TRUE(g_test_image_file != nullptr);
+  string image_contents;
+  string image_path = *g_test_image_file;
+  auto status = GetContents(image_path, &image_contents);
+  ASSERT_TRUE(status.ok()) << status.error_message();
+  const int width = 224;
+  const int height = 224;
+  auto params = InceptionPreprocessingStage::DefaultParamsForType(DT_INT8);
+  InceptionPreprocessingStage preprocess_stage(width, height, DT_INT8, params);
+  Scope scope = Scope::NewRootScope();
+  preprocess_stage.AddToGraph(scope, image_contents);
+  TF_CHECK_OK(scope.status());
+
+  GraphDef graph_def;
+  TF_CHECK_OK(scope.ToGraphDef(&graph_def));
+  std::unique_ptr<Session> session(NewSession(SessionOptions()));
+  TF_CHECK_OK(session->Create(graph_def));
+  std::vector<Tensor> outputs;
+  auto run_status =
+      session->Run({},                                   /*inputs*/
+                   {preprocess_stage.output_name()}, {}, /*target node names */
+                   &outputs);
+  TF_CHECK_OK(run_status);
+  EXPECT_EQ(1, outputs.size());
+  EXPECT_EQ(DT_INT8, outputs[0].dtype());
+  EXPECT_TRUE(outputs[0].shape().IsSameSize({1, 224, 224, 3}));
+}
+
 TEST(InceptionPreprocessingTest, TestImagePreprocessFloat) {
   ASSERT_TRUE(g_test_image_file != nullptr);
   string image_contents;
@@ -85,8 +114,8 @@ TEST(InceptionPreprocessingTest, TestImagePreprocessFloat) {
   ASSERT_TRUE(status.ok()) << status.error_message();
   const int width = 224;
   const int height = 224;
-  const bool is_quantized = false;
-  InceptionPreprocessingStage preprocess_stage(width, height, is_quantized);
+  auto params = InceptionPreprocessingStage::DefaultParamsForType(DT_FLOAT);
+  InceptionPreprocessingStage preprocess_stage(width, height, DT_FLOAT, params);
   Scope scope = Scope::NewRootScope();
   preprocess_stage.AddToGraph(scope, image_contents);
   TF_CHECK_OK(scope.status());
-- 
GitLab


From eed1c1413ff2c45c82142d5d4dd8b6d51a2f5636 Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Wed, 9 Jan 2019 08:11:28 -0800
Subject: [PATCH 0415/2345] Automated rollback of commit
 9f3acf3d89a4ef5f9313a6e2cb2d81e03177e8ef

PiperOrigin-RevId: 228519835
---
 tensorflow/lite/python/BUILD      |  1 +
 tensorflow/lite/python/convert.py | 13 +++++--------
 tensorflow/lite/toco/python/BUILD | 21 ++++++++++++++++++---
 3 files changed, 24 insertions(+), 11 deletions(-)

diff --git a/tensorflow/lite/python/BUILD b/tensorflow/lite/python/BUILD
index 7bfd1a8996..54b925cc05 100644
--- a/tensorflow/lite/python/BUILD
+++ b/tensorflow/lite/python/BUILD
@@ -74,6 +74,7 @@ py_test(
     data = ["@tflite_mobilenet_ssd_quant_protobuf//:tflite_graph.pb"],
     srcs_version = "PY2AND3",
     tags = [
+        "no_oss",
         "no_windows",
     ],
     deps = [
diff --git a/tensorflow/lite/python/convert.py b/tensorflow/lite/python/convert.py
index 9f315fc874..9c60399871 100644
--- a/tensorflow/lite/python/convert.py
+++ b/tensorflow/lite/python/convert.py
@@ -37,14 +37,11 @@ from tensorflow.python.util.tf_export import tf_export as _tf_export
 
 # Lazy load since some of the performance benchmark skylark rules
 # break dependencies.
-if lite_constants.EXPERIMENTAL_USE_TOCO_API_DIRECTLY:
-  _toco_python = LazyLoader(
-      "tensorflow_wrap_toco", globals(),
-      "tensorflow.lite.toco.python."
-      "tensorflow_wrap_toco")
-  del LazyLoader
-else:
-  _toco_python = None
+_toco_python = LazyLoader(
+    "tensorflow_wrap_toco", globals(),
+    "tensorflow.lite.toco.python."
+    "tensorflow_wrap_toco")
+del LazyLoader
 
 # Find the toco_from_protos binary using the resource loader if using from
 # bazel, otherwise we are in a pip where console_scripts already has
diff --git a/tensorflow/lite/toco/python/BUILD b/tensorflow/lite/toco/python/BUILD
index f3ea5b7bfe..8a6e82ec46 100644
--- a/tensorflow/lite/toco/python/BUILD
+++ b/tensorflow/lite/toco/python/BUILD
@@ -10,20 +10,35 @@ load("//tensorflow:tensorflow.bzl", "tf_py_wrap_cc")
 load("//tensorflow:tensorflow.bzl", "tf_py_test")
 load("//tensorflow:tensorflow.bzl", "py_binary")
 
+config_setting(
+    name = "tflite_convert_with_select_tf_ops",
+    define_values = {"tflite_convert_with_select_tf_ops": "true"},
+    visibility = [
+        "//tensorflow/contrib/lite:__subpackages__",
+        "//tensorflow/lite:__subpackages__",
+    ],
+)
+
 cc_library(
     name = "toco_python_api",
     srcs = ["toco_python_api.cc"],
     hdrs = ["toco_python_api.h"],
     deps = [
+        "//third_party/python_runtime:headers",
         "//tensorflow/core:lib",
-        "//tensorflow/core:ops",
         "//tensorflow/lite/toco:model_flags_proto_cc",
         "//tensorflow/lite/toco:toco_flags_proto_cc",
         "//tensorflow/lite/toco:toco_graphviz_dump_options",
         "//tensorflow/lite/toco:toco_port",
         "//tensorflow/lite/toco:toco_tooling",
-        "//third_party/python_runtime:headers",
-    ],
+    ] + select({
+        # This is required when running `tflite_convert` from `bazel`.
+        # It requires to link with TensorFlow Ops to get the op definitions.
+        ":tflite_convert_with_select_tf_ops": [
+            "//tensorflow/core:ops",
+        ],
+        "//conditions:default": [],
+    }),
 )
 
 tf_py_wrap_cc(
-- 
GitLab


From beae80428e5a7d67417dc18561b8ed74724fe4bf Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Wed, 9 Jan 2019 08:32:27 -0800
Subject: [PATCH 0416/2345] Move explicit template instantiations to namespace
 scope.

Fixes compile error on mac os on attached bug causing 1.13
build problems.

PiperOrigin-RevId: 228522576
---
 tensorflow/lite/testing/tflite_driver.cc | 121 ++++++++++++-----------
 1 file changed, 62 insertions(+), 59 deletions(-)

diff --git a/tensorflow/lite/testing/tflite_driver.cc b/tensorflow/lite/testing/tflite_driver.cc
index ffe296432a..a637dc86c0 100644
--- a/tensorflow/lite/testing/tflite_driver.cc
+++ b/tensorflow/lite/testing/tflite_driver.cc
@@ -79,32 +79,7 @@ class TfLiteDriver::Expectation {
     SetTensorData(values, &data_);
   }
 
-  template <>
-  void SetData<string>(const string& csv_values) {
-    string s = absl::HexStringToBytes(csv_values);
-    data_.raw = new char[s.size()];
-    memcpy(data_.raw, s.data(), s.size());
-  }
-
-  bool Check(bool verbose, const TfLiteTensor& tensor) {
-    switch (tensor.type) {
-      case kTfLiteFloat32:
-        return TypedCheck<float>(verbose, tensor);
-      case kTfLiteInt32:
-        return TypedCheck<int32_t>(verbose, tensor);
-      case kTfLiteInt64:
-        return TypedCheck<int64_t>(verbose, tensor);
-      case kTfLiteUInt8:
-        return TypedCheck<uint8_t>(verbose, tensor);
-      case kTfLiteBool:
-        return TypedCheck<bool>(verbose, tensor);
-      case kTfLiteString:
-        return TypedCheck<string>(verbose, tensor);
-      default:
-        fprintf(stderr, "Unsupported type %d in Check\n", tensor.type);
-        return false;
-    }
-  }
+  bool Check(bool verbose, const TfLiteTensor& tensor);
 
  private:
   template <typename T>
@@ -146,49 +121,77 @@ class TfLiteDriver::Expectation {
     return good_output;
   }
 
-  template <>
-  bool TypedCheck<string>(bool verbose, const TfLiteTensor& tensor) {
-    if (tensor.data.raw == nullptr) {
+  TfLitePtrUnion data_;
+  size_t num_elements_;
+};
+
+template <>
+void TfLiteDriver::Expectation::SetData<string>(const string& csv_values) {
+  string s = absl::HexStringToBytes(csv_values);
+  data_.raw = new char[s.size()];
+  memcpy(data_.raw, s.data(), s.size());
+}
+
+template <>
+bool TfLiteDriver::Expectation::TypedCheck<string>(bool verbose,
+                                                   const TfLiteTensor& tensor) {
+  if (tensor.data.raw == nullptr) {
+    if (verbose) {
+      std::cerr << "  got empty string" << std::endl;
+    }
+    return false;
+  }
+  int expected_num_strings = GetStringCount(data_.raw);
+  int returned_num_strings = GetStringCount(tensor.data.raw);
+  if (expected_num_strings != returned_num_strings) {
+    if (verbose) {
+      std::cerr << "  string count differ: got " << returned_num_strings
+                << ", but expected " << expected_num_strings << std::endl;
+    }
+    return false;
+  }
+  for (int i = 0; i < returned_num_strings; ++i) {
+    auto expected_ref = GetString(data_.raw, i);
+    auto returned_ref = GetString(tensor.data.raw, i);
+    if (expected_ref.len != returned_ref.len) {
       if (verbose) {
-        std::cerr << "  got empty string" << std::endl;
+        std::cerr << "  index " << i << ": got string of size "
+                  << returned_ref.len << ", but expected size "
+                  << expected_ref.len << std::endl;
       }
       return false;
     }
-    int expected_num_strings = GetStringCount(data_.raw);
-    int returned_num_strings = GetStringCount(tensor.data.raw);
-    if (expected_num_strings != returned_num_strings) {
+    if (strncmp(expected_ref.str, returned_ref.str, returned_ref.len) != 0) {
       if (verbose) {
-        std::cerr << "  string count differ: got " << returned_num_strings
-                  << ", but expected " << expected_num_strings << std::endl;
+        std::cerr << "  index " << i << ": strings are different" << std::endl;
       }
       return false;
     }
-    for (int i = 0; i < returned_num_strings; ++i) {
-      auto expected_ref = GetString(data_.raw, i);
-      auto returned_ref = GetString(tensor.data.raw, i);
-      if (expected_ref.len != returned_ref.len) {
-        if (verbose) {
-          std::cerr << "  index " << i << ": got string of size "
-                    << returned_ref.len << ", but expected size "
-                    << expected_ref.len << std::endl;
-        }
-        return false;
-      }
-      if (strncmp(expected_ref.str, returned_ref.str, returned_ref.len) != 0) {
-        if (verbose) {
-          std::cerr << "  index " << i << ": strings are different"
-                    << std::endl;
-        }
-        return false;
-      }
-    }
-
-    return true;
   }
 
-  TfLitePtrUnion data_;
-  size_t num_elements_;
-};
+  return true;
+}
+
+bool TfLiteDriver::Expectation::Check(bool verbose,
+                                      const TfLiteTensor& tensor) {
+  switch (tensor.type) {
+    case kTfLiteFloat32:
+      return TypedCheck<float>(verbose, tensor);
+    case kTfLiteInt32:
+      return TypedCheck<int32_t>(verbose, tensor);
+    case kTfLiteInt64:
+      return TypedCheck<int64_t>(verbose, tensor);
+    case kTfLiteUInt8:
+      return TypedCheck<uint8_t>(verbose, tensor);
+    case kTfLiteBool:
+      return TypedCheck<bool>(verbose, tensor);
+    case kTfLiteString:
+      return TypedCheck<string>(verbose, tensor);
+    default:
+      fprintf(stderr, "Unsupported type %d in Check\n", tensor.type);
+      return false;
+  }
+}
 
 TfLiteDriver::TfLiteDriver(bool use_nnapi, const string& delegate_name,
                            bool reference_kernel)
-- 
GitLab


From 5922eee396da936af1451c22e0433071f909eeaa Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 09:15:17 -0800
Subject: [PATCH 0417/2345] Build file changes.

PiperOrigin-RevId: 228528921
---
 tensorflow/BUILD | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 0b4498ef12..29d71c323a 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -385,7 +385,6 @@ package_group(
         "-//third_party/tensorflow/python/estimator",
         "//learning/deepmind/...",
         "//learning/meta_rank/...",
-        "//learning/pathways/...",  # While dataset C++ api requires internals
         "//tensorflow/...",
         "//tensorflow_estimator/contrib/...",
         "//tensorflow_fold/llgtm/...",
-- 
GitLab


From 1ee193a2563d51ee45b401f2cff91f6e480e21db Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Wed, 9 Jan 2019 09:33:54 -0800
Subject: [PATCH 0418/2345] Remove aux-bin from pip package, it's not needed
 any more.

PiperOrigin-RevId: 228531942
---
 tensorflow/tools/pip_package/MANIFEST.in          | 1 -
 tensorflow/tools/pip_package/build_pip_package.sh | 3 ---
 2 files changed, 4 deletions(-)

diff --git a/tensorflow/tools/pip_package/MANIFEST.in b/tensorflow/tools/pip_package/MANIFEST.in
index 272ff4735c..c304e8cf6e 100644
--- a/tensorflow/tools/pip_package/MANIFEST.in
+++ b/tensorflow/tools/pip_package/MANIFEST.in
@@ -6,7 +6,6 @@ recursive-include * *.so
 recursive-include * *.dll
 recursive-include * *.lib
 recursive-include * *.csv
-recursive-include tensorflow/aux-bin *
 recursive-include tensorflow/include/tensorflow *.h
 recursive-include tensorflow/include/Eigen *
 recursive-include tensorflow/include/external *
diff --git a/tensorflow/tools/pip_package/build_pip_package.sh b/tensorflow/tools/pip_package/build_pip_package.sh
index 439b5428b3..27815491d2 100755
--- a/tensorflow/tools/pip_package/build_pip_package.sh
+++ b/tensorflow/tools/pip_package/build_pip_package.sh
@@ -118,9 +118,6 @@ function prepare_src() {
         fi
       fi
     fi
-    mkdir "${TMPDIR}/tensorflow/aux-bin"
-    # Install toco as a binary in aux-bin.
-    cp bazel-bin/tensorflow/lite/python/tflite_convert ${TMPDIR}/tensorflow/aux-bin/
   fi
 
   # protobuf pip package doesn't ship with header files. Copy the headers
-- 
GitLab


From 2da8e8c60343d9e833f1c591c54ee2a27c240842 Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Wed, 9 Jan 2019 10:23:54 -0800
Subject: [PATCH 0419/2345] Change so that error message received if attempting
 to do + between a python literal and tensor with incompatible dtype to come
 from the op

PiperOrigin-RevId: 228541386
---
 tensorflow/python/ops/math_ops.py      |  3 ++-
 tensorflow/python/ops/math_ops_test.py | 18 ++++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index 5bccf5493f..248d092538 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -813,7 +813,8 @@ def _OverrideBinaryOperatorHelper(func, op_name, clazz_object=ops.Tensor):
         return func(x, y, name=name)
       elif not isinstance(y, sparse_tensor.SparseTensor):
         try:
-          y = ops.convert_to_tensor(y, dtype=x.dtype.base_dtype, name="y")
+          y = ops.convert_to_tensor_v2(y, dtype_hint=x.dtype.base_dtype,
+                                       name="y")
         except TypeError:
           # If the RHS is not a tensor, it might be a tensor aware object
           # that can implement the operator with knowledge of itself
diff --git a/tensorflow/python/ops/math_ops_test.py b/tensorflow/python/ops/math_ops_test.py
index b27cf7208c..b4832e09c0 100644
--- a/tensorflow/python/ops/math_ops_test.py
+++ b/tensorflow/python/ops/math_ops_test.py
@@ -22,6 +22,7 @@ import numpy as np
 from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
@@ -664,5 +665,22 @@ class NextAfterTest(test_util.TensorFlowTestCase):
       self.assertAllEqual(math_ops.nextafter(one, two) - one, eps_const)
 
 
+class BinaryOpsTest(test_util.TensorFlowTestCase):
+
+  @test_util.run_in_graph_and_eager_modes
+  def testErrorReceivedIfDtypeMismatchFromOp(self):
+    if context.executing_eagerly():
+      error = errors_impl.InvalidArgumentError
+      error_message = (
+          r"cannot compute Add as input #0\(zero-based\) was expected to be a "
+          r"float tensor but is a int32 tensor \[Op:Add\] name: add/")
+    else:
+      error = TypeError
+      error_message = ("Input 'y' of 'Add' Op has type float32 that does not "
+                       "match type int32 of argument 'x'.")
+    with self.assertRaisesRegexp(error, error_message):
+      a = array_ops.ones([1], dtype=dtypes.int32) + 1.0
+      self.evaluate(a)
+
 if __name__ == "__main__":
   googletest.main()
-- 
GitLab


From 2c76c0a07fbf82d5bc8cfc81e688560d0b7e4537 Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Wed, 9 Jan 2019 10:24:39 -0800
Subject: [PATCH 0420/2345] Add dependency on google_pasta (freshly minted on
 pypi).

PiperOrigin-RevId: 228541538
---
 tensorflow/tools/pip_package/setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index 9567e0aff7..55b7046e30 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -51,6 +51,7 @@ REQUIRED_PACKAGES = [
     'absl-py >= 0.1.6',
     'astor >= 0.6.0',
     'gast >= 0.2.0',
+    'google_pasta >= 0.1.0',
     'keras_applications >= 1.0.6',
     'keras_preprocessing >= 1.0.5',
     'numpy >= 1.13.3',
-- 
GitLab


From 864fe25ff7363730ef81e402d6a10cc758df72cd Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Wed, 9 Jan 2019 10:52:22 -0800
Subject: [PATCH 0421/2345] Align Android gradle build files

Use Gradle 3.1.4 and modernize the TFLite example gradle files.

PiperOrigin-RevId: 228547238
---
 .../lite/examples/android/app/build.gradle    |  6 ++---
 tensorflow/lite/examples/android/build.gradle |  2 +-
 .../lite/g3doc/tfmobile/android_build.md      |  4 +--
 tensorflow/lite/g3doc/using_select_tf_ops.md  |  2 +-
 tensorflow/lite/java/demo/app/build.gradle    | 26 +++++++------------
 tensorflow/lite/java/demo/build.gradle        |  4 ++-
 .../gradle/wrapper/gradle-wrapper.properties  |  2 +-
 .../lite/java/ovic/demo/app/build.gradle      | 21 ++++++---------
 tensorflow/lite/java/ovic/demo/build.gradle   |  4 ++-
 .../gradle/wrapper/gradle-wrapper.properties  |  2 +-
 10 files changed, 33 insertions(+), 40 deletions(-)

diff --git a/tensorflow/lite/examples/android/app/build.gradle b/tensorflow/lite/examples/android/app/build.gradle
index e5f5c7efd1..b372afae19 100644
--- a/tensorflow/lite/examples/android/app/build.gradle
+++ b/tensorflow/lite/examples/android/app/build.gradle
@@ -2,7 +2,7 @@ apply plugin: 'com.android.application'
 
 android {
     compileSdkVersion 26
-    buildToolsVersion '26.0.2'
+    buildToolsVersion '27.0.3'
     defaultConfig {
         applicationId "org.tensorflow.lite.demo"
         minSdkVersion 15
@@ -45,6 +45,6 @@ project.ext.TMP_DIR   = project.buildDir.toString() + '/downloads'
 apply from: "download-models.gradle"
 
 dependencies {
-    compile fileTree(dir: 'libs', include: ['*.jar'])
-    compile 'org.tensorflow:tensorflow-lite:0.0.0-nightly'
+    implementation fileTree(dir: 'libs', include: ['*.jar'])
+    implementation 'org.tensorflow:tensorflow-lite:0.0.0-nightly'
 }
diff --git a/tensorflow/lite/examples/android/build.gradle b/tensorflow/lite/examples/android/build.gradle
index 7c79358e45..7c038ddd46 100644
--- a/tensorflow/lite/examples/android/build.gradle
+++ b/tensorflow/lite/examples/android/build.gradle
@@ -6,7 +6,7 @@ buildscript {
         jcenter()
     }
     dependencies {
-        classpath 'com.android.tools.build:gradle:3.0.1'
+        classpath 'com.android.tools.build:gradle:3.1.4'
 
         // NOTE: Do not place your application dependencies here; they belong
         // in the individual module build.gradle files
diff --git a/tensorflow/lite/g3doc/tfmobile/android_build.md b/tensorflow/lite/g3doc/tfmobile/android_build.md
index 2eb776d10c..f8c0243298 100644
--- a/tensorflow/lite/g3doc/tfmobile/android_build.md
+++ b/tensorflow/lite/g3doc/tfmobile/android_build.md
@@ -91,10 +91,10 @@ following lines to your Gradle build file:
         repositories {
             jcenter()
         }
-	}
+    }
 
     dependencies {
-        compile 'org.tensorflow:tensorflow-android:+'
+        implementation 'org.tensorflow:tensorflow-android:+'
     }
 
 This automatically downloads the latest stable version of TensorFlow as an AAR
diff --git a/tensorflow/lite/g3doc/using_select_tf_ops.md b/tensorflow/lite/g3doc/using_select_tf_ops.md
index aa51f58baa..269774a4b1 100644
--- a/tensorflow/lite/g3doc/using_select_tf_ops.md
+++ b/tensorflow/lite/g3doc/using_select_tf_ops.md
@@ -130,7 +130,7 @@ allprojects {
 }
 
 dependencies {
-    compile 'org.tensorflow:tensorflow-lite-with-select-tf-ops:0.1.100'
+    implementation 'org.tensorflow:tensorflow-lite-with-select-tf-ops:0.1.100'
 }
 ```
 
diff --git a/tensorflow/lite/java/demo/app/build.gradle b/tensorflow/lite/java/demo/app/build.gradle
index b8fc282cb1..8ea16a3417 100644
--- a/tensorflow/lite/java/demo/app/build.gradle
+++ b/tensorflow/lite/java/demo/app/build.gradle
@@ -2,7 +2,7 @@ apply plugin: 'com.android.application'
 
 android {
     compileSdkVersion 26
-    buildToolsVersion "26.0.1"
+    buildToolsVersion "27.0.3"
     defaultConfig {
         applicationId "android.example.com.tflitecamerademo"
         // Required by Camera2 API.
@@ -10,11 +10,6 @@ android {
         targetSdkVersion 26
         versionCode 1
         versionName "1.0"
-
-        // Remove this block.
-        jackOptions {
-            enabled true
-        }
     }
     lintOptions {
         abortOnError false
@@ -40,6 +35,7 @@ repositories {
         url 'https://google.bintray.com/tensorflow'
     }
 }
+
 allprojects {
     repositories {
         // Uncomment if you want to use a local repo.
@@ -48,20 +44,18 @@ allprojects {
     }
 }
 
-
-
 dependencies {
-    compile fileTree(dir: 'libs', include: ['*.jar'])
-    compile 'com.android.support:appcompat-v7:25.2.0'
-    compile 'com.android.support.constraint:constraint-layout:1.0.2'
-    compile 'com.android.support:design:25.2.0'
-    compile 'com.android.support:support-annotations:25.3.1'
-    compile 'com.android.support:support-v13:25.2.0'
+    implementation fileTree(dir: 'libs', include: ['*.jar'])
+    implementation 'com.android.support:appcompat-v7:25.2.0'
+    implementation 'com.android.support.constraint:constraint-layout:1.0.2'
+    implementation 'com.android.support:design:25.2.0'
+    implementation 'com.android.support:support-annotations:25.3.1'
+    implementation 'com.android.support:support-v13:25.2.0'
 
     // Build off of nightly TensorFlow Lite
-    compile 'org.tensorflow:tensorflow-lite:0.0.0-nightly'
+    implementation 'org.tensorflow:tensorflow-lite:0.0.0-nightly'
     // Use local TensorFlow library
-    // compile 'org.tensorflow:tensorflow-lite-local:0.0.0'
+    // implementation 'org.tensorflow:tensorflow-lite-local:0.0.0'
 }
 
 def targetFolder = "src/main/assets"
diff --git a/tensorflow/lite/java/demo/build.gradle b/tensorflow/lite/java/demo/build.gradle
index b78a0b86c9..a88b3fdc70 100644
--- a/tensorflow/lite/java/demo/build.gradle
+++ b/tensorflow/lite/java/demo/build.gradle
@@ -2,10 +2,11 @@
 
 buildscript {
     repositories {
+        google()
         jcenter()
     }
     dependencies {
-        classpath 'com.android.tools.build:gradle:2.3.1'
+        classpath 'com.android.tools.build:gradle:3.1.4'
 
         // NOTE: Do not place your application dependencies here; they belong
         // in the individual module build.gradle files
@@ -14,6 +15,7 @@ buildscript {
 
 allprojects {
     repositories {
+        google()
         jcenter()
     }
 }
diff --git a/tensorflow/lite/java/demo/gradle/wrapper/gradle-wrapper.properties b/tensorflow/lite/java/demo/gradle/wrapper/gradle-wrapper.properties
index fa7a38a0e4..9ff32fe2bb 100644
--- a/tensorflow/lite/java/demo/gradle/wrapper/gradle-wrapper.properties
+++ b/tensorflow/lite/java/demo/gradle/wrapper/gradle-wrapper.properties
@@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-3.3-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.4-all.zip
diff --git a/tensorflow/lite/java/ovic/demo/app/build.gradle b/tensorflow/lite/java/ovic/demo/app/build.gradle
index 4f3a6cdb2f..77f568448a 100644
--- a/tensorflow/lite/java/ovic/demo/app/build.gradle
+++ b/tensorflow/lite/java/ovic/demo/app/build.gradle
@@ -2,18 +2,13 @@ apply plugin: 'com.android.application'
 
 android {
     compileSdkVersion 26
-    buildToolsVersion "26.0.1"
+    buildToolsVersion "27.0.3"
     defaultConfig {
         applicationId "android.example.com.ovicbenchmarker"
         minSdkVersion 15
         targetSdkVersion 26
         versionCode 1
         versionName "1.0"
-
-        // Remove this block.
-        jackOptions {
-            enabled true
-        }
     }
     lintOptions {
         abortOnError false
@@ -41,12 +36,12 @@ repositories {
 }
 
 dependencies {
-    compile fileTree(dir: 'libs', include: ['*.jar'])
-    compile 'com.android.support:appcompat-v7:25.2.0'
-    compile 'com.android.support.constraint:constraint-layout:1.0.2'
-    compile 'com.android.support:design:25.2.0'
-    compile 'com.android.support:support-annotations:25.3.1'
-    compile 'com.android.support:support-v13:25.2.0'
+    implementation fileTree(dir: 'libs', include: ['*.jar'])
+    implementation 'com.android.support:appcompat-v7:25.2.0'
+    implementation 'com.android.support.constraint:constraint-layout:1.0.2'
+    implementation 'com.android.support:design:25.2.0'
+    implementation 'com.android.support:support-annotations:25.3.1'
+    implementation 'com.android.support:support-v13:25.2.0'
 
-    compile 'org.tensorflow:tensorflow-lite:+'
+    implementation 'org.tensorflow:tensorflow-lite:+'
 }
diff --git a/tensorflow/lite/java/ovic/demo/build.gradle b/tensorflow/lite/java/ovic/demo/build.gradle
index b78a0b86c9..a88b3fdc70 100644
--- a/tensorflow/lite/java/ovic/demo/build.gradle
+++ b/tensorflow/lite/java/ovic/demo/build.gradle
@@ -2,10 +2,11 @@
 
 buildscript {
     repositories {
+        google()
         jcenter()
     }
     dependencies {
-        classpath 'com.android.tools.build:gradle:2.3.1'
+        classpath 'com.android.tools.build:gradle:3.1.4'
 
         // NOTE: Do not place your application dependencies here; they belong
         // in the individual module build.gradle files
@@ -14,6 +15,7 @@ buildscript {
 
 allprojects {
     repositories {
+        google()
         jcenter()
     }
 }
diff --git a/tensorflow/lite/java/ovic/demo/gradle/wrapper/gradle-wrapper.properties b/tensorflow/lite/java/ovic/demo/gradle/wrapper/gradle-wrapper.properties
index fa7a38a0e4..9ff32fe2bb 100644
--- a/tensorflow/lite/java/ovic/demo/gradle/wrapper/gradle-wrapper.properties
+++ b/tensorflow/lite/java/ovic/demo/gradle/wrapper/gradle-wrapper.properties
@@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-3.3-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.4-all.zip
-- 
GitLab


From 65cfc6a54bc76912dd16c25b9e13610c4359a6d9 Mon Sep 17 00:00:00 2001
From: Blake Hechtman <blakehechtman@google.com>
Date: Wed, 9 Jan 2019 11:13:04 -0800
Subject: [PATCH 0422/2345] [XLA] Fix Dot operation semantics.

PiperOrigin-RevId: 228551710
---
 tensorflow/compiler/xla/g3doc/operation_semantics.md | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/tensorflow/compiler/xla/g3doc/operation_semantics.md b/tensorflow/compiler/xla/g3doc/operation_semantics.md
index 7377ed729b..59ba9bb658 100644
--- a/tensorflow/compiler/xla/g3doc/operation_semantics.md
+++ b/tensorflow/compiler/xla/g3doc/operation_semantics.md
@@ -871,9 +871,7 @@ DotGeneral performs the sum of products over contracting dimensions specified
 in 'dimension_numbers'.
 
 Associated contracting dimension numbers from the 'lhs' and 'rhs' do not need
-to be the same, but must be listed in the same order in both
-'lhs/rhs_contracting_dimensions' arrays and have the same dimension sizes.
-There must be exactly one contracting dimension on both 'lhs' and 'rhs'.
+to be the same and but must have the same dimension sizes.
 
 Example with contracting dimension numbers:
 
@@ -892,10 +890,8 @@ DotGeneral(lhs, rhs, dnums) -> { {6.0, 12.0},
 {15.0, 30.0} }
 ```
 
-Associated batch dimension numbers from the 'lhs' and 'rhs' must have the same
-dimension number, must be listed in the same order in both arrays, must
-have the same dimension sizes, and must be ordered before contracting and
-non-contracting/non-batch dimension numbers.
+Associated batch dimension numbers from the 'lhs' and 'rhs' must
+have the same dimension sizes.
 
 Example with batch dimension numbers (batch size 2, 2x2 matrices):
 
-- 
GitLab


From d987224e93ba153201a7c3958594396af56a255d Mon Sep 17 00:00:00 2001
From: Andiry Xu <andiryxu@google.com>
Date: Wed, 9 Jan 2019 11:24:20 -0800
Subject: [PATCH 0423/2345] Typo fix

PiperOrigin-RevId: 228554042
---
 tensorflow/core/grappler/costs/graph_properties.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/grappler/costs/graph_properties.cc b/tensorflow/core/grappler/costs/graph_properties.cc
index cfbd340f08..1ed96ca12a 100644
--- a/tensorflow/core/grappler/costs/graph_properties.cc
+++ b/tensorflow/core/grappler/costs/graph_properties.cc
@@ -1463,9 +1463,9 @@ class SymbolicShapeRefiner {
       // Due to the cost of EvaluateNode(), we run it only for certain op types
       // (white listed) and small integer tensors.
 
-      const int max_elelment_size = 17;  // Max up to 4x4 matrix or similar.
+      const int max_element_size = 17;  // Max up to 4x4 matrix or similar.
       if (AllOutputValuesKnown(c) || !AllInputValuesKnown(c) ||
-          !ShouldUpdateOutputValues(c, max_elelment_size)) {
+          !ShouldUpdateOutputValues(c, max_element_size)) {
         return Status::OK();
       }
       UpdateOutputValues(node, c).IgnoreError();  // This is optional.
-- 
GitLab


From 9e4f493abffbcb1efc7ea2b1e9bce0afff8122ce Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Wed, 9 Jan 2019 11:31:52 -0800
Subject: [PATCH 0424/2345] Reduce max_workspace_size_bytes so the GPU memory
 allocation doesn't exceed the limit.

PiperOrigin-RevId: 228555498
---
 tensorflow/contrib/tensorrt/test/quantization_mnist_test.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py b/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
index e7d6ec4ad3..79dde2a6b5 100644
--- a/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
+++ b/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
@@ -144,7 +144,10 @@ class QuantizationAwareTrainingMNISTTest(test_util.TensorFlowTestCase):
           outputs=[OUTPUT_NODE_NAME],
           max_batch_size=max_batch_size,
           precision_mode='INT8',
-          max_workspace_size_bytes=4096 << 19,
+          # There is a 2GB GPU memory limit for each test, so we set
+          # max_workspace_size_bytes to 256MB to leave enough room for TF
+          # runtime to allocate GPU memory.
+          max_workspace_size_bytes=1 << 28,
           minimum_segment_size=2,
           use_calibration=False,
       )
-- 
GitLab


From 06974e29b1e9f4b4b5120acca2889bfa3bcf055d Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Wed, 9 Jan 2019 11:39:26 -0800
Subject: [PATCH 0425/2345] Internal Cleanup.

PiperOrigin-RevId: 228557086
---
 tensorflow/python/keras/optimizer_v2/BUILD    |   4 -
 tensorflow/python/keras/optimizer_v2/nadam.py |   3 +-
 .../keras/optimizer_v2/optimizer_v2_test.py   | 115 ++++++++++--------
 3 files changed, 62 insertions(+), 60 deletions(-)

diff --git a/tensorflow/python/keras/optimizer_v2/BUILD b/tensorflow/python/keras/optimizer_v2/BUILD
index 2b990c9334..da757edc74 100644
--- a/tensorflow/python/keras/optimizer_v2/BUILD
+++ b/tensorflow/python/keras/optimizer_v2/BUILD
@@ -177,11 +177,7 @@ py_test(
     srcs = ["optimizer_v2_test.py"],
     shard_count = 8,
     tags = [
-        "manual",
-        "no_gpu",
-        "no_oss",
         "no_windows",
-        "notap",
     ],
     deps = [
         ":optimizer_v2",
diff --git a/tensorflow/python/keras/optimizer_v2/nadam.py b/tensorflow/python/keras/optimizer_v2/nadam.py
index 627258f455..7141c6a079 100644
--- a/tensorflow/python/keras/optimizer_v2/nadam.py
+++ b/tensorflow/python/keras/optimizer_v2/nadam.py
@@ -84,8 +84,7 @@ class Nadam(optimizer_v2.OptimizerV2):
     """
 
     # Backwards compatiblity with keras NAdam optimizer.
-    if 'schedule_decay' in kwargs:
-      kwargs['decay'] = kwargs.pop('schedule_decay', 0.004)
+    kwargs['decay'] = kwargs.pop('schedule_decay', 0.004)
     super(Nadam, self).__init__(name, **kwargs)
     self._set_hyper('learning_rate', kwargs.get('lr', learning_rate))
     self._set_hyper('decay', self._initial_decay)
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
index b41e4aa064..93a6a7e85e 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
@@ -18,10 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import os
-import tempfile
-
-from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.python import keras
@@ -33,22 +29,25 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.keras import backend
 from tensorflow.python.keras import callbacks
+from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import optimizers
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.keras.engine import input_layer
-from tensorflow.python.keras.engine import saving
 from tensorflow.python.keras.engine import sequential
 from tensorflow.python.keras.engine import training
 from tensorflow.python.keras.layers import core
+from tensorflow.python.keras.optimizer_v2 import adadelta
+from tensorflow.python.keras.optimizer_v2 import adagrad
 from tensorflow.python.keras.optimizer_v2 import adam
+from tensorflow.python.keras.optimizer_v2 import adamax
 from tensorflow.python.keras.optimizer_v2 import gradient_descent
-from tensorflow.python.keras.optimizer_v2 import optimizer_v2
+from tensorflow.python.keras.optimizer_v2 import nadam
+from tensorflow.python.keras.optimizer_v2 import rmsprop
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import clip_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variables
-from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
 from tensorflow.python.training import momentum
 from tensorflow.python.training import training_util
@@ -492,23 +491,20 @@ class OptimizerTest(test.TestCase):
     opt.iterations = global_step
     var = resource_variable_ops.ResourceVariable([1.0, 2.0],
                                                  dtype=dtypes.float32)
+    self.evaluate(variables.global_variables_initializer())
+    init_step_value = self.evaluate(global_step)
     loss = lambda: 3 * var
     opt_op = opt.minimize(loss, [var])
     self.evaluate(variables.global_variables_initializer())
-    init_step_value = self.evaluate(global_step)
     self.evaluate(opt_op)
     new_step_value = self.evaluate(global_step)
     self.assertEqual(new_step_value, init_step_value + 1)
 
 
-class OptimizersCompatibilityTest(test.TestCase, parameterized.TestCase):
+@keras_parameterized.run_with_all_model_types
+class OptimizersCompatibilityTest(keras_parameterized.TestCase):
 
-  @parameterized.named_parameters(
-      ('adadelta', 'adadelta', True), ('adagrad', 'adagrad', True),
-      ('adam', 'adam', True), ('adamax', 'adamax', True),
-      ('nadam', 'nadam', True), ('momentum', 'momentum', True),
-      ('rmsprop', 'rmsprop', True), ('sgd', 'sgd', False))
-  def testOptimizersCompatibility(self, opt_str, test_weights):
+  def _testOptimizersCompatibility(self, opt_v1, opt_v2, test_weights=True):
     np.random.seed(1331)
     with self.cached_session():
       train_samples = 20
@@ -522,43 +518,63 @@ class OptimizersCompatibilityTest(test.TestCase, parameterized.TestCase):
       y = keras.utils.to_categorical(y)
 
       num_hidden = 5
-      model = testing_utils.get_small_sequential_mlp(
+      model_v1 = testing_utils.get_small_sequential_mlp(
           num_hidden=num_hidden, num_classes=num_classes, input_dim=input_dim)
+      model_v1.compile(opt_v1, loss='categorical_crossentropy', metrics=[])
+      model_v1.fit(x, y, batch_size=5, epochs=1)
 
-      old_mode = os.environ.get('TF2_BEHAVIOR', None)
-      # Disable tf2 to create V1 optimizer.
-      disable_tf2()
-      if opt_str == 'momentum':
-        opt_v1 = optimizers.SGD(momentum=0.9)
-      else:
-        opt_v1 = optimizers.get(opt_str)
-
-      # Test compile and fit with v1 optimizer.
-      model.compile(opt_v1, loss='categorical_crossentropy', metrics=[])
-      model.fit(x, y, batch_size=5, epochs=1)
-      model_dir = tempfile.mkdtemp()
-      gfile.MakeDirs(model_dir)
-      file_name = os.path.join(model_dir, 'model.h5')
-      model.save(file_name)
-
-      enable_tf2()
-      # Test load and fit with v2 optimizer.
-      model_2 = saving.load_model(file_name)
-      opt_v2 = model_2.optimizer
-      self.assertIsInstance(opt_v2, optimizer_v2.OptimizerV2)
-      # set_weights is called inside load_model but exception is swallowed,
-      # this call checks the weights can be set correctly.
+      model_v2 = testing_utils.get_small_sequential_mlp(
+          num_hidden=num_hidden, num_classes=num_classes, input_dim=input_dim)
+      model_v2.set_weights(model_v1.get_weights())
+      model_v2.compile(opt_v2, loss='categorical_crossentropy', metrics=[])
+      model_v2._make_train_function()
       if test_weights:
         opt_v2.set_weights(opt_v1.get_weights())
 
-      hist_1 = model.fit(x, y, batch_size=5, epochs=1, shuffle=False)
-      hist_2 = model_2.fit(x, y, batch_size=5, epochs=1, shuffle=False)
-      self.assertAllClose(model.get_weights(), model_2.get_weights())
-      self.assertAllClose(model.get_weights(), model_2.get_weights())
+      hist_1 = model_v1.fit(x, y, batch_size=5, epochs=1, shuffle=False)
+      hist_2 = model_v2.fit(x, y, batch_size=5, epochs=1, shuffle=False)
+      self.assertAllClose(model_v1.get_weights(), model_v2.get_weights())
       self.assertAllClose(hist_1.history['loss'], hist_2.history['loss'])
 
-      if old_mode is not None:
-        os.environ['TF2_BEHAVIOR'] = old_mode
+  def testAdadeltaCompatibility(self):
+    opt_v1 = optimizers.Adadelta(lr=0.01)
+    opt_v2 = adadelta.Adadelta(learning_rate=0.01)
+    self._testOptimizersCompatibility(opt_v1, opt_v2)
+
+  def testAdagradCompatibility(self):
+    opt_v1 = optimizers.Adagrad(lr=0.01)
+    opt_v2 = adagrad.Adagrad(learning_rate=0.01)
+    self._testOptimizersCompatibility(opt_v1, opt_v2)
+
+  def testAdamCompatibility(self):
+    opt_v1 = optimizers.Adam()
+    opt_v2 = adam.Adam()
+    self._testOptimizersCompatibility(opt_v1, opt_v2)
+
+  def testAdamaxCompatibility(self):
+    opt_v1 = optimizers.Adamax(lr=0.01)
+    opt_v2 = adamax.Adamax(learning_rate=0.01)
+    self._testOptimizersCompatibility(opt_v1, opt_v2)
+
+  def testNadamCompatibility(self):
+    opt_v1 = optimizers.Nadam(lr=0.001)
+    opt_v2 = nadam.Nadam(learning_rate=0.001)
+    self._testOptimizersCompatibility(opt_v1, opt_v2)
+
+  def testMomentumCompatibility(self):
+    opt_v1 = optimizers.SGD(lr=0.01, momentum=0.9)
+    opt_v2 = gradient_descent.SGD(learning_rate=0.01, momentum=0.9)
+    self._testOptimizersCompatibility(opt_v1, opt_v2)
+
+  def testRMSpropCompatibility(self):
+    opt_v1 = optimizers.RMSprop()
+    opt_v2 = rmsprop.RMSprop()
+    self._testOptimizersCompatibility(opt_v1, opt_v2)
+
+  def testSGDCompatibility(self):
+    opt_v1 = optimizers.SGD(lr=0.01)
+    opt_v2 = gradient_descent.SGD(learning_rate=0.01)
+    self._testOptimizersCompatibility(opt_v1, opt_v2, False)
 
   def testNumericEquivalenceForNesterovMomentum(self):
     np.random.seed(1331)
@@ -636,15 +652,6 @@ class OptimizersCompatibilityTest(test.TestCase, parameterized.TestCase):
       self.assertAllClose(hist_k_v1.history['loss'], hist_k_v2.history['loss'])
 
 
-def disable_tf2():
-  if 'TF2_BEHAVIOR' in os.environ:
-    del os.environ['TF2_BEHAVIOR']
-
-
-def enable_tf2():
-  os.environ['TF2_BEHAVIOR'] = 'enabled'
-
-
 # Note: These tests are kept in a separate class to avoid bugs in some
 # distributions of Python that break AutoGraph which is used by tf.function.
 class OptimizerWithFunctionTest(test.TestCase):
-- 
GitLab


From f5f079ca12c376eb2b86389973661f27054136af Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Wed, 9 Jan 2019 11:44:32 -0800
Subject: [PATCH 0426/2345] Add missing include needed for windows build. Fixes
 #24521

PiperOrigin-RevId: 228558062
---
 tensorflow/core/util/stats_calculator.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/core/util/stats_calculator.h b/tensorflow/core/util/stats_calculator.h
index e191737bb2..5005ee08a4 100644
--- a/tensorflow/core/util/stats_calculator.h
+++ b/tensorflow/core/util/stats_calculator.h
@@ -18,6 +18,7 @@ limitations under the License.
 
 #include <stdlib.h>
 
+#include <algorithm>
 #include <cmath>
 #include <limits>
 #include <map>
-- 
GitLab


From 245925078f574983593e42154091e07c311e30bd Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Wed, 9 Jan 2019 11:45:24 -0800
Subject: [PATCH 0427/2345] Remove the implementation of TPU Strategy
 initialize and finalize

PiperOrigin-RevId: 228558184
---
 .../contrib/distribute/python/tpu_strategy.py | 25 -------------------
 1 file changed, 25 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index f498f58b25..00360bf996 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -51,9 +51,6 @@ from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import nest
 
 
-_TPU_INITIALIZE_SYSTEM_COLLECTION = "TPU_STRATEGY_INITIALIZE"
-
-
 def initialize_tpu_system(cluster_resolver=None):
   """Initialize the TPU devices in a separate session and graph.
 
@@ -463,28 +460,6 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
     """
     initialize_tpu_system(self._tpu_cluster_resolver)
 
-  def _initialize(self):
-    if context.executing_eagerly():
-      # TODO(priyag): Add appopriate call here when eager is supported for TPUs.
-      raise NotImplementedError("Eager mode not supported in TPUStrategy.")
-    else:
-      # TODO(jhseu): We need this hack because DistributionStrategies must be
-      # pickleable for copy.deepcopy(). Remove when initialize_system goes away.
-      graph = ops.get_default_graph()
-      tpu_init = graph.get_collection(_TPU_INITIALIZE_SYSTEM_COLLECTION)
-      if tpu_init:
-        return tpu_init
-      graph.add_to_collection(_TPU_INITIALIZE_SYSTEM_COLLECTION,
-                              tpu.initialize_system())
-      return graph.get_collection(_TPU_INITIALIZE_SYSTEM_COLLECTION)
-
-  def _finalize(self):
-    if context.executing_eagerly():
-      # TODO(priyag): Add appopriate call here when eager is supported for TPUs.
-      raise NotImplementedError("Eager mode not supported in TPUStrategy.")
-    else:
-      return []
-
   def _create_variable(self, next_creator, *args, **kwargs):
     """Create a TPUMirroredVariable. See `DistributionStrategy.scope`."""
     colocate_with = kwargs.pop("colocate_with", None)
-- 
GitLab


From bebf6b3d5d11e0ae09edab0a3700e0e4000ac842 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 11:57:48 -0800
Subject: [PATCH 0428/2345] Scope GBDT resource variables.

PiperOrigin-RevId: 228560356
---
 .../contrib/boosted_trees/python/ops/model_ops.py      | 10 +++++-----
 .../python/training/functions/gbdt_batch.py            |  5 +++--
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/python/ops/model_ops.py b/tensorflow/contrib/boosted_trees/python/ops/model_ops.py
index fca22c71a8..c3685b54e2 100644
--- a/tensorflow/contrib/boosted_trees/python/ops/model_ops.py
+++ b/tensorflow/contrib/boosted_trees/python/ops/model_ops.py
@@ -62,8 +62,8 @@ class TreeEnsembleVariableSavable(saver.BaseSaverBuilder.SaveableObject):
         saver.BaseSaverBuilder.SaveSpec(ensemble_config, slice_spec,
                                         name + "_config"),
     ]
-    super(TreeEnsembleVariableSavable,
-          self).__init__(tree_ensemble_handle, specs, name)
+    super(TreeEnsembleVariableSavable, self).__init__(tree_ensemble_handle,
+                                                      specs, name)
     self._tree_ensemble_handle = tree_ensemble_handle
     self._create_op = create_op
 
@@ -115,7 +115,7 @@ class TreeEnsembleVariable(tracking.TrackableResource):
 
   def _gather_saveables_for_checkpoint(self):
     return {
-        "tree_ensemble_variable":
+        self.resource_handle.op.name + "/tree_ensemble_variable":
             functools.partial(
                 TreeEnsembleVariableSavable,
                 tree_ensemble_handle=self.resource_handle,
@@ -131,8 +131,8 @@ def tree_ensemble_variable(stamp_token,
 
   Args:
     stamp_token: The initial stamp token value for the ensemble resource.
-    tree_ensemble_config: A `Tensor` of type `string`.
-      Serialized proto of the tree ensemble.
+    tree_ensemble_config: A `Tensor` of type `string`. Serialized proto of the
+      tree ensemble.
     name: A name for the ensemble variable.
     container: An optional `string`. Defaults to `""`.
 
diff --git a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py
index a5951fb737..e78ec476ab 100644
--- a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py
+++ b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py
@@ -566,9 +566,10 @@ class GradientBoostedDecisionTreeModel(object):
     # Determine if ensemble is colocated with the inputs.
     if self._ensemble_handle.device != input_deps[0].device:
       # Create a local ensemble and get its local stamp.
-      with ops.name_scope("local_ensemble", "TreeEnsembleVariable") as name:
+      with ops.name_scope("local_ensemble", "TreeEnsembleVariable"):
         local_ensemble_handle = (
-            gen_model_ops.decision_tree_ensemble_resource_handle_op(name=name))
+            gen_model_ops.decision_tree_ensemble_resource_handle_op(
+                self._ensemble_handle.op.name + "/local_ensemble"))
         create_op = gen_model_ops.create_tree_ensemble_variable(
             local_ensemble_handle, stamp_token=-1, tree_ensemble_config="")
         with ops.control_dependencies([create_op]):
-- 
GitLab


From 8d9872cebb57219acbfaad57e5203b3a407ac28e Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Wed, 9 Jan 2019 11:57:49 -0800
Subject: [PATCH 0429/2345] Fix typos in comments.

PiperOrigin-RevId: 228560360
---
 .../compiler/xla/service/gpu/cudnn_conv_algorithm_picker.cc     | 2 +-
 tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.h         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.cc b/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.cc
index 057d0d2364..309b0aca64 100644
--- a/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.cc
+++ b/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.cc
@@ -250,7 +250,7 @@ CudnnConvAlgorithmPicker::PickBestAlgorithm(
     VLOG(3) << "Trying algorithm " << AlgorithmToString(alg) << " for "
             << instr->ToString();
 
-    // Use assignment insetad of brace-list to make GCC 4.9 happy.
+    // Use assignment instead of brace-list to make GCC 4.9 happy.
     RunConvOptions options;
     options.profile_result = &profile_result;
     options.algo_override = alg;
diff --git a/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.h b/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.h
index 7cc325bce3..25b2461ca6 100644
--- a/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.h
+++ b/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.h
@@ -32,7 +32,7 @@ struct RunConvOptions {
   // Nullable output-parameter pointer for profiling results.
   se::dnn::ProfileResult* profile_result = nullptr;
 
-  // Use this algorithm, instead the one from the instrcution.
+  // Use this algorithm, instead of the one from the instruction.
   absl::optional<se::dnn::AlgorithmDesc> algo_override;
 };
 
-- 
GitLab


From a14adaa2329fb46cb472b949ee52546c2516a21e Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne <reedwm@google.com>
Date: Wed, 9 Jan 2019 12:00:29 -0800
Subject: [PATCH 0430/2345] Automated rollback of changelist 205730535

PiperOrigin-RevId: 228560984
---
 tensorflow/core/BUILD                         | 11 ----
 .../core/common_runtime/gpu/gpu_device.cc     | 51 +------------------
 .../core/common_runtime/gpu/gpu_device.h      | 11 ----
 .../gpu/gpu_device_kernel_check.cu.cc         | 37 --------------
 .../gpu/gpu_device_kernel_check.h             | 32 ------------
 5 files changed, 1 insertion(+), 141 deletions(-)
 delete mode 100644 tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.cu.cc
 delete mode 100644 tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 6a842a432f..e6af9211b5 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -87,7 +87,6 @@ load(
     "tf_gen_op_libs",
     "tf_generate_proto_text_sources",
     "tf_genrule_cmd_append_to_srcs",
-    "tf_gpu_kernel_library",
     "tf_opts_nortti_if_android",
     "transitive_hdrs",
 )
@@ -3163,15 +3162,6 @@ cc_library(
     ],
 )
 
-tf_gpu_kernel_library(
-    name = "gpu_device_kernel_check",
-    srcs = ["common_runtime/gpu/gpu_device_kernel_check.cu.cc"],
-    hdrs = ["common_runtime/gpu/gpu_device_kernel_check.h"],
-    deps = [
-        "//tensorflow/core:stream_executor",
-    ],
-)
-
 GPU_RUNTIME_HEADERS = [
     "common_runtime/gpu/cuda_host_allocator.h",
     "common_runtime/gpu/gpu_bfc_allocator.h",
@@ -3210,7 +3200,6 @@ tf_cuda_library(
         ":core_cpu_lib",
         ":framework",
         ":framework_internal",
-        ":gpu_device_kernel_check",
         ":gpu_id_impl",
         ":gpu_init_impl",
         ":gpu_lib",
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.cc b/tensorflow/core/common_runtime/gpu/gpu_device.cc
index 891b6e0e2a..010fdff4e9 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.cc
@@ -31,7 +31,6 @@ limitations under the License.
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/common_runtime/device_factory.h"
-#include "tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_event_mgr.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_id.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_id_manager.h"
@@ -389,7 +388,7 @@ Status BaseGPUDevice::Init(const SessionOptions& options) {
     }
   }
 
-  return CheckGPU();
+  return Status::OK();
 }
 
 bool BaseGPUDevice::RequiresRecordingAccessedTensors() const {
@@ -908,54 +907,6 @@ Allocator* BaseGPUDevice::GetScopedAllocator(AllocatorAttributes attr,
   return gpu_allocator_;
 }
 
-Status BaseGPUDevice::CheckGPU() {
-  se::Stream* stream = tensorflow_gpu_device_info()->stream;
-  TF_RETURN_IF_ERROR(stream->BlockHostUntilDone());
-  Tensor device_tensor(gpu_allocator_, DT_FLOAT, {});
-  if (!device_tensor.IsInitialized()) {
-    return errors::ResourceExhausted("Failed to allocate ", sizeof(float),
-                                     " bytes on the GPU for initialization "
-                                     "checks");
-  }
-  float* val_dev = device_tensor.scalar<float>().data();
-  const cudaStream_t cu_stream = *reinterpret_cast<const cudaStream_t*>(
-      stream->implementation()->GpuStreamMemberHack());
-  {
-    se::cuda::ScopedActivateExecutorContext scoped_activation{stream->parent()};
-    run_test_kernel(val_dev, cu_stream);
-    // We have to use the CUDA runtime function cudaPeekAtLastError here,
-    // because 'stream' does not provide a way to check if a kernel launch
-    // succeeds. Calling 'stream->BlockHostUntilDone()', which internally calls
-    // 'cuCtxSynchronize()', does not catch all kernel launch errors.
-    cudaError_t cuda_error = cudaPeekAtLastError();
-    if (cuda_error == cudaSuccess) {
-      cuda_error = cudaDeviceSynchronize();
-    }
-    TF_RETURN_IF_ERROR(CudaErrorToStatus(cuda_error, *stream));
-  }
-
-  float val_host = 0.;
-  stream->ThenMemcpy(&val_host, se::DeviceMemoryBase(val_dev, sizeof(float)),
-                     sizeof(float));
-  TF_RETURN_IF_ERROR(stream->BlockHostUntilDone());
-  if (val_host != 12345.) {
-    return errors::Internal(
-        "GPU kernel for initialization returned wrong value: ", val_host);
-  }
-  return Status::OK();
-}
-
-Status BaseGPUDevice::CudaErrorToStatus(cudaError_t cuda_error,
-                                        const se::Stream& stream) {
-  if (cuda_error != cudaSuccess) {
-    return errors::Internal(
-        "Failed to run GPU kernel for the initialization check. Received "
-        "error ",
-        cudaGetErrorName(cuda_error), " after running GPU kernel.");
-  }
-  return Status::OK();
-}
-
 const int BaseGPUDeviceFactory::InterconnectMap::kSameDeviceStrength = 1000;
 const int BaseGPUDeviceFactory::InterconnectMap::kStreamExecutorStrength = 1;
 
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.h b/tensorflow/core/common_runtime/gpu/gpu_device.h
index 86622dad49..d002d02c51 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.h
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.h
@@ -26,7 +26,6 @@ limitations under the License.
 #include <vector>
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
-#include "cuda/include/cuda_runtime_api.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_event_mgr.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_id.h"
@@ -122,12 +121,6 @@ class BaseGPUDevice : public LocalDevice {
   se::StreamExecutor* executor_;  // not owned
   std::unique_ptr<ScopedAllocatorMgr> scoped_allocator_mgr_;
 
-  // Returns a Status corresponding to a cudaError_t. The CUDA error must have
-  // been obtained from a CUDA kernel launch used to check if the GPU is
-  // initialized properly.
-  virtual Status CudaErrorToStatus(cudaError_t cuda_error,
-                                   const se::Stream& stream);
-
  private:
   struct StreamGroup {
     se::Stream* compute = nullptr;
@@ -168,10 +161,6 @@ class BaseGPUDevice : public LocalDevice {
   Status MaybeCopyTensorToGPU(const AllocatorAttributes& alloc_attrs,
                               const Tensor& from, Tensor* to,
                               StatusCallback done);
-
-  // Checks that the GPU is capable of doing work, by running a test kernel on
-  // it.
-  Status CheckGPU();
 };
 
 class BaseGPUDeviceFactory : public DeviceFactory {
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.cu.cc b/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.cu.cc
deleted file mode 100644
index 017565195b..0000000000
--- a/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.cu.cc
+++ /dev/null
@@ -1,37 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#if GOOGLE_CUDA
-
-#include "tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h"
-#include "tensorflow/stream_executor/cuda/cuda_activation.h"
-
-namespace {
-__global__ void test_kernel(float* val) {
-  if (blockIdx.x == 0 && threadIdx.x == 0) {
-    (*val) = 12345.;
-  }
-}
-}  // namespace
-
-namespace tensorflow {
-
-void run_test_kernel(float* val, cudaStream_t cu_stream) {
-  test_kernel<<<1, 1, 0, cu_stream>>>(val);
-}
-
-}  // namespace tensorflow
-
-#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h b/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h
deleted file mode 100644
index 064fb7a49f..0000000000
--- a/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_DEVICE_KERNEL_CHECK_H_
-#define TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_DEVICE_KERNEL_CHECK_H_
-
-#if GOOGLE_CUDA
-
-#include "tensorflow/core/platform/stream_executor.h"
-
-namespace tensorflow {
-
-// Runs a GPU kernel to test that it functions correctly. Sets 'val' to 12345.
-void run_test_kernel(float* val, cudaStream_t cu_stream);
-
-}  // namespace tensorflow
-
-#endif  // GOOGLE_CUDA
-
-#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_DEVICE_KERNEL_CHECK_H_
-- 
GitLab


From 4fe7980e6b33d075a4a9fcfa40310af85f071cff Mon Sep 17 00:00:00 2001
From: Smit Hinsu <hinsu@google.com>
Date: Wed, 9 Jan 2019 12:09:12 -0800
Subject: [PATCH 0431/2345] Automated rollback of commit
 7009febfed6721cbf664b1ccfd44358392c3d5ca

PiperOrigin-RevId: 228562832
---
 tensorflow/contrib/tensorrt/BUILD | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index c17dea02f1..3d34b91a74 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -551,7 +551,6 @@ cuda_py_test(
     ],
     tags = [
         "no_cuda_on_cpu_tap",
-        "no_oss",  # TODO(b/121194394): re-enable in OSS after OOM is fixed.
         "no_pip",
         "no_tap",  # It is not able to download the mnist data.
         "no_windows",
-- 
GitLab


From 57d4745d24195699f93d1674171a03345269d2c1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 12:10:02 -0800
Subject: [PATCH 0432/2345] Compute reshape permutation statically.

Aside from presumably being more efficient generally, this enables using the routines in this file inside of XLA. (which currently throws an error
`Input 0 to InvertPermutation operator must be a compile-time constant.`)

PiperOrigin-RevId: 228563007
---
 tensorflow/python/ops/linalg/linear_operator_util.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/ops/linalg/linear_operator_util.py b/tensorflow/python/ops/linalg/linear_operator_util.py
index 54d04e4a70..6c18943dab 100644
--- a/tensorflow/python/ops/linalg/linear_operator_util.py
+++ b/tensorflow/python/ops/linalg/linear_operator_util.py
@@ -481,9 +481,9 @@ def _reshape_for_efficiency(a,
 
   # Permutation to put the extra dims at the end.
   perm = (
-      array_ops.concat(
-          (math_ops.range(b_extra_ndims, b.shape.ndims),
-           math_ops.range(0, b_extra_ndims)), 0))
+      np.concatenate(
+          (np.arange(b_extra_ndims, b.shape.ndims),
+           np.arange(0, b_extra_ndims)), 0))
   b_extra_on_end = array_ops.transpose(b, perm=perm)
 
   # Now squash this end into one long dim.
@@ -497,7 +497,7 @@ def _reshape_for_efficiency(a,
     y_extra_shape = array_ops.concat(
         (array_ops.shape(y)[:-1], [b_main_sh[-1]], b_extra_sh), 0)
     y_extra_on_end = array_ops.reshape(y, y_extra_shape)
-    return array_ops.transpose(
-        y_extra_on_end, perm=array_ops.invert_permutation(perm))
+    inverse_perm = np.argsort(perm)
+    return array_ops.transpose(y_extra_on_end, perm=inverse_perm)
 
   return a, b_squashed_end, reshape_inv, still_need_to_transpose
-- 
GitLab


From 01d9a34786ea0da087855f88ac32d0d7085c8266 Mon Sep 17 00:00:00 2001
From: Nick Felt <nickfelt@google.com>
Date: Wed, 9 Jan 2019 12:16:00 -0800
Subject: [PATCH 0433/2345] Mark tf.summary v1 API classes and
 tf.train.summary_iterator() as v1-only

The tf.summary.FileWriter and FileWriterCache classes are only for v1 summary writing; TF 2.0 provides the v2 create_file_writer() API instead.

The tf.summary python protos are being removed as part of the overall effort to remove protos from the TF 2.0 python API. The recourse for users will be accessing these bindings from within TensorBoard - the exact package path is still TBD, but when we have one we can add a migration hint.

As for tf.train.summary_iterator(), this can be replaced by just using tf.io.record_iterator() and parsing each record with Event.FromString() using the Event proto symbol, wherever in TensorBoard it ends up.

PiperOrigin-RevId: 228564320
---
 tensorflow/python/__init__.py                 |  10 +-
 tensorflow/python/summary/summary.py          |   5 +-
 tensorflow/python/summary/summary_iterator.py |   2 +-
 tensorflow/python/summary/writer/writer.py    |   2 +-
 .../python/summary/writer/writer_cache.py     |   2 +-
 .../python/tools/api/generator/doc_srcs.py    |   1 +
 .../api/golden/v2/tensorflow.-event.pbtxt     |  74 ---------
 ...rflow.-summary-metadata.-plugin-data.pbtxt |  18 ---
 .../v2/tensorflow.-summary-metadata.pbtxt     |  40 -----
 .../v2/tensorflow.-summary.-audio.pbtxt       |  36 -----
 .../v2/tensorflow.-summary.-image.pbtxt       |  30 ----
 .../v2/tensorflow.-summary.-value.pbtxt       |  74 ---------
 .../api/golden/v2/tensorflow.-summary.pbtxt   | 144 ------------------
 .../tools/api/golden/v2/tensorflow.pbtxt      |  12 --
 .../golden/v2/tensorflow.summary.-event.pbtxt |  74 ---------
 ...ensorflow.summary.-file-writer-cache.pbtxt |  16 --
 .../v2/tensorflow.summary.-file-writer.pbtxt  |  50 ------
 ...sorflow.summary.-summary-description.pbtxt |  12 --
 .../tensorflow.summary.-summary.-audio.pbtxt  |  36 -----
 .../tensorflow.summary.-summary.-image.pbtxt  |  30 ----
 .../tensorflow.summary.-summary.-value.pbtxt  |  74 ---------
 .../v2/tensorflow.summary.-summary.pbtxt      | 144 ------------------
 ...sorflow.summary.-tagged-run-metadata.pbtxt |  18 ---
 .../api/golden/v2/tensorflow.summary.pbtxt    |  24 ---
 .../api/golden/v2/tensorflow.train.pbtxt      |   4 -
 tensorflow/tools/compatibility/renames_v2.py  |  10 ++
 26 files changed, 22 insertions(+), 920 deletions(-)
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-event.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-summary-metadata.-plugin-data.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-summary-metadata.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-summary.-audio.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-summary.-image.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-summary.-value.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-summary.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.summary.-event.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.summary.-file-writer-cache.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.summary.-file-writer.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.summary.-summary-description.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.-audio.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.-image.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.-value.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.summary.-tagged-run-metadata.pbtxt

diff --git a/tensorflow/python/__init__.py b/tensorflow/python/__init__.py
index 207123cb14..31f9dce8e8 100644
--- a/tensorflow/python/__init__.py
+++ b/tensorflow/python/__init__.py
@@ -150,7 +150,7 @@ nn.rnn_cell = rnn_cell
 # pylint: disable=undefined-variable
 tf_export(v1=['AttrValue'])(AttrValue)
 tf_export(v1=['ConfigProto'])(ConfigProto)
-tf_export('Event', 'summary.Event')(Event)
+tf_export(v1=['Event', 'summary.Event'])(Event)
 tf_export(v1=['GPUOptions'])(GPUOptions)
 tf_export(v1=['GraphDef'])(GraphDef)
 tf_export(v1=['GraphOptions'])(GraphOptions)
@@ -163,10 +163,10 @@ tf_export(v1=['OptimizerOptions'])(OptimizerOptions)
 tf_export(v1=['RunMetadata'])(RunMetadata)
 tf_export(v1=['RunOptions'])(RunOptions)
 tf_export(v1=['SessionLog', 'summary.SessionLog'])(SessionLog)
-tf_export('Summary', 'summary.Summary')(Summary)
-tf_export('summary.SummaryDescription')(SummaryDescription)
-tf_export('SummaryMetadata')(SummaryMetadata)
-tf_export('summary.TaggedRunMetadata')(TaggedRunMetadata)
+tf_export(v1=['Summary', 'summary.Summary'])(Summary)
+tf_export(v1=['summary.SummaryDescription'])(SummaryDescription)
+tf_export(v1=['SummaryMetadata'])(SummaryMetadata)
+tf_export(v1=['summary.TaggedRunMetadata'])(TaggedRunMetadata)
 tf_export(v1=['TensorInfo'])(TensorInfo)
 # pylint: enable=undefined-variable
 
diff --git a/tensorflow/python/summary/summary.py b/tensorflow/python/summary/summary.py
index 0c13016712..a01feb3dde 100644
--- a/tensorflow/python/summary/summary.py
+++ b/tensorflow/python/summary/summary.py
@@ -13,9 +13,10 @@
 # limitations under the License.
 # ==============================================================================
 
-"""Tensor summaries for exporting information about a model.
+"""Operations for writing summary data, for use in analysis and visualization.
 
-See the [Summary](https://tensorflow.org/api_guides/python/summary) guide.
+See the [Summaries and
+TensorBoard](https://www.tensorflow.org/guide/summaries_and_tensorboard) guide.
 """
 
 from __future__ import absolute_import
diff --git a/tensorflow/python/summary/summary_iterator.py b/tensorflow/python/summary/summary_iterator.py
index 321b11ffb7..3675c235cf 100644
--- a/tensorflow/python/summary/summary_iterator.py
+++ b/tensorflow/python/summary/summary_iterator.py
@@ -24,7 +24,7 @@ from tensorflow.python.lib.io import tf_record
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export('train.summary_iterator')
+@tf_export(v1=['train.summary_iterator'])
 def summary_iterator(path):
   # pylint: disable=line-too-long
   """An iterator for reading `Event` protocol buffers from an event file.
diff --git a/tensorflow/python/summary/writer/writer.py b/tensorflow/python/summary/writer/writer.py
index 78217b503f..a66be4f833 100644
--- a/tensorflow/python/summary/writer/writer.py
+++ b/tensorflow/python/summary/writer/writer.py
@@ -279,7 +279,7 @@ class SummaryToEventTransformer(object):
     self.event_writer.add_event(event)
 
 
-@tf_export("summary.FileWriter")
+@tf_export(v1=["summary.FileWriter"])
 class FileWriter(SummaryToEventTransformer):
   """Writes `Summary` protocol buffers to event files.
 
diff --git a/tensorflow/python/summary/writer/writer_cache.py b/tensorflow/python/summary/writer/writer_cache.py
index 645fa28a37..c62a7ce1a3 100644
--- a/tensorflow/python/summary/writer/writer_cache.py
+++ b/tensorflow/python/summary/writer/writer_cache.py
@@ -25,7 +25,7 @@ from tensorflow.python.summary.writer.writer import FileWriter
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export('summary.FileWriterCache')
+@tf_export(v1=['summary.FileWriterCache'])
 class FileWriterCache(object):
   """Cache for file writers.
 
diff --git a/tensorflow/python/tools/api/generator/doc_srcs.py b/tensorflow/python/tools/api/generator/doc_srcs.py
index b567eead3d..28bf0e9d01 100644
--- a/tensorflow/python/tools/api/generator/doc_srcs.py
+++ b/tensorflow/python/tools/api/generator/doc_srcs.py
@@ -61,6 +61,7 @@ _TENSORFLOW_DOC_SOURCES = {
     'signal': DocSource(docstring_module_name='ops.signal.signal'),
     'sparse': DocSource(docstring_module_name='ops.sparse_ops'),
     'strings': DocSource(docstring_module_name='ops.string_ops'),
+    'summary': DocSource(docstring_module_name='summary.summary'),
     'sysconfig': DocSource(docstring_module_name='platform.sysconfig'),
     'test': DocSource(docstring_module_name='platform.test'),
     'train': DocSource(docstring_module_name='training.training'),
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-event.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-event.pbtxt
deleted file mode 100644
index 3b75a1735b..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-event.pbtxt
+++ /dev/null
@@ -1,74 +0,0 @@
-path: "tensorflow.Event"
-tf_proto {
-  descriptor {
-    name: "Event"
-    field {
-      name: "wall_time"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_DOUBLE
-    }
-    field {
-      name: "step"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "file_version"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-      oneof_index: 0
-    }
-    field {
-      name: "graph_def"
-      number: 4
-      label: LABEL_OPTIONAL
-      type: TYPE_BYTES
-      oneof_index: 0
-    }
-    field {
-      name: "summary"
-      number: 5
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.Summary"
-      oneof_index: 0
-    }
-    field {
-      name: "log_message"
-      number: 6
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.LogMessage"
-      oneof_index: 0
-    }
-    field {
-      name: "session_log"
-      number: 7
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.SessionLog"
-      oneof_index: 0
-    }
-    field {
-      name: "tagged_run_metadata"
-      number: 8
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.TaggedRunMetadata"
-      oneof_index: 0
-    }
-    field {
-      name: "meta_graph_def"
-      number: 9
-      label: LABEL_OPTIONAL
-      type: TYPE_BYTES
-      oneof_index: 0
-    }
-    oneof_decl {
-      name: "what"
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-summary-metadata.-plugin-data.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-summary-metadata.-plugin-data.pbtxt
deleted file mode 100644
index a66b74b315..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-summary-metadata.-plugin-data.pbtxt
+++ /dev/null
@@ -1,18 +0,0 @@
-path: "tensorflow.SummaryMetadata.PluginData"
-tf_proto {
-  descriptor {
-    name: "PluginData"
-    field {
-      name: "plugin_name"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "content"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_BYTES
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-summary-metadata.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-summary-metadata.pbtxt
deleted file mode 100644
index c02575b962..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-summary-metadata.pbtxt
+++ /dev/null
@@ -1,40 +0,0 @@
-path: "tensorflow.SummaryMetadata"
-tf_proto {
-  descriptor {
-    name: "SummaryMetadata"
-    field {
-      name: "plugin_data"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.SummaryMetadata.PluginData"
-    }
-    field {
-      name: "display_name"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "summary_description"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    nested_type {
-      name: "PluginData"
-      field {
-        name: "plugin_name"
-        number: 1
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-      field {
-        name: "content"
-        number: 2
-        label: LABEL_OPTIONAL
-        type: TYPE_BYTES
-      }
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-summary.-audio.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-summary.-audio.pbtxt
deleted file mode 100644
index 94f712073e..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-summary.-audio.pbtxt
+++ /dev/null
@@ -1,36 +0,0 @@
-path: "tensorflow.Summary.Audio"
-tf_proto {
-  descriptor {
-    name: "Audio"
-    field {
-      name: "sample_rate"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_FLOAT
-    }
-    field {
-      name: "num_channels"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "length_frames"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "encoded_audio_string"
-      number: 4
-      label: LABEL_OPTIONAL
-      type: TYPE_BYTES
-    }
-    field {
-      name: "content_type"
-      number: 5
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-summary.-image.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-summary.-image.pbtxt
deleted file mode 100644
index fc1acb483b..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-summary.-image.pbtxt
+++ /dev/null
@@ -1,30 +0,0 @@
-path: "tensorflow.Summary.Image"
-tf_proto {
-  descriptor {
-    name: "Image"
-    field {
-      name: "height"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_INT32
-    }
-    field {
-      name: "width"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_INT32
-    }
-    field {
-      name: "colorspace"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_INT32
-    }
-    field {
-      name: "encoded_image_string"
-      number: 4
-      label: LABEL_OPTIONAL
-      type: TYPE_BYTES
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-summary.-value.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-summary.-value.pbtxt
deleted file mode 100644
index feb84b6ee9..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-summary.-value.pbtxt
+++ /dev/null
@@ -1,74 +0,0 @@
-path: "tensorflow.Summary.Value"
-tf_proto {
-  descriptor {
-    name: "Value"
-    field {
-      name: "node_name"
-      number: 7
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "tag"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "metadata"
-      number: 9
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.SummaryMetadata"
-    }
-    field {
-      name: "simple_value"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_FLOAT
-      oneof_index: 0
-    }
-    field {
-      name: "obsolete_old_style_histogram"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_BYTES
-      oneof_index: 0
-    }
-    field {
-      name: "image"
-      number: 4
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.Summary.Image"
-      oneof_index: 0
-    }
-    field {
-      name: "histo"
-      number: 5
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.HistogramProto"
-      oneof_index: 0
-    }
-    field {
-      name: "audio"
-      number: 6
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.Summary.Audio"
-      oneof_index: 0
-    }
-    field {
-      name: "tensor"
-      number: 8
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.TensorProto"
-      oneof_index: 0
-    }
-    oneof_decl {
-      name: "value"
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-summary.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-summary.pbtxt
deleted file mode 100644
index b2bdff7171..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-summary.pbtxt
+++ /dev/null
@@ -1,144 +0,0 @@
-path: "tensorflow.Summary"
-tf_proto {
-  descriptor {
-    name: "Summary"
-    field {
-      name: "value"
-      number: 1
-      label: LABEL_REPEATED
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.Summary.Value"
-    }
-    nested_type {
-      name: "Image"
-      field {
-        name: "height"
-        number: 1
-        label: LABEL_OPTIONAL
-        type: TYPE_INT32
-      }
-      field {
-        name: "width"
-        number: 2
-        label: LABEL_OPTIONAL
-        type: TYPE_INT32
-      }
-      field {
-        name: "colorspace"
-        number: 3
-        label: LABEL_OPTIONAL
-        type: TYPE_INT32
-      }
-      field {
-        name: "encoded_image_string"
-        number: 4
-        label: LABEL_OPTIONAL
-        type: TYPE_BYTES
-      }
-    }
-    nested_type {
-      name: "Audio"
-      field {
-        name: "sample_rate"
-        number: 1
-        label: LABEL_OPTIONAL
-        type: TYPE_FLOAT
-      }
-      field {
-        name: "num_channels"
-        number: 2
-        label: LABEL_OPTIONAL
-        type: TYPE_INT64
-      }
-      field {
-        name: "length_frames"
-        number: 3
-        label: LABEL_OPTIONAL
-        type: TYPE_INT64
-      }
-      field {
-        name: "encoded_audio_string"
-        number: 4
-        label: LABEL_OPTIONAL
-        type: TYPE_BYTES
-      }
-      field {
-        name: "content_type"
-        number: 5
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-    }
-    nested_type {
-      name: "Value"
-      field {
-        name: "node_name"
-        number: 7
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-      field {
-        name: "tag"
-        number: 1
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-      field {
-        name: "metadata"
-        number: 9
-        label: LABEL_OPTIONAL
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.SummaryMetadata"
-      }
-      field {
-        name: "simple_value"
-        number: 2
-        label: LABEL_OPTIONAL
-        type: TYPE_FLOAT
-        oneof_index: 0
-      }
-      field {
-        name: "obsolete_old_style_histogram"
-        number: 3
-        label: LABEL_OPTIONAL
-        type: TYPE_BYTES
-        oneof_index: 0
-      }
-      field {
-        name: "image"
-        number: 4
-        label: LABEL_OPTIONAL
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.Summary.Image"
-        oneof_index: 0
-      }
-      field {
-        name: "histo"
-        number: 5
-        label: LABEL_OPTIONAL
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.HistogramProto"
-        oneof_index: 0
-      }
-      field {
-        name: "audio"
-        number: 6
-        label: LABEL_OPTIONAL
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.Summary.Audio"
-        oneof_index: 0
-      }
-      field {
-        name: "tensor"
-        number: 8
-        label: LABEL_OPTIONAL
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.TensorProto"
-        oneof_index: 0
-      }
-      oneof_decl {
-        name: "value"
-      }
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 0cf1fe0e3c..7e5f86d7e6 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -8,10 +8,6 @@ tf_module {
     name: "DType"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "Event"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
   member {
     name: "GradientTape"
     mtype: "<type \'type\'>"
@@ -40,14 +36,6 @@ tf_module {
     name: "SparseTensor"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "Summary"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
-  member {
-    name: "SummaryMetadata"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
   member {
     name: "Tensor"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.summary.-event.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.summary.-event.pbtxt
deleted file mode 100644
index eb99d0f533..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.summary.-event.pbtxt
+++ /dev/null
@@ -1,74 +0,0 @@
-path: "tensorflow.summary.Event"
-tf_proto {
-  descriptor {
-    name: "Event"
-    field {
-      name: "wall_time"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_DOUBLE
-    }
-    field {
-      name: "step"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "file_version"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-      oneof_index: 0
-    }
-    field {
-      name: "graph_def"
-      number: 4
-      label: LABEL_OPTIONAL
-      type: TYPE_BYTES
-      oneof_index: 0
-    }
-    field {
-      name: "summary"
-      number: 5
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.Summary"
-      oneof_index: 0
-    }
-    field {
-      name: "log_message"
-      number: 6
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.LogMessage"
-      oneof_index: 0
-    }
-    field {
-      name: "session_log"
-      number: 7
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.SessionLog"
-      oneof_index: 0
-    }
-    field {
-      name: "tagged_run_metadata"
-      number: 8
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.TaggedRunMetadata"
-      oneof_index: 0
-    }
-    field {
-      name: "meta_graph_def"
-      number: 9
-      label: LABEL_OPTIONAL
-      type: TYPE_BYTES
-      oneof_index: 0
-    }
-    oneof_decl {
-      name: "what"
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.summary.-file-writer-cache.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.summary.-file-writer-cache.pbtxt
deleted file mode 100644
index 2a5b63dcea..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.summary.-file-writer-cache.pbtxt
+++ /dev/null
@@ -1,16 +0,0 @@
-path: "tensorflow.summary.FileWriterCache"
-tf_class {
-  is_instance: "<class \'tensorflow.python.summary.writer.writer_cache.FileWriterCache\'>"
-  is_instance: "<type \'object\'>"
-  member_method {
-    name: "__init__"
-  }
-  member_method {
-    name: "clear"
-    argspec: "args=[], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get"
-    argspec: "args=[\'logdir\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.summary.-file-writer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.summary.-file-writer.pbtxt
deleted file mode 100644
index 6b65b0ace3..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.summary.-file-writer.pbtxt
+++ /dev/null
@@ -1,50 +0,0 @@
-path: "tensorflow.summary.FileWriter"
-tf_class {
-  is_instance: "<class \'tensorflow.python.summary.writer.writer.FileWriter\'>"
-  is_instance: "<class \'tensorflow.python.summary.writer.writer.SummaryToEventTransformer\'>"
-  is_instance: "<type \'object\'>"
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'logdir\', \'graph\', \'max_queue\', \'flush_secs\', \'graph_def\', \'filename_suffix\', \'session\'], varargs=None, keywords=None, defaults=[\'None\', \'10\', \'120\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_event"
-    argspec: "args=[\'self\', \'event\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "add_graph"
-    argspec: "args=[\'self\', \'graph\', \'global_step\', \'graph_def\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_meta_graph"
-    argspec: "args=[\'self\', \'meta_graph_def\', \'global_step\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_run_metadata"
-    argspec: "args=[\'self\', \'run_metadata\', \'tag\', \'global_step\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_session_log"
-    argspec: "args=[\'self\', \'session_log\', \'global_step\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_summary"
-    argspec: "args=[\'self\', \'summary\', \'global_step\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "close"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "flush"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_logdir"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "reopen"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary-description.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary-description.pbtxt
deleted file mode 100644
index 4a8b59cf02..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary-description.pbtxt
+++ /dev/null
@@ -1,12 +0,0 @@
-path: "tensorflow.summary.SummaryDescription"
-tf_proto {
-  descriptor {
-    name: "SummaryDescription"
-    field {
-      name: "type_hint"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.-audio.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.-audio.pbtxt
deleted file mode 100644
index 8b271cf58f..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.-audio.pbtxt
+++ /dev/null
@@ -1,36 +0,0 @@
-path: "tensorflow.summary.Summary.Audio"
-tf_proto {
-  descriptor {
-    name: "Audio"
-    field {
-      name: "sample_rate"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_FLOAT
-    }
-    field {
-      name: "num_channels"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "length_frames"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "encoded_audio_string"
-      number: 4
-      label: LABEL_OPTIONAL
-      type: TYPE_BYTES
-    }
-    field {
-      name: "content_type"
-      number: 5
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.-image.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.-image.pbtxt
deleted file mode 100644
index dbbc02dd05..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.-image.pbtxt
+++ /dev/null
@@ -1,30 +0,0 @@
-path: "tensorflow.summary.Summary.Image"
-tf_proto {
-  descriptor {
-    name: "Image"
-    field {
-      name: "height"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_INT32
-    }
-    field {
-      name: "width"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_INT32
-    }
-    field {
-      name: "colorspace"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_INT32
-    }
-    field {
-      name: "encoded_image_string"
-      number: 4
-      label: LABEL_OPTIONAL
-      type: TYPE_BYTES
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.-value.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.-value.pbtxt
deleted file mode 100644
index 4176171cd9..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.-value.pbtxt
+++ /dev/null
@@ -1,74 +0,0 @@
-path: "tensorflow.summary.Summary.Value"
-tf_proto {
-  descriptor {
-    name: "Value"
-    field {
-      name: "node_name"
-      number: 7
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "tag"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "metadata"
-      number: 9
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.SummaryMetadata"
-    }
-    field {
-      name: "simple_value"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_FLOAT
-      oneof_index: 0
-    }
-    field {
-      name: "obsolete_old_style_histogram"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_BYTES
-      oneof_index: 0
-    }
-    field {
-      name: "image"
-      number: 4
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.Summary.Image"
-      oneof_index: 0
-    }
-    field {
-      name: "histo"
-      number: 5
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.HistogramProto"
-      oneof_index: 0
-    }
-    field {
-      name: "audio"
-      number: 6
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.Summary.Audio"
-      oneof_index: 0
-    }
-    field {
-      name: "tensor"
-      number: 8
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.TensorProto"
-      oneof_index: 0
-    }
-    oneof_decl {
-      name: "value"
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.pbtxt
deleted file mode 100644
index d6c5e3a87a..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.pbtxt
+++ /dev/null
@@ -1,144 +0,0 @@
-path: "tensorflow.summary.Summary"
-tf_proto {
-  descriptor {
-    name: "Summary"
-    field {
-      name: "value"
-      number: 1
-      label: LABEL_REPEATED
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.Summary.Value"
-    }
-    nested_type {
-      name: "Image"
-      field {
-        name: "height"
-        number: 1
-        label: LABEL_OPTIONAL
-        type: TYPE_INT32
-      }
-      field {
-        name: "width"
-        number: 2
-        label: LABEL_OPTIONAL
-        type: TYPE_INT32
-      }
-      field {
-        name: "colorspace"
-        number: 3
-        label: LABEL_OPTIONAL
-        type: TYPE_INT32
-      }
-      field {
-        name: "encoded_image_string"
-        number: 4
-        label: LABEL_OPTIONAL
-        type: TYPE_BYTES
-      }
-    }
-    nested_type {
-      name: "Audio"
-      field {
-        name: "sample_rate"
-        number: 1
-        label: LABEL_OPTIONAL
-        type: TYPE_FLOAT
-      }
-      field {
-        name: "num_channels"
-        number: 2
-        label: LABEL_OPTIONAL
-        type: TYPE_INT64
-      }
-      field {
-        name: "length_frames"
-        number: 3
-        label: LABEL_OPTIONAL
-        type: TYPE_INT64
-      }
-      field {
-        name: "encoded_audio_string"
-        number: 4
-        label: LABEL_OPTIONAL
-        type: TYPE_BYTES
-      }
-      field {
-        name: "content_type"
-        number: 5
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-    }
-    nested_type {
-      name: "Value"
-      field {
-        name: "node_name"
-        number: 7
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-      field {
-        name: "tag"
-        number: 1
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-      field {
-        name: "metadata"
-        number: 9
-        label: LABEL_OPTIONAL
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.SummaryMetadata"
-      }
-      field {
-        name: "simple_value"
-        number: 2
-        label: LABEL_OPTIONAL
-        type: TYPE_FLOAT
-        oneof_index: 0
-      }
-      field {
-        name: "obsolete_old_style_histogram"
-        number: 3
-        label: LABEL_OPTIONAL
-        type: TYPE_BYTES
-        oneof_index: 0
-      }
-      field {
-        name: "image"
-        number: 4
-        label: LABEL_OPTIONAL
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.Summary.Image"
-        oneof_index: 0
-      }
-      field {
-        name: "histo"
-        number: 5
-        label: LABEL_OPTIONAL
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.HistogramProto"
-        oneof_index: 0
-      }
-      field {
-        name: "audio"
-        number: 6
-        label: LABEL_OPTIONAL
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.Summary.Audio"
-        oneof_index: 0
-      }
-      field {
-        name: "tensor"
-        number: 8
-        label: LABEL_OPTIONAL
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.TensorProto"
-        oneof_index: 0
-      }
-      oneof_decl {
-        name: "value"
-      }
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.summary.-tagged-run-metadata.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.summary.-tagged-run-metadata.pbtxt
deleted file mode 100644
index 27c8873320..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.summary.-tagged-run-metadata.pbtxt
+++ /dev/null
@@ -1,18 +0,0 @@
-path: "tensorflow.summary.TaggedRunMetadata"
-tf_proto {
-  descriptor {
-    name: "TaggedRunMetadata"
-    field {
-      name: "tag"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "run_metadata"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_BYTES
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.summary.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.summary.pbtxt
index 61670bd151..c59f1b8474 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.summary.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.summary.pbtxt
@@ -1,33 +1,9 @@
 path: "tensorflow.summary"
 tf_module {
-  member {
-    name: "Event"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
-  member {
-    name: "FileWriter"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "FileWriterCache"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "Summary"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
-  member {
-    name: "SummaryDescription"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
   member {
     name: "SummaryWriter"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "TaggedRunMetadata"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
   member_method {
     name: "create_file_writer"
     argspec: "args=[\'logdir\', \'max_queue\', \'flush_millis\', \'filename_suffix\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
index c72564e598..a3ace15ca2 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
@@ -140,8 +140,4 @@ tf_module {
     name: "sdca_shrink_l1"
     argspec: "args=[\'weights\', \'l1\', \'l2\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "summary_iterator"
-    argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None"
-  }
 }
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index e9085e94b3..b757699f63 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -34,6 +34,7 @@ renames = {
     'tf.ConfigProto': 'tf.compat.v1.ConfigProto',
     'tf.DeviceSpec': 'tf.compat.v1.DeviceSpec',
     'tf.Dimension': 'tf.compat.v1.Dimension',
+    'tf.Event': 'tf.compat.v1.Event',
     'tf.FIFOQueue': 'tf.queue.FIFOQueue',
     'tf.FixedLenFeature': 'tf.io.FixedLenFeature',
     'tf.FixedLenSequenceFeature': 'tf.io.FixedLenSequenceFeature',
@@ -73,6 +74,8 @@ renames = {
     'tf.SparseConditionalAccumulator': 'tf.sparse.SparseConditionalAccumulator',
     'tf.SparseFeature': 'tf.io.SparseFeature',
     'tf.SparseTensorValue': 'tf.compat.v1.SparseTensorValue',
+    'tf.Summary': 'tf.compat.v1.Summary',
+    'tf.SummaryMetadata': 'tf.compat.v1.SummaryMetadata',
     'tf.TFRecordReader': 'tf.compat.v1.TFRecordReader',
     'tf.TensorInfo': 'tf.compat.v1.TensorInfo',
     'tf.TextLineReader': 'tf.compat.v1.TextLineReader',
@@ -608,8 +611,14 @@ renames = {
     'tf.string_strip': 'tf.strings.strip',
     'tf.string_to_hash_bucket_fast': 'tf.strings.to_hash_bucket_fast',
     'tf.string_to_hash_bucket_strong': 'tf.strings.to_hash_bucket_strong',
+    'tf.summary.Event': 'tf.compat.v1.summary.Event',
+    'tf.summary.FileWriter': 'tf.compat.v1.summary.FileWriter',
+    'tf.summary.FileWriterCache': 'tf.compat.v1.summary.FileWriterCache',
     'tf.summary.SessionLog': 'tf.compat.v1.summary.SessionLog',
     'tf.summary.audio': 'tf.compat.v1.summary.audio',
+    'tf.summary.Summary': 'tf.compat.v1.summary.Summary',
+    'tf.summary.SummaryDescription': 'tf.compat.v1.summary.SummaryDescription',
+    'tf.summary.TaggedRunMetadata': 'tf.compat.v1.summary.TaggedRunMetadata',
     'tf.summary.get_summary_description': 'tf.compat.v1.summary.get_summary_description',
     'tf.summary.histogram': 'tf.compat.v1.summary.histogram',
     'tf.summary.image': 'tf.compat.v1.summary.image',
@@ -713,6 +722,7 @@ renames = {
     'tf.train.slice_input_producer': 'tf.compat.v1.train.slice_input_producer',
     'tf.train.start_queue_runners': 'tf.compat.v1.train.start_queue_runners',
     'tf.train.string_input_producer': 'tf.compat.v1.train.string_input_producer',
+    'tf.train.summary_iterator': 'tf.compat.v1.train.summary_iterator',
     'tf.train.update_checkpoint_state': 'tf.compat.v1.train.update_checkpoint_state',
     'tf.train.warm_start': 'tf.compat.v1.train.warm_start',
     'tf.train.write_graph': 'tf.io.write_graph',
-- 
GitLab


From 9a5dcb4c6b7b60557e996106402c46475307c4cd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 12:18:30 -0800
Subject: [PATCH 0434/2345] A new Eager C API: Profiler. The profiler starts
 profiling when it is created. It stops when it is destroyed. Its output
 format is tfprof proto.

PiperOrigin-RevId: 228564754
---
 tensorflow/c/eager/BUILD                      | 32 +++++++-
 tensorflow/c/eager/c_api.cc                   |  2 +-
 tensorflow/c/eager/c_api_experimental.cc      | 23 ++++++
 tensorflow/c/eager/c_api_experimental.h       | 18 +++++
 tensorflow/c/eager/c_api_experimental_test.cc | 79 +++++++++++++++++++
 tensorflow/c/eager/c_api_internal.h           |  8 ++
 tensorflow/core/common_runtime/eager/BUILD    | 28 +++++++
 .../core/common_runtime/eager/context.cc      | 32 +++++++-
 .../core/common_runtime/eager/context.h       | 17 +++-
 .../core/common_runtime/eager/execute.cc      |  2 +-
 .../core/common_runtime/eager/profiler.cc     | 77 ++++++++++++++++++
 .../core/common_runtime/eager/profiler.h      | 63 +++++++++++++++
 tensorflow/python/pywrap_tfe.i                |  3 +
 13 files changed, 375 insertions(+), 9 deletions(-)
 create mode 100644 tensorflow/c/eager/c_api_experimental_test.cc
 create mode 100644 tensorflow/core/common_runtime/eager/profiler.cc
 create mode 100644 tensorflow/core/common_runtime/eager/profiler.h

diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
index 9f09ad1fc3..3ea1724d1e 100644
--- a/tensorflow/c/eager/BUILD
+++ b/tensorflow/c/eager/BUILD
@@ -3,11 +3,10 @@ licenses(["notice"])  # Apache 2.0
 
 load(
     "//tensorflow:tensorflow.bzl",
-    "tf_cuda_cc_test",
-    "tf_cc_test",
     "tf_copts",
-    "tfe_xla_copts",
+    "tf_cuda_cc_test",
     "tf_cuda_library",
+    "tfe_xla_copts",
 )
 
 tf_cuda_library(
@@ -29,6 +28,7 @@ tf_cuda_library(
             "//tensorflow/c:c_api_internal",
             "//tensorflow/core:core_cpu",
             "//tensorflow/core/common_runtime/eager:attr_builder",
+            "//tensorflow/core/common_runtime/eager:profiler",
             "//tensorflow/core/common_runtime/eager:context",
             "//tensorflow/core/common_runtime/eager:eager_executor",
             "//tensorflow/core/common_runtime/eager:execute",
@@ -89,6 +89,7 @@ tf_cuda_library(
         "//tensorflow/core/common_runtime/eager:eager_executor",
         "//tensorflow/core/common_runtime/eager:eager_operation",
         "//tensorflow/core/common_runtime/eager:kernel_and_device",
+        "//tensorflow/core/common_runtime/eager:profiler",
         "//tensorflow/core/common_runtime/eager:tensor_handle",
         "//tensorflow/core/distributed_runtime:remote_device",
         "//tensorflow/core/distributed_runtime:server_lib",
@@ -204,6 +205,31 @@ tf_cuda_library(
     ],
 )
 
+tf_cuda_cc_test(
+    name = "c_api_experimental_test",
+    size = "small",
+    srcs = [
+        "c_api_experimental_test.cc",
+    ],
+    extra_copts = tfe_xla_copts(),
+    tags = [
+        "guitar",
+        "multi_gpu",
+    ],
+    deps = [
+        ":c_api_experimental",
+        ":c_api_test_util",
+        "//tensorflow/c:c_test_util",
+        "//tensorflow/cc/profiler",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core/profiler:protos_all_cc",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
 cc_library(
     name = "tape",
     hdrs = ["tape.h"],
diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index 027d752f42..d5a391a98d 100755
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -774,7 +774,7 @@ void TFE_ContextExportRunMetadata(TFE_Context* ctx, TF_Buffer* buf,
   if (!status->status.ok()) return;
   tensorflow::mutex_lock ml(*ctx->context.MetadataMu());
   status->status = MessageToBuffer(*ctx->context.RunMetadataProto(), buf);
-  ctx->context.RunMetadataProto()->Clear();
+  ctx->context.ClearRunMetadata();
 }
 
 namespace {
diff --git a/tensorflow/c/eager/c_api_experimental.cc b/tensorflow/c/eager/c_api_experimental.cc
index 3461d81b93..1ce03fb226 100644
--- a/tensorflow/c/eager/c_api_experimental.cc
+++ b/tensorflow/c/eager/c_api_experimental.cc
@@ -18,6 +18,29 @@ limitations under the License.
 #include "tensorflow/c/c_api.h"
 #include "tensorflow/c/eager/c_api_internal.h"
 
+using tensorflow::string;
+
 void TFE_OpConsumeInput(TFE_Op* op, TFE_TensorHandle* h, TF_Status* status) {
   op->operation.ConsumeInput(h->handle);
 }
+
+TFE_Profiler* TFE_NewProfiler(TFE_Context* ctx) {
+  return new TFE_Profiler(ctx);
+}
+
+void TFE_DeleteProfiler(TFE_Profiler* profiler) { delete profiler; }
+
+void TFE_ProfilerSerializeToString(TFE_Context* ctx, TFE_Profiler* profiler,
+                                   TF_Buffer* buf, TF_Status* status) {
+  TFE_ContextAsyncWait(ctx, status);
+  if (!status->status.ok()) return;
+  string content;
+  status->status = profiler->profiler->SerializeToString(&content);
+  void* data = tensorflow::port::Malloc(content.length());
+  content.copy(static_cast<char*>(data), content.length(), 0);
+  buf->data = data;
+  buf->length = content.length();
+  buf->data_deallocator = [](void* data, size_t length) {
+    tensorflow::port::Free(data);
+  };
+}
diff --git a/tensorflow/c/eager/c_api_experimental.h b/tensorflow/c/eager/c_api_experimental.h
index 4ee6c066ee..9eb80f5216 100644
--- a/tensorflow/c/eager/c_api_experimental.h
+++ b/tensorflow/c/eager/c_api_experimental.h
@@ -25,6 +25,24 @@ extern "C" {
 TF_CAPI_EXPORT extern void TFE_OpConsumeInput(TFE_Op* op, TFE_TensorHandle* h,
                                               TF_Status* status);
 
+// A profiler which will start profiling when creating the object and will stop
+// when the object is destroyed. It will profile all operations run under the
+// given TFE_Context. Multiple instance of it can be created, but at most one
+// of them will profile for each TFE_Context.
+// Thread-safety: TFE_Profiler is thread-safe.
+typedef struct TFE_Profiler TFE_Profiler;
+
+TF_CAPI_EXPORT extern TFE_Profiler* TFE_NewProfiler(TFE_Context* ctx);
+TF_CAPI_EXPORT extern void TFE_DeleteProfiler(TFE_Profiler* profiler);
+
+// The output string is a binary string of tensorflow.tfprof.ProfileProto.
+// User can write the string to file for offline analysis by tfprof command-line
+// tools or graphical user interface.
+TF_CAPI_EXPORT extern void TFE_ProfilerSerializeToString(TFE_Context* ctx,
+                                                         TFE_Profiler* profiler,
+                                                         TF_Buffer* buf,
+                                                         TF_Status* status);
+
 #ifdef __cplusplus
 } /* end extern "C" */
 #endif
diff --git a/tensorflow/c/eager/c_api_experimental_test.cc b/tensorflow/c/eager/c_api_experimental_test.cc
new file mode 100644
index 0000000000..9b4fca9d45
--- /dev/null
+++ b/tensorflow/c/eager/c_api_experimental_test.cc
@@ -0,0 +1,79 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/c/eager/c_api_experimental.h"
+
+#include <string.h>
+#include "absl/strings/match.h"
+#include "tensorflow/c/eager/c_api_test_util.h"
+#include "tensorflow/cc/profiler/profiler.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/protobuf.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/platform/test_benchmark.h"
+#include "tensorflow/core/profiler/tfprof_log.pb.h"
+
+using tensorflow::string;
+
+namespace {
+
+void ExecuteWithProfiling(bool async) {
+  TF_Status* status = TF_NewStatus();
+  TFE_ContextOptions* opts = TFE_NewContextOptions();
+  TFE_ContextOptionsSetAsync(opts, static_cast<unsigned char>(async));
+  TFE_Context* ctx = TFE_NewContext(opts, status);
+  TFE_Profiler* profiler = TFE_NewProfiler(ctx);
+  CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  TFE_DeleteContextOptions(opts);
+
+  TFE_TensorHandle* m = TestMatrixTensorHandle();
+  TFE_Op* matmul = MatMulOp(ctx, m, m);
+  TFE_TensorHandle* retvals[1] = {nullptr};
+  int num_retvals = 1;
+  TFE_Execute(matmul, &retvals[0], &num_retvals, status);
+  EXPECT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  TFE_DeleteOp(matmul);
+  TFE_DeleteTensorHandle(m);
+
+  ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  ASSERT_EQ(1, num_retvals);
+  TF_Buffer* profiler_result = TF_NewBuffer();
+  TFE_ProfilerSerializeToString(ctx, profiler, profiler_result, status);
+  TFE_DeleteProfiler(profiler);
+  ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  tensorflow::tfprof::ProfileProto profile_proto;
+  EXPECT_TRUE(profile_proto.ParseFromString(
+      {reinterpret_cast<const char*>(profiler_result->data),
+       profiler_result->length}));
+  TF_DeleteBuffer(profiler_result);
+
+  TF_Tensor* t = TFE_TensorHandleResolve(retvals[0], status);
+  TFE_DeleteTensorHandle(retvals[0]);
+  TFE_DeleteContext(ctx);
+  ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  float product[4] = {0};
+  EXPECT_EQ(sizeof(product), TF_TensorByteSize(t));
+  memcpy(&product[0], TF_TensorData(t), TF_TensorByteSize(t));
+  TF_DeleteTensor(t);
+  EXPECT_EQ(7, product[0]);
+  EXPECT_EQ(10, product[1]);
+  EXPECT_EQ(15, product[2]);
+  EXPECT_EQ(22, product[3]);
+  TF_DeleteStatus(status);
+}
+TEST(CAPI, ExecuteWithTracing) { ExecuteWithProfiling(false); }
+TEST(CAPI, ExecuteWithTracingAsync) { ExecuteWithProfiling(true); }
+
+}  // namespace
diff --git a/tensorflow/c/eager/c_api_internal.h b/tensorflow/c/eager/c_api_internal.h
index 67bc1bcd24..7b1035f631 100644
--- a/tensorflow/c/eager/c_api_internal.h
+++ b/tensorflow/c/eager/c_api_internal.h
@@ -34,6 +34,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/eager/eager_executor.h"
 #include "tensorflow/core/common_runtime/eager/eager_operation.h"
 #include "tensorflow/core/common_runtime/eager/kernel_and_device.h"
+#include "tensorflow/core/common_runtime/eager/profiler.h"
 #include "tensorflow/core/common_runtime/eager/tensor_handle.h"
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/common_runtime/rendezvous_mgr.h"
@@ -100,6 +101,13 @@ struct TFE_Op {
   tensorflow::EagerOperation operation;
 };
 
+struct TFE_Profiler {
+  TFE_Profiler(TFE_Context* ctx)
+      : profiler(tensorflow::EagerProfiler::Create(&ctx->context)) {}
+
+  std::unique_ptr<tensorflow::EagerProfiler> profiler;
+};
+
 namespace tensorflow {
 // Set an AttrValue on the op. Doesn't handle the list types.
 void SetOpAttrValueScalar(TFE_Context* ctx, TFE_Op* op,
diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD
index 77e3246df0..cabbddb77d 100644
--- a/tensorflow/core/common_runtime/eager/BUILD
+++ b/tensorflow/core/common_runtime/eager/BUILD
@@ -88,6 +88,34 @@ tf_cuda_library(
     ],
 )
 
+tf_cuda_library(
+    name = "profiler",
+    srcs = [
+        "profiler.cc",
+    ],
+    hdrs = [
+        "profiler.h",
+    ],
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        ":context",
+        "//tensorflow/cc/profiler",
+    ] + select({
+        "//tensorflow:android": [
+            "//tensorflow/core:android_tensorflow_lib_lite",
+        ],
+        "//conditions:default": [
+            "//tensorflow/core:core_cpu_lib",
+            "//tensorflow/core:framework",
+            "//tensorflow/core:framework_internal",
+            "//tensorflow/core:lib",
+            "//tensorflow/core:lib_internal",
+            "//tensorflow/core:protos_all_cc",
+            "//tensorflow/core:session_options",
+        ],
+    }),
+)
+
 tf_cuda_library(
     name = "tensor_handle",
     srcs = [
diff --git a/tensorflow/core/common_runtime/eager/context.cc b/tensorflow/core/common_runtime/eager/context.cc
index 2212bda534..1d93d2bbe6 100644
--- a/tensorflow/core/common_runtime/eager/context.cc
+++ b/tensorflow/core/common_runtime/eager/context.cc
@@ -234,6 +234,29 @@ Status EagerContext::FindDeviceByName(const string& name, Device** result) {
   return Status::OK();
 }
 
+void EagerContext::ClearRunMetadata() {
+  if (metadata_listener_ != nullptr) {
+    metadata_listener_->BeforeClearRunMetadata();
+  }
+  run_metadata_.Clear();
+}
+
+Status EagerContext::RegisterRunMetadataListener(
+    RunMetadataListener* listener) {
+  mutex_lock l(metadata_mu_);
+  if (metadata_listener_ != nullptr) {
+    return Status(error::Code::INVALID_ARGUMENT,
+                  "Cannot run two eager profiler at the same time");
+  }
+  metadata_listener_ = listener;
+  return Status::OK();
+}
+
+void EagerContext::ClearRunMetadataListener() {
+  mutex_lock l(metadata_mu_);
+  metadata_listener_ = nullptr;
+}
+
 void EagerContext::StartStep() {
   mutex_lock ml(metadata_mu_);
   num_active_steps_++;
@@ -317,10 +340,15 @@ void EagerContext::AddKernelToCache(Fprint128 cache_key,
   gtl::InsertOrUpdate(&kernel_cache_, cache_key, kernel);
 }
 
+bool EagerContext::ShouldStoreMetadata() {
+  mutex_lock ml(metadata_mu_);
+  return should_store_metadata_.load() || metadata_listener_ != nullptr;
+}
+
 void EagerContext::SetShouldStoreMetadata(bool value) {
+  mutex_lock ml(metadata_mu_);
   should_store_metadata_.store(value);
-  if (!value) {
-    mutex_lock ml(metadata_mu_);
+  if (!value || metadata_listener_ != nullptr) {
     run_metadata_.Clear();
   }
 }
diff --git a/tensorflow/core/common_runtime/eager/context.h b/tensorflow/core/common_runtime/eager/context.h
index 5ff6b3ffbd..b83f5707da 100644
--- a/tensorflow/core/common_runtime/eager/context.h
+++ b/tensorflow/core/common_runtime/eager/context.h
@@ -29,6 +29,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/eager/kernel_and_device.h"
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/common_runtime/rendezvous_mgr.h"
+#include "tensorflow/core/example/example.pb.h"
 #ifndef __ANDROID__
 #include "tensorflow/core/distributed_runtime/eager/eager_client.h"
 #include "tensorflow/core/distributed_runtime/server_lib.h"
@@ -66,6 +67,12 @@ enum ContextDevicePlacementPolicy {
   DEVICE_PLACEMENT_SILENT_FOR_INT32 = 3,
 };
 
+class RunMetadataListener {
+ public:
+  virtual ~RunMetadataListener() {}
+  virtual void BeforeClearRunMetadata() = 0;
+};
+
 class EagerContext {
  public:
   // TODO: remove this constructor once we migrate all callers to the next one.
@@ -172,10 +179,15 @@ class EagerContext {
   void ReleaseDeviceMgr() { local_device_manager_.release(); }
 
   // TODO(apassos) clean up RunMetadata storage.
-  mutex* MetadataMu() { return &metadata_mu_; }
-  bool ShouldStoreMetadata() { return should_store_metadata_.load(); }
+  mutex* MetadataMu() LOCK_RETURNED(metadata_mu_) { return &metadata_mu_; }
+  bool ShouldStoreMetadata() LOCKS_EXCLUDED(metadata_mu_);
   void SetShouldStoreMetadata(bool value);
   RunMetadata* RunMetadataProto() { return &run_metadata_; }
+  void ClearRunMetadata() EXCLUSIVE_LOCKS_REQUIRED(metadata_mu_);
+
+  Status RegisterRunMetadataListener(RunMetadataListener* listener)
+      LOCKS_EXCLUDED(metadata_mu_);
+  void ClearRunMetadataListener() LOCKS_EXCLUDED(metadata_mu_);
 
   void StartStep();
   void EndStep();
@@ -269,6 +281,7 @@ class EagerContext {
   std::atomic<bool> should_store_metadata_{false};
   mutex metadata_mu_;
   RunMetadata run_metadata_ GUARDED_BY(metadata_mu_);
+  RunMetadataListener* metadata_listener_ GUARDED_BY(metadata_mu_) = nullptr;
   GraphCollector graph_collector_;
   const bool log_device_placement_;
   // EagerExecutor for async execution.
diff --git a/tensorflow/core/common_runtime/eager/execute.cc b/tensorflow/core/common_runtime/eager/execute.cc
index 79806c3c73..9d2401929d 100644
--- a/tensorflow/core/common_runtime/eager/execute.cc
+++ b/tensorflow/core/common_runtime/eager/execute.cc
@@ -752,8 +752,8 @@ Status EagerExecute(EagerContext* ctx, Device* device,
     maybe_stats->set_all_end_rel_micros(nanos / EnvTime::kMicrosToNanos -
                                         maybe_stats->all_start_micros());
     maybe_stats->set_all_end_rel_nanos(nanos - maybe_stats->all_start_nanos());
-    mutex_lock ml(*ctx->MetadataMu());
     if (ctx->ShouldStoreMetadata()) {
+      mutex_lock ml(*ctx->MetadataMu());
       {
         GraphCollector* collector = ctx->GetGraphCollector();
         mutex_lock mll(collector->mu);
diff --git a/tensorflow/core/common_runtime/eager/profiler.cc b/tensorflow/core/common_runtime/eager/profiler.cc
new file mode 100644
index 0000000000..d670d6f344
--- /dev/null
+++ b/tensorflow/core/common_runtime/eager/profiler.cc
@@ -0,0 +1,77 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/common_runtime/eager/profiler.h"
+#include "tensorflow/cc/profiler/profiler.h"
+#include "tensorflow/core/framework/graph.pb.h"
+#include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/protobuf/config.pb.h"
+
+namespace tensorflow {
+
+/*static*/ std::unique_ptr<EagerProfiler> EagerProfiler::Create(
+    EagerContext* const context) {
+  return absl::WrapUnique(new EagerProfiler(context));
+}
+
+void EagerProfiler::BeforeClearRunMetadata() {
+  mutex_lock l(mutex_);
+  run_metadata_.MergeFrom(*context_->RunMetadataProto());
+}
+
+Status EagerProfiler::Status() {
+  mutex_lock l(mutex_);
+  return status_;
+}
+
+Status EagerProfiler::SerializeToString(string* content) {
+  {
+    mutex_lock l(mutex_);
+    if (!status_.ok()) return status_;
+  }
+  RunMetadata metadata;
+  GetMergetRunMetadata(&metadata);
+
+  // TODO(fishx): update tfprof to use a lighter representation instead of
+  // GraphDef.
+  GraphDef graph;
+  std::unique_ptr<tfprof::Profiler> tfprof(new tfprof::Profiler(graph));
+  tfprof->AddStep(0, metadata);
+  return tfprof->SerializeToString(content);
+}
+
+EagerProfiler::EagerProfiler(EagerContext* const context) : context_(context) {
+  LOG(INFO) << "Eager Profiler started.";
+
+  status_ = context_->RegisterRunMetadataListener(this);
+  if (!status_.ok()) {
+    LOG(INFO) << "Eager Profiler failed to start. Another profiler is running.";
+    return;
+  }
+}
+
+EagerProfiler::~EagerProfiler() {
+  context_->ClearRunMetadataListener();
+  LOG(INFO) << "Eager Profiler ended with status:" << status_;
+}
+
+void EagerProfiler::GetMergetRunMetadata(RunMetadata* metadata) {
+  mutex_lock ml(*context_->MetadataMu());
+  mutex_lock l(mutex_);
+  *metadata = run_metadata_;
+  metadata->MergeFrom(*context_->RunMetadataProto());
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/eager/profiler.h b/tensorflow/core/common_runtime/eager/profiler.h
new file mode 100644
index 0000000000..cadbfdb498
--- /dev/null
+++ b/tensorflow/core/common_runtime/eager/profiler.h
@@ -0,0 +1,63 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_PROFILER_H_
+#define TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_PROFILER_H_
+
+#include "tensorflow/core/common_runtime/eager/context.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/protobuf/config.pb.h"
+
+namespace tensorflow {
+
+// A profiler which will start profiling when creating the object and will stop
+// when the object is destroyed. It will profile all operations run under the
+// given EagerContext.
+// Multiple instances of it can be created, but at most one of them will profile
+// for each EagerContext. Status() will return OK only for the instance that is
+// profiling.
+// Thread-safety: TFE_Profiler is thread-safe.
+class EagerProfiler : RunMetadataListener {
+ public:
+  // Creates and EagerProfiler and starts profiling.
+  static std::unique_ptr<EagerProfiler> Create(EagerContext* const context);
+
+  // Deletes an exsiting Profiler and enables starting a new one.
+  ~EagerProfiler() override;
+
+  void BeforeClearRunMetadata() override LOCKS_EXCLUDED(mutex_)
+      EXCLUSIVE_LOCKS_REQUIRED(context_->MetadataMu());
+  tensorflow::Status Status() LOCKS_EXCLUDED(mutex_);
+
+  tensorflow::Status SerializeToString(string* content) LOCKS_EXCLUDED(mutex_);
+
+ private:
+  // Constructs an instance of the class and starts profiling
+  explicit EagerProfiler(EagerContext* const context);
+
+  // Profiler is neither copyable or movable.
+  EagerProfiler(const EagerProfiler&) = delete;
+  EagerProfiler& operator=(const EagerProfiler&) = delete;
+
+  void GetMergetRunMetadata(RunMetadata* metadata) LOCKS_EXCLUDED(mutex_);
+
+  RunMetadata run_metadata_ GUARDED_BY(mutex_);
+  tensorflow::Status status_ GUARDED_BY(mutex_);
+  EagerContext* const context_;
+  mutex mutex_;
+};
+
+}  // namespace tensorflow
+#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_PROFILER_H_
diff --git a/tensorflow/python/pywrap_tfe.i b/tensorflow/python/pywrap_tfe.i
index 733d471ca2..0620e0345c 100755
--- a/tensorflow/python/pywrap_tfe.i
+++ b/tensorflow/python/pywrap_tfe.i
@@ -32,6 +32,9 @@ limitations under the License.
 %rename("%s") TFE_ContextSetServerDef;
 %rename("%s") TFE_ContextAsyncWait;
 %rename("%s") TFE_ContextAsyncClearError;
+%rename("%s") TFE_NewProfiler;
+%rename("%s") TFE_DeleteProfiler;
+%rename("%s") TFE_ProfilerSerializeToString;
 %rename("%s") TFE_OpNameGetAttrType;
 %rename("%s") TFE_Py_InitEagerTensor;
 %rename("%s") TFE_Py_SetEagerTensorProfiler;
-- 
GitLab


From 52718ef7adf0f0a292dcd43432587c818f895ce2 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Wed, 9 Jan 2019 12:18:32 -0800
Subject: [PATCH 0435/2345] Add `axis` property to CosineProximity loss and
 metric. Refactor cosine proximity tests for better clarity.

PiperOrigin-RevId: 228564762
---
 tensorflow/python/keras/losses.py             | 26 ++++--
 tensorflow/python/keras/losses_test.py        | 93 ++++++++++++-------
 tensorflow/python/keras/metrics.py            | 12 ++-
 tensorflow/python/keras/metrics_test.py       | 58 ++++++++----
 ...rflow.keras.losses.-cosine-proximity.pbtxt |  2 +-
 .../golden/v1/tensorflow.keras.losses.pbtxt   |  4 +-
 ...flow.keras.metrics.-cosine-proximity.pbtxt |  2 +-
 .../golden/v1/tensorflow.keras.metrics.pbtxt  |  4 +-
 ...rflow.keras.losses.-cosine-proximity.pbtxt |  2 +-
 .../golden/v2/tensorflow.keras.losses.pbtxt   |  4 +-
 ...flow.keras.metrics.-cosine-proximity.pbtxt |  2 +-
 .../golden/v2/tensorflow.keras.metrics.pbtxt  |  4 +-
 12 files changed, 143 insertions(+), 70 deletions(-)

diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index 5f9278823c..66780de0f0 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -821,15 +821,15 @@ def poisson(y_true, y_pred):
               'keras.metrics.cosine',
               'keras.losses.cosine_proximity',
               'keras.losses.cosine')
-def cosine_proximity(y_true, y_pred):
-  y_true = nn.l2_normalize(y_true, axis=-1)
-  y_pred = nn.l2_normalize(y_pred, axis=-1)
-  return -math_ops.reduce_sum(y_true * y_pred, axis=-1)
+def cosine_proximity(y_true, y_pred, axis=-1):
+  y_true = nn.l2_normalize(y_true, axis=axis)
+  y_pred = nn.l2_normalize(y_pred, axis=axis)
+  return -math_ops.reduce_sum(y_true * y_pred, axis=axis)
 
 
 @keras_export('keras.losses.CosineProximity')
 class CosineProximity(Loss):
-  """Computes the cosine distance between `y_true` and `y_pred`.
+  """Computes the cosine proximity between `y_true` and `y_pred`.
 
   Usage:
 
@@ -845,8 +845,22 @@ class CosineProximity(Loss):
   model = keras.models.Model(inputs, outputs)
   model.compile('sgd', loss=tf.losses.CosineProximity())
   ```
+
+  Args:
+    axis: (Optional) Defaults to -1. The dimension along which the cosine
+      proximity is computed.
+    reduction: (Optional) Type of `tf.losses.Reduction` to apply to loss.
+      Default value is `SUM_OVER_BATCH_SIZE`.
+    name: Optional name for the op.
   """
 
+  def __init__(self,
+               axis=-1,
+               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               name=None):
+    super(CosineProximity, self).__init__(reduction=reduction, name=name)
+    self.axis = axis
+
   def call(self, y_true, y_pred):
     """Calculates the cosine proximity loss.
 
@@ -859,7 +873,7 @@ class CosineProximity(Loss):
     """
     y_pred = ops.convert_to_tensor(y_pred)
     y_true = math_ops.cast(y_true, y_pred.dtype)
-    return cosine_proximity(y_true, y_pred)
+    return cosine_proximity(y_true, y_pred, axis=self.axis)
 
 
 # Aliases.
diff --git a/tensorflow/python/keras/losses_test.py b/tensorflow/python/keras/losses_test.py
index 5d6cd1be55..004c30f84d 100644
--- a/tensorflow/python/keras/losses_test.py
+++ b/tensorflow/python/keras/losses_test.py
@@ -485,59 +485,88 @@ class MeanSquaredLogarithmicErrorTest(test.TestCase):
 @test_util.run_all_in_graph_and_eager_modes
 class CosineProximityTest(test.TestCase):
 
+  def l2_norm(self, x, axis):
+    epsilon = 1e-12
+    square_sum = np.sum(np.square(x), axis=axis, keepdims=True)
+    x_inv_norm = 1 / np.sqrt(np.maximum(square_sum, epsilon))
+    return np.multiply(x, x_inv_norm)
+
+  def setup(self, axis=1):
+    self.np_y_true = np.asarray([[1, 9, 2], [-5, -2, 6]], dtype=np.float32)
+    self.np_y_pred = np.asarray([[4, 8, 12], [8, 1, 3]], dtype=np.float32)
+
+    y_true = self.l2_norm(self.np_y_true, axis)
+    y_pred = self.l2_norm(self.np_y_pred, axis)
+    self.expected_loss = -np.sum(np.multiply(y_true, y_pred), axis=(axis,))
+
+    self.y_true = constant_op.constant(self.np_y_true)
+    self.y_pred = constant_op.constant(self.np_y_pred)
+
   def test_config(self):
     cosine_obj = keras.losses.CosineProximity(
-        reduction=losses_impl.ReductionV2.SUM, name='cosine_loss')
+        axis=2, reduction=losses_impl.ReductionV2.SUM, name='cosine_loss')
     self.assertEqual(cosine_obj.name, 'cosine_loss')
     self.assertEqual(cosine_obj.reduction, losses_impl.ReductionV2.SUM)
+    self.assertEqual(cosine_obj.axis, 2)
 
   def test_unweighted(self):
+    self.setup()
     cosine_obj = keras.losses.CosineProximity()
-    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
-    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
-                                  shape=(2, 3),
-                                  dtype=dtypes.float32)
-    loss = cosine_obj(y_true, y_pred)
-    self.assertAlmostEqual(self.evaluate(loss), -0.18722, 3)
+    loss = cosine_obj(self.y_true, self.y_pred)
+    expected_loss = np.mean(self.expected_loss)
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
 
   def test_scalar_weighted(self):
+    self.setup()
     cosine_obj = keras.losses.CosineProximity()
-    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
-    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
-                                  shape=(2, 3),
-                                  dtype=dtypes.float32)
-    loss = cosine_obj(y_true, y_pred, sample_weight=2.3)
-    self.assertAlmostEqual(self.evaluate(loss), -0.43060, 3)
+    sample_weight = 2.3
+    loss = cosine_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    expected_loss = np.mean(self.expected_loss * sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
 
   def test_sample_weighted(self):
+    self.setup()
     cosine_obj = keras.losses.CosineProximity()
-    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
-    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
-                                  shape=(2, 3),
-                                  dtype=dtypes.float32)
-    sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
-    loss = cosine_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAlmostEqual(self.evaluate(loss), 0.15599, 3)
+    sample_weight = np.asarray([1.2, 3.4])
+    loss = cosine_obj(
+        self.y_true,
+        self.y_pred,
+        sample_weight=constant_op.constant(sample_weight))
+    expected_loss = np.mean(self.expected_loss * sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
 
   def test_timestep_weighted(self):
+    self.setup()
     cosine_obj = keras.losses.CosineProximity()
-    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3, 1))
-    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
-                                  shape=(2, 3, 1),
-                                  dtype=dtypes.float32)
-    sample_weight = constant_op.constant([3, 6, 5, 0, 4, 2], shape=(2, 3))
-    loss = cosine_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAlmostEqual(self.evaluate(loss), -2.0000, 3)
+    np_y_true = self.np_y_true.reshape((2, 3, 1))
+    np_y_pred = self.np_y_pred.reshape((2, 3, 1))
+    sample_weight = np.asarray([3, 6, 5, 0, 4, 2]).reshape((2, 3))
+
+    y_true = self.l2_norm(np_y_true, 2)
+    y_pred = self.l2_norm(np_y_pred, 2)
+    expected_loss = -np.sum(np.multiply(y_true, y_pred), axis=(2,))
+
+    y_true = constant_op.constant(np_y_true)
+    y_pred = constant_op.constant(np_y_pred)
+    loss = cosine_obj(
+        y_true, y_pred, sample_weight=constant_op.constant(sample_weight))
+
+    expected_loss = np.mean(expected_loss * sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
 
   def test_zero_weighted(self):
+    self.setup()
     cosine_obj = keras.losses.CosineProximity()
-    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
-    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
-                                  shape=(2, 3),
-                                  dtype=dtypes.float32)
-    loss = cosine_obj(y_true, y_pred, sample_weight=0)
+    loss = cosine_obj(self.y_true, self.y_pred, sample_weight=0)
     self.assertAlmostEqual(self.evaluate(loss), 0., 3)
 
+  def test_axis(self):
+    self.setup(axis=1)
+    cosine_obj = keras.losses.CosineProximity(axis=1)
+    loss = cosine_obj(self.y_true, self.y_pred)
+    expected_loss = np.mean(self.expected_loss)
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+
 
 @test_util.run_all_in_graph_and_eager_modes
 class BinaryCrossentropyTest(test.TestCase):
diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 2c2700ffa3..a13d8747d6 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -1443,8 +1443,16 @@ class CosineProximity(MeanMetricWrapper):
   ```
   """
 
-  def __init__(self, name='cosine_proximity', dtype=None):
-    super(CosineProximity, self).__init__(cosine, name, dtype=dtype)
+  def __init__(self, name='cosine_proximity', dtype=None, axis=-1):
+    """Creates a `CosineProximity` instance.
+
+    Args:
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+      axis: (Optional) Defaults to -1. The dimension along which the cosine
+        proximity is computed.
+    """
+    super(CosineProximity, self).__init__(cosine, name, dtype=dtype, axis=axis)
 
   @classmethod
   def from_config(cls, config):
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index 4f8406f0e3..42da1dfb99 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -1049,8 +1049,26 @@ class SpecificityAtSensitivityTest(test.TestCase, parameterized.TestCase):
 @test_util.run_all_in_graph_and_eager_modes
 class CosineProximityTest(test.TestCase):
 
+  def l2_norm(self, x, axis):
+    epsilon = 1e-12
+    square_sum = np.sum(np.square(x), axis=axis, keepdims=True)
+    x_inv_norm = 1 / np.sqrt(np.maximum(square_sum, epsilon))
+    return np.multiply(x, x_inv_norm)
+
+  def setup(self, axis=1):
+    self.np_y_true = np.asarray([[1, 9, 2], [-5, -2, 6]], dtype=np.float32)
+    self.np_y_pred = np.asarray([[4, 8, 12], [8, 1, 3]], dtype=np.float32)
+
+    y_true = self.l2_norm(self.np_y_true, axis)
+    y_pred = self.l2_norm(self.np_y_pred, axis)
+    self.expected_loss = -np.sum(np.multiply(y_true, y_pred), axis=(axis,))
+
+    self.y_true = constant_op.constant(self.np_y_true)
+    self.y_pred = constant_op.constant(self.np_y_pred)
+
   def test_config(self):
-    cosine_obj = metrics.CosineProximity(name='my_cos', dtype=dtypes.int32)
+    cosine_obj = metrics.CosineProximity(
+        axis=2, name='my_cos', dtype=dtypes.int32)
     self.assertEqual(cosine_obj.name, 'my_cos')
     self.assertEqual(cosine_obj._dtype, dtypes.int32)
 
@@ -1060,29 +1078,33 @@ class CosineProximityTest(test.TestCase):
     self.assertEqual(cosine_obj2._dtype, dtypes.int32)
 
   def test_unweighted(self):
+    self.setup()
     cosine_obj = metrics.CosineProximity()
     self.evaluate(variables.variables_initializer(cosine_obj.variables))
-
-    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
-                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
-    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
-                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
-
-    update_op = cosine_obj.update_state(y_true, y_pred)
-    self.evaluate(update_op)
-    result = cosine_obj.result()
-    self.assertAllClose(-0.60723, result, atol=1e-5)
+    loss = cosine_obj(self.y_true, self.y_pred)
+    expected_loss = np.mean(self.expected_loss)
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
 
   def test_weighted(self):
+    self.setup()
     cosine_obj = metrics.CosineProximity()
     self.evaluate(variables.variables_initializer(cosine_obj.variables))
-    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
-                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
-    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
-                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
-    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
-    result = cosine_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAllClose(-0.59916, self.evaluate(result), atol=1e-5)
+    sample_weight = np.asarray([1.2, 3.4])
+    loss = cosine_obj(
+        self.y_true,
+        self.y_pred,
+        sample_weight=constant_op.constant(sample_weight))
+    expected_loss = np.sum(
+        self.expected_loss * sample_weight) / np.sum(sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+
+  def test_axis(self):
+    self.setup(axis=1)
+    cosine_obj = metrics.CosineProximity(axis=1)
+    self.evaluate(variables.variables_initializer(cosine_obj.variables))
+    loss = cosine_obj(self.y_true, self.y_pred)
+    expected_loss = np.mean(self.expected_loss)
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
 
 
 @test_util.run_all_in_graph_and_eager_modes
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-cosine-proximity.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-cosine-proximity.pbtxt
index de35966f48..4952a76291 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-cosine-proximity.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-cosine-proximity.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'axis\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'sum_over_batch_size\', \'None\'], "
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
index 532d2b2f88..b71e89883e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
@@ -74,11 +74,11 @@ tf_module {
   }
   member_method {
     name: "cosine"
-    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'y_true\', \'y_pred\', \'axis\'], varargs=None, keywords=None, defaults=[\'-1\'], "
   }
   member_method {
     name: "cosine_proximity"
-    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'y_true\', \'y_pred\', \'axis\'], varargs=None, keywords=None, defaults=[\'-1\'], "
   }
   member_method {
     name: "deserialize"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-cosine-proximity.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-cosine-proximity.pbtxt
index d095ed42ef..c50638740f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-cosine-proximity.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-cosine-proximity.pbtxt
@@ -89,7 +89,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'cosine_proximity\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'dtype\', \'axis\'], varargs=None, keywords=None, defaults=[\'cosine_proximity\', \'None\', \'-1\'], "
   }
   member_method {
     name: "add_loss"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
index 743a8ae34a..54f5b21f55 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
@@ -122,11 +122,11 @@ tf_module {
   }
   member_method {
     name: "cosine"
-    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'y_true\', \'y_pred\', \'axis\'], varargs=None, keywords=None, defaults=[\'-1\'], "
   }
   member_method {
     name: "cosine_proximity"
-    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'y_true\', \'y_pred\', \'axis\'], varargs=None, keywords=None, defaults=[\'-1\'], "
   }
   member_method {
     name: "deserialize"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-cosine-proximity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-cosine-proximity.pbtxt
index de35966f48..4952a76291 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-cosine-proximity.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-cosine-proximity.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'axis\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'sum_over_batch_size\', \'None\'], "
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
index ef4fd66a7a..673593ed6c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
@@ -78,11 +78,11 @@ tf_module {
   }
   member_method {
     name: "cosine"
-    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'y_true\', \'y_pred\', \'axis\'], varargs=None, keywords=None, defaults=[\'-1\'], "
   }
   member_method {
     name: "cosine_proximity"
-    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'y_true\', \'y_pred\', \'axis\'], varargs=None, keywords=None, defaults=[\'-1\'], "
   }
   member_method {
     name: "deserialize"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-cosine-proximity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-cosine-proximity.pbtxt
index d095ed42ef..c50638740f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-cosine-proximity.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-cosine-proximity.pbtxt
@@ -89,7 +89,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'cosine_proximity\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'dtype\', \'axis\'], varargs=None, keywords=None, defaults=[\'cosine_proximity\', \'None\', \'-1\'], "
   }
   member_method {
     name: "add_loss"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
index 743a8ae34a..54f5b21f55 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
@@ -122,11 +122,11 @@ tf_module {
   }
   member_method {
     name: "cosine"
-    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'y_true\', \'y_pred\', \'axis\'], varargs=None, keywords=None, defaults=[\'-1\'], "
   }
   member_method {
     name: "cosine_proximity"
-    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'y_true\', \'y_pred\', \'axis\'], varargs=None, keywords=None, defaults=[\'-1\'], "
   }
   member_method {
     name: "deserialize"
-- 
GitLab


From 0d15dbe9c5df49c6a9757fedbe47f893837363db Mon Sep 17 00:00:00 2001
From: Mark Heffernan <meheff@google.com>
Date: Wed, 9 Jan 2019 12:31:46 -0800
Subject: [PATCH 0436/2345] Remove deprecated methods in ShapeUtil. No
 functional change. Replace with calls to Shape methods.

PiperOrigin-RevId: 228566953
---
 tensorflow/c/eager/c_api_debug.cc             |   4 +-
 .../compiler/xla/service/scatter_expander.cc  |   2 +-
 .../compiler/xla/service/shape_inference.cc   |   4 +-
 tensorflow/compiler/xla/shape.cc              |   4 +-
 tensorflow/compiler/xla/shape_util.cc         | 107 ++++++++----------
 tensorflow/compiler/xla/shape_util.h          |  44 +------
 tensorflow/compiler/xla/shape_util_test.cc    |   4 -
 tensorflow/compiler/xla/tests/tuple_test.cc   |   5 +-
 8 files changed, 61 insertions(+), 113 deletions(-)

diff --git a/tensorflow/c/eager/c_api_debug.cc b/tensorflow/c/eager/c_api_debug.cc
index 52b0824552..ffcd5ace0b 100644
--- a/tensorflow/c/eager/c_api_debug.cc
+++ b/tensorflow/c/eager/c_api_debug.cc
@@ -83,7 +83,7 @@ TF_CAPI_EXPORT extern TFE_TensorDebugInfo* TFE_TensorHandleTensorDebugInfo(
       }
     }
 
-    if (xla::ShapeUtil::IsTuple(padded_shape)) {
+    if (padded_shape.IsTuple()) {
       if (xla::ShapeUtil::TupleElementCount(padded_shape) != 2) {
         // Currently, the only case of XlaTensor containing a tuple shape is to
         // represent 64 bit ints, doubles, and complex numbers (we don't support
@@ -99,7 +99,7 @@ TF_CAPI_EXPORT extern TFE_TensorDebugInfo* TFE_TensorHandleTensorDebugInfo(
       xla::Shape shape0 = xla::ShapeUtil::GetTupleElementShape(padded_shape, 0);
       const xla::Shape& shape1 =
           xla::ShapeUtil::GetTupleElementShape(padded_shape, 1);
-      if (xla::ShapeUtil::IsTuple(shape0) || xla::ShapeUtil::IsTuple(shape1)) {
+      if (shape0.IsTuple() || shape1.IsTuple()) {
         status->status = tensorflow::errors::InvalidArgument(
             "XlaTensors should not contain nested tuples. Shape: ",
             padded_shape.DebugString());
diff --git a/tensorflow/compiler/xla/service/scatter_expander.cc b/tensorflow/compiler/xla/service/scatter_expander.cc
index 8b9955faf8..036c3c36f6 100644
--- a/tensorflow/compiler/xla/service/scatter_expander.cc
+++ b/tensorflow/compiler/xla/service/scatter_expander.cc
@@ -59,7 +59,7 @@ static StatusOr<HloInstruction*> CanonicalizeScatterIndices(
   TF_ASSIGN_OR_RETURN(
       HloInstruction * transposed_scatter_indices,
       TransposeIndexVectorDimToLast(scatter_indices, index_vector_dim));
-  if (ShapeUtil::Rank(scatter_indices->shape()) == index_vector_dim + 1 &&
+  if (scatter_indices->shape().rank() == index_vector_dim + 1 &&
       scatter_indices->shape().dimensions(index_vector_dim) == 1) {
     auto new_shape =
         ShapeUtil::DeleteDimension(index_vector_dim, scatter_indices->shape());
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index b729dd7660..1d3f84af95 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -2358,7 +2358,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
   };
 
   // Check the shapes of computation parameters and return types.
-  if (!ShapeUtil::ShapeIs(condition.result(), PRED, {})) {
+  if (!ShapeUtil::Equal(condition.result(), ShapeUtil::MakeShape(PRED, {}))) {
     return InvalidArgument("Condition must return a boolean; got %s.",
                            shape_string());
   }
@@ -2378,7 +2378,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
     const Shape& predicate, const Shape& true_operand,
     const Shape& false_operand, const ProgramShape& true_computation,
     const ProgramShape& false_computation) {
-  if (!ShapeUtil::ShapeIs(predicate, PRED, {})) {
+  if (!ShapeUtil::Equal(predicate, ShapeUtil::MakeShape(PRED, {}))) {
     return InvalidArgument("Predicate must be a boolean; got %s.",
                            ShapeUtil::HumanString(predicate));
   }
diff --git a/tensorflow/compiler/xla/shape.cc b/tensorflow/compiler/xla/shape.cc
index ea4f722ef4..1a029efe85 100644
--- a/tensorflow/compiler/xla/shape.cc
+++ b/tensorflow/compiler/xla/shape.cc
@@ -80,7 +80,7 @@ string Shape::ToString(bool print_layout) const {
 }
 
 bool Shape::is_static() const {
-  if (ShapeUtil::IsTuple(*this)) {
+  if (IsTuple()) {
     for (const Shape& subshape : tuple_shapes_) {
       if (!subshape.is_static()) {
         return false;
@@ -91,7 +91,7 @@ bool Shape::is_static() const {
 }
 
 void Shape::DeleteDimension(int64 dim_to_delete) {
-  CHECK(ShapeUtil::IsArray(*this));
+  CHECK(IsArray());
   CHECK_GE(dim_to_delete, 0);
   CHECK_LT(dim_to_delete, dimensions_.size());
   dimensions_.erase(dimensions_.begin() + dim_to_delete);
diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc
index 7f359311c3..235b065585 100644
--- a/tensorflow/compiler/xla/shape_util.cc
+++ b/tensorflow/compiler/xla/shape_util.cc
@@ -146,7 +146,7 @@ bool CompareShapes(const Shape& lhs, const Shape& rhs, bool compare_layouts,
     return false;
   }
 
-  for (int i = 0; i < ShapeUtil::Rank(lhs); ++i) {
+  for (int i = 0; i < lhs.rank(); ++i) {
     if (lhs.is_dynamic_dimension(i) != rhs.is_dynamic_dimension(i)) {
       VLOG(3)
           << "CompareShapes: lhs and rhs have different dynamic dimensions.";
@@ -208,11 +208,6 @@ StatusOr<Shape> MakeShapeWithLayoutInternal(
   return equal;
 }
 
-/* static */ int64 ShapeUtil::Rank(const Shape& shape) {
-  CHECK(shape.IsArray()) << "Non-arrays do not have a rank, shape: " << shape;
-  return shape.dimensions_size();
-}
-
 /* static */ int64 ShapeUtil::TrueRank(const Shape& shape) {
   int64 accum = 0;
   for (int64 dimension : shape.dimensions()) {
@@ -344,7 +339,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
 
 /* static */ void ShapeUtil::AppendMajorDimension(int bound, Shape* shape) {
   CHECK(LayoutUtil::IsDenseArray(*shape));
-  shape->mutable_layout()->add_minor_to_major(Rank(*shape));
+  shape->mutable_layout()->add_minor_to_major(shape->rank());
   shape->add_dimensions(bound);
   TF_DCHECK_OK(ValidateShape(*shape));
 }
@@ -359,7 +354,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
 }
 
 /* static */ bool ShapeUtil::ElementHasBitWidth(const Shape& shape, int bits) {
-  if (!IsArray(shape)) {
+  if (!shape.IsArray()) {
     return false;
   }
   return primitive_util::BitWidth(shape.element_type()) == bits;
@@ -401,26 +396,24 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
   return primitive_util::IsFloatingPointType(shape.element_type());
 }
 
-/* static */ bool ShapeUtil::IsArray(const Shape& shape) {
-  return IsArrayPrimitiveType(shape.element_type());
-}
-
 /* static */ bool ShapeUtil::IsNestedTuple(const Shape& shape) {
-  return IsTuple(shape) && absl::c_any_of(shape.tuple_shapes(), IsTuple);
+  return shape.IsTuple() &&
+         absl::c_any_of(shape.tuple_shapes(),
+                        [](const Shape& s) { return s.IsTuple(); });
 }
 
 /* static */ bool ShapeUtil::IsEmptyTuple(const Shape& shape) {
-  return IsTuple(shape) && TupleElementCount(shape) == 0;
+  return shape.IsTuple() && TupleElementCount(shape) == 0;
 }
 
 /* static */ int64 ShapeUtil::TupleElementCount(const Shape& shape) {
-  CHECK(IsTuple(shape)) << HumanString(shape);
+  CHECK(shape.IsTuple()) << HumanString(shape);
   return shape.tuple_shapes_size();
 }
 
 /* static */ const Shape& ShapeUtil::GetTupleElementShape(const Shape& shape,
                                                           int64 index) {
-  CHECK(IsTuple(shape));
+  CHECK(shape.IsTuple());
   CHECK_GT(TupleElementCount(shape), index);
   TF_DCHECK_OK(ValidateShapeWithOptionalLayout(shape.tuple_shapes(index)));
   return shape.tuple_shapes(index);
@@ -436,7 +429,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
 /* static */ Shape ShapeUtil::SliceTuple(const Shape& tuple, int64 start,
                                          int64 limit) {
   TF_DCHECK_OK(ValidateShapeWithOptionalLayout(tuple));
-  CHECK(IsTuple(tuple));
+  CHECK(tuple.IsTuple());
   CHECK_LE(start, TupleElementCount(tuple));
   CHECK_LE(limit, TupleElementCount(tuple));
 
@@ -453,15 +446,9 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
                                               complex_shape.element_type()));
 }
 
-/* static */ bool ShapeUtil::ShapeIs(const Shape& shape,
-                                     PrimitiveType element_type,
-                                     std::initializer_list<int64> dimensions) {
-  return Equal(shape, MakeShape(element_type, dimensions));
-}
-
 /* static */ int64 ShapeUtil::ElementsIn(const Shape& shape) {
-  DCHECK(IsArray(shape)) << ShapeUtil::HumanString(shape);
-  DCHECK_EQ(shape.dimensions_size(), Rank(shape));
+  DCHECK(shape.IsArray()) << ShapeUtil::HumanString(shape);
+  DCHECK_EQ(shape.dimensions_size(), shape.rank());
   if (shape.dimensions().size() == 1) {
     return shape.dimensions()[0];
   }
@@ -471,8 +458,8 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
 }
 
 /* static */ int64 ShapeUtil::ElementsInRecursive(const Shape& shape) {
-  CHECK(IsArray(shape) || IsTuple(shape));
-  if (IsArray(shape)) {
+  CHECK(shape.IsArray() || shape.IsTuple());
+  if (shape.IsArray()) {
     return ElementsIn(shape);
   }
   int64 count = 0;
@@ -505,7 +492,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
 }
 
 /* static */ string ShapeUtil::HumanString(const Shape& shape) {
-  if (IsTuple(shape)) {
+  if (shape.IsTuple()) {
     string text = "(";
     const char* prefix = "";
     for (const Shape& elem_shape : shape.tuple_shapes()) {
@@ -529,7 +516,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
 }
 
 /* static */ string ShapeUtil::HumanStringWithLayout(const Shape& shape) {
-  if (IsTuple(shape)) {
+  if (shape.IsTuple()) {
     string text = "(";
     const char* prefix = "";
     for (const Shape& elem_shape : shape.tuple_shapes()) {
@@ -545,7 +532,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
     StrAppend(&result, (i > 0) ? "," : "", shape.dimensions(i));
   }
   result += "]";
-  if (!IsScalar(shape) && IsArray(shape)) {
+  if (!IsScalar(shape) && shape.IsArray()) {
     if (LayoutUtil::HasLayout(shape)) {
       StrAppend(&result, LayoutUtil::HumanString(shape.layout()));
     }
@@ -580,8 +567,8 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
 
 /* static */ bool ShapeUtil::CompatibleIgnoringElementType(const Shape& lhs,
                                                            const Shape& rhs) {
-  if (IsArray(lhs)) {
-    return IsArray(rhs) && SameDimensions(lhs, rhs);
+  if (lhs.IsArray()) {
+    return rhs.IsArray() && SameDimensions(lhs, rhs);
   } else if (lhs.element_type() == TUPLE) {
     return rhs.element_type() == TUPLE &&
            absl::c_equal(lhs.tuple_shapes(), rhs.tuple_shapes(),
@@ -594,8 +581,8 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
 
 /* static */ bool ShapeUtil::CompatibleIgnoringFpPrecision(const Shape& lhs,
                                                            const Shape& rhs) {
-  if (IsArray(lhs)) {
-    return IsArray(rhs) && SameElementTypeIgnoringFpPrecision(lhs, rhs) &&
+  if (lhs.IsArray()) {
+    return rhs.IsArray() && SameElementTypeIgnoringFpPrecision(lhs, rhs) &&
            CompatibleIgnoringElementType(lhs, rhs);
   } else if (lhs.element_type() == TUPLE) {
     return rhs.element_type() == TUPLE &&
@@ -615,7 +602,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
 /* static */ int64 ShapeUtil::GetDimensionNumber(const Shape& shape,
                                                  int64 dimension_number) {
   if (dimension_number < 0) {
-    dimension_number += Rank(shape);
+    dimension_number += shape.rank();
   }
   CHECK_GE(dimension_number, 0);
   return dimension_number;
@@ -669,7 +656,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
   TF_DCHECK_OK(ValidateShape(shape));
   if (shape.element_type() == TUPLE) {
     return ByteSizeOfTupleIndexTable(shape, pointer_size);
-  } else if (IsArray(shape)) {
+  } else if (shape.IsArray()) {
     int64 byte_size = ByteSizeOfElements(shape);
     if (LayoutUtil::IsSparseArray(shape)) {
       byte_size += ByteSizeOfSparseIndices(shape);
@@ -755,10 +742,10 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
     return Status::OK();
   }
 
-  if (LayoutUtil::IsSparseArray(shape) && Rank(shape) == 0) {
+  if (LayoutUtil::IsSparseArray(shape) && shape.rank() == 0) {
     return InvalidArgument("sparse arrays must have rank > 0");
   }
-  for (int64 i = 0; i < Rank(shape); ++i) {
+  for (int64 i = 0; i < shape.rank(); ++i) {
     int64 dimension = shape.dimensions(i);
     if (dimension < 0) {
       return InvalidArgument(
@@ -774,7 +761,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
 /* static */ Status ShapeUtil::ValidateShapeSize(const Shape& shape) {
   VLOG(3) << "Validating shape size: " << ShapeUtil::HumanString(shape);
 
-  if (!IsArray(shape)) {
+  if (!shape.IsArray()) {
     return Status::OK();
   }
 
@@ -867,7 +854,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
                                           ShapeIndexView index) {
   const Shape* subshape = &shape;
   for (auto i : index) {
-    if (!IsTuple(*subshape) || i >= subshape->tuple_shapes_size() || i < 0) {
+    if (!subshape->IsTuple() || i >= subshape->tuple_shapes_size() || i < 0) {
       return false;
     }
     subshape = &subshape->tuple_shapes(i);
@@ -879,7 +866,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
                                                  ShapeIndexView index) {
   const Shape* return_shape = &shape;
   for (auto i : index) {
-    CHECK(IsTuple(*return_shape))
+    CHECK(return_shape->IsTuple())
         << "Invalid index " << index << " for shape " << shape;
     return_shape = &return_shape->tuple_shapes(i);
   }
@@ -890,7 +877,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
     const Shape& shape, ShapeIndexView index) {
   const Shape* return_shape = &shape;
   for (auto i : index) {
-    if (!IsTuple(*return_shape) || i < 0 ||
+    if (!return_shape->IsTuple() || i < 0 ||
         i >= return_shape->tuple_shapes_size()) {
       return InvalidArgument(
           "Shape index %s not a valid subshape index for tuple with shape %s",
@@ -905,7 +892,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
                                                   ShapeIndexView index) {
   Shape* return_shape = shape;
   for (auto i : index) {
-    CHECK(IsTuple(*return_shape));
+    CHECK(return_shape->IsTuple());
     return_shape = return_shape->mutable_tuple_shapes(i);
   }
   return return_shape;
@@ -913,11 +900,11 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
 
 /* static */
 bool ShapeUtil::IsLeafIndex(const Shape& shape, const ShapeIndex& index) {
-  return !IsTuple(GetSubshape(shape, index));
+  return !GetSubshape(shape, index).IsTuple();
 }
 
 /* static */ int64 ShapeUtil::GetLeafCount(const Shape& shape) {
-  if (!IsTuple(shape)) {
+  if (!shape.IsTuple()) {
     return 1;
   }
   int64 count = 0;
@@ -1081,8 +1068,8 @@ Status ForEachMutableSubshapeHelper(
 /* static */ std::tuple<bool, std::vector<int64>, std::vector<int64>>
 ShapeUtil::InsertedOrDeleted1SizedDimensions(const Shape& shape_pre,
                                              const Shape& shape_post) {
-  CHECK(IsArray(shape_pre));
-  CHECK(IsArray(shape_post));
+  CHECK(shape_pre.IsArray());
+  CHECK(shape_post.IsArray());
 
   auto nil = std::make_tuple(false, std::vector<int64>(), std::vector<int64>());
 
@@ -1129,7 +1116,7 @@ ShapeUtil::InsertedOrDeleted1SizedDimensions(const Shape& shape_pre,
     auto unmodified_dim_pair =
         i < unmodified_dims.size()
             ? unmodified_dims[i]
-            : std::make_pair(Rank(shape_pre), Rank(shape_post));
+            : std::make_pair(shape_pre.rank(), shape_post.rank());
     if (!check_modified_dims(prior_unmodified_dim_pair, unmodified_dim_pair)) {
       return nil;
     }
@@ -1141,8 +1128,8 @@ ShapeUtil::InsertedOrDeleted1SizedDimensions(const Shape& shape_pre,
 /* static */ std::vector<std::pair<int64, int64>>
 ShapeUtil::DimensionsUnmodifiedByReshape(const Shape& input_shape,
                                          const Shape& output_shape) {
-  CHECK(IsArray(input_shape));
-  CHECK(IsArray(output_shape));
+  CHECK(input_shape.IsArray());
+  CHECK(output_shape.IsArray());
 
   // Unmodified dimensions are merely common factors of rank 1.
   auto common_factors = CommonFactors(AsInt64Slice(input_shape.dimensions()),
@@ -1192,8 +1179,8 @@ ShapeUtil::DimensionsUnmodifiedByReshape(const Shape& input_shape,
 
 /* static */ bool ShapeUtil::ReshapeIsBitcast(const Shape& input_shape,
                                               const Shape& output_shape) {
-  CHECK(IsArray(input_shape));
-  CHECK(IsArray(output_shape));
+  CHECK(input_shape.IsArray());
+  CHECK(output_shape.IsArray());
   CHECK(LayoutUtil::HasLayout(input_shape));
   CHECK(LayoutUtil::HasLayout(output_shape));
 
@@ -1321,12 +1308,12 @@ ShapeUtil::DimensionsUnmodifiedByReshape(const Shape& input_shape,
     Shape output_shape_dim0_major = MakeShapeWithDescendingLayout(
         output_shape.element_type(), AsInt64Slice(output_shape.dimensions()));
 
-    for (int64 input_dim = 0; input_dim < Rank(input_shape); ++input_dim) {
+    for (int64 input_dim = 0; input_dim < input_shape.rank(); ++input_dim) {
       if (input_shape.dimensions(input_dim) <= 1) {
         continue;
       }
 
-      std::vector<int64> input_unit_index(Rank(input_shape), 0);
+      std::vector<int64> input_unit_index(input_shape.rank(), 0);
       input_unit_index[input_dim] = 1;
       int64 logical_linear_index =
           IndexUtil::MultidimensionalIndexToLinearIndex(input_shape_dim0_major,
@@ -1352,11 +1339,11 @@ ShapeUtil::DimensionsUnmodifiedByReshape(const Shape& input_shape,
 
 /* static */ absl::optional<Shape> ShapeUtil::AlignLayouts(
     const Shape& input_shape, const Shape& output_shape) {
-  CHECK(IsArray(input_shape));
-  CHECK(IsArray(output_shape));
+  CHECK(input_shape.IsArray());
+  CHECK(output_shape.IsArray());
 
-  int64 input_rank = Rank(input_shape);
-  int64 output_rank = Rank(output_shape);
+  int64 input_rank = input_shape.rank();
+  int64 output_rank = output_shape.rank();
 
   // First, calculate an alignment of the dimensions. A consecutive sequence of
   // input dimensions and output dimensions belong to the same alignment part if
@@ -1493,14 +1480,14 @@ ShapeUtil::DimensionsUnmodifiedByReshape(const Shape& input_shape,
 
 /* static */ Shape ShapeUtil::DeleteDimension(int64 dim_to_delete,
                                               Shape shape) {
-  CHECK(IsArray(shape));
+  CHECK(shape.IsArray());
   shape.DeleteDimension(dim_to_delete);
   return shape;
 }
 
 /* static */ Shape ShapeUtil::FilterDimensions(
     const std::function<bool(int64)>& p, Shape shape) {
-  CHECK(IsArray(shape));
+  CHECK(shape.IsArray());
   std::vector<int64> dims_to_delete;
   for (int64 i = shape.dimensions().size() - 1; i >= 0; --i) {
     if (!p(i)) {
diff --git a/tensorflow/compiler/xla/shape_util.h b/tensorflow/compiler/xla/shape_util.h
index c8295e85ce..e98c6e024b 100644
--- a/tensorflow/compiler/xla/shape_util.h
+++ b/tensorflow/compiler/xla/shape_util.h
@@ -185,7 +185,7 @@ class ShapeUtil {
   // may not actually be able to store this number of elements. See
   // LayoutUtil::MaxSparseElements(shape) to obtain the maximum number of
   // elements that can be stored in a sparse shape.
-  // Precondition: IsArray(shape)
+  // Precondition: shape.IsArray()
   static int64 ElementsIn(const Shape& shape);
 
   // As ElementsIn(), but recurses through tuples.
@@ -296,11 +296,6 @@ class ShapeUtil {
   // As Equal, but allow one of lhs and rhs to be F16 while the other is F32.
   static bool EqualIgnoringFpPrecision(const Shape& lhs, const Shape& rhs);
 
-  // Returns the rank (number of dimensions) of the given shape.
-  // Precondition: !IsTuple(shape)
-  ABSL_DEPRECATED("Use `Shape::rank` instead.")
-  static int64 Rank(const Shape& shape);
-
   // Returns the number of dimensions for which the dimension is not (trivially)
   // 1. e.g., f32[2x1x1] has a true rank of 1D, the other dimensions are just
   // fluff. Note that zero dimensions are included in the true rank, e.g.,
@@ -314,10 +309,10 @@ class ShapeUtil {
   // Scalar-specific
 
   static bool IsScalar(const Shape& shape) {
-    return IsArray(shape) && Rank(shape) == 0;
+    return shape.IsArray() && shape.rank() == 0;
   }
   static bool IsEffectiveScalar(const Shape& shape) {
-    return IsArray(shape) && TrueRank(shape) == 0;
+    return shape.IsArray() && TrueRank(shape) == 0;
   }
 
   // Returns whether "shape" is a scalar (array) with the given element_type.
@@ -457,31 +452,6 @@ class ShapeUtil {
   // that floating point numbers are signed.
   static bool ElementIsSigned(const Shape& shape);
 
-  // Returns whether the shape is a tuple.
-  ABSL_DEPRECATED("Use Shape::IsTuple instead.")
-  static bool IsTuple(const Shape& shape) {
-    return shape.element_type() == TUPLE;
-  }
-
-  // Returns whether the shape is an opaque value (i.e. an 'existential' typed
-  // value that is passed to CustomCall operations).
-  ABSL_DEPRECATED("Use Shape::IsOpaque instead.")
-  static bool IsOpaque(const Shape& shape) {
-    return shape.element_type() == OPAQUE;
-  }
-
-  // Returns whether the shape is an token value used for ordering
-  // side-effecting operations.
-  ABSL_DEPRECATED("Use Shape::IsToken instead.")
-  static bool IsToken(const Shape& shape) {
-    return shape.element_type() == TOKEN;
-  }
-
-  // Returns whether the shape is an array.  Note that scalars are considered
-  // arrays.
-  ABSL_DEPRECATED("Use Shape::IsArray instead.")
-  static bool IsArray(const Shape& shape);
-
   // Returns whether the given primitive type corresponds to an array shape.
   static bool IsArrayPrimitiveType(PrimitiveType primitive_type);
 
@@ -511,12 +481,6 @@ class ShapeUtil {
   // shape.
   static Shape ComplexComponentShape(const Shape& complex_shape);
 
-  // Shorthand for testing whether a shape is of a given element type and
-  // sequence of dimensions.
-  ABSL_DEPRECATED("Use Equal() instead.")
-  static bool ShapeIs(const Shape& shape, PrimitiveType element_type,
-                      std::initializer_list<int64> dimensions);
-
   // Returns true if the given shape has a subshape at the given index.
   static bool IndexIsValid(const Shape& shape, ShapeIndexView index);
 
@@ -764,7 +728,7 @@ class ShapeUtil {
     if (ShapeUtil::IsZeroElementArray(shape)) {
       return Status::OK();
     }
-    CHECK_EQ(Rank(shape), base.size());
+    CHECK_EQ(shape.rank(), base.size());
     CHECK_EQ(incr.size(), base.size());
     CHECK_EQ(count.size(), base.size());
     const int64 rank = LayoutUtil::MinorToMajor(shape).size();
diff --git a/tensorflow/compiler/xla/shape_util_test.cc b/tensorflow/compiler/xla/shape_util_test.cc
index 8e7c208193..61b4e73e06 100644
--- a/tensorflow/compiler/xla/shape_util_test.cc
+++ b/tensorflow/compiler/xla/shape_util_test.cc
@@ -538,10 +538,6 @@ TEST(ShapeUtilTest, InsertedOrDeleted1SizedDimensions) {
       ShapeUtil::InsertedOrDeleted1SizedDimensions(shape0, shape2)));
 }
 
-TEST(ShapeUtilTest, ShapeIs) {
-  EXPECT_FALSE(ShapeUtil::ShapeIs(ShapeUtil::MakeShape(PRED, {2}), PRED, {}));
-}
-
 TEST(ShapeUtilTest, ForEachIndex) {
   struct ShapeDimensionAndNumberInvocations {
     std::vector<int64> dimensions;
diff --git a/tensorflow/compiler/xla/tests/tuple_test.cc b/tensorflow/compiler/xla/tests/tuple_test.cc
index 9c586bdeb0..426d6c84ee 100644
--- a/tensorflow/compiler/xla/tests/tuple_test.cc
+++ b/tensorflow/compiler/xla/tests/tuple_test.cc
@@ -176,8 +176,9 @@ XLA_TEST_F(TupleTest, AddTupleElements) {
       {2.f, 4.f, 6.f},  // row 0
       {5.f, 7.f, 9.f},  // row 1
   });
-  ASSERT_TRUE(ShapeUtil::ShapeIs(vector_shape, F32, {3}));
-  ASSERT_TRUE(ShapeUtil::ShapeIs(matrix_shape, F32, {/*y=*/2, /*x=*/3}));
+  ASSERT_TRUE(ShapeUtil::Equal(vector_shape, ShapeUtil::MakeShape(F32, {3})));
+  ASSERT_TRUE(ShapeUtil::Equal(matrix_shape,
+                               ShapeUtil::MakeShape(F32, {/*y=*/2, /*x=*/3})));
   ComputeAndCompareR2<float>(&builder, expected, {}, error_spec_);
 }
 
-- 
GitLab


From 4053e17dbb7a1be85553e5173c2e08fb9653d33f Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Wed, 9 Jan 2019 12:33:36 -0800
Subject: [PATCH 0437/2345] Generate IDE projects automatically for various
 platforms

PiperOrigin-RevId: 228567285
---
 tensorflow/lite/experimental/micro/BUILD      |   4 +
 tensorflow/lite/experimental/micro/README.md  | 259 ++++++++++++++----
 .../experimental/micro/bluepill/debug_log.cc  |  27 ++
 .../lite/experimental/micro/debug_log.cc      |  41 +++
 .../lite/experimental/micro/debug_log.h       |  23 ++
 .../experimental/micro/debug_log_numbers.cc   | 185 +++++++++++++
 .../experimental/micro/debug_log_numbers.h    |  28 ++
 .../examples/micro_speech/CMSIS/Makefile.inc  |  43 +++
 .../micro/examples/micro_speech/Makefile.inc  | 183 +++++--------
 .../micro_speech/disco_f746ng/Makefile.inc    |   7 +
 .../disco_f746ng/audio_provider.cc            | 182 ++++++++++++
 .../micro_speech/disco_f746ng/timer.cc        |  24 ++
 .../examples/micro_speech/osx/Makefile.inc    |   6 +-
 .../examples/micro_speech/preprocessor.cc     |   9 +-
 .../lite/experimental/micro/mbed/debug_log.cc |  24 ++
 .../experimental/micro/micro_error_reporter.h |  22 +-
 .../micro/riscv32_mcu/debug_log.cc            | 172 ------------
 .../experimental/micro/tools/make/Makefile    | 101 ++++---
 .../micro/tools/make/helper_functions.inc     | 117 ++++++++
 .../tools/make/targets/bluepill_makefile.inc  |   5 +-
 .../tools/make/targets/mbed_makefile.inc      |   4 +
 .../micro/tools/make/targets/osx_makefile.inc |   2 +-
 .../make/templates/AUDIO_DISCO_F746NG.lib.tpl |   1 +
 .../make/templates/BSP_DISCO_F746NG.lib.tpl   |   1 +
 .../micro/tools/make/templates/Makefile.tpl   |  26 ++
 .../tools/make/templates/README_MAKE.md.tpl   |  29 ++
 .../tools/make/templates/README_MBED.md.tpl   |  48 ++++
 .../make/templates/SDRAM_DISCO_F746NG.lib.tpl |   1 +
 .../tools/make/templates/mbed-os.lib.tpl      |   1 +
 .../tools/make/templates/mbed_app.json.tpl    |   7 +
 30 files changed, 1170 insertions(+), 412 deletions(-)
 create mode 100644 tensorflow/lite/experimental/micro/bluepill/debug_log.cc
 create mode 100644 tensorflow/lite/experimental/micro/debug_log.cc
 create mode 100644 tensorflow/lite/experimental/micro/debug_log.h
 create mode 100644 tensorflow/lite/experimental/micro/debug_log_numbers.cc
 create mode 100644 tensorflow/lite/experimental/micro/debug_log_numbers.h
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/Makefile.inc
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/disco_f746ng/Makefile.inc
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/disco_f746ng/audio_provider.cc
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/disco_f746ng/timer.cc
 create mode 100644 tensorflow/lite/experimental/micro/mbed/debug_log.cc
 create mode 100644 tensorflow/lite/experimental/micro/tools/make/helper_functions.inc
 create mode 100644 tensorflow/lite/experimental/micro/tools/make/targets/mbed_makefile.inc
 create mode 100644 tensorflow/lite/experimental/micro/tools/make/templates/AUDIO_DISCO_F746NG.lib.tpl
 create mode 100644 tensorflow/lite/experimental/micro/tools/make/templates/BSP_DISCO_F746NG.lib.tpl
 create mode 100644 tensorflow/lite/experimental/micro/tools/make/templates/Makefile.tpl
 create mode 100644 tensorflow/lite/experimental/micro/tools/make/templates/README_MAKE.md.tpl
 create mode 100644 tensorflow/lite/experimental/micro/tools/make/templates/README_MBED.md.tpl
 create mode 100644 tensorflow/lite/experimental/micro/tools/make/templates/SDRAM_DISCO_F746NG.lib.tpl
 create mode 100644 tensorflow/lite/experimental/micro/tools/make/templates/mbed-os.lib.tpl
 create mode 100644 tensorflow/lite/experimental/micro/tools/make/templates/mbed_app.json.tpl

diff --git a/tensorflow/lite/experimental/micro/BUILD b/tensorflow/lite/experimental/micro/BUILD
index e11159868e..2d00ef76f4 100644
--- a/tensorflow/lite/experimental/micro/BUILD
+++ b/tensorflow/lite/experimental/micro/BUILD
@@ -12,6 +12,8 @@ load(
 cc_library(
     name = "micro_framework",
     srcs = [
+        "debug_log.cc",
+        "debug_log_numbers.cc",
         "micro_error_reporter.cc",
         "micro_interpreter.cc",
         "micro_mutable_op_resolver.cc",
@@ -19,6 +21,8 @@ cc_library(
     ],
     hdrs = [
         "compatibility.h",
+        "debug_log.h",
+        "debug_log_numbers.h",
         "micro_error_reporter.h",
         "micro_interpreter.h",
         "micro_mutable_op_resolver.h",
diff --git a/tensorflow/lite/experimental/micro/README.md b/tensorflow/lite/experimental/micro/README.md
index 464c7b6ad7..173f91fe90 100644
--- a/tensorflow/lite/experimental/micro/README.md
+++ b/tensorflow/lite/experimental/micro/README.md
@@ -1,46 +1,142 @@
 # TensorFlow Lite for Microcontrollers
 
-This an experimental port of TensorFlow Lite aimed at micro controllers and other devices with only kilobytes of memory. It doesn't require any operating system support, any standard C or C++ libraries, or dynamic memory allocation, so it's designed to be portable even to 'bare metal' systems. The core runtime fits in 16KB on a Cortex M3, and with enough operators to run a speech keyword detection model, takes up a total of 22KB.
+This an experimental port of TensorFlow Lite aimed at micro controllers and
+other devices with only kilobytes of memory. It doesn't require any operating
+system support, any standard C or C++ libraries, or dynamic memory allocation,
+so it's designed to be portable even to 'bare metal' systems. The core runtime
+fits in 16KB on a Cortex M3, and with enough operators to run a speech keyword
+detection model, takes up a total of 22KB.
 
 The design goals are for the framework to be:
 
-- **Readable**: We want embedded software engineers to be able to understand what's required to run ML inference without having to study research papers. We've tried to keep the code base small, modular, and have reference implementations of all operations to help with this.
-
-- **Easy to modify**: We know that there are a lot of different platforms and requirements in the embedded world, and we don't expect to cover all of them in one framework. Instead, we're hoping that it can be a good starting point for developers to build on top of to meet their own needs. For example, we tried to make it easy to replace the implementations of key computational operators that are often crucial for performance, without having to touch the data flow and other runtime code. We want it to make more sense to use our workflow to handle things like model import and less-important operations, and customize the parts that matter, rather than having to reimplement everything in your own engine.
-
-- **Well-tested**: If you're modifying code, you need to know if your changes are correct. Having an easy way to test lets you develop much faster. To help there, we've written tests for all the components, and we've made sure that the tests can be run on almost any platform, with no dependencies apart from the ability to log text to a debug console somewhere. We also provide an easy way to run all the tests on-device as part of an automated test framework, and we use qemu/Renode emulation so that tests can be run even without physical devices present.
-
-- **Easy to integrate**: We want to be as open a system as possible, and use the best code available for each platform. To do that, we're going to rely on projects like [CMSIS-NN](https://www.keil.com/pack/doc/CMSIS/NN/html/index.html), [uTensor](https://github.com/uTensor/uTensor), and other vendor libraries to handle as much performance-critical code as possible. We know that there are an increasing number of options to accelerate neural networks on microcontrollers, so we're aiming to be a good host for deploying those hardware technologies too.
-
-- **Compatible**: We're using the same file schema, interpreter API, and kernel interface as regular TensorFlow Lite, so we leverage the large existing set of tools, documentation, and examples for the project. The biggest barrier to deploying ML models is getting them from a training environment into a form that's easy to run inference on, so we see reusing this rich ecosystem as being crucial to being easily usable. We also hope to integrate this experimental work back into the main codebase in the future.
+-   **Readable**: We want embedded software engineers to be able to understand
+    what's required to run ML inference without having to study research papers.
+    We've tried to keep the code base small, modular, and have reference
+    implementations of all operations to help with this.
+
+-   **Easy to modify**: We know that there are a lot of different platforms and
+    requirements in the embedded world, and we don't expect to cover all of them
+    in one framework. Instead, we're hoping that it can be a good starting point
+    for developers to build on top of to meet their own needs. For example, we
+    tried to make it easy to replace the implementations of key computational
+    operators that are often crucial for performance, without having to touch
+    the data flow and other runtime code. We want it to make more sense to use
+    our workflow to handle things like model import and less-important
+    operations, and customize the parts that matter, rather than having to
+    reimplement everything in your own engine.
+
+-   **Well-tested**: If you're modifying code, you need to know if your changes
+    are correct. Having an easy way to test lets you develop much faster. To
+    help there, we've written tests for all the components, and we've made sure
+    that the tests can be run on almost any platform, with no dependencies apart
+    from the ability to log text to a debug console somewhere. We also provide
+    an easy way to run all the tests on-device as part of an automated test
+    framework, and we use qemu/Renode emulation so that tests can be run even
+    without physical devices present.
+
+-   **Easy to integrate**: We want to be as open a system as possible, and use
+    the best code available for each platform. To do that, we're going to rely
+    on projects like
+    [CMSIS-NN](https://www.keil.com/pack/doc/CMSIS/NN/html/index.html),
+    [uTensor](https://github.com/uTensor/uTensor), and other vendor libraries to
+    handle as much performance-critical code as possible. We know that there are
+    an increasing number of options to accelerate neural networks on
+    microcontrollers, so we're aiming to be a good host for deploying those
+    hardware technologies too.
+
+-   **Compatible**: We're using the same file schema, interpreter API, and
+    kernel interface as regular TensorFlow Lite, so we leverage the large
+    existing set of tools, documentation, and examples for the project. The
+    biggest barrier to deploying ML models is getting them from a training
+    environment into a form that's easy to run inference on, so we see reusing
+    this rich ecosystem as being crucial to being easily usable. We also hope to
+    integrate this experimental work back into the main codebase in the future.
 
 To meet those goals, we've made some tradeoffs:
 
-- **Simple C++**: To help with readability, our code is written in a modern version of C++, but we generally treat it as a "better C", rather relying on more complex features such as template meta-programming. As mentioned earlier, we avoid any use of dynamic memory allocation (new/delete) or the standard C/C++ libraries, so we believe this should still be fairly portable. It does mean that some older devices with C-only toolchains won't be supported, but we're hoping that the reference operator implementations (which are simple C-like functions) can still be useful in those cases. The interfaces are also designed to be C-only, so it should be possible to integrate the resulting library with pure C projects.
-
-- **Interpreted**: Code generation is a popular pattern for embedded code, because it gives standalone code that's easy to modify and step through, but we've chosen to go with an interpreted approach. In our internal microcontroller work we've found that using an extremely stripped-down interpreter with almost no dependencies gives us a lot of the same advantages, but is easier to maintain. For example, when new updates come out for the underlying library, you can just merge your local modifications in a single step, rather than having to regenerate new code and then patch in any changes you subsequently made. The coarse granularity of the interpreted primitives means that each operation call typically takes hundreds of thousands of instruction cycles at least, so we don't see noticeable performance gains from avoiding what's essentially a single switch statement at the interpreter level to call each operation. We're still working on improving the packaging though, for example we're considering having the ability to snapshot all the source files and headers used for a particular model, being able to compile the code and data together as a library, and then access it through a minimal set of C interface calls which hide the underlying complexity.
-
-- **Flatbuffers**: We represent our models using [the standard flatbuffer schema used by the rest of TensorFlow Lite](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/schema/schema.fbs), with the difference that we always keep it in read-only program memory (typically flash) rather than relying on having a file system to read it from. This is a good fit because flatbuffer's serialized format is designed to be mapped into memory without requiring any extra memory allocations or modifications to access it. All of the functions to read model values work directly on the serialized bytes, and large sections of data like weights are directly accessible as sequential C-style arrays of their data type, with no strides or unpacking needed. We do get a lot of value from using flatbuffers, but there is a cost in complexity. The flat buffer library code is all inline [inside the main headers](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/schema/schema_generated.h), but it isn't straightforward to inspect their implementations, and the model data structures aren't easy to comprehend from the debugger. The header for the schema itself also has to be periodically updated when new information is added to the file format, though we try to handle that transparently for most developers by checking in a pre-generated version.
-
-- **Code Duplication**: Some of the code in this prototype largely duplicates the logic in other parts of the TensorFlow Lite code base, for example the operator wrappers. We've tried to keep share as much as we can between the two interpreters, but there are some assumptions built into the original runtime that make this difficult. We'll be working on modularizing the main interpreter so that we can move to an entirely shared system.
-
-This initial preview release is designed to get early feedback, and is not intended to be a final product. It only includes enough operations to run a simple keyword recognition model, and the implementations are not optimized. We're hoping this will be a good way to get feedback and collaborate to improve the framework.
-
-## Getting Started
+-   **Simple C++**: To help with readability, our code is written in a modern
+    version of C++, but we generally treat it as a "better C", rather relying on
+    more complex features such as template meta-programming. As mentioned
+    earlier, we avoid any use of dynamic memory allocation (new/delete) or the
+    standard C/C++ libraries, so we believe this should still be fairly
+    portable. It does mean that some older devices with C-only toolchains won't
+    be supported, but we're hoping that the reference operator implementations
+    (which are simple C-like functions) can still be useful in those cases. The
+    interfaces are also designed to be C-only, so it should be possible to
+    integrate the resulting library with pure C projects.
+
+-   **Interpreted**: Code generation is a popular pattern for embedded code,
+    because it gives standalone code that's easy to modify and step through, but
+    we've chosen to go with an interpreted approach. In our internal
+    microcontroller work we've found that using an extremely stripped-down
+    interpreter with almost no dependencies gives us a lot of the same
+    advantages, but is easier to maintain. For example, when new updates come
+    out for the underlying library, you can just merge your local modifications
+    in a single step, rather than having to regenerate new code and then patch
+    in any changes you subsequently made. The coarse granularity of the
+    interpreted primitives means that each operation call typically takes
+    hundreds of thousands of instruction cycles at least, so we don't see
+    noticeable performance gains from avoiding what's essentially a single
+    switch statement at the interpreter level to call each operation. We're
+    still working on improving the packaging though, for example we're
+    considering having the ability to snapshot all the source files and headers
+    used for a particular model, being able to compile the code and data
+    together as a library, and then access it through a minimal set of C
+    interface calls which hide the underlying complexity.
+
+-   **Flatbuffers**: We represent our models using
+    [the standard flatbuffer schema used by the rest of TensorFlow Lite](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/schema/schema.fbs),
+    with the difference that we always keep it in read-only program memory
+    (typically flash) rather than relying on having a file system to read it
+    from. This is a good fit because flatbuffer's serialized format is designed
+    to be mapped into memory without requiring any extra memory allocations or
+    modifications to access it. All of the functions to read model values work
+    directly on the serialized bytes, and large sections of data like weights
+    are directly accessible as sequential C-style arrays of their data type,
+    with no strides or unpacking needed. We do get a lot of value from using
+    flatbuffers, but there is a cost in complexity. The flat buffer library code
+    is all inline
+    [inside the main headers](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/schema/schema_generated.h),
+    but it isn't straightforward to inspect their implementations, and the model
+    data structures aren't easy to comprehend from the debugger. The header for
+    the schema itself also has to be periodically updated when new information
+    is added to the file format, though we try to handle that transparently for
+    most developers by checking in a pre-generated version.
+
+-   **Code Duplication**: Some of the code in this prototype largely duplicates
+    the logic in other parts of the TensorFlow Lite code base, for example the
+    operator wrappers. We've tried to keep share as much as we can between the
+    two interpreters, but there are some assumptions built into the original
+    runtime that make this difficult. We'll be working on modularizing the main
+    interpreter so that we can move to an entirely shared system.
+
+This initial preview release is designed to get early feedback, and is not
+intended to be a final product. It only includes enough operations to run a
+simple keyword recognition model, and the implementations are not optimized.
+We're hoping this will be a good way to get feedback and collaborate to improve
+the framework.
+
+## Getting Started with Make
 
 Building requires a Linux or OS X machine.
 
- - Open a terminal
- - Download the TensorFlow source with `git clone https://github.com/tensorflow/tensorflow.git`
- - Enter the source root directory by running `cd tensorflow`
- - Download the dependencies by running `tensorflow/lite/experimental/micro/tools/make/download_dependencies.sh`. This may take a few minutes
- - Build and test the library with `make -f tensorflow/lite/experimental/micro/tools/make/Makefile test`
+-   Open a terminal
+-   Download the TensorFlow source with `git clone
+    https://github.com/tensorflow/tensorflow.git`
+-   Enter the source root directory by running `cd tensorflow`
+-   Download the dependencies by running
+    `tensorflow/lite/experimental/micro/tools/make/download_dependencies.sh`.
+    This may take a few minutes
+-   Build and test the library with `make -f
+    tensorflow/lite/experimental/micro/tools/make/Makefile test`
 
 You should see a series of compilation steps, followed by `~~~ALL TESTS
 PASSED~~~` for the various tests of the code that it will run. If there's an
 error, you should get an informative message from make about what went wrong.
 
-These tests are all built as simple binaries with few dependencies, so you can run them manually. For example, here's how to run the depthwise convolution test, and its output:
+These tests are all built as simple binaries with few dependencies, so you can
+run them manually. For example, here's how to run the depthwise convolution
+test, and its output:
 
 ```
 tensorflow/lite/experimental/micro/tools/make/gen/linux_x86_64/bin/tensorflow/lite/experimental/micro/kernels/depthwise_conv_test
@@ -53,7 +149,9 @@ Testing SimpleTestReluQuantized
 ~ALL TESTS PASSED~~~
 ```
 
-Looking at the [depthwise_conv_test.cc](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/experimental/micro/kernels/depthwise_conv_test.cc) code, you'll see a sequence that looks like this:
+Looking at the
+[depthwise_conv_test.cc](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/experimental/micro/kernels/depthwise_conv_test.cc)
+code, you'll see a sequence that looks like this:
 
 ```
 ...
@@ -74,19 +172,41 @@ output, and the test harness that runs the binary during the make process knows
 that everything ran correctly. If there's an error, the lack of the expected
 string lets the harness know that the test failed.
 
-So, why are we running tests in this complicated way? So far, we've been building binaries that run locally on the Mac OS or Linux machine you're building on, but this approach becomes important when we're targeting simple micro controller devices.
+So, why are we running tests in this complicated way? So far, we've been
+building binaries that run locally on the Mac OS or Linux machine you're
+building on, but this approach becomes important when we're targeting simple
+micro controller devices.
 
 ## Building for the "Blue Pill" STM32F103
 
-The goal of this library is to enable machine learning on resource-constrained micro controllers and DSPs, and as part of that we've targeted the ["Blue Pill" STM32F103-compatible development board](https://github.com/google/stm32_bare_lib) as a cheap and popular platform. It only has 20KB of RAM and 64KB of flash, so it's a good device to ensure we can run efficiently on small chips.
-
-It's fairly easy to [buy and wire up a physical board](https://github.com/google/stm32_bare_lib#wiring-up-your-blue-pill), but even if you don't have an actual device, the [Renode project](https://renode.io/) makes it easy to run a faithful emulation on your desktop machine. You'll need [Docker](https://www.docker.com/) installed, but once you have that set up, try running the following command:
-
-`make -f tensorflow/lite/experimental/micro/tools/make/Makefile TARGET=bluepill test`
-
-You should see a similar set of outputs as you did in the previous section, with the addition of some extra Docker logging messages. These are because we're using Docker to run the Renode micro controller emulation tool, and the tests themselves are being run on a simulated STM32F103 device. The communication channels between an embedded device and the host are quite limited, so the test harness looks at the output of the debug log to see if tests have passed, just as it did in the previous section. This makes it a very flexible way to run cross-platform tests, even when a platform has no operating system facilities, as long as it can output debugging text logs.
-
-To understand what's happening here, try running the same depthwise convolution test, but through the emulated device test harness, with the following command:
+The goal of this library is to enable machine learning on resource-constrained
+micro controllers and DSPs, and as part of that we've targeted the
+["Blue Pill" STM32F103-compatible development board](https://github.com/google/stm32_bare_lib)
+as a cheap and popular platform. It only has 20KB of RAM and 64KB of flash, so
+it's a good device to ensure we can run efficiently on small chips.
+
+It's fairly easy to
+[buy and wire up a physical board](https://github.com/google/stm32_bare_lib#wiring-up-your-blue-pill),
+but even if you don't have an actual device, the
+[Renode project](https://renode.io/) makes it easy to run a faithful emulation
+on your desktop machine. You'll need [Docker](https://www.docker.com/)
+installed, but once you have that set up, try running the following command:
+
+`make -f tensorflow/lite/experimental/micro/tools/make/Makefile TARGET=bluepill
+test`
+
+You should see a similar set of outputs as you did in the previous section, with
+the addition of some extra Docker logging messages. These are because we're
+using Docker to run the Renode micro controller emulation tool, and the tests
+themselves are being run on a simulated STM32F103 device. The communication
+channels between an embedded device and the host are quite limited, so the test
+harness looks at the output of the debug log to see if tests have passed, just
+as it did in the previous section. This makes it a very flexible way to run
+cross-platform tests, even when a platform has no operating system facilities,
+as long as it can output debugging text logs.
+
+To understand what's happening here, try running the same depthwise convolution
+test, but through the emulated device test harness, with the following command:
 
 ```
 tensorflow/lite/experimental/micro/testing/test_bluepill_binary.sh \
@@ -115,7 +235,7 @@ LOGS:
 03:27:32.4834 [DEBUG] cpu.uartSemihosting: [+0.18ms host +0s virt 0s virt from start]   Testing SimpleTestReluQuantized
 03:27:32.4838 [DEBUG] cpu.uartSemihosting: [+0.4ms host +0s virt 0s virt from start]   4/4 tests passed
 03:27:32.4839 [DEBUG] cpu.uartSemihosting: [+41µs host +0s virt 0s virt from start]   ~~~ALL TESTS PASSED~~~
-03:27:32.4839 [DEBUG] cpu.uartSemihosting: [+5µs host +0s virt 0s virt from start]   
+03:27:32.4839 [DEBUG] cpu.uartSemihosting: [+5µs host +0s virt 0s virt from start]
 ...
 tensorflow/lite/experimental/micro/tools/make/gen/bluepill_cortex-m3/bin/tensorflow/lite/experimental/micro/kernels/depthwise_conv_test: PASS
 ```
@@ -128,16 +248,19 @@ than your desktop. We hope that the simplicity of this testing approach will
 help make adding support for new platforms as easy as possible.
 
 ## Building for "Hifive1" SiFive FE310 development board
-We've targeted the ["HiFive1" Arduino-compatible development board](https://www.sifive.com/boards/hifive1) as a test platform for RISC-V MCU.
 
-Similar to Blue Pill setup, you will need Docker installed. The binary can be executed on either HiFive1 board or emulated using [Renode project](https://renode.io/) on your desktop machine.
+We've targeted the
+["HiFive1" Arduino-compatible development board](https://www.sifive.com/boards/hifive1)
+as a test platform for RISC-V MCU.
+
+Similar to Blue Pill setup, you will need Docker installed. The binary can be
+executed on either HiFive1 board or emulated using
+[Renode project](https://renode.io/) on your desktop machine.
 
 The following instructions builds and transfers the source files to the Docker
-```
-docker build -t riscv_build \
--f {PATH_TO_TENSORFLOW_ROOT_DIR}/tensorflow/lite/experimental/micro/testing/Dockerfile.riscv \
-{PATH_TO_TENSORFLOW_ROOT_DIR}/tensorflow/lite/experimental/micro/testing/
-```
+`docker build -t riscv_build \ -f
+{PATH_TO_TENSORFLOW_ROOT_DIR}/tensorflow/lite/experimental/micro/testing/Dockerfile.riscv
+\ {PATH_TO_TENSORFLOW_ROOT_DIR}/tensorflow/lite/experimental/micro/testing/`
 
 You should see output that looks something like this:
 
@@ -160,18 +283,25 @@ Successfully tagged riscv_build:latest
 
 Building micro_speech_test binary
 
- - Lauch the Docker that we just created using: `docker run -it-v /tmp/copybara_out:/workspace riscv_build:latest bash`
- - Enter the source root directory by running `cd /workspace`
- - Download the dependencies by running `./tensorflow/lite/experimental/micro/tools/make/download_dependencies.sh`. This may take a few minutes.
- - Set the path to RISC-V tools: `export PATH=${PATH}:/workspace/tensorflow/lite/experimental/micro/tools/make/downloads/riscv_toolchain/bin/`
- - Build the binary: `make -f tensorflow/lite/experimental/micro/tools/make/Makefile TARGET=riscv32_mcu`
+-   Lauch the Docker that we just created using: `docker run -it-v
+    /tmp/copybara_out:/workspace riscv_build:latest bash`
+-   Enter the source root directory by running `cd /workspace`
+-   Download the dependencies by running
+    `./tensorflow/lite/experimental/micro/tools/make/download_dependencies.sh`.
+    This may take a few minutes.
+-   Set the path to RISC-V tools: `export
+    PATH=${PATH}:/workspace/tensorflow/lite/experimental/micro/tools/make/downloads/riscv_toolchain/bin/`
+-   Build the binary: `make -f
+    tensorflow/lite/experimental/micro/tools/make/Makefile TARGET=riscv32_mcu`
 
 Lauching Renode to test the binary, currently this set up is not automated.
 
- - Until https://github.com/renode/renode/pull/30 is in the Docker image, patch the change manully: `sed -E -i 's/"rv32g"/"rv32imac"/g' /opt/renode/platforms/cpus/sifive-fe310.repl`
-
+-   Until https://github.com/renode/renode/pull/30 is in the Docker image, patch
+    the change manully: `sed -E -i 's/"rv32g"/"rv32imac"/g'
+    /opt/renode/platforms/cpus/sifive-fe310.repl`
 
- - Execute the binary on Renode: `renode -P 5000 --disable-xwt -e 's @/workspace/tensorflow/lite/experimental/micro/testing/sifive_fe310.resc'`
+-   Execute the binary on Renode: `renode -P 5000 --disable-xwt -e 's
+    @/workspace/tensorflow/lite/experimental/micro/testing/sifive_fe310.resc'`
 
 You should see the following log with the magic string `~~~ALL TEST PASSED~~~`:
 
@@ -238,3 +368,24 @@ JFlashLiteExe 2. Device = AMA3B1KK-KBR 3. Interface = SWD at 1000 kHz 4. Data
 file =
 tensorflow/lite/experimental/micro/tools/make/gen/apollo3evb_cortex-m4/bin/pushbutton_cmsis_speech_test.bin
 5. Prog Addr = 0x0000C000
+
+## Generating Project Files
+
+It's not always easy or convenient to use a makefile-based build process,
+especially if you're working on a product that uses a different IDE for the rest
+of its code. To address that, it's possible to generate standalone project
+folders for various popular build systems. These projects are self-contained,
+with only the headers and source files needed by a particular binary, and
+include project files to make loading them into an IDE easy. These can be
+auto-generated for any target you can compile using the main Make system, using
+a command like this (making sure you've run `download_dependencies.sh` first):
+
+```
+make -f tensorflow/lite/experimental/micro/tools/make/Makefile TARGET=mbed TAGS="CMSIS disco_f746ng" generate_micro_speech_main_test_mbed_project
+```
+
+This will create a folder in
+`tensorflow/lite/experimental/micro/tools/make/gen/mbed_cortex-m4/prj/micro_speech_main_test/mbed`
+that contains the source and header files, some Mbed configuration files, and a
+README. You should then be able to copy this directory to another machine, and
+use it just like any other Mbed project.
diff --git a/tensorflow/lite/experimental/micro/bluepill/debug_log.cc b/tensorflow/lite/experimental/micro/bluepill/debug_log.cc
new file mode 100644
index 0000000000..4812a91849
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/bluepill/debug_log.cc
@@ -0,0 +1,27 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/experimental/micro/debug_log.h"
+
+// For Arm Cortex-M devices, calling SYS_WRITE0 will output the zero-terminated
+// string pointed to by R1 to any debug console that's attached to the system.
+extern "C" void DebugLog(const char* s) {
+  asm("mov r0, #0x04\n"  // SYS_WRITE0
+      "mov r1, %[str]\n"
+      "bkpt #0xAB\n"
+      :
+      : [ str ] "r"(s)
+      : "r0", "r1");
+}
diff --git a/tensorflow/lite/experimental/micro/debug_log.cc b/tensorflow/lite/experimental/micro/debug_log.cc
new file mode 100644
index 0000000000..3d4ca44d76
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/debug_log.cc
@@ -0,0 +1,41 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Reference implementation of the DebugLog() function that's required for a
+// platform to support the TensorFlow Lite for Microcontrollers library. This is
+// the only function that's absolutely required to be available on a target
+// device, since it's used for communicating test results back to the host so
+// that we can verify the implementation is working correctly.
+// It's designed to be as easy as possible to supply an implementation though.
+// On platforms that have a POSIX stack or C library, it can be written as a
+// single call to `fprintf(stderr, "%s", s)` to output a string to the error
+// stream of the console, but if there's no OS or C library available, there's
+// almost always an equivalent way to write out a string to some serial
+// interface that can be used instead. For example on Arm M-series MCUs, calling
+// the `bkpt #0xAB` assembler instruction will output the string in r1 to
+// whatever debug serial connection is available. If you're running mbed, you
+// can do the same by creating `Serial pc(USBTX, USBRX)` and then calling
+// `pc.printf("%s", s)`.
+// To add an equivalent function for your own platform, create your own
+// implementation file, and place it in a subfolder with named after the OS
+// you're targeting. For example, see the Cortex M bare metal version in
+// tensorflow/lite/experimental/micro/bluepill/debug_log.cc or the mbed one on
+// tensorflow/lite/experimental/micro/mbed/debug_log.cc.
+
+#include "tensorflow/lite/experimental/micro/debug_log.h"
+
+#include <cstdio>
+
+extern "C" void DebugLog(const char* s) { fprintf(stderr, "%s", s); }
diff --git a/tensorflow/lite/experimental/micro/debug_log.h b/tensorflow/lite/experimental/micro/debug_log.h
new file mode 100644
index 0000000000..c0e395c376
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/debug_log.h
@@ -0,0 +1,23 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICRO_DEBUG_LOG_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICRO_DEBUG_LOG_H_
+
+// This function should be implemented by each target platform, and provide a
+// way for strings to be output to some text stream. For more information, see
+// tensorflow/lite/experimental/micro/debug_log.cc.
+extern "C" void DebugLog(const char* s);
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_DEBUG_LOG_H_
diff --git a/tensorflow/lite/experimental/micro/debug_log_numbers.cc b/tensorflow/lite/experimental/micro/debug_log_numbers.cc
new file mode 100644
index 0000000000..8e86730674
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/debug_log_numbers.cc
@@ -0,0 +1,185 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Implements debug logging for numbers by converting them into strings and then
+// calling the main DebugLog(char*) function. These are separated into a
+// different file so that platforms can just implement the string output version
+// of DebugLog() and then get the numerical variations without requiring any
+// more code.
+
+#include "tensorflow/lite/experimental/micro/debug_log_numbers.h"
+
+#include "tensorflow/lite/experimental/micro/debug_log.h"
+
+namespace {
+
+// All input buffers to the number conversion functions must be this long.
+static const int kFastToBufferSize = 48;
+
+// Reverses a zero-terminated string in-place.
+char* ReverseStringInPlace(char* start, char* end) {
+  char* p1 = start;
+  char* p2 = end - 1;
+  while (p1 < p2) {
+    char tmp = *p1;
+    *p1++ = *p2;
+    *p2-- = tmp;
+  }
+  return start;
+}
+
+// Appends a string to a string, in-place. You need to pass in the maximum
+// string length as the second argument.
+char* StrCatStr(char* main, int main_max_length, const char* to_append) {
+  char* current = main;
+  while (*current != 0) {
+    ++current;
+  }
+  char* current_end = main + (main_max_length - 1);
+  while ((*to_append != 0) && (current < current_end)) {
+    *current = *to_append;
+    ++current;
+    ++to_append;
+  }
+  *current = 0;
+  return current;
+}
+
+// Populates the provided buffer with an ASCII representation of the number.
+char* FastUInt32ToBufferLeft(uint32_t i, char* buffer, int base) {
+  char* start = buffer;
+  do {
+    int32_t digit = i % base;
+    char character;
+    if (digit < 10) {
+      character = '0' + digit;
+    } else {
+      character = 'a' + (digit - 10);
+    }
+    *buffer++ = character;
+    i /= base;
+  } while (i > 0);
+  *buffer = 0;
+  ReverseStringInPlace(start, buffer);
+  return buffer;
+}
+
+// Populates the provided buffer with an ASCII representation of the number.
+char* FastInt32ToBufferLeft(int32_t i, char* buffer) {
+  uint32_t u = i;
+  if (i < 0) {
+    *buffer++ = '-';
+    u = -u;
+  }
+  return FastUInt32ToBufferLeft(u, buffer, 10);
+}
+
+// Converts a number to a string and appends it to another.
+char* StrCatInt32(char* main, int main_max_length, int32_t number) {
+  char number_string[kFastToBufferSize];
+  FastInt32ToBufferLeft(number, number_string);
+  return StrCatStr(main, main_max_length, number_string);
+}
+
+// Converts a number to a string and appends it to another.
+char* StrCatUInt32(char* main, int main_max_length, uint32_t number, int base) {
+  char number_string[kFastToBufferSize];
+  FastUInt32ToBufferLeft(number, number_string, base);
+  return StrCatStr(main, main_max_length, number_string);
+}
+
+// Populates the provided buffer with ASCII representation of the float number.
+// Avoids the use of any floating point instructions (since these aren't
+// supported on many microcontrollers) and as a consequence prints values with
+// power-of-two exponents.
+char* FastFloatToBufferLeft(float f, char* buffer) {
+  char* current = buffer;
+  char* current_end = buffer + (kFastToBufferSize - 1);
+  // Access the bit fields of the floating point value to avoid requiring any
+  // float instructions. These constants are derived from IEEE 754.
+  const uint32_t sign_mask = 0x80000000;
+  const uint32_t exponent_mask = 0x7f800000;
+  const int32_t exponent_shift = 23;
+  const int32_t exponent_bias = 127;
+  const uint32_t fraction_mask = 0x007fffff;
+  const uint32_t u = *reinterpret_cast<uint32_t*>(&f);
+  const int32_t exponent =
+      ((u & exponent_mask) >> exponent_shift) - exponent_bias;
+  const uint32_t fraction = (u & fraction_mask);
+  // Expect ~0x2B1B9D3 for fraction.
+  if (u & sign_mask) {
+    *current = '-';
+    current += 1;
+  }
+  *current = 0;
+  // These are special cases for infinities and not-a-numbers.
+  if (exponent == 128) {
+    if (fraction == 0) {
+      current = StrCatStr(current, (current_end - current), "Inf");
+      return current;
+    } else {
+      current = StrCatStr(current, (current_end - current), "NaN");
+      return current;
+    }
+  }
+  // 0x007fffff (8388607) represents 0.99... for the fraction, so to print the
+  // correct decimal digits we need to scale our value before passing it to the
+  // conversion function. This scale should be 10000000/8388608 = 1.1920928955.
+  // We can approximate this using multiply-adds and right-shifts using the
+  // values in this array. The 1. portion of the number string is printed out
+  // in a fixed way before the fraction, below.
+  const int32_t scale_shifts_size = 13;
+  const int8_t scale_shifts[13] = {3,  4,  8,  11, 13, 14, 17,
+                                   18, 19, 20, 21, 22, 23};
+  uint32_t scaled_fraction = fraction;
+  for (int i = 0; i < scale_shifts_size; ++i) {
+    scaled_fraction += (fraction >> scale_shifts[i]);
+  }
+  *current = '1';
+  current += 1;
+  *current = '.';
+  current += 1;
+  *current = 0;
+  current = StrCatUInt32(current, (current_end - current), scaled_fraction, 10);
+  current = StrCatStr(current, (current_end - current), "*2^");
+  current = StrCatInt32(current, (current_end - current), exponent);
+  return current;
+}
+
+}  // namespace
+
+extern "C" void DebugLogInt32(int32_t i) {
+  char number_string[kFastToBufferSize];
+  FastInt32ToBufferLeft(i, number_string);
+  DebugLog(number_string);
+}
+
+extern "C" void DebugLogUInt32(uint32_t i) {
+  char number_string[kFastToBufferSize];
+  FastUInt32ToBufferLeft(i, number_string, 10);
+  DebugLog(number_string);
+}
+
+extern "C" void DebugLogHex(uint32_t i) {
+  char number_string[kFastToBufferSize];
+  FastUInt32ToBufferLeft(i, number_string, 16);
+  DebugLog(number_string);
+}
+
+extern "C" void DebugLogFloat(float i) {
+  char number_string[kFastToBufferSize];
+  FastFloatToBufferLeft(i, number_string);
+  DebugLog(number_string);
+}
diff --git a/tensorflow/lite/experimental/micro/debug_log_numbers.h b/tensorflow/lite/experimental/micro/debug_log_numbers.h
new file mode 100644
index 0000000000..d889e75173
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/debug_log_numbers.h
@@ -0,0 +1,28 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICRO_DEBUG_LOG_NUMBERS_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICRO_DEBUG_LOG_NUMBERS_H_
+
+#include <cstdint>
+
+// Output numbers to the debug logging stream.
+extern "C" {
+void DebugLogInt32(int32_t i);
+void DebugLogUInt32(uint32_t i);
+void DebugLogHex(uint32_t i);
+void DebugLogFloat(float i);
+}
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_DEBUG_LOG_NUMBERS_H_
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/Makefile.inc b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/Makefile.inc
new file mode 100644
index 0000000000..3d560510ad
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/Makefile.inc
@@ -0,0 +1,43 @@
+# Settings for targets that use the CMSIS library.
+ifneq ($(filter CMSIS,$(ALL_TAGS)),)
+  INCLUDES += \
+    -isystem$(MAKEFILE_DIR)/downloads/cmsis/CMSIS/Core/Include/ \
+    -isystem$(MAKEFILE_DIR)/downloads/cmsis/CMSIS/DSP/Include/ \
+    -I$(MAKEFILE_DIR)/downloads/CMSIS_ext/
+
+  CMSIS_PREPROCESSOR_SRCS := \
+    tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/hanning.cc \
+    tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/sin_1k.cc \
+
+  CMSIS_PREPROCESSOR_HDRS := \
+    tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/hanning.h \
+    tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/sin_1k.h \
+    third_party/CMSIS_ext/arm_cmplx_mag_squared_q10p6.h
+
+  PREPROCESSOR_TEST_SRCS += $(CMSIS_PREPROCESSOR_SRCS)
+  PREPROCESSOR_TEST_HDRS += $(CMSIS_PREPROCESSOR_HDRS)
+
+  FEATURE_PROVIDER_TEST_SRCS += $(CMSIS_PREPROCESSOR_SRCS)
+  FEATURE_PROVIDER_TEST_HDRS += $(CMSIS_PREPROCESSOR_HDRS)
+
+  MICRO_SPEECH_SRCS += $(CMSIS_PREPROCESSOR_SRCS)
+  MICRO_SPEECH_HDRS += $(CMSIS_PREPROCESSOR_HDRS)
+
+  THIRD_PARTY_CC_SRCS += \
+    third_party/CMSIS_ext/arm_cmplx_mag_squared_q10p6.c \
+    third_party/cmsis/CMSIS/DSP/Source/BasicMathFunctions/arm_mult_q15.c \
+    third_party/cmsis/CMSIS/DSP/Source/TransformFunctions/arm_rfft_init_q15.c \
+    third_party/cmsis/CMSIS/DSP/Source/TransformFunctions/arm_rfft_q15.c \
+    third_party/cmsis/CMSIS/DSP/Source/TransformFunctions/arm_cfft_q15.c \
+    third_party/cmsis/CMSIS/DSP/Source/TransformFunctions/arm_cfft_radix4_q15.c \
+    third_party/cmsis/CMSIS/DSP/Source/TransformFunctions/arm_bitreversal2.S \
+    third_party/cmsis/CMSIS/DSP/Source/CommonTables/arm_const_structs.c \
+    third_party/cmsis/CMSIS/DSP/Source/CommonTables/arm_common_tables.c \
+    third_party/cmsis/CMSIS/DSP/Source/StatisticsFunctions/arm_mean_q15.c \
+    third_party/cmsis/CMSIS/DSP/Source/StatisticsFunctions/arm_max_q7.c
+
+  THIRD_PARTY_CC_HDRS += \
+    third_party/cmsis/CMSIS/DSP/Include/arm_common_tables.h \
+    third_party/cmsis/CMSIS/DSP/Include/arm_const_structs.h
+
+endif
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/Makefile.inc b/tensorflow/lite/experimental/micro/examples/micro_speech/Makefile.inc
index 6f7563c02f..49aace3d7d 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/Makefile.inc
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/Makefile.inc
@@ -1,140 +1,62 @@
 
-# Tests loading and running a speech model.
 MICRO_SPEECH_TEST_SRCS := \
 tensorflow/lite/experimental/micro/examples/micro_speech/micro_speech_test.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/tiny_conv_model_data.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/no_features_data.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/yes_features_data.cc
-MICRO_SPEECH_TEST_SRCS := $(call specialize,$(MICRO_SPEECH_TEST_SRCS))
-ALL_SRCS += $(MICRO_SPEECH_TEST_SRCS)
-MICRO_SPEECH_TEST_OBJS := $(addprefix $(OBJDIR), \
-$(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(MICRO_SPEECH_TEST_SRCS))))
-MICRO_SPEECH_TEST_BINARY := $(BINDIR)micro_speech_test
-ALL_BINARIES += $(MICRO_SPEECH_TEST_BINARY)
-$(MICRO_SPEECH_TEST_BINARY): $(MICRO_SPEECH_TEST_OBJS) $(MICROLITE_LIB_PATH)
-	@mkdir -p $(dir $@)
-	$(CXX) $(CXXFLAGS) $(INCLUDES) \
-	-o $(MICRO_SPEECH_TEST_BINARY) $(MICRO_SPEECH_TEST_OBJS) \
-	$(LIBFLAGS) $(MICROLITE_LIB_PATH) $(LDFLAGS) $(MICROLITE_LIBS)
-micro_speech_test: $(MICRO_SPEECH_TEST_BINARY)
-micro_speech_test_bin: $(MICRO_SPEECH_TEST_BINARY).bin
-test_micro_speech: $(MICRO_SPEECH_TEST_BINARY)
-	$(TEST_SCRIPT) $(MICRO_SPEECH_TEST_BINARY) '~~~ALL TESTS PASSED~~~'
-
-# Source files that are used by multiple preprocessor tests.
-PREPROCESSOR_TEST_SHARED_SRCS := \
+
+MICRO_SPEECH_TEST_HDRS := \
+tensorflow/lite/experimental/micro/examples/micro_speech/tiny_conv_model_data.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/no_features_data.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/yes_features_data.h \
+
+PREPROCESSOR_TEST_SRCS := \
+tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor_test.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/no_30ms_sample_data.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/yes_30ms_sample_data.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/no_power_spectrum_data.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/yes_power_spectrum_data.cc
 
-# Test the float reference code for feature generation.
-PREPROCESSOR_REFERENCE_TEST_SRCS = \
-$(PREPROCESSOR_TEST_SHARED_SRCS) \
-tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc
-PREPROCESSOR_REFERENCE_TEST_SRCS := $(call specialize,$(PREPROCESSOR_REFERENCE_TEST_SRCS))
-ALL_SRCS += $(PREPROCESSOR_REFERENCE_TEST_SRCS)
-PREPROCESSOR_REFERENCE_TEST_OBJS := $(addprefix $(OBJDIR), \
-$(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(PREPROCESSOR_REFERENCE_TEST_SRCS))))
-PREPROCESSOR_REFERENCE_TEST_BINARY := $(BINDIR)preprocessor_reference_test
-ALL_BINARIES += $(PREPROCESSOR_REFERENCE_TEST_BINARY)
-$(PREPROCESSOR_REFERENCE_TEST_BINARY): $(PREPROCESSOR_REFERENCE_TEST_OBJS) $(MICROLITE_LIB_PATH)
-	@mkdir -p $(dir $@)
-	$(CXX) $(CXXFLAGS) $(INCLUDES) \
-	-o $(PREPROCESSOR_REFERENCE_TEST_BINARY) $(PREPROCESSOR_REFERENCE_TEST_OBJS) \
-	$(LIBFLAGS) $(MICROLITE_LIB_PATH) $(LDFLAGS) $(MICROLITE_LIBS)
-preprocessor_reference_test: $(PREPROCESSOR_REFERENCE_TEST_BINARY)
-preprocessor_reference_test_bin: $(PREPROCESSOR_REFERENCE_TEST_BINARY).bin
-test_preprocessor_reference: $(PREPROCESSOR_REFERENCE_TEST_BINARY)
-	$(TEST_SCRIPT) $(PREPROCESSOR_REFERENCE_TEST_BINARY) '~~~ALL TESTS PASSED~~~'
-
-# Test the fixed point reference code for feature generation.
-PREPROCESSOR_FIXED_TEST_SRCS = \
-$(PREPROCESSOR_TEST_SHARED_SRCS) \
-tensorflow/lite/experimental/micro/examples/micro_speech/fixed_point/preprocessor.cc
-PREPROCESSOR_FIXED_TEST_SRCS := $(call specialize,$(PREPROCESSOR_FIXED_TEST_SRCS))
-ALL_SRCS += $(PREPROCESSOR_FIXED_TEST_SRCS)
-PREPROCESSOR_FIXED_TEST_OBJS := $(addprefix $(OBJDIR), \
-$(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(PREPROCESSOR_FIXED_TEST_SRCS))))
-PREPROCESSOR_FIXED_TEST_BINARY := $(BINDIR)preprocessor_fixed_test
-ALL_BINARIES += $(PREPROCESSOR_FIXED_TEST_BINARY)
-$(PREPROCESSOR_FIXED_TEST_BINARY): $(PREPROCESSOR_FIXED_TEST_OBJS) $(MICROLITE_LIB_PATH)
-	@mkdir -p $(dir $@)
-	$(CXX) $(CXXFLAGS) $(INCLUDES) \
-	-o $(PREPROCESSOR_FIXED_TEST_BINARY) $(PREPROCESSOR_FIXED_TEST_OBJS) \
-	$(LIBFLAGS) $(MICROLITE_LIB_PATH) $(LDFLAGS) $(MICROLITE_LIBS)
-preprocessor_fixed_test: $(PREPROCESSOR_FIXED_TEST_BINARY)
-preprocessor_fixed_test_bin: $(PREPROCESSOR_FIXED_TEST_BINARY).bin
-test_preprocessor_fixed: $(PREPROCESSOR_FIXED_TEST_BINARY)
-	$(TEST_SCRIPT) $(PREPROCESSOR_FIXED_TEST_BINARY) '~~~ALL TESTS PASSED~~~'
+PREPROCESSOR_TEST_HDRS := \
+tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/no_30ms_sample_data.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/yes_30ms_sample_data.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/no_power_spectrum_data.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/yes_power_spectrum_data.h
 
-# Tests the audio provider module.
 AUDIO_PROVIDER_TEST_SRCS := \
 tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider_test.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc
-AUDIO_PROVIDER_TEST_SRCS := $(call specialize,$(AUDIO_PROVIDER_TEST_SRCS))
-ALL_SRCS += $(AUDIO_PROVIDER_TEST_SRCS)
-AUDIO_PROVIDER_TEST_OBJS := $(addprefix $(OBJDIR), \
-$(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(AUDIO_PROVIDER_TEST_SRCS))))
-AUDIO_PROVIDER_TEST_BINARY := $(BINDIR)audio_provider_test
-ALL_BINARIES += $(AUDIO_PROVIDER_TEST_BINARY)
-$(AUDIO_PROVIDER_TEST_BINARY): $(AUDIO_PROVIDER_TEST_OBJS) $(MICROLITE_LIB_PATH)
-	@mkdir -p $(dir $@)
-	$(CXX) $(CXXFLAGS) $(INCLUDES) \
-	-o $(AUDIO_PROVIDER_TEST_BINARY) $(AUDIO_PROVIDER_TEST_OBJS) \
-	$(LIBFLAGS) $(MICROLITE_LIB_PATH) $(LDFLAGS) $(MICROLITE_LIBS)
-audio_provider_test: $(AUDIO_PROVIDER_TEST_BINARY)
-audio_provider_test_bin: $(AUDIO_PROVIDER_TEST_BINARY).bin
-test_audio_provider: $(AUDIO_PROVIDER_TEST_BINARY)
-	$(TEST_SCRIPT) $(AUDIO_PROVIDER_TEST_BINARY) '~~~ALL TESTS PASSED~~~'
 
-# Tests the feature provider module.
+AUDIO_PROVIDER_TEST_HDRS := \
+tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h \
+
 FEATURE_PROVIDER_TEST_SRCS := \
 tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider_test.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.cc
-FEATURE_PROVIDER_TEST_SRCS := $(call specialize,$(FEATURE_PROVIDER_TEST_SRCS))
-ALL_SRCS += $(FEATURE_PROVIDER_TEST_SRCS)
-FEATURE_PROVIDER_TEST_OBJS := $(addprefix $(OBJDIR), \
-$(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(FEATURE_PROVIDER_TEST_SRCS))))
-FEATURE_PROVIDER_TEST_BINARY := $(BINDIR)feature_provider_test
-ALL_BINARIES += $(FEATURE_PROVIDER_TEST_BINARY)
-$(FEATURE_PROVIDER_TEST_BINARY): $(FEATURE_PROVIDER_TEST_OBJS) $(MICROLITE_LIB_PATH)
-	@mkdir -p $(dir $@)
-	$(CXX) $(CXXFLAGS) $(INCLUDES) \
-	-o $(FEATURE_PROVIDER_TEST_BINARY) $(FEATURE_PROVIDER_TEST_OBJS) \
-	$(LIBFLAGS) $(MICROLITE_LIB_PATH) $(LDFLAGS) $(MICROLITE_LIBS)
-feature_provider_test: $(FEATURE_PROVIDER_TEST_BINARY)
-feature_provider_test_bin: $(FEATURE_PROVIDER_TEST_BINARY).bin
-test_feature_provider: $(FEATURE_PROVIDER_TEST_BINARY)
-	$(TEST_SCRIPT) $(FEATURE_PROVIDER_TEST_BINARY) '~~~ALL TESTS PASSED~~~'
 
-# Tests the feature provider module.
+FEATURE_PROVIDER_TEST_HDRS := \
+tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h
+
 RECOGNIZE_COMMANDS_TEST_SRCS := \
 tensorflow/lite/experimental/micro/examples/micro_speech/recognize_commands_test.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/recognize_commands.cc
-RECOGNIZE_COMMANDS_TEST_SRCS := $(call specialize,$(RECOGNIZE_COMMANDS_TEST_SRCS))
-ALL_SRCS += $(RECOGNIZE_COMMANDS_TEST_SRCS)
-RECOGNIZE_COMMANDS_TEST_OBJS := $(addprefix $(OBJDIR), \
-$(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(RECOGNIZE_COMMANDS_TEST_SRCS))))
-RECOGNIZE_COMMANDS_TEST_BINARY := $(BINDIR)recognize_commands_test
-ALL_BINARIES += $(RECOGNIZE_COMMANDS_TEST_BINARY)
-$(RECOGNIZE_COMMANDS_TEST_BINARY): $(RECOGNIZE_COMMANDS_TEST_OBJS) $(MICROLITE_LIB_PATH)
-	@mkdir -p $(dir $@)
-	$(CXX) $(CXXFLAGS) $(INCLUDES) \
-	-o $(RECOGNIZE_COMMANDS_TEST_BINARY) $(RECOGNIZE_COMMANDS_TEST_OBJS) \
-	$(LIBFLAGS) $(MICROLITE_LIB_PATH) $(LDFLAGS) $(MICROLITE_LIBS)
-recognize_commands_test: $(RECOGNIZE_COMMANDS_TEST_BINARY)
-recognize_commands_test_bin: $(RECOGNIZE_COMMANDS_TEST_BINARY).bin
-test_recognize_commands: $(RECOGNIZE_COMMANDS_TEST_BINARY)
-	$(TEST_SCRIPT) $(RECOGNIZE_COMMANDS_TEST_BINARY) '~~~ALL TESTS PASSED~~~'
 
-# Builds a standalone speech command recognizer binary.
+RECOGNIZE_COMMANDS_TEST_HDRS := \
+tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/recognize_commands.h
+
 MICRO_SPEECH_SRCS := \
 tensorflow/lite/experimental/micro/examples/micro_speech/main.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.cc \
@@ -145,19 +67,40 @@ tensorflow/lite/experimental/micro/examples/micro_speech/no_features_data.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/yes_features_data.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/tiny_conv_model_data.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/recognize_commands.cc
-MICRO_SPEECH_SRCS := $(call specialize,$(MICRO_SPEECH_SRCS))
-ALL_SRCS += $(MICRO_SPEECH_SRCS)
-MICRO_SPEECH_OBJS := $(addprefix $(OBJDIR), \
-$(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(MICRO_SPEECH_SRCS))))
-MICRO_SPEECH_BINARY := $(BINDIR)micro_speech
-ALL_BINARIES += $(MICRO_SPEECH_BINARY)
-$(MICRO_SPEECH_BINARY): $(MICRO_SPEECH_OBJS) $(MICROLITE_LIB_PATH)
-	@mkdir -p $(dir $@)
-	$(CXX) $(CXXFLAGS) $(INCLUDES) \
-	-o $(MICRO_SPEECH_BINARY) $(MICRO_SPEECH_OBJS) \
-	$(LIBFLAGS) $(MICROLITE_LIB_PATH) $(LDFLAGS) $(MICROLITE_LIBS)
-micro_speech: $(MICRO_SPEECH_BINARY)
-micro_speech_bin: $(MICRO_SPEECH_BINARY).bin
+
+MICRO_SPEECH_HDRS := \
+tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/no_features_data.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/yes_features_data.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/tiny_conv_model_data.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/recognize_commands.h
 
 # Find any platform-specific rules for this example.
 include $(wildcard tensorflow/lite/experimental/micro/examples/micro_speech/*/Makefile.inc)
+
+# Tests loading and running a speech model.
+$(eval $(call microlite_test,micro_speech_test,\
+$(MICRO_SPEECH_TEST_SRCS),$(MICRO_SPEECH_TEST_HDRS)))
+
+# Test the code for feature generation.
+$(eval $(call microlite_test,preprocessor_test,\
+$(PREPROCESSOR_TEST_SRCS), $(PREPROCESSOR_TEST_HDRS)))
+
+# Tests the audio provider module.
+$(eval $(call microlite_test,audio_provider_test,\
+$(AUDIO_PROVIDER_TEST_SRCS),$(AUDIO_PROVIDER_TEST_HDRS)))
+
+# Tests the feature provider module.
+$(eval $(call microlite_test,feature_provider_test,\
+$(FEATURE_PROVIDER_TEST_SRCS),$(FEATURE_PROVIDER_TEST_HDRS)))
+
+# Tests the feature provider module.
+$(eval $(call microlite_test,recognize_commands_test,\
+$(RECOGNIZE_COMMANDS_TEST_SRCS),$(RECOGNIZE_COMMANDS_TEST_HDRS)))
+
+# Builds a standalone speech command recognizer binary.
+$(eval $(call microlite_test,micro_speech,\
+$(MICRO_SPEECH_SRCS),$(MICRO_SPEECH_HDRS)))
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/disco_f746ng/Makefile.inc b/tensorflow/lite/experimental/micro/examples/micro_speech/disco_f746ng/Makefile.inc
new file mode 100644
index 0000000000..5585ed7269
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/disco_f746ng/Makefile.inc
@@ -0,0 +1,7 @@
+# Settings for the Discovery STM32F746NG board.
+ifneq ($(filter disco_f746ng,$(ALL_TAGS)),)
+  MBED_PROJECT_FILES += \
+    AUDIO_DISCO_F746NG.lib \
+    BSP_DISCO_F746NG.lib \
+    SDRAM_DISCO_F746NG.lib
+endif
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/disco_f746ng/audio_provider.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/disco_f746ng/audio_provider.cc
new file mode 100644
index 0000000000..06647d0c53
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/disco_f746ng/audio_provider.cc
@@ -0,0 +1,182 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h"
+
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
+
+#include "AUDIO_DISCO_F746NG.h"
+#include "SDRAM_DISCO_F746NG.h"
+#include "mbed.h"  // NOLINT
+
+namespace {
+
+bool g_is_audio_initialized = false;
+constexpr int kAudioCaptureBufferSize = kAudioSampleFrequency * 0.5;
+int16_t g_audio_capture_buffer[kAudioCaptureBufferSize];
+int16_t g_audio_output_buffer[kMaxAudioSampleSize];
+int32_t g_latest_audio_timestamp = 0;
+
+// For a full example of how to access audio on the STM32F746NG board, see
+// https://os.mbed.com/teams/ST/code/DISCO-F746NG_AUDIO_demo/
+AUDIO_DISCO_F746NG g_audio_device;
+SDRAM_DISCO_F746NG g_sdram_device;
+
+typedef enum {
+  BUFFER_OFFSET_NONE = 0,
+  BUFFER_OFFSET_HALF = 1,
+  BUFFER_OFFSET_FULL = 2,
+} BUFFER_StateTypeDef;
+
+#define AUDIO_BLOCK_SIZE ((uint32_t)2048)
+#define AUDIO_BUFFER_IN SDRAM_DEVICE_ADDR /* In SDRAM */
+#define AUDIO_BUFFER_OUT \
+  (SDRAM_DEVICE_ADDR + (AUDIO_BLOCK_SIZE * 2)) /* In SDRAM */
+__IO uint32_t g_audio_rec_buffer_state = BUFFER_OFFSET_NONE;
+
+uint8_t SetSysClock_PLL_HSE_200MHz() {
+  RCC_ClkInitTypeDef RCC_ClkInitStruct;
+  RCC_OscInitTypeDef RCC_OscInitStruct;
+
+  // Enable power clock
+  __PWR_CLK_ENABLE();
+
+  // Enable HSE oscillator and activate PLL with HSE as source
+  RCC_OscInitStruct.OscillatorType = RCC_OSCILLATORTYPE_HSE;
+  RCC_OscInitStruct.HSEState = RCC_HSE_ON; /* External xtal on OSC_IN/OSC_OUT */
+
+  // Warning: this configuration is for a 25 MHz xtal clock only
+  RCC_OscInitStruct.PLL.PLLState = RCC_PLL_ON;
+  RCC_OscInitStruct.PLL.PLLSource = RCC_PLLSOURCE_HSE;
+  RCC_OscInitStruct.PLL.PLLM = 25;   // VCO input clock = 1 MHz (25 MHz / 25)
+  RCC_OscInitStruct.PLL.PLLN = 400;  // VCO output clock = 400 MHz (1 MHz * 400)
+  RCC_OscInitStruct.PLL.PLLP = RCC_PLLP_DIV2;  // PLLCLK = 200 MHz (400 MHz / 2)
+  RCC_OscInitStruct.PLL.PLLQ = 8;  // USB clock = 50 MHz (400 MHz / 8)
+
+  if (HAL_RCC_OscConfig(&RCC_OscInitStruct) != HAL_OK) {
+    return 0;  // FAIL
+  }
+
+  // Activate the OverDrive to reach the 216 MHz Frequency
+  if (HAL_PWREx_EnableOverDrive() != HAL_OK) {
+    return 0;  // FAIL
+  }
+
+  // Select PLL as system clock source and configure the HCLK, PCLK1 and PCLK2
+  // clocks dividers
+  RCC_ClkInitStruct.ClockType = (RCC_CLOCKTYPE_SYSCLK | RCC_CLOCKTYPE_HCLK |
+                                 RCC_CLOCKTYPE_PCLK1 | RCC_CLOCKTYPE_PCLK2);
+  RCC_ClkInitStruct.SYSCLKSource = RCC_SYSCLKSOURCE_PLLCLK;  // 200 MHz
+  RCC_ClkInitStruct.AHBCLKDivider = RCC_SYSCLK_DIV1;         // 200 MHz
+  RCC_ClkInitStruct.APB1CLKDivider = RCC_HCLK_DIV4;          //  50 MHz
+  RCC_ClkInitStruct.APB2CLKDivider = RCC_HCLK_DIV2;          // 100 MHz
+
+  if (HAL_RCC_ClockConfig(&RCC_ClkInitStruct, FLASH_LATENCY_7) != HAL_OK) {
+    return 0;  // FAIL
+  }
+  HAL_RCC_MCOConfig(RCC_MCO1, RCC_MCO1SOURCE_HSE, RCC_MCODIV_4);
+  return 1;  // OK
+}
+
+TfLiteStatus InitAudioRecording(tflite::ErrorReporter* error_reporter) {
+  SetSysClock_PLL_HSE_200MHz();
+
+  // Initialize SDRAM buffers.
+  memset((uint16_t*)AUDIO_BUFFER_IN, 0, AUDIO_BLOCK_SIZE * 2);
+  memset((uint16_t*)AUDIO_BUFFER_OUT, 0, AUDIO_BLOCK_SIZE * 2);
+  g_audio_rec_buffer_state = BUFFER_OFFSET_NONE;
+
+  // Start Recording.
+  g_audio_device.IN_Record((uint16_t*)AUDIO_BUFFER_IN, AUDIO_BLOCK_SIZE);
+
+  // Also play results out to headphone jack.
+  g_audio_device.OUT_SetAudioFrameSlot(CODEC_AUDIOFRAME_SLOT_02);
+  g_audio_device.OUT_Play((uint16_t*)AUDIO_BUFFER_OUT, AUDIO_BLOCK_SIZE * 2);
+
+  return kTfLiteOk;
+}
+
+void CaptureSamples(const int16_t* sample_data) {
+  const int sample_size = AUDIO_BLOCK_SIZE / (sizeof(int16_t) * 2);
+  const int32_t time_in_ms =
+      g_latest_audio_timestamp + (sample_size / (kAudioSampleFrequency / 1000));
+
+  const int32_t start_sample_offset =
+      g_latest_audio_timestamp * (kAudioSampleFrequency / 1000);
+  for (int i = 0; i < sample_size; ++i) {
+    const int capture_index =
+        (start_sample_offset + i) % kAudioCaptureBufferSize;
+    g_audio_capture_buffer[capture_index] =
+        (sample_data[(i * 2) + 0] / 2) + (sample_data[(i * 2) + 1] / 2);
+  }
+  // This is how we let the outside world know that new audio data has arrived.
+  g_latest_audio_timestamp = time_in_ms;
+}
+
+}  // namespace
+
+// These callbacks need to be linkable symbols, because they override weak
+// default versions.
+void BSP_AUDIO_IN_TransferComplete_CallBack(void) {
+  g_audio_rec_buffer_state = BUFFER_OFFSET_FULL;
+  /* Copy recorded 1st half block */
+  memcpy((uint16_t*)(AUDIO_BUFFER_OUT), (uint16_t*)(AUDIO_BUFFER_IN),
+         AUDIO_BLOCK_SIZE);
+  CaptureSamples(reinterpret_cast<int16_t*>(AUDIO_BUFFER_IN));
+  return;
+}
+
+// Another weak symbol override.
+void BSP_AUDIO_IN_HalfTransfer_CallBack(void) {
+  g_audio_rec_buffer_state = BUFFER_OFFSET_HALF;
+  /* Copy recorded 2nd half block */
+  memcpy((uint16_t*)(AUDIO_BUFFER_OUT + (AUDIO_BLOCK_SIZE)),
+         (uint16_t*)(AUDIO_BUFFER_IN + (AUDIO_BLOCK_SIZE)), AUDIO_BLOCK_SIZE);
+  CaptureSamples(
+      reinterpret_cast<int16_t*>(AUDIO_BUFFER_IN + AUDIO_BLOCK_SIZE));
+  return;
+}
+
+// Main entry point for getting audio data.
+TfLiteStatus GetAudioSamples(tflite::ErrorReporter* error_reporter,
+                             int start_ms, int duration_ms,
+                             int* audio_samples_size, int16_t** audio_samples) {
+  if (!g_is_audio_initialized) {
+    TfLiteStatus init_status = InitAudioRecording(error_reporter);
+    if (init_status != kTfLiteOk) {
+      return init_status;
+    }
+    g_is_audio_initialized = true;
+  }
+  // This should only be called when the main thread notices that the latest
+  // audio sample data timestamp has changed, so that there's new data in the
+  // capture ring buffer. The ring buffer will eventually wrap around and
+  // overwrite the data, but the assumption is that the main thread is checking
+  // often enough and the buffer is large enough that this call will be made
+  // before that happens.
+  const int start_offset = start_ms * (kAudioSampleFrequency / 1000);
+  const int duration_sample_count =
+      duration_ms * (kAudioSampleFrequency / 1000);
+  for (int i = 0; i < duration_sample_count; ++i) {
+    const int capture_index = (start_offset + i) % kAudioCaptureBufferSize;
+    g_audio_output_buffer[i] = g_audio_capture_buffer[capture_index];
+  }
+
+  *audio_samples_size = kMaxAudioSampleSize;
+  *audio_samples = g_audio_output_buffer;
+  return kTfLiteOk;
+}
+
+int32_t LatestAudioTimestamp() { return g_latest_audio_timestamp; }
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/disco_f746ng/timer.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/disco_f746ng/timer.cc
new file mode 100644
index 0000000000..a8f0fe4bd5
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/disco_f746ng/timer.cc
@@ -0,0 +1,24 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/timer.h"
+
+namespace {
+int32_t g_current_time = 0;
+}
+
+void SetTimeInMilliseconds(int32_t time) { g_current_time = time; }
+
+int32_t TimeInMilliseconds() { return g_current_time; }
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/osx/Makefile.inc b/tensorflow/lite/experimental/micro/examples/micro_speech/osx/Makefile.inc
index 89d107cfe0..8f8b33a9fa 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/osx/Makefile.inc
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/osx/Makefile.inc
@@ -1,6 +1,8 @@
-#Settings for Mac OS platforms.
+# Settings for Mac OS platforms.
 ifeq ($(TARGET), osx)
-  MICROLITE_LIBS += \
+  LINKER_FLAGS := \
     -framework Foundation \
     -framework AudioToolbox
+
+  MICROLITE_LIBS += $(LINKER_FLAGS)
 endif
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc
index 7b7db173ab..f8858aad72 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc
@@ -32,6 +32,9 @@ limitations under the License.
 
 namespace {
 
+// Needed because some platforms don't have M_PI defined.
+constexpr float kPi = 3.14159265358979323846f;
+
 // Performs a discrete Fourier transform on the real inputs. This corresponds to
 // rdft() in the FFT package at http://www.kurims.kyoto-u.ac.jp/~ooura/fft.html,
 // and to kiss_fftr() in KISSFFT at https://github.com/mborgerding/kissfft.
@@ -48,11 +51,11 @@ void CalculateDiscreteFourierTransform(float* time_series, int time_series_size,
   for (int i = 0; i < time_series_size / 2; ++i) {
     float real = 0;
     for (int j = 0; j < time_series_size; ++j) {
-      real += time_series[j] * cos(j * i * M_PI * 2 / time_series_size);
+      real += time_series[j] * cos(j * i * kPi * 2 / time_series_size);
     }
     float imaginary = 0;
     for (int j = 0; j < time_series_size; ++j) {
-      imaginary -= time_series[j] * sin(j * i * M_PI * 2 / time_series_size);
+      imaginary -= time_series[j] * sin(j * i * kPi * 2 / time_series_size);
     }
     fourier_output[(i * 2) + 0] = real;
     fourier_output[(i * 2) + 1] = imaginary;
@@ -63,7 +66,7 @@ void CalculateDiscreteFourierTransform(float* time_series, int time_series_size,
 // of the current sample window are weighted more heavily than those at the end.
 void CalculatePeriodicHann(int window_length, float* window_function) {
   for (int i = 0; i < window_length; ++i) {
-    window_function[i] = 0.5 - 0.5 * cos((2 * M_PI * i) / window_length);
+    window_function[i] = 0.5 - 0.5 * cos((2 * kPi * i) / window_length);
   }
 }
 
diff --git a/tensorflow/lite/experimental/micro/mbed/debug_log.cc b/tensorflow/lite/experimental/micro/mbed/debug_log.cc
new file mode 100644
index 0000000000..d4a4a5a842
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/mbed/debug_log.cc
@@ -0,0 +1,24 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/experimental/micro/debug_log.h"
+
+#include <mbed.h>
+
+// On mbed platforms, we set up a serial port and write to it for debug logging.
+extern "C" void DebugLog(const char* s) {
+  static Serial pc(USBTX, USBRX);
+  pc.printf("%s", s);
+}
diff --git a/tensorflow/lite/experimental/micro/micro_error_reporter.h b/tensorflow/lite/experimental/micro/micro_error_reporter.h
index 0ab853ec2a..6c18367c95 100644
--- a/tensorflow/lite/experimental/micro/micro_error_reporter.h
+++ b/tensorflow/lite/experimental/micro/micro_error_reporter.h
@@ -17,26 +17,8 @@ limitations under the License.
 
 #include "tensorflow/lite/core/api/error_reporter.h"
 #include "tensorflow/lite/experimental/micro/compatibility.h"
-
-#ifdef TF_LITE_MCU_DEBUG_LOG
-// These functions should be supplied by the micro target library
-extern "C" {
-#include <stdint.h>
-void DebugLog(const char* s);
-void DebugLogInt32(int32_t i);
-void DebugLogUInt32(uint32_t i);
-void DebugLogHex(uint32_t i);
-void DebugLogFloat(float i);
-}
-#else  // TF_LITE_MCU_DEBUG_LOG
-#include <cstdint>
-#include <cstdio>
-static void inline DebugLog(const char* s) { fprintf(stderr, "%s", s); }
-static void inline DebugLogInt32(int32_t i) { fprintf(stderr, "%d", i); }
-static void inline DebugLogUInt32(uint32_t i) { fprintf(stderr, "%d", i); }
-static void inline DebugLogHex(uint32_t i) { fprintf(stderr, "0x%8x", i); }
-static void inline DebugLogFloat(float i) { fprintf(stderr, "%f", i); }
-#endif  // TF_LITE_MCU_DEBUG_LOG
+#include "tensorflow/lite/experimental/micro/debug_log.h"
+#include "tensorflow/lite/experimental/micro/debug_log_numbers.h"
 
 namespace tflite {
 
diff --git a/tensorflow/lite/experimental/micro/riscv32_mcu/debug_log.cc b/tensorflow/lite/experimental/micro/riscv32_mcu/debug_log.cc
index f0065267bf..d1c2df866e 100644
--- a/tensorflow/lite/experimental/micro/riscv32_mcu/debug_log.cc
+++ b/tensorflow/lite/experimental/micro/riscv32_mcu/debug_log.cc
@@ -15,176 +15,4 @@ limitations under the License.
 
 #include <stdio.h>
 
-namespace {
-
-// All input buffers to the number conversion functions must be this long.
-static const int kFastToBufferSize = 48;
-
-// Reverses a zero-terminated string in-place.
-char* ReverseStringInPlace(char* start, char* end) {
-  char* p1 = start;
-  char* p2 = end - 1;
-  while (p1 < p2) {
-    char tmp = *p1;
-    *p1++ = *p2;
-    *p2-- = tmp;
-  }
-  return start;
-}
-
-// Appends a string to a string, in-place. You need to pass in the maximum
-// string length as the second argument.
-char* StrCatStr(char* main, int main_max_length, char* to_append) {
-  char* current = main;
-  while (*current != 0) {
-    ++current;
-  }
-  char* current_end = main + (main_max_length - 1);
-  while ((*to_append != 0) && (current < current_end)) {
-    *current = *to_append;
-    ++current;
-    ++to_append;
-  }
-  *current = 0;
-  return current;
-}
-
-char* StrCpy(char* main, int main_max_length, const char* source) {
-  char* current = main;
-  char* current_end = main + (main_max_length - 1);
-  while ((*source != 0) && (current < current_end)) {
-    *current = *source;
-    ++current;
-    ++source;
-  }
-  *current = 0;
-  return current;
-}
-
-// Populates the provided buffer with an ASCII representation of the number.
-char* FastUInt32ToBufferLeft(uint32_t i, char* buffer, int base) {
-  char* start = buffer;
-  do {
-    int32_t digit = i % base;
-    char character;
-    if (digit < 10) {
-      character = '0' + digit;
-    } else {
-      character = 'a' + (digit - 10);
-    }
-    *buffer++ = character;
-    i /= base;
-  } while (i > 0);
-  *buffer = 0;
-  ReverseStringInPlace(start, buffer);
-  return buffer;
-}
-
-// Populates the provided buffer with an ASCII representation of the number.
-char* FastInt32ToBufferLeft(int32_t i, char* buffer) {
-  uint32_t u = i;
-  if (i < 0) {
-    *buffer++ = '-';
-    u = -u;
-  }
-  return FastUInt32ToBufferLeft(u, buffer, 10);
-}
-
-// Converts a number to a string and appends it to another.
-char* StrCatInt32(char* main, int main_max_length, int32_t number) {
-  char number_string[kFastToBufferSize];
-  FastInt32ToBufferLeft(number, number_string);
-  return StrCatStr(main, main_max_length, number_string);
-}
-
-// Converts a number to a string and appends it to another.
-char* StrCatUInt32(char* main, int main_max_length, uint32_t number, int base) {
-  char number_string[kFastToBufferSize];
-  FastUInt32ToBufferLeft(number, number_string, base);
-  return StrCatStr(main, main_max_length, number_string);
-}
-
-// Populates the provided buffer with ASCII representation of the float number.
-// Avoids the use of any floating point instructions (since these aren't
-// supported on many microcontrollers) and as a consequence prints values with
-// power-of-two exponents.
-char* FastFloatToBufferLeft(float i, char* buffer) {
-  char* current = buffer;
-  char* current_end = buffer + (kFastToBufferSize - 1);
-  // Access the bit fields of the floating point value to avoid requiring any
-  // float instructions. These constants are derived from IEEE 754.
-  const uint32_t sign_mask = 0x80000000;
-  const uint32_t exponent_mask = 0x7f800000;
-  const int32_t exponent_shift = 23;
-  const int32_t exponent_bias = 127;
-  const uint32_t fraction_mask = 0x007fffff;
-  const uint32_t u = *(uint32_t*)(&i);
-  const int32_t exponent =
-      ((u & exponent_mask) >> exponent_shift) - exponent_bias;
-  const uint32_t fraction = (u & fraction_mask);
-  // Expect ~0x2B1B9D3 for fraction.
-  if (u & sign_mask) {
-    *current = '-';
-    current += 1;
-  }
-  *current = 0;
-  // These are special cases for infinities and not-a-numbers.
-  if (exponent == 128) {
-    if (fraction == 0) {
-      current = StrCatStr(current, (current_end - current), "Inf");
-      return current;
-    } else {
-      current = StrCatStr(current, (current_end - current), "NaN");
-      return current;
-    }
-  }
-  // 0x007fffff represents 0.99... for the fraction, so to print the correct
-  // decimal digits we need to scale our value before passing it to the
-  // conversion function. This scale should be 10000000/8388608 = 1.1920928955.
-  // We can approximate this using multipy-adds and right-shifts using the
-  // values in this array.
-  const int32_t scale_shifts_size = 13;
-  const int8_t scale_shifts[13] = {3,  4,  8,  11, 13, 14, 17,
-                                   18, 19, 20, 21, 22, 23};
-  uint32_t scaled_fraction = fraction;
-  for (int i = 0; i < scale_shifts_size; ++i) {
-    scaled_fraction += (fraction >> scale_shifts[i]);
-  }
-  *current = '1';
-  current += 1;
-  *current = '.';
-  current += 1;
-  *current = 0;
-  current = StrCatUInt32(current, (current_end - current), scaled_fraction, 10);
-  current = StrCatStr(current, (current_end - current), "*2^");
-  current = StrCatInt32(current, (current_end - current), exponent);
-  return current;
-}
-
-}  // namespace
-
 extern "C" void DebugLog(const char* s) { puts(s); }
-
-extern "C" void DebugLogInt32(int32_t i) {
-  char number_string[kFastToBufferSize];
-  FastInt32ToBufferLeft(i, number_string);
-  DebugLog(number_string);
-}
-
-extern "C" void DebugLogUInt32(uint32_t i) {
-  char number_string[kFastToBufferSize];
-  FastUInt32ToBufferLeft(i, number_string, 10);
-  DebugLog(number_string);
-}
-
-extern "C" void DebugLogHex(uint32_t i) {
-  char number_string[kFastToBufferSize];
-  FastUInt32ToBufferLeft(i, number_string, 16);
-  DebugLog(number_string);
-}
-
-extern "C" void DebugLogFloat(float i) {
-  char number_string[kFastToBufferSize];
-  FastFloatToBufferLeft(i, number_string);
-  DebugLog(number_string);
-}
diff --git a/tensorflow/lite/experimental/micro/tools/make/Makefile b/tensorflow/lite/experimental/micro/tools/make/Makefile
index f9a2dffae2..fde195118b 100644
--- a/tensorflow/lite/experimental/micro/tools/make/Makefile
+++ b/tensorflow/lite/experimental/micro/tools/make/Makefile
@@ -1,5 +1,9 @@
+
 MAKEFILE_DIR := tensorflow/lite/experimental/micro/tools/make
 
+# Pull in some convenience functions.
+include $(MAKEFILE_DIR)/helper_functions.inc
+
 # Try to figure out the host system
 HOST_OS :=
 ifeq ($(OS),Windows_NT)
@@ -21,37 +25,10 @@ HOST_ARCH := $(shell if [[ $(shell uname -m) =~ i[345678]86 ]]; then echo x86_32
 TARGET := $(HOST_OS)
 TARGET_ARCH := $(HOST_ARCH)
 
-# Look for platform or target-specific implementation files to replace reference
-# implementations with, given a tag. These are expected to occur in subfolders
-# of a directory where a reference implementation exists, and have the same
-# interface and header file. For example,
-# tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc
-# defines a module for supplying audio data, but since no platform or OS can be
-# presumed, it just always returns zeroes for its samples. The MacOS-specific
-# tensorflow/lite/experimental/micro/examples/micro_speech/osx/audio_provider.cc
-# has an implementation that relies on CoreAudio, and there are equivalent
-# versions for other operating systems. 
-# All lists of source files are put through this substitution process with the
-# tags of their target OS and architecture, so that implementations can be added
-# by simply placing them in the file tree, with no changes to the build files
-# needed.
-# One confusing thing about this implementation is that we're using wildcard to
-# act as a 'does file exist?' function, rather than expanding an expression.
-# Wildcard will return an empty string if given a plain file path with no actual
-# wildcards, if the file doesn't exist, so taking the first word of the list
-# between that and the reference path will pick the specialized one if it's
-# available.
-substitute_specialized_implementation = \
-  $(firstword $(wildcard $(dir $(1))$(2)/$(notdir $(1))) $(wildcard $(1)))
-substitute_specialized_implementations = \
-  $(foreach source,$(1),$(call substitute_specialized_implementation,$(source),$(2)))
-# Here we're first looking for specialized implementations in ref_dir/$(TARGET)
-# and then ref_dir/$(TARGET_ARCH), before falling back to ref_dir's
-# implementation.
-# The argument to this function should be a list of space-separated file paths,
-# with any wildcards already expanded.
-specialize = \
-  $(call substitute_specialized_implementations,$(call substitute_specialized_implementations,$(1),$(TARGET)),$(TARGET_ARCH))
+# Specify TAGS on the command line to add a particular set of specialized
+# implementations, for example TAGS="CMSIS disco_f746ng" to target a Discovery
+# STM32F746NG board, using the CMSIS library's implementations where possible.
+ALL_TAGS := $(TAGS) $(TARGET)
 
 INCLUDES := \
 -I. \
@@ -89,6 +66,9 @@ MICROLITE_TEST_SRCS := \
 $(wildcard tensorflow/lite/experimental/micro/*test.cc) \
 $(wildcard tensorflow/lite/experimental/micro/kernels/*test.cc)
 
+MICROLITE_TEST_HDRS := \
+$(wildcard tensorflow/lite/experimental/micro/testing/*.h)
+
 MICROLITE_CC_BASE_SRCS := \
 $(wildcard tensorflow/lite/experimental/micro/*.cc) \
 $(wildcard tensorflow/lite/experimental/micro/kernels/*.cc) \
@@ -101,12 +81,58 @@ tensorflow/lite/kernels/internal/quantization_util.cc
 MICROLITE_CC_SRCS := $(filter-out $(MICROLITE_TEST_SRCS), $(MICROLITE_CC_BASE_SRCS))
 MICROLITE_CC_SRCS := $(call specialize,$(MICROLITE_CC_SRCS))
 
+MICROLITE_CC_HDRS := \
+$(wildcard tensorflow/lite/experimental/micro/*.h) \
+$(wildcard tensorflow/lite/experimental/micro/kernels/*.h) \
+LICENSE \
+tensorflow/lite/c/c_api_internal.h \
+tensorflow/lite/c/builtin_op_data.h \
+tensorflow/lite/core/api/error_reporter.h \
+tensorflow/lite/core/api/flatbuffer_conversions.h \
+tensorflow/lite/core/api/op_resolver.h \
+tensorflow/lite/kernels/kernel_util.h \
+tensorflow/lite/kernels/op_macros.h \
+tensorflow/lite/kernels/padding.h \
+tensorflow/lite/kernels/internal/common.h \
+tensorflow/lite/kernels/internal/compatibility.h \
+tensorflow/lite/kernels/internal/reference/depthwiseconv_float.h \
+tensorflow/lite/kernels/internal/reference/depthwiseconv_uint8.h \
+tensorflow/lite/kernels/internal/reference/fully_connected.h \
+tensorflow/lite/kernels/internal/reference/softmax.h \
+tensorflow/lite/kernels/internal/round.h \
+tensorflow/lite/kernels/internal/tensor_ctypes.h \
+tensorflow/lite/kernels/internal/types.h \
+tensorflow/lite/kernels/internal/quantization_util.h \
+tensorflow/lite/schema/schema_generated.h \
+tensorflow/lite/version.h
+
+THIRD_PARTY_CC_HDRS := \
+third_party/gemmlowp/fixedpoint/fixedpoint.h \
+third_party/gemmlowp/fixedpoint/fixedpoint_sse.h \
+third_party/gemmlowp/internal/detect_platform.h \
+third_party/gemmlowp/LICENSE \
+third_party/flatbuffers/include/flatbuffers/base.h \
+third_party/flatbuffers/include/flatbuffers/stl_emulation.h \
+third_party/flatbuffers/include/flatbuffers/flatbuffers.h \
+third_party/flatbuffers/LICENSE.txt
+
+MAKE_PROJECT_FILES := \
+  README_MAKE.md \
+  Makefile
+
+MBED_PROJECT_FILES := \
+  README_MBED.md \
+  mbed-os.lib \
+  mbed_app.json
+
 # These target-specific makefiles should modify or replace options like
 # CXXFLAGS or LIBS to work for a specific targetted architecture. All logic
 # based on platforms or architectures should happen within these files, to
 # keep this main makefile focused on the sources and dependencies.
 include $(wildcard $(MAKEFILE_DIR)/targets/*_makefile.inc)
 
+ALL_TAGS += $(TARGET_ARCH)
+
 ALL_SRCS := \
 	$(MICROLITE_CC_SRCS) \
 	$(MICROLITE_TEST_SRCS)
@@ -116,6 +142,7 @@ GENDIR := $(MAKEFILE_DIR)/gen/$(TARGET)_$(TARGET_ARCH)/
 OBJDIR := $(GENDIR)obj/
 BINDIR := $(GENDIR)bin/
 LIBDIR := $(GENDIR)lib/
+PRJDIR := $(GENDIR)prj/
 
 MICROLITE_LIB_PATH := $(LIBDIR)$(MICROLITE_LIB_NAME)
 
@@ -142,7 +169,7 @@ $(OBJDIR)%.o: %.c
 	@mkdir -p $(dir $@)
 	$(CC) $(CCFLAGS) $(INCLUDES) -c $< -o $@
 
-  # For normal manually-created TensorFlow ASM source files.
+# For normal manually-created TensorFlow ASM source files.
 $(OBJDIR)%.o: %.S
 	@mkdir -p $(dir $@)
 	$(CC) $(CCFLAGS) $(INCLUDES) -c $< -o $@
@@ -170,13 +197,9 @@ $(BINDIR)%_test : $(OBJDIR)%_test.o $(MICROLITE_LIB_PATH)
 $(BINDIR)%.test_target: $(BINDIR)%_test
 	$(TEST_SCRIPT) $< '~~~ALL TESTS PASSED~~~'
 
-# snease: Add %.bin rule here since BINDIR is now defined
-# These are microcontroller-specific rules for converting the ELF output
-# of the linker into a binary image that can be loaded directly.
-OBJCOPY := $(TARGET_TOOLCHAIN_PREFIX)objcopy
-$(BINDIR)%.bin: $(BINDIR)%
-	@mkdir -p $(dir $@)
-	$(OBJCOPY) $< $@ -O binary
+# Generate standalone makefile projects for all of the test targets.
+$(foreach TEST_TARGET,$(MICROLITE_TEST_SRCS),\
+$(eval $(call microlite_test,$(notdir $(basename $(TEST_TARGET))),$(TEST_TARGET))))
 
 test: test_micro_speech $(MICROLITE_TEST_TARGETS)
 
diff --git a/tensorflow/lite/experimental/micro/tools/make/helper_functions.inc b/tensorflow/lite/experimental/micro/tools/make/helper_functions.inc
new file mode 100644
index 0000000000..87c002635f
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/tools/make/helper_functions.inc
@@ -0,0 +1,117 @@
+
+# Reverses a space-separated list of words.
+reverse = $(if $(1),$(call reverse,$(wordlist 2,$(words $(1)),$(1)))) $(firstword $(1))
+
+# Look for platform or target-specific implementation files to replace reference
+# implementations with, given a tag. These are expected to occur in subfolders
+# of a directory where a reference implementation exists, and have the same
+# interface and header file. For example,
+# tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc
+# defines a module for supplying audio data, but since no platform or OS can be
+# presumed, it just always returns zeroes for its samples. The MacOS-specific
+# tensorflow/lite/experimental/micro/examples/micro_speech/osx/audio_provider.cc
+# has an implementation that relies on CoreAudio, and there are equivalent
+# versions for other operating systems.
+# The specific implementation yielded by the first tag in the list that produces
+# a match is returned, else the reference version if none of the tags produce a
+# match.
+# All lists of source files are put through this substitution process with the
+# tags of their target OS and architecture, so that implementations can be added
+# by simply placing them in the file tree, with no changes to the build files
+# needed.
+# One confusing thing about this implementation is that we're using wildcard to
+# act as a 'does file exist?' function, rather than expanding an expression.
+# Wildcard will return an empty string if given a plain file path with no actual
+# wildcards, if the file doesn't exist, so taking the first word of the list
+# between that and the reference path will pick the specialized one if it's
+# available.
+substitute_specialized_implementation = \
+  $(firstword $(wildcard $(dir $(1))$(2)/$(notdir $(1))) $(wildcard $(1)))
+substitute_specialized_implementations = \
+  $(foreach source,$(1),$(call substitute_specialized_implementation,$(source),$(2)))
+# Here we're first looking for specialized implementations in ref_dir/$(TAG1)
+# and then ref_dir/$(TAG2), etc, before falling back to ref_dir's
+# implementation.
+# The argument to this function should be a list of space-separated file paths,
+# with any wildcards already expanded.
+define specialize_on_tags
+$(if $(2),$(call substitute_specialized_implementations,$(call specialize_on_tags,$(1),$(wordlist 2,$(words $(2)),$(2))),$(firstword $(2))),$(1))
+endef
+# The entry point that most targets should use to find implementation-specific
+# versions of their source files. The only argument is a list of file paths.
+specialize = $(call specialize_on_tags,$(1),$(strip $(call reverse,$(ALL_TAGS))))
+
+# Creates a set of rules to build a standalone makefile project for an
+# executable, including all of the source and header files required in a
+# separate folder and a simple makefile.
+# Arguments are:
+# 1 - Project type (make, mbed, etc).
+# 2 - Project file template name.
+# 3 - Name of executable.
+# 4 - List of C/C++ source files needed to build the target.
+# 5 - List of C/C++ header files needed to build the target.
+# 6 - Linker flags required.
+# 7 - C++ compilation flags needed.
+# Calling eval on the output will create a <Name>_makefile target that you
+# can invoke to create the standalone project.
+define generate_project
+$(PRJDIR)$(3)/$(1)/%: %
+	@mkdir -p $$(dir $$@)
+	cp $$< $$@
+
+$(PRJDIR)$(3)/$(1)/third_party/%: tensorflow/lite/experimental/micro/tools/make/downloads/%
+	@mkdir -p $$(dir $$@)
+	cp $$< $$@
+
+$(PRJDIR)$(3)/$(1)/%: tensorflow/lite/experimental/micro/tools/make/templates/%.tpl
+	@mkdir -p $$(dir $$@)
+	sed -E 's#\%\{SRCS\}\%#$(4)#g' $$< | \
+	sed -E 's#\%\{EXECUTABLE\}\%#$(3)#g' | \
+	sed -E 's#\%\{LINKER_FLAGS\}\%#$(6)#g' | \
+	sed -E 's#\%\{CXX_FLAGS\}\%#$(7)#g' > $$@
+
+generate_$(3)_$(1)_project: $(addprefix $(PRJDIR)$(3)/$(1)/, $(4) $(5) $(2))
+endef
+
+# Specialized version of generate_project for TF Lite Micro test targets that
+# automatically includes standard library files, so you just need to pass the
+# test name and any extra source files required.
+# Arguments are:
+# 1 - Name of test.
+# 2 - C/C++ source files implementing the test.
+# 3 - C/C++ header files needed for the test.
+# Calling eval on the output will create targets that you can invoke to
+# generate the standalone project.
+define generate_microlite_projects
+$(call generate_project,make,$(MAKE_PROJECT_FILES),$(1),$(MICROLITE_CC_SRCS) $(THIRD_PARTY_CC_SRCS) $(2),$(MICROLITE_CC_HDRS) $(THIRD_PARTY_CC_HDRS) $(MICROLITE_TEST_HDRS) $(3),$(MICROLITE_LIBS),$(CXXFLAGS))
+$(call generate_project,mbed,$(MBED_PROJECT_FILES),$(1),$(MICROLITE_CC_SRCS) $(THIRD_PARTY_CC_SRCS) $(2),$(MICROLITE_CC_HDRS) $(THIRD_PARTY_CC_HDRS) $(MICROLITE_TEST_HDRS) $(3),$(MICROLITE_LIBS),$(CXXFLAGS))
+endef
+
+
+# Handles the details of generating a binary target, including specializing
+# for the current platform, and generating project file targets.
+# Arguments are:
+# 1 - Name of test.
+# 2 - C/C++ source files implementing the test.
+# 3 - C/C++ header files needed for the test.
+# Calling eval on the output will create the targets that you need.
+define microlite_test
+$(1)_LOCAL_SRCS := $(2)
+$(1)_LOCAL_SRCS := $$(call specialize,$$($(1)_LOCAL_SRCS))
+ALL_SRCS += $$($(1)_LOCAL_SRCS)
+$(1)_LOCAL_HDRS := $(3)
+$(1)_LOCAL_OBJS := $$(addprefix $$(OBJDIR), \
+$$(patsubst %.cc,%.o,$$(patsubst %.c,%.o,$$($(1)_LOCAL_SRCS))))
+$(1)_BINARY := $$(BINDIR)$(1)
+ALL_BINARIES += $$($(1)_BINARY)
+$$($(1)_BINARY): $$($(1)_LOCAL_OBJS) $$(MICROLITE_LIB_PATH)
+	@mkdir -p $$(dir $$@)
+	$$(CXX) $$(CXXFLAGS) $$(INCLUDES) \
+	-o $$($(1)_BINARY) $$($(1)_LOCAL_OBJS) \
+	$$(LIBFLAGS) $$(MICROLITE_LIB_PATH) $$(LDFLAGS) $$(MICROLITE_LIBS)
+$(1): $$($(1)_BINARY)
+$(1)_bin: $$($(1)_BINARY).bin
+test_$(1): $$($(1)_BINARY)
+	$$(TEST_SCRIPT) $$($(1)_BINARY) '~~~ALL TESTS PASSED~~~'
+$(eval $(call generate_microlite_projects,$(1),$(call specialize,$(2)),$(3)))
+endef
diff --git a/tensorflow/lite/experimental/micro/tools/make/targets/bluepill_makefile.inc b/tensorflow/lite/experimental/micro/tools/make/targets/bluepill_makefile.inc
index 5e3105a109..b344f844bc 100644
--- a/tensorflow/lite/experimental/micro/tools/make/targets/bluepill_makefile.inc
+++ b/tensorflow/lite/experimental/micro/tools/make/targets/bluepill_makefile.inc
@@ -47,7 +47,10 @@ ifeq ($(TARGET), bluepill)
   MICROLITE_CC_SRCS += \
     $(wildcard $(MAKEFILE_DIR)/downloads/stm32_bare_lib/source/*.c) \
     $(wildcard $(MAKEFILE_DIR)/downloads/stm32_bare_lib/source/*.cc)
-    TEST_SCRIPT := tensorflow/lite/experimental/micro/testing/test_bluepill_binary.sh
+  EXCLUDED_SRCS := \
+    $(MAKEFILE_DIR)/downloads/stm32_bare_lib/source/debug_log.c
+  MICROLITE_CC_SRCS := $(filter-out $(EXCLUDED_SRCS), $(MICROLITE_CC_SRCS))
+  TEST_SCRIPT := tensorflow/lite/experimental/micro/testing/test_bluepill_binary.sh
   # These are tests that don't currently work on the blue pill.
   EXCLUDED_TESTS := \
     tensorflow/lite/experimental/micro/micro_interpreter_test.cc \
diff --git a/tensorflow/lite/experimental/micro/tools/make/targets/mbed_makefile.inc b/tensorflow/lite/experimental/micro/tools/make/targets/mbed_makefile.inc
new file mode 100644
index 0000000000..161ff34cdb
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/tools/make/targets/mbed_makefile.inc
@@ -0,0 +1,4 @@
+# Settings for mbed platforms.
+ifeq ($(TARGET), mbed)
+  TARGET_ARCH := cortex-m4
+endif
\ No newline at end of file
diff --git a/tensorflow/lite/experimental/micro/tools/make/targets/osx_makefile.inc b/tensorflow/lite/experimental/micro/tools/make/targets/osx_makefile.inc
index 0e8aad1993..3b91eeff9f 100644
--- a/tensorflow/lite/experimental/micro/tools/make/targets/osx_makefile.inc
+++ b/tensorflow/lite/experimental/micro/tools/make/targets/osx_makefile.inc
@@ -1,4 +1,4 @@
-#Settings for Mac OS platforms.
+# Settings for Mac OS platforms.
 ifeq ($(TARGET), osx)
 
   PLATFORM_FLAGS = \
diff --git a/tensorflow/lite/experimental/micro/tools/make/templates/AUDIO_DISCO_F746NG.lib.tpl b/tensorflow/lite/experimental/micro/tools/make/templates/AUDIO_DISCO_F746NG.lib.tpl
new file mode 100644
index 0000000000..11dae1ea16
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/tools/make/templates/AUDIO_DISCO_F746NG.lib.tpl
@@ -0,0 +1 @@
+https://os.mbed.com/teams/ST/code/AUDIO_DISCO_F746NG/#7046ce26b7ed
diff --git a/tensorflow/lite/experimental/micro/tools/make/templates/BSP_DISCO_F746NG.lib.tpl b/tensorflow/lite/experimental/micro/tools/make/templates/BSP_DISCO_F746NG.lib.tpl
new file mode 100644
index 0000000000..48dc131707
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/tools/make/templates/BSP_DISCO_F746NG.lib.tpl
@@ -0,0 +1 @@
+https://os.mbed.com/teams/ST/code/BSP_DISCO_F746NG/#df2ea349c37a
diff --git a/tensorflow/lite/experimental/micro/tools/make/templates/Makefile.tpl b/tensorflow/lite/experimental/micro/tools/make/templates/Makefile.tpl
new file mode 100644
index 0000000000..74d54f1ebe
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/tools/make/templates/Makefile.tpl
@@ -0,0 +1,26 @@
+SRCS := \
+%{SRCS}%
+
+OBJS := \
+$(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(SRCS)))
+
+INCLUDES := \
+-I. \
+-I./third_party/gemmlowp \
+-I./third_party/flatbuffers/include
+
+CXXFLAGS += %{CXX_FLAGS}%
+
+LDFLAGS += %{LINKER_FLAGS}%
+
+%.o: %.cc
+	$(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@
+
+%.o: %.c
+	$(CC) $(CCFLAGS) $(INCLUDES) -c $< -o $@
+
+%{EXECUTABLE}% : $(OBJS)
+	$(CXX) $(LDFLAGS) $(OBJS) \
+	-o $@
+
+all: %{EXECUTABLE}%
diff --git a/tensorflow/lite/experimental/micro/tools/make/templates/README_MAKE.md.tpl b/tensorflow/lite/experimental/micro/tools/make/templates/README_MAKE.md.tpl
new file mode 100644
index 0000000000..7906a3226a
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/tools/make/templates/README_MAKE.md.tpl
@@ -0,0 +1,29 @@
+# TensorFlow Lite Micro Make Project
+
+This folder has been autogenerated by TensorFlow, and contains source, header,
+and project files needed to build a single TensorFlow Lite Micro target using
+the make tool.
+
+## Usage
+
+To build this, run:
+
+```
+make
+```
+
+This should attempt to build the target locally on your platform, using the
+standard Makefile variables like CFLAGS, CC, CXX, and so on.
+
+## Project Generation
+
+See
+[tensorflow/lite/experimental/micro](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/experimental/micro)
+for details on how projects like this can be generated from the main source
+tree.
+
+## License
+
+TensorFlow's code is covered by the Apache2 License included in the repository,
+and third party dependencies are covered by their respective licenses, in the
+third_party folder of this package.
diff --git a/tensorflow/lite/experimental/micro/tools/make/templates/README_MBED.md.tpl b/tensorflow/lite/experimental/micro/tools/make/templates/README_MBED.md.tpl
new file mode 100644
index 0000000000..2682236edf
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/tools/make/templates/README_MBED.md.tpl
@@ -0,0 +1,48 @@
+# TensorFlow Lite Micro Mbed Project
+
+This folder has been autogenerated by TensorFlow, and contains source, header,
+and project files needed to build a single TensorFlow Lite Micro target using
+the Mbed command line interface.
+
+## Usage
+
+To load the dependencies this code requires, run:
+
+```
+mbed config root .
+mbed deploy
+```
+
+TensorFlow requires C++ 11, so you'll need to update your profiles to reflect
+this. Here's a short Python command that does that:
+
+```
+python -c 'import fileinput, glob;
+for filename in glob.glob("mbed-os/tools/profiles/*.json"):
+  for line in fileinput.input(filename, inplace=True):
+    print line.replace("\"-std=gnu++98\"","\"-std=c++11\", \"-fpermissive\"")'
+```
+
+With that setting updated, you should now be able to compile:
+
+```
+mbed compile -m auto -t GCC_ARM
+```
+
+If this works, it will give you a .bin file that you can flash onto the device
+you're targeting. For example, using a Discovery STM3246G board, you can deploy
+it by copying the bin to the volume mounted as a USB drive, just by dragging
+over the file.
+
+## Project Generation
+
+See
+[tensorflow/lite/experimental/micro](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/experimental/micro)
+for details on how projects like this can be generated from the main source
+tree.
+
+## License
+
+TensorFlow's code is covered by the Apache2 License included in the repository,
+and third party dependencies are covered by their respective licenses, in the
+third_party folder of this package.
diff --git a/tensorflow/lite/experimental/micro/tools/make/templates/SDRAM_DISCO_F746NG.lib.tpl b/tensorflow/lite/experimental/micro/tools/make/templates/SDRAM_DISCO_F746NG.lib.tpl
new file mode 100644
index 0000000000..e2ccd7b81b
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/tools/make/templates/SDRAM_DISCO_F746NG.lib.tpl
@@ -0,0 +1 @@
+https://os.mbed.com/teams/ST/code/SDRAM_DISCO_F746NG/#370f402a2219
diff --git a/tensorflow/lite/experimental/micro/tools/make/templates/mbed-os.lib.tpl b/tensorflow/lite/experimental/micro/tools/make/templates/mbed-os.lib.tpl
new file mode 100644
index 0000000000..69fff22f33
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/tools/make/templates/mbed-os.lib.tpl
@@ -0,0 +1 @@
+https://github.com/ARMmbed/mbed-os/#6a0a86538c0b9b2bfcc4583b1e2b7fea8f4e71e9
diff --git a/tensorflow/lite/experimental/micro/tools/make/templates/mbed_app.json.tpl b/tensorflow/lite/experimental/micro/tools/make/templates/mbed_app.json.tpl
new file mode 100644
index 0000000000..1c547369fb
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/tools/make/templates/mbed_app.json.tpl
@@ -0,0 +1,7 @@
+{
+    "config": {
+	"main-stack-size": {
+            "value": 65536
+	}
+    }
+}
-- 
GitLab


From 8fefcf7b8b236baf94674ca8fcb0777fd48c2ccd Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Wed, 9 Jan 2019 12:42:37 -0800
Subject: [PATCH 0438/2345] Roll-forward: PR #24081: Split convolution
 invocation into preparation and actual invocation

Please approve this CL. It will be submitted automatically, and its GitHub pull request will be marked as merged.

Imported from GitHub PR #24081

- split DoConvolve into:

PrepareForConvolution
DoConvolve

- split DoConvolveBackwardData into:

PrepareForConvolutionBackwardData
DoConvolveBackwardData

- split DoConvolveBackwardFilter into:

PrepareForConvolutionBackwardFilter
DoConvolveBackwardFilter

PrepareForConvolutionXXX would allocate scratch memory.
DoConolveXXX would invoke actual convolution algorithms.

Implement forward convoution, backward input convolution, backward filter
convolution on CUDA path.

Copybara import of the project:

  - fe9a9dbb8d0ef118b15beb4724f256190fd04d13 Split convolution invocation into preparation and actual ... by Wen-Heng (Jack) Chung <whchung@gmail.com>
  - baeda0d8b5da58a97d28ddce264c80c58d937699 Merge fe9a9dbb8d0ef118b15beb4724f256190fd04d13 into bbe33... by Wen-Heng (Jack) Chung <whchung@gmail.com>

PiperOrigin-RevId: 228568812
---
 tensorflow/stream_executor/cuda/cuda_dnn.cc | 471 +++++++++++++++-----
 tensorflow/stream_executor/cuda/cuda_dnn.h  | 194 +++++++-
 tensorflow/stream_executor/dnn.h            | 207 +++++++--
 tensorflow/stream_executor/stream.cc        | 172 +++++--
 4 files changed, 859 insertions(+), 185 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index a34aa9354d..0bd953fe3b 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -2799,7 +2799,7 @@ void LogCudaProto(const dnn::ConvolutionProto& conv, float profile_time_ms,
 }  // namespace
 
 template <class T>
-port::Status CudnnSupport::DoConvolveImpl(
+port::Status CudnnSupport::PrepareForConvolutionImpl(
     Stream* stream, const dnn::BatchDescriptor& input_descriptor,
     const DeviceMemory<T>& input_data,
     const dnn::FilterDescriptor& filter_descriptor,
@@ -2808,6 +2808,34 @@ port::Status CudnnSupport::DoConvolveImpl(
     const dnn::BatchDescriptor& output_descriptor, DeviceMemory<T>* output_data,
     dnn::DataType accumulator_type, ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+  cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
+  CudnnTensorDescriptor input_nd(input_descriptor, cudnn_type);
+  CudnnTensorDescriptor output_nd(output_descriptor, cudnn_type);
+  CudnnFilterDescriptor filter(filter_descriptor, cudnn_type);
+  CudnnConvolutionDescriptor conv(convolution_descriptor,
+                                  ToCudnnDataType(accumulator_type));
+
+  auto cudnn = cudnn_->GetHandle(parent_, stream);
+
+  SE_ASSIGN_OR_RETURN(*algorithm_desc,
+                      GetCudnnConvolutionForwardAlgorithm(
+                          stream, cudnn, algorithm_config, input_nd, filter,
+                          conv, output_nd, scratch_allocator, scratch_memory));
+
+  return port::Status::OK();
+}
+
+template <class T>
+port::Status CudnnSupport::DoConvolveImpl(
+    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+    const DeviceMemory<T>& input_data,
+    const dnn::FilterDescriptor& filter_descriptor,
+    const DeviceMemory<T>& filter_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::BatchDescriptor& output_descriptor, DeviceMemory<T>* output_data,
+    dnn::DataType accumulator_type, const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
     dnn::ProfileResult* output_profile_result) {
   cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
   CudnnTensorDescriptor input_nd(input_descriptor, cudnn_type);
@@ -2830,12 +2858,6 @@ port::Status CudnnSupport::DoConvolveImpl(
 
   const bool is_profiling = output_profile_result != nullptr;
 
-  DeviceMemory<uint8> scratch;
-  SE_ASSIGN_OR_RETURN(dnn::AlgorithmDesc algo_desc,
-                      GetCudnnConvolutionForwardAlgorithm(
-                          stream, cudnn, algorithm_config, input_nd, filter,
-                          conv, output_nd, scratch_allocator, &scratch));
-
   std::unique_ptr<CUDATimer, TimerDeleter> timer;
   if (is_profiling) {
     timer.reset(new CUDATimer(parent_));  // NOLINT
@@ -2851,7 +2873,7 @@ port::Status CudnnSupport::DoConvolveImpl(
   // memory. See nvbugs/2138754, b/80018418.
   if (CUDNN_VERSION < 7300) {
     SE_RETURN_IF_ERROR([&] {
-      if (algo_desc.algo_id() != CUDNN_CONVOLUTION_FWD_ALGO_FFT_TILING) {
+      if (algorithm_desc.algo_id() != CUDNN_CONVOLUTION_FWD_ALGO_FFT_TILING) {
         return port::Status::OK();
       }
       if (input_descriptor.ndims() < 3) {
@@ -2876,7 +2898,8 @@ port::Status CudnnSupport::DoConvolveImpl(
     }());
   }
 
-  if (algo_desc.algo_id() == CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
+  if (algorithm_desc.algo_id() ==
+          CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
       !ShouldIncludeWinogradNonfusedAlgo(input_descriptor, output_descriptor)) {
     return port::Status(port::error::FAILED_PRECONDITION,
                         "This configuration has potential integer overflow in "
@@ -2888,24 +2911,25 @@ port::Status CudnnSupport::DoConvolveImpl(
       /*alpha=*/alpha, /*srcDesc=*/input_nd.handle(),
       /*srcData=*/input_data.opaque(), /*filterDesc=*/filter.handle(),
       /*filterData=*/filter_data.opaque(), /*convDesc=*/conv.handle(),
-      /*algo=*/ToConvForwardAlgo(algo_desc), /*workSpace=*/scratch.opaque(),
-      /*workSpaceSizeInBytes=*/scratch.size(), /*beta=*/beta,
+      /*algo=*/ToConvForwardAlgo(algorithm_desc),
+      /*workSpace=*/scratch_memory->opaque(),
+      /*workSpaceSizeInBytes=*/scratch_memory->size(), /*beta=*/beta,
       /*yDesc=*/output_nd.handle(), /*y=*/output_data->opaque()));
 
   if (is_profiling) {
     if (!timer->Stop(AsCUDAStream(stream))) {
       return port::Status(port::error::INTERNAL, "Failed to stop timer");
     }
-    output_profile_result->set_algorithm(algo_desc);
+    output_profile_result->set_algorithm(algorithm_desc);
     output_profile_result->set_elapsed_time_in_ms(
         timer->GetElapsedMilliseconds());
-    output_profile_result->set_scratch_size(scratch.size());
+    output_profile_result->set_scratch_size(scratch_memory->size());
 
     LogCudaProto(
-        GenerateConvProto<T>(dnn::ConvolutionKind::FORWARD, input_descriptor,
-                             filter_descriptor, output_descriptor, algo_desc,
-                             convolution_descriptor, dalpha, dbeta,
-                             accumulator_type, dnn::ActivationMode::kNone),
+        GenerateConvProto<T>(
+            dnn::ConvolutionKind::FORWARD, input_descriptor, filter_descriptor,
+            output_descriptor, algorithm_desc, convolution_descriptor, dalpha,
+            dbeta, accumulator_type, dnn::ActivationMode::kNone),
         output_profile_result->elapsed_time_in_ms(), stream->parent());
   }
 
@@ -3310,7 +3334,7 @@ port::Status CudnnSupport::DoBatchNormalizationBackwardImpl(
   return port::Status::OK();
 }
 
-bool CudnnSupport::DoConvolve(
+bool CudnnSupport::PrepareForConvolution(
     Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
     const DeviceMemory<float>& input_data,
     const dnn::FilterDescriptor& filter_descriptor,
@@ -3319,12 +3343,70 @@ bool CudnnSupport::DoConvolve(
     const dnn::BatchDescriptor& output_descriptor,
     DeviceMemory<float>* output_data, ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+  return IsStatusOk(PrepareForConvolutionImpl<float>(
+                        stream, batch_descriptor, input_data, filter_descriptor,
+                        filter_data, convolution_descriptor, output_descriptor,
+                        output_data, dnn::DataType::kFloat, scratch_allocator,
+                        algorithm_config, algorithm_desc, scratch_memory),
+                    /*report_error=*/true);
+}
+
+bool CudnnSupport::PrepareForConvolution(
+    Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
+    const DeviceMemory<double>& input_data,
+    const dnn::FilterDescriptor& filter_descriptor,
+    const DeviceMemory<double>& filter_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
+    const dnn::AlgorithmConfig& algorithm_config,
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+  return IsStatusOk(PrepareForConvolutionImpl<double>(
+                        stream, batch_descriptor, input_data, filter_descriptor,
+                        filter_data, convolution_descriptor, output_descriptor,
+                        output_data, dnn::DataType::kDouble, scratch_allocator,
+                        algorithm_config, algorithm_desc, scratch_memory),
+                    /*report_error=*/true);
+}
+
+bool CudnnSupport::PrepareForConvolution(
+    Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
+    const DeviceMemory<Eigen::half>& input_data,
+    const dnn::FilterDescriptor& filter_descriptor,
+    const DeviceMemory<Eigen::half>& filter_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<Eigen::half>* output_data, ScratchAllocator* scratch_allocator,
+    const dnn::AlgorithmConfig& algorithm_config,
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+  dnn::DataType acc_type =
+      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
+          ? dnn::DataType::kFloat
+          : dnn::DataType::kHalf;
+  return IsStatusOk(
+      PrepareForConvolutionImpl<Eigen::half>(
+          stream, batch_descriptor, input_data, filter_descriptor, filter_data,
+          convolution_descriptor, output_descriptor, output_data, acc_type,
+          scratch_allocator, algorithm_config, algorithm_desc, scratch_memory),
+      /*report_error=*/true);
+}
+
+bool CudnnSupport::DoConvolve(
+    Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
+    const DeviceMemory<float>& input_data,
+    const dnn::FilterDescriptor& filter_descriptor,
+    const DeviceMemory<float>& filter_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<float>* output_data, const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
     dnn::ProfileResult* output_profile_result) {
   return IsStatusOk(
       DoConvolveImpl(stream, batch_descriptor, input_data, filter_descriptor,
                      filter_data, convolution_descriptor, output_descriptor,
-                     output_data, dnn::DataType::kFloat, scratch_allocator,
-                     algorithm_config, output_profile_result),
+                     output_data, dnn::DataType::kFloat, algorithm_desc,
+                     scratch_memory, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3335,14 +3417,14 @@ bool CudnnSupport::DoConvolve(
     const DeviceMemory<double>& filter_data,
     const dnn::ConvolutionDescriptor& convolution_descriptor,
     const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
+    DeviceMemory<double>* output_data, const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
     dnn::ProfileResult* output_profile_result) {
   return IsStatusOk(
       DoConvolveImpl(stream, batch_descriptor, input_data, filter_descriptor,
                      filter_data, convolution_descriptor, output_descriptor,
-                     output_data, dnn::DataType::kDouble, scratch_allocator,
-                     algorithm_config, output_profile_result),
+                     output_data, dnn::DataType::kDouble, algorithm_desc,
+                     scratch_memory, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3353,8 +3435,9 @@ bool CudnnSupport::DoConvolve(
     const DeviceMemory<Eigen::half>& filter_data,
     const dnn::ConvolutionDescriptor& convolution_descriptor,
     const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<Eigen::half>* output_data, ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
+    DeviceMemory<Eigen::half>* output_data,
+    const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
     dnn::ProfileResult* output_profile_result) {
   dnn::DataType acc_type =
       CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
@@ -3363,7 +3446,7 @@ bool CudnnSupport::DoConvolve(
   return IsStatusOk(
       DoConvolveImpl(stream, batch_descriptor, input_data, filter_descriptor,
                      filter_data, convolution_descriptor, output_descriptor,
-                     output_data, acc_type, scratch_allocator, algorithm_config,
+                     output_data, acc_type, algorithm_desc, scratch_memory,
                      output_profile_result),
       /*report_error=*/!output_profile_result);
 }
@@ -3499,7 +3582,7 @@ bool CudnnSupport::DoTransformTensor(Stream* stream,
 }
 
 template <class T>
-port::Status CudnnSupport::DoConvolveBackwardDataImpl(
+port::Status CudnnSupport::PrepareForConvolutionBackwardDataImpl(
     Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
     const DeviceMemory<T>& filter_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3509,6 +3592,36 @@ port::Status CudnnSupport::DoConvolveBackwardDataImpl(
     DeviceMemory<T>* backward_input_data, dnn::DataType accumulator_type,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+  cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
+  auto cudnn = cudnn_->GetHandle(parent_, stream);
+
+  CudnnTensorDescriptor out_back_nd(output_descriptor, cudnn_type);
+  CudnnTensorDescriptor in_back_nd(input_descriptor, cudnn_type);
+  CudnnFilterDescriptor filter(filter_descriptor, cudnn_type);
+  CudnnConvolutionDescriptor conv(convolution_descriptor,
+                                  ToCudnnDataType(accumulator_type));
+
+  SE_ASSIGN_OR_RETURN(
+      *algorithm_desc,
+      GetCudnnConvolutionBackwardDataAlgorithm(
+          stream, cudnn, algorithm_config, in_back_nd, filter, conv,
+          out_back_nd, scratch_allocator, scratch_memory));
+
+  return port::Status::OK();
+}
+
+template <class T>
+port::Status CudnnSupport::DoConvolveBackwardDataImpl(
+    Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+    const DeviceMemory<T>& filter_data,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<T> backward_output_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::BatchDescriptor& input_descriptor,
+    DeviceMemory<T>* backward_input_data, dnn::DataType accumulator_type,
+    const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
     dnn::ProfileResult* output_profile_result) {
   cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
   // Alpha is the scaling factor for input.
@@ -3532,12 +3645,6 @@ port::Status CudnnSupport::DoConvolveBackwardDataImpl(
 
   const bool is_profiling = output_profile_result != nullptr;
 
-  DeviceMemory<uint8> scratch;
-  SE_ASSIGN_OR_RETURN(dnn::AlgorithmDesc algo_desc,
-                      GetCudnnConvolutionBackwardDataAlgorithm(
-                          stream, cudnn, algorithm_config, in_back_nd, filter,
-                          conv, out_back_nd, scratch_allocator, &scratch));
-
   std::unique_ptr<CUDATimer, TimerDeleter> timer;
   if (is_profiling) {
     timer.reset(new CUDATimer(parent_));  // NOLINT
@@ -3549,7 +3656,8 @@ port::Status CudnnSupport::DoConvolveBackwardDataImpl(
     }
   }
 
-  if (algo_desc.algo_id() == CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
+  if (algorithm_desc.algo_id() ==
+          CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
       !ShouldIncludeWinogradNonfusedAlgo(input_descriptor, output_descriptor)) {
     return port::Status(port::error::FAILED_PRECONDITION,
                         "This configuration has potential integer overflow in "
@@ -3559,42 +3667,42 @@ port::Status CudnnSupport::DoConvolveBackwardDataImpl(
   // Cudnn 7.1.4 has a bug if the workspace of the following convolution is not
   // zero-initialized, nvbugs/2254619.
   if (CUDNN_VERSION >= 7000 && CUDNN_VERSION < 7300 &&
-      algo_desc.algo_id() == CUDNN_CONVOLUTION_BWD_DATA_ALGO_1 &&
-      cudnn_type == CUDNN_DATA_HALF && algo_desc.tensor_ops_enabled() &&
+      algorithm_desc.algo_id() == CUDNN_CONVOLUTION_BWD_DATA_ALGO_1 &&
+      cudnn_type == CUDNN_DATA_HALF && algorithm_desc.tensor_ops_enabled() &&
       input_descriptor.layout() == dnn::DataLayout::kBatchYXDepth &&
       filter_descriptor.layout() == dnn::FilterLayout::kOutputInputYX &&
       output_descriptor.layout() == dnn::DataLayout::kBatchDepthYX &&
       (convolution_descriptor.vertical_filter_stride() > 1 ||
        convolution_descriptor.horizontal_filter_stride() > 1)) {
-    stream->ThenMemZero(&scratch, scratch.size());
+    stream->ThenMemZero(scratch_memory, scratch_memory->size());
   }
 
-  RETURN_IF_CUDNN_ERROR(
-      cudnnConvolutionBackwardData(cudnn.handle(),
-                                   /*alpha=*/alpha,
-                                   /*wDesc=*/filter.handle(),
-                                   /*w=*/filter_data.opaque(),
-                                   /*dyDesc=*/out_back_nd.handle(),
-                                   /*dy=*/backward_output_data.opaque(),
-                                   /*convDesc=*/conv.handle(),
-                                   /*algo=*/ToConvBackwardDataAlgo(algo_desc),
-                                   /*workSpace=*/scratch.opaque(),
-                                   /*workSpaceSizeInBytes=*/scratch.size(),
-                                   /*beta=*/beta,
-                                   /*dxDesc=*/in_back_nd.handle(),
-                                   /*dx=*/backward_input_data->opaque()));
+  RETURN_IF_CUDNN_ERROR(cudnnConvolutionBackwardData(
+      cudnn.handle(),
+      /*alpha=*/alpha,
+      /*wDesc=*/filter.handle(),
+      /*w=*/filter_data.opaque(),
+      /*dyDesc=*/out_back_nd.handle(),
+      /*dy=*/backward_output_data.opaque(),
+      /*convDesc=*/conv.handle(),
+      /*algo=*/ToConvBackwardDataAlgo(algorithm_desc),
+      /*workSpace=*/scratch_memory->opaque(),
+      /*workSpaceSizeInBytes=*/scratch_memory->size(),
+      /*beta=*/beta,
+      /*dxDesc=*/in_back_nd.handle(),
+      /*dx=*/backward_input_data->opaque()));
   if (is_profiling) {
     if (!timer->Stop(AsCUDAStream(stream))) {
       return port::Status(port::error::INTERNAL, "Failed to stop timer");
     }
-    output_profile_result->set_algorithm(algo_desc);
+    output_profile_result->set_algorithm(algorithm_desc);
     output_profile_result->set_elapsed_time_in_ms(
         timer->GetElapsedMilliseconds());
-    output_profile_result->set_scratch_size(scratch.size());
+    output_profile_result->set_scratch_size(scratch_memory->size());
 
     LogCudaProto(GenerateConvProto<T>(
                      dnn::ConvolutionKind::BACKWARD_DATA, input_descriptor,
-                     filter_descriptor, output_descriptor, algo_desc,
+                     filter_descriptor, output_descriptor, algorithm_desc,
                      convolution_descriptor, dalpha, dbeta, accumulator_type,
                      dnn::ActivationMode::kNone),
                  output_profile_result->elapsed_time_in_ms(), stream->parent());
@@ -3603,7 +3711,7 @@ port::Status CudnnSupport::DoConvolveBackwardDataImpl(
   return port::Status::OK();
 }
 
-bool CudnnSupport::DoConvolveBackwardData(
+bool CudnnSupport::PrepareForConvolutionBackwardData(
     Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
     const DeviceMemory<double>& filter_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3613,17 +3721,17 @@ bool CudnnSupport::DoConvolveBackwardData(
     DeviceMemory<double>* backward_input_data,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
-    dnn::ProfileResult* output_profile_result) {
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
   return IsStatusOk(
-      DoConvolveBackwardDataImpl(
+      PrepareForConvolutionBackwardDataImpl(
           stream, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
           backward_input_data, dnn::DataType::kDouble, scratch_allocator,
-          algorithm_config, output_profile_result),
-      /*report_error=*/!output_profile_result);
+          algorithm_config, algorithm_desc, scratch_memory),
+      /*report_error=*/true);
 }
 
-bool CudnnSupport::DoConvolveBackwardData(
+bool CudnnSupport::PrepareForConvolutionBackwardData(
     Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
     const DeviceMemory<float>& filter_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3633,17 +3741,17 @@ bool CudnnSupport::DoConvolveBackwardData(
     DeviceMemory<float>* backward_input_data,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
-    dnn::ProfileResult* output_profile_result) {
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
   return IsStatusOk(
-      DoConvolveBackwardDataImpl(
+      PrepareForConvolutionBackwardDataImpl(
           stream, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
           backward_input_data, dnn::DataType::kFloat, scratch_allocator,
-          algorithm_config, output_profile_result),
-      /*report_error=*/!output_profile_result);
+          algorithm_config, algorithm_desc, scratch_memory),
+      /*report_error=*/true);
 }
 
-bool CudnnSupport::DoConvolveBackwardData(
+bool CudnnSupport::PrepareForConvolutionBackwardData(
     Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
     const DeviceMemory<Eigen::half>& filter_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3653,22 +3761,86 @@ bool CudnnSupport::DoConvolveBackwardData(
     DeviceMemory<Eigen::half>* backward_input_data,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
-    dnn::ProfileResult* output_profile_result) {
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
   dnn::DataType acc_type =
       CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
           ? dnn::DataType::kFloat
           : dnn::DataType::kHalf;
   return IsStatusOk(
-      DoConvolveBackwardDataImpl(
+      PrepareForConvolutionBackwardDataImpl(
           stream, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
           backward_input_data, acc_type, scratch_allocator, algorithm_config,
-          output_profile_result),
+          algorithm_desc, scratch_memory),
+      /*report_error=*/true);
+}
+
+bool CudnnSupport::DoConvolveBackwardData(
+    Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+    const DeviceMemory<double>& filter_data,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<double> backward_output_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::BatchDescriptor& input_descriptor,
+    DeviceMemory<double>* backward_input_data,
+    const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
+    dnn::ProfileResult* output_profile_result) {
+  return IsStatusOk(
+      DoConvolveBackwardDataImpl(
+          stream, filter_descriptor, filter_data, output_descriptor,
+          backward_output_data, convolution_descriptor, input_descriptor,
+          backward_input_data, dnn::DataType::kDouble, algorithm_desc,
+          scratch_memory, output_profile_result),
+      /*report_error=*/!output_profile_result);
+}
+
+bool CudnnSupport::DoConvolveBackwardData(
+    Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+    const DeviceMemory<float>& filter_data,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<float> backward_output_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::BatchDescriptor& input_descriptor,
+    DeviceMemory<float>* backward_input_data,
+    const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
+    dnn::ProfileResult* output_profile_result) {
+  return IsStatusOk(
+      DoConvolveBackwardDataImpl(
+          stream, filter_descriptor, filter_data, output_descriptor,
+          backward_output_data, convolution_descriptor, input_descriptor,
+          backward_input_data, dnn::DataType::kFloat, algorithm_desc,
+          scratch_memory, output_profile_result),
+      /*report_error=*/!output_profile_result);
+}
+
+bool CudnnSupport::DoConvolveBackwardData(
+    Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+    const DeviceMemory<Eigen::half>& filter_data,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<Eigen::half> backward_output_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::BatchDescriptor& input_descriptor,
+    DeviceMemory<Eigen::half>* backward_input_data,
+    const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
+    dnn::ProfileResult* output_profile_result) {
+  dnn::DataType acc_type =
+      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
+          ? dnn::DataType::kFloat
+          : dnn::DataType::kHalf;
+  return IsStatusOk(
+      DoConvolveBackwardDataImpl(stream, filter_descriptor, filter_data,
+                                 output_descriptor, backward_output_data,
+                                 convolution_descriptor, input_descriptor,
+                                 backward_input_data, acc_type, algorithm_desc,
+                                 scratch_memory, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
 template <class T>
-port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
+port::Status CudnnSupport::PrepareForConvolutionBackwardFilterImpl(
     Stream* stream, const dnn::BatchDescriptor& input_descriptor,
     const DeviceMemory<T>& input_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3678,6 +3850,36 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
     DeviceMemory<T>* backward_filter_data, dnn::DataType accumulator_type,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+  cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
+  auto cudnn = cudnn_->GetHandle(parent_, stream);
+
+  CudnnTensorDescriptor out_back_nd(output_descriptor, cudnn_type);
+  CudnnTensorDescriptor input_nd(input_descriptor, cudnn_type);
+  CudnnFilterDescriptor filter(filter_descriptor, cudnn_type);
+  CudnnConvolutionDescriptor conv(convolution_descriptor,
+                                  ToCudnnDataType(accumulator_type));
+
+  SE_ASSIGN_OR_RETURN(
+      *algorithm_desc,
+      GetCudnnConvolutionBackwardFilterAlgorithm(
+          stream, cudnn, algorithm_config, input_nd, filter, conv, out_back_nd,
+          scratch_allocator, scratch_memory));
+
+  return port::Status::OK();
+}
+
+template <class T>
+port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
+    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+    const DeviceMemory<T>& input_data,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<T> backward_output_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::FilterDescriptor& filter_descriptor,
+    DeviceMemory<T>* backward_filter_data, dnn::DataType accumulator_type,
+    const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
     dnn::ProfileResult* output_profile_result) {
   cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
   // Alpha is the scaling factor for input.
@@ -3701,12 +3903,6 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
 
   const bool is_profiling = output_profile_result != nullptr;
 
-  DeviceMemory<uint8> scratch;
-  SE_ASSIGN_OR_RETURN(dnn::AlgorithmDesc algo_desc,
-                      GetCudnnConvolutionBackwardFilterAlgorithm(
-                          stream, cudnn, algorithm_config, input_nd, filter,
-                          conv, out_back_nd, scratch_allocator, &scratch));
-
   std::unique_ptr<CUDATimer, TimerDeleter> timer;
   if (is_profiling) {
     timer.reset(new CUDATimer(parent_));  // NOLINT
@@ -3722,7 +3918,8 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
   // results. See nvbugs/2072856
   if (CUDNN_VERSION < 7300) {
     SE_RETURN_IF_ERROR([&] {
-      if (algo_desc.algo_id() != CUDNN_CONVOLUTION_BWD_FILTER_ALGO_FFT_TILING) {
+      if (algorithm_desc.algo_id() !=
+          CUDNN_CONVOLUTION_BWD_FILTER_ALGO_FFT_TILING) {
         return port::Status::OK();
       }
       if (output_descriptor.height() > 1 && output_descriptor.width() > 1) {
@@ -3748,7 +3945,8 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
     }());
   }
 
-  if (algo_desc.algo_id() == CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
+  if (algorithm_desc.algo_id() ==
+          CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
       !ShouldIncludeWinogradNonfusedAlgo(input_descriptor, output_descriptor)) {
     return port::Status(port::error::FAILED_PRECONDITION,
                         "This configuration has potential integer overflow in "
@@ -3764,7 +3962,7 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
   //
   // See nvbugs/2379553.
   if (CUDNN_VERSION >= 7100 && CUDNN_VERSION < 7300 &&
-      algo_desc.algo_id() == CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1 &&
+      algorithm_desc.algo_id() == CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1 &&
       cudnn_type == CUDNN_DATA_HALF &&
       input_descriptor.layout() == dnn::DataLayout::kBatchYXDepth &&
       filter_descriptor.layout() == dnn::FilterLayout::kOutputYXInput &&
@@ -3782,9 +3980,9 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
       /*diffDesc=*/out_back_nd.handle(),
       /*diffData=*/backward_output_data.opaque(),
       /*convDesc=*/conv.handle(),
-      /*algo=*/ToConvBackwardFilterAlgo(algo_desc),
-      /*workSpace=*/scratch.opaque(),
-      /*workSpaceSizeInBytes=*/scratch.size(),
+      /*algo=*/ToConvBackwardFilterAlgo(algorithm_desc),
+      /*workSpace=*/scratch_memory->opaque(),
+      /*workSpaceSizeInBytes=*/scratch_memory->size(),
       /*beta=*/beta,
       /*gradDesc=*/filter.handle(),
       /*dw=*/backward_filter_data->opaque()));
@@ -3792,14 +3990,14 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
     if (!timer->Stop(AsCUDAStream(stream))) {
       return port::Status(port::error::INTERNAL, "Failed to stop timer");
     }
-    output_profile_result->set_algorithm(algo_desc);
+    output_profile_result->set_algorithm(algorithm_desc);
     output_profile_result->set_elapsed_time_in_ms(
         timer->GetElapsedMilliseconds());
-    output_profile_result->set_scratch_size(scratch.size());
+    output_profile_result->set_scratch_size(scratch_memory->size());
 
     LogCudaProto(GenerateConvProto<T>(
                      dnn::ConvolutionKind::BACKWARD_FILTER, input_descriptor,
-                     filter_descriptor, output_descriptor, algo_desc,
+                     filter_descriptor, output_descriptor, algorithm_desc,
                      convolution_descriptor, dalpha, dbeta, accumulator_type,
                      dnn::ActivationMode::kNone),
                  output_profile_result->elapsed_time_in_ms(), stream->parent());
@@ -3808,7 +4006,7 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
   return port::Status::OK();
 }
 
-bool CudnnSupport::DoConvolveBackwardFilter(
+bool CudnnSupport::PrepareForConvolutionBackwardFilter(
     Stream* stream, const dnn::BatchDescriptor& input_descriptor,
     const DeviceMemory<double>& input_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3818,18 +4016,17 @@ bool CudnnSupport::DoConvolveBackwardFilter(
     DeviceMemory<double>* backward_filter_data,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
-    dnn::ProfileResult* output_profile_result) {
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
   return IsStatusOk(
-      DoConvolveBackwardFilterImpl(
+      PrepareForConvolutionBackwardFilterImpl(
           stream, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, dnn::DataType::kDouble,
-
-          scratch_allocator, algorithm_config, output_profile_result),
-      /*report_error=*/!output_profile_result);
+          backward_filter_data, dnn::DataType::kDouble, scratch_allocator,
+          algorithm_config, algorithm_desc, scratch_memory),
+      /*report_error=*/true);
 }
 
-bool CudnnSupport::DoConvolveBackwardFilter(
+bool CudnnSupport::PrepareForConvolutionBackwardFilter(
     Stream* stream, const dnn::BatchDescriptor& input_descriptor,
     const DeviceMemory<float>& input_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3839,18 +4036,17 @@ bool CudnnSupport::DoConvolveBackwardFilter(
     DeviceMemory<float>* backward_filter_data,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
-    dnn::ProfileResult* output_profile_result) {
-  return IsStatusOk(DoConvolveBackwardFilterImpl(
-                        stream, input_descriptor, input_data, output_descriptor,
-                        backward_output_data, convolution_descriptor,
-                        filter_descriptor, backward_filter_data,
-
-                        dnn::DataType::kFloat, scratch_allocator,
-                        algorithm_config, output_profile_result),
-                    /*report_error=*/!output_profile_result);
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+  return IsStatusOk(
+      PrepareForConvolutionBackwardFilterImpl(
+          stream, input_descriptor, input_data, output_descriptor,
+          backward_output_data, convolution_descriptor, filter_descriptor,
+          backward_filter_data, dnn::DataType::kFloat, scratch_allocator,
+          algorithm_config, algorithm_desc, scratch_memory),
+      /*report_error=*/true);
 }
 
-bool CudnnSupport::DoConvolveBackwardFilter(
+bool CudnnSupport::PrepareForConvolutionBackwardFilter(
     Stream* stream, const dnn::BatchDescriptor& input_descriptor,
     const DeviceMemory<Eigen::half>& input_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3860,20 +4056,83 @@ bool CudnnSupport::DoConvolveBackwardFilter(
     DeviceMemory<Eigen::half>* backward_filter_data,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
-    dnn::ProfileResult* output_profile_result) {
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
   dnn::DataType acc_type =
       CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
           ? dnn::DataType::kFloat
           : dnn::DataType::kHalf;
   return IsStatusOk(
-      DoConvolveBackwardFilterImpl(
+      PrepareForConvolutionBackwardFilterImpl(
           stream, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
           backward_filter_data, acc_type, scratch_allocator, algorithm_config,
-          output_profile_result),
+          algorithm_desc, scratch_memory),
+      /*report_error=*/true);
+}
+
+bool CudnnSupport::DoConvolveBackwardFilter(
+    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+    const DeviceMemory<double>& input_data,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<double> backward_output_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::FilterDescriptor& filter_descriptor,
+    DeviceMemory<double>* backward_filter_data,
+    const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
+    dnn::ProfileResult* output_profile_result) {
+  return IsStatusOk(
+      DoConvolveBackwardFilterImpl(
+          stream, input_descriptor, input_data, output_descriptor,
+          backward_output_data, convolution_descriptor, filter_descriptor,
+          backward_filter_data, dnn::DataType::kDouble, algorithm_desc,
+          scratch_memory, output_profile_result),
+      /*report_error=*/!output_profile_result);
+}
+
+bool CudnnSupport::DoConvolveBackwardFilter(
+    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+    const DeviceMemory<float>& input_data,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<float> backward_output_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::FilterDescriptor& filter_descriptor,
+    DeviceMemory<float>* backward_filter_data,
+    const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
+    dnn::ProfileResult* output_profile_result) {
+  return IsStatusOk(
+      DoConvolveBackwardFilterImpl(
+          stream, input_descriptor, input_data, output_descriptor,
+          backward_output_data, convolution_descriptor, filter_descriptor,
+          backward_filter_data, dnn::DataType::kFloat, algorithm_desc,
+          scratch_memory, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
+bool CudnnSupport::DoConvolveBackwardFilter(
+    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+    const DeviceMemory<Eigen::half>& input_data,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<Eigen::half> backward_output_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::FilterDescriptor& filter_descriptor,
+    DeviceMemory<Eigen::half>* backward_filter_data,
+    const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
+    dnn::ProfileResult* output_profile_result) {
+  dnn::DataType acc_type =
+      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
+          ? dnn::DataType::kFloat
+          : dnn::DataType::kHalf;
+  return IsStatusOk(DoConvolveBackwardFilterImpl(
+                        stream, input_descriptor, input_data, output_descriptor,
+                        backward_output_data, convolution_descriptor,
+                        filter_descriptor, backward_filter_data, acc_type,
+                        algorithm_desc, scratch_memory, output_profile_result),
+                    /*report_error=*/!output_profile_result);
+}
+
 template <class T>
 port::Status CudnnSupport::DoConvolveBackwardBiasImpl(
     Stream* stream, const dnn::BatchDescriptor& input_descriptor,
diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.h b/tensorflow/stream_executor/cuda/cuda_dnn.h
index 4cce3c5626..d7514981d5 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.h
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.h
@@ -258,6 +258,43 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<float>* scale_backprop,
       DeviceMemory<float>* offset_backprop) override;
 
+  bool PrepareForConvolution(
+      Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
+      const DeviceMemory<float>& input_data,
+      const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<float>& filter_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<float>* output_data, ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) override;
+
+  bool PrepareForConvolution(
+      Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
+      const DeviceMemory<double>& input_data,
+      const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<double>& filter_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) override;
+
+  bool PrepareForConvolution(
+      Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
+      const DeviceMemory<Eigen::half>& input_data,
+      const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<Eigen::half>& filter_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<Eigen::half>* output_data,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) override;
+
   bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
                   const DeviceMemory<float>& input_data,
                   const dnn::FilterDescriptor& filter_descriptor,
@@ -265,8 +302,8 @@ class CudnnSupport : public dnn::DnnSupport {
                   const dnn::ConvolutionDescriptor& convolution_descriptor,
                   const dnn::BatchDescriptor& output_descriptor,
                   DeviceMemory<float>* output_data,
-                  ScratchAllocator* scratch_allocator,
-                  const dnn::AlgorithmConfig& algorithm_config,
+                  const dnn::AlgorithmDesc& algorithm_desc,
+                  DeviceMemory<uint8>* scratch_memory,
                   dnn::ProfileResult* output_profile_result) override;
 
   bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
@@ -276,8 +313,8 @@ class CudnnSupport : public dnn::DnnSupport {
                   const dnn::ConvolutionDescriptor& convolution_descriptor,
                   const dnn::BatchDescriptor& output_descriptor,
                   DeviceMemory<double>* output_data,
-                  ScratchAllocator* scratch_allocator,
-                  const dnn::AlgorithmConfig& algorithm_config,
+                  const dnn::AlgorithmDesc& algorithm_desc,
+                  DeviceMemory<uint8>* scratch_memory,
                   dnn::ProfileResult* output_profile_result) override;
 
   bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
@@ -287,8 +324,8 @@ class CudnnSupport : public dnn::DnnSupport {
                   const dnn::ConvolutionDescriptor& convolution_descriptor,
                   const dnn::BatchDescriptor& output_descriptor,
                   DeviceMemory<Eigen::half>* output_data,
-                  ScratchAllocator* scratch_allocator,
-                  const dnn::AlgorithmConfig& algorithm_config,
+                  const dnn::AlgorithmDesc& algorithm_desc,
+                  DeviceMemory<uint8>* scratch_memory,
                   dnn::ProfileResult* output_profile_result) override;
 
   bool DoFusedConvolve(
@@ -390,7 +427,20 @@ class CudnnSupport : public dnn::DnnSupport {
     return false;
   }
 
-  bool DoConvolveBackwardData(
+  bool PrepareForConvolutionBackwardData(
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<float>& filter_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<float> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
+      DeviceMemory<float>* backward_input_data,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) override;
+
+  bool PrepareForConvolutionBackwardData(
       Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
       const DeviceMemory<double>& filter_data,
       const dnn::BatchDescriptor& output_descriptor,
@@ -400,6 +450,32 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<double>* backward_input_data,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) override;
+
+  bool PrepareForConvolutionBackwardData(
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<Eigen::half>& filter_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<Eigen::half> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
+      DeviceMemory<Eigen::half>* backward_input_data,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) override;
+
+  bool DoConvolveBackwardData(
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<double>& filter_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<double> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
+      DeviceMemory<double>* backward_input_data,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result) override;
 
   bool DoConvolveBackwardData(
@@ -410,8 +486,8 @@ class CudnnSupport : public dnn::DnnSupport {
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::BatchDescriptor& input_descriptor,
       DeviceMemory<float>* backward_input_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result) override;
 
   bool DoConvolveBackwardData(
@@ -422,11 +498,11 @@ class CudnnSupport : public dnn::DnnSupport {
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::BatchDescriptor& input_descriptor,
       DeviceMemory<Eigen::half>* backward_input_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result) override;
 
-  bool DoConvolveBackwardFilter(
+  bool PrepareForConvolutionBackwardFilter(
       Stream* stream, const dnn::BatchDescriptor& input_descriptor,
       const DeviceMemory<double>& input_data,
       const dnn::BatchDescriptor& output_descriptor,
@@ -436,9 +512,10 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<double>* backward_filter_data,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
-      dnn::ProfileResult* output_profile_result) override;
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) override;
 
-  bool DoConvolveBackwardFilter(
+  bool PrepareForConvolutionBackwardFilter(
       Stream* stream, const dnn::BatchDescriptor& input_descriptor,
       const DeviceMemory<float>& input_data,
       const dnn::BatchDescriptor& output_descriptor,
@@ -448,9 +525,10 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<float>* backward_filter_data,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
-      dnn::ProfileResult* output_profile_result) override;
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) override;
 
-  bool DoConvolveBackwardFilter(
+  bool PrepareForConvolutionBackwardFilter(
       Stream* stream, const dnn::BatchDescriptor& input_descriptor,
       const DeviceMemory<Eigen::half>& input_data,
       const dnn::BatchDescriptor& output_descriptor,
@@ -460,6 +538,43 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<Eigen::half>* backward_filter_data,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) override;
+
+  bool DoConvolveBackwardFilter(
+      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+      const DeviceMemory<double>& input_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<double> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::FilterDescriptor& filter_descriptor,
+      DeviceMemory<double>* backward_filter_data,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
+      dnn::ProfileResult* output_profile_result) override;
+
+  bool DoConvolveBackwardFilter(
+      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+      const DeviceMemory<float>& input_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<float> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::FilterDescriptor& filter_descriptor,
+      DeviceMemory<float>* backward_filter_data,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
+      dnn::ProfileResult* output_profile_result) override;
+
+  bool DoConvolveBackwardFilter(
+      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+      const DeviceMemory<Eigen::half>& input_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<Eigen::half> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::FilterDescriptor& filter_descriptor,
+      DeviceMemory<Eigen::half>* backward_filter_data,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result) override;
 
   bool DoConvolveBackwardBias(
@@ -677,7 +792,7 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<U>* offset_backprop);
 
   template <class T>
-  port::Status DoConvolveImpl(
+  port::Status PrepareForConvolutionImpl(
       Stream* stream, const dnn::BatchDescriptor& input_descriptor,
       const DeviceMemory<T>& input_data,
       const dnn::FilterDescriptor& filter_descriptor,
@@ -687,6 +802,19 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<T>* output_data, dnn::DataType accumulator_type,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory);
+
+  template <class T>
+  port::Status DoConvolveImpl(
+      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+      const DeviceMemory<T>& input_data,
+      const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<T>& filter_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<T>* output_data, dnn::DataType accumulator_type,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result);
 
   template <typename ElementType, typename BiasType, typename ScaleType>
@@ -707,7 +835,7 @@ class CudnnSupport : public dnn::DnnSupport {
       dnn::ProfileResult* output_profile_result);
 
   template <class T>
-  port::Status DoConvolveBackwardDataImpl(
+  port::Status PrepareForConvolutionBackwardDataImpl(
       Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
       const DeviceMemory<T>& filter_data,
       const dnn::BatchDescriptor& output_descriptor,
@@ -717,19 +845,45 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<T>* backward_input_data, dnn::DataType accumulator_type,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory);
+
+  template <class T>
+  port::Status DoConvolveBackwardDataImpl(
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<T>& filter_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<T> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
+      DeviceMemory<T>* backward_input_data, dnn::DataType accumulator_type,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result);
 
   template <class T>
-  port::Status DoConvolveBackwardFilterImpl(
+  port::Status PrepareForConvolutionBackwardFilterImpl(
       Stream* stream, const dnn::BatchDescriptor& input_descriptor,
       const DeviceMemory<T>& input_data,
-      const dnn::BatchDescriptor& output_descriptor,
+      const dnn::BatchDescriptor& output_descriptor_in,
       DeviceMemory<T> backward_output_data,
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::FilterDescriptor& filter_descriptor,
       DeviceMemory<T>* backward_filter_data, dnn::DataType accumulator_type,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory);
+
+  template <class T>
+  port::Status DoConvolveBackwardFilterImpl(
+      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+      const DeviceMemory<T>& input_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<T> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::FilterDescriptor& filter_descriptor,
+      DeviceMemory<T>* backward_filter_data, dnn::DataType accumulator_type,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result);
 
   template <class T>
diff --git a/tensorflow/stream_executor/dnn.h b/tensorflow/stream_executor/dnn.h
index ff7b02cf6b..f5a77d6525 100644
--- a/tensorflow/stream_executor/dnn.h
+++ b/tensorflow/stream_executor/dnn.h
@@ -738,6 +738,7 @@ class PoolingDescriptor {
 class AlgorithmDesc {
  public:
   typedef int64 Index;
+  AlgorithmDesc() : AlgorithmDesc(0, false) {}
   AlgorithmDesc(Index a, bool use_tensor_ops) {
     proto_.set_algo_id(a);
     proto_.set_math_type(use_tensor_ops ? AlgorithmProto::TENSOR_OP_MATH
@@ -1186,6 +1187,52 @@ class DnnSupport {
     return false;
   }
 
+  virtual bool PrepareForConvolution(
+      Stream* stream, const BatchDescriptor& batch_descriptor,
+      const DeviceMemory<float>& input_data,
+      const FilterDescriptor& filter_descriptor,
+      const DeviceMemory<float>& filter_data,
+      const ConvolutionDescriptor& convolution_descriptor,
+      const BatchDescriptor& output_descriptor,
+      DeviceMemory<float>* output_data, ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+    *algorithm_desc = {};
+    *scratch_memory = {};
+    return true;
+  }
+
+  virtual bool PrepareForConvolution(
+      Stream* stream, const BatchDescriptor& batch_descriptor,
+      const DeviceMemory<double>& input_data,
+      const FilterDescriptor& filter_descriptor,
+      const DeviceMemory<double>& filter_data,
+      const ConvolutionDescriptor& convolution_descriptor,
+      const BatchDescriptor& output_descriptor,
+      DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+    *algorithm_desc = {};
+    *scratch_memory = {};
+    return true;
+  }
+
+  virtual bool PrepareForConvolution(
+      Stream* stream, const BatchDescriptor& batch_descriptor,
+      const DeviceMemory<Eigen::half>& input_data,
+      const FilterDescriptor& filter_descriptor,
+      const DeviceMemory<Eigen::half>& filter_data,
+      const ConvolutionDescriptor& convolution_descriptor,
+      const BatchDescriptor& output_descriptor,
+      DeviceMemory<Eigen::half>* output_data,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+    *algorithm_desc = {};
+    *scratch_memory = {};
+    return true;
+  }
+
   // Enqueues a single-precision convolution operation onto the stream.
   //
   // Arguments (all borrowed):
@@ -1199,10 +1246,10 @@ class DnnSupport {
   //  output_descriptor: dimensions of the output layer.
   //  output_data: un-owned device memory region in which to place the
   //    convolution result.
-  //  scratch_allocator: un-owned, may-be-null object that may allocate scratch
-  //    space in order to speed up the convolution operation.
-  //  algorithm_config: specifies which algorithm should be used for the
+  //  algorithm_desc: specifies which algorithm should be used for the
   //    operation.
+  //  scratch: un-owned device memory for scratch space in order to speed up
+  //    the convolution operation.
   //  output_profile_result: the output profile result for this call. The
   //    profiling is only enabled when this is not nullptr.
   //
@@ -1227,8 +1274,9 @@ class DnnSupport {
       const DeviceMemory<float>& filter_data,
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<float>* output_data, ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      DeviceMemory<float>* output_data,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       ProfileResult* output_profile_result) = 0;
 
   // Enqueues a double-precision convolution operation onto the stream.
@@ -1240,8 +1288,9 @@ class DnnSupport {
       const DeviceMemory<double>& filter_data,
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      DeviceMemory<double>* output_data,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result) = 0;
 
   // Enqueues a half-precision convolution operation onto the stream.
@@ -1254,8 +1303,8 @@ class DnnSupport {
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::BatchDescriptor& output_descriptor,
       DeviceMemory<Eigen::half>* output_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       ProfileResult* output_profile_result) = 0;
 
   // Return a list of algorithms supported by the forward convolution pass.
@@ -1311,6 +1360,54 @@ class DnnSupport {
       const BatchDescriptor& output_descriptor,
       DeviceMemory<float>* output_data) = 0;
 
+  virtual bool PrepareForConvolutionBackwardData(
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<float>& filter_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<float> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
+      DeviceMemory<float>* backward_input_data,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+    *algorithm_desc = {};
+    *scratch_memory = {};
+    return true;
+  }
+
+  virtual bool PrepareForConvolutionBackwardData(
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<double>& filter_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<double> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
+      DeviceMemory<double>* backward_input_data,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+    *algorithm_desc = {};
+    *scratch_memory = {};
+    return true;
+  }
+
+  virtual bool PrepareForConvolutionBackwardData(
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<Eigen::half>& filter_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<Eigen::half> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
+      DeviceMemory<Eigen::half>* backward_input_data,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+    *algorithm_desc = {};
+    *scratch_memory = {};
+    return true;
+  }
+
   // Enqueues a single-precision backward convolution (for data) operation onto
   // the stream.
   //
@@ -1330,15 +1427,15 @@ class DnnSupport {
   //  scratch_allocator: un-owned, may-be-null object that may allocate scratch
   //    space in order to speed up the convolution operation.
   virtual bool DoConvolveBackwardData(
-      Stream* stream, const FilterDescriptor& filter_descriptor,
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
       const DeviceMemory<float>& filter_data,
-      const BatchDescriptor& output_descriptor,
+      const dnn::BatchDescriptor& output_descriptor,
       DeviceMemory<float> backward_output_data,
-      const ConvolutionDescriptor& convolution_descriptor,
-      const BatchDescriptor& input_descriptor,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
       DeviceMemory<float>* backward_input_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       ProfileResult* output_profile_result) = 0;
 
   // Return a list of algorithms supported by the backward convolution pass for
@@ -1348,28 +1445,76 @@ class DnnSupport {
       std::vector<AlgorithmDesc>* out_algorithms);
 
   virtual bool DoConvolveBackwardData(
-      Stream* stream, const FilterDescriptor& filter_descriptor,
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
       const DeviceMemory<double>& filter_data,
-      const BatchDescriptor& output_descriptor,
+      const dnn::BatchDescriptor& output_descriptor,
       DeviceMemory<double> backward_output_data,
-      const ConvolutionDescriptor& convolution_descriptor,
-      const BatchDescriptor& input_descriptor,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
       DeviceMemory<double>* backward_input_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       ProfileResult* output_profile_result) = 0;
 
   virtual bool DoConvolveBackwardData(
-      Stream* stream, const FilterDescriptor& filter_descriptor,
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
       const DeviceMemory<Eigen::half>& filter_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<Eigen::half> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
+      DeviceMemory<Eigen::half>* backward_input_data,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
+      ProfileResult* output_profile_result) = 0;
+
+  virtual bool PrepareForConvolutionBackwardFilter(
+      Stream* stream, const BatchDescriptor& input_descriptor,
+      const DeviceMemory<float>& input_data,
+      const BatchDescriptor& output_descriptor,
+      DeviceMemory<float> backward_output_data,
+      const ConvolutionDescriptor& convolution_descriptor,
+      const FilterDescriptor& filter_descriptor,
+      DeviceMemory<float>* backward_filter_data,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+    *algorithm_desc = {};
+    *scratch_memory = {};
+    return true;
+  }
+
+  virtual bool PrepareForConvolutionBackwardFilter(
+      Stream* stream, const BatchDescriptor& input_descriptor,
+      const DeviceMemory<double>& input_data,
+      const BatchDescriptor& output_descriptor,
+      DeviceMemory<double> backward_output_data,
+      const ConvolutionDescriptor& convolution_descriptor,
+      const FilterDescriptor& filter_descriptor,
+      DeviceMemory<double>* backward_filter_data,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+    *algorithm_desc = {};
+    *scratch_memory = {};
+    return true;
+  }
+
+  virtual bool PrepareForConvolutionBackwardFilter(
+      Stream* stream, const BatchDescriptor& input_descriptor,
+      const DeviceMemory<Eigen::half>& input_data,
       const BatchDescriptor& output_descriptor,
       DeviceMemory<Eigen::half> backward_output_data,
       const ConvolutionDescriptor& convolution_descriptor,
-      const BatchDescriptor& input_descriptor,
-      DeviceMemory<Eigen::half>* backward_input_data,
+      const FilterDescriptor& filter_descriptor,
+      DeviceMemory<Eigen::half>* backward_filter_data,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
-      ProfileResult* output_profile_result) = 0;
+      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+    *algorithm_desc = {};
+    *scratch_memory = {};
+    return true;
+  }
 
   // Enqueues a single-precision backward convolution (for filter) operation
   // onto the stream.
@@ -1398,8 +1543,8 @@ class DnnSupport {
       const ConvolutionDescriptor& convolution_descriptor,
       const FilterDescriptor& filter_descriptor,
       DeviceMemory<float>* backward_filter_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       ProfileResult* output_profile_result) = 0;
 
   // Return a list of algorithms supported by the backward convolution pass for
@@ -1416,8 +1561,8 @@ class DnnSupport {
       const ConvolutionDescriptor& convolution_descriptor,
       const FilterDescriptor& filter_descriptor,
       DeviceMemory<double>* backward_filter_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       ProfileResult* output_profile_result) = 0;
 
   virtual bool DoConvolveBackwardFilter(
@@ -1428,8 +1573,8 @@ class DnnSupport {
       const ConvolutionDescriptor& convolution_descriptor,
       const FilterDescriptor& filter_descriptor,
       DeviceMemory<Eigen::half>* backward_filter_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       ProfileResult* output_profile_result) = 0;
 
   // Enqueues a single-precision backward convolution (for bias) operation onto
diff --git a/tensorflow/stream_executor/stream.cc b/tensorflow/stream_executor/stream.cc
index 1b14e4c339..1befc18e19 100644
--- a/tensorflow/stream_executor/stream.cc
+++ b/tensorflow/stream_executor/stream.cc
@@ -549,11 +549,16 @@ Stream &Stream::ThenConvolveWithScratch(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      CheckError(dnn->DoConvolve(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      CheckError(dnn->PrepareForConvolution(
           this, input_descriptor, input_data, filter_descriptor, filter_data,
           convolution_descriptor, output_descriptor, output, scratch_allocator,
-          dnn::AlgorithmConfig(),
-          /*output_profile_result=*/nullptr));
+          dnn::AlgorithmConfig(), &algorithm_desc, &scratch_memory));
+      CheckError(dnn->DoConvolve(
+          this, input_descriptor, input_data, filter_descriptor, filter_data,
+          convolution_descriptor, output_descriptor, output, algorithm_desc,
+          &scratch_memory, nullptr));
     } else {
       SetErrorAndLogNoDnnSupport();
     }
@@ -576,11 +581,16 @@ Stream &Stream::ThenConvolveWithScratch(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      CheckError(dnn->DoConvolve(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      CheckError(dnn->PrepareForConvolution(
           this, input_descriptor, input_data, filter_descriptor, filter_data,
           convolution_descriptor, output_descriptor, output, scratch_allocator,
-          dnn::AlgorithmConfig(),
-          /*output_profile_result=*/nullptr));
+          dnn::AlgorithmConfig(), &algorithm_desc, &scratch_memory));
+      CheckError(dnn->DoConvolve(
+          this, input_descriptor, input_data, filter_descriptor, filter_data,
+          convolution_descriptor, output_descriptor, output, algorithm_desc,
+          &scratch_memory, nullptr));
     } else {
       SetErrorAndLogNoDnnSupport();
     }
@@ -758,10 +768,18 @@ Stream &Stream::ThenConvolveWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      auto status = dnn->DoConvolve(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      auto status = dnn->PrepareForConvolution(
           this, input_descriptor, input_data, filter_descriptor, filter_data,
           convolution_descriptor, output_descriptor, output, scratch_allocator,
-          algorithm_config, output_profile_result);
+          algorithm_config, &algorithm_desc, &scratch_memory);
+      if (status) {
+        status = dnn->DoConvolve(
+            this, input_descriptor, input_data, filter_descriptor, filter_data,
+            convolution_descriptor, output_descriptor, output, algorithm_desc,
+            &scratch_memory, output_profile_result);
+      }
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -789,10 +807,18 @@ Stream &Stream::ThenConvolveWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      auto status = dnn->DoConvolve(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      auto status = dnn->PrepareForConvolution(
           this, input_descriptor, input_data, filter_descriptor, filter_data,
           convolution_descriptor, output_descriptor, output, scratch_allocator,
-          algorithm_config, output_profile_result);
+          algorithm_config, &algorithm_desc, &scratch_memory);
+      if (status) {
+        status = dnn->DoConvolve(
+            this, input_descriptor, input_data, filter_descriptor, filter_data,
+            convolution_descriptor, output_descriptor, output, algorithm_desc,
+            &scratch_memory, output_profile_result);
+      }
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -820,10 +846,18 @@ Stream &Stream::ThenConvolveWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      auto status = dnn->DoConvolve(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      auto status = dnn->PrepareForConvolution(
           this, input_descriptor, input_data, filter_descriptor, filter_data,
           convolution_descriptor, output_descriptor, output, scratch_allocator,
-          algorithm_config, output_profile_result);
+          algorithm_config, &algorithm_desc, &scratch_memory);
+      if (status) {
+        status = dnn->DoConvolve(
+            this, input_descriptor, input_data, filter_descriptor, filter_data,
+            convolution_descriptor, output_descriptor, output, algorithm_desc,
+            &scratch_memory, output_profile_result);
+      }
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -969,10 +1003,17 @@ Stream &Stream::ThenConvolveBackwardDataWithScratch(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      CheckError(dnn->DoConvolveBackwardData(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      CheckError(dnn->PrepareForConvolutionBackwardData(
           this, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
           backward_input_data, scratch_allocator, dnn::AlgorithmConfig(),
+          &algorithm_desc, &scratch_memory));
+      CheckError(dnn->DoConvolveBackwardData(
+          this, filter_descriptor, filter_data, output_descriptor,
+          backward_output_data, convolution_descriptor, input_descriptor,
+          backward_input_data, algorithm_desc, &scratch_memory,
           /*output_profile_result=*/nullptr));
     } else {
       SetErrorAndLogNoDnnSupport();
@@ -999,11 +1040,20 @@ Stream &Stream::ThenConvolveBackwardDataWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      auto status = dnn->DoConvolveBackwardData(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      auto status = dnn->PrepareForConvolutionBackwardData(
           this, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
           backward_input_data, scratch_allocator, algorithm_config,
-          output_profile_result);
+          &algorithm_desc, &scratch_memory);
+      if (status) {
+        status = dnn->DoConvolveBackwardData(
+            this, filter_descriptor, filter_data, output_descriptor,
+            backward_output_data, convolution_descriptor, input_descriptor,
+            backward_input_data, algorithm_desc, &scratch_memory,
+            output_profile_result);
+      }
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -1032,11 +1082,20 @@ Stream &Stream::ThenConvolveBackwardDataWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      auto status = dnn->DoConvolveBackwardData(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      auto status = dnn->PrepareForConvolutionBackwardData(
           this, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
           backward_input_data, scratch_allocator, algorithm_config,
-          output_profile_result);
+          &algorithm_desc, &scratch_memory);
+      if (status) {
+        status = dnn->DoConvolveBackwardData(
+            this, filter_descriptor, filter_data, output_descriptor,
+            backward_output_data, convolution_descriptor, input_descriptor,
+            backward_input_data, algorithm_desc, &scratch_memory,
+            output_profile_result);
+      }
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -1065,11 +1124,20 @@ Stream &Stream::ThenConvolveBackwardDataWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      auto status = dnn->DoConvolveBackwardData(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      auto status = dnn->PrepareForConvolutionBackwardData(
           this, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
           backward_input_data, scratch_allocator, algorithm_config,
-          output_profile_result);
+          &algorithm_desc, &scratch_memory);
+      if (status) {
+        status = dnn->DoConvolveBackwardData(
+            this, filter_descriptor, filter_data, output_descriptor,
+            backward_output_data, convolution_descriptor, input_descriptor,
+            backward_input_data, algorithm_desc, &scratch_memory,
+            output_profile_result);
+      }
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -1096,10 +1164,17 @@ Stream &Stream::ThenConvolveBackwardDataWithScratch(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      CheckError(dnn->DoConvolveBackwardData(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      CheckError(dnn->PrepareForConvolutionBackwardData(
           this, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
           backward_input_data, scratch_allocator, dnn::AlgorithmConfig(),
+          &algorithm_desc, &scratch_memory));
+      CheckError(dnn->DoConvolveBackwardData(
+          this, filter_descriptor, filter_data, output_descriptor,
+          backward_output_data, convolution_descriptor, input_descriptor,
+          backward_input_data, algorithm_desc, &scratch_memory,
           /*output_profile_result=*/nullptr));
     } else {
       SetErrorAndLogNoDnnSupport();
@@ -1138,10 +1213,17 @@ Stream &Stream::ThenConvolveBackwardFilterWithScratch(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      CheckError(dnn->DoConvolveBackwardFilter(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      CheckError(dnn->PrepareForConvolutionBackwardFilter(
           this, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
           backward_filter_data, scratch_allocator, dnn::AlgorithmConfig(),
+          &algorithm_desc, &scratch_memory));
+      CheckError(dnn->DoConvolveBackwardFilter(
+          this, input_descriptor, input_data, output_descriptor,
+          backward_output_data, convolution_descriptor, filter_descriptor,
+          backward_filter_data, algorithm_desc, &scratch_memory,
           /*output_profile_result=*/nullptr));
     } else {
       SetErrorAndLogNoDnnSupport();
@@ -1168,11 +1250,20 @@ Stream &Stream::ThenConvolveBackwardFilterWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      auto status = dnn->DoConvolveBackwardFilter(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      auto status = dnn->PrepareForConvolutionBackwardFilter(
           this, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
           backward_filter_data, scratch_allocator, algorithm_config,
-          output_profile_result);
+          &algorithm_desc, &scratch_memory);
+      if (status) {
+        status = dnn->DoConvolveBackwardFilter(
+            this, input_descriptor, input_data, output_descriptor,
+            backward_output_data, convolution_descriptor, filter_descriptor,
+            backward_filter_data, algorithm_desc, &scratch_memory,
+            output_profile_result);
+      }
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -1201,11 +1292,20 @@ Stream &Stream::ThenConvolveBackwardFilterWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      auto status = dnn->DoConvolveBackwardFilter(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      auto status = dnn->PrepareForConvolutionBackwardFilter(
           this, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
           backward_filter_data, scratch_allocator, algorithm_config,
-          output_profile_result);
+          &algorithm_desc, &scratch_memory);
+      if (status) {
+        status = dnn->DoConvolveBackwardFilter(
+            this, input_descriptor, input_data, output_descriptor,
+            backward_output_data, convolution_descriptor, filter_descriptor,
+            backward_filter_data, algorithm_desc, &scratch_memory,
+            output_profile_result);
+      }
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -1232,10 +1332,17 @@ Stream &Stream::ThenConvolveBackwardFilterWithScratch(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      CheckError(dnn->DoConvolveBackwardFilter(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      CheckError(dnn->PrepareForConvolutionBackwardFilter(
           this, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
           backward_filter_data, scratch_allocator, dnn::AlgorithmConfig(),
+          &algorithm_desc, &scratch_memory));
+      CheckError(dnn->DoConvolveBackwardFilter(
+          this, input_descriptor, input_data, output_descriptor,
+          backward_output_data, convolution_descriptor, filter_descriptor,
+          backward_filter_data, algorithm_desc, &scratch_memory,
           /*output_profile_result=*/nullptr));
     } else {
       SetErrorAndLogNoDnnSupport();
@@ -1262,11 +1369,20 @@ Stream &Stream::ThenConvolveBackwardFilterWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      auto status = dnn->DoConvolveBackwardFilter(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      auto status = dnn->PrepareForConvolutionBackwardFilter(
           this, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
           backward_filter_data, scratch_allocator, algorithm_config,
-          output_profile_result);
+          &algorithm_desc, &scratch_memory);
+      if (status) {
+        status = dnn->DoConvolveBackwardFilter(
+            this, input_descriptor, input_data, output_descriptor,
+            backward_output_data, convolution_descriptor, filter_descriptor,
+            backward_filter_data, algorithm_desc, &scratch_memory,
+            output_profile_result);
+      }
       if (!status && !output_profile_result) {
         SetError();
       }
-- 
GitLab


From 04266f6c791113ec9e0098e52579a4c53e9acc0b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 12:55:40 -0800
Subject: [PATCH 0439/2345] Fix off-by-one error in time calculation for eta
 ecm3531 platform.

PiperOrigin-RevId: 228571040
---
 .../lite/experimental/micro/tools/make/targets/ecm3531/_main.c  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/experimental/micro/tools/make/targets/ecm3531/_main.c b/tensorflow/lite/experimental/micro/tools/make/targets/ecm3531/_main.c
index 93941e119f..2764f3ba50 100644
--- a/tensorflow/lite/experimental/micro/tools/make/targets/ecm3531/_main.c
+++ b/tensorflow/lite/experimental/micro/tools/make/targets/ecm3531/_main.c
@@ -85,7 +85,7 @@ void EtaPrintExecutionTime(uint64_t time_ms) {
       time_ms = time_ms / 10;
       time_string[k1] = (char)(0x30 + c);
     }
-    for (k1 = 4; k1 > 0; k1--) {  // print out 1 char at a time
+    for (k1 = 4; k1 >= 0; k1--) {  // print out 1 char at a time
       EtaCspUartPutc(&g_sUart1, time_string[k1]);
     }
   } else {
-- 
GitLab


From 998a22ab58d8e5faa1f041afea1e322797d70329 Mon Sep 17 00:00:00 2001
From: Shivani Agrawal <shivaniagrawal@google.com>
Date: Wed, 9 Jan 2019 13:05:00 -0800
Subject: [PATCH 0440/2345] [tf.data] document update to correct the
 non-working example.

PiperOrigin-RevId: 228572720
---
 tensorflow/python/data/experimental/ops/stats_aggregator.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/data/experimental/ops/stats_aggregator.py b/tensorflow/python/data/experimental/ops/stats_aggregator.py
index d5fcc033ab..3e4c66be27 100644
--- a/tensorflow/python/data/experimental/ops/stats_aggregator.py
+++ b/tensorflow/python/data/experimental/ops/stats_aggregator.py
@@ -45,7 +45,7 @@ class StatsAggregator(object):
 
   # Apply `StatsOptions` to associate `dataset` with `aggregator`.
   options = dataset_ops.Options()
-  options.experimental_stats = tf.data.experimental.StatsOptions(aggregator)
+  options.experimental_stats.aggregator = aggregator
   dataset = dataset.with_options(options)
   ```
 
-- 
GitLab


From 02747c96978bc76427cf45d32ecb24a866520043 Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Wed, 9 Jan 2019 13:06:08 -0800
Subject: [PATCH 0441/2345] Fix tensor_utils_test

Update another test target relying on lite/kernels:test_util, which
pulls in TF and requires the tf_cc_test build rule.

A follow-up CL will remove this requirement for this and other kernel
tests.

PiperOrigin-RevId: 228572896
---
 tensorflow/lite/kernels/internal/BUILD | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/kernels/internal/BUILD b/tensorflow/lite/kernels/internal/BUILD
index 74a11a72ef..b734b2d6cc 100644
--- a/tensorflow/lite/kernels/internal/BUILD
+++ b/tensorflow/lite/kernels/internal/BUILD
@@ -1,3 +1,4 @@
+load("//tensorflow:tensorflow.bzl", "tf_cc_test")
 load("//tensorflow/lite:build_def.bzl", "tflite_copts")
 load("//tensorflow/lite:special_rules.bzl", "tflite_portable_test_suite")
 
@@ -555,10 +556,11 @@ cc_library(
     ],
 )
 
-cc_test(
+# TODO(b/122597976): Eliminate TF dependency from lite/kernels:test_util,
+# in turn eliminating the need to use tf_cc_test for any dependent tests.
+tf_cc_test(
     name = "tensor_utils_test",
     srcs = ["tensor_utils_test.cc"],
-    copts = NEON_FLAGS_IF_APPLICABLE,
     linkopts = select({
         "//tensorflow:android": [
             "-fPIE -pie",
-- 
GitLab


From 4495df6cb19802fabaae9999a3b3fdcfe88b268d Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Wed, 9 Jan 2019 13:32:55 -0800
Subject: [PATCH 0442/2345] Update types in accuracy tool.

PiperOrigin-RevId: 228577610
---
 tensorflow/lite/tools/accuracy/utils.cc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tensorflow/lite/tools/accuracy/utils.cc b/tensorflow/lite/tools/accuracy/utils.cc
index c19dc1ff7c..953892b8dd 100644
--- a/tensorflow/lite/tools/accuracy/utils.cc
+++ b/tensorflow/lite/tools/accuracy/utils.cc
@@ -38,6 +38,12 @@ DataType GetTFDataType(TfLiteType tflite_type) {
       return DT_FLOAT;
     case kTfLiteUInt8:
       return DT_UINT8;
+    case kTfLiteInt8:
+      return DT_INT8;
+    case kTfLiteInt32:
+      return DT_INT32;
+    case kTfLiteInt64:
+      return DT_INT64;
     default:
       return DT_INVALID;
   }
-- 
GitLab


From 043e72fa2cab247290e0015682e8bc818bee5a15 Mon Sep 17 00:00:00 2001
From: Grzegorz Pawelczak <grzegorzp@graphcore.ai>
Date: Wed, 9 Jan 2019 21:41:09 +0000
Subject: [PATCH 0443/2345] Fix copyright year and run clang format

---
 tensorflow/compiler/xla/service/sort_simplifier.cc      | 6 ++----
 tensorflow/compiler/xla/service/sort_simplifier_test.cc | 2 +-
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/xla/service/sort_simplifier.cc b/tensorflow/compiler/xla/service/sort_simplifier.cc
index ee91762253..6136dc278c 100644
--- a/tensorflow/compiler/xla/service/sort_simplifier.cc
+++ b/tensorflow/compiler/xla/service/sort_simplifier.cc
@@ -92,8 +92,7 @@ StatusOr<bool> RemoveUnusedOperandFromSort(HloInstruction* sort) {
   }
   return true;
 }
-
-} // namespace
+}  // namespace
 
 StatusOr<bool> SortSimplifier::Run(HloModule* module) {
   VLOG(2) << "HLO module before SortSimplifier:";
@@ -109,8 +108,7 @@ StatusOr<bool> SortSimplifier::Run(HloModule* module) {
   }
 
   for (HloInstruction* sort_instr : sort_instrs) {
-    TF_ASSIGN_OR_RETURN(bool result,
-                        RemoveUnusedOperandFromSort(sort_instr));
+    TF_ASSIGN_OR_RETURN(bool result, RemoveUnusedOperandFromSort(sort_instr));
     changed |= result;
   }
 
diff --git a/tensorflow/compiler/xla/service/sort_simplifier_test.cc b/tensorflow/compiler/xla/service/sort_simplifier_test.cc
index 5a81e537de..4df833bbeb 100644
--- a/tensorflow/compiler/xla/service/sort_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/sort_simplifier_test.cc
@@ -1,4 +1,4 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
-- 
GitLab


From 0dc03e940edcab6e6680990557634b335d3be4e9 Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Wed, 9 Jan 2019 13:45:56 -0800
Subject: [PATCH 0444/2345] Fixed target name in documentation

PiperOrigin-RevId: 228580269
---
 tensorflow/lite/experimental/micro/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/experimental/micro/README.md b/tensorflow/lite/experimental/micro/README.md
index 173f91fe90..931c8d9d31 100644
--- a/tensorflow/lite/experimental/micro/README.md
+++ b/tensorflow/lite/experimental/micro/README.md
@@ -381,7 +381,7 @@ auto-generated for any target you can compile using the main Make system, using
 a command like this (making sure you've run `download_dependencies.sh` first):
 
 ```
-make -f tensorflow/lite/experimental/micro/tools/make/Makefile TARGET=mbed TAGS="CMSIS disco_f746ng" generate_micro_speech_main_test_mbed_project
+make -f tensorflow/lite/experimental/micro/tools/make/Makefile TARGET=mbed TAGS="CMSIS disco_f746ng" generate_micro_speech_mbed_project
 ```
 
 This will create a folder in
-- 
GitLab


From 9545dccbd749a626c5451421b090ed1c70125d6a Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Wed, 9 Jan 2019 13:48:27 -0800
Subject: [PATCH 0445/2345] graph_partition: Fix absl::Hash related issue in
 Windows build. PiperOrigin-RevId: 228580775

---
 tensorflow/core/graph/graph_partition.cc | 35 ++++++++++++++++--------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/tensorflow/core/graph/graph_partition.cc b/tensorflow/core/graph/graph_partition.cc
index be0cac50a1..00c7a5b091 100644
--- a/tensorflow/core/graph/graph_partition.cc
+++ b/tensorflow/core/graph/graph_partition.cc
@@ -67,16 +67,14 @@ struct DupRecvKey {
                       c.recv_output_on_host);
   }
 
-  friend bool operator==(const DupRecvKey& x, const DupRecvKey& y);
+  friend bool operator==(const DupRecvKey& x, const DupRecvKey& y) {
+    return (x.src_node_id == y.src_node_id) &&
+           (x.src_output_slot == y.src_output_slot) &&
+           (x.dst_graph == y.dst_graph) &&
+           (x.recv_output_on_host == y.recv_output_on_host);
+  }
 };
 
-bool operator==(const DupRecvKey& x, const DupRecvKey& y) {
-  return (x.src_node_id == y.src_node_id) &&
-         (x.src_output_slot == y.src_output_slot) &&
-         (x.dst_graph == y.dst_graph) &&
-         (x.recv_output_on_host == y.recv_output_on_host);
-}
-
 // struct used to store the recvs, so that start times can be properly updated
 struct RecvInfo {
   NodeDef* recv;
@@ -88,7 +86,22 @@ typedef absl::flat_hash_map<DupRecvKey, RecvInfo> DupRecvTable;
 
 // A map used to store memory types for the inputs/outputs of every node.
 // The key is a pair of ints consisting of a node id and input/output index.
-typedef absl::flat_hash_map<std::pair<int, int>, MemoryType> MemoryTypeMap;
+// TODO(power): migrate back to std::pair when absl::Hash is fixed for MSVC.
+struct NodePort {
+  int node_id;
+  int index;
+
+  friend bool operator==(const NodePort& x, const NodePort& y) {
+    return x.node_id == y.node_id && x.index == y.index;
+  }
+
+  template <typename H>
+  friend H AbslHashValue(H h, const NodePort& c) {
+    return H::combine(std::move(h), c.node_id, c.index);
+  }
+};
+
+typedef absl::flat_hash_map<NodePort, MemoryType> MemoryTypeMap;
 
 // We collect the following information about the graph before performing
 // graph partitioning.
@@ -552,10 +565,10 @@ Status BuildMemoryDeviceInfo(const Graph& g, GraphInfo* info) {
 
     int node_id = node->id();
     info->device_types[node_id] = DeviceType(parsed.type);
-    for (size_t i = 0; i < input_memory_types.size(); ++i) {
+    for (int i = 0; i < input_memory_types.size(); ++i) {
       info->input_types[{node_id, i}] = input_memory_types[i];
     }
-    for (size_t i = 0; i < output_memory_types.size(); ++i) {
+    for (int i = 0; i < output_memory_types.size(); ++i) {
       info->output_types[{node_id, i}] = output_memory_types[i];
     }
   }
-- 
GitLab


From 2b2c042367675fdac67a12bc9bae9689babae07d Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Wed, 9 Jan 2019 13:51:30 -0800
Subject: [PATCH 0446/2345] [Grappler] Add identity nodes to read function
 outputs in GrapplerFunctionItem instantiations

PiperOrigin-RevId: 228581341
---
 .../core/grappler/costs/graph_properties.cc   |   4 +-
 .../optimizers/constant_folding_test.cc       |   2 +-
 .../grappler/optimizers/function_optimizer.cc | 129 +++++++++---
 .../optimizers/function_optimizer_test.cc     |   2 +-
 .../optimizers/meta_optimizer_test.cc         | 111 ++++++++--
 tensorflow/core/grappler/utils/BUILD          |   1 +
 tensorflow/core/grappler/utils/functions.cc   | 196 ++++++++++++------
 tensorflow/core/grappler/utils/functions.h    |  83 ++++----
 .../core/grappler/utils/functions_test.cc     | 105 +++++++---
 .../core/grappler/utils/grappler_test.cc      |  74 ++++---
 .../core/grappler/utils/grappler_test.h       |   7 +
 11 files changed, 513 insertions(+), 201 deletions(-)

diff --git a/tensorflow/core/grappler/costs/graph_properties.cc b/tensorflow/core/grappler/costs/graph_properties.cc
index 1ed96ca12a..863a5087ae 100644
--- a/tensorflow/core/grappler/costs/graph_properties.cc
+++ b/tensorflow/core/grappler/costs/graph_properties.cc
@@ -669,7 +669,7 @@ class SymbolicShapeRefiner {
     ctx->output_tensor_protos.resize(grappler_function_item.output_size(),
                                      nullptr);
     for (auto const& out_arg : grappler_function_item.outputs()) {
-      if (out_arg.output_tensors.size() > 1) {
+      if (out_arg.output_nodes.size() > 1) {
         // TODO(jmdecker): Handle case of multiple output tensors
         return errors::Unimplemented(
             "Output arguments with multiple output tensors are not yet "
@@ -678,7 +678,7 @@ class SymbolicShapeRefiner {
 
       // It is guaranteed that output_tensors does not contain any control
       // inputs, so port_id >= 0.
-      TensorId out_tensor = ParseTensorName(out_arg.output_tensors[0]);
+      TensorId out_tensor = ParseTensorName(out_arg.output_nodes[0]);
 
       const NodeDef* retnode = gv.GetNode(out_tensor.node());
       if (retnode == nullptr) {
diff --git a/tensorflow/core/grappler/optimizers/constant_folding_test.cc b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
index 192f48272f..d7cabf5a8b 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding_test.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
@@ -1601,7 +1601,7 @@ TEST_F(ConstantFoldingTest, SplitRemoval) {
   AddNode("split_dim", "Const", {}, {}, &want);
   AddNode("s1", "Identity", {"in1", AsControlDependency("split_dim")}, {},
           &want);
-  AddNode("s2", "Split", {"in2", "split_dim"}, {}, &want);
+  AddNode("s2", "Split", {"split_dim", "in2"}, {}, &want);
   AddNode("out", "Add", {"s1", "s2"}, {}, &want);
 
   CompareGraphs(want, got);
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index d074676c3d..e9c30fee25 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -806,8 +806,17 @@ Status SpecializeFunction(const NodeDef& func_node, const FunctionDef& func,
   // update outputs for the fetch nodes, so we just skip them.
   std::vector<std::pair<int, int>> output_mapping;
   if (!signature.is_in_fetch_set) {
-    TF_RETURN_IF_ERROR(
-        RemoveUnusedOutputs(signature.active_outputs, &item, &output_mapping));
+    int num_func_outputs = 0;
+    for (const auto& out_arg : item.outputs()) {
+      num_func_outputs += out_arg.output_nodes.size();
+    }
+
+    absl::flat_hash_set<int> remove;
+    for (int i = 0; i < num_func_outputs; ++i) {
+      if (!signature.active_outputs.count(i)) remove.insert(i);
+    }
+
+    TF_RETURN_IF_ERROR(RemoveFunctionOutputs(remove, &item, &output_mapping));
   }
 
   // TODO(ezhulenev): Push down known input shapes.
@@ -962,8 +971,10 @@ NodeDef InlinedFunctionInputsNode(const NodeDef& func_node,
 
 // Create an IdentityN node to hook the function outputs to: this ensures that
 // the function body is fully evaluated before its fanout gets scheduled.
-NodeDef InlinedFunctionOutputsNode(const NodeDef& func_node,
-                                   const GrapplerFunctionItem& item) {
+NodeDef InlinedFunctionOutputsNode(
+    const NodeDef& func_node, const GrapplerFunctionItem& item,
+    const absl::flat_hash_map<absl::string_view, absl::string_view>
+        output_tensors) {
   NodeDef outputs;
   outputs.set_name(func_node.name());
   outputs.set_op("IdentityN");
@@ -972,7 +983,8 @@ NodeDef InlinedFunctionOutputsNode(const NodeDef& func_node,
       (*outputs.mutable_attr())["T"].mutable_list();
 
   for (const OutputArgExpansion& output_arg : item.outputs()) {
-    for (const string& output_tensor : output_arg.output_tensors) {
+    for (const string& output_node : output_arg.output_nodes) {
+      const absl::string_view output_tensor = output_tensors.at(output_node);
       type_list->add_type(output_arg.data_type);
       outputs.add_input(strings::StrCat(func_node.name(), "/", output_tensor));
     }
@@ -1004,29 +1016,51 @@ Status InlineDirectFunctionCall(const NodeDef& func_node,
   }
 
   // Mapping from input placeholder name to function input position.
-  int idx = 0;
-  absl::flat_hash_map<string, int> input_placeholders_idx;
+  absl::flat_hash_map<absl::string_view, int> input_placeholders_idx;
   for (const InputArgExpansion& input_arg : item.inputs()) {
     for (const string& placeholder : input_arg.placeholders) {
-      input_placeholders_idx[placeholder] = idx++;
+      const int idx = input_placeholders_idx.size();
+      input_placeholders_idx[placeholder] = idx;
+    }
+  }
+
+  // Bypass identity nodes added to the graph in place of function outputs.
+  absl::flat_hash_set<absl::string_view> output_nodes;
+  for (const OutputArgExpansion& output_arg : item.outputs()) {
+    for (const string& output_node : output_arg.output_nodes) {
+      output_nodes.insert(output_node);
     }
   }
 
+  // For each function output value we added an identity node that reads the
+  // tensor from one of the function body nodes. When we inline function into
+  // the main graph we want to bypass these nodes, so we keep a mapping from
+  // 'output node name' -> 'output tensor name'.
+  absl::flat_hash_map<absl::string_view, absl::string_view> output_tensors;
+
   // Hook inlined function inputs to IdentityN node.
   NodeDef* func_inputs = optimized_graph->add_node();
   *func_inputs = InlinedFunctionInputsNode(func_node, item);
 
   for (NodeDef& func_body_node : *item.mutable_function_body().mutable_node()) {
-    if (item.IsInputPlaceholder(func_body_node.name())) {
-      // Turn input placeholders into identity nodes.
+    const string& node_name = func_body_node.name();
+
+    // Skip output identity node, and update a mapping to the output tensor.
+    if (IsIdentity(func_body_node) && output_nodes.count(node_name)) {
+      output_tensors.emplace(node_name, func_body_node.input(0));
+      continue;
+    }
+
+    // Turn placeholders added in place of input arguments into identity nodes.
+    const auto input_placeholder_idx = input_placeholders_idx.find(node_name);
+    if (input_placeholder_idx != input_placeholders_idx.end()) {
       CHECK_EQ(0, func_body_node.input_size());
       func_body_node.set_op("Identity");
       (*func_body_node.mutable_attr())["T"] = func_body_node.attr().at("dtype");
       func_body_node.mutable_attr()->erase("dtype");
       func_body_node.mutable_attr()->erase("shape");
-      int input_idx = input_placeholders_idx[func_body_node.name()];
-      func_body_node.add_input(
-          strings::StrCat(func_inputs->name(), ":", input_idx));
+      func_body_node.add_input(strings::StrCat(func_inputs->name(), ":",
+                                               input_placeholder_idx->second));
     } else {
       // Update the input names if any.
       for (string& input : *func_body_node.mutable_input()) {
@@ -1082,9 +1116,12 @@ Status InlineDirectFunctionCall(const NodeDef& func_node,
     }
   }
 
+  DCHECK(output_tensors.size() == item.output_size())
+      << "Each function output must be mapped to an output tensor";
+
   // Hook inlined function outputs to IdentityN node.
   NodeDef* func_outputs = optimized_graph->add_node();
-  *func_outputs = InlinedFunctionOutputsNode(func_node, item);
+  *func_outputs = InlinedFunctionOutputsNode(func_node, item, output_tensors);
 
   return Status::OK();
 }
@@ -1363,11 +1400,11 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
   }
 
   // Mapping from input placeholder name to function input position.
-  int idx = 0;
   absl::flat_hash_map<absl::string_view, int> input_placeholders_idx;
   for (const InputArgExpansion& input_arg : item.inputs()) {
     for (const string& placeholder : input_arg.placeholders) {
-      input_placeholders_idx[placeholder] = idx++;
+      const int idx = input_placeholders_idx.size();
+      input_placeholders_idx[placeholder] = idx;
     }
   }
 
@@ -1378,8 +1415,11 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
   // same device as their corresponding input nodes.
 
   for (NodeDef& func_body_node : *item.graph.mutable_node()) {
-    if (item.IsInputPlaceholder(func_body_node.name())) {
-      const int input_idx = input_placeholders_idx[func_body_node.name()];
+    const auto input_placeholder_idx =
+        input_placeholders_idx.find(func_body_node.name());
+
+    if (input_placeholder_idx != input_placeholders_idx.end()) {
+      const int input_idx = input_placeholder_idx->second;
       const GraphView::OutputPort output_port =
           ctx->graph_view().GetRegularFanin({&func_node, input_idx});
 
@@ -1441,16 +1481,23 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
   // optimized graph: turn placeholders into identities, update nodes
   // connectivity, etc...
 
+  const auto inlined_node_name = [&func_node](const string& name) -> string {
+    return AddPrefixToNodeName(name, /*prefix=*/func_node.name());
+  };
+
   for (NodeDef& func_body_node : *placed_graph_def.mutable_node()) {
-    if (item.IsInputPlaceholder(func_body_node.name())) {
-      // Turn input placeholders into identity node.
+    const string& node_name = func_body_node.name();
+
+    // Turn placeholders added in place of input arguments into identity nodes.
+    const auto input_placeholder_idx = input_placeholders_idx.find(node_name);
+    if (input_placeholder_idx != input_placeholders_idx.end()) {
       DCHECK_EQ(0, func_body_node.input_size());
       func_body_node.set_op("Identity");
       (*func_body_node.mutable_attr())["T"] = func_body_node.attr().at("dtype");
       func_body_node.mutable_attr()->erase("dtype");
       func_body_node.mutable_attr()->erase("shape");
-      const int input_idx = input_placeholders_idx[func_body_node.name()];
-      func_body_node.add_input(strings::StrCat(inputs[input_idx].ToString()));
+      const int input_idx = input_placeholder_idx->second;
+      func_body_node.add_input(inputs[input_idx].ToString());
 
       // All side effects must happen before inputs can start executing.
       for (const string& hb_node : happens_before) {
@@ -1460,7 +1507,7 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
     } else {
       // Update inputs of the regular function body nodes.
       for (string& input : *func_body_node.mutable_input()) {
-        input = AddPrefixToNodeName(input, /*prefix=*/func_node.name());
+        input = inlined_node_name(input);
       }
       if (func_body_node.input_size() == 0 && !empty_inputs_hook.empty()) {
         *func_body_node.add_input() = empty_inputs_hook[0];
@@ -1494,7 +1541,7 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
   for (NodeDef& func_body_node : *placed_graph_def.mutable_node()) {
     if (!IsFreeOfSideEffect(func_body_node, &ctx->function_library())) {
       int num_fanouts = placed_graph_view.NumFanouts(
-          func_body_node, /*include_controlling_nodes=*/true);
+          func_body_node, /*include_controlled_nodes=*/true);
 
       // If the node doesn't have any outgoing edges and we do not have any
       // nodes in the `happens_after` set, we can't inline a function and
@@ -1514,11 +1561,36 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
     }
   }
 
+  // Identity nodes added to the function body in place of function outputs.
+  absl::flat_hash_set<string> output_nodes;
+  for (const OutputArgExpansion& output_arg : item.outputs()) {
+    for (const string& output_node : output_arg.output_nodes) {
+      output_nodes.insert(inlined_node_name(output_node));
+    }
+  }
+
+  // For each function output value we added an identity node that reads the
+  // tensor from one of the function body nodes. When we inline function into
+  // the main graph we want to bypass these nodes, so we keep a mapping from
+  // 'output node name' -> 'output tensor name'.
+  absl::flat_hash_map<string, string> output_tensors;
+
   // Move all the nodes to the optimized graph after successful preprocessing.
   for (NodeDef& func_body_node : *placed_graph_def.mutable_node()) {
+    const string& node_name = func_body_node.name();
+
+    // Skip output identity node, and add a mapping to the output tensor.
+    if (IsIdentity(func_body_node) && output_nodes.count(node_name)) {
+      output_tensors.emplace(node_name, func_body_node.input(0));
+      continue;
+    }
+
     optimized_graph->add_node()->Swap(&func_body_node);
   }
 
+  DCHECK(output_tensors.size() == item.output_size())
+      << "Each function output must be mapped to an output tensor";
+
   // TODO(ezhulenev): Inline nested indirect function calls.
 
   // Indirect function call is fully inlined into the optimized graph, and we do
@@ -1526,10 +1598,13 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
   // mapping from old output tensors, to the outputs of inlined nodes.
   int output_idx = 0;
   for (const OutputArgExpansion& output : item.outputs()) {
-    for (const string& output_tensor : output.output_tensors) {
+    for (const string& output_node : output.output_nodes) {
+      const string inlined_output = inlined_node_name(output_node);
+      const string& output_tensor = output_tensors.at(inlined_output);
+
       const SafeTensorId from_tensor(func_node.name(), output_idx++);
-      const SafeTensorId to_tensor = ParseTensorName(
-          AddPrefixToNodeName(output_tensor, /*prefix=*/func_node.name()));
+      const SafeTensorId to_tensor = ParseTensorName(output_tensor);
+
       ctx->AddTensorMapping(from_tensor, to_tensor);
     }
   }
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
index 79da7dfa2d..cebd002bed 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
@@ -660,7 +660,7 @@ TEST_F(FunctionOptimizerTest, InlineSymbolicGradient_IdentityFunc) {
   test::ExpectTensorEqual<float>(expected[0], optimized[0]);
 }
 
-TEST_F(FunctionOptimizerTest, InlineSymbolicGradient_NoInlineFunc) {
+TEST_F(FunctionOptimizerTest, InlineSymbolicGradientNoInlineFunc) {
   FunctionOptimizer optimizer(RewriterConfig::ON);
 
   FunctionDef func = FunctionDefHelper::Define(
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc b/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
index 12db5d6ca9..a061f6194a 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
@@ -231,7 +231,7 @@ TEST_F(MetaOptimizerTest, RunToggleOptimizersAndCustomGraphOptimizerTwice) {
 TEST_F(MetaOptimizerTest, OptimizeFunctionLibrary) {
   using test::function::NDef;
 
-  // Enable ony function optimization.
+  // Enable only function optimization.
   ConfigProto config_proto;
   auto& rewriter_config =
       *config_proto.mutable_graph_options()->mutable_rewrite_options();
@@ -300,7 +300,7 @@ TEST_F(MetaOptimizerTest, OptimizeFunctionLibrary) {
                                            output.library());
 
   // Specialized and optimized functions should be added to the graph.
-  EXPECT_EQ(6, optimized_flib.num_functions());
+  EXPECT_EQ(5, optimized_flib.num_functions());
 
   // Get a specialized function name.
   const auto specialized_name = [](const string& fn, const string& node,
@@ -314,25 +314,22 @@ TEST_F(MetaOptimizerTest, OptimizeFunctionLibrary) {
       specialized_name("MyQuadratic", "quadratic", "tf_graph");
 
   // MySquare should be specialized and optimized for 3 instantiations:
-  //   1. 'square' node in the main graph
-  //   2. 'square' node in the MyQuadratic specialization (not in a fetch set)
-  //   3. 'quadratic' node in the MyQuadratic specialization (is in a fetch set)
+  //   1.  'square' node in the main graph
+  //   2.  'square' node in the MyQuadratic specialization
+  //   3*. 'quadratic' node in the MyQuadratic specialization
+  //        has identical instantiation context to #2
 
   const string optimized_1 = specialized_name("MySquare", "square", "tf_graph");
   const string optimized_2 =
       specialized_name("MySquare", "square", optimized_0);
-  const string optimized_3 =
-      specialized_name("MySquare", "quadratic", optimized_0);
 
   const FunctionDef* optimized_func_0 = optimized_flib.Find(optimized_0);
   const FunctionDef* optimized_func_1 = optimized_flib.Find(optimized_1);
   const FunctionDef* optimized_func_2 = optimized_flib.Find(optimized_2);
-  const FunctionDef* optimized_func_3 = optimized_flib.Find(optimized_3);
 
   ASSERT_NE(optimized_func_0, nullptr);
   ASSERT_NE(optimized_func_1, nullptr);
   ASSERT_NE(optimized_func_2, nullptr);
-  ASSERT_NE(optimized_func_3, nullptr);
 
   // Graph should call optimized function.
   int count = 0;
@@ -351,13 +348,13 @@ TEST_F(MetaOptimizerTest, OptimizeFunctionLibrary) {
     if (node.name() == "square" && ++count) {
       EXPECT_EQ(optimized_2, node.op());
     } else if (node.name() == "quadratic" && ++count) {
-      EXPECT_EQ(optimized_3, node.op());
+      EXPECT_EQ(optimized_2, node.op());
     }
   }
   EXPECT_EQ(2, count);
 
-  const std::vector<const FunctionDef*> optimized_funcs = {
-      optimized_func_1, optimized_func_2, optimized_func_3};
+  const std::vector<const FunctionDef*> optimized_funcs = {optimized_func_1,
+                                                           optimized_func_2};
 
   // MyMul should be inlined into all optimized versions of MySquare.
   for (const FunctionDef* optimized_func : optimized_funcs) {
@@ -403,6 +400,96 @@ TEST_F(MetaOptimizerTest, OptimizeFunctionLibrary) {
   test::ExpectTensorEqual<int>(tensors_expected[1], tensors[1]);
 }
 
+TEST_F(MetaOptimizerTest, OptimizeFunctionLibraryPruneUnusedOutputs) {
+  using test::function::NDef;
+
+  ConfigProto config_proto;
+  MetaOptimizer optimizer(nullptr, config_proto);
+
+  // MyMul computes x*y three times and has three output values.
+  FunctionDef my_mul = FunctionDefHelper::Create(
+      "MyMul", {"x:T", "y:T"}, {"z0:T", "z1:T", "z2:T"}, {"T: {float, int32}"},
+      {{{"output0"}, "Mul", {"x", "y"}, {{"T", "$T"}}},
+       {{"output1"}, "Mul", {"x", "y"}, {{"T", "$T"}}},
+       {{"output2"}, "Mul", {"x", "y"}, {{"T", "$T"}}}},
+      /* Mapping between function returns and function node outputs. */
+      {{"z0", "output0:z:0"}, {"z1", "output1:z:0"}, {"z2", "output2:z:0"}});
+
+  // Call MyMyl and forward all three outputs.
+  FunctionDef my_fwd = FunctionDefHelper::Create(
+      "Fwd", {"x:T", "y:T"}, {"z0:T", "z1:T", "z2:T"}, {"T: {float, int32}"},
+      {{{"output"}, "MyMul", {"x", "y"}, {{"T", "$T"}}}},
+      /* Mapping between function returns and function node outputs. */
+      {{"z0", "output:z0:0"}, {"z1", "output:z1:0"}, {"z2", "output:z2:0"}});
+
+  // Mark both functions as `_noinline` to trigger specialization.
+  (*my_mul.mutable_attr())["_noinline"].set_b(true);
+  (*my_fwd.mutable_attr())["_noinline"].set_b(true);
+  std::vector<FunctionDef> function_library = {my_mul, my_fwd};
+
+  // Tensorflow graph:
+  //   a = Placeholder[T=float]
+  //   b = Placeholder[T=float]
+  //   fwd = Fwd(a, b)
+  //
+  // Fetch fwd:2 via Identity node.
+  GrapplerItem item;
+  item.id = "tf_graph";
+  item.fetch = {"ret"};
+  item.graph = test::function::GDef(
+      {NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("b", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("fwd", "Fwd", {"a", "b"}, {{"T", DT_FLOAT}}, kDevice),
+       NDef("ret", "Identity", {"fwd:2"}, {{"T", DT_FLOAT}}, kDevice)},
+      function_library);
+
+  GraphDef output;
+  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &output));
+
+  FunctionLibraryDefinition optimized_flib(OpRegistry::Global(),
+                                           output.library());
+
+  // Specialized functions should be added to the graph.
+  EXPECT_EQ(3, optimized_flib.num_functions());
+
+  // Expected names of the specialized functions.
+  const string specialized_my_fwd = "Fwd_specialized_for_fwd_at_tf_graph";
+  const string specialized_my_mul =
+      absl::StrCat("MyMul_specialized_for_output_at_", specialized_my_fwd);
+
+  // Specialized MyMul should have just one output argument.
+  FunctionDef expected_my_mul = FunctionDefHelper::Create(
+      specialized_my_mul, {"x:float", "y:float"}, {"z2:float"}, {},
+      {{{"output2"}, "Mul", {"x", "y"}, {{"T", DT_FLOAT}}}},
+      /* Mapping between function returns and function node outputs. */
+      {{"z2", "output2:z:0"}});
+
+  // Specialized Fwd should also have just one output argument.
+  FunctionDef expected_my_fwd = FunctionDefHelper::Create(
+      specialized_my_fwd, {"x:float", "y:float"}, {"z2:float"}, {},
+      {{{"output"}, specialized_my_mul, {"x", "y"}, {{"T", DT_FLOAT}}}},
+      /* Mapping between function returns and function node outputs. */
+      {{"z2", "output:z2:0"}});
+
+  const FunctionDef* my_mul_spec = optimized_flib.Find(specialized_my_mul);
+  const FunctionDef* my_fwd_spec = optimized_flib.Find(specialized_my_fwd);
+
+  ASSERT_NE(my_mul_spec, nullptr);
+  ASSERT_NE(my_fwd_spec, nullptr);
+
+  CompareFunctions(expected_my_mul, *my_mul_spec);
+  CompareFunctions(expected_my_fwd, *my_fwd_spec);
+
+  item.feed.emplace_back("a", test::AsScalar<float>(2.0f));
+  item.feed.emplace_back("b", test::AsScalar<float>(4.0f));
+  auto tensors_expected = EvaluateFetchNodes(item);
+
+  GrapplerItem optimized = item.WithGraph(std::move(output));
+  auto tensors = EvaluateFetchNodes(optimized);
+
+  test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
+}
+
 TEST_F(MetaOptimizerTest, OptimizeFunctionLibraryPruneFunctionBody) {
   using test::function::NDef;
 
diff --git a/tensorflow/core/grappler/utils/BUILD b/tensorflow/core/grappler/utils/BUILD
index ff15541cdc..1fd0a02b65 100644
--- a/tensorflow/core/grappler/utils/BUILD
+++ b/tensorflow/core/grappler/utils/BUILD
@@ -182,6 +182,7 @@ cc_library(
         "@com_google_absl//absl/algorithm:container",
         "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/container:flat_hash_set",
+        "@com_google_absl//absl/container:inlined_vector",
         "@com_google_absl//absl/strings",
     ],
 )
diff --git a/tensorflow/core/grappler/utils/functions.cc b/tensorflow/core/grappler/utils/functions.cc
index 7c2180ae40..150728d030 100644
--- a/tensorflow/core/grappler/utils/functions.cc
+++ b/tensorflow/core/grappler/utils/functions.cc
@@ -328,13 +328,12 @@ GrapplerFunctionItem::GrapplerFunctionItem(
   for (const InputArgExpansion& input_arg : input_arg_expansions_) {
     for (const string& placeholder : input_arg.placeholders) {
       feed.push_back({placeholder, Tensor()});
-      input_arg_placeholders_.insert(placeholder);
     }
   }
   // Fill the fetch nodes with outputs.
   for (const OutputArgExpansion& output_arg : output_arg_expansions_) {
-    for (const string& output_tensor : output_arg.output_tensors) {
-      fetch.push_back(output_tensor);
+    for (const string& output_node : output_arg.output_nodes) {
+      fetch.push_back(output_node);
     }
   }
 
@@ -357,11 +356,6 @@ const std::size_t GrapplerFunctionItem::input_size() const {
   return input_arg_expansions_.size();
 }
 
-bool GrapplerFunctionItem::IsInputPlaceholder(const string& node_name) const {
-  return input_arg_placeholders_.find(node_name) !=
-         input_arg_placeholders_.end();
-}
-
 const std::vector<OutputArgExpansion>& GrapplerFunctionItem::outputs() const {
   return output_arg_expansions_;
 }
@@ -514,12 +508,18 @@ Status MakeGrapplerFunctionItem(const FunctionDef& func,
 
   // TODO(ezhulenev): support functions with tensor sequence inputs/outputs
 
-  // Make sure that there is no tensor sequences in outputs
+  // Make sure that there are no tensor lists in inputs or outputs.
+  for (const OpDef::ArgDef& input : signature.input_arg()) {
+    if (!input.type_list_attr().empty() || !input.number_attr().empty()) {
+      return errors::InvalidArgument(
+          "Inputs with lists of tensors are not supported. Input: ",
+          input.name());
+    }
+  }
   for (const OpDef::ArgDef& output : signature.output_arg()) {
     if (!output.type_list_attr().empty() || !output.number_attr().empty()) {
       return errors::InvalidArgument(
-          "Outputs with sequence of tensors are not supported. Unsupported "
-          "output: ",
+          "Outputs with lists of tensors are not supported. Output: ",
           output.name());
     }
   }
@@ -529,13 +529,6 @@ Status MakeGrapplerFunctionItem(const FunctionDef& func,
 
   // For each input argument create a placeholder in function body.
   for (const OpDef::ArgDef& input : signature.input_arg()) {
-    if (!input.type_list_attr().empty() || !input.number_attr().empty()) {
-      return errors::InvalidArgument(
-          "Inputs with sequence of tensors are not supported. Unsupported "
-          "input: ",
-          input.name());
-    }
-
     DataType input_data_type;
     TF_RETURN_IF_ERROR(instantiation.GetArgType(input, &input_data_type));
 
@@ -554,8 +547,25 @@ Status MakeGrapplerFunctionItem(const FunctionDef& func,
     inputs.push_back(std::move(input_expansion));
   }
 
-  // Add all function nodes to the function body
+  // Keep names of all nodes in the function body to guarantee that we do not
+  // add an identity with a duplicate name.
+  absl::flat_hash_set<absl::string_view> func_body_nodes;
+
+  // Generate unique output node name: "${out_arg_name}_output_node_${index}".
+  const auto output_node_name = [&func_body_nodes](const OpDef::ArgDef& out,
+                                                   int index) -> string {
+    string name = absl::StrCat(out.name(), "_output_node_", index);
+    int i = 1;
+    while (func_body_nodes.find(name) != func_body_nodes.end()) {
+      name = absl::StrCat(out.name(), "_output_node_", index, "_", i++);
+    }
+    return name;
+  };
+
+  // Add all function nodes to the function body.
   for (const NodeDef& func_def_node : func.node_def()) {
+    func_body_nodes.insert(func_def_node.name());
+
     NodeDef* new_node = function_body.add_node();
     *new_node = func_def_node;
 
@@ -578,8 +588,13 @@ Status MakeGrapplerFunctionItem(const FunctionDef& func,
 
   std::vector<OutputArgExpansion> outputs;
   outputs.reserve(signature.output_arg_size());
-  // Add function outputs
+
+  // For each function output argument we create an Identity node in the
+  // function body, that reads output tensor from the function body node.
   for (const OpDef::ArgDef& out : signature.output_arg()) {
+    DataType output_data_type;
+    TF_RETURN_IF_ERROR(instantiation.GetArgType(out, &output_data_type));
+
     std::vector<string> output_tensors;
     auto ret = func.ret().find(out.name());
     TF_RETURN_IF_ERROR(
@@ -589,13 +604,23 @@ Status MakeGrapplerFunctionItem(const FunctionDef& func,
             // Otherwise output must be one of the function inputs
             : connectivity.ExpandFunctionDefInput(out.name(), &output_tensors));
 
-    DataType output_data_type;
-    TF_RETURN_IF_ERROR(instantiation.GetArgType(out, &output_data_type));
+    absl::InlinedVector<string, 1> output_nodes;
+    for (int i = 0; i < output_tensors.size(); ++i) {
+      const string& output_tensor = output_tensors[i];
+
+      NodeDef* identity = function_body.add_node();
+      identity->set_name(output_node_name(out, i));
+      identity->set_op("Identity");
+      (*identity->mutable_attr())["T"].set_type(output_data_type);
+      identity->add_input(output_tensor);
+
+      output_nodes.push_back(identity->name());
+    }
 
     OutputArgExpansion output{/*output_name=*/out.name(),
                               /*data_type=*/output_data_type,
                               /*is_ref=*/out.is_ref(),
-                              /*output_tensors=*/std::move(output_tensors)};
+                              /*output_nodes=*/std::move(output_nodes)};
     outputs.push_back(std::move(output));
   }
 
@@ -663,7 +688,6 @@ Status ReplaceInputWithConst(const NodeDef& input_const, int input_index,
 
   // Delete placeholder from input expansion.
   string placeholder_name = input_arg_expansion->placeholders[placeholder_idx];
-  item->input_arg_placeholders_.erase(placeholder_name);
   input_arg_expansion->placeholders.erase(
       input_arg_expansion->placeholders.begin() + placeholder_idx);
 
@@ -687,43 +711,46 @@ Status ReplaceInputWithConst(const NodeDef& input_const, int input_index,
   return Status::OK();
 }
 
-Status RemoveUnusedOutputs(const absl::flat_hash_set<int>& active_outputs,
-                           GrapplerFunctionItem* item,
-                           std::vector<std::pair<int, int>>* output_mapping) {
+Status RemoveFunctionOutputs(const absl::flat_hash_set<int>& remove_outputs,
+                             GrapplerFunctionItem* item,
+                             std::vector<std::pair<int, int>>* output_mapping) {
   DCHECK(output_mapping->empty());
 
-  // Do some sanity checking of the active outputs positions.
-  for (int active_output : active_outputs) {
-    if (active_output < 0 || active_output >= item->output_size()) {
+  // Code below assumes that we do not support tensor list outputs and there is
+  // a 1-to-1 mapping between output tensor and output argument expansion.
+  for (const OutputArgExpansion& out_arg : item->outputs()) {
+    DCHECK(out_arg.output_nodes.size() == 1)
+        << "Output arg expansion must have single output";
+  }
+
+  // Do some sanity checking of the removed outputs positions.
+  for (int remove_output : remove_outputs) {
+    if (remove_output < 0 || remove_output >= item->output_size()) {
       return errors::InvalidArgument(
-          "Active output position is out of bound: active_output=",
-          active_output, " num_output_args=", item->output_size());
+          "Function output index is out of bound: index=", remove_output,
+          " max_output_index=", item->output_size());
     }
   }
 
-  absl::flat_hash_set<const OutputArgExpansion*> unused_output_args;
-
-  const auto is_unused_output_arg = [&](const OutputArgExpansion& output) {
-    return unused_output_args.find(&output) != unused_output_args.end();
+  absl::flat_hash_set<const OutputArgExpansion*> remove_output_args;
+  const auto is_remove_output_arg = [&](const OutputArgExpansion& output) {
+    return remove_output_args.find(&output) != remove_output_args.end();
   };
 
   for (int i = 0; i < item->output_size(); ++i) {
     const OutputArgExpansion& output = item->output(i);
-    DCHECK(output.output_tensors.size() == 1)
-        << "Output arg expansion must have single tensor";
-
-    if (active_outputs.find(i) == active_outputs.end()) {
-      VLOG(3) << "Remove unused output: output_name=" << output.output_name
-              << " output_position=" << i;
-      unused_output_args.insert(&output);
-    } else if (!unused_output_args.empty()) {
+    if (remove_outputs.find(i) != remove_outputs.end()) {
+      VLOG(3) << "Remove functions output: output_name=" << output.output_name
+              << "(index = " << i << ")";
+      remove_output_args.insert(&output);
+    } else if (!remove_output_args.empty()) {
       // Add output mapping only if output position changed.
-      output_mapping->push_back({i, i - unused_output_args.size()});
+      output_mapping->push_back({i, i - remove_output_args.size()});
     }
   }
 
   auto& o = item->output_arg_expansions_;
-  o.erase(std::remove_if(o.begin(), o.end(), is_unused_output_arg), o.end());
+  o.erase(std::remove_if(o.begin(), o.end(), is_remove_output_arg), o.end());
 
   return Status::OK();
 }
@@ -735,6 +762,55 @@ Status MakeFunctionDef(const GrapplerFunctionItem& item,
   func->mutable_signature()->set_description(item.description());
   func->mutable_signature()->set_is_stateful(item.is_stateful());
 
+  // Keep track of placeholders that were added to the graph in place of
+  // expanded function input arguments.
+  absl::flat_hash_set<absl::string_view> input_placeholders;
+  for (const InputArgExpansion& input_arg : item.inputs()) {
+    for (const string& placeholder : input_arg.placeholders) {
+      input_placeholders.insert(placeholder);
+    }
+  }
+
+  // Keep track of identity nodes that were added to the graph in place of
+  // expanded function output arguments.
+  absl::flat_hash_set<absl::string_view> output_nodes;
+  for (const OutputArgExpansion& output_arg : item.outputs()) {
+    for (const string& output_node : output_arg.output_nodes) {
+      output_nodes.insert(output_node);
+    }
+  }
+
+  // If the output identity node was not modified by any optimizer, we can
+  // bypass it and returns the function value from its input.
+  absl::flat_hash_map<absl::string_view, string> output_tensors;
+  for (const NodeDef& func_body_node : item.function_body().node()) {
+    if (!IsIdentity(func_body_node)) continue;
+
+    const string& node_name = func_body_node.name();
+    if (output_nodes.find(node_name) != output_nodes.end()) {
+      // Grappler optimizers might optimize nodes in the fanin of the output
+      // node, and forward their control dependencies. We can't express control
+      // dependencies in a function signature, so we have to keep the node.
+      if (func_body_node.input_size() == 1) {
+        VLOG(3) << "Bypass function output node: " << node_name << " -> "
+                << func_body_node.input(0);
+        output_tensors.emplace(node_name, func_body_node.input(0));
+      } else {
+        VLOG(3) << "Keep function output node: " << node_name;
+      }
+    }
+  }
+
+  // Return output tensor name (input of the output node) if it's safe to bypass
+  // output node, otherwise returns the output node name.
+  const auto output_tensor =
+      [&output_tensors](const OutputArgExpansion& output_arg) -> const string& {
+    const string& output_node = output_arg.output_nodes[0];
+    const auto is_output_tensor = output_tensors.find(output_node);
+    return is_output_tensor == output_tensors.end() ? output_node
+                                                    : is_output_tensor->second;
+  };
+
   // Build a GrapplerFunctionConnectivity from inputs and new function body.
   GrapplerFunctionConnectivity connectivity;
   TF_RETURN_IF_ERROR(
@@ -742,8 +818,8 @@ Status MakeFunctionDef(const GrapplerFunctionItem& item,
 
   // Add function input arguments.
   for (const InputArgExpansion& input_arg : item.inputs()) {
-    CHECK(input_arg.placeholders.size() == 1)  // do some sanity checking
-        << "Inputs of tensor sequences are not supported";
+    DCHECK(input_arg.placeholders.size() == 1)  // do some sanity checking
+        << "Inputs of tensor lists are not supported";
 
     OpDef::ArgDef arg_def;
     arg_def.set_name(input_arg.input_name);
@@ -754,8 +830,8 @@ Status MakeFunctionDef(const GrapplerFunctionItem& item,
 
   // Add function output arguments.
   for (const OutputArgExpansion& output_arg : item.outputs()) {
-    CHECK(output_arg.output_tensors.size() == 1)  // do some sanity checking
-        << "Outputs of tensor sequences are not supported";
+    DCHECK(output_arg.output_nodes.size() == 1)  // do some sanity checking
+        << "Outputs of tensor lists are not supported";
 
     OpDef::ArgDef arg_def;
     arg_def.set_name(output_arg.output_name);
@@ -763,11 +839,9 @@ Status MakeFunctionDef(const GrapplerFunctionItem& item,
     arg_def.set_is_ref(output_arg.is_ref);
     *func->mutable_signature()->add_output_arg() = arg_def;
 
-    string ret;
-    for (const string& output_tensor : output_arg.output_tensors) {
-      TF_RETURN_IF_ERROR(connectivity.AsFunctionDefInput(output_tensor, &ret));
-      (*func->mutable_ret())[output_arg.output_name] = ret;
-    }
+    TF_RETURN_IF_ERROR(connectivity.AsFunctionDefInput(
+        output_tensor(output_arg),
+        &(*func->mutable_ret())[output_arg.output_name]));
   }
 
   // Copy function definition specific attributes.
@@ -778,12 +852,16 @@ Status MakeFunctionDef(const GrapplerFunctionItem& item,
   }
 
   // Copy function body nodes to the FunctionDef and update input format
-  for (const NodeDef& func_body_node : item.function_body().node()) {
-    // Do not copy input placeholders
-    if (item.IsInputPlaceholder(func_body_node.name())) continue;
+  for (const NodeDef& func_node : item.function_body().node()) {
+    const string& name = func_node.name();
+
+    // Do not copy input placeholders.
+    if (IsPlaceholder(func_node) && input_placeholders.count(name)) continue;
+    // Do not copy output nodes that we bypassed.
+    if (IsIdentity(func_node) && output_tensors.count(name)) continue;
 
     NodeDef* func_def_node = func->add_node_def();
-    *func_def_node = func_body_node;
+    *func_def_node = func_node;
     TF_RETURN_IF_ERROR(connectivity.AsFunctionDefNode(func_def_node));
   }
 
diff --git a/tensorflow/core/grappler/utils/functions.h b/tensorflow/core/grappler/utils/functions.h
index ce8a3e5ac7..d5a41e7473 100644
--- a/tensorflow/core/grappler/utils/functions.h
+++ b/tensorflow/core/grappler/utils/functions.h
@@ -21,6 +21,7 @@ limitations under the License.
 
 #include "absl/container/flat_hash_map.h"
 #include "absl/container/flat_hash_set.h"
+#include "absl/container/inlined_vector.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/framework/function.pb.h"
@@ -32,6 +33,21 @@ limitations under the License.
 namespace tensorflow {
 namespace grappler {
 
+// WARNING(ezhulenev): Currently we do not support functions with inputs or
+// outputs instantiated into multiple tensors. This can happen if the
+// input/output type is 'T*N' or 'list(type)'. This is enforced by multiple
+// checks across this file and also function_optimizer.cc. InputArgExpansion and
+// OutputArgExpansion already support lists of tensors, but that's pretty much
+// it, all other code is written with assumption that expansions are always of
+// size 1. MakeGrapplerFunctionItem will gracefully fail with Status error.
+//
+// This is a low priority feature, because in practice we don't see a lot (any
+// at all?) functions with such arguments. Tensorflow-Eager always produces
+// functions with plain input/output arguments.
+
+// TODO(ezhulenev): Support inputs and outputs of type 'T*N'.
+// TODO(ezhulenev): Support inputs and outputs of type 'list(type)'.
+
 // Depending on the function instantiation attributes, input argument to the
 // function might be a single tensor, list of tensors of the same type, or a
 // list of tensors of different types.
@@ -39,30 +55,23 @@ namespace grappler {
 // InputArgExpansion keeps track of the placeholders that were added to the
 // function body in place of function inputs and a resolved input data type.
 struct InputArgExpansion {
-  // TODO(ezhulenev): Add support for functions with tensor sequence inputs of
-  // different data types.
-  // TODO(ezhulenev): Support type parametrized inputs?
-  string input_name;                 // name of the function input argument
-  DataType data_type;                // input data type
-  bool is_ref;                       // if true, inputs are required to be refs
-  std::vector<string> placeholders;  // names of placeholder nodes in the
-                                     // function body
+  string input_name;
+  DataType data_type;
+  bool is_ref;
+  absl::InlinedVector<string, 1> placeholders;
 };
 
 // Depending on the function instantiation attributes, output argument is mapped
 // to one or more outputs of one of the function body nodes.
 //
-// OutputArgExpansion keeps mapping from a function output arg to the output
-// tensors of a function body nodes and a resolved output data type
+// OutputArgExpansion keeps track of the Identity nodes that were added to the
+// function body to forward output tensors. Adding these output nodes allows
+// nested function inlining and specialization (see function optimizer).
 struct OutputArgExpansion {
-  // TODO(ezhulenev): Add support for functions with tensor sequence outputs of
-  // different data types.
-  // TODO(ezhulenev): Support type parametrized outputs?
-  string output_name;                  // name of the function output argument
-  DataType data_type;                  // output data type
-  bool is_ref;                         // if true, outputs are refs
-  std::vector<string> output_tensors;  // names of output tensor from the
-                                       // function body nodes
+  string output_name;
+  DataType data_type;
+  bool is_ref;
+  absl::InlinedVector<string, 1> output_nodes;
 };
 
 // FunctionDef uses different connectivity encoding for the function body nodes,
@@ -87,7 +96,7 @@ class GrapplerFunctionConnectivity {
   // When expanding inputs in function def format, single input might be
   // expanded into multiple tensors. When converting back to the function def
   // format from graph def format, it's always a 1-to-1 relationship.
-  // FunctionDef built from GrapplerFunctionItem is always specialized to it's
+  // FunctionDef built from GrapplerFunctionItem is always specialized to its
   // instantiation attributes and length of input args (and node def outputs) is
   // known.
 
@@ -144,8 +153,6 @@ class GrapplerFunctionItem : public GrapplerItem {
 
   const string& description() const;
 
-  bool IsInputPlaceholder(const string& node_name) const;
-
   const std::vector<InputArgExpansion>& inputs() const;
   const InputArgExpansion& input(int i) const;
   const std::size_t input_size() const;
@@ -168,10 +175,9 @@ class GrapplerFunctionItem : public GrapplerItem {
                                          GrapplerFunctionItem*);
   friend Status ReplaceInputWithConst(const NodeDef&, int,
                                       GrapplerFunctionItem*);
-  friend Status RemoveUnusedOutputs(
-      const absl::flat_hash_set<int>& active_outputs,
-      GrapplerFunctionItem* item,
-      std::vector<std::pair<int, int>>* output_mapping);
+  friend Status RemoveFunctionOutputs(const absl::flat_hash_set<int>&,
+                                      GrapplerFunctionItem*,
+                                      std::vector<std::pair<int, int>>*);
 
   GrapplerFunctionItem(string func_name, string description,
                        AttrSlice func_attr,
@@ -187,16 +193,14 @@ class GrapplerFunctionItem : public GrapplerItem {
   std::vector<InputArgExpansion> input_arg_expansions_;
   std::vector<OutputArgExpansion> output_arg_expansions_;
 
-  std::set<string> input_arg_placeholders_;
-
   bool is_stateful_ = false;
 };
 
 // Check if function input/output types are fully defined only at instantiation
-// time (parametrized by it's instantiation node).
+// time (parametrized by its instantiation node).
 bool HasParametrizedType(const FunctionDef& func);
 
-// Check if a function body is parametrized by it's instantiation node. Function
+// Check if a function body is parametrized by its instantiation node. Function
 // body is parametrized, if it has at least one node with a 'placeholder'
 // attribute.
 bool HasParametrizedBody(const FunctionDef& func);
@@ -228,15 +232,16 @@ Status RegisterGrapplerFunctionConnectivity(
 Status ReplaceInputWithConst(const NodeDef& input_const, int input_index,
                              GrapplerFunctionItem* item);
 
-// Remove function output arguments that do not have any active outputs (output
-// tensor connected to other node inputs or in a fetch set). Active outputs uses
-// GraphDef output position encoding, and multiple active outputs could
-// potentially be connected to the same output argument (in case of tensor list
-// outputs). Add output mapping for all active outputs that changed it's output
-// position (std::pair<old position, new position>).
-Status RemoveUnusedOutputs(const absl::flat_hash_set<int>& active_outputs,
-                           GrapplerFunctionItem* item,
-                           std::vector<std::pair<int, int>>* output_mapping);
+// Removes outputs from instantiated grappler function item. Function node
+// outputs use GraphDef output index encoding, and multiple outputs might belong
+// to the same output argument expansion (in case of tensor list outputs). For
+// all active function outputs that changed its output index, this function adds
+// an output mapping (std::pair<old index, new index>).
+Status RemoveFunctionOutputs(const absl::flat_hash_set<int>& remove_outputs,
+                             GrapplerFunctionItem* item,
+                             std::vector<std::pair<int, int>>* output_mapping);
+
+// TODO(ezhulennev, b/120103818): Add RemoveFunctionInputs.
 
 // Make a GrapplerFunctionItem from the function definition and function
 // instantiation attributes (caller node attributes). Returns error if the given
@@ -251,7 +256,7 @@ Status MakeGrapplerFunctionItem(const FunctionDef& func,
 // fully defined (no type or body parametrization).
 // TODO(ezhulenev): Support parametrized functions without fully defined
 // instantiation attributes? Do we ever want to optimize parametrized function
-// without specializing it to it's instantiation attributes (at least types)?
+// without specializing it to its instantiation attributes (at least types)?
 Status MakeGrapplerFunctionItem(const FunctionDef& func,
                                 const FunctionLibraryDefinition& flib,
                                 int graph_def_version,
diff --git a/tensorflow/core/grappler/utils/functions_test.cc b/tensorflow/core/grappler/utils/functions_test.cc
index 29d6100d23..c49920c79c 100644
--- a/tensorflow/core/grappler/utils/functions_test.cc
+++ b/tensorflow/core/grappler/utils/functions_test.cc
@@ -249,15 +249,16 @@ TEST_F(FunctionsTest, FromSimpleFunctionDef) {
                                         flib, TF_GRAPH_DEF_VERSION, &item));
 
   EXPECT_EQ("XTimesTwo", item.id);
-  EXPECT_EQ(4, item.function_body().node_size());
+  EXPECT_EQ(5, item.function_body().node_size());
 
   EXPECT_EQ(1, item.input_size());
   EXPECT_EQ("x", item.input(0).input_name);
-  EXPECT_EQ(std::vector<string>{"x"}, item.input(0).placeholders);
+  ASSERT_EQ(1, item.input(0).placeholders.size());
+  EXPECT_EQ("x", item.input(0).placeholders[0]);
 
   EXPECT_EQ(1, item.output_size());
   EXPECT_EQ("y", item.output(0).output_name);
-  EXPECT_EQ("y", item.output(0).output_tensors[0]);
+  EXPECT_EQ("y_output_node_0", item.output(0).output_nodes[0]);
 
   int count = 0;
   for (const NodeDef &node : item.function_body().node()) {
@@ -279,9 +280,13 @@ TEST_F(FunctionsTest, FromSimpleFunctionDef) {
       EXPECT_EQ(2, node.input_size());
       EXPECT_EQ("x", node.input(0));
       EXPECT_EQ("scale", node.input(1));
+    } else if (node.name() == "y_output_node_0" && ++count) {
+      EXPECT_EQ("Identity", node.op());
+      ASSERT_EQ(1, node.input_size());
+      EXPECT_EQ("y", node.input(0));
     }
   }
-  EXPECT_EQ(4, count);
+  EXPECT_EQ(5, count);
 }
 
 TEST_F(FunctionsTest, FromFunctionDefWithMultiOutputNodes) {
@@ -326,7 +331,7 @@ TEST_F(FunctionsTest, FromFunctionDefWithMultiOutputNodes) {
                                         flib, TF_GRAPH_DEF_VERSION, &item));
 
   EXPECT_EQ("SubGrad", item.id);
-  EXPECT_EQ(12, item.function_body().node_size());
+  EXPECT_EQ(14, item.function_body().node_size());
 
   ASSERT_EQ(3, item.input_size());
   EXPECT_EQ("x", item.input(0).input_name);
@@ -334,8 +339,8 @@ TEST_F(FunctionsTest, FromFunctionDefWithMultiOutputNodes) {
   EXPECT_EQ("dz", item.input(2).input_name);
 
   ASSERT_EQ(2, item.output_size());
-  EXPECT_EQ("dx", item.output(0).output_tensors[0]);
-  EXPECT_EQ("dy", item.output(1).output_tensors[0]);
+  EXPECT_EQ("dx_output_node_0", item.output(0).output_nodes[0]);
+  EXPECT_EQ("dy_output_node_0", item.output(1).output_nodes[0]);
 
   int count = 0;
   for (const NodeDef &node : item.function_body().node()) {
@@ -359,9 +364,17 @@ TEST_F(FunctionsTest, FromFunctionDefWithMultiOutputNodes) {
       EXPECT_EQ(2, node.input_size());
       EXPECT_EQ("gy", node.input(0));
       EXPECT_EQ("rx:1", node.input(1));
+    } else if (node.name() == "dx_output_node_0" && ++count) {
+      EXPECT_EQ("Identity", node.op());
+      ASSERT_EQ(1, node.input_size());
+      EXPECT_EQ("dx", node.input(0));
+    } else if (node.name() == "dy_output_node_0" && ++count) {
+      EXPECT_EQ("Identity", node.op());
+      ASSERT_EQ(1, node.input_size());
+      EXPECT_EQ("dy", node.input(0));
     }
   }
-  EXPECT_EQ(6, count);
+  EXPECT_EQ(8, count);
 }
 
 TEST_F(FunctionsTest, FromFunctionDefWithNestedFuncs) {
@@ -472,7 +485,7 @@ TEST_F(FunctionsTest, FromFunctionDefWithOutputMappings) {
                                         flib, TF_GRAPH_DEF_VERSION, &item));
 
   EXPECT_EQ(1, item.output_size());
-  EXPECT_EQ("Exp", item.output(0).output_tensors[0]);
+  EXPECT_EQ("out_output_node_0", item.output(0).output_nodes[0]);
 
   int count = 0;
   for (const NodeDef &node : item.function_body().node()) {
@@ -488,9 +501,13 @@ TEST_F(FunctionsTest, FromFunctionDefWithOutputMappings) {
       EXPECT_EQ("Exp", node.op());
       EXPECT_EQ(1, node.input_size());
       EXPECT_EQ("Linear_func", node.input(0));
+    } else if (node.name() == "out_output_node_0" && ++count) {
+      EXPECT_EQ("Identity", node.op());
+      ASSERT_EQ(1, node.input_size());
+      EXPECT_EQ("Exp", node.input(0));
     }
   }
-  EXPECT_EQ(3, count);
+  EXPECT_EQ(4, count);
 }
 
 TEST_F(FunctionsTest, FromFunctionDefWithInputForwarding) {
@@ -517,27 +534,44 @@ TEST_F(FunctionsTest, FromFunctionDefWithInputForwarding) {
                                         flib, TF_GRAPH_DEF_VERSION, &item));
 
   EXPECT_EQ("ForwardInputs", item.id);
-  EXPECT_EQ(5, item.function_body().node_size());
+  EXPECT_EQ(8, item.function_body().node_size());
 
   EXPECT_EQ(3, item.output_size());
-  EXPECT_EQ("in0", item.output(0).output_tensors[0]);
-  EXPECT_EQ("arg2", item.output(1).output_tensors[0]);
-  EXPECT_EQ("arg3", item.output(2).output_tensors[0]);
+  EXPECT_EQ("out0_output_node_0", item.output(0).output_nodes[0]);
+  EXPECT_EQ("arg2_output_node_0", item.output(1).output_nodes[0]);
+  EXPECT_EQ("arg3_output_node_0", item.output(2).output_nodes[0]);
 
   int count = 0;
+
+  const auto is_arg_placeholder = [](const string &name) {
+    return name == "in0" || name == "in1" || name == "arg2" || name == "arg3" ||
+           name == "arg4";
+  };
+
   for (const NodeDef &node : item.function_body().node()) {
-    EXPECT_TRUE(node.name() == "in0" || node.name() == "in1" ||
-                node.name() == "arg2" || node.name() == "arg3" ||
-                node.name() == "arg4");
-    count++;
-    EXPECT_EQ("Placeholder", node.op());
-    if (node.name() == "arg3") {
-      EXPECT_EQ(DT_INT32, node.attr().at("dtype").type());
-    } else {
-      EXPECT_EQ(DT_FLOAT, node.attr().at("dtype").type());
+    if (is_arg_placeholder(node.name()) && node.op() == "Placeholder") {
+      count++;
+      if (node.name() == "arg3") {
+        EXPECT_EQ(DT_INT32, node.attr().at("dtype").type());
+      } else {
+        EXPECT_EQ(DT_FLOAT, node.attr().at("dtype").type());
+      }
+      continue;
+    }
+
+    EXPECT_EQ("Identity", node.op());
+    ASSERT_EQ(1, node.input_size());
+    EXPECT_TRUE(is_arg_placeholder(node.input(0)));
+
+    if (node.name() == "out0_output_node_0" && ++count) {
+      EXPECT_EQ("in0", node.input(0));
+    } else if (node.name() == "arg2_output_node_0" && ++count) {
+      EXPECT_EQ("arg2", node.input(0));
+    } else if (node.name() == "arg3_output_node_0" && ++count) {
+      EXPECT_EQ("arg3", node.input(0));
     }
   }
-  EXPECT_EQ(5, count);
+  EXPECT_EQ(8, count);
 }
 
 TEST_F(FunctionsTest, FromFunctionDefWithoutInput) {
@@ -566,16 +600,22 @@ TEST_F(FunctionsTest, FromFunctionDefWithoutInput) {
 
   EXPECT_EQ(0, item.input_size());
   EXPECT_EQ(1, item.output_size());
-  EXPECT_EQ("o", item.output(0).output_tensors[0]);
+  EXPECT_EQ("o_output_node_0", item.output(0).output_nodes[0]);
+  EXPECT_EQ(3, item.function_body().node_size());
 
-  EXPECT_EQ(2, item.function_body().node_size());
   const NodeDef &two = item.function_body().node(0);
   EXPECT_EQ("two", two.name());
   EXPECT_EQ(0, two.input_size());
+
   const NodeDef &cast = item.function_body().node(1);
   EXPECT_EQ("o", cast.name());
   EXPECT_EQ(1, cast.input_size());
   EXPECT_EQ("two", cast.input(0));
+
+  const NodeDef &retval = item.function_body().node(2);
+  EXPECT_EQ("o_output_node_0", retval.name());
+  EXPECT_EQ(1, retval.input_size());
+  EXPECT_EQ("o", retval.input(0));
 }
 
 TEST_F(FunctionsTest, FromFunctionDefWithSideEffectfulOps) {
@@ -674,7 +714,7 @@ TEST_F(FunctionsTest, ReplaceInputWithConst) {
   EXPECT_EQ(2, item.input_size());
   EXPECT_EQ(1, item.output_size());
 
-  ASSERT_EQ(3, item.function_body().node_size());
+  ASSERT_EQ(4, item.function_body().node_size());
 
   const NodeDef &input_x = item.function_body().node(0);
   const NodeDef &input_y = item.function_body().node(1);
@@ -748,8 +788,9 @@ TEST_F(FunctionsTest, SwapFunctionBodyAndMakeFunctionDef) {
       {{"z", "output:z:0"}});
 
   GraphDef id_func_body = test::function::GDef(
-      {/* pass input to output through identity */
-       NDef("output", "Identity", {"x"}, {{"T", "float"}})});
+      {/* Read and return input argument through Identity node. */
+       NDef("read_x", "Identity", {"x"}, {{"T", "float"}}),
+       NDef("z_output_node_0", "Identity", {"read_x"}, {{"T", "float"}})});
 
   protobuf::Map<string, AttrValue> func_instantiation_attr;
   func_instantiation_attr["T"].set_type(DT_FLOAT);
@@ -772,15 +813,15 @@ TEST_F(FunctionsTest, SwapFunctionBodyAndMakeFunctionDef) {
   // Check that graph body was updated.
   int count = 0;
   for (const NodeDef &node : specialized.node_def()) {
-    if (node.name() == "output" && ++count) {
+    if (node.name() == "read_x" && ++count) {
       EXPECT_EQ("Identity", node.op());
       EXPECT_EQ("x:0", node.input(0));
     }
   }
   EXPECT_EQ(1, count);
 
-  // And return tensor mapping was updated with a new output name (z->output).
-  EXPECT_EQ("output:output:0", (*specialized.mutable_ret())["z"]);
+  // And return tensor mapping was updated with a new output name (z->read_x).
+  EXPECT_EQ("read_x:output:0", (*specialized.mutable_ret())["z"]);
 }
 
 TEST_F(FunctionsTest, FunctionDefGrapplerFunctionItemRoundTrip) {
diff --git a/tensorflow/core/grappler/utils/grappler_test.cc b/tensorflow/core/grappler/utils/grappler_test.cc
index 5ef1cf444b..1b4b9f9a51 100644
--- a/tensorflow/core/grappler/utils/grappler_test.cc
+++ b/tensorflow/core/grappler/utils/grappler_test.cc
@@ -14,7 +14,9 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/grappler/utils/grappler_test.h"
+
 #include <memory>
+
 #include "absl/algorithm/container.h"
 #include "tensorflow/core/framework/attr_value_util.h"
 #include "tensorflow/core/grappler/utils.h"
@@ -25,6 +27,46 @@ limitations under the License.
 namespace tensorflow {
 namespace grappler {
 
+namespace {
+void CompareGraphNodes(protobuf::RepeatedPtrField<NodeDef>* want,
+                       protobuf::RepeatedPtrField<NodeDef>* got) {
+  auto comparator = [](const NodeDef& n1, const NodeDef& n2) -> bool {
+    return n1.name() < n2.name();
+  };
+
+  std::sort(want->begin(), want->end(), comparator);
+  std::sort(got->begin(), got->end(), comparator);
+
+  ASSERT_EQ(want->size(), got->size());
+
+  for (int i = 0; i < want->size(); ++i) {
+    NodeDef& want_node = (*want)[i];
+    NodeDef& got_node = (*got)[i];
+
+    EXPECT_EQ(want_node.op(), got_node.op());
+    EXPECT_EQ(want_node.name(), got_node.name());
+    EXPECT_EQ(want_node.device(), got_node.device());
+    ASSERT_EQ(want_node.input_size(), got_node.input_size());
+
+    // Order of control dependencies doesn't matter, so we sort them first.
+    const auto is_control = [](const string& input) -> bool {
+      return ParseTensorName(input).index() < 0;
+    };
+
+    auto want_inputs = want_node.mutable_input();
+    auto got_inputs = got_node.mutable_input();
+    std::sort(absl::c_find_if(*want_inputs, is_control), want_inputs->end());
+    std::sort(absl::c_find_if(*got_inputs, is_control), got_inputs->end());
+
+    for (int j = 0; j < want_node.input_size(); ++j) {
+      const TensorId want_tensor = ParseTensorName(want_node.input(j));
+      const TensorId got_tensor = ParseTensorName(got_node.input(j));
+      EXPECT_EQ(want_tensor.ToString(), got_tensor.ToString());
+    }
+  }
+}
+}  // namespace
+
 GrapplerTest::GrapplerTest() {
   // Turn off all the automatic optimizations to ensure that we run the graph
   // exactly as it is given to us. This ensures that we can compare the results
@@ -96,35 +138,11 @@ NodeDef* GrapplerTest::AddNode(
 }
 
 void GrapplerTest::CompareGraphs(GraphDef want, GraphDef got) const {
-  auto comparator = [](const NodeDef& n1, const NodeDef& n2) -> bool {
-    return n1.name() < n2.name();
-  };
-  std::sort(want.mutable_node()->begin(), want.mutable_node()->end(),
-            comparator);
-  std::sort(got.mutable_node()->begin(), got.mutable_node()->end(), comparator);
-
-  for (int i = 0; i < want.node_size(); ++i) {
-    std::sort(want.mutable_node(i)->mutable_input()->begin(),
-              want.mutable_node(i)->mutable_input()->end());
-  }
-  for (int i = 0; i < got.node_size(); ++i) {
-    std::sort(got.mutable_node(i)->mutable_input()->begin(),
-              got.mutable_node(i)->mutable_input()->end());
-  }
-
-  ASSERT_EQ(want.node_size(), got.node_size());
-  for (int i = 0; i < want.node_size(); ++i) {
-    EXPECT_EQ(want.node(i).op(), got.node(i).op());
-    EXPECT_EQ(want.node(i).name(), got.node(i).name());
-    EXPECT_EQ(want.node(i).device(), got.node(i).device());
+  CompareGraphNodes(want.mutable_node(), got.mutable_node());
+}
 
-    ASSERT_EQ(want.node(i).input_size(), got.node(i).input_size());
-    for (int j = 0; j < want.node(i).input_size(); ++j) {
-      const TensorId want_tensor = ParseTensorName(want.node(i).input(j));
-      const TensorId got_tensor = ParseTensorName(got.node(i).input(j));
-      EXPECT_EQ(want_tensor.ToString(), got_tensor.ToString());
-    }
-  }
+void GrapplerTest::CompareFunctions(FunctionDef want, FunctionDef got) const {
+  CompareGraphNodes(want.mutable_node_def(), got.mutable_node_def());
 }
 
 void GrapplerTest::CompareNodes(const NodeDef& want, const NodeDef& got) const {
diff --git a/tensorflow/core/grappler/utils/grappler_test.h b/tensorflow/core/grappler/utils/grappler_test.h
index 20fd04c1c6..26c1db3740 100644
--- a/tensorflow/core/grappler/utils/grappler_test.h
+++ b/tensorflow/core/grappler/utils/grappler_test.h
@@ -62,6 +62,13 @@ class GrapplerTest : public ::testing::Test {
   // equality, and adds all failuires to the current test.
   void CompareNodes(const NodeDef& want, const NodeDef& got) const;
 
+  // Checks if two functions are equal. Both functions must have the same set of
+  // nodes with the same inputs and attributes. Nodes can be in different order.
+  //
+  // NOTE: This function uses EXPECT/ASSERT macros to check node properties
+  // equality, and adds all failures to the current test.
+  void CompareFunctions(FunctionDef want, FunctionDef got) const;
+
   // Checks if node 'src' is directly connected to the input($position) of
   // 'dst'.
   bool IsNodesDirectlyConnected(const NodeMap& node_map, const string& src,
-- 
GitLab


From 0a88318ac73a3d30dfee4ab1541070cbf82fe05c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 14:04:07 -0800
Subject: [PATCH 0447/2345] Add
 `tf.distribute.Strategy.experimental_make_numpy_iterator()` function.

PiperOrigin-RevId: 228584021
---
 tensorflow/contrib/distribute/python/BUILD    |   4 +
 .../python/collective_all_reduce_strategy.py  |   8 +-
 .../collective_all_reduce_strategy_test.py    |   7 ++
 .../contrib/distribute/python/keras_test.py   |  58 +++------
 .../distribute/python/mirrored_strategy.py    |  30 +++++
 .../python/mirrored_strategy_multigpu_test.py |   3 +
 .../distribute/python/one_device_strategy.py  |   9 +-
 .../python/one_device_strategy_test.py        |   4 +
 .../python/parameter_server_strategy.py       |  31 +++++
 .../python/parameter_server_strategy_test.py  |  12 ++
 .../distribute/python/strategy_test_lib.py    |  29 +++++
 .../contrib/distribute/python/tpu_strategy.py |   9 ++
 tensorflow/python/distribute/BUILD            |  32 +++++
 .../python/distribute/distribute_lib.py       |  72 +++++++++++-
 .../python/distribute/mirrored_strategy.py    |  10 ++
 tensorflow/python/distribute/numpy_dataset.py |  97 +++++++++++++++
 .../python/distribute/numpy_dataset_test.py   |  44 +++++++
 .../distribute/parameter_server_strategy.py   |  13 +-
 .../engine/distributed_training_utils.py      | 111 +-----------------
 tensorflow/python/keras/engine/training.py    |  62 +++++-----
 .../keras/engine/training_distributed.py      |   6 -
 tensorflow/python/training/optimizer.py       |  13 +-
 ...orflow.distribute.-mirrored-strategy.pbtxt |   4 +
 ...orflow.distribute.-strategy-extended.pbtxt |   4 +
 .../v1/tensorflow.distribute.-strategy.pbtxt  |   4 +
 ...orflow.distribute.-mirrored-strategy.pbtxt |   4 +
 ...orflow.distribute.-strategy-extended.pbtxt |   4 +
 .../v2/tensorflow.distribute.-strategy.pbtxt  |   4 +
 28 files changed, 490 insertions(+), 198 deletions(-)
 create mode 100644 tensorflow/python/distribute/numpy_dataset.py
 create mode 100644 tensorflow/python/distribute/numpy_dataset_test.py

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 7ee12812af..d4758d7518 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -131,6 +131,7 @@ py_library(
         "//tensorflow/python:math_ops",
         "//tensorflow/python/distribute:distribute_lib",
         "//tensorflow/python/distribute:input_lib",
+        "//tensorflow/python/distribute:numpy_dataset",
         "//tensorflow/python/distribute:reduce_util",
         "//tensorflow/python/distribute:values",
         "//tensorflow/python/eager:context",
@@ -153,6 +154,7 @@ py_library(
         "//tensorflow/python/distribute:cross_device_utils",
         "//tensorflow/python/distribute:input_lib",
         "//tensorflow/python/distribute:multi_worker_util",
+        "//tensorflow/python/distribute:numpy_dataset",
         "//tensorflow/python/distribute:values",
         "//tensorflow/python/eager:context",
     ],
@@ -178,6 +180,7 @@ py_library(
         "//tensorflow/python/eager:backprop",
         "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:test",
+        "//third_party/py/numpy",
     ],
 )
 
@@ -303,6 +306,7 @@ py_library(
         "//tensorflow/python:tensor_util",
         "//tensorflow/python:util",
         "//tensorflow/python/distribute:input_lib",
+        "//tensorflow/python/distribute:numpy_dataset",
         "//tensorflow/python/distribute:reduce_util",
         "//tensorflow/python/distribute:values",
     ],
diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
index f6361cb6e8..39756b32c5 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
@@ -28,6 +28,7 @@ from tensorflow.python.distribute import device_util
 from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import input_lib
 from tensorflow.python.distribute import multi_worker_util
+from tensorflow.python.distribute import numpy_dataset
 from tensorflow.python.distribute import values
 from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
@@ -86,6 +87,7 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
     else:
       local_devices = ("/device:CPU:0",)
     self._worker_device = device_util.canonicalize("/device:CPU:0")
+    self._host_input_device = numpy_dataset.SingleDevice(self._worker_device)
 
     self._collective_keys = cross_device_utils.CollectiveKeys()
     self._initialize_local(local_devices)
@@ -121,6 +123,7 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
                                                 task_id)
 
     self._worker_device = "/job:%s/task:%d" % (task_type, task_id)
+    self._host_input_device = numpy_dataset.SingleDevice(self._worker_device)
     if num_gpus_per_worker:
       local_devices = tuple(
           "%s/device:GPU:%d" % (self._worker_device, i)
@@ -157,6 +160,9 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
     if colocate_with is None:
       device_map = self._device_map
       logical_device = 0  # TODO(josh11b): Get logical device from scope here.
+    elif isinstance(colocate_with, numpy_dataset.SingleDevice):
+      with ops.device(colocate_with.device):
+        return next_creator(*args, **kwargs)
     else:
       device_map = colocate_with.device_map
       logical_device = colocate_with.logical_device
@@ -347,4 +353,4 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
   # TODO(priyag): Delete this once all strategies use global batch size.
   @property
   def _global_batch_size(self):
-    return False
+    return True
diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
index 4c8c01a216..c3e9c55e96 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
@@ -466,6 +466,13 @@ class LocalCollectiveAllReduceStrategy(
     with self.cached_session(config=config, target=target):
       self._test_all_reduce_mean_gradient_tape(distribution)
 
+  def testNumpyIterator(self):
+    num_gpus = 2
+    if context.num_gpus() < num_gpus:
+      self.skipTest('Not enough GPUs')
+    strategy, _, _ = self._get_test_object(None, None, num_gpus)
+    self._test_numpy_iterator(strategy)
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 1cb5fa30a3..40916afcfa 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -429,15 +429,6 @@ class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase,
 class TestDistributionStrategyWithNumpyArrays(test.TestCase,
                                               parameterized.TestCase):
 
-  @combinations.generate(strategy_for_numpy_input_combinations())
-  def test_creating_var_with_numpy_arrays(self, distribution):
-    with self.cached_session():
-      x = np.asarray(np.random.random((64, 3)), dtype=np.float32)
-      var_x = distributed_training_utils.get_var_for_numpy(distribution, x)
-      val = self.evaluate(var_x.value())
-      # Verify that the numpy value is copied to the variable.
-      self.assertAllEqual(x, val)
-
   @combinations.generate(strategy_for_numpy_input_combinations())
   def test_calculating_input_params_no_steps_no_batch_size(self, distribution):
     # Calculate the per_replica_batch_size scaling factor for strategies
@@ -576,26 +567,26 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
         metrics = ['mae']
         model.compile(optimizer, loss, metrics=metrics)
 
-      inputs = np.zeros((64, 3), dtype=np.float32)
-      targets = np.zeros((64, 4), dtype=np.float32)
+        inputs = np.zeros((64, 3), dtype=np.float32)
+        targets = np.zeros((64, 4), dtype=np.float32)
 
-      # Call fit with validation data
-      model.fit(inputs, targets, epochs=1, batch_size=2, verbose=0,
-                validation_data=(inputs, targets))
+        # Call fit with validation data
+        model.fit(inputs, targets, epochs=1, batch_size=2, verbose=0,
+                  validation_data=(inputs, targets))
 
-      # TODO(anjalisridhar): We need tests for when the batch size and steps are
-      # smaller and results in a 0 batch_size and steps value.
-      model.evaluate(inputs, targets)
-      # with steps
-      model.evaluate(inputs, targets, steps=2)
-      # with batch_size
-      model.evaluate(inputs, targets, batch_size=8)
+        # TODO(anjalisridhar): We need tests for when the batch size and steps
+        # are smaller and results in a 0 batch_size and steps value.
+        model.evaluate(inputs, targets)
+        # with steps
+        model.evaluate(inputs, targets, steps=2)
+        # with batch_size
+        model.evaluate(inputs, targets, batch_size=8)
 
-      model.predict(inputs)
-      # with steps
-      model.predict(inputs, steps=2)
-      # with batch_size
-      model.predict(inputs, batch_size=8)
+        model.predict(inputs)
+        # with steps
+        model.predict(inputs, steps=2)
+        # with batch_size
+        model.predict(inputs, batch_size=8)
 
   @combinations.generate(strategy_for_numpy_input_combinations())
   def test_calling_model_with_nested_numpy_arrays(self, distribution):
@@ -1192,21 +1183,6 @@ class TestDistributionStrategyValidation(test.TestCase,
           metrics = ['mae', keras.metrics.CategoricalAccuracy()]
           model.compile(optimizer, loss, metrics=metrics)
 
-  @combinations.generate(all_strategy_combinations_minus_default())
-  def test_loop_in_scope(self, distribution):
-    with self.cached_session():
-      with self.assertRaisesRegexp(
-          RuntimeError, 'should not be run inside the tf.distribute.Strategy'):
-        with distribution.scope():
-          x = keras.layers.Input(shape=(3,), name='input')
-          y = keras.layers.Dense(4, name='dense')(x)
-          model = keras.Model(x, y)
-          optimizer = gradient_descent.GradientDescentOptimizer(0.001)
-          loss = 'mse'
-          model.compile(optimizer, loss)
-          input_array = np.zeros((3, 3), dtype=np.float32)
-          model.predict(input_array)
-
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index e3ab2bf19e..5fa36fb402 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -104,6 +104,36 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
                                 auto_shard_dataset)
     super(MirroredStrategy, self).__init__(extended)
 
+  # Override to change the documentation to reflect the different handling of
+  # global vs. local batch size between core and contrib.
+  def experimental_make_numpy_iterator(  # pylint: disable=useless-super-delegation
+      self, numpy_input, batch_size, num_epochs=1, shuffle=1024, session=None):
+    """Makes an iterator for input provided via a nest of numpy arrays.
+
+    NOTE: The `batch_size` argument here has different behavior for this
+    contrib version of `MirroredStrategy`.
+
+    Args:
+      numpy_input: A nest of NumPy input arrays that will be distributed evenly
+        across all replicas.
+      batch_size: The number of entries from the array we should consume in one
+        step of the computation, across all replicas. This is the per-replica
+        batch size. The global batch size will be this times
+        `num_replicas_in_sync`.
+      num_epochs: The number of times to iterate through the examples. A value
+        of `None` means repeat forever.
+      shuffle: Size of buffer to use for shuffling the input examples.
+        Use `None` to disable shuffling.
+      session: (TensorFlow v1.x graph execution only) A session used for
+        initialization.
+
+    Returns:
+      An `tf.distribute.InputIterator` which returns inputs for each step of the
+      computation.  User should call `initialize` on the returned iterator.
+    """
+    return super(MirroredStrategy, self).experimental_make_numpy_iterator(
+        numpy_input, batch_size, num_epochs, shuffle, session)
+
 
 class MirroredExtended(CoreMirroredExtended):
   """Implementation of (contrib) MirroredStrategy."""
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index 9821828b2d..59d711ae01 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -116,6 +116,9 @@ class MirroredTwoDeviceDistributionTest(
     self._test_input_fn_iterator(iterator, distribution.extended.worker_devices,
                                  expected_values)
 
+  def testNumpyIterator(self, distribution):
+    self._test_numpy_iterator(distribution)
+
   def testGlobalStepUpdate(self, distribution):
     self._test_global_step_update(distribution)
 
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py
index fb470f8546..34b0c31087 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 from tensorflow.python.distribute import device_util
 from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import input_lib
+from tensorflow.python.distribute import numpy_dataset
 from tensorflow.python.distribute import values
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -50,8 +51,8 @@ class OneDeviceExtended(distribute_lib.DistributionStrategyExtended):
     super(OneDeviceExtended, self).__init__(container_strategy)
     self._device = device
     self._default_device = device
-    worker = device_util.canonicalize("/device:CPU:0")
-    worker_device_pairs = [(worker, [self._device])]
+    self._input_device = device_util.canonicalize("/device:CPU:0")
+    worker_device_pairs = [(self._input_device, [self._device])]
     device_map = values.SingleDeviceMap(device)
     self._input_workers = input_lib.InputWorkers(
         device_map, worker_device_pairs)
@@ -82,6 +83,10 @@ class OneDeviceExtended(distribute_lib.DistributionStrategyExtended):
     return input_lib.InputFunctionIterator(
         input_fn, self._input_workers, [distribute_lib.InputContext()])
 
+  def _experimental_make_numpy_dataset(self, numpy_input, session):
+    return numpy_dataset.one_host_numpy_dataset(
+        numpy_input, self._input_device, session)
+
   def _broadcast_to(self, tensor, destinations):
     del destinations
     return tensor
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy_test.py b/tensorflow/contrib/distribute/python/one_device_strategy_test.py
index 2403dc8f12..f81466a6c7 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy_test.py
@@ -59,6 +59,10 @@ class OneDeviceStrategyTest(
     self._test_input_fn_iterator(
         iterator, d.extended.worker_devices, expected_values)
 
+  @test_util.run_in_graph_and_eager_modes
+  def testNumpyIterator(self):
+    self._test_numpy_iterator(self._get_distribution_strategy())
+
   def testAllReduceSum(self):
     self._test_all_reduce_sum(self._get_distribution_strategy())
 
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy.py b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
index bb0b8eb992..0785427c2c 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
@@ -89,6 +89,37 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy):
     super(ParameterServerStrategy, self).__init__(
         ParameterServerExtended(self, num_gpus_per_worker))
 
+  # Override to change the documentation to reflect the different handling of
+  # global vs. local batch size between core and contrib.
+  def experimental_make_numpy_iterator(  # pylint: disable=useless-super-delegation
+      self, numpy_input, batch_size, num_epochs=1, shuffle=1024, session=None):
+    """Makes an iterator for input provided via a nest of numpy arrays.
+
+    NOTE: The `batch_size` argument here has different behavior for this
+    contrib version of `ParameterServerStrategy`.
+
+    Args:
+      numpy_input: A nest of NumPy input arrays that will be distributed evenly
+        across all replicas.
+      batch_size: The number of entries from the array we should consume in one
+        step of the computation, across all replicas. This is the per-replica
+        batch size. The global batch size will be this times
+        `num_replicas_in_sync`.
+      num_epochs: The number of times to iterate through the examples. A value
+        of `None` means repeat forever.
+      shuffle: Size of buffer to use for shuffling the input examples.
+        Use `None` to disable shuffling.
+      session: (TensorFlow v1.x graph execution only) A session used for
+        initialization.
+
+    Returns:
+      An `tf.distribute.InputIterator` which returns inputs for each step of the
+      computation.  User should call `initialize` on the returned iterator.
+    """
+    return super(ParameterServerStrategy,
+                 self).experimental_make_numpy_iterator(
+                     numpy_input, batch_size, num_epochs, shuffle, session)
+
 
 class ParameterServerExtended(CoreParameterServerExtended):
   """Implementation of ParameterServerStrategy."""
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
index 7836687e7d..802809e7c7 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
@@ -893,5 +893,17 @@ class ParameterServerStrategyWithChiefTest(ParameterServerStrategyTestBase,
       strategy.extended.call_for_each_replica(f)
 
 
+class LocalParameterServerStrategyTest(strategy_test_lib.DistributionTestBase,
+                                       parameterized.TestCase):
+
+  @combinations.generate(combinations.combine(mode=['graph', 'eager'],
+                                              use_core_strategy=[True, False],
+                                              required_gpus=2))
+  def testNumpyIterator(self, use_core_strategy):
+    strategy, _, _ = create_test_objects(
+        num_gpus=2, use_core_strategy=use_core_strategy)
+    self._test_numpy_iterator(strategy)
+
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/distribute/python/strategy_test_lib.py b/tensorflow/contrib/distribute/python/strategy_test_lib.py
index 4fbd630cf7..7455cbd02a 100644
--- a/tensorflow/contrib/distribute/python/strategy_test_lib.py
+++ b/tensorflow/contrib/distribute/python/strategy_test_lib.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import numpy as np
+
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import distribution_strategy_context as ds_context
@@ -295,6 +297,33 @@ class DistributionTestBase(test.TestCase):
       global_step_values = self.evaluate(global_step_tensors)
       self.assertEqual((1,) * len(global_step_tensors), global_step_values)
 
+  def _test_numpy_iterator(self, strategy):
+    with strategy.scope(), self.cached_session() as sess:
+      x = np.asarray([[1, 2], [6, 12], [2, 4],
+                      [5, 10], [3, 6], [4, 8]])
+      y = np.asarray([5, 4, 3, 2, 1, 0])
+      batch_size = 6
+      if not strategy.extended._global_batch_size:  # pylint: disable=protected-access
+        batch_size = batch_size // strategy.num_replicas_in_sync
+      i = strategy.experimental_make_numpy_iterator(
+          (x, y), batch_size=batch_size, num_epochs=2, shuffle=None,
+          session=sess)
+      self.evaluate(i.initialize())
+
+      def run_and_concatenate(strategy, i):
+        x, y = strategy.experimental_run(lambda z: z, i)
+        x, y = self.evaluate((strategy.unwrap(x), strategy.unwrap(y)))
+        return np.concatenate(x), np.concatenate(y)
+
+      x_1, y_1 = run_and_concatenate(strategy, i)
+      self.assertAllEqual(x, x_1)
+      self.assertAllEqual(y, y_1)
+      x_2, y_2 = run_and_concatenate(strategy, i)
+      self.assertAllEqual(x, x_2)
+      self.assertAllEqual(y, y_2)
+      with self.assertRaises(errors.OutOfRangeError):
+        run_and_concatenate(strategy, i)
+
 
 class OneDeviceDistributionTestBase(test.TestCase):
   """Some tests that should work with any one-device DistributionStrategy."""
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 00360bf996..3f89c5869e 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -34,6 +34,7 @@ from tensorflow.python.distribute import cross_device_ops as cross_device_ops_li
 from tensorflow.python.distribute import device_util
 from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import input_lib
+from tensorflow.python.distribute import numpy_dataset
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.distribute import values
 from tensorflow.python.distribute.cluster_resolver import tpu_cluster_resolver as resolver_lib
@@ -303,6 +304,11 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
         functools.partial(self._call_dataset_fn, dataset_fn),
         self._input_workers)
 
+  def _experimental_make_numpy_dataset(self, numpy_input, session):
+    return numpy_dataset.one_host_numpy_dataset(
+        numpy_input, numpy_dataset.SingleDevice(self.get_host_cpu_device(0)),
+        session)
+
   # TODO(priyag): Deal with OutOfRange errors once b/111349762 is fixed.
   # TODO(sourabhbajaj): Remove the initial_loop_values parameter when we have
   # a mechanism to infer the outputs of `fn`. Pending b/110550782.
@@ -466,6 +472,9 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
     if colocate_with is None:
       device_map = self._device_map
       logical_device = 0  # TODO(josh11b): Get logical device from scope here.
+    elif isinstance(colocate_with, numpy_dataset.SingleDevice):
+      with ops.device(colocate_with.device):
+        return next_creator(*args, **kwargs)
     else:
       device_map = colocate_with.device_map
       logical_device = colocate_with.logical_device
diff --git a/tensorflow/python/distribute/BUILD b/tensorflow/python/distribute/BUILD
index faaa61934a..a6a1c470b4 100644
--- a/tensorflow/python/distribute/BUILD
+++ b/tensorflow/python/distribute/BUILD
@@ -124,6 +124,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":device_util",
+        ":numpy_dataset",
         ":reduce_util",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:constant_op",
@@ -221,6 +222,7 @@ py_library(
         ":distribute_lib",
         ":input_lib",
         ":multi_worker_util",
+        ":numpy_dataset",
         ":reduce_util",
         ":shared_variable_creator",
         ":values",
@@ -249,6 +251,7 @@ py_library(
     deps = [
         ":input_lib",
         ":mirrored_strategy",
+        ":numpy_dataset",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:framework_ops",
@@ -276,6 +279,35 @@ py_library(
     ],
 )
 
+py_library(
+    name = "numpy_dataset",
+    srcs = ["numpy_dataset.py"],
+    deps = [
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:util",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/eager:context",
+        "//third_party/py/numpy",
+    ],
+)
+
+py_test(
+    name = "numpy_dataset_test",
+    size = "small",
+    srcs = ["numpy_dataset_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":numpy_dataset",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python/eager:test",
+        "//third_party/py/numpy",
+    ],
+)
+
 py_library(
     name = "input_lib",
     srcs = ["input_lib.py"],
diff --git a/tensorflow/python/distribute/distribute_lib.py b/tensorflow/python/distribute/distribute_lib.py
index 4ad8cc00b8..31213ab647 100644
--- a/tensorflow/python/distribute/distribute_lib.py
+++ b/tensorflow/python/distribute/distribute_lib.py
@@ -26,6 +26,7 @@ import enum
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import device_util
 from tensorflow.python.distribute import distribution_strategy_context
+from tensorflow.python.distribute import numpy_dataset
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.eager import context as eager_context
 from tensorflow.python.framework import constant_op
@@ -360,7 +361,7 @@ class DistributionStrategy(object):
     return self._extended._distribute_dataset(dataset_fn)  # pylint: disable=protected-access
 
   def make_dataset_iterator(self, dataset):
-    """Makes an iterator for input provided via input_dataset.
+    """Makes an iterator for input provided via `dataset`.
 
     Data from the given dataset will be distributed evenly across all the
     compute replicas. We will assume that the input dataset is batched by the
@@ -418,6 +419,40 @@ class DistributionStrategy(object):
     return self.extended._make_input_fn_iterator(  # pylint: disable=protected-access
         input_fn, replication_mode=replication_mode)
 
+  def experimental_make_numpy_iterator(
+      self, numpy_input, batch_size, num_epochs=1, shuffle=1024, session=None):
+    """Makes an iterator for input provided via a nest of numpy arrays.
+
+    Args:
+      numpy_input: A nest of NumPy input arrays that will be distributed evenly
+        across all replicas. Note that lists of Numpy arrays are stacked,
+        as that is normal `tf.data.Dataset` behavior.
+      batch_size: The number of entries from the array we should consume in one
+        step of the computation, across all replicas. This is the global batch
+        size. It should be divisible by `num_replicas_in_sync`.
+      num_epochs: The number of times to iterate through the examples. A value
+        of `None` means repeat forever.
+      shuffle: Size of buffer to use for shuffling the input examples.
+        Use `None` to disable shuffling.
+      session: (TensorFlow v1.x graph execution only) A session used for
+        initialization.
+
+    Returns:
+      An `tf.distribute.InputIterator` which returns inputs for each step of the
+      computation.  User should call `initialize` on the returned iterator.
+    """
+    ds = self.extended.experimental_make_numpy_dataset(
+        numpy_input, session=session)
+    if shuffle:
+      ds = ds.shuffle(shuffle)
+    if num_epochs != 1:
+      ds = ds.repeat(num_epochs)
+    # We need to use the drop_remainder argument to get a known static
+    # input shape which is required for TPUs.
+    drop_remainder = self.extended.experimental_require_static_shapes
+    ds = ds.batch(batch_size, drop_remainder=drop_remainder)
+    return self.make_dataset_iterator(ds)
+
   def experimental_run(self, fn, input_iterator=None):
     """Runs ops in `fn` on each replica, with inputs from `input_iterator`.
 
@@ -1083,6 +1118,29 @@ class DistributionStrategyExtended(object):
   def _make_input_fn_iterator(self, input_fn, replication_mode):
     raise NotImplementedError("must be implemented in descendants")
 
+  def experimental_make_numpy_dataset(self, numpy_input, session=None):
+    """Makes a dataset for input provided via a numpy array.
+
+    This avoids adding `numpy_input` as a large constant in the graph,
+    and copies the data to the machine or machines that will be processing
+    the input.
+
+    Args:
+      numpy_input: A nest of NumPy input arrays that will be distributed evenly
+        across all replicas. Note that lists of Numpy arrays are stacked,
+        as that is normal `tf.data.Dataset` behavior.
+      session: (TensorFlow v1.x graph execution only) A session used for
+        initialization.
+
+    Returns:
+      A `tf.data.Dataset` representing `numpy_input`.
+    """
+    _require_cross_replica_context_extended(self)
+    return self._experimental_make_numpy_dataset(numpy_input, session=session)
+
+  def _experimental_make_numpy_dataset(self, numpy_input, session):
+    raise NotImplementedError("must be implemented in descendants")
+
   def broadcast_to(self, tensor, destinations):
     """Mirror a tensor on one device to all worker devices.
 
@@ -1660,6 +1718,18 @@ class _DefaultDistributionExtended(DistributionStrategyExtended):
                               replication_mode=InputReplicationMode.PER_WORKER):
     return input_fn(InputContext()).make_initializable_iterator()
 
+  def _experimental_make_numpy_dataset(self, numpy_input, session):
+    numpy_flat = nest.flatten(numpy_input)
+    vars_flat = tuple(
+        variable_scope.variable(array_ops.zeros(i.shape, i.dtype),
+                                trainable=False, use_resource=True)
+        for i in numpy_flat
+    )
+    for v, i in zip(vars_flat, numpy_flat):
+      numpy_dataset.init_var_from_numpy(v, i, session)
+    vars_nested = nest.pack_sequence_as(numpy_input, vars_flat)
+    return dataset_ops.Dataset.from_tensor_slices(vars_nested)
+
   def _broadcast_to(self, tensor, destinations):
     if destinations is None:
       return tensor
diff --git a/tensorflow/python/distribute/mirrored_strategy.py b/tensorflow/python/distribute/mirrored_strategy.py
index 37b493d0f7..c0a39d4b55 100644
--- a/tensorflow/python/distribute/mirrored_strategy.py
+++ b/tensorflow/python/distribute/mirrored_strategy.py
@@ -29,6 +29,7 @@ from tensorflow.python.distribute import device_util
 from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import input_lib
 from tensorflow.python.distribute import multi_worker_util
+from tensorflow.python.distribute import numpy_dataset
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.distribute import shared_variable_creator
 from tensorflow.python.distribute import values
@@ -460,6 +461,7 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
     self._input_workers = input_lib.InputWorkers(self._device_map)
     self._inferred_cross_device_ops = cross_device_ops_lib.choose_the_best(
         devices)
+    self._host_input_device = numpy_dataset.SingleDevice("/cpu:0")
 
   def _initialize_multi_worker(self, devices):
     """Initializes the object for multi-worker training."""
@@ -488,6 +490,7 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
     # their ops will end up on the cpu device of its first worker, e.g.
     # "/job:worker/task:0/device:CPU:0". Note this is not used in replica mode.
     self._default_device = workers[0]
+    self._host_input_device = numpy_dataset.SingleDevice(workers[0])
 
     self._device_map = values.ReplicaDeviceMap(devices)
     self._input_workers = input_lib.InputWorkers(
@@ -501,6 +504,9 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
     if colocate_with is None:
       device_map = self._device_map
       logical_device = 0  # TODO(josh11b): Get logical device from scope here.
+    elif isinstance(colocate_with, numpy_dataset.SingleDevice):
+      with ops.device(colocate_with.device):
+        return next_creator(*args, **kwargs)
     else:
       device_map = colocate_with.device_map
       logical_device = colocate_with.logical_device
@@ -571,6 +577,10 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
     return input_lib.InputFunctionIterator(
         input_fn, self._input_workers, input_contexts)
 
+  def _experimental_make_numpy_dataset(self, numpy_input, session):
+    return numpy_dataset.one_host_numpy_dataset(
+        numpy_input, self._host_input_device, session)
+
   # TODO(priyag): Deal with OutOfRange errors once b/111349762 is fixed.
   def _experimental_run_steps_on_iterator(self, fn, iterator, iterations,
                                           initial_loop_values=None):
diff --git a/tensorflow/python/distribute/numpy_dataset.py b/tensorflow/python/distribute/numpy_dataset.py
new file mode 100644
index 0000000000..5881e4cd59
--- /dev/null
+++ b/tensorflow/python/distribute/numpy_dataset.py
@@ -0,0 +1,97 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Code for creating a dataset out of a NumPy array."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.eager import context
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.util import nest
+
+
+def init_var_from_numpy(input_var, numpy_input, session):
+  """Initialize `input_var` to `numpy_input` using `session` in graph mode."""
+  with ops.init_scope():
+    if context.executing_eagerly():
+      input_var.assign(numpy_input)
+      return
+
+    assert session is not None
+    session.run(input_var.initializer)
+
+    start_placeholder = array_ops.placeholder(dtypes.int64, ())
+    end_placeholder = array_ops.placeholder(dtypes.int64, ())
+    slice_placeholder = array_ops.placeholder(input_var.dtype)
+    assign_slice_op = input_var[start_placeholder:end_placeholder].assign(
+        slice_placeholder)
+
+    # If each batch element is > 64 MB, then we copy each batch element
+    # individually. Otherwise, the slices will be < 128 MB. There might be
+    # padding which might mean that the slices are 128 MB even if the size of
+    # the tensor allocated is less than 128 MB.  This formula gives slices with
+    # size: ceil(64 MB / byte size per batch element) bytes.  Using ceil()
+    # guarantees we get a number >= 1.
+
+    # Calculate the size of each batch element.
+    byte_size_per_batch_element = (
+        np.prod(numpy_input.shape[1:]) * input_var.dtype.size)
+
+    # Calculate number of elements we want to copy per slice.
+    batch_size_per_slice = int(
+        np.ceil((64 << 20) / byte_size_per_batch_element))
+
+    # Copy slices of the above size starting at 0, except the last slice will be
+    # smaller.
+    start = 0
+    limit = numpy_input.shape[0]
+    while start < limit:
+      end = min(start + batch_size_per_slice, limit)
+      session.run(assign_slice_op, feed_dict={
+          start_placeholder: start,
+          end_placeholder: end,
+          slice_placeholder: numpy_input[start:end]})
+      start = end
+
+
+def one_host_numpy_dataset(numpy_input, colocate_with, session):
+  """Create a dataset on `colocate_with` from `numpy_input`."""
+  def create_colocated_variable(next_creator, *args, **kwargs):
+    kwargs["colocate_with"] = colocate_with
+    return next_creator(*args, **kwargs)
+
+  numpy_flat = nest.flatten(numpy_input)
+  with variable_scope.variable_creator_scope(create_colocated_variable):
+    vars_flat = tuple(variable_scope.variable(array_ops.zeros(i.shape, i.dtype),
+                                              trainable=False)
+                      for i in numpy_flat)
+  for v, i in zip(vars_flat, numpy_flat):
+    init_var_from_numpy(v, i, session)
+  vars_nested = nest.pack_sequence_as(numpy_input, vars_flat)
+  return dataset_ops.Dataset.from_tensor_slices(vars_nested)
+
+
+class SingleDevice(object):
+  """Used with `colocate_with` to create a non-mirrored variable."""
+
+  def __init__(self, device):
+    self.device = device
diff --git a/tensorflow/python/distribute/numpy_dataset_test.py b/tensorflow/python/distribute/numpy_dataset_test.py
new file mode 100644
index 0000000000..04eae1daa2
--- /dev/null
+++ b/tensorflow/python/distribute/numpy_dataset_test.py
@@ -0,0 +1,44 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for numpy_dataset."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.distribute import numpy_dataset
+from tensorflow.python.eager import test
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import variable_scope
+
+
+class InitVarFromNumpyTest(test.TestCase):
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_creating_var_with_numpy_arrays(self):
+    with self.cached_session() as session:
+      x = np.asarray(np.random.random((64, 3)), dtype=np.float32)
+      initial = np.zeros_like(x)
+      var_x = variable_scope.variable(initial)
+      numpy_dataset.init_var_from_numpy(var_x, x, session)
+      val = self.evaluate(var_x.value())
+      # Verify that the numpy value is copied to the variable.
+      self.assertAllEqual(x, val)
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/python/distribute/parameter_server_strategy.py b/tensorflow/python/distribute/parameter_server_strategy.py
index 71fbffdc0d..ac5ee6f589 100644
--- a/tensorflow/python/distribute/parameter_server_strategy.py
+++ b/tensorflow/python/distribute/parameter_server_strategy.py
@@ -26,6 +26,7 @@ from tensorflow.python.distribute import device_util
 from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import input_lib
 from tensorflow.python.distribute import multi_worker_util
+from tensorflow.python.distribute import numpy_dataset
 from tensorflow.python.distribute import values
 from tensorflow.python.distribute.cluster_resolver import SimpleClusterResolver
 from tensorflow.python.distribute.cluster_resolver import TFConfigClusterResolver
@@ -137,6 +138,7 @@ class ParameterServerStrategyExtended(
     assert cluster_spec.as_dict()
 
     worker_device = "/job:%s/task:%d" % (task_type, task_id)
+    self._input_host_device = numpy_dataset.SingleDevice(worker_device)
 
     # Define compute devices which is a list of device strings and one for each
     # replica. When there are GPUs, replicate operations on these GPUs.
@@ -195,6 +197,7 @@ class ParameterServerStrategyExtended(
   def _initialize_local(self, cluster_resolver):
     """Initialize internal devices for local training."""
     worker_device = device_util.canonicalize("/device:CPU:0")
+    self._input_host_device = numpy_dataset.SingleDevice(worker_device)
     num_gpus = cluster_resolver.num_accelerators()
     # Define compute devices which is a list of device strings and one for each
     # replica. When there are GPUs, replicate operations on these GPUs.
@@ -262,6 +265,10 @@ class ParameterServerStrategyExtended(
     return input_lib.InputFunctionIterator(input_fn, self._input_workers,
                                            [input_context])
 
+  def _experimental_make_numpy_dataset(self, numpy_input, session):
+    return numpy_dataset.one_host_numpy_dataset(
+        numpy_input, self._input_host_device, session)
+
   def _broadcast_to(self, tensor, destinations):
     # This is both a fast path for Python constants, and a way to delay
     # converting Python values to a tensor until we know what type it
@@ -329,8 +336,12 @@ class ParameterServerStrategyExtended(
       var_creator = next_creator
 
     if "colocate_with" in kwargs:
+      colocate_with = kwargs["colocate_with"]
+      if isinstance(colocate_with, numpy_dataset.SingleDevice):
+        with ops.device(colocate_with.device):
+          return var_creator(*args, **kwargs)
       with ops.device(None):
-        with ops.colocate_with(kwargs["colocate_with"]):
+        with ops.colocate_with(colocate_with):
           return var_creator(*args, **kwargs)
 
     with ops.colocate_with(None, ignore_existing=True):
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index 1678d84307..b6ef33f700 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -34,25 +34,13 @@ from tensorflow.python.keras import callbacks
 from tensorflow.python.keras import metrics as metrics_module
 from tensorflow.python.keras import optimizers
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
-from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.training import distribution_strategy_context
 from tensorflow.python.training.mode_keys import ModeKeys
 from tensorflow.python.util import nest
 
 
-def validate_not_in_strategy_scope():
-  """Validate fit/eval/predict are not running in DS scope."""
-  if distribution_strategy_context.has_distribution_strategy():
-    if distribution_strategy_context.in_cross_replica_context():
-      raise RuntimeError(
-          'Fit/Eval/Predict should not be run inside the tf.distribute.Strategy'
-          ' scope. Only model creation and compilation should be in '
-          'tf.distribute.Strategy scope.')
-
-
 def set_weights(distribution_strategy, dist_model, weights):
   """Sets the weights of the replicated models.
 
@@ -530,100 +518,11 @@ def get_batch_dimension(iterator):
   return dims[0] if dims else None
 
 
-def get_cpu_device(distribution_strategy):
-  """Returns the CPU device of the TPU host or the default CPU device string.
-
-  Args:
-    distribution_strategy: The DistributionStrategy used to compile the model.
-
-  Returns:
-    A device string which is the TPU host's CPU device in case of
-    TPUDistributionStrategy or the default CPU device string in all other
-    cases.
-
-  Raises:
-    NotImplementedError: We currently don't support copying numpy data to
-    multiple hosts in the case of Cloud TPU pods.
-  """
-  if is_tpu_strategy(distribution_strategy):
-    if distribution_strategy.extended.num_hosts > 1:
-      raise NotImplementedError('TPUDistributionStrategy does not '
-                                'support numpy inputs when running on Cloud'
-                                'TPU pods.')
-    return distribution_strategy.extended.get_host_cpu_device(0)
-  else:
-    # For all strategies except TPUDistributionStrategy
-    # TODO(anjalisridhar): We may need to modify this when we add support for
-    # multi-worker strategy.
-    return '/CPU:0'
-
-
-def get_var_for_numpy(distribution_strategy, x):
-  if isinstance(x, list):
-    var_x = tuple([_get_var_for_numpy(distribution_strategy, single_input)
-                   for single_input in x])
-  else:
-    var_x = _get_var_for_numpy(distribution_strategy, x)
-  return var_x
-
-
-def _get_var_for_numpy(distribution_strategy, input_array):
-  """Creates a variable and assigns the value of the numpy array to it.
-
-  Args:
-    distribution_strategy: The DistributionStrategy used to compile the model.
-    input_array: The input numpy array whose value will be assigned to the
-      variable we create.
-
-  Returns:
-    The variable to which we will copy the value of the input numpy array.
-
-  """
-  with ops.device(get_cpu_device(distribution_strategy)):
-    # Create and initialize a variable on the CPU device. This is the CPU
-    # device of the host in the case of TPUDistributionStrategy.
-    input_var = variables.VariableV1(array_ops.zeros(input_array.shape,
-                                                     input_array.dtype),
-                                     trainable=False, use_resource=True)
-  K.get_session().run(input_var.initializer)
-
-  # Create a placeholder for the numpy array input slices. We copy the value
-  # of the input numpy array to the variable in slices of size 64 MB to avoid
-  # running into memory issues or RPC message limits.
-  start_placeholder = array_ops.placeholder(dtypes.int64, ())
-  end_placeholder = array_ops.placeholder(dtypes.int64, ())
-  slice_placeholder = array_ops.placeholder(input_var.dtype)
-  assign_slice_op = input_var[start_placeholder:end_placeholder].assign(
-      slice_placeholder)
-
-  # If each batch element is > 64 MB, then we copy each batch element
-  # individually. Otherwise, the slices will be < 128 MB. There might be padding
-  # which might mean that the slices are 128 MB even if the size of the
-  # tensor allocated is less than 128 MB.
-  # This formula gives slices with size:
-  # ceil(64 MB / byte size per batch element) bytes.
-  # Using ceil() guarantees we get a number >= 1.
-
-  # Calculate the size of each batch element.
-  byte_size_per_batch_element = np.prod(input_array.shape[1:]) * \
-                                input_var.dtype.size
-
-  # Calculate number of elements we want to copy per slice.
-  batch_size_per_slice = int(np.ceil((64 << 20) / byte_size_per_batch_element))
-
-  # Copy slices of the above size starting at 0, except the last slice will be
-  # smaller.
-  start = 0
-  limit = input_array.shape[0]
-  while start < limit:
-    end = min(start + batch_size_per_slice, limit)
-    K.get_session().run(assign_slice_op, feed_dict={
-        start_placeholder: start,
-        end_placeholder: end,
-        slice_placeholder: input_array[start:end]})
-    start = end
-
-  return input_var
+def list_to_tuple(maybe_list):
+  """Datasets treat lists specially, so switch them to tuples."""
+  if isinstance(maybe_list, list):
+    return tuple(maybe_list)
+  return maybe_list
 
 
 def _get_input_from_iterator(iterator, model):
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 1eda8cf797..a65c2b6413 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -2161,53 +2161,47 @@ class Model(Network):
                        'you should specify the `{steps_name}` argument.'
                        .format(steps_name=steps_name))
 
-    first_x_value = nest.flatten(x)[0]
-    if isinstance(first_x_value, np.ndarray):
-      # We need to use the drop_remainder argument to allow for a static
-      # input shape which is required for TPUs.
-      drop_remainder = self._distribution_strategy.require_static_shapes
-      if y is not None:
-        var_x = distributed_training_utils.get_var_for_numpy(
-            self._distribution_strategy, x)
-        var_y = distributed_training_utils.get_var_for_numpy(
-            self._distribution_strategy, y)
-        if sample_weight is not None:
-          var_sample_weights = distributed_training_utils.get_var_for_numpy(
-              self._distribution_strategy, sample_weight)
-
-          x = dataset_ops.Dataset.from_tensor_slices((var_x, var_y,
-                                                      var_sample_weights))
+    if ops.executing_eagerly_outside_functions():
+      session = None
+    else:
+      session = K.get_session()
+
+    with self._distribution_strategy.scope():
+      first_x_value = nest.flatten(x)[0]
+      if isinstance(first_x_value, np.ndarray):
+        x = distributed_training_utils.list_to_tuple(x)
+        if y is not None:
+          y = distributed_training_utils.list_to_tuple(y)
+          if sample_weight is not None:
+            sample_weight = distributed_training_utils.list_to_tuple(
+                sample_weight)
+            in_tuple = (x, y, sample_weight)
+          else:
+            in_tuple = (x, y)
         else:
-          x = dataset_ops.Dataset.from_tensor_slices((var_x, var_y))
+          in_tuple = x
 
         if shuffle:
           # 1024 is a good buffer size since it is much larger than the average
           # batch size provided by the user and provides sufficient randomness.
           # One thing to keep in mind is the memory usage based on the size of
           # each sample.
-          x = x.shuffle(1024)
-        x = x.repeat()
-        x = x.batch(batch_size, drop_remainder=drop_remainder)
-        y = None
-        sample_weight = None
+          shuffle_buffer = 1024
+        else:
+          shuffle_buffer = None
+        iterator = self._distribution_strategy.experimental_make_numpy_iterator(
+            in_tuple, batch_size, num_epochs=None, shuffle=shuffle_buffer,
+            session=session)
       else:
-        # This case is for the predict call where the dataset only contains
-        # inputs and no targets, i.e. it does not return a tuple
-        var_x = distributed_training_utils.get_var_for_numpy(
-            self._distribution_strategy, x)
-        x = dataset_ops.Dataset.from_tensor_slices(var_x)
-        x = x.batch(batch_size, drop_remainder=drop_remainder)
-
-    assert isinstance(x, dataset_ops.DatasetV2)
+        assert isinstance(x, dataset_ops.DatasetV2)
+        training_utils.validate_dataset_input(x, y, sample_weight,
+                                              validation_split)
+        iterator = self._distribution_strategy.make_dataset_iterator(x)
 
-    with self._distribution_strategy.scope():
-      iterator = self._distribution_strategy.make_dataset_iterator(x)
       init_op = iterator.initialize()
       if not context.executing_eagerly():
         K.get_session().run(init_op)
 
-    training_utils.validate_dataset_input(x, y, sample_weight,
-                                          validation_split)
     return iterator
 
   def _standardize_user_data(self,
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 92a46e399a..0bd79f2b47 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -59,8 +59,6 @@ def fit_distributed(model,
 
   first_x_value = nest.flatten(x)[0]
   if isinstance(first_x_value, np.ndarray):
-    # TODO(b/122314600): Remove the scope validate.
-    distributed_training_utils.validate_not_in_strategy_scope()
     steps_per_epoch, batch_size = (
         distributed_training_utils.get_input_params(
             model._distribution_strategy, first_x_value, steps_per_epoch,
@@ -141,8 +139,6 @@ def evaluate_distributed(model,
   distributed_training_utils.validate_inputs(x, y, model._distribution_strategy)
   first_x_value = nest.flatten(x)[0]
   if isinstance(first_x_value, np.ndarray):
-    # TODO(b/122314600): Remove the scope validate.
-    distributed_training_utils.validate_not_in_strategy_scope()
     steps, batch_size = distributed_training_utils.get_input_params(
         model._distribution_strategy, first_x_value, steps, batch_size)
   batch_size = model._validate_or_infer_batch_size(batch_size, steps, x)
@@ -179,8 +175,6 @@ def predict_distributed(model,
       x, None, model._distribution_strategy)
   first_x_value = nest.flatten(x)[0]
   if isinstance(first_x_value, np.ndarray):
-    # TODO(b/122314600): Remove the scope validate.
-    distributed_training_utils.validate_not_in_strategy_scope()
     steps, batch_size = distributed_training_utils.get_input_params(
         model._distribution_strategy, first_x_value, steps, batch_size)
   batch_size = model._validate_or_infer_batch_size(batch_size, steps, x)
diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py
index eaa563e84a..c6cc0b6044 100644
--- a/tensorflow/python/training/optimizer.py
+++ b/tensorflow/python/training/optimizer.py
@@ -554,14 +554,15 @@ class Optimizer(
     # by most optimizers.  It relies on the subclass implementing the following
     # methods: _create_slots(), _prepare(), _apply_dense(), and _apply_sparse().
 
-    # Handle DistributionStrategy case.
-    if distribute_ctx.get_cross_replica_context():
-      raise RuntimeError("Use `_distributed_apply()` instead of "
-                         "`apply_gradients()` in a cross-replica context.")
-    # TODO(isaprykin): Get rid of `has_distribution_strategy()` check by
+    # TODO(isaprykin): Get rid of `has_strategy()` check by
     # always calling _distributed_apply(), using the default distribution
     # as needed.
-    if distribute_ctx.has_distribution_strategy():
+    if distribute_ctx.has_strategy():
+      # Handle DistributionStrategy case.
+      if distribute_ctx.in_cross_replica_context():
+        raise RuntimeError("Use `_distributed_apply()` instead of "
+                           "`apply_gradients()` in a cross-replica context.")
+
       grads_and_vars = get_filtered_grad_fn(lambda: grads_and_vars)()
       return distribute_ctx.get_replica_context().merge_call(
           self._distributed_apply, args=(grads_and_vars, global_step, name))
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
index b06c73d126..9c29067b6d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
@@ -75,6 +75,10 @@ tf_class {
     name: "experimental_initialize"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "experimental_make_numpy_iterator"
+    argspec: "args=[\'self\', \'numpy_input\', \'batch_size\', \'num_epochs\', \'shuffle\', \'session\'], varargs=None, keywords=None, defaults=[\'1\', \'1024\', \'None\'], "
+  }
   member_method {
     name: "experimental_run"
     argspec: "args=[\'self\', \'fn\', \'input_iterator\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy-extended.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy-extended.pbtxt
index 77706e5713..37b620891f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy-extended.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy-extended.pbtxt
@@ -50,6 +50,10 @@ tf_class {
     name: "colocate_vars_with"
     argspec: "args=[\'self\', \'colocate_with_variable\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "experimental_make_numpy_dataset"
+    argspec: "args=[\'self\', \'numpy_input\', \'session\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "experimental_run_steps_on_iterator"
     argspec: "args=[\'self\', \'fn\', \'iterator\', \'iterations\', \'initial_loop_values\'], varargs=None, keywords=None, defaults=[\'1\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
index 9a1df55142..4aa6f1c4e1 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
@@ -74,6 +74,10 @@ tf_class {
     name: "experimental_initialize"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "experimental_make_numpy_iterator"
+    argspec: "args=[\'self\', \'numpy_input\', \'batch_size\', \'num_epochs\', \'shuffle\', \'session\'], varargs=None, keywords=None, defaults=[\'1\', \'1024\', \'None\'], "
+  }
   member_method {
     name: "experimental_run"
     argspec: "args=[\'self\', \'fn\', \'input_iterator\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
index b06c73d126..9c29067b6d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
@@ -75,6 +75,10 @@ tf_class {
     name: "experimental_initialize"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "experimental_make_numpy_iterator"
+    argspec: "args=[\'self\', \'numpy_input\', \'batch_size\', \'num_epochs\', \'shuffle\', \'session\'], varargs=None, keywords=None, defaults=[\'1\', \'1024\', \'None\'], "
+  }
   member_method {
     name: "experimental_run"
     argspec: "args=[\'self\', \'fn\', \'input_iterator\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy-extended.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy-extended.pbtxt
index 77706e5713..37b620891f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy-extended.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy-extended.pbtxt
@@ -50,6 +50,10 @@ tf_class {
     name: "colocate_vars_with"
     argspec: "args=[\'self\', \'colocate_with_variable\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "experimental_make_numpy_dataset"
+    argspec: "args=[\'self\', \'numpy_input\', \'session\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "experimental_run_steps_on_iterator"
     argspec: "args=[\'self\', \'fn\', \'iterator\', \'iterations\', \'initial_loop_values\'], varargs=None, keywords=None, defaults=[\'1\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
index 9a1df55142..4aa6f1c4e1 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
@@ -74,6 +74,10 @@ tf_class {
     name: "experimental_initialize"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "experimental_make_numpy_iterator"
+    argspec: "args=[\'self\', \'numpy_input\', \'batch_size\', \'num_epochs\', \'shuffle\', \'session\'], varargs=None, keywords=None, defaults=[\'1\', \'1024\', \'None\'], "
+  }
   member_method {
     name: "experimental_run"
     argspec: "args=[\'self\', \'fn\', \'input_iterator\'], varargs=None, keywords=None, defaults=[\'None\'], "
-- 
GitLab


From c94be5c69b9e540c27cad8c1d3f8604606874d00 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 19 Dec 2018 16:08:48 -0800
Subject: [PATCH 0448/2345] initial commit

---
 .../contrib/tensorrt/ops/trt_engine_op.cc     |  4 +-
 .../contrib/tensorrt/test/multiple_types.py   | 94 +++++++++++++++++++
 2 files changed, 96 insertions(+), 2 deletions(-)
 create mode 100644 tensorflow/contrib/tensorrt/test/multiple_types.py

diff --git a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
index 92405906eb..1dce862568 100644
--- a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
@@ -33,8 +33,8 @@ REGISTER_OP("TRTEngineOp")
     .Attr("input_shapes: list(shape)")
     .Attr("output_shapes: list(shape)")
     .Attr("segment_funcdef_name: string")
-    .Attr("InT: list({int8,float16,float32})")
-    .Attr("OutT: list({int8,float16,float32})")
+    .Attr("InT: list({int8,float16,float32,int32})")
+    .Attr("OutT: list({int8,float16,float32,int32})")
     .Attr("static_engine: bool = true")
     .Attr("fixed_input_size: bool = true")
     .Attr("cached_engine_batches: list(int) = []")
diff --git a/tensorflow/contrib/tensorrt/test/multiple_types.py b/tensorflow/contrib/tensorrt/test/multiple_types.py
new file mode 100644
index 0000000000..35b5a8519a
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/test/multiple_types.py
@@ -0,0 +1,94 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Model script to test TF-TensorRT integration."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gen_array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn
+from tensorflow.python.platform import test
+
+
+class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
+
+  def _ConstOp(self, shape, dtype):
+    return constant_op.constant(np.random.randn(*shape), dtype=dtype)
+
+  def GetParams(self):
+    """Testing conversion of BiasAdd MatMul in TF-TRT conversion."""
+    # Note that tf.nn.bias_add supports up to 5 dimensions.
+    input_dims = [100, 4]
+    output_name = "output"
+    g = ops.Graph()
+    with g.as_default():
+      input1 = array_ops.placeholder(
+          dtype=dtypes.float32, shape=input_dims, name='input_float32')
+
+      b1 = self._ConstOp((4, 10), dtypes.float32)
+      x1 = math_ops.matmul(input1, b)
+      b1 = self._ConstOp((1, 10), dtypes.float32)
+      x1 = x1 + b1
+
+      input2 = array_ops.placeholder(
+          dtype=dtypes.int32, shape=input_dims, name='input_int32')
+
+      b2 = self._ConstOp((4, 10), dtypes.float32)
+      x2 = math_ops.matmul(input2, b)
+      b2 = self._ConstOp((1, 10), dtypes.float32)
+      x2 = x2 + b2
+      out = array_ops.concat([x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11],
+                             axis=-1)
+      out = array_ops.squeeze(out, name=output_name)
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=[input_name],
+        input_dims=[input_dims],
+        output_names=[output_name],
+        expected_output_dims=[(4, 6680)])
+
+  def GetConversionParams(self, run_params):
+    """Return a ConversionParams for test."""
+    conversion_params = super(BiasaddMatMulTest,
+                              self).GetConversionParams(run_params)
+    return conversion_params._replace(
+        max_batch_size=4,
+        maximum_cached_engines=1,
+        # Disable layout optimizer, since it will convert BiasAdd with NHWC
+        # format to NCHW format under four dimentional input.
+        rewriter_config=trt_test.OptimizerDisabledRewriterConfig())
+
+  def ExpectedEnginesToBuild(self, run_params):
+    """Return the expected engines to build."""
+    return ["TRTEngineOp_0"]
+
+  def ShouldRunTest(self, run_params):
+    """Whether to run the test."""
+    # TODO(aaroey): Trt 4.0 forbids conversion for tensors with rank <3 in int8
+    # mode, which is a bug. Re-enable this when trt library is fixed.
+    return not trt_test.IsQuantizationMode(run_params.precision_mode)
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From 62d6bdef48601770d6fc0e335b5602a4a86c6d00 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 21 Dec 2018 11:45:00 -0800
Subject: [PATCH 0449/2345] rename test

---
 tensorflow/contrib/tensorrt/BUILD             |  1 +
 .../test/{multiple_types.py => int_test.py}   | 26 ++++++++++---------
 2 files changed, 15 insertions(+), 12 deletions(-)
 rename tensorflow/contrib/tensorrt/test/{multiple_types.py => int_test.py} (86%)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 3d34b91a74..e58a259a18 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -493,6 +493,7 @@ cuda_py_tests(
         "test/const_broadcast_test.py",
         "test/conv2d_test.py",
         "test/identity_output_test.py",
+        "test/int_test.py",
         "test/manual_test.py",
         "test/memory_alignment_test.py",
         "test/multi_connection_neighbor_engine_test.py",
diff --git a/tensorflow/contrib/tensorrt/test/multiple_types.py b/tensorflow/contrib/tensorrt/test/int_test.py
similarity index 86%
rename from tensorflow/contrib/tensorrt/test/multiple_types.py
rename to tensorflow/contrib/tensorrt/test/int_test.py
index 35b5a8519a..a4d625a213 100644
--- a/tensorflow/contrib/tensorrt/test/multiple_types.py
+++ b/tensorflow/contrib/tensorrt/test/int_test.py
@@ -40,33 +40,35 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
     """Testing conversion of BiasAdd MatMul in TF-TRT conversion."""
     # Note that tf.nn.bias_add supports up to 5 dimensions.
     input_dims = [100, 4]
-    output_name = "output"
     g = ops.Graph()
     with g.as_default():
+      # float part
       input1 = array_ops.placeholder(
           dtype=dtypes.float32, shape=input_dims, name='input_float32')
-
       b1 = self._ConstOp((4, 10), dtypes.float32)
       x1 = math_ops.matmul(input1, b)
       b1 = self._ConstOp((1, 10), dtypes.float32)
       x1 = x1 + b1
 
+      # int part
       input2 = array_ops.placeholder(
           dtype=dtypes.int32, shape=input_dims, name='input_int32')
-
-      b2 = self._ConstOp((4, 10), dtypes.float32)
+      b2 = self._ConstOp((4, 10), dtypes.int32)
       x2 = math_ops.matmul(input2, b)
-      b2 = self._ConstOp((1, 10), dtypes.float32)
+      b2 = self._ConstOp((1, 10), dtypes.int32)
       x2 = x2 + b2
-      out = array_ops.concat([x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11],
-                             axis=-1)
-      out = array_ops.squeeze(out, name=output_name)
+
+      # combine
+      #y = x1 * x2
+
+      out1 = array_ops.identity(x1, name='output_float32')
+      out2 = array_ops.identity(x2, name='output_int32')
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
-        input_names=[input_name],
-        input_dims=[input_dims],
-        output_names=[output_name],
-        expected_output_dims=[(4, 6680)])
+        input_names=['input_float32', 'input_int32'],
+        input_dims=[[100, 4], [100, 4]],
+        output_names=['output_int32', 'output_float32'],
+        expected_output_dims=[(100, 10), (100, 10)])
 
   def GetConversionParams(self, run_params):
     """Return a ConversionParams for test."""
-- 
GitLab


From dd88f64811365d313423e9b28148921ae0f92003 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 21 Dec 2018 14:17:04 -0800
Subject: [PATCH 0450/2345] fix segfault in this case. should add a check to
 GetDeviceAndAllocator

---
 .../contrib/tensorrt/convert/convert_graph.cc | 13 ++++++++
 .../contrib/tensorrt/convert/convert_nodes.cc | 30 ++++++++++++------
 .../contrib/tensorrt/segment/segment.cc       |  2 +-
 tensorflow/contrib/tensorrt/test/int_test.py  | 31 ++++++++++---------
 4 files changed, 51 insertions(+), 25 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index bf2de94e04..ffe1840c68 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -356,10 +356,18 @@ tensorflow::Status GetEngineInfo(
     if (segment_nodes.count(node_name) == 0) continue;
     auto node = *it;
     auto node_device = node->requested_device();
+    LOG(INFO) << "requested: " << node_device;
+    DeviceNameUtils::ParsedName parsed_name;
+    if (DeviceNameUtils::ParseFullName(node_device, &parsed_name) && parsed_name.type == "CPU") {
+      LOG(INFO) << "type: " << parsed_name.type;
+      LOG(INFO) << "clear!";
+      node_device.clear();
+    }
     if (!node_device.empty()) {
       segment_devices.insert(node_device);
     } else {
       if (node->has_assigned_device_name()) {
+        LOG(INFO) << "assigned: " << node->assigned_device_name();
         segment_devices.insert(node->assigned_device_name());
       } else {
         VLOG(2) << "Node " << node->name()
@@ -897,6 +905,8 @@ std::pair<int, tensorflow::Allocator*> GetDeviceAndAllocator(
       StrAppend(&msg, ". Will get the allocator from first one.");
       LOG(WARNING) << msg;
     }
+    LOG(INFO) << "here12";
+    LOG(INFO) << devices[0]->name();
     tensorflow::AllocatorAttributes alloc_attr;
     cuda_device_id = devices[0]->tensorflow_gpu_device_info()->gpu_id;
     dev_allocator = devices[0]->GetAllocator(alloc_attr);
@@ -1008,8 +1018,11 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
   VLOG(1) << "Current cuda device is " << old_cuda_device;
   std::vector<Node*> engine_nodes;
   engine_nodes.resize(engine_segments.size());
+  LOG(INFO) << "here " << engine_segments.size();
   for (int i = 0; i < engine_segments.size(); ++i) {
     auto& engine = engine_segments.at(i);
+
+    LOG(INFO) << "here4 ";
     // Partition the workspace size by the average of node ratio and segment
     // graphdef size
     engine.max_workspace_size_bytes =
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 179d631a3e..9d517b3be2 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -694,6 +694,14 @@ void ReorderCKtoKC(const TRT_ShapedWeights& iweights,
           ostrides);
       break;
     }
+    case tensorflow::DataType::DT_INT32: {
+      Reorder2(
+          {k, c}, static_cast<int const*>(iweights.GetValues()),
+          istrides,
+          static_cast<int*>(const_cast<void*>(oweights->GetValues())),
+          ostrides);
+      break;
+    }
     default:
       LOG(FATAL) << "Unsupported type in reorder expected fp32 or fp16 but got "
                  << DataTypeString(iweights.type_);
@@ -1725,6 +1733,12 @@ Status BinaryTensorOpTensor(OpConverterParams* params,
         "Unsupported binary op broadcast scheme for op ", node_def.name(), ": ",
         status.error_message());
   }
+  TFAttrs attrs(node_def);
+  nvinfer1::DataType dtype = attrs.get<nvinfer1::DataType>("T");
+  if (dtype == nvinfer1::DataType::kINT32) {
+    return errors::Unimplemented(
+        "Binary op ", node_def.op(), " does not support INT32, at ", node_def.name());
+  }
   if (params->validation_only) return Status::OK();
 
   const nvinfer1::ITensor* tensor_l = nullptr;
@@ -1741,8 +1755,6 @@ Status BinaryTensorOpTensor(OpConverterParams* params,
   }
 
   // Check type consistency.
-  TFAttrs attrs(node_def);
-  nvinfer1::DataType dtype = attrs.get<nvinfer1::DataType>("T");
   TFTRT_CHECK_EQ_TYPE(tensor_l->getType(), dtype)
       << DebugString(tensor_l->getType()) << " vs " << DebugString(dtype);
   TFTRT_CHECK_EQ_TYPE(tensor_r->getType(), dtype)
@@ -3450,12 +3462,12 @@ tensorflow::Status ConvertMatMul(OpConverterParams* params) {
 
   TFAttrs attrs(node_def);
   // TODO(jie): INT32 should be converted?
-  tensorflow::DataType tf_dtype = attrs.get<tensorflow::DataType>("T");
-  if (tf_dtype != DataType::DT_FLOAT && tf_dtype != DataType::DT_HALF) {
-    return errors::Unimplemented("Data type is not supported, for node ",
-                                 node_def.name(), " got ",
-                                 DataTypeString(tf_dtype));
-  }
+  // tensorflow::DataType tf_dtype = attrs.get<tensorflow::DataType>("T");
+  // if (tf_dtype != DataType::DT_FLOAT && tf_dtype != DataType::DT_HALF) {
+  //   return errors::Unimplemented("Data type is not supported, for node ",
+  //                                node_def.name(), " got ",
+  //                                DataTypeString(tf_dtype));
+  // }
   bool transpose_a = attrs.get<bool>("transpose_a");
   bool transpose_b = attrs.get<bool>("transpose_b");
 
@@ -3844,7 +3856,7 @@ tensorflow::Status ConvertSegmentToGraphDef(
       marker_nodes.insert(node_name);
       auto seg_node = segment_def->add_node();
       tensorflow::NodeDefBuilder builder(node_name, "Identity");
-      auto status = builder.Input(connection.inside_node_name, 0, dtype)
+      auto status = builder.Input(connection.inside_node_name, connection.inside_port, dtype)
                         .Finalize(seg_node);
       VLOG(1) << "Constructing output " << node_name << " for the edge "
               << connection.inside_node_name << ":" << connection.inside_port
diff --git a/tensorflow/contrib/tensorrt/segment/segment.cc b/tensorflow/contrib/tensorrt/segment/segment.cc
index 084a96e0fa..171ae41575 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.cc
+++ b/tensorflow/contrib/tensorrt/segment/segment.cc
@@ -441,7 +441,7 @@ tensorflow::Status SegmentGraph(
     } else {
       const Status status = candidate_fn(node->tf_node());
       if (!status.ok()) {
-        VLOG(1) << "Not a TF-TRT candidate, "
+        LOG(INFO) << "Not a TF-TRT candidate, "
                 << "(Op type: " << node->tf_node()->type_string() << "), "
                 << "(Op name: " << node->name() << "), "
                 << "(Reason: " << status << ")";
diff --git a/tensorflow/contrib/tensorrt/test/int_test.py b/tensorflow/contrib/tensorrt/test/int_test.py
index a4d625a213..b5d69a799f 100644
--- a/tensorflow/contrib/tensorrt/test/int_test.py
+++ b/tensorflow/contrib/tensorrt/test/int_test.py
@@ -43,39 +43,40 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
     g = ops.Graph()
     with g.as_default():
       # float part
-      input1 = array_ops.placeholder(
-          dtype=dtypes.float32, shape=input_dims, name='input_float32')
-      b1 = self._ConstOp((4, 10), dtypes.float32)
-      x1 = math_ops.matmul(input1, b)
-      b1 = self._ConstOp((1, 10), dtypes.float32)
-      x1 = x1 + b1
+      # input1 = array_ops.placeholder(
+      #     dtype=dtypes.float32, shape=input_dims, name='input_float32')
+      # b1 = self._ConstOp((4, 10), dtypes.float32)
+      # x1 = math_ops.matmul(input1, b1)
+      # b1 = self._ConstOp((1, 10), dtypes.float32)
+      # x1 = x1 + b1
 
       # int part
       input2 = array_ops.placeholder(
           dtype=dtypes.int32, shape=input_dims, name='input_int32')
+      #with g.device("/GPU:0"):
       b2 = self._ConstOp((4, 10), dtypes.int32)
-      x2 = math_ops.matmul(input2, b)
-      b2 = self._ConstOp((1, 10), dtypes.int32)
-      x2 = x2 + b2
+      x2 = math_ops.matmul(input2, b2)
+      b2 = self._ConstOp((10,), dtypes.int32)
+      x2 = nn.bias_add(x2, b2)
 
       # combine
       #y = x1 * x2
 
-      out1 = array_ops.identity(x1, name='output_float32')
+      #out1 = array_ops.identity(x1, name='output_float32')
       out2 = array_ops.identity(x2, name='output_int32')
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
-        input_names=['input_float32', 'input_int32'],
-        input_dims=[[100, 4], [100, 4]],
-        output_names=['output_int32', 'output_float32'],
-        expected_output_dims=[(100, 10), (100, 10)])
+        input_names=['input_int32'],
+        input_dims=[[100, 4]],
+        output_names=['output_int32'],
+        expected_output_dims=[(100, 10)])
 
   def GetConversionParams(self, run_params):
     """Return a ConversionParams for test."""
     conversion_params = super(BiasaddMatMulTest,
                               self).GetConversionParams(run_params)
     return conversion_params._replace(
-        max_batch_size=4,
+        max_batch_size=100,
         maximum_cached_engines=1,
         # Disable layout optimizer, since it will convert BiasAdd with NHWC
         # format to NCHW format under four dimentional input.
-- 
GitLab


From d72dae711ec00e9b9b3e60b0c70fda616a21bbc1 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 21 Dec 2018 15:28:20 -0800
Subject: [PATCH 0451/2345] Exclude CPU devices via segmenter. Prevent BiasAdd
 in int32

---
 .../contrib/tensorrt/convert/convert_graph.cc | 13 +++--------
 .../contrib/tensorrt/convert/convert_nodes.cc | 22 +++++++++++--------
 .../contrib/tensorrt/segment/segment.cc       | 21 +++++++++++++++++-
 tensorflow/contrib/tensorrt/test/int_test.py  |  5 ++---
 4 files changed, 38 insertions(+), 23 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index ffe1840c68..646451d802 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -355,19 +355,14 @@ tensorflow::Status GetEngineInfo(
     const auto& node_name = (*it)->name();
     if (segment_nodes.count(node_name) == 0) continue;
     auto node = *it;
+    // TODO: check for CPU device here
+    // If device is CPU, we should've caught that in the segmenter. Fall back here.
+
     auto node_device = node->requested_device();
-    LOG(INFO) << "requested: " << node_device;
-    DeviceNameUtils::ParsedName parsed_name;
-    if (DeviceNameUtils::ParseFullName(node_device, &parsed_name) && parsed_name.type == "CPU") {
-      LOG(INFO) << "type: " << parsed_name.type;
-      LOG(INFO) << "clear!";
-      node_device.clear();
-    }
     if (!node_device.empty()) {
       segment_devices.insert(node_device);
     } else {
       if (node->has_assigned_device_name()) {
-        LOG(INFO) << "assigned: " << node->assigned_device_name();
         segment_devices.insert(node->assigned_device_name());
       } else {
         VLOG(2) << "Node " << node->name()
@@ -905,8 +900,6 @@ std::pair<int, tensorflow::Allocator*> GetDeviceAndAllocator(
       StrAppend(&msg, ". Will get the allocator from first one.");
       LOG(WARNING) << msg;
     }
-    LOG(INFO) << "here12";
-    LOG(INFO) << devices[0]->name();
     tensorflow::AllocatorAttributes alloc_attr;
     cuda_device_id = devices[0]->tensorflow_gpu_device_info()->gpu_id;
     dev_allocator = devices[0]->GetAllocator(alloc_attr);
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 9d517b3be2..0d98da08b0 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -2609,12 +2609,18 @@ tensorflow::Status ConvertBiasAdd(OpConverterParams* params) {
     return errors::InvalidArgument("Input expects tensor and weights, at ",
                                    node_def.name());
   }
+  TFAttrs attrs(node_def);
+  tensorflow::DataType tf_dtype = attrs.get<tensorflow::DataType>("T");
+  if (tf_dtype != DataType::DT_FLOAT && tf_dtype != DataType::DT_HALF) {
+    return errors::Unimplemented("Data type is not supported, for node ",
+                                 node_def.name(), " got ",
+                                 DataTypeString(tf_dtype));
+  }
   if (params->validation_only) return Status::OK();
 
   nvinfer1::ITensor* tensor =
       const_cast<nvinfer1::ITensor*>(inputs.at(0).tensor());
   const nvinfer1::Dims original_dims = tensor->getDimensions();
-  TFAttrs attrs(node_def);
   const string data_format = attrs.get<string>("data_format");
   const int channel_index =
       (data_format == "NHWC" ? original_dims.nbDims - 1 : 0);
@@ -3461,13 +3467,12 @@ tensorflow::Status ConvertMatMul(OpConverterParams* params) {
   }
 
   TFAttrs attrs(node_def);
-  // TODO(jie): INT32 should be converted?
-  // tensorflow::DataType tf_dtype = attrs.get<tensorflow::DataType>("T");
-  // if (tf_dtype != DataType::DT_FLOAT && tf_dtype != DataType::DT_HALF) {
-  //   return errors::Unimplemented("Data type is not supported, for node ",
-  //                                node_def.name(), " got ",
-  //                                DataTypeString(tf_dtype));
-  // }
+  tensorflow::DataType tf_dtype = attrs.get<tensorflow::DataType>("T");
+  if (tf_dtype != DataType::DT_FLOAT && tf_dtype != DataType::DT_HALF) {
+    return errors::Unimplemented("Data type is not supported, for node ",
+                                 node_def.name(), " got ",
+                                 DataTypeString(tf_dtype));
+  }
   bool transpose_a = attrs.get<bool>("transpose_a");
   bool transpose_b = attrs.get<bool>("transpose_b");
 
@@ -3487,7 +3492,6 @@ tensorflow::Status ConvertBatchMatMul(OpConverterParams* params) {
   const auto& node_def = params->node_def;
   TFAttrs attrs(node_def);
 
-  // TODO(jie): INT32 should be converted?
   tensorflow::DataType tf_dtype = attrs.get<tensorflow::DataType>("T");
   if (tf_dtype != tensorflow::DataType::DT_FLOAT &&
       tf_dtype != tensorflow::DataType::DT_HALF) {
diff --git a/tensorflow/contrib/tensorrt/segment/segment.cc b/tensorflow/contrib/tensorrt/segment/segment.cc
index 171ae41575..9f72f1490c 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.cc
+++ b/tensorflow/contrib/tensorrt/segment/segment.cc
@@ -28,6 +28,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/types.h"
+#include "tensorflow/core/util/device_name_utils.h"
 
 namespace tensorflow {
 namespace tensorrt {
@@ -419,6 +420,11 @@ tensorflow::Status SegmentGraph(
   //    segment but are not eligible, using input/output_candidate_fn to
   //    determine the eligibilities;
   // 3. convert the segment into expected return format and return the result.
+  auto get_device_type = [](const string& device) {
+    DeviceNameUtils::ParsedName parsed_name;
+    DeviceNameUtils::ParseFullName(device, &parsed_name);
+    return parsed_name.type;
+  };
 
   // --------------------------------- Step 1 ---------------------------------
   auto graph = std::unique_ptr<SimpleGraph>(new SimpleGraph(tf_graph));
@@ -430,6 +436,11 @@ tensorflow::Status SegmentGraph(
   std::vector<UnionFind<SimpleNode*>> node_segments;
   for (int i = 0; i < graph->num_node_ids(); ++i) {
     SimpleNode* node = graph->FindNodeId(i);
+    const string requested_device =
+        get_device_type(node->tf_node()->requested_device());
+    const string assigned_device =
+        get_device_type(node->tf_node()->assigned_device_name());
+
     if (options.exclude_node_list.count(node->name()) != 0) {
       VLOG(1) << "Not a TF-TRT candidate, "
               << "(Op type: " << node->tf_node()->type_string() << "), "
@@ -438,10 +449,18 @@ tensorflow::Status SegmentGraph(
       unsupported_ops.emplace(node->tf_node()->type_string());
       num_unsupported_ops++;
       node = nullptr;
+    } else if (requested_device == "CPU" || assigned_device == "CPU") {
+      VLOG(1) << "Not a TF-TRT candidate, "
+              << "(Op type: " << node->tf_node()->type_string() << "), "
+              << "(Op name: " << node->name() << "), "
+              << "(Reason: node is assigned to CPU)";
+      unsupported_ops.emplace(node->tf_node()->type_string());
+      num_unsupported_ops++;
+      node = nullptr;
     } else {
       const Status status = candidate_fn(node->tf_node());
       if (!status.ok()) {
-        LOG(INFO) << "Not a TF-TRT candidate, "
+        VLOG(1) << "Not a TF-TRT candidate, "
                 << "(Op type: " << node->tf_node()->type_string() << "), "
                 << "(Op name: " << node->name() << "), "
                 << "(Reason: " << status << ")";
diff --git a/tensorflow/contrib/tensorrt/test/int_test.py b/tensorflow/contrib/tensorrt/test/int_test.py
index b5d69a799f..bb90c34168 100644
--- a/tensorflow/contrib/tensorrt/test/int_test.py
+++ b/tensorflow/contrib/tensorrt/test/int_test.py
@@ -53,7 +53,6 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
       # int part
       input2 = array_ops.placeholder(
           dtype=dtypes.int32, shape=input_dims, name='input_int32')
-      #with g.device("/GPU:0"):
       b2 = self._ConstOp((4, 10), dtypes.int32)
       x2 = math_ops.matmul(input2, b2)
       b2 = self._ConstOp((10,), dtypes.int32)
@@ -84,13 +83,13 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
-    return ["TRTEngineOp_0"]
+    return []
 
   def ShouldRunTest(self, run_params):
     """Whether to run the test."""
     # TODO(aaroey): Trt 4.0 forbids conversion for tensors with rank <3 in int8
     # mode, which is a bug. Re-enable this when trt library is fixed.
-    return not trt_test.IsQuantizationMode(run_params.precision_mode)
+    return not trt_test.IsQuantizationMode(run_params.precision_mode) #and not run_params.dynamic_engine
 
 
 if __name__ == "__main__":
-- 
GitLab


From 12a755183b32681281960050798bae02de4a58cd Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 21 Dec 2018 15:39:45 -0800
Subject: [PATCH 0452/2345] Clean up code

---
 .../contrib/tensorrt/convert/convert_graph.cc |  3 --
 .../contrib/tensorrt/convert/convert_nodes.cc |  8 ----
 tensorflow/contrib/tensorrt/test/int_test.py  | 43 +++++++------------
 3 files changed, 15 insertions(+), 39 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index 646451d802..fd31c356c1 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -1011,11 +1011,8 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
   VLOG(1) << "Current cuda device is " << old_cuda_device;
   std::vector<Node*> engine_nodes;
   engine_nodes.resize(engine_segments.size());
-  LOG(INFO) << "here " << engine_segments.size();
   for (int i = 0; i < engine_segments.size(); ++i) {
     auto& engine = engine_segments.at(i);
-
-    LOG(INFO) << "here4 ";
     // Partition the workspace size by the average of node ratio and segment
     // graphdef size
     engine.max_workspace_size_bytes =
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 0d98da08b0..39f9e220e4 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -694,14 +694,6 @@ void ReorderCKtoKC(const TRT_ShapedWeights& iweights,
           ostrides);
       break;
     }
-    case tensorflow::DataType::DT_INT32: {
-      Reorder2(
-          {k, c}, static_cast<int const*>(iweights.GetValues()),
-          istrides,
-          static_cast<int*>(const_cast<void*>(oweights->GetValues())),
-          ostrides);
-      break;
-    }
     default:
       LOG(FATAL) << "Unsupported type in reorder expected fp32 or fp16 but got "
                  << DataTypeString(iweights.type_);
diff --git a/tensorflow/contrib/tensorrt/test/int_test.py b/tensorflow/contrib/tensorrt/test/int_test.py
index bb90c34168..b9636fa098 100644
--- a/tensorflow/contrib/tensorrt/test/int_test.py
+++ b/tensorflow/contrib/tensorrt/test/int_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Model script to test TF-TensorRT integration."""
+"""Test conversion of graphs involving INT32 tensors and operations."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -37,37 +37,24 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
     return constant_op.constant(np.random.randn(*shape), dtype=dtype)
 
   def GetParams(self):
-    """Testing conversion of BiasAdd MatMul in TF-TRT conversion."""
-    # Note that tf.nn.bias_add supports up to 5 dimensions.
+    """Test exclusion of ops which are not supported in INT32 mode by TF-TRT"""
+    input_name = 'input'
+    output_name = 'output'
     input_dims = [100, 4]
     g = ops.Graph()
     with g.as_default():
-      # float part
-      # input1 = array_ops.placeholder(
-      #     dtype=dtypes.float32, shape=input_dims, name='input_float32')
-      # b1 = self._ConstOp((4, 10), dtypes.float32)
-      # x1 = math_ops.matmul(input1, b1)
-      # b1 = self._ConstOp((1, 10), dtypes.float32)
-      # x1 = x1 + b1
-
-      # int part
-      input2 = array_ops.placeholder(
-          dtype=dtypes.int32, shape=input_dims, name='input_int32')
-      b2 = self._ConstOp((4, 10), dtypes.int32)
-      x2 = math_ops.matmul(input2, b2)
-      b2 = self._ConstOp((10,), dtypes.int32)
-      x2 = nn.bias_add(x2, b2)
-
-      # combine
-      #y = x1 * x2
-
-      #out1 = array_ops.identity(x1, name='output_float32')
-      out2 = array_ops.identity(x2, name='output_int32')
+      x = array_ops.placeholder(
+          dtype=dtypes.int32, shape=input_dims, name=input_name)
+      b = self._ConstOp((4, 10), dtypes.int32)
+      x = math_ops.matmul(x, b)
+      b = self._ConstOp((10,), dtypes.int32)
+      x = nn.bias_add(x, b)
+      x = array_ops.identity(x, name=output_name)
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
-        input_names=['input_int32'],
-        input_dims=[[100, 4]],
-        output_names=['output_int32'],
+        input_names=[input_name],
+        input_dims=[input_dims],
+        output_names=[output_name],
         expected_output_dims=[(100, 10)])
 
   def GetConversionParams(self, run_params):
@@ -89,7 +76,7 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
     """Whether to run the test."""
     # TODO(aaroey): Trt 4.0 forbids conversion for tensors with rank <3 in int8
     # mode, which is a bug. Re-enable this when trt library is fixed.
-    return not trt_test.IsQuantizationMode(run_params.precision_mode) #and not run_params.dynamic_engine
+    return not trt_test.IsQuantizationMode(run_params.precision_mode)
 
 
 if __name__ == "__main__":
-- 
GitLab


From f84227cac2abe03818cecd80ac02b86b8090e376 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 21 Dec 2018 15:41:01 -0800
Subject: [PATCH 0453/2345] Rename test

---
 tensorflow/contrib/tensorrt/BUILD                            | 2 +-
 .../contrib/tensorrt/test/{int_test.py => int32_test.py}     | 5 ++---
 2 files changed, 3 insertions(+), 4 deletions(-)
 rename tensorflow/contrib/tensorrt/test/{int_test.py => int32_test.py} (96%)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index e58a259a18..418e1564bf 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -493,7 +493,7 @@ cuda_py_tests(
         "test/const_broadcast_test.py",
         "test/conv2d_test.py",
         "test/identity_output_test.py",
-        "test/int_test.py",
+        "test/int32_test.py",
         "test/manual_test.py",
         "test/memory_alignment_test.py",
         "test/multi_connection_neighbor_engine_test.py",
diff --git a/tensorflow/contrib/tensorrt/test/int_test.py b/tensorflow/contrib/tensorrt/test/int32_test.py
similarity index 96%
rename from tensorflow/contrib/tensorrt/test/int_test.py
rename to tensorflow/contrib/tensorrt/test/int32_test.py
index b9636fa098..8f272efc91 100644
--- a/tensorflow/contrib/tensorrt/test/int_test.py
+++ b/tensorflow/contrib/tensorrt/test/int32_test.py
@@ -31,7 +31,7 @@ from tensorflow.python.ops import nn
 from tensorflow.python.platform import test
 
 
-class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
+class ExcludeUnsupportedInt32Test(trt_test.TfTrtIntegrationTestBase):
 
   def _ConstOp(self, shape, dtype):
     return constant_op.constant(np.random.randn(*shape), dtype=dtype)
@@ -59,7 +59,7 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
 
   def GetConversionParams(self, run_params):
     """Return a ConversionParams for test."""
-    conversion_params = super(BiasaddMatMulTest,
+    conversion_params = super(ExcludeUnsupportedInt32Test,
                               self).GetConversionParams(run_params)
     return conversion_params._replace(
         max_batch_size=100,
@@ -78,6 +78,5 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
     # mode, which is a bug. Re-enable this when trt library is fixed.
     return not trt_test.IsQuantizationMode(run_params.precision_mode)
 
-
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 1a73fdfa83bd50695a7d374d14a5cb3835d94d9e Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 21 Dec 2018 16:00:20 -0800
Subject: [PATCH 0454/2345] Add extra check incase segmenter does not exclude
 CPU in order to prevent segfault

---
 .../contrib/tensorrt/convert/convert_graph.cc      | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index fd31c356c1..cfde0049fc 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -355,12 +355,18 @@ tensorflow::Status GetEngineInfo(
     const auto& node_name = (*it)->name();
     if (segment_nodes.count(node_name) == 0) continue;
     auto node = *it;
-    // TODO: check for CPU device here
-    // If device is CPU, we should've caught that in the segmenter. Fall back here.
-
     auto node_device = node->requested_device();
     if (!node_device.empty()) {
-      segment_devices.insert(node_device);
+      // If device is CPU, we should've caught that in the segmenter.
+      DeviceNameUtils::ParsedName parsed_name;
+      DeviceNameUtils::ParseFullName(node_device, &parsed_name);
+      if (parsed_name.type == "CPU") {
+        LOG(WARNING) << "Node " << node->name() << " was assigned to the CPU "
+                     << "but did not get excluded by the segmenter. "
+                     << "Attempting to place on GPU.";
+      } else {
+        segment_devices.insert(node_device);
+      }
     } else {
       if (node->has_assigned_device_name()) {
         segment_devices.insert(node->assigned_device_name());
-- 
GitLab


From 399ae14f9829957e0b6eda6cc48f2a99133255e5 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 21 Dec 2018 16:01:39 -0800
Subject: [PATCH 0455/2345] Improve comment

---
 tensorflow/contrib/tensorrt/convert/convert_graph.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index cfde0049fc..d8356b31a0 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -357,7 +357,9 @@ tensorflow::Status GetEngineInfo(
     auto node = *it;
     auto node_device = node->requested_device();
     if (!node_device.empty()) {
-      // If device is CPU, we should've caught that in the segmenter.
+      // If device is CPU, we should've caught that in the segmenter. Don't add
+      // CPU to segment_device because that would cause a segfault in
+      // GetDeviceAndAllocator.
       DeviceNameUtils::ParsedName parsed_name;
       DeviceNameUtils::ParseFullName(node_device, &parsed_name);
       if (parsed_name.type == "CPU") {
-- 
GitLab


From 2434065e691a553a120aef6c16c76f29b55267a7 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 4 Jan 2019 10:40:57 -0800
Subject: [PATCH 0456/2345] Add TopK test. Specify type for topk indices
 output. Fix formatting

---
 tensorflow/contrib/tensorrt/BUILD             |  1 +
 .../contrib/tensorrt/convert/convert_nodes.cc |  8 ++-
 tensorflow/contrib/tensorrt/test/topk_test.py | 59 +++++++++++++++++++
 3 files changed, 66 insertions(+), 2 deletions(-)
 create mode 100644 tensorflow/contrib/tensorrt/test/topk_test.py

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 418e1564bf..b0f55fed44 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -501,6 +501,7 @@ cuda_py_tests(
         "test/quantization_test.py",
         "test/rank_two_test.py",
         "test/reshape_transpose_test.py",
+        "test/topk_test.py",
         "test/vgg_block_nchw_test.py",
         "test/vgg_block_test.py",
     ],
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 39f9e220e4..ee51c287ab 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -1728,8 +1728,9 @@ Status BinaryTensorOpTensor(OpConverterParams* params,
   TFAttrs attrs(node_def);
   nvinfer1::DataType dtype = attrs.get<nvinfer1::DataType>("T");
   if (dtype == nvinfer1::DataType::kINT32) {
-    return errors::Unimplemented(
-        "Binary op ", node_def.op(), " does not support INT32, at ", node_def.name());
+    return errors::Unimplemented("Binary op ", node_def.op(),
+                                 " does not support INT32, at ",
+                                 node_def.name());
   }
   if (params->validation_only) return Status::OK();
 
@@ -3605,6 +3606,9 @@ tensorflow::Status ConvertTopK(OpConverterParams* params) {
 
   nvinfer1::ITensor* output_value_tensor = layer->getOutput(0);
   nvinfer1::ITensor* output_indices_tensor = layer->getOutput(1);
+  // Tensor type for network output is not inferred. Indices should be INT32
+  // (default is float).
+  output_indices_tensor->setType(nvinfer1::DataType::kINT32);
   params->outputs->push_back(TRT_TensorOrWeights(output_value_tensor));
   params->outputs->push_back(TRT_TensorOrWeights(output_indices_tensor));
   return tensorflow::Status::OK();
diff --git a/tensorflow/contrib/tensorrt/test/topk_test.py b/tensorflow/contrib/tensorrt/test/topk_test.py
new file mode 100644
index 0000000000..cb539bbb93
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/test/topk_test.py
@@ -0,0 +1,59 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Model script to test TF-TensorRT integration."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import nn_ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.platform import test
+
+
+class TopKTest(trt_test.TfTrtIntegrationTestBase):
+
+  def GetParams(self):
+    """Testing Top-K in TF-TRT conversion."""
+    dtype = dtypes.float32
+    input_name = "input"
+    input_dims = [100, 100]
+    k = 5
+    g = ops.Graph()
+    with g.as_default():
+      x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name)
+      values, indices = nn_ops.top_k(x, k)
+      values = array_ops.identity(values, name='output_values')
+      indices = array_ops.identity(indices, name='output_indices')
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=[input_name],
+        input_dims=[input_dims],
+        output_names=['output_values', 'output_indices'],
+        expected_output_dims=[(100, k), (100, k)])
+
+  def ExpectedEnginesToBuild(self, run_params):
+    """Return the expected engines to build."""
+    return ["TRTEngineOp_0"]
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From 4d0a420c4b4d1fbe3e666bd377de2a40401177d2 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 4 Jan 2019 10:48:57 -0800
Subject: [PATCH 0457/2345] Fix clang-format and pylint

---
 tensorflow/contrib/tensorrt/convert/convert_nodes.cc | 6 ++++--
 tensorflow/contrib/tensorrt/test/topk_test.py        | 3 ---
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index ee51c287ab..9c0d79b810 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -3856,8 +3856,10 @@ tensorflow::Status ConvertSegmentToGraphDef(
       marker_nodes.insert(node_name);
       auto seg_node = segment_def->add_node();
       tensorflow::NodeDefBuilder builder(node_name, "Identity");
-      auto status = builder.Input(connection.inside_node_name, connection.inside_port, dtype)
-                        .Finalize(seg_node);
+      auto status =
+          builder
+              .Input(connection.inside_node_name, connection.inside_port, dtype)
+              .Finalize(seg_node);
       VLOG(1) << "Constructing output " << node_name << " for the edge "
               << connection.inside_node_name << ":" << connection.inside_port
               << " -> " << connection.outside_node_name << ":"
diff --git a/tensorflow/contrib/tensorrt/test/topk_test.py b/tensorflow/contrib/tensorrt/test/topk_test.py
index cb539bbb93..2edf7cb587 100644
--- a/tensorflow/contrib/tensorrt/test/topk_test.py
+++ b/tensorflow/contrib/tensorrt/test/topk_test.py
@@ -18,10 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import numpy as np
-
 from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
-from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import nn_ops
-- 
GitLab


From 0107c9155260f9aeb32d7660f7a9e8a8a5126850 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 4 Jan 2019 14:18:37 -0800
Subject: [PATCH 0458/2345] Remove segmenter rule against CPU ops.

---
 tensorflow/contrib/tensorrt/convert/convert_graph.cc |  2 ++
 tensorflow/contrib/tensorrt/segment/segment.cc       | 12 ------------
 tensorflow/contrib/tensorrt/test/int32_test.py       |  7 ++++---
 3 files changed, 6 insertions(+), 15 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index d8356b31a0..1ed6bbe5a3 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -371,6 +371,8 @@ tensorflow::Status GetEngineInfo(
       }
     } else {
       if (node->has_assigned_device_name()) {
+        // It appears that nodes will not have assigned devices at this point in
+        // execution.
         segment_devices.insert(node->assigned_device_name());
       } else {
         VLOG(2) << "Node " << node->name()
diff --git a/tensorflow/contrib/tensorrt/segment/segment.cc b/tensorflow/contrib/tensorrt/segment/segment.cc
index 9f72f1490c..6f2a7846d7 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.cc
+++ b/tensorflow/contrib/tensorrt/segment/segment.cc
@@ -436,10 +436,6 @@ tensorflow::Status SegmentGraph(
   std::vector<UnionFind<SimpleNode*>> node_segments;
   for (int i = 0; i < graph->num_node_ids(); ++i) {
     SimpleNode* node = graph->FindNodeId(i);
-    const string requested_device =
-        get_device_type(node->tf_node()->requested_device());
-    const string assigned_device =
-        get_device_type(node->tf_node()->assigned_device_name());
 
     if (options.exclude_node_list.count(node->name()) != 0) {
       VLOG(1) << "Not a TF-TRT candidate, "
@@ -449,14 +445,6 @@ tensorflow::Status SegmentGraph(
       unsupported_ops.emplace(node->tf_node()->type_string());
       num_unsupported_ops++;
       node = nullptr;
-    } else if (requested_device == "CPU" || assigned_device == "CPU") {
-      VLOG(1) << "Not a TF-TRT candidate, "
-              << "(Op type: " << node->tf_node()->type_string() << "), "
-              << "(Op name: " << node->name() << "), "
-              << "(Reason: node is assigned to CPU)";
-      unsupported_ops.emplace(node->tf_node()->type_string());
-      num_unsupported_ops++;
-      node = nullptr;
     } else {
       const Status status = candidate_fn(node->tf_node());
       if (!status.ok()) {
diff --git a/tensorflow/contrib/tensorrt/test/int32_test.py b/tensorflow/contrib/tensorrt/test/int32_test.py
index 8f272efc91..50af816d8e 100644
--- a/tensorflow/contrib/tensorrt/test/int32_test.py
+++ b/tensorflow/contrib/tensorrt/test/int32_test.py
@@ -41,13 +41,14 @@ class ExcludeUnsupportedInt32Test(trt_test.TfTrtIntegrationTestBase):
     input_name = 'input'
     output_name = 'output'
     input_dims = [100, 4]
+    dtype = dtypes.int32
     g = ops.Graph()
     with g.as_default():
       x = array_ops.placeholder(
-          dtype=dtypes.int32, shape=input_dims, name=input_name)
-      b = self._ConstOp((4, 10), dtypes.int32)
+          dtype=dtype, shape=input_dims, name=input_name)
+      b = self._ConstOp((4, 10), dtype)
       x = math_ops.matmul(x, b)
-      b = self._ConstOp((10,), dtypes.int32)
+      b = self._ConstOp((10,), dtype)
       x = nn.bias_add(x, b)
       x = array_ops.identity(x, name=output_name)
     return trt_test.TfTrtIntegrationTestParams(
-- 
GitLab


From a5037695df5cc9eee60ea31fc1136c92786b57b1 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 4 Jan 2019 14:25:17 -0800
Subject: [PATCH 0459/2345] Update comment on fix to prevent segfault

---
 tensorflow/contrib/tensorrt/convert/convert_graph.cc | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index 1ed6bbe5a3..df4bc4777c 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -357,15 +357,15 @@ tensorflow::Status GetEngineInfo(
     auto node = *it;
     auto node_device = node->requested_device();
     if (!node_device.empty()) {
-      // If device is CPU, we should've caught that in the segmenter. Don't add
-      // CPU to segment_device because that would cause a segfault in
-      // GetDeviceAndAllocator.
+      // If device is CPU, treat as if no device was assigned. Don't add CPU to
+      // segment_device because that would cause a segfault in
+      // GetDeviceAndAllocator. This is because GetDeviceAndAllocator assumes
+      // any already set device is a GPU.
       DeviceNameUtils::ParsedName parsed_name;
       DeviceNameUtils::ParseFullName(node_device, &parsed_name);
       if (parsed_name.type == "CPU") {
-        LOG(WARNING) << "Node " << node->name() << " was assigned to the CPU "
-                     << "but did not get excluded by the segmenter. "
-                     << "Attempting to place on GPU.";
+        VLOG(1) << "Node " << node->name() << " was assigned to the CPU. "
+                << "Attempting to place on GPU.";
       } else {
         segment_devices.insert(node_device);
       }
-- 
GitLab


From 55d34860cd43967a7a19ff9c6c9605168aa191cf Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Wed, 9 Jan 2019 22:50:01 +0000
Subject: [PATCH 0460/2345] Address additional review comments

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/ops/ragged/ragged_getitem.py     | 2 +-
 tensorflow/python/ops/ragged/ragged_tensor_test.py | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/ops/ragged/ragged_getitem.py b/tensorflow/python/ops/ragged/ragged_getitem.py
index f5ac3e9a2a..a7124bf8e5 100644
--- a/tensorflow/python/ops/ragged/ragged_getitem.py
+++ b/tensorflow/python/ops/ragged/ragged_getitem.py
@@ -169,7 +169,7 @@ def _ragged_getitem(rt_input, key_list):
       # will simply be ignored as it will be processed later anyway.
       try:
         if row_key >= len(starts):
-          raise IndexError('row key {} out of bounds'.format(row_key))
+          raise IndexError('Row key {} out of bounds'.format(row_key))
       except TypeError:
         pass
     row = rt_input.values[starts[row_key]:limits[row_key]]
diff --git a/tensorflow/python/ops/ragged/ragged_tensor_test.py b/tensorflow/python/ops/ragged/ragged_tensor_test.py
index 0c9506c731..d92accb0a0 100644
--- a/tensorflow/python/ops/ragged/ragged_tensor_test.py
+++ b/tensorflow/python/ops/ragged/ragged_tensor_test.py
@@ -1226,9 +1226,8 @@ class RaggedTensorTest(ragged_test_util.RaggedTensorTestCase,
     r = ragged_factory_ops.constant(values)
     i = 0
     for elem in r:
-      value = values[i]
+      self.assertAllEqual(elem, values[i])
       i += 1
-      self.assertAllEqual(elem, value)
 
 if __name__ == '__main__':
   googletest.main()
-- 
GitLab


From 0fb8e5f9c0149c4cfaf8ec41b1293a947bc91895 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 14:52:30 -0800
Subject: [PATCH 0461/2345] This change makes errors more informative in case
 of functions. We now report two additional things: 1. The line number/file
 name of the culprit node inside the function. 2. The function calling stack

For eg, consider the following code:

import tensorflow as tf
tf.enable_eager_execution()

@tf.contrib.eager.defun
def fn3(x):
  return x + 2

@tf.contrib.eager.defun
def fn2(x):
  tf.assert_equal(fn3(x), 3)
  return 2

@tf.contrib.eager.defun
def fn(x):
  return fn2(x)

@tf.contrib.eager.defun
def main(argv):
  x = fn(2)
  print(x)

if __name__ == '__main__':
  tf.app.run()

In this, the error message would contain the following:
InvalidArgumentError: assertion failed: [] [Condition x == y did not hold element-wise:] [x (PartitionedCall:0) = ] [4] [y (assert_equal/y:0) = ] [3]
	 [[node assert_equal/Assert/Assert (defined at filename:line_num) ]]
	 [[StatefulPartitionedCall]]
	 [[StatefulPartitionedCall]] [Op:StatefulPartitionedCall]

Function calling stack:
main -> fn -> fn2

PiperOrigin-RevId: 228593653
---
 .../core/kernels/partitioned_function_ops.cc  |  9 ++-
 tensorflow/core/lib/core/errors.h             |  4 ++
 tensorflow/python/eager/function.py           | 61 ++++++++++++++++---
 tensorflow/python/eager/function_test.py      | 24 ++++++++
 .../python/framework/error_interpolation.py   |  8 ++-
 5 files changed, 92 insertions(+), 14 deletions(-)

diff --git a/tensorflow/core/kernels/partitioned_function_ops.cc b/tensorflow/core/kernels/partitioned_function_ops.cc
index e493dca188..cadb83d8cf 100644
--- a/tensorflow/core/kernels/partitioned_function_ops.cc
+++ b/tensorflow/core/kernels/partitioned_function_ops.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/framework/types.pb.h"
 #include "tensorflow/core/grappler/optimizers/meta_optimizer.h"
+#include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/protobuf/config.pb.h"
 #include "tensorflow/core/protobuf/rewriter_config.pb.h"
 #include "tensorflow/core/util/ptr_util.h"
@@ -213,10 +214,14 @@ class PartitionedCallOp : public AsyncOpKernel {
     run_opts.rendezvous = rendez;
 
     std::vector<Tensor>* rets = new std::vector<Tensor>;
+    const string& func_name = func_.name();
     lib->Run(run_opts, handle, inputs, rets,
-             [rets, rendez, done, ctx](const Status& status) {
+             [rets, rendez, done, ctx, func_name](const Status& status) {
                if (!status.ok()) {
-                 ctx->SetStatus(status);
+                 const string function_and_msg =
+                     strings::StrCat(errors::FormatFunctionForError(func_name),
+                                     " ", status.error_message());
+                 ctx->SetStatus(Status(status.code(), function_and_msg));
                } else {
                  for (int i = 0; i < rets->size(); ++i) {
                    ctx->set_output(i, (*rets)[i]);
diff --git a/tensorflow/core/lib/core/errors.h b/tensorflow/core/lib/core/errors.h
index d5cbe6c616..4815f7c2cc 100644
--- a/tensorflow/core/lib/core/errors.h
+++ b/tensorflow/core/lib/core/errors.h
@@ -150,6 +150,10 @@ string FormatColocationNodeForError(const T& names) {
       });
 }
 
+inline string FormatFunctionForError(const string& name) {
+  return strings::StrCat("{{function_node ", name, "}}");
+}
+
 // The CanonicalCode() for non-errors.
 using ::tensorflow::error::OK;
 
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 67c633726f..83cd140158 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -41,6 +41,8 @@ from tensorflow.python.framework import c_api_util
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import device as pydev
 from tensorflow.python.framework import dtypes as dtypes_module
+from tensorflow.python.framework import error_interpolation
+from tensorflow.python.framework import errors
 from tensorflow.python.framework import func_graph as func_graph_module
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_spec
@@ -131,6 +133,46 @@ def _parse_func_attrs(attributes):
   return attrs
 
 
+class _InterpolateFunctionError(object):
+  """Context Manager that interpolates the exception from 'top_level_func'."""
+
+  def __init__(self, top_level_func):
+    self._func = top_level_func
+
+  def __enter__(self):
+    pass
+
+  def __exit__(self, typ, exc, tb):
+    if not exc or not isinstance(exc, errors.OpError):
+      return False
+    message = compat.as_text(exc.message)
+    _, tags = error_interpolation.parse_message(message)
+    g = None
+    func_stack = []
+    # pylint: disable=protected-access
+    for t in tags:
+      if t.type == "function_node":
+        if t.name == compat.as_str(self._func.name):
+          g = self._func._graph
+        elif g:
+          next_func = g._get_function(t.name)
+          if next_func is not None and isinstance(next_func,
+                                                  _EagerDefinedFunction):
+            g = next_func._graph
+        if g:
+          func_stack.append(g.name)
+        else:
+          func_stack.append("<unknown>")
+    # pylint: enable=protected-access
+    if g:
+      message = error_interpolation.interpolate(message, g)
+      message += "\n\nFunction call stack:\n"
+      message += " -> ".join(func_stack)
+      message += "\n"
+      exc._message = message  # pylint: disable=protected-access
+    return False
+
+
 def _forward_name(n):
   """The name of a generated forward defun named n."""
   return "__forward_%s_%s" % (n, ops.uid())
@@ -279,13 +321,14 @@ class _EagerDefinedFunction(object):
             "Arguments and signature arguments do not match: %s %s " %
             (len(args), len(list(self.signature.input_arg))))
       function_call_options = ctx.get_function_call_options()
-      outputs = functional_ops.partitioned_call(
-          args=args,
-          f=self,
-          tout=self._output_types,
-          executing_eagerly=executing_eagerly,
-          config=function_call_options.config_proto_serialized,
-          executor_type=function_call_options.executor_type)
+      with _InterpolateFunctionError(self):
+        outputs = functional_ops.partitioned_call(
+            args=args,
+            f=self,
+            tout=self._output_types,
+            executing_eagerly=executing_eagerly,
+            config=function_call_options.config_proto_serialized,
+            executor_type=function_call_options.executor_type)
 
     if executing_eagerly:
       return outputs
@@ -401,8 +444,8 @@ class Function(object):
     """
     return self._call_flat(
         (t for t in nest.flatten((args, kwargs))
-         if isinstance(
-             t, (ops.Tensor, resource_variable_ops.ResourceVariable))))
+         if isinstance(t, (ops.Tensor,
+                           resource_variable_ops.ResourceVariable))))
 
   def _call_flat(self, args):
     """Executes the wrapped function.
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index 55a9cc4e92..1966b259bf 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -45,6 +45,7 @@ from tensorflow.python.framework import test_util
 from tensorflow.python.keras.engine import training as keras_training
 from tensorflow.python.layers import convolutional
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import clip_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import init_ops
@@ -2063,6 +2064,29 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     # function itself is not involved in a reference cycle.
     self.assertIs(None, weak_fn())
 
+  def testFunctionStackInErrorMessage(self):
+
+    @def_function.function()
+    def fn3(x):
+      return x + 2
+
+    @def_function.function()
+    def fn2(x):
+      check_ops.assert_equal(fn3(x), 3)
+      return 2
+
+    @def_function.function()
+    def fn(x):
+      return fn2(x)
+
+    try:
+      fn(2)
+      self.assertFail()
+    except errors.InvalidArgumentError as e:
+      self.assertIn('fn -> fn2', e.message)
+      self.assertIn('node assert_equal/Assert/Assert (defined at', e.message)
+      self.assertNotIn('fn3', e.message)
+
 
 if __name__ == '__main__':
   ops.enable_eager_execution(
diff --git a/tensorflow/python/framework/error_interpolation.py b/tensorflow/python/framework/error_interpolation.py
index 5e1bed8e0e..af83b70a46 100644
--- a/tensorflow/python/framework/error_interpolation.py
+++ b/tensorflow/python/framework/error_interpolation.py
@@ -31,7 +31,7 @@ import six
 
 from tensorflow.python.util import tf_stack
 
-_NAME_REGEX = r"[A-Za-z0-9.][A-Za-z0-9_.\-/]*?"
+_NAME_REGEX = r"[A-Za-z0-9_.][A-Za-z0-9_.\-/]*?"
 _TAG_REGEX = r"{{{{({name}) ({name})}}}}".format(name=_NAME_REGEX)
 _INTERPOLATION_REGEX = r"^(.*?)({tag})".format(tag=_TAG_REGEX)
 _INTERPOLATION_PATTERN = re.compile(_INTERPOLATION_REGEX, re.DOTALL)
@@ -45,7 +45,7 @@ _BAD_FILE_SUBSTRINGS = [
 ]
 
 
-def _parse_message(message):
+def parse_message(message):
   """Parses the message.
 
   Splits the message into separators and tags. Tags are named tuples
@@ -376,7 +376,7 @@ def interpolate(error_message, graph):
   Returns:
     The string with tags of the form {{type name}} interpolated.
   """
-  seps, tags = _parse_message(error_message)
+  seps, tags = parse_message(error_message)
   subs = []
   end_msg = collections.defaultdict(list)
   tagged_ops = []
@@ -404,6 +404,8 @@ def interpolate(error_message, graph):
         msg = "node %s%s placed on device %s " % (
             ops[0].name, field_dict["defined_at"], field_dict["devices"])
         end_msg["colocations"].append(field_dict["devs_and_colocs"])
+    if tag.type == "function_node":
+      msg = ""
     subs.append(msg)
 
   if "source_nodes" in end_msg:
-- 
GitLab


From 00a3be761b620761a71effb793e2440096f1eebd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 14:55:31 -0800
Subject: [PATCH 0462/2345] Add LinearOperator.inverse method.

PiperOrigin-RevId: 228594130
---
 .../linalg/linear_operator_algebra_test.py    |  48 ++++++
 .../linalg/linear_operator_block_diag_test.py |  38 +++--
 .../linalg/linear_operator_diag_test.py       |   9 +-
 .../linalg/linear_operator_identity_test.py   |  24 ++-
 .../linalg/linear_operator_kronecker_test.py  |  37 +++--
 .../linalg/linear_operator_zeros_test.py      |   3 +-
 .../ops/linalg/inverse_registrations.py       | 108 +++++++++++++
 tensorflow/python/ops/linalg/linalg.py        |   1 +
 .../python/ops/linalg/linear_operator.py      |  25 +++
 .../ops/linalg/linear_operator_algebra.py     |  71 +++++++++
 .../ops/linalg/linear_operator_test_util.py   |  25 ++-
 ...w.linalg.-linear-operator-block-diag.pbtxt |   4 +
 ...ow.linalg.-linear-operator-circulant.pbtxt |   4 +
 ...linalg.-linear-operator-circulant2-d.pbtxt |   4 +
 ...linalg.-linear-operator-circulant3-d.pbtxt |   4 +
 ....linalg.-linear-operator-composition.pbtxt |   4 +
 ...sorflow.linalg.-linear-operator-diag.pbtxt |   4 +
 ....linalg.-linear-operator-full-matrix.pbtxt |   4 +
 ...low.linalg.-linear-operator-identity.pbtxt |   4 +
 ...ow.linalg.-linear-operator-inversion.pbtxt | 142 ++++++++++++++++++
 ...ow.linalg.-linear-operator-kronecker.pbtxt |   4 +
 ...alg.-linear-operator-low-rank-update.pbtxt |   4 +
 ...lg.-linear-operator-lower-triangular.pbtxt |   4 +
 ...alg.-linear-operator-scaled-identity.pbtxt |   4 +
 ...orflow.linalg.-linear-operator-zeros.pbtxt |   4 +
 .../tensorflow.linalg.-linear-operator.pbtxt  |   4 +
 .../api/golden/v1/tensorflow.linalg.pbtxt     |   4 +
 ...w.linalg.-linear-operator-block-diag.pbtxt |   4 +
 ...ow.linalg.-linear-operator-circulant.pbtxt |   4 +
 ...linalg.-linear-operator-circulant2-d.pbtxt |   4 +
 ...linalg.-linear-operator-circulant3-d.pbtxt |   4 +
 ....linalg.-linear-operator-composition.pbtxt |   4 +
 ...sorflow.linalg.-linear-operator-diag.pbtxt |   4 +
 ....linalg.-linear-operator-full-matrix.pbtxt |   4 +
 ...low.linalg.-linear-operator-identity.pbtxt |   4 +
 ...ow.linalg.-linear-operator-inversion.pbtxt | 142 ++++++++++++++++++
 ...ow.linalg.-linear-operator-kronecker.pbtxt |   4 +
 ...alg.-linear-operator-low-rank-update.pbtxt |   4 +
 ...lg.-linear-operator-lower-triangular.pbtxt |   4 +
 ...alg.-linear-operator-scaled-identity.pbtxt |   4 +
 ...orflow.linalg.-linear-operator-zeros.pbtxt |   4 +
 .../tensorflow.linalg.-linear-operator.pbtxt  |   4 +
 .../api/golden/v2/tensorflow.linalg.pbtxt     |   4 +
 43 files changed, 761 insertions(+), 32 deletions(-)
 create mode 100644 tensorflow/python/ops/linalg/inverse_registrations.py
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-inversion.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-inversion.pbtxt

diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_algebra_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_algebra_test.py
index 8e296c026c..ec78a3ffe0 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_algebra_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_algebra_test.py
@@ -30,6 +30,8 @@ _CHOLESKY_DECOMPS = linear_operator_algebra._CHOLESKY_DECOMPS
 _MATMUL = linear_operator_algebra._MATMUL
 _registered_cholesky = linear_operator_algebra._registered_cholesky
 _registered_matmul = linear_operator_algebra._registered_matmul
+_INVERSES = linear_operator_algebra._INVERSES
+_registered_inverse = linear_operator_algebra._registered_inverse
 # pylint: enable=protected-access
 
 
@@ -129,5 +131,51 @@ class MatmulTest(test.TestCase):
       self.assertEqual(v, _registered_matmul(k[0], k[1]))
 
 
+class InverseTest(test.TestCase):
+
+  def testRegistration(self):
+
+    class CustomLinOp(linear_operator.LinearOperator):
+
+      def _matmul(self, a):
+        pass
+
+      def _shape(self):
+        return tensor_shape.TensorShape([1, 1])
+
+      def _shape_tensor(self):
+        pass
+
+    # Register Inverse to a lambda that spits out the name parameter
+    @linear_operator_algebra.RegisterInverse(CustomLinOp)
+    def _inverse(a):  # pylint: disable=unused-argument,unused-variable
+      return "OK"
+
+    with self.assertRaisesRegexp(ValueError, "singular"):
+      CustomLinOp(dtype=None, is_non_singular=False).inverse()
+
+    self.assertEqual("OK", CustomLinOp(
+        dtype=None, is_non_singular=True).inverse())
+
+  def testRegistrationFailures(self):
+
+    class CustomLinOp(linear_operator.LinearOperator):
+      pass
+
+    with self.assertRaisesRegexp(TypeError, "must be callable"):
+      linear_operator_algebra.RegisterInverse(CustomLinOp)("blah")
+
+    # First registration is OK
+    linear_operator_algebra.RegisterInverse(CustomLinOp)(lambda a: None)
+
+    # Second registration fails
+    with self.assertRaisesRegexp(ValueError, "has already been registered"):
+      linear_operator_algebra.RegisterInverse(CustomLinOp)(lambda a: None)
+
+  def testExactRegistrationsAllMatch(self):
+    for (k, v) in _INVERSES.items():
+      self.assertEqual(v, _registered_inverse(k[0]))
+
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_block_diag_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_block_diag_test.py
index f0cc5d709f..96e6e3c04c 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_block_diag_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_block_diag_test.py
@@ -155,20 +155,38 @@ class SquareLinearOperatorBlockDiagTest(
         is_self_adjoint=True,
     )
     cholesky_factor = operator.cholesky()
-    self.assertTrue(isinstance(
+    self.assertIsInstance(
         cholesky_factor,
-        block_diag.LinearOperatorBlockDiag))
+        block_diag.LinearOperatorBlockDiag)
     self.assertEqual(2, len(cholesky_factor.operators))
-    self.assertTrue(
-        isinstance(
-            cholesky_factor.operators[0],
-            lower_triangular.LinearOperatorLowerTriangular)
+    self.assertIsInstance(
+        cholesky_factor.operators[0],
+        lower_triangular.LinearOperatorLowerTriangular)
+    self.assertIsInstance(
+        cholesky_factor.operators[1],
+        lower_triangular.LinearOperatorLowerTriangular
     )
-    self.assertTrue(
-        isinstance(
-            cholesky_factor.operators[1],
-            lower_triangular.LinearOperatorLowerTriangular)
+
+  def test_block_diag_inverse_type(self):
+    matrix = [[1., 0.], [0., 1.]]
+    operator = block_diag.LinearOperatorBlockDiag(
+        [
+            linalg.LinearOperatorFullMatrix(
+                matrix,
+                is_non_singular=True,
+            ),
+            linalg.LinearOperatorFullMatrix(
+                matrix,
+                is_non_singular=True,
+            ),
+        ],
+        is_non_singular=True,
     )
+    inverse = operator.inverse()
+    self.assertIsInstance(
+        inverse,
+        block_diag.LinearOperatorBlockDiag)
+    self.assertEqual(2, len(inverse.operators))
 
   def test_is_non_singular_auto_set(self):
     # Matrix with two positive eigenvalues, 11 and 8.
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py
index dcbc0dd7c9..4d7a31be87 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py
@@ -194,9 +194,12 @@ class LinearOperatorDiagTest(
         is_positive_definite=True,
         is_self_adjoint=True,
     )
-    self.assertTrue(isinstance(
-        operator.cholesky(),
-        linalg.LinearOperatorDiag))
+    self.assertIsInstance(operator.cholesky(), linalg.LinearOperatorDiag)
+
+  def test_diag_inverse_type(self):
+    diag = [1., 3., 5., 8.]
+    operator = linalg.LinearOperatorDiag(diag, is_non_singular=True)
+    self.assertIsInstance(operator.inverse(), linalg.LinearOperatorDiag)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
index 2da5e712d7..14b5228bca 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
@@ -265,9 +265,14 @@ class LinearOperatorIdentityTest(
         is_positive_definite=True,
         is_self_adjoint=True,
     )
-    self.assertTrue(isinstance(
-        operator.cholesky(),
-        linalg_lib.LinearOperatorIdentity))
+    self.assertIsInstance(
+        operator.cholesky(), linalg_lib.LinearOperatorIdentity)
+
+  def test_identity_inverse_type(self):
+    operator = linalg_lib.LinearOperatorIdentity(
+        num_rows=2, is_non_singular=True)
+    self.assertIsInstance(
+        operator.inverse(), linalg_lib.LinearOperatorIdentity)
 
 
 class LinearOperatorScaledIdentityTest(
@@ -491,10 +496,19 @@ class LinearOperatorScaledIdentityTest(
         is_positive_definite=True,
         is_self_adjoint=True,
     )
-    self.assertTrue(isinstance(
+    self.assertIsInstance(
         operator.cholesky(),
-        linalg_lib.LinearOperatorScaledIdentity))
+        linalg_lib.LinearOperatorScaledIdentity)
 
+  def test_scaled_identity_inverse_type(self):
+    operator = linalg_lib.LinearOperatorScaledIdentity(
+        num_rows=2,
+        multiplier=3.,
+        is_non_singular=True,
+    )
+    self.assertIsInstance(
+        operator.inverse(),
+        linalg_lib.LinearOperatorScaledIdentity)
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_kronecker_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_kronecker_test.py
index 513b246803..54ccc0c5f6 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_kronecker_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_kronecker_test.py
@@ -100,7 +100,7 @@ class SquareLinearOperatorKroneckerTest(
 
   @property
   def _tests_to_skip(self):
-    return ["det", "solve", "solve_with_broadcast"]
+    return ["det", "inverse", "solve", "solve_with_broadcast"]
 
   def _operator_and_matrix(
       self, build_info, dtype, use_placeholder,
@@ -211,20 +211,33 @@ class SquareLinearOperatorKroneckerTest(
         is_self_adjoint=True,
     )
     cholesky_factor = operator.cholesky()
-    self.assertTrue(isinstance(
+    self.assertIsInstance(
         cholesky_factor,
-        kronecker.LinearOperatorKronecker))
+        kronecker.LinearOperatorKronecker)
     self.assertEqual(2, len(cholesky_factor.operators))
-    self.assertTrue(
-        isinstance(
-            cholesky_factor.operators[0],
-            lower_triangular.LinearOperatorLowerTriangular)
-    )
-    self.assertTrue(
-        isinstance(
-            cholesky_factor.operators[1],
-            lower_triangular.LinearOperatorLowerTriangular)
+    self.assertIsInstance(
+        cholesky_factor.operators[0],
+        lower_triangular.LinearOperatorLowerTriangular)
+    self.assertIsInstance(
+        cholesky_factor.operators[1],
+        lower_triangular.LinearOperatorLowerTriangular)
+
+  def test_kronecker_inverse_type(self):
+    matrix = [[1., 0.], [0., 1.]]
+    operator = kronecker.LinearOperatorKronecker(
+        [
+            linalg.LinearOperatorFullMatrix(
+                matrix, is_non_singular=True),
+            linalg.LinearOperatorFullMatrix(
+                matrix, is_non_singular=True),
+        ],
+        is_non_singular=True,
     )
+    inverse = operator.inverse()
+    self.assertIsInstance(
+        inverse,
+        kronecker.LinearOperatorKronecker)
+    self.assertEqual(2, len(inverse.operators))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_zeros_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_zeros_test.py
index eb0b8ef127..10651d3c8a 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_zeros_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_zeros_test.py
@@ -36,7 +36,8 @@ class LinearOperatorZerosTest(
 
   @property
   def _tests_to_skip(self):
-    return ["cholesky", "log_abs_det", "solve", "solve_with_broadcast"]
+    return [
+        "cholesky", "log_abs_det", "inverse", "solve", "solve_with_broadcast"]
 
   @property
   def _operator_build_infos(self):
diff --git a/tensorflow/python/ops/linalg/inverse_registrations.py b/tensorflow/python/ops/linalg/inverse_registrations.py
new file mode 100644
index 0000000000..b89bdd240e
--- /dev/null
+++ b/tensorflow/python/ops/linalg/inverse_registrations.py
@@ -0,0 +1,108 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Registrations for LinearOperator.inverse."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.ops.linalg import linear_operator
+from tensorflow.python.ops.linalg import linear_operator_algebra
+from tensorflow.python.ops.linalg import linear_operator_block_diag
+from tensorflow.python.ops.linalg import linear_operator_circulant
+from tensorflow.python.ops.linalg import linear_operator_diag
+from tensorflow.python.ops.linalg import linear_operator_identity
+from tensorflow.python.ops.linalg import linear_operator_inversion
+from tensorflow.python.ops.linalg import linear_operator_kronecker
+
+
+# By default, return LinearOperatorInversion which switched the .matmul
+# and .solve methods.
+@linear_operator_algebra.RegisterInverse(linear_operator.LinearOperator)
+def _inverse_linear_operator(linop):
+  return linear_operator_inversion.LinearOperatorInversion(
+      linop,
+      is_non_singular=linop.is_non_singular,
+      is_self_adjoint=linop.is_self_adjoint,
+      is_positive_definite=linop.is_positive_definite,
+      is_square=linop.is_square)
+
+
+@linear_operator_algebra.RegisterInverse(
+    linear_operator_diag.LinearOperatorDiag)
+def _inverse_diag(diag_operator):
+  return linear_operator_diag.LinearOperatorDiag(
+      1. / diag_operator.diag,
+      is_non_singular=diag_operator.is_non_singular,
+      is_self_adjoint=diag_operator.is_self_adjoint,
+      is_positive_definite=diag_operator.is_positive_definite,
+      is_square=True)
+
+
+@linear_operator_algebra.RegisterInverse(
+    linear_operator_identity.LinearOperatorIdentity)
+def _inverse_identity(identity_operator):
+  return identity_operator
+
+
+@linear_operator_algebra.RegisterInverse(
+    linear_operator_identity.LinearOperatorScaledIdentity)
+def _inverse_scaled_identity(identity_operator):
+  return linear_operator_identity.LinearOperatorScaledIdentity(
+      num_rows=identity_operator._num_rows,  # pylint: disable=protected-access
+      multiplier=1. / identity_operator.multiplier,
+      is_non_singular=identity_operator.is_non_singular,
+      is_self_adjoint=True,
+      is_positive_definite=identity_operator.is_positive_definite,
+      is_square=True)
+
+
+@linear_operator_algebra.RegisterInverse(
+    linear_operator_block_diag.LinearOperatorBlockDiag)
+def _inverse_block_diag(block_diag_operator):
+    # We take the inverse of each block on the diagonal.
+  return linear_operator_block_diag.LinearOperatorBlockDiag(
+      operators=[
+          operator.inverse() for operator in block_diag_operator.operators],
+      is_non_singular=block_diag_operator.is_non_singular,
+      is_self_adjoint=block_diag_operator.is_self_adjoint,
+      is_positive_definite=block_diag_operator.is_positive_definite,
+      is_square=True)
+
+
+@linear_operator_algebra.RegisterInverse(
+    linear_operator_kronecker.LinearOperatorKronecker)
+def _inverse_kronecker(kronecker_operator):
+    # Inverse decomposition of a Kronecker product is the Kronecker product
+    # of inverse decompositions.
+  return linear_operator_kronecker.LinearOperatorKronecker(
+      operators=[
+          operator.inverse() for operator in kronecker_operator.operators],
+      is_non_singular=kronecker_operator.is_non_singular,
+      is_self_adjoint=kronecker_operator.is_self_adjoint,
+      is_positive_definite=kronecker_operator.is_positive_definite,
+      is_square=True)
+
+
+@linear_operator_algebra.RegisterInverse(
+    linear_operator_circulant.LinearOperatorCirculant)
+def _inverse_circulant(circulant_operator):
+  # Inverting the spectrum is sufficient to get the inverse.
+  return linear_operator_circulant.LinearOperatorCirculant(
+      spectrum=1. / circulant_operator.spectrum,
+      is_non_singular=circulant_operator.is_non_singular,
+      is_self_adjoint=circulant_operator.is_self_adjoint,
+      is_positive_definite=circulant_operator.is_positive_definite,
+      is_square=True)
diff --git a/tensorflow/python/ops/linalg/linalg.py b/tensorflow/python/ops/linalg/linalg.py
index ac4fd4ebc6..eebe741337 100644
--- a/tensorflow/python/ops/linalg/linalg.py
+++ b/tensorflow/python/ops/linalg/linalg.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 # go/tf-wildcard-import
 # pylint: disable=wildcard-import,unused-import
 from tensorflow.python.ops.linalg import cholesky_registrations as _cholesky_registrations
+from tensorflow.python.ops.linalg import inverse_registrations as _inverse_registrations
 from tensorflow.python.ops.linalg import linear_operator_algebra as _linear_operator_algebra
 from tensorflow.python.ops.linalg import matmul_registrations as _matmul_registrations
 from tensorflow.python.ops.linalg.linalg_impl import *
diff --git a/tensorflow/python/ops/linalg/linear_operator.py b/tensorflow/python/ops/linalg/linear_operator.py
index 6be81f4b34..4c99e86dc5 100644
--- a/tensorflow/python/ops/linalg/linear_operator.py
+++ b/tensorflow/python/ops/linalg/linear_operator.py
@@ -847,6 +847,31 @@ class LinearOperator(object):
 
       return self._solvevec(rhs, adjoint=adjoint)
 
+  def inverse(self, name="inverse"):
+    """Returns the Inverse of this `LinearOperator`.
+
+    Given `A` representing this `LinearOperator`, return a `LinearOperator`
+    representing `A^-1`.
+
+    Args:
+      name: A name scope to use for ops added by this method.
+
+    Returns:
+      `LinearOperator` representing inverse of this matrix.
+
+    Raises:
+      ValueError: When the `LinearOperator` is not hinted to be `non_singular`.
+    """
+    if self.is_square is False:  # pylint: disable=g-bool-id-comparison
+      raise ValueError("Cannot take the Inverse: This operator represents "
+                       "a non square matrix.")
+    if self.is_non_singular is False:  # pylint: disable=g-bool-id-comparison
+      raise ValueError("Cannot take the Inverse: This operator represents "
+                       "a singular matrix.")
+
+    with self._name_scope(name):
+      return linear_operator_algebra.inverse(self)
+
   def cholesky(self, name="cholesky"):
     """Returns a Cholesky factor as a `LinearOperator`.
 
diff --git a/tensorflow/python/ops/linalg/linear_operator_algebra.py b/tensorflow/python/ops/linalg/linear_operator_algebra.py
index 7b99066e4c..c1513fdb38 100644
--- a/tensorflow/python/ops/linalg/linear_operator_algebra.py
+++ b/tensorflow/python/ops/linalg/linear_operator_algebra.py
@@ -27,6 +27,7 @@ from tensorflow.python.util import tf_inspect
 
 _CHOLESKY_DECOMPS = {}
 _MATMUL = {}
+_INVERSES = {}
 
 
 def _registered_function(type_list, registry):
@@ -55,6 +56,11 @@ def _registered_matmul(type_a, type_b):
   return _registered_function([type_a, type_b], _MATMUL)
 
 
+def _registered_inverse(type_a):
+  """Get the Cholesky function registered for class a."""
+  return _registered_function([type_a], _INVERSES)
+
+
 def cholesky(lin_op_a, name=None):
   """Get the Cholesky factor associated to lin_op_a.
 
@@ -103,6 +109,29 @@ def matmul(lin_op_a, lin_op_b, name=None):
     return matmul_fn(lin_op_a, lin_op_b)
 
 
+def inverse(lin_op_a, name=None):
+  """Get the Inverse associated to lin_op_a.
+
+  Args:
+    lin_op_a: The LinearOperator to decompose.
+    name: Name to use for this operation.
+
+  Returns:
+    A LinearOperator that represents the inverse of `lin_op_a`.
+
+  Raises:
+    NotImplementedError: If no Inverse method is defined for the LinearOperator
+      type of `lin_op_a`.
+  """
+  inverse_fn = _registered_inverse(type(lin_op_a))
+  if inverse_fn is None:
+    raise ValueError("No inverse registered for {}".format(
+        type(lin_op_a)))
+
+  with ops.name_scope(name, "Inverse"):
+    return inverse_fn(lin_op_a)
+
+
 class RegisterCholesky(object):
   """Decorator to register a Cholesky implementation function.
 
@@ -189,3 +218,45 @@ class RegisterMatmul(object):
           self._key[1].__name__))
     _MATMUL[self._key] = matmul_fn
     return matmul_fn
+
+
+class RegisterInverse(object):
+  """Decorator to register an Inverse implementation function.
+
+  Usage:
+
+  @linear_operator_algebra.RegisterInverse(lin_op.LinearOperatorIdentity)
+  def _inverse_identity(lin_op_a):
+    # Return the identity matrix.
+  """
+
+  def __init__(self, lin_op_cls_a):
+    """Initialize the LinearOperator registrar.
+
+    Args:
+      lin_op_cls_a: the class of the LinearOperator to decompose.
+    """
+    self._key = (lin_op_cls_a,)
+
+  def __call__(self, inverse_fn):
+    """Perform the Inverse registration.
+
+    Args:
+      inverse_fn: The function to use for the Inverse.
+
+    Returns:
+      inverse_fn
+
+    Raises:
+      TypeError: if inverse_fn is not a callable.
+      ValueError: if a Inverse function has already been registered for
+        the given argument classes.
+    """
+    if not callable(inverse_fn):
+      raise TypeError(
+          "inverse_fn must be callable, received: {}".format(inverse_fn))
+    if self._key in _INVERSES:
+      raise ValueError("Inverse({}) has already been registered to: {}".format(
+          self._key[0].__name__, _INVERSES[self._key]))
+    _INVERSES[self._key] = inverse_fn
+    return inverse_fn
diff --git a/tensorflow/python/ops/linalg/linear_operator_test_util.py b/tensorflow/python/ops/linalg/linear_operator_test_util.py
index e50f572b5f..a957c84dc1 100644
--- a/tensorflow/python/ops/linalg/linear_operator_test_util.py
+++ b/tensorflow/python/ops/linalg/linear_operator_test_util.py
@@ -336,6 +336,22 @@ class LinearOperatorDerivedClassTest(test.TestCase):
     self._skip_if_tests_to_skip_contains("solve_with_broadcast")
     self._test_solve(with_batch=False)
 
+  def _test_inverse(self):
+    for use_placeholder in self._use_placeholder_options:
+      for build_info in self._operator_build_infos:
+        for dtype in self._dtypes_to_test:
+          with self.session(graph=ops.Graph()) as sess:
+            sess.graph.seed = random_seed.DEFAULT_GRAPH_SEED
+            operator, mat = self._operator_and_matrix(
+                build_info, dtype, use_placeholder=use_placeholder)
+            op_inverse_v, mat_inverse_v = sess.run([
+                operator.inverse().to_dense(), linalg.inv(mat)])
+            self.assertAC(op_inverse_v, mat_inverse_v)
+
+  def test_inverse(self):
+    self._skip_if_tests_to_skip_contains("inverse")
+    self._test_inverse()
+
   def test_trace(self):
     self._skip_if_tests_to_skip_contains("trace")
     for use_placeholder in self._use_placeholder_options:
@@ -463,7 +479,14 @@ class NonSquareLinearOperatorDerivedClassTest(LinearOperatorDerivedClassTest):
   @property
   def _tests_to_skip(self):
     """List of test names to skip."""
-    return ["cholesky", "solve", "solve_with_broadcast", "det", "log_abs_det"]
+    return [
+        "cholesky",
+        "inverse",
+        "solve",
+        "solve_with_broadcast",
+        "det",
+        "log_abs_det"
+    ]
 
   @property
   def _operator_build_infos(self):
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-block-diag.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-block-diag.pbtxt
index 773c74e64d..c7a50969b5 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-block-diag.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-block-diag.pbtxt
@@ -95,6 +95,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant.pbtxt
index 533544d21f..3900c752c8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant.pbtxt
@@ -116,6 +116,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt
index e3926eb6d4..7b876099af 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt
@@ -116,6 +116,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt
index ba209df782..5bddba8e79 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt
@@ -116,6 +116,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-composition.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-composition.pbtxt
index 081fb0e08b..62ba8bb59e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-composition.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-composition.pbtxt
@@ -95,6 +95,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-diag.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-diag.pbtxt
index 2014a04301..0803feeabd 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-diag.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-diag.pbtxt
@@ -95,6 +95,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-full-matrix.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-full-matrix.pbtxt
index 9a87ae9687..6def32864b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-full-matrix.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-full-matrix.pbtxt
@@ -91,6 +91,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-identity.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-identity.pbtxt
index 33afb835ce..dbf1ac82d3 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-identity.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-identity.pbtxt
@@ -92,6 +92,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-inversion.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-inversion.pbtxt
new file mode 100644
index 0000000000..6a3fe4dd66
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-inversion.pbtxt
@@ -0,0 +1,142 @@
+path: "tensorflow.linalg.LinearOperatorInversion"
+tf_class {
+  is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_inversion.LinearOperatorInversion\'>"
+  is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "batch_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "domain_dimension"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "graph_parents"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_non_singular"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_positive_definite"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_self_adjoint"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_square"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "operator"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "range_dimension"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "tensor_rank"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'operator\', \'is_non_singular\', \'is_self_adjoint\', \'is_positive_definite\', \'is_square\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_to_tensor"
+    argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
+  }
+  member_method {
+    name: "assert_non_singular"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
+  }
+  member_method {
+    name: "assert_positive_definite"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_positive_definite\'], "
+  }
+  member_method {
+    name: "assert_self_adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_self_adjoint\'], "
+  }
+  member_method {
+    name: "batch_shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
+  }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
+  member_method {
+    name: "determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
+  }
+  member_method {
+    name: "diag_part"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'diag_part\'], "
+  }
+  member_method {
+    name: "domain_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
+  }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
+  member_method {
+    name: "log_abs_determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
+  }
+  member_method {
+    name: "matmul"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'matmul\'], "
+  }
+  member_method {
+    name: "matvec"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'matvec\'], "
+  }
+  member_method {
+    name: "range_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'range_dimension_tensor\'], "
+  }
+  member_method {
+    name: "shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'shape_tensor\'], "
+  }
+  member_method {
+    name: "solve"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'solve\'], "
+  }
+  member_method {
+    name: "solvevec"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'solve\'], "
+  }
+  member_method {
+    name: "tensor_rank_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'tensor_rank_tensor\'], "
+  }
+  member_method {
+    name: "to_dense"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'to_dense\'], "
+  }
+  member_method {
+    name: "trace"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'trace\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-kronecker.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-kronecker.pbtxt
index a9078c8ab5..85d902b977 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-kronecker.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-kronecker.pbtxt
@@ -95,6 +95,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt
index 4cfa3bb30d..638d82a599 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt
@@ -115,6 +115,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt
index a87649133f..ab1b04bd3c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt
@@ -91,6 +91,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt
index 3265646784..961969aac5 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt
@@ -96,6 +96,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-zeros.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-zeros.pbtxt
index 49d8890c89..e76738a964 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-zeros.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-zeros.pbtxt
@@ -91,6 +91,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator.pbtxt
index c89dc067b3..b35cd69da4 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator.pbtxt
@@ -90,6 +90,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.pbtxt
index 9f7b422fab..5e49b75c31 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.pbtxt
@@ -36,6 +36,10 @@ tf_module {
     name: "LinearOperatorIdentity"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "LinearOperatorInversion"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "LinearOperatorKronecker"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-block-diag.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-block-diag.pbtxt
index 773c74e64d..c7a50969b5 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-block-diag.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-block-diag.pbtxt
@@ -95,6 +95,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant.pbtxt
index 533544d21f..3900c752c8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant.pbtxt
@@ -116,6 +116,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt
index e3926eb6d4..7b876099af 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt
@@ -116,6 +116,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt
index ba209df782..5bddba8e79 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt
@@ -116,6 +116,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-composition.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-composition.pbtxt
index 081fb0e08b..62ba8bb59e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-composition.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-composition.pbtxt
@@ -95,6 +95,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-diag.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-diag.pbtxt
index 2014a04301..0803feeabd 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-diag.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-diag.pbtxt
@@ -95,6 +95,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-full-matrix.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-full-matrix.pbtxt
index 9a87ae9687..6def32864b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-full-matrix.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-full-matrix.pbtxt
@@ -91,6 +91,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-identity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-identity.pbtxt
index 33afb835ce..dbf1ac82d3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-identity.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-identity.pbtxt
@@ -92,6 +92,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-inversion.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-inversion.pbtxt
new file mode 100644
index 0000000000..6a3fe4dd66
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-inversion.pbtxt
@@ -0,0 +1,142 @@
+path: "tensorflow.linalg.LinearOperatorInversion"
+tf_class {
+  is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_inversion.LinearOperatorInversion\'>"
+  is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "batch_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "domain_dimension"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "graph_parents"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_non_singular"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_positive_definite"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_self_adjoint"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_square"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "operator"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "range_dimension"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "tensor_rank"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'operator\', \'is_non_singular\', \'is_self_adjoint\', \'is_positive_definite\', \'is_square\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_to_tensor"
+    argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
+  }
+  member_method {
+    name: "assert_non_singular"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
+  }
+  member_method {
+    name: "assert_positive_definite"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_positive_definite\'], "
+  }
+  member_method {
+    name: "assert_self_adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_self_adjoint\'], "
+  }
+  member_method {
+    name: "batch_shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
+  }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
+  member_method {
+    name: "determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
+  }
+  member_method {
+    name: "diag_part"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'diag_part\'], "
+  }
+  member_method {
+    name: "domain_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
+  }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
+  member_method {
+    name: "log_abs_determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
+  }
+  member_method {
+    name: "matmul"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'matmul\'], "
+  }
+  member_method {
+    name: "matvec"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'matvec\'], "
+  }
+  member_method {
+    name: "range_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'range_dimension_tensor\'], "
+  }
+  member_method {
+    name: "shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'shape_tensor\'], "
+  }
+  member_method {
+    name: "solve"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'solve\'], "
+  }
+  member_method {
+    name: "solvevec"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'solve\'], "
+  }
+  member_method {
+    name: "tensor_rank_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'tensor_rank_tensor\'], "
+  }
+  member_method {
+    name: "to_dense"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'to_dense\'], "
+  }
+  member_method {
+    name: "trace"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'trace\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-kronecker.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-kronecker.pbtxt
index a9078c8ab5..85d902b977 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-kronecker.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-kronecker.pbtxt
@@ -95,6 +95,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt
index 4cfa3bb30d..638d82a599 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt
@@ -115,6 +115,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt
index a87649133f..ab1b04bd3c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt
@@ -91,6 +91,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt
index 3265646784..961969aac5 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt
@@ -96,6 +96,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-zeros.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-zeros.pbtxt
index 49d8890c89..e76738a964 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-zeros.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-zeros.pbtxt
@@ -91,6 +91,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator.pbtxt
index c89dc067b3..b35cd69da4 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator.pbtxt
@@ -90,6 +90,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt
index 3e1e2e3d54..f9119cdd5f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt
@@ -36,6 +36,10 @@ tf_module {
     name: "LinearOperatorIdentity"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "LinearOperatorInversion"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "LinearOperatorKronecker"
     mtype: "<type \'type\'>"
-- 
GitLab


From 4b2b456ea4353864478e608a587ea9b5f7eb696c Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Wed, 9 Jan 2019 15:00:20 -0800
Subject: [PATCH 0463/2345] Remove a deprecation warning from
 tf.saved_model.save

PiperOrigin-RevId: 228594973
---
 tensorflow/python/saved_model/save.py       | 2 +-
 tensorflow/python/saved_model/utils_impl.py | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index d79b5fe3a6..9db6d03ed0 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -222,7 +222,7 @@ def _normalize_outputs(outputs, function_name, signature_key):
 
 
 def _tensor_dict_to_tensorinfo(tensor_dict):
-  return {key: utils_impl.build_tensor_info(value)
+  return {key: utils_impl.build_tensor_info_internal(value)
           for key, value in tensor_dict.items()}
 
 
diff --git a/tensorflow/python/saved_model/utils_impl.py b/tensorflow/python/saved_model/utils_impl.py
index d43cfdf2b3..a82007fd54 100644
--- a/tensorflow/python/saved_model/utils_impl.py
+++ b/tensorflow/python/saved_model/utils_impl.py
@@ -60,6 +60,11 @@ def build_tensor_info(tensor):
   """
   if context.executing_eagerly():
     raise RuntimeError("build_tensor_info is not supported in Eager mode.")
+  return build_tensor_info_internal(tensor)
+
+
+def build_tensor_info_internal(tensor):
+  """Utility function to build TensorInfo proto from a Tensor."""
   tensor_info = meta_graph_pb2.TensorInfo(
       dtype=dtypes.as_dtype(tensor.dtype).as_datatype_enum,
       tensor_shape=tensor.get_shape().as_proto())
-- 
GitLab


From 00a074f82ebaebb436830b13978b3c24e514869d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 15:04:45 -0800
Subject: [PATCH 0464/2345] Precalculate eager op's cache key, so this somewhat
 expensive operation doesn't have to be repeated for each inference.

PiperOrigin-RevId: 228595957
---
 .../core/common_runtime/eager/attr_builder.cc       | 13 ++++++++++++-
 tensorflow/core/common_runtime/eager/attr_builder.h | 11 +++++++++--
 .../core/common_runtime/eager/attr_builder_test.cc  | 13 +++++++++++++
 tensorflow/lite/delegates/flex/kernel.cc            |  6 ++++++
 4 files changed, 40 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/common_runtime/eager/attr_builder.cc b/tensorflow/core/common_runtime/eager/attr_builder.cc
index a750f8cbba..689b04274f 100644
--- a/tensorflow/core/common_runtime/eager/attr_builder.cc
+++ b/tensorflow/core/common_runtime/eager/attr_builder.cc
@@ -125,6 +125,7 @@ Status AttrTypeMapForOp(const char* op_name, const AttrTypeMap** out,
   template <>                                                                \
   AttrBuilder& AttrBuilder::Set(StringPiece attr_name, value_type&& value) { \
     value_field.push_back(std::make_pair(string(attr_name), value));         \
+    cached_cache_key_ = absl::nullopt;                                       \
     return *this;                                                            \
   }
 
@@ -231,7 +232,17 @@ inline tensorflow::Fprint128 CacheKeyHelper(StringPiece s, uint64 b) {
 
 }  // namespace
 
-tensorflow::Fprint128 AttrBuilder::CacheKey(const string& device) const {
+tensorflow::Fprint128 AttrBuilder::CacheKey(const string& device) {
+  if (!cached_cache_key_ || device != device_for_cached_cache_key_) {
+    cached_cache_key_ = BuildCacheKeyForDevice(device);
+    device_for_cached_cache_key_ = device;
+  }
+
+  return *cached_cache_key_;
+}
+
+tensorflow::Fprint128 AttrBuilder::BuildCacheKeyForDevice(
+    const string& device) const {
   tensorflow::Fprint128 f = tensorflow::Fingerprint128(op_name_);
   f = tensorflow::FingerprintCat128(f, tensorflow::Fingerprint128(device));
   if (node_def_ != nullptr) {
diff --git a/tensorflow/core/common_runtime/eager/attr_builder.h b/tensorflow/core/common_runtime/eager/attr_builder.h
index 5e0172dfd3..aa64b5f59b 100644
--- a/tensorflow/core/common_runtime/eager/attr_builder.h
+++ b/tensorflow/core/common_runtime/eager/attr_builder.h
@@ -28,6 +28,7 @@ limitations under the License.
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/gtl/inlined_vector.h"
+#include "tensorflow/core/lib/gtl/optional.h"
 #include "tensorflow/core/platform/fingerprint.h"
 #include "tensorflow/core/util/tensor_slice_reader_cache.h"
 
@@ -74,7 +75,7 @@ Status AttrTypeByName(const AttrTypeMap& m, const string& attr_name,
 // AttrBuilder a;
 // a.NumInputs(2);
 // a.Set("T", TF_FLOAT);
-// uint64 cache_key = a.CacheKey("cpu:0");
+// tensorflow::Fprint128 cache_key = a.CacheKey("cpu:0");
 // const NodeDef& n = a.BuildNodeDef();
 //
 // Note that all calls to Set and NumInputs should happen before calling
@@ -100,10 +101,11 @@ class AttrBuilder {
   AttrBuilder& Set(StringPiece attr_name, T&& value) {
     MayBeInitializeNodeDef();
     SetInAttrValueMap(node_def_->mutable_attr(), string(attr_name), value);
+    cached_cache_key_ = absl::nullopt;
     return *this;
   }
 
-  tensorflow::Fprint128 CacheKey(const string& device) const;
+  tensorflow::Fprint128 CacheKey(const string& device);
 
   void FillAttrValueMap(AttrValueMap* m) const { FillAttrValueMap(m, true); }
   const NodeDef& BuildNodeDef();
@@ -112,6 +114,8 @@ class AttrBuilder {
   template <class T>
   using AttrVec = tensorflow::gtl::InlinedVector<std::pair<string, T>, 2>;
 
+  tensorflow::Fprint128 BuildCacheKeyForDevice(const string& device) const;
+
   void MayBeInitializeNodeDef();
   // Fill `m` with the attr-value pairs set via AttrBuilder::Set() so far, as
   // well as any default attr-value pairs from the associated op_def, if there
@@ -148,6 +152,9 @@ class AttrBuilder {
   int num_inputs_;
   std::unique_ptr<NodeDef> node_def_;
   bool node_def_finalized_;
+
+  absl::optional<tensorflow::Fprint128> cached_cache_key_;
+  string device_for_cached_cache_key_;
 };  // namespace tensorflow
 
 template <>
diff --git a/tensorflow/core/common_runtime/eager/attr_builder_test.cc b/tensorflow/core/common_runtime/eager/attr_builder_test.cc
index 220cc6f5ce..8245660bfc 100644
--- a/tensorflow/core/common_runtime/eager/attr_builder_test.cc
+++ b/tensorflow/core/common_runtime/eager/attr_builder_test.cc
@@ -67,5 +67,18 @@ TEST(AttrTypeMap, Lookup) {
   EXPECT_NE(is_list, 0);
 }
 
+TEST(AttrTypeMap, CacheKey) {
+  AttrBuilder a("op_name");
+  a.NumInputs(2);
+  a.Set("T", TF_FLOAT);
+  tensorflow::Fprint128 cache_key = a.CacheKey("cpu:0");
+
+  ASSERT_FALSE(cache_key == a.CacheKey("cpu:1"));
+  ASSERT_TRUE(cache_key == a.CacheKey("cpu:0"));
+
+  a.Set("x", 1.0);
+  ASSERT_FALSE(cache_key == a.CacheKey("cpu:0"));
+}
+
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/lite/delegates/flex/kernel.cc b/tensorflow/lite/delegates/flex/kernel.cc
index d2853cf9f8..2e0fc22ad6 100644
--- a/tensorflow/lite/delegates/flex/kernel.cc
+++ b/tensorflow/lite/delegates/flex/kernel.cc
@@ -245,6 +245,12 @@ class OpNode {
       op_->MutableAttrs()->Set(attr.first, attr.second);
     }
 
+    // Precalculating a cache key saves about 10% of inference time for very
+    // small models.
+    tensorflow::Device* device = op_->Device();
+    op_->MutableAttrs()->CacheKey(device == nullptr ? "unspecified"
+                                                    : device->name());
+
     return tensorflow::Status::OK();
   }
 
-- 
GitLab


From d7cde5bd2cf5d0d7a69773422716590b79c7bed2 Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Wed, 9 Jan 2019 15:18:43 -0800
Subject: [PATCH 0465/2345] Local collective all reduce with eager mode: wraps
 invocations to collective allreduce into a defun; Only broadcast send/recv
 initial values on the first replica of each worker.

PiperOrigin-RevId: 228598348
---
 .../python/collective_all_reduce_strategy.py  | 71 ++++++++++++-------
 .../collective_all_reduce_strategy_test.py    | 26 ++++---
 .../python/distribute/cross_device_ops.py     | 15 +---
 .../python/distribute/cross_device_utils.py   | 28 +++++---
 4 files changed, 85 insertions(+), 55 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
index 39756b32c5..a3538dfc50 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
@@ -31,6 +31,7 @@ from tensorflow.python.distribute import multi_worker_util
 from tensorflow.python.distribute import numpy_dataset
 from tensorflow.python.distribute import values
 from tensorflow.python.eager import context
+from tensorflow.python.eager import tape
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import collective_ops
@@ -91,6 +92,7 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
 
     self._collective_keys = cross_device_utils.CollectiveKeys()
     self._initialize_local(local_devices)
+    # TODO(yuefengz): remove num_gpus_per_worker from CollectiveAllReduce.
     self._cross_device_ops = cross_device_ops_lib.CollectiveAllReduce(
         num_workers=self._num_workers,
         num_gpus_per_worker=num_gpus_per_worker,
@@ -166,16 +168,17 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
     else:
       device_map = colocate_with.device_map
       logical_device = colocate_with.logical_device
-    group_size = device_map.num_replicas_in_graph * self._num_workers
-    group_key = self._collective_keys.get_group_key(self.worker_devices)
 
     def _real_mirrored_creator(devices, *args, **kwargs):
       """Creates one MirroredVariable on the current worker."""
-      value_list = []
       unique_var_name = ops.get_default_graph().unique_name(
           kwargs["name"], mark_as_used=False).rstrip("/")
+      # pylint: disable=protected-access
       collective_instance_key = self._collective_keys.get_instance_key(
           key_id=unique_var_name)
+      # Only the first device participles in the broadcast of initial values.
+      group_key = self._collective_keys.get_group_key([devices[0]])
+      group_size = self._num_workers
       if "initial_value" not in kwargs:
         raise ValueError("Initial value must be specified.")
       initial_value = kwargs["initial_value"]
@@ -184,9 +187,33 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
       else:
         initial_value_fn = lambda: initial_value
 
+      value_list = []
       for i, d in enumerate(devices):
-        with ops.device(d):
-          if i > 0:
+        with ops.init_scope(), ops.device(d):
+          if i == 0:
+            # The initial value fn makes sure variables all initialized to
+            # same values. The first device of the chief worker will send their
+            # variable values to other workers.
+            def _overridden_initial_value_fn(device=d, index=i):  # pylint: disable=g-missing-docstring
+              with ops.device(device):
+                initial_value = initial_value_fn()
+                assert not callable(initial_value)
+                initial_value = ops.convert_to_tensor(initial_value)
+
+                assert index == 0, index
+                if self._num_workers > 1:
+                  if self._is_chief:
+                    bcast_send = collective_ops.broadcast_send(
+                        initial_value, initial_value.shape, initial_value.dtype,
+                        group_size, group_key, collective_instance_key)
+                    with ops.control_dependencies([bcast_send]):
+                      return array_ops.identity(initial_value)
+                  else:
+                    return collective_ops.broadcast_recv(
+                        initial_value.shape, initial_value.dtype, group_size,
+                        group_key, collective_instance_key)
+                return initial_value
+          else:
             # Give replicas meaningful distinct names:
             var0name = value_list[0].name.split(":")[0]
             # We append a / to variable names created on replicas with id > 0 to
@@ -194,30 +221,22 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
             # name as the absolute name of the variable.
             kwargs["name"] = "%s/replica_%d/" % (var0name, i)
 
-          # The initial value fn makes sure variables all initialized to
-          # same values. The first device of the chief worker will send their
-          # variable values to other devices and other workers.
-          def _overridden_initial_value_fn(device=d, index=i):  # pylint: disable=g-missing-docstring
-            with ops.device(device):
-              initial_value = initial_value_fn()
-              assert not callable(initial_value)
-              initial_value = ops.convert_to_tensor(initial_value)
-
-              if self._is_chief and index == 0:
-                bcast_send = collective_ops.broadcast_send(
-                    initial_value, initial_value.shape, initial_value.dtype,
-                    group_size, group_key, collective_instance_key)
-                with ops.control_dependencies([bcast_send]):
-                  return array_ops.identity(initial_value)
-              else:
-                return collective_ops.broadcast_recv(
-                    initial_value.shape, initial_value.dtype, group_size,
-                    group_key, collective_instance_key)
+            # Variables on non-first replica get initial values from the
+            # variables created on the first device of each worker.
+            def _overridden_initial_value_fn(device=d, index=i):
+              assert index > 0
+              with ops.device(device):
+                if context.executing_eagerly():
+                  return array_ops.identity(value_list[0].value())
+                else:
+                  return array_ops.identity(value_list[0].initial_value)
 
           kwargs["initial_value"] = _overridden_initial_value_fn
-
           with context.context().device_policy(context.DEVICE_PLACEMENT_SILENT):
-            v = next_creator(*args, **kwargs)
+            # Don't record operations (e.g. other variable reads) during
+            # variable creation.
+            with tape.stop_recording():
+              v = next_creator(*args, **kwargs)
 
           if i == 0:
             actual_var_name = v.name.split(":")[0]
diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
index c3e9c55e96..682da44f5d 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
@@ -192,6 +192,7 @@ class CollectiveAllReduceStrategyTestBase(
       image = random_ops.random_uniform([2, 28, 28])
       label = random_ops.random_uniform([2, 1], maxval=10, dtype=dtypes.int32)
       logits = model(image, training=True)
+      # TODO(yuefengz): make loss a callable for eager mode.
       loss = losses.sparse_softmax_cross_entropy(labels=label, logits=logits)
       optimizer = adam.AdamOptimizer(learning_rate=1e-4)
       train_op = optimizer.minimize(loss,
@@ -403,24 +404,31 @@ class LocalCollectiveAllReduceStrategy(
     strategy_test_lib.TwoDeviceDistributionTestBase,
     parameterized.TestCase):
 
-  def testMinimizeLossGraph(self, num_gpus=2):
+  @combinations.generate(
+      combinations.combine(
+          mode=['graph', 'eager'], num_gpus=[2, 4], required_gpus=2))
+  def testMinimizeLoss(self, num_gpus):
     # Collective ops doesn't support strategy with one device.
     if context.num_gpus() < num_gpus:
       self.skipTest('Not enough GPUs')
-    self._test_minimize_loss_graph(None, None, num_gpus)
+    if context.executing_eagerly():
+      strategy, _, _ = self._get_test_object(None, None, num_gpus)
+      self._test_minimize_loss_eager(strategy)
+    else:
+      self._test_minimize_loss_graph(None, None, num_gpus)
 
-  def testComplexModel(self, num_gpus=2):
-    # Collective ops doesn't support strategy with one device.
+  @combinations.generate(
+      combinations.combine(mode=['graph'], num_gpus=[2, 4], required_gpus=2))
+  def testComplexModel(self, num_gpus):
     if context.num_gpus() < num_gpus:
       self.skipTest('Not enough GPUs')
     self._test_complex_model(None, None, num_gpus)
 
+  @combinations.generate(
+      combinations.combine(mode=['graph', 'eager'], required_gpus=2))
   def testMakeInputFnIterator(self, num_gpus=2):
-    # Collective ops doesn't support strategy with one device.
-    if context.num_gpus() < num_gpus:
-      self.skipTest('Not enough GPUs')
-    dataset_fn = lambda: dataset_ops.Dataset.range(10)
-    expected_values = [[i, i+1] for i in range(0, 10, 2)]
+    dataset_fn = lambda: dataset_ops.Dataset.range(5 * num_gpus)
+    expected_values = [range(i, i + num_gpus) for i in range(0, 10, num_gpus)]
 
     input_fn = self._input_fn_to_test_input_context(
         dataset_fn,
diff --git a/tensorflow/python/distribute/cross_device_ops.py b/tensorflow/python/distribute/cross_device_ops.py
index 9575301d97..9729302c6d 100644
--- a/tensorflow/python/distribute/cross_device_ops.py
+++ b/tensorflow/python/distribute/cross_device_ops.py
@@ -323,6 +323,9 @@ class ReductionToOneDeviceCrossDeviceOps(CrossDeviceOps):
     assert check_destinations(destinations)
     devices = get_devices_from(destinations)
     reduce_to_device = self.reduce_to_device or devices[0]
+    logging.log_first_n(
+        logging.INFO,
+        "Reduce to %s then broadcast to %r." % (reduce_to_device, devices), 10)
     reduced = _simple_reduce(per_replica_value, reduce_to_device,
                              self.accumulation_fn, reduce_op)
     return self.broadcast(reduced, destinations)
@@ -839,9 +842,6 @@ class CollectiveAllReduce(CrossDeviceOps):
     if cross_device_utils.contains_indexed_slices(per_replica_value):
       raise ValueError(
           "`IndexSlices` is not supported for Collective All-Reduce.")
-    if context.executing_eagerly():
-      raise ValueError(
-          "Eager execution is not supported for Collective All-Reduce")
 
     all_reduced = self._batch_all_reduce(reduce_op, [per_replica_value])[0]
     device_map, logical_device = get_device_map_from(destinations)
@@ -865,9 +865,6 @@ class CollectiveAllReduce(CrossDeviceOps):
     if cross_device_utils.contains_indexed_slices(value_destination_pairs):
       raise ValueError(
           "`IndexSlices` is not supported for Collective All-Reduce.")
-    if context.executing_eagerly():
-      raise ValueError(
-          "Eager execution is not supported for Collective All-Reduce")
 
     all_devices_match = _all_devices_match(value_destination_pairs)
     if all_devices_match:
@@ -886,9 +883,6 @@ class CollectiveAllReduce(CrossDeviceOps):
 
   def _batch_all_reduce(self, reduce_op, per_replica_values):
     """All-reduce across all workers in a batch."""
-    if context.executing_eagerly():
-      raise ValueError(
-          "Eager execution with collective ops is not supported yet.")
 
     logging.log_first_n(
         logging.INFO, "Collective All-reduce invoked with batches size = %d, "
@@ -949,12 +943,9 @@ def _has_dgx1_like_links(gpu_links):
 
 def _choose_all_reduce_algorithm(device_links):
   if _has_dgx1_like_links(device_links):
-    logging.info("Configured hierarchical_copy with num_packs=%d",
-                 len(device_links))
     return AllReduceCrossDeviceOps(
         "hierarchical_copy", num_packs=len(device_links))
   else:
-    logging.info("Configured nccl all-reduce.")
     return AllReduceCrossDeviceOps("nccl", num_packs=1)
 
 
diff --git a/tensorflow/python/distribute/cross_device_utils.py b/tensorflow/python/distribute/cross_device_utils.py
index 5b4b3a6f97..e8066dd467 100644
--- a/tensorflow/python/distribute/cross_device_utils.py
+++ b/tensorflow/python/distribute/cross_device_utils.py
@@ -23,6 +23,8 @@ import threading
 
 from tensorflow.python.distribute import all_reduce
 from tensorflow.python.distribute import values as value_lib
+from tensorflow.python.eager import context
+from tensorflow.python.eager import def_function
 from tensorflow.python.framework import device as pydev
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -353,15 +355,25 @@ def build_collective_reduce(input_tensors,
   num_devices = len(devices)
   group_key = collective_keys.get_group_key(devices)
   instance_key = collective_keys.get_instance_key()
-  out_tensors = []
   subdiv_offsets = [0]  # TODO(tucker): maybe support non-default subdiv spec
-  for d in range(num_devices):
-    with ops.device(devices[d]):
-      reduce_op = collective_ops.all_reduce(
-          input_tensors[d], group_size, group_key, instance_key, reduction_op,
-          unary_op, subdiv_offsets)
-      out_tensors.append(reduce_op)
-  return out_tensors
+
+  def collective_all_reduce():
+    """Call collective allreduce."""
+    assert not context.executing_eagerly()
+    out_tensors = []
+    for d in range(num_devices):
+      with ops.device(devices[d]):
+        reduce_op = collective_ops.all_reduce(
+            input_tensors[d], group_size, group_key, instance_key, reduction_op,
+            unary_op, subdiv_offsets)
+        out_tensors.append(reduce_op)
+    return out_tensors
+
+  if context.executing_eagerly():
+    # Collective ops will block unless they are executed concurrently such as in
+    # a graph or a defun.
+    collective_all_reduce = def_function.function(collective_all_reduce)
+  return collective_all_reduce()
 
 
 def sum_grad_and_var_all_reduce(grad_and_vars,
-- 
GitLab


From 615182adb3d01fd8357e574bd8920c0995c6bbb8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 15:42:02 -0800
Subject: [PATCH 0466/2345] Fix issue with Subclassed models and generator
 input.

PiperOrigin-RevId: 228602224
---
 tensorflow/python/keras/callbacks.py          |  9 +---
 tensorflow/python/keras/callbacks_test.py     | 42 -------------------
 .../python/keras/engine/training_generator.py |  4 --
 .../keras/engine/training_generator_test.py   | 21 ++++------
 4 files changed, 8 insertions(+), 68 deletions(-)

diff --git a/tensorflow/python/keras/callbacks.py b/tensorflow/python/keras/callbacks.py
index 589cd992d6..b9074669eb 100644
--- a/tensorflow/python/keras/callbacks.py
+++ b/tensorflow/python/keras/callbacks.py
@@ -123,14 +123,6 @@ def configure_callbacks(callbacks,
       'metrics': callback_metrics,
   }
   callback_list.set_params(callback_params)
-
-  if (do_validation and not model._distribution_strategy and
-      not model.run_eagerly):
-    # Need to create the eval_function before start of the first epoch
-    # because TensorBoard callback on_epoch_begin adds summary to the
-    # list of fetches of the eval_function
-    callback_model._make_eval_function()
-
   callback_list.model.stop_training = False
   return callback_list
 # pylint: enable=protected-access
@@ -1373,6 +1365,7 @@ class TensorBoard(Callback):
       self._epoch = epoch
       # pylint: disable=protected-access
       # add the histogram summary op if it should run this epoch
+      self.model._make_eval_function()
       if self.merged not in self.model._eval_function.fetches:
         self.model._eval_function.fetches.append(self.merged)
         self.model._eval_function.fetch_callbacks[
diff --git a/tensorflow/python/keras/callbacks_test.py b/tensorflow/python/keras/callbacks_test.py
index ef469c5e4f..d5ffdf789a 100644
--- a/tensorflow/python/keras/callbacks_test.py
+++ b/tensorflow/python/keras/callbacks_test.py
@@ -31,7 +31,6 @@ import numpy as np
 
 from tensorflow.core.framework import summary_pb2
 from tensorflow.python import keras
-from tensorflow.python.framework import ops
 from tensorflow.python.framework import random_seed
 from tensorflow.python.framework import test_util
 from tensorflow.python.keras import keras_parameterized
@@ -1397,47 +1396,6 @@ class KerasCallbacksTest(test.TestCase):
             callbacks=cbks,
             epochs=1)
 
-  @test_util.run_deprecated_v1
-  def test_fit_generator_with_callback(self):
-
-    class TestCallback(keras.callbacks.Callback):
-
-      def set_model(self, model):
-        # Check the model operations for the optimizer operations that
-        # the _make_train_function adds under a named scope for the
-        # optimizer. This ensurs the full model is populated before the
-        # set_model callback is called.
-        optimizer_name_scope = 'training/' + model.optimizer.__class__.__name__
-        graph_def = ops.get_default_graph().as_graph_def()
-        for node in graph_def.node:
-          if node.name.startswith(optimizer_name_scope):
-            return
-        raise RuntimeError('The optimizer operations are not present in the '
-                           'model graph when the Callback.set_model function '
-                           'is called')
-    np.random.seed(1337)
-
-    def generator():
-      x = np.random.randn(10, 100).astype(np.float32)
-      y = np.random.randn(10, 10).astype(np.float32)
-      while True:
-        yield x, y
-
-    with self.cached_session():
-      model = testing_utils.get_small_sequential_mlp(
-          num_hidden=10, num_classes=10, input_dim=100)
-      model.compile(
-          loss='categorical_crossentropy',
-          optimizer='sgd',
-          metrics=['accuracy'])
-      model.fit_generator(
-          generator(),
-          steps_per_epoch=2,
-          epochs=1,
-          validation_data=generator(),
-          validation_steps=2,
-          callbacks=[TestCallback()],
-          verbose=0)
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/engine/training_generator.py b/tensorflow/python/keras/engine/training_generator.py
index d1699b2827..07de934259 100644
--- a/tensorflow/python/keras/engine/training_generator.py
+++ b/tensorflow/python/keras/engine/training_generator.py
@@ -430,12 +430,8 @@ def _make_enqueued_generator(generator,
 def _make_execution_function(model, mode, class_weight=None):
   """Makes function to run one step of model execution."""
   if mode == ModeKeys.TRAIN:
-    if not context.executing_eagerly():
-      model._make_fit_function()
     f = functools.partial(model.train_on_batch, class_weight=class_weight)
   elif mode == ModeKeys.TEST:
-    if not context.executing_eagerly():
-      model._make_eval_function()
     f = model.test_on_batch
   else:
     # Match signature of other modes to allow
diff --git a/tensorflow/python/keras/engine/training_generator_test.py b/tensorflow/python/keras/engine/training_generator_test.py
index 90c45dfcb7..6b754c18b3 100644
--- a/tensorflow/python/keras/engine/training_generator_test.py
+++ b/tensorflow/python/keras/engine/training_generator_test.py
@@ -66,8 +66,7 @@ class TestGeneratorMethods(keras_parameterized.TestCase):
   @unittest.skipIf(
       os.name == 'nt',
       'use_multiprocessing=True does not work on windows properly.')
-  # TODO(b/120940700): Bug with subclassed model inputs.
-  @keras_parameterized.run_with_all_model_types(exclude_models='subclass')
+  @keras_parameterized.run_with_all_model_types
   @keras_parameterized.run_all_keras_modes
   def test_fit_generator_method(self):
     model = testing_utils.get_small_mlp(
@@ -107,8 +106,7 @@ class TestGeneratorMethods(keras_parameterized.TestCase):
   @unittest.skipIf(
       os.name == 'nt',
       'use_multiprocessing=True does not work on windows properly.')
-  # TODO(b/120940700): Bug with subclassed model inputs.
-  @keras_parameterized.run_with_all_model_types(exclude_models='subclass')
+  @keras_parameterized.run_with_all_model_types
   @keras_parameterized.run_all_keras_modes
   def test_evaluate_generator_method(self):
     model = testing_utils.get_small_mlp(
@@ -173,8 +171,7 @@ class TestGeneratorMethods(keras_parameterized.TestCase):
                             max_queue_size=10,
                             workers=0)
 
-  # TODO(b/120940700): Bug with subclassed model inputs.
-  @keras_parameterized.run_with_all_model_types(exclude_models='subclass')
+  @keras_parameterized.run_with_all_model_types
   @keras_parameterized.run_all_keras_modes
   def test_generator_methods_with_sample_weights(self):
     model = testing_utils.get_small_mlp(
@@ -208,8 +205,7 @@ class TestGeneratorMethods(keras_parameterized.TestCase):
                              max_queue_size=10,
                              use_multiprocessing=False)
 
-  # TODO(b/120940700): Bug with subclassed model inputs.
-  @keras_parameterized.run_with_all_model_types(exclude_models='subclass')
+  @keras_parameterized.run_with_all_model_types
   @keras_parameterized.run_all_keras_modes
   def test_generator_methods_invalid_use_case(self):
 
@@ -249,8 +245,7 @@ class TestGeneratorMethods(keras_parameterized.TestCase):
                                max_queue_size=10,
                                use_multiprocessing=False)
 
-  # TODO(b/120940700): Bug with subclassed model inputs.
-  @keras_parameterized.run_with_all_model_types(exclude_models='subclass')
+  @keras_parameterized.run_with_all_model_types
   @keras_parameterized.run_all_keras_modes
   def test_generator_input_to_fit_eval_predict(self):
     val_data = np.ones([10, 10], np.float32), np.ones([10, 1], np.float32)
@@ -275,8 +270,7 @@ class TestGeneratorMethods(keras_parameterized.TestCase):
 
 class TestGeneratorMethodsWithSequences(keras_parameterized.TestCase):
 
-  # TODO(b/120940700): Bug with subclassed model inputs.
-  @keras_parameterized.run_with_all_model_types(exclude_models='subclass')
+  @keras_parameterized.run_with_all_model_types
   @keras_parameterized.run_all_keras_modes
   def test_training_with_sequences(self):
 
@@ -307,8 +301,7 @@ class TestGeneratorMethodsWithSequences(keras_parameterized.TestCase):
                         workers=0,
                         use_multiprocessing=False)
 
-  # TODO(b/120940700): Bug with subclassed model inputs.
-  @keras_parameterized.run_with_all_model_types(exclude_models='subclass')
+  @keras_parameterized.run_with_all_model_types
   @keras_parameterized.run_all_keras_modes
   def test_sequence_input_to_fit_eval_predict(self):
     val_data = np.ones([10, 10], np.float32), np.ones([10, 1], np.float32)
-- 
GitLab


From ac40a68cd5fac2227fb6c4086b2eb01a7dc726c4 Mon Sep 17 00:00:00 2001
From: Samantha Andow <samdow@fb.com>
Date: Wed, 26 Dec 2018 08:58:44 -0800
Subject: [PATCH 0467/2345] Fix iterable bug

Co-authored-by: melissagrueter <melissagrueter@fb.com>
Co-authored-by: irenedea <irenedea@fb.com>
---
 tensorflow/java/src/gen/cc/op_specs.cc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/java/src/gen/cc/op_specs.cc b/tensorflow/java/src/gen/cc/op_specs.cc
index 4f5a491d25..4024efedef 100644
--- a/tensorflow/java/src/gen/cc/op_specs.cc
+++ b/tensorflow/java/src/gen/cc/op_specs.cc
@@ -91,11 +91,6 @@ class TypeResolver {
 
 Type TypeResolver::TypeOf(const OpDef_ArgDef& arg_def, bool* iterable_out) {
   *iterable_out = false;
-  if (!arg_def.number_attr().empty()) {
-    // when number_attr is set, argument has to be a list of tensors
-    *iterable_out = true;
-    visited_attrs_.insert(std::make_pair(arg_def.number_attr(), Type::Int()));
-  }
   Type type = Type::Wildcard();
   if (arg_def.type() != DataType::DT_INVALID) {
     type = Type::ForDataType(arg_def.type());
@@ -122,6 +117,11 @@ Type TypeResolver::TypeOf(const OpDef_ArgDef& arg_def, bool* iterable_out) {
     LOG(FATAL) << "Cannot resolve data type of argument \"" << arg_def.name()
                << "\" in operation \"" << op_def_.name() << "\"";
   }
+  if (!arg_def.number_attr().empty()) {
+    // when number_attr is set, argument has to be a list of tensors
+    *iterable_out = true;
+    visited_attrs_.insert(std::make_pair(arg_def.number_attr(), Type::Int()));
+  }
   return type;
 }
 
-- 
GitLab


From fe13148bf6afcb85d71c05391d168302d8d4bf2d Mon Sep 17 00:00:00 2001
From: Frank Chen <frankchn@google.com>
Date: Wed, 9 Jan 2019 16:24:30 -0800
Subject: [PATCH 0468/2345] Change TPU Cluster Resolver doc string to clarify
 current / future features

PiperOrigin-RevId: 228610010
---
 .../cluster_resolver/tpu_cluster_resolver.py          | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
index 529a443412..72b9990701 100644
--- a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
+++ b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
@@ -192,11 +192,12 @@ class TPUClusterResolver(ClusterResolver):
     for the IP addresses and ports of each Cloud TPU listed.
 
     Args:
-      tpu: Either a string, or a list of strings corresponding to the TPUs to
-        use. If the single string is the empty string, the string 'local', or a
-        string that begins with 'grpc://' or '/bns', then it is assumed to not
-        correspond with a Cloud TPU and will instead be passed as the session
-        master and no ClusterSpec propagation will be done.
+      tpu: A string corresponding to the TPU to use. If the string is the empty
+        string, the string 'local', or a string that begins with 'grpc://' or
+        '/bns', then it is assumed to not correspond with a Cloud TPU and will
+        instead be passed as the session master and no ClusterSpec propagation
+        will be done. In the future, this may also support a list of strings
+        when multiple Cloud TPUs are used.
       zone: Zone where the TPUs are located. If omitted or empty, we will assume
         that the zone of the TPU is the same as the zone of the GCE VM, which we
         will try to discover from the GCE metadata service.
-- 
GitLab


From 4bb6da8e92da8d75456ee00b264e4c02877c5d7d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 16:35:23 -0800
Subject: [PATCH 0469/2345] Enable TPUEstimator to ignore non predict mode
 exporters rather than just hard failing.

PiperOrigin-RevId: 228611812
---
 tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 2317300041..2df5b3e4eb 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -2186,7 +2186,9 @@ class TPUEstimator(estimator_lib.Estimator):
       eval_on_tpu: If False, evaluation runs on CPU or GPU. In this case, the
         model_fn must return `EstimatorSpec` when called with `mode` as `EVAL`.
       export_to_tpu: If True, `export_savedmodel()` exports a metagraph for
-        serving on TPU besides the one on CPU.
+        serving on TPU besides the one on CPU. Note that unsupported export
+        modes such as EVAL will be ignored. For those modes, only a CPU model
+        will be exported. Currently, export_to_tpu only supports PREDICT.
       warm_start_from: Optional string filepath to a checkpoint or SavedModel to
         warm-start from, or a `tf.estimator.WarmStartSettings` object to fully
         configure warm-starting.  If the string filepath is provided instead of
@@ -2277,10 +2279,9 @@ class TPUEstimator(estimator_lib.Estimator):
                                export_tags=None,
                                check_variables=True):
     if self._export_to_tpu and mode != model_fn_lib.ModeKeys.PREDICT:
-      raise NotImplementedError(
-          'TPUEstimator only handles mode PREDICT for exporting '
-          'when `export_to_tpu` is `True`; '
-          'got {}.'.format(mode))
+      logging.warning('TPUEstimator only handles mode PREDICT for exporting '
+                      'when `export_to_tpu` is `True`; Mode {} will be ignored '
+                      'for TPU.'.format(mode))
 
     (super(TPUEstimator, self)._add_meta_graph_for_mode(
         builder,
@@ -2291,7 +2292,7 @@ class TPUEstimator(estimator_lib.Estimator):
         export_tags=export_tags,
         check_variables=check_variables))
 
-    if self._export_to_tpu:
+    if self._export_to_tpu and mode == model_fn_lib.ModeKeys.PREDICT:
       input_receiver_fn_map = {
           _REWRITE_FOR_INFERENCE_MODE: input_receiver_fn_map[mode]
       }
-- 
GitLab


From 1a768c9dc3895b3188b19d36a630f0e8ba5c85a0 Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Wed, 9 Jan 2019 16:46:17 -0800
Subject: [PATCH 0470/2345] Populate entry parameter's dynamic dimension when
 using XlaBuilder::SetDynamicBinding

Since now that we have is_dynamic_dimension field, we can set the field when calling SetDynamicBinding. ShapeInference will soon start populating this field based on the parameter shape.

PiperOrigin-RevId: 228613510
---
 tensorflow/compiler/xla/client/xla_builder.cc | 23 +++++++++++++++++++
 .../compiler/xla/client/xla_builder_test.cc   | 20 ++++++++++++++++
 2 files changed, 43 insertions(+)

diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc
index 7a04c0ec69..59e156eb72 100644
--- a/tensorflow/compiler/xla/client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_builder.cc
@@ -247,6 +247,29 @@ Status XlaBuilder::SetDynamicBinding(int64 dynamic_size_param_num,
                                      int64 target_param_num,
                                      ShapeIndex target_param_index,
                                      int64 target_dim_num) {
+  bool param_exists = false;
+  for (HloInstructionProto& instr : instructions_) {
+    if (instr.opcode() == HloOpcodeString(HloOpcode::kParameter) &&
+        instr.parameter_number() == target_param_num) {
+      param_exists = true;
+      Shape param_shape(instr.shape());
+      Shape* param_shape_ptr = &param_shape;
+      for (int64 index : target_param_index) {
+        param_shape_ptr = param_shape_ptr->mutable_tuple_shapes(index);
+      }
+      param_shape_ptr->set_dynamic_dimension(target_dim_num,
+                                             /*is_dynamic=*/true);
+      *instr.mutable_shape() = param_shape.ToProto();
+    }
+  }
+
+  if (!param_exists) {
+    return InvalidArgument(
+        "Asked to mark parameter %lld as dynamic sized parameter, but the "
+        "doesn't exists",
+        target_param_num);
+  }
+
   TF_RETURN_IF_ERROR(dynamic_parameter_binding_.Bind(
       DynamicParameterBinding::DynamicParameter{dynamic_size_param_num,
                                                 dynamic_size_param_index},
diff --git a/tensorflow/compiler/xla/client/xla_builder_test.cc b/tensorflow/compiler/xla/client/xla_builder_test.cc
index ba929a1200..abc11b4732 100644
--- a/tensorflow/compiler/xla/client/xla_builder_test.cc
+++ b/tensorflow/compiler/xla/client/xla_builder_test.cc
@@ -446,6 +446,26 @@ TEST_F(XlaBuilderTest, ProtoMatches) {
   EXPECT_EQ(c0_string, c1_string);
 }
 
+TEST_F(XlaBuilderTest, DynamicParameter) {
+  std::vector<XlaComputation> computations;
+  XlaBuilder b("builder");
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {5}), ShapeUtil::MakeShape(F32, {6})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  Parameter(&b, 1, ShapeUtil::MakeShape(U32, {}), "p1");
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/1,
+                                   /*dynamic_size_param_index=*/{},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{1},
+                                   /*target_dim_num=*/0));
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b, /*root=*/p0));
+  const Shape& param_shape = module->entry_computation()
+                                 ->parameter_instruction(0)
+                                 ->shape()
+                                 .tuple_shapes(1);
+  EXPECT_TRUE(param_shape.is_dynamic_dimension(0));
+}
+
 TEST_F(XlaBuilderTest, AfterAllWithNonTokenOperands) {
   XlaBuilder b(TestName());
   AfterAll(&b, {CreateToken(&b), ConstantR0<float>(&b, 1.0)});
-- 
GitLab


From 8e55a2f3fe768451b0e1ac8ae657f5ecfcdd0123 Mon Sep 17 00:00:00 2001
From: Dimitris Vardoulakis <dimvar@google.com>
Date: Wed, 9 Jan 2019 16:55:33 -0800
Subject: [PATCH 0471/2345] [TF/XLA] Fix bug in AR/CRS combiner. Don't recur to
 the arguments when comparing cross-module AllReduces for equality.

PiperOrigin-RevId: 228615009
---
 tensorflow/compiler/xla/service/ar_crs_combiner.cc   | 12 +++++++++---
 .../compiler/xla/service/ar_crs_combiner_test.cc     |  5 +++--
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/xla/service/ar_crs_combiner.cc b/tensorflow/compiler/xla/service/ar_crs_combiner.cc
index 4a227d3b5c..35a32f537b 100644
--- a/tensorflow/compiler/xla/service/ar_crs_combiner.cc
+++ b/tensorflow/compiler/xla/service/ar_crs_combiner.cc
@@ -190,6 +190,15 @@ bool ArCrsCombiner::InstructionsComputeSameValue(
   if (opcode1 != i2->opcode() || operands1.size() != i2->operands().size()) {
     return false;
   }
+  auto eq_computations = [](const HloComputation* a, const HloComputation* b) {
+    return *a == *b;
+  };
+  if (i1->IsCrossModuleAllReduce()) {
+    return i1->Identical(*i2,
+                         /*eq_operands=*/std::equal_to<const HloInstruction*>(),
+                         eq_computations,
+                         /*layout_sensitive=*/false);
+  }
   visited_pairs->emplace(min_uid, max_uid);
   for (int i = 0; i < operands1.size(); ++i) {
     auto operand1 = operands1[i];
@@ -215,9 +224,6 @@ bool ArCrsCombiner::InstructionsComputeSameValue(
   // InstructionsComputeSameValue earlier.
   auto eq_instructions = [](const HloInstruction* i1,
                             const HloInstruction* i2) -> bool { return true; };
-  auto eq_computations = [](const HloComputation* a, const HloComputation* b) {
-    return *a == *b;
-  };
   return i1->Identical(*i2, eq_instructions, eq_computations,
                        /*layout_sensitive=*/false);
 }
diff --git a/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc b/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc
index caa57296f4..b12b63b2dd 100644
--- a/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc
+++ b/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc
@@ -360,6 +360,7 @@ HloModule foobar
 
 ENTRY %entrycomp (p: bf16[]) -> (f32[], f32[]) {
   %p = bf16[] parameter(0)
+  %constant.bf16 = bf16[] constant(1)
 
   %all-reduce.ar.1 = bf16[]
       all-reduce(%p),
@@ -377,7 +378,7 @@ ENTRY %entrycomp (p: bf16[]) -> (f32[], f32[]) {
       sharding={maximal device=0}
 
   %all-reduce.ar.2 = bf16[]
-      all-reduce(%p),
+      all-reduce(%constant.bf16),
       replica_groups={{0},{1}},
       all_reduce_id=1,
       to_apply=%sum.bf16,
@@ -407,7 +408,7 @@ ENTRY %entrycomp (p: bf16[]) -> (f32[], f32[]) {
   EXPECT_TRUE(changed);
   EXPECT_THAT(module->entry_computation()->root_instruction(),
               op::Tuple(op::AllReduce(op::Convert(op::Parameter())),
-                        op::AllReduce(op::Convert(op::Parameter()))));
+                        op::AllReduce(op::Convert(op::Constant()))));
   auto crs_after =
       module->entry_computation()->root_instruction()->operands()[0];
   auto replica_groups_after = crs_after->replica_groups();
-- 
GitLab


From 7f9c7afd8829d20df25ee04f1999074290d7a45a Mon Sep 17 00:00:00 2001
From: Andiry Xu <andiryxu@google.com>
Date: Wed, 9 Jan 2019 16:58:27 -0800
Subject: [PATCH 0472/2345] Fix AllOutputValuesKnown logic.

PiperOrigin-RevId: 228615404
---
 .../core/grappler/costs/graph_properties.cc     | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/grappler/costs/graph_properties.cc b/tensorflow/core/grappler/costs/graph_properties.cc
index 863a5087ae..e869f7830c 100644
--- a/tensorflow/core/grappler/costs/graph_properties.cc
+++ b/tensorflow/core/grappler/costs/graph_properties.cc
@@ -1087,15 +1087,20 @@ class SymbolicShapeRefiner {
         c->output_tensor_protos.size() < ic->num_outputs()) {
       return false;
     } else {
+      // Checks if we can get output value via either output_tensor_proto or
+      // output_tensors_as_shapes.
       for (int i = 0; i < ic->num_outputs(); i++) {
-        if (c->output_tensor_protos.size() <= i ||
-            c->output_tensor_protos[i] == nullptr) {
-          return false;
+        if (c->output_tensor_protos.size() > i &&
+            c->output_tensor_protos[i] != nullptr) {
+          continue;
         }
-        if (c->output_tensors_as_shapes.size() <= i ||
-            !ic->FullyDefined(c->output_tensors_as_shapes[i])) {
-          return false;
+        if (c->output_tensors_as_shapes.size() > i &&
+            ic->FullyDefined(c->output_tensors_as_shapes[i])) {
+          continue;
         }
+
+        // Unknown for output[i].
+        return false;
       }
     }
     return true;
-- 
GitLab


From c2aa5390828b39863f9baaa2e5baad34788c7dec Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Wed, 9 Jan 2019 17:11:46 -0800
Subject: [PATCH 0473/2345] Update basic rnn test case.

PiperOrigin-RevId: 228617690
---
 tensorflow/lite/kernels/basic_rnn_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/kernels/basic_rnn_test.cc b/tensorflow/lite/kernels/basic_rnn_test.cc
index c71849fff3..9eb20444a6 100644
--- a/tensorflow/lite/kernels/basic_rnn_test.cc
+++ b/tensorflow/lite/kernels/basic_rnn_test.cc
@@ -240,7 +240,7 @@ class HybridRNNOpModel : public RNNOpModel {
 
   TensorType tensor_type_;
 
-  void SetWeights(int weights_idx, std::vector<float> f) {
+  void SetWeights(int weights_idx, const std::vector<float>& f) {
     if (tensor_type_ == TensorType_UINT8) {
       SymmetricQuantizeAndPopulate(weights_idx, f);
     } else {
-- 
GitLab


From 12f810aa76194935959cc0d414f7de6eeaa234b9 Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Wed, 9 Jan 2019 17:14:32 -0800
Subject: [PATCH 0474/2345] [XLA:GPU] Enhance unrolling heuristics for column
 reduction.

Previously, we enable unrolling only when the reduce operands are of small data
types. This change adds a simple analysis to count the number of tensors that
can be vectorized and can't be vectorized in order to decide whether unrolling
is beneficial for the kernel.

Add test cases.

PiperOrigin-RevId: 228618121
---
 .../xla/service/gpu/ir_emitter_unnested.cc    | 106 ++++++++++++--
 .../xla/service/gpu/ir_emitter_unnested.h     |   8 +-
 .../gpu/tests/gpu_kernel_tiling_test.cc       | 129 +++++++++++++++++-
 3 files changed, 228 insertions(+), 15 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index 2bc4912155..34ddeb1d41 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -3403,11 +3403,101 @@ std::tuple<int64, int64, int64> GetReductionToVectorDimensions(
   return std::make_tuple(num_reduced_major, num_kept, num_reduced_minor);
 }
 
+// Returns true if all the transitive users of hlo before hitting users in
+// use_chain_endings are elementwise operations.
+bool AreUsersElementwise(const HloInstruction* hlo,
+                         const ConstHloInstructionSet& use_chain_endings) {
+  return absl::c_all_of(hlo->users(), [&](const HloInstruction* user) {
+    return use_chain_endings.count(user) ||
+           (user->IsElementwise() &&
+            AreUsersElementwise(user, use_chain_endings));
+  });
+}
+
+// Returns the number of fusion inputs that have the same dimension as the
+// given shape, and involve in only elementwise operations.
+int64 NumInputsInvolveInOnlyElementwiseOps(
+    const HloInstruction* unnested_hlo, const Shape& op_shape,
+    const ConstHloInstructionSet& use_chain_endings) {
+  return absl::c_count_if(
+      unnested_hlo->fused_parameters(), [&](const HloInstruction* parameter) {
+        const Shape& parameter_shape = parameter->shape();
+        return ShapeUtil::SameDimensions(op_shape, parameter_shape) &&
+               AreUsersElementwise(parameter, use_chain_endings);
+      });
+}
+
+// Returns the number of fusion inputs that have more elements than the given
+// shape.
+int64 NumInputsWithMoreElementsThan(const HloInstruction* unnested_hlo,
+                                    const Shape& shape) {
+  int64 num_elements = ShapeUtil::ElementsIn(shape);
+  return absl::c_count_if(
+      unnested_hlo->fused_parameters(), [&](const HloInstruction* parameter) {
+        return ShapeUtil::ElementsIn(parameter->shape()) > num_elements;
+      });
+}
+
+// The benefit of unrolling a kInput fusion that is a column reduction comes
+// from the vectorization of non-reduction fusion outputs and fusion inputs.
+// On the other hand, unrolling can also introduce factors that can cause
+// the kernel to run slower. This routine uses a simple heuristic to estimate
+// the benefit as well as the overhead of unrolling in order to decide whether
+// unrolling is beneficial for the given kInput fusion.
+bool IsUnrollingColumnReductionBeneficial(const HloInstruction* unnested_hlo,
+                                          const Shape& input_shape,
+                                          int64 num_kept) {
+  // TODO(b/122468062): Need further investigate to see whether we can
+  // remove the constraint on IsPowerOfTwo.
+  if (!IsPowerOfTwo(static_cast<uint64>(num_kept))) {
+    return false;
+  }
+
+  if (unnested_hlo->opcode() == HloOpcode::kReduce) {
+    return true;
+  }
+
+  CHECK_EQ(unnested_hlo->opcode(), HloOpcode::kFusion);
+  int64 can_be_vectorized = 0;
+  int64 cannot_be_vectorized = 0;
+  const HloInstruction* fused_root = unnested_hlo->fused_expression_root();
+  ConstHloInstructionSet use_chain_endings;
+  if (fused_root->opcode() == HloOpcode::kReduce) {
+    use_chain_endings.insert(fused_root);
+    // Atomic.add of the reduction result can't be vectorized.
+    cannot_be_vectorized++;
+  } else {
+    CHECK_EQ(fused_root->opcode(), HloOpcode::kTuple);
+    for (const HloInstruction* instr : fused_root->operands()) {
+      if (instr->opcode() == HloOpcode::kReduce) {
+        // Atomic.add of the reduction result can't be vectorized.
+        cannot_be_vectorized++;
+      } else {
+        // Write of the non-reduction result can be vectorized.
+        can_be_vectorized++;
+      }
+      use_chain_endings.insert(instr);
+    }
+  }
+  // Fusion inputs that have the same dimension as the reduce input and
+  // only involve in elementwise operations can be vectorized.
+  can_be_vectorized += NumInputsInvolveInOnlyElementwiseOps(
+      unnested_hlo, input_shape, use_chain_endings);
+  // Fusion inputs with more elements than the reduce op input must participate
+  // in non-elementwise operations and we assume that they are not vectorizable
+  // for the purpose of estimating the benefit of unrolling. If the kernel is
+  // unrolled even with such an assumption,  and the accesses to those inputs
+  // turn out to be vectorizable, the compiler will still vectorize them.
+  cannot_be_vectorized +=
+      NumInputsWithMoreElementsThan(unnested_hlo, input_shape);
+  return can_be_vectorized >= cannot_be_vectorized;
+}
+
 }  // namespace
 
 std::tuple<KernelMappingScheme, bool>
 IrEmitterUnnested::ComputeMappingSchemeAndReductionKind(
-    const HloInstruction* first_reduce) {
+    const HloInstruction* unnested_hlo, const HloInstruction* first_reduce) {
   int64 depth = 1;
   int64 height = 1;
   int64 width = 1;
@@ -3437,15 +3527,6 @@ IrEmitterUnnested::ComputeMappingSchemeAndReductionKind(
     height = num_reduced_major;
     width = num_kept;
     is_row_reduction = false;
-    // Assume unrolling is beneficial only when we can vectorize the loads
-    // of small data types.
-    auto is_unrolling_beneficial = [&] {
-      // TODO(b/122468062): Need further investigate to see whether we can
-      // remove the constraint on IsPowerOfTwo.
-      return IsPowerOfTwo(static_cast<uint64>(num_kept)) &&
-             primitive_util::BitWidth(
-                 first_reduce->operand(0)->shape().element_type()) <= 16;
-    };
     // Column reduction without transpose doesn't require communication among
     // threads processing elements in the same tile. The current implementation
     // only support the use of one hardware thread block to process one block of
@@ -3454,7 +3535,8 @@ IrEmitterUnnested::ComputeMappingSchemeAndReductionKind(
     // num_threads_x and tile_size_x to allow a bigger hardware thread block.
     int64 hw_threads_per_block_limit =
         ThreadsPerBlockLimit(ir_emitter_context_->device_description());
-    if (is_unrolling_beneficial()) {
+    if (IsUnrollingColumnReductionBeneficial(unnested_hlo, input_shape,
+                                             num_kept)) {
       tile_size_x = std::min(2 * hw_threads_per_block_limit, num_kept);
       num_threads_x = tile_size_x / 2;
       dilated_x = false;
@@ -3539,7 +3621,7 @@ Status IrEmitterUnnested::EmitReductionToVector(HloInstruction* unnested_hlo) {
   bool is_row_reduction;
   llvm_ir::KernelMappingScheme mapping_scheme;
   std::tie(mapping_scheme, is_row_reduction) =
-      ComputeMappingSchemeAndReductionKind(first_reduce);
+      ComputeMappingSchemeAndReductionKind(unnested_hlo, first_reduce);
   ReductionCodegenInfo reduction_info(&mapping_scheme, is_row_reduction);
   KernelCodeGenerator kernel_generator(
       /*tile_element_generator=*/
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
index eb0ea90e14..21b842bb2c 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
@@ -217,9 +217,13 @@ class IrEmitterUnnested : public IrEmitter {
   Status EmitReductionToVector(HloInstruction* unnested_hlo);
 
   // Computes the KernelMappingScheme for the reduce HLO and indicates whether
-  // the reduction is a row reduction.
+  // the reduction is a row reduction. For an un-fused reduce op, unnested_hlo
+  // and first_reduce are the same instruction. For a kInput fusion,
+  // unnested_hlo is the fusion instruction while first_reduce is the first
+  // reduce op.
   std::tuple<llvm_ir::KernelMappingScheme, bool>
-  ComputeMappingSchemeAndReductionKind(const HloInstruction* first_reduce);
+  ComputeMappingSchemeAndReductionKind(const HloInstruction* unnested_hlo,
+                                       const HloInstruction* first_reduce);
 
   // Emits code for an in-place scatter, modifying `thunk`s launch dimensions in
   // the process. `scatter` may be fused, scatter indices are taken from
diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_kernel_tiling_test.cc b/tensorflow/compiler/xla/service/gpu/tests/gpu_kernel_tiling_test.cc
index 8004ebdc8e..869724db60 100644
--- a/tensorflow/compiler/xla/service/gpu/tests/gpu_kernel_tiling_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_kernel_tiling_test.cc
@@ -270,7 +270,7 @@ TEST_F(GpuKernelTilingTest, TransposedInputWithoutUnsafeUseTiled) {
         kind=kLoop, calls=fused_computation
     })";
 
-  // Check that a call to llvm.nvvm.barrier0 is not generated.
+  // Check that a call to llvm.nvvm.barrier0 is generated.
   auto hlo_module =
       ParseHloString(kHloString, ConfigWithoutLayoutAssignment()).ValueOrDie();
   CompileAndVerifyIr(std::move(hlo_module),
@@ -284,6 +284,133 @@ TEST_F(GpuKernelTilingTest, TransposedInputWithoutUnsafeUseTiled) {
   EXPECT_TRUE(RunAndCompareNoHloPasses(kHloString, ErrorSpec{0.0}));
 }
 
+TEST_F(GpuKernelTilingTest, ColumnReductionWithPowerOf2OutputElementsUnrolled) {
+  const char *const kHloString = R"(
+  HloModule column_reduce_powerof2
+
+  reduction {
+    x = f32[] parameter(0)
+    y = f32[] parameter(1)
+    ROOT add = f32[] add(x, y)
+  }
+
+  ENTRY kernel_entry {
+    constant0 = f32[] constant(0)
+    arg1 = f16[1024,512]{1,0} parameter(0)
+    arg1_conv = f32[1024,512]{1,0} convert(arg1)
+    ROOT reduce = f32[512]{0} reduce(arg1_conv, constant0), dimensions={0}, to_apply=reduction
+  })";
+
+  // Check that two calls to llvm.nvvm.atomic are generated.
+  auto hlo_module =
+      ParseHloString(kHloString, ConfigWithoutLayoutAssignment()).ValueOrDie();
+  CompileAndVerifyIr(std::move(hlo_module),
+                     R"(
+; CHECK-LABEL: define void @fusion
+; CHECK: call float @llvm.nvvm.atomic.load.add.f32.p0f32
+; CHECK: call float @llvm.nvvm.atomic.load.add.f32.p0f32
+; CHECK-NOT: call float @llvm.nvvm.atomic.load.add.f32.p0f32
+; CHECK: }
+)",
+                     /*match_optimized_ir=*/true);
+  // Check that the kernel runs correctly.
+  EXPECT_TRUE(RunAndCompareNoHloPasses(kHloString, ErrorSpec{1.0e-5, 1.0e-5}));
+}
+
+TEST_F(GpuKernelTilingTest,
+       ColumnReductionWithInputLargerThenReduceInputNotUnrolled) {
+  const char *const kHloString = R"(
+  HloModule larger_than_reduce_input_parameter
+
+  reduction22 {
+    x = f32[] parameter(0)
+    y = f32[] parameter(1)
+    ROOT add = f32[] add(x, y)
+  }
+
+  fused_computation {
+    constant0 = f32[] constant(0)
+    arg.1 = f16[1024,512]{1,0} parameter(0)
+    arg.2 = f16[1027,513]{1,0} parameter(1)
+    arg1.conv = f32[1024,512]{1,0} convert(arg.1)
+    arg2.conv = f32[1027,513]{1,0} convert(arg.2)
+    slice2 = f32[1024,512]{1,0} slice(arg2.conv), slice={[2:1026], [1:513]}
+    add2 = f32[1024,512]{1,0} add(arg1.conv, slice2)
+    ROOT reduce = f32[512]{0} reduce(add2, constant0), dimensions={0},
+      to_apply=reduction22
+  }
+
+  ENTRY kernel_entry {
+    arg1 = f16[1024,512]{1,0} parameter(0)
+    arg2 = f16[1027,513]{1,0} parameter(1)
+    ROOT fusion = f32[512]{0} fusion(arg1, arg2), kind=kInput,
+      calls=fused_computation
+  })";
+
+  // Check that one call to llvm.nvvm.atomic is generated.
+  auto hlo_module =
+      ParseHloString(kHloString, ConfigWithoutLayoutAssignment()).ValueOrDie();
+  CompileAndVerifyIr(std::move(hlo_module),
+                     R"(
+; CHECK-LABEL: define void @fusion
+; CHECK: call float @llvm.nvvm.atomic.load.add.f32.p0f32
+; CHECK-NOT: call float @llvm.nvvm.atomic.load.add.f32.p0f32
+; CHECK: }
+)",
+                     /*match_optimized_ir=*/true);
+  // Check that the kernel runs correctly.
+  EXPECT_TRUE(RunAndCompareNoHloPasses(kHloString, ErrorSpec{1.0e-5, 1.0e-5}));
+}
+
+TEST_F(GpuKernelTilingTest, ColumnReductionMOFUnrolled) {
+  const char *const kHloString = R"(
+  HloModule column_reduce_powerof2_mof
+
+  reduction22 {
+    x = f32[] parameter(0)
+    y = f32[] parameter(1)
+    ROOT add = f32[] add(x, y)
+  }
+
+  fused_computation {
+    constant0 = f32[] constant(0)
+    arg.1 = f16[1024,512]{1,0} parameter(0)
+    arg.2 = f16[1024,512]{1,0} parameter(1)
+    arg1.conv = f32[1024,512]{1,0} convert(arg.1)
+    arg2.conv = f32[1024,512]{1,0} convert(arg.2)
+    reduce1 = f32[512]{0} reduce(arg1.conv, constant0), dimensions={0},
+      to_apply=reduction22
+    reduce2 = f32[512]{0} reduce(arg2.conv, constant0), dimensions={0},
+      to_apply=reduction22
+    add = f32[1024,512]{1,0} add(arg1.conv, arg2.conv)
+    ROOT tuple = (f32[512]{0}, f32[512]{0}, f32[1024,512]{1,0})
+      tuple(reduce1, reduce2, add)
+  }
+
+  ENTRY kernel_entry {
+    arg1 = f16[1024,512]{1,0} parameter(0)
+    arg2 = f16[1024,512]{1,0} parameter(1)
+    ROOT fusion = (f32[512]{0}, f32[512]{0}, f32[1024,512]{1,0})
+      fusion(arg1, arg2), kind=kInput, calls=fused_computation
+  })";
+
+  // Check that four calls to llvm.nvvm.atomic are generated.
+  auto hlo_module =
+      ParseHloString(kHloString, ConfigWithoutLayoutAssignment()).ValueOrDie();
+  CompileAndVerifyIr(std::move(hlo_module),
+                     R"(
+; CHECK-LABEL: define void @fusion
+; CHECK: call float @llvm.nvvm.atomic.load.add.f32.p0f32
+; CHECK: call float @llvm.nvvm.atomic.load.add.f32.p0f32
+; CHECK: call float @llvm.nvvm.atomic.load.add.f32.p0f32
+; CHECK: call float @llvm.nvvm.atomic.load.add.f32.p0f32
+; CHECK-NOT: call float @llvm.nvvm.atomic.load.add.f32.p0f32
+; CHECK: }
+)",
+                     /*match_optimized_ir=*/true);
+  // Check that the kernel runs correctly.
+  EXPECT_TRUE(RunAndCompareNoHloPasses(kHloString, ErrorSpec{1.0e-5, 1.0e-5}));
+}
 }  // namespace
 }  // namespace gpu
 }  // namespace xla
-- 
GitLab


From 84bf8020f3a99dc49116417a6430c7a51e349e14 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Wed, 9 Jan 2019 17:25:33 -0800
Subject: [PATCH 0475/2345] Set device on Identity ops representing inlined
 function inputs/outputs.

This fixes a strange multi-GPU performance issue where the identities from inlined cond_v2 branch functions (part of the control flow lowering pass) were being placed alone on a GPU, creating unnecessary device traffic. Ideally the placer wouldn't be this stupid or grappler would remove the useless identities, but this fix is easier for now.

This also adds a simple benchmark testing nested function calls, which currently are inlined. This benchmark doesn't expose the original problem (all of the ops, including the identities added by inlining, are placed on a single device regardless), but I've included it in case it catches any future regressions. I also increased the number of iterations, as I found this made the results more stable.

PiperOrigin-RevId: 228619628
---
 tensorflow/core/common_runtime/function.cc |  4 ++++
 tensorflow/python/eager/benchmarks_test.py | 25 +++++++++++++++++++++-
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/common_runtime/function.cc b/tensorflow/core/common_runtime/function.cc
index 24a0eee0df..48f32df275 100644
--- a/tensorflow/core/common_runtime/function.cc
+++ b/tensorflow/core/common_runtime/function.cc
@@ -104,6 +104,10 @@ static Node* AddIdentity(Graph* g, Endpoint input) {
   NodeDef ndef;
   ndef.set_name(g->NewName(kNodeLabel));
   ndef.set_op("Identity");
+  // NOTE(skyewm): we explicitly set the device here to address a multi-GPU
+  // performance issue where this Identity would be placed alone on a GPU,
+  // causing unnecessary device traffic. See b/122483225 for details.
+  ndef.set_device(input.node->def().device());
   ndef.add_input(input.name());
   AddNodeAttr("T", BaseType(input.dtype()), &ndef);
   Status s;
diff --git a/tensorflow/python/eager/benchmarks_test.py b/tensorflow/python/eager/benchmarks_test.py
index 9191c8e689..62c4a12cbf 100644
--- a/tensorflow/python/eager/benchmarks_test.py
+++ b/tensorflow/python/eager/benchmarks_test.py
@@ -140,7 +140,7 @@ class MicroBenchmarks(test.Benchmark):
     self._m_2_by_2 = random_ops.random_uniform((2, 2))
     self._m_100_by_784 = random_ops.random_uniform((100, 784))
     self._num_iters_2_by_2 = 30000
-    self._num_iters_100_by_784 = 1000
+    self._num_iters_100_by_784 = 30000
 
   def _run(self, func, num_iters, execution_mode=None):
     # call func to maybe warm up the GPU
@@ -370,6 +370,19 @@ class MicroBenchmarks(test.Benchmark):
     func = lambda: f(m, m, transpose_b=transpose_b)
     self._run(func, num_iters, execution_mode=execution_mode)
 
+  def _benchmark_nested_defun_matmul(self, m, transpose_b, num_iters):
+    inner = function.defun(math_ops.matmul)
+
+    @function.defun
+    def outer(a, b, c, transpose_b):
+      return math_ops.matmul(inner(a, b, transpose_b=transpose_b), c)
+
+    func = lambda: outer(m, m, m, transpose_b=transpose_b)
+    # Warmup before benchmark
+    for _ in range(1000):
+      func()
+    self._run(func, num_iters)
+
   def _benchmark_defun_matmul_forward_backward(self,
                                                m,
                                                transpose_b,
@@ -525,6 +538,11 @@ class MicroBenchmarks(test.Benchmark):
           num_iters=self._num_iters_2_by_2,
           execution_mode=context.ASYNC)
 
+  def benchmark_nested_defun_matmul_2_by_2(self):
+    m = self._m_2_by_2.cpu()
+    self._benchmark_nested_defun_matmul(
+        m, transpose_b=False, num_iters=self._num_iters_2_by_2)
+
   # Benchmarks for AA.T, A of dimension 100 by 784.
   def benchmark_np_matmul_100_by_784(self):
     self._benchmark_np_matmul(
@@ -614,6 +632,11 @@ class MicroBenchmarks(test.Benchmark):
       self._benchmark_defun_matmul(
           m, transpose_b=True, num_iters=self._num_iters_100_by_784)
 
+  def benchmark_nested_defun_matmul_100_by_784(self):
+    m = self._m_100_by_784.gpu()
+    self._benchmark_nested_defun_matmul(
+        m, transpose_b=True, num_iters=self._num_iters_100_by_784)
+
   def benchmark_defun_without_signature(self):
 
     def func(t1, t2, t3, t4, t5, t6, t7, t8):
-- 
GitLab


From 0d53e22c582605d4f7f2a26b179a637a2d469206 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 17:30:56 -0800
Subject: [PATCH 0476/2345] Refactor pfor tests into multiple files.

PiperOrigin-RevId: 228620325
---
 tensorflow/python/ops/parallel_for/BUILD      |  47 +-
 .../python/ops/parallel_for/array_test.py     | 274 +++++++
 .../ops/parallel_for/control_flow_ops_test.py | 693 ++----------------
 .../python/ops/parallel_for/gradients_test.py |   2 +
 .../python/ops/parallel_for/math_test.py      | 405 ++++++++++
 .../python/ops/parallel_for/test_util.py      |  59 ++
 6 files changed, 830 insertions(+), 650 deletions(-)
 create mode 100644 tensorflow/python/ops/parallel_for/array_test.py
 create mode 100644 tensorflow/python/ops/parallel_for/math_test.py
 create mode 100644 tensorflow/python/ops/parallel_for/test_util.py

diff --git a/tensorflow/python/ops/parallel_for/BUILD b/tensorflow/python/ops/parallel_for/BUILD
index 07fc9433a2..1028963b1a 100644
--- a/tensorflow/python/ops/parallel_for/BUILD
+++ b/tensorflow/python/ops/parallel_for/BUILD
@@ -15,11 +15,13 @@ py_library(
         "control_flow_ops.py",
         "gradients.py",
         "pfor.py",
+        "test_util.py",
     ],
     srcs_version = "PY2AND3",
     deps = [
         ":control_flow_ops",
         ":gradients",
+        ":test_util",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:check_ops",
         "//tensorflow/python:constant_op",
@@ -83,12 +85,25 @@ py_library(
     ],
 )
 
+py_library(
+    name = "test_util",
+    srcs = ["test_util.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":pfor_lib",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:util",
+        "//tensorflow/python:variables",
+    ],
+)
+
 cuda_py_test(
     name = "control_flow_ops_test",
-    size = "large",
     srcs = ["control_flow_ops_test.py"],
     additional_deps = [
         ":control_flow_ops",
+        ":test_util",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:gradients",
@@ -101,6 +116,34 @@ cuda_py_test(
     ],
 )
 
+cuda_py_test(
+    name = "array_test",
+    srcs = ["array_test.py"],
+    additional_deps = [
+        ":control_flow_ops",
+        ":test_util",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:random_ops",
+        "//tensorflow/python:util",
+        "//tensorflow/python/eager:backprop",
+    ],
+)
+
+cuda_py_test(
+    name = "math_test",
+    srcs = ["math_test.py"],
+    additional_deps = [
+        ":control_flow_ops",
+        ":test_util",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:random_ops",
+        "//tensorflow/python:util",
+    ],
+    tags = ["optonly"],  # Too slow in non-opt mode
+)
+
 py_library(
     name = "gradients",
     srcs = ["gradients.py"],
@@ -115,7 +158,6 @@ py_library(
 
 cuda_py_test(
     name = "gradients_test",
-    size = "large",
     srcs = ["gradients_test.py"],
     additional_deps = [
         ":control_flow_ops",
@@ -128,4 +170,5 @@ cuda_py_test(
         "//tensorflow/python:random_ops",
         "//tensorflow/python/ops/losses",
     ],
+    tags = ["optonly"],  # Too slow in non-opt mode
 )
diff --git a/tensorflow/python/ops/parallel_for/array_test.py b/tensorflow/python/ops/parallel_for/array_test.py
new file mode 100644
index 0000000000..7f0c0f5b99
--- /dev/null
+++ b/tensorflow/python/ops/parallel_for/array_test.py
@@ -0,0 +1,274 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for vectorization of array kernels."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.eager import backprop
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import nn
+from tensorflow.python.ops import random_ops
+from tensorflow.python.ops import tensor_array_grad  # pylint: disable=unused-import
+from tensorflow.python.ops.parallel_for import control_flow_ops as pfor_control_flow_ops
+from tensorflow.python.ops.parallel_for.test_util import PForTestCase
+from tensorflow.python.platform import test
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class ArrayTest(PForTestCase):
+
+  def test_gather(self):
+    x = random_ops.random_uniform([3, 3, 3])
+
+    def loop_fn(i):
+      outputs = []
+      x_i = array_ops.gather(x, i)
+      for y in [x, x_i]:
+        axes = [0, 2, -1] if y == x else [0]
+        for axis in axes:
+          outputs.append(array_ops.gather(y, 2, axis=axis))
+          outputs.append(array_ops.gather(y, i, axis=axis))
+          outputs.append(array_ops.gather(y, [i], axis=axis))
+          outputs.append(array_ops.gather(y, [i, 2], axis=axis))
+          outputs.append(array_ops.gather(y, [[2, i], [i, 1]], axis=axis))
+      return outputs
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 20)
+
+  def test_shape(self):
+    x = random_ops.random_uniform([3, 2, 3])
+
+    def loop_fn(i):
+      x_i = array_ops.gather(x, i)
+      return array_ops.shape(x_i), array_ops.shape(x_i, out_type=dtypes.int64)
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.int32, dtypes.int64])
+
+  def test_size(self):
+    x = random_ops.random_uniform([3, 2, 3])
+
+    def loop_fn(i):
+      x_i = array_ops.gather(x, i)
+      return array_ops.size(x_i), array_ops.size(x_i, out_type=dtypes.int64)
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.int32, dtypes.int64])
+
+  def test_rank(self):
+    x = random_ops.random_uniform([3, 2, 3])
+
+    def loop_fn(i):
+      x_i = array_ops.gather(x, i)
+      return array_ops.rank(x_i)
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.int32])
+
+  def test_shape_n(self):
+    x = random_ops.random_uniform([3, 2, 3])
+    y = random_ops.random_uniform([3])
+
+    def loop_fn(i):
+      x_i = array_ops.gather(x, i)
+      y_i = array_ops.gather(y, i)
+      return array_ops.shape_n([x_i, x, y, y_i]), array_ops.shape_n(
+          [x_i, x, y, y_i], out_type=dtypes.int64)
+
+    self._test_loop_fn(
+        loop_fn, 3, loop_fn_dtypes=[dtypes.int32] * 4 + [dtypes.int64] * 4)
+
+  def test_reshape(self):
+    x = random_ops.random_uniform([3, 2, 3])
+
+    def loop_fn(i):
+      x1 = array_ops.gather(x, i)
+      return array_ops.reshape(x1, [-1]), array_ops.reshape(x1, [1, 3, 1, -1])
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
+
+  def test_expand_dims(self):
+    x = random_ops.random_uniform([3, 2, 3])
+
+    def loop_fn(i):
+      x1 = array_ops.gather(x, i)
+      return array_ops.expand_dims(
+          x1, axis=-1), array_ops.expand_dims(
+              x1, axis=1)
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
+
+  def test_slice(self):
+    x = random_ops.random_uniform([3, 2, 3])
+
+    def loop_fn(i):
+      x1 = array_ops.gather(x, i)
+      return array_ops.slice(x1, begin=(0, 1), size=(2, 1))
+
+    self._test_loop_fn(loop_fn, 3)
+
+  def test_tile(self):
+    x = random_ops.random_uniform([3, 2, 3])
+
+    def loop_fn(i):
+      x1 = array_ops.gather(x, i)
+      return array_ops.tile(x1, [2, 1])
+
+    self._test_loop_fn(loop_fn, 3)
+
+  def test_tile_loop_dependent(self):
+    x = random_ops.random_uniform([3, 2, 3])
+
+    def loop_fn(i):
+      x1 = array_ops.gather(x, i)
+      return array_ops.tile(x1, [i, 1])
+
+    with self.assertRaisesRegexp(ValueError, "expected to be loop invariant"):
+      pfor_control_flow_ops.pfor(loop_fn, 2)
+
+  def test_pack(self):
+    x = random_ops.random_uniform([3, 2, 3])
+    y = random_ops.random_uniform([2, 3])
+
+    def loop_fn(i):
+      x1 = array_ops.gather(x, i)
+      return array_ops.stack([x1, y], axis=-1)
+
+    self._test_loop_fn(loop_fn, 1)
+
+  def test_unpack(self):
+    x = random_ops.random_uniform([3, 2, 3, 4])
+
+    def loop_fn(i):
+      x_i = array_ops.gather(x, i)
+      return array_ops.unstack(
+          x_i, 4, axis=-1), array_ops.unstack(
+              x_i, 3, axis=1)
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 7)
+
+  def test_pad(self):
+    x = random_ops.random_uniform([3, 2, 3])
+    padding = constant_op.constant([[1, 2], [3, 4]])
+
+    def loop_fn(i):
+      x1 = array_ops.gather(x, i)
+      return array_ops.pad(x1, padding, mode="CONSTANT")
+
+    self._test_loop_fn(loop_fn, 3)
+
+  def test_split(self):
+    x = random_ops.random_uniform([3, 2, 3])
+
+    def loop_fn(i):
+      x1 = array_ops.gather(x, i)
+      return array_ops.split(x1, 2, axis=0), array_ops.split(x1, 3, axis=-1)
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 5)
+
+  def test_split_v(self):
+    x = random_ops.random_uniform([3, 6, 3])
+
+    def loop_fn(i):
+      x1 = array_ops.gather(x, i)
+      return (array_ops.split(x1, [2, 1, 3], axis=0),
+              array_ops.split(x1, [3], axis=-1))
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 4)
+
+  def test_transpose(self):
+    x = random_ops.random_uniform([3, 2, 3, 4])
+
+    def loop_fn(i):
+      x1 = array_ops.gather(x, i)
+      return array_ops.transpose(x1, [2, 1, 0])
+
+    self._test_loop_fn(loop_fn, 3)
+
+  def test_zeros_like(self):
+    x = random_ops.random_uniform([3, 2, 3])
+
+    def loop_fn(i):
+      x1 = array_ops.gather(x, i)
+      z = array_ops.zeros_like(x1),
+      return z, z + x1
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
+
+  def test_concat_v2(self):
+    x = random_ops.random_uniform([3, 2, 3])
+    y = random_ops.random_uniform([2, 3])
+
+    def loop_fn(i):
+      x1 = array_ops.gather(x, i)
+      return array_ops.concat(
+          [x1, x1, y], axis=0), array_ops.concat(
+              [x1, x1, y], axis=-1)
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
+
+  def test_unary_cwise_ops(self):
+    for op in [array_ops.identity, array_ops.stop_gradient]:
+      with backprop.GradientTape(persistent=True) as g:
+        x = random_ops.random_uniform([3, 5])
+        g.watch(x)
+
+      # pylint: disable=cell-var-from-loop
+      def loop_fn(i):
+        with g:
+          x1 = array_ops.gather(x, i)
+          y = op(x1) + x1
+          loss = nn.l2_loss(y)
+        return op(x), y, g.gradient(loss, x1)
+
+      # pylint: enable=cell-var-from-loop
+
+      self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 3)
+
+  def test_identity_n(self):
+    x = random_ops.random_uniform([3, 4])
+
+    def loop_fn(i):
+      return array_ops.identity_n([x, array_ops.gather(x, i)])
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
+
+  def test_matrix_diag_part(self):
+    x = random_ops.random_uniform([3, 4, 2])
+
+    def loop_fn(i):
+      return array_ops.matrix_diag_part(array_ops.gather(x, i))
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32])
+
+  def test_strided_slice(self):
+    with backprop.GradientTape(persistent=True) as g:
+      x = random_ops.random_uniform([3, 3, 4, 4, 2, 2, 2])
+      g.watch(x)
+
+    def loop_fn(i):
+      with g:
+        x_i = array_ops.gather(x, i)
+        y = x_i[:2, ::2, 1::3, ..., array_ops.newaxis, 1]
+        loss = nn.l2_loss(y)
+      return y, g.gradient(loss, x_i)
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
index 933bddd8cc..1a8f639d43 100644
--- a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
+++ b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
@@ -34,7 +34,6 @@ from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import bitwise_ops
-from tensorflow.python.ops import clip_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import data_flow_ops
 from tensorflow.python.ops import functional_ops
@@ -50,40 +49,13 @@ from tensorflow.python.ops import tensor_array_grad  # pylint: disable=unused-im
 from tensorflow.python.ops import tensor_array_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.ops.parallel_for import control_flow_ops as pfor_control_flow_ops
+from tensorflow.python.ops.parallel_for.test_util import PForTestCase
 from tensorflow.python.platform import test
 from tensorflow.python.util import nest
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class PForTest(test.TestCase):
-
-  def _run_targets(self, targets1, targets2=None, run_init=True):
-    targets1 = nest.flatten(targets1)
-    targets2 = ([] if targets2 is None else nest.flatten(targets2))
-    assert len(targets1) == len(targets2) or not targets2
-    if run_init:
-      init = variables.global_variables_initializer()
-      self.evaluate(init)
-    return self.evaluate(targets1 + targets2)
-
-  def run_and_assert_equal(self, targets1, targets2):
-    outputs = self._run_targets(targets1, targets2)
-    outputs = nest.flatten(outputs)  # flatten SparseTensorValues
-    n = len(outputs) // 2
-    for i in range(n):
-      if outputs[i + n].dtype != np.object:
-        self.assertAllClose(outputs[i + n], outputs[i], rtol=1e-4, atol=1e-5)
-      else:
-        self.assertAllEqual(outputs[i + n], outputs[i])
-
-  def _test_loop_fn(self, loop_fn, iters,
-                    loop_fn_dtypes=dtypes.float32,
-                    parallel_iterations=None):
-    t1 = pfor_control_flow_ops.pfor(loop_fn, iters=iters,
-                                    parallel_iterations=parallel_iterations)
-    t2 = pfor_control_flow_ops.for_loop(loop_fn, loop_fn_dtypes, iters=iters,
-                                        parallel_iterations=parallel_iterations)
-    self.run_and_assert_equal(t1, t2)
+class PForTest(PForTestCase):
 
   def test_op_conversion_fallback_to_while_loop(self):
     # Note that we used top_k op for this test. If a converter gets defined for
@@ -129,246 +101,7 @@ class PForTest(test.TestCase):
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class ArrayTest(PForTest):
-
-  def test_gather(self):
-    x = random_ops.random_uniform([3, 3, 3])
-
-    def loop_fn(i):
-      outputs = []
-      x_i = array_ops.gather(x, i)
-      for y in [x, x_i]:
-        axes = [0, 2, -1] if y == x else [0]
-        for axis in axes:
-          outputs.append(array_ops.gather(y, 2, axis=axis))
-          outputs.append(array_ops.gather(y, i, axis=axis))
-          outputs.append(array_ops.gather(y, [i], axis=axis))
-          outputs.append(array_ops.gather(y, [i, 2], axis=axis))
-          outputs.append(array_ops.gather(y, [[2, i], [i, 1]], axis=axis))
-      return outputs
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 20)
-
-  def test_shape(self):
-    x = random_ops.random_uniform([3, 2, 3])
-
-    def loop_fn(i):
-      x_i = array_ops.gather(x, i)
-      return array_ops.shape(x_i), array_ops.shape(x_i, out_type=dtypes.int64)
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.int32, dtypes.int64])
-
-  def test_size(self):
-    x = random_ops.random_uniform([3, 2, 3])
-
-    def loop_fn(i):
-      x_i = array_ops.gather(x, i)
-      return array_ops.size(x_i), array_ops.size(x_i, out_type=dtypes.int64)
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.int32, dtypes.int64])
-
-  def test_rank(self):
-    x = random_ops.random_uniform([3, 2, 3])
-
-    def loop_fn(i):
-      x_i = array_ops.gather(x, i)
-      return array_ops.rank(x_i)
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.int32])
-
-  def test_shape_n(self):
-    x = random_ops.random_uniform([3, 2, 3])
-    y = random_ops.random_uniform([3])
-
-    def loop_fn(i):
-      x_i = array_ops.gather(x, i)
-      y_i = array_ops.gather(y, i)
-      return array_ops.shape_n([x_i, x, y, y_i]), array_ops.shape_n(
-          [x_i, x, y, y_i], out_type=dtypes.int64)
-
-    self._test_loop_fn(
-        loop_fn, 3, loop_fn_dtypes=[dtypes.int32] * 4 + [dtypes.int64] * 4)
-
-  def test_reshape(self):
-    x = random_ops.random_uniform([3, 2, 3])
-
-    def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      return array_ops.reshape(x1, [-1]), array_ops.reshape(x1, [1, 3, 1, -1])
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
-
-  def test_expand_dims(self):
-    x = random_ops.random_uniform([3, 2, 3])
-
-    def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      return array_ops.expand_dims(
-          x1, axis=-1), array_ops.expand_dims(
-              x1, axis=1)
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
-
-  def test_slice(self):
-    x = random_ops.random_uniform([3, 2, 3])
-
-    def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      return array_ops.slice(x1, begin=(0, 1), size=(2, 1))
-
-    self._test_loop_fn(loop_fn, 3)
-
-  def test_tile(self):
-    x = random_ops.random_uniform([3, 2, 3])
-
-    def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      return array_ops.tile(x1, [2, 1])
-
-    self._test_loop_fn(loop_fn, 3)
-
-  def test_tile_loop_dependent(self):
-    x = random_ops.random_uniform([3, 2, 3])
-
-    def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      return array_ops.tile(x1, [i, 1])
-
-    with self.assertRaisesRegexp(ValueError, "expected to be loop invariant"):
-      pfor_control_flow_ops.pfor(loop_fn, 2)
-
-  def test_pack(self):
-    x = random_ops.random_uniform([3, 2, 3])
-    y = random_ops.random_uniform([2, 3])
-
-    def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      return array_ops.stack([x1, y], axis=-1)
-
-    self._test_loop_fn(loop_fn, 1)
-
-  def test_unpack(self):
-    x = random_ops.random_uniform([3, 2, 3, 4])
-
-    def loop_fn(i):
-      x_i = array_ops.gather(x, i)
-      return array_ops.unstack(
-          x_i, 4, axis=-1), array_ops.unstack(
-              x_i, 3, axis=1)
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 7)
-
-  def test_pad(self):
-    x = random_ops.random_uniform([3, 2, 3])
-    padding = constant_op.constant([[1, 2], [3, 4]])
-
-    def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      return array_ops.pad(x1, padding, mode="CONSTANT")
-
-    self._test_loop_fn(loop_fn, 3)
-
-  def test_split(self):
-    x = random_ops.random_uniform([3, 2, 3])
-
-    def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      return array_ops.split(x1, 2, axis=0), array_ops.split(x1, 3, axis=-1)
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 5)
-
-  def test_split_v(self):
-    x = random_ops.random_uniform([3, 6, 3])
-
-    def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      return (array_ops.split(x1, [2, 1, 3], axis=0),
-              array_ops.split(x1, [3], axis=-1))
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 4)
-
-  def test_transpose(self):
-    x = random_ops.random_uniform([3, 2, 3, 4])
-
-    def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      return array_ops.transpose(x1, [2, 1, 0])
-
-    self._test_loop_fn(loop_fn, 3)
-
-  def test_zeros_like(self):
-    x = random_ops.random_uniform([3, 2, 3])
-
-    def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      z = array_ops.zeros_like(x1),
-      return z, z + x1
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
-
-  def test_concat_v2(self):
-    x = random_ops.random_uniform([3, 2, 3])
-    y = random_ops.random_uniform([2, 3])
-
-    def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      return array_ops.concat(
-          [x1, x1, y], axis=0), array_ops.concat(
-              [x1, x1, y], axis=-1)
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
-
-  def test_unary_cwise_ops(self):
-    for op in [array_ops.identity, array_ops.stop_gradient]:
-      with backprop.GradientTape(persistent=True) as g:
-        x = random_ops.random_uniform([3, 5])
-        g.watch(x)
-
-      # pylint: disable=cell-var-from-loop
-      def loop_fn(i):
-        with g:
-          x1 = array_ops.gather(x, i)
-          y = op(x1) + x1
-          loss = nn.l2_loss(y)
-        return op(x), y, g.gradient(loss, x1)
-
-      # pylint: enable=cell-var-from-loop
-
-      self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 3)
-
-  def test_identity_n(self):
-    x = random_ops.random_uniform([3, 4])
-
-    def loop_fn(i):
-      return array_ops.identity_n([x, array_ops.gather(x, i)])
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
-
-  def test_matrix_diag_part(self):
-    x = random_ops.random_uniform([3, 4, 2])
-
-    def loop_fn(i):
-      return array_ops.matrix_diag_part(array_ops.gather(x, i))
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32])
-
-  def test_strided_slice(self):
-    with backprop.GradientTape(persistent=True) as g:
-      x = random_ops.random_uniform([3, 3, 4, 4, 2, 2, 2])
-      g.watch(x)
-
-    def loop_fn(i):
-      with g:
-        x_i = array_ops.gather(x, i)
-        y = x_i[:2, ::2, 1::3, ..., array_ops.newaxis, 1]
-        loss = nn.l2_loss(y)
-      return y, g.gradient(loss, x_i)
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
-
-
-@test_util.run_all_in_graph_and_eager_modes
-class BitwiseTest(PForTest):
+class BitwiseTest(PForTestCase):
 
   def test_unary_cwise(self):
     for op in [bitwise_ops.invert]:
@@ -408,376 +141,7 @@ class BitwiseTest(PForTest):
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class MathTest(PForTest):
-
-  def test_unary_cwise_ops(self):
-    complex_ops = [
-        math_ops.angle,
-        math_ops.imag,
-        math_ops.complex_abs,
-        math_ops.real,
-        math_ops.conj,
-    ]
-    real_ops = [
-        lambda x: math_ops.acosh(1 + math_ops.square(x)),
-        math_ops.abs,
-        math_ops.acos,
-        math_ops.asin,
-        math_ops.asinh,
-        math_ops.atan,
-        math_ops.atanh,
-        math_ops.bessel_i0e,
-        math_ops.bessel_i1e,
-        math_ops.cos,
-        math_ops.cosh,
-        math_ops.digamma,
-        math_ops.erf,
-        math_ops.erfc,
-        math_ops.exp,
-        math_ops.expm1,
-        math_ops.inv,
-        math_ops.is_finite,
-        math_ops.is_inf,
-        math_ops.lgamma,
-        math_ops.log,
-        math_ops.log1p,
-        math_ops.neg,
-        math_ops.negative,
-        math_ops.reciprocal,
-        math_ops.rint,
-        math_ops.round,
-        math_ops.rsqrt,
-        math_ops.sigmoid,
-        math_ops.sign,
-        math_ops.sin,
-        math_ops.sinh,
-        math_ops.sqrt,
-        math_ops.square,
-        math_ops.tan,
-        math_ops.tanh,
-        math_ops.tanh,
-        nn.elu,
-        nn.relu,
-        nn.relu6,
-        nn.selu,
-        nn.softplus,
-        nn.softsign,
-    ]
-    for op in complex_ops + real_ops:
-      with backprop.GradientTape(persistent=True) as g:
-        x = random_ops.random_uniform([3, 5])
-        g.watch(x)
-        if op in complex_ops:
-          y = random_ops.random_uniform([3, 5])
-          g.watch(y)
-          x = math_ops.complex(x, y)
-
-      # pylint: disable=cell-var-from-loop
-      output_dtypes = []
-      def loop_fn(i):
-        with g:
-          x1 = array_ops.gather(x, i)
-          y1 = op(x1)
-          outputs = [op(x), y1]
-          if y1.dtype == dtypes.float32:
-            loss = math_ops.reduce_sum(y1 * y1)
-          else:
-            loss = None
-        if loss is not None:
-          grad = g.gradient(loss, x1)
-          if grad is not None:
-            outputs.append(grad)
-        del output_dtypes[:]
-        output_dtypes.extend([t.dtype for t in outputs])
-        return outputs
-
-      # pylint: enable=cell-var-from-loop
-
-      self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=output_dtypes)
-
-  def test_unary_cwise_no_grad(self):
-    for op in [math_ops.ceil,
-               math_ops.floor,
-               math_ops.logical_not]:
-      x = random_ops.random_uniform([3, 5])
-      if op == math_ops.logical_not:
-        x = x > 0
-
-      # pylint: disable=cell-var-from-loop
-      def loop_fn(i):
-        return op(array_ops.gather(x, i))
-
-      # pylint: enable=cell-var-from-loop
-
-      self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=x.dtype)
-
-  def test_binary_cwise_ops(self):
-    logical_ops = [
-        math_ops.logical_and,
-        math_ops.logical_or,
-        math_ops.logical_xor
-    ]
-
-    # Wrapper functions restricting the range of inputs of zeta and polygamma.
-    def safe_polygamma(x, y):
-      return math_ops.polygamma(
-          math_ops.round(clip_ops.clip_by_value(y, 1, 10)),
-          x * x + 1)
-
-    def safe_zeta(x, y):
-      return math_ops.zeta(x * x + 1, y * y)
-
-    float_ops = [
-        math_ops.add,
-        math_ops.add_v2,
-        math_ops.atan2,
-        math_ops.complex,
-        math_ops.div,
-        math_ops.divide,
-        math_ops.div_no_nan,
-        math_ops.equal,
-        math_ops.floor_div,
-        math_ops.floor_mod,
-        math_ops.greater,
-        math_ops.greater_equal,
-        math_ops.igamma,
-        math_ops.igammac,
-        math_ops.igamma_grad_a,
-        math_ops.less,
-        math_ops.less_equal,
-        math_ops.maximum,
-        math_ops.minimum,
-        math_ops.mod,
-        math_ops.multiply,
-        math_ops.not_equal,
-        math_ops.pow,
-        math_ops.squared_difference,
-        math_ops.subtract,
-        math_ops.truncate_mod,
-        safe_polygamma,
-        safe_zeta,
-    ]
-    for op in logical_ops + float_ops:
-      x = random_ops.random_uniform([7, 3, 5])
-      y = random_ops.random_uniform([3, 5])
-      if op in logical_ops:
-        x = x > 0
-        y = y > 0
-
-      output_dtypes = []
-      # pylint: disable=cell-var-from-loop
-      def loop_fn(i):
-        x1 = array_ops.gather(x, i)
-        y1 = array_ops.gather(y, i)
-        outputs = [op(x, y), op(x1, y), op(x, y1), op(x1, y1), op(x1, x1)]
-        del output_dtypes[:]
-        output_dtypes.extend([t.dtype for t in outputs])
-        return outputs
-      # pylint: enable=cell-var-from-loop
-
-      self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=output_dtypes)
-
-  def test_approximate_equal(self):
-    x = random_ops.random_uniform([3, 5])
-    y = random_ops.random_uniform([3, 5])
-
-    def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      y1 = array_ops.gather(y, i)
-      return math_ops.approximate_equal(x1, y1)
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.bool])
-
-  def test_addn(self):
-    x = random_ops.random_uniform([2, 3, 5])
-    y = random_ops.random_uniform([3, 5])
-    z = random_ops.random_uniform([3, 5])
-
-    def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      return math_ops.add_n([x1, y, z])
-
-    self._test_loop_fn(loop_fn, 2)
-
-  def test_matmul(self):
-    for tr_a in (True, False):
-      for tr_b in (True, False):
-        for stack_a in (True, False):
-          for stack_b in (True, False):
-            shape_a = (5, 3) if tr_a else (3, 5)
-            if stack_a:
-              shape_a = (2,) + shape_a
-            shape_b = (7, 5) if tr_b else (5, 7)
-            if stack_b:
-              shape_b = (2,) + shape_b
-
-            x = random_ops.random_uniform(shape_a)
-            y = random_ops.random_uniform(shape_b)
-
-            # pylint: disable=cell-var-from-loop
-            def loop_fn(i):
-              a = array_ops.gather(x, i) if stack_a else x
-              b = array_ops.gather(y, i) if stack_b else y
-              return math_ops.matmul(a, b, transpose_a=tr_a, transpose_b=tr_b)
-
-            # pylint: enable=cell-var-from-loop
-
-            self._test_loop_fn(loop_fn, 2)
-
-  def test_batch_matmul(self):
-    for tr_a in (True, False):
-      for tr_b in (True, False):
-        for stack_a in (True, False):
-          for stack_b in (True, False):
-            shape_a = (4, 5, 3) if tr_a else (4, 3, 5)
-            if stack_a:
-              shape_a = (2,) + shape_a
-            shape_b = (4, 7, 5) if tr_b else (4, 5, 7)
-            if stack_b:
-              shape_b = (2,) + shape_b
-
-            x = random_ops.random_uniform(shape_a)
-            y = random_ops.random_uniform(shape_b)
-
-            # pylint: disable=cell-var-from-loop
-            def loop_fn(i):
-              a = array_ops.gather(x, i) if stack_a else x
-              b = array_ops.gather(y, i) if stack_b else y
-              return math_ops.matmul(a, b, transpose_a=tr_a, transpose_b=tr_b)
-
-            # pylint: enable=cell-var-from-loop
-
-            self._test_loop_fn(loop_fn, 2)
-
-  def test_reduction(self):
-    x = random_ops.random_uniform([2, 3, 4, 5])
-    for op in [
-        math_ops.reduce_sum, math_ops.reduce_prod, math_ops.reduce_max,
-        math_ops.reduce_min
-    ]:
-      for axis in ([1], None, [0, 2]):
-        for keepdims in (True, False):
-
-          # pylint: disable=cell-var-from-loop
-          def loop_fn(i):
-            a = array_ops.gather(x, i)
-            return op(a, axis=axis, keepdims=keepdims)
-
-          # pylint: enable=cell-var-from-loop
-
-          self._test_loop_fn(loop_fn, 2)
-
-  def test_cum_sum(self):
-    x = random_ops.random_uniform([2, 3, 4, 5])
-    for axis in (1, -2):
-      for exclusive in (True, False):
-        for reverse in (True, False):
-
-          # pylint: disable=cell-var-from-loop
-          def loop_fn(i):
-            a = array_ops.gather(x, i)
-            return math_ops.cumsum(
-                a, axis=axis, exclusive=exclusive, reverse=reverse)
-
-          # pylint: enable=cell-var-from-loop
-
-          self._test_loop_fn(loop_fn, 2)
-
-  def test_cum_prod(self):
-    x = random_ops.random_uniform([2, 3, 4, 5])
-    for axis in (1, -2):
-      for exclusive in (True, False):
-        for reverse in (True, False):
-
-          # pylint: disable=cell-var-from-loop
-          def loop_fn(i):
-            a = array_ops.gather(x, i)
-            return math_ops.cumprod(
-                a, axis=axis, exclusive=exclusive, reverse=reverse)
-
-          # pylint: enable=cell-var-from-loop
-
-          self._test_loop_fn(loop_fn, 2)
-
-  def test_bias_add(self):
-    x_shape = [2, 3, 4, 5, 6]
-    x = random_ops.random_uniform(x_shape)
-    for data_format in ("NCHW", "NHWC"):
-      with backprop.GradientTape(persistent=True) as g:
-        bias_dim = 2 if data_format == "NCHW" else -1
-        bias_shape = x_shape[bias_dim]
-        bias = random_ops.random_uniform([bias_shape])
-        g.watch(bias)
-
-      # pylint: disable=cell-var-from-loop
-      def loop_fn(i):
-        with g:
-          a = array_ops.gather(x, i)
-          y = nn.bias_add(a, bias, data_format=data_format)
-          loss = math_ops.reduce_sum(y * y)
-        return y, g.gradient(loss, bias)
-      # pylint: enable=cell-var-from-loop
-
-      self._test_loop_fn(
-          loop_fn, 2, loop_fn_dtypes=[dtypes.float32, dtypes.float32])
-
-  def test_unsorted_segment_sum(self):
-    t = random_ops.random_uniform([3, 3, 2])
-    segment_ids = constant_op.constant([[0, 0, 2], [0, 1, 2], [2, 2, 2]])
-    num_segments = 3
-
-    def loop_fn(i):
-      data = array_ops.gather(t, i)
-      data_0 = array_ops.gather(t, 0)
-      seg_ids = array_ops.gather(segment_ids, i)
-      return (math_ops.unsorted_segment_sum(data, seg_ids, num_segments),
-              math_ops.unsorted_segment_sum(data_0, seg_ids, num_segments))
-
-    self._test_loop_fn(loop_fn, 3, [dtypes.float32] * 2)
-
-  def test_cast(self):
-    x = constant_op.constant([[1], [2]])
-    y = constant_op.constant([[1.0], [2.0]])
-
-    def loop_fn(i):
-      return (math_ops.cast(array_ops.gather(x, i), dtypes.float32),
-              math_ops.cast(array_ops.gather(y, i), dtypes.int32))
-
-    self._test_loop_fn(
-        loop_fn, 2, loop_fn_dtypes=[dtypes.float32, dtypes.int32])
-
-  def test_tanh_axpy(self):
-    a = constant_op.constant(3.)
-    x = random_ops.random_uniform([4, 5])
-    y = random_ops.random_uniform([6, 5])
-    n = x.shape[0]
-
-    def loop_fn(i):
-      return math_ops.tanh(a * array_ops.gather(x, i) + array_ops.gather(y, i))
-
-    self._test_loop_fn(loop_fn, n)
-
-  def test_select(self):
-    cond = constant_op.constant([True, False])
-    a = random_ops.random_uniform([2, 3, 5])
-    b = random_ops.random_uniform([2, 3, 5])
-    for cond_shape in [2], [2, 3], [2, 3, 5]:
-      cond = random_ops.random_uniform(cond_shape) > 0.5
-
-      # pylint: disable=cell-var-from-loop
-      def loop_fn(i):
-        a_i = array_ops.gather(a, i)
-        b_i = array_ops.gather(b, i)
-        cond_i = array_ops.gather(cond, i)
-        return array_ops.where(cond_i, a_i, b_i)
-
-      # pylint: enable=cell-var-from-loop
-
-      self._test_loop_fn(loop_fn, 2)
-
-
-@test_util.run_all_in_graph_and_eager_modes
-class NNTest(PForTest):
+class NNTest(PForTestCase):
 
   def test_conv2d(self):
     x = random_ops.random_uniform([3, 2, 12, 12, 3])
@@ -956,7 +320,7 @@ class NNTest(PForTest):
     self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
 
 
-class RandomTest(PForTest):
+class RandomTest(PForTestCase):
 
   # The random values generated in the two implementations are not guaranteed to
   # match. So we only check the returned shapes.
@@ -1009,8 +373,9 @@ class RandomTest(PForTest):
     self._test_loop_fn(loop_fn, 5)
 
 
-class LoggingTest(PForTest):
+class LoggingTest(PForTestCase):
 
+  @test_util.run_v1_only("b/122612051")
   def test_print(self):
     x = random_ops.random_uniform([3, 5])
 
@@ -1031,8 +396,9 @@ class LoggingTest(PForTest):
       sess.run(pfor_control_flow_ops.pfor(loop_fn, 3))
 
 
-class TensorArrayTest(PForTest):
+class TensorArrayTest(PForTestCase):
 
+  @test_util.run_v1_only("b/122612051")
   def test_create_outside_and_read(self):
 
     ta = tensor_array_ops.TensorArray(
@@ -1043,6 +409,7 @@ class TensorArrayTest(PForTest):
 
     self._test_loop_fn(loop_fn, 2, [dtypes.int32] * 2)
 
+  @test_util.run_v1_only("b/122612051")
   def test_create_outside_and_gather(self):
 
     ta = tensor_array_ops.TensorArray(
@@ -1053,6 +420,7 @@ class TensorArrayTest(PForTest):
 
     self._test_loop_fn(loop_fn, 2, [dtypes.int32] * 2)
 
+  @test_util.run_v1_only("b/122612051")
   def test_create_outside_and_write_and_scatter(self):
 
     t = tensor_array_ops.TensorArray(dtypes.int32, 10, clear_after_read=False)
@@ -1074,6 +442,7 @@ class TensorArrayTest(PForTest):
     output2 = self._run_targets(out2)
     self.assertAllClose(output2, output1)
 
+  @test_util.run_v1_only("b/122612051")
   def test_create_inside_and_write(self):
 
     def loop_fn(i):
@@ -1085,6 +454,7 @@ class TensorArrayTest(PForTest):
 
     self._test_loop_fn(loop_fn, 3, [dtypes.int32] * 2)
 
+  @test_util.run_v1_only("b/122612051")
   def test_create_inside_and_scatter(self):
 
     def loop_fn(i):
@@ -1097,6 +467,7 @@ class TensorArrayTest(PForTest):
 
     self._test_loop_fn(loop_fn, 3, [dtypes.int32] * 2)
 
+  @test_util.run_v1_only("b/122612051")
   def test_create_inside_and_read(self):
 
     def loop_fn(i):
@@ -1109,6 +480,7 @@ class TensorArrayTest(PForTest):
 
     self._test_loop_fn(loop_fn, 2, [dtypes.int32] * 3)
 
+  @test_util.run_v1_only("b/122612051")
   def test_create_inside_and_gather(self):
 
     def loop_fn(i):
@@ -1121,6 +493,7 @@ class TensorArrayTest(PForTest):
 
     self._test_loop_fn(loop_fn, 2, [dtypes.int32] * 3)
 
+  @test_util.run_v1_only("b/122612051")
   def test_grad(self):
     x = random_ops.random_uniform([3, 2])
     ta = tensor_array_ops.TensorArray(
@@ -1140,8 +513,9 @@ class TensorArrayTest(PForTest):
       self.assertAllClose(actual_grad, computed_grad)
 
 
-class StackTest(PForTest):
+class StackTest(PForTestCase):
 
+  @test_util.run_v1_only("b/122612051")
   def test_stack_inside_loop_invariant(self):
 
     def loop_fn(_):
@@ -1157,6 +531,7 @@ class StackTest(PForTest):
 
     self._test_loop_fn(loop_fn, 2, [dtypes.int32] * 2)
 
+  @test_util.run_v1_only("b/122612051")
   def test_stack_inside_push_loop_dependent(self):
 
     def loop_fn(i):
@@ -1172,6 +547,7 @@ class StackTest(PForTest):
 
     self._test_loop_fn(loop_fn, 2, [dtypes.int32] * 2)
 
+  @test_util.run_v1_only("b/122612051")
   def test_stack_outside_pop(self):
     s = data_flow_ops.stack_v2(max_size=4, elem_type=dtypes.int32)
     op = data_flow_ops.stack_push_v2(s, 5)
@@ -1195,6 +571,7 @@ class StackTest(PForTest):
     self.assertAllEqual([6, 6], v2)
     self.assertAllEqual(5, v3)
 
+  @test_util.run_v1_only("b/122612051")
   def test_stack_outside_push(self):
     s = data_flow_ops.stack_v2(max_size=4, elem_type=dtypes.int32)
 
@@ -1207,7 +584,7 @@ class StackTest(PForTest):
 
 # TODO(agarwal): test nested while_loops. This currently requires converting a
 # tf.cond.
-class ControlFlowTest(PForTest):
+class ControlFlowTest(PForTestCase):
 
   def test_while_outside_loop(self):
 
@@ -1218,6 +595,7 @@ class ControlFlowTest(PForTest):
 
     self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.int32])
 
+  @test_util.run_v1_only("b/122612051")
   def test_invariant_while(self):
 
     def loop_fn(_):
@@ -1225,6 +603,7 @@ class ControlFlowTest(PForTest):
 
     self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.int32])
 
+  @test_util.run_v1_only("b/122612051")
   def test_invariant_while_with_control_dependency(self):
 
     def loop_fn(i):
@@ -1234,6 +613,7 @@ class ControlFlowTest(PForTest):
 
     self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.int32])
 
+  @test_util.run_v1_only("b/122612051")
   def test_while_with_stateful_ops(self):
 
     def loop_fn(_):
@@ -1243,6 +623,7 @@ class ControlFlowTest(PForTest):
 
     self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.int32])
 
+  @test_util.run_v1_only("b/122612051")
   def test_while_unstacked_condition(self):
 
     def loop_fn(i):
@@ -1251,6 +632,7 @@ class ControlFlowTest(PForTest):
 
     self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.int32, dtypes.int32])
 
+  @test_util.run_v1_only("b/122612051")
   def test_while(self):
     x = random_ops.random_uniform([3, 5])
     lengths = constant_op.constant([4, 0, 2])
@@ -1266,6 +648,7 @@ class ControlFlowTest(PForTest):
 
     self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32])
 
+  @test_util.run_v1_only("b/122612051")
   def test_while_jacobian(self):
     x = random_ops.random_uniform([1, 3])
     y = random_ops.random_uniform([3, 3])
@@ -1293,6 +676,7 @@ class ControlFlowTest(PForTest):
       out, expected = sess.run([out, expected_output])
       self.assertAllClose(expected, out)
 
+  @test_util.run_v1_only("b/122612051")
   def test_tensor_array_as_loop_variable(self):
 
     def loop_fn(i):
@@ -1308,6 +692,7 @@ class ControlFlowTest(PForTest):
 
     self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.int32])
 
+  @test_util.run_v1_only("b/122612051")
   def test_read_tensor_array_partitioned_indices(self):
     # Note that tensor array values are pfor loop dependent, and the while loop
     # termination condition is also dependent on pfor iteration.
@@ -1325,6 +710,7 @@ class ControlFlowTest(PForTest):
 
     self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.int32])
 
+  @test_util.run_v1_only("b/122612051")
   def test_external_while_loop_grad(self):
     # Here we test that external while_loops that are extended from inside pfor
     # (due to gradient calls) are not actually converted. If the below was
@@ -1350,6 +736,7 @@ class ControlFlowTest(PForTest):
       self.assertAllEqual([1, 1, 1],
                           sess.run(pfor_control_flow_ops.pfor(loop_fn, 3)))
 
+  @test_util.run_v1_only("b/122612051")
   def test_tensor_array_grad(self):
     inp = constant_op.constant(np.random.rand(3, 4, 2), dtype=dtypes.float32)
     ta = tensor_array_ops.TensorArray(dtypes.float32, size=3)
@@ -1447,13 +834,15 @@ def create_dynamic_lstm(cell_fn, batch_size, state_size, max_steps):
   return pfor_output, tf_output
 
 
-class RNNTest(PForTest):
+class RNNTest(PForTestCase):
 
+  @test_util.run_v1_only("b/122612051")
   def test_dynamic_rnn(self):
     pfor_outputs, tf_outputs = create_dynamic_lstm(rnn_cell.BasicRNNCell,
                                                    3, 5, 7)
     self.run_and_assert_equal(pfor_outputs, tf_outputs)
 
+  @test_util.run_v1_only("b/122612051")
   def test_dynamic_lstm(self):
     pfor_outputs, tf_outputs = create_dynamic_lstm(rnn_cell.BasicLSTMCell,
                                                    3, 5, 7)
@@ -1576,8 +965,9 @@ class Benchmarks(test.Benchmark):
       self._run(tf_outputs, 100, name="tf_rnn")
 
 
-class SparseTest(PForTest):
+class SparseTest(PForTestCase):
 
+  @test_util.run_v1_only("b/122612051")
   def test_var_loop_len(self):
     num_iters = array_ops.placeholder(dtypes.int32)
 
@@ -1589,6 +979,7 @@ class SparseTest(PForTest):
     with self.cached_session() as sess:
       sess.run(pfor, feed_dict={num_iters: 3})
 
+  @test_util.run_v1_only("b/122612051")
   def test_sparse_result_none_stacked(self):
     num_iters = 10
 
@@ -1605,6 +996,7 @@ class SparseTest(PForTest):
     manual = sparse_tensor.SparseTensor(indices, values, dense_shapes)
     self.run_and_assert_equal(pfor, manual)
 
+  @test_util.run_v1_only("b/122612051")
   def test_sparse_result_all_stacked(self):
     num_iters = 10
 
@@ -1620,6 +1012,7 @@ class SparseTest(PForTest):
                                         (num_iters, num_iters))
     self.run_and_assert_equal(pfor, manual)
 
+  @test_util.run_v1_only("b/122612051")
   def test_sparse_result_indices_stacked(self):
     num_iters = 10
 
@@ -1634,6 +1027,7 @@ class SparseTest(PForTest):
                                         [1] * num_iters, (num_iters, num_iters))
     self.run_and_assert_equal(pfor, manual)
 
+  @test_util.run_v1_only("b/122612051")
   def test_sparse_result_values_stacked(self):
     num_iters = 10
 
@@ -1648,6 +1042,7 @@ class SparseTest(PForTest):
                                         (num_iters, num_iters))
     self.run_and_assert_equal(pfor, manual)
 
+  @test_util.run_v1_only("b/122612051")
   def test_sparse_result_shapes_stacked(self):
     num_iters = 10
 
@@ -1661,6 +1056,7 @@ class SparseTest(PForTest):
                                         [1] * num_iters, (num_iters, num_iters))
     self.run_and_assert_equal(pfor, manual)
 
+  @test_util.run_v1_only("b/122612051")
   def test_sparse_result_shapes_stacked_2D(self):
     num_iters = 10
 
@@ -1677,7 +1073,7 @@ class SparseTest(PForTest):
     self.run_and_assert_equal(pfor, manual)
 
 
-class ParsingTest(PForTest):
+class ParsingTest(PForTestCase):
 
   def test_decode_csv(self):
     csv_tensor = constant_op.constant([["1:2:3"], ["::"], ["7:8:9"]])
@@ -1689,6 +1085,7 @@ class ParsingTest(PForTest):
 
     self._test_loop_fn(loop_fn, iters=3, loop_fn_dtypes=[dtypes.int32] * 3)
 
+  @test_util.run_v1_only("b/122612051")
   def test_parse_single_example(self):
 
     def _int64_feature(*values):
diff --git a/tensorflow/python/ops/parallel_for/gradients_test.py b/tensorflow/python/ops/parallel_for/gradients_test.py
index 4342833e3e..133e790992 100644
--- a/tensorflow/python/ops/parallel_for/gradients_test.py
+++ b/tensorflow/python/ops/parallel_for/gradients_test.py
@@ -29,6 +29,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.keras.engine import training as keras_training
 from tensorflow.python.layers import layers as tf_layers
 from tensorflow.python.ops import array_ops
@@ -338,6 +339,7 @@ def create_fc_per_eg_jacobians(batch_size, activation_size, num_layers):
   return jacobians, per_eg_jacobians_pfor, per_eg_jacobians_while
 
 
+@test_util.run_v1_only("b/122612051")
 class GradientsTest(test.TestCase):
 
   def run_and_assert_equal(self, targets1, targets2, atol=1e-4, rtol=1e-4):
diff --git a/tensorflow/python/ops/parallel_for/math_test.py b/tensorflow/python/ops/parallel_for/math_test.py
new file mode 100644
index 0000000000..db88f4fe03
--- /dev/null
+++ b/tensorflow/python/ops/parallel_for/math_test.py
@@ -0,0 +1,405 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for vectorization of math kernels."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.eager import backprop
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import clip_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn
+from tensorflow.python.ops import random_ops
+from tensorflow.python.ops import tensor_array_grad  # pylint: disable=unused-import
+from tensorflow.python.ops.parallel_for.test_util import PForTestCase
+from tensorflow.python.platform import test
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class MathTest(PForTestCase):
+
+  def test_unary_cwise_ops(self):
+    complex_ops = [
+        math_ops.angle,
+        math_ops.imag,
+        math_ops.complex_abs,
+        math_ops.real,
+        math_ops.conj,
+    ]
+    real_ops = [
+        lambda x: math_ops.acosh(1 + math_ops.square(x)),
+        math_ops.abs,
+        math_ops.acos,
+        math_ops.asin,
+        math_ops.asinh,
+        math_ops.atan,
+        math_ops.atanh,
+        math_ops.bessel_i0e,
+        math_ops.bessel_i1e,
+        math_ops.cos,
+        math_ops.cosh,
+        math_ops.digamma,
+        math_ops.erf,
+        math_ops.erfc,
+        math_ops.exp,
+        math_ops.expm1,
+        math_ops.inv,
+        math_ops.is_finite,
+        math_ops.is_inf,
+        math_ops.lgamma,
+        math_ops.log,
+        math_ops.log1p,
+        math_ops.neg,
+        math_ops.negative,
+        math_ops.reciprocal,
+        math_ops.rint,
+        math_ops.round,
+        math_ops.rsqrt,
+        math_ops.sigmoid,
+        math_ops.sign,
+        math_ops.sin,
+        math_ops.sinh,
+        math_ops.sqrt,
+        math_ops.square,
+        math_ops.tan,
+        math_ops.tanh,
+        math_ops.tanh,
+        nn.elu,
+        nn.relu,
+        nn.relu6,
+        nn.selu,
+        nn.softplus,
+        nn.softsign,
+    ]
+    for op in complex_ops + real_ops:
+      with backprop.GradientTape(persistent=True) as g:
+        x = random_ops.random_uniform([3, 5])
+        g.watch(x)
+        if op in complex_ops:
+          y = random_ops.random_uniform([3, 5])
+          g.watch(y)
+          x = math_ops.complex(x, y)
+
+      # pylint: disable=cell-var-from-loop
+      output_dtypes = []
+      def loop_fn(i):
+        with g:
+          x1 = array_ops.gather(x, i)
+          y1 = op(x1)
+          outputs = [op(x), y1]
+          if y1.dtype == dtypes.float32:
+            loss = math_ops.reduce_sum(y1 * y1)
+          else:
+            loss = None
+        if loss is not None:
+          grad = g.gradient(loss, x1)
+          if grad is not None:
+            outputs.append(grad)
+        del output_dtypes[:]
+        output_dtypes.extend([t.dtype for t in outputs])
+        return outputs
+
+      # pylint: enable=cell-var-from-loop
+
+      self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=output_dtypes)
+
+  def test_unary_cwise_no_grad(self):
+    for op in [math_ops.ceil,
+               math_ops.floor,
+               math_ops.logical_not]:
+      x = random_ops.random_uniform([3, 5])
+      if op == math_ops.logical_not:
+        x = x > 0
+
+      # pylint: disable=cell-var-from-loop
+      def loop_fn(i):
+        return op(array_ops.gather(x, i))
+
+      # pylint: enable=cell-var-from-loop
+
+      self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=x.dtype)
+
+  def test_binary_cwise_ops(self):
+    logical_ops = [
+        math_ops.logical_and,
+        math_ops.logical_or,
+        math_ops.logical_xor
+    ]
+
+    # Wrapper functions restricting the range of inputs of zeta and polygamma.
+    def safe_polygamma(x, y):
+      return math_ops.polygamma(
+          math_ops.round(clip_ops.clip_by_value(y, 1, 10)),
+          x * x + 1)
+
+    def safe_zeta(x, y):
+      return math_ops.zeta(x * x + 1, y * y)
+
+    float_ops = [
+        math_ops.add,
+        math_ops.add_v2,
+        math_ops.atan2,
+        math_ops.complex,
+        math_ops.div,
+        math_ops.divide,
+        math_ops.div_no_nan,
+        math_ops.equal,
+        math_ops.floor_div,
+        math_ops.floor_mod,
+        math_ops.greater,
+        math_ops.greater_equal,
+        math_ops.igamma,
+        math_ops.igammac,
+        math_ops.igamma_grad_a,
+        math_ops.less,
+        math_ops.less_equal,
+        math_ops.maximum,
+        math_ops.minimum,
+        math_ops.mod,
+        math_ops.multiply,
+        math_ops.not_equal,
+        math_ops.pow,
+        math_ops.squared_difference,
+        math_ops.subtract,
+        math_ops.truncate_mod,
+        safe_polygamma,
+        safe_zeta,
+    ]
+    for op in logical_ops + float_ops:
+      x = random_ops.random_uniform([7, 3, 5])
+      y = random_ops.random_uniform([3, 5])
+      if op in logical_ops:
+        x = x > 0
+        y = y > 0
+
+      output_dtypes = []
+      # pylint: disable=cell-var-from-loop
+      def loop_fn(i):
+        x1 = array_ops.gather(x, i)
+        y1 = array_ops.gather(y, i)
+        outputs = [op(x, y), op(x1, y), op(x, y1), op(x1, y1), op(x1, x1)]
+        del output_dtypes[:]
+        output_dtypes.extend([t.dtype for t in outputs])
+        return outputs
+      # pylint: enable=cell-var-from-loop
+
+      self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=output_dtypes)
+
+  def test_approximate_equal(self):
+    x = random_ops.random_uniform([3, 5])
+    y = random_ops.random_uniform([3, 5])
+
+    def loop_fn(i):
+      x1 = array_ops.gather(x, i)
+      y1 = array_ops.gather(y, i)
+      return math_ops.approximate_equal(x1, y1)
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.bool])
+
+  def test_addn(self):
+    x = random_ops.random_uniform([2, 3, 5])
+    y = random_ops.random_uniform([3, 5])
+    z = random_ops.random_uniform([3, 5])
+
+    def loop_fn(i):
+      x1 = array_ops.gather(x, i)
+      return math_ops.add_n([x1, y, z])
+
+    self._test_loop_fn(loop_fn, 2)
+
+  def test_matmul(self):
+    for tr_a in (True, False):
+      for tr_b in (True, False):
+        for stack_a in (True, False):
+          for stack_b in (True, False):
+            shape_a = (5, 3) if tr_a else (3, 5)
+            if stack_a:
+              shape_a = (2,) + shape_a
+            shape_b = (7, 5) if tr_b else (5, 7)
+            if stack_b:
+              shape_b = (2,) + shape_b
+
+            x = random_ops.random_uniform(shape_a)
+            y = random_ops.random_uniform(shape_b)
+
+            # pylint: disable=cell-var-from-loop
+            def loop_fn(i):
+              a = array_ops.gather(x, i) if stack_a else x
+              b = array_ops.gather(y, i) if stack_b else y
+              return math_ops.matmul(a, b, transpose_a=tr_a, transpose_b=tr_b)
+
+            # pylint: enable=cell-var-from-loop
+
+            self._test_loop_fn(loop_fn, 2)
+
+  def test_batch_matmul(self):
+    for tr_a in (True, False):
+      for tr_b in (True, False):
+        for stack_a in (True, False):
+          for stack_b in (True, False):
+            shape_a = (4, 5, 3) if tr_a else (4, 3, 5)
+            if stack_a:
+              shape_a = (2,) + shape_a
+            shape_b = (4, 7, 5) if tr_b else (4, 5, 7)
+            if stack_b:
+              shape_b = (2,) + shape_b
+
+            x = random_ops.random_uniform(shape_a)
+            y = random_ops.random_uniform(shape_b)
+
+            # pylint: disable=cell-var-from-loop
+            def loop_fn(i):
+              a = array_ops.gather(x, i) if stack_a else x
+              b = array_ops.gather(y, i) if stack_b else y
+              return math_ops.matmul(a, b, transpose_a=tr_a, transpose_b=tr_b)
+
+            # pylint: enable=cell-var-from-loop
+
+            self._test_loop_fn(loop_fn, 2)
+
+  def test_reduction(self):
+    x = random_ops.random_uniform([2, 3, 4, 5])
+    for op in [
+        math_ops.reduce_sum, math_ops.reduce_prod, math_ops.reduce_max,
+        math_ops.reduce_min
+    ]:
+      for axis in ([1], None, [0, 2]):
+        for keepdims in (True, False):
+
+          # pylint: disable=cell-var-from-loop
+          def loop_fn(i):
+            a = array_ops.gather(x, i)
+            return op(a, axis=axis, keepdims=keepdims)
+
+          # pylint: enable=cell-var-from-loop
+
+          self._test_loop_fn(loop_fn, 2)
+
+  def test_cum_sum(self):
+    x = random_ops.random_uniform([2, 3, 4, 5])
+    for axis in (1, -2):
+      for exclusive in (True, False):
+        for reverse in (True, False):
+
+          # pylint: disable=cell-var-from-loop
+          def loop_fn(i):
+            a = array_ops.gather(x, i)
+            return math_ops.cumsum(
+                a, axis=axis, exclusive=exclusive, reverse=reverse)
+
+          # pylint: enable=cell-var-from-loop
+
+          self._test_loop_fn(loop_fn, 2)
+
+  def test_cum_prod(self):
+    x = random_ops.random_uniform([2, 3, 4, 5])
+    for axis in (1, -2):
+      for exclusive in (True, False):
+        for reverse in (True, False):
+
+          # pylint: disable=cell-var-from-loop
+          def loop_fn(i):
+            a = array_ops.gather(x, i)
+            return math_ops.cumprod(
+                a, axis=axis, exclusive=exclusive, reverse=reverse)
+
+          # pylint: enable=cell-var-from-loop
+
+          self._test_loop_fn(loop_fn, 2)
+
+  def test_bias_add(self):
+    x_shape = [2, 3, 4, 5, 6]
+    x = random_ops.random_uniform(x_shape)
+    for data_format in ("NCHW", "NHWC"):
+      with backprop.GradientTape(persistent=True) as g:
+        bias_dim = 2 if data_format == "NCHW" else -1
+        bias_shape = x_shape[bias_dim]
+        bias = random_ops.random_uniform([bias_shape])
+        g.watch(bias)
+
+      # pylint: disable=cell-var-from-loop
+      def loop_fn(i):
+        with g:
+          a = array_ops.gather(x, i)
+          y = nn.bias_add(a, bias, data_format=data_format)
+          loss = math_ops.reduce_sum(y * y)
+        return y, g.gradient(loss, bias)
+      # pylint: enable=cell-var-from-loop
+
+      self._test_loop_fn(
+          loop_fn, 2, loop_fn_dtypes=[dtypes.float32, dtypes.float32])
+
+  def test_unsorted_segment_sum(self):
+    t = random_ops.random_uniform([3, 3, 2])
+    segment_ids = constant_op.constant([[0, 0, 2], [0, 1, 2], [2, 2, 2]])
+    num_segments = 3
+
+    def loop_fn(i):
+      data = array_ops.gather(t, i)
+      data_0 = array_ops.gather(t, 0)
+      seg_ids = array_ops.gather(segment_ids, i)
+      return (math_ops.unsorted_segment_sum(data, seg_ids, num_segments),
+              math_ops.unsorted_segment_sum(data_0, seg_ids, num_segments))
+
+    self._test_loop_fn(loop_fn, 3, [dtypes.float32] * 2)
+
+  def test_cast(self):
+    x = constant_op.constant([[1], [2]])
+    y = constant_op.constant([[1.0], [2.0]])
+
+    def loop_fn(i):
+      return (math_ops.cast(array_ops.gather(x, i), dtypes.float32),
+              math_ops.cast(array_ops.gather(y, i), dtypes.int32))
+
+    self._test_loop_fn(
+        loop_fn, 2, loop_fn_dtypes=[dtypes.float32, dtypes.int32])
+
+  def test_tanh_axpy(self):
+    a = constant_op.constant(3.)
+    x = random_ops.random_uniform([4, 5])
+    y = random_ops.random_uniform([6, 5])
+    n = x.shape[0]
+
+    def loop_fn(i):
+      return math_ops.tanh(a * array_ops.gather(x, i) + array_ops.gather(y, i))
+
+    self._test_loop_fn(loop_fn, n)
+
+  def test_select(self):
+    cond = constant_op.constant([True, False])
+    a = random_ops.random_uniform([2, 3, 5])
+    b = random_ops.random_uniform([2, 3, 5])
+    for cond_shape in [2], [2, 3], [2, 3, 5]:
+      cond = random_ops.random_uniform(cond_shape) > 0.5
+
+      # pylint: disable=cell-var-from-loop
+      def loop_fn(i):
+        a_i = array_ops.gather(a, i)
+        b_i = array_ops.gather(b, i)
+        cond_i = array_ops.gather(cond, i)
+        return array_ops.where(cond_i, a_i, b_i)
+
+      # pylint: enable=cell-var-from-loop
+
+      self._test_loop_fn(loop_fn, 2)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/ops/parallel_for/test_util.py b/tensorflow/python/ops/parallel_for/test_util.py
new file mode 100644
index 0000000000..7b4ef2239e
--- /dev/null
+++ b/tensorflow/python/ops/parallel_for/test_util.py
@@ -0,0 +1,59 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test utility."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.framework import dtypes
+from tensorflow.python.ops import variables
+from tensorflow.python.ops.parallel_for import control_flow_ops as pfor_control_flow_ops
+from tensorflow.python.platform import test
+from tensorflow.python.util import nest
+
+
+class PForTestCase(test.TestCase):
+  """Base class for test cases."""
+
+  def _run_targets(self, targets1, targets2=None, run_init=True):
+    targets1 = nest.flatten(targets1)
+    targets2 = ([] if targets2 is None else nest.flatten(targets2))
+    assert len(targets1) == len(targets2) or not targets2
+    if run_init:
+      init = variables.global_variables_initializer()
+      self.evaluate(init)
+    return self.evaluate(targets1 + targets2)
+
+  def run_and_assert_equal(self, targets1, targets2):
+    outputs = self._run_targets(targets1, targets2)
+    outputs = nest.flatten(outputs)  # flatten SparseTensorValues
+    n = len(outputs) // 2
+    for i in range(n):
+      if outputs[i + n].dtype != np.object:
+        self.assertAllClose(outputs[i + n], outputs[i], rtol=1e-4, atol=1e-5)
+      else:
+        self.assertAllEqual(outputs[i + n], outputs[i])
+
+  def _test_loop_fn(self, loop_fn, iters,
+                    loop_fn_dtypes=dtypes.float32,
+                    parallel_iterations=None):
+    t1 = pfor_control_flow_ops.pfor(loop_fn, iters=iters,
+                                    parallel_iterations=parallel_iterations)
+    t2 = pfor_control_flow_ops.for_loop(loop_fn, loop_fn_dtypes, iters=iters,
+                                        parallel_iterations=parallel_iterations)
+    self.run_and_assert_equal(t1, t2)
-- 
GitLab


From d852dec831a4a4c30c3018187e52e25010bf9711 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 17:42:18 -0800
Subject: [PATCH 0477/2345] Use %zu format specifier for size_t printf
 argument.

This fixes -Wformat error in 32-bit Android build.

PiperOrigin-RevId: 228621967
---
 tensorflow/core/lib/core/status.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/lib/core/status.cc b/tensorflow/core/lib/core/status.cc
index 3076c09337..0b63f66f6d 100644
--- a/tensorflow/core/lib/core/status.cc
+++ b/tensorflow/core/lib/core/status.cc
@@ -200,7 +200,7 @@ Status StatusGroup::as_status() const {
          const std::pair<error::Code, int>& b) { return a.second < b.second; });
 
   fmt.push_back(
-      strings::Printf("Combined status information from %lu operations:\n",
+      strings::Printf("Combined status information from %zu operations:\n",
                       num_ok_ + children_.size()));
 
   for (const auto& p : count_vec) {
-- 
GitLab


From 0b1b49fd5a39a5834d897dc61068e68f0649e313 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 17:45:02 -0800
Subject: [PATCH 0478/2345] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 228622263

---
 tensorflow/go/op/wrappers.go | 52 ++++++++++++++++++++++++++++++------
 1 file changed, 44 insertions(+), 8 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 208c15d75a..6f6fb793a0 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -7477,6 +7477,19 @@ func Conv2DBackpropFilterUseCudnnOnGpu(value bool) Conv2DBackpropFilterAttr {
 	}
 }
 
+// Conv2DBackpropFilterExplicitPaddings sets the optional explicit_paddings attribute to value.
+//
+// value: If `padding` is `"EXPLICIT"`, the list of explicit padding amounts. For the ith
+// dimension, the amount of padding inserted before and after the dimension is
+// `explicit_paddings[2 * i]` and `explicit_paddings[2 * i + 1]`, respectively. If
+// `padding` is not `"EXPLICIT"`, `explicit_paddings` must be empty.
+// If not specified, defaults to <>
+func Conv2DBackpropFilterExplicitPaddings(value []int64) Conv2DBackpropFilterAttr {
+	return func(m optionalAttr) {
+		m["explicit_paddings"] = value
+	}
+}
+
 // Conv2DBackpropFilterDataFormat sets the optional data_format attribute to value.
 //
 // value: Specify the data format of the input and output data. With the
@@ -8176,7 +8189,7 @@ func RegexReplaceReplaceGlobal(value bool) RegexReplaceAttr {
 // Arguments:
 //	input: The text to be processed.
 //	pattern: The regular expression to match the input.
-//	rewrite: The rewrite to be applied to the matched expresion.
+//	rewrite: The rewrite to be applied to the matched expression.
 //
 // Returns The text after applying pattern and rewrite.
 func RegexReplace(scope *Scope, input tf.Output, pattern tf.Output, rewrite tf.Output, optional ...RegexReplaceAttr) (output tf.Output) {
@@ -11605,15 +11618,12 @@ func NthElement(scope *Scope, input tf.Output, n tf.Output, optional ...NthEleme
 //
 // Arguments:
 //
-//	segment_ids: A tensor whose shape is a prefix of `data.shape`.END
-//   }
-//   out_arg {
-//     name: "output"
-//     description: <<END
-// Has same shape as data, except for the first `segment_ids.rank`
+//	segment_ids: A tensor whose shape is a prefix of `data.shape`.
+//
+//
+// Returns Has same shape as data, except for the first `segment_ids.rank`
 // dimensions, which are replaced with a single dimension which has size
 // `num_segments`.
-//
 func UnsortedSegmentMax(scope *Scope, data tf.Output, segment_ids tf.Output, num_segments tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
@@ -33501,6 +33511,19 @@ func Conv2DBackpropInputUseCudnnOnGpu(value bool) Conv2DBackpropInputAttr {
 	}
 }
 
+// Conv2DBackpropInputExplicitPaddings sets the optional explicit_paddings attribute to value.
+//
+// value: If `padding` is `"EXPLICIT"`, the list of explicit padding amounts. For the ith
+// dimension, the amount of padding inserted before and after the dimension is
+// `explicit_paddings[2 * i]` and `explicit_paddings[2 * i + 1]`, respectively. If
+// `padding` is not `"EXPLICIT"`, `explicit_paddings` must be empty.
+// If not specified, defaults to <>
+func Conv2DBackpropInputExplicitPaddings(value []int64) Conv2DBackpropInputAttr {
+	return func(m optionalAttr) {
+		m["explicit_paddings"] = value
+	}
+}
+
 // Conv2DBackpropInputDataFormat sets the optional data_format attribute to value.
 //
 // value: Specify the data format of the input and output data. With the
@@ -35308,6 +35331,19 @@ func Conv2DUseCudnnOnGpu(value bool) Conv2DAttr {
 	}
 }
 
+// Conv2DExplicitPaddings sets the optional explicit_paddings attribute to value.
+//
+// value: If `padding` is `"EXPLICIT"`, the list of explicit padding amounts. For the ith
+// dimension, the amount of padding inserted before and after the dimension is
+// `explicit_paddings[2 * i]` and `explicit_paddings[2 * i + 1]`, respectively. If
+// `padding` is not `"EXPLICIT"`, `explicit_paddings` must be empty.
+// If not specified, defaults to <>
+func Conv2DExplicitPaddings(value []int64) Conv2DAttr {
+	return func(m optionalAttr) {
+		m["explicit_paddings"] = value
+	}
+}
+
 // Conv2DDataFormat sets the optional data_format attribute to value.
 //
 // value: Specify the data format of the input and output data. With the
-- 
GitLab


From 767a1fe74600fadee070fbaa88cbb5f5917170bd Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Wed, 9 Jan 2019 18:31:55 -0800
Subject: [PATCH 0479/2345] Delete unused
 ServiceExecutableRunOptions::xla_intra_op_thread_pool; NFC

PiperOrigin-RevId: 228627868
---
 tensorflow/compiler/xla/client/local_client.cc     |  5 ++---
 tensorflow/compiler/xla/service/hlo_runner.cc      |  4 +---
 tensorflow/compiler/xla/service/service.cc         |  4 +---
 .../xla/service/service_executable_run_options.h   | 14 +++-----------
 .../compiler/xla/tests/xla_hlo_profile_test.cc     |  5 ++---
 5 files changed, 9 insertions(+), 23 deletions(-)

diff --git a/tensorflow/compiler/xla/client/local_client.cc b/tensorflow/compiler/xla/client/local_client.cc
index 049cd15738..48b5f94538 100644
--- a/tensorflow/compiler/xla/client/local_client.cc
+++ b/tensorflow/compiler/xla/client/local_client.cc
@@ -164,9 +164,8 @@ StatusOr<ScopedShapedBuffer> LocalExecutable::Run(
   //    ExecutableRunOptions.eigen_intra_op_thread_pool.
   // *) The thread pool used for XLA CPU ops is from
   //    backend_->eigen_intra_op_thread_pool().
-  ServiceExecutableRunOptions service_options(
-      run_options, backend_->StreamBorrower(),
-      backend_->eigen_intra_op_thread_pool());
+  ServiceExecutableRunOptions service_options(run_options,
+                                              backend_->StreamBorrower());
 
   if (executable_->dumping_snapshot()) {
     return ExecuteAndDump(&service_options, arguments);
diff --git a/tensorflow/compiler/xla/service/hlo_runner.cc b/tensorflow/compiler/xla/service/hlo_runner.cc
index 5a9b820a9d..d7d66ae1c4 100644
--- a/tensorflow/compiler/xla/service/hlo_runner.cc
+++ b/tensorflow/compiler/xla/service/hlo_runner.cc
@@ -383,9 +383,7 @@ ServiceExecutableRunOptions HloRunner::GetServiceRunOptionsForDevice(
   if (device_assignment != nullptr) {
     run_options.set_device_assignment(device_assignment);
   }
-  return ServiceExecutableRunOptions(
-      run_options, backend().StreamBorrower(),
-      /*xla_intra_op_thread_pool=*/backend().eigen_intra_op_thread_pool());
+  return ServiceExecutableRunOptions(run_options, backend().StreamBorrower());
 }
 
 Backend& HloRunner::backend() {
diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc
index 4a8d272261..2732d498d8 100644
--- a/tensorflow/compiler/xla/service/service.cc
+++ b/tensorflow/compiler/xla/service/service.cc
@@ -553,9 +553,7 @@ StatusOr<GlobalDataHandle> Service::ExecuteAndRegisterResult(
     options.set_intra_op_thread_pool(
         backend->eigen_intra_op_thread_pool_device());
     options.set_device_assignment(&device_assignment);
-    run_options.emplace_back(
-        options, backend->StreamBorrower(),
-        /*xla_intra_op_thread_pool=*/backend->eigen_intra_op_thread_pool());
+    run_options.emplace_back(options, backend->StreamBorrower());
   }
 
   if (options_.number_of_replicas() == 1) {
diff --git a/tensorflow/compiler/xla/service/service_executable_run_options.h b/tensorflow/compiler/xla/service/service_executable_run_options.h
index dbfed628bf..6bee671056 100644
--- a/tensorflow/compiler/xla/service/service_executable_run_options.h
+++ b/tensorflow/compiler/xla/service/service_executable_run_options.h
@@ -32,12 +32,10 @@ class ServiceExecutableRunOptions {
   ServiceExecutableRunOptions()
       : ServiceExecutableRunOptions(ExecutableRunOptions()) {}
 
-  explicit ServiceExecutableRunOptions(
-      ExecutableRunOptions run_options, StreamBorrower borrow_stream = nullptr,
-      tensorflow::thread::ThreadPool* xla_intra_op_thread_pool = nullptr)
+  explicit ServiceExecutableRunOptions(ExecutableRunOptions run_options,
+                                       StreamBorrower borrow_stream = nullptr)
       : run_options_(std::move(run_options)),
-        borrow_stream_(std::move(borrow_stream)),
-        xla_intra_op_thread_pool_(xla_intra_op_thread_pool) {}
+        borrow_stream_(std::move(borrow_stream)) {}
 
   // Returns reference or pointer to `ExecutableRunOptions` member.
   const ExecutableRunOptions& run_options() const { return run_options_; }
@@ -56,15 +54,9 @@ class ServiceExecutableRunOptions {
                : Status(tensorflow::error::UNIMPLEMENTED, "No stream cache");
   }
 
-  // Returns reference to thread pool for execution of XLA ops on CPU backend.
-  tensorflow::thread::ThreadPool* xla_intra_op_thread_pool() const {
-    return xla_intra_op_thread_pool_;
-  }
-
  private:
   ExecutableRunOptions run_options_;
   StreamBorrower borrow_stream_;
-  tensorflow::thread::ThreadPool* xla_intra_op_thread_pool_;
 };
 
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc b/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc
index 1538f2afba..c7337e8caa 100644
--- a/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc
+++ b/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc
@@ -174,9 +174,8 @@ void ExecuteAndFetchProfile(string* profile_output, LocalClient* client,
   exec_run_options.set_allocator(backend->memory_allocator());
   exec_run_options.set_intra_op_thread_pool(
       backend->eigen_intra_op_thread_pool_device());
-  ServiceExecutableRunOptions run_options(
-      exec_run_options, /*borrow_stream=*/nullptr,
-      backend->eigen_intra_op_thread_pool());
+  ServiceExecutableRunOptions run_options(exec_run_options,
+                                          /*borrow_stream=*/nullptr);
   std::vector<const ShapedBuffer*> args = {&lhs_arg, &rhs_arg};
   TF_ASSERT_OK_AND_ASSIGN(
       auto execution_result,
-- 
GitLab


From 07b238033996f813381f1e1a0410c877c13c6c94 Mon Sep 17 00:00:00 2001
From: Aurelien Geron <aurelien.geron@gmail.com>
Date: Thu, 10 Jan 2019 12:41:40 +0800
Subject: [PATCH 0480/2345] Rename Function to ConcreteFunction

---
 .../python/mirrored_strategy_multigpu_test.py |  2 +-
 tensorflow/python/eager/def_function.py       |  9 ++--
 tensorflow/python/eager/function.py           | 50 ++++++++++---------
 tensorflow/python/eager/wrap_function.py      |  2 +-
 tensorflow/python/keras/backend.py            |  2 +-
 .../saved_model/function_deserialization.py   | 12 ++---
 tensorflow/python/saved_model/save.py         |  2 +-
 7 files changed, 42 insertions(+), 37 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index 59d711ae01..2925b8c88c 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -1266,7 +1266,7 @@ class MirroredStrategyDefunTest(test.TestCase):
                             self.evaluate(device_result))
 
       for defun in defuns:
-        # PolymorphicFunctions are specialized to the current device stack, so
+        # `PolymorphicFunction`s are specialized to the current device stack, so
         # call_for_each has one trace per device. To check that the expected set
         # of variables was accessed on each trace, we first retrieve each
         # device-specific graph function.
diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py
index 4c22a12c1c..b3f361b92d 100644
--- a/tensorflow/python/eager/def_function.py
+++ b/tensorflow/python/eager/def_function.py
@@ -408,7 +408,8 @@ class PolymorphicFunction(object):
     return self._function_spec
 
   def get_initialization_function(self, *args, **kwargs):
-    """Returns a `Function` object which initializes this function's variables.
+    """Returns a `ConcreteFunction` object which initializes this function's
+    variables.
 
     Requires that this function hasn't been accessed yet through either calling
     it or calling get_concrete_function. Fails if we cannot build an initializer
@@ -420,7 +421,8 @@ class PolymorphicFunction(object):
       **kwargs: keyword arguments to the python callable.
 
     Returns:
-      A `Function` object which initializes the variables of this function.
+      A `ConcreteFunction` object which initializes the variables of this
+      function.
 
     Raises:
       RuntimeError: if called after the variables have been initialized.
@@ -468,7 +470,8 @@ class PolymorphicFunction(object):
     # pylint: enable=protected-access
 
   def get_concrete_function(self, *args, **kwargs):
-    """Returns a `Function` object specialized to inputs and execution context.
+    """Returns a `ConcreteFunction` object specialized to inputs and execution
+    context.
 
     If this `PolymorphicFunction` was created with an `input_signature`, `args`
     and `kwargs` may be omitted. With an input signature there is only one
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 83cd140158..3bd7df9502 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -271,8 +271,8 @@ class _EagerDefinedFunction(object):
   def call(self, ctx, args):
     """Calls this function with `args` as inputs.
 
-    Function execution respects device annotations only if the function won't
-    be compiled with xla.
+    `ConcreteFunction` execution respects device annotations only if the
+    function won't be compiled with xla.
 
     Args:
       ctx: a Context object
@@ -340,15 +340,15 @@ class _EagerDefinedFunction(object):
       return outputs
 
 
-class Function(object):
+class ConcreteFunction(object):
   """Callable object encapsulating a function definition and its gradient.
 
-  `Function` is a callable that encapsulates a function definition and
+  `ConcreteFunction` is a callable that encapsulates a function definition and
   is differentiable under `tf.GradientTape` objects.
   """
 
   def __init__(self, func_graph, attrs=None, signature=None):
-    """Initialize a Function.
+    """Initialize a `ConcreteFunction`.
 
     Args:
       func_graph: An instance of FuncGraph: the function body to wrap.
@@ -384,8 +384,8 @@ class Function(object):
       *args: Tensors or Variables. Positional arguments are only accepted when
         they correspond one-to-one with arguments of the traced Python function.
       **kwargs: Tensors or Variables specified by name. When
-        `get_concrete_function` was called to create this `Function`, each
-        Tensor input was given a name, defaulting to the name of the Python
+        `get_concrete_function` was called to create this `ConcreteFunction`,
+        each Tensor input was given a name, defaulting to the name of the Python
         function's argument but possibly overridden by the `name=` argument to
         `tf.TensorSpec`. These names become the argument names for the concrete
         function.
@@ -394,7 +394,7 @@ class Function(object):
       The result of applying the TF function on the given Tensors.
 
     Raises:
-      AssertionError: If this `Function` was not created through
+      AssertionError: If this `ConcreteFunction` was not created through
         `get_concrete_function`.
       ValueError: If arguments contains anything other than Tensors or
         Variables.
@@ -478,7 +478,7 @@ class Function(object):
         tensor_inputs.append(
             ops.convert_to_tensor(arg, self._signature[i].dtype))
       else:
-        raise ValueError("All inputs to `Function`s must be Tensors; "
+        raise ValueError("All inputs to `ConcreteFunction`s must be Tensors; "
                          "on invocation of %s, the %d-th input (%s) was not a "
                          "Tensor." % (self._func_graph.name, i, str(arg)))
     args = tensor_inputs + self._captured_inputs
@@ -504,7 +504,8 @@ class Function(object):
     return self._build_call_outputs(outputs)
 
   def _register_gradient(self, name):
-    """Registers the gradient for the current Function under the given name.
+    """Registers the gradient for the current `ConcreteFunction` under the given
+    name.
 
     The gradient rewrites an inference call op to a forward call op, but does
     not modify a pre-existing forward call op. It then computes the gradient
@@ -545,7 +546,7 @@ class Function(object):
 
   @property
   def name(self):
-    """Function name."""
+    """`ConcreteFunction` name."""
     return self._inference_function.name
 
   @property
@@ -672,7 +673,7 @@ class Function(object):
         grad for grad in func_graph_module.flatten(gradients_wrt_inputs)
         if grad is not None)
     backwards_graph.structured_outputs = gradients_wrt_inputs
-    self._backward_graph_function = Function(
+    self._backward_graph_function = ConcreteFunction(
         backwards_graph, attrs=backward_function_attr)
 
     forward_function_attr = _parse_func_attrs({
@@ -1087,14 +1088,15 @@ class PolymorphicFunction(object):
     graph_function = self._get_concrete_function_internal_garbage_collected(
         *args, **kwargs)
     # We're returning this concrete function to someone, and they may keep a
-    # reference to the FuncGraph without keeping a reference to the Function
-    # object. So we won't clean up the reference cycles manually and instead
-    # will leave them to Python's garbage collector.
+    # reference to the FuncGraph without keeping a reference to the
+    # ConcreteFunction object. So we won't clean up the reference cycles
+    # manually and instead will leave them to Python's garbage collector.
     graph_function._garbage_collector.release()  # pylint: disable=protected-access
     return graph_function
 
   def get_concrete_function(self, *args, **kwargs):
-    """Returns a `Function` object specialized to inputs and execution context.
+    """Returns a `ConcreteFunction` object specialized to inputs and execution
+    context.
 
     Args:
       *args: inputs to specialize on.
@@ -1292,7 +1294,7 @@ class PolymorphicFunction(object):
             self._function_spec.arg_names[:arglen]
             + [self._function_spec.vararg_name] *
             (arglen - len(self._function_spec.arg_names)))
-        graph_function = Function(
+        graph_function = ConcreteFunction(
             func_graph_module.func_graph_from_py_func(
                 self._name,
                 self._python_function,
@@ -1311,10 +1313,10 @@ class PolymorphicFunction(object):
         # Save information about non-Tensor arguments with the concrete
         # function. Used to serialize PolymorphicFunctions.
         graph_function._python_call_signature = python_call_signature
-        # Tell the Function to clean up its graph once it goes out of
-        # scope. Function does not do this in its constructor since it gets used
-        # in some places (like Keras) where the FuncGraph lives longer than the
-        # Function.
+        # Tell the ConcreteFunction to clean up its graph once it goes out of
+        # scope. ConcreteFunction does not do this in its constructor since it
+        # gets used in some places (like Keras) where the FuncGraph lives longer
+        # than the ConcreteFunction.
         graph_function._garbage_collector = _FunctionGarbageCollector(
             graph_function.graph)
         # pylint: enable=protected-access
@@ -1335,7 +1337,7 @@ def register(func, *args, **kwargs):
     **kwargs: input keyword arguments for the Python function.
 
   Returns:
-    a `Function` object specialized to inputs and execution context.
+    a `ConcreteFunction` object specialized to inputs and execution context.
 
   Raises:
     ValueError: When the input function is not a defun wrapped python function.
@@ -1706,7 +1708,7 @@ def defun_with_attributes(func=None,
       whitelisted attribute name is allowed. Unwhitelisted attribute name or
       unsupported value will result into ValueError. `func_name` is also one of
       the whitelisted argument which is a python string, and sets the name for
-      this `Function` in the graph.
+      this `ConcreteFunction` in the graph.
     autograph: same as defun()'s autograph.
 
   Returns:
@@ -1828,7 +1830,7 @@ class _PolymorphicFunctionGarbageCollector(object):
 
 
 class _FunctionGarbageCollector(object):
-  """Cleans up reference cycles when a Function goes out of scope."""
+  """Cleans up reference cycles when a `ConcreteFunction` goes out of scope."""
 
   def __init__(self, func_graph):
     self._func_graph = func_graph
diff --git a/tensorflow/python/eager/wrap_function.py b/tensorflow/python/eager/wrap_function.py
index 6f978feb3c..61b4876830 100644
--- a/tensorflow/python/eager/wrap_function.py
+++ b/tensorflow/python/eager/wrap_function.py
@@ -60,7 +60,7 @@ class VariableHolder(object):
 
 
 # TODO(allenl): make this checkpointable
-class WrappedFunction(function.Function):
+class WrappedFunction(function.ConcreteFunction):
   """Wraps a tf V1 piece of code in a function."""
 
   def __init__(self, fn_graph, variable_holder, attrs=None, signature=None):
diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index 6693243a68..2e53ee4ec9 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -3138,7 +3138,7 @@ class EagerExecutionFunction(object):
     # the relevant subgraph?
     graph.inputs = self.inputs + list(graph.captures.values())
     graph.outputs = self.outputs
-    graph_fn = eager_function.Function(graph)
+    graph_fn = eager_function.ConcreteFunction(graph)
     graph_fn._num_positional_args = len(self.inputs)
     graph_fn._arg_keywords = []
     self._graph_fn = graph_fn
diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py
index dc839cf61c..bc10c81fbe 100644
--- a/tensorflow/python/saved_model/function_deserialization.py
+++ b/tensorflow/python/saved_model/function_deserialization.py
@@ -60,7 +60,7 @@ def recreate_polymorphic_function(
 
   Args:
     saved_polymorphic_function: SavedPolymorphicFunction proto.
-    functions: map from function name to Function.
+    functions: map from function name to `ConcreteFunction`.
 
   Returns:
     A PolymorphicFunction.
@@ -123,8 +123,8 @@ def load_function_def_library(library):
     library: FunctionDefLibrary proto message.
 
   Returns:
-    Map of original function names in the library to instances of `Function`
-    without captured inputs.
+    Map of original function names in the library to instances of
+    `ConcreteFunction` without captured inputs.
 
   Raises:
     ValueError: if functions dependencies have a cycle.
@@ -142,7 +142,7 @@ def load_function_def_library(library):
       copy = _fix_fdef(fdef, name_mapping)
 
       func_graph = function_def_lib.function_def_to_graph(copy)
-      func = function_lib.Function(func_graph)
+      func = function_lib.ConcreteFunction(func_graph)
       func.add_to_graph(import_graph)
 
       name_mapping[fdef.signature.name] = func.name
@@ -209,8 +209,8 @@ def _list_function_deps(fdef):
 
 def _clean_function_name(name):
   """Vanity function to keep the function names comprehensible."""
-  # Note: each time a function is wrapped into `function_lib.Function` its
-  # name becomes "__inference_<orig>_xyz".
+  # Note: each time a function is wrapped into `function_lib.ConcreteFunction`
+  # its name becomes "__inference_<orig>_xyz".
   match = re.search(r"^__inference_(.*)_\d+$", name)
   if match:
     return match.group(1)
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index 9db6d03ed0..89fa845b75 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -169,7 +169,7 @@ def _canonicalize_signatures(signatures):
              "converted to concrete functions using "
              "`f.get_concrete_function(...)`.").format(signature_function))
       signature_function = signature_function.get_concrete_function()
-    elif not isinstance(signature_function, defun.Function):
+    elif not isinstance(signature_function, defun.ConcreteFunction):
       raise ValueError(
           ("Expected a TensorFlow function to generate a signature for, but "
            "got {}. Python functions may be decorated with "
-- 
GitLab


From 10b4e7755c9c3c94cc9adc2d0bd0ffe975d718d7 Mon Sep 17 00:00:00 2001
From: Aurelien Geron <aurelien.geron@gmail.com>
Date: Thu, 10 Jan 2019 14:08:58 +0800
Subject: [PATCH 0481/2345] Rename PolymorphicFunction to Function

---
 .../python/mirrored_strategy_multigpu_test.py |  2 +-
 tensorflow/python/eager/def_function.py       | 36 +++++------
 tensorflow/python/eager/function.py           | 61 +++++++++----------
 tensorflow/python/eager/function_test.py      |  8 +--
 .../python/keras/engine/training_utils.py     |  2 +-
 .../saved_model/function_deserialization.py   | 23 ++++---
 .../saved_model/function_serialization.py     | 29 +++++----
 tensorflow/python/saved_model/load.py         | 21 ++++---
 .../saved_model/nested_structure_coder.py     |  8 +--
 tensorflow/python/saved_model/save.py         | 43 +++++++------
 .../python/training/checkpointable/util.py    |  3 +-
 11 files changed, 116 insertions(+), 120 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index 2925b8c88c..b3b4fa85d3 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -1266,7 +1266,7 @@ class MirroredStrategyDefunTest(test.TestCase):
                             self.evaluate(device_result))
 
       for defun in defuns:
-        # `PolymorphicFunction`s are specialized to the current device stack, so
+        # `Function`s are specialized to the current device stack, so
         # call_for_each has one trace per device. To check that the expected set
         # of variables was accessed on each trace, we first retrieve each
         # device-specific graph function.
diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py
index b3f361b92d..6471841780 100644
--- a/tensorflow/python/eager/def_function.py
+++ b/tensorflow/python/eager/def_function.py
@@ -202,13 +202,13 @@ class UnliftedInitializerVariable(resource_variable_ops.ResourceVariable):
     self._cached_shape_as_list = None
 
 
-class PolymorphicFunction(object):
+class Function(object):
   """Wrapper class for the graph functions defined for a Python function.
 
   See the documentation for `tf.function` for more information on the semantics
   of defined functions.
 
-  PolymorphicFunction is thread-compatible.
+  `Function` is thread-compatible.
   """
 
   def __init__(self,
@@ -217,7 +217,7 @@ class PolymorphicFunction(object):
                input_signature=None,
                autograph=True,
                experimental_autograph_options=None):
-    """Initializes a polymorphic function.
+    """Initializes a `Function`.
 
     Args:
       python_function: the function to be wrapped.
@@ -280,10 +280,10 @@ class PolymorphicFunction(object):
   def _initialize(self, args, kwds, add_initializers_to=None):
     """Initializes, on the first call.
 
-    Creates two polymorphic functions, one that will allow creation of variables
+    Creates two `Function`s, one that will allow creation of variables
     and one that won't.
 
-    Additionally runs a trace for the polymorphic function that allows creation
+    Additionally runs a trace for the `Function` that allows creation
     of variables.
 
     Args:
@@ -447,7 +447,7 @@ class PolymorphicFunction(object):
 
   @property
   def _cached_input_signatures(self):
-    """All input signatures used to call this PolymorphicFunction."""
+    """All input signatures used to call this `Function`."""
     seen = list()
     # We are using a list so that:
     #  - the returned collection is deterministic, and
@@ -473,15 +473,15 @@ class PolymorphicFunction(object):
     """Returns a `ConcreteFunction` object specialized to inputs and execution
     context.
 
-    If this `PolymorphicFunction` was created with an `input_signature`, `args`
-    and `kwargs` may be omitted. With an input signature there is only one
-    concrete function associated with this `PolymorphicFunction`.
+    If this `Function` was created with an `input_signature`, `args` and
+    `kwargs` may be omitted. With an input signature there is only one
+    concrete function associated with this `Function`.
 
     If there is no fixed `input_signature` associated with this
-    `PolymorphicFunction`, positional and keyword arguments to
-    `get_concrete_function` follow the same rules as input signature
-    specification, with `tf.TensorSpec` objects describing `tf.Tensor`s which
-    will be passed to the concrete function.
+    `Function`, positional and keyword arguments to `get_concrete_function`
+    follow the same rules as input signature specification, with `tf.TensorSpec`
+    objects describing `tf.Tensor`s which will be passed to the concrete
+    function.
 
     Each `tf.Tensor` argument to the concrete function must have a unique name,
     either because it is the only one associated with a named argument of the
@@ -566,8 +566,8 @@ class PolymorphicFunction(object):
   def __get__(self, instance, owner):
     """Makes it possible to defun instance methods."""
     del owner
-    # `instance` here is the instance that this `PolymorphicFunction` was
-    # accessed through; e.g., for
+    # `instance` here is the instance that this `Function` was accessed through
+    # e.g., for
     #
     #   class Foo(object):
     #
@@ -576,10 +576,10 @@ class PolymorphicFunction(object):
     #       ...
     #
     #   foo = Foo()
-    #   foo.bar()  # `foo.bar` is a `PolymorphicFunction` instance
+    #   foo.bar()  # `foo.bar` is a `Function` instance
     #
     # then `instance` will be `foo` (and `owner` will be `Foo`).  We create a
-    # new instance of PolymorphicFunction here to allow different instances each
+    # new instance of `Function` here to allow different instances each
     # to create variables once, thereby allowing methods to be decorated with
     # tf.function. Keeps a cache to avoid retracing the function every time the
     # descriptor is accessed.
@@ -835,7 +835,7 @@ def function(func=None,
       name = "function"
     return tf_decorator.make_decorator(
         inner_function,
-        PolymorphicFunction(
+        Function(
             inner_function,
             name,
             input_signature=input_signature,
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 3bd7df9502..26ad52f59d 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -997,16 +997,16 @@ class FunctionSpec(object):
       return inputs, {}
 
 
-class PolymorphicFunction(object):
+class Function(object):
   """Wrapper class for the graph functions defined for a Python function.
 
   See the documentation for `defun` for more information on the semantics of
   defined functions.
 
-  PolymorphicFunction class is thread-compatible meaning that minimal
-  usage of defuns (defining and calling) is thread-safe, but if users call other
-  methods or invoke the base `python_function` themselves, external
-  synchronization is necessary.
+  `Function` class is thread-compatible meaning that minimal usage of defuns
+  (defining and calling) is thread-safe, but if users call other methods or
+  invoke the base `python_function` themselves, external synchronization is
+  necessary.
   """
 
   def __init__(self,
@@ -1015,7 +1015,7 @@ class PolymorphicFunction(object):
                input_signature=None,
                attributes=None,
                autograph=True):
-    """Initializes a polymorphic function.
+    """Initializes a `Function`.
 
     Args:
       python_function: the function to be wrapped.
@@ -1042,14 +1042,14 @@ class PolymorphicFunction(object):
     self._name = name
     self._autograph = autograph
     self._function_cache = collections.OrderedDict()
-    self._garbage_collector = _PolymorphicFunctionGarbageCollector(
+    self._garbage_collector = _FunctionGarbageCollector(
         self._function_cache)
     self._function_attributes = attributes or {}
 
     self._lock = threading.Lock()
     # _descriptor_cache is a of instance of a class to an instance-specific
-    # PolymorphicFunction, used to make sure defun-decorated methods create
-    # different functions for each instance.
+    # `Function`, used to make sure defun-decorated methods create different
+    # functions for each instance.
     self._descriptor_cache = weakref.WeakKeyDictionary()
 
   def __call__(self, *args, **kwargs):
@@ -1169,8 +1169,8 @@ class PolymorphicFunction(object):
   def __get__(self, instance, owner):
     """Makes it possible to defun instance methods."""
     del owner
-    # `instance` here is the instance that this `PolymorphicFunction` was
-    # accessed through; e.g., for
+    # `instance` here is the instance that this `Function` was accessed through
+    # e.g., for
     #
     #   class Foo(object):
     #
@@ -1179,26 +1179,25 @@ class PolymorphicFunction(object):
     #       ...
     #
     #   foo = Foo()
-    #   foo.bar()  # `foo.bar` is a `PolymorphicFunction` instance
+    #   foo.bar()  # `foo.bar` is a `Function` instance
     #
     # then `instance` will be `foo` (and `owner` will be `Foo`).  We create a
-    # new instance of PolymorphicFunction here to allow different instances each
+    # new instance of `Function` here to allow different instances each
     # to create variables once, thereby allowing methods to be decorated with
     # defun. Keeps a cache to avoid retracing the function every time the
     # descriptor is accessed.
     if instance not in self._descriptor_cache:
       if instance is None:
         return self
-      # If there is no instance-specific polymorphic func in the cache,
-      # we construct an instance-specific polymorphic function
-      # that uses a weak reference to the instance (so that the instance will
-      # be correctly gc'd).
+      # If there is no instance-specific `Function` in the cache, we construct
+      # an instance-specific `Function` that uses a weak reference to the
+      # instance (so that the instance will be correctly gc'd).
 
       # And finally add the wrapped function to the description cache
       self._descriptor_cache[instance] = class_method_to_instance_method(
           self, instance)
 
-    # Return the cached polymorphic function for the instance
+    # Return the cached `Function` for the instance
     return self._descriptor_cache[instance]
 
   def _cache_key(self, args, kwargs):
@@ -1256,8 +1255,8 @@ class PolymorphicFunction(object):
   def _maybe_define_function(self, args, kwargs):
     """Gets a function for these inputs, defining it if necessary.
 
-    `args` and `kwargs` can be None if this `PolymorphicFunction` was created
-    with an `input_signature`.
+    `args` and `kwargs` can be None if this `Function` was created with an
+    `input_signature`.
 
     Args:
       args: The varargs for the Python function.
@@ -1311,13 +1310,13 @@ class PolymorphicFunction(object):
               _encode_arg_for_serialization(arg) for arg in args)
         # pylint: disable=protected-access
         # Save information about non-Tensor arguments with the concrete
-        # function. Used to serialize PolymorphicFunctions.
+        # function. Used to serialize `Function`s.
         graph_function._python_call_signature = python_call_signature
         # Tell the ConcreteFunction to clean up its graph once it goes out of
         # scope. ConcreteFunction does not do this in its constructor since it
         # gets used in some places (like Keras) where the FuncGraph lives longer
         # than the ConcreteFunction.
-        graph_function._garbage_collector = _FunctionGarbageCollector(
+        graph_function._garbage_collector = _ConcreteFunctionGarbageCollector(
             graph_function.graph)
         # pylint: enable=protected-access
         self._function_cache[cache_key] = graph_function
@@ -1325,14 +1324,14 @@ class PolymorphicFunction(object):
 
 
 def register(func, *args, **kwargs):
-  """Register a specialization of a PolymorphicFunction into the graph.
+  """Register a specialization of a `Function` into the graph.
 
   This won't actually call the function with the inputs, and only put the
   function definition into graph. Register function with different input param
   will result into multiple version of functions registered in graph.
 
   Args:
-    func: the PolymorphicFunction instance that generated by a @defun
+    func: the `Function` instance that generated by a @defun
     *args: input arguments for the Python function.
     **kwargs: input keyword arguments for the Python function.
 
@@ -1342,7 +1341,7 @@ def register(func, *args, **kwargs):
   Raises:
     ValueError: When the input function is not a defun wrapped python function.
   """
-  if not isinstance(func, PolymorphicFunction):
+  if not isinstance(func, Function):
     raise ValueError("Only defun function is allowed to be registered. "
                      "Got type: %s" % type(func))
   concrete_func = func.get_concrete_function(*args, **kwargs)
@@ -1729,7 +1728,7 @@ def defun_with_attributes(func=None,
       name = "function"
     return tf_decorator.make_decorator(
         function,
-        PolymorphicFunction(
+        Function(
             function,
             name,
             input_signature=input_signature,
@@ -1761,7 +1760,7 @@ class _WeakrefSelf(object):
 
 
 def class_method_to_instance_method(original_function, instance):
-  """Constructs a new PolymorphicFunction with `self` bound."""
+  """Constructs a new `Function` with `self` bound."""
   weak_instance = weakref.ref(instance)
 
   # Note: while we could bind to a weakref proxy instead, that causes the
@@ -1769,8 +1768,8 @@ def class_method_to_instance_method(original_function, instance):
   bound_method = types_lib.MethodType(original_function.python_function,
                                       _WeakrefSelf(weak_instance))
 
-  # original_function is expected to be of one of the two PolymorphicFunction
-  # types (defined either in function.py or def_function.py).
+  # original_function is expected to be of one of the two `Function` types
+  # (defined either in function.py or def_function.py).
   assert hasattr(original_function, "_name")
   assert hasattr(original_function, "_autograph")
   assert hasattr(original_function, "_input_signature")
@@ -1812,7 +1811,7 @@ def class_method_to_instance_method(original_function, instance):
   return wrapped_instance_func
 
 
-class _PolymorphicFunctionGarbageCollector(object):
+class _FunctionGarbageCollector(object):
   """Cleans up cycles when a defun goes out of scope."""
 
   def __init__(self, cache):
@@ -1829,7 +1828,7 @@ class _PolymorphicFunctionGarbageCollector(object):
       pass
 
 
-class _FunctionGarbageCollector(object):
+class _ConcreteFunctionGarbageCollector(object):
   """Cleans up reference cycles when a `ConcreteFunction` goes out of scope."""
 
   def __init__(self, func_graph):
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index 1966b259bf..3bb4f2452c 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -966,8 +966,8 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     self.assertAllEqual([[3.0]], self.evaluate(y))
 
     # Break the reference cycle between the MiniModel and the defun:
-    # MiniModel --(through its `call` method)--> PolymorphicFunction
-    # PolymorphicFunction --(instancemethod on MiniModel)--> MiniModel
+    # `MiniModel` --(through its `call` method)--> `Function`
+    # `Function` --(instancemethod on `MiniModel`)--> `MiniModel`
     del model.call
 
   # Note: The ConfigProto below unfortunately only configures graph
@@ -1535,7 +1535,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
           t = constant_op.constant([[1.0, 2.0], [3.0, 4.0]])
           add(t, t)
 
-  def testRegisterPolymorphicFunction(self):
+  def testRegisterFunction(self):
     @function.defun
     def add(x, y):
       return math_ops.add(x, y)
@@ -1818,7 +1818,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     self.assertAllEqual(instance_call_one, instance_call_two)
     self.assertAllEqual(instance_call_one, class_call)
 
-  def testDecoratedMethodUniquePolymorphicFuncPerInstance(self):
+  def testDecoratedMethodUniqueFunctionPerInstance(self):
     m = DefunnedMiniModel()
     n = DefunnedMiniModel()
 
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index 949f1e400d..6072d09428 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -1402,7 +1402,7 @@ def trace_model_call(model, input_signature=None):
     ValueError: if input signature cannot be inferred from the model.
   """
   if input_signature is None:
-    if isinstance(model.call, def_function.PolymorphicFunction):
+    if isinstance(model.call, def_function.Function):
       input_signature = model.call.input_signature
 
   if input_signature is None:
diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py
index bc10c81fbe..f1c43cb7a4 100644
--- a/tensorflow/python/saved_model/function_deserialization.py
+++ b/tensorflow/python/saved_model/function_deserialization.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tools for deserializing PolymorphicFunctions."""
+"""Tools for deserializing `Function`s."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -54,25 +54,24 @@ def _inputs_compatible(args, stored_inputs):
   return True
 
 
-def recreate_polymorphic_function(
-    saved_polymorphic_function, functions):
-  """Creates a PolymorphicFunction from a SavedPolymorphicFunction.
+def recreate_function(saved_function, concrete_functions):
+  """Creates a `Function` from a `SavedPolymorphicFunction`.
 
   Args:
-    saved_polymorphic_function: SavedPolymorphicFunction proto.
-    functions: map from function name to `ConcreteFunction`.
+    saved_function: `SavedPolymorphicFunction` proto.
+    concrete_functions: map from function name to `ConcreteFunction`.
 
   Returns:
-    A PolymorphicFunction.
+    A `Function`.
   """
-  # TODO(andresp): Construct a PolymorphicFunction with the cache populated
-  # instead of creating a new PolymorphicFunction backed by a Python layer to
+  # TODO(andresp): Construct a `Function` with the cache populated
+  # instead of creating a new `Function` backed by a Python layer to
   # glue things together. Current approach is nesting functions deeper for each
   # serialization cycle.
 
   coder = nested_structure_coder.StructureCoder()
   function_spec_tuple = coder.decode_proto(
-      saved_polymorphic_function.function_spec_tuple)
+      saved_function.function_spec_tuple)
   function_spec = function_lib.FunctionSpec.from_tuple(function_spec_tuple)
 
   # TODO(mdan): We may enable autograph once exceptions are supported.
@@ -81,8 +80,8 @@ def recreate_polymorphic_function(
     """Calls a restored function."""
     # TODO(allenl): Functions saved with input_signatures should revive with
     # input_signatures.
-    for monomorphic_function in saved_polymorphic_function.monomorphic_function:
-      function_obj = functions[monomorphic_function.concrete_function]
+    for monomorphic_function in saved_function.monomorphic_function:
+      function_obj = concrete_functions[monomorphic_function.concrete_function]
       canonicalized_original_inputs = coder.decode_proto(
           monomorphic_function.canonicalized_input)
 
diff --git a/tensorflow/python/saved_model/function_serialization.py b/tensorflow/python/saved_model/function_serialization.py
index 014a4ef04c..b645a7c964 100644
--- a/tensorflow/python/saved_model/function_serialization.py
+++ b/tensorflow/python/saved_model/function_serialization.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tools for serializing PolymorphicFunctions."""
+"""Tools for serializing `Function`s."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -24,15 +24,14 @@ from tensorflow.python.saved_model import nested_structure_coder
 from tensorflow.python.saved_model import saved_object_graph_pb2
 
 
-def serialize_polymorphic_function(polymorphic_function, node_ids):
-  """Build a SavedPolymorphicProto."""
+def serialize_function(function, node_ids):
+  """Build a SavedPolymorphicFunction proto."""
   coder = nested_structure_coder.StructureCoder()
   proto = saved_object_graph_pb2.SavedPolymorphicFunction()
 
   proto.function_spec_tuple.CopyFrom(
-      coder.encode_structure(polymorphic_function.function_spec.as_tuple()))  # pylint: disable=protected-access
-  for signature, concrete_function in list_all_concrete_functions(
-      polymorphic_function):
+      coder.encode_structure(function.function_spec.as_tuple()))  # pylint: disable=protected-access
+  for signature, concrete_function in list_all_concrete_functions(function):
     bound_inputs = []
     try:
       for capture in concrete_function.captured_inputs:
@@ -52,23 +51,23 @@ def serialize_polymorphic_function(polymorphic_function, node_ids):
   return proto
 
 
-def list_all_concrete_functions(polymorphic_function):
-  """Given a polymorphic function, returns all of its concrete functions.
+def list_all_concrete_functions(function):
+  """Given a `Function`, returns all of its concrete functions.
 
   Args:
-    polymorphic_function: Instance of `PolymorphicFunction`.
+    function: Instance of `Function`.
 
   Returns:
-    A list of tuples in the form (signature, concrete_function), where concrete
-    function is an instance of `Function`.
+    A list of tuples in the form (signature, concrete_function), where
+    `concrete_function` is an instance of `ConcreteFunction`.
   """
-  input_signature = polymorphic_function._input_signature  # pylint: disable=protected-access
+  input_signature = function._input_signature  # pylint: disable=protected-access
   if input_signature is not None:
-    polymorphic_function.get_concrete_function()
+    function.get_concrete_function()
   concrete_functions = []
-  for signature in polymorphic_function._cached_input_signatures:  # pylint: disable=protected-access
+  for signature in function._cached_input_signatures:  # pylint: disable=protected-access
     if any(isinstance(arg, defun_lib.UnknownArgument) for arg in signature):
       continue
-    concrete_function = polymorphic_function.get_concrete_function(*signature)
+    concrete_function = function.get_concrete_function(*signature)
     concrete_functions.append((signature, concrete_function))
   return concrete_functions
diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
index bbb1485612..ae71a228ea 100644
--- a/tensorflow/python/saved_model/load.py
+++ b/tensorflow/python/saved_model/load.py
@@ -42,15 +42,16 @@ class _Loader(object):
     self._asset_file_def = meta_graph.asset_file_def
     self._proto = object_graph_proto
     self._export_dir = export_dir
-    self._functions = function_deserialization.load_function_def_library(
-        meta_graph.graph_def.library)
+    self._concrete_functions = (
+        function_deserialization.load_function_def_library(
+            meta_graph.graph_def.library))
     self._load_all()
     self._bind_function_captures()
     self._restore_checkpoint()
 
   def _bind_function_captures(self):
     """Setup captured tensors in restored concrete functions."""
-    seen_functions = set()
+    seen_concrete_functions = set()
     for object_proto in self._proto.nodes:
       if object_proto.WhichOneof("kind") == "function":
         for monomorphic_function in object_proto.function.monomorphic_function:
@@ -63,18 +64,18 @@ class _Loader(object):
               for node_id in monomorphic_function.bound_inputs
               if self._proto.nodes[node_id].WhichOneof("kind") == "variable"
           ]
-          if name in seen_functions:
-            if self._functions[name]._captured_inputs != bound_inputs:  # pylint: disable=protected-access
+          if name in seen_concrete_functions:
+            if self._concrete_functions[name]._captured_inputs != bound_inputs:  # pylint: disable=protected-access
               raise NotImplementedError(
                   "Function %s is used more than once with different "
                   "captured inputs." % name)
           else:
-            seen_functions.add(name)
+            seen_concrete_functions.add(name)
             # TODO(andresp): This is only injecting the captured inputs into the
             # concrete function, note that we did not modify the FuncGraph
             # itself.
-            self._functions[name]._captured_inputs = bound_inputs  # pylint: disable=protected-access
-            self._functions[name]._func_graph.variables = bound_variables  # pylint: disable=protected-access
+            self._concrete_functions[name]._captured_inputs = bound_inputs  # pylint: disable=protected-access
+            self._concrete_functions[name]._func_graph.variables = bound_variables  # pylint: disable=protected-access
 
   def _get_tensor_from_node(self, node_id):
     obj = self._nodes[node_id]
@@ -137,8 +138,8 @@ class _Loader(object):
     return tracking.TrackableAsset(filename)
 
   def _recreate_function(self, proto):
-    return function_deserialization.recreate_polymorphic_function(
-        proto, self._functions)
+    return function_deserialization.recreate_function(
+        proto, self._concrete_functions)
 
   def _recreate_variable(self, proto):
     # TODO(andresp): Can we use the checkpointed value as initializer?
diff --git a/tensorflow/python/saved_model/nested_structure_coder.py b/tensorflow/python/saved_model/nested_structure_coder.py
index 410ebda5c1..31ee239f13 100644
--- a/tensorflow/python/saved_model/nested_structure_coder.py
+++ b/tensorflow/python/saved_model/nested_structure_coder.py
@@ -14,14 +14,14 @@
 # ==============================================================================
 """Module that encodes (decodes) nested structures into (from) protos.
 
-The intended use is to serialize everything needed to restore a
-PolymorphicFunction that was saved into a SavedModel. This may include concrete
-function inputs and outputs, signatures, function specs, etc.
+The intended use is to serialize everything needed to restore a `Function` that
+was saved into a SavedModel. This may include concrete function inputs and
+outputs, signatures, function specs, etc.
 
 Example use:
 coder = nested_structure_coder.StructureCoder()
 # Encode into proto.
-signature_proto = coder.encode_structure(polymorphic_function.input_signature)
+signature_proto = coder.encode_structure(function.input_signature)
 # Decode into a Python object.
 restored_signature = coder.decode_proto(signature_proto)
 """
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index 89fa845b75..31f6aad4cf 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -68,17 +68,17 @@ class _SaveableView(object):
     self.nodes = checkpointable_objects
     self.node_ids = node_ids
     self.slot_variables = slot_variables
-    self.polymorphic_functions = util.ObjectIdentityDictionary()
+    self.functions = util.ObjectIdentityDictionary()
 
-    # Also add polymorphic functions as nodes.
+    # Also add `Function`s as nodes.
     for obj in self.nodes:
-      self.polymorphic_functions[obj] = self._list_polymorphic_functions(obj)
-      for function in self.polymorphic_functions[obj].values():
+      self.functions[obj] = self._list_functions(obj)
+      for function in self.functions[obj].values():
         if function not in self.node_ids:
           self.node_ids[function] = len(self.nodes)
           self.nodes.append(function)
         # Force listing the concrete functions for the side effects:
-        #  - populate the cache for polymorphic functions that have an
+        #  - populate the cache for `Function`s that have an
         #  input_signature and have not been called.
         #  - force side effects of creation of concrete functions, e.g. create
         #  variables on first run.
@@ -94,20 +94,20 @@ class _SaveableView(object):
       assert self.node_ids[node] == node_id
       object_proto = proto.nodes.add()
       object_proto.slot_variables.extend(self.slot_variables.get(node, ()))
-      if isinstance(node, def_function.PolymorphicFunction):
+      if isinstance(node, def_function.Function):
         continue
       for child in node._checkpoint_dependencies:  # pylint: disable=protected-access
         child_proto = object_proto.children.add()
         child_proto.node_id = self.node_ids[child.ref]
         child_proto.local_name = child.name
-      for local_name, ref_function in self.polymorphic_functions[node].items():
+      for local_name, ref_function in self.functions[node].items():
         child_proto = object_proto.children.add()
         child_proto.node_id = self.node_ids[ref_function]
         child_proto.local_name = local_name
 
-  def _list_polymorphic_functions(self, checkpointable_object):
-    """Return a dict of polymorphic functions of a checkpointable."""
-    polymorphic_functions = dict()
+  def _list_functions(self, checkpointable_object):
+    """Return a dict of `Function`s of a checkpointable."""
+    functions = dict()
     for attribute_name in dir(checkpointable_object):
       try:
         attribute_value = getattr(checkpointable_object, attribute_name, None)
@@ -115,16 +115,16 @@ class _SaveableView(object):
         # We really don't want to throw an exception just because some object's
         # attribute accessor is broken.
         attribute_value = None
-      if isinstance(attribute_value, def_function.PolymorphicFunction):
-        polymorphic_functions[attribute_name] = attribute_value
-    return polymorphic_functions
+      if isinstance(attribute_value, def_function.Function):
+        functions[attribute_name] = attribute_value
+    return functions
 
 
 def _find_function_to_export(saveable_view):
   """Iterate over `root`'s attributes, finding traced functions."""
   exported_function = None
   previous_attribute_name = None
-  functions = saveable_view.polymorphic_functions[saveable_view.root]
+  functions = saveable_view.functions[saveable_view.root]
   for name, value in sorted(functions.items()):
     if exported_function is not None:
       raise ValueError(
@@ -158,8 +158,7 @@ def _canonicalize_signatures(signatures):
         signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signatures}
   concrete_signatures = {}
   for serving_key, signature_function in signatures.items():
-    if isinstance(signature_function, (defun.PolymorphicFunction,
-                                       def_function.PolymorphicFunction)):
+    if isinstance(signature_function, (defun.Function, def_function.Function)):
       input_signature = signature_function._input_signature  # pylint: disable=protected-access
       if input_signature is None:
         raise ValueError(
@@ -547,12 +546,12 @@ def _fill_meta_graph_def(meta_graph_def, saveable_view, signature_functions,
   # the exported graph (thus the `to_graph` argument).
   saver = object_saver.freeze(object_map=object_map, to_graph=exported_graph)
 
-  # We must instantiate and list all concrete functions of polymorphic functions
-  # while in eager mode so they end up added to the graph and can later be used
-  # by the object based saved model.
+  # We must instantiate and list all concrete functions of `Function`s while in
+  # eager mode so they end up added to the graph and can later be used by the
+  # object based saved model.
   concrete_functions = []
   for obj in accessible_objects:
-    for function in saveable_view.polymorphic_functions[obj].values():
+    for function in saveable_view.functions[obj].values():
       concrete_functions.extend(
           function_serialization.list_all_concrete_functions(function))
 
@@ -613,9 +612,9 @@ def _write_object_proto(obj, proto, asset_file_def_index, node_ids):
     proto.variable.trainable = obj.trainable
     proto.variable.dtype = obj.dtype.as_datatype_enum
     proto.variable.shape.CopyFrom(obj.shape.as_proto())
-  elif isinstance(obj, def_function.PolymorphicFunction):
+  elif isinstance(obj, def_function.Function):
     proto.function.CopyFrom(
-        function_serialization.serialize_polymorphic_function(obj, node_ids))
+        function_serialization.serialize_function(obj, node_ids))
   else:
     proto.user_object.SetInParent()
 
diff --git a/tensorflow/python/training/checkpointable/util.py b/tensorflow/python/training/checkpointable/util.py
index a45263f5c6..4a80b25a98 100644
--- a/tensorflow/python/training/checkpointable/util.py
+++ b/tensorflow/python/training/checkpointable/util.py
@@ -1748,8 +1748,7 @@ class Checkpoint(tracking.Checkpointable):
     """
     super(Checkpoint, self).__init__()
     for k, v in sorted(kwargs.items(), key=lambda item: item[0]):
-      if not isinstance(v, (base.CheckpointableBase,
-                            def_function.PolymorphicFunction)):
+      if not isinstance(v, (base.CheckpointableBase, def_function.Function)):
         raise ValueError(
             ("`Checkpoint` was expecting a checkpointable object (an object "
              "derived from `CheckpointableBase`), got %s. If you believe this "
-- 
GitLab


From ebb3429856441149e41388dfbea59496f8dbf17b Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <penporn@google.com>
Date: Wed, 9 Jan 2019 22:30:09 -0800
Subject: [PATCH 0482/2345] Remove a TODO about moving choose() out of
 TensorContraction.h since it is now in
 unsupported/Eigen/CXX11/scr/Tensor/TensorMeta.h.

PiperOrigin-RevId: 228648231
---
 tensorflow/core/kernels/eigen_spatial_convolutions.h | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tensorflow/core/kernels/eigen_spatial_convolutions.h b/tensorflow/core/kernels/eigen_spatial_convolutions.h
index 86d8c98ee6..8b19813940 100644
--- a/tensorflow/core/kernels/eigen_spatial_convolutions.h
+++ b/tensorflow/core/kernels/eigen_spatial_convolutions.h
@@ -1683,8 +1683,6 @@ EIGEN_DEVICE_FUNC
     kernel_dims[0] = kernelChannels * kernelRows * kernelCols;
     kernel_dims[1] = kernelFilters;
   }
-  // TODO(yangke): choose() is defined in TensorContraction.h -- consider
-  // moving it to somewhere more "common".
   return choose(
       Cond<internal::traits<Input>::Layout == ColMajor>(),
       kernel.reshape(kernel_dims)
-- 
GitLab


From 6d3cbd2bcc165f8de064643719991db9753c8dca Mon Sep 17 00:00:00 2001
From: Smit Hinsu <hinsu@google.com>
Date: Wed, 9 Jan 2019 23:57:12 -0800
Subject: [PATCH 0483/2345] Update TensorRT SegmentGraph function to return
 node pointers

This is in preparation to share the graph segmentation code between XLA and
TensorRT.

TESTED with existing unit tests

PiperOrigin-RevId: 228654815
---
 .../contrib/tensorrt/convert/convert_graph.cc | 51 ++++++++++---------
 .../contrib/tensorrt/convert/convert_nodes.cc | 15 +++---
 .../contrib/tensorrt/convert/convert_nodes.h  |  3 +-
 .../contrib/tensorrt/segment/segment.cc       | 17 +++----
 tensorflow/contrib/tensorrt/segment/segment.h |  8 +--
 .../contrib/tensorrt/segment/segment_test.cc  |  5 +-
 6 files changed, 53 insertions(+), 46 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index bf2de94e04..eef647473a 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -334,13 +334,12 @@ struct EdgePtrCompare {
 tensorflow::Status GetEngineInfo(
     const tensorflow::Graph* g,
     const tensorflow::grappler::GraphProperties& graph_properties,
-    const std::set<string>& segment_nodes,
+    const std::set<const Node*>& segment_nodes,
     const std::unordered_map<string, tensorflow::Node*>& node_map,
     const std::vector<tensorflow::Node*>& reverse_topo_order,
     EngineInfo* info) {
-  std::vector<int> subgraph_node_ids;  // Topologically sorted node ids.
-  std::set<string> subgraph_node_names = segment_nodes;
-  std::set<int> added_const_node_ids;  // Used to prevent double insertion.
+  std::vector<const Node*> subgraph_nodes;  // Topologically sorted nodes.
+  std::set<const Node*> added_const_nodes;  // Used to prevent double insertion.
   std::set<string> segment_devices;
 
   // Map from src_node_name+port to the unique port numbers of the TRT op, where
@@ -352,9 +351,8 @@ tensorflow::Status GetEngineInfo(
   std::unordered_map<string, int> input_to_engine_port, output_to_engine_port;
   for (auto it = reverse_topo_order.rbegin(); it != reverse_topo_order.rend();
        ++it) {
-    const auto& node_name = (*it)->name();
-    if (segment_nodes.count(node_name) == 0) continue;
-    auto node = *it;
+    const Node* node = *it;
+    if (segment_nodes.count(node) == 0) continue;
     auto node_device = node->requested_device();
     if (!node_device.empty()) {
       segment_devices.insert(node_device);
@@ -366,8 +364,11 @@ tensorflow::Status GetEngineInfo(
                 << " neither have requested device nor assigned device";
       }
     }
+    subgraph_nodes.push_back(node);
+
     const int node_id = node->id();
-    subgraph_node_ids.push_back(node_id);
+    const string& node_name = node->name();
+
     // Create input connections. Sort edges first to make determnistic since
     // in_edges is a set of pointers.
     std::vector<const tensorflow::Edge*> in_edges(node->in_edges().begin(),
@@ -375,7 +376,7 @@ tensorflow::Status GetEngineInfo(
     std::sort(in_edges.begin(), in_edges.end(), EdgePtrCompare());
     for (const auto edge : in_edges) {
       auto input_node = edge->src();
-      if (input_node->IsSource() || segment_nodes.count(input_node->name())) {
+      if (input_node->IsSource() || segment_nodes.count(input_node)) {
         continue;
       }
       if (edge->IsControlEdge()) {
@@ -392,12 +393,11 @@ tensorflow::Status GetEngineInfo(
         //
         // Note that the segmenter already ensure that the constant data input
         // is valid and suppported by the engine.
-        if (!added_const_node_ids.insert(input_node->id()).second) {
+        if (!added_const_nodes.insert(input_node).second) {
           // Already added before.
           continue;
         }
         VLOG(1) << "Adding const node " << input_node->name();
-        QCHECK(subgraph_node_names.insert(input_node->name()).second);
         // Since we already add (duplicate) the const input node to the segment
         // graphdef, it's now not a data dependency any more, but to make the
         // dependency correct we still add a control dependency.
@@ -428,7 +428,7 @@ tensorflow::Status GetEngineInfo(
     std::sort(out_edges.begin(), out_edges.end(), EdgePtrCompare());
     for (const auto edge : out_edges) {
       auto output_node = edge->dst();
-      if (output_node->IsSink() || segment_nodes.count(output_node->name())) {
+      if (output_node->IsSink() || segment_nodes.count(output_node)) {
         continue;
       }
       if (edge->IsControlEdge()) {
@@ -456,12 +456,11 @@ tensorflow::Status GetEngineInfo(
   }  // For each segment node in topological order.
 
   // Construct the const nodes first.
-  subgraph_node_ids.insert(subgraph_node_ids.begin(),
-                           added_const_node_ids.begin(),
-                           added_const_node_ids.end());
+  subgraph_nodes.insert(subgraph_nodes.begin(), added_const_nodes.begin(),
+                        added_const_nodes.end());
   TF_RETURN_IF_ERROR(ConvertSegmentToGraphDef(
-      g, graph_properties, subgraph_node_names, subgraph_node_ids,
-      &info->connections, &info->segment_graph_def, &info->engine_name));
+      g, graph_properties, subgraph_nodes, &info->connections,
+      &info->segment_graph_def, &info->engine_name));
   // TODO(sami): This should not happen once segmenter is updated.
   if (segment_devices.size() == 1) {
     info->device = *segment_devices.begin();
@@ -1033,27 +1032,31 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
     cudaSetDevice(cuda_device_id);
     auto status = CreateTRTNode(engine_segments, i, params.max_batch_size,
                                 &graph, alloc.get(), &engine_nodes);
-    // If status is ok, we successfully added the node to the graph and can
-    // remove segment ops. Otherwise graph is not modified.
+
     string msg = StrCat("TensorRT node ", engine.engine_name,
                         " added for segment ", i, " consisting of ",
                         converted_segments.at(i).first.size(), " nodes");
     if (status.ok()) {
       LOG(INFO) << msg << " succeeded.";
-      for (auto node_name : converted_segments.at(i).first) {
-        graph.RemoveNode(node_map.at(node_name));
-      }
     } else {
       // Graph is not modified.
       LOG(WARNING) << msg << " failed: " << status << ". Fallback to TF...";
     }
     if (VLOG_IS_ON(1)) {
       msg = "Segment consists of nodes: ";
-      for (const string& node_name : converted_segments.at(i).first) {
-        StrAppend(&msg, node_name, ", ");
+      for (const Node* node : converted_segments.at(i).first) {
+        StrAppend(&msg, node->name(), ", ");
       }
       VLOG(1) << msg;
     }
+
+    // If status is ok, we successfully added the node to the graph and can
+    // remove segment ops. Otherwise graph is not modified.
+    if (status.ok()) {
+      for (const Node* node : converted_segments.at(i).first) {
+        graph.RemoveNode(const_cast<Node*>(node));
+      }
+    }
   }
   cudaSetDevice(old_cuda_device);
   graph.ToGraphDef(params.output_graph_def);
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 179d631a3e..7b0c4b446d 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -3780,8 +3780,7 @@ tensorflow::Status ConvertGraphDefToEngine(
 tensorflow::Status ConvertSegmentToGraphDef(
     const tensorflow::Graph* graph,
     const tensorflow::grappler::GraphProperties& graph_properties,
-    const std::set<string>& subgraph_node_names,
-    const std::vector<int>& subgraph_node_ids,  // In topological order
+    const std::vector<const Node*>& subgraph_nodes,  // In topological order
     std::vector<EngineConnection>* connections,
     tensorflow::GraphDef* segment_def, string* common_scope) {
   std::set<string> marker_nodes;
@@ -3855,11 +3854,10 @@ tensorflow::Status ConvertSegmentToGraphDef(
 
   std::unordered_map<int, int> old_to_new_id_map;
   // Copy internal nodes to new graphdef
-  string local_scope = graph->FindNodeId(*subgraph_node_ids.begin())->name();
-  for (const auto node_id : subgraph_node_ids) {
-    const auto node = graph->FindNodeId(node_id);
+  string local_scope = subgraph_nodes.front()->name();
+  for (const Node* node : subgraph_nodes) {
     local_scope = GetCommonNameScope(local_scope, node->name());
-    old_to_new_id_map[node_id] = segment_def->node_size();
+    old_to_new_id_map[node->id()] = segment_def->node_size();
     auto snode = segment_def->add_node();
     snode->CopyFrom(node->def());
     VLOG(2) << "Copying " << snode->name() << " to subgraph";
@@ -3877,6 +3875,11 @@ tensorflow::Status ConvertSegmentToGraphDef(
             << placeholder_name;
     snode->set_input(connection.inside_port, placeholder_name);
   }
+  std::set<string> subgraph_node_names;
+  for (const Node* node : subgraph_nodes) {
+    subgraph_node_names.insert(node->name());
+  }
+
   // Remove control inputs that are not inside the segment.
   for (int i = 0; i < segment_def->node_size(); ++i) {
     auto snode = segment_def->mutable_node(i);
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.h b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
index 54e19b7395..8f2271ee3f 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
@@ -128,8 +128,7 @@ struct EngineInfo {
 tensorflow::Status ConvertSegmentToGraphDef(
     const tensorflow::Graph* graph,
     const tensorflow::grappler::GraphProperties& graph_properties,
-    const std::set<string>& subgraph_node_names,
-    const std::vector<int>& subgraph_node_ids,
+    const std::vector<const Node*>& subgraph_nodes,
     std::vector<EngineConnection>* connections,
     tensorflow::GraphDef* segment_def, string* common_scope);
 
diff --git a/tensorflow/contrib/tensorrt/segment/segment.cc b/tensorflow/contrib/tensorrt/segment/segment.cc
index 084a96e0fa..ecaffa3023 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.cc
+++ b/tensorflow/contrib/tensorrt/segment/segment.cc
@@ -673,10 +673,11 @@ tensorflow::Status SegmentGraph(
   // --------------------------------- Step 3 ---------------------------------
   // Convert the segments into the expected return format
   for (const auto& itr : sg_map) {
-    const std::set<const tensorflow::Node*, NodePtrCompare>& segment_nodes =
-        itr.second;
+    const string& segment_root = itr.first;
+    // Return format does not require set comparator.
+    std::set<const Node*> segment_nodes(itr.second.begin(), itr.second.end());
     if (VLOG_IS_ON(1)) {
-      string s = "parent=" + itr.first + ":";
+      string s = "parent=" + segment_root + ":";
       for (auto node : segment_nodes) s += " " + node->name();
       VLOG(1) << "Segment " << segments->size() << ": " << s;
     }
@@ -689,12 +690,10 @@ tensorflow::Status SegmentGraph(
     }
 
     // TODO(sami): Make segmenter placement aware once trtscopes are in place
-    std::set<string> segment_node_names;
-    for (auto node : itr.second) segment_node_names.insert(node->name());
-    const auto& dev_itr = device_maps.find(itr.first);
+    const auto& dev_itr = device_maps.find(segment_root);
     if (dev_itr == device_maps.end() || dev_itr->second.empty()) {
       VLOG(1) << "No device assigned to segment " << segments->size();
-      segments->emplace_back(std::make_pair(segment_node_names, string()));
+      segments->emplace_back(std::make_pair(segment_nodes, string()));
     } else if (dev_itr->second.size() > 1) {
       string s("Segment ");
       StrAppend(&s, segments->size(), " has multiple devices attached: ");
@@ -703,10 +702,10 @@ tensorflow::Status SegmentGraph(
       }
       LOG(WARNING) << s << " choosing " << *(dev_itr->second.begin());
       segments->emplace_back(
-          std::make_pair(segment_node_names, *(dev_itr->second.begin())));
+          std::make_pair(segment_nodes, *(dev_itr->second.begin())));
     } else {
       segments->emplace_back(
-          std::make_pair(segment_node_names, *(dev_itr->second.begin())));
+          std::make_pair(segment_nodes, *(dev_itr->second.begin())));
     }
   }
   if (VLOG_IS_ON(1)) {
diff --git a/tensorflow/contrib/tensorrt/segment/segment.h b/tensorflow/contrib/tensorrt/segment/segment.h
index b9693aad1b..6cc92cdb5d 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.h
+++ b/tensorflow/contrib/tensorrt/segment/segment.h
@@ -29,10 +29,10 @@ namespace tensorflow {
 namespace tensorrt {
 namespace segment {
 
-// Vector of segments, each entry contains a set of node names and a device name
-// in the segment.
-// TODO(aaroey): use node pointer instead of node name.
-using SegmentNodesVector = std::vector<std::pair<std::set<string>, string>>;
+// Vector of segments, each entry contains a set of node pointers and a device
+// name in the segment.
+using SegmentNodesVector =
+    std::vector<std::pair<std::set<const Node*>, string>>;
 
 struct SegmentOptions {
   // Segment must contain at least this many nodes.
diff --git a/tensorflow/contrib/tensorrt/segment/segment_test.cc b/tensorflow/contrib/tensorrt/segment/segment_test.cc
index 4805ef9c61..4ac02327ae 100644
--- a/tensorflow/contrib/tensorrt/segment/segment_test.cc
+++ b/tensorflow/contrib/tensorrt/segment/segment_test.cc
@@ -75,7 +75,10 @@ class SegmentTest : public ::testing::Test {
                        const std::vector<std::set<string>>& expected_segments) {
     EXPECT_EQ(expected_segments.size(), segments.size());
     for (int i = 0; i < segments.size(); ++i) {
-      const auto& segment_node_names = segments[i].first;
+      std::set<string> segment_node_names;
+      for (const Node* node : segments[i].first) {
+        segment_node_names.insert(node->name());
+      }
       const auto& expected = expected_segments[i];
       for (const auto& name : expected) {
         EXPECT_TRUE(segment_node_names.count(name))
-- 
GitLab


From 1e104bb923494b667a0ac8c28dac0d8fe516239c Mon Sep 17 00:00:00 2001
From: Grzegorz Pawelczak <grzegorzp@graphcore.ai>
Date: Thu, 10 Jan 2019 08:36:05 +0000
Subject: [PATCH 0484/2345] Fix nit in build file

---
 tensorflow/compiler/xla/service/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 7be180c624..4b691c7340 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -3484,9 +3484,9 @@ tf_cc_test(
     srcs = ["sort_simplifier_test.cc"],
     deps = [
         ":hlo_matchers",
-        ":sort_simplifier",
         ":pattern_matcher",
         ":pattern_matcher_gmock",
+        ":sort_simplifier",
         "//tensorflow/compiler/xla:test",
         "//tensorflow/compiler/xla/service:hlo_parser",
         "//tensorflow/compiler/xla/tests:hlo_test_base",
-- 
GitLab


From 439d4ec9568a259f665efd878791530a6f5ca8f2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 01:02:17 -0800
Subject: [PATCH 0485/2345] compat: Update forward compatibility horizon to
 2019-01-10

PiperOrigin-RevId: 228661930
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index dfee972cab..638bd445a6 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 9)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 10)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 2f7e56a65104aeca0a7fcc98a7b10fb2cff9b1a3 Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Thu, 10 Jan 2019 02:46:36 -0800
Subject: [PATCH 0486/2345] Remove partitioned_info as argument to initializer
 in Keras make_variable to work with the v2 initializers

PiperOrigin-RevId: 228675005
---
 tensorflow/python/keras/engine/base_layer_utils.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/keras/engine/base_layer_utils.py b/tensorflow/python/keras/engine/base_layer_utils.py
index 60d0cf3391..f1b1f42c22 100644
--- a/tensorflow/python/keras/engine/base_layer_utils.py
+++ b/tensorflow/python/keras/engine/base_layer_utils.py
@@ -25,6 +25,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.keras import backend
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import init_ops
+from tensorflow.python.ops import init_ops_v2
 from tensorflow.python.ops import variables as tf_variables
 from tensorflow.python.util import nest
 
@@ -55,7 +56,6 @@ def make_variable(name,
                   shape=None,
                   dtype=dtypes.float32,
                   initializer=None,
-                  partition_info=None,
                   trainable=None,
                   caching_device=None,
                   validate_shape=True,
@@ -76,14 +76,12 @@ def make_variable(name,
   rid of this temporary solution.
 
   TODO(fchollet): remove this method when no longer needed.
-  TODO(fchollet): handle `partitioner` argument.
 
   Arguments:
     name: Variable name.
     shape: Variable shape.
     dtype: The type of the variable. Defaults to `self.dtype` or `float32`.
     initializer: Initializer instance (callable).
-    partition_info: Not handled at this time.
     trainable: Whether the variable should be part of the layer's
       "trainable_variables" (e.g. variables, biases)
       or "non_trainable_variables" (e.g. BatchNorm mean, stddev).
@@ -123,8 +121,9 @@ def make_variable(name,
       # Instantiate initializer if provided initializer is a type object.
       if isinstance(initializer, type(init_ops.Initializer)):
         initializer = initializer(dtype=dtype)
-      init_val = lambda: initializer(  # pylint: disable=g-long-lambda
-          shape, dtype=dtype, partition_info=partition_info)
+      elif isinstance(initializer, type(init_ops_v2.Initializer)):
+        initializer = initializer()
+      init_val = lambda: initializer(shape, dtype=dtype)
       variable_dtype = dtype.base_dtype
   if use_resource is None:
     use_resource = True
-- 
GitLab


From ef088e54734e0412bc47e4f33d1df7c0672fdd87 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 04:21:44 -0800
Subject: [PATCH 0487/2345] Add a test for soft matching of inputs.

PiperOrigin-RevId: 228686125
---
 tensorflow/python/saved_model/load_test.py | 31 ++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index f7020e4a45..8b34414d25 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -29,6 +29,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import tensor_spec
 from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import variables
+from tensorflow.python.saved_model import function_serialization
 from tensorflow.python.saved_model import load
 from tensorflow.python.saved_model import save
 from tensorflow.python.training.checkpointable import tracking
@@ -312,6 +313,36 @@ class LoadTest(test.TestCase):
     x = constant_op.constant(1.0)
     self.assertAllEqual(imported(x).numpy(), 3.0)
 
+  def test_soft_matching(self):
+
+    @def_function.function(
+        input_signature=[tensor_spec.TensorSpec([None], dtypes.int32)])
+    def func(x):
+      return 2 * x
+
+    root = tracking.Checkpointable()
+    root.f = func
+
+    self.assertAllEqual([2], root.f(constant_op.constant([1])).numpy())
+    self.assertAllEqual([2, 4], root.f(constant_op.constant([1, 2])).numpy())
+
+    self.assertEqual(
+        1, len(function_serialization.list_all_concrete_functions(root.f)))
+
+    imported = self.cycle(root)
+
+    with self.assertRaises(AssertionError):
+      # We cannot call the function with a constant of shape ().
+      self.assertEqual(7, imported.f(constant_op.constant(2)).numpy())
+
+    # TODO(vbardiovsky): When classes are revived with input_signatures, we
+    # should also check that the calls below are not generating any more
+    # concrete functions.
+    self.assertAllEqual([2, 4, 6, 8],
+                        imported.f(constant_op.constant([1, 2, 3, 4])).numpy())
+    self.assertAllEqual([2, 4, 6],
+                        imported.f(constant_op.constant([1, 2, 3])).numpy())
+
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 2af977ba922fe498e30c1fcc30942709f75d2e78 Mon Sep 17 00:00:00 2001
From: Nutti <nutti.metro@gmail.com>
Date: Thu, 10 Jan 2019 23:07:08 +0900
Subject: [PATCH 0488/2345] Fix: typo in SavedModelBundle.java

---
 .../java/src/main/java/org/tensorflow/SavedModelBundle.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/java/src/main/java/org/tensorflow/SavedModelBundle.java b/tensorflow/java/src/main/java/org/tensorflow/SavedModelBundle.java
index 49594e6b47..e653373f85 100644
--- a/tensorflow/java/src/main/java/org/tensorflow/SavedModelBundle.java
+++ b/tensorflow/java/src/main/java/org/tensorflow/SavedModelBundle.java
@@ -84,7 +84,7 @@ public class SavedModelBundle implements AutoCloseable {
    * <p>This method is a shorthand for:
    *
    * <pre>{@code
-   * SavedModelBundler.loader().withTags(tags).load();
+   * SavedModelBundle.loader().withTags(tags).load();
    * }</pre>
    *
    * @param exportDir the directory path containing a saved model.
-- 
GitLab


From 9ad0810fd55096fc86a58300c5a2710b2f3b5175 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Thu, 10 Jan 2019 08:15:42 -0800
Subject: [PATCH 0489/2345] [XLA:Python] Fix build error in opensource due to
 xla::int64/protobuf_int64 differences.

PiperOrigin-RevId: 228713538
---
 tensorflow/compiler/xla/python/local_computation_builder.i | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/python/local_computation_builder.i b/tensorflow/compiler/xla/python/local_computation_builder.i
index cc96d84724..66ecee5c4d 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.i
+++ b/tensorflow/compiler/xla/python/local_computation_builder.i
@@ -171,7 +171,7 @@ bool HandleStringAttribute(PyObject* o,
 
 bool HandleRepeatedInt64Attribute(
     PyObject* o, const char* attr_name,
-    tensorflow::protobuf::RepeatedField<int64>* field) {
+    tensorflow::protobuf::RepeatedField<tensorflow::protobuf_int64>* field) {
   PyObject* seq = PyObject_GetAttrString(o, attr_name);
   if (!seq) {
     return false;
-- 
GitLab


From 4e462bc347ddadeb939e4861f77fa349c083fe64 Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Thu, 10 Jan 2019 09:31:51 -0800
Subject: [PATCH 0490/2345] Update unit-test to enable for OSS

PiperOrigin-RevId: 228724452
---
 tensorflow/lite/models/smartreply/BUILD           |  1 -
 .../lite/models/smartreply/predictor_test.cc      | 15 +++++++++------
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/tensorflow/lite/models/smartreply/BUILD b/tensorflow/lite/models/smartreply/BUILD
index a3f4e61cb5..fccbf79eb9 100644
--- a/tensorflow/lite/models/smartreply/BUILD
+++ b/tensorflow/lite/models/smartreply/BUILD
@@ -58,7 +58,6 @@ tf_cc_test(
         "//tensorflow/lite/models:testdata/smartreply_samples.tsv",
         "@tflite_smartreply//:smartreply.tflite",
     ],
-    tags = ["no_oss"],
     deps = [
         ":predictor_lib",
         "//tensorflow/core:test",
diff --git a/tensorflow/lite/models/smartreply/predictor_test.cc b/tensorflow/lite/models/smartreply/predictor_test.cc
index 9bdd7b537a..9ce8ce9a77 100644
--- a/tensorflow/lite/models/smartreply/predictor_test.cc
+++ b/tensorflow/lite/models/smartreply/predictor_test.cc
@@ -31,12 +31,16 @@ namespace custom {
 namespace smartreply {
 namespace {
 
-const char kModelName[] = "smartreply_ondevice_model.bin";
 const char kSamples[] = "smartreply_samples.tsv";
 
-string TestDataPath() {
+string GetModelFilePath() {
+  return "third_party/tensorflow/lite/models/testdata/"
+         "smartreply_ondevice_model.bin";
+}
+
+string GetSamplesFilePath() {
   return string(absl::StrCat(tensorflow::testing::TensorFlowSrcRoot(), "/",
-                             "lite/models/testdata/"));
+                             "lite/models/testdata/", kSamples));
 }
 
 MATCHER_P(IncludeAnyResponesIn, expected_response, "contains the response") {
@@ -57,8 +61,7 @@ class PredictorTest : public ::testing::Test {
   ~PredictorTest() override {}
 
   void SetUp() override {
-    model_ = tflite::FlatBufferModel::BuildFromFile(
-        absl::StrCat(TestDataPath(), "/", kModelName).c_str());
+    model_ = tflite::FlatBufferModel::BuildFromFile(GetModelFilePath().c_str());
     ASSERT_NE(model_.get(), nullptr);
   }
 
@@ -123,7 +126,7 @@ TEST_F(PredictorTest, BatchTest) {
   int total_triggers = 0;
 
   string line;
-  std::ifstream fin(absl::StrCat(TestDataPath(), "/", kSamples));
+  std::ifstream fin(GetSamplesFilePath());
   while (std::getline(fin, line)) {
     const std::vector<string> fields = absl::StrSplit(line, '\t');
     if (fields.empty()) {
-- 
GitLab


From 73b6d9262ea6d1578374ecb63560ad0fcad13e35 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 09:55:14 -0800
Subject: [PATCH 0491/2345] Split Keras TPU correctness tests by models.

PiperOrigin-RevId: 228727689
---
 tensorflow/contrib/distribute/python/BUILD    |  30 +-
 ...test.py => keras_correctness_test_base.py} | 261 ++++--------------
 .../python/keras_dnn_correctness_test.py      | 163 +++++++++++
 .../keras_image_model_correctness_test.py     |  92 ++++++
 4 files changed, 337 insertions(+), 209 deletions(-)
 rename tensorflow/contrib/distribute/python/{keras_correctness_test.py => keras_correctness_test_base.py} (58%)
 create mode 100644 tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
 create mode 100644 tensorflow/contrib/distribute/python/keras_image_model_correctness_test.py

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index d4758d7518..4c25f49f16 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -657,7 +657,11 @@ cuda_py_test(
 py_library(
     name = "keras_correctness_test_lib",
     testonly = 1,
-    srcs = ["keras_correctness_test.py"],
+    srcs = [
+        "keras_correctness_test_base.py",
+        "keras_dnn_correctness_test.py",
+        "keras_image_model_correctness_test.py",
+    ],
     deps = [
         ":combinations",
         "//tensorflow/contrib/distribute/python:mirrored_strategy",
@@ -673,8 +677,9 @@ py_library(
 )
 
 cuda_py_test(
-    name = "keras_correctness_test",
-    srcs = ["keras_correctness_test.py"],
+    name = "keras_dnn_correctness_test",
+    size = "medium",
+    srcs = ["keras_dnn_correctness_test.py"],
     additional_deps = [
         ":keras_correctness_test_lib",
     ],
@@ -690,6 +695,25 @@ cuda_py_test(
     ],
 )
 
+cuda_py_test(
+    name = "keras_image_model_correctness_test",
+    size = "medium",
+    srcs = ["keras_image_model_correctness_test.py"],
+    additional_deps = [
+        ":keras_correctness_test_lib",
+    ],
+    # Shard count is set to an odd number to distribute tasks across
+    # shards more evenly.
+    shard_count = 31,
+    tags = [
+        "multi_and_single_gpu",
+        "no_oss",  # TODO(b/117919883): Fix python error.
+        "no_pip",
+        "no_windows_gpu",
+        "notsan",
+    ],
+)
+
 py_library(
     name = "metrics_v1_test_lib",
     testonly = 1,
diff --git a/tensorflow/contrib/distribute/python/keras_correctness_test.py b/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
similarity index 58%
rename from tensorflow/contrib/distribute/python/keras_correctness_test.py
rename to tensorflow/contrib/distribute/python/keras_correctness_test_base.py
index 3abdee2c0e..e13984b26d 100644
--- a/tensorflow/contrib/distribute/python/keras_correctness_test.py
+++ b/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
@@ -30,8 +30,6 @@ from tensorflow.python.eager import context
 from tensorflow.python.eager import test
 from tensorflow.python.framework import random_seed
 from tensorflow.python.keras.engine import distributed_training_utils
-from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_descent_keras
-from tensorflow.python.training import gradient_descent
 
 _RANDOM_SEED = 1337
 _EVAL_STEPS = 20
@@ -62,24 +60,13 @@ def all_strategy_combinations_with_graph_mode():
   return combinations.combine(distribution=all_strategies, mode=['graph'])
 
 
-def strategy_and_input_combinations():
-  def cnn_model_with_batch_norm(**kwargs):
-    return _create_cnn_model(with_batch_norm=True, **kwargs)
-
+def all_strategy_and_input_config_combinations():
   return (
       combinations.times(
           combinations.combine(distribution=all_strategies),
           combinations.combine(mode=['graph', 'eager'],
                                use_numpy=[True, False],
-                               use_validation_data=[True, False]),
-          combinations.combine(model_with_data=[
-              ModelWithData('dnn', _create_dnn_model, _dnn_training_data),
-              ModelWithData('cnn', _create_cnn_model, _cnn_training_data),
-              ModelWithData('cnn_batch_norm',
-                            cnn_model_with_batch_norm,
-                            _cnn_training_data,
-                            with_batch_norm=True),
-          ])))
+                               use_validation_data=[True, False])))
 
 
 class MaybeDistributionScope(object):
@@ -100,97 +87,6 @@ class MaybeDistributionScope(object):
       self._scope = None
 
 
-class ModelWithData(object):
-  """An object giving a good name in combinations.
-
-  The model_fn must take two arguments: initial_weights and distribution.
-  """
-
-  def __init__(self, name, model_fn, data_fn, with_batch_norm=False):
-    self.name = name
-    self.model_fn = model_fn
-    self.data_fn = data_fn
-    self.with_batch_norm = with_batch_norm
-
-  def __repr__(self):
-    return self.name
-
-
-def _dnn_training_data():
-  # TODO(xiejw): Change this back to 10000, once we support final partial
-  # batch.
-  num_samples = 9984
-  x_train = np.random.rand(num_samples, 1)
-  y_train = 3 * x_train
-  x_train = x_train.astype('float32')
-  y_train = y_train.astype('float32')
-  x_predict = [[1.], [2.], [3.], [4.]]
-  return x_train, y_train, x_predict
-
-
-def _create_dnn_model(initial_weights=None, distribution=None):
-  with MaybeDistributionScope(distribution):
-    # We add few non-linear layers to make it non-trivial.
-    model = keras.Sequential()
-    model.add(keras.layers.Dense(10, activation='relu', input_shape=(1,)))
-    model.add(keras.layers.Dense(10, activation='relu'))
-    model.add(keras.layers.Dense(10, activation='relu'))
-    model.add(keras.layers.Dense(1))
-
-    if initial_weights:
-      model.set_weights(initial_weights)
-
-    model.compile(
-        loss=keras.losses.mean_squared_error,
-        optimizer=gradient_descent_keras.SGD(0.5),
-        metrics=['mse'])
-    return model
-
-
-def _cnn_training_data(count=_GLOBAL_BATCH_SIZE * _EVAL_STEPS,
-                       shape=(28, 28, 3), num_classes=10):
-  centers = np.random.randn(num_classes, *shape)
-
-  features = []
-  labels = []
-  for _ in range(count):
-    label = np.random.randint(0, num_classes, size=1)[0]
-    offset = np.random.normal(loc=0, scale=0.1, size=np.prod(shape))
-    offset = offset.reshape(shape)
-    labels.append(label)
-    features.append(centers[label] + offset)
-
-  x_train = np.asarray(features, dtype=np.float32)
-  y_train = np.asarray(labels, dtype=np.float32).reshape((count, 1))
-  x_predict = x_train
-  return x_train, y_train, x_predict
-
-
-def _create_cnn_model(initial_weights=None, distribution=None,
-                      with_batch_norm=False):
-  with MaybeDistributionScope(distribution):
-    image = keras.layers.Input(shape=(28, 28, 3), name='image')
-    c1 = keras.layers.Conv2D(
-        name='conv1', filters=16, kernel_size=(3, 3), strides=(4, 4))(
-            image)
-    if with_batch_norm:
-      c1 = keras.layers.BatchNormalization(name='bn1')(c1)
-    c1 = keras.layers.MaxPooling2D(pool_size=(2, 2))(c1)
-    logits = keras.layers.Dense(
-        10, activation='softmax', name='pred')(
-            keras.layers.Flatten()(c1))
-    model = keras.Model(inputs=[image], outputs=[logits])
-
-    if initial_weights:
-      model.set_weights(initial_weights)
-
-    model.compile(
-        optimizer=gradient_descent.GradientDescentOptimizer(learning_rate=0.1),
-        loss='sparse_categorical_crossentropy',
-        metrics=['sparse_categorical_accuracy'])
-  return model
-
-
 def batch_wrapper(dataset, batch_size, distribution, repeat=None):
   if repeat:
     dataset = dataset.repeat(repeat)
@@ -350,6 +246,11 @@ def compare_results(results_with_ds, results_without_ds, distribution,
         msg='Fail to assert {}.'.format(key))
 
 
+def should_skip_tpu_with_eager(distribution):
+  return (context.executing_eagerly() and
+          isinstance(distribution, tpu_strategy.TPUStrategy))
+
+
 class LearningRateBatchScheduler(keras.callbacks.Callback):
 
   def __init__(self, update_freq=None):
@@ -364,107 +265,60 @@ class LearningRateBatchScheduler(keras.callbacks.Callback):
     keras.backend.set_value(self.model.optimizer.lr, lr)
 
 
-class TestDistributionStrategyCorrectness(test.TestCase,
-                                          parameterized.TestCase):
+class TestDistributionStrategyCorrectnessBase(test.TestCase,
+                                              parameterized.TestCase):
+  """Model agnostic testing infra to test correctness of Keras models."""
+
+  def set_up_test_config(self, use_numpy=False,
+                         use_validation_data=False,
+                         with_batch_norm=False):
+    self.use_numpy = use_numpy
+    self.use_validation_data = use_validation_data
+    self.with_batch_norm = with_batch_norm
 
-  def _should_skip_tpu_with_eager(self, distribution):
-    return (context.executing_eagerly() and
-            isinstance(distribution, tpu_strategy.TPUStrategy))
+    keras.backend.set_image_data_format('channels_last')
+    np.random.seed(_RANDOM_SEED)
+    random_seed.set_random_seed(_RANDOM_SEED)
 
-  @combinations.generate(all_strategy_combinations_with_eager_and_graph_modes())
-  def test_metric_correctness(self, distribution):
-    if self._should_skip_tpu_with_eager(distribution):
-      self.skipTest('TPUStrategy does not support eager mode now.')
+  def get_data(self):
+    num_samples = 10000
+    x_train = np.random.randint(0, 2, num_samples)
+    x_train = np.reshape(x_train, (num_samples, 1))
+    y_train = x_train
+    return (x_train.astype('float32'), y_train.astype('float32'), None)
 
-    with self.cached_session():
-      keras.backend.set_image_data_format('channels_last')
-      num_samples = 10000
-
-      x_train = np.random.randint(0, 2, num_samples)
-      x_train = np.reshape(x_train, (num_samples, 1))
-      y_train = x_train
-      x_train = x_train.astype('float32')
-      y_train = y_train.astype('float32')
-
-      # Create identity model.
-      with distribution.scope():
-        model = keras.Sequential()
-        model.add(
-            keras.layers.Dense(1, input_shape=(1,), kernel_initializer='ones'))
-        model.compile(
-            loss=keras.losses.mean_squared_error,
-            optimizer=gradient_descent.GradientDescentOptimizer(0.5),
-            metrics=[keras.metrics.BinaryAccuracy()])
-
-      batch_size = 64
-      batch_size = get_batch_size(batch_size, distribution)
-      train_dataset = dataset_ops.Dataset.from_tensor_slices((x_train, y_train))
-      train_dataset = batch_wrapper(train_dataset, batch_size, distribution)
-
-      history = model.fit(x=train_dataset, epochs=2, steps_per_epoch=10)
-      self.assertEqual(history.history['binary_accuracy'], [1.0, 1.0])
-
-  @combinations.generate(all_strategy_combinations_with_eager_and_graph_modes())
-  def test_eval_metrics_correctness(self, distribution):
-    if self._should_skip_tpu_with_eager(distribution):
-      self.skipTest('TPUStrategy does not support eager mode now.')
+  def get_model(self, distribution=None):
+    raise NotImplementedError
 
-    with self.cached_session():
-      with distribution.scope():
-        model = keras.Sequential()
-        model.add(
-            keras.layers.Dense(
-                3, activation='relu', input_dim=4, kernel_initializer='ones'))
-        model.add(
-            keras.layers.Dense(
-                1, activation='sigmoid', kernel_initializer='ones'))
-        model.compile(
-            loss='mae',
-            metrics=['accuracy', keras.metrics.BinaryAccuracy()],
-            optimizer=gradient_descent.GradientDescentOptimizer(0.001))
-
-      # verify correctness of stateful and stateless metrics.
-      x = np.ones((100, 4)).astype('float32')
-      y = np.ones((100, 1)).astype('float32')
-      dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat()
-      dataset = batch_wrapper(dataset, 4, distribution)
-      outs = model.evaluate(dataset, steps=10)
-      self.assertEqual(outs[1], 1.)
-      self.assertEqual(outs[2], 1.)
-
-      y = np.zeros((100, 1)).astype('float32')
-      dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat()
-      dataset = batch_wrapper(dataset, 4, distribution)
-      outs = model.evaluate(dataset, steps=10)
-      self.assertEqual(outs[1], 0.)
-      self.assertEqual(outs[2], 0.)
-
-  @combinations.generate(strategy_and_input_combinations())
-  def test_correctness(self, distribution, use_numpy, use_validation_data,
-                       model_with_data):
-    if self._should_skip_tpu_with_eager(distribution):
+  def skip_unsupported_test_configuration(self, distribution):
+    if should_skip_tpu_with_eager(distribution):
       self.skipTest('TPUStrategy does not support eager mode now.')
 
-    if context.executing_eagerly() and use_numpy:
+    if context.executing_eagerly() and self.use_numpy:
       self.skipTest('Numpy as inputs is not supported with strategy in eager.')
 
-    if context.executing_eagerly() and use_validation_data:
-      self.skipTest('TODO')
+    if context.executing_eagerly() and self.use_validation_data:
+      self.skipTest('TODO(hongjunchoi): Add test logic for using validation '
+                    'data for eager execution.')
+    return
 
+  def run_correctness_test(self,
+                           distribution,
+                           use_numpy,
+                           use_validation_data,
+                           with_batch_norm=False):
     with self.cached_session():
-      keras.backend.set_image_data_format('channels_last')
-      np.random.seed(_RANDOM_SEED)
-      random_seed.set_random_seed(_RANDOM_SEED)
+      self.set_up_test_config(use_numpy, use_validation_data, with_batch_norm)
+      self.skip_unsupported_test_configuration(distribution)
 
-      model_fn, data_fn = model_with_data.model_fn, model_with_data.data_fn
       # Train, eval, and predict datasets are created with the same input numpy
       # arrays.
-      x_train, y_train, x_predict = data_fn()
+      x_train, y_train, x_predict = self.get_data()
 
       # The model is built once and the initial weights are saved.
       # This is used to initialize the model for both the distribution and
       # non-distribution run.
-      model = model_fn()
+      model = self.get_model()
       initial_weights = model.get_weights()
 
       def input_fn(dist):
@@ -472,15 +326,15 @@ class TestDistributionStrategyCorrectness(test.TestCase,
             use_numpy, use_validation_data, dist, x_train, y_train, x_predict)
 
       results_with_ds = fit_eval_and_predict(
-          initial_weights, input_fn=input_fn, model_fn=model_fn,
+          initial_weights, input_fn=input_fn, model_fn=self.get_model,
           distribution=distribution)
       results_without_ds = fit_eval_and_predict(
-          initial_weights, input_fn=input_fn, model_fn=model_fn,
+          initial_weights, input_fn=input_fn, model_fn=self.get_model,
           distribution=None)
 
       # First, special case, for multi-replica distributed training, batch norm
       # is not aggregated globally. So it is expected to have different weights.
-      if (model_with_data.with_batch_norm and
+      if (self.with_batch_norm and
           distribution.num_replicas_in_sync > 1):
         with self.assertRaises(AssertionError):
           compare_results(results_with_ds, results_without_ds, distribution,
@@ -489,21 +343,16 @@ class TestDistributionStrategyCorrectness(test.TestCase,
         compare_results(results_with_ds, results_without_ds, distribution,
                         testcase=self)
 
-  @combinations.generate(all_strategy_combinations_with_graph_mode())
-  def test_dynamic_lr(self, distribution):
-
+  def run_dynamic_lr_test(self, distribution):
     with self.cached_session():
+      self.set_up_test_config()
+      self.skip_unsupported_test_configuration(distribution)
 
-      keras.backend.set_image_data_format('channels_last')
-      np.random.seed(_RANDOM_SEED)
-      random_seed.set_random_seed(_RANDOM_SEED)
-
-      x_train, y_train, _ = _dnn_training_data()
-
-      model = _create_dnn_model()
+      x_train, y_train, _ = self.get_data()
+      model = self.get_model()
       initial_weights = model.get_weights()
-
       update_freq = None
+
       if (isinstance(distribution, tpu_strategy.TPUStrategy) and
           distribution.extended.steps_per_run > 1):
         # For TPUStrategy with steps_per_run > 1, the callback is not invoked
@@ -530,10 +379,10 @@ class TestDistributionStrategyCorrectness(test.TestCase,
         return training_inputs, eval_inputs, predict_inputs
 
       results_with_ds = fit_eval_and_predict(
-          initial_weights, input_fn=input_fn, model_fn=_create_dnn_model,
+          initial_weights, input_fn=input_fn, model_fn=self.get_model,
           distribution=distribution)
       results_without_ds = fit_eval_and_predict(
-          initial_weights, input_fn=input_fn, model_fn=_create_dnn_model,
+          initial_weights, input_fn=input_fn, model_fn=self.get_model,
           distribution=None)
       compare_results(results_with_ds, results_without_ds, distribution,
                       testcase=self)
diff --git a/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py b/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
new file mode 100644
index 0000000000..f21b1c6752
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
@@ -0,0 +1,163 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Correctness tests for tf.keras DNN model using DistributionStrategy."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.distribute.python import combinations
+from tensorflow.contrib.distribute.python import keras_correctness_test_base
+from tensorflow.python import keras
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.eager import test
+from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_descent_keras
+from tensorflow.python.training import gradient_descent
+
+
+class TestDistributionStrategyDnnCorrectness(
+    keras_correctness_test_base.TestDistributionStrategyCorrectnessBase):
+
+  def get_model(self, initial_weights=None, distribution=None):
+    with keras_correctness_test_base.MaybeDistributionScope(distribution):
+      # We add few non-linear layers to make it non-trivial.
+      model = keras.Sequential()
+      model.add(keras.layers.Dense(10, activation='relu', input_shape=(1,)))
+      model.add(keras.layers.Dense(10, activation='relu'))
+      model.add(keras.layers.Dense(10, activation='relu'))
+      model.add(keras.layers.Dense(1))
+
+      if initial_weights:
+        model.set_weights(initial_weights)
+
+      model.compile(
+          loss=keras.losses.mean_squared_error,
+          optimizer=gradient_descent_keras.SGD(0.5),
+          metrics=['mse'])
+      return model
+
+  def get_data(self):
+    # TODO(xiejw): Change this back to 10000, once we support final partial
+    # batch.
+    num_samples = 9984
+    x_train = np.random.rand(num_samples, 1)
+    y_train = 3 * x_train
+    x_train = x_train.astype('float32')
+    y_train = y_train.astype('float32')
+    x_predict = [[1.], [2.], [3.], [4.]]
+    return x_train, y_train, x_predict
+
+  @combinations.generate(keras_correctness_test_base.
+                         all_strategy_and_input_config_combinations())
+  def test_dnn_correctness(self, distribution, use_numpy, use_validation_data):
+    self.run_correctness_test(distribution, use_numpy, use_validation_data)
+
+  @combinations.generate(keras_correctness_test_base.
+                         all_strategy_combinations_with_graph_mode())
+  def test_dnn_with_dynamic_learning_rate(self, distribution):
+    self.run_dynamic_lr_test(distribution)
+
+
+class TestDistributionStrategyDnnMetricCorrectness(
+    keras_correctness_test_base.TestDistributionStrategyCorrectnessBase):
+
+  def get_model(self, distribution=None):
+    with distribution.scope():
+      model = keras.Sequential()
+      model.add(keras.layers.Dense(1,
+                                   input_shape=(1,),
+                                   kernel_initializer='ones'))
+      model.compile(
+          loss=keras.losses.mean_squared_error,
+          optimizer=gradient_descent.GradientDescentOptimizer(0.5),
+          metrics=[keras.metrics.BinaryAccuracy()])
+    return model
+
+  def run_metric_correctness_test(self, distribution):
+    with self.cached_session():
+      self.set_up_test_config()
+      self.skip_unsupported_test_configuration(distribution)
+
+      x_train, y_train, _ = self.get_data()
+      model = self.get_model(distribution=distribution)
+
+      batch_size = 64
+      batch_size = (keras_correctness_test_base.
+                    get_batch_size(batch_size, distribution))
+      train_dataset = dataset_ops.Dataset.from_tensor_slices((x_train, y_train))
+      train_dataset = (keras_correctness_test_base.
+                       batch_wrapper(train_dataset, batch_size, distribution))
+
+      history = model.fit(x=train_dataset, epochs=2, steps_per_epoch=10)
+      self.assertEqual(history.history['binary_accuracy'], [1.0, 1.0])
+
+  @combinations.generate(keras_correctness_test_base.
+                         all_strategy_combinations_with_eager_and_graph_modes())
+  def test_simple_dnn_metric_correctness(self, distribution):
+    self.run_metric_correctness_test(distribution)
+
+
+class TestDistributionStrategyDnnMetricEvalCorrectness(
+    keras_correctness_test_base.TestDistributionStrategyCorrectnessBase):
+
+  def get_model(self, distribution=None):
+    with distribution.scope():
+      model = keras.Sequential()
+      model.add(
+          keras.layers.Dense(
+              3, activation='relu', input_dim=4, kernel_initializer='ones'))
+      model.add(
+          keras.layers.Dense(
+              1, activation='sigmoid', kernel_initializer='ones'))
+      model.compile(
+          loss='mae',
+          metrics=['accuracy', keras.metrics.BinaryAccuracy()],
+          optimizer=gradient_descent.GradientDescentOptimizer(0.001))
+    return model
+
+  def run_eval_metrics_correctness_test(self, distribution):
+    with self.cached_session():
+      self.set_up_test_config()
+      self.skip_unsupported_test_configuration(distribution)
+
+      model = self.get_model(distribution=distribution)
+
+      # verify correctness of stateful and stateless metrics.
+      x = np.ones((100, 4)).astype('float32')
+      y = np.ones((100, 1)).astype('float32')
+      dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat()
+      dataset = (keras_correctness_test_base.
+                 batch_wrapper(dataset, 4, distribution))
+      outs = model.evaluate(dataset, steps=10)
+      self.assertEqual(outs[1], 1.)
+      self.assertEqual(outs[2], 1.)
+
+      y = np.zeros((100, 1)).astype('float32')
+      dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat()
+      dataset = (keras_correctness_test_base.
+                 batch_wrapper(dataset, 4, distribution))
+      outs = model.evaluate(dataset, steps=10)
+      self.assertEqual(outs[1], 0.)
+      self.assertEqual(outs[2], 0.)
+
+  @combinations.generate(keras_correctness_test_base.
+                         all_strategy_combinations_with_eager_and_graph_modes())
+  def test_identity_model_metric_eval_correctness(self, distribution):
+    self.run_eval_metrics_correctness_test(distribution)
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/contrib/distribute/python/keras_image_model_correctness_test.py b/tensorflow/contrib/distribute/python/keras_image_model_correctness_test.py
new file mode 100644
index 0000000000..f625664372
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/keras_image_model_correctness_test.py
@@ -0,0 +1,92 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Correctness tests for tf.keras CNN models using DistributionStrategy."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.distribute.python import combinations
+from tensorflow.contrib.distribute.python import keras_correctness_test_base
+from tensorflow.python import keras
+from tensorflow.python.eager import test
+from tensorflow.python.training import gradient_descent
+
+
+class DistributionStrategyCnnCorrectnessTest(
+    keras_correctness_test_base.TestDistributionStrategyCorrectnessBase):
+
+  def get_model(self, initial_weights=None, distribution=None):
+    with keras_correctness_test_base.MaybeDistributionScope(distribution):
+      image = keras.layers.Input(shape=(28, 28, 3), name='image')
+      c1 = keras.layers.Conv2D(
+          name='conv1', filters=16, kernel_size=(3, 3), strides=(4, 4))(
+              image)
+      if self.with_batch_norm:
+        c1 = keras.layers.BatchNormalization(name='bn1')(c1)
+      c1 = keras.layers.MaxPooling2D(pool_size=(2, 2))(c1)
+      logits = keras.layers.Dense(
+          10, activation='softmax', name='pred')(
+              keras.layers.Flatten()(c1))
+      model = keras.Model(inputs=[image], outputs=[logits])
+
+      if initial_weights:
+        model.set_weights(initial_weights)
+
+      model.compile(
+          optimizer=gradient_descent.GradientDescentOptimizer(
+              learning_rate=0.1),
+          loss='sparse_categorical_crossentropy',
+          metrics=['sparse_categorical_accuracy'])
+
+    return model
+
+  def get_data(self,
+               count=keras_correctness_test_base._GLOBAL_BATCH_SIZE
+               * keras_correctness_test_base._EVAL_STEPS,
+               shape=(28, 28, 3),
+               num_classes=10):
+    centers = np.random.randn(num_classes, *shape)
+
+    features = []
+    labels = []
+    for _ in range(count):
+      label = np.random.randint(0, num_classes, size=1)[0]
+      offset = np.random.normal(loc=0, scale=0.1, size=np.prod(shape))
+      offset = offset.reshape(shape)
+      labels.append(label)
+      features.append(centers[label] + offset)
+
+    x_train = np.asarray(features, dtype=np.float32)
+    y_train = np.asarray(labels, dtype=np.float32).reshape((count, 1))
+    x_predict = x_train
+    return x_train, y_train, x_predict
+
+  @combinations.generate(keras_correctness_test_base.
+                         all_strategy_and_input_config_combinations())
+  def test_cnn_correctness(self, distribution, use_numpy, use_validation_data):
+    self.run_correctness_test(distribution, use_numpy, use_validation_data)
+
+  @combinations.generate(keras_correctness_test_base.
+                         all_strategy_and_input_config_combinations())
+  def test_cnn_with_batch_norm_correctness(self, distribution, use_numpy,
+                                           use_validation_data):
+    self.run_correctness_test(distribution, use_numpy, use_validation_data,
+                              with_batch_norm=True)
+
+
+if __name__ == '__main__':
+  test.main()
-- 
GitLab


From 1210b6506dbde52a5bf5c33d56f3c52dfbd505e7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 10:28:19 -0800
Subject: [PATCH 0492/2345] fix links to renamed markdown documentation

PiperOrigin-RevId: 228733282
---
 tensorflow/lite/g3doc/apis.md | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tensorflow/lite/g3doc/apis.md b/tensorflow/lite/g3doc/apis.md
index b15159ce41..1a05142bc4 100644
--- a/tensorflow/lite/g3doc/apis.md
+++ b/tensorflow/lite/g3doc/apis.md
@@ -1,4 +1,3 @@
-
 # TensorFlow Lite APIs
 
 TensorFlow Lite provides programming APIs in C++ and Java, and in both cases
@@ -8,8 +7,7 @@ no surprise that the APIs try to avoid unnecessary copies at the expense of
 convenience.  Similarly, consistency with TensorFlow APIs was not an explicit
 goal and some variance is to be expected.
 
-There is also a Python API for TensorFlow Lite described
-[here](../toco/g3doc/python_api.md#interpreter).
+There is also a [Python API for TensorFlow Lite](g3doc/convert/python_api.md).
 
 ## C++
 
-- 
GitLab


From 3c4b35297e4b44baaf4cc7ed5b099c586ffa049b Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Thu, 10 Jan 2019 10:44:31 -0800
Subject: [PATCH 0493/2345] Re-apply "Don't check for alignment before calling
 into a tiled dot (Eigen or LLVM)" with fix

PiperOrigin-RevId: 228736280
---
 tensorflow/compiler/xla/service/cpu/BUILD     |   2 -
 .../cpu/cpu_eigen_tensor_alignment_test.cc    |  38 ------
 .../xla/service/cpu/dot_op_emitter.cc         | 117 ++++++++++++------
 3 files changed, 77 insertions(+), 80 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index de62aa60aa..a197bdddc8 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -385,7 +385,6 @@ cc_library(
     srcs = ["dot_op_emitter.cc"],
     hdrs = [
         "dot_op_emitter.h",
-        "dot_op_emitter_internal.h",
     ],
     deps = [
         ":cpu_options",
@@ -1029,7 +1028,6 @@ tf_cc_test(
     size = "small",
     srcs = ["cpu_eigen_tensor_alignment_test.cc"],
     deps = [
-        ":dot_op_emitter",
         ":ir_emission_utils",
         ":target_machine_features_fake",
         "//tensorflow/compiler/xla:test",
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc b/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc
index 823bdf259c..485769a373 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc
@@ -13,7 +13,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h"
 #include "tensorflow/compiler/xla/service/cpu/ir_emission_utils.h"
 #include "tensorflow/compiler/xla/service/cpu/target_machine_features_fake.h"
 #include "tensorflow/compiler/xla/service/hlo_parser.h"
@@ -23,48 +22,11 @@ namespace xla {
 namespace cpu {
 namespace {
 
-using internal::DotImplementationStrategy;
-using internal::DotInfo;
-using internal::GetDotImplementationStrategy;
-
 // Test that we don't call into Eigen with tensors too small to be aligned
 // reliably.
 
 class CpuEigenTensorAlignmentTest : public ::testing::Test {};
 
-TEST_F(CpuEigenTensorAlignmentTest, EigenDotAlignment) {
-  string hlo_string = R"(
-HloModule DotOperation
-
-ENTRY DotOperation {
-  arg0 = f32[5,256] parameter(0)
-  arg1 = f32[256,1024] parameter(1)
-  ROOT dot = f32[5,1024] dot(arg0, arg1), lhs_contracting_dims={1}, rhs_contracting_dims={0}
-}
-)";
-
-  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
-                          ParseHloString(hlo_string));
-
-  HloInstruction* dot = module->entry_computation()->root_instruction();
-
-  TargetMachineFeaturesWithFakeAlignmentLogic target_machine_with_no_alignment(
-      [](int64 size) { return 1; });
-
-  EXPECT_EQ(GetDotImplementationStrategy(HloModuleConfig{}, DotInfo(*dot),
-                                         target_machine_with_no_alignment),
-            DotImplementationStrategy::kNaiveLlvmIr);
-
-  TargetMachineFeaturesWithFakeAlignmentLogic
-      target_machine_with_full_alignment([](int64 size) {
-        return TargetMachineFeatures::kEigenExpectedTensorAlignment;
-      });
-
-  EXPECT_NE(GetDotImplementationStrategy(HloModuleConfig{}, DotInfo(*dot),
-                                         target_machine_with_full_alignment),
-            DotImplementationStrategy::kNaiveLlvmIr);
-}
-
 TEST_F(CpuEigenTensorAlignmentTest, EigenConvAlignment) {
   string hlo_string = R"(
 HloModule ConvOperation
diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
index e8a84ebe6b..b018e0cd46 100644
--- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
@@ -14,7 +14,6 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/compiler/xla/service/cpu/dot_op_emitter.h"
-#include "tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h"
 
 #include <memory>
 #include <vector>
@@ -43,17 +42,63 @@ namespace xla {
 using llvm_ir::SetToFirstInsertPoint;
 
 namespace cpu {
-
-using internal::DotImplementationStrategy;
-using internal::DotInfo;
-using internal::GetDotImplementationStrategy;
-
 namespace {
 // Returns true if we should call into multi-threaded Eigen routines.
 bool ShouldUseMultiThreadedEigen(const HloModuleConfig& config) {
   return config.debug_options().xla_cpu_multi_thread_eigen();
 }
 
+// Represents a dot operation.  We use this in lieu of an `HloInstruction`
+// because we want to be able to create this for the "inner" dot operation in a
+// batch dot, for which there is no separate HLO instruction.
+struct DotInfo {
+  Shape lhs_shape;
+  Shape rhs_shape;
+  Shape result_shape;
+  DotDimensionNumbers dim_nums;
+
+  explicit DotInfo(const HloInstruction& instr) {
+    CHECK_EQ(instr.opcode(), HloOpcode::kDot);
+    lhs_shape = instr.operand(0)->shape();
+    rhs_shape = instr.operand(1)->shape();
+    result_shape = instr.shape();
+    dim_nums = instr.dot_dimension_numbers();
+  }
+};
+
+// Dictates how a dot operation is implemented.
+enum class DotImplementationStrategy {
+  // The dot operation is lowered into LLVM IR that implements a naive nested
+  // loop that computes the result one element at a time.  This is our
+  // "fallback"; we don't really want this to kick in for any non-trival dot
+  // operation.
+  kNaiveLlvmIr,
+
+  // The dot operation is lowered into LLVM IR that implements a tiled
+  // Matrix*Vector operation.  This strategy also allows fusing in a bias add
+  // into the dot.  The matrix can be row major or column major, both are
+  // supported.
+  kTiledLlvmIrGemv,
+
+  // The dot operation is lowered into LLVM IR that implemetns a tiled
+  // Matrix*Matrix operation.  No fusions are supported.  The two inputs
+  // and the output have to be row major.
+  kTiledLlvmIrGemm,
+
+  // The dot operation is lowered into a call into an Eigen routine.  No fusions
+  // are supported today.  The two inputs and the output have to be row major.
+  // However, we do allow transposing either the LHS or the RHS as part of the
+  // GEMM -- we expose this flexibility as flexibility in the contraction
+  // dimensions, but we can also see this as flexibility in the input layouts.
+  kEigen,
+};
+
+// Returns the implementation strategy for a dot with the configuration
+// `dot_info`.
+DotImplementationStrategy GetDotImplementationStrategy(
+    const HloModuleConfig& config, const DotInfo& dot_info,
+    const TargetMachineFeatures& target_machine_features);
+
 // Helper class for emitting LLVM IR to perform the dot operation.
 class DotOpEmitter {
  public:
@@ -190,9 +235,8 @@ void DotOpEmitter::EmitTiledLlvmIrGemm() {
   }
 
   int64 size_bytes = m * n * ShapeUtil::ByteSizeOfPrimitiveType(primitive_type);
-  b_->CreateMemSet(
-      target, b_->getInt8(0), size_bytes,
-      target_machine_features_.minimum_alignment_for_allocation(size_bytes));
+  b_->CreateMemSet(target, b_->getInt8(0), /*Size=*/size_bytes,
+                   /*Align=*/1);
 
   int64 max_target_vector_width =
       target_machine_features_.vector_register_num_elements(
@@ -696,40 +740,34 @@ absl::optional<int64> ProfitableToMakeDotOperandColumnMajor(
   return {};
 }
 
-namespace internal {
 namespace {
 // Return whether the given shape is rank 2.
 bool IsRank2(const Shape& shape) { return shape.rank() == 2; }
 
+bool IsSimpleLayout(const Layout& layout) {
+  return layout.tiles().empty() && layout.format() == DENSE;
+}
+
 // In a gemm operation where output = lhs * rhs, check whether the given shapes
 // are valid for the operation.
-bool AreAlignedGemmShapes(
-    const Shape& lhs_shape, const Shape& rhs_shape, const Shape& output_shape,
-    const TargetMachineFeatures& target_machine_features) {
-  // The inputs and the output must
-  // 1) be matrices with no padding, and
-  // 2) have an allowed element type.
-  PrimitiveType output_primitive_type = output_shape.element_type();
-  if (!(output_primitive_type == F64 || output_primitive_type == F32 ||
-        output_primitive_type == F16)) {
-    return false;
-  }
-
-  if (!(IsRank2(lhs_shape) && IsRank2(rhs_shape) && IsRank2(output_shape))) {
-    return false;
-  }
-
-  auto is_aligned = [&](const Shape& shape) {
-    return GetMinimumAlignmentForArray(shape, target_machine_features) >=
-           TargetMachineFeatures::kEigenExpectedTensorAlignment;
-  };
-
-  if (!is_aligned(lhs_shape) || !is_aligned(rhs_shape) ||
-      !is_aligned(output_shape)) {
-    return false;
+bool AreGemmShapes(const Shape& lhs_shape, const Shape& rhs_shape,
+                   const Shape& output_shape,
+                   const TargetMachineFeatures& target_machine_features) {
+  CHECK(!lhs_shape.has_layout() || IsSimpleLayout(lhs_shape.layout()))
+      << lhs_shape.DebugString();
+  CHECK(!rhs_shape.has_layout() || IsSimpleLayout(rhs_shape.layout()))
+      << rhs_shape.DebugString();
+  CHECK(!output_shape.has_layout() || IsSimpleLayout(output_shape.layout()))
+      << output_shape.DebugString();
+
+  switch (output_shape.element_type()) {
+    case F64:
+    case F32:
+    case F16:
+      return IsRank2(lhs_shape) && IsRank2(rhs_shape) && IsRank2(output_shape);
+    default:
+      return false;
   }
-
-  return true;
 }
 
 bool IsAlignedGemm(const DotInfo& dot_info,
@@ -739,8 +777,8 @@ bool IsAlignedGemm(const DotInfo& dot_info,
     return false;
   }
 
-  return AreAlignedGemmShapes(dot_info.lhs_shape, dot_info.rhs_shape,
-                              dot_info.result_shape, target_machine_features);
+  return AreGemmShapes(dot_info.lhs_shape, dot_info.rhs_shape,
+                       dot_info.result_shape, target_machine_features);
 }
 
 bool CanEmitTiledLlvmIrGemm(
@@ -781,7 +819,6 @@ bool CanEmitTiledLlvmIrGemm(
 
   return true;
 }
-}  // namespace
 
 DotImplementationStrategy GetDotImplementationStrategy(
     const HloModuleConfig& config, const DotInfo& dot_info,
@@ -806,7 +843,7 @@ DotImplementationStrategy GetDotImplementationStrategy(
 
   return DotImplementationStrategy::kNaiveLlvmIr;
 }
-}  // namespace internal
+}  // namespace
 
 bool DotImplementationCanHandleTranspose(
     const HloInstruction& dot_instr,
-- 
GitLab


From eae721281d6c5bdd1e84a17de56c1f2cb911fd76 Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Thu, 10 Jan 2019 11:07:54 -0800
Subject: [PATCH 0494/2345] [XLA] Move over some uses of CreateDS / CreateDUS
 to new API

Move over some uses of the R1-index API to use the span-of-scalars one.

PiperOrigin-RevId: 228740894
---
 .../xla/service/algebraic_simplifier_test.cc  | 34 ++++++++++-----
 .../cpu/cpu_instruction_fusion_test.cc        | 43 ++++++++++++-------
 2 files changed, 51 insertions(+), 26 deletions(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index e39321f1f3..916b953cfc 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -3774,12 +3774,16 @@ TEST_F(AlgebraicSimplifierTest, TrivialDynamicSlice) {
   HloComputation::Builder builder(TestName());
 
   Shape shape = ShapeUtil::MakeShape(F32, {10, 100, 1000});
+  std::vector<HloInstruction*> params;
+  for (int i = 0; i < 3; ++i) {
+    params.push_back(builder.AddInstruction(HloInstruction::CreateParameter(
+        i + 1, ShapeUtil::MakeShape(U32, {}), "slice_indices")));
+  }
   builder.AddInstruction(HloInstruction::CreateDynamicSlice(
       shape,
       builder.AddInstruction(
           HloInstruction::CreateParameter(0, shape, "slice_from")),
-      builder.AddInstruction(HloInstruction::CreateParameter(
-          1, ShapeUtil::MakeShape(U32, {3}), "slice_indices")),
+      params,
       /*slice_sizes=*/{10, 100, 1000}));
 
   auto computation = m->AddEntryComputation(builder.Build());
@@ -3798,28 +3802,35 @@ TEST_F(AlgebraicSimplifierTest, TrivialDynamicUpdateSlice) {
   Shape full_shape = ShapeUtil::MakeShape(F32, {10, 100, 1000});
   Shape slice_shape = ShapeUtil::MakeShape(F32, {10, 1, 1000});
 
+  std::vector<HloInstruction*> slice_indices, update_indices;
+  for (int i = 0; i < 3; ++i) {
+    slice_indices.push_back(
+        builder.AddInstruction(HloInstruction::CreateParameter(
+            i + 1, ShapeUtil::MakeShape(U32, {}), "slice_indices")));
+    update_indices.push_back(
+        builder.AddInstruction(HloInstruction::CreateParameter(
+            i + 5, ShapeUtil::MakeShape(U32, {}), "update_indices")));
+  }
   HloInstruction* slice =
       builder.AddInstruction(HloInstruction::CreateDynamicSlice(
           slice_shape,
           builder.AddInstruction(
               HloInstruction::CreateParameter(0, full_shape, "slice_from")),
-          builder.AddInstruction(HloInstruction::CreateParameter(
-              1, ShapeUtil::MakeShape(U32, {3}), "slice_indices")),
+          slice_indices,
           /*slice_sizes=*/{10, 1, 1000}));
 
   builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
       slice_shape,
       builder.AddInstruction(
-          HloInstruction::CreateParameter(2, slice_shape, "to_update")),
-      slice,
-      builder.AddInstruction(HloInstruction::CreateParameter(
-          3, ShapeUtil::MakeShape(U32, {3}), "update_indices"))));
+          HloInstruction::CreateParameter(4, slice_shape, "to_update")),
+      slice, update_indices));
 
   auto computation = m->AddEntryComputation(builder.Build());
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   EXPECT_THAT(computation->root_instruction(),
-              GmockMatch(m::DynamicSlice(m::Parameter(), m::Parameter())));
+              GmockMatch(m::DynamicSlice(m::Parameter(), m::Parameter(),
+                                         m::Parameter(), m::Parameter())));
 }
 
 // Test that two consecutive broadcasts can be merged to one.
@@ -4480,9 +4491,10 @@ TEST_F(AlgebraicSimplifierTest, DynamicUpdateSliceZeroUpdate) {
   HloInstruction* const update = builder.AddInstruction(
       HloInstruction::CreateParameter(1, update_shape, "update"));
   HloInstruction* const start_indices = builder.AddInstruction(
-      HloInstruction::CreateConstant(LiteralUtil::CreateR1<int>({0})));
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int>({})));
   builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
-      dslice_shape, operand, update, start_indices));
+      dslice_shape, operand, update,
+      std::initializer_list<HloInstruction*>({start_indices})));
   const HloComputation* const computation =
       m->AddEntryComputation(builder.Build());
 
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc b/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc
index 527df0bd1c..46c1d4c38e 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc
@@ -332,7 +332,7 @@ TEST_F(OpcodeFusionTest, Exponential_Reshape_Negate) {
 TEST_F(OpcodeFusionTest, Broadcast_Reshape_DynamicSlice_Tanh) {
   HloComputation::Builder builder(TestName());
   Shape param_shape = ShapeUtil::MakeShape(F32, {8});
-  Shape starts_shape = ShapeUtil::MakeShape(F32, {2});
+  Shape starts_shape = ShapeUtil::MakeShape(F32, {});
   Shape broadcast_shape = ShapeUtil::MakeShape(F32, {1, 8, 8});
   Shape reshape_shape = ShapeUtil::MakeShape(F32, {8, 8});
   Shape dynamic_slice_shape = ShapeUtil::MakeShape(F32, {4, 4});
@@ -340,13 +340,15 @@ TEST_F(OpcodeFusionTest, Broadcast_Reshape_DynamicSlice_Tanh) {
       HloInstruction::CreateParameter(0, param_shape, "param"));
   HloInstruction* param1 = builder.AddInstruction(
       HloInstruction::CreateParameter(1, starts_shape, "starts"));
+  HloInstruction* param2 = builder.AddInstruction(
+      HloInstruction::CreateParameter(2, starts_shape, "starts"));
   HloInstruction* broadcast2 = builder.AddInstruction(
       HloInstruction::CreateBroadcast(broadcast_shape, param0, {1}));
   HloInstruction* reshape3 = builder.AddInstruction(
       HloInstruction::CreateReshape(reshape_shape, broadcast2));
   HloInstruction* dynamic_slice4 =
       builder.AddInstruction(HloInstruction::CreateDynamicSlice(
-          dynamic_slice_shape, reshape3, param1, {4, 4}));
+          dynamic_slice_shape, reshape3, {param1, param2}, {4, 4}));
   builder.AddInstruction(HloInstruction::CreateUnary(
       dynamic_slice_shape, HloOpcode::kTanh, dynamic_slice4));
 
@@ -356,7 +358,8 @@ TEST_F(OpcodeFusionTest, Broadcast_Reshape_DynamicSlice_Tanh) {
   RunFusionAndCheckOpcodesWereFused(
       module.get(),
       {HloOpcode::kTanh, HloOpcode::kDynamicSlice, HloOpcode::kReshape,
-       HloOpcode::kBroadcast, HloOpcode::kParameter, HloOpcode::kParameter});
+       HloOpcode::kBroadcast, HloOpcode::kParameter, HloOpcode::kParameter,
+       HloOpcode::kParameter});
 }
 
 TEST_F(OpcodeFusionTest, Broadcast_Negate) {
@@ -381,14 +384,16 @@ TEST_F(OpcodeFusionTest, Broadcast_Negate) {
 TEST_F(OpcodeFusionTest, DynamicSlice_Negate) {
   HloComputation::Builder builder(TestName());
   Shape param_shape = ShapeUtil::MakeShape(F32, {4});
-  Shape slice_shape = ShapeUtil::MakeShape(F32, {1});
+  Shape slice_shape = ShapeUtil::MakeShape(F32, {});
   Shape result_shape = ShapeUtil::MakeShape(F32, {2});
   HloInstruction* param0 = builder.AddInstruction(
       HloInstruction::CreateParameter(0, param_shape, "param"));
   HloInstruction* param1 = builder.AddInstruction(
       HloInstruction::CreateParameter(1, slice_shape, "starts"));
-  HloInstruction* dynamic_slice2 = builder.AddInstruction(
-      HloInstruction::CreateDynamicSlice(result_shape, param0, param1, {2}));
+  HloInstruction* dynamic_slice2 =
+      builder.AddInstruction(HloInstruction::CreateDynamicSlice(
+          result_shape, param0,
+          std::initializer_list<HloInstruction*>({param1}), {2}));
   builder.AddInstruction(HloInstruction::CreateUnary(
       result_shape, HloOpcode::kNegate, dynamic_slice2));
 
@@ -548,28 +553,36 @@ TEST_F(OpcodeFusionTest, DynamicSliceWithDynamicUpdateSlice) {
   Shape full_shape = ShapeUtil::MakeShape(F32, {10, 100, 1000});
   Shape slice_shape = ShapeUtil::MakeShape(F32, {10, 1, 1000});
 
+  std::vector<HloInstruction*> slice_indices, update_indices;
+  for (int i = 0; i < 3; ++i) {
+    slice_indices.push_back(
+        builder.AddInstruction(HloInstruction::CreateParameter(
+            1 + i, ShapeUtil::MakeShape(U32, {}), "slice_indices")));
+    update_indices.push_back(
+        builder.AddInstruction(HloInstruction::CreateParameter(
+            5 + i, ShapeUtil::MakeShape(U32, {}), "update_indices")));
+  }
   HloInstruction* slice =
       builder.AddInstruction(HloInstruction::CreateDynamicSlice(
           slice_shape,
           builder.AddInstruction(
               HloInstruction::CreateParameter(0, full_shape, "slice_from")),
-          builder.AddInstruction(HloInstruction::CreateParameter(
-              1, ShapeUtil::MakeShape(U32, {3}), "slice_indices")),
+          slice_indices,
           /*slice_sizes=*/{10, 1, 1000}));
 
   builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
       full_shape,
       builder.AddInstruction(
-          HloInstruction::CreateParameter(2, full_shape, "to_update")),
-      slice,
-      builder.AddInstruction(HloInstruction::CreateParameter(
-          3, ShapeUtil::MakeShape(U32, {3}), "update_indices"))));
+          HloInstruction::CreateParameter(4, full_shape, "to_update")),
+      slice, update_indices));
 
   module->AddEntryComputation(builder.Build());
   RunFusionAndCheckOpcodesWereFused(
-      module.get(), {HloOpcode::kDynamicSlice, HloOpcode::kDynamicUpdateSlice,
-                     HloOpcode::kParameter, HloOpcode::kParameter,
-                     HloOpcode::kParameter, HloOpcode::kParameter});
+      module.get(),
+      {HloOpcode::kDynamicSlice, HloOpcode::kDynamicUpdateSlice,
+       HloOpcode::kParameter, HloOpcode::kParameter, HloOpcode::kParameter,
+       HloOpcode::kParameter, HloOpcode::kParameter, HloOpcode::kParameter,
+       HloOpcode::kParameter, HloOpcode::kParameter});
 }
 
 TEST_F(OpcodeFusionTest, MessOfFusibleNodes) {
-- 
GitLab


From 8ccb11412614e7d9b702d9a56b6f876af8779f72 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 11:24:15 -0800
Subject: [PATCH 0495/2345] Allows variable targets in gradient tapes. This
 allows using the native python `max` method on lists of variables inside of
 gradient tapes.

PiperOrigin-RevId: 228743966
---
 tensorflow/python/eager/backprop.py      |  9 +++++----
 tensorflow/python/eager/backprop_test.py | 16 ++++++++++++----
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index 7d99f6e95a..42db726a7b 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -930,11 +930,12 @@ class GradientTape(object):
                             "gradient in order to compute higher order "
                             "derrivatives.", 1)
 
-    flat_targets = nest.flatten(target)
-    for t in flat_targets:
+    flat_targets = []
+    for t in nest.flatten(target):
       if resource_variable_ops.is_resource_variable(t):
-        raise ValueError("GradientTape.gradient is not supported for variable "
-                         "targets.")
+        with self:
+          t = ops.convert_to_tensor(t)
+      flat_targets.append(t)
 
     flat_sources = nest.flatten(sources)
     flat_sources = [_handle_or_self(x) for x in flat_sources]
diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py
index 22ae6f74cb..5f4fda8897 100644
--- a/tensorflow/python/eager/backprop_test.py
+++ b/tensorflow/python/eager/backprop_test.py
@@ -354,6 +354,16 @@ class BackpropTest(test.TestCase):
       loss += v * v
     self.assertAllEqual(t.gradient(loss, v), 2.0)
 
+  def testPythonMax(self):
+    x = [resource_variable_ops.ResourceVariable(2.),
+         resource_variable_ops.ResourceVariable(3.),
+         resource_variable_ops.ResourceVariable(5.)]
+    with backprop.GradientTape() as t:
+      f = max(x)
+    grad = t.gradient(f, x)
+    self.assertAllEqual(self.evaluate(f), 5.)
+    self.assertAllEqual(self.evaluate(grad), [None, None, 1.0])
+
   def testAutomaticWatchedVariables(self):
     with backprop.GradientTape() as t:
       self.assertEqual(0, len(t.watched_variables()))
@@ -674,10 +684,8 @@ class BackpropTest(test.TestCase):
     with backprop.GradientTape() as g:
       x = variables.Variable([3.0])
       y = variables.Variable([2.0])
-    with self.assertRaisesRegexp(
-        ValueError,
-        'GradientTape.gradient is not supported for variable targets.'):
-      g.gradient(x, y)
+    grad = g.gradient(x, y)
+    self.assertAllEqual(grad, None)
 
   @test_util.run_in_graph_and_eager_modes
   @test_util.run_v1_only('b/120545219')
-- 
GitLab


From 32a8626373897a05195dfb3d895385a7584037cc Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Thu, 10 Jan 2019 11:26:10 -0800
Subject: [PATCH 0496/2345] [Java]: Correct some comments.

PiperOrigin-RevId: 228744342
---
 .../org/tensorflow/processor/OperatorProcessor.java  |  2 +-
 .../java/org/tensorflow/op/annotation/Operator.java  | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/tensorflow/java/src/gen/java/org/tensorflow/processor/OperatorProcessor.java b/tensorflow/java/src/gen/java/org/tensorflow/processor/OperatorProcessor.java
index 1b7bcdab35..df1426ad75 100644
--- a/tensorflow/java/src/gen/java/org/tensorflow/processor/OperatorProcessor.java
+++ b/tensorflow/java/src/gen/java/org/tensorflow/processor/OperatorProcessor.java
@@ -340,7 +340,7 @@ public final class OperatorProcessor extends AbstractProcessor {
                     + "{@link $T @Operator} is exposed\n"
                     + "by this API or one of its subgroup.\n<p>Example usage:\n<pre>{@code\n"
                     + "try (Graph g = new Graph()) {\n"
-                    + "  Ops ops = new Ops(g);\n"
+                    + "  Ops ops = Ops.create(g);\n"
                     + "  // Operations are typed classes with convenience\n"
                     + "  // builders in Ops.\n"
                     + "  Constant three = ops.constant(3);\n"
diff --git a/tensorflow/java/src/main/java/org/tensorflow/op/annotation/Operator.java b/tensorflow/java/src/main/java/org/tensorflow/op/annotation/Operator.java
index 3782240edb..38f466c574 100644
--- a/tensorflow/java/src/main/java/org/tensorflow/op/annotation/Operator.java
+++ b/tensorflow/java/src/main/java/org/tensorflow/op/annotation/Operator.java
@@ -25,11 +25,11 @@ import java.lang.annotation.Target;
  * Annotation used by classes to make TensorFlow operations conveniently accessible via {@code
  * org.tensorflow.op.Ops}.
  *
- * <p>An annotation processor (TODO: not yet implemented) builds the {@code Ops} class by
- * aggregating all classes annotated as {@code @Operator}s. Each annotated class <b>must</b> have at
- * least one public static factory method named {@code create} that accepts a {@link
- * org.tensorflow.op.Scope} as its first argument. The processor then adds a convenience method in
- * the {@code Ops} class. For example:
+ * <p>An annotation processor ({@code org.tensorflow.processor.OperatorProcessor}) builds the
+ * {@code Ops} class by aggregating all classes annotated as {@code @Operator}s. Each annotated
+ * class <b>must</b> have at least one public static factory method named {@code create} that
+ * accepts a {@link org.tensorflow.op.Scope} as its first argument. The processor then adds a
+ * convenience method in the {@code Ops} class. For example:
  *
  * <pre>{@code
  * @Operator
@@ -45,7 +45,7 @@ import java.lang.annotation.Target;
  * <pre>{@code
  * import org.tensorflow.op.Ops;
  * ...
- * Ops ops = new Ops(graph);
+ * Ops ops = Ops.create(graph);
  * ...
  * ops.myOp(operand);
  * // and has exactly the same effect as calling
-- 
GitLab


From 519efc7913ef2c6e5271c90727c6a1dc43817efb Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Thu, 10 Jan 2019 11:27:18 -0800
Subject: [PATCH 0497/2345] [XLA] Move over more uses of CreateDS / CreateDUS
 to new API

Move over more uses of the R1-index API to use the span-of-scalars one.

PiperOrigin-RevId: 228744562
---
 .../xla/service/hlo_dataflow_analysis_test.cc | 47 +++++++++++--------
 .../xla/service/hlo_evaluator_test.cc         | 28 ++++++-----
 2 files changed, 45 insertions(+), 30 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc b/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc
index 1d165ac35f..888886865b 100644
--- a/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc
@@ -1970,12 +1970,13 @@ TEST_F(DoesNotUseOperandBufferTest, FusedDynamicUpdateSlice) {
 
   // Create a DynamicUpdateSlice instruction of tuple element 1.
   auto starts = builder.AddInstruction(
-      HloInstruction::CreateConstant(LiteralUtil::CreateR1<int32>({2})));
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(2)));
   auto update = builder.AddInstruction(HloInstruction::CreateConstant(
       LiteralUtil::CreateR1<float>({2.f, 2.f, 2.f})));
   auto dynamic_update_slice =
       builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
-          data_shape, gte1, update, starts));
+          data_shape, gte1, update,
+          std::initializer_list<HloInstruction*>({starts})));
   builder.AddInstruction(
       HloInstruction::CreateTuple({gte0, dynamic_update_slice}));
 
@@ -2012,12 +2013,13 @@ TEST_F(DoesNotUseOperandBufferTest, IndirectUses) {
 
   // Create a DynamicUpdateSlice instruction of tuple element 1.
   auto starts = builder.AddInstruction(
-      HloInstruction::CreateConstant(LiteralUtil::CreateR1<int32>({2})));
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(2)));
   auto update = builder.AddInstruction(HloInstruction::CreateConstant(
       LiteralUtil::CreateR1<float>({2.f, 2.f, 2.f})));
   auto dynamic_update_slice =
       builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
-          data_shape, gte1, update, starts));
+          data_shape, gte1, update,
+          std::initializer_list<HloInstruction*>({starts})));
   builder.AddInstruction(
       HloInstruction::CreateTuple({gte0, dynamic_update_slice}));
 
@@ -2150,17 +2152,17 @@ TEST_F(CanShareOperandBufferWithUserTest,
 
   auto param = builder.AddInstruction(
       HloInstruction::CreateParameter(0, data_shape, "param0"));
-  auto index = builder.AddInstruction(
-      HloInstruction::CreateConstant(LiteralUtil::CreateR1<int64>({0, 0})));
-  auto ds = builder.AddInstruction(
-      HloInstruction::CreateDynamicSlice(slice_shape, param, index, {1, 2, 2}));
+  auto zero = builder.AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int64>(0)));
+  auto ds = builder.AddInstruction(HloInstruction::CreateDynamicSlice(
+      slice_shape, param, {zero, zero}, {1, 2, 2}));
 
-  auto dus = builder.AddInstruction(
-      HloInstruction::CreateDynamicUpdateSlice(data_shape, param, ds, index));
+  auto dus = builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
+      data_shape, param, ds, {zero, zero}));
 
   BuildModule(builder.Build());
   auto fusion = computation_->CreateFusionInstruction(
-      {dus, ds, index}, HloInstruction::FusionKind::kLoop);
+      {dus, ds, zero}, HloInstruction::FusionKind::kLoop);
   RunAnalysis();
 
   EXPECT_TRUE(
@@ -2219,12 +2221,13 @@ TEST_F(CanShareOperandBufferWithUserTest, FusedDynamicUpdateSlice) {
 
   // Create a DynamicUpdateSlice instruction of tuple element 1.
   auto starts = builder.AddInstruction(
-      HloInstruction::CreateConstant(LiteralUtil::CreateR1<int32>({2})));
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(2)));
   auto update = builder.AddInstruction(HloInstruction::CreateConstant(
       LiteralUtil::CreateR1<float>({2.f, 2.f, 2.f})));
   auto dynamic_update_slice =
       builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
-          data_shape, gte1, update, starts));
+          data_shape, gte1, update,
+          std::initializer_list<HloInstruction*>({starts})));
   builder.AddInstruction(
       HloInstruction::CreateTuple({gte0, dynamic_update_slice}));
 
@@ -2259,12 +2262,13 @@ TEST_F(CanShareOperandBufferWithUserTest,
 
   // Create a DynamicUpdateSlice instruction of tuple element 1.
   auto starts = builder.AddInstruction(
-      HloInstruction::CreateConstant(LiteralUtil::CreateR1<int32>({2})));
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(2)));
   auto update = builder.AddInstruction(HloInstruction::CreateConstant(
       LiteralUtil::CreateR1<float>({2.f, 2.f, 2.f})));
   auto dynamic_update_slice =
       builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
-          data_shape_bf16, convert1, update, starts));
+          data_shape_bf16, convert1, update,
+          std::initializer_list<HloInstruction*>({starts})));
 
   auto convert2 = builder.AddInstruction(
       HloInstruction::CreateConvert(data_shape, dynamic_update_slice));
@@ -2290,10 +2294,13 @@ TEST_F(CanShareOperandBufferWithUserTest, DynamicUpdateSliceCanShare) {
       HloInstruction::CreateParameter(0, data_shape, "data"));
   auto update = builder.AddInstruction(
       HloInstruction::CreateParameter(1, update_shape, "update"));
-  auto starts = builder.AddInstruction(
-      HloInstruction::CreateParameter(2, starts_shape, "starts"));
+  auto start0 = builder.AddInstruction(
+      HloInstruction::CreateParameter(2, starts_shape, "start0"));
+  auto start1 = builder.AddInstruction(
+      HloInstruction::CreateParameter(3, starts_shape, "start1"));
+
   auto dus = builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
-      data_shape, data, update, starts));
+      data_shape, data, update, {start0, start1}));
 
   BuildModuleAndRunAnalysis(builder.Build());
 
@@ -2304,7 +2311,9 @@ TEST_F(CanShareOperandBufferWithUserTest, DynamicUpdateSliceCanShare) {
   EXPECT_FALSE(
       dataflow_analysis_->CanShareOperandBufferWithUser(update, {}, dus, {}));
   EXPECT_FALSE(
-      dataflow_analysis_->CanShareOperandBufferWithUser(starts, {}, dus, {}));
+      dataflow_analysis_->CanShareOperandBufferWithUser(start0, {}, dus, {}));
+  EXPECT_FALSE(
+      dataflow_analysis_->CanShareOperandBufferWithUser(start1, {}, dus, {}));
 }
 
 TEST_F(CanShareOperandBufferWithUserTest, ScatterCanShare) {
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
index 319f0db97a..674df0016a 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
@@ -1739,12 +1739,14 @@ TEST_P(HloEvaluatorBf16Test, DynamicSlice) {
   HloInstruction* operand = b.AddInstruction(
       HloInstruction::CreateConstant(std::move(operand_literal)));
 
-  auto start_indices = b.AddInstruction(
-      HloInstruction::CreateConstant(LiteralUtil::CreateR1<int32>({0, 1})));
+  auto zero = b.AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(0)));
+  auto one = b.AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(1)));
 
   Shape shape = ShapeUtil::MakeShape(F32, {2, 3});
-  b.AddInstruction(HloInstruction::CreateDynamicSlice(shape, operand,
-                                                      start_indices, {2, 3}));
+  b.AddInstruction(
+      HloInstruction::CreateDynamicSlice(shape, operand, {zero, one}, {2, 3}));
   m_->AddEntryComputation(b.Build());
 
   Literal result = Evaluate();
@@ -1775,12 +1777,14 @@ TEST_P(HloEvaluatorBf16Test, DynamicSliceModSlice) {
   HloInstruction* operand = b.AddInstruction(
       HloInstruction::CreateConstant(std::move(operand_literal)));
 
-  auto start_indices = b.AddInstruction(
-      HloInstruction::CreateConstant(LiteralUtil::CreateR1<int32>({2, 1})));
+  auto two = b.AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(2)));
+  auto one = b.AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(1)));
 
   Shape shape = ShapeUtil::MakeShape(F32, {2, 3});
-  b.AddInstruction(HloInstruction::CreateDynamicSlice(shape, operand,
-                                                      start_indices, {2, 3}));
+  b.AddInstruction(
+      HloInstruction::CreateDynamicSlice(shape, operand, {two, one}, {2, 3}));
   m_->AddEntryComputation(b.Build());
 
   Literal result = Evaluate();
@@ -1809,15 +1813,17 @@ TEST_P(HloEvaluatorBf16Test, DynamicSliceUpdate) {
   HloInstruction* operand = b.AddInstruction(
       HloInstruction::CreateConstant(std::move(operand_literal)));
 
-  auto start_indices = b.AddInstruction(
-      HloInstruction::CreateConstant(LiteralUtil::CreateR1<int64>({0, 1})));
+  auto zero = b.AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(0)));
+  auto one = b.AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(1)));
 
   auto update = b.AddInstruction(HloInstruction::CreateConstant(
       LiteralUtil::CreateR2<double>({{-2.0, -3.0}, {-6.0, -7.0}})));
 
   Shape shape = ShapeUtil::MakeShape(F64, {2, 3});
   b.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
-      shape, operand, update, start_indices));
+      shape, operand, update, {zero, one}));
   m_->AddEntryComputation(b.Build());
 
   Literal result = Evaluate();
-- 
GitLab


From 4644d4e5f26cf72905d1ce40194a9733c4f8252f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 11:29:29 -0800
Subject: [PATCH 0498/2345] [convergence_tools]: Tracing the subset of the ops
 that are in the execution path.

PiperOrigin-RevId: 228744967
---
 .../contrib/tpu/python/tpu/tensor_tracer.py   | 53 +++++++++++++++++--
 .../contrib/tpu/python/tpu/tpu_estimator.py   |  3 +-
 2 files changed, 52 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py b/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py
index c7e29860e5..bf492e78a1 100644
--- a/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py
+++ b/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py
@@ -59,6 +59,7 @@ _REASON_SCALAR_GET_TRACED = 'traced-scalar'
 _REASON_TENSOR_GET_TRACED = 'traced-tensor'
 _REASON_USER_INCLUDED = 'traced-user-included'
 _REASON_USER_EXCLUDED = 'not-traced-user-excluded'
+_REASON_NOT_EXECUTED = 'not-traced-not-in-exec-path'
 _REASON_NON_NUMERIC_TENSOR = 'not-traced-non-numeric-tensor'
 _MARKER_SECTION_BEGIN = '!!!!!!! section-begin:'
 _MARKER_SECTION_END = '!!!!!!! section-end:'
@@ -864,7 +865,8 @@ class TensorTracer(object):
     raise RuntimeError('Tensor trace fun for %s is not yet implemented'
                        %self._trace_mode)
 
-  def _skip_op(self, op_id, op, user_included, user_excluded):
+  def _skip_op(self, op_id, op, user_included, user_excluded,
+               in_exec_path=True):
     """Returns True if we should not trace Op."""
 
     if user_included:
@@ -875,6 +877,10 @@ class TensorTracer(object):
       self._instrument_records[op.name] = TensorTracer.reason(
           op_id, _REASON_USER_EXCLUDED)
       return True
+    if not in_exec_path:
+      self._instrument_records[op.name] = TensorTracer.reason(
+          op_id, _REASON_NOT_EXECUTED)
+      return True
     if not self._inside_op_range(op_id):
       self._instrument_records[op.name] = TensorTracer.reason(
           op_id, _REASON_OUTSIDE_OP_RANGE)
@@ -946,6 +952,40 @@ class TensorTracer(object):
           op_id, _REASON_TENSOR_GET_TRACED)
       return False
 
+  def _filter_execution_path_operations(self, operations, fetches):
+    """Returns the set of ops in the execution path to compute given fetches."""
+    # If no fetch provided, then return all operations.
+    if fetches is None:
+      return set(operations)
+    # Convert to list, if a single element is provided.
+    if not isinstance(fetches, (list, tuple)):
+      fetches = [fetches]
+    # If a tensor is given as fetch, convert it to op.
+    op_fetches = []
+    for fetch in fetches:
+      if isinstance(fetch, ops.Operation):
+        op_fetches.append(fetch)
+      elif isinstance(fetch, ops.Tensor):
+        op_fetches.append(fetch.op)
+      else:
+        raise RuntimeError('Given fetch:%s is neither a tensor nor an op.'
+                           %fetch)
+
+    execution_path_operations = set(op_fetches)
+    traverse_stack = list(op_fetches)
+    while True:
+      if not traverse_stack:
+        break
+      head_op = traverse_stack.pop()
+      input_ops = [tensor_input.op for tensor_input in head_op.inputs]
+      input_ops.extend(head_op.control_inputs)
+
+      for input_op in input_ops:
+        if input_op not in execution_path_operations:
+          execution_path_operations.add(input_op)
+          traverse_stack.append(input_op)
+    return execution_path_operations
+
   def _pre_tracing(self, graph):
     """Work needs to be done prior to TPU or CPU tracing."""
 
@@ -987,13 +1027,15 @@ class TensorTracer(object):
                                   _TENSOR_TRACER_CHECKPOINT))
     return checkpoint_operations
 
-  def trace_tpu(self, graph, result_tensor, num_replicas=None):
+  def trace_tpu(self, graph, result_tensor, num_replicas=None, fetches=None):
     """Traces the tensors generated by TPU Ops in a TF graph.
 
     Args:
       graph: the graph of Ops executed on the TPU.
       result_tensor: a result tensor of evaluating the graph.
       num_replicas: number of replicas used on the TPU.
+      fetches: the list of fetches given to session.run, used to determine the
+      ops in execution path. If None, the whole graph will be traced.
 
     Returns:
       A tuple (result_tensor_copy, tracing_ops), where:
@@ -1022,6 +1064,10 @@ class TensorTracer(object):
     result_tensor_copy = self._add_replica_id_to_graph(num_replicas,
                                                        result_tensor)
     (operations, succeed, sorted_or_cycle) = self._pre_tracing(graph)
+    # Filter out the operations that won't be executed.
+    # if fetches=None, then ops_in_exec_path = set(operations)
+    ops_in_exec_path = self._filter_execution_path_operations(operations,
+                                                              fetches)
     tracing_ops = []
     checkpoint_operations = self._get_checkpoints(graph)
 
@@ -1030,7 +1076,8 @@ class TensorTracer(object):
         continue
       user_included = self._is_user_included_op(op)
       user_excluded = self._is_user_excluded_op(op)
-      if self._skip_op(op_id, op, user_included, user_excluded):
+      in_exec_path = op in ops_in_exec_path
+      if self._skip_op(op_id, op, user_included, user_excluded, in_exec_path):
         continue
       for i in range(len(op.outputs)):
         out_tensor = op.outputs[i]
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 2df5b3e4eb..075ecbc52a 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -1412,7 +1412,8 @@ class _ModelFnWrapper(object):
       if tensor_tracer.TensorTracer.is_enabled():
         tt = tensor_tracer.TensorTracer()
         loss, tracing_ops = tt.trace_tpu(ops.get_default_graph(), loss,
-                                         self._ctx.num_replicas)
+                                         self._ctx.num_replicas,
+                                         fetches=[loss, train_op])
 
       # We must run train_op to update the variables prior to running the
       # outfeed.
-- 
GitLab


From 53bd138262df6e141c0e196a262f4ae760c2cf5f Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Thu, 10 Jan 2019 11:34:00 -0800
Subject: [PATCH 0499/2345] [Java]: Add place to put unittests for generated Op
 wrappers.

PiperOrigin-RevId: 228745935
---
 tensorflow/java/BUILD                         | 13 ++++++
 .../op/core/GeneratedOperationsTest.java      | 43 +++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 tensorflow/java/src/test/java/org/tensorflow/op/core/GeneratedOperationsTest.java

diff --git a/tensorflow/java/BUILD b/tensorflow/java/BUILD
index 10808e162e..af5503f2ad 100644
--- a/tensorflow/java/BUILD
+++ b/tensorflow/java/BUILD
@@ -295,6 +295,19 @@ tf_java_test(
     ],
 )
 
+tf_java_test(
+    name = "GeneratedOperationsTest",
+    size = "small",
+    srcs = ["src/test/java/org/tensorflow/op/core/GeneratedOperationsTest.java"],
+    javacopts = JAVACOPTS,
+    test_class = "org.tensorflow.op.core.GeneratedOperationsTest",
+    deps = [
+        ":tensorflow",
+        ":testutil",
+        "@junit",
+    ],
+)
+
 tf_java_test(
     name = "GradientsTest",
     size = "small",
diff --git a/tensorflow/java/src/test/java/org/tensorflow/op/core/GeneratedOperationsTest.java b/tensorflow/java/src/test/java/org/tensorflow/op/core/GeneratedOperationsTest.java
new file mode 100644
index 0000000000..42d126c3c4
--- /dev/null
+++ b/tensorflow/java/src/test/java/org/tensorflow/op/core/GeneratedOperationsTest.java
@@ -0,0 +1,43 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.op.core;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.tensorflow.Graph;
+import org.tensorflow.Operand;
+import org.tensorflow.Session;
+import org.tensorflow.Tensor;
+import org.tensorflow.op.Ops;
+
+@RunWith(JUnit4.class)
+public final class GeneratedOperationsTest {
+
+  @Test
+  public void tensorInputTensorOutput() {
+    try (Graph g = new Graph();
+        Session sess = new Session(g)) {
+      Ops ops = Ops.create(g);
+      Operand<Integer> x = ops.math().add(ops.constant(1), ops.constant(2));
+      try (Tensor<Integer> result = sess.runner().fetch(x).run().get(0).expect(Integer.class)) {
+        assertEquals(3, result.intValue());
+      }
+    }
+  }
+}
-- 
GitLab


From 1095ed8fe6808a70edb66d7970e9462db791e690 Mon Sep 17 00:00:00 2001
From: James Ring <sjr@google.com>
Date: Thu, 10 Jan 2019 11:48:55 -0800
Subject: [PATCH 0500/2345] Add TF_StepId and TF_ExpectedOutputDataType

PiperOrigin-RevId: 228748943
---
 tensorflow/c/kernels.cc      | 9 +++++++++
 tensorflow/c/kernels.h       | 8 ++++++++
 tensorflow/c/kernels_test.cc | 6 ++++++
 3 files changed, 23 insertions(+)

diff --git a/tensorflow/c/kernels.cc b/tensorflow/c/kernels.cc
index 2a4eaecb6c..673aed558a 100644
--- a/tensorflow/c/kernels.cc
+++ b/tensorflow/c/kernels.cc
@@ -158,3 +158,12 @@ void TF_SetOutput(TF_OpKernelContext* ctx, int i, const TF_Tensor* tensor,
     cc_ctx->set_output(i, cc_tensor);
   }
 }
+
+TF_DataType TF_ExpectedOutputDataType(TF_OpKernelContext* ctx, int i) {
+  auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelContext*>(ctx);
+  return static_cast<TF_DataType>(cc_ctx->expected_output_dtype(i));
+}
+
+int64_t TF_StepId(TF_OpKernelContext* ctx) {
+  return reinterpret_cast<::tensorflow::OpKernelContext*>(ctx)->step_id();
+}
diff --git a/tensorflow/c/kernels.h b/tensorflow/c/kernels.h
index cefc30bcdf..721a4aca0b 100644
--- a/tensorflow/c/kernels.h
+++ b/tensorflow/c/kernels.h
@@ -111,6 +111,14 @@ TF_CAPI_EXPORT extern void TF_SetOutput(TF_OpKernelContext* ctx, int i,
                                         const TF_Tensor* tensor,
                                         TF_Status* status);
 
+// Returns the expected output data type of the ith output. If i < 0 or
+// i >= TF_NumOutputs(ctx), the program aborts.
+TF_CAPI_EXPORT extern TF_DataType TF_ExpectedOutputDataType(
+    TF_OpKernelContext* ctx, int i);
+
+// Returns the step ID of the given context.
+TF_CAPI_EXPORT extern int64_t TF_StepId(TF_OpKernelContext* ctx);
+
 #ifdef __cplusplus
 } /* end extern "C" */
 #endif
diff --git a/tensorflow/c/kernels_test.cc b/tensorflow/c/kernels_test.cc
index e659ee3c3d..fdeb04c84c 100644
--- a/tensorflow/c/kernels_test.cc
+++ b/tensorflow/c/kernels_test.cc
@@ -41,6 +41,9 @@ static void* MyCreateFunc(TF_OpKernelConstruction* ctx) {
 static void MyComputeFunc(void* kernel, TF_OpKernelContext* ctx) {
   struct MyCustomKernel* s = static_cast<struct MyCustomKernel*>(kernel);
   s->compute_called = true;
+  if (ctx != nullptr) {
+    EXPECT_EQ(43, TF_StepId(ctx));
+  }
 }
 
 static void MyDeleteFunc(void* kernel) {
@@ -155,6 +158,8 @@ TEST(TestKernel, TestInputAndOutputCount) {
     TF_SetOutput(ctx, 24, input, s);
     EXPECT_EQ(TF_OUT_OF_RANGE, TF_GetCode(s));
 
+    EXPECT_EQ(TF_UINT8, TF_ExpectedOutputDataType(ctx, 0));
+
     TF_DeleteStatus(s);
     if (input != nullptr) {
       TF_DeleteTensor(input);
@@ -175,6 +180,7 @@ TEST(TestKernel, TestInputAndOutputCount) {
     OpKernelContext::Params p;
     DummyDevice dummy_device(nullptr, false);
     p.device = &dummy_device;
+    p.step_id = 43;
 
     Tensor t(tensorflow::uint8(123));
 
-- 
GitLab


From d5977dfe93b95f6902cb9df28525a1dc50e4704d Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Thu, 10 Jan 2019 11:53:59 -0800
Subject: [PATCH 0501/2345] Internal Cleanup.

PiperOrigin-RevId: 228749766
---
 tensorflow/python/keras/BUILD | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 9f8da126bf..fb0161a677 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -482,13 +482,14 @@ cuda_py_test(
 
 tf_py_test(
     name = "pooling_test",
-    size = "large",
+    size = "medium",
     srcs = ["layers/pooling_test.py"],
     additional_deps = [
         ":keras",
         "@absl_py//absl/testing:parameterized",
         "//tensorflow/python:client_testlib",
     ],
+    shard_count = 8,
 )
 
 tf_py_test(
-- 
GitLab


From 84572bff5cf02b1722ab763982a92be44e7719ef Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Thu, 10 Jan 2019 12:18:36 -0800
Subject: [PATCH 0502/2345] Fix collective_all_reduce_strategy test failure.

PiperOrigin-RevId: 228754463
---
 .../distribute/python/collective_all_reduce_strategy_test.py   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
index 682da44f5d..62ff4b178e 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
@@ -426,7 +426,8 @@ class LocalCollectiveAllReduceStrategy(
 
   @combinations.generate(
       combinations.combine(mode=['graph', 'eager'], required_gpus=2))
-  def testMakeInputFnIterator(self, num_gpus=2):
+  def testMakeInputFnIterator(self):
+    num_gpus = 2
     dataset_fn = lambda: dataset_ops.Dataset.range(5 * num_gpus)
     expected_values = [range(i, i + num_gpus) for i in range(0, 10, num_gpus)]
 
-- 
GitLab


From 02630688f2fa239ce1d1551caa8a2f3559c248e3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 12:25:01 -0800
Subject: [PATCH 0503/2345] Automated rollback of commit
 73b6d9262ea6d1578374ecb63560ad0fcad13e35

PiperOrigin-RevId: 228755594
---
 tensorflow/contrib/distribute/python/BUILD    |  30 +-
 ...test_base.py => keras_correctness_test.py} | 261 ++++++++++++++----
 .../python/keras_dnn_correctness_test.py      | 163 -----------
 .../keras_image_model_correctness_test.py     |  92 ------
 4 files changed, 209 insertions(+), 337 deletions(-)
 rename tensorflow/contrib/distribute/python/{keras_correctness_test_base.py => keras_correctness_test.py} (58%)
 delete mode 100644 tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
 delete mode 100644 tensorflow/contrib/distribute/python/keras_image_model_correctness_test.py

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 4c25f49f16..d4758d7518 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -657,11 +657,7 @@ cuda_py_test(
 py_library(
     name = "keras_correctness_test_lib",
     testonly = 1,
-    srcs = [
-        "keras_correctness_test_base.py",
-        "keras_dnn_correctness_test.py",
-        "keras_image_model_correctness_test.py",
-    ],
+    srcs = ["keras_correctness_test.py"],
     deps = [
         ":combinations",
         "//tensorflow/contrib/distribute/python:mirrored_strategy",
@@ -677,9 +673,8 @@ py_library(
 )
 
 cuda_py_test(
-    name = "keras_dnn_correctness_test",
-    size = "medium",
-    srcs = ["keras_dnn_correctness_test.py"],
+    name = "keras_correctness_test",
+    srcs = ["keras_correctness_test.py"],
     additional_deps = [
         ":keras_correctness_test_lib",
     ],
@@ -695,25 +690,6 @@ cuda_py_test(
     ],
 )
 
-cuda_py_test(
-    name = "keras_image_model_correctness_test",
-    size = "medium",
-    srcs = ["keras_image_model_correctness_test.py"],
-    additional_deps = [
-        ":keras_correctness_test_lib",
-    ],
-    # Shard count is set to an odd number to distribute tasks across
-    # shards more evenly.
-    shard_count = 31,
-    tags = [
-        "multi_and_single_gpu",
-        "no_oss",  # TODO(b/117919883): Fix python error.
-        "no_pip",
-        "no_windows_gpu",
-        "notsan",
-    ],
-)
-
 py_library(
     name = "metrics_v1_test_lib",
     testonly = 1,
diff --git a/tensorflow/contrib/distribute/python/keras_correctness_test_base.py b/tensorflow/contrib/distribute/python/keras_correctness_test.py
similarity index 58%
rename from tensorflow/contrib/distribute/python/keras_correctness_test_base.py
rename to tensorflow/contrib/distribute/python/keras_correctness_test.py
index e13984b26d..3abdee2c0e 100644
--- a/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
+++ b/tensorflow/contrib/distribute/python/keras_correctness_test.py
@@ -30,6 +30,8 @@ from tensorflow.python.eager import context
 from tensorflow.python.eager import test
 from tensorflow.python.framework import random_seed
 from tensorflow.python.keras.engine import distributed_training_utils
+from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_descent_keras
+from tensorflow.python.training import gradient_descent
 
 _RANDOM_SEED = 1337
 _EVAL_STEPS = 20
@@ -60,13 +62,24 @@ def all_strategy_combinations_with_graph_mode():
   return combinations.combine(distribution=all_strategies, mode=['graph'])
 
 
-def all_strategy_and_input_config_combinations():
+def strategy_and_input_combinations():
+  def cnn_model_with_batch_norm(**kwargs):
+    return _create_cnn_model(with_batch_norm=True, **kwargs)
+
   return (
       combinations.times(
           combinations.combine(distribution=all_strategies),
           combinations.combine(mode=['graph', 'eager'],
                                use_numpy=[True, False],
-                               use_validation_data=[True, False])))
+                               use_validation_data=[True, False]),
+          combinations.combine(model_with_data=[
+              ModelWithData('dnn', _create_dnn_model, _dnn_training_data),
+              ModelWithData('cnn', _create_cnn_model, _cnn_training_data),
+              ModelWithData('cnn_batch_norm',
+                            cnn_model_with_batch_norm,
+                            _cnn_training_data,
+                            with_batch_norm=True),
+          ])))
 
 
 class MaybeDistributionScope(object):
@@ -87,6 +100,97 @@ class MaybeDistributionScope(object):
       self._scope = None
 
 
+class ModelWithData(object):
+  """An object giving a good name in combinations.
+
+  The model_fn must take two arguments: initial_weights and distribution.
+  """
+
+  def __init__(self, name, model_fn, data_fn, with_batch_norm=False):
+    self.name = name
+    self.model_fn = model_fn
+    self.data_fn = data_fn
+    self.with_batch_norm = with_batch_norm
+
+  def __repr__(self):
+    return self.name
+
+
+def _dnn_training_data():
+  # TODO(xiejw): Change this back to 10000, once we support final partial
+  # batch.
+  num_samples = 9984
+  x_train = np.random.rand(num_samples, 1)
+  y_train = 3 * x_train
+  x_train = x_train.astype('float32')
+  y_train = y_train.astype('float32')
+  x_predict = [[1.], [2.], [3.], [4.]]
+  return x_train, y_train, x_predict
+
+
+def _create_dnn_model(initial_weights=None, distribution=None):
+  with MaybeDistributionScope(distribution):
+    # We add few non-linear layers to make it non-trivial.
+    model = keras.Sequential()
+    model.add(keras.layers.Dense(10, activation='relu', input_shape=(1,)))
+    model.add(keras.layers.Dense(10, activation='relu'))
+    model.add(keras.layers.Dense(10, activation='relu'))
+    model.add(keras.layers.Dense(1))
+
+    if initial_weights:
+      model.set_weights(initial_weights)
+
+    model.compile(
+        loss=keras.losses.mean_squared_error,
+        optimizer=gradient_descent_keras.SGD(0.5),
+        metrics=['mse'])
+    return model
+
+
+def _cnn_training_data(count=_GLOBAL_BATCH_SIZE * _EVAL_STEPS,
+                       shape=(28, 28, 3), num_classes=10):
+  centers = np.random.randn(num_classes, *shape)
+
+  features = []
+  labels = []
+  for _ in range(count):
+    label = np.random.randint(0, num_classes, size=1)[0]
+    offset = np.random.normal(loc=0, scale=0.1, size=np.prod(shape))
+    offset = offset.reshape(shape)
+    labels.append(label)
+    features.append(centers[label] + offset)
+
+  x_train = np.asarray(features, dtype=np.float32)
+  y_train = np.asarray(labels, dtype=np.float32).reshape((count, 1))
+  x_predict = x_train
+  return x_train, y_train, x_predict
+
+
+def _create_cnn_model(initial_weights=None, distribution=None,
+                      with_batch_norm=False):
+  with MaybeDistributionScope(distribution):
+    image = keras.layers.Input(shape=(28, 28, 3), name='image')
+    c1 = keras.layers.Conv2D(
+        name='conv1', filters=16, kernel_size=(3, 3), strides=(4, 4))(
+            image)
+    if with_batch_norm:
+      c1 = keras.layers.BatchNormalization(name='bn1')(c1)
+    c1 = keras.layers.MaxPooling2D(pool_size=(2, 2))(c1)
+    logits = keras.layers.Dense(
+        10, activation='softmax', name='pred')(
+            keras.layers.Flatten()(c1))
+    model = keras.Model(inputs=[image], outputs=[logits])
+
+    if initial_weights:
+      model.set_weights(initial_weights)
+
+    model.compile(
+        optimizer=gradient_descent.GradientDescentOptimizer(learning_rate=0.1),
+        loss='sparse_categorical_crossentropy',
+        metrics=['sparse_categorical_accuracy'])
+  return model
+
+
 def batch_wrapper(dataset, batch_size, distribution, repeat=None):
   if repeat:
     dataset = dataset.repeat(repeat)
@@ -246,11 +350,6 @@ def compare_results(results_with_ds, results_without_ds, distribution,
         msg='Fail to assert {}.'.format(key))
 
 
-def should_skip_tpu_with_eager(distribution):
-  return (context.executing_eagerly() and
-          isinstance(distribution, tpu_strategy.TPUStrategy))
-
-
 class LearningRateBatchScheduler(keras.callbacks.Callback):
 
   def __init__(self, update_freq=None):
@@ -265,60 +364,107 @@ class LearningRateBatchScheduler(keras.callbacks.Callback):
     keras.backend.set_value(self.model.optimizer.lr, lr)
 
 
-class TestDistributionStrategyCorrectnessBase(test.TestCase,
-                                              parameterized.TestCase):
-  """Model agnostic testing infra to test correctness of Keras models."""
-
-  def set_up_test_config(self, use_numpy=False,
-                         use_validation_data=False,
-                         with_batch_norm=False):
-    self.use_numpy = use_numpy
-    self.use_validation_data = use_validation_data
-    self.with_batch_norm = with_batch_norm
+class TestDistributionStrategyCorrectness(test.TestCase,
+                                          parameterized.TestCase):
 
-    keras.backend.set_image_data_format('channels_last')
-    np.random.seed(_RANDOM_SEED)
-    random_seed.set_random_seed(_RANDOM_SEED)
+  def _should_skip_tpu_with_eager(self, distribution):
+    return (context.executing_eagerly() and
+            isinstance(distribution, tpu_strategy.TPUStrategy))
 
-  def get_data(self):
-    num_samples = 10000
-    x_train = np.random.randint(0, 2, num_samples)
-    x_train = np.reshape(x_train, (num_samples, 1))
-    y_train = x_train
-    return (x_train.astype('float32'), y_train.astype('float32'), None)
+  @combinations.generate(all_strategy_combinations_with_eager_and_graph_modes())
+  def test_metric_correctness(self, distribution):
+    if self._should_skip_tpu_with_eager(distribution):
+      self.skipTest('TPUStrategy does not support eager mode now.')
 
-  def get_model(self, distribution=None):
-    raise NotImplementedError
+    with self.cached_session():
+      keras.backend.set_image_data_format('channels_last')
+      num_samples = 10000
+
+      x_train = np.random.randint(0, 2, num_samples)
+      x_train = np.reshape(x_train, (num_samples, 1))
+      y_train = x_train
+      x_train = x_train.astype('float32')
+      y_train = y_train.astype('float32')
+
+      # Create identity model.
+      with distribution.scope():
+        model = keras.Sequential()
+        model.add(
+            keras.layers.Dense(1, input_shape=(1,), kernel_initializer='ones'))
+        model.compile(
+            loss=keras.losses.mean_squared_error,
+            optimizer=gradient_descent.GradientDescentOptimizer(0.5),
+            metrics=[keras.metrics.BinaryAccuracy()])
+
+      batch_size = 64
+      batch_size = get_batch_size(batch_size, distribution)
+      train_dataset = dataset_ops.Dataset.from_tensor_slices((x_train, y_train))
+      train_dataset = batch_wrapper(train_dataset, batch_size, distribution)
+
+      history = model.fit(x=train_dataset, epochs=2, steps_per_epoch=10)
+      self.assertEqual(history.history['binary_accuracy'], [1.0, 1.0])
+
+  @combinations.generate(all_strategy_combinations_with_eager_and_graph_modes())
+  def test_eval_metrics_correctness(self, distribution):
+    if self._should_skip_tpu_with_eager(distribution):
+      self.skipTest('TPUStrategy does not support eager mode now.')
 
-  def skip_unsupported_test_configuration(self, distribution):
-    if should_skip_tpu_with_eager(distribution):
+    with self.cached_session():
+      with distribution.scope():
+        model = keras.Sequential()
+        model.add(
+            keras.layers.Dense(
+                3, activation='relu', input_dim=4, kernel_initializer='ones'))
+        model.add(
+            keras.layers.Dense(
+                1, activation='sigmoid', kernel_initializer='ones'))
+        model.compile(
+            loss='mae',
+            metrics=['accuracy', keras.metrics.BinaryAccuracy()],
+            optimizer=gradient_descent.GradientDescentOptimizer(0.001))
+
+      # verify correctness of stateful and stateless metrics.
+      x = np.ones((100, 4)).astype('float32')
+      y = np.ones((100, 1)).astype('float32')
+      dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat()
+      dataset = batch_wrapper(dataset, 4, distribution)
+      outs = model.evaluate(dataset, steps=10)
+      self.assertEqual(outs[1], 1.)
+      self.assertEqual(outs[2], 1.)
+
+      y = np.zeros((100, 1)).astype('float32')
+      dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat()
+      dataset = batch_wrapper(dataset, 4, distribution)
+      outs = model.evaluate(dataset, steps=10)
+      self.assertEqual(outs[1], 0.)
+      self.assertEqual(outs[2], 0.)
+
+  @combinations.generate(strategy_and_input_combinations())
+  def test_correctness(self, distribution, use_numpy, use_validation_data,
+                       model_with_data):
+    if self._should_skip_tpu_with_eager(distribution):
       self.skipTest('TPUStrategy does not support eager mode now.')
 
-    if context.executing_eagerly() and self.use_numpy:
+    if context.executing_eagerly() and use_numpy:
       self.skipTest('Numpy as inputs is not supported with strategy in eager.')
 
-    if context.executing_eagerly() and self.use_validation_data:
-      self.skipTest('TODO(hongjunchoi): Add test logic for using validation '
-                    'data for eager execution.')
-    return
+    if context.executing_eagerly() and use_validation_data:
+      self.skipTest('TODO')
 
-  def run_correctness_test(self,
-                           distribution,
-                           use_numpy,
-                           use_validation_data,
-                           with_batch_norm=False):
     with self.cached_session():
-      self.set_up_test_config(use_numpy, use_validation_data, with_batch_norm)
-      self.skip_unsupported_test_configuration(distribution)
+      keras.backend.set_image_data_format('channels_last')
+      np.random.seed(_RANDOM_SEED)
+      random_seed.set_random_seed(_RANDOM_SEED)
 
+      model_fn, data_fn = model_with_data.model_fn, model_with_data.data_fn
       # Train, eval, and predict datasets are created with the same input numpy
       # arrays.
-      x_train, y_train, x_predict = self.get_data()
+      x_train, y_train, x_predict = data_fn()
 
       # The model is built once and the initial weights are saved.
       # This is used to initialize the model for both the distribution and
       # non-distribution run.
-      model = self.get_model()
+      model = model_fn()
       initial_weights = model.get_weights()
 
       def input_fn(dist):
@@ -326,15 +472,15 @@ class TestDistributionStrategyCorrectnessBase(test.TestCase,
             use_numpy, use_validation_data, dist, x_train, y_train, x_predict)
 
       results_with_ds = fit_eval_and_predict(
-          initial_weights, input_fn=input_fn, model_fn=self.get_model,
+          initial_weights, input_fn=input_fn, model_fn=model_fn,
           distribution=distribution)
       results_without_ds = fit_eval_and_predict(
-          initial_weights, input_fn=input_fn, model_fn=self.get_model,
+          initial_weights, input_fn=input_fn, model_fn=model_fn,
           distribution=None)
 
       # First, special case, for multi-replica distributed training, batch norm
       # is not aggregated globally. So it is expected to have different weights.
-      if (self.with_batch_norm and
+      if (model_with_data.with_batch_norm and
           distribution.num_replicas_in_sync > 1):
         with self.assertRaises(AssertionError):
           compare_results(results_with_ds, results_without_ds, distribution,
@@ -343,16 +489,21 @@ class TestDistributionStrategyCorrectnessBase(test.TestCase,
         compare_results(results_with_ds, results_without_ds, distribution,
                         testcase=self)
 
-  def run_dynamic_lr_test(self, distribution):
+  @combinations.generate(all_strategy_combinations_with_graph_mode())
+  def test_dynamic_lr(self, distribution):
+
     with self.cached_session():
-      self.set_up_test_config()
-      self.skip_unsupported_test_configuration(distribution)
 
-      x_train, y_train, _ = self.get_data()
-      model = self.get_model()
+      keras.backend.set_image_data_format('channels_last')
+      np.random.seed(_RANDOM_SEED)
+      random_seed.set_random_seed(_RANDOM_SEED)
+
+      x_train, y_train, _ = _dnn_training_data()
+
+      model = _create_dnn_model()
       initial_weights = model.get_weights()
-      update_freq = None
 
+      update_freq = None
       if (isinstance(distribution, tpu_strategy.TPUStrategy) and
           distribution.extended.steps_per_run > 1):
         # For TPUStrategy with steps_per_run > 1, the callback is not invoked
@@ -379,10 +530,10 @@ class TestDistributionStrategyCorrectnessBase(test.TestCase,
         return training_inputs, eval_inputs, predict_inputs
 
       results_with_ds = fit_eval_and_predict(
-          initial_weights, input_fn=input_fn, model_fn=self.get_model,
+          initial_weights, input_fn=input_fn, model_fn=_create_dnn_model,
           distribution=distribution)
       results_without_ds = fit_eval_and_predict(
-          initial_weights, input_fn=input_fn, model_fn=self.get_model,
+          initial_weights, input_fn=input_fn, model_fn=_create_dnn_model,
           distribution=None)
       compare_results(results_with_ds, results_without_ds, distribution,
                       testcase=self)
diff --git a/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py b/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
deleted file mode 100644
index f21b1c6752..0000000000
--- a/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
+++ /dev/null
@@ -1,163 +0,0 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Correctness tests for tf.keras DNN model using DistributionStrategy."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-
-from tensorflow.contrib.distribute.python import combinations
-from tensorflow.contrib.distribute.python import keras_correctness_test_base
-from tensorflow.python import keras
-from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.eager import test
-from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_descent_keras
-from tensorflow.python.training import gradient_descent
-
-
-class TestDistributionStrategyDnnCorrectness(
-    keras_correctness_test_base.TestDistributionStrategyCorrectnessBase):
-
-  def get_model(self, initial_weights=None, distribution=None):
-    with keras_correctness_test_base.MaybeDistributionScope(distribution):
-      # We add few non-linear layers to make it non-trivial.
-      model = keras.Sequential()
-      model.add(keras.layers.Dense(10, activation='relu', input_shape=(1,)))
-      model.add(keras.layers.Dense(10, activation='relu'))
-      model.add(keras.layers.Dense(10, activation='relu'))
-      model.add(keras.layers.Dense(1))
-
-      if initial_weights:
-        model.set_weights(initial_weights)
-
-      model.compile(
-          loss=keras.losses.mean_squared_error,
-          optimizer=gradient_descent_keras.SGD(0.5),
-          metrics=['mse'])
-      return model
-
-  def get_data(self):
-    # TODO(xiejw): Change this back to 10000, once we support final partial
-    # batch.
-    num_samples = 9984
-    x_train = np.random.rand(num_samples, 1)
-    y_train = 3 * x_train
-    x_train = x_train.astype('float32')
-    y_train = y_train.astype('float32')
-    x_predict = [[1.], [2.], [3.], [4.]]
-    return x_train, y_train, x_predict
-
-  @combinations.generate(keras_correctness_test_base.
-                         all_strategy_and_input_config_combinations())
-  def test_dnn_correctness(self, distribution, use_numpy, use_validation_data):
-    self.run_correctness_test(distribution, use_numpy, use_validation_data)
-
-  @combinations.generate(keras_correctness_test_base.
-                         all_strategy_combinations_with_graph_mode())
-  def test_dnn_with_dynamic_learning_rate(self, distribution):
-    self.run_dynamic_lr_test(distribution)
-
-
-class TestDistributionStrategyDnnMetricCorrectness(
-    keras_correctness_test_base.TestDistributionStrategyCorrectnessBase):
-
-  def get_model(self, distribution=None):
-    with distribution.scope():
-      model = keras.Sequential()
-      model.add(keras.layers.Dense(1,
-                                   input_shape=(1,),
-                                   kernel_initializer='ones'))
-      model.compile(
-          loss=keras.losses.mean_squared_error,
-          optimizer=gradient_descent.GradientDescentOptimizer(0.5),
-          metrics=[keras.metrics.BinaryAccuracy()])
-    return model
-
-  def run_metric_correctness_test(self, distribution):
-    with self.cached_session():
-      self.set_up_test_config()
-      self.skip_unsupported_test_configuration(distribution)
-
-      x_train, y_train, _ = self.get_data()
-      model = self.get_model(distribution=distribution)
-
-      batch_size = 64
-      batch_size = (keras_correctness_test_base.
-                    get_batch_size(batch_size, distribution))
-      train_dataset = dataset_ops.Dataset.from_tensor_slices((x_train, y_train))
-      train_dataset = (keras_correctness_test_base.
-                       batch_wrapper(train_dataset, batch_size, distribution))
-
-      history = model.fit(x=train_dataset, epochs=2, steps_per_epoch=10)
-      self.assertEqual(history.history['binary_accuracy'], [1.0, 1.0])
-
-  @combinations.generate(keras_correctness_test_base.
-                         all_strategy_combinations_with_eager_and_graph_modes())
-  def test_simple_dnn_metric_correctness(self, distribution):
-    self.run_metric_correctness_test(distribution)
-
-
-class TestDistributionStrategyDnnMetricEvalCorrectness(
-    keras_correctness_test_base.TestDistributionStrategyCorrectnessBase):
-
-  def get_model(self, distribution=None):
-    with distribution.scope():
-      model = keras.Sequential()
-      model.add(
-          keras.layers.Dense(
-              3, activation='relu', input_dim=4, kernel_initializer='ones'))
-      model.add(
-          keras.layers.Dense(
-              1, activation='sigmoid', kernel_initializer='ones'))
-      model.compile(
-          loss='mae',
-          metrics=['accuracy', keras.metrics.BinaryAccuracy()],
-          optimizer=gradient_descent.GradientDescentOptimizer(0.001))
-    return model
-
-  def run_eval_metrics_correctness_test(self, distribution):
-    with self.cached_session():
-      self.set_up_test_config()
-      self.skip_unsupported_test_configuration(distribution)
-
-      model = self.get_model(distribution=distribution)
-
-      # verify correctness of stateful and stateless metrics.
-      x = np.ones((100, 4)).astype('float32')
-      y = np.ones((100, 1)).astype('float32')
-      dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat()
-      dataset = (keras_correctness_test_base.
-                 batch_wrapper(dataset, 4, distribution))
-      outs = model.evaluate(dataset, steps=10)
-      self.assertEqual(outs[1], 1.)
-      self.assertEqual(outs[2], 1.)
-
-      y = np.zeros((100, 1)).astype('float32')
-      dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat()
-      dataset = (keras_correctness_test_base.
-                 batch_wrapper(dataset, 4, distribution))
-      outs = model.evaluate(dataset, steps=10)
-      self.assertEqual(outs[1], 0.)
-      self.assertEqual(outs[2], 0.)
-
-  @combinations.generate(keras_correctness_test_base.
-                         all_strategy_combinations_with_eager_and_graph_modes())
-  def test_identity_model_metric_eval_correctness(self, distribution):
-    self.run_eval_metrics_correctness_test(distribution)
-
-
-if __name__ == '__main__':
-  test.main()
diff --git a/tensorflow/contrib/distribute/python/keras_image_model_correctness_test.py b/tensorflow/contrib/distribute/python/keras_image_model_correctness_test.py
deleted file mode 100644
index f625664372..0000000000
--- a/tensorflow/contrib/distribute/python/keras_image_model_correctness_test.py
+++ /dev/null
@@ -1,92 +0,0 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Correctness tests for tf.keras CNN models using DistributionStrategy."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-
-from tensorflow.contrib.distribute.python import combinations
-from tensorflow.contrib.distribute.python import keras_correctness_test_base
-from tensorflow.python import keras
-from tensorflow.python.eager import test
-from tensorflow.python.training import gradient_descent
-
-
-class DistributionStrategyCnnCorrectnessTest(
-    keras_correctness_test_base.TestDistributionStrategyCorrectnessBase):
-
-  def get_model(self, initial_weights=None, distribution=None):
-    with keras_correctness_test_base.MaybeDistributionScope(distribution):
-      image = keras.layers.Input(shape=(28, 28, 3), name='image')
-      c1 = keras.layers.Conv2D(
-          name='conv1', filters=16, kernel_size=(3, 3), strides=(4, 4))(
-              image)
-      if self.with_batch_norm:
-        c1 = keras.layers.BatchNormalization(name='bn1')(c1)
-      c1 = keras.layers.MaxPooling2D(pool_size=(2, 2))(c1)
-      logits = keras.layers.Dense(
-          10, activation='softmax', name='pred')(
-              keras.layers.Flatten()(c1))
-      model = keras.Model(inputs=[image], outputs=[logits])
-
-      if initial_weights:
-        model.set_weights(initial_weights)
-
-      model.compile(
-          optimizer=gradient_descent.GradientDescentOptimizer(
-              learning_rate=0.1),
-          loss='sparse_categorical_crossentropy',
-          metrics=['sparse_categorical_accuracy'])
-
-    return model
-
-  def get_data(self,
-               count=keras_correctness_test_base._GLOBAL_BATCH_SIZE
-               * keras_correctness_test_base._EVAL_STEPS,
-               shape=(28, 28, 3),
-               num_classes=10):
-    centers = np.random.randn(num_classes, *shape)
-
-    features = []
-    labels = []
-    for _ in range(count):
-      label = np.random.randint(0, num_classes, size=1)[0]
-      offset = np.random.normal(loc=0, scale=0.1, size=np.prod(shape))
-      offset = offset.reshape(shape)
-      labels.append(label)
-      features.append(centers[label] + offset)
-
-    x_train = np.asarray(features, dtype=np.float32)
-    y_train = np.asarray(labels, dtype=np.float32).reshape((count, 1))
-    x_predict = x_train
-    return x_train, y_train, x_predict
-
-  @combinations.generate(keras_correctness_test_base.
-                         all_strategy_and_input_config_combinations())
-  def test_cnn_correctness(self, distribution, use_numpy, use_validation_data):
-    self.run_correctness_test(distribution, use_numpy, use_validation_data)
-
-  @combinations.generate(keras_correctness_test_base.
-                         all_strategy_and_input_config_combinations())
-  def test_cnn_with_batch_norm_correctness(self, distribution, use_numpy,
-                                           use_validation_data):
-    self.run_correctness_test(distribution, use_numpy, use_validation_data,
-                              with_batch_norm=True)
-
-
-if __name__ == '__main__':
-  test.main()
-- 
GitLab


From a76a7e8545c6f0255d7607ca817fa92ed7b08349 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 12:29:21 -0800
Subject: [PATCH 0504/2345] Use CUB-based row reduction for sparse_xent_op. The
 Eigen implementation seems to trigger a bug on Volta GPUs.

PiperOrigin-RevId: 228756250
---
 tensorflow/core/kernels/BUILD                 |  5 ++-
 tensorflow/core/kernels/sparse_xent_op.cc     |  9 +++--
 tensorflow/core/kernels/sparse_xent_op.h      | 29 ++++++++++++++--
 .../core/kernels/sparse_xent_op_gpu.cu.cc     | 34 +++++++++++++++++--
 4 files changed, 65 insertions(+), 12 deletions(-)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 5794ba520e..1a20e8628e 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -4487,7 +4487,10 @@ tf_kernel_library(
     deps = SPARSE_DEPS + [
         ":bounds_check",
         "//third_party/eigen3",
-    ],
+    ] + if_cuda([
+        ":reduction_ops",
+        "@cub_archive//:cub",
+    ]),
 )
 
 tf_kernel_library(
diff --git a/tensorflow/core/kernels/sparse_xent_op.cc b/tensorflow/core/kernels/sparse_xent_op.cc
index f84ffd5323..37d4d0661c 100644
--- a/tensorflow/core/kernels/sparse_xent_op.cc
+++ b/tensorflow/core/kernels/sparse_xent_op.cc
@@ -90,9 +90,8 @@ class SparseSoftmaxXentWithLogitsOp : public OpKernel {
             context, CheckInvalidLabelIndex<Index>(labels, logits.dim_size(1)));
       }
       functor::SparseXentFunctor<Device, T, Index> functor;
-      functor(context->eigen_device<Device>(), logits.matrix<T>(),
-              labels.vec<Index>(), scratch.vec<T>(), loss_out->vec<T>(),
-              back_out->matrix<T>());
+      functor(context, logits.matrix<T>(), labels.vec<Index>(),
+              scratch.vec<T>(), loss_out->vec<T>(), back_out->matrix<T>());
     }
   }
 };
@@ -102,11 +101,11 @@ class SparseSoftmaxXentWithLogitsOp : public OpKernel {
 namespace functor {
 template <typename T, typename Index>
 struct SparseXentFunctor<CPUDevice, T, Index> {
-  void operator()(const CPUDevice& d, typename TTypes<T>::ConstMatrix logits,
+  void operator()(OpKernelContext* ctx, typename TTypes<T>::ConstMatrix logits,
                   typename TTypes<Index>::ConstVec labels,
                   typename TTypes<T>::Vec scratch, typename TTypes<T>::Vec loss,
                   typename TTypes<T>::Matrix backprop) {
-    SparseXentEigenImpl<CPUDevice, T, Index>::Compute(d, logits, labels,
+    SparseXentEigenImpl<CPUDevice, T, Index>::Compute(ctx, logits, labels,
                                                       scratch, loss, backprop);
   }
 };
diff --git a/tensorflow/core/kernels/sparse_xent_op.h b/tensorflow/core/kernels/sparse_xent_op.h
index 6ba7931ab5..5e462424ed 100644
--- a/tensorflow/core/kernels/sparse_xent_op.h
+++ b/tensorflow/core/kernels/sparse_xent_op.h
@@ -18,6 +18,7 @@ limitations under the License.
 // Functor definition for SparseXentOp, must be compilable by nvcc.
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/platform/macros.h"
@@ -128,6 +129,26 @@ class SparseXentGradGenerator {
 
 namespace functor {
 
+template <typename Device, typename T>
+struct RowMaxReduction {
+  // Computes the maximum across the rows of logits
+  //
+  // logits: batch_size, num_classes.
+  // maximum: temporary tensor, dims: batch_size, 1
+  static inline void Compute(OpKernelContext* ctx,
+                             typename TTypes<T>::ConstMatrix logits,
+                             typename TTypes<T>::Vec maximum) {
+#if !defined(EIGEN_HAS_INDEX_LIST)
+    Eigen::array<int, 1> along_row;
+    along_row[0] = 1;
+#else
+    Eigen::IndexList<Eigen::type2index<1> > along_row;
+#endif
+    Device d = ctx->eigen_device<Device>();
+    To32Bit(maximum).device(d) = To32Bit(logits).maximum(along_row);
+  }
+};
+
 // Functor used by SparseXentOp to do the computations.
 template <typename Device, typename T, typename Index>
 struct SparseXentFunctor {
@@ -138,7 +159,7 @@ struct SparseXentFunctor {
   // scratch: temporary tensor, dims: batch_size, 1
   // loss: output tensor for the loss, dims: batch_size.
   // backprop: output tensor for the backprop, dims: batch_size, num_classes.
-  void operator()(const Device& d, typename TTypes<T>::ConstMatrix logits,
+  void operator()(OpKernelContext* ctx, typename TTypes<T>::ConstMatrix logits,
                   typename TTypes<Index>::ConstVec labels,
                   typename TTypes<T>::Vec scratch, typename TTypes<T>::Vec loss,
                   typename TTypes<T>::Matrix backprop);
@@ -149,7 +170,8 @@ struct SparseXentFunctor {
 // specializations for both device types.
 template <typename Device, typename T, typename Index>
 struct SparseXentEigenImpl {
-  static void Compute(const Device& d, typename TTypes<T>::ConstMatrix logits,
+  static void Compute(OpKernelContext* ctx,
+                      typename TTypes<T>::ConstMatrix logits,
                       typename TTypes<Index>::ConstVec labels,
                       typename TTypes<T>::Vec scratch,
                       typename TTypes<T>::Vec loss,
@@ -188,8 +210,9 @@ struct SparseXentEigenImpl {
 #endif
 
     // scratch = max_logits along classes.
-    To32Bit(scratch).device(d) = To32Bit(logits).maximum(along_class);
+    RowMaxReduction<Device, T>::Compute(ctx, logits, scratch);
 
+    Device d = ctx->eigen_device<Device>();
     // backprop = logits - max_logits.
     To32Bit(backprop).device(d) =
         To32Bit(logits) -
diff --git a/tensorflow/core/kernels/sparse_xent_op_gpu.cu.cc b/tensorflow/core/kernels/sparse_xent_op_gpu.cu.cc
index d053966028..5fe15352c3 100644
--- a/tensorflow/core/kernels/sparse_xent_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/sparse_xent_op_gpu.cu.cc
@@ -20,22 +20,50 @@ limitations under the License.
 #include "tensorflow/core/kernels/sparse_xent_op.h"
 
 #include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/kernels/reduction_gpu_kernels.cu.h"
+#include "tensorflow/core/kernels/reduction_ops_common.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
 
 typedef Eigen::GpuDevice GPUDevice;
 
+namespace functor {
+
+// Partial specialization for a GPUDevice, that uses the CUB implementation
+// from reduction_gpu_kernels.cu.h.
+template <typename T>
+struct RowMaxReduction<GPUDevice, T> {
+  // Computes the maximum across the rows of logits
+  //
+  // logits: batch_size, num_classes.
+  // maximum: temporary tensor, dims: batch_size, 1
+  static inline void Compute(OpKernelContext* ctx,
+                             typename TTypes<T>::ConstMatrix logits,
+                             typename TTypes<T>::Vec maximum) {
+    const int kBatchDim = 0;
+    const int kClassDim = 1;
+    const int rows = logits.dimension(kBatchDim);
+    const int cols = logits.dimension(kClassDim);
+
+    typedef const Eigen::array<TTypes<float>::Tensor::Index, 1>& ReductionAxes;
+    Constants<GPUDevice> constants;
+    cub::Max op;
+    functor::ReduceImpl<T, cub::Max, T*, const T*, ReductionAxes>(
+        ctx, maximum.data(), logits.data(), 2, rows, cols, 1, 1, constants.kOne,
+        op);
+  }
+};
+
 // Partial specialization for a GPUDevice, that uses the Eigen implementation
 // from XentEigenImpl.
-namespace functor {
 template <typename T, typename Index>
 struct SparseXentFunctor<GPUDevice, T, Index> {
-  void operator()(const GPUDevice& d, typename TTypes<T>::ConstMatrix logits,
+  void operator()(OpKernelContext* ctx, typename TTypes<T>::ConstMatrix logits,
                   typename TTypes<Index>::ConstVec labels,
                   typename TTypes<T>::Vec scratch, typename TTypes<T>::Vec loss,
                   typename TTypes<T>::Matrix backprop) {
-    SparseXentEigenImpl<GPUDevice, T, Index>::Compute(d, logits, labels,
+    SparseXentEigenImpl<GPUDevice, T, Index>::Compute(ctx, logits, labels,
                                                       scratch, loss, backprop);
   }
 };
-- 
GitLab


From 0b3c3c55e177b35d38ba33170ebe2baa3f5badff Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Thu, 10 Jan 2019 12:30:40 -0800
Subject: [PATCH 0505/2345] Be resilient towards graphs without source/sink
 connectivity.

PiperOrigin-RevId: 228756459
---
 tensorflow/compiler/jit/mark_for_compilation_pass.cc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass.cc b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
index 6618e3a58a..50afec020d 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
@@ -677,6 +677,11 @@ Status MarkForCompilationPass::Run(
   VLOG(1) << "flags->tf_xla_auto_jit = " << flags->tf_xla_auto_jit;
   const FunctionLibraryDefinition* fld = options.flib_def;
 
+  // Deadness analysis expects a graph with source and sink edges properly
+  // connected but sometimes the incoming graph does not follow this invariant.
+  // So fix up the source and sink edges before calling into deadness analysis.
+  FixupSourceAndSinkEdges(options.graph->get());
+
   std::unique_ptr<DeadnessAnalysis> deadness;
   {
     XLA_SCOPED_LOGGING_TIMER_LEVEL("DeadnessAnalysis", 1);
-- 
GitLab


From 9c45814aac694a6454d312225121dd4c4cd33de9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 12:45:38 -0800
Subject: [PATCH 0506/2345] Update documentation of `tf.distribute.Strategy
 methods` to clarify and/or correct whether they expect global or per-replica
 batching.

PiperOrigin-RevId: 228758901
---
 .../python/collective_all_reduce_strategy.py  |  8 ++++
 .../distribute/python/mirrored_strategy.py    | 26 +++++++++++
 .../distribute/python/one_device_strategy.py  |  1 +
 .../python/parameter_server_strategy.py       | 26 +++++++++++
 .../contrib/distribute/python/tpu_strategy.py |  9 +++-
 .../python/distribute/distribute_lib.py       | 44 ++++++++++++-------
 .../python/distribute/mirrored_strategy.py    |  8 ++++
 .../distribute/parameter_server_strategy.py   |  8 ++++
 8 files changed, 113 insertions(+), 17 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
index a3538dfc50..eee0754325 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
@@ -372,4 +372,12 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
   # TODO(priyag): Delete this once all strategies use global batch size.
   @property
   def _global_batch_size(self):
+    """`make_dataset_iterator` and `make_numpy_iterator` use global batch size.
+
+    `distribute_dataset` and `make_input_fn_iterator` assume per-replica
+    batching.
+
+    Returns:
+      Boolean.
+    """
     return True
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index 5fa36fb402..2e23a51ee5 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -104,6 +104,31 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
                                 auto_shard_dataset)
     super(MirroredStrategy, self).__init__(extended)
 
+  # Override to change the documentation to reflect the different handling of
+  # global vs. local batch size between core and contrib.
+  def make_dataset_iterator(self, dataset):  # pylint: disable=useless-super-delegation
+    """Makes an iterator for input provided via `dataset`.
+
+    NOTE: The batch size of the `dataset` argument is treated differently for
+    this contrib version of `MirroredStrategy`.
+
+    Data from the given dataset will be distributed evenly across all the
+    compute replicas. We will assume that the input dataset is batched by the
+    per-replica batch size.
+
+    The user could also use `make_input_fn_iterator` if they want to
+    customize which input is fed to which replica/worker etc.
+
+    Args:
+      dataset: `tf.data.Dataset` that will be distributed evenly across all
+        replicas.
+
+    Returns:
+      An `tf.distribute.InputIterator` which returns inputs for each step of the
+      computation.  User should call `initialize` on the returned iterator.
+    """
+    return super(MirroredStrategy, self).make_dataset_iterator(dataset)
+
   # Override to change the documentation to reflect the different handling of
   # global vs. local batch size between core and contrib.
   def experimental_make_numpy_iterator(  # pylint: disable=useless-super-delegation
@@ -180,4 +205,5 @@ class MirroredExtended(CoreMirroredExtended):
   # TODO(priyag): Delete this once all strategies use global batch size.
   @property
   def _global_batch_size(self):
+    """The contrib version of Mirrored strategy uses per-replica batch size."""
     return False
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py
index 34b0c31087..836cb7cc41 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy.py
@@ -199,6 +199,7 @@ class OneDeviceExtended(distribute_lib.DistributionStrategyExtended):
   # TODO(priyag): Delete this once all strategies use global batch size.
   @property
   def _global_batch_size(self):
+    """Global and per-replica batching are equivalent for OneDeviceStrategy."""
     return True
 
 
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy.py b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
index 0785427c2c..0cefef7545 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
@@ -89,6 +89,31 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy):
     super(ParameterServerStrategy, self).__init__(
         ParameterServerExtended(self, num_gpus_per_worker))
 
+  # Override to change the documentation to reflect the different handling of
+  # global vs. local batch size between core and contrib.
+  def make_dataset_iterator(self, dataset):  # pylint: disable=useless-super-delegation
+    """Makes an iterator for input provided via `dataset`.
+
+    NOTE: The batch size of the `dataset` argument is treated differently for
+    this contrib version of `ParameterServerStrategy`.
+
+    Data from the given dataset will be distributed evenly across all the
+    compute replicas. We will assume that the input dataset is batched by the
+    per-replica batch size.
+
+    The user could also use `make_input_fn_iterator` if they want to
+    customize which input is fed to which replica/worker etc.
+
+    Args:
+      dataset: `tf.data.Dataset` that will be distributed evenly across all
+        replicas.
+
+    Returns:
+      An `tf.distribute.InputIterator` which returns inputs for each step of the
+      computation.  User should call `initialize` on the returned iterator.
+    """
+    return super(ParameterServerStrategy, self).make_dataset_iterator(dataset)
+
   # Override to change the documentation to reflect the different handling of
   # global vs. local batch size between core and contrib.
   def experimental_make_numpy_iterator(  # pylint: disable=useless-super-delegation
@@ -143,4 +168,5 @@ class ParameterServerExtended(CoreParameterServerExtended):
   # TODO(priyag): Delete this once all strategies use global batch size.
   @property
   def _global_batch_size(self):
+    """The contrib version of PS strategy uses per-replica batch size."""
     return False
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 3f89c5869e..518c704b89 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -295,7 +295,6 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
 
   def _make_dataset_iterator(self, dataset):
     """Make iterators for each of the TPU hosts."""
-
     return input_lib.DatasetIterator(dataset, self._input_workers,
                                      self._num_replicas_in_sync)
 
@@ -667,6 +666,14 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
   # TODO(priyag): Delete this once all strategies use global batch size.
   @property
   def _global_batch_size(self):
+    """`make_dataset_iterator` and `make_numpy_iterator` use global batch size.
+
+    `distribute_dataset` and `make_input_fn_iterator` assume per-replica
+    batching.
+
+    Returns:
+      Boolean.
+    """
     return True
 
 
diff --git a/tensorflow/python/distribute/distribute_lib.py b/tensorflow/python/distribute/distribute_lib.py
index 31213ab647..5fe77e5478 100644
--- a/tensorflow/python/distribute/distribute_lib.py
+++ b/tensorflow/python/distribute/distribute_lib.py
@@ -210,12 +210,14 @@ class _SameScopeAgainContext(object):
 # TODO(yuefengz): add more replication modes.
 @tf_export("distribute.InputReplicationMode")
 class InputReplicationMode(enum.Enum):
-  """Replication mode for input function."""
+  """Replication mode for input function.
 
-  # The input function will be called on each worker independently, creating as
-  # many input pipelines as number of workers. Replicas will dequeue from the
-  # local Dataset on their worker. Distribution Strategy doesn't manage any
-  # state sharing between such separate input pipelines.
+  * `PER_WORKER`: The input function will be called on each worker
+    independently, creating as many input pipelines as number of workers.
+    Replicas will dequeue from the local Dataset on their worker.
+    `tf.distribute.Strategy` doesn't manage any state sharing between such
+    separate input pipelines.
+  """
   PER_WORKER = "PER_WORKER"
 
 
@@ -353,7 +355,8 @@ class DistributionStrategy(object):
     ```
 
     Args:
-      dataset_fn: A function that returns a `tf.data.Dataset`.
+      dataset_fn: A function that returns a `tf.data.Dataset` with per-replica
+        batching.
 
     Returns:
       A `PerReplicaDataset` that will produce data for each replica.
@@ -390,28 +393,36 @@ class DistributionStrategy(object):
     """Returns an iterator split across replicas created from an input function.
 
     The `input_fn` should take an `tf.distribute.InputContext` object where
-    information about input sharding can be accessed:
+    information about batching and input sharding can be accessed:
 
     ```
     def input_fn(input_context):
-      d = tf.data.Dataset.from_tensors([[1.]]).repeat()
+      batch_size = input_context.get_per_replica_batch_size(global_batch_size)
+      d = tf.data.Dataset.from_tensors([[1.]]).repeat().batch(batch_size)
       return d.shard(input_context.num_input_pipelines,
                      input_context.input_pipeline_id)
     with strategy.scope():
-      iterator = strategy.make_input_fn_iterator(
-          input_fn)
-      replica_results = strategy.extended.call_for_each_replica(
-          replica_fn, iterator.get_next())
+      iterator = strategy.make_input_fn_iterator(input_fn)
+      replica_results = strategy.experimental_run(replica_fn, iterator)
     ```
 
+    The `tf.data.Dataset` returned by `input_fn` should have a per-replica
+    batch size, which may be computed using
+    `input_context.get_per_replica_batch_size`.
+
     Args:
-      input_fn: A function that returns a `tf.data.Dataset`. This function is
-        expected to take an `tf.distribute.InputContext` object.
+      input_fn: A function taking a `tf.distribute.InputContext` object and
+        returning a `tf.data.Dataset`.
       replication_mode: an enum value of `tf.distribute.InputReplicationMode`.
-        Only `PER_WORKER` is supported currently.
+        Only `PER_WORKER` is supported currently, which means there will be
+        a single call to `input_fn` per worker. Replicas will dequeue from the
+        local `tf.data.Dataset` on their worker.
 
     Returns:
-      An iterator object that can be initialized and fetched next element.
+      An iterator object that should first be `.initialize()`-ed. It may then
+      either be passed to `strategy.experimental_run()` or you can
+      `iterator.get_next()` to get the next value to pass to
+      `strategy.extended.call_for_each_replica()`.
     """
     if replication_mode != InputReplicationMode.PER_WORKER:
       raise ValueError(
@@ -1813,6 +1824,7 @@ class _DefaultDistributionExtended(DistributionStrategyExtended):
   # TODO(priyag): Delete this once all strategies use global batch size.
   @property
   def _global_batch_size(self):
+    """Global and per-replica batching are equivalent for this strategy."""
     return True
 
 
diff --git a/tensorflow/python/distribute/mirrored_strategy.py b/tensorflow/python/distribute/mirrored_strategy.py
index c0a39d4b55..1ed7eef1ed 100644
--- a/tensorflow/python/distribute/mirrored_strategy.py
+++ b/tensorflow/python/distribute/mirrored_strategy.py
@@ -782,6 +782,14 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
   # TODO(priyag): Delete this once all strategies use global batch size.
   @property
   def _global_batch_size(self):
+    """`make_dataset_iterator` and `make_numpy_iterator` use global batch size.
+
+    `distribute_dataset` and `make_input_fn_iterator` assume per-replica
+    batching.
+
+    Returns:
+      Boolean.
+    """
     return True
 
 
diff --git a/tensorflow/python/distribute/parameter_server_strategy.py b/tensorflow/python/distribute/parameter_server_strategy.py
index ac5ee6f589..9e865082f1 100644
--- a/tensorflow/python/distribute/parameter_server_strategy.py
+++ b/tensorflow/python/distribute/parameter_server_strategy.py
@@ -536,4 +536,12 @@ class ParameterServerStrategyExtended(
   # TODO(priyag): Delete this once all strategies use global batch size.
   @property
   def _global_batch_size(self):
+    """`make_dataset_iterator` and `make_numpy_iterator` use global batch size.
+
+    `distribute_dataset` and `make_input_fn_iterator` assume per-replica
+    batching.
+
+    Returns:
+      Boolean.
+    """
     return True
-- 
GitLab


From a23420e75ddf38b92c93fe80e0d83ea7b70b11b1 Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Thu, 10 Jan 2019 12:47:37 -0800
Subject: [PATCH 0507/2345] Enable test in OSS

PiperOrigin-RevId: 228759239
---
 tensorflow/lite/kernels/BUILD | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index 50deac9bb6..7a4b6b8644 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -585,9 +585,6 @@ tf_cc_test(
     name = "bidirectional_sequence_rnn_test",
     size = "small",
     srcs = ["bidirectional_sequence_rnn_test.cc"],
-    tags = [
-        "tflite_not_portable",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
-- 
GitLab


From b1876c7c25cb1d40c79a77c21f1ae05deca36cc4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 12:52:05 -0800
Subject: [PATCH 0508/2345] Migrate to new names: * get_distribution_strategy
 -> get_strategy * has_distribution_strategy -> has_strategy

PiperOrigin-RevId: 228759939
---
 tensorflow/contrib/distribute/__init__.py             | 2 ++
 tensorflow/contrib/optimizer_v2/optimizer_v2.py       | 3 +--
 tensorflow/python/distribute/input_lib.py             | 2 +-
 tensorflow/python/distribute/values.py                | 4 ++--
 tensorflow/python/eager/tape.py                       | 4 ++--
 tensorflow/python/framework/test_util.py              | 2 +-
 tensorflow/python/keras/engine/training.py            | 4 ++--
 tensorflow/python/keras/layers/normalization.py       | 9 +++++----
 tensorflow/python/keras/optimizer_v2/optimizer_v2.py  | 8 +++-----
 tensorflow/python/keras/optimizers.py                 | 2 +-
 tensorflow/python/training/optimizer.py               | 5 ++---
 tensorflow/python/training/session_manager.py         | 2 +-
 tensorflow/python/training/slot_creator.py            | 6 ++----
 tensorflow/python/training/sync_replicas_optimizer.py | 3 +--
 14 files changed, 26 insertions(+), 30 deletions(-)

diff --git a/tensorflow/contrib/distribute/__init__.py b/tensorflow/contrib/distribute/__init__.py
index 4c3f9b8f02..1bcc453a7e 100644
--- a/tensorflow/contrib/distribute/__init__.py
+++ b/tensorflow/contrib/distribute/__init__.py
@@ -64,7 +64,9 @@ _allowed_symbols = [
     'get_distribution_strategy',
     'get_loss_reduction',
     'get_replica_context',
+    'get_strategy',
     'has_distribution_strategy',
+    'has_strategy',
     'in_cross_replica_context',
     'require_replica_context',
     'run_standard_tensorflow_server',
diff --git a/tensorflow/contrib/optimizer_v2/optimizer_v2.py b/tensorflow/contrib/optimizer_v2/optimizer_v2.py
index 7fb23abc38..1323ed014c 100644
--- a/tensorflow/contrib/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/contrib/optimizer_v2/optimizer_v2.py
@@ -843,8 +843,7 @@ class OptimizerV2(optimizer_v1.Optimizer):
       scale_loss_by_num_replicas = (
           distribute_lib.get_loss_reduction() == ds_reduce_util.ReduceOp.MEAN)
     if scale_loss_by_num_replicas:
-      num_replicas = \
-        distribute_ctx.get_distribution_strategy().num_replicas_in_sync
+      num_replicas = distribute_ctx.get_strategy().num_replicas_in_sync
       if num_replicas > 1:
         loss_value *= 1. / num_replicas
     return loss_value
diff --git a/tensorflow/python/distribute/input_lib.py b/tensorflow/python/distribute/input_lib.py
index 022595146d..c64eea1604 100644
--- a/tensorflow/python/distribute/input_lib.py
+++ b/tensorflow/python/distribute/input_lib.py
@@ -689,7 +689,7 @@ class MultiStepContext(object):
       if reduce_op is None:
         self._last_step_outputs[name] = output
       else:
-        distribution = distribution_strategy_context.get_distribution_strategy()
+        distribution = distribution_strategy_context.get_strategy()
         self._last_step_outputs[name] = distribution.reduce(reduce_op, output)
     else:
       assert reduce_op is not None
diff --git a/tensorflow/python/distribute/values.py b/tensorflow/python/distribute/values.py
index a9dcabdab6..2a57d5cfc7 100644
--- a/tensorflow/python/distribute/values.py
+++ b/tensorflow/python/distribute/values.py
@@ -411,11 +411,11 @@ def _assign_on_device(device, variable, tensor):
 
 
 def _assert_strategy(strategy):
-  if not distribution_strategy_context.has_distribution_strategy():
+  if not distribution_strategy_context.has_strategy():
     raise RuntimeError(
         'Need to be inside "with strategy.scope()" for %s' %
         (strategy,))
-  current_strategy = distribution_strategy_context.get_distribution_strategy()
+  current_strategy = distribution_strategy_context.get_strategy()
   if current_strategy is not strategy:
     raise RuntimeError(
         "Mixing different tf.distribute.Strategy objects: %s is not %s" %
diff --git a/tensorflow/python/eager/tape.py b/tensorflow/python/eager/tape.py
index e501b403a3..56b68b9eea 100644
--- a/tensorflow/python/eager/tape.py
+++ b/tensorflow/python/eager/tape.py
@@ -61,7 +61,7 @@ def watch(tape, tensor):
 
 def watch_variable(tape, variable):
   """Marks this variable to be watched by the given tape."""
-  strategy = distribution_strategy_context.get_distribution_strategy()
+  strategy = distribution_strategy_context.get_strategy()
   if distribution_strategy_context.get_replica_context():
     variables = [strategy.extended.value_container(variable)]
   else:
@@ -76,7 +76,7 @@ def variable_accessed(variable):
   Args:
     variable: variable to be watched.
   """
-  strategy = distribution_strategy_context.get_distribution_strategy()
+  strategy = distribution_strategy_context.get_strategy()
   if distribution_strategy_context.get_replica_context():
     variables = [strategy.extended.value_container(variable)]
   else:
diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index 21d21cc7f4..6d01d3bf54 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -735,7 +735,7 @@ def assert_no_garbage_created(f):
     """Sets DEBUG_SAVEALL, runs the test, and checks for new garbage."""
     # Force-load `distribution_strategy_context` to prevent GC at
     # test time when using eager. Remove once b/117329403 is resolved.
-    tape.distribution_strategy_context.get_distribution_strategy()
+    tape.distribution_strategy_context.get_strategy()
 
     gc.disable()
     previous_debug_flags = gc.get_debug()
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index a65c2b6413..0b1743af38 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -216,14 +216,14 @@ class Model(Network):
       self._distribution_strategy = distribute
       self._compile_distribution = True
     else:
-      if distribution_strategy_context.has_distribution_strategy():
+      if distribution_strategy_context.has_strategy():
         # When the user builds the model in the DS scope and cross replica
         # context we want distribution strategy to be set but when building the
         # replica copies of the models internally we should not be compiling
         # with distribution strategy and use the default compilation path.
         if distribution_strategy_context.in_cross_replica_context():
           self._distribution_strategy = (
-              distribution_strategy_context.get_distribution_strategy())
+              distribution_strategy_context.get_strategy())
 
     # Validate that arguments passed by the user to `compile` are supported by
     # DistributionStrategy.
diff --git a/tensorflow/python/keras/layers/normalization.py b/tensorflow/python/keras/layers/normalization.py
index 98bc10881b..c174c8ddd6 100644
--- a/tensorflow/python/keras/layers/normalization.py
+++ b/tensorflow/python/keras/layers/normalization.py
@@ -417,8 +417,8 @@ class BatchNormalizationV2(Layer):
       # since TPUStrategy does not implement replica local variables.
       # Remove this hack once we support TPULocalVariables.
       is_tpu_strategy = False
-      if distribution_strategy_context.has_distribution_strategy():
-        distribute = distribution_strategy_context.get_distribution_strategy()
+      if distribution_strategy_context.has_strategy():
+        distribute = distribution_strategy_context.get_strategy()
         if distribute.__class__.__name__ == 'TPUStrategy':
           is_tpu_strategy = True
 
@@ -474,7 +474,7 @@ class BatchNormalizationV2(Layer):
       momentum = ops.convert_to_tensor(self.momentum)
     if training_value or training_value is None:
       if distribution_strategy_context.in_cross_replica_context():
-        strategy = distribution_strategy_context.get_distribution_strategy()
+        strategy = distribution_strategy_context.get_strategy()
         mean_update = strategy.extended.update(
             self.moving_mean, self._assign_moving_average,
             (mean, self.momentum))
@@ -666,7 +666,8 @@ class BatchNormalizationV2(Layer):
         scale, offset = _compose_transforms(r, d, scale, offset)
 
       if distribution_strategy_context.in_cross_replica_context():
-        strategy = distribution_strategy_context.get_distribution_strategy()
+        strategy = distribution_strategy_context.get_strategy()
+
         def _do_update(var, value):
           """Compute the updates for mean and variance."""
           if in_eager_mode and not self.trainable:
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
index 98f87a41ae..894af66f5d 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
@@ -290,8 +290,7 @@ class OptimizerV2(checkpointable.CheckpointableBase):
   @staticmethod
   def _scale_loss(loss_value):
     if distribute_lib.get_loss_reduction() == ds_reduce_util.ReduceOp.MEAN:
-      num_replicas = \
-        distribute_ctx.get_distribution_strategy().num_replicas_in_sync
+      num_replicas = distribute_ctx.get_strategy().num_replicas_in_sync
       if num_replicas > 1:
         loss_value *= (1. / num_replicas)
     return loss_value
@@ -349,7 +348,7 @@ class OptimizerV2(checkpointable.CheckpointableBase):
     """
     grads_and_vars = _filter_grads(grads_and_vars)
     var_list = [v for (_, v) in grads_and_vars]
-    if distribute_ctx.has_distribution_strategy():
+    if distribute_ctx.has_strategy():
       reduced_grads = merge_grads(grads_and_vars)
       grads_and_vars = zip(reduced_grads, var_list)
 
@@ -900,8 +899,7 @@ def _var_key(var):
   """
 
   # pylint: disable=protected-access
-  if distribute_ctx.has_distribution_strategy() and hasattr(
-      var, "_primary_var"):
+  if distribute_ctx.has_strategy() and hasattr(var, "_primary_var"):
     var = var._primary_var
   if hasattr(var, "op"):
     return var._shared_name
diff --git a/tensorflow/python/keras/optimizers.py b/tensorflow/python/keras/optimizers.py
index 12168e5574..82fb555b57 100644
--- a/tensorflow/python/keras/optimizers.py
+++ b/tensorflow/python/keras/optimizers.py
@@ -740,7 +740,7 @@ class TFOptimizer(Optimizer, checkpointable.CheckpointableBase):
     return self.optimizer.compute_gradients(loss, params)
 
   def get_updates(self, loss, params):
-    if distribution_strategy_context.has_distribution_strategy():
+    if distribution_strategy_context.has_strategy():
       self.updates = []
 
       if not params:
diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py
index c6cc0b6044..8076ed31bf 100644
--- a/tensorflow/python/training/optimizer.py
+++ b/tensorflow/python/training/optimizer.py
@@ -521,8 +521,7 @@ class Optimizer(
   @staticmethod
   def _scale_loss(loss_value):
     if distribute_lib.get_loss_reduction() == ds_reduce_util.ReduceOp.MEAN:
-      num_replicas = \
-        distribute_ctx.get_distribution_strategy().num_replicas_in_sync
+      num_replicas = distribute_ctx.get_strategy().num_replicas_in_sync
       if num_replicas > 1:
         loss_value *= (1. / num_replicas)
     return loss_value
@@ -816,7 +815,7 @@ class Optimizer(
     v = self._non_slot_dict.get(key, None)
     if v is None:
       self._maybe_initialize_checkpointable()
-      distribution_strategy = distribute_ctx.get_distribution_strategy()
+      distribution_strategy = distribute_ctx.get_strategy()
       with distribution_strategy.colocate_vars_with(colocate_with):
         if eager:
           restored_initial_value = self._preload_simple_restoration(
diff --git a/tensorflow/python/training/session_manager.py b/tensorflow/python/training/session_manager.py
index cd22e7e7de..7bd0891e35 100644
--- a/tensorflow/python/training/session_manager.py
+++ b/tensorflow/python/training/session_manager.py
@@ -186,7 +186,7 @@ class SessionManager(object):
     # This is required to so that we initialize the TPU device before
     # restoring from checkpoint since we'll be placing variables on the device
     # and TPUInitialize wipes out the memory of the device.
-    strategy = distribution_strategy_context.get_distribution_strategy()
+    strategy = distribution_strategy_context.get_strategy()
     if strategy and hasattr(strategy.extended,
                             "_experimental_initialize_system"):
       strategy.extended._experimental_initialize_system()  # pylint: disable=protected-access
diff --git a/tensorflow/python/training/slot_creator.py b/tensorflow/python/training/slot_creator.py
index bc1137e200..abe1253b00 100644
--- a/tensorflow/python/training/slot_creator.py
+++ b/tensorflow/python/training/slot_creator.py
@@ -121,8 +121,7 @@ def create_slot(primary, val, name, colocate_with_primary=True):
     prefix = primary.op.name
   with variable_scope.variable_scope(None, prefix + "/" + name):
     if colocate_with_primary:
-      distribution_strategy = (
-          distribution_strategy_context.get_distribution_strategy())
+      distribution_strategy = distribution_strategy_context.get_strategy()
       with distribution_strategy.colocate_vars_with(primary):
         return _create_slot_var(primary, val, "", validate_shape, None, None)
     else:
@@ -159,8 +158,7 @@ def create_slot_with_initializer(primary, initializer, shape, dtype, name,
     prefix = primary.op.name
   with variable_scope.variable_scope(None, prefix + "/" + name):
     if colocate_with_primary:
-      distribution_strategy = (
-          distribution_strategy_context.get_distribution_strategy())
+      distribution_strategy = distribution_strategy_context.get_strategy()
       with distribution_strategy.colocate_vars_with(primary):
         return _create_slot_var(primary, initializer, "", validate_shape, shape,
                                 dtype)
diff --git a/tensorflow/python/training/sync_replicas_optimizer.py b/tensorflow/python/training/sync_replicas_optimizer.py
index cd4590db7f..0079ecc98b 100644
--- a/tensorflow/python/training/sync_replicas_optimizer.py
+++ b/tensorflow/python/training/sync_replicas_optimizer.py
@@ -260,8 +260,7 @@ class SyncReplicasOptimizer(optimizer.Optimizer):
     # local_anchor op will be placed on this worker task by default.
     local_anchor = control_flow_ops.no_op()
     # Colocating local_step variable prevents it being placed on the PS.
-    distribution_strategy = (
-        distribution_strategy_context.get_distribution_strategy())
+    distribution_strategy = distribution_strategy_context.get_strategy()
     with distribution_strategy.colocate_vars_with(local_anchor):
       self._local_step = variable_scope.variable(
           initial_value=0,
-- 
GitLab


From 0418e820833f88bc3545644d0aed81f51022a359 Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Thu, 10 Jan 2019 13:09:58 -0800
Subject: [PATCH 0509/2345] TfLite calibration library.

TfLite calibration library to capture activation ranges of tensors.

PiperOrigin-RevId: 228762955
---
 .../lite/tools/optimize/calibration_common.h  |  61 ++++
 .../lite/tools/optimize/calibration_logger.h  |  85 +++++
 .../lite/tools/optimize/calibration_reader.cc |  55 +++
 .../lite/tools/optimize/calibration_reader.h  |  55 +++
 tensorflow/lite/tools/optimize/calibrator.cc  | 345 ++++++++++++++++++
 tensorflow/lite/tools/optimize/calibrator.h   |  64 ++++
 .../lite/tools/optimize/calibrator_test.cc    | 189 ++++++++++
 .../tools/optimize/logging_op_resolver.cc     |  61 ++++
 .../lite/tools/optimize/logging_op_resolver.h |  59 +++
 .../optimize/logging_op_resolver_test.cc      | 141 +++++++
 .../lite/tools/optimize/node_info_delegate.cc |  66 ++++
 .../lite/tools/optimize/node_info_delegate.h  |  67 ++++
 .../tools/optimize/node_info_delegate_test.cc | 152 ++++++++
 13 files changed, 1400 insertions(+)
 create mode 100644 tensorflow/lite/tools/optimize/calibration_common.h
 create mode 100644 tensorflow/lite/tools/optimize/calibration_logger.h
 create mode 100644 tensorflow/lite/tools/optimize/calibration_reader.cc
 create mode 100644 tensorflow/lite/tools/optimize/calibration_reader.h
 create mode 100644 tensorflow/lite/tools/optimize/calibrator.cc
 create mode 100644 tensorflow/lite/tools/optimize/calibrator.h
 create mode 100644 tensorflow/lite/tools/optimize/calibrator_test.cc
 create mode 100644 tensorflow/lite/tools/optimize/logging_op_resolver.cc
 create mode 100644 tensorflow/lite/tools/optimize/logging_op_resolver.h
 create mode 100644 tensorflow/lite/tools/optimize/logging_op_resolver_test.cc
 create mode 100644 tensorflow/lite/tools/optimize/node_info_delegate.cc
 create mode 100644 tensorflow/lite/tools/optimize/node_info_delegate.h
 create mode 100644 tensorflow/lite/tools/optimize/node_info_delegate_test.cc

diff --git a/tensorflow/lite/tools/optimize/calibration_common.h b/tensorflow/lite/tools/optimize/calibration_common.h
new file mode 100644
index 0000000000..1ff2d3f18a
--- /dev/null
+++ b/tensorflow/lite/tools/optimize/calibration_common.h
@@ -0,0 +1,61 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TOOLS_OPTIMIZE_CALIBRATION_COMMON_H_
+#define TENSORFLOW_LITE_TOOLS_OPTIMIZE_CALIBRATION_COMMON_H_
+
+#include <unordered_map>
+#include <unordered_set>
+
+#include "tensorflow/lite/mutable_op_resolver.h"
+
+namespace tflite {
+namespace optimize {
+namespace calibration {
+using BuiltinOperatorKey = std::pair<BuiltinOperator, int>;
+
+using BuiltinOpsSet = std::unordered_set<
+    BuiltinOperatorKey,
+    op_resolver_hasher::OperatorKeyHasher<BuiltinOperatorKey>>;
+
+template <typename T>
+class BuiltinOpsMap
+    : public std::unordered_map<
+          BuiltinOperatorKey, T,
+          op_resolver_hasher::OperatorKeyHasher<BuiltinOperatorKey>> {};
+
+// An alias for |TfLiteRegistration.invoke|.
+using KernelEvalFuncPtr = TfLiteStatus (*)(TfLiteContext*, TfLiteNode*);
+
+enum class OperatorTensorType { kNone, kInput, kOutput, kIntermediate };
+
+// Information about an operator in the TfLite graph.
+struct OperatorInfo {
+  int node_index;
+  std::string name;
+  BuiltinOperator builtin_op_code;
+  bool is_custom_op;
+  std::vector<int> inputs;
+  std::vector<int> outputs;
+  // Inputs that need to be logged.
+  std::vector<int> loggable_inputs;
+  // Outputs that need to be logged.
+  std::vector<int> loggable_outputs;
+  const TfLiteRegistration* registration;
+};
+
+}  // namespace calibration
+}  // namespace optimize
+}  // namespace tflite
+#endif  // TENSORFLOW_LITE_TOOLS_OPTIMIZE_CALIBRATION_COMMON_H_
diff --git a/tensorflow/lite/tools/optimize/calibration_logger.h b/tensorflow/lite/tools/optimize/calibration_logger.h
new file mode 100644
index 0000000000..8fd380423a
--- /dev/null
+++ b/tensorflow/lite/tools/optimize/calibration_logger.h
@@ -0,0 +1,85 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TOOLS_OPTIMIZE_CALIBRATION_LOGGER_H_
+#define TENSORFLOW_LITE_TOOLS_OPTIMIZE_CALIBRATION_LOGGER_H_
+
+#include <unordered_map>
+
+#include "tensorflow/lite/c/c_api_internal.h"
+
+namespace tflite {
+namespace optimize {
+namespace calibration {
+class MinMax {
+ public:
+  void Update(const float* values, size_t tensor_size) {
+    // TODO(shashishekhar): Really slow implementation, optimize
+    if (tensor_size <= 0) return;
+
+    if (!has_values_) {
+      min_ = max_ = values[0];
+      has_values_ = true;
+      return;
+    }
+
+    // We are only logging absolute min/max here.
+    // TODO(shashishekhar): Make it possible to use weighted/moving average.
+    for (size_t i = 0; i < tensor_size; i++) {
+      float val = values[i];
+      if (min_ > val) {
+        min_ = val;
+      } else if (max_ < val) {
+        max_ = val;
+      }
+    }
+  }
+
+  bool HasValues() const { return has_values_; }
+
+  TfLiteStatus Get(float* min_val, float* max_val) const {
+    if (!has_values_) return kTfLiteError;
+    *min_val = min_;
+    *max_val = max_;
+    return kTfLiteOk;
+  }
+
+ private:
+  bool has_values_;
+  float min_, max_;
+};
+
+// Captures min max values for tensors.
+class Logger {
+ public:
+  // Log the value for tensor at |tensor_index| which has |tensor_values|
+  void LogTensorValue(int tensor_index, const float* tensor_values,
+                      size_t tensor_size) {
+    tensor_id_to_stats_map_[tensor_index].Update(tensor_values, tensor_size);
+  }
+
+  // Returns a map from tensor_index -> observed min max values.
+  const std::unordered_map<int, MinMax>& GetCalibrationValues() const {
+    return tensor_id_to_stats_map_;
+  }
+
+ private:
+  std::unordered_map<int, MinMax> tensor_id_to_stats_map_;
+};
+
+}  // namespace calibration
+}  // namespace optimize
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TOOLS_OPTIMIZE_CALIBRATION_LOGGER_H_
diff --git a/tensorflow/lite/tools/optimize/calibration_reader.cc b/tensorflow/lite/tools/optimize/calibration_reader.cc
new file mode 100644
index 0000000000..b01a62bd6c
--- /dev/null
+++ b/tensorflow/lite/tools/optimize/calibration_reader.cc
@@ -0,0 +1,55 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/optimize/calibration_reader.h"
+
+#include "absl/memory/memory.h"
+
+namespace tflite {
+namespace optimize {
+namespace calibration {
+TfLiteStatus CalibrationReader::GetTensorStatsAsMap(
+    std::unordered_map<int, CalibrationStats>* tensor_id_to_stats_map) const {
+  tensor_id_to_stats_map->clear();
+  for (const auto& tensorid_stat : logger_->GetCalibrationValues()) {
+    auto minmax = tensorid_stat.second;
+    CalibrationReader::CalibrationStats stats;
+    TF_LITE_ENSURE_STATUS(minmax.Get(&stats.min, &stats.max));
+    tensor_id_to_stats_map->insert({tensorid_stat.first, stats});
+  }
+
+  return kTfLiteOk;
+}
+
+TfLiteStatus CalibrationReader::AddCalibrationToModel(ModelT* model) const {
+  if (!model || model->subgraphs.empty()) {
+    return kTfLiteError;
+  }
+  const auto& subgraph = model->subgraphs[0];
+  for (const auto& tensorid_stat : logger_->GetCalibrationValues()) {
+    auto minmax = tensorid_stat.second;
+    float min, max;
+    TF_LITE_ENSURE_STATUS(minmax.Get(&min, &max));
+    auto quant_params = absl::make_unique<tflite::QuantizationParametersT>();
+    quant_params->min.push_back(min);
+    quant_params->max.push_back(max);
+    subgraph->tensors[tensorid_stat.first]->quantization =
+        std::move(quant_params);
+  }
+
+  return kTfLiteOk;
+}
+}  // namespace calibration
+}  // namespace optimize
+}  // namespace tflite
diff --git a/tensorflow/lite/tools/optimize/calibration_reader.h b/tensorflow/lite/tools/optimize/calibration_reader.h
new file mode 100644
index 0000000000..af0da1bb38
--- /dev/null
+++ b/tensorflow/lite/tools/optimize/calibration_reader.h
@@ -0,0 +1,55 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TOOLS_OPTIMIZE_CALIBRATION_READER_H_
+#define TENSORFLOW_LITE_TOOLS_OPTIMIZE_CALIBRATION_READER_H_
+
+#include <unordered_map>
+
+#include "tensorflow/lite/context.h"
+#include "tensorflow/lite/model.h"
+#include "tensorflow/lite/tools/optimize/calibration_logger.h"
+
+namespace tflite {
+namespace optimize {
+namespace calibration {
+// Warning: This is not a public API and subject to change.
+//
+// Reads calibrator data collected by running the interpreter through
+// a calibration set.
+class CalibrationReader {
+ public:
+  struct CalibrationStats {
+    float min;
+    float max;
+  };
+  explicit CalibrationReader(const Logger* logger) : logger_(logger) {}
+
+  // Gets a map from tensor index to recorded calibration values.
+  virtual TfLiteStatus GetTensorStatsAsMap(
+      std::unordered_map<int, CalibrationStats>* tensor_id_to_stats_map) const;
+
+  // Annotates the tensors in the given model with statistics captured during
+  // calibration.
+  virtual TfLiteStatus AddCalibrationToModel(ModelT* model) const;
+
+  virtual ~CalibrationReader() {}
+
+ private:
+  const Logger* logger_;
+};
+}  // namespace calibration
+}  // namespace optimize
+}  // namespace tflite
+#endif  // TENSORFLOW_LITE_TOOLS_OPTIMIZE_CALIBRATION_READER_H_
diff --git a/tensorflow/lite/tools/optimize/calibrator.cc b/tensorflow/lite/tools/optimize/calibrator.cc
new file mode 100644
index 0000000000..0e817f9346
--- /dev/null
+++ b/tensorflow/lite/tools/optimize/calibrator.cc
@@ -0,0 +1,345 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/optimize/calibrator.h"
+
+#include <fstream>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#include "absl/memory/memory.h"
+#include "tensorflow/lite/core/api/error_reporter.h"
+#include "tensorflow/lite/core/api/op_resolver.h"
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/model.h"
+#include "tensorflow/lite/op_resolver.h"
+#include "tensorflow/lite/schema/schema_generated.h"
+#include "tensorflow/lite/string_util.h"
+#include "tensorflow/lite/tools/optimize/calibration_common.h"
+#include "tensorflow/lite/tools/optimize/calibration_logger.h"
+#include "tensorflow/lite/tools/optimize/calibration_reader.h"
+#include "tensorflow/lite/tools/optimize/logging_op_resolver.h"
+#include "tensorflow/lite/tools/optimize/node_info_delegate.h"
+
+namespace tflite {
+namespace optimize {
+namespace calibration {
+
+namespace {
+
+// Calibrator is used to hold information that can be accessed during kernel
+// invocations.
+// TfLite kernel invocations are C functions and cannot look at the global
+// structure of the graph. Calibrator allows the kernel invoke functions to
+// access the global structure of graph and know which node is currently being
+// executed. This also allows us to write a simple kernel invoke wrapper
+// (see LoggingEval) that can work for most builtin ops.
+class Calibrator {
+ public:
+  Calibrator(const std::unordered_map<const TfLiteNode*, OperatorInfo>&
+                 node_ptr_opinfo_map,
+             std::unique_ptr<LoggingOpResolver> logging_op_resolver)
+      : node_ptr_opinfo_map_(node_ptr_opinfo_map),
+        logging_op_resolver_(std::move(logging_op_resolver)) {
+    logger_ = absl::make_unique<Logger>();
+  }
+
+  // Returns the wrapped kernel invoke function |TfLiteRegistration.invoke|.
+  KernelEvalFuncPtr GetKernelInvoke(const TfLiteNode* node) const;
+
+  // Gets the instance of logger associated with the current context.
+  Logger* GetLogger() const { return logger_.get(); }
+
+  // Gets the operator information about the given TfLiteNode.
+  const OperatorInfo& GetOpInfo(const TfLiteNode* node) const {
+    return node_ptr_opinfo_map_.at(node);
+  }
+
+ private:
+  std::unordered_map<const TfLiteNode*, OperatorInfo> node_ptr_opinfo_map_;
+  std::unique_ptr<LoggingOpResolver> logging_op_resolver_;
+  const std::unordered_map<int, OperatorInfo> index_opinfo_;
+  std::unique_ptr<Logger> logger_;
+};
+
+KernelEvalFuncPtr Calibrator::GetKernelInvoke(const TfLiteNode* node) const {
+  auto op_info = node_ptr_opinfo_map_.at(node);
+  return logging_op_resolver_->GetWrappedKernelInvoke(op_info.builtin_op_code,
+                                                      1);
+}
+
+// A registry of |Calibrator| objects per |TfLiteContext|.
+// This global registry is needed to access |Calibrator| objects in the kernel
+// invoke functions i.e. |TfLiteRegistration.invoke|.
+// Kernel invoke functions are C functions that have limited access to
+// |TfLiteContext|. Kernel invoke functions don't have access to global state of
+// graph. That means during a kernel invocation, the function cannot know which
+// node it was invoked for. E.g. in case of a model with |Conv| op at two
+// locations, there is no easy way for the Conv.invoke function to disambiguate
+// the calls.
+//
+// For calibration we solve this problem by creating a map of calibrators
+// per |TfLiteContext|. This map is |GlobalCalibrationRegistry|.
+//
+// This registry is then accessed using a global getter function:
+// |GetCalibratorRegistry|.
+// E.g.
+// TfLiteStatus SomeKernelInvokeFn(TfLiteContext* context, TfLiteNode* node) {
+//   .... code ....
+//   auto registry = GetCalibratorRegistry();
+//   auto calibrator = registry->GetCalibrator(context);
+//   ..... code ....
+//  }
+//
+// This way the kernel invoke functions can get the access to the Calibrator
+// object associated with the |TfLiteContext|.
+class GlobalCalibratorRegistry {
+ public:
+  // Get the |Calibrator| associated with given context, returns null if no
+  // calibrator is associated with the given context.
+  Calibrator* GetCalibrator(const TfLiteContext* context) const {
+    if (calibrator_registry_.find(context) == calibrator_registry_.cend()) {
+      return nullptr;
+    }
+    return calibrator_registry_.at(context).get();
+  }
+
+  // Removes the association between calibrator and context.
+  // Note: This deletes the calibrator as well.
+  void RemoveCalibrator(const TfLiteContext* context) {
+    calibrator_registry_.erase(context);
+  }
+
+  // Creates an instance of |Calibrator|.
+  // Registry owns the |Calibrator| object which can be deleted by calling
+  // |RemoveCalibrator|.
+  TfLiteStatus CreateCalibrator(
+      const TfLiteContext* context,
+      const std::unordered_map<const TfLiteNode*, OperatorInfo>& node_to_opinfo,
+      std::unique_ptr<LoggingOpResolver> logging_op_resolver,
+      Calibrator** calibrator_ptr, ErrorReporter* reporter) {
+    if (calibrator_registry_.find(context) != calibrator_registry_.cend()) {
+      reporter->Report(
+          "Failed to create calibrator, context already registered.");
+      return kTfLiteError;
+    }
+    std::unique_ptr<Calibrator> calibrator = absl::make_unique<Calibrator>(
+        node_to_opinfo, std::move(logging_op_resolver));
+    calibrator_registry_[context] = std::move(calibrator);
+    *calibrator_ptr = calibrator_registry_.at(context).get();
+    return kTfLiteOk;
+  }
+
+ private:
+  std::unordered_map<const TfLiteContext*, std::unique_ptr<Calibrator>>
+      calibrator_registry_;
+};
+
+GlobalCalibratorRegistry* GetCalibratorRegistry() {
+  static GlobalCalibratorRegistry* registry = new GlobalCalibratorRegistry();
+  return registry;
+}
+
+// A wrapper implementation for |TfLiteRegistration.invoke| that logs inputs,
+// invokes the wrapped implementation and then logs the outputs.
+TfLiteStatus LoggingEval(TfLiteContext* context, TfLiteNode* node) {
+  Calibrator* calibrator = GetCalibratorRegistry()->GetCalibrator(context);
+
+  if (!calibrator) {
+    context->ReportError(context, "No calibrator found for context.");
+    return kTfLiteError;
+  }
+
+  auto kernel_invoke = calibrator->GetKernelInvoke(node);
+  auto logger = calibrator->GetLogger();
+  auto op_info = calibrator->GetOpInfo(node);
+
+  for (int i : op_info.loggable_inputs) {
+    auto tensor = context->tensors[i];
+    logger->LogTensorValue(i, tensor.data.f, tensor.bytes / sizeof(float));
+  }
+
+  auto status = kernel_invoke(context, node);
+  // TODO(shashishekhar): An intermediate tensor in graph will get logged twice
+  // once as an input and second time as output. This doesn't change the min max
+  // values but is inefficient.
+  // Using moving average will also break this.
+
+  for (int i : op_info.loggable_outputs) {
+    auto tensor = context->tensors[i];
+    logger->LogTensorValue(i, tensor.data.f, tensor.bytes / sizeof(float));
+  }
+
+  return status;
+}
+
+// Returns the loggable tensors. Not all inputs and outputs need to be logged.
+// For example, const weight tensors which have buffers associated with them
+// don't need to be logged.
+std::vector<int> GetLoggableTensorIndices(
+    const std::vector<int>& tensor_indices,
+    const flatbuffers::Vector<flatbuffers::Offset<Tensor>>* tensors,
+    const flatbuffers::Vector<flatbuffers::Offset<Buffer>>* tensor_buffers) {
+  std::vector<int> loggable;
+  for (auto tensor_index : tensor_indices) {
+    auto tensor = tensors->Get(tensor_index);
+    auto buffer_index = tensor->buffer();
+    bool has_no_buffer =
+        buffer_index == 0 || (tensor_buffers->Get(buffer_index) == nullptr);
+    if (has_no_buffer && tensor->type() == tflite::TensorType_FLOAT32) {
+      loggable.push_back(tensor_index);
+    }
+  }
+  return loggable;
+}
+
+// Creates a mapping between the static model graph and the runtime TfLiteNode*
+// nodes in the graph for the given context.
+// This is done by querying the TfLiteContext for node and registrations using
+// the |NodeInfoDelegateObserver|.
+TfLiteStatus GetNodeOpInfoMapAndContext(
+    const std::unordered_map<int, OperatorInfo>& node_to_opinfo,
+    tflite::Interpreter* const interpreter,
+    std::unordered_map<const TfLiteNode*, OperatorInfo>* node_ptr_opinfo_map,
+    const TfLiteContext** context
+
+) {
+  NodeInfoDelegateObserver delegate_observer(node_to_opinfo,
+                                             node_ptr_opinfo_map);
+  NodeInfoDelegateParams delegate_params;
+  delegate_params.delegate_observer = &delegate_observer;
+  TfLiteDelegate logging_delegate = CreateNodeInfoDelegate(&delegate_params);
+
+  auto modify_status = interpreter->ModifyGraphWithDelegate(&logging_delegate);
+  if (modify_status != kTfLiteOk) {
+    return kTfLiteError;
+  }
+  *context = delegate_observer.GetContext();
+  return kTfLiteOk;
+}
+
+string GetOpName(const tflite::OperatorCode& opcode) {
+  if (opcode.custom_code() != nullptr) {
+    return opcode.custom_code()->str();
+  }
+  return tflite::EnumNamesBuiltinOperator()[opcode.builtin_code()];
+}
+
+// A |CalibrationReader| that owns the Calibrator.
+class Reader : public CalibrationReader {
+ public:
+  Reader(const TfLiteContext* context, const Logger* logger)
+      : CalibrationReader(logger), context_(context) {}
+
+  ~Reader() override { GetCalibratorRegistry()->RemoveCalibrator(context_); }
+
+ private:
+  const TfLiteContext* context_;
+};
+
+}  // namespace
+
+TfLiteStatus BuildLoggingInterpreter(
+    const FlatBufferModel& model, const OpResolver& op_resolver,
+    std::unique_ptr<Interpreter>* interpreter,
+    std::unique_ptr<CalibrationReader>* calibration_reader) {
+  auto tflite_model = model.GetModel();
+  auto subgraphs = tflite_model->subgraphs();
+  auto tensor_buffers = tflite_model->buffers();
+
+  if (subgraphs->size() != 1) {
+    model.error_reporter()->Report(
+        "Only models with a single subgraph are supported, model had %d "
+        "subgraphs",
+        subgraphs->size());
+    return kTfLiteError;
+  }
+
+  // Populate the node index to operator info map.
+  // We want to collect this information so we can use it during runtime to
+  // log details of which inputs and outputs.
+  // At runtime TFLite kernel invoke functions can only look into their
+  // own node in the graph (TFLiteNode*) and some limited context information.
+  auto primary_subgraph = subgraphs->Get(0);
+  auto operator_codes = tflite_model->operator_codes();
+  auto operators = primary_subgraph->operators();
+  auto tensors = primary_subgraph->tensors();
+  std::unordered_map<int, OperatorInfo> node_to_opinfo;
+  BuiltinOpsSet op_and_versions;
+
+  for (size_t i = 0; i < operators->size(); i++) {
+    OperatorInfo op_info;
+    op_info.node_index = i;
+    auto op = operators->Get(i);
+    auto operator_code = operator_codes->Get(op->opcode_index());
+    op_info.builtin_op_code = operator_code->builtin_code();
+    op_info.name = GetOpName(*operator_code);
+    op_info.is_custom_op = operator_code->custom_code() != nullptr;
+
+    auto op_inputs = op->inputs();
+    auto op_outputs = op->outputs();
+    op_info.inputs = std::vector<int>(op_inputs->begin(), op_inputs->end());
+    op_info.outputs = std::vector<int>(op_outputs->begin(), op_outputs->end());
+    op_info.loggable_inputs =
+        GetLoggableTensorIndices(op_info.inputs, tensors, tensor_buffers);
+    op_info.loggable_outputs =
+        GetLoggableTensorIndices(op_info.outputs, tensors, tensor_buffers);
+    if (!op_info.is_custom_op) {
+      op_info.registration = op_resolver.FindOp(operator_code->builtin_code(),
+                                                operator_code->version());
+    } else {
+      op_info.registration =
+          op_resolver.FindOp(op_info.name.c_str(), operator_code->version());
+    }
+    node_to_opinfo[i] = op_info;
+    op_and_versions.insert({op_info.builtin_op_code, operator_code->version()});
+  }
+
+  // Prepare the logging op resolver to use |LoggingEval| for kernel
+  // invocations.
+  auto logging_op_resolver = absl::make_unique<LoggingOpResolver>(
+      op_and_versions, op_resolver, LoggingEval);
+  tflite::InterpreterBuilder(model, *logging_op_resolver)(interpreter);
+
+  if (!(*interpreter)) {
+    model.error_reporter()->Report("Failed to construct interpreter");
+    return kTfLiteError;
+  }
+
+  // Compute the mapping between runtime and static graph structure, i.e.
+  // (TfLiteContext, TfLiteNode) -> OperatorInfo
+  std::unordered_map<const TfLiteNode*, OperatorInfo> node_ptr_opinfo_map;
+  const TfLiteContext* context = nullptr;
+  GetNodeOpInfoMapAndContext(node_to_opinfo, interpreter->get(),
+                             &node_ptr_opinfo_map, &context);
+
+  Calibrator* calibrator = nullptr;
+  // Register a calibrator object for the context. This can be accessed
+  // during invocations by the logging kernels.
+  TF_LITE_ENSURE_STATUS(GetCalibratorRegistry()->CreateCalibrator(
+      context, node_ptr_opinfo_map, std::move(logging_op_resolver), &calibrator,
+      model.error_reporter()));
+  *calibration_reader = std::unique_ptr<CalibrationReader>(
+      new Reader(context, calibrator->GetLogger()));
+
+  return kTfLiteOk;
+}
+
+}  // namespace calibration
+}  // namespace optimize
+}  // namespace tflite
diff --git a/tensorflow/lite/tools/optimize/calibrator.h b/tensorflow/lite/tools/optimize/calibrator.h
new file mode 100644
index 0000000000..ab3cb27eb7
--- /dev/null
+++ b/tensorflow/lite/tools/optimize/calibrator.h
@@ -0,0 +1,64 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TOOLS_OPTIMIZE_CALIBRATOR_H_
+#define TENSORFLOW_LITE_TOOLS_OPTIMIZE_CALIBRATOR_H_
+
+#include <unordered_map>
+
+#include "flatbuffers/flatbuffers.h"  // TF:flatbuffers
+#include "tensorflow/lite/core/api/op_resolver.h"
+#include "tensorflow/lite/model.h"
+#include "tensorflow/lite/tools/optimize/calibration_reader.h"
+
+namespace tflite {
+namespace optimize {
+namespace calibration {
+
+// Warning: This is not a public API and subject to change.
+
+// Builds a interpreter that logs the calibration data in memory.
+// The calibration data can be recovered using |calibration_reader|.
+//
+// Sample usage:
+// std::unique_ptr<Interpreter> interpreter;
+// std::unique_ptr<CalibrationReader> calibration_reader;
+// BuiltinOpResolver resolver = ...
+// FlatBufferModel model = ..
+//
+// BuildLoggingInterpreter(model, resolver, &interpreter,
+//  &calibration_reader);
+//
+//
+// * Allocate tensors...
+// * Call interpreter->invoke on calibration dataset.
+//
+// Calibration data can be read either directly by calling
+// std::unordered_map<int,  CalibrationStats>> tensor_index_to_stats;
+// calibration_reader->GetTensorStatsAsMap(&tensor_index_to_stats);
+//
+// or adding calibration data to model itself.
+// ModelT * original_floating_point_model = ...
+// calibration_reader->AddCalibrationToModel(original_floating_point_model);
+//
+TfLiteStatus BuildLoggingInterpreter(
+    const FlatBufferModel& model, const OpResolver& op_resolver,
+    std::unique_ptr<Interpreter>* interpreter,
+    std::unique_ptr<CalibrationReader>* calibration_reader);
+
+}  // namespace calibration
+}  // namespace optimize
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TOOLS_OPTIMIZE_CALIBRATOR_H_
diff --git a/tensorflow/lite/tools/optimize/calibrator_test.cc b/tensorflow/lite/tools/optimize/calibrator_test.cc
new file mode 100644
index 0000000000..bbbcc70fae
--- /dev/null
+++ b/tensorflow/lite/tools/optimize/calibrator_test.cc
@@ -0,0 +1,189 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <cstring>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+#include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/model.h"
+#include "tensorflow/lite/tools/optimize/calibrator.h"
+
+namespace tflite {
+namespace optimize {
+namespace calibration {
+namespace {
+
+TEST(CalibratorTest, CalibrationStatsAreCollected) {
+  auto model = FlatBufferModel::BuildFromFile(
+      "third_party/tensorflow/lite/testdata/multi_add.bin");
+  ASSERT_TRUE(model);
+  std::unique_ptr<Interpreter> interpreter;
+  std::unique_ptr<CalibrationReader> reader;
+  auto status = BuildLoggingInterpreter(
+      *model, ops::builtin::BuiltinOpResolver{}, &interpreter, &reader);
+  EXPECT_EQ(kTfLiteOk, status);
+
+  ASSERT_TRUE(interpreter);
+  ASSERT_TRUE(reader);
+  std::unordered_map<int, CalibrationReader::CalibrationStats> stats;
+  status = reader->GetTensorStatsAsMap(&stats);
+  EXPECT_EQ(kTfLiteOk, status);
+  EXPECT_TRUE(stats.empty());
+
+  status = interpreter->AllocateTensors();
+  ASSERT_EQ(kTfLiteOk, status);
+  // Model does the following:
+  // 0        1       2        3
+  // |        |__ ____|        |
+  // |           |             |
+  // |          Add(tensor:4)  |
+  // |____ ______|______ ______|
+  //      |             |
+  //      Add          Add
+  //      |             |
+  //    Output:5      Output:6
+
+  const size_t tensor_size = 1 * 8 * 8 * 3;
+
+  std::vector<float> ones(tensor_size, 1.0f);
+  // Fill input tensor i with i+1, i.e. input[0] = 1.0f, input[1] = 2.0f,
+  // input[2] = 3.0f
+
+  for (size_t i = 0; i < interpreter->inputs().size(); i++) {
+    int input_tensor_idx = interpreter->inputs()[i];
+    TfLiteTensor* tensor = interpreter->tensor(input_tensor_idx);
+    ASSERT_EQ(tensor->bytes, tensor_size * sizeof(float));
+    for (size_t j = 0; j < tensor_size; j++) {
+      tensor->data.f[j] = i + 1;
+    }
+  }
+  status = interpreter->Invoke();
+  ASSERT_EQ(kTfLiteOk, status);
+  const float eps = 1e-6f;
+  // Verify that tensor 5: is 6
+  // Verify that tensor 6: is 9
+  TfLiteTensor* tensor = interpreter->tensor(interpreter->outputs()[0]);
+  for (size_t i = 0; i < tensor_size; i++) {
+    EXPECT_NEAR(tensor->data.f[i], 6.0f, eps);
+  }
+  tensor = interpreter->tensor(interpreter->outputs()[1]);
+  for (size_t i = 0; i < tensor_size; i++) {
+    EXPECT_NEAR(tensor->data.f[i], 9.0f, eps);
+  }
+
+  // Verify that min max of tensors.
+  status = reader->GetTensorStatsAsMap(&stats);
+  EXPECT_EQ(kTfLiteOk, status);
+  EXPECT_EQ(7, stats.size());
+  // Check inputs
+  for (int tensor_idx = 0; tensor_idx < 4; tensor_idx++) {
+    EXPECT_NEAR(stats.at(tensor_idx).min, tensor_idx + 1, eps);
+    EXPECT_NEAR(stats.at(tensor_idx).max, tensor_idx + 1, eps);
+  }
+  // Check tensor 4 max.
+  EXPECT_NEAR(stats.at(4).min, 5, eps);
+  EXPECT_NEAR(stats.at(4).max, 5, eps);
+
+  // Check outputs
+  EXPECT_NEAR(stats.at(5).min, 6, eps);
+  EXPECT_NEAR(stats.at(5).max, 6, eps);
+
+  EXPECT_NEAR(stats.at(6).min, 9, eps);
+  EXPECT_NEAR(stats.at(6).max, 9, eps);
+}
+
+TEST(CalibratorTest, MultipleInvokes) {
+  auto model = FlatBufferModel::BuildFromFile(
+      "third_party/tensorflow/lite/testdata/multi_add.bin");
+  ASSERT_TRUE(model);
+  std::unique_ptr<Interpreter> interpreter;
+  std::unique_ptr<CalibrationReader> reader;
+  auto status = BuildLoggingInterpreter(
+      *model, ops::builtin::BuiltinOpResolver{}, &interpreter, &reader);
+  EXPECT_EQ(kTfLiteOk, status);
+
+  ASSERT_TRUE(interpreter);
+  ASSERT_TRUE(reader);
+  status = interpreter->AllocateTensors();
+
+  EXPECT_EQ(kTfLiteOk, status);
+  const size_t tensor_size = 1 * 8 * 8 * 3;
+  // Fill input tensor i with i+1, i.e. input[0] = 1.0f, input[1] = 2.0f,
+  // input[2] = 3.0f
+
+  for (size_t i = 0; i < interpreter->inputs().size(); i++) {
+    int input_tensor_idx = interpreter->inputs()[i];
+    TfLiteTensor* tensor = interpreter->tensor(input_tensor_idx);
+    ASSERT_EQ(tensor->bytes, tensor_size * sizeof(float));
+    for (size_t j = 0; j < tensor_size; j++) {
+      tensor->data.f[j] = i + 1;
+    }
+  }
+  status = interpreter->Invoke();
+  ASSERT_EQ(kTfLiteOk, status);
+  const float eps = 1e-6f;
+  // Verify that min max of tensors.
+  std::unordered_map<int, CalibrationReader::CalibrationStats> stats;
+  status = reader->GetTensorStatsAsMap(&stats);
+  EXPECT_EQ(kTfLiteOk, status);
+  EXPECT_EQ(7, stats.size());
+  const float expected_values[7] = {
+      1.0f,  // input 0
+      2.0f,  // input 1
+      3.0f,  // input 2
+      4.0f,  // input 3
+      5.0f,  // Add(1, 2)
+      6.0f,  // Output 5: Add(0, Add(1,2))
+      9.0f,  // Output 6: Add(Add(1,2), 3)
+  };
+  for (int tensor_idx = 0; tensor_idx < 7; tensor_idx++) {
+    EXPECT_NEAR(stats.at(tensor_idx).min, expected_values[tensor_idx], eps);
+    EXPECT_NEAR(stats.at(tensor_idx).max, expected_values[tensor_idx], eps);
+  }
+  // Set input[0][0] = 1.5 and input[0][1] = 0.5 this should change the values
+  // only for input[0] and tensor 4 and ouputs 5, 6.
+  TfLiteTensor* input0 = interpreter->tensor(0);
+  input0->data.f[0] = 1.5f;
+  input0->data.f[1] = 0.5f;
+  status = interpreter->Invoke();
+  ASSERT_EQ(kTfLiteOk, status);
+  status = reader->GetTensorStatsAsMap(&stats);
+  EXPECT_EQ(kTfLiteOk, status);
+  EXPECT_EQ(7, stats.size());
+  EXPECT_NEAR(stats.at(0).min, 0.5f, eps);
+  EXPECT_NEAR(stats.at(0).max, 1.5f, eps);
+
+  for (int tensor_idx = 1; tensor_idx < 5; tensor_idx++) {
+    EXPECT_NEAR(stats.at(tensor_idx).min, expected_values[tensor_idx], eps);
+    EXPECT_NEAR(stats.at(tensor_idx).max, expected_values[tensor_idx], eps);
+  }
+
+  EXPECT_NEAR(stats.at(5).min, 5.5f, eps);
+  EXPECT_NEAR(stats.at(5).max, 6.5f, eps);
+
+  EXPECT_NEAR(stats.at(6).min, 9.0f, eps);
+  EXPECT_NEAR(stats.at(6).max, 9.0f, eps);
+}
+
+}  // namespace
+}  // namespace calibration
+}  // namespace optimize
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  // On Linux, add: FLAGS_logtostderr = true;
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/lite/tools/optimize/logging_op_resolver.cc b/tensorflow/lite/tools/optimize/logging_op_resolver.cc
new file mode 100644
index 0000000000..7633ebb8dd
--- /dev/null
+++ b/tensorflow/lite/tools/optimize/logging_op_resolver.cc
@@ -0,0 +1,61 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/optimize/logging_op_resolver.h"
+
+#include "absl/memory/memory.h"
+
+namespace tflite {
+namespace optimize {
+namespace calibration {
+
+LoggingOpResolver::LoggingOpResolver(const BuiltinOpsSet& ops_to_replace,
+                                     const OpResolver& base_resolver,
+                                     KernelEvalFuncPtr logging_eval_fn) {
+  for (const auto& op_and_version : ops_to_replace) {
+    const TfLiteRegistration* base_registration =
+        base_resolver.FindOp(op_and_version.first, op_and_version.second);
+    BuiltinOperatorKey key = op_and_version;
+    builtin_op_evalfn_map_[key] = base_registration->invoke;
+    std::unique_ptr<TfLiteRegistration> logging_registation =
+        absl::make_unique<TfLiteRegistration>(*base_registration);
+    logging_registation->invoke = logging_eval_fn;
+    builtin_op_registration_map_[key] = std::move(logging_registation);
+  }
+}
+
+const TfLiteRegistration* LoggingOpResolver::FindOp(BuiltinOperator op,
+                                                    int version) const {
+  BuiltinOperatorKey key = {op, version};
+  if (builtin_op_registration_map_.find(key) !=
+      builtin_op_registration_map_.end()) {
+    return builtin_op_registration_map_.at(key).get();
+  }
+
+  return nullptr;
+}
+
+KernelEvalFuncPtr LoggingOpResolver::GetWrappedKernelInvoke(BuiltinOperator op,
+                                                            int version) const {
+  return builtin_op_evalfn_map_.at({op, version});
+}
+
+const TfLiteRegistration* LoggingOpResolver::FindOp(const char* op,
+                                                    int version) const {
+  // TODO(b/121374947): Support custom ops as well.
+  return nullptr;
+}
+}  // namespace calibration
+}  // namespace optimize
+}  // namespace tflite
diff --git a/tensorflow/lite/tools/optimize/logging_op_resolver.h b/tensorflow/lite/tools/optimize/logging_op_resolver.h
new file mode 100644
index 0000000000..58a3a0fe3c
--- /dev/null
+++ b/tensorflow/lite/tools/optimize/logging_op_resolver.h
@@ -0,0 +1,59 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TOOLS_OPTIMIZE_LOGGING_OP_RESOLVER_H_
+#define TENSORFLOW_LITE_TOOLS_OPTIMIZE_LOGGING_OP_RESOLVER_H_
+
+#include <set>
+#include <unordered_map>
+
+#include "tensorflow/lite/core/api/op_resolver.h"
+#include "tensorflow/lite/mutable_op_resolver.h"
+#include "tensorflow/lite/op_resolver.h"
+#include "tensorflow/lite/tools/optimize/calibration_common.h"
+
+namespace tflite {
+namespace optimize {
+namespace calibration {
+// A resolver that replaces the kernel invocations with a wrapper
+// eval function.
+class LoggingOpResolver : public OpResolver {
+ public:
+  // Creates an instance of |LoggingOpResolver|.
+  // All |TfLiteRegistration.invoke| functions are replaced by
+  // |logging_eval_fn|.
+  // TODO(shashishekhar): This interface needs to change for custom ops and
+  // BuiltinOps that need special logging implementations.
+  LoggingOpResolver(const BuiltinOpsSet& ops_to_replace,
+                    const OpResolver& base_resolver,
+                    KernelEvalFuncPtr logging_eval_fn);
+
+  const TfLiteRegistration* FindOp(BuiltinOperator op,
+                                   int version) const override;
+
+  KernelEvalFuncPtr GetWrappedKernelInvoke(BuiltinOperator op,
+                                           int version) const;
+  const TfLiteRegistration* FindOp(const char* op, int version) const override;
+
+ private:
+  BuiltinOpsMap<std::unique_ptr<TfLiteRegistration>>
+      builtin_op_registration_map_;
+  BuiltinOpsMap<KernelEvalFuncPtr> builtin_op_evalfn_map_;
+};
+
+}  // namespace calibration
+}  // namespace optimize
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TOOLS_OPTIMIZE_LOGGING_OP_RESOLVER_H_
diff --git a/tensorflow/lite/tools/optimize/logging_op_resolver_test.cc b/tensorflow/lite/tools/optimize/logging_op_resolver_test.cc
new file mode 100644
index 0000000000..18c29abec6
--- /dev/null
+++ b/tensorflow/lite/tools/optimize/logging_op_resolver_test.cc
@@ -0,0 +1,141 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/optimize/logging_op_resolver.h"
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+#include "tensorflow/lite/mutable_op_resolver.h"
+
+namespace tflite {
+namespace optimize {
+namespace calibration {
+namespace {
+
+TfLiteStatus ConvPrepare(TfLiteContext* context, TfLiteNode* node) {
+  return kTfLiteOk;
+}
+
+TfLiteStatus ConvEval(TfLiteContext* context, TfLiteNode* node) {
+  return kTfLiteOk;
+}
+
+TfLiteStatus AddPrepare(TfLiteContext* context, TfLiteNode* node) {
+  return kTfLiteOk;
+}
+
+TfLiteStatus AddEval(TfLiteContext* context, TfLiteNode* node) {
+  return kTfLiteOk;
+}
+
+TfLiteStatus WrappingInvoke(TfLiteContext* context, TfLiteNode* node) {
+  return kTfLiteOk;
+}
+
+TEST(LoggingOpResolverTest, KernelInvokesAreReplaced) {
+  MutableOpResolver base_resolver;
+  TfLiteRegistration conv_registration = {
+      .prepare = ConvPrepare,
+      .invoke = ConvEval,
+  };
+  base_resolver.AddBuiltin(BuiltinOperator_CONV_2D, &conv_registration);
+
+  TfLiteRegistration add_registration = {
+      .prepare = AddPrepare,
+      .invoke = AddEval,
+  };
+  base_resolver.AddBuiltin(BuiltinOperator_ADD, &add_registration);
+  BuiltinOpsSet ops_to_replace = {
+      {BuiltinOperator_CONV_2D, /*version*/ 1},
+      {BuiltinOperator_ADD, /*version*/ 1},
+  };
+
+  LoggingOpResolver resolver(ops_to_replace, base_resolver, WrappingInvoke);
+
+  auto reg = resolver.FindOp(BuiltinOperator_CONV_2D, 1);
+
+  EXPECT_EQ(reg->builtin_code, BuiltinOperator_CONV_2D);
+  EXPECT_TRUE(reg->prepare == ConvPrepare);
+  EXPECT_TRUE(reg->invoke == WrappingInvoke);
+
+  reg = resolver.FindOp(BuiltinOperator_ADD, 1);
+
+  EXPECT_EQ(reg->builtin_code, BuiltinOperator_ADD);
+  EXPECT_TRUE(reg->prepare == AddPrepare);
+  EXPECT_TRUE(reg->invoke == WrappingInvoke);
+}
+
+TEST(LoggingOpResolverTest, OriginalKernelInvokesAreRetained) {
+  MutableOpResolver base_resolver;
+  TfLiteRegistration conv_registration = {
+      .prepare = ConvPrepare,
+      .invoke = ConvEval,
+  };
+  base_resolver.AddBuiltin(BuiltinOperator_CONV_2D, &conv_registration);
+
+  TfLiteRegistration add_registration = {
+      .prepare = AddPrepare,
+      .invoke = AddEval,
+  };
+  base_resolver.AddBuiltin(BuiltinOperator_ADD, &add_registration);
+  BuiltinOpsSet ops_to_replace = {
+      {BuiltinOperator_CONV_2D, /*version*/ 1},
+      {BuiltinOperator_ADD, /*version*/ 1},
+  };
+
+  LoggingOpResolver resolver(ops_to_replace, base_resolver, WrappingInvoke);
+  auto kernel_invoke =
+      resolver.GetWrappedKernelInvoke(BuiltinOperator_CONV_2D, 1);
+  EXPECT_TRUE(kernel_invoke == ConvEval);
+  kernel_invoke = resolver.GetWrappedKernelInvoke(BuiltinOperator_ADD, 1);
+  EXPECT_TRUE(kernel_invoke == AddEval);
+}
+
+TEST(LoggingOpResolverTest, OnlyOpsInReplacementSetAreReplaces) {
+  MutableOpResolver base_resolver;
+  TfLiteRegistration conv_registration = {
+      .prepare = ConvPrepare,
+      .invoke = ConvEval,
+  };
+  base_resolver.AddBuiltin(BuiltinOperator_CONV_2D, &conv_registration);
+
+  TfLiteRegistration add_registration = {
+      .prepare = AddPrepare,
+      .invoke = AddEval,
+  };
+  base_resolver.AddBuiltin(BuiltinOperator_ADD, &add_registration);
+  // Only replace conv2d
+  BuiltinOpsSet ops_to_replace = {
+      {BuiltinOperator_CONV_2D, /*version*/ 1},
+  };
+
+  LoggingOpResolver resolver(ops_to_replace, base_resolver, WrappingInvoke);
+  auto reg = resolver.FindOp(BuiltinOperator_CONV_2D, 1);
+  EXPECT_EQ(reg->builtin_code, BuiltinOperator_CONV_2D);
+  EXPECT_TRUE(reg->prepare == ConvPrepare);
+  EXPECT_TRUE(reg->invoke == WrappingInvoke);
+
+  reg = resolver.FindOp(BuiltinOperator_ADD, 1);
+  EXPECT_EQ(nullptr, reg);
+}
+
+}  // namespace
+}  // namespace calibration
+}  // namespace optimize
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  // On Linux, add: FLAGS_logtostderr = true;
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/lite/tools/optimize/node_info_delegate.cc b/tensorflow/lite/tools/optimize/node_info_delegate.cc
new file mode 100644
index 0000000000..ccaa69373f
--- /dev/null
+++ b/tensorflow/lite/tools/optimize/node_info_delegate.cc
@@ -0,0 +1,66 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/optimize/node_info_delegate.h"
+
+namespace tflite {
+namespace optimize {
+namespace calibration {
+
+namespace {
+// The prepare function for delegate that forwards the prepare call to the
+// delegate observer in node info delegate params.
+// The function simply calls a delegate observer OnDelegatePrepareMethod.
+TfLiteStatus NodeInfoDelegatePrepare(TfLiteContext* context,
+                                     TfLiteDelegate* delegate) {
+  if (delegate == nullptr) return TfLiteStatus::kTfLiteError;
+
+  NodeInfoDelegateParams* params =
+      reinterpret_cast<NodeInfoDelegateParams*>(delegate->data_);
+  return params->delegate_observer->OnDelegatePrepareCalled(context);
+}
+}  // namespace
+
+TfLiteDelegate CreateNodeInfoDelegate(NodeInfoDelegateParams* params) {
+  return {.data_ = params,
+          .Prepare = NodeInfoDelegatePrepare,
+          .CopyFromBufferHandle = nullptr,
+          .CopyToBufferHandle = nullptr,
+          .FreeBufferHandle = nullptr};
+}
+
+TfLiteStatus NodeInfoDelegateObserver::OnDelegatePrepareCalled(
+    TfLiteContext* context) {
+  context_ = context;
+  const size_t num_nodes = node_index_opinfo_map_.size();
+  for (size_t node_index = 0; node_index < num_nodes; node_index++) {
+    TfLiteNode* node = nullptr;
+    TfLiteRegistration* reg = nullptr;
+    TF_LITE_ENSURE_STATUS(
+        context->GetNodeAndRegistration(context, node_index, &node, &reg));
+    auto op_info = node_index_opinfo_map_.at(node_index);
+    op_info.registration = reg;
+    node_ptr_opinfo_map_->insert({node, op_info});
+  }
+
+  if (node_ptr_opinfo_map_->size() != node_index_opinfo_map_.size()) {
+    // Something wrong.
+    return kTfLiteError;
+  }
+  return kTfLiteOk;
+}
+
+}  // namespace calibration
+}  // namespace optimize
+}  // namespace tflite
diff --git a/tensorflow/lite/tools/optimize/node_info_delegate.h b/tensorflow/lite/tools/optimize/node_info_delegate.h
new file mode 100644
index 0000000000..8ee2ce1978
--- /dev/null
+++ b/tensorflow/lite/tools/optimize/node_info_delegate.h
@@ -0,0 +1,67 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TOOLS_OPTIMIZE_NODE_INFO_DELEGATE_H_
+#define TENSORFLOW_LITE_TOOLS_OPTIMIZE_NODE_INFO_DELEGATE_H_
+
+#include <unordered_map>
+
+#include "tensorflow/lite/context.h"
+#include "tensorflow/lite/tools/optimize/calibration_common.h"
+
+namespace tflite {
+namespace optimize {
+namespace calibration {
+
+// An interface for delegate observer that can listen to TfLiteDelegate::Prepare
+// calls.
+class DelegateObserver {
+ public:
+  virtual TfLiteStatus OnDelegatePrepareCalled(TfLiteContext* context) = 0;
+  virtual ~DelegateObserver() {}
+};
+
+// The parameters for the node info delegate.
+struct NodeInfoDelegateParams {
+  DelegateObserver* delegate_observer;
+};
+
+// Creates a delegate with the given |params|.
+TfLiteDelegate CreateNodeInfoDelegate(NodeInfoDelegateParams* params);
+
+// A delegate observer that can construct the map from TfLiteNode* ->
+// OperatorInfo.
+class NodeInfoDelegateObserver : public DelegateObserver {
+ public:
+  NodeInfoDelegateObserver(
+      const std::unordered_map<int, OperatorInfo>& node_index_to_op,
+      std::unordered_map<const TfLiteNode*, OperatorInfo>* node_ptr_opinfo_map)
+      : node_index_opinfo_map_(node_index_to_op),
+        node_ptr_opinfo_map_(node_ptr_opinfo_map) {}
+
+  TfLiteStatus OnDelegatePrepareCalled(TfLiteContext* context) override;
+
+  // Returns the context that was used to called the prepare method.
+  const TfLiteContext* GetContext() const { return context_; }
+
+ private:
+  const TfLiteContext* context_ = nullptr;
+  const std::unordered_map<int, OperatorInfo>& node_index_opinfo_map_;
+  std::unordered_map<const TfLiteNode*, OperatorInfo>* node_ptr_opinfo_map_;
+};
+
+}  // namespace calibration
+}  // namespace optimize
+}  // namespace tflite
+#endif  // TENSORFLOW_LITE_TOOLS_OPTIMIZE_NODE_INFO_DELEGATE_H_
diff --git a/tensorflow/lite/tools/optimize/node_info_delegate_test.cc b/tensorflow/lite/tools/optimize/node_info_delegate_test.cc
new file mode 100644
index 0000000000..e762d5c014
--- /dev/null
+++ b/tensorflow/lite/tools/optimize/node_info_delegate_test.cc
@@ -0,0 +1,152 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <unordered_map>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+#include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/model.h"
+#include "tensorflow/lite/tools/optimize/node_info_delegate.h"
+
+namespace tflite {
+namespace optimize {
+namespace calibration {
+namespace {
+
+class TestDelegateObserver : public DelegateObserver {
+ public:
+  explicit TestDelegateObserver(TfLiteStatus status_to_return)
+      : status_to_return_(status_to_return) {}
+
+  TfLiteStatus OnDelegatePrepareCalled(TfLiteContext* context) override {
+    num_times_called_++;
+    return status_to_return_;
+  }
+  int num_times_called() { return num_times_called_; }
+
+ private:
+  int num_times_called_ = 0;
+  TfLiteStatus status_to_return_;
+};
+
+TEST(NodeInfoDelegateTest, DelegateObserverIsCalled) {
+  TestDelegateObserver observer(kTfLiteOk);
+  NodeInfoDelegateParams params;
+  params.delegate_observer = &observer;
+  auto model = FlatBufferModel::BuildFromFile(
+      "third_party/tensorflow/lite/testdata/multi_add.bin");
+  ASSERT_TRUE(model);
+  std::unique_ptr<Interpreter> interpreter;
+  ASSERT_EQ(InterpreterBuilder(*model,
+                               ops::builtin::BuiltinOpResolver{})(&interpreter),
+            kTfLiteOk);
+  ASSERT_TRUE(interpreter);
+  EXPECT_EQ(0, observer.num_times_called());
+  TfLiteDelegate delegate = CreateNodeInfoDelegate(&params);
+
+  auto status = interpreter->ModifyGraphWithDelegate(&delegate);
+  EXPECT_EQ(kTfLiteOk, status);
+  EXPECT_EQ(1, observer.num_times_called());
+}
+
+TEST(NodeInfoDelegateTest, ObserverErrorCausesModifyGraphFailure) {
+  // Observer returns error
+  TestDelegateObserver observer(kTfLiteError);
+  NodeInfoDelegateParams params;
+  params.delegate_observer = &observer;
+  auto model = FlatBufferModel::BuildFromFile(
+      "third_party/tensorflow/lite/testdata/multi_add.bin");
+  ASSERT_TRUE(model);
+  std::unique_ptr<Interpreter> interpreter;
+  ASSERT_EQ(InterpreterBuilder(*model,
+                               ops::builtin::BuiltinOpResolver{})(&interpreter),
+            kTfLiteOk);
+  ASSERT_TRUE(interpreter);
+  TfLiteDelegate delegate = CreateNodeInfoDelegate(&params);
+
+  auto status = interpreter->ModifyGraphWithDelegate(&delegate);
+  EXPECT_EQ(kTfLiteError, status);
+}
+
+TEST(NodeInfoDelegateTest, NodeInfoDelegateObserver) {
+  auto model = FlatBufferModel::BuildFromFile(
+      "third_party/tensorflow/lite/testdata/multi_add.bin");
+  ASSERT_TRUE(model);
+
+  std::unordered_map<int, OperatorInfo> index_to_opinfo;
+  auto primary_subgraph = model->GetModel()->subgraphs()->Get(0);
+  auto operators = primary_subgraph->operators();
+  auto subgraph_tensors = primary_subgraph->tensors();
+  for (size_t i = 0; i < operators->size(); i++) {
+    OperatorInfo info;
+    auto op_inputs = operators->Get(i)->inputs();
+    auto op_outputs = operators->Get(i)->outputs();
+    info.inputs = std::vector<int>(op_inputs->begin(), op_inputs->end());
+    info.outputs = std::vector<int>(op_outputs->begin(), op_outputs->end());
+    index_to_opinfo[i] = info;
+  }
+
+  std::unordered_map<const TfLiteNode*, OperatorInfo> node_to_opinfo;
+  NodeInfoDelegateObserver observer(index_to_opinfo, &node_to_opinfo);
+  NodeInfoDelegateParams params;
+  params.delegate_observer = &observer;
+  std::unique_ptr<Interpreter> interpreter;
+  ASSERT_EQ(InterpreterBuilder(*model,
+                               ops::builtin::BuiltinOpResolver{})(&interpreter),
+            kTfLiteOk);
+  ASSERT_TRUE(interpreter);
+
+  TfLiteDelegate delegate = CreateNodeInfoDelegate(&params);
+
+  auto status = interpreter->ModifyGraphWithDelegate(&delegate);
+  EXPECT_EQ(kTfLiteOk, status);
+  EXPECT_EQ(index_to_opinfo.size(), node_to_opinfo.size());
+  EXPECT_EQ(interpreter->nodes_size(), node_to_opinfo.size());
+
+  for (const auto& node_and_opinfo : node_to_opinfo) {
+    const TfLiteNode* tflite_node = node_and_opinfo.first;
+    const OperatorInfo& info = node_and_opinfo.second;
+    ASSERT_EQ(tflite_node->inputs->size, info.inputs.size());
+    ASSERT_EQ(tflite_node->outputs->size, info.outputs.size());
+
+    for (size_t input_index = 0; input_index < info.inputs.size();
+         input_index++) {
+      const TfLiteTensor* tflite_tensor =
+          interpreter->tensor(tflite_node->inputs->data[input_index]);
+      EXPECT_EQ(tflite_tensor->name,
+                subgraph_tensors->Get(info.inputs[input_index])->name()->str());
+    }
+
+    for (size_t output_index = 0; output_index < info.outputs.size();
+         output_index++) {
+      const TfLiteTensor* tflite_tensor =
+          interpreter->tensor(tflite_node->outputs->data[output_index]);
+      EXPECT_EQ(
+          tflite_tensor->name,
+          subgraph_tensors->Get(info.outputs[output_index])->name()->str());
+    }
+  }
+}
+
+}  // namespace
+}  // namespace calibration
+}  // namespace optimize
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  // On Linux, add: FLAGS_logtostderr = true;
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
-- 
GitLab


From adfde389863219e3f2c90e566ba620d48fd44978 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Thu, 10 Jan 2019 13:18:59 -0800
Subject: [PATCH 0510/2345] Needed this change to compile after rebasing

---
 tensorflow/contrib/tensorrt/resources/trt_resources.h | 2 +-
 tensorflow/contrib/tensorrt/segment/segment.cc        | 7 -------
 2 files changed, 1 insertion(+), 8 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/resources/trt_resources.h b/tensorflow/contrib/tensorrt/resources/trt_resources.h
index 8d877b392f..aac9e5c7bd 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_resources.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_resources.h
@@ -48,7 +48,7 @@ class TRTCalibrationResource : public tensorflow::ResourceBase {
     allocator_.reset();
   }
 
-  string DebugString() const override {
+  string DebugString() override {
     std::stringstream oss;
     using std::dec;
     using std::endl;
diff --git a/tensorflow/contrib/tensorrt/segment/segment.cc b/tensorflow/contrib/tensorrt/segment/segment.cc
index 6f2a7846d7..084a96e0fa 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.cc
+++ b/tensorflow/contrib/tensorrt/segment/segment.cc
@@ -28,7 +28,6 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/types.h"
-#include "tensorflow/core/util/device_name_utils.h"
 
 namespace tensorflow {
 namespace tensorrt {
@@ -420,11 +419,6 @@ tensorflow::Status SegmentGraph(
   //    segment but are not eligible, using input/output_candidate_fn to
   //    determine the eligibilities;
   // 3. convert the segment into expected return format and return the result.
-  auto get_device_type = [](const string& device) {
-    DeviceNameUtils::ParsedName parsed_name;
-    DeviceNameUtils::ParseFullName(device, &parsed_name);
-    return parsed_name.type;
-  };
 
   // --------------------------------- Step 1 ---------------------------------
   auto graph = std::unique_ptr<SimpleGraph>(new SimpleGraph(tf_graph));
@@ -436,7 +430,6 @@ tensorflow::Status SegmentGraph(
   std::vector<UnionFind<SimpleNode*>> node_segments;
   for (int i = 0; i < graph->num_node_ids(); ++i) {
     SimpleNode* node = graph->FindNodeId(i);
-
     if (options.exclude_node_list.count(node->name()) != 0) {
       VLOG(1) << "Not a TF-TRT candidate, "
               << "(Op type: " << node->tf_node()->type_string() << "), "
-- 
GitLab


From 1271cb81425692e81a905cb0e7265c5b9c512093 Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Thu, 10 Jan 2019 13:10:22 -0800
Subject: [PATCH 0511/2345] [Grappler] Fix updating ports when removing regular
 fanins. Cleaned up some mutation logic and separated out adding and removing
 of regular and controlling fanins.

PiperOrigin-RevId: 228763029
---
 .../core/grappler/mutable_graph_view.cc       | 310 ++++++++------
 tensorflow/core/grappler/mutable_graph_view.h |  78 ++--
 .../core/grappler/mutable_graph_view_test.cc  | 396 +++++++++++-------
 3 files changed, 470 insertions(+), 314 deletions(-)

diff --git a/tensorflow/core/grappler/mutable_graph_view.cc b/tensorflow/core/grappler/mutable_graph_view.cc
index 5df8bd8113..a8a90d1a57 100644
--- a/tensorflow/core/grappler/mutable_graph_view.cc
+++ b/tensorflow/core/grappler/mutable_graph_view.cc
@@ -39,6 +39,22 @@ bool IsTensorIdPortValid(const TensorId& tensor_id) {
   return tensor_id.index() >= Graph::kControlSlot;
 }
 
+bool IsTensorIdRegular(const TensorId& tensor_id) {
+  return tensor_id.index() > Graph::kControlSlot;
+}
+
+bool IsTensorIdControlling(const TensorId& tensor_id) {
+  return tensor_id.index() == Graph::kControlSlot;
+}
+
+bool IsOutputPortRegular(const MutableGraphView::OutputPort& port) {
+  return port.port_id > Graph::kControlSlot;
+}
+
+bool IsOutputPortControlling(const MutableGraphView::OutputPort& port) {
+  return port.port_id == Graph::kControlSlot;
+}
+
 // Determines if node is an Identity where it's first regular input is a Switch
 // node.
 bool IsIdentityConsumingSwitch(const MutableGraphView& graph,
@@ -46,7 +62,10 @@ bool IsIdentityConsumingSwitch(const MutableGraphView& graph,
   if ((IsIdentity(node) || IsIdentityNSingleInput(node)) &&
       node.input_size() > 0) {
     TensorId tensor_id = ParseTensorName(node.input(0));
-    if (tensor_id.index() == Graph::kControlSlot) return false;
+    if (IsTensorIdControlling(tensor_id)) {
+      return false;
+    }
+
     NodeDef* input_node = graph.GetNode(tensor_id.node());
     return IsSwitch(*input_node);
   }
@@ -83,7 +102,7 @@ void MutableGraphView::AddAndDedupFanouts(NodeDef* node) {
   while (pos <= last_pos) {
     TensorId tensor_id = ParseTensorName(node->input(pos));
     absl::string_view input_node_name = tensor_id.node();
-    bool is_control_input = IsControlInput(tensor_id);
+    bool is_control_input = IsTensorIdControlling(tensor_id);
     bool can_dedup_control_with_regular_input =
         CanDedupControlWithRegularInput(*this, input_node_name);
     bool can_dedup_control =
@@ -246,15 +265,13 @@ bool MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
   auto control_fanouts =
       GetFanout(GraphView::OutputPort(from_node, Graph::kControlSlot));
 
-  const string from_control_input = absl::StrCat("^", from_node->name());
-  const string to_control_input = absl::StrCat("^", to_node->name());
-
   for (const InputPort& control_port : control_fanouts) {
     // Node can't be control dependency of itself.
     if (control_port.node == to_node) continue;
 
     NodeDef* node = control_port.node;
     modified |= RemoveControllingFaninInternal(node, from_node);
+    // TODO(lyandy): Handle Switch control dependencies.
     modified |= AddFaninInternal(node, {to_node, Graph::kControlSlot});
   }
 
@@ -276,7 +293,7 @@ bool MutableGraphView::AddFaninInternal(NodeDef* node,
                                         const OutputPort& fanin) {
   int num_non_controlling_fanins =
       NumFanins(*node, /*include_controlling_nodes=*/false);
-  bool input_is_control = fanin.port_id == Graph::kControlSlot;
+  bool input_is_control = IsOutputPortControlling(fanin);
   bool can_dedup_control_with_regular_input =
       CanDedupControlWithRegularInput(*this, *fanin.node);
   // Don't add duplicate control dependencies.
@@ -296,12 +313,12 @@ bool MutableGraphView::AddFaninInternal(NodeDef* node,
       input_is_control ? Graph::kControlSlot : num_non_controlling_fanins;
 
   node->add_input(TensorIdToString({fanin.node->name(), fanin.port_id}));
-  if (fanin.port_id > Graph::kControlSlot) {
-    int node_input_size = node->input_size() - 1;
+  if (IsOutputPortRegular(fanin)) {
+    int last_node_input = node->input_size() - 1;
     // If there are control dependencies in node, move newly inserted fanin to
     // be before such control dependencies.
-    if (num_non_controlling_fanins < node_input_size) {
-      node->mutable_input()->SwapElements(node_input_size,
+    if (num_non_controlling_fanins < last_node_input) {
+      node->mutable_input()->SwapElements(last_node_input,
                                           num_non_controlling_fanins);
     }
   }
@@ -327,9 +344,9 @@ bool MutableGraphView::AddFaninInternal(NodeDef* node, const TensorId& fanin) {
   return AddFaninInternal(node, {fanin_node, fanin.index()});
 }
 
-bool MutableGraphView::AddFanin(absl::string_view node_name,
-                                const TensorId& fanin) {
-  if (!IsTensorIdPortValid(fanin)) {
+bool MutableGraphView::AddRegularFanin(absl::string_view node_name,
+                                       const TensorId& fanin) {
+  if (!IsTensorIdRegular(fanin)) {
     return false;
   }
   NodeDef* node = GetNode(node_name);
@@ -339,60 +356,153 @@ bool MutableGraphView::AddFanin(absl::string_view node_name,
   return AddFaninInternal(node, fanin);
 }
 
-bool MutableGraphView::RemoveFanins(NodeDef* node,
-                                    absl::Span<const TensorId> fanins) {
-  bool modified = false;
-  auto mutable_inputs = node->mutable_input();
-  int curr_pos = 0;
-  int num_inputs = node->input_size();
-  for (int i = 0; i < num_inputs; ++i) {
-    TensorId tensor_id = ParseTensorName(node->input(i));
-    bool remove_fanin =
-        std::find(fanins.begin(), fanins.end(), tensor_id) != fanins.end();
-    bool update_fanin = !remove_fanin && modified;
-    if (remove_fanin || update_fanin) {
-      OutputPort fanin(nodes()[tensor_id.node()], tensor_id.index());
-
-      InputPort input;
-      input.node = node;
-      input.port_id =
-          tensor_id.index() == Graph::kControlSlot ? Graph::kControlSlot : i;
-
-      auto& fanouts_set = fanouts()[fanin];
-      if (remove_fanin) {
-        fanouts_set.erase(input);
-      } else {
-        // Shift inputs to be retained.
-        if (tensor_id.index() > Graph::kControlSlot) {
-          fanouts_set.erase(input);
-          fanouts_set.insert(InputPort(node, i));
+bool MutableGraphView::AddControllingFanin(absl::string_view node_name,
+                                           const TensorId& fanin) {
+  NodeDef* node = GetNode(node_name);
+  if (node == nullptr) {
+    return false;
+  }
+  NodeDef* fanin_node = GetNode(fanin.node());
+  if (fanin_node == nullptr) {
+    return false;
+  }
+
+  if (!IsSwitch(*fanin_node)) {
+    return AddFaninInternal(node, {fanin_node, Graph::kControlSlot});
+  } else {
+    if (IsTensorIdControlling(fanin)) {
+      // Cannot add a Switch node control dependency.
+      return false;
+    }
+    // We can't anchor control dependencies directly on the switch node: unlike
+    // other nodes only one of the outputs of the switch node will be generated
+    // when the switch node is executed, and we need to make sure the control
+    // dependency is only triggered when the corresponding output is triggered.
+    // We start by looking for an identity node connected to the output of the
+    // switch node, and use it to anchor the control dependency.
+    auto fanouts = GetFanouts(*fanin_node, /*include_controlled_nodes=*/false);
+    for (auto fanout : fanouts) {
+      if (IsIdentity(*fanout.node) || IsIdentityNSingleInput(*fanout.node)) {
+        if (ParseTensorName(fanout.node->input(0)) == fanin) {
+          return AddFaninInternal(node, {fanout.node, Graph::kControlSlot});
         }
-        mutable_inputs->SwapElements(i, curr_pos++);
       }
-      UpdateMaxRegularOutputPortForRemovedFanin(fanin, fanouts_set);
+    }
+    // We haven't found an existing node where we can anchor the control
+    // dependency: add a new identity node.
+    string ctrl_dep_name = AddPrefixToNodeName(
+        absl::StrCat(fanin.node(), "_", fanin.index()), kMutableGraphViewCtrl);
 
+    // Reuse a previously created node, if possible.
+    NodeDef* ctrl_dep_node = GetNode(ctrl_dep_name);
+    if (ctrl_dep_node == nullptr) {
+      NodeDef new_node;
+      new_node.set_name(ctrl_dep_name);
+      new_node.set_op("Identity");
+      new_node.set_device(fanin_node->device());
+      (*new_node.mutable_attr())["T"].set_type(
+          fanin_node->attr().at("T").type());
+      new_node.add_input(TensorIdToString(fanin));
+      ctrl_dep_node = AddNode(std::move(new_node));
+    }
+    return AddFaninInternal(node, {ctrl_dep_node, Graph::kControlSlot});
+  }
+}
+
+bool MutableGraphView::RemoveRegularFaninInternal(NodeDef* node,
+                                                  const TensorId& fanin) {
+  auto remove_input = [this, node](const TensorId& tensor_id, int port,
+                                   bool update_max_port) {
+    OutputPort fanin_port(nodes()[tensor_id.node()], tensor_id.index());
+    InputPort input(node, port);
+
+    absl::flat_hash_set<InputPort>* fanouts_set = &fanouts()[fanin_port];
+    fanouts_set->erase(input);
+    if (update_max_port) {
+      UpdateMaxRegularOutputPortForRemovedFanin(fanin_port, *fanouts_set);
+    }
+    return fanouts_set;
+  };
+
+  auto mutable_inputs = node->mutable_input();
+  bool modified = false;
+  const int num_inputs = node->input_size();
+  int i;
+  int curr_pos = 0;
+  for (i = 0; i < num_inputs; ++i) {
+    TensorId tensor_id = ParseTensorName(node->input(i));
+    if (IsTensorIdControlling(tensor_id)) {
+      break;
+    }
+    if (tensor_id == fanin) {
+      remove_input(tensor_id, i, /*update_max_port=*/true);
       modified = true;
+    } else if (modified) {
+      // Regular inputs will need to have their ports updated.
+      auto fanouts_set = remove_input(tensor_id, i, /*update_max_port=*/false);
+      fanouts_set->insert({node, curr_pos});
+      // Shift inputs to be retained.
+      mutable_inputs->SwapElements(i, curr_pos++);
     } else {
       // Skip inputs to be retained until first modification.
       curr_pos++;
     }
   }
-  if (modified) {
-    mutable_inputs->DeleteSubrange(curr_pos, num_inputs - curr_pos);
+
+  if (modified && curr_pos < i) {
+    // Remove fanins from node inputs.
+    mutable_inputs->DeleteSubrange(curr_pos, i - curr_pos);
   }
+
   return modified;
 }
 
-bool MutableGraphView::RemoveFanin(absl::string_view node_name,
-                                   const TensorId& fanin) {
-  if (!IsTensorIdPortValid(fanin)) {
+bool MutableGraphView::RemoveRegularFanin(absl::string_view node_name,
+                                          const TensorId& fanin) {
+  if (!IsTensorIdRegular(fanin)) {
     return false;
   }
   NodeDef* node = GetNode(node_name);
   if (node == nullptr) {
     return false;
   }
-  return RemoveFanins(node, {fanin});
+  return RemoveRegularFaninInternal(node, fanin);
+}
+
+bool MutableGraphView::RemoveControllingFaninInternal(NodeDef* node,
+                                                      NodeDef* fanin_node) {
+  for (int i = node->input_size() - 1; i >= 0; --i) {
+    TensorId tensor_id = ParseTensorName(node->input(i));
+    if (tensor_id.index() > Graph::kControlSlot) {
+      break;
+    }
+    if (tensor_id.node() == fanin_node->name()) {
+      fanouts()[{fanin_node, Graph::kControlSlot}].erase(
+          {node, Graph::kControlSlot});
+      node->mutable_input()->SwapElements(i, node->input_size() - 1);
+      node->mutable_input()->RemoveLast();
+      return true;
+    }
+  }
+  return false;
+}
+
+bool MutableGraphView::RemoveControllingFaninInternal(
+    NodeDef* node, absl::string_view fanin_node_name) {
+  NodeDef* fanin = GetNode(fanin_node_name);
+  if (fanin == nullptr) {
+    return false;
+  }
+  return RemoveControllingFaninInternal(node, fanin);
+}
+
+bool MutableGraphView::RemoveControllingFanin(
+    absl::string_view node_name, absl::string_view fanin_node_name) {
+  NodeDef* node = GetNode(node_name);
+  if (node == nullptr) {
+    return false;
+  }
+  return RemoveControllingFaninInternal(node, fanin_node_name);
 }
 
 bool MutableGraphView::RemoveAllFanins(absl::string_view node_name,
@@ -432,10 +542,18 @@ bool MutableGraphView::UpdateFanin(absl::string_view node_name,
 
   // When replacing a non control dependency fanin with a control dependency, or
   // vice versa, remove and add, so ports can be updated properly in fanout(s).
-  if (from_fanin.index() == Graph::kControlSlot ||
-      to_fanin.index() == Graph::kControlSlot) {
-    bool modified = RemoveFanins(node, {from_fanin});
-    modified |= AddFaninInternal(node, to_fanin);
+  bool from_fanin_is_control = IsTensorIdControlling(from_fanin);
+  if (from_fanin_is_control || IsTensorIdControlling(to_fanin)) {
+    bool modified = false;
+    if (from_fanin_is_control) {
+      modified |= RemoveControllingFaninInternal(node, from_fanin.node());
+    } else {
+      modified |= RemoveRegularFaninInternal(node, from_fanin);
+    }
+    if (modified) {
+      AddFaninInternal(node, to_fanin);
+    }
+
     return modified;
   }
 
@@ -456,7 +574,7 @@ bool MutableGraphView::UpdateFanin(absl::string_view node_name,
       InputPort old_input;
       old_input.node = node;
       old_input.port_id =
-          from_fanin.index() == Graph::kControlSlot ? Graph::kControlSlot : i;
+          IsTensorIdControlling(from_fanin) ? Graph::kControlSlot : i;
       if (from_fanin_port_fanouts == nullptr) {
         OutputPort from_fanin_port(from_fanin_node, from_fanin.index());
         from_fanin_port_fanouts = &fanouts()[from_fanin_port];
@@ -466,7 +584,7 @@ bool MutableGraphView::UpdateFanin(absl::string_view node_name,
       InputPort new_input;
       new_input.node = node;
       new_input.port_id =
-          to_fanin.index() == Graph::kControlSlot ? Graph::kControlSlot : i;
+          IsTensorIdControlling(to_fanin) ? Graph::kControlSlot : i;
       if (to_fanin_port_fanouts == nullptr) {
         OutputPort to_fanin_port(to_fanin_node, to_fanin.index());
         to_fanin_port_fanouts = &fanouts()[to_fanin_port];
@@ -493,75 +611,6 @@ bool MutableGraphView::UpdateFanin(absl::string_view node_name,
   return modified;
 }
 
-bool MutableGraphView::AddControllingFanin(absl::string_view node_name,
-                                           const TensorId& fanin) {
-  NodeDef* node = GetNode(node_name);
-  if (node == nullptr) {
-    return false;
-  }
-  NodeDef* fanin_node = GetNode(fanin.node());
-  if (fanin_node == nullptr) {
-    return false;
-  }
-  if (fanin.index() == Graph::kControlSlot) {
-    return AddFaninInternal(node, {fanin_node, Graph::kControlSlot});
-  }
-
-  if (!IsSwitch(*fanin_node)) {
-    return AddFaninInternal(node, {fanin_node, Graph::kControlSlot});
-  } else {
-    // We can't anchor control dependencies directly on the switch node: unlike
-    // other nodes only one of the outputs of the switch node will be generated
-    // when the switch node is executed, and we need to make sure the control
-    // dependency is only triggered when the corresponding output is triggered.
-    // We start by looking for an identity node connected to the output of the
-    // switch node, and use it to anchor the control dependency.
-    auto fanouts = GetFanouts(*fanin_node, /*include_controlled_nodes=*/false);
-    for (auto fanout : fanouts) {
-      if (IsIdentity(*fanout.node) || IsIdentityNSingleInput(*fanout.node)) {
-        if (ParseTensorName(fanout.node->input(0)) == fanin) {
-          return AddFaninInternal(node, {fanout.node, Graph::kControlSlot});
-        }
-      }
-    }
-    // We haven't found an existing node where we can anchor the control
-    // dependency: add a new identity node.
-    string ctrl_dep_name = AddPrefixToNodeName(
-        absl::StrCat(fanin.node(), "_", fanin.index()), kMutableGraphViewCtrl);
-
-    NodeDef* ctrl_dep_node = GetNode(ctrl_dep_name);
-    if (ctrl_dep_node == nullptr) {
-      NodeDef new_node;
-      new_node.set_name(ctrl_dep_name);
-      new_node.set_op("Identity");
-      new_node.set_device(fanin_node->device());
-      (*new_node.mutable_attr())["T"].set_type(
-          fanin_node->attr().at("T").type());
-      new_node.add_input(TensorIdToString(fanin));
-      ctrl_dep_node = AddNode(std::move(new_node));
-    }
-    return AddFaninInternal(node, {ctrl_dep_node, Graph::kControlSlot});
-  }
-}
-
-bool MutableGraphView::RemoveControllingFaninInternal(NodeDef* node,
-                                                      NodeDef* fanin_node) {
-  for (int i = node->input_size() - 1; i >= 0; --i) {
-    TensorId tensor_id = ParseTensorName(node->input(i));
-    if (tensor_id.index() > Graph::kControlSlot) {
-      break;
-    }
-    if (tensor_id.node() == fanin_node->name()) {
-      fanouts()[{fanin_node, Graph::kControlSlot}].erase(
-          {node, Graph::kControlSlot});
-      node->mutable_input()->SwapElements(i, node->input_size() - 1);
-      node->mutable_input()->RemoveLast();
-      return true;
-    }
-  }
-  return false;
-}
-
 void MutableGraphView::DeleteNodes(const std::set<string>& nodes_to_delete) {
   for (const string& node_name_to_delete : nodes_to_delete)
     RemoveFaninsInternal(nodes().at(node_name_to_delete),
@@ -575,21 +624,18 @@ void MutableGraphView::RemoveFaninsInternal(NodeDef* deleted_node,
                                             bool keep_controlling_fanins) {
   for (int i = 0; i < deleted_node->input_size(); ++i) {
     TensorId tensor_id = ParseTensorName(deleted_node->input(i));
-    if (keep_controlling_fanins && tensor_id.index() < 0) {
+    if (keep_controlling_fanins && IsTensorIdControlling(tensor_id)) {
       break;
     }
     OutputPort fanin(nodes()[tensor_id.node()], tensor_id.index());
 
     InputPort input;
     input.node = deleted_node;
-    if (tensor_id.index() < 0)
-      input.port_id = Graph::kControlSlot;
-    else
-      input.port_id = i;
-
-    auto& fanouts_set = fanouts()[fanin];
-    fanouts_set.erase(input);
-    UpdateMaxRegularOutputPortForRemovedFanin(fanin, fanouts_set);
+    input.port_id = IsTensorIdControlling(tensor_id) ? Graph::kControlSlot : i;
+
+    absl::flat_hash_set<InputPort>* fanouts_set = &fanouts()[fanin];
+    fanouts_set->erase(input);
+    UpdateMaxRegularOutputPortForRemovedFanin(fanin, *fanouts_set);
   }
 }
 
diff --git a/tensorflow/core/grappler/mutable_graph_view.h b/tensorflow/core/grappler/mutable_graph_view.h
index 93f0958d5e..f07254068d 100644
--- a/tensorflow/core/grappler/mutable_graph_view.h
+++ b/tensorflow/core/grappler/mutable_graph_view.h
@@ -78,39 +78,17 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // This will return true iff the nodes are modified.
   bool UpdateFanouts(absl::string_view from_node, absl::string_view to_node);
 
-  // Adds fanin to node `node_name`. If the node or fanin do not exist in the
-  // graph, nothing will be modified in the graph. If fanin is a control
-  // dependency, existing control dependencies will be checked first before
-  // adding. Otherwise fanin will be added after existing non control dependency
-  // inputs.
-  //
-  // This will return true iff the node is modified. If a control dependency
-  // already exists, the node will not be modified.
-  bool AddFanin(absl::string_view node_name, const TensorId& fanin);
-
-  // Removes fanin from node `node_name`. If the node or fanin do not exist in
-  // the graph, nothing will be modified in the graph. If there are multiple
-  // inputs that match the fanin, all of them will be removed.
-  //
-  // This will return true iff the node is modified. If no inputs match the
-  // fanin, the node will not be modified.
-  bool RemoveFanin(absl::string_view node_name, const TensorId& fanin);
-
-  // Removes all fanins from node `node_name`. Control dependencies will be
-  // retained if keep_controlling_fanins is true.
-  //
-  // This will return true iff the node is modified.
-  bool RemoveAllFanins(absl::string_view node_name,
-                       bool keep_controlling_fanins);
-
-  // Replaces all fanins `from_fanin` with `to_fanin` in node `node_name`. If
-  // the fanins or node do not exist, nothing will be modified in the graph.
+  // Adds regular fanin `fanin` to node `node_name`. If the node or fanin do not
+  // exist in the graph, nothing will be modified in the graph. Otherwise fanin
+  // will be added after existing non control dependency fanins. Control
+  // dependencies will be deduped. To add control dependencies, use
+  // AddControllingFanin.
   //
   // This will return true iff the node is modified.
-  bool UpdateFanin(absl::string_view node_name, const TensorId& from_fanin,
-                   const TensorId& to_fanin);
+  bool AddRegularFanin(absl::string_view node_name, const TensorId& fanin);
 
-  // Adds a control dependency to the target node named `node_name`.
+  // Adds control dependency `fanin` to the target node named `node_name`. To
+  // add regular fanins, use AddRegularFanin.
   //
   // Case 1: If the fanin is not a Switch node, the control dependency is simply
   // added to the target node:
@@ -130,6 +108,37 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // This will return true iff the node is modified.
   bool AddControllingFanin(absl::string_view node_name, const TensorId& fanin);
 
+  // Removes regular fanin `fanin` from node `node_name`. If the node or fanin
+  // do not exist in the graph, nothing will be modified in the graph. If there
+  // are multiple inputs that match the fanin, all of them will be removed. To
+  // remove controlling fanins, use RemoveControllingFanin.
+  //
+  // This will return true iff the node is modified.
+  bool RemoveRegularFanin(absl::string_view node_name, const TensorId& fanin);
+
+  // Removes control dependency `fanin_node_name` from the target node named
+  // `node_name`. If the node or fanin do not exist in the graph, nothing will
+  // be modified in the graph. To remove regular fanins, use RemoveRegualrFanin.
+  //
+  // This will return true iff the node is modified.
+  bool RemoveControllingFanin(absl::string_view node_name,
+                              absl::string_view fanin_node_name);
+
+  // Removes all fanins from node `node_name`. Control dependencies will be
+  // retained if keep_controlling_fanins is true.
+  //
+  // This will return true iff the node is modified.
+  bool RemoveAllFanins(absl::string_view node_name,
+                       bool keep_controlling_fanins);
+
+  // Replaces all fanins `from_fanin` with `to_fanin` in node `node_name`. If
+  // the fanins or node do not exist, nothing will be modified in the graph.
+  // Control dependencies will be deduped.
+  //
+  // This will return true iff the node is modified.
+  bool UpdateFanin(absl::string_view node_name, const TensorId& from_fanin,
+                   const TensorId& to_fanin);
+
   // Deletes nodes from the graph.
   void DeleteNodes(const std::set<string>& nodes_to_delete);
 
@@ -189,7 +198,14 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // Removes any fanin in node that matches to a fanin in fanins.
   //
   // This will return true iff the node is modified.
-  bool RemoveFanins(NodeDef* node, absl::Span<const TensorId> fanins);
+  bool RemoveRegularFaninInternal(NodeDef* node, const TensorId& fanin);
+
+  // Removes controlling fanin `fanin_node_name` from node if such controlling
+  // fanin exists.
+  //
+  // This will return true iff the node is modified.
+  bool RemoveControllingFaninInternal(NodeDef* node,
+                                      absl::string_view fanin_node_name);
 
   // Removes controlling fanin `fanin_node` from node if such controlling fanin
   // exists.
diff --git a/tensorflow/core/grappler/mutable_graph_view_test.cc b/tensorflow/core/grappler/mutable_graph_view_test.cc
index 526ca85ec0..2048c67e3e 100644
--- a/tensorflow/core/grappler/mutable_graph_view_test.cc
+++ b/tensorflow/core/grappler/mutable_graph_view_test.cc
@@ -186,10 +186,10 @@ GraphDef SimpleMutateFaninGraph() {
 void CompareNodeInputs(const MutableGraphView& graph, const NodeDef* expected,
                        NodeDef* actual) {
   ASSERT_EQ(actual->input_size(), expected->input_size());
-  int port;
   for (int i = 0; i < actual->input_size(); ++i) {
     EXPECT_EQ(actual->input(i), expected->input(i));
     TensorId tensor_id = ParseTensorName(expected->input(i));
+    int port;
     if (tensor_id.index() == Graph::kControlSlot) {
       port = Graph::kControlSlot;
     } else {
@@ -203,8 +203,9 @@ void CompareNodeInputs(const MutableGraphView& graph, const NodeDef* expected,
   }
 }
 
-void TestAddFanin(absl::string_view node_name, const TensorId& fanin_to_add,
-                  bool modified, const NodeDef* expected_node) {
+void TestAddRegularFanin(absl::string_view node_name,
+                         const TensorId& fanin_to_add, bool modified,
+                         const NodeDef* expected_node) {
   GraphDef graph_def = SimpleMutateFaninGraph();
 
   MutableGraphView graph(&graph_def);
@@ -216,70 +217,71 @@ void TestAddFanin(absl::string_view node_name, const TensorId& fanin_to_add,
     EXPECT_NE(node, nullptr);
   }
 
-  EXPECT_EQ(modified, graph.AddFanin(node_name, fanin_to_add));
+  EXPECT_EQ(modified, graph.AddRegularFanin(node_name, fanin_to_add));
   if (expected_node != nullptr) {
     CompareNodeInputs(graph, expected_node, node);
   }
 }
 
-TEST(MutableGraphViewTest, AddFanin) {
+TEST(MutableGraphViewTest, AddRegularFanin) {
   NodeDef expected_node;
   // Add input to node with 1 input 0 controls.
   expected_node = NDef("", "", {"a", "b:1"});
-  TestAddFanin("foo_1", {"b", 1}, /*modified=*/true, &expected_node);
+  TestAddRegularFanin("foo_1", {"b", 1}, /*modified=*/true, &expected_node);
   // Add input to node with multiple inputs and 0 controls.
   expected_node = NDef("", "", {"b", "a:1", "a:1", "b:2"});
-  TestAddFanin("foo_3", {"b", 2}, /*modified=*/true, &expected_node);
+  TestAddRegularFanin("foo_3", {"b", 2}, /*modified=*/true, &expected_node);
   // Add input to node with 1 input multiple controls.
   expected_node = NDef("", "", {"b", "a", "^c"});
-  TestAddFanin("foo_2", {"a", 0}, /*modified=*/true, &expected_node);
+  TestAddRegularFanin("foo_2", {"a", 0}, /*modified=*/true, &expected_node);
   // Add input to node with multiple inputs and controls.
   expected_node = NDef("", "", {"a", "b:2", "b:2", "a:1", "^d", "^c"});
-  TestAddFanin("foo_4", {"a", 1}, /*modified=*/true, &expected_node);
+  TestAddRegularFanin("foo_4", {"a", 1}, /*modified=*/true, &expected_node);
   // Add input to node with 0 inputs 0 controls.
   expected_node = NDef("", "", {"a:1"});
-  TestAddFanin("foo_5", {"a", 1}, /*modified=*/true, &expected_node);
+  TestAddRegularFanin("foo_5", {"a", 1}, /*modified=*/true, &expected_node);
   // Add input to node with 0 inputs multiple controls.
   expected_node = NDef("", "", {"c:1", "^b", "^a"});
-  TestAddFanin("foo_6", {"c", 1}, /*modified=*/true, &expected_node);
+  TestAddRegularFanin("foo_6", {"c", 1}, /*modified=*/true, &expected_node);
 
   // Add control to node with 1 input 0 controls.
-  expected_node = NDef("", "", {"a", "^b"});
-  TestAddFanin("foo_1", {"b", Graph::kControlSlot}, /*modified=*/true,
-               &expected_node);
+  expected_node = NDef("", "", {"a"});
+  TestAddRegularFanin("foo_1", {"b", Graph::kControlSlot}, /*modified=*/false,
+                      &expected_node);
   // Add control to node with multiple inputs and 0 controls.
-  expected_node = NDef("", "", {"b", "a:1", "a:1", "^c"});
-  TestAddFanin("foo_3", {"c", Graph::kControlSlot}, /*modified=*/true,
-               &expected_node);
+  expected_node = NDef("", "", {"b", "a:1", "a:1"});
+  TestAddRegularFanin("foo_3", {"c", Graph::kControlSlot}, /*modified=*/false,
+                      &expected_node);
   // Add control to node with 1 input multiple controls.
-  expected_node = NDef("", "", {"b", "^a", "^c", "^d"});
-  TestAddFanin("foo_2", {"d", Graph::kControlSlot}, /*modified=*/true,
-               &expected_node);
+  expected_node = NDef("", "", {"b", "^a", "^c"});
+  TestAddRegularFanin("foo_2", {"d", Graph::kControlSlot}, /*modified=*/false,
+                      &expected_node);
   // Add control to node with multiple input multiple controls.
   expected_node = NDef("", "", {"a", "b:2", "b:2", "^c", "^d"});
-  TestAddFanin("foo_4", {"a", Graph::kControlSlot}, /*modified=*/false,
-               &expected_node);
+  TestAddRegularFanin("foo_4", {"a", Graph::kControlSlot},
+                      /*modified=*/false, &expected_node);
   // Add control to node with 0 inputs 0 controls.
-  expected_node = NDef("", "", {"^a"});
-  TestAddFanin("foo_5", {"a", Graph::kControlSlot}, /*modified=*/true,
-               &expected_node);
+  expected_node = NDef("", "", {});
+  TestAddRegularFanin("foo_5", {"a", Graph::kControlSlot}, /*modified=*/false,
+                      &expected_node);
   // Add control to node with 0 inputs multiple controls.
-  expected_node = NDef("", "", {"^a", "^b", "^c"});
-  TestAddFanin("foo_6", {"c", Graph::kControlSlot}, /*modified=*/true,
-               &expected_node);
+  expected_node = NDef("", "", {"^a", "^b"});
+  TestAddRegularFanin("foo_6", {"c", Graph::kControlSlot}, /*modified=*/false,
+                      &expected_node);
   // Add control to node with control that already exists.
   expected_node = NDef("", "", {"b", "^a", "^c"});
-  TestAddFanin("foo_2", {"a", Graph::kControlSlot}, /*modified=*/false,
-               &expected_node);
+  TestAddRegularFanin("foo_2", {"a", Graph::kControlSlot},
+                      /*modified=*/false, &expected_node);
 
   // Add fanin to node where node is missing.
-  TestAddFanin("foo_missing", {"a", 0}, /*modified=*/false, nullptr);
+  TestAddRegularFanin("foo_missing", {"a", 0}, /*modified=*/false, nullptr);
   // Add fanin to node where fanin is missing.
   expected_node = NDef("", "", {"a"});
-  TestAddFanin("foo_1", {"bar_missing", 0}, /*modified=*/false, &expected_node);
+  TestAddRegularFanin("foo_1", {"bar_missing", 0}, /*modified=*/false,
+                      &expected_node);
   // Add fanin to node where node and fanin are missing.
-  TestAddFanin("foo_missing", {"bar_missing", 0}, /*modified=*/false,
-               /*expected_node=*/nullptr);
+  TestAddRegularFanin("foo_missing", {"bar_missing", 0}, /*modified=*/false,
+                      /*expected_node=*/nullptr);
 }
 
 void CheckFanout(const MutableGraphView& graph, const TensorId& fanin,
@@ -292,9 +294,9 @@ void CheckFanout(const MutableGraphView& graph, const TensorId& fanin,
   }
 }
 
-void TestRemoveFanin(absl::string_view node_name,
-                     const TensorId& fanin_to_remove, bool modified,
-                     const NodeDef* expected_node) {
+void TestRemoveRegularFanin(absl::string_view node_name,
+                            const TensorId& fanin_to_remove, bool modified,
+                            const NodeDef* expected_node) {
   GraphDef graph_def = SimpleMutateFaninGraph();
 
   MutableGraphView graph(&graph_def);
@@ -306,7 +308,7 @@ void TestRemoveFanin(absl::string_view node_name,
     EXPECT_NE(nullptr, node);
   }
 
-  EXPECT_EQ(modified, graph.RemoveFanin(node_name, fanin_to_remove));
+  EXPECT_EQ(modified, graph.RemoveRegularFanin(node_name, fanin_to_remove));
   if (expected_node != nullptr) {
     CompareNodeInputs(graph, expected_node, node);
     if (modified) {
@@ -315,63 +317,68 @@ void TestRemoveFanin(absl::string_view node_name,
   }
 }
 
-TEST(MutableGraphViewTest, RemoveFanin) {
+TEST(MutableGraphViewTest, RemoveRegularFanin) {
   NodeDef expected_node;
   // Remove input from node with 1 input 0 controls.
   expected_node = NDef("", "", {});
-  TestRemoveFanin("foo_1", {"a", 0}, /*modified=*/true, &expected_node);
+  TestRemoveRegularFanin("foo_1", {"a", 0}, /*modified=*/true, &expected_node);
   // Remove input from node with multiple inputs and 0 controls.
   expected_node = NDef("", "", {"b"});
-  TestRemoveFanin("foo_3", {"a", 1}, /*modified=*/true, &expected_node);
+  TestRemoveRegularFanin("foo_3", {"a", 1}, /*modified=*/true, &expected_node);
   // Remove input from node with 1 input multiple controls.
   expected_node = NDef("", "", {"^a", "^c"});
-  TestRemoveFanin("foo_2", {"b", 0}, /*modified=*/true, &expected_node);
+  TestRemoveRegularFanin("foo_2", {"b", 0}, /*modified=*/true, &expected_node);
   // Remove input from node with multiple inputs and controls.
   expected_node = NDef("", "", {"a", "^c", "^d"});
-  TestRemoveFanin("foo_4", {"b", 2}, /*modified=*/true, &expected_node);
+  TestRemoveRegularFanin("foo_4", {"b", 2}, /*modified=*/true, &expected_node);
+  // Remove input from node with multiple inputs and controls, and results in
+  // shifting of ports.
+  expected_node = NDef("", "", {"b:2", "b:2", "^c", "^d"});
+  TestRemoveRegularFanin("foo_4", {"a", 0}, /*modified=*/true, &expected_node);
 
   // Remove control from node with 1 input multiple controls.
-  expected_node = NDef("", "", {"b", "^c"});
-  TestRemoveFanin("foo_2", {"a", Graph::kControlSlot}, /*modified=*/true,
-                  &expected_node);
+  expected_node = NDef("", "", {"b", "^a", "^c"});
+  TestRemoveRegularFanin("foo_2", {"a", Graph::kControlSlot},
+                         /*modified=*/false, &expected_node);
   // Remove control from node with multiple input multiple controls.
-  expected_node = NDef("", "", {"a", "b:2", "b:2", "^c"});
-  TestRemoveFanin("foo_4", {"d", Graph::kControlSlot}, /*modified=*/true,
-                  &expected_node);
+  expected_node = NDef("", "", {"a", "b:2", "b:2", "^c", "^d"});
+  TestRemoveRegularFanin("foo_4", {"d", Graph::kControlSlot},
+                         /*modified=*/false, &expected_node);
   // Remove control from node with 0 inputs multiple controls.
-  expected_node = NDef("", "", {"^b"});
-  TestRemoveFanin("foo_6", {"a", Graph::kControlSlot}, /*modified=*/true,
-                  &expected_node);
+  expected_node = NDef("", "", {"^a", "^b"});
+  TestRemoveRegularFanin("foo_6", {"a", Graph::kControlSlot},
+                         /*modified=*/false, &expected_node);
 
   // Remove input from node with 0 inputs 0 controls.
   expected_node = NDef("", "", {});
-  TestRemoveFanin("foo_5", {"a", 1}, /*modified=*/false, &expected_node);
+  TestRemoveRegularFanin("foo_5", {"a", 1}, /*modified=*/false, &expected_node);
   // Remove input from node with 0 inputs multiple controls.
   expected_node = NDef("", "", {"^a", "^b"});
-  TestRemoveFanin("foo_6", {"a", 1}, /*modified=*/false, &expected_node);
+  TestRemoveRegularFanin("foo_6", {"a", 1}, /*modified=*/false, &expected_node);
+
   // Remove control from node with 1 input 0 controls.
   expected_node = NDef("", "", {"a"});
-  TestRemoveFanin("foo_1", {"b", Graph::kControlSlot}, /*modified=*/false,
-                  &expected_node);
+  TestRemoveRegularFanin("foo_1", {"b", Graph::kControlSlot},
+                         /*modified=*/false, &expected_node);
   // Remove control from node with multiple inputs and 0 controls.
   expected_node = NDef("", "", {"b", "a:1", "a:1"});
-  TestRemoveFanin("foo_3", {"c", Graph::kControlSlot}, /*modified=*/false,
-                  &expected_node);
+  TestRemoveRegularFanin("foo_3", {"c", Graph::kControlSlot},
+                         /*modified=*/false, &expected_node);
   // Remove control from node with 0 inputs 0 controls.
   expected_node = NDef("", "", {});
-  TestRemoveFanin("foo_5", {"a", Graph::kControlSlot}, /*modified=*/false,
-                  &expected_node);
+  TestRemoveRegularFanin("foo_5", {"a", Graph::kControlSlot},
+                         /*modified=*/false, &expected_node);
 
   // Remove fanin from node where node is missing.
-  TestRemoveFanin("foo_missing", {"a", 0}, /*modified=*/false,
-                  /*expected_node=*/nullptr);
+  TestRemoveRegularFanin("foo_missing", {"a", 0}, /*modified=*/false,
+                         /*expected_node=*/nullptr);
   // Remove fanin from node where fanin is missing.
   expected_node = NDef("", "", {"a"});
-  TestRemoveFanin("foo_1", {"bar_missing", 0}, /*modified=*/false,
-                  &expected_node);
+  TestRemoveRegularFanin("foo_1", {"bar_missing", 0}, /*modified=*/false,
+                         &expected_node);
   // Remove fanin from node where node and fanin are missing.
-  TestRemoveFanin("foo_missing", {"bar_missing", 0}, /*modified=*/false,
-                  /*expected_node=*/nullptr);
+  TestRemoveRegularFanin("foo_missing", {"bar_missing", 0}, /*modified=*/false,
+                         /*expected_node=*/nullptr);
 }
 
 void TestRemoveAllFanins(absl::string_view node_name,
@@ -540,56 +547,59 @@ TEST(MutableGraphViewTest, DedupControllingFaninsOnGraphInit) {
   MutableGraphView graph(&graph_def);
 
   EXPECT_EQ(graph.graph()->node_size(), 11);
+  NodeDef expected;
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
-  ASSERT_EQ(a->input_size(), 0);
+  expected = NDef("", "", {});
+  CompareNodeInputs(graph, &expected, a);
+
   NodeDef* b = graph.GetNode("b");
   ASSERT_NE(b, nullptr);
-  ASSERT_EQ(b->input_size(), 0);
+  CompareNodeInputs(graph, &expected, b);
+
   NodeDef* c = graph.GetNode("c");
   ASSERT_NE(c, nullptr);
-  ASSERT_EQ(c->input_size(), 0);
+  CompareNodeInputs(graph, &expected, c);
+
   NodeDef* d = graph.GetNode("d");
   ASSERT_NE(d, nullptr);
-  ASSERT_EQ(d->input_size(), 1);
-  EXPECT_EQ(d->input(0), "c:1");
+  expected = NDef("", "", {"c:1"});
+  CompareNodeInputs(graph, &expected, d);
+
   NodeDef* foo_1 = graph.GetNode("foo_1");
   ASSERT_NE(foo_1, nullptr);
-  ASSERT_EQ(foo_1->input_size(), 2);
-  EXPECT_EQ(foo_1->input(0), "a");
-  EXPECT_EQ(foo_1->input(1), "b:1");
+  expected = NDef("", "", {"a", "b:1"});
+  CompareNodeInputs(graph, &expected, foo_1);
+
   NodeDef* foo_2 = graph.GetNode("foo_2");
   ASSERT_NE(foo_2, nullptr);
-  ASSERT_EQ(foo_2->input_size(), 2);
-  EXPECT_EQ(foo_2->input(0), "a");
-  EXPECT_EQ(foo_2->input(1), "^b");
+  expected = NDef("", "", {"a", "^b"});
+  CompareNodeInputs(graph, &expected, foo_2);
+
   NodeDef* foo_3 = graph.GetNode("foo_3");
   ASSERT_NE(foo_3, nullptr);
-  ASSERT_EQ(foo_3->input_size(), 2);
-  EXPECT_EQ(foo_3->input(0), "a");
-  EXPECT_EQ(foo_3->input(1), "b:1");
+  expected = NDef("", "", {"a", "b:1"});
+  CompareNodeInputs(graph, &expected, foo_3);
+
   NodeDef* foo_4 = graph.GetNode("foo_4");
   ASSERT_NE(foo_4, nullptr);
-  ASSERT_EQ(foo_4->input_size(), 2);
-  EXPECT_EQ(foo_4->input(0), "a:2");
-  EXPECT_EQ(foo_4->input(1), "b:1");
+  expected = NDef("", "", {"a:2", "b:1"});
+  CompareNodeInputs(graph, &expected, foo_4);
+
   NodeDef* foo_5 = graph.GetNode("foo_5");
   ASSERT_NE(foo_5, nullptr);
-  ASSERT_EQ(foo_5->input_size(), 2);
-  EXPECT_EQ(foo_5->input(0), "a:2");
-  EXPECT_EQ(foo_5->input(1), "b:1");
+  expected = NDef("", "", {"a:2", "b:1"});
+  CompareNodeInputs(graph, &expected, foo_5);
+
   NodeDef* foo_6 = graph.GetNode("foo_6");
   ASSERT_NE(foo_6, nullptr);
-  ASSERT_EQ(foo_6->input_size(), 2);
-  EXPECT_EQ(foo_6->input(0), "d");
-  EXPECT_EQ(foo_6->input(1), "^d");
+  expected = NDef("", "", {"d", "^d"});
+  CompareNodeInputs(graph, &expected, foo_6);
+
   NodeDef* foo_7 = graph.GetNode("foo_7");
   ASSERT_NE(foo_7, nullptr);
-  ASSERT_EQ(foo_7->input_size(), 4);
-  EXPECT_EQ(foo_7->input(0), "a:3");
-  EXPECT_EQ(foo_7->input(1), "b:2");
-  EXPECT_EQ(foo_7->input(2), "d");
-  EXPECT_EQ(foo_7->input(3), "^d");
+  expected = NDef("", "", {"a:3", "b:2", "d", "^d"});
+  CompareNodeInputs(graph, &expected, foo_7);
 }
 
 TEST(MutableGraphViewTest, DedupControllingFaninsOnAddFanin) {
@@ -601,17 +611,18 @@ TEST(MutableGraphViewTest, DedupControllingFaninsOnAddFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddFanin("b", {"a", 2}));
+  EXPECT_TRUE(graph.AddRegularFanin("b", {"a", 2}));
+  NodeDef expected;
   NodeDef* b = graph.GetNode("b");
   ASSERT_NE(b, nullptr);
-  ASSERT_EQ(b->input_size(), 1);
-  EXPECT_EQ(b->input(0), "a:2");
+  expected = NDef("", "", {"a:2"});
+  CompareNodeInputs(graph, &expected, b);
 
-  EXPECT_FALSE(graph.AddFanin("c", {"a", Graph::kControlSlot}));
+  EXPECT_FALSE(graph.AddControllingFanin("c", {"a", Graph::kControlSlot}));
   NodeDef* c = graph.GetNode("c");
   ASSERT_NE(c, nullptr);
-  ASSERT_EQ(c->input_size(), 1);
-  EXPECT_EQ(c->input(0), "a:1");
+  expected = NDef("", "", {"a:1"});
+  CompareNodeInputs(graph, &expected, c);
 }
 
 TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnAddFanin) {
@@ -622,32 +633,30 @@ TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnAddFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddFanin("c", {"b", 2}));
+  EXPECT_TRUE(graph.AddRegularFanin("c", {"b", 2}));
+  NodeDef expected;
   NodeDef* c = graph.GetNode("c");
   ASSERT_NE(c, nullptr);
-  ASSERT_EQ(c->input_size(), 1);
-  EXPECT_EQ(c->input(0), "b:2");
-  EXPECT_TRUE(graph.AddFanin("c", {"b", Graph::kControlSlot}));
-  ASSERT_EQ(c->input_size(), 2);
-  EXPECT_EQ(c->input(0), "b:2");
-  EXPECT_EQ(c->input(1), "^b");
-  EXPECT_FALSE(graph.AddFanin("c", {"b", Graph::kControlSlot}));
-  ASSERT_EQ(c->input_size(), 2);
-  EXPECT_EQ(c->input(0), "b:2");
-  EXPECT_EQ(c->input(1), "^b");
-
-  EXPECT_TRUE(graph.AddFanin("d", {"b", Graph::kControlSlot}));
+  expected = NDef("", "", {"b:2"});
+  CompareNodeInputs(graph, &expected, c);
+  EXPECT_TRUE(graph.AddControllingFanin("c", {"b", Graph::kControlSlot}));
+  expected = NDef("", "", {"b:2", "^b"});
+  CompareNodeInputs(graph, &expected, c);
+  EXPECT_FALSE(graph.AddControllingFanin("c", {"b", Graph::kControlSlot}));
+  expected = NDef("", "", {"b:2", "^b"});
+  CompareNodeInputs(graph, &expected, c);
+
+  EXPECT_TRUE(graph.AddControllingFanin("d", {"b", Graph::kControlSlot}));
   NodeDef* d = graph.GetNode("d");
   ASSERT_NE(d, nullptr);
-  ASSERT_EQ(d->input_size(), 1);
-  EXPECT_EQ(d->input(0), "^b");
-  EXPECT_FALSE(graph.AddFanin("d", {"b", Graph::kControlSlot}));
-  ASSERT_EQ(d->input_size(), 1);
-  EXPECT_EQ(d->input(0), "^b");
-  EXPECT_TRUE(graph.AddFanin("d", {"b", 3}));
-  ASSERT_EQ(d->input_size(), 2);
-  EXPECT_EQ(d->input(0), "b:3");
-  EXPECT_EQ(d->input(1), "^b");
+  expected = NDef("", "", {"^b"});
+  CompareNodeInputs(graph, &expected, d);
+  EXPECT_FALSE(graph.AddControllingFanin("d", {"b", Graph::kControlSlot}));
+  expected = NDef("", "", {"^b"});
+  CompareNodeInputs(graph, &expected, d);
+  EXPECT_TRUE(graph.AddRegularFanin("d", {"b", 3}));
+  expected = NDef("", "", {"b:3", "^b"});
+  CompareNodeInputs(graph, &expected, d);
 }
 
 TEST(MutableGraphViewTest, DedupControllingFaninsOnUpdateFanin) {
@@ -662,8 +671,8 @@ TEST(MutableGraphViewTest, DedupControllingFaninsOnUpdateFanin) {
   EXPECT_TRUE(graph.UpdateFanin("c", {"a", 1}, {"b", 2}));
   NodeDef* c = graph.GetNode("c");
   ASSERT_NE(c, nullptr);
-  ASSERT_EQ(c->input_size(), 1);
-  EXPECT_EQ(c->input(0), "b:2");
+  NodeDef expected = NDef("", "", {"b:2"});
+  CompareNodeInputs(graph, &expected, c);
 }
 
 TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnUpdateFanin) {
@@ -677,23 +686,22 @@ TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnUpdateFanin) {
 
   EXPECT_TRUE(graph.UpdateFanin("d", {"b", Graph::kControlSlot},
                                 {"c", Graph::kControlSlot}));
+  NodeDef expected;
   NodeDef* d = graph.GetNode("d");
   ASSERT_NE(d, nullptr);
-  ASSERT_EQ(d->input_size(), 2);
-  EXPECT_EQ(d->input(0), "c");
-  EXPECT_EQ(d->input(1), "^c");
+  expected = NDef("", "", {"c", "^c"});
+  CompareNodeInputs(graph, &expected, d);
 
   EXPECT_TRUE(graph.UpdateFanin("e", {"b", 0}, {"c", 3}));
   NodeDef* e = graph.GetNode("e");
   ASSERT_NE(e, nullptr);
-  ASSERT_EQ(e->input_size(), 2);
-  EXPECT_EQ(e->input(0), "c:3");
-  EXPECT_EQ(e->input(1), "^c");
+  expected = NDef("", "", {"c:3", "^c"});
+  CompareNodeInputs(graph, &expected, e);
 
   EXPECT_TRUE(graph.UpdateFanin("e", {"c", 3}, {"c", Graph::kControlSlot}));
   ASSERT_NE(e, nullptr);
-  ASSERT_EQ(e->input_size(), 1);
-  EXPECT_EQ(e->input(0), "^c");
+  expected = NDef("", "", {"^c"});
+  CompareNodeInputs(graph, &expected, e);
 }
 
 TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnAddFanin) {
@@ -705,14 +713,17 @@ TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnAddFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddFanin("c", {"a", 3}));
+  EXPECT_TRUE(graph.AddRegularFanin("c", {"a", 3}));
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
+
   auto fanouts = graph.GetFanouts(*a, /*include_controlled_nodes=*/true);
   EXPECT_EQ(fanouts.size(), 2);
+
   NodeDef* b = graph.GetNode("b");
   ASSERT_NE(b, nullptr);
   EXPECT_EQ(fanouts.count(MutableGraphView::InputPort(b, 0)), 1);
+
   NodeDef* c = graph.GetNode("c");
   ASSERT_NE(c, nullptr);
   EXPECT_EQ(fanouts.count(MutableGraphView::InputPort(c, 0)), 1);
@@ -727,11 +738,13 @@ TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnRemoveFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.RemoveFanin("c", {"a", 2}));
+  EXPECT_TRUE(graph.RemoveRegularFanin("c", {"a", 2}));
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
+
   auto fanouts = graph.GetFanouts(*a, /*include_controlled_nodes=*/true);
   EXPECT_EQ(fanouts.size(), 1);
+
   NodeDef* b = graph.GetNode("b");
   ASSERT_NE(b, nullptr);
   EXPECT_EQ(fanouts.count(MutableGraphView::InputPort(b, 0)), 1);
@@ -746,11 +759,13 @@ TEST(MutableGraphViewTest, KeepMaxRegularOutputPortOnRemoveFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.RemoveFanin("b", {"a", 1}));
+  EXPECT_TRUE(graph.RemoveRegularFanin("b", {"a", 1}));
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
+
   auto fanouts = graph.GetFanouts(*a, /*include_controlled_nodes=*/true);
   EXPECT_EQ(fanouts.size(), 1);
+
   NodeDef* c = graph.GetNode("c");
   ASSERT_NE(c, nullptr);
   EXPECT_EQ(fanouts.count(MutableGraphView::InputPort(c, 0)), 1);
@@ -768,13 +783,17 @@ TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnUpdateFanin) {
   EXPECT_TRUE(graph.UpdateFanin("c", {"a", 2}, {"b", 3}));
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
+
   auto a_fanouts = graph.GetFanouts(*a, /*include_controlled_nodes=*/true);
   EXPECT_EQ(a_fanouts.size(), 1);
+
   NodeDef* b = graph.GetNode("b");
   ASSERT_NE(b, nullptr);
   EXPECT_EQ(a_fanouts.count(MutableGraphView::InputPort(b, 0)), 1);
+
   auto b_fanouts = graph.GetFanouts(*b, /*include_controlled_nodes=*/true);
   EXPECT_EQ(b_fanouts.size(), 1);
+
   NodeDef* c = graph.GetNode("c");
   ASSERT_NE(c, nullptr);
   EXPECT_EQ(b_fanouts.count(MutableGraphView::InputPort(c, 0)), 1);
@@ -795,12 +814,15 @@ TEST(MutableGraphViewTest, AddControllingFaninMissing) {
   EXPECT_FALSE(graph.AddControllingFanin("c", {"d", Graph::kControlSlot}));
 
   ASSERT_EQ(graph.graph()->node_size(), 2);
+  NodeDef expected;
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
-  ASSERT_EQ(a->input_size(), 0);
+  expected = NDef("", "", {});
+  CompareNodeInputs(graph, &expected, a);
+
   NodeDef* b = graph.GetNode("b");
   ASSERT_NE(b, nullptr);
-  ASSERT_EQ(b->input_size(), 0);
+  CompareNodeInputs(graph, &expected, b);
 }
 
 TEST(MutableGraphViewTest, AddControllingFaninExistingControl) {
@@ -814,13 +836,16 @@ TEST(MutableGraphViewTest, AddControllingFaninExistingControl) {
   EXPECT_FALSE(graph.AddControllingFanin("a", {"b", Graph::kControlSlot}));
 
   ASSERT_EQ(graph.graph()->node_size(), 2);
+  NodeDef expected;
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
-  ASSERT_EQ(a->input_size(), 1);
-  EXPECT_EQ(a->input(0), "^b");
+  expected = NDef("", "", {"^b"});
+  CompareNodeInputs(graph, &expected, a);
+
   NodeDef* b = graph.GetNode("b");
   ASSERT_NE(b, nullptr);
-  ASSERT_EQ(b->input_size(), 0);
+  expected = NDef("", "", {});
+  CompareNodeInputs(graph, &expected, b);
 }
 
 TEST(MutableGraphViewTest, AddControllingFaninNotSwitch) {
@@ -834,13 +859,16 @@ TEST(MutableGraphViewTest, AddControllingFaninNotSwitch) {
   EXPECT_FALSE(graph.AddControllingFanin("a", {"b", 2}));
 
   ASSERT_EQ(graph.graph()->node_size(), 2);
+  NodeDef expected;
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
-  ASSERT_EQ(a->input_size(), 1);
-  EXPECT_EQ(a->input(0), "^b");
+  expected = NDef("", "", {"^b"});
+  CompareNodeInputs(graph, &expected, a);
+
   NodeDef* b = graph.GetNode("b");
   ASSERT_NE(b, nullptr);
-  ASSERT_EQ(b->input_size(), 0);
+  expected = NDef("", "", {});
+  CompareNodeInputs(graph, &expected, b);
 }
 
 TEST(MutableGraphViewTest, AddControllingFaninSwitchWithIdentity) {
@@ -857,8 +885,8 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithIdentity) {
   ASSERT_EQ(graph.graph()->node_size(), 3);
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
-  ASSERT_EQ(a->input_size(), 1);
-  EXPECT_EQ(a->input(0), "^identity");
+  NodeDef expected = NDef("", "", {"^identity"});
+  CompareNodeInputs(graph, &expected, a);
 }
 
 TEST(MutableGraphViewTest, AddControllingFaninSwitchWithNoExistingIdentity) {
@@ -874,14 +902,16 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithNoExistingIdentity) {
   EXPECT_FALSE(graph.AddControllingFanin("a", {"switch", 0}));
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
+  NodeDef expected;
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
-  ASSERT_EQ(a->input_size(), 1);
-  EXPECT_EQ(a->input(0), "^ConstantFoldingCtrl/switch_0");
+  expected = NDef("", "", {"^ConstantFoldingCtrl/switch_0"});
+  CompareNodeInputs(graph, &expected, a);
+
   NodeDef* identity = graph.GetNode("ConstantFoldingCtrl/switch_0");
   ASSERT_NE(identity, nullptr);
-  ASSERT_EQ(identity->input_size(), 1);
-  EXPECT_EQ(identity->input(0), "switch");
+  expected = NDef("", "", {"switch"});
+  CompareNodeInputs(graph, &expected, identity);
   EXPECT_EQ(identity->op(), "Identity");
   EXPECT_EQ(identity->device(), kDevice);
   ASSERT_TRUE(identity->attr().count("T"));
@@ -902,8 +932,72 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithExistingAddedIdentity) {
   ASSERT_EQ(graph.graph()->node_size(), 3);
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
-  ASSERT_EQ(a->input_size(), 1);
-  EXPECT_EQ(a->input(0), "^ConstantFoldingCtrl/switch_0");
+  NodeDef expected = NDef("", "", {"^ConstantFoldingCtrl/switch_0"});
+  CompareNodeInputs(graph, &expected, a);
+}
+
+TEST(MutableGraphViewTest, RemoveControllingFaninMissing) {
+  // Actual node.op() is not important in this test.
+  GraphDef graph_def = test::function::GDef(
+      {
+          NDef("a", "NotImportant", {}, {}),
+          NDef("b", "NotImportant", {}, {}),
+          NDef("c", "NotImportant", {}, {}),
+          NDef("d", "NotImportant", {"^a", "^b"}),
+      },
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_FALSE(graph.RemoveControllingFanin("d", "c"));
+
+  ASSERT_EQ(graph.graph()->node_size(), 4);
+  NodeDef* d = graph.GetNode("d");
+  ASSERT_NE(d, nullptr);
+  NodeDef expected = NDef("", "", {"^a", "^b"});
+  CompareNodeInputs(graph, &expected, d);
+}
+
+TEST(MutableGraphViewTest, RemoveControllingFaninExisting) {
+  // Actual node.op() is not important in this test.
+  GraphDef graph_def = test::function::GDef(
+      {
+          NDef("a", "NotImportant", {}, {}),
+          NDef("b", "NotImportant", {}, {}),
+          NDef("c", "NotImportant", {}, {}),
+          NDef("d", "NotImportant", {"^a", "^b", "^c"}),
+      },
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_TRUE(graph.RemoveControllingFanin("d", "a"));
+  EXPECT_FALSE(graph.RemoveControllingFanin("d", "a"));
+
+  ASSERT_EQ(graph.graph()->node_size(), 4);
+  NodeDef* d = graph.GetNode("d");
+  ASSERT_NE(d, nullptr);
+  NodeDef expected = NDef("", "", {"^c", "^b"});
+  CompareNodeInputs(graph, &expected, d);
+}
+
+TEST(MutableGraphViewTest, RemoveControllingFaninOnRegularFanin) {
+  // Actual node.op() is not important in this test.
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {"a"}),
+       NDef("c", "NotImportant", {"a", "b", "^c"})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_FALSE(graph.RemoveControllingFanin("c", "a"));
+  EXPECT_FALSE(graph.RemoveControllingFanin("c", "b"));
+
+  ASSERT_EQ(graph.graph()->node_size(), 3);
+  NodeDef* c = graph.GetNode("c");
+  ASSERT_NE(c, nullptr);
+  NodeDef expected = NDef("", "", {"a", "b", "^c"});
+  CompareNodeInputs(graph, &expected, c);
 }
 
 TEST(MutableGraphViewTest, DeleteNodes) {
-- 
GitLab


From 80e29253740b251d7632f613ceb414205248ce93 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Thu, 10 Jan 2019 14:06:26 -0800
Subject: [PATCH 0512/2345] [TF:XLA] Bump open source llvm revision to r350820

PiperOrigin-RevId: 228773540
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index a7f3665a31..06caa200e4 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -500,11 +500,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "83a4f199742f3d6892994dd6dc46d6a53019aedaa28590b460c120f3dfc7bc47",
-        strip_prefix = "llvm-671f057a2cf137914be6c786daf8000469adebab",
+        sha256 = "b85f0571c5d2997142cd379eda3def286b26671fabb8453c5d56fee6e0698f52",
+        strip_prefix = "llvm-437f3bdb2355b0f453a4b4824e4b74ed6b6a73a5",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/671f057a2cf137914be6c786daf8000469adebab.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/671f057a2cf137914be6c786daf8000469adebab.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/437f3bdb2355b0f453a4b4824e4b74ed6b6a73a5.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/437f3bdb2355b0f453a4b4824e4b74ed6b6a73a5.tar.gz",
         ],
     )
 
-- 
GitLab


From 462fccab21bc9a6ad0377a0c3998b47291eea279 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Thu, 10 Jan 2019 14:38:16 -0800
Subject: [PATCH 0513/2345] [TF:XLA] Fix failure in sort_ops_test when built in
 -UNDEBUG mode.

The current shape function fails to set a shape for the second output tensor.

PiperOrigin-RevId: 228779527
---
 tensorflow/compiler/tf2xla/ops/xla_ops.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/tf2xla/ops/xla_ops.cc b/tensorflow/compiler/tf2xla/ops/xla_ops.cc
index ab77984684..af641131ed 100644
--- a/tensorflow/compiler/tf2xla/ops/xla_ops.cc
+++ b/tensorflow/compiler/tf2xla/ops/xla_ops.cc
@@ -369,7 +369,11 @@ REGISTER_OP("XlaKeyValueSort")
     .Output("sorted_values: V")
     .Attr("K: realnumbertype")
     .Attr("V: type")
-    .SetShapeFn(shape_inference::UnchangedShape)
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      c->set_output(0, c->input(0));
+      c->set_output(1, c->input(1));
+      return Status::OK();
+    })
     .Doc(R"doc(
 Wraps the XLA Sort operator, documented at
  https://www.tensorflow.org/performance/xla/operation_semantics#sort
-- 
GitLab


From 12c9bb638e8e428bccca0ee8f76c26776f75d5b0 Mon Sep 17 00:00:00 2001
From: Saurabh Deoras <sdeoras@gmail.com>
Date: Thu, 10 Jan 2019 14:49:47 -0800
Subject: [PATCH 0514/2345] use smaller slice length for 32-bit arch

Signed-off-by: Saurabh Deoras <sdeoras@gmail.com>
---
 tensorflow/go/attrs.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/go/attrs.go b/tensorflow/go/attrs.go
index f86c5737bc..ed1a1f0b54 100644
--- a/tensorflow/go/attrs.go
+++ b/tensorflow/go/attrs.go
@@ -170,7 +170,8 @@ func listAttribute(op *Operation, cname *C.char, meta C.TF_AttrMetadata) (interf
 			}
 			// A []C.int64_t slice backed by C memory.
 			// See: https://github.com/golang/go/wiki/cgo#turning-c-arrays-into-go-slices
-			slice := (*[1 << 30]C.int64_t)(unsafe.Pointer(dim))[:numDim:numDim]
+			// Using [1<<27] instead of [1<<30] so it works on 32-bit architecture
+			slice := (*[1 << 27]C.int64_t)(unsafe.Pointer(dim))[:numDim:numDim]
 			list[i] = makeCShape(slice)
 		}
 		return list, nil
-- 
GitLab


From f4fd5aae1d19607497047f5d5ae2d9de3aec4112 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 14:45:30 -0800
Subject: [PATCH 0515/2345] Introduced eager profiler python api.

PiperOrigin-RevId: 228780846
---
 tensorflow/python/BUILD                  |  2 +
 tensorflow/python/eager/BUILD            | 24 +++++++++
 tensorflow/python/eager/profiler.py      | 69 ++++++++++++++++++++++++
 tensorflow/python/eager/profiler_test.py | 44 +++++++++++++++
 tensorflow/python/pywrap_tfe.i           |  2 +
 5 files changed, 141 insertions(+)
 create mode 100644 tensorflow/python/eager/profiler.py
 create mode 100644 tensorflow/python/eager/profiler_test.py

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 933ccf1c7b..c5a8994b80 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -174,6 +174,7 @@ py_library(
         "//tensorflow/python/distribute",
         "//tensorflow/python/distribute:estimator_training",
         "//tensorflow/python/eager:def_function",
+        "//tensorflow/python/eager:profiler",
         "//tensorflow/python/eager:remote",
         "//tensorflow/python/ops/distributions",
         "//tensorflow/python/ops/linalg",
@@ -4117,6 +4118,7 @@ tf_py_wrap_cc(
         "//tensorflow/c:python_api",
         "//tensorflow/c:tf_status_helper",
         "//tensorflow/c/eager:c_api",
+        "//tensorflow/c/eager:c_api_experimental",
         "//tensorflow/core/distributed_runtime/rpc:grpc_rpc_factory_registration",
         "//tensorflow/core/distributed_runtime/rpc:grpc_server_lib",
         "//tensorflow/core/distributed_runtime/rpc:grpc_session",
diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD
index 1f1cb22d58..3a35734051 100644
--- a/tensorflow/python/eager/BUILD
+++ b/tensorflow/python/eager/BUILD
@@ -25,6 +25,7 @@ cc_library(
         "//tensorflow/c:c_api",
         "//tensorflow/c:c_api_internal",
         "//tensorflow/c/eager:c_api",
+        "//tensorflow/c/eager:c_api_experimental",
         "//tensorflow/c/eager:c_api_internal",
         "//tensorflow/c/eager:tape",
         "//tensorflow/core:framework",
@@ -55,6 +56,7 @@ py_library(
         ":execute",
         ":function",
         ":graph_only_ops",
+        ":profiler",
         ":tape",
         ":test",
         ":wrap_function",
@@ -89,6 +91,28 @@ py_library(
     ],
 )
 
+py_library(
+    name = "profiler",
+    srcs = ["profiler.py"],
+    srcs_version = "PY2AND3",
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        ":context",
+        "//tensorflow/python:pywrap_tensorflow",
+    ],
+)
+
+cuda_py_test(
+    name = "profiler_test",
+    srcs = ["profiler_test.py"],
+    additional_deps = [
+        ":profiler",
+        ":test",
+        "//tensorflow/core/profiler:protos_all_py",
+        "//tensorflow/python:constant_op",
+    ],
+)
+
 py_library(
     name = "tape",
     srcs = ["tape.py"],
diff --git a/tensorflow/python/eager/profiler.py b/tensorflow/python/eager/profiler.py
new file mode 100644
index 0000000000..016bca3e3c
--- /dev/null
+++ b/tensorflow/python/eager/profiler.py
@@ -0,0 +1,69 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Profiler for eager mode."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import threading
+
+from tensorflow.python import pywrap_tensorflow
+from tensorflow.python.eager import context
+from tensorflow.python.framework import c_api_util
+
+_profiler = None
+_profiler_lock = threading.Lock()
+
+
+def start():
+  """Start profiling.
+
+  Only one active profiling session is allowed.
+
+  Raises:
+    AssertionError: If another profiling session is running.
+  """
+  global _profiler
+  if _profiler is not None:
+    raise AssertionError('Another profiler is running.')
+  with _profiler_lock:
+    _profiler = pywrap_tensorflow.TFE_NewProfiler(context.context()._handle)  # pylint: disable=protected-access
+
+
+def stop():
+  """Stop current profiling session and return its result.
+
+  Returns:
+    A binary string of tensorflow.tfprof.ProfileProto. User can write the string
+    to file for offline analysis by tfprof command-line tools or graphical user
+    interface.
+
+  Raises:
+    AssertionError: If there is no active profiling session.
+  """
+  global _profiler
+  if _profiler is None:
+    raise AssertionError('Cannot stop profiling. No profiler is running.')
+  with c_api_util.tf_buffer() as buffer_:
+    pywrap_tensorflow.TFE_ProfilerSerializeToString(
+        context.context()._handle,  # pylint: disable=protected-access
+        _profiler,
+        buffer_)
+    result = pywrap_tensorflow.TF_GetBuffer(buffer_)
+  with _profiler_lock:
+    pywrap_tensorflow.TFE_DeleteProfiler(_profiler)
+    _profiler = None
+  return result
diff --git a/tensorflow/python/eager/profiler_test.py b/tensorflow/python/eager/profiler_test.py
new file mode 100644
index 0000000000..b3fe2efabe
--- /dev/null
+++ b/tensorflow/python/eager/profiler_test.py
@@ -0,0 +1,44 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for eager profiler."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.core.profiler import tfprof_log_pb2
+from tensorflow.python.eager import profiler
+from tensorflow.python.eager import test
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import test_util
+
+
+class ProfilerTest(test_util.TensorFlowTestCase):
+
+  def test_profile(self):
+    profiler.start()
+    three = constant_op.constant(3)
+    five = constant_op.constant(5)
+    product = three * five
+    self.assertAllEqual(15, product)
+    profile_result = profiler.stop()
+    profile_pb = tfprof_log_pb2.ProfileProto()
+    profile_pb.ParseFromString(profile_result)
+    profile_pb_str = '%s' % profile_pb
+    self.assertTrue('Mul' in profile_pb_str)
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/python/pywrap_tfe.i b/tensorflow/python/pywrap_tfe.i
index 0620e0345c..d0ba244853 100755
--- a/tensorflow/python/pywrap_tfe.i
+++ b/tensorflow/python/pywrap_tfe.i
@@ -77,6 +77,7 @@ limitations under the License.
 %{
 #include "tensorflow/python/eager/pywrap_tfe.h"
 #include "tensorflow/c/c_api_experimental.h"
+#include "tensorflow/c/eager/c_api_experimental.h"
 %}
 
 %typemap(in) (const void* proto) {
@@ -233,6 +234,7 @@ limitations under the License.
 
 %include "tensorflow/python/eager/pywrap_tfe.h"
 %include "tensorflow/c/c_api_experimental.h"
+%include "tensorflow/c/eager/c_api_experimental.h"
 
 // Clear all typemaps.
 %typemap(out) TF_DataType;
-- 
GitLab


From 4df903f8fff67700567abb68ae46c4cddab67cee Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Thu, 10 Jan 2019 14:51:46 -0800
Subject: [PATCH 0516/2345] Splitting metrics_test file - moving confusion
 matrix related metric tests into a separate file.

PiperOrigin-RevId: 228781953
---
 tensorflow/python/keras/BUILD                 |  15 +-
 .../keras/metrics_confusion_matrix_test.py    | 745 ++++++++++++++++++
 tensorflow/python/keras/metrics_test.py       | 713 -----------------
 3 files changed, 759 insertions(+), 714 deletions(-)
 create mode 100644 tensorflow/python/keras/metrics_confusion_matrix_test.py

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index fb0161a677..536acca298 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -395,7 +395,7 @@ tf_py_test(
 
 tf_py_test(
     name = "metrics_functional_test",
-    size = "medium",
+    size = "small",
     srcs = ["metrics_functional_test.py"],
     additional_deps = [
         ":keras",
@@ -417,6 +417,19 @@ tf_py_test(
     shard_count = 4,
 )
 
+tf_py_test(
+    name = "metrics_confusion_matrix_test",
+    size = "medium",
+    srcs = ["metrics_confusion_matrix_test.py"],
+    additional_deps = [
+        ":keras",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+    ],
+    shard_count = 4,
+)
+
 tf_py_test(
     name = "applications_test",
     size = "enormous",
diff --git a/tensorflow/python/keras/metrics_confusion_matrix_test.py b/tensorflow/python/keras/metrics_confusion_matrix_test.py
new file mode 100644
index 0000000000..37d2880088
--- /dev/null
+++ b/tensorflow/python/keras/metrics_confusion_matrix_test.py
@@ -0,0 +1,745 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for Keras metrics functions."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+import numpy as np
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
+from tensorflow.python.keras import metrics
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import random_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class FalsePositivesTest(test.TestCase):
+
+  def test_config(self):
+    fp_obj = metrics.FalsePositives(name='my_fp', thresholds=[0.4, 0.9])
+    self.assertEqual(fp_obj.name, 'my_fp')
+    self.assertEqual(len(fp_obj.variables), 1)
+    self.assertEqual(fp_obj.thresholds, [0.4, 0.9])
+
+    # Check save and restore config
+    fp_obj2 = metrics.FalsePositives.from_config(fp_obj.get_config())
+    self.assertEqual(fp_obj2.name, 'my_fp')
+    self.assertEqual(len(fp_obj2.variables), 1)
+    self.assertEqual(fp_obj2.thresholds, [0.4, 0.9])
+
+  def test_unweighted(self):
+    fp_obj = metrics.FalsePositives()
+    self.evaluate(variables.variables_initializer(fp_obj.variables))
+
+    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
+                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
+    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
+                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
+
+    update_op = fp_obj.update_state(y_true, y_pred)
+    self.evaluate(update_op)
+    result = fp_obj.result()
+    self.assertAllClose(7., result)
+
+  def test_weighted(self):
+    fp_obj = metrics.FalsePositives()
+    self.evaluate(variables.variables_initializer(fp_obj.variables))
+    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
+                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
+    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
+                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
+    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
+    result = fp_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAllClose(14., self.evaluate(result))
+
+  def test_unweighted_with_thresholds(self):
+    fp_obj = metrics.FalsePositives(thresholds=[0.15, 0.5, 0.85])
+    self.evaluate(variables.variables_initializer(fp_obj.variables))
+
+    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
+                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
+    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
+                                   (1, 1, 1, 1)))
+
+    update_op = fp_obj.update_state(y_true, y_pred)
+    self.evaluate(update_op)
+    result = fp_obj.result()
+    self.assertAllClose([7., 4., 2.], result)
+
+  def test_weighted_with_thresholds(self):
+    fp_obj = metrics.FalsePositives(thresholds=[0.15, 0.5, 0.85])
+    self.evaluate(variables.variables_initializer(fp_obj.variables))
+
+    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
+                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
+    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
+                                   (1, 1, 1, 1)))
+    sample_weight = ((1.0, 2.0, 3.0, 5.0), (7.0, 11.0, 13.0, 17.0),
+                     (19.0, 23.0, 29.0, 31.0), (5.0, 15.0, 10.0, 0))
+
+    result = fp_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAllClose([125., 42., 12.], self.evaluate(result))
+
+  def test_threshold_limit(self):
+    with self.assertRaisesRegexp(
+        ValueError,
+        r'Threshold values must be in \[0, 1\]. Invalid values: \[-1, 2\]'):
+      metrics.FalsePositives(thresholds=[-1, 0.5, 2])
+
+    with self.assertRaisesRegexp(
+        ValueError,
+        r'Threshold values must be in \[0, 1\]. Invalid values: \[None\]'):
+      metrics.FalsePositives(thresholds=[None])
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class FalseNegativesTest(test.TestCase):
+
+  def test_config(self):
+    fn_obj = metrics.FalseNegatives(name='my_fn', thresholds=[0.4, 0.9])
+    self.assertEqual(fn_obj.name, 'my_fn')
+    self.assertEqual(len(fn_obj.variables), 1)
+    self.assertEqual(fn_obj.thresholds, [0.4, 0.9])
+
+    # Check save and restore config
+    fn_obj2 = metrics.FalseNegatives.from_config(fn_obj.get_config())
+    self.assertEqual(fn_obj2.name, 'my_fn')
+    self.assertEqual(len(fn_obj2.variables), 1)
+    self.assertEqual(fn_obj2.thresholds, [0.4, 0.9])
+
+  def test_unweighted(self):
+    fn_obj = metrics.FalseNegatives()
+    self.evaluate(variables.variables_initializer(fn_obj.variables))
+
+    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
+                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
+    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
+                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
+
+    update_op = fn_obj.update_state(y_true, y_pred)
+    self.evaluate(update_op)
+    result = fn_obj.result()
+    self.assertAllClose(3., result)
+
+  def test_weighted(self):
+    fn_obj = metrics.FalseNegatives()
+    self.evaluate(variables.variables_initializer(fn_obj.variables))
+    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
+                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
+    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
+                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
+    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
+    result = fn_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAllClose(5., self.evaluate(result))
+
+  def test_unweighted_with_thresholds(self):
+    fn_obj = metrics.FalseNegatives(thresholds=[0.15, 0.5, 0.85])
+    self.evaluate(variables.variables_initializer(fn_obj.variables))
+
+    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
+                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
+    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
+                                   (1, 1, 1, 1)))
+
+    update_op = fn_obj.update_state(y_true, y_pred)
+    self.evaluate(update_op)
+    result = fn_obj.result()
+    self.assertAllClose([1., 4., 6.], result)
+
+  def test_weighted_with_thresholds(self):
+    fn_obj = metrics.FalseNegatives(thresholds=[0.15, 0.5, 0.85])
+    self.evaluate(variables.variables_initializer(fn_obj.variables))
+
+    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
+                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
+    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
+                                   (1, 1, 1, 1)))
+    sample_weight = ((3.0,), (5.0,), (7.0,), (4.0,))
+
+    result = fn_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAllClose([4., 16., 23.], self.evaluate(result))
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class TrueNegativesTest(test.TestCase):
+
+  def test_config(self):
+    tn_obj = metrics.TrueNegatives(name='my_tn', thresholds=[0.4, 0.9])
+    self.assertEqual(tn_obj.name, 'my_tn')
+    self.assertEqual(len(tn_obj.variables), 1)
+    self.assertEqual(tn_obj.thresholds, [0.4, 0.9])
+
+    # Check save and restore config
+    tn_obj2 = metrics.TrueNegatives.from_config(tn_obj.get_config())
+    self.assertEqual(tn_obj2.name, 'my_tn')
+    self.assertEqual(len(tn_obj2.variables), 1)
+    self.assertEqual(tn_obj2.thresholds, [0.4, 0.9])
+
+  def test_unweighted(self):
+    tn_obj = metrics.TrueNegatives()
+    self.evaluate(variables.variables_initializer(tn_obj.variables))
+
+    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
+                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
+    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
+                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
+
+    update_op = tn_obj.update_state(y_true, y_pred)
+    self.evaluate(update_op)
+    result = tn_obj.result()
+    self.assertAllClose(3., result)
+
+  def test_weighted(self):
+    tn_obj = metrics.TrueNegatives()
+    self.evaluate(variables.variables_initializer(tn_obj.variables))
+    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
+                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
+    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
+                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
+    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
+    result = tn_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAllClose(4., self.evaluate(result))
+
+  def test_unweighted_with_thresholds(self):
+    tn_obj = metrics.TrueNegatives(thresholds=[0.15, 0.5, 0.85])
+    self.evaluate(variables.variables_initializer(tn_obj.variables))
+
+    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
+                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
+    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
+                                   (1, 1, 1, 1)))
+
+    update_op = tn_obj.update_state(y_true, y_pred)
+    self.evaluate(update_op)
+    result = tn_obj.result()
+    self.assertAllClose([2., 5., 7.], result)
+
+  def test_weighted_with_thresholds(self):
+    tn_obj = metrics.TrueNegatives(thresholds=[0.15, 0.5, 0.85])
+    self.evaluate(variables.variables_initializer(tn_obj.variables))
+
+    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
+                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
+    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
+                                   (1, 1, 1, 1)))
+    sample_weight = ((0.0, 2.0, 3.0, 5.0),)
+
+    result = tn_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAllClose([5., 15., 23.], self.evaluate(result))
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class TruePositivesTest(test.TestCase):
+
+  def test_config(self):
+    tp_obj = metrics.TruePositives(name='my_tp', thresholds=[0.4, 0.9])
+    self.assertEqual(tp_obj.name, 'my_tp')
+    self.assertEqual(len(tp_obj.variables), 1)
+    self.assertEqual(tp_obj.thresholds, [0.4, 0.9])
+
+    # Check save and restore config
+    tp_obj2 = metrics.TruePositives.from_config(tp_obj.get_config())
+    self.assertEqual(tp_obj2.name, 'my_tp')
+    self.assertEqual(len(tp_obj2.variables), 1)
+    self.assertEqual(tp_obj2.thresholds, [0.4, 0.9])
+
+  def test_unweighted(self):
+    tp_obj = metrics.TruePositives()
+    self.evaluate(variables.variables_initializer(tp_obj.variables))
+
+    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
+                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
+    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
+                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
+
+    update_op = tp_obj.update_state(y_true, y_pred)
+    self.evaluate(update_op)
+    result = tp_obj.result()
+    self.assertAllClose(7., result)
+
+  def test_weighted(self):
+    tp_obj = metrics.TruePositives()
+    self.evaluate(variables.variables_initializer(tp_obj.variables))
+    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
+                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
+    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
+                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
+    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
+    result = tp_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAllClose(12., self.evaluate(result))
+
+  def test_unweighted_with_thresholds(self):
+    tp_obj = metrics.TruePositives(thresholds=[0.15, 0.5, 0.85])
+    self.evaluate(variables.variables_initializer(tp_obj.variables))
+
+    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
+                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
+    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
+                                   (1, 1, 1, 1)))
+
+    update_op = tp_obj.update_state(y_true, y_pred)
+    self.evaluate(update_op)
+    result = tp_obj.result()
+    self.assertAllClose([6., 3., 1.], result)
+
+  def test_weighted_with_thresholds(self):
+    tp_obj = metrics.TruePositives(thresholds=[0.15, 0.5, 0.85])
+    self.evaluate(variables.variables_initializer(tp_obj.variables))
+
+    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
+                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
+    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
+                                   (1, 1, 1, 1)))
+
+    result = tp_obj(y_true, y_pred, sample_weight=37.)
+    self.assertAllClose([222., 111., 37.], self.evaluate(result))
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class PrecisionTest(test.TestCase):
+
+  def test_config(self):
+    p_obj = metrics.Precision(name='my_precision', thresholds=[0.4, 0.9])
+    self.assertEqual(p_obj.name, 'my_precision')
+    self.assertEqual(len(p_obj.variables), 2)
+    self.assertEqual([v.name for v in p_obj.variables],
+                     ['true_positives:0', 'false_positives:0'])
+    self.assertEqual(p_obj.thresholds, [0.4, 0.9])
+
+    # Check save and restore config
+    p_obj2 = metrics.Precision.from_config(p_obj.get_config())
+    self.assertEqual(p_obj2.name, 'my_precision')
+    self.assertEqual(len(p_obj2.variables), 2)
+    self.assertEqual(p_obj2.thresholds, [0.4, 0.9])
+
+  def test_value_is_idempotent(self):
+    p_obj = metrics.Precision(thresholds=[0.3, 0.72])
+    y_pred = random_ops.random_uniform(shape=(10, 3))
+    y_true = random_ops.random_uniform(shape=(10, 3))
+    update_op = p_obj.update_state(y_true, y_pred)
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+
+    # Run several updates.
+    for _ in range(10):
+      self.evaluate(update_op)
+
+    # Then verify idempotency.
+    initial_precision = self.evaluate(p_obj.result())
+    for _ in range(10):
+      self.assertArrayNear(initial_precision, self.evaluate(p_obj.result()),
+                           1e-3)
+
+  def test_unweighted(self):
+    p_obj = metrics.Precision()
+    y_pred = constant_op.constant([1, 0, 1, 0], shape=(1, 4))
+    y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+    result = p_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.5, self.evaluate(result))
+
+  def test_unweighted_all_incorrect(self):
+    p_obj = metrics.Precision(thresholds=[0.5])
+    inputs = np.random.randint(0, 2, size=(100, 1))
+    y_pred = constant_op.constant(inputs)
+    y_true = constant_op.constant(1 - inputs)
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+    result = p_obj(y_true, y_pred)
+    self.assertAlmostEqual(0, self.evaluate(result))
+
+  def test_weighted(self):
+    p_obj = metrics.Precision()
+    y_pred = constant_op.constant([[1, 0, 1, 0], [1, 0, 1, 0]])
+    y_true = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+    result = p_obj(
+        y_true,
+        y_pred,
+        sample_weight=constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]]))
+    weighted_tp = 3.0 + 4.0
+    weighted_positives = (1.0 + 3.0) + (4.0 + 2.0)
+    expected_precision = weighted_tp / weighted_positives
+    self.assertAlmostEqual(expected_precision, self.evaluate(result))
+
+  def test_div_by_zero(self):
+    p_obj = metrics.Precision()
+    y_pred = constant_op.constant([0, 0, 0, 0])
+    y_true = constant_op.constant([0, 0, 0, 0])
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+    result = p_obj(y_true, y_pred)
+    self.assertEqual(0, self.evaluate(result))
+
+  def test_unweighted_with_threshold(self):
+    p_obj = metrics.Precision(thresholds=[0.5, 0.7])
+    y_pred = constant_op.constant([1, 0, 0.6, 0], shape=(1, 4))
+    y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+    result = p_obj(y_true, y_pred)
+    self.assertArrayNear([0.5, 0.], self.evaluate(result), 0)
+
+  def test_weighted_with_threshold(self):
+    p_obj = metrics.Precision(thresholds=[0.5, 1.])
+    y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
+    y_pred = constant_op.constant([[1, 0], [0.6, 0]],
+                                  shape=(2, 2),
+                                  dtype=dtypes.float32)
+    weights = constant_op.constant([[4, 0], [3, 1]],
+                                   shape=(2, 2),
+                                   dtype=dtypes.float32)
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+    result = p_obj(y_true, y_pred, sample_weight=weights)
+    weighted_tp = 0 + 3.
+    weighted_positives = (0 + 3.) + (4. + 0.)
+    expected_precision = weighted_tp / weighted_positives
+    self.assertArrayNear([expected_precision, 0], self.evaluate(result), 1e-3)
+
+  def test_multiple_updates(self):
+    p_obj = metrics.Precision(thresholds=[0.5, 1.])
+    y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
+    y_pred = constant_op.constant([[1, 0], [0.6, 0]],
+                                  shape=(2, 2),
+                                  dtype=dtypes.float32)
+    weights = constant_op.constant([[4, 0], [3, 1]],
+                                   shape=(2, 2),
+                                   dtype=dtypes.float32)
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+    update_op = p_obj.update_state(y_true, y_pred, sample_weight=weights)
+    for _ in range(2):
+      self.evaluate(update_op)
+
+    weighted_tp = (0 + 3.) + (0 + 3.)
+    weighted_positives = ((0 + 3.) + (4. + 0.)) + ((0 + 3.) + (4. + 0.))
+    expected_precision = weighted_tp / weighted_positives
+    self.assertArrayNear([expected_precision, 0], self.evaluate(p_obj.result()),
+                         1e-3)
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class RecallTest(test.TestCase):
+
+  def test_config(self):
+    r_obj = metrics.Recall(name='my_recall', thresholds=[0.4, 0.9])
+    self.assertEqual(r_obj.name, 'my_recall')
+    self.assertEqual(len(r_obj.variables), 2)
+    self.assertEqual([v.name for v in r_obj.variables],
+                     ['true_positives:0', 'false_negatives:0'])
+    self.assertEqual(r_obj.thresholds, [0.4, 0.9])
+
+    # Check save and restore config
+    r_obj2 = metrics.Recall.from_config(r_obj.get_config())
+    self.assertEqual(r_obj2.name, 'my_recall')
+    self.assertEqual(len(r_obj2.variables), 2)
+    self.assertEqual(r_obj2.thresholds, [0.4, 0.9])
+
+  def test_value_is_idempotent(self):
+    r_obj = metrics.Recall(thresholds=[0.3, 0.72])
+    y_pred = random_ops.random_uniform(shape=(10, 3))
+    y_true = random_ops.random_uniform(shape=(10, 3))
+    update_op = r_obj.update_state(y_true, y_pred)
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+
+    # Run several updates.
+    for _ in range(10):
+      self.evaluate(update_op)
+
+    # Then verify idempotency.
+    initial_recall = self.evaluate(r_obj.result())
+    for _ in range(10):
+      self.assertArrayNear(initial_recall, self.evaluate(r_obj.result()), 1e-3)
+
+  def test_unweighted(self):
+    r_obj = metrics.Recall()
+    y_pred = constant_op.constant([1, 0, 1, 0], shape=(1, 4))
+    y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+    result = r_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.5, self.evaluate(result))
+
+  def test_unweighted_all_incorrect(self):
+    r_obj = metrics.Recall(thresholds=[0.5])
+    inputs = np.random.randint(0, 2, size=(100, 1))
+    y_pred = constant_op.constant(inputs)
+    y_true = constant_op.constant(1 - inputs)
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+    result = r_obj(y_true, y_pred)
+    self.assertAlmostEqual(0, self.evaluate(result))
+
+  def test_weighted(self):
+    r_obj = metrics.Recall()
+    y_pred = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]])
+    y_true = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+    result = r_obj(
+        y_true,
+        y_pred,
+        sample_weight=constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]]))
+    weighted_tp = 3.0 + 1.0
+    weighted_t = (2.0 + 3.0) + (4.0 + 1.0)
+    expected_recall = weighted_tp / weighted_t
+    self.assertAlmostEqual(expected_recall, self.evaluate(result))
+
+  def test_div_by_zero(self):
+    r_obj = metrics.Recall()
+    y_pred = constant_op.constant([0, 0, 0, 0])
+    y_true = constant_op.constant([0, 0, 0, 0])
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+    result = r_obj(y_true, y_pred)
+    self.assertEqual(0, self.evaluate(result))
+
+  def test_unweighted_with_threshold(self):
+    r_obj = metrics.Recall(thresholds=[0.5, 0.7])
+    y_pred = constant_op.constant([1, 0, 0.6, 0], shape=(1, 4))
+    y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+    result = r_obj(y_true, y_pred)
+    self.assertArrayNear([0.5, 0.], self.evaluate(result), 0)
+
+  def test_weighted_with_threshold(self):
+    r_obj = metrics.Recall(thresholds=[0.5, 1.])
+    y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
+    y_pred = constant_op.constant([[1, 0], [0.6, 0]],
+                                  shape=(2, 2),
+                                  dtype=dtypes.float32)
+    weights = constant_op.constant([[1, 4], [3, 2]],
+                                   shape=(2, 2),
+                                   dtype=dtypes.float32)
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+    result = r_obj(y_true, y_pred, sample_weight=weights)
+    weighted_tp = 0 + 3.
+    weighted_positives = (0 + 3.) + (4. + 0.)
+    expected_recall = weighted_tp / weighted_positives
+    self.assertArrayNear([expected_recall, 0], self.evaluate(result), 1e-3)
+
+  def test_multiple_updates(self):
+    r_obj = metrics.Recall(thresholds=[0.5, 1.])
+    y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
+    y_pred = constant_op.constant([[1, 0], [0.6, 0]],
+                                  shape=(2, 2),
+                                  dtype=dtypes.float32)
+    weights = constant_op.constant([[1, 4], [3, 2]],
+                                   shape=(2, 2),
+                                   dtype=dtypes.float32)
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+    update_op = r_obj.update_state(y_true, y_pred, sample_weight=weights)
+    for _ in range(2):
+      self.evaluate(update_op)
+
+    weighted_tp = (0 + 3.) + (0 + 3.)
+    weighted_positives = ((0 + 3.) + (4. + 0.)) + ((0 + 3.) + (4. + 0.))
+    expected_recall = weighted_tp / weighted_positives
+    self.assertArrayNear([expected_recall, 0], self.evaluate(r_obj.result()),
+                         1e-3)
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class SensitivityAtSpecificityTest(test.TestCase, parameterized.TestCase):
+
+  def test_config(self):
+    s_obj = metrics.SensitivityAtSpecificity(
+        0.4, num_thresholds=100, name='sensitivity_at_specificity_1')
+    self.assertEqual(s_obj.name, 'sensitivity_at_specificity_1')
+    self.assertLen(s_obj.variables, 4)
+    self.assertEqual(s_obj.specificity, 0.4)
+    self.assertEqual(s_obj.num_thresholds, 100)
+
+    # Check save and restore config
+    s_obj2 = metrics.SensitivityAtSpecificity.from_config(s_obj.get_config())
+    self.assertEqual(s_obj2.name, 'sensitivity_at_specificity_1')
+    self.assertLen(s_obj2.variables, 4)
+    self.assertEqual(s_obj2.specificity, 0.4)
+    self.assertEqual(s_obj2.num_thresholds, 100)
+
+  def test_value_is_idempotent(self):
+    s_obj = metrics.SensitivityAtSpecificity(0.7)
+    y_pred = random_ops.random_uniform((10, 3),
+                                       maxval=1,
+                                       dtype=dtypes.float32,
+                                       seed=1)
+    y_true = random_ops.random_uniform((10, 3),
+                                       maxval=2,
+                                       dtype=dtypes.int64,
+                                       seed=1)
+    update_op = s_obj.update_state(y_true, y_pred)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+
+    # Run several updates.
+    for _ in range(10):
+      self.evaluate(update_op)
+
+    # Then verify idempotency.
+    initial_sensitivity = self.evaluate(s_obj.result())
+    for _ in range(10):
+      self.assertAlmostEqual(initial_sensitivity, self.evaluate(s_obj.result()),
+                             1e-3)
+
+  def test_unweighted_all_correct(self):
+    s_obj = metrics.SensitivityAtSpecificity(0.7)
+    inputs = np.random.randint(0, 2, size=(100, 1))
+    y_pred = constant_op.constant(inputs, dtype=dtypes.float32)
+    y_true = constant_op.constant(inputs)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+    result = s_obj(y_true, y_pred)
+    self.assertAlmostEqual(1, self.evaluate(result))
+
+  def test_unweighted_high_specificity(self):
+    s_obj = metrics.SensitivityAtSpecificity(0.8)
+    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.1, 0.45, 0.5, 0.8, 0.9]
+    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
+
+    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
+    y_true = constant_op.constant(label_values)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+    result = s_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.8, self.evaluate(result))
+
+  def test_unweighted_low_specificity(self):
+    s_obj = metrics.SensitivityAtSpecificity(0.4)
+    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26]
+    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
+
+    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
+    y_true = constant_op.constant(label_values)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+    result = s_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.6, self.evaluate(result))
+
+  @parameterized.parameters([dtypes.bool, dtypes.int32, dtypes.float32])
+  def test_weighted(self, label_dtype):
+    s_obj = metrics.SensitivityAtSpecificity(0.4)
+    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26]
+    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
+    weight_values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+
+    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
+    y_true = math_ops.cast(label_values, dtype=label_dtype)
+    weights = constant_op.constant(weight_values)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+    result = s_obj(y_true, y_pred, sample_weight=weights)
+    self.assertAlmostEqual(0.675, self.evaluate(result))
+
+  def test_invalid_specificity(self):
+    with self.assertRaisesRegexp(
+        ValueError, r'`specificity` must be in the range \[0, 1\].'):
+      metrics.SensitivityAtSpecificity(-1)
+
+  def test_invalid_num_thresholds(self):
+    with self.assertRaisesRegexp(ValueError, '`num_thresholds` must be > 0.'):
+      metrics.SensitivityAtSpecificity(0.4, num_thresholds=-1)
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class SpecificityAtSensitivityTest(test.TestCase, parameterized.TestCase):
+
+  def test_config(self):
+    s_obj = metrics.SpecificityAtSensitivity(
+        0.4, num_thresholds=100, name='specificity_at_sensitivity_1')
+    self.assertEqual(s_obj.name, 'specificity_at_sensitivity_1')
+    self.assertLen(s_obj.variables, 4)
+    self.assertEqual(s_obj.sensitivity, 0.4)
+    self.assertEqual(s_obj.num_thresholds, 100)
+
+    # Check save and restore config
+    s_obj2 = metrics.SpecificityAtSensitivity.from_config(s_obj.get_config())
+    self.assertEqual(s_obj2.name, 'specificity_at_sensitivity_1')
+    self.assertLen(s_obj2.variables, 4)
+    self.assertEqual(s_obj2.sensitivity, 0.4)
+    self.assertEqual(s_obj2.num_thresholds, 100)
+
+  def test_value_is_idempotent(self):
+    s_obj = metrics.SpecificityAtSensitivity(0.7)
+    y_pred = random_ops.random_uniform((10, 3),
+                                       maxval=1,
+                                       dtype=dtypes.float32,
+                                       seed=1)
+    y_true = random_ops.random_uniform((10, 3),
+                                       maxval=2,
+                                       dtype=dtypes.int64,
+                                       seed=1)
+    update_op = s_obj.update_state(y_true, y_pred)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+
+    # Run several updates.
+    for _ in range(10):
+      self.evaluate(update_op)
+
+    # Then verify idempotency.
+    initial_specificity = self.evaluate(s_obj.result())
+    for _ in range(10):
+      self.assertAlmostEqual(initial_specificity, self.evaluate(s_obj.result()),
+                             1e-3)
+
+  def test_unweighted_all_correct(self):
+    s_obj = metrics.SpecificityAtSensitivity(0.7)
+    inputs = np.random.randint(0, 2, size=(100, 1))
+    y_pred = constant_op.constant(inputs, dtype=dtypes.float32)
+    y_true = constant_op.constant(inputs)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+    result = s_obj(y_true, y_pred)
+    self.assertAlmostEqual(1, self.evaluate(result))
+
+  def test_unweighted_high_sensitivity(self):
+    s_obj = metrics.SpecificityAtSensitivity(0.8)
+    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.1, 0.45, 0.5, 0.8, 0.9]
+    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
+
+    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
+    y_true = constant_op.constant(label_values)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+    result = s_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.4, self.evaluate(result))
+
+  def test_unweighted_low_sensitivity(self):
+    s_obj = metrics.SpecificityAtSensitivity(0.4)
+    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26]
+    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
+
+    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
+    y_true = constant_op.constant(label_values)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+    result = s_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.6, self.evaluate(result))
+
+  @parameterized.parameters([dtypes.bool, dtypes.int32, dtypes.float32])
+  def test_weighted(self, label_dtype):
+    s_obj = metrics.SpecificityAtSensitivity(0.4)
+    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26]
+    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
+    weight_values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+
+    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
+    y_true = math_ops.cast(label_values, dtype=label_dtype)
+    weights = constant_op.constant(weight_values)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+    result = s_obj(y_true, y_pred, sample_weight=weights)
+    self.assertAlmostEqual(0.4, self.evaluate(result))
+
+  def test_invalid_sensitivity(self):
+    with self.assertRaisesRegexp(
+        ValueError, r'`sensitivity` must be in the range \[0, 1\].'):
+      metrics.SpecificityAtSensitivity(-1)
+
+  def test_invalid_num_thresholds(self):
+    with self.assertRaisesRegexp(ValueError, '`num_thresholds` must be > 0.'):
+      metrics.SpecificityAtSensitivity(0.4, num_thresholds=-1)
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index 42da1dfb99..f902838c11 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -20,7 +20,6 @@ from __future__ import print_function
 
 import math
 import os
-from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.python.eager import context
@@ -33,8 +32,6 @@ from tensorflow.python.keras import layers
 from tensorflow.python.keras import metrics
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 from tensorflow.python.training.checkpointable import util as checkpointable_utils
@@ -336,716 +333,6 @@ class KerasAccuracyTest(test.TestCase):
       self.assertAlmostEqual(result, 0.71, 2)  # 2.5/2.7
 
 
-@test_util.run_all_in_graph_and_eager_modes
-class FalsePositivesTest(test.TestCase):
-
-  def test_config(self):
-    fp_obj = metrics.FalsePositives(name='my_fp', thresholds=[0.4, 0.9])
-    self.assertEqual(fp_obj.name, 'my_fp')
-    self.assertEqual(len(fp_obj.variables), 1)
-    self.assertEqual(fp_obj.thresholds, [0.4, 0.9])
-
-    # Check save and restore config
-    fp_obj2 = metrics.FalsePositives.from_config(fp_obj.get_config())
-    self.assertEqual(fp_obj2.name, 'my_fp')
-    self.assertEqual(len(fp_obj2.variables), 1)
-    self.assertEqual(fp_obj2.thresholds, [0.4, 0.9])
-
-  def test_unweighted(self):
-    fp_obj = metrics.FalsePositives()
-    self.evaluate(variables.variables_initializer(fp_obj.variables))
-
-    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
-                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
-    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
-                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
-
-    update_op = fp_obj.update_state(y_true, y_pred)
-    self.evaluate(update_op)
-    result = fp_obj.result()
-    self.assertAllClose(7., result)
-
-  def test_weighted(self):
-    fp_obj = metrics.FalsePositives()
-    self.evaluate(variables.variables_initializer(fp_obj.variables))
-    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
-                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
-    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
-                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
-    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
-    result = fp_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAllClose(14., self.evaluate(result))
-
-  def test_unweighted_with_thresholds(self):
-    fp_obj = metrics.FalsePositives(thresholds=[0.15, 0.5, 0.85])
-    self.evaluate(variables.variables_initializer(fp_obj.variables))
-
-    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
-                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
-    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
-                                   (1, 1, 1, 1)))
-
-    update_op = fp_obj.update_state(y_true, y_pred)
-    self.evaluate(update_op)
-    result = fp_obj.result()
-    self.assertAllClose([7., 4., 2.], result)
-
-  def test_weighted_with_thresholds(self):
-    fp_obj = metrics.FalsePositives(thresholds=[0.15, 0.5, 0.85])
-    self.evaluate(variables.variables_initializer(fp_obj.variables))
-
-    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
-                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
-    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
-                                   (1, 1, 1, 1)))
-    sample_weight = ((1.0, 2.0, 3.0, 5.0), (7.0, 11.0, 13.0, 17.0),
-                     (19.0, 23.0, 29.0, 31.0), (5.0, 15.0, 10.0, 0))
-
-    result = fp_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAllClose([125., 42., 12.], self.evaluate(result))
-
-  def test_threshold_limit(self):
-    with self.assertRaisesRegexp(
-        ValueError,
-        r'Threshold values must be in \[0, 1\]. Invalid values: \[-1, 2\]'):
-      metrics.FalsePositives(thresholds=[-1, 0.5, 2])
-
-    with self.assertRaisesRegexp(
-        ValueError,
-        r'Threshold values must be in \[0, 1\]. Invalid values: \[None\]'):
-      metrics.FalsePositives(thresholds=[None])
-
-
-@test_util.run_all_in_graph_and_eager_modes
-class FalseNegativesTest(test.TestCase):
-
-  def test_config(self):
-    fn_obj = metrics.FalseNegatives(name='my_fn', thresholds=[0.4, 0.9])
-    self.assertEqual(fn_obj.name, 'my_fn')
-    self.assertEqual(len(fn_obj.variables), 1)
-    self.assertEqual(fn_obj.thresholds, [0.4, 0.9])
-
-    # Check save and restore config
-    fn_obj2 = metrics.FalseNegatives.from_config(fn_obj.get_config())
-    self.assertEqual(fn_obj2.name, 'my_fn')
-    self.assertEqual(len(fn_obj2.variables), 1)
-    self.assertEqual(fn_obj2.thresholds, [0.4, 0.9])
-
-  def test_unweighted(self):
-    fn_obj = metrics.FalseNegatives()
-    self.evaluate(variables.variables_initializer(fn_obj.variables))
-
-    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
-                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
-    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
-                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
-
-    update_op = fn_obj.update_state(y_true, y_pred)
-    self.evaluate(update_op)
-    result = fn_obj.result()
-    self.assertAllClose(3., result)
-
-  def test_weighted(self):
-    fn_obj = metrics.FalseNegatives()
-    self.evaluate(variables.variables_initializer(fn_obj.variables))
-    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
-                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
-    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
-                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
-    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
-    result = fn_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAllClose(5., self.evaluate(result))
-
-  def test_unweighted_with_thresholds(self):
-    fn_obj = metrics.FalseNegatives(thresholds=[0.15, 0.5, 0.85])
-    self.evaluate(variables.variables_initializer(fn_obj.variables))
-
-    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
-                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
-    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
-                                   (1, 1, 1, 1)))
-
-    update_op = fn_obj.update_state(y_true, y_pred)
-    self.evaluate(update_op)
-    result = fn_obj.result()
-    self.assertAllClose([1., 4., 6.], result)
-
-  def test_weighted_with_thresholds(self):
-    fn_obj = metrics.FalseNegatives(thresholds=[0.15, 0.5, 0.85])
-    self.evaluate(variables.variables_initializer(fn_obj.variables))
-
-    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
-                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
-    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
-                                   (1, 1, 1, 1)))
-    sample_weight = ((3.0,), (5.0,), (7.0,), (4.0,))
-
-    result = fn_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAllClose([4., 16., 23.], self.evaluate(result))
-
-
-@test_util.run_all_in_graph_and_eager_modes
-class TrueNegativesTest(test.TestCase):
-
-  def test_config(self):
-    tn_obj = metrics.TrueNegatives(name='my_tn', thresholds=[0.4, 0.9])
-    self.assertEqual(tn_obj.name, 'my_tn')
-    self.assertEqual(len(tn_obj.variables), 1)
-    self.assertEqual(tn_obj.thresholds, [0.4, 0.9])
-
-    # Check save and restore config
-    tn_obj2 = metrics.TrueNegatives.from_config(tn_obj.get_config())
-    self.assertEqual(tn_obj2.name, 'my_tn')
-    self.assertEqual(len(tn_obj2.variables), 1)
-    self.assertEqual(tn_obj2.thresholds, [0.4, 0.9])
-
-  def test_unweighted(self):
-    tn_obj = metrics.TrueNegatives()
-    self.evaluate(variables.variables_initializer(tn_obj.variables))
-
-    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
-                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
-    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
-                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
-
-    update_op = tn_obj.update_state(y_true, y_pred)
-    self.evaluate(update_op)
-    result = tn_obj.result()
-    self.assertAllClose(3., result)
-
-  def test_weighted(self):
-    tn_obj = metrics.TrueNegatives()
-    self.evaluate(variables.variables_initializer(tn_obj.variables))
-    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
-                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
-    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
-                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
-    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
-    result = tn_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAllClose(4., self.evaluate(result))
-
-  def test_unweighted_with_thresholds(self):
-    tn_obj = metrics.TrueNegatives(thresholds=[0.15, 0.5, 0.85])
-    self.evaluate(variables.variables_initializer(tn_obj.variables))
-
-    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
-                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
-    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
-                                   (1, 1, 1, 1)))
-
-    update_op = tn_obj.update_state(y_true, y_pred)
-    self.evaluate(update_op)
-    result = tn_obj.result()
-    self.assertAllClose([2., 5., 7.], result)
-
-  def test_weighted_with_thresholds(self):
-    tn_obj = metrics.TrueNegatives(thresholds=[0.15, 0.5, 0.85])
-    self.evaluate(variables.variables_initializer(tn_obj.variables))
-
-    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
-                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
-    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
-                                   (1, 1, 1, 1)))
-    sample_weight = ((0.0, 2.0, 3.0, 5.0),)
-
-    result = tn_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAllClose([5., 15., 23.], self.evaluate(result))
-
-
-@test_util.run_all_in_graph_and_eager_modes
-class TruePositivesTest(test.TestCase):
-
-  def test_config(self):
-    tp_obj = metrics.TruePositives(name='my_tp', thresholds=[0.4, 0.9])
-    self.assertEqual(tp_obj.name, 'my_tp')
-    self.assertEqual(len(tp_obj.variables), 1)
-    self.assertEqual(tp_obj.thresholds, [0.4, 0.9])
-
-    # Check save and restore config
-    tp_obj2 = metrics.TruePositives.from_config(tp_obj.get_config())
-    self.assertEqual(tp_obj2.name, 'my_tp')
-    self.assertEqual(len(tp_obj2.variables), 1)
-    self.assertEqual(tp_obj2.thresholds, [0.4, 0.9])
-
-  def test_unweighted(self):
-    tp_obj = metrics.TruePositives()
-    self.evaluate(variables.variables_initializer(tp_obj.variables))
-
-    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
-                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
-    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
-                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
-
-    update_op = tp_obj.update_state(y_true, y_pred)
-    self.evaluate(update_op)
-    result = tp_obj.result()
-    self.assertAllClose(7., result)
-
-  def test_weighted(self):
-    tp_obj = metrics.TruePositives()
-    self.evaluate(variables.variables_initializer(tp_obj.variables))
-    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
-                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
-    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
-                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
-    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
-    result = tp_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAllClose(12., self.evaluate(result))
-
-  def test_unweighted_with_thresholds(self):
-    tp_obj = metrics.TruePositives(thresholds=[0.15, 0.5, 0.85])
-    self.evaluate(variables.variables_initializer(tp_obj.variables))
-
-    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
-                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
-    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
-                                   (1, 1, 1, 1)))
-
-    update_op = tp_obj.update_state(y_true, y_pred)
-    self.evaluate(update_op)
-    result = tp_obj.result()
-    self.assertAllClose([6., 3., 1.], result)
-
-  def test_weighted_with_thresholds(self):
-    tp_obj = metrics.TruePositives(thresholds=[0.15, 0.5, 0.85])
-    self.evaluate(variables.variables_initializer(tp_obj.variables))
-
-    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
-                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
-    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
-                                   (1, 1, 1, 1)))
-
-    result = tp_obj(y_true, y_pred, sample_weight=37.)
-    self.assertAllClose([222., 111., 37.], self.evaluate(result))
-
-
-@test_util.run_all_in_graph_and_eager_modes
-class PrecisionTest(test.TestCase):
-
-  def test_config(self):
-    p_obj = metrics.Precision(name='my_precision', thresholds=[0.4, 0.9])
-    self.assertEqual(p_obj.name, 'my_precision')
-    self.assertEqual(len(p_obj.variables), 2)
-    self.assertEqual([v.name for v in p_obj.variables],
-                     ['true_positives:0', 'false_positives:0'])
-    self.assertEqual(p_obj.thresholds, [0.4, 0.9])
-
-    # Check save and restore config
-    p_obj2 = metrics.Precision.from_config(p_obj.get_config())
-    self.assertEqual(p_obj2.name, 'my_precision')
-    self.assertEqual(len(p_obj2.variables), 2)
-    self.assertEqual(p_obj2.thresholds, [0.4, 0.9])
-
-  def test_value_is_idempotent(self):
-    p_obj = metrics.Precision(thresholds=[0.3, 0.72])
-    y_pred = random_ops.random_uniform(shape=(10, 3))
-    y_true = random_ops.random_uniform(shape=(10, 3))
-    update_op = p_obj.update_state(y_true, y_pred)
-    self.evaluate(variables.variables_initializer(p_obj.variables))
-
-    # Run several updates.
-    for _ in range(10):
-      self.evaluate(update_op)
-
-    # Then verify idempotency.
-    initial_precision = self.evaluate(p_obj.result())
-    for _ in range(10):
-      self.assertArrayNear(initial_precision, self.evaluate(p_obj.result()),
-                           1e-3)
-
-  def test_unweighted(self):
-    p_obj = metrics.Precision()
-    y_pred = constant_op.constant([1, 0, 1, 0], shape=(1, 4))
-    y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
-    self.evaluate(variables.variables_initializer(p_obj.variables))
-    result = p_obj(y_true, y_pred)
-    self.assertAlmostEqual(0.5, self.evaluate(result))
-
-  def test_unweighted_all_incorrect(self):
-    p_obj = metrics.Precision(thresholds=[0.5])
-    inputs = np.random.randint(0, 2, size=(100, 1))
-    y_pred = constant_op.constant(inputs)
-    y_true = constant_op.constant(1 - inputs)
-    self.evaluate(variables.variables_initializer(p_obj.variables))
-    result = p_obj(y_true, y_pred)
-    self.assertAlmostEqual(0, self.evaluate(result))
-
-  def test_weighted(self):
-    p_obj = metrics.Precision()
-    y_pred = constant_op.constant([[1, 0, 1, 0], [1, 0, 1, 0]])
-    y_true = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])
-    self.evaluate(variables.variables_initializer(p_obj.variables))
-    result = p_obj(
-        y_true,
-        y_pred,
-        sample_weight=constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]]))
-    weighted_tp = 3.0 + 4.0
-    weighted_positives = (1.0 + 3.0) + (4.0 + 2.0)
-    expected_precision = weighted_tp / weighted_positives
-    self.assertAlmostEqual(expected_precision, self.evaluate(result))
-
-  def test_div_by_zero(self):
-    p_obj = metrics.Precision()
-    y_pred = constant_op.constant([0, 0, 0, 0])
-    y_true = constant_op.constant([0, 0, 0, 0])
-    self.evaluate(variables.variables_initializer(p_obj.variables))
-    result = p_obj(y_true, y_pred)
-    self.assertEqual(0, self.evaluate(result))
-
-  def test_unweighted_with_threshold(self):
-    p_obj = metrics.Precision(thresholds=[0.5, 0.7])
-    y_pred = constant_op.constant([1, 0, 0.6, 0], shape=(1, 4))
-    y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
-    self.evaluate(variables.variables_initializer(p_obj.variables))
-    result = p_obj(y_true, y_pred)
-    self.assertArrayNear([0.5, 0.], self.evaluate(result), 0)
-
-  def test_weighted_with_threshold(self):
-    p_obj = metrics.Precision(thresholds=[0.5, 1.])
-    y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
-    y_pred = constant_op.constant([[1, 0], [0.6, 0]],
-                                  shape=(2, 2),
-                                  dtype=dtypes.float32)
-    weights = constant_op.constant([[4, 0], [3, 1]],
-                                   shape=(2, 2),
-                                   dtype=dtypes.float32)
-    self.evaluate(variables.variables_initializer(p_obj.variables))
-    result = p_obj(y_true, y_pred, sample_weight=weights)
-    weighted_tp = 0 + 3.
-    weighted_positives = (0 + 3.) + (4. + 0.)
-    expected_precision = weighted_tp / weighted_positives
-    self.assertArrayNear([expected_precision, 0], self.evaluate(result), 1e-3)
-
-  def test_multiple_updates(self):
-    p_obj = metrics.Precision(thresholds=[0.5, 1.])
-    y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
-    y_pred = constant_op.constant([[1, 0], [0.6, 0]],
-                                  shape=(2, 2),
-                                  dtype=dtypes.float32)
-    weights = constant_op.constant([[4, 0], [3, 1]],
-                                   shape=(2, 2),
-                                   dtype=dtypes.float32)
-    self.evaluate(variables.variables_initializer(p_obj.variables))
-    update_op = p_obj.update_state(y_true, y_pred, sample_weight=weights)
-    for _ in range(2):
-      self.evaluate(update_op)
-
-    weighted_tp = (0 + 3.) + (0 + 3.)
-    weighted_positives = ((0 + 3.) + (4. + 0.)) + ((0 + 3.) + (4. + 0.))
-    expected_precision = weighted_tp / weighted_positives
-    self.assertArrayNear([expected_precision, 0], self.evaluate(p_obj.result()),
-                         1e-3)
-
-
-@test_util.run_all_in_graph_and_eager_modes
-class RecallTest(test.TestCase):
-
-  def test_config(self):
-    r_obj = metrics.Recall(name='my_recall', thresholds=[0.4, 0.9])
-    self.assertEqual(r_obj.name, 'my_recall')
-    self.assertEqual(len(r_obj.variables), 2)
-    self.assertEqual([v.name for v in r_obj.variables],
-                     ['true_positives:0', 'false_negatives:0'])
-    self.assertEqual(r_obj.thresholds, [0.4, 0.9])
-
-    # Check save and restore config
-    r_obj2 = metrics.Recall.from_config(r_obj.get_config())
-    self.assertEqual(r_obj2.name, 'my_recall')
-    self.assertEqual(len(r_obj2.variables), 2)
-    self.assertEqual(r_obj2.thresholds, [0.4, 0.9])
-
-  def test_value_is_idempotent(self):
-    r_obj = metrics.Recall(thresholds=[0.3, 0.72])
-    y_pred = random_ops.random_uniform(shape=(10, 3))
-    y_true = random_ops.random_uniform(shape=(10, 3))
-    update_op = r_obj.update_state(y_true, y_pred)
-    self.evaluate(variables.variables_initializer(r_obj.variables))
-
-    # Run several updates.
-    for _ in range(10):
-      self.evaluate(update_op)
-
-    # Then verify idempotency.
-    initial_recall = self.evaluate(r_obj.result())
-    for _ in range(10):
-      self.assertArrayNear(initial_recall, self.evaluate(r_obj.result()), 1e-3)
-
-  def test_unweighted(self):
-    r_obj = metrics.Recall()
-    y_pred = constant_op.constant([1, 0, 1, 0], shape=(1, 4))
-    y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
-    self.evaluate(variables.variables_initializer(r_obj.variables))
-    result = r_obj(y_true, y_pred)
-    self.assertAlmostEqual(0.5, self.evaluate(result))
-
-  def test_unweighted_all_incorrect(self):
-    r_obj = metrics.Recall(thresholds=[0.5])
-    inputs = np.random.randint(0, 2, size=(100, 1))
-    y_pred = constant_op.constant(inputs)
-    y_true = constant_op.constant(1 - inputs)
-    self.evaluate(variables.variables_initializer(r_obj.variables))
-    result = r_obj(y_true, y_pred)
-    self.assertAlmostEqual(0, self.evaluate(result))
-
-  def test_weighted(self):
-    r_obj = metrics.Recall()
-    y_pred = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]])
-    y_true = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])
-    self.evaluate(variables.variables_initializer(r_obj.variables))
-    result = r_obj(
-        y_true,
-        y_pred,
-        sample_weight=constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]]))
-    weighted_tp = 3.0 + 1.0
-    weighted_t = (2.0 + 3.0) + (4.0 + 1.0)
-    expected_recall = weighted_tp / weighted_t
-    self.assertAlmostEqual(expected_recall, self.evaluate(result))
-
-  def test_div_by_zero(self):
-    r_obj = metrics.Recall()
-    y_pred = constant_op.constant([0, 0, 0, 0])
-    y_true = constant_op.constant([0, 0, 0, 0])
-    self.evaluate(variables.variables_initializer(r_obj.variables))
-    result = r_obj(y_true, y_pred)
-    self.assertEqual(0, self.evaluate(result))
-
-  def test_unweighted_with_threshold(self):
-    r_obj = metrics.Recall(thresholds=[0.5, 0.7])
-    y_pred = constant_op.constant([1, 0, 0.6, 0], shape=(1, 4))
-    y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
-    self.evaluate(variables.variables_initializer(r_obj.variables))
-    result = r_obj(y_true, y_pred)
-    self.assertArrayNear([0.5, 0.], self.evaluate(result), 0)
-
-  def test_weighted_with_threshold(self):
-    r_obj = metrics.Recall(thresholds=[0.5, 1.])
-    y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
-    y_pred = constant_op.constant([[1, 0], [0.6, 0]],
-                                  shape=(2, 2),
-                                  dtype=dtypes.float32)
-    weights = constant_op.constant([[1, 4], [3, 2]],
-                                   shape=(2, 2),
-                                   dtype=dtypes.float32)
-    self.evaluate(variables.variables_initializer(r_obj.variables))
-    result = r_obj(y_true, y_pred, sample_weight=weights)
-    weighted_tp = 0 + 3.
-    weighted_positives = (0 + 3.) + (4. + 0.)
-    expected_recall = weighted_tp / weighted_positives
-    self.assertArrayNear([expected_recall, 0], self.evaluate(result), 1e-3)
-
-  def test_multiple_updates(self):
-    r_obj = metrics.Recall(thresholds=[0.5, 1.])
-    y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
-    y_pred = constant_op.constant([[1, 0], [0.6, 0]],
-                                  shape=(2, 2),
-                                  dtype=dtypes.float32)
-    weights = constant_op.constant([[1, 4], [3, 2]],
-                                   shape=(2, 2),
-                                   dtype=dtypes.float32)
-    self.evaluate(variables.variables_initializer(r_obj.variables))
-    update_op = r_obj.update_state(y_true, y_pred, sample_weight=weights)
-    for _ in range(2):
-      self.evaluate(update_op)
-
-    weighted_tp = (0 + 3.) + (0 + 3.)
-    weighted_positives = ((0 + 3.) + (4. + 0.)) + ((0 + 3.) + (4. + 0.))
-    expected_recall = weighted_tp / weighted_positives
-    self.assertArrayNear([expected_recall, 0], self.evaluate(r_obj.result()),
-                         1e-3)
-
-
-@test_util.run_all_in_graph_and_eager_modes
-class SensitivityAtSpecificityTest(test.TestCase, parameterized.TestCase):
-
-  def test_config(self):
-    s_obj = metrics.SensitivityAtSpecificity(
-        0.4, num_thresholds=100, name='sensitivity_at_specificity_1')
-    self.assertEqual(s_obj.name, 'sensitivity_at_specificity_1')
-    self.assertLen(s_obj.variables, 4)
-    self.assertEqual(s_obj.specificity, 0.4)
-    self.assertEqual(s_obj.num_thresholds, 100)
-
-    # Check save and restore config
-    s_obj2 = metrics.SensitivityAtSpecificity.from_config(s_obj.get_config())
-    self.assertEqual(s_obj2.name, 'sensitivity_at_specificity_1')
-    self.assertLen(s_obj2.variables, 4)
-    self.assertEqual(s_obj2.specificity, 0.4)
-    self.assertEqual(s_obj2.num_thresholds, 100)
-
-  def test_value_is_idempotent(self):
-    s_obj = metrics.SensitivityAtSpecificity(0.7)
-    y_pred = random_ops.random_uniform((10, 3),
-                                       maxval=1,
-                                       dtype=dtypes.float32,
-                                       seed=1)
-    y_true = random_ops.random_uniform((10, 3),
-                                       maxval=2,
-                                       dtype=dtypes.int64,
-                                       seed=1)
-    update_op = s_obj.update_state(y_true, y_pred)
-    self.evaluate(variables.variables_initializer(s_obj.variables))
-
-    # Run several updates.
-    for _ in range(10):
-      self.evaluate(update_op)
-
-    # Then verify idempotency.
-    initial_sensitivity = self.evaluate(s_obj.result())
-    for _ in range(10):
-      self.assertAlmostEqual(initial_sensitivity, self.evaluate(s_obj.result()),
-                             1e-3)
-
-  def test_unweighted_all_correct(self):
-    s_obj = metrics.SensitivityAtSpecificity(0.7)
-    inputs = np.random.randint(0, 2, size=(100, 1))
-    y_pred = constant_op.constant(inputs, dtype=dtypes.float32)
-    y_true = constant_op.constant(inputs)
-    self.evaluate(variables.variables_initializer(s_obj.variables))
-    result = s_obj(y_true, y_pred)
-    self.assertAlmostEqual(1, self.evaluate(result))
-
-  def test_unweighted_high_specificity(self):
-    s_obj = metrics.SensitivityAtSpecificity(0.8)
-    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.1, 0.45, 0.5, 0.8, 0.9]
-    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
-
-    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
-    y_true = constant_op.constant(label_values)
-    self.evaluate(variables.variables_initializer(s_obj.variables))
-    result = s_obj(y_true, y_pred)
-    self.assertAlmostEqual(0.8, self.evaluate(result))
-
-  def test_unweighted_low_specificity(self):
-    s_obj = metrics.SensitivityAtSpecificity(0.4)
-    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26]
-    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
-
-    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
-    y_true = constant_op.constant(label_values)
-    self.evaluate(variables.variables_initializer(s_obj.variables))
-    result = s_obj(y_true, y_pred)
-    self.assertAlmostEqual(0.6, self.evaluate(result))
-
-  @parameterized.parameters([dtypes.bool, dtypes.int32, dtypes.float32])
-  def test_weighted(self, label_dtype):
-    s_obj = metrics.SensitivityAtSpecificity(0.4)
-    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26]
-    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
-    weight_values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
-
-    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
-    y_true = math_ops.cast(label_values, dtype=label_dtype)
-    weights = constant_op.constant(weight_values)
-    self.evaluate(variables.variables_initializer(s_obj.variables))
-    result = s_obj(y_true, y_pred, sample_weight=weights)
-    self.assertAlmostEqual(0.675, self.evaluate(result))
-
-  def test_invalid_specificity(self):
-    with self.assertRaisesRegexp(
-        ValueError, r'`specificity` must be in the range \[0, 1\].'):
-      metrics.SensitivityAtSpecificity(-1)
-
-  def test_invalid_num_thresholds(self):
-    with self.assertRaisesRegexp(ValueError, '`num_thresholds` must be > 0.'):
-      metrics.SensitivityAtSpecificity(0.4, num_thresholds=-1)
-
-
-@test_util.run_all_in_graph_and_eager_modes
-class SpecificityAtSensitivityTest(test.TestCase, parameterized.TestCase):
-
-  def test_config(self):
-    s_obj = metrics.SpecificityAtSensitivity(
-        0.4, num_thresholds=100, name='specificity_at_sensitivity_1')
-    self.assertEqual(s_obj.name, 'specificity_at_sensitivity_1')
-    self.assertLen(s_obj.variables, 4)
-    self.assertEqual(s_obj.sensitivity, 0.4)
-    self.assertEqual(s_obj.num_thresholds, 100)
-
-    # Check save and restore config
-    s_obj2 = metrics.SpecificityAtSensitivity.from_config(s_obj.get_config())
-    self.assertEqual(s_obj2.name, 'specificity_at_sensitivity_1')
-    self.assertLen(s_obj2.variables, 4)
-    self.assertEqual(s_obj2.sensitivity, 0.4)
-    self.assertEqual(s_obj2.num_thresholds, 100)
-
-  def test_value_is_idempotent(self):
-    s_obj = metrics.SpecificityAtSensitivity(0.7)
-    y_pred = random_ops.random_uniform((10, 3),
-                                       maxval=1,
-                                       dtype=dtypes.float32,
-                                       seed=1)
-    y_true = random_ops.random_uniform((10, 3),
-                                       maxval=2,
-                                       dtype=dtypes.int64,
-                                       seed=1)
-    update_op = s_obj.update_state(y_true, y_pred)
-    self.evaluate(variables.variables_initializer(s_obj.variables))
-
-    # Run several updates.
-    for _ in range(10):
-      self.evaluate(update_op)
-
-    # Then verify idempotency.
-    initial_specificity = self.evaluate(s_obj.result())
-    for _ in range(10):
-      self.assertAlmostEqual(initial_specificity, self.evaluate(s_obj.result()),
-                             1e-3)
-
-  def test_unweighted_all_correct(self):
-    s_obj = metrics.SpecificityAtSensitivity(0.7)
-    inputs = np.random.randint(0, 2, size=(100, 1))
-    y_pred = constant_op.constant(inputs, dtype=dtypes.float32)
-    y_true = constant_op.constant(inputs)
-    self.evaluate(variables.variables_initializer(s_obj.variables))
-    result = s_obj(y_true, y_pred)
-    self.assertAlmostEqual(1, self.evaluate(result))
-
-  def test_unweighted_high_sensitivity(self):
-    s_obj = metrics.SpecificityAtSensitivity(0.8)
-    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.1, 0.45, 0.5, 0.8, 0.9]
-    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
-
-    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
-    y_true = constant_op.constant(label_values)
-    self.evaluate(variables.variables_initializer(s_obj.variables))
-    result = s_obj(y_true, y_pred)
-    self.assertAlmostEqual(0.4, self.evaluate(result))
-
-  def test_unweighted_low_sensitivity(self):
-    s_obj = metrics.SpecificityAtSensitivity(0.4)
-    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26]
-    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
-
-    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
-    y_true = constant_op.constant(label_values)
-    self.evaluate(variables.variables_initializer(s_obj.variables))
-    result = s_obj(y_true, y_pred)
-    self.assertAlmostEqual(0.6, self.evaluate(result))
-
-  @parameterized.parameters([dtypes.bool, dtypes.int32, dtypes.float32])
-  def test_weighted(self, label_dtype):
-    s_obj = metrics.SpecificityAtSensitivity(0.4)
-    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26]
-    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
-    weight_values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
-
-    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
-    y_true = math_ops.cast(label_values, dtype=label_dtype)
-    weights = constant_op.constant(weight_values)
-    self.evaluate(variables.variables_initializer(s_obj.variables))
-    result = s_obj(y_true, y_pred, sample_weight=weights)
-    self.assertAlmostEqual(0.4, self.evaluate(result))
-
-  def test_invalid_sensitivity(self):
-    with self.assertRaisesRegexp(
-        ValueError, r'`sensitivity` must be in the range \[0, 1\].'):
-      metrics.SpecificityAtSensitivity(-1)
-
-  def test_invalid_num_thresholds(self):
-    with self.assertRaisesRegexp(ValueError, '`num_thresholds` must be > 0.'):
-      metrics.SpecificityAtSensitivity(0.4, num_thresholds=-1)
-
-
 @test_util.run_all_in_graph_and_eager_modes
 class CosineProximityTest(test.TestCase):
 
-- 
GitLab


From 7e3077a22530e253ab31417abb18de5bb60779ef Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Thu, 10 Jan 2019 14:55:53 -0800
Subject: [PATCH 0517/2345] Add the data file to the data section in BUILD.

PiperOrigin-RevId: 228782621
---
 tensorflow/lite/java/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/lite/java/BUILD b/tensorflow/lite/java/BUILD
index adf7bc9087..a9d4a422c5 100644
--- a/tensorflow/lite/java/BUILD
+++ b/tensorflow/lite/java/BUILD
@@ -121,6 +121,7 @@ java_test(
         "src/testdata/int64.bin",
         "src/testdata/invalid_model.bin",
         "src/testdata/quantized.bin",
+        "src/testdata/string.bin",
         "src/testdata/uint8.bin",
         "src/testdata/with_custom_op.lite",
     ],
-- 
GitLab


From d08fa433af627143e23624e9aaadf32373f4db0e Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Thu, 10 Jan 2019 15:27:19 -0800
Subject: [PATCH 0518/2345] Fix error message when Subclassing Network without
 calling super's __init__ first.

PiperOrigin-RevId: 228788216
---
 tensorflow/python/keras/engine/network.py       |  8 +++++---
 tensorflow/python/keras/engine/topology_test.py | 10 ++++++++++
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/keras/engine/network.py b/tensorflow/python/keras/engine/network.py
index 8e130aef40..6c3691a30c 100644
--- a/tensorflow/python/keras/engine/network.py
+++ b/tensorflow/python/keras/engine/network.py
@@ -335,9 +335,11 @@ class Network(base_layer.Layer):
     if not getattr(self, '_setattr_tracking', True):
       super(Network, self).__setattr__(name, value)
       return
-    if (isinstance(value, (base_layer.Layer,
-                           data_structures.CheckpointableDataStructure))
-        or checkpointable_layer_utils.has_weights(value)):
+
+    if all(
+        isinstance(v, (base_layer.Layer,
+                       data_structures.CheckpointableDataStructure)) or
+        checkpointable_layer_utils.has_weights(v) for v in nest.flatten(value)):
       try:
         self._is_graph_network
       except AttributeError:
diff --git a/tensorflow/python/keras/engine/topology_test.py b/tensorflow/python/keras/engine/topology_test.py
index e9aeae75ce..6c2835aece 100644
--- a/tensorflow/python/keras/engine/topology_test.py
+++ b/tensorflow/python/keras/engine/topology_test.py
@@ -920,6 +920,16 @@ class TopologyConstructionTest(keras_parameterized.TestCase):
       yaml_str = model.to_yaml()
       keras.models.model_from_yaml(yaml_str)
 
+  def test_subclassed_error_if_init_not_called(self):
+
+    class MyNetwork(network_lib.Network):
+
+      def __init__(self):
+        self._foo = [keras.layers.Dense(10), keras.layers.Dense(10)]
+
+    with self.assertRaisesRegexp(RuntimeError, 'forgot to call'):
+      MyNetwork()
+
 
 class DeferredModeTest(test.TestCase):
 
-- 
GitLab


From d6623c9341cda24851827e5069fec810860f1616 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 16:08:49 -0800
Subject: [PATCH 0519/2345] A context manager api for eager profiler.

PiperOrigin-RevId: 228796096
---
 tensorflow/python/eager/profiler.py | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/tensorflow/python/eager/profiler.py b/tensorflow/python/eager/profiler.py
index 016bca3e3c..f1d4373085 100644
--- a/tensorflow/python/eager/profiler.py
+++ b/tensorflow/python/eager/profiler.py
@@ -23,6 +23,7 @@ import threading
 from tensorflow.python import pywrap_tensorflow
 from tensorflow.python.eager import context
 from tensorflow.python.framework import c_api_util
+from tensorflow.python.platform import gfile
 
 _profiler = None
 _profiler_lock = threading.Lock()
@@ -67,3 +68,26 @@ def stop():
     pywrap_tensorflow.TFE_DeleteProfiler(_profiler)
     _profiler = None
   return result
+
+
+class Profiler(object):
+  """Context-manager eager profiler api.
+
+  Example usage:
+  ```python
+  with Profiler("/path/to/save/result"):
+    # do some work
+  ```
+  """
+
+  def __init__(self, filename):
+    self._filename = filename
+
+  def __enter__(self):
+    start()
+
+  def __exit__(self, typ, value, tb):
+    result = stop()
+    with gfile.Open(self._filename, 'wb') as f:
+      f.write(result)
+
-- 
GitLab


From 0b3ef83939b9deb2a32c5afae254555f395e0935 Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Thu, 10 Jan 2019 16:15:12 -0800
Subject: [PATCH 0520/2345] [Grappler] Update UpdateFanouts and UpdateFanin to
 skip mutations when to_node is a Switch that is being added as a control
 dependency.

PiperOrigin-RevId: 228797109
---
 .../core/grappler/mutable_graph_view.cc       | 115 ++++++-----
 tensorflow/core/grappler/mutable_graph_view.h |  11 +-
 .../core/grappler/mutable_graph_view_test.cc  | 181 ++++++++++++++----
 3 files changed, 209 insertions(+), 98 deletions(-)

diff --git a/tensorflow/core/grappler/mutable_graph_view.cc b/tensorflow/core/grappler/mutable_graph_view.cc
index a8a90d1a57..5acdb02dee 100644
--- a/tensorflow/core/grappler/mutable_graph_view.cc
+++ b/tensorflow/core/grappler/mutable_graph_view.cc
@@ -185,6 +185,9 @@ NodeDef* MutableGraphView::AddNode(NodeDef&& node) {
 
 bool MutableGraphView::UpdateFanouts(absl::string_view from_node,
                                      absl::string_view to_node) {
+  if (from_node == to_node) {
+    return false;
+  }
   NodeDef* from_node_ptr = GetNode(from_node);
   NodeDef* to_node_ptr = GetNode(to_node);
   if (from_node_ptr && to_node_ptr) {
@@ -218,6 +221,35 @@ bool MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
     fanouts()[output_port].erase(input_port);
   };
 
+  bool modified = false;
+
+  // For the control fanouts we do not know the input index in a NodeDef,
+  // so we have to traverse all control inputs.
+
+  auto control_fanouts =
+      GetFanout(GraphView::OutputPort(from_node, Graph::kControlSlot));
+
+  bool to_node_is_switch = IsSwitch(*to_node);
+  for (const InputPort& control_port : control_fanouts) {
+    // Node can't be control dependency of itself.
+    if (control_port.node == to_node) continue;
+
+    // Can't add Switch node as a control dependency.
+    if (to_node_is_switch) {
+      // This will only be printed to the log if we are trying to add a Switch
+      // as a control dependency, which if allowed will make the graph invalid.
+      LOG(WARNING) << absl::Substitute(
+          "Can't update fanouts from '$0' to '$1', to node is being added as a "
+          "control dependency when it is a Switch.",
+          from_node, to_node);
+      return false;
+    }
+
+    NodeDef* node = control_port.node;
+    modified |= RemoveControllingFaninInternal(node, from_node);
+    modified |= AddFaninInternal(node, {to_node, Graph::kControlSlot});
+  }
+
   // First we update regular fanouts. For the regular fanouts
   // `input_port:port_id` is the input index in NodeDef.
 
@@ -228,7 +260,6 @@ bool MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
   // input to some other node.
   int keep_max_regular_output_port = -1;
 
-  bool modified = false;
   for (const Edge& edge : regular_edges) {
     const OutputPort output_port = edge.src;
     const InputPort input_port = edge.dst;
@@ -259,22 +290,6 @@ bool MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
     modified = true;
   }
 
-  // For the control fanouts we do not know the input index in a NodeDef,
-  // so we have to traverse all control inputs.
-
-  auto control_fanouts =
-      GetFanout(GraphView::OutputPort(from_node, Graph::kControlSlot));
-
-  for (const InputPort& control_port : control_fanouts) {
-    // Node can't be control dependency of itself.
-    if (control_port.node == to_node) continue;
-
-    NodeDef* node = control_port.node;
-    modified |= RemoveControllingFaninInternal(node, from_node);
-    // TODO(lyandy): Handle Switch control dependencies.
-    modified |= AddFaninInternal(node, {to_node, Graph::kControlSlot});
-  }
-
   // Because we update all regular fanouts of `from_node`, we can just copy
   // the value `num_regular_outputs`.
   max_regular_output_port()[to_node] = max_regular_output_port()[from_node];
@@ -371,7 +386,7 @@ bool MutableGraphView::AddControllingFanin(absl::string_view node_name,
     return AddFaninInternal(node, {fanin_node, Graph::kControlSlot});
   } else {
     if (IsTensorIdControlling(fanin)) {
-      // Cannot add a Switch node control dependency.
+      // Can't add a Switch node control dependency.
       return false;
     }
     // We can't anchor control dependencies directly on the switch node: unlike
@@ -410,11 +425,10 @@ bool MutableGraphView::AddControllingFanin(absl::string_view node_name,
 }
 
 bool MutableGraphView::RemoveRegularFaninInternal(NodeDef* node,
-                                                  const TensorId& fanin) {
-  auto remove_input = [this, node](const TensorId& tensor_id, int port,
-                                   bool update_max_port) {
-    OutputPort fanin_port(nodes()[tensor_id.node()], tensor_id.index());
-    InputPort input(node, port);
+                                                  const OutputPort& fanin) {
+  auto remove_input = [this, node](const OutputPort& fanin_port,
+                                   int node_input_port, bool update_max_port) {
+    InputPort input(node, node_input_port);
 
     absl::flat_hash_set<InputPort>* fanouts_set = &fanouts()[fanin_port];
     fanouts_set->erase(input);
@@ -434,12 +448,14 @@ bool MutableGraphView::RemoveRegularFaninInternal(NodeDef* node,
     if (IsTensorIdControlling(tensor_id)) {
       break;
     }
-    if (tensor_id == fanin) {
-      remove_input(tensor_id, i, /*update_max_port=*/true);
+    if (tensor_id.node() == fanin.node->name() &&
+        tensor_id.index() == fanin.port_id) {
+      remove_input(fanin, i, /*update_max_port=*/true);
       modified = true;
     } else if (modified) {
       // Regular inputs will need to have their ports updated.
-      auto fanouts_set = remove_input(tensor_id, i, /*update_max_port=*/false);
+      OutputPort fanin_port(nodes()[tensor_id.node()], tensor_id.index());
+      auto fanouts_set = remove_input(fanin_port, i, /*update_max_port=*/false);
       fanouts_set->insert({node, curr_pos});
       // Shift inputs to be retained.
       mutable_inputs->SwapElements(i, curr_pos++);
@@ -466,7 +482,11 @@ bool MutableGraphView::RemoveRegularFanin(absl::string_view node_name,
   if (node == nullptr) {
     return false;
   }
-  return RemoveRegularFaninInternal(node, fanin);
+  NodeDef* fanin_node = GetNode(fanin.node());
+  if (fanin_node == nullptr) {
+    return false;
+  }
+  return RemoveRegularFaninInternal(node, {fanin_node, fanin.index()});
 }
 
 bool MutableGraphView::RemoveControllingFaninInternal(NodeDef* node,
@@ -487,22 +507,17 @@ bool MutableGraphView::RemoveControllingFaninInternal(NodeDef* node,
   return false;
 }
 
-bool MutableGraphView::RemoveControllingFaninInternal(
-    NodeDef* node, absl::string_view fanin_node_name) {
-  NodeDef* fanin = GetNode(fanin_node_name);
-  if (fanin == nullptr) {
-    return false;
-  }
-  return RemoveControllingFaninInternal(node, fanin);
-}
-
 bool MutableGraphView::RemoveControllingFanin(
     absl::string_view node_name, absl::string_view fanin_node_name) {
   NodeDef* node = GetNode(node_name);
   if (node == nullptr) {
     return false;
   }
-  return RemoveControllingFaninInternal(node, fanin_node_name);
+  NodeDef* fanin = GetNode(fanin_node_name);
+  if (fanin == nullptr) {
+    return false;
+  }
+  return RemoveControllingFaninInternal(node, fanin);
 }
 
 bool MutableGraphView::RemoveAllFanins(absl::string_view node_name,
@@ -540,30 +555,36 @@ bool MutableGraphView::UpdateFanin(absl::string_view node_name,
     return false;
   }
 
+  NodeDef* from_fanin_node = GetNode(from_fanin.node());
+  NodeDef* to_fanin_node = GetNode(to_fanin.node());
+  if (from_fanin_node == nullptr || to_fanin_node == nullptr) {
+    return false;
+  }
+
   // When replacing a non control dependency fanin with a control dependency, or
   // vice versa, remove and add, so ports can be updated properly in fanout(s).
   bool from_fanin_is_control = IsTensorIdControlling(from_fanin);
-  if (from_fanin_is_control || IsTensorIdControlling(to_fanin)) {
+  bool to_fanin_is_control = IsTensorIdControlling(to_fanin);
+  if (to_fanin_is_control && IsSwitch(*to_fanin_node)) {
+    // Can't add Switch node as a control dependency.
+    return false;
+  }
+  if (from_fanin_is_control || to_fanin_is_control) {
     bool modified = false;
     if (from_fanin_is_control) {
-      modified |= RemoveControllingFaninInternal(node, from_fanin.node());
+      modified |= RemoveControllingFaninInternal(node, from_fanin_node);
     } else {
-      modified |= RemoveRegularFaninInternal(node, from_fanin);
+      modified |= RemoveRegularFaninInternal(
+          node, {from_fanin_node, from_fanin.index()});
     }
     if (modified) {
-      AddFaninInternal(node, to_fanin);
+      AddFaninInternal(node, {to_fanin_node, to_fanin.index()});
     }
 
     return modified;
   }
 
   // In place mutation, requires no shifting of ports.
-  NodeDef* from_fanin_node = GetNode(from_fanin.node());
-  NodeDef* to_fanin_node = GetNode(to_fanin.node());
-  if (from_fanin_node == nullptr || to_fanin_node == nullptr) {
-    return false;
-  }
-
   string to_fanin_string = TensorIdToString(to_fanin);
   int num_inputs = node->input_size();
   bool modified = false;
diff --git a/tensorflow/core/grappler/mutable_graph_view.h b/tensorflow/core/grappler/mutable_graph_view.h
index f07254068d..0a441d9115 100644
--- a/tensorflow/core/grappler/mutable_graph_view.h
+++ b/tensorflow/core/grappler/mutable_graph_view.h
@@ -195,17 +195,10 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // already exists, the node will not be modified.
   bool AddFaninInternal(NodeDef* node, const TensorId& fanin);
 
-  // Removes any fanin in node that matches to a fanin in fanins.
+  // Removes all instances of regular fanin `fanin` from node `node`.
   //
   // This will return true iff the node is modified.
-  bool RemoveRegularFaninInternal(NodeDef* node, const TensorId& fanin);
-
-  // Removes controlling fanin `fanin_node_name` from node if such controlling
-  // fanin exists.
-  //
-  // This will return true iff the node is modified.
-  bool RemoveControllingFaninInternal(NodeDef* node,
-                                      absl::string_view fanin_node_name);
+  bool RemoveRegularFaninInternal(NodeDef* node, const OutputPort& fanin);
 
   // Removes controlling fanin `fanin_node` from node if such controlling fanin
   // exists.
diff --git a/tensorflow/core/grappler/mutable_graph_view_test.cc b/tensorflow/core/grappler/mutable_graph_view_test.cc
index 2048c67e3e..b44e1a360d 100644
--- a/tensorflow/core/grappler/mutable_graph_view_test.cc
+++ b/tensorflow/core/grappler/mutable_graph_view_test.cc
@@ -168,21 +168,6 @@ TEST(MutableGraphViewTest, AddAndUpdateFanoutsWithoutSelfLoops) {
   EXPECT_EQ(new_bar_fanouts.count(MutableGraphView::InputPort(foo_2, -1)), 1);
 }
 
-GraphDef SimpleMutateFaninGraph() {
-  // Actual node.op() is not important in this test.
-  GraphDef graph_def = test::function::GDef(
-      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {}, {}),
-       NDef("c", "NotImportant", {}, {}), NDef("d", "NotImportant", {}, {}),
-       NDef("foo_1", "NotImportant", {"a"}),
-       NDef("foo_2", "NotImportant", {"b", "^a", "^c"}),
-       NDef("foo_3", "NotImportant", {"b", "a:1", "a:1"}),
-       NDef("foo_4", "NotImportant", {"a", "b:2", "b:2", "^c", "^d"}),
-       NDef("foo_5", "NotImportant", {}),
-       NDef("foo_6", "NotImportant", {"^a", "^b"})},
-      /*funcs=*/{});
-  return graph_def;
-}
-
 void CompareNodeInputs(const MutableGraphView& graph, const NodeDef* expected,
                        NodeDef* actual) {
   ASSERT_EQ(actual->input_size(), expected->input_size());
@@ -203,6 +188,94 @@ void CompareNodeInputs(const MutableGraphView& graph, const NodeDef* expected,
   }
 }
 
+TEST(MutableGraphViewTest, UpdateFanoutsToSwitchWithControlFromSwitch) {
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "Switch", {}, {}),
+       NDef("c", "NotImportant", {}, {}), NDef("d", "NotImportant", {}, {}),
+       NDef("e", "NotImportant", {"c", "b", "^a", "^d"})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_FALSE(graph.UpdateFanouts("a", "b"));
+  EXPECT_FALSE(graph.UpdateFanouts("d", "b"));
+
+  EXPECT_EQ(graph.graph()->node_size(), 5);
+
+  NodeDef expected = NDef("", "", {}, {});
+  NodeDef* a = graph.GetNode("a");
+  ASSERT_NE(a, nullptr);
+  CompareNodeInputs(graph, &expected, a);
+
+  NodeDef* b = graph.GetNode("b");
+  ASSERT_NE(b, nullptr);
+  CompareNodeInputs(graph, &expected, b);
+
+  NodeDef* c = graph.GetNode("c");
+  ASSERT_NE(c, nullptr);
+  CompareNodeInputs(graph, &expected, c);
+
+  NodeDef* d = graph.GetNode("d");
+  ASSERT_NE(d, nullptr);
+  CompareNodeInputs(graph, &expected, d);
+
+  NodeDef* e = graph.GetNode("e");
+  ASSERT_NE(e, nullptr);
+  expected = NDef("", "", {"c", "b", "^a", "^d"});
+  CompareNodeInputs(graph, &expected, e);
+}
+
+TEST(MutableGraphViewTest, UpdateFanoutsToSwitchWithNoControlFromSwitch) {
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "Switch", {}, {}),
+       NDef("c", "NotImportant", {}, {}), NDef("d", "NotImportant", {}, {}),
+       NDef("e", "NotImportant", {"c", "b", "^a", "^d"})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_TRUE(graph.UpdateFanouts("c", "b"));
+
+  EXPECT_EQ(graph.graph()->node_size(), 5);
+
+  NodeDef expected = NDef("", "", {}, {});
+  NodeDef* a = graph.GetNode("a");
+  ASSERT_NE(a, nullptr);
+  CompareNodeInputs(graph, &expected, a);
+
+  NodeDef* b = graph.GetNode("b");
+  ASSERT_NE(b, nullptr);
+  CompareNodeInputs(graph, &expected, b);
+
+  NodeDef* c = graph.GetNode("c");
+  ASSERT_NE(c, nullptr);
+  CompareNodeInputs(graph, &expected, c);
+
+  NodeDef* d = graph.GetNode("d");
+  ASSERT_NE(d, nullptr);
+  CompareNodeInputs(graph, &expected, d);
+
+  NodeDef* e = graph.GetNode("e");
+  ASSERT_NE(e, nullptr);
+  expected = NDef("", "", {"b", "b", "^a", "^d"});
+  CompareNodeInputs(graph, &expected, e);
+}
+
+GraphDef SimpleMutateFaninGraph() {
+  // Actual node.op() is not important in this test.
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {}, {}),
+       NDef("c", "NotImportant", {}, {}), NDef("d", "NotImportant", {}, {}),
+       NDef("foo_1", "NotImportant", {"a"}),
+       NDef("foo_2", "NotImportant", {"b", "^a", "^c"}),
+       NDef("foo_3", "NotImportant", {"b", "a:1", "a:1"}),
+       NDef("foo_4", "NotImportant", {"a", "b:2", "b:2", "^c", "^d"}),
+       NDef("foo_5", "NotImportant", {}),
+       NDef("foo_6", "NotImportant", {"^a", "^b"})},
+      /*funcs=*/{});
+  return graph_def;
+}
+
 void TestAddRegularFanin(absl::string_view node_name,
                          const TensorId& fanin_to_add, bool modified,
                          const NodeDef* expected_node) {
@@ -525,23 +598,53 @@ TEST(MutableGraphViewTest, UpdateFanin) {
                   /*modified=*/false, /*expected_node=*/nullptr);
 }
 
+void TestUpdateFaninFromFaninToNodeAsSwitchControl(const TensorId& fanin) {
+  string tensor_id_str = TensorIdToString(fanin);
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "Switch", {}, {}),
+       NDef("c", "NotImportant", {tensor_id_str})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_FALSE(graph.UpdateFanin("c", fanin, {"b", Graph::kControlSlot}));
+
+  EXPECT_EQ(graph.graph()->node_size(), 3);
+
+  NodeDef expected;
+  NodeDef* a = graph.GetNode("a");
+  ASSERT_NE(a, nullptr);
+  expected = NDef("", "", {});
+  CompareNodeInputs(graph, &expected, a);
+
+  NodeDef* b = graph.GetNode("b");
+  ASSERT_NE(b, nullptr);
+  CompareNodeInputs(graph, &expected, b);
+
+  NodeDef* c = graph.GetNode("c");
+  ASSERT_NE(c, nullptr);
+  expected = NDef("", "", {tensor_id_str});
+  CompareNodeInputs(graph, &expected, c);
+}
+
+TEST(MutableGraphViewTest, UpdateFaninToNodeAsSwitchControl) {
+  TestUpdateFaninFromFaninToNodeAsSwitchControl({"a", 0});
+  TestUpdateFaninFromFaninToNodeAsSwitchControl({"a", 1});
+  TestUpdateFaninFromFaninToNodeAsSwitchControl({"a", Graph::kControlSlot});
+}
+
 TEST(MutableGraphViewTest, DedupControllingFaninsOnGraphInit) {
-  // Actual node.op() is not important in this test.
   GraphDef graph_def = test::function::GDef(
-      {
-          NDef("a", "NotImportant", {}, {}),
-          NDef("b", "NotImportant", {}, {}),
-          NDef("c", "Switch", {}, {}),
-          NDef("d", "Identity", {"c:1"}),
-          NDef("foo_1", "IdentityN", {"a", "b:1", "^b"}),
-          NDef("foo_2", "IdentityN", {"a", "^b", "^b"}),
-          NDef("foo_3", "IdentityN", {"a", "b:1", "^b", "^b"}),
-          NDef("foo_4", "IdentityN", {"a:2", "b:1", "^b", "^b", "^a", "^a"}),
-          NDef("foo_5", "NotImportant", {"a:2", "b:1", "^b", "^b", "^a", "^a"}),
-          NDef("foo_6", "Identity", {"d", "^d"}),
-          NDef("foo_7", "NotImportant",
-               {"a:3", "b:2", "d", "^d", "^d", "^a", "^b", "^a", "^b"}),
-      },
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {}, {}),
+       NDef("c", "Switch", {}, {}), NDef("d", "Identity", {"c:1"}),
+       NDef("foo_1", "IdentityN", {"a", "b:1", "^b"}),
+       NDef("foo_2", "IdentityN", {"a", "^b", "^b"}),
+       NDef("foo_3", "IdentityN", {"a", "b:1", "^b", "^b"}),
+       NDef("foo_4", "IdentityN", {"a:2", "b:1", "^b", "^b", "^a", "^a"}),
+       NDef("foo_5", "NotImportant", {"a:2", "b:1", "^b", "^b", "^a", "^a"}),
+       NDef("foo_6", "Identity", {"d", "^d"}),
+       NDef("foo_7", "NotImportant",
+            {"a:3", "b:2", "d", "^d", "^d", "^a", "^b", "^a", "^b"})},
       /*funcs=*/{});
 
   MutableGraphView graph(&graph_def);
@@ -939,12 +1042,9 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithExistingAddedIdentity) {
 TEST(MutableGraphViewTest, RemoveControllingFaninMissing) {
   // Actual node.op() is not important in this test.
   GraphDef graph_def = test::function::GDef(
-      {
-          NDef("a", "NotImportant", {}, {}),
-          NDef("b", "NotImportant", {}, {}),
-          NDef("c", "NotImportant", {}, {}),
-          NDef("d", "NotImportant", {"^a", "^b"}),
-      },
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {}, {}),
+       NDef("c", "NotImportant", {}, {}),
+       NDef("d", "NotImportant", {"^a", "^b"})},
       /*funcs=*/{});
 
   MutableGraphView graph(&graph_def);
@@ -961,12 +1061,9 @@ TEST(MutableGraphViewTest, RemoveControllingFaninMissing) {
 TEST(MutableGraphViewTest, RemoveControllingFaninExisting) {
   // Actual node.op() is not important in this test.
   GraphDef graph_def = test::function::GDef(
-      {
-          NDef("a", "NotImportant", {}, {}),
-          NDef("b", "NotImportant", {}, {}),
-          NDef("c", "NotImportant", {}, {}),
-          NDef("d", "NotImportant", {"^a", "^b", "^c"}),
-      },
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {}, {}),
+       NDef("c", "NotImportant", {}, {}),
+       NDef("d", "NotImportant", {"^a", "^b", "^c"})},
       /*funcs=*/{});
 
   MutableGraphView graph(&graph_def);
-- 
GitLab


From f497b36f3250a4e26051be3f3616e7ad233f3cc8 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Thu, 10 Jan 2019 16:17:39 -0800
Subject: [PATCH 0521/2345] Remove the experimental initialize and finalize
 API.

PiperOrigin-RevId: 228797485
---
 .../python/distribute/distribute_lib.py       | 42 -------------------
 ...orflow.distribute.-mirrored-strategy.pbtxt | 16 -------
 .../v1/tensorflow.distribute.-strategy.pbtxt  | 16 -------
 ...orflow.distribute.-mirrored-strategy.pbtxt | 16 -------
 .../v2/tensorflow.distribute.-strategy.pbtxt  | 16 -------
 5 files changed, 106 deletions(-)

diff --git a/tensorflow/python/distribute/distribute_lib.py b/tensorflow/python/distribute/distribute_lib.py
index 5fe77e5478..66112dceda 100644
--- a/tensorflow/python/distribute/distribute_lib.py
+++ b/tensorflow/python/distribute/distribute_lib.py
@@ -505,42 +505,6 @@ class DistributionStrategy(object):
     """DEPRECATED: use extended.broadcast_to() instead."""
     return self._extended.broadcast_to(tensor, destinations)
 
-  @doc_controls.do_not_generate_docs  # Use experimental_initialize() instead.
-  def initialize(self):
-    """DEPRECATED: Use `experimental_initialize()` instead."""
-    return self._extended._initialize()  # pylint: disable=protected-access
-
-  def experimental_initialize(self):
-    """Any initialization to be done before running any computations.
-
-    In eager mode, it executes any initialization as a side effect.
-    In graph mode, it creates the initialization ops and returns them.
-
-    For example, TPU initialize_system ops.
-
-    Returns:
-      A list of ops to execute.
-    """
-    return self._extended._initialize()  # pylint: disable=protected-access
-
-  @doc_controls.do_not_generate_docs  # Use experimental_finalize() instead.
-  def finalize(self):
-    """DEPRECATED: Use `experimental_finalize()` instead."""
-    return self._extended._finalize()  # pylint: disable=protected-access
-
-  def experimental_finalize(self):
-    """Any final actions to be done at the end of all computations.
-
-    In eager mode, it executes any finalize actions as a side effect.
-    In graph mode, it creates the finalize ops and returns them.
-
-    For example, TPU shutdown ops.
-
-    Returns:
-      A list of ops to execute.
-    """
-    return self._extended._finalize()  # pylint: disable=protected-access
-
   @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
   def run_steps_on_dataset(self, fn, iterator, iterations=1,
                            initial_loop_values=None):
@@ -1171,12 +1135,6 @@ class DistributionStrategyExtended(object):
   def _broadcast_to(self, tensor, destinations):
     raise NotImplementedError("must be implemented in descendants")
 
-  def _initialize(self):
-    return []
-
-  def _finalize(self):
-    return []
-
   def experimental_run_steps_on_iterator(self, fn, iterator, iterations=1,
                                          initial_loop_values=None):
     """Run `fn` with input from `iterator` for `iterations` times.
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
index 9c29067b6d..863ee9c210 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
@@ -67,14 +67,6 @@ tf_class {
     name: "distribute_dataset"
     argspec: "args=[\'self\', \'dataset_fn\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "experimental_finalize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "experimental_initialize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "experimental_make_numpy_iterator"
     argspec: "args=[\'self\', \'numpy_input\', \'batch_size\', \'num_epochs\', \'shuffle\', \'session\'], varargs=None, keywords=None, defaults=[\'1\', \'1024\', \'None\'], "
@@ -83,18 +75,10 @@ tf_class {
     name: "experimental_run"
     argspec: "args=[\'self\', \'fn\', \'input_iterator\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "finalize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "group"
     argspec: "args=[\'self\', \'value\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "initialize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "make_dataset_iterator"
     argspec: "args=[\'self\', \'dataset\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
index 4aa6f1c4e1..1c06f7873e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
@@ -66,14 +66,6 @@ tf_class {
     name: "distribute_dataset"
     argspec: "args=[\'self\', \'dataset_fn\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "experimental_finalize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "experimental_initialize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "experimental_make_numpy_iterator"
     argspec: "args=[\'self\', \'numpy_input\', \'batch_size\', \'num_epochs\', \'shuffle\', \'session\'], varargs=None, keywords=None, defaults=[\'1\', \'1024\', \'None\'], "
@@ -82,18 +74,10 @@ tf_class {
     name: "experimental_run"
     argspec: "args=[\'self\', \'fn\', \'input_iterator\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "finalize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "group"
     argspec: "args=[\'self\', \'value\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "initialize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "make_dataset_iterator"
     argspec: "args=[\'self\', \'dataset\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
index 9c29067b6d..863ee9c210 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
@@ -67,14 +67,6 @@ tf_class {
     name: "distribute_dataset"
     argspec: "args=[\'self\', \'dataset_fn\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "experimental_finalize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "experimental_initialize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "experimental_make_numpy_iterator"
     argspec: "args=[\'self\', \'numpy_input\', \'batch_size\', \'num_epochs\', \'shuffle\', \'session\'], varargs=None, keywords=None, defaults=[\'1\', \'1024\', \'None\'], "
@@ -83,18 +75,10 @@ tf_class {
     name: "experimental_run"
     argspec: "args=[\'self\', \'fn\', \'input_iterator\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "finalize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "group"
     argspec: "args=[\'self\', \'value\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "initialize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "make_dataset_iterator"
     argspec: "args=[\'self\', \'dataset\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
index 4aa6f1c4e1..1c06f7873e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
@@ -66,14 +66,6 @@ tf_class {
     name: "distribute_dataset"
     argspec: "args=[\'self\', \'dataset_fn\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "experimental_finalize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "experimental_initialize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "experimental_make_numpy_iterator"
     argspec: "args=[\'self\', \'numpy_input\', \'batch_size\', \'num_epochs\', \'shuffle\', \'session\'], varargs=None, keywords=None, defaults=[\'1\', \'1024\', \'None\'], "
@@ -82,18 +74,10 @@ tf_class {
     name: "experimental_run"
     argspec: "args=[\'self\', \'fn\', \'input_iterator\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "finalize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "group"
     argspec: "args=[\'self\', \'value\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "initialize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "make_dataset_iterator"
     argspec: "args=[\'self\', \'dataset\'], varargs=None, keywords=None, defaults=None"
-- 
GitLab


From a7443e6ad57bd60693cf58a9dad3f462fdf5bc3d Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Thu, 10 Jan 2019 16:20:26 -0800
Subject: [PATCH 0522/2345] Allow scalar shapes in the Python converter API

Scalars are represented as empty lists in the shape. As the
converter and runtime now support scalars, allow such empty
(scalar) shape values in the Python converter API. Also
add a test to validate conversion using string types.

PiperOrigin-RevId: 228797882
---
 tensorflow/lite/python/BUILD        |  1 +
 tensorflow/lite/python/lite.py      |  5 ++-
 tensorflow/lite/python/lite_test.py | 65 ++++++++++++++++++++++++++---
 3 files changed, 63 insertions(+), 8 deletions(-)

diff --git a/tensorflow/lite/python/BUILD b/tensorflow/lite/python/BUILD
index 54b925cc05..4949d7c92e 100644
--- a/tensorflow/lite/python/BUILD
+++ b/tensorflow/lite/python/BUILD
@@ -72,6 +72,7 @@ py_test(
     name = "lite_test",
     srcs = ["lite_test.py"],
     data = ["@tflite_mobilenet_ssd_quant_protobuf//:tflite_graph.pb"],
+    shard_count = 4,
     srcs_version = "PY2AND3",
     tags = [
         "no_oss",
diff --git a/tensorflow/lite/python/lite.py b/tensorflow/lite/python/lite.py
index 5ffa8c426b..3b0aa02b7c 100644
--- a/tensorflow/lite/python/lite.py
+++ b/tensorflow/lite/python/lite.py
@@ -435,15 +435,16 @@ class TFLiteConverter(object):
     if self._has_valid_tensors():
       for tensor in self._input_tensors:
         shape = tensor.get_shape()
-        if not shape or not shape.as_list():
+        if not shape:
           raise ValueError("Provide an input shape for input array "
                            "'{0}'.".format(_tensor_name(tensor)))
+        # Note that shape_list might be empty for scalar shapes.
         shape_list = shape.as_list()
         if None in shape_list[1:]:
           raise ValueError(
               "None is only supported in the 1st dimension. Tensor '{0}' has "
               "invalid shape '{1}'.".format(_tensor_name(tensor), shape_list))
-        elif shape_list[0] is None:
+        elif shape_list and shape_list[0] is None:
           self._set_batch_size(batch_size=1)
 
     # Get quantization stats. Ensures there is one stat per name if the stats
diff --git a/tensorflow/lite/python/lite_test.py b/tensorflow/lite/python/lite_test.py
index 1f9c768b44..83fd56bf1d 100644
--- a/tensorflow/lite/python/lite_test.py
+++ b/tensorflow/lite/python/lite_test.py
@@ -113,6 +113,35 @@ class FromSessionTest(test_util.TensorFlowTestCase):
     self.assertTrue(([1, 16, 16, 3] == output_details[0]['shape']).all())
     self.assertEqual((0., 0.), output_details[0]['quantization'])
 
+  def testString(self):
+    in_tensor = array_ops.placeholder(shape=[4], dtype=dtypes.string)
+    out_tensor = array_ops.reshape(in_tensor, shape=[2, 2])
+    sess = session.Session()
+
+    # Convert model and ensure model is not None.
+    converter = lite.TFLiteConverter.from_session(sess, [in_tensor],
+                                                  [out_tensor])
+    tflite_model = converter.convert()
+    self.assertTrue(tflite_model)
+
+    # Check values from converted model.
+    interpreter = Interpreter(model_content=tflite_model)
+    interpreter.allocate_tensors()
+
+    input_details = interpreter.get_input_details()
+    self.assertEqual(1, len(input_details))
+    self.assertEqual('Placeholder', input_details[0]['name'])
+    self.assertEqual(np.object_, input_details[0]['dtype'])
+    self.assertTrue(([4] == input_details[0]['shape']).all())
+
+    output_details = interpreter.get_output_details()
+    self.assertEqual(1, len(output_details))
+    self.assertEqual('Reshape', output_details[0]['name'])
+    self.assertEqual(np.object_, output_details[0]['dtype'])
+    self.assertTrue(([2, 2] == output_details[0]['shape']).all())
+    # TODO(b/122659643): Test setting/getting string data via the python
+    # interpreter API after support has been added.
+
   def testQuantization(self):
     in_tensor_1 = array_ops.placeholder(
         shape=[1, 16, 16, 3], dtype=dtypes.float32, name='inputA')
@@ -223,18 +252,42 @@ class FromSessionTest(test_util.TensorFlowTestCase):
     self.assertEqual('Provide an input shape for input array \'Placeholder\'.',
                      str(error.exception))
 
-  def testSizeEmptyInvalid(self):
+  def testScalarValid(self):
+    # Construct a graph using a scalar (empty shape) input.
     in_tensor = array_ops.placeholder(dtype=dtypes.float32, shape=[])
     out_tensor = in_tensor + in_tensor
     sess = session.Session()
 
-    # Test empty shape.
+    # Test conversion with the scalar input shape.
     converter = lite.TFLiteConverter.from_session(sess, [in_tensor],
                                                   [out_tensor])
-    with self.assertRaises(ValueError) as error:
-      converter.convert()
-    self.assertEqual('Provide an input shape for input array \'Placeholder\'.',
-                     str(error.exception))
+    tflite_model = converter.convert()
+    self.assertTrue(tflite_model)
+
+    # Check values from converted model.
+    interpreter = Interpreter(model_content=tflite_model)
+    interpreter.allocate_tensors()
+
+    input_details = interpreter.get_input_details()
+    self.assertEqual(1, len(input_details))
+    self.assertEqual('Placeholder', input_details[0]['name'])
+    self.assertEqual(np.float32, input_details[0]['dtype'])
+    self.assertTrue(([] == input_details[0]['shape']).all())
+
+    output_details = interpreter.get_output_details()
+    self.assertEqual(1, len(output_details))
+    self.assertEqual('add', output_details[0]['name'])
+    self.assertEqual(np.float32, output_details[0]['dtype'])
+    self.assertTrue(([] == input_details[0]['shape']).all())
+
+    # Validate inference using the scalar inputs/outputs.
+    test_input = np.array(4.0, dtype=np.float32)
+    expected_output = np.array(8.0, dtype=np.float32)
+    interpreter.set_tensor(input_details[0]['index'], test_input)
+    interpreter.invoke()
+
+    output_data = interpreter.get_tensor(output_details[0]['index'])
+    self.assertTrue((expected_output == output_data).all())
 
   def testSizeInvalid(self):
     in_tensor = array_ops.placeholder(
-- 
GitLab


From 63a6b272919a1621b6ecc57b4960047e2c61d654 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 16:22:20 -0800
Subject: [PATCH 0523/2345] Internal change

PiperOrigin-RevId: 228798154
---
 .../feature_column/feature_column_v2.py       | 18 ++----
 .../feature_column/feature_column_v2_test.py  | 56 ++++++++++++++++++-
 2 files changed, 60 insertions(+), 14 deletions(-)

diff --git a/tensorflow/python/feature_column/feature_column_v2.py b/tensorflow/python/feature_column/feature_column_v2.py
index d4e3fc5d2d..73e3001fc0 100644
--- a/tensorflow/python/feature_column/feature_column_v2.py
+++ b/tensorflow/python/feature_column/feature_column_v2.py
@@ -2780,12 +2780,8 @@ class NumericColumn(
     """See 'FeatureColumn` base class."""
     _check_config_keys(config, cls._fields)
     kwargs = config.copy()
-    # TODO(b/118820158): Simplify if deserialize_keras_object supports None.
-    if config['normalizer_fn']:
-      kwargs['normalizer_fn'] = utils.deserialize_keras_object(
-          config['normalizer_fn'], custom_objects=custom_objects)
-    else:
-      kwargs['normalizer_fn'] = None
+    kwargs['normalizer_fn'] = utils.deserialize_keras_object(
+        config['normalizer_fn'], custom_objects=custom_objects)
     kwargs['dtype'] = dtypes.as_dtype(config['dtype'])
     return cls(**kwargs)
 
@@ -3168,12 +3164,8 @@ class EmbeddingColumn(
     kwargs = config.copy()
     kwargs['categorical_column'] = deserialize_feature_column(
         config['categorical_column'], custom_objects, columns_by_name)
-    # TODO(b/118820158): Simplify if deserialize_keras_object supports None.
-    if config['initializer']:
-      kwargs['initializer'] = utils.deserialize_keras_object(
-          config['initializer'], custom_objects=custom_objects)
-    else:
-      kwargs['initializer'] = None
+    kwargs['initializer'] = utils.deserialize_keras_object(
+        config['initializer'], custom_objects=custom_objects)
     return cls(**kwargs)
 
 
@@ -4681,7 +4673,7 @@ def deserialize_feature_column(config,
           IdentityCategoricalColumn, IndicatorColumn, NumericColumn,
           SequenceCategoricalColumn, SequenceDenseColumn, SharedEmbeddingColumn,
           VocabularyFileCategoricalColumn, VocabularyListCategoricalColumn,
-          WeightedCategoricalColumn
+          WeightedCategoricalColumn, init_ops.TruncatedNormal
       ]
   }
   if columns_by_name is None:
diff --git a/tensorflow/python/feature_column/feature_column_v2_test.py b/tensorflow/python/feature_column/feature_column_v2_test.py
index aeaa8df8b5..137d5e0a8c 100644
--- a/tensorflow/python/feature_column/feature_column_v2_test.py
+++ b/tensorflow/python/feature_column/feature_column_v2_test.py
@@ -40,6 +40,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import lookup_ops
 from tensorflow.python.ops import parsing_ops
 from tensorflow.python.ops import partitioned_variables
@@ -7147,7 +7148,60 @@ class EmbeddingColumnTest(test.TestCase):
                           self.evaluate(predictions))
 
   @test_util.run_deprecated_v1
-  def test_serialization(self):
+  def test_serialization_with_default_initializer(self):
+
+    # Build columns.
+    categorical_column = fc.categorical_column_with_identity(
+        key='aaa', num_buckets=3)
+    embedding_column = fc.embedding_column(categorical_column, dimension=2)
+
+    self.assertEqual([categorical_column], embedding_column.parents)
+
+    config = embedding_column._get_config()
+    self.assertEqual({
+        'categorical_column': {
+            'class_name': 'IdentityCategoricalColumn',
+            'config': {
+                'number_buckets': 3,
+                'key': 'aaa',
+                'default_value': None
+            }
+        },
+        'ckpt_to_load_from': None,
+        'combiner': 'mean',
+        'dimension': 2,
+        'initializer': {
+            'class_name': 'TruncatedNormal',
+            'config': {
+                'dtype': 'float32',
+                'stddev': 0.7071067811865475,
+                'seed': None,
+                'mean': 0.0
+            }
+        },
+        'max_norm': None,
+        'tensor_name_in_ckpt': None,
+        'trainable': True
+    }, config)
+
+    custom_objects = {'TruncatedNormal': init_ops.TruncatedNormal}
+    new_embedding_column = fc.EmbeddingColumn._from_config(
+        config, custom_objects=custom_objects)
+    self.assertEqual(embedding_column._get_config(),
+                     new_embedding_column._get_config())
+    self.assertIsNot(categorical_column,
+                     new_embedding_column.categorical_column)
+
+    new_embedding_column = fc.EmbeddingColumn._from_config(
+        config,
+        custom_objects=custom_objects,
+        columns_by_name={categorical_column.name: categorical_column})
+    self.assertEqual(embedding_column._get_config(),
+                     new_embedding_column._get_config())
+    self.assertIs(categorical_column, new_embedding_column.categorical_column)
+
+  @test_util.run_deprecated_v1
+  def test_serialization_with_custom_initializer(self):
 
     def _initializer(shape, dtype, partition_info):
       del shape, dtype, partition_info
-- 
GitLab


From d2490b91e3f3d7771a58986a2d1446a32053eb19 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 16:27:21 -0800
Subject: [PATCH 0524/2345] Use Cub-based tensor reduction in multinomial op.
 The Eigen implementation seems to trigger a bug on Volta GPUs.

PiperOrigin-RevId: 228798930
---
 tensorflow/core/kernels/BUILD                    |  7 +++++--
 tensorflow/core/kernels/multinomial_op_gpu.cu.cc | 14 ++++++++++----
 2 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 1a20e8628e..0045c5dd19 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -4982,11 +4982,14 @@ tf_kernel_library(
         ":random_op",
         ":random_ops",
         ":stateless_random_ops",
+        "//third_party/eigen3",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//third_party/eigen3",
-    ],
+    ] + if_cuda([
+        ":reduction_ops",
+        "@cub_archive//:cub",
+    ]),
 )
 
 tf_cuda_cc_test(
diff --git a/tensorflow/core/kernels/multinomial_op_gpu.cu.cc b/tensorflow/core/kernels/multinomial_op_gpu.cu.cc
index 5cc5877cce..62e38694c8 100644
--- a/tensorflow/core/kernels/multinomial_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/multinomial_op_gpu.cu.cc
@@ -22,9 +22,10 @@ limitations under the License.
 #include <assert.h>
 #include <stdio.h>
 
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/kernels/random_op.h"
+#include "tensorflow/core/kernels/reduction_gpu_kernels.cu.h"
+#include "tensorflow/core/kernels/reduction_ops_common.h"
 #include "tensorflow/core/lib/random/philox_random.h"
 #include "tensorflow/core/lib/random/random_distributions.h"
 #include "tensorflow/core/util/cuda_kernel_helper.h"
@@ -67,7 +68,6 @@ struct MultinomialFunctor<GPUDevice, T, OutputType> {
                                                  noises.size(), Dist());
 
 #if defined(EIGEN_HAS_INDEX_LIST)
-    Eigen::IndexList<Eigen::type2index<2>> kTwo;
     Eigen::IndexList<int, int, int> bsc;
     bsc.set(0, batch_size);
     bsc.set(1, num_samples);
@@ -80,7 +80,6 @@ struct MultinomialFunctor<GPUDevice, T, OutputType> {
     Eigen::IndexList<Eigen::type2index<1>, int, Eigen::type2index<1>> oso;
     oso.set(1, num_samples);
 #else
-    Eigen::array<int, 1> kTwo{2};
     Eigen::array<int, 3> bsc{batch_size, num_samples, num_classes};
     Eigen::array<int, 3> boc{batch_size, 1, num_classes};
     Eigen::array<int, 3> oso{1, num_samples, 1};
@@ -98,7 +97,14 @@ struct MultinomialFunctor<GPUDevice, T, OutputType> {
         ((-((To32Bit(noises) + 2e-30f).log())).log());
 
     // Max-reduce along classes for each (batch, sample).
-    To32Bit(maxima).device(d) = To32Bit(scores).reshape(bsc).maximum(kTwo);
+    typedef const Eigen::array<TTypes<float>::Tensor::Index, 1>& ReductionAxes;
+    Constants<GPUDevice> constants;
+    cub::Max op;
+    functor::ReduceImpl<float, cub::Max, float*, const float*, ReductionAxes>(
+        /*ctx=*/ctx, /*out=*/maxima.data(), /*in=*/scores.data(), /*in_rank=*/2,
+        /*in_dim0=*/batch_size * num_samples,
+        /*in_dim1=*/num_classes, /*in_dim2=*/1, /*out_rank=*/1,
+        /*reduction_axes=*/constants.kOne, /*Op=*/op);
 
     // Necessary for atomicMax() inside the kernel.
     output.device(d) = output.constant(0LL);
-- 
GitLab


From 40874676a61cc1823355016953a6ac6222f8b404 Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Thu, 10 Jan 2019 16:43:49 -0800
Subject: [PATCH 0525/2345] Make nccl work in eager mode: wrap nccl ops in a
 defun; remove control dependencies on NcclAllReduce

PiperOrigin-RevId: 228801431
---
 tensorflow/python/BUILD                       |  2 ++
 .../python/framework/auto_control_deps.py     |  1 +
 tensorflow/python/ops/nccl_ops.py             | 34 ++++++++++++-------
 3 files changed, 25 insertions(+), 12 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index c5a8994b80..82c542f51d 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -5869,6 +5869,8 @@ py_library(
     deps = [
         ":framework_for_generated_wrappers",
         ":nccl_ops_gen",
+        "//tensorflow/python/eager:context",
+        "//tensorflow/python/eager:def_function",
     ],
 )
 
diff --git a/tensorflow/python/framework/auto_control_deps.py b/tensorflow/python/framework/auto_control_deps.py
index a7d61417bf..da76a84e55 100644
--- a/tensorflow/python/framework/auto_control_deps.py
+++ b/tensorflow/python/framework/auto_control_deps.py
@@ -35,6 +35,7 @@ ASYNC_STATEFUL_OPS = [
     "CollectiveReduce",
     "CollectiveBcastSend",
     "CollectiveBcastRecv",
+    "NcclAllReduce",
 ]
 
 
diff --git a/tensorflow/python/ops/nccl_ops.py b/tensorflow/python/ops/nccl_ops.py
index 6259ce0f94..6c8685cf63 100644
--- a/tensorflow/python/ops/nccl_ops.py
+++ b/tensorflow/python/ops/nccl_ops.py
@@ -19,6 +19,8 @@ from __future__ import print_function
 
 import threading
 
+from tensorflow.python.eager import context
+from tensorflow.python.eager import def_function
 from tensorflow.python.framework import device
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import gen_nccl_ops
@@ -211,19 +213,27 @@ def _apply_all_reduce(reduction, tensors):
     raise ValueError('Must pass >0 tensors to all reduce operations')
 
   shared_name = _get_shared_name()
-  res = []
 
-  for t in tensors:
-    _check_device(t)
-    with ops.device(t.device):
-      res.append(
-          gen_nccl_ops.nccl_all_reduce(
-              input=t,
-              reduction=reduction,
-              num_devices=len(tensors),
-              shared_name=shared_name))
-
-  return res
+  def _all_reduce():
+    """Call nccl allreduce."""
+    res = []
+    for t in tensors:
+      _check_device(t)
+      with ops.device(t.device):
+        res.append(
+            gen_nccl_ops.nccl_all_reduce(
+                input=t,
+                reduction=reduction,
+                num_devices=len(tensors),
+                shared_name=shared_name))
+    return res
+
+  if context.executing_eagerly():
+    # Nccl ops will block unless they are executed concurrently such as in a
+    # graph or a defun.
+    return def_function.function(_all_reduce)()
+  else:
+    return _all_reduce()
 
 
 def _apply_reduce(reduction, tensors):
-- 
GitLab


From e2760cb89f08c3c31ad0322f5b2d8532d0c11b7d Mon Sep 17 00:00:00 2001
From: Ayush Dubey <ayushd@google.com>
Date: Thu, 10 Jan 2019 16:44:20 -0800
Subject: [PATCH 0526/2345] Enable a soft ordering for collectives which adds
 dependencies as node attrs.

Before this change, OrderCollectives would add control edges between collective
ops to make their execution sequential.  This change introduces a technique of
encoding the ordering information as a node attribute.  In future changes, the
collective executor will parse these changes and enforce a fine-grained
ordering between collective ops.

Additionally, this change also
* restricts ordering to CollectiveReduce nodes,
* and fixes a bug so that this ordering is invoked from both DirectSession and
MasterSession.

PiperOrigin-RevId: 228801501
---
 tensorflow/core/BUILD                         |   1 +
 .../common_runtime/build_graph_options.cc     |  13 ++
 .../core/common_runtime/build_graph_options.h |   6 +
 .../core/common_runtime/direct_session.cc     |  11 +-
 .../common_runtime/graph_execution_state.cc   |   7 +
 .../distributed_runtime/master_session.cc     |  45 ++---
 tensorflow/core/graph/collective_order.cc     | 161 +++++++++++++++---
 tensorflow/core/graph/collective_order.h      |  16 +-
 .../core/graph/collective_order_test.cc       |  71 +++++++-
 tensorflow/core/ops/collective_ops.cc         |   1 +
 10 files changed, 268 insertions(+), 64 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index e6af9211b5..690fa4a19e 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -2819,6 +2819,7 @@ tf_cuda_library(
         ":protos_all_cc",
         "//third_party/eigen3",
         "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/container:flat_hash_set",
     ],
 )
 
diff --git a/tensorflow/core/common_runtime/build_graph_options.cc b/tensorflow/core/common_runtime/build_graph_options.cc
index 00f7a8e645..b095fcfa3b 100644
--- a/tensorflow/core/common_runtime/build_graph_options.cc
+++ b/tensorflow/core/common_runtime/build_graph_options.cc
@@ -35,6 +35,19 @@ string BuildGraphOptions::DebugString() const {
   if (collective_graph_key != kNoCollectiveGraphKey) {
     strings::StrAppend(&rv, "\ncollective_graph_key: ", collective_graph_key);
   }
+  string collective_order_str;
+  switch (collective_order) {
+    case GraphCollectiveOrder::kNone:
+      collective_order_str = "none";
+      break;
+    case GraphCollectiveOrder::kEdges:
+      collective_order_str = "edges";
+      break;
+    case GraphCollectiveOrder::kAttrs:
+      collective_order_str = "attrs";
+      break;
+  }
+  strings::StrAppend(&rv, "\ncollective_order: ", collective_order_str);
   return rv;
 }
 
diff --git a/tensorflow/core/common_runtime/build_graph_options.h b/tensorflow/core/common_runtime/build_graph_options.h
index 3d0f242ea5..24b71cc741 100644
--- a/tensorflow/core/common_runtime/build_graph_options.h
+++ b/tensorflow/core/common_runtime/build_graph_options.h
@@ -18,6 +18,7 @@ limitations under the License.
 
 #include <vector>
 
+#include "tensorflow/core/graph/collective_order.h"
 #include "tensorflow/core/platform/types.h"
 #include "tensorflow/core/protobuf/config.pb.h"
 
@@ -34,6 +35,11 @@ struct BuildGraphOptions {
   static const int64 kNoCollectiveGraphKey = 0;
   int64 collective_graph_key = kNoCollectiveGraphKey;
 
+  // If not `kNone`, order all CollectiveReduce operations statically and
+  // deterministically.  If `kEdges`, encode dependencies as explicit control
+  // edges, if `kAttrs` encode as attribute on collective op.
+  GraphCollectiveOrder collective_order = GraphCollectiveOrder::kNone;
+
   string DebugString() const;
 };
 
diff --git a/tensorflow/core/common_runtime/direct_session.cc b/tensorflow/core/common_runtime/direct_session.cc
index 36f1a92aab..8a1c648f61 100644
--- a/tensorflow/core/common_runtime/direct_session.cc
+++ b/tensorflow/core/common_runtime/direct_session.cc
@@ -45,7 +45,6 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/versions.pb.h"
 #include "tensorflow/core/graph/algorithm.h"
-#include "tensorflow/core/graph/collective_order.h"
 #include "tensorflow/core/graph/graph.h"
 #include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/graph/graph_partition.h"
@@ -1192,6 +1191,10 @@ Status DirectSession::CreateExecutors(
   options.use_function_convention = !run_state_args->is_partial_run;
   options.collective_graph_key =
       callable_options.run_options().experimental().collective_graph_key();
+  if (options_.config.experimental()
+          .collective_deterministic_sequential_execution()) {
+    options.collective_order = GraphCollectiveOrder::kEdges;
+  }
 
   std::unique_ptr<FunctionInfo> func_info(new FunctionInfo);
   std::unique_ptr<ExecutorsAndKeys> ek(new ExecutorsAndKeys);
@@ -1523,12 +1526,6 @@ Status DirectSession::CreateGraphs(
     CopyGraph(*execution_state->full_graph(), run_state_args->graph.get());
   }
 
-  // Make collective execution order deterministic if needed.
-  if (options_.config.experimental()
-          .collective_deterministic_sequential_execution()) {
-    TF_RETURN_IF_ERROR(OrderCollectives(&client_graph->graph));
-  }
-
   // Partition the graph across devices.
   PartitionOptions popts;
   popts.node_to_loc = [](const Node* node) {
diff --git a/tensorflow/core/common_runtime/graph_execution_state.cc b/tensorflow/core/common_runtime/graph_execution_state.cc
index 9ecbc34f5f..8875883f62 100644
--- a/tensorflow/core/common_runtime/graph_execution_state.cc
+++ b/tensorflow/core/common_runtime/graph_execution_state.cc
@@ -32,6 +32,7 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor.pb.h"
 #include "tensorflow/core/framework/versions.pb.h"
 #include "tensorflow/core/graph/algorithm.h"
+#include "tensorflow/core/graph/collective_order.h"
 #include "tensorflow/core/graph/graph.h"
 #include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/graph/subgraph.h"
@@ -819,6 +820,12 @@ Status GraphExecutionState::BuildGraph(const BuildGraphOptions& options,
     }
   }
 
+  // Make collective execution order deterministic if needed.
+  if (options.collective_order != GraphCollectiveOrder::kNone) {
+    TF_RETURN_IF_ERROR(
+        OrderCollectives(optimized_graph.get(), options.collective_order));
+  }
+
   // Copy the extracted graph in order to make its node ids dense,
   // since the local CostModel used to record its stats is sized by
   // the largest node id.
diff --git a/tensorflow/core/distributed_runtime/master_session.cc b/tensorflow/core/distributed_runtime/master_session.cc
index 2bb4375831..48b72fb948 100644
--- a/tensorflow/core/distributed_runtime/master_session.cc
+++ b/tensorflow/core/distributed_runtime/master_session.cc
@@ -292,8 +292,8 @@ class MasterSession::ReffedClientGraph : public core::RefCounted {
     if (tot >= 0.1 * 1048576.0) {
       bytes = strings::Printf("[%.1fMB] ", tot / 1048576.0);
     }
-    return strings::StrCat(bytes, stats.node_name(), " = ",
-                           details.type_string, details.detail_text);
+    return strings::StrCat(bytes, stats.node_name(), " = ", details.type_string,
+                           details.detail_text);
   }
 
   // Send/Recv nodes that are the result of client-added
@@ -1081,17 +1081,18 @@ void CopyAndSortStrings(size_t size,
 }  // namespace
 
 void BuildBuildGraphOptions(const RunStepRequestWrapper& req,
+                            const ConfigProto& config,
                             BuildGraphOptions* opts) {
   CallableOptions* callable_opts = &opts->callable_options;
-  CopyAndSortStrings(req.num_feeds(),
-                     [&req](size_t i) { return req.feed_name(i); },
-                     callable_opts->mutable_feed());
-  CopyAndSortStrings(req.num_fetches(),
-                     [&req](size_t i) { return req.fetch_name(i); },
-                     callable_opts->mutable_fetch());
-  CopyAndSortStrings(req.num_targets(),
-                     [&req](size_t i) { return req.target_name(i); },
-                     callable_opts->mutable_target());
+  CopyAndSortStrings(
+      req.num_feeds(), [&req](size_t i) { return req.feed_name(i); },
+      callable_opts->mutable_feed());
+  CopyAndSortStrings(
+      req.num_fetches(), [&req](size_t i) { return req.fetch_name(i); },
+      callable_opts->mutable_fetch());
+  CopyAndSortStrings(
+      req.num_targets(), [&req](size_t i) { return req.target_name(i); },
+      callable_opts->mutable_target());
 
   if (!req.options().debug_options().debug_tensor_watch_opts().empty()) {
     *callable_opts->mutable_run_options()->mutable_debug_options() =
@@ -1100,19 +1101,23 @@ void BuildBuildGraphOptions(const RunStepRequestWrapper& req,
 
   opts->collective_graph_key =
       req.options().experimental().collective_graph_key();
+  if (config.experimental().collective_deterministic_sequential_execution()) {
+    opts->collective_order = GraphCollectiveOrder::kEdges;
+  }
 }
 
 void BuildBuildGraphOptions(const PartialRunSetupRequest& req,
                             BuildGraphOptions* opts) {
   CallableOptions* callable_opts = &opts->callable_options;
-  CopyAndSortStrings(req.feed_size(), [&req](size_t i) { return req.feed(i); },
-                     callable_opts->mutable_feed());
-  CopyAndSortStrings(req.fetch_size(),
-                     [&req](size_t i) { return req.fetch(i); },
-                     callable_opts->mutable_fetch());
-  CopyAndSortStrings(req.target_size(),
-                     [&req](size_t i) { return req.target(i); },
-                     callable_opts->mutable_target());
+  CopyAndSortStrings(
+      req.feed_size(), [&req](size_t i) { return req.feed(i); },
+      callable_opts->mutable_feed());
+  CopyAndSortStrings(
+      req.fetch_size(), [&req](size_t i) { return req.fetch(i); },
+      callable_opts->mutable_fetch());
+  CopyAndSortStrings(
+      req.target_size(), [&req](size_t i) { return req.target(i); },
+      callable_opts->mutable_target());
 
   // TODO(cais): Add TFDBG support to partial runs.
 }
@@ -1852,7 +1857,7 @@ Status MasterSession::DoRunWithLocalExecution(
 
   // Prepare.
   BuildGraphOptions bgopts;
-  BuildBuildGraphOptions(req, &bgopts);
+  BuildBuildGraphOptions(req, session_opts_.config, &bgopts);
   ReffedClientGraph* rcg = nullptr;
   int64 count;
   TF_RETURN_IF_ERROR(StartStep(bgopts, false, &rcg, &count));
diff --git a/tensorflow/core/graph/collective_order.cc b/tensorflow/core/graph/collective_order.cc
index bd8fd767ee..bbba9264ce 100644
--- a/tensorflow/core/graph/collective_order.cc
+++ b/tensorflow/core/graph/collective_order.cc
@@ -14,55 +14,70 @@ limitations under the License.
 ==============================================================================*/
 #include "tensorflow/core/graph/collective_order.h"
 
+#include "absl/container/flat_hash_map.h"
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/graph/algorithm.h"
 
 namespace tensorflow {
+namespace {
 
-Status OrderCollectives(Graph* graph) {
-  // `instance_keys[i]` corresponds to `collective_nodes[i]`
-  std::vector<Node*> collective_nodes;
-  std::vector<int32> instance_keys;
-  // node -> set of collectives on which node depends.
-  std::unordered_map<Node*, std::unordered_set<int32>> node_dependencies;
+// Find all CollectiveReduce nodes and the existing data dependencies between
+// them.
+Status DiscoverDataDependencies(
+    const Graph* graph, std::vector<Node*>* collective_nodes,
+    std::vector<int32>* instance_keys,
+    absl::flat_hash_map<Node*, absl::flat_hash_set<int32>>* data_dependencies) {
   Status s;
-
   // Algorithm: do Reverse DFS starting at sink.  `node_leave` is called when
-  // all parents of `node` have been visited.  At that point, the collectives
-  // on which this node depends on are up to date.  For this node's children,
-  // add all these collectives.  Also, if this node is collective, add as a
-  // dependency for the children.
-  auto node_leave = [&collective_nodes, &instance_keys, &node_dependencies,
+  // all parents of `node` have been visited.  At that point,
+  // `data_dependencies[node]` is a list containing `instance_key` of every
+  // `CollectiveReduce` on which `node` has a data dependency.
+  // For this node's children, add all these instance keys.  Also, if this node
+  // is collective, add as a dependency for the children.
+  auto node_leave = [collective_nodes, instance_keys, data_dependencies,
                      &s](Node* node) {
     int32 instance_key;
-    if (node->IsCollective()) {
+    bool enter_node =
+        node->IsCollective() && node->type_string() == "CollectiveReduce";
+    if (enter_node) {
       Status get_attr_status =
           GetNodeAttr(node->attrs(), "instance_key", &instance_key);
       s.Update(get_attr_status);
-      collective_nodes.push_back(node);
-      instance_keys.push_back(instance_key);
+      collective_nodes->push_back(node);
+      instance_keys->push_back(instance_key);
       VLOG(2) << "collective node " << node->DebugString();
     }
-    const auto& node_deps = node_dependencies[node];
+    const auto& node_deps = (*data_dependencies)[node];
     for (const Edge* out_edge : node->out_edges()) {
-      auto& child_deps = node_dependencies[out_edge->dst()];
+      auto& child_deps = (*data_dependencies)[out_edge->dst()];
       child_deps.insert(node_deps.begin(), node_deps.end());
-      if (node->IsCollective() && s.ok()) {
+      if (enter_node && s.ok()) {
         child_deps.insert(instance_key);
       }
     }
   };
   ReverseDFS(*graph, nullptr, node_leave);
-  if (!s.ok()) return s;
+  return s;
+}
 
-  // For all pairs of collective nodes n1 and n2 on the same device, if n1 does
-  // not depend on n2 and n2 does not depend on n1, then they are potentially
-  // concurrent.  Add an arbitrary, deterministic control edge between them.
+// Given a list of `collective_nodes` and `data_dependencies` between the
+// collective nodes, create control dependencies between concurrent collectives
+// and store in `dependency_edges`.
+// If there exists an edge a -> b then `dependency_edges[a]` contains `b`
+Status CreateControlDependencies(
+    const std::vector<Node*>& collective_nodes,
+    const std::vector<int32>& instance_keys,
+    absl::flat_hash_map<Node*, absl::flat_hash_set<int32>>* data_dependencies,
+    absl::flat_hash_map<Node*, absl::flat_hash_set<Node*>>* dependency_edges) {
+  // If there exists some path a -> ... -> b then `all_paths[a]` contains `b`
+  absl::flat_hash_map<Node*, absl::flat_hash_set<Node*>> all_paths;
   for (int i = 0; i < collective_nodes.size() - 1; i++) {
-    if (!collective_nodes[i]->IsCollective()) {
+    if (!collective_nodes[i]->IsCollective() ||
+        collective_nodes[i]->type_string() != "CollectiveReduce") {
       return errors::Internal("Unexpected node ",
                               collective_nodes[i]->DebugString());
     }
-    const auto& deps_i = node_dependencies[collective_nodes[i]];
+    const auto& deps_i = (*data_dependencies)[collective_nodes[i]];
     for (int j = i + 1; j < collective_nodes.size(); j++) {
       if (collective_nodes[i]->requested_device() !=
           collective_nodes[j]->requested_device()) {
@@ -74,17 +89,48 @@ Status OrderCollectives(Graph* graph) {
                                 " on 2 nodes with the same device ",
                                 collective_nodes[i]->requested_device());
       }
-      const auto& deps_j = node_dependencies[collective_nodes[j]];
+      const auto& deps_j = (*data_dependencies)[collective_nodes[j]];
       if (deps_i.find(instance_keys[j]) == deps_i.end() &&
           deps_j.find(instance_keys[i]) == deps_j.end()) {
         int src_idx = instance_keys[i] < instance_keys[j] ? i : j;
         int dst_idx = instance_keys[i] < instance_keys[j] ? j : i;
         Node* src_node = collective_nodes[src_idx];
         Node* dst_node = collective_nodes[dst_idx];
-        VLOG(1) << "Adding control edge from node " << src_node->name()
+        VLOG(1) << "Adding control dependency from node " << src_node->name()
                 << " instance " << instance_keys[src_idx] << " to node "
                 << dst_node->name() << " instance " << instance_keys[dst_idx];
-        graph->AddControlEdge(src_node, dst_node);
+        (*dependency_edges)[src_node].insert(dst_node);
+        auto& src_paths = all_paths[src_node];
+        src_paths.insert(dst_node);
+        for (Node* downstream_node : all_paths[dst_node]) {
+          src_paths.insert(downstream_node);
+        }
+      }
+    }
+  }
+
+  // Prune dependency edges so that if there are edges a -> b, b -> c, and a ->
+  // c, then remove a -> c.  This dependency would be handled naturally during
+  // op scheduling.
+  for (int i = 0; i < collective_nodes.size(); ++i) {
+    Node* node = collective_nodes[i];
+    auto& neighbor_set = (*dependency_edges)[node];
+    std::vector<Node*> neighbor_list(neighbor_set.begin(), neighbor_set.end());
+    // For all n1, n2 in `neighbor_list` if there is a path from n1 -> n2 then
+    // eliminate n2 from `neighbor_set` and `neighbor_list`.  We remove from
+    // `neighbor_list` by replacing with a `nullptr`, hence the `nullptr` checks
+    // below.
+    for (int j = 0; j < neighbor_list.size(); ++j) {
+      Node* n1 = neighbor_list[j];
+      if (n1 == nullptr) continue;
+      auto& n1_paths = all_paths[n1];
+      for (int k = 0; k < neighbor_list.size(); ++k) {
+        Node* n2 = neighbor_list[k];
+        if (j == k || n2 == nullptr) continue;
+        if (n1_paths.find(n2) != n1_paths.end()) {
+          neighbor_set.erase(n2);
+          neighbor_list[k] = nullptr;
+        }
       }
     }
   }
@@ -92,4 +138,65 @@ Status OrderCollectives(Graph* graph) {
   return Status::OK();
 }
 
+// Insert control dependencies defined by `dependency_edges` in `graph`.  If
+// `order_type` is `kEdges`, insert explicit control edges, else if `order_type`
+// is `kAttrs`, encode depdencies as an attribute on collective node.
+Status InsertControlDependencies(
+    Graph* graph, GraphCollectiveOrder order_type,
+    const absl::flat_hash_map<Node*, absl::flat_hash_set<Node*>>&
+        dependency_edges) {
+  if (order_type == GraphCollectiveOrder::kEdges) {
+    for (const auto& pair : dependency_edges) {
+      Node* src_node = pair.first;
+      for (Node* dst_node : pair.second) {
+        graph->AddControlEdge(src_node, dst_node);
+      }
+    }
+  } else if (order_type == GraphCollectiveOrder::kAttrs) {
+    // `wait_for` is the inverse of `dependency_edges`, i.e. `wait_for[node]`
+    // contains the list of instance keys for which `node` must wait.
+    absl::flat_hash_map<Node*, absl::flat_hash_set<int32>> wait_for;
+    for (const auto& pair : dependency_edges) {
+      int32 src_instance;
+      TF_RETURN_IF_ERROR(
+          GetNodeAttr(pair.first->attrs(), "instance_key", &src_instance));
+      for (Node* dst_node : pair.second) {
+        wait_for[dst_node].insert(src_instance);
+      }
+    }
+    for (const auto& pair : wait_for) {
+      std::vector<int32> wait_for_list(pair.second.begin(), pair.second.end());
+      pair.first->ClearAttr("wait_for");
+      pair.first->AddAttr("wait_for", wait_for_list);
+    }
+  } else {
+    return errors::Internal("Unexpected GraphCollectiveOrder type ",
+                            static_cast<int>(order_type));
+  }
+  return Status::OK();
+}
+
+}  // namespace
+
+Status OrderCollectives(Graph* graph, GraphCollectiveOrder order_type) {
+  // `instance_keys[i]` corresponds to `collective_nodes[i]`
+  std::vector<Node*> collective_nodes;
+  std::vector<int32> instance_keys;
+  // node -> set of collectives on which node depends.
+  absl::flat_hash_map<Node*, absl::flat_hash_set<int32>> data_dependencies;
+  TF_RETURN_IF_ERROR(DiscoverDataDependencies(
+      graph, &collective_nodes, &instance_keys, &data_dependencies));
+
+  if (collective_nodes.empty()) return Status::OK();
+
+  absl::flat_hash_map<Node*, absl::flat_hash_set<Node*>> dependency_edges;
+  // For all pairs of collective nodes n1 and n2 on the same device, if n1 does
+  // not depend on n2 and n2 does not depend on n1, then they are potentially
+  // concurrent.  Create an arbitrary, deterministic ordering between them.
+  TF_RETURN_IF_ERROR(CreateControlDependencies(
+      collective_nodes, instance_keys, &data_dependencies, &dependency_edges));
+
+  return InsertControlDependencies(graph, order_type, dependency_edges);
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/graph/collective_order.h b/tensorflow/core/graph/collective_order.h
index 66210a2ab4..67a1427a96 100644
--- a/tensorflow/core/graph/collective_order.h
+++ b/tensorflow/core/graph/collective_order.h
@@ -19,11 +19,17 @@ limitations under the License.
 
 namespace tensorflow {
 
-// Introduces control edges between potentially concurrent CollectiveOps to make
-// their execution order deterministic. This may be used to execute collectives
-// in the same order across all workers in a distributed execution, if all
-// workers are executing the same graph.
-Status OrderCollectives(Graph* graph);
+enum class GraphCollectiveOrder { kNone, kEdges, kAttrs };
+
+// Introduces a deterministic execution order between potentially concurrent
+// CollectiveOps.  This may be used to execute collectives in the same order
+// across all workers in a distributed execution, if all workers are executing
+// the same graph.
+// If `order_type` is `kEdges`, introduce the ordering in the form of explicit
+// control edges between collective graph nodes.  If `order_type` is `kAttrs`,
+// add an attribute to the node which may be used by collective executor to
+// ensure the required ordering.
+Status OrderCollectives(Graph* graph, GraphCollectiveOrder order_type);
 
 }  // namespace tensorflow
 
diff --git a/tensorflow/core/graph/collective_order_test.cc b/tensorflow/core/graph/collective_order_test.cc
index b0ced7b007..241c98b549 100644
--- a/tensorflow/core/graph/collective_order_test.cc
+++ b/tensorflow/core/graph/collective_order_test.cc
@@ -59,6 +59,23 @@ void VerifyGraph(const Graph& graph,
               UnorderedElementsAreArray(expected_collective_control_edges));
 }
 
+// Verifies that the `wait_for` attribute on collective nodes matches
+// `wait_for_map`.
+void VerifyAttrs(
+    const Graph& graph,
+    const std::unordered_map<string, std::vector<int32>> wait_for_map) {
+  for (const Node* node : graph.nodes()) {
+    if (node->IsCollective() ||
+        wait_for_map.find(node->name()) == wait_for_map.end()) {
+      continue;
+    }
+    std::vector<int32> wait_for_actual;
+    TF_EXPECT_OK(GetNodeAttr(node->attrs(), "wait_for", &wait_for_actual));
+    auto wait_for_expected = wait_for_map.at(node->name());
+    EXPECT_THAT(wait_for_actual, UnorderedElementsAreArray(wait_for_expected));
+  }
+}
+
 Node* CollectiveReduceNode(GraphDefBuilder* builder, Node* input,
                            const string& name, const string& device,
                            int instance_key) {
@@ -123,11 +140,17 @@ std::unique_ptr<Graph> InitGraph() {
 // added after calling `OrderCollectives`: c2_0 -> c3_0 and c2_1 -> c3_1.
 TEST(CollectiveOrderTest, SimpleOrder) {
   std::unique_ptr<Graph> graph = InitGraph();
-  TF_EXPECT_OK(OrderCollectives(graph.get()));
+  TF_EXPECT_OK(OrderCollectives(graph.get(), GraphCollectiveOrder::kEdges));
   VerifyGraph(*graph, {"c1_0", "c1_1", "c2_0", "c2_1", "c3_0", "c3_1"},
               {{"c2_0", "c3_0"}, {"c2_1", "c3_1"}});
 }
 
+TEST(CollectiveOrderTest, SimpleOrderAttr) {
+  std::unique_ptr<Graph> graph = InitGraph();
+  TF_EXPECT_OK(OrderCollectives(graph.get(), GraphCollectiveOrder::kAttrs));
+  VerifyAttrs(*graph, {{"c3_0", {2}}, {"c3_1", {2}}});
+}
+
 // Initialize the following graph:
 //
 //         a
@@ -162,12 +185,50 @@ std::unique_ptr<Graph> InitGraph2() {
 }
 
 // Tests that in the graph created by `InitGraph2`, we add the following control
-// edges after calling `OrderCollectives`: c2 -> c3, c3 -> c4, and c2 -> c4.
+// edges after calling `OrderCollectives`: c2 -> c3, c3 -> c4.  c2->c4 is
+// pruned because it follows from the other two edges.
 TEST(CollectiveOrderTest, SimpleOrder2) {
   std::unique_ptr<Graph> graph = InitGraph2();
-  TF_EXPECT_OK(OrderCollectives(graph.get()));
-  VerifyGraph(*graph, {"c1", "c2", "c3", "c4"},
-              {{"c2", "c3"}, {"c3", "c4"}, {"c2", "c4"}});
+  TF_EXPECT_OK(OrderCollectives(graph.get(), GraphCollectiveOrder::kEdges));
+  VerifyGraph(*graph, {"c1", "c2", "c3", "c4"}, {{"c2", "c3"}, {"c3", "c4"}});
+}
+
+// Initialize the following graph:
+//
+//         w   x   y   z
+//         |   |   |   |
+//         c1  c2  c3  c4
+//
+std::unique_ptr<Graph> InitGraphForPruning() {
+  GraphDefBuilder builder(GraphDefBuilder::kFailImmediately);
+  const string dev0 = "/job:localhost/replica:0/task:0/device:CPU:0";
+  Node* w = ops::SourceOp("TestParams",
+                          builder.opts().WithName("w").WithDevice(dev0));
+  Node* x = ops::SourceOp("TestParams",
+                          builder.opts().WithName("x").WithDevice(dev0));
+  Node* y = ops::SourceOp("TestParams",
+                          builder.opts().WithName("y").WithDevice(dev0));
+  Node* z = ops::SourceOp("TestParams",
+                          builder.opts().WithName("z").WithDevice(dev0));
+  CollectiveReduceNode(&builder, w, "c1", dev0, 1);
+  CollectiveReduceNode(&builder, x, "c2", dev0, 2);
+  CollectiveReduceNode(&builder, y, "c3", dev0, 3);
+  CollectiveReduceNode(&builder, z, "c4", dev0, 4);
+
+  std::unique_ptr<Graph> graph = absl::make_unique<Graph>(OpRegistry::Global());
+  Status s = GraphDefBuilderToGraph(builder, graph.get());
+  if (!s.ok()) {
+    LOG(FATAL) << "Error building graph " << s;
+  }
+  return graph;
+}
+
+// Tests that in the graph created by `InitGraphForPruning`, we only add c1 ->
+// c2, c2 -> c3, c3 -> c4, and other edges are pruned away.
+TEST(CollectiveOrderTest, Pruning) {
+  std::unique_ptr<Graph> graph = InitGraphForPruning();
+  TF_EXPECT_OK(OrderCollectives(graph.get(), GraphCollectiveOrder::kAttrs));
+  VerifyAttrs(*graph, {{"c4", {3}}, {"c3", {2}}, {"c2", {1}}});
 }
 
 }  // namespace
diff --git a/tensorflow/core/ops/collective_ops.cc b/tensorflow/core/ops/collective_ops.cc
index d6157a69df..e45a8a9b36 100644
--- a/tensorflow/core/ops/collective_ops.cc
+++ b/tensorflow/core/ops/collective_ops.cc
@@ -28,6 +28,7 @@ REGISTER_OP("CollectiveReduce")
     .Attr("merge_op: {'Min', 'Max', 'Mul', 'Add'}")
     .Attr("final_op: {'Id', 'Div'}")
     .Attr("subdiv_offsets: list(int)")
+    .Attr("wait_for: list(int) = []")
     .SetIsStateful()
     .SetShapeFn(shape_inference::UnchangedShape);
 
-- 
GitLab


From 0364aef0ba5385b540632d692fcb2843ef382047 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 16:54:28 -0800
Subject: [PATCH 0527/2345] Add registration for (A^-1)^-1 = A.

PiperOrigin-RevId: 228803130
---
 .../python/ops/linalg/inverse_registrations.py       | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/ops/linalg/inverse_registrations.py b/tensorflow/python/ops/linalg/inverse_registrations.py
index b89bdd240e..12d1e7554c 100644
--- a/tensorflow/python/ops/linalg/inverse_registrations.py
+++ b/tensorflow/python/ops/linalg/inverse_registrations.py
@@ -40,6 +40,12 @@ def _inverse_linear_operator(linop):
       is_square=linop.is_square)
 
 
+@linear_operator_algebra.RegisterInverse(
+    linear_operator_inversion.LinearOperatorInversion)
+def _inverse_inverse_linear_operator(linop_inversion):
+  return linop_inversion.operator
+
+
 @linear_operator_algebra.RegisterInverse(
     linear_operator_diag.LinearOperatorDiag)
 def _inverse_diag(diag_operator):
@@ -72,7 +78,7 @@ def _inverse_scaled_identity(identity_operator):
 @linear_operator_algebra.RegisterInverse(
     linear_operator_block_diag.LinearOperatorBlockDiag)
 def _inverse_block_diag(block_diag_operator):
-    # We take the inverse of each block on the diagonal.
+  # We take the inverse of each block on the diagonal.
   return linear_operator_block_diag.LinearOperatorBlockDiag(
       operators=[
           operator.inverse() for operator in block_diag_operator.operators],
@@ -85,8 +91,8 @@ def _inverse_block_diag(block_diag_operator):
 @linear_operator_algebra.RegisterInverse(
     linear_operator_kronecker.LinearOperatorKronecker)
 def _inverse_kronecker(kronecker_operator):
-    # Inverse decomposition of a Kronecker product is the Kronecker product
-    # of inverse decompositions.
+  # Inverse decomposition of a Kronecker product is the Kronecker product
+  # of inverse decompositions.
   return linear_operator_kronecker.LinearOperatorKronecker(
       operators=[
           operator.inverse() for operator in kronecker_operator.operators],
-- 
GitLab


From 5269e3db18693866373a8c774936bbb7515fa027 Mon Sep 17 00:00:00 2001
From: Katherine Wu <kathywu@google.com>
Date: Thu, 10 Jan 2019 17:04:19 -0800
Subject: [PATCH 0528/2345] Move Keras saving and loading functions into
 keras/saving/, and resolve circular dependencies with the SavedModel code.

* saving.py has been split by format (hdf5 and model config)
* saving.py will be deleted after dependencies to keras.engine.saving are all fixed.

PiperOrigin-RevId: 228804552
---
 tensorflow/contrib/saved_model/BUILD          |  29 -
 .../python/saved_model/keras_saved_model.py   | 397 +-------
 .../saved_model/keras_saved_model_test.py     | 538 ----------
 tensorflow/python/keras/BUILD                 |  64 +-
 tensorflow/python/keras/__init__.py           |   2 -
 tensorflow/python/keras/engine/network.py     |   8 +-
 tensorflow/python/keras/engine/saving.py      | 945 +-----------------
 tensorflow/python/keras/engine/training.py    |   3 +-
 .../python/keras/engine/training_utils.py     |  56 --
 .../keras/engine/training_utils_test.py       | 179 ----
 .../keras/layers/cudnn_recurrent_test.py      |   2 +-
 tensorflow/python/keras/models.py             |  13 +-
 tensorflow/python/keras/saving/BUILD          |  72 --
 tensorflow/python/keras/saving/__init__.py    |  16 +-
 tensorflow/python/keras/saving/hdf5_format.py | 895 +++++++++++++++++
 .../hdf5_format_test.py}                      |  20 +-
 .../python/keras/saving/model_config.py       |  96 ++
 tensorflow/python/keras/saving/saved_model.py |  12 +-
 .../python/keras/saving/saving_utils.py       |  60 +-
 .../python/keras/saving/saving_utils_test.py  | 209 ++++
 tensorflow/python/saved_model/BUILD           |   2 +-
 21 files changed, 1369 insertions(+), 2249 deletions(-)
 delete mode 100644 tensorflow/contrib/saved_model/python/saved_model/keras_saved_model_test.py
 delete mode 100644 tensorflow/python/keras/saving/BUILD
 create mode 100644 tensorflow/python/keras/saving/hdf5_format.py
 rename tensorflow/python/keras/{engine/saving_test.py => saving/hdf5_format_test.py} (98%)
 create mode 100644 tensorflow/python/keras/saving/model_config.py
 create mode 100644 tensorflow/python/keras/saving/saving_utils_test.py

diff --git a/tensorflow/contrib/saved_model/BUILD b/tensorflow/contrib/saved_model/BUILD
index 269443b2c6..f0242a3b40 100644
--- a/tensorflow/contrib/saved_model/BUILD
+++ b/tensorflow/contrib/saved_model/BUILD
@@ -84,35 +84,6 @@ py_library(
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
     deps = [
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:lib",
-        "//tensorflow/python:metrics",
-        "//tensorflow/python:platform",
-        "//tensorflow/python:saver",
-        "//tensorflow/python:util",
-        "//tensorflow/python/estimator:estimator_py",
-        "//tensorflow/python/keras:engine",
-        "//tensorflow/python/saved_model",
-    ],
-)
-
-py_test(
-    name = "keras_saved_model_test",
-    size = "medium",
-    srcs = ["python/saved_model/keras_saved_model_test.py"],
-    srcs_version = "PY2AND3",
-    tags = [
-        "no_oss",  # TODO(b/119349471): Re-enable
-        "no_windows",
-    ],
-    deps = [
-        ":keras_saved_model",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:training",
-        "//tensorflow/python/estimator:estimator_py",
         "//tensorflow/python/keras",
-        "//third_party/py/numpy",
-        "@absl_py//absl/testing:parameterized",
     ],
 )
diff --git a/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model.py b/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model.py
index 2a4b6eae36..0392ed9eee 100644
--- a/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model.py
+++ b/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model.py
@@ -18,398 +18,9 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import os
-import six
+from tensorflow.python.keras import saving
 
-from tensorflow.python.client import session
-from tensorflow.python.framework import ops
-from tensorflow.python.keras import backend as K
-from tensorflow.python.keras import models as models_lib
-from tensorflow.python.keras import optimizers
-from tensorflow.python.keras.engine import sequential
-from tensorflow.python.keras.engine import training_utils
-from tensorflow.python.keras.metrics import Metric
-from tensorflow.python.keras.models import model_from_json
-from tensorflow.python.lib.io import file_io
-from tensorflow.python.ops import variables
-from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.saved_model import builder as saved_model_builder
-from tensorflow.python.saved_model import constants
-from tensorflow.python.saved_model import save as save_lib
-from tensorflow.python.saved_model import utils_impl as saved_model_utils
-from tensorflow.python.training import saver as saver_lib
-from tensorflow.python.training.checkpointable import util as checkpointable_utils
-from tensorflow.python.util import compat
-from tensorflow.python.util import nest
-from tensorflow_estimator.python.estimator import keras as estimator_keras_util
-from tensorflow_estimator.python.estimator import model_fn as model_fn_lib
-from tensorflow_estimator.python.estimator.export import export as export_helpers
 
-
-def save_keras_model(
-    model, saved_model_path, custom_objects=None, as_text=None,
-    input_signature=None, serving_only=False):
-  """Saves a `tf.keras.Model` into Tensorflow SavedModel format.
-
-  `save_model` generates new files/folders under the `saved_model_path` folder:
-  1) a checkpoint containing the model weights.
-  2) a saved_model.pb file containing the model's MetaGraphs. The prediction
-     graph is always exported. The evaluaton and training graphs are exported
-     if the following conditions are met:
-     - Evaluation: model loss is defined.
-     - Training: model is compiled with an optimizer defined under `tf.train`.
-       This is because `tf.keras.optimizers.Optimizer` instances cannot be
-       saved to checkpoints.
-  3) Model's json configuration, if model.get_config() has been implemented.
-     This file can be used to reload the model using
-     tf.keras.models.model_from_json(). Note that if any custom objects were
-     used, they should be passed to the `custom_object` argument when loading
-     the model.
-
-  Model limitations:
-  - Sequential and functional models can always be saved.
-  - Subclassed models can only be saved when `serving_only=True`. This is due to
-    the current implementation copying the model in order to export the training
-    and evaluation graphs. Because the topology of subclassed models cannot be
-    determined, the subclassed models cannot be cloned. Subclassed models will
-    be entirely exportable in the future.
-
-  Note that each mode is exported in separate graphs, so different modes do not
-  share variables. To use the train graph with evaluation or prediction graphs,
-  create a new checkpoint if variable values have been updated.
-
-  Example:
-
-  ```python
-  import tensorflow as tf
-
-  # Create a tf.keras model.
-  model = tf.keras.Sequential()
-  model.add(tf.keras.layers.Dense(1, input_shape=[10]))
-  model.summary()
-
-  # Save the tf.keras model in the SavedModel format.
-  saved_to_path = tf.contrib.saved_model.save_keras_model(
-        model, '/tmp/my_simple_tf_keras_saved_model')
-
-  # Load the saved keras model back.
-  model_prime = tf.contrib.saved_model.load_keras_model(saved_to_path)
-  model_prime.summary()
-  ```
-
-  Args:
-    model: A `tf.keras.Model` to be saved. If the model is subclassed, the flag
-      `serving_only` must be set to True.
-    saved_model_path: a string specifying the path to the SavedModel directory.
-      The SavedModel will be saved to a timestamped folder created within this
-      directory.
-    custom_objects: Optional dictionary mapping string names to custom classes
-      or functions (e.g. custom loss functions).
-    as_text: whether to write the `SavedModel` proto in text format. Currently
-      unavailable in serving-only mode.
-    input_signature: A possibly nested sequence of `tf.TensorSpec` objects, used
-      to specify the expected model inputs. `input_signature`'s nested structure
-      should match the expected nested structure of the inputs to the model. If
-      this is not set, this function will attempt to infer the input shapes and
-      dtypes from the model. Note that if the model is subclassed, the tensor
-      inputs to the call function should be nested in the first argument (this
-      is a general requirement for using subclassed models with Keras functions
-      .fit(), .predict(), etc.).
-    serving_only: Export only the outputs produced from calling the model in
-      predict mode. The losses, optimizer, and other training configurations are
-      not saved. If the SavedModel will only be used for serving (rather than
-      retraining), or if the model is subclassed, this can be set to True.
-
-  Returns:
-    String path to the SavedModel folder, a subdirectory of `saved_model_path`.
-
-  Raises:
-    NotImplementedError: If the model is a subclassed model, and serving_only is
-      False.
-    ValueError: If the input signature cannot be inferred from the model.
-  """
-  export_dir = export_helpers.get_timestamped_export_dir(saved_model_path)
-
-  if serving_only:
-    save_lib.save(
-        model, export_dir,
-        signatures=training_utils.trace_model_call(model, input_signature))
-  else:
-    _save_v1_format(model, export_dir, custom_objects, as_text, input_signature)
-
-  try:
-    _export_model_json(model, export_dir)
-  except NotImplementedError:
-    logging.warning('Skipped saving model JSON, subclassed model does not have '
-                    'get_config() defined.')
-
-  return export_dir
-
-
-def _export_model_json(model, saved_model_path):
-  """Saves model configuration as a json string under assets folder."""
-  model_json = model.to_json()
-  model_json_filepath = os.path.join(
-      saved_model_utils.get_or_create_assets_dir(saved_model_path),
-      compat.as_text(constants.SAVED_MODEL_FILENAME_JSON))
-  file_io.write_string_to_file(model_json_filepath, model_json)
-
-
-def _export_model_variables(model, saved_model_path):
-  """Saves model weights in checkpoint format under variables folder."""
-  saved_model_utils.get_or_create_variables_dir(saved_model_path)
-  checkpoint_prefix = saved_model_utils.get_variables_path(saved_model_path)
-  model.save_weights(checkpoint_prefix, save_format='tf', overwrite=True)
-  return checkpoint_prefix
-
-
-def _save_v1_format(model, path, custom_objects, as_text, input_signature):
-  """Exports model to v1 SavedModel format."""
-  if not model._is_graph_network:
-    if isinstance(model, sequential.Sequential):
-      # If input shape is not directly set in the model, the exported model
-      # will infer the expected shapes of the input from the model.
-      if not model.built and input_signature is None:
-        raise ValueError(
-            'Sequential model\'s input shape is unknown. Please build the '
-            'model, or use the input_signature argument to specify the '
-            'model inputs.')
-    else:
-      raise NotImplementedError(
-          'Subclassed models can only be exported for serving. Please set '
-          'argument serving_only=True.')
-
-  builder = saved_model_builder._SavedModelBuilder(path)
-
-  # Manually save variables to export them in an object-based checkpoint. This
-  # skips the `builder.add_meta_graph_and_variables()` step, which saves a
-  # named-based checkpoint.
-  # TODO(b/113134168): Add fn to Builder to save with object-based saver.
-  # TODO(b/113178242): This should only export the model json structure. Only
-  # one save is needed once the weights can be copied from the model to clone.
-  checkpoint_path = _export_model_variables(model, path)
-
-  # Export each mode. Use ModeKeys enums defined for `Estimator` to ensure that
-  # Keras models and `Estimator`s are exported with the same format.
-  # Every time a mode is exported, the code checks to see if new variables have
-  # been created (e.g. optimizer slot variables). If that is the case, the
-  # checkpoint is re-saved to include the new variables.
-  export_args = {'builder': builder,
-                 'model': model,
-                 'custom_objects': custom_objects,
-                 'checkpoint_path': checkpoint_path,
-                 'input_signature': input_signature}
-
-  has_saved_vars = False
-  if model.optimizer:
-    # TODO(kathywu): Verify this works with v2 optimizer.
-    if isinstance(model.optimizer, optimizers.TFOptimizer):
-      _export_mode(model_fn_lib.ModeKeys.TRAIN, has_saved_vars, **export_args)
-      has_saved_vars = True
-      _export_mode(model_fn_lib.ModeKeys.EVAL, has_saved_vars, **export_args)
-    else:
-      logging.warning(
-          'Model was compiled with an optimizer, but the optimizer is not from '
-          '`tf.train` (e.g. `tf.train.AdagradOptimizer`). Only the serving '
-          'graph was exported. The train and evaluate graphs were not added to '
-          'the SavedModel.')
-  _export_mode(model_fn_lib.ModeKeys.PREDICT, has_saved_vars, **export_args)
-
-  builder.save(as_text)
-
-
-def _get_var_list(model):
-  """Returns list of all checkpointed saveable objects in the model."""
-  return checkpointable_utils.named_saveables(model)
-
-
-def create_placeholder(spec):
-  return K.placeholder(shape=spec.shape, dtype=spec.dtype, name=spec.name)
-
-
-def _export_mode(
-    mode, has_saved_vars, builder, model, custom_objects, checkpoint_path,
-    input_signature):
-  """Exports a model, and optionally saves new vars from the clone model.
-
-  Args:
-    mode: A `tf.estimator.ModeKeys` string.
-    has_saved_vars: A `boolean` indicating whether the SavedModel has already
-      exported variables.
-    builder: A `SavedModelBuilder` object.
-    model: A `tf.keras.Model` object.
-    custom_objects: A dictionary mapping string names to custom classes
-      or functions.
-    checkpoint_path: String path to checkpoint.
-    input_signature: Nested TensorSpec containing the expected inputs. Can be
-      `None`, in which case the signature will be inferred from the model.
-
-  Raises:
-    ValueError: If the train/eval mode is being exported, but the model does
-      not have an optimizer.
-  """
-  compile_clone = (mode != model_fn_lib.ModeKeys.PREDICT)
-  if compile_clone and not model.optimizer:
-    raise ValueError(
-        'Model does not have an optimizer. Cannot export mode %s' % mode)
-
-  model_graph = ops.get_default_graph()
-  with ops.Graph().as_default() as g:
-
-    K.set_learning_phase(mode == model_fn_lib.ModeKeys.TRAIN)
-
-    if input_signature is None:
-      input_tensors = None
-    else:
-      input_tensors = nest.map_structure(create_placeholder, input_signature)
-
-    # Clone the model into blank graph. This will create placeholders for inputs
-    # and targets.
-    clone = models_lib.clone_and_build_model(
-        model, input_tensors=input_tensors, custom_objects=custom_objects,
-        compile_clone=compile_clone)
-
-    # Make sure that iterations variable is added to the global step collection,
-    # to ensure that, when the SavedModel graph is loaded, the iterations
-    # variable is returned by `tf.train.get_global_step()`. This is required for
-    # compatibility with the SavedModelEstimator.
-    if compile_clone:
-      g.add_to_collection(ops.GraphKeys.GLOBAL_STEP, clone.optimizer.iterations)
-
-    # Extract update and train ops from train/test/predict functions.
-    train_op = None
-    if mode == model_fn_lib.ModeKeys.TRAIN:
-      clone._make_train_function()
-      train_op = clone.train_function.updates_op
-    elif mode == model_fn_lib.ModeKeys.EVAL:
-      clone._make_test_function()
-    else:
-      clone._make_predict_function()
-    g.get_collection_ref(ops.GraphKeys.UPDATE_OPS).extend(clone.state_updates)
-
-    clone_var_list = checkpointable_utils.named_saveables(clone)
-
-    with session.Session().as_default():
-      if has_saved_vars:
-        # Confirm all variables in the clone have an entry in the checkpoint.
-        status = clone.load_weights(checkpoint_path)
-        status.assert_existing_objects_matched()
-      else:
-        # Confirm that variables between the clone and model match up exactly,
-        # not counting optimizer objects. Optimizer objects are ignored because
-        # if the model has not trained, the slot variables will not have been
-        # created yet.
-        # TODO(b/113179535): Replace with checkpointable equivalence.
-        _assert_same_non_optimizer_objects(model, model_graph, clone, g)
-
-        # TODO(b/113178242): Use value transfer for checkpointable objects.
-        clone.load_weights(checkpoint_path)
-
-        # Add graph and variables to SavedModel.
-        # TODO(b/113134168): Switch to add_meta_graph_and_variables.
-        clone.save_weights(checkpoint_path, save_format='tf', overwrite=True)
-        builder._has_saved_variables = True
-
-    # Add graph to the SavedModel builder.
-    builder.add_meta_graph(
-        model_fn_lib.EXPORT_TAG_MAP[mode],
-        signature_def_map=_create_signature_def_map(clone, mode),
-        saver=saver_lib.Saver(clone_var_list),
-        init_op=variables.local_variables_initializer(),
-        train_op=train_op)
-    return None
-
-
-def _create_signature_def_map(model, mode):
-  """Creates a SignatureDef map from a Keras model."""
-  inputs_dict = {name: x for name, x in zip(model.input_names, model.inputs)}
-  if model.optimizer:
-    targets_dict = {x.name.split(':')[0]: x
-                    for x in model.targets if x is not None}
-    inputs_dict.update(targets_dict)
-  outputs_dict = {name: x
-                  for name, x in zip(model.output_names, model.outputs)}
-  metrics = estimator_keras_util._convert_keras_metrics_to_estimator(model)
-
-  # Add metric variables to the `LOCAL_VARIABLES` collection. Metric variables
-  # are by default not added to any collections. We are doing this here, so
-  # that metric variables get initialized.
-  local_vars = set(ops.get_collection(ops.GraphKeys.LOCAL_VARIABLES))
-  vars_to_add = set()
-  if metrics is not None:
-    for key, value in six.iteritems(metrics):
-      if isinstance(value, Metric):
-        vars_to_add.update(value.variables)
-        # Convert Metric instances to (value_tensor, update_op) tuple.
-        metrics[key] = (value.result(), value.updates[0])
-  # Remove variables that are in the local variables collection already.
-  vars_to_add = vars_to_add.difference(local_vars)
-  for v in vars_to_add:
-    ops.add_to_collection(ops.GraphKeys.LOCAL_VARIABLES, v)
-
-  export_outputs = model_fn_lib.export_outputs_for_mode(
-      mode,
-      predictions=outputs_dict,
-      loss=model.total_loss if model.optimizer else None,
-      metrics=metrics)
-  return export_helpers.build_all_signature_defs(
-      inputs_dict,
-      export_outputs=export_outputs,
-      serving_only=(mode == model_fn_lib.ModeKeys.PREDICT))
-
-
-def _assert_same_non_optimizer_objects(model, model_graph, clone, clone_graph):  # pylint: disable=unused-argument
-  """Asserts model and clone contain the same checkpointable objects."""
-
-  # TODO(fchollet, kathywu): make sure this works in eager mode.
-  return True
-
-
-def load_keras_model(saved_model_path):
-  """Loads a keras.Model from SavedModel.
-
-  load_model reinstantiates model state by:
-  1) loading model topology from json (this will eventually come
-     from metagraph).
-  2) loading model weights from checkpoint.
-
-  Example:
-
-  ```python
-  import tensorflow as tf
-
-  # Create a tf.keras model.
-  model = tf.keras.Sequential()
-  model.add(tf.keras.layers.Dense(1, input_shape=[10]))
-  model.summary()
-
-  # Save the tf.keras model in the SavedModel format.
-  saved_to_path = tf.contrib.saved_model.save_keras_model(
-        model, '/tmp/my_simple_tf_keras_saved_model')
-
-  # Load the saved keras model back.
-  model_prime = tf.contrib.saved_model.load_keras_model(saved_to_path)
-  model_prime.summary()
-  ```
-
-  Args:
-    saved_model_path: a string specifying the path to an existing SavedModel.
-
-  Returns:
-    a keras.Model instance.
-  """
-  # restore model topology from json string
-  model_json_filepath = os.path.join(
-      compat.as_bytes(saved_model_path),
-      compat.as_bytes(constants.ASSETS_DIRECTORY),
-      compat.as_bytes(constants.SAVED_MODEL_FILENAME_JSON))
-  model_json = file_io.read_file_to_string(model_json_filepath)
-  model = model_from_json(model_json)
-
-  # restore model weights
-  checkpoint_prefix = os.path.join(
-      compat.as_text(saved_model_path),
-      compat.as_text(constants.VARIABLES_DIRECTORY),
-      compat.as_text(constants.VARIABLES_FILENAME))
-  model.load_weights(checkpoint_prefix)
-  return model
+# TODO(kathywu): Remove all contrib callers, switch to tf.keras.
+save_keras_model = saving.export
+load_keras_model = saving.load_from_saved_model
diff --git a/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model_test.py b/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model_test.py
deleted file mode 100644
index fbf8138493..0000000000
--- a/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model_test.py
+++ /dev/null
@@ -1,538 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-# pylint: disable=protected-access
-"""Tests for saving/loading function for keras Model."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-import shutil
-
-from absl.testing import parameterized
-import numpy as np
-
-from tensorflow.contrib.saved_model.python.saved_model import keras_saved_model
-from tensorflow.python import keras
-from tensorflow.python.client import session
-from tensorflow.python.eager import context
-from tensorflow.python.estimator import model_fn as model_fn_lib
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_spec
-from tensorflow.python.framework import test_util
-from tensorflow.python.keras.engine import training
-from tensorflow.python.keras.utils import tf_utils
-from tensorflow.python.ops import array_ops
-from tensorflow.python.platform import test
-from tensorflow.python.saved_model import loader_impl
-from tensorflow.python.saved_model import signature_constants
-from tensorflow.python.training import training as training_module
-
-
-class TestModelSavingandLoading(test.TestCase):
-
-  def _save_model_dir(self, dirname='saved_model'):
-    temp_dir = self.get_temp_dir()
-    self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True)
-    return os.path.join(temp_dir, dirname)
-
-  def test_saving_sequential_model(self):
-    with self.cached_session():
-      model = keras.models.Sequential()
-      model.add(keras.layers.Dense(2, input_shape=(3,)))
-      model.add(keras.layers.RepeatVector(3))
-      model.add(keras.layers.TimeDistributed(keras.layers.Dense(3)))
-      model.compile(
-          loss=keras.losses.MSE,
-          optimizer=keras.optimizers.RMSprop(lr=0.0001),
-          metrics=[keras.metrics.categorical_accuracy],
-          sample_weight_mode='temporal')
-      x = np.random.random((1, 3))
-      y = np.random.random((1, 3, 3))
-      model.train_on_batch(x, y)
-
-      ref_y = model.predict(x)
-
-      temp_saved_model = self._save_model_dir()
-      output_path = keras_saved_model.save_keras_model(model, temp_saved_model)
-
-      loaded_model = keras_saved_model.load_keras_model(output_path)
-      y = loaded_model.predict(x)
-      self.assertAllClose(ref_y, y, atol=1e-05)
-
-  @test_util.run_in_graph_and_eager_modes
-  def test_saving_sequential_model_without_compile(self):
-    with self.cached_session():
-      model = keras.models.Sequential()
-      model.add(keras.layers.Dense(2, input_shape=(3,)))
-      model.add(keras.layers.RepeatVector(3))
-      model.add(keras.layers.TimeDistributed(keras.layers.Dense(3)))
-
-      x = np.random.random((1, 3))
-      ref_y = model.predict(x)
-
-      temp_saved_model = self._save_model_dir()
-      output_path = keras_saved_model.save_keras_model(model, temp_saved_model)
-      loaded_model = keras_saved_model.load_keras_model(output_path)
-
-      y = loaded_model.predict(x)
-      self.assertAllClose(ref_y, y, atol=1e-05)
-
-  def test_saving_functional_model(self):
-    with self.cached_session():
-      inputs = keras.layers.Input(shape=(3,))
-      x = keras.layers.Dense(2)(inputs)
-      output = keras.layers.Dense(3)(x)
-
-      model = keras.models.Model(inputs, output)
-      model.compile(
-          loss=keras.losses.MSE,
-          optimizer=keras.optimizers.RMSprop(lr=0.0001),
-          metrics=[keras.metrics.categorical_accuracy])
-      x = np.random.random((1, 3))
-      y = np.random.random((1, 3))
-      model.train_on_batch(x, y)
-
-      ref_y = model.predict(x)
-
-      temp_saved_model = self._save_model_dir()
-      output_path = keras_saved_model.save_keras_model(model, temp_saved_model)
-      loaded_model = keras_saved_model.load_keras_model(output_path)
-
-      y = loaded_model.predict(x)
-      self.assertAllClose(ref_y, y, atol=1e-05)
-
-  @test_util.run_in_graph_and_eager_modes
-  def test_saving_functional_model_without_compile(self):
-    with self.cached_session():
-      inputs = keras.layers.Input(shape=(3,))
-      x = keras.layers.Dense(2)(inputs)
-      output = keras.layers.Dense(3)(x)
-
-      model = keras.models.Model(inputs, output)
-
-      x = np.random.random((1, 3))
-      y = np.random.random((1, 3))
-
-      ref_y = model.predict(x)
-
-      temp_saved_model = self._save_model_dir()
-      output_path = keras_saved_model.save_keras_model(model, temp_saved_model)
-      loaded_model = keras_saved_model.load_keras_model(output_path)
-
-      y = loaded_model.predict(x)
-      self.assertAllClose(ref_y, y, atol=1e-05)
-
-  @test_util.run_in_graph_and_eager_modes
-  def test_saving_with_tf_optimizer(self):
-    with self.cached_session():
-      model = keras.models.Sequential()
-      model.add(keras.layers.Dense(2, input_shape=(3,)))
-      model.add(keras.layers.Dense(3))
-      model.compile(
-          loss='mse',
-          optimizer=training_module.RMSPropOptimizer(0.1),
-          metrics=['acc'])
-
-      x = np.random.random((1, 3))
-      y = np.random.random((1, 3))
-      model.train_on_batch(x, y)
-      ref_y = model.predict(x)
-
-      temp_saved_model = self._save_model_dir()
-      output_path = keras_saved_model.save_keras_model(model, temp_saved_model)
-      loaded_model = keras_saved_model.load_keras_model(output_path)
-      loaded_model.compile(
-          loss='mse',
-          optimizer=training_module.RMSPropOptimizer(0.1),
-          metrics=['acc'])
-      y = loaded_model.predict(x)
-      self.assertAllClose(ref_y, y, atol=1e-05)
-
-      # test that new updates are the same with both models
-      x = np.random.random((1, 3))
-      y = np.random.random((1, 3))
-
-      ref_loss = model.train_on_batch(x, y)
-      loss = loaded_model.train_on_batch(x, y)
-      self.assertAllClose(ref_loss, loss, atol=1e-05)
-
-      ref_y = model.predict(x)
-      y = loaded_model.predict(x)
-      self.assertAllClose(ref_y, y, atol=1e-05)
-
-      # test saving/loading again
-      temp_saved_model2 = self._save_model_dir('saved_model_2')
-      output_path2 = keras_saved_model.save_keras_model(
-          loaded_model, temp_saved_model2)
-      loaded_model = keras_saved_model.load_keras_model(output_path2)
-      y = loaded_model.predict(x)
-      self.assertAllClose(ref_y, y, atol=1e-05)
-
-  def test_saving_subclassed_model_raise_error(self):
-    # For now, saving subclassed model should raise an error. It should be
-    # avoided later with loading from SavedModel.pb.
-
-    class SubclassedModel(training.Model):
-
-      def __init__(self):
-        super(SubclassedModel, self).__init__()
-        self.layer1 = keras.layers.Dense(3)
-        self.layer2 = keras.layers.Dense(1)
-
-      def call(self, inp):
-        return self.layer2(self.layer1(inp))
-
-    model = SubclassedModel()
-
-    temp_saved_model = self._save_model_dir()
-    with self.assertRaises(NotImplementedError):
-      keras_saved_model.save_keras_model(model, temp_saved_model)
-
-
-class LayerWithLearningPhase(keras.engine.base_layer.Layer):
-
-  def call(self, x):
-    phase = keras.backend.learning_phase()
-    output = tf_utils.smart_cond(
-        phase, lambda: x * 0, lambda: array_ops.identity(x))
-    if not context.executing_eagerly():
-      output._uses_learning_phase = True  # pylint: disable=protected-access
-    return output
-
-  def compute_output_shape(self, input_shape):
-    return input_shape
-
-
-def functional_model(uses_learning_phase=True):
-  inputs = keras.layers.Input(shape=(3,))
-  x = keras.layers.Dense(2)(inputs)
-  x = keras.layers.Dense(3)(x)
-  if uses_learning_phase:
-    x = LayerWithLearningPhase()(x)
-  return keras.models.Model(inputs, x)
-
-
-def sequential_model(uses_learning_phase=True):
-  model = keras.models.Sequential()
-  model.add(keras.layers.Dense(2, input_shape=(3,)))
-  model.add(keras.layers.Dense(3))
-  if uses_learning_phase:
-    model.add(LayerWithLearningPhase())
-  return model
-
-
-def sequential_model_without_input_shape(uses_learning_phase=True):
-  model = keras.models.Sequential()
-  model.add(keras.layers.Dense(2))
-  model.add(keras.layers.Dense(3))
-  if uses_learning_phase:
-    model.add(LayerWithLearningPhase())
-  return model
-
-
-class Subclassed(keras.models.Model):
-
-  def __init__(self):
-    super(Subclassed, self).__init__()
-    self.dense1 = keras.layers.Dense(2)
-    self.dense2 = keras.layers.Dense(3)
-
-  def call(self, inputs):
-    x = self.dense1(inputs)
-    x = self.dense2(x)
-    return x
-
-
-def subclassed_model():
-  return Subclassed()
-
-
-def load_model(sess, path, mode):
-  tags = model_fn_lib.EXPORT_TAG_MAP[mode]
-  if mode == model_fn_lib.ModeKeys.PREDICT:
-    sig_def_key = signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
-  else:
-    sig_def_key = mode
-
-  meta_graph_def = loader_impl.load(sess, tags, path)
-  inputs = {
-      k: sess.graph.get_tensor_by_name(v.name)
-      for k, v in meta_graph_def.signature_def[sig_def_key].inputs.items()}
-  outputs = {
-      k: sess.graph.get_tensor_by_name(v.name)
-      for k, v in meta_graph_def.signature_def[sig_def_key].outputs.items()}
-  return inputs, outputs, meta_graph_def
-
-
-@test_util.run_all_in_graph_and_eager_modes
-class TestModelSavedModelExport(test.TestCase, parameterized.TestCase):
-
-  def _save_model_dir(self, dirname='saved_model'):
-    temp_dir = self.get_temp_dir()
-    self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True)
-    return os.path.join(temp_dir, dirname)
-
-  @parameterized.parameters(
-      {
-          'model_builder': functional_model,
-          'uses_learning_phase': True,
-          'optimizer': training_module.AdadeltaOptimizer(),
-          'train_before_export': True},
-      {
-          'model_builder': functional_model,
-          'uses_learning_phase': True,
-          'optimizer': training_module.AdadeltaOptimizer(),
-          'train_before_export': False},
-      {
-          'model_builder': functional_model,
-          'uses_learning_phase': False,
-          'optimizer': None,
-          'train_before_export': False},
-      {
-          'model_builder': sequential_model,
-          'uses_learning_phase': True,
-          'optimizer': training_module.AdadeltaOptimizer(),
-          'train_before_export': True},
-      {
-          'model_builder': sequential_model,
-          'uses_learning_phase': True,
-          'optimizer': training_module.AdadeltaOptimizer(),
-          'train_before_export': False},
-      {
-          'model_builder': sequential_model,
-          'uses_learning_phase': False,
-          'optimizer': None,
-          'train_before_export': False},
-      {
-          'model_builder': sequential_model_without_input_shape,
-          'uses_learning_phase': True,
-          'optimizer': training_module.AdadeltaOptimizer(),
-          'train_before_export': False})
-  def testSaveAndLoadSavedModelExport(
-      self, model_builder, uses_learning_phase, optimizer, train_before_export):
-    saved_model_path = self._save_model_dir()
-    with self.session(graph=ops.Graph()):
-      np.random.seed(130)
-      input_arr = np.random.random((1, 3))
-      target_arr = np.random.random((1, 3))
-
-      model = model_builder(uses_learning_phase)
-      if optimizer is not None:
-        model.compile(
-            loss='mse',
-            optimizer=optimizer,
-            metrics=['mae'])
-        if train_before_export:
-          model.train_on_batch(input_arr, target_arr)
-
-        ref_loss, ref_mae = model.evaluate(input_arr, target_arr)
-
-      ref_predict = model.predict(input_arr)
-
-      # Export SavedModel
-      output_path = keras_saved_model.save_keras_model(model, saved_model_path)
-
-    input_name = model.input_names[0]
-    output_name = model.output_names[0]
-    target_name = output_name + '_target'
-
-    # Load predict graph, and test predictions
-    with session.Session(graph=ops.Graph()) as sess:
-      inputs, outputs, _ = load_model(sess, output_path,
-                                      model_fn_lib.ModeKeys.PREDICT)
-
-      predictions = sess.run(outputs[output_name],
-                             {inputs[input_name]: input_arr})
-      self.assertAllClose(ref_predict, predictions, atol=1e-05)
-
-    if optimizer:
-      # Load eval graph, and test predictions, loss and metric values
-      with session.Session(graph=ops.Graph()) as sess:
-        inputs, outputs, _ = load_model(sess, output_path,
-                                        model_fn_lib.ModeKeys.EVAL)
-
-        # First obtain the loss and predictions, and run the metric update op by
-        # feeding in the inputs and targets.
-        loss, predictions, _ = sess.run(
-            (outputs['loss'], outputs['predictions/' + output_name],
-             outputs['metrics/mean_absolute_error/update_op']), {
-                 inputs[input_name]: input_arr,
-                 inputs[target_name]: target_arr
-             })
-
-        # The metric value should be run after the update op, to ensure that it
-        # reflects the correct value.
-        metric_value = sess.run(outputs['metrics/mean_absolute_error/value'])
-
-        self.assertEqual(int(train_before_export),
-                         sess.run(training_module.get_global_step()))
-        self.assertAllClose(ref_loss, loss, atol=1e-05)
-        self.assertAllClose(ref_mae, metric_value, atol=1e-05)
-        self.assertAllClose(ref_predict, predictions, atol=1e-05)
-
-      # Load train graph, and check for the train op, and prediction values
-      with session.Session(graph=ops.Graph()) as sess:
-        inputs, outputs, meta_graph_def = load_model(
-            sess, output_path, model_fn_lib.ModeKeys.TRAIN)
-        self.assertEqual(int(train_before_export),
-                         sess.run(training_module.get_global_step()))
-        self.assertIn('loss', outputs)
-        self.assertIn('metrics/mean_absolute_error/update_op', outputs)
-        self.assertIn('metrics/mean_absolute_error/value', outputs)
-        self.assertIn('predictions/' + output_name, outputs)
-
-        # Train for a step
-        train_op = loader_impl.get_train_op(meta_graph_def)
-        train_outputs, _ = sess.run(
-            [outputs, train_op], {inputs[input_name]: input_arr,
-                                  inputs[target_name]: target_arr})
-        self.assertEqual(int(train_before_export) + 1,
-                         sess.run(training_module.get_global_step()))
-
-        if uses_learning_phase:
-          self.assertAllClose(
-              [[0, 0, 0]], train_outputs['predictions/' + output_name],
-              atol=1e-05)
-        else:
-          self.assertNotAllClose(
-              [[0, 0, 0]], train_outputs['predictions/' + output_name],
-              atol=1e-05)
-
-  def testSaveAndLoadSavedModelWithCustomObject(self):
-    saved_model_path = self._save_model_dir()
-    with session.Session(graph=ops.Graph()) as sess:
-      def relu6(x):
-        return keras.backend.relu(x, max_value=6)
-      inputs = keras.layers.Input(shape=(1,))
-      outputs = keras.layers.Activation(relu6)(inputs)
-      model = keras.models.Model(inputs, outputs)
-      output_path = keras_saved_model.save_keras_model(
-          model, saved_model_path, custom_objects={'relu6': relu6})
-    with session.Session(graph=ops.Graph()) as sess:
-      inputs, outputs, _ = load_model(sess, output_path,
-                                      model_fn_lib.ModeKeys.PREDICT)
-      input_name = model.input_names[0]
-      output_name = model.output_names[0]
-      predictions = sess.run(
-          outputs[output_name], {inputs[input_name]: [[7], [-3], [4]]})
-      self.assertAllEqual([[6], [0], [4]], predictions)
-
-  def testAssertModelCloneSameObjectsIgnoreOptimizer(self):
-    input_arr = np.random.random((1, 3))
-    target_arr = np.random.random((1, 3))
-
-    model_graph = ops.Graph()
-    clone_graph = ops.Graph()
-
-    # Create two models with the same layers but different optimizers.
-    with session.Session(graph=model_graph):
-      inputs = keras.layers.Input(shape=(3,))
-      x = keras.layers.Dense(2)(inputs)
-      x = keras.layers.Dense(3)(x)
-      model = keras.models.Model(inputs, x)
-
-      model.compile(loss='mse', optimizer=training_module.AdadeltaOptimizer())
-      model.train_on_batch(input_arr, target_arr)
-
-    with session.Session(graph=clone_graph):
-      inputs = keras.layers.Input(shape=(3,))
-      x = keras.layers.Dense(2)(inputs)
-      x = keras.layers.Dense(3)(x)
-      clone = keras.models.Model(inputs, x)
-      clone.compile(loss='mse', optimizer=keras.optimizers.RMSprop(lr=0.0001))
-      clone.train_on_batch(input_arr, target_arr)
-
-    keras_saved_model._assert_same_non_optimizer_objects(
-        model, model_graph, clone, clone_graph)
-
-  def testAssertModelCloneSameObjectsThrowError(self):
-    input_arr = np.random.random((1, 3))
-    target_arr = np.random.random((1, 3))
-
-    model_graph = ops.Graph()
-    clone_graph = ops.Graph()
-
-    # Create two models with the same layers but different optimizers.
-    with session.Session(graph=model_graph):
-      inputs = keras.layers.Input(shape=(3,))
-      x = keras.layers.Dense(2)(inputs)
-      x = keras.layers.Dense(3)(x)
-      model = keras.models.Model(inputs, x)
-
-      model.compile(loss='mse', optimizer=training_module.AdadeltaOptimizer())
-      model.train_on_batch(input_arr, target_arr)
-
-    with session.Session(graph=clone_graph):
-      inputs = keras.layers.Input(shape=(3,))
-      x = keras.layers.Dense(2)(inputs)
-      x = keras.layers.Dense(4)(x)
-      x = keras.layers.Dense(3)(x)
-      clone = keras.models.Model(inputs, x)
-      clone.compile(loss='mse', optimizer=keras.optimizers.RMSprop(lr=0.0001))
-      clone.train_on_batch(input_arr, target_arr)
-
-  def testSaveSequentialModelWithoutInputShapes(self):
-    model = sequential_model_without_input_shape(True)
-    # A Sequential model that hasn't been built should raise an error.
-    with self.assertRaisesRegexp(ValueError, 'Please build the model'):
-      keras_saved_model.save_keras_model(model, '')
-
-    saved_model_path = self._save_model_dir()
-    output_path = keras_saved_model.save_keras_model(
-        model, saved_model_path,
-        input_signature=tensor_spec.TensorSpec(shape=(10, 11, 12, 13, 14),
-                                               dtype=dtypes.float32,
-                                               name='spec_input'))
-
-    with session.Session(graph=ops.Graph()) as sess:
-      inputs, outputs, _ = load_model(sess, output_path,
-                                      model_fn_lib.ModeKeys.PREDICT)
-      self.assertEqual(5, inputs[next(iter(inputs.keys()))].shape.ndims)
-      self.assertEqual(5, outputs[next(iter(outputs.keys()))].shape.ndims)
-      self.assertEqual(3, outputs[next(iter(outputs.keys()))].shape[-1])
-
-  @test_util.run_v2_only
-  @parameterized.parameters(
-      {
-          'model_builder': sequential_model_without_input_shape,
-          'input_signature': [tensor_spec.TensorSpec(shape=[None, 3],
-                                                     dtype=dtypes.float32)]},
-      {
-          'model_builder': subclassed_model,
-          'input_signature': [tensor_spec.TensorSpec(shape=[None, 3],
-                                                     dtype=dtypes.float32)]})
-  def testServingOnly(self, model_builder, input_signature):
-    saved_model_path = self._save_model_dir()
-    input_arr = np.random.random((5, 3)).astype(np.float32)
-    model = model_builder()
-    ref_predict = model.predict(input_arr)
-
-    output_path = keras_saved_model.save_keras_model(
-        model, saved_model_path, serving_only=True,
-        input_signature=input_signature)
-
-    # Load predict graph, and test predictions
-    with session.Session(graph=ops.Graph()) as sess:
-      inputs, outputs, _ = load_model(sess, output_path,
-                                      model_fn_lib.ModeKeys.PREDICT)
-      predictions = sess.run(outputs[next(iter(outputs.keys()))],
-                             {inputs[next(iter(inputs.keys()))]: input_arr})
-      self.assertAllClose(ref_predict, predictions, atol=1e-05)
-
-
-if __name__ == '__main__':
-  test.main()
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 536acca298..5f44a1b4e8 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -61,13 +61,11 @@ py_library(
         ":engine",
         ":layers",
         ":pil_for_keras",
-        "@keras_applications_archive//:keras_applications",
+        ":saving",
         "//tensorflow/python:training",
         "//tensorflow/python/keras/optimizer_v2",
-        # TODO(kathywu): move saving into engine after resolving circular
-        # dependencies between Keras and SavedModel
-        "//tensorflow/python/keras/saving",
         "//tensorflow/python/saved_model",
+        "@keras_applications_archive//:keras_applications",
     ],
 )
 
@@ -152,6 +150,7 @@ py_library(
         ":losses",
         ":optimizers",
         ":regularizers",
+        ":saving",
         "//tensorflow/python/data",
         "//tensorflow/python/distribute:reduce_util",
         "//tensorflow/python/training/checkpointable:data_structures",
@@ -160,6 +159,28 @@ py_library(
     ],
 )
 
+py_library(
+    name = "saving",
+    srcs = [
+        "saving/__init__.py",
+        "saving/hdf5_format.py",
+        "saving/model_config.py",
+        "saving/saved_model.py",
+        "saving/saving_utils.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":backend",
+        ":engine_utils",
+        ":optimizers",
+        "//tensorflow/python:lib",
+        "//tensorflow/python:mode_keys",
+        "//tensorflow/python:saver",
+        "//tensorflow/python/saved_model",
+        "//tensorflow/python/saved_model/model_utils",
+    ],
+)
+
 py_library(
     name = "activations",
     srcs = [
@@ -248,6 +269,7 @@ py_library(
 py_library(
     name = "engine_utils",
     srcs = [
+        "utils/conv_utils.py",
         "utils/data_utils.py",
         "utils/io_utils.py",
         "utils/losses_utils.py",
@@ -276,7 +298,6 @@ py_library(
         "layers/recurrent.py",
         "layers/serialization.py",
         "layers/wrappers.py",
-        "utils/conv_utils.py",
         "utils/generic_utils.py",
         "utils/layer_utils.py",
         "utils/tf_utils.py",
@@ -1001,9 +1022,9 @@ tf_py_test(
 )
 
 tf_py_test(
-    name = "saving_test",
+    name = "hdf5_format_test",
     size = "medium",
-    srcs = ["engine/saving_test.py"],
+    srcs = ["saving/hdf5_format_test.py"],
     additional_deps = [
         ":keras",
         "@absl_py//absl/testing:parameterized",
@@ -1064,3 +1085,32 @@ tf_py_test(
     ],
     tags = ["notsan"],
 )
+
+tf_py_test(
+    name = "saved_model_test",
+    size = "medium",
+    srcs = ["saving/saved_model_test.py"],
+    additional_deps = [
+        ":keras",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+    ],
+    tags = [
+        "no_oss",  # TODO(b/119349471): Re-enable
+        "no_windows",
+    ],
+)
+
+tf_py_test(
+    name = "saving_utils_test",
+    size = "medium",
+    srcs = ["saving/saving_utils_test.py"],
+    additional_deps = [
+        ":keras",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+    ],
+    tags = ["notsan"],
+)
diff --git a/tensorflow/python/keras/__init__.py b/tensorflow/python/keras/__init__.py
index e59744f64d..f024b9b59a 100644
--- a/tensorflow/python/keras/__init__.py
+++ b/tensorflow/python/keras/__init__.py
@@ -42,8 +42,6 @@ from tensorflow.python.keras import wrappers
 from tensorflow.python.keras.layers import Input
 from tensorflow.python.keras.models import Model
 from tensorflow.python.keras.models import Sequential
-from tensorflow.python.keras.saving.saved_model import export
-from tensorflow.python.keras.saving.saved_model import load_from_saved_model
 
 from tensorflow.python.util.tf_export import keras_export
 
diff --git a/tensorflow/python/keras/engine/network.py b/tensorflow/python/keras/engine/network.py
index 6c3691a30c..991950993f 100644
--- a/tensorflow/python/keras/engine/network.py
+++ b/tensorflow/python/keras/engine/network.py
@@ -37,8 +37,8 @@ from tensorflow.python.framework import tensor_shape
 from tensorflow.python.keras import backend
 from tensorflow.python.keras.engine import base_layer
 from tensorflow.python.keras.engine import base_layer_utils
-from tensorflow.python.keras.engine import saving
 from tensorflow.python.keras.engine import training_utils
+from tensorflow.python.keras.saving import hdf5_format
 from tensorflow.python.keras.utils import generic_utils
 from tensorflow.python.keras.utils import layer_utils
 from tensorflow.python.keras.utils import tf_utils
@@ -1366,7 +1366,7 @@ class Network(base_layer.Layer):
         return
     if save_format == 'h5':
       with h5py.File(filepath, 'w') as f:
-        saving.save_weights_to_hdf5_group(f, self.layers)
+        hdf5_format.save_weights_to_hdf5_group(f, self.layers)
     else:
       if context.executing_eagerly():
         session = None
@@ -1466,9 +1466,9 @@ class Network(base_layer.Layer):
       if 'layer_names' not in f.attrs and 'model_weights' in f:
         f = f['model_weights']
       if by_name:
-        saving.load_weights_from_hdf5_group_by_name(f, self.layers)
+        hdf5_format.load_weights_from_hdf5_group_by_name(f, self.layers)
       else:
-        saving.load_weights_from_hdf5_group(f, self.layers)
+        hdf5_format.load_weights_from_hdf5_group(f, self.layers)
 
   def _updated_config(self):
     """Util shared between different serialization methods.
diff --git a/tensorflow/python/keras/engine/saving.py b/tensorflow/python/keras/engine/saving.py
index 0604a389a9..b4da86d984 100644
--- a/tensorflow/python/keras/engine/saving.py
+++ b/tensorflow/python/keras/engine/saving.py
@@ -14,950 +14,11 @@
 # ==============================================================================
 # pylint: disable=protected-access
 """Model saving utilities.
+
+Everything has been moved to keras/saving/. This file will be deleted soon.
 """
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import json
-import os
-
-import numpy as np
-from six.moves import zip  # pylint: disable=redefined-builtin
-
-from tensorflow.python.framework import ops
-from tensorflow.python.keras import backend as K
-from tensorflow.python.keras import optimizers
-from tensorflow.python.keras.utils import conv_utils
-from tensorflow.python.keras.utils.io_utils import ask_to_proceed_with_overwrite
-from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.util import serialization
-from tensorflow.python.util.tf_export import keras_export
-
-# pylint: disable=g-import-not-at-top
-try:
-  import h5py
-  HDF5_OBJECT_HEADER_LIMIT = 64512
-except ImportError:
-  h5py = None
-
-try:
-  import yaml
-except ImportError:
-  yaml = None
-# pylint: enable=g-import-not-at-top
-
-
-@keras_export('keras.models.save_model')
-def save_model(model, filepath, overwrite=True, include_optimizer=True):
-  """Saves a model to a HDF5 file.
-
-  The saved model contains:
-      - the model's configuration (topology)
-      - the model's weights
-      - the model's optimizer's state (if any)
-
-  Thus the saved model can be reinstantiated in
-  the exact same state, without any of the code
-  used for model definition or training.
-
-  Arguments:
-      model: Keras model instance to be saved.
-      filepath: One of the following:
-          - String, path where to save the model
-          - `h5py.File` object where to save the model
-      overwrite: Whether we should overwrite any existing
-          model at the target location, or instead
-          ask the user with a manual prompt.
-      include_optimizer: If True, save optimizer's state together.
-
-  Raises:
-      ImportError: if h5py is not available.
-  """
-
-  if h5py is None:
-    raise ImportError('`save_model` requires h5py.')
-
-  from tensorflow.python.keras import __version__ as keras_version  # pylint: disable=g-import-not-at-top
-
-  # TODO(psv) Add warning when we save models that contain non-serializable
-  # entities like metrics added using `add_metric` and losses added using
-  # `add_loss.`
-
-  if not isinstance(filepath, h5py.File):
-    # If file exists and should not be overwritten.
-    if not overwrite and os.path.isfile(filepath):
-      proceed = ask_to_proceed_with_overwrite(filepath)
-      if not proceed:
-        return
-
-    f = h5py.File(filepath, mode='w')
-    opened_new_file = True
-  else:
-    f = filepath
-    opened_new_file = False
-
-  try:
-    f.attrs['keras_version'] = str(keras_version).encode('utf8')
-    f.attrs['backend'] = K.backend().encode('utf8')
-    f.attrs['model_config'] = json.dumps(
-        {
-            'class_name': model.__class__.__name__,
-            'config': model.get_config()
-        },
-        default=serialization.get_json_type).encode('utf8')
-
-    model_weights_group = f.create_group('model_weights')
-    model_layers = model.layers
-    save_weights_to_hdf5_group(model_weights_group, model_layers)
-
-    if include_optimizer and model.optimizer:
-      if isinstance(model.optimizer, optimizers.TFOptimizer):
-        logging.warning(
-            'TensorFlow optimizers do not '
-            'make it possible to access '
-            'optimizer attributes or optimizer state '
-            'after instantiation. '
-            'As a result, we cannot save the optimizer '
-            'as part of the model save file.'
-            'You will have to compile your model again after loading it. '
-            'Prefer using a Keras optimizer instead '
-            '(see keras.io/optimizers).')
-      else:
-        f.attrs['training_config'] = json.dumps(
-            {
-                'optimizer_config': {
-                    'class_name': model.optimizer.__class__.__name__,
-                    'config': model.optimizer.get_config()
-                },
-                'loss': model.loss,
-                'metrics': model._compile_metrics,
-                'weighted_metrics': model._compile_weighted_metrics,
-                'sample_weight_mode': model.sample_weight_mode,
-                'loss_weights': model.loss_weights,
-            },
-            default=serialization.get_json_type).encode('utf8')
-
-        # Save optimizer weights.
-        symbolic_weights = getattr(model.optimizer, 'weights')
-        if symbolic_weights:
-          optimizer_weights_group = f.create_group('optimizer_weights')
-          weight_values = K.batch_get_value(symbolic_weights)
-          weight_names = []
-          for w, val in zip(symbolic_weights, weight_values):
-            name = str(w.name)
-            weight_names.append(name.encode('utf8'))
-          optimizer_weights_group.attrs['weight_names'] = weight_names
-          for name, val in zip(weight_names, weight_values):
-            param_dset = optimizer_weights_group.create_dataset(
-                name, val.shape, dtype=val.dtype)
-            if not val.shape:
-              # scalar
-              param_dset[()] = val
-            else:
-              param_dset[:] = val
-    f.flush()
-  finally:
-    if opened_new_file:
-      f.close()
-
-
-@keras_export('keras.models.load_model')
-def load_model(filepath, custom_objects=None, compile=True):  # pylint: disable=redefined-builtin
-  """Loads a model saved via `save_model`.
-
-  Arguments:
-      filepath: One of the following:
-          - String, path to the saved model
-          - `h5py.File` object from which to load the model
-      custom_objects: Optional dictionary mapping names
-          (strings) to custom classes or functions to be
-          considered during deserialization.
-      compile: Boolean, whether to compile the model
-          after loading.
-
-  Returns:
-      A Keras model instance. If an optimizer was found
-      as part of the saved model, the model is already
-      compiled. Otherwise, the model is uncompiled and
-      a warning will be displayed. When `compile` is set
-      to False, the compilation is omitted without any
-      warning.
-
-  Raises:
-      ImportError: if h5py is not available.
-      ValueError: In case of an invalid savefile.
-  """
-  if h5py is None:
-    raise ImportError('`load_model` requires h5py.')
-
-  if not custom_objects:
-    custom_objects = {}
-
-  def convert_custom_objects(obj):
-    """Handles custom object lookup.
-
-    Arguments:
-        obj: object, dict, or list.
-
-    Returns:
-        The same structure, where occurrences
-            of a custom object name have been replaced
-            with the custom object.
-    """
-    if isinstance(obj, list):
-      deserialized = []
-      for value in obj:
-        deserialized.append(convert_custom_objects(value))
-      return deserialized
-    if isinstance(obj, dict):
-      deserialized = {}
-      for key, value in obj.items():
-        deserialized[key] = convert_custom_objects(value)
-      return deserialized
-    if obj in custom_objects:
-      return custom_objects[obj]
-    return obj
-
-  opened_new_file = not isinstance(filepath, h5py.File)
-  if opened_new_file:
-    f = h5py.File(filepath, mode='r')
-  else:
-    f = filepath
-
-  model = None
-  try:
-    # instantiate model
-    model_config = f.attrs.get('model_config')
-    if model_config is None:
-      raise ValueError('No model found in config file.')
-    model_config = json.loads(model_config.decode('utf-8'))
-    model = model_from_config(model_config, custom_objects=custom_objects)
-
-    # set weights
-    load_weights_from_hdf5_group(f['model_weights'], model.layers)
-
-    if compile:
-      # instantiate optimizer
-      training_config = f.attrs.get('training_config')
-      if training_config is None:
-        logging.warning('No training configuration found in save file: '
-                        'the model was *not* compiled. Compile it manually.')
-        return model
-      training_config = json.loads(training_config.decode('utf-8'))
-      optimizer_config = training_config['optimizer_config']
-      optimizer = optimizers.deserialize(
-          optimizer_config, custom_objects=custom_objects)
-
-      # Recover loss functions and metrics.
-      loss = convert_custom_objects(training_config['loss'])
-      metrics = convert_custom_objects(training_config['metrics'])
-      weighted_metrics = convert_custom_objects(
-          training_config.get('weighted_metrics', None))
-      sample_weight_mode = training_config['sample_weight_mode']
-      loss_weights = training_config['loss_weights']
-
-      # Compile model.
-      model.compile(
-          optimizer=optimizer,
-          loss=loss,
-          metrics=metrics,
-          weighted_metrics=weighted_metrics,
-          loss_weights=loss_weights,
-          sample_weight_mode=sample_weight_mode)
-
-      # Set optimizer weights.
-      if 'optimizer_weights' in f:
-        # Build train function (to get weight updates).
-        # Models that aren't graph networks must wait until they are called
-        # with data to _make_train_function() and so can't load optimizer
-        # weights.
-        if model._is_graph_network:  # pylint: disable=protected-access
-          model._make_train_function()
-          optimizer_weights_group = f['optimizer_weights']
-          optimizer_weight_names = [
-              n.decode('utf8')
-              for n in optimizer_weights_group.attrs['weight_names']
-          ]
-          optimizer_weight_values = [
-              optimizer_weights_group[n] for n in optimizer_weight_names
-          ]
-          try:
-            model.optimizer.set_weights(optimizer_weight_values)
-          except ValueError:
-            logging.warning('Error in loading the saved optimizer '
-                            'state. As a result, your model is '
-                            'starting with a freshly initialized '
-                            'optimizer.')
-        else:
-          logging.warning('Sequential models without an `input_shape` '
-                          'passed to the first layer cannot reload their '
-                          'optimizer state. As a result, your model is'
-                          'starting with a freshly initialized optimizer.')
-
-  finally:
-    if opened_new_file:
-      f.close()
-  return model
-
-
-@keras_export('keras.models.model_from_config')
-def model_from_config(config, custom_objects=None):
-  """Instantiates a Keras model from its config.
-
-  Arguments:
-      config: Configuration dictionary.
-      custom_objects: Optional dictionary mapping names
-          (strings) to custom classes or functions to be
-          considered during deserialization.
-
-  Returns:
-      A Keras model instance (uncompiled).
-
-  Raises:
-      TypeError: if `config` is not a dictionary.
-  """
-  if isinstance(config, list):
-    raise TypeError('`model_from_config` expects a dictionary, not a list. '
-                    'Maybe you meant to use '
-                    '`Sequential.from_config(config)`?')
-  from tensorflow.python.keras.layers import deserialize  # pylint: disable=g-import-not-at-top
-  return deserialize(config, custom_objects=custom_objects)
-
-
-@keras_export('keras.models.model_from_yaml')
-def model_from_yaml(yaml_string, custom_objects=None):
-  """Parses a yaml model configuration file and returns a model instance.
-
-  Arguments:
-      yaml_string: YAML string encoding a model configuration.
-      custom_objects: Optional dictionary mapping names
-          (strings) to custom classes or functions to be
-          considered during deserialization.
-
-  Returns:
-      A Keras model instance (uncompiled).
-
-  Raises:
-      ImportError: if yaml module is not found.
-  """
-  if yaml is None:
-    raise ImportError('Requires yaml module installed (`pip install pyyaml`).')
-  config = yaml.load(yaml_string)
-  from tensorflow.python.keras.layers import deserialize  # pylint: disable=g-import-not-at-top
-  return deserialize(config, custom_objects=custom_objects)
-
-
-@keras_export('keras.models.model_from_json')
-def model_from_json(json_string, custom_objects=None):
-  """Parses a JSON model configuration file and returns a model instance.
-
-  Arguments:
-      json_string: JSON string encoding a model configuration.
-      custom_objects: Optional dictionary mapping names
-          (strings) to custom classes or functions to be
-          considered during deserialization.
-
-  Returns:
-      A Keras model instance (uncompiled).
-  """
-  config = json.loads(json_string)
-  from tensorflow.python.keras.layers import deserialize  # pylint: disable=g-import-not-at-top
-  return deserialize(config, custom_objects=custom_objects)
-
-
-def preprocess_weights_for_loading(layer,
-                                   weights,
-                                   original_keras_version=None,
-                                   original_backend=None):
-  """Preprocess layer weights between different Keras formats.
-
-  Converts layers weights from Keras 1 format to Keras 2 and also weights of
-  CuDNN layers in Keras 2.
-
-  Arguments:
-      layer: Layer instance.
-      weights: List of weights values (Numpy arrays).
-      original_keras_version: Keras version for the weights, as a string.
-      original_backend: Keras backend the weights were trained with,
-          as a string.
-
-  Returns:
-      A list of weights values (Numpy arrays).
-  """
-  def convert_nested_bidirectional(weights):
-    """Converts layers nested in `Bidirectional` wrapper.
-
-    This function uses `preprocess_weights_for_loading()` for converting
-    layers.
-
-    Arguments:
-        weights: List of weights values (Numpy arrays).
-
-    Returns:
-        A list of weights values (Numpy arrays).
-    """
-    num_weights_per_layer = len(weights) // 2
-    forward_weights = preprocess_weights_for_loading(
-        layer.forward_layer, weights[:num_weights_per_layer],
-        original_keras_version, original_backend)
-    backward_weights = preprocess_weights_for_loading(
-        layer.backward_layer, weights[num_weights_per_layer:],
-        original_keras_version, original_backend)
-    return forward_weights + backward_weights
-
-  def convert_nested_time_distributed(weights):
-    """Converts layers nested in `TimeDistributed` wrapper.
-
-    This function uses `preprocess_weights_for_loading()` for converting nested
-    layers.
-
-    Arguments:
-        weights: List of weights values (Numpy arrays).
-
-    Returns:
-        A list of weights values (Numpy arrays).
-    """
-    return preprocess_weights_for_loading(
-        layer.layer, weights, original_keras_version, original_backend)
-
-  def convert_nested_model(weights):
-    """Converts layers nested in `Model` or `Sequential`.
-
-    This function uses `preprocess_weights_for_loading()` for converting nested
-    layers.
-
-    Arguments:
-        weights: List of weights values (Numpy arrays).
-
-    Returns:
-        A list of weights values (Numpy arrays).
-    """
-    new_weights = []
-    # trainable weights
-    for sublayer in layer.layers:
-      num_weights = len(sublayer.trainable_weights)
-      if num_weights > 0:
-        new_weights.extend(preprocess_weights_for_loading(
-            layer=sublayer,
-            weights=weights[:num_weights],
-            original_keras_version=original_keras_version,
-            original_backend=original_backend))
-        weights = weights[num_weights:]
-
-    # non-trainable weights
-    for sublayer in layer.layers:
-      num_weights = len([l for l in sublayer.weights
-                         if l not in sublayer.trainable_weights])
-      if num_weights > 0:
-        new_weights.extend(preprocess_weights_for_loading(
-            layer=sublayer,
-            weights=weights[:num_weights],
-            original_keras_version=original_keras_version,
-            original_backend=original_backend))
-        weights = weights[num_weights:]
-    return new_weights
-
-  # Convert layers nested in Bidirectional/Model/Sequential.
-  # Both transformation should be ran for both Keras 1->2 conversion
-  # and for conversion of CuDNN layers.
-  if layer.__class__.__name__ == 'Bidirectional':
-    weights = convert_nested_bidirectional(weights)
-  if layer.__class__.__name__ == 'TimeDistributed':
-    weights = convert_nested_time_distributed(weights)
-  elif layer.__class__.__name__ in ['Model', 'Sequential']:
-    weights = convert_nested_model(weights)
-
-  if original_keras_version == '1':
-    if layer.__class__.__name__ == 'TimeDistributed':
-      weights = preprocess_weights_for_loading(
-          layer.layer, weights, original_keras_version, original_backend)
-
-    if layer.__class__.__name__ == 'Conv1D':
-      shape = weights[0].shape
-      # Handle Keras 1.1 format
-      if shape[:2] != (layer.kernel_size[0], 1) or shape[3] != layer.filters:
-        # Legacy shape:
-        # (filters, input_dim, filter_length, 1)
-        assert shape[0] == layer.filters and shape[2:] == (layer.kernel_size[0],
-                                                           1)
-        weights[0] = np.transpose(weights[0], (2, 3, 1, 0))
-      weights[0] = weights[0][:, 0, :, :]
-
-    if layer.__class__.__name__ == 'Conv2D':
-      if layer.data_format == 'channels_first':
-        # old: (filters, stack_size, kernel_rows, kernel_cols)
-        # new: (kernel_rows, kernel_cols, stack_size, filters)
-        weights[0] = np.transpose(weights[0], (2, 3, 1, 0))
-
-    if layer.__class__.__name__ == 'Conv2DTranspose':
-      if layer.data_format == 'channels_last':
-        # old: (kernel_rows, kernel_cols, stack_size, filters)
-        # new: (kernel_rows, kernel_cols, filters, stack_size)
-        weights[0] = np.transpose(weights[0], (0, 1, 3, 2))
-      if layer.data_format == 'channels_first':
-        # old: (filters, stack_size, kernel_rows, kernel_cols)
-        # new: (kernel_rows, kernel_cols, filters, stack_size)
-        weights[0] = np.transpose(weights[0], (2, 3, 0, 1))
-
-    if layer.__class__.__name__ == 'Conv3D':
-      if layer.data_format == 'channels_first':
-        # old: (filters, stack_size, ...)
-        # new: (..., stack_size, filters)
-        weights[0] = np.transpose(weights[0], (2, 3, 4, 1, 0))
-
-    if layer.__class__.__name__ == 'GRU':
-      if len(weights) == 9:
-        kernel = np.concatenate([weights[0], weights[3], weights[6]], axis=-1)
-        recurrent_kernel = np.concatenate(
-            [weights[1], weights[4], weights[7]], axis=-1)
-        bias = np.concatenate([weights[2], weights[5], weights[8]], axis=-1)
-        weights = [kernel, recurrent_kernel, bias]
-
-    if layer.__class__.__name__ == 'LSTM':
-      if len(weights) == 12:
-        # old: i, c, f, o
-        # new: i, f, c, o
-        kernel = np.concatenate(
-            [weights[0], weights[6], weights[3], weights[9]], axis=-1)
-        recurrent_kernel = np.concatenate(
-            [weights[1], weights[7], weights[4], weights[10]], axis=-1)
-        bias = np.concatenate(
-            [weights[2], weights[8], weights[5], weights[11]], axis=-1)
-        weights = [kernel, recurrent_kernel, bias]
-
-    if layer.__class__.__name__ == 'ConvLSTM2D':
-      if len(weights) == 12:
-        kernel = np.concatenate(
-            [weights[0], weights[6], weights[3], weights[9]], axis=-1)
-        recurrent_kernel = np.concatenate(
-            [weights[1], weights[7], weights[4], weights[10]], axis=-1)
-        bias = np.concatenate(
-            [weights[2], weights[8], weights[5], weights[11]], axis=-1)
-        if layer.data_format == 'channels_first':
-          # old: (filters, stack_size, kernel_rows, kernel_cols)
-          # new: (kernel_rows, kernel_cols, stack_size, filters)
-          kernel = np.transpose(kernel, (2, 3, 1, 0))
-          recurrent_kernel = np.transpose(recurrent_kernel, (2, 3, 1, 0))
-        weights = [kernel, recurrent_kernel, bias]
-
-  conv_layers = ['Conv1D', 'Conv2D', 'Conv3D', 'Conv2DTranspose', 'ConvLSTM2D']
-  if layer.__class__.__name__ in conv_layers:
-    if original_backend == 'theano':
-      weights[0] = conv_utils.convert_kernel(weights[0])
-      if layer.__class__.__name__ == 'ConvLSTM2D':
-        weights[1] = conv_utils.convert_kernel(weights[1])
-    if K.int_shape(layer.weights[0]) != weights[0].shape:
-      weights[0] = np.transpose(weights[0], (3, 2, 0, 1))
-      if layer.__class__.__name__ == 'ConvLSTM2D':
-        weights[1] = np.transpose(weights[1], (3, 2, 0, 1))
-
-  # convert CuDNN layers
-  return _convert_rnn_weights(layer, weights)
-
-
-def _convert_rnn_weights(layer, weights):
-  """Converts weights for RNN layers between native and CuDNN format.
-
-  Input kernels for each gate are transposed and converted between Fortran
-  and C layout, recurrent kernels are transposed. For LSTM biases are summed/
-  split in half, for GRU biases are reshaped.
-
-  Weights can be converted in both directions between `LSTM` and`CuDNNSLTM`
-  and between `CuDNNGRU` and `GRU(reset_after=True)`. Default `GRU` is not
-  compatible with `CuDNNGRU`.
-
-  For missing biases in `LSTM`/`GRU` (`use_bias=False`) no conversion is made.
-
-  Arguments:
-      layer: Target layer instance.
-      weights: List of source weights values (input kernels, recurrent
-          kernels, [biases]) (Numpy arrays).
-
-  Returns:
-      A list of converted weights values (Numpy arrays).
-
-  Raises:
-      ValueError: for incompatible GRU layer/weights or incompatible biases
-  """
-
-  def transform_kernels(kernels, func, n_gates):
-    """Transforms kernel for each gate separately using given function.
-
-    Arguments:
-        kernels: Stacked array of kernels for individual gates.
-        func: Function applied to kernel of each gate.
-        n_gates: Number of gates (4 for LSTM, 3 for GRU).
-
-    Returns:
-        Stacked array of transformed kernels.
-    """
-    return np.hstack([func(k) for k in np.hsplit(kernels, n_gates)])
-
-  def transpose_input(from_cudnn):
-    """Makes a function that transforms input kernels from/to CuDNN format.
-
-    It keeps the shape, but changes between the layout (Fortran/C). Eg.:
-
-    ```
-    Keras                 CuDNN
-    [[0, 1, 2],  <--->  [[0, 2, 4],
-     [3, 4, 5]]          [1, 3, 5]]
-    ```
-
-    It can be passed to `transform_kernels()`.
-
-    Arguments:
-        from_cudnn: `True` if source weights are in CuDNN format, `False`
-            if they're in plain Keras format.
-
-    Returns:
-        Function that converts input kernel to the other format.
-    """
-    order = 'F' if from_cudnn else 'C'
-
-    def transform(kernel):
-      return kernel.T.reshape(kernel.shape, order=order)
-
-    return transform
-
-  target_class = layer.__class__.__name__
-
-  # convert the weights between CuDNNLSTM and LSTM
-  if target_class in ['LSTM', 'CuDNNLSTM'] and len(weights) == 3:
-    # determine if we're loading a CuDNNLSTM layer
-    # from the number of bias weights:
-    # CuDNNLSTM has (units * 8) weights; while LSTM has (units * 4)
-    # if there's no bias weight in the file, skip this conversion
-    units = weights[1].shape[0]
-    bias_shape = weights[2].shape
-    n_gates = 4
-
-    if bias_shape == (2 * units * n_gates,):
-      source = 'CuDNNLSTM'
-    elif bias_shape == (units * n_gates,):
-      source = 'LSTM'
-    else:
-      raise ValueError('Invalid bias shape: ' + str(bias_shape))
-
-    def convert_lstm_weights(weights, from_cudnn=True):
-      """Converts the weights between CuDNNLSTM and LSTM.
-
-      Arguments:
-        weights: Original weights.
-        from_cudnn: Indicates whether original weights are from CuDNN layer.
-
-      Returns:
-        Updated weights compatible with LSTM.
-      """
-
-      # Transpose (and reshape) input and recurrent kernels
-      kernels = transform_kernels(weights[0], transpose_input(from_cudnn),
-                                  n_gates)
-      recurrent_kernels = transform_kernels(weights[1], lambda k: k.T, n_gates)
-      if from_cudnn:
-        # merge input and recurrent biases into a single set
-        biases = np.sum(np.split(weights[2], 2, axis=0), axis=0)
-      else:
-        # Split single set of biases evenly to two sets. The way of
-        # splitting doesn't matter as long as the two sets sum is kept.
-        biases = np.tile(0.5 * weights[2], 2)
-      return [kernels, recurrent_kernels, biases]
-
-    if source != target_class:
-      weights = convert_lstm_weights(weights, from_cudnn=source == 'CuDNNLSTM')
-
-  # convert the weights between CuDNNGRU and GRU(reset_after=True)
-  if target_class in ['GRU', 'CuDNNGRU'] and len(weights) == 3:
-    # We can determine the source of the weights from the shape of the bias.
-    # If there is no bias we skip the conversion since
-    # CuDNNGRU always has biases.
-
-    units = weights[1].shape[0]
-    bias_shape = weights[2].shape
-    n_gates = 3
-
-    def convert_gru_weights(weights, from_cudnn=True):
-      """Converts the weights between CuDNNGRU and GRU.
-
-      Arguments:
-        weights: Original weights.
-        from_cudnn: Indicates whether original weights are from CuDNN layer.
-
-      Returns:
-        Updated weights compatible with GRU.
-      """
-
-      kernels = transform_kernels(weights[0], transpose_input(from_cudnn),
-                                  n_gates)
-      recurrent_kernels = transform_kernels(weights[1], lambda k: k.T, n_gates)
-      biases = np.array(weights[2]).reshape((2, -1) if from_cudnn else -1)
-      return [kernels, recurrent_kernels, biases]
-
-    if bias_shape == (2 * units * n_gates,):
-      source = 'CuDNNGRU'
-    elif bias_shape == (2, units * n_gates):
-      source = 'GRU(reset_after=True)'
-    elif bias_shape == (units * n_gates,):
-      source = 'GRU(reset_after=False)'
-    else:
-      raise ValueError('Invalid bias shape: ' + str(bias_shape))
-
-    if target_class == 'CuDNNGRU':
-      target = 'CuDNNGRU'
-    elif layer.reset_after:
-      target = 'GRU(reset_after=True)'
-    else:
-      target = 'GRU(reset_after=False)'
-
-    # only convert between different types
-    if source != target:
-      types = (source, target)
-      if 'GRU(reset_after=False)' in types:
-        raise ValueError('%s is not compatible with %s' % types)
-      if source == 'CuDNNGRU':
-        weights = convert_gru_weights(weights, from_cudnn=True)
-      elif source == 'GRU(reset_after=True)':
-        weights = convert_gru_weights(weights, from_cudnn=False)
-
-  return weights
-
-
-def save_weights_to_hdf5_group(f, layers):
-  """Saves the weights of a list of layers to a HDF5 group.
-
-  Arguments:
-      f: HDF5 group.
-      layers: List of layer instances.
-  """
-  from tensorflow.python.keras import __version__ as keras_version  # pylint: disable=g-import-not-at-top
-
-  save_attributes_to_hdf5_group(
-      f, 'layer_names', [layer.name.encode('utf8') for layer in layers])
-  f.attrs['backend'] = K.backend().encode('utf8')
-  f.attrs['keras_version'] = str(keras_version).encode('utf8')
-
-  # On TPUs, modifying the graph between session.runs() triggers some expensive
-  # recompilation overhead. To avoid this, we build up the full set of tensors
-  # to save before fetching weights, thus only modifying the graph once.
-  layer_weights_dict = {}
-  for layer in layers:
-    layer_weights_dict[layer.name] = [ops.convert_to_tensor(w)
-                                      for w in layer.weights]
-
-  for layer in layers:
-    g = f.create_group(layer.name)
-    symbolic_weights = layer_weights_dict[layer.name]
-    weight_values = K.batch_get_value(symbolic_weights)
-    weight_names = []
-    for i, (w, val) in enumerate(zip(symbolic_weights, weight_values)):
-      if hasattr(w, 'name') and w.name:
-        name = str(w.name)
-      else:
-        name = 'param_' + str(i)
-      weight_names.append(name.encode('utf8'))
-    save_attributes_to_hdf5_group(g, 'weight_names', weight_names)
-    for name, val in zip(weight_names, weight_values):
-      param_dset = g.create_dataset(name, val.shape, dtype=val.dtype)
-      if not val.shape:
-        # scalar
-        param_dset[()] = val
-      else:
-        param_dset[:] = val
-
-
-def load_weights_from_hdf5_group(f, layers):
-  """Implements topological (order-based) weight loading.
-
-  Arguments:
-      f: A pointer to a HDF5 group.
-      layers: a list of target layers.
-
-  Raises:
-      ValueError: in case of mismatch between provided layers
-          and weights file.
-  """
-  if 'keras_version' in f.attrs:
-    original_keras_version = f.attrs['keras_version'].decode('utf8')
-  else:
-    original_keras_version = '1'
-  if 'backend' in f.attrs:
-    original_backend = f.attrs['backend'].decode('utf8')
-  else:
-    original_backend = None
-
-  filtered_layers = []
-  for layer in layers:
-    weights = layer.weights
-    if weights:
-      filtered_layers.append(layer)
-
-  layer_names = load_attributes_from_hdf5_group(f, 'layer_names')
-  filtered_layer_names = []
-  for name in layer_names:
-    g = f[name]
-    weight_names = load_attributes_from_hdf5_group(g, 'weight_names')
-    if weight_names:
-      filtered_layer_names.append(name)
-  layer_names = filtered_layer_names
-  if len(layer_names) != len(filtered_layers):
-    raise ValueError('You are trying to load a weight file '
-                     'containing ' + str(len(layer_names)) +
-                     ' layers into a model with ' + str(len(filtered_layers)) +
-                     ' layers.')
-
-  # We batch weight value assignments in a single backend call
-  # which provides a speedup in TensorFlow.
-  weight_value_tuples = []
-  for k, name in enumerate(layer_names):
-    g = f[name]
-    weight_names = load_attributes_from_hdf5_group(g, 'weight_names')
-    weight_values = [np.asarray(g[weight_name]) for weight_name in weight_names]
-    layer = filtered_layers[k]
-    symbolic_weights = layer.weights
-    weight_values = preprocess_weights_for_loading(
-        layer, weight_values, original_keras_version, original_backend)
-    if len(weight_values) != len(symbolic_weights):
-      raise ValueError('Layer #' + str(k) + ' (named "' + layer.name +
-                       '" in the current model) was found to '
-                       'correspond to layer ' + name + ' in the save file. '
-                       'However the new layer ' + layer.name + ' expects ' +
-                       str(len(symbolic_weights)) +
-                       ' weights, but the saved weights have ' +
-                       str(len(weight_values)) + ' elements.')
-    weight_value_tuples += zip(symbolic_weights, weight_values)
-  K.batch_set_value(weight_value_tuples)
-
-
-def load_weights_from_hdf5_group_by_name(f, layers):
-  """Implements name-based weight loading.
-
-  (instead of topological weight loading).
-
-  Layers that have no matching name are skipped.
-
-  Arguments:
-      f: A pointer to a HDF5 group.
-      layers: a list of target layers.
-
-  Raises:
-      ValueError: in case of mismatch between provided layers
-          and weights file.
-  """
-  if 'keras_version' in f.attrs:
-    original_keras_version = f.attrs['keras_version'].decode('utf8')
-  else:
-    original_keras_version = '1'
-  if 'backend' in f.attrs:
-    original_backend = f.attrs['backend'].decode('utf8')
-  else:
-    original_backend = None
-
-  # New file format.
-  layer_names = load_attributes_from_hdf5_group(f, 'layer_names')
-
-  # Reverse index of layer name to list of layers with name.
-  index = {}
-  for layer in layers:
-    if layer.name:
-      index.setdefault(layer.name, []).append(layer)
-
-  # We batch weight value assignments in a single backend call
-  # which provides a speedup in TensorFlow.
-  weight_value_tuples = []
-  for k, name in enumerate(layer_names):
-    g = f[name]
-    weight_names = load_attributes_from_hdf5_group(g, 'weight_names')
-    weight_values = [np.asarray(g[weight_name]) for weight_name in weight_names]
-
-    for layer in index.get(name, []):
-      symbolic_weights = layer.weights
-      weight_values = preprocess_weights_for_loading(
-          layer, weight_values, original_keras_version, original_backend)
-      if len(weight_values) != len(symbolic_weights):
-        raise ValueError('Layer #' + str(k) + ' (named "' + layer.name +
-                         '") expects ' + str(len(symbolic_weights)) +
-                         ' weight(s), but the saved weights' + ' have ' +
-                         str(len(weight_values)) + ' element(s).')
-      # Set values.
-      for i in range(len(weight_values)):
-        if K.int_shape(symbolic_weights[i]) != weight_values[i].shape:
-          raise ValueError('Layer #' + str(k) +' (named "' + layer.name +
-                           '"), weight ' + str(symbolic_weights[i]) +
-                           ' has shape {}'.format(K.int_shape(
-                               symbolic_weights[i])) +
-                           ', but the saved weight has shape ' +
-                           str(weight_values[i].shape) + '.')
-
-        else:
-          weight_value_tuples.append((symbolic_weights[i], weight_values[i]))
-  K.batch_set_value(weight_value_tuples)
-
-
-def save_attributes_to_hdf5_group(group, name, data):
-  """Saves attributes (data) of the specified name into the HDF5 group.
-
-  This method deals with an inherent problem of HDF5 file which is not
-  able to store data larger than HDF5_OBJECT_HEADER_LIMIT bytes.
-
-  Arguments:
-      group: A pointer to a HDF5 group.
-      name: A name of the attributes to save.
-      data: Attributes data to store.
-
-  Raises:
-    RuntimeError: If any single attribute is too large to be saved.
-  """
-  # Check that no item in `data` is larger than `HDF5_OBJECT_HEADER_LIMIT`
-  # because in that case even chunking the array would not make the saving
-  # possible.
-  bad_attributes = [x for x in data if len(x) > HDF5_OBJECT_HEADER_LIMIT]
-
-  # Expecting this to never be true.
-  if bad_attributes:
-    raise RuntimeError('The following attributes cannot be saved to HDF5 '
-                       'file because they are larger than %d bytes: %s' %
-                       (HDF5_OBJECT_HEADER_LIMIT,
-                        ', '.join([x for x in bad_attributes])))
-
-  data_npy = np.asarray(data)
-
-  num_chunks = 1
-  chunked_data = np.array_split(data_npy, num_chunks)
-
-  # This will never loop forever thanks to the test above.
-  while any(x.nbytes > HDF5_OBJECT_HEADER_LIMIT for x in chunked_data):
-    num_chunks += 1
-    chunked_data = np.array_split(data_npy, num_chunks)
-
-  if num_chunks > 1:
-    for chunk_id, chunk_data in enumerate(chunked_data):
-      group.attrs['%s%d' % (name, chunk_id)] = chunk_data
-  else:
-    group.attrs[name] = data
-
-
-def load_attributes_from_hdf5_group(group, name):
-  """Loads attributes of the specified name from the HDF5 group.
-
-  This method deals with an inherent problem
-  of HDF5 file which is not able to store
-  data larger than HDF5_OBJECT_HEADER_LIMIT bytes.
-
-  Arguments:
-      group: A pointer to a HDF5 group.
-      name: A name of the attributes to load.
-
-  Returns:
-      data: Attributes data.
-  """
-  if name in group.attrs:
-    data = [n.decode('utf8') for n in group.attrs[name]]
-  else:
-    data = []
-    chunk_id = 0
-    while '%s%d' % (name, chunk_id) in group.attrs:
-      data.extend(
-          [n.decode('utf8') for n in group.attrs['%s%d' % (name, chunk_id)]])
-      chunk_id += 1
-  return data
+from tensorflow.python.keras.saving import *  # pylint: disable=wildcard-import
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 0b1743af38..5614c93842 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -40,6 +40,7 @@ from tensorflow.python.keras.engine import training_generator
 from tensorflow.python.keras.engine import training_utils
 from tensorflow.python.keras.engine.network import Network
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
+from tensorflow.python.keras.saving import saving_utils
 from tensorflow.python.keras.utils import data_utils
 from tensorflow.python.keras.utils.generic_utils import slice_arrays
 from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions
@@ -1702,7 +1703,7 @@ class Model(Network):
 
   @property
   def _default_save_signature(self):
-    return training_utils.trace_model_call(self)
+    return saving_utils.trace_model_call(self)
 
   def _set_sample_weight_attributes(self, sample_weight_mode,
                                     skip_target_weighing_indices):
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index 949f1e400d..a4ee6c1ba5 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -29,12 +29,10 @@ from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.data.ops import readers
 from tensorflow.python.eager import context
-from tensorflow.python.eager import def_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
-from tensorflow.python.framework import tensor_spec
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras import callbacks as cbks
@@ -1387,60 +1385,6 @@ def generic_output_names(outputs_list):
   return ['output_%d' % (i + 1) for i in range(len(outputs_list))]
 
 
-def trace_model_call(model, input_signature=None):
-  """Trace the model call to create a tf.function for exporting a Keras model.
-
-  Args:
-    model: A Keras model.
-    input_signature: optional, a list of tf.TensorSpec objects specifying the
-      inputs to the model.
-
-  Returns:
-    A tf.function wrapping the model's call function with input signatures set.
-
-  Raises:
-    ValueError: if input signature cannot be inferred from the model.
-  """
-  if input_signature is None:
-    if isinstance(model.call, def_function.PolymorphicFunction):
-      input_signature = model.call.input_signature
-
-  if input_signature is None:
-    try:
-      inputs = model.inputs
-      input_names = model.input_names
-    except AttributeError:
-      raise ValueError(
-          'Model {} cannot be saved because the input shapes have not been '
-          'set. Usually, input shapes are automatically determined from calling'
-          ' .fit() or .predict(). To manually set the shapes, call '
-          'model._set_inputs(inputs).'.format(model))
-    input_specs = []
-    for input_tensor, input_name in zip(inputs, input_names):
-      input_specs.append(tensor_spec.TensorSpec(
-          shape=input_tensor.shape, dtype=input_tensor.dtype,
-          name=input_name))
-    # The input signature of the call function is a list with one element, since
-    # all tensor inputs must be passed in as the first argument.
-    input_signature = [input_specs] if len(input_specs) > 1 else input_specs
-
-  # TODO(mdan): Should the model's call be autographed by default?
-  @def_function.function(input_signature=input_signature, autograph=False)
-  def _wrapped_model(*args):
-    """A concrete tf.function that wraps the model's call function."""
-    # When given a single input, Keras models will call the model on the tensor
-    # rather than a list consisting of the single tensor.
-    inputs = args[0] if len(input_signature) == 1 else list(args)
-    outputs_list = nest.flatten(model(inputs=inputs))
-    try:
-      output_names = model.output_names
-    except AttributeError:
-      output_names = generic_output_names(outputs_list)
-    return {name: output for name, output in zip(output_names, outputs_list)}
-
-  return _wrapped_model
-
-
 def set_run_eagerly_for_dict_structure(model, x):
   """Set model.run_eagerly to true if x is dict structure.
 
diff --git a/tensorflow/python/keras/engine/training_utils_test.py b/tensorflow/python/keras/engine/training_utils_test.py
index d0b2567b55..d8d8c58610 100644
--- a/tensorflow/python/keras/engine/training_utils_test.py
+++ b/tensorflow/python/keras/engine/training_utils_test.py
@@ -18,33 +18,18 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import os
-
 from absl.testing import parameterized
 import numpy as np
 
 
-from tensorflow.python import keras
-from tensorflow.python.client import session as session_lib
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import readers
 from tensorflow.python.eager import context
-from tensorflow.python.eager import def_function
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_spec
 from tensorflow.python.framework import tensor_util
-from tensorflow.python.keras import backend as K
 from tensorflow.python.keras import keras_parameterized
-from tensorflow.python.keras import testing_utils
 from tensorflow.python.keras.engine import training_utils
 from tensorflow.python.keras.utils import tf_utils
-from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
-from tensorflow.python.saved_model import loader
-from tensorflow.python.saved_model import save as save_lib
-from tensorflow.python.saved_model import signature_constants
-from tensorflow.python.saved_model import tag_constants
 
 
 class ModelInputsTest(test.TestCase):
@@ -105,170 +90,6 @@ class ModelInputsTest(test.TestCase):
       self.assertTrue(tf_utils.is_symbolic_tensor(vals['b']))
 
 
-class TraceModelCallTest(keras_parameterized.TestCase):
-
-  def _assert_all_close(self, expected, actual):
-    if not context.executing_eagerly():
-      with self.cached_session() as sess:
-        K._initialize_variables(sess)
-        self.assertAllClose(expected, actual)
-    else:
-      self.assertAllClose(expected, actual)
-
-  @keras_parameterized.run_with_all_model_types
-  @keras_parameterized.run_all_keras_modes
-  def test_trace_model_outputs(self):
-    input_dim = 5 if testing_utils.get_model_type() == 'functional' else None
-    model = testing_utils.get_small_mlp(10, 3, input_dim)
-    inputs = array_ops.ones((8, 5))
-
-    if input_dim is None:
-      with self.assertRaisesRegexp(ValueError,
-                                   'input shapes have not been set'):
-        training_utils.trace_model_call(model)
-      model._set_inputs(inputs)
-
-    fn = training_utils.trace_model_call(model)
-    signature_outputs = fn(inputs)
-    expected_outputs = {model.output_names[0]: model(inputs)}
-
-    self._assert_all_close(expected_outputs, signature_outputs)
-
-  @keras_parameterized.run_with_all_model_types
-  @keras_parameterized.run_all_keras_modes
-  def test_trace_model_outputs_after_fitting(self):
-    input_dim = 5 if testing_utils.get_model_type() == 'functional' else None
-    model = testing_utils.get_small_mlp(10, 3, input_dim)
-    model.compile(optimizer='sgd', loss='mse')
-    model.fit(x=np.random.random((8, 5)),
-              y=np.random.random((8, 3)), epochs=2)
-
-    inputs = array_ops.ones((8, 5))
-
-    fn = training_utils.trace_model_call(model)
-    signature_outputs = fn(inputs)
-    expected_outputs = {model.output_names[0]: model(inputs)}
-
-    self._assert_all_close(expected_outputs, signature_outputs)
-
-  @keras_parameterized.run_with_all_model_types(exclude_models='sequential')
-  @keras_parameterized.run_all_keras_modes
-  def test_trace_multi_io_model_outputs(self):
-    input_dim = 5
-    num_classes = 3
-    num_classes_b = 4
-    input_a = keras.layers.Input(shape=(input_dim,), name='input_a')
-    input_b = keras.layers.Input(shape=(input_dim,), name='input_b')
-
-    dense = keras.layers.Dense(num_classes, name='dense')
-    dense2 = keras.layers.Dense(num_classes_b, name='dense2')
-    dropout = keras.layers.Dropout(0.5, name='dropout')
-    branch_a = [input_a, dense]
-    branch_b = [input_b, dense, dense2, dropout]
-
-    model = testing_utils.get_multi_io_model(branch_a, branch_b)
-
-    input_a_np = np.random.random((10, input_dim)).astype(np.float32)
-    input_b_np = np.random.random((10, input_dim)).astype(np.float32)
-
-    if testing_utils.get_model_type() == 'subclass':
-      with self.assertRaisesRegexp(ValueError,
-                                   'input shapes have not been set'):
-        training_utils.trace_model_call(model)
-
-    model.compile(optimizer='sgd', loss='mse')
-    model.fit(x=[np.random.random((8, input_dim)).astype(np.float32),
-                 np.random.random((8, input_dim)).astype(np.float32)],
-              y=[np.random.random((8, num_classes)).astype(np.float32),
-                 np.random.random((8, num_classes_b)).astype(np.float32)],
-              epochs=2)
-
-    fn = training_utils.trace_model_call(model)
-    signature_outputs = fn([input_a_np, input_b_np])
-    outputs = model([input_a_np, input_b_np])
-    expected_outputs = {model.output_names[0]: outputs[0],
-                        model.output_names[1]: outputs[1]}
-
-    self._assert_all_close(expected_outputs, signature_outputs)
-
-  @keras_parameterized.run_all_keras_modes
-  def test_specify_input_signature(self):
-    model = testing_utils.get_small_sequential_mlp(10, 3, None)
-    inputs = array_ops.ones((8, 5))
-
-    with self.assertRaisesRegexp(ValueError, 'input shapes have not been set'):
-      training_utils.trace_model_call(model)
-
-    fn = training_utils.trace_model_call(
-        model, [tensor_spec.TensorSpec(shape=[None, 5], dtype=dtypes.float32)])
-    signature_outputs = fn(inputs)
-    expected_outputs = {model.output_names[0]: model(inputs)}
-    self._assert_all_close(expected_outputs, signature_outputs)
-
-  @keras_parameterized.run_all_keras_modes
-  def test_subclassed_model_with_input_signature(self):
-
-    class Model(keras.Model):
-
-      def __init__(self):
-        super(Model, self).__init__()
-        self.dense = keras.layers.Dense(3, name='dense')
-
-      @def_function.function(
-          input_signature=[[tensor_spec.TensorSpec([None, 5], dtypes.float32),
-                            tensor_spec.TensorSpec([None], dtypes.float32)]],)
-      def call(self, inputs, *args):
-        x, y = inputs
-        return self.dense(x) + y
-
-    model = Model()
-    fn = training_utils.trace_model_call(model)
-    x = array_ops.ones((8, 5), dtype=dtypes.float32)
-    y = array_ops.ones((3,), dtype=dtypes.float32)
-    expected_outputs = {'output_1': model([x, y])}
-    signature_outputs = fn([x, y])
-    self._assert_all_close(expected_outputs, signature_outputs)
-
-
-def _import_and_infer(save_dir, inputs):
-  """Import a SavedModel into a TF 1.x-style graph and run `signature_key`."""
-  graph = ops.Graph()
-  with graph.as_default(), session_lib.Session() as session:
-    model = loader.load(session, [tag_constants.SERVING], save_dir)
-    signature = model.signature_def[
-        signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY]
-    assert set(inputs.keys()) == set(signature.inputs.keys())
-    feed_dict = {}
-    for arg_name in inputs.keys():
-      feed_dict[graph.get_tensor_by_name(signature.inputs[arg_name].name)] = (
-          inputs[arg_name])
-    output_dict = {}
-    for output_name, output_tensor_info in signature.outputs.items():
-      output_dict[output_name] = graph.get_tensor_by_name(
-          output_tensor_info.name)
-    return session.run(output_dict, feed_dict=feed_dict)
-
-
-class ModelSaveTest(keras_parameterized.TestCase):
-
-  @keras_parameterized.run_with_all_model_types
-  @keras_parameterized.run_all_keras_modes(always_skip_v1=True)
-  def test_model_save(self):
-    input_dim = 5
-    model = testing_utils.get_small_mlp(10, 3, input_dim)
-    inputs = array_ops.ones((8, 5))
-
-    if testing_utils.get_model_type() == 'subclass':
-      model._set_inputs(inputs)
-
-    save_dir = os.path.join(self.get_temp_dir(), 'saved_model')
-    save_lib.save(model, save_dir)
-
-    self.assertAllClose(
-        {model.output_names[0]: model.predict_on_batch(inputs)},
-        _import_and_infer(save_dir, {model.input_names[0]: np.ones((8, 5))}))
-
-
 class DatasetUtilsTest(test.TestCase, parameterized.TestCase):
 
   @parameterized.named_parameters(
diff --git a/tensorflow/python/keras/layers/cudnn_recurrent_test.py b/tensorflow/python/keras/layers/cudnn_recurrent_test.py
index 36f2d2fa38..1e305baefd 100644
--- a/tensorflow/python/keras/layers/cudnn_recurrent_test.py
+++ b/tensorflow/python/keras/layers/cudnn_recurrent_test.py
@@ -467,7 +467,7 @@ class CuDNNV1OnlyTest(keras_parameterized.TestCase):
 
     def assert_not_compatible(src, dest, message):
       with self.assertRaises(ValueError) as ex:
-        keras.engine.saving.preprocess_weights_for_loading(
+        keras.saving.preprocess_weights_for_loading(
             dest,
             get_layer_weights(src))
       self.assertIn(message, str(ex.exception))
diff --git a/tensorflow/python/keras/models.py b/tensorflow/python/keras/models.py
index 851de6e71f..9bc5aa2be5 100644
--- a/tensorflow/python/keras/models.py
+++ b/tensorflow/python/keras/models.py
@@ -22,13 +22,14 @@ from __future__ import print_function
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras import metrics as metrics_module
 from tensorflow.python.keras import optimizers
-from tensorflow.python.keras.engine import saving
 from tensorflow.python.keras.engine import sequential
 from tensorflow.python.keras.engine import training
 from tensorflow.python.keras.engine.base_layer import Layer
 from tensorflow.python.keras.engine.input_layer import Input
 from tensorflow.python.keras.engine.input_layer import InputLayer
 from tensorflow.python.keras.engine.network import Network
+from tensorflow.python.keras.saving import hdf5_format
+from tensorflow.python.keras.saving import model_config
 from tensorflow.python.keras.utils import generic_utils
 from tensorflow.python.keras.utils.generic_utils import CustomObjectScope
 from tensorflow.python.util import nest
@@ -38,11 +39,11 @@ from tensorflow.python.util.tf_export import keras_export
 # API entries importable from `keras.models`:
 Model = training.Model  # pylint: disable=invalid-name
 Sequential = sequential.Sequential  # pylint: disable=invalid-name
-save_model = saving.save_model
-load_model = saving.load_model
-model_from_config = saving.model_from_config
-model_from_yaml = saving.model_from_yaml
-model_from_json = saving.model_from_json
+save_model = hdf5_format.save_model
+load_model = hdf5_format.load_model
+model_from_config = model_config.model_from_config
+model_from_yaml = model_config.model_from_yaml
+model_from_json = model_config.model_from_json
 
 
 def _clone_layer(layer):
diff --git a/tensorflow/python/keras/saving/BUILD b/tensorflow/python/keras/saving/BUILD
deleted file mode 100644
index a78d58e876..0000000000
--- a/tensorflow/python/keras/saving/BUILD
+++ /dev/null
@@ -1,72 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-# Description:
-#   Keras saving and loading libraries.
-load("//tensorflow:tensorflow.bzl", "py_test")
-
-package(default_visibility = ["//tensorflow:__subpackages__"])
-
-licenses(["notice"])  # Apache 2.0
-
-exports_files(["LICENSE"])
-
-py_library(
-    name = "saving",
-    srcs = ["__init__.py"],
-    deps = [":saved_model"],
-)
-
-py_library(
-    name = "saved_model",
-    srcs = [
-        "saved_model.py",
-        "saving_utils.py",
-    ],
-    srcs_version = "PY2AND3",
-    visibility = ["//visibility:public"],
-    deps = [
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:lib",
-        "//tensorflow/python:metrics",
-        "//tensorflow/python:mode_keys",
-        "//tensorflow/python:platform",
-        "//tensorflow/python:saver",
-        "//tensorflow/python:util",
-        "//tensorflow/python/keras:engine",
-        "//tensorflow/python/saved_model",
-        "//tensorflow/python/saved_model/model_utils",
-    ],
-)
-
-py_test(
-    name = "saved_model_test",
-    size = "medium",
-    srcs = ["saved_model_test.py"],
-    srcs_version = "PY2AND3",
-    tags = [
-        "no_oss",  # TODO(b/119349471): Re-enable
-        "no_windows",
-    ],
-    deps = [
-        ":saved_model",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:mode_keys",
-        "//tensorflow/python/keras",
-        "//third_party/py/numpy",
-        "@absl_py//absl/testing:parameterized",
-    ],
-)
diff --git a/tensorflow/python/keras/saving/__init__.py b/tensorflow/python/keras/saving/__init__.py
index 8ff9f3b74e..bb4db68124 100644
--- a/tensorflow/python/keras/saving/__init__.py
+++ b/tensorflow/python/keras/saving/__init__.py
@@ -12,10 +12,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Utils for saving a Keras Model or Estimator to the SavedModel format."""
+"""Utils for saving and loading Keras Models."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.keras.saving.hdf5_format import load_attributes_from_hdf5_group
+from tensorflow.python.keras.saving.hdf5_format import load_model
+from tensorflow.python.keras.saving.hdf5_format import load_weights_from_hdf5_group
+from tensorflow.python.keras.saving.hdf5_format import load_weights_from_hdf5_group_by_name
+from tensorflow.python.keras.saving.hdf5_format import preprocess_weights_for_loading
+from tensorflow.python.keras.saving.hdf5_format import save_attributes_to_hdf5_group
+from tensorflow.python.keras.saving.hdf5_format import save_model
+from tensorflow.python.keras.saving.hdf5_format import save_weights_to_hdf5_group
+from tensorflow.python.keras.saving.model_config import model_from_config
+from tensorflow.python.keras.saving.model_config import model_from_json
+from tensorflow.python.keras.saving.model_config import model_from_yaml
 from tensorflow.python.keras.saving.saved_model import export
 from tensorflow.python.keras.saving.saved_model import load_from_saved_model
+from tensorflow.python.keras.saving.saving_utils import trace_model_call
+
+
diff --git a/tensorflow/python/keras/saving/hdf5_format.py b/tensorflow/python/keras/saving/hdf5_format.py
new file mode 100644
index 0000000000..cb925c30a6
--- /dev/null
+++ b/tensorflow/python/keras/saving/hdf5_format.py
@@ -0,0 +1,895 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# pylint: disable=protected-access
+"""Functions for saving and loading a Keras Model from HDF5 format.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import json
+import os
+
+import numpy as np
+from six.moves import zip  # pylint: disable=redefined-builtin
+
+from tensorflow.python.framework import ops
+from tensorflow.python.keras import backend as K
+from tensorflow.python.keras import optimizers
+from tensorflow.python.keras.saving import model_config as model_config_lib
+from tensorflow.python.keras.utils import conv_utils
+from tensorflow.python.keras.utils.io_utils import ask_to_proceed_with_overwrite
+from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.util import serialization
+from tensorflow.python.util.tf_export import keras_export
+
+# pylint: disable=g-import-not-at-top
+try:
+  import h5py
+  HDF5_OBJECT_HEADER_LIMIT = 64512
+except ImportError:
+  h5py = None
+# pylint: enable=g-import-not-at-top
+
+
+@keras_export('keras.models.save_model')
+def save_model(model, filepath, overwrite=True, include_optimizer=True):
+  """Saves a model to a HDF5 file.
+
+  The saved model contains:
+      - the model's configuration (topology)
+      - the model's weights
+      - the model's optimizer's state (if any)
+
+  Thus the saved model can be reinstantiated in
+  the exact same state, without any of the code
+  used for model definition or training.
+
+  Arguments:
+      model: Keras model instance to be saved.
+      filepath: One of the following:
+          - String, path where to save the model
+          - `h5py.File` object where to save the model
+      overwrite: Whether we should overwrite any existing
+          model at the target location, or instead
+          ask the user with a manual prompt.
+      include_optimizer: If True, save optimizer's state together.
+
+  Raises:
+      ImportError: if h5py is not available.
+  """
+
+  if h5py is None:
+    raise ImportError('`save_model` requires h5py.')
+
+  from tensorflow.python.keras import __version__ as keras_version  # pylint: disable=g-import-not-at-top
+
+  # TODO(psv) Add warning when we save models that contain non-serializable
+  # entities like metrics added using `add_metric` and losses added using
+  # `add_loss.`
+
+  if not isinstance(filepath, h5py.File):
+    # If file exists and should not be overwritten.
+    if not overwrite and os.path.isfile(filepath):
+      proceed = ask_to_proceed_with_overwrite(filepath)
+      if not proceed:
+        return
+
+    f = h5py.File(filepath, mode='w')
+    opened_new_file = True
+  else:
+    f = filepath
+    opened_new_file = False
+
+  try:
+    f.attrs['keras_version'] = str(keras_version).encode('utf8')
+    f.attrs['backend'] = K.backend().encode('utf8')
+    f.attrs['model_config'] = json.dumps(
+        {
+            'class_name': model.__class__.__name__,
+            'config': model.get_config()
+        },
+        default=serialization.get_json_type).encode('utf8')
+
+    model_weights_group = f.create_group('model_weights')
+    model_layers = model.layers
+    save_weights_to_hdf5_group(model_weights_group, model_layers)
+
+    if include_optimizer and model.optimizer:
+      if isinstance(model.optimizer, optimizers.TFOptimizer):
+        logging.warning(
+            'TensorFlow optimizers do not '
+            'make it possible to access '
+            'optimizer attributes or optimizer state '
+            'after instantiation. '
+            'As a result, we cannot save the optimizer '
+            'as part of the model save file.'
+            'You will have to compile your model again after loading it. '
+            'Prefer using a Keras optimizer instead '
+            '(see keras.io/optimizers).')
+      else:
+        f.attrs['training_config'] = json.dumps(
+            {
+                'optimizer_config': {
+                    'class_name': model.optimizer.__class__.__name__,
+                    'config': model.optimizer.get_config()
+                },
+                'loss': model.loss,
+                'metrics': model._compile_metrics,
+                'weighted_metrics': model._compile_weighted_metrics,
+                'sample_weight_mode': model.sample_weight_mode,
+                'loss_weights': model.loss_weights,
+            },
+            default=serialization.get_json_type).encode('utf8')
+
+        # Save optimizer weights.
+        symbolic_weights = getattr(model.optimizer, 'weights')
+        if symbolic_weights:
+          optimizer_weights_group = f.create_group('optimizer_weights')
+          weight_values = K.batch_get_value(symbolic_weights)
+          weight_names = []
+          for w, val in zip(symbolic_weights, weight_values):
+            name = str(w.name)
+            weight_names.append(name.encode('utf8'))
+          optimizer_weights_group.attrs['weight_names'] = weight_names
+          for name, val in zip(weight_names, weight_values):
+            param_dset = optimizer_weights_group.create_dataset(
+                name, val.shape, dtype=val.dtype)
+            if not val.shape:
+              # scalar
+              param_dset[()] = val
+            else:
+              param_dset[:] = val
+    f.flush()
+  finally:
+    if opened_new_file:
+      f.close()
+
+
+@keras_export('keras.models.load_model')
+def load_model(filepath, custom_objects=None, compile=True):  # pylint: disable=redefined-builtin
+  """Loads a model saved via `save_model`.
+
+  Arguments:
+      filepath: One of the following:
+          - String, path to the saved model
+          - `h5py.File` object from which to load the model
+      custom_objects: Optional dictionary mapping names
+          (strings) to custom classes or functions to be
+          considered during deserialization.
+      compile: Boolean, whether to compile the model
+          after loading.
+
+  Returns:
+      A Keras model instance. If an optimizer was found
+      as part of the saved model, the model is already
+      compiled. Otherwise, the model is uncompiled and
+      a warning will be displayed. When `compile` is set
+      to False, the compilation is omitted without any
+      warning.
+
+  Raises:
+      ImportError: if h5py is not available.
+      ValueError: In case of an invalid savefile.
+  """
+  if h5py is None:
+    raise ImportError('`load_model` requires h5py.')
+
+  if not custom_objects:
+    custom_objects = {}
+
+  def convert_custom_objects(obj):
+    """Handles custom object lookup.
+
+    Arguments:
+        obj: object, dict, or list.
+
+    Returns:
+        The same structure, where occurrences
+            of a custom object name have been replaced
+            with the custom object.
+    """
+    if isinstance(obj, list):
+      deserialized = []
+      for value in obj:
+        deserialized.append(convert_custom_objects(value))
+      return deserialized
+    if isinstance(obj, dict):
+      deserialized = {}
+      for key, value in obj.items():
+        deserialized[key] = convert_custom_objects(value)
+      return deserialized
+    if obj in custom_objects:
+      return custom_objects[obj]
+    return obj
+
+  opened_new_file = not isinstance(filepath, h5py.File)
+  if opened_new_file:
+    f = h5py.File(filepath, mode='r')
+  else:
+    f = filepath
+
+  model = None
+  try:
+    # instantiate model
+    model_config = f.attrs.get('model_config')
+    if model_config is None:
+      raise ValueError('No model found in config file.')
+    model_config = json.loads(model_config.decode('utf-8'))
+    model = model_config_lib.model_from_config(model_config,
+                                               custom_objects=custom_objects)
+
+    # set weights
+    load_weights_from_hdf5_group(f['model_weights'], model.layers)
+
+    if compile:
+      # instantiate optimizer
+      training_config = f.attrs.get('training_config')
+      if training_config is None:
+        logging.warning('No training configuration found in save file: '
+                        'the model was *not* compiled. Compile it manually.')
+        return model
+      training_config = json.loads(training_config.decode('utf-8'))
+      optimizer_config = training_config['optimizer_config']
+      optimizer = optimizers.deserialize(
+          optimizer_config, custom_objects=custom_objects)
+
+      # Recover loss functions and metrics.
+      loss = convert_custom_objects(training_config['loss'])
+      metrics = convert_custom_objects(training_config['metrics'])
+      weighted_metrics = convert_custom_objects(
+          training_config.get('weighted_metrics', None))
+      sample_weight_mode = training_config['sample_weight_mode']
+      loss_weights = training_config['loss_weights']
+
+      # Compile model.
+      model.compile(
+          optimizer=optimizer,
+          loss=loss,
+          metrics=metrics,
+          weighted_metrics=weighted_metrics,
+          loss_weights=loss_weights,
+          sample_weight_mode=sample_weight_mode)
+
+      # Set optimizer weights.
+      if 'optimizer_weights' in f:
+        # Build train function (to get weight updates).
+        # Models that aren't graph networks must wait until they are called
+        # with data to _make_train_function() and so can't load optimizer
+        # weights.
+        if model._is_graph_network:  # pylint: disable=protected-access
+          model._make_train_function()
+          optimizer_weights_group = f['optimizer_weights']
+          optimizer_weight_names = [
+              n.decode('utf8')
+              for n in optimizer_weights_group.attrs['weight_names']
+          ]
+          optimizer_weight_values = [
+              optimizer_weights_group[n] for n in optimizer_weight_names
+          ]
+          try:
+            model.optimizer.set_weights(optimizer_weight_values)
+          except ValueError:
+            logging.warning('Error in loading the saved optimizer '
+                            'state. As a result, your model is '
+                            'starting with a freshly initialized '
+                            'optimizer.')
+        else:
+          logging.warning('Sequential models without an `input_shape` '
+                          'passed to the first layer cannot reload their '
+                          'optimizer state. As a result, your model is'
+                          'starting with a freshly initialized optimizer.')
+
+  finally:
+    if opened_new_file:
+      f.close()
+  return model
+
+
+def preprocess_weights_for_loading(layer,
+                                   weights,
+                                   original_keras_version=None,
+                                   original_backend=None):
+  """Preprocess layer weights between different Keras formats.
+
+  Converts layers weights from Keras 1 format to Keras 2 and also weights of
+  CuDNN layers in Keras 2.
+
+  Arguments:
+      layer: Layer instance.
+      weights: List of weights values (Numpy arrays).
+      original_keras_version: Keras version for the weights, as a string.
+      original_backend: Keras backend the weights were trained with,
+          as a string.
+
+  Returns:
+      A list of weights values (Numpy arrays).
+  """
+  def convert_nested_bidirectional(weights):
+    """Converts layers nested in `Bidirectional` wrapper.
+
+    This function uses `preprocess_weights_for_loading()` for converting
+    layers.
+
+    Arguments:
+        weights: List of weights values (Numpy arrays).
+
+    Returns:
+        A list of weights values (Numpy arrays).
+    """
+    num_weights_per_layer = len(weights) // 2
+    forward_weights = preprocess_weights_for_loading(
+        layer.forward_layer, weights[:num_weights_per_layer],
+        original_keras_version, original_backend)
+    backward_weights = preprocess_weights_for_loading(
+        layer.backward_layer, weights[num_weights_per_layer:],
+        original_keras_version, original_backend)
+    return forward_weights + backward_weights
+
+  def convert_nested_time_distributed(weights):
+    """Converts layers nested in `TimeDistributed` wrapper.
+
+    This function uses `preprocess_weights_for_loading()` for converting nested
+    layers.
+
+    Arguments:
+        weights: List of weights values (Numpy arrays).
+
+    Returns:
+        A list of weights values (Numpy arrays).
+    """
+    return preprocess_weights_for_loading(
+        layer.layer, weights, original_keras_version, original_backend)
+
+  def convert_nested_model(weights):
+    """Converts layers nested in `Model` or `Sequential`.
+
+    This function uses `preprocess_weights_for_loading()` for converting nested
+    layers.
+
+    Arguments:
+        weights: List of weights values (Numpy arrays).
+
+    Returns:
+        A list of weights values (Numpy arrays).
+    """
+    new_weights = []
+    # trainable weights
+    for sublayer in layer.layers:
+      num_weights = len(sublayer.trainable_weights)
+      if num_weights > 0:
+        new_weights.extend(preprocess_weights_for_loading(
+            layer=sublayer,
+            weights=weights[:num_weights],
+            original_keras_version=original_keras_version,
+            original_backend=original_backend))
+        weights = weights[num_weights:]
+
+    # non-trainable weights
+    for sublayer in layer.layers:
+      num_weights = len([l for l in sublayer.weights
+                         if l not in sublayer.trainable_weights])
+      if num_weights > 0:
+        new_weights.extend(preprocess_weights_for_loading(
+            layer=sublayer,
+            weights=weights[:num_weights],
+            original_keras_version=original_keras_version,
+            original_backend=original_backend))
+        weights = weights[num_weights:]
+    return new_weights
+
+  # Convert layers nested in Bidirectional/Model/Sequential.
+  # Both transformation should be ran for both Keras 1->2 conversion
+  # and for conversion of CuDNN layers.
+  if layer.__class__.__name__ == 'Bidirectional':
+    weights = convert_nested_bidirectional(weights)
+  if layer.__class__.__name__ == 'TimeDistributed':
+    weights = convert_nested_time_distributed(weights)
+  elif layer.__class__.__name__ in ['Model', 'Sequential']:
+    weights = convert_nested_model(weights)
+
+  if original_keras_version == '1':
+    if layer.__class__.__name__ == 'TimeDistributed':
+      weights = preprocess_weights_for_loading(
+          layer.layer, weights, original_keras_version, original_backend)
+
+    if layer.__class__.__name__ == 'Conv1D':
+      shape = weights[0].shape
+      # Handle Keras 1.1 format
+      if shape[:2] != (layer.kernel_size[0], 1) or shape[3] != layer.filters:
+        # Legacy shape:
+        # (filters, input_dim, filter_length, 1)
+        assert shape[0] == layer.filters and shape[2:] == (layer.kernel_size[0],
+                                                           1)
+        weights[0] = np.transpose(weights[0], (2, 3, 1, 0))
+      weights[0] = weights[0][:, 0, :, :]
+
+    if layer.__class__.__name__ == 'Conv2D':
+      if layer.data_format == 'channels_first':
+        # old: (filters, stack_size, kernel_rows, kernel_cols)
+        # new: (kernel_rows, kernel_cols, stack_size, filters)
+        weights[0] = np.transpose(weights[0], (2, 3, 1, 0))
+
+    if layer.__class__.__name__ == 'Conv2DTranspose':
+      if layer.data_format == 'channels_last':
+        # old: (kernel_rows, kernel_cols, stack_size, filters)
+        # new: (kernel_rows, kernel_cols, filters, stack_size)
+        weights[0] = np.transpose(weights[0], (0, 1, 3, 2))
+      if layer.data_format == 'channels_first':
+        # old: (filters, stack_size, kernel_rows, kernel_cols)
+        # new: (kernel_rows, kernel_cols, filters, stack_size)
+        weights[0] = np.transpose(weights[0], (2, 3, 0, 1))
+
+    if layer.__class__.__name__ == 'Conv3D':
+      if layer.data_format == 'channels_first':
+        # old: (filters, stack_size, ...)
+        # new: (..., stack_size, filters)
+        weights[0] = np.transpose(weights[0], (2, 3, 4, 1, 0))
+
+    if layer.__class__.__name__ == 'GRU':
+      if len(weights) == 9:
+        kernel = np.concatenate([weights[0], weights[3], weights[6]], axis=-1)
+        recurrent_kernel = np.concatenate(
+            [weights[1], weights[4], weights[7]], axis=-1)
+        bias = np.concatenate([weights[2], weights[5], weights[8]], axis=-1)
+        weights = [kernel, recurrent_kernel, bias]
+
+    if layer.__class__.__name__ == 'LSTM':
+      if len(weights) == 12:
+        # old: i, c, f, o
+        # new: i, f, c, o
+        kernel = np.concatenate(
+            [weights[0], weights[6], weights[3], weights[9]], axis=-1)
+        recurrent_kernel = np.concatenate(
+            [weights[1], weights[7], weights[4], weights[10]], axis=-1)
+        bias = np.concatenate(
+            [weights[2], weights[8], weights[5], weights[11]], axis=-1)
+        weights = [kernel, recurrent_kernel, bias]
+
+    if layer.__class__.__name__ == 'ConvLSTM2D':
+      if len(weights) == 12:
+        kernel = np.concatenate(
+            [weights[0], weights[6], weights[3], weights[9]], axis=-1)
+        recurrent_kernel = np.concatenate(
+            [weights[1], weights[7], weights[4], weights[10]], axis=-1)
+        bias = np.concatenate(
+            [weights[2], weights[8], weights[5], weights[11]], axis=-1)
+        if layer.data_format == 'channels_first':
+          # old: (filters, stack_size, kernel_rows, kernel_cols)
+          # new: (kernel_rows, kernel_cols, stack_size, filters)
+          kernel = np.transpose(kernel, (2, 3, 1, 0))
+          recurrent_kernel = np.transpose(recurrent_kernel, (2, 3, 1, 0))
+        weights = [kernel, recurrent_kernel, bias]
+
+  conv_layers = ['Conv1D', 'Conv2D', 'Conv3D', 'Conv2DTranspose', 'ConvLSTM2D']
+  if layer.__class__.__name__ in conv_layers:
+    if original_backend == 'theano':
+      weights[0] = conv_utils.convert_kernel(weights[0])
+      if layer.__class__.__name__ == 'ConvLSTM2D':
+        weights[1] = conv_utils.convert_kernel(weights[1])
+    if K.int_shape(layer.weights[0]) != weights[0].shape:
+      weights[0] = np.transpose(weights[0], (3, 2, 0, 1))
+      if layer.__class__.__name__ == 'ConvLSTM2D':
+        weights[1] = np.transpose(weights[1], (3, 2, 0, 1))
+
+  # convert CuDNN layers
+  return _convert_rnn_weights(layer, weights)
+
+
+def _convert_rnn_weights(layer, weights):
+  """Converts weights for RNN layers between native and CuDNN format.
+
+  Input kernels for each gate are transposed and converted between Fortran
+  and C layout, recurrent kernels are transposed. For LSTM biases are summed/
+  split in half, for GRU biases are reshaped.
+
+  Weights can be converted in both directions between `LSTM` and`CuDNNSLTM`
+  and between `CuDNNGRU` and `GRU(reset_after=True)`. Default `GRU` is not
+  compatible with `CuDNNGRU`.
+
+  For missing biases in `LSTM`/`GRU` (`use_bias=False`) no conversion is made.
+
+  Arguments:
+      layer: Target layer instance.
+      weights: List of source weights values (input kernels, recurrent
+          kernels, [biases]) (Numpy arrays).
+
+  Returns:
+      A list of converted weights values (Numpy arrays).
+
+  Raises:
+      ValueError: for incompatible GRU layer/weights or incompatible biases
+  """
+
+  def transform_kernels(kernels, func, n_gates):
+    """Transforms kernel for each gate separately using given function.
+
+    Arguments:
+        kernels: Stacked array of kernels for individual gates.
+        func: Function applied to kernel of each gate.
+        n_gates: Number of gates (4 for LSTM, 3 for GRU).
+
+    Returns:
+        Stacked array of transformed kernels.
+    """
+    return np.hstack([func(k) for k in np.hsplit(kernels, n_gates)])
+
+  def transpose_input(from_cudnn):
+    """Makes a function that transforms input kernels from/to CuDNN format.
+
+    It keeps the shape, but changes between the layout (Fortran/C). Eg.:
+
+    ```
+    Keras                 CuDNN
+    [[0, 1, 2],  <--->  [[0, 2, 4],
+     [3, 4, 5]]          [1, 3, 5]]
+    ```
+
+    It can be passed to `transform_kernels()`.
+
+    Arguments:
+        from_cudnn: `True` if source weights are in CuDNN format, `False`
+            if they're in plain Keras format.
+
+    Returns:
+        Function that converts input kernel to the other format.
+    """
+    order = 'F' if from_cudnn else 'C'
+
+    def transform(kernel):
+      return kernel.T.reshape(kernel.shape, order=order)
+
+    return transform
+
+  target_class = layer.__class__.__name__
+
+  # convert the weights between CuDNNLSTM and LSTM
+  if target_class in ['LSTM', 'CuDNNLSTM'] and len(weights) == 3:
+    # determine if we're loading a CuDNNLSTM layer
+    # from the number of bias weights:
+    # CuDNNLSTM has (units * 8) weights; while LSTM has (units * 4)
+    # if there's no bias weight in the file, skip this conversion
+    units = weights[1].shape[0]
+    bias_shape = weights[2].shape
+    n_gates = 4
+
+    if bias_shape == (2 * units * n_gates,):
+      source = 'CuDNNLSTM'
+    elif bias_shape == (units * n_gates,):
+      source = 'LSTM'
+    else:
+      raise ValueError('Invalid bias shape: ' + str(bias_shape))
+
+    def convert_lstm_weights(weights, from_cudnn=True):
+      """Converts the weights between CuDNNLSTM and LSTM.
+
+      Arguments:
+        weights: Original weights.
+        from_cudnn: Indicates whether original weights are from CuDNN layer.
+
+      Returns:
+        Updated weights compatible with LSTM.
+      """
+
+      # Transpose (and reshape) input and recurrent kernels
+      kernels = transform_kernels(weights[0], transpose_input(from_cudnn),
+                                  n_gates)
+      recurrent_kernels = transform_kernels(weights[1], lambda k: k.T, n_gates)
+      if from_cudnn:
+        # merge input and recurrent biases into a single set
+        biases = np.sum(np.split(weights[2], 2, axis=0), axis=0)
+      else:
+        # Split single set of biases evenly to two sets. The way of
+        # splitting doesn't matter as long as the two sets sum is kept.
+        biases = np.tile(0.5 * weights[2], 2)
+      return [kernels, recurrent_kernels, biases]
+
+    if source != target_class:
+      weights = convert_lstm_weights(weights, from_cudnn=source == 'CuDNNLSTM')
+
+  # convert the weights between CuDNNGRU and GRU(reset_after=True)
+  if target_class in ['GRU', 'CuDNNGRU'] and len(weights) == 3:
+    # We can determine the source of the weights from the shape of the bias.
+    # If there is no bias we skip the conversion since
+    # CuDNNGRU always has biases.
+
+    units = weights[1].shape[0]
+    bias_shape = weights[2].shape
+    n_gates = 3
+
+    def convert_gru_weights(weights, from_cudnn=True):
+      """Converts the weights between CuDNNGRU and GRU.
+
+      Arguments:
+        weights: Original weights.
+        from_cudnn: Indicates whether original weights are from CuDNN layer.
+
+      Returns:
+        Updated weights compatible with GRU.
+      """
+
+      kernels = transform_kernels(weights[0], transpose_input(from_cudnn),
+                                  n_gates)
+      recurrent_kernels = transform_kernels(weights[1], lambda k: k.T, n_gates)
+      biases = np.array(weights[2]).reshape((2, -1) if from_cudnn else -1)
+      return [kernels, recurrent_kernels, biases]
+
+    if bias_shape == (2 * units * n_gates,):
+      source = 'CuDNNGRU'
+    elif bias_shape == (2, units * n_gates):
+      source = 'GRU(reset_after=True)'
+    elif bias_shape == (units * n_gates,):
+      source = 'GRU(reset_after=False)'
+    else:
+      raise ValueError('Invalid bias shape: ' + str(bias_shape))
+
+    if target_class == 'CuDNNGRU':
+      target = 'CuDNNGRU'
+    elif layer.reset_after:
+      target = 'GRU(reset_after=True)'
+    else:
+      target = 'GRU(reset_after=False)'
+
+    # only convert between different types
+    if source != target:
+      types = (source, target)
+      if 'GRU(reset_after=False)' in types:
+        raise ValueError('%s is not compatible with %s' % types)
+      if source == 'CuDNNGRU':
+        weights = convert_gru_weights(weights, from_cudnn=True)
+      elif source == 'GRU(reset_after=True)':
+        weights = convert_gru_weights(weights, from_cudnn=False)
+
+  return weights
+
+
+def save_weights_to_hdf5_group(f, layers):
+  """Saves the weights of a list of layers to a HDF5 group.
+
+  Arguments:
+      f: HDF5 group.
+      layers: List of layer instances.
+  """
+  from tensorflow.python.keras import __version__ as keras_version  # pylint: disable=g-import-not-at-top
+
+  save_attributes_to_hdf5_group(
+      f, 'layer_names', [layer.name.encode('utf8') for layer in layers])
+  f.attrs['backend'] = K.backend().encode('utf8')
+  f.attrs['keras_version'] = str(keras_version).encode('utf8')
+
+  # On TPUs, modifying the graph between session.runs() triggers some expensive
+  # recompilation overhead. To avoid this, we build up the full set of tensors
+  # to save before fetching weights, thus only modifying the graph once.
+  layer_weights_dict = {}
+  for layer in layers:
+    layer_weights_dict[layer.name] = [ops.convert_to_tensor(w)
+                                      for w in layer.weights]
+
+  for layer in layers:
+    g = f.create_group(layer.name)
+    symbolic_weights = layer_weights_dict[layer.name]
+    weight_values = K.batch_get_value(symbolic_weights)
+    weight_names = []
+    for i, (w, val) in enumerate(zip(symbolic_weights, weight_values)):
+      if hasattr(w, 'name') and w.name:
+        name = str(w.name)
+      else:
+        name = 'param_' + str(i)
+      weight_names.append(name.encode('utf8'))
+    save_attributes_to_hdf5_group(g, 'weight_names', weight_names)
+    for name, val in zip(weight_names, weight_values):
+      param_dset = g.create_dataset(name, val.shape, dtype=val.dtype)
+      if not val.shape:
+        # scalar
+        param_dset[()] = val
+      else:
+        param_dset[:] = val
+
+
+def load_weights_from_hdf5_group(f, layers):
+  """Implements topological (order-based) weight loading.
+
+  Arguments:
+      f: A pointer to a HDF5 group.
+      layers: a list of target layers.
+
+  Raises:
+      ValueError: in case of mismatch between provided layers
+          and weights file.
+  """
+  if 'keras_version' in f.attrs:
+    original_keras_version = f.attrs['keras_version'].decode('utf8')
+  else:
+    original_keras_version = '1'
+  if 'backend' in f.attrs:
+    original_backend = f.attrs['backend'].decode('utf8')
+  else:
+    original_backend = None
+
+  filtered_layers = []
+  for layer in layers:
+    weights = layer.weights
+    if weights:
+      filtered_layers.append(layer)
+
+  layer_names = load_attributes_from_hdf5_group(f, 'layer_names')
+  filtered_layer_names = []
+  for name in layer_names:
+    g = f[name]
+    weight_names = load_attributes_from_hdf5_group(g, 'weight_names')
+    if weight_names:
+      filtered_layer_names.append(name)
+  layer_names = filtered_layer_names
+  if len(layer_names) != len(filtered_layers):
+    raise ValueError('You are trying to load a weight file '
+                     'containing ' + str(len(layer_names)) +
+                     ' layers into a model with ' + str(len(filtered_layers)) +
+                     ' layers.')
+
+  # We batch weight value assignments in a single backend call
+  # which provides a speedup in TensorFlow.
+  weight_value_tuples = []
+  for k, name in enumerate(layer_names):
+    g = f[name]
+    weight_names = load_attributes_from_hdf5_group(g, 'weight_names')
+    weight_values = [np.asarray(g[weight_name]) for weight_name in weight_names]
+    layer = filtered_layers[k]
+    symbolic_weights = layer.weights
+    weight_values = preprocess_weights_for_loading(
+        layer, weight_values, original_keras_version, original_backend)
+    if len(weight_values) != len(symbolic_weights):
+      raise ValueError('Layer #' + str(k) + ' (named "' + layer.name +
+                       '" in the current model) was found to '
+                       'correspond to layer ' + name + ' in the save file. '
+                       'However the new layer ' + layer.name + ' expects ' +
+                       str(len(symbolic_weights)) +
+                       ' weights, but the saved weights have ' +
+                       str(len(weight_values)) + ' elements.')
+    weight_value_tuples += zip(symbolic_weights, weight_values)
+  K.batch_set_value(weight_value_tuples)
+
+
+def load_weights_from_hdf5_group_by_name(f, layers):
+  """Implements name-based weight loading.
+
+  (instead of topological weight loading).
+
+  Layers that have no matching name are skipped.
+
+  Arguments:
+      f: A pointer to a HDF5 group.
+      layers: a list of target layers.
+
+  Raises:
+      ValueError: in case of mismatch between provided layers
+          and weights file.
+  """
+  if 'keras_version' in f.attrs:
+    original_keras_version = f.attrs['keras_version'].decode('utf8')
+  else:
+    original_keras_version = '1'
+  if 'backend' in f.attrs:
+    original_backend = f.attrs['backend'].decode('utf8')
+  else:
+    original_backend = None
+
+  # New file format.
+  layer_names = load_attributes_from_hdf5_group(f, 'layer_names')
+
+  # Reverse index of layer name to list of layers with name.
+  index = {}
+  for layer in layers:
+    if layer.name:
+      index.setdefault(layer.name, []).append(layer)
+
+  # We batch weight value assignments in a single backend call
+  # which provides a speedup in TensorFlow.
+  weight_value_tuples = []
+  for k, name in enumerate(layer_names):
+    g = f[name]
+    weight_names = load_attributes_from_hdf5_group(g, 'weight_names')
+    weight_values = [np.asarray(g[weight_name]) for weight_name in weight_names]
+
+    for layer in index.get(name, []):
+      symbolic_weights = layer.weights
+      weight_values = preprocess_weights_for_loading(
+          layer, weight_values, original_keras_version, original_backend)
+      if len(weight_values) != len(symbolic_weights):
+        raise ValueError('Layer #' + str(k) + ' (named "' + layer.name +
+                         '") expects ' + str(len(symbolic_weights)) +
+                         ' weight(s), but the saved weights' + ' have ' +
+                         str(len(weight_values)) + ' element(s).')
+      # Set values.
+      for i in range(len(weight_values)):
+        if K.int_shape(symbolic_weights[i]) != weight_values[i].shape:
+          raise ValueError('Layer #' + str(k) +' (named "' + layer.name +
+                           '"), weight ' + str(symbolic_weights[i]) +
+                           ' has shape {}'.format(K.int_shape(
+                               symbolic_weights[i])) +
+                           ', but the saved weight has shape ' +
+                           str(weight_values[i].shape) + '.')
+
+        else:
+          weight_value_tuples.append((symbolic_weights[i], weight_values[i]))
+  K.batch_set_value(weight_value_tuples)
+
+
+def save_attributes_to_hdf5_group(group, name, data):
+  """Saves attributes (data) of the specified name into the HDF5 group.
+
+  This method deals with an inherent problem of HDF5 file which is not
+  able to store data larger than HDF5_OBJECT_HEADER_LIMIT bytes.
+
+  Arguments:
+      group: A pointer to a HDF5 group.
+      name: A name of the attributes to save.
+      data: Attributes data to store.
+
+  Raises:
+    RuntimeError: If any single attribute is too large to be saved.
+  """
+  # Check that no item in `data` is larger than `HDF5_OBJECT_HEADER_LIMIT`
+  # because in that case even chunking the array would not make the saving
+  # possible.
+  bad_attributes = [x for x in data if len(x) > HDF5_OBJECT_HEADER_LIMIT]
+
+  # Expecting this to never be true.
+  if bad_attributes:
+    raise RuntimeError('The following attributes cannot be saved to HDF5 '
+                       'file because they are larger than %d bytes: %s' %
+                       (HDF5_OBJECT_HEADER_LIMIT,
+                        ', '.join([x for x in bad_attributes])))
+
+  data_npy = np.asarray(data)
+
+  num_chunks = 1
+  chunked_data = np.array_split(data_npy, num_chunks)
+
+  # This will never loop forever thanks to the test above.
+  while any(x.nbytes > HDF5_OBJECT_HEADER_LIMIT for x in chunked_data):
+    num_chunks += 1
+    chunked_data = np.array_split(data_npy, num_chunks)
+
+  if num_chunks > 1:
+    for chunk_id, chunk_data in enumerate(chunked_data):
+      group.attrs['%s%d' % (name, chunk_id)] = chunk_data
+  else:
+    group.attrs[name] = data
+
+
+def load_attributes_from_hdf5_group(group, name):
+  """Loads attributes of the specified name from the HDF5 group.
+
+  This method deals with an inherent problem
+  of HDF5 file which is not able to store
+  data larger than HDF5_OBJECT_HEADER_LIMIT bytes.
+
+  Arguments:
+      group: A pointer to a HDF5 group.
+      name: A name of the attributes to load.
+
+  Returns:
+      data: Attributes data.
+  """
+  if name in group.attrs:
+    data = [n.decode('utf8') for n in group.attrs[name]]
+  else:
+    data = []
+    chunk_id = 0
+    while '%s%d' % (name, chunk_id) in group.attrs:
+      data.extend(
+          [n.decode('utf8') for n in group.attrs['%s%d' % (name, chunk_id)]])
+      chunk_id += 1
+  return data
diff --git a/tensorflow/python/keras/engine/saving_test.py b/tensorflow/python/keras/saving/hdf5_format_test.py
similarity index 98%
rename from tensorflow/python/keras/engine/saving_test.py
rename to tensorflow/python/keras/saving/hdf5_format_test.py
index 92fac6f242..3b496e70c1 100644
--- a/tensorflow/python/keras/engine/saving_test.py
+++ b/tensorflow/python/keras/saving/hdf5_format_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #,============================================================================
-"""Tests for model saving."""
+"""Tests for model saving in the HDF5 format."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -31,8 +31,8 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.keras import optimizers
-from tensorflow.python.keras.engine import saving
 from tensorflow.python.keras.engine import training
+from tensorflow.python.keras.saving import hdf5_format
 from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import random_ops
@@ -174,17 +174,17 @@ class TestWeightSavingAndLoading(test.TestCase, parameterized.TestCase):
     ]
     for layer, weights, input_shape in cases:
       layer.build(input_shape)
-      _ = keras.engine.saving.preprocess_weights_for_loading(
+      _ = hdf5_format.preprocess_weights_for_loading(
           layer, weights, original_keras_version='1')
 
     model = keras.models.Sequential([keras.layers.Dense(2, input_dim=2)])
-    _ = keras.engine.saving.preprocess_weights_for_loading(
+    _ = hdf5_format.preprocess_weights_for_loading(
         model, model.weights, original_keras_version='1')
 
     x = keras.Input((2,))
     y = keras.layers.Dense(2)(x)
     model = keras.models.Model(x, y)
-    _ = keras.engine.saving.preprocess_weights_for_loading(
+    _ = hdf5_format.preprocess_weights_for_loading(
         model, model.weights, original_keras_version='1')
 
   @parameterized.named_parameters(
@@ -215,7 +215,7 @@ class TestWeightSavingAndLoading(test.TestCase, parameterized.TestCase):
       layer = layer_class(**layer_args)
       layer.build(input_shape=layer_args.get('input_shape'))
       weights1 = layer.get_weights()
-      weights2 = keras.engine.saving.preprocess_weights_for_loading(
+      weights2 = hdf5_format.preprocess_weights_for_loading(
           layer, weights1)
       _ = [
           self.assertAllClose(x, y, rtol=1e-05)
@@ -274,7 +274,7 @@ class TestWeightSavingAndLoading(test.TestCase, parameterized.TestCase):
                         metrics=[keras.metrics.categorical_accuracy])
 
       f_ref_model = h5py.File(h5_path, 'w')
-      saving.save_weights_to_hdf5_group(f_ref_model, ref_model.layers)
+      hdf5_format.save_weights_to_hdf5_group(f_ref_model, ref_model.layers)
 
       f_model = h5py.File(h5_path, 'r')
       model = keras.models.Sequential()
@@ -288,7 +288,7 @@ class TestWeightSavingAndLoading(test.TestCase, parameterized.TestCase):
                                  r'Layer #0 \(named \"d1\"\) expects 1 '
                                  r'weight\(s\), but the saved weights have 2 '
                                  r'element\(s\)\.'):
-      saving.load_weights_from_hdf5_group_by_name(f_model, model.layers)
+      hdf5_format.load_weights_from_hdf5_group_by_name(f_model, model.layers)
 
   @test_util.run_deprecated_v1
   def test_sequential_weight_loading_group_name_with_incorrect_shape(self):
@@ -312,7 +312,7 @@ class TestWeightSavingAndLoading(test.TestCase, parameterized.TestCase):
                         metrics=[keras.metrics.categorical_accuracy])
 
       f_ref_model = h5py.File(h5_path, 'w')
-      saving.save_weights_to_hdf5_group(f_ref_model, ref_model.layers)
+      hdf5_format.save_weights_to_hdf5_group(f_ref_model, ref_model.layers)
 
       f_model = h5py.File(h5_path, 'r')
       model = keras.models.Sequential()
@@ -328,7 +328,7 @@ class TestWeightSavingAndLoading(test.TestCase, parameterized.TestCase):
                                    r'shape=\(3, 10\) dtype=float32> has '
                                    r'shape \(3, 10\), but the saved weight has '
                                    r'shape \(3, 5\)\.'):
-        saving.load_weights_from_hdf5_group_by_name(f_model, model.layers)
+        hdf5_format.load_weights_from_hdf5_group_by_name(f_model, model.layers)
 
 
 class TestWholeModelSaving(test.TestCase):
diff --git a/tensorflow/python/keras/saving/model_config.py b/tensorflow/python/keras/saving/model_config.py
new file mode 100644
index 0000000000..7f59ecd7df
--- /dev/null
+++ b/tensorflow/python/keras/saving/model_config.py
@@ -0,0 +1,96 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# pylint: disable=protected-access
+"""Functions that save the model's config into different formats.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import json
+
+from tensorflow.python.util.tf_export import keras_export
+
+# pylint: disable=g-import-not-at-top
+try:
+  import yaml
+except ImportError:
+  yaml = None
+# pylint: enable=g-import-not-at-top
+
+
+@keras_export('keras.models.model_from_config')
+def model_from_config(config, custom_objects=None):
+  """Instantiates a Keras model from its config.
+
+  Arguments:
+      config: Configuration dictionary.
+      custom_objects: Optional dictionary mapping names
+          (strings) to custom classes or functions to be
+          considered during deserialization.
+
+  Returns:
+      A Keras model instance (uncompiled).
+
+  Raises:
+      TypeError: if `config` is not a dictionary.
+  """
+  if isinstance(config, list):
+    raise TypeError('`model_from_config` expects a dictionary, not a list. '
+                    'Maybe you meant to use '
+                    '`Sequential.from_config(config)`?')
+  from tensorflow.python.keras.layers import deserialize  # pylint: disable=g-import-not-at-top
+  return deserialize(config, custom_objects=custom_objects)
+
+
+@keras_export('keras.models.model_from_yaml')
+def model_from_yaml(yaml_string, custom_objects=None):
+  """Parses a yaml model configuration file and returns a model instance.
+
+  Arguments:
+      yaml_string: YAML string encoding a model configuration.
+      custom_objects: Optional dictionary mapping names
+          (strings) to custom classes or functions to be
+          considered during deserialization.
+
+  Returns:
+      A Keras model instance (uncompiled).
+
+  Raises:
+      ImportError: if yaml module is not found.
+  """
+  if yaml is None:
+    raise ImportError('Requires yaml module installed (`pip install pyyaml`).')
+  config = yaml.load(yaml_string)
+  from tensorflow.python.keras.layers import deserialize  # pylint: disable=g-import-not-at-top
+  return deserialize(config, custom_objects=custom_objects)
+
+
+@keras_export('keras.models.model_from_json')
+def model_from_json(json_string, custom_objects=None):
+  """Parses a JSON model configuration file and returns a model instance.
+
+  Arguments:
+      json_string: JSON string encoding a model configuration.
+      custom_objects: Optional dictionary mapping names
+          (strings) to custom classes or functions to be
+          considered during deserialization.
+
+  Returns:
+      A Keras model instance (uncompiled).
+  """
+  config = json.loads(json_string)
+  from tensorflow.python.keras.layers import deserialize  # pylint: disable=g-import-not-at-top
+  return deserialize(config, custom_objects=custom_objects)
diff --git a/tensorflow/python/keras/saving/saved_model.py b/tensorflow/python/keras/saving/saved_model.py
index 5de6d2657f..06e02acf32 100644
--- a/tensorflow/python/keras/saving/saved_model.py
+++ b/tensorflow/python/keras/saving/saved_model.py
@@ -24,12 +24,8 @@ import six
 from tensorflow.python.client import session
 from tensorflow.python.framework import ops
 from tensorflow.python.keras import backend as K
-from tensorflow.python.keras import models as models_lib
 from tensorflow.python.keras import optimizers
-from tensorflow.python.keras.engine import sequential
-from tensorflow.python.keras.engine import training_utils
-from tensorflow.python.keras.metrics import Metric
-from tensorflow.python.keras.models import model_from_json
+from tensorflow.python.keras.saving import model_from_json
 from tensorflow.python.keras.saving import saving_utils
 from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import variables
@@ -135,7 +131,7 @@ def export(
   if serving_only:
     save_lib.save(
         model, export_dir,
-        signatures=training_utils.trace_model_call(model, input_signature))
+        signatures=saving_utils.trace_model_call(model, input_signature))
   else:
     _save_v1_format(model, export_dir, custom_objects, as_text, input_signature)
 
@@ -167,6 +163,8 @@ def _export_model_variables(model, saved_model_path):
 
 def _save_v1_format(model, path, custom_objects, as_text, input_signature):
   """Exports model to v1 SavedModel format."""
+  from tensorflow.python.keras.engine import sequential  # pylint: disable=g-import-not-at-top
+
   if not model._is_graph_network:
     if isinstance(model, sequential.Sequential):
       # If input shape is not directly set in the model, the exported model
@@ -250,6 +248,7 @@ def _export_mode(
     ValueError: If the train/eval mode is being exported, but the model does
       not have an optimizer.
   """
+  from tensorflow.python.keras import models as models_lib  # pylint: disable=g-import-not-at-top
   compile_clone = (mode != mode_keys.ModeKeys.PREDICT)
   if compile_clone and not model.optimizer:
     raise ValueError(
@@ -339,6 +338,7 @@ def _create_signature_def_map(model, mode):
   local_vars = set(ops.get_collection(ops.GraphKeys.LOCAL_VARIABLES))
   vars_to_add = set()
   if metrics is not None:
+    from tensorflow.python.keras.metrics import Metric  # pylint: disable=g-import-not-at-top
     for key, value in six.iteritems(metrics):
       if isinstance(value, Metric):
         vars_to_add.update(value.variables)
diff --git a/tensorflow/python/keras/saving/saving_utils.py b/tensorflow/python/keras/saving/saving_utils.py
index ddd98aa581..0b643453c9 100644
--- a/tensorflow/python/keras/saving/saving_utils.py
+++ b/tensorflow/python/keras/saving/saving_utils.py
@@ -18,7 +18,9 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.python.keras import metrics
+from tensorflow.python.eager import def_function
+from tensorflow.python.framework import tensor_spec
+from tensorflow.python.util import nest
 
 
 def extract_model_metrics(model):
@@ -33,6 +35,7 @@ def extract_model_metrics(model):
     Dictionary mapping metric names to tuples of (value, update) ops. May return
     `None` if the model does not contain any metrics.
   """
+  from tensorflow.python.keras import metrics  # pylint: disable=g-import-not-at-top
   if not getattr(model, '_compile_metrics', None):
     return None
 
@@ -43,3 +46,58 @@ def extract_model_metrics(model):
     m(model._compile_metrics_tensors[metric_name])
     eval_metric_ops[metric_name] = m
   return eval_metric_ops
+
+
+def trace_model_call(model, input_signature=None):
+  """Trace the model call to create a tf.function for exporting a Keras model.
+
+  Args:
+    model: A Keras model.
+    input_signature: optional, a list of tf.TensorSpec objects specifying the
+      inputs to the model.
+
+  Returns:
+    A tf.function wrapping the model's call function with input signatures set.
+
+  Raises:
+    ValueError: if input signature cannot be inferred from the model.
+  """
+  if input_signature is None:
+    if isinstance(model.call, def_function.PolymorphicFunction):
+      input_signature = model.call.input_signature
+
+  if input_signature is None:
+    try:
+      inputs = model.inputs
+      input_names = model.input_names
+    except AttributeError:
+      raise ValueError(
+          'Model {} cannot be saved because the input shapes have not been '
+          'set. Usually, input shapes are automatically determined from calling'
+          ' .fit() or .predict(). To manually set the shapes, call '
+          'model._set_inputs(inputs).'.format(model))
+    input_specs = []
+    for input_tensor, input_name in zip(inputs, input_names):
+      input_specs.append(tensor_spec.TensorSpec(
+          shape=input_tensor.shape, dtype=input_tensor.dtype,
+          name=input_name))
+    # The input signature of the call function is a list with one element, since
+    # all tensor inputs must be passed in as the first argument.
+    input_signature = [input_specs] if len(input_specs) > 1 else input_specs
+
+  # TODO(mdan): Should the model's call be autographed by default?
+  @def_function.function(input_signature=input_signature, autograph=False)
+  def _wrapped_model(*args):
+    """A concrete tf.function that wraps the model's call function."""
+    # When given a single input, Keras models will call the model on the tensor
+    # rather than a list consisting of the single tensor.
+    inputs = args[0] if len(input_signature) == 1 else list(args)
+    outputs_list = nest.flatten(model(inputs=inputs))
+    try:
+      output_names = model.output_names
+    except AttributeError:
+      from tensorflow.python.keras.engine import training_utils  # pylint: disable=g-import-not-at-top
+      output_names = training_utils.generic_output_names(outputs_list)
+    return {name: output for name, output in zip(output_names, outputs_list)}
+
+  return _wrapped_model
diff --git a/tensorflow/python/keras/saving/saving_utils_test.py b/tensorflow/python/keras/saving/saving_utils_test.py
new file mode 100644
index 0000000000..ae267e283c
--- /dev/null
+++ b/tensorflow/python/keras/saving/saving_utils_test.py
@@ -0,0 +1,209 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for saving utility functions."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+
+import numpy as np
+
+
+from tensorflow.python import keras
+from tensorflow.python.client import session as session_lib
+from tensorflow.python.eager import context
+from tensorflow.python.eager import def_function
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_spec
+from tensorflow.python.keras import backend as K
+from tensorflow.python.keras import keras_parameterized
+from tensorflow.python.keras import testing_utils
+from tensorflow.python.keras.saving import saving_utils
+from tensorflow.python.ops import array_ops
+from tensorflow.python.platform import test
+from tensorflow.python.saved_model import loader
+from tensorflow.python.saved_model import save as save_lib
+from tensorflow.python.saved_model import signature_constants
+from tensorflow.python.saved_model import tag_constants
+
+
+class TraceModelCallTest(keras_parameterized.TestCase):
+
+  def _assert_all_close(self, expected, actual):
+    if not context.executing_eagerly():
+      with self.cached_session() as sess:
+        K._initialize_variables(sess)
+        self.assertAllClose(expected, actual)
+    else:
+      self.assertAllClose(expected, actual)
+
+  @keras_parameterized.run_with_all_model_types
+  @keras_parameterized.run_all_keras_modes
+  def test_trace_model_outputs(self):
+    input_dim = 5 if testing_utils.get_model_type() == 'functional' else None
+    model = testing_utils.get_small_mlp(10, 3, input_dim)
+    inputs = array_ops.ones((8, 5))
+
+    if input_dim is None:
+      with self.assertRaisesRegexp(ValueError,
+                                   'input shapes have not been set'):
+        saving_utils.trace_model_call(model)
+      model._set_inputs(inputs)
+
+    fn = saving_utils.trace_model_call(model)
+    signature_outputs = fn(inputs)
+    expected_outputs = {model.output_names[0]: model(inputs)}
+
+    self._assert_all_close(expected_outputs, signature_outputs)
+
+  @keras_parameterized.run_with_all_model_types
+  @keras_parameterized.run_all_keras_modes
+  def test_trace_model_outputs_after_fitting(self):
+    input_dim = 5 if testing_utils.get_model_type() == 'functional' else None
+    model = testing_utils.get_small_mlp(10, 3, input_dim)
+    model.compile(optimizer='sgd', loss='mse')
+    model.fit(x=np.random.random((8, 5)),
+              y=np.random.random((8, 3)), epochs=2)
+
+    inputs = array_ops.ones((8, 5))
+
+    fn = saving_utils.trace_model_call(model)
+    signature_outputs = fn(inputs)
+    expected_outputs = {model.output_names[0]: model(inputs)}
+
+    self._assert_all_close(expected_outputs, signature_outputs)
+
+  @keras_parameterized.run_with_all_model_types(exclude_models='sequential')
+  @keras_parameterized.run_all_keras_modes
+  def test_trace_multi_io_model_outputs(self):
+    input_dim = 5
+    num_classes = 3
+    num_classes_b = 4
+    input_a = keras.layers.Input(shape=(input_dim,), name='input_a')
+    input_b = keras.layers.Input(shape=(input_dim,), name='input_b')
+
+    dense = keras.layers.Dense(num_classes, name='dense')
+    dense2 = keras.layers.Dense(num_classes_b, name='dense2')
+    dropout = keras.layers.Dropout(0.5, name='dropout')
+    branch_a = [input_a, dense]
+    branch_b = [input_b, dense, dense2, dropout]
+
+    model = testing_utils.get_multi_io_model(branch_a, branch_b)
+
+    input_a_np = np.random.random((10, input_dim)).astype(np.float32)
+    input_b_np = np.random.random((10, input_dim)).astype(np.float32)
+
+    if testing_utils.get_model_type() == 'subclass':
+      with self.assertRaisesRegexp(ValueError,
+                                   'input shapes have not been set'):
+        saving_utils.trace_model_call(model)
+
+    model.compile(optimizer='sgd', loss='mse')
+    model.fit(x=[np.random.random((8, input_dim)).astype(np.float32),
+                 np.random.random((8, input_dim)).astype(np.float32)],
+              y=[np.random.random((8, num_classes)).astype(np.float32),
+                 np.random.random((8, num_classes_b)).astype(np.float32)],
+              epochs=2)
+
+    fn = saving_utils.trace_model_call(model)
+    signature_outputs = fn([input_a_np, input_b_np])
+    outputs = model([input_a_np, input_b_np])
+    expected_outputs = {model.output_names[0]: outputs[0],
+                        model.output_names[1]: outputs[1]}
+
+    self._assert_all_close(expected_outputs, signature_outputs)
+
+  @keras_parameterized.run_all_keras_modes
+  def test_specify_input_signature(self):
+    model = testing_utils.get_small_sequential_mlp(10, 3, None)
+    inputs = array_ops.ones((8, 5))
+
+    with self.assertRaisesRegexp(ValueError, 'input shapes have not been set'):
+      saving_utils.trace_model_call(model)
+
+    fn = saving_utils.trace_model_call(
+        model, [tensor_spec.TensorSpec(shape=[None, 5], dtype=dtypes.float32)])
+    signature_outputs = fn(inputs)
+    expected_outputs = {model.output_names[0]: model(inputs)}
+    self._assert_all_close(expected_outputs, signature_outputs)
+
+  @keras_parameterized.run_all_keras_modes
+  def test_subclassed_model_with_input_signature(self):
+
+    class Model(keras.Model):
+
+      def __init__(self):
+        super(Model, self).__init__()
+        self.dense = keras.layers.Dense(3, name='dense')
+
+      @def_function.function(
+          input_signature=[[tensor_spec.TensorSpec([None, 5], dtypes.float32),
+                            tensor_spec.TensorSpec([None], dtypes.float32)]],)
+      def call(self, inputs, *args):
+        x, y = inputs
+        return self.dense(x) + y
+
+    model = Model()
+    fn = saving_utils.trace_model_call(model)
+    x = array_ops.ones((8, 5), dtype=dtypes.float32)
+    y = array_ops.ones((3,), dtype=dtypes.float32)
+    expected_outputs = {'output_1': model([x, y])}
+    signature_outputs = fn([x, y])
+    self._assert_all_close(expected_outputs, signature_outputs)
+
+
+def _import_and_infer(save_dir, inputs):
+  """Import a SavedModel into a TF 1.x-style graph and run `signature_key`."""
+  graph = ops.Graph()
+  with graph.as_default(), session_lib.Session() as session:
+    model = loader.load(session, [tag_constants.SERVING], save_dir)
+    signature = model.signature_def[
+        signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY]
+    assert set(inputs.keys()) == set(signature.inputs.keys())
+    feed_dict = {}
+    for arg_name in inputs.keys():
+      feed_dict[graph.get_tensor_by_name(signature.inputs[arg_name].name)] = (
+          inputs[arg_name])
+    output_dict = {}
+    for output_name, output_tensor_info in signature.outputs.items():
+      output_dict[output_name] = graph.get_tensor_by_name(
+          output_tensor_info.name)
+    return session.run(output_dict, feed_dict=feed_dict)
+
+
+class ModelSaveTest(keras_parameterized.TestCase):
+
+  @keras_parameterized.run_with_all_model_types
+  @keras_parameterized.run_all_keras_modes(always_skip_v1=True)
+  def test_model_save(self):
+    input_dim = 5
+    model = testing_utils.get_small_mlp(10, 3, input_dim)
+    inputs = array_ops.ones((8, 5))
+
+    if testing_utils.get_model_type() == 'subclass':
+      model._set_inputs(inputs)
+
+    save_dir = os.path.join(self.get_temp_dir(), 'saved_model')
+    save_lib.save(model, save_dir)
+
+    self.assertAllClose(
+        {model.output_names[0]: model.predict_on_batch(inputs)},
+        _import_and_infer(save_dir, {model.input_names[0]: np.ones((8, 5))}))
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/python/saved_model/BUILD b/tensorflow/python/saved_model/BUILD
index 8da6ff5142..8a54888582 100644
--- a/tensorflow/python/saved_model/BUILD
+++ b/tensorflow/python/saved_model/BUILD
@@ -71,7 +71,7 @@ py_library(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:lib",
         "//tensorflow/python:platform",
-        "//tensorflow/python:training",
+        "//tensorflow/python:saver",
         "//tensorflow/python:util",
         "//tensorflow/python:variables",
     ],
-- 
GitLab


From bcf94f9cf2a4cfe8d2b7a76a1fe1be553a591edd Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Thu, 10 Jan 2019 17:04:59 -0800
Subject: [PATCH 0529/2345] Minor refactor of
 _WhileBodyGradFuncGraph._capture_helper.

It was getting kind of long. This also makes minor stylistic changes to the moved code.

PiperOrigin-RevId: 228804655
---
 tensorflow/python/ops/while_v2.py | 61 +++++++++++++++++++------------
 1 file changed, 37 insertions(+), 24 deletions(-)

diff --git a/tensorflow/python/ops/while_v2.py b/tensorflow/python/ops/while_v2.py
index f5a51bb1bc..cefadfb098 100644
--- a/tensorflow/python/ops/while_v2.py
+++ b/tensorflow/python/ops/while_v2.py
@@ -724,31 +724,9 @@ class _WhileBodyGradFuncGraph(util.WhileBodyFuncGraph):
     if captured_tensor is not None:
       return captured_tensor
 
+    # Resource tensors are not accumulated and handled specially.
     if tensor.dtype == dtypes.resource:
-      # Resource-type tensors are not accumulated.
-      # If a resource tensor exists in the loop body it must either be a loop
-      # input or an output of a nested While op inside the loop body which
-      # had captured the external resource.
-      if tensor in self._forward_graph.inputs:
-        index = self._forward_graph.inputs.index(tensor)
-      elif tensor.op.type == "While":
-        # Captured resources occur at the same index in the lists of inputs and
-        # outputs of a while op. So we lookup the input of `tensor.op` at the
-        # same index as the index of `tensor` in the `tensor.op.outputs`.
-        index = self._forward_graph.inputs.index(
-            tensor.op.inputs[tensor.value_index])
-      else:
-        raise ValueError(
-            "Taking gradient of a while loop which creates"
-            " a resource in its body is not supported: %s" % str(tensor))
-      # This must be a loop invariant.
-      assert self._forward_graph.inputs[index] == self._forward_graph.outputs[
-          index], "Resource tensors must be loop invariants %s." % str(
-              self._forward_graph._while.inputs[index])
-      tensor_in_outer_graph = self._forward_graph._while.inputs[index]
-      self._indirect_captures[tensor] = self.capture(
-          tensor_in_outer_graph, whitelisted=True)
-      return self._indirect_captures[tensor]
+      return self._resource_capture_helper(tensor)
 
     # Create or find an existing accumulator output for `tensor` in the forward
     # graph, and fetch from this accumulator in the gradient graph to get the
@@ -789,6 +767,41 @@ class _WhileBodyGradFuncGraph(util.WhileBodyFuncGraph):
     self.popped_tensor_lists[captured_accumulator] = new_tensor_list
     return captured_tensor
 
+  def _resource_capture_helper(self, tensor):
+    """Returns the captured resource tensor.
+
+    Resource-type tensors are not accumulated. If a resource tensor exists in
+    the loop body it must either be a loop input or an output of a nested While
+    op inside the loop body which had captured the external resource.
+
+    Args:
+      tensor: the external resource Tensor to be captured.
+
+    Returns:
+      Tensor in this graph.
+    """
+    assert tensor.dtype == dtypes.resource
+    if tensor in self._forward_graph.inputs:
+      index = self._forward_graph.inputs.index(tensor)
+    elif tensor.op.type == "While":
+      # Captured resources occur at the same index in the lists of inputs and
+      # outputs of a while op. So we lookup the input of `tensor.op` at the
+      # same index as the index of `tensor` in the `tensor.op.outputs`.
+      index = self._forward_graph.inputs.index(
+          tensor.op.inputs[tensor.value_index])
+    else:
+      raise ValueError(
+          "Taking gradient of a while loop which creates "
+          "a resource in its body is not supported: %s" % tensor)
+    # This must be a loop invariant.
+    assert self._forward_graph.inputs[index] == self._forward_graph.outputs[
+        index], ("Resource tensors must be loop invariants %s." %
+                 self._forward_graph._while.inputs[index])
+    tensor_in_outer_graph = self._forward_graph._while.inputs[index]
+    self._indirect_captures[tensor] = self.capture(
+        tensor_in_outer_graph, whitelisted=True)
+    return self._indirect_captures[tensor]
+
 
 def _check_shapes_compat(output_tensors, shape_invariants, input_tensors):
   for (t, shape, input_t) in zip(output_tensors, shape_invariants,
-- 
GitLab


From 3fc2b09b601e9d2c3e1bdd80d299c31b3adb0c0d Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Thu, 10 Jan 2019 17:19:40 -0800
Subject: [PATCH 0530/2345] Centralize the logic that infers accumulator type
 from element type.

PiperOrigin-RevId: 228806506
---
 tensorflow/stream_executor/cuda/cuda_dnn.cc | 132 ++++++++++----------
 1 file changed, 68 insertions(+), 64 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index 0bd953fe3b..60648075c0 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -2710,6 +2710,23 @@ cudnnDataType_t GetRnnComputeType(dnn::DataType data_type) {
   }
 }
 
+dnn::DataType GetConvAccumulatorType(dnn::DataType data_type) {
+  switch (data_type) {
+    case dnn::DataType::kFloat:
+    case dnn::DataType::kDouble:
+      return data_type;
+    case dnn::DataType::kHalf:
+      return CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
+                 ? dnn::DataType::kFloat
+                 : dnn::DataType::kHalf;
+    case dnn::DataType::kInt8:
+    case dnn::DataType::kInt32:
+      return dnn::DataType::kInt32;
+    default:
+      LOG(FATAL) << "Invalid DNN data type: " << static_cast<int>(data_type);
+  }
+}
+
 // Determines whether we can safely perform a winograd non-fused convolution for
 // the given input and output shapes.  This works around b/68264959, an integer
 // overflow in cuDNNv5 and cuDNNv6.
@@ -3405,8 +3422,8 @@ bool CudnnSupport::DoConvolve(
   return IsStatusOk(
       DoConvolveImpl(stream, batch_descriptor, input_data, filter_descriptor,
                      filter_data, convolution_descriptor, output_descriptor,
-                     output_data, dnn::DataType::kFloat, algorithm_desc,
-                     scratch_memory, output_profile_result),
+                     output_data, GetConvAccumulatorType(dnn::DataType::kFloat),
+                     algorithm_desc, scratch_memory, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3423,8 +3440,9 @@ bool CudnnSupport::DoConvolve(
   return IsStatusOk(
       DoConvolveImpl(stream, batch_descriptor, input_data, filter_descriptor,
                      filter_data, convolution_descriptor, output_descriptor,
-                     output_data, dnn::DataType::kDouble, algorithm_desc,
-                     scratch_memory, output_profile_result),
+                     output_data,
+                     GetConvAccumulatorType(dnn::DataType::kDouble),
+                     algorithm_desc, scratch_memory, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3439,15 +3457,11 @@ bool CudnnSupport::DoConvolve(
     const dnn::AlgorithmDesc& algorithm_desc,
     DeviceMemory<uint8>* scratch_memory,
     dnn::ProfileResult* output_profile_result) {
-  dnn::DataType acc_type =
-      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
-          ? dnn::DataType::kFloat
-          : dnn::DataType::kHalf;
   return IsStatusOk(
       DoConvolveImpl(stream, batch_descriptor, input_data, filter_descriptor,
                      filter_data, convolution_descriptor, output_descriptor,
-                     output_data, acc_type, algorithm_desc, scratch_memory,
-                     output_profile_result),
+                     output_data, GetConvAccumulatorType(dnn::DataType::kHalf),
+                     algorithm_desc, scratch_memory, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3465,13 +3479,13 @@ bool CudnnSupport::DoFusedConvolve(
     const dnn::AlgorithmConfig& algorithm_config,
     dnn::ProfileResult* output_profile_result) {
   return IsStatusOk(
-      DoFusedConvolveImpl(stream, conv_input_descriptor, conv_input_data,
-                          conv_input_scale, filter_descriptor, filter_data,
-                          convolution_descriptor, side_input_data,
-                          side_input_scale, bias_descriptor, biases,
-                          activation_mode, output_descriptor, output_data,
-                          dnn::DataType::kDouble, scratch_allocator,
-                          algorithm_config, output_profile_result),
+      DoFusedConvolveImpl(
+          stream, conv_input_descriptor, conv_input_data, conv_input_scale,
+          filter_descriptor, filter_data, convolution_descriptor,
+          side_input_data, side_input_scale, bias_descriptor, biases,
+          activation_mode, output_descriptor, output_data,
+          GetConvAccumulatorType(dnn::DataType::kDouble), scratch_allocator,
+          algorithm_config, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3489,13 +3503,13 @@ bool CudnnSupport::DoFusedConvolve(
     const dnn::AlgorithmConfig& algorithm_config,
     dnn::ProfileResult* output_profile_result) {
   return IsStatusOk(
-      DoFusedConvolveImpl(stream, conv_input_descriptor, conv_input_data,
-                          conv_input_scale, filter_descriptor, filter_data,
-                          convolution_descriptor, side_input_data,
-                          side_input_scale, bias_descriptor, biases,
-                          activation_mode, output_descriptor, output_data,
-                          dnn::DataType::kFloat, scratch_allocator,
-                          algorithm_config, output_profile_result),
+      DoFusedConvolveImpl(
+          stream, conv_input_descriptor, conv_input_data, conv_input_scale,
+          filter_descriptor, filter_data, convolution_descriptor,
+          side_input_data, side_input_scale, bias_descriptor, biases,
+          activation_mode, output_descriptor, output_data,
+          GetConvAccumulatorType(dnn::DataType::kFloat), scratch_allocator,
+          algorithm_config, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3513,17 +3527,14 @@ bool CudnnSupport::DoFusedConvolve(
     DeviceMemory<Eigen::half>* output_data, ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
     dnn::ProfileResult* output_profile_result) {
-  dnn::DataType acc_type =
-      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
-          ? dnn::DataType::kFloat
-          : dnn::DataType::kHalf;
   return IsStatusOk(
       DoFusedConvolveImpl(
           stream, conv_input_descriptor, conv_input_data, conv_input_scale,
           filter_descriptor, filter_data, convolution_descriptor,
           side_input_data, side_input_scale, bias_descriptor, biases,
-          activation_mode, output_descriptor, output_data, acc_type,
-          scratch_allocator, algorithm_config, output_profile_result),
+          activation_mode, output_descriptor, output_data,
+          GetConvAccumulatorType(dnn::DataType::kHalf), scratch_allocator,
+          algorithm_config, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3549,13 +3560,13 @@ bool CudnnSupport::DoFusedConvolve(
     return false;
   }
   return IsStatusOk(
-      DoFusedConvolveImpl(stream, conv_input_descriptor, conv_input_data,
-                          conv_input_scale, filter_descriptor, filter_data,
-                          convolution_descriptor, side_input_data,
-                          side_input_scale, bias_descriptor, biases,
-                          activation_mode, output_descriptor, output_data,
-                          dnn::DataType::kInt32, scratch_allocator,
-                          algorithm_config, output_profile_result),
+      DoFusedConvolveImpl(
+          stream, conv_input_descriptor, conv_input_data, conv_input_scale,
+          filter_descriptor, filter_data, convolution_descriptor,
+          side_input_data, side_input_scale, bias_descriptor, biases,
+          activation_mode, output_descriptor, output_data,
+          GetConvAccumulatorType(dnn::DataType::kInt8), scratch_allocator,
+          algorithm_config, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3790,8 +3801,8 @@ bool CudnnSupport::DoConvolveBackwardData(
       DoConvolveBackwardDataImpl(
           stream, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, dnn::DataType::kDouble, algorithm_desc,
-          scratch_memory, output_profile_result),
+          backward_input_data, GetConvAccumulatorType(dnn::DataType::kDouble),
+          algorithm_desc, scratch_memory, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3810,8 +3821,8 @@ bool CudnnSupport::DoConvolveBackwardData(
       DoConvolveBackwardDataImpl(
           stream, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, dnn::DataType::kFloat, algorithm_desc,
-          scratch_memory, output_profile_result),
+          backward_input_data, GetConvAccumulatorType(dnn::DataType::kFloat),
+          algorithm_desc, scratch_memory, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3826,16 +3837,12 @@ bool CudnnSupport::DoConvolveBackwardData(
     const dnn::AlgorithmDesc& algorithm_desc,
     DeviceMemory<uint8>* scratch_memory,
     dnn::ProfileResult* output_profile_result) {
-  dnn::DataType acc_type =
-      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
-          ? dnn::DataType::kFloat
-          : dnn::DataType::kHalf;
   return IsStatusOk(
-      DoConvolveBackwardDataImpl(stream, filter_descriptor, filter_data,
-                                 output_descriptor, backward_output_data,
-                                 convolution_descriptor, input_descriptor,
-                                 backward_input_data, acc_type, algorithm_desc,
-                                 scratch_memory, output_profile_result),
+      DoConvolveBackwardDataImpl(
+          stream, filter_descriptor, filter_data, output_descriptor,
+          backward_output_data, convolution_descriptor, input_descriptor,
+          backward_input_data, GetConvAccumulatorType(dnn::DataType::kHalf),
+          algorithm_desc, scratch_memory, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -4085,8 +4092,8 @@ bool CudnnSupport::DoConvolveBackwardFilter(
       DoConvolveBackwardFilterImpl(
           stream, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, dnn::DataType::kDouble, algorithm_desc,
-          scratch_memory, output_profile_result),
+          backward_filter_data, GetConvAccumulatorType(dnn::DataType::kDouble),
+          algorithm_desc, scratch_memory, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -4105,8 +4112,8 @@ bool CudnnSupport::DoConvolveBackwardFilter(
       DoConvolveBackwardFilterImpl(
           stream, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, dnn::DataType::kFloat, algorithm_desc,
-          scratch_memory, output_profile_result),
+          backward_filter_data, GetConvAccumulatorType(dnn::DataType::kFloat),
+          algorithm_desc, scratch_memory, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -4121,16 +4128,13 @@ bool CudnnSupport::DoConvolveBackwardFilter(
     const dnn::AlgorithmDesc& algorithm_desc,
     DeviceMemory<uint8>* scratch_memory,
     dnn::ProfileResult* output_profile_result) {
-  dnn::DataType acc_type =
-      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
-          ? dnn::DataType::kFloat
-          : dnn::DataType::kHalf;
-  return IsStatusOk(DoConvolveBackwardFilterImpl(
-                        stream, input_descriptor, input_data, output_descriptor,
-                        backward_output_data, convolution_descriptor,
-                        filter_descriptor, backward_filter_data, acc_type,
-                        algorithm_desc, scratch_memory, output_profile_result),
-                    /*report_error=*/!output_profile_result);
+  return IsStatusOk(
+      DoConvolveBackwardFilterImpl(
+          stream, input_descriptor, input_data, output_descriptor,
+          backward_output_data, convolution_descriptor, filter_descriptor,
+          backward_filter_data, GetConvAccumulatorType(dnn::DataType::kHalf),
+          algorithm_desc, scratch_memory, output_profile_result),
+      /*report_error=*/!output_profile_result);
 }
 
 template <class T>
-- 
GitLab


From 578bd3a2765f32112ef69bf355b4e1221b1c723b Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Thu, 10 Jan 2019 17:33:58 -0800
Subject: [PATCH 0531/2345] Move clustering ops to core.

PiperOrigin-RevId: 228808275
---
 tensorflow/contrib/factorization/BUILD        | 24 +----
 .../contrib/factorization/kernels/BUILD       | 28 ------
 .../factorization/ops/clustering_ops.cc       | 91 -------------------
 .../python/ops/clustering_ops.py              | 15 +--
 tensorflow/core/BUILD                         |  3 +
 .../api_def_KMC2ChainInitialization.pbtxt     | 30 ++++++
 ...api_def_KmeansPlusPlusInitialization.pbtxt | 44 +++++++++
 .../base_api/api_def_NearestNeighbors.pbtxt   | 43 +++++++++
 .../api_def_KMC2ChainInitialization.pbtxt     |  4 +
 ...api_def_KmeansPlusPlusInitialization.pbtxt |  4 +
 .../python_api/api_def_NearestNeighbors.pbtxt |  4 +
 tensorflow/core/kernels/BUILD                 | 27 ++++++
 .../kernels/clustering_ops.cc                 | 10 +-
 .../kernels/clustering_ops_test.cc            |  0
 tensorflow/core/ops/clustering_ops.cc         | 43 +++++++++
 tensorflow/python/BUILD                       |  8 ++
 16 files changed, 221 insertions(+), 157 deletions(-)
 delete mode 100644 tensorflow/contrib/factorization/ops/clustering_ops.cc
 create mode 100644 tensorflow/core/api_def/base_api/api_def_KMC2ChainInitialization.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_KmeansPlusPlusInitialization.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_NearestNeighbors.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_KMC2ChainInitialization.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_KmeansPlusPlusInitialization.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_NearestNeighbors.pbtxt
 rename tensorflow/{contrib/factorization => core}/kernels/clustering_ops.cc (99%)
 rename tensorflow/{contrib/factorization => core}/kernels/clustering_ops_test.cc (100%)
 create mode 100644 tensorflow/core/ops/clustering_ops.cc

diff --git a/tensorflow/contrib/factorization/BUILD b/tensorflow/contrib/factorization/BUILD
index e344d7a23b..cb86efb8da 100644
--- a/tensorflow/contrib/factorization/BUILD
+++ b/tensorflow/contrib/factorization/BUILD
@@ -28,7 +28,6 @@ tf_custom_op_py_library(
         "python/ops/wals.py",
     ],
     dso = [
-        ":python/ops/_clustering_ops.so",
         ":python/ops/_factorization_ops.so",
     ],
     kernels = [
@@ -38,12 +37,12 @@ tf_custom_op_py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":factorization_ops_test_utils_py",
-        ":gen_clustering_ops",
         ":gen_factorization_ops",
         "//tensorflow/contrib/framework:framework_py",
         "//tensorflow/contrib/util:util_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:check_ops",
+        "//tensorflow/python:clustering_ops_gen",
         "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:data_flow_ops",
         "//tensorflow/python:embedding_ops",
@@ -77,17 +76,6 @@ py_library(
     ],
 )
 
-# Ops
-tf_custom_op_library(
-    name = "python/ops/_clustering_ops.so",
-    srcs = [
-        "ops/clustering_ops.cc",
-    ],
-    deps = [
-        "//tensorflow/contrib/factorization/kernels:clustering_ops",
-    ],
-)
-
 tf_custom_op_library(
     name = "python/ops/_factorization_ops.so",
     srcs = [
@@ -100,26 +88,16 @@ tf_custom_op_library(
 )
 
 tf_gen_op_libs([
-    "clustering_ops",
     "factorization_ops",
 ])
 
 cc_library(
     name = "all_ops",
     deps = [
-        ":clustering_ops_op_lib",
         ":factorization_ops_op_lib",
     ],
 )
 
-tf_gen_op_wrapper_py(
-    name = "gen_clustering_ops",
-    out = "python/ops/gen_clustering_ops.py",
-    deps = [
-        ":clustering_ops_op_lib",
-    ],
-)
-
 tf_gen_op_wrapper_py(
     name = "gen_factorization_ops",
     out = "python/ops/gen_factorization_ops.py",
diff --git a/tensorflow/contrib/factorization/kernels/BUILD b/tensorflow/contrib/factorization/kernels/BUILD
index ea8b9a17a2..23d7e088d0 100644
--- a/tensorflow/contrib/factorization/kernels/BUILD
+++ b/tensorflow/contrib/factorization/kernels/BUILD
@@ -11,7 +11,6 @@ load("//tensorflow:tensorflow.bzl", "tf_cc_test")
 cc_library(
     name = "all_kernels",
     deps = [
-        ":clustering_ops",
         ":masked_matmul_ops",
         ":wals_solver_ops",
         "@protobuf_archive//:protobuf_headers",
@@ -29,17 +28,6 @@ cc_library(
     alwayslink = 1,
 )
 
-cc_library(
-    name = "clustering_ops",
-    srcs = ["clustering_ops.cc"],
-    deps = [
-        "//tensorflow/core:framework_headers_lib",
-        "//third_party/eigen3",
-        "@protobuf_archive//:protobuf_headers",
-    ],
-    alwayslink = 1,
-)
-
 cc_library(
     name = "masked_matmul_ops",
     srcs = ["masked_matmul_ops.cc"],
@@ -51,19 +39,3 @@ cc_library(
     ],
     alwayslink = 1,
 )
-
-tf_cc_test(
-    name = "clustering_ops_test",
-    srcs = ["clustering_ops_test.cc"],
-    deps = [
-        ":clustering_ops",
-        "//tensorflow/contrib/factorization:clustering_ops_op_lib",
-        "//tensorflow/core:core_cpu",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:protos_all_cc",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-        "//tensorflow/core:testlib",
-    ],
-)
diff --git a/tensorflow/contrib/factorization/ops/clustering_ops.cc b/tensorflow/contrib/factorization/ops/clustering_ops.cc
deleted file mode 100644
index 2686702c1d..0000000000
--- a/tensorflow/contrib/factorization/ops/clustering_ops.cc
+++ /dev/null
@@ -1,91 +0,0 @@
-// Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not
-// use this file except in compliance with the License.  You may obtain a copy
-// of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
-// License for the specific language governing permissions and limitations under
-// the License.
-// ==============================================================================
-
-#include "tensorflow/core/framework/common_shape_fns.h"
-#include "tensorflow/core/framework/op.h"
-
-namespace tensorflow {
-
-REGISTER_OP("KmeansPlusPlusInitialization")
-    .Input("points: float32")
-    .Input("num_to_sample: int64")
-    .Input("seed: int64")
-    .Input("num_retries_per_sample: int64")
-    .Output("samples: float32")
-    .SetShapeFn(shape_inference::UnknownShape)
-    .Doc(R"(
-Selects num_to_sample rows of input using the KMeans++ criterion.
-
-Rows of points are assumed to be input points. One row is selected at random.
-Subsequent rows are sampled with probability proportional to the squared L2
-distance from the nearest row selected thus far till num_to_sample rows have
-been sampled.
-
-points: Matrix of shape (n, d). Rows are assumed to be input points.
-num_to_sample: Scalar. The number of rows to sample. This value must not be
-  larger than n.
-seed: Scalar. Seed for initializing the random number generator.
-num_retries_per_sample: Scalar. For each row that is sampled, this parameter
-  specifies the number of additional points to draw from the current
-  distribution before selecting the best. If a negative value is specified, a
-  heuristic is used to sample O(log(num_to_sample)) additional points.
-samples: Matrix of shape (num_to_sample, d). The sampled rows.
-)");
-
-REGISTER_OP("KMC2ChainInitialization")
-    .Input("distances: float32")
-    .Input("seed: int64")
-    .Output("index: int64")
-    .SetShapeFn(shape_inference::ScalarShape)
-    .Doc(R"(
-Returns the index of a data point that should be added to the seed set.
-
-Entries in distances are assumed to be squared distances of candidate points to
-the already sampled centers in the seed set. The op constructs one Markov chain
-of the k-MC^2 algorithm and returns the index of one candidate point to be added
-as an additional cluster center.
-
-distances: Vector with squared distances to the closest previously sampled
-  cluster center for each candidate point.
-seed: Scalar. Seed for initializing the random number generator.
-index: Scalar with the index of the sampled point.
-)");
-
-REGISTER_OP("NearestNeighbors")
-    .Input("points: float32")
-    .Input("centers: float32")
-    .Input("k: int64")
-    .Output("nearest_center_indices: int64")
-    .Output("nearest_center_distances: float32")
-    .SetShapeFn(shape_inference::UnknownShape)
-    .Doc(R"(
-Selects the k nearest centers for each point.
-
-Rows of points are assumed to be input points. Rows of centers are assumed to be
-the list of candidate centers. For each point, the k centers that have least L2
-distance to it are computed.
-
-points: Matrix of shape (n, d). Rows are assumed to be input points.
-centers: Matrix of shape (m, d). Rows are assumed to be centers.
-k: Scalar. Number of nearest centers to return for each point. If k is larger
-  than m, then only m centers are returned.
-nearest_center_indices: Matrix of shape (n, min(m, k)). Each row contains the
-  indices of the centers closest to the corresponding point, ordered by
-  increasing distance.
-nearest_center_distances: Matrix of shape (n, min(m, k)). Each row contains the
-  squared L2 distance to the corresponding center in nearest_center_indices.
-)");
-
-}  // namespace tensorflow
diff --git a/tensorflow/contrib/factorization/python/ops/clustering_ops.py b/tensorflow/contrib/factorization/python/ops/clustering_ops.py
index 84e80791f4..d48b89cbac 100644
--- a/tensorflow/contrib/factorization/python/ops/clustering_ops.py
+++ b/tensorflow/contrib/factorization/python/ops/clustering_ops.py
@@ -18,28 +18,23 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.factorization.python.ops import gen_clustering_ops
-# go/tf-wildcard-import
-# pylint: disable=wildcard-import
-from tensorflow.contrib.factorization.python.ops.gen_clustering_ops import *
-# pylint: enable=wildcard-import
-from tensorflow.contrib.util import loader
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import gen_clustering_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn_impl
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops.embedding_ops import embedding_lookup
-from tensorflow.python.platform import resource_loader
-
-_clustering_ops = loader.load_op_library(
-    resource_loader.get_path_to_datafile('_clustering_ops.so'))
+# go/tf-wildcard-import
+# pylint: disable=wildcard-import
+from tensorflow.python.ops.gen_clustering_ops import *
+# pylint: enable=wildcard-import
 
 # Euclidean distance between vectors U and V is defined as \\(||U - V||_F\\)
 # which is the square root of the sum of the absolute squares of the elements
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 690fa4a19e..20f5815423 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1074,6 +1074,7 @@ tf_gen_op_libs(
         "tensor_forest_ops",
         "candidate_sampling_ops",
         "checkpoint_ops",
+        "clustering_ops",
         "collective_ops",
         "control_flow_ops",
         "ctc_ops",
@@ -1228,6 +1229,7 @@ cc_library(
         ":tensor_forest_ops_op_lib",
         ":candidate_sampling_ops_op_lib",
         ":checkpoint_ops_op_lib",
+        ":clustering_ops_op_lib",
         ":collective_ops_op_lib",
         ":control_flow_ops_op_lib",
         ":ctc_ops_op_lib",
@@ -1382,6 +1384,7 @@ cc_library(
         "//tensorflow/core/kernels:tensor_forest_ops",
         "//tensorflow/core/kernels:candidate_sampler_ops",
         "//tensorflow/core/kernels:checkpoint_ops",
+        "//tensorflow/core/kernels:clustering_ops",
         "//tensorflow/core/kernels:collective_ops",
         "//tensorflow/core/kernels:control_flow_ops",
         "//tensorflow/core/kernels:ctc_ops",
diff --git a/tensorflow/core/api_def/base_api/api_def_KMC2ChainInitialization.pbtxt b/tensorflow/core/api_def/base_api/api_def_KMC2ChainInitialization.pbtxt
new file mode 100644
index 0000000000..c6ff4b9e2d
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_KMC2ChainInitialization.pbtxt
@@ -0,0 +1,30 @@
+op {
+  graph_op_name: "KMC2ChainInitialization"
+  visibility: HIDDEN
+  in_arg {
+    name: "distances"
+    description: <<END
+Vector with squared distances to the closest previously sampled cluster center
+for each candidate point.
+END
+  }
+  in_arg {
+    name: "seed"
+    description: <<END
+Scalar. Seed for initializing the random number generator.
+END
+  }
+  out_arg {
+    name: "index"
+    description: <<END
+Scalar with the index of the sampled point.
+END
+  }
+  summary: "Returns the index of a data point that should be added to the seed set."
+  description: <<END
+Entries in distances are assumed to be squared distances of candidate points to
+the already sampled centers in the seed set. The op constructs one Markov chain
+of the k-MC^2 algorithm and returns the index of one candidate point to be added
+as an additional cluster center.
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_KmeansPlusPlusInitialization.pbtxt b/tensorflow/core/api_def/base_api/api_def_KmeansPlusPlusInitialization.pbtxt
new file mode 100644
index 0000000000..461820486b
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_KmeansPlusPlusInitialization.pbtxt
@@ -0,0 +1,44 @@
+op {
+  graph_op_name: "KmeansPlusPlusInitialization"
+  visibility: HIDDEN
+  in_arg {
+    name: "points"
+    description: <<END
+Matrix of shape (n, d). Rows are assumed to be input points.
+END
+  }
+  in_arg {
+    name: "num_to_sample"
+    description: <<END
+Scalar. The number of rows to sample. This value must not be larger than n.
+END
+  }
+  in_arg {
+    name: "seed"
+    description: <<END
+Scalar. Seed for initializing the random number generator.
+END
+  }
+  in_arg {
+    name: "num_retries_per_sample"
+    description: <<END
+Scalar. For each row that is sampled, this parameter
+specifies the number of additional points to draw from the current
+distribution before selecting the best. If a negative value is specified, a
+heuristic is used to sample O(log(num_to_sample)) additional points.
+END
+  }
+  out_arg {
+    name: "samples"
+    description: <<END
+Matrix of shape (num_to_sample, d). The sampled rows.
+END
+  }
+  summary: "Selects num_to_sample rows of input using the KMeans++ criterion."
+  description: <<END
+Rows of points are assumed to be input points. One row is selected at random.
+Subsequent rows are sampled with probability proportional to the squared L2
+distance from the nearest row selected thus far till num_to_sample rows have
+been sampled.
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_NearestNeighbors.pbtxt b/tensorflow/core/api_def/base_api/api_def_NearestNeighbors.pbtxt
new file mode 100644
index 0000000000..2bdf68fb9f
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_NearestNeighbors.pbtxt
@@ -0,0 +1,43 @@
+op {
+  graph_op_name: "NearestNeighbors"
+  visibility: HIDDEN
+  in_arg {
+    name: "points"
+    description: <<END
+Matrix of shape (n, d). Rows are assumed to be input points.
+END
+  }
+  in_arg {
+    name: "centers"
+    description: <<END
+Matrix of shape (m, d). Rows are assumed to be centers.
+END
+  }
+  in_arg {
+    name: "k"
+    description: <<END
+Number of nearest centers to return for each point. If k is larger than m, then
+only m centers are returned.
+END
+  }
+  out_arg {
+    name: "nearest_center_indices"
+    description: <<END
+Matrix of shape (n, min(m, k)). Each row contains the indices of the centers
+closest to the corresponding point, ordered by increasing distance.
+END
+  }
+  out_arg {
+    name: "nearest_center_distances"
+    description: <<END
+Matrix of shape (n, min(m, k)). Each row contains the squared L2 distance to the
+corresponding center in nearest_center_indices.
+END
+  }
+  summary: "Selects the k nearest centers for each point."
+  description: <<END
+Rows of points are assumed to be input points. Rows of centers are assumed to be
+the list of candidate centers. For each point, the k centers that have least L2
+distance to it are computed.
+END
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_KMC2ChainInitialization.pbtxt b/tensorflow/core/api_def/python_api/api_def_KMC2ChainInitialization.pbtxt
new file mode 100644
index 0000000000..3e8ec113f8
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_KMC2ChainInitialization.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "KMC2ChainInitialization"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_KmeansPlusPlusInitialization.pbtxt b/tensorflow/core/api_def/python_api/api_def_KmeansPlusPlusInitialization.pbtxt
new file mode 100644
index 0000000000..83aa3a3a69
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_KmeansPlusPlusInitialization.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "KmeansPlusPlusInitialization"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_NearestNeighbors.pbtxt b/tensorflow/core/api_def/python_api/api_def_NearestNeighbors.pbtxt
new file mode 100644
index 0000000000..ad05bb3486
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_NearestNeighbors.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "NearestNeighbors"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 0045c5dd19..4db0505a8f 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -152,6 +152,33 @@ tf_kernel_library(
     ],
 )
 
+tf_kernel_library(
+    name = "clustering_ops",
+    prefix = "clustering_ops",
+    deps = [
+        "//tensorflow/core:clustering_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:framework_headers_lib",
+        "//tensorflow/core:lib",
+    ],
+)
+
+tf_cc_test(
+    name = "clustering_ops_test",
+    srcs = ["clustering_ops_test.cc"],
+    deps = [
+        ":clustering_ops",
+        "//tensorflow/core:clustering_ops_op_lib",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+    ],
+)
+
 tf_kernel_library(
     name = "collective_ops",
     prefix = "collective_ops",
diff --git a/tensorflow/contrib/factorization/kernels/clustering_ops.cc b/tensorflow/core/kernels/clustering_ops.cc
similarity index 99%
rename from tensorflow/contrib/factorization/kernels/clustering_ops.cc
rename to tensorflow/core/kernels/clustering_ops.cc
index 025534d540..7e1a1fdcd2 100644
--- a/tensorflow/contrib/factorization/kernels/clustering_ops.cc
+++ b/tensorflow/core/kernels/clustering_ops.cc
@@ -392,7 +392,7 @@ class NearestNeighborsOp : public OpKernel {
       for (; start < limit; ++start) {
         const int64 start_row = num_points * start / num_units;
         const int64 limit_row = num_points * (start + 1) / num_units;
-        CHECK_LE(limit_row, num_points);
+        DCHECK_LE(limit_row, num_points);
         const int64 num_rows = limit_row - start_row;
         auto points_shard = points.middleRows(start_row, num_rows);
         const Eigen::VectorXf points_half_squared_norm =
@@ -430,7 +430,7 @@ class NearestNeighborsOp : public OpKernel {
       const Eigen::Ref<const Eigen::VectorXf>& centers_half_squared_norm,
       const Eigen::Ref<MatrixXi64RowMajor>& nearest_center_indices,
       const Eigen::Ref<MatrixXfRowMajor>& nearest_center_distances) {
-    CHECK_LE(k, centers.rows());
+    DCHECK_LE(k, centers.rows());
     if (centers.rows() <= kNearestNeighborsCentersMaxBlockSize) {
       FindKNearestCentersOneBlock(k, points, points_half_squared_norm, centers,
                                   centers_half_squared_norm,
@@ -451,7 +451,7 @@ class NearestNeighborsOp : public OpKernel {
       const Eigen::Ref<const Eigen::VectorXf>& centers_half_squared_norm,
       Eigen::Ref<MatrixXi64RowMajor> nearest_center_indices,
       Eigen::Ref<MatrixXfRowMajor> nearest_center_distances) {
-    CHECK_LE(k, centers.rows());
+    DCHECK_LE(k, centers.rows());
     const int64 num_points = points.rows();
     const MatrixXfRowMajor inner_product = points * centers.transpose();
     // Find nearest neighbors.
@@ -500,8 +500,8 @@ class NearestNeighborsOp : public OpKernel {
       Eigen::Ref<MatrixXfRowMajor> nearest_center_distances) {
     const int64 num_points = points.rows();
     const int64 num_centers = centers.rows();
-    CHECK_LE(k, num_centers);
-    CHECK_GT(num_centers, kNearestNeighborsCentersMaxBlockSize);
+    DCHECK_LE(k, num_centers);
+    DCHECK_GT(num_centers, kNearestNeighborsCentersMaxBlockSize);
     // Store nearest neighbors with first block of centers directly into the
     // output matrices.
     int64 out_k = std::min(k, kNearestNeighborsCentersMaxBlockSize);
diff --git a/tensorflow/contrib/factorization/kernels/clustering_ops_test.cc b/tensorflow/core/kernels/clustering_ops_test.cc
similarity index 100%
rename from tensorflow/contrib/factorization/kernels/clustering_ops_test.cc
rename to tensorflow/core/kernels/clustering_ops_test.cc
diff --git a/tensorflow/core/ops/clustering_ops.cc b/tensorflow/core/ops/clustering_ops.cc
new file mode 100644
index 0000000000..189f00730b
--- /dev/null
+++ b/tensorflow/core/ops/clustering_ops.cc
@@ -0,0 +1,43 @@
+// Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License.  You may obtain a copy
+// of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+// License for the specific language governing permissions and limitations under
+// the License.
+// ==============================================================================
+
+#include "tensorflow/core/framework/common_shape_fns.h"
+#include "tensorflow/core/framework/op.h"
+
+namespace tensorflow {
+
+REGISTER_OP("KmeansPlusPlusInitialization")
+    .Input("points: float32")
+    .Input("num_to_sample: int64")
+    .Input("seed: int64")
+    .Input("num_retries_per_sample: int64")
+    .Output("samples: float32")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("KMC2ChainInitialization")
+    .Input("distances: float32")
+    .Input("seed: int64")
+    .Output("index: int64")
+    .SetShapeFn(shape_inference::ScalarShape);
+
+REGISTER_OP("NearestNeighbors")
+    .Input("points: float32")
+    .Input("centers: float32")
+    .Input("k: int64")
+    .Output("nearest_center_indices: int64")
+    .Output("nearest_center_distances: float32")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+}  // namespace tensorflow
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 82c542f51d..659f0735cb 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -1680,6 +1680,14 @@ tf_gen_op_wrapper_private_py(
     ],
 )
 
+tf_gen_op_wrapper_private_py(
+    name = "clustering_ops_gen",
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        "//tensorflow/core:clustering_ops_op_lib",
+    ],
+)
+
 tf_gen_op_wrapper_private_py(
     name = "collective_ops_gen",
     visibility = ["//tensorflow:internal"],
-- 
GitLab


From bde6a28b2c8df473aa674aa6b9ee3ba4aa43db2f Mon Sep 17 00:00:00 2001
From: Samantha Andow <samdow@fb.com>
Date: Thu, 10 Jan 2019 18:04:53 -0800
Subject: [PATCH 0532/2345] Added unit test for InputList fix

@Co-authored-by: irenedea<irenedea@fb.com>
@Co-authored-by: melissagrueter<melissagrueter@fb.com>
---
 .../op/core/GeneratedOperationsTest.java        | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/tensorflow/java/src/test/java/org/tensorflow/op/core/GeneratedOperationsTest.java b/tensorflow/java/src/test/java/org/tensorflow/op/core/GeneratedOperationsTest.java
index 42d126c3c4..49c4ff639e 100644
--- a/tensorflow/java/src/test/java/org/tensorflow/op/core/GeneratedOperationsTest.java
+++ b/tensorflow/java/src/test/java/org/tensorflow/op/core/GeneratedOperationsTest.java
@@ -17,6 +17,7 @@ package org.tensorflow.op.core;
 
 import static org.junit.Assert.assertEquals;
 
+import java.util.ArrayList;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
@@ -40,4 +41,20 @@ public final class GeneratedOperationsTest {
       }
     }
   }
+
+  @Test
+  public void testListInputTensorOutput() {
+    try (Graph g = new Graph();
+        Session sess = new Session(g)) {
+      Ops ops = Ops.create(g);
+      ArrayList<Operand<Integer>> inputs = new ArrayList<Operand<Integer>>();
+      inputs.add(ops.constant(1));
+      inputs.add(ops.constant(2));
+      inputs.add(ops.constant(3));
+      Operand<Integer> x = ops.math().addN(inputs);
+      try (Tensor<Integer> result = sess.runner().fetch(x).run().get(0).expect(Integer.class)) {
+        assertEquals(6, result.intValue());
+      }
+    }
+  }
 }
-- 
GitLab


From b0571c92bd5893ab2ceadd00be2b928821b9d490 Mon Sep 17 00:00:00 2001
From: Davide Libenzi <dlibenzi@google.com>
Date: Thu, 10 Jan 2019 18:01:40 -0800
Subject: [PATCH 0533/2345] Automated rollback of commit
 55fda7fb66f1774ac5271e523259faa7d2a44d0c

PiperOrigin-RevId: 228811469
---
 .../xla/service/cpu/cpu_executable.cc         |  29 +++-
 .../xla/service/gpu/gpu_executable.cc         |  26 +++-
 tensorflow/compiler/xrt/kernels/BUILD         |   1 +
 .../compiler/xrt/kernels/xrt_execute_op.cc    |  20 +++
 tensorflow/compiler/xrt/tests/raw_api_test.cc | 127 +++++++++++++++++-
 tensorflow/compiler/xrt/xrt_state.cc          |  43 +++++-
 tensorflow/compiler/xrt/xrt_state.h           |  23 +++-
 7 files changed, 252 insertions(+), 17 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/cpu_executable.cc b/tensorflow/compiler/xla/service/cpu/cpu_executable.cc
index 412c2715b9..23d0af3423 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_executable.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_executable.cc
@@ -213,6 +213,8 @@ StatusOr<ScopedShapedBuffer> CpuExecutable::CreateResultShapedBuffer(
       /*on_host_shape=*/result_shape(),
       /*on_device_shape=*/result_shape(), run_options->allocator(),
       stream->parent()->device_ordinal());
+  const HloInputOutputAliasConfig& input_output_alias =
+      module().input_output_alias_config();
 
   // Move OwningDeviceMemory values which contain the array(s) of the result
   // into the respective location in ScopedShapedBuffer which is returned to the
@@ -232,12 +234,31 @@ StatusOr<ScopedShapedBuffer> CpuExecutable::CreateResultShapedBuffer(
         TF_ASSIGN_OR_RETURN(
             const BufferAllocation::Slice slice,
             this->assignment_->GetUniqueSlice(src, buffer_source->index()));
-        CHECK(!slice.allocation()->is_entry_computation_parameter());
-
         const BufferAllocation::Index buffer_index = slice.index();
         OwningDeviceMemory& buffer = buffers[buffer_index];
-        CHECK(!buffer.is_null() || buffer.size() == 0);
-        *device_memory = buffer.Forget();
+        if (!slice.allocation()->is_entry_computation_parameter()) {
+          // If the buffer coming out of the result is from a parameter, the
+          // owning buffer will be null, and that means the caller aliased some
+          // parameter buffer to an output one (via the
+          // HloInputOutputAliasConfig API). If that is the case, the caller
+          // will receive a partially complete scoped shaped buffer, which they
+          // will have to fill up on return. Unfortunately the interface to the
+          // execute APIs are ShapedBuffer pointer based, which assumes caller
+          // ownership, and hence a buffer coming from there cannot be part of
+          // the new ScopedShapedBuffer we create for the result (which assumes
+          // ownership).
+          *device_memory = buffer.Forget();
+        } else {
+          auto output_alias = input_output_alias.GetAliasedOutput(
+              slice.allocation()->parameter_number(),
+              slice.allocation()->param_shape_index());
+          CHECK(output_alias)
+              << "Ouput buffer is coming from parameter "
+              << slice.allocation()->parameter_number() << " at index "
+              << slice.allocation()->param_shape_index()
+              << ", but no alias exists";
+          CHECK_EQ(*output_alias, index);
+        }
         return Status::OK();
       }));
   return std::move(result_buffer);
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_executable.cc b/tensorflow/compiler/xla/service/gpu/gpu_executable.cc
index 128cecd765..434060ad89 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_executable.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_executable.cc
@@ -310,12 +310,34 @@ StatusOr<ScopedShapedBuffer> GpuExecutable::ExecuteOnStream(
         TF_ASSIGN_OR_RETURN(
             const BufferAllocation::Slice slice,
             this->assignment_->GetUniqueSlice(src_hlo, sources[0]->index()));
-        CHECK(!slice.allocation()->is_entry_computation_parameter());
 
         se::DeviceMemoryBase src_base =
             buffer_allocations->GetDeviceAddress(slice.index());
         CHECK(!src_base.is_null() || src_base.size() == 0);
-        *device_memory = src_base;
+        if (!slice.allocation()->is_entry_computation_parameter()) {
+          // If the buffer coming out of the result is from a parameter, it
+          // means the caller aliased some parameter buffer to an output one
+          // (via the HloInputOutputAliasConfig API). If that is the case, the
+          // caller will receive a partially complete scoped shaped buffer,
+          // which they will have to fill up on return.
+          // Unfortunately the interface to the execute APIs are ShapedBuffer
+          // pointer based, which assumes caller ownership, and hence a buffer
+          // coming from there cannot be part of the new ScopedShapedBuffer we
+          // create for the result (which assumes ownership).
+          *device_memory = src_base;
+        } else {
+          const HloInputOutputAliasConfig& input_output_alias =
+              module().input_output_alias_config();
+          auto output_alias = input_output_alias.GetAliasedOutput(
+              slice.allocation()->parameter_number(),
+              slice.allocation()->param_shape_index());
+          CHECK(output_alias)
+              << "Ouput buffer is coming from parameter "
+              << slice.allocation()->parameter_number() << " at index "
+              << slice.allocation()->param_shape_index()
+              << ", but no alias exists";
+          CHECK_EQ(*output_alias, index);
+        }
         buffers_in_result.insert(src_base);
         return Status::OK();
       }));
diff --git a/tensorflow/compiler/xrt/kernels/BUILD b/tensorflow/compiler/xrt/kernels/BUILD
index 78d093ec4a..d38f41ab43 100644
--- a/tensorflow/compiler/xrt/kernels/BUILD
+++ b/tensorflow/compiler/xrt/kernels/BUILD
@@ -55,6 +55,7 @@ cc_library(
         "//tensorflow/compiler/xla/client:xla_computation",
         "//tensorflow/compiler/xla/service:compiler",
         "//tensorflow/compiler/xla/service:computation_placer",
+        "//tensorflow/compiler/xla/service:hlo",
         "//tensorflow/compiler/xrt:xrt_proto",
         "//tensorflow/compiler/xrt:xrt_utils",
         "//tensorflow/core:core_cpu_internal",
diff --git a/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc b/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc
index 7f0ac123a5..116c193cab 100644
--- a/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc
+++ b/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc
@@ -19,6 +19,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/service/computation_placer.h"
+#include "tensorflow/compiler/xla/service/hlo_input_output_alias_config.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/statusor.h"
@@ -228,6 +229,25 @@ Status XRTExecuteOp::DoWork(OpKernelContext* context) {
   TF_RETURN_IF_ERROR(XRTTupleAllocation::CreateFromBuffer(
       shaped_buffer, device_ref.backend(), device_ref.device_ordinal(),
       &output_tuple));
+
+  // The ScopedShapedBuffer returned by the executable Run() API, in case of
+  // input/output buffer aliasing, might have holes in it, which need to be
+  // filled using the proper input tuples buffers which are the source of
+  // aliasing.
+  const xla::HloInputOutputAliasConfig& input_output_alias =
+      executable->executable()->module().input_output_alias_config();
+  auto alias_function =
+      [&](const xla::ShapeIndex& output_index,
+          const xla::HloInputOutputAliasConfig::Alias& alias) -> Status {
+    TF_RET_CHECK(alias.parameter_number < input_tuples.size());
+    return alias.kind == xla::HloInputOutputAliasConfig::AliasKind::kUserAlias
+               ? output_tuple->AliasBufferFrom(
+                     *input_tuples[alias.parameter_number],
+                     alias.parameter_index, output_index)
+               : Status::OK();
+  };
+  TF_RETURN_IF_ERROR(input_output_alias.ForEachAliasWithStatus(alias_function));
+
   if (config_proto.return_exploded_tuple() &&
       output_tuple->on_device_shape().IsTuple()) {
     int64 tuple_element_count =
diff --git a/tensorflow/compiler/xrt/tests/raw_api_test.cc b/tensorflow/compiler/xrt/tests/raw_api_test.cc
index be0c4b9392..8e70cc3959 100644
--- a/tensorflow/compiler/xrt/tests/raw_api_test.cc
+++ b/tensorflow/compiler/xrt/tests/raw_api_test.cc
@@ -96,14 +96,21 @@ xla::LiteralProto FloatMatrix(
   return array.ToProto();
 }
 
+xla::Literal ReadOutputLiteral(const std::vector<Tensor>& outputs, size_t idx) {
+  xla::LiteralProto response;
+  CHECK(response.ParseFromString(outputs[idx].scalar<string>()()));
+  return xla::Literal::CreateFromProto(response).ValueOrDie();
+}
+
 bool CompareLiteralProtos(const xla::LiteralProto& a,
                           const xla::LiteralProto& b) {
   auto l_a = xla::Literal::CreateFromProto(a).ValueOrDie();
   auto l_b = xla::Literal::CreateFromProto(b).ValueOrDie();
   bool equal = l_a == l_b;
   if (!equal) {
-    LOG(INFO) << "LiteralProtos don't match: " << a.DebugString()
-              << " != " << b.DebugString();
+    LOG(INFO) << "LiteralProtos don't match:\n"
+              << a.DebugString() << "\n!=\n"
+              << b.DebugString();
   }
   return equal;
 }
@@ -113,8 +120,19 @@ bool CompareLiteralToLiteralProto(const xla::Literal& a,
   auto l_b = xla::Literal::CreateFromProto(b).ValueOrDie();
   bool equal = a == l_b;
   if (!equal) {
-    LOG(INFO) << "Literal and LiteralProto don't match "
-              << a.ToProto().DebugString() << " != " << b.DebugString();
+    LOG(INFO) << "Literal and LiteralProto don't match:\n"
+              << a.ToProto().DebugString() << "\n!=\n"
+              << b.DebugString();
+  }
+  return equal;
+}
+
+bool CompareLiterals(const xla::Literal& a, const xla::Literal& b) {
+  bool equal = a == b;
+  if (!equal) {
+    LOG(INFO) << "Literals don't match:\n"
+              << a.ToProto().DebugString() << "\n!=\n"
+              << b.ToProto().DebugString();
   }
   return equal;
 }
@@ -939,6 +957,107 @@ TEST(RawApiTest, LeakCompilationReference) {
   TF_EXPECT_OK(session.Run({c_handle.handle}, &outputs));
 }
 
+TEST(RawApiTest, CompileAndExecuteWithReusedBuffers) {
+  xla::Shape element_shape = xla::ShapeUtil::MakeShape(xla::F32, {2});
+  xla::Shape shape =
+      xla::ShapeUtil::MakeTupleShape({element_shape, element_shape});
+  xla::Shape return_shape = xla::ShapeUtil::MakeTupleShape(
+      {element_shape, element_shape, element_shape, element_shape});
+  xla::XlaBuilder builder("ReuseBuffer");
+  auto param = xla::Parameter(&builder, 0, shape, "param");
+  auto p0 = xla::GetTupleElement(param, 0);
+  auto p1 = xla::GetTupleElement(param, 1);
+  auto add = xla::Add(p0, p1);
+  auto sub = xla::Sub(p0, p1);
+  xla::Tuple(&builder, {add, sub, p0, p1});
+
+  // Flip the tuple literals in the input handle.
+  builder.SetUpAlias({1}, 0, {0});
+  builder.SetUpAlias({0}, 0, {1});
+
+  auto computation = builder.Build().ValueOrDie();
+
+  auto literal0 = xla::LiteralUtil::CreateR1<float>({1.0f, 2.0f});
+  auto literal1 = xla::LiteralUtil::CreateR1<float>({5.0f, 9.0f});
+  auto literal = xla::LiteralUtil::MakeTuple({&literal0, &literal1});
+
+  xrt::XLAAllocation param_alloc;
+  *param_alloc.mutable_value() = literal.ToProto();
+
+  xrt::XLAComputation c;
+  auto config = c.mutable_config();
+  auto shapes = config->mutable_program_shape();
+  *shapes->add_parameters() = shape.ToProto();
+  *shapes->mutable_result() = return_shape.ToProto();
+  StoreComputationSnapshot(computation, c.mutable_hlo_snapshot());
+
+  xrt::XRTExecutionConfig e;
+  e.set_release_input_handles(false);
+  e.set_release_compilation_handle(true);
+
+  Scope root = Scope::NewRootScope().WithDevice(DeviceFromFlag());
+  ClientSession session(root);
+  auto e_config =
+      ops::Const(root.WithDevice("/device:CPU:0"), e.SerializeAsString());
+  auto c_data =
+      ops::Const(root.WithDevice("/device:CPU:0"), c.SerializeAsString());
+  auto c_handle = ops::XRTCompile(root, c_data);
+  auto param_value = ops::Const(root.WithDevice("/device:CPU:0"),
+                                param_alloc.SerializeAsString());
+  auto param_handle = ops::XRTAllocate(root, param_value);
+  TF_ASSERT_OK(root.status());
+
+  std::vector<Tensor> outputs;
+  TF_EXPECT_OK(session.Run({param_handle}, &outputs));
+
+  int64 alloc_handle = outputs[0].scalar<int64>()();
+
+  // Note that we release the result handle immediately, but since we aliased
+  // the output buffers onto the input allocation ones (held in alloc_handle),
+  // we can fetch the result from there.
+  auto result =
+      ops::XRTExecute(root, c_handle.handle, e_config, {Input(alloc_handle)});
+  auto read_back = ops::XRTReadLiteral(root, result);
+  auto release = ops::XRTReleaseAllocationHandle(
+      root.WithControlDependencies(read_back), result);
+  TF_ASSERT_OK(root.status());
+
+  outputs.clear();
+  TF_EXPECT_OK(session.Run(tensorflow::ClientSession::FeedType(), {read_back},
+                           {release}, &outputs));
+
+  xla::Literal exec_literal = ReadOutputLiteral(outputs, 0);
+  auto exec_literal_parts = exec_literal.DecomposeTuple();
+  ASSERT_EQ(exec_literal_parts.size(), 4);
+
+  EXPECT_TRUE(CompareLiterals(exec_literal_parts[2], literal0));
+  EXPECT_TRUE(CompareLiterals(exec_literal_parts[3], literal1));
+
+  // Now we read back the original input handle values, which at this point
+  // should contain the result of the XLA computation.
+  auto read_handle = ops::XRTReadLiteral(root, Input(alloc_handle));
+  TF_ASSERT_OK(root.status());
+  auto release_handle = ops::XRTReleaseAllocationHandle(
+      root.WithControlDependencies(read_handle), Input(alloc_handle));
+  TF_ASSERT_OK(root.status());
+
+  outputs.clear();
+  TF_EXPECT_OK(session.Run(tensorflow::ClientSession::FeedType(), {read_handle},
+                           {release_handle}, &outputs));
+
+  xla::Literal return_literal = ReadOutputLiteral(outputs, 0);
+
+  auto expected_literal0 = xla::LiteralUtil::CreateR1<float>({6.0f, 11.0f});
+  auto expected_literal1 = xla::LiteralUtil::CreateR1<float>({-4.0f, -7.0f});
+  // The first element of the computation returned tuple would be the add
+  // (expected_literal0), but since we flipped the buffers, the sub
+  // (expected_literal1) should come first.
+  auto expected_literal =
+      xla::LiteralUtil::MakeTuple({&expected_literal1, &expected_literal0});
+
+  EXPECT_TRUE(CompareLiterals(return_literal, expected_literal));
+}
+
 TEST(RawApiTest, CompileAndExecuteWithS64Argument) {
   xrt::XLAAllocation p0;
   *p0.mutable_value() = xla::LiteralUtil::CreateR0<int64>(11031965).ToProto();
diff --git a/tensorflow/compiler/xrt/xrt_state.cc b/tensorflow/compiler/xrt/xrt_state.cc
index 343460ff10..13c275aaca 100644
--- a/tensorflow/compiler/xrt/xrt_state.cc
+++ b/tensorflow/compiler/xrt/xrt_state.cc
@@ -133,7 +133,8 @@ Status AllocateScopedShapedBuffer(
 XRTBufferAllocation::XRTBufferAllocation(const se::DeviceMemoryBase& allocation,
                                          int device_ordinal,
                                          xla::DeviceMemoryAllocator* allocator)
-    : allocation_(allocation),
+    : size_(allocation.size()),
+      allocation_(allocation),
       device_ordinal_(device_ordinal),
       allocator_(allocator) {
   if (VLOG_IS_ON(2)) {
@@ -223,8 +224,19 @@ Status XRTTupleAllocation::ToLiteral(xla::Backend* backend, int device_ordinal,
                                      xla::Literal* literal) {
   auto transfer_manager = backend->transfer_manager();
   TF_ASSIGN_OR_RETURN(auto stream, backend->BorrowStream(device_ordinal));
+
+  // Validate the allocation buffers as if nulls gets to
+  // TransferLiteralFromDevice() a CHECK is issued.
+  xla::ShapedBuffer shaped_buffer = ToShapedBuffer();
+  for (auto& index_buffer : shaped_buffer.buffers()) {
+    if (index_buffer.second.is_null()) {
+      return errors::InvalidArgument("Literal buffer at index ",
+                                     index_buffer.first.ToString(),
+                                     " has been released");
+    }
+  }
   TF_ASSIGN_OR_RETURN(*literal, transfer_manager->TransferLiteralFromDevice(
-                                    stream.get(), ToShapedBuffer()));
+                                    stream.get(), shaped_buffer));
   return Status::OK();
 }
 
@@ -505,11 +517,34 @@ xla::ShapedBuffer XRTTupleAllocation::ToShapedBuffer() {
   return shaped_buffer;
 }
 
+Status XRTTupleAllocation::AliasBufferFrom(const XRTTupleAllocation& source,
+                                           const xla::ShapeIndex& source_index,
+                                           const xla::ShapeIndex& dest_index) {
+  XRTBufferAllocation* source_buffer = source.buffers_.element(source_index);
+  XRTBufferAllocation* dest_buffer = buffers_.element(dest_index);
+  // We allow the destination size being zero, because there are cases where we
+  // are coming in later filling in null/uninitialized device buffers.
+  // In all other cases, the size of the new buffer must match.
+  if (source_buffer->size() != dest_buffer->size() &&
+      dest_buffer->size() != 0) {
+    return errors::InvalidArgument(
+        "Source buffer at index ", source_index.ToString(),
+        " does not match the size of destination buffer at index ",
+        dest_index.ToString(), ": ", source_buffer->size(), " vs ",
+        dest_buffer->size());
+  }
+  *buffers_.mutable_element(dest_index) = source_buffer;
+  source_buffer->Ref();
+  dest_buffer->Unref();
+  return Status::OK();
+}
+
 xla::ShapeTree<xla::MaybeOwningDeviceMemory>
-XRTTupleAllocation::ToDeviceMemoryTree(bool release) {
+XRTTupleAllocation::ToDeviceMemoryTree(
+    const std::function<bool(const xla::ShapeIndex&)>& release_checker) {
   xla::ShapeTree<xla::MaybeOwningDeviceMemory> shaped_tree(on_device_shape());
   for (const auto& buffer : buffers_) {
-    if (!release) {
+    if (!release_checker(buffer.first)) {
       *shaped_tree.mutable_element(buffer.first) = buffer.second->allocation();
     } else {
       *shaped_tree.mutable_element(buffer.first) = xla::OwningDeviceMemory(
diff --git a/tensorflow/compiler/xrt/xrt_state.h b/tensorflow/compiler/xrt/xrt_state.h
index 4aac37737e..1b32b6a20f 100644
--- a/tensorflow/compiler/xrt/xrt_state.h
+++ b/tensorflow/compiler/xrt/xrt_state.h
@@ -18,6 +18,7 @@ limitations under the License.
 #ifndef TENSORFLOW_COMPILER_XRT_XRT_STATE_H_
 #define TENSORFLOW_COMPILER_XRT_XRT_STATE_H_
 
+#include <functional>
 #include <memory>
 #include <string>
 #include <vector>
@@ -58,7 +59,14 @@ class XRTBufferAllocation : public core::RefCounted {
   // freed when the reference count drops to zero.
   void DiscardAllocation();
 
+  // Returns the expected size of the allocation. Since DiscardAllocation() will
+  // set allocation_ to {null,0}, and since later we might want to replace the
+  // discarded buffer with a new one, we need to be able to verify the size
+  // compatibility.
+  uint64 size() const { return size_; }
+
  private:
+  uint64 size_ = 0;
   se::DeviceMemoryBase allocation_;
   int device_ordinal_;
   xla::DeviceMemoryAllocator* allocator_;
@@ -168,9 +176,18 @@ class XRTTupleAllocation : public ResourceBase {
   // the same shape as on_host_shape.
   xla::ShapedBuffer ToShapedBuffer();
 
-  // Returns the device memory tree of this allocation. If 'release' is set, the
-  // ownership of the device memory is transferred to the result.
-  xla::ShapeTree<xla::MaybeOwningDeviceMemory> ToDeviceMemoryTree(bool release);
+  // Aliases the source buffer at source_index into the current tuple allocation
+  // dest_index.
+  Status AliasBufferFrom(const XRTTupleAllocation& source,
+                         const xla::ShapeIndex& source_index,
+                         const xla::ShapeIndex& dest_index);
+
+  // Returns the device memory tree of this allocation. If the release_checker
+  // function returns true for a given index, the ownership of the device memory
+  // at that index is transferred to the result. Every attempt to read the value
+  // at that index will fail.
+  xla::ShapeTree<xla::MaybeOwningDeviceMemory> ToDeviceMemoryTree(
+      const std::function<bool(const xla::ShapeIndex&)>& release_checker);
 
   string DebugString() const override { return "XLA allocation handle"; }
 
-- 
GitLab


From bdd76e2f04b17512d5c64a294975e7feb1231fab Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 18:22:45 -0800
Subject: [PATCH 0534/2345] Update ops-related pbtxt files.

PiperOrigin-RevId: 228813872
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 132 ++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 |  69 +++++++++
 2 files changed, 201 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 9698673dfe..15a4a10189 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -12758,6 +12758,77 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "CollectiveReduce"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "data"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_HALF
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "group_size"
+    type: "int"
+  }
+  attr {
+    name: "group_key"
+    type: "int"
+  }
+  attr {
+    name: "instance_key"
+    type: "int"
+  }
+  attr {
+    name: "merge_op"
+    type: "string"
+    allowed_values {
+      list {
+        s: "Min"
+        s: "Max"
+        s: "Mul"
+        s: "Add"
+      }
+    }
+  }
+  attr {
+    name: "final_op"
+    type: "string"
+    allowed_values {
+      list {
+        s: "Id"
+        s: "Div"
+      }
+    }
+  }
+  attr {
+    name: "subdiv_offsets"
+    type: "list(int)"
+  }
+  attr {
+    name: "wait_for"
+    type: "list(int)"
+    default_value {
+      list {
+      }
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "CompareAndBitpack"
   input_arg {
@@ -30125,6 +30196,44 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "KMC2ChainInitialization"
+  input_arg {
+    name: "distances"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "seed"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "index"
+    type: DT_INT64
+  }
+}
+op {
+  name: "KmeansPlusPlusInitialization"
+  input_arg {
+    name: "points"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "num_to_sample"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "seed"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "num_retries_per_sample"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "samples"
+    type: DT_FLOAT
+  }
+}
 op {
   name: "L2Loss"
   input_arg {
@@ -38646,6 +38755,29 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "NearestNeighbors"
+  input_arg {
+    name: "points"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "centers"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "k"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "nearest_center_indices"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "nearest_center_distances"
+    type: DT_FLOAT
+  }
+}
 op {
   name: "Neg"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 4a1b3a6f2d..9949c00aa4 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -5417,6 +5417,14 @@ op {
     name: "subdiv_offsets"
     type: "list(int)"
   }
+  attr {
+    name: "wait_for"
+    type: "list(int)"
+    default_value {
+      list {
+      }
+    }
+  }
   is_stateful: true
 }
 op {
@@ -15070,6 +15078,44 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "KMC2ChainInitialization"
+  input_arg {
+    name: "distances"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "seed"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "index"
+    type: DT_INT64
+  }
+}
+op {
+  name: "KmeansPlusPlusInitialization"
+  input_arg {
+    name: "points"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "num_to_sample"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "seed"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "num_retries_per_sample"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "samples"
+    type: DT_FLOAT
+  }
+}
 op {
   name: "L2Loss"
   input_arg {
@@ -18960,6 +19006,29 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "NearestNeighbors"
+  input_arg {
+    name: "points"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "centers"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "k"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "nearest_center_indices"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "nearest_center_distances"
+    type: DT_FLOAT
+  }
+}
 op {
   name: "Neg"
   input_arg {
-- 
GitLab


From a4b7a2575485aa4918597d1974b299b886f4a1cd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 18:47:26 -0800
Subject: [PATCH 0535/2345] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 228816108

---
 tensorflow/go/op/wrappers.go | 103 ++++++++++++++++++++++++++++++++++-
 1 file changed, 102 insertions(+), 1 deletion(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 6f6fb793a0..f8a5b5e7e1 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -4526,12 +4526,26 @@ func CollectiveBcastSend(scope *Scope, input tf.Output, group_size int64, group_
 	return op.Output(0)
 }
 
+// CollectiveReduceAttr is an optional argument to CollectiveReduce.
+type CollectiveReduceAttr func(optionalAttr)
+
+// CollectiveReduceWaitFor sets the optional wait_for attribute to value.
+// If not specified, defaults to <>
+func CollectiveReduceWaitFor(value []int64) CollectiveReduceAttr {
+	return func(m optionalAttr) {
+		m["wait_for"] = value
+	}
+}
+
 // Mutually reduces multiple tensors of identical type and shape.
-func CollectiveReduce(scope *Scope, input tf.Output, group_size int64, group_key int64, instance_key int64, merge_op string, final_op string, subdiv_offsets []int64) (data tf.Output) {
+func CollectiveReduce(scope *Scope, input tf.Output, group_size int64, group_key int64, instance_key int64, merge_op string, final_op string, subdiv_offsets []int64, optional ...CollectiveReduceAttr) (data tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	attrs := map[string]interface{}{"group_size": group_size, "group_key": group_key, "instance_key": instance_key, "merge_op": merge_op, "final_op": final_op, "subdiv_offsets": subdiv_offsets}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
 		Type: "CollectiveReduce",
 		Input: []tf.Input{
@@ -6846,6 +6860,35 @@ func Div(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	return op.Output(0)
 }
 
+// Selects the k nearest centers for each point.
+//
+// Rows of points are assumed to be input points. Rows of centers are assumed to be
+// the list of candidate centers. For each point, the k centers that have least L2
+// distance to it are computed.
+//
+// Arguments:
+//	points: Matrix of shape (n, d). Rows are assumed to be input points.
+//	centers: Matrix of shape (m, d). Rows are assumed to be centers.
+//	k: Number of nearest centers to return for each point. If k is larger than m, then
+// only m centers are returned.
+//
+// Returns Matrix of shape (n, min(m, k)). Each row contains the indices of the centers
+// closest to the corresponding point, ordered by increasing distance.Matrix of shape (n, min(m, k)). Each row contains the squared L2 distance to the
+// corresponding center in nearest_center_indices.
+func NearestNeighbors(scope *Scope, points tf.Output, centers tf.Output, k tf.Output) (nearest_center_indices tf.Output, nearest_center_distances tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "NearestNeighbors",
+		Input: []tf.Input{
+			points, centers, k,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
 // Returns x * y element-wise.
 //
 // *NOTE*: `Multiply` supports broadcasting. More about broadcasting
@@ -7649,6 +7692,33 @@ func TensorSliceDataset(scope *Scope, components []tf.Output, output_shapes []tf
 	return op.Output(0)
 }
 
+// Returns the index of a data point that should be added to the seed set.
+//
+// Entries in distances are assumed to be squared distances of candidate points to
+// the already sampled centers in the seed set. The op constructs one Markov chain
+// of the k-MC^2 algorithm and returns the index of one candidate point to be added
+// as an additional cluster center.
+//
+// Arguments:
+//	distances: Vector with squared distances to the closest previously sampled cluster center
+// for each candidate point.
+//	seed: Scalar. Seed for initializing the random number generator.
+//
+// Returns Scalar with the index of the sampled point.
+func KMC2ChainInitialization(scope *Scope, distances tf.Output, seed tf.Output) (index tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "KMC2ChainInitialization",
+		Input: []tf.Input{
+			distances, seed,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Computes hyperbolic sine of x element-wise.
 func Sinh(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
@@ -10408,6 +10478,37 @@ func BiasAddV1(scope *Scope, value tf.Output, bias tf.Output) (output tf.Output)
 	return op.Output(0)
 }
 
+// Selects num_to_sample rows of input using the KMeans++ criterion.
+//
+// Rows of points are assumed to be input points. One row is selected at random.
+// Subsequent rows are sampled with probability proportional to the squared L2
+// distance from the nearest row selected thus far till num_to_sample rows have
+// been sampled.
+//
+// Arguments:
+//	points: Matrix of shape (n, d). Rows are assumed to be input points.
+//	num_to_sample: Scalar. The number of rows to sample. This value must not be larger than n.
+//	seed: Scalar. Seed for initializing the random number generator.
+//	num_retries_per_sample: Scalar. For each row that is sampled, this parameter
+// specifies the number of additional points to draw from the current
+// distribution before selecting the best. If a negative value is specified, a
+// heuristic is used to sample O(log(num_to_sample)) additional points.
+//
+// Returns Matrix of shape (num_to_sample, d). The sampled rows.
+func KmeansPlusPlusInitialization(scope *Scope, points tf.Output, num_to_sample tf.Output, seed tf.Output, num_retries_per_sample tf.Output) (samples tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "KmeansPlusPlusInitialization",
+		Input: []tf.Input{
+			points, num_to_sample, seed, num_retries_per_sample,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Shuffle dimensions of x according to a permutation.
 //
 // The output `y` has the same rank as `x`. The shapes of `x` and `y` satisfy:
-- 
GitLab


From aeddcb55fadc0ad9085d71d736ba49c6b69620bf Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Thu, 10 Jan 2019 18:53:55 -0800
Subject: [PATCH 0536/2345] Make `steps_per_epoch`, `steps`, `validation_steps`
 optional for non-infinite datasets.

(Note: if your dataset is infinite but that fact is unknowable (cardinality=UNKNOWN), and you forget the `steps` argument,
your model will train forever, and ever, and ever)

PiperOrigin-RevId: 228816626
---
 tensorflow/python/keras/callbacks.py          |  15 +-
 .../engine/distributed_training_utils.py      |  11 +-
 tensorflow/python/keras/engine/training.py    |  44 +--
 .../python/keras/engine/training_arrays.py    | 256 ++++++++++--------
 .../keras/engine/training_dataset_test.py     |  44 ++-
 .../keras/engine/training_distributed.py      |  48 ++--
 .../keras/engine/training_eager_test.py       |   3 +-
 .../python/keras/engine/training_generator.py |  70 ++++-
 .../python/keras/engine/training_utils.py     |  63 ++++-
 9 files changed, 368 insertions(+), 186 deletions(-)

diff --git a/tensorflow/python/keras/callbacks.py b/tensorflow/python/keras/callbacks.py
index b9074669eb..e8985c2306 100644
--- a/tensorflow/python/keras/callbacks.py
+++ b/tensorflow/python/keras/callbacks.py
@@ -688,15 +688,14 @@ class ProgbarLogger(Callback):
     if self.verbose:
       if self.epochs > 1:
         print('Epoch %d/%d' % (epoch + 1, self.epochs))
-      self.progbar = Progbar(
-          target=self.target,
-          verbose=self.verbose,
-          stateful_metrics=self.stateful_metrics,
-          unit_name='step' if self.use_steps else 'sample')
+    self.progbar = Progbar(
+        target=self.target,
+        verbose=self.verbose,
+        stateful_metrics=self.stateful_metrics,
+        unit_name='step' if self.use_steps else 'sample')
 
   def on_batch_begin(self, batch, logs=None):
-    if self.seen < self.target:
-      self.log_values = []
+    self.log_values = []
 
   def on_batch_end(self, batch, logs=None):
     logs = logs or {}
@@ -715,7 +714,7 @@ class ProgbarLogger(Callback):
 
     # Skip progbar update for the last batch;
     # will be handled by on_epoch_end.
-    if self.verbose and self.seen < self.target:
+    if self.verbose and (self.target is None or self.seen < self.target):
       self.progbar.update(self.seen, self.log_values)
 
   def on_epoch_end(self, epoch, logs=None):
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index b6ef33f700..0185e87cb0 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -525,6 +525,15 @@ def list_to_tuple(maybe_list):
   return maybe_list
 
 
+def get_iterator(dataset, distribution_strategy):
+  with distribution_strategy.scope():
+    iterator = distribution_strategy.make_dataset_iterator(dataset)
+    init_op = iterator.initialize()
+    if not context.executing_eagerly():
+      K.get_session().run(init_op)
+  return iterator
+
+
 def _get_input_from_iterator(iterator, model):
   """Get elements from the iterator and verify the input shape and type."""
   next_element = iterator.get_next()
@@ -541,8 +550,6 @@ def _get_input_from_iterator(iterator, model):
     x, y, sample_weights = next_element
 
   # Validate that all the elements in x and y are of the same type and shape.
-  # We can then pass the first element of x and y to `_standardize_weights`
-  # below and be confident of the output.
   validate_distributed_dataset_inputs(
       model._distribution_strategy, x, y, sample_weights)
   return x, y, sample_weights
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 5614c93842..3e6c13232c 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -896,7 +896,8 @@ class Model(Network):
           validation_steps=validation_steps,
           workers=0,
           shuffle=shuffle,
-          initial_epoch=initial_epoch)
+          initial_epoch=initial_epoch,
+          steps_name='steps_per_epoch')
     else:
       return training_arrays.fit_loop(
           self,
@@ -913,7 +914,8 @@ class Model(Network):
           shuffle=shuffle,
           initial_epoch=initial_epoch,
           steps_per_epoch=steps_per_epoch,
-          validation_steps=validation_steps)
+          validation_steps=validation_steps,
+          steps_name='steps_per_epoch')
 
   def evaluate(self,
                x=None,
@@ -1500,7 +1502,8 @@ class Model(Network):
         workers=workers,
         use_multiprocessing=use_multiprocessing,
         shuffle=shuffle,
-        initial_epoch=initial_epoch)
+        initial_epoch=initial_epoch,
+        steps_name='steps_per_epoch')
 
   def evaluate_generator(self,
                          generator,
@@ -2136,7 +2139,7 @@ class Model(Network):
       shuffle: Boolean whether to shuffle the training data before each epoch.
 
     Returns:
-      Iterator for reading the dataset `x`.
+      Dataset instance.
 
     Raises:
       ValueError: In case of invalid user-provided data.
@@ -2167,7 +2170,8 @@ class Model(Network):
     else:
       session = K.get_session()
 
-    with self._distribution_strategy.scope():
+    strategy = self._distribution_strategy
+    with strategy.scope():
       first_x_value = nest.flatten(x)[0]
       if isinstance(first_x_value, np.ndarray):
         x = distributed_training_utils.list_to_tuple(x)
@@ -2190,20 +2194,20 @@ class Model(Network):
           shuffle_buffer = 1024
         else:
           shuffle_buffer = None
-        iterator = self._distribution_strategy.experimental_make_numpy_iterator(
-            in_tuple, batch_size, num_epochs=None, shuffle=shuffle_buffer,
-            session=session)
+        ds = strategy.extended.experimental_make_numpy_dataset(in_tuple,
+                                                               session=session)
+        if shuffle_buffer:
+          ds = ds.shuffle(shuffle_buffer)
+        ds = ds.repeat()
+        # We need to use the drop_remainder argument to get a known static
+        # input shape which is required for TPUs.
+        drop_remainder = strategy.extended.experimental_require_static_shapes
+        x = ds.batch(batch_size, drop_remainder=drop_remainder)
       else:
         assert isinstance(x, dataset_ops.DatasetV2)
         training_utils.validate_dataset_input(x, y, sample_weight,
                                               validation_split)
-        iterator = self._distribution_strategy.make_dataset_iterator(x)
-
-      init_op = iterator.initialize()
-      if not context.executing_eagerly():
-        K.get_session().run(init_op)
-
-    return iterator
+    return x
 
   def _standardize_user_data(self,
                              x,
@@ -2276,7 +2280,7 @@ class Model(Network):
       ValueError: In case of invalid user-provided data.
       RuntimeError: If the model was never compiled.
     """
-    if isinstance(x, (dataset_ops.DatasetV2, dataset_ops.DatasetV1)):
+    if isinstance(x, (dataset_ops.DatasetV1, dataset_ops.DatasetV2)):
       # Graph mode dataset. We'll pass the dataset as-is (unless
       # `extract_tensors_from_dataset` is True, in which case we extract
       # the tensors from the dataset and we output them.
@@ -2313,7 +2317,7 @@ class Model(Network):
       # If input data is a dataset iterator in graph mode or if it is an eager
       # iterator and only one batch of samples is required, we fetch the data
       # tensors from the iterator and then standardize them.
-      if isinstance(x, (dataset_ops.DatasetV2, dataset_ops.DatasetV1)):
+      if isinstance(x, (dataset_ops.DatasetV1, dataset_ops.DatasetV2)):
         x_input, y_input, _ = training_utils.extract_tensors_from_dataset(x)
       else:
         x_input = x
@@ -2433,7 +2437,7 @@ class Model(Network):
       feed_input_shapes = self._feed_input_shapes
 
     # Standardize the inputs.
-    if not isinstance(x, (dataset_ops.DatasetV2, dataset_ops.DatasetV1)):
+    if not isinstance(x, (dataset_ops.DatasetV1, dataset_ops.DatasetV2)):
       # TODO(fchollet): run static checks with dataset output shape(s).
       x = training_utils.standardize_input_data(
           x,
@@ -2514,8 +2518,8 @@ class Model(Network):
                          str(x[0].shape[0]) + ' samples')
 
     # If dictionary inputs were provided, we return a dictionary as well.
-    if dict_inputs and not isinstance(x, (dataset_ops.DatasetV2,
-                                          dataset_ops.DatasetV1)):
+    if dict_inputs and not isinstance(x, (dataset_ops.DatasetV1,
+                                          dataset_ops.DatasetV2)):
       x = dict(zip(feed_input_names, x))
     return x, y, sample_weights
 
diff --git a/tensorflow/python/keras/engine/training_arrays.py b/tensorflow/python/keras/engine/training_arrays.py
index ab7d455bfa..a356f8b328 100644
--- a/tensorflow/python/keras/engine/training_arrays.py
+++ b/tensorflow/python/keras/engine/training_arrays.py
@@ -41,90 +41,6 @@ except ImportError:
   issparse = None
 
 
-def _get_model_feed(model, mode):
-  if mode == ModeKeys.PREDICT:
-    feed = model._feed_inputs
-  else:
-    feed = (
-        model._feed_inputs + model._feed_targets + model._feed_sample_weights)
-  return feed
-
-
-def _validate_arguments(steps_per_epoch, validation_steps, kwargs):
-  for k in kwargs:
-    if k != 'steps':
-      raise ValueError('Invalid argument passed: {}'.format(k))
-
-  # Validate inputs when in training mode.
-  if validation_steps and steps_per_epoch is None:
-    raise ValueError('Can only use `validation_steps` '
-                     'when doing step-wise '
-                     'training, i.e. `steps_per_epoch` '
-                     'must be set.')
-
-
-def _print_train_info(inputs, val_inputs, steps_per_epoch, verbose):
-  if (val_inputs and steps_per_epoch is None and verbose and inputs and
-      hasattr(inputs[0], 'shape') and hasattr(val_inputs[0], 'shape')):
-    print('Train on %d samples, validate on %d samples' %
-          (inputs[0].shape[0], val_inputs[0].shape[0]))
-
-
-def _get_num_samples_or_steps(ins, batch_size, steps_per_epoch):
-  """Returns total number of samples (when training in batch mode) or steps."""
-  if steps_per_epoch:
-    return steps_per_epoch
-  return training_utils.check_num_samples(ins, batch_size, steps_per_epoch,
-                                          'steps_per_epoch')
-
-
-def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
-  """Prepare feed values to the model execution function.
-
-  Arguments:
-    model: Model to prepare feed values for.
-    inputs: List or dict of model inputs.
-    targets: Optional list of model targets.
-    sample_weights: Optional list of sample weight arrays.
-    mode: One of ModeKeys.TRAIN/ModeKeys.TEST/ModeKeys.PREDICT.
-
-  Returns:
-    Feed values for the model in the given mode.
-  """
-  if model._distribution_strategy:
-    def get_distributed_inputs():
-      return distributed_training_utils._prepare_feed_values(
-          model, inputs, targets, sample_weights, mode)
-
-    # In the eager case, we want to call the input method per step, so return
-    # a lambda from here that can be called. Note that this is applicable only
-    # in Distribution Strategy case as it follows the same code path for both
-    # eager and graph modes.
-    # TODO(priyag,omalleyt): Either we should move the training DS with
-    # EagerIterator to use training_generator code path, or figure out how to
-    # set a symbolic Iterator out of a Dataset when in eager mode.
-    if context.executing_eagerly():
-      return get_distributed_inputs
-    else:
-      return get_distributed_inputs()
-
-  inputs = training_utils.ModelInputs(inputs).as_list()
-  targets = targets or []
-  sample_weights = sample_weights or []
-  ins = inputs + targets + sample_weights
-  if mode == ModeKeys.TRAIN and not isinstance(K.symbolic_learning_phase(),
-                                               int):
-    ins += [True]
-  return ins
-
-
-def _make_execution_function(model, mode):
-  """Makes function to run one step of model execution."""
-  if model._distribution_strategy:
-    return distributed_training_utils._make_execution_function(model, mode)
-  return model._make_execution_function(mode)
-
-
 def model_iteration(model,
                     inputs,
                     targets=None,
@@ -142,6 +58,7 @@ def model_iteration(model,
                     validation_steps=None,
                     mode=ModeKeys.TRAIN,
                     validation_in_fit=False,
+                    steps_name='steps',
                     **kwargs):
   """Loop function for arrays of data with modes TRAIN/TEST/PREDICT.
 
@@ -173,6 +90,9 @@ def model_iteration(model,
         weights when using a tf.distribute.Strategy. The input is deprecated as
         it is not required if the user creates a distributed model under the
         distribution strategy scope rather than passing it to compile.
+      steps_name: The string name of the steps argument, either `steps`,
+        `validation_steps`, or `steps_per_epoch`. Only used for error message
+        formatting.
       **kwargs: Additional arguments for backwards compatibility.
 
   Returns:
@@ -185,23 +105,25 @@ def model_iteration(model,
   """
   # Backwards compatibility.
   if 'steps' in kwargs:
-    steps_per_epoch = kwargs['steps']
+    steps_per_epoch = kwargs.pop('steps')
+  if kwargs:
+    raise TypeError('Unknown arguments: %s' % (kwargs,))
+
+  # In case we were passed a dataset, we extract symbolic tensors from it.
+  reset_dataset_after_each_epoch = False
+  original_dataset = None
+  is_dataset = isinstance(inputs,
+                          (dataset_ops.DatasetV1, dataset_ops.DatasetV2))
+  # TODO(fchollet): consider moving `steps_per_epoch` inference to
+  # _standardize_user_data and set reset_dataset_after_each_epoch as an
+  # attribute on the dataset instance.
+  if is_dataset:
+    original_dataset = inputs
+    if steps_per_epoch is None:
+      reset_dataset_after_each_epoch = True
+      steps_per_epoch = training_utils.infer_steps_for_dataset(
+          inputs, steps_per_epoch, epochs=epochs, steps_name=steps_name)
 
-  # In case we are passed datasets, we extract symbolic tensors from them.
-  if isinstance(inputs, (dataset_ops.DatasetV2, dataset_ops.DatasetV1)):
-    inputs, targets, sample_weights = model._standardize_user_data(
-        inputs,
-        steps_name='steps_per_epoch',
-        steps=steps_per_epoch,
-        extract_tensors_from_dataset=True)
-  if isinstance(val_inputs, (dataset_ops.DatasetV2, dataset_ops.DatasetV1)):
-    val_inputs, val_targets, val_sample_weights = model._standardize_user_data(
-        val_inputs,
-        steps_name='validation_steps',
-        steps=validation_steps,
-        extract_tensors_from_dataset=True)
-
-  _validate_arguments(steps_per_epoch, validation_steps, kwargs)
   if mode == ModeKeys.TRAIN:
     _print_train_info(inputs, val_inputs, steps_per_epoch, verbose)
 
@@ -212,7 +134,7 @@ def model_iteration(model,
 
   # Get step function and loop type.
   f = _make_execution_function(model, mode)
-  use_steps = steps_per_epoch is not None
+  use_steps = is_dataset or steps_per_epoch is not None
   do_validation = val_inputs is not None
 
   # Convert Eager Tensors to NumPy arrays to support batching/shuffling.
@@ -221,8 +143,11 @@ def model_iteration(model,
 
   # Prepare input data.
   ins = _prepare_feed_values(model, inputs, targets, sample_weights, mode)
-  num_samples_or_steps = _get_num_samples_or_steps(ins, batch_size,
-                                                   steps_per_epoch)
+  if not is_dataset:
+    num_samples_or_steps = _get_num_samples_or_steps(ins, batch_size,
+                                                     steps_per_epoch)
+  else:
+    num_samples_or_steps = steps_per_epoch
 
   # Configure callbacks.
   count_mode = 'steps' if use_steps else 'samples'
@@ -278,7 +203,15 @@ def model_iteration(model,
 
     if use_steps:
       # Step-wise loop.
-      for step in range(steps_per_epoch):
+      if steps_per_epoch is None:
+        # Loop over dataset until `OutOfRangeError` is raised.
+        target_steps = np.inf
+      else:
+        # Loop over dataset for the specified number of steps.
+        target_steps = steps_per_epoch
+
+      step = 0
+      while step < target_steps:
         batch_logs = {'batch': step, 'size': 1}
         callbacks._call_batch_hook(mode, 'begin', step, batch_logs)
         progbar.on_batch_begin(step, batch_logs)
@@ -289,13 +222,26 @@ def model_iteration(model,
           actual_inputs = ins() if callable(ins) else ins
           batch_outs = f(actual_inputs)
         except errors.OutOfRangeError:
-          logging.warning('Your dataset iterator ran out of data; '
-                          'interrupting training. Make sure that your dataset '
-                          'can generate at least `steps_per_epoch * epochs` '
-                          'batches (in this case, %d batches). You may need to'
-                          'use the repeat() function when building your '
-                          'dataset.' % steps_per_epoch * epochs)
+          if original_dataset is None:
+            # We ran out of batches while the user passed an iterator (legacy).
+            logging.warning(
+                'Your dataset iterator ran out of data; '
+                'interrupting training. Make sure that your iterator '
+                'can generate at least `%s * epochs` '
+                'batches (in this case, %d batches). You may need to'
+                'use the repeat() function when building your '
+                'dataset.' % (steps_name, steps_per_epoch * epochs))
+            callbacks.model.stop_training = True
+          else:
+            # The dataset passed by the user ran out of batches.
+            # Now we know the cardinality of the dataset.
+            if step > 0:
+              steps_per_epoch = step
+              aggregator.num_samples_or_steps = steps_per_epoch
+              progbar.params['steps'] = steps_per_epoch
+              progbar.progbar.target = steps_per_epoch
           break
+
         if not isinstance(batch_outs, list):
           batch_outs = [batch_outs]
 
@@ -312,6 +258,7 @@ def model_iteration(model,
         batch_logs = cbks.make_logs(model, batch_logs, batch_outs, mode)
         callbacks._call_batch_hook(mode, 'end', step, batch_logs)
         progbar.on_batch_end(step, batch_logs)
+        step += 1
 
         if callbacks.model.stop_training:
           break
@@ -385,7 +332,8 @@ def model_iteration(model,
           callbacks=callbacks,
           verbose=0,
           mode=ModeKeys.TEST,
-          validation_in_fit=True)
+          validation_in_fit=True,
+          steps_name='validation_steps')
       if not isinstance(val_results, list):
         val_results = [val_results]
       epoch_logs = cbks.make_logs(
@@ -394,7 +342,11 @@ def model_iteration(model,
     if mode == ModeKeys.TRAIN:
       # Epochs only apply to `fit`.
       callbacks.on_epoch_end(epoch, epoch_logs)
-      progbar.on_epoch_end(epoch, epoch_logs)
+    progbar.on_epoch_end(epoch, epoch_logs)
+
+    # Recreate dataset iterator for the next epoch.
+    if reset_dataset_after_each_epoch and epoch < epochs - 1:
+      ins = _prepare_feed_values(model, original_dataset, None, None, mode)
 
   callbacks._call_end_hook(mode)
 
@@ -410,6 +362,86 @@ def model_iteration(model,
   return results
 
 
+def _get_model_feed(model, mode):
+  if mode == ModeKeys.PREDICT:
+    feed = model._feed_inputs
+  else:
+    feed = (
+        model._feed_inputs + model._feed_targets + model._feed_sample_weights)
+  return feed
+
+
+def _print_train_info(inputs, val_inputs, steps_per_epoch, verbose):
+  if (val_inputs and steps_per_epoch is None and verbose and inputs and
+      hasattr(inputs[0], 'shape') and hasattr(val_inputs[0], 'shape')):
+    print('Train on %d samples, validate on %d samples' %
+          (inputs[0].shape[0], val_inputs[0].shape[0]))
+
+
+def _get_num_samples_or_steps(ins, batch_size, steps_per_epoch):
+  """Returns total number of samples (when training in batch mode) or steps."""
+  if steps_per_epoch:
+    return steps_per_epoch
+  return training_utils.check_num_samples(ins, batch_size, steps_per_epoch,
+                                          'steps_per_epoch')
+
+
+def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
+  """Prepare feed values to the model execution function.
+
+  Arguments:
+    model: Model to prepare feed values for.
+    inputs: List or dict of model inputs.
+    targets: Optional list of model targets.
+    sample_weights: Optional list of sample weight arrays.
+    mode: One of ModeKeys.TRAIN/ModeKeys.TEST/ModeKeys.PREDICT.
+
+  Returns:
+    Feed values for the model in the given mode.
+  """
+  if model._distribution_strategy:
+    if isinstance(inputs, (dataset_ops.DatasetV1, dataset_ops.DatasetV2)):
+      inputs = distributed_training_utils.get_iterator(
+          inputs, model._distribution_strategy)
+
+    def get_distributed_inputs():
+      return distributed_training_utils._prepare_feed_values(
+          model, inputs, targets, sample_weights, mode)
+
+    # In the eager case, we want to call the input method per step, so return
+    # a lambda from here that can be called. Note that this is applicable only
+    # in Distribution Strategy case as it follows the same code path for both
+    # eager and graph modes.
+    # TODO(priyag,omalleyt): Either we should move the training DS with
+    # EagerIterator to use training_generator code path, or figure out how to
+    # set a symbolic Iterator out of a Dataset when in eager mode.
+    if context.executing_eagerly():
+      return get_distributed_inputs
+    else:
+      return get_distributed_inputs()
+
+  if isinstance(inputs, (dataset_ops.DatasetV1, dataset_ops.DatasetV2)):
+    inputs, targets, sample_weights = model._standardize_user_data(
+        inputs,
+        extract_tensors_from_dataset=True)
+
+  inputs = training_utils.ModelInputs(inputs).as_list()
+  targets = targets or []
+  sample_weights = sample_weights or []
+  ins = inputs + targets + sample_weights
+  if mode == ModeKeys.TRAIN and not isinstance(K.symbolic_learning_phase(),
+                                               int):
+    ins += [True]  # Add learning phase value.
+  return ins
+
+
+def _make_execution_function(model, mode):
+  """Makes function to run one step of model execution."""
+  if model._distribution_strategy:
+    return distributed_training_utils._make_execution_function(model, mode)
+  return model._make_execution_function(mode)
+
+
 # For backwards compatibility for internal users of these loops.
 fit_loop = functools.partial(model_iteration, mode=ModeKeys.TRAIN)
 test_loop = functools.partial(
diff --git a/tensorflow/python/keras/engine/training_dataset_test.py b/tensorflow/python/keras/engine/training_dataset_test.py
index 40d42c13f0..0ce01ec606 100644
--- a/tensorflow/python/keras/engine/training_dataset_test.py
+++ b/tensorflow/python/keras/engine/training_dataset_test.py
@@ -23,6 +23,7 @@ import logging
 import numpy as np
 
 from tensorflow.python import keras
+from tensorflow.python.data.experimental.ops import cardinality
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.eager import context
 from tensorflow.python.framework import test_util as tf_test_util
@@ -164,7 +165,7 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
     inputs = np.zeros((10, 3), np.float32)
     targets = np.zeros((10, 4), np.float32)
     dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
-    dataset = dataset.repeat(100)
+    dataset = dataset.repeat()  # Infinite dataset.
     dataset = dataset.batch(10)
 
     model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1)
@@ -215,6 +216,7 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
       model.fit(dataset, dataset,
                 epochs=1, steps_per_epoch=2, verbose=0)
 
+    # With an infinite dataset, `steps_per_epoch`/`steps` argument is required.
     with self.assertRaisesRegexp(
         ValueError, 'the `steps_per_epoch` argument'):
       model.fit(dataset, epochs=1, verbose=0)
@@ -322,6 +324,46 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
                                    r'expected (.*?) to have shape \(3,\)'):
         model.train_on_batch(dataset)
 
+  @keras_parameterized.run_with_all_model_types
+  @keras_parameterized.run_all_keras_modes
+  def test_finite_dataset_known_cardinality_no_steps_arg(self):
+    model = testing_utils.get_small_mlp(1, 4, input_dim=3)
+    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    model.compile(optimizer, 'mse',
+                  run_eagerly=testing_utils.should_run_eagerly())
+
+    inputs = np.zeros((100, 3), dtype=np.float32)
+    targets = np.random.randint(0, 4, size=100, dtype=np.int32)
+    dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
+    dataset = dataset.batch(10)
+
+    history = model.fit(dataset, epochs=2, verbose=1)
+    self.assertEqual(len(history.history['loss']), 2)
+    model.evaluate(dataset)
+    out = model.predict(dataset)
+    self.assertEqual(out.shape[0], 100)
+
+  @keras_parameterized.run_with_all_model_types
+  @keras_parameterized.run_all_keras_modes
+  def test_finite_dataset_unknown_cardinality_no_steps_arg(self):
+    model = testing_utils.get_small_mlp(1, 4, input_dim=3)
+    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    model.compile(optimizer, 'mse',
+                  run_eagerly=testing_utils.should_run_eagerly())
+
+    inputs = np.zeros((100, 3), dtype=np.float32)
+    targets = np.random.randint(0, 4, size=100, dtype=np.int32)
+    dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
+    dataset = dataset.filter(lambda x, y: True).batch(10)
+    self.assertEqual(keras.backend.get_value(cardinality.cardinality(dataset)),
+                     cardinality.UNKNOWN)
+
+    history = model.fit(dataset, epochs=2, verbose=1)
+    self.assertEqual(len(history.history['loss']), 2)
+    model.evaluate(dataset)
+    out = model.predict(dataset)
+    self.assertEqual(out.shape[0], 100)
+
 
 class TestMetricsWithDatasetIterators(keras_parameterized.TestCase):
 
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 0bd79f2b47..58a8637a84 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -65,7 +65,7 @@ def fit_distributed(model,
             batch_size, is_training=True))
   batch_size = model._validate_or_infer_batch_size(
       batch_size, steps_per_epoch, x)
-  iterator = model._distribution_standardize_user_data(
+  dataset = model._distribution_standardize_user_data(
       x, y,
       sample_weight=sample_weight,
       class_weight=class_weight,
@@ -76,7 +76,7 @@ def fit_distributed(model,
       validation_split=validation_split,
       shuffle=shuffle)
 
-  val_iterator = None
+  val_dataset = None
   if validation_data:
     val_x, val_y, val_sample_weights = model._unpack_validation_data(
         validation_data)
@@ -87,7 +87,7 @@ def fit_distributed(model,
       validation_steps, _ = distributed_training_utils.get_input_params(
           model._distribution_strategy, first_valx_value, validation_steps,
           batch_size)
-    val_iterator = model._distribution_standardize_user_data(
+    val_dataset = model._distribution_standardize_user_data(
         val_x, val_y,
         sample_weight=val_sample_weights,
         class_weight=None,
@@ -104,23 +104,23 @@ def fit_distributed(model,
   if distributed_training_utils.is_tpu_strategy(model._distribution_strategy):
     return experimental_tpu_fit_loop(
         model,
-        iterator,
+        dataset,
         epochs=epochs,
         verbose=verbose,
         callbacks=callbacks,
-        val_iterator=val_iterator,
+        val_dataset=val_dataset,
         initial_epoch=initial_epoch,
         steps_per_epoch=steps_per_epoch,
         validation_steps=validation_steps)
   else:
     return training_arrays.fit_loop(
         model,
-        iterator,
+        dataset,
         batch_size=batch_size,
         epochs=epochs,
         verbose=verbose,
         callbacks=callbacks,
-        val_inputs=val_iterator,
+        val_inputs=val_dataset,
         shuffle=shuffle,
         initial_epoch=initial_epoch,
         steps_per_epoch=steps_per_epoch,
@@ -142,7 +142,7 @@ def evaluate_distributed(model,
     steps, batch_size = distributed_training_utils.get_input_params(
         model._distribution_strategy, first_x_value, steps, batch_size)
   batch_size = model._validate_or_infer_batch_size(batch_size, steps, x)
-  iterator = model._distribution_standardize_user_data(
+  dataset = model._distribution_standardize_user_data(
       x, y,
       sample_weight=sample_weight,
       batch_size=batch_size,
@@ -153,11 +153,11 @@ def evaluate_distributed(model,
   if distributed_training_utils.is_tpu_strategy(model._distribution_strategy):
     # TODO(fchollet): why aren't callbacks supported here?
     return experimental_tpu_test_loop(
-        model, iterator=iterator, verbose=verbose, steps=steps)
+        model, dataset, verbose=verbose, steps=steps)
   else:
     return training_arrays.test_loop(
         model,
-        inputs=iterator,
+        inputs=dataset,
         batch_size=batch_size,
         verbose=verbose,
         steps=steps,
@@ -178,7 +178,7 @@ def predict_distributed(model,
     steps, batch_size = distributed_training_utils.get_input_params(
         model._distribution_strategy, first_x_value, steps, batch_size)
   batch_size = model._validate_or_infer_batch_size(batch_size, steps, x)
-  iterator = model._distribution_standardize_user_data(
+  dataset = model._distribution_standardize_user_data(
       x,
       batch_size=batch_size,
       check_steps=True,
@@ -187,11 +187,11 @@ def predict_distributed(model,
   if distributed_training_utils.is_tpu_strategy(model._distribution_strategy):
     # TODO(fchollet): why aren't callbacks supported here?
     return experimental_tpu_predict_loop(
-        model, iterator, verbose=verbose, steps=steps)
+        model, dataset, verbose=verbose, steps=steps)
   else:
     return training_arrays.predict_loop(
         model,
-        iterator,
+        dataset,
         batch_size=batch_size,
         verbose=verbose,
         steps=steps,
@@ -199,19 +199,19 @@ def predict_distributed(model,
 
 
 def experimental_tpu_fit_loop(model,
-                              iterator,
+                              dataset,
                               epochs=100,
                               verbose=1,
                               callbacks=None,
                               initial_epoch=0,
                               steps_per_epoch=None,
-                              val_iterator=None,
+                              val_dataset=None,
                               validation_steps=None):
   """Fit loop for training with TPU DistributionStrategy.
 
   Arguments:
       model: Keras Model instance.
-      iterator: Iterator that returns inputs and targets
+      dataset: Dataset that returns inputs and targets
       epochs: Number of times to iterate over the data
       verbose: Integer, Verbosity mode, 0, 1 or 2
       callbacks: List of callbacks to be called during training
@@ -220,7 +220,7 @@ def experimental_tpu_fit_loop(model,
       steps_per_epoch: Total number of steps (batches of samples)
           before declaring one epoch finished and starting the
           next epoch. Ignored with the default value of `None`.
-      val_iterator: Iterator for validation data.
+      val_dataset: Dataset for validation data.
       validation_steps: Number of steps to run validation for
           (only if doing validation from data tensors).
           Ignored with the default value of `None`.
@@ -231,7 +231,9 @@ def experimental_tpu_fit_loop(model,
   Raises:
       ValueError: in case of invalid arguments.
   """
+  # TODO(fchollet): add support for `steps_per_epoch=None` in TPU loops.
   current_strategy = model._distribution_strategy
+  iterator = distributed_training_utils.get_iterator(dataset, current_strategy)
   scope = current_strategy.scope()
   scope.__enter__()
 
@@ -373,7 +375,7 @@ def experimental_tpu_fit_loop(model,
 
       val_outs = experimental_tpu_test_loop(  # pylint: disable=undefined-variable
           model,
-          val_iterator,
+          val_dataset,
           steps=validation_steps,
           verbose=verbose)
       if not isinstance(val_outs, list):
@@ -397,14 +399,14 @@ def experimental_tpu_fit_loop(model,
 
 
 def experimental_tpu_test_loop(model,
-                               iterator,
+                               dataset,
                                verbose=0,
                                steps=None):
   """Test loop for evaluating with TPU DistributionStrategy.
 
   Arguments:
       model: Keras Model instance.
-      iterator: Iterator for input data.
+      dataset: Dataset for input data.
       verbose: Integer, Verbosity mode 0 or 1.
       steps: Total number of steps (batches of samples)
           before declaring predictions finished.
@@ -417,6 +419,7 @@ def experimental_tpu_test_loop(model,
       the display labels for the outputs.
   """
   current_strategy = model._distribution_strategy
+  iterator = distributed_training_utils.get_iterator(dataset, current_strategy)
   scope = current_strategy.scope()
   scope.__enter__()
 
@@ -520,12 +523,12 @@ def experimental_tpu_test_loop(model,
   return outs
 
 
-def experimental_tpu_predict_loop(model, iterator, verbose=0, steps=None):
+def experimental_tpu_predict_loop(model, dataset, verbose=0, steps=None):
   """Predict loop for predicting with TPU DistributionStrategy.
 
   Arguments:
       model: Keras Model instance.
-      iterator: Iterator for input data.
+      dataset: Dataset for input data.
       verbose: Integer, Verbosity mode 0 or 1.
       steps: Total number of steps (batches of samples)
           before declaring `_predict_loop` finished.
@@ -537,6 +540,7 @@ def experimental_tpu_predict_loop(model, iterator, verbose=0, steps=None):
       (if the model has multiple outputs).
   """
   current_strategy = model._distribution_strategy
+  iterator = distributed_training_utils.get_iterator(dataset, current_strategy)
   scope = current_strategy.scope()
   scope.__enter__()
 
diff --git a/tensorflow/python/keras/engine/training_eager_test.py b/tensorflow/python/keras/engine/training_eager_test.py
index a6e2c24ec2..84f1fa0efc 100644
--- a/tensorflow/python/keras/engine/training_eager_test.py
+++ b/tensorflow/python/keras/engine/training_eager_test.py
@@ -156,7 +156,7 @@ class TrainingTest(keras_parameterized.TestCase):
     dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat(10).batch(5)
     iterator = dataset_ops.make_one_shot_iterator(dataset)
     validation_dataset = dataset_ops.Dataset.from_tensor_slices(
-        (x, y)).repeat(10).batch(5)
+        (x, y)).repeat().batch(5)  # Infinite dataset.
     validation_iterator = dataset_ops.make_one_shot_iterator(validation_dataset)
 
     with self.assertRaisesRegexp(
@@ -169,6 +169,7 @@ class TrainingTest(keras_parameterized.TestCase):
           ValueError, r'provide either `batch_size` or `validation_steps`'):
         model.fit(iterator, steps_per_epoch=2, epochs=1, verbose=0,
                   validation_data=(x, y))
+    # Step argument is required for infinite datasets.
     with self.assertRaisesRegexp(ValueError,
                                  'specify the `validation_steps` argument.'):
       model.fit(iterator, steps_per_epoch=2, epochs=1, verbose=0,
diff --git a/tensorflow/python/keras/engine/training_generator.py b/tensorflow/python/keras/engine/training_generator.py
index 07de934259..8a5e24856c 100644
--- a/tensorflow/python/keras/engine/training_generator.py
+++ b/tensorflow/python/keras/engine/training_generator.py
@@ -54,6 +54,7 @@ def model_iteration(model,
                     initial_epoch=0,
                     mode=ModeKeys.TRAIN,
                     batch_size=None,
+                    steps_name='steps',
                     **kwargs):
   """Loop function for arrays of data with modes TRAIN/TEST/PREDICT.
 
@@ -93,6 +94,9 @@ def model_iteration(model,
       mode: One of ModeKeys.TRAIN/ModeKeys.TEST/ModeKeys.PREDICT.
       batch_size: Integer batch size or None if unknown. Will only be used if
         `data` is in NumPy/Tensor format.
+      steps_name: The string name of the steps argument, either `steps`,
+        `validation_steps`, or `steps_per_epoch`. Only used for error message
+        formatting.
       **kwargs: Additional arguments for backwards compatibility. `steps` is
         accepted as an alias for `steps_per_epoch`.
 
@@ -107,6 +111,18 @@ def model_iteration(model,
   if 'steps' in kwargs:
     steps_per_epoch = kwargs['steps']
 
+  # Determine the number of steps per epoch and whether we should reset the
+  # dataset at the end of each epoch.
+  reset_dataset_after_each_epoch = False
+  original_dataset = None
+  is_dataset = isinstance(data, (dataset_ops.DatasetV2, dataset_ops.DatasetV1))
+  if is_dataset:
+    original_dataset = data
+    if steps_per_epoch is None:
+      reset_dataset_after_each_epoch = True
+      steps_per_epoch = training_utils.infer_steps_for_dataset(
+          data, steps_per_epoch, epochs=epochs, steps_name=steps_name)
+
   # Convert to a format that supports `next(generator)`.
   generator, steps_per_epoch = convert_to_generator_like(
       data,
@@ -118,7 +134,7 @@ def model_iteration(model,
   do_validation = validation_data is not None
   should_set_learning_phase = context.executing_eagerly() and model.run_eagerly
   is_sequence = isinstance(generator, data_utils.Sequence)
-  _validate_arguments(is_sequence, use_multiprocessing, workers,
+  _validate_arguments(is_sequence, is_dataset, use_multiprocessing, workers,
                       steps_per_epoch, validation_data, validation_steps, mode,
                       kwargs)
 
@@ -175,10 +191,36 @@ def model_iteration(model,
       callbacks.on_epoch_begin(epoch, epoch_logs)
     progbar.on_epoch_begin(epoch, epoch_logs)
 
-    for step in range(steps_per_epoch):
+    if steps_per_epoch is None:
+      # Loop over dataset until `OutOfRangeError` is raised.
+      target_steps = np.inf
+    else:
+      # Loop over dataset for the specified number of steps.
+      target_steps = steps_per_epoch
+
+    step = 0
+    while step < target_steps:
       batch_data = _get_next_batch(output_generator, mode)
       if batch_data is None:
-        callbacks.model.stop_training = True
+        if not is_dataset:
+          # We ran out of batches while the user passed an iterator (legacy).
+          logging.warning(
+              'Your dataset iterator ran out of data; '
+              'interrupting training. Make sure that your iterator '
+              'can generate at least `%s * epochs` '
+              'batches (in this case, %d batches). You may need to'
+              'use the repeat() function when building your '
+              'dataset.' % (steps_name, steps_per_epoch * epochs))
+          callbacks.model.stop_training = True
+        else:
+          # The dataset passed by the user ran out of batches.
+          # Now we know the cardinality of the dataset.
+          # assert steps_per_epoch is None
+          if step > 0:
+            steps_per_epoch = step
+            aggregator.num_samples_or_steps = steps_per_epoch
+            progbar.params['steps'] = steps_per_epoch
+            progbar.progbar.target = steps_per_epoch
         break
 
       # `batch_size` used for validation data if validation
@@ -203,6 +245,7 @@ def model_iteration(model,
       batch_logs = cbks.make_logs(model, batch_logs, batch_outs, mode)
       callbacks._call_batch_hook(mode, 'end', step, batch_logs)
       progbar.on_batch_end(step, batch_logs)
+      step += 1
 
       if callbacks.model.stop_training:
         break
@@ -226,7 +269,8 @@ def model_iteration(model,
           max_queue_size=max_queue_size,
           callbacks=callbacks,
           verbose=0,
-          mode=ModeKeys.TEST)
+          mode=ModeKeys.TEST,
+          steps_name='validation_steps')
 
       if not isinstance(val_results, list):
         val_results = [val_results]
@@ -236,7 +280,11 @@ def model_iteration(model,
     if mode == ModeKeys.TRAIN:
       # Epochs only apply to `fit`.
       callbacks.on_epoch_end(epoch, epoch_logs)
-      progbar.on_epoch_end(epoch, epoch_logs)
+    progbar.on_epoch_end(epoch, epoch_logs)
+
+    # Recreate dataset iterator for the next epoch.
+    if reset_dataset_after_each_epoch and epoch < epochs - 1:
+      generator = dataset_ops.make_one_shot_iterator(original_dataset)
 
   callbacks._call_end_hook(mode)
 
@@ -263,9 +311,7 @@ def _get_next_batch(output_generator, mode):
   """Retrieves the next batch of input data."""
   try:
     generator_output = next(output_generator)
-  except (errors.OutOfRangeError, StopIteration):
-    # Returning `None` will trigger looping to stop.
-    logging.warning('Your dataset iterator ran out of data.')
+  except (StopIteration, errors.OutOfRangeError):
     return None
   if not isinstance(generator_output, tuple):
     if mode == ModeKeys.PREDICT:
@@ -283,7 +329,7 @@ def _get_next_batch(output_generator, mode):
   return generator_output
 
 
-def _validate_arguments(is_sequence, use_multiprocessing, workers,
+def _validate_arguments(is_sequence, is_dataset, use_multiprocessing, workers,
                         steps_per_epoch, validation_data, validation_steps,
                         mode, kwargs):
   """Raises errors if arguments are invalid.
@@ -291,6 +337,7 @@ def _validate_arguments(is_sequence, use_multiprocessing, workers,
   Arguments:
     is_sequence: Boolean, whether data is a `keras.utils.data_utils.Sequence`
       instance.
+    is_dataset: Boolean, whether data is a dataset instance.
     use_multiprocessing: Boolean. If `True`, use process-based threading. If
       unspecified, `use_multiprocessing` will default to `False`. Note that
       because this implementation relies on multiprocessing, you should not pass
@@ -322,15 +369,14 @@ def _validate_arguments(is_sequence, use_multiprocessing, workers,
                     ' Please consider using the `keras.utils.Sequence`'
                     ' class.'))
 
-  if steps_per_epoch is None:
+  if steps_per_epoch is None and not is_dataset:
     arg_name = 'steps_per_epoch' if mode == ModeKeys.TRAIN else 'steps'
     raise ValueError('Please specify the number of steps via the '
                      '`{}` argument.'.format(arg_name))
 
   val_gen = (
       data_utils.is_generator_or_sequence(validation_data) or
-      isinstance(validation_data, iterator_ops.EagerIterator) or
-      isinstance(validation_data, dataset_ops.DatasetV2))
+      isinstance(validation_data, iterator_ops.EagerIterator))
   if (val_gen and not isinstance(validation_data, data_utils.Sequence) and
       not validation_steps):
     raise ValueError('Please specify the `validation_steps` argument.')
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index a4ee6c1ba5..0c3beb4681 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -25,6 +25,7 @@ import copy
 import numpy as np
 import six
 
+from tensorflow.python.data.experimental.ops import cardinality
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.data.ops import readers
@@ -106,6 +107,8 @@ class MetricsAggregator(Aggregator):
     self.results[1:] = batch_outs[1:]
 
   def finalize(self):
+    if not self.results:
+      raise ValueError('Empty training data.')
     self.results[0] /= self.num_samples_or_steps
 
 
@@ -952,13 +955,12 @@ def check_steps_argument(input_data, steps, steps_name):
         but not provided.
   """
   # TODO(fchollet): allow datasets with steps=None if cardinality is known.
-  is_x_dataset = isinstance(input_data, (iterator_ops.Iterator,
-                                         iterator_ops.EagerIterator,
-                                         dataset_ops.DatasetV2))
-  if (input_data is None or is_x_dataset or has_symbolic_tensors(input_data) or
+  is_x_iterator = isinstance(input_data, (iterator_ops.Iterator,
+                                          iterator_ops.EagerIterator))
+  if (input_data is None or is_x_iterator or has_symbolic_tensors(input_data) or
       (isinstance(input_data, list) and not input_data)):
     if steps is None:
-      input_type_str = 'a Dataset' if is_x_dataset else 'data tensors'
+      input_type_str = 'a Dataset iterator' if is_x_iterator else 'data tensors'
       raise ValueError('When using {input_type} as input to a model, you should'
                        ' specify the `{steps_name}` argument.'.format(
                            input_type=input_type_str, steps_name=steps_name))
@@ -1208,8 +1210,6 @@ def is_dataset_or_iterator(data):
 def extract_tensors_from_dataset(dataset):
   """Extract a tuple of tensors `inputs, targets, sample_weight` from a dataset.
 
-  Works only for graph mode.
-
   Arguments:
     dataset: Dataset instance.
 
@@ -1217,7 +1217,9 @@ def extract_tensors_from_dataset(dataset):
     Tuple of tensors `x, y, weights`. `y` and `weights` entry may be None.
   """
   iterator = dataset_ops.make_initializable_iterator(dataset)
-  K.get_session().run(iterator.initializer)
+  init_op = iterator.initializer
+  if not context.executing_eagerly():
+    K.get_session().run(init_op)
   inputs, targets, sample_weight = unpack_iterator_input(iterator)
   return inputs, targets, sample_weight
 
@@ -1256,6 +1258,51 @@ def unpack_iterator_input(iterator):
   return x, y, weights
 
 
+def infer_steps_for_dataset(dataset, steps, epochs=1, steps_name='steps'):
+  """Infers steps_per_epoch needed to loop through a dataset.
+
+  Arguments:
+      dataset: Input data of type tf.data.Dataset.
+      steps: Number of steps to draw from the dataset (may be None if unknown).
+      epochs: Number of times to iterate over the dataset.
+      steps_name: The string name of the steps argument, either `steps`,
+        `validation_steps`, or `steps_per_epoch`. Only used for error message
+        formatting.
+
+  Returns:
+    Integer or `None`. Inferred number of steps to loop through the dataset.
+    `None` is returned if the size of the dataset is unknown and `steps` was
+    not specified.
+
+  Raises:
+    ValueError: In case of invalid argument values.
+  """
+  assert isinstance(dataset, dataset_ops.DatasetV2)
+  size = K.get_value(cardinality.cardinality(dataset))
+  if size == cardinality.INFINITE and steps is None:
+    raise ValueError('When passing an infinitely repeating dataset, you '
+                     'must specify the `%s` argument.' % (steps_name,))
+  if size != cardinality.UNKNOWN:
+    if steps is not None and steps * epochs > size:
+      if epochs > 1:
+        raise ValueError('The dataset you passed contains %s batches, but you '
+                         'passed `epochs=%s` and `%s=%s`, which is a total of '
+                         '%s steps. We cannot draw that many steps from this '
+                         'dataset. We suggest to set `%s=%s`.' %
+                         (size, epochs, steps_name, steps, steps * epochs,
+                          steps_name, size // epochs))
+      else:
+        raise ValueError('The dataset you passed contains %s batches, but you '
+                         'passed `%s=%s`. We cannot draw that many steps from '
+                         'this dataset. We suggest to set `%s=%s`.' %
+                         (size, steps_name, steps, steps_name, size))
+  if steps is None:
+    if size >= 0:
+      return size
+    return None
+  return steps
+
+
 class ModelInputs(object):
   """Encapsulates model inputs.
 
-- 
GitLab


From 7305fd64837a2137cd1ace2c548f390a348f2f67 Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Thu, 10 Jan 2019 18:58:10 -0800
Subject: [PATCH 0537/2345] Put all calibration resources into resource
 manager. This fixes a race condition triggered in int8 mode: during
 calibration, when two sess.run()s fetch two different sets of ops and both of
 them trigger the computation of the TRTEngineOp, two different TRTEngineOp
 will be created, and dev_tensors_ will only be allocated once by the first op
 by the device's resource manager via LookupOrCreate(). As a result, the
 second TRTEngineOp will have a empty dev_tensors_ vector which will result in
 std::out_of_range error when the op tries to access it later.

We should consider marking TRTEngineOp as stateful, the problem is it uses TF
functions which has to be stateless, but we can use function library as key to
cache them.

PiperOrigin-RevId: 228816949
---
 .../contrib/tensorrt/kernels/trt_engine_op.cc     | 15 +++++++++------
 .../contrib/tensorrt/kernels/trt_engine_op.h      |  6 ------
 tensorflow/contrib/tensorrt/ops/trt_engine_op.cc  |  3 +++
 .../contrib/tensorrt/resources/trt_resources.h    |  8 ++++++++
 4 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
index bad568644b..1f6409341c 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
@@ -175,11 +175,13 @@ void TRTEngineOp::ExecuteNativeSegment(OpKernelContext* ctx,
   lib->Run(opts, native_func_, inputs, outputs,
            [this, ctx, outputs, helper](const tensorflow::Status& s) {
              tensorflow::core::ScopedUnref sc(helper);
-             VLOG(1) << "Native Segment completed";
              if (!s.ok()) {
+               LOG(ERROR) << "Failed to execute native segment " << this->name()
+                          << ": " << s;
                ctx->SetStatus(s);
                return;
              }
+             VLOG(1) << "Native Segment completed";
              for (size_t t = 0; t < outputs->size(); ++t) {
                ctx->set_output(t, outputs->at(t));
              }
@@ -219,7 +221,8 @@ void TRTEngineOp::ExecuteCalibration(OpKernelContext* ctx,
       return;
     }
     // Check the allocated buffer is sufficient for input
-    const auto device_tensor = dev_tensors_.at(i).AccessTensor(ctx);
+    const auto device_tensor =
+        calib_res->device_tensors_.at(i).AccessTensor(ctx);
     CHECK_EQ(t.TotalBytes(), device_tensor->TotalBytes());
     input_data.emplace(StrCat(kInputPHName, i), data_address);
   }
@@ -536,7 +539,7 @@ tensorflow::Status TRTEngineOp::AllocateCalibrationResources(
   const int batch_size = ctx->input(0).dim_size(0);
   const int num_inputs = ctx->num_inputs();
   std::vector<tensorflow::PartialTensorShape> shapes;
-  dev_tensors_.resize(num_inputs);
+  cres->device_tensors_.resize(num_inputs);
   VLOG(1) << " Constructing calibrator";
   for (int i = 0; i < num_inputs; i++) {
     // allocate workspace on device for inputs
@@ -544,19 +547,19 @@ tensorflow::Status TRTEngineOp::AllocateCalibrationResources(
     shapes.emplace_back(t.shape());
     Tensor* device_tensor;
     TF_RETURN_IF_ERROR(ctx->allocate_persistent(
-        t.dtype(), t.shape(), &dev_tensors_.at(i), &device_tensor));
+        t.dtype(), t.shape(), &cres->device_tensors_.at(i), &device_tensor));
     CHECK_EQ(t.TotalBytes(), device_tensor->TotalBytes());
     void* device_address = GetTensorAddress(device_tensor);
     if (device_address == nullptr) {
       return tensorflow::errors::InvalidArgument(
           "Unsupported data type encountered in input ", i);
     }
-    device_buffers_.emplace(
+    cres->device_buffers_.emplace(
         StrCat(kInputPHName, i),
         std::pair<void*, size_t>(device_address, device_tensor->TotalBytes()));
   }
   cres->calibrator_.reset(
-      new TRTInt8Calibrator(device_buffers_, batch_size, name()));
+      new TRTInt8Calibrator(cres->device_buffers_, batch_size, name()));
   const string label(name());
   auto segment_graph = &segment_graph_;
   const int platform_gpu_id =
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
index b545f497f3..ceb7483642 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
@@ -98,12 +98,6 @@ class TRTEngineOp : public AsyncOpKernel {
   // GraphDef representation of the segment.
   GraphDef segment_graph_;
 
-  // Lookup table for temporary staging areas of input tensors for calibration.
-  std::unordered_map<string, std::pair<void*, size_t>> device_buffers_;
-
-  // Temporary staging areas for calibration inputs.
-  std::vector<PersistentTensor> dev_tensors_;
-
   // Engine Precision mode.
   int precision_mode_;
 
diff --git a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
index 92405906eb..0f0688a280 100644
--- a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
@@ -28,6 +28,9 @@ namespace shape_inference {
 extern Status TRTEngineOpShapeInference(InferenceContext* c);
 }
 
+// TODO(laigd): consider making this op stateful. The only problem is it uses TF
+// function which has to be stateless, but we can use function library as the
+// key to cache the instantiated functions for different executor subgraphs.
 REGISTER_OP("TRTEngineOp")
     .Attr("serialized_segment: string")
     .Attr("input_shapes: list(shape)")
diff --git a/tensorflow/contrib/tensorrt/resources/trt_resources.h b/tensorflow/contrib/tensorrt/resources/trt_resources.h
index 8d877b392f..11930f3671 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_resources.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_resources.h
@@ -20,12 +20,14 @@ limitations under the License.
 #include <sstream>
 #include <string>
 #include <thread>
+#include <unordered_map>
 #include <vector>
 
 #include "tensorflow/contrib/tensorrt/convert/utils.h"
 #include "tensorflow/contrib/tensorrt/log/trt_logger.h"
 #include "tensorflow/contrib/tensorrt/resources/trt_allocator.h"
 #include "tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h"
+#include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 
 #if GOOGLE_CUDA
@@ -62,6 +64,12 @@ class TRTCalibrationResource : public tensorflow::ResourceBase {
     return oss.str();
   }
 
+  // Lookup table for temporary staging areas of input tensors for calibration.
+  std::unordered_map<string, std::pair<void*, size_t>> device_buffers_;
+
+  // Temporary staging areas for calibration inputs.
+  std::vector<PersistentTensor> device_tensors_;
+
   std::unique_ptr<TRTInt8Calibrator> calibrator_;
   TrtUniquePtrType<nvinfer1::IBuilder> builder_;
   TrtUniquePtrType<nvinfer1::ICudaEngine> engine_;
-- 
GitLab


From ed7a8b4dc39abf4b3a82d0f8edda267caf2e5451 Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Thu, 10 Jan 2019 19:05:20 -0800
Subject: [PATCH 0538/2345] Add logging of samples/steps count in each line in
 verbose=2 mode for fit method.

PiperOrigin-RevId: 228817708
---
 tensorflow/python/keras/utils/generic_utils.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/keras/utils/generic_utils.py b/tensorflow/python/keras/utils/generic_utils.py
index 454854618c..8b5cdadd45 100644
--- a/tensorflow/python/keras/utils/generic_utils.py
+++ b/tensorflow/python/keras/utils/generic_utils.py
@@ -391,9 +391,8 @@ class Progbar(object):
         sys.stdout.write('\n')
 
       if self.target is not None:
-        numdigits = int(np.floor(np.log10(self.target))) + 1
-        barstr = '%%%dd/%d [' % (numdigits, self.target)
-        bar = barstr % current
+        numdigits = int(np.log10(self.target)) + 1
+        bar = ('%' + str(numdigits) + 'd/%d') % (current, self.target)
         prog = float(current) / self.target
         prog_width = int(self.width * prog)
         if prog_width > 0:
@@ -456,7 +455,10 @@ class Progbar(object):
       sys.stdout.flush()
 
     elif self.verbose == 2:
-      if self.target is None or current >= self.target:
+      if self.target is not None and current >= self.target:
+        numdigits = int(np.log10(self.target)) + 1
+        count = ('%' + str(numdigits) + 'd/%d') % (current, self.target)
+        info = count + info
         for k in self._values_order:
           info += ' - %s:' % k
           avg = np.mean(self._values[k][0] / max(1, self._values[k][1]))
-- 
GitLab


From 03fb5daa9cc0c84c52ab800843dcef22580f64c2 Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Thu, 10 Jan 2019 19:11:40 -0800
Subject: [PATCH 0539/2345] Simplify PrepareForConvolution: * Change the
 virtual functions overloaded on DeviceMemory<T> to a single function
 DeviceMemoryBase. * Create a non-virtual, template wrapper that takes
 DeviceMemory<T>.

PiperOrigin-RevId: 228818212
---
 tensorflow/stream_executor/cuda/cuda_dnn.cc | 325 ++++----------------
 tensorflow/stream_executor/cuda/cuda_dnn.h  | 167 +---------
 tensorflow/stream_executor/dnn.h            | 172 ++---------
 tensorflow/stream_executor/stream.cc        | 173 ++++++-----
 tensorflow/stream_executor/stream.h         |  10 +
 5 files changed, 216 insertions(+), 631 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index 60648075c0..7bae6c207e 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -995,9 +995,11 @@ cudnnDataType_t ToCudnnDataType(
     dnn::DataLayout data_layout = dnn::DataLayout::kBatchDepthYX) {
   switch (data_type) {
     case dnn::DataType::kFloat:
+      return CUDNN_DATA_FLOAT;
     case dnn::DataType::kDouble:
+      return CUDNN_DATA_DOUBLE;
     case dnn::DataType::kHalf:
-      return static_cast<cudnnDataType_t>(data_type);
+      return CUDNN_DATA_HALF;
     case dnn::DataType::kInt8:
       return data_layout == dnn::DataLayout::kBatchDepthYX4 ? CUDNN_DATA_INT8x4
                                                             : CUDNN_DATA_INT8;
@@ -1008,6 +1010,15 @@ cudnnDataType_t ToCudnnDataType(
   }
 }
 
+cudnnDataType_t ToCudnnDataType(dnn::DataType data_type,
+                                dnn::FilterLayout filter_layout) {
+  if (data_type == dnn::DataType::kInt8 &&
+      filter_layout == dnn::FilterLayout::kOutputInputYX4) {
+    return CUDNN_DATA_INT8x4;
+  }
+  return ToCudnnDataType(data_type);
+}
+
 template <typename T>
 cudnnDataType_t GetCudnnDataType(
     dnn::DataLayout data_layout = dnn::DataLayout::kBatchDepthYX) {
@@ -2815,30 +2826,60 @@ void LogCudaProto(const dnn::ConvolutionProto& conv, float profile_time_ms,
 
 }  // namespace
 
-template <class T>
-port::Status CudnnSupport::PrepareForConvolutionImpl(
-    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-    const DeviceMemory<T>& input_data,
+port::Status CudnnSupport::DoPrepareForConvolution(
+    dnn::ConvolutionKind kind, dnn::DataType element_type, Stream* stream,
+    const dnn::BatchDescriptor& input_descriptor, DeviceMemoryBase input_data,
     const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<T>& filter_data,
+    DeviceMemoryBase filter_data, const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemoryBase output_data,
     const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& output_descriptor, DeviceMemory<T>* output_data,
-    dnn::DataType accumulator_type, ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
-  CudnnTensorDescriptor input_nd(input_descriptor, cudnn_type);
-  CudnnTensorDescriptor output_nd(output_descriptor, cudnn_type);
-  CudnnFilterDescriptor filter(filter_descriptor, cudnn_type);
-  CudnnConvolutionDescriptor conv(convolution_descriptor,
-                                  ToCudnnDataType(accumulator_type));
+    ScratchAllocator* scratch_allocator, dnn::AlgorithmDesc* algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory) {
+  CudnnTensorDescriptor input_nd(
+      input_descriptor,
+      ToCudnnDataType(element_type, input_descriptor.layout()));
+  CudnnFilterDescriptor filter_nd(
+      filter_descriptor,
+      ToCudnnDataType(element_type, filter_descriptor.layout()));
+  CudnnTensorDescriptor output_nd(
+      output_descriptor,
+      ToCudnnDataType(element_type, output_descriptor.layout()));
+  CudnnConvolutionDescriptor conv(
+      convolution_descriptor,
+      ToCudnnDataType(GetConvAccumulatorType(element_type)));
 
   auto cudnn = cudnn_->GetHandle(parent_, stream);
 
-  SE_ASSIGN_OR_RETURN(*algorithm_desc,
-                      GetCudnnConvolutionForwardAlgorithm(
-                          stream, cudnn, algorithm_config, input_nd, filter,
-                          conv, output_nd, scratch_allocator, scratch_memory));
+  switch (kind) {
+    case dnn::ConvolutionKind::FORWARD: {
+      SE_ASSIGN_OR_RETURN(
+          *algorithm_desc,
+          GetCudnnConvolutionForwardAlgorithm(
+              stream, cudnn, algorithm_config, input_nd, filter_nd, conv,
+              output_nd, scratch_allocator, scratch_memory));
+      break;
+    }
+    case dnn::ConvolutionKind::BACKWARD_DATA: {
+      SE_ASSIGN_OR_RETURN(
+          *algorithm_desc,
+          GetCudnnConvolutionBackwardDataAlgorithm(
+              stream, cudnn, algorithm_config, input_nd, filter_nd, conv,
+              output_nd, scratch_allocator, scratch_memory));
+      break;
+    }
+    case dnn::ConvolutionKind::BACKWARD_FILTER: {
+      SE_ASSIGN_OR_RETURN(
+          *algorithm_desc,
+          GetCudnnConvolutionBackwardFilterAlgorithm(
+              stream, cudnn, algorithm_config, input_nd, filter_nd, conv,
+              output_nd, scratch_allocator, scratch_memory));
+      break;
+    }
+    default:
+      return port::InternalError(
+          absl::StrCat("Unexpected convolution kind ", static_cast<int>(kind)));
+  }
 
   return port::Status::OK();
 }
@@ -3351,64 +3392,6 @@ port::Status CudnnSupport::DoBatchNormalizationBackwardImpl(
   return port::Status::OK();
 }
 
-bool CudnnSupport::PrepareForConvolution(
-    Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
-    const DeviceMemory<float>& input_data,
-    const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<float>& filter_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<float>* output_data, ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  return IsStatusOk(PrepareForConvolutionImpl<float>(
-                        stream, batch_descriptor, input_data, filter_descriptor,
-                        filter_data, convolution_descriptor, output_descriptor,
-                        output_data, dnn::DataType::kFloat, scratch_allocator,
-                        algorithm_config, algorithm_desc, scratch_memory),
-                    /*report_error=*/true);
-}
-
-bool CudnnSupport::PrepareForConvolution(
-    Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
-    const DeviceMemory<double>& input_data,
-    const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<double>& filter_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  return IsStatusOk(PrepareForConvolutionImpl<double>(
-                        stream, batch_descriptor, input_data, filter_descriptor,
-                        filter_data, convolution_descriptor, output_descriptor,
-                        output_data, dnn::DataType::kDouble, scratch_allocator,
-                        algorithm_config, algorithm_desc, scratch_memory),
-                    /*report_error=*/true);
-}
-
-bool CudnnSupport::PrepareForConvolution(
-    Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
-    const DeviceMemory<Eigen::half>& input_data,
-    const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<Eigen::half>& filter_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<Eigen::half>* output_data, ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  dnn::DataType acc_type =
-      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
-          ? dnn::DataType::kFloat
-          : dnn::DataType::kHalf;
-  return IsStatusOk(
-      PrepareForConvolutionImpl<Eigen::half>(
-          stream, batch_descriptor, input_data, filter_descriptor, filter_data,
-          convolution_descriptor, output_descriptor, output_data, acc_type,
-          scratch_allocator, algorithm_config, algorithm_desc, scratch_memory),
-      /*report_error=*/true);
-}
-
 bool CudnnSupport::DoConvolve(
     Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
     const DeviceMemory<float>& input_data,
@@ -3592,36 +3575,6 @@ bool CudnnSupport::DoTransformTensor(Stream* stream,
   return IsStatusOk(status, /*report_error=*/true);
 }
 
-template <class T>
-port::Status CudnnSupport::PrepareForConvolutionBackwardDataImpl(
-    Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<T>& filter_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<T> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& input_descriptor,
-    DeviceMemory<T>* backward_input_data, dnn::DataType accumulator_type,
-    ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
-  auto cudnn = cudnn_->GetHandle(parent_, stream);
-
-  CudnnTensorDescriptor out_back_nd(output_descriptor, cudnn_type);
-  CudnnTensorDescriptor in_back_nd(input_descriptor, cudnn_type);
-  CudnnFilterDescriptor filter(filter_descriptor, cudnn_type);
-  CudnnConvolutionDescriptor conv(convolution_descriptor,
-                                  ToCudnnDataType(accumulator_type));
-
-  SE_ASSIGN_OR_RETURN(
-      *algorithm_desc,
-      GetCudnnConvolutionBackwardDataAlgorithm(
-          stream, cudnn, algorithm_config, in_back_nd, filter, conv,
-          out_back_nd, scratch_allocator, scratch_memory));
-
-  return port::Status::OK();
-}
-
 template <class T>
 port::Status CudnnSupport::DoConvolveBackwardDataImpl(
     Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
@@ -3722,70 +3675,6 @@ port::Status CudnnSupport::DoConvolveBackwardDataImpl(
   return port::Status::OK();
 }
 
-bool CudnnSupport::PrepareForConvolutionBackwardData(
-    Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<double>& filter_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<double> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& input_descriptor,
-    DeviceMemory<double>* backward_input_data,
-    ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  return IsStatusOk(
-      PrepareForConvolutionBackwardDataImpl(
-          stream, filter_descriptor, filter_data, output_descriptor,
-          backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, dnn::DataType::kDouble, scratch_allocator,
-          algorithm_config, algorithm_desc, scratch_memory),
-      /*report_error=*/true);
-}
-
-bool CudnnSupport::PrepareForConvolutionBackwardData(
-    Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<float>& filter_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<float> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& input_descriptor,
-    DeviceMemory<float>* backward_input_data,
-    ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  return IsStatusOk(
-      PrepareForConvolutionBackwardDataImpl(
-          stream, filter_descriptor, filter_data, output_descriptor,
-          backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, dnn::DataType::kFloat, scratch_allocator,
-          algorithm_config, algorithm_desc, scratch_memory),
-      /*report_error=*/true);
-}
-
-bool CudnnSupport::PrepareForConvolutionBackwardData(
-    Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<Eigen::half>& filter_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<Eigen::half> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& input_descriptor,
-    DeviceMemory<Eigen::half>* backward_input_data,
-    ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  dnn::DataType acc_type =
-      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
-          ? dnn::DataType::kFloat
-          : dnn::DataType::kHalf;
-  return IsStatusOk(
-      PrepareForConvolutionBackwardDataImpl(
-          stream, filter_descriptor, filter_data, output_descriptor,
-          backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, acc_type, scratch_allocator, algorithm_config,
-          algorithm_desc, scratch_memory),
-      /*report_error=*/true);
-}
-
 bool CudnnSupport::DoConvolveBackwardData(
     Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
     const DeviceMemory<double>& filter_data,
@@ -3846,36 +3735,6 @@ bool CudnnSupport::DoConvolveBackwardData(
       /*report_error=*/!output_profile_result);
 }
 
-template <class T>
-port::Status CudnnSupport::PrepareForConvolutionBackwardFilterImpl(
-    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-    const DeviceMemory<T>& input_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<T> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::FilterDescriptor& filter_descriptor,
-    DeviceMemory<T>* backward_filter_data, dnn::DataType accumulator_type,
-    ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
-  auto cudnn = cudnn_->GetHandle(parent_, stream);
-
-  CudnnTensorDescriptor out_back_nd(output_descriptor, cudnn_type);
-  CudnnTensorDescriptor input_nd(input_descriptor, cudnn_type);
-  CudnnFilterDescriptor filter(filter_descriptor, cudnn_type);
-  CudnnConvolutionDescriptor conv(convolution_descriptor,
-                                  ToCudnnDataType(accumulator_type));
-
-  SE_ASSIGN_OR_RETURN(
-      *algorithm_desc,
-      GetCudnnConvolutionBackwardFilterAlgorithm(
-          stream, cudnn, algorithm_config, input_nd, filter, conv, out_back_nd,
-          scratch_allocator, scratch_memory));
-
-  return port::Status::OK();
-}
-
 template <class T>
 port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
     Stream* stream, const dnn::BatchDescriptor& input_descriptor,
@@ -4013,70 +3872,6 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
   return port::Status::OK();
 }
 
-bool CudnnSupport::PrepareForConvolutionBackwardFilter(
-    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-    const DeviceMemory<double>& input_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<double> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::FilterDescriptor& filter_descriptor,
-    DeviceMemory<double>* backward_filter_data,
-    ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  return IsStatusOk(
-      PrepareForConvolutionBackwardFilterImpl(
-          stream, input_descriptor, input_data, output_descriptor,
-          backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, dnn::DataType::kDouble, scratch_allocator,
-          algorithm_config, algorithm_desc, scratch_memory),
-      /*report_error=*/true);
-}
-
-bool CudnnSupport::PrepareForConvolutionBackwardFilter(
-    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-    const DeviceMemory<float>& input_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<float> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::FilterDescriptor& filter_descriptor,
-    DeviceMemory<float>* backward_filter_data,
-    ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  return IsStatusOk(
-      PrepareForConvolutionBackwardFilterImpl(
-          stream, input_descriptor, input_data, output_descriptor,
-          backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, dnn::DataType::kFloat, scratch_allocator,
-          algorithm_config, algorithm_desc, scratch_memory),
-      /*report_error=*/true);
-}
-
-bool CudnnSupport::PrepareForConvolutionBackwardFilter(
-    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-    const DeviceMemory<Eigen::half>& input_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<Eigen::half> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::FilterDescriptor& filter_descriptor,
-    DeviceMemory<Eigen::half>* backward_filter_data,
-    ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  dnn::DataType acc_type =
-      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
-          ? dnn::DataType::kFloat
-          : dnn::DataType::kHalf;
-  return IsStatusOk(
-      PrepareForConvolutionBackwardFilterImpl(
-          stream, input_descriptor, input_data, output_descriptor,
-          backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, acc_type, scratch_allocator, algorithm_config,
-          algorithm_desc, scratch_memory),
-      /*report_error=*/true);
-}
-
 bool CudnnSupport::DoConvolveBackwardFilter(
     Stream* stream, const dnn::BatchDescriptor& input_descriptor,
     const DeviceMemory<double>& input_data,
diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.h b/tensorflow/stream_executor/cuda/cuda_dnn.h
index d7514981d5..383a99e118 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.h
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.h
@@ -258,43 +258,6 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<float>* scale_backprop,
       DeviceMemory<float>* offset_backprop) override;
 
-  bool PrepareForConvolution(
-      Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
-      const DeviceMemory<float>& input_data,
-      const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<float>& filter_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<float>* output_data, ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory) override;
-
-  bool PrepareForConvolution(
-      Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
-      const DeviceMemory<double>& input_data,
-      const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<double>& filter_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory) override;
-
-  bool PrepareForConvolution(
-      Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
-      const DeviceMemory<Eigen::half>& input_data,
-      const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<Eigen::half>& filter_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<Eigen::half>* output_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory) override;
-
   bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
                   const DeviceMemory<float>& input_data,
                   const dnn::FilterDescriptor& filter_descriptor,
@@ -427,45 +390,6 @@ class CudnnSupport : public dnn::DnnSupport {
     return false;
   }
 
-  bool PrepareForConvolutionBackwardData(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<float>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<float> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<float>* backward_input_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory) override;
-
-  bool PrepareForConvolutionBackwardData(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<double>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<double> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<double>* backward_input_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory) override;
-
-  bool PrepareForConvolutionBackwardData(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<Eigen::half>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<Eigen::half> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<Eigen::half>* backward_input_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory) override;
-
   bool DoConvolveBackwardData(
       Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
       const DeviceMemory<double>& filter_data,
@@ -502,45 +426,6 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result) override;
 
-  bool PrepareForConvolutionBackwardFilter(
-      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-      const DeviceMemory<double>& input_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<double> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::FilterDescriptor& filter_descriptor,
-      DeviceMemory<double>* backward_filter_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory) override;
-
-  bool PrepareForConvolutionBackwardFilter(
-      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-      const DeviceMemory<float>& input_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<float> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::FilterDescriptor& filter_descriptor,
-      DeviceMemory<float>* backward_filter_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory) override;
-
-  bool PrepareForConvolutionBackwardFilter(
-      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-      const DeviceMemory<Eigen::half>& input_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<Eigen::half> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::FilterDescriptor& filter_descriptor,
-      DeviceMemory<Eigen::half>* backward_filter_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory) override;
-
   bool DoConvolveBackwardFilter(
       Stream* stream, const dnn::BatchDescriptor& input_descriptor,
       const DeviceMemory<double>& input_data,
@@ -791,19 +676,6 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<T>* x_backprop, DeviceMemory<U>* scale_backprop,
       DeviceMemory<U>* offset_backprop);
 
-  template <class T>
-  port::Status PrepareForConvolutionImpl(
-      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-      const DeviceMemory<T>& input_data,
-      const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<T>& filter_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<T>* output_data, dnn::DataType accumulator_type,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory);
-
   template <class T>
   port::Status DoConvolveImpl(
       Stream* stream, const dnn::BatchDescriptor& input_descriptor,
@@ -834,19 +706,6 @@ class CudnnSupport : public dnn::DnnSupport {
       const dnn::AlgorithmConfig& algorithm_config,
       dnn::ProfileResult* output_profile_result);
 
-  template <class T>
-  port::Status PrepareForConvolutionBackwardDataImpl(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<T>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<T> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<T>* backward_input_data, dnn::DataType accumulator_type,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory);
-
   template <class T>
   port::Status DoConvolveBackwardDataImpl(
       Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
@@ -860,19 +719,6 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result);
 
-  template <class T>
-  port::Status PrepareForConvolutionBackwardFilterImpl(
-      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-      const DeviceMemory<T>& input_data,
-      const dnn::BatchDescriptor& output_descriptor_in,
-      DeviceMemory<T> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::FilterDescriptor& filter_descriptor,
-      DeviceMemory<T>* backward_filter_data, dnn::DataType accumulator_type,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory);
-
   template <class T>
   port::Status DoConvolveBackwardFilterImpl(
       Stream* stream, const dnn::BatchDescriptor& input_descriptor,
@@ -938,6 +784,19 @@ class CudnnSupport : public dnn::DnnSupport {
       ScratchAllocator* workspace_allocator,
       dnn::ProfileResult* output_profile_result);
 
+ private:
+  port::Status DoPrepareForConvolution(
+      dnn::ConvolutionKind kind, dnn::DataType element_type, Stream* stream,
+      const dnn::BatchDescriptor& input_descriptor, DeviceMemoryBase input_data,
+      const dnn::FilterDescriptor& filter_descriptor,
+      DeviceMemoryBase filter_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemoryBase output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::AlgorithmConfig& algorithm_config,
+      ScratchAllocator* scratch_allocator, dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) override;
+
   SE_DISALLOW_COPY_AND_ASSIGN(CudnnSupport);
 };
 
diff --git a/tensorflow/stream_executor/dnn.h b/tensorflow/stream_executor/dnn.h
index f5a77d6525..f5eef29109 100644
--- a/tensorflow/stream_executor/dnn.h
+++ b/tensorflow/stream_executor/dnn.h
@@ -942,11 +942,7 @@ class VersionInfo {
 //   burden.
 // * Poor error handling: the API should return Status objects.
 //
-// Things worth trying:
-// * Move functions that are not actually common back to the backends. Then,
-//   callers may use dynamic_cast to access specific backends. This may not be
-//   that hard, as many of the callers are Stream::ThenXxx functions.
-// * Change all the returned bools to Status.
+// PrepareForConvolution is an example for how new APIs should be written.
 class DnnSupport {
  public:
   DnnSupport() {}
@@ -1187,50 +1183,24 @@ class DnnSupport {
     return false;
   }
 
-  virtual bool PrepareForConvolution(
-      Stream* stream, const BatchDescriptor& batch_descriptor,
-      const DeviceMemory<float>& input_data,
-      const FilterDescriptor& filter_descriptor,
-      const DeviceMemory<float>& filter_data,
-      const ConvolutionDescriptor& convolution_descriptor,
-      const BatchDescriptor& output_descriptor,
-      DeviceMemory<float>* output_data, ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-    *algorithm_desc = {};
-    *scratch_memory = {};
-    return true;
-  }
-
-  virtual bool PrepareForConvolution(
-      Stream* stream, const BatchDescriptor& batch_descriptor,
-      const DeviceMemory<double>& input_data,
+  template <typename ElementType>
+  port::Status PrepareForConvolution(
+      ConvolutionKind kind, Stream* stream,
+      const BatchDescriptor& batch_descriptor,
+      DeviceMemory<ElementType> input_data,
       const FilterDescriptor& filter_descriptor,
-      const DeviceMemory<double>& filter_data,
-      const ConvolutionDescriptor& convolution_descriptor,
+      DeviceMemory<ElementType> filter_data,
       const BatchDescriptor& output_descriptor,
-      DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-    *algorithm_desc = {};
-    *scratch_memory = {};
-    return true;
-  }
-
-  virtual bool PrepareForConvolution(
-      Stream* stream, const BatchDescriptor& batch_descriptor,
-      const DeviceMemory<Eigen::half>& input_data,
-      const FilterDescriptor& filter_descriptor,
-      const DeviceMemory<Eigen::half>& filter_data,
+      DeviceMemory<ElementType> output_data,
       const ConvolutionDescriptor& convolution_descriptor,
-      const BatchDescriptor& output_descriptor,
-      DeviceMemory<Eigen::half>* output_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-    *algorithm_desc = {};
-    *scratch_memory = {};
-    return true;
+      const AlgorithmConfig& algorithm_config,
+      ScratchAllocator* scratch_allocator, AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) {
+    return DoPrepareForConvolution(
+        kind, ToDataType<ElementType>::value, stream, batch_descriptor,
+        input_data, filter_descriptor, filter_data, output_descriptor,
+        output_data, convolution_descriptor, algorithm_config,
+        scratch_allocator, algorithm_desc, scratch_memory);
   }
 
   // Enqueues a single-precision convolution operation onto the stream.
@@ -1360,54 +1330,6 @@ class DnnSupport {
       const BatchDescriptor& output_descriptor,
       DeviceMemory<float>* output_data) = 0;
 
-  virtual bool PrepareForConvolutionBackwardData(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<float>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<float> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<float>* backward_input_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-    *algorithm_desc = {};
-    *scratch_memory = {};
-    return true;
-  }
-
-  virtual bool PrepareForConvolutionBackwardData(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<double>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<double> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<double>* backward_input_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-    *algorithm_desc = {};
-    *scratch_memory = {};
-    return true;
-  }
-
-  virtual bool PrepareForConvolutionBackwardData(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<Eigen::half>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<Eigen::half> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<Eigen::half>* backward_input_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-    *algorithm_desc = {};
-    *scratch_memory = {};
-    return true;
-  }
-
   // Enqueues a single-precision backward convolution (for data) operation onto
   // the stream.
   //
@@ -1468,54 +1390,6 @@ class DnnSupport {
       DeviceMemory<uint8>* scratch_memory,
       ProfileResult* output_profile_result) = 0;
 
-  virtual bool PrepareForConvolutionBackwardFilter(
-      Stream* stream, const BatchDescriptor& input_descriptor,
-      const DeviceMemory<float>& input_data,
-      const BatchDescriptor& output_descriptor,
-      DeviceMemory<float> backward_output_data,
-      const ConvolutionDescriptor& convolution_descriptor,
-      const FilterDescriptor& filter_descriptor,
-      DeviceMemory<float>* backward_filter_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-    *algorithm_desc = {};
-    *scratch_memory = {};
-    return true;
-  }
-
-  virtual bool PrepareForConvolutionBackwardFilter(
-      Stream* stream, const BatchDescriptor& input_descriptor,
-      const DeviceMemory<double>& input_data,
-      const BatchDescriptor& output_descriptor,
-      DeviceMemory<double> backward_output_data,
-      const ConvolutionDescriptor& convolution_descriptor,
-      const FilterDescriptor& filter_descriptor,
-      DeviceMemory<double>* backward_filter_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-    *algorithm_desc = {};
-    *scratch_memory = {};
-    return true;
-  }
-
-  virtual bool PrepareForConvolutionBackwardFilter(
-      Stream* stream, const BatchDescriptor& input_descriptor,
-      const DeviceMemory<Eigen::half>& input_data,
-      const BatchDescriptor& output_descriptor,
-      DeviceMemory<Eigen::half> backward_output_data,
-      const ConvolutionDescriptor& convolution_descriptor,
-      const FilterDescriptor& filter_descriptor,
-      DeviceMemory<Eigen::half>* backward_filter_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-    *algorithm_desc = {};
-    *scratch_memory = {};
-    return true;
-  }
-
   // Enqueues a single-precision backward convolution (for filter) operation
   // onto the stream.
   //
@@ -2516,6 +2390,20 @@ class DnnSupport {
   }
 
  private:
+  virtual port::Status DoPrepareForConvolution(
+      ConvolutionKind kind, DataType element_type, Stream* stream,
+      const BatchDescriptor& batch_descriptor, DeviceMemoryBase input_data,
+      const FilterDescriptor& filter_descriptor, DeviceMemoryBase filter_data,
+      const BatchDescriptor& output_descriptor, DeviceMemoryBase output_data,
+      const ConvolutionDescriptor& convolution_descriptor,
+      const AlgorithmConfig& algorithm_config,
+      ScratchAllocator* scratch_allocator, AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) {
+    *algorithm_desc = {};
+    *scratch_memory = {};
+    return port::Status::OK();
+  }
+
   SE_DISALLOW_COPY_AND_ASSIGN(DnnSupport);
 };
 
diff --git a/tensorflow/stream_executor/stream.cc b/tensorflow/stream_executor/stream.cc
index 1befc18e19..9937861d88 100644
--- a/tensorflow/stream_executor/stream.cc
+++ b/tensorflow/stream_executor/stream.cc
@@ -551,10 +551,11 @@ Stream &Stream::ThenConvolveWithScratch(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      CheckError(dnn->PrepareForConvolution(
-          this, input_descriptor, input_data, filter_descriptor, filter_data,
-          convolution_descriptor, output_descriptor, output, scratch_allocator,
-          dnn::AlgorithmConfig(), &algorithm_desc, &scratch_memory));
+      CheckStatus(dnn->PrepareForConvolution(
+          dnn::ConvolutionKind::FORWARD, this, input_descriptor, input_data,
+          filter_descriptor, filter_data, output_descriptor, *output,
+          convolution_descriptor, dnn::AlgorithmConfig(), scratch_allocator,
+          &algorithm_desc, &scratch_memory));
       CheckError(dnn->DoConvolve(
           this, input_descriptor, input_data, filter_descriptor, filter_data,
           convolution_descriptor, output_descriptor, output, algorithm_desc,
@@ -583,10 +584,11 @@ Stream &Stream::ThenConvolveWithScratch(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      CheckError(dnn->PrepareForConvolution(
-          this, input_descriptor, input_data, filter_descriptor, filter_data,
-          convolution_descriptor, output_descriptor, output, scratch_allocator,
-          dnn::AlgorithmConfig(), &algorithm_desc, &scratch_memory));
+      CheckStatus(dnn->PrepareForConvolution(
+          dnn::ConvolutionKind::FORWARD, this, input_descriptor, input_data,
+          filter_descriptor, filter_data, output_descriptor, *output,
+          convolution_descriptor, dnn::AlgorithmConfig(), scratch_allocator,
+          &algorithm_desc, &scratch_memory));
       CheckError(dnn->DoConvolve(
           this, input_descriptor, input_data, filter_descriptor, filter_data,
           convolution_descriptor, output_descriptor, output, algorithm_desc,
@@ -770,10 +772,13 @@ Stream &Stream::ThenConvolveWithAlgorithm(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      auto status = dnn->PrepareForConvolution(
-          this, input_descriptor, input_data, filter_descriptor, filter_data,
-          convolution_descriptor, output_descriptor, output, scratch_allocator,
-          algorithm_config, &algorithm_desc, &scratch_memory);
+      auto status =
+          dnn->PrepareForConvolution(
+                 dnn::ConvolutionKind::FORWARD, this, input_descriptor,
+                 input_data, filter_descriptor, filter_data, output_descriptor,
+                 *output, convolution_descriptor, algorithm_config,
+                 scratch_allocator, &algorithm_desc, &scratch_memory)
+              .ok();
       if (status) {
         status = dnn->DoConvolve(
             this, input_descriptor, input_data, filter_descriptor, filter_data,
@@ -809,10 +814,13 @@ Stream &Stream::ThenConvolveWithAlgorithm(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      auto status = dnn->PrepareForConvolution(
-          this, input_descriptor, input_data, filter_descriptor, filter_data,
-          convolution_descriptor, output_descriptor, output, scratch_allocator,
-          algorithm_config, &algorithm_desc, &scratch_memory);
+      auto status =
+          dnn->PrepareForConvolution(
+                 dnn::ConvolutionKind::FORWARD, this, input_descriptor,
+                 input_data, filter_descriptor, filter_data, output_descriptor,
+                 *output, convolution_descriptor, algorithm_config,
+                 scratch_allocator, &algorithm_desc, &scratch_memory)
+              .ok();
       if (status) {
         status = dnn->DoConvolve(
             this, input_descriptor, input_data, filter_descriptor, filter_data,
@@ -848,10 +856,13 @@ Stream &Stream::ThenConvolveWithAlgorithm(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      auto status = dnn->PrepareForConvolution(
-          this, input_descriptor, input_data, filter_descriptor, filter_data,
-          convolution_descriptor, output_descriptor, output, scratch_allocator,
-          algorithm_config, &algorithm_desc, &scratch_memory);
+      auto status =
+          dnn->PrepareForConvolution(
+                 dnn::ConvolutionKind::FORWARD, this, input_descriptor,
+                 input_data, filter_descriptor, filter_data, output_descriptor,
+                 *output, convolution_descriptor, algorithm_config,
+                 scratch_allocator, &algorithm_desc, &scratch_memory)
+              .ok();
       if (status) {
         status = dnn->DoConvolve(
             this, input_descriptor, input_data, filter_descriptor, filter_data,
@@ -1005,11 +1016,12 @@ Stream &Stream::ThenConvolveBackwardDataWithScratch(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      CheckError(dnn->PrepareForConvolutionBackwardData(
-          this, filter_descriptor, filter_data, output_descriptor,
-          backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, scratch_allocator, dnn::AlgorithmConfig(),
-          &algorithm_desc, &scratch_memory));
+      CheckStatus(dnn->PrepareForConvolution(
+          dnn::ConvolutionKind::BACKWARD_DATA, this, input_descriptor,
+          *backward_input_data, filter_descriptor, filter_data,
+          output_descriptor, backward_output_data, convolution_descriptor,
+          dnn::AlgorithmConfig(), scratch_allocator, &algorithm_desc,
+          &scratch_memory));
       CheckError(dnn->DoConvolveBackwardData(
           this, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
@@ -1042,11 +1054,14 @@ Stream &Stream::ThenConvolveBackwardDataWithAlgorithm(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      auto status = dnn->PrepareForConvolutionBackwardData(
-          this, filter_descriptor, filter_data, output_descriptor,
-          backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, scratch_allocator, algorithm_config,
-          &algorithm_desc, &scratch_memory);
+      auto status =
+          dnn->PrepareForConvolution(
+                 dnn::ConvolutionKind::BACKWARD_DATA, this, input_descriptor,
+                 *backward_input_data, filter_descriptor, filter_data,
+                 output_descriptor, backward_output_data,
+                 convolution_descriptor, algorithm_config, scratch_allocator,
+                 &algorithm_desc, &scratch_memory)
+              .ok();
       if (status) {
         status = dnn->DoConvolveBackwardData(
             this, filter_descriptor, filter_data, output_descriptor,
@@ -1084,11 +1099,14 @@ Stream &Stream::ThenConvolveBackwardDataWithAlgorithm(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      auto status = dnn->PrepareForConvolutionBackwardData(
-          this, filter_descriptor, filter_data, output_descriptor,
-          backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, scratch_allocator, algorithm_config,
-          &algorithm_desc, &scratch_memory);
+      auto status =
+          dnn->PrepareForConvolution(
+                 dnn::ConvolutionKind::BACKWARD_DATA, this, input_descriptor,
+                 *backward_input_data, filter_descriptor, filter_data,
+                 output_descriptor, backward_output_data,
+                 convolution_descriptor, algorithm_config, scratch_allocator,
+                 &algorithm_desc, &scratch_memory)
+              .ok();
       if (status) {
         status = dnn->DoConvolveBackwardData(
             this, filter_descriptor, filter_data, output_descriptor,
@@ -1126,11 +1144,14 @@ Stream &Stream::ThenConvolveBackwardDataWithAlgorithm(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      auto status = dnn->PrepareForConvolutionBackwardData(
-          this, filter_descriptor, filter_data, output_descriptor,
-          backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, scratch_allocator, algorithm_config,
-          &algorithm_desc, &scratch_memory);
+      auto status =
+          dnn->PrepareForConvolution(
+                 dnn::ConvolutionKind::BACKWARD_DATA, this, input_descriptor,
+                 *backward_input_data, filter_descriptor, filter_data,
+                 output_descriptor, backward_output_data,
+                 convolution_descriptor, algorithm_config, scratch_allocator,
+                 &algorithm_desc, &scratch_memory)
+              .ok();
       if (status) {
         status = dnn->DoConvolveBackwardData(
             this, filter_descriptor, filter_data, output_descriptor,
@@ -1166,11 +1187,12 @@ Stream &Stream::ThenConvolveBackwardDataWithScratch(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      CheckError(dnn->PrepareForConvolutionBackwardData(
-          this, filter_descriptor, filter_data, output_descriptor,
-          backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, scratch_allocator, dnn::AlgorithmConfig(),
-          &algorithm_desc, &scratch_memory));
+      CheckStatus(dnn->PrepareForConvolution(
+          dnn::ConvolutionKind::BACKWARD_DATA, this, input_descriptor,
+          *backward_input_data, filter_descriptor, filter_data,
+          output_descriptor, backward_output_data, convolution_descriptor,
+          dnn::AlgorithmConfig(), scratch_allocator, &algorithm_desc,
+          &scratch_memory));
       CheckError(dnn->DoConvolveBackwardData(
           this, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
@@ -1215,11 +1237,12 @@ Stream &Stream::ThenConvolveBackwardFilterWithScratch(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      CheckError(dnn->PrepareForConvolutionBackwardFilter(
-          this, input_descriptor, input_data, output_descriptor,
-          backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, scratch_allocator, dnn::AlgorithmConfig(),
-          &algorithm_desc, &scratch_memory));
+      CheckStatus(dnn->PrepareForConvolution(
+          dnn::ConvolutionKind::BACKWARD_FILTER, this, input_descriptor,
+          input_data, filter_descriptor, *backward_filter_data,
+          output_descriptor, backward_output_data, convolution_descriptor,
+          dnn::AlgorithmConfig(), scratch_allocator, &algorithm_desc,
+          &scratch_memory));
       CheckError(dnn->DoConvolveBackwardFilter(
           this, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
@@ -1252,11 +1275,14 @@ Stream &Stream::ThenConvolveBackwardFilterWithAlgorithm(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      auto status = dnn->PrepareForConvolutionBackwardFilter(
-          this, input_descriptor, input_data, output_descriptor,
-          backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, scratch_allocator, algorithm_config,
-          &algorithm_desc, &scratch_memory);
+      auto status =
+          dnn->PrepareForConvolution(
+                 dnn::ConvolutionKind::BACKWARD_FILTER, this, input_descriptor,
+                 input_data, filter_descriptor, *backward_filter_data,
+                 output_descriptor, backward_output_data,
+                 convolution_descriptor, algorithm_config, scratch_allocator,
+                 &algorithm_desc, &scratch_memory)
+              .ok();
       if (status) {
         status = dnn->DoConvolveBackwardFilter(
             this, input_descriptor, input_data, output_descriptor,
@@ -1294,11 +1320,14 @@ Stream &Stream::ThenConvolveBackwardFilterWithAlgorithm(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      auto status = dnn->PrepareForConvolutionBackwardFilter(
-          this, input_descriptor, input_data, output_descriptor,
-          backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, scratch_allocator, algorithm_config,
-          &algorithm_desc, &scratch_memory);
+      auto status =
+          dnn->PrepareForConvolution(
+                 dnn::ConvolutionKind::BACKWARD_FILTER, this, input_descriptor,
+                 input_data, filter_descriptor, *backward_filter_data,
+                 output_descriptor, backward_output_data,
+                 convolution_descriptor, algorithm_config, scratch_allocator,
+                 &algorithm_desc, &scratch_memory)
+              .ok();
       if (status) {
         status = dnn->DoConvolveBackwardFilter(
             this, input_descriptor, input_data, output_descriptor,
@@ -1334,11 +1363,12 @@ Stream &Stream::ThenConvolveBackwardFilterWithScratch(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      CheckError(dnn->PrepareForConvolutionBackwardFilter(
-          this, input_descriptor, input_data, output_descriptor,
-          backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, scratch_allocator, dnn::AlgorithmConfig(),
-          &algorithm_desc, &scratch_memory));
+      CheckStatus(dnn->PrepareForConvolution(
+          dnn::ConvolutionKind::BACKWARD_FILTER, this, input_descriptor,
+          input_data, filter_descriptor, *backward_filter_data,
+          output_descriptor, backward_output_data, convolution_descriptor,
+          dnn::AlgorithmConfig(), scratch_allocator, &algorithm_desc,
+          &scratch_memory));
       CheckError(dnn->DoConvolveBackwardFilter(
           this, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
@@ -1371,11 +1401,14 @@ Stream &Stream::ThenConvolveBackwardFilterWithAlgorithm(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      auto status = dnn->PrepareForConvolutionBackwardFilter(
-          this, input_descriptor, input_data, output_descriptor,
-          backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, scratch_allocator, algorithm_config,
-          &algorithm_desc, &scratch_memory);
+      auto status =
+          dnn->PrepareForConvolution(
+                 dnn::ConvolutionKind::BACKWARD_FILTER, this, input_descriptor,
+                 input_data, filter_descriptor, *backward_filter_data,
+                 output_descriptor, backward_output_data,
+                 convolution_descriptor, algorithm_config, scratch_allocator,
+                 &algorithm_desc, &scratch_memory)
+              .ok();
       if (status) {
         status = dnn->DoConvolveBackwardFilter(
             this, input_descriptor, input_data, output_descriptor,
diff --git a/tensorflow/stream_executor/stream.h b/tensorflow/stream_executor/stream.h
index d5e2fdf58d..85d65c1061 100644
--- a/tensorflow/stream_executor/stream.h
+++ b/tensorflow/stream_executor/stream.h
@@ -2099,6 +2099,16 @@ class Stream {
     ok_ = false;
   }
 
+  // Checks the status and logs the error message, if any.
+  void CheckStatus(port::Status status) LOCKS_EXCLUDED(mu_) {
+    if (status.ok()) {
+      return;
+    }
+    LOG(ERROR) << status;
+    mutex_lock lock(mu_);
+    ok_ = false;
+  }
+
   void SetError() { CheckError(false /* = operation_retcode */); }
 
   void SetErrorAndLogNoDnnSupport() {
-- 
GitLab


From 96b579c4d2f592b8183b34abe6cb155e7796809c Mon Sep 17 00:00:00 2001
From: Aurelien Geron <aurelien.geron@gmail.com>
Date: Fri, 11 Jan 2019 11:53:21 +0800
Subject: [PATCH 0540/2345] Fix issue in function_test.py: Function to
 ConcreteFunction

---
 tensorflow/python/eager/function_test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index 3bb4f2452c..73692e3efc 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -1729,8 +1729,8 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
       return x
 
     graph_function = foo.get_concrete_function(constant_op.constant(1.0))
-    with self.assertRaisesRegexp(ValueError, 'All inputs to `Function`s must '
-                                 'be Tensors;.*'):
+    with self.assertRaisesRegexp(
+        ValueError, 'All inputs to `ConcreteFunction`s must be Tensors;.*'):
       graph_function('Not a Tensor.')
 
   def testSwapImplementationWithGrapplerPlugin(self):
-- 
GitLab


From 764109a35299f55f7da5cbf6d3d8fc01a06668b6 Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Thu, 10 Jan 2019 20:35:37 -0800
Subject: [PATCH 0541/2345] Reenable test cases that are broken by TRT 4.0
 since TF is now built against TRT 5.0.

PiperOrigin-RevId: 228824151
---
 .../tensorrt/test/batch_matmul_test.py        | 22 -------------------
 .../tensorrt/test/biasadd_matmul_test.py      |  6 -----
 .../tensorrt/test/identity_output_test.py     |  6 -----
 .../contrib/tensorrt/test/rank_two_test.py    |  6 -----
 4 files changed, 40 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/test/batch_matmul_test.py b/tensorflow/contrib/tensorrt/test/batch_matmul_test.py
index f42308ecb7..384b84cb02 100644
--- a/tensorflow/contrib/tensorrt/test/batch_matmul_test.py
+++ b/tensorflow/contrib/tensorrt/test/batch_matmul_test.py
@@ -86,28 +86,6 @@ class BatchMatMulTest(trt_test.TfTrtIntegrationTestBase):
     """Return the expected engines to run."""
     return ["TRTEngineOp_1"]
 
-  def ShouldRunTest(self, run_params):
-    """Whether to run the test."""
-    # TODO(aaroey): Trt library will fail like:
-    #
-    # ../builder/cudnnBuilder2.cpp:685:
-    # virtual std::vector<nvinfer1::query::Ports<
-    #     nvinfer1::query::TensorRequirements>>
-    # nvinfer1::builder::Node::getSupportedFormats(
-    #     const nvinfer1::query::Ports<nvinfer1::query::AbstractTensor>&,
-    #     const nvinfer1::cudnn::HardwareContext&,
-    #     nvinfer1::builder::Format::Type,
-    #     const nvinfer1::builder::FormatTypeHack&) const:
-    # Assertion `sf' failed.
-    #
-    # To reproduce, run:
-    # bazel test -c opt --copt=-mavx \
-    #   --test_arg=BatchMatMulTest.testTfTrt_ToolConversion_INT8_DynamicEngine \
-    #   tensorflow/contrib/tensorrt:batch_matmul_test
-    #
-    # Investigate and fix it.
-    return not trt_test.IsQuantizationMode(run_params.precision_mode)
-
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py b/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
index 053b38ff1c..49f5e4cc9f 100644
--- a/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
+++ b/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
@@ -130,12 +130,6 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
     """Return the expected engines to build."""
     return ["TRTEngineOp_0"]
 
-  def ShouldRunTest(self, run_params):
-    """Whether to run the test."""
-    # TODO(aaroey): Trt 4.0 forbids conversion for tensors with rank <3 in int8
-    # mode, which is a bug. Re-enable this when trt library is fixed.
-    return not trt_test.IsQuantizationMode(run_params.precision_mode)
-
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/tensorrt/test/identity_output_test.py b/tensorflow/contrib/tensorrt/test/identity_output_test.py
index d1c6dd5204..196ccdbe9d 100644
--- a/tensorflow/contrib/tensorrt/test/identity_output_test.py
+++ b/tensorflow/contrib/tensorrt/test/identity_output_test.py
@@ -69,12 +69,6 @@ class IdentityTest(trt_test.TfTrtIntegrationTestBase):
     """Return the expected engines to build."""
     return ['TRTEngineOp_0']
 
-  def ShouldRunTest(self, run_params):
-    """Whether to run the test."""
-    # TODO(aaroey): Trt 4.0 forbids conversion for tensors with rank <3 in int8
-    # mode, which is a bug. Re-enable this when trt library is fixed.
-    return not trt_test.IsQuantizationMode(run_params.precision_mode)
-
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/tensorrt/test/rank_two_test.py b/tensorflow/contrib/tensorrt/test/rank_two_test.py
index 563232fc12..dd3b72d212 100644
--- a/tensorflow/contrib/tensorrt/test/rank_two_test.py
+++ b/tensorflow/contrib/tensorrt/test/rank_two_test.py
@@ -80,12 +80,6 @@ class RankTwoTest(trt_test.TfTrtIntegrationTestBase):
         ],
     }
 
-  def ShouldRunTest(self, run_params):
-    """Whether to run the test."""
-    # TODO(aaroey): Trt 4.0 forbids conversion for tensors with rank <3 in int8
-    # mode, which is a bug. Re-enable this when trt library is fixed.
-    return not trt_test.IsQuantizationMode(run_params.precision_mode)
-
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 9d1679d8f3fbc76f2b2a74cce7e8014f8d4543bc Mon Sep 17 00:00:00 2001
From: Aurelien Geron <aurelien.geron@gmail.com>
Date: Fri, 11 Jan 2019 13:13:49 +0800
Subject: [PATCH 0542/2345] Rename `SavedPolymorphicFunction` to `SavedFunction
 and `MonomorphicFunction` to `ConcreteFunction` and `concrete_function`
 attribute (in `ConcreteFunction` proto) to `name`

---
 .../python/saved_model/function_deserialization.py     | 10 +++++-----
 .../python/saved_model/function_serialization.py       |  8 ++++----
 tensorflow/python/saved_model/load.py                  |  8 ++++----
 tensorflow/python/saved_model/saved_object_graph.proto | 10 +++++-----
 4 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py
index f1c43cb7a4..41694fb749 100644
--- a/tensorflow/python/saved_model/function_deserialization.py
+++ b/tensorflow/python/saved_model/function_deserialization.py
@@ -55,10 +55,10 @@ def _inputs_compatible(args, stored_inputs):
 
 
 def recreate_function(saved_function, concrete_functions):
-  """Creates a `Function` from a `SavedPolymorphicFunction`.
+  """Creates a `Function` from a `SavedFunction`.
 
   Args:
-    saved_function: `SavedPolymorphicFunction` proto.
+    saved_function: `SavedFunction` proto.
     concrete_functions: map from function name to `ConcreteFunction`.
 
   Returns:
@@ -80,10 +80,10 @@ def recreate_function(saved_function, concrete_functions):
     """Calls a restored function."""
     # TODO(allenl): Functions saved with input_signatures should revive with
     # input_signatures.
-    for monomorphic_function in saved_function.monomorphic_function:
-      function_obj = concrete_functions[monomorphic_function.concrete_function]
+    for concrete_function in saved_function.concrete_function:
+      function_obj = concrete_functions[concrete_function.name]
       canonicalized_original_inputs = coder.decode_proto(
-          monomorphic_function.canonicalized_input)
+          concrete_function.canonicalized_input)
 
       try:
         can_args, can_kwargs = function_spec.canonicalize_function_inputs(
diff --git a/tensorflow/python/saved_model/function_serialization.py b/tensorflow/python/saved_model/function_serialization.py
index b645a7c964..2868ca6f66 100644
--- a/tensorflow/python/saved_model/function_serialization.py
+++ b/tensorflow/python/saved_model/function_serialization.py
@@ -25,9 +25,9 @@ from tensorflow.python.saved_model import saved_object_graph_pb2
 
 
 def serialize_function(function, node_ids):
-  """Build a SavedPolymorphicFunction proto."""
+  """Build a SavedFunction proto."""
   coder = nested_structure_coder.StructureCoder()
-  proto = saved_object_graph_pb2.SavedPolymorphicFunction()
+  proto = saved_object_graph_pb2.SavedFunction()
 
   proto.function_spec_tuple.CopyFrom(
       coder.encode_structure(function.function_spec.as_tuple()))  # pylint: disable=protected-access
@@ -43,8 +43,8 @@ def serialize_function(function, node_ids):
           "captures tensor %s which is unsupported or not reachable from root.",
           concrete_function.name, capture)
       continue
-    function_proto = proto.monomorphic_function.add()
-    function_proto.concrete_function = concrete_function.name
+    function_proto = proto.concrete_function.add()
+    function_proto.name = concrete_function.name
     function_proto.canonicalized_input.CopyFrom(
         coder.encode_structure(signature))
     function_proto.bound_inputs.extend(bound_inputs)
diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
index ae71a228ea..8c4f1c9f74 100644
--- a/tensorflow/python/saved_model/load.py
+++ b/tensorflow/python/saved_model/load.py
@@ -54,14 +54,14 @@ class _Loader(object):
     seen_concrete_functions = set()
     for object_proto in self._proto.nodes:
       if object_proto.WhichOneof("kind") == "function":
-        for monomorphic_function in object_proto.function.monomorphic_function:
-          name = monomorphic_function.concrete_function
+        for concrete_function in object_proto.function.concrete_function:
+          name = concrete_function.name
           bound_inputs = [
               self._get_tensor_from_node(node_id)
-              for node_id in monomorphic_function.bound_inputs]
+              for node_id in concrete_function.bound_inputs]
           bound_variables = [
               self._nodes[node_id]
-              for node_id in monomorphic_function.bound_inputs
+              for node_id in concrete_function.bound_inputs
               if self._proto.nodes[node_id].WhichOneof("kind") == "variable"
           ]
           if name in seen_concrete_functions:
diff --git a/tensorflow/python/saved_model/saved_object_graph.proto b/tensorflow/python/saved_model/saved_object_graph.proto
index 1e2514b7f7..6b44a75e3f 100644
--- a/tensorflow/python/saved_model/saved_object_graph.proto
+++ b/tensorflow/python/saved_model/saved_object_graph.proto
@@ -51,7 +51,7 @@ message SavedObject {
   oneof kind {
     SavedUserObject user_object = 4;
     SavedAsset asset = 5;
-    SavedPolymorphicFunction function = 6;
+    SavedFunction function = 6;
     SavedVariable variable = 7;
   }
 }
@@ -78,16 +78,16 @@ message SavedAsset {
 }
 
 // A function with multiple signatures, possibly with non-Tensor arguments.
-message SavedPolymorphicFunction {
-  repeated SavedMonomorphicFunction monomorphic_function = 1;
+message SavedFunction {
+  repeated SavedConcreteFunction concrete_function = 1;
   // Tuple representing a `FunctionSpec`.
   // TODO(vbardiovsky): Make this a proto.
   StructuredValue function_spec_tuple = 2;
 }
 
-message SavedMonomorphicFunction {
+message SavedConcreteFunction {
   // A reference to a TensorFlow function in the MetaGraph's FunctionDefLibrary
-  string concrete_function = 1;
+  string name = 1;
 
   // Bound inputs to the function. The SavedObjects identified by the node ids
   // given here are appended as extra inputs to the caller-supplied inputs.
-- 
GitLab


From 489f1dc4573eabbff59c149031984ea8319a7bf8 Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <penporn@google.com>
Date: Thu, 10 Jan 2019 22:07:22 -0800
Subject: [PATCH 0543/2345] Fix typo.

PiperOrigin-RevId: 228831583
---
 configure.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.py b/configure.py
index bb59063f79..adc9ef9cac 100644
--- a/configure.py
+++ b/configure.py
@@ -1694,7 +1694,7 @@ def main():
   config_info_line('noaws', 'Disable AWS S3 filesystem support.')
   config_info_line('nogcp', 'Disable GCP support.')
   config_info_line('nohdfs', 'Disable HDFS support.')
-  config_info_line('noignite', 'Disable Apacha Ignite support.')
+  config_info_line('noignite', 'Disable Apache Ignite support.')
   config_info_line('nokafka', 'Disable Apache Kafka support.')
   config_info_line('nonccl', 'Disable NVIDIA NCCL support.')
 
-- 
GitLab


From 30afb286a47474dc68b61e572028e3b96b43af6a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 22:23:11 -0800
Subject: [PATCH 0544/2345] Split off SeparableConv and ConvTranspose tests and
 parameterize many test so they can be properly sharded.

PiperOrigin-RevId: 228832550
---
 tensorflow/python/keras/BUILD                 |  30 +-
 .../python/keras/layers/convolutional_test.py | 547 ++++--------------
 .../layers/convolutional_transpose_test.py    | 209 +++++++
 .../layers/separable_convolutional_test.py    | 167 ++++++
 4 files changed, 502 insertions(+), 451 deletions(-)
 create mode 100644 tensorflow/python/keras/layers/convolutional_transpose_test.py
 create mode 100644 tensorflow/python/keras/layers/separable_convolutional_test.py

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 5f44a1b4e8..ef8cb23a09 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -487,9 +487,9 @@ tf_py_test(
     shard_count = 2,
 )
 
-tf_py_test(
+cuda_py_test(
     name = "convolutional_test",
-    size = "large",
+    size = "medium",
     srcs = ["layers/convolutional_test.py"],
     additional_deps = [
         ":keras",
@@ -497,7 +497,19 @@ tf_py_test(
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
     ],
-    shard_count = 11,
+    shard_count = 8,
+)
+
+cuda_py_test(
+    name = "convolutional_transpose_test",
+    size = "medium",
+    srcs = ["layers/convolutional_transpose_test.py"],
+    additional_deps = [
+        ":keras",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+    ],
 )
 
 cuda_py_test(
@@ -659,6 +671,18 @@ tf_py_test(
     shard_count = 4,
 )
 
+cuda_py_test(
+    name = "separable_convolutional_test",
+    size = "medium",
+    srcs = ["layers/separable_convolutional_test.py"],
+    additional_deps = [
+        ":keras",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+    ],
+)
+
 cuda_py_test(
     name = "unified_lstm_test",
     size = "medium",
diff --git a/tensorflow/python/keras/layers/convolutional_test.py b/tensorflow/python/keras/layers/convolutional_test.py
index eedf66ff67..9140ce426e 100644
--- a/tensorflow/python/keras/layers/convolutional_test.py
+++ b/tensorflow/python/keras/layers/convolutional_test.py
@@ -18,8 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import copy
-
+from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.python import keras
@@ -30,39 +29,32 @@ from tensorflow.python.platform import test
 
 
 @keras_parameterized.run_all_keras_modes
-class Convolution1DTest(keras_parameterized.TestCase):
+class Conv1DTest(keras_parameterized.TestCase):
 
-  def _run_test(self, kwargs, arg, values):
+  def _run_test(self, kwargs):
     num_samples = 2
     stack_size = 3
     length = 7
 
-    test_kwargs = copy.copy(kwargs)
-    for value in values:
-      test_kwargs[arg] = value
-      with self.cached_session(use_gpu=True):
-        testing_utils.layer_test(
-            keras.layers.Conv1D,
-            kwargs=test_kwargs,
-            input_shape=(num_samples, length, stack_size))
-
-  def test_conv1d(self):
-    kwargs = {
-        'filters': 2,
-        'kernel_size': 3,
-    }
-
-    self._run_test(kwargs, 'padding', ['valid', 'same', 'causal'])
-    self._run_test(kwargs, 'strides', [2])
-    self._run_test(kwargs, 'dilation_rate', [2])
-
-    kwargs = {
-        'filters': 2,
-        'kernel_size': 3,
-        'padding': 'same',
-    }
-    self._run_test(kwargs, 'dilation_rate', [2])
-    self._run_test(kwargs, 'dilation_rate', [3])
+    with self.cached_session(use_gpu=True):
+      testing_utils.layer_test(
+          keras.layers.Conv1D,
+          kwargs=kwargs,
+          input_shape=(num_samples, length, stack_size))
+
+  @parameterized.named_parameters(
+      ('padding_valid', {'padding': 'valid'}),
+      ('padding_same', {'padding': 'same'}),
+      ('padding_same_dilation_2', {'padding': 'same', 'dilation_rate': 2}),
+      ('padding_same_dilation_3', {'padding': 'same', 'dilation_rate': 3}),
+      ('padding_causal', {'padding': 'causal'}),
+      ('strides', {'strides': 2}),
+      ('dilation_rate', {'dilation_rate': 2}),
+  )
+  def test_conv1d(self, kwargs):
+    kwargs['filters'] = 2
+    kwargs['kernel_size'] = 3
+    self._run_test(kwargs)
 
   def test_conv1d_regularizers(self):
     kwargs = {
@@ -74,7 +66,7 @@ class Convolution1DTest(keras_parameterized.TestCase):
         'activity_regularizer': 'l2',
         'strides': 1
     }
-    with self.session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       layer = keras.layers.Conv1D(**kwargs)
       layer.build((None, 5, 2))
       self.assertEqual(len(layer.losses), 2)
@@ -93,7 +85,7 @@ class Convolution1DTest(keras_parameterized.TestCase):
         'bias_constraint': b_constraint,
         'strides': 1
     }
-    with self.session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       layer = keras.layers.Conv1D(**kwargs)
       layer.build((None, 5, 2))
       self.assertEqual(layer.kernel.constraint, k_constraint)
@@ -103,41 +95,33 @@ class Convolution1DTest(keras_parameterized.TestCase):
 @keras_parameterized.run_all_keras_modes
 class Conv2DTest(keras_parameterized.TestCase):
 
-  def _run_test(self, kwargs, arg, values):
+  def _run_test(self, kwargs):
     num_samples = 2
     stack_size = 3
     num_row = 7
     num_col = 6
 
-    test_kwargs = copy.copy(kwargs)
-    for value in values:
-      test_kwargs[arg] = value
-      with self.cached_session(use_gpu=True):
-        testing_utils.layer_test(
-            keras.layers.Conv2D,
-            kwargs=test_kwargs,
-            input_shape=(num_samples, num_row, num_col, stack_size))
-
-  def test_conv2d(self):
-    kwargs = {
-        'filters': 2,
-        'kernel_size': (3, 3),
-    }
-
-    self._run_test(kwargs, 'padding', ['valid', 'same'])
-    self._run_test(kwargs, 'strides', [(2, 2)])
-    if test.is_gpu_available(cuda_only=True):
+    with self.cached_session(use_gpu=True):
+      testing_utils.layer_test(
+          keras.layers.Conv2D,
+          kwargs=kwargs,
+          input_shape=(num_samples, num_row, num_col, stack_size))
+
+  @parameterized.named_parameters(
+      ('padding_valid', {'padding': 'valid'}),
+      ('padding_same', {'padding': 'same'}),
+      ('padding_same_dilation_2', {'padding': 'same', 'dilation_rate': 2}),
+      ('strides', {'strides': (2, 2)}),
+      ('dilation_rate', {'dilation_rate': (2, 2)}),
       # Only runs on GPU with CUDA, channels_first is not supported on CPU.
       # TODO(b/62340061): Support channels_first on CPU.
-      self._run_test(kwargs, 'data_format', ['channels_first'])
-    self._run_test(kwargs, 'dilation_rate', [(2, 2)])
-
-    kwargs = {
-        'filters': 2,
-        'kernel_size': 3,
-        'padding': 'same',
-    }
-    self._run_test(kwargs, 'dilation_rate', [2])
+      ('data_format', {'data_format': 'channels_first'}),
+  )
+  def test_conv2d(self, kwargs):
+    kwargs['filters'] = 2
+    kwargs['kernel_size'] = (3, 3)
+    if 'data_format' not in kwargs or test.is_gpu_available(cuda_only=True):
+      self._run_test(kwargs)
 
   def test_conv2d_regularizers(self):
     kwargs = {
@@ -149,7 +133,7 @@ class Conv2DTest(keras_parameterized.TestCase):
         'activity_regularizer': 'l2',
         'strides': 1
     }
-    with self.session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       layer = keras.layers.Conv2D(**kwargs)
       layer.build((None, 5, 5, 2))
       self.assertEqual(len(layer.losses), 2)
@@ -168,381 +152,43 @@ class Conv2DTest(keras_parameterized.TestCase):
         'bias_constraint': b_constraint,
         'strides': 1
     }
-    with self.session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       layer = keras.layers.Conv2D(**kwargs)
       layer.build((None, 5, 5, 2))
       self.assertEqual(layer.kernel.constraint, k_constraint)
       self.assertEqual(layer.bias.constraint, b_constraint)
 
 
-@keras_parameterized.run_all_keras_modes
-class Conv2DTransposeTest(keras_parameterized.TestCase):
-
-  def _run_test(self, kwargs, arg, values):
-    num_samples = 2
-    stack_size = 3
-    num_row = 7
-    num_col = 6
-
-    test_kwargs = copy.copy(kwargs)
-    for value in values:
-      test_kwargs[arg] = value
-      with self.cached_session(use_gpu=True):
-        testing_utils.layer_test(
-            keras.layers.Conv2DTranspose,
-            kwargs=test_kwargs,
-            input_shape=(num_samples, num_row, num_col, stack_size))
-
-  def test_conv2dtranspose(self):
-    kwargs = {
-        'filters': 2,
-        'kernel_size': (3, 3),
-    }
-
-    self._run_test(kwargs, 'padding', ['valid', 'same'])
-    self._run_test(kwargs, 'strides', [(2, 2)])
-    if test.is_gpu_available(cuda_only=True):
-      self._run_test(kwargs, 'data_format', ['channels_first'])
-
-    kwargs['strides'] = (2, 2)
-    self._run_test(kwargs, 'output_padding', [(1, 1)])
-
-  def test_conv2dtranspose_regularizers(self):
-    kwargs = {
-        'filters': 3,
-        'kernel_size': 3,
-        'padding': 'valid',
-        'kernel_regularizer': 'l2',
-        'bias_regularizer': 'l2',
-        'activity_regularizer': 'l2',
-        'strides': 1
-    }
-    with self.session(use_gpu=True):
-      layer = keras.layers.Conv2DTranspose(**kwargs)
-      layer.build((None, 5, 5, 2))
-      self.assertEqual(len(layer.losses), 2)
-      layer(keras.backend.variable(np.ones((1, 5, 5, 2))))
-      self.assertEqual(len(layer.losses), 3)
-
-  def test_conv2dtranspose_constraints(self):
-    k_constraint = lambda x: x
-    b_constraint = lambda x: x
-
-    kwargs = {
-        'filters': 3,
-        'kernel_size': 3,
-        'padding': 'valid',
-        'kernel_constraint': k_constraint,
-        'bias_constraint': b_constraint,
-        'strides': 1
-    }
-    with self.session(use_gpu=True):
-      layer = keras.layers.Conv2DTranspose(**kwargs)
-      layer.build((None, 5, 5, 2))
-      self.assertEqual(layer.kernel.constraint, k_constraint)
-      self.assertEqual(layer.bias.constraint, b_constraint)
-
-  def test_conv2d_transpose_dilation(self):
-    testing_utils.layer_test(keras.layers.Conv2DTranspose,
-                             kwargs={'filters': 2,
-                                     'kernel_size': 3,
-                                     'padding': 'same',
-                                     'data_format': 'channels_last',
-                                     'dilation_rate': (2, 2)},
-                             input_shape=(2, 5, 6, 3))
-
-    input_data = np.arange(48).reshape((1, 4, 4, 3)).astype(np.float32)
-    expected_output = np.float32([[192, 228, 192, 228],
-                                  [336, 372, 336, 372],
-                                  [192, 228, 192, 228],
-                                  [336, 372, 336, 372]]).reshape((1, 4, 4, 1))
-    testing_utils.layer_test(keras.layers.Conv2DTranspose,
-                             input_data=input_data,
-                             kwargs={'filters': 1,
-                                     'kernel_size': 3,
-                                     'padding': 'same',
-                                     'data_format': 'channels_last',
-                                     'dilation_rate': (2, 2),
-                                     'kernel_initializer': 'ones'},
-                             expected_output=expected_output)
-
-
-@keras_parameterized.run_all_keras_modes
-class Conv3DTransposeTest(keras_parameterized.TestCase):
-
-  def _run_test(self, kwargs, arg, values):
-    num_samples = 2
-    stack_size = 3
-    num_row = 7
-    num_col = 6
-    depth = 5
-
-    test_kwargs = copy.copy(kwargs)
-    for value in values:
-      test_kwargs[arg] = value
-      with self.cached_session(use_gpu=True):
-        testing_utils.layer_test(
-            keras.layers.Conv3DTranspose,
-            kwargs=test_kwargs,
-            input_shape=(num_samples, depth, num_row, num_col, stack_size))
-
-  def test_conv3dtranspose(self):
-    kwargs = {
-        'filters': 2,
-        'kernel_size': (3, 3, 3),
-    }
-
-    self._run_test(kwargs, 'padding', ['valid', 'same'])
-    self._run_test(kwargs, 'strides', [(2, 2, 2)])
-    if test.is_gpu_available(cuda_only=True):
-      self._run_test(kwargs, 'data_format', ['channels_first'])
-
-    kwargs['strides'] = (2, 2, 2)
-    self._run_test(kwargs, 'output_padding', [(1, 1, 1)])
-
-  def test_conv3dtranspose_regularizers(self):
-    kwargs = {
-        'filters': 3,
-        'kernel_size': 3,
-        'padding': 'valid',
-        'kernel_regularizer': 'l2',
-        'bias_regularizer': 'l2',
-        'activity_regularizer': 'l2',
-        'strides': 1
-    }
-    with self.session(use_gpu=True):
-      layer = keras.layers.Conv3DTranspose(**kwargs)
-      layer.build((None, 5, 5, 5, 2))
-      self.assertEqual(len(layer.losses), 2)
-      layer(keras.backend.variable(np.ones((1, 5, 5, 5, 2))))
-      self.assertEqual(len(layer.losses), 3)
-
-  def test_conv3dtranspose_constraints(self):
-    k_constraint = lambda x: x
-    b_constraint = lambda x: x
-
-    kwargs = {
-        'filters': 3,
-        'kernel_size': 3,
-        'padding': 'valid',
-        'kernel_constraint': k_constraint,
-        'bias_constraint': b_constraint,
-        'strides': 1
-    }
-    with self.session(use_gpu=True):
-      layer = keras.layers.Conv3DTranspose(**kwargs)
-      layer.build((None, 5, 5, 5, 2))
-      self.assertEqual(layer.kernel.constraint, k_constraint)
-      self.assertEqual(layer.bias.constraint, b_constraint)
-
-  def test_conv3dtranspose_dynamic_shape(self):
-    input_data = np.random.random((1, 3, 3, 3, 3)).astype(np.float32)
-    with self.session(use_gpu=True):
-      # Won't raise error here.
-      testing_utils.layer_test(
-          keras.layers.Conv3DTranspose,
-          kwargs={
-              'data_format': 'channels_last',
-              'filters': 3,
-              'kernel_size': 3
-          },
-          input_shape=(None, None, None, None, 3),
-          input_data=input_data)
-      if test.is_gpu_available(cuda_only=True):
-        testing_utils.layer_test(
-            keras.layers.Conv3DTranspose,
-            kwargs={
-                'data_format': 'channels_first',
-                'filters': 3,
-                'kernel_size': 3
-            },
-            input_shape=(None, 3, None, None, None),
-            input_data=input_data)
-
-
-@keras_parameterized.run_all_keras_modes
-class SeparableConv1DTest(keras_parameterized.TestCase):
-
-  def _run_test(self, kwargs, arg, values):
-    num_samples = 2
-    stack_size = 3
-    length = 7
-
-    test_kwargs = copy.copy(kwargs)
-    for value in values:
-      test_kwargs[arg] = value
-      with self.cached_session(use_gpu=True):
-        testing_utils.layer_test(
-            keras.layers.SeparableConv1D,
-            kwargs=test_kwargs,
-            input_shape=(num_samples, length, stack_size))
-
-  def test_separable_conv1d(self):
-    kwargs = {
-        'filters': 2,
-        'kernel_size': 3,
-    }
-
-    self._run_test(kwargs, 'padding', ['valid', 'same', 'causal'])
-    self._run_test(kwargs, 'strides', [2])
-    self._run_test(kwargs, 'dilation_rate', [2])
-    self._run_test(kwargs, 'depth_multiplier', [2])
-
-    kwargs = {
-        'filters': 2,
-        'kernel_size': 3,
-        'padding': 'same',
-    }
-    self._run_test(kwargs, 'dilation_rate', [2])
-
-  def test_separable_conv1d_regularizers(self):
-    kwargs = {
-        'filters': 3,
-        'kernel_size': 3,
-        'padding': 'valid',
-        'depthwise_regularizer': 'l2',
-        'pointwise_regularizer': 'l2',
-        'bias_regularizer': 'l2',
-        'activity_regularizer': 'l2',
-        'strides': 1
-    }
-    with self.session(use_gpu=True):
-      layer = keras.layers.SeparableConv1D(**kwargs)
-      layer.build((None, 5, 2))
-      self.assertEqual(len(layer.losses), 3)
-      layer(keras.backend.variable(np.ones((1, 5, 2))))
-      self.assertEqual(len(layer.losses), 4)
-
-  def test_separable_conv1d_constraints(self):
-    d_constraint = lambda x: x
-    p_constraint = lambda x: x
-    b_constraint = lambda x: x
-
-    kwargs = {
-        'filters': 3,
-        'kernel_size': 3,
-        'padding': 'valid',
-        'pointwise_constraint': p_constraint,
-        'depthwise_constraint': d_constraint,
-        'bias_constraint': b_constraint,
-        'strides': 1
-    }
-    with self.session(use_gpu=True):
-      layer = keras.layers.SeparableConv1D(**kwargs)
-      layer.build((None, 5, 2))
-      self.assertEqual(layer.depthwise_kernel.constraint, d_constraint)
-      self.assertEqual(layer.pointwise_kernel.constraint, p_constraint)
-      self.assertEqual(layer.bias.constraint, b_constraint)
-
-
-@keras_parameterized.run_all_keras_modes
-class SeparableConv2DTest(keras_parameterized.TestCase):
-
-  def _run_test(self, kwargs, arg, values):
-    num_samples = 2
-    stack_size = 3
-    num_row = 7
-    num_col = 6
-
-    test_kwargs = copy.copy(kwargs)
-    for value in values:
-      test_kwargs[arg] = value
-      with self.cached_session(use_gpu=True):
-        testing_utils.layer_test(
-            keras.layers.SeparableConv2D,
-            kwargs=test_kwargs,
-            input_shape=(num_samples, num_row, num_col, stack_size))
-
-  def test_separable_conv2d(self):
-    kwargs = {
-        'filters': 2,
-        'kernel_size': 3,
-    }
-
-    self._run_test(kwargs, 'padding', ['valid', 'same'])
-    self._run_test(kwargs, 'strides', [2])
-    if test.is_gpu_available(cuda_only=True):
-      self._run_test(kwargs, 'data_format', ['channels_first'])
-    self._run_test(kwargs, 'dilation_rate', [2])
-    self._run_test(kwargs, 'depth_multiplier', [2])
-
-    kwargs = {
-        'filters': 2,
-        'kernel_size': 3,
-        'padding': 'same',
-    }
-    self._run_test(kwargs, 'dilation_rate', [2])
-
-  def test_separable_conv2d_regularizers(self):
-    kwargs = {
-        'filters': 3,
-        'kernel_size': 3,
-        'padding': 'valid',
-        'depthwise_regularizer': 'l2',
-        'pointwise_regularizer': 'l2',
-        'bias_regularizer': 'l2',
-        'activity_regularizer': 'l2',
-        'strides': 1
-    }
-    with self.session(use_gpu=True):
-      layer = keras.layers.SeparableConv2D(**kwargs)
-      layer.build((None, 5, 5, 2))
-      self.assertEqual(len(layer.losses), 3)
-      layer(keras.backend.variable(np.ones((1, 5, 5, 2))))
-      self.assertEqual(len(layer.losses), 4)
-
-  def test_separable_conv2d_constraints(self):
-    d_constraint = lambda x: x
-    p_constraint = lambda x: x
-    b_constraint = lambda x: x
-
-    kwargs = {
-        'filters': 3,
-        'kernel_size': 3,
-        'padding': 'valid',
-        'pointwise_constraint': p_constraint,
-        'depthwise_constraint': d_constraint,
-        'bias_constraint': b_constraint,
-        'strides': 1
-    }
-    with self.session(use_gpu=True):
-      layer = keras.layers.SeparableConv2D(**kwargs)
-      layer.build((None, 5, 5, 2))
-      self.assertEqual(layer.depthwise_kernel.constraint, d_constraint)
-      self.assertEqual(layer.pointwise_kernel.constraint, p_constraint)
-      self.assertEqual(layer.bias.constraint, b_constraint)
-
-
 @keras_parameterized.run_all_keras_modes
 class Conv3DTest(keras_parameterized.TestCase):
 
-  def _run_test(self, kwargs, arg, values):
+  def _run_test(self, kwargs):
     num_samples = 2
     stack_size = 3
     num_row = 7
     num_col = 6
     depth = 5
 
-    test_kwargs = copy.copy(kwargs)
-    for value in values:
-      test_kwargs[arg] = value
-      with self.cached_session(use_gpu=True):
-        testing_utils.layer_test(
-            keras.layers.Conv3D,
-            kwargs=test_kwargs,
-            input_shape=(num_samples, depth, num_row, num_col, stack_size))
-
-  def test_conv3d(self):
-    kwargs = {
-        'filters': 2,
-        'kernel_size': (3, 3, 3),
-    }
-
-    self._run_test(kwargs, 'padding', ['valid', 'same'])
-    self._run_test(kwargs, 'strides', [(2, 2, 2)])
-    self._run_test(kwargs, 'dilation_rate', [(2, 2, 2)])
-    if test.is_gpu_available(cuda_only=True):
-      self._run_test(kwargs, 'data_format', ['channels_first'])
+    with self.cached_session(use_gpu=True):
+      testing_utils.layer_test(
+          keras.layers.Conv3D,
+          kwargs=kwargs,
+          input_shape=(num_samples, depth, num_row, num_col, stack_size))
+
+  @parameterized.named_parameters(
+      ('padding_valid', {'padding': 'valid'}),
+      ('padding_same', {'padding': 'same'}),
+      ('strides', {'strides': (2, 2, 2)}),
+      ('dilation_rate', {'dilation_rate': (2, 2, 2)}),
+      # Only runs on GPU with CUDA, channels_first is not supported on CPU.
+      # TODO(b/62340061): Support channels_first on CPU.
+      ('data_format', {'data_format': 'channels_first'}),
+  )
+  def test_conv3d(self, kwargs):
+    kwargs['filters'] = 2
+    kwargs['kernel_size'] = (3, 3, 3)
+    if 'data_format' not in kwargs or test.is_gpu_available(cuda_only=True):
+      self._run_test(kwargs)
 
   def test_conv3d_regularizers(self):
     kwargs = {
@@ -554,7 +200,7 @@ class Conv3DTest(keras_parameterized.TestCase):
         'activity_regularizer': 'l2',
         'strides': 1
     }
-    with self.session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       layer = keras.layers.Conv3D(**kwargs)
       layer.build((None, 5, 5, 5, 2))
       self.assertEqual(len(layer.losses), 2)
@@ -574,7 +220,7 @@ class Conv3DTest(keras_parameterized.TestCase):
         'bias_constraint': b_constraint,
         'strides': 1
     }
-    with self.session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       layer = keras.layers.Conv3D(**kwargs)
       layer.build((None, 5, 5, 5, 2))
       self.assertEqual(layer.kernel.constraint, k_constraint)
@@ -582,7 +228,7 @@ class Conv3DTest(keras_parameterized.TestCase):
 
   def test_conv3d_dynamic_shape(self):
     input_data = np.random.random((1, 3, 3, 3, 3)).astype(np.float32)
-    with self.session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       # Won't raise error here.
       testing_utils.layer_test(
           keras.layers.Conv3D,
@@ -615,7 +261,7 @@ class ZeroPaddingTest(keras_parameterized.TestCase):
     shape = (num_samples, num_steps, input_dim)
     inputs = np.ones(shape)
 
-    with self.session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       # basic test
       testing_utils.layer_test(
           keras.layers.ZeroPadding1D,
@@ -746,7 +392,7 @@ class ZeroPaddingTest(keras_parameterized.TestCase):
     inputs = np.ones((num_samples, input_len_dim1, input_len_dim2,
                       input_len_dim3, stack_size))
 
-    with self.session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       # basic test
       testing_utils.layer_test(
           keras.layers.ZeroPadding3D,
@@ -778,7 +424,7 @@ class ZeroPaddingTest(keras_parameterized.TestCase):
 class UpSamplingTest(keras_parameterized.TestCase):
 
   def test_upsampling_1d(self):
-    with self.session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       testing_utils.layer_test(
           keras.layers.UpSampling1D, kwargs={'size': 2}, input_shape=(3, 5, 4))
 
@@ -932,7 +578,7 @@ class CroppingTest(keras_parameterized.TestCase):
     input_len_dim1 = 2
     inputs = np.random.rand(num_samples, time_length, input_len_dim1)
 
-    with self.session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       testing_utils.layer_test(
           keras.layers.Cropping1D,
           kwargs={'cropping': (2, 2)},
@@ -1068,30 +714,34 @@ class CroppingTest(keras_parameterized.TestCase):
 @keras_parameterized.run_all_keras_modes
 class DepthwiseConv2DTest(keras_parameterized.TestCase):
 
-  def _run_test(self, kwargs, arg, values):
+  def _run_test(self, kwargs):
     num_samples = 2
     stack_size = 3
     num_row = 7
     num_col = 6
 
-    test_kwargs = copy.copy(kwargs)
-    for value in values:
-      test_kwargs[arg] = value
-      with self.cached_session(use_gpu=True):
-        testing_utils.layer_test(
-            keras.layers.DepthwiseConv2D,
-            kwargs=test_kwargs,
-            input_shape=(num_samples, num_row, num_col, stack_size))
-
-  def test_depthwise_conv2d(self):
-    kwargs = {'kernel_size': (3, 3)}
-
-    self._run_test(kwargs, 'padding', ['valid', 'same'])
-    self._run_test(kwargs, 'strides', [(2, 2)])
-    if test.is_gpu_available(cuda_only=True):
-      self._run_test(kwargs, 'data_format', ['channels_first'])
-    self._run_test(kwargs, 'depth_multiplier', [1, 2])
-
+    with self.cached_session(use_gpu=True):
+      testing_utils.layer_test(
+          keras.layers.DepthwiseConv2D,
+          kwargs=kwargs,
+          input_shape=(num_samples, num_row, num_col, stack_size))
+
+  @parameterized.named_parameters(
+      ('padding_valid', {'padding': 'valid'}),
+      ('padding_same', {'padding': 'same'}),
+      ('strides', {'strides': (2, 2)}),
+      # Only runs on GPU with CUDA, channels_first is not supported on CPU.
+      # TODO(b/62340061): Support channels_first on CPU.
+      ('data_format', {'data_format': 'channels_first'}),
+      ('depth_multiplier_1', {'depth_multiplier': 1}),
+      ('depth_multiplier_2', {'depth_multiplier': 2}),
+  )
+  def test_depthwise_conv2d(self, kwargs):
+    kwargs['kernel_size'] = (3, 3)
+    if 'data_format' not in kwargs or test.is_gpu_available(cuda_only=True):
+      self._run_test(kwargs)
+
+  def test_depthwise_conv2d_full(self):
     kwargs = {
         'kernel_size': 3,
         'padding': 'valid',
@@ -1103,8 +753,9 @@ class DepthwiseConv2DTest(keras_parameterized.TestCase):
         'depthwise_constraint': 'unit_norm',
         'use_bias': True,
         'strides': (2, 2),
+        'depth_multiplier': 1,
     }
-    self._run_test(kwargs, 'depth_multiplier', [1])
+    self._run_test(kwargs)
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/layers/convolutional_transpose_test.py b/tensorflow/python/keras/layers/convolutional_transpose_test.py
new file mode 100644
index 0000000000..dd73d22d51
--- /dev/null
+++ b/tensorflow/python/keras/layers/convolutional_transpose_test.py
@@ -0,0 +1,209 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for convolutional transpose layers."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+import numpy as np
+
+from tensorflow.python import keras
+from tensorflow.python.keras import keras_parameterized
+from tensorflow.python.keras import testing_utils
+from tensorflow.python.platform import test
+
+
+@keras_parameterized.run_all_keras_modes
+class Conv2DTransposeTest(keras_parameterized.TestCase):
+
+  def _run_test(self, kwargs):
+    num_samples = 2
+    stack_size = 3
+    num_row = 7
+    num_col = 6
+
+    with self.cached_session(use_gpu=True):
+      testing_utils.layer_test(
+          keras.layers.Conv2DTranspose,
+          kwargs=kwargs,
+          input_shape=(num_samples, num_row, num_col, stack_size))
+
+  @parameterized.named_parameters(
+      ('padding_valid', {'padding': 'valid'}),
+      ('padding_same', {'padding': 'same'}),
+      ('strides', {'strides': (2, 2)}),
+      # Only runs on GPU with CUDA, channels_first is not supported on CPU.
+      # TODO(b/62340061): Support channels_first on CPU.
+      ('data_format', {'data_format': 'channels_first'}),
+      ('strides_output_padding', {'strides': (2, 2), 'output_padding': (1, 1)}),
+  )
+  def test_conv2d_transpose(self, kwargs):
+    kwargs['filters'] = 2
+    kwargs['kernel_size'] = (3, 3)
+    if 'data_format' not in kwargs or test.is_gpu_available(cuda_only=True):
+      self._run_test(kwargs)
+
+  def test_conv2d_transpose_regularizers(self):
+    kwargs = {
+        'filters': 3,
+        'kernel_size': 3,
+        'padding': 'valid',
+        'kernel_regularizer': 'l2',
+        'bias_regularizer': 'l2',
+        'activity_regularizer': 'l2',
+        'strides': 1
+    }
+    with self.cached_session(use_gpu=True):
+      layer = keras.layers.Conv2DTranspose(**kwargs)
+      layer.build((None, 5, 5, 2))
+      self.assertEqual(len(layer.losses), 2)
+      layer(keras.backend.variable(np.ones((1, 5, 5, 2))))
+      self.assertEqual(len(layer.losses), 3)
+
+  def test_conv2d_transpose_constraints(self):
+    k_constraint = lambda x: x
+    b_constraint = lambda x: x
+
+    kwargs = {
+        'filters': 3,
+        'kernel_size': 3,
+        'padding': 'valid',
+        'kernel_constraint': k_constraint,
+        'bias_constraint': b_constraint,
+        'strides': 1
+    }
+    with self.cached_session(use_gpu=True):
+      layer = keras.layers.Conv2DTranspose(**kwargs)
+      layer.build((None, 5, 5, 2))
+      self.assertEqual(layer.kernel.constraint, k_constraint)
+      self.assertEqual(layer.bias.constraint, b_constraint)
+
+  def test_conv2d_transpose_dilation(self):
+    testing_utils.layer_test(keras.layers.Conv2DTranspose,
+                             kwargs={'filters': 2,
+                                     'kernel_size': 3,
+                                     'padding': 'same',
+                                     'data_format': 'channels_last',
+                                     'dilation_rate': (2, 2)},
+                             input_shape=(2, 5, 6, 3))
+
+    input_data = np.arange(48).reshape((1, 4, 4, 3)).astype(np.float32)
+    expected_output = np.float32([[192, 228, 192, 228],
+                                  [336, 372, 336, 372],
+                                  [192, 228, 192, 228],
+                                  [336, 372, 336, 372]]).reshape((1, 4, 4, 1))
+    testing_utils.layer_test(keras.layers.Conv2DTranspose,
+                             input_data=input_data,
+                             kwargs={'filters': 1,
+                                     'kernel_size': 3,
+                                     'padding': 'same',
+                                     'data_format': 'channels_last',
+                                     'dilation_rate': (2, 2),
+                                     'kernel_initializer': 'ones'},
+                             expected_output=expected_output)
+
+
+@keras_parameterized.run_all_keras_modes
+class Conv3DTransposeTest(keras_parameterized.TestCase):
+
+  def _run_test(self, kwargs):
+    num_samples = 2
+    stack_size = 3
+    num_row = 7
+    num_col = 6
+    depth = 5
+
+    with self.cached_session(use_gpu=True):
+      testing_utils.layer_test(
+          keras.layers.Conv3DTranspose,
+          kwargs=kwargs,
+          input_shape=(num_samples, depth, num_row, num_col, stack_size))
+
+  @parameterized.named_parameters(
+      ('padding_valid', {'padding': 'valid'}),
+      ('padding_same', {'padding': 'same'}),
+      ('strides', {'strides': (2, 2, 2)}),
+      # Only runs on GPU with CUDA, channels_first is not supported on CPU.
+      # TODO(b/62340061): Support channels_first on CPU.
+      ('data_format', {'data_format': 'channels_first'}),
+      ('strides_output_padding', {'strides': (2, 2, 2),
+                                  'output_padding': (1, 1, 1)}),
+  )
+  def test_conv3d_transpose(self, kwargs):
+    kwargs['filters'] = 2
+    kwargs['kernel_size'] = (3, 3, 3)
+    if 'data_format' not in kwargs or test.is_gpu_available(cuda_only=True):
+      self._run_test(kwargs)
+
+  def test_conv3d_transpose_regularizers(self):
+    kwargs = {
+        'filters': 3,
+        'kernel_size': 3,
+        'padding': 'valid',
+        'kernel_regularizer': 'l2',
+        'bias_regularizer': 'l2',
+        'activity_regularizer': 'l2',
+        'strides': 1
+    }
+    with self.cached_session(use_gpu=True):
+      layer = keras.layers.Conv3DTranspose(**kwargs)
+      layer.build((None, 5, 5, 5, 2))
+      self.assertEqual(len(layer.losses), 2)
+      layer(keras.backend.variable(np.ones((1, 5, 5, 5, 2))))
+      self.assertEqual(len(layer.losses), 3)
+
+  def test_conv3d_transpose_constraints(self):
+    k_constraint = lambda x: x
+    b_constraint = lambda x: x
+
+    kwargs = {
+        'filters': 3,
+        'kernel_size': 3,
+        'padding': 'valid',
+        'kernel_constraint': k_constraint,
+        'bias_constraint': b_constraint,
+        'strides': 1
+    }
+    with self.cached_session(use_gpu=True):
+      layer = keras.layers.Conv3DTranspose(**kwargs)
+      layer.build((None, 5, 5, 5, 2))
+      self.assertEqual(layer.kernel.constraint, k_constraint)
+      self.assertEqual(layer.bias.constraint, b_constraint)
+
+  def test_conv3d_transpose_dynamic_shape(self):
+    input_data = np.random.random((1, 3, 3, 3, 3)).astype(np.float32)
+    with self.cached_session(use_gpu=True):
+      # Won't raise error here.
+      testing_utils.layer_test(
+          keras.layers.Conv3DTranspose,
+          kwargs={
+              'data_format': 'channels_last',
+              'filters': 3,
+              'kernel_size': 3
+          },
+          input_shape=(None, None, None, None, 3),
+          input_data=input_data)
+      if test.is_gpu_available(cuda_only=True):
+        testing_utils.layer_test(
+            keras.layers.Conv3DTranspose,
+            kwargs={
+                'data_format': 'channels_first',
+                'filters': 3,
+                'kernel_size': 3
+            },
+            input_shape=(None, 3, None, None, None),
+            input_data=input_data)
diff --git a/tensorflow/python/keras/layers/separable_convolutional_test.py b/tensorflow/python/keras/layers/separable_convolutional_test.py
new file mode 100644
index 0000000000..8234bfe704
--- /dev/null
+++ b/tensorflow/python/keras/layers/separable_convolutional_test.py
@@ -0,0 +1,167 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for separable convolutional layers."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+import numpy as np
+
+from tensorflow.python import keras
+from tensorflow.python.keras import keras_parameterized
+from tensorflow.python.keras import testing_utils
+from tensorflow.python.platform import test
+
+
+@keras_parameterized.run_all_keras_modes
+class SeparableConv1DTest(keras_parameterized.TestCase):
+
+  def _run_test(self, kwargs):
+    num_samples = 2
+    stack_size = 3
+    length = 7
+
+    with self.cached_session(use_gpu=True):
+      testing_utils.layer_test(
+          keras.layers.SeparableConv1D,
+          kwargs=kwargs,
+          input_shape=(num_samples, length, stack_size))
+
+  @parameterized.named_parameters(
+      ('padding_valid', {'padding': 'valid'}),
+      ('padding_same', {'padding': 'same'}),
+      ('padding_same_dilation_2', {'padding': 'same', 'dilation_rate': 2}),
+      ('padding_causal', {'padding': 'causal'}),
+      ('strides', {'strides': 2}),
+      ('dilation_rate', {'dilation_rate': 2}),
+      ('depth_multiplier', {'depth_multiplier': 2}),
+  )
+  def test_separable_conv1d(self, kwargs):
+    kwargs['filters'] = 2
+    kwargs['kernel_size'] = 3
+    self._run_test(kwargs)
+
+  def test_separable_conv1d_regularizers(self):
+    kwargs = {
+        'filters': 3,
+        'kernel_size': 3,
+        'padding': 'valid',
+        'depthwise_regularizer': 'l2',
+        'pointwise_regularizer': 'l2',
+        'bias_regularizer': 'l2',
+        'activity_regularizer': 'l2',
+        'strides': 1
+    }
+    with self.cached_session(use_gpu=True):
+      layer = keras.layers.SeparableConv1D(**kwargs)
+      layer.build((None, 5, 2))
+      self.assertEqual(len(layer.losses), 3)
+      layer(keras.backend.variable(np.ones((1, 5, 2))))
+      self.assertEqual(len(layer.losses), 4)
+
+  def test_separable_conv1d_constraints(self):
+    d_constraint = lambda x: x
+    p_constraint = lambda x: x
+    b_constraint = lambda x: x
+
+    kwargs = {
+        'filters': 3,
+        'kernel_size': 3,
+        'padding': 'valid',
+        'pointwise_constraint': p_constraint,
+        'depthwise_constraint': d_constraint,
+        'bias_constraint': b_constraint,
+        'strides': 1
+    }
+    with self.cached_session(use_gpu=True):
+      layer = keras.layers.SeparableConv1D(**kwargs)
+      layer.build((None, 5, 2))
+      self.assertEqual(layer.depthwise_kernel.constraint, d_constraint)
+      self.assertEqual(layer.pointwise_kernel.constraint, p_constraint)
+      self.assertEqual(layer.bias.constraint, b_constraint)
+
+
+@keras_parameterized.run_all_keras_modes
+class SeparableConv2DTest(keras_parameterized.TestCase):
+
+  def _run_test(self, kwargs):
+    num_samples = 2
+    stack_size = 3
+    num_row = 7
+    num_col = 6
+
+    with self.cached_session(use_gpu=True):
+      testing_utils.layer_test(
+          keras.layers.SeparableConv2D,
+          kwargs=kwargs,
+          input_shape=(num_samples, num_row, num_col, stack_size))
+
+  @parameterized.named_parameters(
+      ('padding_valid', {'padding': 'valid'}),
+      ('padding_same', {'padding': 'same'}),
+      ('padding_same_dilation_2', {'padding': 'same', 'dilation_rate': 2}),
+      ('strides', {'strides': 2}),
+      # Only runs on GPU with CUDA, channels_first is not supported on CPU.
+      # TODO(b/62340061): Support channels_first on CPU.
+      ('data_format', {'data_format': 'channels_first'}),
+      ('dilation_rate', {'dilation_rate': 2}),
+      ('depth_multiplier', {'depth_multiplier': 2}),
+  )
+  def test_separable_conv2d(self, kwargs):
+    kwargs['filters'] = 2
+    kwargs['kernel_size'] = 3
+    if 'data_format' not in kwargs or test.is_gpu_available(cuda_only=True):
+      self._run_test(kwargs)
+
+  def test_separable_conv2d_regularizers(self):
+    kwargs = {
+        'filters': 3,
+        'kernel_size': 3,
+        'padding': 'valid',
+        'depthwise_regularizer': 'l2',
+        'pointwise_regularizer': 'l2',
+        'bias_regularizer': 'l2',
+        'activity_regularizer': 'l2',
+        'strides': 1
+    }
+    with self.cached_session(use_gpu=True):
+      layer = keras.layers.SeparableConv2D(**kwargs)
+      layer.build((None, 5, 5, 2))
+      self.assertEqual(len(layer.losses), 3)
+      layer(keras.backend.variable(np.ones((1, 5, 5, 2))))
+      self.assertEqual(len(layer.losses), 4)
+
+  def test_separable_conv2d_constraints(self):
+    d_constraint = lambda x: x
+    p_constraint = lambda x: x
+    b_constraint = lambda x: x
+
+    kwargs = {
+        'filters': 3,
+        'kernel_size': 3,
+        'padding': 'valid',
+        'pointwise_constraint': p_constraint,
+        'depthwise_constraint': d_constraint,
+        'bias_constraint': b_constraint,
+        'strides': 1
+    }
+    with self.cached_session(use_gpu=True):
+      layer = keras.layers.SeparableConv2D(**kwargs)
+      layer.build((None, 5, 5, 2))
+      self.assertEqual(layer.depthwise_kernel.constraint, d_constraint)
+      self.assertEqual(layer.pointwise_kernel.constraint, p_constraint)
+      self.assertEqual(layer.bias.constraint, b_constraint)
-- 
GitLab


From 8fcb0f87ffe41b072ed8ed893254d293e3a53b93 Mon Sep 17 00:00:00 2001
From: Smit Hinsu <hinsu@google.com>
Date: Thu, 10 Jan 2019 23:17:26 -0800
Subject: [PATCH 0545/2345] Automated rollback of commit
 68e869eb40ff0acd03515336bf31eecbabf97adc

PiperOrigin-RevId: 228836014
---
 tensorflow/contrib/tensorrt/BUILD | 20 +-------------------
 1 file changed, 1 insertion(+), 19 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 3d34b91a74..003a7e0b0a 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -500,6 +500,7 @@ cuda_py_tests(
         "test/quantization_test.py",
         "test/rank_two_test.py",
         "test/reshape_transpose_test.py",
+        "test/unary_test.py",
         "test/vgg_block_nchw_test.py",
         "test/vgg_block_test.py",
     ],
@@ -515,25 +516,6 @@ cuda_py_tests(
     ],
 )
 
-cuda_py_tests(
-    name = "tf_trt_integration_test_no_oss",
-    srcs = [
-        "test/unary_test.py",
-    ],
-    additional_deps = [
-        ":tf_trt_integration_test_base",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_test_lib",
-    ],
-    tags = [
-        "no_cuda_on_cpu_tap",
-        "no_oss",  # TODO(b/117274186): re-enable in OSS after crash fixed
-        "no_pip",  # TODO(b/117274186): re-enable in OSS after crash fixed
-        "no_windows",
-        "nomac",
-    ],
-)
-
 cuda_py_test(
     name = "quantization_mnist_test",
     srcs = ["test/quantization_mnist_test.py"],
-- 
GitLab


From 4bfe33c0f94c75a72b81242d31daad104d8d7b77 Mon Sep 17 00:00:00 2001
From: Dayananda-V <dayanandasiet@gmail.com>
Date: Fri, 11 Jan 2019 13:08:46 +0530
Subject: [PATCH 0546/2345] TF Keras conv_utils missing test cases add

Certain basic unit test cases add in conv_utils(//tensorflow/python/keras:conv_utils_test)
---
 .../python/keras/utils/conv_utils_test.py     | 112 ++++++++++++++++++
 1 file changed, 112 insertions(+)

diff --git a/tensorflow/python/keras/utils/conv_utils_test.py b/tensorflow/python/keras/utils/conv_utils_test.py
index eb2a360bfd..75dc718b70 100644
--- a/tensorflow/python/keras/utils/conv_utils_test.py
+++ b/tensorflow/python/keras/utils/conv_utils_test.py
@@ -52,6 +52,118 @@ input_shapes = [
 ]
 
 
+class TestBasicConvUtilsTest(test.TestCase):
+  def test_convert_data_format(self):
+    self.assertEqual('NCDHW',
+                     conv_utils.convert_data_format('channels_first', 5))
+    self.assertEqual('NCHW',
+                     conv_utils.convert_data_format('channels_first', 4))
+    self.assertEqual('NCW',
+                     conv_utils.convert_data_format('channels_first', 3))
+    self.assertEqual('NHWC',
+                     conv_utils.convert_data_format('channels_last', 4))
+    self.assertEqual('NWC',
+                     conv_utils.convert_data_format('channels_last', 3))
+    self.assertEqual('NDHWC',
+                     conv_utils.convert_data_format('channels_last', 5))
+
+    with self.assertRaises(ValueError):
+      conv_utils.convert_data_format('invalid', 2)
+
+  def test_normalize_tuple(self):
+    self.assertEqual((2, 2, 2),
+                     conv_utils.normalize_tuple(2, n=3, name='strides'))
+    self.assertEqual((2, 1, 2),
+                     conv_utils.normalize_tuple((2, 1, 2), n=3, name='strides'))
+
+    with self.assertRaises(ValueError):
+      conv_utils.normalize_tuple((2, 1), n=3, name='strides')
+
+    with self.assertRaises(ValueError):
+      conv_utils.normalize_tuple(None, n=3, name='strides')
+
+  def test_normalize_data_format(self):
+    self.assertEqual(
+        'channels_last', conv_utils.normalize_data_format('Channels_Last'))
+    self.assertEqual(
+        'channels_first', conv_utils.normalize_data_format('CHANNELS_FIRST'))
+
+    with self.assertRaises(ValueError):
+      conv_utils.normalize_data_format('invalid')
+
+  def test_normalize_padding(self):
+    self.assertEqual('same', conv_utils.normalize_padding('SAME'))
+    self.assertEqual('valid', conv_utils.normalize_padding('VALID'))
+
+    with self.assertRaises(ValueError):
+      conv_utils.normalize_padding('invalid')
+
+  def test_conv_output_length(self):
+    self.assertEqual(4, conv_utils.conv_output_length(4, 2, 'same', 1, 1))
+    self.assertEqual(2, conv_utils.conv_output_length(4, 2, 'same', 2, 1))
+    self.assertEqual(3, conv_utils.conv_output_length(4, 2, 'valid', 1, 1))
+    self.assertEqual(2, conv_utils.conv_output_length(4, 2, 'valid', 2, 1))
+    self.assertEqual(5, conv_utils.conv_output_length(4, 2, 'full', 1, 1))
+    self.assertEqual(3, conv_utils.conv_output_length(4, 2, 'full', 2, 1))
+    self.assertEqual(2, conv_utils.conv_output_length(5, 2, 'valid', 2, 2))
+
+  def test_conv_input_length(self):
+    self.assertEqual(3, conv_utils.conv_input_length(4, 2, 'same', 1))
+    self.assertEqual(2, conv_utils.conv_input_length(2, 2, 'same', 2))
+    self.assertEqual(4, conv_utils.conv_input_length(3, 2, 'valid', 1))
+    self.assertEqual(4, conv_utils.conv_input_length(2, 2, 'valid', 2))
+    self.assertEqual(3, conv_utils.conv_input_length(4, 2, 'full', 1))
+    self.assertEqual(4, conv_utils.conv_input_length(3, 2, 'full', 2))
+
+  def test_deconv_output_length(self):
+    self.assertEqual(4, conv_utils.deconv_output_length(4, 2, 'same', stride=1))
+    self.assertEqual(8, conv_utils.deconv_output_length(4, 2, 'same', stride=2))
+    self.assertEqual(5,
+                     conv_utils.deconv_output_length(4, 2, 'valid', stride=1))
+    self.assertEqual(8,
+                     conv_utils.deconv_output_length(4, 2, 'valid', stride=2))
+    self.assertEqual(3, conv_utils.deconv_output_length(4, 2, 'full', stride=1))
+    self.assertEqual(6, conv_utils.deconv_output_length(4, 2, 'full', stride=2))
+    self.assertEqual(5,
+                     conv_utils.deconv_output_length(4, 2, 'same',
+                                                     output_padding=2,
+                                                     stride=1))
+    self.assertEqual(7,
+                     conv_utils.deconv_output_length(4, 2, 'same',
+                                                     output_padding=1,
+                                                     stride=2))
+    self.assertEqual(7,
+                     conv_utils.deconv_output_length(4, 2, 'valid',
+                                                     output_padding=2,
+                                                     stride=1))
+    self.assertEqual(9,
+                     conv_utils.deconv_output_length(4, 2, 'valid',
+                                                     output_padding=1,
+                                                     stride=2))
+    self.assertEqual(5,
+                     conv_utils.deconv_output_length(4, 2, 'full',
+                                                     output_padding=2,
+                                                     stride=1))
+    self.assertEqual(7,
+                     conv_utils.deconv_output_length(4, 2, 'full',
+                                                     output_padding=1,
+                                                     stride=2))
+    self.assertEqual(5,
+                     conv_utils.deconv_output_length(4, 2, 'same',
+                                                     output_padding=1,
+                                                     stride=1,
+                                                     dilation=2))
+    self.assertEqual(12,
+                     conv_utils.deconv_output_length(4, 2, 'valid',
+                                                     output_padding=2,
+                                                     stride=2,
+                                                     dilation=3))
+    self.assertEqual(6,
+                     conv_utils.deconv_output_length(4, 2, 'full',
+                                                     output_padding=2,
+                                                     stride=2,
+                                                     dilation=3))
+
 @parameterized.parameters(input_shapes)
 class TestConvUtils(test.TestCase, parameterized.TestCase):
 
-- 
GitLab


From cfe3f839c852bb590f7b349376a2c3bfeb0a031f Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Fri, 11 Jan 2019 00:04:54 -0800
Subject: [PATCH 0547/2345] Update Eigen to commit ad3bcd81cc49.

PiperOrigin-RevId: 228839247
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 06caa200e4..ca4feb691e 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -138,11 +138,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "eigen_archive",
         build_file = clean_dep("//third_party:eigen.BUILD"),
-        sha256 = "9de38f2d162c51599b802f7c36d9f3773980d19ac908c61638f8344d2c10e1ca",
-        strip_prefix = "eigen-eigen-88fc23324517",
+        sha256 = "df54d805573871d03aaef6d27d45439c18ff6e8db215a3b922b2daefd6914147",
+        strip_prefix = "eigen-eigen-ad3bcd81cc49",
         urls = [
-            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/88fc23324517..tar.gz",
-            "https://bitbucket.org/eigen/eigen/get/88fc23324517.tar.gz",
+            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/ad3bcd81cc49..tar.gz",
+            "https://bitbucket.org/eigen/eigen/get/ad3bcd81cc49.tar.gz",
         ],
     )
 
-- 
GitLab


From ac345af940a5bfc2d205aa14001450b10ee01c16 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 01:02:23 -0800
Subject: [PATCH 0548/2345] compat: Update forward compatibility horizon to
 2019-01-11

PiperOrigin-RevId: 228844118
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 638bd445a6..4028302362 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 10)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 11)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From bac29e41394d029e94659b6d1df833237056b764 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 01:04:23 -0800
Subject: [PATCH 0549/2345] Added simple nested structure serialization on
 input.

PiperOrigin-RevId: 228844725
---
 tensorflow/python/eager/function.py           | 30 +++++++++-------
 .../saved_model/function_deserialization.py   | 35 +++++++++++--------
 .../saved_model/function_serialization.py     |  5 ++-
 tensorflow/python/saved_model/load_test.py    | 33 +++++++++++++++++
 4 files changed, 74 insertions(+), 29 deletions(-)

diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 83cd140158..9a604620fa 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -814,17 +814,22 @@ class UnknownArgument(object):
   pass
 
 
-def _encode_arg_for_serialization(arg):
-  """A representation for this argument, for serializing signatures."""
-  if isinstance(arg, ops.Tensor):
-    return tensor_spec.TensorSpec(arg.shape, arg.dtype)
-  if isinstance(arg, int):
-    return arg
-  if isinstance(arg, float):
-    return arg
-  if isinstance(arg, bool):
-    return arg
-  return UnknownArgument()
+def _convert_inputs_to_signature(inputs):
+  """Convert a list of potentially nested inputs to a signature."""
+
+  def encode_arg(arg, name=None):
+    """A representation for this argument, for converting into signatures."""
+    if isinstance(arg, ops.Tensor):
+      return tensor_spec.TensorSpec(arg.shape, arg.dtype, name)
+    if isinstance(arg, (int, float, bool)):
+      return arg
+    return UnknownArgument()
+
+  # We are using the flattened paths to name the TensorSpecs. We need an
+  # explicit name for them downstream.
+  flattened_with_paths = nest.flatten_with_joined_string_paths(inputs)
+  mapped = [encode_arg(arg, path) for path, arg in flattened_with_paths]
+  return nest.pack_sequence_as(inputs, mapped)
 
 
 pywrap_tensorflow.RegisterType("Tensor", ops.Tensor)
@@ -1305,8 +1310,7 @@ class PolymorphicFunction(object):
         if self._input_signature:
           python_call_signature = self._input_signature
         else:
-          python_call_signature = tuple(
-              _encode_arg_for_serialization(arg) for arg in args)
+          python_call_signature = tuple(_convert_inputs_to_signature(args))
         # pylint: disable=protected-access
         # Save information about non-Tensor arguments with the concrete
         # function. Used to serialize PolymorphicFunctions.
diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py
index dc839cf61c..3cf27fd5a4 100644
--- a/tensorflow/python/saved_model/function_deserialization.py
+++ b/tensorflow/python/saved_model/function_deserialization.py
@@ -38,19 +38,25 @@ def _is_tensor(t):
 
 def _inputs_compatible(args, stored_inputs):
   """Checks whether function arguments are compatible with parameters."""
-  # TODO(vbardiovsky): The compatibility check should be about the signature,
-  # not the flattened version of it.
   if len(args) != len(stored_inputs):
     return False
-  for a, b in zip(args, stored_inputs):
-    if _is_tensor(a):
-      if not isinstance(b, tensor_spec.TensorSpec):
-        return False
-      if a.dtype != b.dtype or not b.shape.is_compatible_with(a.shape):
-        return False
-    else:
-      if a != b:
-        return False
+
+  for arg, stored_input in zip(args, stored_inputs):
+    if not function_lib.is_same_structure(arg, stored_input):
+      return False
+
+    flattened_arg = nest.flatten(arg)
+    flattened_stored_input = nest.flatten(stored_input)
+
+    for a, b in zip(flattened_arg, flattened_stored_input):
+      if _is_tensor(a):
+        if not isinstance(b, tensor_spec.TensorSpec):
+          return False
+        if a.dtype != b.dtype or not b.shape.is_compatible_with(a.shape):
+          return False
+      else:
+        if a != b:
+          return False
   return True
 
 
@@ -98,11 +104,10 @@ def recreate_polymorphic_function(
       except ValueError:
         continue
 
-      canonicalized_inputs = nest.flatten(can_args)
-
-      if _inputs_compatible(canonicalized_inputs,
+      if _inputs_compatible(can_args,
                             canonicalized_original_inputs):
-        filtered_inputs = [t for t in canonicalized_inputs if _is_tensor(t)]
+        flattened_inputs = nest.flatten(can_args)
+        filtered_inputs = [t for t in flattened_inputs if _is_tensor(t)]
         flattened_outputs = function_obj._call_flat(filtered_inputs)  # pylint: disable=protected-access
         # TODO(vbardiovsky): Rebuild output structure.
         single_output, = flattened_outputs
diff --git a/tensorflow/python/saved_model/function_serialization.py b/tensorflow/python/saved_model/function_serialization.py
index 014a4ef04c..c826ffa02c 100644
--- a/tensorflow/python/saved_model/function_serialization.py
+++ b/tensorflow/python/saved_model/function_serialization.py
@@ -22,6 +22,7 @@ from tensorflow.python.eager import function as defun_lib
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.saved_model import nested_structure_coder
 from tensorflow.python.saved_model import saved_object_graph_pb2
+from tensorflow.python.util import nest
 
 
 def serialize_polymorphic_function(polymorphic_function, node_ids):
@@ -67,7 +68,9 @@ def list_all_concrete_functions(polymorphic_function):
     polymorphic_function.get_concrete_function()
   concrete_functions = []
   for signature in polymorphic_function._cached_input_signatures:  # pylint: disable=protected-access
-    if any(isinstance(arg, defun_lib.UnknownArgument) for arg in signature):
+    flattened = nest.flatten(signature)
+    if any(isinstance(arg, defun_lib.UnknownArgument) for arg in flattened):
+      logging.info("Unsupported signature for serialization: %s.", signature)
       continue
     concrete_function = polymorphic_function.get_concrete_function(*signature)
     concrete_functions.append((signature, concrete_function))
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index 8b34414d25..32bb841556 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -185,6 +185,39 @@ class LoadTest(test.TestCase):
     self.assertEqual(4, imported.f(constant_op.constant(2), True).numpy())
     self.assertEqual(7, imported.f(constant_op.constant(2)).numpy())
 
+  def test_structured_inputs(self):
+
+    def func(x, training=True):
+      # x is a nested structure, we care about one particular tensor.
+      _, (a, b) = x
+      if training:
+        return 2 * a["a"] + b
+      else:
+        return 7
+
+    root = tracking.Checkpointable()
+    root.f = def_function.function(func)
+
+    x = constant_op.constant(10)
+    y = constant_op.constant(11)
+
+    input1 = [6, ({"a": x}, y)]
+    input2 = [7, ({"a": x}, y)]  # Not compatible with input1 signature.
+    input3 = [6, ({"a": y}, x)]  # Compatible with input1 signature.
+
+    # Note: by only calling f(input1) before serialization, only inputs with
+    # matching signature will be valid on the loaded model.
+    self.assertEqual(31, root.f(input1).numpy())
+
+    imported = self.cycle(root)
+
+    with self.assertRaisesRegexp(AssertionError,
+                                 "Could not find matching function to call.*"):
+      imported.f(input2)
+
+    self.assertEqual(31, imported.f(input1).numpy())
+    self.assertEqual(32, imported.f(input3).numpy())
+
   def test_positional_arguments(self):
     def func(x, training=False, abc=7.1, defg=7.7):
       del abc
-- 
GitLab


From c2ab6dc7a19cb121d22f36e03107315aeb5e3513 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Fri, 11 Jan 2019 01:31:06 -0800
Subject: [PATCH 0550/2345] Add default comparators which can be used for
 sorting.

For sorting, we will allow soon to specify comparator computations.
This adds default comparators which implement the current comparison
behavior. For floating point values special logic is used to create a total
order.

PiperOrigin-RevId: 228847705
---
 tensorflow/compiler/xla/client/lib/BUILD      |  33 ++++
 .../compiler/xla/client/lib/comparators.cc    | 159 ++++++++++++++++++
 .../compiler/xla/client/lib/comparators.h     |  47 ++++++
 .../xla/client/lib/comparators_test.cc        | 150 +++++++++++++++++
 tensorflow/compiler/xla/literal_util.cc       |   7 +-
 5 files changed, 393 insertions(+), 3 deletions(-)
 create mode 100644 tensorflow/compiler/xla/client/lib/comparators.cc
 create mode 100644 tensorflow/compiler/xla/client/lib/comparators.h
 create mode 100644 tensorflow/compiler/xla/client/lib/comparators_test.cc

diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD
index df1ee330f1..27f9c61848 100644
--- a/tensorflow/compiler/xla/client/lib/BUILD
+++ b/tensorflow/compiler/xla/client/lib/BUILD
@@ -91,6 +91,39 @@ xla_test(
     ],
 )
 
+cc_library(
+    name = "comparators",
+    srcs = ["comparators.cc"],
+    hdrs = ["comparators.h"],
+    deps = [
+        ":constants",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:types",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/client:xla_builder",
+        "//tensorflow/compiler/xla/client:xla_computation",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:span",
+    ],
+)
+
+xla_test(
+    name = "comparators_test",
+    srcs = ["comparators_test.cc"],
+    deps = [
+        ":comparators",
+        ":constants",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:test",
+        "//tensorflow/compiler/xla:types",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/client:xla_builder",
+        "//tensorflow/compiler/xla/tests:client_library_test_base",
+        "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+        "@com_google_absl//absl/container:inlined_vector",
+    ],
+)
+
 cc_library(
     name = "constants",
     srcs = ["constants.cc"],
diff --git a/tensorflow/compiler/xla/client/lib/comparators.cc b/tensorflow/compiler/xla/client/lib/comparators.cc
new file mode 100644
index 0000000000..c620c9841a
--- /dev/null
+++ b/tensorflow/compiler/xla/client/lib/comparators.cc
@@ -0,0 +1,159 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/client/lib/comparators.h"
+
+#include <limits>
+#include <string>
+#include <vector>
+
+#include "absl/strings/str_cat.h"
+#include "absl/types/span.h"
+#include "tensorflow/compiler/xla/client/lib/constants.h"
+#include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/compiler/xla/client/xla_computation.h"
+#include "tensorflow/compiler/xla/primitive_util.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/types.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+
+namespace xla {
+namespace {
+
+using XlaOpGenerator = XlaOp (*)(const XlaOp&, const XlaOp&,
+                                 absl::Span<const int64>);
+
+XlaOp BitcastConvertFloatingPointToIntegral(const XlaOp& value,
+                                            int64 bit_width) {
+  PrimitiveType signed_type;
+  PrimitiveType unsigned_type;
+  XlaOp max_value;
+  switch (bit_width) {
+    case 16:
+      max_value =
+          ConstantR0(value.builder(),
+                     static_cast<uint16>(std::numeric_limits<int16>::max()));
+      signed_type = S16;
+      unsigned_type = U16;
+      break;
+    case 32:
+      max_value =
+          ConstantR0(value.builder(),
+                     static_cast<uint32>(std::numeric_limits<int32>::max()));
+      signed_type = S32;
+      unsigned_type = U32;
+      break;
+    case 64:
+      max_value =
+          ConstantR0(value.builder(),
+                     static_cast<uint64>(std::numeric_limits<int64>::max()));
+      signed_type = S64;
+      unsigned_type = U64;
+      break;
+    default:
+      return value.builder()->ReportError(
+          InvalidArgument("Invalid bit width %lld for Comparator floating "
+                          "point parameter.",
+                          bit_width));
+  }
+  // Switch from a floating point value to a integer value in such a way that
+  // when using the integer value to compare, we get the same result for normal
+  // values, and -Nan is treated as the smallest value, and Nan is treated as
+  // the largest value.
+  // If f is a float, and
+  // x = bit_cast<int32>(f);
+  // y = x < 0 ? numeric_limits<int32>::max() - x : x;
+  // then y is ordered as an int32 such that finite values have the obvious
+  // order, -0 is ordered before 0, and -NaN and NaN appear at the beginning
+  // and end of the ordering.
+  // Note that in order to avoid -x to overflow, we calculate
+  // numeric_limits<int32>::max() - x as unsigned, and then convert back to
+  // signed.
+  auto signed_value = BitcastConvertType(value, signed_type);
+  auto unsigned_value = BitcastConvertType(value, unsigned_type);
+  auto flipped_value =
+      BitcastConvertType(Sub(max_value, unsigned_value), signed_type);
+  auto is_negative = Lt(signed_value, Zero(value.builder(), signed_type));
+  return Select(is_negative, flipped_value, signed_value);
+}
+
+XlaComputation CreateScalarComparisonComputation(
+    const string& name, const std::vector<PrimitiveType>& operand_types,
+    XlaBuilder* builder, XlaOpGenerator generator) {
+  // Create a default computation where we compare only the first two
+  // parameters of type 'operand_types[0]'.
+  auto b = builder->CreateSubBuilder(name);
+  if (operand_types.empty()) {
+    b->ReportError(InvalidArgument("operand_types should not be empty"));
+    return b->BuildAndNoteError();
+  }
+
+  int64 parameter_count = 0;
+  XlaOp first_lhs_param;
+  XlaOp first_rhs_param;
+
+  // For each type in 'operand_types' we create two parameters of this type. The
+  // idea is that this computation can be used by n-ary Sort, and potentially
+  // should support comparing also the other operands of sort. In this default
+  // computation, however, we will not actually use any parameters except the
+  // first two.
+  for (auto operand_type : operand_types) {
+    auto scalar_shape = ShapeUtil::MakeShape(operand_type, {});
+    auto lhs_param = Parameter(b.get(), parameter_count * 2, scalar_shape,
+                               absl::StrCat("p.", parameter_count, ".lhs"));
+    auto rhs_param = Parameter(b.get(), parameter_count * 2 + 1, scalar_shape,
+                               absl::StrCat("p.", parameter_count, ".rhs"));
+    if (parameter_count == 0) {
+      first_lhs_param = lhs_param;
+      first_rhs_param = rhs_param;
+    }
+    ++parameter_count;
+  }
+  if (primitive_util::IsFloatingPointType(operand_types[0])) {
+    PrimitiveType compare_type = operand_types[0];
+    // Special-case handling for BF16. We currently do not support direct
+    // comparisons with BF16, so we convert to F32 and then use the F32
+    // comparison logic.
+    if (compare_type == BF16) {
+      compare_type = F32;
+      first_lhs_param = ConvertElementType(first_lhs_param, F32);
+      first_rhs_param = ConvertElementType(first_rhs_param, F32);
+    }
+    int64 bit_width = primitive_util::BitWidth(compare_type);
+    first_lhs_param =
+        BitcastConvertFloatingPointToIntegral(first_lhs_param, bit_width);
+    first_rhs_param =
+        BitcastConvertFloatingPointToIntegral(first_rhs_param, bit_width);
+  }
+  generator(first_lhs_param, first_rhs_param, {});
+  return b->BuildAndNoteError();
+}
+}  // namespace
+
+// Creates a scalar less-than computation and returns it.
+XlaComputation CreateScalarLtComputation(
+    const std::vector<PrimitiveType>& operand_types, XlaBuilder* builder) {
+  return CreateScalarComparisonComputation("compare-less-than", operand_types,
+                                           builder, Lt);
+}
+
+// Creates a scalar greater-than computation and returns it.
+XlaComputation CreateScalarGtComputation(
+    const std::vector<PrimitiveType>& operand_types, XlaBuilder* builder) {
+  return CreateScalarComparisonComputation("compare-greater-than",
+                                           operand_types, builder, Gt);
+}
+
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/client/lib/comparators.h b/tensorflow/compiler/xla/client/lib/comparators.h
new file mode 100644
index 0000000000..cbcfc227dd
--- /dev/null
+++ b/tensorflow/compiler/xla/client/lib/comparators.h
@@ -0,0 +1,47 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_CLIENT_LIB_COMPARATORS_H_
+#define TENSORFLOW_COMPILER_XLA_CLIENT_LIB_COMPARATORS_H_
+
+#include <vector>
+
+#include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/compiler/xla/client/xla_computation.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+
+namespace xla {
+
+// Creates a scalar less-than computation and returns it. The created
+// computation has 2 * 'operand_types.size()' many parameters, where parameters
+// 2 * i and 2 * i + 1 are a scalar with primitive type 'operand_types[i]'. The
+// computation compares the first two parameters. For floating point types, a
+// total order is created where
+// -NaN < -infinity < ... < -0 < 0 < ... < infinity < NaN
+XlaComputation CreateScalarLtComputation(
+    const std::vector<PrimitiveType>& operand_types, XlaBuilder* builder);
+
+// Creates a scalar greater-than computation and returns it. The created
+// computation has 2 * 'operand_types.size()' many parameters, where parameters
+// 2 * i and 2 * i + 1 are a scalar with primitive type 'operand_types[i]'. The
+// computation compares the first two parameters. For floating point types, a
+// total order is created where
+// NaN > infinity > ... > 0 > -0 > ... > -infinity > -NaN
+XlaComputation CreateScalarGtComputation(
+    const std::vector<PrimitiveType>& operand_types, XlaBuilder* builder);
+
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_CLIENT_LIB_COMPARATORS_H_
diff --git a/tensorflow/compiler/xla/client/lib/comparators_test.cc b/tensorflow/compiler/xla/client/lib/comparators_test.cc
new file mode 100644
index 0000000000..efec984eb2
--- /dev/null
+++ b/tensorflow/compiler/xla/client/lib/comparators_test.cc
@@ -0,0 +1,150 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/client/lib/comparators.h"
+
+#include <limits>
+#include <vector>
+
+#include "absl/container/inlined_vector.h"
+#include "tensorflow/compiler/xla/client/lib/constants.h"
+#include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/compiler/xla/primitive_util.h"
+#include "tensorflow/compiler/xla/test.h"
+#include "tensorflow/compiler/xla/tests/client_library_test_base.h"
+#include "tensorflow/compiler/xla/tests/test_macros.h"
+#include "tensorflow/compiler/xla/types.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+
+namespace xla {
+namespace {
+
+class ComparatorsTest : public ClientLibraryTestBase {
+ public:
+  ComparatorsTest() : builder_(TestName()) {}
+  XlaBuilder* builder() { return &builder_; }
+
+ private:
+  XlaBuilder builder_;
+};
+
+template <
+    PrimitiveType type,
+    typename T = typename primitive_util::PrimitiveTypeToNative<type>::type>
+void BuildComparatorAndComparisons(ComparatorsTest* test,
+                                   bool compare_less_than,
+                                   absl::InlinedVector<bool, 10>* expected) {
+  auto compare = compare_less_than
+                     ? CreateScalarLtComputation({type}, test->builder())
+                     : CreateScalarGtComputation({type}, test->builder());
+
+  auto negative_nan = ConstantR0<T>(
+      test->builder(), -T(std::numeric_limits<float>::quiet_NaN()));
+  auto positive_nan = ConstantR0<T>(test->builder(),
+                                    T(std::numeric_limits<float>::quiet_NaN()));
+  auto negative_zero = ConstantR0<T>(test->builder(), T(-0.));
+  auto positive_zero = ConstantR0<T>(test->builder(), T(0.));
+  auto negative_infinity = MinValue(test->builder(), type);
+  auto positive_infinity = MaxValue(test->builder(), type);
+
+  // List the values in the expected sorting order from smallest to largest.
+  std::vector<XlaOp> all_constants{negative_nan,      negative_infinity,
+                                   negative_zero,     positive_zero,
+                                   positive_infinity, positive_nan};
+
+  // Do pairwise comparisons.
+  std::vector<XlaOp> all_comparisons;
+  for (const XlaOp& lhs_constant : all_constants) {
+    for (const XlaOp& rhs_constant : all_constants) {
+      all_comparisons.push_back(Broadcast(
+          Call(test->builder(), compare, {lhs_constant, rhs_constant}), {1}));
+    }
+  }
+
+  // Concantenate the comparison results.
+  ConcatInDim(test->builder(), all_comparisons, 0);
+
+  // If we use less-than comparisons, we expect the comparison to result in true
+  // if the lhs value to be compared appears earlier in 'all_constants' than the
+  // rhs value. Likewise, if we use greater-than comparisons, we expect the
+  // comparison to return true if the rhs value appears earlier in
+  // 'all_constants' than the lhs value.
+  expected->clear();
+  for (int i = 0; i < all_constants.size(); ++i) {
+    for (int j = 0; j < all_constants.size(); ++j) {
+      expected->push_back(compare_less_than ? i < j : i > j);
+    }
+  }
+}
+
+XLA_TEST_F(ComparatorsTest, CompareLtBF16) {
+  absl::InlinedVector<bool, 10> expected;
+  BuildComparatorAndComparisons<BF16>(this, /*compare_less_than=*/true,
+                                      &expected);
+  ComputeAndCompareR1<bool>(builder(), expected, {});
+}
+
+XLA_TEST_F(ComparatorsTest, CompareGtBF16) {
+  absl::InlinedVector<bool, 10> expected;
+  BuildComparatorAndComparisons<BF16>(this, /*compare_less_than=*/false,
+                                      &expected);
+  ComputeAndCompareR1<bool>(builder(), expected, {});
+}
+
+// The interpreter doesn't support S16, and F16 results in comparisons with S16.
+XLA_TEST_F(ComparatorsTest, DISABLED_ON_INTERPRETER(CompareLtF16)) {
+  absl::InlinedVector<bool, 10> expected;
+  BuildComparatorAndComparisons<F16>(this, /*compare_less_than=*/true,
+                                     &expected);
+  ComputeAndCompareR1<bool>(builder(), expected, {});
+}
+
+XLA_TEST_F(ComparatorsTest, DISABLED_ON_INTERPRETER(CompareGtF16)) {
+  absl::InlinedVector<bool, 10> expected;
+  BuildComparatorAndComparisons<F16>(this, /*compare_less_than=*/false,
+                                     &expected);
+  ComputeAndCompareR1<bool>(builder(), expected, {});
+}
+
+XLA_TEST_F(ComparatorsTest, CompareLtF32) {
+  absl::InlinedVector<bool, 10> expected;
+  BuildComparatorAndComparisons<F32>(this, /*compare_less_than=*/true,
+                                     &expected);
+  ComputeAndCompareR1<bool>(builder(), expected, {});
+}
+
+XLA_TEST_F(ComparatorsTest, CompareGtF32) {
+  absl::InlinedVector<bool, 10> expected;
+  BuildComparatorAndComparisons<F32>(this, /*compare_less_than=*/false,
+                                     &expected);
+  ComputeAndCompareR1<bool>(builder(), expected, {});
+}
+
+XLA_TEST_F(ComparatorsTest, CompareLtF64) {
+  absl::InlinedVector<bool, 10> expected;
+  BuildComparatorAndComparisons<F64>(this, /*compare_less_than=*/true,
+                                     &expected);
+  ComputeAndCompareR1<bool>(builder(), expected, {});
+}
+
+XLA_TEST_F(ComparatorsTest, CompareGtF64) {
+  absl::InlinedVector<bool, 10> expected;
+  BuildComparatorAndComparisons<F64>(this, /*compare_less_than=*/false,
+                                     &expected);
+  ComputeAndCompareR1<bool>(builder(), expected, {});
+}
+
+}  // namespace
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/literal_util.cc b/tensorflow/compiler/xla/literal_util.cc
index 67eab1bff8..5881e43e96 100644
--- a/tensorflow/compiler/xla/literal_util.cc
+++ b/tensorflow/compiler/xla/literal_util.cc
@@ -106,12 +106,16 @@ Literal ConvertType(LiteralSlice literal) {
   switch (primitive_type) {
     case U8:
       return LiteralUtil::CreateR0<uint8>(0);
+    case U16:
+      return LiteralUtil::CreateR0<uint16>(0);
     case U32:
       return LiteralUtil::CreateR0<uint32>(0);
     case U64:
       return LiteralUtil::CreateR0<uint64>(0);
     case S8:
       return LiteralUtil::CreateR0<int8>(0);
+    case S16:
+      return LiteralUtil::CreateR0<int16>(0);
     case S32:
       return LiteralUtil::CreateR0<int32>(0);
     case S64:
@@ -128,9 +132,6 @@ Literal ConvertType(LiteralSlice literal) {
       return LiteralUtil::CreateR0<complex64>(0);
     case PRED:
       return LiteralUtil::CreateR0<bool>(false);
-    case S16:
-    case U16:
-      LOG(FATAL) << "u16/s16 literals not yet implemented";
     case TUPLE:
       LOG(FATAL) << "tuple element type cannot take on value of 0";
     case OPAQUE:
-- 
GitLab


From bb87d0021f12bc2e10d4cbb12e669012b2897d41 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 02:22:26 -0800
Subject: [PATCH 0551/2345] Update ConvLSTMCell to accept shape as list or
 tuple

PiperOrigin-RevId: 228852641
---
 tensorflow/contrib/rnn/python/ops/rnn_cell.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/rnn/python/ops/rnn_cell.py b/tensorflow/contrib/rnn/python/ops/rnn_cell.py
index 8a1c09f171..0702b2a723 100644
--- a/tensorflow/contrib/rnn/python/ops/rnn_cell.py
+++ b/tensorflow/contrib/rnn/python/ops/rnn_cell.py
@@ -2071,7 +2071,7 @@ class ConvLSTMCell(rnn_cell_impl.RNNCell):
       conv_ndims: Convolution dimensionality (1, 2 or 3).
       input_shape: Shape of the input as int tuple, excluding the batch size.
       output_channels: int, number of output channels of the conv LSTM.
-      kernel_shape: Shape of kernel as in tuple (of size 1,2 or 3).
+      kernel_shape: Shape of kernel as an int tuple (of size 1, 2 or 3).
       use_bias: (bool) Use bias in convolutions.
       skip_connection: If set to `True`, concatenate the input to the
         output of the conv LSTM. Default: `False`.
@@ -2092,7 +2092,7 @@ class ConvLSTMCell(rnn_cell_impl.RNNCell):
     self._conv_ndims = conv_ndims
     self._input_shape = input_shape
     self._output_channels = output_channels
-    self._kernel_shape = kernel_shape
+    self._kernel_shape = list(kernel_shape)
     self._use_bias = use_bias
     self._forget_bias = forget_bias
     self._skip_connection = skip_connection
@@ -2172,7 +2172,7 @@ def _conv(args, filter_size, num_features, bias, bias_start=0.0):
   Args:
     args: a Tensor or a list of Tensors of dimension 3D, 4D or 5D,
     batch x n, Tensors.
-    filter_size: int tuple of filter height and width.
+    filter_size: int tuple of filter shape (of size 1, 2 or 3).
     num_features: int, number of features.
     bias: Whether to use biases in the convolution layer.
     bias_start: starting value to initialize the bias; 0 by default.
-- 
GitLab


From cdbaf19b7bb6b6ccfc1f4123f5121448683cbfa3 Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Fri, 11 Jan 2019 03:03:43 -0800
Subject: [PATCH 0552/2345] [Go]: Allow code in the "master" branch to work
 against the latest C library (1.12)

Fixes #23257 for now.
A more principled way of ensuring that we don't end up in this situation
as the Go API is updated to use new C API features remains to be investigated.

PiperOrigin-RevId: 228856302
---
 tensorflow/go/tensor_handle.go | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tensorflow/go/tensor_handle.go b/tensorflow/go/tensor_handle.go
index 3b06773dd1..09192eccef 100644
--- a/tensorflow/go/tensor_handle.go
+++ b/tensorflow/go/tensor_handle.go
@@ -123,13 +123,22 @@ func (th *TensorHandle) DeviceName() (string, error) {
 // BackingDeviceName returns the name of the device in whose memory the tensor
 // handle resides. This function will block till the operation that produces
 // `h` has completed.
+//
+// WARNING: The implementation currently returns the same as DeviceName().
+// After TensoFlow 1.13's C library is released, this implementation will
+// be updated to return what the documentation says!
 func (th *TensorHandle) BackingDeviceName() (string, error) {
+	// TODO(ashankar): Restore after TensorFlow 1.13 is released.
+	// See https://github.com/tensorflow/tensorflow/issues/23257#issuecomment-433751410
+	return th.DeviceName()
+	/*
 	status := newStatus()
 	name := C.TFE_TensorHandleBackingDeviceName(th.c, status.c)
 	if err := status.Err(); err != nil {
 		return "", err
 	}
 	return C.GoString(name), nil
+	*/
 }
 
 // ToTensor returns the Tensor referenced by th. It may block if this tensor is
-- 
GitLab


From 6ba0adcb8051583c892a7f1120c87276e7cf1a11 Mon Sep 17 00:00:00 2001
From: Albin Joy <albin.joy@huawei.com>
Date: Fri, 11 Jan 2019 17:40:47 +0530
Subject: [PATCH 0553/2345] Additional test on make_csv_dataset

Verify the delim and NAvalue are compatible with different inputs.
---
 .../kernel_tests/make_csv_dataset_test.py     | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py
index 3b7b335e70..14677fa6bf 100644
--- a/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py
@@ -449,6 +449,29 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
         header=True,
     )
 
+  def testMakeCSVDataset_withNAValuesAndFieldDelim(self):
+    """Tests that datasets can be created from different delim and na_value.
+    """
+    column_names = ["col%d" % i for i in range(5)]
+    inputs = [["0 1 2 3 4", "5 6 7 8 9"], ["10 11 12 13 14", "15 16 17 ? 19"]]
+    expected_output = [[0, 1, 2, 3, 4], [5, 6, 7, 8, 9],
+                       [10, 11, 12, 13, 14], [15, 16, 17, 0, 19]]
+    label = "col0"
+
+    self._test_dataset(
+        inputs,
+        expected_output=expected_output,
+        expected_keys=column_names,
+        column_names=column_names,
+        label_name=label,
+        batch_size=1,
+        num_epochs=1,
+        shuffle=False,
+        header=False,
+        na_value="?",
+        field_delim=" ",
+    )
+
   def testMakeCSVDataset_withSelectCols(self):
     record_defaults = [
         constant_op.constant([], dtypes.int32),
-- 
GitLab


From 24bfc9b93c6b51dbdd1a1a4d271f49ee13bf0ceb Mon Sep 17 00:00:00 2001
From: Dayananda V <dayanandasiet@gmail.com>
Date: Mon, 7 Jan 2019 11:23:07 +0530
Subject: [PATCH 0554/2345] Fix several DeprecationWarning: invlid escape
 sequence

---
 tensorflow/contrib/learn/python/learn/utils/gc_test.py | 4 ++--
 tensorflow/contrib/session_bundle/exporter.py          | 4 ++--
 tensorflow/contrib/session_bundle/gc_test.py           | 2 +-
 tensorflow/python/kernel_tests/scatter_nd_ops_test.py  | 8 ++++----
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/learn/python/learn/utils/gc_test.py b/tensorflow/contrib/learn/python/learn/utils/gc_test.py
index e7d091e18a..af93e517f5 100644
--- a/tensorflow/contrib/learn/python/learn/utils/gc_test.py
+++ b/tensorflow/contrib/learn/python/learn/utils/gc_test.py
@@ -36,10 +36,10 @@ def _create_parser(base_dir):
     # Modify the path object for RegEx match for Windows Paths
     if os.name == "nt":
       match = re.match(
-          "^" + compat.as_str_any(base_dir).replace("\\", "/") + "/(\\d+)$",
+          r"^" + compat.as_str_any(base_dir).replace("\\", "/") + r"/(\d+)$",
           compat.as_str_any(path.path).replace("\\", "/"))
     else:
-      match = re.match("^" + compat.as_str_any(base_dir) + "/(\\d+)$",
+      match = re.match(r"^" + compat.as_str_any(base_dir) + r"/(\d+)$",
                        compat.as_str_any(path.path))
     if not match:
       return None
diff --git a/tensorflow/contrib/session_bundle/exporter.py b/tensorflow/contrib/session_bundle/exporter.py
index 08983337fc..f3efd292cf 100644
--- a/tensorflow/contrib/session_bundle/exporter.py
+++ b/tensorflow/contrib/session_bundle/exporter.py
@@ -304,10 +304,10 @@ class Exporter(object):
       def parser(path):
         if os.name == "nt":
           match = re.match(
-              "^" + export_dir_base.replace("\\", "/") + "/(\\d{8})$",
+              r"^" + export_dir_base.replace("\\", "/") + r"/(\d{8})$",
               path.path.replace("\\", "/"))
         else:
-          match = re.match("^" + export_dir_base + "/(\\d{8})$", path.path)
+          match = re.match(r"^" + export_dir_base + r"/(\d{8})$", path.path)
         if not match:
           return None
         return path._replace(export_version=int(match.group(1)))
diff --git a/tensorflow/contrib/session_bundle/gc_test.py b/tensorflow/contrib/session_bundle/gc_test.py
index 8faf3ef3d4..02725bb1cb 100644
--- a/tensorflow/contrib/session_bundle/gc_test.py
+++ b/tensorflow/contrib/session_bundle/gc_test.py
@@ -104,7 +104,7 @@ class GcTest(test_util.TensorFlowTestCase):
 
     # create a simple parser that pulls the export_version from the directory.
     def parser(path):
-      match = re.match("^" + base_dir + "/(\\d+)$", path.path)
+      match = re.match(r"^" + base_dir + r"/(\d+)$", path.path)
       if not match:
         return None
       return path._replace(export_version=int(match.group(1)))
diff --git a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
index e1cd9e7622..7ae2df6b01 100644
--- a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
+++ b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
@@ -325,7 +325,7 @@ class StatefulScatterNdTest(test.TestCase):
     shape = np.array([2, 2, 2])
     ref = variables.Variable(array_ops.zeros(shape, dtypes.int32))
     with self.assertRaisesWithPredicateMatch(
-        ValueError, "The outer \\d+ dimensions of indices\\.shape="):
+        ValueError, r"The outer \d+ dimensions of indices\.shape="):
       state_ops.scatter_nd_update(ref, indices, updates)
 
   @test_util.run_deprecated_v1
@@ -335,7 +335,7 @@ class StatefulScatterNdTest(test.TestCase):
     shape = np.array([2, 2, 2])
     ref = variables.Variable(array_ops.zeros(shape, dtypes.int32))
     with self.assertRaisesWithPredicateMatch(
-        ValueError, "The inner \\d+ dimensions of input\\.shape="):
+        ValueError, r"The inner \d+ dimensions of input\.shape="):
       state_ops.scatter_nd_update(ref, indices, updates)
 
   @test_util.run_deprecated_v1
@@ -539,7 +539,7 @@ class ScatterNdTest(test.TestCase):
     updates = array_ops.zeros([2, 2, 2], dtypes.int32)
     shape = np.array([2, 2, 2])
     with self.assertRaisesWithPredicateMatch(
-        ValueError, "The outer \\d+ dimensions of indices\\.shape="):
+        ValueError, r"The outer \d+ dimensions of indices\.shape="):
       self.scatter_nd(indices, updates, shape)
 
   @test_util.run_deprecated_v1
@@ -548,7 +548,7 @@ class ScatterNdTest(test.TestCase):
     updates = array_ops.zeros([2, 2], dtypes.int32)
     shape = np.array([2, 2, 2])
     with self.assertRaisesWithPredicateMatch(
-        ValueError, "The inner \\d+ dimensions of (input|output)\\.shape="):
+        ValueError, r"The inner \d+ dimensions of (input|output)\.shape="):
       self.scatter_nd(indices, updates, shape)
 
   @test_util.run_deprecated_v1
-- 
GitLab


From b86e0ff748dd66f7450bf6859cc881ff5f1c34ba Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Fri, 11 Jan 2019 05:39:19 -0800
Subject: [PATCH 0555/2345] Added nested structure reconstruction on output.

PiperOrigin-RevId: 228868264
---
 tensorflow/python/eager/function.py           | 25 +++++++++----
 .../saved_model/function_deserialization.py   |  8 ++---
 .../saved_model/function_serialization.py     |  6 +++-
 tensorflow/python/saved_model/load.py         | 35 ++++++++++++++-----
 tensorflow/python/saved_model/load_test.py    | 30 ++++++++++++++++
 .../saved_model/saved_object_graph.proto      |  6 +++-
 6 files changed, 87 insertions(+), 23 deletions(-)

diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 9a604620fa..e21ad4e2ee 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -560,9 +560,14 @@ class Function(object):
 
   @property
   def outputs(self):
-    """Returns tensors in `self.graph` corresponding to return values."""
+    """Returns tensors in `self.graph` corresponding to returned tensors."""
     return self._func_graph.outputs
 
+  @property
+  def structured_outputs(self):
+    """Returns outputs in `self.graph` as returned by the original function."""
+    return self._func_graph.structured_outputs
+
   @property
   def captured_inputs(self):
     """Returns external Tensors captured by this function.
@@ -814,8 +819,16 @@ class UnknownArgument(object):
   pass
 
 
-def _convert_inputs_to_signature(inputs):
-  """Convert a list of potentially nested inputs to a signature."""
+def convert_structure_to_signature(structure):
+  """Convert a potentially nested structure to a signature.
+
+  Args:
+    structure: Structure to convert.
+
+  Returns:
+    Identical structure that has TensorSpec objects instead of Tensors and
+    UknownArgument instead of any unsupported types.
+  """
 
   def encode_arg(arg, name=None):
     """A representation for this argument, for converting into signatures."""
@@ -827,9 +840,9 @@ def _convert_inputs_to_signature(inputs):
 
   # We are using the flattened paths to name the TensorSpecs. We need an
   # explicit name for them downstream.
-  flattened_with_paths = nest.flatten_with_joined_string_paths(inputs)
+  flattened_with_paths = nest.flatten_with_joined_string_paths(structure)
   mapped = [encode_arg(arg, path) for path, arg in flattened_with_paths]
-  return nest.pack_sequence_as(inputs, mapped)
+  return nest.pack_sequence_as(structure, mapped)
 
 
 pywrap_tensorflow.RegisterType("Tensor", ops.Tensor)
@@ -1310,7 +1323,7 @@ class PolymorphicFunction(object):
         if self._input_signature:
           python_call_signature = self._input_signature
         else:
-          python_call_signature = tuple(_convert_inputs_to_signature(args))
+          python_call_signature = tuple(convert_structure_to_signature(args))
         # pylint: disable=protected-access
         # Save information about non-Tensor arguments with the concrete
         # function. Used to serialize PolymorphicFunctions.
diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py
index 3cf27fd5a4..ec6cefb606 100644
--- a/tensorflow/python/saved_model/function_deserialization.py
+++ b/tensorflow/python/saved_model/function_deserialization.py
@@ -90,8 +90,7 @@ def recreate_polymorphic_function(
     for monomorphic_function in saved_polymorphic_function.monomorphic_function:
       function_obj = functions[monomorphic_function.concrete_function]
       canonicalized_original_inputs = coder.decode_proto(
-          monomorphic_function.canonicalized_input)
-
+          monomorphic_function.canonicalized_input_signature)
       try:
         can_args, can_kwargs = function_spec.canonicalize_function_inputs(
             *args, **kwargs)
@@ -108,10 +107,7 @@ def recreate_polymorphic_function(
                             canonicalized_original_inputs):
         flattened_inputs = nest.flatten(can_args)
         filtered_inputs = [t for t in flattened_inputs if _is_tensor(t)]
-        flattened_outputs = function_obj._call_flat(filtered_inputs)  # pylint: disable=protected-access
-        # TODO(vbardiovsky): Rebuild output structure.
-        single_output, = flattened_outputs
-        return single_output
+        return function_obj._call_flat(filtered_inputs)  # pylint: disable=protected-access
 
     raise AssertionError(
         "Could not find matching function to call for arguments: %s" % (args,))
diff --git a/tensorflow/python/saved_model/function_serialization.py b/tensorflow/python/saved_model/function_serialization.py
index c826ffa02c..267852ef4a 100644
--- a/tensorflow/python/saved_model/function_serialization.py
+++ b/tensorflow/python/saved_model/function_serialization.py
@@ -47,8 +47,12 @@ def serialize_polymorphic_function(polymorphic_function, node_ids):
       continue
     function_proto = proto.monomorphic_function.add()
     function_proto.concrete_function = concrete_function.name
-    function_proto.canonicalized_input.CopyFrom(
+    function_proto.canonicalized_input_signature.CopyFrom(
         coder.encode_structure(signature))
+    structured_outputs = defun_lib.convert_structure_to_signature(
+        concrete_function.structured_outputs)
+    function_proto.output_signature.CopyFrom(
+        coder.encode_structure(structured_outputs))
     function_proto.bound_inputs.extend(bound_inputs)
   return proto
 
diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
index bbb1485612..d184133fe9 100644
--- a/tensorflow/python/saved_model/load.py
+++ b/tensorflow/python/saved_model/load.py
@@ -27,6 +27,7 @@ from tensorflow.python.ops import variables
 from tensorflow.python.saved_model import constants
 from tensorflow.python.saved_model import function_deserialization
 from tensorflow.python.saved_model import loader_impl
+from tensorflow.python.saved_model import nested_structure_coder
 from tensorflow.python.saved_model import saved_object_graph_pb2
 from tensorflow.python.saved_model import utils_impl as saved_model_utils
 from tensorflow.python.training.checkpointable import tracking
@@ -45,12 +46,13 @@ class _Loader(object):
     self._functions = function_deserialization.load_function_def_library(
         meta_graph.graph_def.library)
     self._load_all()
-    self._bind_function_captures()
+    self._setup_concrete_functions()
     self._restore_checkpoint()
 
-  def _bind_function_captures(self):
-    """Setup captured tensors in restored concrete functions."""
+  def _setup_concrete_functions(self):
+    """Setup captured tensors and output structure in restored functions."""
     seen_functions = set()
+    coder = nested_structure_coder.StructureCoder()
     for object_proto in self._proto.nodes:
       if object_proto.WhichOneof("kind") == "function":
         for monomorphic_function in object_proto.function.monomorphic_function:
@@ -64,17 +66,32 @@ class _Loader(object):
               if self._proto.nodes[node_id].WhichOneof("kind") == "variable"
           ]
           if name in seen_functions:
-            if self._functions[name]._captured_inputs != bound_inputs:  # pylint: disable=protected-access
-              raise NotImplementedError(
-                  "Function %s is used more than once with different "
-                  "captured inputs." % name)
+            raise RuntimeError(
+                "Monomorphic function with a duplicate name: %s." % name)
           else:
             seen_functions.add(name)
             # TODO(andresp): This is only injecting the captured inputs into the
             # concrete function, note that we did not modify the FuncGraph
             # itself.
-            self._functions[name]._captured_inputs = bound_inputs  # pylint: disable=protected-access
-            self._functions[name]._func_graph.variables = bound_variables  # pylint: disable=protected-access
+            function = self._functions[name]
+            function._captured_inputs = bound_inputs  # pylint: disable=protected-access
+            function._func_graph.variables = bound_variables  # pylint: disable=protected-access
+            # By setting the structured_outputs directly, we can rely on this
+            # function_lib.Function object to perform the output repacking
+            # logic. The only limitation of that logic is that it only works
+            # with output that is convertible to Tensors and the conversion
+            # always happens. For example tf.TensorShape([2, 3]) will be
+            # converted to Tensor representing [2, 3].
+            original_outputs = coder.decode_proto(
+                monomorphic_function.output_signature)
+            # The original_outputs here had Tensors converted to TensorSpecs, so
+            # the restored function's structured_outputs field will not be
+            # exactly the same. Fortunately the repacking logic cares only about
+            # the structure.
+            # TODO(vbardiovsky): Should we just replicate the structures, with
+            # Nones instead of real objects? Decide when we start solving
+            # idempotency.
+            function._func_graph.structured_outputs = original_outputs  # pylint: disable=protected-access
 
   def _get_tensor_from_node(self, node_id):
     obj = self._nodes[node_id]
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index 32bb841556..defd269012 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import collections
 import os
 import tempfile
 
@@ -218,6 +219,35 @@ class LoadTest(test.TestCase):
     self.assertEqual(31, imported.f(input1).numpy())
     self.assertEqual(32, imported.f(input3).numpy())
 
+  def test_structured_output(self):
+
+    # Use fields with non-alphabetical order
+    named_tuple_type = collections.namedtuple("NamedTupleHello", ["b", "a"])
+
+    def func(input1, input2):
+      named_tuple = named_tuple_type(a=input1 + input2, b=input1 * input2)
+      return [named_tuple, input2, {"x": 0.5}]
+
+    root = tracking.Checkpointable()
+    root.f = def_function.function(func)
+
+    result = root.f(constant_op.constant(2), constant_op.constant(3))
+
+    self.assertEqual(5, result[0].a.numpy())
+    self.assertEqual(6, result[0].b.numpy())
+    self.assertEqual(["b", "a"], list(result[0]._asdict().keys()))
+    self.assertEqual(3, result[1].numpy())
+    self.assertEqual(0.5, result[2]["x"].numpy())
+
+    imported = self.cycle(root)
+
+    result = imported.f(constant_op.constant(2), constant_op.constant(5))
+    self.assertEqual(7, result[0].a.numpy())
+    self.assertEqual(10, result[0].b.numpy())
+    self.assertEqual(["b", "a"], list(result[0]._asdict().keys()))
+    self.assertEqual(5, result[1].numpy())
+    self.assertEqual(0.5, result[2]["x"].numpy())
+
   def test_positional_arguments(self):
     def func(x, training=False, abc=7.1, defg=7.7):
       del abc
diff --git a/tensorflow/python/saved_model/saved_object_graph.proto b/tensorflow/python/saved_model/saved_object_graph.proto
index 1e2514b7f7..a870eee383 100644
--- a/tensorflow/python/saved_model/saved_object_graph.proto
+++ b/tensorflow/python/saved_model/saved_object_graph.proto
@@ -96,7 +96,11 @@ message SavedMonomorphicFunction {
   repeated int32 bound_inputs = 2;
   // Input in canonicalized form that was received to create this concrete
   // function.
-  StructuredValue canonicalized_input = 3;
+  StructuredValue canonicalized_input_signature = 3;
+  // Output that was the return value of this function after replacing all
+  // Tensors with TensorSpecs. This can be an arbitrary nested function and will
+  // be used to reconstruct the full structure from pure tensors.
+  StructuredValue output_signature = 4;
 }
 
 // Represents a Variable that is initialized by loading the contents from the
-- 
GitLab


From b044b5977e6c0425d4eab66d7f9daa11e161e3ee Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Fri, 11 Jan 2019 06:01:22 -0800
Subject: [PATCH 0556/2345] Support S16 and U16 in the HLO evaluator

This allows to enable ComparatorsTest.CompareLtF16 and ComparatorsTest.CompareGtF16 tests.
While at it, also enable other evaluator tests which are already passing.

PiperOrigin-RevId: 228869860
---
 .../xla/client/lib/comparators_test.cc        |  5 ++--
 tensorflow/compiler/xla/literal.cc            |  4 +++
 tensorflow/compiler/xla/literal_test.cc       | 24 +++++++++++++++--
 tensorflow/compiler/xla/service/BUILD         |  2 ++
 .../compiler/xla/service/hlo_evaluator.cc     | 26 +++++++++----------
 .../hlo_evaluator_typed_visitor_int16.cc      | 22 ++++++++++++++++
 .../hlo_evaluator_typed_visitor_uint16.cc     | 22 ++++++++++++++++
 tensorflow/compiler/xla/tests/tuple_test.cc   |  3 +--
 tensorflow/compiler/xla/tests/while_test.cc   |  4 +--
 9 files changed, 89 insertions(+), 23 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor_int16.cc
 create mode 100644 tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor_uint16.cc

diff --git a/tensorflow/compiler/xla/client/lib/comparators_test.cc b/tensorflow/compiler/xla/client/lib/comparators_test.cc
index efec984eb2..598956803b 100644
--- a/tensorflow/compiler/xla/client/lib/comparators_test.cc
+++ b/tensorflow/compiler/xla/client/lib/comparators_test.cc
@@ -103,15 +103,14 @@ XLA_TEST_F(ComparatorsTest, CompareGtBF16) {
   ComputeAndCompareR1<bool>(builder(), expected, {});
 }
 
-// The interpreter doesn't support S16, and F16 results in comparisons with S16.
-XLA_TEST_F(ComparatorsTest, DISABLED_ON_INTERPRETER(CompareLtF16)) {
+XLA_TEST_F(ComparatorsTest, CompareLtF16) {
   absl::InlinedVector<bool, 10> expected;
   BuildComparatorAndComparisons<F16>(this, /*compare_less_than=*/true,
                                      &expected);
   ComputeAndCompareR1<bool>(builder(), expected, {});
 }
 
-XLA_TEST_F(ComparatorsTest, DISABLED_ON_INTERPRETER(CompareGtF16)) {
+XLA_TEST_F(ComparatorsTest, CompareGtF16) {
   absl::InlinedVector<bool, 10> expected;
   BuildComparatorAndComparisons<F16>(this, /*compare_less_than=*/false,
                                      &expected);
diff --git a/tensorflow/compiler/xla/literal.cc b/tensorflow/compiler/xla/literal.cc
index 6c44189930..81874fa405 100644
--- a/tensorflow/compiler/xla/literal.cc
+++ b/tensorflow/compiler/xla/literal.cc
@@ -1319,9 +1319,11 @@ StatusOr<Literal> ConvertIfDestTypeMatches(const LiteralBase& src_literal,
                                                            bitcast);
     CONVERT_IF_TYPES_MATCH(PRED)
     CONVERT_IF_TYPES_MATCH(S8)
+    CONVERT_IF_TYPES_MATCH(S16)
     CONVERT_IF_TYPES_MATCH(S32)
     CONVERT_IF_TYPES_MATCH(S64)
     CONVERT_IF_TYPES_MATCH(U8)
+    CONVERT_IF_TYPES_MATCH(U16)
     CONVERT_IF_TYPES_MATCH(U32)
     CONVERT_IF_TYPES_MATCH(U64)
     CONVERT_IF_TYPES_MATCH(F16)
@@ -1357,9 +1359,11 @@ StatusOr<Literal> ConvertSwitch(const LiteralBase& literal,
                                             bitcast);
     CONVERT_IF_DEST_TYPE_MATCHES(PRED)
     CONVERT_IF_DEST_TYPE_MATCHES(S8)
+    CONVERT_IF_DEST_TYPE_MATCHES(S16)
     CONVERT_IF_DEST_TYPE_MATCHES(S32)
     CONVERT_IF_DEST_TYPE_MATCHES(S64)
     CONVERT_IF_DEST_TYPE_MATCHES(U8)
+    CONVERT_IF_DEST_TYPE_MATCHES(U16)
     CONVERT_IF_DEST_TYPE_MATCHES(U32)
     CONVERT_IF_DEST_TYPE_MATCHES(U64)
     CONVERT_IF_DEST_TYPE_MATCHES(F16)
diff --git a/tensorflow/compiler/xla/literal_test.cc b/tensorflow/compiler/xla/literal_test.cc
index 0a71cfd5ab..a4088c6d11 100644
--- a/tensorflow/compiler/xla/literal_test.cc
+++ b/tensorflow/compiler/xla/literal_test.cc
@@ -1237,11 +1237,21 @@ TEST_F(LiteralUtilTest, ConvertIfTypesMatch) {
     {{0, 19, 0, 21}, {22, 0, 24, 0}},
     {{26, 0, 28, 0}, {0, 31, 0, 33}},
   }}, layout_r4_dim0major_);
+  auto s16 = LiteralUtil::CreateR4WithLayout<int16>({{
+    {{10, 0, 12, 0}, {0, 15, 0, 17}},
+    {{0, 19, 0, 21}, {22, 0, 24, 0}},
+    {{26, 0, 28, 0}, {0, 31, 0, 33}},
+  }}, layout_r4_dim0major_);
   auto s32 = LiteralUtil::CreateR4WithLayout<int32>({{
     {{10, 0, 12, 0}, {0, 15, 0, 17}},
     {{0, 19, 0, 21}, {22, 0, 24, 0}},
     {{26, 0, 28, 0}, {0, 31, 0, 33}},
   }}, layout_r4_dim0major_);
+  auto u16 = LiteralUtil::CreateR4WithLayout<uint16>({{
+    {{10, 0, 12, 0}, {0, 15, 0, 17}},
+    {{0, 19, 0, 21}, {22, 0, 24, 0}},
+    {{26, 0, 28, 0}, {0, 31, 0, 33}},
+  }}, layout_r4_dim0major_);
   auto u32 = LiteralUtil::CreateR4WithLayout<uint32>({{
     {{10, 0, 12, 0}, {0, 15, 0, 17}},
     {{0, 19, 0, 21}, {22, 0, 24, 0}},
@@ -1301,6 +1311,12 @@ TEST_F(LiteralUtilTest, ConvertIfTypesMatch) {
   // clang-format on
   Literal conv;
 
+  conv = s8.Convert(U16).ConsumeValueOrDie();
+  EXPECT_EQ(conv, u16);
+
+  conv = s8.Convert(S16).ConsumeValueOrDie();
+  EXPECT_EQ(conv, s16);
+
   conv = s8.Convert(U32).ConsumeValueOrDie();
   EXPECT_EQ(conv, u32);
 
@@ -1352,10 +1368,14 @@ TEST_F(LiteralUtilTest, ConvertIfTypesMatch) {
   conv = f16.Convert(C64).ConsumeValueOrDie();
   EXPECT_EQ(conv, c64);
 
+  conv = s32.Convert(S16).ConsumeValueOrDie();
+  EXPECT_EQ(conv, s16);
+
+  conv = s32.Convert(U16).ConsumeValueOrDie();
+  EXPECT_EQ(conv, u16);
+
   EXPECT_EQ(s32.Convert(TUPLE).status().code(),
             tensorflow::error::UNIMPLEMENTED);
-  EXPECT_EQ(s32.Convert(S16).status().code(), tensorflow::error::UNIMPLEMENTED);
-  EXPECT_EQ(s32.Convert(U16).status().code(), tensorflow::error::UNIMPLEMENTED);
   EXPECT_EQ(c64.Convert(F32).status().code(), tensorflow::error::UNIMPLEMENTED);
   EXPECT_EQ(c64.Convert(S32).status().code(), tensorflow::error::UNIMPLEMENTED);
 }
diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 728adb67a0..ff14c500e3 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -227,9 +227,11 @@ cc_library(
         "hlo_evaluator_typed_visitor_double.cc",
         "hlo_evaluator_typed_visitor_float.cc",
         "hlo_evaluator_typed_visitor_half.cc",
+        "hlo_evaluator_typed_visitor_int16.cc",
         "hlo_evaluator_typed_visitor_int32.cc",
         "hlo_evaluator_typed_visitor_int64.cc",
         "hlo_evaluator_typed_visitor_int8.cc",
+        "hlo_evaluator_typed_visitor_uint16.cc",
         "hlo_evaluator_typed_visitor_uint32.cc",
         "hlo_evaluator_typed_visitor_uint64.cc",
         "hlo_evaluator_typed_visitor_uint8.cc",
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index b8d95fb244..407615f842 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -150,22 +150,14 @@ HloEvaluator::HloEvaluator(int64 max_loop_iterations)
   typed_visitors_[U8] =
       absl::make_unique<HloEvaluatorTypedVisitor<uint8>>(this);
   typed_visitors_[U16] =
-      absl::make_unique<FunctionVisitor>([](HloInstruction*) {
-        return Unimplemented(
-            "HloEvaluator::HloEvaluatorTypedVisitor: unhandled primitive type: "
-            "U16.");
-      });
+      absl::make_unique<HloEvaluatorTypedVisitor<uint16>>(this);
   typed_visitors_[U32] =
       absl::make_unique<HloEvaluatorTypedVisitor<uint32>>(this);
   typed_visitors_[U64] =
       absl::make_unique<HloEvaluatorTypedVisitor<uint64>>(this);
   typed_visitors_[S8] = absl::make_unique<HloEvaluatorTypedVisitor<int8>>(this);
   typed_visitors_[S16] =
-      absl::make_unique<FunctionVisitor>([](HloInstruction*) {
-        return Unimplemented(
-            "HloEvaluator::HloEvaluatorTypedVisitor: unhandled primitive type: "
-            "S16.");
-      });
+      absl::make_unique<HloEvaluatorTypedVisitor<int16>>(this);
   typed_visitors_[S32] =
       absl::make_unique<HloEvaluatorTypedVisitor<int32>>(this);
   typed_visitors_[S64] =
@@ -595,8 +587,11 @@ Status HloEvaluator::HandleCompare(HloInstruction* compare) {
           evaluated_[compare],
           Compare<uint8>(compare->shape(), opcode, lhs_literal, rhs_literal));
     } break;
-    case U16:
-      return Unimplemented("unhandled primitive type: U16.");
+    case U16: {
+      TF_ASSIGN_OR_RETURN(
+          evaluated_[compare],
+          Compare<uint16>(compare->shape(), opcode, lhs_literal, rhs_literal));
+    } break;
     case U32: {
       TF_ASSIGN_OR_RETURN(
           evaluated_[compare],
@@ -612,8 +607,11 @@ Status HloEvaluator::HandleCompare(HloInstruction* compare) {
           evaluated_[compare],
           Compare<int8>(compare->shape(), opcode, lhs_literal, rhs_literal));
     } break;
-    case S16:
-      return Unimplemented("unhandled primitive type: S16.");
+    case S16: {
+      TF_ASSIGN_OR_RETURN(
+          evaluated_[compare],
+          Compare<int16>(compare->shape(), opcode, lhs_literal, rhs_literal));
+    } break;
     case S32: {
       TF_ASSIGN_OR_RETURN(
           evaluated_[compare],
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor_int16.cc b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor_int16.cc
new file mode 100644
index 0000000000..e54285a157
--- /dev/null
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor_int16.cc
@@ -0,0 +1,22 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h"
+
+#include "tensorflow/compiler/xla/service/hlo_evaluator.h"
+
+namespace xla {
+template class HloEvaluatorTypedVisitor<int16>;
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor_uint16.cc b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor_uint16.cc
new file mode 100644
index 0000000000..cc708952d2
--- /dev/null
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor_uint16.cc
@@ -0,0 +1,22 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h"
+
+#include "tensorflow/compiler/xla/service/hlo_evaluator.h"
+
+namespace xla {
+template class HloEvaluatorTypedVisitor<uint16>;
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/tests/tuple_test.cc b/tensorflow/compiler/xla/tests/tuple_test.cc
index 426d6c84ee..0def0577e0 100644
--- a/tensorflow/compiler/xla/tests/tuple_test.cc
+++ b/tensorflow/compiler/xla/tests/tuple_test.cc
@@ -513,8 +513,7 @@ XLA_TEST_F(TupleTest, ComplexTuples) {
 
 class TupleHloTest : public HloTestBase {};
 
-// Disabled on the interpreter because bitcast doesn't exist on the interpreter.
-XLA_TEST_F(TupleHloTest, DISABLED_ON_INTERPRETER(BitcastAfterGTE)) {
+XLA_TEST_F(TupleHloTest, BitcastAfterGTE) {
   const char* testcase = R"(
     HloModule m, is_scheduled=true
 
diff --git a/tensorflow/compiler/xla/tests/while_test.cc b/tensorflow/compiler/xla/tests/while_test.cc
index 6d5f276e82..b1566dc8ad 100644
--- a/tensorflow/compiler/xla/tests/while_test.cc
+++ b/tensorflow/compiler/xla/tests/while_test.cc
@@ -901,7 +901,7 @@ XLA_TEST_F(WhileTest, WhileWithDynamicUpdateSlice) {
 // Per backend the values generated can be different as the different backends
 // use different random number generators.
 // TODO(b/32240857): Extend test to verify outputs.
-XLA_TEST_F(WhileTest, DISABLED_ON_INTERPRETER(WhileWithPrngScalarResult)) {
+XLA_TEST_F(WhileTest, WhileWithPrngScalarResult) {
   auto v6s32 = ShapeUtil::MakeShape(S32, {6});
 
   // Create a computation for the condition: repeat for count iterations.
@@ -1146,7 +1146,7 @@ XLA_TEST_F(WhileTest, NestedWhileWithScalarResult) {
 // while (f(result).get<0>()) {
 //   result = result + 1;
 // }
-XLA_TEST_F(WhileTest, DISABLED_ON_INTERPRETER(WhileWithCallInsideCondition)) {
+XLA_TEST_F(WhileTest, WhileWithCallInsideCondition) {
   auto result_shape = ShapeUtil::MakeShape(S32, {});
 
   // Create a computation for the condition: repeat for 5 iterations.
-- 
GitLab


From 277636caf62cddf094806129239c6ae6788c636a Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Fri, 11 Jan 2019 09:28:34 -0800
Subject: [PATCH 0557/2345] Corrected ResourceVariable constructor
 documentation wrt validate_shape

PiperOrigin-RevId: 228894072
---
 tensorflow/python/ops/resource_variable_ops.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 2d46e21074..1f2ff9f197 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -219,7 +219,7 @@ class ResourceVariable(variables.VariableV1):
                initial_value=None,
                trainable=True,
                collections=None,
-               validate_shape=True,
+               validate_shape=True,  # pylint: disable=unused-argument
                caching_device=None,
                name=None,
                dtype=None,
@@ -231,8 +231,7 @@ class ResourceVariable(variables.VariableV1):
 
     Args:
       initial_value: A `Tensor`, or Python object convertible to a `Tensor`,
-        which is the initial value for the Variable. The initial value must have
-        a shape specified unless `validate_shape` is set to False. Can also be a
+        which is the initial value for the Variable. Can also be a
         callable with no argument that returns the initial value when called.
         (Note that initializer functions from init_ops.py must first be bound
          to a shape before being used here.)
@@ -292,7 +291,6 @@ class ResourceVariable(variables.VariableV1):
           initial_value=initial_value,
           trainable=trainable,
           collections=collections,
-          validate_shape=validate_shape,
           caching_device=caching_device,
           name=name,
           dtype=dtype,
@@ -307,12 +305,10 @@ class ResourceVariable(variables.VariableV1):
       return "<tf.Variable '%s' shape=%s dtype=%s>" % (
           self.name, self.get_shape(), self.dtype.name)
 
-  # pylint: disable=unused-argument
   def _init_from_args(self,
                       initial_value=None,
                       trainable=True,
                       collections=None,
-                      validate_shape=True,
                       caching_device=None,
                       name=None,
                       dtype=None,
-- 
GitLab


From 196715b526a695678891ec11d13b7290b3f6861a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 09:32:51 -0800
Subject: [PATCH 0558/2345] Remove unnecessary dependency on :pooling_ops from
 NN_DEPS (used by :bias_op and others.)

PiperOrigin-RevId: 228894672
---
 tensorflow/core/kernels/BUILD                  | 1 -
 tensorflow/core/kernels/fused_batch_norm_op.cc | 1 -
 2 files changed, 2 deletions(-)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 4db0505a8f..0792b398f1 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -3820,7 +3820,6 @@ NN_DEPS = [
     ":conv_2d",
     ":eigen_contraction_kernel",
     ":ops_util",
-    ":pooling_ops",
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
diff --git a/tensorflow/core/kernels/fused_batch_norm_op.cc b/tensorflow/core/kernels/fused_batch_norm_op.cc
index dbd3bb05db..48b339508b 100644
--- a/tensorflow/core/kernels/fused_batch_norm_op.cc
+++ b/tensorflow/core/kernels/fused_batch_norm_op.cc
@@ -18,7 +18,6 @@ limitations under the License.
 #if GOOGLE_CUDA
 #define EIGEN_USE_GPU
 #include "tensorflow/core/kernels/conv_2d.h"
-#include "tensorflow/core/kernels/conv_ops_gpu.h"
 #include "tensorflow/core/util/stream_executor_util.h"
 #endif
 
-- 
GitLab


From 93a4a83b08f0290926bca774013f4667e4634bf5 Mon Sep 17 00:00:00 2001
From: Dheeraj Rajaram Reddy <dheeraj98reddy@gmail.com>
Date: Fri, 11 Jan 2019 23:33:02 +0530
Subject: [PATCH 0559/2345] Add take_api golden API to pbtxt

---
 .../tools/api/golden/v2/tensorflow.data.experimental.pbtxt    | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
index a262c0f799..abc98a74b6 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
@@ -180,6 +180,10 @@ tf_module {
     name: "shuffle_and_repeat"
     argspec: "args=[\'buffer_size\', \'count\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
+  member_method {
+    name: "take_while"
+    argspec: "args=[\'predicate\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "unbatch"
     argspec: "args=[], varargs=None, keywords=None, defaults=None"
-- 
GitLab


From 8d0896bc0020c68faaf56b4240db6da144b584d4 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 11 Jan 2019 10:07:16 -0800
Subject: [PATCH 0560/2345] Make TopK test more explicit

---
 tensorflow/contrib/tensorrt/test/topk_test.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/test/topk_test.py b/tensorflow/contrib/tensorrt/test/topk_test.py
index 2edf7cb587..efbfb26898 100644
--- a/tensorflow/contrib/tensorrt/test/topk_test.py
+++ b/tensorflow/contrib/tensorrt/test/topk_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import constant_op
 from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
@@ -37,7 +38,8 @@ class TopKTest(trt_test.TfTrtIntegrationTestBase):
     g = ops.Graph()
     with g.as_default():
       x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name)
-      values, indices = nn_ops.top_k(x, k)
+      k_tensor = constant_op.constant(k, dtype=tf.int32, name="Const")
+      values, indices = nn_ops.top_k(x, k_tensor, name="TopK")
       values = array_ops.identity(values, name='output_values')
       indices = array_ops.identity(indices, name='output_indices')
     return trt_test.TfTrtIntegrationTestParams(
@@ -49,7 +51,7 @@ class TopKTest(trt_test.TfTrtIntegrationTestBase):
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
-    return ["TRTEngineOp_0"]
+    return {"TRTEngineOp_0" : ["Const", "TopK"]}
 
 
 if __name__ == "__main__":
-- 
GitLab


From 1546c90bfea6e6e072eb889f4986a8db78c10ab4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 10:03:13 -0800
Subject: [PATCH 0561/2345] Add bitwise ops to ragged_dispatch

PiperOrigin-RevId: 228899228
---
 tensorflow/python/ops/ragged/ragged_dispatch.py      | 7 +++++++
 tensorflow/python/ops/ragged/ragged_dispatch_test.py | 7 +++++++
 2 files changed, 14 insertions(+)

diff --git a/tensorflow/python/ops/ragged/ragged_dispatch.py b/tensorflow/python/ops/ragged/ragged_dispatch.py
index 52c4142346..970d88663a 100644
--- a/tensorflow/python/ops/ragged/ragged_dispatch.py
+++ b/tensorflow/python/ops/ragged/ragged_dispatch.py
@@ -25,6 +25,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import clip_ops
+from tensorflow.python.ops import gen_bitwise_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import parsing_ops
 from tensorflow.python.ops import string_ops
@@ -282,6 +283,7 @@ _UNARY_ELEMENTWISE_OPS = [
     array_ops.zeros_like,
     array_ops.zeros_like_v2,
     clip_ops.clip_by_value,
+    gen_bitwise_ops.invert,
     math_ops.abs,
     math_ops.acos,
     math_ops.acosh,
@@ -348,6 +350,11 @@ _UNARY_LIST_ELEMENTWISE_OPS = [
 ]
 
 _BINARY_ELEMENTWISE_OPS = [
+    gen_bitwise_ops.bitwise_and,
+    gen_bitwise_ops.bitwise_or,
+    gen_bitwise_ops.bitwise_xor,
+    gen_bitwise_ops.left_shift,
+    gen_bitwise_ops.right_shift,
     math_ops.add,
     math_ops.atan2,
     math_ops.complex,
diff --git a/tensorflow/python/ops/ragged/ragged_dispatch_test.py b/tensorflow/python/ops/ragged/ragged_dispatch_test.py
index 0c546accc3..1aa612633e 100644
--- a/tensorflow/python/ops/ragged/ragged_dispatch_test.py
+++ b/tensorflow/python/ops/ragged/ragged_dispatch_test.py
@@ -29,6 +29,7 @@ from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import clip_ops
+from tensorflow.python.ops import gen_bitwise_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import parsing_ops
 from tensorflow.python.ops import string_ops
@@ -121,9 +122,15 @@ BINARY_BOOL_OPS = [
     math_ops.logical_xor,
 ]
 UNARY_INT_OPS = [
+    gen_bitwise_ops.invert,
     string_ops.unicode_script,
 ]
 BINARY_INT_OPS = [
+    gen_bitwise_ops.bitwise_and,
+    gen_bitwise_ops.bitwise_or,
+    gen_bitwise_ops.bitwise_xor,
+    gen_bitwise_ops.left_shift,
+    gen_bitwise_ops.right_shift,
     math_ops.truncatediv,
     math_ops.truncatemod,
 ]
-- 
GitLab


From 78032512e66dd8a829dcfce338c4fb50e3b9ed49 Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Fri, 11 Jan 2019 10:18:57 -0800
Subject: [PATCH 0562/2345] Lifted *_slice_info methods to Variable

That way they are easier to delete once PartitionInfo is gone.

PiperOrigin-RevId: 228901926
---
 .../python/ops/resource_variable_ops.py       | 11 ----------
 tensorflow/python/ops/variables.py            | 22 +++++++++----------
 2 files changed, 11 insertions(+), 22 deletions(-)

diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 1f2ff9f197..2681deb78c 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -730,17 +730,6 @@ class ResourceVariable(variables.VariableV1):
     return gen_state_ops.resource_count_up_to(self.handle, limit=limit,
                                               T=self.dtype)
 
-  def _set_save_slice_info(self, save_slice_info):
-    """Sets the slice info for this `ResourceVariable`.
-
-    Args:
-      save_slice_info: A `Variable.SaveSliceInfo` object.
-    """
-    self._save_slice_info = save_slice_info
-
-  def _get_save_slice_info(self):
-    return self._save_slice_info
-
   def _read_variable_op(self):
     if self.trainable:
       tape.variable_accessed(self)
diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index f9fc72a6da..d765936399 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -1049,6 +1049,17 @@ class Variable(six.with_metaclass(VariableMetaclass,
     return RefVariable(variable_def=variable_def,
                        import_scope=import_scope)
 
+  def _set_save_slice_info(self, save_slice_info):
+    """Sets the slice info for this `Variable`.
+
+    Args:
+      save_slice_info: A `Variable.SaveSliceInfo` object.
+    """
+    self._save_slice_info = save_slice_info
+
+  def _get_save_slice_info(self):
+    return self._save_slice_info
+
   class SaveSliceInfo(object):
     """Information on how to save this Variable as a slice.
 
@@ -2295,17 +2306,6 @@ class RefVariable(VariableV1):
         " if you want a new python Tensor object.", 1)
     return self ** other
 
-  def _set_save_slice_info(self, save_slice_info):
-    """Sets the slice info for this `Variable`.
-
-    Args:
-      save_slice_info: A `Variable.SaveSliceInfo` object.
-    """
-    self._save_slice_info = save_slice_info
-
-  def _get_save_slice_info(self):
-    return self._save_slice_info
-
 
 def _try_guard_against_uninitialized_dependencies(name, initial_value):
   """Attempt to guard against dependencies on uninitialized variables.
-- 
GitLab


From a4f5558529a56d72bdcec9c0c8d0a3fb2a91fb64 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Fri, 11 Jan 2019 10:33:58 -0800
Subject: [PATCH 0563/2345] Fix typos in deprecation messages for convolutional
 layers for TF 2.0 API.

PiperOrigin-RevId: 228904460
---
 tensorflow/python/layers/convolutional.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/tensorflow/python/layers/convolutional.py b/tensorflow/python/layers/convolutional.py
index 5d4805e245..03344c844d 100644
--- a/tensorflow/python/layers/convolutional.py
+++ b/tensorflow/python/layers/convolutional.py
@@ -117,7 +117,7 @@ class Conv1D(keras_layers.Conv1D, base.Layer):
 
 @deprecation.deprecated(
     date=None,
-    instructions='Use keras.layers.conv1d instead.')
+    instructions='Use tf.keras.layers.Conv1D instead.')
 @tf_export(v1=['layers.conv1d'])
 def conv1d(inputs,
            filters,
@@ -316,7 +316,7 @@ class Conv2D(keras_layers.Conv2D, base.Layer):
 
 @deprecation.deprecated(
     date=None,
-    instructions='Use keras.layers.conv2d instead.')
+    instructions='Use tf.keras.layers.Conv2D instead.')
 @tf_export(v1=['layers.conv2d'])
 def conv2d(inputs,
            filters,
@@ -523,7 +523,7 @@ class Conv3D(keras_layers.Conv3D, base.Layer):
 
 @deprecation.deprecated(
     date=None,
-    instructions='Use keras.layers.conv3d instead.')
+    instructions='Use tf.keras.layers.Conv3D instead.')
 @tf_export(v1=['layers.conv3d'])
 def conv3d(inputs,
            filters,
@@ -853,7 +853,7 @@ class SeparableConv2D(keras_layers.SeparableConv2D, base.Layer):
 
 @deprecation.deprecated(
     date=None,
-    instructions='Use keras.layers.separable_conv1d instead.')
+    instructions='Use tf.keras.layers.SeparableConv1D instead.')
 @tf_export(v1=['layers.separable_conv1d'])
 def separable_conv1d(inputs,
                      filters,
@@ -973,7 +973,7 @@ def separable_conv1d(inputs,
 
 @deprecation.deprecated(
     date=None,
-    instructions='Use keras.layers.separable_conv2d instead.')
+    instructions='Use tf.keras.layers.SeparableConv2D instead.')
 @tf_export(v1=['layers.separable_conv2d'])
 def separable_conv2d(inputs,
                      filters,
@@ -1183,7 +1183,7 @@ class Conv2DTranspose(keras_layers.Conv2DTranspose, base.Layer):
 
 @deprecation.deprecated(
     date=None,
-    instructions='Use keras.layers.conv2d_transpose instead.')
+    instructions='Use tf.keras.layers.Conv2DTranspose instead.')
 @tf_export(v1=['layers.conv2d_transpose'])
 def conv2d_transpose(inputs,
                      filters,
@@ -1363,7 +1363,7 @@ class Conv3DTranspose(keras_layers.Conv3DTranspose, base.Layer):
 
 @deprecation.deprecated(
     date=None,
-    instructions='Use keras.layers.conv3d_transpose instead.')
+    instructions='Use tf.keras.layers.Conv3DTranspose instead.')
 @tf_export(v1=['layers.conv3d_transpose'])
 def conv3d_transpose(inputs,
                      filters,
-- 
GitLab


From 1dea159cd3cbde8763544ecc961ef11c9d748e50 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Fri, 11 Jan 2019 10:34:40 -0800
Subject: [PATCH 0564/2345] [tf.data] Adding a benchmark for parallel
 interleave and using it to improve the performance of the v2 implementation.

PiperOrigin-RevId: 228904615
---
 .../data/parallel_interleave_dataset_op.cc    |   8 +-
 .../python/data/experimental/benchmarks/BUILD |  16 +++
 .../parallel_interleave_benchmark.py          | 103 ++++++++++++++++++
 3 files changed, 124 insertions(+), 3 deletions(-)
 create mode 100644 tensorflow/python/data/experimental/benchmarks/parallel_interleave_benchmark.py

diff --git a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
index dc2663d1e0..336fa52b49 100644
--- a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
+++ b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
@@ -28,6 +28,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/threadpool.h"
 #include "tensorflow/core/lib/gtl/cleanup.h"
 #include "tensorflow/core/lib/random/random.h"
+#include "tensorflow/core/platform/cpu_info.h"
 #include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
@@ -202,7 +203,7 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
             thread_pool_(new thread::ThreadPool(
                 Env::Default(), ThreadOptions(),
                 "data_parallel_interleave_worker_pool",
-                dataset()->cycle_length_ /* num_threads */,
+                port::NumSchedulableCPUs() /* num_threads */,
                 false /* low_latency_hint */)) {
         std::vector<string> components =
             str_util::Split(params.prefix, "::", str_util::SkipEmpty());
@@ -262,7 +263,7 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
         return model::MakeAsyncInterleaveManyNode(
             std::move(args),
             {model::MakeParameter("parallelism", num_parallel_calls_, /*min=*/1,
-                                  /*max=*/dataset()->cycle_length_)});
+                                  /*max=*/port::NumSchedulableCPUs())});
       }
 
       Status SaveInternal(IteratorStateWriter* writer) override {
@@ -574,8 +575,9 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
         RecordStart(ctx.get());
         auto cleanup = gtl::MakeCleanup([this, ctx] { RecordStop(ctx.get()); });
         auto busy = [this]() EXCLUSIVE_LOCKS_REQUIRED(*mu_) -> bool {
+          // TODO(jsimsa): Autotune the buffer size.
           return num_calls_ >= num_parallel_calls_->value ||
-                 future_elements_.size() >= dataset()->cycle_length_;
+                 future_elements_.size() >= 2 * dataset()->cycle_length_;
         };
         while (true) {
           mutex_lock l(*mu_);
diff --git a/tensorflow/python/data/experimental/benchmarks/BUILD b/tensorflow/python/data/experimental/benchmarks/BUILD
index 651dfd6857..2443f7edad 100644
--- a/tensorflow/python/data/experimental/benchmarks/BUILD
+++ b/tensorflow/python/data/experimental/benchmarks/BUILD
@@ -124,6 +124,22 @@ py_test(
     ],
 )
 
+py_test(
+    name = "parallel_interleave_benchmark",
+    srcs = ["parallel_interleave_benchmark.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:session",
+        "//tensorflow/python/data/experimental/ops:interleave_ops",
+        "//tensorflow/python/data/experimental/ops:optimization",
+        "//tensorflow/python/data/experimental/ops:sleep",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//third_party/py/numpy",
+    ],
+)
+
 py_test(
     name = "rejection_resample_benchmark",
     srcs = ["rejection_resample_benchmark.py"],
diff --git a/tensorflow/python/data/experimental/benchmarks/parallel_interleave_benchmark.py b/tensorflow/python/data/experimental/benchmarks/parallel_interleave_benchmark.py
new file mode 100644
index 0000000000..f5108e5ff4
--- /dev/null
+++ b/tensorflow/python/data/experimental/benchmarks/parallel_interleave_benchmark.py
@@ -0,0 +1,103 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Benchmarks for `tf.data.experimental.parallel_interleave()`."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import time
+
+import numpy as np
+
+from tensorflow.python.client import session
+from tensorflow.python.data.experimental.ops import interleave_ops
+from tensorflow.python.data.experimental.ops import optimization
+from tensorflow.python.data.experimental.ops import sleep
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import ops
+from tensorflow.python.platform import test
+
+
+def _make_fake_dataset_fn():
+  """Returns a dataset that emulates a remote storage data source.
+
+  Returns a dataset factory which creates a dataset with 100 elements that
+  emulates the performance characteristic of a file-based dataset stored in a
+  remote storage. In particular, the first element will take an order of
+  magnitude longer to produce than the remaining elements (1s vs. 1ms).
+  """
+
+  def fake_dataset_fn(unused):
+    del unused
+
+    def make_dataset(time_us, num_elements):
+      return dataset_ops.Dataset.range(num_elements).apply(sleep.sleep(time_us))
+
+    return make_dataset(1000 * 1000, 0).concatenate(make_dataset(1000,
+                                                                 100)).take(100)
+
+  return fake_dataset_fn
+
+
+class ParallelInterleaveBenchmark(test.Benchmark):
+  """Benchmarks for `tf.data.experimental.parallel_interleave()`."""
+
+  def _benchmark(self, dataset_fn, iters, num_elements):
+    with ops.Graph().as_default():
+      iterator = dataset_ops.make_one_shot_iterator(dataset_fn())
+      next_element = iterator.get_next()
+      with session.Session() as sess:
+        deltas = []
+        for _ in range(iters):
+          start = time.time()
+          for _ in range(num_elements):
+            sess.run(next_element.op)
+          end = time.time()
+          deltas.append(end - start)
+
+    mean_wall_time = np.mean(deltas) / num_elements
+    self.report_benchmark(iters=iters, wall_time=mean_wall_time)
+
+  def benchmark_sequential_interleave(self):
+
+    def dataset_fn():
+      return dataset_ops.Dataset.range(1).repeat().interleave(
+          _make_fake_dataset_fn(), cycle_length=10)
+
+    self._benchmark(dataset_fn=dataset_fn, iters=10, num_elements=100)
+
+  def benchmark_parallel_interleave_v1(self):
+    """Benchmark for parallel interleave that does not support autotuning."""
+
+    def dataset_fn():
+      return dataset_ops.Dataset.range(1).repeat().apply(
+          interleave_ops.parallel_interleave(
+              _make_fake_dataset_fn(), cycle_length=10))
+
+    self._benchmark(dataset_fn=dataset_fn, iters=100, num_elements=1000)
+
+  def benchmark_parallel_interleave_v2(self):
+    """Benchmark for parallel interleave that supports autotuning."""
+
+    def dataset_fn():
+      return dataset_ops.Dataset.range(1).repeat().interleave(
+          _make_fake_dataset_fn(),
+          cycle_length=10, num_parallel_calls=optimization.AUTOTUNE)
+
+    self._benchmark(dataset_fn=dataset_fn, iters=100, num_elements=1000)
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From 104c9dac9cf69f2b3f409d9b874dc82795c5569d Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Fri, 11 Jan 2019 10:40:40 -0800
Subject: [PATCH 0565/2345] replace keras.optimizers with keras v2 optimizer in
 1.0.

PiperOrigin-RevId: 228905734
---
 tensorflow/python/keras/BUILD                 |   2 +-
 .../feature_columns_integration_test.py       |  13 +-
 .../python/keras/engine/sequential_test.py    |  41 ++--
 .../python/keras/engine/topology_test.py      |   8 +-
 .../keras/engine/training_dataset_test.py     |  26 ++-
 .../python/keras/engine/training_gpu_test.py  |   3 +-
 .../keras/layers/cudnn_recurrent_test.py      |  13 +-
 tensorflow/python/keras/layers/gru_test.py    |  16 +-
 tensorflow/python/keras/layers/lstm_test.py   |  26 +--
 .../python/keras/layers/pooling_test.py       |   3 +-
 .../python/keras/layers/recurrent_test.py     | 204 ++++++++++--------
 .../python/keras/layers/simplernn_test.py     |   6 +-
 .../python/keras/layers/wrappers_test.py      |   7 +-
 tensorflow/python/keras/metrics_test.py       |   3 +-
 .../python/keras/model_subclassing_test.py    |  81 ++++---
 .../python/keras/optimizer_v2/adadelta.py     |   2 +-
 .../python/keras/optimizer_v2/adagrad.py      |   2 +-
 tensorflow/python/keras/optimizer_v2/adam.py  |   2 +-
 .../python/keras/optimizer_v2/adamax.py       |   2 +-
 tensorflow/python/keras/optimizer_v2/ftrl.py  |   2 +-
 .../keras/optimizer_v2/gradient_descent.py    |   2 +-
 tensorflow/python/keras/optimizer_v2/nadam.py |   2 +-
 .../python/keras/optimizer_v2/optimizer_v2.py |   2 +-
 .../python/keras/optimizer_v2/rmsprop.py      |   2 +-
 tensorflow/python/keras/optimizers.py         |  39 +---
 tensorflow/python/keras/optimizers_test.py    |   3 +-
 tensorflow/python/keras/testing_utils.py      |  10 +-
 ...ensorflow.keras.optimizers.-adadelta.pbtxt |  45 +++-
 ...tensorflow.keras.optimizers.-adagrad.pbtxt |  45 +++-
 .../tensorflow.keras.optimizers.-adam.pbtxt   |  45 +++-
 .../tensorflow.keras.optimizers.-adamax.pbtxt |  45 +++-
 .../tensorflow.keras.optimizers.-nadam.pbtxt  |  45 +++-
 ...nsorflow.keras.optimizers.-optimizer.pbtxt |  43 +++-
 ...nsorflow.keras.optimizers.-r-m-sprop.pbtxt |  45 +++-
 .../tensorflow.keras.optimizers.-s-g-d.pbtxt  |  45 +++-
 35 files changed, 596 insertions(+), 284 deletions(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index ef8cb23a09..0ff2aaffcc 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -398,7 +398,7 @@ tf_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:training",
     ],
-    shard_count = 4,
+    shard_count = 8,
     tags = ["notsan"],
 )
 
diff --git a/tensorflow/python/keras/engine/feature_columns_integration_test.py b/tensorflow/python/keras/engine/feature_columns_integration_test.py
index b3f8cfe725..3bed40b084 100644
--- a/tensorflow/python/keras/engine/feature_columns_integration_test.py
+++ b/tensorflow/python/keras/engine/feature_columns_integration_test.py
@@ -27,7 +27,6 @@ from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import metrics as metrics_module
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.platform import test
-from tensorflow.python.training import rmsprop
 
 
 class TestDNNModel(keras.models.Model):
@@ -57,7 +56,7 @@ class FeatureColumnsIntegrationTest(keras_parameterized.TestCase):
         keras.layers.Dense(20, activation='softmax')
     ])
     model.compile(
-        optimizer=rmsprop.RMSPropOptimizer(1e-3),
+        optimizer='rmsprop',
         loss='categorical_crossentropy',
         metrics=['accuracy'],
         run_eagerly=testing_utils.should_run_eagerly())
@@ -79,7 +78,7 @@ class FeatureColumnsIntegrationTest(keras_parameterized.TestCase):
         keras.layers.Dense(20, activation='softmax')
     ])
     model.compile(
-        optimizer=rmsprop.RMSPropOptimizer(1e-3),
+        optimizer='rmsprop',
         loss='categorical_crossentropy',
         metrics=['accuracy'],
         run_eagerly=testing_utils.should_run_eagerly())
@@ -103,7 +102,7 @@ class FeatureColumnsIntegrationTest(keras_parameterized.TestCase):
     dnn_model = TestDNNModel([col_a, col_b], 20)
 
     dnn_model.compile(
-        optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
+        optimizer='rmsprop',
         loss='categorical_crossentropy',
         metrics=['accuracy'],
         run_eagerly=testing_utils.should_run_eagerly())
@@ -124,7 +123,7 @@ class FeatureColumnsIntegrationTest(keras_parameterized.TestCase):
     dnn_model = TestDNNModel([col_a, col_b], 20)
 
     dnn_model.compile(
-        optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
+        optimizer='rmsprop',
         loss='categorical_crossentropy',
         metrics=['accuracy'],
         run_eagerly=testing_utils.should_run_eagerly())
@@ -155,7 +154,7 @@ class FeatureColumnsIntegrationTest(keras_parameterized.TestCase):
 
     model = keras.models.Model([feature_layer], [output])
 
-    optimizer = rmsprop.RMSPropOptimizer(learning_rate=0.001)
+    optimizer = 'rmsprop'
     loss = 'mse'
     loss_weights = [1., 0.5]
     model.compile(
@@ -184,7 +183,7 @@ class FeatureColumnsIntegrationTest(keras_parameterized.TestCase):
 
     model = keras.models.Model([fc1, fc2], [output])
 
-    optimizer = rmsprop.RMSPropOptimizer(learning_rate=0.001)
+    optimizer = 'rmsprop'
     loss = 'mse'
     loss_weights = [1., 0.5]
     model.compile(
diff --git a/tensorflow/python/keras/engine/sequential_test.py b/tensorflow/python/keras/engine/sequential_test.py
index 30a41e39b7..8e5d6fe93a 100644
--- a/tensorflow/python/keras/engine/sequential_test.py
+++ b/tensorflow/python/keras/engine/sequential_test.py
@@ -30,7 +30,6 @@ from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
-from tensorflow.python.training import rmsprop
 
 
 class TestSequential(keras_parameterized.TestCase):
@@ -57,16 +56,20 @@ class TestSequential(keras_parameterized.TestCase):
 
     model = testing_utils.get_small_sequential_mlp(
         num_hidden, num_classes, input_dim)
-    model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3),
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        loss='mse',
+        optimizer='rmsprop',
+        run_eagerly=testing_utils.should_run_eagerly())
     x = np.random.random((batch_size, input_dim))
     y = np.random.random((batch_size, num_classes))
     model.fit(x, y, epochs=1)
     model.pop()
     self.assertEqual(len(model.layers), 1)
     self.assertEqual(model.output_shape, (None, num_hidden))
-    model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3),
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        loss='mse',
+        optimizer='rmsprop',
+        run_eagerly=testing_utils.should_run_eagerly())
     y = np.random.random((batch_size, num_hidden))
     model.fit(x, y, epochs=1)
 
@@ -92,7 +95,7 @@ class TestSequential(keras_parameterized.TestCase):
     model = testing_utils.get_small_sequential_mlp(num_hidden, num_classes)
     model.compile(
         loss='mse',
-        optimizer=rmsprop.RMSPropOptimizer(1e-3),
+        optimizer='rmsprop',
         metrics=[keras.metrics.CategoricalAccuracy()],
         run_eagerly=testing_utils.should_run_eagerly())
     self.assertEqual(len(model.layers), 2)
@@ -117,7 +120,7 @@ class TestSequential(keras_parameterized.TestCase):
     model = testing_utils.get_small_sequential_mlp(num_hidden, num_classes)
     model.compile(
         loss='mse',
-        optimizer=rmsprop.RMSPropOptimizer(1e-3),
+        optimizer='rmsprop',
         metrics=[keras.metrics.CategoricalAccuracy()],
         run_eagerly=testing_utils.should_run_eagerly())
     self.assertEqual(len(model.layers), 2)
@@ -148,7 +151,7 @@ class TestSequential(keras_parameterized.TestCase):
         else:
           model = testing_utils.get_small_sequential_mlp(10, 4, input_dim=3)
         model.compile(
-            optimizer=rmsprop.RMSPropOptimizer(1e-3),
+            optimizer='rmsprop',
             loss='categorical_crossentropy',
             metrics=['accuracy'])
         return model
@@ -264,7 +267,7 @@ class TestSequential(keras_parameterized.TestCase):
     model = testing_utils.get_small_sequential_mlp(num_hidden, num_classes)
     model.compile(
         loss='mse',
-        optimizer=rmsprop.RMSPropOptimizer(1e-3),
+        optimizer='rmsprop',
         metrics=[keras.metrics.CategoricalAccuracy()],
         run_eagerly=testing_utils.should_run_eagerly())
     self.assertFalse(model.built)
@@ -310,8 +313,10 @@ class TestSequential(keras_parameterized.TestCase):
     inner_model = testing_utils.get_small_sequential_mlp(4, 5)
     model.add(inner_model)
 
-    model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3),
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        loss='mse',
+        optimizer='rmsprop',
+        run_eagerly=testing_utils.should_run_eagerly())
     x = np.random.random((2, 6))
     y = np.random.random((2, 5))
     model.fit(x, y, epochs=1)
@@ -352,8 +357,10 @@ class TestSequentialEagerIntegration(keras_parameterized.TestCase):
     model.add(keras.layers.Dense(4, activation='relu'))
     model.add(keras.layers.Dense(5, activation='softmax'))
 
-    model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3),
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        loss='mse',
+        optimizer='rmsprop',
+        run_eagerly=testing_utils.should_run_eagerly())
 
     x = np.random.random((2, 6))
     y = np.random.random((2, 5))
@@ -363,8 +370,10 @@ class TestSequentialEagerIntegration(keras_parameterized.TestCase):
   def test_build_before_fit(self):
     # Fix for b/112433577
     model = testing_utils.get_small_sequential_mlp(4, 5)
-    model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3),
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        loss='mse',
+        optimizer='rmsprop',
+        run_eagerly=testing_utils.should_run_eagerly())
 
     model.build((None, 6))
 
@@ -378,7 +387,7 @@ class TestSequentialEagerIntegration(keras_parameterized.TestCase):
     model = testing_utils.get_small_sequential_mlp(
         num_hidden=10, num_classes=num_classes)
     model.compile(
-        rmsprop.RMSPropOptimizer(learning_rate=0.001),
+        'rmsprop',
         metrics=['acc'],
         weighted_metrics=['mae'],
         loss='categorical_crossentropy',
diff --git a/tensorflow/python/keras/engine/topology_test.py b/tensorflow/python/keras/engine/topology_test.py
index 6c2835aece..08d21193a0 100644
--- a/tensorflow/python/keras/engine/topology_test.py
+++ b/tensorflow/python/keras/engine/topology_test.py
@@ -30,12 +30,10 @@ from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.keras.engine import input_layer as input_layer_lib
 from tensorflow.python.keras.engine import network as network_lib
-from tensorflow.python.keras.optimizer_v2 import gradient_descent
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.platform import test
-from tensorflow.python.training import rmsprop
 
 try:
   import yaml  # pylint:disable=g-import-not-at-top
@@ -860,7 +858,7 @@ class TopologyConstructionTest(keras_parameterized.TestCase):
     x = np.ones((100, 2))
     y = np.ones((100, 2))
     model.compile(
-        optimizer=gradient_descent.SGD(),
+        optimizer='sgd',
         loss='mse',
         run_eagerly=testing_utils.should_run_eagerly())
     loss = model.train_on_batch(x, y)
@@ -909,7 +907,7 @@ class TopologyConstructionTest(keras_parameterized.TestCase):
     model.add(keras.layers.Dense(3))
     model.compile(
         loss='mse',
-        optimizer=gradient_descent.SGD(),
+        optimizer='sgd',
         metrics=['acc'],
         run_eagerly=testing_utils.should_run_eagerly())
 
@@ -1161,7 +1159,7 @@ class DefaultShapeInferenceBehaviorTest(keras_parameterized.TestCase):
     x = keras.layers.wrappers.TimeDistributed(s)(x)
     model = keras.Model(inputs=inputs, outputs=x)
     model.compile(
-        optimizer=rmsprop.RMSPropOptimizer(1e-3),
+        optimizer='rmsprop',
         loss='mse',
         run_eagerly=testing_utils.should_run_eagerly())
 
diff --git a/tensorflow/python/keras/engine/training_dataset_test.py b/tensorflow/python/keras/engine/training_dataset_test.py
index 0ce01ec606..1c67100f20 100644
--- a/tensorflow/python/keras/engine/training_dataset_test.py
+++ b/tensorflow/python/keras/engine/training_dataset_test.py
@@ -33,7 +33,6 @@ from tensorflow.python.keras import testing_utils
 from tensorflow.python.ops.losses import losses_impl
 from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.training.rmsprop import RMSPropOptimizer
 
 
 class TestTrainingWithDatasetIterators(keras_parameterized.TestCase):
@@ -42,7 +41,7 @@ class TestTrainingWithDatasetIterators(keras_parameterized.TestCase):
   @keras_parameterized.run_all_keras_modes
   def test_training_and_eval_methods_on_iterators_single_io(self):
     model = testing_utils.get_small_mlp(1, 4, input_dim=3)
-    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    optimizer = 'rmsprop'
     loss = 'mse'
     metrics = ['mae', metrics_module.CategoricalAccuracy()]
     model.compile(optimizer, loss, metrics=metrics,
@@ -103,7 +102,7 @@ class TestTrainingWithDatasetIterators(keras_parameterized.TestCase):
   @keras_parameterized.run_all_keras_modes
   def test_iterators_running_out_of_data(self):
     model = testing_utils.get_small_mlp(1, 4, input_dim=3)
-    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    optimizer = 'rmsprop'
     loss = 'mse'
     metrics = ['mae']
     model.compile(optimizer, loss, metrics=metrics,
@@ -134,7 +133,7 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
       self.skipTest('b/120673224')
 
     model = testing_utils.get_small_mlp(1, 4, input_dim=3)
-    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    optimizer = 'rmsprop'
     loss = 'mse'
     metrics = ['mae']
     model.compile(optimizer, loss, metrics=metrics,
@@ -156,7 +155,7 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
   @keras_parameterized.run_all_keras_modes
   def test_training_and_eval_methods_on_dataset(self):
     model = testing_utils.get_small_mlp(1, 4, input_dim=3)
-    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    optimizer = 'rmsprop'
     loss = 'mse'
     metrics = ['mae', metrics_module.CategoricalAccuracy()]
     model.compile(optimizer, loss, metrics=metrics,
@@ -231,7 +230,7 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
   @keras_parameterized.run_all_keras_modes
   def test_dataset_with_sample_weights(self):
     model = testing_utils.get_small_mlp(1, 4, input_dim=3)
-    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    optimizer = 'rmsprop'
     loss = 'mse'
     metrics = ['mae', metrics_module.CategoricalAccuracy()]
     model.compile(optimizer, loss, metrics=metrics,
@@ -253,7 +252,7 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
   @keras_parameterized.run_all_keras_modes
   def test_dataset_with_sparse_labels(self):
     model = testing_utils.get_small_mlp(1, 4, input_dim=3)
-    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    optimizer = 'rmsprop'
     for loss in ['sparse_categorical_crossentropy',
                  losses_impl.sparse_softmax_cross_entropy]:
       model.compile(optimizer, loss,
@@ -279,9 +278,8 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
         return keras.backend.sum(inputs) + self.w * 0
 
     model = keras.Sequential([SumLayer(input_shape=(2,))])
-    model.compile(RMSPropOptimizer(learning_rate=0.001),
-                  loss='mae',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        'rmsprop', loss='mae', run_eagerly=testing_utils.should_run_eagerly())
 
     inputs = np.zeros((40, 2), dtype=np.float32)
     inputs[10:20, :] = 2
@@ -299,7 +297,7 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
   def test_dataset_input_shape_validation(self):
     with self.cached_session():
       model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
-      model.compile(optimizer=RMSPropOptimizer(learning_rate=0.001), loss='mse')
+      model.compile(optimizer='rmsprop', loss='mse')
 
       # User forgets to batch the dataset
       inputs = np.zeros((10, 3))
@@ -328,7 +326,7 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
   @keras_parameterized.run_all_keras_modes
   def test_finite_dataset_known_cardinality_no_steps_arg(self):
     model = testing_utils.get_small_mlp(1, 4, input_dim=3)
-    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    optimizer = 'rmsprop'
     model.compile(optimizer, 'mse',
                   run_eagerly=testing_utils.should_run_eagerly())
 
@@ -347,7 +345,7 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
   @keras_parameterized.run_all_keras_modes
   def test_finite_dataset_unknown_cardinality_no_steps_arg(self):
     model = testing_utils.get_small_mlp(1, 4, input_dim=3)
-    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    optimizer = 'rmsprop'
     model.compile(optimizer, 'mse',
                   run_eagerly=testing_utils.should_run_eagerly())
 
@@ -381,7 +379,7 @@ class TestMetricsWithDatasetIterators(keras_parameterized.TestCase):
     model.compile(
         loss='binary_crossentropy',
         metrics=['accuracy', metrics_module.BinaryAccuracy()],
-        optimizer=RMSPropOptimizer(learning_rate=0.001),
+        optimizer='rmsprop',
         run_eagerly=testing_utils.should_run_eagerly())
 
     np.random.seed(123)
diff --git a/tensorflow/python/keras/engine/training_gpu_test.py b/tensorflow/python/keras/engine/training_gpu_test.py
index 45dcfe4399..ddc947339d 100644
--- a/tensorflow/python/keras/engine/training_gpu_test.py
+++ b/tensorflow/python/keras/engine/training_gpu_test.py
@@ -25,7 +25,6 @@ from tensorflow.python.framework import test_util
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras.layers.convolutional import Conv2D
 from tensorflow.python.platform import test
-from tensorflow.python.training import rmsprop
 
 
 class TrainingGPUTest(test.TestCase):
@@ -65,7 +64,7 @@ class TrainingGPUTest(test.TestCase):
                            bias_initializer='ones')(input_tensor)
       simple_model = keras.models.Model(inputs=input_tensor,
                                         outputs=predictions)
-      simple_model.compile(optimizer=rmsprop.RMSPropOptimizer(1e-3), loss=loss)
+      simple_model.compile(optimizer='rmsprop', loss=loss)
       return simple_model
 
     if test.is_gpu_available(cuda_only=True):
diff --git a/tensorflow/python/keras/layers/cudnn_recurrent_test.py b/tensorflow/python/keras/layers/cudnn_recurrent_test.py
index 1e305baefd..c7d8d82ee2 100644
--- a/tensorflow/python/keras/layers/cudnn_recurrent_test.py
+++ b/tensorflow/python/keras/layers/cudnn_recurrent_test.py
@@ -31,7 +31,6 @@ from tensorflow.python.keras.optimizer_v2.rmsprop import RMSprop
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 from tensorflow.python.training import gradient_descent
-from tensorflow.python.training.rmsprop import RMSPropOptimizer
 
 
 @keras_parameterized.run_all_keras_modes
@@ -408,8 +407,7 @@ class CuDNNV1OnlyTest(keras_parameterized.TestCase):
     model.add(
         keras.layers.Bidirectional(
             rnn(output_dim), merge_mode=mode, input_shape=(None, dim)))
-    model.compile(
-        loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
+    model.compile(loss='mse', optimizer='rmsprop')
     model.fit(x, y, epochs=1, batch_size=1)
 
     # test config
@@ -425,8 +423,7 @@ class CuDNNV1OnlyTest(keras_parameterized.TestCase):
             merge_mode=mode,
             input_shape=(None, dim)))
     model.add(keras.layers.Bidirectional(rnn(output_dim), merge_mode=mode))
-    model.compile(
-        loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
+    model.compile(loss='mse', optimizer=R'rmsprop')
     model.fit(x, y, epochs=1, batch_size=1)
 
     # test with functional API
@@ -435,8 +432,7 @@ class CuDNNV1OnlyTest(keras_parameterized.TestCase):
         rnn(output_dim), merge_mode=mode)(
             inputs)
     model = keras.Model(inputs, outputs)
-    model.compile(
-        loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
+    model.compile(loss='mse', optimizer=R'rmsprop')
     model.fit(x, y, epochs=1, batch_size=1)
 
     # Bidirectional and stateful
@@ -445,8 +441,7 @@ class CuDNNV1OnlyTest(keras_parameterized.TestCase):
         rnn(output_dim, stateful=True), merge_mode=mode)(
             inputs)
     model = keras.Model(inputs, outputs)
-    model.compile(
-        loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
+    model.compile(loss='mse', optimizer='rmsprop')
     model.fit(x, y, epochs=1, batch_size=1)
 
   @test_util.run_gpu_only
diff --git a/tensorflow/python/keras/layers/gru_test.py b/tensorflow/python/keras/layers/gru_test.py
index e2b65de661..0e9dde8eae 100644
--- a/tensorflow/python/keras/layers/gru_test.py
+++ b/tensorflow/python/keras/layers/gru_test.py
@@ -26,7 +26,6 @@ from tensorflow.python.framework import test_util as tf_test_util
 from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.platform import test
-from tensorflow.python.training.rmsprop import RMSPropOptimizer
 
 
 @keras_parameterized.run_all_keras_modes
@@ -51,8 +50,8 @@ class GRULayerTest(keras_parameterized.TestCase):
     layer = keras.layers.GRU(units, input_shape=(None, embedding_dim))
     model = keras.models.Sequential()
     model.add(layer)
-    model.compile(RMSPropOptimizer(0.01), 'mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        'rmsprop', 'mse', run_eagerly=testing_utils.should_run_eagerly())
     x = np.random.random((num_samples, timesteps, embedding_dim))
     y = np.random.random((num_samples, units))
     model.train_on_batch(x, y)
@@ -99,8 +98,8 @@ class GRULayerTest(keras_parameterized.TestCase):
                                  reset_after=True)
     output = gru_layer(inputs)
     gru_model = keras.models.Model(inputs, output)
-    gru_model.compile(RMSPropOptimizer(0.01), 'mse',
-                      run_eagerly=testing_utils.should_run_eagerly())
+    gru_model.compile(
+        'rmsprop', 'mse', run_eagerly=testing_utils.should_run_eagerly())
     gru_model.fit(x_train, y_train)
     gru_model.predict(x_train)
 
@@ -112,9 +111,10 @@ class GRULayerTest(keras_parameterized.TestCase):
     model = keras.models.Sequential()
     model.add(keras.layers.Masking(input_shape=(3, 4)))
     model.add(layer_class(units=5, return_sequences=True, unroll=False))
-    model.compile(loss='categorical_crossentropy',
-                  optimizer=RMSPropOptimizer(0.01),
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        loss='categorical_crossentropy',
+        optimizer='rmsprop',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.fit(inputs, targets, epochs=1, batch_size=2, verbose=1)
 
   def test_statefulness_GRU(self):
diff --git a/tensorflow/python/keras/layers/lstm_test.py b/tensorflow/python/keras/layers/lstm_test.py
index 38c0177e39..38e165653e 100644
--- a/tensorflow/python/keras/layers/lstm_test.py
+++ b/tensorflow/python/keras/layers/lstm_test.py
@@ -28,7 +28,6 @@ from tensorflow.python.keras import testing_utils
 from tensorflow.python.platform import test
 from tensorflow.python.training import adam
 from tensorflow.python.training import gradient_descent
-from tensorflow.python.training.rmsprop import RMSPropOptimizer
 
 
 @keras_parameterized.run_all_keras_modes
@@ -68,8 +67,8 @@ class LSTMLayerTest(keras_parameterized.TestCase):
     layer = keras.layers.LSTM(units, input_shape=(None, embedding_dim))
     model = keras.models.Sequential()
     model.add(layer)
-    model.compile(RMSPropOptimizer(0.001), 'mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        'rmsprop', 'mse', run_eagerly=testing_utils.should_run_eagerly())
 
     x = np.random.random((num_samples, timesteps, embedding_dim))
     y = np.random.random((num_samples, units))
@@ -126,9 +125,10 @@ class LSTMLayerTest(keras_parameterized.TestCase):
     model = keras.models.Sequential()
     model.add(keras.layers.Masking(input_shape=(3, 4)))
     model.add(layer_class(units=5, return_sequences=True, unroll=False))
-    model.compile(loss='categorical_crossentropy',
-                  optimizer=RMSPropOptimizer(0.01),
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        loss='categorical_crossentropy',
+        optimizer='rmsprop',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.fit(inputs, targets, epochs=1, batch_size=2, verbose=1)
 
   def test_masking_with_stacking_LSTM(self):
@@ -139,9 +139,10 @@ class LSTMLayerTest(keras_parameterized.TestCase):
     model.add(keras.layers.Masking(input_shape=(3, 4)))
     lstm_cells = [keras.layers.LSTMCell(10), keras.layers.LSTMCell(5)]
     model.add(keras.layers.RNN(lstm_cells, return_sequences=True, unroll=False))
-    model.compile(loss='categorical_crossentropy',
-                  optimizer=RMSPropOptimizer(0.01),
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        loss='categorical_crossentropy',
+        optimizer='rmsprop',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.fit(inputs, targets, epochs=1, batch_size=2, verbose=1)
 
   def test_from_config_LSTM(self):
@@ -246,9 +247,10 @@ class LSTMLayerTest(keras_parameterized.TestCase):
     output = keras.layers.LSTM(units)(inputs, initial_state=initial_state)
 
     model = keras.models.Model([inputs] + initial_state, output)
-    model.compile(loss='categorical_crossentropy',
-                  optimizer=RMSPropOptimizer(0.01),
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        loss='categorical_crossentropy',
+        optimizer='rmsprop',
+        run_eagerly=testing_utils.should_run_eagerly())
 
     inputs = np.random.random((num_samples, timesteps, embedding_dim))
     initial_state = [np.random.random((num_samples, units))
diff --git a/tensorflow/python/keras/layers/pooling_test.py b/tensorflow/python/keras/layers/pooling_test.py
index 936e73ecf9..67df4d7a25 100644
--- a/tensorflow/python/keras/layers/pooling_test.py
+++ b/tensorflow/python/keras/layers/pooling_test.py
@@ -25,7 +25,6 @@ from tensorflow.python.eager import context
 from tensorflow.python.framework import test_util as tf_test_util
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.platform import test
-from tensorflow.python.training import rmsprop
 
 
 class GlobalPoolingTest(test.TestCase):
@@ -48,7 +47,7 @@ class GlobalPoolingTest(test.TestCase):
     model = keras.Sequential()
     model.add(keras.layers.Masking(mask_value=0., input_shape=(3, 4)))
     model.add(keras.layers.GlobalAveragePooling1D())
-    model.compile(loss='mae', optimizer=rmsprop.RMSPropOptimizer(0.001))
+    model.compile(loss='mae', optimizer='rmsprop')
 
     model_input = np.random.random((2, 3, 4))
     model_input[0, 1:, :] = 0
diff --git a/tensorflow/python/keras/layers/recurrent_test.py b/tensorflow/python/keras/layers/recurrent_test.py
index e9bf788740..ddea2f4eae 100644
--- a/tensorflow/python/keras/layers/recurrent_test.py
+++ b/tensorflow/python/keras/layers/recurrent_test.py
@@ -40,7 +40,6 @@ from tensorflow.python.ops import special_math_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variables as variables_lib
 from tensorflow.python.platform import test
-from tensorflow.python.training import rmsprop
 from tensorflow.python.training.checkpointable import util as checkpointable_util
 from tensorflow.python.util import nest
 
@@ -73,9 +72,10 @@ class RNNTest(keras_parameterized.TestCase):
     layer = keras.layers.RNN(cell)
     y = layer(x)
     model = keras.models.Model(x, y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32)))
 
     # Test stacking.
@@ -85,9 +85,10 @@ class RNNTest(keras_parameterized.TestCase):
     layer = keras.layers.RNN(cells)
     y = layer(x)
     model = keras.models.Model(x, y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32)))
 
   def test_minimal_rnn_cell_non_layer_multiple_states(self):
@@ -114,9 +115,10 @@ class RNNTest(keras_parameterized.TestCase):
     layer = keras.layers.RNN(cell)
     y = layer(x)
     model = keras.models.Model(x, y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32)))
 
     # Test stacking.
@@ -128,9 +130,10 @@ class RNNTest(keras_parameterized.TestCase):
     self.assertEqual(layer.cell.output_size, 32)
     y = layer(x)
     model = keras.models.Model(x, y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32)))
 
   def test_minimal_rnn_cell_layer(self):
@@ -169,9 +172,10 @@ class RNNTest(keras_parameterized.TestCase):
     layer = keras.layers.RNN(cell)
     y = layer(x)
     model = keras.models.Model(x, y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32)))
 
     # Test basic case serialization.
@@ -194,9 +198,10 @@ class RNNTest(keras_parameterized.TestCase):
     layer = keras.layers.RNN(cells)
     y = layer(x)
     model = keras.models.Model(x, y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32)))
 
     # Test stacked RNN serialization.
@@ -234,9 +239,10 @@ class RNNTest(keras_parameterized.TestCase):
     y = keras.layers.Lambda(lambda t: array_ops.transpose(t, [1, 0, 2]))(y)
 
     model = keras.models.Model(x, y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         np.zeros((batch, time_step, embedding_dim)),
         np.zeros((batch, time_step, units)))
@@ -253,9 +259,10 @@ class RNNTest(keras_parameterized.TestCase):
 
     y = keras.layers.Lambda(lambda t: array_ops.transpose(t, [1, 0, 2]))(y)
     model = keras.models.Model(x, y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         np.zeros((batch, time_step, embedding_dim)),
         np.zeros((batch, time_step, cell_units[-1])))
@@ -269,9 +276,10 @@ class RNNTest(keras_parameterized.TestCase):
         units, time_major=True, return_sequences=True)(mask)
     y = keras.layers.Lambda(lambda t: array_ops.transpose(t, [1, 0, 2]))(rnn)
     model = keras.models.Model(x, y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         np.zeros((batch, time_step, embedding_dim)),
         np.zeros((batch, time_step, units)))
@@ -282,9 +290,10 @@ class RNNTest(keras_parameterized.TestCase):
     y = rnn_1(x)
 
     model = keras.models.Model(x, y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         np.zeros((batch, time_step, embedding_dim)),
         np.zeros((batch, time_step, units)))
@@ -357,9 +366,10 @@ class RNNTest(keras_parameterized.TestCase):
     y = layer(x, constants=c)
 
     model = keras.models.Model([x, c], y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         [np.zeros((6, 5, 5)), np.zeros((6, 3))],
         np.zeros((6, 32))
@@ -396,9 +406,10 @@ class RNNTest(keras_parameterized.TestCase):
     layer = keras.layers.recurrent.RNN(cells)
     y = layer(x, constants=c)
     model = keras.models.Model([x, c], y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         [np.zeros((6, 5, 5)), np.zeros((6, 3))],
         np.zeros((6, 32))
@@ -411,9 +422,10 @@ class RNNTest(keras_parameterized.TestCase):
     layer = keras.layers.recurrent.RNN(cells)
     y = layer(x, constants=c)
     model = keras.models.Model([x, c], y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         [np.zeros((6, 5, 5)), np.zeros((6, 3))],
         np.zeros((6, 32))
@@ -484,9 +496,10 @@ class RNNTest(keras_parameterized.TestCase):
     layer = keras.layers.RNN(cell)
     y = layer(x, initial_state=s, constants=c)
     model = keras.models.Model([x, s, c], y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         [np.zeros((6, 5, 5)), np.zeros((6, 32)), np.zeros((6, 3))],
         np.zeros((6, 32))
@@ -615,9 +628,10 @@ class RNNTest(keras_parameterized.TestCase):
       layer = keras.layers.RNN(cell)
       y = layer(x)
       model = keras.models.Model(x, y)
-      model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                    loss='mse',
-                    run_eagerly=testing_utils.should_run_eagerly())
+      model.compile(
+          optimizer='rmsprop',
+          loss='mse',
+          run_eagerly=testing_utils.should_run_eagerly())
 
       # Test basic case serialization.
       x_np = np.random.random((6, 5, 5))
@@ -638,9 +652,10 @@ class RNNTest(keras_parameterized.TestCase):
       layer = keras.layers.RNN(cells)
       y = layer(x)
       model = keras.models.Model(x, y)
-      model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                    loss='mse',
-                    run_eagerly=testing_utils.should_run_eagerly())
+      model.compile(
+          optimizer='rmsprop',
+          loss='mse',
+          run_eagerly=testing_utils.should_run_eagerly())
 
       # Test stacked RNN serialization.
       x_np = np.random.random((6, 5, 5))
@@ -706,9 +721,10 @@ class RNNTest(keras_parameterized.TestCase):
     y = np.random.random((2, 2))
     model = keras.models.Sequential()
     model.add(rnn(2))
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.fit(x, y, epochs=1, batch_size=1)
 
     # check whether the model variables are present in the
@@ -740,9 +756,10 @@ class RNNTest(keras_parameterized.TestCase):
                        [None, unit_a, unit_b])
 
     model = keras.models.Model(x, y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         np.zeros((batch, time_step, input_a, input_b)),
         np.zeros((batch, unit_a, unit_b)))
@@ -757,9 +774,10 @@ class RNNTest(keras_parameterized.TestCase):
     layer = keras.layers.RNN(cells)
     y = layer(x)
     model = keras.models.Model(x, y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         np.zeros((batch, time_step, input_a, input_b)),
         np.zeros((batch, unit_a * 4, unit_b * 4)))
@@ -781,9 +799,10 @@ class RNNTest(keras_parameterized.TestCase):
     y = layer(x, initial_state=s)
 
     model = keras.models.Model([x, s], y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch([
         np.zeros((batch, time_step, input_a, input_b)),
         np.zeros((batch, unit_a, unit_b))
@@ -819,9 +838,10 @@ class RNNTest(keras_parameterized.TestCase):
                        [None, state_size])
 
     model = keras.models.Model(x, y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         np.zeros((batch, time_step, input_size)),
         np.zeros((batch, input_size)))
@@ -875,9 +895,10 @@ class RNNTest(keras_parameterized.TestCase):
     self.assertEqual(outputs[1].shape.as_list(), [None, o2, o3])
 
     model = keras.models.Model((input_1, input_2), outputs)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         [np.zeros((batch, t, i1)), np.zeros((batch, t, i2, i3))],
         [np.zeros((batch, o1)), np.zeros((batch, o2, o3))])
@@ -897,9 +918,10 @@ class RNNTest(keras_parameterized.TestCase):
     self.assertEqual(outputs[1].shape.as_list(), [None, o2, o3])
 
     model = keras.models.Model([input_1, input_2], outputs)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         [np.zeros((batch, t, i1)),
          np.zeros((batch, t, i2, i3))],
@@ -926,9 +948,10 @@ class RNNTest(keras_parameterized.TestCase):
     self.assertEqual(s2.shape.as_list(), [None, o2, o3])
 
     model = keras.models.Model([input_1, input_2], [output1, output2])
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         [np.zeros((batch, t, i1)),
          np.zeros((batch, t, i2, i3))],
@@ -951,9 +974,10 @@ class RNNTest(keras_parameterized.TestCase):
     self.assertEqual(s2.shape.as_list(), [None, o2, o3])
 
     model = keras.models.Model([input_1, input_2], [output1, output2])
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         [np.zeros((batch, t, i1)),
          np.zeros((batch, t, i2, i3))],
@@ -985,9 +1009,10 @@ class RNNTest(keras_parameterized.TestCase):
 
     model = keras.models.Model([input_1, input_2, init_s1, init_s2],
                                [output1, output2])
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         [np.zeros((batch, t, i1)),
          np.zeros((batch, t, i2, i3)),
@@ -1017,9 +1042,10 @@ class RNNTest(keras_parameterized.TestCase):
 
     model = keras.models.Model([input_1, input_2, init_s1, init_s2],
                                [output1, output2])
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         [np.zeros((batch, t, i1)),
          np.zeros((batch, t, i2, i3)),
@@ -1093,9 +1119,10 @@ class RNNTest(keras_parameterized.TestCase):
     y, s = keras.layers.RNN(
         Cell(), return_state=True)(x_masked, initial_state=s_0)
     model = keras.models.Model([x, s_0], [y, s])
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
 
     # last time step masked
     x_np = np.array([[[1.], [2.], [0.]]])
@@ -1118,9 +1145,10 @@ class RNNTest(keras_parameterized.TestCase):
       masked_input = mask(x)
       y = layer(masked_input)
       model = keras.models.Model(x, y)
-      model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                    loss='mse',
-                    run_eagerly=testing_utils.should_run_eagerly())
+      model.compile(
+          optimizer='rmsprop',
+          loss='mse',
+          run_eagerly=testing_utils.should_run_eagerly())
 
       np_x = np.ones((6, 5, 5))
       result_1 = model.predict(np_x)
diff --git a/tensorflow/python/keras/layers/simplernn_test.py b/tensorflow/python/keras/layers/simplernn_test.py
index 0e599dade9..0ee074d19c 100644
--- a/tensorflow/python/keras/layers/simplernn_test.py
+++ b/tensorflow/python/keras/layers/simplernn_test.py
@@ -26,7 +26,6 @@ from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.platform import test
 from tensorflow.python.training import gradient_descent
-from tensorflow.python.training.rmsprop import RMSPropOptimizer
 
 
 @keras_parameterized.run_all_keras_modes
@@ -51,7 +50,7 @@ class SimpleRNNLayerTest(keras_parameterized.TestCase):
     layer = keras.layers.SimpleRNN(units, input_shape=(None, embedding_dim))
     model = keras.models.Sequential()
     model.add(layer)
-    model.compile(RMSPropOptimizer(0.01), 'mse')
+    model.compile('rmsprop', 'mse')
     x = np.random.random((num_samples, timesteps, embedding_dim))
     y = np.random.random((num_samples, units))
     model.train_on_batch(x, y)
@@ -107,8 +106,7 @@ class SimpleRNNLayerTest(keras_parameterized.TestCase):
     model = keras.models.Sequential()
     model.add(keras.layers.Masking(input_shape=(3, 4)))
     model.add(layer_class(units=5, return_sequences=True, unroll=False))
-    model.compile(loss='categorical_crossentropy',
-                  optimizer=RMSPropOptimizer(0.01))
+    model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
     model.fit(inputs, targets, epochs=1, batch_size=2, verbose=1)
 
   def test_from_config_SimpleRNN(self):
diff --git a/tensorflow/python/keras/layers/wrappers_test.py b/tensorflow/python/keras/layers/wrappers_test.py
index 46d5487b2c..00e5276d6a 100644
--- a/tensorflow/python/keras/layers/wrappers_test.py
+++ b/tensorflow/python/keras/layers/wrappers_test.py
@@ -27,7 +27,6 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import test_util as tf_test_util
 from tensorflow.python.platform import test
 from tensorflow.python.training.checkpointable import util as checkpointable_util
-from tensorflow.python.training.rmsprop import RMSPropOptimizer
 
 
 class _RNNCellWithConstants(keras.layers.Layer):
@@ -77,7 +76,7 @@ class TimeDistributedTest(test.TestCase):
     model.add(
         keras.layers.TimeDistributed(
             keras.layers.Dense(2), input_shape=(3, 4)))
-    model.compile(optimizer=RMSPropOptimizer(0.01), loss='mse')
+    model.compile(optimizer='rmsprop', loss='mse')
     model.fit(
         np.random.random((10, 3, 4)),
         np.random.random((10, 3, 2)),
@@ -98,7 +97,7 @@ class TimeDistributedTest(test.TestCase):
     model.add(
         keras.layers.TimeDistributed(
             keras.layers.Dense(2), input_shape=(3, 4), batch_size=10))
-    model.compile(optimizer=RMSPropOptimizer(0.01), loss='mse')
+    model.compile(optimizer='rmsprop', loss='mse')
     model.fit(
         np.random.random((10, 3, 4)),
         np.random.random((10, 3, 2)),
@@ -277,7 +276,7 @@ class BidirectionalTest(test.TestCase):
         model.add(
             keras.layers.Bidirectional(
                 rnn(output_dim), merge_mode=mode, input_shape=(timesteps, dim)))
-        model.compile(optimizer=RMSPropOptimizer(0.01), loss='mse')
+        model.compile(optimizer='rmsprop', loss='mse')
         model.fit(x, y, epochs=1, batch_size=1)
 
         # check whether the model variables are present in the
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index f902838c11..6cf61d33d0 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -35,7 +35,6 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 from tensorflow.python.training.checkpointable import util as checkpointable_utils
-from tensorflow.python.training.rmsprop import RMSPropOptimizer
 
 
 @test_util.run_all_in_graph_and_eager_modes
@@ -977,7 +976,7 @@ def _get_model(compile_metrics):
   model.compile(
       loss='mae',
       metrics=compile_metrics,
-      optimizer=RMSPropOptimizer(learning_rate=0.001),
+      optimizer='rmsprop',
       run_eagerly=testing_utils.should_run_eagerly())
   return model
 
diff --git a/tensorflow/python/keras/model_subclassing_test.py b/tensorflow/python/keras/model_subclassing_test.py
index cf64e00d20..6d8ff9d847 100644
--- a/tensorflow/python/keras/model_subclassing_test.py
+++ b/tensorflow/python/keras/model_subclassing_test.py
@@ -36,7 +36,6 @@ from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.platform import test
 from tensorflow.python.training.checkpointable import data_structures
-from tensorflow.python.training.rmsprop import RMSPropOptimizer
 
 try:
   import h5py  # pylint:disable=g-import-not-at-top
@@ -572,7 +571,7 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
                             use_bn=True)
     model.compile(
         loss='mse',
-        optimizer=RMSPropOptimizer(learning_rate=0.001),
+        optimizer='rmsprop',
         metrics=['acc', keras.metrics.CategoricalAccuracy()],
         run_eagerly=testing_utils.should_run_eagerly())
 
@@ -590,10 +589,11 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
     model = MultiIOTestModel(num_classes=num_classes,
                              use_dp=True,
                              use_bn=True)
-    model.compile(loss='mse',
-                  optimizer=RMSPropOptimizer(learning_rate=0.001),
-                  metrics=['acc'],
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        loss='mse',
+        optimizer='rmsprop',
+        metrics=['acc'],
+        run_eagerly=testing_utils.should_run_eagerly())
 
     x1 = np.ones((num_samples, input_dim))
     x2 = np.ones((num_samples, input_dim))
@@ -611,7 +611,8 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
     with self.cached_session():
       model = SimpleTestModel(num_classes=num_classes, use_dp=True, use_bn=True)
       model.compile(
-          loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001),
+          loss='mse',
+          optimizer='rmsprop',
           run_eagerly=testing_utils.should_run_eagerly())
 
       x = np.ones((num_samples, input_dim), dtype=np.float32)
@@ -643,7 +644,8 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
     self.assertEqual(len(model.weights), 0)
 
     model.compile(
-        loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001),
+        loss='mse',
+        optimizer='rmsprop',
         run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch([x1, x2], [y1, y2])
 
@@ -675,7 +677,8 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
 
     model = BNNet()
     model.compile(
-        loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001),
+        loss='mse',
+        optimizer='rmsprop',
         run_eagerly=testing_utils.should_run_eagerly())
     y_ref = model.predict(x)
 
@@ -707,7 +710,8 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
     y = model.predict(x)
     self.assertEqual(np.sum(y), np.sum(x))
     model.compile(
-        loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001),
+        loss='mse',
+        optimizer='rmsprop',
         run_eagerly=testing_utils.should_run_eagerly())
     loss = model.train_on_batch(x, y)
     self.assertGreater(loss, 0.1)
@@ -727,7 +731,8 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
 
     model = MultiIOTestModel(num_classes=num_classes, use_bn=True)
     model.compile(
-        loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001),
+        loss='mse',
+        optimizer='rmsprop',
         run_eagerly=testing_utils.should_run_eagerly())
     model.fit([x1, x2], [y1, y2], epochs=2, batch_size=32, verbose=0)
     model.fit({'input_1': x1, 'input_2': x2},
@@ -738,7 +743,8 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
 
     model = MultiIOTestModel(num_classes=num_classes, use_bn=True)
     model.compile(
-        loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001),
+        loss='mse',
+        optimizer='rmsprop',
         run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch([x1, x2], [y1, y2])
     model.train_on_batch({'input_1': x1, 'input_2': x2},
@@ -758,7 +764,8 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
 
     model = MultiIOTestModel(num_classes=num_classes, use_bn=True)
     model.compile(
-        loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001),
+        loss='mse',
+        optimizer='rmsprop',
         run_eagerly=testing_utils.should_run_eagerly())
     model.evaluate([x1, x2], [y1, y2])
     model.test_on_batch([x1, x2], [y1, y2])
@@ -782,7 +789,8 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
 
     model = MultiIOTestModel(num_classes=num_classes, use_bn=True)
     model.compile(
-        loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001),
+        loss='mse',
+        optimizer='rmsprop',
         run_eagerly=testing_utils.should_run_eagerly())
     model.fit([x1, x2], [y1, y2], epochs=2, batch_size=32, verbose=0)
     y_ref_1, y_ref_2 = model.predict([x1, x2])
@@ -818,10 +826,11 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
     input_dim = 50
 
     model = NestedTestModel1(num_classes=num_classes)
-    model.compile(loss='mse',
-                  optimizer=RMSPropOptimizer(learning_rate=0.001),
-                  metrics=['acc'],
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        loss='mse',
+        optimizer='rmsprop',
+        metrics=['acc'],
+        run_eagerly=testing_utils.should_run_eagerly())
 
     x = np.ones((num_samples, input_dim))
     y = np.zeros((num_samples, num_classes))
@@ -841,10 +850,11 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
     input_dim = 50
 
     model = NestedTestModel2(num_classes=num_classes)
-    model.compile(loss='mse',
-                  optimizer=RMSPropOptimizer(learning_rate=0.001),
-                  metrics=['acc'],
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        loss='mse',
+        optimizer='rmsprop',
+        metrics=['acc'],
+        run_eagerly=testing_utils.should_run_eagerly())
 
     x = np.ones((num_samples, input_dim))
     y = np.zeros((num_samples, num_classes))
@@ -864,10 +874,11 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
     input_dim = 50
 
     model = get_nested_model_3(input_dim=input_dim, num_classes=num_classes)
-    model.compile(loss='mse',
-                  optimizer=RMSPropOptimizer(learning_rate=0.001),
-                  metrics=['acc'],
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        loss='mse',
+        optimizer='rmsprop',
+        metrics=['acc'],
+        run_eagerly=testing_utils.should_run_eagerly())
 
     x = np.ones((num_samples, input_dim))
     y = np.zeros((num_samples, num_classes))
@@ -898,10 +909,11 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
         return self.bn(x)
 
     model = keras.Sequential([Inner()])
-    model.compile(loss='mse',
-                  optimizer=RMSPropOptimizer(learning_rate=0.001),
-                  metrics=['acc'],
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        loss='mse',
+        optimizer='rmsprop',
+        metrics=['acc'],
+        run_eagerly=testing_utils.should_run_eagerly())
 
     x = np.ones((num_samples, input_dim))
     y = np.zeros((num_samples, num_classes))
@@ -938,7 +950,8 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
     y = model.predict(x)
     self.assertEqual(np.sum(y), np.sum(x))
     model.compile(
-        loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001),
+        loss='mse',
+        optimizer='rmsprop',
         run_eagerly=testing_utils.should_run_eagerly())
     loss = model.train_on_batch(x, y)
     self.assertGreater(loss, 0.1)
@@ -956,7 +969,7 @@ class GraphSpecificModelSubclassingTests(test.TestCase):
       model = SimpleTestModel(num_classes=num_classes,
                               use_dp=True,
                               use_bn=True)
-      model.compile(loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
+      model.compile(loss='mse', optimizer='rmsprop')
 
       x = array_ops.ones((num_samples, input_dim))
       y = array_ops.zeros((num_samples, num_classes))
@@ -974,7 +987,7 @@ class GraphSpecificModelSubclassingTests(test.TestCase):
       model = MultiIOTestModel(num_classes=num_classes,
                                use_dp=True,
                                use_bn=True)
-      model.compile(loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
+      model.compile(loss='mse', optimizer='rmsprop')
 
       x1 = array_ops.ones((num_samples, input_dim))
       x2 = array_ops.ones((num_samples, input_dim))
@@ -1062,7 +1075,7 @@ class GraphSpecificModelSubclassingTests(test.TestCase):
       model = MultiIOTestModel(num_classes=num_classes,
                                use_dp=True,
                                use_bn=True)
-      model.compile(loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
+      model.compile(loss='mse', optimizer='rmsprop')
 
       x1 = np.ones((num_samples, input_dim))
       x2 = np.ones((num_samples, input_dim))
diff --git a/tensorflow/python/keras/optimizer_v2/adadelta.py b/tensorflow/python/keras/optimizer_v2/adadelta.py
index cb9b7d1015..b6cba263bc 100644
--- a/tensorflow/python/keras/optimizer_v2/adadelta.py
+++ b/tensorflow/python/keras/optimizer_v2/adadelta.py
@@ -25,7 +25,7 @@ from tensorflow.python.training import training_ops
 from tensorflow.python.util.tf_export import keras_export
 
 
-@keras_export('keras.optimizers.Adadelta', v1=[])
+@keras_export('keras.optimizers.Adadelta')
 class Adadelta(optimizer_v2.OptimizerV2):
   r"""Optimizer that implements the Adadelta algorithm.
 
diff --git a/tensorflow/python/keras/optimizer_v2/adagrad.py b/tensorflow/python/keras/optimizer_v2/adagrad.py
index 7e4a153678..01c722521f 100644
--- a/tensorflow/python/keras/optimizer_v2/adagrad.py
+++ b/tensorflow/python/keras/optimizer_v2/adagrad.py
@@ -30,7 +30,7 @@ from tensorflow.python.ops import state_ops
 from tensorflow.python.util.tf_export import keras_export
 
 
-@keras_export('keras.optimizers.Adagrad', v1=[])
+@keras_export('keras.optimizers.Adagrad')
 class Adagrad(optimizer_v2.OptimizerV2):
   r"""Optimizer that implements the Adagrad algorithm.
 
diff --git a/tensorflow/python/keras/optimizer_v2/adam.py b/tensorflow/python/keras/optimizer_v2/adam.py
index a05bc7e96c..263004c9df 100644
--- a/tensorflow/python/keras/optimizer_v2/adam.py
+++ b/tensorflow/python/keras/optimizer_v2/adam.py
@@ -26,7 +26,7 @@ from tensorflow.python.training import training_ops
 from tensorflow.python.util.tf_export import keras_export
 
 
-@keras_export('keras.optimizers.Adam', v1=[])
+@keras_export('keras.optimizers.Adam')
 class Adam(optimizer_v2.OptimizerV2):
   """Optimizer that implements the Adam algorithm.
 
diff --git a/tensorflow/python/keras/optimizer_v2/adamax.py b/tensorflow/python/keras/optimizer_v2/adamax.py
index 4a4660d328..9a1b63f6c6 100644
--- a/tensorflow/python/keras/optimizer_v2/adamax.py
+++ b/tensorflow/python/keras/optimizer_v2/adamax.py
@@ -27,7 +27,7 @@ from tensorflow.python.training import training_ops
 from tensorflow.python.util.tf_export import keras_export
 
 
-@keras_export('keras.optimizers.Adamax', v1=[])
+@keras_export('keras.optimizers.Adamax')
 class Adamax(optimizer_v2.OptimizerV2):
   """Optimizer that implements the Adamax algorithm.
 
diff --git a/tensorflow/python/keras/optimizer_v2/ftrl.py b/tensorflow/python/keras/optimizer_v2/ftrl.py
index 15515616b2..a86fd8d89d 100644
--- a/tensorflow/python/keras/optimizer_v2/ftrl.py
+++ b/tensorflow/python/keras/optimizer_v2/ftrl.py
@@ -24,7 +24,7 @@ from tensorflow.python.training import training_ops
 from tensorflow.python.util.tf_export import keras_export
 
 
-@keras_export('keras.optimizers.Ftrl', v1=[])
+@keras_export('keras.optimizers.Ftrl')
 class Ftrl(optimizer_v2.OptimizerV2):
   """Optimizer that implements the FTRL algorithm.
 
diff --git a/tensorflow/python/keras/optimizer_v2/gradient_descent.py b/tensorflow/python/keras/optimizer_v2/gradient_descent.py
index adb4ae2bf8..c444f969f6 100644
--- a/tensorflow/python/keras/optimizer_v2/gradient_descent.py
+++ b/tensorflow/python/keras/optimizer_v2/gradient_descent.py
@@ -24,7 +24,7 @@ from tensorflow.python.training import training_ops
 from tensorflow.python.util.tf_export import keras_export
 
 
-@keras_export("keras.optimizers.SGD", v1=[])
+@keras_export("keras.optimizers.SGD")
 class SGD(optimizer_v2.OptimizerV2):
   """Stochastic gradient descent and momentum optimizer.
 
diff --git a/tensorflow/python/keras/optimizer_v2/nadam.py b/tensorflow/python/keras/optimizer_v2/nadam.py
index 7141c6a079..4c94b1358a 100644
--- a/tensorflow/python/keras/optimizer_v2/nadam.py
+++ b/tensorflow/python/keras/optimizer_v2/nadam.py
@@ -26,7 +26,7 @@ from tensorflow.python.ops import state_ops
 from tensorflow.python.util.tf_export import keras_export
 
 
-@keras_export('keras.optimizers.Nadam', v1=[])
+@keras_export('keras.optimizers.Nadam')
 class Nadam(optimizer_v2.OptimizerV2):
   r"""Optimizer that implements the NAdam algorithm.
 
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
index 894af66f5d..91dab9338d 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
@@ -69,7 +69,7 @@ def _deduplicate_indexed_slices(values, indices):
 
 
 @six.add_metaclass(abc.ABCMeta)
-@keras_export("keras.optimizers.Optimizer", v1=[])
+@keras_export("keras.optimizers.Optimizer")
 class OptimizerV2(checkpointable.CheckpointableBase):
   """Updated base class for optimizers.
 
diff --git a/tensorflow/python/keras/optimizer_v2/rmsprop.py b/tensorflow/python/keras/optimizer_v2/rmsprop.py
index 83b81cbe5d..d459d9811e 100644
--- a/tensorflow/python/keras/optimizer_v2/rmsprop.py
+++ b/tensorflow/python/keras/optimizer_v2/rmsprop.py
@@ -29,7 +29,7 @@ from tensorflow.python.training import training_ops
 from tensorflow.python.util.tf_export import keras_export
 
 
-@keras_export("keras.optimizers.RMSprop", v1=[])
+@keras_export("keras.optimizers.RMSprop")
 class RMSprop(optimizer_v2.OptimizerV2):
   r"""Optimizer that implements the RMSprop algorithm.
 
diff --git a/tensorflow/python/keras/optimizers.py b/tensorflow/python/keras/optimizers.py
index 82fb555b57..8c3bf110d4 100644
--- a/tensorflow/python/keras/optimizers.py
+++ b/tensorflow/python/keras/optimizers.py
@@ -22,7 +22,6 @@ from __future__ import print_function
 import six
 from six.moves import zip  # pylint: disable=redefined-builtin
 
-from tensorflow.python import tf2
 from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.framework import ops
 from tensorflow.python.keras import backend as K
@@ -45,7 +44,6 @@ from tensorflow.python.training.checkpointable import base as checkpointable
 from tensorflow.python.util.tf_export import keras_export
 
 
-@keras_export(v1=['keras.optimizers.Optimizer'])
 class Optimizer(object):
   """Abstract optimizer base class.
 
@@ -159,7 +157,6 @@ class Optimizer(object):
     return cls(**config)
 
 
-@keras_export(v1=['keras.optimizers.SGD'])
 class SGD(Optimizer):
   """Stochastic gradient descent optimizer.
 
@@ -224,7 +221,6 @@ class SGD(Optimizer):
     return dict(list(base_config.items()) + list(config.items()))
 
 
-@keras_export(v1=['keras.optimizers.RMSprop'])
 class RMSprop(Optimizer):
   """RMSProp optimizer.
 
@@ -291,7 +287,6 @@ class RMSprop(Optimizer):
     return dict(list(base_config.items()) + list(config.items()))
 
 
-@keras_export(v1=['keras.optimizers.Adagrad'])
 class Adagrad(Optimizer):
   """Adagrad optimizer.
 
@@ -358,7 +353,6 @@ class Adagrad(Optimizer):
     return dict(list(base_config.items()) + list(config.items()))
 
 
-@keras_export(v1=['keras.optimizers.Adadelta'])
 class Adadelta(Optimizer):
   """Adadelta optimizer.
 
@@ -442,7 +436,6 @@ class Adadelta(Optimizer):
     return dict(list(base_config.items()) + list(config.items()))
 
 
-@keras_export(v1=['keras.optimizers.Adam'])
 class Adam(Optimizer):
   """Adam optimizer.
 
@@ -539,7 +532,6 @@ class Adam(Optimizer):
     return dict(list(base_config.items()) + list(config.items()))
 
 
-@keras_export(v1=['keras.optimizers.Adamax'])
 class Adamax(Optimizer):
   """Adamax optimizer from Adam paper's Section 7.
 
@@ -623,7 +615,6 @@ class Adamax(Optimizer):
     return dict(list(base_config.items()) + list(config.items()))
 
 
-@keras_export(v1=['keras.optimizers.Nadam'])
 class Nadam(Optimizer):
   """Nesterov Adam optimizer.
 
@@ -808,27 +799,15 @@ def deserialize(config, custom_objects=None):
   Returns:
       A Keras Optimizer instance.
   """
-  if tf2.enabled():
-    all_classes = {
-        'adadelta': adadelta_v2.Adadelta,
-        'adagrad': adagrad_v2.Adagrad,
-        'adam': adam_v2.Adam,
-        'adamax': adamax_v2.Adamax,
-        'nadam': nadam_v2.Nadam,
-        'rmsprop': rmsprop_v2.RMSprop,
-        'sgd': gradient_descent_v2.SGD
-    }
-  else:
-    all_classes = {
-        'adadelta': Adadelta,
-        'adagrad': Adagrad,
-        'adam': Adam,
-        'adamax': Adamax,
-        'nadam': Nadam,
-        'rmsprop': RMSprop,
-        'sgd': SGD,
-        'tfoptimizer': TFOptimizer
-    }
+  all_classes = {
+      'adadelta': adadelta_v2.Adadelta,
+      'adagrad': adagrad_v2.Adagrad,
+      'adam': adam_v2.Adam,
+      'adamax': adamax_v2.Adamax,
+      'nadam': nadam_v2.Nadam,
+      'rmsprop': rmsprop_v2.RMSprop,
+      'sgd': gradient_descent_v2.SGD
+  }
 
   # Make deserialization case-insensitive for built-in optimizers.
   if config['class_name'].lower() in all_classes:
diff --git a/tensorflow/python/keras/optimizers_test.py b/tensorflow/python/keras/optimizers_test.py
index de24bf0536..606e711483 100644
--- a/tensorflow/python/keras/optimizers_test.py
+++ b/tensorflow/python/keras/optimizers_test.py
@@ -62,7 +62,7 @@ class KerasOptimizersTest(test.TestCase):
     np.testing.assert_equal(
         keras.backend.get_value(model.optimizer.iterations),
         126)  # 63 steps per epoch
-    assert history.history['acc'][-1] >= target
+    self.assertGreaterEqual(history.history['acc'][-1], target)
     config = keras.optimizers.serialize(optimizer)
     optim = keras.optimizers.deserialize(config)
     new_config = keras.optimizers.serialize(optim)
@@ -190,7 +190,6 @@ class KerasOptimizersTest(test.TestCase):
     self.assertIs(graph_weak(), None)
     self.assertIs(optimizer_weak(), None)
 
-  @test_util.run_in_graph_and_eager_modes
   def test_tf_optimizer_iterations(self):
     with self.cached_session():
       optimizer = keras.optimizers.TFOptimizer(AdamOptimizer(0.01))
diff --git a/tensorflow/python/keras/testing_utils.py b/tensorflow/python/keras/testing_utils.py
index 6448f87d25..eff0f39b6d 100644
--- a/tensorflow/python/keras/testing_utils.py
+++ b/tensorflow/python/keras/testing_utils.py
@@ -32,7 +32,6 @@ from tensorflow.python.keras.optimizer_v2 import adamax as adamax_v2
 from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_descent_v2
 from tensorflow.python.keras.optimizer_v2 import nadam as nadam_v2
 from tensorflow.python.keras.optimizer_v2 import rmsprop as rmsprop_v2
-from tensorflow.python.training.rmsprop import RMSPropOptimizer
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util import tf_inspect
 
@@ -169,10 +168,13 @@ def layer_test(layer_cls, kwargs=None, input_shape=None, input_dtype=None,
   # See b/120160788 for more details. This should be mitigated after 2.0.
   model = keras.models.Model(x, layer(x))
   if _thread_local_data.run_eagerly is not None:
-    model.compile(RMSPropOptimizer(0.01), 'mse', weighted_metrics=['acc'],
-                  run_eagerly=should_run_eagerly())
+    model.compile(
+        'rmsprop',
+        'mse',
+        weighted_metrics=['acc'],
+        run_eagerly=should_run_eagerly())
   else:
-    model.compile(RMSPropOptimizer(0.01), 'mse', weighted_metrics=['acc'])
+    model.compile('rmsprop', 'mse', weighted_metrics=['acc'])
   model.train_on_batch(input_data, actual_output)
 
   # test as first layer in Sequential API
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adadelta.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adadelta.pbtxt
index b9ce154bdd..5426269793 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adadelta.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adadelta.pbtxt
@@ -1,15 +1,36 @@
 path: "tensorflow.keras.optimizers.Adadelta"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.optimizers.Adadelta\'>"
-  is_instance: "<class \'tensorflow.python.keras.optimizers.Optimizer\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.adadelta.Adadelta\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "iterations"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'lr\', \'rho\', \'epsilon\', \'decay\'], varargs=None, keywords=kwargs, defaults=[\'1.0\', \'0.95\', \'None\', \'0.0\'], "
+    argspec: "args=[\'self\', \'learning_rate\', \'rho\', \'epsilon\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'0.001\', \'0.95\', \'1e-07\', \'Adadelta\'], "
+  }
+  member_method {
+    name: "add_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\', \'initializer\'], varargs=None, keywords=None, defaults=[\'zeros\'], "
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'trainable\', \'synchronization\', \'aggregation\'], varargs=None, keywords=None, defaults=[\'None\', \'zeros\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply_gradients"
+    argspec: "args=[\'self\', \'grads_and_vars\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'cls\', \'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "get_config"
@@ -19,6 +40,14 @@ tf_class {
     name: "get_gradients"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "get_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "get_updates"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
@@ -27,8 +56,16 @@ tf_class {
     name: "get_weights"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "minimize"
+    argspec: "args=[\'self\', \'loss\', \'var_list\', \'grad_loss\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "set_weights"
     argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "variables"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
 }
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adagrad.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adagrad.pbtxt
index d0dc9e37a3..c39fe6ba4f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adagrad.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adagrad.pbtxt
@@ -1,15 +1,36 @@
 path: "tensorflow.keras.optimizers.Adagrad"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.optimizers.Adagrad\'>"
-  is_instance: "<class \'tensorflow.python.keras.optimizers.Optimizer\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.adagrad.Adagrad\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "iterations"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'lr\', \'epsilon\', \'decay\'], varargs=None, keywords=kwargs, defaults=[\'0.01\', \'None\', \'0.0\'], "
+    argspec: "args=[\'self\', \'learning_rate\', \'initial_accumulator_value\', \'epsilon\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'0.001\', \'0.1\', \'1e-07\', \'Adagrad\'], "
+  }
+  member_method {
+    name: "add_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\', \'initializer\'], varargs=None, keywords=None, defaults=[\'zeros\'], "
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'trainable\', \'synchronization\', \'aggregation\'], varargs=None, keywords=None, defaults=[\'None\', \'zeros\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply_gradients"
+    argspec: "args=[\'self\', \'grads_and_vars\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'cls\', \'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "get_config"
@@ -19,6 +40,14 @@ tf_class {
     name: "get_gradients"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "get_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "get_updates"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
@@ -27,8 +56,16 @@ tf_class {
     name: "get_weights"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "minimize"
+    argspec: "args=[\'self\', \'loss\', \'var_list\', \'grad_loss\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "set_weights"
     argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "variables"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
 }
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adam.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adam.pbtxt
index 06815fa99a..05d46d380b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adam.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adam.pbtxt
@@ -1,15 +1,36 @@
 path: "tensorflow.keras.optimizers.Adam"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.optimizers.Adam\'>"
-  is_instance: "<class \'tensorflow.python.keras.optimizers.Optimizer\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.adam.Adam\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "iterations"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'lr\', \'beta_1\', \'beta_2\', \'epsilon\', \'decay\', \'amsgrad\'], varargs=None, keywords=kwargs, defaults=[\'0.001\', \'0.9\', \'0.999\', \'None\', \'0.0\', \'False\'], "
+    argspec: "args=[\'self\', \'learning_rate\', \'beta_1\', \'beta_2\', \'epsilon\', \'amsgrad\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'0.001\', \'0.9\', \'0.999\', \'1e-07\', \'False\', \'Adam\'], "
+  }
+  member_method {
+    name: "add_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\', \'initializer\'], varargs=None, keywords=None, defaults=[\'zeros\'], "
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'trainable\', \'synchronization\', \'aggregation\'], varargs=None, keywords=None, defaults=[\'None\', \'zeros\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply_gradients"
+    argspec: "args=[\'self\', \'grads_and_vars\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'cls\', \'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "get_config"
@@ -19,6 +40,14 @@ tf_class {
     name: "get_gradients"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "get_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "get_updates"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
@@ -27,8 +56,16 @@ tf_class {
     name: "get_weights"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "minimize"
+    argspec: "args=[\'self\', \'loss\', \'var_list\', \'grad_loss\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "set_weights"
     argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "variables"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
 }
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adamax.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adamax.pbtxt
index 47b55fdb44..88fd96c71d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adamax.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adamax.pbtxt
@@ -1,15 +1,36 @@
 path: "tensorflow.keras.optimizers.Adamax"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.optimizers.Adamax\'>"
-  is_instance: "<class \'tensorflow.python.keras.optimizers.Optimizer\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.adamax.Adamax\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "iterations"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'lr\', \'beta_1\', \'beta_2\', \'epsilon\', \'decay\'], varargs=None, keywords=kwargs, defaults=[\'0.002\', \'0.9\', \'0.999\', \'None\', \'0.0\'], "
+    argspec: "args=[\'self\', \'learning_rate\', \'beta_1\', \'beta_2\', \'epsilon\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'0.001\', \'0.9\', \'0.999\', \'1e-07\', \'Adamax\'], "
+  }
+  member_method {
+    name: "add_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\', \'initializer\'], varargs=None, keywords=None, defaults=[\'zeros\'], "
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'trainable\', \'synchronization\', \'aggregation\'], varargs=None, keywords=None, defaults=[\'None\', \'zeros\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply_gradients"
+    argspec: "args=[\'self\', \'grads_and_vars\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'cls\', \'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "get_config"
@@ -19,6 +40,14 @@ tf_class {
     name: "get_gradients"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "get_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "get_updates"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
@@ -27,8 +56,16 @@ tf_class {
     name: "get_weights"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "minimize"
+    argspec: "args=[\'self\', \'loss\', \'var_list\', \'grad_loss\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "set_weights"
     argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "variables"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
 }
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-nadam.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-nadam.pbtxt
index 8c63a7dda9..7b824e7f0b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-nadam.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-nadam.pbtxt
@@ -1,15 +1,36 @@
 path: "tensorflow.keras.optimizers.Nadam"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.optimizers.Nadam\'>"
-  is_instance: "<class \'tensorflow.python.keras.optimizers.Optimizer\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.nadam.Nadam\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "iterations"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'lr\', \'beta_1\', \'beta_2\', \'epsilon\', \'schedule_decay\'], varargs=None, keywords=kwargs, defaults=[\'0.002\', \'0.9\', \'0.999\', \'None\', \'0.004\'], "
+    argspec: "args=[\'self\', \'learning_rate\', \'beta_1\', \'beta_2\', \'epsilon\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'0.001\', \'0.9\', \'0.999\', \'1e-07\', \'Nadam\'], "
+  }
+  member_method {
+    name: "add_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\', \'initializer\'], varargs=None, keywords=None, defaults=[\'zeros\'], "
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'trainable\', \'synchronization\', \'aggregation\'], varargs=None, keywords=None, defaults=[\'None\', \'zeros\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply_gradients"
+    argspec: "args=[\'self\', \'grads_and_vars\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'cls\', \'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "get_config"
@@ -19,6 +40,14 @@ tf_class {
     name: "get_gradients"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "get_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "get_updates"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
@@ -27,8 +56,16 @@ tf_class {
     name: "get_weights"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "minimize"
+    argspec: "args=[\'self\', \'loss\', \'var_list\', \'grad_loss\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "set_weights"
     argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "variables"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
 }
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-optimizer.pbtxt
index 53d64dae93..58b7f27491 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-optimizer.pbtxt
@@ -1,14 +1,35 @@
 path: "tensorflow.keras.optimizers.Optimizer"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.optimizers.Optimizer\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "iterations"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\'], varargs=None, keywords=kwargs, defaults=None"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\', \'initializer\'], varargs=None, keywords=None, defaults=[\'zeros\'], "
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'trainable\', \'synchronization\', \'aggregation\'], varargs=None, keywords=None, defaults=[\'None\', \'zeros\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply_gradients"
+    argspec: "args=[\'self\', \'grads_and_vars\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'cls\', \'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "get_config"
@@ -18,6 +39,14 @@ tf_class {
     name: "get_gradients"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "get_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "get_updates"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
@@ -26,8 +55,16 @@ tf_class {
     name: "get_weights"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "minimize"
+    argspec: "args=[\'self\', \'loss\', \'var_list\', \'grad_loss\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "set_weights"
     argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "variables"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
 }
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-r-m-sprop.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-r-m-sprop.pbtxt
index a1e9b8cceb..8de796edde 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-r-m-sprop.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-r-m-sprop.pbtxt
@@ -1,15 +1,36 @@
 path: "tensorflow.keras.optimizers.RMSprop"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.optimizers.RMSprop\'>"
-  is_instance: "<class \'tensorflow.python.keras.optimizers.Optimizer\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.rmsprop.RMSprop\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "iterations"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'lr\', \'rho\', \'epsilon\', \'decay\'], varargs=None, keywords=kwargs, defaults=[\'0.001\', \'0.9\', \'None\', \'0.0\'], "
+    argspec: "args=[\'self\', \'learning_rate\', \'rho\', \'momentum\', \'epsilon\', \'centered\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'0.001\', \'0.9\', \'0.0\', \'1e-07\', \'False\', \'RMSprop\'], "
+  }
+  member_method {
+    name: "add_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\', \'initializer\'], varargs=None, keywords=None, defaults=[\'zeros\'], "
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'trainable\', \'synchronization\', \'aggregation\'], varargs=None, keywords=None, defaults=[\'None\', \'zeros\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply_gradients"
+    argspec: "args=[\'self\', \'grads_and_vars\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'cls\', \'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "get_config"
@@ -19,6 +40,14 @@ tf_class {
     name: "get_gradients"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "get_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "get_updates"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
@@ -27,8 +56,16 @@ tf_class {
     name: "get_weights"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "minimize"
+    argspec: "args=[\'self\', \'loss\', \'var_list\', \'grad_loss\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "set_weights"
     argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "variables"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
 }
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-s-g-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-s-g-d.pbtxt
index a67fefb1ba..393eeb3d6c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-s-g-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-s-g-d.pbtxt
@@ -1,15 +1,36 @@
 path: "tensorflow.keras.optimizers.SGD"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.optimizers.SGD\'>"
-  is_instance: "<class \'tensorflow.python.keras.optimizers.Optimizer\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.gradient_descent.SGD\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "iterations"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'lr\', \'momentum\', \'decay\', \'nesterov\'], varargs=None, keywords=kwargs, defaults=[\'0.01\', \'0.0\', \'0.0\', \'False\'], "
+    argspec: "args=[\'self\', \'learning_rate\', \'momentum\', \'nesterov\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'0.001\', \'0.0\', \'False\', \'SGD\'], "
+  }
+  member_method {
+    name: "add_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\', \'initializer\'], varargs=None, keywords=None, defaults=[\'zeros\'], "
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'trainable\', \'synchronization\', \'aggregation\'], varargs=None, keywords=None, defaults=[\'None\', \'zeros\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply_gradients"
+    argspec: "args=[\'self\', \'grads_and_vars\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'cls\', \'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "get_config"
@@ -19,6 +40,14 @@ tf_class {
     name: "get_gradients"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "get_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "get_updates"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
@@ -27,8 +56,16 @@ tf_class {
     name: "get_weights"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "minimize"
+    argspec: "args=[\'self\', \'loss\', \'var_list\', \'grad_loss\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "set_weights"
     argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "variables"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
 }
-- 
GitLab


From 99ed07a8e162e2c30b591ee6f7dcc8e5fb57838d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 10:56:15 -0800
Subject: [PATCH 0566/2345] Switch callers from deprecated
 tf.distribute.Strategy.colocate_vars_with() to its replacement
 tf.distribute.Strategy.extended.colocate_vars_with().

PiperOrigin-RevId: 228908656
---
 .../distribute/python/parameter_server_strategy.py        | 6 +++---
 .../distribute/python/parameter_server_strategy_test.py   | 4 ++--
 tensorflow/python/distribute/parameter_server_strategy.py | 6 +++---
 tensorflow/python/keras/layers/normalization.py           | 8 ++++----
 tensorflow/python/training/optimizer.py                   | 2 +-
 tensorflow/python/training/slot_creator.py                | 4 ++--
 tensorflow/python/training/sync_replicas_optimizer.py     | 2 +-
 7 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy.py b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
index 0cefef7545..0da269204f 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
@@ -70,9 +70,9 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy):
   variables.
 
   3) It is also not recommended to open a colocation scope (i.e. calling
-  `tf.colocate_with`) under the strategy's scope. For colocating variables,
-  use `distribution.colocate_vars_with` instead. Colocation of ops will possibly
-  create conflicts of device assignment.
+  `tf.colocate_with`) under the strategy's scope. For colocating variables, use
+  `strategy.extended.colocate_vars_with` instead. Colocation of ops will
+  possibly create conflicts of device assignment.
   """
 
   def __init__(self, num_gpus_per_worker=0):
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
index 802809e7c7..2d4f8a9be5 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
@@ -190,7 +190,7 @@ class ParameterServerStrategyTestBase(
                          '/job:worker/replica:0/task:0/%s' % last_part_device)
 
         # The colocate_vars_with can override the distribution's device.
-        with d.colocate_vars_with(x):
+        with d.extended.colocate_vars_with(x):
           y = variable_scope.get_variable(
               'y', initializer=20.0,
               aggregation=variable_scope.VariableAggregation.SUM)
@@ -344,7 +344,7 @@ class ParameterServerStrategyTestBase(
         self.assertEqual(e.device, device_util.canonicalize('/device:GPU:2'))
 
         # The colocate_vars_with can override the distribution's device.
-        with d.colocate_vars_with(x):
+        with d.extended.colocate_vars_with(x):
           y = variable_scope.get_variable(
               'y', initializer=20.0,
               aggregation=variable_scope.VariableAggregation.SUM)
diff --git a/tensorflow/python/distribute/parameter_server_strategy.py b/tensorflow/python/distribute/parameter_server_strategy.py
index 9e865082f1..8093f8ccb0 100644
--- a/tensorflow/python/distribute/parameter_server_strategy.py
+++ b/tensorflow/python/distribute/parameter_server_strategy.py
@@ -78,9 +78,9 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy):
   variables.
 
   2) It is also not recommended to open a colocation scope (i.e. calling
-  `tf.colocate_with`) under the strategy's scope. For colocating variables,
-  use `distribution.colocate_vars_with` instead. Colocation of ops will possibly
-  create conflicts of device assignment.
+  `tf.colocate_with`) under the strategy's scope. For colocating variables, use
+  `strategy.extended.colocate_vars_with` instead. Colocation of ops will
+  possibly create conflicts of device assignment.
   """
 
   def __init__(self):
diff --git a/tensorflow/python/keras/layers/normalization.py b/tensorflow/python/keras/layers/normalization.py
index c174c8ddd6..9167a747cd 100644
--- a/tensorflow/python/keras/layers/normalization.py
+++ b/tensorflow/python/keras/layers/normalization.py
@@ -392,16 +392,16 @@ class BatchNormalizationV2(Layer):
               aggregation=tf_variables.VariableAggregation.MEAN)
           return var
 
-        with distribution_strategy_context.get_distribution_strategy(
-        ).colocate_vars_with(self.moving_mean):
+        with distribution_strategy_context.get_strategy(
+        ).extended.colocate_vars_with(self.moving_mean):
           self.renorm_mean = _renorm_variable('renorm_mean', param_shape)
           self.renorm_mean_weight = _renorm_variable('renorm_mean_weight', ())
         # We initialize renorm_stddev to 0, and maintain the (0-initialized)
         # renorm_stddev_weight. This allows us to (1) mix the average
         # stddev with the minibatch stddev early in training, and (2) compute
         # the unbiased average stddev by dividing renorm_stddev by the weight.
-        with distribution_strategy_context.get_distribution_strategy(
-        ).colocate_vars_with(self.moving_variance):
+        with distribution_strategy_context.get_strategy(
+        ).extended.colocate_vars_with(self.moving_variance):
           self.renorm_stddev = _renorm_variable('renorm_stddev', param_shape)
           self.renorm_stddev_weight = _renorm_variable('renorm_stddev_weight',
                                                        ())
diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py
index 8076ed31bf..a591dc9fec 100644
--- a/tensorflow/python/training/optimizer.py
+++ b/tensorflow/python/training/optimizer.py
@@ -816,7 +816,7 @@ class Optimizer(
     if v is None:
       self._maybe_initialize_checkpointable()
       distribution_strategy = distribute_ctx.get_strategy()
-      with distribution_strategy.colocate_vars_with(colocate_with):
+      with distribution_strategy.extended.colocate_vars_with(colocate_with):
         if eager:
           restored_initial_value = self._preload_simple_restoration(
               name=name, shape=None)
diff --git a/tensorflow/python/training/slot_creator.py b/tensorflow/python/training/slot_creator.py
index abe1253b00..0868cfdea8 100644
--- a/tensorflow/python/training/slot_creator.py
+++ b/tensorflow/python/training/slot_creator.py
@@ -122,7 +122,7 @@ def create_slot(primary, val, name, colocate_with_primary=True):
   with variable_scope.variable_scope(None, prefix + "/" + name):
     if colocate_with_primary:
       distribution_strategy = distribution_strategy_context.get_strategy()
-      with distribution_strategy.colocate_vars_with(primary):
+      with distribution_strategy.extended.colocate_vars_with(primary):
         return _create_slot_var(primary, val, "", validate_shape, None, None)
     else:
       return _create_slot_var(primary, val, "", validate_shape, None, None)
@@ -159,7 +159,7 @@ def create_slot_with_initializer(primary, initializer, shape, dtype, name,
   with variable_scope.variable_scope(None, prefix + "/" + name):
     if colocate_with_primary:
       distribution_strategy = distribution_strategy_context.get_strategy()
-      with distribution_strategy.colocate_vars_with(primary):
+      with distribution_strategy.extended.colocate_vars_with(primary):
         return _create_slot_var(primary, initializer, "", validate_shape, shape,
                                 dtype)
     else:
diff --git a/tensorflow/python/training/sync_replicas_optimizer.py b/tensorflow/python/training/sync_replicas_optimizer.py
index 0079ecc98b..21e9a99e7c 100644
--- a/tensorflow/python/training/sync_replicas_optimizer.py
+++ b/tensorflow/python/training/sync_replicas_optimizer.py
@@ -261,7 +261,7 @@ class SyncReplicasOptimizer(optimizer.Optimizer):
     local_anchor = control_flow_ops.no_op()
     # Colocating local_step variable prevents it being placed on the PS.
     distribution_strategy = distribution_strategy_context.get_strategy()
-    with distribution_strategy.colocate_vars_with(local_anchor):
+    with distribution_strategy.extended.colocate_vars_with(local_anchor):
       self._local_step = variable_scope.variable(
           initial_value=0,
           trainable=False,
-- 
GitLab


From 1d1aeb64b91effd4329965cc7954a3e21820c88b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 11:16:53 -0800
Subject: [PATCH 0567/2345] Copy remote tensors to be local in ResolveTensor if
 they are Remote.

PiperOrigin-RevId: 228912733
---
 tensorflow/c/eager/c_api.cc                   | 30 ++++++++++++++-----
 tensorflow/c/eager/c_api_test.cc              |  7 +----
 .../core/common_runtime/eager/tensor_handle.h |  4 +--
 3 files changed, 26 insertions(+), 15 deletions(-)

diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index d5a391a98d..f0408677d4 100755
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -443,15 +443,15 @@ TF_Tensor* TFE_TensorHandleResolve(TFE_TensorHandle* h, TF_Status* status) {
     return nullptr;
   }
   // TODO(agarwal): move this implementation inside TFE_TensorHandle.
-  tensorflow::Device* d = nullptr;
-  tensorflow::Device* op_device = nullptr;
   const tensorflow::Tensor* t = nullptr;
-  status->status = h->handle->TensorAndDevice(&t, &d, &op_device);
-  if (!status->status.ok()) return nullptr;
   tensorflow::TensorHandle* h_cpu = nullptr;
-  if (!IsCPU(d)) {
-    status->status = h->handle->CopyToDevice(
-        h->handle->Context(), h->handle->Context()->HostCPU(), &h_cpu);
+  tensorflow::Device* d = nullptr;
+  tensorflow::Device* op_device = nullptr;
+
+  if (h->handle->IsRemote()) {
+    status->status = EagerCopyToDevice(
+        h->handle, h->handle->Context(),
+        h->handle->Context()->HostCPU()->name().c_str(), &h_cpu);
     if (!status->status.ok()) {
       return nullptr;
     }
@@ -460,6 +460,22 @@ TF_Tensor* TFE_TensorHandleResolve(TFE_TensorHandle* h, TF_Status* status) {
       h_cpu->Unref();
       return nullptr;
     }
+  } else {
+    status->status = h->handle->TensorAndDevice(&t, &d, &op_device);
+    if (!status->status.ok()) return nullptr;
+
+    if (!IsCPU(d)) {
+      status->status = h->handle->CopyToDevice(
+          h->handle->Context(), h->handle->Context()->HostCPU(), &h_cpu);
+      if (!status->status.ok()) {
+        return nullptr;
+      }
+      status->status = h_cpu->TensorAndDevice(&t, &d, &op_device);
+      if (!status->status.ok()) {
+        h_cpu->Unref();
+        return nullptr;
+      }
+    }
   }
   TF_Tensor* retval = tensorflow::TF_TensorFromTensor(*t, status);
   if (h_cpu != nullptr) {
diff --git a/tensorflow/c/eager/c_api_test.cc b/tensorflow/c/eager/c_api_test.cc
index 6b39b79ee8..3d1ca4fb4b 100644
--- a/tensorflow/c/eager/c_api_test.cc
+++ b/tensorflow/c/eager/c_api_test.cc
@@ -175,13 +175,8 @@ void TestRemoteExecute(bool async) {
   TFE_Execute(matmul, &retvals[0], &num_retvals, status);
   EXPECT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
 
-  auto* retval_task0 = TFE_TensorHandleCopyToDevice(
-      retvals[0], ctx, "/job:localhost/replica:0/task:0/device:CPU:0", status);
-  ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
-
-  TF_Tensor* t = TFE_TensorHandleResolve(retval_task0, status);
+  TF_Tensor* t = TFE_TensorHandleResolve(retvals[0], status);
   ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
-  TFE_DeleteTensorHandle(retval_task0);
   float product[4] = {0};
   EXPECT_EQ(sizeof(product), TF_TensorByteSize(t));
   memcpy(&product[0], TF_TensorData(t), TF_TensorByteSize(t));
diff --git a/tensorflow/core/common_runtime/eager/tensor_handle.h b/tensorflow/core/common_runtime/eager/tensor_handle.h
index 1c81087f1f..3eae3c7cc3 100644
--- a/tensorflow/core/common_runtime/eager/tensor_handle.h
+++ b/tensorflow/core/common_runtime/eager/tensor_handle.h
@@ -147,6 +147,8 @@ class TensorHandle : public core::RefCounted {
            (ctx_ == nullptr || ctx_->HostCPU() == device_);
   }
 
+  bool IsRemote();
+
  private:
   // If the contents of the Tensor pointed to by this handle is yet to be
   // computed by a EagerNode, this function will block till that compuatation is
@@ -156,8 +158,6 @@ class TensorHandle : public core::RefCounted {
 
   bool IsReady();
 
-  bool IsRemote();
-
   // Id for the EagerNode that will compute the value pointed to by this handle.
   // If the value is 0, the handle is already ready, but not vice-versa.
   const uint64 node_id_;
-- 
GitLab


From 3c4a3e36f4c1c998fb31e2feca09b793f2b4a550 Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Fri, 11 Jan 2019 11:45:42 -0800
Subject: [PATCH 0568/2345] Whitelists TensorFlow Operations for use with the
 Keras Functional API.

Raw TensorFlow functions can now be used in conjunction with the Keras
Functional API during model creation. This obviates the need for users to create Lambda layers in most cases when using the Functional API. Like Lambda layers, TensorFlow functions that result in Variable creation or assign ops are not supported.

PiperOrigin-RevId: 228918545
---
 tensorflow/python/keras/BUILD                 |  11 ++
 tensorflow/python/keras/engine/base_layer.py  |  87 +++++++++++-
 .../python/keras/engine/base_layer_utils.py   |  98 +++++++++++++
 tensorflow/python/keras/engine/network.py     |  62 ++++++++-
 .../python/keras/layers/serialization.py      |   1 +
 .../keras/layers/tensorflow_op_layer_test.py  | 131 ++++++++++++++++++
 6 files changed, 388 insertions(+), 2 deletions(-)
 create mode 100644 tensorflow/python/keras/layers/tensorflow_op_layer_test.py

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 0ff2aaffcc..719a4a04c4 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -474,6 +474,17 @@ tf_py_test(
     ],
 )
 
+tf_py_test(
+    name = "tensorflow_op_layer_test",
+    size = "medium",
+    srcs = ["layers/tensorflow_op_layer_test.py"],
+    additional_deps = [
+        ":keras",
+        "@absl_py//absl/testing:parameterized",
+        "//tensorflow/python:client_testlib",
+    ],
+)
+
 tf_py_test(
     name = "convolutional_recurrent_test",
     size = "large",
diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index f7c51f7993..47900e7533 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -25,7 +25,9 @@ import itertools
 import numpy as np
 from six.moves import zip  # pylint: disable=redefined-builtin
 
+from tensorflow.core.framework import node_def_pb2
 from tensorflow.python.eager import context
+from tensorflow.python.eager import function
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import func_graph
 from tensorflow.python.framework import ops
@@ -509,7 +511,6 @@ class Layer(checkpointable.CheckpointableBase):
       ValueError: if the layer's `call` method returns None (an invalid value).
     """
     input_list = nest.flatten(inputs)
-
     if context.executing_eagerly():
       # Accept NumPy inputs by converting to Tensors when executing eagerly.
       if all(isinstance(x, (np.ndarray, float, int)) for x in input_list):
@@ -522,6 +523,13 @@ class Layer(checkpointable.CheckpointableBase):
     # models using the functional API).
     build_graph = tf_utils.are_all_symbolic_tensors(input_list)
 
+    if build_graph:
+      # Only create Keras history if at least one tensor originates from a
+      # `keras.Input`. Otherwise this Layer may be being used outside the Keras
+      # framework.
+      if base_layer_utils.uses_keras_input_layers(inputs):
+        base_layer_utils.create_keras_history(inputs)
+
     # Handle Keras mask propagation from previous layer to current layer.
     previous_mask = None
     if build_graph and (not hasattr(self, '_compute_previous_mask') or
@@ -1748,6 +1756,83 @@ class Node(object):
     }
 
 
+class TensorFlowOpLayer(Layer):
+  """Wraps a TensorFlow Operation in a Layer.
+
+  This class is used internally by the Functional API. When a user
+  uses a raw TensorFlow Operation on symbolic tensors originating
+  from an `Input` Layer, the resultant operation will be wrapped
+  with this Layer object in order to make the operation compatible
+  with the Keras API.
+
+  This Layer will create a new, identical operation (except for inputs
+  and outputs) every time it is called. If `run_eagerly` is `True`,
+  the op creation and calculation will happen inside an Eager function.
+
+  Instances of this Layer are created when `autolambda` is called, which
+  is whenever a Layer's `__call__` encounters symbolic inputs that do
+  not have Keras metadata, or when a Network's `__init__` encounters
+  outputs that do not have Keras metadata.
+
+  Attributes:
+    node_def: String, the serialized NodeDef of the Op this layer will wrap.
+    constants: Dict of NumPy arrays, the values of any Tensors needed for this
+      Operation that do not originate from a Keras `Input` Layer. Since all
+      placeholders must come from Keras `Input` Layers, these Tensors must be
+      treated as constant in the Functional API.
+    name: String, the name of the Layer.
+    trainable: Bool, whether this Layer is trainable. Currently Variables are
+      not supported, and so this parameter has no effect.
+    dtype: The default dtype of this Layer. Inherited from `Layer` and has no
+      effect on this class, however is used in `get_config`.
+  """
+
+  def __init__(self,
+               node_def,
+               constants=None,
+               name=None,
+               trainable=True,
+               dtype=None):
+    super(TensorFlowOpLayer, self).__init__(
+        name=name, trainable=trainable, dtype=dtype)
+    self.node_def = node_def_pb2.NodeDef.FromString(node_def)
+    self.constants = constants or {}
+
+  def call(self, inputs):
+    if context.executing_eagerly():
+      return self._defun_call(inputs)
+    return self._make_op(inputs)
+
+  def _make_op(self, inputs):
+    inputs = nest.flatten(inputs)
+    graph = inputs[0].graph
+    with graph.as_default():
+      for index, constant in self.constants.items():
+        constant = ops.convert_to_tensor(constant)
+        inputs.insert(index, constant)
+
+      self.node_def.name = graph.unique_name(self.node_def.name)
+      c_op = ops._create_c_op(graph, self.node_def, inputs, control_inputs=[])
+      op = graph._create_op_from_tf_operation(c_op)
+
+      if len(op.outputs) == 1:
+        return op.outputs[0]
+      return op.outputs
+
+  @function.defun
+  def _defun_call(self, inputs):
+    """Wraps the op creation method in an Eager function for `run_eagerly`."""
+    return self._make_op(inputs)
+
+  def get_config(self):
+    config = super(TensorFlowOpLayer, self).get_config()
+    config.update({
+        'node_def': self.node_def.SerializeToString(),
+        'constants': self.constants
+    })
+    return config
+
+
 def default(method):
   """Decorates a method to detect overrides in subclasses."""
   method._is_default = True
diff --git a/tensorflow/python/keras/engine/base_layer_utils.py b/tensorflow/python/keras/engine/base_layer_utils.py
index f1b1f42c22..0d0257a1dd 100644
--- a/tensorflow/python/keras/engine/base_layer_utils.py
+++ b/tensorflow/python/keras/engine/base_layer_utils.py
@@ -229,3 +229,101 @@ def have_all_keras_metadata(tensors):
 
 def generate_placeholders_from_shape(shape):
   return array_ops.placeholder(shape=shape, dtype=backend.floatx())
+
+
+def create_keras_history(tensors):
+  """Wraps TensorFlow Operations for compatibility with the Functional API.
+
+  This method checks to see if a Tensor in `tensors` is missing Keras metadata
+  and has its origin in a Keras `Input` Layer. If so, this method will replace
+  the raw TensorFlow Operations that created this tensor with
+  `TensorFlowOpLayer`
+  instances that create identical operations.
+
+  Any Tensors not originating from a Keras `Input` Layer will be treated as
+  constants when constructing `TensorFlowOpLayer` instances.
+
+  Arguments:
+    tensors: A structure of Tensors, some of which come from raw TensorFlow
+      operations and need to have Keras metadata assigned to them.
+  """
+
+  _create_keras_history_helper(tensors, set())
+
+
+def _create_keras_history_helper(tensors, processed_ops=None):
+  """Helper method for `create_keras_history`.
+
+  Arguments:
+    tensors: A structure of Tensors for which to create Keras metadata.
+    processed_ops: TensorFlow operations that have already been wrapped in
+      `TensorFlowOpLayer` instances.
+
+  Returns:
+    The updated set of TensorFlow Operations that have been wrapped
+    in `TensorFlowOpLayer` instances.
+  """
+  # Import of `base_layer` needed in order to create `TensorFlowOpLayer`.
+  # Cannot be imported at top because of circular dependencies.
+  # TODO(omalleyt): Resolve circular dependency.
+  from tensorflow.python.keras.engine import base_layer  # pylint: disable=g-import-not-at-top
+  tensor_list = nest.flatten(tensors)
+  for tensor in tensor_list:
+    if getattr(tensor, '_keras_history', None) is not None:
+      continue
+    op = tensor.op  # The Op that created this Tensor.
+    if op not in processed_ops:
+      # Recursively set `_keras_history`.
+      op_inputs = list(op.inputs)
+      constants = {}
+      layer_inputs = []
+      for i, op_input in enumerate(op_inputs):
+        if uses_keras_input_layers(op_input):
+          layer_inputs.append(op_input)
+        else:
+          # Treat any value not originating from a `keras.Input` as
+          # a constant (Variables currently have `Placeholder` op type
+          # when originating from an eager context
+          # so can't be supported.
+          constants[i] = backend.function([], [op_input])([])
+      processed_ops = _create_keras_history_helper(layer_inputs, processed_ops)
+      name = op.name
+      node_def = op.node_def.SerializeToString()
+      op_layer = base_layer.TensorFlowOpLayer(
+          node_def, constants=constants, name=name)
+      op_layer._add_inbound_node(  # pylint: disable=protected-access
+          layer_inputs, op.outputs)
+      processed_ops.update([op])
+  return processed_ops
+
+
+def uses_keras_input_layers(tensors):
+  """Checks if at least one Tensor in `tensors` originates from a Keras `Input`.
+
+  If so, the Functional API is being used.
+
+  Arguments:
+    tensors: An arbitrary nested structure of Tensors.
+
+  Returns:
+    Bool, whether at least one Tensor originates from a Keras `Input`.
+  """
+  checked_tensors = set()
+  input_tensors = nest.flatten(tensors)
+
+  while input_tensors:
+    if any(
+        getattr(tensor, '_keras_history', None) is not None
+        for tensor in input_tensors):
+      return True
+    checked_tensors.update(input_tensors)
+    new_input_tensors = set()
+    for tensor in input_tensors:
+      try:
+        new_input_tensors.update(tensor.op.inputs)
+      except AttributeError:
+        # In case `tensor` is a Variable created in an Eager
+        # context
+        pass
+    input_tensors = list(new_input_tensors - checked_tensors)
+  return False
diff --git a/tensorflow/python/keras/engine/network.py b/tensorflow/python/keras/engine/network.py
index 991950993f..cca9874e8b 100644
--- a/tensorflow/python/keras/engine/network.py
+++ b/tensorflow/python/keras/engine/network.py
@@ -69,8 +69,64 @@ except ImportError:
 class Network(base_layer.Layer):
   """A `Network` is a composition of layers.
 
-  It is the topological form of a "model". A `Model`
+  `Network` is the topological form of a "model". A `Model`
   is simply a `Network` with added training routines.
+
+  Two types of `Networks` exist: Graph Networks and Subclass Networks. Graph
+  networks are used in the Keras Functional and Sequential APIs. Subclassed
+  networks are used when a user subclasses the `Model` class. In general,
+  more Keras features are supported with Graph Networks than with Subclassed
+  Networks, specifically:
+
+  - Model cloning (`keras.models.clone`)
+  - Serialization (`model.get_config()/from_config`, `model.to_json()/to_yaml()`
+  - Whole-model saving (`model.save()`)
+
+  A Graph Network can be instantiated by passing two arguments to `__init__`.
+  The first argument is the `keras.Input` Tensors that represent the inputs
+  to the Network. The second argument specifies the output Tensors that
+  represent the outputs of this Network. Both arguments can be a nested
+  structure of Tensors.
+
+  Example:
+
+  ```
+  inputs = {'x1': keras.Input(shape=(10,)), 'x2': keras.Input(shape=(1,))}
+  t = keras.layers.Dense(1, activation='relu')(inputs['x1'])
+  outputs = keras.layers.Add()([t, inputs['x2'])
+  network = Network(inputs, outputs)
+  ```
+
+  A Graph Network constructed using the Functional API can also include raw
+  TensorFlow functions, with the exception of functions that create Variables
+  or assign ops.
+
+  Example:
+
+  ```
+  inputs = keras.Input(shape=(10,))
+  x = keras.layers.Dense(1)(inputs)
+  outputs = tf.nn.relu(x)
+  network = Network(inputs, outputs)
+  ```
+
+  Subclassed Networks can be instantiated via `name` and (optional) `dynamic`
+  keyword arguments. Subclassed Networks keep track of their Layers, and their
+  `call` method can be overridden. Subclassed Networks are typically created
+  indirectly, by subclassing the `Model` class.
+
+  Example:
+
+  ```
+  class MyModel(keras.Model):
+    def __init__(self):
+      super(MyModel, self).__init__(name='my_model', dynamic=False)
+
+      self.layer1 = keras.layers.Dense(10, activation='relu')
+
+    def call(self, inputs):
+      return self.layer1(inputs)
+  ```
   """
 
   def __init__(self, *args, **kwargs):  # pylint: disable=super-init-not-called
@@ -174,6 +230,10 @@ class Network(base_layer.Layer):
     self._nested_inputs = inputs
     self.inputs = nest.flatten(inputs)
     self.outputs = nest.flatten(outputs)
+
+    if any(not hasattr(tensor, '_keras_history') for tensor in self.outputs):
+      base_layer_utils.create_keras_history(self._nested_outputs)
+
     self._validate_graph_inputs_and_outputs()
 
     self._base_init(name=name)
diff --git a/tensorflow/python/keras/layers/serialization.py b/tensorflow/python/keras/layers/serialization.py
index cef078a28c..e4c508b0f3 100644
--- a/tensorflow/python/keras/layers/serialization.py
+++ b/tensorflow/python/keras/layers/serialization.py
@@ -20,6 +20,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.keras.engine.base_layer import TensorFlowOpLayer
 from tensorflow.python.keras.engine.input_layer import Input
 from tensorflow.python.keras.engine.input_layer import InputLayer
 from tensorflow.python.keras.layers.advanced_activations import *
diff --git a/tensorflow/python/keras/layers/tensorflow_op_layer_test.py b/tensorflow/python/keras/layers/tensorflow_op_layer_test.py
new file mode 100644
index 0000000000..3268bf8b64
--- /dev/null
+++ b/tensorflow/python/keras/layers/tensorflow_op_layer_test.py
@@ -0,0 +1,131 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test for allowing TF ops to work with Keras Functional API."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+import numpy as np
+
+from tensorflow.python import keras
+from tensorflow.python.framework import ops
+from tensorflow.python.keras import keras_parameterized
+from tensorflow.python.keras import testing_utils
+from tensorflow.python.keras.optimizer_v2 import adam
+from tensorflow.python.ops import gen_nn_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.platform import test
+from tensorflow.python.util import nest
+
+
+def _single_op_at_end():
+  inputs = keras.Input(shape=(10,))
+  x = keras.layers.Dense(10)(inputs)
+  outputs = gen_nn_ops.relu(x, name='hey')
+  return inputs, outputs
+
+
+def _multiple_ops_at_end():
+  inputs = keras.Input(shape=(10,))
+  inputs = keras.Input(shape=(10,))
+  x = keras.layers.Dense(10)(inputs)
+  x = gen_nn_ops.relu(x, name='hey')
+  outputs = gen_nn_ops.relu(x, name='hey2')
+  return inputs, outputs
+
+
+def _single_op_in_middle():
+  inputs = keras.Input(shape=(10,))
+  x = keras.layers.Dense(10)(inputs)
+  x = gen_nn_ops.relu(x, name='hey')
+  outputs = keras.layers.Dense(10)(x)
+  return inputs, outputs
+
+
+def _multiple_ops_in_middle():
+  inputs = keras.Input(shape=(10,))
+  x = keras.layers.Dense(10)(inputs)
+  x = gen_nn_ops.relu(x, name='hey')
+  x = gen_nn_ops.relu(x, name='hey2')
+  outputs = keras.layers.Dense(10)(x)
+  return inputs, outputs
+
+
+def _single_standalone_branch():
+  inputs = keras.Input(shape=(10,))
+  x = keras.layers.Dense(10)(inputs)
+  outputs = x * 2
+  return inputs, outputs
+
+
+def _single_op_with_attrs():
+  inputs = keras.Input(shape=(10,))
+  x = math_ops.reduce_mean(inputs, axis=1, keepdims=True)
+  outputs = keras.layers.Dense(10)(x)
+  return inputs, outputs
+
+
+@keras_parameterized.run_all_keras_modes
+class AutoLambdaTest(keras_parameterized.TestCase):
+
+  @parameterized.named_parameters(
+      ('single_op_at_end', _single_op_at_end),
+      ('multiple_ops_at_end', _multiple_ops_at_end),
+      ('single_op_in_middle', _single_op_in_middle),
+      ('multiple_ops_in_middle', _multiple_ops_in_middle),
+      ('single_standalone_branch', _single_standalone_branch),
+      ('single_op_with_attrs', _single_op_with_attrs))
+  def test_autolambda(self, model_fn):
+    inputs, outputs = model_fn()
+    model = keras.Model(inputs, outputs)
+    model.compile(
+        adam.Adam(0.001), 'mse', run_eagerly=testing_utils.should_run_eagerly())
+
+    np_inputs = nest.map_structure(lambda x: np.ones((10, 10), 'float32'),
+                                   inputs)
+    np_outputs = nest.map_structure(lambda x: np.ones((10, 10), 'float32'),
+                                    outputs)
+    model.fit(np_inputs, np_outputs, batch_size=2)
+
+  def test_numerical_correctness_simple(self):
+    x = ops.convert_to_tensor([[-1., 0., -2., 1.]])
+    inputs = keras.Input(shape=(4,))
+    outputs = gen_nn_ops.relu(inputs)
+    model = keras.Model(inputs, outputs)
+    y = self.evaluate(model(x))
+    self.assertAllClose(y, [[0., 0., 0., 1.]])
+
+  def test_numerical_correctness_with_attrs(self):
+    x = ops.convert_to_tensor([[1.5, 1.5], [2.5, 3.5]])
+    inputs = keras.Input(shape=(10,))
+    outputs = math_ops.reduce_mean(inputs, axis=1)
+    model = keras.Model(inputs, outputs)
+    y = self.evaluate(model(x))
+    self.assertAllClose(y, [1.5, 3.])
+
+  def test_serialization(self):
+    x = ops.convert_to_tensor([-1., 0., -2., 1.])
+    inputs = keras.Input(shape=(4,))
+    outputs = gen_nn_ops.relu(inputs)
+    model1 = keras.Model(inputs, outputs)
+    y1 = self.evaluate(model1(x))
+    model2 = model1.from_config(model1.get_config())
+    y2 = self.evaluate(model2(x))
+    self.assertAllClose(y1, y2)
+
+
+if __name__ == '__main__':
+  test.main()
-- 
GitLab


From 80b129e2f6af86bf097861e586b2440ef18b6557 Mon Sep 17 00:00:00 2001
From: James Ring <sjr@google.com>
Date: Fri, 11 Jan 2019 11:49:42 -0800
Subject: [PATCH 0569/2345] Make c_api depend on model_loader_lite instead of
 model_loader

In the near future, some kernels will be implemented against the C API,
and will therefore depend on //tensorflow/c:c_api. c_api currently
depends on saved_models/loader, which depends on all of tensorflow,
which depends on all kernels. This will be a circular dependency when C
kernel implementations are introduced.

PiperOrigin-RevId: 228919312
---
 tensorflow/c/BUILD                              | 2 +-
 tensorflow/core/distributed_runtime/eager/BUILD | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index 9d267e9e59..4e2fe34d28 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -83,7 +83,7 @@ tf_cuda_library(
         ],
         "//conditions:default": [
             ":c_api_internal",
-            "//tensorflow/cc/saved_model:loader",
+            "//tensorflow/cc/saved_model:loader_lite",
             "//tensorflow/cc:gradients",
             "//tensorflow/cc:ops",
             "//tensorflow/cc:grad_ops",
diff --git a/tensorflow/core/distributed_runtime/eager/BUILD b/tensorflow/core/distributed_runtime/eager/BUILD
index 55b2657e74..6f08943e2d 100644
--- a/tensorflow/core/distributed_runtime/eager/BUILD
+++ b/tensorflow/core/distributed_runtime/eager/BUILD
@@ -84,6 +84,7 @@ tf_cc_test(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:tensorflow",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core/common_runtime/eager:tensor_handle",
-- 
GitLab


From 2273a9c4e0d0ec3bd03be25887953d6abf8d09ad Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Fri, 11 Jan 2019 11:50:57 -0800
Subject: [PATCH 0570/2345] Fix pip issues with lazy loader and binary count

PiperOrigin-RevId: 228919530
---
 tensorflow/lite/python/interpreter.py      | 40 ++++++++++++++--------
 tensorflow/lite/tools/pip_package/setup.py | 12 ++++++-
 2 files changed, 37 insertions(+), 15 deletions(-)

diff --git a/tensorflow/lite/python/interpreter.py b/tensorflow/lite/python/interpreter.py
index a6183d13b5..a1325f0b1f 100644
--- a/tensorflow/lite/python/interpreter.py
+++ b/tensorflow/lite/python/interpreter.py
@@ -19,20 +19,32 @@ from __future__ import print_function
 
 import sys
 import numpy as np
-from tensorflow.python.util.lazy_loader import LazyLoader
-from tensorflow.python.util.tf_export import tf_export as _tf_export
-
-# Lazy load since some of the performance benchmark skylark rules
-# break dependencies. Must use double quotes to match code internal rewrite
-# rule.
-# pylint: disable=g-inconsistent-quotes
-_interpreter_wrapper = LazyLoader(
-    "_interpreter_wrapper", globals(),
-    "tensorflow.lite.python.interpreter_wrapper."
-    "tensorflow_wrap_interpreter_wrapper")
-# pylint: enable=g-inconsistent-quotes
-
-del LazyLoader
+
+# pylint: disable=g-import-not-at-top
+try:
+  from tensorflow.python.util.lazy_loader import LazyLoader
+  from tensorflow.python.util.tf_export import tf_export as _tf_export
+
+  # Lazy load since some of the performance benchmark skylark rules
+  # break dependencies. Must use double quotes to match code internal rewrite
+  # rule.
+  # pylint: disable=g-inconsistent-quotes
+  _interpreter_wrapper = LazyLoader(
+      "_interpreter_wrapper", globals(),
+      "tensorflow.lite.python.interpreter_wrapper."
+      "tensorflow_wrap_interpreter_wrapper")
+  # pylint: enable=g-inconsistent-quotes
+
+  del LazyLoader
+except ImportError:
+  # When full Tensorflow Python PIP is not available do not use lazy load
+  # and instead uf the tflite_runtime path.
+  from tflite_runtime.lite.python import interpreter_wrapper as _interpreter_wrapper
+
+  def tf_export_dummy(*x, **kwargs):
+    del x, kwargs
+    return lambda x: x
+  _tf_export = tf_export_dummy
 
 
 @_tf_export('lite.Interpreter')
diff --git a/tensorflow/lite/tools/pip_package/setup.py b/tensorflow/lite/tools/pip_package/setup.py
index 82f9ac5b8f..c5141c1753 100644
--- a/tensorflow/lite/tools/pip_package/setup.py
+++ b/tensorflow/lite/tools/pip_package/setup.py
@@ -57,12 +57,22 @@ RELATIVE_MAKEFILE_PATH = os.path.join(RELATIVE_MAKE_DIR, 'Makefile')
 DOWNLOAD_SCRIPT_PATH = os.path.join(MAKE_DIR, 'download_dependencies.sh')
 
 
+# Check physical memory and if we are on a reasonable non small SOC machine
+# with more than 4GB, use all the CPUs, otherwisxe only 1.
+def get_build_cpus():
+  physical_bytes = os.sysconf('SC_PAGESIZE') * os.sysconf('SC_PHYS_PAGES')
+  if physical_bytes < (1<<30) * 4:
+    return 1
+  else:
+    return multiprocessing.cpu_count()
+
+
 def make_args(target='', quiet=True):
   """Construct make command line."""
   args = (['make', 'SHELL=/bin/bash', '-C', TENSORFLOW_DIR]
           + MAKE_CROSS_OPTIONS +
           ['-f', RELATIVE_MAKEFILE_PATH, '-j',
-           str(multiprocessing.cpu_count())])
+           str(get_build_cpus())])
   if quiet:
     args.append('--quiet')
   if target:
-- 
GitLab


From fa5a0f85cd542c8fc2ec2b96813aafcf8231dd04 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Fri, 11 Jan 2019 11:53:15 -0800
Subject: [PATCH 0571/2345] Add some infrastructure for type registrations, use
 it to revive dictionaries

Lots of boilerplate for handling version numbers. This should make it reasonably easy to maintain backwards compatibility by appending new wrapper versions over time.

The main reason I want this is so we can eventually save a .signatures attribute without special casing, which requires dictionaries be revived as such. But hopefully the same infrastructure will work for Keras Layers. Those will need their own proto format, presumably another oneof in SavedUserObject.

PiperOrigin-RevId: 228919938
---
 tensorflow/python/saved_model/BUILD           |  23 +++
 tensorflow/python/saved_model/load.py         |  36 ++--
 tensorflow/python/saved_model/load_test.py    |  10 ++
 .../python/saved_model/revived_types.py       | 167 ++++++++++++++++++
 .../python/saved_model/revived_types_test.py  | 110 ++++++++++++
 tensorflow/python/saved_model/save.py         |  11 +-
 .../saved_model/saved_object_graph.proto      |   8 +-
 .../python/training/checkpointable/BUILD      |   1 +
 .../checkpointable/data_structures.py         |  12 ++
 9 files changed, 363 insertions(+), 15 deletions(-)
 create mode 100644 tensorflow/python/saved_model/revived_types.py
 create mode 100644 tensorflow/python/saved_model/revived_types_test.py

diff --git a/tensorflow/python/saved_model/BUILD b/tensorflow/python/saved_model/BUILD
index 8a54888582..ebbefd7cde 100644
--- a/tensorflow/python/saved_model/BUILD
+++ b/tensorflow/python/saved_model/BUILD
@@ -271,6 +271,7 @@ py_library(
         ":builder",
         ":constants",
         ":function_serialization",
+        ":revived_types",
         ":saved_object_graph_py",
         ":signature_constants",
         ":signature_def_utils",
@@ -320,6 +321,7 @@ py_library(
         ":constants",
         ":function_deserialization",
         ":loader",
+        ":revived_types",
         ":saved_object_graph_py",
         ":utils",
         "//tensorflow/python:function",
@@ -345,6 +347,27 @@ tf_py_test(
     ],
 )
 
+py_library(
+    name = "revived_types",
+    srcs = [
+        "revived_types.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":saved_object_graph_py",
+    ],
+)
+
+tf_py_test(
+    name = "revived_types_test",
+    srcs = ["revived_types_test.py"],
+    additional_deps = [
+        ":revived_types",
+        ":saved_object_graph_py",
+        "//tensorflow/python:client_testlib",
+    ],
+)
+
 py_library(
     name = "function_serialization",
     srcs = [
diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
index d184133fe9..93a081b62b 100644
--- a/tensorflow/python/saved_model/load.py
+++ b/tensorflow/python/saved_model/load.py
@@ -28,6 +28,7 @@ from tensorflow.python.saved_model import constants
 from tensorflow.python.saved_model import function_deserialization
 from tensorflow.python.saved_model import loader_impl
 from tensorflow.python.saved_model import nested_structure_coder
+from tensorflow.python.saved_model import revived_types
 from tensorflow.python.saved_model import saved_object_graph_pb2
 from tensorflow.python.saved_model import utils_impl as saved_model_utils
 from tensorflow.python.training.checkpointable import tracking
@@ -103,11 +104,17 @@ class _Loader(object):
 
   def _load_all(self):
     """Load all saved objects and wire their properties."""
-    self._nodes = [self._recreate(proto) for proto in self._proto.nodes]
+    self._nodes = []
+    node_setters = []
+    for proto in self._proto.nodes:
+      node, setter = self._recreate(proto)
+      self._nodes.append(node)
+      node_setters.append(setter)
     # After creating the objects, construct the edges between the objects.
-    for obj, object_proto in zip(self._nodes, self._proto.nodes):
+    for obj, object_proto, setter in zip(self._nodes, self._proto.nodes,
+                                         node_setters):
       for reference in object_proto.children:
-        setattr(obj, reference.local_name, self._nodes[reference.node_id])
+        setter(obj, reference.local_name, self._nodes[reference.node_id])
         # Note: if an object has an attribute `__call__` add a class method
         # that allows `obj()` syntax to work. This is done per-instance to
         # allow `callable` to be used to find out if an object is callable.
@@ -137,30 +144,32 @@ class _Loader(object):
 
   def _recreate_user_object(self, proto):
     """Instantiates a SavedUserObject."""
-    del proto
+    looked_up = revived_types.deserialize(proto)
+    if looked_up is None:
+      # Note: each user object has its own class. This allows to make each one
+      # individually callable by adding a `__call__` method to the classes of
+      # the objects instances that have a `__call__` property.
 
-    # Note: each user object has its own class. This allows to make each one
-    # individually callable by adding a `__call__` method to the classes of
-    # the objects instances that have a `__call__` property.
-    class _UserObject(tracking.Checkpointable):
-      pass
+      class _UserObject(tracking.Checkpointable):
+        pass
 
-    return _UserObject()
+      return _UserObject(), setattr
+    return looked_up
 
   def _recreate_asset(self, proto):
     filename = os.path.join(
         saved_model_utils.get_assets_dir(self._export_dir),
         self._asset_file_def[proto.asset_file_def_index].filename)
-    return tracking.TrackableAsset(filename)
+    return tracking.TrackableAsset(filename), setattr
 
   def _recreate_function(self, proto):
     return function_deserialization.recreate_polymorphic_function(
-        proto, self._functions)
+        proto, self._functions), setattr
 
   def _recreate_variable(self, proto):
     # TODO(andresp): Can we use the checkpointed value as initializer?
     dummy_value = init_ops.Zeros(dtype=proto.dtype)(shape=proto.shape)
-    return variables.Variable(dummy_value, trainable=proto.trainable)
+    return variables.Variable(dummy_value, trainable=proto.trainable), setattr
 
 
 def _call_attribute(instance, *args, **kwargs):
@@ -191,3 +200,4 @@ def load(export_dir):
         "Currently only SavedModels exported with `tf.saved_model.save` may be "
         "imported. Other SavedModels may eventually be supported via load().")
   return root
+
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index defd269012..1d6d1fe88c 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -406,6 +406,16 @@ class LoadTest(test.TestCase):
     self.assertAllEqual([2, 4, 6],
                         imported.f(constant_op.constant([1, 2, 3])).numpy())
 
+  def test_dict(self):
+    root = tracking.Checkpointable()
+    root.variables = dict(a=variables.Variable(1.))
+    root.variables["b"] = variables.Variable(2.)
+    root.variables["c"] = 1
+    imported = self.cycle(root)
+    self.assertEqual(1., imported.variables["a"].numpy())
+    self.assertEqual(2., imported.variables["b"].numpy())
+    self.assertEqual(set(["a", "b"]), set(imported.variables.keys()))
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/saved_model/revived_types.py b/tensorflow/python/saved_model/revived_types.py
new file mode 100644
index 0000000000..ae063202bb
--- /dev/null
+++ b/tensorflow/python/saved_model/revived_types.py
@@ -0,0 +1,167 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Handles types registrations for tf.saved_model.load."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.core.framework import versions_pb2
+from tensorflow.python.saved_model import saved_object_graph_pb2
+
+
+class VersionedTypeRegistration(object):
+  """Holds information about one version of a revived type."""
+
+  def __init__(self, object_factory, version, min_producer_version,
+               min_consumer_version, bad_consumers=None, setter=setattr):
+    """Identify a revived type version.
+
+    Args:
+      object_factory: A callable which takes a SavedUserObject proto and returns
+        a checkpointable object. Dependencies are added later via `setter`.
+      version: An integer, the producer version of this wrapper type. When
+        making incompatible changes to a wrapper, add a new
+        `VersionedTypeRegistration` with an incremented `version`. The most
+        recent version will be saved, and all registrations with a matching
+        identifier will be searched for the highest compatible version to use
+        when loading.
+      min_producer_version: The minimum producer version number required to use
+        this `VersionedTypeRegistration` when loading a proto.
+      min_consumer_version: `VersionedTypeRegistration`s with a version number
+        less than `min_consumer_version` will not be used to load a proto saved
+        with this object. `min_consumer_version` should be set to the lowest
+        version number which can successfully load protos saved by this
+        object. If no matching registration is available on load, the object
+        will be revived with a generic checkpointable type.
+
+        `min_consumer_version` and `bad_consumers` are a blunt tool, and using
+        them will generally break forward compatibility: previous versions of
+        TensorFlow will revive newly saved objects as opaque checkpointable
+        objects rather than wrapped objects. When updating wrappers, prefer
+        saving new information but preserving compatibility with previous
+        wrapper versions. They are, however, useful for ensuring that
+        previously-released buggy wrapper versions degrade gracefully rather
+        than throwing exceptions when presented with newly-saved SavedModels.
+      bad_consumers: A list of consumer versions which are incompatible (in
+        addition to any version less than `min_consumer_version`).
+      setter: A callable with the same signature as `setattr` to use when adding
+        dependencies to generated objects.
+    """
+    self.setter = setter
+    self.identifier = None  # Set after registration
+    self._object_factory = object_factory
+    self.version = version
+    self._min_consumer_version = min_consumer_version
+    self._min_producer_version = min_producer_version
+    if bad_consumers is None:
+      bad_consumers = []
+    self._bad_consumers = bad_consumers
+
+  def to_proto(self):
+    """Create a SavedUserObject proto."""
+    # For now wrappers just use dependencies to save their state, so the
+    # SavedUserObject doesn't depend on the object being saved.
+    # TODO(allenl): Add a wrapper which uses its own proto.
+    return saved_object_graph_pb2.SavedUserObject(
+        identifier=self.identifier,
+        version=versions_pb2.VersionDef(
+            producer=self.version,
+            min_consumer=self._min_consumer_version,
+            bad_consumers=self._bad_consumers))
+
+  def from_proto(self, proto):
+    """Recreate a checkpointable object from a SavedUserObject proto."""
+    return self._object_factory(proto)
+
+  def should_load(self, proto):
+    """Checks if this object should load the SavedUserObject `proto`."""
+    if proto.identifier != self.identifier:
+      return False
+    if self.version < proto.version.min_consumer:
+      return False
+    if proto.version.producer < self._min_producer_version:
+      return False
+    for bad_version in proto.version.bad_consumers:
+      if self.version == bad_version:
+        return False
+    return True
+
+
+# string identifier -> (predicate, [VersionedTypeRegistration])
+_REVIVED_TYPE_REGISTRY = {}
+_TYPE_IDENTIFIERS = []
+
+
+def register_revived_type(identifier, predicate, versions):
+  """Register a type for revived objects.
+
+  Args:
+    identifier: A unique string identifying this class of objects.
+    predicate: A Boolean predicate for this registration. Takes a
+      checkpointable object as an argument. If True, `type_registration` may be
+      used to save and restore the object.
+    versions: A list of `VersionedTypeRegistration` objects.
+  """
+  # Keep registrations in order of version. We always use the highest matching
+  # version (respecting the min consumer version and bad consumers).
+  versions.sort(key=lambda reg: reg.version, reverse=True)
+  if not versions:
+    raise AssertionError("Need at least one version of a registered type.")
+  version_numbers = set()
+  for registration in versions:
+    # Copy over the identifier for use in generating protos
+    registration.identifier = identifier
+    if registration.version in version_numbers:
+      raise AssertionError(
+          "Got multiple registrations with version {} for type {}".format(
+              registration.version, identifier))
+    version_numbers.add(registration.version)
+  if identifier in _REVIVED_TYPE_REGISTRY:
+    raise AssertionError(
+        "Duplicate registrations for type {}".format(identifier))
+
+  _REVIVED_TYPE_REGISTRY[identifier] = (predicate, versions)
+  _TYPE_IDENTIFIERS.append(identifier)
+
+
+def serialize(obj):
+  """Create a SavedUserObject from a checkpointable object."""
+  for identifier in _TYPE_IDENTIFIERS:
+    predicate, versions = _REVIVED_TYPE_REGISTRY[identifier]
+    if predicate(obj):
+      # Always uses the most recent version to serialize.
+      return versions[0].to_proto()
+  return None
+
+
+def deserialize(proto):
+  """Create a checkpointable object from a SavedUserObject proto.
+
+  Args:
+    proto: A SavedUserObject to deserialize.
+
+  Returns:
+    A tuple of (checkpointable, assignment_fn) where assignment_fn has the same
+    signature as setattr and should be used to add dependencies to
+    `checkpointable` when they are available.
+  """
+  _, type_registrations = _REVIVED_TYPE_REGISTRY.get(
+      proto.identifier, (None, None))
+  if type_registrations is not None:
+    for type_registration in type_registrations:
+      if type_registration.should_load(proto):
+        return (type_registration.from_proto(proto), type_registration.setter)
+  return None
diff --git a/tensorflow/python/saved_model/revived_types_test.py b/tensorflow/python/saved_model/revived_types_test.py
new file mode 100644
index 0000000000..98e10cfec0
--- /dev/null
+++ b/tensorflow/python/saved_model/revived_types_test.py
@@ -0,0 +1,110 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for revived type matching."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.core.framework import versions_pb2
+from tensorflow.python.platform import test
+from tensorflow.python.saved_model import revived_types
+from tensorflow.python.saved_model import saved_object_graph_pb2
+from tensorflow.python.training.checkpointable import tracking
+
+
+class CustomTestClass(tracking.Checkpointable):
+
+  def __init__(self, version):
+    self.version = version
+
+
+revived_types.register_revived_type(
+    "test_type",
+    lambda obj: isinstance(obj, CustomTestClass),
+    versions=[
+        revived_types.VersionedTypeRegistration(
+            object_factory=lambda _: CustomTestClass(1),
+            version=1, min_producer_version=1,
+            min_consumer_version=1),
+        revived_types.VersionedTypeRegistration(
+            object_factory=lambda _: CustomTestClass(2),
+            version=2, min_producer_version=2, min_consumer_version=1),
+        revived_types.VersionedTypeRegistration(
+            object_factory=lambda _: CustomTestClass(3),
+            version=3, min_producer_version=3, min_consumer_version=2),
+        revived_types.VersionedTypeRegistration(
+            object_factory=lambda _: CustomTestClass(4),
+            version=4, min_producer_version=4, min_consumer_version=2,
+            bad_consumers=[3]),
+    ]
+)
+
+
+class RegistrationMatchingTest(test.TestCase):
+
+  def test_save_typecheck(self):
+    self.assertIs(revived_types.serialize(tracking.Checkpointable()), None)
+
+  def test_load_identifier_not_found(self):
+    nothing_matches = revived_types.deserialize(
+        saved_object_graph_pb2.SavedUserObject(
+            identifier="_unregistered_type",
+            version=versions_pb2.VersionDef(
+                producer=1,
+                min_consumer=1,
+                bad_consumers=[])))
+    self.assertIs(nothing_matches, None)
+
+  def test_most_recent_version_saved(self):
+    serialized = revived_types.serialize(CustomTestClass(None))
+    self.assertEqual([3], serialized.version.bad_consumers)
+    deserialized, _ = revived_types.deserialize(serialized)
+    self.assertIsInstance(deserialized, CustomTestClass)
+    self.assertEqual(4, deserialized.version)
+
+  def test_min_consumer_version(self):
+    nothing_matches = revived_types.deserialize(
+        saved_object_graph_pb2.SavedUserObject(
+            identifier="test_type",
+            version=versions_pb2.VersionDef(
+                producer=5,
+                min_consumer=5,
+                bad_consumers=[])))
+    self.assertIs(nothing_matches, None)
+
+  def test_bad_versions(self):
+    deserialized, _ = revived_types.deserialize(
+        saved_object_graph_pb2.SavedUserObject(
+            identifier="test_type",
+            version=versions_pb2.VersionDef(
+                producer=5,
+                min_consumer=1,
+                bad_consumers=[4, 3])))
+    self.assertEqual(2, deserialized.version)
+
+  def test_min_producer_version(self):
+    deserialized, _ = revived_types.deserialize(
+        saved_object_graph_pb2.SavedUserObject(
+            identifier="test_type",
+            version=versions_pb2.VersionDef(
+                producer=3,
+                min_consumer=0,
+                bad_consumers=[])))
+    self.assertEqual(3, deserialized.version)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index 9db6d03ed0..0f98766553 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -22,6 +22,7 @@ import collections
 import functools
 import os
 
+from tensorflow.core.framework import versions_pb2
 from tensorflow.core.protobuf import meta_graph_pb2
 from tensorflow.core.protobuf import saved_model_pb2
 from tensorflow.python.eager import context
@@ -38,6 +39,7 @@ from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.saved_model import builder_impl
 from tensorflow.python.saved_model import constants
 from tensorflow.python.saved_model import function_serialization
+from tensorflow.python.saved_model import revived_types
 from tensorflow.python.saved_model import saved_object_graph_pb2
 from tensorflow.python.saved_model import signature_constants
 from tensorflow.python.saved_model import signature_def_utils
@@ -617,7 +619,14 @@ def _write_object_proto(obj, proto, asset_file_def_index, node_ids):
     proto.function.CopyFrom(
         function_serialization.serialize_polymorphic_function(obj, node_ids))
   else:
-    proto.user_object.SetInParent()
+    registered_type_proto = revived_types.serialize(obj)
+    if registered_type_proto is None:
+      # Fallback for types with no matching registration
+      registered_type_proto = saved_object_graph_pb2.SavedUserObject(
+          identifier="_generic_user_object",
+          version=versions_pb2.VersionDef(
+              producer=1, min_consumer=1, bad_consumers=[]))
+    proto.user_object.CopyFrom(registered_type_proto)
 
 
 @tf_export("saved_model.save", v1=["saved_model.experimental.save"])
diff --git a/tensorflow/python/saved_model/saved_object_graph.proto b/tensorflow/python/saved_model/saved_object_graph.proto
index a870eee383..a322726e1e 100644
--- a/tensorflow/python/saved_model/saved_object_graph.proto
+++ b/tensorflow/python/saved_model/saved_object_graph.proto
@@ -3,6 +3,7 @@ syntax = "proto3";
 import "tensorflow/core/protobuf/checkpointable_object_graph.proto";
 import "tensorflow/core/framework/tensor_shape.proto";
 import "tensorflow/core/framework/types.proto";
+import "tensorflow/core/framework/versions.proto";
 import "tensorflow/python/saved_model/struct.proto";
 
 option cc_enable_arenas = true;
@@ -62,7 +63,12 @@ message SavedObject {
 //
 // This object cannot be evaluated as a tensor, and therefore cannot be bound
 // to an input of a function.
-message SavedUserObject {}
+message SavedUserObject {
+  // Corresponds to a registration of the type to use in the loading program.
+  string identifier = 1;
+  // Version information from the producer of this SavedUserObject.
+  VersionDef version = 2;
+}
 
 // A SavedAsset represents a file in a SavedModel.
 //
diff --git a/tensorflow/python/training/checkpointable/BUILD b/tensorflow/python/training/checkpointable/BUILD
index 855dc4fb68..a39462732f 100644
--- a/tensorflow/python/training/checkpointable/BUILD
+++ b/tensorflow/python/training/checkpointable/BUILD
@@ -74,6 +74,7 @@ py_library(
     deps = [
         ":base",
         ":layer_utils",
+        "//tensorflow/python/saved_model:revived_types",
     ],
 )
 
diff --git a/tensorflow/python/training/checkpointable/data_structures.py b/tensorflow/python/training/checkpointable/data_structures.py
index 817552f326..c46af68155 100644
--- a/tensorflow/python/training/checkpointable/data_structures.py
+++ b/tensorflow/python/training/checkpointable/data_structures.py
@@ -19,10 +19,12 @@ from __future__ import print_function
 
 import collections
 import copy
+import operator
 
 import six
 
 from tensorflow.python.ops import variables
+from tensorflow.python.saved_model import revived_types
 from tensorflow.python.training.checkpointable import base
 from tensorflow.python.training.checkpointable import layer_utils
 
@@ -727,3 +729,13 @@ class _DictWrapper(Mapping, collections.MutableMapping):
   def update(self, *args, **kwargs):
     for key, value in dict(*args, **kwargs).items():
       self[key] = value
+
+revived_types.register_revived_type(
+    "checkpointable_dict_wrapper",
+    lambda obj: isinstance(obj, _DictWrapper),
+    versions=[revived_types.VersionedTypeRegistration(
+        object_factory=lambda _: _DictWrapper({}),
+        version=1,
+        min_producer_version=1,
+        min_consumer_version=1,
+        setter=operator.setitem)])
-- 
GitLab


From d2d8584845253fe30e1a43394f281d045b23492c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 11:58:08 -0800
Subject: [PATCH 0572/2345] Automated rollback of commit
 196715b526a695678891ec11d13b7290b3f6861a

PiperOrigin-RevId: 228920631
---
 tensorflow/core/kernels/BUILD                  | 1 +
 tensorflow/core/kernels/fused_batch_norm_op.cc | 1 +
 2 files changed, 2 insertions(+)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 0792b398f1..4db0505a8f 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -3820,6 +3820,7 @@ NN_DEPS = [
     ":conv_2d",
     ":eigen_contraction_kernel",
     ":ops_util",
+    ":pooling_ops",
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
diff --git a/tensorflow/core/kernels/fused_batch_norm_op.cc b/tensorflow/core/kernels/fused_batch_norm_op.cc
index 48b339508b..dbd3bb05db 100644
--- a/tensorflow/core/kernels/fused_batch_norm_op.cc
+++ b/tensorflow/core/kernels/fused_batch_norm_op.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #if GOOGLE_CUDA
 #define EIGEN_USE_GPU
 #include "tensorflow/core/kernels/conv_2d.h"
+#include "tensorflow/core/kernels/conv_ops_gpu.h"
 #include "tensorflow/core/util/stream_executor_util.h"
 #endif
 
-- 
GitLab


From 3567f27bf3fe437def0588c92ebe8ef982e74e47 Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Fri, 11 Jan 2019 12:03:13 -0800
Subject: [PATCH 0573/2345] Support parallel_iterations in while_v2.

PiperOrigin-RevId: 228921645
---
 .../core/common_runtime/lower_while_op.cc     | 28 +++++++++++++------
 .../common_runtime/lower_while_op_test.cc     |  2 ++
 tensorflow/core/ops/functional_ops.cc         |  1 +
 .../kernel_tests/control_flow_ops_py_test.py  |  2 --
 tensorflow/python/ops/control_flow_ops.py     |  1 +
 tensorflow/python/ops/while_v2.py             |  4 +++
 6 files changed, 28 insertions(+), 10 deletions(-)

diff --git a/tensorflow/core/common_runtime/lower_while_op.cc b/tensorflow/core/common_runtime/lower_while_op.cc
index 8b68c31a72..7552838ca1 100644
--- a/tensorflow/core/common_runtime/lower_while_op.cc
+++ b/tensorflow/core/common_runtime/lower_while_op.cc
@@ -53,9 +53,10 @@ using NodeOut = NodeBuilder::NodeOut;
 class LowerWhileHelper {
  public:
   static Status Run(Node* while_op, const string& cond_fn_name,
-                    const string& body_fn_name, Graph* graph,
-                    const FunctionLibraryDefinition& flib) {
-    LowerWhileHelper helper(while_op, cond_fn_name, body_fn_name, graph, flib);
+                    const string& body_fn_name, int parallel_iterations,
+                    Graph* graph, const FunctionLibraryDefinition& flib) {
+    LowerWhileHelper helper(while_op, cond_fn_name, body_fn_name,
+                            parallel_iterations, graph, flib);
     return helper.RunInternal();
   }
 
@@ -64,8 +65,8 @@ class LowerWhileHelper {
   // and body functions named `cond_fn_name` and `body_fn_name` respectively in
   // the given graph.
   LowerWhileHelper(Node* while_op, const string& cond_fn_name,
-                   const string& body_fn_name, Graph* graph,
-                   const FunctionLibraryDefinition& flib);
+                   const string& body_fn_name, int parallel_iterations,
+                   Graph* graph, const FunctionLibraryDefinition& flib);
 
   Status RunInternal();
 
@@ -132,6 +133,8 @@ class LowerWhileHelper {
   const FunctionLibraryDefinition& flib_;
   // Name of the `while_op_`.
   string name_;
+  // Max number of parallel_iterations for the while loop.
+  const int parallel_iterations_;
 
   NodeDebugInfo debug_info_;
   NodeBuilder cond_call_builder_;
@@ -147,12 +150,14 @@ class LowerWhileHelper {
 };
 
 LowerWhileHelper::LowerWhileHelper(Node* while_op, const string& cond_fn_name,
-                                   const string& body_fn_name, Graph* graph,
+                                   const string& body_fn_name,
+                                   int parallel_iterations, Graph* graph,
                                    const FunctionLibraryDefinition& flib)
     : while_op_(while_op),
       graph_(graph),
       flib_(flib),
       name_(while_op->name()),
+      parallel_iterations_(parallel_iterations),
       debug_info_(*while_op_),
       cond_call_builder_(NewName("cond"), cond_fn_name, graph->op_registry(),
                          &debug_info_),
@@ -194,6 +199,7 @@ Status LowerWhileHelper::CreateEnterNodes() {
                                    graph_->op_registry(), &debug_info_)
                            .Input(NodeOut(edge->src(), edge->src_output()))
                            .Attr("frame_name", name_)
+                           .Attr("parallel_iterations", parallel_iterations_)
                            .Finalize(graph_, &enter_node));
     enter_nodes_[edge->dst_input()] = enter_node;
   }
@@ -392,9 +398,15 @@ Status RewriteWhileNode(Node* n, Graph* g,
   if (body_attr == nullptr) {
     return errors::InvalidArgument("While body function missing");
   }
+  const AttrValue* parallel_iterations_attr =
+      n->attrs().Find("parallel_iterations");
+  if (parallel_iterations_attr == nullptr) {
+    return errors::InvalidArgument("parallel_iterations attr missing");
+  }
 
-  TF_RETURN_IF_ERROR(LowerWhileHelper::Run(n, cond_attr->func().name(),
-                                           body_attr->func().name(), g, flib));
+  TF_RETURN_IF_ERROR(LowerWhileHelper::Run(
+      n, cond_attr->func().name(), body_attr->func().name(),
+      parallel_iterations_attr->i(), g, flib));
   g->RemoveNode(n);
 
   return Status::OK();
diff --git a/tensorflow/core/common_runtime/lower_while_op_test.cc b/tensorflow/core/common_runtime/lower_while_op_test.cc
index 24fd4ed5bb..fcb10bc75d 100644
--- a/tensorflow/core/common_runtime/lower_while_op_test.cc
+++ b/tensorflow/core/common_runtime/lower_while_op_test.cc
@@ -66,6 +66,7 @@ TEST(LowerWhileOpTest, Simple) {
                    .Attr("T", {DT_INT32})
                    .Attr("cond", cond_func)
                    .Attr("body", body_func)
+                   .Attr("parallel_iterations", 100)
                    .Attr(LowerIfWhilePass::kLowerUsingSwitchMergeAttr, true)
                    .Finalize(root.graph(), &while_node));
   TF_ASSERT_OK(root.DoShapeInference(while_node));
@@ -97,6 +98,7 @@ TEST(LowerWhileOpTest, Simple) {
   for (const auto* op : graph->op_nodes()) {
     if (op->IsEnter()) {
       ++enter_count;
+      ASSERT_EQ(op->attrs().Find("parallel_iterations")->i(), 100);
     }
     if (op->IsExit()) {
       ++exit_count;
diff --git a/tensorflow/core/ops/functional_ops.cc b/tensorflow/core/ops/functional_ops.cc
index 5e0bdd888c..be440ed728 100644
--- a/tensorflow/core/ops/functional_ops.cc
+++ b/tensorflow/core/ops/functional_ops.cc
@@ -170,6 +170,7 @@ REGISTER_OP("While")
     .Attr("cond: func")
     .Attr("body: func")
     .Attr("output_shapes: list(shape) = []")
+    .Attr("parallel_iterations: int = 10")
     .SetIsStateful()
     .SetShapeFn([](shape_inference::InferenceContext* c) {
       std::vector<PartialTensorShape> output_shapes;
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 2949038728..25977510fb 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -1065,7 +1065,6 @@ class ControlFlowTest(test.TestCase):
       self.assertEqual(printed.contents(), "D\nD\n")
 
   # Microbenchmark: 256,000 iterations/s.
-  @test_util.disable_control_flow_v2("b/116630618 (Times out)")
   def testWhile_1(self):
     with self.cached_session():
       n = constant_op.constant(0)
@@ -2245,7 +2244,6 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose(216.0, grad_a_val)
       self.assertAllClose(81.0, grad_v_val)
 
-  @test_util.disable_control_flow_v2("b/116630618 (parallel_iters: times out)")
   @test_util.run_deprecated_v1
   def testWhileGrad_Mul(self):
     self._testWhileGrad_Mul(use_gpu=False, p_iters=1)
diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py
index 2b481bded5..3b3bfbbfc2 100644
--- a/tensorflow/python/ops/control_flow_ops.py
+++ b/tensorflow/python/ops/control_flow_ops.py
@@ -3492,6 +3492,7 @@ def while_loop(cond,
         body,
         loop_vars,
         shape_invariants=shape_invariants,
+        parallel_iterations=parallel_iterations,
         maximum_iterations=maximum_iterations,
         name=name,
         return_same_structure=return_same_structure)
diff --git a/tensorflow/python/ops/while_v2.py b/tensorflow/python/ops/while_v2.py
index cefadfb098..50011e34b1 100644
--- a/tensorflow/python/ops/while_v2.py
+++ b/tensorflow/python/ops/while_v2.py
@@ -57,6 +57,7 @@ def while_loop(cond,
                body,
                loop_vars,
                shape_invariants=None,
+               parallel_iterations=10,
                maximum_iterations=None,
                name=None,
                return_same_structure=True):
@@ -215,6 +216,7 @@ def while_loop(cond,
         util.create_new_tf_function(cond_graph),
         util.create_new_tf_function(body_graph),
         output_shapes=[t.shape for t in body_graph.outputs],
+        parallel_iterations=parallel_iterations,
         name=scope)
 
     _copy_handle_data(body_graph.outputs, outputs)
@@ -257,6 +259,7 @@ def _WhileGrad(op, *grads):  # pylint: disable=invalid-name
 
   maximum_iterations = op.get_attr(
       "_maximum_iterations") if _is_in_xla_context() else None
+  parallel_iterations = op.get_attr("parallel_iterations")
   assert not _is_in_xla_context() or maximum_iterations is not None
   maximum_iterations = _validate_and_convert_to_tensor(maximum_iterations)
 
@@ -325,6 +328,7 @@ def _WhileGrad(op, *grads):  # pylint: disable=invalid-name
       util.create_new_tf_function(cond_grad_graph),
       util.create_new_tf_function(body_grad_graph),
       output_shapes=[t.shape for t in body_grad_graph.outputs],
+      parallel_iterations=parallel_iterations,
       name="%s_grad" % while_op.name)
 
   _copy_handle_data(body_grad_graph.outputs, outputs)
-- 
GitLab


From 36d26bdbe45a2539de079aaf94a6c1c72bae5342 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Fri, 11 Jan 2019 12:09:44 -0800
Subject: [PATCH 0574/2345] [TF:XLA] Bump open source llvm revision to r350887

PiperOrigin-RevId: 228922762
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index ca4feb691e..bea57b8ed7 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -500,11 +500,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "b85f0571c5d2997142cd379eda3def286b26671fabb8453c5d56fee6e0698f52",
-        strip_prefix = "llvm-437f3bdb2355b0f453a4b4824e4b74ed6b6a73a5",
+        sha256 = "1b5e02d72413ae0de88a4457ff549434ab9c7b4f583f628eb6f29a9d199e280f",
+        strip_prefix = "llvm-ec5ab87c7698f6ede8905f833db67cd16449e360",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/437f3bdb2355b0f453a4b4824e4b74ed6b6a73a5.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/437f3bdb2355b0f453a4b4824e4b74ed6b6a73a5.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/ec5ab87c7698f6ede8905f833db67cd16449e360.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/ec5ab87c7698f6ede8905f833db67cd16449e360.tar.gz",
         ],
     )
 
-- 
GitLab


From 2193a41cdde2a1e9ea870e5c8df4dc84f0aabf92 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 12:18:27 -0800
Subject: [PATCH 0575/2345] Update ops-related pbtxt files.

PiperOrigin-RevId: 228924111
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 40 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 |  7 ++++
 2 files changed, 47 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 15a4a10189..8705492c66 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -80322,6 +80322,46 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "While"
+  input_arg {
+    name: "input"
+    type_list_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_list_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "cond"
+    type: "func"
+  }
+  attr {
+    name: "body"
+    type: "func"
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    default_value {
+      list {
+      }
+    }
+  }
+  attr {
+    name: "parallel_iterations"
+    type: "int"
+    default_value {
+      i: 10
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "WholeFileReader"
   output_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 9949c00aa4..012c03a834 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -38987,6 +38987,13 @@ op {
       }
     }
   }
+  attr {
+    name: "parallel_iterations"
+    type: "int"
+    default_value {
+      i: 10
+    }
+  }
   is_stateful: true
 }
 op {
-- 
GitLab


From 5a4bf97a3167ef6a725e689ef5b6cb1a7bedfa73 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Fri, 11 Jan 2019 12:31:33 -0800
Subject: [PATCH 0576/2345] Improve GDB interaction with the XLA CPU JIT

 - Register JITted code with GDB
 - Emit unwind information for JITed functions

With these changes GDB can unwind through JITted frames.

PiperOrigin-RevId: 228925859
---
 .../xla/service/cpu/simple_orc_jit.cc         | 29 ++++++++++++++++++-
 .../compiler/xla/service/cpu/simple_orc_jit.h | 15 ++++++++++
 .../compiler/xla/service/llvm_ir/llvm_util.cc |  4 +++
 3 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
index 296f39a485..12ab6360c5 100644
--- a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
+++ b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
@@ -116,13 +116,26 @@ SimpleOrcJIT::SimpleOrcJIT(const llvm::TargetOptions& target_options,
                 orc_jit_memory_mapper::GetInstance());
             result.Resolver = symbol_resolver_;
             return result;
+          },
+          /*NotifyLoaded=*/
+          llvm::orc::LegacyRTDyldObjectLinkingLayer::NotifyLoadedFtor(),
+          /*NotifyFinalized=*/
+          [this](VModuleKeyT, const llvm::object::ObjectFile& object,
+                 const llvm::RuntimeDyld::LoadedObjectInfo& object_info) {
+            this->NotifyObjectFinalized(object, object_info);
+          },
+          /*NotifyFreed=*/
+          [this](VModuleKeyT, const llvm::object::ObjectFile& object) {
+            this->NotifyObjectFreed(object);
           }),
       compile_layer_(object_layer_,
                      CompilerFunctor(target_machine_.get(), &disassembler_,
                                      opt_level, optimize_for_size,
                                      enable_fast_math, disable_expensive_passes,
                                      std::move(pre_optimization_hook),
-                                     std::move(post_optimization_hook))) {
+                                     std::move(post_optimization_hook))),
+      gdb_jit_event_listener_(
+          llvm::JITEventListener::createGDBRegistrationListener()) {
   VLOG(1) << "CPU target: " << target_machine_->getTargetCPU().str()
           << " features: " << target_machine_->getTargetFeatureString().str();
 }
@@ -147,6 +160,20 @@ llvm::JITSymbol SimpleOrcJIT::ResolveRuntimeSymbol(const std::string& name) {
   return symbol_info;
 }
 
+void SimpleOrcJIT::NotifyObjectFinalized(
+    const llvm::object::ObjectFile& object,
+    const llvm::RuntimeDyld::LoadedObjectInfo& object_info) {
+  uint64_t key = static_cast<uint64_t>(
+      reinterpret_cast<uintptr_t>(object.getData().data()));
+  gdb_jit_event_listener_->notifyObjectLoaded(key, object, object_info);
+}
+
+void SimpleOrcJIT::NotifyObjectFreed(const llvm::object::ObjectFile& object) {
+  uint64_t key = static_cast<uint64_t>(
+      reinterpret_cast<uintptr_t>(object.getData().data()));
+  gdb_jit_event_listener_->notifyFreeingObject(key);
+}
+
 SimpleOrcJIT::VModuleKeyT SimpleOrcJIT::AddModule(
     std::unique_ptr<llvm::Module> module) {
   auto key = execution_session_.allocateVModule();
diff --git a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.h b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.h
index 78406ba143..3307c2f93d 100644
--- a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.h
+++ b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.h
@@ -21,6 +21,7 @@ limitations under the License.
 #include <vector>
 
 #include "llvm/ADT/Triple.h"
+#include "llvm/ExecutionEngine/JITEventListener.h"
 #include "llvm/ExecutionEngine/Orc/Core.h"
 #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
@@ -99,6 +100,11 @@ class SimpleOrcJIT {
  private:
   llvm::JITSymbol ResolveRuntimeSymbol(const std::string& name);
 
+  void NotifyObjectFinalized(
+      const llvm::object::ObjectFile& object,
+      const llvm::RuntimeDyld::LoadedObjectInfo& object_info);
+  void NotifyObjectFreed(const llvm::object::ObjectFile& object);
+
   std::vector<VModuleKeyT> module_keys_;
   std::unique_ptr<llvm::TargetMachine> target_machine_;
   const Disassembler disassembler_;
@@ -107,6 +113,15 @@ class SimpleOrcJIT {
   std::shared_ptr<llvm::orc::SymbolResolver> symbol_resolver_;
   ObjLayerT object_layer_;
   CompileLayerT compile_layer_;
+
+  // Non owning pointer to a JIT event listener that registers the JIT events
+  // with an attached GDB.
+  //
+  // Note: we get a pointer to this event listener using
+  // `createGDBRegistrationListener` which makes it look like we're supposed to
+  // free this, but the function is poorly named and really just returns a
+  // pointer to a static object.
+  llvm::JITEventListener* gdb_jit_event_listener_;
 };
 
 }  // namespace cpu
diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc
index 75c4245f5f..83af5d2543 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc
@@ -621,6 +621,10 @@ llvm::Function* CreateFunction(llvm::FunctionType* function_type,
   function->setCallingConv(llvm::CallingConv::C);
   function->addFnAttr("no-frame-pointer-elim", "false");
 
+  // Generate unwind information so that GDB can crawl through the stack frames
+  // created by the JIT compiled code.
+  function->setHasUWTable();
+
   if (enable_fast_math) {
     function->addFnAttr("unsafe-fp-math", "true");
     function->addFnAttr("no-infs-fp-math", "true");
-- 
GitLab


From 25abe10651f6f0a4177d28874bcd11567708e02f Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Fri, 11 Jan 2019 12:34:47 -0800
Subject: [PATCH 0577/2345] [XLA:CPU] Implement batch dot

For now we lower a batch dot to N non-batch dot operations.  In the future we
may consider a more direct lowering.  This current CL is still better than what
we had because:

 - We don't blow up compile time by unrolling the batch dot
 - We avoid the slice and concat in the cases where they wouldn't get optimized away

After this CL DotDecomposer can be simplified, but I'll wait a bit and do that
in a separate CL to make it easier to roll this CL back if necessary.

PiperOrigin-RevId: 228926335
---
 tensorflow/compiler/xla/service/cpu/BUILD     |   2 +
 .../compiler/xla/service/cpu/cpu_compiler.cc  |   2 +-
 .../xla/service/cpu/dot_op_emitter.cc         | 179 +++++++++++++++++-
 .../compiler/xla/service/cpu/ir_emitter.cc    |   4 -
 .../compiler/xla/service/llvm_ir/ir_array.cc  |   4 +-
 .../compiler/xla/service/llvm_ir/ir_array.h   |  12 +-
 6 files changed, 185 insertions(+), 18 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index a197bdddc8..3f501fb64e 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -94,6 +94,7 @@ cc_library(
         ":target_machine_features",
         "@com_google_absl//absl/types:span",
         "//tensorflow/compiler/tf2xla:cpu_function_runtime",
+        "//tensorflow/compiler/xla/service:hlo_casting_utils",
         "//tensorflow/compiler/xla/service:map_inliner",
         "//tensorflow/compiler/xla/service:hlo_get_dimension_size_rewriter",
         "//tensorflow/compiler/xla/service:scatter_expander",
@@ -399,6 +400,7 @@ cc_library(
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/compiler/xla/service:hlo_casting_utils",
         "//tensorflow/compiler/xla/service:hlo_module_config",
         "//tensorflow/compiler/xla/service/llvm_ir:ir_array",
         "//tensorflow/compiler/xla/service/llvm_ir:kernel_support_library",
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
index a6d92ce10e..23c568236b 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
@@ -258,7 +258,7 @@ Status CpuCompiler::RunHloPassesThroughLayoutAssn(
   // pass.
   pipeline.AddPass<CallInliner>();
   pipeline.AddPass<BatchDotSimplification>();
-  pipeline.AddPass<DotDecomposer>();
+  pipeline.AddPass<DotDecomposer>(/*decompose_batch_dot=*/false);
   auto cost_model = [](HloInstruction* conv) {
     // We need a cost model for CPUs. Currently, do nothing.
     return false;
diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
index b018e0cd46..dba28aa51a 100644
--- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
@@ -28,6 +28,8 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/cpu/target_machine_features.h"
 #include "tensorflow/compiler/xla/service/cpu/tiled_dot_emitter.h"
 #include "tensorflow/compiler/xla/service/cpu/vector_support_library.h"
+#include "tensorflow/compiler/xla/service/hlo_casting_utils.h"
+#include "tensorflow/compiler/xla/service/hlo_instructions.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/kernel_support_library.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h"
@@ -57,6 +59,8 @@ struct DotInfo {
   Shape result_shape;
   DotDimensionNumbers dim_nums;
 
+  DotInfo() = default;
+
   explicit DotInfo(const HloInstruction& instr) {
     CHECK_EQ(instr.opcode(), HloOpcode::kDot);
     lhs_shape = instr.operand(0)->shape();
@@ -843,6 +847,163 @@ DotImplementationStrategy GetDotImplementationStrategy(
 
   return DotImplementationStrategy::kNaiveLlvmIr;
 }
+
+Status EmitNonBatchDotOperation(
+    DotInfo dot_info, string hlo_name, const llvm_ir::IrArray& target_array,
+    const llvm_ir::IrArray& lhs_array, const llvm_ir::IrArray& rhs_array,
+    const llvm_ir::IrArray* addend_array,
+    llvm::Value* executable_run_options_value, llvm::IRBuilder<>* b,
+    const HloModuleConfig& hlo_module_config,
+    const TargetMachineFeatures& target_machine_features) {
+  PrimitiveType type = target_array.GetShape().element_type();
+  TF_RET_CHECK(F16 == type || F32 == type || F64 == type || C64 == type);
+  DotOpEmitter dot_emitter(std::move(dot_info), std::move(hlo_name),
+                           target_array, lhs_array, rhs_array, addend_array,
+                           executable_run_options_value, b, hlo_module_config,
+                           target_machine_features);
+  return dot_emitter.Emit();
+}
+
+Shape DropFirstDim(const Shape& shape) {
+  absl::Span<int64 const> array_shape_dims(shape.dimensions());
+  array_shape_dims.remove_prefix(1);
+  return ShapeUtil::MakeShapeWithDescendingLayout(shape.element_type(),
+                                                  array_shape_dims);
+}
+
+Shape CollapseFirstNDims(const Shape& shape, int64 n) {
+  absl::Span<int64 const> input_shape_dims(shape.dimensions());
+  int64 prefix_dim =
+      std::accumulate(input_shape_dims.begin(), input_shape_dims.begin() + n,
+                      1ll, std::multiplies<int64>());
+  DimensionVector result_dims;
+  result_dims.push_back(prefix_dim);
+  std::copy(input_shape_dims.begin() + n, input_shape_dims.end(),
+            std::back_inserter(result_dims));
+  return ShapeUtil::MakeShapeWithDescendingLayout(shape.element_type(),
+                                                  result_dims);
+}
+
+llvm_ir::IrArray CollapseFirstNDims(llvm::IRBuilder<>* b,
+                                    const llvm_ir::IrArray& array, int64 n) {
+  llvm::Module* module = b->GetInsertBlock()->getParent()->getParent();
+  const Shape& shape = array.GetShape();
+  CHECK(shape.has_layout() &&
+        LayoutUtil::IsMonotonicWithDim0Major(shape.layout()));
+  CHECK_GE(shape.dimensions_size(), n);
+  Shape new_shape = CollapseFirstNDims(shape, n);
+  llvm::Value* new_value = b->CreateBitCast(
+      array.GetBasePointer(),
+      llvm_ir::ShapeToIrType(new_shape, module)->getPointerTo());
+  return llvm_ir::IrArray(new_value, std::move(new_shape));
+}
+
+Status ValidateDotDimensionNumbers(const DotDimensionNumbers& dim_numbers) {
+  // Checks some invariants that do not hold in general, but DotDecomposer
+  // should have established for us.  This is just a debugging aid.
+  TF_RET_CHECK(dim_numbers.lhs_contracting_dimensions_size() == 1);
+  std::vector<int64> batch_dim_numbers(dim_numbers.lhs_batch_dimensions_size());
+  absl::c_iota(batch_dim_numbers, 0);
+  TF_RET_CHECK(
+      absl::c_equal(batch_dim_numbers, dim_numbers.lhs_batch_dimensions()));
+  TF_RET_CHECK(
+      absl::c_equal(batch_dim_numbers, dim_numbers.rhs_batch_dimensions()));
+  return Status::OK();
+}
+
+// Slice out the inner array at batch index `batch_index` from `outer_array`.
+llvm_ir::IrArray SliceOutInnerArray(llvm_ir::IrArray outer_array,
+                                    llvm::Value* batch_index,
+                                    llvm::IRBuilder<>* b) {
+  llvm::Module* module = b->GetInsertBlock()->getParent()->getParent();
+
+  Shape inner_shape = DropFirstDim(outer_array.GetShape());
+  llvm_ir::IrArray::Index slice_index(b->getInt64Ty());
+  slice_index.push_back(batch_index);
+  slice_index.InsertAt(
+      /*index=*/1, outer_array.GetShape().dimensions_size() - 1,
+      b->getInt64(0));
+  llvm::Value* slice_ptr = outer_array.EmitArrayElementAddress(slice_index, b);
+  llvm::Type* slice_ptr_type =
+      llvm_ir::ShapeToIrType(inner_shape, module)->getPointerTo();
+  return llvm_ir::IrArray(b->CreateBitCast(slice_ptr, slice_ptr_type),
+                          std::move(inner_shape));
+}
+
+Status EmitBatchDotOperation(
+    const HloInstruction& dot, const llvm_ir::IrArray& target_array,
+    const llvm_ir::IrArray& lhs_array, const llvm_ir::IrArray& rhs_array,
+    llvm::Value* executable_run_options_value, llvm::IRBuilder<>* b,
+    const HloModuleConfig& hlo_module_config,
+    const TargetMachineFeatures& target_machine_features) {
+  TF_RETURN_IF_ERROR(ValidateDotDimensionNumbers(dot.dot_dimension_numbers()));
+
+  // Lower a batch dot into a sequence of non-batch dot operations.
+
+  int64 num_batch_dims =
+      dot.dot_dimension_numbers().lhs_batch_dimensions_size();
+
+  // First reshape the inputs to make sure we only have one batch dimension.
+  // This is a no-op bitcast because the operands have to be in row-major layout
+  // (enforced in CpuLayoutAssignment), and the batch dimensions are the leading
+  // dimensions (established by DotDecomposer and checked by
+  // ValidateDotDimensionNumbers above).
+  llvm_ir::IrArray lhs_array_reshaped =
+      CollapseFirstNDims(b, lhs_array, num_batch_dims);
+  llvm_ir::IrArray rhs_array_reshaped =
+      CollapseFirstNDims(b, rhs_array, num_batch_dims);
+  llvm_ir::IrArray target_array_reshaped =
+      CollapseFirstNDims(b, target_array, num_batch_dims);
+
+  int64 batch_count = lhs_array_reshaped.GetShape().dimensions(0);
+
+  KernelSupportLibrary ksl(b);
+
+  return ksl.ForWithStatus(
+      "bdot", /*start=*/0, /*end=*/batch_count, /*step=*/1,
+      [&](llvm::Value* indvar) {
+        DotDimensionNumbers adjusted_dim_numbers = dot.dot_dimension_numbers();
+        adjusted_dim_numbers.clear_lhs_batch_dimensions();
+        adjusted_dim_numbers.clear_rhs_batch_dimensions();
+
+        // Create a DotInfo representing the "inner" non-batch dot operation.
+        DotInfo dot_info;
+        dot_info.lhs_shape = DropFirstDim(lhs_array_reshaped.GetShape());
+        dot_info.rhs_shape = DropFirstDim(rhs_array_reshaped.GetShape());
+        dot_info.result_shape = DropFirstDim(target_array_reshaped.GetShape());
+        dot_info.dim_nums = dot.dot_dimension_numbers();
+        dot_info.dim_nums.clear_lhs_batch_dimensions();
+        dot_info.dim_nums.clear_rhs_batch_dimensions();
+
+        dot_info.dim_nums.set_lhs_contracting_dimensions(
+            0,
+            dot_info.dim_nums.lhs_contracting_dimensions(0) - num_batch_dims);
+        dot_info.dim_nums.set_rhs_contracting_dimensions(
+            0,
+            dot_info.dim_nums.rhs_contracting_dimensions(0) - num_batch_dims);
+
+        llvm_ir::IrArray lhs_slice =
+            SliceOutInnerArray(lhs_array_reshaped, /*batch_index=*/indvar, b);
+        llvm_ir::IrArray rhs_slice =
+            SliceOutInnerArray(rhs_array_reshaped, /*batch_index=*/indvar, b);
+        llvm_ir::IrArray target_slice = SliceOutInnerArray(
+            target_array_reshaped, /*batch_index=*/indvar, b);
+
+        // Emit the inner non-batch dot operation.
+        return EmitNonBatchDotOperation(
+            dot_info, dot.name(), target_slice, lhs_slice, rhs_slice, nullptr,
+            executable_run_options_value, b, hlo_module_config,
+            target_machine_features);
+      });
+}
+
+bool IsBatchDot(const HloInstruction& instr) {
+  if (auto* dot_instr = DynCast<HloDotInstruction>(&instr)) {
+    return dot_instr->dot_dimension_numbers().lhs_batch_dimensions_size() > 0;
+  }
+
+  return false;
+}
 }  // namespace
 
 bool DotImplementationCanHandleTranspose(
@@ -884,13 +1045,17 @@ Status EmitDotOperation(const HloInstruction& dot,
   // enclosing computation.
   CHECK(dot.parent()->root_instruction()->outer_dimension_partitions().empty());
 
-  PrimitiveType type = target_array.GetShape().element_type();
-  TF_RET_CHECK(F16 == type || F32 == type || F64 == type || C64 == type);
-  DotOpEmitter dot_emitter(DotInfo(dot), dot.name(), target_array, lhs_array,
-                           rhs_array, addend_array,
-                           executable_run_options_value, b, hlo_module_config,
-                           target_machine_features);
-  return dot_emitter.Emit();
+  if (IsBatchDot(dot)) {
+    TF_RET_CHECK(addend_array == nullptr);
+    return EmitBatchDotOperation(dot, target_array, lhs_array, rhs_array,
+                                 executable_run_options_value, b,
+                                 hlo_module_config, target_machine_features);
+  }
+
+  return EmitNonBatchDotOperation(DotInfo(dot), dot.name(), target_array,
+                                  lhs_array, rhs_array, addend_array,
+                                  executable_run_options_value, b,
+                                  hlo_module_config, target_machine_features);
 }
 }  // namespace cpu
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index 91e3693354..b27a232557 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -944,10 +944,6 @@ Status IrEmitter::HandleDot(HloInstruction* dot) {
       /*instruction=*/*dot, /*operands=*/{lhs, rhs},
       /*supported_types=*/{F16, F32, F64, C64}));
   const DotDimensionNumbers& dnums = dot->dot_dimension_numbers();
-  if (dnums.lhs_batch_dimensions_size() > 0 ||
-      dnums.rhs_batch_dimensions_size() > 0) {
-    return Unimplemented("Dot with batch dimensions not implemented.");
-  }
 
   if (dnums.lhs_contracting_dimensions_size() != 1) {
     // This is disallowed by ShapeInference today.
diff --git a/tensorflow/compiler/xla/service/llvm_ir/ir_array.cc b/tensorflow/compiler/xla/service/llvm_ir/ir_array.cc
index f658761dff..8ee07ae833 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/ir_array.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/ir_array.cc
@@ -104,8 +104,8 @@ IrArray::Index::Index(absl::Span<llvm::Value* const> multidim,
   CHECK(LayoutUtil::HasLayout(shape));
 }
 
-IrArray::IrArray(llvm::Value* base_ptr, const Shape& shape)
-    : base_ptr_(base_ptr), shape_(&shape) {
+IrArray::IrArray(llvm::Value* base_ptr, Shape shape)
+    : base_ptr_(base_ptr), shape_(std::move(shape)) {
   TF_CHECK_OK(ShapeUtil::ValidateShape(shape));
   CHECK(base_ptr_->getType()->isPointerTy());
   int depth = 0;
diff --git a/tensorflow/compiler/xla/service/llvm_ir/ir_array.h b/tensorflow/compiler/xla/service/llvm_ir/ir_array.h
index a483f7051f..b706ebd311 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/ir_array.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/ir_array.h
@@ -130,6 +130,11 @@ class IrArray {
       CHECK_LE(index, size());
       mutable_multidim().insert(mutable_multidim().begin() + index, value);
     }
+    void InsertAt(int64 index, int64 count, llvm::Value* value) {
+      CHECK_LE(index, size());
+      mutable_multidim().insert(mutable_multidim().begin() + index, count,
+                                value);
+    }
 
     using iterator = std::vector<llvm::Value*>::iterator;
     using const_iterator = std::vector<llvm::Value*>::const_iterator;
@@ -222,11 +227,11 @@ class IrArray {
   };
 
   // Default constructor. Constructs an IrArray in a null status.
-  IrArray() : base_ptr_(nullptr), shape_(nullptr) {}
+  IrArray() : base_ptr_(nullptr) {}
 
   // Construct an IrArray with the given base pointer and shape. base_ptr is a
   // pointer type pointing to the first element(lowest address) of the array.
-  IrArray(llvm::Value* base_ptr, const Shape& shape);
+  IrArray(llvm::Value* base_ptr, Shape shape);
 
   // Default implementations of copying and moving.
   IrArray(IrArray&& other) = default;
@@ -238,7 +243,6 @@ class IrArray {
   llvm::Type* GetElementLlvmType() const { return element_type_; }
 
   const Shape& GetShape() const {
-    CHECK(shape_ != nullptr);
     return *shape_;
   }
 
@@ -333,7 +337,7 @@ class IrArray {
   llvm::Type* element_type_;
 
   // Shape of the XLA array.
-  const Shape* shape_;
+  absl::optional<Shape> shape_;
 
   // The list of key/value pairs used when attaching metadata to emitted
   // loads/stores for this array.  They keys are the metadata kinds and the
-- 
GitLab


From a7a85ad829bc2401d4df4fc5400a99f5069ec8ba Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Fri, 11 Jan 2019 12:48:10 -0800
Subject: [PATCH 0578/2345] Internal test cleanup.

PiperOrigin-RevId: 228928263
---
 tensorflow/python/keras/BUILD              |  4 ++--
 tensorflow/python/keras/layers/gru_test.py | 15 ++++++++-------
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 719a4a04c4..d36aaaf632 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -640,7 +640,7 @@ tf_py_test(
 
 tf_py_test(
     name = "gru_test",
-    size = "large",
+    size = "medium",
     srcs = ["layers/gru_test.py"],
     additional_deps = [
         ":keras",
@@ -648,7 +648,7 @@ tf_py_test(
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
     ],
-    shard_count = 2,
+    shard_count = 4,
     tags = ["notsan"],  # http://b/62136390
 )
 
diff --git a/tensorflow/python/keras/layers/gru_test.py b/tensorflow/python/keras/layers/gru_test.py
index 0e9dde8eae..91183a4d73 100644
--- a/tensorflow/python/keras/layers/gru_test.py
+++ b/tensorflow/python/keras/layers/gru_test.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.python import keras
@@ -68,17 +69,17 @@ class GRULayerTest(keras_parameterized.TestCase):
                 'recurrent_dropout': 0.1},
         input_shape=(num_samples, timesteps, embedding_dim))
 
-  def test_implementation_mode_GRU(self):
+  @parameterized.parameters([0, 1, 2])
+  def test_implementation_mode_GRU(self, implementation_mode):
     num_samples = 2
     timesteps = 3
     embedding_dim = 4
     units = 2
-    for mode in [0, 1, 2]:
-      testing_utils.layer_test(
-          keras.layers.GRU,
-          kwargs={'units': units,
-                  'implementation': mode},
-          input_shape=(num_samples, timesteps, embedding_dim))
+    testing_utils.layer_test(
+        keras.layers.GRU,
+        kwargs={'units': units,
+                'implementation': implementation_mode},
+        input_shape=(num_samples, timesteps, embedding_dim))
 
   def test_reset_after_GRU(self):
     num_samples = 2
-- 
GitLab


From 2cecb70a63372d1ff510e7708a6e581bbe167efd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 12:58:15 -0800
Subject: [PATCH 0579/2345] Adapt SRUCell to keras RNN layer.

PiperOrigin-RevId: 228930242
---
 .../python/kernel_tests/core_rnn_cell_test.py | 29 +++++++++++++++++++
 tensorflow/contrib/rnn/python/ops/rnn_cell.py |  8 +++--
 2 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py
index a0d013c618..7bad4a60a1 100644
--- a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py
+++ b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py
@@ -210,6 +210,35 @@ class RNNCellTest(test.TestCase):
         # Smoke test
         self.assertAllClose(res[0], [[0.509682, 0.509682]])
 
+  def testSRUCellKerasRNN(self):
+    """Tests that SRUCell works with keras RNN layer."""
+    cell = contrib_rnn_cell.SRUCell(10)
+    seq_input = ops.convert_to_tensor(
+        np.random.rand(2, 3, 5), name="seq_input", dtype=dtypes.float32)
+    rnn_layer = keras_layers.RNN(cell=cell)
+    rnn_outputs_keras = rnn_layer(seq_input)
+    with self.cached_session() as sess:
+      sess.run([variables_lib.global_variables_initializer()])
+      self.assertEqual(sess.run(rnn_outputs_keras).shape, (2, 10))
+
+  def testSRUCellBiasType(self):
+    """Tests that the bias' dtype is properly set."""
+    cell = contrib_rnn_cell.SRUCell(10)
+    cell.build((2, 3, 5))
+    self.assertEqual(cell._bias.dtype, dtypes.float32_ref)
+
+    cell = contrib_rnn_cell.SRUCell(10, dtype=dtypes.int32)
+    cell.build((2, 3, 5))
+    self.assertEqual(cell._bias.dtype, dtypes.int32_ref)
+
+    cell_input = ops.convert_to_tensor(
+        np.random.rand(2, 5), name="cell_input", dtype=dtypes.float16)
+    cell_state = ops.convert_to_tensor(
+        np.random.rand(2, 10), name="cell_state", dtype=dtypes.float16)
+    cell = contrib_rnn_cell.SRUCell(10)
+    cell(cell_input, [cell_state])
+    self.assertEqual(cell._bias.dtype, dtypes.float16_ref)
+
   def testSRUCellWithDiffSize(self):
     with self.cached_session() as sess:
       with variable_scope.variable_scope(
diff --git a/tensorflow/contrib/rnn/python/ops/rnn_cell.py b/tensorflow/contrib/rnn/python/ops/rnn_cell.py
index 0702b2a723..a2179cae0c 100644
--- a/tensorflow/contrib/rnn/python/ops/rnn_cell.py
+++ b/tensorflow/contrib/rnn/python/ops/rnn_cell.py
@@ -2744,10 +2744,12 @@ class SRUCell(rnn_cell_impl.LayerRNNCell):
     name: (optional) String, the name of the layer. Layers with the same name
       will share weights, but to avoid mistakes we require reuse=True in such
       cases.
+    **kwargs: Additional keyword arguments.
   """
 
-  def __init__(self, num_units, activation=None, reuse=None, name=None):
-    super(SRUCell, self).__init__(_reuse=reuse, name=name)
+  def __init__(self, num_units, activation=None, reuse=None, name=None,
+               **kwargs):
+    super(SRUCell, self).__init__(_reuse=reuse, name=name, **kwargs)
     self._num_units = num_units
     self._activation = activation or math_ops.tanh
 
@@ -2777,7 +2779,7 @@ class SRUCell(rnn_cell_impl.LayerRNNCell):
     self._bias = self.add_variable(
         rnn_cell_impl._BIAS_VARIABLE_NAME,  # pylint: disable=protected-access
         shape=[2 * self._num_units],
-        initializer=init_ops.constant_initializer(0.0, dtype=self.dtype))
+        initializer=init_ops.zeros_initializer)
 
     self._built = True
 
-- 
GitLab


From 937ff1b4cf1d954806e075d66198429a6d2312be Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Fri, 11 Jan 2019 13:05:56 -0800
Subject: [PATCH 0580/2345] [XLA] Add complex128 support.

Minimally tested at the moment (tested via the TF tests).

PiperOrigin-RevId: 228931680
---
 .../jit/mark_for_compilation_pass_test.cc     | 10 +-
 tensorflow/compiler/jit/xla_cpu_device.cc     |  4 +-
 .../compiler/jit/xla_interpreter_device.cc    |  4 +-
 tensorflow/compiler/tests/binary_ops_test.py  |  2 +-
 tensorflow/compiler/tests/build_defs.bzl      |  2 +-
 tensorflow/compiler/tests/unary_ops_test.py   |  2 +-
 tensorflow/compiler/tf2xla/kernels/cast_op.cc |  4 +-
 .../compiler/tf2xla/kernels/const_op.cc       | 11 +++
 .../compiler/tf2xla/kernels/matmul_op.cc      |  4 +-
 tensorflow/compiler/tf2xla/lib/util.cc        |  6 ++
 tensorflow/compiler/tf2xla/type_util.cc       |  3 +
 tensorflow/compiler/tf2xla/xla_op_registry.h  |  9 +-
 .../compiler/xla/client/lib/constants.h       |  2 +
 tensorflow/compiler/xla/client/lib/math.cc    |  4 +-
 tensorflow/compiler/xla/literal.cc            | 98 +++++++++++++++----
 tensorflow/compiler/xla/literal_comparison.cc | 58 +++++++++--
 tensorflow/compiler/xla/literal_test.cc       | 56 ++++++++++-
 tensorflow/compiler/xla/literal_util.cc       | 14 +++
 tensorflow/compiler/xla/primitive_util.cc     |  7 +-
 tensorflow/compiler/xla/primitive_util.h      | 10 ++
 .../compiler/xla/python/numpy_bridge.cc       | 11 +++
 tensorflow/compiler/xla/python/xla_client.py  |  1 +
 tensorflow/compiler/xla/service/BUILD         |  2 +
 .../xla/service/algebraic_simplifier.cc       |  3 +
 .../xla/service/cpu/dot_op_emitter.cc         |  3 +-
 .../compiler/xla/service/cpu/ir_emitter.cc    | 12 ++-
 .../compiler/xla/service/gpu/gemm_thunk.cc    |  8 ++
 .../xla/service/gpu/ir_emission_utils.cc      |  3 +-
 .../compiler/xla/service/hlo_evaluator.cc     | 97 ++++++++++++++++--
 .../xla/service/hlo_evaluator_typed_visitor.h | 15 ++-
 .../hlo_evaluator_typed_visitor_complex128.cc | 22 +++++
 .../compiler/xla/service/hlo_parser_test.cc   | 11 +++
 .../compiler/xla/service/llvm_ir/llvm_util.cc | 11 ++-
 .../compiler/xla/service/shape_inference.cc   |  4 +-
 .../xla/service/shape_inference_test.cc       |  8 +-
 tensorflow/compiler/xla/shape_util.cc         |  3 +
 .../xla/tests/client_library_test_base.h      | 15 ++-
 tensorflow/compiler/xla/types.h               |  1 +
 tensorflow/compiler/xla/xla_data.proto        |  6 +-
 39 files changed, 462 insertions(+), 84 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor_complex128.cc

diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc
index bf2c5508ea..c2b6250f73 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc
@@ -151,7 +151,7 @@ TEST(XlaCompilationTest, CompilableCycles) {
   EXPECT_EQ(clusters["A"], clusters["C"]);
 }
 
-TEST(XlaCompilationTest, Complex128Unsupported) {
+TEST(XlaCompilationTest, StringUnsupported) {
   std::unique_ptr<Graph> graph(new Graph(OpRegistry::Global()));
   GraphDef graphdef;
   {
@@ -159,10 +159,10 @@ TEST(XlaCompilationTest, Complex128Unsupported) {
     Node* a = ops::SourceOp(
         "Const", builder.opts()
                      .WithName("A")
-                     .WithAttr("dtype", DT_COMPLEX128)
-                     .WithAttr("value", Tensor(DT_COMPLEX128, TensorShape())));
-    Node* b = ops::UnaryOp("Neg", a, builder.opts().WithName("B"));
-    ops::BinaryOp("MatMul", a, b, builder.opts().WithName("C"));
+                     .WithAttr("dtype", DT_STRING)
+                     .WithAttr("value", Tensor(DT_STRING, TensorShape())));
+    Node* b = ops::UnaryOp("EncodeBase64", a, builder.opts().WithName("B"));
+    ops::BinaryOp("StringSplit", a, b, builder.opts().WithName("C"));
     TF_EXPECT_OK(GraphDefBuilderToGraph(builder, graph.get()));
   }
 
diff --git a/tensorflow/compiler/jit/xla_cpu_device.cc b/tensorflow/compiler/jit/xla_cpu_device.cc
index e9770647e7..94dc61d55f 100644
--- a/tensorflow/compiler/jit/xla_cpu_device.cc
+++ b/tensorflow/compiler/jit/xla_cpu_device.cc
@@ -83,9 +83,9 @@ REGISTER_LOCAL_DEVICE_FACTORY(DEVICE_XLA_CPU, XlaCpuDeviceFactory);
 
 // Kernel registrations
 
-constexpr std::array<DataType, 12> kAllXlaCpuTypes = {
+constexpr std::array<DataType, 13> kAllXlaCpuTypes = {
     {DT_UINT8, DT_QUINT8, DT_INT8, DT_QINT8, DT_INT32, DT_QINT32, DT_INT64,
-     DT_HALF, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64, DT_BOOL}};
+     DT_HALF, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64, DT_COMPLEX128, DT_BOOL}};
 
 REGISTER_XLA_LAUNCH_KERNEL(DEVICE_XLA_CPU, XlaLocalLaunchOp, kAllXlaCpuTypes);
 REGISTER_XLA_COMPILE_KERNEL(DEVICE_XLA_CPU, XlaCompileOp, kAllXlaCpuTypes);
diff --git a/tensorflow/compiler/jit/xla_interpreter_device.cc b/tensorflow/compiler/jit/xla_interpreter_device.cc
index 4007309ed1..e1a5824061 100644
--- a/tensorflow/compiler/jit/xla_interpreter_device.cc
+++ b/tensorflow/compiler/jit/xla_interpreter_device.cc
@@ -26,9 +26,9 @@ namespace tensorflow {
 const char* const DEVICE_XLA_INTERPRETER = "XLA_INTERPRETER";
 const char* const DEVICE_INTERPRETER_XLA_JIT = "XLA_INTERPRETER_JIT";
 
-constexpr std::array<DataType, 9> kExecAllTypes = {
+constexpr std::array<DataType, 10> kExecAllTypes = {
     {DT_INT8, DT_INT32, DT_INT64, DT_HALF, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64,
-     DT_BOOL, DT_BFLOAT16}};
+     DT_COMPLEX128, DT_BOOL, DT_BFLOAT16}};
 
 class XlaInterpreterDeviceFactory : public DeviceFactory {
  public:
diff --git a/tensorflow/compiler/tests/binary_ops_test.py b/tensorflow/compiler/tests/binary_ops_test.py
index 9a5423c1b2..a3651b4b0d 100644
--- a/tensorflow/compiler/tests/binary_ops_test.py
+++ b/tensorflow/compiler/tests/binary_ops_test.py
@@ -400,7 +400,7 @@ class BinaryOpsTest(xla_test.XLATestCase):
 
   def testComplexOps(self):
     for dtype in self.complex_types:
-      ctypes = {np.complex64: np.float32}
+      ctypes = {np.complex64: np.float32, np.complex128: np.float64}
       self._testBinary(
           math_ops.complex,
           np.array([[[[-1, 2], [2, 0]]]], dtype=ctypes[dtype]),
diff --git a/tensorflow/compiler/tests/build_defs.bzl b/tensorflow/compiler/tests/build_defs.bzl
index 447a7de2cb..be9766c4ef 100644
--- a/tensorflow/compiler/tests/build_defs.bzl
+++ b/tensorflow/compiler/tests/build_defs.bzl
@@ -64,7 +64,7 @@ def tf_xla_py_test(
         if backend == "cpu":
             backend_args += [
                 "--test_device=XLA_CPU",
-                "--types=DT_HALF,DT_FLOAT,DT_DOUBLE,DT_UINT8,DT_QUINT8,DT_INT8,DT_QINT8,DT_INT32,DT_QINT32,DT_INT64,DT_BOOL,DT_COMPLEX64",
+                "--types=DT_HALF,DT_FLOAT,DT_DOUBLE,DT_UINT8,DT_QUINT8,DT_INT8,DT_QINT8,DT_INT32,DT_QINT32,DT_INT64,DT_BOOL,DT_COMPLEX64,DT_COMPLEX128",
             ]
         elif backend == "gpu":
             backend_args += [
diff --git a/tensorflow/compiler/tests/unary_ops_test.py b/tensorflow/compiler/tests/unary_ops_test.py
index 95c9e7ffd4..3c2875ba47 100644
--- a/tensorflow/compiler/tests/unary_ops_test.py
+++ b/tensorflow/compiler/tests/unary_ops_test.py
@@ -647,7 +647,7 @@ class UnaryOpsTest(xla_test.XLATestCase):
           np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype),
           expected=np.tan(np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype)))
 
-      ctypes = {np.complex64: np.float32}
+      ctypes = {np.complex64: np.float32, np.complex128: np.float64}
       self._assertOpOutputMatchesExpected(
           math_ops.abs,
           np.array([[3 - 4j, -1j, np.inf]], dtype=dtype),
diff --git a/tensorflow/compiler/tf2xla/kernels/cast_op.cc b/tensorflow/compiler/tf2xla/kernels/cast_op.cc
index 8cc2479dd5..ce8131eeb4 100644
--- a/tensorflow/compiler/tf2xla/kernels/cast_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/cast_op.cc
@@ -79,8 +79,8 @@ class BitcastOp : public XlaOpKernel {
     if (src_dtype_ == dst_dtype_) {
       output = input;
     } else {
-      // The only complex type in XLA is C64, so error out if the bitcast has a
-      // complex source or destination type and the bitcast is not trivial.
+      // Error out if the bitcast has a complex source or destination type and
+      // the bitcast is not trivial.
       OP_REQUIRES(ctx,
                   !xla::primitive_util::IsComplexType(src_type_) &&
                       !xla::primitive_util::IsComplexType(dst_type_),
diff --git a/tensorflow/compiler/tf2xla/kernels/const_op.cc b/tensorflow/compiler/tf2xla/kernels/const_op.cc
index dff8af8002..ff6c54e47c 100644
--- a/tensorflow/compiler/tf2xla/kernels/const_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/const_op.cc
@@ -83,6 +83,17 @@ class ConstOp : public XlaOpKernel {
             return;
           }
           break;
+        case DT_COMPLEX128:
+          if (proto_.scomplex_val_size() == 2) {
+            ctx->SetOutput(
+                0,
+                xla::Broadcast(xla::ConstantR0<xla::complex128>(
+                                   b, xla::complex128(proto_.dcomplex_val(0),
+                                                      proto_.dcomplex_val(1))),
+                               shape.dim_sizes()));
+            return;
+          }
+          break;
         case DT_INT32:
           if (proto_.int_val_size() == 1) {
             ctx->SetOutput(
diff --git a/tensorflow/compiler/tf2xla/kernels/matmul_op.cc b/tensorflow/compiler/tf2xla/kernels/matmul_op.cc
index 6440770c29..f36e002525 100644
--- a/tensorflow/compiler/tf2xla/kernels/matmul_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/matmul_op.cc
@@ -24,8 +24,8 @@ limitations under the License.
 namespace tensorflow {
 namespace {
 
-constexpr std::array<DataType, 5> kMatmulTypes = {
-    {DT_HALF, DT_BFLOAT16, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64}};
+constexpr std::array<DataType, 6> kMatmulTypes = {
+    {DT_HALF, DT_BFLOAT16, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64, DT_COMPLEX128}};
 
 class MatMulOp : public XlaOpKernel {
  public:
diff --git a/tensorflow/compiler/tf2xla/lib/util.cc b/tensorflow/compiler/tf2xla/lib/util.cc
index c0bd172d17..06eda41611 100644
--- a/tensorflow/compiler/tf2xla/lib/util.cc
+++ b/tensorflow/compiler/tf2xla/lib/util.cc
@@ -54,6 +54,9 @@ xla::XlaOp FloatLiteral(xla::XlaBuilder* builder, xla::PrimitiveType type,
     case xla::C64:
       return xla::ConstantR0<xla::complex64>(builder, value);
       break;
+    case xla::C128:
+      return xla::ConstantR0<xla::complex128>(builder, value);
+      break;
     default:
       LOG(FATAL) << "unhandled element type " << type;
   }
@@ -90,6 +93,9 @@ xla::XlaOp IntegerLiteral(xla::XlaBuilder* builder, xla::PrimitiveType type,
     case xla::C64:
       literal = xla::LiteralUtil::CreateR0<complex64>(value);
       break;
+    case xla::C128:
+      literal = xla::LiteralUtil::CreateR0<complex128>(value);
+      break;
     case xla::PRED:
       LOG(FATAL) << "pred element type is not integral";
     case xla::S16:
diff --git a/tensorflow/compiler/tf2xla/type_util.cc b/tensorflow/compiler/tf2xla/type_util.cc
index d00b137662..732f957d73 100644
--- a/tensorflow/compiler/tf2xla/type_util.cc
+++ b/tensorflow/compiler/tf2xla/type_util.cc
@@ -69,6 +69,9 @@ Status DataTypeToPrimitiveType(DataType data_type, xla::PrimitiveType* type) {
     case tensorflow::DT_COMPLEX64:
       *type = xla::C64;
       return Status::OK();
+    case tensorflow::DT_COMPLEX128:
+      *type = xla::C128;
+      return Status::OK();
     default:
       return errors::InvalidArgument(
           "Unsupported type in DataTypeToPrimitiveType ",
diff --git a/tensorflow/compiler/tf2xla/xla_op_registry.h b/tensorflow/compiler/tf2xla/xla_op_registry.h
index 0bdd4a1085..ce3b6b298c 100644
--- a/tensorflow/compiler/tf2xla/xla_op_registry.h
+++ b/tensorflow/compiler/tf2xla/xla_op_registry.h
@@ -47,13 +47,14 @@ extern const char* const DEVICE_XLA_GPU;
 
 constexpr std::array<DataType, 4> kFloatTypes = {
     {DT_HALF, DT_FLOAT, DT_DOUBLE, DT_BFLOAT16}};
-constexpr std::array<DataType, 11> kNumericTypes = {
+constexpr std::array<DataType, 12> kNumericTypes = {
     {DT_UINT8, DT_UINT32, DT_UINT64, DT_INT8, DT_INT32, DT_INT64, DT_HALF,
-     DT_FLOAT, DT_DOUBLE, DT_COMPLEX64, DT_BFLOAT16}};
+     DT_FLOAT, DT_DOUBLE, DT_COMPLEX64, DT_COMPLEX128, DT_BFLOAT16}};
 
-constexpr std::array<DataType, 14> kCpuAllTypes = {
+constexpr std::array<DataType, 15> kCpuAllTypes = {
     {DT_UINT8, DT_QUINT8, DT_UINT32, DT_UINT64, DT_INT8, DT_QINT8, DT_INT32,
-     DT_QINT32, DT_INT64, DT_HALF, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64, DT_BOOL}};
+     DT_QINT32, DT_INT64, DT_HALF, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64,
+     DT_COMPLEX128, DT_BOOL}};
 
 constexpr std::array<DataType, 15> kGpuAllTypes = {
     {DT_UINT8, DT_QUINT8, DT_UINT32, DT_UINT64, DT_INT8, DT_QINT8, DT_INT32,
diff --git a/tensorflow/compiler/xla/client/lib/constants.h b/tensorflow/compiler/xla/client/lib/constants.h
index 81624614c1..a38282e8db 100644
--- a/tensorflow/compiler/xla/client/lib/constants.h
+++ b/tensorflow/compiler/xla/client/lib/constants.h
@@ -56,6 +56,8 @@ XlaOp ConstantR0WithType(XlaBuilder* builder, PrimitiveType type, T value) {
       return ConstantR0<double>(builder, static_cast<double>(value));
     case C64:
       return ConstantR0<complex64>(builder, static_cast<complex64>(value));
+    case C128:
+      return ConstantR0<complex128>(builder, static_cast<complex128>(value));
     case U8:
       return ConstantR0<uint8>(builder, static_cast<uint8>(value));
     case U32:
diff --git a/tensorflow/compiler/xla/client/lib/math.cc b/tensorflow/compiler/xla/client/lib/math.cc
index 36fdda39b4..3d0e3a2b93 100644
--- a/tensorflow/compiler/xla/client/lib/math.cc
+++ b/tensorflow/compiler/xla/client/lib/math.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/lib/math.h"
 
 #include "tensorflow/compiler/xla/client/lib/constants.h"
+#include "tensorflow/compiler/xla/primitive_util.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 
@@ -323,7 +324,8 @@ XlaOp MaybeConjugate(XlaOp x, bool conjugate) {
   XlaBuilder* builder = x.builder();
   return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
     TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x));
-    auto perform_conj = shape.element_type() == C64 && conjugate;
+    auto perform_conj =
+        primitive_util::IsComplexType(shape.element_type()) && conjugate;
     return perform_conj ? Conj(x) : x;
   });
 }
diff --git a/tensorflow/compiler/xla/literal.cc b/tensorflow/compiler/xla/literal.cc
index 81874fa405..8600e8752c 100644
--- a/tensorflow/compiler/xla/literal.cc
+++ b/tensorflow/compiler/xla/literal.cc
@@ -29,10 +29,12 @@ limitations under the License.
 #include "absl/strings/str_join.h"
 #include "absl/types/span.h"
 #include "tensorflow/compiler/xla/index_util.h"
+#include "tensorflow/compiler/xla/primitive_util.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/util.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/hash/hash.h"
 #include "tensorflow/core/platform/logging.h"
@@ -411,6 +413,7 @@ Status LiteralBase::Piece::CopyFrom(const LiteralBase::Piece& src) {
       COPY_ELEMENTS(F32, float);
       COPY_ELEMENTS(F64, double);
       COPY_ELEMENTS(C64, complex64);
+      COPY_ELEMENTS(C128, complex128);
       COPY_ELEMENTS(PRED, bool);
 #undef COPY_ELEMENTS
       default:
@@ -548,6 +551,9 @@ Status MutableLiteralBase::CopySliceFrom(const LiteralSlice& src_literal,
     case C64:
       return CopySliceFromInternal<complex64>(src_literal, src_base, dest_base,
                                               copy_size);
+    case C128:
+      return CopySliceFromInternal<complex128>(src_literal, src_base, dest_base,
+                                               copy_size);
     case PRED:
       return CopySliceFromInternal<bool>(src_literal, src_base, dest_base,
                                          copy_size);
@@ -766,6 +772,8 @@ Literal LiteralBase::Slice(absl::Span<const int64> start_indices,
       return SliceInternal<double>(result_shape, start_indices);
     case C64:
       return SliceInternal<complex64>(result_shape, start_indices);
+    case C128:
+      return SliceInternal<complex128>(result_shape, start_indices);
     default:
       LOG(FATAL) << "not yet implemented: "
                  << PrimitiveType_Name(result_shape.element_type());
@@ -814,6 +822,10 @@ string LiteralBase::GetAsString(absl::Span<const int64> multi_index,
       complex64 c = Get<complex64>(multi_index, shape_index);
       return StrCat("(", c.real(), ", ", c.imag(), ")");
     }
+    case C128: {
+      complex128 c = Get<complex128>(multi_index, shape_index);
+      return StrCat("(", c.real(), ", ", c.imag(), ")");
+    }
     default:
       LOG(FATAL) << PrimitiveType_Name(subshape.element_type());
   }
@@ -868,6 +880,11 @@ string LiteralBase::GetSparseElementAsString(
           GetSparseElement<complex64>(sparse_element_number, shape_index);
       return StrCat("(", c.real(), ", ", c.imag(), ")");
     }
+    case C128: {
+      complex128 c =
+          GetSparseElement<complex128>(sparse_element_number, shape_index);
+      return StrCat("(", c.real(), ", ", c.imag(), ")");
+    }
     default:
       LOG(FATAL) << "Invalid element type for sparse arrays: "
                  << PrimitiveType_Name(subshape.element_type());
@@ -996,6 +1013,9 @@ void LiteralBase::Piece::SortSparseElements() {
     case C64:
       SortSparseElementsInternal<complex64>();
       break;
+    case C128:
+      SortSparseElementsInternal<complex128>();
+      break;
     case F16:
       SortSparseElementsInternal<half>();
       break;
@@ -1230,7 +1250,24 @@ Literal ConvertBetweenNativeTypesWithConverter(const LiteralBase& src_literal,
 }
 
 template <typename NativeSrcT, typename NativeDestT>
-Literal ConvertBetweenNativeTypes(const LiteralBase& src_literal) {
+typename std::enable_if<(std::is_same<NativeSrcT, Eigen::half>::value) &&
+                            (std::is_same<NativeDestT, complex64>::value ||
+                             std::is_same<NativeDestT, complex128>::value),
+                        Literal>::type
+ConvertBetweenNativeTypes(const LiteralBase& src_literal) {
+  auto converter = [](NativeSrcT src) {
+    return NativeDestT(static_cast<typename NativeDestT::value_type>(src));
+  };
+  return ConvertBetweenNativeTypesWithConverter<NativeSrcT, NativeDestT>(
+      src_literal, converter);
+}
+
+template <typename NativeSrcT, typename NativeDestT>
+typename std::enable_if<(!std::is_same<NativeSrcT, Eigen::half>::value) ||
+                            (!std::is_same<NativeDestT, complex64>::value &&
+                             !std::is_same<NativeDestT, complex128>::value),
+                        Literal>::type
+ConvertBetweenNativeTypes(const LiteralBase& src_literal) {
   auto converter = [](NativeSrcT src) { return static_cast<NativeDestT>(src); };
   return ConvertBetweenNativeTypesWithConverter<NativeSrcT, NativeDestT>(
       src_literal, converter);
@@ -1274,22 +1311,6 @@ BitcastBetweenNativeTypes(const LiteralBase& src_literal) {
   LOG(FATAL) << "Invalid bitcast between types of different sizes.";
 }
 
-template <PrimitiveType primitive_src_type>
-Literal ConvertToC64(const LiteralBase& src_literal) {
-  CHECK(src_literal.shape().IsArray());
-  Literal result_literal(
-      ShapeUtil::ChangeElementType(src_literal.shape(), C64));
-  using NativeSrcT =
-      typename primitive_util::PrimitiveTypeToNative<primitive_src_type>::type;
-  absl::Span<const NativeSrcT> src_data = src_literal.data<NativeSrcT>();
-  absl::Span<complex64> dest_data = result_literal.data<complex64>();
-  int64 num_elements = src_literal.element_count();
-  for (int64 i = 0; i < num_elements; ++i) {
-    dest_data[i] = complex64(static_cast<float>(src_data[i]), 0);
-  }
-  return result_literal;
-}
-
 template <PrimitiveType primitive_src_type, PrimitiveType primitive_dest_type>
 Literal ConvertIfTypesMatch(const LiteralBase& src_literal, bool bitcast) {
   CHECK_EQ(primitive_src_type, src_literal.shape().element_type());
@@ -1332,10 +1353,15 @@ StatusOr<Literal> ConvertIfDestTypeMatches(const LiteralBase& src_literal,
     CONVERT_IF_TYPES_MATCH(BF16)
 #undef CONVERT_IF_TYPES_MATCH
     case C64:
-      if (!bitcast) {
-        return ConvertToC64<primitive_src_type>(src_literal);
+      if (bitcast) {
+        break;
       }
-      break;
+      return ConvertIfTypesMatch<primitive_src_type, C64>(src_literal, false);
+    case C128:
+      if (bitcast) {
+        break;
+      }
+      return ConvertIfTypesMatch<primitive_src_type, C128>(src_literal, false);
     // Other types are not yet supported.
     default:
       break;
@@ -1485,6 +1511,8 @@ bool LiteralBase::Piece::EqualElements(const LiteralBase::Piece& other) const {
       return EqualElementsInternal<bfloat16>(other, &multi_index);
     case C64:
       return EqualElementsInternal<complex64>(other, &multi_index);
+    case C128:
+      return EqualElementsInternal<complex128>(other, &multi_index);
     default:
       LOG(FATAL) << "Unimplemented: LiteralBase::Piece::EqualElements for type "
                  << PrimitiveType_Name(subshape().element_type());
@@ -1628,6 +1656,9 @@ bool LiteralBase::IsAllComplex(complex64 value) const {
     case C64:
       return AllElementsEqualValue<complex64>(root_piece().data<complex64>(),
                                               value);
+    case C128:
+      return AllElementsEqualValue<complex128>(root_piece().data<complex128>(),
+                                               value);
     default:
       return false;
   }
@@ -1707,6 +1738,11 @@ bool LiteralBase::IsAllFirst() const {
               auto data = piece.data<uint64>();
               return AllElementsEqualValue<uint64>(data, data[0]);
             }
+
+            case C128: {
+              auto data = piece.data<complex128>();
+              return AllElementsEqualValue<complex128>(data, data[0]);
+            }
             default:
               return false;
           }
@@ -1756,6 +1792,8 @@ bool LiteralBase::IsR1Iota() const {
         return Get<bfloat16>({idx}) == static_cast<bfloat16>(idx);
       case C64:
         return Get<complex64>({idx}) == complex64(idx, 0.0f);
+      case C128:
+        return Get<complex128>({idx}) == complex128(idx, 0.0f);
       case PRED:
         return Get<bool>({idx}) == idx;
       // token, opaque, tuple, etc. are all not iota.
@@ -1799,6 +1837,8 @@ bool LiteralBase::IsZero(absl::Span<const int64> indices) const {
       return Get<double>(indices) == 0.0;
     case C64:
       return Get<complex64>(indices) == complex64(0.0f, 0.0f);
+    case C128:
+      return Get<complex128>(indices) == complex128(0.0f, 0.0f);
     case F16:
       return Get<half>(indices) == static_cast<half>(0.0f);
     case BF16:
@@ -1886,6 +1926,12 @@ void LiteralBase::Piece::WriteToProto(LiteralProto* proto) const {
         proto->add_c64s(value.imag());
       }
       break;
+    case C128:
+      for (complex128 value : data<complex128>()) {
+        proto->add_c128s(value.real());
+        proto->add_c128s(value.imag());
+      }
+      break;
     case TUPLE:
     case TOKEN:
       // Nothing to do but assign the shape which is done above.
@@ -2018,7 +2064,17 @@ Status LiteralBase::Piece::CopyFromProto(const LiteralProto& proto) {
       for (int64 i = 0; i < complex_data.size(); ++i) {
         complex_data[i] = complex64{proto.c64s(i * 2), proto.c64s(i * 2 + 1)};
       }
-    } break;
+      break;
+    }
+    case C128: {
+      auto complex_data = data<complex128>();
+      TF_RET_CHECK(proto.c128s_size() == complex_data.size() * 2);
+      for (int64 i = 0; i < complex_data.size(); ++i) {
+        complex_data[i] =
+            complex128{proto.c128s(i * 2), proto.c128s(i * 2 + 1)};
+      }
+      break;
+    }
     case TUPLE:
       return InvalidArgument("Should not be called on tuple shapes: %s",
                              ShapeUtil::HumanString(subshape()));
diff --git a/tensorflow/compiler/xla/literal_comparison.cc b/tensorflow/compiler/xla/literal_comparison.cc
index 258bc966b1..69efa06d39 100644
--- a/tensorflow/compiler/xla/literal_comparison.cc
+++ b/tensorflow/compiler/xla/literal_comparison.cc
@@ -90,6 +90,12 @@ bool CompareEqual<complex64>(complex64 lhs, complex64 rhs,
   return CompareEqual<float>(lhs.real(), rhs.real(), multi_index) &&
          CompareEqual<float>(lhs.imag(), rhs.imag(), multi_index);
 }
+template <>
+bool CompareEqual<complex128>(complex128 lhs, complex128 rhs,
+                              absl::Span<const int64> multi_index) {
+  return CompareEqual<double>(lhs.real(), rhs.real(), multi_index) &&
+         CompareEqual<double>(lhs.imag(), rhs.imag(), multi_index);
+}
 
 template <typename NativeT, typename UnsignedT>
 Status MakeBitwiseErrorStatus(NativeT lhs, NativeT rhs,
@@ -143,6 +149,14 @@ Status MakeErrorStatus(complex64 lhs, complex64 rhs,
   }
   return MakeErrorStatus(lhs.imag(), rhs.imag(), multi_index);
 }
+template <>
+Status MakeErrorStatus(complex128 lhs, complex128 rhs,
+                       absl::Span<const int64> multi_index) {
+  if (!CompareEqual<double>(lhs.real(), rhs.real(), multi_index)) {
+    return MakeErrorStatus(lhs.real(), rhs.real(), multi_index);
+  }
+  return MakeErrorStatus(lhs.imag(), rhs.imag(), multi_index);
+}
 
 // A recursive function which iterates through every index of expected and
 // actual literal and compares their values elementwise. Returns true if all
@@ -197,13 +211,6 @@ bool NanMismatch(NativeT expected, NativeT actual, bool relaxed_nans) {
   }
 }
 
-template <>
-bool NanMismatch<complex64>(complex64 expected, complex64 actual,
-                            bool relaxed_nans) {
-  return NanMismatch<float>(expected.real(), actual.real(), relaxed_nans) ||
-         NanMismatch<float>(expected.imag(), actual.imag(), relaxed_nans);
-}
-
 template <>
 bool NanMismatch<half>(half expected, half actual, bool relaxed_nans) {
   return NanMismatch<float>(static_cast<float>(expected),
@@ -232,6 +239,11 @@ string FpValueToString<complex64>(complex64 value) {
   return absl::StrFormat("%8.4g + %8.4fi", value.real(), value.imag());
 }
 
+template <>
+string FpValueToString<complex128>(complex128 value) {
+  return absl::StrFormat("%8.4g + %8.4fi", value.real(), value.imag());
+}
+
 // Returns the absolute value of the given floating point value. This function
 // is used instead of std::abs directly in order to allow type-dependent
 // implementations for NearComparator.
@@ -434,7 +446,7 @@ class NearComparator {
     mismatches_.data<bool>()[linear_index] = true;
   }
 
-  // For complex64 types, we compare real and imaginary parts individually.
+  // For complex types, we compare real and imaginary parts individually.
   void CompareValues(complex64 expected, complex64 actual, int64 linear_index) {
     bool mismatch = false;
     CompareValues<float>(expected.real(), actual.real(), linear_index);
@@ -457,6 +469,29 @@ class NearComparator {
     mismatches_.data<bool>()[linear_index] = mismatch;
   }
 
+  void CompareValues(complex128 expected, complex128 actual,
+                     int64 linear_index) {
+    bool mismatch = false;
+    CompareValues<double>(expected.real(), actual.real(), linear_index);
+    if (mismatches_.data<bool>()[linear_index] == true) {
+      mismatch = true;
+      // Delay the mismatch count increase for real part, instead increase
+      // mismatch by 1 for the entire complex number.
+      num_mismatches_--;
+    }
+    CompareValues<double>(expected.imag(), actual.imag(), linear_index);
+    if (mismatches_.data<bool>()[linear_index] == true) {
+      mismatch = true;
+      // Delay the mismatch count increase for imag part, instead increase
+      // mismatch by 1 for the entire complex number.
+      num_mismatches_--;
+    }
+    if (mismatch == true) {
+      num_mismatches_++;
+    }
+    mismatches_.data<bool>()[linear_index] = mismatch;
+  }
+
   // Compares the two literals elementwise.
   void CompareLiterals() {
     // Fast path optimization for the case were layouts match.
@@ -665,6 +700,9 @@ Status EqualHelper(const LiteralSlice& expected, const LiteralSlice& actual) {
     case C64:
       result = Equal<complex64>(expected, actual, index, 0);
       break;
+    case C128:
+      result = Equal<complex128>(expected, actual, index, 0);
+      break;
     case TUPLE: {
       for (int i = 0; i < ShapeUtil::TupleElementCount(expected.shape()); ++i) {
         result.Update(EqualHelper(LiteralSlice(expected, {i}),
@@ -749,6 +787,10 @@ Status NearHelper(const LiteralSlice& expected, const LiteralSlice& actual,
         return NearComparator<complex64>::Compare(
             expected, actual, error, detailed_message, miscompare_callback);
         break;
+      case C128:
+        return NearComparator<complex128>::Compare(
+            expected, actual, error, detailed_message, miscompare_callback);
+        break;
       default:
         LOG(FATAL) << "Unsupported primitive type in near comparator: "
                    << PrimitiveType_Name(expected.shape().element_type())
diff --git a/tensorflow/compiler/xla/literal_test.cc b/tensorflow/compiler/xla/literal_test.cc
index a4088c6d11..e67bb5c32f 100644
--- a/tensorflow/compiler/xla/literal_test.cc
+++ b/tensorflow/compiler/xla/literal_test.cc
@@ -118,6 +118,9 @@ TEST_F(LiteralUtilTest, LiteralScalarToString) {
   auto c64_lit = LiteralUtil::CreateR0<complex64>({3.14f, 2.78f});
   EXPECT_EQ("c64[] (3.14, 2.78)", c64_lit.ToString());
 
+  auto c128_lit = LiteralUtil::CreateR0<complex128>({3.14f, 2.78f});
+  EXPECT_EQ("c128[] (3.14, 2.78)", c128_lit.ToString());
+
   auto bf16_lit = LiteralUtil::CreateR0<bfloat16>(static_cast<bfloat16>(0.5f));
   EXPECT_EQ("bf16[] 0.5", bf16_lit.ToString());
 
@@ -469,6 +472,21 @@ TEST_F(LiteralUtilTest, C64Equality) {
   EXPECT_NE(vector, vector_reversed);
 }
 
+TEST_F(LiteralUtilTest, C128Equality) {
+  // Test equality with tuples.
+  auto vector = LiteralUtil::CreateR1<complex128>({{1.0, 2.0}, {3.0, 4.0}});
+
+  // Tuple with the same elements. One element is shared with the original
+  // tuple, the other is a clone of the element in the original tuple.
+  auto vector_clone =
+      LiteralUtil::CreateR1<complex128>({{1.0, 2.0}, {3.0, 4.0}});
+  EXPECT_EQ(vector, vector_clone);
+
+  auto vector_reversed =
+      LiteralUtil::CreateR1<complex128>({{3.0, 4.0}, {1.0, 2.0}});
+  EXPECT_NE(vector, vector_reversed);
+}
+
 TEST_F(LiteralUtilTest, IsAllTuple) {
   auto element1 = LiteralUtil::CreateR0<float>(0.0);
   auto element2 = LiteralUtil::CreateR2<float>({{0.0, 0.0}, {0.0, 0.0}});
@@ -836,6 +854,13 @@ TEST_F(LiteralUtilTest, PopulateR1C64) {
   EXPECT_EQ(output, expected);
 }
 
+TEST_F(LiteralUtilTest, PopulateR1C128) {
+  Literal output(ShapeUtil::MakeShape(C128, {1}));
+  output.PopulateR1<complex128>({{77, 88}});
+  auto expected = LiteralUtil::CreateR1<complex128>({{77, 88}});
+  EXPECT_EQ(output, expected);
+}
+
 TEST_F(LiteralUtilTest, PopulateR2C64) {
   Literal output(ShapeUtil::MakeShape(C64, {2, 2}));
   output.PopulateR2<complex64>({{{7, 8}, {9, 10}}, {{1, 2}, {3, 4}}});
@@ -897,6 +922,14 @@ TEST_F(LiteralUtilTest, PopulateWithValueR2C64) {
   EXPECT_EQ(output, expected);
 }
 
+TEST_F(LiteralUtilTest, PopulateWithValueR2C128) {
+  Literal output(ShapeUtil::MakeShape(C128, {2, 2}));
+  output.PopulateWithValue<complex128>({4, 2});
+  auto expected =
+      LiteralUtil::CreateR2<complex128>({{{4, 2}, {4, 2}}, {{4, 2}, {4, 2}}});
+  EXPECT_EQ(output, expected);
+}
+
 TEST_F(LiteralUtilTest, PopulateWithValueR0F16) {
   Literal output(ShapeUtil::MakeShape(F16, {}));
   half h(0.25f);
@@ -1308,7 +1341,11 @@ TEST_F(LiteralUtilTest, ConvertIfTypesMatch) {
     {{0.0f, 19.0f, 0.0f, 21.0f}, {22.0f, 0.0f, 24.0f, 0.0f}},
     {{26.0f, 0.0f, 28.0f, 0.0f}, {0.0f, 31.0f, 0.0f, 33.0f}},
   }}, layout_r4_dim0major_);
-  // clang-format on
+  auto c128 = LiteralUtil::CreateR4WithLayout<complex128>({{
+    {{10.0, 0.0, 12.0, 0.0}, {0.0, 15.0, 0.0, 17.0}},
+    {{0.0, 19.0, 0.0, 21.0}, {22.0, 0.0, 24.0, 0.0}},
+    {{26.0, 0.0, 28.0, 0.0}, {0.0, 31.0, 0.0, 33.0}},
+  }}, layout_r4_dim0major_);  // clang-format on
   Literal conv;
 
   conv = s8.Convert(U16).ConsumeValueOrDie();
@@ -1374,10 +1411,20 @@ TEST_F(LiteralUtilTest, ConvertIfTypesMatch) {
   conv = s32.Convert(U16).ConsumeValueOrDie();
   EXPECT_EQ(conv, u16);
 
+  conv = s32.Convert(C128).ConsumeValueOrDie();
+  EXPECT_EQ(conv, c128);
+
+  conv = f16.Convert(C128).ConsumeValueOrDie();
+  EXPECT_EQ(conv, c128);
+
   EXPECT_EQ(s32.Convert(TUPLE).status().code(),
             tensorflow::error::UNIMPLEMENTED);
   EXPECT_EQ(c64.Convert(F32).status().code(), tensorflow::error::UNIMPLEMENTED);
   EXPECT_EQ(c64.Convert(S32).status().code(), tensorflow::error::UNIMPLEMENTED);
+  EXPECT_EQ(c128.Convert(F32).status().code(),
+            tensorflow::error::UNIMPLEMENTED);
+  EXPECT_EQ(c128.Convert(S32).status().code(),
+            tensorflow::error::UNIMPLEMENTED);
 }
 
 TEST_F(LiteralUtilTest, BitcastConvert) {
@@ -1739,7 +1786,8 @@ TEST_F(LiteralUtilTest, CreateFromShapeZeroInitialized) {
 
   Literal tuple = Literal::CreateFromShape(ShapeUtil::MakeTupleShape(
       {ShapeUtil::MakeShape(F64, {}), ShapeUtil::MakeShape(PRED, {2}),
-       ShapeUtil::MakeShape(U64, {2, 1}), ShapeUtil::MakeShape(C64, {})}));
+       ShapeUtil::MakeShape(U64, {2, 1}), ShapeUtil::MakeShape(C64, {}),
+       ShapeUtil::MakeShape(C128, {})}));
 
   EXPECT_EQ(tuple.Get<double>({}, {0}), 0.0);
   EXPECT_EQ(tuple.Get<bool>({0}, {1}), false);
@@ -1747,6 +1795,7 @@ TEST_F(LiteralUtilTest, CreateFromShapeZeroInitialized) {
   EXPECT_EQ(tuple.Get<uint64>({0, 0}, {2}), 0);
   EXPECT_EQ(tuple.Get<uint64>({1, 0}, {2}), 0);
   EXPECT_EQ(tuple.Get<complex64>({}, {3}), complex64(0.0f, 0.0f));
+  EXPECT_EQ(tuple.Get<complex128>({}, {4}), complex128(0.0, 0.0));
 }
 
 TEST_F(LiteralUtilTest, ProtoRoundTrip) {
@@ -1756,6 +1805,8 @@ TEST_F(LiteralUtilTest, ProtoRoundTrip) {
   auto vector_int8 = LiteralUtil::CreateR1<int8>({-128, 0, 2, 4, 7, 56, 127});
   auto vector_uint8 = LiteralUtil::CreateR1<uint8>({128, 0, 2, 56, 127, 255});
   auto vector_c64 = LiteralUtil::CreateR1<complex64>({{1.0, 2.0}, {3.0, 4.0}});
+  auto vector_c128 =
+      LiteralUtil::CreateR1<complex128>({{1.0, 2.0}, {3.0, 4.0}});
   auto vector_bfloat16 = LiteralUtil::CreateR1<bfloat16>(
       {bfloat16{-1.0}, bfloat16{2.0}, bfloat16{-3.0}});
   auto vector_half =
@@ -1776,6 +1827,7 @@ TEST_F(LiteralUtilTest, ProtoRoundTrip) {
   EXPECT_EQ(vector_int8, to_from_proto(vector_int8));
   EXPECT_EQ(vector_uint8, to_from_proto(vector_uint8));
   EXPECT_EQ(vector_c64, to_from_proto(vector_c64));
+  EXPECT_EQ(vector_c128, to_from_proto(vector_c128));
   EXPECT_EQ(vector_bfloat16, to_from_proto(vector_bfloat16));
   EXPECT_EQ(matrix_pred, to_from_proto(matrix_pred));
   EXPECT_EQ(tuple, to_from_proto(tuple));
diff --git a/tensorflow/compiler/xla/literal_util.cc b/tensorflow/compiler/xla/literal_util.cc
index 5881e43e96..26b029c8d0 100644
--- a/tensorflow/compiler/xla/literal_util.cc
+++ b/tensorflow/compiler/xla/literal_util.cc
@@ -130,6 +130,8 @@ Literal ConvertType(LiteralSlice literal) {
       return LiteralUtil::CreateR0<double>(0);
     case C64:
       return LiteralUtil::CreateR0<complex64>(0);
+    case C128:
+      return LiteralUtil::CreateR0<complex128>(0);
     case PRED:
       return LiteralUtil::CreateR0<bool>(false);
     case TUPLE:
@@ -165,6 +167,8 @@ Literal ConvertType(LiteralSlice literal) {
       return LiteralUtil::CreateR0<double>(1);
     case C64:
       return LiteralUtil::CreateR0<complex64>(1);
+    case C128:
+      return LiteralUtil::CreateR0<complex128>(1);
     case PRED:
       return LiteralUtil::CreateR0<bool>(true);
     case S16:
@@ -201,6 +205,8 @@ Literal ConvertType(LiteralSlice literal) {
           -std::numeric_limits<double>::infinity());
     case C64:
       LOG(FATAL) << "C64 element type has no minimum value";
+    case C128:
+      LOG(FATAL) << "C128 element type has no minimum value";
     case PRED:
       return LiteralUtil::CreateR0<bool>(false);
     case S16:
@@ -345,6 +351,10 @@ Literal ConvertType(LiteralSlice literal) {
         new_literal.Set<complex64>(to_multi_index,
                                    literal.Get<complex64>(from_multi_index));
         break;
+      case C128:
+        new_literal.Set<complex128>(to_multi_index,
+                                    literal.Get<complex128>(from_multi_index));
+        break;
       default:
         LOG(FATAL) << "Unhandled primitive element type: "
                    << PrimitiveType_Name(literal.shape().element_type());
@@ -393,6 +403,10 @@ Literal ConvertType(LiteralSlice literal) {
       return LiteralUtil::CreateR0<int64>(literal.GetFirstElement<int64>());
     case U64:
       return LiteralUtil::CreateR0<uint64>(literal.GetFirstElement<uint64>());
+
+    case C128:
+      return LiteralUtil::CreateR0<complex128>(
+          literal.GetFirstElement<complex128>());
     default:
       LOG(FATAL) << "Unhandled primitive type "
                  << literal.shape().element_type();
diff --git a/tensorflow/compiler/xla/primitive_util.cc b/tensorflow/compiler/xla/primitive_util.cc
index 00ad01fc40..04b40efbed 100644
--- a/tensorflow/compiler/xla/primitive_util.cc
+++ b/tensorflow/compiler/xla/primitive_util.cc
@@ -27,7 +27,7 @@ bool IsFloatingPointType(PrimitiveType type) {
   return type == F16 || type == F32 || type == F64 || type == BF16;
 }
 
-bool IsComplexType(PrimitiveType type) { return type == C64; }
+bool IsComplexType(PrimitiveType type) { return type == C64 || type == C128; }
 
 bool IsSignedIntegralType(PrimitiveType type) {
   return type == S8 || type == S16 || type == S32 || type == S64;
@@ -67,6 +67,9 @@ int BitWidth(PrimitiveType type) {
     case C64:
       return 64;
 
+    case C128:
+      return 128;
+
     case TUPLE:
       LOG(FATAL) << "TUPLE is an invalid type for BitWidth";
 
@@ -82,6 +85,8 @@ PrimitiveType ComplexComponentType(PrimitiveType complex_type) {
   switch (complex_type) {
     case C64:
       return F32;
+    case C128:
+      return F64;
     default:
       LOG(FATAL) << "Primitive type is not complex: "
                  << PrimitiveType_Name(complex_type);
diff --git a/tensorflow/compiler/xla/primitive_util.h b/tensorflow/compiler/xla/primitive_util.h
index 70603b6fed..739a7c409c 100644
--- a/tensorflow/compiler/xla/primitive_util.h
+++ b/tensorflow/compiler/xla/primitive_util.h
@@ -126,6 +126,11 @@ inline PrimitiveType NativeToPrimitiveType<complex64>() {
   return C64;
 }
 
+template <>
+inline PrimitiveType NativeToPrimitiveType<complex128>() {
+  return C128;
+}
+
 bool IsFloatingPointType(PrimitiveType type);
 
 bool IsComplexType(PrimitiveType type);
@@ -225,6 +230,11 @@ struct PrimitiveTypeToNative<C64> {
   using type = complex64;
 };
 
+template <>
+struct PrimitiveTypeToNative<C128> {
+  using type = complex128;
+};
+
 // Returns the lower-case name of the given primitive type.
 const string& LowercasePrimitiveTypeName(PrimitiveType s);
 
diff --git a/tensorflow/compiler/xla/python/numpy_bridge.cc b/tensorflow/compiler/xla/python/numpy_bridge.cc
index 9c90550825..52c5c621f7 100644
--- a/tensorflow/compiler/xla/python/numpy_bridge.cc
+++ b/tensorflow/compiler/xla/python/numpy_bridge.cc
@@ -54,6 +54,8 @@ int PrimitiveTypeToNumpyType(PrimitiveType primitive_type) {
       return NPY_FLOAT64;
     case C64:
       return NPY_COMPLEX64;
+    case C128:
+      return NPY_COMPLEX128;
     case TUPLE:
       return NPY_OBJECT;
     default:
@@ -89,6 +91,8 @@ PrimitiveType NumpyTypeToPrimitiveType(int np_type) {
       return F64;
     case NPY_COMPLEX64:
       return C64;
+    case NPY_COMPLEX128:
+      return C128;
     case NPY_OBJECT:
       return TUPLE;
     default:
@@ -111,6 +115,7 @@ bool NumpyTypeIsValid(int np_type) {
     case NPY_FLOAT32:
     case NPY_FLOAT64:
     case NPY_COMPLEX64:
+    case NPY_COMPLEX128:
     case NPY_OBJECT:
       return true;
     default:
@@ -430,6 +435,9 @@ Status CopyNumpyArrayToLiteral(int np_type, PyArrayObject* py_array,
     case NPY_COMPLEX64:
       CopyNumpyArrayToLiteral<complex64>(py_array, literal);
       break;
+    case NPY_COMPLEX128:
+      CopyNumpyArrayToLiteral<complex128>(py_array, literal);
+      break;
     default:
       return InvalidArgument(
           "No XLA literal container for Numpy type number: %d", np_type);
@@ -470,6 +478,9 @@ void CopyLiteralToNumpyArray(int np_type, const LiteralSlice& literal,
     case NPY_COMPLEX64:
       CopyLiteralToNumpyArray<complex64>(literal, py_array);
       break;
+    case NPY_COMPLEX128:
+      CopyLiteralToNumpyArray<complex128>(literal, py_array);
+      break;
     default:
       LOG(FATAL) << "No XLA literal container for Numpy type" << np_type;
   }
diff --git a/tensorflow/compiler/xla/python/xla_client.py b/tensorflow/compiler/xla/python/xla_client.py
index 4e71121c09..1684cb20e6 100644
--- a/tensorflow/compiler/xla/python/xla_client.py
+++ b/tensorflow/compiler/xla/python/xla_client.py
@@ -199,6 +199,7 @@ XLA_ELEMENT_TYPE_TO_DTYPE = {
     xla_data_pb2.F32: np.dtype('float32'),
     xla_data_pb2.F64: np.dtype('float64'),
     xla_data_pb2.C64: np.dtype('complex64'),
+    xla_data_pb2.C128: np.dtype('complex128'),
     xla_data_pb2.TUPLE: np.dtype(np.object),
 }
 
diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index ff14c500e3..b324a810ce 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -223,6 +223,7 @@ cc_library(
         "hlo_evaluator_typed_visitor.h",
         "hlo_evaluator_typed_visitor_bfloat16.cc",
         "hlo_evaluator_typed_visitor_bool.cc",
+        "hlo_evaluator_typed_visitor_complex128.cc",
         "hlo_evaluator_typed_visitor_complex64.cc",
         "hlo_evaluator_typed_visitor_double.cc",
         "hlo_evaluator_typed_visitor_float.cc",
@@ -259,6 +260,7 @@ cc_library(
         "@com_google_absl//absl/container:inlined_vector",
         "@com_google_absl//absl/container:node_hash_map",
         "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/meta:type_traits",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/types:optional",
         "@com_google_absl//absl/types:span",
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index 5ac746c9f3..4d6042b0d4 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -877,6 +877,9 @@ Status AlgebraicSimplifierVisitor::HandleDivide(HloInstruction* divide) {
       case C64:
         TF_RETURN_IF_ERROR(InvertConstant<complex64>(*b, &new_literal));
         break;
+      case C128:
+        TF_RETURN_IF_ERROR(InvertConstant<complex128>(*b, &new_literal));
+        break;
       default:
         return Status::OK();
     }
diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
index dba28aa51a..48510181bd 100644
--- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
@@ -856,7 +856,8 @@ Status EmitNonBatchDotOperation(
     const HloModuleConfig& hlo_module_config,
     const TargetMachineFeatures& target_machine_features) {
   PrimitiveType type = target_array.GetShape().element_type();
-  TF_RET_CHECK(F16 == type || F32 == type || F64 == type || C64 == type);
+  TF_RET_CHECK(F16 == type || F32 == type || F64 == type || C64 == type ||
+               C128 == type);
   DotOpEmitter dot_emitter(std::move(dot_info), std::move(hlo_name),
                            target_array, lhs_array, rhs_array, addend_array,
                            executable_run_options_value, b, hlo_module_config,
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index b27a232557..f8a997045a 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -239,10 +239,12 @@ Status IrEmitter::HandleCopy(HloInstruction* copy) {
 int IrEmitter::MinimumAlignmentForPrimitiveType(PrimitiveType primitive_type) {
   int64 byte_size = ShapeUtil::ByteSizeOfPrimitiveType(primitive_type);
   DCHECK_GE(byte_size, 0);
-  // Largest scalar is a complex64 so we don't need to worry about the
+  // Largest scalar is a complex128 so we don't need to worry about the
   // int64->int truncation here.
-  DCHECK_LE(byte_size, 8);
-  return byte_size;
+  DCHECK_LE(byte_size, 16);
+
+  // Allocations may be 8-byte aligned if part of a small block.
+  return std::min(8LL, byte_size);
 }
 
 int64 IrEmitter::ByteSizeOf(const Shape& shape) const {
@@ -942,7 +944,7 @@ Status IrEmitter::HandleDot(HloInstruction* dot) {
   auto rhs = dot->operand(1);
   TF_RETURN_IF_ERROR(ElementTypesSameAndSupported(
       /*instruction=*/*dot, /*operands=*/{lhs, rhs},
-      /*supported_types=*/{F16, F32, F64, C64}));
+      /*supported_types=*/{F16, F32, F64, C64, C128}));
   const DotDimensionNumbers& dnums = dot->dot_dimension_numbers();
 
   if (dnums.lhs_contracting_dimensions_size() != 1) {
@@ -1114,7 +1116,7 @@ Status IrEmitter::HandleConvolution(HloInstruction* convolution) {
   auto rhs = convolution->operand(1);
   TF_RETURN_IF_ERROR(ElementTypesSameAndSupported(
       /*instruction=*/*convolution, /*operands=*/{lhs, rhs},
-      /*supported_types=*/{F16, F32, C64}));
+      /*supported_types=*/{F16, F32, C64, C128}));
 
   // TODO(tonywy): Add PotentiallyImplementedAsMKLCovolution to support
   // different data layouts.
diff --git a/tensorflow/compiler/xla/service/gpu/gemm_thunk.cc b/tensorflow/compiler/xla/service/gpu/gemm_thunk.cc
index b8fbe7d2bc..86c9bc6a34 100644
--- a/tensorflow/compiler/xla/service/gpu/gemm_thunk.cc
+++ b/tensorflow/compiler/xla/service/gpu/gemm_thunk.cc
@@ -206,6 +206,8 @@ auto GetGemmFn(PrimitiveType type) -> decltype(&DoGemm<float>) {
       return &DoGemm<double>;
     case C64:
       return &DoGemm<std::complex<float>>;
+    case C128:
+      return &DoGemm<std::complex<double>>;
     default:
       LOG(FATAL) << "Unsupported type.";
   }
@@ -221,6 +223,8 @@ auto GetGemmWithAlgorithmFn(PrimitiveType type)
       return &DoGemmWithAlgorithm<double>;
     case C64:
       return &DoGemmWithAlgorithm<std::complex<float>>;
+    case C128:
+      return &DoGemmWithAlgorithm<std::complex<double>>;
     default:
       LOG(FATAL) << "Unsupported type.";
   }
@@ -235,6 +239,8 @@ auto GetGemmAutotuneFn(PrimitiveType type) -> decltype(&DoGemmAutotune<float>) {
       return &DoGemmAutotune<double>;
     case C64:
       return &DoGemmAutotune<std::complex<float>>;
+    case C128:
+      return &DoGemmAutotune<std::complex<double>>;
     default:
       LOG(FATAL) << "Unsupported type.";
   }
@@ -255,6 +261,8 @@ se::blas::ComputationType GetBlasComputationType(PrimitiveType type) {
       return se::blas::ComputationType::kF64;
     case C64:
       return se::blas::ComputationType::kComplexF32;
+    case C128:
+      return se::blas::ComputationType::kComplexF64;
     default:
       LOG(FATAL) << "Unsupported type.";
   }
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc b/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc
index caccb18899..82bdd677d9 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc
@@ -54,7 +54,8 @@ bool AreValidGemmShapes(const Shape& lhs_shape, const Shape& rhs_shape,
   PrimitiveType output_primitive_type = output_shape.element_type();
   bool type_is_allowed =
       (output_primitive_type == F16 || output_primitive_type == F32 ||
-       output_primitive_type == F64 || output_primitive_type == C64);
+       output_primitive_type == F64 || output_primitive_type == C64 ||
+       output_primitive_type == C128);
   return type_is_allowed && IsRank2(lhs_shape, batch_dimensions_size) &&
          IsRank2(rhs_shape, batch_dimensions_size) &&
          IsRank2(output_shape, batch_dimensions_size) &&
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index 407615f842..927e751d38 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -136,6 +136,37 @@ StatusOr<Literal> Compare<complex64>(const Shape& shape, HloOpcode opcode,
   return std::move(result);
 }
 
+template <>
+StatusOr<Literal> Compare<complex128>(const Shape& shape, HloOpcode opcode,
+                                      LiteralSlice lhs_literal,
+                                      LiteralSlice rhs_literal) {
+  std::function<bool(complex128, complex128)> compare_op;
+  switch (opcode) {
+    case HloOpcode::kEq:
+      compare_op = [](complex128 lhs_el, complex128 rhs_el) {
+        return lhs_el == rhs_el;
+      };
+      break;
+    case HloOpcode::kNe:
+      compare_op = [](complex128 lhs_el, complex128 rhs_el) {
+        return lhs_el != rhs_el;
+      };
+      break;
+    default:
+      LOG(FATAL) << "unhandled HLO opcode for conversion to Comparison: "
+                 << HloOpcodeString(opcode);
+  }
+
+  Literal result(shape);
+  TF_RETURN_IF_ERROR(
+      result.Populate<bool>([&](absl::Span<const int64> multi_index) {
+        return compare_op(lhs_literal.Get<complex128>(multi_index),
+                          rhs_literal.Get<complex128>(multi_index));
+      }));
+
+  return std::move(result);
+}
+
 }  // namespace
 
 // Note that unsupported types by the typed visitor does not necessarily imply
@@ -170,6 +201,8 @@ HloEvaluator::HloEvaluator(int64 max_loop_iterations)
       absl::make_unique<HloEvaluatorTypedVisitor<double>>(this);
   typed_visitors_[C64] =
       absl::make_unique<HloEvaluatorTypedVisitor<complex64>>(this);
+  typed_visitors_[C128] =
+      absl::make_unique<HloEvaluatorTypedVisitor<complex128>>(this);
 
   // Most of the evaluator computations we use don't support BF16 (e.g.,
   // std::ceil, std::tanh). To make evaluator work with BF16, we set all
@@ -500,6 +533,13 @@ Status HloEvaluator::HandleReal(HloInstruction* real) {
       TF_ASSIGN_OR_RETURN(evaluated_[real], std::move(result_or));
       break;
     }
+    case C128: {
+      auto result_or = ElementWiseUnaryOpImpl<float, complex128>(
+          real, [](complex128 elem_operand) { return std::real(elem_operand); },
+          GetEvaluatedLiteralFor(operand));
+      TF_ASSIGN_OR_RETURN(evaluated_[real], std::move(result_or));
+      break;
+    }
     case F16: {
       auto result_or = ElementWiseUnaryOpImpl<Eigen::half, Eigen::half>(
           real, [](Eigen::half elem_operand) { return elem_operand; },
@@ -530,11 +570,29 @@ Status HloEvaluator::HandleReal(HloInstruction* real) {
 }
 
 Status HloEvaluator::HandleImag(HloInstruction* imag) {
-  auto result_or = ElementWiseUnaryOpImpl<float, complex64>(
-      imag, [](complex64 elem_operand) { return std::imag(elem_operand); },
-      GetEvaluatedLiteralFor(imag->operand(0)));
+  auto operand = imag->operand(0);
+  switch (operand->shape().element_type()) {
+    case C64: {
+      auto result_or = ElementWiseUnaryOpImpl<float, complex64>(
+          imag, [](complex64 elem_operand) { return std::imag(elem_operand); },
+          GetEvaluatedLiteralFor(imag->operand(0)));
+
+      TF_ASSIGN_OR_RETURN(evaluated_[imag], std::move(result_or));
+      break;
+    }
+    case C128: {
+      auto result_or = ElementWiseUnaryOpImpl<double, complex128>(
+          imag, [](complex128 elem_operand) { return std::imag(elem_operand); },
+          GetEvaluatedLiteralFor(imag->operand(0)));
+
+      TF_ASSIGN_OR_RETURN(evaluated_[imag], std::move(result_or));
+      break;
+    }
+    default:
+      LOG(FATAL) << "HandleImag: unknown/unhandled primitive type: "
+                 << PrimitiveType_Name(operand->shape().element_type());
+  }
 
-  TF_ASSIGN_OR_RETURN(evaluated_[imag], std::move(result_or));
   return Status::OK();
 }
 
@@ -544,11 +602,27 @@ Status HloEvaluator::HandleComplex(HloInstruction* complex) {
   TF_RET_CHECK(ShapeUtil::Compatible(real.shape(), imag.shape()));
 
   Literal result(complex->shape());
-  TF_RETURN_IF_ERROR(
-      result.Populate<complex64>([&](absl::Span<const int64> multi_index) {
-        return std::complex<float>(real.Get<float>(multi_index),
-                                   imag.Get<float>(multi_index));
-      }));
+  switch (complex->shape().element_type()) {
+    case C64: {
+      TF_RETURN_IF_ERROR(
+          result.Populate<complex64>([&](absl::Span<const int64> multi_index) {
+            return std::complex<float>(real.Get<float>(multi_index),
+                                       imag.Get<float>(multi_index));
+          }));
+      break;
+    }
+    case C128: {
+      TF_RETURN_IF_ERROR(
+          result.Populate<complex128>([&](absl::Span<const int64> multi_index) {
+            return std::complex<float>(real.Get<double>(multi_index),
+                                       imag.Get<double>(multi_index));
+          }));
+      break;
+    }
+    default:
+      LOG(FATAL) << "HandleComplex: unknown/unhandled primitive type: "
+                 << PrimitiveType_Name(complex->shape().element_type());
+  }
 
   evaluated_[complex] = std::move(result);
   return Status::OK();
@@ -647,6 +721,11 @@ Status HloEvaluator::HandleCompare(HloInstruction* compare) {
                           Compare<complex64>(compare->shape(), opcode,
                                              lhs_literal, rhs_literal));
     } break;
+    case C128: {
+      TF_ASSIGN_OR_RETURN(evaluated_[compare],
+                          Compare<complex128>(compare->shape(), opcode,
+                                              lhs_literal, rhs_literal));
+    } break;
     default:
       LOG(FATAL) << "HandleCompare: unknown primitive type: "
                  << PrimitiveType_Name(lhs->shape().element_type());
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
index 698b177310..f3f400e46f 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
@@ -22,6 +22,7 @@ limitations under the License.
 #include "absl/base/casts.h"
 #include "absl/container/inlined_vector.h"
 #include "absl/memory/memory.h"
+#include "absl/meta/type_traits.h"
 #include "absl/types/optional.h"
 #include "tensorflow/compiler/xla/array2d.h"
 #include "tensorflow/compiler/xla/literal_util.h"
@@ -39,9 +40,8 @@ namespace xla {
 // Anyway this is relatively safe as-is because hlo_evaluator_typed_visitor.h is
 // a "private" header that's not exposed outside of hlo_evaluator.cc.
 template <typename T>
-using is_complex_t = std::is_same<T, complex64>;
-template <typename T>
-using is_complex64_t = std::is_same<T, complex64>;
+using is_complex_t =
+    absl::disjunction<std::is_same<T, complex64>, std::is_same<T, complex128>>;
 
 // It's UB to use std::sort with std::less<float>, because of NaNs. Define
 // "safe" less functions which are actually strict weak orders. -NaN and NaN
@@ -212,7 +212,7 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
 
   template <
       typename NativeT,
-      typename std::enable_if<is_complex64_t<NativeT>::value>::type* = nullptr>
+      typename std::enable_if<is_complex_t<NativeT>::value>::type* = nullptr>
   Status HandleAbs(HloInstruction* abs) {
     const Literal& operand_literal =
         parent_->GetEvaluatedLiteralFor(abs->operand(0));
@@ -231,6 +231,8 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
     // specifying the ElementwiseT explicitly as C64 is needed below.
     if (abs->operand(0)->shape().element_type() == C64) {
       return HandleAbs<complex64>(abs);
+    } else if (abs->operand(0)->shape().element_type() == C128) {
+      return HandleAbs<complex128>(abs);
     }
     return HandleAbs<ElementwiseT>(abs);
   }
@@ -1616,6 +1618,10 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
         TF_ASSIGN_OR_RETURN(parent_->evaluated_[map], MapImpl<complex64>(map));
         break;
       }
+      case C128: {
+        TF_ASSIGN_OR_RETURN(parent_->evaluated_[map], MapImpl<complex128>(map));
+        break;
+      }
       default:
         LOG(FATAL) << "HandleMap: unhandled primitive type for "
                       "input operand: "
@@ -3040,6 +3046,7 @@ extern template class HloEvaluatorTypedVisitor<Eigen::half, float>;
 extern template class HloEvaluatorTypedVisitor<float>;
 extern template class HloEvaluatorTypedVisitor<double>;
 extern template class HloEvaluatorTypedVisitor<complex64>;
+extern template class HloEvaluatorTypedVisitor<complex128>;
 extern template class HloEvaluatorTypedVisitor<bfloat16, float>;
 
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor_complex128.cc b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor_complex128.cc
new file mode 100644
index 0000000000..1f48140ee4
--- /dev/null
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor_complex128.cc
@@ -0,0 +1,22 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h"
+
+#include "tensorflow/compiler/xla/service/hlo_evaluator.h"
+
+namespace xla {
+template class HloEvaluatorTypedVisitor<complex128>;
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/hlo_parser_test.cc b/tensorflow/compiler/xla/service/hlo_parser_test.cc
index 76b8a5bc11..74ef8a2fba 100644
--- a/tensorflow/compiler/xla/service/hlo_parser_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser_test.cc
@@ -551,6 +551,17 @@ ENTRY %Transpose.v2 () -> s32[1,2,3] {
   ROOT %transpose = s32[1,2,3]{2,1,0} transpose(s32[1,2,3]{2,1,0} %constant), dimensions={0,1,2}
 }
 
+)"
+},
+{
+"TransposeC128",
+R"(HloModule TransposeC128_module
+
+ENTRY %Transpose.v3 (input: c128[1,2,3]) -> c128[1,2,3] {
+  %input = c128[1,2,3]{2,1,0} parameter(0)
+  ROOT %transpose = c128[1,2,3]{2,1,0} transpose(c128[1,2,3]{2,1,0} %input), dimensions={0,1,2}
+}
+
 )"
 },
 // Dynamic slice
diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc
index 83af5d2543..807296329c 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc
@@ -188,7 +188,16 @@ llvm::Type* PrimitiveTypeToIrType(PrimitiveType element_type,
       }
       return cplx_t;
     }
-    // A Tuple contains an array of pointers. Use i8*.
+    case C128: {
+      auto cplx_t = module->getTypeByName("complex128");
+      if (cplx_t == nullptr) {
+        return llvm::StructType::create(
+            {llvm::Type::getDoubleTy(module->getContext()),
+             llvm::Type::getDoubleTy(module->getContext())},
+            "complex128", /*isPacked=*/true);
+      }
+      return cplx_t;
+    }  // A Tuple contains an array of pointers. Use i8*.
     case TUPLE:
     // An Opaque is like a void*, use i8*.
     case OPAQUE:
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index 1d3f84af95..c2e014ad42 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -906,6 +906,8 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
                                                         broadcast_dimensions));
       if (lhs.element_type() == F32 && rhs.element_type() == F32) {
         return ShapeUtil::ChangeElementType(shape, C64);
+      } else if (lhs.element_type() == F64 && rhs.element_type() == F64) {
+        return ShapeUtil::ChangeElementType(shape, C128);
       } else {
         return Unimplemented("Complex component type is not implemented.");
       }
@@ -1733,7 +1735,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
     case FFT:
     case IFFT:
       if (in.element_type() != C64) {
-        return InvalidArgument("%s requires C64 input type, found %s.",
+        return InvalidArgument("%s requires complex input type, found %s.",
                                FftType_Name(fft_type),
                                PrimitiveType_Name(in.element_type()));
       }
diff --git a/tensorflow/compiler/xla/service/shape_inference_test.cc b/tensorflow/compiler/xla/service/shape_inference_test.cc
index bb00bfd2ee..26120a06b8 100644
--- a/tensorflow/compiler/xla/service/shape_inference_test.cc
+++ b/tensorflow/compiler/xla/service/shape_inference_test.cc
@@ -35,6 +35,7 @@ class ShapeInferenceTest : public ::testing::Test {
  protected:
   // Some handy scalar shapes.
   const Shape s32_ = ShapeUtil::MakeShape(S32, {});
+  const Shape f16_ = ShapeUtil::MakeShape(F16, {});
   const Shape f32_ = ShapeUtil::MakeShape(F32, {});
   const Shape f64_ = ShapeUtil::MakeShape(F64, {});
   const Shape pred_ = ShapeUtil::MakeShape(PRED, {});
@@ -260,8 +261,8 @@ TEST_F(ShapeInferenceTest, Complex) {
   ASSERT_FALSE(complex_shape(pred_, pred_, {}).ok());
   // Component types must match.
   ASSERT_FALSE(complex_shape(f32_, f64_, {}).ok());
-  // Only F32->C64 supported.
-  ASSERT_FALSE(complex_shape(f64_, f64_, {}).ok());
+  // Only F32->C64 and F64->C128 supported.
+  ASSERT_FALSE(complex_shape(f16_, f16_, {}).ok());
   // Validate correct uses.
   Shape c64_32 = ShapeUtil::MakeShape(C64, {32});
   TF_ASSERT_OK_AND_ASSIGN(Shape result, complex_shape(f32_, f32_, {}));
@@ -285,6 +286,9 @@ TEST_F(ShapeInferenceTest, Complex) {
   ASSERT_TRUE(ShapeUtil::Equal(result, c64_32_64));
   TF_ASSERT_OK_AND_ASSIGN(result, complex_shape(matrix_32_64_, f32_, {}));
   ASSERT_TRUE(ShapeUtil::Equal(result, c64_32_64));
+
+  TF_ASSERT_OK_AND_ASSIGN(result, complex_shape(f64_, f64_, {}));
+  ASSERT_TRUE(ShapeUtil::Equal(result, ShapeUtil::MakeShape(C128, {})));
 }
 
 TEST_F(ShapeInferenceTest, VariadicOpTuplify) {
diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc
index 235b065585..9821790cb3 100644
--- a/tensorflow/compiler/xla/shape_util.cc
+++ b/tensorflow/compiler/xla/shape_util.cc
@@ -378,6 +378,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
     case U32:
     case U64:
     case C64:
+    case C128:
     case TUPLE:
     case OPAQUE:
     case TOKEN:
@@ -639,6 +640,8 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
       return sizeof(double);
     case C64:
       return sizeof(complex64);
+    case C128:
+      return sizeof(complex128);
     case TOKEN:
       // Tokens require no space.
       return 0;
diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.h b/tensorflow/compiler/xla/tests/client_library_test_base.h
index 65a23dd883..3f65ed7fce 100644
--- a/tensorflow/compiler/xla/tests/client_library_test_base.h
+++ b/tensorflow/compiler/xla/tests/client_library_test_base.h
@@ -431,7 +431,8 @@ void ClientLibraryTestBase::ComputeAndCompareR0(
                     std::is_same<NativeT, double>::value ||
                     std::is_same<NativeT, bfloat16>::value ||
                     std::is_same<NativeT, half>::value ||
-                    std::is_same<NativeT, complex64>::value,
+                    std::is_same<NativeT, complex64>::value ||
+                    std::is_same<NativeT, complex128>::value,
                 "Float or complex type required when specifying an ErrorSpec");
   Literal expected_literal = LiteralUtil::CreateR0<NativeT>(expected);
   ClientLibraryTestBase::ComputeAndCompareLiteral(builder, expected_literal,
@@ -455,7 +456,8 @@ void ClientLibraryTestBase::ComputeAndCompareR1(
                     std::is_same<NativeT, double>::value ||
                     std::is_same<NativeT, bfloat16>::value ||
                     std::is_same<NativeT, half>::value ||
-                    std::is_same<NativeT, complex64>::value,
+                    std::is_same<NativeT, complex64>::value ||
+                    std::is_same<NativeT, complex128>::value,
                 "Float or complex type required when specifying an ErrorSpec");
   Literal expected_literal = LiteralUtil::CreateR1<NativeT>(expected);
   ClientLibraryTestBase::ComputeAndCompareLiteral(builder, expected_literal,
@@ -480,7 +482,8 @@ void ClientLibraryTestBase::ComputeAndCompareR2(
                     std::is_same<NativeT, double>::value ||
                     std::is_same<NativeT, bfloat16>::value ||
                     std::is_same<NativeT, half>::value ||
-                    std::is_same<NativeT, complex64>::value,
+                    std::is_same<NativeT, complex64>::value ||
+                    std::is_same<NativeT, complex128>::value,
                 "Float or complex type required when specifying an ErrorSpec");
   Literal expected_literal =
       LiteralUtil::CreateR2FromArray2D<NativeT>(expected);
@@ -506,7 +509,8 @@ void ClientLibraryTestBase::ComputeAndCompareR3(
                     std::is_same<NativeT, double>::value ||
                     std::is_same<NativeT, bfloat16>::value ||
                     std::is_same<NativeT, half>::value ||
-                    std::is_same<NativeT, complex64>::value,
+                    std::is_same<NativeT, complex64>::value ||
+                    std::is_same<NativeT, complex128>::value,
                 "Float or complex type required when specifying an ErrorSpec");
   Literal expected_literal =
       LiteralUtil::CreateR3FromArray3D<NativeT>(expected);
@@ -532,7 +536,8 @@ void ClientLibraryTestBase::ComputeAndCompareR4(
                     std::is_same<NativeT, double>::value ||
                     std::is_same<NativeT, bfloat16>::value ||
                     std::is_same<NativeT, half>::value ||
-                    std::is_same<NativeT, complex64>::value,
+                    std::is_same<NativeT, complex64>::value ||
+                    std::is_same<NativeT, complex128>::value,
                 "Float or complex type required when specifying an ErrorSpec");
   Literal expected_literal =
       LiteralUtil::CreateR4FromArray4D<NativeT>(expected);
diff --git a/tensorflow/compiler/xla/types.h b/tensorflow/compiler/xla/types.h
index b645acb700..daf678f690 100644
--- a/tensorflow/compiler/xla/types.h
+++ b/tensorflow/compiler/xla/types.h
@@ -41,6 +41,7 @@ using ::tensorflow::uint32;
 using ::tensorflow::uint64;
 
 using complex64 = std::complex<float>;
+using complex128 = std::complex<double>;
 
 using ::Eigen::half;
 
diff --git a/tensorflow/compiler/xla/xla_data.proto b/tensorflow/compiler/xla/xla_data.proto
index d029669cbe..a64e2f5df5 100644
--- a/tensorflow/compiler/xla/xla_data.proto
+++ b/tensorflow/compiler/xla/xla_data.proto
@@ -56,6 +56,7 @@ enum PrimitiveType {
 
   // Complex values of fixed width.
   C64 = 15;  // Paired F32 (real, imag), as in std::complex<float>.
+  C128 = 18;  // Paired F64 (real, imag), as in std::complex<double>.
 
   // A tuple is a polymorphic sequence; e.g. a shape that holds different
   // sub-shapes. They are used for things like returning multiple values from a
@@ -75,7 +76,7 @@ enum PrimitiveType {
   // primitive type will have empty dimensions and tuple_shapes fields.
   TOKEN = 17;
 
-  // Next = 18
+  // Next = 19
 }
 
 // Describes the padding configuration for Pad operation. The padding amount on
@@ -367,6 +368,7 @@ message LiteralProto {
   repeated float f32s = 8;
   repeated double f64s = 9;
   repeated float c64s = 12;  // Stored as interleaved real, imag floats.
+  repeated double c128s = 18;  // Stored as interleaved real, imag doubles.
   repeated LiteralProto tuple_literals = 10;
   // The F16s, BF16s, U16s and S16s are encoded in little endian byte order
   bytes f16s = 11;
@@ -374,7 +376,7 @@ message LiteralProto {
   bytes u16s = 16;
   bytes s16s = 17;
   repeated int64 sparse_indices = 14;
-  // Next = 18
+  // Next = 19
 }
 
 message WindowDimension {
-- 
GitLab


From 5bc8f987097b9cccc63c3aa310595f719edd1344 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Fri, 11 Jan 2019 13:18:50 -0800
Subject: [PATCH 0581/2345] [XLA:Python] Remove unused variable to silence
 warning.

PiperOrigin-RevId: 228933374
---
 tensorflow/compiler/xla/python/local_computation_builder.i | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tensorflow/compiler/xla/python/local_computation_builder.i b/tensorflow/compiler/xla/python/local_computation_builder.i
index 66ecee5c4d..39bff3f571 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.i
+++ b/tensorflow/compiler/xla/python/local_computation_builder.i
@@ -794,8 +794,6 @@ tensorflow::ImportNumpy();
   }
   dimension_numbers.set_kernel_input_feature_dimension(value);
 
-  PyObject* o;
-
   if (!HandleRepeatedInt64Attribute(
         $input, "input_spatial_dimensions",
         dimension_numbers.mutable_input_spatial_dimensions())) {
-- 
GitLab


From 655d3387a362b5c4b1070fa847824acb3b6ae644 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Fri, 11 Jan 2019 13:24:09 -0800
Subject: [PATCH 0582/2345] Print disk usage after building stuff.

PiperOrigin-RevId: 228934277
---
 tensorflow/tools/ci_build/ci_parameterized_build.sh | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tensorflow/tools/ci_build/ci_parameterized_build.sh b/tensorflow/tools/ci_build/ci_parameterized_build.sh
index 41b9f241d5..0631806dc5 100755
--- a/tensorflow/tools/ci_build/ci_parameterized_build.sh
+++ b/tensorflow/tools/ci_build/ci_parameterized_build.sh
@@ -613,6 +613,13 @@ if [[ "${DO_DOCKER}" == "1" ]]; then
   fi
 fi
 
+# Set a disk usage trap.
+function finish {
+    echo "Finished script... disk usage report in ${TMP_DIR}"
+    du -k -d 2 ${TMP_DIR} | sort -n -r
+}
+trap finish EXIT
+
 chmod +x ${TMP_SCRIPT}
 
 # Map TF_BUILD container types to containers we actually have.
-- 
GitLab


From a1515c0edfd45bccf942564dad1d302ca4d0d491 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 13:29:56 -0800
Subject: [PATCH 0583/2345] Automated rollback of commit
 73b6d9262ea6d1578374ecb63560ad0fcad13e35

PiperOrigin-RevId: 228935205
---
 tensorflow/contrib/distribute/python/BUILD    |  30 +-
 ...test.py => keras_correctness_test_base.py} | 266 ++++--------------
 .../python/keras_dnn_correctness_test.py      | 163 +++++++++++
 .../keras_image_model_correctness_test.py     |  92 ++++++
 4 files changed, 342 insertions(+), 209 deletions(-)
 rename tensorflow/contrib/distribute/python/{keras_correctness_test.py => keras_correctness_test_base.py} (58%)
 create mode 100644 tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
 create mode 100644 tensorflow/contrib/distribute/python/keras_image_model_correctness_test.py

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index d4758d7518..4c25f49f16 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -657,7 +657,11 @@ cuda_py_test(
 py_library(
     name = "keras_correctness_test_lib",
     testonly = 1,
-    srcs = ["keras_correctness_test.py"],
+    srcs = [
+        "keras_correctness_test_base.py",
+        "keras_dnn_correctness_test.py",
+        "keras_image_model_correctness_test.py",
+    ],
     deps = [
         ":combinations",
         "//tensorflow/contrib/distribute/python:mirrored_strategy",
@@ -673,8 +677,9 @@ py_library(
 )
 
 cuda_py_test(
-    name = "keras_correctness_test",
-    srcs = ["keras_correctness_test.py"],
+    name = "keras_dnn_correctness_test",
+    size = "medium",
+    srcs = ["keras_dnn_correctness_test.py"],
     additional_deps = [
         ":keras_correctness_test_lib",
     ],
@@ -690,6 +695,25 @@ cuda_py_test(
     ],
 )
 
+cuda_py_test(
+    name = "keras_image_model_correctness_test",
+    size = "medium",
+    srcs = ["keras_image_model_correctness_test.py"],
+    additional_deps = [
+        ":keras_correctness_test_lib",
+    ],
+    # Shard count is set to an odd number to distribute tasks across
+    # shards more evenly.
+    shard_count = 31,
+    tags = [
+        "multi_and_single_gpu",
+        "no_oss",  # TODO(b/117919883): Fix python error.
+        "no_pip",
+        "no_windows_gpu",
+        "notsan",
+    ],
+)
+
 py_library(
     name = "metrics_v1_test_lib",
     testonly = 1,
diff --git a/tensorflow/contrib/distribute/python/keras_correctness_test.py b/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
similarity index 58%
rename from tensorflow/contrib/distribute/python/keras_correctness_test.py
rename to tensorflow/contrib/distribute/python/keras_correctness_test_base.py
index 3abdee2c0e..9f3cab49a8 100644
--- a/tensorflow/contrib/distribute/python/keras_correctness_test.py
+++ b/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
@@ -30,8 +30,6 @@ from tensorflow.python.eager import context
 from tensorflow.python.eager import test
 from tensorflow.python.framework import random_seed
 from tensorflow.python.keras.engine import distributed_training_utils
-from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_descent_keras
-from tensorflow.python.training import gradient_descent
 
 _RANDOM_SEED = 1337
 _EVAL_STEPS = 20
@@ -62,24 +60,13 @@ def all_strategy_combinations_with_graph_mode():
   return combinations.combine(distribution=all_strategies, mode=['graph'])
 
 
-def strategy_and_input_combinations():
-  def cnn_model_with_batch_norm(**kwargs):
-    return _create_cnn_model(with_batch_norm=True, **kwargs)
-
+def all_strategy_and_input_config_combinations():
   return (
       combinations.times(
           combinations.combine(distribution=all_strategies),
           combinations.combine(mode=['graph', 'eager'],
                                use_numpy=[True, False],
-                               use_validation_data=[True, False]),
-          combinations.combine(model_with_data=[
-              ModelWithData('dnn', _create_dnn_model, _dnn_training_data),
-              ModelWithData('cnn', _create_cnn_model, _cnn_training_data),
-              ModelWithData('cnn_batch_norm',
-                            cnn_model_with_batch_norm,
-                            _cnn_training_data,
-                            with_batch_norm=True),
-          ])))
+                               use_validation_data=[True, False])))
 
 
 class MaybeDistributionScope(object):
@@ -100,97 +87,6 @@ class MaybeDistributionScope(object):
       self._scope = None
 
 
-class ModelWithData(object):
-  """An object giving a good name in combinations.
-
-  The model_fn must take two arguments: initial_weights and distribution.
-  """
-
-  def __init__(self, name, model_fn, data_fn, with_batch_norm=False):
-    self.name = name
-    self.model_fn = model_fn
-    self.data_fn = data_fn
-    self.with_batch_norm = with_batch_norm
-
-  def __repr__(self):
-    return self.name
-
-
-def _dnn_training_data():
-  # TODO(xiejw): Change this back to 10000, once we support final partial
-  # batch.
-  num_samples = 9984
-  x_train = np.random.rand(num_samples, 1)
-  y_train = 3 * x_train
-  x_train = x_train.astype('float32')
-  y_train = y_train.astype('float32')
-  x_predict = [[1.], [2.], [3.], [4.]]
-  return x_train, y_train, x_predict
-
-
-def _create_dnn_model(initial_weights=None, distribution=None):
-  with MaybeDistributionScope(distribution):
-    # We add few non-linear layers to make it non-trivial.
-    model = keras.Sequential()
-    model.add(keras.layers.Dense(10, activation='relu', input_shape=(1,)))
-    model.add(keras.layers.Dense(10, activation='relu'))
-    model.add(keras.layers.Dense(10, activation='relu'))
-    model.add(keras.layers.Dense(1))
-
-    if initial_weights:
-      model.set_weights(initial_weights)
-
-    model.compile(
-        loss=keras.losses.mean_squared_error,
-        optimizer=gradient_descent_keras.SGD(0.5),
-        metrics=['mse'])
-    return model
-
-
-def _cnn_training_data(count=_GLOBAL_BATCH_SIZE * _EVAL_STEPS,
-                       shape=(28, 28, 3), num_classes=10):
-  centers = np.random.randn(num_classes, *shape)
-
-  features = []
-  labels = []
-  for _ in range(count):
-    label = np.random.randint(0, num_classes, size=1)[0]
-    offset = np.random.normal(loc=0, scale=0.1, size=np.prod(shape))
-    offset = offset.reshape(shape)
-    labels.append(label)
-    features.append(centers[label] + offset)
-
-  x_train = np.asarray(features, dtype=np.float32)
-  y_train = np.asarray(labels, dtype=np.float32).reshape((count, 1))
-  x_predict = x_train
-  return x_train, y_train, x_predict
-
-
-def _create_cnn_model(initial_weights=None, distribution=None,
-                      with_batch_norm=False):
-  with MaybeDistributionScope(distribution):
-    image = keras.layers.Input(shape=(28, 28, 3), name='image')
-    c1 = keras.layers.Conv2D(
-        name='conv1', filters=16, kernel_size=(3, 3), strides=(4, 4))(
-            image)
-    if with_batch_norm:
-      c1 = keras.layers.BatchNormalization(name='bn1')(c1)
-    c1 = keras.layers.MaxPooling2D(pool_size=(2, 2))(c1)
-    logits = keras.layers.Dense(
-        10, activation='softmax', name='pred')(
-            keras.layers.Flatten()(c1))
-    model = keras.Model(inputs=[image], outputs=[logits])
-
-    if initial_weights:
-      model.set_weights(initial_weights)
-
-    model.compile(
-        optimizer=gradient_descent.GradientDescentOptimizer(learning_rate=0.1),
-        loss='sparse_categorical_crossentropy',
-        metrics=['sparse_categorical_accuracy'])
-  return model
-
-
 def batch_wrapper(dataset, batch_size, distribution, repeat=None):
   if repeat:
     dataset = dataset.repeat(repeat)
@@ -289,6 +185,7 @@ def get_correctness_test_inputs(use_numpy, use_validation_data,
 
 def fit_eval_and_predict(
     initial_weights, input_fn, model_fn, distribution=None):
+  """Generates results for fit/predict/evaluate for given model."""
   model = model_fn(initial_weights=initial_weights, distribution=distribution)
   training_inputs, eval_inputs, predict_inputs = input_fn(distribution)
 
@@ -317,6 +214,8 @@ def fit_eval_and_predict(
 
 def compare_results(results_with_ds, results_without_ds, distribution,
                     testcase):
+  """Compares results of model compiled with/without distribution strategy."""
+
   default_tolerance = 1e-5
   tol_table = {}
 
@@ -350,7 +249,13 @@ def compare_results(results_with_ds, results_without_ds, distribution,
         msg='Fail to assert {}.'.format(key))
 
 
+def should_skip_tpu_with_eager(distribution):
+  return (context.executing_eagerly() and
+          isinstance(distribution, tpu_strategy.TPUStrategy))
+
+
 class LearningRateBatchScheduler(keras.callbacks.Callback):
+  """Scheduler that dynamically sets the learning rate of model."""
 
   def __init__(self, update_freq=None):
     self._update_freq = update_freq
@@ -364,107 +269,60 @@ class LearningRateBatchScheduler(keras.callbacks.Callback):
     keras.backend.set_value(self.model.optimizer.lr, lr)
 
 
-class TestDistributionStrategyCorrectness(test.TestCase,
-                                          parameterized.TestCase):
+class TestDistributionStrategyCorrectnessBase(test.TestCase,
+                                              parameterized.TestCase):
+  """Model agnostic testing infra to test correctness of Keras models."""
 
-  def _should_skip_tpu_with_eager(self, distribution):
-    return (context.executing_eagerly() and
-            isinstance(distribution, tpu_strategy.TPUStrategy))
+  def set_up_test_config(self, use_numpy=False,
+                         use_validation_data=False,
+                         with_batch_norm=False):
+    self.use_numpy = use_numpy
+    self.use_validation_data = use_validation_data
+    self.with_batch_norm = with_batch_norm
 
-  @combinations.generate(all_strategy_combinations_with_eager_and_graph_modes())
-  def test_metric_correctness(self, distribution):
-    if self._should_skip_tpu_with_eager(distribution):
-      self.skipTest('TPUStrategy does not support eager mode now.')
+    keras.backend.set_image_data_format('channels_last')
+    np.random.seed(_RANDOM_SEED)
+    random_seed.set_random_seed(_RANDOM_SEED)
 
-    with self.cached_session():
-      keras.backend.set_image_data_format('channels_last')
-      num_samples = 10000
-
-      x_train = np.random.randint(0, 2, num_samples)
-      x_train = np.reshape(x_train, (num_samples, 1))
-      y_train = x_train
-      x_train = x_train.astype('float32')
-      y_train = y_train.astype('float32')
-
-      # Create identity model.
-      with distribution.scope():
-        model = keras.Sequential()
-        model.add(
-            keras.layers.Dense(1, input_shape=(1,), kernel_initializer='ones'))
-        model.compile(
-            loss=keras.losses.mean_squared_error,
-            optimizer=gradient_descent.GradientDescentOptimizer(0.5),
-            metrics=[keras.metrics.BinaryAccuracy()])
-
-      batch_size = 64
-      batch_size = get_batch_size(batch_size, distribution)
-      train_dataset = dataset_ops.Dataset.from_tensor_slices((x_train, y_train))
-      train_dataset = batch_wrapper(train_dataset, batch_size, distribution)
-
-      history = model.fit(x=train_dataset, epochs=2, steps_per_epoch=10)
-      self.assertEqual(history.history['binary_accuracy'], [1.0, 1.0])
-
-  @combinations.generate(all_strategy_combinations_with_eager_and_graph_modes())
-  def test_eval_metrics_correctness(self, distribution):
-    if self._should_skip_tpu_with_eager(distribution):
-      self.skipTest('TPUStrategy does not support eager mode now.')
+  def get_data(self):
+    num_samples = 10000
+    x_train = np.random.randint(0, 2, num_samples)
+    x_train = np.reshape(x_train, (num_samples, 1))
+    y_train = x_train
+    return (x_train.astype('float32'), y_train.astype('float32'), None)
 
-    with self.cached_session():
-      with distribution.scope():
-        model = keras.Sequential()
-        model.add(
-            keras.layers.Dense(
-                3, activation='relu', input_dim=4, kernel_initializer='ones'))
-        model.add(
-            keras.layers.Dense(
-                1, activation='sigmoid', kernel_initializer='ones'))
-        model.compile(
-            loss='mae',
-            metrics=['accuracy', keras.metrics.BinaryAccuracy()],
-            optimizer=gradient_descent.GradientDescentOptimizer(0.001))
-
-      # verify correctness of stateful and stateless metrics.
-      x = np.ones((100, 4)).astype('float32')
-      y = np.ones((100, 1)).astype('float32')
-      dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat()
-      dataset = batch_wrapper(dataset, 4, distribution)
-      outs = model.evaluate(dataset, steps=10)
-      self.assertEqual(outs[1], 1.)
-      self.assertEqual(outs[2], 1.)
-
-      y = np.zeros((100, 1)).astype('float32')
-      dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat()
-      dataset = batch_wrapper(dataset, 4, distribution)
-      outs = model.evaluate(dataset, steps=10)
-      self.assertEqual(outs[1], 0.)
-      self.assertEqual(outs[2], 0.)
-
-  @combinations.generate(strategy_and_input_combinations())
-  def test_correctness(self, distribution, use_numpy, use_validation_data,
-                       model_with_data):
-    if self._should_skip_tpu_with_eager(distribution):
+  def get_model(self, distribution=None):
+    raise NotImplementedError
+
+  def skip_unsupported_test_configuration(self, distribution):
+    if should_skip_tpu_with_eager(distribution):
       self.skipTest('TPUStrategy does not support eager mode now.')
 
-    if context.executing_eagerly() and use_numpy:
+    if context.executing_eagerly() and self.use_numpy:
       self.skipTest('Numpy as inputs is not supported with strategy in eager.')
 
-    if context.executing_eagerly() and use_validation_data:
-      self.skipTest('TODO')
+    if context.executing_eagerly() and self.use_validation_data:
+      self.skipTest('TODO(hongjunchoi): Add test logic for using validation '
+                    'data for eager execution.')
+    return
 
+  def run_correctness_test(self,
+                           distribution,
+                           use_numpy,
+                           use_validation_data,
+                           with_batch_norm=False):
     with self.cached_session():
-      keras.backend.set_image_data_format('channels_last')
-      np.random.seed(_RANDOM_SEED)
-      random_seed.set_random_seed(_RANDOM_SEED)
+      self.set_up_test_config(use_numpy, use_validation_data, with_batch_norm)
+      self.skip_unsupported_test_configuration(distribution)
 
-      model_fn, data_fn = model_with_data.model_fn, model_with_data.data_fn
       # Train, eval, and predict datasets are created with the same input numpy
       # arrays.
-      x_train, y_train, x_predict = data_fn()
+      x_train, y_train, x_predict = self.get_data()
 
       # The model is built once and the initial weights are saved.
       # This is used to initialize the model for both the distribution and
       # non-distribution run.
-      model = model_fn()
+      model = self.get_model()
       initial_weights = model.get_weights()
 
       def input_fn(dist):
@@ -472,15 +330,15 @@ class TestDistributionStrategyCorrectness(test.TestCase,
             use_numpy, use_validation_data, dist, x_train, y_train, x_predict)
 
       results_with_ds = fit_eval_and_predict(
-          initial_weights, input_fn=input_fn, model_fn=model_fn,
+          initial_weights, input_fn=input_fn, model_fn=self.get_model,
           distribution=distribution)
       results_without_ds = fit_eval_and_predict(
-          initial_weights, input_fn=input_fn, model_fn=model_fn,
+          initial_weights, input_fn=input_fn, model_fn=self.get_model,
           distribution=None)
 
       # First, special case, for multi-replica distributed training, batch norm
       # is not aggregated globally. So it is expected to have different weights.
-      if (model_with_data.with_batch_norm and
+      if (self.with_batch_norm and
           distribution.num_replicas_in_sync > 1):
         with self.assertRaises(AssertionError):
           compare_results(results_with_ds, results_without_ds, distribution,
@@ -489,21 +347,16 @@ class TestDistributionStrategyCorrectness(test.TestCase,
         compare_results(results_with_ds, results_without_ds, distribution,
                         testcase=self)
 
-  @combinations.generate(all_strategy_combinations_with_graph_mode())
-  def test_dynamic_lr(self, distribution):
-
+  def run_dynamic_lr_test(self, distribution):
     with self.cached_session():
+      self.set_up_test_config()
+      self.skip_unsupported_test_configuration(distribution)
 
-      keras.backend.set_image_data_format('channels_last')
-      np.random.seed(_RANDOM_SEED)
-      random_seed.set_random_seed(_RANDOM_SEED)
-
-      x_train, y_train, _ = _dnn_training_data()
-
-      model = _create_dnn_model()
+      x_train, y_train, _ = self.get_data()
+      model = self.get_model()
       initial_weights = model.get_weights()
-
       update_freq = None
+
       if (isinstance(distribution, tpu_strategy.TPUStrategy) and
           distribution.extended.steps_per_run > 1):
         # For TPUStrategy with steps_per_run > 1, the callback is not invoked
@@ -512,6 +365,7 @@ class TestDistributionStrategyCorrectness(test.TestCase,
         update_freq = distribution.extended.steps_per_run
 
       def input_fn(dist):
+        """Generates training test given test configuration."""
         training_epochs = 2
         global_batch_size = 64
         batch_size = get_batch_size(global_batch_size, dist)
@@ -530,10 +384,10 @@ class TestDistributionStrategyCorrectness(test.TestCase,
         return training_inputs, eval_inputs, predict_inputs
 
       results_with_ds = fit_eval_and_predict(
-          initial_weights, input_fn=input_fn, model_fn=_create_dnn_model,
+          initial_weights, input_fn=input_fn, model_fn=self.get_model,
           distribution=distribution)
       results_without_ds = fit_eval_and_predict(
-          initial_weights, input_fn=input_fn, model_fn=_create_dnn_model,
+          initial_weights, input_fn=input_fn, model_fn=self.get_model,
           distribution=None)
       compare_results(results_with_ds, results_without_ds, distribution,
                       testcase=self)
diff --git a/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py b/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
new file mode 100644
index 0000000000..f21b1c6752
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
@@ -0,0 +1,163 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Correctness tests for tf.keras DNN model using DistributionStrategy."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.distribute.python import combinations
+from tensorflow.contrib.distribute.python import keras_correctness_test_base
+from tensorflow.python import keras
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.eager import test
+from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_descent_keras
+from tensorflow.python.training import gradient_descent
+
+
+class TestDistributionStrategyDnnCorrectness(
+    keras_correctness_test_base.TestDistributionStrategyCorrectnessBase):
+
+  def get_model(self, initial_weights=None, distribution=None):
+    with keras_correctness_test_base.MaybeDistributionScope(distribution):
+      # We add few non-linear layers to make it non-trivial.
+      model = keras.Sequential()
+      model.add(keras.layers.Dense(10, activation='relu', input_shape=(1,)))
+      model.add(keras.layers.Dense(10, activation='relu'))
+      model.add(keras.layers.Dense(10, activation='relu'))
+      model.add(keras.layers.Dense(1))
+
+      if initial_weights:
+        model.set_weights(initial_weights)
+
+      model.compile(
+          loss=keras.losses.mean_squared_error,
+          optimizer=gradient_descent_keras.SGD(0.5),
+          metrics=['mse'])
+      return model
+
+  def get_data(self):
+    # TODO(xiejw): Change this back to 10000, once we support final partial
+    # batch.
+    num_samples = 9984
+    x_train = np.random.rand(num_samples, 1)
+    y_train = 3 * x_train
+    x_train = x_train.astype('float32')
+    y_train = y_train.astype('float32')
+    x_predict = [[1.], [2.], [3.], [4.]]
+    return x_train, y_train, x_predict
+
+  @combinations.generate(keras_correctness_test_base.
+                         all_strategy_and_input_config_combinations())
+  def test_dnn_correctness(self, distribution, use_numpy, use_validation_data):
+    self.run_correctness_test(distribution, use_numpy, use_validation_data)
+
+  @combinations.generate(keras_correctness_test_base.
+                         all_strategy_combinations_with_graph_mode())
+  def test_dnn_with_dynamic_learning_rate(self, distribution):
+    self.run_dynamic_lr_test(distribution)
+
+
+class TestDistributionStrategyDnnMetricCorrectness(
+    keras_correctness_test_base.TestDistributionStrategyCorrectnessBase):
+
+  def get_model(self, distribution=None):
+    with distribution.scope():
+      model = keras.Sequential()
+      model.add(keras.layers.Dense(1,
+                                   input_shape=(1,),
+                                   kernel_initializer='ones'))
+      model.compile(
+          loss=keras.losses.mean_squared_error,
+          optimizer=gradient_descent.GradientDescentOptimizer(0.5),
+          metrics=[keras.metrics.BinaryAccuracy()])
+    return model
+
+  def run_metric_correctness_test(self, distribution):
+    with self.cached_session():
+      self.set_up_test_config()
+      self.skip_unsupported_test_configuration(distribution)
+
+      x_train, y_train, _ = self.get_data()
+      model = self.get_model(distribution=distribution)
+
+      batch_size = 64
+      batch_size = (keras_correctness_test_base.
+                    get_batch_size(batch_size, distribution))
+      train_dataset = dataset_ops.Dataset.from_tensor_slices((x_train, y_train))
+      train_dataset = (keras_correctness_test_base.
+                       batch_wrapper(train_dataset, batch_size, distribution))
+
+      history = model.fit(x=train_dataset, epochs=2, steps_per_epoch=10)
+      self.assertEqual(history.history['binary_accuracy'], [1.0, 1.0])
+
+  @combinations.generate(keras_correctness_test_base.
+                         all_strategy_combinations_with_eager_and_graph_modes())
+  def test_simple_dnn_metric_correctness(self, distribution):
+    self.run_metric_correctness_test(distribution)
+
+
+class TestDistributionStrategyDnnMetricEvalCorrectness(
+    keras_correctness_test_base.TestDistributionStrategyCorrectnessBase):
+
+  def get_model(self, distribution=None):
+    with distribution.scope():
+      model = keras.Sequential()
+      model.add(
+          keras.layers.Dense(
+              3, activation='relu', input_dim=4, kernel_initializer='ones'))
+      model.add(
+          keras.layers.Dense(
+              1, activation='sigmoid', kernel_initializer='ones'))
+      model.compile(
+          loss='mae',
+          metrics=['accuracy', keras.metrics.BinaryAccuracy()],
+          optimizer=gradient_descent.GradientDescentOptimizer(0.001))
+    return model
+
+  def run_eval_metrics_correctness_test(self, distribution):
+    with self.cached_session():
+      self.set_up_test_config()
+      self.skip_unsupported_test_configuration(distribution)
+
+      model = self.get_model(distribution=distribution)
+
+      # verify correctness of stateful and stateless metrics.
+      x = np.ones((100, 4)).astype('float32')
+      y = np.ones((100, 1)).astype('float32')
+      dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat()
+      dataset = (keras_correctness_test_base.
+                 batch_wrapper(dataset, 4, distribution))
+      outs = model.evaluate(dataset, steps=10)
+      self.assertEqual(outs[1], 1.)
+      self.assertEqual(outs[2], 1.)
+
+      y = np.zeros((100, 1)).astype('float32')
+      dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat()
+      dataset = (keras_correctness_test_base.
+                 batch_wrapper(dataset, 4, distribution))
+      outs = model.evaluate(dataset, steps=10)
+      self.assertEqual(outs[1], 0.)
+      self.assertEqual(outs[2], 0.)
+
+  @combinations.generate(keras_correctness_test_base.
+                         all_strategy_combinations_with_eager_and_graph_modes())
+  def test_identity_model_metric_eval_correctness(self, distribution):
+    self.run_eval_metrics_correctness_test(distribution)
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/contrib/distribute/python/keras_image_model_correctness_test.py b/tensorflow/contrib/distribute/python/keras_image_model_correctness_test.py
new file mode 100644
index 0000000000..f625664372
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/keras_image_model_correctness_test.py
@@ -0,0 +1,92 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Correctness tests for tf.keras CNN models using DistributionStrategy."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.distribute.python import combinations
+from tensorflow.contrib.distribute.python import keras_correctness_test_base
+from tensorflow.python import keras
+from tensorflow.python.eager import test
+from tensorflow.python.training import gradient_descent
+
+
+class DistributionStrategyCnnCorrectnessTest(
+    keras_correctness_test_base.TestDistributionStrategyCorrectnessBase):
+
+  def get_model(self, initial_weights=None, distribution=None):
+    with keras_correctness_test_base.MaybeDistributionScope(distribution):
+      image = keras.layers.Input(shape=(28, 28, 3), name='image')
+      c1 = keras.layers.Conv2D(
+          name='conv1', filters=16, kernel_size=(3, 3), strides=(4, 4))(
+              image)
+      if self.with_batch_norm:
+        c1 = keras.layers.BatchNormalization(name='bn1')(c1)
+      c1 = keras.layers.MaxPooling2D(pool_size=(2, 2))(c1)
+      logits = keras.layers.Dense(
+          10, activation='softmax', name='pred')(
+              keras.layers.Flatten()(c1))
+      model = keras.Model(inputs=[image], outputs=[logits])
+
+      if initial_weights:
+        model.set_weights(initial_weights)
+
+      model.compile(
+          optimizer=gradient_descent.GradientDescentOptimizer(
+              learning_rate=0.1),
+          loss='sparse_categorical_crossentropy',
+          metrics=['sparse_categorical_accuracy'])
+
+    return model
+
+  def get_data(self,
+               count=keras_correctness_test_base._GLOBAL_BATCH_SIZE
+               * keras_correctness_test_base._EVAL_STEPS,
+               shape=(28, 28, 3),
+               num_classes=10):
+    centers = np.random.randn(num_classes, *shape)
+
+    features = []
+    labels = []
+    for _ in range(count):
+      label = np.random.randint(0, num_classes, size=1)[0]
+      offset = np.random.normal(loc=0, scale=0.1, size=np.prod(shape))
+      offset = offset.reshape(shape)
+      labels.append(label)
+      features.append(centers[label] + offset)
+
+    x_train = np.asarray(features, dtype=np.float32)
+    y_train = np.asarray(labels, dtype=np.float32).reshape((count, 1))
+    x_predict = x_train
+    return x_train, y_train, x_predict
+
+  @combinations.generate(keras_correctness_test_base.
+                         all_strategy_and_input_config_combinations())
+  def test_cnn_correctness(self, distribution, use_numpy, use_validation_data):
+    self.run_correctness_test(distribution, use_numpy, use_validation_data)
+
+  @combinations.generate(keras_correctness_test_base.
+                         all_strategy_and_input_config_combinations())
+  def test_cnn_with_batch_norm_correctness(self, distribution, use_numpy,
+                                           use_validation_data):
+    self.run_correctness_test(distribution, use_numpy, use_validation_data,
+                              with_batch_norm=True)
+
+
+if __name__ == '__main__':
+  test.main()
-- 
GitLab


From b6e1f16f12ad9408fb8c983bc98bd62234ed8e39 Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Fri, 11 Jan 2019 13:47:13 -0800
Subject: [PATCH 0584/2345] Constant-fold GetDimensionSize ops in XLABuilder.

This CL implements constant-folding for GetDimensionSize ops if there is no dynamic dimension.

- If the operand of GetDimensionSize is constant dimension, consider GetDimensionSize as constant.

- Replace GetDimensionSize with a constant instruction in the constant subgraph for constant folding.

PiperOrigin-RevId: 228938348
---
 tensorflow/compiler/xla/client/xla_builder.cc | 61 ++++++++++++++++---
 .../xla/tests/compute_constant_test.cc        | 19 +++---
 2 files changed, 62 insertions(+), 18 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc
index 59e156eb72..c98f5b1140 100644
--- a/tensorflow/compiler/xla/client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_builder.cc
@@ -211,11 +211,21 @@ void XlaBuilder::IsConstantVisitor(const int64 op_handle,
       }
       // TODO(b/32495713): We aren't checking the called computations.
       break;
+    case HloOpcode::kGetDimensionSize: {
+      int64 dimension_number = instr.dimensions(0);
+      const HloInstructionProto& operand =
+          *(LookUpInstructionByHandle(instr.operand_ids(0)).ValueOrDie());
+      Shape operand_shape(operand.shape());
+      if (operand_shape.is_dynamic_dimension(dimension_number)) {
+        *is_constant = false;
+      }
+      break;
+    }
 
     // Non functional ops.
     case HloOpcode::kRng:
     case HloOpcode::kAllReduce:
-      // TODO(b/33009255): Implmement constant folding for cross replica sum.
+      // TODO(b/33009255): Implement constant folding for cross replica sum.
     case HloOpcode::kInfeed:
     case HloOpcode::kOutfeed:
     case HloOpcode::kCall:
@@ -2541,21 +2551,56 @@ StatusOr<XlaComputation> XlaBuilder::BuildConstantSubGraph(
     worklist.pop();
     TF_ASSIGN_OR_RETURN(const HloInstructionProto* instr_proto,
                         LookUpInstructionByHandle(handle));
-    for (int64 id : instr_proto->operand_ids()) {
-      if (related_ops.insert(id).second) {
-        worklist.push(id);
+
+    if (instr_proto->opcode() ==
+        HloOpcodeString(HloOpcode::kGetDimensionSize)) {
+      // At this point, BuildConstantSubGraph should never encounter a
+      // GetDimensionSize with a dynamic dimension. IsConstant check would have
+      // failed at the beginning of this function.
+      //
+      // Replace GetDimensionSize with a Constant representing the static bound
+      // of the shape.
+      int64 dimension = instr_proto->dimensions(0);
+      int64 operand_handle = instr_proto->operand_ids(0);
+      TF_ASSIGN_OR_RETURN(const HloInstructionProto* operand_proto,
+                          LookUpInstructionByHandle(operand_handle));
+
+      TF_RET_CHECK(!operand_proto->shape().is_dynamic_dimension(dimension));
+      auto constant_dimension_size =
+          static_cast<uint32>(operand_proto->shape().dimensions(dimension));
+
+      Literal literal = LiteralUtil::CreateR0(constant_dimension_size);
+
+      HloInstructionProto const_instr;
+      *const_instr.mutable_shape() = literal.shape().ToProto();
+      *const_instr.mutable_literal() = literal.ToProto();
+      *const_instr.mutable_opcode() = HloOpcodeString(HloOpcode::kConstant);
+      *const_instr.mutable_name() = HloOpcodeString(HloOpcode::kConstant);
+      const_instr.set_id(handle);
+      *entry.add_instructions() =
+          const_instr;  // Add to the result constant graph.
+    } else {
+      for (int64 id : instr_proto->operand_ids()) {
+        if (related_ops.insert(id).second) {
+          worklist.push(id);
+        }
+      }
+      for (int64 called_id : instr_proto->called_computation_ids()) {
+        related_calls.insert(called_id);
       }
-    }
-    for (int64 called_id : instr_proto->called_computation_ids()) {
-      related_calls.insert(called_id);
     }
   }
 
   // Add related ops to the computation.
   for (int64 id : related_ops) {
-    auto* instr = entry.add_instructions();
     TF_ASSIGN_OR_RETURN(const HloInstructionProto* instr_src,
                         LookUpInstructionByHandle(id));
+
+    if (instr_src->opcode() == HloOpcodeString(HloOpcode::kGetDimensionSize)) {
+      continue;
+    }
+    auto* instr = entry.add_instructions();
+
     *instr = *instr_src;
     // Ensures that the instruction names are unique among the graph.
     const string& new_name =
diff --git a/tensorflow/compiler/xla/tests/compute_constant_test.cc b/tensorflow/compiler/xla/tests/compute_constant_test.cc
index 3b0414a604..960f496977 100644
--- a/tensorflow/compiler/xla/tests/compute_constant_test.cc
+++ b/tensorflow/compiler/xla/tests/compute_constant_test.cc
@@ -151,19 +151,18 @@ TEST_F(ComputeConstantTest, DirectParamMissing) {
   }
 }
 
-TEST_F(ComputeConstantTest, IndirectParamMissing) {
+TEST_F(ComputeConstantTest, GetDimensionSize) {
   for (ClientType client_type : client_types) {
     Client* client = ClientOrDie(platform_, client_type);
     XlaBuilder b(TestName());
-    auto computation =
-        Add(ConstantR0<float>(&b, 1.0f),
-            Parameter(&b, 0, ShapeUtil::MakeShape(F32, {}), "param"));
-    EXPECT_FALSE(IsConstant(computation, &b));
-
-    auto value = ComputeConstantScalar<float>(client, computation, &b);
-    EXPECT_TRUE(
-        absl::StrContains(value.status().ToString(), "depends on a parameter"))
-        << value.status();
+    auto add =
+        Add(ConstantR1<float>(&b, {1.0f}), ConstantR1<float>(&b, {1.0f}));
+    auto get_dimension_size = GetDimensionSize(add, 0);
+    EXPECT_TRUE(IsConstant(get_dimension_size, &b));
+
+    TF_ASSERT_OK_AND_ASSIGN(auto value, ComputeConstantScalar<uint32>(
+                                            client, get_dimension_size, &b));
+    EXPECT_EQ(value, 1);
   }
 }
 
-- 
GitLab


From b96c6716234797aadaec84d1952c309c76cbba8a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 13:53:33 -0800
Subject: [PATCH 0585/2345] Auto-detect number of hosts, cores, etc in
 mid-level TPU embedding API.

PiperOrigin-RevId: 228939416
---
 tensorflow/contrib/tpu/BUILD                  |  3 +-
 .../contrib/tpu/python/tpu/tpu_embedding.py   | 44 +++++++------------
 2 files changed, 18 insertions(+), 29 deletions(-)

diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index b9c225b44a..0fe9d0cdfc 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -453,7 +453,8 @@ py_library(
     srcs = ["python/tpu/tpu_embedding.py"],
     srcs_version = "PY2AND3",
     deps = [
-        "//tensorflow/contrib/tpu:tpu_ops",
+        ":tpu_lib",
+        ":tpu_ops",
         "//tensorflow/contrib/tpu/proto:tpu_embedding_configuration_proto_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:framework_for_generated_wrappers",
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py b/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py
index ccba8a46c7..1a909a3ac6 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py
@@ -28,6 +28,7 @@ from tensorflow.contrib.framework.python.framework import experimental
 from tensorflow.contrib.tpu.ops import gen_tpu_ops
 from tensorflow.contrib.tpu.proto import tpu_embedding_configuration_pb2 as elc
 from tensorflow.contrib.tpu.python.ops import tpu_ops
+from tensorflow.contrib.tpu.python.tpu import tpu_system_metadata as tpu_system_metadata_lib
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
@@ -43,19 +44,6 @@ from tensorflow.python.ops import variables
 TRAINING = elc.TPUEmbeddingConfiguration.TRAINING
 INFERENCE = elc.TPUEmbeddingConfiguration.INFERENCE
 
-# TODO(shizhiw): A better interface is to make `num_hosts` and
-# `num_cores_per_host` optional parameters for `TPUEmbedding`
-# constructor. Usually they can be automatically detected, but
-# user can also specify them for debugging (b/112112496).
-# Auto-detection can be done with `tpu_system_metadata.py`.
-_MASTER_JOB = 'tpu_worker'
-_HOST_PATTERN = '/job:tpu_worker/task:{}/device:CPU:0'
-_NUM_CORES_PER_HOST = 8
-
-_TEST_MASTER_JOB = None
-_TEST_HOST = '/replica:0/task:0/device:CPU:0'
-_TEST_NUM_CORES_PER_HOST = 2
-
 
 class TableConfig(
     collections.namedtuple(
@@ -301,10 +289,9 @@ class TPUEmbedding(object):
                table_to_config_dict,
                feature_to_table_dict,
                batch_size,
-               num_hosts,
                mode,
-               optimization_parameters=None,
-               tpu_embedding_test=False):
+               master,
+               optimization_parameters=None):
     """API for using TPU for embedding lookups.
 
     Args:
@@ -315,12 +302,11 @@ class TPUEmbedding(object):
         to string of table name. Feature refers to ids to lookup in embedding
         table, e.g. `sp_ids` argument to `tf.nn.embedding_lookup_sparse()`.
       batch_size: An `int` representing the global batch size.
-      num_hosts: An `int` representing the number of TPU hosts.
       mode: `TRAINING` or `INFERENCE`.
+      master: A `string` representing the TensorFlow master to use.
       optimization_parameters: `AdagradParameters`, `AdamParameters`,
         `Stochasticgradientdescentparameters`. Must be set in training and must
         be `None` in inference.
-      tpu_embedding_test: A `bool`. Only used for testing.
 
     Raises:
       ValueError: if any input is invalid.
@@ -337,15 +323,17 @@ class TPUEmbedding(object):
 
     self._batch_size = batch_size
 
-    if tpu_embedding_test:
-      self._num_hosts = 1
-      self._hosts = [_TEST_HOST]
-      self._num_cores_per_host = _TEST_NUM_CORES_PER_HOST
-    else:
-      self._num_hosts = num_hosts
-      self._hosts = [_HOST_PATTERN.format(i) for i in range(self._num_hosts)]
-      self._num_cores_per_host = _NUM_CORES_PER_HOST
-    self._num_cores = self._num_cores_per_host * self._num_hosts
+    self._master = master
+    self._tpu_system_metadata = (
+        tpu_system_metadata_lib._query_tpu_system_metadata(self._master))  # pylint: disable=protected-access
+    if self._tpu_system_metadata.num_cores == 0:
+      raise ValueError('TPUEmbedding needs TPUs, but master {} does not have '
+                       'TPUs.'.format(self._master))
+    self._num_hosts = self._tpu_system_metadata.num_hosts
+    self._hosts = [device.name for device in self._tpu_system_metadata.devices
+                   if 'device:CPU:' in device.name]
+    self._num_cores_per_host = self._tpu_system_metadata.num_of_cores_per_host
+    self._num_cores = self._tpu_system_metadata.num_cores
 
     _validate_batch_size(self._batch_size, self._num_cores)
     self._batch_size_per_core = self._batch_size // self._num_cores
@@ -389,7 +377,7 @@ class TPUEmbedding(object):
     Returns:
       A list of device names for CPU hosts.
     """
-    return self._hosts
+    return copy.copy(self._hosts)
 
   # TODO(shizhiw): change to num_tensor_cores_per_host to be more explicit and
   # to be consistent with `tpu_embedding_configuration.proto`.
-- 
GitLab


From 7ae5dd13e3c0c92db1f866bdbe63590a431ff3f4 Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <penporn@google.com>
Date: Fri, 11 Jan 2019 13:54:29 -0800
Subject: [PATCH 0586/2345] Fix --config=mkl performance regression: Prevent
 grappler from generating a _FusedConv2D node that doesn't have a
 corresponding TF-MKL node. For example, MKL graph rewriter doesn't support
 _FusedConv2D with BatchNorm yet so any Conv2D that gets fused with BatchNorm
 will miss using MKL-DNN's convolutions.

PiperOrigin-RevId: 228939593
---
 tensorflow/core/grappler/optimizers/BUILD       | 2 +-
 tensorflow/core/grappler/optimizers/remapper.cc | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 7e29cee86a..a67d6ac5ea 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -683,7 +683,7 @@ tf_cc_test(
     ],
 )
 
-cc_library(
+tf_kernel_library(
     name = "remapper",
     srcs = ["remapper.cc"],
     hdrs = [
diff --git a/tensorflow/core/grappler/optimizers/remapper.cc b/tensorflow/core/grappler/optimizers/remapper.cc
index 0869e3b49b..193772fcda 100644
--- a/tensorflow/core/grappler/optimizers/remapper.cc
+++ b/tensorflow/core/grappler/optimizers/remapper.cc
@@ -817,6 +817,9 @@ Status Remapper::Optimize(Cluster* /*cluster*/, const GrapplerItem& item,
       continue;
     }
 
+// TODO(penporn):
+// Remove this once TF-MKL supports _FusedConv2D with these operations.
+#ifndef INTEL_MKL
     // Remap Conv2D+Squeeze+BiasAdd into the _FusedConv2D+Squeeze.
     if (FindConv2DWithSqueezeAndBias(ctx, &node,
                                      &conv2d_with_squeeze_and_bias)) {
@@ -839,6 +842,7 @@ Status Remapper::Optimize(Cluster* /*cluster*/, const GrapplerItem& item,
                          &invalidated_nodes);
       continue;
     }
+#endif  // !INTEL_MKL
 
     // Infer properties lazily in case they are not needed.
     if (!ctx.inferred_graph_properties && IsFusedBatchNormCandidate(node)) {
-- 
GitLab


From 03f9fc0b5e65b5979720bdbd4c466c319afd0597 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 14:08:45 -0800
Subject: [PATCH 0587/2345] Add an explicit dependency on :pooling_ops to
 tensorflow/core:all_kernels_impl.

PiperOrigin-RevId: 228942375
---
 tensorflow/core/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 20f5815423..ee9ceeae89 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1409,6 +1409,7 @@ cc_library(
         "//tensorflow/core/kernels:parameterized_truncated_normal_op",
         "//tensorflow/core/kernels:parsing",
         "//tensorflow/core/kernels:partitioned_function_ops",
+        "//tensorflow/core/kernels:pooling_ops",
         "//tensorflow/core/kernels:ragged_ops",
         "//tensorflow/core/kernels:random_ops",
         "//tensorflow/core/kernels:random_poisson_op",
-- 
GitLab


From d745e4f0a96dd782f8a2e3ca0f7fd846901d9764 Mon Sep 17 00:00:00 2001
From: Katherine Wu <kathywu@google.com>
Date: Fri, 11 Jan 2019 14:22:50 -0800
Subject: [PATCH 0588/2345] Add recall and precision at K to metrics.

This change adds the options to set top_k and class_id to the Recall and Precision metric classes. The top_k is computed for dense labels, which differs from tf.metrics.recall_at_k, which expects sparse labels.

PiperOrigin-RevId: 228944786
---
 tensorflow/python/keras/metrics.py            | 152 +++++++++++----
 .../keras/metrics_confusion_matrix_test.py    | 178 +++++++++++++++++-
 .../python/keras/utils/metrics_utils.py       |  51 ++++-
 .../tensorflow.keras.metrics.-precision.pbtxt |   2 +-
 .../v1/tensorflow.keras.metrics.-recall.pbtxt |   2 +-
 .../tensorflow.keras.metrics.-precision.pbtxt |   2 +-
 .../v2/tensorflow.keras.metrics.-recall.pbtxt |   2 +-
 7 files changed, 340 insertions(+), 49 deletions(-)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index a13d8747d6..c04487f175 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -52,6 +52,7 @@ from tensorflow.python.keras.utils.generic_utils import to_list
 from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import confusion_matrix
+from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
@@ -792,8 +793,11 @@ class _ConfusionMatrixConditionCount(Metric):
       Update op.
     """
     return metrics_utils.update_confusion_matrix_variables(
-        {self._confusion_matrix_cond: self.accumulator}, y_true, y_pred,
-        self.thresholds, sample_weight)
+        {self._confusion_matrix_cond: self.accumulator},
+        y_true,
+        y_pred,
+        thresholds=self.thresholds,
+        sample_weight=sample_weight)
 
   def result(self):
     if len(self.thresholds) == 1:
@@ -1029,6 +1033,15 @@ class Precision(Metric):
   If `sample_weight` is `None`, weights default to 1.
   Use `sample_weight` of 0 to mask values.
 
+  If `top_k` is set, we'll calculate precision as how often on average a class
+  among the top-k classes with the highest predicted values of a batch entry is
+  correct and can be found in the label for that entry.
+
+  If `class_id` is specified, we calculate precision by considering only the
+  entries in the batch for which `class_id` is above the threshold and/or in the
+  top-k highest predictions, and computing the fraction of them for which
+  `class_id` is indeed a correct label.
+
   Usage:
 
   ```python
@@ -1045,22 +1058,37 @@ class Precision(Metric):
   ```
   """
 
-  def __init__(self, thresholds=None, name=None, dtype=None):
+  def __init__(self,
+               thresholds=None,
+               top_k=None,
+               class_id=None,
+               name=None,
+               dtype=None):
     """Creates a `Precision` instance.
 
     Args:
-      thresholds: (Optional) Defaults to 0.5. A float value or a python
-        list/tuple of float threshold values in [0, 1]. A threshold is compared
-        with prediction values to determine the truth value of predictions
-        (i.e., above the threshold is `true`, below is `false`). One metric
-        value is generated for each threshold value.
+      thresholds: (Optional) A float value or a python list/tuple of float
+        threshold values in [0, 1]. A threshold is compared with prediction
+        values to determine the truth value of predictions (i.e., above the
+        threshold is `true`, below is `false`). One metric value is generated
+        for each threshold value. If neither thresholds nor top_k are set, the
+        default is to calculate precision with `thresholds=0.5`.
+      top_k: (Optional) Unset by default. An int value specifying the top-k
+        predictions to consider when calculating precision.
+      class_id: (Optional) Integer class ID for which we want binary metrics.
+        This must be in the half-open interval `[0, num_classes)`, where
+        `num_classes` is the last dimension of predictions.
       name: (Optional) string name of the metric instance.
       dtype: (Optional) data type of the metric result.
     """
     super(Precision, self).__init__(name=name, dtype=dtype)
     self.init_thresholds = thresholds
+    self.top_k = top_k
+    self.class_id = class_id
+
+    default_threshold = 0.5 if top_k is None else metrics_utils.NEG_INF
     self.thresholds = metrics_utils.parse_init_thresholds(
-        thresholds, default_threshold=0.5)
+        thresholds, default_threshold=default_threshold)
     self.tp = self.add_weight(
         'true_positives',
         shape=(len(self.thresholds),),
@@ -1074,8 +1102,9 @@ class Precision(Metric):
     """Accumulates true positive and false positive statistics.
 
     Args:
-      y_true: The ground truth values.
-      y_pred: The predicted values.
+      y_true: The ground truth values, with the same dimensions as `y_pred`.
+        Will be cast to `bool`.
+      y_pred: The predicted values. Each element must be in the range `[0, 1]`.
       sample_weight: Optional weighting of each example. Defaults to 1. Can be a
         `Tensor` whose rank is either 0, or the same rank as `y_true`, and must
         be broadcastable to `y_true`.
@@ -1083,10 +1112,17 @@ class Precision(Metric):
     Returns:
       Update op.
     """
-    return metrics_utils.update_confusion_matrix_variables({
-        metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.tp,
-        metrics_utils.ConfusionMatrix.FALSE_POSITIVES: self.fp
-    }, y_true, y_pred, self.thresholds, sample_weight)
+    return metrics_utils.update_confusion_matrix_variables(
+        {
+            metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.tp,
+            metrics_utils.ConfusionMatrix.FALSE_POSITIVES: self.fp
+        },
+        y_true,
+        y_pred,
+        thresholds=self.thresholds,
+        top_k=self.top_k,
+        class_id=self.class_id,
+        sample_weight=sample_weight)
 
   def result(self):
     result = math_ops.div_no_nan(self.tp, self.tp + self.fp)
@@ -1098,7 +1134,11 @@ class Precision(Metric):
       K.set_value(v, np.zeros((num_thresholds,)))
 
   def get_config(self):
-    config = {'thresholds': self.init_thresholds}
+    config = {
+        'thresholds': self.init_thresholds,
+        'top_k': self.top_k,
+        'class_id': self.class_id
+    }
     base_config = super(Precision, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
 
@@ -1119,6 +1159,14 @@ class Recall(Metric):
   If `sample_weight` is `None`, weights default to 1.
   Use `sample_weight` of 0 to mask values.
 
+  If `top_k` is set, recall will be computed as how often on average a class
+  among the labels of a batch entry is in the top-k predictions.
+
+  If `class_id` is specified, we calculate recall by considering only the
+  entries in the batch for which `class_id` is in the label, and computing the
+  fraction of them for which `class_id` is above the threshold and/or in the
+  top-k predictions.
+
   Usage:
 
   ```python
@@ -1135,22 +1183,37 @@ class Recall(Metric):
   ```
   """
 
-  def __init__(self, thresholds=None, name=None, dtype=None):
+  def __init__(self,
+               thresholds=None,
+               top_k=None,
+               class_id=None,
+               name=None,
+               dtype=None):
     """Creates a `Recall` instance.
 
     Args:
-      thresholds: (Optional) Defaults to 0.5. A float value or a python
-        list/tuple of float threshold values in [0, 1]. A threshold is compared
-        with prediction values to determine the truth value of predictions
-        (i.e., above the threshold is `true`, below is `false`). One metric
-        value is generated for each threshold value.
+      thresholds: (Optional) A float value or a python list/tuple of float
+        threshold values in [0, 1]. A threshold is compared with prediction
+        values to determine the truth value of predictions (i.e., above the
+        threshold is `true`, below is `false`). One metric value is generated
+        for each threshold value. If neither thresholds nor top_k are set, the
+        default is to calculate recall with `thresholds=0.5`.
+      top_k: (Optional) Unset by default. An int value specifying the top-k
+        predictions to consider when calculating recall.
+      class_id: (Optional) Integer class ID for which we want binary metrics.
+        This must be in the half-open interval `[0, num_classes)`, where
+        `num_classes` is the last dimension of predictions.
       name: (Optional) string name of the metric instance.
       dtype: (Optional) data type of the metric result.
     """
     super(Recall, self).__init__(name=name, dtype=dtype)
     self.init_thresholds = thresholds
+    self.top_k = top_k
+    self.class_id = class_id
+
+    default_threshold = 0.5 if top_k is None else metrics_utils.NEG_INF
     self.thresholds = metrics_utils.parse_init_thresholds(
-        thresholds, default_threshold=0.5)
+        thresholds, default_threshold=default_threshold)
     self.tp = self.add_weight(
         'true_positives',
         shape=(len(self.thresholds),),
@@ -1164,8 +1227,9 @@ class Recall(Metric):
     """Accumulates true positive and false negative statistics.
 
     Args:
-      y_true: The ground truth values.
-      y_pred: The predicted values.
+      y_true: The ground truth values, with the same dimensions as `y_pred`.
+        Will be cast to `bool`.
+      y_pred: The predicted values. Each element must be in the range `[0, 1]`.
       sample_weight: Optional weighting of each example. Defaults to 1. Can be a
         `Tensor` whose rank is either 0, or the same rank as `y_true`, and must
         be broadcastable to `y_true`.
@@ -1173,10 +1237,17 @@ class Recall(Metric):
     Returns:
       Update op.
     """
-    return metrics_utils.update_confusion_matrix_variables({
-        metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.tp,
-        metrics_utils.ConfusionMatrix.FALSE_NEGATIVES: self.fn
-    }, y_true, y_pred, self.thresholds, sample_weight)
+    return metrics_utils.update_confusion_matrix_variables(
+        {
+            metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.tp,
+            metrics_utils.ConfusionMatrix.FALSE_NEGATIVES: self.fn
+        },
+        y_true,
+        y_pred,
+        thresholds=self.thresholds,
+        top_k=self.top_k,
+        class_id=self.class_id,
+        sample_weight=sample_weight)
 
   def result(self):
     result = math_ops.div_no_nan(self.tp, self.tp + self.fn)
@@ -1188,7 +1259,11 @@ class Recall(Metric):
       K.set_value(v, np.zeros((num_thresholds,)))
 
   def get_config(self):
-    config = {'thresholds': self.init_thresholds}
+    config = {
+        'thresholds': self.init_thresholds,
+        'top_k': self.top_k,
+        'class_id': self.class_id
+    }
     base_config = super(Recall, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
 
@@ -1244,12 +1319,17 @@ class SensitivitySpecificityBase(Metric):
     Returns:
       Update op.
     """
-    return metrics_utils.update_confusion_matrix_variables({
-        metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.tp,
-        metrics_utils.ConfusionMatrix.TRUE_NEGATIVES: self.tn,
-        metrics_utils.ConfusionMatrix.FALSE_POSITIVES: self.fp,
-        metrics_utils.ConfusionMatrix.FALSE_NEGATIVES: self.fn,
-    }, y_true, y_pred, self.thresholds, sample_weight)
+    return metrics_utils.update_confusion_matrix_variables(
+        {
+            metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.tp,
+            metrics_utils.ConfusionMatrix.TRUE_NEGATIVES: self.tn,
+            metrics_utils.ConfusionMatrix.FALSE_POSITIVES: self.fp,
+            metrics_utils.ConfusionMatrix.FALSE_NEGATIVES: self.fn,
+        },
+        y_true,
+        y_pred,
+        thresholds=self.thresholds,
+        sample_weight=sample_weight)
 
   def reset_states(self):
     num_thresholds = len(self.thresholds)
diff --git a/tensorflow/python/keras/metrics_confusion_matrix_test.py b/tensorflow/python/keras/metrics_confusion_matrix_test.py
index 37d2880088..6db034b721 100644
--- a/tensorflow/python/keras/metrics_confusion_matrix_test.py
+++ b/tensorflow/python/keras/metrics_confusion_matrix_test.py
@@ -318,18 +318,23 @@ class TruePositivesTest(test.TestCase):
 class PrecisionTest(test.TestCase):
 
   def test_config(self):
-    p_obj = metrics.Precision(name='my_precision', thresholds=[0.4, 0.9])
+    p_obj = metrics.Precision(
+        name='my_precision', thresholds=[0.4, 0.9], top_k=15, class_id=12)
     self.assertEqual(p_obj.name, 'my_precision')
     self.assertEqual(len(p_obj.variables), 2)
     self.assertEqual([v.name for v in p_obj.variables],
                      ['true_positives:0', 'false_positives:0'])
     self.assertEqual(p_obj.thresholds, [0.4, 0.9])
+    self.assertEqual(p_obj.top_k, 15)
+    self.assertEqual(p_obj.class_id, 12)
 
     # Check save and restore config
     p_obj2 = metrics.Precision.from_config(p_obj.get_config())
     self.assertEqual(p_obj2.name, 'my_precision')
     self.assertEqual(len(p_obj2.variables), 2)
     self.assertEqual(p_obj2.thresholds, [0.4, 0.9])
+    self.assertEqual(p_obj2.top_k, 15)
+    self.assertEqual(p_obj2.class_id, 12)
 
   def test_value_is_idempotent(self):
     p_obj = metrics.Precision(thresholds=[0.3, 0.72])
@@ -431,23 +436,110 @@ class PrecisionTest(test.TestCase):
     self.assertArrayNear([expected_precision, 0], self.evaluate(p_obj.result()),
                          1e-3)
 
+  def test_unweighted_top_k(self):
+    p_obj = metrics.Precision(top_k=3)
+    y_pred = constant_op.constant([0.2, 0.1, 0.5, 0, 0.2], shape=(1, 5))
+    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+    result = p_obj(y_true, y_pred)
+    self.assertAlmostEqual(1. / 3, self.evaluate(result))
+
+  def test_weighted_top_k(self):
+    p_obj = metrics.Precision(top_k=3)
+    y_pred1 = constant_op.constant([0.2, 0.1, 0.4, 0, 0.2], shape=(1, 5))
+    y_true1 = constant_op.constant([0, 1, 1, 0, 1], shape=(1, 5))
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+    self.evaluate(
+        p_obj(
+            y_true1,
+            y_pred1,
+            sample_weight=constant_op.constant([[1, 4, 2, 3, 5]])))
+
+    y_pred2 = constant_op.constant([0.2, 0.6, 0.4, 0.2, 0.2], shape=(1, 5))
+    y_true2 = constant_op.constant([1, 0, 1, 1, 1], shape=(1, 5))
+    result = p_obj(y_true2, y_pred2, sample_weight=constant_op.constant(3))
+
+    tp = (2 + 5) + (3 + 3)
+    predicted_positives = (1 + 2 + 5) + (3 + 3 + 3)
+    expected_precision = tp / predicted_positives
+    self.assertAlmostEqual(expected_precision, self.evaluate(result))
+
+  def test_unweighted_class_id(self):
+    p_obj = metrics.Precision(class_id=2)
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+
+    y_pred = constant_op.constant([0.2, 0.1, 0.6, 0, 0.2], shape=(1, 5))
+    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
+    result = p_obj(y_true, y_pred)
+    self.assertAlmostEqual(1, self.evaluate(result))
+    self.assertAlmostEqual(1, self.evaluate(p_obj.tp))
+    self.assertAlmostEqual(0, self.evaluate(p_obj.fp))
+
+    y_pred = constant_op.constant([0.2, 0.1, 0, 0, 0.2], shape=(1, 5))
+    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
+    result = p_obj(y_true, y_pred)
+    self.assertAlmostEqual(1, self.evaluate(result))
+    self.assertAlmostEqual(1, self.evaluate(p_obj.tp))
+    self.assertAlmostEqual(0, self.evaluate(p_obj.fp))
+
+    y_pred = constant_op.constant([0.2, 0.1, 0.6, 0, 0.2], shape=(1, 5))
+    y_true = constant_op.constant([0, 1, 0, 0, 0], shape=(1, 5))
+    result = p_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.5, self.evaluate(result))
+    self.assertAlmostEqual(1, self.evaluate(p_obj.tp))
+    self.assertAlmostEqual(1, self.evaluate(p_obj.fp))
+
+  def test_unweighted_top_k_and_class_id(self):
+    p_obj = metrics.Precision(class_id=2, top_k=2)
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+
+    y_pred = constant_op.constant([0.2, 0.6, 0.3, 0, 0.2], shape=(1, 5))
+    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
+    result = p_obj(y_true, y_pred)
+    self.assertAlmostEqual(1, self.evaluate(result))
+    self.assertAlmostEqual(1, self.evaluate(p_obj.tp))
+    self.assertAlmostEqual(0, self.evaluate(p_obj.fp))
+
+    y_pred = constant_op.constant([1, 1, 0.9, 1, 1], shape=(1, 5))
+    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
+    result = p_obj(y_true, y_pred)
+    self.assertAlmostEqual(1, self.evaluate(result))
+    self.assertAlmostEqual(1, self.evaluate(p_obj.tp))
+    self.assertAlmostEqual(0, self.evaluate(p_obj.fp))
+
+  def test_unweighted_top_k_and_threshold(self):
+    p_obj = metrics.Precision(thresholds=.7, top_k=2)
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+
+    y_pred = constant_op.constant([0.2, 0.8, 0.6, 0, 0.2], shape=(1, 5))
+    y_true = constant_op.constant([0, 1, 1, 0, 1], shape=(1, 5))
+    result = p_obj(y_true, y_pred)
+    self.assertAlmostEqual(1, self.evaluate(result))
+    self.assertAlmostEqual(1, self.evaluate(p_obj.tp))
+    self.assertAlmostEqual(0, self.evaluate(p_obj.fp))
+
 
 @test_util.run_all_in_graph_and_eager_modes
 class RecallTest(test.TestCase):
 
   def test_config(self):
-    r_obj = metrics.Recall(name='my_recall', thresholds=[0.4, 0.9])
+    r_obj = metrics.Recall(
+        name='my_recall', thresholds=[0.4, 0.9], top_k=15, class_id=12)
     self.assertEqual(r_obj.name, 'my_recall')
     self.assertEqual(len(r_obj.variables), 2)
     self.assertEqual([v.name for v in r_obj.variables],
                      ['true_positives:0', 'false_negatives:0'])
     self.assertEqual(r_obj.thresholds, [0.4, 0.9])
+    self.assertEqual(r_obj.top_k, 15)
+    self.assertEqual(r_obj.class_id, 12)
 
     # Check save and restore config
     r_obj2 = metrics.Recall.from_config(r_obj.get_config())
     self.assertEqual(r_obj2.name, 'my_recall')
     self.assertEqual(len(r_obj2.variables), 2)
     self.assertEqual(r_obj2.thresholds, [0.4, 0.9])
+    self.assertEqual(r_obj2.top_k, 15)
+    self.assertEqual(r_obj2.class_id, 12)
 
   def test_value_is_idempotent(self):
     r_obj = metrics.Recall(thresholds=[0.3, 0.72])
@@ -548,6 +640,88 @@ class RecallTest(test.TestCase):
     self.assertArrayNear([expected_recall, 0], self.evaluate(r_obj.result()),
                          1e-3)
 
+  def test_unweighted_top_k(self):
+    r_obj = metrics.Recall(top_k=3)
+    y_pred = constant_op.constant([0.2, 0.1, 0.5, 0, 0.2], shape=(1, 5))
+    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+    result = r_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.5, self.evaluate(result))
+
+  def test_weighted_top_k(self):
+    r_obj = metrics.Recall(top_k=3)
+    y_pred1 = constant_op.constant([0.2, 0.1, 0.4, 0, 0.2], shape=(1, 5))
+    y_true1 = constant_op.constant([0, 1, 1, 0, 1], shape=(1, 5))
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+    self.evaluate(
+        r_obj(
+            y_true1,
+            y_pred1,
+            sample_weight=constant_op.constant([[1, 4, 2, 3, 5]])))
+
+    y_pred2 = constant_op.constant([0.2, 0.6, 0.4, 0.2, 0.2], shape=(1, 5))
+    y_true2 = constant_op.constant([1, 0, 1, 1, 1], shape=(1, 5))
+    result = r_obj(y_true2, y_pred2, sample_weight=constant_op.constant(3))
+
+    tp = (2 + 5) + (3 + 3)
+    positives = (4 + 2 + 5) + (3 + 3 + 3 + 3)
+    expected_recall = tp / positives
+    self.assertAlmostEqual(expected_recall, self.evaluate(result))
+
+  def test_unweighted_class_id(self):
+    r_obj = metrics.Recall(class_id=2)
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+
+    y_pred = constant_op.constant([0.2, 0.1, 0.6, 0, 0.2], shape=(1, 5))
+    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
+    result = r_obj(y_true, y_pred)
+    self.assertAlmostEqual(1, self.evaluate(result))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.tp))
+    self.assertAlmostEqual(0, self.evaluate(r_obj.fn))
+
+    y_pred = constant_op.constant([0.2, 0.1, 0, 0, 0.2], shape=(1, 5))
+    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
+    result = r_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.5, self.evaluate(result))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.tp))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.fn))
+
+    y_pred = constant_op.constant([0.2, 0.1, 0.6, 0, 0.2], shape=(1, 5))
+    y_true = constant_op.constant([0, 1, 0, 0, 0], shape=(1, 5))
+    result = r_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.5, self.evaluate(result))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.tp))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.fn))
+
+  def test_unweighted_top_k_and_class_id(self):
+    r_obj = metrics.Recall(class_id=2, top_k=2)
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+
+    y_pred = constant_op.constant([0.2, 0.6, 0.3, 0, 0.2], shape=(1, 5))
+    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
+    result = r_obj(y_true, y_pred)
+    self.assertAlmostEqual(1, self.evaluate(result))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.tp))
+    self.assertAlmostEqual(0, self.evaluate(r_obj.fn))
+
+    y_pred = constant_op.constant([1, 1, 0.9, 1, 1], shape=(1, 5))
+    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
+    result = r_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.5, self.evaluate(result))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.tp))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.fn))
+
+  def test_unweighted_top_k_and_threshold(self):
+    r_obj = metrics.Recall(thresholds=.7, top_k=2)
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+
+    y_pred = constant_op.constant([0.2, 0.8, 0.6, 0, 0.2], shape=(1, 5))
+    y_true = constant_op.constant([1, 1, 1, 0, 1], shape=(1, 5))
+    result = r_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.25, self.evaluate(result))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.tp))
+    self.assertAlmostEqual(3, self.evaluate(r_obj.fn))
+
 
 @test_util.run_all_in_graph_and_eager_modes
 class SensitivityAtSpecificityTest(test.TestCase, parameterized.TestCase):
diff --git a/tensorflow/python/keras/utils/metrics_utils.py b/tensorflow/python/keras/utils/metrics_utils.py
index 6125b95b6e..7db128e8ab 100644
--- a/tensorflow/python/keras/utils/metrics_utils.py
+++ b/tensorflow/python/keras/utils/metrics_utils.py
@@ -33,11 +33,15 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import weights_broadcast_ops
 from tensorflow.python.util import tf_decorator
 
 
+NEG_INF = -1e10
+
+
 def update_state_wrapper(update_state_fn):
   """Decorator to wrap metric `update_state()` with `add_update()`.
 
@@ -120,16 +124,18 @@ def weakmethod(method):
 
 
 def assert_thresholds_range(thresholds):
-  invalid_thresholds = [t for t in thresholds if t is None or t < 0 or t > 1]
-  if invalid_thresholds:
-    raise ValueError(
-        'Threshold values must be in [0, 1]. Invalid values: {}'.format(
-            invalid_thresholds))
+  if thresholds is not None:
+    invalid_thresholds = [t for t in thresholds if t is None or t < 0 or t > 1]
+    if invalid_thresholds:
+      raise ValueError(
+          'Threshold values must be in [0, 1]. Invalid values: {}'.format(
+              invalid_thresholds))
 
 
 def parse_init_thresholds(thresholds, default_threshold=0.5):
+  if thresholds is not None:
+    assert_thresholds_range(to_list(thresholds))
   thresholds = to_list(default_threshold if thresholds is None else thresholds)
-  assert_thresholds_range(thresholds)
   return thresholds
 
 
@@ -144,6 +150,8 @@ def update_confusion_matrix_variables(variables_to_update,
                                       y_true,
                                       y_pred,
                                       thresholds,
+                                      top_k=None,
+                                      class_id=None,
                                       sample_weight=None):
   """Returns op to update the given confusion matrix variables.
 
@@ -170,7 +178,11 @@ def update_confusion_matrix_variables(variables_to_update,
     y_pred: A floating point `Tensor` of arbitrary shape and whose values are in
       the range `[0, 1]`.
     thresholds: A float value or a python list or tuple of float thresholds in
-      `[0, 1]`.
+      `[0, 1]`, or NEG_INF (used when top_k is set).
+    top_k: Optional int, indicates that the positive labels should be limited to
+      the top k predictions.
+    class_id: Optional int, limits the prediction and labels to the class
+      specified by this argument.
     sample_weight: Optional `Tensor` whose rank is either 0, or the same rank as
       `y_true`, and must be broadcastable to `y_true` (i.e., all dimensions must
       be either `1`, or the same as the corresponding `y_true` dimension).
@@ -219,6 +231,12 @@ def update_confusion_matrix_variables(variables_to_update,
         math_ops.cast(y_pred, dtype=dtypes.float32),
         math_ops.cast(y_true, dtype=dtypes.bool), sample_weight)
 
+  if top_k is not None:
+    y_pred = _filter_top_k(y_pred, top_k)
+  if class_id is not None:
+    y_true = y_true[..., class_id]
+    y_pred = y_pred[..., class_id]
+
   thresholds = to_list(thresholds)
   num_thresholds = len(thresholds)
   num_predictions = array_ops.size(y_pred)
@@ -282,3 +300,22 @@ def update_confusion_matrix_variables(variables_to_update,
           weighted_assign_add(label, pred, weights_tiled,
                               variables_to_update[matrix_cond]))
   return control_flow_ops.group(update_ops)
+
+
+def _filter_top_k(x, k):
+  """Filters top-k values in the last dim of x and set the rest to NEG_INF.
+
+  Used for computing top-k prediction values in dense labels (which has the same
+  shape as predictions) for recall and precision top-k metrics.
+
+  Args:
+    x: tensor with any dimensions.
+    k: the number of values to keep.
+
+  Returns:
+    tensor with same shape and dtype as x.
+  """
+  _, top_k_idx = nn_ops.top_k(x, k, sorted=False)
+  top_k_mask = math_ops.reduce_sum(
+      array_ops.one_hot(top_k_idx, x.shape[-1], axis=-1), axis=-2)
+  return x * top_k_mask + NEG_INF * (1 - top_k_mask)
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt
index e81ecfe3f6..794da447d7 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt
@@ -87,7 +87,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'thresholds\', \'top_k\', \'class_id\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_loss"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt
index f8470b94d7..cae610b9ea 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt
@@ -87,7 +87,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'thresholds\', \'top_k\', \'class_id\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_loss"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt
index e81ecfe3f6..794da447d7 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt
@@ -87,7 +87,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'thresholds\', \'top_k\', \'class_id\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_loss"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt
index f8470b94d7..cae610b9ea 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt
@@ -87,7 +87,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'thresholds\', \'top_k\', \'class_id\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_loss"
-- 
GitLab


From eacf1972e653069683e739ba401bb55a8cd58ce5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 14:24:48 -0800
Subject: [PATCH 0589/2345] Start device tracer when running eager profiler.

PiperOrigin-RevId: 228945120
---
 tensorflow/c/eager/BUILD                      | 18 +++--
 tensorflow/c/eager/c_api_experimental_test.cc | 27 +++++++-
 tensorflow/core/common_runtime/eager/BUILD    |  1 +
 .../core/common_runtime/eager/profiler.cc     | 68 ++++++++++++++-----
 .../core/common_runtime/eager/profiler.h      | 10 +--
 5 files changed, 98 insertions(+), 26 deletions(-)

diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
index 3ea1724d1e..710850b5f2 100644
--- a/tensorflow/c/eager/BUILD
+++ b/tensorflow/c/eager/BUILD
@@ -8,6 +8,15 @@ load(
     "tf_cuda_library",
     "tfe_xla_copts",
 )
+load(
+    "//tensorflow/core:platform/default/build_config.bzl",
+    "tf_additional_device_tracer_test_flags",
+    "tf_kernel_tests_linkstatic",
+)
+load(
+    "//tensorflow/core:platform/default/build_config_root.bzl",
+    "tf_cuda_tests_tags",
+)
 
 tf_cuda_library(
     name = "c_api",
@@ -211,11 +220,10 @@ tf_cuda_cc_test(
     srcs = [
         "c_api_experimental_test.cc",
     ],
-    extra_copts = tfe_xla_copts(),
-    tags = [
-        "guitar",
-        "multi_gpu",
-    ],
+    args =
+        ["--heap_check=local"] + tf_additional_device_tracer_test_flags(),
+    linkstatic = tf_kernel_tests_linkstatic(),
+    tags = tf_cuda_tests_tags() + ["nomac"],
     deps = [
         ":c_api_experimental",
         ":c_api_test_util",
diff --git a/tensorflow/c/eager/c_api_experimental_test.cc b/tensorflow/c/eager/c_api_experimental_test.cc
index 9b4fca9d45..c3d29bd2e0 100644
--- a/tensorflow/c/eager/c_api_experimental_test.cc
+++ b/tensorflow/c/eager/c_api_experimental_test.cc
@@ -16,9 +16,9 @@ limitations under the License.
 #include "tensorflow/c/eager/c_api_experimental.h"
 
 #include <string.h>
-#include "absl/strings/match.h"
 #include "tensorflow/c/eager/c_api_test_util.h"
 #include "tensorflow/cc/profiler/profiler.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/protobuf.h"
 #include "tensorflow/core/platform/test.h"
@@ -27,8 +27,15 @@ limitations under the License.
 
 using tensorflow::string;
 
+namespace tensorflow {
 namespace {
 
+static bool HasSubstr(absl::string_view base, absl::string_view substr) {
+  bool ok = str_util::StrContains(base, substr);
+  EXPECT_TRUE(ok) << base << ", expected substring " << substr;
+  return ok;
+}
+
 void ExecuteWithProfiling(bool async) {
   TF_Status* status = TF_NewStatus();
   TFE_ContextOptions* opts = TFE_NewContextOptions();
@@ -42,6 +49,16 @@ void ExecuteWithProfiling(bool async) {
   TFE_Op* matmul = MatMulOp(ctx, m, m);
   TFE_TensorHandle* retvals[1] = {nullptr};
   int num_retvals = 1;
+
+  // Run op on GPU if it is present.
+  string gpu_device_name;
+  if (GetDeviceName(ctx, &gpu_device_name, "GPU")) {
+    TFE_OpSetDevice(matmul, gpu_device_name.c_str(), status);
+    ASSERT_TRUE(TF_GetCode(status) == TF_OK) << TF_Message(status);
+    const char* device_name = TFE_OpGetDevice(matmul, status);
+    ASSERT_TRUE(strstr(device_name, "GPU:0") != nullptr);
+  }
+
   TFE_Execute(matmul, &retvals[0], &num_retvals, status);
   EXPECT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
   TFE_DeleteOp(matmul);
@@ -57,6 +74,13 @@ void ExecuteWithProfiling(bool async) {
   EXPECT_TRUE(profile_proto.ParseFromString(
       {reinterpret_cast<const char*>(profiler_result->data),
        profiler_result->length}));
+  string profile_proto_str = profile_proto.DebugString();
+  if (!gpu_device_name.empty()) {
+    EXPECT_TRUE(HasSubstr(profile_proto_str, "gpu:0"));
+    // device name with "stream:all" is collected by Device Tracer.
+    EXPECT_TRUE(HasSubstr(profile_proto_str, "stream:all"));
+  }
+  EXPECT_TRUE(HasSubstr(profile_proto_str, "cpu:0"));
   TF_DeleteBuffer(profiler_result);
 
   TF_Tensor* t = TFE_TensorHandleResolve(retvals[0], status);
@@ -77,3 +101,4 @@ TEST(CAPI, ExecuteWithTracing) { ExecuteWithProfiling(false); }
 TEST(CAPI, ExecuteWithTracingAsync) { ExecuteWithProfiling(true); }
 
 }  // namespace
+}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD
index cabbddb77d..2697aea783 100644
--- a/tensorflow/core/common_runtime/eager/BUILD
+++ b/tensorflow/core/common_runtime/eager/BUILD
@@ -112,6 +112,7 @@ tf_cuda_library(
             "//tensorflow/core:lib_internal",
             "//tensorflow/core:protos_all_cc",
             "//tensorflow/core:session_options",
+            "//tensorflow/core:device_tracer",
         ],
     }),
 )
diff --git a/tensorflow/core/common_runtime/eager/profiler.cc b/tensorflow/core/common_runtime/eager/profiler.cc
index d670d6f344..46449d0771 100644
--- a/tensorflow/core/common_runtime/eager/profiler.cc
+++ b/tensorflow/core/common_runtime/eager/profiler.cc
@@ -15,7 +15,11 @@ limitations under the License.
 
 #include "tensorflow/core/common_runtime/eager/profiler.h"
 #include "tensorflow/cc/profiler/profiler.h"
+#include "tensorflow/core/common_runtime/eager/context.h"
+#include "tensorflow/core/common_runtime/step_stats_collector.h"
 #include "tensorflow/core/framework/graph.pb.h"
+#include "tensorflow/core/platform/device_tracer.h"
+#include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/protobuf/config.pb.h"
 
@@ -37,18 +41,28 @@ Status EagerProfiler::Status() {
 }
 
 Status EagerProfiler::SerializeToString(string* content) {
-  {
-    mutex_lock l(mutex_);
-    if (!status_.ok()) return status_;
+  mutex_lock l(mutex_);
+  if (!status_.ok()) return status_;
+  Stop();
+
+  // Get profiling data from device tracer
+  if (device_tracer_ != nullptr) {
+    std::unique_ptr<StepStatsCollector> step_stats_collector(
+        new StepStatsCollector(run_metadata_.mutable_step_stats()));
+    tensorflow::Status s = device_tracer_->Collect(step_stats_collector.get());
+    if (!s.ok()) {
+      device_tracer_.reset(nullptr);
+      LOG(WARNING) << "Failed to collect data from device tracer. "
+                   << s.error_message();
+    }
+    step_stats_collector->Finalize();
   }
-  RunMetadata metadata;
-  GetMergetRunMetadata(&metadata);
 
   // TODO(fishx): update tfprof to use a lighter representation instead of
   // GraphDef.
   GraphDef graph;
   std::unique_ptr<tfprof::Profiler> tfprof(new tfprof::Profiler(graph));
-  tfprof->AddStep(0, metadata);
+  tfprof->AddStep(0, run_metadata_);
   return tfprof->SerializeToString(content);
 }
 
@@ -57,21 +71,43 @@ EagerProfiler::EagerProfiler(EagerContext* const context) : context_(context) {
 
   status_ = context_->RegisterRunMetadataListener(this);
   if (!status_.ok()) {
-    LOG(INFO) << "Eager Profiler failed to start. Another profiler is running.";
+    context_ = nullptr;
+    LOG(WARNING)
+        << "Eager Profiler failed to start. Another profiler is running.";
     return;
   }
-}
 
-EagerProfiler::~EagerProfiler() {
-  context_->ClearRunMetadataListener();
-  LOG(INFO) << "Eager Profiler ended with status:" << status_;
+  // TODO(fishx): Allow user disable device tracer.
+  device_tracer_ = CreateDeviceTracer();
+  if (!device_tracer_) {
+    LOG(WARNING) << "Continue profiling without device tracer. "
+                 << "Failed to create device tracer.";
+    return;
+  }
+  class Status s = device_tracer_->Start();
+  if (!s.ok()) {
+    device_tracer_.reset(nullptr);
+    LOG(WARNING) << "Continue profiling without device tracer. "
+                 << s.error_message();
+  }
 }
 
-void EagerProfiler::GetMergetRunMetadata(RunMetadata* metadata) {
-  mutex_lock ml(*context_->MetadataMu());
-  mutex_lock l(mutex_);
-  *metadata = run_metadata_;
-  metadata->MergeFrom(*context_->RunMetadataProto());
+EagerProfiler::~EagerProfiler() { Stop(); }
+
+void EagerProfiler::Stop() {
+  if (context_ != nullptr) {
+    context_->ClearRunMetadataListener();
+    run_metadata_.MergeFrom(*context_->RunMetadataProto());
+    context_ = nullptr;
+    if (device_tracer_ != nullptr) {
+      tensorflow::Status s = device_tracer_->Stop();
+      if (!s.ok()) {
+        device_tracer_.reset(nullptr);
+        LOG(WARNING) << "Failed to stop device tracer. " << s.error_message();
+      }
+    }
+    LOG(INFO) << "Eager Profiler ended with status:" << status_;
+  }
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/eager/profiler.h b/tensorflow/core/common_runtime/eager/profiler.h
index cadbfdb498..62df869e94 100644
--- a/tensorflow/core/common_runtime/eager/profiler.h
+++ b/tensorflow/core/common_runtime/eager/profiler.h
@@ -17,14 +17,15 @@ limitations under the License.
 
 #include "tensorflow/core/common_runtime/eager/context.h"
 #include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/platform/device_tracer.h"
 #include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/protobuf/config.pb.h"
 
 namespace tensorflow {
 
 // A profiler which will start profiling when creating the object and will stop
-// when the object is destroyed. It will profile all operations run under the
-// given EagerContext.
+// when either the object is destroyed or SerializedToString is called. It will
+// profile all operations run under the given EagerContext.
 // Multiple instances of it can be created, but at most one of them will profile
 // for each EagerContext. Status() will return OK only for the instance that is
 // profiling.
@@ -51,11 +52,12 @@ class EagerProfiler : RunMetadataListener {
   EagerProfiler(const EagerProfiler&) = delete;
   EagerProfiler& operator=(const EagerProfiler&) = delete;
 
-  void GetMergetRunMetadata(RunMetadata* metadata) LOCKS_EXCLUDED(mutex_);
+  void Stop() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
 
   RunMetadata run_metadata_ GUARDED_BY(mutex_);
   tensorflow::Status status_ GUARDED_BY(mutex_);
-  EagerContext* const context_;
+  std::unique_ptr<DeviceTracer> device_tracer_ GUARDED_BY(mutex_);
+  EagerContext* context_ GUARDED_BY(mutex_);
   mutex mutex_;
 };
 
-- 
GitLab


From e51fa30400b3a469b03111b25344bc47bdff96bd Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Fri, 11 Jan 2019 14:38:07 -0800
Subject: [PATCH 0590/2345] Add a mutex for each (ICudaEngine,
 IExecutionContext) pair since some of their operations are not thread safe.

This fixed one of the issues mentioned in
#23853

PiperOrigin-RevId: 228947504
---
 tensorflow/contrib/tensorrt/BUILD             |  1 +
 .../contrib/tensorrt/kernels/trt_engine_op.cc | 71 +++++++++----------
 .../contrib/tensorrt/kernels/trt_engine_op.h  | 27 ++++---
 3 files changed, 55 insertions(+), 44 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 003a7e0b0a..73bb1fe0c5 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -97,6 +97,7 @@ cc_library(
         ":trt_plugins",
         ":trt_resources",
         ":utils",
+        "@com_google_absl//absl/memory",
         "//tensorflow/core:gpu_headers_lib",
         "//tensorflow/core:lib_proto_parsing",
         "//tensorflow/core:stream_executor_headers_lib",
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
index 1f6409341c..ae5e64e823 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
@@ -16,6 +16,7 @@ limitations under the License.
 
 #include <algorithm>
 
+#include "absl/memory/memory.h"
 #include "tensorflow/contrib/tensorrt/convert/convert_nodes.h"
 #include "tensorflow/contrib/tensorrt/convert/utils.h"
 #include "tensorflow/contrib/tensorrt/log/trt_logger.h"
@@ -284,16 +285,14 @@ void TRTEngineOp::ComputeAsync(OpKernelContext* ctx,
   }
 
   const int num_batch = ctx->input(0).shape().dim_size(0);
-  auto& engine_ctx_pair = GetEngine(smallest_engine, ctx);
-  auto& trt_engine_ptr = engine_ctx_pair.first;
-  if (!trt_engine_ptr) {
+  EngineContext* engine_context = GetEngine(smallest_engine, ctx);
+  if (!engine_context->cuda_engine) {
     LOG(WARNING) << "Engine retrieval for batch size " << num_batch
                  << " failed. Running native segment for " << name();
     ExecuteNativeSegment(ctx, helper);
     return;
   }
-  const bool retry = ExecuteTrtEngine(ctx, num_batch, trt_engine_ptr.get(),
-                                      engine_ctx_pair.second.get());
+  const bool retry = ExecuteTrtEngine(ctx, num_batch, engine_context);
   if (retry) {
     LOG(WARNING) << "Failed to execute engine, "
                  << "retrying with native segment for " << name();
@@ -302,18 +301,17 @@ void TRTEngineOp::ComputeAsync(OpKernelContext* ctx,
   }
 }
 
-bool TRTEngineOp::ExecuteTrtEngine(
-    OpKernelContext* ctx, const int num_batch,
-    nvinfer1::ICudaEngine* trt_engine_ptr,
-    nvinfer1::IExecutionContext* trt_execution_context_ptr) {
+bool TRTEngineOp::ExecuteTrtEngine(OpKernelContext* ctx, const int num_batch,
+                                   EngineContext* engine_context) {
+  auto& cuda_engine = engine_context->cuda_engine;
+
   VLOG(1) << "Executing TRT engine: " << name();
   const bool kRetry = true;
   const int num_binding = ctx->num_inputs() + ctx->num_outputs();
   std::vector<void*> buffers(num_binding);
   for (int i = 0; i < ctx->num_inputs(); i++) {
     const string input_name = StrCat(kInputPHName, i);
-    const int binding_index =
-        trt_engine_ptr->getBindingIndex(input_name.c_str());
+    const int binding_index = cuda_engine->getBindingIndex(input_name.c_str());
     if (binding_index == -1) {
       LOG(ERROR) << "Input node not found, at " << input_name;
       return kRetry;
@@ -326,7 +324,7 @@ bool TRTEngineOp::ExecuteTrtEngine(
                  << " vs " << input_shape.dim_size(0);
       return kRetry;
     }
-    auto dtype = trt_engine_ptr->getBindingDataType(binding_index);
+    auto dtype = cuda_engine->getBindingDataType(binding_index);
     switch (dtype) {
       case nvinfer1::DataType::kFLOAT:
         buffers[binding_index] = (void*)(input_tensor.flat<float>().data());
@@ -349,13 +347,12 @@ bool TRTEngineOp::ExecuteTrtEngine(
   for (int i = 0; i < ctx->num_outputs(); i++) {
     // Create an output tensor
     const string output_name = StrCat(kOutputPHName, i);
-    const int binding_index =
-        trt_engine_ptr->getBindingIndex(output_name.c_str());
+    const int binding_index = cuda_engine->getBindingIndex(output_name.c_str());
     Tensor* output_tensor = nullptr;
 
     TensorShape output_shape;
     if (binding_index != -1) {
-      auto dims = trt_engine_ptr->getBindingDimensions(binding_index);
+      auto dims = cuda_engine->getBindingDimensions(binding_index);
       std::vector<int> trt_shape(dims.nbDims + 1);
       trt_shape[0] = num_batch;
       for (int j = 0; j < dims.nbDims; j++) trt_shape[j + 1] = dims.d[j];
@@ -377,7 +374,7 @@ bool TRTEngineOp::ExecuteTrtEngine(
       // TODO(aaroey): ideally we should retry, fix this.
       return !kRetry;
     }
-    auto dtype = trt_engine_ptr->getBindingDataType(binding_index);
+    auto dtype = cuda_engine->getBindingDataType(binding_index);
     switch (dtype) {
       case nvinfer1::DataType::kFLOAT:
         buffers[binding_index] =
@@ -405,9 +402,12 @@ bool TRTEngineOp::ExecuteTrtEngine(
                                                 ->implementation()
                                                 ->GpuStreamMemberHack()));
 
+  // nvinfer1::IExecutionContext::enqueue is not thread safe and we need a mutex
+  // for it.
+  tensorflow::mutex_lock lock(engine_context->mu);
   // TODO(jie): trt enqueue does not return error
-  auto ret = trt_execution_context_ptr->enqueue(num_batch, &buffers[0], *stream,
-                                                nullptr);
+  auto ret = engine_context->execution_context->enqueue(num_batch, &buffers[0],
+                                                        *stream, nullptr);
   if (!ret) {
     LOG(WARNING) << "Failed to enqueue batch for TRT engine: " << name();
     return kRetry;
@@ -421,8 +421,8 @@ TRTEngineOp::~TRTEngineOp() {
   // We need to manually destroy the engine and execution context before
   // the allocator is destructed.
   for (auto& eng : engine_map_) {
-    eng.second.first.reset();
-    eng.second.second.reset();
+    eng.second->cuda_engine.reset();
+    eng.second->execution_context.reset();
   }
   allocator_.reset();
 }
@@ -440,11 +440,9 @@ nvinfer1::IGpuAllocator* TRTEngineOp::GetAllocator(OpKernelContext* ctx) {
   return allocator_.get();
 }
 
-TRTEngineOp::EngineCtxPair& TRTEngineOp::GetEngine(int batch_size,
+TRTEngineOp::EngineContext* TRTEngineOp::GetEngine(int batch_size,
                                                    OpKernelContext* ctx) {
-  static EngineCtxPair null_pair = {
-      TrtUniquePtrType<nvinfer1::ICudaEngine>(nullptr),
-      TrtUniquePtrType<nvinfer1::IExecutionContext>(nullptr)};
+  static EngineContext empty_context;
   // TODO(sami): This method needs to be re-written to use resource manager and
   // with LRU mechanism option.
   tensorflow::mutex_lock lock(engine_mutex_);
@@ -452,14 +450,14 @@ TRTEngineOp::EngineCtxPair& TRTEngineOp::GetEngine(int batch_size,
   if (static_engine_) {
     if (engine_map_.size()) {
       if (engine_map_.begin()->first >= batch_size) {
-        return engine_map_.begin()->second;
+        return engine_map_.begin()->second.get();
       }
-      return null_pair;
+      return &empty_context;
     }
     TrtUniquePtrType<IRuntime> infer(nvinfer1::createInferRuntime(logger));
     auto allocator = GetAllocator(ctx);
     if (allocator == nullptr) {
-      return null_pair;
+      return &empty_context;
     }
     infer->setGpuAllocator(allocator);
     TrtUniquePtrType<nvinfer1::ICudaEngine> static_engine(
@@ -468,16 +466,16 @@ TRTEngineOp::EngineCtxPair& TRTEngineOp::GetEngine(int batch_size,
                                      PluginFactoryTensorRT::GetInstance()));
     auto raw_static_engine = static_engine.get();
     const auto max_batch_size = raw_static_engine->getMaxBatchSize();
-    engine_map_[max_batch_size] = {
+    engine_map_[max_batch_size] = absl::make_unique<EngineContext>(
         std::move(static_engine),
         TrtUniquePtrType<nvinfer1::IExecutionContext>(
-            raw_static_engine->createExecutionContext())};
+            raw_static_engine->createExecutionContext()));
     // Runtime is safe to delete after engine creation
     serialized_segment_.clear();
     if (max_batch_size < batch_size) {
-      return null_pair;
+      return &empty_context;
     }
-    return engine_map_.at(max_batch_size);
+    return engine_map_.at(max_batch_size).get();
   }  // static_engine_
 
   // Handle the dynamic engine case.
@@ -487,7 +485,7 @@ TRTEngineOp::EngineCtxPair& TRTEngineOp::GetEngine(int batch_size,
     nvinfer1::IGpuAllocator* allocator = nullptr;
     allocator = GetAllocator(ctx);
     if (allocator == nullptr) {
-      return null_pair;
+      return &empty_context;
     }
     std::vector<tensorflow::PartialTensorShape> shapes;
     for (int i = 0; i < ctx->num_inputs(); ++i) {
@@ -508,18 +506,19 @@ TRTEngineOp::EngineCtxPair& TRTEngineOp::GetEngine(int batch_size,
         // This means it fail to build the engine even when the network is built
         // successfully, probably due to internal issues. In this case we don't
         // retry in the future.
-        engine_map_[batch_size] = {nullptr, nullptr};
+        engine_map_[batch_size] = absl::make_unique<EngineContext>();
       }
       LOG(WARNING) << "Engine creation for batch size " << batch_size
                    << " failed " << status;
-      return null_pair;
+      return &empty_context;
     }
     VLOG(1) << "Conversion is done";
     TrtUniquePtrType<nvinfer1::IExecutionContext> exec_context(
         engine->createExecutionContext());
-    engine_map_[batch_size] = {std::move(engine), std::move(exec_context)};
+    engine_map_[batch_size] = absl::make_unique<EngineContext>(
+        std::move(engine), std::move(exec_context));
   }
-  return engine_map_.at(batch_size);
+  return engine_map_.at(batch_size).get();
 }
 
 tensorflow::Status TRTEngineOp::AllocateCalibrationResources(
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
index ceb7483642..38ea224063 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
@@ -27,6 +27,7 @@ limitations under the License.
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/platform/thread_annotations.h"
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
@@ -51,6 +52,21 @@ class TRTEngineOp : public AsyncOpKernel {
   ~TRTEngineOp();
 
  private:
+  // TODO(samikama): context should go to a resource manager!
+  struct EngineContext {
+    EngineContext() {}  // Creates an empty context.
+    EngineContext(
+        TrtUniquePtrType<nvinfer1::ICudaEngine>&& input_cuda_engine,
+        TrtUniquePtrType<nvinfer1::IExecutionContext>&& input_execution_context)
+        : cuda_engine(std::move(input_cuda_engine)),
+          execution_context(std::move(input_execution_context)) {}
+
+    mutex mu;
+    TrtUniquePtrType<nvinfer1::ICudaEngine> cuda_engine;
+    TrtUniquePtrType<nvinfer1::IExecutionContext> execution_context
+        GUARDED_BY(mu);
+  };
+
   // Execute calibration
   void ExecuteCalibration(OpKernelContext* ctx, AsyncHelper* helper);
 
@@ -63,18 +79,13 @@ class TRTEngineOp : public AsyncOpKernel {
   // Execute the tensorrt engine. Returns whether we need to retry by running
   // the native segment.
   bool ExecuteTrtEngine(OpKernelContext* ctx, const int num_batch,
-                        nvinfer1::ICudaEngine* trt_engine_ptr,
-                        nvinfer1::IExecutionContext* trt_execution_context_ptr);
+                        EngineContext* engine_context);
 
   // Allocate necessary resources for calibration
   Status AllocateCalibrationResources(OpKernelContext* ctx,
                                       TRTCalibrationResource** cr);
 
-  // TODO(samikama): context should go to a resource manager!
-  typedef std::pair<TrtUniquePtrType<nvinfer1::ICudaEngine>,
-                    TrtUniquePtrType<nvinfer1::IExecutionContext>>
-      EngineCtxPair;
-  EngineCtxPair& GetEngine(int batch_size, OpKernelContext* ctx);
+  EngineContext* GetEngine(int batch_size, OpKernelContext* ctx);
 
   // Return engine batch closest to input batch.
   int GetEngineBatch(OpKernelContext* ctx);
@@ -82,7 +93,7 @@ class TRTEngineOp : public AsyncOpKernel {
   nvinfer1::IGpuAllocator* GetAllocator(OpKernelContext* ctx);
 
   // map to keep engines and their execution context for given batch size.
-  std::unordered_map<int, EngineCtxPair> engine_map_;
+  std::unordered_map<int, std::unique_ptr<EngineContext>> engine_map_;
   std::vector<string> input_nodes_;
   std::vector<string> output_nodes_;
 
-- 
GitLab


From 67c18c4f8be9544dac4f65e932327872228b8cec Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Fri, 11 Jan 2019 14:50:39 -0800
Subject: [PATCH 0591/2345] Increase coverage of Keras integration tests across
 model types and runtime modes.

PiperOrigin-RevId: 228949803
---
 tensorflow/python/keras/BUILD               |   3 +-
 tensorflow/python/keras/integration_test.py | 446 +++++++-------------
 2 files changed, 164 insertions(+), 285 deletions(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index d36aaaf632..d1ec5f0ac3 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -333,9 +333,8 @@ tf_py_test(
         "@absl_py//absl/testing:parameterized",
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
-        "//tensorflow/python:layers",
-        "//tensorflow/python:nn",
     ],
+    shard_count = 12,
     tags = ["notsan"],
 )
 
diff --git a/tensorflow/python/keras/integration_test.py b/tensorflow/python/keras/integration_test.py
index fbe3508f07..febee44cd4 100644
--- a/tensorflow/python/keras/integration_test.py
+++ b/tensorflow/python/keras/integration_test.py
@@ -22,299 +22,179 @@ import numpy as np
 
 from tensorflow.python import keras
 from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import test_util
+from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import testing_utils
-from tensorflow.python.layers import core as tf_core_layers
-from tensorflow.python.ops import nn
 from tensorflow.python.ops import rnn_cell
 from tensorflow.python.platform import test
 
 
-class KerasIntegrationTest(test.TestCase):
+@keras_parameterized.run_with_all_model_types
+@keras_parameterized.run_all_keras_modes
+class VectorClassificationIntegrationTest(keras_parameterized.TestCase):
+
+  def test_vector_classification(self):
+    np.random.seed(1337)
+    (x_train, y_train), _ = testing_utils.get_test_data(
+        train_samples=150,
+        test_samples=0,
+        input_shape=(10,),
+        num_classes=2)
+    y_train = keras.utils.to_categorical(y_train)
+
+    model = testing_utils.get_model_from_layers(
+        [keras.layers.Dense(16, activation='relu'),
+         keras.layers.Dropout(0.1),
+         keras.layers.Dense(y_train.shape[-1], activation='softmax')],
+        input_shape=x_train.shape[1:])
+    model.compile(loss='categorical_crossentropy',
+                  optimizer=keras.optimizer_v2.adam.Adam(0.01),
+                  metrics=['accuracy'],
+                  run_eagerly=testing_utils.should_run_eagerly())
+    history = model.fit(x_train, y_train, epochs=10, batch_size=16,
+                        validation_data=(x_train, y_train),
+                        verbose=2)
+    self.assertGreater(history.history['val_acc'][-1], 0.7)
+    _, val_acc = model.evaluate(x_train, y_train)
+    self.assertAlmostEqual(history.history['val_acc'][-1], val_acc)
+    predictions = model.predict(x_train)
+    self.assertEqual(predictions.shape, (x_train.shape[0], 2))
 
-  def test_version(self):
-    self.assertTrue(keras.__version__.endswith('-tf'))
-
-  def test_vector_classification_sequential(self):
-    with self.cached_session():
-      np.random.seed(1337)
-      (x_train, y_train), _ = testing_utils.get_test_data(
-          train_samples=100,
-          test_samples=0,
-          input_shape=(10,),
-          num_classes=2)
-      y_train = keras.utils.to_categorical(y_train)
-
-      model = keras.models.Sequential([
-          keras.layers.Dense(16,
-                             activation='relu',
-                             input_shape=x_train.shape[1:]),
-          keras.layers.Dropout(0.1),
-          keras.layers.Dense(y_train.shape[-1], activation='softmax')
-      ])
-      model.compile(loss='categorical_crossentropy',
-                    optimizer=keras.optimizers.Adam(lr=0.1),
-                    metrics=['accuracy'])
-      history = model.fit(x_train, y_train, epochs=10, batch_size=16,
-                          validation_data=(x_train, y_train),
-                          verbose=2)
-      self.assertGreater(history.history['val_acc'][-1], 0.7)
-
-  @test_util.run_deprecated_v1
-  def test_vector_classification_functional(self):
-    with self.cached_session():
-      np.random.seed(1337)
-      (x_train, y_train), _ = testing_utils.get_test_data(
-          train_samples=100,
-          test_samples=0,
-          input_shape=(20,),
-          num_classes=2)
-      y_train = keras.utils.to_categorical(y_train)
-
-      inputs = keras.layers.Input(shape=x_train.shape[1:])
-      x = keras.layers.Dense(16, activation='relu')(inputs)
-      x = keras.layers.Dropout(0.1)(x)
-      outputs = keras.layers.Dense(y_train.shape[-1], activation='softmax')(x)
-
-      model = keras.models.Model(inputs, outputs)
-      model.compile(loss='categorical_crossentropy',
-                    optimizer=keras.optimizers.Adam(lr=0.1),
-                    metrics=['accuracy'])
-      history = model.fit(x_train, y_train, epochs=10, batch_size=16,
-                          validation_data=(x_train, y_train),
-                          verbose=2)
-      self.assertGreater(history.history['val_acc'][-1], 0.7)
-
-  @test_util.run_deprecated_v1
-  def test_temporal_classification_sequential(self):
-    with self.cached_session():
-      np.random.seed(1337)
-      (x_train, y_train), _ = testing_utils.get_test_data(
-          train_samples=100,
-          test_samples=0,
-          input_shape=(4, 10),
-          num_classes=2)
-      y_train = keras.utils.to_categorical(y_train)
-
-      model = keras.models.Sequential()
-      model.add(keras.layers.LSTM(5, return_sequences=True,
-                                  input_shape=x_train.shape[1:]))
-      model.add(keras.layers.GRU(y_train.shape[-1], activation='softmax'))
-      model.compile(loss='categorical_crossentropy',
-                    optimizer=keras.optimizers.Adam(lr=0.1),
-                    metrics=['accuracy'])
-      history = model.fit(x_train, y_train, epochs=15, batch_size=16,
-                          validation_data=(x_train, y_train),
-                          verbose=2)
-      self.assertGreater(history.history['val_acc'][-1], 0.7)
-
-  @test_util.run_deprecated_v1
-  def test_temporal_classification_sequential_tf_rnn(self):
-    with self.cached_session():
-      np.random.seed(1337)
-      (x_train, y_train), _ = testing_utils.get_test_data(
-          train_samples=100,
-          test_samples=0,
-          input_shape=(4, 10),
-          num_classes=2)
-      y_train = keras.utils.to_categorical(y_train)
-
-      model = keras.models.Sequential()
-      model.add(keras.layers.RNN(rnn_cell.LSTMCell(5), return_sequences=True,
-                                 input_shape=x_train.shape[1:]))
-      model.add(keras.layers.RNN(rnn_cell.GRUCell(y_train.shape[-1],
-                                                  activation='softmax',
-                                                  dtype=dtypes.float32)))
-      model.compile(loss='categorical_crossentropy',
-                    optimizer=keras.optimizers.Adam(lr=0.1),
-                    metrics=['accuracy'])
-      history = model.fit(x_train, y_train, epochs=15, batch_size=16,
-                          validation_data=(x_train, y_train),
-                          verbose=2)
-      self.assertGreater(history.history['val_acc'][-1], 0.7)
-
-  def test_image_classification_sequential(self):
-    with self.cached_session():
-      np.random.seed(1337)
-      (x_train, y_train), _ = testing_utils.get_test_data(
-          train_samples=100,
-          test_samples=0,
-          input_shape=(12, 12, 3),
-          num_classes=2)
-      y_train = keras.utils.to_categorical(y_train)
-
-      model = keras.models.Sequential()
-      model.add(keras.layers.Conv2D(
-          4, 3,
-          padding='same',
-          activation='relu',
-          input_shape=x_train.shape[1:]))
-      model.add(keras.layers.Conv2D(
-          8, 3,
-          padding='same',
-          activation='relu'))
-      model.add(keras.layers.Conv2D(
-          16, 3,
-          padding='same',
-          activation='relu'))
-      model.add(keras.layers.Flatten())
-      model.add(keras.layers.Dense(y_train.shape[-1], activation='softmax'))
-      model.compile(loss='categorical_crossentropy',
-                    optimizer=keras.optimizers.SGD(lr=0.01, momentum=0.8),
-                    metrics=['accuracy'])
-      history = model.fit(x_train, y_train, epochs=10, batch_size=16,
-                          validation_data=(x_train, y_train),
-                          verbose=2)
-      self.assertGreater(history.history['val_acc'][-1], 0.7)
-
-  def test_video_classification_functional(self):
-    with self.cached_session():
-      np.random.seed(1337)
-      (x_train, y_train), _ = testing_utils.get_test_data(
-          train_samples=100,
-          test_samples=0,
-          input_shape=(4, 8, 8, 3),
-          num_classes=3)
-      y_train = keras.utils.to_categorical(y_train)
-
-      inputs = keras.layers.Input(shape=x_train.shape[1:])
-      x = keras.layers.TimeDistributed(
-          keras.layers.Conv2D(4, 3, activation='relu'))(inputs)
-      x = keras.layers.BatchNormalization()(x)
-      x = keras.layers.TimeDistributed(keras.layers.GlobalMaxPooling2D())(x)
-      x = keras.layers.Conv1D(8, 3, activation='relu')(x)
-      x = keras.layers.Flatten()(x)
-      outputs = keras.layers.Dense(y_train.shape[-1], activation='softmax')(x)
-
-      model = keras.models.Model(inputs, outputs)
-      model.compile(loss='categorical_crossentropy',
-                    optimizer=keras.optimizers.SGD(lr=0.01, momentum=0.8),
-                    metrics=['accuracy'])
-      history = model.fit(x_train, y_train, epochs=10, batch_size=16,
-                          validation_data=(x_train, y_train),
-                          verbose=2)
-      self.assertGreater(history.history['val_acc'][-1], 0.7)
-
-  def test_vector_classification_shared_sequential(self):
+  def test_vector_classification_shared_model(self):
     # Test that Sequential models that feature internal updates
     # and internal losses can be shared.
-    with self.cached_session():
-      np.random.seed(1337)
-      (x_train, y_train), _ = testing_utils.get_test_data(
-          train_samples=100,
-          test_samples=0,
-          input_shape=(10,),
-          num_classes=2)
-      y_train = keras.utils.to_categorical(y_train)
-
-      base_model = keras.models.Sequential([
-          keras.layers.Dense(16,
-                             activation='relu',
-                             kernel_regularizer=keras.regularizers.l2(1e-5),
-                             bias_regularizer=keras.regularizers.l2(1e-5),
-                             input_shape=x_train.shape[1:]),
-          keras.layers.BatchNormalization(),
-      ])
-      x = keras.layers.Input(x_train.shape[1:])
-      y = base_model(x)
-      y = keras.layers.Dense(y_train.shape[-1], activation='softmax')(y)
-      model = keras.models.Model(x, y)
-      model.compile(loss='categorical_crossentropy',
-                    optimizer=keras.optimizers.Adam(lr=0.1),
-                    metrics=['accuracy'])
+    np.random.seed(1337)
+    (x_train, y_train), _ = testing_utils.get_test_data(
+        train_samples=150,
+        test_samples=0,
+        input_shape=(10,),
+        num_classes=2)
+    y_train = keras.utils.to_categorical(y_train)
+
+    base_model = testing_utils.get_model_from_layers(
+        [keras.layers.Dense(16,
+                            activation='relu',
+                            kernel_regularizer=keras.regularizers.l2(1e-5),
+                            bias_regularizer=keras.regularizers.l2(1e-5)),
+         keras.layers.BatchNormalization()],
+        input_shape=x_train.shape[1:])
+    x = keras.layers.Input(x_train.shape[1:])
+    y = base_model(x)
+    y = keras.layers.Dense(y_train.shape[-1], activation='softmax')(y)
+    model = keras.models.Model(x, y)
+    model.compile(loss='categorical_crossentropy',
+                  optimizer=keras.optimizer_v2.adam.Adam(0.01),
+                  metrics=['accuracy'],
+                  run_eagerly=testing_utils.should_run_eagerly())
+    if not testing_utils.should_run_eagerly():
       self.assertEqual(len(model.losses), 2)
       self.assertEqual(len(model.updates), 2)
-      history = model.fit(x_train, y_train, epochs=10, batch_size=16,
-                          validation_data=(x_train, y_train),
-                          verbose=2)
-      self.assertGreater(history.history['val_acc'][-1], 0.7)
-
-  def test_vector_classification_shared_model(self):
-    # Test that functional models that feature internal updates
-    # and internal losses can be shared.
-    with self.cached_session():
-      np.random.seed(1337)
-      (x_train, y_train), _ = testing_utils.get_test_data(
-          train_samples=100,
-          test_samples=0,
-          input_shape=(10,),
-          num_classes=2)
-      y_train = keras.utils.to_categorical(y_train)
-
-      inputs = keras.layers.Input(x_train.shape[1:])
-      x = keras.layers.Dense(16,
-                             activation='relu',
-                             kernel_regularizer=keras.regularizers.l2(1e-5),
-                             bias_regularizer=keras.regularizers.l2(1e-5),
-                             input_shape=x_train.shape[1:])(inputs)
-      x = keras.layers.BatchNormalization()(x)
-      base_model = keras.models.Model(inputs, x)
-
-      x = keras.layers.Input(x_train.shape[1:])
-      y = base_model(x)
-      y = keras.layers.Dense(y_train.shape[-1], activation='softmax')(y)
-      model = keras.models.Model(x, y)
-      model.compile(loss='categorical_crossentropy',
-                    optimizer=keras.optimizers.Adam(lr=0.1),
-                    metrics=['accuracy'])
-      history = model.fit(x_train, y_train, epochs=10, batch_size=16,
-                          validation_data=(x_train, y_train),
-                          verbose=2)
-      self.assertGreater(history.history['val_acc'][-1], 0.7)
-
-  def test_embedding_with_clipnorm(self):
-    with self.cached_session():
-      model = keras.models.Sequential()
-      model.add(keras.layers.Embedding(input_dim=1, output_dim=1))
-      model.compile(optimizer=keras.optimizers.SGD(clipnorm=0.1), loss='mse')
-      model.fit(np.array([[0]]), np.array([[[0.5]]]), epochs=1)
-
-  def test_using_tf_layers_in_keras_sequential_model(self):
-    with self.cached_session():
-      np.random.seed(1337)
-      (x_train, y_train), _ = testing_utils.get_test_data(
-          train_samples=100,
-          test_samples=0,
-          input_shape=(10,),
-          num_classes=2)
-
-      model = keras.models.Sequential()
-      model.add(tf_core_layers.Dense(32, activation=nn.relu, input_shape=(10,)))
-      model.add(tf_core_layers.Dense(2, activation=nn.softmax))
-      model.summary()
-
-      y_train = keras.utils.to_categorical(y_train)
-      model.compile(loss='categorical_crossentropy',
-                    optimizer=keras.optimizers.Adam(lr=0.1),
-                    metrics=['accuracy'])
-      history = model.fit(x_train, y_train, epochs=10, batch_size=16,
-                          validation_data=(x_train, y_train),
-                          verbose=0)
-      self.assertGreater(history.history['val_acc'][-1], 0.7)
-
-  def test_using_tf_layers_in_keras_functional_model(self):
-    with self.cached_session():
-      np.random.seed(1337)
-      (x_train, y_train), _ = testing_utils.get_test_data(
-          train_samples=100,
-          test_samples=0,
-          input_shape=(10,),
-          num_classes=2)
-      y_train = keras.utils.to_categorical(y_train)
-
-      inputs = keras.Input(shape=(10,))
-      x = tf_core_layers.Dense(32, activation=nn.relu)(inputs)
-      outputs = tf_core_layers.Dense(2, activation=nn.softmax)(x)
-      model = keras.Model(inputs, outputs)
-      model.summary()
-
-      model.compile(loss='categorical_crossentropy',
-                    optimizer=keras.optimizers.Adam(lr=0.1),
-                    metrics=['accuracy'])
-      history = model.fit(x_train, y_train, epochs=10, batch_size=16,
-                          validation_data=(x_train, y_train),
-                          verbose=0)
-      self.assertGreater(history.history['val_acc'][-1], 0.7)
+    history = model.fit(x_train, y_train, epochs=10, batch_size=16,
+                        validation_data=(x_train, y_train),
+                        verbose=2)
+    self.assertGreater(history.history['val_acc'][-1], 0.7)
+    _, val_acc = model.evaluate(x_train, y_train)
+    self.assertAlmostEqual(history.history['val_acc'][-1], val_acc)
+    predictions = model.predict(x_train)
+    self.assertEqual(predictions.shape, (x_train.shape[0], 2))
+
+
+# See b/122473407
+@keras_parameterized.run_all_keras_modes(always_skip_v1=True)
+class TimeseriesClassificationIntegrationTest(keras_parameterized.TestCase):
+
+  @keras_parameterized.run_with_all_model_types
+  def test_timeseries_classification(self):
+    np.random.seed(1337)
+    (x_train, y_train), _ = testing_utils.get_test_data(
+        train_samples=250,
+        test_samples=0,
+        input_shape=(4, 10),
+        num_classes=2)
+    y_train = keras.utils.to_categorical(y_train)
+
+    layers = [
+        keras.layers.LSTM(5, return_sequences=True),
+        keras.layers.GRU(y_train.shape[-1], activation='softmax')
+    ]
+    model = testing_utils.get_model_from_layers(
+        layers, input_shape=x_train.shape[1:])
+    model.compile(loss='categorical_crossentropy',
+                  optimizer=keras.optimizer_v2.adam.Adam(0.01),
+                  metrics=['accuracy'],
+                  run_eagerly=testing_utils.should_run_eagerly())
+    history = model.fit(x_train, y_train, epochs=15, batch_size=16,
+                        validation_data=(x_train, y_train),
+                        verbose=2)
+    self.assertGreater(history.history['val_acc'][-1], 0.7)
+    _, val_acc = model.evaluate(x_train, y_train)
+    self.assertAlmostEqual(history.history['val_acc'][-1], val_acc)
+    predictions = model.predict(x_train)
+    self.assertEqual(predictions.shape, (250, 2))
+
+  def test_timeseries_classification_sequential_tf_rnn(self):
+    np.random.seed(1337)
+    (x_train, y_train), _ = testing_utils.get_test_data(
+        train_samples=150,
+        test_samples=0,
+        input_shape=(4, 10),
+        num_classes=2)
+    y_train = keras.utils.to_categorical(y_train)
+
+    model = keras.models.Sequential()
+    model.add(keras.layers.RNN(rnn_cell.LSTMCell(5), return_sequences=True,
+                               input_shape=x_train.shape[1:]))
+    model.add(keras.layers.RNN(rnn_cell.GRUCell(y_train.shape[-1],
+                                                activation='softmax',
+                                                dtype=dtypes.float32)))
+    model.compile(loss='categorical_crossentropy',
+                  optimizer=keras.optimizer_v2.adam.Adam(0.01),
+                  metrics=['accuracy'],
+                  run_eagerly=testing_utils.should_run_eagerly())
+    history = model.fit(x_train, y_train, epochs=15, batch_size=16,
+                        validation_data=(x_train, y_train),
+                        verbose=2)
+    self.assertGreater(history.history['val_acc'][-1], 0.7)
+    _, val_acc = model.evaluate(x_train, y_train)
+    self.assertAlmostEqual(history.history['val_acc'][-1], val_acc)
+    predictions = model.predict(x_train)
+    self.assertEqual(predictions.shape, (150, 2))
+
+
+@keras_parameterized.run_with_all_model_types
+@keras_parameterized.run_all_keras_modes
+class ImageClassificationIntegrationTest(keras_parameterized.TestCase):
+
+  def test_image_classification(self):
+    np.random.seed(1337)
+    (x_train, y_train), _ = testing_utils.get_test_data(
+        train_samples=250,
+        test_samples=0,
+        input_shape=(10, 10, 3),
+        num_classes=2)
+    y_train = keras.utils.to_categorical(y_train)
+
+    layers = [
+        keras.layers.Conv2D(4, 3, padding='same', activation='relu'),
+        keras.layers.Conv2D(8, 3, padding='same'),
+        keras.layers.BatchNormalization(),
+        keras.layers.Conv2D(8, 3, padding='same'),
+        keras.layers.Flatten(),
+        keras.layers.Dense(y_train.shape[-1], activation='softmax')
+    ]
+    model = testing_utils.get_model_from_layers(
+        layers, input_shape=x_train.shape[1:])
+    model.compile(loss='categorical_crossentropy',
+                  optimizer=keras.optimizer_v2.adam.Adam(0.01),
+                  metrics=['accuracy'],
+                  run_eagerly=testing_utils.should_run_eagerly())
+    history = model.fit(x_train, y_train, epochs=10, batch_size=16,
+                        validation_data=(x_train, y_train),
+                        verbose=2)
+    self.assertGreater(history.history['val_acc'][-1], 0.7)
+    _, val_acc = model.evaluate(x_train, y_train)
+    self.assertAlmostEqual(history.history['val_acc'][-1], val_acc)
+    predictions = model.predict(x_train)
+    self.assertEqual(predictions.shape, (250, 2))
 
 
 if __name__ == '__main__':
-- 
GitLab


From 892ea4dcc74be16fbfb0e31cedb3a41b807bb6b1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 15:00:37 -0800
Subject: [PATCH 0592/2345] Add annotation to eager mode.

With these annotations, when user turns on tracing, it will display op name like "BiasAddGrad" instead of kernel name "void tensorflow::BiasGradNCHW_SharedAtomics<float>(float const*, float*, int, int, int, int)".

PiperOrigin-RevId: 228951394
---
 .../core/common_runtime/eager/kernel_and_device.cc   | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/common_runtime/eager/kernel_and_device.cc b/tensorflow/core/common_runtime/eager/kernel_and_device.cc
index 354f96c440..4096763b11 100644
--- a/tensorflow/core/common_runtime/eager/kernel_and_device.cc
+++ b/tensorflow/core/common_runtime/eager/kernel_and_device.cc
@@ -28,6 +28,7 @@ limitations under the License.
 #include "tensorflow/core/lib/gtl/stl_util.h"
 #include "tensorflow/core/platform/fingerprint.h"
 #include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/platform/tracing.h"
 #include "tensorflow/core/public/version.h"
 #include "tensorflow/core/util/tensor_slice_reader_cache.h"
 
@@ -129,7 +130,16 @@ Status KernelAndDevice::Run(ScopedStepContainer* step_container,
     device_->ComputeAsync(async, &context, [&done]() { done.Notify(); });
     done.WaitForNotification();
   } else {
-    device_->Compute(kernel_.get(), &context);
+    const string& op_name = kernel_->name();
+    // If tracing if off, the overheads of ScopedAnnotation and ScopedActivity
+    // are negligible.
+    if (device_->TraceUsingAnnotations()) {
+      tracing::ScopedAnnotation activity(op_name, kernel_->type_string());
+      device_->Compute(kernel_.get(), &context);
+    } else {
+      tracing::ScopedActivity activity(op_name, kernel_->type_string());
+      device_->Compute(kernel_.get(), &context);
+    }
   }
   if (!context.status().ok()) return context.status();
 
-- 
GitLab


From e25754186b415cb668632a938d7c24ed2d925bb8 Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Fri, 11 Jan 2019 15:07:17 -0800
Subject: [PATCH 0593/2345] [XLA] Move more XlaBuilder users over to the new
 DS/DUS API

Move more tests and bridge op implementations over to the new
span-of-scalars API for DynamicSlice and DynamicUpdateSlice.

Also fix a bug in DynamicIndexSplitter to handle the case of an
already-empty index span.

PiperOrigin-RevId: 228952769
---
 .../compiler/tf2xla/kernels/random_ops.cc     | 19 ++++++---
 .../tf2xla/kernels/tensor_array_ops.cc        | 23 ++++++-----
 .../tf2xla/kernels/tensor_list_ops.cc         | 13 +++---
 .../xla/service/dynamic_index_splitter.cc     | 22 +++++-----
 .../xla/service/hlo_cost_analysis_test.cc     |  5 ++-
 .../compiler/xla/tests/dot_operation_test.cc  | 40 +++++++++++--------
 6 files changed, 70 insertions(+), 52 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/random_ops.cc b/tensorflow/compiler/tf2xla/kernels/random_ops.cc
index 2d92056e4f..01b047f732 100644
--- a/tensorflow/compiler/tf2xla/kernels/random_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/random_ops.cc
@@ -160,17 +160,24 @@ class RandomShuffleOp : public XlaOpKernel {
         -> xla::StatusOr<std::vector<xla::XlaOp>> {
       auto swaps = loop_vars[0];
       auto indices = loop_vars[1];
-      i = xla::Reshape(i, {1});
+      // TODO(b/118437727): The absl::Span nonsense is only necessary because
+      // the deprecated overload creates ambiguity for the single-element span
+      // case. Remove it once the deprecated overload is gone.
       // temp = indices[i]
-      auto temp = xla::DynamicSlice(indices, i, {1});
+      auto temp =
+          xla::DynamicSlice(indices, absl::Span<const xla::XlaOp>({i}), {1});
       // swap_index = swaps[i]
-      auto swap_index = xla::DynamicSlice(swaps, i, {1});
+      auto swap_index = xla::Reshape(
+          xla::DynamicSlice(swaps, absl::Span<const xla::XlaOp>({i}), {1}), {});
       // swap_value = indices[swaps[i]]
-      auto swap_value = xla::DynamicSlice(indices, swap_index, {1});
+      auto swap_value = xla::DynamicSlice(
+          indices, absl::Span<const xla::XlaOp>({swap_index}), {1});
       // indices[i] = indices[swaps[i]]
-      indices = xla::DynamicUpdateSlice(indices, swap_value, i);
+      indices = xla::DynamicUpdateSlice(indices, swap_value,
+                                        absl::Span<const xla::XlaOp>({i}));
       // indices[swaps[i]] = temp
-      indices = xla::DynamicUpdateSlice(indices, temp, swap_index);
+      indices = xla::DynamicUpdateSlice(
+          indices, temp, absl::Span<const xla::XlaOp>({swap_index}));
       return std::vector<xla::XlaOp>{swaps, indices};
     };
     // for i in range(n):
diff --git a/tensorflow/compiler/tf2xla/kernels/tensor_array_ops.cc b/tensorflow/compiler/tf2xla/kernels/tensor_array_ops.cc
index 939d7e1951..22e64789a3 100644
--- a/tensorflow/compiler/tf2xla/kernels/tensor_array_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/tensor_array_ops.cc
@@ -123,7 +123,8 @@ Status GetTensorArrayShape(const XlaResource* resource,
 xla::XlaOp DynamicAddSlice(xla::XlaBuilder* builder, const xla::XlaOp& operand,
                            const xla::XlaOp& update,
                            absl::Span<const int64> update_dims,
-                           const xla::XlaOp& start_indices, DataType dtype) {
+                           absl::Span<const xla::XlaOp> start_indices,
+                           DataType dtype) {
   xla::XlaOp current = xla::DynamicSlice(operand, start_indices, update_dims);
   xla::XlaOp sum =
       dtype == DT_BOOL ? xla::Or(current, update) : xla::Add(current, update);
@@ -212,9 +213,9 @@ class TensorArrayWriteOp : public XlaOpKernel {
     xla::XlaOp flow = ctx->Input(3);
 
     // start_indices of the DynamicUpdateSlice are [index, 0, 0, ..., 0].
-    auto start_indices =
-        xla::Pad(xla::Reshape(index, {1}), xla::ConstantR0<int32>(b, 0),
-                 xla::MakeEdgePaddingConfig({{0, elem_shape.dims()}}));
+    std::vector<xla::XlaOp> start_indices(elem_shape.dims() + 1,
+                                          xla::ConstantR0<int32>(b, 0));
+    start_indices[0] = index;
 
     TensorShape slice_shape = elem_shape;
     slice_shape.InsertDim(0, 1LL);
@@ -263,9 +264,9 @@ class TensorArrayReadOp : public XlaOpKernel {
     xla::XlaOp index = ctx->Input(1);
 
     // start_indices of the DynamicSlice are [index, 0, 0, ..., 0].
-    auto start_indices =
-        xla::Pad(xla::Reshape(index, {1}), xla::ConstantR0<int32>(b, 0),
-                 xla::MakeEdgePaddingConfig({{0, ta_shape.dims() - 1}}));
+    std::vector<xla::XlaOp> start_indices(ta_shape.dims(),
+                                          xla::ConstantR0<int32>(b, 0));
+    start_indices[0] = index;
 
     auto slice_shape = ta_shape.dim_sizes();
     slice_shape[0] = 1LL;
@@ -419,10 +420,10 @@ class TensorArrayScatterOp : public XlaOpKernel {
         auto slice = xla::Slice(value, value_starts, value_ends, value_strides);
 
         // start_indices of the DynamicUpdateSlice are [index, 0, 0, ..., 0].
-        auto index = xla::Slice(indices, {i}, {i + 1}, {1});
-        auto start_indices =
-            xla::Pad(xla::Reshape(index, {1}), xla::ConstantR0<int32>(b, 0),
-                     xla::MakeEdgePaddingConfig({{0, elem_shape.dims()}}));
+        auto index = xla::Reshape(xla::Slice(indices, {i}, {i + 1}, {1}), {});
+        std::vector<xla::XlaOp> start_indices(elem_shape.dims() + 1,
+                                              xla::ConstantR0<int32>(b, 0));
+        start_indices[0] = index;
         ta = DynamicAddSlice(b, ta, slice, slice_dims, start_indices, dtype_);
       }
     }
diff --git a/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc b/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc
index 747f591a1b..ba3d7eb00d 100644
--- a/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc
@@ -155,9 +155,9 @@ class TensorListPushBackOp : public XlaOpKernel {
     xla::XlaOp value = ctx->Input(1);
 
     // start_indices of the DynamicUpdateSlice are [index, 0, 0, ..., 0].
-    auto start_indices =
-        xla::Pad(xla::Reshape(index, {1}), xla::ConstantR0<int32>(b, 0),
-                 xla::MakeEdgePaddingConfig({{0, elem_shape.dims()}}));
+    std::vector<xla::XlaOp> start_indices(elem_shape.dims() + 1,
+                                          xla::ConstantR0<int32>(b, 0));
+    start_indices[0] = index;
 
     TensorShape slice_shape = elem_shape;
     slice_shape.InsertDim(0, 1LL);
@@ -197,10 +197,9 @@ class TensorListPopBackOp : public XlaOpKernel {
     index = index - xla::ConstantR0<int32>(b, 1);
 
     // start_indices of the DynamicSlice are [index, 0, 0, ..., 0].
-    auto start_indices =
-        xla::Pad(xla::Reshape(index, {1}), xla::ConstantR0<int32>(b, 0),
-                 xla::MakeEdgePaddingConfig({{0, shape.dims() - 1}}));
-
+    std::vector<xla::XlaOp> start_indices(shape.dims(),
+                                          xla::ConstantR0<int32>(b, 0));
+    start_indices[0] = index;
     auto slice_shape = shape.dim_sizes();
     slice_shape[0] = 1LL;
 
diff --git a/tensorflow/compiler/xla/service/dynamic_index_splitter.cc b/tensorflow/compiler/xla/service/dynamic_index_splitter.cc
index ede4a5afdb..e34adfd2d2 100644
--- a/tensorflow/compiler/xla/service/dynamic_index_splitter.cc
+++ b/tensorflow/compiler/xla/service/dynamic_index_splitter.cc
@@ -45,17 +45,10 @@ StatusOr<bool> DynamicIndexSplitter::Run(HloModule* module) {
       }
       auto parent = dynamic_op->parent();
       bool is_update = dynamic_op->opcode() == HloOpcode::kDynamicUpdateSlice;
-      int64 index_operand_number = Cast<HloDynamicIndexInstruction>(dynamic_op)
-                                       ->first_index_operand_number();
-      auto index_operand = dynamic_op->mutable_operand(index_operand_number);
-      if (ShapeUtil::IsScalar(index_operand->shape())) {
-        // This DS/DUS already uses scalar indices.
-        continue;
-      }
-      TF_RET_CHECK(index_operand->shape().rank() == 1);
-      int64 num_indices = index_operand->shape().dimensions(0);
+      int64 num_indices = dynamic_op->operand(0)->shape().rank();
+
       if (num_indices == 0) {
-        // If the operand dimension is 0, directly replace R0 DS/DUS with the
+        // If the operand rank is 0, directly replace R0 DS/DUS with the
         // operand (for DS) or update (for DUS).
         if (is_update) {
           TF_CHECK_OK(parent->ReplaceInstruction(
@@ -67,6 +60,15 @@ StatusOr<bool> DynamicIndexSplitter::Run(HloModule* module) {
         changed = true;
         continue;
       }
+
+      int64 index_operand_number = Cast<HloDynamicIndexInstruction>(dynamic_op)
+                                       ->first_index_operand_number();
+      auto index_operand = dynamic_op->mutable_operand(index_operand_number);
+      if (ShapeUtil::IsScalar(index_operand->shape())) {
+        // This DS/DUS already uses scalar indices.
+        continue;
+      }
+      TF_RET_CHECK(index_operand->shape().rank() == 1);
       auto index_element_type = index_operand->shape().element_type();
       std::vector<HloInstruction*> index_array;
       for (int64 dim = 0; dim < num_indices; ++dim) {
diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis_test.cc b/tensorflow/compiler/xla/service/hlo_cost_analysis_test.cc
index ff32faf298..82dd57d3c6 100644
--- a/tensorflow/compiler/xla/service/hlo_cost_analysis_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_cost_analysis_test.cc
@@ -529,7 +529,8 @@ TEST_F(HloCostAnalysisTest, DynamicSlice) {
   // Test the analysis on a slice.
   XlaBuilder builder("dynamic-slice");
   auto x = Parameter(&builder, 0, ShapeUtil::MakeShape(F32, {2}), "x");
-  DynamicSlice(x, ConstantR1<int32>(&builder, {1}), {1});
+  DynamicSlice(x, absl::Span<const XlaOp>({ConstantR0<int32>(&builder, 1)}),
+               {1});
   auto hlo_module = BuildHloGraph(&builder);
 
   // Run HLO cost analysis.
@@ -545,7 +546,7 @@ TEST_F(HloCostAnalysisTest, DynamicUpdateSlice) {
   XlaBuilder builder("dynamic-update-slice");
   auto x = Parameter(&builder, 0, ShapeUtil::MakeShape(F32, {2}), "x");
   DynamicUpdateSlice(x, ConstantR1<float>(&builder, {1.0}),
-                     ConstantR1<int32>(&builder, {1}));
+                     absl::Span<const XlaOp>({ConstantR0<int32>(&builder, 1)}));
   auto hlo_module = BuildHloGraph(&builder);
 
   // Run HLO cost analysis.
diff --git a/tensorflow/compiler/xla/tests/dot_operation_test.cc b/tensorflow/compiler/xla/tests/dot_operation_test.cc
index 2e02968ac5..33d816292c 100644
--- a/tensorflow/compiler/xla/tests/dot_operation_test.cc
+++ b/tensorflow/compiler/xla/tests/dot_operation_test.cc
@@ -919,8 +919,9 @@ XLA_TEST_F(DotOperationTest, DotOfGatherOptimizationWithConstRHSClassicMM) {
   XlaBuilder builder(TestName());
   auto lhs_constant = ConstantR2FromArray2D(&builder, *constant_lhs_array);
   auto rhs_constant = ConstantR2FromArray2D(&builder, *constant_rhs_array);
-  auto start_constant = ConstantR1<int32>(&builder, {1, 0});
-  auto dynamic_slice = DynamicSlice(lhs_constant, start_constant, {1, 6});
+  auto one = ConstantR0<int32>(&builder, 1);
+  auto zero = ConstantR0<int32>(&builder, 0);
+  auto dynamic_slice = DynamicSlice(lhs_constant, {one, zero}, {1, 6});
 
   DotDimensionNumbers dot_dnums;
   dot_dnums.add_lhs_contracting_dimensions(1);
@@ -946,8 +947,9 @@ XLA_TEST_F(DotOperationTest, DotOfGatherOptimizationWithConstLHSClassicMM) {
   XlaBuilder builder(TestName());
   auto lhs_constant = ConstantR2FromArray2D(&builder, *constant_lhs_array);
   auto rhs_constant = ConstantR2FromArray2D(&builder, *constant_rhs_array);
-  auto start_constant = ConstantR1<int32>(&builder, {0, 1});
-  auto dynamic_slice = DynamicSlice(rhs_constant, start_constant, {6, 1});
+  auto zero = ConstantR0<int32>(&builder, 0);
+  auto one = ConstantR0<int32>(&builder, 1);
+  auto dynamic_slice = DynamicSlice(rhs_constant, {zero, one}, {6, 1});
 
   DotDimensionNumbers dot_dnums;
   dot_dnums.add_lhs_contracting_dimensions(1);
@@ -975,8 +977,9 @@ XLA_TEST_F(DotOperationTest,
   XlaBuilder builder(TestName());
   auto lhs_constant = ConstantR2FromArray2D(&builder, *constant_lhs_array);
   auto rhs_constant = ConstantR2FromArray2D(&builder, *constant_rhs_array);
-  auto start_constant = ConstantR1<int32>(&builder, {0, 1});
-  auto dynamic_slice = DynamicSlice(lhs_constant, start_constant, {6, 1});
+  auto zero = ConstantR0<int32>(&builder, 0);
+  auto one = ConstantR0<int32>(&builder, 1);
+  auto dynamic_slice = DynamicSlice(lhs_constant, {zero, one}, {6, 1});
 
   DotDimensionNumbers dot_dnums;
   dot_dnums.add_lhs_contracting_dimensions(0);
@@ -1002,8 +1005,9 @@ XLA_TEST_F(DotOperationTest, DotOfGatherOptimizationWithConstLHSReverseMM) {
   XlaBuilder builder(TestName());
   auto lhs_constant = ConstantR2FromArray2D(&builder, *constant_lhs_array);
   auto rhs_constant = ConstantR2FromArray2D(&builder, *constant_rhs_array);
-  auto start_constant = ConstantR1<int32>(&builder, {1, 0});
-  auto dynamic_slice = DynamicSlice(rhs_constant, start_constant, {1, 6});
+  auto zero = ConstantR0<int32>(&builder, 0);
+  auto one = ConstantR0<int32>(&builder, 1);
+  auto dynamic_slice = DynamicSlice(rhs_constant, {one, zero}, {1, 6});
 
   DotDimensionNumbers dot_dnums;
   dot_dnums.add_lhs_contracting_dimensions(0);
@@ -1034,8 +1038,9 @@ XLA_TEST_F(DotOperationTest, DotOfGatherOptimizationWithConstRHSRows) {
   XlaBuilder builder(TestName());
   auto lhs_constant = ConstantR2FromArray2D(&builder, *constant_lhs_array);
   auto rhs_constant = ConstantR2FromArray2D(&builder, *constant_rhs_array);
-  auto start_constant = ConstantR1<int32>(&builder, {0, 1});
-  auto dynamic_slice = DynamicSlice(lhs_constant, start_constant, {6, 1});
+  auto zero = ConstantR0<int32>(&builder, 0);
+  auto one = ConstantR0<int32>(&builder, 1);
+  auto dynamic_slice = DynamicSlice(lhs_constant, {zero, one}, {6, 1});
 
   DotDimensionNumbers dot_dnums;
   dot_dnums.add_lhs_contracting_dimensions(0);
@@ -1066,8 +1071,9 @@ XLA_TEST_F(DotOperationTest, DotOfGatherOptimizationWithConstLHSRows) {
   XlaBuilder builder(TestName());
   auto lhs_constant = ConstantR2FromArray2D(&builder, *constant_lhs_array);
   auto rhs_constant = ConstantR2FromArray2D(&builder, *constant_rhs_array);
-  auto start_constant = ConstantR1<int32>(&builder, {0, 1});
-  auto dynamic_slice = DynamicSlice(rhs_constant, start_constant, {6, 1});
+  auto zero = ConstantR0<int32>(&builder, 0);
+  auto one = ConstantR0<int32>(&builder, 1);
+  auto dynamic_slice = DynamicSlice(rhs_constant, {zero, one}, {6, 1});
 
   DotDimensionNumbers dot_dnums;
   dot_dnums.add_lhs_contracting_dimensions(0);
@@ -1090,8 +1096,9 @@ XLA_TEST_F(DotOperationTest, DotOfGatherOptimizationWithConstRHSCols) {
   XlaBuilder builder(TestName());
   auto lhs_constant = ConstantR2FromArray2D(&builder, *constant_lhs_array);
   auto rhs_constant = ConstantR2FromArray2D(&builder, *constant_rhs_array);
-  auto start_constant = ConstantR1<int32>(&builder, {1, 0});
-  auto dynamic_slice = DynamicSlice(lhs_constant, start_constant, {1, 6});
+  auto zero = ConstantR0<int32>(&builder, 0);
+  auto one = ConstantR0<int32>(&builder, 1);
+  auto dynamic_slice = DynamicSlice(lhs_constant, {one, zero}, {1, 6});
 
   DotDimensionNumbers dot_dnums;
   dot_dnums.add_lhs_contracting_dimensions(1);
@@ -1114,8 +1121,9 @@ XLA_TEST_F(DotOperationTest, DotOfGatherOptimizationWithConstLHSCols) {
   XlaBuilder builder(TestName());
   auto lhs_constant = ConstantR2FromArray2D(&builder, *constant_lhs_array);
   auto rhs_constant = ConstantR2FromArray2D(&builder, *constant_rhs_array);
-  auto start_constant = ConstantR1<int32>(&builder, {1, 0});
-  auto dynamic_slice = DynamicSlice(rhs_constant, start_constant, {1, 6});
+  auto zero = ConstantR0<int32>(&builder, 0);
+  auto one = ConstantR0<int32>(&builder, 1);
+  auto dynamic_slice = DynamicSlice(rhs_constant, {one, zero}, {1, 6});
 
   DotDimensionNumbers dot_dnums;
   dot_dnums.add_lhs_contracting_dimensions(1);
-- 
GitLab


From 0186c2d8d3735c2ea58d00f331eaa70b5ecacc42 Mon Sep 17 00:00:00 2001
From: Davide Libenzi <dlibenzi@google.com>
Date: Fri, 11 Jan 2019 15:12:56 -0800
Subject: [PATCH 0594/2345] Implement an XRT allocation op that takes a Tensor
 (with optional layout) as input.

PiperOrigin-RevId: 228953639
---
 tensorflow/compiler/tf2xla/shape_util.cc      |  93 ++++++++++++-
 tensorflow/compiler/tf2xla/shape_util.h       |  15 ++-
 tensorflow/compiler/xrt/kernels/BUILD         |   7 +-
 .../compiler/xrt/kernels/xrt_state_ops.cc     |  11 ++
 .../compiler/xrt/kernels/xrt_state_ops.h      | 108 +++++++++++++++
 tensorflow/compiler/xrt/ops/xrt_state_ops.cc  |  33 ++++-
 tensorflow/compiler/xrt/tests/BUILD           |   1 +
 tensorflow/compiler/xrt/tests/raw_api_test.cc | 124 ++++++++++++++++++
 tensorflow/compiler/xrt/xrt_state.cc          |   2 +-
 tensorflow/compiler/xrt/xrt_state.h           |   2 +-
 10 files changed, 383 insertions(+), 13 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/shape_util.cc b/tensorflow/compiler/tf2xla/shape_util.cc
index cb04a75d95..8997b2f5c6 100644
--- a/tensorflow/compiler/tf2xla/shape_util.cc
+++ b/tensorflow/compiler/tf2xla/shape_util.cc
@@ -44,6 +44,43 @@ Status PopulateInfeedLayoutVector(const xla::Shape& shape,
   return Status::OK();
 }
 
+// Populate the output layout unless the minor_to_major array contains all -1
+// value, in which case the layout is considered missing and the API returns
+// false.
+xla::StatusOr<bool> MakeLayout(absl::Span<const int64> minor_to_major,
+                               xla::Layout* layout) {
+  if (std::all_of(minor_to_major.begin(), minor_to_major.end(),
+                  [](int64 dim) { return dim == -1; })) {
+    return false;
+  }
+  std::vector<bool> dim_present(minor_to_major.size(), false);
+  for (auto dim : minor_to_major) {
+    if (dim < 0 || dim >= minor_to_major.size()) {
+      return errors::InvalidArgument("Layout dimension out of range: dim=", dim,
+                                     " rank=", minor_to_major.size());
+    }
+    if (dim_present[dim]) {
+      return errors::InvalidArgument("Repeated layout dimension: dim=", dim);
+    }
+    dim_present[dim] = true;
+  }
+  *layout = xla::LayoutUtil::MakeLayout(minor_to_major);
+  return true;
+}
+
+Status AssignLayout(
+    absl::Span<const int64> minor_to_major,
+    const std::function<xla::Layout(const xla::Shape&)>& layout_func,
+    xla::Shape* shape) {
+  xla::Layout layout;
+  TF_ASSIGN_OR_RETURN(bool has_layout, MakeLayout(minor_to_major, &layout));
+  if (!has_layout && layout_func) {
+    layout = layout_func(*shape);
+  }
+  *shape->mutable_layout() = layout;
+  return Status::OK();
+}
+
 }  // namespace
 
 // Convert an XLA Shape into the equivalent TensorFlow shape.
@@ -84,10 +121,64 @@ xla::Shape TensorShapeToXLAShape(xla::PrimitiveType type,
   return xla::ShapeUtil::MakeShapeWithLayout(type, dimensions, layout);
 }
 
-xla::StatusOr<std::vector<int>> GetInfeedLayoutVector(const xla::Shape& shape) {
+xla::StatusOr<std::vector<int>> GetShapeLayoutVector(const xla::Shape& shape) {
   std::vector<int> layouts;
   TF_RETURN_IF_ERROR(PopulateInfeedLayoutVector(shape, &layouts));
   return layouts;
 }
 
+Status GetShapeWithLayout(
+    const xla::Shape& input_shape, absl::Span<const int64> minor_to_major,
+    const std::function<xla::Layout(const xla::Shape&)>& layout_func,
+    xla::Shape* output_shape) {
+  if (input_shape.IsTuple()) {
+    int64 tuple_elements = xla::ShapeUtil::TupleElementCount(input_shape);
+    std::vector<xla::Shape> shapes;
+    shapes.reserve(tuple_elements);
+    size_t position = 0;
+    for (int64 i = 0; i < tuple_elements; ++i) {
+      const xla::Shape& shape =
+          xla::ShapeUtil::GetTupleElementShape(input_shape, i);
+      if (shape.IsTuple()) {
+        return errors::InvalidArgument(
+            "Nested tuples not supported: ",
+            xla::ShapeUtil::HumanString(input_shape));
+      }
+      int64 rank = shape.rank();
+      if (position + rank > minor_to_major.size()) {
+        return errors::InvalidArgument(
+            "Not enough layout attribute elements: position=", position,
+            " rank=", rank, " elements=", minor_to_major.size());
+      }
+      shapes.push_back(shape);
+      TF_RETURN_IF_ERROR(AssignLayout(
+          absl::Span<const int64>(minor_to_major).subspan(position, rank),
+          layout_func, &shapes.back()));
+      position += rank;
+
+      VLOG(4) << "Shape[" << i
+              << "] = " << xla::ShapeUtil::HumanStringWithLayout(shapes.back());
+    }
+    if (position != minor_to_major.size()) {
+      return errors::InvalidArgument(
+          "Too many elements passed in the layout attribute: position=",
+          position, " size=", minor_to_major.size());
+    }
+    *output_shape = xla::ShapeUtil::MakeTupleShape(shapes);
+  } else {
+    int64 rank = input_shape.rank();
+    if (rank != minor_to_major.size()) {
+      return errors::InvalidArgument(
+          "Wrong number of layout attribute elements: rank=", rank,
+          " elements=", minor_to_major.size());
+    }
+    *output_shape = input_shape;
+    TF_RETURN_IF_ERROR(AssignLayout(minor_to_major, layout_func, output_shape));
+
+    VLOG(4) << "Shape[] = "
+            << xla::ShapeUtil::HumanStringWithLayout(*output_shape);
+  }
+  return Status::OK();
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/tf2xla/shape_util.h b/tensorflow/compiler/tf2xla/shape_util.h
index cf52bf46e7..e775c4462c 100644
--- a/tensorflow/compiler/tf2xla/shape_util.h
+++ b/tensorflow/compiler/tf2xla/shape_util.h
@@ -45,12 +45,23 @@ xla::Shape TensorShapeToXLAShape(xla::PrimitiveType type,
                                  const TensorShape& tensor_shape);
 
 // Given an XLA shape with layouts, builds a layout vector in the form able to
-// be fed to an InfeedEnqueue/InfeedEnqueueTuple ops.
+// be fed to ops like InfeedEnqueue/InfeedEnqueueTuple/XRTAllocateV2/....
 // THe returned vector is a linearized sequence of the minor-to-major values of
 // the layouts held within the input shape.
 // In case the input shape is a tuple, the minor-to-major values will be in the
 // order of the tuple elements within the tuple shape.
-xla::StatusOr<std::vector<int>> GetInfeedLayoutVector(const xla::Shape& shape);
+// If a shape (or a subshape of a tuple shape) has missing layout, a rank long
+// sequence of -1 values will be emittted.
+xla::StatusOr<std::vector<int>> GetShapeLayoutVector(const xla::Shape& shape);
+
+// Given the input shape and a linearized sequence of the minor-to-major values
+// of the layouts, create the output shape by rewriting the input shape layouts.
+// If a layout is missing (has -1 values) for a matching tuple subshape, the
+// layout_func will be called, if not nullptr.
+Status GetShapeWithLayout(
+    const xla::Shape& input_shape, absl::Span<const int64> minor_to_major,
+    const std::function<xla::Layout(const xla::Shape&)>& layout_func,
+    xla::Shape* output_shape);
 
 }  // namespace tensorflow
 
diff --git a/tensorflow/compiler/xrt/kernels/BUILD b/tensorflow/compiler/xrt/kernels/BUILD
index d38f41ab43..dc02fd272f 100644
--- a/tensorflow/compiler/xrt/kernels/BUILD
+++ b/tensorflow/compiler/xrt/kernels/BUILD
@@ -11,20 +11,15 @@ cc_library(
     name = "xrt_state_ops",
     hdrs = ["xrt_state_ops.h"],
     deps = [
+        "//tensorflow/compiler/tf2xla:common",
         "//tensorflow/compiler/tf2xla:xla_compiler",
-        "//tensorflow/compiler/xla:debug_options_flags",
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:status_macros",
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:xla_data_proto",
-        "//tensorflow/compiler/xla/client:client_library",
-        "//tensorflow/compiler/xla/client:compile_only_client",
         "//tensorflow/compiler/xla/client:local_client",
-        "//tensorflow/compiler/xla/client:xla_computation",
-        "//tensorflow/compiler/xla/service:compiler",
         "//tensorflow/compiler/xla/service:computation_placer",
-        "//tensorflow/compiler/xla/service:hlo_proto",
         "//tensorflow/compiler/xrt:xrt_proto",
         "//tensorflow/compiler/xrt:xrt_utils",
         "//tensorflow/core:core_cpu_internal",
diff --git a/tensorflow/compiler/xrt/kernels/xrt_state_ops.cc b/tensorflow/compiler/xrt/kernels/xrt_state_ops.cc
index 1a5bfac337..6a7f106525 100644
--- a/tensorflow/compiler/xrt/kernels/xrt_state_ops.cc
+++ b/tensorflow/compiler/xrt/kernels/xrt_state_ops.cc
@@ -37,6 +37,17 @@ REGISTER_KERNEL_BUILDER(Name("XRTAllocate")
                             .HostMemory("handle"),
                         XRTAllocateOp<XRTGenericDeviceAccessor>);
 
+REGISTER_KERNEL_BUILDER(Name("XRTAllocateFromTensor")
+                            .Device(DEVICE_XLA_GPU)
+                            .HostMemory("inputs")
+                            .HostMemory("handle"),
+                        XRTAllocateFromTensorOp<XRTGenericDeviceAccessor>);
+REGISTER_KERNEL_BUILDER(Name("XRTAllocateFromTensor")
+                            .Device(DEVICE_XLA_CPU)
+                            .HostMemory("inputs")
+                            .HostMemory("handle"),
+                        XRTAllocateFromTensorOp<XRTGenericDeviceAccessor>);
+
 REGISTER_KERNEL_BUILDER(Name("XRTSubTuple")
                             .Device(DEVICE_XLA_GPU)
                             .HostMemory("base_handle")
diff --git a/tensorflow/compiler/xrt/kernels/xrt_state_ops.h b/tensorflow/compiler/xrt/kernels/xrt_state_ops.h
index c8def16bbc..e2c223b3db 100644
--- a/tensorflow/compiler/xrt/kernels/xrt_state_ops.h
+++ b/tensorflow/compiler/xrt/kernels/xrt_state_ops.h
@@ -19,10 +19,14 @@ limitations under the License.
 #ifndef TENSORFLOW_COMPILER_XRT_KERNELS_XRT_STATE_OPS_H_
 #define TENSORFLOW_COMPILER_XRT_KERNELS_XRT_STATE_OPS_H_
 
+#include <functional>
 #include <memory>
 #include <string>
 
+#include "tensorflow/compiler/tf2xla/literal_util.h"
+#include "tensorflow/compiler/tf2xla/shape_util.h"
 #include "tensorflow/compiler/xla/client/local_client.h"
+#include "tensorflow/compiler/xla/layout_util.h"
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/statusor.h"
@@ -30,6 +34,7 @@ limitations under the License.
 #include "tensorflow/compiler/xrt/xrt.pb.h"
 #include "tensorflow/compiler/xrt/xrt_device.h"
 #include "tensorflow/compiler/xrt/xrt_state.h"
+#include "tensorflow/core/common_runtime/dma_helper.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/framework/tensor.h"
@@ -200,6 +205,109 @@ class XRTAllocateOp : public OpKernel {
   }
 };
 
+// Op that allocates memory for a tensor (with optional layout) and transfers it
+// to the device, returning an allocation handle.
+template <class DeviceAccessor>
+class XRTAllocateFromTensorOp : public OpKernel {
+ public:
+  explicit XRTAllocateFromTensorOp(OpKernelConstruction* ctx) : OpKernel(ctx) {
+    bool make_tuple = false;
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("shapes", &tf_shapes_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("dtypes", &dtypes_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("make_tuple", &make_tuple));
+    if (ctx->HasAttr("layouts")) {
+      OP_REQUIRES_OK(ctx, ctx->GetAttr("layouts", &minor_to_major_));
+    }
+    OP_REQUIRES(
+        ctx, tf_shapes_.size() == dtypes_.size(),
+        errors::InvalidArgument("shapes and dtypes must be the same length"));
+    std::vector<xla::Shape> xla_shapes;
+    for (int i = 0; i < tf_shapes_.size(); i++) {
+      xla::Shape xla_shape;
+      OP_REQUIRES_OK(
+          ctx, TensorShapeToXLAShape(dtypes_[i], tf_shapes_[i], &xla_shape));
+      xla_shapes.push_back(xla_shape);
+    }
+    if (xla_shapes.size() > 1 || make_tuple) {
+      shape_ = xla::ShapeUtil::MakeTupleShape(xla_shapes);
+    } else {
+      shape_.Swap(&xla_shapes.front());
+    }
+    if (!minor_to_major_.empty()) {
+      xla::Shape shape_with_layouts;
+      OP_REQUIRES_OK(ctx, GetShapeWithLayout(shape_, minor_to_major_,
+                                             /*layout_func=*/nullptr,
+                                             &shape_with_layouts));
+      shape_.Swap(&shape_with_layouts);
+    }
+  }
+
+  ~XRTAllocateFromTensorOp() override = default;
+  XRTAllocateFromTensorOp(const XRTAllocateFromTensorOp&) = delete;
+  XRTAllocateFromTensorOp& operator=(const XRTAllocateFromTensorOp&) = delete;
+
+  void Compute(OpKernelContext* ctx) override {
+    VLOG(1) << "XRTAllocateFromTensorOp::Compute";
+
+    OpInputList values;
+    OP_REQUIRES_OK(ctx, ctx->input_list("inputs", &values));
+    OP_REQUIRES(ctx, values.size() == tf_shapes_.size(),
+                errors::InvalidArgument(
+                    "Wrong number of inputs to XRTAllocateFromTensor: ",
+                    values.size(), " vs. ", tf_shapes_.size()));
+
+    std::vector<const char*> tensors_data;
+    for (size_t i = 0; i < values.size(); ++i) {
+      const Tensor& input_tensor = values[i];
+      OP_REQUIRES(ctx, input_tensor.dtype() == dtypes_[i],
+                  errors::InvalidArgument(
+                      "Input tensor type and input dtype do not match"));
+      // We allow the requested on-device shape to differ from the shape of the
+      // input tensor, as long as they have the same number of elements.
+      OP_REQUIRES(
+          ctx,
+          input_tensor.shape().num_elements() == tf_shapes_[i].num_elements(),
+          errors::InvalidArgument(
+              "Input tensor must have the number of elements specified "
+              "in the matching input shape: ",
+              input_tensor.shape().num_elements(), " vs. ",
+              tf_shapes_[i].num_elements(), " at index ", i));
+      tensors_data.push_back(
+          static_cast<const char*>(DMAHelper::base(&input_tensor)));
+    }
+    // Use the buffer straight out of the input tensors to create the literal.
+    xla::BorrowingLiteral literal =
+        shape_.IsTuple() ? xla::BorrowingLiteral(tensors_data, shape_)
+                         : xla::BorrowingLiteral(tensors_data.front(), shape_);
+    ResourceMgr* rm;
+    OP_REQUIRES_OK(ctx, DeviceAccessor::GetResourceManager(ctx, &rm));
+
+    // We are guaranteed that the underlying device object won't be deleted out
+    // from under us, while the ScopedRef is live.
+    class DeviceAccessor::ScopedRef device_ref;
+    OP_REQUIRES_OK(ctx, DeviceAccessor::InitScopedRef(ctx, &device_ref));
+
+    XRTTupleAllocation* allocation;
+    OP_REQUIRES_OK(ctx, XRTTupleAllocation::CreateAndTransfer(
+                            literal, device_ref.backend(),
+                            device_ref.device_ordinal(), &allocation));
+
+    // Intern takes ownership of our reference to allocation.
+    int64 key;
+    OP_REQUIRES_OK(ctx, allocation->Intern(rm, &key));
+
+    Tensor output(DT_INT64, TensorShape({}));
+    output.scalar<int64>()() = key;
+    ctx->set_output(0, output);
+  }
+
+ private:
+  std::vector<TensorShape> tf_shapes_;
+  DataTypeVector dtypes_;
+  std::vector<int64> minor_to_major_;
+  xla::Shape shape_;
+};
+
 // Op that takes a tuple handle input and returns a handle to a sub-tuple of the
 // input.
 template <bool discard_, class DeviceAccessor>
diff --git a/tensorflow/compiler/xrt/ops/xrt_state_ops.cc b/tensorflow/compiler/xrt/ops/xrt_state_ops.cc
index db58f0797d..2e743fec49 100644
--- a/tensorflow/compiler/xrt/ops/xrt_state_ops.cc
+++ b/tensorflow/compiler/xrt/ops/xrt_state_ops.cc
@@ -26,12 +26,41 @@ REGISTER_OP("XRTAllocate")
     .SetShapeFn(tensorflow::shape_inference::ScalarShape)
     .Doc(
         R"(
-Reads a literal proto and transfers it to TPU device memory.
+Reads a literal proto and transfers it to device memory.
 
-'allocation' is a serialized xrt::TPUAllocation proto.
+'allocation' is a serialized xrt::XLAAllocation proto.
 'handle' is an id that can be used in other ops to refer to the allocation.
 )");
 
+REGISTER_OP("XRTAllocateFromTensor")
+    .Input("inputs: dtypes")
+    .Output("handle: int64")
+    .Attr("dtypes: list(type)")
+    .Attr("shapes: list(shape)")
+    .Attr("layouts: list(int) = []")
+    .Attr("make_tuple: bool = false")
+    .SetShapeFn(tensorflow::shape_inference::ScalarShape)
+    .Doc(
+        R"(
+Reads a list of tensors with optional layouts, and transfers it to device
+memory.
+
+inputs: The tensors holding the input data.
+shapes: The shapes which the tensors should have on device. The i-th shape
+corresponds to the i-th input. The shapes, together with the (optional)
+layouts, helps creating the fully qualified shape of the data on the device.
+The shapes can differ from the corresponding input one, as long as the total
+number of elements matches. In other words, it is possible to feed an input
+tensor with shape {8} and have a corresponding shape {2,2,2}.
+layouts: A vector holding the requested layout in minor-to-major sequence.
+If empty, the default layout wil be used.
+For a tuple, the layouts vector holds a linearized minor-to-major numbers
+for all the tuple leaves, in the order they appear within the tuple.
+The elements within the layouts sequence corresponding to a given tuple
+subshape can be set to -1, to leave such subshape to the default shape.
+handle: An id that can be used in other ops to refer to the allocation.
+)");
+
 REGISTER_OP("XRTSubTuple")
     .Input("base_handle: int64")
     .Input("shape_index: int32")
diff --git a/tensorflow/compiler/xrt/tests/BUILD b/tensorflow/compiler/xrt/tests/BUILD
index be44a3474a..3a19327e5b 100644
--- a/tensorflow/compiler/xrt/tests/BUILD
+++ b/tensorflow/compiler/xrt/tests/BUILD
@@ -24,6 +24,7 @@ cc_library(
         "//tensorflow/cc:client_session",
         "//tensorflow/cc:ops",
         "//tensorflow/cc:scope",
+        "//tensorflow/compiler/tf2xla:common",
         "//tensorflow/compiler/tf2xla:xla_compiler",
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:literal_util",
diff --git a/tensorflow/compiler/xrt/tests/raw_api_test.cc b/tensorflow/compiler/xrt/tests/raw_api_test.cc
index 8e70cc3959..1111f82405 100644
--- a/tensorflow/compiler/xrt/tests/raw_api_test.cc
+++ b/tensorflow/compiler/xrt/tests/raw_api_test.cc
@@ -22,6 +22,8 @@ limitations under the License.
 #include "tensorflow/cc/framework/ops.h"
 #include "tensorflow/cc/framework/scope.h"
 #include "tensorflow/cc/ops/standard_ops.h"
+#include "tensorflow/compiler/tf2xla/literal_util.h"
+#include "tensorflow/compiler/tf2xla/shape_util.h"
 #include "tensorflow/compiler/xla/client/client_library.h"
 #include "tensorflow/compiler/xla/client/local_client.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
@@ -53,6 +55,14 @@ string DeviceFromFlag() {
   return absl::StrCat("/device:", xla_test_device, ":0");
 }
 
+std::vector<int> GetAttrLayout(absl::Span<const int64> minor_to_mayor) {
+  std::vector<int> layout;
+  for (auto dim : minor_to_mayor) {
+    layout.push_back(static_cast<int>(dim));
+  }
+  return layout;
+}
+
 xla::LiteralProto TwoElementTuple() {
   auto array = xla::LiteralUtil::CreateR1<float>({1.0f, 3.0f});
   auto matrix = xla::LiteralUtil::CreateR2({{4, 5}, {6, 7}});
@@ -233,6 +243,120 @@ xla::ProgramShape XlaCompiledProgramShape(
       ->ComputeProgramShape();
 }
 
+TEST(RawApiTest, AllocFromTensor) {
+  xla::Literal literal =
+      xla::LiteralUtil::CreateR2<float>({{4.0f, 5.0f}, {6.0f, 7.0f}});
+  Tensor tensor;
+  TF_ASSERT_OK(LiteralToHostTensor(literal, DT_FLOAT, &tensor));
+
+  Scope root = Scope::NewRootScope().WithDevice(DeviceFromFlag());
+  std::vector<int> layout =
+      GetAttrLayout(literal.shape().layout().minor_to_major());
+  ops::XRTAllocateFromTensor::Attrs alloc_attrs =
+      ops::XRTAllocateFromTensor::Layouts(layout);
+  auto handle =
+      ops::XRTAllocateFromTensor(root, {tensor}, {tensor.shape()}, alloc_attrs);
+  auto read_back = ops::XRTReadLiteralAndRelease(root, handle);
+  TF_ASSERT_OK(root.status());
+
+  ClientSession session(root);
+  std::vector<Tensor> outputs;
+  TF_EXPECT_OK(session.Run({read_back}, &outputs));
+  EXPECT_EQ(outputs.size(), 1);
+
+  xla::LiteralProto response;
+  EXPECT_TRUE(response.ParseFromString(outputs[0].scalar<string>()()));
+  EXPECT_TRUE(CompareLiteralToLiteralProto(literal, response));
+}
+
+TEST(RawApiTest, AllocFromTensorTuple) {
+  xla::Literal literal0 =
+      xla::LiteralUtil::CreateR2<float>({{4.0f, 5.0f}, {6.0f, 7.0f}});
+  xla::Literal literal1 =
+      xla::LiteralUtil::CreateR2<float>({{14.0f, -5.0f}, {16.0f, 17.0f}});
+  xla::Literal literal = xla::LiteralUtil::MakeTuple({&literal0, &literal1});
+  Tensor tensor0;
+  TF_ASSERT_OK(LiteralToHostTensor(literal0, DT_FLOAT, &tensor0));
+  Tensor tensor1;
+  TF_ASSERT_OK(LiteralToHostTensor(literal1, DT_FLOAT, &tensor1));
+
+  Scope root = Scope::NewRootScope().WithDevice(DeviceFromFlag());
+  std::vector<int> layout = GetShapeLayoutVector(literal.shape()).ValueOrDie();
+  ops::XRTAllocateFromTensor::Attrs alloc_attrs =
+      ops::XRTAllocateFromTensor::Layouts(layout);
+  auto handle = ops::XRTAllocateFromTensor(root, {tensor0, tensor1},
+                                           {tensor0.shape(), tensor1.shape()},
+                                           alloc_attrs);
+  auto read_back = ops::XRTReadLiteralAndRelease(root, handle);
+  TF_ASSERT_OK(root.status());
+
+  ClientSession session(root);
+  std::vector<Tensor> outputs;
+  TF_EXPECT_OK(session.Run({read_back}, &outputs));
+  EXPECT_EQ(outputs.size(), 1);
+
+  xla::LiteralProto response;
+  EXPECT_TRUE(response.ParseFromString(outputs[0].scalar<string>()()));
+  EXPECT_TRUE(CompareLiteralToLiteralProto(literal, response));
+}
+
+TEST(RawApiTest, AllocFromTensorTupleSingle) {
+  xla::Literal literal0 =
+      xla::LiteralUtil::CreateR2<float>({{4.0f, 5.0f}, {6.0f, 7.0f}});
+  xla::Literal literal = xla::LiteralUtil::MakeTuple({&literal0});
+  Tensor tensor0;
+  TF_ASSERT_OK(LiteralToHostTensor(literal0, DT_FLOAT, &tensor0));
+
+  Scope root = Scope::NewRootScope().WithDevice(DeviceFromFlag());
+  std::vector<int> layout = GetShapeLayoutVector(literal.shape()).ValueOrDie();
+  ops::XRTAllocateFromTensor::Attrs alloc_attrs =
+      ops::XRTAllocateFromTensor::Layouts(layout).MakeTuple(true);
+  auto handle = ops::XRTAllocateFromTensor(root, {tensor0}, {tensor0.shape()},
+                                           alloc_attrs);
+  auto read_back = ops::XRTReadLiteralAndRelease(root, handle);
+  TF_ASSERT_OK(root.status());
+
+  ClientSession session(root);
+  std::vector<Tensor> outputs;
+  TF_EXPECT_OK(session.Run({read_back}, &outputs));
+  EXPECT_EQ(outputs.size(), 1);
+
+  xla::LiteralProto response;
+  EXPECT_TRUE(response.ParseFromString(outputs[0].scalar<string>()()));
+  EXPECT_TRUE(CompareLiteralToLiteralProto(literal, response));
+}
+
+TEST(RawApiTest, AllocFromTensorRelayout) {
+  xla::Literal literal =
+      xla::LiteralUtil::CreateR2<float>({{4.0f, 5.0f}, {6.0f, 7.0f}});
+  Tensor tensor;
+  TF_ASSERT_OK(LiteralToHostTensor(literal, DT_FLOAT, &tensor));
+
+  Scope root = Scope::NewRootScope().WithDevice(DeviceFromFlag());
+  // Use inverse array layout with the tensor data above.
+  std::vector<int> layout({0, 1});
+  ops::XRTAllocateFromTensor::Attrs alloc_attrs =
+      ops::XRTAllocateFromTensor::Layouts(layout);
+  auto handle =
+      ops::XRTAllocateFromTensor(root, {tensor}, {tensor.shape()}, alloc_attrs);
+  auto read_back = ops::XRTReadLiteralAndRelease(root, handle);
+  TF_ASSERT_OK(root.status());
+
+  ClientSession session(root);
+  std::vector<Tensor> outputs;
+  TF_EXPECT_OK(session.Run({read_back}, &outputs));
+  EXPECT_EQ(outputs.size(), 1);
+
+  xla::LiteralProto response;
+  EXPECT_TRUE(response.ParseFromString(outputs[0].scalar<string>()()));
+  // We have sent literal's data (in array layout) with a attribute layout
+  // {0,1}, so the expected literal read from device needs to be changed
+  // accordingly.
+  xla::Literal expected_literal =
+      xla::LiteralUtil::CreateR2<float>({{4.0f, 6.0f}, {5.0f, 7.0f}});
+  EXPECT_TRUE(CompareLiteralToLiteralProto(expected_literal, response));
+}
+
 TEST(RawApiTest, AllocAndRewrite) {
   xrt::XLAAllocation alloc;
   *alloc.mutable_value() =
diff --git a/tensorflow/compiler/xrt/xrt_state.cc b/tensorflow/compiler/xrt/xrt_state.cc
index 13c275aaca..1e2a9584f8 100644
--- a/tensorflow/compiler/xrt/xrt_state.cc
+++ b/tensorflow/compiler/xrt/xrt_state.cc
@@ -182,7 +182,7 @@ XRTTupleAllocation::~XRTTupleAllocation() {
 }
 
 /*static*/ Status XRTTupleAllocation::CreateAndTransfer(
-    const xla::Literal& literal, xla::Backend* backend, int device_ordinal,
+    const xla::LiteralBase& literal, xla::Backend* backend, int device_ordinal,
     XRTTupleAllocation** allocation) {
   auto transfer_manager = backend->transfer_manager();
   auto allocator = backend->memory_allocator();
diff --git a/tensorflow/compiler/xrt/xrt_state.h b/tensorflow/compiler/xrt/xrt_state.h
index 1b32b6a20f..ddf2656e6f 100644
--- a/tensorflow/compiler/xrt/xrt_state.h
+++ b/tensorflow/compiler/xrt/xrt_state.h
@@ -88,7 +88,7 @@ class XRTTupleAllocation : public ResourceBase {
   // Allocates new device memory buffers sufficient to store literal, transfers
   // literal to that memory, and returns a XRTTupleAllocation handle to the
   // allocated buffers.
-  static Status CreateAndTransfer(const xla::Literal& literal,
+  static Status CreateAndTransfer(const xla::LiteralBase& literal,
                                   xla::Backend* backend, int device_ordinal,
                                   XRTTupleAllocation** allocation);
 
-- 
GitLab


From 8ec7b777318a4f93dfb2f34bc6a0e89e48bc4ea3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 15:27:00 -0800
Subject: [PATCH 0595/2345] Enable quantization support for ReduceMax and
 ReduceMin

PiperOrigin-RevId: 228955777
---
 .../lite/toco/graph_transformations/hardcode_min_max.cc       | 4 +++-
 tensorflow/lite/toco/graph_transformations/quantize.cc        | 3 ++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/toco/graph_transformations/hardcode_min_max.cc b/tensorflow/lite/toco/graph_transformations/hardcode_min_max.cc
index 2e41767095..6882a19801 100644
--- a/tensorflow/lite/toco/graph_transformations/hardcode_min_max.cc
+++ b/tensorflow/lite/toco/graph_transformations/hardcode_min_max.cc
@@ -440,6 +440,8 @@ bool HardcodeMinMaxForLstmCell(Model* model, Operator* op) {
     case OperatorType::kGather:
     case OperatorType::kTranspose:
     case OperatorType::kMean:
+    case OperatorType::kReduceMax:
+    case OperatorType::kReduceMin:
       changed = HardcodeMinMaxFromFirstInput(model, op);
       break;
     case OperatorType::kSum:
@@ -448,7 +450,7 @@ bool HardcodeMinMaxForLstmCell(Model* model, Operator* op) {
       // in special circumstances like when computing expected value using
       // reduce_sum the input range and the output range matches. Hence the
       // below code would act as a fallback. If a fake_quant node is observed in
-      // the output that takes precendence over the hard coding logic below.
+      // the output that takes precedence over the hard coding logic below.
       changed = HardcodeMinMaxFromFirstInput(model, op);
       if (changed) {
         LOG(WARNING) << "Using the input range for output in reduce_sum op."
diff --git a/tensorflow/lite/toco/graph_transformations/quantize.cc b/tensorflow/lite/toco/graph_transformations/quantize.cc
index 2fa80f2eda..ee65f92e00 100644
--- a/tensorflow/lite/toco/graph_transformations/quantize.cc
+++ b/tensorflow/lite/toco/graph_transformations/quantize.cc
@@ -66,7 +66,8 @@ bool SupportsQuantization(const Operator& op) {
          type == OperatorType::kPack || type == OperatorType::kTopK_V2 ||
          type == OperatorType::kRandomUniform ||
          type == OperatorType::kResizeNearestNeighbor ||
-         type == OperatorType::kPRelu;
+         type == OperatorType::kPRelu || type == OperatorType::kReduceMax ||
+         type == OperatorType::kReduceMin;
 }
 
 // The quantized op allows output arrays of type float using
-- 
GitLab


From a8bec4ea824a3ad98dcf2065c6933f3bdb49580d Mon Sep 17 00:00:00 2001
From: Bairen Yi <byronyi@clustar.ai>
Date: Fri, 11 Jan 2019 23:41:39 +0000
Subject: [PATCH 0596/2345] Add support of collective ops for GDR

Fixes #18232.

Signed-off-by: Bairen Yi <byronyi@clustar.ai>
---
 tensorflow/contrib/gdr/BUILD                  |  21 +++
 .../gdr/gdr_collective_executor_mgr.cc        | 160 ++++++++++++++++++
 .../contrib/gdr/gdr_collective_executor_mgr.h |  56 ++++++
 tensorflow/contrib/gdr/gdr_server_lib.cc      |  30 +++-
 tensorflow/contrib/gdr/gdr_worker.cc          |  41 +++++
 tensorflow/contrib/gdr/gdr_worker.h           |   4 +
 .../rpc_collective_executor_mgr.h             |   2 +-
 7 files changed, 310 insertions(+), 4 deletions(-)
 create mode 100644 tensorflow/contrib/gdr/gdr_collective_executor_mgr.cc
 create mode 100644 tensorflow/contrib/gdr/gdr_collective_executor_mgr.h

diff --git a/tensorflow/contrib/gdr/BUILD b/tensorflow/contrib/gdr/BUILD
index 704be917b3..89314d162b 100644
--- a/tensorflow/contrib/gdr/BUILD
+++ b/tensorflow/contrib/gdr/BUILD
@@ -100,12 +100,33 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "gdr_collective_executor_mgr",
+    srcs = ["gdr_collective_executor_mgr.cc"],
+    hdrs = ["gdr_collective_executor_mgr.h"],
+    deps = [
+        ":gdr_memory_manager",
+        "//tensorflow/core:core_cpu_internal",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core/distributed_runtime:cancellable_call",
+        "//tensorflow/core/distributed_runtime:collective_param_resolver_distributed",
+        "//tensorflow/core/distributed_runtime:device_resolver_distributed",
+        "//tensorflow/core/distributed_runtime:request_id",
+        "//tensorflow/core/distributed_runtime:rpc_collective_executor_mgr",
+        "//tensorflow/core/distributed_runtime:worker_cache",
+        "//tensorflow/core/distributed_runtime:worker_env",
+        "//tensorflow/core/distributed_runtime:worker_interface",
+    ],
+)
+
 cc_library(
     name = "gdr_server_lib",
     srcs = ["gdr_server_lib.cc"],
     hdrs = ["gdr_server_lib.h"],
     linkstatic = 1,  # Seems to be needed since alwayslink is broken in bazel
     deps = [
+        ":gdr_collective_executor_mgr",
         ":gdr_memory_manager",
         ":gdr_rendezvous_mgr",
         ":gdr_worker",
diff --git a/tensorflow/contrib/gdr/gdr_collective_executor_mgr.cc b/tensorflow/contrib/gdr/gdr_collective_executor_mgr.cc
new file mode 100644
index 0000000000..b84710d26e
--- /dev/null
+++ b/tensorflow/contrib/gdr/gdr_collective_executor_mgr.cc
@@ -0,0 +1,160 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/contrib/gdr/gdr_collective_executor_mgr.h"
+
+#include "tensorflow/core/common_runtime/base_collective_executor.h"
+#include "tensorflow/core/common_runtime/collective_executor_mgr.h"
+#include "tensorflow/core/common_runtime/collective_rma_local.h"
+#include "tensorflow/core/common_runtime/dma_helper.h"
+#include "tensorflow/core/distributed_runtime/cancellable_call.h"
+#include "tensorflow/core/distributed_runtime/collective_param_resolver_distributed.h"
+#include "tensorflow/core/distributed_runtime/device_resolver_distributed.h"
+#include "tensorflow/core/distributed_runtime/request_id.h"
+#include "tensorflow/core/distributed_runtime/worker_cache.h"
+#include "tensorflow/core/lib/random/random.h"
+
+namespace tensorflow {
+
+class WorkerCacheInterface;
+
+namespace {
+
+class RecvBufCall : public CancellableCall {
+ public:
+  RecvBufCall(int64 step_id, const string& peer_device, const string& peer_task,
+              const string& key, Device* to_device,
+              DeviceContext* to_device_ctx,
+              const AllocatorAttributes& to_alloc_attr, Tensor* to_tensor,
+              const DeviceLocality& client_locality,
+              const DeviceLocality& server_locality,
+              CancellationManager* cancel_mgr, WorkerCacheInterface* wc)
+      : CancellableCall(cancel_mgr, peer_task, wc) {
+    req_.set_step_id(step_id);
+    req_.set_buf_rendezvous_key(key);
+    *req_.mutable_client_locality() = client_locality;
+    *req_.mutable_server_locality() = server_locality;
+    req_.set_num_bytes(to_tensor->TotalBytes());
+    req_.set_buf_ptr(reinterpret_cast<int64>(DMAHelper::base(to_tensor)));
+    req_.set_src_device(peer_device);
+    req_.set_dst_device(to_device->name());
+    req_.set_request_id(GetUniqueRequestId());
+  }
+
+  ~RecvBufCall() override {}
+
+  void IssueCall(const StatusCallback& done) override {
+    wi_->RecvBufAsync(&opts_, &req_, &resp_, done);
+  }
+
+  RecvBufRequest req_;
+  RecvBufResponse resp_;
+};
+
+class CollectiveRemoteAccessDistributed : public CollectiveRemoteAccessLocal {
+ public:
+  CollectiveRemoteAccessDistributed(const DeviceMgr* dev_mgr,
+                                    DeviceResolverInterface* dev_resolver,
+                                    WorkerCacheInterface* worker_cache,
+                                    int64 step_id,
+                                    RemoteMemoryManager* remote_memory_manager)
+      : CollectiveRemoteAccessLocal(dev_mgr, dev_resolver, step_id),
+        worker_cache_(worker_cache),
+        remote_memory_manager_(remote_memory_manager) {}
+
+  ~CollectiveRemoteAccessDistributed() override {}
+
+  void RecvFromPeer(const string& peer_device, const string& peer_task,
+                    bool peer_is_local, const string& key, Device* to_device,
+                    DeviceContext* to_device_ctx,
+                    const AllocatorAttributes& to_alloc_attr, Tensor* to_tensor,
+                    const DeviceLocality& client_locality,
+                    int dev_to_dev_stream_index,
+                    const StatusCallback& done) override {
+    if (peer_is_local) {
+      CollectiveRemoteAccessLocal::RecvFromPeer(
+          peer_device, peer_task, peer_is_local, key, to_device, to_device_ctx,
+          to_alloc_attr, to_tensor, client_locality, dev_to_dev_stream_index,
+          done);
+      return;
+    }
+
+    // State that needs to be threaded through a couple of async calls
+    // in order to make this function completely non-blocking.
+    struct State {
+      DeviceLocality server_locality;
+      std::unique_ptr<RecvBufCall> call;
+    };
+    State* state = new State;
+
+    // Logic to be executed on the RecvBufAsync callback.
+    auto recv_buf_callback = [this, state, peer_task, to_device, to_alloc_attr,
+                              to_device_ctx, to_tensor, dev_to_dev_stream_index,
+                              done](const Status& s) {
+      if (s.ok()) {
+        remote_memory_manager_->TensorFromTransportOptions(
+            to_tensor, state->call->resp_.transport_options(), to_device,
+            to_device_ctx, to_alloc_attr.on_host(), done);
+      }
+      if (!s.ok() && errors::IsFailedPrecondition(s)) {
+        dev_resolver_->ClearTask(peer_task);
+      }
+
+      delete state;
+    };
+
+    // Logic to execute once we have the device locality for the server-side
+    // device.
+    auto dev_locality_callback = [this, state, peer_device, peer_task, key,
+                                  to_device, to_device_ctx, to_alloc_attr,
+                                  to_tensor, client_locality,
+                                  recv_buf_callback](const Status& s) {
+      if (!s.ok()) {
+        recv_buf_callback(s);
+      } else {
+        state->call.reset(new RecvBufCall(
+            step_id_, peer_device, peer_task, key, to_device, to_device_ctx,
+            to_alloc_attr, to_tensor, client_locality, state->server_locality,
+            &cancel_mgr_, worker_cache_));
+        state->call->Start(recv_buf_callback);
+      }
+    };
+
+    dev_resolver_->GetLocalityAsync(
+        peer_device, peer_task, &state->server_locality, dev_locality_callback);
+  }
+
+  void StartAbort(const Status& s) override {
+    CollectiveRemoteAccessLocal::StartAbort(s);
+    cancel_mgr_.StartCancel();
+  }
+
+ protected:
+  WorkerCacheInterface* worker_cache_;  // Not owned
+  CancellationManager cancel_mgr_;
+  RemoteMemoryManager* remote_memory_manager_;
+};
+
+}  // namespace
+
+CollectiveExecutor* GdrCollectiveExecutorMgr::Create(int64 step_id) {
+  CollectiveRemoteAccessDistributed* rma =
+      new CollectiveRemoteAccessDistributed(dev_mgr_, dev_resolver_.get(),
+                                            worker_cache_, step_id,
+                                            remote_memory_manager_);
+  return new BaseCollectiveExecutor(this, rma, step_id, dev_mgr_,
+                                    &gpu_ring_order_);
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/contrib/gdr/gdr_collective_executor_mgr.h b/tensorflow/contrib/gdr/gdr_collective_executor_mgr.h
new file mode 100644
index 0000000000..a713c1c6b0
--- /dev/null
+++ b/tensorflow/contrib/gdr/gdr_collective_executor_mgr.h
@@ -0,0 +1,56 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CONTRIB_GDR_GDR_COLLECTIVE_EXECUTOR_MGR_H_
+#define TENSORFLOW_CONTRIB_GDR_GDR_COLLECTIVE_EXECUTOR_MGR_H_
+
+#include "tensorflow/contrib/gdr/gdr_memory_manager.h"
+#include "tensorflow/core/distributed_runtime/rpc_collective_executor_mgr.h"
+#include "tensorflow/core/framework/collective.h"
+
+namespace tensorflow {
+class CollectiveParamResolverDistributed;
+class ConfigProto;
+class DeviceMgr;
+class DeviceResolverDistributed;
+class WorkerCacheInterface;
+class StepSequenceRequest;
+class StepSequenceResponse;
+
+// An implementation of CollectiveExecutorMgr for a distributed environment
+// that uses WorkerInterface::RecvBufAsync to route data transfers over RDMA.
+class GdrCollectiveExecutorMgr : public RpcCollectiveExecutorMgr {
+ public:
+  GdrCollectiveExecutorMgr(
+      const ConfigProto& config, const DeviceMgr* dev_mgr,
+      std::unique_ptr<DeviceResolverDistributed> dev_resolver,
+      std::unique_ptr<CollectiveParamResolverDistributed> param_resolver,
+      WorkerCacheInterface* worker_cache, const string& task_name,
+      RemoteMemoryManager* remote_memory_manager)
+      : RpcCollectiveExecutorMgr(config, dev_mgr, std::move(dev_resolver),
+                                 std::move(param_resolver), worker_cache,
+                                 task_name),
+        remote_memory_manager_(remote_memory_manager) {}
+
+  ~GdrCollectiveExecutorMgr() override {}
+
+ protected:
+  virtual CollectiveExecutor* Create(int64 step_id) override;
+
+ private:
+  RemoteMemoryManager* remote_memory_manager_;  // Not owned.
+};
+
+}  // namespace tensorflow
+#endif  // TENSORFLOW_CONTRIB_GDR_GDR_COLLECTIVE_EXECUTOR_MGR_H_
diff --git a/tensorflow/contrib/gdr/gdr_server_lib.cc b/tensorflow/contrib/gdr/gdr_server_lib.cc
index dc0d5d548b..9f27d70008 100644
--- a/tensorflow/contrib/gdr/gdr_server_lib.cc
+++ b/tensorflow/contrib/gdr/gdr_server_lib.cc
@@ -16,11 +16,13 @@ limitations under the License.
 #include "tensorflow/contrib/gdr/gdr_server_lib.h"
 
 #include "grpc/support/alloc.h"
+#include "tensorflow/contrib/gdr/gdr_collective_executor_mgr.h"
 #include "tensorflow/contrib/gdr/gdr_memory_manager.h"
 #include "tensorflow/contrib/gdr/gdr_rendezvous_mgr.h"
 #include "tensorflow/contrib/gdr/gdr_worker.h"
-
-#include "grpc/support/alloc.h"
+#include "tensorflow/core/common_runtime/collective_rma_local.h"
+#include "tensorflow/core/distributed_runtime/collective_param_resolver_distributed.h"
+#include "tensorflow/core/distributed_runtime/device_resolver_distributed.h"
 
 namespace tensorflow {
 
@@ -57,10 +59,32 @@ Status GdrServer::Init() {
     return std::unique_ptr<GdrWorker>(
         new GdrWorker(env, config, remote_memory_manager_.get()));
   };
+  CollectiveMgrCreationFunction collective_mgr_func = [this](
+      const ConfigProto& config, const WorkerEnv* env,
+      WorkerCacheInterface* worker_cache) {
+
+    string unused;
+    string default_worker_name;
+    DeviceNameUtils::SplitDeviceName(env->device_mgr->ListDevices()[0]->name(),
+                                     &default_worker_name, &unused);
+
+    std::unique_ptr<DeviceResolverDistributed> dev_resolver(
+        new DeviceResolverDistributed(env->device_mgr, worker_cache,
+                                      default_worker_name));
+    std::unique_ptr<CollectiveParamResolverDistributed> param_resolver(
+        new CollectiveParamResolverDistributed(config, env->device_mgr,
+                                               dev_resolver.get(), worker_cache,
+                                               default_worker_name));
+    return new GdrCollectiveExecutorMgr(
+        config, env->device_mgr, std::move(dev_resolver),
+        std::move(param_resolver), worker_cache, default_worker_name,
+        remote_memory_manager_.get());
 
+  };
   TF_RETURN_IF_ERROR(remote_memory_manager_->Init());
 
-  return GrpcServer::Init(nullptr, rendezvous_mgr_func, nullptr, worker_func);
+  return GrpcServer::Init(nullptr, rendezvous_mgr_func, collective_mgr_func,
+                          worker_func);
 }
 
 Status GdrServer::Start() {
diff --git a/tensorflow/contrib/gdr/gdr_worker.cc b/tensorflow/contrib/gdr/gdr_worker.cc
index 016e5ea27b..29ee4c9375 100644
--- a/tensorflow/contrib/gdr/gdr_worker.cc
+++ b/tensorflow/contrib/gdr/gdr_worker.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/contrib/gdr/gdr_worker.h"
 
+#include "tensorflow/core/common_runtime/buf_rendezvous.h"
 #include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/common_runtime/device_mgr.h"
 #include "tensorflow/core/common_runtime/dma_helper.h"
@@ -29,10 +30,13 @@ limitations under the License.
 #include "tensorflow/core/distributed_runtime/worker_cache.h"
 #include "tensorflow/core/distributed_runtime/worker_session.h"
 #include "tensorflow/core/framework/cancellation.h"
+#include "tensorflow/core/framework/collective.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/tracing.h"
+#include "tensorflow/core/protobuf/transport_options.pb.h"
+#include "tensorflow/core/protobuf/worker.pb.h"
 
 namespace tensorflow {
 
@@ -145,4 +149,41 @@ void GdrWorker::GrpcRecvTensorAsync(CallOptions* opts,
       });
 }
 
+void GdrWorker::RecvBufAsync(CallOptions* opts, const RecvBufRequest* request,
+                             RecvBufResponse* response, StatusCallback done) {
+  // This is an RDMA enabled implementation augmenting grpc.
+  Status s = recv_tensor_recent_request_ids_.TrackUnique(
+      request->request_id(), "RecvBuf (GdrWorker)", *request);
+  if (!s.ok()) {
+    done(s);
+    return;
+  }
+  CollectiveExecutor::Handle ce_handle(
+      env_->collective_executor_mgr->FindOrCreate(request->step_id()), true);
+  CollectiveRemoteAccess* rma = ce_handle.get()->remote_access();
+  rma->buf_rendezvous()->ConsumeBuf(
+      request->buf_rendezvous_key(),
+      [this, request, response, done](const Status& status,
+                                      BufRendezvous::Hook* hook) {
+        Status s = status;
+        if (s.ok()) {
+          if (!DMAHelper::CanUseDMA(hook->prod_value)) {
+            s = errors::Internal("Tensor value for key ",
+                                 request->buf_rendezvous_key(),
+                                 " is not of a type supported by RecvBuf");
+          }
+        }
+        if (s.ok()) {
+          remote_memory_manager_->TransportOptionsFromTensor(
+              response->mutable_transport_options(), *hook->prod_value,
+              hook->prod_dev, hook->prod_ctx, hook->prod_attr.on_host(),
+              [this, response, done, hook](const Status& s) {
+                response->set_send_start_micros(env_->env->NowMicros());
+                done(s);
+                BufRendezvous::DoneWithHook(hook);
+              });
+        }
+      });
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/contrib/gdr/gdr_worker.h b/tensorflow/contrib/gdr/gdr_worker.h
index 39f11e6bde..7af03d16fc 100644
--- a/tensorflow/contrib/gdr/gdr_worker.h
+++ b/tensorflow/contrib/gdr/gdr_worker.h
@@ -38,6 +38,10 @@ class GdrWorker : public GrpcWorker {
                                    ::grpc::ByteBuffer* response,
                                    StatusCallback done) override;
 
+  virtual void RecvBufAsync(CallOptions* opts, const RecvBufRequest* request,
+                            RecvBufResponse* response,
+                            StatusCallback done) override;
+
  private:
   RemoteMemoryManager* remote_memory_manager_;  // Not owned
   RecentRequestIds recv_tensor_recent_request_ids_;
diff --git a/tensorflow/core/distributed_runtime/rpc_collective_executor_mgr.h b/tensorflow/core/distributed_runtime/rpc_collective_executor_mgr.h
index c9581fa00f..98eb146770 100644
--- a/tensorflow/core/distributed_runtime/rpc_collective_executor_mgr.h
+++ b/tensorflow/core/distributed_runtime/rpc_collective_executor_mgr.h
@@ -56,7 +56,7 @@ class RpcCollectiveExecutorMgr : public CollectiveExecutorMgr {
   void RetireStepId(int64 graph_key, int64 step_id) override;
 
  protected:
-  CollectiveExecutor* Create(int64 step_id) override;
+  virtual CollectiveExecutor* Create(int64 step_id) override;
 
   WorkerCacheInterface* const worker_cache_;  // Not owned.
   const string task_name_;
-- 
GitLab


From 175962fc170ba3f3dc0fc85261500361ffeeca77 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Fri, 11 Jan 2019 15:54:46 -0800
Subject: [PATCH 0597/2345] Debug disk usage of parameterized build.

PiperOrigin-RevId: 228960346
---
 tensorflow/tools/ci_build/ci_parameterized_build.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/ci_build/ci_parameterized_build.sh b/tensorflow/tools/ci_build/ci_parameterized_build.sh
index 0631806dc5..62c1e014d5 100755
--- a/tensorflow/tools/ci_build/ci_parameterized_build.sh
+++ b/tensorflow/tools/ci_build/ci_parameterized_build.sh
@@ -614,11 +614,11 @@ if [[ "${DO_DOCKER}" == "1" ]]; then
 fi
 
 # Set a disk usage trap.
-function finish {
+function debug_disk_usage {
     echo "Finished script... disk usage report in ${TMP_DIR}"
     du -k -d 2 ${TMP_DIR} | sort -n -r
 }
-trap finish EXIT
+# trap debug_disk_usage EXIT
 
 chmod +x ${TMP_SCRIPT}
 
@@ -655,6 +655,8 @@ echo ""
 echo "Parameterized build ends with ${RESULT} at: $(date) "\
 "(Elapsed time: $((END_TIME - START_TIME)) s)"
 
+# Dump disk usage
+debug_disk_usage
 
 # Clean up temporary directory if it exists
 if [[ ! -z "${TMP_DIR}" ]]; then
-- 
GitLab


From 3f594e1b481ee16fe3a3aa08c87dc53158250de7 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Fri, 11 Jan 2019 16:01:26 -0800
Subject: [PATCH 0598/2345] [tf.data] Replacing usage of
 `std::unique_ptr<Foo>(new Foo(...))` with `absl::make_unique<Foo>(...)` when
 possible and wrapping it `absl::WrapUnique<Foo>` otherwise.

PiperOrigin-RevId: 228961343
---
 .../core/kernels/data/batch_dataset_op.cc     |  4 +-
 .../core/kernels/data/cache_dataset_ops.cc    | 35 ++++++-----
 .../core/kernels/data/captured_function.cc    | 15 ++---
 .../kernels/data/concatenate_dataset_op.cc    |  4 +-
 .../experimental/assert_next_dataset_op.cc    |  4 +-
 .../data/experimental/csv_dataset_op.cc       |  6 +-
 .../dense_to_sparse_batch_dataset_op.cc       |  4 +-
 .../directed_interleave_dataset_op.cc         |  4 +-
 .../group_by_reducer_dataset_op.cc            |  4 +-
 .../group_by_window_dataset_op.cc             |  4 +-
 .../experimental/ignore_errors_dataset_op.cc  |  4 +-
 .../data/experimental/indexed_dataset_op.cc   |  7 +--
 .../data/experimental/lmdb_dataset_op.cc      |  5 +-
 .../experimental/map_and_batch_dataset_op.cc  |  3 +-
 .../experimental/matching_files_dataset_op.cc |  4 +-
 .../non_serializable_dataset_op.cc            |  4 +-
 .../numa_map_and_batch_dataset_op.cc          |  7 +--
 .../parallel_interleave_dataset_op.cc         |  5 +-
 .../experimental/parse_example_dataset_op.cc  |  4 +-
 .../data/experimental/random_dataset_op.cc    |  4 +-
 .../data/experimental/scan_dataset_op.cc      |  4 +-
 .../set_stats_aggregator_dataset_op.cc        |  5 +-
 .../data/experimental/sleep_dataset_op.cc     |  3 +-
 .../experimental/sliding_window_dataset_op.cc |  5 +-
 .../data/experimental/sql_dataset_op.cc       |  4 +-
 .../data/experimental/stats_aggregator_ops.cc |  4 +-
 .../data/experimental/stats_dataset_ops.cc    |  9 ++-
 .../experimental/threadpool_dataset_op.cc     | 17 +++--
 .../data/experimental/to_tf_record_op.cc      | 13 ++--
 .../data/experimental/unbatch_dataset_op.cc   |  4 +-
 .../data/experimental/unique_dataset_op.cc    |  4 +-
 .../data/filter_by_component_dataset_op.cc    |  4 +-
 .../core/kernels/data/filter_dataset_op.cc    |  3 +-
 .../core/kernels/data/flat_map_dataset_op.cc  |  4 +-
 .../core/kernels/data/generator_dataset_op.cc |  8 +--
 .../kernels/data/interleave_dataset_op.cc     |  4 +-
 tensorflow/core/kernels/data/iterator_ops.cc  | 27 ++++----
 .../core/kernels/data/map_dataset_op.cc       |  3 +-
 tensorflow/core/kernels/data/map_defun_op.cc  |  2 +-
 .../kernels/data/multi_device_iterator_ops.cc | 21 +++----
 tensorflow/core/kernels/data/optional_ops.h   |  7 ++-
 .../kernels/data/padded_batch_dataset_op.cc   |  4 +-
 .../data/parallel_interleave_dataset_op.cc    | 14 ++---
 .../kernels/data/parallel_map_dataset_op.cc   |  5 +-
 .../kernels/data/parallel_map_iterator.cc     |  3 +-
 .../core/kernels/data/prefetch_dataset_op.cc  |  9 +--
 .../core/kernels/data/range_dataset_op.cc     |  4 +-
 .../core/kernels/data/reader_dataset_ops.cc   | 63 ++++++++++---------
 .../core/kernels/data/repeat_dataset_op.cc    | 12 ++--
 .../core/kernels/data/shuffle_dataset_op.cc   | 35 ++++++-----
 .../kernels/data/single_threaded_executor.cc  |  4 +-
 .../data/single_threaded_executor_test.cc     |  8 +--
 .../core/kernels/data/skip_dataset_op.cc      |  8 +--
 .../data/sparse_tensor_slice_dataset_op.cc    |  4 +-
 .../core/kernels/data/take_dataset_op.cc      |  8 +--
 .../core/kernels/data/tensor_dataset_op.cc    |  4 +-
 .../kernels/data/tensor_slice_dataset_op.cc   |  4 +-
 .../core/kernels/data/window_dataset.cc       |  4 +-
 .../core/kernels/data/window_dataset_op.cc    |  4 +-
 .../core/kernels/data/zip_dataset_op.cc       |  4 +-
 60 files changed, 246 insertions(+), 241 deletions(-)

diff --git a/tensorflow/core/kernels/data/batch_dataset_op.cc b/tensorflow/core/kernels/data/batch_dataset_op.cc
index 1f8d2bdbae..f9ce0d9642 100644
--- a/tensorflow/core/kernels/data/batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/batch_dataset_op.cc
@@ -79,8 +79,8 @@ class BatchDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(new Iterator(
-          Iterator::Params{this, strings::StrCat(prefix, "::Batch")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::Batch")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/cache_dataset_ops.cc b/tensorflow/core/kernels/data/cache_dataset_ops.cc
index 535f49cff8..343157de6f 100644
--- a/tensorflow/core/kernels/data/cache_dataset_ops.cc
+++ b/tensorflow/core/kernels/data/cache_dataset_ops.cc
@@ -69,8 +69,8 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new FileIterator({this, strings::StrCat(prefix, "::FileCache")}));
+      return absl::make_unique<FileIterator>(
+          FileIterator::Params{this, strings::StrCat(prefix, "::FileCache")});
     }
 
     const DataTypeVector& output_dtypes() const override {
@@ -325,7 +325,7 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
           }
           filename_ = strings::StrCat(dataset()->filename_, "_", shard_id_);
           lockfile_ = strings::StrCat(filename_, ".lockfile");
-          writer_.reset(new BundleWriter(dataset()->env_, filename_));
+          writer_ = absl::make_unique<BundleWriter>(dataset()->env_, filename_);
           return Status::OK();
         }
 
@@ -385,7 +385,8 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
             // conditions are not met since BundleWriter's constructor creates
             // new temp files which can delete the temp files created by a
             // BundleWriter in another Session.
-            writer_.reset(new BundleWriter(dataset()->env_, filename_));
+            writer_ =
+                absl::make_unique<BundleWriter>(dataset()->env_, filename_);
             lockfile_created_ = true;
             return Status::OK();
           }
@@ -537,12 +538,14 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
         // `FileReaderIterator` and seek to the `cur_index`.
         switch (mode_) {
           case Mode::read:
-            iterator_.reset(new FileReaderIterator(
-                {dataset(), strings::StrCat(prefix(), "Impl")}));
+            iterator_ = absl::make_unique<FileReaderIterator>(
+                FileReaderIterator::Params{dataset(),
+                                           strings::StrCat(prefix(), "Impl")});
             break;
           case Mode::write:
-            iterator_.reset(new FileWriterIterator(
-                {dataset(), strings::StrCat(prefix(), "Impl")}));
+            iterator_ = absl::make_unique<FileWriterIterator>(
+                FileWriterIterator::Params{dataset(),
+                                           strings::StrCat(prefix(), "Impl")});
         }
       }
 
@@ -573,8 +576,8 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new MemoryIterator({this, strings::StrCat(prefix, "::MemoryCache")}));
+      return absl::make_unique<MemoryIterator>(MemoryIterator::Params{
+          this, strings::StrCat(prefix, "::MemoryCache")});
     }
 
     const DataTypeVector& output_dtypes() const override {
@@ -933,12 +936,16 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
       void InitializeIterator() EXCLUSIVE_LOCKS_REQUIRED(mu_) {
         switch (mode_) {
           case Mode::read:
-            iterator_.reset(new MemoryReaderIterator(
-                {dataset(), strings::StrCat(prefix(), "Impl")}, cache_));
+            iterator_ = absl::make_unique<MemoryReaderIterator>(
+                MemoryReaderIterator::Params{dataset(),
+                                             strings::StrCat(prefix(), "Impl")},
+                cache_);
             break;
           case Mode::write:
-            iterator_.reset(new MemoryWriterIterator(
-                {dataset(), strings::StrCat(prefix(), "Impl")}, cache_));
+            iterator_ = absl::make_unique<MemoryWriterIterator>(
+                MemoryWriterIterator::Params{dataset(),
+                                             strings::StrCat(prefix(), "Impl")},
+                cache_);
         }
       }
 
diff --git a/tensorflow/core/kernels/data/captured_function.cc b/tensorflow/core/kernels/data/captured_function.cc
index 973b6b0604..99b745b4c4 100644
--- a/tensorflow/core/kernels/data/captured_function.cc
+++ b/tensorflow/core/kernels/data/captured_function.cc
@@ -25,7 +25,6 @@ limitations under the License.
 #include "tensorflow/core/lib/random/random.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/notification.h"
-#include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
 namespace data {
@@ -114,8 +113,8 @@ Status CapturedFunction::Create(
   OpInputList inputs;
   TF_RETURN_IF_ERROR(ctx->input_list(argument, &inputs));
   std::vector<Tensor> arguments(inputs.begin(), inputs.end());
-  *out_function = WrapUnique(new CapturedFunction(func, std::move(arguments),
-                                                  use_inter_op_parallelism));
+  *out_function = absl::WrapUnique(new CapturedFunction(
+      func, std::move(arguments), use_inter_op_parallelism));
   return Status::OK();
 }
 
@@ -144,8 +143,10 @@ Status CapturedFunction::Instantiate(
     ret_types.push_back(ret_type);
   }
 
-  instantiated_captured_function->reset(new InstantiatedCapturedFunction(
-      lib, f_handle, std::move(ret_types), *ctx->runner(), this));
+  *instantiated_captured_function =
+      absl::WrapUnique<InstantiatedCapturedFunction>(
+          new InstantiatedCapturedFunction(lib, f_handle, std::move(ret_types),
+                                           *ctx->runner(), this));
   return Status::OK();
 }
 
@@ -422,11 +423,11 @@ void InstantiatedCapturedFunction::RunAsync(
   // (such as queue kernels) that depend on the non-nullness of
   // `OpKernelContext::cancellation_manager()`, but additional effort
   // will be required to plumb it through the `IteratorContext`.
-  CancellationManager* c_mgr = new CancellationManager;
+  CancellationManager* c_mgr = new CancellationManager();
   f_opts.cancellation_manager = c_mgr;
   std::shared_ptr<SimpleStepStatsCollector> stats_collector;
   if (ctx->model() || ctx->stats_aggregator()) {
-    stats_collector = MakeUnique<SimpleStepStatsCollector>();
+    stats_collector = absl::make_unique<SimpleStepStatsCollector>();
   }
   f_opts.stats_collector = stats_collector.get();
 
diff --git a/tensorflow/core/kernels/data/concatenate_dataset_op.cc b/tensorflow/core/kernels/data/concatenate_dataset_op.cc
index 066b2c9aef..1d7c3a65d5 100644
--- a/tensorflow/core/kernels/data/concatenate_dataset_op.cc
+++ b/tensorflow/core/kernels/data/concatenate_dataset_op.cc
@@ -63,8 +63,8 @@ class ConcatenateDatasetOp : public BinaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::Concatenate")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::Concatenate")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/assert_next_dataset_op.cc b/tensorflow/core/kernels/data/experimental/assert_next_dataset_op.cc
index 3e87f484b9..eb54713360 100644
--- a/tensorflow/core/kernels/data/experimental/assert_next_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/assert_next_dataset_op.cc
@@ -61,8 +61,8 @@ class AssertNextDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::Assert")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::AssertNext")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/csv_dataset_op.cc b/tensorflow/core/kernels/data/experimental/csv_dataset_op.cc
index f6f58fc430..4435c2a131 100644
--- a/tensorflow/core/kernels/data/experimental/csv_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/csv_dataset_op.cc
@@ -12,8 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-
-// See docs in ../ops/parsing_ops.cc.
 #include "tensorflow/core/framework/common_shape_fns.h"
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/op.h"
@@ -159,8 +157,8 @@ class CSVDatasetOp : public DatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::CSV")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::CSV")});
     }
 
     const DataTypeVector& output_dtypes() const override { return out_type_; }
diff --git a/tensorflow/core/kernels/data/experimental/dense_to_sparse_batch_dataset_op.cc b/tensorflow/core/kernels/data/experimental/dense_to_sparse_batch_dataset_op.cc
index 97e64dd744..31f081a727 100644
--- a/tensorflow/core/kernels/data/experimental/dense_to_sparse_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/dense_to_sparse_batch_dataset_op.cc
@@ -96,8 +96,8 @@ class DenseToSparseBatchDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(new Iterator(
-          {this, strings::StrCat(prefix, "::DenseToSparseBatch")}));
+      return absl::make_unique<Iterator>(typename Iterator::Params{
+          this, strings::StrCat(prefix, "::DenseToSparseBatch")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/directed_interleave_dataset_op.cc b/tensorflow/core/kernels/data/experimental/directed_interleave_dataset_op.cc
index d8bb696167..f55718a006 100644
--- a/tensorflow/core/kernels/data/experimental/directed_interleave_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/directed_interleave_dataset_op.cc
@@ -93,8 +93,8 @@ class DirectedInterleaveDatasetOp : public DatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(new Iterator(
-          {this, strings::StrCat(prefix, "::DirectedInterleave")}));
+      return absl::make_unique<Iterator>(Iterator::Params{
+          this, strings::StrCat(prefix, "::DirectedInterleave")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/group_by_reducer_dataset_op.cc b/tensorflow/core/kernels/data/experimental/group_by_reducer_dataset_op.cc
index 5f0c01be4b..56159593a9 100644
--- a/tensorflow/core/kernels/data/experimental/group_by_reducer_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/group_by_reducer_dataset_op.cc
@@ -90,8 +90,8 @@ class GroupByReducerDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::GroupByReducer")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::GroupByReducer")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/group_by_window_dataset_op.cc b/tensorflow/core/kernels/data/experimental/group_by_window_dataset_op.cc
index 11491e00db..49122807b2 100644
--- a/tensorflow/core/kernels/data/experimental/group_by_window_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/group_by_window_dataset_op.cc
@@ -89,8 +89,8 @@ class GroupByWindowDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::GroupByWindow")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::GroupByWindow")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/ignore_errors_dataset_op.cc b/tensorflow/core/kernels/data/experimental/ignore_errors_dataset_op.cc
index d445d9c809..5e07bdb32e 100644
--- a/tensorflow/core/kernels/data/experimental/ignore_errors_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/ignore_errors_dataset_op.cc
@@ -45,8 +45,8 @@ class IgnoreErrorsDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::IgnoreErrors")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::IgnoreErrors")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/indexed_dataset_op.cc b/tensorflow/core/kernels/data/experimental/indexed_dataset_op.cc
index 83eeed7892..e75e6e4b80 100644
--- a/tensorflow/core/kernels/data/experimental/indexed_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/indexed_dataset_op.cc
@@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/resource_mgr.h"
@@ -424,7 +423,7 @@ class IdentityIndexedDatasetOp : public IndexedDatasetOpKernel {
 
     Status MaterializeDataset(
         std::shared_ptr<MaterializedIndexedDataset>* materialized) override {
-      materialized->reset(new Materialized(this));
+      (*materialized) = std::make_shared<Materialized>(this);
       return Status::OK();
     }
 
@@ -441,8 +440,8 @@ class IdentityIndexedDatasetOp : public IndexedDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(new Iterator(
-          {this, strings::StrCat(prefix, "::IdentityIndexedDataset")}));
+      return absl::make_unique<Iterator>(Iterator::Params{
+          this, strings::StrCat(prefix, "::IdentityIndexedDataset")});
     }
 
     string DebugString() const override {
diff --git a/tensorflow/core/kernels/data/experimental/lmdb_dataset_op.cc b/tensorflow/core/kernels/data/experimental/lmdb_dataset_op.cc
index 6248eb775e..cf900f1336 100644
--- a/tensorflow/core/kernels/data/experimental/lmdb_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/lmdb_dataset_op.cc
@@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-
 #include <sys/stat.h>
 
 #include "tensorflow/core/framework/dataset.h"
@@ -52,8 +51,8 @@ class LMDBDatasetOp : public DatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::LMDB")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::LMDB")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc b/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
index a6348e4647..5d8e7cfabe 100644
--- a/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
@@ -32,7 +32,6 @@ limitations under the License.
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/cpu_info.h"
 #include "tensorflow/core/platform/tracing.h"
-#include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
 namespace data {
@@ -163,7 +162,7 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return MakeUnique<Iterator>(
+      return absl::make_unique<Iterator>(
           Iterator::Params{this, strings::StrCat(prefix, "::MapAndBatch")},
           map_func_);
     }
diff --git a/tensorflow/core/kernels/data/experimental/matching_files_dataset_op.cc b/tensorflow/core/kernels/data/experimental/matching_files_dataset_op.cc
index aa27a13416..381b9691d1 100644
--- a/tensorflow/core/kernels/data/experimental/matching_files_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/matching_files_dataset_op.cc
@@ -61,8 +61,8 @@ class MatchingFilesDatasetOp : public DatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::MatchingFiles")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::MatchingFiles")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/non_serializable_dataset_op.cc b/tensorflow/core/kernels/data/experimental/non_serializable_dataset_op.cc
index 61811ea14e..9ca8e33b94 100644
--- a/tensorflow/core/kernels/data/experimental/non_serializable_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/non_serializable_dataset_op.cc
@@ -53,8 +53,8 @@ class NonSerializableDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::NonSerializable")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::NonSerializable")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/numa_map_and_batch_dataset_op.cc b/tensorflow/core/kernels/data/experimental/numa_map_and_batch_dataset_op.cc
index 2b1aec358c..643b6460e8 100644
--- a/tensorflow/core/kernels/data/experimental/numa_map_and_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/numa_map_and_batch_dataset_op.cc
@@ -32,7 +32,6 @@ limitations under the License.
 #include "tensorflow/core/platform/cpu_info.h"
 #include "tensorflow/core/platform/numa.h"
 #include "tensorflow/core/platform/tracing.h"
-#include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
 namespace data {
@@ -121,8 +120,8 @@ class NumaMapAndBatchDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::NumaMapAndBatch")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::NumaMapAndBatch")});
     }
 
     const DataTypeVector& output_dtypes() const override {
@@ -321,7 +320,7 @@ class NumaMapAndBatchDatasetOp : public UnaryDatasetOpKernel {
         }
         workers_.resize(num_workers);
         for (size_t i = 0; i < num_workers; ++i) {
-          workers_[i] = MakeUnique<NumaWorkerBlock>(this);
+          workers_[i] = absl::make_unique<NumaWorkerBlock>(this);
           TF_RETURN_IF_ERROR(
               workers_[i]->manager.Restore(ctx, reader, this, i));
         }
diff --git a/tensorflow/core/kernels/data/experimental/parallel_interleave_dataset_op.cc b/tensorflow/core/kernels/data/experimental/parallel_interleave_dataset_op.cc
index 1c19119d88..f6d522078d 100644
--- a/tensorflow/core/kernels/data/experimental/parallel_interleave_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/parallel_interleave_dataset_op.cc
@@ -26,7 +26,6 @@ limitations under the License.
 #include "tensorflow/core/lib/core/threadpool.h"
 #include "tensorflow/core/lib/gtl/cleanup.h"
 #include "tensorflow/core/lib/random/random.h"
-#include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
 namespace data {
@@ -113,8 +112,8 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(new Iterator(
-          {this, strings::StrCat(prefix, "::ParallelInterleave")}));
+      return absl::make_unique<Iterator>(Iterator::Params{
+          this, strings::StrCat(prefix, "::ParallelInterleave")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/parse_example_dataset_op.cc b/tensorflow/core/kernels/data/experimental/parse_example_dataset_op.cc
index ea99a8b32c..0057405734 100644
--- a/tensorflow/core/kernels/data/experimental/parse_example_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/parse_example_dataset_op.cc
@@ -183,8 +183,8 @@ class ParseExampleDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      std::unique_ptr<ParallelMapFunctor> parse_example_functor(
-          new ParseExampleFunctor(this));
+      std::unique_ptr<ParallelMapFunctor> parse_example_functor =
+          absl::make_unique<ParseExampleFunctor>(this);
       return NewParallelMapIterator(
           {this, strings::StrCat(prefix, "::ParseExample")}, input_,
           std::move(parse_example_functor), num_parallel_calls_, sloppy_,
diff --git a/tensorflow/core/kernels/data/experimental/random_dataset_op.cc b/tensorflow/core/kernels/data/experimental/random_dataset_op.cc
index 6d85cd5c45..114bb6a856 100644
--- a/tensorflow/core/kernels/data/experimental/random_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/random_dataset_op.cc
@@ -56,8 +56,8 @@ class RandomDatasetOp : public DatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::Random")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::Random")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc b/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc
index 87263b5606..55e22c1cac 100644
--- a/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc
@@ -84,8 +84,8 @@ class ScanDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::Scan")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::Scan")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/set_stats_aggregator_dataset_op.cc b/tensorflow/core/kernels/data/experimental/set_stats_aggregator_dataset_op.cc
index fe128005fa..67bb1e160b 100644
--- a/tensorflow/core/kernels/data/experimental/set_stats_aggregator_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/set_stats_aggregator_dataset_op.cc
@@ -13,6 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 #include <memory>
+
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/stats_aggregator.h"
@@ -114,8 +115,8 @@ class SetStatsAggregatorDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(new Iterator(
-          {this, strings::StrCat(prefix, "::SetStatsAggregator")}));
+      return absl::make_unique<Iterator>(Iterator::Params{
+          this, strings::StrCat(prefix, "::SetStatsAggregator")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/sleep_dataset_op.cc b/tensorflow/core/kernels/data/experimental/sleep_dataset_op.cc
index d2fb8ac4f3..9d63690622 100644
--- a/tensorflow/core/kernels/data/experimental/sleep_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/sleep_dataset_op.cc
@@ -15,7 +15,6 @@ limitations under the License.
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
 namespace data {
@@ -55,7 +54,7 @@ class SleepDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return MakeUnique<Iterator>(
+      return absl::make_unique<Iterator>(
           Iterator::Params{this, strings::StrCat(prefix, "::Sleep")});
     }
 
diff --git a/tensorflow/core/kernels/data/experimental/sliding_window_dataset_op.cc b/tensorflow/core/kernels/data/experimental/sliding_window_dataset_op.cc
index 1ce4fbd313..c5851eaf86 100644
--- a/tensorflow/core/kernels/data/experimental/sliding_window_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/sliding_window_dataset_op.cc
@@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-
 #include <deque>
 #include <vector>
 
@@ -86,8 +85,8 @@ class SlidingWindowDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(new Iterator(
-          Iterator::Params{this, strings::StrCat(prefix, "::Slide")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::Slide")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/sql_dataset_op.cc b/tensorflow/core/kernels/data/experimental/sql_dataset_op.cc
index c16d8ed02c..84f6fba36d 100644
--- a/tensorflow/core/kernels/data/experimental/sql_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/sql_dataset_op.cc
@@ -91,8 +91,8 @@ class SqlDatasetOp : public DatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::Sql")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::Sql")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/stats_aggregator_ops.cc b/tensorflow/core/kernels/data/experimental/stats_aggregator_ops.cc
index 894465e181..1d1b788b6c 100644
--- a/tensorflow/core/kernels/data/experimental/stats_aggregator_ops.cc
+++ b/tensorflow/core/kernels/data/experimental/stats_aggregator_ops.cc
@@ -108,8 +108,8 @@ class StatsAggregatorHandleOp
  private:
   Status CreateResource(StatsAggregatorResource** ret) override
       EXCLUSIVE_LOCKS_REQUIRED(mu_) {
-    *ret = new StatsAggregatorResource(
-        std::unique_ptr<StatsAggregator>(new StatsAggregatorImpl));
+    *ret =
+        new StatsAggregatorResource(absl::make_unique<StatsAggregatorImpl>());
     return Status::OK();
   }
 
diff --git a/tensorflow/core/kernels/data/experimental/stats_dataset_ops.cc b/tensorflow/core/kernels/data/experimental/stats_dataset_ops.cc
index 1961f25df8..bf96be4eb0 100644
--- a/tensorflow/core/kernels/data/experimental/stats_dataset_ops.cc
+++ b/tensorflow/core/kernels/data/experimental/stats_dataset_ops.cc
@@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-
 #include "tensorflow/core/example/example.pb.h"
 #include "tensorflow/core/example/feature.pb.h"
 #include "tensorflow/core/framework/dataset.h"
@@ -63,8 +62,8 @@ class LatencyStatsDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::LatencyStats")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::LatencyStats")});
     }
 
     const DataTypeVector& output_dtypes() const override {
@@ -173,8 +172,8 @@ class BytesProducedStatsDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(new Iterator(
-          {this, strings::StrCat(prefix, "::BytesProducedStats")}));
+      return absl::make_unique<Iterator>(Iterator::Params{
+          this, strings::StrCat(prefix, "::BytesProducedStats")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc b/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
index fab3cab7da..7a16cda0f3 100644
--- a/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
@@ -12,13 +12,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-
 #include <memory>
+
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/lib/core/threadpool.h"
-#include "tensorflow/core/util/ptr_util.h"
 #include "tensorflow/core/util/work_sharder.h"
 
 namespace tensorflow {
@@ -154,8 +153,8 @@ class ThreadPoolDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::ThreadPool")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::ThreadPool")});
     }
 
     const DataTypeVector& output_dtypes() const override {
@@ -261,8 +260,8 @@ class MaxIntraOpParallelismDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(new Iterator(
-          {this, strings::StrCat(prefix, "::MaxIntraOpParallelism")}));
+      return absl::make_unique<Iterator>(Iterator::Params{
+          this, strings::StrCat(prefix, "::MaxIntraOpParallelism")});
     }
 
     const DataTypeVector& output_dtypes() const override {
@@ -362,7 +361,7 @@ class PrivateThreadPoolDatasetOp : public UnaryDatasetOpKernel {
         : DatasetBase(DatasetContext(ctx)),
           input_(input),
           num_threads_(num_threads) {
-      thread_pool_ = MakeUnique<thread::ThreadPool>(
+      thread_pool_ = absl::make_unique<thread::ThreadPool>(
           ctx->env(), ThreadOptions{}, "data_private_threadpool", num_threads,
           /*low_latency_hint=*/false);
       input_->Ref();
@@ -372,8 +371,8 @@ class PrivateThreadPoolDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::PrivateThreadPool")}));
+      return absl::make_unique<Iterator>(Iterator::Params{
+          this, strings::StrCat(prefix, "::PrivateThreadPool")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/to_tf_record_op.cc b/tensorflow/core/kernels/data/experimental/to_tf_record_op.cc
index 7728baf150..6cf6198432 100644
--- a/tensorflow/core/kernels/data/experimental/to_tf_record_op.cc
+++ b/tensorflow/core/kernels/data/experimental/to_tf_record_op.cc
@@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/function_handle_cache.h"
 #include "tensorflow/core/framework/op_kernel.h"
@@ -59,18 +58,18 @@ class ToTFRecordOp : public AsyncOpKernel {
       std::unique_ptr<WritableFile> file;
       OP_REQUIRES_OK_ASYNC(ctx, ctx->env()->NewWritableFile(filename, &file),
                            done);
-      std::unique_ptr<io::RecordWriter> writer;
-      writer.reset(new io::RecordWriter(
-          file.get(), io::RecordWriterOptions::CreateRecordWriterOptions(
-                          compression_type)));
+      std::unique_ptr<io::RecordWriter> writer =
+          absl::make_unique<io::RecordWriter>(
+              file.get(), io::RecordWriterOptions::CreateRecordWriterOptions(
+                              compression_type));
 
       DatasetBase* dataset;
       OP_REQUIRES_OK_ASYNC(
           ctx, GetDatasetFromVariantTensor(ctx->input(0), &dataset), done);
       std::unique_ptr<IteratorBase> iterator;
       IteratorContext::Params params(ctx);
-      std::unique_ptr<FunctionHandleCache> function_handle_cache(
-          new FunctionHandleCache(params.lib));
+      std::unique_ptr<FunctionHandleCache> function_handle_cache =
+          absl::make_unique<FunctionHandleCache>(params.lib);
       params.function_handle_cache = function_handle_cache.get();
       IteratorContext iter_ctx(std::move(params));
 
diff --git a/tensorflow/core/kernels/data/experimental/unbatch_dataset_op.cc b/tensorflow/core/kernels/data/experimental/unbatch_dataset_op.cc
index 2626ec3ed7..cb26fd3e43 100644
--- a/tensorflow/core/kernels/data/experimental/unbatch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/unbatch_dataset_op.cc
@@ -58,8 +58,8 @@ class UnbatchDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::Unbatch")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::Unbatch")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/unique_dataset_op.cc b/tensorflow/core/kernels/data/experimental/unique_dataset_op.cc
index 23dd9ff612..57865c45fc 100644
--- a/tensorflow/core/kernels/data/experimental/unique_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/unique_dataset_op.cc
@@ -58,8 +58,8 @@ class UniqueDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::Unique")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::Unique")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/filter_by_component_dataset_op.cc b/tensorflow/core/kernels/data/filter_by_component_dataset_op.cc
index 784f987286..3b9b319ea9 100644
--- a/tensorflow/core/kernels/data/filter_by_component_dataset_op.cc
+++ b/tensorflow/core/kernels/data/filter_by_component_dataset_op.cc
@@ -64,8 +64,8 @@ class FilterByLastComponentDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<Iterator>(new Iterator(
-          {this, strings::StrCat(prefix, "::FilterByLastComponent")}));
+      return absl::make_unique<Iterator>(Iterator::Params{
+          this, strings::StrCat(prefix, "::FilterByLastComponent")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/filter_dataset_op.cc b/tensorflow/core/kernels/data/filter_dataset_op.cc
index 30b2fc5db8..483d42c809 100644
--- a/tensorflow/core/kernels/data/filter_dataset_op.cc
+++ b/tensorflow/core/kernels/data/filter_dataset_op.cc
@@ -22,7 +22,6 @@ limitations under the License.
 #include "tensorflow/core/lib/gtl/cleanup.h"
 #include "tensorflow/core/lib/random/random.h"
 #include "tensorflow/core/lib/strings/str_util.h"
-#include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
 namespace data {
@@ -109,7 +108,7 @@ class FilterDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return MakeUnique<Iterator>(
+      return absl::make_unique<Iterator>(
           Iterator::Params{this, strings::StrCat(prefix, "::Filter")},
           filter_pred_);
     }
diff --git a/tensorflow/core/kernels/data/flat_map_dataset_op.cc b/tensorflow/core/kernels/data/flat_map_dataset_op.cc
index efa76ab34b..3f01ac5569 100644
--- a/tensorflow/core/kernels/data/flat_map_dataset_op.cc
+++ b/tensorflow/core/kernels/data/flat_map_dataset_op.cc
@@ -67,8 +67,8 @@ class FlatMapDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::FlatMap")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::FlatMap")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/generator_dataset_op.cc b/tensorflow/core/kernels/data/generator_dataset_op.cc
index 48697ec6c8..5dff2be39d 100644
--- a/tensorflow/core/kernels/data/generator_dataset_op.cc
+++ b/tensorflow/core/kernels/data/generator_dataset_op.cc
@@ -12,11 +12,11 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
+#include "tensorflow/core/kernels/data/generator_dataset_op.h"
+
 #include <iterator>
 #include <vector>
 
-#include "tensorflow/core/kernels/data/generator_dataset_op.h"
-
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/kernels/data/captured_function.h"
@@ -44,8 +44,8 @@ class GeneratorDatasetOp::Dataset : public DatasetBase {
 
   std::unique_ptr<IteratorBase> MakeIteratorInternal(
       const string& prefix) const override {
-    return std::unique_ptr<IteratorBase>(
-        new Iterator({this, strings::StrCat(prefix, "::Generator")}));
+    return absl::make_unique<Iterator>(
+        Iterator::Params{this, strings::StrCat(prefix, "::Generator")});
   }
 
   const DataTypeVector& output_dtypes() const override { return output_types_; }
diff --git a/tensorflow/core/kernels/data/interleave_dataset_op.cc b/tensorflow/core/kernels/data/interleave_dataset_op.cc
index 1a5e6edb5b..69310bcff2 100644
--- a/tensorflow/core/kernels/data/interleave_dataset_op.cc
+++ b/tensorflow/core/kernels/data/interleave_dataset_op.cc
@@ -89,8 +89,8 @@ class InterleaveDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::Interleave")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::Interleave")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/iterator_ops.cc b/tensorflow/core/kernels/data/iterator_ops.cc
index 81e26d35c0..a8c3f7f77f 100644
--- a/tensorflow/core/kernels/data/iterator_ops.cc
+++ b/tensorflow/core/kernels/data/iterator_ops.cc
@@ -38,7 +38,6 @@ limitations under the License.
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/public/session_options.h"
-#include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
 namespace data {
@@ -137,7 +136,7 @@ class IteratorResource : public ResourceBase {
     std::unique_ptr<ProcessFunctionLibraryRuntime> pflr(nullptr);
     TF_RETURN_IF_ERROR(ctx->function_library()->Clone(&flib_def, &pflr, &lib));
     TF_RETURN_IF_ERROR(flib_def->AddLibrary(graph_def.library()));
-    std::unique_ptr<State> new_state = MakeUnique<State>(
+    std::unique_ptr<State> new_state = absl::make_unique<State>(
         std::move(flib_def), std::move(pflr), lib, nullptr /* iterator */);
 
     TF_RETURN_IF_ERROR(
@@ -211,7 +210,7 @@ class IteratorResource : public ResourceBase {
     }
 
     new_state->function_handle_cache =
-        MakeUnique<FunctionHandleCache>(new_state->lib);
+        absl::make_unique<FunctionHandleCache>(new_state->lib);
     // Create new iterator.
     std::unique_ptr<IteratorBase> iterator;
     IteratorContext::Params params(ctx);
@@ -247,7 +246,7 @@ class IteratorResource : public ResourceBase {
         : flib_def(flib_def),
           pflr(pflr),
           lib(lib),
-          function_handle_cache(MakeUnique<FunctionHandleCache>(lib)),
+          function_handle_cache(absl::make_unique<FunctionHandleCache>(lib)),
           iterator(std::move(iterator)) {}
 
     State(std::shared_ptr<FunctionLibraryDefinition> flib_def,
@@ -434,7 +433,7 @@ class IteratorStateVariant {
     SerializationContext::Params params;
     params.flib_def = ctx->function_library()->GetFunctionLibraryDefinition();
     SerializationContext serialization_ctx(params);
-    data_ = MakeUnique<VariantTensorData>();
+    data_ = absl::make_unique<VariantTensorData>();
     data_->set_type_name(TypeName());
     VariantTensorDataWriter writer(data_.get());
     TF_RETURN_IF_ERROR(iterator_resource->Save(&serialization_ctx, &writer));
@@ -448,10 +447,10 @@ class IteratorStateVariant {
       return false;
     }
     std::unique_ptr<VariantTensorData> tensor_data =
-        MakeUnique<VariantTensorData>();
+        absl::make_unique<VariantTensorData>();
     std::swap(*tensor_data, data);
     std::unique_ptr<VariantTensorDataReader> reader =
-        MakeUnique<VariantTensorDataReader>(tensor_data.get());
+        absl::make_unique<VariantTensorDataReader>(tensor_data.get());
     status_ = reader->status();
     if (!status_.ok()) {
       return false;
@@ -585,9 +584,9 @@ FunctionLibraryRuntime* IteratorHandleOp::CreatePrivateFLR(
   *device_mgr = absl::make_unique<DeviceMgr>(RenamedDevice::NewRenamedDevice(
       ctx->device()->name(), down_cast<Device*>(ctx->device()),
       false /* owns_underlying */, false /* isolate_session_state */));
-  *flib_def = MakeUnique<FunctionLibraryDefinition>(
+  *flib_def = absl::make_unique<FunctionLibraryDefinition>(
       *ctx->function_library()->GetFunctionLibraryDefinition());
-  *pflr = MakeUnique<ProcessFunctionLibraryRuntime>(
+  *pflr = absl::make_unique<ProcessFunctionLibraryRuntime>(
       device_mgr->get(), ctx->env(), graph_def_version_, flib_def->get(),
       OptimizerOptions{} /* TODO(mrry): OptimizerOptions? */,
       nullptr /* TODO(mrry): ClusterFLR */);
@@ -679,9 +678,10 @@ class ToSingleElementOp : public AsyncOpKernel {
       std::unique_ptr<IteratorBase> iterator;
       IteratorContext::Params params(ctx);
       std::unique_ptr<FunctionHandleCache> function_handle_cache =
-          MakeUnique<FunctionHandleCache>(params.lib);
+          absl::make_unique<FunctionHandleCache>(params.lib);
       params.function_handle_cache = function_handle_cache.get();
-      std::unique_ptr<ResourceMgr> resource_mgr = MakeUnique<ResourceMgr>();
+      std::unique_ptr<ResourceMgr> resource_mgr =
+          absl::make_unique<ResourceMgr>();
       params.resource_mgr = resource_mgr.get();
       IteratorContext iter_ctx(std::move(params));
 
@@ -769,9 +769,10 @@ class ReduceDatasetOp : public AsyncOpKernel {
 
       IteratorContext::Params params(ctx);
       std::unique_ptr<FunctionHandleCache> function_handle_cache =
-          MakeUnique<FunctionHandleCache>(params.lib);
+          absl::make_unique<FunctionHandleCache>(params.lib);
       params.function_handle_cache = function_handle_cache.get();
-      std::unique_ptr<ResourceMgr> resource_mgr = MakeUnique<ResourceMgr>();
+      std::unique_ptr<ResourceMgr> resource_mgr =
+          absl::make_unique<ResourceMgr>();
       params.resource_mgr = resource_mgr.get();
       IteratorContext iter_ctx(std::move(params));
       std::unique_ptr<InstantiatedCapturedFunction> instantiated_captured_func;
diff --git a/tensorflow/core/kernels/data/map_dataset_op.cc b/tensorflow/core/kernels/data/map_dataset_op.cc
index 02c0199a0c..95f4c1c891 100644
--- a/tensorflow/core/kernels/data/map_dataset_op.cc
+++ b/tensorflow/core/kernels/data/map_dataset_op.cc
@@ -19,7 +19,6 @@ limitations under the License.
 #include "tensorflow/core/kernels/data/captured_function.h"
 #include "tensorflow/core/kernels/data/dataset_utils.h"
 #include "tensorflow/core/lib/random/random.h"
-#include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
 namespace data {
@@ -120,7 +119,7 @@ class MapDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return MakeUnique<Iterator>(
+      return absl::make_unique<Iterator>(
           Iterator::Params{this, strings::StrCat(prefix, "::Map")}, map_func_);
     }
 
diff --git a/tensorflow/core/kernels/data/map_defun_op.cc b/tensorflow/core/kernels/data/map_defun_op.cc
index 705b0393de..8122048702 100644
--- a/tensorflow/core/kernels/data/map_defun_op.cc
+++ b/tensorflow/core/kernels/data/map_defun_op.cc
@@ -92,7 +92,7 @@ class MapDefunOp : public AsyncOpKernel {
       // We use a different cancellation manager each time the function is run
       // to avoid the race condition between a function run error and other
       // functions being cancelled as a result.
-      CancellationManager* c_mgr = new CancellationManager;
+      CancellationManager* c_mgr = new CancellationManager();
       CancellationToken token = parent_mgr->get_cancellation_token();
       const bool success = parent_mgr->RegisterCallback(
           token, [c_mgr]() { c_mgr->StartCancel(); });
diff --git a/tensorflow/core/kernels/data/multi_device_iterator_ops.cc b/tensorflow/core/kernels/data/multi_device_iterator_ops.cc
index 05528e0ee0..eb08d2b28a 100644
--- a/tensorflow/core/kernels/data/multi_device_iterator_ops.cc
+++ b/tensorflow/core/kernels/data/multi_device_iterator_ops.cc
@@ -81,9 +81,8 @@ class MultiDeviceIterator : public ResourceBase {
     ++incarnation_id_;
     *incarnation_id = incarnation_id_;
 
-    multi_device_buffer_.reset(
-        new MultiDeviceBuffer(devices_.size(), max_buffer_size, incarnation_id_,
-                              std::move(iterator)));
+    multi_device_buffer_ = absl::make_unique<MultiDeviceBuffer>(
+        devices_.size(), max_buffer_size, incarnation_id_, std::move(iterator));
     return Status::OK();
   }
 
@@ -217,10 +216,11 @@ class MultiDeviceIterator : public ResourceBase {
     void EnsureBackgroundThreadStarted(IteratorContext* ctx)
         EXCLUSIVE_LOCKS_REQUIRED(mu_) {
       if (!background_thread_) {
-        background_thread_.reset(ctx->env()->StartThread(
+        auto ctx_copy = std::make_shared<IteratorContext>(*ctx);
+        background_thread_ = absl::WrapUnique<Thread>(ctx->env()->StartThread(
             {}, "tf_data_multi_device_iterator",
             std::bind(&MultiDeviceIterator::MultiDeviceBuffer::BackgroundThread,
-                      this, new IteratorContext(*ctx))));
+                      this, std::move(ctx_copy))));
       }
     }
 
@@ -258,12 +258,11 @@ class MultiDeviceIterator : public ResourceBase {
       }
     }
 
-    void BackgroundThread(IteratorContext* ctx) {
+    void BackgroundThread(std::shared_ptr<IteratorContext> ctx) {
       {
         mutex_lock l(mu_);
         background_thread_started_ = true;
       }
-      std::unique_ptr<IteratorContext> cleanup(ctx);
       int shard_to_fetch = 0;
       while (true) {
         HostBufferElement elem;
@@ -284,8 +283,8 @@ class MultiDeviceIterator : public ResourceBase {
           }
         }
 
-        elem.status =
-            host_iterator_->GetNext(ctx, &elem.value, &elem.end_of_sequence);
+        elem.status = host_iterator_->GetNext(ctx.get(), &elem.value,
+                                              &elem.end_of_sequence);
 
         if (elem.status.ok() && elem.end_of_sequence) {
           end_of_iterator = true;
@@ -397,8 +396,8 @@ class MultiDeviceIteratorHandleOp : public OpKernel {
         std::unique_ptr<ProcessFunctionLibraryRuntime> pflr(nullptr);
         OP_REQUIRES_OK(context, context->function_library()->Clone(
                                     &flib_def, &pflr, &lib));
-        std::unique_ptr<FunctionHandleCache> function_handle_cache(
-            new FunctionHandleCache(lib));
+        std::unique_ptr<FunctionHandleCache> function_handle_cache =
+            absl::make_unique<FunctionHandleCache>(lib);
         ResourceMgr* mgr = context->resource_manager();
         OP_REQUIRES_OK(context, cinfo_.Init(mgr, def()));
 
diff --git a/tensorflow/core/kernels/data/optional_ops.h b/tensorflow/core/kernels/data/optional_ops.h
index ef14e84311..7089a423d7 100644
--- a/tensorflow/core/kernels/data/optional_ops.h
+++ b/tensorflow/core/kernels/data/optional_ops.h
@@ -44,8 +44,9 @@ class OptionalVariant {
 
   // Create an `OptionalVariant` with the actual value given by the tuple of
   // tensors in `values`.
-  explicit OptionalVariant(std::vector<Tensor> values)
-      : values_(new std::vector<Tensor>(std::move(values))) {}
+  explicit OptionalVariant(std::vector<Tensor> values) {
+    values_ = std::make_shared<std::vector<Tensor>>(std::move(values));
+  }
 
   OptionalVariant(const OptionalVariant& other) : values_(other.values_) {}
 
@@ -79,7 +80,7 @@ class OptionalVariant {
       return false;
     }
     if (has_value) {
-      values_.reset(new std::vector<Tensor>(data.tensors()));
+      values_ = std::make_shared<std::vector<Tensor>>(data.tensors());
     } else {
       values_.reset();
     }
diff --git a/tensorflow/core/kernels/data/padded_batch_dataset_op.cc b/tensorflow/core/kernels/data/padded_batch_dataset_op.cc
index 0fff4c5370..41ea36263c 100644
--- a/tensorflow/core/kernels/data/padded_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/padded_batch_dataset_op.cc
@@ -135,8 +135,8 @@ class PaddedBatchDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::PaddedBatch")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::PaddedBatch")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
index 336fa52b49..add123f34a 100644
--- a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
+++ b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
@@ -29,7 +29,6 @@ limitations under the License.
 #include "tensorflow/core/lib/gtl/cleanup.h"
 #include "tensorflow/core/lib/random/random.h"
 #include "tensorflow/core/platform/cpu_info.h"
-#include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
 namespace data {
@@ -120,7 +119,7 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return MakeUnique<ParallelInterleaveIterator>(
+      return absl::make_unique<ParallelInterleaveIterator>(
           ParallelInterleaveIterator::Params{
               this, strings::StrCat(prefix, "::ParallelInterleaveV2")},
           sloppy_);
@@ -200,7 +199,7 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
                 params.dataset->num_parallel_calls_, mu_, cond_var_)),
             sloppy_(sloppy),
             current_elements_(params.dataset->cycle_length_),
-            thread_pool_(new thread::ThreadPool(
+            thread_pool_(absl::make_unique<thread::ThreadPool>(
                 Env::Default(), ThreadOptions(),
                 "data_parallel_interleave_worker_pool",
                 port::NumSchedulableCPUs() /* num_threads */,
@@ -513,15 +512,16 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
         if (!current_elements_manager_) {
           auto new_ctx = std::make_shared<IteratorContext>(*ctx);
           current_elements_manager_ =
-              WrapUnique<Thread>(ctx->env()->StartThread(
+              absl::WrapUnique<Thread>(ctx->env()->StartThread(
                   {}, "tf_data_parallel_interleave_current",
                   [this, new_ctx]() { CurrentElementsManager(new_ctx); }));
         }
         if (!future_elements_manager_) {
           auto new_ctx = std::make_shared<IteratorContext>(*ctx);
-          future_elements_manager_ = WrapUnique<Thread>(ctx->env()->StartThread(
-              {}, "tf_data_parallel_interleave_future",
-              [this, new_ctx]() { FutureElementsManager(new_ctx); }));
+          future_elements_manager_ =
+              absl::WrapUnique<Thread>(ctx->env()->StartThread(
+                  {}, "tf_data_parallel_interleave_future",
+                  [this, new_ctx]() { FutureElementsManager(new_ctx); }));
         }
       }
 
diff --git a/tensorflow/core/kernels/data/parallel_map_dataset_op.cc b/tensorflow/core/kernels/data/parallel_map_dataset_op.cc
index c0002c86d8..e2c9057cb2 100644
--- a/tensorflow/core/kernels/data/parallel_map_dataset_op.cc
+++ b/tensorflow/core/kernels/data/parallel_map_dataset_op.cc
@@ -102,9 +102,10 @@ class ParallelMapDatasetOp : public UnaryDatasetOpKernel {
         const string& prefix) const override {
       std::unique_ptr<ParallelMapFunctor> parallel_map_functor(nullptr);
       if (indices_.empty()) {
-        parallel_map_functor.reset(new ParallelMapDatasetFunctor(this));
+        parallel_map_functor =
+            absl::make_unique<ParallelMapDatasetFunctor>(this);
       } else {
-        parallel_map_functor.reset(new ShortCircuitFunctor(this));
+        parallel_map_functor = absl::make_unique<ShortCircuitFunctor>(this);
       }
       return NewParallelMapIterator(
           {this, strings::StrCat(prefix, "::ParallelMap")}, input_,
diff --git a/tensorflow/core/kernels/data/parallel_map_iterator.cc b/tensorflow/core/kernels/data/parallel_map_iterator.cc
index 9a4ce981d0..be91de12fe 100644
--- a/tensorflow/core/kernels/data/parallel_map_iterator.cc
+++ b/tensorflow/core/kernels/data/parallel_map_iterator.cc
@@ -24,7 +24,6 @@ limitations under the License.
 #include "tensorflow/core/framework/stats_aggregator.h"
 #include "tensorflow/core/lib/gtl/cleanup.h"
 #include "tensorflow/core/platform/cpu_info.h"
-#include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
 namespace data {
@@ -410,7 +409,7 @@ std::unique_ptr<IteratorBase> NewParallelMapIterator(
     const DatasetBase* input_dataset,
     std::unique_ptr<ParallelMapFunctor> parallel_map_functor,
     int32 num_parallel_calls, bool sloppy, bool preserve_cardinality) {
-  return MakeUnique<ParallelMapIterator>(
+  return absl::make_unique<ParallelMapIterator>(
       params, input_dataset,
       ParallelMapIterator::Params{std::move(parallel_map_functor),
                                   num_parallel_calls, sloppy,
diff --git a/tensorflow/core/kernels/data/prefetch_dataset_op.cc b/tensorflow/core/kernels/data/prefetch_dataset_op.cc
index 08d6de4bf9..1f2dd6ca95 100644
--- a/tensorflow/core/kernels/data/prefetch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/prefetch_dataset_op.cc
@@ -42,8 +42,8 @@ class PrefetchDatasetOp::Dataset : public DatasetBase {
 
   std::unique_ptr<IteratorBase> MakeIteratorInternal(
       const string& prefix) const override {
-    return std::unique_ptr<IteratorBase>(
-        new Iterator({this, strings::StrCat(prefix, "::Prefetch")}));
+    return absl::make_unique<Iterator>(
+        Iterator::Params{this, strings::StrCat(prefix, "::Prefetch")});
   }
 
   const DataTypeVector& output_dtypes() const override {
@@ -266,8 +266,9 @@ class PrefetchDatasetOp::Dataset : public DatasetBase {
     Status EnsurePrefetchThreadStarted(IteratorContext* ctx)
         EXCLUSIVE_LOCKS_REQUIRED(mu_) {
       if (!prefetch_thread_) {
-        std::shared_ptr<IteratorContext> new_ctx(new IteratorContext(*ctx));
-        prefetch_thread_.reset(ctx->env()->StartThread(
+        std::shared_ptr<IteratorContext> new_ctx =
+            std::make_shared<IteratorContext>(*ctx);
+        prefetch_thread_ = absl::WrapUnique<Thread>(ctx->env()->StartThread(
             {}, "tf_data_prefetch",
             [this, new_ctx]() { PrefetchThread(new_ctx); }));
       }
diff --git a/tensorflow/core/kernels/data/range_dataset_op.cc b/tensorflow/core/kernels/data/range_dataset_op.cc
index 580702f741..aa14d27d5c 100644
--- a/tensorflow/core/kernels/data/range_dataset_op.cc
+++ b/tensorflow/core/kernels/data/range_dataset_op.cc
@@ -53,8 +53,8 @@ class RangeDatasetOp : public DatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::Range")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::Range")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/reader_dataset_ops.cc b/tensorflow/core/kernels/data/reader_dataset_ops.cc
index 971fd2a436..789f9c859a 100644
--- a/tensorflow/core/kernels/data/reader_dataset_ops.cc
+++ b/tensorflow/core/kernels/data/reader_dataset_ops.cc
@@ -91,8 +91,8 @@ class TextLineDatasetOp : public DatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::TextLine")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::TextLine")});
     }
 
     const DataTypeVector& output_dtypes() const override {
@@ -221,20 +221,20 @@ class TextLineDatasetOp : public DatasetOpKernel {
         // Actually move on to next file.
         TF_RETURN_IF_ERROR(env->NewRandomAccessFile(
             dataset()->filenames_[current_file_index_], &file_));
-        input_stream_.reset(
-            new io::RandomAccessInputStream(file_.get(), false));
+        input_stream_ =
+            absl::make_unique<io::RandomAccessInputStream>(file_.get(), false);
 
         if (dataset()->use_compression_) {
-          zlib_input_stream_.reset(new io::ZlibInputStream(
+          zlib_input_stream_ = absl::make_unique<io::ZlibInputStream>(
               input_stream_.get(), dataset()->options_.input_buffer_size,
-              dataset()->options_.input_buffer_size, dataset()->options_));
-          buffered_input_stream_.reset(new io::BufferedInputStream(
+              dataset()->options_.input_buffer_size, dataset()->options_);
+          buffered_input_stream_ = absl::make_unique<io::BufferedInputStream>(
               zlib_input_stream_.get(), dataset()->options_.input_buffer_size,
-              false));
+              false);
         } else {
-          buffered_input_stream_.reset(new io::BufferedInputStream(
+          buffered_input_stream_ = absl::make_unique<io::BufferedInputStream>(
               input_stream_.get(), dataset()->options_.input_buffer_size,
-              false));
+              false);
         }
         return Status::OK();
       }
@@ -344,11 +344,12 @@ class FixedLengthRecordDatasetOp : public DatasetOpKernel {
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
       if (compression_type_.empty()) {
-        return std::unique_ptr<IteratorBase>(new UncompressedIterator(
-            {this, strings::StrCat(prefix, "::FixedLengthRecord")}));
+        return absl::make_unique<UncompressedIterator>(
+            UncompressedIterator::Params{
+                this, strings::StrCat(prefix, "::FixedLengthRecord")});
       } else {
-        return std::unique_ptr<IteratorBase>(new CompressedIterator(
-            {this, strings::StrCat(prefix, "::FixedLengthRecord")}));
+        return absl::make_unique<CompressedIterator>(CompressedIterator::Params{
+            this, strings::StrCat(prefix, "::FixedLengthRecord")});
       }
     }
 
@@ -452,8 +453,8 @@ class FixedLengthRecordDatasetOp : public DatasetOpKernel {
           }
           TF_RETURN_IF_ERROR(ctx->env()->NewRandomAccessFile(
               dataset()->filenames_[current_file_index_], &file_));
-          input_buffer_.reset(
-              new io::InputBuffer(file_.get(), dataset()->buffer_size_));
+          input_buffer_ = absl::make_unique<io::InputBuffer>(
+              file_.get(), dataset()->buffer_size_);
           TF_RETURN_IF_ERROR(
               input_buffer_->SkipNBytes(dataset()->header_bytes_));
         } while (true);
@@ -495,8 +496,8 @@ class FixedLengthRecordDatasetOp : public DatasetOpKernel {
           file_pos_limit_ = file_size - dataset()->footer_bytes_;
           TF_RETURN_IF_ERROR(ctx->env()->NewRandomAccessFile(
               dataset()->filenames_[current_file_index_], &file_));
-          input_buffer_.reset(
-              new io::InputBuffer(file_.get(), dataset()->buffer_size_));
+          input_buffer_ = absl::make_unique<io::InputBuffer>(
+              file_.get(), dataset()->buffer_size_);
           TF_RETURN_IF_ERROR(input_buffer_->Seek(current_pos));
         }
 
@@ -612,13 +613,14 @@ class FixedLengthRecordDatasetOp : public DatasetOpKernel {
                 dataset()->compression_type_ == "ZLIB"
                     ? io::ZlibCompressionOptions::DEFAULT()
                     : io::ZlibCompressionOptions::GZIP();
-            file_stream_.reset(new io::RandomAccessInputStream(file_.get()));
-            buffered_input_stream_.reset(new io::ZlibInputStream(
+            file_stream_ =
+                absl::make_unique<io::RandomAccessInputStream>(file_.get());
+            buffered_input_stream_ = absl::make_unique<io::ZlibInputStream>(
                 file_stream_.get(), dataset()->buffer_size_,
-                dataset()->buffer_size_, zlib_options));
+                dataset()->buffer_size_, zlib_options);
           } else {
-            buffered_input_stream_.reset(new io::BufferedInputStream(
-                file_.get(), dataset()->buffer_size_));
+            buffered_input_stream_ = absl::make_unique<io::BufferedInputStream>(
+                file_.get(), dataset()->buffer_size_);
           }
           TF_RETURN_IF_ERROR(
               buffered_input_stream_->SkipNBytes(dataset()->header_bytes_));
@@ -672,10 +674,11 @@ class FixedLengthRecordDatasetOp : public DatasetOpKernel {
               dataset()->compression_type_ == "ZLIB"
                   ? io::ZlibCompressionOptions::DEFAULT()
                   : io::ZlibCompressionOptions::GZIP();
-          file_stream_.reset(new io::RandomAccessInputStream(file_.get()));
-          buffered_input_stream_.reset(new io::ZlibInputStream(
+          file_stream_ =
+              absl::make_unique<io::RandomAccessInputStream>(file_.get());
+          buffered_input_stream_ = absl::make_unique<io::ZlibInputStream>(
               file_stream_.get(), dataset()->buffer_size_,
-              dataset()->buffer_size_, zlib_options));
+              dataset()->buffer_size_, zlib_options);
           lookahead_cache_.clear();
           TF_RETURN_IF_ERROR(buffered_input_stream_->SkipNBytes(
               current_pos - dataset()->footer_bytes_));
@@ -763,8 +766,8 @@ class TFRecordDatasetOp : public DatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::TFRecord")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::TFRecord")});
     }
 
     const DataTypeVector& output_dtypes() const override {
@@ -885,8 +888,8 @@ class TFRecordDatasetOp : public DatasetOpKernel {
         const string& next_filename =
             dataset()->filenames_[current_file_index_];
         TF_RETURN_IF_ERROR(env->NewRandomAccessFile(next_filename, &file_));
-        reader_.reset(
-            new io::SequentialRecordReader(file_.get(), dataset()->options_));
+        reader_ = absl::make_unique<io::SequentialRecordReader>(
+            file_.get(), dataset()->options_);
         return Status::OK();
       }
 
diff --git a/tensorflow/core/kernels/data/repeat_dataset_op.cc b/tensorflow/core/kernels/data/repeat_dataset_op.cc
index 8100f2695b..ef507ffdd1 100644
--- a/tensorflow/core/kernels/data/repeat_dataset_op.cc
+++ b/tensorflow/core/kernels/data/repeat_dataset_op.cc
@@ -51,14 +51,14 @@ class RepeatDatasetOp : public UnaryDatasetOpKernel {
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
       if (count_ < 0) {
-        return std::unique_ptr<IteratorBase>(new ForeverIterator(
-            {this, strings::StrCat(prefix, "::ForeverRepeat")}));
+        return absl::make_unique<ForeverIterator>(ForeverIterator::Params{
+            this, strings::StrCat(prefix, "::ForeverRepeat")});
       } else if (count_ == 0) {
-        return std::unique_ptr<IteratorBase>(new EmptyIterator(
-            {this, strings::StrCat(prefix, "::EmptyRepeat")}));
+        return absl::make_unique<EmptyIterator>(EmptyIterator::Params{
+            this, strings::StrCat(prefix, "::EmptyRepeat")});
       } else {
-        return std::unique_ptr<IteratorBase>(new FiniteIterator(
-            {this, strings::StrCat(prefix, "::FiniteRepeat")}));
+        return absl::make_unique<FiniteIterator>(FiniteIterator::Params{
+            this, strings::StrCat(prefix, "::FiniteRepeat")});
       }
     }
 
diff --git a/tensorflow/core/kernels/data/shuffle_dataset_op.cc b/tensorflow/core/kernels/data/shuffle_dataset_op.cc
index 4c380c1fa2..e0c435718a 100644
--- a/tensorflow/core/kernels/data/shuffle_dataset_op.cc
+++ b/tensorflow/core/kernels/data/shuffle_dataset_op.cc
@@ -23,7 +23,6 @@ limitations under the License.
 #include "tensorflow/core/lib/random/philox_random.h"
 #include "tensorflow/core/lib/random/random.h"
 #include "tensorflow/core/lib/random/random_distributions.h"
-#include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
 namespace data {
@@ -80,8 +79,9 @@ class ShuffleDatasetOpBase : public UnaryDatasetOpKernel {
             num_elements_(0),
             parent_generator_(seed, seed2),
             generator_(&parent_generator_) {
-        buffer_.reset(new std::vector<Tensor>[params.dataset->buffer_size_]);
-        slices_.push_back(MakeUnique<Slice>(0, 0));
+        buffer_ = absl::make_unique<std::vector<Tensor>[]>(
+            params.dataset->buffer_size_);
+        slices_.push_back(absl::make_unique<Slice>(0, 0));
       }
 
       Status GetNextInternal(IteratorContext* ctx,
@@ -124,7 +124,7 @@ class ShuffleDatasetOpBase : public UnaryDatasetOpKernel {
             }
             epoch_++;
             int64 n = slices_.back()->end;
-            slices_.push_back(MakeUnique<Slice>(n, n));
+            slices_.push_back(absl::make_unique<Slice>(n, n));
             TF_RETURN_IF_ERROR(this->dataset()->input_->MakeIterator(
                 ctx, this->prefix(), &input_impl_));
           }
@@ -273,7 +273,8 @@ class ShuffleDatasetOpBase : public UnaryDatasetOpKernel {
               reader->ReadScalar(this->full_name("slices_size"), &temp));
           slices_size = static_cast<size_t>(temp);
         }
-        buffer_.reset(new std::vector<Tensor>[this->dataset()->buffer_size_]);
+        buffer_ = absl::make_unique<std::vector<Tensor>[]>(
+            this->dataset()->buffer_size_);
         for (size_t i = 0; i < slices_size; ++i) {
           int64 start;
           TF_RETURN_IF_ERROR(reader->ReadScalar(
@@ -281,7 +282,7 @@ class ShuffleDatasetOpBase : public UnaryDatasetOpKernel {
           int64 end;
           TF_RETURN_IF_ERROR(reader->ReadScalar(
               this->full_name(strings::StrCat("slices_end_", i)), &end));
-          slices_.push_back(MakeUnique<Slice>(start, end));
+          slices_.push_back(absl::make_unique<Slice>(start, end));
           for (size_t j = start; j < end; ++j) {
             size_t index = j % this->dataset()->buffer_size_;
             int64 list_size;
@@ -399,8 +400,9 @@ class ShuffleDatasetOp : public ShuffleDatasetOpBase {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(new Iterator(
-          {this, strings::StrCat(prefix, "::Shuffle")}, seed_, seed2_));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::Shuffle")}, seed_,
+          seed2_);
     }
 
    protected:
@@ -578,9 +580,11 @@ class ShuffleDatasetOp : public ShuffleDatasetOpBase {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new ShuffleDatasetBase::Iterator<ShuffleDatasetBase>(
-              {this, strings::StrCat(prefix, "::Shuffle")}, seed_, seed2_));
+      return absl::make_unique<
+          ShuffleDatasetBase::Iterator<ShuffleDatasetBase>>(
+          ShuffleDatasetBase::Iterator<ShuffleDatasetBase>::Params{
+              this, strings::StrCat(prefix, "::Shuffle")},
+          seed_, seed2_);
     }
 
    protected:
@@ -663,10 +667,11 @@ class ShuffleAndRepeatDatasetOp : public ShuffleDatasetOpBase {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new ShuffleDatasetBase::Iterator<ShuffleDatasetBase>(
-              {this, strings::StrCat(prefix, "::ShuffleAndRepeat")}, seed_,
-              seed2_));
+      return absl::make_unique<
+          ShuffleDatasetBase::Iterator<ShuffleDatasetBase>>(
+          ShuffleDatasetBase::Iterator<ShuffleDatasetBase>::Params{
+              this, strings::StrCat(prefix, "::ShuffleAndRepeat")},
+          seed_, seed2_);
     }
 
    protected:
diff --git a/tensorflow/core/kernels/data/single_threaded_executor.cc b/tensorflow/core/kernels/data/single_threaded_executor.cc
index 89e3881037..aab4bfe640 100644
--- a/tensorflow/core/kernels/data/single_threaded_executor.cc
+++ b/tensorflow/core/kernels/data/single_threaded_executor.cc
@@ -376,8 +376,8 @@ static SingleThreadedExecutorRegistrar registrar;
 Status NewSingleThreadedExecutor(const LocalExecutorParams& params,
                                  std::unique_ptr<const Graph> graph,
                                  Executor** executor) {
-  std::unique_ptr<SingleThreadedExecutorImpl> impl(
-      new SingleThreadedExecutorImpl(params));
+  std::unique_ptr<SingleThreadedExecutorImpl> impl =
+      absl::make_unique<SingleThreadedExecutorImpl>(params);
   TF_RETURN_IF_ERROR(impl->Initialize(*graph));
   *executor = impl.release();
   return Status::OK();
diff --git a/tensorflow/core/kernels/data/single_threaded_executor_test.cc b/tensorflow/core/kernels/data/single_threaded_executor_test.cc
index 7bb51fb8b5..df669e53d3 100644
--- a/tensorflow/core/kernels/data/single_threaded_executor_test.cc
+++ b/tensorflow/core/kernels/data/single_threaded_executor_test.cc
@@ -139,7 +139,7 @@ Rendezvous::ParsedKey Key(const string& sender, const uint64 incarnation,
 
 TEST_F(ExecutorTest, SimpleAdd) {
   // c = a + b
-  std::unique_ptr<Graph> g(new Graph(OpRegistry::Global()));
+  std::unique_ptr<Graph> g = absl::make_unique<Graph>(OpRegistry::Global());
   auto in0 = test::graph::Arg(g.get(), 0, DT_FLOAT);
   auto in1 = test::graph::Arg(g.get(), 0, DT_FLOAT);
   auto tmp = test::graph::Add(g.get(), in0, in1);
@@ -163,7 +163,7 @@ TEST_F(ExecutorTest, SelfAdd) {
   //
   // b <- v10
   // All nodes are executed by one thread.
-  std::unique_ptr<Graph> g(new Graph(OpRegistry::Global()));
+  std::unique_ptr<Graph> g = absl::make_unique<Graph>(OpRegistry::Global());
   auto v = test::graph::Arg(g.get(), 0, DT_FLOAT);
   const int N = 10;
   for (int i = 1; i <= N; ++i) {
@@ -219,7 +219,7 @@ void BuildTree(int N, Graph* g) {
 }
 
 TEST_F(ExecutorTest, RandomTree) {
-  std::unique_ptr<Graph> g(new Graph(OpRegistry::Global()));
+  std::unique_ptr<Graph> g = absl::make_unique<Graph>(OpRegistry::Global());
   BuildTree(4096, g.get());
   Create(std::move(g));
   FunctionCallFrame call_frame({DT_FLOAT}, {DT_FLOAT});
@@ -231,7 +231,7 @@ TEST_F(ExecutorTest, RandomTree) {
 }
 
 TEST_F(ExecutorTest, OpError) {
-  std::unique_ptr<Graph> g(new Graph(OpRegistry::Global()));
+  std::unique_ptr<Graph> g = absl::make_unique<Graph>(OpRegistry::Global());
   auto zero = test::graph::Constant(g.get(), V(0.0));
   auto inf = test::graph::Unary(g.get(), "Reciprocal", zero);
   auto check = test::graph::CheckNumerics(g.get(), inf, "message");
diff --git a/tensorflow/core/kernels/data/skip_dataset_op.cc b/tensorflow/core/kernels/data/skip_dataset_op.cc
index e321066a71..5b85a10edf 100644
--- a/tensorflow/core/kernels/data/skip_dataset_op.cc
+++ b/tensorflow/core/kernels/data/skip_dataset_op.cc
@@ -50,11 +50,11 @@ class SkipDatasetOp : public UnaryDatasetOpKernel {
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
       if (count_ < 0) {
-        return std::unique_ptr<IteratorBase>(
-            new EmptyIterator({this, strings::StrCat(prefix, "::EmptySkip")}));
+        return absl::make_unique<EmptyIterator>(EmptyIterator::Params{
+            this, strings::StrCat(prefix, "::EmptySkip")});
       } else {
-        return std::unique_ptr<IteratorBase>(new FiniteIterator(
-            {this, strings::StrCat(prefix, "::FiniteSkip")}));
+        return absl::make_unique<FiniteIterator>(FiniteIterator::Params{
+            this, strings::StrCat(prefix, "::FiniteSkip")});
       }
     }
 
diff --git a/tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc b/tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc
index be105f8170..d8d7cd204d 100644
--- a/tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc
+++ b/tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc
@@ -41,8 +41,8 @@ class Dataset : public DatasetBase {
 
   std::unique_ptr<IteratorBase> MakeIteratorInternal(
       const string& prefix) const override {
-    return std::unique_ptr<IteratorBase>(
-        new Iterator({this, strings::StrCat(prefix, "::SparseTensorSlice")}));
+    return absl::make_unique<Iterator>(typename Iterator::Params{
+        this, strings::StrCat(prefix, "::SparseTensorSlice")});
   }
 
   const DataTypeVector& output_dtypes() const override { return dtypes_; }
diff --git a/tensorflow/core/kernels/data/take_dataset_op.cc b/tensorflow/core/kernels/data/take_dataset_op.cc
index 0a3d586953..0dd0c0c80d 100644
--- a/tensorflow/core/kernels/data/take_dataset_op.cc
+++ b/tensorflow/core/kernels/data/take_dataset_op.cc
@@ -50,11 +50,11 @@ class TakeDatasetOp : public UnaryDatasetOpKernel {
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
       if (count_ == 0) {
-        return std::unique_ptr<IteratorBase>(
-            new EmptyIterator({this, strings::StrCat(prefix, "::EmptyTake")}));
+        return absl::make_unique<EmptyIterator>(EmptyIterator::Params{
+            this, strings::StrCat(prefix, "::EmptyTake")});
       } else {
-        return std::unique_ptr<IteratorBase>(new FiniteIterator(
-            {this, strings::StrCat(prefix, "::FiniteTake")}));
+        return absl::make_unique<FiniteIterator>(FiniteIterator::Params{
+            this, strings::StrCat(prefix, "::FiniteTake")});
       }
     }
 
diff --git a/tensorflow/core/kernels/data/tensor_dataset_op.cc b/tensorflow/core/kernels/data/tensor_dataset_op.cc
index 98c23f23b2..a44dbd0d4d 100644
--- a/tensorflow/core/kernels/data/tensor_dataset_op.cc
+++ b/tensorflow/core/kernels/data/tensor_dataset_op.cc
@@ -50,8 +50,8 @@ class TensorDatasetOp : public DatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::FromTensor")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::FromTensor")});
     }
 
     const DataTypeVector& output_dtypes() const override { return dtypes_; }
diff --git a/tensorflow/core/kernels/data/tensor_slice_dataset_op.cc b/tensorflow/core/kernels/data/tensor_slice_dataset_op.cc
index 4ba2bde718..54dcd7eb7d 100644
--- a/tensorflow/core/kernels/data/tensor_slice_dataset_op.cc
+++ b/tensorflow/core/kernels/data/tensor_slice_dataset_op.cc
@@ -71,8 +71,8 @@ class TensorSliceDatasetOp : public DatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::TensorSlice")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::TensorSlice")});
     }
 
     const DataTypeVector& output_dtypes() const override { return dtypes_; }
diff --git a/tensorflow/core/kernels/data/window_dataset.cc b/tensorflow/core/kernels/data/window_dataset.cc
index c295631550..dc27702f1e 100644
--- a/tensorflow/core/kernels/data/window_dataset.cc
+++ b/tensorflow/core/kernels/data/window_dataset.cc
@@ -31,8 +31,8 @@ class WindowDataset : public DatasetBase {
 
   std::unique_ptr<IteratorBase> MakeIteratorInternal(
       const string& prefix) const override {
-    return std::unique_ptr<IteratorBase>(
-        new Iterator({this, strings::StrCat(prefix, "::Window")}));
+    return absl::make_unique<Iterator>(
+        Iterator::Params{this, strings::StrCat(prefix, "::Window")});
   }
 
   const DataTypeVector& output_dtypes() const override { return output_types_; }
diff --git a/tensorflow/core/kernels/data/window_dataset_op.cc b/tensorflow/core/kernels/data/window_dataset_op.cc
index ae13ae5da8..0b24c11891 100644
--- a/tensorflow/core/kernels/data/window_dataset_op.cc
+++ b/tensorflow/core/kernels/data/window_dataset_op.cc
@@ -78,8 +78,8 @@ class WindowDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(new Iterator(
-          Iterator::Params{this, strings::StrCat(prefix, "::Window")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::Window")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/zip_dataset_op.cc b/tensorflow/core/kernels/data/zip_dataset_op.cc
index 1760e63a9e..cdc2969fc2 100644
--- a/tensorflow/core/kernels/data/zip_dataset_op.cc
+++ b/tensorflow/core/kernels/data/zip_dataset_op.cc
@@ -62,8 +62,8 @@ class ZipDatasetOp : public DatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::Zip")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::Zip")});
     }
 
     const DataTypeVector& output_dtypes() const override {
-- 
GitLab


From 5704143a6790225f7f08533fa0225136095688b1 Mon Sep 17 00:00:00 2001
From: Nick Felt <nickfelt@google.com>
Date: Fri, 11 Jan 2019 16:03:25 -0800
Subject: [PATCH 0599/2345] Check syntax when parsing ApiDef textprotos

This change ensures that syntax errors in ApiDef textprotos (e.g. mistyping a field name, which I did) fail the overall API generation rule instead of having parsing fail silently and produce an empty result for that single file.

PiperOrigin-RevId: 228961785
---
 tensorflow/core/BUILD                         |   1 +
 tensorflow/core/framework/op_gen_lib.cc       |  12 +-
 tensorflow/core/framework/op_gen_lib_test.cc  |  15 ++-
 tensorflow/core/platform/default/protobuf.h   |   1 +
 tensorflow/core/util/proto/BUILD              |  15 +++
 tensorflow/core/util/proto/proto_utils.cc     |  51 +++++++-
 tensorflow/core/util/proto/proto_utils.h      |  31 +++++
 .../core/util/proto/proto_utils_test.cc       | 112 ++++++++++++++++++
 tensorflow/js/ops/ts_op_gen_test.cc           |  15 +--
 9 files changed, 237 insertions(+), 16 deletions(-)
 create mode 100644 tensorflow/core/util/proto/proto_utils_test.cc

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index ee9ceeae89..b655fdcbff 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1017,6 +1017,7 @@ cc_library(
         ":lib",
         ":lib_internal",
         ":protos_all_cc",
+        "//tensorflow/core/util/proto:proto_utils",
     ],
 )
 
diff --git a/tensorflow/core/framework/op_gen_lib.cc b/tensorflow/core/framework/op_gen_lib.cc
index 505ab54775..92a7038a40 100644
--- a/tensorflow/core/framework/op_gen_lib.cc
+++ b/tensorflow/core/framework/op_gen_lib.cc
@@ -23,6 +23,7 @@ limitations under the License.
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/protobuf.h"
+#include "tensorflow/core/util/proto/proto_utils.h"
 
 namespace tensorflow {
 
@@ -488,14 +489,21 @@ Status ApiDefMap::LoadFile(Env* env, const string& filename) {
   if (filename.empty()) return Status::OK();
   string contents;
   TF_RETURN_IF_ERROR(ReadFileToString(env, filename, &contents));
-  TF_RETURN_IF_ERROR(LoadApiDef(contents));
+  Status status = LoadApiDef(contents);
+  if (!status.ok()) {
+    // Return failed status annotated with filename to aid in debugging.
+    return Status(status.code(),
+                  strings::StrCat("Error parsing ApiDef file ", filename, ": ",
+                                  status.error_message()));
+  }
   return Status::OK();
 }
 
 Status ApiDefMap::LoadApiDef(const string& api_def_file_contents) {
   const string contents = PBTxtFromMultiline(api_def_file_contents);
   ApiDefs api_defs;
-  protobuf::TextFormat::ParseFromString(contents, &api_defs);
+  TF_RETURN_IF_ERROR(
+      proto_utils::ParseTextFormatFromString(contents, &api_defs));
   for (const auto& api_def : api_defs.op()) {
     // Check if the op definition is loaded. If op definition is not
     // loaded, then we just skip this ApiDef.
diff --git a/tensorflow/core/framework/op_gen_lib_test.cc b/tensorflow/core/framework/op_gen_lib_test.cc
index e0e77c7449..6b43d7dc68 100644
--- a/tensorflow/core/framework/op_gen_lib_test.cc
+++ b/tensorflow/core/framework/op_gen_lib_test.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/core/framework/op_gen_lib.h"
 
 #include "tensorflow/core/framework/op_def.pb.h"
+#include "tensorflow/core/lib/core/error_codes.pb.h"
 #include "tensorflow/core/platform/test.h"
 
 namespace tensorflow {
@@ -39,7 +40,7 @@ constexpr char kTestOpList[] = R"(op {
     version: 123
     explanation: "foo"
   }
-)";
+})";
 
 constexpr char kTestApiDef[] = R"(op {
   graph_op_name: "testop"
@@ -455,6 +456,18 @@ op {
   ASSERT_EQ(tensorflow::error::FAILED_PRECONDITION, status.code());
 }
 
+TEST(OpGenLibTest, ApiDefInvalidSyntax) {
+  const string api_def = R"pb(
+    op { bad_op_name: "testop" }
+  )pb";
+
+  OpList op_list;
+  ApiDefMap api_map(op_list);
+  // Loading with invalid syntax (e.g. unrecognized field name) should fail.
+  auto status = api_map.LoadApiDef(api_def);
+  ASSERT_EQ(tensorflow::error::INVALID_ARGUMENT, status.code());
+}
+
 TEST(OpGenLibTest, ApiDefUpdateDocs) {
   const string op_list1 = R"(op {
   name: "testop"
diff --git a/tensorflow/core/platform/default/protobuf.h b/tensorflow/core/platform/default/protobuf.h
index 2708d6ebda..aeef2d9b88 100644
--- a/tensorflow/core/platform/default/protobuf.h
+++ b/tensorflow/core/platform/default/protobuf.h
@@ -23,6 +23,7 @@ limitations under the License.
 #include "google/protobuf/descriptor.h"
 #include "google/protobuf/descriptor.pb.h"
 #include "google/protobuf/dynamic_message.h"
+#include "google/protobuf/io/tokenizer.h"
 #include "google/protobuf/text_format.h"
 #include "google/protobuf/util/json_util.h"
 #include "google/protobuf/util/type_resolver_util.h"
diff --git a/tensorflow/core/util/proto/BUILD b/tensorflow/core/util/proto/BUILD
index 7e549c7764..b990f0a749 100644
--- a/tensorflow/core/util/proto/BUILD
+++ b/tensorflow/core/util/proto/BUILD
@@ -68,5 +68,20 @@ cc_library(
     deps = [
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:platform_base",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+tf_cc_test(
+    name = "proto_utils_test",
+    srcs = ["proto_utils_test.cc"],
+    deps = [
+        ":proto_utils",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+        "@com_google_googletest//:gtest_main",
     ],
 )
diff --git a/tensorflow/core/util/proto/proto_utils.cc b/tensorflow/core/util/proto/proto_utils.cc
index 201f05a129..f106414139 100644
--- a/tensorflow/core/util/proto/proto_utils.cc
+++ b/tensorflow/core/util/proto/proto_utils.cc
@@ -13,11 +13,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include "tensorflow/core/util/proto/proto_utils.h"
+
+#include "absl/strings/string_view.h"
+#include "absl/strings/substitute.h"
 #include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/protobuf.h"
 
-#include "tensorflow/core/util/proto/proto_utils.h"
-
 namespace tensorflow {
 namespace proto_utils {
 
@@ -66,5 +69,49 @@ bool IsCompatibleType(FieldDescriptor::Type field_type, DataType dtype) {
   }
 }
 
+Status ParseTextFormatFromString(absl::string_view input,
+                                 protobuf::Message* output) {
+  DCHECK(output != nullptr) << "output must be non NULL";
+  // When checks are disabled, instead log the error and return an error status.
+  if (output == nullptr) {
+    LOG(ERROR) << "output must be non NULL";
+    return Status(error::INVALID_ARGUMENT, "output must be non NULL");
+  }
+  string err;
+  StringErrorCollector err_collector(&err, /*one-indexing=*/true);
+  protobuf::TextFormat::Parser parser;
+  parser.RecordErrorsTo(&err_collector);
+  if (!parser.ParseFromString(string(input), output)) {
+    return Status(error::INVALID_ARGUMENT, err);
+  }
+  return Status::OK();
+}
+
+StringErrorCollector::StringErrorCollector(string* error_text)
+    : StringErrorCollector(error_text, false) {}
+
+StringErrorCollector::StringErrorCollector(string* error_text,
+                                           bool one_indexing)
+    : error_text_(error_text), index_offset_(one_indexing ? 1 : 0) {
+  DCHECK(error_text_ != nullptr) << "error_text must be non NULL";
+  // When checks are disabled, just log and then ignore added errors/warnings.
+  if (error_text_ == nullptr) {
+    LOG(ERROR) << "error_text must be non NULL";
+  }
+}
+
+void StringErrorCollector::AddError(int line, int column,
+                                    const string& message) {
+  if (error_text_ != nullptr) {
+    absl::SubstituteAndAppend(error_text_, "$0($1): $2\n", line + index_offset_,
+                              column + index_offset_, message);
+  }
+}
+
+void StringErrorCollector::AddWarning(int line, int column,
+                                      const string& message) {
+  AddError(line, column, message);
+}
+
 }  // namespace proto_utils
 }  // namespace tensorflow
diff --git a/tensorflow/core/util/proto/proto_utils.h b/tensorflow/core/util/proto/proto_utils.h
index d5e0b9006c..9451e317a1 100644
--- a/tensorflow/core/util/proto/proto_utils.h
+++ b/tensorflow/core/util/proto/proto_utils.h
@@ -16,7 +16,9 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_UTIL_PROTO_PROTO_UTILS_H_
 #define TENSORFLOW_CORE_UTIL_PROTO_PROTO_UTILS_H_
 
+#include "absl/strings/string_view.h"
 #include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/protobuf.h"
 
 namespace tensorflow {
@@ -27,6 +29,35 @@ using tensorflow::protobuf::FieldDescriptor;
 // Returns true if the proto field type can be converted to the tensor dtype.
 bool IsCompatibleType(FieldDescriptor::Type field_type, DataType dtype);
 
+// Parses a text-formatted protobuf from a string into the given Message* output
+// and returns status OK if valid, or INVALID_ARGUMENT with an accompanying
+// parser error message if the text format is invalid.
+Status ParseTextFormatFromString(absl::string_view input,
+                                 protobuf::Message* output);
+
+class StringErrorCollector : public protobuf::io::ErrorCollector {
+ public:
+  // String error_text is unowned and must remain valid during the use of
+  // StringErrorCollector.
+  explicit StringErrorCollector(string* error_text);
+  // If one_indexing is set to true, all line and column numbers will be
+  // increased by one for cases when provided indices are 0-indexed and
+  // 1-indexed error messages are desired
+  StringErrorCollector(string* error_text, bool one_indexing);
+  StringErrorCollector(const StringErrorCollector&) = delete;
+  StringErrorCollector& operator=(const StringErrorCollector&) = delete;
+
+  // Implementation of protobuf::io::ErrorCollector::AddError.
+  void AddError(int line, int column, const string& message) override;
+
+  // Implementation of protobuf::io::ErrorCollector::AddWarning.
+  void AddWarning(int line, int column, const string& message) override;
+
+ private:
+  string* const error_text_;
+  const int index_offset_;
+};
+
 }  // namespace proto_utils
 }  // namespace tensorflow
 
diff --git a/tensorflow/core/util/proto/proto_utils_test.cc b/tensorflow/core/util/proto/proto_utils_test.cc
new file mode 100644
index 0000000000..7f6312a718
--- /dev/null
+++ b/tensorflow/core/util/proto/proto_utils_test.cc
@@ -0,0 +1,112 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/util/proto/proto_utils.h"
+
+#include <gmock/gmock.h>
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/protobuf.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+
+using proto_utils::ParseTextFormatFromString;
+using proto_utils::StringErrorCollector;
+using ::testing::ContainsRegex;
+
+TEST(ParseTextFormatFromStringTest, Success) {
+  protobuf::DescriptorProto output;
+  TF_ASSERT_OK(ParseTextFormatFromString("name: \"foo\"", &output));
+  EXPECT_EQ(output.name(), "foo");
+}
+
+TEST(ParseTextFormatFromStringTest, ErrorOnInvalidSyntax) {
+  protobuf::DescriptorProto output;
+  Status status = ParseTextFormatFromString("name: foo", &output);
+  EXPECT_EQ(status.code(), error::INVALID_ARGUMENT);
+  EXPECT_THAT(status.error_message(), ContainsRegex("foo"));
+  EXPECT_FALSE(output.has_name());
+}
+
+TEST(ParseTextFormatFromStringTest, ErrorOnUnknownFieldName) {
+  protobuf::DescriptorProto output;
+  Status status = ParseTextFormatFromString("badname: \"foo\"", &output);
+  EXPECT_EQ(status.code(), error::INVALID_ARGUMENT);
+  EXPECT_THAT(status.error_message(), ContainsRegex("badname"));
+  EXPECT_FALSE(output.has_name());
+}
+
+TEST(ParseTextFormatFromStringTest, DiesOnNullOutputPointer) {
+#ifndef NDEBUG
+  ASSERT_DEATH(ParseTextFormatFromString("foo", nullptr).IgnoreError(),
+               "output.*non NULL");
+#else
+  // Under NDEBUG we don't die but should still return an error status.
+  Status status = ParseTextFormatFromString("foo", nullptr);
+  EXPECT_EQ(status.code(), error::INVALID_ARGUMENT);
+  EXPECT_THAT(status.error_message(), ContainsRegex("output.*non NULL"));
+#endif
+}
+
+TEST(StringErrorCollectorTest, AppendsError) {
+  string err;
+  StringErrorCollector collector(&err);
+  collector.AddError(1, 2, "foo");
+  EXPECT_EQ("1(2): foo\n", err);
+}
+
+TEST(StringErrorCollectorTest, AppendsWarning) {
+  string err;
+  StringErrorCollector collector(&err);
+  collector.AddWarning(1, 2, "foo");
+  EXPECT_EQ("1(2): foo\n", err);
+}
+
+TEST(StringErrorCollectorTest, AppendsMultipleError) {
+  string err;
+  StringErrorCollector collector(&err);
+  collector.AddError(1, 2, "foo");
+  collector.AddError(3, 4, "bar");
+  EXPECT_EQ("1(2): foo\n3(4): bar\n", err);
+}
+
+TEST(StringErrorCollectorTest, AppendsMultipleWarning) {
+  string err;
+  StringErrorCollector collector(&err);
+  collector.AddWarning(1, 2, "foo");
+  collector.AddWarning(3, 4, "bar");
+  EXPECT_EQ("1(2): foo\n3(4): bar\n", err);
+}
+
+TEST(StringErrorCollectorTest, OffsetWorks) {
+  string err;
+  StringErrorCollector collector(&err, true);
+  collector.AddError(1, 2, "foo");
+  collector.AddWarning(3, 4, "bar");
+  EXPECT_EQ("2(3): foo\n4(5): bar\n", err);
+}
+
+TEST(StringErrorCollectorTest, DiesOnNullErrorText) {
+#ifndef NDEBUG
+  ASSERT_DEATH(StringErrorCollector(nullptr), "error_text.*non NULL");
+#else
+  // Under NDEBUG we don't die and instead AddError/AddWarning just do nothing.
+  StringErrorCollector collector(nullptr);
+  collector.AddError(1, 2, "foo");
+  collector.AddWarning(3, 4, "bar");
+#endif
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/js/ops/ts_op_gen_test.cc b/tensorflow/js/ops/ts_op_gen_test.cc
index 03241689b5..1c51dd030f 100644
--- a/tensorflow/js/ops/ts_op_gen_test.cc
+++ b/tensorflow/js/ops/ts_op_gen_test.cc
@@ -112,22 +112,15 @@ import {createTensorsTypeOpAttr, nodeBackend} from './op_utils';
 }
 
 TEST(TsOpGenTest, InputSingleAndList) {
-  const string api_def = R"(
-op {
-  name: "Foo"
-  input_arg {
-    name: "images"
-    type_attr: "T"
-    number_attr: "N"
-  }
-}
-)";
+  const string api_def = R"pb(
+    op { graph_op_name: "Foo" arg_order: "dim" arg_order: "images" }
+  )pb";
 
   string ts_file_text;
   GenerateTsOpFileText("", api_def, &ts_file_text);
 
   const string expected = R"(
-export function Foo(images: tfc.Tensor[], dim: tfc.Tensor): tfc.Tensor {
+export function Foo(dim: tfc.Tensor, images: tfc.Tensor[]): tfc.Tensor {
 )";
   ExpectContainsStr(ts_file_text, expected);
 }
-- 
GitLab


From 5c89fd3016286e8c3f71b29174a7d6efde90d611 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 16:07:23 -0800
Subject: [PATCH 0600/2345] Automated rollback of commit
 196715b526a695678891ec11d13b7290b3f6861a

PiperOrigin-RevId: 228962444
---
 tensorflow/core/kernels/BUILD                  | 1 -
 tensorflow/core/kernels/fused_batch_norm_op.cc | 1 -
 2 files changed, 2 deletions(-)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 4db0505a8f..0792b398f1 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -3820,7 +3820,6 @@ NN_DEPS = [
     ":conv_2d",
     ":eigen_contraction_kernel",
     ":ops_util",
-    ":pooling_ops",
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
diff --git a/tensorflow/core/kernels/fused_batch_norm_op.cc b/tensorflow/core/kernels/fused_batch_norm_op.cc
index dbd3bb05db..48b339508b 100644
--- a/tensorflow/core/kernels/fused_batch_norm_op.cc
+++ b/tensorflow/core/kernels/fused_batch_norm_op.cc
@@ -18,7 +18,6 @@ limitations under the License.
 #if GOOGLE_CUDA
 #define EIGEN_USE_GPU
 #include "tensorflow/core/kernels/conv_2d.h"
-#include "tensorflow/core/kernels/conv_ops_gpu.h"
 #include "tensorflow/core/util/stream_executor_util.h"
 #endif
 
-- 
GitLab


From 2f2f865d9857f8d2c84d7a2554e41563947ac6cc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 16:10:53 -0800
Subject: [PATCH 0601/2345] Test to show errors in conv2d_transpose

PiperOrigin-RevId: 228962978
---
 tensorflow/lite/build_def.bzl                 |  1 +
 tensorflow/lite/testing/generate_examples.py  | 42 +++++++++++++++++++
 .../testing/generated_examples_zip_test.cc    |  2 +
 3 files changed, 45 insertions(+)

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index 343ec60f2a..03b217c120 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -232,6 +232,7 @@ def generated_test_models():
         "constant",
         "control_dep",
         "conv",
+        "conv2d_transpose",
         "conv_with_shared_weights",
         "conv_to_depthwiseconv_with_shared_weights",
         "depthwiseconv",
diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index 87e7d7eb02..d2c275f787 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -3222,6 +3222,48 @@ def make_slice_tests(zip_path):
   make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
 
 
+def make_conv2d_transpose_tests(zip_path):
+  """Make a set of tests to do transpose_conv."""
+
+  test_parameters = [{
+      "input_shape": [[1, 50, 54, 3]],
+      "filter_shape": [[1, 1, 8, 3], [1, 2, 8, 3], [1, 3, 8, 3], [1, 4, 8, 3]],
+      "output_shape": [[1, 100, 108, 8]],
+  }]
+
+  def build_graph(parameters):
+    """Build a transpose_conv graph given `parameters`."""
+    input_tensor = tf.placeholder(
+        dtype=tf.float32, name="input", shape=parameters["input_shape"])
+
+    filter_tensor = tf.placeholder(
+        dtype=tf.float32, name="filter", shape=parameters["filter_shape"])
+
+    out = tf.nn.conv2d_transpose(
+        input_tensor,
+        filter_tensor,
+        output_shape=parameters["output_shape"],
+        padding="SAME",
+        strides=(1, 2, 2, 1))
+
+    input_tensors = [input_tensor, filter_tensor]
+    return input_tensors, [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    values = [
+        create_tensor_data(np.float32, parameters["input_shape"]),
+        create_tensor_data(np.float32, parameters["filter_shape"])
+    ]
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_success=4)
+
+
 # Since compute output_shape is fairly complicated for
 # tf.nn.conv2d_backprop_input input_sizes argument, so we here first perform a
 # "conv2d" operation to get the output, then we use the output to feed in
diff --git a/tensorflow/lite/testing/generated_examples_zip_test.cc b/tensorflow/lite/testing/generated_examples_zip_test.cc
index a9a31ad088..bade6590a4 100644
--- a/tensorflow/lite/testing/generated_examples_zip_test.cc
+++ b/tensorflow/lite/testing/generated_examples_zip_test.cc
@@ -105,6 +105,8 @@ std::map<string, string> kBrokenTests = {
 
     // Strided Slice chooses the wrong dimension.
     {R"(^\/strided_slice_buggy)", "119786029"},
+
+    {R"(^\/conv2d_transpose_filter_shape=\[1,4,8,3\])", "122735908"},
 };
 
 // Allows test data to be unarchived into a temporary directory and makes
-- 
GitLab


From b523734dd86bb943c490a5c25c358014922a797e Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 11 Jan 2019 16:18:21 -0800
Subject: [PATCH 0602/2345] [XLA] Don't re-seed RNG on every call to
 HloEvaluator::Evaluate.

Some tests -- in particular, tests that do reductions and so invoke a small
subcomputation many times -- invoke HloEvaluator::Evaluate() many times.
Previously, each such invocation was creating an std::random_device.

Creating std::random_device is very expensive -- it touches /dev/urandom and so
on.  This was creating a lot of overhead (25%+) in these tests.  So instead, we
do it only once, and do a simple atomic op to generate new seeds.

PiperOrigin-RevId: 228963974
---
 tensorflow/compiler/xla/service/hlo_evaluator.cc | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index 927e751d38..bcfa96d17f 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -258,14 +258,18 @@ StatusOr<Literal> HloEvaluator::Evaluate(
   for (const auto& literal_ptr : arg_literals) {
     arg_literals_.push_back(&*literal_ptr);
   }
+
+  // Re-seed RNG, either from the configuration's seed or a monotonic
+  // per-evaluator seed (which prevents two evaluators from returning the same
+  // random sequence).
   if (computation.parent()->config().seed()) {
     seed_ = computation.parent()->config().seed();
   } else {
-    std::random_device rd;
-    seed_ = rd();
+    // Start global_seed at a (true) random value.
+    static std::atomic<uint64> global_seed{std::random_device()()};
+    seed_ = global_seed.fetch_add(1);
   }
-
-  engine_ = std::minstd_rand0(seed_);
+  engine_.seed(seed_);
 
   TF_RETURN_IF_ERROR(computation.Accept(this));
   return GetEvaluatedLiteralFor(computation.root_instruction()).Clone();
-- 
GitLab


From 3bf87a030a7d05a347b541047dc9308edaafb7d7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 16:46:46 -0800
Subject: [PATCH 0603/2345] move eager profiler to core/profiler/lib.

PiperOrigin-RevId: 228967775
---
 tensorflow/c/eager/BUILD                      |  4 +-
 tensorflow/c/eager/c_api_internal.h           |  2 +-
 tensorflow/core/common_runtime/eager/BUILD    | 29 -------------
 tensorflow/core/profiler/lib/BUILD            | 42 +++++++++++++++++++
 .../lib/eager_profiler.cc}                    |  2 +-
 .../lib/eager_profiler.h}                     |  0
 6 files changed, 46 insertions(+), 33 deletions(-)
 create mode 100644 tensorflow/core/profiler/lib/BUILD
 rename tensorflow/core/{common_runtime/eager/profiler.cc => profiler/lib/eager_profiler.cc} (98%)
 rename tensorflow/core/{common_runtime/eager/profiler.h => profiler/lib/eager_profiler.h} (100%)

diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
index 710850b5f2..51665fb9db 100644
--- a/tensorflow/c/eager/BUILD
+++ b/tensorflow/c/eager/BUILD
@@ -37,7 +37,6 @@ tf_cuda_library(
             "//tensorflow/c:c_api_internal",
             "//tensorflow/core:core_cpu",
             "//tensorflow/core/common_runtime/eager:attr_builder",
-            "//tensorflow/core/common_runtime/eager:profiler",
             "//tensorflow/core/common_runtime/eager:context",
             "//tensorflow/core/common_runtime/eager:eager_executor",
             "//tensorflow/core/common_runtime/eager:execute",
@@ -71,6 +70,7 @@ tf_cuda_library(
         "//tensorflow/core/distributed_runtime:remote_device",
         "//tensorflow/core/distributed_runtime:server_lib",
         "//tensorflow/core/distributed_runtime:worker_env",
+        "//tensorflow/core/profiler/lib:eager_profiler",
         "//tensorflow/core:gpu_runtime",
     ],
 )
@@ -98,7 +98,6 @@ tf_cuda_library(
         "//tensorflow/core/common_runtime/eager:eager_executor",
         "//tensorflow/core/common_runtime/eager:eager_operation",
         "//tensorflow/core/common_runtime/eager:kernel_and_device",
-        "//tensorflow/core/common_runtime/eager:profiler",
         "//tensorflow/core/common_runtime/eager:tensor_handle",
         "//tensorflow/core/distributed_runtime:remote_device",
         "//tensorflow/core/distributed_runtime:server_lib",
@@ -111,6 +110,7 @@ tf_cuda_library(
         "//tensorflow/core/distributed_runtime/rpc:grpc_worker_service",
         "//tensorflow/core/distributed_runtime/rpc:rpc_rendezvous_mgr",
         "//tensorflow/core/distributed_runtime/rpc/eager:grpc_eager_client",
+        "//tensorflow/core/profiler/lib:eager_profiler",
     ],
 )
 
diff --git a/tensorflow/c/eager/c_api_internal.h b/tensorflow/c/eager/c_api_internal.h
index 7b1035f631..d330aa4290 100644
--- a/tensorflow/c/eager/c_api_internal.h
+++ b/tensorflow/c/eager/c_api_internal.h
@@ -34,7 +34,6 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/eager/eager_executor.h"
 #include "tensorflow/core/common_runtime/eager/eager_operation.h"
 #include "tensorflow/core/common_runtime/eager/kernel_and_device.h"
-#include "tensorflow/core/common_runtime/eager/profiler.h"
 #include "tensorflow/core/common_runtime/eager/tensor_handle.h"
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/common_runtime/rendezvous_mgr.h"
@@ -53,6 +52,7 @@ limitations under the License.
 #include "tensorflow/core/lib/gtl/stl_util.h"
 #include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/platform/thread_annotations.h"
+#include "tensorflow/core/profiler/lib/eager_profiler.h"
 #include "tensorflow/core/public/version.h"
 
 struct TFE_ContextOptions {
diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD
index 2697aea783..77e3246df0 100644
--- a/tensorflow/core/common_runtime/eager/BUILD
+++ b/tensorflow/core/common_runtime/eager/BUILD
@@ -88,35 +88,6 @@ tf_cuda_library(
     ],
 )
 
-tf_cuda_library(
-    name = "profiler",
-    srcs = [
-        "profiler.cc",
-    ],
-    hdrs = [
-        "profiler.h",
-    ],
-    visibility = ["//tensorflow:internal"],
-    deps = [
-        ":context",
-        "//tensorflow/cc/profiler",
-    ] + select({
-        "//tensorflow:android": [
-            "//tensorflow/core:android_tensorflow_lib_lite",
-        ],
-        "//conditions:default": [
-            "//tensorflow/core:core_cpu_lib",
-            "//tensorflow/core:framework",
-            "//tensorflow/core:framework_internal",
-            "//tensorflow/core:lib",
-            "//tensorflow/core:lib_internal",
-            "//tensorflow/core:protos_all_cc",
-            "//tensorflow/core:session_options",
-            "//tensorflow/core:device_tracer",
-        ],
-    }),
-)
-
 tf_cuda_library(
     name = "tensor_handle",
     srcs = [
diff --git a/tensorflow/core/profiler/lib/BUILD b/tensorflow/core/profiler/lib/BUILD
new file mode 100644
index 0000000000..3d46606cbd
--- /dev/null
+++ b/tensorflow/core/profiler/lib/BUILD
@@ -0,0 +1,42 @@
+package(
+    default_visibility = [
+        "//tensorflow:internal",
+        "//tensorflow_models:__subpackages__",
+    ],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+load(
+    "//tensorflow:tensorflow.bzl",
+    "tf_cuda_library",
+)
+
+tf_cuda_library(
+    name = "eager_profiler",
+    srcs = [
+        "eager_profiler.cc",
+    ],
+    hdrs = [
+        "eager_profiler.h",
+    ],
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        "//tensorflow/core/common_runtime/eager:context",
+        "//tensorflow/cc/profiler",
+    ] + select({
+        "//tensorflow:android": [
+            "//tensorflow/core:android_tensorflow_lib_lite",
+        ],
+        "//conditions:default": [
+            "//tensorflow/core:core_cpu_lib",
+            "//tensorflow/core:framework",
+            "//tensorflow/core:framework_internal",
+            "//tensorflow/core:lib",
+            "//tensorflow/core:lib_internal",
+            "//tensorflow/core:protos_all_cc",
+            "//tensorflow/core:session_options",
+            "//tensorflow/core:device_tracer",
+        ],
+    }),
+)
diff --git a/tensorflow/core/common_runtime/eager/profiler.cc b/tensorflow/core/profiler/lib/eager_profiler.cc
similarity index 98%
rename from tensorflow/core/common_runtime/eager/profiler.cc
rename to tensorflow/core/profiler/lib/eager_profiler.cc
index 46449d0771..7702febbc3 100644
--- a/tensorflow/core/common_runtime/eager/profiler.cc
+++ b/tensorflow/core/profiler/lib/eager_profiler.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/core/common_runtime/eager/profiler.h"
+#include "tensorflow/core/profiler/lib/eager_profiler.h"
 #include "tensorflow/cc/profiler/profiler.h"
 #include "tensorflow/core/common_runtime/eager/context.h"
 #include "tensorflow/core/common_runtime/step_stats_collector.h"
diff --git a/tensorflow/core/common_runtime/eager/profiler.h b/tensorflow/core/profiler/lib/eager_profiler.h
similarity index 100%
rename from tensorflow/core/common_runtime/eager/profiler.h
rename to tensorflow/core/profiler/lib/eager_profiler.h
-- 
GitLab


From 5846ad65a0cd37586f248fea2fb2c2015bb1221a Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 11 Jan 2019 17:24:57 -0800
Subject: [PATCH 0604/2345] Add outfeed support to replay_computation.

Without this, computations that outfeed will hang.

PiperOrigin-RevId: 228972323
---
 .../compiler/xla/service/gpu/xfeed_queue.h    |  13 ++
 tensorflow/compiler/xla/tools/BUILD           |   1 +
 .../compiler/xla/tools/replay_computation.cc  | 180 +++++++++++++-----
 3 files changed, 142 insertions(+), 52 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/xfeed_queue.h b/tensorflow/compiler/xla/service/gpu/xfeed_queue.h
index dd46ff433b..167c038420 100644
--- a/tensorflow/compiler/xla/service/gpu/xfeed_queue.h
+++ b/tensorflow/compiler/xla/service/gpu/xfeed_queue.h
@@ -47,6 +47,10 @@ class XfeedQueue {
   // Blocks until the queue is non-empty, then returns the buffer at the head of
   // the queue.
   BufferType BlockingGetNextDestination() {
+    for (const auto& callback : before_get_next_dest_callbacks_) {
+      callback();
+    }
+
     bool became_empty;
     BufferType current_buffer;
     {
@@ -69,6 +73,10 @@ class XfeedQueue {
   void RegisterOnEmptyCallback(std::function<void()> callback) {
     on_empty_callbacks_.push_back(std::move(callback));
   }
+  void RegisterBeforeGetNextDestinationCallback(
+      std::function<void()> callback) {
+    before_get_next_dest_callbacks_.push_back(std::move(callback));
+  }
 
  private:
   tensorflow::mutex mu_;
@@ -82,6 +90,11 @@ class XfeedQueue {
   // List of callbacks which will be called when 'enqueued_buffers_' becomes
   // empty.
   std::vector<std::function<void()>> on_empty_callbacks_;
+
+  // List of callbacks which will be called before BlockingGetNextDestination()
+  // is called. This lets you e.g. call EnqueueDestination() for each call to
+  // BlockingGetNextDestination().
+  std::vector<std::function<void()>> before_get_next_dest_callbacks_;
 };
 
 }  // namespace gpu
diff --git a/tensorflow/compiler/xla/tools/BUILD b/tensorflow/compiler/xla/tools/BUILD
index 99b32c19a5..a7ef32fd35 100644
--- a/tensorflow/compiler/xla/tools/BUILD
+++ b/tensorflow/compiler/xla/tools/BUILD
@@ -95,6 +95,7 @@ cc_library(
         "//tensorflow/compiler/xla/service:hlo_parser",
         "//tensorflow/compiler/xla/service:hlo_proto",
         "//tensorflow/compiler/xla/service/gpu:infeed_manager",
+        "//tensorflow/compiler/xla/service/gpu:outfeed_manager",
         "//tensorflow/compiler/xla/tests:test_utils",
         "//tensorflow/core:framework_internal",
         "//tensorflow/core:lib",
diff --git a/tensorflow/compiler/xla/tools/replay_computation.cc b/tensorflow/compiler/xla/tools/replay_computation.cc
index 75023f6edf..c01a47b510 100644
--- a/tensorflow/compiler/xla/tools/replay_computation.cc
+++ b/tensorflow/compiler/xla/tools/replay_computation.cc
@@ -51,6 +51,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/execution_options_util.h"
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/service/gpu/infeed_manager.h"
+#include "tensorflow/compiler/xla/service/gpu/outfeed_manager.h"
 #include "tensorflow/compiler/xla/service/hlo.pb.h"
 #include "tensorflow/compiler/xla/service/hlo_parser.h"
 #include "tensorflow/compiler/xla/shape_util.h"
@@ -73,7 +74,17 @@ namespace {
 // fields.
 struct Options {
   string fake_infeed_shape;
-  bool generate_fake_infeed = false;
+  string fake_outfeed_shape;
+
+  // generate_fake_infeed == true is a safe default: If the model has 0 or 1
+  // infeeds, then it will work like normal.  If the model has more than one
+  // infeed, it will be an error, but that wouldn't have worked anyway if you
+  // hadn't passed generate_fake_infeed.
+  //
+  // Same for generate_fake_outfeed.
+  bool generate_fake_infeed = true;
+  bool generate_fake_outfeed = true;
+
   bool use_fake_data = false;
   bool print_result = true;
   int num_runs = 1;
@@ -96,6 +107,83 @@ std::unique_ptr<LocalExecutable> CompileExecutable(const HloSnapshot& module,
       .ValueOrDie();
 }
 
+absl::optional<Shape> GetXfeedShape(bool is_infeed,
+                                    const HloModuleProto& module,
+                                    const Options& opts) {
+  std::vector<HloInstructionProto> xfeed_instrs;
+  for (const auto& comp : module.computations()) {
+    for (const auto& instruction : comp.instructions()) {
+      if (instruction.opcode() == HloOpcodeString(is_infeed
+                                                      ? HloOpcode::kInfeed
+                                                      : HloOpcode::kOutfeed)) {
+        xfeed_instrs.push_back(instruction);
+      }
+    }
+  }
+
+  auto log_xfeed_instrs = [&] {
+    for (const auto& infeed : xfeed_instrs) {
+      LOG(ERROR) << "  " << ShapeUtil::HumanString(Shape(infeed.shape())) << " "
+                 << infeed.name();
+    }
+  };
+
+  auto find_instruction_from_id_or_die = [&](int64 id) {
+    for (const auto& comp : module.computations()) {
+      for (const auto& instruction : comp.instructions()) {
+        if (instruction.id() == id) {
+          return instruction;
+        }
+      }
+    }
+    LOG(FATAL) << "No instruction with id " << id;
+  };
+
+  absl::optional<Shape> xfeed_shape;
+  string xfeed_name = is_infeed ? "infeed" : "outfeed";
+  string fake_xfeed_shape =
+      is_infeed ? opts.fake_infeed_shape : opts.fake_outfeed_shape;
+  bool generate_fake_xfeed =
+      is_infeed ? opts.generate_fake_infeed : opts.generate_fake_outfeed;
+  if (!fake_xfeed_shape.empty()) {
+    xfeed_shape = std::move(ParseShape(fake_xfeed_shape)).ValueOrDie();
+  } else if (generate_fake_xfeed) {
+    CHECK_LT(xfeed_instrs.size(), 2)
+        << "--generate_fake_" << xfeed_name
+        << " only works if the model has 0 or 1 " << xfeed_name << " ops.";
+    if (xfeed_instrs.empty()) {
+      LOG(INFO) << "Not generating fake " << xfeed_name
+                << " shape; model has no " << xfeed_name << "s.";
+    } else if (xfeed_instrs.size() == 1) {
+      // kInfeed instructions should have a shape (buffer, token).  kOutfeed
+      // instructions should have operand 0 of shape `buffer`. We want to xfeed
+      // just `buffer`.
+      xfeed_shape = is_infeed
+                        ? Shape(xfeed_instrs.front().shape()).tuple_shapes(0)
+                        : Shape(find_instruction_from_id_or_die(
+                                    xfeed_instrs.front().operand_ids(0))
+                                    .shape());
+      LOG(INFO) << "Generating fake " << xfeed_name << " with inferred shape: "
+                << ShapeUtil::HumanString(*xfeed_shape);
+    } else {
+      LOG(ERROR) << "--generate_fake_" << xfeed_name
+                 << " only works if the model has 0 or 1 " << xfeed_name
+                 << " ops, but this model has " << xfeed_instrs.size()
+                 << " of them:";
+      log_xfeed_instrs();
+      LOG(FATAL) << "Can't run model with --generate_fake_infeed.";
+    }
+  } else if (!xfeed_instrs.empty()) {
+    LOG(ERROR) << "Model contains " << xfeed_instrs.size() << " " << xfeed_name
+               << " instruction(s), but neither --generate_fake_" << xfeed_name
+               << " nor --fake_" << xfeed_name
+               << "_shape was specified.  Execution will likely hang.";
+    log_xfeed_instrs();
+  }
+
+  return xfeed_shape;
+}
+
 // Invokes the given computation passing arbitrary data for every (unbound)
 // parameter if use_fake_data, Otherwise use recorded data if available.
 //
@@ -142,56 +230,37 @@ StatusOr<Literal> ReplayComputation(const HloSnapshot& module,
     }
   }
 
-  bool provide_infeed = false;
-  Shape infeed_shape;
-  if (!opts.fake_infeed_shape.empty()) {
-    StatusOr<Shape> shape_status = ParseShape(opts.fake_infeed_shape);
-    TF_CHECK_OK(shape_status.status());
-    infeed_shape = std::move(shape_status).ValueOrDie();
-    provide_infeed = true;
-  } else if (opts.generate_fake_infeed) {
-    for (const auto& comp : computation.proto().computations()) {
-      for (const auto& instruction : comp.instructions()) {
-        if (instruction.opcode() == HloOpcodeString(HloOpcode::kInfeed)) {
-          CHECK(!provide_infeed)
-              << "--generate_fake_infeed only works if the model has 0 or 1 "
-                 "infeed ops, but this one has >= 2.";
-          provide_infeed = true;
-          // The kInfeed instruction should have a shape (buffer, token).  We
-          // want to infeed just `buffer`.
-          infeed_shape = Shape(instruction.shape()).tuple_shapes(0);
-          LOG(INFO) << "Generating fake infeed shape for inferred shape: "
-                    << ShapeUtil::HumanString(infeed_shape);
-        }
-      }
-    }
+  if (absl::optional<Shape> infeed_shape = GetXfeedShape(
+          /*is_infeed=*/true, computation.proto(), opts)) {
+    auto infeed_data = std::make_shared<Literal>(
+        std::move(MakeFakeLiteral(*infeed_shape)).ValueOrDie());
+    xla::gpu::GetOrCreateInfeedManager()
+        ->RegisterBeforeGetNextDestinationCallback([infeed_data, client] {
+          TF_CHECK_OK(client->TransferToInfeed(*infeed_data));
+        });
   }
-  // We only instantiate the thread pool if the user has requested that a
-  // concurrent infeed occur via the fake_infeed_shape, or when
-  // --generate_fake_infeed is passed and there exists an infeed operation in
-  // the HloSnapshot.
-  absl::optional<tensorflow::thread::ThreadPool> pool;
-  Literal data;
-  if (provide_infeed) {
-    data = std::move(MakeFakeLiteral(infeed_shape)).ValueOrDie();
-  }
-  auto transfer_infeed = [&data, client]() {
-    TF_CHECK_OK(client->TransferToInfeed(data));
-  };
-  if (provide_infeed) {
-    pool.emplace(tensorflow::Env::Default(), "infeed",
-                 /*num_threads=*/1);
-    pool->Schedule([transfer_infeed]() {
-      // There may be several infeed buffers needed, however we don't know how
-      // many. If we proactively transfer too many infeed buffers, we may run
-      // out of memory. If we transfer too few infeed buffers, the program will
-      // hang. Therefore, we register a callback that is called when the infeed
-      // becomes empty, and in this callback we will transfer another fake
-      // infeed.
-      auto infeed_manager = xla::gpu::GetOrCreateInfeedManager();
-      infeed_manager->RegisterOnEmptyCallback(transfer_infeed);
-      transfer_infeed();
-    });
+
+  absl::optional<tensorflow::thread::ThreadPool> outfeed_thread_pool;
+  if (absl::optional<Shape> outfeed_shape = GetXfeedShape(
+          /*is_infeed=*/false, computation.proto(), opts)) {
+    // For each an outfeed that runs, enqueue a task that will consume it.  We
+    // need a thread pool because the act of running an outfeed blocks on there
+    // being a destination available, and the act of making a destination
+    // available blocks on there being outfeed data available.
+    outfeed_thread_pool.emplace(tensorflow::Env::Default(), "infeed",
+                                /*num_threads=*/1);
+    auto consume_outfeed = [client, outfeed_shape] {
+      TF_CHECK_OK(
+          client->TransferFromOutfeedLocal(*outfeed_shape, /*device_ordinal=*/0)
+              .status());
+      VLOG(1) << "Received outfeed data of shape "
+              << ShapeUtil::HumanStringWithLayout(*outfeed_shape);
+    };
+    xla::gpu::GetOrCreateOutfeedManager()
+        ->RegisterBeforeGetNextDestinationCallback(
+            [consume_outfeed, &outfeed_thread_pool] {
+              outfeed_thread_pool->Schedule(consume_outfeed);
+            });
   }
 
   // Do not attempt to run the executable if num_runs is less than 1.
@@ -306,8 +375,10 @@ int RealMain(absl::Span<char* const> args, const Options& opts) {
 
   for (int64 i = 0; i < executables.size(); ++i) {
     LocalExecutable* executable = executables[i].get();
+    LOG(ERROR) << "Running iteration " << i;
     StatusOr<Literal> result_status =
         ReplayComputation(snapshots[i], executable, client, opts);
+    LOG(ERROR) << "iteration complete.";
     if (!result_status.ok()) {
       fprintf(stderr, "%s: error: %s\n", args[i],
               result_status.status().ToString().c_str());
@@ -352,9 +423,14 @@ int main(int argc, char** argv) {
                        "Number of times to run each computation"),
       tensorflow::Flag("fake_infeed_shape", &opts.fake_infeed_shape,
                        "Shape of fake data to construct for (infinite) infeed"),
+      tensorflow::Flag("fake_outfeed_shape", &opts.fake_outfeed_shape,
+                       "Shape of fake data to outfeed from computation"),
       tensorflow::Flag("generate_fake_infeed", &opts.generate_fake_infeed,
-                       "Whether a fake infeed shape should be generated "
-                       "derived from the computation"),
+                       "Whether a fake infeed shape should be derived "
+                       "from the computation"),
+      tensorflow::Flag("generate_fake_outfeed", &opts.generate_fake_outfeed,
+                       "Whether a fake outfeed shape should be derived "
+                       "from the computation"),
   };
   xla::string usage = tensorflow::Flags::Usage(argv[0], flag_list);
   bool parse_ok = tensorflow::Flags::Parse(&argc, argv, flag_list);
-- 
GitLab


From 0eca4f6281579e4db607429c9dd97c9146008d65 Mon Sep 17 00:00:00 2001
From: Peter Ma <pcma@google.com>
Date: Fri, 11 Jan 2019 17:31:01 -0800
Subject: [PATCH 0605/2345] Readability improvement on graph_properties: - Use
 'ic' for InferenceContext*, and 'ctx' for NodeContext* to be consistent
 within graph_properties (there are places where 'c' is used for either one in
 short functions). - The inner loop for iterating over input nodes in
 UpdateNode is redundant, and thus removed. - UpdateMergeNode is renamed to
 UpdateMerge in consistence with other Update functions; - Some other minor
 changes.

PiperOrigin-RevId: 228972916
---
 tensorflow/core/grappler/costs/BUILD          |   2 -
 .../core/grappler/costs/graph_properties.cc   | 298 ++++++++----------
 .../core/grappler/costs/graph_properties.h    |   4 +-
 3 files changed, 142 insertions(+), 162 deletions(-)

diff --git a/tensorflow/core/grappler/costs/BUILD b/tensorflow/core/grappler/costs/BUILD
index 92294379b5..35ca93d934 100644
--- a/tensorflow/core/grappler/costs/BUILD
+++ b/tensorflow/core/grappler/costs/BUILD
@@ -41,8 +41,6 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":utils",
-        "@com_google_absl//absl/memory",
-        "@com_google_absl//absl/strings",
         "//tensorflow/core/grappler/utils:functions",
         "//tensorflow/core/grappler/utils:topological_sort",
         "//tensorflow/core/grappler:mutable_graph_view",
diff --git a/tensorflow/core/grappler/costs/graph_properties.cc b/tensorflow/core/grappler/costs/graph_properties.cc
index e869f7830c..9f5966c4af 100644
--- a/tensorflow/core/grappler/costs/graph_properties.cc
+++ b/tensorflow/core/grappler/costs/graph_properties.cc
@@ -15,12 +15,6 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/costs/graph_properties.h"
 
-#include <limits>
-#include <list>
-#include <queue>
-#include <unordered_map>
-#include <unordered_set>
-#include "absl/memory/memory.h"
 #include "tensorflow/core/framework/common_shape_fns.h"
 #include "tensorflow/core/framework/function.pb.h"
 #include "tensorflow/core/framework/node_def_util.h"
@@ -44,7 +38,6 @@ limitations under the License.
 
 namespace tensorflow {
 namespace grappler {
-using TensorVector = gtl::InlinedVector<TensorValue, 4>;
 
 namespace {
 
@@ -52,6 +45,7 @@ using shape_inference::DimensionHandle;
 using shape_inference::InferenceContext;
 using shape_inference::ShapeAndType;
 using shape_inference::ShapeHandle;
+using TensorVector = gtl::InlinedVector<TensorValue, 4>;
 
 template <typename Handle>
 struct HashHandle {
@@ -416,6 +410,7 @@ NodeDef MakeConstNodeDefFromShape(InferenceContext* ic,
   return MakeConstNodeDefFromTensorProto(
       ic, MakeTensorProtoFromShape(ic, shape, tensor_as_shape, dtype), dtype);
 }
+
 }  // namespace
 
 // Queue of nodes to process. Nodes can be enqueued in any order, but will be
@@ -607,8 +602,8 @@ class SymbolicShapeRefiner {
                                           " was not found in the graph.");
       }
 
-      InferenceContext* input_inference_context = GetContext(input_node);
-      if (input_inference_context == nullptr) {
+      InferenceContext* input_ic = GetContext(input_node);
+      if (input_ic == nullptr) {
         return errors::FailedPrecondition(
             "Inference context has not been created for ", input_tensor.node());
       }
@@ -616,8 +611,8 @@ class SymbolicShapeRefiner {
       int output_port_num = input_tensor.index();
       AttrValue attr_output_shape;
       TensorShapeProto proto;
-      const auto& handle = input_inference_context->output(output_port_num);
-      input_inference_context->ShapeHandleToProto(handle, &proto);
+      const auto& handle = input_ic->output(output_port_num);
+      input_ic->ShapeHandleToProto(handle, &proto);
       // There may be dim.size < -1 in SymbolicShapeRefiner. Change those to -1.
       for (int i = 0; i < proto.dim_size(); i++) {
         if (proto.dim(i).size() < -1) {
@@ -715,145 +710,132 @@ class SymbolicShapeRefiner {
     return Status::OK();
   }
 
+  // Prepares input shapes/values/handles, then runs shape inference, and
+  // finally sets output shapes/values/handles.
   Status UpdateNode(const NodeDef* node, bool* refined) {
-    NodeContext* node_context = GetNodeContext(node);
-    if (node_context == nullptr) {
+    NodeContext* ctx = GetNodeContext(node);
+    if (ctx == nullptr) {
       TF_RETURN_IF_ERROR(AddNode(node));
-      node_context = CHECK_NOTNULL(GetNodeContext(node));
+      ctx = CHECK_NOTNULL(GetNodeContext(node));
       *refined = true;
     }
 
     // Check if the shapes of the nodes in the fan-in of this node have changed,
     // and if they have, update the node input shapes.
-    InferenceContext* inference_context = node_context->inference_context.get();
-    std::vector<Tensor> const_values(inference_context->num_inputs());
-    std::vector<const Tensor*> input_tensors(inference_context->num_inputs(),
-                                             nullptr);
-    std::vector<ShapeHandle> input_tensors_as_shapes(
-        inference_context->num_inputs());
-    node_context->input_tensor_protos.resize(inference_context->num_inputs(),
-                                             nullptr);
-
-    for (int dst_input = 0; dst_input < inference_context->num_inputs();
-         ++dst_input) {
-      GraphView::InputPort port(node, dst_input);
-      for (const GraphView::OutputPort fanin : graph_.GetFanin(port)) {
-        int src_output = fanin.port_id;
-        const NodeDef* input = fanin.node;
-        NodeContext* c = GetNodeContext(input);
-        if (c == nullptr) {
-          return errors::FailedPrecondition(
-              "Input ", dst_input, " ('", input->name(), "') for '",
-              node->name(),
-              "' was not previously added to SymbolicShapeRefiner.");
-        }
+    InferenceContext* ic = ctx->inference_context.get();
+    std::vector<Tensor> const_values(ic->num_inputs());
+    std::vector<const Tensor*> input_tensors(ic->num_inputs(), nullptr);
+    std::vector<ShapeHandle> input_tensors_as_shapes(ic->num_inputs());
+    ctx->input_tensor_protos.resize(ic->num_inputs(), nullptr);
+
+    for (int dst_input = 0; dst_input < ic->num_inputs(); ++dst_input) {
+      const GraphView::InputPort port(node, dst_input);
+      const GraphView::OutputPort fanin = graph_.GetRegularFanin(port);
+      int src_output = fanin.port_id;
+      const NodeDef* src = fanin.node;
+      NodeContext* src_ctx = GetNodeContext(src);
+      InferenceContext* src_ic = src_ctx->inference_context.get();
+      if (src_ctx == nullptr) {
+        return errors::FailedPrecondition(
+            "Input ", dst_input, " ('", src->name(), "') for '", node->name(),
+            "' was not previously added to SymbolicShapeRefiner.");
+      }
 
-        if (src_output >= c->inference_context->num_outputs())
-          return errors::OutOfRange("src_output = ", src_output,
-                                    ", but num_outputs is only ",
-                                    c->inference_context->num_outputs());
+      if (src_output >= src_ic->num_outputs()) {
+        return errors::OutOfRange("src_output = ", src_output,
+                                  ", but num_outputs is only ",
+                                  src_ic->num_outputs());
+      }
 
-        // Propagate input node's NodeContext info to the current node's
-        // NodeContext:
-        // output_tensor_protos to input_tensor_protos and input_tensors, and
-        // output_tensors_as_shapes to input_tensors_as_shapes.
+      // Propagate input node's NodeContext info to the current node's
+      // NodeContext:
+      // output_tensor_protos to input_tensor_protos and input_tensors, and
+      // output_tensors_as_shapes to input_tensors_as_shapes.
 
-        if (c->output_tensors_as_shapes.size() > src_output) {
-          input_tensors_as_shapes[dst_input] =
-              c->output_tensors_as_shapes[src_output];
-        }
+      if (src_ctx->output_tensors_as_shapes.size() > src_output) {
+        input_tensors_as_shapes[dst_input] =
+            src_ctx->output_tensors_as_shapes[src_output];
+      }
 
-        if (c->output_tensor_protos.size() > src_output) {
-          auto* tensor_proto = c->output_tensor_protos[src_output];
-          if (tensor_proto != nullptr &&
-              const_values[dst_input].FromProto(*tensor_proto)) {
-            input_tensors[dst_input] = &const_values[dst_input];
-            node_context->input_tensor_protos[dst_input] = tensor_proto;
-
-            if (!inference_context->FullyDefined(
-                    input_tensors_as_shapes[dst_input])) {
-              // Shape from a Const is not fully defined when the Const has
-              // value -1 (e.g., Reshape(x, Const(-1)) to reshape an arbitrary
-              // tensor x to a vector).
-              // It's possible that the same Const with -1 is used in many
-              // places, but that doesn't mean the resultant shapes are
-              // identical. e.g., x1 = Reshape(x, c) and y1 = Reshape(y, c),
-              // where c is -1. In this case, shape inference yields both x1 and
-              // y1 as rank 1, size unknown, but still the shapes of x1 and y1
-              // can be different. (even if we use different Const(-1) for x1
-              // and x2, graph optimzier may merge them to single Const through
-              // duplicate removal.)
-              // If we reuse output_tensors_as_shapes to input_tensors_as_shapes
-              // by copying ShapeHandle, they share the same Shape object, and
-              // SymbolicShapeManager, later in InferStatically(), assigns the
-              // same symbolic dim value (unique value < -1); in the above
-              // Reshape example, the shapes of x1 and y1 become, for example,
-              // [-278] and graph optimizer may yield incorrect output 'cause it
-              // assumes x1 and y1 have the same shape.
-              // To prevent this, we re-create a ShapeHandle from the Const
-              // tensor, instead of reusing output_tensors_as_shapes (so that
-              // ShapeHandles of the const fanouts have the same values,
-              // but different Shape objects -- SymbolicShapeManager assigns
-              // different symbol id to each fanout shape).
-              // TODO(dyoon): clean up the way values are propagated.
-              MaybeTensorValueToShape(inference_context,
-                                      const_values[dst_input],
-                                      &input_tensors_as_shapes[dst_input]);
-            }
+      if (src_ctx->output_tensor_protos.size() > src_output) {
+        auto* tensor_proto = src_ctx->output_tensor_protos[src_output];
+        if (tensor_proto != nullptr &&
+            const_values[dst_input].FromProto(*tensor_proto)) {
+          input_tensors[dst_input] = &const_values[dst_input];
+          ctx->input_tensor_protos[dst_input] = tensor_proto;
+
+          if (!ic->FullyDefined(input_tensors_as_shapes[dst_input])) {
+            // Shape from a Const is not fully defined when the Const has
+            // value -1 (e.g., Reshape(x, Const(-1)) to reshape an arbitrary
+            // tensor x to a vector).
+            // It's possible that the same Const with -1 is used in many
+            // places, but that doesn't mean the resultant shapes are
+            // identical. e.g., x1 = Reshape(x, c) and y1 = Reshape(y, c),
+            // where c is -1. In this case, shape inference yields both x1 and
+            // y1 as rank 1, size unknown, but still the shapes of x1 and y1
+            // can be different. (even if we use different Const(-1) for x1
+            // and x2, graph optimzier may merge them to single Const through
+            // duplicate removal.)
+            // If we reuse output_tensors_as_shapes to input_tensors_as_shapes
+            // by copying ShapeHandle, they share the same Shape object, and
+            // SymbolicShapeManager, later in InferStatically(), assigns the
+            // same symbolic dim value (unique value < -1); in the above
+            // Reshape example, the shapes of x1 and y1 become, for example,
+            // [-278] and graph optimizer may yield incorrect output 'cause it
+            // assumes x1 and y1 have the same shape.
+            // To prevent this, we re-create a ShapeHandle from the Const
+            // tensor, instead of reusing output_tensors_as_shapes (so that
+            // ShapeHandles of the const fanouts have the same values,
+            // but different Shape objects -- SymbolicShapeManager assigns
+            // different symbol id to each fanout shape).
+            // TODO(dyoon): clean up the way values are propagated.
+            MaybeTensorValueToShape(ic, const_values[dst_input],
+                                    &input_tensors_as_shapes[dst_input]);
           }
         }
+      }
 
-        DCHECK_GE(dst_input, 0);
-        // NOTE: we check only shape is refined; we do not (yet) check whether
-        // tensor value is refined.
-        if (!*refined && !inference_context->input(dst_input).SameHandle(
-                             c->inference_context->output(src_output))) {
-          *refined = true;
-        }
-        inference_context->SetInput(dst_input,
-                                    c->inference_context->output(src_output));
-
-        if (!*refined &&
-            inference_context->requested_input_tensor_as_partial_shape(
-                dst_input)) {
-          // The input value may have changed. Since we have no way to know if
-          // that's indeed the case, err on the safe side.
-          *refined = true;
-        }
+      // NOTE: we check only shape is refined; we do not (yet) check whether
+      // tensor value is refined.
+      if (!*refined &&
+          !ic->input(dst_input).SameHandle(src_ic->output(src_output))) {
+        *refined = true;
+      }
+      ic->SetInput(dst_input, src_ic->output(src_output));
 
-        // Also propagate handle shape and dtype of edges which are carrying
-        // resource handles.
-        if (node_context->input_types[dst_input] == DT_RESOURCE) {
-          auto* outputs =
-              c->inference_context->output_handle_shapes_and_types(src_output);
-          if (!outputs) continue;
-          auto* inputs =
-              inference_context->input_handle_shapes_and_types(dst_input);
-
-          if (!inputs || !EquivalentShapesAndTypes(*outputs, *inputs)) {
-            *refined = true;
-          }
-          inference_context->set_input_handle_shapes_and_types(dst_input,
-                                                               *outputs);
-        }
+      if (!*refined && ic->requested_input_tensor_as_partial_shape(dst_input)) {
+        // The input value may have changed. Since we have no way to know if
+        // that's indeed the case, err on the safe side.
+        *refined = true;
+      }
+
+      // Also propagate handle shape and dtype of edges which are carrying
+      // resource handles.
+      if (ctx->input_types[dst_input] == DT_RESOURCE) {
+        auto* outputs = src_ic->output_handle_shapes_and_types(src_output);
+        if (!outputs) continue;
+        auto* inputs = ic->input_handle_shapes_and_types(dst_input);
+
+        if (!inputs || !EquivalentShapesAndTypes(*outputs, *inputs))
+          *refined = true;
+        ic->set_input_handle_shapes_and_types(dst_input, *outputs);
       }
     }
 
     // Make sure we schedule the fanout of resources (which have no input)
     // whenever the resources are updated.
-    *refined |= inference_context->num_inputs() == 0;
+    *refined |= ic->num_inputs() == 0;
 
     if (!*refined) {
       // No input shape has changed, we're done.
       return Status::OK();
     }
 
-    node_context->inference_context->set_input_tensors(input_tensors);
-    node_context->inference_context->set_input_tensors_as_shapes(
-        input_tensors_as_shapes);
+    ic->set_input_tensors(input_tensors);
+    ic->set_input_tensors_as_shapes(input_tensors_as_shapes);
 
     // Properly handle function nodes.
-    if (node_context->op_data && node_context->op_data->is_function_op) {
+    if (ctx->op_data && ctx->op_data->is_function_op) {
       // TODO(jmdecker): Detect if the input shapes have changed for this
       // function. Note that when we hit a function call node, refined will be
       // true, as the updates to the call node will have changed, even if it's
@@ -870,7 +852,7 @@ class SymbolicShapeRefiner {
     }
 
     // Update the shapes of the outputs.
-    return InferShapes(*node, node_context);
+    return InferShapes(*node, ctx);
   }
 
   Status SetUnknownShape(const NodeDef* node, int output_port) {
@@ -1681,45 +1663,45 @@ Status GraphProperties::RelaxEnqueueShapesAndMergeTypes(
 
 // Compute the output shape of the merge node as the union of the available
 // input shapes.
-Status GraphProperties::UpdateMergeNode(SymbolicShapeRefiner* shape_refiner,
-                                        const NodeDef* node,
-                                        bool* new_shapes) const {
-  InferenceContext* c = shape_refiner->GetContext(node);
-  if (!c) {
+Status GraphProperties::UpdateMerge(SymbolicShapeRefiner* shape_refiner,
+                                    const NodeDef* node,
+                                    bool* new_shapes) const {
+  InferenceContext* ic = shape_refiner->GetContext(node);
+  if (!ic) {
     // Now we can run shape inference
     TF_RETURN_IF_ERROR(shape_refiner->AddNode(node));
-    c = CHECK_NOTNULL(shape_refiner->GetContext(node));
+    ic = CHECK_NOTNULL(shape_refiner->GetContext(node));
     *new_shapes = true;
 
     // Infer the shape of the second output once and for all since it never
     // changes.
-    ShapeHandle out1 = c->Scalar();
-    c->set_output(1, out1);
+    ShapeHandle out1 = ic->Scalar();
+    ic->set_output(1, out1);
   }
 
   ShapeHandle out;
   bool out_initialized = false;
-  for (const GraphView::Edge fanin :
-       shape_refiner->graph().GetFaninEdges(*node, false)) {
-    InferenceContext* in = shape_refiner->GetContext(fanin.src.node);
-    if (!in) {
+  for (const GraphView::Edge fanin : shape_refiner->graph().GetFaninEdges(
+           *node, /*include_controlling_edges=*/false)) {
+    InferenceContext* src_ic = shape_refiner->GetContext(fanin.src.node);
+    if (!src_ic) {
       // Handling a loop for the first time, the back edge won't have any shape
       // info.
       continue;
     }
-    ShapeHandle input = in->output(fanin.src.port_id);
+    ShapeHandle input = src_ic->output(fanin.src.port_id);
     CHECK_EQ(fanin.dst.node, node);
-    c->SetInput(fanin.dst.port_id, input);
+    ic->SetInput(fanin.dst.port_id, input);
     if (!out_initialized) {
       out_initialized = true;
       out = input;
-      continue;
+    } else {
+      out = shape_refiner->OutputAsUnion(node, 0, input, out);
     }
-    out = shape_refiner->OutputAsUnion(node, 0, input, out);
   }
 
-  if (*new_shapes || !shape_refiner->EquivalentShapes(out, c->output(0))) {
-    c->set_output(0, out);
+  if (*new_shapes || !shape_refiner->EquivalentShapes(out, ic->output(0))) {
+    ic->set_output(0, out);
     *new_shapes = true;
   }
 
@@ -1729,26 +1711,26 @@ Status GraphProperties::UpdateMergeNode(SymbolicShapeRefiner* shape_refiner,
 // Manually propagate the input shape for Enter nodes.
 Status GraphProperties::UpdateEnter(SymbolicShapeRefiner* shape_refiner,
                                     const NodeDef* node, bool* new_shapes) {
-  auto enter_ctx = shape_refiner->GetContext(node);
-  if (!enter_ctx) {
+  InferenceContext* ic = shape_refiner->GetContext(node);
+  if (!ic) {
     TF_RETURN_IF_ERROR(shape_refiner->UpdateNode(node, new_shapes));
-    enter_ctx = shape_refiner->GetContext(node);
+    ic = shape_refiner->GetContext(node);
   }
 
-  GraphView::InputPort inp(node, 0);
-  GraphView::OutputPort fanin = shape_refiner->graph().GetRegularFanin(inp);
+  GraphView::InputPort port(node, 0);
+  GraphView::OutputPort fanin = shape_refiner->graph().GetRegularFanin(port);
 
-  InferenceContext* in = shape_refiner->GetContext(fanin.node);
-  ShapeHandle input = in->output(fanin.port_id);
-  if (!enter_ctx->output(0).SameHandle(input)) {
-    enter_ctx->SetInput(0, input);
-    enter_ctx->set_output(0, input);
+  InferenceContext* src_ic = shape_refiner->GetContext(fanin.node);
+  ShapeHandle input = src_ic->output(fanin.port_id);
+  if (!ic->output(0).SameHandle(input)) {
+    ic->SetInput(0, input);
+    ic->set_output(0, input);
     *new_shapes = true;
   }
-  auto* outputs = in->output_handle_shapes_and_types(fanin.port_id);
+  auto* outputs = src_ic->output_handle_shapes_and_types(fanin.port_id);
   if (outputs) {
-    enter_ctx->set_input_handle_shapes_and_types(0, *outputs);
-    enter_ctx->set_output_handle_shapes_and_types(0, *outputs);
+    ic->set_input_handle_shapes_and_types(0, *outputs);
+    ic->set_output_handle_shapes_and_types(0, *outputs);
     *new_shapes = true;
   }
   return Status::OK();
@@ -1764,7 +1746,7 @@ Status GraphProperties::UpdateShapes(
     TF_RETURN_IF_ERROR(UpdateEnter(shape_refiner, n, new_shapes));
   } else if (IsMerge(*n)) {
     // Properly handle merge nodes.
-    TF_RETURN_IF_ERROR(UpdateMergeNode(shape_refiner, n, new_shapes));
+    TF_RETURN_IF_ERROR(UpdateMerge(shape_refiner, n, new_shapes));
   } else if (IsEnqueue(*n)) {
     // Make sure the shapes of enqueued tensors are propagated to the queue
     // itself.
@@ -1811,8 +1793,8 @@ Status GraphProperties::PropagateShapes(
       TF_RETURN_IF_ERROR(
           UpdateShapes(shape_refiner, resource_handles, n, &updated));
       if (updated) {
-        for (const GraphView::InputPort& fanout :
-             shape_refiner->graph().GetFanouts(*n, false)) {
+        for (const auto& fanout : shape_refiner->graph().GetFanouts(
+                 *n, /*include_controlled_nodes=*/false)) {
           new_shapes->push(fanout.node);
         }
         // Make sure the corresponding queue nodes are (re)processed.
@@ -1837,7 +1819,7 @@ Status GraphProperties::PropagateShapes(
 Status GraphProperties::UpdateQueue(const NodeDef* queue_node,
                                     SymbolicShapeRefiner* shape_refiner,
                                     bool* new_shapes) {
-  auto ctx = shape_refiner->GetNodeContext(queue_node);
+  auto* ctx = shape_refiner->GetNodeContext(queue_node);
   if (!ctx) {
     TF_RETURN_IF_ERROR(shape_refiner->AddNode(queue_node));
     ctx = CHECK_NOTNULL(shape_refiner->GetNodeContext(queue_node));
diff --git a/tensorflow/core/grappler/costs/graph_properties.h b/tensorflow/core/grappler/costs/graph_properties.h
index 0635222fe9..3fcad6eb1b 100644
--- a/tensorflow/core/grappler/costs/graph_properties.h
+++ b/tensorflow/core/grappler/costs/graph_properties.h
@@ -111,8 +111,8 @@ class GraphProperties {
 
   // Update the output shapes of a Merge node, and enqueue its fanout in
   // new_shapes if needed.
-  Status UpdateMergeNode(SymbolicShapeRefiner* shape_refiner,
-                         const NodeDef* node, bool* new_shapes) const;
+  Status UpdateMerge(SymbolicShapeRefiner* shape_refiner, const NodeDef* node,
+                     bool* new_shapes) const;
   // Process the Enter node, and enqueue its fanout in new_shapes if needed.
   static Status UpdateEnter(SymbolicShapeRefiner* shape_refiner,
                             const NodeDef* node, bool* new_shapes);
-- 
GitLab


From 28093291532750e8f1529938f1a44d3b6eceb3b1 Mon Sep 17 00:00:00 2001
From: Aurelien Geron <aurelien.geron@gmail.com>
Date: Sat, 12 Jan 2019 09:40:46 +0800
Subject: [PATCH 0606/2345] Fix typo, renamed _setup_concrete_functions()

---
 tensorflow/python/saved_model/load.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
index b86e2f0aec..b30d392099 100644
--- a/tensorflow/python/saved_model/load.py
+++ b/tensorflow/python/saved_model/load.py
@@ -51,7 +51,7 @@ class _Loader(object):
     self._setup_concrete_functions()
     self._restore_checkpoint()
 
-  def _bind_function_captures(self):
+  def _setup_concrete_functions(self):
     """Setup captured tensors and output structure in restored concrete
     functions."""
     seen_concrete_functions = set()
-- 
GitLab


From c6bb5cd0447a0af2764c195fb14d218df8ae6471 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 17:40:46 -0800
Subject: [PATCH 0607/2345] Fix Windows build by undefining unwanted ERROR
 macro.

PiperOrigin-RevId: 228974009
---
 tensorflow/stream_executor/cuda/cuda_gpu_executor.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
index 4874d096ad..5b33d97594 100644
--- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
+++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
@@ -53,6 +53,10 @@ limitations under the License.
 #include "tensorflow/stream_executor/stream_executor_pimpl.h"
 #include "tensorflow/stream_executor/timer.h"
 
+// LOG(ERROR) uses a const named ERROR, so a macro with the same name is
+// always unwanted. This happens on Windows that defines such a macro.
+#undef ERROR
+
 #ifdef PLATFORMS_GPUS_CUDA_DYNAMIC_LIBCUDA_DYNAMIC_LIBCUDA_H_
 #error \
     "No driver calls in this file, wrap driver functionality in cuda_driver.cc."
-- 
GitLab


From f6d74c3653a8a2c3c1bdcbe09be04b97b3f952cb Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Fri, 11 Jan 2019 17:43:18 -0800
Subject: [PATCH 0608/2345] [Resubmit][TF2XLA] Use dynamic dimension in SizeOp.

This is needed if the number of elements in a tensor can be dynamic.

Resubmit after xla_builder support constant-folding get_dynamic_size.

PiperOrigin-RevId: 228974238
---
 tensorflow/compiler/tf2xla/kernels/shape_op.cc | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/shape_op.cc b/tensorflow/compiler/tf2xla/kernels/shape_op.cc
index 12830816ec..85b0367f73 100644
--- a/tensorflow/compiler/tf2xla/kernels/shape_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/shape_op.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_helpers.h"
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
+#include "tensorflow/compiler/xla/client/lib/constants.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
 #include "tensorflow/core/framework/tensor_shape.h"
@@ -91,14 +92,20 @@ class SizeOp : public XlaOpKernel {
 
   void Compile(XlaOpKernelContext* ctx) override {
     const TensorShape input_shape = ctx->InputShape(0);
-    const int64 size = input_shape.num_elements();
-    OP_REQUIRES(ctx, FastBoundsCheck(size, std::numeric_limits<int32>::max()),
+    OP_REQUIRES(ctx,
+                FastBoundsCheck(input_shape.num_elements(),
+                                std::numeric_limits<int32>::max()),
                 errors::InvalidArgument("Size does not work for tensors > "
                                         "int32 max."));
     Tensor size_constant(DT_INT32, TensorShape({}));
-    size_constant.scalar<int32>()() = static_cast<int32>(size);
-
-    ctx->SetConstantOutput(0, size_constant);
+    const int rank = input_shape.dims();
+    xla::XlaBuilder* builder = ctx->builder();
+    auto size = xla::One(builder, xla::U32);
+    for (int64 i = 0; i < rank; ++i) {
+      size = xla::Mul(size, xla::GetDimensionSize(ctx->Input(0), i));
+    }
+    size = xla::ConvertElementType(size, xla::S32);
+    ctx->SetOutput(0, size);
   }
 };
 
-- 
GitLab


From 706cce3f2395806757fc9f3ffeb07c132f13c993 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 11 Jan 2019 17:49:28 -0800
Subject: [PATCH 0609/2345] [XLA] Speed up reduce_window_test by using iota
 rather than generating random numbers.

1.3x faster on some of the large testcases.

PiperOrigin-RevId: 228974823
---
 tensorflow/compiler/xla/array.h               | 13 ++++++--
 .../compiler/xla/tests/reduce_window_test.cc  | 31 +++++++++++--------
 2 files changed, 29 insertions(+), 15 deletions(-)

diff --git a/tensorflow/compiler/xla/array.h b/tensorflow/compiler/xla/array.h
index 58cc157585..529e7f77ce 100644
--- a/tensorflow/compiler/xla/array.h
+++ b/tensorflow/compiler/xla/array.h
@@ -272,6 +272,15 @@ class Array {
     std::iota(&values_[0], &values_[0] + num_elements(), value);
   }
 
+  // Fills the array with a repeating sequence:
+  //   [value, value + 1, ..., value + length - 1, value, ... ]
+  void FillRepeatedIota(const T& value, int64 length) {
+    for (int64 i = 0; i < num_elements(); i += length) {
+      std::iota(&values_[i], &values_[std::min(i + length, num_elements())],
+                value);
+    }
+  }
+
   // Fills the array with the sequence i*multiplier for i=0,1,...
   void FillWithMultiples(const T& multiplier) {
     for (int64 i = 0; i < num_elements(); ++i) {
@@ -280,11 +289,11 @@ class Array {
   }
 
   // Fills the array with random normal variables with the specified mean.
-  void FillRandom(const T& value, const double mean = 0.0,
+  void FillRandom(const T& stddev, const double mean = 0.0,
                   const int seed = 12345) {
     std::mt19937 g(seed);
     std::normal_distribution<double> distribution(mean,
-                                                  static_cast<double>(value));
+                                                  static_cast<double>(stddev));
     for (int64 i = 0; i < num_elements(); ++i) {
       values_[i] = static_cast<T>(distribution(g));
     }
diff --git a/tensorflow/compiler/xla/tests/reduce_window_test.cc b/tensorflow/compiler/xla/tests/reduce_window_test.cc
index 22fe4a2670..16c67d94c7 100644
--- a/tensorflow/compiler/xla/tests/reduce_window_test.cc
+++ b/tensorflow/compiler/xla/tests/reduce_window_test.cc
@@ -607,7 +607,10 @@ class R4ReduceWindowTest : public ReduceWindowTestBase,
 
     Array4D<float> input(param.base_bounds[0], param.base_bounds[1],
                          param.base_bounds[2], param.base_bounds[3]);
-    input.FillRandom(0.1f, 0.1f);
+    // Choose a prime iota length so that each window sees a unique set of
+    // values. (Technically, the requirement is that the iota length is
+    // relatively prime to all of the dimensions involved in the reduce-window.)
+    input.FillRepeatedIota(0, 137);
     Literal input_literal = LiteralUtil::CreateR4FromArray4DWithLayout(
         input, LayoutUtil::MakeLayout(param.layout));
     XlaOp parameter;
@@ -623,9 +626,9 @@ class R4ReduceWindowTest : public ReduceWindowTestBase,
         CreateConstantFromLiteral(LiteralUtil::CreateR0(kInitValue), &b);
     CHECK(param.reducer == kAdd || param.reducer == kMax);
     auto reducer = param.reducer;
-    if (use_bfloat16() && Product(param.window_bounds) > 128) {
-      // To avoid numerical issues, force the reducer to be kMax for large bf16
-      // windows.
+    if (use_bfloat16()) {
+      // To avoid numerical issues, force the reducer to be kMax for bf16
+      // inputs.
       reducer = kMax;
     }
 
@@ -949,16 +952,16 @@ struct R3ReduceWindowTestData {
      /*padding=*/Padding::kValid, /*reducer=*/Reducer::kAdd},
     {/*base_bounds=*/{95, 202, 251}, /*window_bounds=*/{95, 202, 251},
      /*strides=*/{1, 1, 1}, /*layout=*/{2, 1, 0},
-     /*padding=*/Padding::kValid, /*reducer=*/Reducer::kAdd},
+     /*padding=*/Padding::kValid, /*reducer=*/Reducer::kMax},
     {/*base_bounds=*/{999, 57, 3}, /*window_bounds=*/{999, 57, 3},
      /*strides=*/{1, 1, 1}, /*layout=*/{2, 1, 0},
      /*padding=*/Padding::kValid, /*reducer=*/Reducer::kAdd},
     {/*base_bounds=*/{178, 302, 64}, /*window_bounds=*/{178, 302, 64},
      /*strides=*/{1, 1, 1}, /*layout=*/{2, 1, 0},
-     /*padding=*/Padding::kValid, /*reducer=*/Reducer::kAdd},
+     /*padding=*/Padding::kValid, /*reducer=*/Reducer::kMax},
     {/*base_bounds=*/{63, 261, 257}, /*window_bounds=*/{63, 261, 257},
      /*strides=*/{1, 1, 1}, /*layout=*/{2, 1, 0},
-     /*padding=*/Padding::kValid, /*reducer=*/Reducer::kAdd},
+     /*padding=*/Padding::kValid, /*reducer=*/Reducer::kMax},
     {/*base_bounds=*/{10003, 10, 5}, /*window_bounds=*/{9999, 7, 3},
      /*strides=*/{1, 1, 1}, /*layout=*/{2, 1, 0},
      /*padding=*/Padding::kValid, /*reducer=*/Reducer::kAdd},
@@ -1001,17 +1004,19 @@ TEST_P(R3ReduceWindowTest, DoIt) {
   const float kInitValue = 0.0f;
   Array3D<float> input(param.base_bounds[0], param.base_bounds[1],
                        param.base_bounds[2]);
-  input.FillRandom(0.1f, 0.1f);
+  // Choose a prime iota length so that each window sees a unique set of values.
+  // (Technically, the requirement is that the iota length is relatively prime
+  // to all of the dimensions involved in the reduce-window.)
+  input.FillRepeatedIota(0, 137);
   Literal input_literal = LiteralUtil::CreateR3FromArray3DWithLayout(
       input, LayoutUtil::MakeLayout(param.layout));
   auto reducer = param.reducer;
   if (use_bfloat16()) {
     input_literal = LiteralUtil::ConvertF32ToBF16(input_literal);
-    if (Product(param.window_bounds) > 128) {
-      // To avoid numerical issues, force the reducer to be kMax for large bf16
-      // windows.
-      reducer = kMax;
-    }
+
+    // To avoid numerical issues, force the reducer to be kMax for bf16
+    // inputs.
+    reducer = kMax;
   }
 
   XlaOp parameter = Parameter(&b, 0, input_literal.shape(), "input");
-- 
GitLab


From 32f4097f8ac09039c010e59200f4f0e056990fcc Mon Sep 17 00:00:00 2001
From: Peter Ma <pcma@google.com>
Date: Fri, 11 Jan 2019 18:08:53 -0800
Subject: [PATCH 0610/2345] Modify UpdateMerge() so that it passes
 input/output_handle_shapes_and_types correctly.

PiperOrigin-RevId: 228976515
---
 .../core/grappler/costs/graph_properties.cc   |   9 +-
 .../grappler/costs/graph_properties_test.cc   |  38 +++
 .../while_loop_var_handle_op.pbtxt            | 291 ++++++++++++++++++
 3 files changed, 337 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/core/grappler/costs/graph_properties_testdata/while_loop_var_handle_op.pbtxt

diff --git a/tensorflow/core/grappler/costs/graph_properties.cc b/tensorflow/core/grappler/costs/graph_properties.cc
index 9f5966c4af..59696f39f8 100644
--- a/tensorflow/core/grappler/costs/graph_properties.cc
+++ b/tensorflow/core/grappler/costs/graph_properties.cc
@@ -1680,6 +1680,7 @@ Status GraphProperties::UpdateMerge(SymbolicShapeRefiner* shape_refiner,
   }
 
   ShapeHandle out;
+  const std::vector<ShapeAndType>* out_handle = nullptr;
   bool out_initialized = false;
   for (const GraphView::Edge fanin : shape_refiner->graph().GetFaninEdges(
            *node, /*include_controlling_edges=*/false)) {
@@ -1690,18 +1691,24 @@ Status GraphProperties::UpdateMerge(SymbolicShapeRefiner* shape_refiner,
       continue;
     }
     ShapeHandle input = src_ic->output(fanin.src.port_id);
-    CHECK_EQ(fanin.dst.node, node);
     ic->SetInput(fanin.dst.port_id, input);
+    auto* input_handle =
+        src_ic->output_handle_shapes_and_types(fanin.src.port_id);
+    if (input_handle)
+      ic->set_input_handle_shapes_and_types(fanin.dst.port_id, *input_handle);
     if (!out_initialized) {
       out_initialized = true;
       out = input;
+      out_handle = input_handle;
     } else {
+      // Note here only out, not out_handle, is modified.
       out = shape_refiner->OutputAsUnion(node, 0, input, out);
     }
   }
 
   if (*new_shapes || !shape_refiner->EquivalentShapes(out, ic->output(0))) {
     ic->set_output(0, out);
+    if (out_handle) ic->set_output_handle_shapes_and_types(0, *out_handle);
     *new_shapes = true;
   }
 
diff --git a/tensorflow/core/grappler/costs/graph_properties_test.cc b/tensorflow/core/grappler/costs/graph_properties_test.cc
index c7f53b2cde..0a7697a213 100644
--- a/tensorflow/core/grappler/costs/graph_properties_test.cc
+++ b/tensorflow/core/grappler/costs/graph_properties_test.cc
@@ -341,6 +341,44 @@ TEST_F(GraphPropertiesTest, VarHandles) {
   EXPECT_EQ(7, prop.shape().dim(1).size());
 }
 
+TEST_F(GraphPropertiesTest, WhileLoopWithVarHandleOpInput) {
+  // Test graph is first generated in python using:
+  /*
+    i0 = tf.constant(0)
+    v = tf.get_variable(initializer=i0, name='loop_var', use_resource=True)
+    def cond(i, x):
+      return i < 3
+    def body(i, x):
+      return i + 1, x + x
+    v, y = tf.while_loop(cond, body, loop_vars=[v, tf.constant(1)])
+  */
+  // and then modified by hand such that the ReadVariableOp is inside the loop
+  // body instead of outside the while loop (which is the case when constructed
+  // using the python API), such that we have the following pattern: VarHandleOp
+  // -> Enter -> Switch -> ReadVariableOp -> other parts of loop body. Note
+  // DT_RESOURCE is passed all the way until ReadVariableOp.
+  GrapplerItem item;
+  string filename = io::JoinPath(testing::TensorFlowSrcRoot(), kTestDataPath,
+                                 "while_loop_var_handle_op.pbtxt");
+  TF_CHECK_OK(ReadGraphDefFromFile(filename, &item.graph));
+  GraphProperties properties(item);
+  TF_CHECK_OK(properties.InferStatically(false));
+
+  std::vector<string> resource_nodes{
+      "loop_var",       "while/Enter",         "while/Merge", "while/Switch",
+      "while/Identity", "while/NextIteration", "while/Exit"};
+  for (const string& node : resource_nodes) {
+    const auto props = properties.GetOutputProperties(node);
+    EXPECT_GE(props.size(), 1);  // Merge has 2 outputs.
+    EXPECT_EQ("resource: []", PropToString(props[0]));
+  }
+
+  // After ReadVariableOp, the shape should be recovered.
+  const auto props = properties.GetOutputProperties("while/ReadVariableOp");
+  EXPECT_EQ(1, props.size());
+  EXPECT_EQ("int32: []", PropToString(props[0]));
+}
+
 TEST_F(GraphPropertiesTest, QueueWithOnlyDequeue_NoShapeAttr) {
   tensorflow::Scope root = tensorflow::Scope::NewRootScope();
   auto q1 = ops::FIFOQueue(root.WithOpName("Queue1"), {DataType::DT_FLOAT});
diff --git a/tensorflow/core/grappler/costs/graph_properties_testdata/while_loop_var_handle_op.pbtxt b/tensorflow/core/grappler/costs/graph_properties_testdata/while_loop_var_handle_op.pbtxt
new file mode 100644
index 0000000000..048b8a23dc
--- /dev/null
+++ b/tensorflow/core/grappler/costs/graph_properties_testdata/while_loop_var_handle_op.pbtxt
@@ -0,0 +1,291 @@
+node {
+  name: "Const"
+  op: "Const"
+  attr {
+    key: "dtype"
+    value { type: DT_INT32 }
+  }
+  attr {
+    key: "value"
+    value {
+      tensor {
+        dtype: DT_INT32
+        tensor_shape {}
+        int_val: 0
+      }
+    }
+  }
+}
+node {
+  name: "loop_var"
+  op: "VarHandleOp"
+  attr {
+    key: "_class"
+    value { list { s: "loc:@loop_var" } }
+  }
+  attr {
+    key: "container"
+    value { s: "" }
+  }
+  attr {
+    key: "dtype"
+    value { type: DT_INT32 }
+  }
+  attr {
+    key: "shape"
+    value { shape {} }
+  }
+  attr {
+    key: "shared_name"
+    value { s: "loop_var" }
+  }
+}
+node {
+  name: "Const_1"
+  op: "Const"
+  attr {
+    key: "dtype"
+    value { type: DT_INT32 }
+  }
+  attr {
+    key: "value"
+    value {
+      tensor {
+        dtype: DT_INT32
+        tensor_shape {}
+        int_val: 1
+      }
+    }
+  }
+}
+node {
+  name: "while/Enter"
+  op: "Enter"
+  input: "loop_var"
+  attr {
+    key: "T"
+    value { type: DT_RESOURCE }
+  }
+  attr {
+    key: "frame_name"
+    value { s: "while/while_context" }
+  }
+  attr {
+    key: "is_constant"
+    value { b: false }
+  }
+  attr {
+    key: "parallel_iterations"
+    value { i: 10 }
+  }
+}
+node {
+  name: "while/Enter_1"
+  op: "Enter"
+  input: "Const_1"
+  attr {
+    key: "T"
+    value { type: DT_INT32 }
+  }
+  attr {
+    key: "frame_name"
+    value { s: "while/while_context" }
+  }
+  attr {
+    key: "is_constant"
+    value { b: false }
+  }
+  attr {
+    key: "parallel_iterations"
+    value { i: 10 }
+  }
+}
+node {
+  name: "while/Merge"
+  op: "Merge"
+  input: "while/Enter"
+  input: "while/NextIteration"
+  attr {
+    key: "N"
+    value { i: 2 }
+  }
+  attr {
+    key: "T"
+    value { type: DT_RESOURCE }
+  }
+}
+node {
+  name: "while/Merge_1"
+  op: "Merge"
+  input: "while/Enter_1"
+  input: "while/NextIteration_1"
+  attr {
+    key: "N"
+    value { i: 2 }
+  }
+  attr {
+    key: "T"
+    value { type: DT_INT32 }
+  }
+}
+node {
+  name: "while/Less/y"
+  op: "Const"
+  input: "^while/Merge"
+  attr {
+    key: "dtype"
+    value { type: DT_INT32 }
+  }
+  attr {
+    key: "value"
+    value {
+      tensor {
+        dtype: DT_INT32
+        tensor_shape {}
+        int_val: 3
+      }
+    }
+  }
+}
+node {
+  name: "while/Less"
+  op: "Less"
+  input: "while/Merge"
+  input: "while/Less/y"
+  attr {
+    key: "T"
+    value { type: DT_INT32 }
+  }
+}
+node { name: "while/LoopCond" op: "LoopCond" input: "while/Less" }
+node {
+  name: "while/Switch"
+  op: "Switch"
+  input: "while/Merge"
+  input: "while/LoopCond"
+  attr {
+    key: "T"
+    value { type: DT_RESOURCE }
+  }
+  attr {
+    key: "_class"
+    value { list { s: "loc:@while/Merge" } }
+  }
+}
+node {
+  name: "while/Switch_1"
+  op: "Switch"
+  input: "while/Merge_1"
+  input: "while/LoopCond"
+  attr {
+    key: "T"
+    value { type: DT_INT32 }
+  }
+  attr {
+    key: "_class"
+    value { list { s: "loc:@while/Merge_1" } }
+  }
+}
+node {
+  name: "while/Identity"
+  op: "Identity"
+  input: "while/Switch:1"
+  attr {
+    key: "T"
+    value { type: DT_RESOURCE }
+  }
+}
+node {
+  name: "while/ReadVariableOp"
+  op: "ReadVariableOp"
+  input: "while/Identity"
+  attr {
+    key: "dtype"
+    value { type: DT_INT32 }
+  }
+}
+node {
+  name: "while/Identity_1"
+  op: "Identity"
+  input: "while/Switch_1:1"
+  attr {
+    key: "T"
+    value { type: DT_INT32 }
+  }
+}
+node {
+  name: "while/add/y"
+  op: "Const"
+  input: "^while/ReadVariableOp"
+  attr {
+    key: "dtype"
+    value { type: DT_INT32 }
+  }
+  attr {
+    key: "value"
+    value {
+      tensor {
+        dtype: DT_INT32
+        tensor_shape {}
+        int_val: 1
+      }
+    }
+  }
+}
+node {
+  name: "while/add"
+  op: "Add"
+  input: "while/ReadVariableOp"
+  input: "while/add/y"
+  attr {
+    key: "T"
+    value { type: DT_INT32 }
+  }
+}
+node {
+  name: "while/add_1"
+  op: "Add"
+  input: "while/Identity_1"
+  input: "while/Identity_1"
+  attr {
+    key: "T"
+    value { type: DT_INT32 }
+  }
+}
+node {
+  name: "while/NextIteration"
+  op: "NextIteration"
+  input: "while/Identity"
+  attr {
+    key: "T"
+    value { type: DT_RESOURCE }
+  }
+}
+node {
+  name: "while/NextIteration_1"
+  op: "NextIteration"
+  input: "while/add_1"
+  attr {
+    key: "T"
+    value { type: DT_INT32 }
+  }
+}
+node {
+  name: "while/Exit"
+  op: "Exit"
+  input: "while/Switch"
+  attr {
+    key: "T"
+    value { type: DT_RESOURCE }
+  }
+}
+node {
+  name: "while/Exit_1"
+  op: "Exit"
+  input: "while/Switch_1"
+  attr {
+    key: "T"
+    value { type: DT_INT32 }
+  }
+}
+versions { producer: 27 }
-- 
GitLab


From aca318aaee0944c8305feb8c57ed888c718973ed Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Fri, 11 Jan 2019 18:09:15 -0800
Subject: [PATCH 0611/2345] Call multi-device functions natively from eager
 runtime

PiperOrigin-RevId: 228976541
---
 tensorflow/compiler/tests/eager_test.py       |  12 +-
 tensorflow/core/common_runtime/eager/BUILD    |   5 +
 .../core/common_runtime/eager/attr_builder.cc |   2 +-
 .../common_runtime/eager/attr_builder_test.cc |   4 +-
 .../core/common_runtime/eager/context.h       |  12 +-
 .../eager/copy_to_device_node.h               |   2 +-
 .../core/common_runtime/eager/execute.cc      | 340 ++++++++++++------
 .../core/common_runtime/eager/execute.h       |  12 +-
 .../core/common_runtime/eager/execute_node.h  |   2 +-
 .../common_runtime/eager/kernel_and_device.cc | 243 ++++++++++---
 .../common_runtime/eager/kernel_and_device.h  | 180 ++++++++--
 .../eager/kernel_and_device_test.cc           |  38 +-
 .../common_runtime/eager/tensor_handle.cc     |  67 ++++
 .../core/common_runtime/eager/tensor_handle.h |  55 +--
 tensorflow/python/eager/function.py           |  68 ++--
 tensorflow/python/eager/function_test.py      | 265 ++++++++++++++
 tensorflow/python/eager/pywrap_tfe_test.py    |   5 +
 tensorflow/python/ops/functional_ops.py       |  18 +-
 tensorflow/python/util/function_utils.py      |  14 +
 19 files changed, 1044 insertions(+), 300 deletions(-)

diff --git a/tensorflow/compiler/tests/eager_test.py b/tensorflow/compiler/tests/eager_test.py
index 2af32b537b..c36c12a0e3 100644
--- a/tensorflow/compiler/tests/eager_test.py
+++ b/tensorflow/compiler/tests/eager_test.py
@@ -463,7 +463,7 @@ class EagerFunctionTest(xla_test.XLATestCase):
       def f(x, y):
         return x[0::2, y:, ...]
 
-      x = array_ops.ones([2, 3, 4])
+      x = array_ops.ones([2, 3, 4], dtype=dtypes.float32)
       y = array_ops.ones([], dtype=dtypes.int32)
       with backprop.GradientTape() as tape:
         tape.watch(x)
@@ -479,15 +479,15 @@ class EagerFunctionTest(xla_test.XLATestCase):
 
       @function.defun
       def times_two(x):
-        return 2 * x
+        return 2. * x
 
       @function.defun
       def two_x_plus_1(x):
-        return times_two(x) + 1
+        return times_two(x) + 1.
 
-      x = constant_op.constant([2, 3, 4])
+      x = constant_op.constant([2., 3., 4.])
       y = two_x_plus_1(x)
-      self.assertAllEqual([5, 7, 9], y.numpy())
+      self.assertAllEqual([5., 7., 9.], y.numpy())
 
   def testNestedDefunWithVariable(self):
     with self.test_scope():
@@ -506,7 +506,7 @@ class EagerFunctionTest(xla_test.XLATestCase):
       x = constant_op.constant(3.0)
       y = f(x)
 
-    self.assertEqual(75, y.numpy())
+    self.assertEqual(75.0, y.numpy())
 
   def testNestedDefunInGradientTape(self):
     with self.test_scope():
diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD
index 77e3246df0..06cbe8049f 100644
--- a/tensorflow/core/common_runtime/eager/BUILD
+++ b/tensorflow/core/common_runtime/eager/BUILD
@@ -165,6 +165,7 @@ tf_cuda_library(
             "//tensorflow/core:lib",
             "//tensorflow/core:lib_internal",
             "//tensorflow/core:protos_all_cc",
+            "//tensorflow/core/grappler/optimizers:meta_optimizer",
         ],
     }),
 )
@@ -181,10 +182,14 @@ tf_cc_test(
         "//tensorflow/cc:scope",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:core_cpu_internal",
+        "//tensorflow/core:direct_session",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
+        "//tensorflow/core/kernels:constant_op",
+        "//tensorflow/core/kernels:matmul_op",
         "@com_google_absl//absl/memory",
     ],
 )
diff --git a/tensorflow/core/common_runtime/eager/attr_builder.cc b/tensorflow/core/common_runtime/eager/attr_builder.cc
index 689b04274f..77be4c951e 100644
--- a/tensorflow/core/common_runtime/eager/attr_builder.cc
+++ b/tensorflow/core/common_runtime/eager/attr_builder.cc
@@ -42,7 +42,7 @@ const uint32 kIsList = 1U << 31;
 AttrTypeMap* DefaultFunctionAttrTypeMap() {
   AttrTypeMap* map = new AttrTypeMap();
   (*map)["executor_type"] = TF_ATTR_STRING;
-  (*map)["config"] = TF_ATTR_STRING;
+  (*map)["config_proto"] = TF_ATTR_STRING;
   return map;
 }
 
diff --git a/tensorflow/core/common_runtime/eager/attr_builder_test.cc b/tensorflow/core/common_runtime/eager/attr_builder_test.cc
index 8245660bfc..31c998a670 100644
--- a/tensorflow/core/common_runtime/eager/attr_builder_test.cc
+++ b/tensorflow/core/common_runtime/eager/attr_builder_test.cc
@@ -41,8 +41,10 @@ TEST(AttrTypeMap, Lookup) {
   Status s = AttrTypeMapForOp("SomeFunctionName", &m, &is_function);
   EXPECT_TRUE(s.ok());
   EXPECT_TRUE(is_function);
+  ASSERT_NE(m->end(), m->find("executor_type"));
   EXPECT_EQ(TF_ATTR_STRING, m->find("executor_type")->second);
-  EXPECT_EQ(TF_ATTR_STRING, m->find("config")->second);
+  ASSERT_NE(m->end(), m->find("config_proto"));
+  EXPECT_EQ(TF_ATTR_STRING, m->find("config_proto")->second);
 
   is_function = true;
   s = AttrTypeMapForOp("MatMul", &m, &is_function);
diff --git a/tensorflow/core/common_runtime/eager/context.h b/tensorflow/core/common_runtime/eager/context.h
index b83f5707da..d8807cf839 100644
--- a/tensorflow/core/common_runtime/eager/context.h
+++ b/tensorflow/core/common_runtime/eager/context.h
@@ -93,6 +93,8 @@ class EagerContext {
     return pflr_->GetFLR(d->name());
   }
 
+  ProcessFunctionLibraryRuntime* pflr() const { return pflr_.get(); }
+
   // True if running in asynchronous mode.
   bool Async() const;
 
@@ -138,7 +140,7 @@ class EagerContext {
 
   Status FindDeviceByName(const string& name, Device** result);
 
-  Device* HostCPU() { return devices_[0]; }
+  Device* HostCPU() const { return devices_[0]; }
 
   GraphCollector* GetGraphCollector() { return &graph_collector_; }
 
@@ -152,10 +154,10 @@ class EagerContext {
 
   void AddKernelToCache(Fprint128 cache_key, KernelAndDevice* kernel);
 
-  bool LogDevicePlacement() { return log_device_placement_; }
-  bool LogMemory() { return log_memory_; }
+  bool LogDevicePlacement() const { return log_device_placement_; }
+  bool LogMemory() const { return log_memory_; }
 
-  Rendezvous* GetRendezvous() { return rendezvous_; }
+  Rendezvous* GetRendezvous() const { return rendezvous_; }
   CollectiveExecutorMgrInterface* collective_executor_mgr() {
     return (collective_executor_mgr_ != nullptr)
                ? collective_executor_mgr_.get()
@@ -171,7 +173,7 @@ class EagerContext {
     return (local_device_manager_ != nullptr) ? local_device_manager_.get()
                                               : local_unowned_device_manager_;
   }
-  const tensorflow::DeviceMgr* remote_device_mgr() {
+  const tensorflow::DeviceMgr* remote_device_mgr() const {
     return remote_device_manager_.get();
   }
 
diff --git a/tensorflow/core/common_runtime/eager/copy_to_device_node.h b/tensorflow/core/common_runtime/eager/copy_to_device_node.h
index 5bc3bb689e..a807e7f68d 100644
--- a/tensorflow/core/common_runtime/eager/copy_to_device_node.h
+++ b/tensorflow/core/common_runtime/eager/copy_to_device_node.h
@@ -30,7 +30,7 @@ class CopyToDeviceNode : public EagerNode {
         src_(src),
         dstd_(dstd),
         ctx_(ctx),
-        dst_(new TensorHandle(id, dstd_, dstd_, src->dtype, ctx)) {
+        dst_(new TensorHandle(id, dstd_, dstd_, nullptr, src->dtype, ctx)) {
     src_->Ref();
     dst_->Ref();
   }
diff --git a/tensorflow/core/common_runtime/eager/execute.cc b/tensorflow/core/common_runtime/eager/execute.cc
index 9d2401929d..9df1511bc7 100644
--- a/tensorflow/core/common_runtime/eager/execute.cc
+++ b/tensorflow/core/common_runtime/eager/execute.cc
@@ -24,6 +24,9 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/eager/execute_node.h"
 #include "tensorflow/core/common_runtime/eager/kernel_and_device.h"
 #include "tensorflow/core/common_runtime/eager/tensor_handle.h"
+#include "tensorflow/core/framework/function.h"
+#include "tensorflow/core/framework/node_def_util.h"
+#include "tensorflow/core/framework/types.pb.h"
 #include "tensorflow/core/lib/core/errors.h"
 #ifndef __ANDROID__
 #include "tensorflow/core/distributed_runtime/eager/eager_client.h"
@@ -81,18 +84,20 @@ int StepStatsDeviceIndex(StepStats* step_stats, EagerContext* ctx,
 // tensor handle.
 //
 // The passed in *handle will be Unreffed if it is replaced.
-Status MaybeCopyInputToExpectedDevice(EagerOperation* op, int i,
-                                      const Device* expected_device,
+//
+// `op_device` is passed in explicitly because `op->device()` might be unset
+// and we might have selected some specific device to run this op on.
+Status MaybeCopyInputToExpectedDevice(EagerOperation* op,
+                                      const Device* op_device, int i,
+                                      const Device* expected_input_device,
                                       RunMetadata* run_metadata,
                                       TensorHandle** handle) {
   EagerContext* ctx = op->EagerContext();
   Device* handle_device = (*handle)->device();
   const Device* actual_device =
       handle_device == nullptr ? ctx->HostCPU() : handle_device;
-  const Device* op_device =
-      op->Device() == nullptr ? ctx->HostCPU() : op->Device();
 
-  if (expected_device != actual_device) {
+  if (expected_input_device != actual_device) {
     switch (ctx->GetDevicePlacementPolicy()) {
       case DEVICE_PLACEMENT_SILENT_FOR_INT32:
         // TODO(xpan): See if we could bubble python related error up
@@ -108,7 +113,7 @@ Status MaybeCopyInputToExpectedDevice(EagerOperation* op, int i,
             "Tensors on conflicting devices:"
             " cannot compute ",
             op->Name(), " as input #", i, " was expected to be on ",
-            expected_device->name(), " but is actually on ",
+            expected_input_device->name(), " but is actually on ",
             actual_device->name(), " (operation running on ", op_device->name(),
             ")",
             " Tensors can be copied explicitly using .gpu() or .cpu() "
@@ -119,9 +124,10 @@ Status MaybeCopyInputToExpectedDevice(EagerOperation* op, int i,
             " may slow down your model");
       case DEVICE_PLACEMENT_WARN:
         LOG(WARNING) << "before computing " << op->Name() << " input #" << i
-                     << " was expected to be on " << expected_device->name()
-                     << " but is actually on " << actual_device->name()
-                     << " (operation running on " << op_device->name()
+                     << " was expected to be on "
+                     << expected_input_device->name() << " but is actually on "
+                     << actual_device->name() << " (operation running on "
+                     << op_device->name()
                      << "). This triggers a copy which can be a performance "
                         "bottleneck.";
         break;
@@ -133,7 +139,7 @@ Status MaybeCopyInputToExpectedDevice(EagerOperation* op, int i,
     auto pre_time_nanos = Env::Default()->NowNanos();
     TensorHandle* result_handle = nullptr;
     Status status = EagerCopyToDevice(
-        *handle, ctx, expected_device->name().c_str(), &result_handle);
+        *handle, ctx, expected_input_device->name().c_str(), &result_handle);
     if (run_metadata != nullptr) {
       auto* step_stats = run_metadata->mutable_step_stats();
       MaybeInitializeStepStats(step_stats, ctx);
@@ -155,10 +161,10 @@ Status MaybeCopyInputToExpectedDevice(EagerOperation* op, int i,
     }
     if (!status.ok()) {
       if (result_handle != nullptr) result_handle->Unref();
-      return errors::Internal("Failed copying input tensor from ",
-                              actual_device->name(), " to ",
-                              expected_device->name(), " in order to run ",
-                              op->Name(), ": ", status.error_message());
+      return errors::Internal(
+          "Failed copying input tensor from ", actual_device->name(), " to ",
+          expected_input_device->name(), " in order to run ", op->Name(), ": ",
+          status.error_message());
     }
 
     (*handle)->Unref();
@@ -168,19 +174,18 @@ Status MaybeCopyInputToExpectedDevice(EagerOperation* op, int i,
 }
 
 Status ValidateInputTypeAndPlacement(EagerContext* ctx, Device* op_device,
-                                     EagerOperation* op, const OpKernel* kernel,
+                                     EagerOperation* op,
+                                     const KernelAndDevice* kernel,
                                      RunMetadata* run_metadata) {
-  Device* host_device = ctx->HostCPU();
-  const MemoryTypeVector& memtypes = kernel->input_memory_types();
-  if (memtypes.size() != op->Inputs().size()) {
-    return errors::InvalidArgument("expected ", memtypes.size(),
+  if (kernel->num_inputs() != op->Inputs().size()) {
+    return errors::InvalidArgument("expected ", kernel->num_inputs(),
                                    " inputs, got ", op->Inputs().size());
   }
   for (int i = 0; i < op->Inputs().size(); ++i) {
-    const Device* expected_device =
-        memtypes[i] == HOST_MEMORY ? host_device : op_device;
+    const Device* expected_device = kernel->InputDevice(i);
     TF_RETURN_IF_ERROR(MaybeCopyInputToExpectedDevice(
-        op, i, expected_device, run_metadata, &((*op->MutableInputs())[i])));
+        op, op_device, i, expected_device, run_metadata,
+        &((*op->MutableInputs())[i])));
     tensorflow::TensorHandle* handle = op->Inputs()[i];
     if (handle->dtype != kernel->input_type(i)) {
       return errors::InvalidArgument(
@@ -246,6 +251,89 @@ bool OnSameTask(EagerContext* ctx, Device* first, Device* second) {
          first->parsed_name().task == second->parsed_name().task;
 }
 
+inline tensorflow::Fprint128 FingerprintCat128(const tensorflow::Fprint128& a,
+                                               const tensorflow::Fprint128& b) {
+  return {tensorflow::FingerprintCat64(a.low64, b.low64),
+          tensorflow::FingerprintCat64(a.high64, b.high64)};
+}
+
+Status FindDeviceFromName(const EagerContext* ctx, const char* device_name,
+                          Device** device) {
+  *device = ctx->HostCPU();
+  if (device_name == nullptr || strlen(device_name) == 0) {
+    return Status::OK();
+  }
+
+  auto status = ctx->local_device_mgr()->LookupDevice(device_name, device);
+  if (status.ok()) {
+    return status;
+  }
+
+  if (ctx->remote_device_mgr() != nullptr) {
+    return ctx->remote_device_mgr()->LookupDevice(device_name, device);
+  }
+
+  return status;
+}
+
+bool IsMultiDevice(const FunctionDef* fdef, const string& op_device) {
+  if (fdef == nullptr) {
+    // Primitive op.
+    return false;
+  }
+
+  // Run all functions as multi-device.
+  return true;
+
+  // We can eliminate some overhead by running simple functions using regular
+  // CallOp kernel. However, it is tricky to figure out which functions should
+  // be run using CallOp. Also, currently CallOp runs neither optimization
+  // passes (needed for TPU/XLA) nor grappler.
+  // Here are some cases where a function should be run in multi-device mode:
+  //  - Function takes at least two resources on different devices.
+  //  - Function takes a resource on deviceA and a body op explicitly placed
+  //  on deviceB.
+  //  - Function has a colocation constraint.
+  //  - Function has an explicit device annotation (which might not be using
+  //    full canonical device name) different from op_device. Note that false
+  //    positives are ok.
+  //  - Function has a node or a (node) attribute that can potentially make
+  //    the function multi-device after a rewrite pass (e.g. various XLA/TPU
+  //    special nodes and attributes)
+}
+
+Status AddInputDevicesToCacheKey(const EagerContext* ctx,
+                                 const EagerOperation* op,
+                                 std::vector<Device*>* input_dev_ptrs,
+                                 Fprint128* cache_key) {
+  input_dev_ptrs->reserve(op->Inputs().size());
+  Device* cpu_device = ctx->HostCPU();
+  for (TensorHandle* tensor_handle : op->Inputs()) {
+    string device_name;
+    if (tensor_handle->dtype == DT_RESOURCE) {
+      // Use the resource's actual device because it is the device that will
+      // influence partitioning the multi-device function.
+      const Tensor* tensor;
+      TF_RETURN_IF_ERROR(tensor_handle->Tensor(&tensor));
+      const ResourceHandle& handle = tensor->flat<ResourceHandle>()(0);
+      device_name = handle.device();
+
+      Device* input_device;
+      TF_RETURN_IF_ERROR(
+          FindDeviceFromName(ctx, device_name.c_str(), &input_device));
+      input_dev_ptrs->push_back(input_device);
+    } else if (MTypeFromDType(tensor_handle->dtype) == HOST_MEMORY) {
+      input_dev_ptrs->push_back(cpu_device);
+    } else {
+      Device* device = tensor_handle->device();
+      device_name = device != nullptr ? device->name() : cpu_device->name();
+      input_dev_ptrs->push_back(device == nullptr ? cpu_device : device);
+    }
+    *cache_key = FingerprintCat128(*cache_key, Fingerprint128(device_name));
+  }
+  return Status::OK();
+}
+
 Status EagerLocalExecute(EagerOperation* op,
                          gtl::InlinedVector<TensorHandle*, 2>* retvals,
                          int* num_retvals) {
@@ -254,18 +342,34 @@ Status EagerLocalExecute(EagerOperation* op,
   if (!status.ok()) return status;
   Device* device = op->Device();
 
-  Fprint128 cache_key = op->MutableAttrs()->CacheKey(
-      device == nullptr ? "unspecified" : device->name());
+  const string& maybe_unspecified_device_name =
+      device == nullptr ? "unspecified" : device->name();
+  Fprint128 cache_key =
+      op->MutableAttrs()->CacheKey(maybe_unspecified_device_name);
+
+  bool is_multi_device_function = IsMultiDevice(
+      ctx->FindFunctionDef(op->Name()), maybe_unspecified_device_name);
+
+  std::vector<Device*> input_dev_ptrs;
+  if (is_multi_device_function) {
+    TF_RETURN_IF_ERROR(
+        AddInputDevicesToCacheKey(ctx, op, &input_dev_ptrs, &cache_key));
+  }
+
   KernelAndDevice* kernel = ctx->GetCachedKernel(cache_key);
   if (kernel == nullptr) {
+    VLOG(2) << "Creating new kernel for " << op->Name() << " on device "
+            << maybe_unspecified_device_name;
     // If we are running a function on explicitly requested TPU,
     // compile it with XLA.
     // Note that it is not ideal, but currently ok, to set this
     // attribute after computing the kernel cache key above.
+    bool compile_with_xla = false;
     if (op->is_function() && device != nullptr &&
         (device->device_type() == "TPU" || device->device_type() == "XLA_GPU" ||
          device->device_type() == "XLA_CPU")) {
       op->MutableAttrs()->Set(kXlaCompileAttr, true);
+      compile_with_xla = true;
     }
 
     const NodeDef& ndef = op->MutableAttrs()->BuildNodeDef();
@@ -273,7 +377,6 @@ Status EagerLocalExecute(EagerOperation* op,
       status = SelectDevice(ndef, ctx, &device);
       if (!status.ok()) return status;
     }
-    CHECK(device != nullptr);
     if (ctx->LogDevicePlacement()) {
       LOG(INFO) << "Executing op " << ndef.op() << " in device "
                 << device->name();
@@ -285,9 +388,35 @@ Status EagerLocalExecute(EagerOperation* op,
           "Unable to find a FunctionLibraryRuntime corresponding to device ",
           device->name());
     }
-    kernel = new KernelAndDevice(ctx->GetRendezvous(), ctx->LogMemory(),
-                                 ctx->GetCollectiveExecutorHandle());
-    status = KernelAndDevice::Init(ndef, flr, ctx->runner(), kernel);
+    GraphCollector* graph_collector = nullptr;
+    if (ctx->ShouldStoreMetadata()) {
+      graph_collector = ctx->GetGraphCollector();
+    }
+    // Treat the function as multi_device only when we are not compiling
+    // it wholly with XLA. When compiling wholly with XLA, flr->CreateKernel
+    // will create an XlaLaunchOp kernel to compile and run the function.
+    if (is_multi_device_function && !compile_with_xla) {
+      // Multi-device functions don't use the rendezvous from eager context.
+      // If we use that rendezvous, multiple concurrent calls to the same
+      // function will likely result in collisions. However, this also means
+      // that we don't support legitimate sending/receiving across function
+      // boundary.
+      VLOG(2) << "Running " << ndef.op() << " using multi-device function. "
+              << "compile_with_xla=" << compile_with_xla
+              << ". Full node_def=" << ndef.DebugString();
+      kernel = new KernelAndDeviceFunc(
+          flr, ctx->pflr(), std::move(input_dev_ptrs), ctx->runner(),
+          ctx->GetCollectiveExecutorHandle(), ctx->HostCPU());
+    } else {
+      VLOG(2) << "Running " << ndef.op() << " using op kernel. "
+              << "compile_with_xla=" << compile_with_xla
+              << ". Full node_def=" << ndef.DebugString();
+      kernel = new KernelAndDeviceOp(
+          ctx->GetRendezvous(), ctx->LogMemory(), flr, ctx->runner(),
+          ctx->GetCollectiveExecutorHandle(), ctx->HostCPU());
+    }
+
+    status = kernel->Init(ndef, graph_collector);
     if (!status.ok()) {
       delete kernel;
       return status;
@@ -309,7 +438,7 @@ Status EagerLocalExecute(EagerOperation* op,
     device = kernel->device();
   }
   status = ValidateInputTypeAndPlacement(
-      ctx, device, op, kernel->kernel(),
+      ctx, device, op, kernel,
       ctx->ShouldStoreMetadata() ? ctx->RunMetadataProto() : nullptr);
   if (!status.ok()) return status;
   std::unique_ptr<NodeExecStats> maybe_stats;
@@ -333,12 +462,15 @@ Status EagerLocalExecute(EagerOperation* op,
   if (ctx->Async()) {
     // Note that for async mode, execution order will make sure that all
     // input handles are ready before executing them.
-    // TODO(agarwal): Consider executing "cheap" kernels inline for performance.
+    // TODO(agarwal): Consider executing "cheap" kernels inline for
+    // performance.
     tensorflow::uint64 id = ctx->NextId();
     for (int i = 0; i < *num_retvals; ++i) {
-      (*retvals)[i] = new TensorHandle(id, /* d= */ kernel->OutputDevice(i),
-                                       /* op_device= */ kernel->device(),
-                                       output_dtypes[i], ctx);
+      (*retvals)[i] = new TensorHandle(
+          id, /* d= */ kernel->OutputDevice(i),
+          /* op_device= */ kernel->device(),
+          /* resource_device= */ kernel->OutputResourceDevice(i),
+          output_dtypes[i], ctx);
     }
     EagerNode* node = new ExecuteNode(
         id, ctx, op->Device(), op->Inputs(), kernel, maybe_stats.release(),
@@ -347,9 +479,9 @@ Status EagerLocalExecute(EagerOperation* op,
   } else {
     // Execute checks if retvals[i] is nullptr or not to figure if it needs to
     // allocate it.
-    status = EagerExecute(ctx, op->Device(), op->Inputs(), kernel,
-                          maybe_stats.get(), maybe_step_stats, graph_collector,
-                          retvals->data(), *num_retvals);
+    status = EagerKernelExecute(ctx, op->Device(), op->Inputs(), kernel,
+                                maybe_stats.get(), maybe_step_stats,
+                                graph_collector, retvals->data(), *num_retvals);
   }
 
   return status;
@@ -361,9 +493,9 @@ std::function<void()> GetRemoteTensorDestructor(
     uint64 op_id, int output_num) {
   return [ctx, eager_client, context_id, op_id, output_num]() {
     if (!ctx->HasActiveRemoteContext(context_id)) {
-      // This means that this tensor was pointing to a remote device, which has
-      // been changed out from under us. Simply return since there is nothing we
-      // can do.
+      // This means that this tensor was pointing to a remote device, which
+      // has been changed out from under us. Simply return since there is
+      // nothing we can do.
       return tensorflow::Status::OK();
     }
 
@@ -423,10 +555,10 @@ Status EagerRemoteSendTensor(EagerContext* ctx, TensorHandle* h,
 
   Device* tensor_handle_device = h->device();
 
-  // AsProtoTensorContent doesn't work when the tensor is on the GPU, hence copy
-  // it to the CPU before copying it out.
-  // TODO(nareshmodi): this is currently slow, but can be fixed by making tensor
-  // handles aware of more than one device.
+  // AsProtoTensorContent doesn't work when the tensor is on the GPU, hence
+  // copy it to the CPU before copying it out.
+  // TODO(nareshmodi): this is currently slow, but can be fixed by making
+  // tensor handles aware of more than one device.
   TensorHandle* actual_handle;
   if (tensor_handle_device != nullptr &&
       tensor_handle_device->device_type() != "CPU") {
@@ -458,7 +590,8 @@ Status EagerRemoteSendTensor(EagerContext* ctx, TensorHandle* h,
 
   *result = new TensorHandle(id, /*output_num=*/0, /*remote_shape_node_id=*/0,
                              tensor->dtype(), std::move(destructor),
-                             recv_device, recv_device, ctx);
+                             /*d=*/recv_device, /*op_device=*/recv_device,
+                             /*resource_device=*/nullptr, ctx);
   (*result)->SetRemoteShape(MakeUnique<TensorShape>(tensor->shape()));
 
   actual_handle->Unref();
@@ -497,7 +630,7 @@ Status EagerRemoteExecute(EagerOperation* op, TensorHandle** retvals,
       // TODO(b/110044833): It's possible the same tensor gets copied to the
       // remote device repeatedly.
       TF_RETURN_IF_ERROR(MaybeCopyInputToExpectedDevice(
-          op, i, op->Device(), /* run_metadata= */ nullptr,
+          op, op->Device(), i, op->Device(), /* run_metadata= */ nullptr,
           &(*op->MutableInputs())[i]));
     }
 
@@ -537,14 +670,25 @@ Status EagerRemoteExecute(EagerOperation* op, TensorHandle** retvals,
 
   const tensorflow::uint64 id = remote_op->id();
   for (int i = 0; i < *num_retvals; i++) {
-    // TODO(nareshmodi): Change the callback to instead add the decref to a list
-    // of pending decrefs that we can send as a batch with the next execute.
+    // TODO(nareshmodi): Change the callback to instead add the decref to a
+    // list of pending decrefs that we can send as a batch with the next
+    // execute.
     std::function<void()> destructor =
         GetRemoteTensorDestructor(ctx, eager_client, context_id, id, i);
 
-    retvals[i] = new TensorHandle(remote_op->id(), i, remote_node_id,
-                                  output_dtypes[i], std::move(destructor),
-                                  op_device, op_device, op->EagerContext());
+    // The device_ and resource_device_ or this TensorHandle are not correct.
+    // It is pretty hard to make it correct because for multi-device functions,
+    // we don't know the output device until the function is instantiated.
+    // Luckily, we don't need to know the correct remote device here. We just
+    // need to know that it is remote. If we need to copy this tensor to this
+    // process, the remote end will know the correct device of this handle.
+    retvals[i] = new TensorHandle(
+        remote_op->id(), i, remote_node_id, output_dtypes[i],
+        std::move(destructor),
+        /*d=*/op_device, /*op_device=*/op_device,
+        /*resource_device=*/output_dtypes[i] == DT_RESOURCE ? op_device
+                                                            : nullptr,
+        op->EagerContext());
   }
 
   if (is_async) {
@@ -619,43 +763,56 @@ bool IsPinnableOp(const string& op_type) {
 // "TF_EAGER_ENABLE_SMALL_TENSOR_CPU_PINNING" to "0" or "false".
 Status MaybeUpdateOpDevice(EagerOperation* op) {
   EagerContext* ctx = op->EagerContext();
-  bool device_set_for_resource_variable = false;
   bool all_inputs_eligible_for_cpu_pinning =
       ctx->PinSmallOpsToCPU() && IsPinnableOp(op->Name());
-
+  Device* op_device = op->Device() == nullptr ? ctx->HostCPU() : op->Device();
   for (int i = 0; i < op->Inputs().size(); ++i) {
-    Device* input_op_device = op->Inputs()[i]->op_device();
-    VLOG(2) << "for op " << op->Name() << " input " << i << " "
-            << DataTypeString(op->Inputs()[i]->dtype) << " "
-            << (input_op_device == nullptr ? "cpu" : input_op_device->name())
-            << " " << (op->Device() == nullptr ? "cpu" : op->Device()->name());
-    if (op->Inputs()[i]->dtype == DT_RESOURCE &&
-        (input_op_device != op->Device() || input_op_device == nullptr)) {
-      Device* d = input_op_device == nullptr ? ctx->HostCPU() : input_op_device;
-      VLOG(1) << "Changing device of operation " << op->Name() << " to "
-              << d->name() << " because input #" << i
-              << " is a resource in this device.";
-      op->SetDevice(d);
-
-      device_set_for_resource_variable = true;
+    TensorHandle* tensor_handle = op->Inputs()[i];
+    if (tensor_handle->dtype == DT_RESOURCE) {
+      Device* resource_device = tensor_handle->resource_device();
+      VLOG(2) << "for op " << op->Name() << " input " << i << " "
+              << DataTypeString(tensor_handle->dtype)
+              << " input device = " << resource_device->name()
+              << ", op device = " << op_device->name();
+      // We check for `op->Device() == nullptr` because it can be later
+      // interpreted as unspecified device and a different device can
+      // be selected based on device priority. If any input to an op
+      // is a resource we must pin it to prevent different device selection.
+      // TODO(iga): null device can mean "unspecified" or "CPU". Clean this up.
+      if (resource_device != op_device || op->Device() == nullptr) {
+        VLOG(1) << (resource_device != op_device ? "Changing " : "Setting ")
+                << "device of operation " << op->Name() << " to "
+                << resource_device->name() << " because input #" << i
+                << " is a resource in this device.";
+        op->SetDevice(resource_device);
+      }
       all_inputs_eligible_for_cpu_pinning = false;
+      // No point in looking at other inputs. If there are other resources,
+      // they must have the same device and we already declared the op to be
+      // ineligible for CPU pinning.
+      break;
     } else if (all_inputs_eligible_for_cpu_pinning) {
-      TensorHandle* handle = op->Inputs()[i];
+      Device* input_device = tensor_handle->device();
+      input_device = input_device == nullptr ? ctx->HostCPU() : input_device;
+      VLOG(2) << "for op " << op->Name() << " input " << i << " "
+              << DataTypeString(tensor_handle->dtype)
+              << " input device = " << input_device->name()
+              << ", op device = " << op_device->name();
 
       // Input is on CPU.
-      if (input_op_device != nullptr && input_op_device != ctx->HostCPU()) {
+      if (input_device != ctx->HostCPU()) {
         all_inputs_eligible_for_cpu_pinning = false;
         continue;
       }
 
-      if (handle->dtype != DataType::DT_INT32 &&
-          handle->dtype != DataType::DT_INT64) {
+      if (tensor_handle->dtype != DataType::DT_INT32 &&
+          tensor_handle->dtype != DataType::DT_INT64) {
         all_inputs_eligible_for_cpu_pinning = false;
         continue;
       }
 
       int64 num_elements;
-      TF_RETURN_IF_ERROR(handle->NumElements(&num_elements));
+      TF_RETURN_IF_ERROR(tensor_handle->NumElements(&num_elements));
       if (num_elements > 64) {
         all_inputs_eligible_for_cpu_pinning = false;
       }
@@ -697,12 +854,12 @@ Status EagerExecute(EagerOperation* op,
   return EagerRemoteExecute(op, retvals->data(), num_retvals);
 }
 
-Status EagerExecute(EagerContext* ctx, Device* device,
-                    const gtl::InlinedVector<TensorHandle*, 4>& op_inputs,
-                    KernelAndDevice* kernel, NodeExecStats* maybe_stats,
-                    StepStats* maybe_step_stats,
-                    GraphCollector* graph_collector, TensorHandle** retvals,
-                    int num_retvals) {
+Status EagerKernelExecute(EagerContext* ctx, Device* device,
+                          const gtl::InlinedVector<TensorHandle*, 4>& op_inputs,
+                          KernelAndDevice* kernel, NodeExecStats* maybe_stats,
+                          StepStats* maybe_step_stats,
+                          GraphCollector* graph_collector,
+                          TensorHandle** retvals, int num_retvals) {
   if (device == nullptr) {
     // TODO(apassos) debug how the assignment below might return a different
     // device from the one requested above.
@@ -710,8 +867,6 @@ Status EagerExecute(EagerContext* ctx, Device* device,
   }
 
   std::vector<Tensor> outputs(1);
-  const MemoryTypeVector* output_memory_types = nullptr;
-  output_memory_types = &kernel->kernel()->output_memory_types();
 
   // If there are multiple references to a TensorHandle in 'op_inputs' we must
   // increment the reference count of the corresponding Tensor or risk it being
@@ -790,8 +945,8 @@ Status EagerExecute(EagerContext* ctx, Device* device,
                            /* op_device= */ device, ctx);
     } else {
       // In the async case, the retval is not a nullptr, and its device is
-      // already set since all TensorHandles always have their device set during
-      // construction.
+      // already set since all TensorHandles always have their device set
+      // during construction.
       DCHECK_EQ(device, retvals[i]->op_device());
       DCHECK_EQ(kernel->OutputDevice(i), retvals[i]->device());
 
@@ -822,25 +977,6 @@ Status LocalEagerCopyToDevice(TensorHandle* h, EagerContext* ctx, Device* dstd,
   }
 }
 
-Status FindDeviceFromName(EagerContext* ctx, const char* device_name,
-                          Device** device) {
-  *device = ctx->HostCPU();
-  if (device_name == nullptr || strlen(device_name) == 0) {
-    return Status::OK();
-  }
-
-  auto status = ctx->local_device_mgr()->LookupDevice(device_name, device);
-  if (status.ok()) {
-    return status;
-  }
-
-  if (ctx->remote_device_mgr() != nullptr) {
-    return ctx->remote_device_mgr()->LookupDevice(device_name, device);
-  }
-
-  return status;
-}
-
 Status ExecuteSend(EagerContext* ctx, tensorflow::Device* device,
                    TensorHandle* h, StringPiece wire_id,
                    const string& recv_device) {
@@ -902,8 +1038,8 @@ Status ExecuteRecv(EagerContext* ctx, tensorflow::Device* device,
   return Status::OK();
 }
 
-// This gets a unique wire ID. We add a random identifier so that if the worker
-// has other clients that it is servicing, we don't have any collision.
+// This gets a unique wire ID. We add a random identifier so that if the
+// worker has other clients that it is servicing, we don't have any collision.
 string GetUniqueWireID() {
   static tensorflow::uint64 random_seed = random::New64();
   static tensorflow::mutex wireid_mutex(tensorflow::LINKER_INITIALIZED);
diff --git a/tensorflow/core/common_runtime/eager/execute.h b/tensorflow/core/common_runtime/eager/execute.h
index 6143a52d4b..80d381cecd 100644
--- a/tensorflow/core/common_runtime/eager/execute.h
+++ b/tensorflow/core/common_runtime/eager/execute.h
@@ -43,12 +43,12 @@ Status EagerExecute(
 
 // Low-level utility to execute the kernel specified by kernel on device device,
 // with the inputs op_inputs, in the context ctx.
-Status EagerExecute(EagerContext* ctx, Device* device,
-                    const gtl::InlinedVector<TensorHandle*, 4>& op_inputs,
-                    KernelAndDevice* kernel, NodeExecStats* maybe_stats,
-                    StepStats* maybe_step_stats,
-                    GraphCollector* graph_collector, TensorHandle** retvals,
-                    int num_retvals);
+Status EagerKernelExecute(EagerContext* ctx, Device* device,
+                          const gtl::InlinedVector<TensorHandle*, 4>& op_inputs,
+                          KernelAndDevice* kernel, NodeExecStats* maybe_stats,
+                          StepStats* maybe_step_stats,
+                          GraphCollector* graph_collector,
+                          TensorHandle** retvals, int num_retvals);
 
 // Low-level utility to copy a tensor handle from one device to another.
 Status EagerCopyToDevice(TensorHandle* h, EagerContext* ctx,
diff --git a/tensorflow/core/common_runtime/eager/execute_node.h b/tensorflow/core/common_runtime/eager/execute_node.h
index a99d509dd6..4459e3221b 100644
--- a/tensorflow/core/common_runtime/eager/execute_node.h
+++ b/tensorflow/core/common_runtime/eager/execute_node.h
@@ -64,7 +64,7 @@ class ExecuteNode : public EagerNode {
   }
 
   tensorflow::Status Run() override {
-    const Status status = EagerExecute(
+    const Status status = EagerKernelExecute(
         ctx_, op_device_, inputs_, kernel_, maybe_stats_.get(),
         maybe_step_stats_, graph_collector_, retvals_.begin(), retvals_.size());
     if (status.ok()) {
diff --git a/tensorflow/core/common_runtime/eager/kernel_and_device.cc b/tensorflow/core/common_runtime/eager/kernel_and_device.cc
index 4096763b11..943b183d63 100644
--- a/tensorflow/core/common_runtime/eager/kernel_and_device.cc
+++ b/tensorflow/core/common_runtime/eager/kernel_and_device.cc
@@ -17,12 +17,17 @@ limitations under the License.
 
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/common_runtime/eager/attr_builder.h"
+#include "tensorflow/core/common_runtime/process_function_library_runtime.h"
 #include "tensorflow/core/common_runtime/rendezvous_mgr.h"
 #include "tensorflow/core/common_runtime/step_stats_collector.h"
 #include "tensorflow/core/framework/allocator.h"
+#include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/framework/node_def.pb.h"
+#include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/framework/step_stats.pb.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/framework/types.pb.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/gtl/map_util.h"
 #include "tensorflow/core/lib/gtl/stl_util.h"
@@ -31,31 +36,77 @@ limitations under the License.
 #include "tensorflow/core/platform/tracing.h"
 #include "tensorflow/core/public/version.h"
 #include "tensorflow/core/util/tensor_slice_reader_cache.h"
+#ifndef __ANDROID__
+#include "tensorflow/core/grappler/optimizers/meta_optimizer.h"
+#endif
 
 namespace tensorflow {
 
-// static
-Status KernelAndDevice::Init(const NodeDef& ndef, FunctionLibraryRuntime* flr,
-                             std::function<void(std::function<void()>)>* runner,
-                             KernelAndDevice* out) {
+KernelAndDeviceFunc::~KernelAndDeviceFunc() {
+  if (handle_ != kInvalidHandle) {
+    Status status = pflr_->ReleaseHandle(handle_);
+    if (!status.ok()) {
+      LOG(INFO) << "Ignoring error status when releasing multi-device function "
+                   "handle "
+                << status.ToString();
+    }
+  }
+}
+
+Status KernelAndDeviceOp::Init(const NodeDef& ndef,
+                               GraphCollector* graph_collector) {
   OpKernel* k = nullptr;
-  TF_RETURN_IF_ERROR(flr->CreateKernel(ndef, &k));
-  out->device_ = flr->device();
-  out->kernel_.reset(k);
-  out->flr_ = flr;
-  out->runner_ = runner;
-  out->default_runner_ = [](std::function<void()> f) { f(); };
-
-  // Update output_dtypes_.
+  TF_RETURN_IF_ERROR(flr_->CreateKernel(ndef, &k));
+  kernel_.reset(k);
+  return Status::OK();
+}
+
+Status KernelAndDeviceFunc::Init(const NodeDef& ndef,
+                                 GraphCollector* graph_collector) {
   const OpDef* op_def = nullptr;
   const FunctionDef* function_def =
-      flr->GetFunctionLibraryDefinition()->Find(ndef.op());
+      flr_->GetFunctionLibraryDefinition()->Find(ndef.op());
   if (function_def != nullptr) {
     op_def = &(function_def->signature());
   } else {
     TF_RETURN_IF_ERROR(OpDefForOp(ndef.op().c_str(), &op_def));
   }
-  return OutputTypesForNode(ndef, *op_def, &out->output_dtypes_);
+  TF_RETURN_IF_ERROR(
+      InOutTypesForNode(ndef, *op_def, &input_dtypes_, &output_dtypes_));
+
+  FunctionLibraryRuntime::InstantiateOptions options;
+  options.target = device_->name();
+  options.is_multi_device_function = true;
+  for (const Device* device : input_devices_) {
+    options.input_devices.push_back(device->name());
+  }
+
+  const auto& it = ndef.attr().find("executor_type");
+  if (it != ndef.attr().end()) {
+    options.executor_type = it->second.s();
+  }
+#ifndef __ANDROID__
+  // Android tf library does not include grappler.
+  const auto& config_it = ndef.attr().find("config_proto");
+  if (it != ndef.attr().end()) {
+    ConfigProto config_proto;
+    if (!config_proto.ParseFromString(config_it->second.s())) {
+      return errors::InvalidArgument(
+          "Failed to parse config_proto attribute as tensorflow::ConfigProto "
+          "proto.");
+    }
+    options.optimize_graph_fn =
+        std::bind(grappler::OptimizeGraph, std::placeholders::_1,
+                  std::placeholders::_2, std::placeholders::_3,
+                  std::placeholders::_4, config_proto, std::placeholders::_5);
+  }
+#endif
+  options.graph_collector = graph_collector;
+
+  TF_RETURN_IF_ERROR(
+      pflr_->Instantiate(ndef.op(), AttrSlice(ndef), options, &handle_));
+  return pflr_->GetOutputDevices(handle_, &output_devices_);
+  return Status::OK();
 }
 
 Status KernelAndDevice::Run(const gtl::InlinedVector<TensorValue, 4>& inputs,
@@ -69,11 +120,39 @@ Status KernelAndDevice::Run(const gtl::InlinedVector<TensorValue, 4>& inputs,
                    graph_collector);
 }
 
-Status KernelAndDevice::Run(ScopedStepContainer* step_container,
-                            const gtl::InlinedVector<TensorValue, 4>& inputs,
-                            std::vector<Tensor>* outputs, NodeExecStats* stats,
-                            StepStats* step_stats,
-                            GraphCollector* graph_collector) {
+namespace {
+void UpdateStats(OpKernelContext* context,
+                 StepStatsCollector* step_stats_collector,
+                 NodeExecStats* stats) {
+  for (const auto& allocator_pair : context->ConsumeWrappedAllocators()) {
+    AllocatorMemoryUsed* memory = stats->add_memory();
+    memory->set_allocator_name(allocator_pair.first->Name());
+    auto sizes = allocator_pair.second->GetSizes();
+    memory->set_total_bytes(std::get<0>(sizes));
+    memory->set_peak_bytes(std::get<1>(sizes));
+    memory->set_live_bytes(std::get<2>(sizes));
+
+    AllocatorStats allocator_stats;
+    allocator_pair.first->GetStats(&allocator_stats);
+    memory->set_allocator_bytes_in_use(allocator_stats.bytes_in_use);
+    allocator_pair.second->GetRecordsAndUnRef();
+  }
+  auto* ms = stats->mutable_memory_stats();
+  ms->set_temp_memory_size(context->temp_memory_allocated());
+  for (const auto& alloc_id : context->persistent_alloc_ids()) {
+    ms->mutable_persistent_tensor_alloc_ids()->Add(alloc_id);
+  }
+
+  ms->set_persistent_memory_size(context->persistent_memory_allocated());
+  step_stats_collector->Finalize();
+}
+}  // anonymous namespace
+
+Status KernelAndDeviceOp::Run(ScopedStepContainer* step_container,
+                              const gtl::InlinedVector<TensorValue, 4>& inputs,
+                              std::vector<Tensor>* outputs,
+                              NodeExecStats* stats, StepStats* step_stats,
+                              GraphCollector* graph_collector) {
   std::vector<AllocatorAttributes> out_attrs(kernel_->num_outputs());
   for (size_t i = 0; i < out_attrs.size(); ++i) {
     out_attrs[i].set_on_host(kernel_->output_memory_types()[i] ==
@@ -108,11 +187,7 @@ Status KernelAndDevice::Run(ScopedStepContainer* step_container,
     params.stats_collector = step_stats_collector.get();
     params.graph_collector = graph_collector;
   }
-  if (runner_ == nullptr) {
-    params.runner = &default_runner_;
-  } else {
-    params.runner = runner_;
-  }
+  params.runner = runner_ != nullptr ? runner_ : &default_runner_;
 
   params.step_container = step_container;
   params.collective_executor =
@@ -148,37 +223,111 @@ Status KernelAndDevice::Run(ScopedStepContainer* step_container,
     outputs->push_back(Tensor(*context.mutable_output(i)));
   }
   if (stats != nullptr) {
-    for (const auto& allocator_pair : context.ConsumeWrappedAllocators()) {
-      AllocatorMemoryUsed* memory = stats->add_memory();
-      memory->set_allocator_name(allocator_pair.first->Name());
-      auto sizes = allocator_pair.second->GetSizes();
-      memory->set_total_bytes(std::get<0>(sizes));
-      memory->set_peak_bytes(std::get<1>(sizes));
-      memory->set_live_bytes(std::get<2>(sizes));
-
-      AllocatorStats allocator_stats;
-      allocator_pair.first->GetStats(&allocator_stats);
-      memory->set_allocator_bytes_in_use(allocator_stats.bytes_in_use);
-      allocator_pair.second->GetRecordsAndUnRef();
-    }
-    auto* ms = stats->mutable_memory_stats();
-    ms->set_temp_memory_size(context.temp_memory_allocated());
-    for (const auto& alloc_id : context.persistent_alloc_ids()) {
-      ms->mutable_persistent_tensor_alloc_ids()->Add(alloc_id);
-    }
+    UpdateStats(&context, step_stats_collector.get(), stats);
+  }
+  return Status::OK();
+}
+
+Status KernelAndDeviceFunc::Run(
+    ScopedStepContainer* step_container,
+    const gtl::InlinedVector<TensorValue, 4>& inputs,
+    std::vector<Tensor>* outputs, NodeExecStats* stats, StepStats* step_stats,
+    GraphCollector* graph_collector) {
+  FunctionLibraryRuntime::Options opts;
+  // We don't pass rendezvous from eager context because we can get tensor
+  // name collisions in send/recv ops when running multiple instances
+  // of the same multi-device function concurrently. Instead, we ask the
+  // function library runtime to create a new for this call. We could have
+  // created one here but it requires more state to be kept in
+  // KernelAndDeviceFunc.
+  opts.rendezvous = nullptr;
+  opts.create_rendezvous = true;
+  opts.cancellation_manager = &cm_;
+  // eager runtime does not yet support collective ops.
+  opts.collective_executor = nullptr;
+  opts.allow_dead_tensors = true;
+  opts.step_container = step_container;
+  opts.collective_executor =
+      collective_executor_ ? collective_executor_->get() : nullptr;
 
-    ms->set_persistent_memory_size(context.persistent_memory_allocated());
+  std::unique_ptr<StepStatsCollector> step_stats_collector;
+  if (stats != nullptr) {
+    step_stats_collector.reset(new StepStatsCollector(step_stats));
+  }
+  opts.stats_collector = step_stats_collector.get();
+  opts.runner = (runner_ == nullptr) ? &default_runner_ : runner_;
+
+  Notification done;
+  Status status;
+  outputs->clear();
+  std::vector<Tensor> input_vector;
+  input_vector.reserve(inputs.size());
+  for (const TensorValue& tensor_value : inputs) {
+    input_vector.push_back(*tensor_value.tensor);
+  }
+
+  flr_->Run(opts, handle_, input_vector, outputs,
+            [&status, &done](const Status& s) {
+              status = s;
+              done.Notify();
+            });
+  done.WaitForNotification();
+
+  if (step_stats_collector != nullptr) {
     step_stats_collector->Finalize();
   }
-  return Status::OK();
+  return status;
+}
+
+tensorflow::Device* KernelAndDeviceOp::OutputDevice(int idx) const {
+  if (kernel_->output_memory_types()[idx] == HOST_MEMORY) {
+    return nullptr;
+  }
+  return device_;
 }
 
-tensorflow::Device* KernelAndDevice::OutputDevice(int idx) const {
-  if (device_ != nullptr &&
-      kernel_->output_memory_types()[idx] == HOST_MEMORY) {
+tensorflow::Device* KernelAndDeviceFunc::OutputDevice(int idx) const {
+  if (output_dtypes_[idx] == DT_RESOURCE) {
     return nullptr;
   }
+  return output_devices_[idx];
+}
+
+tensorflow::Device* KernelAndDeviceOp::OutputResourceDevice(int idx) const {
+  if (kernel_->output_type(idx) == DT_RESOURCE) {
+    return device_;
+  }
+  return nullptr;
+}
+
+tensorflow::Device* KernelAndDeviceFunc::OutputResourceDevice(int idx) const {
+  if (output_dtypes_[idx] == DT_RESOURCE) {
+    return output_devices_[idx];
+  }
+  return nullptr;
+}
+
+DataType KernelAndDeviceOp::input_type(int i) const {
+  return kernel_->input_type(i);
+}
+
+DataType KernelAndDeviceFunc::input_type(int i) const {
+  return input_dtypes_[i];
+}
+
+Device* KernelAndDeviceOp::InputDevice(int i) const {
+  if (kernel_->input_memory_types()[i] == HOST_MEMORY) {
+    return host_cpu_device_;
+  }
   return device_;
 }
 
+Device* KernelAndDeviceFunc::InputDevice(int i) const {
+  if (input_dtypes_[i] == DT_RESOURCE) {
+    return host_cpu_device_;
+  } else {
+    return input_devices_[i];
+  }
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/eager/kernel_and_device.h b/tensorflow/core/common_runtime/eager/kernel_and_device.h
index f7a5b76518..c4ea99f53e 100644
--- a/tensorflow/core/common_runtime/eager/kernel_and_device.h
+++ b/tensorflow/core/common_runtime/eager/kernel_and_device.h
@@ -38,8 +38,13 @@ namespace tensorflow {
 // include the proto header
 class NodeExecStats;
 class StepStats;
+class ProcessFunctionLibraryRuntime;
+class FunctionLibraryRuntime;
 
-// KernelAndDevice encapsulates an instantiated kernel and the device it is on.
+// KernelAndDevice encapsulates the logic needed to run a computation eagerly.
+// The computation can be a single instantiated kernel (implemented by
+// KernelAndDeviceOp below) or a multi-device function (implemented by
+// KernelAndDeviceFunc below).
 //
 // Also see:
 // https://www.tensorflow.org/code/tensorflow/core/common_runtime/kernel_benchmark_testlib.h
@@ -47,60 +52,171 @@ class StepStats;
 // https://www.tensorflow.org/code/tensorflow/core/kernels/ops_testutil.h
 class KernelAndDevice {
  public:
-  // Populates 'out' with a kernel appropriate for 'ndef'.
+  // Populates this with a kernel appropriate for 'ndef'.
   //
   // The provided FunctionLibraryRuntime MUST outlive all calls to
   // Run() on the returned KernelAndDevice.
-  static Status Init(const NodeDef& ndef, FunctionLibraryRuntime* flr,
-                     std::function<void(std::function<void()>)>* runner,
-                     KernelAndDevice* out);
-
-  KernelAndDevice(tensorflow::Rendezvous* rendez, bool log_memory)
-      : KernelAndDevice(rendez, log_memory, nullptr) {}
+  virtual Status Init(const NodeDef& ndef, GraphCollector* graph_collector) = 0;
 
+  // Non-multi-device functions are run using regular CallOp and look like
+  // primitive operations from KernelAndDevice perspective.
   KernelAndDevice(
-      tensorflow::Rendezvous* rendez, bool log_memory,
-      std::unique_ptr<CollectiveExecutor::Handle> collective_executor)
-      : device_(nullptr),
-        flr_(nullptr),
-        rendez_(rendez),
-        log_memory_(log_memory),
+      FunctionLibraryRuntime* flr,
+      std::function<void(std::function<void()>)>* runner,
+      std::unique_ptr<CollectiveExecutor::Handle> collective_executor,
+      Device* host_cpu_device)
+      : device_(flr->device()),
+        host_cpu_device_(host_cpu_device),
+        flr_(flr),
+        runner_(runner),
+        default_runner_([](std::function<void()> f) { f(); }),
         collective_executor_(std::move(collective_executor)) {}
 
+  virtual ~KernelAndDevice() {}
+
   // TODO(ashankar): Handle list-valued inputs.
   Status Run(const gtl::InlinedVector<TensorValue, 4>& inputs,
              std::vector<Tensor>* outputs, NodeExecStats* stats,
              StepStats* step_stats, GraphCollector* graph_collector);
 
-  Status Run(ScopedStepContainer* step_container,
-             const gtl::InlinedVector<TensorValue, 4>& inputs,
-             std::vector<Tensor>* outputs, NodeExecStats* stats,
-             StepStats* step_stats, GraphCollector* graph_collector);
-
-  Device* OutputDevice(int idx) const;
-
-  const OpKernel* kernel() const { return kernel_.get(); }
-
+  virtual Status Run(ScopedStepContainer* step_container,
+                     const gtl::InlinedVector<TensorValue, 4>& inputs,
+                     std::vector<Tensor>* outputs, NodeExecStats* stats,
+                     StepStats* step_stats,
+                     GraphCollector* graph_collector) = 0;
+
+  virtual Device* InputDevice(int i) const = 0;
+  virtual Device* OutputDevice(int idx) const = 0;
+  // If idx'th output is a resource, returns the device backing the resource.
+  // Else, returns nullptr.
+  virtual Device* OutputResourceDevice(int idx) const = 0;
+
+  // Returns nullptr for functions.
+  virtual const OpKernel* kernel() const = 0;
+
+  // Returns the device on which this kernel will run. In the case of
+  // multi-device functions, this is the default device that is passed to the
+  // placer but actual computation can happen on a different set of devices.
+  // Also, outputs can be produced on devices different from what this method
+  // returns.
   Device* device() const { return device_; }
 
-  const DataTypeVector& output_dtypes() { return output_dtypes_; }
+  virtual const DataTypeVector& output_dtypes() const = 0;
 
- private:
+  virtual DataType input_type(int i) const = 0;
+  virtual int num_inputs() const = 0;
+  virtual int num_outputs() const = 0;
+
+ protected:
   // TODO(apassos) Consider a shared cancellation manager. Note that this
   // cancellation manager is not useful to actually cancel anything, and is
   // provided here only for the few kernels which can't handle one being
   // missing.
   CancellationManager cm_;
+  Device* const device_;           // non-null
+  Device* const host_cpu_device_;  // non-null
+  FunctionLibraryRuntime* const flr_;
+  std::function<void(std::function<void()>)>* const runner_;
+  std::function<void(std::function<void()>)> default_runner_;
+  const std::unique_ptr<CollectiveExecutor::Handle> collective_executor_;
+};
+
+// Represents an op kernel and the device it will be run on.
+class KernelAndDeviceOp final : public KernelAndDevice {
+ public:
+  KernelAndDeviceOp(
+      tensorflow::Rendezvous* rendez, bool log_memory,
+      FunctionLibraryRuntime* flr,
+      std::function<void(std::function<void()>)>* runner,
+      std::unique_ptr<CollectiveExecutor::Handle> collective_executor,
+      Device* host_cpu_device)
+      : KernelAndDevice(flr, runner, std::move(collective_executor),
+                        host_cpu_device),
+        rendez_(rendez),
+        log_memory_(log_memory) {}
+
+  Status Init(const NodeDef& ndef, GraphCollector* graph_collector) override;
+
+  using KernelAndDevice::Run;
+
+  Status Run(ScopedStepContainer* step_container,
+             const gtl::InlinedVector<TensorValue, 4>& inputs,
+             std::vector<Tensor>* outputs, NodeExecStats* stats,
+             StepStats* step_stats, GraphCollector* graph_collector) override;
+
+  const OpKernel* kernel() const override { return kernel_.get(); }
+
+  Device* InputDevice(int i) const override;
+  Device* OutputDevice(int idx) const override;
+  Device* OutputResourceDevice(int idx) const override;
+
+  DataType input_type(int i) const override;
+  const DataTypeVector& output_dtypes() const {
+    return kernel_->output_types();
+  }
+  int num_inputs() const override { return kernel_->num_inputs(); }
+  int num_outputs() const override { return kernel_->num_outputs(); }
+
+ private:
   std::unique_ptr<OpKernel> kernel_;
-  Device* device_;
-  FunctionLibraryRuntime* flr_;
+  Rendezvous* const rendez_;
   checkpoint::TensorSliceReaderCacheWrapper slice_reader_cache_;
-  Rendezvous* rendez_;
-  DataTypeVector output_dtypes_;
-  std::function<void(std::function<void()>)>* runner_;
-  std::function<void(std::function<void()>)> default_runner_;
   const bool log_memory_;
-  const std::unique_ptr<CollectiveExecutor::Handle> collective_executor_;
+};
+
+// Represents a multi-device function. Functions can also be run using
+// various function-calling kernels including CallOp and PartitionedCallOp.
+// In such cases, KernelAndDeviceOp is used.
+class KernelAndDeviceFunc final : public KernelAndDevice {
+ public:
+  KernelAndDeviceFunc(
+      FunctionLibraryRuntime* flr, ProcessFunctionLibraryRuntime* pflr,
+      std::vector<Device*> input_devices,
+      std::function<void(std::function<void()>)>* runner,
+      std::unique_ptr<CollectiveExecutor::Handle> collective_executor,
+      Device* host_cpu_device)
+      : KernelAndDevice(flr, runner, std::move(collective_executor),
+                        host_cpu_device),
+        pflr_(pflr),
+        handle_(kInvalidHandle),
+        input_devices_(std::move(input_devices)) {}
+
+  virtual ~KernelAndDeviceFunc();
+
+  Status Init(const NodeDef& ndef, GraphCollector* graph_collector) override;
+
+  using KernelAndDevice::Run;
+
+  Status Run(ScopedStepContainer* step_container,
+             const gtl::InlinedVector<TensorValue, 4>& inputs,
+             std::vector<Tensor>* outputs, NodeExecStats* stats,
+             StepStats* step_stats, GraphCollector* graph_collector) override;
+
+  const OpKernel* kernel() const override { return nullptr; }
+
+  Device* InputDevice(int i) const override;
+  Device* OutputDevice(int idx) const override;
+  Device* OutputResourceDevice(int idx) const override;
+
+  DataType input_type(int i) const override;
+  const DataTypeVector& output_dtypes() const override {
+    return output_dtypes_;
+  }
+  int num_inputs() const override { return input_dtypes_.size(); }
+  int num_outputs() const override { return output_dtypes_.size(); }
+
+ private:
+  ProcessFunctionLibraryRuntime* const pflr_;
+  FunctionLibraryRuntime::Handle handle_;
+  // CPU devices are null. Resource handles' devices are actual backing
+  // devices.
+  std::vector<Device*> output_devices_;
+  // CPU devices are not null. Resource handles' devices are actual backing
+  // devices.
+  std::vector<Device*> input_devices_;
+
+  DataTypeVector input_dtypes_;
+  DataTypeVector output_dtypes_;
 };
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc b/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc
index 8d6db96779..703f3eb9b7 100644
--- a/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc
+++ b/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc
@@ -27,10 +27,13 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/device_mgr.h"
 #include "tensorflow/core/common_runtime/eager/attr_builder.h"
 #include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/common_runtime/process_function_library_runtime.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/platform/test_benchmark.h"
+#include "tensorflow/core/protobuf/config.pb.h"
 #include "tensorflow/core/public/version.h"
+#include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
 namespace {
@@ -41,20 +44,27 @@ class TestEnv {
     std::vector<std::unique_ptr<Device>> devices;
     devices.push_back(
         DeviceFactory::NewDevice("CPU", {}, "/job:a/replica:0/task:0"));
+    cpu_device_ = devices.back().get();
     device_mgr_ = absl::make_unique<DeviceMgr>(std::move(devices));
-    flib_runtime_ = NewFunctionLibraryRuntime(
-        device_mgr_.get(), Env::Default(), device_mgr_->ListDevices()[0],
-        TF_GRAPH_DEF_VERSION, &flib_def_, nullptr, {}, nullptr);
-  }
+    OptimizerOptions opts;
+    pflr_ = tensorflow::MakeUnique<ProcessFunctionLibraryRuntime>(
+        device_mgr_.get(), Env::Default(), TF_GRAPH_DEF_VERSION, &flib_def_,
+        opts, /*default_thread_pool=*/nullptr, /*cluster_flr=*/nullptr);
 
-  FunctionLibraryRuntime* function_library_runtime() const {
-    return flib_runtime_.get();
+    flr_ = pflr_->GetFLR("/job:a/replica:0/task:0/device:CPU:0");
+    CHECK(flr_ != nullptr);
   }
 
+  FunctionLibraryRuntime* function_library_runtime() const { return flr_; }
+  ProcessFunctionLibraryRuntime* pflr() const { return pflr_.get(); }
+  Device* cpu_device() { return cpu_device_; }
+
  private:
   FunctionLibraryDefinition flib_def_;
   std::unique_ptr<DeviceMgr> device_mgr_;
-  std::unique_ptr<FunctionLibraryRuntime> flib_runtime_;
+  FunctionLibraryRuntime* flr_;
+  std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_;
+  Device* cpu_device_;
 };
 
 void BM_CreateGraph(int iters) {
@@ -106,11 +116,11 @@ void BM_KernelAndDeviceInit(int iters) {
                    .NumInputs(2)
                    .BuildNodeDef());
   TestEnv env;
-  KernelAndDevice k(nullptr, false);
+  KernelAndDeviceOp k(nullptr, false, env.function_library_runtime(), nullptr,
+                      nullptr, env.cpu_device());
   tensorflow::testing::StartTiming();
   for (int i = 0; i < iters; ++i) {
-    TF_CHECK_OK(KernelAndDevice::Init(ndef, env.function_library_runtime(),
-                                      nullptr, &k));
+    TF_CHECK_OK(k.Init(ndef, nullptr));
   }
 }
 BENCHMARK(BM_KernelAndDeviceInit);
@@ -129,12 +139,12 @@ void BM_KernelAndDeviceRun(int iters) {
                    .NumInputs(inputs.size())
                    .BuildNodeDef());
   TestEnv env;
-  KernelAndDevice kernel(nullptr, false);
-  TF_CHECK_OK(KernelAndDevice::Init(ndef, env.function_library_runtime(),
-                                    nullptr, &kernel));
+  KernelAndDeviceOp k(nullptr, false, env.function_library_runtime(), nullptr,
+                      nullptr, env.cpu_device());
+  TF_CHECK_OK(k.Init(ndef, nullptr));
   tensorflow::testing::StartTiming();
   for (int i = 0; i < iters; ++i) {
-    TF_CHECK_OK(kernel.Run(inputs, &outputs, nullptr, nullptr, nullptr));
+    TF_CHECK_OK(k.Run(inputs, &outputs, nullptr, nullptr, nullptr));
   }
 }
 BENCHMARK(BM_KernelAndDeviceRun);
diff --git a/tensorflow/core/common_runtime/eager/tensor_handle.cc b/tensorflow/core/common_runtime/eager/tensor_handle.cc
index 47a856a36e..eaca6d7b32 100644
--- a/tensorflow/core/common_runtime/eager/tensor_handle.cc
+++ b/tensorflow/core/common_runtime/eager/tensor_handle.cc
@@ -31,6 +31,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/rendezvous_mgr.h"
 #include "tensorflow/core/framework/rendezvous.h"
 #include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/types.pb.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
 #include "tensorflow/core/lib/gtl/inlined_vector.h"
 #include "tensorflow/core/lib/gtl/map_util.h"
@@ -43,6 +44,61 @@ limitations under the License.
 
 namespace tensorflow {
 
+TensorHandle::TensorHandle(const class Tensor& t, Device* d, Device* op_device,
+                           EagerContext* ctx)
+    : dtype(t.dtype()),
+      node_id_(0),
+      tensor_(t),
+      device_(d),
+      op_device_(op_device),
+      resource_device_(GetResourceDevice(t, ctx)),
+      remote_op_id_(-1),
+      remote_output_num_(-1),
+      remote_shape_node_id_(-1),
+      ctx_(ctx),
+      is_ready_(true) {}
+
+TensorHandle::TensorHandle(uint64 node_id, Device* d, Device* op_device,
+                           Device* resource_device, DataType dtype,
+                           EagerContext* ctx)
+    : dtype(dtype),
+      node_id_(node_id),
+      tensor_(dtype),
+      device_(d),
+      op_device_(op_device),
+      resource_device_(resource_device),
+      remote_op_id_(-1),
+      remote_output_num_(-1),
+      remote_shape_node_id_(-1),
+      ctx_(ctx),
+      is_ready_(ctx == nullptr) {
+  DCHECK_GT(node_id_, 0);
+  DCHECK(dtype == DT_RESOURCE ? resource_device_ != nullptr
+                              : resource_device_ == nullptr);
+}
+
+TensorHandle::TensorHandle(int64 op_id, int32 output_num,
+                           uint64 remote_shape_node_id, DataType dtype,
+                           std::function<void()> call_on_destroy, Device* d,
+                           Device* op_device, Device* resource_device,
+                           EagerContext* ctx)
+    : dtype(dtype),
+      node_id_(0),
+      device_(d),
+      op_device_(op_device),
+      resource_device_(resource_device),
+      remote_op_id_(op_id),
+      remote_output_num_(output_num),
+      remote_shape_node_id_(remote_shape_node_id),
+      call_on_destroy_(std::move(call_on_destroy)),
+      ctx_(ctx),
+      is_ready_(true) {
+  DCHECK(IsRemote()) << "Op ID and output num should be >= 0. Op ID: " << op_id
+                     << ", Output num: " << output_num;
+  DCHECK(dtype == DT_RESOURCE ? resource_device_ != nullptr
+                              : resource_device_ == nullptr);
+}
+
 bool TensorHandle::IsReady() {
   if (node_id_ == 0) return true;
   mutex_lock l(ctx_mutex_);
@@ -245,4 +301,15 @@ Status TensorHandle::CopyToDevice(EagerContext* ctx, tensorflow::Device* dstd,
   return status;
 }
 
+Device* GetResourceDevice(const Tensor& t, EagerContext* ctx) {
+  if (t.dtype() != DT_RESOURCE) {
+    return nullptr;
+  }
+  const ResourceHandle& resource_handle = t.flat<ResourceHandle>()(0);
+  const auto& map = *ctx->device_map();
+  auto it = map.find(resource_handle.device());
+  DCHECK(it != map.end());
+  return it->second;
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/eager/tensor_handle.h b/tensorflow/core/common_runtime/eager/tensor_handle.h
index 3eae3c7cc3..a4727d6f76 100644
--- a/tensorflow/core/common_runtime/eager/tensor_handle.h
+++ b/tensorflow/core/common_runtime/eager/tensor_handle.h
@@ -48,50 +48,15 @@ namespace tensorflow {
 // (unrelated to python TensorHandle).
 class TensorHandle : public core::RefCounted {
  public:
-  TensorHandle(const Tensor& t, Device* d, Device* op_device, EagerContext* ctx)
-      : dtype(t.dtype()),
-        node_id_(0),
-        tensor_(t),
-        device_(d),
-        op_device_(op_device),
-        remote_op_id_(-1),
-        remote_output_num_(-1),
-        remote_shape_node_id_(-1),
-        ctx_(ctx),
-        is_ready_(true) {}
-
-  TensorHandle(uint64 node_id, Device* d, Device* op_device, DataType dtype,
-               EagerContext* ctx)
-      : dtype(dtype),
-        node_id_(node_id),
-        tensor_(dtype),
-        device_(d),
-        op_device_(op_device),
-        remote_op_id_(-1),
-        remote_output_num_(-1),
-        remote_shape_node_id_(-1),
-        ctx_(ctx),
-        is_ready_(ctx == nullptr) {
-    DCHECK_GT(node_id_, 0);
-  }
+  TensorHandle(const Tensor& t, Device* d, Device* op_device,
+               EagerContext* ctx);
+  TensorHandle(uint64 node_id, Device* d, Device* op_device,
+               Device* resource_device, DataType dtype, EagerContext* ctx);
 
   // Remote tensor handle constructor.
   TensorHandle(int64 op_id, int32 output_num, uint64 remote_shape_node_id,
                DataType dtype, std::function<void()> call_on_destroy, Device* d,
-               Device* op_device, EagerContext* ctx)
-      : dtype(dtype),
-        node_id_(0),
-        device_(d),
-        op_device_(op_device),
-        remote_op_id_(op_id),
-        remote_output_num_(output_num),
-        remote_shape_node_id_(remote_shape_node_id),
-        call_on_destroy_(std::move(call_on_destroy)),
-        ctx_(ctx),
-        is_ready_(true) {
-    DCHECK(IsRemote()) << "Op ID and output num should be >= 0. Op ID: "
-                       << op_id << ", Output num: " << output_num;
-  }
+               Device* op_device, Device* resource_device, EagerContext* ctx);
 
   ~TensorHandle() override {
     if (call_on_destroy_) {
@@ -104,8 +69,8 @@ class TensorHandle : public core::RefCounted {
   Status TensorValue(tensorflow::TensorValue* t);
 
   tensorflow::Device* device() const { return device_; }
-
   tensorflow::Device* op_device() const { return op_device_; }
+  tensorflow::Device* resource_device() const { return resource_device_; }
 
   Status TensorAndDevice(const tensorflow::Tensor** tensor,
                          tensorflow::Device** device,
@@ -178,6 +143,10 @@ class TensorHandle : public core::RefCounted {
   // device_ for constant tensors.
   tensorflow::Device* const op_device_;
 
+  // If the tensor dtype is DT_RESOURCE, resource_device_ holds the device
+  // backing the resource. Else resource_device_ is nullptr.
+  tensorflow::Device* const resource_device_;
+
   // IDs required when this class is representing a remote tensor handle.
   const int64 remote_op_id_;
   const int32 remote_output_num_;
@@ -198,6 +167,10 @@ class TensorHandle : public core::RefCounted {
   bool is_ready_ GUARDED_BY(ctx_mutex_);
 };
 
+// If tensor's dtype is DT_RESOURCE, returns the device backing the resource.
+// Else, returns nullptr.
+Device* GetResourceDevice(const Tensor& t, EagerContext* ctx);
+
 }  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_TENSOR_HANDLE_H_
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index e21ad4e2ee..bbf8630a2e 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -52,6 +52,7 @@ from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import compat
+from tensorflow.python.util import function_utils
 from tensorflow.python.util import memory
 from tensorflow.python.util import nest
 from tensorflow.python.util import tf_decorator
@@ -192,7 +193,7 @@ def _inference_name(n):
 # so it doesn't have the definition-generating logic and is just a container for
 # an already-defined function.
 class _EagerDefinedFunction(object):
-  """Callable with the interface of `framework.function._DefinedFunction.`
+  """Callable with the interface of `framework.function._DefinedFunction`.
 
   `_EagerDefinedFunction` encapsulates a function definition and its properties,
   and it provides a method for calling the encapsulated function. Some Ops
@@ -284,51 +285,60 @@ class _EagerDefinedFunction(object):
     Raises:
       ValueError: if the number of arguments is incorrect.
     """
+    if len(args) != len(self.signature.input_arg):
+      raise ValueError(
+          "Arguments and signature arguments do not match: %s %s " %
+          (len(args), len(list(self.signature.input_arg))))
+
+    function_call_options = ctx.get_function_call_options()
+    if function_call_options.config_proto_serialized is None:
+      config = function_utils.get_disabled_rewriter_config()
+    else:
+      config = function_call_options.config_proto_serialized
+    executor_type = function_call_options.executor_type or ""
 
     executing_eagerly = ctx.executing_eagerly()
-
-    if self._graph._xla_compile:  # pylint: disable=protected-access
-      # XLA compilation relies upon a custom kernel creator to run functions.
-      signature = self.signature
-      if executing_eagerly:
+    if executing_eagerly:
+      with _InterpolateFunctionError(self):
         outputs = execute.execute(
-            str(signature.name),
+            str(self.signature.name),
             num_outputs=self._num_outputs,
             inputs=args,
-            attrs=None,
+            attrs=("executor_type", executor_type,
+                   "config_proto", config),
             ctx=ctx)
+      # Replace empty list with None
+      outputs = outputs or None
+    elif self._graph._xla_compile:  # pylint: disable=protected-access
+      g = ops.get_default_graph()
+      self.add_to_graph(g)
+      signature = self.signature
+      op = g.create_op(
+          signature.name,
+          [ops.internal_convert_to_tensor(x, ctx=ctx) for x in args],
+          tuple(dtypes_module.DType(x.type) for x in signature.output_arg),
+          op_def=signature,
+          name="FunctionCall",
+          compute_shapes=False)
+      outputs = op.outputs
+      if not outputs:
+        return op
+      if isinstance(outputs, (ops.Tensor, type(None))):
+        outputs = [outputs]
       else:
-        g = ops.get_default_graph()
-        self.add_to_graph(g)
-        op = g.create_op(
-            signature.name,
-            [ops.internal_convert_to_tensor(x, ctx=ctx) for x in args],
-            tuple(dtypes_module.DType(x.type) for x in signature.output_arg),
-            op_def=signature,
-            name="FunctionCall",
-            compute_shapes=False)
-        outputs = op.outputs
-        if not outputs:
-          return op
-        outputs = [outputs] if isinstance(
-            outputs, (ops.Tensor, type(None))) else list(outputs)
+        outputs = list(outputs)
     else:
       # TODO(akshayka): Either remove this if the FunctionLibraryRuntime
       # creates `PartitionedCallOp` kernels by default, or remove the previous
       # branch if a TPU kernel is registered for `PartitionedCall`.
-      if len(args) != len(self.signature.input_arg):
-        raise ValueError(
-            "Arguments and signature arguments do not match: %s %s " %
-            (len(args), len(list(self.signature.input_arg))))
-      function_call_options = ctx.get_function_call_options()
       with _InterpolateFunctionError(self):
         outputs = functional_ops.partitioned_call(
             args=args,
             f=self,
             tout=self._output_types,
             executing_eagerly=executing_eagerly,
-            config=function_call_options.config_proto_serialized,
-            executor_type=function_call_options.executor_type)
+            config=config,
+            executor_type=executor_type)
 
     if executing_eagerly:
       return outputs
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index 1966b259bf..68c581cbf8 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -19,6 +19,7 @@ from __future__ import print_function
 
 import collections
 import functools
+import itertools
 from multiprocessing.pool import ThreadPool
 import sys
 import weakref
@@ -48,6 +49,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import clip_ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import gen_resource_variable_ops
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import list_ops
 from tensorflow.python.ops import math_ops
@@ -2065,6 +2067,10 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     self.assertIs(None, weak_fn())
 
   def testFunctionStackInErrorMessage(self):
+    if context.executing_eagerly():
+      # TODO(b/122736651): Remove this skipTest once fixed.
+      self.skipTest('Error interpolation is not working when function is '
+                    'invoked without PartitionedCallOp.')
 
     @def_function.function()
     def fn3(x):
@@ -2088,6 +2094,265 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
       self.assertNotIn('fn3', e.message)
 
 
+class MultiDeviceTest(test.TestCase, parameterized.TestCase):
+
+  def testMultiDeviceOutput(self):
+    """Tests that functions can produce outputs on multiple devices."""
+    if not context.context().num_gpus():
+      self.skipTest('No GPUs found.')
+
+    @function.defun
+    def func(a, b, transpose_a):
+      with ops.device('/device:CPU:0'):
+        m1 = math_ops.matmul(a, b, transpose_a=transpose_a)
+      with ops.device('/device:GPU:0'):
+        m2 = math_ops.matmul(a, b, transpose_a=transpose_a)
+      return m1, m2
+
+    t = constant_op.constant([[1.0, 2.0], [3.0, 4.0]])
+    m1, m2 = func(t, t, transpose_a=True)
+    self.assertAllEqual(m1.numpy(), [[10, 14], [14, 20]])
+    self.assertRegexpMatches(m1.backing_device, 'CPU')
+    self.assertAllEqual(m2.numpy(), [[10, 14], [14, 20]])
+    self.assertRegexpMatches(m2.backing_device, 'GPU')
+
+  def testEmptyBody(self):
+    if not context.context().num_gpus():
+      self.skipTest('No GPUs found.')
+
+    @function.defun
+    def func(a, b):
+      return b, a
+
+    with ops.device('/device:CPU:0'):
+      a = constant_op.constant(3.0)
+    with ops.device('/device:GPU:0'):
+      b = constant_op.constant(5.0)
+
+    m1, m2 = func(a, b)
+    self.assertAllEqual(m1.numpy(), 5.0)
+    self.assertRegexpMatches(m1.backing_device, 'GPU')
+    self.assertAllEqual(m2.numpy(), 3.0)
+    self.assertRegexpMatches(m2.backing_device, 'CPU')
+
+  def testMultiDeviceInt32(self):
+    """Tests that multi-device functions can take and output INT32s.
+
+    When an INT32 device tensor is fed into a function, it is copied to CPU
+    by the eager runtime. The function sees all INT32 inputs on CPU.
+
+    We set allocator attribute 'on_host' for INT32 outputs. They can be
+    partitioned into the GPU component function, but will be allocated on
+    CPU nevertheless.
+
+    There is experimental support for `ints_on_device` in
+    FunctionLibraryRuntime now. We can try that.
+
+    """
+    if not context.context().num_gpus():
+      self.skipTest('No GPUs found.')
+
+    with ops.device('/device:CPU:0'):
+      int_cpu = constant_op.constant(3, dtype=dtypes.int32)
+      resource = resource_variable_ops.ResourceVariable(5, dtype=dtypes.int32)
+    with ops.device('/device:GPU:0'):
+      int_gpu = constant_op.constant(7, dtype=dtypes.int32)
+
+    @function.defun
+    def func(int_cpu, resource, int_gpu):
+      with ops.device('/device:CPU:0'):
+        m1 = int_cpu * resource + int_gpu
+      with ops.device('/device:GPU:0'):
+        # This computation will happen on GPU but m2 will be copied to CPU.
+        m2 = int_gpu * resource + int_cpu + 1
+      return m1, m2
+
+    m1, m2 = func(int_cpu, resource, int_gpu)
+    self.assertAllEqual(m1.numpy(), 22)
+    self.assertRegexpMatches(m1.backing_device, 'CPU')
+    self.assertAllEqual(m2.numpy(), 39)
+    self.assertRegexpMatches(m2.backing_device, 'CPU')
+
+    # flip arguments
+    m1, m2 = func(int_gpu, resource, int_cpu)
+    self.assertAllEqual(m1.numpy(), 38)
+    self.assertRegexpMatches(m1.backing_device, 'CPU')
+    self.assertAllEqual(m2.numpy(), 23)
+    self.assertRegexpMatches(m2.backing_device, 'CPU')
+
+  def testMultiDeviceColocateWith(self):
+    """Tests that function's outputs respect colocation constraints."""
+    if not context.context().num_gpus():
+      self.skipTest('No GPUs found.')
+
+    @function.defun
+    def func(a, b):
+      with ops.colocate_with(a):
+        ra = 2 * a
+      with ops.colocate_with(b):
+        rb = 3 * b
+      return ra, rb
+
+    devices = ['/device:CPU:0', '/device:GPU:0']
+    for dev1, dev2 in itertools.product(devices, devices):
+      with ops.device(dev1):
+        a = constant_op.constant(1.0)
+      with ops.device(dev2):
+        b = constant_op.constant(10.0)
+
+      ra, rb = func(a, b)
+      self.assertEqual(ra.numpy(), 2.0)
+      self.assertRegexpMatches(ra.backing_device, dev1)
+      self.assertEqual(rb.numpy(), 30.0)
+      self.assertRegexpMatches(rb.backing_device, dev2)
+
+  def testMultiDeviceResources(self):
+    if not context.context().num_gpus():
+      self.skipTest('No GPUs found.')
+
+    with ops.device('/device:CPU:0'):
+      c1 = resource_variable_ops.ResourceVariable(2.0)
+      c2 = resource_variable_ops.ResourceVariable(7.0)
+    with ops.device('/device:GPU:0'):
+      g1 = resource_variable_ops.ResourceVariable(3.0)
+      g2 = resource_variable_ops.ResourceVariable(5.0)
+
+    @function.defun
+    def func(resource1, resource2):
+      with ops.device('/device:CPU:0'):
+        result1 = resource1 * g2
+      with ops.device('/device:GPU:0'):
+        result2 = resource2 * c2
+      return result1, result2
+
+    r1, r2 = func(c1, g1)
+    self.assertEqual(r1.numpy(), 10.0)
+    self.assertRegexpMatches(r1.backing_device, 'CPU')
+    self.assertEqual(r2.numpy(), 21.0)
+    self.assertRegexpMatches(r2.backing_device, 'GPU')
+
+    # Call with flipped inputs. Check that we look at resource's
+    # device and reinstantiates the function when inputs' devices change.
+    r1, r2 = func(g1, c1)
+    self.assertEqual(r1.numpy(), 15.0)
+    self.assertRegexpMatches(r1.backing_device, 'CPU')
+    self.assertEqual(r2.numpy(), 14.0)
+    self.assertRegexpMatches(r2.backing_device, 'GPU')
+
+  def testOutputResources(self):
+    if not context.context().num_gpus():
+      self.skipTest('No GPUs found.')
+
+    with ops.device('/device:CPU:0'):
+      c1 = resource_variable_ops.ResourceVariable(2.0)
+    with ops.device('/device:GPU:0'):
+      g1 = resource_variable_ops.ResourceVariable(3.0)
+
+    @function.defun
+    def func(resource1, resource2):
+      with ops.device('/device:CPU:0'):
+        result1 = resource1 * 5
+      with ops.device('/device:GPU:0'):
+        result2 = resource2 * 7
+      return result1, resource1.handle, result2, resource2.handle
+
+    r1, res1, r2, res2 = func(c1, g1)
+    self.assertEqual(r1.numpy(), 10.0)
+    self.assertRegexpMatches(r1.backing_device, 'CPU')
+    self.assertEqual(r2.numpy(), 21.0)
+    self.assertRegexpMatches(r2.backing_device, 'GPU')
+
+    def check_handle(handle, expected_value):
+      self.assertRegexpMatches(handle.backing_device, 'CPU')
+      tensor = gen_resource_variable_ops.read_variable_op(
+          handle, dtypes.float32)
+      self.assertEqual(tensor.numpy(), expected_value)
+
+    # Check that handles returned from functions are on CPU and an op using
+    # the resource handle is correctly placed on the device backing the
+    # resource.
+    check_handle(res1, 2.0)
+    check_handle(res2, 3.0)
+
+    # Call with flipped inputs to make sure the same the function is
+    # reinstantiated and eager runtime does not mess up the device assignment
+    # for ops consuming handles returned from defuns.
+    r1, res1, r2, res2 = func(g1, c1)
+    self.assertEqual(r1.numpy(), 15.0)
+    self.assertRegexpMatches(r1.backing_device, 'CPU')
+    self.assertEqual(r2.numpy(), 14.0)
+    self.assertRegexpMatches(r2.backing_device, 'GPU')
+    check_handle(res1, 3.0)
+    check_handle(res2, 2.0)
+
+  def testComplexInputOutputDevicePattern(self):
+    """Tests input/output mapping logic in partitioning."""
+    if not context.context().num_gpus():
+      self.skipTest('No GPUs found.')
+
+    with ops.device('/device:CPU:0'):
+      rc0 = resource_variable_ops.ResourceVariable(2.0)
+      rc1 = resource_variable_ops.ResourceVariable(3.0)
+      cc0 = constant_op.constant(5.0)
+      cc1 = constant_op.constant(7.0)
+    with ops.device('/device:GPU:0'):
+      rg0 = resource_variable_ops.ResourceVariable(11.0)
+      rg1 = resource_variable_ops.ResourceVariable(13.0)
+      cg0 = constant_op.constant(17.0)
+      cg1 = constant_op.constant(19.0)
+
+    # Make sure tensors are on expected devices.
+    for tensor in [cc0, cc1]:
+      self.assertRegexpMatches(tensor.backing_device, 'CPU:0')
+    for tensor in [cg0, cg1]:
+      self.assertRegexpMatches(tensor.backing_device, 'GPU:0')
+
+    @function.defun
+    def func(rc0, cc0, cg0, rc1, cg1, rg0, rg1, cc1):
+      with ops.device('/device:CPU:0'):
+        m1 = rc0 * cg0
+      with ops.device('/device:GPU:0'):
+        m2 = rg0 * cc0
+
+      with ops.device('/device:CPU:0'):
+        r1 = 1000.0 * m2 + rc1 * cg1
+      with ops.device('/device:GPU:0'):
+        r2 = 1000.0 * m1 + rg1 * cc1
+
+      return r1, r2, m2, m1
+
+    r1, r2, m2, m1 = func(rc0, cc0, cg0, rc1, cg1, rg0, rg1, cc1)
+    self.assertRegexpMatches(m1.backing_device, 'CPU')
+    self.assertRegexpMatches(r1.backing_device, 'CPU')
+    self.assertRegexpMatches(m2.backing_device, 'GPU')
+    self.assertRegexpMatches(r2.backing_device, 'GPU')
+    self.assertEqual(m1.numpy(), 34.0)
+    self.assertEqual(r1.numpy(), 55000.0 + 3.0 * 19.0)
+    self.assertEqual(m2.numpy(), 55.0)
+    self.assertEqual(r2.numpy(), 34000.0 + 13.0 * 7.0)
+
+  def testArgumentPrunning(self):
+    """Tests functions taking unnecessary arguments."""
+    if not context.context().num_gpus():
+      self.skipTest('No GPUs found.')
+
+    with ops.device('/device:CPU:0'):
+      c1 = constant_op.constant(5.0)
+      c2 = constant_op.constant(7.0)
+
+    with ops.device('/device:GPU:0'):
+      g1 = constant_op.constant(11.0)
+      g2 = constant_op.constant(13.0)
+      g3 = constant_op.constant(17.0)
+
+    @function.defun
+    def func(g1, g2, c1, g3, c2):  # pylint: disable=unused-argument
+      # arguments g1 and g2 are unused and can be pruned by grappler.
+      return c1 * g3 * c2
+
+    result = func(g1, g2, c1, g3, c2)
+    self.assertEqual(result.numpy(), 5.0 * 7.0 * 17.0)
+
 if __name__ == '__main__':
   ops.enable_eager_execution(
       config=config_pb2.ConfigProto(device_count={'CPU': 4}))
diff --git a/tensorflow/python/eager/pywrap_tfe_test.py b/tensorflow/python/eager/pywrap_tfe_test.py
index 669fa08488..2abd4678af 100644
--- a/tensorflow/python/eager/pywrap_tfe_test.py
+++ b/tensorflow/python/eager/pywrap_tfe_test.py
@@ -254,6 +254,11 @@ class Tests(test.TestCase):
         "Value for attr 'num_split' of -1 must be at least minimum 1"):
       array_ops.split(value=[1, 2, 3], num_or_size_splits=-1)
 
+    with self.assertRaisesRegexp(
+        Exception,
+        "Value for attr 'num_split' of 0 must be at least minimum 1"):
+      array_ops.split(value=[1, 2, 3], num_or_size_splits=0)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/ops/functional_ops.py b/tensorflow/python/ops/functional_ops.py
index df4be1d65a..b240d1e465 100644
--- a/tensorflow/python/ops/functional_ops.py
+++ b/tensorflow/python/ops/functional_ops.py
@@ -19,7 +19,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.framework import attr_value_pb2
 from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
@@ -40,6 +39,7 @@ from tensorflow.python.ops.gen_functional_ops import remote_call
 from tensorflow.python.ops.gen_functional_ops import symbolic_gradient
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import compat
+from tensorflow.python.util import function_utils
 from tensorflow.python.util import nest
 from tensorflow.python.util.tf_export import tf_export
 
@@ -1024,17 +1024,6 @@ def For(start,
   return ret
 # pylint: enable=invalid-name,protected-access
 
-_rewriter_config_optimizer_disabled = None
-
-def _get_disabled_rewriter_config():
-  global _rewriter_config_optimizer_disabled
-  if _rewriter_config_optimizer_disabled is None:
-    config = config_pb2.ConfigProto()
-    rewriter_config = config.graph_options.rewrite_options
-    rewriter_config.disable_meta_optimizer = True
-    _rewriter_config_optimizer_disabled = config.SerializeToString()
-  return _rewriter_config_optimizer_disabled
-
 
 def partitioned_call(args, f, tout=None, executing_eagerly=None, config=None,
                      executor_type=None):
@@ -1071,7 +1060,7 @@ def partitioned_call(args, f, tout=None, executing_eagerly=None, config=None,
     executing_eagerly = context.executing_eagerly()
 
   if config is None:
-    config = _get_disabled_rewriter_config()
+    config = function_utils.get_disabled_rewriter_config()
 
   if executor_type is None:
     executor_type = ""
@@ -1103,7 +1092,8 @@ def partitioned_call(args, f, tout=None, executing_eagerly=None, config=None,
   # When running in graph mode, the graph and function graphs are optimized
   # (i.e. run through grappler) per the session options, so we can disable any
   # eager-specific rewriting.
-  config_proto = attr_value_pb2.AttrValue(s=_get_disabled_rewriter_config())
+  config_proto = attr_value_pb2.AttrValue(
+      s=function_utils.get_disabled_rewriter_config())
 
   graph = ops.get_default_graph()
   f.add_to_graph(graph)
diff --git a/tensorflow/python/util/function_utils.py b/tensorflow/python/util/function_utils.py
index a56dfbff8e..84e45bec6f 100644
--- a/tensorflow/python/util/function_utils.py
+++ b/tensorflow/python/util/function_utils.py
@@ -22,6 +22,7 @@ import functools
 
 import six
 
+from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.util import tf_decorator
 from tensorflow.python.util import tf_inspect
 
@@ -113,3 +114,16 @@ def get_func_code(func):
       return None
   else:
     raise ValueError('Argument must be callable')
+
+
+_rewriter_config_optimizer_disabled = None
+
+
+def get_disabled_rewriter_config():
+  global _rewriter_config_optimizer_disabled
+  if _rewriter_config_optimizer_disabled is None:
+    config = config_pb2.ConfigProto()
+    rewriter_config = config.graph_options.rewrite_options
+    rewriter_config.disable_meta_optimizer = True
+    _rewriter_config_optimizer_disabled = config.SerializeToString()
+  return _rewriter_config_optimizer_disabled
-- 
GitLab


From ef32bf6e22f6c9ec701d32b8625e87047aa08d15 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 11 Jan 2019 18:11:21 -0800
Subject: [PATCH 0612/2345] [XLA] Don't copy parameters in HloEvaluator.

Previously, every kParameter instruction resulted in a copy of the parameter's
literal.  This is unnecessary.

Speeds up one testcase in reduce_window_test_gpu from 17s to 13s.

PiperOrigin-RevId: 228976711
---
 .../compiler/xla/service/hlo_evaluator.cc      |  6 +++++-
 .../compiler/xla/service/hlo_evaluator.h       | 18 ++++++++++++++++--
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index bcfa96d17f..7589c99202 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -416,15 +416,19 @@ Status HloEvaluator::HandleGetDimensionSize(
 }
 
 Status HloEvaluator::HandleParameter(HloInstruction* parameter) {
+  // Nothing to do other than sanity checks. Parameters' values are stored in
+  // arg_literals_.
   CHECK_LT(parameter->parameter_number(), arg_literals_.size());
+
+#ifndef NDEBUG
   const Literal* input_literal = arg_literals_[parameter->parameter_number()];
   VLOG(2) << "Parameter evaluated to: " << input_literal->ToString();
   DCHECK(ShapeUtil::Equal(parameter->shape(), input_literal->shape()))
       << "parameter shape is: " << ShapeUtil::HumanString(parameter->shape())
       << ", but input literal shape is: "
       << ShapeUtil::HumanString(input_literal->shape());
+#endif
 
-  evaluated_[parameter] = input_literal->Clone();
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.h b/tensorflow/compiler/xla/service/hlo_evaluator.h
index 604b861913..ccb8af4fb0 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.h
@@ -243,13 +243,21 @@ class HloEvaluator : public DfsHloVisitorWithDefault {
   };
 
   // Returns the already-evaluated literal result for the instruction.
+  //
   // A Constant instruction is considered evaluated and its literal will be
   // returned directly without looking up the cache.
+  //
+  // Similarly, a Parameter instruction is considered evaluated and its literal
+  // is looked up in arg_literals.
+  //
   // Crash with log if the given instruction has not been evaluated previously.
   const Literal& GetEvaluatedLiteralFor(const HloInstruction* hlo) {
     if (hlo->IsConstant()) {
       return hlo->literal();
     }
+    if (hlo->opcode() == HloOpcode::kParameter) {
+      return *arg_literals_.at(hlo->parameter_number());
+    }
     auto it = evaluated_.find(hlo);
     CHECK(it != evaluated_.end())
         << "could not find evaluated value for: " << hlo->ToString();
@@ -257,12 +265,18 @@ class HloEvaluator : public DfsHloVisitorWithDefault {
   }
 
   // Tracks the HLO instruction and its evaluated literal result.
+  //
+  // Parameters and constants aren't stored here, see implementation of
+  // GetEvaluatedLiteralFor.
+  //
   // TODO(b/35950897): have better memory management here to free instructions
   // that are no longer a parent for any other subsequent instruction in
   // post-orderring.
+  //
   // Must be cleared for each evaluation.
-  // Storing Literal in place require the container to have pointer stability so
-  // we cannot use flat_hash_map any more.
+  //
+  // Storing Literal in place requires the container to have pointer stability
+  // so we cannot use flat_hash_map any more.
   absl::node_hash_map<const HloInstruction*, Literal> evaluated_;
 
   // Use fast path that uses eigen in the evaluator.
-- 
GitLab


From ff2cd836a36281f9fc01a9a04c2a4ea71ab42235 Mon Sep 17 00:00:00 2001
From: Ian Langmore <langmore@google.com>
Date: Fri, 11 Jan 2019 18:17:15 -0800
Subject: [PATCH 0613/2345] Auto-set `is_self_adjoint` hint for
 LinearOperatorScaledIdentity.

PiperOrigin-RevId: 228977102
---
 .../linalg/linear_operator_identity_test.py         |  3 ++-
 .../python/ops/linalg/linear_operator_identity.py   | 13 ++++++++++---
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
index 14b5228bca..ea9ee99a58 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
@@ -463,7 +463,7 @@ class LinearOperatorScaledIdentityTest(
         is_positive_definite=False, is_non_singular=True)
     self.assertFalse(operator.is_positive_definite)
     self.assertTrue(operator.is_non_singular)
-    self.assertTrue(operator.is_self_adjoint is None)
+    self.assertTrue(operator.is_self_adjoint)  # Auto-set due to real multiplier
 
   def test_identity_matmul(self):
     operator1 = linalg_lib.LinearOperatorIdentity(num_rows=2)
@@ -510,5 +510,6 @@ class LinearOperatorScaledIdentityTest(
         operator.inverse(),
         linalg_lib.LinearOperatorScaledIdentity)
 
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/ops/linalg/linear_operator_identity.py b/tensorflow/python/ops/linalg/linear_operator_identity.py
index 32b222cb2a..694557e50a 100644
--- a/tensorflow/python/ops/linalg/linear_operator_identity.py
+++ b/tensorflow/python/ops/linalg/linear_operator_identity.py
@@ -588,12 +588,19 @@ class LinearOperatorScaledIdentity(BaseLinearOperatorIdentity):
     """
     self._assert_proper_shapes = assert_proper_shapes
 
-    if not is_square:
-      raise ValueError("A ScaledIdentity operator is always square.")
-
     with ops.name_scope(name, values=[multiplier, num_rows]):
       self._multiplier = ops.convert_to_tensor(multiplier, name="multiplier")
 
+      # Check and auto-set hints.
+      if not self._multiplier.dtype.is_complex:
+        if is_self_adjoint is False:  # pylint: disable=g-bool-id-comparison
+          raise ValueError("A real diagonal operator is always self adjoint.")
+        else:
+          is_self_adjoint = True
+
+      if not is_square:
+        raise ValueError("A ScaledIdentity operator is always square.")
+
       super(LinearOperatorScaledIdentity, self).__init__(
           dtype=self._multiplier.dtype,
           is_non_singular=is_non_singular,
-- 
GitLab


From a79cdcf41968cdadce2c65ec4933a9cf5998910b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 18:18:03 -0800
Subject: [PATCH 0614/2345] [tf.keras] Add warning message when non-shuffled
 dataset input is passed when `shuffle` parameter of fit() is set to True .

PiperOrigin-RevId: 228977169
---
 tensorflow/python/keras/engine/training.py    | 23 ++++++++++++++-----
 .../python/keras/engine/training_utils.py     | 21 +++++++++++++++++
 .../keras/engine/training_utils_test.py       | 14 +++++++++++
 3 files changed, 52 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 3e6c13232c..598766db74 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -824,6 +824,10 @@ class Model(Network):
       # Make sure that y, sample_weights, validation_split are not passed.
       training_utils.validate_dataset_input(x, y, sample_weight,
                                             validation_split)
+      if (isinstance(x, (dataset_ops.DatasetV1, dataset_ops.DatasetV2))
+          and shuffle):
+        training_utils.verify_dataset_shuffled(x)
+
       return self.fit_generator(
           x,
           steps_per_epoch=steps_per_epoch,
@@ -2155,15 +2159,19 @@ class Model(Network):
       raise NotImplementedError('`sample_weight` is currently not supported '
                                 'when using TPUStrategy.')
 
-    # Validates `steps` argument right at the beginning since we use it to
-    # construct the dataset object.
+    # Validates `steps` and `shuffle` arguments right at the beginning
+    # since we use it to construct the dataset object.
     # TODO(anjalisridhar): Remove this check once we refactor the
     # _standardize_user_data code path. This check is already present elsewhere
     # in the codebase.
-    if check_steps and isinstance(x, dataset_ops.DatasetV2) and steps is None:
-      raise ValueError('When using Datasets as input, '
-                       'you should specify the `{steps_name}` argument.'
-                       .format(steps_name=steps_name))
+    if isinstance(x, dataset_ops.DatasetV2):
+      if shuffle:
+        training_utils.verify_dataset_shuffled(x)
+
+      if check_steps and steps is None:
+        raise ValueError('When using Datasets as input, '
+                         'you should specify the `{steps_name}` argument.'
+                         .format(steps_name=steps_name))
 
     if ops.executing_eagerly_outside_functions():
       session = None
@@ -2286,6 +2294,9 @@ class Model(Network):
       # the tensors from the dataset and we output them.
       training_utils.validate_dataset_input(x, y, sample_weight,
                                             validation_split)
+      if shuffle:
+        training_utils.verify_dataset_shuffled(x)
+
       is_dataset = True
       if extract_tensors_from_dataset:
         # We do this for `train_on_batch`/etc.
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index 0c3beb4681..f7cbf4101a 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -45,6 +45,7 @@ from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensi
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import weights_broadcast_ops
+from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import nest
 
 
@@ -1200,6 +1201,26 @@ def assert_not_shuffled(dataset):
     raise ValueError('Could not assert that dataset is not shuffled.')
 
 
+def verify_dataset_shuffled(x):
+  """Verifies that the dataset is shuffled.
+
+  Args:
+    x: Dataset passed as an input to the model.
+
+  Raises:
+    ValueError: if the dataset is not already shuffled.
+  """
+  assert isinstance(x, dataset_ops.DatasetV2)
+  try:
+    assert_not_shuffled(x)
+  except ValueError:
+    # Dataset may or may not be shuffled.
+    return
+  else:
+    logging.warning('Expected a shuffled dataset but input dataset `x` is '
+                    'not shuffled. Please invoke `shuffle()` on input dataset.')
+
+
 def is_dataset_or_iterator(data):
   return isinstance(data, (dataset_ops.DatasetV1,
                            dataset_ops.DatasetV2,
diff --git a/tensorflow/python/keras/engine/training_utils_test.py b/tensorflow/python/keras/engine/training_utils_test.py
index d8d8c58610..ec44252c9b 100644
--- a/tensorflow/python/keras/engine/training_utils_test.py
+++ b/tensorflow/python/keras/engine/training_utils_test.py
@@ -30,6 +30,7 @@ from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras.engine import training_utils
 from tensorflow.python.keras.utils import tf_utils
 from tensorflow.python.platform import test
+from tensorflow.python.platform import tf_logging as logging
 
 
 class ModelInputsTest(test.TestCase):
@@ -187,6 +188,19 @@ class DatasetUtilsTest(test.TestCase, parameterized.TestCase):
       with self.assertRaises(expected_error):
         training_utils.assert_not_shuffled(dataset_fn())
 
+  def test_verify_dataset_shuffled(self):
+    dataset = dataset_ops.Dataset.range(5)
+    training_utils.assert_not_shuffled(dataset)
+
+    with test.mock.patch.object(logging, 'warning') as mock_log:
+      training_utils.verify_dataset_shuffled(dataset)
+      self.assertRegexpMatches(
+          str(mock_log.call_args),
+          'input dataset `x` is not shuffled.')
+
+    shuffled_dataset = dataset.shuffle(10)
+    training_utils.verify_dataset_shuffled(shuffled_dataset)
+
 
 class StandardizeWeightsTest(keras_parameterized.TestCase):
 
-- 
GitLab


From ba7f78b9b0535f38e3e12dcb0b095fc7a6e7bca5 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Fri, 11 Jan 2019 18:28:05 -0800
Subject: [PATCH 0615/2345] Adds v2 AUC metric.

PiperOrigin-RevId: 228978078
---
 tensorflow/python/keras/metrics.py            | 307 ++++++++++++++++--
 .../keras/metrics_confusion_matrix_test.py    | 249 ++++++++++++--
 tensorflow/python/keras/metrics_test.py       |  65 ++--
 .../python/keras/utils/metrics_utils.py       |  11 +
 4 files changed, 556 insertions(+), 76 deletions(-)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index c04487f175..ec09ad9ba8 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -1089,11 +1089,11 @@ class Precision(Metric):
     default_threshold = 0.5 if top_k is None else metrics_utils.NEG_INF
     self.thresholds = metrics_utils.parse_init_thresholds(
         thresholds, default_threshold=default_threshold)
-    self.tp = self.add_weight(
+    self.true_positives = self.add_weight(
         'true_positives',
         shape=(len(self.thresholds),),
         initializer=init_ops.zeros_initializer)
-    self.fp = self.add_weight(
+    self.false_positives = self.add_weight(
         'false_positives',
         shape=(len(self.thresholds),),
         initializer=init_ops.zeros_initializer)
@@ -1114,8 +1114,8 @@ class Precision(Metric):
     """
     return metrics_utils.update_confusion_matrix_variables(
         {
-            metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.tp,
-            metrics_utils.ConfusionMatrix.FALSE_POSITIVES: self.fp
+            metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.true_positives,
+            metrics_utils.ConfusionMatrix.FALSE_POSITIVES: self.false_positives
         },
         y_true,
         y_pred,
@@ -1125,7 +1125,8 @@ class Precision(Metric):
         sample_weight=sample_weight)
 
   def result(self):
-    result = math_ops.div_no_nan(self.tp, self.tp + self.fp)
+    result = math_ops.div_no_nan(self.true_positives,
+                                 self.true_positives + self.false_positives)
     return result[0] if len(self.thresholds) == 1 else result
 
   def reset_states(self):
@@ -1214,11 +1215,11 @@ class Recall(Metric):
     default_threshold = 0.5 if top_k is None else metrics_utils.NEG_INF
     self.thresholds = metrics_utils.parse_init_thresholds(
         thresholds, default_threshold=default_threshold)
-    self.tp = self.add_weight(
+    self.true_positives = self.add_weight(
         'true_positives',
         shape=(len(self.thresholds),),
         initializer=init_ops.zeros_initializer)
-    self.fn = self.add_weight(
+    self.false_negatives = self.add_weight(
         'false_negatives',
         shape=(len(self.thresholds),),
         initializer=init_ops.zeros_initializer)
@@ -1239,8 +1240,8 @@ class Recall(Metric):
     """
     return metrics_utils.update_confusion_matrix_variables(
         {
-            metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.tp,
-            metrics_utils.ConfusionMatrix.FALSE_NEGATIVES: self.fn
+            metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.true_positives,
+            metrics_utils.ConfusionMatrix.FALSE_NEGATIVES: self.false_negatives
         },
         y_true,
         y_pred,
@@ -1250,7 +1251,8 @@ class Recall(Metric):
         sample_weight=sample_weight)
 
   def result(self):
-    result = math_ops.div_no_nan(self.tp, self.tp + self.fn)
+    result = math_ops.div_no_nan(self.true_positives,
+                                 self.true_positives + self.false_negatives)
     return result[0] if len(self.thresholds) == 1 else result
 
   def reset_states(self):
@@ -1281,19 +1283,19 @@ class SensitivitySpecificityBase(Metric):
     if num_thresholds <= 0:
       raise ValueError('`num_thresholds` must be > 0.')
     self.value = value
-    self.tp = self.add_weight(
+    self.true_positives = self.add_weight(
         'true_positives',
         shape=(num_thresholds,),
         initializer=init_ops.zeros_initializer)
-    self.tn = self.add_weight(
+    self.true_negatives = self.add_weight(
         'true_negatives',
         shape=(num_thresholds,),
         initializer=init_ops.zeros_initializer)
-    self.fp = self.add_weight(
+    self.false_positives = self.add_weight(
         'false_positives',
         shape=(num_thresholds,),
         initializer=init_ops.zeros_initializer)
-    self.fn = self.add_weight(
+    self.false_negatives = self.add_weight(
         'false_negatives',
         shape=(num_thresholds,),
         initializer=init_ops.zeros_initializer)
@@ -1321,10 +1323,10 @@ class SensitivitySpecificityBase(Metric):
     """
     return metrics_utils.update_confusion_matrix_variables(
         {
-            metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.tp,
-            metrics_utils.ConfusionMatrix.TRUE_NEGATIVES: self.tn,
-            metrics_utils.ConfusionMatrix.FALSE_POSITIVES: self.fp,
-            metrics_utils.ConfusionMatrix.FALSE_NEGATIVES: self.fn,
+            metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.true_positives,
+            metrics_utils.ConfusionMatrix.TRUE_NEGATIVES: self.true_negatives,
+            metrics_utils.ConfusionMatrix.FALSE_POSITIVES: self.false_positives,
+            metrics_utils.ConfusionMatrix.FALSE_NEGATIVES: self.false_negatives,
         },
         y_true,
         y_pred,
@@ -1395,7 +1397,8 @@ class SensitivityAtSpecificity(SensitivitySpecificityBase):
 
   def result(self):
     # Calculate specificities at all the thresholds.
-    specificities = math_ops.div_no_nan(self.tn, self.tn + self.fp)
+    specificities = math_ops.div_no_nan(
+        self.true_negatives, self.true_negatives + self.false_positives)
 
     # Find the index of the threshold where the specificity is closest to the
     # given specificity.
@@ -1404,8 +1407,9 @@ class SensitivityAtSpecificity(SensitivitySpecificityBase):
     min_index = math_ops.cast(min_index, dtypes.int32)
 
     # Compute sensitivity at that index.
-    return math_ops.div_no_nan(self.tp[min_index],
-                               self.tp[min_index] + self.fn[min_index])
+    return math_ops.div_no_nan(
+        self.true_positives[min_index],
+        self.true_positives[min_index] + self.false_negatives[min_index])
 
   def get_config(self):
     config = {
@@ -1474,7 +1478,8 @@ class SpecificityAtSensitivity(SensitivitySpecificityBase):
 
   def result(self):
     # Calculate sensitivities at all the thresholds.
-    sensitivities = math_ops.div_no_nan(self.tp, self.tp + self.fn)
+    sensitivities = math_ops.div_no_nan(
+        self.true_positives, self.true_positives + self.false_negatives)
 
     # Find the index of the threshold where the sensitivity is closest to the
     # given specificity.
@@ -1483,8 +1488,9 @@ class SpecificityAtSensitivity(SensitivitySpecificityBase):
     min_index = math_ops.cast(min_index, dtypes.int32)
 
     # Compute specificity at that index.
-    return math_ops.div_no_nan(self.tn[min_index],
-                               self.tn[min_index] + self.fp[min_index])
+    return math_ops.div_no_nan(
+        self.true_negatives[min_index],
+        self.true_negatives[min_index] + self.false_positives[min_index])
 
   def get_config(self):
     config = {
@@ -1495,6 +1501,259 @@ class SpecificityAtSensitivity(SensitivitySpecificityBase):
     return dict(list(base_config.items()) + list(config.items()))
 
 
+class AUC(Metric):
+  """Computes the approximate AUC (Area under the curve) via a Riemann sum.
+
+  This metric creates four local variables, `true_positives`, `true_negatives`,
+  `false_positives` and `false_negatives` that are used to compute the AUC.
+  To discretize the AUC curve, a linearly spaced set of thresholds is used to
+  compute pairs of recall and precision values. The area under the ROC-curve is
+  therefore computed using the height of the recall values by the false positive
+  rate, while the area under the PR-curve is the computed using the height of
+  the precision values by the recall.
+
+  This value is ultimately returned as `auc`, an idempotent operation that
+  computes the area under a discretized curve of precision versus recall values
+  (computed using the aforementioned variables). The `num_thresholds` variable
+  controls the degree of discretization with larger numbers of thresholds more
+  closely approximating the true AUC. The quality of the approximation may vary
+  dramatically depending on `num_thresholds`.
+
+  For best results, `predictions` should be distributed approximately uniformly
+  in the range [0, 1] and not peaked around 0 or 1. The quality of the AUC
+  approximation may be poor if this is not the case. Setting `summation_method`
+  to 'minoring' or 'majoring' can help quantify the error in the approximation
+  by providing lower or upper bound estimate of the AUC.
+
+  If `sample_weight` is `None`, weights default to 1.
+  Use `sample_weight` of 0 to mask values.
+
+  Usage:
+
+  ```python
+  m = tf.keras.metrics.AUC(num_thresholds=3)
+  m.update_state([0, 0, 1, 1], [0, 0.5, 0.3, 0.9])
+
+  # threshold values are [0 - 1e-7, 0.5, 1 + 1e-7]
+  # tp = [2, 1, 0], fp = [2, 0, 0], fn = [0, 1, 2], tn = [0, 2, 2]
+  # recall = [1, 0.5, 0], fp_rate = [1, 0, 0]
+  # auc = ((((1+0.5)/2)*(1-0))+ (((0.5+0)/2)*(0-0))) = 0.75
+
+  print('Final result: ', m.result().numpy())  # Final result: 0.75
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', loss='mse', metrics=[tf.keras.metrics.AUC()])
+  ```
+  """
+
+  def __init__(self,
+               num_thresholds=200,
+               curve=metrics_utils.AUCCurve.ROC,
+               summation_method=metrics_utils.AUCSummationMethod.INTERPOLATION,
+               name=None,
+               dtype=None):
+    """Creates an `AUC` instance.
+
+    Args:
+      num_thresholds: (Optional) Defaults to 200. The number of thresholds to
+        use when discretizing the roc curve. Values must be > 1.
+      curve: (Optional) Specifies the name of the curve to be computed, 'ROC'
+        [default] or 'PR' for the Precision-Recall-curve.
+      summation_method: (Optional) Specifies the Riemann summation method used
+        (https://en.wikipedia.org/wiki/Riemann_sum): 'interpolation' [default],
+          applies mid-point summation scheme for `ROC`. For PR-AUC, interpolates
+          (true/false) positives but not the ratio that is precision (see Davis
+          & Goadrich 2006 for details); 'minoring' that applies left summation
+          for increasing intervals and right summation for decreasing intervals;
+          'majoring' that does the opposite.
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+    """
+    # Validate configurations.
+    if num_thresholds <= 1:
+      raise ValueError('`num_thresholds` must be > 1.')
+    if curve not in list(metrics_utils.AUCCurve):
+      raise ValueError('Invalid curve: "{}". Valid options are: "{}"'.format(
+          curve, list(metrics_utils.AUCCurve)))
+    if summation_method not in list(metrics_utils.AUCSummationMethod):
+      raise ValueError(
+          'Invalid summation method: "{}". Valid options are: "{}"'.format(
+              summation_method, list(metrics_utils.AUCSummationMethod)))
+
+    # Update properties.
+    self.num_thresholds = num_thresholds
+    self.curve = curve
+    self.summation_method = summation_method
+    super(AUC, self).__init__(name=name, dtype=dtype)
+
+    # Create metric variables
+    self.true_positives = self.add_weight(
+        'true_positives',
+        shape=(num_thresholds,),
+        initializer=init_ops.zeros_initializer)
+    self.true_negatives = self.add_weight(
+        'true_negatives',
+        shape=(num_thresholds,),
+        initializer=init_ops.zeros_initializer)
+    self.false_positives = self.add_weight(
+        'false_positives',
+        shape=(num_thresholds,),
+        initializer=init_ops.zeros_initializer)
+    self.false_negatives = self.add_weight(
+        'false_negatives',
+        shape=(num_thresholds,),
+        initializer=init_ops.zeros_initializer)
+
+    # Compute `num_thresholds` thresholds in [0, 1]
+    thresholds = [
+        (i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2)
+    ]
+    self.thresholds = [0.0 - K.epsilon()] + thresholds + [1.0 + K.epsilon()]
+    # epsilon - to account for floating point imprecisions.
+
+  def update_state(self, y_true, y_pred, sample_weight=None):
+    """Accumulates confusion matrix statistics.
+
+    Args:
+      y_true: The ground truth values.
+      y_pred: The predicted values.
+      sample_weight: Optional weighting of each example. Defaults to 1. Can be a
+        `Tensor` whose rank is either 0, or the same rank as `y_true`, and must
+        be broadcastable to `y_true`.
+
+    Returns:
+      Update op.
+    """
+    return metrics_utils.update_confusion_matrix_variables({
+        metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.true_positives,
+        metrics_utils.ConfusionMatrix.TRUE_NEGATIVES: self.true_negatives,
+        metrics_utils.ConfusionMatrix.FALSE_POSITIVES: self.false_positives,
+        metrics_utils.ConfusionMatrix.FALSE_NEGATIVES: self.false_negatives,
+    }, y_true, y_pred, self.thresholds, sample_weight=sample_weight)
+
+  def interpolate_pr_auc(self):
+    """Interpolation formula inspired by section 4 of Davis & Goadrich 2006.
+
+    https://www.biostat.wisc.edu/~page/rocpr.pdf
+
+    Note here we derive & use a closed formula not present in the paper
+    as follows:
+
+      Precision = TP / (TP + FP) = TP / P
+
+    Modeling all of TP (true positive), FP (false positive) and their sum
+    P = TP + FP (predicted positive) as varying linearly within each interval
+    [A, B] between successive thresholds, we get
+
+      Precision slope = dTP / dP
+                      = (TP_B - TP_A) / (P_B - P_A)
+                      = (TP - TP_A) / (P - P_A)
+      Precision = (TP_A + slope * (P - P_A)) / P
+
+    The area within the interval is (slope / total_pos_weight) times
+
+      int_A^B{Precision.dP} = int_A^B{(TP_A + slope * (P - P_A)) * dP / P}
+      int_A^B{Precision.dP} = int_A^B{slope * dP + intercept * dP / P}
+
+    where intercept = TP_A - slope * P_A = TP_B - slope * P_B, resulting in
+
+      int_A^B{Precision.dP} = TP_B - TP_A + intercept * log(P_B / P_A)
+
+    Bringing back the factor (slope / total_pos_weight) we'd put aside, we get
+
+      slope * [dTP + intercept *  log(P_B / P_A)] / total_pos_weight
+
+    where dTP == TP_B - TP_A.
+
+    Note that when P_A == 0 the above calculation simplifies into
+
+      int_A^B{Precision.dTP} = int_A^B{slope * dTP} = slope * (TP_B - TP_A)
+
+    which is really equivalent to imputing constant precision throughout the
+    first bucket having >0 true positives.
+
+    Returns:
+      pr_auc: an approximation of the area under the P-R curve.
+    """
+    dtp = self.true_positives[:self.num_thresholds -
+                              1] - self.true_positives[1:]
+    p = self.true_positives + self.false_positives
+    dp = p[:self.num_thresholds - 1] - p[1:]
+
+    prec_slope = math_ops.div_no_nan(
+        dtp, math_ops.maximum(dp, 0), name='prec_slope')
+    intercept = self.true_positives[1:] - math_ops.multiply(prec_slope, p[1:])
+
+    safe_p_ratio = array_ops.where(
+        math_ops.logical_and(p[:self.num_thresholds - 1] > 0, p[1:] > 0),
+        math_ops.div_no_nan(
+            p[:self.num_thresholds - 1],
+            math_ops.maximum(p[1:], 0),
+            name='recall_relative_ratio'),
+        array_ops.ones_like(p[1:]))
+
+    return math_ops.reduce_sum(
+        math_ops.div_no_nan(
+            prec_slope * (dtp + intercept * math_ops.log(safe_p_ratio)),
+            math_ops.maximum(self.true_positives[1:] + self.false_negatives[1:],
+                             0),
+            name='pr_auc_increment'),
+        name='interpolate_pr_auc')
+
+  def result(self):
+    if (self.curve == metrics_utils.AUCCurve.PR and
+        self.summation_method == metrics_utils.AUCSummationMethod.INTERPOLATION
+       ):
+      # This use case is different and is handled separately.
+      return self.interpolate_pr_auc()
+
+    # Set `x` and `y` values for the curves based on `curve` config.
+    recall = math_ops.div_no_nan(self.true_positives,
+                                 self.true_positives + self.false_negatives)
+    if self.curve == metrics_utils.AUCCurve.ROC:
+      fp_rate = math_ops.div_no_nan(self.false_positives,
+                                    self.false_positives + self.true_negatives)
+      x = fp_rate
+      y = recall
+    else:  # curve == 'PR'.
+      precision = math_ops.div_no_nan(
+          self.true_positives, self.true_positives + self.false_positives)
+      x = recall
+      y = precision
+
+    # Find the rectangle heights based on `summation_method`.
+    if self.summation_method == metrics_utils.AUCSummationMethod.INTERPOLATION:
+      # Note: the case ('PR', 'interpolation') has been handled above.
+      heights = (y[:self.num_thresholds - 1] + y[1:]) / 2.
+    elif self.summation_method == metrics_utils.AUCSummationMethod.MINORING:
+      heights = math_ops.minimum(y[:self.num_thresholds - 1], y[1:])
+    else:  # self.summation_method = metrics_utils.AUCSummationMethod.MAJORING:
+      heights = math_ops.maximum(y[:self.num_thresholds - 1], y[1:])
+
+    # Sum up the areas of all the rectangles.
+    return math_ops.reduce_sum(
+        math_ops.multiply(x[:self.num_thresholds - 1] - x[1:], heights),
+        name=self.name)
+
+  def reset_states(self):
+    num_thresholds = len(self.thresholds)
+    for v in self.variables:
+      K.set_value(v, np.zeros((num_thresholds,)))
+
+  def get_config(self):
+    config = {
+        'num_thresholds': self.num_thresholds,
+        'curve': self.curve,
+        'summation_method': self.summation_method,
+    }
+    base_config = super(AUC, self).get_config()
+    return dict(list(base_config.items()) + list(config.items()))
+
+
 @keras_export('keras.metrics.CosineProximity')
 class CosineProximity(MeanMetricWrapper):
   """Computes the cosine distance between the labels and predictions.
diff --git a/tensorflow/python/keras/metrics_confusion_matrix_test.py b/tensorflow/python/keras/metrics_confusion_matrix_test.py
index 6db034b721..6a0fcfed07 100644
--- a/tensorflow/python/keras/metrics_confusion_matrix_test.py
+++ b/tensorflow/python/keras/metrics_confusion_matrix_test.py
@@ -25,6 +25,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import test_util
 from tensorflow.python.keras import metrics
+from tensorflow.python.keras.utils import metrics_utils
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import variables
@@ -472,22 +473,22 @@ class PrecisionTest(test.TestCase):
     y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
     result = p_obj(y_true, y_pred)
     self.assertAlmostEqual(1, self.evaluate(result))
-    self.assertAlmostEqual(1, self.evaluate(p_obj.tp))
-    self.assertAlmostEqual(0, self.evaluate(p_obj.fp))
+    self.assertAlmostEqual(1, self.evaluate(p_obj.true_positives))
+    self.assertAlmostEqual(0, self.evaluate(p_obj.false_positives))
 
     y_pred = constant_op.constant([0.2, 0.1, 0, 0, 0.2], shape=(1, 5))
     y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
     result = p_obj(y_true, y_pred)
     self.assertAlmostEqual(1, self.evaluate(result))
-    self.assertAlmostEqual(1, self.evaluate(p_obj.tp))
-    self.assertAlmostEqual(0, self.evaluate(p_obj.fp))
+    self.assertAlmostEqual(1, self.evaluate(p_obj.true_positives))
+    self.assertAlmostEqual(0, self.evaluate(p_obj.false_positives))
 
     y_pred = constant_op.constant([0.2, 0.1, 0.6, 0, 0.2], shape=(1, 5))
     y_true = constant_op.constant([0, 1, 0, 0, 0], shape=(1, 5))
     result = p_obj(y_true, y_pred)
     self.assertAlmostEqual(0.5, self.evaluate(result))
-    self.assertAlmostEqual(1, self.evaluate(p_obj.tp))
-    self.assertAlmostEqual(1, self.evaluate(p_obj.fp))
+    self.assertAlmostEqual(1, self.evaluate(p_obj.true_positives))
+    self.assertAlmostEqual(1, self.evaluate(p_obj.false_positives))
 
   def test_unweighted_top_k_and_class_id(self):
     p_obj = metrics.Precision(class_id=2, top_k=2)
@@ -497,15 +498,15 @@ class PrecisionTest(test.TestCase):
     y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
     result = p_obj(y_true, y_pred)
     self.assertAlmostEqual(1, self.evaluate(result))
-    self.assertAlmostEqual(1, self.evaluate(p_obj.tp))
-    self.assertAlmostEqual(0, self.evaluate(p_obj.fp))
+    self.assertAlmostEqual(1, self.evaluate(p_obj.true_positives))
+    self.assertAlmostEqual(0, self.evaluate(p_obj.false_positives))
 
     y_pred = constant_op.constant([1, 1, 0.9, 1, 1], shape=(1, 5))
     y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
     result = p_obj(y_true, y_pred)
     self.assertAlmostEqual(1, self.evaluate(result))
-    self.assertAlmostEqual(1, self.evaluate(p_obj.tp))
-    self.assertAlmostEqual(0, self.evaluate(p_obj.fp))
+    self.assertAlmostEqual(1, self.evaluate(p_obj.true_positives))
+    self.assertAlmostEqual(0, self.evaluate(p_obj.false_positives))
 
   def test_unweighted_top_k_and_threshold(self):
     p_obj = metrics.Precision(thresholds=.7, top_k=2)
@@ -515,8 +516,8 @@ class PrecisionTest(test.TestCase):
     y_true = constant_op.constant([0, 1, 1, 0, 1], shape=(1, 5))
     result = p_obj(y_true, y_pred)
     self.assertAlmostEqual(1, self.evaluate(result))
-    self.assertAlmostEqual(1, self.evaluate(p_obj.tp))
-    self.assertAlmostEqual(0, self.evaluate(p_obj.fp))
+    self.assertAlmostEqual(1, self.evaluate(p_obj.true_positives))
+    self.assertAlmostEqual(0, self.evaluate(p_obj.false_positives))
 
 
 @test_util.run_all_in_graph_and_eager_modes
@@ -676,22 +677,22 @@ class RecallTest(test.TestCase):
     y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
     result = r_obj(y_true, y_pred)
     self.assertAlmostEqual(1, self.evaluate(result))
-    self.assertAlmostEqual(1, self.evaluate(r_obj.tp))
-    self.assertAlmostEqual(0, self.evaluate(r_obj.fn))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.true_positives))
+    self.assertAlmostEqual(0, self.evaluate(r_obj.false_negatives))
 
     y_pred = constant_op.constant([0.2, 0.1, 0, 0, 0.2], shape=(1, 5))
     y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
     result = r_obj(y_true, y_pred)
     self.assertAlmostEqual(0.5, self.evaluate(result))
-    self.assertAlmostEqual(1, self.evaluate(r_obj.tp))
-    self.assertAlmostEqual(1, self.evaluate(r_obj.fn))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.true_positives))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.false_negatives))
 
     y_pred = constant_op.constant([0.2, 0.1, 0.6, 0, 0.2], shape=(1, 5))
     y_true = constant_op.constant([0, 1, 0, 0, 0], shape=(1, 5))
     result = r_obj(y_true, y_pred)
     self.assertAlmostEqual(0.5, self.evaluate(result))
-    self.assertAlmostEqual(1, self.evaluate(r_obj.tp))
-    self.assertAlmostEqual(1, self.evaluate(r_obj.fn))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.true_positives))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.false_negatives))
 
   def test_unweighted_top_k_and_class_id(self):
     r_obj = metrics.Recall(class_id=2, top_k=2)
@@ -701,15 +702,15 @@ class RecallTest(test.TestCase):
     y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
     result = r_obj(y_true, y_pred)
     self.assertAlmostEqual(1, self.evaluate(result))
-    self.assertAlmostEqual(1, self.evaluate(r_obj.tp))
-    self.assertAlmostEqual(0, self.evaluate(r_obj.fn))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.true_positives))
+    self.assertAlmostEqual(0, self.evaluate(r_obj.false_negatives))
 
     y_pred = constant_op.constant([1, 1, 0.9, 1, 1], shape=(1, 5))
     y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
     result = r_obj(y_true, y_pred)
     self.assertAlmostEqual(0.5, self.evaluate(result))
-    self.assertAlmostEqual(1, self.evaluate(r_obj.tp))
-    self.assertAlmostEqual(1, self.evaluate(r_obj.fn))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.true_positives))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.false_negatives))
 
   def test_unweighted_top_k_and_threshold(self):
     r_obj = metrics.Recall(thresholds=.7, top_k=2)
@@ -719,8 +720,8 @@ class RecallTest(test.TestCase):
     y_true = constant_op.constant([1, 1, 1, 0, 1], shape=(1, 5))
     result = r_obj(y_true, y_pred)
     self.assertAlmostEqual(0.25, self.evaluate(result))
-    self.assertAlmostEqual(1, self.evaluate(r_obj.tp))
-    self.assertAlmostEqual(3, self.evaluate(r_obj.fn))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.true_positives))
+    self.assertAlmostEqual(3, self.evaluate(r_obj.false_negatives))
 
 
 @test_util.run_all_in_graph_and_eager_modes
@@ -915,5 +916,205 @@ class SpecificityAtSensitivityTest(test.TestCase, parameterized.TestCase):
       metrics.SpecificityAtSensitivity(0.4, num_thresholds=-1)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class AUCTest(test.TestCase):
+
+  def setup(self):
+    self.num_thresholds = 3
+    self.y_pred = constant_op.constant([0, 0.5, 0.3, 0.9], dtype=dtypes.float32)
+    self.y_true = constant_op.constant([0, 0, 1, 1])
+    self.sample_weight = [1, 2, 3, 4]
+
+    # threshold values are [0 - 1e-7, 0.5, 1 + 1e-7]
+    # y_pred when threshold = 0 - 1e-7  : [1, 1, 1, 1]
+    # y_pred when threshold = 0.5       : [0, 0, 0, 1]
+    # y_pred when threshold = 1 + 1e-7  : [0, 0, 0, 0]
+
+    # without sample_weight:
+    # tp = np.sum([[0, 0, 1, 1], [0, 0, 0, 1], [0, 0, 0, 0]], axis=1)
+    # fp = np.sum([[1, 1, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]], axis=1)
+    # fn = np.sum([[0, 0, 0, 0], [0, 0, 1, 0], [0, 0, 1, 1]], axis=1)
+    # tn = np.sum([[0, 0, 0, 0], [1, 1, 0, 0], [1, 1, 0, 0]], axis=1)
+
+    # tp = [2, 1, 0], fp = [2, 0, 0], fn = [0, 1, 2], tn = [0, 2, 2]
+
+    # with sample_weight:
+    # tp = np.sum([[0, 0, 3, 4], [0, 0, 0, 4], [0, 0, 0, 0]], axis=1)
+    # fp = np.sum([[1, 2, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]], axis=1)
+    # fn = np.sum([[0, 0, 0, 0], [0, 0, 3, 0], [0, 0, 3, 4]], axis=1)
+    # tn = np.sum([[0, 0, 0, 0], [1, 2, 0, 0], [1, 2, 0, 0]], axis=1)
+
+    # tp = [7, 4, 0], fp = [3, 0, 0], fn = [0, 3, 7], tn = [0, 3, 3]
+
+  def test_config(self):
+    auc_obj = metrics.AUC(
+        num_thresholds=100,
+        curve=metrics_utils.AUCCurve.PR,
+        summation_method=metrics_utils.AUCSummationMethod.MAJORING,
+        name='auc_1')
+    self.assertEqual(auc_obj.name, 'auc_1')
+    self.assertEqual(len(auc_obj.variables), 4)
+    self.assertEqual(auc_obj.num_thresholds, 100)
+    self.assertEqual(auc_obj.curve, metrics_utils.AUCCurve.PR)
+    self.assertEqual(auc_obj.summation_method,
+                     metrics_utils.AUCSummationMethod.MAJORING)
+
+    # Check save and restore config
+    auc_obj2 = metrics.AUC.from_config(auc_obj.get_config())
+    self.assertEqual(auc_obj2.name, 'auc_1')
+    self.assertEqual(len(auc_obj2.variables), 4)
+    self.assertEqual(auc_obj2.num_thresholds, 100)
+    self.assertEqual(auc_obj2.curve, metrics_utils.AUCCurve.PR)
+    self.assertEqual(auc_obj2.summation_method,
+                     metrics_utils.AUCSummationMethod.MAJORING)
+
+  def test_value_is_idempotent(self):
+    self.setup()
+    auc_obj = metrics.AUC(num_thresholds=3)
+    self.evaluate(variables.variables_initializer(auc_obj.variables))
+
+    # Run several updates.
+    update_op = auc_obj.update_state(self.y_true, self.y_pred)
+    for _ in range(10):
+      self.evaluate(update_op)
+
+    # Then verify idempotency.
+    initial_auc = self.evaluate(auc_obj.result())
+    for _ in range(10):
+      self.assertAllClose(initial_auc, self.evaluate(auc_obj.result()), 1e-3)
+
+  def test_unweighted_all_correct(self):
+    self.setup()
+    auc_obj = metrics.AUC()
+    self.evaluate(variables.variables_initializer(auc_obj.variables))
+    result = auc_obj(self.y_true, self.y_true)
+    self.assertEqual(self.evaluate(result), 1)
+
+  def test_unweighted(self):
+    self.setup()
+    auc_obj = metrics.AUC(num_thresholds=self.num_thresholds)
+    self.evaluate(variables.variables_initializer(auc_obj.variables))
+    result = auc_obj(self.y_true, self.y_pred)
+
+    # tp = [2, 1, 0], fp = [2, 0, 0], fn = [0, 1, 2], tn = [0, 2, 2]
+    # recall = [2/2, 1/(1+1), 0] = [1, 0.5, 0]
+    # fp_rate = [2/2, 0, 0] = [1, 0, 0]
+    # heights = [(1 + 0.5)/2, (0.5 + 0)/2] = [0.75, 0.25]
+    # widths = [(1 - 0), (0 - 0)] = [1, 0]
+    expected_result = (0.75 * 1 + 0.25 * 0)
+    self.assertAllClose(self.evaluate(result), expected_result, 1e-3)
+
+  def test_weighted_roc_interpolation(self):
+    self.setup()
+    auc_obj = metrics.AUC(num_thresholds=self.num_thresholds)
+    self.evaluate(variables.variables_initializer(auc_obj.variables))
+    result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)
+
+    # tp = [7, 4, 0], fp = [3, 0, 0], fn = [0, 3, 7], tn = [0, 3, 3]
+    # recall = [7/7, 4/(4+3), 0] = [1, 0.571, 0]
+    # fp_rate = [3/3, 0, 0] = [1, 0, 0]
+    # heights = [(1 + 0.571)/2, (0.571 + 0)/2] = [0.7855, 0.2855]
+    # widths = [(1 - 0), (0 - 0)] = [1, 0]
+    expected_result = (0.7855 * 1 + 0.2855 * 0)
+    self.assertAllClose(self.evaluate(result), expected_result, 1e-3)
+
+  def test_weighted_roc_majoring(self):
+    self.setup()
+    auc_obj = metrics.AUC(
+        num_thresholds=self.num_thresholds,
+        summation_method=metrics_utils.AUCSummationMethod.MAJORING)
+    self.evaluate(variables.variables_initializer(auc_obj.variables))
+    result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)
+
+    # tp = [7, 4, 0], fp = [3, 0, 0], fn = [0, 3, 7], tn = [0, 3, 3]
+    # recall = [7/7, 4/(4+3), 0] = [1, 0.571, 0]
+    # fp_rate = [3/3, 0, 0] = [1, 0, 0]
+    # heights = [max(1, 0.571), max(0.571, 0)] = [1, 0.571]
+    # widths = [(1 - 0), (0 - 0)] = [1, 0]
+    expected_result = (1 * 1 + 0.571 * 0)
+    self.assertAllClose(self.evaluate(result), expected_result, 1e-3)
+
+  def test_weighted_roc_minoring(self):
+    self.setup()
+    auc_obj = metrics.AUC(
+        num_thresholds=self.num_thresholds,
+        summation_method=metrics_utils.AUCSummationMethod.MINORING)
+    self.evaluate(variables.variables_initializer(auc_obj.variables))
+    result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)
+
+    # tp = [7, 4, 0], fp = [3, 0, 0], fn = [0, 3, 7], tn = [0, 3, 3]
+    # recall = [7/7, 4/(4+3), 0] = [1, 0.571, 0]
+    # fp_rate = [3/3, 0, 0] = [1, 0, 0]
+    # heights = [min(1, 0.571), min(0.571, 0)] = [0.571, 0]
+    # widths = [(1 - 0), (0 - 0)] = [1, 0]
+    expected_result = (0.571 * 1 + 0 * 0)
+    self.assertAllClose(self.evaluate(result), expected_result, 1e-3)
+
+  def test_weighted_pr_majoring(self):
+    self.setup()
+    auc_obj = metrics.AUC(
+        num_thresholds=self.num_thresholds,
+        curve=metrics_utils.AUCCurve.PR,
+        summation_method=metrics_utils.AUCSummationMethod.MAJORING)
+    self.evaluate(variables.variables_initializer(auc_obj.variables))
+    result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)
+
+    # tp = [7, 4, 0], fp = [3, 0, 0], fn = [0, 3, 7], tn = [0, 3, 3]
+    # precision = [7/(7+3), 4/4, 0] = [0.7, 1, 0]
+    # recall = [7/7, 4/(4+3), 0] = [1, 0.571, 0]
+    # heights = [max(0.7, 1), max(1, 0)] = [1, 1]
+    # widths = [(1 - 0.571), (0.571 - 0)] = [0.429, 0.571]
+    expected_result = (1 * 0.429 + 1 * 0.571)
+    self.assertAllClose(self.evaluate(result), expected_result, 1e-3)
+
+  def test_weighted_pr_minoring(self):
+    self.setup()
+    auc_obj = metrics.AUC(
+        num_thresholds=self.num_thresholds,
+        curve=metrics_utils.AUCCurve.PR,
+        summation_method=metrics_utils.AUCSummationMethod.MINORING)
+    self.evaluate(variables.variables_initializer(auc_obj.variables))
+    result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)
+
+    # tp = [7, 4, 0], fp = [3, 0, 0], fn = [0, 3, 7], tn = [0, 3, 3]
+    # precision = [7/(7+3), 4/4, 0] = [0.7, 1, 0]
+    # recall = [7/7, 4/(4+3), 0] = [1, 0.571, 0]
+    # heights = [min(0.7, 1), min(1, 0)] = [0.7, 0]
+    # widths = [(1 - 0.571), (0.571 - 0)] = [0.429, 0.571]
+    expected_result = (0.7 * 0.429 + 0 * 0.571)
+    self.assertAllClose(self.evaluate(result), expected_result, 1e-3)
+
+  def test_weighted_pr_interpolation(self):
+    self.setup()
+    auc_obj = metrics.AUC(
+        num_thresholds=self.num_thresholds,
+        curve=metrics_utils.AUCCurve.PR,
+        summation_method=metrics_utils.AUCSummationMethod.INTERPOLATION)
+    self.evaluate(variables.variables_initializer(auc_obj.variables))
+    result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)
+
+    # auc = (slope / Total Pos) * [dTP - intercept * log(Pb/Pa)]
+
+    # tp = [7, 4, 0], fp = [3, 0, 0], fn = [0, 3, 7], tn = [0, 3, 3]
+    # P = tp + fp = [10, 4, 0]
+    # dTP = [7-4, 4-0] = [3, 4]
+    # dP = [10-4, 4-0] = [6, 4]
+    # slope = dTP/dP = [0.5, 1]
+    # intercept = (TPa+(slope*Pa) = [(4 - 0.5*4), (0 - 1*0)] = [2, 0]
+    # (Pb/Pa) = (Pb/Pa) if Pb > 0 AND Pa > 0 else 1 = [10/4, 4/0] = [2.5, 1]
+    # auc * TotalPos = [(0.5 * (3 + 2 * log(2.5))), (1 * (4 + 0))]
+    #                = [2.416, 4]
+    # auc = [2.416, 4]/(tp[1:]+fn[1:])
+    expected_result = (2.416/7 + 4/7)
+    self.assertAllClose(self.evaluate(result), expected_result, 1e-3)
+
+  def test_invalid_num_thresholds(self):
+    with self.assertRaisesRegexp(ValueError, '`num_thresholds` must be > 1.'):
+      metrics.AUC(num_thresholds=-1)
+
+    with self.assertRaisesRegexp(ValueError, '`num_thresholds` must be > 1.'):
+      metrics.AUC(num_thresholds=1)
+
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index 6cf61d33d0..d28a615c02 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -1031,11 +1031,11 @@ class ResetStatesTest(keras_parameterized.TestCase):
     x = np.concatenate((np.ones((50, 4)), np.ones((50, 4))))
     y = np.concatenate((np.ones((50, 1)), np.zeros((50, 1))))
     model.evaluate(x, y)
-    self.assertEqual(self.evaluate(p_obj.tp), 50.)
-    self.assertEqual(self.evaluate(p_obj.fp), 50.)
+    self.assertEqual(self.evaluate(p_obj.true_positives), 50.)
+    self.assertEqual(self.evaluate(p_obj.false_positives), 50.)
     model.evaluate(x, y)
-    self.assertEqual(self.evaluate(p_obj.tp), 50.)
-    self.assertEqual(self.evaluate(p_obj.fp), 50.)
+    self.assertEqual(self.evaluate(p_obj.true_positives), 50.)
+    self.assertEqual(self.evaluate(p_obj.false_positives), 50.)
 
   def test_reset_states_recall(self):
     r_obj = metrics.Recall()
@@ -1043,11 +1043,11 @@ class ResetStatesTest(keras_parameterized.TestCase):
     x = np.concatenate((np.ones((50, 4)), np.zeros((50, 4))))
     y = np.concatenate((np.ones((50, 1)), np.ones((50, 1))))
     model.evaluate(x, y)
-    self.assertEqual(self.evaluate(r_obj.tp), 50.)
-    self.assertEqual(self.evaluate(r_obj.fn), 50.)
+    self.assertEqual(self.evaluate(r_obj.true_positives), 50.)
+    self.assertEqual(self.evaluate(r_obj.false_negatives), 50.)
     model.evaluate(x, y)
-    self.assertEqual(self.evaluate(r_obj.tp), 50.)
-    self.assertEqual(self.evaluate(r_obj.fn), 50.)
+    self.assertEqual(self.evaluate(r_obj.true_positives), 50.)
+    self.assertEqual(self.evaluate(r_obj.false_negatives), 50.)
 
   def test_reset_states_sensitivity_at_specificity(self):
     s_obj = metrics.SensitivityAtSpecificity(0.5, num_thresholds=1)
@@ -1056,16 +1056,13 @@ class ResetStatesTest(keras_parameterized.TestCase):
                         np.ones((25, 4))))
     y = np.concatenate((np.ones((25, 1)), np.zeros((25, 1)), np.ones((25, 1)),
                         np.zeros((25, 1))))
-    model.evaluate(x, y)
-    self.assertEqual(self.evaluate(s_obj.tp), 25.)
-    self.assertEqual(self.evaluate(s_obj.fp), 25.)
-    self.assertEqual(self.evaluate(s_obj.fn), 25.)
-    self.assertEqual(self.evaluate(s_obj.tn), 25.)
-    model.evaluate(x, y)
-    self.assertEqual(self.evaluate(s_obj.tp), 25.)
-    self.assertEqual(self.evaluate(s_obj.fp), 25.)
-    self.assertEqual(self.evaluate(s_obj.fn), 25.)
-    self.assertEqual(self.evaluate(s_obj.tn), 25.)
+
+    for _ in range(2):
+      model.evaluate(x, y)
+      self.assertEqual(self.evaluate(s_obj.true_positives), 25.)
+      self.assertEqual(self.evaluate(s_obj.false_positives), 25.)
+      self.assertEqual(self.evaluate(s_obj.false_negatives), 25.)
+      self.assertEqual(self.evaluate(s_obj.true_negatives), 25.)
 
   def test_reset_states_specificity_at_sensitivity(self):
     s_obj = metrics.SpecificityAtSensitivity(0.5, num_thresholds=1)
@@ -1074,16 +1071,28 @@ class ResetStatesTest(keras_parameterized.TestCase):
                         np.ones((25, 4))))
     y = np.concatenate((np.ones((25, 1)), np.zeros((25, 1)), np.ones((25, 1)),
                         np.zeros((25, 1))))
-    model.evaluate(x, y)
-    self.assertEqual(self.evaluate(s_obj.tp), 25.)
-    self.assertEqual(self.evaluate(s_obj.fp), 25.)
-    self.assertEqual(self.evaluate(s_obj.fn), 25.)
-    self.assertEqual(self.evaluate(s_obj.tn), 25.)
-    model.evaluate(x, y)
-    self.assertEqual(self.evaluate(s_obj.tp), 25.)
-    self.assertEqual(self.evaluate(s_obj.fp), 25.)
-    self.assertEqual(self.evaluate(s_obj.fn), 25.)
-    self.assertEqual(self.evaluate(s_obj.tn), 25.)
+
+    for _ in range(2):
+      model.evaluate(x, y)
+      self.assertEqual(self.evaluate(s_obj.true_positives), 25.)
+      self.assertEqual(self.evaluate(s_obj.false_positives), 25.)
+      self.assertEqual(self.evaluate(s_obj.false_negatives), 25.)
+      self.assertEqual(self.evaluate(s_obj.true_negatives), 25.)
+
+  def test_reset_states_auc(self):
+    auc_obj = metrics.AUC(num_thresholds=3)
+    model = _get_model([auc_obj])
+    x = np.concatenate((np.ones((25, 4)), np.zeros((25, 4)), np.zeros((25, 4)),
+                        np.ones((25, 4))))
+    y = np.concatenate((np.ones((25, 1)), np.zeros((25, 1)), np.ones((25, 1)),
+                        np.zeros((25, 1))))
+
+    for _ in range(2):
+      model.evaluate(x, y)
+      self.assertEqual(self.evaluate(auc_obj.true_positives[1]), 25.)
+      self.assertEqual(self.evaluate(auc_obj.false_positives[1]), 25.)
+      self.assertEqual(self.evaluate(auc_obj.false_negatives[1]), 25.)
+      self.assertEqual(self.evaluate(auc_obj.true_negatives[1]), 25.)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/keras/utils/metrics_utils.py b/tensorflow/python/keras/utils/metrics_utils.py
index 7db128e8ab..108d6ee642 100644
--- a/tensorflow/python/keras/utils/metrics_utils.py
+++ b/tensorflow/python/keras/utils/metrics_utils.py
@@ -146,6 +146,17 @@ class ConfusionMatrix(Enum):
   FALSE_NEGATIVES = 'fn'
 
 
+class AUCCurve(Enum):
+  ROC = 'ROC'
+  PR = 'PR'
+
+
+class AUCSummationMethod(Enum):
+  INTERPOLATION = 'interpolation'
+  MAJORING = 'majoring'
+  MINORING = 'minoring'
+
+
 def update_confusion_matrix_variables(variables_to_update,
                                       y_true,
                                       y_pred,
-- 
GitLab


From bf87f8347ce24735d9d22173e289d7860a40498f Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Fri, 11 Jan 2019 19:03:45 -0800
Subject: [PATCH 0616/2345] Increase the tolerance value for
 quantization_mnist_test.

PiperOrigin-RevId: 228980452
---
 tensorflow/contrib/tensorrt/test/quantization_mnist_test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py b/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
index 79dde2a6b5..966476dc3e 100644
--- a/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
+++ b/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
@@ -274,7 +274,7 @@ class QuantizationAwareTrainingMNISTTest(test_util.TensorFlowTestCase):
         num_epochs=None,
         model_dir=model_dir)['accuracy']
     logging.info('accuracy_tf_native: %f', accuracy_tf_native)
-    self.assertAllClose(accuracy_tf_native, 0.9662)
+    self.assertAllClose(0.9662, accuracy_tf_native, rtol=1e-3, atol=1e-3)
 
     if trt_convert.get_linked_tensorrt_version()[0] < 5:
       return
@@ -286,7 +286,7 @@ class QuantizationAwareTrainingMNISTTest(test_util.TensorFlowTestCase):
         num_epochs=None,
         model_dir=model_dir)['accuracy']
     logging.info('accuracy_tf_trt: %f', accuracy_tf_trt)
-    self.assertAllClose(accuracy_tf_trt, 0.9677)
+    self.assertAllClose(0.9675, accuracy_tf_trt, rtol=1e-3, atol=1e-3)
 
 
 if __name__ == '__main__':
-- 
GitLab


From 0a958b9d697df62df0206e5e76665ee72e95aed9 Mon Sep 17 00:00:00 2001
From: Nick Felt <nickfelt@google.com>
Date: Fri, 11 Jan 2019 19:11:17 -0800
Subject: [PATCH 0617/2345] fix logic adding generated API directory to
 tensorflow.__path__

---
 tensorflow/api_template_v1.__init__.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tensorflow/api_template_v1.__init__.py b/tensorflow/api_template_v1.__init__.py
index b293d0f153..eeca8f0d56 100644
--- a/tensorflow/api_template_v1.__init__.py
+++ b/tensorflow/api_template_v1.__init__.py
@@ -62,12 +62,15 @@ if '__all__' in vars():
   vars()['__all__'].append('contrib')
 
 from tensorflow.python.platform import flags  # pylint: disable=g-import-not-at-top
-from tensorflow.python.platform import app  # pylint: disable=g-import-not-at-top
-app.flags = flags
+# The 'app' module will be imported as part of the placeholder section above.
+app.flags = flags  # pylint: disable=undefined-variable
+
+# Also use 'app' module (choice is arbitrary) to derive the API directory below.
+_API_MODULE = app  # pylint: disable=undefined-variable
 
 # Make sure directory containing top level submodules is in
 # the __path__ so that "from tensorflow.foo import bar" works.
-_tf_api_dir = _os.path.dirname(_os.path.dirname(app.__file__))  # pylint: disable=undefined-variable
+_tf_api_dir = _os.path.dirname(_os.path.dirname(_API_MODULE.__file__))  # pylint: disable=undefined-variable
 if not hasattr(_current_module, '__path__'):
   __path__ = [_tf_api_dir]
 elif _tf_api_dir not in __path__:
-- 
GitLab


From a423ca472579cfcbf79c8044f6791a1c744df94f Mon Sep 17 00:00:00 2001
From: Bairen Yi <byronyi@clustar.ai>
Date: Sat, 12 Jan 2019 05:03:58 +0000
Subject: [PATCH 0618/2345] Fix CHECK error as rxe does not have numa_node

Signed-off-by: Bairen Yi <byronyi@clustar.ai>
---
 tensorflow/contrib/gdr/gdr_memory_manager.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/gdr/gdr_memory_manager.cc b/tensorflow/contrib/gdr/gdr_memory_manager.cc
index ce18751515..7321e97319 100644
--- a/tensorflow/contrib/gdr/gdr_memory_manager.cc
+++ b/tensorflow/contrib/gdr/gdr_memory_manager.cc
@@ -73,7 +73,10 @@ int TryToReadNumaNode(ibv_device* device) {
 
   std::ifstream ifs(filename.c_str());
   string content;
-  CHECK(std::getline(ifs, content));
+  const auto& ret = std::getline(ifs, content);
+  if (!ret) {
+    return port::kNUMANoAffinity;
+  }
 
   int32 value;
   if (strings::safe_strto32(content, &value)) {
-- 
GitLab


From 66e36030a8fce773c883ae1857fd067564138b27 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 21:10:22 -0800
Subject: [PATCH 0619/2345] Switch callers away from deprecated methods of
 tf.distribute.Strategy.

PiperOrigin-RevId: 228987556
---
 .../python/checkpoint_utils_test.py           |  2 +-
 .../collective_all_reduce_strategy_test.py    |  8 ++--
 .../python/keras_optimizer_v2_test.py         |  3 +-
 .../distribute/python/metrics_v1_test.py      |  6 +--
 .../distribute/python/minimize_loss_test.py   | 28 ++++++-----
 .../distribute/python/moving_averages_test.py |  6 +--
 .../distribute/python/optimizer_v2_test.py    |  7 +--
 .../python/parameter_server_strategy_test.py  | 14 +++---
 .../contrib/distribute/python/step_fn.py      |  4 +-
 .../distribute/python/strategy_test_lib.py    | 48 ++++++++++---------
 tensorflow/python/distribute/values.py        | 25 ++++++----
 .../engine/distributed_training_utils.py      |  2 +-
 .../python/keras/optimizer_v2/optimizer_v2.py |  4 +-
 tensorflow/python/training/moving_averages.py |  4 +-
 14 files changed, 88 insertions(+), 73 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/checkpoint_utils_test.py b/tensorflow/contrib/distribute/python/checkpoint_utils_test.py
index 31bd0e996a..3ef8b9574a 100644
--- a/tensorflow/contrib/distribute/python/checkpoint_utils_test.py
+++ b/tensorflow/contrib/distribute/python/checkpoint_utils_test.py
@@ -71,7 +71,7 @@ class CheckpointUtilsWithDistributionStrategyTest(
 
     with ops.Graph().as_default() as g, distribution.scope():
       if in_replica_mode:
-        distribution.call_for_each_replica(init_and_verify, args=[g])
+        distribution.extended.call_for_each_replica(init_and_verify, args=[g])
       else:
         init_and_verify(g)
 
diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
index 62ff4b178e..9b6236fd9f 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
@@ -123,7 +123,7 @@ class CollectiveAllReduceStrategyTestBase(
       def step():
         """Perform one optimization step."""
         # Run forward & backward to get gradients, variables list.
-        g_v = d.call_for_each_replica(grad_fn, args=[one])
+        g_v = d.extended.call_for_each_replica(grad_fn, args=[one])
         # Update the variables using the gradients and the update() function.
         before_list = []
         after_list = []
@@ -135,7 +135,7 @@ class CollectiveAllReduceStrategyTestBase(
             g = d.extended.reduce_to(
                 reduce_util.ReduceOp.SUM, g, destinations=v)
             with ops.control_dependencies(
-                d.update(v, update, g, grouped=False)):
+                d.extended.update(v, update, args=(g,), group=False)):
               after_list.append(d.extended.read_var(v))
         return before_list, after_list
 
@@ -203,7 +203,7 @@ class CollectiveAllReduceStrategyTestBase(
          self.cached_session(config=config,
                              target=master_target) as sess:
       with d.scope():
-        train_op = d.call_for_each_replica(model_fn)
+        train_op = d.extended.call_for_each_replica(model_fn)
         train_op = d.group(d.unwrap(train_op))
 
       sess.run(variables.global_variables_initializer())
@@ -226,7 +226,7 @@ class CollectiveAllReduceStrategyTestBase(
                 1.0, 10.0, dtype=dtypes.float32))
         return array_ops.identity(x)
 
-      x = distribution.call_for_each_replica(model_fn)
+      x = distribution.extended.call_for_each_replica(model_fn)
       reduced_x = distribution.reduce(reduce_util.ReduceOp.MEAN, x)
       x = distribution.unwrap(x)[0]
 
diff --git a/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py b/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
index cce93b3c10..5349794334 100644
--- a/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
+++ b/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
@@ -65,7 +65,8 @@ class MirroredStrategyOptimizerV2Test(test.TestCase, parameterized.TestCase):
 
     devices = ['/device:GPU:0', '/device:CPU:0']
     with distribution.scope():
-      (var, m, v, op, counter) = distribution.call_for_each_replica(create_fn)
+      (var, m, v, op,
+       counter) = distribution.extended.call_for_each_replica(create_fn)
       self.evaluate(variables.global_variables_initializer())
       var_val = [2.0, 2.0, 2.0]
       self.assertAllClose(
diff --git a/tensorflow/contrib/distribute/python/metrics_v1_test.py b/tensorflow/contrib/distribute/python/metrics_v1_test.py
index 7472f6dde4..96006276f6 100644
--- a/tensorflow/contrib/distribute/python/metrics_v1_test.py
+++ b/tensorflow/contrib/distribute/python/metrics_v1_test.py
@@ -99,12 +99,12 @@ class MetricsV1Test(test.TestCase, parameterized.TestCase):
           dataset_fn).make_initializable_iterator()
       if isinstance(distribution, tpu_strategy.TPUStrategy):
         def step_fn(ctx, inputs):
-          value, update = distribution.call_for_each_replica(
+          value, update = distribution.extended.call_for_each_replica(
               metric_fn, args=(inputs,))
           ctx.set_non_tensor_output(name="value", output=value)
           return distribution.group(update)
 
-        ctx = distribution.run_steps_on_dataset(
+        ctx = distribution.extended.experimental_run_steps_on_iterator(
             step_fn, iterator, iterations=distribution.extended.steps_per_run)
         update = ctx.run_op
         value = ctx.non_tensor_outputs["value"]
@@ -114,7 +114,7 @@ class MetricsV1Test(test.TestCase, parameterized.TestCase):
             distribution.num_replicas_in_sync *
             distribution.extended.steps_per_run)
       else:
-        value, update = distribution.call_for_each_replica(
+        value, update = distribution.extended.call_for_each_replica(
             metric_fn, args=(iterator.get_next(),))
         update = distribution.group(update)
         # TODO(josh11b): Once we switch to using a global batch size for input,
diff --git a/tensorflow/contrib/distribute/python/minimize_loss_test.py b/tensorflow/contrib/distribute/python/minimize_loss_test.py
index b0e24a53f6..37e6b418a0 100644
--- a/tensorflow/contrib/distribute/python/minimize_loss_test.py
+++ b/tensorflow/contrib/distribute/python/minimize_loss_test.py
@@ -67,12 +67,13 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
       def step_fn(ctx, inputs):
         del ctx  # Unused
         return distribution.group(
-            distribution.call_for_each_replica(model_fn, args=(inputs,)))
+            distribution.extended.call_for_each_replica(
+                model_fn, args=(inputs,)))
 
       iterator = self._get_iterator(distribution.distribute_dataset(dataset_fn))
 
       def run_step():
-        return distribution.run_steps_on_dataset(
+        return distribution.extended.experimental_run_steps_on_iterator(
             step_fn, iterator, iterations=2).run_op
 
       if not context.executing_eagerly():
@@ -105,7 +106,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
 
       def run_step():
         return distribution.group(
-            distribution.call_for_each_replica(
+            distribution.extended.call_for_each_replica(
                 model_fn, args=(iterator.get_next(),)))
 
       if not context.executing_eagerly():
@@ -157,12 +158,13 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
       def step_fn(ctx, inputs):
         del ctx  # Unused
         return distribution.group(
-            distribution.call_for_each_replica(model_fn, args=(inputs,)))
+            distribution.extended.call_for_each_replica(
+                model_fn, args=(inputs,)))
 
       iterator = self._get_iterator(distribution.distribute_dataset(dataset_fn))
 
       def run_step():
-        return distribution.run_steps_on_dataset(
+        return distribution.extended.experimental_run_steps_on_iterator(
             step_fn, iterator, iterations=1).run_op
 
       if not context.executing_eagerly():
@@ -189,7 +191,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
 
       self.assertEqual(
           get_expected_variables(optimizer_fn,
-                                 len(distribution.parameter_devices)),
+                                 len(distribution.extended.parameter_devices)),
           set(created_variables))
 
   @combinations.generate(
@@ -222,7 +224,8 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
       def step_fn(ctx, inputs):
         del ctx  # Unused
         fetches = distribution.unwrap(
-            distribution.call_for_each_replica(model_fn, args=(inputs,)))
+            distribution.extended.call_for_each_replica(
+                model_fn, args=(inputs,)))
         if update_ops_in_cross_replica_mode:
           fetches += tuple(ops.get_collection(ops.GraphKeys.UPDATE_OPS))
         return control_flow_ops.group(fetches)
@@ -230,7 +233,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
       iterator = self._get_iterator(distribution.distribute_dataset(dataset_fn))
 
       def run_step():
-        return distribution.run_steps_on_dataset(
+        return distribution.extended.experimental_run_steps_on_iterator(
             step_fn, iterator, iterations=1).run_op
 
       if not context.executing_eagerly():
@@ -316,12 +319,13 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
       def step_fn(ctx, inputs):
         del ctx  # Unused
         return distribution.group(
-            distribution.call_for_each_replica(model_fn, args=(inputs,)))
+            distribution.extended.call_for_each_replica(
+                model_fn, args=(inputs,)))
 
       iterator = self._get_iterator(distribution.distribute_dataset(dataset_fn))
 
       def run_step():
-        return distribution.run_steps_on_dataset(
+        return distribution.extended.experimental_run_steps_on_iterator(
             step_fn, iterator, iterations=1).run_op
 
       if not context.executing_eagerly():
@@ -398,7 +402,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
         return (train_op, loss)
 
       def step_fn(output_context, inputs):
-        (train_op, loss) = distribution.call_for_each_replica(
+        (train_op, loss) = distribution.extended.call_for_each_replica(
             model_fn, args=(output_context, inputs))
         output_context.set_last_step_output(
             name="cross_replica_loss_reduced",
@@ -425,7 +429,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
             "cross_replica_loss_not_reduced":
             distribution.unwrap(distribution.broadcast(initial_loss()))
         }
-        ctx = distribution.run_steps_on_dataset(
+        ctx = distribution.extended.experimental_run_steps_on_iterator(
             step_fn, iterator, iterations=2,
             initial_loop_values=initial_loop_values)
 
diff --git a/tensorflow/contrib/distribute/python/moving_averages_test.py b/tensorflow/contrib/distribute/python/moving_averages_test.py
index 8f13e9153e..c4622cdd2a 100644
--- a/tensorflow/contrib/distribute/python/moving_averages_test.py
+++ b/tensorflow/contrib/distribute/python/moving_averages_test.py
@@ -53,7 +53,7 @@ class AssignMovingAveragesTest(test.TestCase, parameterized.TestCase):
       return var, assign
 
     with distribution.scope(), self.cached_session() as sess:
-      var, assign = distribution.call_for_each_replica(replica_fn)
+      var, assign = distribution.extended.call_for_each_replica(replica_fn)
       variables.global_variables_initializer().run()
       self.assertAllClose([10.0, 11.0], var.eval())
       sess.run(distribution.unwrap(assign))
@@ -79,7 +79,7 @@ class AssignMovingAveragesTest(test.TestCase, parameterized.TestCase):
       return var, assign.op
 
     with distribution.scope(), self.cached_session() as sess:
-      var, assign_op = distribution.call_for_each_replica(replica_fn)
+      var, assign_op = distribution.extended.call_for_each_replica(replica_fn)
       variables.global_variables_initializer().run()
       self.assertAllClose([0.0, 0.0], var.eval())
       sess.run(distribution.unwrap(assign_op))
@@ -152,7 +152,7 @@ class AssignMovingAveragesTest(test.TestCase, parameterized.TestCase):
       return var, assign
 
     with distribution.scope(), self.cached_session() as sess:
-      var, assign = distribution.call_for_each_replica(replica_fn)
+      var, assign = distribution.extended.call_for_each_replica(replica_fn)
       variables.global_variables_initializer().run()
       self.assertAllClose([10.0, 11.0], var.eval())
       sess.run(distribution.unwrap(assign))
diff --git a/tensorflow/contrib/distribute/python/optimizer_v2_test.py b/tensorflow/contrib/distribute/python/optimizer_v2_test.py
index fa4705af7c..a832ef61f9 100644
--- a/tensorflow/contrib/distribute/python/optimizer_v2_test.py
+++ b/tensorflow/contrib/distribute/python/optimizer_v2_test.py
@@ -49,9 +49,10 @@ class MinimizeLossOptimizerV2Test(test.TestCase, parameterized.TestCase):
         iterator = ds.make_initializable_iterator()
 
       def run_step():
-        return control_flow_ops.group(distribution.unwrap(
-            distribution.call_for_each_replica(
-                model_fn, args=(iterator.get_next(),))))
+        return control_flow_ops.group(
+            distribution.unwrap(
+                distribution.extended.call_for_each_replica(
+                    model_fn, args=(iterator.get_next(),))))
 
       if not context.executing_eagerly():
         with self.cached_session() as sess:
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
index 2d4f8a9be5..c7760a8431 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
@@ -236,7 +236,7 @@ class ParameterServerStrategyTestBase(
         self.assertIn('/job:ps/', h.device)
         return y_add, z_add, f
 
-      y, z, f = d.call_for_each_replica(model_fn)
+      y, z, f = d.extended.call_for_each_replica(model_fn)
       self.assertNotEqual(y, None)
       self.assertNotEqual(z, None)
       self.assertNotEqual(f, None)
@@ -252,7 +252,7 @@ class ParameterServerStrategyTestBase(
       self, task_type, task_id, num_gpus, use_core_strategy=False):
     d, _, sess_config = self._get_test_objects(
         task_type, task_id, num_gpus, use_core_strategy=use_core_strategy)
-    num_shards = len(d.parameter_devices)
+    num_shards = len(d.extended.parameter_devices)
     partitioner = partitioned_variables.fixed_size_partitioner(num_shards)
     with ops.Graph().as_default(), \
          self.cached_session(target=self._default_target,
@@ -286,7 +286,7 @@ class ParameterServerStrategyTestBase(
 
         return x_add
 
-      x = d.call_for_each_replica(model_fn)
+      x = d.extended.call_for_each_replica(model_fn)
 
       if context.num_gpus() >= 1:
         variables.global_variables_initializer().run()
@@ -387,7 +387,7 @@ class ParameterServerStrategyTestBase(
             device_util.canonicalize(h.device))
         return y_add, z_add, f
 
-      y, z, f = d.call_for_each_replica(model_fn)
+      y, z, f = d.extended.call_for_each_replica(model_fn)
       self.assertNotEqual(y, None)
       self.assertNotEqual(z, None)
       self.assertNotEqual(f, None)
@@ -438,7 +438,7 @@ class ParameterServerStrategyTestBase(
         train_op = control_flow_ops.group(x_add, y_add, z_add)
         return x, y, z, train_op
 
-      x, y, z, train_op = d.call_for_each_replica(model_fn)
+      x, y, z, train_op = d.extended.call_for_each_replica(model_fn)
       train_op = d.group(train_op)
 
       if context.num_gpus() < d.extended._num_gpus_per_worker:
@@ -519,7 +519,7 @@ class ParameterServerStrategyTestBase(
       def step():
         """Perform one optimization step."""
         # Run forward & backward to get gradients, variables list.
-        g_v = d.call_for_each_replica(grad_fn, args=(one,))
+        g_v = d.extended.call_for_each_replica(grad_fn, args=(one,))
         # Update the variables using the gradients and the update() function.
         before_list = []
         after_list = []
@@ -531,7 +531,7 @@ class ParameterServerStrategyTestBase(
             g = d.extended.reduce_to(
                 reduce_util.ReduceOp.SUM, g, destinations=v)
             with ops.control_dependencies(
-                d.update(v, update, g, grouped=False)):
+                d.extended.update(v, update, args=(g,), group=False)):
               after_list.append(d.extended.read_var(v))
         return before_list, after_list
 
diff --git a/tensorflow/contrib/distribute/python/step_fn.py b/tensorflow/contrib/distribute/python/step_fn.py
index faeb96bcb7..24b95f748b 100644
--- a/tensorflow/contrib/distribute/python/step_fn.py
+++ b/tensorflow/contrib/distribute/python/step_fn.py
@@ -99,7 +99,7 @@ class StandardSingleLossStep(StandardInputStep):
         gradients_fn = backprop.implicit_grad(self._loss_fn)
         gradients_fn = optimizer_lib.get_filtered_grad_fn(gradients_fn)
 
-        grads_and_vars = self.distribution.call_for_each_replica(
+        grads_and_vars = self.distribution.extended.call_for_each_replica(
             gradients_fn, args=(ctx, inputs))
         # If threads use layers, then we need to run the first step
         # sequentially, so that layers.build() is not executed in parallel.
@@ -109,6 +109,6 @@ class StandardSingleLossStep(StandardInputStep):
             self.distribution, grads_and_vars)
 
       # TODO(priyag): Return the outputs, context, etc as well.
-      ctx = self.distribution.run_steps_on_dataset(
+      ctx = self.distribution.extended.experimental_run_steps_on_iterator(
           step_fn, self._iterator, self._iterations_per_step)
       return ctx.run_op
diff --git a/tensorflow/contrib/distribute/python/strategy_test_lib.py b/tensorflow/contrib/distribute/python/strategy_test_lib.py
index 7455cbd02a..2e2ee92b6e 100644
--- a/tensorflow/contrib/distribute/python/strategy_test_lib.py
+++ b/tensorflow/contrib/distribute/python/strategy_test_lib.py
@@ -45,25 +45,26 @@ class _TestException(Exception):
   pass
 
 
-# May be the argument to either distribution.call_for_each_replica() or
+# May be the argument to either distribution.extended.call_for_each_replica() or
 # get_replica_context().merge_call()
 def _raise_exception_fn(_=None):
   raise _TestException()
 
 
-# Must be the argument to a distribution.call_for_each_replica() call, calls a
-# get_replica_context().merge_call() that raises an exception.
+# Must be the argument to a distribution.extended.call_for_each_replica() call,
+# calls a get_replica_context().merge_call() that raises an exception.
 def _merge_raises_fn():
   ds_context.get_replica_context().merge_call(_raise_exception_fn)
 
 
 # Must be the argument to a get_replica_context().merge_call() call, calls
-# dist.call_for_each_replica() with a function that raises an exception.
+# dist.extended.call_for_each_replica() with a function that raises an
+# exception.
 def _call_raises_fn(dist):
-  dist.call_for_each_replica(_raise_exception_fn)
+  dist.extended.call_for_each_replica(_raise_exception_fn)
 
 
-# Must be the argument to a distribution.call_for_each_replica() call,
+# Must be the argument to a distribution.extended.call_for_each_replica() call,
 # calls a get_replica_context().merge_call() that calls a
 # call_for_each_replica() that raises an exception.
 def _merge_call_raises_fn():
@@ -71,15 +72,16 @@ def _merge_call_raises_fn():
 
 
 # Must be the argument to a get_replica_context().merge_call() call, calls
-# dist.call_for_each_replica() with a function that calls a
+# dist.extended.call_for_each_replica() with a function that calls a
 # get_replica_context().merge_call() that raises an exception.
 def _call_merge_raises_fn(dist):
-  dist.call_for_each_replica(_merge_raises_fn)
+  dist.extended.call_for_each_replica(_merge_raises_fn)
 
 
-# Must be the argument to a distribution.call_for_each_replica() call, calls a
-# get_replica_context().merge_call() that calls a call_for_each_replica() that
-# calls a get_replica_context().merge_call() that raises an exception.
+# Must be the argument to a distribution.extended.call_for_each_replica() call,
+# calls a get_replica_context().merge_call() that calls a
+# call_for_each_replica() that calls a get_replica_context().merge_call() that
+# raises an exception.
 def _merge_call_merge_raises_fn():
   ds_context.get_replica_context().merge_call(_call_merge_raises_fn)
 
@@ -110,7 +112,7 @@ class DistributionTestBase(test.TestCase):
       def step():
         """Perform one optimization step."""
         # Run forward & backward to get gradients, variables list.
-        g_v = d.call_for_each_replica(grad_fn, args=(one,))
+        g_v = d.extended.call_for_each_replica(grad_fn, args=(one,))
 
         # Update the variables using the gradients and the update() function.
         before_list = []
@@ -122,8 +124,8 @@ class DistributionTestBase(test.TestCase):
           with ops.control_dependencies([fetched]):
             g = d.extended.reduce_to(
                 reduce_util.ReduceOp.SUM, g, destinations=v)
-            with ops.control_dependencies(d.update(
-                v, update, g, grouped=False)):
+            with ops.control_dependencies(d.extended.update(
+                v, update, args=(g,), group=False)):
               after_list.append(d.extended.read_var(v))
         return before_list, after_list
 
@@ -166,7 +168,7 @@ class DistributionTestBase(test.TestCase):
       def step():
         """Perform one optimization step."""
         # Run forward & backward to get gradients, variables list.
-        g_v = d.call_for_each_replica(grad_fn, args=(one,))
+        g_v = d.extended.call_for_each_replica(grad_fn, args=(one,))
 
         # Update the variables using the gradients and the update() function.
         before_list = []
@@ -177,8 +179,8 @@ class DistributionTestBase(test.TestCase):
           with ops.control_dependencies([fetched]):
             g = d.extended.reduce_to(
                 reduce_util.ReduceOp.SUM, g, destinations=v)
-            with ops.control_dependencies(d.update(
-                v, update, g, grouped=False)):
+            with ops.control_dependencies(d.extended.update(
+                v, update, args=(g,), group=False)):
               after_list.append(d.extended.read_var(v))
         return before_list, after_list
 
@@ -206,20 +208,20 @@ class DistributionTestBase(test.TestCase):
         self.assertFalse(expected_devices[replica_id])
         expected_devices[replica_id] = True
 
-      d.call_for_each_replica(mark_devices_fn)
+      d.extended.call_for_each_replica(mark_devices_fn)
       self.assertAllEqual(expected_devices,
                           [True] * len(d.extended.worker_devices))
 
   def _test_call_and_merge_exceptions(self, dist):
     with dist.scope():
       with self.assertRaises(_TestException):
-        dist.call_for_each_replica(_raise_exception_fn)
+        dist.extended.call_for_each_replica(_raise_exception_fn)
       with self.assertRaises(_TestException):
-        dist.call_for_each_replica(_merge_raises_fn)
+        dist.extended.call_for_each_replica(_merge_raises_fn)
       with self.assertRaises(_TestException):
-        dist.call_for_each_replica(_merge_call_raises_fn)
+        dist.extended.call_for_each_replica(_merge_call_raises_fn)
       with self.assertRaises(_TestException):
-        dist.call_for_each_replica(_merge_call_merge_raises_fn)
+        dist.extended.call_for_each_replica(_merge_call_merge_raises_fn)
 
   def _input_fn_to_test_input_context(self,
                                       dataset_fn,
@@ -291,7 +293,7 @@ class DistributionTestBase(test.TestCase):
         value = global_step.read_value()
         return train_op, value
 
-      train_ops, value = strategy.call_for_each_replica(model_fn)
+      train_ops, value = strategy.extended.call_for_each_replica(model_fn)
       self.evaluate(strategy.group(train_ops))
       global_step_tensors = strategy.unwrap(value)
       global_step_values = self.evaluate(global_step_tensors)
diff --git a/tensorflow/python/distribute/values.py b/tensorflow/python/distribute/values.py
index 2a57d5cfc7..ec0ea4bc2d 100644
--- a/tensorflow/python/distribute/values.py
+++ b/tensorflow/python/distribute/values.py
@@ -653,8 +653,9 @@ class MirroredVariable(DistributedVariable, Mirrored,
         return f(v, *args, **kwargs)
 
       # We are calling assign on the mirrored variable in cross replica context,
-      # use `strategy.update()` to update the variable.
-      return self._distribute_strategy.update(self, f, *args, **kwargs)
+      # use `strategy.extended.update()` to update the variable.
+      return self._distribute_strategy.extended.update(
+          self, f, args=args, kwargs=kwargs)
     else:
       _assert_replica_context(self._distribute_strategy)
       # We are calling an assign function on the mirrored variable in replica
@@ -669,7 +670,8 @@ class MirroredVariable(DistributedVariable, Mirrored,
 
       def merge_fn(strategy, value, *other_args, **other_kwargs):
         v = _apply_aggregation(strategy, value, self._aggregation, self)
-        return strategy.update(self, f, v, *other_args, **other_kwargs)
+        return strategy.extended.update(
+            self, f, args=(v,) + other_args, kwargs=other_kwargs)
 
       return distribution_strategy_context.get_replica_context().merge_call(
           merge_fn, args=args, kwargs=kwargs)
@@ -721,7 +723,7 @@ class MirroredVariable(DistributedVariable, Mirrored,
 # allowing instances of the class to be used as tensors.
 def _tensor_conversion_mirrored(var, dtype=None, name=None, as_ref=False):
   # Try to avoid assignments to and other mutations of MirroredVariable
-  # state except through a DistributionStrategy.update() call.
+  # state except through a DistributionStrategy.extended.update() call.
   assert not as_ref
   return ops.internal_convert_to_tensor(
       var.get(), dtype=dtype, name=name, as_ref=as_ref)
@@ -914,7 +916,8 @@ class TPUMirroredVariable(checkpointable.CheckpointableBase):
     f = kwargs.pop("f")
     if distribution_strategy_context.in_cross_replica_context():
       if _enclosing_tpu_context() is not None:
-        return self._distribute_strategy.update(self, f, *args, **kwargs)
+        return self._distribute_strategy.extended.update(
+            self, f, args=args, kwargs=kwargs)
 
       update_device = distribute_lib.get_update_device()
       # We are calling update on the mirrored variable in cross replica context.
@@ -924,7 +927,8 @@ class TPUMirroredVariable(checkpointable.CheckpointableBase):
         v = self._get(device=update_device)
         return f(v, *args, **kwargs)
 
-      return self._distribute_strategy.update(self, f, *args, **kwargs)
+      return self._distribute_strategy.extended.update(
+          self, f, args=args, kwargs=kwargs)
     else:
       _assert_replica_context(self._distribute_strategy)
       # We are calling an assign function on the mirrored variable in replica
@@ -939,7 +943,8 @@ class TPUMirroredVariable(checkpointable.CheckpointableBase):
 
       def merge_fn(strategy, value, *other_args, **other_kwargs):
         v = _apply_aggregation(strategy, value, self._aggregation, self)
-        return strategy.update(self, f, v, *other_args, **other_kwargs)
+        return strategy.extended.update(
+            self, f, args=(v,) + other_args, kwargs=other_kwargs)
 
       return distribution_strategy_context.get_replica_context().merge_call(
           merge_fn, args=args, kwargs=kwargs)
@@ -1459,7 +1464,8 @@ class AggregatingVariable(checkpointable.CheckpointableBase):
 
       # We are calling an assign function in cross replica context, wrap it in
       # an update call.
-      return self._distribute_strategy.update(self, f, *args, **kwargs)
+      return self._distribute_strategy.extended.update(
+          self, f, args=args, kwargs=kwargs)
     else:
       replica_context = distribution_strategy_context.get_replica_context()
       assert replica_context
@@ -1473,7 +1479,8 @@ class AggregatingVariable(checkpointable.CheckpointableBase):
 
       def merge_fn(strategy, value, *other_args, **other_kwargs):
         v = _apply_aggregation(strategy, value, self._aggregation, self)
-        return strategy.update(self, f, v, *other_args, **other_kwargs)
+        return strategy.extended.update(
+            self, f, args=(v,) + other_args, kwargs=other_kwargs)
 
       return replica_context.merge_call(merge_fn, args=args, kwargs=kwargs)
 
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index 0185e87cb0..27ad6a5c0b 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -816,7 +816,7 @@ def _make_eager_execution_function(model, mode):
   with K.get_graph().as_default(), strategy.scope():
     # Create train ops on each of the devices when we call
     # `_per_device_fit_function`.
-    (grouped_inputs, grouped_outputs) = strategy.call_for_each_replica(
+    (grouped_inputs, grouped_outputs) = strategy.extended.call_for_each_replica(
         _per_device_function, args=(model._distributed_model,))
 
     # Unwrap all the per device values returned from `call_for_each_replica`.
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
index 91dab9338d..0415c97a0b 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
@@ -876,8 +876,8 @@ def merge_grads(grads_and_vars):
   """Merge gradients from different replicas."""
 
   def merge_grad_fn(strategy, grads_and_vars):
-    reduced_grads = strategy.batch_reduce(ds_reduce_util.ReduceOp.SUM,
-                                          grads_and_vars)
+    reduced_grads = strategy.extended.batch_reduce_to(
+        ds_reduce_util.ReduceOp.SUM, grads_and_vars)
     return reduced_grads
 
   return distribute_ctx.get_replica_context().merge_call(
diff --git a/tensorflow/python/training/moving_averages.py b/tensorflow/python/training/moving_averages.py
index 4b267dfb98..6efcab28c5 100644
--- a/tensorflow/python/training/moving_averages.py
+++ b/tensorflow/python/training/moving_averages.py
@@ -98,12 +98,12 @@ def assign_moving_average(variable, value, decay, zero_debias=True, name=None):
       def merge_fn(strategy, v, value):
         value = strategy.extended.reduce_to(
             ds_reduce_util.ReduceOp.MEAN, value, v)
-        return strategy.update(v, update_fn, value)
+        return strategy.extended.update(v, update_fn, args=(value,))
 
       return replica_context.merge_call(merge_fn, args=(variable, value))
     else:
       strategy = distribution_strategy_context.get_cross_replica_context()
-      return strategy.update(variable, update_fn, value)
+      return strategy.extended.update(variable, update_fn, args=(value,))
 
 
 def weighted_moving_average(value,
-- 
GitLab


From 7d707dcce6f7755790a23271bedc1c737b643dc7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 12 Jan 2019 01:02:32 -0800
Subject: [PATCH 0620/2345] compat: Update forward compatibility horizon to
 2019-01-12

PiperOrigin-RevId: 229000037
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 4028302362..1f4840ff38 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 11)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 12)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 3a10998daf52ea922aa7ff895af3012afe956866 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Sat, 12 Jan 2019 15:56:44 -0800
Subject: [PATCH 0621/2345] [XLA] Don't cache Executables resulting from
 Client::Execute().

Previously, Client::Execute() would cache the Executables it created forever.
This is surprising, and it causes long-running tests (the primary user of the
plain Client API) to consume a lot of resources.

Change the Client API implementation so that we don't cache these Executables
at all.

PiperOrigin-RevId: 229038268
---
 tensorflow/compiler/xla/client/client.cc    | 86 ++++++++++-----------
 tensorflow/compiler/xla/client/client.h     | 10 +++
 tensorflow/compiler/xla/rpc/grpc_service.cc | 16 ++++
 tensorflow/compiler/xla/rpc/grpc_service.h  |  7 ++
 tensorflow/compiler/xla/service/service.cc  | 51 ++++++++----
 tensorflow/compiler/xla/service/service.h   |  5 +-
 6 files changed, 114 insertions(+), 61 deletions(-)

diff --git a/tensorflow/compiler/xla/client/client.cc b/tensorflow/compiler/xla/client/client.cc
index 43127cae1e..4f020bcec2 100644
--- a/tensorflow/compiler/xla/client/client.cc
+++ b/tensorflow/compiler/xla/client/client.cc
@@ -278,53 +278,51 @@ StatusOr<std::unique_ptr<GlobalData>> Client::Execute(
     const XlaComputation& computation, absl::Span<GlobalData* const> arguments,
     const ExecutionOptions* execution_options,
     ExecutionProfile* execution_profile) {
-  if (execution_options != nullptr &&
-      execution_options->device_handles_size() > 1) {
-    std::vector<XlaComputationInstance> computation_instances = {
-        XlaComputationInstance{
-            computation,
-            std::vector<GlobalData*>(arguments.begin(), arguments.end()),
-            *execution_options, execution_profile}};
-    TF_ASSIGN_OR_RETURN(auto results, ExecuteParallel(computation_instances));
-    // The result selection is a bit hacky, but better than assuming it is
-    // device 0.
-    //
-    // TODO(b/118493728): Allow Execute to return one result per computation.
-    for (int64 i = 0; i < results.size(); i++) {
-      TF_ASSIGN_OR_RETURN(const Shape& shape, GetShape(*results[i]));
-      if (!ShapeUtil::IsEmptyTuple(shape)) {
-        VLOG(3) << "Fetching result from device " << i << ": "
-                << ShapeUtil::HumanString(shape);
-        return std::move(results[i]);
-      }
+  // Create an ExecutionOptions if necessary, or set its DeviceHandles.
+  absl::optional<ExecutionOptions> options_storage;
+  if (!execution_options || execution_options->device_handles().empty()) {
+    if (execution_options) {
+      options_storage.emplace(*execution_options);
+    } else {
+      options_storage.emplace(CreateDefaultExecutionOptions());
     }
-    TF_RET_CHECK(!results.empty());
-    VLOG(1) << "Defaulting to device 0 result";
-    return std::move(results[0]);
-  }
-
-  // The argument shapes affect how the computation is compiled.
-  std::vector<Shape> arg_shapes(arguments.size());
-  for (int i = 0; i < arguments.size(); i++) {
-    TF_ASSIGN_OR_RETURN(arg_shapes[i], GetShape(*arguments[i]));
-  }
-
-  TF_ASSIGN_OR_RETURN(auto handle,
-                      Compile(computation, arg_shapes, execution_options));
-
-  TF_ASSIGN_OR_RETURN(auto result,
-                      Execute(handle, arguments, execution_profile));
-
-  if (execution_profile != nullptr) {
-    if (VLOG_IS_ON(1)) {
-      TF_ASSIGN_OR_RETURN(
-          auto execution_stats,
-          ExecutionStatsAsString(computation, *execution_profile));
-      VLOG(1) << execution_stats;
+    execution_options = &*options_storage;
+
+    TF_ASSIGN_OR_RETURN(auto device_handles,
+                        GetDeviceHandles(/*device_count=*/1));
+    TF_RET_CHECK(!device_handles.empty());
+    *options_storage->add_device_handles() = std::move(device_handles[0]);
+  }
+
+  std::vector<XlaComputationInstance> computation_instances = {
+      XlaComputationInstance{
+          computation,
+          std::vector<GlobalData*>(arguments.begin(), arguments.end()),
+          *execution_options, execution_profile}};
+
+  // Instead of invoking Compile() and Execute(), invoke
+  // Service::ExecuteParallel() to execute our one computation.  Compile()
+  // caches the executable forever, which isn't what we want.
+  VLOG(1) << "Making ExecuteParallel request: "
+          << execution_options->DebugString();
+  TF_ASSIGN_OR_RETURN(auto results, ExecuteParallel(computation_instances));
+  VLOG(1) << "ExecuteParallel request done.";
+
+  // The result selection is a bit hacky, but better than assuming it is
+  // device 0.
+  //
+  // TODO(b/118493728): Allow Execute to return one result per computation.
+  for (int64 i = 0; i < results.size(); i++) {
+    TF_ASSIGN_OR_RETURN(const Shape& shape, GetShape(*results[i]));
+    if (!ShapeUtil::IsEmptyTuple(shape)) {
+      VLOG(3) << "Fetching result from device " << i << ": "
+              << ShapeUtil::HumanString(shape);
+      return std::move(results[i]);
     }
   }
-
-  return std::move(result);
+  TF_RET_CHECK(!results.empty());
+  VLOG(1) << "Defaulting to device 0 result";
+  return std::move(results[0]);
 }
 
 StatusOr<std::vector<std::unique_ptr<GlobalData>>> Client::ExecuteParallel(
diff --git a/tensorflow/compiler/xla/client/client.h b/tensorflow/compiler/xla/client/client.h
index d0ac4703c6..eff8713ac3 100644
--- a/tensorflow/compiler/xla/client/client.h
+++ b/tensorflow/compiler/xla/client/client.h
@@ -52,6 +52,12 @@ class Client {
   //   need to live beyond this call.)
   // * If execution_options.device_handles should be empty. If you need
   //   non-empty device handles, call 'Execute' instead.
+  //
+  // TODO(b/122731460): This call caches the resulting Executable in the Service
+  // *forever*.  If you're only going to run the computation once, you may want
+  // to call the Execute(const XlaComputation&) overload.  If you're going to
+  // run the computation more than once but you want control over when the
+  // Executable is unloaded, use the LocalClient API.
   StatusOr<ExecutionHandle> Compile(
       const XlaComputation& computation,
       absl::Span<const Shape> argument_shapes,
@@ -76,6 +82,10 @@ class Client {
   //   device is chosen by the service.
   // * If execution_profile is not nullptr then the pointed-to ExecutionProfile
   //   will be filled with profile data from the execution.
+  //
+  // TODO(b/122731460): The given computation is compiled and then thrown away
+  // immediately after it's run.  If you want control over how long the
+  // resulting Executable lives, use the LocalClient API.
   StatusOr<std::unique_ptr<GlobalData>> Execute(
       const XlaComputation& computation,
       absl::Span<GlobalData* const> arguments,
diff --git a/tensorflow/compiler/xla/rpc/grpc_service.cc b/tensorflow/compiler/xla/rpc/grpc_service.cc
index d8123a6de2..22b4218fbd 100644
--- a/tensorflow/compiler/xla/rpc/grpc_service.cc
+++ b/tensorflow/compiler/xla/rpc/grpc_service.cc
@@ -47,6 +47,14 @@ namespace xla {
   });
 }
 
+::grpc::Status GRPCService::GetDeviceHandles(::grpc::ServerContext* context,
+                                             const GetDeviceHandlesRequest* arg,
+                                             GetDeviceHandlesResponse* result) {
+  return DelegateRPC([this, arg, result]() {
+    return service_->GetDeviceHandles(arg, result);
+  });
+}
+
 ::grpc::Status GRPCService::Compile(::grpc::ServerContext* /*context*/,
                                     const CompileRequest* arg,
                                     CompileResponse* result) {
@@ -61,6 +69,14 @@ namespace xla {
       [this, arg, result]() { return service_->Execute(arg, result); });
 }
 
+::grpc::Status GRPCService::ExecuteGraphParallel(
+    ::grpc::ServerContext* /*context*/, const ExecuteGraphParallelRequest* arg,
+    ExecuteParallelResponse* result) {
+  return DelegateRPC([this, arg, result]() {
+    return service_->ExecuteGraphParallel(arg, result);
+  });
+}
+
 ::grpc::Status GRPCService::WaitForExecution(::grpc::ServerContext* context,
                                              const WaitForExecutionRequest* arg,
                                              WaitForExecutionResponse* result) {
diff --git a/tensorflow/compiler/xla/rpc/grpc_service.h b/tensorflow/compiler/xla/rpc/grpc_service.h
index 3e586b288a..b546704f73 100644
--- a/tensorflow/compiler/xla/rpc/grpc_service.h
+++ b/tensorflow/compiler/xla/rpc/grpc_service.h
@@ -39,6 +39,10 @@ class GRPCService : public grpc::XlaService::Service {
                                   const DeconstructTupleRequest* arg,
                                   DeconstructTupleResponse* result) override;
 
+  ::grpc::Status GetDeviceHandles(::grpc::ServerContext* context,
+                                  const GetDeviceHandlesRequest* arg,
+                                  GetDeviceHandlesResponse* result) override;
+
   ::grpc::Status Compile(::grpc::ServerContext* context,
                          const CompileRequest* arg,
                          CompileResponse* result) override;
@@ -46,6 +50,9 @@ class GRPCService : public grpc::XlaService::Service {
   ::grpc::Status Execute(::grpc::ServerContext* context,
                          const ExecuteRequest* arg,
                          ExecuteResponse* result) override;
+  ::grpc::Status ExecuteGraphParallel(::grpc::ServerContext* context,
+                                      const ExecuteGraphParallelRequest* arg,
+                                      ExecuteParallelResponse* result) override;
 
   ::grpc::Status WaitForExecution(::grpc::ServerContext* context,
                                   const WaitForExecutionRequest* arg,
diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc
index 2732d498d8..32707de700 100644
--- a/tensorflow/compiler/xla/service/service.cc
+++ b/tensorflow/compiler/xla/service/service.cc
@@ -524,13 +524,13 @@ Service::ExecuteParallelAndRegisterResult(
 
 StatusOr<GlobalDataHandle> Service::ExecuteAndRegisterResult(
     Executable* executable,
-    const absl::Span<const std::vector<const ShapedBuffer*>> arguments,
-    Backend* backend, const string& result_tag, ExecutionProfile* profile) {
+    absl::Span<const std::vector<const ShapedBuffer*>> arguments,
+    Backend* backend, const DeviceHandle& device_handle,
+    const string& result_tag, ExecutionProfile* profile) {
   // Set up streams.
   std::vector<StreamPool::Ptr> streams;
 
-  TF_ASSIGN_OR_RETURN(auto replicas,
-                      Replicas(*backend, SingleComputationDeviceHandle()));
+  TF_ASSIGN_OR_RETURN(auto replicas, Replicas(*backend, device_handle));
   TF_RET_CHECK(!replicas.empty());
   for (se::StreamExecutor* executor : replicas) {
     TF_ASSIGN_OR_RETURN(StreamPool::Ptr stream,
@@ -538,10 +538,11 @@ StatusOr<GlobalDataHandle> Service::ExecuteAndRegisterResult(
     streams.push_back(std::move(stream));
   }
 
-  TF_ASSIGN_OR_RETURN(DeviceAssignment device_assignment,
-                      backend->computation_placer()->AssignDevices(
-                          options_.number_of_replicas(),
-                          /*computation_count=*/1));
+  DeviceAssignment device_assignment(options_.number_of_replicas(),
+                                     /*computation_count=*/1);
+  for (int64 replica = 0; replica < replicas.size(); ++replica) {
+    device_assignment(replica, 0) = replicas[replica]->device_ordinal();
+  }
 
   // Set up run options.
   std::vector<ServiceExecutableRunOptions> run_options;
@@ -710,14 +711,33 @@ Status Service::ExecuteGraphParallel(const ExecuteGraphParallelRequest* arg,
     }
   }
 
-  // Execute the generated executables in parallel and return the device
-  // handles for each computation's output.
+  // If we have multiple executables to run, execute them all in parallel.  But
+  // if we only have one executable, execute it using the vanilla, non-parallel
+  // call.
+  //
+  // We do this because the Client API uses ExecuteGraphParallel when it wants
+  // to compile and run one computation without caching the executable, but not
+  // all backends support the async StreamExecutor API required by
+  // ExecuteParallelAndRegisterResult.
+  //
+  // TODO(b/122731460): Consolidate Execute{,Parallel}AndRegisterResult; they do
+  // basically the same thing.
   ExecutionProfile profile;
-  TF_ASSIGN_OR_RETURN(
-      std::vector<GlobalDataHandle> outputs,
-      ExecuteParallelAndRegisterResult(executable_ptrs, all_arguments,
-                                       execute_backend_.get(), device_handles,
-                                       computation_names, &profile));
+  std::vector<GlobalDataHandle> outputs;
+  if (executable_ptrs.size() == 1) {
+    TF_ASSIGN_OR_RETURN(
+        auto output,
+        ExecuteAndRegisterResult(executable_ptrs[0], all_arguments[0],
+                                 execute_backend_.get(), device_handles[0],
+                                 computation_names[0], &profile));
+    outputs.push_back(std::move(output));
+  } else {
+    TF_ASSIGN_OR_RETURN(
+        outputs, ExecuteParallelAndRegisterResult(
+                     executable_ptrs, all_arguments, execute_backend_.get(),
+                     device_handles, computation_names, &profile));
+  }
+
   for (const GlobalDataHandle& output : outputs) {
     ExecuteResponse response;
     *response.mutable_output() = output;
@@ -903,6 +923,7 @@ Status Service::Execute(const ExecuteRequest* arg, ExecuteResponse* result) {
       *result->mutable_output(),
       ExecuteAndRegisterResult(executable.get(), replicated_arguments,
                                execute_backend_.get(),
+                               SingleComputationDeviceHandle(),
                                "result of " + executable->module().name(),
                                result->mutable_profile()));
 
diff --git a/tensorflow/compiler/xla/service/service.h b/tensorflow/compiler/xla/service/service.h
index abd3ee5a05..6e31bea728 100644
--- a/tensorflow/compiler/xla/service/service.h
+++ b/tensorflow/compiler/xla/service/service.h
@@ -250,8 +250,9 @@ class Service : public ServiceInterface {
   // ExecutionProfile object which will be filled in with profile data.
   StatusOr<GlobalDataHandle> ExecuteAndRegisterResult(
       Executable* executable,
-      const absl::Span<const std::vector<const ShapedBuffer*>> arguments,
-      Backend* backend, const string& result_tag, ExecutionProfile* profile);
+      absl::Span<const std::vector<const ShapedBuffer*>> arguments,
+      Backend* backend, const DeviceHandle& device_handle,
+      const string& result_tag, ExecutionProfile* profile);
 
   // Runs the given executables with the given arguments and register the result
   // from each executable in the allocation tracker. The handles of the result
-- 
GitLab


From 8b6251446c730269bc698db83acd0787f44f3645 Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Sat, 12 Jan 2019 17:07:42 -0800
Subject: [PATCH 0622/2345] Add a test for paren handling.

PiperOrigin-RevId: 229041154
---
 tensorflow/tools/compatibility/ast_edits.py      |  5 ++++-
 tensorflow/tools/compatibility/ast_edits_test.py | 14 ++++++++++++++
 .../tools/compatibility/tf_upgrade_v2_test.py    | 16 ++++++++++++++++
 3 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/tensorflow/tools/compatibility/ast_edits.py b/tensorflow/tools/compatibility/ast_edits.py
index 2254c223ce..940be4c23c 100644
--- a/tensorflow/tools/compatibility/ast_edits.py
+++ b/tensorflow/tools/compatibility/ast_edits.py
@@ -289,8 +289,11 @@ class _PastaEditVisitor(ast.NodeVisitor):
       reordered = function_reorders[full_name]
       new_keywords = []
       for idx, arg in enumerate(node.args):
+        if sys.version_info[:2] >= (3, 5) and isinstance(arg, ast.Starred):
+          continue  # Can't move Starred to keywords
         keyword_arg = reordered[idx]
-        new_keywords.append(ast.keyword(arg=keyword_arg, value=arg))
+        keyword = ast.keyword(arg=keyword_arg, value=arg)
+        new_keywords.append(keyword)
 
       if new_keywords:
         self.add_log(node.lineno, node.col_offset,
diff --git a/tensorflow/tools/compatibility/ast_edits_test.py b/tensorflow/tools/compatibility/ast_edits_test.py
index 4accd8fe29..ff8b8d6982 100644
--- a/tensorflow/tools/compatibility/ast_edits_test.py
+++ b/tensorflow/tools/compatibility/ast_edits_test.py
@@ -190,6 +190,20 @@ class TestAstEdits(test_util.TensorFlowTestCase):
     _, new_text = self._upgrade(RenameKeywordSpec(), text)
     self.assertEqual(new_text, text)
 
+  def testKeywordReorderWithParens(self):
+    """Test that we get the expected result if there are parens around args."""
+    text = "f((a), ( ( b ) ))\n"
+    acceptable_outputs = [
+        # No change is a valid output
+        text,
+        # Also cases where all arguments are fully specified are allowed
+        "f(a=(a), b=( ( b ) ))\n",
+        # Making the parens canonical is ok
+        "f(a=(a), b=((b)))\n",
+    ]
+    _, new_text = self._upgrade(ReorderKeywordSpec(), text)
+    self.assertIn(new_text, acceptable_outputs)
+
   def testKeywordReorder(self):
     """Test that we get the expected result if kw2 is now before kw1."""
     text = "f(a, b, kw1=c, kw2=d)\n"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 0a85cb39c8..0dde5045aa 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -912,6 +912,22 @@ tf.print('abc')
     self.assertIn("tf.cond", errors[0])
     self.assertIn("requires manual check", errors[0])
 
+  def testParens(self):
+    text = """
+def _log_prob(self, x):
+  return tf.reduce_logsumexp(
+      (self.mixture_distribution.logits + self.distribution.log_prob(
+          x[..., tf.newaxis])),
+          axis=-1)"""
+    expected_text = """
+def _log_prob(self, x):
+  return tf.reduce_logsumexp(
+      input_tensor=(self.mixture_distribution.logits + self.distribution.log_prob(
+          x[..., tf.newaxis])),
+          axis=-1)"""
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(expected_text, new_text)
+
 
 class TestUpgradeFiles(test_util.TensorFlowTestCase):
 
-- 
GitLab


From 4eb7eaa1814d5f0822adf5f61d327f68f0b70982 Mon Sep 17 00:00:00 2001
From: Dheeraj Rajaram Reddy <dheeraj98reddy@gmail.com>
Date: Fri, 11 Jan 2019 23:33:02 +0530
Subject: [PATCH 0623/2345] Add take_while golden API to pbtxt

---
 .../tools/api/golden/v2/tensorflow.data.experimental.pbtxt    | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
index a262c0f799..abc98a74b6 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
@@ -180,6 +180,10 @@ tf_module {
     name: "shuffle_and_repeat"
     argspec: "args=[\'buffer_size\', \'count\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
+  member_method {
+    name: "take_while"
+    argspec: "args=[\'predicate\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "unbatch"
     argspec: "args=[], varargs=None, keywords=None, defaults=None"
-- 
GitLab


From 898738bce44c0cfdaa36b1dfc865239eb48fa999 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 13 Jan 2019 01:02:14 -0800
Subject: [PATCH 0624/2345] compat: Update forward compatibility horizon to
 2019-01-13

PiperOrigin-RevId: 229060750
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 1f4840ff38..f977d0d580 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 12)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 13)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 3e6d902b6e8180fbfe9de0bb2e5069068cb5cbc5 Mon Sep 17 00:00:00 2001
From: Kashif Rasul <kashif.rasul@gmail.com>
Date: Sun, 13 Jan 2019 13:35:32 +0100
Subject: [PATCH 0625/2345] removed deprecated cuda driver API calls

---
 .../stream_executor/cuda/cuda_driver.cc       | 34 +++++++++----------
 tensorflow/stream_executor/cuda/cuda_driver.h | 10 ------
 .../stream_executor/cuda/cuda_gpu_executor.cc | 22 +++++++-----
 3 files changed, 31 insertions(+), 35 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/cuda_driver.cc b/tensorflow/stream_executor/cuda/cuda_driver.cc
index ca7a717bdb..3b6e1bef66 100644
--- a/tensorflow/stream_executor/cuda/cuda_driver.cc
+++ b/tensorflow/stream_executor/cuda/cuda_driver.cc
@@ -1290,15 +1290,26 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                            CUdevice device) {
   *cc_major = 0;
   *cc_minor = 0;
-  CUresult result = cuDeviceComputeCapability(cc_major, cc_minor, device);
-  if (result == CUDA_SUCCESS) {
-    return port::Status::OK();
+
+  CUresult res = 
+      cuDeviceGetAttribute(cc_major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, device);
+  if (res != CUDA_SUCCESS) {
+    return port::Status(
+      port::error::INTERNAL,
+      port::Printf("failed to get compute capability major for device: %s; %d",
+                   ToString(res).c_str(), device));
   }
 
-  return port::Status(
+  res = 
+      cuDeviceGetAttribute(cc_minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, device);
+  if (res != CUDA_SUCCESS) {
+    return port::Status(
       port::error::INTERNAL,
-      port::Printf("failed to get compute capability for device: %s; %d",
-                   ToString(result).c_str(), device));
+      port::Printf("failed to get compute capability minor for device: %s; %d",
+                   ToString(res).c_str(), device));
+  }
+
+  return port::Status::OK();
 }
 
 // Helper function that turns the integer output of cuDeviceGetAttribute to type
@@ -1398,17 +1409,6 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
   return true;
 }
 
-/* static */ bool CUDADriver::GetDeviceProperties(CUdevprop *device_properties,
-                                                  int device_ordinal) {
-  CUresult res = cuDeviceGetProperties(device_properties, device_ordinal);
-  if (res != CUDA_SUCCESS) {
-    LOG(ERROR) << "failed to query device properties: " << ToString(res);
-    return false;
-  }
-
-  return true;
-}
-
 /* static */ port::StatusOr<int> CUDADriver::GetDeviceAttribute(
     CUdevice_attribute attribute, CUdevice device) {
   int val;
diff --git a/tensorflow/stream_executor/cuda/cuda_driver.h b/tensorflow/stream_executor/cuda/cuda_driver.h
index 447422739d..825455d448 100644
--- a/tensorflow/stream_executor/cuda/cuda_driver.h
+++ b/tensorflow/stream_executor/cuda/cuda_driver.h
@@ -407,16 +407,6 @@ class CUDADriver {
   // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g9c3e1414f0ad901d3278a4d6645fc266
   static bool GetGridLimits(int *x, int *y, int *z, CUdevice device);
 
-  // Returns a grab-bag of device properties in a caller-owned device_properties
-  // structure for device_ordinal via cuDeviceGetProperties.
-  //
-  // This call is deprecated in the NVIDIA driver API; its replacement is
-  // GetDeviceAttribute
-  //
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE__DEPRECATED.html#group__CUDA__DEVICE__DEPRECATED_1g65a5b4e25186bd257df80b98c98cffe6
-  static bool GetDeviceProperties(CUdevprop *device_properties,
-                                  int device_ordinal);
-
   // Gets a specific integer-valued property about the given device.
   //
   // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g9c3e1414f0ad901d3278a4d6645fc266
diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
index 5b33d97594..7705e7e8f2 100644
--- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
+++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
@@ -1043,18 +1043,24 @@ DeviceDescription *CUDAExecutor::PopulateDeviceDescription() const {
     builder.set_numa_node(numa_node);
   }
 
-  CUdevprop prop;
-  if (CUDADriver::GetDeviceProperties(&prop, device_ordinal_)) {
-    builder.set_threads_per_block_limit(prop.maxThreadsPerBlock);
+  {
+    builder.set_threads_per_block_limit(
+      CUDADriver::GetDeviceAttribute(
+          CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_BLOCK, device_)
+          .ValueOrDie());
 
     ThreadDim thread_dim_limit;
-    thread_dim_limit.x = prop.maxThreadsDim[0];
-    thread_dim_limit.y = prop.maxThreadsDim[1];
-    thread_dim_limit.z = prop.maxThreadsDim[2];
+    thread_dim_limit.x = CUDADriver::GetDeviceAttribute(
+        CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_X, device_).ValueOrDie();
+    thread_dim_limit.y = CUDADriver::GetDeviceAttribute(
+        CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_Y, device_).ValueOrDie();
+    thread_dim_limit.z = CUDADriver::GetDeviceAttribute(
+        CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_Z, device_).ValueOrDie();
     builder.set_thread_dim_limit(thread_dim_limit);
 
-    float clock_rate_ghz = static_cast<float>(prop.clockRate) / 1e6;
-    builder.set_clock_rate_ghz(clock_rate_ghz);
+    int clockRate = CUDADriver::GetDeviceAttribute(
+        CU_DEVICE_ATTRIBUTE_CLOCK_RATE, device_).ValueOrDie();
+    builder.set_clock_rate_ghz(static_cast<float>(clockRate) / 1e6);
   }
 
   {
-- 
GitLab


From 9a9e7655f1a4121342ffd2ee51dad7a908c40f36 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 13 Jan 2019 23:41:33 +0000
Subject: [PATCH 0626/2345] Fix warnings in feature_column.py

This fix fixes warnings in feature_column.py where
`math_ops.to_int64` has been deprecated and replaced with
`math_ops.cast`.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .../python/feature_column/feature_column.py   | 22 +++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/tensorflow/python/feature_column/feature_column.py b/tensorflow/python/feature_column/feature_column.py
index a858d92608..f474d8a4fe 100644
--- a/tensorflow/python/feature_column/feature_column.py
+++ b/tensorflow/python/feature_column/feature_column.py
@@ -2361,7 +2361,7 @@ class _BucketizedColumn(_DenseColumn, _CategoricalColumn,
     del trainable
     input_tensor = inputs.get(self)
     return array_ops.one_hot(
-        indices=math_ops.to_int64(input_tensor),
+        indices=math_ops.cast(input_tensor, dtypes.int64),
         depth=len(self.boundaries) + 1,
         on_value=1.,
         off_value=0.)
@@ -2391,9 +2391,9 @@ class _BucketizedColumn(_DenseColumn, _CategoricalColumn,
         array_ops.reshape(input_tensor, (-1,)) +
         (len(self.boundaries) + 1) * i2)
 
-    indices = math_ops.to_int64(array_ops.transpose(array_ops.stack((i1, i2))))
-    dense_shape = math_ops.to_int64(array_ops.stack(
-        [batch_size, source_dimension]))
+    indices = math_ops.cast(array_ops.transpose(array_ops.stack((i1, i2))), dtypes.int64)
+    dense_shape = math_ops.cast(array_ops.stack(
+        [batch_size, source_dimension]), dtypes.int64)
     sparse_tensor = sparse_tensor_lib.SparseTensor(
         indices=indices,
         values=bucket_indices,
@@ -2829,7 +2829,7 @@ class _VocabularyFileCategoricalColumn(
     if input_tensor.dtype.is_integer:
       # `index_table_from_file` requires 64-bit integer keys.
       key_dtype = dtypes.int64
-      input_tensor = math_ops.to_int64(input_tensor)
+      input_tensor = math_ops.cast(input_tensor, dtypes.int64)
 
     return lookup_ops.index_table_from_file(
         vocabulary_file=self.vocabulary_file,
@@ -2881,7 +2881,7 @@ class _VocabularyListCategoricalColumn(
     if input_tensor.dtype.is_integer:
       # `index_table_from_tensor` requires 64-bit integer keys.
       key_dtype = dtypes.int64
-      input_tensor = math_ops.to_int64(input_tensor)
+      input_tensor = math_ops.cast(input_tensor, dtypes.int64)
 
     return lookup_ops.index_table_from_tensor(
         vocabulary_list=tuple(self.vocabulary_list),
@@ -2924,9 +2924,9 @@ class _IdentityCategoricalColumn(
           'Invalid input, not integer. key: {} dtype: {}'.format(
               self.key, input_tensor.dtype))
 
-    values = math_ops.to_int64(input_tensor.values, name='values')
-    num_buckets = math_ops.to_int64(self.num_buckets, name='num_buckets')
-    zero = math_ops.to_int64(0, name='zero')
+    values = math_ops.cast(input_tensor.values, dtypes.int64, name='values')
+    num_buckets = math_ops.cast(self.num_buckets, dtypes.int64, name='num_buckets')
+    zero = math_ops.cast(0, dtypes.int64, name='zero')
     if self.default_value is None:
       # Fail if values are out-of-range.
       assert_less = check_ops.assert_less(
@@ -2944,7 +2944,7 @@ class _IdentityCategoricalColumn(
               values < zero, values >= num_buckets, name='out_of_range'),
           array_ops.fill(
               dims=array_ops.shape(values),
-              value=math_ops.to_int64(self.default_value),
+              value=math_ops.cast(self.default_value, dtypes.int64),
               name='default_values'),
           values)
 
@@ -3256,7 +3256,7 @@ def _sequence_length_from_sparse_tensor(sp_tensor, num_elements=1):
     # Example: orig tensor [[1, 2], [3]], col_ids = (0, 1, 1),
     # row_ids = (0, 0, 1), seq_length = [2, 1]. If num_elements = 2,
     # these will get grouped, and the final seq_length is [1, 1]
-    seq_length = math_ops.to_int64(math_ops.ceil(seq_length / num_elements))
+    seq_length = math_ops.cast(math_ops.ceil(seq_length / num_elements), dtypes.int64)
 
     # If the last n rows do not have ids, seq_length will have shape
     # [batch_size - n]. Pad the remaining values with zeros.
-- 
GitLab


From fdaf63b38b511126d66aa9b3da254df8c85e47e2 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 13 Jan 2019 23:45:42 +0000
Subject: [PATCH 0627/2345] Line too long (line-too-long) fix

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/feature_column/feature_column.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/feature_column/feature_column.py b/tensorflow/python/feature_column/feature_column.py
index f474d8a4fe..84db3d3ac2 100644
--- a/tensorflow/python/feature_column/feature_column.py
+++ b/tensorflow/python/feature_column/feature_column.py
@@ -2391,7 +2391,8 @@ class _BucketizedColumn(_DenseColumn, _CategoricalColumn,
         array_ops.reshape(input_tensor, (-1,)) +
         (len(self.boundaries) + 1) * i2)
 
-    indices = math_ops.cast(array_ops.transpose(array_ops.stack((i1, i2))), dtypes.int64)
+    indices = math_ops.cast(
+        array_ops.transpose(array_ops.stack((i1, i2))), dtypes.int64)
     dense_shape = math_ops.cast(array_ops.stack(
         [batch_size, source_dimension]), dtypes.int64)
     sparse_tensor = sparse_tensor_lib.SparseTensor(
@@ -2925,7 +2926,8 @@ class _IdentityCategoricalColumn(
               self.key, input_tensor.dtype))
 
     values = math_ops.cast(input_tensor.values, dtypes.int64, name='values')
-    num_buckets = math_ops.cast(self.num_buckets, dtypes.int64, name='num_buckets')
+    num_buckets = math_ops.cast(
+        self.num_buckets, dtypes.int64, name='num_buckets')
     zero = math_ops.cast(0, dtypes.int64, name='zero')
     if self.default_value is None:
       # Fail if values are out-of-range.
@@ -3256,7 +3258,8 @@ def _sequence_length_from_sparse_tensor(sp_tensor, num_elements=1):
     # Example: orig tensor [[1, 2], [3]], col_ids = (0, 1, 1),
     # row_ids = (0, 0, 1), seq_length = [2, 1]. If num_elements = 2,
     # these will get grouped, and the final seq_length is [1, 1]
-    seq_length = math_ops.cast(math_ops.ceil(seq_length / num_elements), dtypes.int64)
+    seq_length = math_ops.cast(
+        math_ops.ceil(seq_length / num_elements), dtypes.int64)
 
     # If the last n rows do not have ids, seq_length will have shape
     # [batch_size - n]. Pad the remaining values with zeros.
-- 
GitLab


From ac4d3682939dac303082097d25ebda805409acde Mon Sep 17 00:00:00 2001
From: Suharsh Sivakumar <suharshs@google.com>
Date: Sun, 13 Jan 2019 18:02:02 -0800
Subject: [PATCH 0628/2345] Add quantized_dimension field to quantization
 params.

PiperOrigin-RevId: 229109989
---
 tensorflow/lite/schema/schema.fbs         | 15 ++++++++++--
 tensorflow/lite/schema/schema_generated.h | 29 ++++++++++++++++++-----
 2 files changed, 36 insertions(+), 8 deletions(-)

diff --git a/tensorflow/lite/schema/schema.fbs b/tensorflow/lite/schema/schema.fbs
index aca926a3b9..13f47d2cfc 100644
--- a/tensorflow/lite/schema/schema.fbs
+++ b/tensorflow/lite/schema/schema.fbs
@@ -64,9 +64,20 @@ table QuantizationParameters {
   scale:[float];  // For dequantizing the tensor's values.
   zero_point:[long];
 
-  // If this is not none, the quantization parameters above are ignored and the
-  // value of the QuantizationDetails union below should be used.
+  // If this is not none, the other quantization parameters (i.e. min, max,
+  // scale, zero_point fields above) are ignored and the value of the
+  // QuantizationDetails union should be used.
   details:QuantizationDetails;
+
+  // Specifies the dimension of the Tensor's shape that the scales and
+  // zero_points correspond to. For example, a tensor t, with dims=[4, 3, 2, 1]
+  // with quantization params:
+  //   scale=[1.0, 2.0, 3.0], zero_point=[1, 2, 3], quantization_dimension=1
+  // will be quantized across the second dimension of t.
+  //   t[:, 0, :, :] will have scale[0]=1.0, zero_point[0]=1
+  //   t[:, 1, :, :] will have scale[1]=2.0, zero_point[0]=2
+  //   t[:, 2, :, :] will have scale[2]=3.0, zero_point[0]=3
+  quantized_dimension:int;
 }
 
 table Tensor {
diff --git a/tensorflow/lite/schema/schema_generated.h b/tensorflow/lite/schema/schema_generated.h
index 1d153cc717..a1174e79f7 100755
--- a/tensorflow/lite/schema/schema_generated.h
+++ b/tensorflow/lite/schema/schema_generated.h
@@ -2335,7 +2335,9 @@ struct QuantizationParametersT : public flatbuffers::NativeTable {
   std::vector<float> scale;
   std::vector<int64_t> zero_point;
   QuantizationDetailsUnion details;
-  QuantizationParametersT() {
+  int32_t quantized_dimension;
+  QuantizationParametersT()
+      : quantized_dimension(0) {
   }
 };
 
@@ -2347,7 +2349,8 @@ struct QuantizationParameters FLATBUFFERS_FINAL_CLASS : private flatbuffers::Tab
     VT_SCALE = 8,
     VT_ZERO_POINT = 10,
     VT_DETAILS_TYPE = 12,
-    VT_DETAILS = 14
+    VT_DETAILS = 14,
+    VT_QUANTIZED_DIMENSION = 16
   };
   const flatbuffers::Vector<float> *min() const {
     return GetPointer<const flatbuffers::Vector<float> *>(VT_MIN);
@@ -2371,6 +2374,9 @@ struct QuantizationParameters FLATBUFFERS_FINAL_CLASS : private flatbuffers::Tab
   const CustomQuantization *details_as_CustomQuantization() const {
     return details_type() == QuantizationDetails_CustomQuantization ? static_cast<const CustomQuantization *>(details()) : nullptr;
   }
+  int32_t quantized_dimension() const {
+    return GetField<int32_t>(VT_QUANTIZED_DIMENSION, 0);
+  }
   bool Verify(flatbuffers::Verifier &verifier) const {
     return VerifyTableStart(verifier) &&
            VerifyOffset(verifier, VT_MIN) &&
@@ -2384,6 +2390,7 @@ struct QuantizationParameters FLATBUFFERS_FINAL_CLASS : private flatbuffers::Tab
            VerifyField<uint8_t>(verifier, VT_DETAILS_TYPE) &&
            VerifyOffset(verifier, VT_DETAILS) &&
            VerifyQuantizationDetails(verifier, details(), details_type()) &&
+           VerifyField<int32_t>(verifier, VT_QUANTIZED_DIMENSION) &&
            verifier.EndTable();
   }
   QuantizationParametersT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const;
@@ -2416,6 +2423,9 @@ struct QuantizationParametersBuilder {
   void add_details(flatbuffers::Offset<void> details) {
     fbb_.AddOffset(QuantizationParameters::VT_DETAILS, details);
   }
+  void add_quantized_dimension(int32_t quantized_dimension) {
+    fbb_.AddElement<int32_t>(QuantizationParameters::VT_QUANTIZED_DIMENSION, quantized_dimension, 0);
+  }
   explicit QuantizationParametersBuilder(flatbuffers::FlatBufferBuilder &_fbb)
         : fbb_(_fbb) {
     start_ = fbb_.StartTable();
@@ -2435,8 +2445,10 @@ inline flatbuffers::Offset<QuantizationParameters> CreateQuantizationParameters(
     flatbuffers::Offset<flatbuffers::Vector<float>> scale = 0,
     flatbuffers::Offset<flatbuffers::Vector<int64_t>> zero_point = 0,
     QuantizationDetails details_type = QuantizationDetails_NONE,
-    flatbuffers::Offset<void> details = 0) {
+    flatbuffers::Offset<void> details = 0,
+    int32_t quantized_dimension = 0) {
   QuantizationParametersBuilder builder_(_fbb);
+  builder_.add_quantized_dimension(quantized_dimension);
   builder_.add_details(details);
   builder_.add_zero_point(zero_point);
   builder_.add_scale(scale);
@@ -2453,7 +2465,8 @@ inline flatbuffers::Offset<QuantizationParameters> CreateQuantizationParametersD
     const std::vector<float> *scale = nullptr,
     const std::vector<int64_t> *zero_point = nullptr,
     QuantizationDetails details_type = QuantizationDetails_NONE,
-    flatbuffers::Offset<void> details = 0) {
+    flatbuffers::Offset<void> details = 0,
+    int32_t quantized_dimension = 0) {
   return tflite::CreateQuantizationParameters(
       _fbb,
       min ? _fbb.CreateVector<float>(*min) : 0,
@@ -2461,7 +2474,8 @@ inline flatbuffers::Offset<QuantizationParameters> CreateQuantizationParametersD
       scale ? _fbb.CreateVector<float>(*scale) : 0,
       zero_point ? _fbb.CreateVector<int64_t>(*zero_point) : 0,
       details_type,
-      details);
+      details,
+      quantized_dimension);
 }
 
 flatbuffers::Offset<QuantizationParameters> CreateQuantizationParameters(flatbuffers::FlatBufferBuilder &_fbb, const QuantizationParametersT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
@@ -8263,6 +8277,7 @@ inline void QuantizationParameters::UnPackTo(QuantizationParametersT *_o, const
   { auto _e = zero_point(); if (_e) { _o->zero_point.resize(_e->size()); for (flatbuffers::uoffset_t _i = 0; _i < _e->size(); _i++) { _o->zero_point[_i] = _e->Get(_i); } } };
   { auto _e = details_type(); _o->details.type = _e; };
   { auto _e = details(); if (_e) _o->details.value = QuantizationDetailsUnion::UnPack(_e, details_type(), _resolver); };
+  { auto _e = quantized_dimension(); _o->quantized_dimension = _e; };
 }
 
 inline flatbuffers::Offset<QuantizationParameters> QuantizationParameters::Pack(flatbuffers::FlatBufferBuilder &_fbb, const QuantizationParametersT* _o, const flatbuffers::rehasher_function_t *_rehasher) {
@@ -8279,6 +8294,7 @@ inline flatbuffers::Offset<QuantizationParameters> CreateQuantizationParameters(
   auto _zero_point = _o->zero_point.size() ? _fbb.CreateVector(_o->zero_point) : 0;
   auto _details_type = _o->details.type;
   auto _details = _o->details.Pack(_fbb);
+  auto _quantized_dimension = _o->quantized_dimension;
   return tflite::CreateQuantizationParameters(
       _fbb,
       _min,
@@ -8286,7 +8302,8 @@ inline flatbuffers::Offset<QuantizationParameters> CreateQuantizationParameters(
       _scale,
       _zero_point,
       _details_type,
-      _details);
+      _details,
+      _quantized_dimension);
 }
 
 inline TensorT *Tensor::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
-- 
GitLab


From 5b4c85b9d94484e7ba1f29344f0d2672228ed014 Mon Sep 17 00:00:00 2001
From: Jonas Rauber <jonasrauber@users.noreply.github.com>
Date: Mon, 14 Jan 2019 09:33:47 +0100
Subject: [PATCH 0629/2345] fixed typo

---
 tensorflow/python/eager/backprop.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index 42db726a7b..effbc03ae3 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -465,7 +465,7 @@ def val_and_grad_function(f, params=None):
 
 
 def make_vjp(f, params=None, persistent=True):
-  """Returns a function that computes f and is vjp w.r.t. params.
+  """Returns a function that computes f and its vjp w.r.t. params.
 
   The term "vjp" here is an abbreviation for vector-jacobian product.
 
-- 
GitLab


From 2a50782fb68f0b7c9fc4926c7c9e6b05c05abb1c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 00:47:08 -0800
Subject: [PATCH 0630/2345] internal change

PiperOrigin-RevId: 229136654
---
 tensorflow/lite/experimental/examples/lstm/tflite_lstm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/experimental/examples/lstm/tflite_lstm.py b/tensorflow/lite/experimental/examples/lstm/tflite_lstm.py
index 2fe8ebf9e9..461345060b 100644
--- a/tensorflow/lite/experimental/examples/lstm/tflite_lstm.py
+++ b/tensorflow/lite/experimental/examples/lstm/tflite_lstm.py
@@ -184,7 +184,7 @@ class TFLiteLSTMCell(rnn_cell_impl.LayerRNNCell):
       var = self.add_variable(
           name, shape=shape, initializer=initializer, partitioner=partitioner)
       return self._tflite_wrapper.add_input(
-          var, name="name", index_override=index)
+          var, name=name, index_override=index)
 
     weight_initializer = self._initializer
     if self.dtype is None:
-- 
GitLab


From cfbf81335a3de3c31411bad33ff43ffcfda20817 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 01:02:53 -0800
Subject: [PATCH 0631/2345] compat: Update forward compatibility horizon to
 2019-01-14

PiperOrigin-RevId: 229138454
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index f977d0d580..c9f2397fa5 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 13)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 14)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 6dd6ad9fd7353d945227fd87a93ef1f11eccc75e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 01:32:22 -0800
Subject: [PATCH 0632/2345] Set up remote GPU testing.

Currently, we set the tag "local" for GPU tests in order to be able to execute
remote CPU and local GPU tests within the same bazel invocation.

This change introduces the possibility to set REMOTE_GPU_TESTING to enable
GPU tests to also run remotely; given that tags cannot use starlark's select,
we use an autoconfig rule that defines a function returning the tags we want:
"local" by default and "remote-gpu" if REMOTE_GPU_TESTING is set.

The platform is set via exec_compatible_with constraints, so we select on the
"remote-gpu" tag to add a constraint that is only fulfilled by GPU-enabled
platforms.

PiperOrigin-RevId: 229141861
---
 tensorflow/compiler/tests/build_defs.bzl      |  5 ++-
 .../platform/default/build_config_root.bzl    | 12 +++++-
 tensorflow/opensource_only.files              |  4 ++
 tensorflow/tensorflow.bzl                     |  8 +++-
 tensorflow/workspace.bzl                      |  2 +
 third_party/toolchains/BUILD                  | 35 +++++++++++++++
 third_party/toolchains/remote/BUILD           |  0
 third_party/toolchains/remote/BUILD.tpl       |  0
 third_party/toolchains/remote/configure.bzl   | 43 +++++++++++++++++++
 .../toolchains/remote/execution.bzl.tpl       |  2 +
 10 files changed, 107 insertions(+), 4 deletions(-)
 create mode 100644 third_party/toolchains/remote/BUILD
 create mode 100644 third_party/toolchains/remote/BUILD.tpl
 create mode 100644 third_party/toolchains/remote/configure.bzl
 create mode 100644 third_party/toolchains/remote/execution.bzl.tpl

diff --git a/tensorflow/compiler/tests/build_defs.bzl b/tensorflow/compiler/tests/build_defs.bzl
index be9766c4ef..ed580f95b6 100644
--- a/tensorflow/compiler/tests/build_defs.bzl
+++ b/tensorflow/compiler/tests/build_defs.bzl
@@ -5,6 +5,7 @@ load("//tensorflow/compiler/tests:plugin.bzl", "plugins")
 load(
     "//tensorflow/core:platform/default/build_config_root.bzl",
     "tf_cuda_tests_tags",
+    "tf_exec_compatible_with",
 )
 
 def all_backends():
@@ -84,6 +85,7 @@ def tf_xla_py_test(
         else:
             fail("Unknown backend {}".format(backend))
 
+        test_tags = tags + backend_tags
         native.py_test(
             name = test_name,
             srcs = srcs,
@@ -92,7 +94,8 @@ def tf_xla_py_test(
             main = "{}.py".format(name) if main == None else main,
             data = data + backend_data,
             deps = deps + backend_deps,
-            tags = tags + backend_tags,
+            tags = test_tags,
+            exec_compatible_with = tf_exec_compatible_with({"tags": test_tags}),
             **kwargs
         )
         test_names.append(test_name)
diff --git a/tensorflow/core/platform/default/build_config_root.bzl b/tensorflow/core/platform/default/build_config_root.bzl
index 37475feebe..ab05b25d68 100644
--- a/tensorflow/core/platform/default/build_config_root.bzl
+++ b/tensorflow/core/platform/default/build_config_root.bzl
@@ -2,11 +2,19 @@
 # The functions in this file might be referred by tensorflow.bzl. They have to
 # be separate to avoid cyclic references.
 
+load("@local_config_remote_execution//:remote_execution.bzl", "gpu_test_tags")
+
 def tf_cuda_tests_tags():
-    return ["requires-gpu", "local", "gpu"]
+    return ["requires-gpu", "gpu"] + gpu_test_tags()
 
 def tf_sycl_tests_tags():
-    return ["requires-gpu", "local", "gpu"]
+    return ["requires-gpu", "gpu"] + gpu_test_tags()
+
+def tf_exec_compatible_with(kwargs):
+    if ("tags" in kwargs and kwargs["tags"] != None and
+        "remote-gpu" in kwargs["tags"]):
+        return ["@org_tensorflow//third_party/toolchains:gpu_test"]
+    return []
 
 def tf_additional_plugin_deps():
     return select({
diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index bade64dcf8..03b5e5a73c 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -71,6 +71,10 @@ tensorflow/third_party/toolchains/cpus/arm/CROSSTOOL.tpl
 tensorflow/third_party/toolchains/cpus/arm/BUILD
 tensorflow/third_party/toolchains/cpus/py3/BUILD
 tensorflow/third_party/toolchains/cpus/py/BUILD
+tensorflow/third_party/toolchains/remote/configure.bzl
+tensorflow/third_party/toolchains/remote/BUILD.tpl
+tensorflow/third_party/toolchains/remote/BUILD
+tensorflow/third_party/toolchains/remote/execution.bzl.tpl
 tensorflow/third_party/toolchains/BUILD
 tensorflow/third_party/gpus/BUILD
 tensorflow/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_rocm.tpl
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index 1024b686eb..b7da5bf913 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -9,6 +9,7 @@ load(
     "tf_additional_grpc_deps_py",
     "tf_additional_xla_deps_py",
     "tf_cuda_tests_tags",
+    "tf_exec_compatible_with",
     "tf_sycl_tests_tags",
 )
 load(
@@ -785,6 +786,7 @@ def tf_cc_test(
             ],
         ),
         data = data + tf_binary_dynamic_kernel_dsos(kernels),
+        exec_compatible_with = tf_exec_compatible_with(kwargs),
         # Nested select() statements seem not to be supported when passed to
         # linkstatic, and we already have a cuda select() passed in to this
         # function.
@@ -897,6 +899,7 @@ def tf_cuda_only_cc_test(
         args = [],
         kernels = [],
         linkopts = []):
+    tags = tags + tf_cuda_tests_tags()
     native.cc_test(
         name = "%s%s" % (name, "_gpu"),
         srcs = srcs + tf_binary_additional_srcs(),
@@ -919,7 +922,8 @@ def tf_cuda_only_cc_test(
             clean_dep("//tensorflow:darwin"): 1,
             "//conditions:default": 0,
         }),
-        tags = tags + tf_cuda_tests_tags(),
+        tags = tags,
+        exec_compatible_with = tf_exec_compatible_with({"tags": tags}),
     )
 
 register_extension_info(
@@ -983,6 +987,7 @@ def tf_cc_test_mkl(
             }) + _rpath_linkopts(src_to_test_name(src)),
             deps = deps + tf_binary_dynamic_kernel_deps(kernels) + mkl_deps(),
             data = data + tf_binary_dynamic_kernel_dsos(kernels),
+            exec_compatible_with = tf_exec_compatible_with({"tags": tags}),
             linkstatic = linkstatic,
             tags = tags,
             size = size,
@@ -1752,6 +1757,7 @@ def py_test(deps = [], data = [], kernels = [], **kwargs):
             "//conditions:default": [],
             clean_dep("//tensorflow:no_tensorflow_py_deps"): ["//tensorflow/tools/pip_package:win_pip_package_marker"],
         }) + tf_binary_dynamic_kernel_dsos(kernels),
+        exec_compatible_with = tf_exec_compatible_with(kwargs),
         **kwargs
     )
 
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index bea57b8ed7..46c1261be2 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -10,6 +10,7 @@ load("//third_party/py:python_configure.bzl", "python_configure")
 
 load("//third_party/sycl:sycl_configure.bzl", "sycl_configure")
 load("//third_party/systemlibs:syslibs_configure.bzl", "syslibs_configure")
+load("//third_party/toolchains/remote:configure.bzl", "remote_execution_configure")
 load("//third_party/toolchains/clang6:repo.bzl", "clang6_configure")
 load("//third_party/toolchains/cpus/arm:arm_compiler_configure.bzl", "arm_compiler_configure")
 load("//third_party:repo.bzl", "tf_http_archive")
@@ -64,6 +65,7 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     syslibs_configure(name = "local_config_syslibs")
     python_configure(name = "local_config_python")
     rocm_configure(name = "local_config_rocm")
+    remote_execution_configure(name = "local_config_remote_execution")
 
     initialize_third_party()
 
diff --git a/third_party/toolchains/BUILD b/third_party/toolchains/BUILD
index 6ed6e5c367..509b541400 100644
--- a/third_party/toolchains/BUILD
+++ b/third_party/toolchains/BUILD
@@ -4,6 +4,17 @@ package(default_visibility = ["//visibility:public"])
 
 load("//third_party/toolchains/preconfig/generate:containers.bzl", "container_digests")
 
+# Constraint used for platforms below so we can force certain rules to be executed
+# on specific platforms.
+constraint_setting(name = "custom_platforms")
+
+# Constraint for platforms that allow GPU testing (i.e. have a GPU available).
+# This is used in exec_compatible_with of rules that need GPU access.
+constraint_value(
+    name = "gpu_test",
+    constraint_setting = ":custom_platforms",
+)
+
 # TODO(b/122347293): This is the RBE config based on the CPU configuration / image provided
 # in the asci-toolchain setup. Delete this once we switched CPU remote builds to the
 # new platform below.
@@ -68,3 +79,27 @@ platform(
             value:"docker://gcr.io/tensorflow-testing/nosla-cuda10.0-cudnn7-ubuntu14.04@%s"
         }""" % container_digests["cuda10.0-cudnn7-ubuntu14.04"],
 )
+
+# The above platform with GPU support.
+platform(
+    name = "rbe_cuda10.0-cudnn7-ubuntu14.04-gpu",
+    constraint_values = [
+        "@bazel_tools//platforms:x86_64",
+        "@bazel_tools//platforms:linux",
+        ":gpu_test",
+    ],
+    remote_execution_properties = """
+        properties: {
+            name: "container-image"
+            value: "docker://gcr.io/tensorflow-testing/nosla-cuda10.0-cudnn7-ubuntu14.04@%s"
+        }
+        properties: {
+            name: "dockerRuntime"
+            value: "nvidia"
+        }
+        properties: {
+            name: "Pool"
+            value: "gpu-pool"
+        }
+        """ % container_digests["cuda10.0-cudnn7-ubuntu14.04"],
+)
diff --git a/third_party/toolchains/remote/BUILD b/third_party/toolchains/remote/BUILD
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/third_party/toolchains/remote/BUILD.tpl b/third_party/toolchains/remote/BUILD.tpl
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/third_party/toolchains/remote/configure.bzl b/third_party/toolchains/remote/configure.bzl
new file mode 100644
index 0000000000..cc5b984264
--- /dev/null
+++ b/third_party/toolchains/remote/configure.bzl
@@ -0,0 +1,43 @@
+"""Repository rule for remote GPU autoconfiguration.
+
+This rule creates the starlark file
+//third_party/toolchains/remote:execution.bzl
+providing the function `gpu_test_tags`.
+
+`gpu_test_tags` will return:
+
+  * `local`: if `REMOTE_GPU_TESTING` is false, allowing CPU tests to run
+    remotely and GPU tests to run locally in the same bazel invocation.
+  * `remote-gpu`: if `REMOTE_GPU_TESTING` is true; this allows rules to
+    set an execution requirement that enables a GPU-enabled remote platform.
+"""
+
+_REMOTE_GPU_TESTING = "REMOTE_GPU_TESTING"
+
+def _flag_enabled(repository_ctx, flag_name):
+    if flag_name not in repository_ctx.os.environ:
+        return False
+    return repository_ctx.os.environ[flag_name].strip() == "1"
+
+def _remote_execution_configure(repository_ctx):
+    # If we do not support remote gpu test execution, mark them as local, so we
+    # can combine remote builds with local gpu tests.
+    gpu_test_tags = "\"local\""
+    if _flag_enabled(repository_ctx, _REMOTE_GPU_TESTING):
+        gpu_test_tags = "\"remote-gpu\""
+    repository_ctx.template(
+        "remote_execution.bzl",
+        Label("//third_party/toolchains/remote:execution.bzl.tpl"),
+        {
+            "%{gpu_test_tags}": gpu_test_tags,
+        },
+    )
+    repository_ctx.template(
+        "BUILD",
+        Label("//third_party/toolchains/remote:BUILD.tpl"),
+    )
+
+remote_execution_configure = repository_rule(
+    implementation = _remote_execution_configure,
+    environ = [_REMOTE_GPU_TESTING],
+)
diff --git a/third_party/toolchains/remote/execution.bzl.tpl b/third_party/toolchains/remote/execution.bzl.tpl
new file mode 100644
index 0000000000..18858cc0dc
--- /dev/null
+++ b/third_party/toolchains/remote/execution.bzl.tpl
@@ -0,0 +1,2 @@
+def gpu_test_tags():
+    return [%{gpu_test_tags}]
-- 
GitLab


From 4ac9bda9d0259ac42fc60ab6ad0d2520d3dde88a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 02:36:35 -0800
Subject: [PATCH 0633/2345] [TF:XLA] Enable XLA through autojit for all
 tensorflow/python/kernel_test/ tests.

Some test methods are disabled, but all tests now have a new "_xla" version of the test for XLA:GPU testing. This will run 2 different tests. One with XLA and one without.

PiperOrigin-RevId: 229149574
---
 tensorflow/python/BUILD                       |   8 +
 tensorflow/python/framework/function_test.py  |   1 +
 .../python/framework/is_xla_test_true.py      |  29 +++
 tensorflow/python/framework/test_util.py      | 166 ++++++++++++------
 tensorflow/python/kernel_tests/BUILD          | 130 ++++++++++++++
 .../python/kernel_tests/atrous_conv2d_test.py |   1 +
 .../python/kernel_tests/benchmark_test.py     |   2 +
 .../python/kernel_tests/check_ops_test.py     |   4 +
 .../python/kernel_tests/cholesky_op_test.py   |   1 +
 .../python/kernel_tests/concat_op_test.py     |   5 +
 .../kernel_tests/control_flow_ops_py_test.py  |   2 +
 .../kernel_tests/depthtospace_op_test.py      |   1 +
 .../kernel_tests/depthwise_conv_op_test.py    |   2 +
 .../python/kernel_tests/diag_op_test.py       |   2 +
 .../python/kernel_tests/distributions/BUILD   |  17 ++
 .../kernel_tests/functional_ops_test.py       |   4 +
 .../python/kernel_tests/gather_nd_op_test.py  |   3 +
 .../python/kernel_tests/gather_op_test.py     |   2 +
 tensorflow/python/kernel_tests/linalg/BUILD   |  16 ++
 tensorflow/python/kernel_tests/pool_test.py   |   1 +
 .../python/kernel_tests/pooling_ops_test.py   |  20 ++-
 tensorflow/python/kernel_tests/random/BUILD   |   8 +
 .../kernel_tests/random/random_ops_test.py    |   1 +
 .../python/kernel_tests/relu_op_test.py       |   1 +
 .../resource_variable_ops_test.py             |   1 +
 .../python/kernel_tests/scan_ops_test.py      |   2 +
 .../kernel_tests/scatter_nd_ops_test.py       |   1 +
 tensorflow/python/kernel_tests/signal/BUILD   |   8 +
 .../kernel_tests/signal/fft_ops_test.py       |   1 +
 .../kernel_tests/spacetodepth_op_test.py      |   1 +
 .../python/kernel_tests/split_op_test.py      |   1 +
 tensorflow/python/ops/bitwise_ops_test.py     |   3 +
 tensorflow/python/ops/nn_batchnorm_test.py    |   1 +
 .../python/ops/nn_fused_batchnorm_test.py     |   1 +
 tensorflow/python/ops/nn_test.py              |   2 +
 tensorflow/python/ops/parallel_for/BUILD      |   2 +
 .../ops/parallel_for/control_flow_ops_test.py |   1 +
 .../python/ops/parallel_for/gradients_test.py |   1 +
 .../python/ops/special_math_ops_test.py       |   1 +
 tensorflow/tensorflow.bzl                     |  23 ++-
 40 files changed, 422 insertions(+), 55 deletions(-)
 create mode 100644 tensorflow/python/framework/is_xla_test_true.py

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 659f0735cb..2c6672c81e 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -1103,6 +1103,14 @@ py_library(
     ],
 )
 
+# Including this as a dependency will result in tests using
+# :framework_test_lib to use XLA.
+py_library(
+    name = "is_xla_test_true",
+    srcs = ["framework/is_xla_test_true.py"],
+    srcs_version = "PY2AND3",
+)
+
 py_library(
     name = "distributed_framework_test_lib",
     srcs_version = "PY2AND3",
diff --git a/tensorflow/python/framework/function_test.py b/tensorflow/python/framework/function_test.py
index 7543376bcf..3d5a5fe797 100644
--- a/tensorflow/python/framework/function_test.py
+++ b/tensorflow/python/framework/function_test.py
@@ -284,6 +284,7 @@ class FunctionTest(test.TestCase):
         out, = sess.run(dlogits, {logits: x, labels: y})
       self.assertAllClose(out, np.exp(prob - y))
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testCustomGradientError(self):
     dtype = dtypes.float32
 
diff --git a/tensorflow/python/framework/is_xla_test_true.py b/tensorflow/python/framework/is_xla_test_true.py
new file mode 100644
index 0000000000..7ae1e68b36
--- /dev/null
+++ b/tensorflow/python/framework/is_xla_test_true.py
@@ -0,0 +1,29 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Including this as a dependency will result in Tensorflow tests using XLA.
+
+This function is defined by default in test_util.py to False. The test_util then
+attempts to import this module. If this file is made available through the BUILD
+rule, then this function is overridden and will instead cause Tensorflow graphs
+to be compiled with XLA.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+def is_xla_enabled():
+  """Returns true to state XLA should be enabled for Tensorflow tests."""
+  return True
diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index 6d01d3bf54..491249089e 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -82,6 +82,19 @@ from tensorflow.python.util.protobuf import compare
 from tensorflow.python.util.tf_export import tf_export
 
 
+# If the above import is made available through the BUILD rule, then this
+# function is overridden and will instead return True and cause Tensorflow
+# graphs to be compiled with XLA.
+def is_xla_enabled():
+  return False
+
+
+try:
+  from tensorflow.python.framework.is_xla_test_true import is_xla_enabled  # pylint: disable=g-import-not-at-top
+except:
+  pass
+
+
 @tf_export("test.gpu_device_name")
 def gpu_device_name():
   """Returns the name of a GPU device if available or the empty string."""
@@ -97,6 +110,7 @@ def assert_ops_in_graph(expected_ops, graph):
   Args:
     expected_ops: `dict<string, string>` of op name to op type.
     graph: Graph to check.
+
   Returns:
     `dict<string, node>` of node name to node.
 
@@ -149,7 +163,7 @@ def assert_equal_graph_def_v1(actual, expected, checkpoint_v2=False):
     actual: The `GraphDef` we have.
     expected: The `GraphDef` we expected.
     checkpoint_v2: boolean determining whether to ignore randomized attribute
-        values that appear in V2 checkpoints.
+      values that appear in V2 checkpoints.
 
   Raises:
     AssertionError: If the `GraphDef`s do not match.
@@ -360,7 +374,8 @@ def skip_if(condition):
 
   Args:
     condition: Either an expression that can be used in "if not condition"
-               statement, or a callable whose result should be a boolean.
+      statement, or a callable whose result should be a boolean.
+
   Returns:
     The wrapped function
   """
@@ -483,9 +498,11 @@ def disable_control_flow_v2(unused_msg):
   Returns:
     The wrapped function with _disable_control_flow_v2 attr set to True.
   """
+
   def wrapper(func):
     func._disable_control_flow_v2 = True
     return func
+
   return wrapper
 
 
@@ -568,6 +585,7 @@ def assert_no_new_tensors(f):
 
   Args:
     f: The test case to run.
+
   Returns:
     The decorated test case.
   """
@@ -727,6 +745,7 @@ def assert_no_garbage_created(f):
 
   Args:
     f: The function to decorate.
+
   Returns:
     The decorated function.
   """
@@ -799,8 +818,8 @@ def _combine_named_parameters(**kwargs):
   can be computed using `times()`.
 
   Args:
-    **kwargs: keyword arguments of form `option=[possibilities, ...]`
-         or `option=the_only_possibility`.
+    **kwargs: keyword arguments of form `option=[possibilities, ...]` or
+      `option=the_only_possibility`.
 
   Returns:
     a list of dictionaries for each combination. Keys in the dictionaries are
@@ -838,8 +857,8 @@ def generate_combinations_with_testcase_name(**kwargs):
   parameterized tests.
 
   Args:
-    **kwargs: keyword arguments of form `option=[possibilities, ...]`
-         or `option=the_only_possibility`.
+    **kwargs: keyword arguments of form `option=[possibilities, ...]` or
+      `option=the_only_possibility`.
 
   Returns:
     a list of dictionaries for each combination. Keys in the dictionaries are
@@ -866,11 +885,12 @@ def generate_combinations_with_testcase_name(**kwargs):
 def run_all_in_graph_and_eager_modes(cls):
   """Execute all test methods in the given class with and without eager."""
   base_decorator = run_in_graph_and_eager_modes
-  for name, value in cls.__dict__.copy().items():
-    if (callable(value) and
-        name.startswith(unittest.TestLoader.testMethodPrefix) and
-        not (name.startswith("testSkipEager")
-             or name.startswith("test_skip_eager"))):
+  for name in dir(cls):
+    value = getattr(cls, name)
+    if callable(value) and name.startswith(
+        unittest.TestLoader.testMethodPrefix) and not (
+            name.startswith("testSkipEager") or
+            name.startswith("test_skip_eager") or name == "test_session"):
       setattr(cls, name, base_decorator(value))
   return cls
 
@@ -917,8 +937,8 @@ def run_in_graph_and_eager_modes(func=None,
     func: function to be annotated. If `func` is None, this method returns a
       decorator the can be applied to a function. If `func` is not None this
       returns the decorator applied to `func`.
-    config: An optional config_pb2.ConfigProto to use to configure the
-      session when executing graphs.
+    config: An optional config_pb2.ConfigProto to use to configure the session
+      when executing graphs.
     use_gpu: If True, attempt to run as many operations as possible on GPU.
     reset_test: If True, tearDown and SetUp the test case between the two
       executions of the test (once with and once without eager execution).
@@ -932,6 +952,7 @@ def run_in_graph_and_eager_modes(func=None,
       collected elsewhere in the unit test file will not work). Additionally,
       checks that nothing still has a reference to Tensors that the test
       allocated.
+
   Returns:
     Returns a decorator that will run the decorated test method twice:
     once by constructing and executing a graph in a session and once with
@@ -992,9 +1013,10 @@ def py_func_if_in_function(f):
     if not ops.get_default_graph()._building_function:
       return f(*args, **kwds)
 
-    tensor_args, tensor_indices = zip(
-        *[(x, i) for i, x in enumerate(args)
-          if isinstance(x, (ops.Tensor, variables.Variable))])
+    tensor_args, tensor_indices = zip(*[(x, i)
+                                        for i, x in enumerate(args)
+                                        if isinstance(x, (ops.Tensor,
+                                                          variables.Variable))])
 
     def inner_f(*inner_tensor_args):
       my_args = list(args)
@@ -1056,6 +1078,7 @@ def deprecated_graph_mode_only(func=None):
     func: function to be annotated. If `func` is None, this method returns a
       decorator the can be applied to a function. If `func` is not None this
       returns the decorator applied to `func`.
+
   Returns:
     Returns a decorator that will run the decorated test method in graph mode.
   """
@@ -1390,8 +1413,7 @@ class FakeEagerSession(object):
 
 
 class ErrorLoggingSession(session.Session):
-  """Wrapper around a Session that logs errors in run().
-  """
+  """Wrapper around a Session that logs errors in run()."""
 
   def run(self, *args, **kwargs):
     try:
@@ -1405,13 +1427,56 @@ class ErrorLoggingSession(session.Session):
       raise
 
 
+# The description is just for documentation purposes.
+def disable_xla(description):
+
+  def disable_xla_impl(func):
+    """Execute the test method only if xla is not enabled."""
+
+    def decorator(func):
+
+      def decorated(self, *args, **kwargs):
+        if is_xla_enabled():
+          return
+        else:
+          return func(self, *args, **kwargs)
+
+      return decorated
+
+    if func is not None:
+      return decorator(func)
+
+    return decorator
+
+  return disable_xla_impl
+
+
+# The description is just for documentation purposes.
+def disable_all_xla(description):
+
+  def disable_all_impl(cls):
+    """Execute all test methods in this class only if xla is not enabled."""
+    base_decorator = disable_xla
+    for name in dir(cls):
+      value = getattr(cls, name)
+      if callable(value) and name.startswith(
+          "test") and not name == "test_session":
+        setattr(cls, name, base_decorator(value))
+    return cls
+
+  return disable_all_impl
+
+
 @tf_export("test.TestCase")
 class TensorFlowTestCase(googletest.TestCase):
-  """Base class for tests that need to test TensorFlow.
-  """
+  """Base class for tests that need to test TensorFlow."""
 
   def __init__(self, methodName="runTest"):  # pylint: disable=invalid-name
     super(TensorFlowTestCase, self).__init__(methodName)
+    if is_xla_enabled():
+      os.putenv(
+          "TF_XLA_FLAGS", "--tf_xla_auto_jit=2 --tf_xla_min_cluster_size=1 "
+          "--tf_xla_enable_lazy_compilation=false")
     self._threads = []
     self._tempdir = None
     self._cached_session = None
@@ -1489,9 +1554,9 @@ class TensorFlowTestCase(googletest.TestCase):
     ```
 
     Args:
-      stream: The stream whose writes should be captured. This
-        stream must have a file descriptor, support writing via using that
-        file descriptor, and must have a `.flush()` method.
+      stream: The stream whose writes should be captured. This stream must have
+        a file descriptor, support writing via using that file descriptor, and
+        must have a `.flush()` method.
 
     Yields:
       A `CapturedWrites` object that contains all writes to the specified stream
@@ -1840,7 +1905,6 @@ class TensorFlowTestCase(googletest.TestCase):
     self._threads.append(ret)
     return ret
 
-
   # pylint: enable=invalid-name
   @py_func_if_in_function
   def assertNear(self, f1, f2, err, msg=None):
@@ -1857,9 +1921,8 @@ class TensorFlowTestCase(googletest.TestCase):
     """
     # f1 == f2 is needed here as we might have: f1, f2 = inf, inf
     self.assertTrue(
-        f1 == f2 or math.fabs(f1 - f2) <= err,
-        "%f != %f +/- %f%s" % (f1, f2, err, " (%s)" % msg
-                               if msg is not None else ""))
+        f1 == f2 or math.fabs(f1 - f2) <= err, "%f != %f +/- %f%s" %
+        (f1, f2, err, " (%s)" % msg if msg is not None else ""))
 
   @py_func_if_in_function
   def assertArrayNear(self, farray1, farray2, err, msg=None):
@@ -2028,11 +2091,11 @@ class TensorFlowTestCase(googletest.TestCase):
 
     Args:
       a: The expected numpy `ndarray`, or anything that can be converted into a
-         numpy `ndarray` (including Tensor), or any arbitrarily nested of
-         structure of these.
+        numpy `ndarray` (including Tensor), or any arbitrarily nested of
+        structure of these.
       b: The actual numpy `ndarray`, or anything that can be converted into a
-         numpy `ndarray` (including Tensor), or any arbitrarily nested of
-         structure of these.
+        numpy `ndarray` (including Tensor), or any arbitrarily nested of
+        structure of these.
       rtol: relative tolerance.
       atol: absolute tolerance.
       msg: Optional message to report on failure.
@@ -2160,8 +2223,8 @@ class TensorFlowTestCase(googletest.TestCase):
     """Assert element values are all greater than a target value.
 
     Args:
-      a: The numpy `ndarray`, or anything that can be converted into a
-         numpy `ndarray` (including Tensor).
+      a: The numpy `ndarray`, or anything that can be converted into a numpy
+        `ndarray` (including Tensor).
       comparison_target: The target value of comparison.
     """
     a = self._GetNdArray(a)
@@ -2172,8 +2235,8 @@ class TensorFlowTestCase(googletest.TestCase):
     """Assert element values are all less than a target value.
 
     Args:
-      a: The numpy `ndarray`, or anything that can be converted into a
-         numpy `ndarray` (including Tensor).
+      a: The numpy `ndarray`, or anything that can be converted into a numpy
+        `ndarray` (including Tensor).
       comparison_target: The target value of comparison.
     """
     a = self._GetNdArray(a)
@@ -2184,8 +2247,8 @@ class TensorFlowTestCase(googletest.TestCase):
     """Assert element values are all greater than or equal to a target value.
 
     Args:
-      a: The numpy `ndarray`, or anything that can be converted into a
-         numpy `ndarray` (including Tensor).
+      a: The numpy `ndarray`, or anything that can be converted into a numpy
+        `ndarray` (including Tensor).
       comparison_target: The target value of comparison.
     """
     a = self._GetNdArray(a)
@@ -2196,8 +2259,8 @@ class TensorFlowTestCase(googletest.TestCase):
     """Assert element values are all less than or equal to a target value.
 
     Args:
-      a: The numpy `ndarray`, or anything that can be converted into a
-         numpy `ndarray` (including Tensor).
+      a: The numpy `ndarray`, or anything that can be converted into a numpy
+        `ndarray` (including Tensor).
       comparison_target: The target value of comparison.
     """
     a = self._GetNdArray(a)
@@ -2245,7 +2308,7 @@ class TensorFlowTestCase(googletest.TestCase):
 
     Args:
       target: The numpy `ndarray`, or anything that can be converted into a
-         numpy `ndarray` (including Tensor).
+        numpy `ndarray` (including Tensor).
       lower_bound: lower bound of the range
       upper_bound: upper bound of the range
       open_lower_bound: (`bool`) whether the lower bound is open (i.e., > rather
@@ -2279,8 +2342,8 @@ class TensorFlowTestCase(googletest.TestCase):
                  str(upper_bound) + (")" if open_upper_bound else "]"))
 
     violations = (
-        np.less_equal(target, lower_bound)
-        if open_lower_bound else np.less(target, lower_bound))
+        np.less_equal(target, lower_bound) if open_lower_bound else np.less(
+            target, lower_bound))
     violations = np.logical_or(
         violations,
         np.greater_equal(target, upper_bound)
@@ -2299,7 +2362,7 @@ class TensorFlowTestCase(googletest.TestCase):
 
     Args:
       target: The numpy `ndarray`, or anything that can be converted into a
-         numpy `ndarray` (including Tensor).
+        numpy `ndarray` (including Tensor).
       expected_set: (`list`, `tuple` or `set`) The closed set that the elements
         of the value of `target` are expected to fall into.
 
@@ -2321,7 +2384,7 @@ class TensorFlowTestCase(googletest.TestCase):
 
     Args:
       target: The numpy `ndarray`, or anything that can be converted into a
-         numpy `ndarray` (including Tensor).
+        numpy `ndarray` (including Tensor).
       expected_dtype: Expected data type.
     """
     target = self._GetNdArray(target)
@@ -2342,9 +2405,9 @@ class TensorFlowTestCase(googletest.TestCase):
     Args:
       exception_type: The expected type of exception that should be raised.
       expected_err_re_or_predicate: If this is callable, it should be a function
-        of one argument that inspects the passed-in exception and
-        returns True (success) or False (please fail the test). Otherwise, the
-        error message is expected to match this regular expression partially.
+        of one argument that inspects the passed-in exception and returns True
+        (success) or False (please fail the test). Otherwise, the error message
+        is expected to match this regular expression partially.
 
     Returns:
       A context manager to surround code that is expected to raise an
@@ -2445,6 +2508,7 @@ class TensorFlowTestCase(googletest.TestCase):
 
   def _create_session(self, graph, config, force_gpu):
     """See session() for details."""
+
     def prepare_config(config):
       """Returns a config for sessions.
 
@@ -2547,10 +2611,10 @@ def create_local_cluster(num_workers,
   Args:
     num_workers: Number of worker servers to start.
     num_ps: Number of PS servers to start.
-    protocol: Communication protocol.  Allowed values are documented in
-      the documentation of `tf.train.Server`.
-    worker_config: (optional) ConfigProto to initialize workers. Can be used
-      to instantiate multiple devices etc.
+    protocol: Communication protocol.  Allowed values are documented in the
+      documentation of `tf.train.Server`.
+    worker_config: (optional) ConfigProto to initialize workers. Can be used to
+      instantiate multiple devices etc.
     ps_config: (optional) ConfigProto to initialize PS servers.
 
   Returns:
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 276342cbbc..d9ea42374d 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -131,6 +131,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
     ],
     grpc_enabled = True,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -145,6 +146,7 @@ cuda_py_test(
         "//tensorflow/python:platform_benchmark",
     ],
     tags = ["no_windows"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -161,6 +163,7 @@ cuda_py_test(
         "//tensorflow/python:platform",
         "//tensorflow/python:platform_benchmark",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 tf_py_test(
@@ -226,6 +229,7 @@ cuda_py_test(
     ],
     shard_count = 5,
     tags = ["no_windows_gpu"],
+    xla_enable_strict_auto_jit = True,
 )
 
 tf_py_test(
@@ -282,6 +286,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:gradients",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 tf_py_test(
@@ -387,6 +392,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:linalg_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 tf_py_test(
@@ -645,6 +651,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
     ],
     tags = ["optonly"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -659,6 +666,7 @@ cuda_py_test(
         "//tensorflow/python:linalg_ops",
         "//tensorflow/python:math_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -671,6 +679,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:linalg_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -684,6 +693,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:linalg_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -695,6 +705,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:linalg_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -713,6 +724,7 @@ cuda_py_test(
         "//tensorflow/python:platform",
         "//tensorflow/python:random_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 tf_py_test(
@@ -795,6 +807,7 @@ cuda_py_test(
         "//tensorflow/python:resource_variable_ops",
         "//tensorflow/python:variables",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 tf_py_test(
@@ -853,6 +866,7 @@ cuda_py_test(
         "//tensorflow/python:resource_variable_ops",
     ],
     tags = ["noasan"],  # http://b/32635055
+    xla_enable_strict_auto_jit = True,
 )
 
 tf_py_test(
@@ -1159,6 +1173,7 @@ cuda_py_test(
         "//tensorflow/python:nn_grad",
         "//tensorflow/python:nn_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1174,6 +1189,7 @@ cuda_py_test(
         "//tensorflow/python:nn_grad",
         "//tensorflow/python:nn_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 tf_py_test(
@@ -1290,6 +1306,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1311,6 +1328,7 @@ cuda_py_test(
         "noguitar",
         "notap",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1326,6 +1344,7 @@ cuda_py_test(
         "//tensorflow/python:variables",
     ],
     tags = ["notsan"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1342,6 +1361,7 @@ cuda_py_test(
     ],
     shard_count = 2,
     tags = ["no_windows_gpu"],
+    xla_enable_strict_auto_jit = True,
 )
 
 tf_py_test(
@@ -1374,6 +1394,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:math_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1385,6 +1406,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:math_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1413,6 +1435,7 @@ cuda_py_test(
         "noasan",  # times out
         "optonly",  # times out
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1425,6 +1448,7 @@ cuda_py_test(
         "//tensorflow/python:client",
         "//tensorflow/python:client_testlib",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1441,6 +1465,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
     ],
     shard_count = 10,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1455,6 +1480,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
     ],
     shard_count = 20,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1468,6 +1494,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1482,6 +1509,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1497,6 +1525,7 @@ cuda_py_test(
         "//tensorflow/python:nn_grad",
         "//tensorflow/python:nn_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1509,6 +1538,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1527,6 +1557,7 @@ cuda_py_test(
         "//tensorflow/python:framework",
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1542,6 +1573,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:util",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1560,6 +1592,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:util",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1600,6 +1633,7 @@ cuda_py_test(
     ],
     shard_count = 16,
     tags = ["no_gpu"],  # TODO(b/117928656)
+    xla_enable_strict_auto_jit = True,
 )
 
 tf_py_test(
@@ -1640,6 +1674,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:nn_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1654,6 +1689,7 @@ cuda_py_test(
         "//tensorflow/python:nn_grad",
         "//tensorflow/python:nn_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1668,6 +1704,7 @@ cuda_py_test(
         "//tensorflow/python:nn_grad",
         "//tensorflow/python:nn_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1679,6 +1716,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:math_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1692,6 +1730,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:platform",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1707,6 +1746,7 @@ cuda_py_test(
         "//tensorflow/python:state_ops",
         "//tensorflow/python:variables",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1721,6 +1761,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
     ],
     tags = ["no_windows_gpu"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1733,6 +1774,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
     tags = ["manual"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1748,6 +1790,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:gradients",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1762,6 +1805,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:gradients",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1774,6 +1818,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1786,6 +1831,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1812,6 +1858,7 @@ cuda_py_test(
     grpc_enabled = True,
     shard_count = 2,
     tags = ["no_windows"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1827,6 +1874,7 @@ cuda_py_test(
         "//tensorflow/python:gradients",
         "//tensorflow/python:variables",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1841,6 +1889,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:gradients",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1854,6 +1903,7 @@ cuda_py_test(
         "//tensorflow/python:gradients",
         "//tensorflow/python:math_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1881,6 +1931,7 @@ cuda_py_test(
         "noasan",
         "notap",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1898,6 +1949,7 @@ cuda_py_test(
         "//tensorflow/python/ops/linalg",
     ],
     tags = ["no_windows_gpu"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1913,6 +1965,7 @@ cuda_py_test(
         "//tensorflow/python:nn",
         "//tensorflow/python:nn_grad",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1929,6 +1982,7 @@ cuda_py_test(
         "//tensorflow/python:platform",
         "//tensorflow/python/ops/linalg",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1942,6 +1996,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
     tags = ["no_windows_gpu"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1959,6 +2014,7 @@ cuda_py_test(
         "//tensorflow/python:variables",
     ],
     shard_count = 20,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1972,6 +2028,7 @@ cuda_py_test(
         "//tensorflow/python:nn_grad",
         "//tensorflow/python:nn_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1987,6 +2044,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:numerics",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2000,6 +2058,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
     tags = ["no_windows_gpu"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2014,6 +2073,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:variables",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2026,6 +2086,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2040,6 +2101,7 @@ cuda_py_test(
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2058,6 +2120,7 @@ cuda_py_test(
         "//tensorflow/python/eager:function",
     ],
     tags = ["no_windows"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2071,6 +2134,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:string_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2086,6 +2150,7 @@ cuda_py_test(
     ],
     shard_count = 4,
     tags = ["no_windows_gpu"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2106,6 +2171,7 @@ cuda_py_test(
         "noguitar",
         "notap",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2124,6 +2190,7 @@ cuda_py_test(
         "//tensorflow/python/eager:backprop",
         "//tensorflow/python:tf2",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2136,6 +2203,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2148,6 +2216,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2160,6 +2229,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2178,6 +2248,7 @@ cuda_py_test(
         "//tensorflow/python:random_ops",
         "//tensorflow/python:sparse_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2191,6 +2262,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:math_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2203,6 +2275,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:session_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2219,6 +2292,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:gradients",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2233,6 +2307,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:nn_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2246,6 +2321,7 @@ cuda_py_test(
         "//tensorflow/python:nn_grad",
         "//tensorflow/python:nn_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2259,6 +2335,7 @@ cuda_py_test(
         "//tensorflow/python:nn_grad",
         "//tensorflow/python:nn_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2274,6 +2351,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:math_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2288,6 +2366,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
     ],
     tags = ["no_windows_gpu"],
+    xla_enable_strict_auto_jit = True,
 )
 
 tf_py_test(
@@ -2332,6 +2411,7 @@ cuda_py_test(
         "//tensorflow/python:sparse_grad",
         "//tensorflow/python:sparse_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2356,6 +2436,7 @@ cuda_py_test(
         "//tensorflow/python:random_ops",
         "//tensorflow/python:variables",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2370,6 +2451,7 @@ cuda_py_test(
         "//tensorflow/python:gradients",
         "//tensorflow/python:math_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2385,6 +2467,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:math_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2397,6 +2480,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:string_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2409,6 +2493,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:parsing_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2422,6 +2507,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:summary",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2437,6 +2523,7 @@ cuda_py_test(
         "//tensorflow/python:nn_grad",
         "//tensorflow/python:summary",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2467,6 +2554,7 @@ cuda_py_test(
         "//tensorflow/python/eager:context",
     ],
     flaky = 1,  # create_local_cluster sometimes times out.
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2479,6 +2567,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
     ],
     tags = ["no_windows_gpu"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2497,6 +2586,7 @@ cuda_py_test(
         "no_oss",
         "optonly",  # times out
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2509,6 +2599,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2526,6 +2617,7 @@ cuda_py_test(
         "//tensorflow/python:state_ops_gen",
         "//tensorflow/python:variables",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2542,6 +2634,7 @@ cuda_py_test(
         "//tensorflow/python:nn_ops",
         "//tensorflow/python:nn_ops_gen",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2553,6 +2646,7 @@ cuda_py_test(
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2572,6 +2666,7 @@ cuda_py_test(
     tags = [
         "no_gpu",  #  Flaky: b/80127739
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2587,6 +2682,7 @@ cuda_py_test(
         "//tensorflow/python:nn_ops",
     ],
     tags = ["manual"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2600,6 +2696,7 @@ cuda_py_test(
         "//tensorflow/python:nn_grad",
         "//tensorflow/python:nn_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2618,6 +2715,7 @@ cuda_py_test(
     tags = [
         "optonly",  # flaky timeouts unless optimized
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2631,6 +2729,7 @@ cuda_py_test(
         "//tensorflow/python:nn_grad",
         "//tensorflow/python:nn_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2659,6 +2758,7 @@ cuda_py_test(
         "no_oss",
         "optonly",  # times out
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2676,6 +2776,7 @@ cuda_py_test(
     ],
     # TODO(b/118842098): Re-enable this test in Kokoro.
     tags = ["no_oss"],
+    xla_enable_strict_auto_jit = True,
 )
 
 tf_py_test(
@@ -2704,6 +2805,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
     tags = ["manual"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2717,6 +2819,7 @@ cuda_py_test(
         "//tensorflow/python:nn_grad",
         "//tensorflow/python:nn_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2735,6 +2838,7 @@ cuda_py_test(
         "//tensorflow/python:nn_ops_gen",
     ],
     shard_count = 4,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2764,6 +2868,7 @@ cuda_py_test(
         "//tensorflow/python/eager:context",
     ],
     shard_count = 10,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2779,6 +2884,7 @@ cuda_py_test(
     ],
     shard_count = 2,
     tags = ["optonly"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2793,6 +2899,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:gradients",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2809,6 +2916,7 @@ cuda_py_test(
     tags = [
         "no_oss",  # Requires 4GB+ RAM
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2822,6 +2930,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
     ],
     tags = ["no_windows"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2845,6 +2954,7 @@ cuda_py_test(
         "noasan",
         "optonly",  # b/77589990
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2864,6 +2974,7 @@ cuda_py_test(
         "//tensorflow/python:platform",
         "//tensorflow/python:sparse_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 # TODO(gpapan): Revisit the gradient of extract_image_patches_op to resolve
@@ -2880,6 +2991,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
     tags = ["notap"],  # http://b/31080670
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2894,6 +3006,7 @@ cuda_py_test(
         "//tensorflow/python:util",
         "//tensorflow/python:data_flow_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2908,6 +3021,7 @@ cuda_py_test(
         "//tensorflow/python:util",
         "//tensorflow/python:data_flow_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2925,6 +3039,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:variables",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2940,6 +3055,7 @@ cuda_py_test(
         "nomsan",
         "notsan",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2953,6 +3069,7 @@ cuda_py_test(
         "//tensorflow/python:nn_ops",
     ],
     shard_count = 30,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2973,6 +3090,7 @@ cuda_py_test(
         "//tensorflow/python:variables",
     ],
     shard_count = 50,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2993,6 +3111,7 @@ cuda_py_test(
         "//tensorflow/python:variables",
     ],
     shard_count = 50,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -3013,6 +3132,7 @@ cuda_py_test(
         "//tensorflow/python:variables",
     ],
     shard_count = 50,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -3038,6 +3158,7 @@ cuda_py_test(
         "//tensorflow/python:variables",
     ],
     shard_count = 20,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -3055,6 +3176,7 @@ cuda_py_test(
         "//tensorflow/python/ops/linalg",
     ],
     shard_count = 20,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -3068,6 +3190,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
     shard_count = 20,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -3085,6 +3208,7 @@ cuda_py_test(
     data = ["//tensorflow/python/kernel_tests/testdata:self_adjoint_eig_op_test_files"],
     shard_count = 20,
     tags = ["no_windows"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -3100,6 +3224,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
     ],
     shard_count = 20,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -3119,6 +3244,7 @@ cuda_py_test(
         "no_oss",  # b/117185141.
         "nomsan",  # TODO(b/117236102): Re-enable in msan build.
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -3138,6 +3264,7 @@ cuda_py_test(
         "no_windows_gpu",
         "nomsan",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -3153,6 +3280,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
     ],
     shard_count = 20,
+    xla_enable_strict_auto_jit = True,
 )
 
 sycl_py_test(
@@ -3399,6 +3527,7 @@ cuda_py_test(
         "//tensorflow/python:while_v2",
     ],
     grpc_enabled = True,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -3424,4 +3553,5 @@ cuda_py_test(
         "//tensorflow/python:while_v2",
     ],
     grpc_enabled = True,
+    xla_enable_strict_auto_jit = True,
 )
diff --git a/tensorflow/python/kernel_tests/atrous_conv2d_test.py b/tensorflow/python/kernel_tests/atrous_conv2d_test.py
index a13e325835..5d7b2dd30f 100644
--- a/tensorflow/python/kernel_tests/atrous_conv2d_test.py
+++ b/tensorflow/python/kernel_tests/atrous_conv2d_test.py
@@ -139,6 +139,7 @@ class AtrousConv2DTest(test.TestCase):
                   y1.eval(), self.evaluate(y2), rtol=1e-2, atol=1e-2)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")  # larger error range
   def testGradient(self):
     with self.session(use_gpu=True):
       # Input: [batch, height, width, input_depth]
diff --git a/tensorflow/python/kernel_tests/benchmark_test.py b/tensorflow/python/kernel_tests/benchmark_test.py
index bffa5e6e8f..a91f96cf95 100644
--- a/tensorflow/python/kernel_tests/benchmark_test.py
+++ b/tensorflow/python/kernel_tests/benchmark_test.py
@@ -26,6 +26,7 @@ import numpy as np
 from tensorflow.core.util import test_log_pb2
 from tensorflow.python.client import session
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import benchmark
 from tensorflow.python.platform import gfile
@@ -125,6 +126,7 @@ class BenchmarkTest(test.TestCase):
     self.assertFalse(_ran_somebenchmark_2[0])
     self.assertFalse(_ran_somebenchmark_but_shouldnt[0])
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testReportingBenchmark(self):
     tempdir = test.get_temp_dir()
     try:
diff --git a/tensorflow/python/kernel_tests/check_ops_test.py b/tensorflow/python/kernel_tests/check_ops_test.py
index 95bac85027..d5f3696a9d 100644
--- a/tensorflow/python/kernel_tests/check_ops_test.py
+++ b/tensorflow/python/kernel_tests/check_ops_test.py
@@ -889,6 +889,8 @@ class EnsureShapeTest(test.TestCase):
 
   # Dynamic shape check
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA"
+                        )  # Dynamic shapes not supported now with XLA
   def testEnsuresDynamicShape_RaisesError(self):
     placeholder = array_ops.placeholder(dtypes.int32)
     derived = math_ops.divide(placeholder, 3, name="MyDivide")
@@ -902,6 +904,8 @@ class EnsureShapeTest(test.TestCase):
         sess.run(derived, feed_dict={placeholder: feed_val})
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA"
+                        )  # Dynamic shapes not supported now with XLA
   def testEnsuresDynamicShape_RaisesErrorDimUnknown(self):
     placeholder = array_ops.placeholder(dtypes.int32)
     derived = placeholder / 3
diff --git a/tensorflow/python/kernel_tests/cholesky_op_test.py b/tensorflow/python/kernel_tests/cholesky_op_test.py
index a08cfe960d..abb71a672c 100644
--- a/tensorflow/python/kernel_tests/cholesky_op_test.py
+++ b/tensorflow/python/kernel_tests/cholesky_op_test.py
@@ -163,6 +163,7 @@ class CholeskyOpTest(test.TestCase):
     with self.assertRaises(ValueError):
       linalg_ops.cholesky(tensor3)
 
+  @test_util.disable_xla("This test never passed for XLA")  # all nan on XLA
   def testNotInvertibleCPU(self):
     # The input should be invertible.
     with self.session(use_gpu=True):
diff --git a/tensorflow/python/kernel_tests/concat_op_test.py b/tensorflow/python/kernel_tests/concat_op_test.py
index 474760a93f..a968b06127 100644
--- a/tensorflow/python/kernel_tests/concat_op_test.py
+++ b/tensorflow/python/kernel_tests/concat_op_test.py
@@ -33,6 +33,7 @@ from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 
 
+@test_util.disable_all_xla("This test never passed for XLA")
 class ConcatOpTest(test.TestCase):
 
   @test_util.run_deprecated_v1
@@ -640,6 +641,8 @@ class ConcatOpTest(test.TestCase):
         output = self.evaluate(c)
         self.assertAllEqual([[1, 2, 3, 7, 8, 9], [4, 5, 6, 10, 11, 12]], output)
 
+
+@test_util.disable_all_xla("This test never passed for XLA")
 class ConcatOffsetTest(test.TestCase):
 
   def testBasic(self):
@@ -683,6 +686,8 @@ class ConcatOffsetTest(test.TestCase):
       self.evaluate(off)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla(
+      "This test never passed for XLA")  # Different error message on XLA
   def testSizeMismatch(self):
     cdim = constant_op.constant(1, dtypes.int32)
     s0 = constant_op.constant([2, 3, 5], dtypes.int32)
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 25977510fb..870dd8285e 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -2380,6 +2380,8 @@ class ControlFlowTest(test.TestCase):
       self.assertEqual(i_val, 3)
       self.assertAllClose(x_val, 1.0)
 
+  @test_util.disable_xla("This test never passed for XLA"
+                        )  # Resource variable issue for ControlFlowV2
   @test_util.run_gpu_only
   def testGpuResourceAccess(self):
     with ops.device(test.gpu_device_name()):
diff --git a/tensorflow/python/kernel_tests/depthtospace_op_test.py b/tensorflow/python/kernel_tests/depthtospace_op_test.py
index 96c9b5258e..97d3645b61 100644
--- a/tensorflow/python/kernel_tests/depthtospace_op_test.py
+++ b/tensorflow/python/kernel_tests/depthtospace_op_test.py
@@ -295,6 +295,7 @@ class DepthToSpaceTest(test.TestCase):
       actual_vals, expected_vals = self.evaluate([actual, expected])
       self.assertTrue(np.array_equal(actual_vals, expected_vals))
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testAgainstTranspose(self):
     self.compareToTranspose(3, 2, 3, 1, 2, "NHWC", False)
     self.compareToTranspose(3, 2, 3, 2, 2, "NHWC", False)
diff --git a/tensorflow/python/kernel_tests/depthwise_conv_op_test.py b/tensorflow/python/kernel_tests/depthwise_conv_op_test.py
index 5b1a47fb03..6f6d17b4eb 100644
--- a/tensorflow/python/kernel_tests/depthwise_conv_op_test.py
+++ b/tensorflow/python/kernel_tests/depthwise_conv_op_test.py
@@ -481,6 +481,7 @@ class DepthwiseConv2DTest(test.TestCase):
             data_format="NCHW")
 
   @test_util.run_v1_only("b/120545219")
+  @test_util.disable_xla("This test never passed for XLA")
   def testDepthwiseConv2DFilterGrad(self):
     for index, (input_size, filter_size, output_size, stride,
                 padding) in enumerate(CheckGradConfigsToTest()):
@@ -612,6 +613,7 @@ class DepthwiseConv2DTest(test.TestCase):
     cpu_value = _GetVal(use_gpu=False)
     self.assertAllClose(cpu_value, gpu_value, rtol=1e-4, atol=1e-4)
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testDepthwiseConv2DFilterGradCompare(self):
     for index, (input_size, filter_size, output_size, stride,
                 padding) in enumerate(ConfigsToTest()):
diff --git a/tensorflow/python/kernel_tests/diag_op_test.py b/tensorflow/python/kernel_tests/diag_op_test.py
index ed2a9e8e47..b8139918c5 100644
--- a/tensorflow/python/kernel_tests/diag_op_test.py
+++ b/tensorflow/python/kernel_tests/diag_op_test.py
@@ -65,6 +65,7 @@ class MatrixDiagTest(test.TestCase):
       array_ops.matrix_diag(0)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testInvalidShapeAtEval(self):
     with self.session(use_gpu=True):
       v = array_ops.placeholder(dtype=dtypes_lib.float32)
@@ -269,6 +270,7 @@ class MatrixDiagPartTest(test.TestCase):
       array_ops.matrix_diag_part(0)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testInvalidShapeAtEval(self):
     with self.session(use_gpu=True):
       v = array_ops.placeholder(dtype=dtypes_lib.float32)
diff --git a/tensorflow/python/kernel_tests/distributions/BUILD b/tensorflow/python/kernel_tests/distributions/BUILD
index a83622ac9a..7a10ed9852 100644
--- a/tensorflow/python/kernel_tests/distributions/BUILD
+++ b/tensorflow/python/kernel_tests/distributions/BUILD
@@ -23,6 +23,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -42,6 +43,7 @@ cuda_py_test(
         "//tensorflow/python:platform_test",
     ],
     shard_count = 3,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -54,6 +56,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -71,6 +74,7 @@ cuda_py_test(
         "//tensorflow/python:nn_ops",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -86,6 +90,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -105,6 +110,7 @@ cuda_py_test(
         "//tensorflow/python:platform_test",
         "//tensorflow/python:random_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -119,6 +125,7 @@ cuda_py_test(
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -139,6 +146,7 @@ cuda_py_test(
         "noguitar",  # b/110489471
         "notap",  # b/110489471
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -154,6 +162,7 @@ cuda_py_test(
         "//tensorflow/python:nn_ops",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -169,6 +178,7 @@ cuda_py_test(
         "//tensorflow/python:nn_ops",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -184,6 +194,7 @@ cuda_py_test(
         "//tensorflow/python:nn_ops",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -200,6 +211,7 @@ cuda_py_test(
         "//tensorflow/python:platform_test",
     ],
     tags = ["manual"],  # b/69001419
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -224,6 +236,7 @@ cuda_py_test(
         # disable to avoid false positives from scipy.
         "nomsan",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -240,6 +253,7 @@ cuda_py_test(
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:math_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -258,6 +272,7 @@ cuda_py_test(
         "//tensorflow/python:platform_test",
         "//tensorflow/python:variables",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -274,6 +289,7 @@ cuda_py_test(
         "//tensorflow/python:platform_test",
         "//tensorflow/python:variables",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -291,4 +307,5 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
diff --git a/tensorflow/python/kernel_tests/functional_ops_test.py b/tensorflow/python/kernel_tests/functional_ops_test.py
index 8988305bde..92f4e7b39e 100644
--- a/tensorflow/python/kernel_tests/functional_ops_test.py
+++ b/tensorflow/python/kernel_tests/functional_ops_test.py
@@ -58,6 +58,7 @@ def simple_scoped_fn(a, x):
 
 
 @test_util.with_control_flow_v2
+@test_util.disable_all_xla("This test never passed for XLA")
 class FunctionalOpsTest(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
@@ -826,6 +827,8 @@ class FunctionalOpsTest(test.TestCase):
     self.assertAllEqual(Run(100., True), 5050.)
 
   @test_util.run_v1_only("b/120545219")
+  @test_util.disable_xla(
+      "This test never passed for XLA")  # Different error message
   def testWhileError(self):
     for use_gpu in (True, False):
       with ops.Graph().as_default() as g:
@@ -1102,6 +1105,7 @@ class FunctionalOpsTest(test.TestCase):
 
 # TODO(akshayka): Replace `function.Defun` with tf.contrib.eager.defun` in the
 # below test cases.
+@test_util.disable_all_xla("This test never passed for XLA")
 class PartitionedCallTest(test.TestCase):
 
   @test_util.run_deprecated_v1
diff --git a/tensorflow/python/kernel_tests/gather_nd_op_test.py b/tensorflow/python/kernel_tests/gather_nd_op_test.py
index 320ffc9674..76ae2fcb72 100644
--- a/tensorflow/python/kernel_tests/gather_nd_op_test.py
+++ b/tensorflow/python/kernel_tests/gather_nd_op_test.py
@@ -34,6 +34,7 @@ from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 
 
+@test_util.disable_all_xla("This test never passed for XLA")
 class GatherNdTest(test.TestCase):
 
   def _testSimpleDtype(self, dtype):
@@ -56,6 +57,7 @@ class GatherNdTest(test.TestCase):
     self._testSimpleDtype("|S")  # byte strings in python2 + 3
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testEmptyIndicesAndParamsOKButJustEmptyParamsFails(self):
     with self.session(use_gpu=True):
       params = np.ones((3, 3), dtype=np.float32)
@@ -358,6 +360,7 @@ class GatherNdTest(test.TestCase):
       self.assertAllEqual(expected_grads, ops.convert_to_tensor(grads).eval())
 
 
+@test_util.disable_all_xla("This test never passed for XLA")
 class GatherNdOpBenchmark(test.Benchmark):
 
   def benchmark_gather_nd_op(self):
diff --git a/tensorflow/python/kernel_tests/gather_op_test.py b/tensorflow/python/kernel_tests/gather_op_test.py
index 1d6f7955aa..3b8a9a38de 100644
--- a/tensorflow/python/kernel_tests/gather_op_test.py
+++ b/tensorflow/python/kernel_tests/gather_op_test.py
@@ -213,6 +213,8 @@ class GatherTest(test.TestCase, parameterized.TestCase):
         array_ops.gather(params, [[7]], axis=1).eval()
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla(
+      "This test never passed for XLA")  # Different error message.
   def testBadAxis(self):
     with self.session(use_gpu=True):
       params = [0, 1, 2]
diff --git a/tensorflow/python/kernel_tests/linalg/BUILD b/tensorflow/python/kernel_tests/linalg/BUILD
index ba9e64979a..c1aa99cc37 100644
--- a/tensorflow/python/kernel_tests/linalg/BUILD
+++ b/tensorflow/python/kernel_tests/linalg/BUILD
@@ -22,6 +22,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -38,6 +39,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -60,6 +62,7 @@ cuda_py_test(
         "noasan",  # times out, b/63678675
         "optonly",  # times out
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -76,6 +79,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -98,6 +102,7 @@ cuda_py_test(
         "noasan",
         "optonly",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -120,6 +125,7 @@ cuda_py_test(
         "noasan",  # times out, b/63678675
         "optonly",  # times out
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -144,6 +150,7 @@ cuda_py_test(
         "noasan",  # times out, b/63678675
         "optonly",  # times out, b/79171797
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -166,6 +173,7 @@ cuda_py_test(
         "noasan",
         "optonly",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -187,6 +195,7 @@ cuda_py_test(
         "noasan",
         "optonly",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -209,6 +218,7 @@ cuda_py_test(
         "noasan",  # times out, b/63678675
         "optonly",  # times out
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -229,6 +239,7 @@ cuda_py_test(
         "noasan",
         "optonly",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -251,6 +262,7 @@ cuda_py_test(
         "noasan",
         "optonly",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -270,6 +282,7 @@ cuda_py_test(
         "noasan",
         "optonly",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -290,6 +303,7 @@ cuda_py_test(
         "noasan",  # times out
         "optonly",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -310,6 +324,7 @@ cuda_py_test(
         "noasan",
         "optonly",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -328,4 +343,5 @@ cuda_py_test(
     ],
     shard_count = 5,
     tags = ["optonly"],  # Test is flaky without optimization.
+    xla_enable_strict_auto_jit = True,
 )
diff --git a/tensorflow/python/kernel_tests/pool_test.py b/tensorflow/python/kernel_tests/pool_test.py
index 78e786f01c..367c94dd1e 100644
--- a/tensorflow/python/kernel_tests/pool_test.py
+++ b/tensorflow/python/kernel_tests/pool_test.py
@@ -303,6 +303,7 @@ class PoolingTest(test.TestCase):
     self.assertLess(err, err_tolerance)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")  # Much larger error
   def testGradient1D(self):
     with self.session(use_gpu=test.is_gpu_available()):
       for padding in ["SAME", "VALID"]:
diff --git a/tensorflow/python/kernel_tests/pooling_ops_test.py b/tensorflow/python/kernel_tests/pooling_ops_test.py
index c33b59bb99..27a7190434 100644
--- a/tensorflow/python/kernel_tests/pooling_ops_test.py
+++ b/tensorflow/python/kernel_tests/pooling_ops_test.py
@@ -730,6 +730,7 @@ class PoolingTest(test.TestCase):
         t = nn_ops.max_pool(
             t, ksize=ksize, strides=strides, padding="SAME").eval()
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testDepthwiseMaxPoolInvalidConfigs(self):
     self._testDepthwiseMaxPoolInvalidConfig(
         [1, 2, 2, 4], [1, 2, 2, 2], [1, 1, 1, 2],
@@ -1174,6 +1175,7 @@ class PoolingTest(test.TestCase):
           use_gpu=use_gpu)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testMaxPoolGrad(self):
     for (data_format, use_gpu) in GetTestConfigs():
       self._testMaxPoolGradValidPadding1_1(data_format, use_gpu)
@@ -1210,6 +1212,7 @@ class PoolingTest(test.TestCase):
                      [1, window_rows, window_cols, 1],
                      [1, row_stride, col_stride, 1], padding)
 
+  @test_util.disable_xla("This test never passed for XLA")
   def _testMaxPoolGradDirect(self, input_data, output_backprop,
                              expected_input_backprop, input_sizes, output_sizes,
                              window_rows, window_cols, row_stride, col_stride,
@@ -1625,6 +1628,7 @@ class PoolingTest(test.TestCase):
           use_gpu=use_gpu)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testMaxPoolGradGrad(self):
     for (data_format, use_gpu) in GetTestConfigs():
       self._testMaxPoolGradGradValidPadding1_1(data_format, use_gpu)
@@ -1659,6 +1663,7 @@ class PoolingTest(test.TestCase):
         [1, row_stride, col_stride, 1], padding)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testAvgPoolGrad(self):
     for (data_format, use_gpu) in GetTestConfigs():
       self._testAvgPoolGradValidPadding1_1(data_format, use_gpu)
@@ -1818,6 +1823,7 @@ class PoolingTest(test.TestCase):
             padding="SAME")
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testOpEdgeCases(self):
     with self.session(use_gpu=test.is_gpu_available()) as sess:
       pool_funcs = [nn_ops.max_pool, nn_ops.avg_pool]
@@ -1893,9 +1899,17 @@ if __name__ == "__main__":
        padding_) in GetShrunkInceptionMaxPoolShapes():
     setattr(PoolingTest, "testMaxPoolFwd_" + name_,
             GetMaxPoolFwdTest(input_size_, filter_size_, stride_, padding_))
-    setattr(PoolingTest, "testMaxPoolGrad_" + name_,
-            GetMaxPoolGradTest(input_size_, filter_size_, output_size_, stride_,
-                               padding_))
+    if name_ == "maxpool5":
+      setattr(
+          PoolingTest, "testMaxPoolGrad_" + name_,
+          test_util.disable_xla("maxpool5 fails while all others pass")(
+              GetMaxPoolGradTest(input_size_, filter_size_, output_size_,
+                                 stride_, padding_)))
+    else:
+      setattr(
+          PoolingTest, "testMaxPoolGrad_" + name_,
+          GetMaxPoolGradTest(input_size_, filter_size_, output_size_, stride_,
+                             padding_))
     setattr(PoolingTest, "testMaxPoolGradGrad_" + name_,
             GetMaxPoolGradGradTest(input_size_, filter_size_, output_size_,
                                    stride_, padding_))
diff --git a/tensorflow/python/kernel_tests/random/BUILD b/tensorflow/python/kernel_tests/random/BUILD
index dd81306db0..445aaf9dc4 100644
--- a/tensorflow/python/kernel_tests/random/BUILD
+++ b/tensorflow/python/kernel_tests/random/BUILD
@@ -45,6 +45,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:random_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -64,6 +65,7 @@ cuda_py_test(
         "//tensorflow/python:random_ops",
     ],
     shard_count = 3,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -75,6 +77,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:random_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -88,6 +91,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:random_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -103,6 +107,7 @@ cuda_py_test(
         "//tensorflow/python:random_ops",
         "//tensorflow/python:stateless_random_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -120,6 +125,7 @@ cuda_py_test(
     ],
     shard_count = 4,
     tags = ["nozapfhahn"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -137,6 +143,7 @@ cuda_py_test(
         "//tensorflow/python:random_grad",
         "//tensorflow/python:random_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -152,4 +159,5 @@ cuda_py_test(
         "//tensorflow/python:platform",
         "//tensorflow/python:random_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
diff --git a/tensorflow/python/kernel_tests/random/random_ops_test.py b/tensorflow/python/kernel_tests/random/random_ops_test.py
index 1384c3f446..68672a04bb 100644
--- a/tensorflow/python/kernel_tests/random/random_ops_test.py
+++ b/tensorflow/python/kernel_tests/random/random_ops_test.py
@@ -257,6 +257,7 @@ class TruncatedNormalTest(test.TestCase):
       self.assertAllEqual(rnd1, rnd2)
 
 
+@test_util.disable_all_xla("This never passed on XLA")
 class RandomUniformTest(RandomOpTestCommon):
 
   def _Sampler(self, num, minv, maxv, dtype, use_gpu, seed=None):
diff --git a/tensorflow/python/kernel_tests/relu_op_test.py b/tensorflow/python/kernel_tests/relu_op_test.py
index d4ba1ad77d..3b8924904c 100644
--- a/tensorflow/python/kernel_tests/relu_op_test.py
+++ b/tensorflow/python/kernel_tests/relu_op_test.py
@@ -86,6 +86,7 @@ class ReluTest(test.TestCase):
     self.assertAllClose(np_relu, tf_relu)
     self.assertShapeEqual(np_relu, tf_relu)
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testReluInt8x4BadShape(self):
     if not test.is_gpu_available(cuda_only=True):
       self.skipTest("No GPU available")
diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index 2594055119..ec93038f1b 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -741,6 +741,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
 
   @test_util.run_in_graph_and_eager_modes
   @test_util.run_v1_only("b/120545219")
+  @test_util.disable_xla("This test never passed for XLA")
   def testDestroyResource(self):
     v = resource_variable_ops.ResourceVariable(3.0, name="var0")
     self.evaluate(variables.global_variables_initializer())
diff --git a/tensorflow/python/kernel_tests/scan_ops_test.py b/tensorflow/python/kernel_tests/scan_ops_test.py
index 33e491fee1..4e15894fb4 100644
--- a/tensorflow/python/kernel_tests/scan_ops_test.py
+++ b/tensorflow/python/kernel_tests/scan_ops_test.py
@@ -70,6 +70,7 @@ def handle_options(func, x, axis, exclusive, reverse):
   return x
 
 
+@test_util.disable_all_xla("This test never passed for XLA")
 class CumsumTest(test.TestCase):
 
   valid_dtypes = [
@@ -193,6 +194,7 @@ class CumsumTest(test.TestCase):
           self._compareGradient([5, 10], axis, exclusive, reverse)
 
 
+@test_util.disable_all_xla("This test never passed for XLA")
 class CumprodTest(test.TestCase):
 
   valid_dtypes = [
diff --git a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
index e1cd9e7622..c443c23b16 100644
--- a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
+++ b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
@@ -295,6 +295,7 @@ class StatefulScatterNdTest(test.TestCase):
                                     updates).get_shape().as_list(), shape)
 
   @test_util.run_v1_only("b/120545219")
+  @test_util.disable_xla("This test never passed for XLA")
   def testResVarInvalidOutputShape(self):
     res = variables.Variable(
         initial_value=lambda: array_ops.zeros(shape=[], dtype=dtypes.float32),
diff --git a/tensorflow/python/kernel_tests/signal/BUILD b/tensorflow/python/kernel_tests/signal/BUILD
index 8f4e31abe3..554bf38029 100644
--- a/tensorflow/python/kernel_tests/signal/BUILD
+++ b/tensorflow/python/kernel_tests/signal/BUILD
@@ -29,6 +29,7 @@ cuda_py_tests(
         "//tensorflow/python:spectral_ops_test_util",
         "//tensorflow/python/ops/signal",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_tests(
@@ -45,6 +46,7 @@ cuda_py_tests(
     ],
     shard_count = 4,
     tags = ["optonly"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_tests(
@@ -56,6 +58,7 @@ cuda_py_tests(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python/ops/signal",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_tests(
@@ -70,6 +73,7 @@ cuda_py_tests(
         "//tensorflow/python/ops/signal",
         "//tensorflow/python:spectral_ops_test_util",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_tests(
@@ -87,6 +91,7 @@ cuda_py_tests(
         "//tensorflow/python/ops/signal",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_tests(
@@ -104,6 +109,7 @@ cuda_py_tests(
         "//tensorflow/python/ops/signal",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_tests(
@@ -125,6 +131,7 @@ cuda_py_tests(
         "//tensorflow/python/ops/signal",
     ],
     tags = ["nomac"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_tests(
@@ -140,4 +147,5 @@ cuda_py_tests(
         "//tensorflow/python/ops/signal",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
diff --git a/tensorflow/python/kernel_tests/signal/fft_ops_test.py b/tensorflow/python/kernel_tests/signal/fft_ops_test.py
index 5b1053428c..d98a568491 100644
--- a/tensorflow/python/kernel_tests/signal/fft_ops_test.py
+++ b/tensorflow/python/kernel_tests/signal/fft_ops_test.py
@@ -465,6 +465,7 @@ class RFFTOpsTest(BaseFFTOpsTest):
                 gen_complex(complex_dims), rank, (size,) * rank)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testError(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       for rank in VALID_FFT_RANKS:
diff --git a/tensorflow/python/kernel_tests/spacetodepth_op_test.py b/tensorflow/python/kernel_tests/spacetodepth_op_test.py
index e96bc09f36..7f3c381fa1 100644
--- a/tensorflow/python/kernel_tests/spacetodepth_op_test.py
+++ b/tensorflow/python/kernel_tests/spacetodepth_op_test.py
@@ -285,6 +285,7 @@ class SpaceToDepthTest(test.TestCase):
       actual_vals, expected_vals = self.evaluate([actual, expected])
       self.assertTrue(np.array_equal(actual_vals, expected_vals))
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testAgainstTranspose(self):
     self.compareToTranspose(3, 2, 3, 1, 2, "NHWC", False)
     self.compareToTranspose(1, 2, 3, 2, 2, "NHWC", False)
diff --git a/tensorflow/python/kernel_tests/split_op_test.py b/tensorflow/python/kernel_tests/split_op_test.py
index 517db3450f..80004db833 100644
--- a/tensorflow/python/kernel_tests/split_op_test.py
+++ b/tensorflow/python/kernel_tests/split_op_test.py
@@ -373,6 +373,7 @@ class SplitOpTest(test.TestCase):
     assert s1.shape.as_list() == [1]
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testNonexistentDimTensor(self):
     x = array_ops.placeholder(dtypes.int32)
     values = np.zeros([5, 30])
diff --git a/tensorflow/python/ops/bitwise_ops_test.py b/tensorflow/python/ops/bitwise_ops_test.py
index d154b6759b..c182874c7f 100644
--- a/tensorflow/python/ops/bitwise_ops_test.py
+++ b/tensorflow/python/ops/bitwise_ops_test.py
@@ -35,6 +35,7 @@ class BitwiseOpTest(test_util.TensorFlowTestCase):
     super(BitwiseOpTest, self).__init__(method_name)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testBinaryOps(self):
     dtype_list = [dtypes.int8, dtypes.int16, dtypes.int32, dtypes.int64,
                   dtypes.uint8, dtypes.uint16, dtypes.uint32, dtypes.uint64]
@@ -72,6 +73,7 @@ class BitwiseOpTest(test_util.TensorFlowTestCase):
         self.assertAllEqual(truth, popcnt_result)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testInvertOp(self):
     dtype_list = [dtypes.int8, dtypes.int16, dtypes.int32, dtypes.int64,
                   dtypes.uint8, dtypes.uint16, dtypes.uint32, dtypes.uint64]
@@ -97,6 +99,7 @@ class BitwiseOpTest(test_util.TensorFlowTestCase):
           self.assertAllEqual(inverted, expected)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testShiftsWithPositiveLHS(self):
     dtype_list = [np.int8, np.int16, np.int32, np.int64,
                   np.uint8, np.uint16, np.uint32, np.uint64]
diff --git a/tensorflow/python/ops/nn_batchnorm_test.py b/tensorflow/python/ops/nn_batchnorm_test.py
index e978f1d326..fedf8e44c3 100644
--- a/tensorflow/python/ops/nn_batchnorm_test.py
+++ b/tensorflow/python/ops/nn_batchnorm_test.py
@@ -206,6 +206,7 @@ class BatchNormalizationTest(test.TestCase):
                                   2)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testBatchNormGradImpl(self):
     x_shape = [7, 5, 4, 6]
     param_shape = [6]
diff --git a/tensorflow/python/ops/nn_fused_batchnorm_test.py b/tensorflow/python/ops/nn_fused_batchnorm_test.py
index 4bc33ff8bd..4e5cff5c16 100644
--- a/tensorflow/python/ops/nn_fused_batchnorm_test.py
+++ b/tensorflow/python/ops/nn_fused_batchnorm_test.py
@@ -523,6 +523,7 @@ class BatchNormalizationTest(test.TestCase):
             data_format='NHWC',
             is_training=is_training)
 
+  @test_util.disable_xla('This test never passed for XLA')
   def testBatchNormGradShape5(self):
     for is_training in [True, False]:
       x_shape = [0, 7, 11, 4]
diff --git a/tensorflow/python/ops/nn_test.py b/tensorflow/python/ops/nn_test.py
index 82fab74183..74561349ed 100644
--- a/tensorflow/python/ops/nn_test.py
+++ b/tensorflow/python/ops/nn_test.py
@@ -41,6 +41,7 @@ from tensorflow.python.ops.nn_impl import _compute_sampled_logits
 from tensorflow.python.platform import test as test_lib
 
 
+@test_util.disable_all_xla("This test never passed for XLA")
 class ZeroFractionTest(test_lib.TestCase):
 
   def _ZeroFraction(self, x):
@@ -1017,6 +1018,7 @@ class LeakyReluTest(test_lib.TestCase):
 class SwishTest(test_lib.TestCase):
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testValues(self):
     np_values = np.array(
         [np.linspace(-10.0, 0.0, 100),
diff --git a/tensorflow/python/ops/parallel_for/BUILD b/tensorflow/python/ops/parallel_for/BUILD
index 1028963b1a..f54ca558f4 100644
--- a/tensorflow/python/ops/parallel_for/BUILD
+++ b/tensorflow/python/ops/parallel_for/BUILD
@@ -114,6 +114,7 @@ cuda_py_test(
         "//tensorflow/python:random_ops",
         "//tensorflow/python:util",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -171,4 +172,5 @@ cuda_py_test(
         "//tensorflow/python/ops/losses",
     ],
     tags = ["optonly"],  # Too slow in non-opt mode
+    xla_enable_strict_auto_jit = True,
 )
diff --git a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
index 1a8f639d43..8a5830e28f 100644
--- a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
+++ b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
@@ -251,6 +251,7 @@ class NNTest(PForTestCase):
 
     self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 3)
 
+  @test_util.disable_xla("This test never passed for XLA")
   def test_fused_batch_norm(self):
     data_formats = ["NHWC"]
     if test.is_gpu_available():
diff --git a/tensorflow/python/ops/parallel_for/gradients_test.py b/tensorflow/python/ops/parallel_for/gradients_test.py
index 133e790992..66110f12d9 100644
--- a/tensorflow/python/ops/parallel_for/gradients_test.py
+++ b/tensorflow/python/ops/parallel_for/gradients_test.py
@@ -482,6 +482,7 @@ class GradientsTest(test.TestCase):
     pfor_jacobian, while_jacobian = create_lstm_batch_jacobian(8, 4, 2)
     self.run_and_assert_equal(pfor_jacobian, while_jacobian)
 
+  @test_util.disable_xla("This test never passed for XLA")
   def test_dynamic_lstm_batch_jacobian(self):
     pfor_jacobian, while_gradients = create_dynamic_lstm_batch_jacobian(8, 4, 3)
     with session.Session() as sess:
diff --git a/tensorflow/python/ops/special_math_ops_test.py b/tensorflow/python/ops/special_math_ops_test.py
index 94aaebed95..0224e7ebb8 100644
--- a/tensorflow/python/ops/special_math_ops_test.py
+++ b/tensorflow/python/ops/special_math_ops_test.py
@@ -119,6 +119,7 @@ class LBetaTest(test.TestCase):
           special_math_ops.lbeta(x).get_shape())
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.disable_xla('This test never passed for XLA')
   def test_length_1_last_dimension_results_in_one(self):
     # If there is only one coefficient, the formula still works, and we get one
     # as the answer, always.
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index b7da5bf913..37d52cdb39 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -1802,8 +1802,17 @@ def tf_py_test(
         additional_visibility = [],
         kernels = [],
         flaky = 0,
+        xla_enable_strict_auto_jit = False,
         xla_enabled = False,
         grpc_enabled = False):
+    """Create one or more python tests with extra tensorflow dependencies."""
+    xla_test_true_list = []
+
+    # xla_enable_strict_auto_jit is used to run Tensorflow unit tests with all XLA compilable
+    # kernels compiled with XLA.
+    if xla_enable_strict_auto_jit:
+        xla_enabled = True
+        xla_test_true_list += ["//tensorflow/python:is_xla_test_true"]
     if xla_enabled:
         additional_deps = additional_deps + tf_additional_xla_deps_py()
     if grpc_enabled:
@@ -1824,7 +1833,7 @@ def tf_py_test(
         deps = [
             clean_dep("//tensorflow/python:extra_py_tests_deps"),
             clean_dep("//tensorflow/python:gradient_checker"),
-        ] + additional_deps,
+        ] + additional_deps + xla_test_true_list,
     )
 
 register_extension_info(
@@ -1844,8 +1853,12 @@ def cuda_py_test(
         kernels = [],
         tags = [],
         flaky = 0,
+        xla_enable_strict_auto_jit = False,
         xla_enabled = False,
         grpc_enabled = False):
+    # TODO(b/122522101): Don't ignore xla_enable_strict_auto_jit and enable additional
+    # XLA tests once enough compute resources are available.
+    _ignored = [xla_enable_strict_auto_jit]
     if main == None:
         main = name + ".py"
     for config in ["cpu", "gpu"]:
@@ -1868,6 +1881,7 @@ def cuda_py_test(
             shard_count = shard_count,
             tags = test_tags,
             xla_enabled = xla_enabled,
+            xla_enable_strict_auto_jit = False,
         )
 
 register_extension_info(
@@ -1921,6 +1935,7 @@ def py_tests(
         tags = [],
         shard_count = 1,
         prefix = "",
+        xla_enable_strict_auto_jit = False,
         xla_enabled = False,
         grpc_enabled = False):
     for src in srcs:
@@ -1939,6 +1954,7 @@ def py_tests(
             shard_count = shard_count,
             tags = tags,
             xla_enabled = xla_enabled,
+            xla_enable_strict_auto_jit = xla_enable_strict_auto_jit,
         )
 
 def cuda_py_tests(
@@ -1951,8 +1967,12 @@ def cuda_py_tests(
         shard_count = 1,
         tags = [],
         prefix = "",
+        xla_enable_strict_auto_jit = False,
         xla_enabled = False,
         grpc_enabled = False):
+    # TODO(b/122522101): Don't ignore xla_enable_strict_auto_jit and enable additional
+    # XLA tests once enough compute resources are available.
+    _ignored = [xla_enable_strict_auto_jit]
     test_tags = tags + tf_cuda_tests_tags()
     py_tests(
         name = name,
@@ -1966,6 +1986,7 @@ def cuda_py_tests(
         shard_count = shard_count,
         tags = test_tags,
         xla_enabled = xla_enabled,
+        xla_enable_strict_auto_jit = False,
     )
 
 # Creates a genrule named <name> for running tools/proto_text's generator to
-- 
GitLab


From 267263629fd8d612719f61991eb2d2e3cd3b2b01 Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Mon, 14 Jan 2019 02:42:51 -0800
Subject: [PATCH 0634/2345] Change so that tf.random_poisson is renamed to the
 v2 API instead of to compat

PiperOrigin-RevId: 229150058
---
 tensorflow/tools/compatibility/renames_v2.py         | 1 -
 tensorflow/tools/compatibility/tf_upgrade_v2.py      | 2 ++
 tensorflow/tools/compatibility/tf_upgrade_v2_test.py | 9 +++++++++
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index b757699f63..f143a260b9 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -478,7 +478,6 @@ renames = {
     'tf.random_crop': 'tf.image.random_crop',
     'tf.random_gamma': 'tf.random.gamma',
     'tf.random_normal': 'tf.random.normal',
-    'tf.random_poisson': 'tf.compat.v1.random_poisson',
     'tf.random_shuffle': 'tf.random.shuffle',
     'tf.random_uniform': 'tf.random.uniform',
     'tf.read_file': 'tf.io.read_file',
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 2c56cbf36b..994d7f9b7e 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -601,6 +601,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.compat.v1.initializers.truncated_normal",
         "tf.image.resize_images":
             "tf.image.resize",
+        "tf.random_poisson":
+            "tf.random.poisson",
     }
     # pylint: enable=line-too-long
 
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 0dde5045aa..caf63416d8 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -539,6 +539,15 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
         )
     self.assertEqual(new_text, expected_text)
 
+  def testRandomPoissonConversion(self):
+    text1 = "tf.random_poisson(lam, shape, dtype)"
+    text2 = "tf.random.poisson(lam, shape, dtype)"
+    expected_text = "tf.random.poisson(lam=lam, shape=shape, dtype=dtype)"
+    _, unused_report, unused_errors, new_text1 = self._upgrade(text1)
+    self.assertEqual(new_text1, expected_text)
+    _, unused_report, unused_errors, new_text2 = self._upgrade(text2)
+    self.assertEqual(new_text2, expected_text)
+
   def testConvolutionOpUpdate(self):
     text = (
         "tf.nn.convolution(input, filter, padding, strides, dilation_rate, "
-- 
GitLab


From 74a6cca5d867d37e79ec9d780f2c57b926f07a80 Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Mon, 14 Jan 2019 03:04:27 -0800
Subject: [PATCH 0635/2345] Removed a linear scan in dtypes.as_dtype

PiperOrigin-RevId: 229152423
---
 tensorflow/python/framework/dtypes.py | 98 +++++++++++++--------------
 1 file changed, 47 insertions(+), 51 deletions(-)

diff --git a/tensorflow/python/framework/dtypes.py b/tensorflow/python/framework/dtypes.py
index 9a4fe4e93b..9386cca379 100644
--- a/tensorflow/python/framework/dtypes.py
+++ b/tensorflow/python/framework/dtypes.py
@@ -535,29 +535,31 @@ _np_qint32 = np.dtype([("qint32", np.int32, 1)])
 np_resource = np.dtype([("resource", np.ubyte, 1)])
 
 # Standard mappings between types_pb2.DataType values and numpy.dtypes.
-_NP_TO_TF = frozenset([
-    (np.float16, float16),
-    (np.float32, float32),
-    (np.float64, float64),
-    (np.int32, int32),
-    (np.int64, int64),
-    (np.uint8, uint8),
-    (np.uint16, uint16),
-    (np.uint32, uint32),
-    (np.uint64, uint64),
-    (np.int16, int16),
-    (np.int8, int8),
-    (np.complex64, complex64),
-    (np.complex128, complex128),
-    (np.object_, string),
-    (np.bool_, bool),
-    (_np_qint8, qint8),
-    (_np_quint8, quint8),
-    (_np_qint16, qint16),
-    (_np_quint16, quint16),
-    (_np_qint32, qint32),
-    (_np_bfloat16, bfloat16),
-])
+_NP_TO_TF = {
+    np.float16: float16,
+    np.float32: float32,
+    np.float64: float64,
+    np.int32: int32,
+    np.int64: int64,
+    np.uint8: uint8,
+    np.uint16: uint16,
+    np.uint32: uint32,
+    np.uint64: uint64,
+    np.int16: int16,
+    np.int8: int8,
+    np.complex64: complex64,
+    np.complex128: complex128,
+    np.object_: string,
+    np.string_: string,
+    np.unicode_: string,
+    np.bool_: bool,
+    _np_qint8: qint8,
+    _np_quint8: quint8,
+    _np_qint16: qint16,
+    _np_quint16: quint16,
+    _np_qint32: qint32,
+    _np_bfloat16: bfloat16,
+}
 _TF_TO_NP = {
     types_pb2.DT_HALF:
         np.float16,
@@ -664,6 +666,20 @@ _PYTHON_TO_TF = {
     builtins.object: string
 }
 
+_ANY_TO_TF = {}
+_ANY_TO_TF.update(_INTERN_TABLE)
+_ANY_TO_TF.update(_STRING_TO_TF)
+_ANY_TO_TF.update(_PYTHON_TO_TF)
+_ANY_TO_TF.update(_NP_TO_TF)
+
+# Ensure no collisions.
+assert len(_ANY_TO_TF) == sum(len(d) for d in [
+    _INTERN_TABLE,
+    _STRING_TO_TF,
+    _PYTHON_TO_TF,
+    _NP_TO_TF
+])
+
 
 @tf_export("dtypes.as_dtype", "as_dtype")
 def as_dtype(type_value):
@@ -684,36 +700,16 @@ def as_dtype(type_value):
   if isinstance(type_value, DType):
     return type_value
 
-  try:
-    return _INTERN_TABLE[type_value]
-  except KeyError:
-    pass
-
-  try:
-    return _STRING_TO_TF[type_value]
-  except KeyError:
-    pass
+  if isinstance(type_value, np.dtype):
+    try:
+      return _NP_TO_TF[type_value.type]
+    except KeyError:
+      pass
 
   try:
-    return _PYTHON_TO_TF[type_value]
+    return _ANY_TO_TF[type_value]
   except KeyError:
     pass
 
-  if isinstance(type_value, np.dtype):
-    # The numpy dtype for strings is variable length. We can not compare
-    # dtype with a single constant (np.string does not exist) to decide
-    # dtype is a "string" type. We need to compare the dtype.type to be
-    # sure it's a string type.
-    if type_value.type == np.string_ or type_value.type == np.unicode_:
-      return string
-
-  if isinstance(type_value, (type, np.dtype)):
-    for key, val in _NP_TO_TF:
-      try:
-        if key == type_value:
-          return val
-      except TypeError as e:
-        raise TypeError("Cannot convert {} to a dtype. {}".format(
-            type_value, e))
-
-  raise TypeError("Cannot convert value %r to a TensorFlow DType." % type_value)
+  raise TypeError(
+      "Cannot convert value %r to a TensorFlow DType." % type_value)
-- 
GitLab


From 8f593c48c84a2d52d2ba8becf2eaef20250325a0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 03:07:27 -0800
Subject: [PATCH 0636/2345] Correctly read tensor values.

PiperOrigin-RevId: 229152770
---
 tensorflow/python/framework/tensor_util.py    | 95 ++++++-------------
 .../python/framework/tensor_util_test.py      | 30 +++---
 2 files changed, 43 insertions(+), 82 deletions(-)

diff --git a/tensorflow/python/framework/tensor_util.py b/tensorflow/python/framework/tensor_util.py
index 51f71616a1..ca8b067935 100644
--- a/tensorflow/python/framework/tensor_util.py
+++ b/tensorflow/python/framework/tensor_util.py
@@ -598,88 +598,53 @@ def MakeNdarray(tensor):
   dtype = tensor_dtype.as_numpy_dtype
 
   if tensor.tensor_content:
-    return (np.frombuffer(tensor.tensor_content, dtype=dtype).copy()
-            .reshape(shape))
-  elif tensor_dtype == dtypes.float16 or tensor_dtype == dtypes.bfloat16:
+    return (np.frombuffer(tensor.tensor_content,
+                          dtype=dtype).copy().reshape(shape))
+
+  if tensor_dtype == dtypes.string:
+    # np.pad throws on these arrays of type np.object.
+    values = list(tensor.string_val)
+    padding = num_elements - len(values)
+    if padding > 0:
+      last = values[-1] if values else ""
+      values.extend([last] * padding)
+    return np.array(values, dtype=dtype).reshape(shape)
+
+  if tensor_dtype == dtypes.float16 or tensor_dtype == dtypes.bfloat16:
     # the half_val field of the TensorProto stores the binary representation
     # of the fp16: we need to reinterpret this as a proper float16
-    if len(tensor.half_val) == 1:
-      tmp = np.array(tensor.half_val[0], dtype=np.uint16)
-      tmp.dtype = tensor_dtype.as_numpy_dtype
-      return np.repeat(tmp, num_elements).reshape(shape)
-    else:
-      tmp = np.fromiter(tensor.half_val, dtype=np.uint16)
-      tmp.dtype = tensor_dtype.as_numpy_dtype
-      return tmp.reshape(shape)
+    values = np.fromiter(tensor.half_val, dtype=np.uint16)
+    values.dtype = tensor_dtype.as_numpy_dtype
   elif tensor_dtype == dtypes.float32:
-    if len(tensor.float_val) == 1:
-      return np.repeat(
-          np.array(tensor.float_val[0], dtype=dtype),
-          num_elements).reshape(shape)
-    else:
-      return np.fromiter(tensor.float_val, dtype=dtype).reshape(shape)
+    values = np.fromiter(tensor.float_val, dtype=dtype)
   elif tensor_dtype == dtypes.float64:
-    if len(tensor.double_val) == 1:
-      return np.repeat(
-          np.array(tensor.double_val[0], dtype=dtype),
-          num_elements).reshape(shape)
-    else:
-      return np.fromiter(tensor.double_val, dtype=dtype).reshape(shape)
+    values = np.fromiter(tensor.double_val, dtype=dtype)
   elif tensor_dtype in [
       dtypes.int32, dtypes.uint8, dtypes.uint16, dtypes.int16, dtypes.int8,
       dtypes.qint32, dtypes.quint8, dtypes.qint8, dtypes.qint16, dtypes.quint16
   ]:
-    if len(tensor.int_val) == 1:
-      return np.repeat(np.array(tensor.int_val[0], dtype=dtype),
-                       num_elements).reshape(shape)
-    else:
-      return np.fromiter(tensor.int_val, dtype=dtype).reshape(shape)
+    values = np.fromiter(tensor.int_val, dtype=dtype)
   elif tensor_dtype == dtypes.int64:
-    if len(tensor.int64_val) == 1:
-      return np.repeat(
-          np.array(tensor.int64_val[0], dtype=dtype),
-          num_elements).reshape(shape)
-    else:
-      return np.fromiter(tensor.int64_val, dtype=dtype).reshape(shape)
-  elif tensor_dtype == dtypes.string:
-    if len(tensor.string_val) == 1:
-      return np.repeat(
-          np.array(tensor.string_val[0], dtype=dtype),
-          num_elements).reshape(shape)
-    else:
-      return np.array(
-          [x for x in tensor.string_val], dtype=dtype).reshape(shape)
+    values = np.fromiter(tensor.int64_val, dtype=dtype)
   elif tensor_dtype == dtypes.complex64:
     it = iter(tensor.scomplex_val)
-    if len(tensor.scomplex_val) == 2:
-      return np.repeat(
-          np.array(
-              complex(tensor.scomplex_val[0], tensor.scomplex_val[1]),
-              dtype=dtype), num_elements).reshape(shape)
-    else:
-      return np.array(
-          [complex(x[0], x[1]) for x in zip(it, it)],
-          dtype=dtype).reshape(shape)
+    values = np.array([complex(x[0], x[1]) for x in zip(it, it)], dtype=dtype)
   elif tensor_dtype == dtypes.complex128:
     it = iter(tensor.dcomplex_val)
-    if len(tensor.dcomplex_val) == 2:
-      return np.repeat(
-          np.array(
-              complex(tensor.dcomplex_val[0], tensor.dcomplex_val[1]),
-              dtype=dtype), num_elements).reshape(shape)
-    else:
-      return np.array(
-          [complex(x[0], x[1]) for x in zip(it, it)],
-          dtype=dtype).reshape(shape)
+    values = np.array([complex(x[0], x[1]) for x in zip(it, it)], dtype=dtype)
   elif tensor_dtype == dtypes.bool:
-    if len(tensor.bool_val) == 1:
-      return np.repeat(np.array(tensor.bool_val[0], dtype=dtype),
-                       num_elements).reshape(shape)
-    else:
-      return np.fromiter(tensor.bool_val, dtype=dtype).reshape(shape)
+    values = np.fromiter(tensor.bool_val, dtype=dtype)
   else:
     raise TypeError("Unsupported tensor type: %s" % tensor.dtype)
 
+  if values.size == 0:
+    return np.zeros(shape, dtype)
+
+  if values.size != num_elements:
+    values = np.pad(values, (0, num_elements - values.size), "edge")
+
+  return values.reshape(shape)
+
 
 def ShapeEquals(tensor_proto, shape):
   """Returns True if "tensor_proto" has the given "shape".
diff --git a/tensorflow/python/framework/tensor_util_test.py b/tensorflow/python/framework/tensor_util_test.py
index 0033754618..cdacdfaaad 100644
--- a/tensorflow/python/framework/tensor_util_test.py
+++ b/tensorflow/python/framework/tensor_util_test.py
@@ -336,23 +336,16 @@ class TensorUtilTest(test.TestCase):
       self.assertAllClose(np.array([10, 20, 30], dtype=nptype), a)
 
   def testIntTypesWithImplicitRepeat(self):
-    for dtype, nptype in [(dtypes.int64, np.int64),
-                          (dtypes.int32, np.int32),
-                          (dtypes.uint8, np.uint8),
-                          (dtypes.uint16, np.uint16),
-                          (dtypes.int16, np.int16),
-                          (dtypes.int8, np.int8)]:
+    for dtype, nptype in [(dtypes.int64, np.int64), (dtypes.int32, np.int32),
+                          (dtypes.uint8, np.uint8), (dtypes.uint16, np.uint16),
+                          (dtypes.int16, np.int16), (dtypes.int8, np.int8)]:
       self.assertAllEqual(
-          np.array(
-              [[10, 10, 10, 10],
-               [10, 10, 10, 10],
-               [10, 10, 10, 10]],
-              dtype=nptype),
+          np.array([[10, 11, 12, 12], [12, 12, 12, 12], [12, 12, 12, 12]],
+                   dtype=nptype),
           tensor_util.MakeNdarray(
-              tensor_util.make_tensor_proto(
-                  [10],
-                  shape=[3, 4],
-                  dtype=dtype)))
+              tensor_util.make_tensor_proto([10, 11, 12],
+                                            shape=[3, 4],
+                                            dtype=dtype)))
 
   def testIntMixedWithDimension(self):
     # Github issue: 11974
@@ -500,9 +493,12 @@ class TensorUtilTest(test.TestCase):
     self.assertEquals([b"foo"], a)
 
   def testStringWithImplicitRepeat(self):
-    t = tensor_util.make_tensor_proto("f", shape=[3, 4])
+    t = tensor_util.make_tensor_proto(["f", "g"], shape=[3, 4])
     a = tensor_util.MakeNdarray(t)
-    self.assertAllEqual(np.array([[b"f"] * 4] * 3, dtype=np.object), a)
+    self.assertAllEqual(
+        np.array([[b"f", b"g", b"g", b"g"], [b"g", b"g", b"g", b"g"],
+                  [b"g", b"g", b"g", b"g"]],
+                 dtype=np.object), a)
 
   def testStringN(self):
     t = tensor_util.make_tensor_proto([b"foo", b"bar", b"baz"], shape=[1, 3])
-- 
GitLab


From ac2f7471b9f6ba0316d865b65c11f86ee939e68a Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Mon, 14 Jan 2019 03:22:00 -0800
Subject: [PATCH 0637/2345] Fix assert_* conversions, ensure emit warnings and
 change to compat.v1

PiperOrigin-RevId: 229154096
---
 .../tools/compatibility/tf_upgrade_v2.py      | 76 +++++++++++++++++++
 .../tools/compatibility/tf_upgrade_v2_test.py | 39 +++++++++-
 2 files changed, 112 insertions(+), 3 deletions(-)

diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 994d7f9b7e..fd71103cc6 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -603,6 +603,48 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.image.resize",
         "tf.random_poisson":
             "tf.random.poisson",
+        "tf.debugging.assert_greater":
+            "tf.compat.v1.debugging.assert_greater",
+        "tf.debugging.assert_greater_equal":
+            "tf.compat.v1.debugging.assert_greater_equal",
+        "tf.debugging.assert_integer":
+            "tf.compat.v1.debugging.assert_integer",
+        "tf.debugging.assert_less":
+            "tf.compat.v1.debugging.assert_less",
+        "tf.debugging.assert_less_equal":
+            "tf.compat.v1.debugging.assert_less_equal",
+        "tf.debugging.assert_near":
+            "tf.compat.v1.debugging.assert_near",
+        "tf.debugging.assert_negative":
+            "tf.compat.v1.debugging.assert_negative",
+        "tf.debugging.assert_non_negative":
+            "tf.compat.v1.debugging.assert_non_negative",
+        "tf.debugging.assert_non_positive":
+            "tf.compat.v1.debugging.assert_non_positive",
+        "tf.debugging.assert_none_equal":
+            "tf.compat.v1.debugging.assert_none_equal",
+        "tf.debugging.assert_type":
+            "tf.compat.v1.debugging.assert_type",
+        "tf.debugging.assert_positive":
+            "tf.compat.v1.debugging.assert_positive",
+        "tf.debugging.assert_equal":
+            "tf.compat.v1.debugging.assert_equal",
+        "tf.debugging.assert_scalar":
+            "tf.compat.v1.debugging.assert_scalar",
+        "tf.assert_equal":
+            "tf.compat.v1.assert_equal",
+        "tf.assert_less":
+            "tf.compat.v1.assert_less",
+        "tf.assert_greater":
+            "tf.compat.v1.assert_greater",
+        "tf.debugging.assert_rank":
+            "tf.compat.v1.debugging.assert_rank",
+        "tf.debugging.assert_rank_at_least":
+            "tf.compat.v1.debugging.assert_rank_at_least",
+        "tf.debugging.assert_rank_in":
+            "tf.compat.v1.debugging.assert_rank_in",
+        "tf.assert_rank":
+            "tf.compat.v1.assert_rank",
     }
     # pylint: enable=line-too-long
 
@@ -851,10 +893,40 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             assert_return_type_comment,
         "tf.assert_equal":
             assert_return_type_comment,
+        "tf.assert_none_equal":
+            assert_return_type_comment,
+        "tf.assert_less":
+            assert_return_type_comment,
+        "tf.assert_negative":
+            assert_return_type_comment,
+        "tf.assert_positive":
+            assert_return_type_comment,
+        "tf.assert_non_negative":
+            assert_return_type_comment,
+        "tf.assert_non_positive":
+            assert_return_type_comment,
+        "tf.assert_near":
+            assert_return_type_comment,
         "tf.assert_less":
             assert_return_type_comment,
+        "tf.assert_less_equal":
+            assert_return_type_comment,
+        "tf.assert_greater":
+            assert_return_type_comment,
+        "tf.assert_greater_equal":
+            assert_return_type_comment,
+        "tf.assert_integer":
+            assert_return_type_comment,
+        "tf.assert_type":
+            assert_return_type_comment,
+        "tf.assert_scalar":
+            assert_return_type_comment,
         "tf.assert_rank":
             assert_rank_comment,
+        "tf.assert_rank_at_least":
+            assert_rank_comment,
+        "tf.assert_rank_in":
+            assert_rank_comment,
         "tf.debugging.assert_equal":
             assert_return_type_comment,
         "tf.debugging.assert_greater":
@@ -879,6 +951,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             assert_return_type_comment,
         "tf.debugging.assert_positive":
             assert_return_type_comment,
+        "tf.debugging.assert_type":
+            assert_return_type_comment,
+        "tf.debugging.assert_scalar":
+            assert_return_type_comment,
         "tf.debugging.assert_rank":
             assert_rank_comment,
         "tf.debugging.assert_rank_at_least":
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index caf63416d8..0adee6a566 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -114,12 +114,12 @@ class TestUpgrade(test_util.TensorFlowTestCase):
     self.assertTrue(report.find("Failed to parse") != -1)
 
   def testReport(self):
-    text = "tf.assert_near(a)\n"
+    text = "tf.angle(a)\n"
     _, report, unused_errors, unused_new_text = self._upgrade(text)
     # This is not a complete test, but it is a sanity test that a report
     # is generating information.
-    self.assertTrue(report.find("Renamed function `tf.assert_near` to "
-                                "`tf.debugging.assert_near`"))
+    self.assertTrue(report.find("Renamed function `tf.angle` to "
+                                "`tf.math.angle`"))
 
   def testRename(self):
     text = "tf.conj(a)\n"
@@ -937,6 +937,39 @@ def _log_prob(self, x):
     _, unused_report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual(expected_text, new_text)
 
+  def testAssertStatements(self):
+    for name in ["assert_greater", "assert_equal", "assert_none_equal",
+                 "assert_less", "assert_negative", "assert_positive",
+                 "assert_non_negative", "assert_non_positive", "assert_near",
+                 "assert_less", "assert_less_equal", "assert_greater",
+                 "assert_greater_equal", "assert_integer", "assert_type",
+                 "assert_scalar"]:
+      text = "tf.%s(a)" % name
+      expected_text = "tf.compat.v1.%s(a)" % name
+      _, unused_report, errors, new_text = self._upgrade(text)
+      self.assertEqual(expected_text, new_text)
+      self.assertIn("assert_* functions", errors[0])
+
+      text = "tf.debugging.%s(a)" % name
+      expected_text = "tf.compat.v1.debugging.%s(a)" % name
+      _, unused_report, errors, new_text = self._upgrade(text)
+      self.assertEqual(expected_text, new_text)
+      self.assertIn("assert_* functions", errors[0])
+
+  def testAssertRankStatements(self):
+    for name in ["assert_rank", "assert_rank_at_least", "assert_rank_in"]:
+      text = "tf.%s(a)" % name
+      expected_text = "tf.compat.v1.%s(a)" % name
+      _, unused_report, errors, new_text = self._upgrade(text)
+      self.assertEqual(expected_text, new_text)
+      self.assertIn("assert_rank_* functions", errors[0])
+
+      text = "tf.debugging.%s(a)" % name
+      expected_text = "tf.compat.v1.debugging.%s(a)" % name
+      _, unused_report, errors, new_text = self._upgrade(text)
+      self.assertEqual(expected_text, new_text)
+      self.assertIn("assert_rank_* functions", errors[0])
+
 
 class TestUpgradeFiles(test_util.TensorFlowTestCase):
 
-- 
GitLab


From 3ed46c325f70e2f1521b850b43d2b174101d9472 Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Mon, 14 Jan 2019 04:06:24 -0800
Subject: [PATCH 0638/2345] Add conversion for count_nonzero outside of package

PiperOrigin-RevId: 229158255
---
 tensorflow/python/ops/math_ops.py                  |  2 ++
 tensorflow/tools/compatibility/renames_v2.py       |  1 -
 tensorflow/tools/compatibility/tf_upgrade_v2.py    |  7 +++++++
 .../tools/compatibility/tf_upgrade_v2_test.py      | 14 +++++++++++++-
 4 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index 248d092538..2ae29d0eb9 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -1339,6 +1339,8 @@ def reduce_sum(input_tensor, axis=None, keepdims=False, name=None):
 @tf_export(v1=["math.count_nonzero", "count_nonzero"])
 @deprecation.deprecated_args(
     None, "keep_dims is deprecated, use keepdims instead", "keep_dims")
+@deprecation.deprecated_args(
+    None, "reduction_indices is deprecated, use axis instead", "axis")
 def count_nonzero(input_tensor,
                   axis=None,
                   keepdims=None,
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index f143a260b9..fb2303e9dd 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -121,7 +121,6 @@ renames = {
     'tf.container': 'tf.compat.v1.container',
     'tf.convert_to_tensor_or_indexed_slices': 'tf.compat.v1.convert_to_tensor_or_indexed_slices',
     'tf.convert_to_tensor_or_sparse_tensor': 'tf.compat.v1.convert_to_tensor_or_sparse_tensor',
-    'tf.count_nonzero': 'tf.compat.v1.count_nonzero',
     'tf.count_up_to': 'tf.compat.v1.count_up_to',
     'tf.create_partitioned_variables': 'tf.compat.v1.create_partitioned_variables',
     'tf.cross': 'tf.linalg.cross',
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index fd71103cc6..3f9032bb33 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -119,6 +119,11 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.load_file_system_library": {
             "library_filename": "library_location",
         },
+        "tf.count_nonzero": {
+            "input_tensor": "input",
+            "keep_dims": "keepdims",
+            "reduction_indices": "axis",
+        },
         "tf.math.count_nonzero": {
             "input_tensor": "input",
             "keep_dims": "keepdims",
@@ -511,6 +516,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.sort",
         "tf.contrib.framework.argsort":
             "tf.argsort",
+        "tf.count_nonzero":
+            "tf.math.count_nonzero",
         "tf.manip.batch_to_space_nd":
             "tf.batch_to_space",
         "tf.quantize_v2":
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 0adee6a566..23446e30b1 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -506,7 +506,7 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
         "tf.nn.dropout(x, 1 - (1 - func(3 + 4.)), name=\"foo\")\n",
     )
 
-  def testCountNonZeroChanges(self):
+  def testMathCountNonZeroChanges(self):
     text = (
         "tf.math.count_nonzero(input_tensor=input, dtype=dtype, name=name, "
         "reduction_indices=axis, keep_dims=keepdims)\n"
@@ -518,6 +518,18 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
         )
     self.assertEqual(new_text, expected_text)
 
+  def testCountNonZeroChanges(self):
+    text = (
+        "tf.count_nonzero(input_tensor=input, dtype=dtype, name=name, "
+        "reduction_indices=axis, keep_dims=keepdims)\n"
+        )
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    expected_text = (
+        "tf.math.count_nonzero(input=input, dtype=dtype, name=name, "
+        "axis=axis, keepdims=keepdims)\n"
+        )
+    self.assertEqual(new_text, expected_text)
+
   def testRandomMultinomialToRandomCategorical(self):
     text = (
         "tf.random.multinomial(logits, samples, seed, name, output_dtype)\n"
-- 
GitLab


From a1a8d2fd0c6a33cff524e14417d1a7d183f2c129 Mon Sep 17 00:00:00 2001
From: dmitrievanthony <dmitrievanthony@gmail.com>
Date: Mon, 14 Jan 2019 17:13:29 +0300
Subject: [PATCH 0639/2345] Fix Ignite Dataset (V2 API).

---
 tensorflow/contrib/ignite/README.md                        | 4 +++-
 tensorflow/contrib/ignite/python/ops/ignite_dataset_ops.py | 4 ++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/ignite/README.md b/tensorflow/contrib/ignite/README.md
index 5a8c650fb9..47f93115c4 100644
--- a/tensorflow/contrib/ignite/README.md
+++ b/tensorflow/contrib/ignite/README.md
@@ -97,6 +97,7 @@ jdbc:ignite:thin://localhost/> INSERT INTO KITTEN_CACHE VALUES (3, 'LITTLE BALL
 ```python
 >>> import tensorflow as tf
 >>> from tensorflow.contrib.ignite import IgniteDataset
+>>> tf.enable_eager_execution()
 >>>
 >>> dataset = IgniteDataset(cache_name="IMAGES").map(lambda obj: obj['val']['pixels'])
 >>>
@@ -166,6 +167,7 @@ Apache Ignite allows to protect data transfer channels by [SSL](https://en.wikip
 ```python
 >>> import tensorflow as tf
 >>> from tensorflow.contrib.ignite import IgniteDataset
+>>> tf.enable_eager_execution()
 >>>
 >>> dataset = IgniteDataset(cache_name="IMAGES",
                             certfile="client.pem",
@@ -197,7 +199,7 @@ docker run -it -p 10800:10800 dmitrievanthony/ignite-with-mnist
 
 After that you will be able to work with it following way:
 
-![ignite-dataset-mnist](https://s3.amazonaws.com/helloworld23423423ew23/ignite-dataset-mnist.png "Ignite Dataset Mnist")
+![ignite-dataset-mnist](https://s3.amazonaws.com/helloworld23423423ew23/ignite-dataset-mnist-2.png "Ignite Dataset Mnist")
 
 ### IGFS
 
diff --git a/tensorflow/contrib/ignite/python/ops/ignite_dataset_ops.py b/tensorflow/contrib/ignite/python/ops/ignite_dataset_ops.py
index 66e654ca63..3ffceef807 100644
--- a/tensorflow/contrib/ignite/python/ops/ignite_dataset_ops.py
+++ b/tensorflow/contrib/ignite/python/ops/ignite_dataset_ops.py
@@ -735,8 +735,6 @@ class IgniteDataset(dataset_ops.DatasetSource):
       cert_password: Password to be used if the private key is encrypted and a
         password is necessary.
     """
-    super(IgniteDataset, self).__init__()
-
     with IgniteClient(host, port, username, password, certfile, keyfile,
                       cert_password) as client:
       client.handshake()
@@ -760,6 +758,8 @@ class IgniteDataset(dataset_ops.DatasetSource):
         self.cache_type.to_output_types(), self.cache_type.to_output_shapes(),
         self.cache_type.to_output_classes())
 
+    super(IgniteDataset, self).__init__(self._as_variant_tensor())
+
   def _as_variant_tensor(self):
     return gen_dataset_ops.ignite_dataset(self.cache_name, self.host, self.port,
                                           self.local, self.part, self.page_size,
-- 
GitLab


From 0a78b4d689c1d4f40c52377bacd4fef99cbc93c3 Mon Sep 17 00:00:00 2001
From: dmitrievanthony <dmitrievanthony@gmail.com>
Date: Mon, 14 Jan 2019 17:25:10 +0300
Subject: [PATCH 0640/2345] Fix layout of README.md in ignite module.

---
 tensorflow/contrib/ignite/README.md | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/ignite/README.md b/tensorflow/contrib/ignite/README.md
index 47f93115c4..7e3eba5efd 100644
--- a/tensorflow/contrib/ignite/README.md
+++ b/tensorflow/contrib/ignite/README.md
@@ -154,11 +154,13 @@ contains an integration between IGFS and TensorFlow. The integration is based
 on [custom filesystem plugin](https://www.tensorflow.org/extend/add_filesys)
 from TensorFlow side and
 [IGFS Native API](https://ignite.apache.org/features/igfs.html) from Apache
-Ignite side. It has numerous uses, for example: * Checkpoints of state can be
-saved to IGFS for reliability and fault-tolerance. * Training processes
-communicate with TensorBoard by writing event files to a directory, which
-TensorBoard watches. IGFS allows this communication to work even when
-TensorBoard runs in a different process or machine.
+Ignite side. It has numerous uses, for example:
+
+* Checkpoints of state can be saved to IGFS for reliability and
+fault-tolerance.
+* Training processes communicate with TensorBoard by writing event files to a
+directory, which TensorBoard watches. IGFS allows this communication to work
+even when TensorBoard runs in a different process or machine.
 
 ### SSL Connection
 
-- 
GitLab


From ae856965e0b832b75c845da994480bdcf54fa068 Mon Sep 17 00:00:00 2001
From: dmitrievanthony <dmitrievanthony@gmail.com>
Date: Mon, 14 Jan 2019 17:45:35 +0300
Subject: [PATCH 0641/2345] Update Ignite tests

---
 tensorflow/contrib/ignite/python/tests/ignite_dataset_test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/ignite/python/tests/ignite_dataset_test.py b/tensorflow/contrib/ignite/python/tests/ignite_dataset_test.py
index ff5d4c458c..b44b78f075 100644
--- a/tensorflow/contrib/ignite/python/tests/ignite_dataset_test.py
+++ b/tensorflow/contrib/ignite/python/tests/ignite_dataset_test.py
@@ -19,9 +19,9 @@ from __future__ import print_function
 
 import os
 
+import tensorflow as tf
 from tensorflow.contrib.ignite import IgniteDataset
 from tensorflow.python.client import session
-from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.platform import test
@@ -66,7 +66,7 @@ class IgniteDatasetTest(test.TestCase):
     self.assertEqual(dtypes.string, dataset.output_types["val"]["NAME"])
     self.assertEqual(dtypes.int64, dataset.output_types["val"]["VAL"])
 
-    it = dataset_ops.make_one_shot_iterator(dataset)
+    it = tf.compat.v1.data.make_one_shot_iterator(dataset)
     ne = it.get_next()
 
     with session.Session() as sess:
-- 
GitLab


From 4afa2ba93dd606fd3715ef5693dee86fc25e0c77 Mon Sep 17 00:00:00 2001
From: dmitrievanthony <dmitrievanthony@gmail.com>
Date: Mon, 14 Jan 2019 17:57:09 +0300
Subject: [PATCH 0642/2345] Fix Kafka Dataset (V2 API).

---
 tensorflow/contrib/kafka/python/kernel_tests/kafka_test.sh | 0
 tensorflow/contrib/kafka/python/ops/kafka_dataset_ops.py   | 3 ++-
 2 files changed, 2 insertions(+), 1 deletion(-)
 mode change 100644 => 100755 tensorflow/contrib/kafka/python/kernel_tests/kafka_test.sh

diff --git a/tensorflow/contrib/kafka/python/kernel_tests/kafka_test.sh b/tensorflow/contrib/kafka/python/kernel_tests/kafka_test.sh
old mode 100644
new mode 100755
diff --git a/tensorflow/contrib/kafka/python/ops/kafka_dataset_ops.py b/tensorflow/contrib/kafka/python/ops/kafka_dataset_ops.py
index b399e1b6c2..5591c3b0cc 100644
--- a/tensorflow/contrib/kafka/python/ops/kafka_dataset_ops.py
+++ b/tensorflow/contrib/kafka/python/ops/kafka_dataset_ops.py
@@ -52,7 +52,6 @@ class KafkaDataset(dataset_ops.DatasetSource):
       timeout: The timeout value for the Kafka Consumer to wait
                (in millisecond).
     """
-    super(KafkaDataset, self).__init__()
     self._topics = ops.convert_to_tensor(
         topics, dtype=dtypes.string, name="topics")
     self._servers = ops.convert_to_tensor(
@@ -63,6 +62,8 @@ class KafkaDataset(dataset_ops.DatasetSource):
     self._timeout = ops.convert_to_tensor(
         timeout, dtype=dtypes.int64, name="timeout")
 
+    super(KafkaDataset, self).__init__(self._as_variant_tensor())
+
   def _as_variant_tensor(self):
     return gen_dataset_ops.kafka_dataset(self._topics, self._servers,
                                          self._group, self._eof, self._timeout)
-- 
GitLab


From ad683f866b465fa753731be283515bb0a67ad078 Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Mon, 14 Jan 2019 06:51:47 -0800
Subject: [PATCH 0643/2345] Serialize FunctionSpec as proto instead of a tuple.

PiperOrigin-RevId: 229174621
---
 tensorflow/python/eager/function.py           | 51 ++++++++++++-------
 .../saved_model/function_deserialization.py   | 16 ++++--
 .../saved_model/function_serialization.py     | 19 ++++++-
 .../saved_model/saved_object_graph.proto      | 21 ++++++--
 4 files changed, 82 insertions(+), 25 deletions(-)

diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index bbf8630a2e..6429022df3 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -866,14 +866,6 @@ def _deterministic_dict_values(dictionary):
 class FunctionSpec(object):
   """Specification of how to bind arguments to a function."""
 
-  def as_tuple(self):
-    return (self._fullargspec, self._is_method, self._args_to_prepend,
-            self._kwargs_to_include, self.input_signature)
-
-  @staticmethod
-  def from_tuple(spec_tuple):
-    return FunctionSpec(*spec_tuple)
-
   @staticmethod
   def from_function_and_signature(python_function, input_signature):
     """Create a FunctionSpec instance given a python function and signature."""
@@ -920,7 +912,7 @@ class FunctionSpec(object):
     }
     self._default_values_start_index = offset
     if input_signature is None:
-      self.input_signature = None
+      self._input_signature = None
     else:
       if fullargspec.varkw is not None or fullargspec.kwonlyargs:
         raise ValueError("Cannot define a TensorFlow function from a Python "
@@ -931,8 +923,32 @@ class FunctionSpec(object):
         raise TypeError("input_signature must be either a tuple or a "
                         "list, received " + str(type(input_signature)))
 
-      self.input_signature = tuple(input_signature)
-      self.flat_input_signature = tuple(nest.flatten(input_signature))
+      self._input_signature = tuple(input_signature)
+      self._flat_input_signature = tuple(nest.flatten(input_signature))
+
+  @property
+  def fullargspec(self):
+    return self._fullargspec
+
+  @property
+  def is_method(self):
+    return self._is_method
+
+  @property
+  def args_to_prepend(self):
+    return self._args_to_prepend
+
+  @property
+  def kwargs_to_include(self):
+    return self._kwargs_to_include
+
+  @property
+  def input_signature(self):
+    return self._input_signature
+
+  @property
+  def flat_input_signature(self):
+    return self._flat_input_signature
 
   def canonicalize_function_inputs(self, *args, **kwargs):
     """Canonicalizes `args` and `kwargs`.
@@ -980,7 +996,7 @@ class FunctionSpec(object):
         if index is not None:
           arg_indices_to_values[index] = value
           consumed_args.append(arg)
-        elif self.input_signature is not None:
+        elif self._input_signature is not None:
           raise ValueError("Cannot define a TensorFlow function from a Python "
                            "function with keyword arguments when "
                            "input_signature is provided.")
@@ -1003,12 +1019,13 @@ class FunctionSpec(object):
     if need_packing:
       inputs = nest.pack_sequence_as(
           structure=inputs, flat_sequence=flat_inputs)
-    if self.input_signature is None:
+    if self._input_signature is None:
       return inputs, kwargs
     else:
       assert not kwargs
-      signature_relevant_inputs = inputs[:len(self.input_signature)]
-      if not is_same_structure(self.input_signature, signature_relevant_inputs):
+      signature_relevant_inputs = inputs[:len(self._input_signature)]
+      if not is_same_structure(self._input_signature,
+                               signature_relevant_inputs):
         raise ValueError("Structure of Python function inputs does not match "
                          "input_signature.")
       signature_inputs_flat = nest.flatten(signature_relevant_inputs)
@@ -1017,10 +1034,10 @@ class FunctionSpec(object):
         raise ValueError("When input_signature is provided, all inputs to "
                          "the Python function must be Tensors.")
       if any(not spec.is_compatible_with(other) for spec, other in zip(
-          self.flat_input_signature, signature_inputs_flat)):
+          self._flat_input_signature, signature_inputs_flat)):
         raise ValueError("Python inputs incompatible with input_signature: "
                          "inputs (%s), input_signature (%s)" %
-                         (str(inputs), str(self.input_signature)))
+                         (str(inputs), str(self._input_signature)))
       return inputs, {}
 
 
diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py
index ec6cefb606..e19a46ac19 100644
--- a/tensorflow/python/saved_model/function_deserialization.py
+++ b/tensorflow/python/saved_model/function_deserialization.py
@@ -60,6 +60,17 @@ def _inputs_compatible(args, stored_inputs):
   return True
 
 
+def _deserialize_function_spec(function_spec_proto, coder):
+  """Deserialize a FunctionSpec object from its proto representation."""
+  fullargspec = coder.decode_proto(function_spec_proto.fullargspec)
+  is_method = function_spec_proto.is_method
+  args_to_prepend = coder.decode_proto(function_spec_proto.args_to_prepend)
+  kwargs_to_include = coder.decode_proto(function_spec_proto.kwargs_to_include)
+  input_signature = coder.decode_proto(function_spec_proto.input_signature)
+  return function_lib.FunctionSpec(fullargspec, is_method, args_to_prepend,
+                                   kwargs_to_include, input_signature)
+
+
 def recreate_polymorphic_function(
     saved_polymorphic_function, functions):
   """Creates a PolymorphicFunction from a SavedPolymorphicFunction.
@@ -77,9 +88,8 @@ def recreate_polymorphic_function(
   # serialization cycle.
 
   coder = nested_structure_coder.StructureCoder()
-  function_spec_tuple = coder.decode_proto(
-      saved_polymorphic_function.function_spec_tuple)
-  function_spec = function_lib.FunctionSpec.from_tuple(function_spec_tuple)
+  function_spec = _deserialize_function_spec(
+      saved_polymorphic_function.function_spec, coder)
 
   # TODO(mdan): We may enable autograph once exceptions are supported.
   @def_function.function(autograph=False)
diff --git a/tensorflow/python/saved_model/function_serialization.py b/tensorflow/python/saved_model/function_serialization.py
index 267852ef4a..d2c5b33a5f 100644
--- a/tensorflow/python/saved_model/function_serialization.py
+++ b/tensorflow/python/saved_model/function_serialization.py
@@ -25,13 +25,28 @@ from tensorflow.python.saved_model import saved_object_graph_pb2
 from tensorflow.python.util import nest
 
 
+def _serialize_function_spec(function_spec, coder):
+  """Serialize a FunctionSpec object into its proto representation."""
+  proto = saved_object_graph_pb2.FunctionSpec()
+  proto.fullargspec.CopyFrom(coder.encode_structure(function_spec.fullargspec))
+  proto.is_method = function_spec.is_method
+  proto.args_to_prepend.CopyFrom(
+      coder.encode_structure(function_spec.args_to_prepend))
+  proto.kwargs_to_include.CopyFrom(
+      coder.encode_structure(function_spec.kwargs_to_include))
+  proto.input_signature.CopyFrom(
+      coder.encode_structure(function_spec.input_signature))
+  return proto
+
+
 def serialize_polymorphic_function(polymorphic_function, node_ids):
   """Build a SavedPolymorphicProto."""
   coder = nested_structure_coder.StructureCoder()
   proto = saved_object_graph_pb2.SavedPolymorphicFunction()
 
-  proto.function_spec_tuple.CopyFrom(
-      coder.encode_structure(polymorphic_function.function_spec.as_tuple()))  # pylint: disable=protected-access
+  function_spec_proto = _serialize_function_spec(
+      polymorphic_function.function_spec, coder)
+  proto.function_spec.CopyFrom(function_spec_proto)
   for signature, concrete_function in list_all_concrete_functions(
       polymorphic_function):
     bound_inputs = []
diff --git a/tensorflow/python/saved_model/saved_object_graph.proto b/tensorflow/python/saved_model/saved_object_graph.proto
index a322726e1e..f48d2d2b7e 100644
--- a/tensorflow/python/saved_model/saved_object_graph.proto
+++ b/tensorflow/python/saved_model/saved_object_graph.proto
@@ -86,9 +86,7 @@ message SavedAsset {
 // A function with multiple signatures, possibly with non-Tensor arguments.
 message SavedPolymorphicFunction {
   repeated SavedMonomorphicFunction monomorphic_function = 1;
-  // Tuple representing a `FunctionSpec`.
-  // TODO(vbardiovsky): Make this a proto.
-  StructuredValue function_spec_tuple = 2;
+  FunctionSpec function_spec = 2;
 }
 
 message SavedMonomorphicFunction {
@@ -118,3 +116,20 @@ message SavedVariable {
 
   // TODO(andresp): Add save_slice_info_def?
 }
+
+// Represents FunctionSpec used in PolymorphicFunction. This represents a
+// function that has been wrapped as a PolymorphicFunction.
+message FunctionSpec {
+  // Full arg spec from inspect.getfullargspec().
+  StructuredValue fullargspec = 1;
+  // Whether this represents a class method.
+  bool is_method = 2;
+  // Which arguments to always prepend, in case the original function is based
+  // on a functools.partial.
+  StructuredValue args_to_prepend = 3;
+  // Which kwargs to always include, in case the original function is based on a
+  // functools.partial.
+  StructuredValue kwargs_to_include = 4;
+  // The input signature, if specified.
+  StructuredValue input_signature = 5;
+}
-- 
GitLab


From 471b523d9323ebed3f466b7e737e6536f45f4aee Mon Sep 17 00:00:00 2001
From: James Keeling <jtkeeling@google.com>
Date: Mon, 14 Jan 2019 07:28:18 -0800
Subject: [PATCH 0644/2345] Split cwise igammas GPU code into multiple files

This file was a bottleneck during compilation, often taking many minutes to compile. In local testing this change reduces the wall-clock build time for the affected kernels from 141s to 87s.

PiperOrigin-RevId: 229179013
---
 .../kernels/cwise_op_gpu_igammas_double.cu.cc | 29 +++++++++++++++++++
 ...cu.cc => cwise_op_gpu_igammas_float.cu.cc} |  6 ++--
 2 files changed, 32 insertions(+), 3 deletions(-)
 create mode 100644 tensorflow/core/kernels/cwise_op_gpu_igammas_double.cu.cc
 rename tensorflow/core/kernels/{cwise_op_gpu_igammas.cu.cc => cwise_op_gpu_igammas_float.cu.cc} (88%)

diff --git a/tensorflow/core/kernels/cwise_op_gpu_igammas_double.cu.cc b/tensorflow/core/kernels/cwise_op_gpu_igammas_double.cu.cc
new file mode 100644
index 0000000000..2bcc7aa885
--- /dev/null
+++ b/tensorflow/core/kernels/cwise_op_gpu_igammas_double.cu.cc
@@ -0,0 +1,29 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#include "tensorflow/core/kernels/cwise_ops_gpu_common.cu.h"
+#include "tensorflow/core/kernels/cwise_ops_gpu_gradients.cu.h"
+
+namespace tensorflow {
+namespace functor {
+DEFINE_BINARY1(igamma, double);
+DEFINE_BINARY1(igamma_grad_a, double);
+DEFINE_BINARY1(igammac, double);
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/cwise_op_gpu_igammas.cu.cc b/tensorflow/core/kernels/cwise_op_gpu_igammas_float.cu.cc
similarity index 88%
rename from tensorflow/core/kernels/cwise_op_gpu_igammas.cu.cc
rename to tensorflow/core/kernels/cwise_op_gpu_igammas_float.cu.cc
index 508a47deda..e6412216e9 100644
--- a/tensorflow/core/kernels/cwise_op_gpu_igammas.cu.cc
+++ b/tensorflow/core/kernels/cwise_op_gpu_igammas_float.cu.cc
@@ -20,9 +20,9 @@ limitations under the License.
 
 namespace tensorflow {
 namespace functor {
-DEFINE_BINARY2(igamma, float, double);
-DEFINE_BINARY2(igamma_grad_a, float, double);
-DEFINE_BINARY2(igammac, float, double);
+DEFINE_BINARY1(igamma, float);
+DEFINE_BINARY1(igamma_grad_a, float);
+DEFINE_BINARY1(igammac, float);
 }  // namespace functor
 }  // namespace tensorflow
 
-- 
GitLab


From 6dae7e3ebbfd83b9f1d318ba5f2f2baeae17b9f5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 07:40:31 -0800
Subject: [PATCH 0645/2345] Adapt exec_compatible_with for remote testing in
 add_an_op test.

PiperOrigin-RevId: 229180723
---
 tensorflow/examples/adding_an_op/BUILD | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/examples/adding_an_op/BUILD b/tensorflow/examples/adding_an_op/BUILD
index 2b39b3683f..a4d6f204cd 100644
--- a/tensorflow/examples/adding_an_op/BUILD
+++ b/tensorflow/examples/adding_an_op/BUILD
@@ -10,6 +10,7 @@ licenses(["notice"])  # Apache 2.0
 load("//tensorflow:tensorflow.bzl", "tf_custom_op_library")
 load("//tensorflow:tensorflow.bzl", "tf_cuda_tests_tags")
 load("//tensorflow:tensorflow.bzl", "tf_cc_binary")
+load("//tensorflow:tensorflow.bzl", "tf_exec_compatible_with")
 
 exports_files(["LICENSE"])
 
@@ -118,6 +119,7 @@ py_test(
     name = "cuda_op_test",
     size = "small",
     srcs = ["cuda_op_test.py"],
+    exec_compatible_with = tf_exec_compatible_with({"tags": tf_cuda_tests_tags()}),
     srcs_version = "PY2AND3",
     tags = tf_cuda_tests_tags() + ["notap"],
     deps = [
-- 
GitLab


From 22e458382d3001a0cda4e594decf175f2387475e Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Mon, 14 Jan 2019 08:12:13 -0800
Subject: [PATCH 0646/2345] Create quantize multiplier for array.

PiperOrigin-RevId: 229185162
---
 .../kernels/internal/quantization_util.cc     |  9 ++++
 .../lite/kernels/internal/quantization_util.h | 11 +++++
 .../internal/quantization_util_test.cc        | 47 +++++++++++++++++++
 3 files changed, 67 insertions(+)

diff --git a/tensorflow/lite/kernels/internal/quantization_util.cc b/tensorflow/lite/kernels/internal/quantization_util.cc
index 0279d2a922..71eef71372 100644
--- a/tensorflow/lite/kernels/internal/quantization_util.cc
+++ b/tensorflow/lite/kernels/internal/quantization_util.cc
@@ -366,4 +366,13 @@ bool CheckedLog2(const float x, int* log2_result) {
   return std::abs(x_log2_fracpart) < 1e-3;
 }
 
+void QuantizeMultiplierArray(const double* effective_scales, size_t size,
+                             int32_t* effective_scale_significand,
+                             int* effective_shift) {
+  for (size_t i = 0; i < size; ++i) {
+    QuantizeMultiplier(effective_scales[i], &effective_scale_significand[i],
+                       &effective_shift[i]);
+  }
+}
+
 }  // namespace tflite
diff --git a/tensorflow/lite/kernels/internal/quantization_util.h b/tensorflow/lite/kernels/internal/quantization_util.h
index bf313f39cd..5d67c0d027 100644
--- a/tensorflow/lite/kernels/internal/quantization_util.h
+++ b/tensorflow/lite/kernels/internal/quantization_util.h
@@ -275,6 +275,17 @@ void FakeQuantizeArray(const float nudged_scale, const float nudged_min,
 // returns false.
 bool CheckedLog2(const float x, int* log2_result);
 
+// Decomposes an array of double multipliers into a Q0.31 int32 representation
+// of its significand, and shift representation of its exponent.
+//
+// Handles an arbitrary multiplier. The 'shift' output-value is
+// basically the 'floating-point exponent' of the multiplier:
+// Negative for a right-shift (when the multiplier is <1), positive for a
+// left-shift (when the multiplier is >1)
+void QuantizeMultiplierArray(const double* effective_scales, size_t size,
+                             int32_t* effective_scale_significand,
+                             int* effective_shift);
+
 }  // namespace tflite
 
 #endif  // TENSORFLOW_LITE_KERNELS_INTERNAL_QUANTIZATION_UTIL_H_
diff --git a/tensorflow/lite/kernels/internal/quantization_util_test.cc b/tensorflow/lite/kernels/internal/quantization_util_test.cc
index 2f8f771379..ca4ff370ad 100644
--- a/tensorflow/lite/kernels/internal/quantization_util_test.cc
+++ b/tensorflow/lite/kernels/internal/quantization_util_test.cc
@@ -20,6 +20,7 @@ limitations under the License.
 namespace tflite {
 namespace {
 
+using ::testing::ElementsAreArray;
 using ::testing::Pair;
 
 template <class FloatIn, class IntOut>
@@ -406,6 +407,52 @@ TEST(QuantizationUtilTest, CalculateInputRadius) {
   EXPECT_EQ(CalculateInputRadius(4, 2), 503316480);
 }
 
+TEST(QuantizationUtilTest, QuantizeMultiplierArray) {
+  const std::vector<double> weights = {-4,    -2,   -1,  -0.5, -0.25, -0.125, 0,
+                                       0.125, 0.25, 0.5, 1,    2,     4};
+  const int size = weights.size();
+  std::vector<int32> effective_scale_significand(size);
+  std::vector<int> effective_scale_shift(size);
+  QuantizeMultiplierArray(weights.data(), size,
+                          effective_scale_significand.data(),
+                          effective_scale_shift.data());
+  const std::vector<int32> expected_effective_scale_significand = {
+      -1073741824,  // float scale = -4
+      -1073741824,  // float scale = -2
+      -1073741824,  // float scale = -1
+      -1073741824,  // float scale = -0.5
+      -1073741824,  // float scale = -0.25
+      -1073741824,  // float scale = -0.125
+      0,            // float scale = 0
+      1073741824,   // float scale = 0.125
+      1073741824,   // float scale = 0.25
+      1073741824,   // float scale = 0.5
+      1073741824,   // float scale = 1
+      1073741824,   // float scale = 2
+      1073741824,   // float scale = 4
+  };
+
+  const std::vector<int> expected_effective_scale_shift = {
+      3,   // float scale = -4
+      2,   // float scale = -2
+      1,   // float scale = -1
+      0,   // float scale = -0.5
+      -1,  // float scale = -0.25
+      -2,  // float scale = -0.125
+      0,   // float scale = 0
+      -2,  // float scale = 0.125
+      -1,  // float scale = 0.25
+      0,   // float scale = 0.5
+      1,   // float scale = 1
+      2,   // float scale = 2
+      3,   // float scale = 4
+  };
+  EXPECT_THAT(effective_scale_significand,
+              ElementsAreArray(expected_effective_scale_significand));
+  EXPECT_THAT(effective_scale_shift,
+              ElementsAreArray(expected_effective_scale_shift));
+}
+
 }  // namespace
 }  // namespace tflite
 
-- 
GitLab


From dbb3f15afb2ee8d6af7c4a4adc9843314ecfa1cc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 09:13:28 -0800
Subject: [PATCH 0647/2345] Add tests for batch_group_count.

PiperOrigin-RevId: 229194140
---
 tensorflow/compiler/xla/service/BUILD         |   4 +-
 .../service/convolution_group_converter.cc    |  14 +-
 ...cc => convolution_group_converter_test.cc} |  27 ++++
 .../xla/service/hlo_evaluator_typed_visitor.h |  33 +++-
 .../compiler/xla/service/hlo_instructions.cc  |   4 +
 .../compiler/xla/service/shape_inference.cc   |  18 +++
 tensorflow/compiler/xla/tests/BUILD           |  20 +++
 .../conv_depthwise_backprop_filter_test.cc    | 150 ++++++++++++++++++
 8 files changed, 256 insertions(+), 14 deletions(-)
 rename tensorflow/compiler/xla/service/{convolution_feature_group_converter_test.cc => convolution_group_converter_test.cc} (75%)
 create mode 100644 tensorflow/compiler/xla/tests/conv_depthwise_backprop_filter_test.cc

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index b324a810ce..48defc011b 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -1731,9 +1731,9 @@ cc_library(
 )
 
 tf_cc_test(
-    name = "convolution_feature_group_converter_test",
+    name = "convolution_group_converter_test",
     size = "small",
-    srcs = ["convolution_feature_group_converter_test.cc"],
+    srcs = ["convolution_group_converter_test.cc"],
     deps = [
         ":convolution_group_converter",
         ":hlo",
diff --git a/tensorflow/compiler/xla/service/convolution_group_converter.cc b/tensorflow/compiler/xla/service/convolution_group_converter.cc
index 7a24faec17..b3ed27d9a8 100644
--- a/tensorflow/compiler/xla/service/convolution_group_converter.cc
+++ b/tensorflow/compiler/xla/service/convolution_group_converter.cc
@@ -207,7 +207,8 @@ Status ConvolutionVisitor::HandleBatchGroupCount(HloInstruction* convolution) {
     return Status::OK();
   }
 
-  VLOG(2) << "Dealing with batch_group_count " << batch_group_count << "\n";
+  VLOG(2) << "Dealing with batch_group_count " << batch_group_count
+          << " for convolution " << convolution->ToString() << "\n";
 
   auto add = [&](std::unique_ptr<HloInstruction> inst) {
     return computation_->AddInstruction(std::move(inst));
@@ -332,18 +333,19 @@ Status ConvolutionVisitor::HandleBatchGroupCount(HloInstruction* convolution) {
       return computation_->parent()->AddEmbeddedComputation(b.Build(scalar_op));
     };
 
+    auto reduce_window_shape = new_convolution->shape();
+    reduce_window_shape.set_dimensions(output_batch_dimension, 1);
+
     // Ensure that data input to reduce window is of type F32.
     if (primitive_util::BitWidth(new_filter->shape().element_type()) <
         primitive_util::BitWidth(F32)) {
+      reduce_window_shape.set_element_type(F32);
       Shape convert_shape = new_filter->shape();
       convert_shape.set_element_type(F32);
       new_filter =
-          add(HloInstruction::CreateBitcastConvert(convert_shape, new_filter));
+          add(HloInstruction::CreateConvert(convert_shape, new_filter));
     }
 
-    auto reduce_window_shape = new_convolution->shape();
-    reduce_window_shape.set_dimensions(output_batch_dimension, 1);
-
     // Create the reduce window.
     Window window;
     for (int64 i = 0; i < new_convolution->shape().dimensions_size(); ++i) {
@@ -369,7 +371,7 @@ Status ConvolutionVisitor::HandleBatchGroupCount(HloInstruction* convolution) {
 
     // Convert reduced data back to the original data type.
     auto reduce_window_converted =
-        HloInstruction::CreateBitcastConvert(convert_back_shape, reduce_window);
+        HloInstruction::CreateConvert(convert_back_shape, reduce_window);
 
     TF_RETURN_IF_ERROR(computation_->ReplaceWithNewInstruction(
         convolution, std::move(reduce_window_converted)));
diff --git a/tensorflow/compiler/xla/service/convolution_feature_group_converter_test.cc b/tensorflow/compiler/xla/service/convolution_group_converter_test.cc
similarity index 75%
rename from tensorflow/compiler/xla/service/convolution_feature_group_converter_test.cc
rename to tensorflow/compiler/xla/service/convolution_group_converter_test.cc
index d58f157242..585b81a5db 100644
--- a/tensorflow/compiler/xla/service/convolution_feature_group_converter_test.cc
+++ b/tensorflow/compiler/xla/service/convolution_group_converter_test.cc
@@ -94,5 +94,32 @@ ENTRY %Convolve1D1Window_0.v3 (input: f32[1,2,4], filter: f32[1,2,2]) -> f32[1,2
   EXPECT_EQ(root->operand(1)->feature_group_count(), 1);
 }
 
+TEST_F(ConvolutionGroupConverterTest,
+       ConvertBatchGroupCountEqualToInputBatchDim) {
+  string hlo_string = R"(HloModule Convolve1D1Window_0_module
+
+ENTRY %Convolve1D1Window_0.v3 (input: f32[16,19,19,512]{3,2,1,0}, filter: f32[16,19,19,512]{3,2,1,0}) -> f32[3,3,512,1]{3,2,1,0} {
+  %input = f32[16,19,19,512]{3,2,1,0} parameter(0)
+  %filter = f32[16,19,19,512]{3,2,1,0} parameter(1)
+  ROOT %convolution = f32[3,3,512,1]{3,2,1,0} convolution(f32[16,19,19,512]{3,2,1,0} %input, f32[16,19,19,512]{3,2,1,0} %filter), window={size=19x19 pad=1_1x1_1}, dim_labels=f01b_i01o->01fb, batch_group_count=512
+  })";
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseHloString(hlo_string));
+
+  auto computation = module->entry_computation();
+  HloInstruction* root = computation->root_instruction();
+  EXPECT_EQ(root->opcode(), HloOpcode::kConvolution);
+  auto cost_model = [](HloInstruction* conv) { return true; };
+  ConvolutionGroupConverter converter(cost_model, /*convert_batch_groups_only=*/
+                                      true);
+  ASSERT_TRUE(converter.Run(module.get()).ValueOrDie());
+  root = computation->root_instruction();
+  // Make sure the convolution is converted to one with batch_group_count = 1.
+  EXPECT_EQ(root->operand(0)->opcode(), HloOpcode::kConvolution);
+  EXPECT_EQ(root->operand(0)->batch_group_count(), 1);
+  // Verify that the convolution is replaced by a reshape.
+  EXPECT_EQ(root->opcode(), HloOpcode::kReshape);
+}
+
 }  // namespace
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
index f3f400e46f..5245c563ec 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
@@ -1042,15 +1042,13 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
     auto lhs_literal_data = lhs_literal.data<ReturnT>();
     auto rhs_literal_data = rhs_literal.data<ReturnT>();
 
-    int64 feature_group_count = conv->feature_group_count();
-    int64 batch_group_count = conv->batch_group_count();
+    const int64 feature_group_count = conv->feature_group_count();
+    const int64 batch_group_count = conv->batch_group_count();
 
-    // The batch count > 1 case is unimplemented in the HLO evaluator so far.
-    TF_RET_CHECK(batch_group_count == 1);
     auto func = [&window_shape, &dnums, &lhs_shape, &rhs_shape, &window,
                  &lhs_dim_multipliers, &rhs_dim_multipliers, lhs_literal_data,
-                 rhs_literal_data,
-                 feature_group_count](const absl::Span<const int64> out_index) {
+                 rhs_literal_data, feature_group_count,
+                 batch_group_count](const absl::Span<const int64> out_index) {
       // Dimension number applicable for input (lhs).
       const int64 input_batch_dim = dnums.input_batch_dimension();
       const int64 input_z_dim = dnums.input_feature_dimension();
@@ -1063,6 +1061,12 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
 
       const int64 input_z_size =
           ShapeUtil::GetDimension(lhs_shape, input_z_dim);
+
+      const int64 input_batch_size =
+          ShapeUtil::GetDimension(lhs_shape, input_batch_dim);
+
+      const int64 batch_group_size = input_batch_size / batch_group_count;
+
       // The size of an input feature group.
       const int64 input_feature_group_size = input_z_size / feature_group_count;
 
@@ -1078,11 +1082,15 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
       const int64 feature_group_index =
           out_index[output_z_dim] / output_feature_group_size;
 
+      const int64 batch_group_index = out_index[output_z_dim];
+
       ElementwiseT result_val = static_cast<ElementwiseT>(0);
       DimensionVector rhs_spatial_index(dnums.kernel_spatial_dimensions_size(),
                                         0);
 
       // Convolve input feature with kernel.
+      // The mechanism indexes into the correct LHS (input) and RHS (kernel)
+      // locations and accumulates multiplications for a given output index.
       do {
         // Find corresponding spatial dimension index for input (lhs).
         int64 lhs_linear_spatial_index = 0;
@@ -1135,11 +1143,24 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
               feature_group_index * input_feature_group_size + rhs_iz;
 
           int64 lhs_linear_index = lhs_linear_spatial_index;
+
           lhs_linear_index += out_index[output_batch_dim] *
                               lhs_dim_multipliers[input_batch_dim];
+
+          // We are scraping only the diagonal elements in the resultant
+          // convolution output when batch_group_count is greater than 1,
+          // where 1 is the default. No scraping is done in that case.
+          // This approach works out automatically for 'groups' in batches
+          // with group_size > 1, because we already descend down the batch
+          // dimension for the 'output_batch_dim' above.
+          lhs_linear_index +=
+              ((batch_group_index * batch_group_size) % input_batch_size) *
+              lhs_dim_multipliers[input_batch_dim];
+
           lhs_linear_index += iz * lhs_dim_multipliers[input_z_dim];
 
           int64 rhs_linear_index = rhs_linear_spatial_index;
+
           rhs_linear_index += out_index[output_z_dim] *
                               rhs_dim_multipliers[kernel_output_z_dim];
           rhs_linear_index += rhs_iz * rhs_dim_multipliers[kernel_input_z_dim];
diff --git a/tensorflow/compiler/xla/service/hlo_instructions.cc b/tensorflow/compiler/xla/service/hlo_instructions.cc
index c24bf41ff8..3a0d71dd88 100644
--- a/tensorflow/compiler/xla/service/hlo_instructions.cc
+++ b/tensorflow/compiler/xla/service/hlo_instructions.cc
@@ -1702,6 +1702,10 @@ std::vector<string> HloConvolutionInstruction::ExtraAttributesToStringImpl(
     extra.push_back(StrCat("feature_group_count=", feature_group_count_));
   }
 
+  if (batch_group_count_ != 1) {
+    extra.push_back(StrCat("batch_group_count=", batch_group_count_));
+  }
+
   string precision_config_string = PrecisionConfigToString(precision_config_);
   if (!precision_config_string.empty()) {
     extra.push_back(precision_config_string);
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index c2e014ad42..fad7afefaf 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -1539,6 +1539,13 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
         batch_group_count);
   }
 
+  if (batch_group_count > 1 && feature_group_count > 1) {
+    return InvalidArgument(
+        "both batch_group_count %d and feature_group_count %d cannot be "
+        "greater than 1",
+        batch_group_count, feature_group_count);
+  }
+
   if (!ShapeUtil::SameElementTypeIgnoringFpPrecision(lhs, rhs)) {
     return InvalidArgument(
         "Convolution with different element types: %s and %s.",
@@ -1652,6 +1659,17 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
   const int64 kernel_output_features =
       rhs.dimensions(dnums.kernel_output_feature_dimension());
 
+  if (batch_group_count > 1 && input_batch % kernel_output_features != 0) {
+    return InvalidArgument(
+        "Expected output feature dimension (value %d) to be divisible by "
+        "input_batch (value %d) for batch group count %d; "
+        "got <conv>(%s, %s)\n"
+        "Dimension numbers: {%s}.",
+        kernel_output_features, input_batch, batch_group_count,
+        ShapeUtil::HumanString(lhs), ShapeUtil::HumanString(rhs),
+        dnums.DebugString());
+  }
+
   if (input_features % feature_group_count != 0 ||
       input_features / feature_group_count != kernel_input_features) {
     return InvalidArgument(
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 0fd0fc108a..4e7480c9af 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -313,6 +313,26 @@ xla_test(
     ],
 )
 
+xla_test(
+    name = "conv_depthwise_backprop_filter_test",
+    timeout = "long",
+    srcs = ["conv_depthwise_backprop_filter_test.cc"],
+    shard_count = 1,
+    deps = [
+        "//tensorflow/compiler/xla:execution_options_util",
+        "//tensorflow/compiler/xla:status_macros",
+        "//tensorflow/compiler/xla:test",
+        "//tensorflow/compiler/xla/client:xla_computation",
+        "//tensorflow/compiler/xla/service:bfloat16_normalization",
+        "//tensorflow/compiler/xla/service:despecializer",
+        "//tensorflow/compiler/xla/service:hlo_parser",
+        "//tensorflow/compiler/xla/tests:client_library_test_base",
+        "//tensorflow/compiler/xla/tests:hlo_test_base",
+        "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+        "@com_google_absl//absl/types:optional",
+    ],
+)
+
 xla_test(
     name = "grouped_convolution_test",
     timeout = "long",
diff --git a/tensorflow/compiler/xla/tests/conv_depthwise_backprop_filter_test.cc b/tensorflow/compiler/xla/tests/conv_depthwise_backprop_filter_test.cc
new file mode 100644
index 0000000000..90c1971403
--- /dev/null
+++ b/tensorflow/compiler/xla/tests/conv_depthwise_backprop_filter_test.cc
@@ -0,0 +1,150 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "absl/types/optional.h"
+#include "tensorflow/compiler/xla/client/xla_computation.h"
+#include "tensorflow/compiler/xla/execution_options_util.h"
+#include "tensorflow/compiler/xla/service/bfloat16_normalization.h"
+#include "tensorflow/compiler/xla/service/despecializer.h"
+#include "tensorflow/compiler/xla/service/hlo_parser.h"
+#include "tensorflow/compiler/xla/status_macros.h"
+#include "tensorflow/compiler/xla/test.h"
+#include "tensorflow/compiler/xla/tests/client_library_test_base.h"
+#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+#include "tensorflow/compiler/xla/tests/test_macros.h"
+
+namespace xla {
+namespace {
+
+string GetFloatDataType(bool use_bfloat16) {
+  return use_bfloat16 ? "bf16" : "f32";
+}
+
+struct DepthwiseConvolution2DSpec {
+  int64 output_batch, window;
+  std::vector<int64> activation_dims;
+  std::vector<int64> activation_layout;
+  std::vector<int64> kernel_dims;
+  std::vector<int64> kernel_layout;
+  std::vector<int64> output_dims;
+  std::vector<int64> output_layout;
+};
+
+class DepthwiseConvolution2DTest
+    : public HloTestBase,
+      public ::testing::WithParamInterface<
+          ::testing::tuple<DepthwiseConvolution2DSpec, bool>> {};
+
+static std::vector<DepthwiseConvolution2DSpec> GetConv2DTestCases() {
+  std::vector<DepthwiseConvolution2DSpec> config_set;
+  std::vector<std::vector<int64>> config_options = {
+      {16, 5, 5, 2}, {64, 4, 4, 16}, {2, 5, 5, 256}};
+
+  for (auto option : config_options) {
+    int64 feature = option[3];
+    int64 activation_size = option[1];
+    int64 kernel_size = option[2];
+    int64 batch = option[0];
+
+    std::vector<int64> kernel_layout = {3, 2, 1, 0};
+    DepthwiseConvolution2DSpec config;
+    config.output_batch = feature;
+    config.window = kernel_size;
+
+    config.activation_dims = {batch, activation_size, activation_size, feature};
+    config.activation_layout = {0, 3, 2, 1};
+
+    config.kernel_dims = {batch, kernel_size, kernel_size, feature};
+    config.kernel_layout = {0, 2, 3, 1};
+
+    config.output_dims = {3, 3, feature, 1};
+
+    // Try this layout for all kernel shapes.
+    config.output_layout = {3, 2, 0, 1};
+    config_set.push_back(config);
+  }
+
+  return config_set;
+}
+
+string DepthwiseConvolution2DTestDataToString(
+    const ::testing::TestParamInfo<
+        ::testing::tuple<DepthwiseConvolution2DSpec, bool>>& data) {
+  const auto& spec = ::testing::get<0>(data.param);
+  const string data_type = GetFloatDataType(::testing::get<1>(data.param));
+  string str = absl::StrCat(
+      "activation_dims_", absl::StrJoin(spec.activation_dims, "x"),
+      "_activation_layout_", absl::StrJoin(spec.activation_layout, "_"),
+      "_kernel_dims_", absl::StrJoin(spec.kernel_dims, "x"), "_kernel_layout_",
+      absl::StrJoin(spec.kernel_layout, "_"), "_output_dims_",
+      absl::StrJoin(spec.output_dims, "x"), "_output_layout_",
+      absl::StrJoin(spec.output_layout, "_"), data_type);
+
+  // Test names are not allowed to contain the '-' character.
+  absl::c_replace(str, '-', 'n');
+  return str;
+}
+
+string BuildHloTextDepthwiseConvolution2D(
+    const DepthwiseConvolution2DSpec& spec, bool use_bfloat16) {
+  const string data_type = GetFloatDataType(use_bfloat16);
+  return absl::StrFormat(
+      R"(
+    HloModule TensorFlowDepthwiseConv
+
+    ENTRY main {
+      activation = %s[%s]{%s} parameter(0)
+      kernel = %s[%s]{%s} parameter(1)
+      ROOT conv = %s[%s]{%s} convolution(%s[%s]{%s} activation, %s[%s]{%s} kernel),
+          window={size=%dx%d pad=1_1x1_1}, dim_labels=f01b_i01o->01fb,
+          batch_group_count=%d
+    }
+    )",
+      data_type, absl::StrJoin(spec.activation_dims, ","),
+      absl::StrJoin(spec.activation_layout, ","), data_type,
+      absl::StrJoin(spec.kernel_dims, ","),
+      absl::StrJoin(spec.kernel_layout, ","), data_type,
+      absl::StrJoin(spec.output_dims, ","),
+      absl::StrJoin(spec.output_layout, ","), data_type,
+      absl::StrJoin(spec.activation_dims, ","),
+      absl::StrJoin(spec.activation_layout, ","), data_type,
+      absl::StrJoin(spec.kernel_dims, ","),
+      absl::StrJoin(spec.kernel_layout, ","), spec.window, spec.window,
+      spec.output_batch);
+}
+
+XLA_TEST_P(DepthwiseConvolution2DTest, DoIt) {
+  const DepthwiseConvolution2DSpec& spec = ::testing::get<0>(GetParam());
+  bool use_bfloat16 = ::testing::get<1>(GetParam());
+  const string hlo_text =
+      BuildHloTextDepthwiseConvolution2D(spec, use_bfloat16);
+
+  EXPECT_TRUE(RunAndCompare(hlo_text, ErrorSpec{0.01, 0.01},
+                            [](HloModule* module) -> Status {
+                              BFloat16MixedPrecisionRemoval remover;
+                              TF_RETURN_IF_ERROR(remover.Run(module).status());
+                              Despecializer despecializer;
+                              return despecializer.Run(module).status();
+                            }));
+}
+
+INSTANTIATE_TEST_CASE_P(
+    DepthwiseConvolution2DTestWithRandomIndices, DepthwiseConvolution2DTest,
+    ::testing::Combine(::testing::ValuesIn(GetConv2DTestCases()),
+                       ::testing::Bool()),
+    DepthwiseConvolution2DTestDataToString);
+
+}  // namespace
+}  // namespace xla
-- 
GitLab


From b9598f45d4de3baaa745436891907f989446d6b6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 09:30:31 -0800
Subject: [PATCH 0648/2345] Rollback change in Eager mode decorator.

This change causes the Eager mode to be run with all methods in the class, but not all methods in its super classes.

PiperOrigin-RevId: 229196652
---
 tensorflow/python/framework/test_util.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index 491249089e..22cd50321f 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -885,8 +885,7 @@ def generate_combinations_with_testcase_name(**kwargs):
 def run_all_in_graph_and_eager_modes(cls):
   """Execute all test methods in the given class with and without eager."""
   base_decorator = run_in_graph_and_eager_modes
-  for name in dir(cls):
-    value = getattr(cls, name)
+  for name, value in cls.__dict__.copy().items():
     if callable(value) and name.startswith(
         unittest.TestLoader.testMethodPrefix) and not (
             name.startswith("testSkipEager") or
-- 
GitLab


From caf00d258648be6ad19c98830a63431e31209045 Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Mon, 14 Jan 2019 09:33:38 -0800
Subject: [PATCH 0649/2345] Refactoring: move list_all_concrete_functions into
 PolymorphicFunction.

This will enable overriding this function in a "RestoredPolymorphicFunction", which will be helpful for repeated saving/restoring.

PiperOrigin-RevId: 229197202
---
 tensorflow/python/eager/def_function.py       | 23 ++++++++++++++
 .../saved_model/function_serialization.py     | 30 ++-----------------
 tensorflow/python/saved_model/load_test.py    |  5 ++--
 tensorflow/python/saved_model/save.py         |  4 +--
 4 files changed, 30 insertions(+), 32 deletions(-)

diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py
index 4c56358e24..bd70b9dad0 100644
--- a/tensorflow/python/eager/def_function.py
+++ b/tensorflow/python/eager/def_function.py
@@ -30,7 +30,9 @@ from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variable_scope
+from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training.checkpointable import base as checkpointable
+from tensorflow.python.util import nest
 from tensorflow.python.util import tf_decorator
 from tensorflow.python.util.tf_export import tf_export
 
@@ -443,6 +445,27 @@ class PolymorphicFunction(object):
 
     return initialize_variables.get_concrete_function()
 
+  def _list_all_concrete_functions_for_serialization(self):
+    """Returns all of the concrete functions.
+
+    Returns:
+      A list of tuples in the form (signature, concrete_function), where
+      concrete function is an instance of `Function`.
+    """
+    input_signature = self._input_signature
+    if input_signature is not None:
+      self.get_concrete_function()
+    concrete_functions = []
+    for signature in self._cached_input_signatures:
+      flattened = nest.flatten(signature)
+      if any(
+          isinstance(arg, function_lib.UnknownArgument) for arg in flattened):
+        logging.info("Unsupported signature for serialization: %s.", signature)
+        continue
+      concrete_function = self.get_concrete_function(*signature)
+      concrete_functions.append((signature, concrete_function))
+    return concrete_functions
+
   @property
   def _cached_input_signatures(self):
     """All input signatures used to call this PolymorphicFunction."""
diff --git a/tensorflow/python/saved_model/function_serialization.py b/tensorflow/python/saved_model/function_serialization.py
index d2c5b33a5f..e5d3cc4bd2 100644
--- a/tensorflow/python/saved_model/function_serialization.py
+++ b/tensorflow/python/saved_model/function_serialization.py
@@ -22,7 +22,6 @@ from tensorflow.python.eager import function as defun_lib
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.saved_model import nested_structure_coder
 from tensorflow.python.saved_model import saved_object_graph_pb2
-from tensorflow.python.util import nest
 
 
 def _serialize_function_spec(function_spec, coder):
@@ -47,8 +46,9 @@ def serialize_polymorphic_function(polymorphic_function, node_ids):
   function_spec_proto = _serialize_function_spec(
       polymorphic_function.function_spec, coder)
   proto.function_spec.CopyFrom(function_spec_proto)
-  for signature, concrete_function in list_all_concrete_functions(
-      polymorphic_function):
+  all_concrete_functions = \
+      polymorphic_function._list_all_concrete_functions_for_serialization()  # pylint: disable=protected-access
+  for signature, concrete_function in all_concrete_functions:
     bound_inputs = []
     try:
       for capture in concrete_function.captured_inputs:
@@ -70,27 +70,3 @@ def serialize_polymorphic_function(polymorphic_function, node_ids):
         coder.encode_structure(structured_outputs))
     function_proto.bound_inputs.extend(bound_inputs)
   return proto
-
-
-def list_all_concrete_functions(polymorphic_function):
-  """Given a polymorphic function, returns all of its concrete functions.
-
-  Args:
-    polymorphic_function: Instance of `PolymorphicFunction`.
-
-  Returns:
-    A list of tuples in the form (signature, concrete_function), where concrete
-    function is an instance of `Function`.
-  """
-  input_signature = polymorphic_function._input_signature  # pylint: disable=protected-access
-  if input_signature is not None:
-    polymorphic_function.get_concrete_function()
-  concrete_functions = []
-  for signature in polymorphic_function._cached_input_signatures:  # pylint: disable=protected-access
-    flattened = nest.flatten(signature)
-    if any(isinstance(arg, defun_lib.UnknownArgument) for arg in flattened):
-      logging.info("Unsupported signature for serialization: %s.", signature)
-      continue
-    concrete_function = polymorphic_function.get_concrete_function(*signature)
-    concrete_functions.append((signature, concrete_function))
-  return concrete_functions
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index 1d6d1fe88c..e94cda533e 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -30,7 +30,6 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import tensor_spec
 from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import variables
-from tensorflow.python.saved_model import function_serialization
 from tensorflow.python.saved_model import load
 from tensorflow.python.saved_model import save
 from tensorflow.python.training.checkpointable import tracking
@@ -389,8 +388,8 @@ class LoadTest(test.TestCase):
     self.assertAllEqual([2], root.f(constant_op.constant([1])).numpy())
     self.assertAllEqual([2, 4], root.f(constant_op.constant([1, 2])).numpy())
 
-    self.assertEqual(
-        1, len(function_serialization.list_all_concrete_functions(root.f)))
+    concrete_functions = root.f._list_all_concrete_functions_for_serialization()
+    self.assertEqual(1, len(concrete_functions))  # pylint: disable=protected-access
 
     imported = self.cycle(root)
 
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index 0f98766553..207dc2cb74 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -84,7 +84,7 @@ class _SaveableView(object):
         #  input_signature and have not been called.
         #  - force side effects of creation of concrete functions, e.g. create
         #  variables on first run.
-        function_serialization.list_all_concrete_functions(function)
+        function._list_all_concrete_functions_for_serialization()  # pylint: disable=protected-access
 
   @property
   def root(self):
@@ -556,7 +556,7 @@ def _fill_meta_graph_def(meta_graph_def, saveable_view, signature_functions,
   for obj in accessible_objects:
     for function in saveable_view.polymorphic_functions[obj].values():
       concrete_functions.extend(
-          function_serialization.list_all_concrete_functions(function))
+          function._list_all_concrete_functions_for_serialization())  # pylint: disable=protected-access
 
   with exported_graph.as_default():
     signatures = _generate_signatures(signature_functions, resource_map)
-- 
GitLab


From a1a8cf6038812c3f5f81d9ddda11553bd2dcc149 Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Mon, 14 Jan 2019 09:42:00 -0800
Subject: [PATCH 0650/2345] Fix file path for test

PiperOrigin-RevId: 229198411
---
 tensorflow/lite/models/smartreply/predictor_test.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/lite/models/smartreply/predictor_test.cc b/tensorflow/lite/models/smartreply/predictor_test.cc
index 9ce8ce9a77..f4a9453b42 100644
--- a/tensorflow/lite/models/smartreply/predictor_test.cc
+++ b/tensorflow/lite/models/smartreply/predictor_test.cc
@@ -34,8 +34,7 @@ namespace {
 const char kSamples[] = "smartreply_samples.tsv";
 
 string GetModelFilePath() {
-  return "third_party/tensorflow/lite/models/testdata/"
-         "smartreply_ondevice_model.bin";
+  return "external/tflite_smartreply/smartreply.tflite";  // NOLINT
 }
 
 string GetSamplesFilePath() {
-- 
GitLab


From 558974db23ee0ea8845ba4b0b7fa94f70408ac1b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 09:48:57 -0800
Subject: [PATCH 0651/2345] Increase timeout for for a python kernel_test.

There have been several timeouts with //third_party/tensorflow/python/kernel_tests:bias_op_test_xla

PiperOrigin-RevId: 229199505
---
 tensorflow/python/kernel_tests/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index d9ea42374d..7a514875d6 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -1514,7 +1514,7 @@ cuda_py_test(
 
 cuda_py_test(
     name = "bias_op_test",
-    size = "small",
+    size = "medium",
     srcs = ["bias_op_test.py"],
     additional_deps = [
         "//third_party/py/numpy",
-- 
GitLab


From 72678f0315963cbc19c371f90f3605149fece063 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 10:29:25 -0800
Subject: [PATCH 0652/2345] Remove unused methods. Those are obsolete now,
 after the recent refactoring efforts.

PiperOrigin-RevId: 229207631
---
 tensorflow/lite/delegates/flex/buffer_map.cc  | 14 -----
 tensorflow/lite/delegates/flex/buffer_map.h   |  6 ---
 .../lite/delegates/flex/buffer_map_test.cc    | 51 -------------------
 3 files changed, 71 deletions(-)

diff --git a/tensorflow/lite/delegates/flex/buffer_map.cc b/tensorflow/lite/delegates/flex/buffer_map.cc
index 9408efaaa1..0d0c953636 100644
--- a/tensorflow/lite/delegates/flex/buffer_map.cc
+++ b/tensorflow/lite/delegates/flex/buffer_map.cc
@@ -173,19 +173,5 @@ void BufferMap::SetFromTensorFlow(int tensor_index, tensorflow::Tensor tensor) {
   owned_by_tf_.insert(tensor_index);
 }
 
-void BufferMap::RemoveTensor(int tensor_index) {
-  id_to_tensor_.erase(tensor_index);
-}
-
-void BufferMap::RemoveTensorsNotInSet(const std::set<int>& keep) {
-  for (auto it = id_to_tensor_.begin(); it != id_to_tensor_.end();) {
-    if (keep.count(it->first) == 0 && IsTensorFlowTensor(it->first)) {
-      it = id_to_tensor_.erase(it);
-    } else {
-      ++it;
-    }
-  }
-}
-
 }  // namespace flex
 }  // namespace tflite
diff --git a/tensorflow/lite/delegates/flex/buffer_map.h b/tensorflow/lite/delegates/flex/buffer_map.h
index d4724a011d..45bc571f92 100644
--- a/tensorflow/lite/delegates/flex/buffer_map.h
+++ b/tensorflow/lite/delegates/flex/buffer_map.h
@@ -52,12 +52,6 @@ class BufferMap {
   // shallow copy.
   void SetFromTensorFlow(int tensor_index, tensorflow::Tensor tensor);
 
-  // Removes any association of a tensor and the given 'tensor_index'.
-  void RemoveTensor(int tensor_index);
-
-  // Removes all *TF* tensors that are not contained in the given 'keep' set.
-  void RemoveTensorsNotInSet(const std::set<int>& keep);
-
   // Same as above but creates a new tensorflow::Tensor with a copy of the
   // given TfLiteTensor's data.
   void SetFromTfLite(int tensor_index, const TfLiteTensor* tensor);
diff --git a/tensorflow/lite/delegates/flex/buffer_map_test.cc b/tensorflow/lite/delegates/flex/buffer_map_test.cc
index 7b4acbd69d..8edeb280eb 100644
--- a/tensorflow/lite/delegates/flex/buffer_map_test.cc
+++ b/tensorflow/lite/delegates/flex/buffer_map_test.cc
@@ -222,57 +222,6 @@ TEST(BufferMapTest, TensorFlowOverwritesTfLite) {
               ElementsAre(0, 0, 0, 0.123f, 0, 0));
 }
 
-TEST(BufferMapTest, RemoveTensorFlowTensor) {
-  tensorflow::Tensor t1 =
-      MakeTensor<float>({1, 2, 1, 3}, {0, 0, 0, 0.123f, 0, 0});
-  BufferMap buffer_map;
-  buffer_map.SetFromTensorFlow(0, t1);
-  EXPECT_TRUE(buffer_map.HasTensor(0));
-  buffer_map.RemoveTensor(0);
-  EXPECT_FALSE(buffer_map.HasTensor(0));
-}
-
-TEST(BufferMapTest, RemoveTfLiteTensor) {
-  UniqueTfLiteTensor t2 =
-      MakeLiteTensor<int>({1, 2, 4}, {0, 0, 0, 3, 0, 0, 1, 2});
-  BufferMap buffer_map;
-  buffer_map.SetFromTfLite(0, t2.get());
-  EXPECT_TRUE(buffer_map.HasTensor(0));
-  buffer_map.RemoveTensor(0);
-  EXPECT_FALSE(buffer_map.HasTensor(0));
-}
-
-TEST(BufferMapTest, RemoveTensorsInSet) {
-  tensorflow::Tensor t1 =
-      MakeTensor<float>({1, 2, 1, 3}, {0, 0, 0, 0.123f, 0, 0});
-  UniqueTfLiteTensor t2 =
-      MakeLiteTensor<int>({1, 2, 4}, {0, 0, 0, 3, 0, 0, 1, 2});
-  BufferMap buffer_map;
-
-  auto list_existing_tensors = [&buffer_map]() {
-    std::vector<int> result;
-    for (int i : {0, 1, 2, 3}) {
-      if (buffer_map.HasTensor(i)) {
-        result.push_back(i);
-      }
-    }
-    return result;
-  };
-
-  buffer_map.SetFromTensorFlow(0, t1);
-  buffer_map.SetFromTfLite(1, t2.get());
-  buffer_map.SetFromTfLite(2, t2.get());
-  buffer_map.SetFromTensorFlow(3, t1);
-
-  EXPECT_THAT(list_existing_tensors(), ElementsAre(0, 1, 2, 3));
-
-  buffer_map.RemoveTensorsNotInSet({0, 3});
-  EXPECT_THAT(list_existing_tensors(), ElementsAre(0, 1, 2, 3));
-
-  buffer_map.RemoveTensorsNotInSet({});
-  EXPECT_THAT(list_existing_tensors(), ElementsAre(1, 2));
-}
-
 TEST(BufferMapTest, Forwardable) {
   BufferMap buffer_map;
   EXPECT_FALSE(buffer_map.IsForwardable(0));
-- 
GitLab


From 4f2400fd319f7f7d12d90b08cecd6df9116524c6 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Mon, 14 Jan 2019 18:35:31 +0000
Subject: [PATCH 0653/2345] Update kinesis dataset to match v2 API

Since DatasetSource now requires varient_tensor to be passed into constructor,
the kinesis data will need to be updated to fix the incompatibility issue.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/contrib/kinesis/python/ops/kinesis_dataset_ops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/kinesis/python/ops/kinesis_dataset_ops.py b/tensorflow/contrib/kinesis/python/ops/kinesis_dataset_ops.py
index 2b1d478a9b..9479afb180 100644
--- a/tensorflow/contrib/kinesis/python/ops/kinesis_dataset_ops.py
+++ b/tensorflow/contrib/kinesis/python/ops/kinesis_dataset_ops.py
@@ -71,7 +71,6 @@ class KinesisDataset(dataset_ops.DatasetSource):
       interval: The interval for the Kinesis Client to wait before
         it tries to get records again (in millisecond).
     """
-    super(KinesisDataset, self).__init__()
     self._stream = ops.convert_to_tensor(
         stream, dtype=dtypes.string, name="stream")
     self._shard = ops.convert_to_tensor(
@@ -80,6 +79,7 @@ class KinesisDataset(dataset_ops.DatasetSource):
         read_indefinitely, dtype=dtypes.bool, name="read_indefinitely")
     self._interval = ops.convert_to_tensor(
         interval, dtype=dtypes.int64, name="interval")
+    super(KinesisDataset, self).__init__(self._as_variant_tensor())
 
   def _as_variant_tensor(self):
     return gen_dataset_ops.kinesis_dataset(
-- 
GitLab


From 959494c31ed3a0eb84af9057eaa34c2ed4c0799c Mon Sep 17 00:00:00 2001
From: Guangda Lai <31743510+aaroey@users.noreply.github.com>
Date: Mon, 14 Jan 2019 11:15:54 -0800
Subject: [PATCH 0654/2345] fix CI build failures

---
 tensorflow/contrib/tensorrt/test/int32_test.py | 7 +++----
 tensorflow/contrib/tensorrt/test/topk_test.py  | 8 ++++----
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/test/int32_test.py b/tensorflow/contrib/tensorrt/test/int32_test.py
index 50af816d8e..181229c8a7 100644
--- a/tensorflow/contrib/tensorrt/test/int32_test.py
+++ b/tensorflow/contrib/tensorrt/test/int32_test.py
@@ -25,7 +25,6 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
 from tensorflow.python.platform import test
@@ -44,8 +43,7 @@ class ExcludeUnsupportedInt32Test(trt_test.TfTrtIntegrationTestBase):
     dtype = dtypes.int32
     g = ops.Graph()
     with g.as_default():
-      x = array_ops.placeholder(
-          dtype=dtype, shape=input_dims, name=input_name)
+      x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name)
       b = self._ConstOp((4, 10), dtype)
       x = math_ops.matmul(x, b)
       b = self._ConstOp((10,), dtype)
@@ -79,5 +77,6 @@ class ExcludeUnsupportedInt32Test(trt_test.TfTrtIntegrationTestBase):
     # mode, which is a bug. Re-enable this when trt library is fixed.
     return not trt_test.IsQuantizationMode(run_params.precision_mode)
 
-if __name__ == "__main__":
+
+if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/tensorrt/test/topk_test.py b/tensorflow/contrib/tensorrt/test/topk_test.py
index efbfb26898..263ea0cfa1 100644
--- a/tensorflow/contrib/tensorrt/test/topk_test.py
+++ b/tensorflow/contrib/tensorrt/test/topk_test.py
@@ -40,18 +40,18 @@ class TopKTest(trt_test.TfTrtIntegrationTestBase):
       x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name)
       k_tensor = constant_op.constant(k, dtype=tf.int32, name="Const")
       values, indices = nn_ops.top_k(x, k_tensor, name="TopK")
-      values = array_ops.identity(values, name='output_values')
-      indices = array_ops.identity(indices, name='output_indices')
+      values = array_ops.identity(values, name="output_values")
+      indices = array_ops.identity(indices, name="output_indices")
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
         input_dims=[input_dims],
-        output_names=['output_values', 'output_indices'],
+        output_names=["output_values", "output_indices"],
         expected_output_dims=[(100, k), (100, k)])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
-    return {"TRTEngineOp_0" : ["Const", "TopK"]}
+    return {"TRTEngineOp_0": ["Const", "TopK"]}
 
 
 if __name__ == "__main__":
-- 
GitLab


From 5bb89f16124ebbcff38a7bd6ec404cfa54ec9b91 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 11:03:56 -0800
Subject: [PATCH 0655/2345] Cleaning up copying system files to execroot. Copy
 and use nvcc from execroot.

PiperOrigin-RevId: 229214994
---
 third_party/gpus/cuda/BUILD.tpl             |   2 +-
 third_party/gpus/cuda/BUILD.windows.tpl     |   2 +-
 third_party/gpus/cuda_configure.bzl         | 206 ++++++++------------
 third_party/gpus/rocm/BUILD.tpl             |   2 +-
 third_party/gpus/rocm_configure.bzl         | 131 ++++---------
 third_party/tensorrt/BUILD.tpl              |   2 +-
 third_party/tensorrt/tensorrt_configure.bzl |  45 +++--
 7 files changed, 157 insertions(+), 233 deletions(-)

diff --git a/third_party/gpus/cuda/BUILD.tpl b/third_party/gpus/cuda/BUILD.tpl
index f6b497f813..1921ef7c1f 100644
--- a/third_party/gpus/cuda/BUILD.tpl
+++ b/third_party/gpus/cuda/BUILD.tpl
@@ -202,4 +202,4 @@ cc_library(
     visibility = ["//visibility:public"],
 )
 
-%{cuda_include_genrules}
+%{copy_rules}
diff --git a/third_party/gpus/cuda/BUILD.windows.tpl b/third_party/gpus/cuda/BUILD.windows.tpl
index 325d18b9cb..3ed4fd415c 100644
--- a/third_party/gpus/cuda/BUILD.windows.tpl
+++ b/third_party/gpus/cuda/BUILD.windows.tpl
@@ -161,4 +161,4 @@ cc_library(
     visibility = ["//visibility:public"],
 )
 
-%{cuda_include_genrules}
+%{copy_rules}
diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl
index 8de4fe58e5..af8876ba17 100644
--- a/third_party/gpus/cuda_configure.bzl
+++ b/third_party/gpus/cuda_configure.bzl
@@ -1058,7 +1058,7 @@ def _create_dummy_repository(repository_ctx):
               _lib_name("curand", cpu_value),
           "%{cupti_lib}":
               _lib_name("cupti", cpu_value),
-          "%{cuda_include_genrules}":
+          "%{copy_rules}":
               "",
           "%{cuda_headers}":
               "",
@@ -1148,65 +1148,37 @@ def _norm_path(path):
     path = path[:-1]
   return path
 
-
-def symlink_genrule_for_dir(
-        repository_ctx,
-        src_dir,
-        dest_dir,
-        genrule_name,
-        src_files = [],
-        dest_files = []):
-  """Returns a genrule to symlink(or copy if on Windows) a set of files.
-
-    If src_dir is passed, files will be read from the given directory; otherwise
-    we assume files are in src_files and dest_files
-    """
-  if src_dir != None:
-    src_dir = _norm_path(src_dir)
-    dest_dir = _norm_path(dest_dir)
-    files = "\n".join(sorted(_read_dir(repository_ctx, src_dir).splitlines()))
-
-    # Create a list with the src_dir stripped to use for outputs.
-    dest_files = files.replace(src_dir, "").splitlines()
-    src_files = files.splitlines()
-  command = []
-  if not _is_windows(repository_ctx):
-    # We clear folders that might have been generated previously to avoid
-    # undesired inclusions
-    command.append('if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi')
-    command.append('if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi')
-    command.append('if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi')
-    command.append('if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi')
-  outs = []
-  for i in range(len(dest_files)):
-    if dest_files[i] != "":
-      # If we have only one file to link we do not want to use the dest_dir, as
-      # $(@D) will include the full path to the file.
-      dest = "$(@D)/" + dest_dir + dest_files[i] if len(
-          dest_files) != 1 else "$(@D)/" + dest_files[i]
-
-      # Copy the headers to create a sandboxable setup.
-      cmd = "cp -f"
-      command.append(cmd + ' "%s" "%s"' % (src_files[i], dest))
-      outs.append('        "' + dest_dir + dest_files[i] + '",')
-  genrule = _genrule(
-      src_dir,
-      genrule_name,
-      " && ".join(command),
-      "\n".join(outs),
-  )
-  return genrule
-
-
-def _genrule(src_dir, genrule_name, command, outs):
-  """Returns a string with a genrule.
-
-    Genrule executes the given command and produces the given outputs.
-    """
-  return (
-      "genrule(\n" + '    name = "' + genrule_name + '",\n' + "    outs = [\n" +
-      outs + "\n    ],\n" + '    cmd = """\n' + command + '\n   """,\n' + ")\n")
-
+def make_copy_files_rule(repository_ctx, name, srcs, outs):
+  """Returns a rule to copy a set of files."""
+  cmds = []
+  # Copy files.
+  for src, out in zip(srcs, outs):
+    cmds.append('cp -f "%s" $(location %s)' % (src, out))
+  outs = [('        "%s",' % out) for out in outs]
+  return """genrule(
+    name = "%s",
+    outs = [
+%s
+    ],
+    cmd = \"""%s \""",
+)""" % (name, "\n".join(outs), " && ".join(cmds))
+
+def make_copy_dir_rule(repository_ctx, name, src_dir, out_dir):
+  """Returns a rule to recursively copy a directory."""
+  src_dir = _norm_path(src_dir)
+  out_dir = _norm_path(out_dir)
+  outs = _read_dir(repository_ctx, src_dir)
+  outs = [('        "%s",' % out.replace(src_dir, out_dir)) for out in outs]
+  # '@D' already contains the relative path for a single file, see
+  # http://docs.bazel.build/versions/master/be/make-variables.html#predefined_genrule_variables
+  out_dir = "$(@D)/%s" % out_dir if len(outs) > 1 else "$(@D)"
+  return """genrule(
+    name = "%s",
+    outs = [
+%s
+    ],
+    cmd = \"""cp -rf "%s/." "%s/" \""",
+)""" % (name, "\n".join(outs), src_dir, out_dir)
 
 def _read_dir(repository_ctx, src_dir):
   """Returns a string with all files in a directory.
@@ -1233,7 +1205,7 @@ def _read_dir(repository_ctx, src_dir):
         empty_stdout_fine=True,
     )
     result = find_result.stdout
-  return result
+  return sorted(result.splitlines())
 
 
 def _flag_enabled(repository_ctx, flag_name):
@@ -1272,69 +1244,59 @@ def _create_local_cuda_repository(repository_ctx):
   cupti_header_dir = _find_cupti_header_dir(repository_ctx, cuda_config)
   nvvm_libdevice_dir = _find_nvvm_libdevice_dir(repository_ctx, cuda_config)
 
-  # Set up symbolic links for the cuda toolkit by creating genrules to do
-  # symlinking. We create one genrule for each directory we want to track under
-  # cuda_toolkit_path
-  cuda_toolkit_path = cuda_config.cuda_toolkit_path
-  genrules = [
-      symlink_genrule_for_dir(
+  # Create genrule to copy files from the installed CUDA toolkit into execroot.
+  copy_rules = [
+      make_copy_dir_rule(
           repository_ctx,
-          cuda_include_path,
-          "cuda/include",
-          "cuda-include",
-      )
-  ]
-  genrules.append(
-      symlink_genrule_for_dir(
+          name = "cuda-include",
+          src_dir = cuda_include_path,
+          out_dir = "cuda/include",
+      ),
+      make_copy_dir_rule(
           repository_ctx,
-          nvvm_libdevice_dir,
-          "cuda/nvvm/libdevice",
-          "cuda-nvvm",
-      ))
-  genrules.append(
-      symlink_genrule_for_dir(
+          name = "cuda-nvvm",
+          src_dir = nvvm_libdevice_dir,
+          out_dir = "cuda/nvvm/libdevice",
+      ),
+      make_copy_dir_rule(
           repository_ctx,
-          cupti_header_dir,
-          "cuda/extras/CUPTI/include",
-          "cuda-extras",
-      ))
+          name = "cuda-extras",
+          src_dir = cupti_header_dir,
+          out_dir = "cuda/extras/CUPTI/include",
+      ),
+  ]
 
   cuda_libs = _find_libs(repository_ctx, cuda_config)
-  cuda_lib_src = []
-  cuda_lib_dest = []
+  cuda_lib_srcs = []
+  cuda_lib_outs = []
   for lib in cuda_libs.values():
-    cuda_lib_src.append(lib.path)
-    cuda_lib_dest.append("cuda/lib/" + lib.file_name)
-  genrules.append(
-      symlink_genrule_for_dir(
-          repository_ctx,
-          None,
-          "",
-          "cuda-lib",
-          cuda_lib_src,
-          cuda_lib_dest,
-      ))
+    cuda_lib_srcs.append(lib.path)
+    cuda_lib_outs.append("cuda/lib/" + lib.file_name)
+  copy_rules.append(make_copy_files_rule(
+      repository_ctx,
+      name = "cuda-lib",
+      srcs = cuda_lib_srcs,
+      outs = cuda_lib_outs,
+  ))
+
+  copy_rules.append(make_copy_dir_rule(
+      repository_ctx,
+      name = "cuda-bin",
+      src_dir = cuda_config.cuda_toolkit_path + "/bin",
+      out_dir = "cuda/bin"
+  ))
 
-  # Set up the symbolic links for cudnn if cndnn was not installed to
-  # CUDA_TOOLKIT_PATH.
-  included_files = _read_dir(repository_ctx, cuda_include_path).replace(
-      cuda_include_path,
-      "",
-  ).splitlines()
-  if "/cudnn.h" not in included_files:
-    genrules.append(
-        symlink_genrule_for_dir(
-            repository_ctx,
-            None,
-            "cuda/include/",
-            "cudnn-include",
-            [cudnn_header_dir + "/cudnn.h"],
-            ["cudnn.h"],
-        ))
+  # Copy cudnn.h if cuDNN was not installed to CUDA_TOOLKIT_PATH.
+  included_files = _read_dir(repository_ctx, cuda_include_path)
+  if not any([file.endswith("cudnn.h") for file in included_files]):
+    copy_rules.append(make_copy_files_rule(
+        repository_ctx,
+        name = "cudnn-include",
+        srcs = [cudnn_header_dir + "/cudnn.h"],
+        outs = ["cuda/include/cudnn.h"],
+    ))
   else:
-    genrules.append(
-        "filegroup(\n" + '    name = "cudnn-include",\n' + "    srcs = [],\n" +
-        ")\n",)
+    copy_rules.append("filegroup(name = 'cudnn-include')\n")
 
   # Set up BUILD file for cuda/
   _tpl(
@@ -1374,8 +1336,8 @@ def _create_local_cuda_repository(repository_ctx):
               cuda_libs["curand"].file_name,
           "%{cupti_lib}":
               cuda_libs["cupti"].file_name,
-          "%{cuda_include_genrules}":
-              "\n".join(genrules),
+          "%{copy_rules}":
+              "\n".join(copy_rules),
           "%{cuda_headers}": ('":cuda-include",\n' + '        ":cudnn-include",'
                              ),
       },
@@ -1447,11 +1409,9 @@ def _create_local_cuda_repository(repository_ctx):
     # bazel's header check failing.
     cuda_defines["%{extra_no_canonical_prefixes_flags}"] = (
         "flag: \"-fno-canonical-system-headers\"")
-    nvcc_path = str(
-        repository_ctx.path("%s/bin/nvcc%s" % (
-            cuda_config.cuda_toolkit_path,
-            ".exe" if _is_windows(repository_ctx) else "",
-        )))
+    nvcc_path = "cuda/bin/nvcc"
+    if _is_windows(repository_ctx):
+      nvcc_path += ".exe"
     _tpl(
         repository_ctx,
         "crosstool:BUILD",
diff --git a/third_party/gpus/rocm/BUILD.tpl b/third_party/gpus/rocm/BUILD.tpl
index 8258bb3589..655acaec97 100644
--- a/third_party/gpus/rocm/BUILD.tpl
+++ b/third_party/gpus/rocm/BUILD.tpl
@@ -96,4 +96,4 @@ cc_library(
     ],
 )
 
-%{rocm_include_genrules}
+%{copy_rules}
diff --git a/third_party/gpus/rocm_configure.bzl b/third_party/gpus/rocm_configure.bzl
index 6df6799bd7..a57ec9f7a6 100644
--- a/third_party/gpus/rocm_configure.bzl
+++ b/third_party/gpus/rocm_configure.bzl
@@ -14,6 +14,12 @@
     `gfx803,gfx900`.
 """
 
+load(
+    ":cuda_configure.bzl",
+    "make_copy_dir_rule",
+    "make_copy_files_rule",
+)
+
 _GCC_HOST_COMPILER_PATH = "GCC_HOST_COMPILER_PATH"
 _ROCM_TOOLKIT_PATH = "ROCM_TOOLKIT_PATH"
 _TF_ROCM_VERSION = "TF_ROCM_VERSION"
@@ -445,7 +451,7 @@ def _create_dummy_repository(repository_ctx):
             "%{miopen_lib}": _lib_name("miopen", cpu_value),
             "%{rocfft_lib}": _lib_name("rocfft", cpu_value),
             "%{hiprand_lib}": _lib_name("hiprand", cpu_value),
-            "%{rocm_include_genrules}": "",
+            "%{copy_rules}": "",
             "%{rocm_headers}": "",
         },
     )
@@ -510,51 +516,6 @@ def _norm_path(path):
         path = path[:-1]
     return path
 
-def _symlink_genrule_for_dir(
-        repository_ctx,
-        src_dir,
-        dest_dir,
-        genrule_name,
-        src_files = [],
-        dest_files = []):
-    """Returns a genrule to symlink(or copy if on Windows) a set of files.
-
-    If src_dir is passed, files will be read from the given directory; otherwise
-    we assume files are in src_files and dest_files
-    """
-    if src_dir != None:
-        src_dir = _norm_path(src_dir)
-        dest_dir = _norm_path(dest_dir)
-        files = _read_dir(repository_ctx, src_dir)
-
-        # Create a list with the src_dir stripped to use for outputs.
-        dest_files = files.replace(src_dir, "").splitlines()
-        src_files = files.splitlines()
-    command = []
-
-    # We clear folders that might have been generated previously to avoid
-    # undesired inclusions
-    command.append('if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi')
-    command.append('if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi')
-    outs = []
-    for i in range(len(dest_files)):
-        if dest_files[i] != "":
-            # If we have only one file to link we do not want to use the dest_dir, as
-            # $(@D) will include the full path to the file.
-            dest = "$(@D)/" + dest_dir + dest_files[i] if len(dest_files) != 1 else "$(@D)/" + dest_files[i]
-
-            # On Windows, symlink is not supported, so we just copy all the files.
-            cmd = "ln -s"
-            command.append(cmd + ' "%s" "%s"' % (src_files[i], dest))
-            outs.append('        "' + dest_dir + dest_files[i] + '",')
-    genrule = _genrule(
-        src_dir,
-        genrule_name,
-        " && ".join(command),
-        "\n".join(outs),
-    )
-    return genrule
-
 def _genrule(src_dir, genrule_name, command, outs):
     """Returns a string with a genrule.
 
@@ -601,55 +562,49 @@ def _create_local_rocm_repository(repository_ctx):
     """Creates the repository containing files set up to build with ROCm."""
     rocm_config = _get_rocm_config(repository_ctx)
 
-    # Set up symbolic links for the rocm toolkit by creating genrules to do
-    # symlinking. We create one genrule for each directory we want to track under
+    # Copy header and library files to execroot.
     # rocm_toolkit_path
     rocm_toolkit_path = rocm_config.rocm_toolkit_path
-    rocm_include_path = rocm_toolkit_path + "/include"
-    genrules = [_symlink_genrule_for_dir(
-        repository_ctx,
-        rocm_include_path,
-        "rocm/include",
-        "rocm-include",
-    )]
-    genrules.append(_symlink_genrule_for_dir(
-        repository_ctx,
-        rocm_toolkit_path + "/rocfft/include",
-        "rocm/include/rocfft",
-        "rocfft-include",
-    ))
-    genrules.append(_symlink_genrule_for_dir(
-        repository_ctx,
-        rocm_toolkit_path + "/rocblas/include",
-        "rocm/include/rocblas",
-        "rocblas-include",
-    ))
-    genrules.append(_symlink_genrule_for_dir(
-        repository_ctx,
-        rocm_toolkit_path + "/miopen/include",
-        "rocm/include/miopen",
-        "miopen-include",
-    ))
+    copy_rules = [
+        make_copy_dir_rule(
+            repository_ctx,
+            name = "rocm-include",
+            src_dir = rocm_toolkit_path + "/include",
+            out_dir = "rocm/include",
+        ),
+        make_copy_dir_rule(
+            repository_ctx,
+            name = "rocfft-include",
+            src_dir = rocm_toolkit_path + "/rocfft/include",
+            out_dir = "rocm/include/rocfft",
+        ),
+        make_copy_dir_rule(
+            repository_ctx,
+            name = "rocm-blas",
+            src_dir = rocm_toolkit_path + "/rocblas/include",
+            out_dir = "rocm/include/rocblas",
+        ),
+        make_copy_dir_rule(
+            repository_ctx,
+            name = "rocm-miopen",
+            src_dir = rocm_toolkit_path + "/miopen/include",
+            out_dir = "rocm/include/miopen",
+        ),
+    ]
 
     rocm_libs = _find_libs(repository_ctx, rocm_config)
-    rocm_lib_src = []
-    rocm_lib_dest = []
+    rocm_lib_srcs = []
+    rocm_lib_outs = []
     for lib in rocm_libs.values():
-        rocm_lib_src.append(lib.path)
-        rocm_lib_dest.append("rocm/lib/" + lib.file_name)
-    genrules.append(_symlink_genrule_for_dir(
+        rocm_lib_srcs.append(lib.path)
+        rocm_lib_outs.append("rocm/lib/" + lib.file_name)
+    copy_rules.append(make_copy_files_rule(
         repository_ctx,
-        None,
-        "",
-        "rocm-lib",
-        rocm_lib_src,
-        rocm_lib_dest,
+        name = "rocm-lib",
+        srcs = rocm_lib_srcs,
+        outs = rocm_lib_outs,
     ))
 
-    included_files = _read_dir(repository_ctx, rocm_include_path).replace(
-        rocm_include_path,
-        "",
-    ).splitlines()
 
     # Set up BUILD file for rocm/
     _tpl(
@@ -672,7 +627,7 @@ def _create_local_rocm_repository(repository_ctx):
             "%{rocfft_lib}": rocm_libs["rocfft"].file_name,
             "%{hiprand_lib}": rocm_libs["hiprand"].file_name,
             "%{miopen_lib}": rocm_libs["miopen"].file_name,
-            "%{rocm_include_genrules}": "\n".join(genrules),
+            "%{copy_rules}": "\n".join(copy_rules),
             "%{rocm_headers}": ('":rocm-include",\n' +
                                 '":rocfft-include",\n' +
                                 '":rocblas-include",\n' +
diff --git a/third_party/tensorrt/BUILD.tpl b/third_party/tensorrt/BUILD.tpl
index a2c30b8b94..17ab2fa119 100644
--- a/third_party/tensorrt/BUILD.tpl
+++ b/third_party/tensorrt/BUILD.tpl
@@ -29,5 +29,5 @@ cc_library(
 )
 
 
-%{tensorrt_genrules}
+%{copy_rules}
 
diff --git a/third_party/tensorrt/tensorrt_configure.bzl b/third_party/tensorrt/tensorrt_configure.bzl
index 77ee6622d1..d1e116d9dd 100644
--- a/third_party/tensorrt/tensorrt_configure.bzl
+++ b/third_party/tensorrt/tensorrt_configure.bzl
@@ -13,7 +13,8 @@ load(
     "get_cpu_value",
     "find_cuda_define",
     "matches_version",
-    "symlink_genrule_for_dir",
+    "make_copy_dir_rule",
+    "make_copy_files_rule",
 )
 
 _TENSORRT_INSTALL_PATH = "TENSORRT_INSTALL_PATH"
@@ -177,35 +178,43 @@ def _tensorrt_configure_impl(repository_ctx):
     auto_configure_fail(
         "Cannot find TensorRT install path %s." % trt_install_path)
 
-  # Set up the symbolic links for the library files.
+  # Copy the library files.
   trt_lib_version = _trt_lib_version(repository_ctx, trt_install_path)
   trt_libs = _find_trt_libs(repository_ctx, trt_install_path, trt_lib_version)
-  trt_lib_src = []
-  trt_lib_dest = []
+  trt_lib_srcs = []
+  trt_lib_outs = []
   for lib in trt_libs.values():
-    trt_lib_src.append(lib.src_file_path)
-    trt_lib_dest.append(lib.dst_file_name)
-  genrules = [
-      symlink_genrule_for_dir(repository_ctx, None, "tensorrt/lib/",
-                              "tensorrt_lib", trt_lib_src, trt_lib_dest)
-  ]
-
-  # Set up the symbolic links for the header files.
+    trt_lib_srcs.append(lib.src_file_path)
+    trt_lib_outs.append("tensorrt/lib/" + lib.dst_file_name)
+  copy_rules = [make_copy_files_rule(
+      repository_ctx,
+      name = "tensorrt_lib",
+      srcs = trt_lib_srcs,
+      outs = trt_lib_outs,
+  )]
+
+  # Copy the header files header files.
   trt_header_dir = _find_trt_header_dir(repository_ctx, trt_install_path)
-  src_files = [
+  trt_header_srcs = [
       "%s/%s" % (trt_header_dir, header) for header in _TF_TENSORRT_HEADERS
   ]
-  dest_files = _TF_TENSORRT_HEADERS
-  genrules.append(
-      symlink_genrule_for_dir(repository_ctx, None, "tensorrt/include/",
-                              "tensorrt_include", src_files, dest_files))
+  trt_header_outs = [
+      "tensorrt/include/" + header for header in _TF_TENSORRT_HEADERS
+  ]
+  copy_rules.append(
+      make_copy_files_rule(
+          repository_ctx,
+          name = "tensorrt_include",
+          srcs = trt_header_srcs,
+          outs = trt_header_outs,
+  ))
 
   # Set up config file.
   _tpl(repository_ctx, "build_defs.bzl", {"%{tensorrt_is_configured}": "True"})
 
   # Set up BUILD file.
   substitutions = {
-      "%{tensorrt_genrules}": "\n".join(genrules),
+      "%{copy_rules}": "\n".join(copy_rules),
       "%{tensorrt_headers}": '":tensorrt_include"',
   }
   for lib in _TF_TENSORRT_LIBS:
-- 
GitLab


From adf8dac492adb33d4c71a4271f6f003a156eae22 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 11:18:16 -0800
Subject: [PATCH 0656/2345] Implicitly initialize platforms in
 PlatformUtil::GetSupportedPlatforms.

PiperOrigin-RevId: 229218011
---
 tensorflow/compiler/xla/service/platform_util.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/compiler/xla/service/platform_util.cc b/tensorflow/compiler/xla/service/platform_util.cc
index 0491f641fc..886a054562 100644
--- a/tensorflow/compiler/xla/service/platform_util.cc
+++ b/tensorflow/compiler/xla/service/platform_util.cc
@@ -70,6 +70,9 @@ PlatformUtil::GetSupportedPlatforms() {
   for (se::Platform* platform : all_platforms) {
     auto compiler_status = Compiler::GetForPlatform(platform);
     if (compiler_status.ok()) {
+      if (!platform->Initialized()) {
+        TF_RETURN_IF_ERROR(platform->Initialize({}));
+      }
       platforms.push_back(platform);
     } else {
       LOG(INFO) << "platform " << platform->Name() << " present but no "
-- 
GitLab


From 2d58c2502f8971bc9af65c4f8270512f7c53fd2a Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Mon, 14 Jan 2019 11:19:51 -0800
Subject: [PATCH 0657/2345] Modify flaky topology_test

PiperOrigin-RevId: 229218301
---
 tensorflow/python/keras/engine/topology_test.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/engine/topology_test.py b/tensorflow/python/keras/engine/topology_test.py
index 08d21193a0..9022dc9481 100644
--- a/tensorflow/python/keras/engine/topology_test.py
+++ b/tensorflow/python/keras/engine/topology_test.py
@@ -1228,8 +1228,12 @@ class NestedNetworkTest(test.TestCase):
     result = self.evaluate(result_tensor)
     self.assertAllEqual(result, [[2.]])
 
-    output_shape = network.compute_output_shape({'x1': (10, 1), 'x2': (10, 1)})
-    self.assertListEqual(output_shape.as_list(), [10, 1])
+    # TODO(b/122726584): Investigate why concrete batch is flaky in some builds.
+    output_shape = network.compute_output_shape({
+        'x1': (None, 1),
+        'x2': (None, 1)
+    })
+    self.assertListEqual(output_shape.as_list(), [None, 1])
 
   def test_nested_outputs_network(self):
     inputs = keras.Input(shape=(1,))
-- 
GitLab


From f96ef427440f7fe15cb2b493a8de25202fa96cc5 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Mon, 14 Jan 2019 11:20:04 -0800
Subject: [PATCH 0658/2345] Toco FuseBinaryIntoFollowingAffine bugfix

A binary op can't be fused into a following affine op if
the output is consumed by multiple ops.

PiperOrigin-RevId: 229218339
---
 .../fuse_binary_into_following_affine.cc      |  10 +-
 .../toco/graph_transformations/tests/BUILD    |  12 ++
 .../fuse_binary_into_following_affine_test.cc | 148 ++++++++++++++++++
 3 files changed, 167 insertions(+), 3 deletions(-)
 create mode 100644 tensorflow/lite/toco/graph_transformations/tests/fuse_binary_into_following_affine_test.cc

diff --git a/tensorflow/lite/toco/graph_transformations/fuse_binary_into_following_affine.cc b/tensorflow/lite/toco/graph_transformations/fuse_binary_into_following_affine.cc
index 436b639253..9ea8d8fa5b 100644
--- a/tensorflow/lite/toco/graph_transformations/fuse_binary_into_following_affine.cc
+++ b/tensorflow/lite/toco/graph_transformations/fuse_binary_into_following_affine.cc
@@ -218,6 +218,12 @@ void FuseMulOrDivParamsIntoFollowingAffine(Model* model, Operator* following_op,
     return ::tensorflow::Status::OK();
   }
 
+  if (CountOpsWithInput(*model, binary_op->outputs[0]) != 1) {
+    AddMessageF("Not fusing %s because it's consumed by multiple ops",
+                LogName(*binary_op));
+    return ::tensorflow::Status::OK();
+  }
+
   Operator* following_op = GetOpWithInput(*model, binary_op->outputs[0]);
 
   if (!following_op) {
@@ -287,9 +293,7 @@ void FuseMulOrDivParamsIntoFollowingAffine(Model* model, Operator* following_op,
   AddMessageF("Fusing %s into the following %s", LogName(*binary_op),
               LogName(*following_op));
 
-  if (CountOpsWithInput(*model, binary_op->outputs[0]) == 1) {
-    model->EraseArray(binary_op->outputs[0]);
-  }
+  model->EraseArray(binary_op->outputs[0]);
 
   following_op->inputs[0] = binary_op->inputs[index_of_variable_input];
   const auto& old_constant_param_name =
diff --git a/tensorflow/lite/toco/graph_transformations/tests/BUILD b/tensorflow/lite/toco/graph_transformations/tests/BUILD
index bbbedbe3a9..03d331226d 100644
--- a/tensorflow/lite/toco/graph_transformations/tests/BUILD
+++ b/tensorflow/lite/toco/graph_transformations/tests/BUILD
@@ -40,3 +40,15 @@ tf_cc_test(
         "@com_google_googletest//:gtest_main",
     ],
 )
+
+tf_cc_test(
+    name = "fuse_binary_into_following_affine_test",
+    srcs = ["fuse_binary_into_following_affine_test.cc"],
+    deps = [
+        "//tensorflow/lite/toco:graph_transformations",
+        "//tensorflow/lite/toco:model",
+        "//tensorflow/lite/toco:tooling_util",
+        "@com_google_absl//absl/memory",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
diff --git a/tensorflow/lite/toco/graph_transformations/tests/fuse_binary_into_following_affine_test.cc b/tensorflow/lite/toco/graph_transformations/tests/fuse_binary_into_following_affine_test.cc
new file mode 100644
index 0000000000..2cba6824cf
--- /dev/null
+++ b/tensorflow/lite/toco/graph_transformations/tests/fuse_binary_into_following_affine_test.cc
@@ -0,0 +1,148 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+#include "tensorflow/lite/toco/graph_transformations/graph_transformations.h"
+#include "tensorflow/lite/toco/model.h"
+#include "tensorflow/lite/toco/tooling_util.h"
+
+namespace toco {
+
+namespace {
+// A gmock matcher that check that elements of a float vector match to a given
+// tolerance.
+std::vector<testing::Matcher<float>> ArrayFloatNear(
+    const std::vector<float>& values, float max_abs_error = 1e-5) {
+  std::vector<testing::Matcher<float>> matchers;
+  matchers.reserve(values.size());
+  for (const float& v : values) {
+    matchers.emplace_back(testing::FloatNear(v, max_abs_error));
+  }
+  return matchers;
+}
+}  // namespace
+
+class FuseBinaryIntoFollowingAffineTest : public ::testing::Test {
+ protected:
+  FuseBinaryIntoFollowingAffineTest() {}
+
+  void SetUp() override { model_.reset(new Model); }
+
+  void CreateArray(const string& name, const std::vector<int>& shape) {
+    Array& array = model_->GetOrCreateArray(name);
+    array.data_type = ArrayDataType::kFloat;
+    Shape* array_shape = array.mutable_shape();
+    *(array_shape->mutable_dims()) = shape;
+  }
+
+  void CreateConstantArray(const string& name, const std::vector<int>& shape,
+                           const std::vector<float>& data) {
+    CreateArray(name, shape);
+    Array& array = model_->GetOrCreateArray(name);
+    auto& array_buffer = array.GetMutableBuffer<ArrayDataType::kFloat>();
+    int bufsize = 1;
+    for (int dim : shape) {
+      bufsize *= dim;
+    }
+    array_buffer.data.resize(bufsize);
+    float* buf_ptr = array_buffer.data.data();
+    for (int i = 0; i < bufsize; ++i) {
+      buf_ptr[i] = data[i];
+    }
+  }
+
+  std::unique_ptr<Model> model_;
+};
+
+TEST_F(FuseBinaryIntoFollowingAffineTest, FuseMulIntoFullyConnected) {
+  // Creating a model.
+  {
+    CreateArray("Input", {2, 2});
+    CreateConstantArray("MulInput2", {1}, {2.0});
+    CreateArray("MulOutput", {2, 2});
+    CreateConstantArray("FCWeight", {2, 2}, {1.0, 2.0, 3.0, 4.0});
+    CreateConstantArray("FCBias", {1}, {1.0});
+    CreateArray("Output", {2, 2});
+
+    auto* mul_op = new MulOperator;
+    mul_op->inputs = {"Input", "MulInput2"};
+    mul_op->outputs = {"MulOutput"};
+    model_->operators.push_back(std::unique_ptr<Operator>(mul_op));
+
+    auto* fc_op = new FullyConnectedOperator;
+    fc_op->inputs = {"MulOutput", "FCWeight", "FCBias"};
+    fc_op->outputs = {"Output"};
+    model_->operators.push_back(std::unique_ptr<Operator>(fc_op));
+  }
+  toco::FuseBinaryIntoFollowingAffine transformation;
+  bool modified;
+  ASSERT_TRUE(transformation.Run(model_.get(), /*op_index=*/0, &modified).ok());
+  EXPECT_TRUE(modified);
+
+  // `Mul` should be fused into `FullyConnected`. Only 1 op is left.
+  ASSERT_EQ(model_->operators.size(), 1);
+  const auto& op = model_->operators[0];
+  ASSERT_EQ(op->type, OperatorType::kFullyConnected);
+  ASSERT_EQ(op->inputs.size(), 3);
+
+  auto& weights_array = model_->GetArray(op->inputs[1]);
+  EXPECT_THAT(weights_array.GetBuffer<toco::ArrayDataType::kFloat>().data,
+              ElementsAreArray(ArrayFloatNear({2.0, 4.0, 6.0, 8.0})));
+
+  auto& bias_array = model_->GetArray(op->inputs[2]);
+  EXPECT_THAT(bias_array.GetBuffer<toco::ArrayDataType::kFloat>().data,
+              ElementsAreArray(ArrayFloatNear({1.0})));
+}
+
+// This is a regression test of b/121287325. Toco crashes before the fix.
+TEST_F(FuseBinaryIntoFollowingAffineTest, DoNotFuseWithMultipleConsumers) {
+  // Creating a model.
+  {
+    CreateArray("Input", {2, 2});
+    CreateConstantArray("MulInput2", {1}, {2.0});
+    CreateArray("MulOutput", {2, 2});
+    CreateConstantArray("FCWeight", {2, 2}, {1.0, 2.0, 3.0, 4.0});
+    CreateConstantArray("FCBias", {1}, {1.0});
+    CreateArray("Output", {2, 2});
+    CreateArray("AnotherOutput", {2, 2});
+
+    auto* mul_op = new MulOperator;
+    mul_op->inputs = {"Input", "MulInput2"};
+    mul_op->outputs = {"MulOutput"};
+    model_->operators.push_back(std::unique_ptr<Operator>(mul_op));
+
+    auto* fc_op = new FullyConnectedOperator;
+    fc_op->inputs = {"MulOutput", "FCWeight", "FCBias"};
+    fc_op->outputs = {"Output"};
+    model_->operators.push_back(std::unique_ptr<Operator>(fc_op));
+
+    auto identity_op = new TensorFlowIdentityOperator;
+    identity_op->inputs = {"MulOutput"};
+    identity_op->outputs = {"AnotherOutput"};
+    model_->operators.push_back(std::unique_ptr<Operator>(identity_op));
+  }
+
+  toco::FuseBinaryIntoFollowingAffine transformation;
+  bool modified;
+  ASSERT_TRUE(transformation.Run(model_.get(), /*op_index=*/0, &modified).ok());
+  // Do not modify the graph if the binary operator has another output.
+  EXPECT_FALSE(modified);
+  EXPECT_EQ(model_->operators.size(), 3);
+}
+
+}  // namespace toco
-- 
GitLab


From a05eec6f5a6f6efcd9847719e32d2e34197457d8 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Mon, 14 Jan 2019 11:22:15 -0800
Subject: [PATCH 0659/2345] Better error message when using test session in
 eager mode.

PiperOrigin-RevId: 229218872
---
 tensorflow/python/framework/test_util.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index 22cd50321f..e2b906e37c 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -1466,6 +1466,18 @@ def disable_all_xla(description):
   return disable_all_impl
 
 
+class EagerSessionWarner(object):
+
+  def __getattr__(self, attr):
+    raise AttributeError(
+        "Trying to access properties or call methods on the result of "
+        "self.session(), self.cached_session(), etc while eager execution "
+        "is enabled. If you're porting this test case to TF 2.0, either "
+        "adapt the test to work with eager execution or insert a call to "
+        "tf.disable_eager_execution() in the main() function of this test "
+        "file.")
+
+
 @tf_export("test.TestCase")
 class TensorFlowTestCase(googletest.TestCase):
   """Base class for tests that need to test TensorFlow."""
@@ -1714,7 +1726,7 @@ class TensorFlowTestCase(googletest.TestCase):
       the graph building and execution code in a test case.
     """
     if context.executing_eagerly():
-      yield None
+      yield EagerSessionWarner()
     else:
       with self._create_session(graph, config, force_gpu) as sess:
         with self._constrain_devices_and_set_default(sess, use_gpu, force_gpu):
-- 
GitLab


From 4c6b506694ef03f1ff527d6330b10afc056287eb Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Mon, 14 Jan 2019 11:24:27 -0800
Subject: [PATCH 0660/2345] [XLA] Move more XlaBuilder users over to the new
 DS/DUS API

Move more tests and bridge op implementations over to the new
span-of-scalars API for DynamicSlice and DynamicUpdateSlice.

PiperOrigin-RevId: 229219316
---
 .../tf2xla/kernels/strided_slice_op.cc        |  17 +--
 .../compiler/xla/tests/dynamic_ops_test.cc    | 142 +++++++++++-------
 tensorflow/compiler/xla/tests/while_test.cc   |   6 +-
 3 files changed, 100 insertions(+), 65 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/strided_slice_op.cc b/tensorflow/compiler/tf2xla/kernels/strided_slice_op.cc
index 10d990b321..2273b59246 100644
--- a/tensorflow/compiler/tf2xla/kernels/strided_slice_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/strided_slice_op.cc
@@ -288,19 +288,21 @@ class StridedSliceAssignOp : public XlaOpKernel {
     xla::XlaOp rhs = ctx->Input(4);
 
     absl::InlinedVector<int64, 4> dimensions_to_reverse;
-    absl::InlinedVector<int64, 4> slice_begin, slice_dims;
+    absl::InlinedVector<xla::XlaOp, 4> slice_begin;
+    absl::InlinedVector<int64, 4> slice_dims;
     for (int i = 0; i < begin.size(); ++i) {
       // TODO(phawkins): implement strides != 1
       OP_REQUIRES(
           ctx, strides[i] == 1 || strides[i] == -1,
           errors::Unimplemented("Strides != 1 or -1 are not yet implemented"));
       if (strides[i] > 0) {
-        slice_begin.push_back(begin[i]);
+        slice_begin.push_back(xla::ConstantR0<int64>(ctx->builder(), begin[i]));
         slice_dims.push_back(end[i] - begin[i]);
       } else {
         // Negative stride: swap begin and end, add 1 because the interval
         // is semi-open, and mark the dimension to be reversed.
-        slice_begin.push_back(end[i] + 1);
+        slice_begin.push_back(
+            xla::ConstantR0<int64>(ctx->builder(), end[i] + 1));
         slice_dims.push_back(begin[i] - end[i]);
         dimensions_to_reverse.push_back(i);
       }
@@ -311,14 +313,7 @@ class StridedSliceAssignOp : public XlaOpKernel {
     }
     rhs = xla::Reshape(rhs, slice_dims);
 
-    if (lhs_shape.dims() == 0) {
-      // TODO(b/38323843): DynamicUpdateSlice crashes on rank 0 inputs. Fix
-      // and remove this workaround.
-      lhs = rhs;
-    } else {
-      lhs = xla::DynamicUpdateSlice(
-          lhs, rhs, xla::ConstantR1<int64>(ctx->builder(), slice_begin));
-    }
+    lhs = xla::DynamicUpdateSlice(lhs, rhs, slice_begin);
 
     OP_REQUIRES_OK(ctx, ctx->AssignVariable(0, dtype_, lhs));
   }
diff --git a/tensorflow/compiler/xla/tests/dynamic_ops_test.cc b/tensorflow/compiler/xla/tests/dynamic_ops_test.cc
index 7501c6d957..82e2db3614 100644
--- a/tensorflow/compiler/xla/tests/dynamic_ops_test.cc
+++ b/tensorflow/compiler/xla/tests/dynamic_ops_test.cc
@@ -135,11 +135,11 @@ class DynamicSliceTest : public ClientLibraryTestBase {
     XlaBuilder builder(TestName());
     // Initialize and transfer dynamic slice start indices parameter.
     XlaOp starts;
-    std::unique_ptr<GlobalData> start_data = CreateR1Parameter<IndexT>(
-        slice_starts, 0, "slice_starts", &builder, &starts);
+    std::unique_ptr<GlobalData> start_data = CreateR0Parameter<IndexT>(
+        slice_starts[0], 0, "slice_starts", &builder, &starts);
     // Build dynamic slice computation.
     auto input = ConstantLiteral(&builder, input_values);
-    DynamicSlice(input, starts, slice_sizes);
+    DynamicSlice(input, absl::Span<const XlaOp>({starts}), slice_sizes);
     // Run computation and compare against expected values.
     ComputeAndCompareLiteral(&builder, expected_values, {start_data.get()});
   }
@@ -160,14 +160,23 @@ class DynamicSliceTest : public ClientLibraryTestBase {
 
     XlaBuilder builder(TestName());
     // Initialize and transfer dynamic slice start indices parameter.
-    XlaOp starts;
-    std::unique_ptr<GlobalData> start_data = CreateR1Parameter<IndexT>(
-        slice_starts, 0, "slice_starts", &builder, &starts);
+    std::vector<XlaOp> starts(2);
+    std::vector<std::unique_ptr<GlobalData>> start_data(2);
+    for (int i = 0; i < 2; ++i) {
+      start_data[i] = CreateR0Parameter<IndexT>(
+          slice_starts[i], i, "slice_starts", &builder, &starts[i]);
+    }
+
     // Build dynamic slice computation.
     auto input = ConstantLiteral(&builder, input_values);
     DynamicSlice(input, starts, slice_sizes);
     // Run computation and compare against expected values.
-    ComputeAndCompareLiteral(&builder, expected_values, {start_data.get()});
+    std::vector<GlobalData*> argument_ptrs;
+    absl::c_transform(start_data, std::back_inserter(argument_ptrs),
+                      [](const std::unique_ptr<GlobalData>& argument) {
+                        return argument.get();
+                      });
+    ComputeAndCompareLiteral(&builder, expected_values, argument_ptrs);
   }
 
   template <typename IndexT, typename DataT>
@@ -186,14 +195,22 @@ class DynamicSliceTest : public ClientLibraryTestBase {
 
     XlaBuilder builder(TestName());
     // Initialize and transfer dynamic slice start indices parameter.
-    XlaOp starts;
-    std::unique_ptr<GlobalData> start_data = CreateR1Parameter<IndexT>(
-        slice_starts, 0, "slice_starts", &builder, &starts);
+    std::vector<XlaOp> starts(3);
+    std::vector<std::unique_ptr<GlobalData>> start_data(3);
+    for (int i = 0; i < 3; ++i) {
+      start_data[i] = CreateR0Parameter<IndexT>(
+          slice_starts[i], i, "slice_starts", &builder, &starts[i]);
+    }
     // Build dynamic slice computation.
     auto input = ConstantLiteral(&builder, input_values);
     DynamicSlice(input, starts, slice_sizes);
     // Run computation and compare against expected values.
-    ComputeAndCompareLiteral(&builder, expected_values, {start_data.get()});
+    std::vector<GlobalData*> argument_ptrs;
+    absl::c_transform(start_data, std::back_inserter(argument_ptrs),
+                      [](const std::unique_ptr<GlobalData>& argument) {
+                        return argument.get();
+                      });
+    ComputeAndCompareLiteral(&builder, expected_values, argument_ptrs);
   }
 };
 
@@ -372,16 +389,12 @@ class DynamicUpdateSliceTest : public ClientLibraryTestBase {
                       .ValueOrDie());
 
     XlaBuilder builder(TestName());
-    // Initialize and transfer dynamic slice start indices parameter.
-    XlaOp starts;
-    std::unique_ptr<GlobalData> start_data = CreateR1Parameter<IndexT>(
-        slice_starts, 0, "slice_starts", &builder, &starts);
     // Build dynamic slice computation.
     auto input = ConstantLiteral(&builder, input_value);
     auto update = ConstantLiteral(&builder, update_value);
-    DynamicUpdateSlice(input, update, starts);
+    DynamicUpdateSlice(input, update, absl::Span<const XlaOp>({}));
     // Run computation and compare against expected values.
-    ComputeAndCompareLiteral(&builder, expected_value, {start_data.get()});
+    ComputeAndCompareLiteral(&builder, expected_value, {});
   }
 
   template <typename IndexT, typename DataT>
@@ -405,12 +418,12 @@ class DynamicUpdateSliceTest : public ClientLibraryTestBase {
     XlaBuilder builder(TestName());
     // Initialize and transfer dynamic slice start indices parameter.
     XlaOp starts;
-    std::unique_ptr<GlobalData> start_data = CreateR1Parameter<IndexT>(
-        slice_starts, 0, "slice_starts", &builder, &starts);
+    std::unique_ptr<GlobalData> start_data = CreateR0Parameter<IndexT>(
+        slice_starts[0], 0, "slice_starts", &builder, &starts);
     // Build dynamic slice computation.
     auto input = ConstantLiteral(&builder, input_values);
     auto update = ConstantLiteral(&builder, update_values);
-    DynamicUpdateSlice(input, update, starts);
+    DynamicUpdateSlice(input, update, absl::Span<const XlaOp>({starts}));
     // Run computation and compare against expected values.
     ComputeAndCompareLiteral(&builder, expected_values, {start_data.get()});
   }
@@ -435,15 +448,23 @@ class DynamicUpdateSliceTest : public ClientLibraryTestBase {
 
     XlaBuilder builder(TestName());
     // Initialize and transfer dynamic slice start indices parameter.
-    XlaOp starts;
-    std::unique_ptr<GlobalData> start_data = CreateR1Parameter<IndexT>(
-        slice_starts, 0, "slice_starts", &builder, &starts);
+    std::vector<XlaOp> starts(2);
+    std::vector<std::unique_ptr<GlobalData>> start_data(2);
+    for (int i = 0; i < 2; ++i) {
+      start_data[i] = CreateR0Parameter<IndexT>(
+          slice_starts[i], i, "slice_starts", &builder, &starts[i]);
+    }
     // Build dynamic slice computation.
     auto input = ConstantLiteral(&builder, input_values);
     auto update = ConstantLiteral(&builder, update_values);
     DynamicUpdateSlice(input, update, starts);
     // Run computation and compare against expected values.
-    ComputeAndCompareLiteral(&builder, expected_values, {start_data.get()});
+    std::vector<GlobalData*> argument_ptrs;
+    absl::c_transform(start_data, std::back_inserter(argument_ptrs),
+                      [](const std::unique_ptr<GlobalData>& argument) {
+                        return argument.get();
+                      });
+    ComputeAndCompareLiteral(&builder, expected_values, argument_ptrs);
   }
 
   template <typename IndexT, typename DataT>
@@ -466,15 +487,24 @@ class DynamicUpdateSliceTest : public ClientLibraryTestBase {
 
     XlaBuilder builder(TestName());
     // Initialize and transfer dynamic slice start indices parameter.
-    XlaOp starts;
-    std::unique_ptr<GlobalData> start_data = CreateR1Parameter<IndexT>(
-        slice_starts, 0, "slice_starts", &builder, &starts);
+    std::vector<XlaOp> starts(3);
+    std::vector<std::unique_ptr<GlobalData>> start_data(3);
+    for (int i = 0; i < 3; ++i) {
+      start_data[i] = CreateR0Parameter<IndexT>(
+          slice_starts[i], i, "slice_starts", &builder, &starts[i]);
+    }
+
     // Build dynamic slice computation.
     auto input = ConstantLiteral(&builder, input_values);
     auto update = ConstantLiteral(&builder, update_values);
     DynamicUpdateSlice(input, update, starts);
     // Run computation and compare against expected values.
-    ComputeAndCompareLiteral(&builder, expected_values, {start_data.get()});
+    std::vector<GlobalData*> argument_ptrs;
+    absl::c_transform(start_data, std::back_inserter(argument_ptrs),
+                      [](const std::unique_ptr<GlobalData>& argument) {
+                        return argument.get();
+                      });
+    ComputeAndCompareLiteral(&builder, expected_values, argument_ptrs);
   }
 
   template <class T>
@@ -518,8 +548,9 @@ class DynamicUpdateSliceTest : public ClientLibraryTestBase {
     XlaOp update;
     std::unique_ptr<GlobalData> update_data = CreateR3Parameter<T>(
         update_values, 1, "update_values", &builder, &update);
-    auto starts = ConstantR1<int32>(&builder, {index, 0, 0});
-    DynamicUpdateSlice(input, update, starts);
+    auto constant_index = ConstantR0<int32>(&builder, index);
+    auto zero = ConstantR0<int32>(&builder, 0);
+    DynamicUpdateSlice(input, update, {constant_index, zero, zero});
 
     // Run computation and compare against expected values.
     ComputeAndCompareR3<T>(&builder, expected_values,
@@ -720,46 +751,55 @@ void BM_DynamicSlice(int num_iters) {
         {{13, 14, 15, 16}, {17, 18, 19, 20}, {21, 22, 23, 24}}}});
   auto input = ConstantLiteral(&builder, input_literal);
 
+  auto stream =
+      client->mutable_backend()->BorrowStream(device_ordinal).ValueOrDie();
+
   // Create dynamic slice start indices as a parameter: shape [4]
-  auto start_indices_shape = ShapeUtil::MakeShape(S32, {4});
-  auto start_indices =
-      Parameter(&builder, 0, start_indices_shape, "start_indices");
+  auto start_indices_shape = ShapeUtil::MakeShape(S32, {});
+  std::vector<XlaOp> start_indices(4);
+  std::vector<ScopedShapedBuffer> shaped_buffers;
+  std::vector<const Shape*> host_shapes(4);
+  for (int i = 0; i < 4; ++i) {
+    start_indices[i] =
+        Parameter(&builder, i, start_indices_shape, "start_indices");
+    auto start_index_literal = LiteralUtil::CreateR0<int32>(i + 1);
+    // Initialize and transfer parameter buffer.
+    shaped_buffers.emplace_back(
+        client->backend()
+            .transfer_manager()
+            ->AllocateScopedShapedBuffer(start_indices_shape, &allocator,
+                                         /*device_ordinal=*/0)
+            .ConsumeValueOrDie());
+    host_shapes[i] = &shaped_buffers[i].on_host_shape();
+    ASSERT_IS_OK(transfer_manager->TransferLiteralToDevice(
+        stream.get(), start_index_literal, shaped_buffers[i]));
+  }
+
   // Add DynamicSlice op to the computatation.
   DynamicSlice(input, start_indices, {1, 1, 1, 1});
   auto computation = builder.Build().ConsumeValueOrDie();
 
-  // Initialize and transfer parameter buffer.
-  auto buffer = client->backend()
-                    .transfer_manager()
-                    ->AllocateScopedShapedBuffer(
-                        start_indices_shape, &allocator, /*device_ordinal=*/0)
-                    .ConsumeValueOrDie();
-
-  auto start_indices_literal = LiteralUtil::CreateR1<int32>({0, 1, 2, 3});
-  auto stream =
-      client->mutable_backend()->BorrowStream(device_ordinal).ValueOrDie();
-  ASSERT_IS_OK(transfer_manager->TransferLiteralToDevice(
-      stream.get(), start_indices_literal, buffer));
-
   std::unique_ptr<LocalExecutable> executable =
-      client
-          ->Compile(computation, {&buffer.on_host_shape()},
-                    ExecutableBuildOptions())
+      client->Compile(computation, host_shapes, ExecutableBuildOptions())
           .ConsumeValueOrDie();
 
   // Run some warm-up executions.
   ExecutableRunOptions options;
   options.set_allocator(&allocator);
   const int kWarmups = 2;
+  std::vector<const ShapedBuffer*> shaped_buffer_ptrs;
+  absl::c_transform(shaped_buffers, std::back_inserter(shaped_buffer_ptrs),
+                    [](const ScopedShapedBuffer& buffer) { return &buffer; });
+
   for (int i = 0; i < kWarmups; ++i) {
-    auto result = executable->Run({&buffer}, options);
+    auto result = executable->Run(shaped_buffer_ptrs, options);
     ASSERT_TRUE(result.ok());
   }
 
   // Run benchmark.
   tensorflow::testing::StartTiming();
   for (int i = 0; i < num_iters; ++i) {
-    auto result = executable->Run({&buffer}, options);
+    auto result = executable->Run(shaped_buffer_ptrs, options);
     ASSERT_TRUE(result.ok());
   }
 }
diff --git a/tensorflow/compiler/xla/tests/while_test.cc b/tensorflow/compiler/xla/tests/while_test.cc
index b1566dc8ad..85212fa56d 100644
--- a/tensorflow/compiler/xla/tests/while_test.cc
+++ b/tensorflow/compiler/xla/tests/while_test.cc
@@ -861,7 +861,7 @@ XLA_TEST_F(WhileTest, WhileWithDynamicUpdateSlice) {
     // Update.
     auto update = ConvertElementType(Broadcast(out0, {2}), F32);
     // Starts = iteration * 2;
-    auto starts = Reshape(Mul(iteration, ConstantR0<int32>(&builder, 2)), {1});
+    auto starts = Mul(iteration, ConstantR0<int32>(&builder, 2));
     // UpdateSlice.
     auto out1 = DynamicUpdateSlice(input, update, starts);
 
@@ -1299,9 +1299,9 @@ void BM_WhileLoop(int num_iters) {
     auto one = ConstantR0<float>(&builder, 1.0);
     auto update = Broadcast(one, {1, 1024, 1024});
     // Starts = iteration * 2;
-    auto starts = ConstantR1<int32>(&builder, {0, 0, 0});
+    auto zero = ConstantR0<int32>(&builder, 0);
     // UpdateSlice.
-    auto out1 = DynamicUpdateSlice(input, update, starts);
+    auto out1 = DynamicUpdateSlice(input, update, {zero, zero, zero});
     Tuple(&builder, {out0, out1});
     body = builder.Build().ConsumeValueOrDie();
   }
-- 
GitLab


From 8d151b6c60919a9ed871f96c12f68a25831bdf2d Mon Sep 17 00:00:00 2001
From: Doe Hyun Yoon <dyoon@google.com>
Date: Mon, 14 Jan 2019 11:39:36 -0800
Subject: [PATCH 0661/2345] Replace PredictCosts() with
 PredictCostsAndReturnRunMetadata().

PiperOrigin-RevId: 229222942
---
 .../grappler/costs/analytical_cost_estimator.cc | 15 +--------------
 .../grappler/costs/analytical_cost_estimator.h  | 10 +++-------
 .../costs/analytical_cost_estimator_test.cc     |  3 +--
 tensorflow/core/grappler/costs/cost_estimator.h | 17 ++---------------
 .../grappler/costs/measuring_cost_estimator.cc  |  6 +++++-
 .../grappler/costs/measuring_cost_estimator.h   | 10 +++++-----
 tensorflow/python/grappler/cluster.i            |  6 +++---
 tensorflow/python/grappler/cost_analyzer.cc     |  4 ++--
 8 files changed, 22 insertions(+), 49 deletions(-)

diff --git a/tensorflow/core/grappler/costs/analytical_cost_estimator.cc b/tensorflow/core/grappler/costs/analytical_cost_estimator.cc
index 09cddad8ba..5baf306f6f 100644
--- a/tensorflow/core/grappler/costs/analytical_cost_estimator.cc
+++ b/tensorflow/core/grappler/costs/analytical_cost_estimator.cc
@@ -124,22 +124,9 @@ Status AnalyticalCostEstimator::Initialize(const GrapplerItem& item) {
   return Status::OK();
 }
 
-// TODO(b/67607683): unify logic with VirtualCluster logic
 Status AnalyticalCostEstimator::PredictCosts(const GraphDef& optimized_graph,
-                                             CostGraphDef* cost_graph,
+                                             RunMetadata* run_metadata,
                                              Costs* costs) const {
-  RunMetadata run_metadata;
-  auto s =
-      PredictCostsAndReturnRunMetadata(optimized_graph, &run_metadata, costs);
-  if (s.ok() && cost_graph) {
-    cost_graph->Swap(run_metadata.mutable_cost_graph());
-  }
-  return s;
-}
-
-Status AnalyticalCostEstimator::PredictCostsAndReturnRunMetadata(
-    const GraphDef& optimized_graph, RunMetadata* run_metadata,
-    Costs* costs) const {
   GrapplerItem item = item_;
   item.graph = optimized_graph;
 
diff --git a/tensorflow/core/grappler/costs/analytical_cost_estimator.h b/tensorflow/core/grappler/costs/analytical_cost_estimator.h
index 6275c62198..d058ba4115 100644
--- a/tensorflow/core/grappler/costs/analytical_cost_estimator.h
+++ b/tensorflow/core/grappler/costs/analytical_cost_estimator.h
@@ -51,14 +51,10 @@ class AnalyticalCostEstimator : public CostEstimator {
   Status Initialize(const GrapplerItem& item) override;
 
   // Predict the performance of each node of the optimized graph and annotate
-  // the CostGraphDef with the corresponding estimates. Also returns the
+  // the RunMetadata with the corresponding estimates. Also returns the
   // expected cost for the whole graph.
-  Status PredictCosts(const GraphDef& optimized_graph, CostGraphDef* cost_graph,
-                      Costs* cost) const override;
-
-  Status PredictCostsAndReturnRunMetadata(const GraphDef& optimized_graph,
-                                          RunMetadata* run_metadata,
-                                          Costs* cost) const override;
+  Status PredictCosts(const GraphDef& optimized_graph,
+                      RunMetadata* run_metadata, Costs* cost) const override;
 
   const VirtualScheduler* GetScheduler() const { return scheduler_.get(); }
 
diff --git a/tensorflow/core/grappler/costs/analytical_cost_estimator_test.cc b/tensorflow/core/grappler/costs/analytical_cost_estimator_test.cc
index 8a56344378..eb7ee8dc0a 100644
--- a/tensorflow/core/grappler/costs/analytical_cost_estimator_test.cc
+++ b/tensorflow/core/grappler/costs/analytical_cost_estimator_test.cc
@@ -100,8 +100,7 @@ TEST_F(AnalyticalCostEstimatorTest, SimpleTest) {
 
   RunMetadata run_metadata;
   Costs summary;
-  TF_ASSERT_OK(estimator.PredictCostsAndReturnRunMetadata(
-      item.graph, &run_metadata, &summary));
+  TF_ASSERT_OK(estimator.PredictCosts(item.graph, &run_metadata, &summary));
 
   EXPECT_EQ(Costs::NanoSeconds(9151), summary.execution_time);
   // Note there are totally 17 nodes (RandomUniform creates 2 nodes), but
diff --git a/tensorflow/core/grappler/costs/cost_estimator.h b/tensorflow/core/grappler/costs/cost_estimator.h
index 725d81a881..d85ae0b77f 100644
--- a/tensorflow/core/grappler/costs/cost_estimator.h
+++ b/tensorflow/core/grappler/costs/cost_estimator.h
@@ -216,27 +216,14 @@ class CostEstimator {
 
   // Predicts the cost of running the given optimized version of the grappler
   // item.
-  // If a CostGraphDef is passed, it will be populated with detailed information
+  // If a RunMetadata is passed, it will be populated with detailed information
   // about the cost of running each operation of the optimized graph.
   // if a double value is passed, it will be set to a value that reflects the
   // overall cost of running the graph (e.g. the latency of the computation).
   // Returns a status that indicate is the performance could be estimated or
   // not.
   virtual Status PredictCosts(const GraphDef& optimized_graph,
-                              CostGraphDef* cost_graph, Costs* cost) const = 0;
-
-  // TODO(dyoon): Delete PredictCosts() with CostGraphDef as RunMetadata is a
-  // superset of CostGraphDef.
-  // Same method, but returns RunMetadata.
-  virtual Status PredictCostsAndReturnRunMetadata(
-      const GraphDef& optimized_graph, RunMetadata* run_metadata,
-      Costs* cost) const {
-    CostGraphDef* cost_graph = nullptr;
-    if (run_metadata) {
-      cost_graph = run_metadata->mutable_cost_graph();
-    }
-    return PredictCosts(optimized_graph, cost_graph, cost);
-  }
+                              RunMetadata* run_metadata, Costs* cost) const = 0;
 };
 
 }  // end namespace grappler
diff --git a/tensorflow/core/grappler/costs/measuring_cost_estimator.cc b/tensorflow/core/grappler/costs/measuring_cost_estimator.cc
index 833205ac6f..088ce56658 100644
--- a/tensorflow/core/grappler/costs/measuring_cost_estimator.cc
+++ b/tensorflow/core/grappler/costs/measuring_cost_estimator.cc
@@ -51,8 +51,12 @@ Status MeasuringCostEstimator::Initialize(const GrapplerItem& item) {
 }
 
 Status MeasuringCostEstimator::PredictCosts(const GraphDef& optimized_graph,
-                                            CostGraphDef* cost_graph,
+                                            RunMetadata* run_metadata,
                                             Costs* costs) const {
+  CostGraphDef* cost_graph = nullptr;
+  if (run_metadata) {
+    cost_graph = run_metadata->mutable_cost_graph();
+  }
   const bool running_simulation = (cluster_->type() == "virtual");
 
   std::vector<double> times(measurement_steps_);
diff --git a/tensorflow/core/grappler/costs/measuring_cost_estimator.h b/tensorflow/core/grappler/costs/measuring_cost_estimator.h
index 3e741c9199..67145f5241 100644
--- a/tensorflow/core/grappler/costs/measuring_cost_estimator.h
+++ b/tensorflow/core/grappler/costs/measuring_cost_estimator.h
@@ -54,12 +54,12 @@ class MeasuringCostEstimator : public CostEstimator {
   // This implementation always returns OK.
   Status Initialize(const GrapplerItem& item) override;
 
-  // Runs the optimized version of the graph on the cluster, measure
-  // the runtimes of each operation, and annotated the CostGraphDef
-  // with the corresponding measurements.
+  // Runs the optimized version of the graph on the cluster, measures
+  // the runtimes of each operation, and annotates the CostGraphDef of
+  // RunMetadata with the corresponding measurements.
   // Returns the average latency for the whole graph.
-  Status PredictCosts(const GraphDef& optimized_graph, CostGraphDef* cost_graph,
-                      Costs* overall_cost) const override;
+  Status PredictCosts(const GraphDef& optimized_graph,
+                      RunMetadata* run_metadata, Costs* cost) const override;
 
  private:
   Cluster* cluster_;  // Not owned.
diff --git a/tensorflow/python/grappler/cluster.i b/tensorflow/python/grappler/cluster.i
index b0c1f71a85..af9276c508 100644
--- a/tensorflow/python/grappler/cluster.i
+++ b/tensorflow/python/grappler/cluster.i
@@ -176,13 +176,13 @@ tensorflow::Status _GetOpPerformanceDataAndRunTime(
   tensorflow::Status status = cost_measure->Initialize(item);
   if (!status.ok()) return status;
 
-  tensorflow::CostGraphDef cost_graph;
+  tensorflow::RunMetadata run_metadata;
   TF_RETURN_IF_ERROR(
-      cost_measure->PredictCosts(item.graph, &cost_graph, costs));
+      cost_measure->PredictCosts(item.graph, &run_metadata, costs));
 
   if (op_performance_data) {
     *op_performance_data = tensorflow::grappler::CostGraphToOpPerformanceData(
-        cost_graph, item.graph);
+        run_metadata.cost_graph(), item.graph);
   }
   return tensorflow::Status::OK();
 }
diff --git a/tensorflow/python/grappler/cost_analyzer.cc b/tensorflow/python/grappler/cost_analyzer.cc
index bb8c6d5b85..9aa5fbca38 100644
--- a/tensorflow/python/grappler/cost_analyzer.cc
+++ b/tensorflow/python/grappler/cost_analyzer.cc
@@ -44,8 +44,8 @@ void CostAnalyzer::PredictCosts(CostEstimator* cost_estimator,
   TF_CHECK_OK(cost_estimator->Initialize(*item_));
   RunMetadata run_metadata;
   Costs costs;
-  const Status status = cost_estimator->PredictCostsAndReturnRunMetadata(
-      item_->graph, &run_metadata, &costs);
+  const Status status =
+      cost_estimator->PredictCosts(item_->graph, &run_metadata, &costs);
   if (cost_graph) {
     cost_graph->Swap(run_metadata.mutable_cost_graph());
   }
-- 
GitLab


From 0d8b2d70ac5a7387bc46454b40ae4f1388eaaffc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 11:55:32 -0800
Subject: [PATCH 0662/2345] Internal change only

PiperOrigin-RevId: 229225901
---
 tensorflow/python/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 2c6672c81e..905b47ea75 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -11,6 +11,7 @@ visibility = [
     "//tensorflow/lite/toco/python:__pkg__",
     "//tensorflow_models:__subpackages__",
     "//tensorflow_model_optimization:__subpackages__",
+    "//third_party/py/cleverhans:__subpackages__",
     # TODO(aselle): to pass open source test.
     "//bazel_pip/tensorflow/lite/toco/python:__pkg__",
 ]
-- 
GitLab


From 30796c730b0d137b0367d425cbfbfae66c43a9ff Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Mon, 14 Jan 2019 12:02:17 -0800
Subject: [PATCH 0663/2345] Enable Callbacks when a generator is passed to
 `evaluate`/`predict`.

PiperOrigin-RevId: 229227149
---
 tensorflow/python/keras/callbacks_test.py  | 36 ++++++++++++++++++----
 tensorflow/python/keras/engine/training.py |  4 +--
 2 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/keras/callbacks_test.py b/tensorflow/python/keras/callbacks_test.py
index d5ffdf789a..7af9a9a2b2 100644
--- a/tensorflow/python/keras/callbacks_test.py
+++ b/tensorflow/python/keras/callbacks_test.py
@@ -27,6 +27,7 @@ import tempfile
 import threading
 import unittest
 
+from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.core.framework import summary_pb2
@@ -88,6 +89,23 @@ class Counter(keras.callbacks.Callback):
     return _call_and_count
 
 
+def _get_numpy():
+  return np.ones((10, 10)), np.ones((10, 1))
+
+
+def _get_sequence():
+
+  class MySequence(keras.utils.data_utils.Sequence):
+
+    def __getitem__(self, _):
+      return np.ones((2, 10)), np.ones((2, 1))
+
+    def __len__(self):
+      return 5
+
+  return MySequence(), None
+
+
 @keras_parameterized.run_with_all_model_types
 @keras_parameterized.run_all_keras_modes
 class CallbackCountsTest(keras_parameterized.TestCase):
@@ -113,8 +131,10 @@ class CallbackCountsTest(keras_parameterized.TestCase):
         run_eagerly=testing_utils.should_run_eagerly())
     return model
 
-  def test_callback_hooks_are_called_in_fit(self):
-    x, y = np.ones((10, 10)), np.ones((10, 1))
+  @parameterized.named_parameters(('with_numpy', _get_numpy()),
+                                  ('with_sequence', _get_sequence()))
+  def test_callback_hooks_are_called_in_fit(self, data):
+    x, y = data
     val_x, val_y = np.ones((4, 10)), np.ones((4, 1))
 
     model = self._get_model()
@@ -147,8 +167,10 @@ class CallbackCountsTest(keras_parameterized.TestCase):
             'on_train_end': 1
         })
 
-  def test_callback_hooks_are_called_in_evaluate(self):
-    x, y = np.ones((10, 10)), np.ones((10, 1))
+  @parameterized.named_parameters(('with_numpy', _get_numpy()),
+                                  ('with_sequence', _get_sequence()))
+  def test_callback_hooks_are_called_in_evaluate(self, data):
+    x, y = data
 
     model = self._get_model()
     counter = Counter()
@@ -161,8 +183,10 @@ class CallbackCountsTest(keras_parameterized.TestCase):
             'on_test_end': 1
         })
 
-  def test_callback_hooks_are_called_in_predict(self):
-    x = np.ones((10, 10))
+  @parameterized.named_parameters(('with_numpy', _get_numpy()),
+                                  ('with_sequence', _get_sequence()))
+  def test_callback_hooks_are_called_in_predict(self, data):
+    x = data[0]
 
     model = self._get_model()
     counter = Counter()
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 598766db74..9f544ceff4 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -1023,11 +1023,11 @@ class Model(Network):
     # or a non-distributed Dataset or iterator in eager execution.
     if data_utils.is_generator_or_sequence(x):
       training_utils.check_generator_arguments(y, sample_weight)
-      # TODO(fchollet): why aren't callbacks supported here?
       return self.evaluate_generator(
           x,
           steps=steps,
           verbose=verbose,
+          callbacks=callbacks,
           max_queue_size=max_queue_size,
           workers=workers,
           use_multiprocessing=use_multiprocessing)
@@ -1146,11 +1146,11 @@ class Model(Network):
     # Case 2: generator-like. Input is Python generator, or Sequence object,
     # or a non-distributed Dataset or iterator in eager execution.
     if data_utils.is_generator_or_sequence(x):
-      # TODO(fchollet): why aren't callbacks supported here?
       return self.predict_generator(
           x,
           steps=steps,
           verbose=verbose,
+          callbacks=callbacks,
           max_queue_size=max_queue_size,
           workers=workers,
           use_multiprocessing=use_multiprocessing)
-- 
GitLab


From 3768e3e1748a68dcd8e35ec1e28c7b09bacc1b31 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 12:33:14 -0800
Subject: [PATCH 0664/2345] Add `alwayslink = 1` param to optimizer build
 rules. Without this, optimizers might not actually get linked into a binary.

PiperOrigin-RevId: 229232974
---
 tensorflow/core/grappler/optimizers/data/BUILD | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tensorflow/core/grappler/optimizers/data/BUILD b/tensorflow/core/grappler/optimizers/data/BUILD
index 57b0df3cb2..3bfaef4be1 100644
--- a/tensorflow/core/grappler/optimizers/data/BUILD
+++ b/tensorflow/core/grappler/optimizers/data/BUILD
@@ -25,6 +25,7 @@ cc_library(
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:ptr_util",
     ] + tf_protos_all(),
+    alwayslink = 1,
 )
 
 cc_library(
@@ -48,6 +49,7 @@ cc_library(
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
         "//tensorflow/core:lib_internal",
     ] + tf_protos_all(),
+    alwayslink = 1,
 )
 
 tf_cc_test(
@@ -212,6 +214,7 @@ cc_library(
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
         "//tensorflow/core:lib_internal",
     ] + tf_protos_all(),
+    alwayslink = 1,
 )
 
 tf_cc_test(
@@ -248,6 +251,7 @@ cc_library(
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
+    alwayslink = 1,
 )
 
 cc_library(
@@ -266,6 +270,7 @@ cc_library(
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
+    alwayslink = 1,
 )
 
 tf_cc_test(
@@ -297,6 +302,7 @@ cc_library(
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
+    alwayslink = 1,
 )
 
 tf_cc_test(
@@ -333,6 +339,7 @@ cc_library(
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
+    alwayslink = 1,
 )
 
 tf_cc_test(
@@ -370,6 +377,7 @@ cc_library(
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
         "//tensorflow/core:lib_internal",
     ] + tf_protos_all(),
+    alwayslink = 1,
 )
 
 tf_cc_test(
@@ -409,6 +417,7 @@ cc_library(
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
         "//tensorflow/core:lib_internal",
     ] + tf_protos_all(),
+    alwayslink = 1,
 )
 
 tf_cc_test(
@@ -447,6 +456,7 @@ cc_library(
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
+    alwayslink = 1,
 )
 
 tf_cc_test(
@@ -486,6 +496,7 @@ cc_library(
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
         "//tensorflow/core:lib_internal",
     ] + tf_protos_all(),
+    alwayslink = 1,
 )
 
 tf_cc_test(
@@ -521,6 +532,7 @@ cc_library(
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
+    alwayslink = 1,
 )
 
 tf_cc_test(
@@ -555,6 +567,7 @@ cc_library(
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
+    alwayslink = 1,
 )
 
 tf_cc_test(
-- 
GitLab


From b23dc8cb365c0678931b893d6919f20630fab61e Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Mon, 14 Jan 2019 12:36:07 -0800
Subject: [PATCH 0665/2345] Fix Windows build by moving `LOG(ERROR)` from the
 header to the .cc file.

Some Windows header(s) may define ERROR as a macro, causing `LOG(ERROR)` fail to compile. The .cc file doesn't include such header(s), since there are already a lot of uses of ERROR there.

PiperOrigin-RevId: 229233490
---
 tensorflow/stream_executor/stream.cc | 9 +++++++++
 tensorflow/stream_executor/stream.h  | 9 +--------
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/tensorflow/stream_executor/stream.cc b/tensorflow/stream_executor/stream.cc
index 9937861d88..0dceacb228 100644
--- a/tensorflow/stream_executor/stream.cc
+++ b/tensorflow/stream_executor/stream.cc
@@ -5678,4 +5678,13 @@ string Stream::DebugStreamPointers() const {
                       ",impl=", ToVlogString(implementation_.get()), "]");
 }
 
+void Stream::CheckStatus(port::Status status) {
+  if (status.ok()) {
+    return;
+  }
+  LOG(ERROR) << status;
+  mutex_lock lock(mu_);
+  ok_ = false;
+}
+
 }  // namespace stream_executor
diff --git a/tensorflow/stream_executor/stream.h b/tensorflow/stream_executor/stream.h
index 85d65c1061..f698d50e2a 100644
--- a/tensorflow/stream_executor/stream.h
+++ b/tensorflow/stream_executor/stream.h
@@ -2100,14 +2100,7 @@ class Stream {
   }
 
   // Checks the status and logs the error message, if any.
-  void CheckStatus(port::Status status) LOCKS_EXCLUDED(mu_) {
-    if (status.ok()) {
-      return;
-    }
-    LOG(ERROR) << status;
-    mutex_lock lock(mu_);
-    ok_ = false;
-  }
+  void CheckStatus(port::Status status) LOCKS_EXCLUDED(mu_);
 
   void SetError() { CheckError(false /* = operation_retcode */); }
 
-- 
GitLab


From c45a87f7c7080eecf42c430c66adef0e9249f5ac Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 12:40:41 -0800
Subject: [PATCH 0666/2345] LSC: Replace calls to deprecated googletest macros
 *TEST_CASE() with *TEST_SUITE()

PiperOrigin-RevId: 229234323
---
 tensorflow/lite/testing/generated_examples_zip_test.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/lite/testing/generated_examples_zip_test.cc b/tensorflow/lite/testing/generated_examples_zip_test.cc
index bade6590a4..dfd8eb059a 100644
--- a/tensorflow/lite/testing/generated_examples_zip_test.cc
+++ b/tensorflow/lite/testing/generated_examples_zip_test.cc
@@ -287,10 +287,10 @@ struct ZipPathParamName {
   }
 };
 
-INSTANTIATE_TEST_CASE_P(tests, OpsTest,
-                        ::testing::ValuesIn(UnarchiveAndFindTestNames(
-                            *FLAGS_zip_file_path, *FLAGS_tar_file_path)),
-                        ZipPathParamName());
+INSTANTIATE_TEST_SUITE_P(tests, OpsTest,
+                         ::testing::ValuesIn(UnarchiveAndFindTestNames(
+                             *FLAGS_zip_file_path, *FLAGS_tar_file_path)),
+                         ZipPathParamName());
 
 }  // namespace testing
 }  // namespace tflite
-- 
GitLab


From 36af98052e0f92c303a85244eebdf7876e5cce19 Mon Sep 17 00:00:00 2001
From: Kashif Rasul <kashif.rasul@gmail.com>
Date: Mon, 14 Jan 2019 21:53:22 +0100
Subject: [PATCH 0667/2345] fix var name to follow style

---
 tensorflow/stream_executor/cuda/cuda_gpu_executor.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
index 7705e7e8f2..6e0bd38149 100644
--- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
+++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
@@ -1058,9 +1058,9 @@ DeviceDescription *CUDAExecutor::PopulateDeviceDescription() const {
         CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_Z, device_).ValueOrDie();
     builder.set_thread_dim_limit(thread_dim_limit);
 
-    int clockRate = CUDADriver::GetDeviceAttribute(
+    int clock_rate = CUDADriver::GetDeviceAttribute(
         CU_DEVICE_ATTRIBUTE_CLOCK_RATE, device_).ValueOrDie();
-    builder.set_clock_rate_ghz(static_cast<float>(clockRate) / 1e6);
+    builder.set_clock_rate_ghz(static_cast<float>(clock_rate) / 1e6);
   }
 
   {
-- 
GitLab


From 2a33341c8605dceb6b95f48e688347ae5d437781 Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Mon, 14 Jan 2019 12:40:58 -0800
Subject: [PATCH 0668/2345] Add basic while/cond tests to eager/xla tests

PiperOrigin-RevId: 229234361
---
 tensorflow/compiler/tests/eager_test.py | 52 +++++++++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/tensorflow/compiler/tests/eager_test.py b/tensorflow/compiler/tests/eager_test.py
index c36c12a0e3..c9fce39f6c 100644
--- a/tensorflow/compiler/tests/eager_test.py
+++ b/tensorflow/compiler/tests/eager_test.py
@@ -24,6 +24,7 @@ from tensorflow.compiler.tests import xla_test
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
+from tensorflow.python.eager import def_function
 from tensorflow.python.eager import function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -31,6 +32,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.layers import convolutional
 from tensorflow.python.layers import pooling
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import embedding_ops
 from tensorflow.python.ops import gen_random_ops
 from tensorflow.python.ops import init_ops
@@ -555,6 +557,56 @@ class EagerFunctionTest(xla_test.XLATestCase):
     self.assertEqual(9, dy_v0.numpy())
     self.assertEqual(15, dy_v1.numpy())
 
+  def testWhileInDefun(self):
+    with self.test_scope():
+      @def_function.function
+      def f(start):
+        c = lambda x: math_ops.less(x, 13.0)
+        b = lambda x: math_ops.add(x, 1.0)
+        return control_flow_ops.while_loop(c, b, [start])
+
+      y = f(constant_op.constant(3.0))
+    self.assertEqual(13.0, y.numpy())
+
+  def testAutoGraphWhileInDefun(self):
+    with self.test_scope():
+      @def_function.function
+      def f(start):
+        x = start
+        while x < 13.0:
+          x += 1.0
+        return x
+
+      y = f(constant_op.constant(3.0))
+    self.assertEqual(13.0, y.numpy())
+
+  def testCondInDefun(self):
+    with self.test_scope():
+      @def_function.function
+      def f(pred, value):
+        fn1 = lambda: math_ops.add(value, 1.0)
+        fn2 = lambda: math_ops.subtract(value, 1.0)
+        return control_flow_ops.cond(pred, fn1, fn2)
+
+      plus_one = f(constant_op.constant(True), constant_op.constant(10.0))
+      minus_one = f(constant_op.constant(False), constant_op.constant(10.0))
+    self.assertEqual(11.0, plus_one.numpy())
+    self.assertEqual(9.0, minus_one.numpy())
+
+  def testAutoGraphCondInDefun(self):
+    with self.test_scope():
+      @def_function.function
+      def f(pred, value):
+        if pred:
+          return value + 1.0
+        else:
+          return value - 1.0
+
+      plus_one = f(constant_op.constant(True), constant_op.constant(10.0))
+      minus_one = f(constant_op.constant(False), constant_op.constant(10.0))
+    self.assertEqual(11.0, plus_one.numpy())
+    self.assertEqual(9.0, minus_one.numpy())
+
 
 class ExcessivePaddingTest(xla_test.XLATestCase):
   """Test that eager execution works with TPU flattened tensors.
-- 
GitLab


From 418359b3bc408e1948d9e34618a4b5af3e416b56 Mon Sep 17 00:00:00 2001
From: Taylor Robie <taylorrobie@google.com>
Date: Mon, 14 Jan 2019 12:57:02 -0800
Subject: [PATCH 0669/2345] Fix a bug in Sequential models where activity
 regularization was sometimes not applied.

PiperOrigin-RevId: 229236960
---
 tensorflow/python/keras/engine/sequential.py  | 28 ++++++++++-----
 .../python/keras/engine/training_test.py      | 34 +++++++++++--------
 2 files changed, 38 insertions(+), 24 deletions(-)

diff --git a/tensorflow/python/keras/engine/sequential.py b/tensorflow/python/keras/engine/sequential.py
index 3627091198..425c8c3ef6 100644
--- a/tensorflow/python/keras/engine/sequential.py
+++ b/tensorflow/python/keras/engine/sequential.py
@@ -245,8 +245,11 @@ class Sequential(Model):
     if not self.built and self._is_graph_network:
       self._init_graph_network(self.inputs, self.outputs, name=self.name)
 
-    x = inputs
+    outputs = inputs  # handle the corner case where self.layers is empty
     for layer in self.layers:
+      # During each iteration, `inputs` are the inputs to `layer`, and `outputs`
+      # are the outputs of `layer` applied to `inputs`. At the end of each
+      # iteration `inputs` is set to `outputs` to prepare for the next layer.
       kwargs = {}
       argspec = self._layer_call_argspecs[layer].args
       if 'mask' in argspec:
@@ -255,27 +258,34 @@ class Sequential(Model):
         kwargs['training'] = training
 
       if isinstance(layer, Network) and layer._compute_output_and_mask_jointly:
-        x, mask = layer._call_and_compute_mask(x, **kwargs)
+        outputs, mask = layer._call_and_compute_mask(inputs, **kwargs)
       else:
         if not layer.built:
           # Build layer if applicable.
           with ops.name_scope(layer._name_scope()):
-            layer._maybe_build(x)
+            layer._maybe_build(inputs)
           layer.built = True
         if layer.supports_masking:
-          mask = layer.compute_mask(x, mask)
+          mask = layer.compute_mask(inputs, mask)
         else:
           mask = None
 
         if context.executing_eagerly():
-          x = layer(x, **kwargs)
+          # __call__ handles activity regularization.
+          outputs = layer(inputs, **kwargs)
         elif layer.dynamic:
-          x = layer._symbolic_call(x)
+          outputs = layer._symbolic_call(inputs)
+          layer._handle_activity_regularization(inputs, outputs)
         else:
-          x = layer.call(x, **kwargs)
+          outputs = layer.call(inputs, **kwargs)
+          layer._handle_activity_regularization(inputs, outputs)
+
       if not context.executing_eagerly():
-        x._keras_mask = mask
-    return x, mask
+        outputs._keras_mask = mask
+
+      # `outputs` will be the inputs to the next layer.
+      inputs = outputs
+    return outputs, mask
 
   def compute_output_shape(self, input_shape):
     shape = input_shape
diff --git a/tensorflow/python/keras/engine/training_test.py b/tensorflow/python/keras/engine/training_test.py
index b8d4752db4..fa26385cc4 100644
--- a/tensorflow/python/keras/engine/training_test.py
+++ b/tensorflow/python/keras/engine/training_test.py
@@ -350,16 +350,21 @@ class TrainingTest(keras_parameterized.TestCase):
     self.assertEqual(len(out), 2)
 
   @keras_parameterized.run_all_keras_modes
+  @keras_parameterized.run_with_all_model_types
   def test_activity_regularizer_fit(self):
     loss = {}
     for reg in [None, 'l2']:
-      inputs = keras.layers.Input(shape=(10,))
-      x = keras.layers.Dense(
-          10, activation='relu', activity_regularizer=reg,
-          kernel_initializer='ones', use_bias=False)(inputs)
-      outputs = keras.layers.Dense(1, activation='sigmoid',
-                                   kernel_initializer='ones', use_bias=False)(x)
-      model = keras.Model(inputs, outputs)
+      layers = [
+          keras.layers.Dense(
+              10, activation='relu', activity_regularizer=reg,
+              kernel_initializer='ones', use_bias=False),
+          keras.layers.Dense(
+              1, activation='sigmoid', kernel_initializer='ones',
+              use_bias=False),
+      ]
+
+      model = testing_utils.get_model_from_layers(
+          layers, input_shape=(10,))
 
       x = np.ones((10, 10), 'float32')
       y = np.ones((10, 1), 'float32')
@@ -372,15 +377,14 @@ class TrainingTest(keras_parameterized.TestCase):
     self.assertLess(loss[None], loss['l2'])
 
   @keras_parameterized.run_all_keras_modes
+  @keras_parameterized.run_with_all_model_types
   def test_activity_regularizer_loss_value(self):
-    inputs = keras.layers.Input(shape=(10,))
-    outputs = keras.layers.Dense(
-        1,
-        kernel_initializer=keras.initializers.zeros(),
-        bias_initializer=keras.initializers.ones(),
-        activity_regularizer='l2')(
-            inputs)
-    model = keras.Model(inputs, outputs)
+    layer = keras.layers.Dense(
+        1, kernel_initializer=keras.initializers.zeros(),
+        bias_initializer=keras.initializers.ones(), activity_regularizer='l2')
+
+    model = testing_utils.get_model_from_layers([layer], input_shape=(10,))
+
     x = np.ones((10, 10), 'float32')
     y = np.ones((10, 1), 'float32')
     optimizer = RMSPropOptimizer(learning_rate=0.001)
-- 
GitLab


From 5cfd3b3876ea52608b3a64740d71eeef5ffbbec8 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Mon, 14 Jan 2019 12:58:49 -0800
Subject: [PATCH 0670/2345] [tf.data] Turn the `map_parallelization`
 optimization on by default.

This CL:
- turns the `map_parallelization` optimization on by default
- fixes a bug in the optimization code that would result in a segfault if two consecutive map transformations were both parallelized
- fixes a bug in the OptimizationOptions._static_optimizations() method that would cause `apply_default_optimization = False` to override explicitly enabled default optimizations

PiperOrigin-RevId: 229237263
---
 .../core/grappler/optimizers/data/BUILD       |  2 -
 .../optimizers/data/map_parallelization.cc    | 28 +++++----
 .../kernel_tests/optimization/BUILD           |  1 +
 .../optimization/assert_next_dataset_test.py  |  7 +++
 .../optimization/filter_fusion_test.py        |  1 +
 .../optimization/hoist_random_uniform_test.py | 10 +--
 .../optimization/latency_all_edges_test.py    |  1 +
 .../optimization/make_numa_aware_test.py      |  1 +
 .../optimization/map_and_batch_fusion_test.py |  4 ++
 .../map_and_filter_fusion_test.py             |  2 +
 .../optimization/map_fusion_test.py           |  1 +
 .../optimization/map_parallelization_test.py  | 63 +++++++++++++------
 .../optimization/map_vectorization_test.py    |  2 +
 .../optimization/model_dataset_test.py        |  1 +
 .../optimization/noop_elimination_test.py     |  4 ++
 .../optimization/optimize_dataset_test.py     | 48 ++++++++++----
 .../shuffle_and_repeat_fusion_test.py         |  4 ++
 .../experimental/ops/optimization_options.py  | 18 +++---
 .../python/data/kernel_tests/map_test.py      |  2 +-
 19 files changed, 139 insertions(+), 61 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/data/BUILD b/tensorflow/core/grappler/optimizers/data/BUILD
index 3bfaef4be1..d95edc5623 100644
--- a/tensorflow/core/grappler/optimizers/data/BUILD
+++ b/tensorflow/core/grappler/optimizers/data/BUILD
@@ -447,12 +447,10 @@ cc_library(
     deps = [
         ":graph_utils",
         "//tensorflow/core/grappler:mutable_graph_view",
-        "//tensorflow/core:lib",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/clusters:cluster",
-        "//tensorflow/core/grappler/utils:topological_sort",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
diff --git a/tensorflow/core/grappler/optimizers/data/map_parallelization.cc b/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
index 8e49f908a7..d335117585 100644
--- a/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
@@ -28,6 +28,10 @@ namespace tensorflow {
 namespace grappler {
 namespace {
 
+static const char* kMapDataset = "MapDataset";
+static const char* kParallelMapDataset = "ParallelMapDataset";
+static constexpr int kAutotune = -1;
+
 bool CanParallelize(const FunctionDef& function,
                     const FunctionLibraryDefinition& library) {
   if (!function.signature().is_stateful()) return true;
@@ -45,16 +49,17 @@ bool CanParallelize(const FunctionDef& function,
   return true;
 }
 
-NodeDef MakeParallelMap(const NodeDef& map_node, MutableGraphView* graph) {
-  NodeDef parallel_map = map_node;
-  graph_utils::SetUniqueGraphNodeName("parallel_map", graph->graph(),
+NodeDef MakeParallelMap(const string& name, MutableGraphView* graph) {
+  // The inputs of the node to be parallelized could be changed by the
+  // optimization pass, so we need to look it up in the modified graph.
+  int index = graph_utils::FindGraphNodeWithName(name, *graph->graph());
+  DCHECK_NE(index, -1) << "Failed to find node " << name
+                       << " in the optimized graph.";
+  NodeDef parallel_map = graph->graph()->node(index);
+  graph_utils::SetUniqueGraphNodeName(kParallelMapDataset, graph->graph(),
                                       &parallel_map);
-  parallel_map.set_op("ParallelMapDataset");
-  // TODO(b/114475558): We want to set `num_parallel_calls` to a special value,
-  // so that dynamic tunning will pick the optimal value at runtime. Because
-  // this feature is not yet implemented, we set it to 2, which is the smallest
-  // value that introduces parallelism.
-  auto* num_parallel_calls = graph_utils::AddScalarConstNode(2, graph);
+  parallel_map.set_op(kParallelMapDataset);
+  auto* num_parallel_calls = graph_utils::AddScalarConstNode(kAutotune, graph);
   parallel_map.add_input(num_parallel_calls->name());
 
   return parallel_map;
@@ -70,7 +75,7 @@ Status MapParallelization::Optimize(Cluster* cluster, const GrapplerItem& item,
   FunctionLibraryDefinition function_library(OpRegistry::Global(),
                                              item.graph.library());
   auto get_map_node = [](const NodeDef& node) -> const NodeDef* {
-    if (node.op() == "MapDataset") return &node;
+    if (node.op() == kMapDataset) return &node;
     return nullptr;
   };
 
@@ -82,7 +87,8 @@ Status MapParallelization::Optimize(Cluster* cluster, const GrapplerItem& item,
         function_library.Find(map_node->attr().at("f").func().name());
     if (!CanParallelize(*function, function_library)) continue;
 
-    auto* parallel_map = graph.AddNode(MakeParallelMap(*map_node, &graph));
+    auto* parallel_map =
+        graph.AddNode(MakeParallelMap(map_node->name(), &graph));
     graph.UpdateFanouts(map_node->name(), parallel_map->name());
     nodes_to_delete.insert(map_node->name());
   }
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD b/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
index bf868ebe79..93a9aca1f2 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
@@ -190,6 +190,7 @@ py_test(
     ],
     deps = [
         "//tensorflow/python:client_testlib",
+        "//tensorflow/python:constant_op",
         "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/assert_next_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/assert_next_dataset_test.py
index 9b8248a78d..e05dcbd9d5 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/assert_next_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/assert_next_dataset_test.py
@@ -31,11 +31,17 @@ class AssertNextDatasetTest(test_base.DatasetTestBase):
   def testAssertNext(self):
     dataset = dataset_ops.Dataset.from_tensors(0).apply(
         optimization.assert_next(["Map"])).map(lambda x: x)
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
     self.assertDatasetProduces(dataset, expected_output=[0])
 
   def testAssertNextInvalid(self):
     dataset = dataset_ops.Dataset.from_tensors(0).apply(
         optimization.assert_next(["Whoops"])).map(lambda x: x)
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
     self.assertDatasetProduces(
         dataset,
         expected_error=(
@@ -48,6 +54,7 @@ class AssertNextDatasetTest(test_base.DatasetTestBase):
         optimization.assert_next(["Map", "Whoops"])).map(lambda x: x)
     options = dataset_ops.Options()
     options.experimental_autotune = False
+    options.experimental_optimization.apply_default_optimizations = False
     dataset = dataset.with_options(options)
     self.assertDatasetProduces(
         dataset,
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/filter_fusion_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/filter_fusion_test.py
index 3ce921b5ef..525ae2c54e 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/filter_fusion_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/filter_fusion_test.py
@@ -71,6 +71,7 @@ class FilterFusionTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     dataset = dataset.cache()
     options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
     options.experimental_optimization.filter_fusion = True
     dataset = dataset.with_options(options)
     expected_output = []
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/hoist_random_uniform_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/hoist_random_uniform_test.py
index f080891f2e..08a44e572b 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/hoist_random_uniform_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/hoist_random_uniform_test.py
@@ -22,7 +22,6 @@ from absl.testing import parameterized
 from tensorflow.python.data.experimental.ops import optimization
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -65,12 +64,7 @@ class HoistRandomUniformTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   def _testDataset(self, dataset):
     previous_result = 0
-    if context.executing_eagerly():
-      iterator = dataset.__iter__()
-      get_next = iterator._next_internal  # pylint: disable=protected-access
-    else:
-      iterator = dataset_ops.make_one_shot_iterator(dataset)
-      get_next = iterator.get_next
+    get_next = self.getNext(dataset)
     for _ in range(5):
       result = self.evaluate(get_next())
       self.assertLessEqual(1, result)
@@ -91,6 +85,7 @@ class HoistRandomUniformTest(test_base.DatasetTestBase, parameterized.TestCase):
             ["Zip[0]", "Map"] if will_optimize else ["Map"])).map(function)
 
     options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
     options.experimental_optimization.hoist_random_uniform = True
     dataset = dataset.with_options(options)
     self._testDataset(dataset)
@@ -107,6 +102,7 @@ class HoistRandomUniformTest(test_base.DatasetTestBase, parameterized.TestCase):
     dataset = dataset_ops.Dataset.range(5).apply(
         optimization.assert_next(["Zip[0]", "Map"])).map(random_with_capture)
     options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
     options.experimental_optimization.hoist_random_uniform = True
     dataset = dataset.with_options(options)
     self._testDataset(dataset)
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/latency_all_edges_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/latency_all_edges_test.py
index 8af86da852..4fd982d122 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/latency_all_edges_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/latency_all_edges_test.py
@@ -35,6 +35,7 @@ class LatencyAllEdgesTest(stats_dataset_test_base.StatsDatasetTestBase):
             ["LatencyStats", "Map", "LatencyStats", "Prefetch",
              "LatencyStats"])).map(lambda x: x * x).prefetch(1)
     options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
     options.experimental_stats.latency_all_edges = True
     options.experimental_stats.aggregator = aggregator
     dataset = dataset.with_options(options)
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/make_numa_aware_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/make_numa_aware_test.py
index 2386dd5f11..d79ae4387c 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/make_numa_aware_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/make_numa_aware_test.py
@@ -34,6 +34,7 @@ class MakeNumaAwareTest(test_base.DatasetTestBase):
             batching.map_and_batch(lambda x: x * x, 10))
     options = dataset_ops.Options()
     options.experimental_numa_aware = True
+    options.experimental_optimization.apply_default_optimizations = False
     dataset = dataset.with_options(options)
     self.assertDatasetProduces(
         dataset, expected_output=[[x * x for x in range(10)]])
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_batch_fusion_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_batch_fusion_test.py
index e2ff3116ec..dc7bb9d6a3 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_batch_fusion_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_batch_fusion_test.py
@@ -31,6 +31,10 @@ class MapAndBatchFusionTest(test_base.DatasetTestBase):
     dataset = dataset_ops.Dataset.range(10).apply(
         optimization.assert_next(
             ["MapAndBatch"])).map(lambda x: x * x).batch(10)
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    options.experimental_optimization.map_and_batch_fusion = True
+    dataset = dataset.with_options(options)
     self.assertDatasetProduces(
         dataset, expected_output=[[x * x for x in range(10)]])
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_filter_fusion_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_filter_fusion_test.py
index fa1d673065..7b0cc56973 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_filter_fusion_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_filter_fusion_test.py
@@ -83,6 +83,7 @@ class MapAndFilterFusionTest(test_base.DatasetTestBase, parameterized.TestCase):
         optimization.assert_next(
             ["Map", "FilterByLastComponent"])).map(function).filter(predicate)
     options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
     options.experimental_optimization.map_and_filter_fusion = True
     dataset = dataset.with_options(options)
     self._testMapAndFilter(dataset, function, predicate)
@@ -101,6 +102,7 @@ class MapAndFilterFusionTest(test_base.DatasetTestBase, parameterized.TestCase):
         optimization.assert_next(["Map",
                                   "Filter"])).map(function).filter(predicate)
     options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
     options.experimental_optimization.map_and_filter_fusion = True
     dataset = dataset.with_options(options)
     self._testMapAndFilter(dataset, function, predicate)
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_fusion_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_fusion_test.py
index defdaf0440..b3a7304b4e 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_fusion_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_fusion_test.py
@@ -74,6 +74,7 @@ class MapFusionTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     dataset = dataset.cache()
     options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
     options.experimental_optimization.map_fusion = True
     dataset = dataset.with_options(options)
     expected_output = []
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_parallelization_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_parallelization_test.py
index d8dd31fee8..60649cd3ed 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_parallelization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_parallelization_test.py
@@ -22,12 +22,13 @@ from absl.testing import parameterized
 from tensorflow.python.data.experimental.ops import optimization
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import random_ops
+from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 
 
@@ -42,36 +43,58 @@ def _map_parallelization_test_cases():
     with ops.control_dependencies([assert_op]):
       return x
 
-  def random(_):
-    return random_ops.random_uniform([],
-                                     minval=0,
-                                     maxval=10,
-                                     dtype=dtypes.int64,
-                                     seed=42)
-
-  def assert_with_random(x):
-    x = assert_greater(x)
-    return random(x)
-
-  return (("Identity", identity, True), ("Increment", increment, True),
-          ("AssertGreater", assert_greater, True), ("Random", random, False),
-          ("AssertWithRandom", assert_with_random, False))
+  return (("Identity", identity, True),
+          ("Increment", increment, True),
+          ("AssertGreater", assert_greater, True))
 
 
 @test_util.run_all_in_graph_and_eager_modes
 class MapParallelizationTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   @parameterized.named_parameters(*_map_parallelization_test_cases())
-  def testMapParallelization(self, function, should_optimize):
-    next_nodes = ["ParallelMap"] if should_optimize else ["Map"]
+  def testMapParallelization(self, function, should_be_parallel):
+    next_nodes = ["ParallelMap"] if should_be_parallel else ["Map"]
     dataset = dataset_ops.Dataset.range(5).apply(
         optimization.assert_next(next_nodes)).map(function)
     options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    options.experimental_optimization.map_parallelization = True
+    dataset = dataset.with_options(options)
+    self.assertDatasetProduces(
+        dataset, expected_output=[function(x) for x in range(5)])
+
+  def testMapParallelizationWithCapturedConstant(self):
+    """Tests that functions with captured constants are parallelized."""
+
+    captured_t = constant_op.constant(42, dtype=dtypes.int64)
+    def fn(x):
+      return x + captured_t
+    dataset = dataset_ops.Dataset.range(5).apply(
+        optimization.assert_next(["ParallelMap"])).map(fn)
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    options.experimental_optimization.map_parallelization = True
+    dataset = dataset.with_options(options)
+    self.assertDatasetProduces(
+        dataset, expected_output=[x + 42 for x in range(5)])
+
+  def testMapParallelizationWithCapturedVariable(self):
+    """Tests that functions with captured variables are not parallelized."""
+
+    captured_t = variables.Variable(42, dtype=dtypes.int64)
+    def fn(x):
+      return x + captured_t
+    dataset = dataset_ops.Dataset.range(5).apply(
+        optimization.assert_next(["Map"])).map(fn)
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
     options.experimental_optimization.map_parallelization = True
     dataset = dataset.with_options(options)
-    if should_optimize:
-      self.assertDatasetProduces(
-          dataset, expected_output=[function(x) for x in range(5)])
+    self.evaluate(variables.global_variables_initializer())
+    self.assertDatasetProduces(
+        dataset,
+        expected_output=[x + 42 for x in range(5)],
+        requires_initialization=True)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
index 65fa2bac17..405b038eb2 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
@@ -349,6 +349,7 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
       dataset = dataset.map(map_fn, num_parallel_calls)
       dataset = dataset.batch(100)
       options = dataset_ops.Options()
+      options.experimental_optimization.apply_default_optimizations = False
       options.experimental_optimization.map_and_batch_fusion = False
       dataset = dataset.with_options(options)
       return dataset
@@ -357,6 +358,7 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
     optimized = _make_dataset(["Batch", map_node_name]
                               if expect_optimized else [map_node_name, "Batch"])
     options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
     options.experimental_optimization.map_vectorization = True
     optimized = optimized.with_options(options)
     return unoptimized, optimized
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py
index 0f0274b41f..c6b9db0d52 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py
@@ -35,6 +35,7 @@ class ModelDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
         optimization.assert_next(["Model"]))
     options = dataset_ops.Options()
     options.experimental_autotune = True
+    options.experimental_optimization.apply_default_optimizations = False
     dataset = dataset.with_options(options)
 
     iterator = dataset_ops.make_one_shot_iterator(dataset)
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/noop_elimination_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/noop_elimination_test.py
index 8058f53eea..74f620e37d 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/noop_elimination_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/noop_elimination_test.py
@@ -41,6 +41,10 @@ class NoopEliminationTest(test_base.DatasetTestBase):
             ["FiniteRepeat", "FiniteSkip", "Prefetch", "MemoryCacheImpl"]))
     dataset = dataset.repeat(some_tensor).skip(5).take(-1).skip(0).repeat(
         1).prefetch(0).prefetch(1).cache()
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    options.experimental_optimization.noop_elimination = True
+    dataset = dataset.with_options(options)
     self.assertDatasetProduces(dataset, expected_output=range(5))
 
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
index c111567c1c..a08016ca89 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
@@ -106,7 +106,9 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
   def testOptimizationStatefulFunction(self):
     dataset = dataset_ops.Dataset.range(
         10).map(lambda _: random_ops.random_uniform([])).batch(10)
-    dataset = dataset_ops._OptimizeDataset(dataset, [])
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
     get_next = self.getNext(dataset)
     self.evaluate(get_next())
 
@@ -114,7 +116,9 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
   def testSkipEagerOptimizationLargeInputFromTensor(self):
     input_t = array_ops.placeholder(dtypes.int32, (None, None, None))
     dataset = dataset_ops.Dataset.from_tensors(input_t)
-    dataset = dataset_ops._OptimizeDataset(dataset, [])
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
     iterator = dataset_ops.make_initializable_iterator(dataset)
     init_op = iterator.initializer
     get_next = iterator.get_next()
@@ -127,7 +131,9 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
   def testSkipEagerOptimizationLargeInputFromTensorSlices(self):
     input_t = array_ops.placeholder(dtypes.int32, (None, None, None, None))
     dataset = dataset_ops.Dataset.from_tensor_slices(input_t)
-    dataset = dataset_ops._OptimizeDataset(dataset, [])
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
     iterator = dataset_ops.make_initializable_iterator(dataset)
     init_op = iterator.initializer
     get_next = iterator.get_next()
@@ -147,7 +153,10 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     dataset = dataset_ops.Dataset.range(1)
     dataset = dataset.flat_map(flat_map_fn)
-    dataset = dataset_ops._OptimizeDataset(dataset, ["noop_elimination"])
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    options.experimental_optimization.noop_elimination = True
+    dataset = dataset.with_options(options)
     self.assertDatasetProduces(dataset, expected_output=[0])
 
   def testOptimizationNestedDatasetWithModifiedRetval(self):
@@ -163,10 +172,8 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     dataset = dataset_ops.Dataset.range(1)
     dataset = dataset.flat_map(flat_map_fn)
 
-    # TODO(b/120558523): We use Options instead of _OptimizeDataset directly
-    # here because of a bug with chaining _OptimizeDatasets when there are
-    # nested dataset functions
     options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
     options.experimental_optimization.map_and_batch_fusion = True
     dataset = dataset.with_options(options)
     self.assertDatasetProduces(dataset, expected_output=[[0]])
@@ -179,7 +186,9 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
         threadpool.PrivateThreadPool(
             2, display_name="private_thread_pool_%d" % 2))
 
-    dataset = dataset_ops._OptimizeDataset(dataset, [])
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
     self.assertDatasetProduces(
         dataset,
         expected_output=[list(range(10))],
@@ -193,14 +202,20 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     dataset = dataset.apply(optimization.assert_next(["MemoryCacheImpl"]))
     dataset = dataset.skip(0)  # Should be removed by noop elimination
     dataset = dataset.cache()
-    dataset = dataset_ops._OptimizeDataset(dataset, ["noop_elimination"])
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    options.experimental_optimization.noop_elimination = True
+    dataset = dataset.with_options(options)
     self.assertDatasetProduces(dataset, expected_output=[0])
 
   def testOptimizationNonSerializableAsDirectInput(self):
     """Tests that non-serializable dataset can be OptimizeDataset's input."""
     dataset = dataset_ops.Dataset.from_tensors(0)
     dataset = dataset.apply(optimization.non_serializable())
-    dataset = dataset_ops._OptimizeDataset(dataset, ["noop_elimination"])
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    options.experimental_optimization.noop_elimination = True
+    dataset = dataset.with_options(options)
     self.assertDatasetProduces(dataset, expected_output=[0])
 
   @parameterized.named_parameters(_generate_captured_refvar_test_cases())
@@ -217,6 +232,7 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       unoptimized_dataset = dataset_fn(variable)
 
       options = dataset_ops.Options()
+      options.experimental_optimization.apply_default_optimizations = False
       options.experimental_optimization.noop_elimination = True
       options.experimental_optimization.map_and_batch_fusion = True
       optimized_dataset = unoptimized_dataset.with_options(options)
@@ -252,6 +268,7 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     options = dataset_ops.Options()
     expected_optimizations = [
         "map_and_batch_fusion",
+        "map_parallelization",
         "noop_elimination",
         "shuffle_and_repeat_fusion",
     ]
@@ -265,10 +282,15 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     only explicitly enabled optimizations will be applied.
     """
     options = dataset_ops.Options()
-    options.experimental_optimization.hoist_random_uniform = True
     options.experimental_optimization.apply_default_optimizations = False
-    expected_optimizations = ["hoist_random_uniform"]
-    self.assertEqual(options._static_optimizations(), expected_optimizations)
+    options.experimental_optimization.hoist_random_uniform = True
+    options.experimental_optimization.noop_elimination = True
+    expected_optimizations = [
+        "hoist_random_uniform",
+        "noop_elimination",
+    ]
+    self.assertEqual(
+        set(options._static_optimizations()), set(expected_optimizations))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/shuffle_and_repeat_fusion_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/shuffle_and_repeat_fusion_test.py
index 594b59375f..824cc680ab 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/shuffle_and_repeat_fusion_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/shuffle_and_repeat_fusion_test.py
@@ -31,6 +31,10 @@ class ShuffleAndRepeatFusionTest(test_base.DatasetTestBase):
   def testShuffleAndRepeatFusion(self):
     dataset = dataset_ops.Dataset.range(10).apply(
         optimization.assert_next(["ShuffleAndRepeat"])).shuffle(10).repeat(2)
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    options.experimental_optimization.shuffle_and_repeat_fusion = True
+    dataset = dataset.with_options(options)
     get_next = self.getNext(dataset)
 
     for _ in range(2):
diff --git a/tensorflow/python/data/experimental/ops/optimization_options.py b/tensorflow/python/data/experimental/ops/optimization_options.py
index eb95484ac5..9fe1f9499e 100644
--- a/tensorflow/python/data/experimental/ops/optimization_options.py
+++ b/tensorflow/python/data/experimental/ops/optimization_options.py
@@ -104,28 +104,32 @@ class OptimizationOptions(options.OptionsBase):
 
   def _static_optimizations(self):
     """Produces the list of enabled static optimizations."""
-    result = []
-    optimizations_to_enable = [
+    result = set()
+    all_optimizations = [
         "filter_fusion",
         "hoist_random_uniform",
+        "map_and_batch_fusion",
         "map_and_filter_fusion",
-        "map_fusion",
         "map_parallelization",
+        "map_fusion",
         "map_vectorization",
+        "noop_elimination",
+        "shuffle_and_repeat_fusion",
     ]
-    for optimization in optimizations_to_enable:
+    for optimization in all_optimizations:
       if getattr(self, optimization):
-        result.append(optimization)
+        result.add(optimization)
 
     if self.apply_default_optimizations is not False:
       # The following optimizations are turned on by default, unless the
       # user explicitly disables them.
       optimizations_to_disable = [
           "map_and_batch_fusion",
+          "map_parallelization",
           "noop_elimination",
           "shuffle_and_repeat_fusion",
       ]
       for optimization in optimizations_to_disable:
         if getattr(self, optimization) is not False:
-          result.append(optimization)
-    return result
+          result.add(optimization)
+    return list(result)
diff --git a/tensorflow/python/data/kernel_tests/map_test.py b/tensorflow/python/data/kernel_tests/map_test.py
index e84391946f..97ab6b21bc 100644
--- a/tensorflow/python/data/kernel_tests/map_test.py
+++ b/tensorflow/python/data/kernel_tests/map_test.py
@@ -84,7 +84,7 @@ def _make_coordinated_sloppy_dataset(num_elements, num_parallel_calls):
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
+class MapTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   def _buildMapDataset(self, components, count):
 
-- 
GitLab


From 92971d7af78b3ff48d0c66fb57bfde886406bbe8 Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Mon, 14 Jan 2019 13:27:41 -0800
Subject: [PATCH 0671/2345] Add some more pfor benchmarks

PiperOrigin-RevId: 229242903
---
 .../python/ops/parallel_for/gradients_test.py | 50 +++++++++++++++----
 1 file changed, 39 insertions(+), 11 deletions(-)

diff --git a/tensorflow/python/ops/parallel_for/gradients_test.py b/tensorflow/python/ops/parallel_for/gradients_test.py
index 66110f12d9..69635c5a79 100644
--- a/tensorflow/python/ops/parallel_for/gradients_test.py
+++ b/tensorflow/python/ops/parallel_for/gradients_test.py
@@ -70,9 +70,10 @@ def fully_connected_model_fn(batch_size, activation_size, num_layers):
   return inp, model(inp)
 
 
-def lstm_model_fn(batch_size, state_size, steps):
+def lstm_model_fn(batch_size, state_size, steps, inputs_size=None):
+  inputs_size = inputs_size or state_size
   inputs = [
-      random_ops.random_normal([batch_size, state_size]) for _ in range(steps)
+      random_ops.random_normal([batch_size, inputs_size]) for _ in range(steps)
   ]
   cell = rnn_cell.BasicLSTMCell(state_size)
   init_state = cell.zero_state(batch_size, dtypes.float32)
@@ -108,8 +109,9 @@ def create_fc_batch_jacobian(batch_size, activation_size, num_layers):
   return pfor_jacobian, while_jacobian
 
 
-def create_lstm_batch_jacobian(batch_size, state_size, steps):
-  inp, output = lstm_model_fn(batch_size, state_size, steps)
+def create_lstm_batch_jacobian(batch_size, state_size, steps, inputs_size=None):
+  inp, output = lstm_model_fn(batch_size, state_size, steps,
+                              inputs_size=inputs_size)
   pfor_jacobian = gradients.batch_jacobian(output, inp, use_pfor=True)
   while_jacobian = gradients.batch_jacobian(output, inp, use_pfor=False)
   return pfor_jacobian, while_jacobian
@@ -181,9 +183,10 @@ def create_fc_per_eg_grad(batch_size, activation_size, num_layers):
   return pfor_outputs, while_outputs
 
 
-def create_lstm_per_eg_grad(batch_size, state_size, steps):
+def create_lstm_per_eg_grad(batch_size, state_size, steps, inputs_size=None):
+  inputs_size = inputs_size or state_size
   inputs = [
-      random_ops.random_normal([batch_size, state_size]) for _ in range(steps)
+      random_ops.random_normal([batch_size, inputs_size]) for _ in range(steps)
   ]
   cell = rnn_cell.BasicLSTMCell(state_size)
   init_state = cell.zero_state(batch_size, dtypes.float32)
@@ -298,6 +301,16 @@ def create_mnist_per_eg_grad(batch_size, data_format, training):
   return pfor_outputs, while_outputs
 
 
+def create_mnist_batch_jacobian(batch_size, data_format, training):
+  images = random_ops.random_uniform([batch_size, 28, 28])
+  model = Mnist(data_format)
+  logits = model(images, training=training)
+
+  pfor_jacobian = gradients.batch_jacobian(logits, images, use_pfor=True)
+  while_jacobian = gradients.batch_jacobian(logits, images, use_pfor=False)
+  return pfor_jacobian, while_jacobian
+
+
 def create_mnist_per_eg_jacobian(batch_size, data_format, training):
   images = random_ops.random_uniform([batch_size, 28, 28])
   model = Mnist(data_format)
@@ -479,7 +492,8 @@ class GradientsTest(test.TestCase):
     self.run_and_assert_equal(pfor_jacobian, while_jacobian)
 
   def test_lstm_batch_jacobian(self):
-    pfor_jacobian, while_jacobian = create_lstm_batch_jacobian(8, 4, 2)
+    pfor_jacobian, while_jacobian = create_lstm_batch_jacobian(8, 4, 2,
+                                                               inputs_size=128)
     self.run_and_assert_equal(pfor_jacobian, while_jacobian)
 
   @test_util.disable_xla("This test never passed for XLA")
@@ -569,7 +583,7 @@ class GradientsBenchmarks(test.Benchmark):
       for _ in range(iters):
         self.evaluate(targets)
       end = time.time()
-    avg_time_ms = 1000 * (end - begin) / iters
+    avg_time_ms = (1000 * (end - begin)) / iters
     self.report_benchmark(iters=iters, wall_time=avg_time_ms, name=name)
     return avg_time_ms
 
@@ -581,7 +595,8 @@ class GradientsBenchmarks(test.Benchmark):
 
   def benchmark_lstm_batch_jacobian(self):
     with ops.Graph().as_default():
-      pfor_jacobian, while_jacobian = create_lstm_batch_jacobian(100, 32, 8)
+      pfor_jacobian, while_jacobian = create_lstm_batch_jacobian(
+          100, 32, 8, inputs_size=128)
       self._run(pfor_jacobian, 100, name="lstm_batch_jacobian_pfor")
       self._run(while_jacobian, 20, name="lstm_batch_jacobian_while")
 
@@ -630,13 +645,26 @@ class GradientsBenchmarks(test.Benchmark):
 
   def benchmark_mnist_per_eg_jacobian(self):
     with ops.Graph().as_default():
-      data_format = ("channels_first"
-                     if test.is_gpu_available() else "channels_last")
+      if test.is_gpu_available():
+        data_format = "channels_first"
+      else:
+        data_format = "channels_last"
       pfor_outputs, while_outputs = create_mnist_per_eg_jacobian(
           16, data_format, training=True)
       self._run(pfor_outputs, 20, name="mnist_per_eg_jacobian_pfor")
       self._run(while_outputs, 20, name="mnist_per_eg_jacobian_while")
 
+  def benchmark_mnist_batch_jacobian(self):
+    with ops.Graph().as_default():
+      if test.is_gpu_available():
+        data_format = "channels_first"
+      else:
+        data_format = "channels_last"
+      pfor_outputs, while_outputs = create_mnist_batch_jacobian(
+          128, data_format, training=True)
+      self._run(pfor_outputs, 20, name="mnist_batch_jacobian_pfor")
+      self._run(while_outputs, 20, name="mnist_batch_jacobian_while")
+
   def benchmark_fc_per_eg_jacobian(self):
     with ops.Graph().as_default():
       jacobians, per_eg_jacobians_pfor, per_eg_jacobians_while = (
-- 
GitLab


From 26f2dd9346139dfab43d56783ea76dff90dce1eb Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Mon, 14 Jan 2019 13:31:35 -0800
Subject: [PATCH 0672/2345] XLABuilder -- Uniquify constant names when constant
 folding xla_builder

Constant folding can introduce multiple constant nodes in a graph, previously we use the same name for them, which could break the invariance that the names are unique in a graph.

PiperOrigin-RevId: 229243764
---
 tensorflow/compiler/xla/client/xla_builder.cc   |  4 +++-
 .../compiler/xla/tests/compute_constant_test.cc | 17 +++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc
index c98f5b1140..4b360c6563 100644
--- a/tensorflow/compiler/xla/client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_builder.cc
@@ -2575,8 +2575,10 @@ StatusOr<XlaComputation> XlaBuilder::BuildConstantSubGraph(
       *const_instr.mutable_shape() = literal.shape().ToProto();
       *const_instr.mutable_literal() = literal.ToProto();
       *const_instr.mutable_opcode() = HloOpcodeString(HloOpcode::kConstant);
-      *const_instr.mutable_name() = HloOpcodeString(HloOpcode::kConstant);
+
       const_instr.set_id(handle);
+      *const_instr.mutable_name() =
+          GetFullName(const_instr.opcode(), kNameSeparator, const_instr.id());
       *entry.add_instructions() =
           const_instr;  // Add to the result constant graph.
     } else {
diff --git a/tensorflow/compiler/xla/tests/compute_constant_test.cc b/tensorflow/compiler/xla/tests/compute_constant_test.cc
index 960f496977..ef800b8ef6 100644
--- a/tensorflow/compiler/xla/tests/compute_constant_test.cc
+++ b/tensorflow/compiler/xla/tests/compute_constant_test.cc
@@ -166,6 +166,23 @@ TEST_F(ComputeConstantTest, GetDimensionSize) {
   }
 }
 
+TEST_F(ComputeConstantTest, MultipleGetDimensionSize) {
+  for (ClientType client_type : client_types) {
+    Client* client = ClientOrDie(platform_, client_type);
+    XlaBuilder b(TestName());
+    auto add =
+        Add(ConstantR2<float>(&b, {{1.0f}}), ConstantR2<float>(&b, {{1.0f}}));
+    auto get_dimension_size = GetDimensionSize(add, 0);
+    auto get_dimension_size_2 = GetDimensionSize(add, 0);
+    auto add_2 = Add(get_dimension_size, get_dimension_size_2);
+    EXPECT_TRUE(IsConstant(add_2, &b));
+
+    TF_ASSERT_OK_AND_ASSIGN(auto value,
+                            ComputeConstantScalar<uint32>(client, add_2, &b));
+    EXPECT_EQ(value, 2);
+  }
+}
+
 // Test computation of an expression interspersed with param nodes but
 // the expression does not depend on the param nodes.
 TEST_F(ComputeConstantTest, UnrelatedParam) {
-- 
GitLab


From f6709559bebf6cdc9a015d4740226d905ac6f6dc Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Mon, 14 Jan 2019 13:48:19 -0800
Subject: [PATCH 0673/2345] [XLA:GPU] Enhance kLoop fusion unroll heuristic.

Avoid unrolling kLoop fusion with instructions such as kReduce that are likely
translated into loops in LLVM IR and can prevent vectorization.

PiperOrigin-RevId: 229246919
---
 .../xla/service/gpu/ir_emitter_unnested.cc    | 27 ++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index 34ddeb1d41..294a454931 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include <algorithm>
 #include <cstring>
+#include <iterator>
 #include <memory>
 #include <string>
 #include <vector>
@@ -2080,12 +2081,36 @@ Status IrEmitterUnnested::EmitTargetElementLoopInThunk(
   return Status::OK();
 }
 
+namespace {
+
+// Returns true if the fusion contains any instruction that is likely
+// translated to complex LLVM IR, such as loops, and prevent vectorization.
+bool MayPreventVectorization(const HloInstruction& fusion_hlo) {
+  CHECK_EQ(fusion_hlo.opcode(), HloOpcode::kFusion);
+  return absl::c_any_of(
+      fusion_hlo.fused_instructions_computation()->instructions(),
+      [&](const HloInstruction* instr) {
+        switch (instr->opcode()) {
+          case HloOpcode::kReduce:
+          case HloOpcode::kReduceWindow:
+          case HloOpcode::kSort:
+          case HloOpcode::kDot:
+            return true;
+          default:
+            return false;
+        }
+      });
+}
+
+}  // namespace
+
 Status IrEmitterUnnested::EmitTargetElementLoop(
     const HloInstruction& hlo,
     const llvm_ir::ElementGenerator& element_generator) {
   int unroll_factor = 1;
   // Unfused elementwise operations are usually memory bound, unroll them.
-  if (hlo.IsElementwise() || hlo.opcode() == HloOpcode::kFusion) {
+  if (hlo.IsElementwise() ||
+      (hlo.opcode() == HloOpcode::kFusion && !MayPreventVectorization(hlo))) {
     unroll_factor = ComputeMaxUnrollFactor(&hlo);
   }
 
-- 
GitLab


From 4bc799f0b27369b55eb0b6301b6ba9d48811b94e Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Mon, 14 Jan 2019 14:00:07 -0800
Subject: [PATCH 0674/2345] Enable a few control_flow_v2 tests that are now
 working.

PiperOrigin-RevId: 229249264
---
 tensorflow/python/kernel_tests/control_flow_ops_py_test.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 870dd8285e..104e9e82ee 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -1816,7 +1816,6 @@ class ControlFlowTest(test.TestCase):
       self.assertEqual(0.0, sess.run(r, {p: False}))
       self.assertEqual([2.0], sess.run(r1, {p: False}))
 
-  @test_util.disable_control_flow_v2("b/116743589")
   @test_util.run_deprecated_v1
   def testCondWhile_3(self):
     self._testCondWhile_3(use_gpu=False)
@@ -2755,13 +2754,11 @@ class ControlFlowTest(test.TestCase):
       r = gradients_impl.gradients(r, v)[0]
       self.assertAllClose(1024.0, self.evaluate(r))
 
-  @test_util.disable_control_flow_v2("b/117519152")
   @test_util.run_deprecated_v1
   def testWhileCondGrad_Simple(self):
     self._testWhileCondGrad_Simple(use_gpu=False)
     self._testWhileCondGrad_Simple(use_gpu=True)
 
-  @test_util.disable_control_flow_v2("b/117276490")
   @test_util.run_deprecated_v1
   def testWhileCondGrad_UnknownShape(self):
     with self.cached_session() as sess:
@@ -2882,7 +2879,6 @@ class ControlFlowTest(test.TestCase):
       r = gradients_impl.gradients(r.values, values)[0]
       self.assertAllClose(np.array([1024.0, 1024.0]), self.evaluate(r))
 
-  @test_util.disable_control_flow_v2("b/115920078 (gradients)")
   @test_util.run_v1_only("b/120545219")
   def testCallGradInLoop(self):
     with self.cached_session() as sess:
@@ -3045,7 +3041,6 @@ class ControlFlowTest(test.TestCase):
       all_ops = x.graph.get_operations()
       self.assertFalse(any(name in op.name for op in all_ops))
 
-  @test_util.disable_control_flow_v2("b/117954949")
   @test_util.run_deprecated_v1
   def testWhileGradGradFail(self):
     theta = variables.Variable(initial_value=1.)
@@ -3116,7 +3111,6 @@ class ControlFlowTest(test.TestCase):
       self.evaluate(q.initializer)
       self.assertAllClose([1., 1.], self.evaluate(dy_dq))
 
-  @test_util.disable_control_flow_v2("b/115920078 (gradients)")
   @test_util.run_v1_only("b/120545219")
   def testIssue16504(self):
     c = constant_op.constant(np.arange(100), dtype=dtypes.float32)
-- 
GitLab


From 1965757cef0b20c5933f7dd276f41cf4f350d032 Mon Sep 17 00:00:00 2001
From: Guangda Lai <31743510+aaroey@users.noreply.github.com>
Date: Mon, 14 Jan 2019 14:07:27 -0800
Subject: [PATCH 0675/2345] Fix merge conflict correctly

---
 tensorflow/contrib/tensorrt/resources/trt_resources.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tensorrt/resources/trt_resources.h b/tensorflow/contrib/tensorrt/resources/trt_resources.h
index d613c6ae70..11930f3671 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_resources.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_resources.h
@@ -50,7 +50,7 @@ class TRTCalibrationResource : public tensorflow::ResourceBase {
     allocator_.reset();
   }
 
-  string DebugString() override {
+  string DebugString() const override {
     std::stringstream oss;
     using std::dec;
     using std::endl;
-- 
GitLab


From f6593d64cc39e304c77e4ae36530b7c04320bacc Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Mon, 14 Jan 2019 14:08:01 -0800
Subject: [PATCH 0676/2345] Automated rollback of commit
 879af3c38d2405db3ff4c96d6104cdf82a1ab368

PiperOrigin-RevId: 229251035
---
 .../python/data/experimental/ops/batching.py  |  8 ++------
 tensorflow/python/distribute/input_lib.py     | 20 +++----------------
 2 files changed, 5 insertions(+), 23 deletions(-)

diff --git a/tensorflow/python/data/experimental/ops/batching.py b/tensorflow/python/data/experimental/ops/batching.py
index ff8a9182c3..f0cf7f0a99 100644
--- a/tensorflow/python/data/experimental/ops/batching.py
+++ b/tensorflow/python/data/experimental/ops/batching.py
@@ -553,12 +553,8 @@ class _MapAndBatchDataset(dataset_ops.UnaryDataset):
     """See `Dataset.map()` for details."""
     self._input_dataset = input_dataset
 
-    if isinstance(map_func, dataset_ops.StructuredFunctionWrapper):
-      self._map_func = map_func
-    else:
-      self._map_func = dataset_ops.StructuredFunctionWrapper(
-          map_func, "tf.data.experimental.map_and_batch()",
-          dataset=input_dataset)
+    self._map_func = dataset_ops.StructuredFunctionWrapper(
+        map_func, "tf.data.experimental.map_and_batch()", dataset=input_dataset)
     self._batch_size_t = ops.convert_to_tensor(
         batch_size, dtype=dtypes.int64, name="batch_size")
     self._num_parallel_calls_t = ops.convert_to_tensor(
diff --git a/tensorflow/python/distribute/input_lib.py b/tensorflow/python/distribute/input_lib.py
index c64eea1604..3ba80c9287 100644
--- a/tensorflow/python/distribute/input_lib.py
+++ b/tensorflow/python/distribute/input_lib.py
@@ -574,19 +574,12 @@ def _split_dataset_batch(dataset, split_batch_by):
         "The batch operations can be followed by a prefetch.")
 
   batched_dataset = _get_batch_dataset(dataset)
-  prev_dataset = batched_dataset._input_dataset
-
-  num_parallel_calls = None
-  map_func = None
-
   if isinstance(batched_dataset, dataset_ops.BatchDataset):
     batch_size = batched_dataset._batch_size
     drop_remainder = batched_dataset._drop_remainder
   elif isinstance(batched_dataset, batching._MapAndBatchDataset):
     batch_size = batched_dataset._batch_size_t
     drop_remainder = batched_dataset._drop_remainder_t
-    num_parallel_calls = batched_dataset._num_parallel_calls_t
-    map_func = batched_dataset._map_func
 
   prefetch_buffer = None
   if isinstance(dataset, dataset_ops.PrefetchDataset):
@@ -594,6 +587,7 @@ def _split_dataset_batch(dataset, split_batch_by):
   elif (isinstance(dataset, dataset_ops.DatasetV1Adapter)
         and isinstance(dataset._dataset, dataset_ops.PrefetchDataset)):
     prefetch_buffer = dataset._dataset._buffer_size
+  # pylint: enable=protected-access
 
   if tensor_util.is_tensor(batch_size):
     batch_size = tensor_util.constant_value(batch_size)
@@ -601,22 +595,14 @@ def _split_dataset_batch(dataset, split_batch_by):
   if tensor_util.is_tensor(drop_remainder):
     drop_remainder = tensor_util.constant_value(drop_remainder)
 
-  if num_parallel_calls is not None and tensor_util.is_tensor(drop_remainder):
-    num_parallel_calls = tensor_util.constant_value(num_parallel_calls)
-
   if batch_size % split_batch_by:
     raise ValueError(
         "Batch size %s cannot be sharded evenly across replicas %s" % (
             batch_size, split_batch_by))
   new_batch_size = batch_size // split_batch_by
 
-  if isinstance(batched_dataset, dataset_ops.BatchDataset):
-    dataset = prev_dataset.batch(new_batch_size, drop_remainder=drop_remainder)
-  elif isinstance(batched_dataset, batching._MapAndBatchDataset):
-    dataset = prev_dataset.apply(batching.map_and_batch(
-        map_func, new_batch_size, num_parallel_calls, drop_remainder))
-  # pylint: enable=protected-access
-
+  dataset = dataset.apply(batching.unbatch())
+  dataset = dataset.batch(new_batch_size, drop_remainder=drop_remainder)
   if prefetch_buffer is not None:
     dataset = dataset.prefetch(prefetch_buffer)
   return dataset
-- 
GitLab


From f2202f39a6724da9fa4578f82da436a650a59829 Mon Sep 17 00:00:00 2001
From: Yuanzhong Xu <yuanzx@google.com>
Date: Mon, 14 Jan 2019 14:13:33 -0800
Subject: [PATCH 0677/2345] [XLA] Fix HLO verifier for mixed-precision select

kSelect should not be whitelisted.

PiperOrigin-RevId: 229252200
---
 .../compiler/xla/service/hlo_verifier.cc      |  1 -
 .../compiler/xla/service/hlo_verifier_test.cc | 36 +++++++++++++++++++
 2 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc
index 2c69c27ce5..36340d3d78 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier.cc
@@ -702,7 +702,6 @@ Status CheckMixedPrecisionOperands(const HloInstruction* instruction) {
     case HloOpcode::kRecv:
     case HloOpcode::kRecvDone:
     case HloOpcode::kReducePrecision:
-    case HloOpcode::kSelect:
     case HloOpcode::kTupleSelect:
     case HloOpcode::kSend:
     case HloOpcode::kSendDone:
diff --git a/tensorflow/compiler/xla/service/hlo_verifier_test.cc b/tensorflow/compiler/xla/service/hlo_verifier_test.cc
index d27c62a879..de0335a5e8 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier_test.cc
@@ -499,5 +499,41 @@ TEST_F(HloVerifierTest, BitcastCanNotChangeElementType) {
               HasSubstr("Bitcast can not change the element type"));
 }
 
+TEST_F(HloVerifierTest, SelectMixedPrecisionNotAllowed) {
+  const char* const hlo_string = R"(
+  HloModule Module
+
+  ENTRY SelectMixedPrecisionNotAllowed {
+   p0 = pred[] parameter(0)
+   p1 = f32[32] parameter(1)
+   p2 = bf16[32] parameter(2)
+   ROOT select = f32[32] select(p0, p1, p2)
+  }
+  )";
+  TF_ASSERT_OK_AND_ASSIGN(auto module, ParseHloString(hlo_string));
+
+  auto status = verifier().Run(module.get()).status();
+  ASSERT_FALSE(status.ok());
+  EXPECT_THAT(status.error_message(),
+              HasSubstr("Seen floating point types of different precisions"));
+}
+
+TEST_F(HloVerifierTestAllowMixedPrecision, SelectMixedPrecisionAllowed) {
+  const char* const hlo_string = R"(
+  HloModule Module
+
+  ENTRY SelectMixedPrecisionAllowed {
+   p0 = pred[] parameter(0)
+   p1 = f32[32] parameter(1)
+   p2 = bf16[32] parameter(2)
+   ROOT select = f32[32] select(p0, p1, p2)
+  }
+  )";
+  TF_ASSERT_OK_AND_ASSIGN(auto module, ParseHloString(hlo_string));
+
+  auto status = verifier().Run(module.get()).status();
+  ASSERT_TRUE(status.ok());
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 2af4bcf9ded4a872d6675397ebff396a3bdbde5b Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 14 Jan 2019 14:16:43 -0800
Subject: [PATCH 0678/2345] [TF:XLA] Bump open source llvm revision to r350980

PiperOrigin-RevId: 229252812
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 46c1261be2..9a699ae243 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -502,11 +502,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "1b5e02d72413ae0de88a4457ff549434ab9c7b4f583f628eb6f29a9d199e280f",
-        strip_prefix = "llvm-ec5ab87c7698f6ede8905f833db67cd16449e360",
+        sha256 = "066d58d687ea0a97d36135d4aecf6ea31b900cfc74582b2349cadf0c7e4a04fd",
+        strip_prefix = "llvm-06a887ac8a688a23aa09252f44e9bb95797d057f",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/ec5ab87c7698f6ede8905f833db67cd16449e360.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/ec5ab87c7698f6ede8905f833db67cd16449e360.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/06a887ac8a688a23aa09252f44e9bb95797d057f.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/06a887ac8a688a23aa09252f44e9bb95797d057f.tar.gz",
         ],
     )
 
-- 
GitLab


From fec769a899962e2b4792041b4b6baf80b30111c6 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Mon, 14 Jan 2019 14:24:49 -0800
Subject: [PATCH 0679/2345] Enable optimizer v2 lr with eager.

PiperOrigin-RevId: 229254683
---
 .../contrib/distribute/python/keras_test.py   |  3 --
 .../keras/optimizer_v2/adadelta_test.py       | 11 ++--
 .../python/keras/optimizer_v2/adagrad_test.py | 11 ++--
 .../python/keras/optimizer_v2/adam_test.py    | 11 ++--
 .../python/keras/optimizer_v2/adamax_test.py  | 11 ++--
 .../optimizer_v2/gradient_descent_test.py     | 52 ++++++++++++-------
 .../python/keras/optimizer_v2/nadam_test.py   | 15 ++++--
 .../python/keras/optimizer_v2/optimizer_v2.py | 11 +++-
 .../keras/optimizer_v2/optimizer_v2_test.py   | 14 ++---
 .../python/keras/optimizer_v2/rmsprop_test.py | 11 ++--
 .../training/checkpointable/util_test.py      | 10 ++--
 11 files changed, 107 insertions(+), 53 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 40916afcfa..8a607dd070 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -940,9 +940,6 @@ class TestDistributionStrategyWithDatasets(test.TestCase,
   @combinations.generate(all_strategy_combinations())
   def testOptimizerWithCallbacks(self, distribution):
     with self.cached_session():
-      # TODO(b/120946189): Investigate why default strategy + eager fails.
-      if '_Default' in distribution.__class__.__name__:
-        self.skipTest('Disable the test for default strategy.')
       with distribution.scope():
         model = get_model()
         optimizer = gradient_descent_keras.SGD(0.01)
diff --git a/tensorflow/python/keras/optimizer_v2/adadelta_test.py b/tensorflow/python/keras/optimizer_v2/adadelta_test.py
index c95af6a8ad..442a1d3189 100644
--- a/tensorflow/python/keras/optimizer_v2/adadelta_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adadelta_test.py
@@ -170,11 +170,16 @@ class AdadeltaOptimizerTest(test.TestCase):
 
   def testConstructAdadeltaWithLR(self):
     opt = adadelta.Adadelta(lr=1.0, rho=0.9, epsilon=1.)
-    self.assertEqual(opt.lr, 1.0)
     opt_2 = adadelta.Adadelta(learning_rate=0.1, rho=0.9, epsilon=1., lr=1.0)
-    self.assertEqual(opt_2.lr, 1.0)
     opt_3 = adadelta.Adadelta(learning_rate=0.1, rho=0.9, epsilon=1.)
-    self.assertEqual(opt_3.lr, 0.1)
+    self.assertIsInstance(opt.lr, variables.Variable)
+    self.assertIsInstance(opt_2.lr, variables.Variable)
+    self.assertIsInstance(opt_3.lr, variables.Variable)
+
+    self.evaluate(variables.global_variables_initializer())
+    self.assertAllClose(self.evaluate(opt.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_2.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_3.lr), (0.1))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/keras/optimizer_v2/adagrad_test.py b/tensorflow/python/keras/optimizer_v2/adagrad_test.py
index cf6f6a7832..1599a54745 100644
--- a/tensorflow/python/keras/optimizer_v2/adagrad_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adagrad_test.py
@@ -400,11 +400,16 @@ class AdagradOptimizerTest(test.TestCase):
 
   def testConstructAdagradWithLR(self):
     opt = adagrad.Adagrad(lr=1.0)
-    self.assertEqual(opt.lr, 1.0)
     opt_2 = adagrad.Adagrad(learning_rate=0.1, lr=1.0)
-    self.assertEqual(opt_2.lr, 1.0)
     opt_3 = adagrad.Adagrad(learning_rate=0.1)
-    self.assertEqual(opt_3.lr, 0.1)
+    self.assertIsInstance(opt.lr, variables.Variable)
+    self.assertIsInstance(opt_2.lr, variables.Variable)
+    self.assertIsInstance(opt_3.lr, variables.Variable)
+
+    self.evaluate(variables.global_variables_initializer())
+    self.assertAllClose(self.evaluate(opt.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_2.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_3.lr), (0.1))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/keras/optimizer_v2/adam_test.py b/tensorflow/python/keras/optimizer_v2/adam_test.py
index 49a9de41cd..837d0220e1 100644
--- a/tensorflow/python/keras/optimizer_v2/adam_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adam_test.py
@@ -505,11 +505,16 @@ class AdamOptimizerTest(test.TestCase):
 
   def testConstructAdamWithLR(self):
     opt = adam.Adam(lr=1.0)
-    self.assertEqual(opt.lr, 1.0)
     opt_2 = adam.Adam(learning_rate=0.1, lr=1.0)
-    self.assertEqual(opt_2.lr, 1.0)
     opt_3 = adam.Adam(learning_rate=0.1)
-    self.assertEqual(opt_3.lr, 0.1)
+    self.assertIsInstance(opt.lr, variables.Variable)
+    self.assertIsInstance(opt_2.lr, variables.Variable)
+    self.assertIsInstance(opt_3.lr, variables.Variable)
+
+    self.evaluate(variables.global_variables_initializer())
+    self.assertAllClose(self.evaluate(opt.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_2.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_3.lr), (0.1))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/keras/optimizer_v2/adamax_test.py b/tensorflow/python/keras/optimizer_v2/adamax_test.py
index 339c0fe6e6..fe658761c7 100644
--- a/tensorflow/python/keras/optimizer_v2/adamax_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adamax_test.py
@@ -364,11 +364,16 @@ class AdamaxOptimizerTest(test.TestCase):
 
   def testConstructAdamaxWithLR(self):
     opt = adamax.Adamax(lr=1.0)
-    self.assertEqual(opt.lr, 1.0)
     opt_2 = adamax.Adamax(learning_rate=0.1, lr=1.0)
-    self.assertEqual(opt_2.lr, 1.0)
     opt_3 = adamax.Adamax(learning_rate=0.1)
-    self.assertEqual(opt_3.lr, 0.1)
+    self.assertIsInstance(opt.lr, variables.Variable)
+    self.assertIsInstance(opt_2.lr, variables.Variable)
+    self.assertIsInstance(opt_3.lr, variables.Variable)
+
+    self.evaluate(variables.global_variables_initializer())
+    self.assertAllClose(self.evaluate(opt.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_2.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_3.lr), (0.1))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py b/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py
index 9a4178db46..333a6f288e 100644
--- a/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py
+++ b/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py
@@ -263,11 +263,16 @@ class GradientDescentOptimizerTest(test.TestCase):
 
   def testConstructSGDWithLR(self):
     opt = gradient_descent.SGD(lr=1.0)
-    self.assertEqual(opt.lr, 1.0)
     opt_2 = gradient_descent.SGD(learning_rate=0.1, lr=1.0)
-    self.assertEqual(opt_2.lr, 1.0)
     opt_3 = gradient_descent.SGD(learning_rate=0.1)
-    self.assertEqual(opt_3.lr, 0.1)
+    self.assertIsInstance(opt.lr, variables.Variable)
+    self.assertIsInstance(opt_2.lr, variables.Variable)
+    self.assertIsInstance(opt_3.lr, variables.Variable)
+
+    self.evaluate(variables.global_variables_initializer())
+    self.assertAllClose(self.evaluate(opt.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_2.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_3.lr), (0.1))
 
 
 class MomentumOptimizerTest(test.TestCase):
@@ -667,11 +672,16 @@ class MomentumOptimizerTest(test.TestCase):
       opt = gradient_descent.SGD(learning_rate=1.0, momentum=0.9, nesterov=True)
       config = opt.get_config()
       opt2 = gradient_descent.SGD.from_config(config)
-      # assert both are equal float values.
-      self.assertEqual(
-          opt._get_hyper("learning_rate"), opt2._get_hyper("learning_rate"))
-      self.assertEqual(opt._get_hyper("momentum"), opt2._get_hyper("momentum"))
-      # self.assertEqual(opt._get_hyper("decay"), opt2._get_hyper("decay"))
+      lr = opt.lr
+      lr2 = opt2.lr
+      self.evaluate(variables.global_variables_initializer())
+      self.assertAllClose(self.evaluate(lr), self.evaluate(lr2))
+      self.assertAllClose(
+          self.evaluate(opt._get_hyper("momentum")),
+          self.evaluate(opt2._get_hyper("momentum")))
+      self.assertAllClose(
+          self.evaluate(opt._get_hyper("decay")),
+          self.evaluate(opt2._get_hyper("decay")))
       var0 = variables.Variable([[1.0], [2.0]], dtype=dtypes.float32)
       loss = lambda: 3 * var0
       # learning rate variable created when calling minimize.
@@ -679,14 +689,15 @@ class MomentumOptimizerTest(test.TestCase):
       self.evaluate(variables.global_variables_initializer())
       config = opt.get_config()
       opt3 = gradient_descent.SGD.from_config(config)
-      self.assertEqual(
-          self.evaluate(opt._get_hyper("learning_rate")),
-          opt3._get_hyper("learning_rate"))
-      self.assertEqual(
+      lr3 = opt3.lr
+      self.evaluate(variables.global_variables_initializer())
+      self.assertAllClose(self.evaluate(lr), self.evaluate(lr3))
+      self.assertAllClose(
           self.evaluate(opt._get_hyper("momentum")),
-          opt3._get_hyper("momentum"))
-      # self.assertEqual(
-      #     self.evaluate(opt._get_hyper("decay")), opt3._get_hyper("decay"))
+          self.evaluate(opt3._get_hyper("momentum")))
+      self.assertAllClose(
+          self.evaluate(opt._get_hyper("decay")),
+          self.evaluate(opt3._get_hyper("decay")))
       self.assertTrue(opt3.nesterov)
 
   def testNesterovWithoutMomentum(self):
@@ -695,11 +706,16 @@ class MomentumOptimizerTest(test.TestCase):
 
   def testConstructMomentumWithLR(self):
     opt = gradient_descent.SGD(lr=1.0, momentum=0.9)
-    self.assertEqual(opt.lr, 1.0)
     opt_2 = gradient_descent.SGD(learning_rate=0.1, momentum=0.9, lr=1.0)
-    self.assertEqual(opt_2.lr, 1.0)
     opt_3 = gradient_descent.SGD(learning_rate=0.1, momentum=0.9)
-    self.assertEqual(opt_3.lr, 0.1)
+    self.assertIsInstance(opt.lr, variables.Variable)
+    self.assertIsInstance(opt_2.lr, variables.Variable)
+    self.assertIsInstance(opt_3.lr, variables.Variable)
+
+    self.evaluate(variables.global_variables_initializer())
+    self.assertAllClose(self.evaluate(opt.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_2.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_3.lr), (0.1))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/keras/optimizer_v2/nadam_test.py b/tensorflow/python/keras/optimizer_v2/nadam_test.py
index 44fad751bf..b877a80cd8 100644
--- a/tensorflow/python/keras/optimizer_v2/nadam_test.py
+++ b/tensorflow/python/keras/optimizer_v2/nadam_test.py
@@ -161,15 +161,22 @@ class NadamOptimizerTest(test.TestCase):
 
   def testConstructNAdamWithLR(self):
     opt = nadam.Nadam(lr=1.0)
-    self.assertEqual(opt.lr, 1.0)
     opt_2 = nadam.Nadam(learning_rate=0.1, lr=1.0)
-    self.assertEqual(opt_2.lr, 1.0)
     opt_3 = nadam.Nadam(learning_rate=0.1)
-    self.assertEqual(opt_3.lr, 0.1)
+    self.assertIsInstance(opt.lr, variables.Variable)
+    self.assertIsInstance(opt_2.lr, variables.Variable)
+    self.assertIsInstance(opt_3.lr, variables.Variable)
+
+    self.evaluate(variables.global_variables_initializer())
+    self.assertAllClose(self.evaluate(opt.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_2.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_3.lr), (0.1))
 
   def testConstructNAdamWithScheduleDecay(self):
     opt = nadam.Nadam(schedule_decay=0.2)
-    self.assertEqual(opt.decay, 0.2)
+    self.assertIsInstance(opt.decay, variables.Variable)
+    self.evaluate(variables.global_variables_initializer())
+    self.assertAllClose(self.evaluate(opt.decay), (0.2))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
index 0415c97a0b..bc210132b5 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
@@ -95,6 +95,11 @@ class OptimizerV2(checkpointable.CheckpointableBase):
   opt_op.run()
   ```
 
+  ### Thread Compatibility
+
+  The entire optimizer is currently thread compatible, not thread-safe. The user
+  needs to perform synchronization if necessary.
+
   ### Processing gradients before applying them.
 
   Calling `minimize()` takes care of both computing the gradients and
@@ -411,6 +416,8 @@ class OptimizerV2(checkpointable.CheckpointableBase):
         backend.set_value(self._hyper[name], value)
 
   def _get_hyper(self, name, dtype=None):
+    if not self._hypers_created:
+      self._create_hypers()
     value = self._hyper[name]
     if callable(value):
       value = value()
@@ -431,7 +438,7 @@ class OptimizerV2(checkpointable.CheckpointableBase):
       if name == "lr":
         name = "learning_rate"
       if name in self._hyper:
-        return self._hyper[name]
+        return self._get_hyper(name)
       raise e
 
   def __setattr__(self, name, value):
@@ -573,7 +580,7 @@ class OptimizerV2(checkpointable.CheckpointableBase):
 
   def _serialize_hyperparameter(self, hyperparameter_name):
     """Serialize a hyperparameter that can be a float, callable, or Tensor."""
-    value = self._get_hyper(hyperparameter_name)
+    value = self._hyper[hyperparameter_name]
     if callable(value):
       return value()
     if isinstance(value, (ops.Tensor, tf_variables.Variable,
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
index 93a6a7e85e..374ba60460 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
@@ -266,19 +266,19 @@ class OptimizerTest(test.TestCase):
       opt = gradient_descent.SGD(learning_rate=1.0)
       config = opt.get_config()
       opt2 = gradient_descent.SGD.from_config(config)
+      lr = opt._get_hyper('learning_rate')
+      lr2 = opt2._get_hyper('learning_rate')
+      self.evaluate(variables.global_variables_initializer())
       # assert both are equal float values.
-      self.assertEqual(
-          opt._get_hyper('learning_rate'), opt2._get_hyper('learning_rate'))
+      self.assertEqual(self.evaluate(lr), self.evaluate(lr2))
       var0 = variables.Variable([[1.0], [2.0]], dtype=dtypes.float32)
       loss = lambda: 3 * var0
       # learning rate variable created when calling minimize.
       opt.minimize(loss, [var0])
-      self.evaluate(variables.global_variables_initializer())
-      config = opt.get_config()
       opt3 = gradient_descent.SGD.from_config(config)
-      self.assertEqual(
-          self.evaluate(opt._get_hyper('learning_rate')),
-          opt3._get_hyper('learning_rate'))
+      lr3 = opt3._get_hyper('learning_rate')
+      self.evaluate(variables.global_variables_initializer())
+      self.assertEqual(self.evaluate(lr), self.evaluate(lr3))
 
   @test_util.run_in_graph_and_eager_modes
   def testGradClipValue(self):
diff --git a/tensorflow/python/keras/optimizer_v2/rmsprop_test.py b/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
index 67662aae20..d4bc7ce50c 100644
--- a/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
+++ b/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
@@ -434,11 +434,16 @@ class RMSpropOptimizerTest(test.TestCase):
 
   def testConstructRMSpropWithLR(self):
     opt = rmsprop.RMSprop(lr=1.0)
-    self.assertEqual(opt.lr, 1.0)
     opt_2 = rmsprop.RMSprop(learning_rate=0.1, lr=1.0)
-    self.assertEqual(opt_2.lr, 1.0)
     opt_3 = rmsprop.RMSprop(learning_rate=0.1)
-    self.assertEqual(opt_3.lr, 0.1)
+    self.assertIsInstance(opt.lr, variables.Variable)
+    self.assertIsInstance(opt_2.lr, variables.Variable)
+    self.assertIsInstance(opt_3.lr, variables.Variable)
+
+    self.evaluate(variables.global_variables_initializer())
+    self.assertAllClose(self.evaluate(opt.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_2.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_3.lr), (0.1))
 
   def testSlotsUniqueEager(self):
     with context.eager_mode():
diff --git a/tensorflow/python/training/checkpointable/util_test.py b/tensorflow/python/training/checkpointable/util_test.py
index 4987387dc3..78a8282f25 100644
--- a/tensorflow/python/training/checkpointable/util_test.py
+++ b/tensorflow/python/training/checkpointable/util_test.py
@@ -437,8 +437,9 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
     on_create_m_bias_slot = on_create_optimizer.get_slot(
         on_create_model._named_dense.variables[1], "m")
     status.assert_existing_objects_matched()
-    with self.assertRaises(AssertionError):
-      status.assert_consumed()
+    if not context.executing_eagerly():
+      with self.assertRaises(AssertionError):
+        status.assert_consumed()
     # Optimizer slot variables are created when the original variable is
     # restored.
     self.assertAllEqual([1.5], self.evaluate(on_create_m_bias_slot))
@@ -830,8 +831,9 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
     self.assertEqual(12., self.evaluate(new_root.var))
     new_root.optimizer = adam.Adam(0.1)
     slot_status.assert_existing_objects_matched()
-    with self.assertRaisesRegexp(AssertionError, "Unresolved object"):
-      slot_status.assert_consumed()
+    if not context.executing_eagerly():
+      with self.assertRaisesRegexp(AssertionError, "Unresolved object"):
+        slot_status.assert_consumed()
     self.assertEqual(12., self.evaluate(new_root.var))
     if context.executing_eagerly():
       # Slot variables are only created with restoring initializers when
-- 
GitLab


From 6fd7a5c291c2b2f7246def6d5e7551ace5dd6cf9 Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Mon, 14 Jan 2019 15:11:16 -0800
Subject: [PATCH 0680/2345] [Grappler] Change MutableGraphView mutations to
 return Status.

PiperOrigin-RevId: 229263687
---
 tensorflow/core/grappler/BUILD                |   1 +
 .../core/grappler/mutable_graph_view.cc       | 243 ++++++-----
 tensorflow/core/grappler/mutable_graph_view.h |  56 +--
 .../core/grappler/mutable_graph_view_test.cc  | 383 ++++++++++++------
 .../grappler/optimizers/data/filter_fusion.cc |   3 +-
 .../optimizers/data/hoist_random_uniform.cc   |   3 +-
 .../optimizers/data/latency_all_edges.cc      |   2 +-
 .../optimizers/data/make_numa_aware.cc        |   2 +-
 .../optimizers/data/map_and_batch_fusion.cc   |   3 +-
 .../optimizers/data/map_and_filter_fusion.cc  |   3 +-
 .../grappler/optimizers/data/map_fusion.cc    |   3 +-
 .../optimizers/data/map_parallelization.cc    |   3 +-
 .../optimizers/data/map_vectorization.cc      |   3 +-
 .../optimizers/data/noop_elimination.cc       |   2 +-
 .../data/shuffle_and_repeat_fusion.cc         |   3 +-
 15 files changed, 450 insertions(+), 263 deletions(-)

diff --git a/tensorflow/core/grappler/BUILD b/tensorflow/core/grappler/BUILD
index 55d642612b..475fa64201 100644
--- a/tensorflow/core/grappler/BUILD
+++ b/tensorflow/core/grappler/BUILD
@@ -232,5 +232,6 @@ tf_cc_test(
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
         "//tensorflow/core/grappler/inputs:trivial_test_graph_input_yielder",
+        "@com_google_absl//absl/strings",
     ],
 )
diff --git a/tensorflow/core/grappler/mutable_graph_view.cc b/tensorflow/core/grappler/mutable_graph_view.cc
index 5acdb02dee..13a530181c 100644
--- a/tensorflow/core/grappler/mutable_graph_view.cc
+++ b/tensorflow/core/grappler/mutable_graph_view.cc
@@ -27,6 +27,7 @@ limitations under the License.
 #include "tensorflow/core/graph/tensor_id.h"
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/grappler/utils.h"
+#include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
 #include "tensorflow/core/platform/types.h"
 
@@ -35,6 +36,26 @@ namespace grappler {
 
 namespace {
 
+const char kMissingMsg[] = "missing";
+const char kInvalidMsg[] = "invalid";
+const char kNoErrMsg[] = "";
+
+string FaninError(bool tensor_id_valid, bool node_missing) {
+  string s;
+  if (!tensor_id_valid && node_missing) {
+    s = absl::StrCat(" ", kInvalidMsg, "/", kMissingMsg);
+  } else if (!tensor_id_valid) {
+    s = absl::StrCat(" ", kInvalidMsg);
+  } else if (node_missing) {
+    s = absl::StrCat(" ", kMissingMsg);
+  }
+  return s;
+}
+
+string NodeError(bool node_missing) {
+  return node_missing ? absl::StrCat(" ", kMissingMsg) : kNoErrMsg;
+}
+
 bool IsTensorIdPortValid(const TensorId& tensor_id) {
   return tensor_id.index() >= Graph::kControlSlot;
 }
@@ -94,6 +115,8 @@ bool CanDedupControlWithRegularInput(const MutableGraphView& graph,
 }  // namespace
 
 void MutableGraphView::AddAndDedupFanouts(NodeDef* node) {
+  // TODO(lyandy): Checks for self loops, Switch control dependencies and if
+  // fanins exist.
   absl::flat_hash_set<absl::string_view> fanins;
   absl::flat_hash_set<absl::string_view> controlling_fanins;
   int pos = 0;
@@ -183,31 +206,36 @@ NodeDef* MutableGraphView::AddNode(NodeDef&& node) {
   return node_in_graph;
 }
 
-bool MutableGraphView::UpdateFanouts(absl::string_view from_node,
-                                     absl::string_view to_node) {
-  if (from_node == to_node) {
-    return false;
-  }
+Status MutableGraphView::UpdateFanouts(absl::string_view from_node,
+                                       absl::string_view to_node) {
   NodeDef* from_node_ptr = GetNode(from_node);
   NodeDef* to_node_ptr = GetNode(to_node);
   if (from_node_ptr && to_node_ptr) {
     return UpdateFanoutsInternal(from_node_ptr, to_node_ptr);
   } else if (!from_node_ptr) {
-    LOG(WARNING) << absl::Substitute(
+    return errors::Internal(absl::Substitute(
         "Can't update fanouts from '$0' to '$1', from node was not found.",
-        from_node, to_node);
-  } else {
-    LOG(WARNING) << absl::Substitute(
+        from_node, to_node));
+  } else if (!to_node_ptr) {
+    return errors::Internal(absl::Substitute(
         "Can't update fanouts from '$0' to '$1', to node was not found.",
-        from_node, to_node);
+        from_node, to_node));
+  } else {
+    return errors::Internal(
+        absl::Substitute("Can't update fanouts from '$0' to '$1', from and to "
+                         "nodes were not found.",
+                         from_node, to_node));
   }
-  return false;
+  return Status::OK();
 }
 
-bool MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
-                                             NodeDef* to_node) {
+Status MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
+                                               NodeDef* to_node) {
   VLOG(2) << absl::Substitute("Update fanouts from '$0' to '$1'.",
                               from_node->name(), to_node->name());
+  if (from_node == to_node) {
+    return Status::OK();
+  }
 
   // Update internal state with the new output_port->input_port edge.
   const auto add_edge = [this](const OutputPort& output_port,
@@ -221,8 +249,6 @@ bool MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
     fanouts()[output_port].erase(input_port);
   };
 
-  bool modified = false;
-
   // For the control fanouts we do not know the input index in a NodeDef,
   // so we have to traverse all control inputs.
 
@@ -236,18 +262,17 @@ bool MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
 
     // Can't add Switch node as a control dependency.
     if (to_node_is_switch) {
-      // This will only be printed to the log if we are trying to add a Switch
-      // as a control dependency, which if allowed will make the graph invalid.
-      LOG(WARNING) << absl::Substitute(
-          "Can't update fanouts from '$0' to '$1', to node is being added as a "
-          "control dependency when it is a Switch.",
-          from_node, to_node);
-      return false;
+      // Trying to add a Switch as a control dependency, which if allowed will
+      // make the graph invalid.
+      return errors::Internal(
+          absl::Substitute("Can't update fanouts from '$0' to '$1', to node is "
+                           "being added as a Switch control dependency.",
+                           from_node->name(), to_node->name()));
     }
 
     NodeDef* node = control_port.node;
-    modified |= RemoveControllingFaninInternal(node, from_node);
-    modified |= AddFaninInternal(node, {to_node, Graph::kControlSlot});
+    RemoveControllingFaninInternal(node, from_node);
+    AddFaninInternal(node, {to_node, Graph::kControlSlot});
   }
 
   // First we update regular fanouts. For the regular fanouts
@@ -287,7 +312,6 @@ bool MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
     if (CanDedupControlWithRegularInput(*this, *to_node)) {
       RemoveControllingFaninInternal(input_port.node, to_node);
     }
-    modified = true;
   }
 
   // Because we update all regular fanouts of `from_node`, we can just copy
@@ -301,7 +325,7 @@ bool MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
     max_regular_output_port().erase(from_node);
   }
 
-  return modified;
+  return Status::OK();
 }
 
 bool MutableGraphView::AddFaninInternal(NodeDef* node,
@@ -351,43 +375,46 @@ bool MutableGraphView::AddFaninInternal(NodeDef* node,
   return true;
 }
 
-bool MutableGraphView::AddFaninInternal(NodeDef* node, const TensorId& fanin) {
+Status MutableGraphView::AddRegularFanin(absl::string_view node_name,
+                                         const TensorId& fanin) {
+  NodeDef* node = GetNode(node_name);
   NodeDef* fanin_node = GetNode(fanin.node());
-  if (fanin_node == nullptr) {
-    return false;
-  }
-  return AddFaninInternal(node, {fanin_node, fanin.index()});
-}
 
-bool MutableGraphView::AddRegularFanin(absl::string_view node_name,
-                                       const TensorId& fanin) {
-  if (!IsTensorIdRegular(fanin)) {
-    return false;
-  }
-  NodeDef* node = GetNode(node_name);
-  if (node == nullptr) {
-    return false;
+  string node_err = NodeError(/*node_missing=*/node == nullptr);
+  string fanin_err = FaninError(IsTensorIdRegular(fanin),
+                                /*node_missing=*/fanin_node == nullptr);
+  if (!node_err.empty() || !fanin_err.empty()) {
+    return errors::Internal(absl::Substitute(
+        "Can't add$0 fanin '$1' as regular fanin to$2 node '$3'.", fanin_err,
+        fanin.ToString(), node_err, node_name));
   }
-  return AddFaninInternal(node, fanin);
+
+  AddFaninInternal(node, {fanin_node, fanin.index()});
+  return Status::OK();
 }
 
-bool MutableGraphView::AddControllingFanin(absl::string_view node_name,
-                                           const TensorId& fanin) {
+Status MutableGraphView::AddControllingFanin(absl::string_view node_name,
+                                             const TensorId& fanin) {
   NodeDef* node = GetNode(node_name);
-  if (node == nullptr) {
-    return false;
-  }
   NodeDef* fanin_node = GetNode(fanin.node());
-  if (fanin_node == nullptr) {
-    return false;
+
+  string node_err = NodeError(/*node_missing=*/node == nullptr);
+  string fanin_err = FaninError(IsTensorIdPortValid(fanin),
+                                /*node_missing=*/fanin_node == nullptr);
+  if (!node_err.empty() || !fanin_err.empty()) {
+    return errors::Internal(
+        absl::Substitute("Can't add$0 controlling fanin '$1' to$2 node '$3'.",
+                         fanin_err, fanin.ToString(), node_err, node_name));
   }
 
   if (!IsSwitch(*fanin_node)) {
-    return AddFaninInternal(node, {fanin_node, Graph::kControlSlot});
+    AddFaninInternal(node, {fanin_node, Graph::kControlSlot});
   } else {
     if (IsTensorIdControlling(fanin)) {
       // Can't add a Switch node control dependency.
-      return false;
+      return errors::Internal(absl::Substitute(
+          "Can't add Switch as controlling fanin '$0' to node '$1'.",
+          fanin.ToString(), node_name));
     }
     // We can't anchor control dependencies directly on the switch node: unlike
     // other nodes only one of the outputs of the switch node will be generated
@@ -399,7 +426,8 @@ bool MutableGraphView::AddControllingFanin(absl::string_view node_name,
     for (auto fanout : fanouts) {
       if (IsIdentity(*fanout.node) || IsIdentityNSingleInput(*fanout.node)) {
         if (ParseTensorName(fanout.node->input(0)) == fanin) {
-          return AddFaninInternal(node, {fanout.node, Graph::kControlSlot});
+          AddFaninInternal(node, {fanout.node, Graph::kControlSlot});
+          return Status::OK();
         }
       }
     }
@@ -420,8 +448,9 @@ bool MutableGraphView::AddControllingFanin(absl::string_view node_name,
       new_node.add_input(TensorIdToString(fanin));
       ctrl_dep_node = AddNode(std::move(new_node));
     }
-    return AddFaninInternal(node, {ctrl_dep_node, Graph::kControlSlot});
+    AddFaninInternal(node, {ctrl_dep_node, Graph::kControlSlot});
   }
+  return Status::OK();
 }
 
 bool MutableGraphView::RemoveRegularFaninInternal(NodeDef* node,
@@ -473,20 +502,22 @@ bool MutableGraphView::RemoveRegularFaninInternal(NodeDef* node,
   return modified;
 }
 
-bool MutableGraphView::RemoveRegularFanin(absl::string_view node_name,
-                                          const TensorId& fanin) {
-  if (!IsTensorIdRegular(fanin)) {
-    return false;
-  }
+Status MutableGraphView::RemoveRegularFanin(absl::string_view node_name,
+                                            const TensorId& fanin) {
   NodeDef* node = GetNode(node_name);
-  if (node == nullptr) {
-    return false;
-  }
   NodeDef* fanin_node = GetNode(fanin.node());
-  if (fanin_node == nullptr) {
-    return false;
+
+  string node_err = NodeError(/*node_missing=*/node == nullptr);
+  string fanin_err = FaninError(IsTensorIdRegular(fanin),
+                                /*node_missing=*/fanin_node == nullptr);
+  if (!node_err.empty() || !fanin_err.empty()) {
+    return errors::Internal(absl::Substitute(
+        "Can't remove$0 fanin '$1' as regular fanin from$2 node '$3'.",
+        fanin_err, fanin.ToString(), node_err, node_name));
   }
-  return RemoveRegularFaninInternal(node, {fanin_node, fanin.index()});
+
+  RemoveRegularFaninInternal(node, {fanin_node, fanin.index()});
+  return Status::OK();
 }
 
 bool MutableGraphView::RemoveControllingFaninInternal(NodeDef* node,
@@ -507,31 +538,42 @@ bool MutableGraphView::RemoveControllingFaninInternal(NodeDef* node,
   return false;
 }
 
-bool MutableGraphView::RemoveControllingFanin(
+Status MutableGraphView::RemoveControllingFanin(
     absl::string_view node_name, absl::string_view fanin_node_name) {
   NodeDef* node = GetNode(node_name);
-  if (node == nullptr) {
-    return false;
-  }
-  NodeDef* fanin = GetNode(fanin_node_name);
-  if (fanin == nullptr) {
-    return false;
+  NodeDef* fanin_node = GetNode(fanin_node_name);
+
+  string node_err = NodeError(/*node_missing=*/node == nullptr);
+  string fanin_err = NodeError(/*node_missing=*/fanin_node == nullptr);
+  if (!node_err.empty() || !fanin_err.empty()) {
+    return errors::Internal(absl::Substitute(
+        "Can't remove$0 controlling fanin '$1' from$2 node '$3'.", fanin_err,
+        AsControlDependency(string(fanin_node_name)), node_err, node_name));
   }
-  return RemoveControllingFaninInternal(node, fanin);
+
+  RemoveControllingFaninInternal(node, fanin_node);
+  return Status::OK();
 }
 
-bool MutableGraphView::RemoveAllFanins(absl::string_view node_name,
-                                       bool keep_controlling_fanins) {
+Status MutableGraphView::RemoveAllFanins(absl::string_view node_name,
+                                         bool keep_controlling_fanins) {
   NodeDef* node = GetNode(node_name);
-  if (node == nullptr || node->input().empty()) {
-    return false;
+
+  if (node == nullptr) {
+    return errors::Internal(absl::Substitute(
+        "Can't remove all fanins from missing node '$0'.", node_name));
   }
+
+  if (node->input().empty()) {
+    return Status::OK();
+  }
+
   RemoveFaninsInternal(node, keep_controlling_fanins);
   if (keep_controlling_fanins) {
     int num_non_controlling_fanins =
         NumFanins(*node, /*include_controlling_nodes=*/false);
     if (num_non_controlling_fanins == 0) {
-      return false;
+      return Status::OK();
     } else if (num_non_controlling_fanins < node->input_size()) {
       node->mutable_input()->DeleteSubrange(0, num_non_controlling_fanins);
     } else {
@@ -540,35 +582,45 @@ bool MutableGraphView::RemoveAllFanins(absl::string_view node_name,
   } else {
     node->clear_input();
   }
-  return true;
+  return Status::OK();
 }
 
-bool MutableGraphView::UpdateFanin(absl::string_view node_name,
-                                   const TensorId& from_fanin,
-                                   const TensorId& to_fanin) {
-  if (from_fanin == to_fanin || !IsTensorIdPortValid(from_fanin) ||
-      !IsTensorIdPortValid(to_fanin)) {
-    return false;
-  }
+Status MutableGraphView::UpdateFanin(absl::string_view node_name,
+                                     const TensorId& from_fanin,
+                                     const TensorId& to_fanin) {
   NodeDef* node = GetNode(node_name);
-  if (node == nullptr) {
-    return false;
-  }
-
   NodeDef* from_fanin_node = GetNode(from_fanin.node());
   NodeDef* to_fanin_node = GetNode(to_fanin.node());
-  if (from_fanin_node == nullptr || to_fanin_node == nullptr) {
-    return false;
+
+  string node_err = NodeError(/*node_missing=*/node == nullptr);
+  string from_fanin_err =
+      FaninError(IsTensorIdPortValid(from_fanin),
+                 /*node_missing=*/from_fanin_node == nullptr);
+  string to_fanin_err = FaninError(IsTensorIdPortValid(to_fanin),
+                                   /*node_missing=*/to_fanin_node == nullptr);
+  if (!node_err.empty() || !from_fanin_err.empty() || !to_fanin_err.empty()) {
+    return errors::Internal(absl::Substitute(
+        "Can't update$0 fanin '$1' to$2 fanin '$3' in$4 node '$5'.",
+        from_fanin_err, from_fanin.ToString(), to_fanin_err,
+        to_fanin.ToString(), node_err, node_name));
   }
 
   // When replacing a non control dependency fanin with a control dependency, or
   // vice versa, remove and add, so ports can be updated properly in fanout(s).
-  bool from_fanin_is_control = IsTensorIdControlling(from_fanin);
   bool to_fanin_is_control = IsTensorIdControlling(to_fanin);
   if (to_fanin_is_control && IsSwitch(*to_fanin_node)) {
     // Can't add Switch node as a control dependency.
-    return false;
+    return errors::Internal(absl::Substitute(
+        "Can't update fanin '$0' to fanin '$1' in node '$2', to fanin is a "
+        "Switch control dependency.",
+        from_fanin.ToString(), to_fanin.ToString(), node_name));
+  }
+
+  if (from_fanin == to_fanin) {
+    return Status::OK();
   }
+
+  bool from_fanin_is_control = IsTensorIdControlling(from_fanin);
   if (from_fanin_is_control || to_fanin_is_control) {
     bool modified = false;
     if (from_fanin_is_control) {
@@ -580,8 +632,7 @@ bool MutableGraphView::UpdateFanin(absl::string_view node_name,
     if (modified) {
       AddFaninInternal(node, {to_fanin_node, to_fanin.index()});
     }
-
-    return modified;
+    return Status::OK();
   }
 
   // In place mutation, requires no shifting of ports.
@@ -629,10 +680,12 @@ bool MutableGraphView::UpdateFanin(absl::string_view node_name,
     }
   }
 
-  return modified;
+  return Status::OK();
 }
 
 void MutableGraphView::DeleteNodes(const std::set<string>& nodes_to_delete) {
+  // TODO(lyandy): Check if nodes have fanouts before deleting and return
+  // Status.
   for (const string& node_name_to_delete : nodes_to_delete)
     RemoveFaninsInternal(nodes().at(node_name_to_delete),
                          /*keep_controlling_fanins=*/false);
diff --git a/tensorflow/core/grappler/mutable_graph_view.h b/tensorflow/core/grappler/mutable_graph_view.h
index 0a441d9115..59740f2c71 100644
--- a/tensorflow/core/grappler/mutable_graph_view.h
+++ b/tensorflow/core/grappler/mutable_graph_view.h
@@ -28,6 +28,7 @@ limitations under the License.
 #include "tensorflow/core/graph/tensor_id.h"
 #include "tensorflow/core/grappler/graph_view.h"
 #include "tensorflow/core/grappler/op_types.h"
+#include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
@@ -74,18 +75,14 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   //   1. foo1(new_bar:0, new_bar:1, other:0)
   //   2. foo2(new_bar:1, other:1)
   //   3. foo3(other:2, ^new_bar)
-  //
-  // This will return true iff the nodes are modified.
-  bool UpdateFanouts(absl::string_view from_node, absl::string_view to_node);
+  Status UpdateFanouts(absl::string_view from_node, absl::string_view to_node);
 
   // Adds regular fanin `fanin` to node `node_name`. If the node or fanin do not
   // exist in the graph, nothing will be modified in the graph. Otherwise fanin
   // will be added after existing non control dependency fanins. Control
   // dependencies will be deduped. To add control dependencies, use
   // AddControllingFanin.
-  //
-  // This will return true iff the node is modified.
-  bool AddRegularFanin(absl::string_view node_name, const TensorId& fanin);
+  Status AddRegularFanin(absl::string_view node_name, const TensorId& fanin);
 
   // Adds control dependency `fanin` to the target node named `node_name`. To
   // add regular fanins, use AddRegularFanin.
@@ -104,42 +101,35 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // end up with:
   //
   //   fanin -> Identity{N} -^> target node.
-  //
-  // This will return true iff the node is modified.
-  bool AddControllingFanin(absl::string_view node_name, const TensorId& fanin);
+  Status AddControllingFanin(absl::string_view node_name,
+                             const TensorId& fanin);
 
   // Removes regular fanin `fanin` from node `node_name`. If the node or fanin
   // do not exist in the graph, nothing will be modified in the graph. If there
   // are multiple inputs that match the fanin, all of them will be removed. To
   // remove controlling fanins, use RemoveControllingFanin.
-  //
-  // This will return true iff the node is modified.
-  bool RemoveRegularFanin(absl::string_view node_name, const TensorId& fanin);
+  Status RemoveRegularFanin(absl::string_view node_name, const TensorId& fanin);
 
   // Removes control dependency `fanin_node_name` from the target node named
   // `node_name`. If the node or fanin do not exist in the graph, nothing will
   // be modified in the graph. To remove regular fanins, use RemoveRegualrFanin.
-  //
-  // This will return true iff the node is modified.
-  bool RemoveControllingFanin(absl::string_view node_name,
-                              absl::string_view fanin_node_name);
+  Status RemoveControllingFanin(absl::string_view node_name,
+                                absl::string_view fanin_node_name);
 
   // Removes all fanins from node `node_name`. Control dependencies will be
   // retained if keep_controlling_fanins is true.
-  //
-  // This will return true iff the node is modified.
-  bool RemoveAllFanins(absl::string_view node_name,
-                       bool keep_controlling_fanins);
+  Status RemoveAllFanins(absl::string_view node_name,
+                         bool keep_controlling_fanins);
 
   // Replaces all fanins `from_fanin` with `to_fanin` in node `node_name`. If
   // the fanins or node do not exist, nothing will be modified in the graph.
   // Control dependencies will be deduped.
-  //
-  // This will return true iff the node is modified.
-  bool UpdateFanin(absl::string_view node_name, const TensorId& from_fanin,
-                   const TensorId& to_fanin);
+  Status UpdateFanin(absl::string_view node_name, const TensorId& from_fanin,
+                     const TensorId& to_fanin);
 
   // Deletes nodes from the graph.
+  // TODO(lyandy): Check if nodes have fanouts before deleting and return
+  // Status.
   void DeleteNodes(const std::set<string>& nodes_to_delete);
 
  private:
@@ -171,7 +161,7 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   //
   // IMPORTANT: If `from_node` or `to_node` is not in the underlying graph, the
   // behavior is undefined.
-  bool UpdateFanoutsInternal(NodeDef* from_node, NodeDef* to_node);
+  Status UpdateFanoutsInternal(NodeDef* from_node, NodeDef* to_node);
 
   // Removes fanins of the deleted node from internal state. Control
   // dependencies are retained iff keep_controlling_fanins is true.
@@ -181,29 +171,13 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // Adds fanin to node. If fanin is a control dependency, existing control
   // dependencies will be checked first before adding. Otherwise fanin will be
   // added after existing non control dependency inputs.
-  //
-  // This will return true iff the node is modified. If a control dependency
-  // already exists, the node will not be modified.
   bool AddFaninInternal(NodeDef* node, const OutputPort& fanin);
 
-  // Adds fanin to node. If the node or fanin do not exist in the graph, nothing
-  // will be modified in the graph. If fanin is a control dependency, existing
-  // control dependencies will be checked first before adding. Otherwise fanin
-  // will be added after existing non control dependency inputs.
-  //
-  // This will return true iff the node is modified. If a control dependency
-  // already exists, the node will not be modified.
-  bool AddFaninInternal(NodeDef* node, const TensorId& fanin);
-
   // Removes all instances of regular fanin `fanin` from node `node`.
-  //
-  // This will return true iff the node is modified.
   bool RemoveRegularFaninInternal(NodeDef* node, const OutputPort& fanin);
 
   // Removes controlling fanin `fanin_node` from node if such controlling fanin
   // exists.
-  //
-  // This will return true iff the node is modified.
   bool RemoveControllingFaninInternal(NodeDef* node, NodeDef* fanin_node);
 };
 
diff --git a/tensorflow/core/grappler/mutable_graph_view_test.cc b/tensorflow/core/grappler/mutable_graph_view_test.cc
index b44e1a360d..99a69ef263 100644
--- a/tensorflow/core/grappler/mutable_graph_view_test.cc
+++ b/tensorflow/core/grappler/mutable_graph_view_test.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/grappler/mutable_graph_view.h"
+#include "absl/strings/substitute.h"
 #include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/core/framework/function_testlib.h"
 #include "tensorflow/core/framework/types.pb.h"
@@ -44,7 +45,7 @@ TEST(MutableGraphViewTest, AddAndUpdateFanouts) {
   NodeDef* new_bar = graph.AddNode(NDef("new_bar", "NotImportant", {}, {}));
   NodeDef* bar = graph.GetNode("bar");
 
-  EXPECT_TRUE(graph.UpdateFanouts(bar->name(), new_bar->name()));
+  EXPECT_TRUE(graph.UpdateFanouts(bar->name(), new_bar->name()).ok());
 
   // Fanout nodes must have their inputs updated.
   NodeDef* foo_1 = graph.GetNode("foo_1");
@@ -93,7 +94,7 @@ TEST(MutableGraphViewTest, AddAndUpdateFanoutsKeepControls) {
   NodeDef* new_bar = graph.AddNode(NDef("new_bar", "Identity", {"bar_1:2"}));
   NodeDef* bar_2 = graph.GetNode("bar_2");
 
-  EXPECT_TRUE(graph.UpdateFanouts(bar_2->name(), new_bar->name()));
+  EXPECT_TRUE(graph.UpdateFanouts(bar_2->name(), new_bar->name()).ok());
 
   // Fanout nodes must have their inputs updated.
   NodeDef* foo_1 = graph.GetNode("foo_1");
@@ -138,7 +139,7 @@ TEST(MutableGraphViewTest, AddAndUpdateFanoutsWithoutSelfLoops) {
   NodeDef* new_bar = graph.AddNode(NDef("new_bar", "NewBar", {"bar"}, {}));
   NodeDef* bar = graph.GetNode("bar");
 
-  EXPECT_TRUE(graph.UpdateFanouts("bar", new_bar->name()));
+  EXPECT_TRUE(graph.UpdateFanouts("bar", new_bar->name()).ok());
 
   // Foo node must read from `new_bar`.
   NodeDef* foo_1 = graph.GetNode("foo_1");
@@ -197,8 +198,18 @@ TEST(MutableGraphViewTest, UpdateFanoutsToSwitchWithControlFromSwitch) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_FALSE(graph.UpdateFanouts("a", "b"));
-  EXPECT_FALSE(graph.UpdateFanouts("d", "b"));
+  Status s = graph.UpdateFanouts("a", "b");
+  EXPECT_FALSE(s.ok());
+  string expected_msg =
+      "Can't update fanouts from 'a' to 'b', to node is being added as a "
+      "Switch control dependency.";
+  EXPECT_EQ(s.error_message(), expected_msg);
+  s = graph.UpdateFanouts("d", "b");
+  EXPECT_FALSE(s.ok());
+  expected_msg =
+      "Can't update fanouts from 'd' to 'b', to node is being added as a "
+      "Switch control dependency.";
+  EXPECT_EQ(s.error_message(), expected_msg);
 
   EXPECT_EQ(graph.graph()->node_size(), 5);
 
@@ -234,7 +245,7 @@ TEST(MutableGraphViewTest, UpdateFanoutsToSwitchWithNoControlFromSwitch) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.UpdateFanouts("c", "b"));
+  EXPECT_TRUE(graph.UpdateFanouts("c", "b").ok());
 
   EXPECT_EQ(graph.graph()->node_size(), 5);
 
@@ -277,7 +288,8 @@ GraphDef SimpleMutateFaninGraph() {
 }
 
 void TestAddRegularFanin(absl::string_view node_name,
-                         const TensorId& fanin_to_add, bool modified,
+                         const TensorId& fanin_to_add, bool success,
+                         const string& error_msg,
                          const NodeDef* expected_node) {
   GraphDef graph_def = SimpleMutateFaninGraph();
 
@@ -290,7 +302,11 @@ void TestAddRegularFanin(absl::string_view node_name,
     EXPECT_NE(node, nullptr);
   }
 
-  EXPECT_EQ(modified, graph.AddRegularFanin(node_name, fanin_to_add));
+  Status s = graph.AddRegularFanin(node_name, fanin_to_add);
+  EXPECT_EQ(s.ok(), success);
+  if (!success) {
+    EXPECT_EQ(s.error_message(), error_msg);
+  }
   if (expected_node != nullptr) {
     CompareNodeInputs(graph, expected_node, node);
   }
@@ -298,63 +314,92 @@ void TestAddRegularFanin(absl::string_view node_name,
 
 TEST(MutableGraphViewTest, AddRegularFanin) {
   NodeDef expected_node;
+  string error_msg;
   // Add input to node with 1 input 0 controls.
   expected_node = NDef("", "", {"a", "b:1"});
-  TestAddRegularFanin("foo_1", {"b", 1}, /*modified=*/true, &expected_node);
+  TestAddRegularFanin("foo_1", {"b", 1}, /*success=*/true, error_msg,
+                      &expected_node);
   // Add input to node with multiple inputs and 0 controls.
   expected_node = NDef("", "", {"b", "a:1", "a:1", "b:2"});
-  TestAddRegularFanin("foo_3", {"b", 2}, /*modified=*/true, &expected_node);
+  TestAddRegularFanin("foo_3", {"b", 2}, /*success=*/true, error_msg,
+                      &expected_node);
   // Add input to node with 1 input multiple controls.
   expected_node = NDef("", "", {"b", "a", "^c"});
-  TestAddRegularFanin("foo_2", {"a", 0}, /*modified=*/true, &expected_node);
+  TestAddRegularFanin("foo_2", {"a", 0}, /*success=*/true, error_msg,
+                      &expected_node);
   // Add input to node with multiple inputs and controls.
   expected_node = NDef("", "", {"a", "b:2", "b:2", "a:1", "^d", "^c"});
-  TestAddRegularFanin("foo_4", {"a", 1}, /*modified=*/true, &expected_node);
+  TestAddRegularFanin("foo_4", {"a", 1}, /*success=*/true, error_msg,
+                      &expected_node);
   // Add input to node with 0 inputs 0 controls.
   expected_node = NDef("", "", {"a:1"});
-  TestAddRegularFanin("foo_5", {"a", 1}, /*modified=*/true, &expected_node);
+  TestAddRegularFanin("foo_5", {"a", 1}, /*success=*/true, error_msg,
+                      &expected_node);
   // Add input to node with 0 inputs multiple controls.
   expected_node = NDef("", "", {"c:1", "^b", "^a"});
-  TestAddRegularFanin("foo_6", {"c", 1}, /*modified=*/true, &expected_node);
+  TestAddRegularFanin("foo_6", {"c", 1}, /*success=*/true, error_msg,
+                      &expected_node);
 
   // Add control to node with 1 input 0 controls.
   expected_node = NDef("", "", {"a"});
-  TestAddRegularFanin("foo_1", {"b", Graph::kControlSlot}, /*modified=*/false,
-                      &expected_node);
+  error_msg = "Can't add invalid fanin '^b' as regular fanin to node 'foo_1'.";
+  TestAddRegularFanin("foo_1", {"b", Graph::kControlSlot}, /*success=*/false,
+                      error_msg, &expected_node);
   // Add control to node with multiple inputs and 0 controls.
   expected_node = NDef("", "", {"b", "a:1", "a:1"});
-  TestAddRegularFanin("foo_3", {"c", Graph::kControlSlot}, /*modified=*/false,
-                      &expected_node);
+  error_msg = "Can't add invalid fanin '^c' as regular fanin to node 'foo_3'.";
+  TestAddRegularFanin("foo_3", {"c", Graph::kControlSlot}, /*success=*/false,
+                      error_msg, &expected_node);
   // Add control to node with 1 input multiple controls.
   expected_node = NDef("", "", {"b", "^a", "^c"});
-  TestAddRegularFanin("foo_2", {"d", Graph::kControlSlot}, /*modified=*/false,
-                      &expected_node);
+  error_msg = "Can't add invalid fanin '^d' as regular fanin to node 'foo_2'.";
+  TestAddRegularFanin("foo_2", {"d", Graph::kControlSlot}, /*success=*/false,
+                      error_msg, &expected_node);
   // Add control to node with multiple input multiple controls.
   expected_node = NDef("", "", {"a", "b:2", "b:2", "^c", "^d"});
+  error_msg = "Can't add invalid fanin '^a' as regular fanin to node 'foo_4'.";
   TestAddRegularFanin("foo_4", {"a", Graph::kControlSlot},
-                      /*modified=*/false, &expected_node);
+                      /*success=*/false, error_msg, &expected_node);
   // Add control to node with 0 inputs 0 controls.
   expected_node = NDef("", "", {});
-  TestAddRegularFanin("foo_5", {"a", Graph::kControlSlot}, /*modified=*/false,
-                      &expected_node);
+  error_msg = "Can't add invalid fanin '^a' as regular fanin to node 'foo_5'.";
+  TestAddRegularFanin("foo_5", {"a", Graph::kControlSlot}, /*success=*/false,
+                      error_msg, &expected_node);
   // Add control to node with 0 inputs multiple controls.
   expected_node = NDef("", "", {"^a", "^b"});
-  TestAddRegularFanin("foo_6", {"c", Graph::kControlSlot}, /*modified=*/false,
-                      &expected_node);
+  error_msg = "Can't add invalid fanin '^c' as regular fanin to node 'foo_6'.";
+  TestAddRegularFanin("foo_6", {"c", Graph::kControlSlot}, /*success=*/false,
+                      error_msg, &expected_node);
   // Add control to node with control that already exists.
   expected_node = NDef("", "", {"b", "^a", "^c"});
+  error_msg = "Can't add invalid fanin '^a' as regular fanin to node 'foo_2'.";
   TestAddRegularFanin("foo_2", {"a", Graph::kControlSlot},
-                      /*modified=*/false, &expected_node);
+                      /*success=*/false, error_msg, &expected_node);
 
   // Add fanin to node where node is missing.
-  TestAddRegularFanin("foo_missing", {"a", 0}, /*modified=*/false, nullptr);
+  error_msg =
+      "Can't add fanin 'a:0' as regular fanin to missing node 'foo_missing'.";
+  TestAddRegularFanin("foo_missing", {"a", 0}, /*success=*/false, error_msg,
+                      nullptr);
   // Add fanin to node where fanin is missing.
   expected_node = NDef("", "", {"a"});
-  TestAddRegularFanin("foo_1", {"bar_missing", 0}, /*modified=*/false,
+  error_msg =
+      "Can't add missing fanin 'bar_missing:0' as regular fanin to node "
+      "'foo_1'.";
+  TestAddRegularFanin("foo_1", {"bar_missing", 0}, /*success=*/false, error_msg,
                       &expected_node);
   // Add fanin to node where node and fanin are missing.
-  TestAddRegularFanin("foo_missing", {"bar_missing", 0}, /*modified=*/false,
-                      /*expected_node=*/nullptr);
+  error_msg =
+      "Can't add missing fanin 'bar_missing:0' as regular fanin to missing "
+      "node 'foo_missing'.";
+  TestAddRegularFanin("foo_missing", {"bar_missing", 0}, /*success=*/false,
+                      error_msg, /*expected_node=*/nullptr);
+  // Add control fanin to node where node and fanin are missing.
+  error_msg =
+      "Can't add invalid/missing fanin '^bar_missing' as regular fanin to "
+      "missing node 'foo_missing'.";
+  TestAddRegularFanin("foo_missing", {"bar_missing", Graph::kControlSlot},
+                      /*success=*/false, error_msg, /*expected_node=*/nullptr);
 }
 
 void CheckFanout(const MutableGraphView& graph, const TensorId& fanin,
@@ -368,7 +413,8 @@ void CheckFanout(const MutableGraphView& graph, const TensorId& fanin,
 }
 
 void TestRemoveRegularFanin(absl::string_view node_name,
-                            const TensorId& fanin_to_remove, bool modified,
+                            const TensorId& fanin_to_remove, bool success,
+                            const string& error_msg,
                             const NodeDef* expected_node) {
   GraphDef graph_def = SimpleMutateFaninGraph();
 
@@ -381,10 +427,14 @@ void TestRemoveRegularFanin(absl::string_view node_name,
     EXPECT_NE(nullptr, node);
   }
 
-  EXPECT_EQ(modified, graph.RemoveRegularFanin(node_name, fanin_to_remove));
+  Status s = graph.RemoveRegularFanin(node_name, fanin_to_remove);
+  EXPECT_EQ(s.ok(), success);
+  if (!success) {
+    EXPECT_EQ(s.error_message(), error_msg);
+  }
   if (expected_node != nullptr) {
     CompareNodeInputs(graph, expected_node, node);
-    if (modified) {
+    if (success) {
       CheckFanout(graph, fanin_to_remove, node_name);
     }
   }
@@ -392,70 +442,109 @@ void TestRemoveRegularFanin(absl::string_view node_name,
 
 TEST(MutableGraphViewTest, RemoveRegularFanin) {
   NodeDef expected_node;
+  string error_msg;
   // Remove input from node with 1 input 0 controls.
   expected_node = NDef("", "", {});
-  TestRemoveRegularFanin("foo_1", {"a", 0}, /*modified=*/true, &expected_node);
+  TestRemoveRegularFanin("foo_1", {"a", 0}, /*success=*/true, error_msg,
+                         &expected_node);
   // Remove input from node with multiple inputs and 0 controls.
   expected_node = NDef("", "", {"b"});
-  TestRemoveRegularFanin("foo_3", {"a", 1}, /*modified=*/true, &expected_node);
+  TestRemoveRegularFanin("foo_3", {"a", 1}, /*success=*/true, error_msg,
+                         &expected_node);
   // Remove input from node with 1 input multiple controls.
   expected_node = NDef("", "", {"^a", "^c"});
-  TestRemoveRegularFanin("foo_2", {"b", 0}, /*modified=*/true, &expected_node);
+  TestRemoveRegularFanin("foo_2", {"b", 0}, /*success=*/true, error_msg,
+                         &expected_node);
   // Remove input from node with multiple inputs and controls.
   expected_node = NDef("", "", {"a", "^c", "^d"});
-  TestRemoveRegularFanin("foo_4", {"b", 2}, /*modified=*/true, &expected_node);
+  TestRemoveRegularFanin("foo_4", {"b", 2}, /*success=*/true, error_msg,
+                         &expected_node);
   // Remove input from node with multiple inputs and controls, and results in
   // shifting of ports.
   expected_node = NDef("", "", {"b:2", "b:2", "^c", "^d"});
-  TestRemoveRegularFanin("foo_4", {"a", 0}, /*modified=*/true, &expected_node);
+  TestRemoveRegularFanin("foo_4", {"a", 0}, /*success=*/true, error_msg,
+                         &expected_node);
 
   // Remove control from node with 1 input multiple controls.
   expected_node = NDef("", "", {"b", "^a", "^c"});
+  error_msg =
+      "Can't remove invalid fanin '^a' as regular fanin from node 'foo_2'.";
   TestRemoveRegularFanin("foo_2", {"a", Graph::kControlSlot},
-                         /*modified=*/false, &expected_node);
+                         /*success=*/false, error_msg, &expected_node);
   // Remove control from node with multiple input multiple controls.
   expected_node = NDef("", "", {"a", "b:2", "b:2", "^c", "^d"});
+  error_msg =
+      "Can't remove invalid fanin '^d' as regular fanin from node 'foo_4'.";
   TestRemoveRegularFanin("foo_4", {"d", Graph::kControlSlot},
-                         /*modified=*/false, &expected_node);
+                         /*success=*/false, error_msg, &expected_node);
   // Remove control from node with 0 inputs multiple controls.
   expected_node = NDef("", "", {"^a", "^b"});
+  error_msg =
+      "Can't remove invalid fanin '^a' as regular fanin from node 'foo_6'.";
   TestRemoveRegularFanin("foo_6", {"a", Graph::kControlSlot},
-                         /*modified=*/false, &expected_node);
+                         /*success=*/false, error_msg, &expected_node);
 
   // Remove input from node with 0 inputs 0 controls.
   expected_node = NDef("", "", {});
-  TestRemoveRegularFanin("foo_5", {"a", 1}, /*modified=*/false, &expected_node);
+  error_msg = "";
+  TestRemoveRegularFanin("foo_5", {"a", 1}, /*success=*/true, error_msg,
+                         &expected_node);
   // Remove input from node with 0 inputs multiple controls.
   expected_node = NDef("", "", {"^a", "^b"});
-  TestRemoveRegularFanin("foo_6", {"a", 1}, /*modified=*/false, &expected_node);
+  TestRemoveRegularFanin("foo_6", {"a", 1}, /*success=*/true, error_msg,
+                         &expected_node);
 
   // Remove control from node with 1 input 0 controls.
   expected_node = NDef("", "", {"a"});
+  error_msg =
+      "Can't remove invalid fanin '^b' as regular fanin from node 'foo_1'.";
   TestRemoveRegularFanin("foo_1", {"b", Graph::kControlSlot},
-                         /*modified=*/false, &expected_node);
+                         /*success=*/false, error_msg, &expected_node);
   // Remove control from node with multiple inputs and 0 controls.
   expected_node = NDef("", "", {"b", "a:1", "a:1"});
+  error_msg =
+      "Can't remove invalid fanin '^c' as regular fanin from node 'foo_3'.";
   TestRemoveRegularFanin("foo_3", {"c", Graph::kControlSlot},
-                         /*modified=*/false, &expected_node);
+                         /*success=*/false, error_msg, &expected_node);
   // Remove control from node with 0 inputs 0 controls.
   expected_node = NDef("", "", {});
+  error_msg =
+      "Can't remove invalid fanin '^a' as regular fanin from node 'foo_5'.";
   TestRemoveRegularFanin("foo_5", {"a", Graph::kControlSlot},
-                         /*modified=*/false, &expected_node);
+                         /*success=*/false, error_msg, &expected_node);
 
   // Remove fanin from node where node is missing.
-  TestRemoveRegularFanin("foo_missing", {"a", 0}, /*modified=*/false,
+  error_msg =
+      "Can't remove fanin 'a:0' as regular fanin from missing node "
+      "'foo_missing'.";
+  TestRemoveRegularFanin("foo_missing", {"a", 0}, /*success=*/false, error_msg,
                          /*expected_node=*/nullptr);
   // Remove fanin from node where fanin is missing.
   expected_node = NDef("", "", {"a"});
-  TestRemoveRegularFanin("foo_1", {"bar_missing", 0}, /*modified=*/false,
-                         &expected_node);
+  error_msg =
+      "Can't remove missing fanin 'bar_missing:0' as regular fanin from node "
+      "'foo_1'.";
+  TestRemoveRegularFanin("foo_1", {"bar_missing", 0}, /*success=*/false,
+                         error_msg, &expected_node);
   // Remove fanin from node where node and fanin are missing.
-  TestRemoveRegularFanin("foo_missing", {"bar_missing", 0}, /*modified=*/false,
+  error_msg =
+      "Can't remove missing fanin 'bar_missing:0' as regular fanin from "
+      "missing node 'foo_missing'.";
+  TestRemoveRegularFanin("foo_missing", {"bar_missing", 0}, /*success=*/false,
+                         error_msg,
+                         /*expected_node=*/nullptr);
+  // Remove control from node where node and fanin are missing.
+  error_msg =
+      "Can't remove invalid/missing fanin '^bar_missing' as regular fanin from "
+      "missing node 'foo_missing'.";
+  TestRemoveRegularFanin("foo_missing", {"bar_missing", Graph::kControlSlot},
+                         /*success=*/false, error_msg,
                          /*expected_node=*/nullptr);
 }
 
 void TestRemoveAllFanins(absl::string_view node_name,
-                         bool keep_controlling_nodes, bool modified,
+                         bool keep_controlling_nodes, bool success,
+                         const string& error_msg,
                          const NodeDef* expected_node) {
   GraphDef graph_def = SimpleMutateFaninGraph();
 
@@ -470,10 +559,14 @@ void TestRemoveAllFanins(absl::string_view node_name,
     fanin_strings.insert(node->input().begin(), node->input().end());
   }
 
-  EXPECT_EQ(modified, graph.RemoveAllFanins(node_name, keep_controlling_nodes));
+  Status s = graph.RemoveAllFanins(node_name, keep_controlling_nodes);
+  EXPECT_EQ(s.ok(), success);
+  if (!success) {
+    EXPECT_EQ(s.error_message(), error_msg);
+  }
   if (expected_node != nullptr) {
     CompareNodeInputs(graph, expected_node, node);
-    if (modified) {
+    if (success) {
       TensorId tensor_id;
       auto retained_inputs = absl::flat_hash_set<string>(node->input().begin(),
                                                          node->input().end());
@@ -489,48 +582,50 @@ void TestRemoveAllFanins(absl::string_view node_name,
 
 TEST(MutableGraphViewTest, RemoveAllFanins) {
   NodeDef expected_node;
+  string error_msg;
   // Remove all fanins from node with no control dependencies.
   expected_node = NDef("", "", {});
   TestRemoveAllFanins("foo_3", /*keep_controlling_nodes=*/false,
-                      /*modified=*/true, &expected_node);
+                      /*success=*/true, error_msg, &expected_node);
   // Remove all fanins from node with control dependencies.
   TestRemoveAllFanins("foo_4", /*keep_controlling_nodes=*/false,
-                      /*modified=*/true, &expected_node);
+                      /*success=*/true, error_msg, &expected_node);
 
   // Remove all fanins from node with no control dependencies and preserve
   // control dependencies.
   TestRemoveAllFanins("foo_3", /*keep_controlling_nodes=*/true,
-                      /*modified=*/true, &expected_node);
+                      /*success=*/true, error_msg, &expected_node);
   // Remove all fanins from node with control dependencies and preserve control
   // dependencies.
   expected_node = NDef("", "", {"^c", "^d"});
   TestRemoveAllFanins("foo_4", /*keep_controlling_nodes=*/true,
-                      /*modified=*/true, &expected_node);
+                      /*success=*/true, error_msg, &expected_node);
 
   // Remove all fanins from node with no fanins.
   expected_node = NDef("", "", {});
   TestRemoveAllFanins("foo_5", /*keep_controlling_nodes=*/false,
-                      /*modified=*/false, &expected_node);
+                      /*success=*/true, error_msg, &expected_node);
   TestRemoveAllFanins("foo_5", /*keep_controlling_nodes=*/true,
-                      /*modified=*/false, &expected_node);
+                      /*success=*/true, error_msg, &expected_node);
 
   // Remove all fanins from node with only control dependencies.
   TestRemoveAllFanins("foo_6", /*keep_controlling_nodes=*/false,
-                      /*modified=*/true, &expected_node);
+                      /*success=*/true, error_msg, &expected_node);
   expected_node = NDef("", "", {"^a", "^b"});
   TestRemoveAllFanins("foo_6", /*keep_controlling_nodes=*/true,
-                      /*modified=*/false, &expected_node);
+                      /*success=*/true, error_msg, &expected_node);
 
   // Remove all fanins from node where node is missing.
+  error_msg = "Can't remove all fanins from missing node 'foo_missing'.";
   TestRemoveAllFanins("foo_missing", /*keep_controlling_nodes=*/false,
-                      /*modified=*/false, /*expected_node=*/nullptr);
+                      /*success=*/false, error_msg, /*expected_node=*/nullptr);
   TestRemoveAllFanins("foo_missing", /*keep_controlling_nodes=*/true,
-                      /*modified=*/false, /*expected_node=*/nullptr);
+                      /*success=*/false, error_msg, /*expected_node=*/nullptr);
 }
 
 void TestUpdateFanin(absl::string_view node_name, const TensorId& from_fanin,
-                     const TensorId& to_fanin, bool modified,
-                     const NodeDef* expected_node) {
+                     const TensorId& to_fanin, bool success,
+                     const string& error_msg, const NodeDef* expected_node) {
   GraphDef graph_def = SimpleMutateFaninGraph();
 
   MutableGraphView graph(&graph_def);
@@ -542,10 +637,14 @@ void TestUpdateFanin(absl::string_view node_name, const TensorId& from_fanin,
     EXPECT_NE(node, nullptr);
   }
 
-  EXPECT_EQ(modified, graph.UpdateFanin(node_name, from_fanin, to_fanin));
+  Status s = graph.UpdateFanin(node_name, from_fanin, to_fanin);
+  EXPECT_EQ(s.ok(), success);
+  if (!success) {
+    EXPECT_EQ(s.error_message(), error_msg);
+  }
   if (expected_node != nullptr) {
     CompareNodeInputs(graph, expected_node, node);
-    if (modified) {
+    if (success) {
       CheckFanout(graph, from_fanin, node_name);
     }
   }
@@ -553,49 +652,82 @@ void TestUpdateFanin(absl::string_view node_name, const TensorId& from_fanin,
 
 TEST(MutableGraphViewTest, UpdateFanin) {
   NodeDef expected_node;
+  string error_msg;
   // Update fanin from non control to non control.
   expected_node = NDef("", "", {"a", "b:3", "b:3", "^c", "^d"});
-  TestUpdateFanin("foo_4", {"b", 2}, {"b", 3}, /*modified=*/true,
+  TestUpdateFanin("foo_4", {"b", 2}, {"b", 3}, /*success=*/true, error_msg,
                   &expected_node);
   // Update fanin from non control to control.
   expected_node = NDef("", "", {"a", "^c", "^d", "^b"});
   TestUpdateFanin("foo_4", {"b", 2}, {"b", Graph::kControlSlot},
-                  /*modified=*/true, &expected_node);
+                  /*success=*/true, error_msg, &expected_node);
   // Update fanin from control to non control.
   expected_node = NDef("", "", {"a", "b:2", "b:2", "d:1", "^c"});
   TestUpdateFanin("foo_4", {"d", Graph::kControlSlot}, {"d", 1},
-                  /*modified=*/true, &expected_node);
+                  /*success=*/true, error_msg, &expected_node);
   // Update fanin from control to control.
   expected_node = NDef("", "", {"a", "b:2", "b:2", "^d"});
   TestUpdateFanin("foo_4", {"c", Graph::kControlSlot},
-                  {"b", Graph::kControlSlot}, /*modified=*/true,
+                  {"b", Graph::kControlSlot}, /*success=*/true, error_msg,
                   &expected_node);
   // Update fanin from control to existing control.
   expected_node = NDef("", "", {"a", "b:2", "b:2", "^d"});
   TestUpdateFanin("foo_4", {"c", Graph::kControlSlot},
-                  {"d", Graph::kControlSlot}, /*modified=*/true,
+                  {"d", Graph::kControlSlot}, /*success=*/true, error_msg,
                   &expected_node);
 
   // Update fanin of node where from and to fanins are the same.
   expected_node = NDef("", "", {"a"});
-  TestUpdateFanin("foo_1", {"a", -1}, {"a", -1}, /*modified=*/false,
+  TestUpdateFanin("foo_1", {"a", -1}, {"a", -1}, /*success=*/true, error_msg,
                   &expected_node);
-  TestUpdateFanin("foo_1", {"a", 0}, {"a", 0}, /*modified=*/false,
+  TestUpdateFanin("foo_1", {"a", 0}, {"a", 0}, /*success=*/true, error_msg,
                   &expected_node);
-  TestUpdateFanin("foo_1", {"a", 1}, {"a", 1}, /*modified=*/false,
+  TestUpdateFanin("foo_1", {"a", 1}, {"a", 1}, /*success=*/true, error_msg,
                   &expected_node);
+
   // Update fanin of node where node is missing.
-  TestUpdateFanin("foo_missing", {"a", 0}, {"a", 1}, /*modified=*/false,
+  error_msg =
+      "Can't update fanin 'a:0' to fanin 'a:1' in missing node 'foo_missing'.";
+  TestUpdateFanin("foo_missing", {"a", 0}, {"a", 1}, /*success=*/false,
+                  error_msg,
                   /*expected_node=*/nullptr);
   // Update fanin of node where from fanin is missing.
+  error_msg =
+      "Can't update missing fanin 'from_bar_missing:0' to fanin 'a:1' in node "
+      "'foo_1'.";
   TestUpdateFanin("foo_1", {"from_bar_missing", 0}, {"a", 1},
-                  /*modified=*/false, &expected_node);
+                  /*success=*/false, error_msg, &expected_node);
   // Update fanin of node where to fanin is missing.
-  TestUpdateFanin("foo_1", {"a", 0}, {"to_bar_missing", 1}, /*modified=*/false,
-                  &expected_node);
+  error_msg =
+      "Can't update fanin 'a:0' to missing fanin 'to_bar_missing:1' in node "
+      "'foo_1'.";
+  TestUpdateFanin("foo_1", {"a", 0}, {"to_bar_missing", 1}, /*success=*/false,
+                  error_msg, &expected_node);
   // Update fanin of node where from/to fanins and node are missing.
+  error_msg =
+      "Can't update missing fanin 'from_bar_missing:0' to missing fanin "
+      "'to_bar_missing:1' in missing node 'foo_missing'.";
   TestUpdateFanin("foo_missing", {"from_bar_missing", 0}, {"to_bar_missing", 1},
-                  /*modified=*/false, /*expected_node=*/nullptr);
+                  /*success=*/false, error_msg, /*expected_node=*/nullptr);
+  // Update fanin of node where from fanin is invalid.
+  error_msg =
+      "Can't update invalid fanin 'a:-2' to fanin 'a:0' in node 'foo_1'.";
+  TestUpdateFanin("foo_1", {"a", -2}, {"a", 0},
+                  /*success=*/false, error_msg, &expected_node);
+  // Update fanin of node where to fanin is invalid.
+  error_msg =
+      "Can't update fanin 'a:0' to invalid fanin 'a:-2' in node 'foo_1'.";
+  TestUpdateFanin("foo_1", {"a", 0}, {"a", -2},
+                  /*success=*/false, error_msg, &expected_node);
+  // Update fanin of node where from/to fanins are invalid and missing and node
+  // is missing.
+  error_msg =
+      "Can't update invalid/missing fanin 'from_bar_missing:-2' to "
+      "invalid/missing fanin 'to_bar_missing:-3' in missing node "
+      "'foo_missing'.";
+  TestUpdateFanin("foo_missing", {"from_bar_missing", -2},
+                  {"to_bar_missing", -3},
+                  /*success=*/false, error_msg, /*expected_node=*/nullptr);
 }
 
 void TestUpdateFaninFromFaninToNodeAsSwitchControl(const TensorId& fanin) {
@@ -607,7 +739,13 @@ void TestUpdateFaninFromFaninToNodeAsSwitchControl(const TensorId& fanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_FALSE(graph.UpdateFanin("c", fanin, {"b", Graph::kControlSlot}));
+  Status s = graph.UpdateFanin("c", fanin, {"b", Graph::kControlSlot});
+  EXPECT_FALSE(s.ok());
+  string expected_msg = absl::Substitute(
+      "Can't update fanin '$0' to fanin '^b' in node 'c', to fanin is a Switch "
+      "control dependency.",
+      fanin.ToString());
+  EXPECT_EQ(s.error_message(), expected_msg);
 
   EXPECT_EQ(graph.graph()->node_size(), 3);
 
@@ -714,14 +852,14 @@ TEST(MutableGraphViewTest, DedupControllingFaninsOnAddFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddRegularFanin("b", {"a", 2}));
+  EXPECT_TRUE(graph.AddRegularFanin("b", {"a", 2}).ok());
   NodeDef expected;
   NodeDef* b = graph.GetNode("b");
   ASSERT_NE(b, nullptr);
   expected = NDef("", "", {"a:2"});
   CompareNodeInputs(graph, &expected, b);
 
-  EXPECT_FALSE(graph.AddControllingFanin("c", {"a", Graph::kControlSlot}));
+  EXPECT_TRUE(graph.AddControllingFanin("c", {"a", Graph::kControlSlot}).ok());
   NodeDef* c = graph.GetNode("c");
   ASSERT_NE(c, nullptr);
   expected = NDef("", "", {"a:1"});
@@ -736,28 +874,28 @@ TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnAddFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddRegularFanin("c", {"b", 2}));
+  EXPECT_TRUE(graph.AddRegularFanin("c", {"b", 2}).ok());
   NodeDef expected;
   NodeDef* c = graph.GetNode("c");
   ASSERT_NE(c, nullptr);
   expected = NDef("", "", {"b:2"});
   CompareNodeInputs(graph, &expected, c);
-  EXPECT_TRUE(graph.AddControllingFanin("c", {"b", Graph::kControlSlot}));
+  EXPECT_TRUE(graph.AddControllingFanin("c", {"b", Graph::kControlSlot}).ok());
   expected = NDef("", "", {"b:2", "^b"});
   CompareNodeInputs(graph, &expected, c);
-  EXPECT_FALSE(graph.AddControllingFanin("c", {"b", Graph::kControlSlot}));
+  EXPECT_TRUE(graph.AddControllingFanin("c", {"b", Graph::kControlSlot}).ok());
   expected = NDef("", "", {"b:2", "^b"});
   CompareNodeInputs(graph, &expected, c);
 
-  EXPECT_TRUE(graph.AddControllingFanin("d", {"b", Graph::kControlSlot}));
+  EXPECT_TRUE(graph.AddControllingFanin("d", {"b", Graph::kControlSlot}).ok());
   NodeDef* d = graph.GetNode("d");
   ASSERT_NE(d, nullptr);
   expected = NDef("", "", {"^b"});
   CompareNodeInputs(graph, &expected, d);
-  EXPECT_FALSE(graph.AddControllingFanin("d", {"b", Graph::kControlSlot}));
+  EXPECT_TRUE(graph.AddControllingFanin("d", {"b", Graph::kControlSlot}).ok());
   expected = NDef("", "", {"^b"});
   CompareNodeInputs(graph, &expected, d);
-  EXPECT_TRUE(graph.AddRegularFanin("d", {"b", 3}));
+  EXPECT_TRUE(graph.AddRegularFanin("d", {"b", 3}).ok());
   expected = NDef("", "", {"b:3", "^b"});
   CompareNodeInputs(graph, &expected, d);
 }
@@ -771,7 +909,7 @@ TEST(MutableGraphViewTest, DedupControllingFaninsOnUpdateFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.UpdateFanin("c", {"a", 1}, {"b", 2}));
+  EXPECT_TRUE(graph.UpdateFanin("c", {"a", 1}, {"b", 2}).ok());
   NodeDef* c = graph.GetNode("c");
   ASSERT_NE(c, nullptr);
   NodeDef expected = NDef("", "", {"b:2"});
@@ -787,21 +925,24 @@ TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnUpdateFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.UpdateFanin("d", {"b", Graph::kControlSlot},
-                                {"c", Graph::kControlSlot}));
+  EXPECT_TRUE(graph
+                  .UpdateFanin("d", {"b", Graph::kControlSlot},
+                               {"c", Graph::kControlSlot})
+                  .ok());
   NodeDef expected;
   NodeDef* d = graph.GetNode("d");
   ASSERT_NE(d, nullptr);
   expected = NDef("", "", {"c", "^c"});
   CompareNodeInputs(graph, &expected, d);
 
-  EXPECT_TRUE(graph.UpdateFanin("e", {"b", 0}, {"c", 3}));
+  EXPECT_TRUE(graph.UpdateFanin("e", {"b", 0}, {"c", 3}).ok());
   NodeDef* e = graph.GetNode("e");
   ASSERT_NE(e, nullptr);
   expected = NDef("", "", {"c:3", "^c"});
   CompareNodeInputs(graph, &expected, e);
 
-  EXPECT_TRUE(graph.UpdateFanin("e", {"c", 3}, {"c", Graph::kControlSlot}));
+  EXPECT_TRUE(
+      graph.UpdateFanin("e", {"c", 3}, {"c", Graph::kControlSlot}).ok());
   ASSERT_NE(e, nullptr);
   expected = NDef("", "", {"^c"});
   CompareNodeInputs(graph, &expected, e);
@@ -816,7 +957,7 @@ TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnAddFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddRegularFanin("c", {"a", 3}));
+  EXPECT_TRUE(graph.AddRegularFanin("c", {"a", 3}).ok());
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
 
@@ -841,7 +982,7 @@ TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnRemoveFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.RemoveRegularFanin("c", {"a", 2}));
+  EXPECT_TRUE(graph.RemoveRegularFanin("c", {"a", 2}).ok());
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
 
@@ -862,7 +1003,7 @@ TEST(MutableGraphViewTest, KeepMaxRegularOutputPortOnRemoveFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.RemoveRegularFanin("b", {"a", 1}));
+  EXPECT_TRUE(graph.RemoveRegularFanin("b", {"a", 1}).ok());
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
 
@@ -883,7 +1024,7 @@ TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnUpdateFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.UpdateFanin("c", {"a", 2}, {"b", 3}));
+  EXPECT_TRUE(graph.UpdateFanin("c", {"a", 2}, {"b", 3}).ok());
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
 
@@ -910,11 +1051,21 @@ TEST(MutableGraphViewTest, AddControllingFaninMissing) {
 
   MutableGraphView graph(&graph_def);
   // Missing fanin.
-  EXPECT_FALSE(graph.AddControllingFanin("a", {"c", Graph::kControlSlot}));
+  Status s = graph.AddControllingFanin("a", {"c", Graph::kControlSlot});
+  EXPECT_FALSE(s.ok());
+  string expected_msg = "Can't add missing controlling fanin '^c' to node 'a'.";
+  EXPECT_EQ(s.error_message(), expected_msg);
   // Missing node.
-  EXPECT_FALSE(graph.AddControllingFanin("d", {"a", Graph::kControlSlot}));
+  s = graph.AddControllingFanin("d", {"a", Graph::kControlSlot});
+  EXPECT_FALSE(s.ok());
+  expected_msg = "Can't add controlling fanin '^a' to missing node 'd'.";
+  EXPECT_EQ(s.error_message(), expected_msg);
   // Missing node and fanin.
-  EXPECT_FALSE(graph.AddControllingFanin("c", {"d", Graph::kControlSlot}));
+  s = graph.AddControllingFanin("c", {"d", Graph::kControlSlot});
+  EXPECT_FALSE(s.ok());
+  expected_msg =
+      "Can't add missing controlling fanin '^d' to missing node 'c'.";
+  EXPECT_EQ(s.error_message(), expected_msg);
 
   ASSERT_EQ(graph.graph()->node_size(), 2);
   NodeDef expected;
@@ -935,8 +1086,8 @@ TEST(MutableGraphViewTest, AddControllingFaninExistingControl) {
       /*funcs=*/{});
 
   MutableGraphView graph(&graph_def);
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"b", Graph::kControlSlot}));
-  EXPECT_FALSE(graph.AddControllingFanin("a", {"b", Graph::kControlSlot}));
+  EXPECT_TRUE(graph.AddControllingFanin("a", {"b", Graph::kControlSlot}).ok());
+  EXPECT_TRUE(graph.AddControllingFanin("a", {"b", Graph::kControlSlot}).ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 2);
   NodeDef expected;
@@ -958,8 +1109,8 @@ TEST(MutableGraphViewTest, AddControllingFaninNotSwitch) {
       /*funcs=*/{});
 
   MutableGraphView graph(&graph_def);
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"b", 2}));
-  EXPECT_FALSE(graph.AddControllingFanin("a", {"b", 2}));
+  EXPECT_TRUE(graph.AddControllingFanin("a", {"b", 2}).ok());
+  EXPECT_TRUE(graph.AddControllingFanin("a", {"b", 2}).ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 2);
   NodeDef expected;
@@ -982,8 +1133,8 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithIdentity) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}));
-  EXPECT_FALSE(graph.AddControllingFanin("a", {"switch", 0}));
+  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
+  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
   NodeDef* a = graph.GetNode("a");
@@ -1001,8 +1152,8 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithNoExistingIdentity) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}));
-  EXPECT_FALSE(graph.AddControllingFanin("a", {"switch", 0}));
+  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
+  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
   NodeDef expected;
@@ -1029,8 +1180,8 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithExistingAddedIdentity) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}));
-  EXPECT_FALSE(graph.AddControllingFanin("a", {"switch", 0}));
+  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
+  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
   NodeDef* a = graph.GetNode("a");
@@ -1049,7 +1200,7 @@ TEST(MutableGraphViewTest, RemoveControllingFaninMissing) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_FALSE(graph.RemoveControllingFanin("d", "c"));
+  EXPECT_TRUE(graph.RemoveControllingFanin("d", "c").ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 4);
   NodeDef* d = graph.GetNode("d");
@@ -1068,8 +1219,8 @@ TEST(MutableGraphViewTest, RemoveControllingFaninExisting) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.RemoveControllingFanin("d", "a"));
-  EXPECT_FALSE(graph.RemoveControllingFanin("d", "a"));
+  EXPECT_TRUE(graph.RemoveControllingFanin("d", "a").ok());
+  EXPECT_TRUE(graph.RemoveControllingFanin("d", "a").ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 4);
   NodeDef* d = graph.GetNode("d");
@@ -1087,8 +1238,8 @@ TEST(MutableGraphViewTest, RemoveControllingFaninOnRegularFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_FALSE(graph.RemoveControllingFanin("c", "a"));
-  EXPECT_FALSE(graph.RemoveControllingFanin("c", "b"));
+  EXPECT_TRUE(graph.RemoveControllingFanin("c", "a").ok());
+  EXPECT_TRUE(graph.RemoveControllingFanin("c", "b").ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
   NodeDef* c = graph.GetNode("c");
diff --git a/tensorflow/core/grappler/optimizers/data/filter_fusion.cc b/tensorflow/core/grappler/optimizers/data/filter_fusion.cc
index 89b568ecf1..e3f0a7e6c5 100644
--- a/tensorflow/core/grappler/optimizers/data/filter_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/filter_fusion.cc
@@ -109,7 +109,8 @@ Status FilterFusion::Optimize(Cluster* cluster, const GrapplerItem& item,
     const auto* fused_filter_node = graph.AddNode(MakeFusedFilterNode(
         *first_filter_node, *second_filter_node, *fused_predicate, &graph));
 
-    graph.UpdateFanouts(second_filter_node->name(), fused_filter_node->name());
+    TF_RETURN_IF_ERROR(graph.UpdateFanouts(second_filter_node->name(),
+                                           fused_filter_node->name()));
 
     // TODO(prazek): we should run some optimizations on the fused filter
     // functions, or make sure that optimization passes run after filter
diff --git a/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc b/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc
index 60755256d8..6aa450e123 100644
--- a/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc
+++ b/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc
@@ -266,7 +266,8 @@ Status HoistRandomUniform::Optimize(Cluster* cluster, const GrapplerItem& item,
     const auto* stateless_map = graph.AddNode(
         MakeStatelessMap(*map_node, *zip_node, *stateless_func, &graph));
 
-    graph.UpdateFanouts(map_node->name(), stateless_map->name());
+    TF_RETURN_IF_ERROR(
+        graph.UpdateFanouts(map_node->name(), stateless_map->name()));
 
     // TODO(b/116285210): we could also remove map functions from library if
     // they are not used anymore.
diff --git a/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc b/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc
index 52b4b785a3..ff0fdbc131 100644
--- a/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc
+++ b/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc
@@ -97,7 +97,7 @@ Status LatencyAllEdges::Optimize(Cluster* cluster, const GrapplerItem& item,
     }
 
     NodeDef* latency_node = graph.AddNode(MakeLatencyNode(node, &graph));
-    graph.UpdateFanouts(node.name(), latency_node->name());
+    TF_RETURN_IF_ERROR(graph.UpdateFanouts(node.name(), latency_node->name()));
   }
   return Status::OK();
 }
diff --git a/tensorflow/core/grappler/optimizers/data/make_numa_aware.cc b/tensorflow/core/grappler/optimizers/data/make_numa_aware.cc
index 72c27a1d4a..2940fbecd4 100644
--- a/tensorflow/core/grappler/optimizers/data/make_numa_aware.cc
+++ b/tensorflow/core/grappler/optimizers/data/make_numa_aware.cc
@@ -47,7 +47,7 @@ Status MakeNumaAware::Optimize(Cluster* cluster, const GrapplerItem& item,
     if (node.op() != "ExperimentalMapAndBatchDataset") continue;
 
     auto* numa_node = graph.AddNode(MakeNumaAwareNode(node, &graph));
-    graph.UpdateFanouts(node.name(), numa_node->name());
+    TF_RETURN_IF_ERROR(graph.UpdateFanouts(node.name(), numa_node->name()));
     nodes_to_delete.insert(node.name());
   }
   graph.DeleteNodes(nodes_to_delete);
diff --git a/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc b/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc
index 84c4d82f6a..4ff749c57b 100644
--- a/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc
@@ -120,7 +120,8 @@ Status MapAndBatchFusion::Optimize(Cluster* cluster, const GrapplerItem& item,
 
     auto* new_node =
         graph.AddNode(MakeMapAndBatchNode(*map_node, batch_node, &graph));
-    graph.UpdateFanouts(batch_node.name(), new_node->name());
+    TF_RETURN_IF_ERROR(
+        graph.UpdateFanouts(batch_node.name(), new_node->name()));
 
     // Mark the `Map` and `Batch` nodes for removal.
     nodes_to_delete.insert(map_node->name());
diff --git a/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.cc b/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.cc
index 233d7968c8..128a2fefa2 100644
--- a/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.cc
@@ -155,7 +155,8 @@ Status MapAndFilterFusion::Optimize(Cluster* cluster, const GrapplerItem& item,
     const auto* filter_by_component = graph.AddNode(
         MakeFilterByLastComponentNode(*fused_maps, *filter_node, &graph));
 
-    graph.UpdateFanouts(filter_node->name(), filter_by_component->name());
+    TF_RETURN_IF_ERROR(
+        graph.UpdateFanouts(filter_node->name(), filter_by_component->name()));
     TF_RETURN_IF_ERROR(function_library.AddFunctionDef(*fused_function));
 
     // TODO(prazek): we could also remove functions from library if they are not
diff --git a/tensorflow/core/grappler/optimizers/data/map_fusion.cc b/tensorflow/core/grappler/optimizers/data/map_fusion.cc
index 6b8015f96a..504fcc1f5e 100644
--- a/tensorflow/core/grappler/optimizers/data/map_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_fusion.cc
@@ -130,7 +130,8 @@ Status MapFusion::Optimize(Cluster* cluster, const GrapplerItem& item,
     const auto* fused_maps_node = graph.AddNode(
         MakeFusedNode(*parent_map_node, *map_node, *fused_function, &graph));
 
-    graph.UpdateFanouts(map_node->name(), fused_maps_node->name());
+    TF_RETURN_IF_ERROR(
+        graph.UpdateFanouts(map_node->name(), fused_maps_node->name()));
 
     // TODO(prazek): we should run some optimizations on the fused map
     // functions, or make sure that optimization passes run after map
diff --git a/tensorflow/core/grappler/optimizers/data/map_parallelization.cc b/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
index d335117585..1274af7160 100644
--- a/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
@@ -89,7 +89,8 @@ Status MapParallelization::Optimize(Cluster* cluster, const GrapplerItem& item,
 
     auto* parallel_map =
         graph.AddNode(MakeParallelMap(map_node->name(), &graph));
-    graph.UpdateFanouts(map_node->name(), parallel_map->name());
+    TF_RETURN_IF_ERROR(
+        graph.UpdateFanouts(map_node->name(), parallel_map->name()));
     nodes_to_delete.insert(map_node->name());
   }
 
diff --git a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
index 3401dcc6f2..bc1d6d0ea6 100644
--- a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
@@ -264,7 +264,8 @@ Status MapVectorization::Optimize(Cluster* cluster, const GrapplerItem& item,
 
     auto* new_map_node = graph.AddNode(MakeNewMapNode(
         *map_node, batch_node, *new_batch_node, *vectorized_func, &graph));
-    graph.UpdateFanouts(batch_node.name(), new_map_node->name());
+    TF_RETURN_IF_ERROR(
+        graph.UpdateFanouts(batch_node.name(), new_map_node->name()));
 
     // Mark the `Map` and `Batch` nodes for removal.
     nodes_to_delete.insert(map_node->name());
diff --git a/tensorflow/core/grappler/optimizers/data/noop_elimination.cc b/tensorflow/core/grappler/optimizers/data/noop_elimination.cc
index bd405c8329..b71e9e5420 100644
--- a/tensorflow/core/grappler/optimizers/data/noop_elimination.cc
+++ b/tensorflow/core/grappler/optimizers/data/noop_elimination.cc
@@ -79,7 +79,7 @@ Status NoOpElimination::Optimize(Cluster* cluster, const GrapplerItem& item,
     if (!IsNoOp(node, graph)) continue;
 
     NodeDef* const parent = graph_utils::GetInputNode(node, graph);
-    graph.UpdateFanouts(node.name(), parent->name());
+    TF_RETURN_IF_ERROR(graph.UpdateFanouts(node.name(), parent->name()));
 
     nodes_to_delete.insert(node.name());
   }
diff --git a/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.cc b/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.cc
index d9af78d38c..f03648e7e9 100644
--- a/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.cc
@@ -86,7 +86,8 @@ Status ShuffleAndRepeatFusion::Optimize(Cluster* cluster,
 
     NodeDef* shuffle_and_repeat_node =
         graph.AddNode(make_shuffle_and_repeat_node(shuffle_node, repeat_node));
-    graph.UpdateFanouts(repeat_node.name(), shuffle_and_repeat_node->name());
+    TF_RETURN_IF_ERROR(graph.UpdateFanouts(repeat_node.name(),
+                                           shuffle_and_repeat_node->name()));
 
     // Mark the `Shuffle` and `Repeat` nodes for removal.
     nodes_to_delete.insert(shuffle_node.name());
-- 
GitLab


From 01c19850c0fd6b587f44ab5938e2d874744a2cbf Mon Sep 17 00:00:00 2001
From: Dimitris Vardoulakis <dimvar@google.com>
Date: Mon, 14 Jan 2019 15:21:18 -0800
Subject: [PATCH 0681/2345] [TF:XLA] Fix crash in AR/CRS combiner, when a
 cross-module AllReduce is immediately followed by a cross-replica AllReduce.

PiperOrigin-RevId: 229265445
---
 .../compiler/xla/service/ar_crs_combiner.cc   |  7 +-
 .../xla/service/ar_crs_combiner_test.cc       | 67 +++++++++++++++++++
 2 files changed, 72 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/ar_crs_combiner.cc b/tensorflow/compiler/xla/service/ar_crs_combiner.cc
index 35a32f537b..8e327ee84b 100644
--- a/tensorflow/compiler/xla/service/ar_crs_combiner.cc
+++ b/tensorflow/compiler/xla/service/ar_crs_combiner.cc
@@ -249,14 +249,17 @@ void ArCrsCombiner::KeepProvablyEqualInstructionGroups() {
       auto next_0 = instr_0->users()[0];
       auto next_i = instr_i->users()[0];
       absl::flat_hash_map<int64, int64> visited_pairs;
-      do {
+      while (true) {
         if (!InstructionsComputeSameValue(next_0, next_i, &visited_pairs)) {
           all_reduce_map_.erase(all_reduce_id);
           break;
         }
+        if (next_0->IsCrossReplicaAllReduce()) {
+          break;
+        }
         next_0 = next_0->users()[0];
         next_i = next_i->users()[0];
-      } while (!next_0->IsCrossReplicaAllReduce());
+      }
     }
   }
 }
diff --git a/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc b/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc
index b12b63b2dd..08eb77f032 100644
--- a/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc
+++ b/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc
@@ -706,5 +706,72 @@ ENTRY %entrycomp (p: f32[]) -> (f32[], f32[]) {
   EXPECT_FALSE(changed);
 }
 
+TEST_F(ArCrsCombinerTest, ArThenCrsDontCrash) {
+  const char* module_str = R"(
+HloModule foobar
+
+%sum.1 (a: f32[], b: f32[]) -> f32[] {
+  %a = f32[] parameter(0)
+  %b = f32[] parameter(1)
+  ROOT %add = f32[] add(%a, %b)
+}
+
+ENTRY %entrycomp (p: f32[]) -> (f32[], f32[]) {
+  %p = f32[] parameter(0)
+  %constant.f32 = f32[] constant(123)
+
+  %all-reduce.ar.1 = f32[]
+      all-reduce(%p),
+      replica_groups={{0},{1}},
+      all_reduce_id=1,
+      to_apply=%sum.1,
+      sharding={maximal device=0}
+  %all-reduce.1 = f32[]
+      all-reduce(%all-reduce.ar.1),
+      replica_groups={{0,1}},
+      to_apply=%sum.1,
+      sharding={maximal device=0}
+  %multiply.1 = f32[]
+      multiply(%all-reduce.1, %constant.f32),
+      sharding={maximal device=0}
+
+  %all-reduce.ar.2 = f32[]
+      all-reduce(%p),
+      replica_groups={{0},{1}},
+      all_reduce_id=1,
+      to_apply=%sum.1,
+      sharding={maximal device=1}
+  %all-reduce.2 = f32[]
+      all-reduce(%all-reduce.ar.2),
+      replica_groups={{0,1}},
+      to_apply=%sum.1,
+      sharding={maximal device=1}
+  %multiply.2 = f32[]
+      multiply(%all-reduce.2, %constant.f32),
+      sharding={maximal device=1}
+
+  ROOT %tuple = (f32[], f32[])
+      tuple(%all-reduce.1, %all-reduce.2),
+      sharding={{maximal device=0}, {maximal device=1}}
+}
+)";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseAndReturnVerifiedModule(module_str));
+  auto crs_before =
+      module->entry_computation()->root_instruction()->operands()[0];
+  auto replica_groups_before = crs_before->replica_groups();
+  ArCrsCombiner combiner(2);
+  auto changed = combiner.Run(module.get()).ValueOrDie();
+  EXPECT_TRUE(changed);
+  EXPECT_THAT(module->entry_computation()->root_instruction(),
+              op::Tuple(op::AllReduce(op::Parameter()),
+                        op::AllReduce(op::Parameter())));
+  auto crs_after =
+      module->entry_computation()->root_instruction()->operands()[0];
+  auto replica_groups_after = crs_after->replica_groups();
+  CompareReplicaGroups(replica_groups_before, replica_groups_after);
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 18ce5c3f3e634f77fd90168034fa453cb0f5c08b Mon Sep 17 00:00:00 2001
From: ymodak <42785357+ymodak@users.noreply.github.com>
Date: Mon, 14 Jan 2019 15:28:12 -0800
Subject: [PATCH 0682/2345] Update conv_utils_test.py

skipped one line
---
 tensorflow/python/keras/utils/conv_utils_test.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/python/keras/utils/conv_utils_test.py b/tensorflow/python/keras/utils/conv_utils_test.py
index 75dc718b70..a736752ba4 100644
--- a/tensorflow/python/keras/utils/conv_utils_test.py
+++ b/tensorflow/python/keras/utils/conv_utils_test.py
@@ -51,7 +51,6 @@ input_shapes = [
     (1, 2, 3, 4, 7, 2),
 ]
 
-
 class TestBasicConvUtilsTest(test.TestCase):
   def test_convert_data_format(self):
     self.assertEqual('NCDHW',
-- 
GitLab


From ae37e634c2f39b9514c73faab140a4fb980f976d Mon Sep 17 00:00:00 2001
From: Marek Drozdowski <mdrozdowski@nvidia.com>
Date: Fri, 16 Nov 2018 17:01:15 -0800
Subject: [PATCH 0683/2345] initial commit (add lru_cache class, enable
 trt_engine_op to use it)

change deconstructor

Formatting

Add test for changing input shapes

Dynamic input shapes test passes. Static mode and cached_input_shapes not implemented yet

enable to run network in tests multiple times

Fix and improve input shape cache. Change cached_engine_batch_sizes -> cached_engine_input_shapes

Formatting

Implement SerializeShapesString for TensorShapes too

All tests passing

Revert cached_engine_input_shapes -> cached_engine_batches.

Rename cached_engine_batches -> cached_engine_batch_sizes

Improve hash and remove < operator

Forgot to rename hasher

Fix formatting

remove commented out code

remove lru_cache mutex

change deconstructor and declaration order

Formatting and use default deconstructor

Use default deconstructor

Use Resource manager for LRU cache

Implement feedback. Bug with destructor of resources when not using singleton

Move allocator to TRTEngineCacheResource to fix segfault.

Merge changes from e51fa30
---
 tensorflow/contrib/tensorrt/BUILD             |  22 ++
 .../contrib/tensorrt/convert/convert_graph.cc |  13 +-
 .../contrib/tensorrt/convert/convert_graph.h  |   5 +-
 .../contrib/tensorrt/convert/convert_nodes.h  |   2 +-
 .../tensorrt/convert/trt_optimization_pass.cc |   6 +-
 .../contrib/tensorrt/kernels/trt_engine_op.cc | 223 ++++++++++--------
 .../contrib/tensorrt/kernels/trt_engine_op.h  |  29 +--
 .../contrib/tensorrt/ops/trt_engine_op.cc     |   2 +-
 .../contrib/tensorrt/python/trt_convert.py    |   6 +-
 .../tensorrt/python/trt_convert_test.py       |   4 +-
 .../tensorrt/resources/trt_lru_cache.h        | 175 ++++++++++++++
 .../tensorrt/resources/trt_lru_cache_test.cc  |  57 +++++
 tensorflow/contrib/tensorrt/test/base_test.py |  28 +--
 .../tensorrt/test/batch_matmul_test.py        |   4 +-
 .../tensorrt/test/biasadd_matmul_test.py      |   4 +-
 .../binary_tensor_weight_broadcast_test.py    |   4 +-
 .../tensorrt/test/concatenation_test.py       |   4 +-
 .../tensorrt/test/const_broadcast_test.py     |   4 +-
 .../test/dynamic_input_shapes_test.py         | 105 +++++++++
 .../contrib/tensorrt/test/lru_cache_test.py   |  62 +++++
 .../contrib/tensorrt/test/manual_test.py      |   4 +-
 .../tensorrt/test/memory_alignment_test.py    |   4 +-
 .../multi_connection_neighbor_engine_test.py  |   4 +-
 .../tensorrt/test/neighboring_engine_test.py  |   4 +-
 .../tensorrt/test/quantization_test.py        |   4 +-
 .../contrib/tensorrt/test/rank_two_test.py    |   4 +-
 .../tensorrt/test/reshape_transpose_test.py   |   8 +-
 .../test/tf_trt_integration_test_base.py      |  93 ++++----
 .../contrib/tensorrt/test/unary_test.py       |   4 +-
 .../tensorrt/test/vgg_block_nchw_test.py      |   4 +-
 .../contrib/tensorrt/test/vgg_block_test.py   |   4 +-
 31 files changed, 671 insertions(+), 225 deletions(-)
 create mode 100644 tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
 create mode 100644 tensorflow/contrib/tensorrt/resources/trt_lru_cache_test.cc
 create mode 100644 tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py
 create mode 100644 tensorflow/contrib/tensorrt/test/lru_cache_test.py

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 9b798425b6..3df56271de 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -222,6 +222,7 @@ tf_cuda_library(
     hdrs = [
         "resources/trt_int8_calibrator.h",
         "resources/trt_resource_manager.h",
+        "resources/trt_lru_cache.h",
         "resources/trt_resources.h",
     ],
     deps = [
@@ -264,6 +265,21 @@ tf_cc_test(
     ],
 )
 
+tf_cc_test(
+    name = "trt_lru_cache_test",
+    size = "small",
+    srcs = ["resources/trt_lru_cache_test.cc"],
+    tags = [
+        "no_windows",
+        "nomac",
+    ],
+    deps = [
+        ":trt_resources",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
 # Library for the node-level conversion portion of TensorRT operation creation
 tf_cuda_library(
     name = "trt_conversion",
@@ -493,8 +509,13 @@ cuda_py_tests(
         "test/concatenation_test.py",
         "test/const_broadcast_test.py",
         "test/conv2d_test.py",
+        "test/dynamic_input_shapes_test.py",
         "test/identity_output_test.py",
+<<<<<<< ab254f392694eb2f7231526c64fc301ef10a1384
         "test/int32_test.py",
+=======
+        "test/lru_cache_test.py",
+>>>>>>> initial commit (add lru_cache class, enable trt_engine_op to use it)
         "test/manual_test.py",
         "test/memory_alignment_test.py",
         "test/multi_connection_neighbor_engine_test.py",
@@ -550,6 +571,7 @@ cc_library(
     copts = tf_copts(),
     deps = [
         "//tensorflow/core:lib",
+        "//tensorflow/core:framework"
     ],
 )
 
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index b1f631bf5f..87c2d367f8 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -239,7 +239,7 @@ tensorflow::Status ConvertGraphDefToTensorRT(
     const std::vector<string>& output_names, size_t max_batch_size,
     size_t max_workspace_size_bytes, tensorflow::GraphDef* new_graph_def,
     int precision_mode, int minimum_segment_size, bool is_dyn_op,
-    int max_cached_engines, std::vector<int> cached_engine_batches,
+    int max_cached_engines, std::vector<int> cached_engine_batch_sizes,
     bool use_calibration) {
   // Create GrapplerItem.
   tensorflow::grappler::GrapplerItem item;
@@ -301,9 +301,9 @@ tensorflow::Status ConvertGraphDefToTensorRT(
   TF_RETURN_IF_ERROR(GetPrecisionModeName(
       precision_mode, parameters["precision_mode"].mutable_s()));
   parameters["maximum_cached_engines"].set_i(max_cached_engines);
-  if (!cached_engine_batches.empty()) {
-    auto list = parameters["cached_engine_batches"].mutable_list();
-    for (const int batch : cached_engine_batches) {
+  if (!cached_engine_batch_sizes.empty()) {
+    auto list = parameters["cached_engine_batch_sizes"].mutable_list();
+    for (const int batch : cached_engine_batch_sizes) {
       list->add_i(batch);
     }
   }
@@ -689,7 +689,7 @@ tensorflow::Status CreateTRTNode(const std::vector<EngineInfo>& infos, int pos,
   }
 
   if (info.engine_type == EngineInfo::EngineType::TRTStatic &&
-      info.cached_engine_batches.size()) {
+      !info.cached_engine_batch_sizes.empty()) {
     LOG(WARNING) << "Cached engine batches are ignored for static engines";
   }
   tensorflow::NodeDef trt_node;
@@ -703,7 +703,6 @@ tensorflow::Status CreateTRTNode(const std::vector<EngineInfo>& infos, int pos,
           .Attr("serialized_segment", segment_string)
           .Attr("calibration_data", "")
           .Attr("max_cached_engines_count", info.maximum_cached_engines)
-          .Attr("cached_engine_batches", {max_batch_size})
           .Attr("workspace_size_bytes", info.max_workspace_size_bytes)
           .Attr("precision_mode", prec_string)
           .Attr("use_calibration", info.use_calibration)
@@ -984,7 +983,7 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
                                    ? EngineInfo::EngineType::TRTDynamic
                                    : EngineInfo::EngineType::TRTStatic);
     curr_engine.use_calibration = params.use_calibration;
-    curr_engine.cached_engine_batches = params.cached_engine_batches;
+    curr_engine.cached_engine_batch_sizes = params.cached_engine_batch_sizes;
     curr_engine.maximum_cached_engines = params.max_cached_engines;
     StrAppend(&curr_engine.engine_name, "TRTEngineOp_", t);
     status = RegisterSegmentFunctionToFunctionLibrary(
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.h b/tensorflow/contrib/tensorrt/convert/convert_graph.h
index 1f39f56f63..5616bbefd9 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.h
@@ -82,7 +82,7 @@ struct ConversionParams {
   bool fixed_input_size;   // Assume non-batch ranks of input tensors are fixed
   int max_cached_engines;  // maximum number of cached engines
   bool use_calibration;
-  std::vector<int> cached_engine_batches;  // list of cached engines
+  std::vector<int> cached_engine_batch_sizes;  // list of cached engines
 };
 
 // This method extracts calibration information from the resource managers
@@ -101,7 +101,8 @@ tensorflow::Status ConvertGraphDefToTensorRT(
     size_t max_workspace_size_bytes, tensorflow::GraphDef* new_graph_def,
     int precision_mode = 1, int minimum_segment_size = 3,
     bool is_dyn_op = false, int max_cached_engines = 1,
-    std::vector<int> cached_engine_batches = {}, bool use_calibration = true);
+    std::vector<int> cached_engine_batch_sizes = {},
+    bool use_calibration = true);
 
 // Method to call from optimization pass
 tensorflow::Status ConvertAfterShapes(ConversionParams& params);
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.h b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
index 8f2271ee3f..f9f7c0bdf1 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
@@ -108,7 +108,7 @@ struct EngineInfo {
   EngineType engine_type;
   int64 max_workspace_size_bytes;
   int maximum_cached_engines;
-  std::vector<int> cached_engine_batches;
+  std::vector<int> cached_engine_batch_sizes;
   int precision_mode;
   bool use_calibration;
 };
diff --git a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc
index d57f2300f8..e7ca126f55 100644
--- a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc
+++ b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc
@@ -50,8 +50,8 @@ tensorflow::Status TRTOptimizationPass::Init(
   if (params.count("is_dynamic_op")) {
     is_dynamic_op_ = params.at("is_dynamic_op").b();
   }
-  if (params.count("cached_engine_batches")) {
-    auto batch_vec = params.at("cached_engine_batches").list();
+  if (params.count("cached_engine_batch_sizes")) {
+    auto batch_vec = params.at("cached_engine_batch_sizes").list();
     batches_.reserve(batch_vec.i_size());
     for (const auto i : batch_vec.i()) {
       batches_.push_back(i);
@@ -258,7 +258,7 @@ tensorflow::Status TRTOptimizationPass::Optimize(
   cp.graph_properties = &static_graph_properties;
   cp.cluster = cluster;
   cp.is_dyn_op = is_dynamic_op_;
-  cp.cached_engine_batches = batches_;
+  cp.cached_engine_batch_sizes = batches_;
   cp.max_cached_engines = max_cached_batches_;
   cp.use_calibration = use_calibration_;
   auto status = tensorflow::tensorrt::convert::ConvertAfterShapes(cp);
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
index ae5e64e823..e1e64fec68 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
@@ -136,14 +136,13 @@ TRTEngineOp::TRTEngineOp(OpKernelConstruction* context)
   native_func_ = tensorflow::kInvalidHandle;
   OP_REQUIRES_OK(context, context->GetAttr("max_cached_engines_count",
                                            &max_cached_engines_));
-  OP_REQUIRES_OK(context,
-                 context->GetAttr("fixed_input_size", &fixed_input_size_));
-  OP_REQUIRES_OK(context, context->GetAttr("cached_engine_batches",
-                                           &cached_engine_batches_));
-  std::sort(cached_engine_batches_.begin(), cached_engine_batches_.end());
+  OP_REQUIRES_OK(context, context->GetAttr("cached_engine_batch_sizes",
+                                           &cached_engine_batch_sizes_));
+  std::sort(cached_engine_batch_sizes_.begin(),
+            cached_engine_batch_sizes_.end());
   if (VLOG_IS_ON(1)) {
     string s("Engine Batches= ");
-    for (auto i : cached_engine_batches_) {
+    for (auto i : cached_engine_batch_sizes_) {
       StrAppend(&s, i, " ");
     }
     VLOG(1) << s;
@@ -240,32 +239,30 @@ void TRTEngineOp::ExecuteCalibration(OpKernelContext* ctx,
   ExecuteNativeSegment(ctx, helper);
 }
 
-int TRTEngineOp::GetEngineBatch(OpKernelContext* ctx) {
-  int num_batch = ctx->input(0).shape().dim_size(0);
-  int smallest_engine = 0;
-  for (const auto i : cached_engine_batches_) {
-    if (i >= num_batch) {
-      smallest_engine = i;
-      break;
-    }
-  }
-  // TODO(sami): Need an LRU here
-  if (smallest_engine == 0) {
-    if (max_cached_engines_ > cached_engine_batches_.size()) {
-      smallest_engine = num_batch;
-      cached_engine_batches_.push_back(num_batch);
-      VLOG(1) << "Running with batch size " << num_batch;
-    } else {
-      string msg =
-          StrCat("Engine buffer is full. buffer limit=", max_cached_engines_,
-                 ", current entries=");
-      for (auto i : cached_engine_batches_) StrAppend(&msg, i, ",");
-      StrAppend(&msg, " requested batch=", num_batch);
-      LOG(WARNING) << msg;
-      return -1;
+bool TRTEngineOp::GetCompatibleCachedEngine(
+    const std::vector<TensorShape>& actual_input_shapes,
+    std::vector<TensorShape>* engine_input_shapes) {
+  const int batch_size = actual_input_shapes[0].dim_size(0);
+  int smallest_batch_size = -1;
+  // Output shape will always be the same as the input but we will overwrite the
+  // batch size.
+  *engine_input_shapes = actual_input_shapes;
+  for (const int cached_batch_size : cached_engine_batch_sizes_) {
+    // Check if compatible: batch <= cached batch.
+    if (batch_size <= cached_batch_size) {
+      // First case: first compatible engine found
+      // Second case: smaller batch size engine found
+      if ((smallest_batch_size == -1) ||
+          (cached_batch_size < smallest_batch_size)) {
+        smallest_batch_size = cached_batch_size;
+        // Overwrite batch size for output
+        for (int i = 0; i < engine_input_shapes->size(); i++) {
+          (*engine_input_shapes)[i].set_dim(0, smallest_batch_size);
+        }
+      }
     }
   }
-  return smallest_engine;
+  return (smallest_batch_size != -1);
 }
 
 void TRTEngineOp::ComputeAsync(OpKernelContext* ctx,
@@ -276,23 +273,20 @@ void TRTEngineOp::ComputeAsync(OpKernelContext* ctx,
     ExecuteCalibration(ctx, helper);
     return;
   }
-  const int smallest_engine = GetEngineBatch(ctx);
-  if (smallest_engine < 0) {
-    LOG(WARNING) << "Failed to get engine batch, running native segment for "
-                 << name();
-    ExecuteNativeSegment(ctx, helper);
-    return;
+  // Get shapes of inputs to engine.
+  std::vector<tensorflow::TensorShape> input_shapes;
+  for (int i = 0; i < ctx->num_inputs(); ++i) {
+    input_shapes.emplace_back(ctx->input(i).shape());
   }
-
-  const int num_batch = ctx->input(0).shape().dim_size(0);
-  EngineContext* engine_context = GetEngine(smallest_engine, ctx);
+  EngineContext* engine_context = GetEngine(input_shapes, ctx);
   if (!engine_context->cuda_engine) {
-    LOG(WARNING) << "Engine retrieval for batch size " << num_batch
+    LOG(WARNING) << "Engine retrieval for input shapes: "
+                 << TensorShapeUtils::ShapeListString(input_shapes)
                  << " failed. Running native segment for " << name();
     ExecuteNativeSegment(ctx, helper);
     return;
   }
-  const bool retry = ExecuteTrtEngine(ctx, num_batch, engine_context);
+  const bool retry = ExecuteTrtEngine(ctx, engine_context);
   if (retry) {
     LOG(WARNING) << "Failed to execute engine, "
                  << "retrying with native segment for " << name();
@@ -301,12 +295,15 @@ void TRTEngineOp::ComputeAsync(OpKernelContext* ctx,
   }
 }
 
-bool TRTEngineOp::ExecuteTrtEngine(OpKernelContext* ctx, const int num_batch,
+bool TRTEngineOp::ExecuteTrtEngine(OpKernelContext* ctx,
                                    EngineContext* engine_context) {
-  auto& cuda_engine = engine_context->cuda_engine;
-
+  
   VLOG(1) << "Executing TRT engine: " << name();
+  auto& cuda_engine = engine_context->cuda_engine;
   const bool kRetry = true;
+  // All inputs must have the same batch size, so just get it from the first
+  // input.
+  const int num_batch = ctx->input(0).shape().dim_size(0);
   const int num_binding = ctx->num_inputs() + ctx->num_outputs();
   std::vector<void*> buffers(num_binding);
   for (int i = 0; i < ctx->num_inputs(); i++) {
@@ -417,48 +414,45 @@ bool TRTEngineOp::ExecuteTrtEngine(OpKernelContext* ctx, const int num_batch,
   return !kRetry;
 }
 
-TRTEngineOp::~TRTEngineOp() {
-  // We need to manually destroy the engine and execution context before
-  // the allocator is destructed.
-  for (auto& eng : engine_map_) {
-    eng.second->cuda_engine.reset();
-    eng.second->execution_context.reset();
-  }
-  allocator_.reset();
-}
-
-nvinfer1::IGpuAllocator* TRTEngineOp::GetAllocator(OpKernelContext* ctx) {
-  if (allocator_) return allocator_.get();
-  auto device = ctx->device();
-  auto alloc = device->GetAllocator(tensorflow::AllocatorAttributes());
-  if (!alloc) {
-    LOG(ERROR) << "Can't find device allocator for gpu device "
-               << device->name();
-    return nullptr;
-  }
-  allocator_.reset(new TRTDeviceAllocator(alloc));
-  return allocator_.get();
-}
-
-TRTEngineOp::EngineContext* TRTEngineOp::GetEngine(int batch_size,
-                                                   OpKernelContext* ctx) {
+EngineContext* TRTEngineOp::GetEngine(
+    const std::vector<TensorShape>& input_shapes, OpKernelContext* ctx) {
   static EngineContext empty_context;
-  // TODO(sami): This method needs to be re-written to use resource manager and
-  // with LRU mechanism option.
   tensorflow::mutex_lock lock(engine_mutex_);
+  // TODO(tmorris): using first input to get batch size - is this reliable?
+  const int batch_size = input_shapes[0].dim_size(0);
+
+  // Get engine cache
+  TRTEngineCacheResource* cache_res = nullptr;
+  auto status = ctx->resource_manager()->LookupOrCreate(
+      "TRTEngineCache", funcdef_name_, &cache_res,
+      {[this, ctx](TRTEngineCacheResource** cr) -> tensorflow::Status {
+        *cr = new TRTEngineCacheResource(ctx, this->max_cached_engines_);
+        return Status::OK();
+      }});
+  if (!status.ok()) {
+    ctx->SetStatus(status);
+    return &empty_context;
+  }
+  tensorflow::core::ScopedUnref sc(cache_res);
+  auto& cache = cache_res->cache_;
+  auto allocator = cache_res->allocator_.get();
+  if (allocator == nullptr) {
+    return &empty_context;
+  }
 
+  // Handle the static engine case. For static engines, the cache will have a
+  // single element containing the only engine.
   if (static_engine_) {
-    if (engine_map_.size()) {
-      if (engine_map_.begin()->first >= batch_size) {
-        return engine_map_.begin()->second.get();
+    if (cache.size()) {
+      // Batch size of engine must be >= the input batch size
+      // TODO(tmorris): use match compatible function?
+      if (cache.begin()->first[0].dim_size(0) >= batch_size) {
+        return cache.begin()->second.get();
       }
       return &empty_context;
     }
+
     TrtUniquePtrType<IRuntime> infer(nvinfer1::createInferRuntime(logger));
-    auto allocator = GetAllocator(ctx);
-    if (allocator == nullptr) {
-      return &empty_context;
-    }
     infer->setGpuAllocator(allocator);
     TrtUniquePtrType<nvinfer1::ICudaEngine> static_engine(
         infer->deserializeCudaEngine(serialized_segment_.c_str(),
@@ -466,47 +460,71 @@ TRTEngineOp::EngineContext* TRTEngineOp::GetEngine(int batch_size,
                                      PluginFactoryTensorRT::GetInstance()));
     auto raw_static_engine = static_engine.get();
     const auto max_batch_size = raw_static_engine->getMaxBatchSize();
-    engine_map_[max_batch_size] = absl::make_unique<EngineContext>(
-        std::move(static_engine),
-        TrtUniquePtrType<nvinfer1::IExecutionContext>(
-            raw_static_engine->createExecutionContext()));
+    // Static engine will have max_batch_size for batch size so that all inputs
+    // will map to this single engine.
+    std::vector<TensorShape> engine_input_shapes(input_shapes);
+    for (int i = 0; i < engine_input_shapes.size(); i++) {
+      // TODO(tmorris): will all inputs have batch size as first dimension??
+      engine_input_shapes[i].set_dim(0, max_batch_size);
+    }
+    // Assume engine_input_shapes matches the input shapes of the engine.
+    cache.emplace(
+        engine_input_shapes, absl::make_unique<EngineContext>(
+            std::move(static_engine),
+            TrtUniquePtrType<nvinfer1::IExecutionContext>(
+                raw_static_engine->createExecutionContext())));
     // Runtime is safe to delete after engine creation
     serialized_segment_.clear();
     if (max_batch_size < batch_size) {
       return &empty_context;
     }
-    return engine_map_.at(max_batch_size).get();
+    return cache.at(input_shapes).get();
   }  // static_engine_
 
   // Handle the dynamic engine case.
-  auto engine_it = engine_map_.find(batch_size);
-  if (engine_it == engine_map_.end() &&
-      engine_map_.size() < (size_t)max_cached_engines_) {
-    nvinfer1::IGpuAllocator* allocator = nullptr;
-    allocator = GetAllocator(ctx);
-    if (allocator == nullptr) {
-      return &empty_context;
-    }
-    std::vector<tensorflow::PartialTensorShape> shapes;
-    for (int i = 0; i < ctx->num_inputs(); ++i) {
-      shapes.emplace_back(ctx->input(i).shape());
+  // See if there is a compatible engine cached. The batch size should be <= the
+  // cached batch size.
+  std::vector<tensorflow::TensorShape> engine_input_shapes;
+  bool matched_successfully =
+      GetCompatibleCachedEngine(input_shapes, &engine_input_shapes);
+  // If matched, use that engine. Otherwise, we will look in cache for that
+  // exact shape and possibly create a new engine if it is not in cache.
+  if (!matched_successfully) {
+    engine_input_shapes = input_shapes;
+    if (!cached_engine_batch_sizes_.empty()) {
+      // If user has explicitly defined cached_engine_batch_sizes, we should
+      // warn them that their input was non-compatible (batch size too high)
+      LOG(WARNING) << "No compatible cached engine was found for batch size: "
+                   << batch_size << ". A new engine will be created.";
+      cached_engine_batch_sizes_.push_back(batch_size);
     }
+  }
+  tensorflow::Status engine_found_status;
+  cache.at(engine_input_shapes, &engine_found_status);
+
+  if (!engine_found_status.ok()) {
     TrtUniquePtrType<nvinfer1::ICudaEngine> engine;
     bool convert_successfully = false;
     LOG(INFO) << "Building a new TensorRT engine for " << name()
-              << " with batch size " << batch_size;
+              << " input shapes: "
+              << TensorShapeUtils::ShapeListString(engine_input_shapes);
+    // Convert to partial shapes
+    std::vector<PartialTensorShape> partial_shapes;
+    for (int i = 0; i < engine_input_shapes.size(); i++) {
+      partial_shapes.emplace_back(engine_input_shapes[i]);
+    }
     // Up to this point, calibrator_ can never be empty, since otherwise it
     // means calibration_mode_ is true and this path won't get executed.
     auto status = convert::ConvertGraphDefToEngine(
-        segment_graph_, precision_mode_, batch_size, workspace_size_, shapes,
-        &logger, allocator, calibrator_.get(), &engine, use_calibration_,
-        &convert_successfully);
+        segment_graph_, precision_mode_, batch_size, workspace_size_,
+        partial_shapes, &logger, allocator, calibrator_.get(), &engine,
+        use_calibration_, &convert_successfully);
     if (!status.ok()) {
       if (convert_successfully) {
         // This means it fail to build the engine even when the network is built
         // successfully, probably due to internal issues. In this case we don't
         // retry in the future.
-        engine_map_[batch_size] = absl::make_unique<EngineContext>();
+        cache.emplace(engine_input_shapes, absl::make_unique<EngineContext>());
       }
       LOG(WARNING) << "Engine creation for batch size " << batch_size
                    << " failed " << status;
@@ -515,10 +533,11 @@ TRTEngineOp::EngineContext* TRTEngineOp::GetEngine(int batch_size,
     VLOG(1) << "Conversion is done";
     TrtUniquePtrType<nvinfer1::IExecutionContext> exec_context(
         engine->createExecutionContext());
-    engine_map_[batch_size] = absl::make_unique<EngineContext>(
-        std::move(engine), std::move(exec_context));
+    cache.emplace(engine_input_shapes,
+                  absl::make_unique<EngineContext>(std::move(engine),
+                                                   std::move(exec_context)));
   }
-  return engine_map_.at(batch_size).get();
+  return cache.at(engine_input_shapes).get();
 }
 
 tensorflow::Status TRTEngineOp::AllocateCalibrationResources(
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
index 38ea224063..4f39a4c548 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
@@ -22,6 +22,7 @@ limitations under the License.
 #include "tensorflow/contrib/tensorrt/convert/utils.h"
 #include "tensorflow/contrib/tensorrt/log/trt_logger.h"
 #include "tensorflow/contrib/tensorrt/resources/trt_allocator.h"
+#include "tensorflow/contrib/tensorrt/resources/trt_lru_cache.h"
 #include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/framework/op.h"
@@ -49,7 +50,6 @@ class TRTEngineOp : public AsyncOpKernel {
 
   void ComputeAsync(OpKernelContext* context,
                     AsyncOpKernel::DoneCallback done) override;
-  ~TRTEngineOp();
 
  private:
   // TODO(samikama): context should go to a resource manager!
@@ -78,28 +78,25 @@ class TRTEngineOp : public AsyncOpKernel {
 
   // Execute the tensorrt engine. Returns whether we need to retry by running
   // the native segment.
-  bool ExecuteTrtEngine(OpKernelContext* ctx, const int num_batch,
-                        EngineContext* engine_context);
+  bool ExecuteTrtEngine(OpKernelContext* ctx, EngineContext* engine_context);
 
   // Allocate necessary resources for calibration
   Status AllocateCalibrationResources(OpKernelContext* ctx,
                                       TRTCalibrationResource** cr);
 
-  EngineContext* GetEngine(int batch_size, OpKernelContext* ctx);
+  // Get engine for the input shape
+  EngineContext* GetEngine(const std::vector<TensorShape>& input_shapes,
+                           OpKernelContext* ctx);
 
-  // Return engine batch closest to input batch.
-  int GetEngineBatch(OpKernelContext* ctx);
+  // Return engine batch in cached_engne_batch_sizes_ which is closest to input
+  // batch.
+  bool GetCompatibleCachedEngine(
+      const std::vector<TensorShape>& actual_input_shapes,
+      std::vector<TensorShape>* engine_input_shapes);
 
-  nvinfer1::IGpuAllocator* GetAllocator(OpKernelContext* ctx);
-
-  // map to keep engines and their execution context for given batch size.
-  std::unordered_map<int, std::unique_ptr<EngineContext>> engine_map_;
   std::vector<string> input_nodes_;
   std::vector<string> output_nodes_;
 
-  // keep device allocator for TRT.
-  std::unique_ptr<TRTBaseAllocator> allocator_;
-
   // serialized protobuf segment or trt engine depending on static_engine_ flag.
   string serialized_segment_;
 
@@ -119,12 +116,8 @@ class TRTEngineOp : public AsyncOpKernel {
   // Whether to calibrate INT8 engine.
   bool calibration_mode_;
 
-  // Whether non-batch ranks of the inputs are assumed to be fixed or not for
-  // engine construction.
-  bool fixed_input_size_;
-
   // Batches of the cached engines
-  std::vector<int> cached_engine_batches_;
+  std::vector<int> cached_engine_batch_sizes_;
 
   // Maximum number of cached engines
   int max_cached_engines_;
diff --git a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
index 2f761c8b8e..73cc046bc3 100644
--- a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
@@ -40,7 +40,7 @@ REGISTER_OP("TRTEngineOp")
     .Attr("OutT: list({int8,float16,float32,int32})")
     .Attr("static_engine: bool = true")
     .Attr("fixed_input_size: bool = true")
-    .Attr("cached_engine_batches: list(int) = []")
+    .Attr("cached_engine_batch_sizes: list(int) >= 0 = []")
     .Attr("max_cached_engines_count: int = 1")
     .Attr("workspace_size_bytes: int")
     .Attr("precision_mode: {'FP32', 'FP16', 'INT8'}")
diff --git a/tensorflow/contrib/tensorrt/python/trt_convert.py b/tensorflow/contrib/tensorrt/python/trt_convert.py
index 92377edaec..8de17347f8 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert.py
@@ -99,7 +99,7 @@ def get_tensorrt_rewriter_config(rewriter_config=None,
       based on which the TRTEngineOp is created.
     cached_engine_batch_sizes: a list of batch sizes used to create cached
       engines, only used when is_dynamic_op is True. The length of the list
-      should be smaller than maximum_cached_engines, and the dynamic TRT op will
+      should be <= maximum_cached_engines, and the dynamic TRT op will
       use this list to determine the batch sizes of the cached engines, instead
       of making the decision on the fly. This is useful when we know the most
       common batch size(s) the application is going to generate.
@@ -156,7 +156,7 @@ def get_tensorrt_rewriter_config(rewriter_config=None,
     if len(cached_engine_batch_sizes) > maximum_cached_engines:
       raise ValueError("cached_engine_batch_sizes should not contain more than "
                        "maximum_cached_engines items.")
-    optimizer.parameter_map["cached_engine_batches"].list.i.extend(
+    optimizer.parameter_map["cached_engine_batch_sizes"].list.i.extend(
         cached_engine_batch_sizes)
   optimizer.parameter_map["use_calibration"].b = use_calibration
   return rewriter_config_with_trt
@@ -199,7 +199,7 @@ def create_inference_graph(input_graph_def,
       based on which the TRTEngineOp is created.
     cached_engine_batch_sizes: a list of batch sizes used to create cached
       engines, only used when is_dynamic_op is True. The length of the list
-      should be smaller than maximum_cached_engines, and the dynamic TRT op will
+      should be <= maximum_cached_engines, and the dynamic TRT op will
       use this list to determine the batch sizes of the cached engines, instead
       of making the decision on the fly. This is useful when we know the most
       common batch size(s) the application is going to generate.
diff --git a/tensorflow/contrib/tensorrt/python/trt_convert_test.py b/tensorflow/contrib/tensorrt/python/trt_convert_test.py
index a7b2d2ea50..60c31580db 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert_test.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert_test.py
@@ -71,7 +71,7 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
     for key in [
         "minimum_segment_size", "max_batch_size", "is_dynamic_op",
         "max_workspace_size_bytes", "precision_mode", "maximum_cached_engines",
-        "cached_engine_batches"
+        "cached_engine_batch_sizes"
     ]:
       self.assertTrue(key in trt_optimizer.parameter_map)
     self.assertEqual(10, trt_optimizer.parameter_map["minimum_segment_size"].i)
@@ -85,7 +85,7 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
     self.assertEqual(2, trt_optimizer.parameter_map["maximum_cached_engines"].i)
     self.assertEqual(
         [1, 128],
-        trt_optimizer.parameter_map["cached_engine_batches"].list.i)
+        trt_optimizer.parameter_map["cached_engine_batch_sizes"].list.i)
 
   def _GetConfigProto(self):
     """Get ConfigProto for session creation."""
diff --git a/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h b/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
new file mode 100644
index 0000000000..4d63a03b40
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
@@ -0,0 +1,175 @@
+#ifndef TENSORFLOW_CONTRIB_TENSORRT_TRT_LRU_CACHE_H_
+#define TENSORFLOW_CONTRIB_TENSORRT_TRT_LRU_CACHE_H_
+
+#include <list>
+#include <unordered_map>
+
+#include "tensorflow/contrib/tensorrt/convert/utils.h"
+#include "tensorflow/contrib/tensorrt/resources/trt_allocator.h"
+#include "tensorflow/core/framework/resource_mgr.h"
+#include "tensorflow/core/lib/core/errors.h"
+
+#if GOOGLE_CUDA
+#if GOOGLE_TENSORRT
+#include "tensorrt/include/NvInfer.h"
+
+namespace tensorflow {
+namespace tensorrt {
+
+template <class Key, class Value, class HashFunction>
+class LRUCache {
+ public:
+  typedef Value value_type;
+  typedef Key key_type;
+  typedef HashFunction hasher;
+  typedef typename std::unordered_map<key_type, value_type, hasher>::iterator
+      map_iterator;
+
+  LRUCache() : capacity_(0) {}
+  explicit LRUCache(size_t capacity) : capacity_(capacity) {}
+
+  size_t capacity() const { return capacity_; }
+
+  void reserve(size_t capacity) {
+    capacity_ = capacity;
+    DiscardOld();
+  }
+
+  size_t size() const { return objects_.size(); }
+
+  size_t count(const key_type& k) const { return objects_.count(k); }
+
+  value_type& at(key_type k, tensorflow::Status* status_ptr = nullptr) {
+    tensorflow::Status status = Touch(k);
+    if (!status.ok()) {
+      if (status_ptr) {
+        *status_ptr = status;
+      }
+      return not_found_value_;
+    }
+    return objects_.at(k);
+  }
+
+  map_iterator begin() { return objects_.begin(); }
+
+  map_iterator end() { return objects_.end(); }
+
+  template <typename... Args>
+  std::pair<map_iterator, bool> emplace(Args&&... args) {
+    DiscardOld(1);
+    std::pair<map_iterator, bool> result =
+        objects_.emplace(std::forward<Args>(args)...);
+    key_type key = result.first->first;
+    if (result.second) {
+      keys_.push_front(key);
+    } else {
+      Touch(key);
+    }
+    return result;
+  }
+
+ private:
+  std::unordered_map<key_type, value_type, hasher> objects_;
+  std::list<key_type> keys_;
+  size_t capacity_;
+  value_type not_found_value_;
+
+  tensorflow::Status Touch(const key_type& key) {
+    if (!count(key)) {
+      return tensorflow::errors::NotFound("Key not found in cache.");
+    }
+    auto rank = std::find(keys_.begin(), keys_.end(), key);
+    if (rank != keys_.begin()) {
+      keys_.erase(rank);
+      keys_.push_front(key);
+    }
+    return tensorflow::Status::OK();
+  }
+
+  // Creates n free positions in cache
+  tensorflow::Status DiscardOld(size_t n = 0) {
+    if (n > capacity_) {
+      return tensorflow::errors::Internal(
+          "Insufficient capacity in cache (capacity = ", capacity_,
+          ", requested ", n, ")");
+    }
+    while (objects_.size() > (capacity_ - n)) {
+      key_type discard_key = keys_.back();
+      keys_.pop_back();
+      objects_.erase(discard_key);
+    }
+    return tensorflow::Status::OK();
+  }
+};
+
+// Define a hash function for vector<TensorShape> because it is used as the key
+// for the engine cache.
+struct VectorTensorShapeHasher {
+  std::size_t operator()(
+      const std::vector<tensorflow::TensorShape>& key) const {
+    return std::hash<std::string>()(TensorShapeUtils::ShapeListString(key));
+  }
+};
+
+struct EngineContext {
+  EngineContext() {}  // Creates an empty context.
+  EngineContext(
+      TrtUniquePtrType<nvinfer1::ICudaEngine>&& input_cuda_engine,
+      TrtUniquePtrType<nvinfer1::IExecutionContext>&& input_execution_context)
+      : cuda_engine(std::move(input_cuda_engine)),
+        execution_context(std::move(input_execution_context)) {}
+
+  mutex mu;
+  TrtUniquePtrType<nvinfer1::ICudaEngine> cuda_engine;
+  TrtUniquePtrType<nvinfer1::IExecutionContext> execution_context
+      GUARDED_BY(mu);
+};
+
+class TRTEngineCacheResource : public tensorflow::ResourceBase {
+ public:
+  TRTEngineCacheResource(OpKernelContext* ctx, size_t capacity)
+      : cache_(capacity) {
+    auto device = ctx->device();
+    auto alloc = device->GetAllocator(tensorflow::AllocatorAttributes());
+    if (!alloc) {
+      LOG(ERROR) << "Can't find device allocator for gpu device "
+                 << device->name();
+      allocator_ = nullptr;
+    } else {
+      allocator_.reset(new TRTDeviceAllocator(alloc));
+    }
+  }
+
+  string DebugString() override {
+    std::stringstream oss;
+    using std::dec;
+    using std::endl;
+    using std::hex;
+    oss << "TRTEngineCacheResource: ";
+    oss << "TRTBaseAllocator = " << hex << allocator_.get() << dec << ", ";
+    oss << "LRUCache = " << hex << &cache_ << dec << endl;
+    oss << "Containing " << cache_.size() << " entries: " << endl;
+    for (auto& item : cache_) {
+      oss << TensorShapeUtils::ShapeListString(item.first) << ": " << hex
+          << "ICudaEngine: " << item.second.get()->cuda_engine.get() << ", "
+          << "IExecutionContext: " << item.second.get()->execution_context.get()
+          << dec << endl;
+    }
+    return oss.str();
+  }
+
+  // Keep device allocator for TRT.
+  std::unique_ptr<TRTBaseAllocator> allocator_;
+
+  // Declare cache after allocator so that it is destroyed before allocator is.
+  LRUCache<std::vector<TensorShape>, std::unique_ptr<EngineContext>, VectorTensorShapeHasher>
+      cache_;
+};
+
+}  // namespace tensorrt
+}  // namespace tensorflow
+
+#endif  // GOOGLE_TENSORRT
+#endif  // GOOGLE_CUDA
+
+#endif  // TENSORFLOW_CONTRIB_TENSORRT_TRT_LRU_CACHE_H_
diff --git a/tensorflow/contrib/tensorrt/resources/trt_lru_cache_test.cc b/tensorflow/contrib/tensorrt/resources/trt_lru_cache_test.cc
new file mode 100644
index 0000000000..a0959479ce
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/resources/trt_lru_cache_test.cc
@@ -0,0 +1,57 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/contrib/tensorrt/resources/trt_lru_cache.h"
+
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+namespace tensorrt {
+
+TEST(LRUCacheTest, Basic) {
+  LRUCache<int, int, std::hash<int>> cache;
+  cache.reserve(2);
+  // Insert 10
+  cache.emplace(10, 100);
+  EXPECT_EQ(cache.size(), 1);
+  EXPECT_EQ(cache.count(10), 1);
+  EXPECT_EQ(cache.at(10), 100);
+  EXPECT_EQ(cache.count(100), 0);
+  // Insert 20
+  cache.emplace(20, 200);
+  EXPECT_EQ(cache.size(), 2);
+  EXPECT_EQ(cache.count(10), 1);
+  EXPECT_EQ(cache.count(20), 1);
+  EXPECT_EQ(cache.at(10), 100);
+  EXPECT_EQ(cache.at(20), 200);
+  EXPECT_EQ(cache.count(100), 0);
+  EXPECT_EQ(cache.count(200), 0);
+  // Insert 30, Evicting 10
+  cache.emplace(30, 300);
+  EXPECT_EQ(cache.count(10), 0);
+  EXPECT_EQ(cache.count(20), 1);
+  EXPECT_EQ(cache.count(30), 1);
+  // Touch 20
+  cache.at(20);
+  // Insert 40, Evicting 30
+  cache.emplace(40, 400);
+  EXPECT_EQ(cache.count(10), 0);
+  EXPECT_EQ(cache.count(20), 1);
+  EXPECT_EQ(cache.count(30), 0);
+  EXPECT_EQ(cache.count(40), 1);
+}
+
+}  // namespace tensorrt
+}  // namespace tensorflow
diff --git a/tensorflow/contrib/tensorrt/test/base_test.py b/tensorflow/contrib/tensorrt/test/base_test.py
index ff317e43e1..0f6293b166 100644
--- a/tensorflow/contrib/tensorrt/test/base_test.py
+++ b/tensorflow/contrib/tensorrt/test/base_test.py
@@ -68,9 +68,9 @@ class SimpleSingleEngineTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[(100, 6, 6, 6)])
+        expected_output_dims=[[(100, 6, 6, 6)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -125,9 +125,9 @@ class SimpleMultiEnginesTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[(100, 12, 12, 6)])
+        expected_output_dims=[[(100, 12, 12, 6)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -183,9 +183,9 @@ class PartiallyConvertedTestA(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[tuple(input_dims)])
+        expected_output_dims=[[tuple(input_dims)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -253,9 +253,9 @@ class ConstInputTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[tuple(input_dims)])
+        expected_output_dims=[[tuple(input_dims)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -286,9 +286,9 @@ class ConstDataInputSingleEngineTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[tuple(input_dims)])
+        expected_output_dims=[[tuple(input_dims)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -320,9 +320,9 @@ class ConstDataInputMultipleEnginesTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[tuple(input_dims)])
+        expected_output_dims=[[tuple(input_dims)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -369,9 +369,9 @@ class ControlDependencyTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[tuple(input_dims)])
+        expected_output_dims=[[tuple(input_dims)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/batch_matmul_test.py b/tensorflow/contrib/tensorrt/test/batch_matmul_test.py
index 384b84cb02..1b015dd369 100644
--- a/tensorflow/contrib/tensorrt/test/batch_matmul_test.py
+++ b/tensorflow/contrib/tensorrt/test/batch_matmul_test.py
@@ -71,9 +71,9 @@ class BatchMatMulTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name, w1_name, w2_name],
-        input_dims=[input_dims, w1_dims, w2_dims],
+        input_dims=[[input_dims, w1_dims, w2_dims]],
         output_names=[output_name],
-        expected_output_dims=[(12, 5, 8, 7)])
+        expected_output_dims=[[(12, 5, 8, 7)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py b/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
index 49f5e4cc9f..8a5f8ef5d5 100644
--- a/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
+++ b/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
@@ -111,9 +111,9 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[(4, 6680)])
+        expected_output_dims=[[(4, 6680)]])
 
   def GetConversionParams(self, run_params):
     """Return a ConversionParams for test."""
diff --git a/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py b/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
index 169835956c..0083695a50 100644
--- a/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
+++ b/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
@@ -63,9 +63,9 @@ class BinaryTensorWeightBroadcastTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[(5, 23040)])
+        expected_output_dims=[[(5, 23040)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/concatenation_test.py b/tensorflow/contrib/tensorrt/test/concatenation_test.py
index c3576f81d9..e6afe4e3f6 100644
--- a/tensorflow/contrib/tensorrt/test/concatenation_test.py
+++ b/tensorflow/contrib/tensorrt/test/concatenation_test.py
@@ -73,9 +73,9 @@ class ConcatenationTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[(2, 126)])
+        expected_output_dims=[[(2, 126)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/const_broadcast_test.py b/tensorflow/contrib/tensorrt/test/const_broadcast_test.py
index c1c883312d..56a297afb5 100644
--- a/tensorflow/contrib/tensorrt/test/const_broadcast_test.py
+++ b/tensorflow/contrib/tensorrt/test/const_broadcast_test.py
@@ -58,9 +58,9 @@ class ConstBroadcastTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[(5, 12, 12, 1)])
+        expected_output_dims=[[(5, 12, 12, 1)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py b/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py
new file mode 100644
index 0000000000..9985561e1e
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py
@@ -0,0 +1,105 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Script to test TF-TRT INT8 conversion without calibration on Mnist model."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+# pylint: disable=unused-import
+from tensorflow.contrib.tensorrt.python.ops import trt_engine_op
+# pylint: enable=unused-import
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import constant_op
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import nn
+from tensorflow.python.platform import test
+import numpy as np
+
+
+class DynamicInputShapesTest(trt_test.TfTrtIntegrationTestBase):
+
+  def GetParams(self):
+    input_dims = [[[1, 5, 5, 1]], [[10, 5, 5, 1]], [[3, 5, 5, 1]],
+                  [[1, 5, 5, 1]], [[1, 3, 1, 1]], [[2, 9, 9, 1]],
+                  [[1, 224, 224, 1]], [[1, 128, 224, 1]]]
+    expected_output_dims = input_dims
+
+    g = ops.Graph()
+    with g.as_default():
+      x = array_ops.placeholder(shape=(None, None, None, 1),
+                                dtype=dtypes.float32,
+                                name='input')
+      conv_filter1 = constant_op.constant(np.ones([3, 3, 1, 8]),
+                                          name="weights1",
+                                          dtype=dtypes.float32)
+      bias1 = constant_op.constant(np.random.randn(8), dtype=dtypes.float32)
+      x = nn.conv2d(
+          input=x,
+          filter=conv_filter1,
+          strides=[1, 1, 1, 1],
+          padding="SAME",
+          name="conv")
+      x = nn.bias_add(x, bias1)
+      x = nn.relu(x)
+      conv_filter2 = constant_op.constant(np.ones([3, 3, 8, 1]),
+                                          name="weights2",
+                                          dtype=dtypes.float32)
+      bias2 = constant_op.constant(np.random.randn(1), dtype=dtypes.float32)
+      x = nn.conv2d(
+          input=x,
+          filter=conv_filter2,
+          strides=[1, 1, 1, 1],
+          padding="SAME",
+          name="conv")
+      x = nn.bias_add(x, bias2)
+      x = array_ops.identity(x, name='output')
+
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=['input'],
+        input_dims=input_dims,
+        output_names=['output'],
+        expected_output_dims=expected_output_dims)
+
+  def GetConversionParams(self, run_params):
+    """Return a ConversionParams for test."""
+    conversion_params = super(DynamicInputShapesTest,
+                              self).GetConversionParams(run_params)
+    return conversion_params._replace(
+        maximum_cached_engines=10,
+        # Disable layout optimizer, since it will convert BiasAdd with NHWC
+        # format to NCHW format under four dimentional input.
+        rewriter_config=trt_test.OptimizerDisabledRewriterConfig())
+
+  def ExpectedEnginesToBuild(self, run_params):
+    return ["TRTEngineOp_0"]
+
+  def ShouldRunTest(self, run_params):
+    return (run_params.dynamic_engine and
+            not trt_test.IsQuantizationMode(run_params.precision_mode))
+
+  def ExpectedAbsoluteTolerance(self, run_params):
+    """The absolute tolerance to compare floating point results."""
+    return 1.e-03 if run_params.precision_mode == "FP32" else 1.e-01
+
+  def ExpectedRelativeTolerance(self, run_params):
+    """The relative tolerance to compare floating point results."""
+    return 1.e-03 if run_params.precision_mode == "FP32" else 1.e-01
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/contrib/tensorrt/test/lru_cache_test.py b/tensorflow/contrib/tensorrt/test/lru_cache_test.py
new file mode 100644
index 0000000000..74b51932ac
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/test/lru_cache_test.py
@@ -0,0 +1,62 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn
+from tensorflow.python.platform import test
+
+
+class LRUCacheTest(trt_test.TfTrtIntegrationTestBase):
+
+  def GetParams(self):
+    """Test for running network multiple times with different batch size to
+    check that lru_cache works correctly"""
+    dtype = dtypes.float32
+    input_name = "input"
+    input_dims = [[[1, 10, 10, 2]], [[2, 10, 10, 2]], [[4, 10, 10, 2]],
+                  [[2, 10, 10, 2]]]
+    expected_output_dims = [[(1, 10, 10, 1)], [(2, 10, 10, 1)],
+                            [(4, 10, 10, 1)], [(2, 10, 10, 1)]]
+    output_name = "output"
+    g = ops.Graph()
+    with g.as_default():
+      x = array_ops.placeholder(dtype=dtype,
+                                shape=[None, 10, 10, 2],
+                                name=input_name)
+      conv_filter = constant_op.constant(np.random.randn(3, 3, 2, 1),
+                                         dtype=dtypes.float32)
+      x = nn.conv2d(input=x,
+                    filter=conv_filter,
+                    strides=[1, 1, 1, 1],
+                    padding="SAME",
+                    name="conv")
+      bias = constant_op.constant(np.random.randn(1, 10, 10, 1),
+                                  dtype=dtypes.float32)
+      x = math_ops.add(x, bias)
+      x = nn.relu(x)
+      x = array_ops.identity(x, name="output")
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=[input_name],
+        input_dims=input_dims,
+        output_names=[output_name],
+        expected_output_dims=expected_output_dims)
+
+  def ExpectedEnginesToBuild(self, run_params):
+    """Return the expected engines to build."""
+    return ["TRTEngineOp_0"]
+
+  def ShouldRunTest(self, run_params):
+    return (run_params.dynamic_engine and
+            not trt_test.IsQuantizationMode(run_params.precision_mode))
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/tensorrt/test/manual_test.py b/tensorflow/contrib/tensorrt/test/manual_test.py
index 1187c759b4..aad7b9f307 100644
--- a/tensorflow/contrib/tensorrt/test/manual_test.py
+++ b/tensorflow/contrib/tensorrt/test/manual_test.py
@@ -67,9 +67,9 @@ class ManualTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=gdef,
         input_names=params_map['input_names'],
-        input_dims=params_map['input_dims'],
+        input_dims=[params_map['input_dims']],
         output_names=params_map['output_names'],
-        expected_output_dims=params_map['expected_output_dims'])
+        expected_output_dims=[params_map['expected_output_dims']])
 
   def GetConversionParams(self, run_params):
     """Return a ConversionParams for test."""
diff --git a/tensorflow/contrib/tensorrt/test/memory_alignment_test.py b/tensorflow/contrib/tensorrt/test/memory_alignment_test.py
index 104bac43a0..6142e6d8e3 100644
--- a/tensorflow/contrib/tensorrt/test/memory_alignment_test.py
+++ b/tensorflow/contrib/tensorrt/test/memory_alignment_test.py
@@ -62,9 +62,9 @@ class MemoryAlignmentTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[(2, 15, 15, 10)])
+        expected_output_dims=[[(2, 15, 15, 10)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py b/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
index 293f93d8a7..fcf89d3a74 100644
--- a/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
+++ b/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
@@ -75,9 +75,9 @@ class MultiConnectionNeighborEngineTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[(2, 4, 5, 4)])
+        expected_output_dims=[[(2, 4, 5, 4)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py b/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py
index 3e1e4b088b..840344aeba 100644
--- a/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py
+++ b/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py
@@ -59,9 +59,9 @@ class NeighboringEngineTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[(2, 4, 5, 4)])
+        expected_output_dims=[[(2, 4, 5, 4)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/quantization_test.py b/tensorflow/contrib/tensorrt/test/quantization_test.py
index e425a36746..fb09980552 100644
--- a/tensorflow/contrib/tensorrt/test/quantization_test.py
+++ b/tensorflow/contrib/tensorrt/test/quantization_test.py
@@ -60,9 +60,9 @@ def _GetParams(add_quantization_nodes, dtype=dtypes.float32):
   return trt_test.TfTrtIntegrationTestParams(
       gdef=g.as_graph_def(),
       input_names=[input_name],
-      input_dims=[input_dims],
+      input_dims=[[input_dims]],
       output_names=[output_name],
-      expected_output_dims=[(8, 1)])
+      expected_output_dims=[[(8, 1)]])
 
 
 class QuantizationMissingAllRangesTest(trt_test.TfTrtIntegrationTestBase):
diff --git a/tensorflow/contrib/tensorrt/test/rank_two_test.py b/tensorflow/contrib/tensorrt/test/rank_two_test.py
index dd3b72d212..85ce89e151 100644
--- a/tensorflow/contrib/tensorrt/test/rank_two_test.py
+++ b/tensorflow/contrib/tensorrt/test/rank_two_test.py
@@ -63,9 +63,9 @@ class RankTwoTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=input_names,
-        input_dims=input_dims,
+        input_dims=[input_dims],
         output_names=[output_name],
-        expected_output_dims=[tuple(input_dims[1])])
+        expected_output_dims=[[tuple(input_dims[1])]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py b/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py
index 207944468a..530c4992db 100644
--- a/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py
+++ b/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py
@@ -72,9 +72,9 @@ class ReshapeTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[tuple(input_dims)])
+        expected_output_dims=[[tuple(input_dims)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -129,9 +129,9 @@ class TransposeTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[(24, 100, 2, 24)])
+        expected_output_dims=[[(24, 100, 2, 24)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
index 671abba6a6..a105981298 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
@@ -161,7 +161,7 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
     """Return a ConversionParams for test."""
     return ConversionParams(
         max_batch_size=max([
-            dims[0] for dims in self._GetParamsCached().input_dims if len(dims)
+            dims[0] for inp in self._GetParamsCached().input_dims if len(inp) for dims in inp
         ]),
         max_workspace_size_bytes=1 << 25,
         precision_mode=run_params.precision_mode,
@@ -283,13 +283,16 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
   def _RunGraph(self,
                 run_params,
                 gdef,
-                input_data,
+                inputs_data,
                 config,
                 graph_state,
                 num_runs=2):
     """Run given graphdef multiple times."""
     params = self._GetParamsCached()
-    assert len(params.input_names) == len(input_data)
+    for current_input_data in inputs_data:
+      assert len(params.input_names) == len(current_input_data)
+
+    vals = []
     g = ops.Graph()
     with g.as_default():
       io_ops = importer.import_graph_def(
@@ -297,32 +300,38 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
           return_elements=params.input_names + params.output_names,
           name="")
       inputs = [op.outputs[0] for op in io_ops[:len(params.input_names)]]
-      assert len(inputs) == len(input_data)
+      for current_input_data in inputs_data:
+        assert len(inputs) == len(current_input_data)
       outputs = [op.outputs[0] for op in io_ops[len(params.input_names):]]
-    with self.test_session(
-        graph=g, config=config, use_gpu=True, force_gpu=True) as sess:
-      val = None
-      # Defaults to 2 runs to verify result across multiple runs is same.
-      for _ in range(num_runs):
-        self._PrepareRun(graph_state)
-        new_val = sess.run(
-            outputs, {inputs[i]: input_data[i] for i in range(len(inputs))})
-        output_len = len(params.expected_output_dims)
-        self.assertEqual(output_len, len(new_val))
-        for i in range(output_len):
-          self.assertEqual(params.expected_output_dims[i], new_val[i].shape)
-        if val is not None:
-          self.assertAllClose(val, new_val, atol=1.e-06, rtol=1.e-06)
-        val = new_val
-        self.VerifyRun(run_params, graph_state)
-    return val
+      with self.test_session(
+          graph=g, config=config, use_gpu=True, force_gpu=True) as sess:
+        # Run for each input(s) shape
+        for ind in range(len(inputs_data)):
+          val = None
+          # Defaults to 2 runs to verify result across multiple runs is same.
+          for _ in range(num_runs):
+            self._PrepareRun(graph_state)
+            new_val = sess.run(
+                outputs,
+                {inputs[i]: inputs_data[ind][i] for i in range(len(inputs))})
+            output_len = len(params.expected_output_dims[ind])
+            self.assertEqual(output_len, len(new_val))
+            for i in range(output_len):
+              self.assertEqual(list(params.expected_output_dims[ind][i]),
+                               list(new_val[i].shape))
+            if val is not None:
+              self.assertAllClose(val, new_val, atol=1.e-06, rtol=1.e-06)
+            val = new_val
+            self.VerifyRun(run_params, graph_state)
+          vals.append(val)
+    return vals
 
   # Use real data that is representative of the inference dataset
   # for calibration. For this test script it is random data.
-  def _RunCalibration(self, run_params, gdef, input_data, config):
+  def _RunCalibration(self, run_params, gdef, inputs_data, config):
     """Run calibration on given graph."""
     return self._RunGraph(
-        run_params, gdef, input_data, config, GraphState.CALIBRATE, num_runs=5)
+        run_params, gdef, inputs_data, config, GraphState.CALIBRATE, num_runs=5)
 
   def _GetTrtGraphDef(self, run_params, graph_state, gdef):
     """Return trt converted graphdef."""
@@ -474,26 +483,30 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
             dtypes.as_dtype(node.attr["dtype"].type).as_numpy_dtype())
     assert len(params.input_names) == len(input_dtypes)
 
-    input_data = []
-    for i in range(len(params.input_names)):
-      dtype = input_dtypes[params.input_names[i]]
-      # Multiply the input by some constant to avoid all zeros input for integer
-      # types.
-      scale = 10.0 if np.issubdtype(dtype, np.integer) else 1.0
-      dims = params.input_dims[i]
-      # TODO(laigd): add debug options. E.g. we can set the input data to be
-      # continuous natural numbers:
-      # seq = np.arange(np.prod(dims))
-      # seq.resize(dims)
-      # input_data.append(scale * seq.astype(dtype))
-      input_data.append((scale * np.random.random_sample(dims)).astype(dtype))
+    inputs_data = []
+    for inp in params.input_dims:
+      current_input_data = []
+      for i in range(len(params.input_names)):
+        dtype = input_dtypes[params.input_names[i]]
+        # Multiply the input by some constant to avoid all zeros input for integer
+        # types.
+        scale = 10.0 if np.issubdtype(dtype, np.integer) else 1.0
+        dims = inp[i]
+        # TODO(laigd): add debug options. E.g. we can set the input data to be
+        # continuous natural numbers:
+        # seq = np.arange(np.prod(dims))
+        # seq.resize(dims)
+        # input_data.append(scale * seq.astype(dtype))
+        current_input_data.append((scale * np.random.random_sample(dims)).astype(dtype))
+      inputs_data.append(current_input_data)
+
     self._VerifyGraphDef(run_params, input_gdef, GraphState.ORIGINAL)
 
     # Get reference result without running trt.
     config_no_trt = self._GetConfigProto(run_params, GraphState.ORIGINAL)
     logging.info("Running original graph w/o trt, config:\n%s",
                  str(config_no_trt))
-    ref_result = self._RunGraph(run_params, input_gdef, input_data,
+    ref_result = self._RunGraph(run_params, input_gdef, inputs_data,
                                 config_no_trt, GraphState.ORIGINAL)
 
     # Run calibration if necessary.
@@ -503,13 +516,13 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
       calib_config = self._GetConfigProto(run_params, GraphState.CALIBRATE)
       logging.info("Running calibration graph, config:\n%s", str(calib_config))
       if run_params.use_optimizer:
-        result = self._RunCalibration(run_params, input_gdef, input_data,
+        result = self._RunCalibration(run_params, input_gdef, inputs_data,
                                       calib_config)
       else:
         calib_gdef = self._GetTrtGraphDef(run_params, GraphState.CALIBRATE,
                                           input_gdef)
         self._VerifyGraphDef(run_params, calib_gdef, GraphState.CALIBRATE)
-        result = self._RunCalibration(run_params, calib_gdef, input_data,
+        result = self._RunCalibration(run_params, calib_gdef, inputs_data,
                                       calib_config)
       infer_gdef = trt_convert.calib_graph_to_infer_graph(
           calib_gdef, run_params.dynamic_engine)
@@ -532,7 +545,7 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
                                         infer_gdef)
       self._VerifyGraphDef(run_params, infer_gdef, GraphState.INFERENCE)
 
-    result = self._RunGraph(run_params, infer_gdef, input_data, infer_config,
+    result = self._RunGraph(run_params, infer_gdef, inputs_data, infer_config,
                             GraphState.INFERENCE)
     self.assertAllClose(
         ref_result,
diff --git a/tensorflow/contrib/tensorrt/test/unary_test.py b/tensorflow/contrib/tensorrt/test/unary_test.py
index b6e5e32db1..f630d695cf 100644
--- a/tensorflow/contrib/tensorrt/test/unary_test.py
+++ b/tensorflow/contrib/tensorrt/test/unary_test.py
@@ -100,9 +100,9 @@ class UnaryTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name, input2_name],
-        input_dims=[input_dims, input2_dims],
+        input_dims=[[input_dims, input2_dims]],
         output_names=[output_name],
-        expected_output_dims=[(12, 5, 8, 12)])
+        expected_output_dims=[[(12, 5, 8, 12)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py b/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py
index b29626d2c2..1a4746e8d0 100644
--- a/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py
+++ b/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py
@@ -70,9 +70,9 @@ class VGGBlockNCHWTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[(5, 6, 2, 2)])
+        expected_output_dims=[[(5, 6, 2, 2)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/vgg_block_test.py b/tensorflow/contrib/tensorrt/test/vgg_block_test.py
index 9b0b189626..051ced055a 100644
--- a/tensorflow/contrib/tensorrt/test/vgg_block_test.py
+++ b/tensorflow/contrib/tensorrt/test/vgg_block_test.py
@@ -61,9 +61,9 @@ class VGGBlockTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[(5, 2, 2, 6)])
+        expected_output_dims=[[(5, 2, 2, 6)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
-- 
GitLab


From 48891a7ecc94058e9dc18498dd3d0b2a46b75abe Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Mon, 14 Jan 2019 14:07:01 -0800
Subject: [PATCH 0684/2345] Make DebugString() const

---
 tensorflow/contrib/tensorrt/resources/trt_lru_cache.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h b/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
index 4d63a03b40..02407d023a 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
@@ -140,7 +140,7 @@ class TRTEngineCacheResource : public tensorflow::ResourceBase {
     }
   }
 
-  string DebugString() override {
+  string DebugString() const override {
     std::stringstream oss;
     using std::dec;
     using std::endl;
-- 
GitLab


From 9887a0312b59d288773d0a806c335fd46ee7d6c9 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Mon, 14 Jan 2019 14:23:55 -0800
Subject: [PATCH 0685/2345] Fix clang-format

---
 tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc  | 11 +++++------
 tensorflow/contrib/tensorrt/resources/trt_lru_cache.h |  3 ++-
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
index e1e64fec68..06a62fed37 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
@@ -297,7 +297,6 @@ void TRTEngineOp::ComputeAsync(OpKernelContext* ctx,
 
 bool TRTEngineOp::ExecuteTrtEngine(OpKernelContext* ctx,
                                    EngineContext* engine_context) {
-  
   VLOG(1) << "Executing TRT engine: " << name();
   auto& cuda_engine = engine_context->cuda_engine;
   const bool kRetry = true;
@@ -468,11 +467,11 @@ EngineContext* TRTEngineOp::GetEngine(
       engine_input_shapes[i].set_dim(0, max_batch_size);
     }
     // Assume engine_input_shapes matches the input shapes of the engine.
-    cache.emplace(
-        engine_input_shapes, absl::make_unique<EngineContext>(
-            std::move(static_engine),
-            TrtUniquePtrType<nvinfer1::IExecutionContext>(
-                raw_static_engine->createExecutionContext())));
+    cache.emplace(engine_input_shapes,
+                  absl::make_unique<EngineContext>(
+                      std::move(static_engine),
+                      TrtUniquePtrType<nvinfer1::IExecutionContext>(
+                          raw_static_engine->createExecutionContext())));
     // Runtime is safe to delete after engine creation
     serialized_segment_.clear();
     if (max_batch_size < batch_size) {
diff --git a/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h b/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
index 02407d023a..57267e5b71 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
@@ -162,7 +162,8 @@ class TRTEngineCacheResource : public tensorflow::ResourceBase {
   std::unique_ptr<TRTBaseAllocator> allocator_;
 
   // Declare cache after allocator so that it is destroyed before allocator is.
-  LRUCache<std::vector<TensorShape>, std::unique_ptr<EngineContext>, VectorTensorShapeHasher>
+  LRUCache<std::vector<TensorShape>, std::unique_ptr<EngineContext>,
+           VectorTensorShapeHasher>
       cache_;
 };
 
-- 
GitLab


From 2cfc9e6ca77544e90090c45c498d024ffd79fc84 Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Mon, 14 Jan 2019 15:40:37 -0800
Subject: [PATCH 0686/2345] Disable InterpreterFlexTest

This test currently only works with --config=monolithic. Disable for
now until the select .so supports execution against the shared TF lib.

PiperOrigin-RevId: 229269278
---
 tensorflow/lite/java/BUILD | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/lite/java/BUILD b/tensorflow/lite/java/BUILD
index a9d4a422c5..bd9d8f4806 100644
--- a/tensorflow/lite/java/BUILD
+++ b/tensorflow/lite/java/BUILD
@@ -163,6 +163,9 @@ java_test(
         "//tensorflow/lite:testdata/multi_add_flex.bin",
     ],
     javacopts = JAVACOPTS,
+    tags = [
+        "no_oss",  # Currently requires --config=monolithic, b/118895218.
+    ],
     test_class = "org.tensorflow.lite.InterpreterFlexTest",
     visibility = ["//visibility:private"],
     deps = [
-- 
GitLab


From a3a6bd35ffde82f0d6ff00b004497a5b2f97e6b8 Mon Sep 17 00:00:00 2001
From: Katherine Wu <kathywu@google.com>
Date: Mon, 14 Jan 2019 15:59:05 -0800
Subject: [PATCH 0687/2345] Fix bug where TensorSpecs are replaced with
 UnknownArgument, which caused issues with f.get_concrete_function.

PiperOrigin-RevId: 229273199
---
 tensorflow/python/eager/function.py        |  2 +-
 tensorflow/python/saved_model/load_test.py | 17 +++++++++++++++--
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 6429022df3..b5bda35916 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -844,7 +844,7 @@ def convert_structure_to_signature(structure):
     """A representation for this argument, for converting into signatures."""
     if isinstance(arg, ops.Tensor):
       return tensor_spec.TensorSpec(arg.shape, arg.dtype, name)
-    if isinstance(arg, (int, float, bool)):
+    if isinstance(arg, (int, float, bool, tensor_spec.TensorSpec)):
       return arg
     return UnknownArgument()
 
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index e94cda533e..ca9e16d14a 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -37,9 +37,9 @@ from tensorflow.python.training.checkpointable import tracking
 
 class LoadTest(test.TestCase):
 
-  def cycle(self, obj):
+  def cycle(self, obj, signatures=None):
     path = tempfile.mkdtemp(prefix=self.get_temp_dir())
-    save.save(obj, path, signatures={})
+    save.save(obj, path, signatures=signatures or {})
     return load.load(path)
 
   def test_structure_import(self):
@@ -152,6 +152,19 @@ class LoadTest(test.TestCase):
     imported = self.cycle(root)
     self.assertEqual(4., imported.f(constant_op.constant(2.0)).numpy())
 
+  def test_explicit_save_signature(self):
+    @def_function.function
+    def func(x):
+      return 2 * x
+
+    root = tracking.Checkpointable()
+    root.f = func
+
+    imported = self.cycle(
+        root, {"f": root.f.get_concrete_function(
+            tensor_spec.TensorSpec(None, dtypes.float32))})
+    self.assertEqual(4., imported.f(constant_op.constant(2.0)).numpy())
+
   def test_nested_functions(self):
     f = def_function.function(
         lambda x: x*2.0,
-- 
GitLab


From ca4c61a35b36da49e5dd2aeebc33b99587f243ba Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Mon, 14 Jan 2019 16:09:17 -0800
Subject: [PATCH 0688/2345] [Grappler] Relax no-inline constraint for functions
 with Switch/Merge control flow.

Do not inline functions only if they can produce dead outputs.

PiperOrigin-RevId: 229275596
---
 tensorflow/core/grappler/optimizers/BUILD     |   2 +
 .../grappler/optimizers/function_optimizer.cc | 106 ++++++++++--
 .../optimizers/function_optimizer_test.cc     | 157 ++++++++++++++++++
 3 files changed, 251 insertions(+), 14 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index a67d6ac5ea..ee216f80f9 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -145,11 +145,13 @@ cc_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core/grappler:graph_topology_view",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/utils:functions",
+        "//tensorflow/core/grappler/utils:traversal",
         "@com_google_absl//absl/algorithm:container",
         "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/container:flat_hash_set",
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index e9c30fee25..8095d6f31d 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -39,11 +39,13 @@ limitations under the License.
 #include "tensorflow/core/framework/versions.pb.h"
 #include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/graph/tensor_id.h"
+#include "tensorflow/core/grappler/graph_topology_view.h"
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/mutable_graph_view.h"
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/grappler/utils.h"
 #include "tensorflow/core/grappler/utils/functions.h"
+#include "tensorflow/core/grappler/utils/traversal.h"
 #include "tensorflow/core/lib/gtl/map_util.h"
 
 namespace tensorflow {
@@ -264,6 +266,8 @@ class FunctionOptimizerContext {
     InitializeFetchNodes(item);
   }
 
+  int graph_version() const { return graph_version_; }
+
   const RewriterConfig::Toggle opt_level() const { return opt_level_; }
 
   const GrapplerItem::AllowedOptimizations& allowed_optimizations() const {
@@ -1259,6 +1263,74 @@ Status InlineSymbolicGradient(const NodeDef& node,
 // because they are pure functions of input tensors, and can be freely
 // reordered.
 
+struct MaybeDeadOutput {
+  const NodeDef* dead_tensor_src;
+  const NodeDef* output_node_dst;
+};
+
+// Finds all function outputs that might return a dead tensor. This can happen
+// if there is no `Merge` node on the path from the `Switch` node, to the
+// function output.
+Status MaybeDeadOutputs(const FunctionOptimizerContext& ctx,
+                        const GrapplerFunctionItem& item,
+                        std::vector<MaybeDeadOutput>* maybe_dead) {
+  DCHECK(maybe_dead->empty()) << "Input argument must be an empty vector";
+
+  std::vector<const NodeDef*> dead_tensor_srcs;
+  for (const NodeDef& node : item.graph.node()) {
+    if (IsSwitch(node)) {
+      dead_tensor_srcs.push_back(&node);
+      continue;
+    }
+
+    // Regular (aka 'direct') function call can also produce dead tensors if
+    // the function body has mergeless switches.
+    const FunctionDef* func = ctx.function_library().Find(node.op());
+    if (func != nullptr) {
+      GrapplerFunctionItem func_item;
+      TF_RETURN_IF_ERROR(MakeGrapplerFunctionItem(
+          *func, FunctionInstantiationAttributes(*func, node),
+          ctx.function_library(), ctx.graph_version(), &func_item));
+
+      std::vector<MaybeDeadOutput> func_dead_outputs;
+      TF_RETURN_IF_ERROR(MaybeDeadOutputs(ctx, func_item, &func_dead_outputs));
+
+      if (!func_dead_outputs.empty()) dead_tensor_srcs.push_back(&node);
+    }
+  }
+
+  // If we do not have dead tensor sources in the function body, it's
+  // guaranteed that all output tensors can't become dead.
+  if (dead_tensor_srcs.empty()) return Status::OK();
+
+  // Names of the function body nodes that return function output values.
+  absl::flat_hash_set<absl::string_view> output_nodes;
+  for (const auto& output_expansion : item.outputs()) {
+    for (const auto& output_node : output_expansion.output_nodes) {
+      output_nodes.insert(output_node);
+    }
+  }
+
+  GraphTopologyView topology_view;
+  TF_RETURN_IF_ERROR(topology_view.InitializeFromGraph(item.graph));
+
+  for (const NodeDef* dead_tensor_src : dead_tensor_srcs) {
+    DfsTraversal(topology_view, {dead_tensor_src},
+                 TraversalDirection::kFollowOutputs,
+                 // Stop traversal when reached first `Merge` node.
+                 DfsPredicates::Advance(
+                     [](const NodeDef* node) { return !IsMerge(*node); }),
+                 // If we reached output node, add MaybeDeadOutput edge.
+                 DfsCallbacks::PreOrder([&](const NodeDef* node) {
+                   if (output_nodes.find(node->name()) != output_nodes.end()) {
+                     maybe_dead->push_back({dead_tensor_src, node});
+                   }
+                 }));
+  }
+
+  return Status::OK();
+}
+
 // Returns `Status::OK()` iff `node` is an indirect function call of `func`, and
 // we know how to inline it into the main graph, otherwise returns and error
 // indicating why the function call is not inlinable.
@@ -1293,20 +1365,6 @@ Status IsInlinableIndirectFunctionCall(const FunctionOptimizerContext& ctx,
         SummarizeNodeDef(func_node));
   }
 
-  // We can't inline functions with `Switch` nodes in the function body, because
-  // they might have dead tensors as a function output argument (we need all
-  // intermediate tensors to compute the function gradient). `PartitionedCallOp`
-  // invokes functions with `allow_dead_tensors = true` to reset dead flag,
-  // and return default initialized tensors instead of a dead tensors.
-  // TODO(ezhulenev): Do the liveness analysis and add
-  // `IdentitytWithResurrection` nodes after all potentially dead output
-  // tensors?
-  if (absl::c_any_of(func.node_def(), IsSwitch)) {
-    return errors::FailedPrecondition(
-        "Can't inline function with `Switch` nodes in the function body: ",
-        SummarizeNodeDef(func_node));
-  }
-
   // TODO(b/120991525, b/120986912): We need to lower `If` and `While` nodes to
   // `Switch` nodes after function inlining (one more PRE_PLACEMENT pass?), but
   // because of the reason described above we are not sure that it's safe, for
@@ -1346,6 +1404,26 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
                                    ". Error: ", item_status.error_message());
   }
 
+  // `PartitionedCallOp` invokes functions with `allow_dead_tensors = true` to
+  // reset dead flag, and return default initialized tensors instead of a dead
+  // tensors. There is no way to express this in a regular Tensorflow graph, so
+  // we choose not to inline if a function can have dead tensors as an output
+  // position. In practice `mergeless switches` should not exists in a function
+  // body, because tf-eager will only use v2 control flow ops.
+  std::vector<MaybeDeadOutput> maybe_dead_outputs;
+  TF_RETURN_IF_ERROR(MaybeDeadOutputs(*ctx, item, &maybe_dead_outputs));
+  if (!maybe_dead_outputs.empty()) {
+    struct MaybeDeadOutputFormatter {
+      void operator()(string* out, const MaybeDeadOutput& md) const {
+        absl::StrAppend(out, SummarizeNodeDef(*md.dead_tensor_src));
+      }
+    };
+    return errors::FailedPrecondition(
+        "Can't inline function with dead outputs. Dead tensor sources (size = ",
+        maybe_dead_outputs.size(), "): ",
+        absl::StrJoin(maybe_dead_outputs, "\n", MaybeDeadOutputFormatter()));
+  }
+
   GraphView::InputPort control_input_port =
       ctx->graph_view().GetInputPort(func_node.name(), Graph::kControlSlot);
   GraphView::OutputPort control_output_port =
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
index cebd002bed..827b0658c5 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
@@ -1025,6 +1025,163 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithoutSideEffects) {
   test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
 }
 
+TEST_F(FunctionOptimizerTest, InlineIndirectFunctionDoNotInlineDeadOutputs) {
+  using test::function::NDef;
+  using FDH = FunctionDefHelper;
+
+  FunctionOptimizer optimizer(RewriterConfig::AGGRESSIVE);
+
+  // Function output can be dead.
+  FunctionDef dead_outputs = FunctionDefHelper::Create(
+      "DeadOutputs", {"x:T", "cond:bool"}, {"z:T"}, {"T: {float, double}"},
+      {
+          {{"switch"}, "Switch", {"x", "cond"}, {{"T", "$T"}}},
+          {{"if_false"}, "Identity", {"switch:output_false:0"}, {{"T", "$T"}}},
+          {{"if_true"}, "Identity", {"switch:output_true:0"}, {{"T", "$T"}}},
+      },
+      /* Mapping between function returns and function node outputs. */
+      {{"z", "if_false:output:0"}});
+
+  // Simple proxy functions that calls DeadOutputs from the function body.
+  FunctionDef proxy_func = FunctionDefHelper::Create(
+      "Proxy", {"x:T", "cond:bool"}, {"z:T"}, {"T: {float, double}"},
+      {{{"dead"}, "DeadOutputs", {"x", "cond"}, {{"T", "$T"}}}},
+      /* Mapping between function returns and function node outputs. */
+      {{"z", "dead:z:0"}});
+
+  // Build a graph to compute:
+  //   a: float
+  //   b: bool
+  //   fn0 = DeadOutputs(x, b)
+  //   fn1 = Proxy(x, b)
+  //   out0 = Identity(fn0)
+  //   out1 = Identity(fn1)
+  //   return [out0, out1]
+  //
+  GrapplerItem item;
+  item.fetch = {"out0", "out1"};
+  item.graph = test::function::GDef(
+      {NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("b", "Placeholder", {}, {{"dtype", DT_BOOL}}, kDevice),
+
+       NDef("fn0", "PartitionedCall", {"a", "b"},
+            {{"Tin", DataTypeSlice{DT_FLOAT, DT_BOOL}},
+             {"Tout", DataTypeSlice{DT_FLOAT}},
+             {"f", FDH::FunctionRef("DeadOutputs", {{"T", DT_FLOAT}})}},
+            kDevice),
+
+       NDef("fn1", "PartitionedCall", {"a", "b"},
+            {{"Tin", DataTypeSlice{DT_FLOAT, DT_BOOL}},
+             {"Tout", DataTypeSlice{DT_FLOAT}},
+             {"f", FDH::FunctionRef("Proxy", {{"T", DT_FLOAT}})}},
+            kDevice),
+
+       NDef("out0", "Identity", {"fn0"}, {{"T", DT_FLOAT}}, kDevice),
+       NDef("out1", "Identity", {"fn1"}, {{"T", DT_FLOAT}}, kDevice)},
+      // Function library.
+      {dead_outputs, proxy_func});
+
+  GraphDef optimized_graph;
+  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &optimized_graph));
+
+  GraphDef expected = item.graph;
+  CompareGraphs(expected, optimized_graph);
+
+  const Tensor one = test::AsScalar<float>(1.0);
+  item.feed.emplace_back("a", one);
+  item.feed.emplace_back("b", test::AsScalar<bool>(false));
+
+  auto tensors = EvaluateFetchNodes(item);
+  ASSERT_EQ(tensors.size(), 2);
+  test::ExpectTensorEqual<float>(tensors[0], one);
+  test::ExpectTensorEqual<float>(tensors[1], one);
+}
+
+TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithMergedDeadTensors) {
+  using test::function::NDef;
+  using FDH = FunctionDefHelper;
+
+  FunctionOptimizer optimizer(RewriterConfig::AGGRESSIVE);
+
+  // Function output can't be dead because it goes through the Merge node.
+  FunctionDef no_dead_outputs = FunctionDefHelper::Create(
+      "NoDeadOutputs", {"x:T", "cond:bool"}, {"z:T"}, {"T: {float, double}"},
+      {
+          {{"switch"}, "Switch", {"x", "cond"}, {{"T", "$T"}}},
+          {{"if_false"}, "Identity", {"switch:output_false:0"}, {{"T", "$T"}}},
+          {{"if_true"}, "Identity", {"switch:output_true:0"}, {{"T", "$T"}}},
+          {{"merge"},
+           "Merge",
+           {"if_false:output:0", "if_true:output:0"},
+           {{"T", "$T"}, {"N", 2}}},
+      },
+      /* Mapping between function returns and function node outputs. */
+      {{"z", "merge:output:0"}});
+
+  // Build a graph to compute:
+  //   a: float
+  //   b: bool
+  //   d = DeadOutputs(x, b)
+  //   out = Identity(d)
+  //   return out
+  //
+  GrapplerItem item;
+  item.fetch = {"out"};
+  item.graph = test::function::GDef(
+      {NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("b", "Placeholder", {}, {{"dtype", DT_BOOL}}, kDevice),
+
+       NDef("fn", "PartitionedCall", {"a", "b"},
+            {{"Tin", DataTypeSlice{DT_FLOAT, DT_BOOL}},
+             {"Tout", DataTypeSlice{DT_FLOAT}},
+             {"f", FDH::FunctionRef("NoDeadOutputs", {{"T", DT_FLOAT}})}},
+            kDevice),
+
+       NDef("out", "Identity", {"fn"}, {{"T", DT_FLOAT}}, kDevice)},
+      // Function library.
+      {no_dead_outputs});
+
+  GraphDef optimized_graph;
+  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &optimized_graph));
+
+  GraphDef expected = test::function::GDef(
+      {NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("b", "Placeholder", {}, {{"dtype", DT_BOOL}}, kDevice),
+
+       // Function body of a first function call inlined into the graph.
+       NDef("fn/x", "Identity", {"a:0"}, {{"T", DT_FLOAT}}, kDevice),
+       NDef("fn/cond", "Identity", {"b:0"}, {{"T", DT_BOOL}}, kDevice),
+       NDef("fn/switch", "Switch", {"fn/x:0", "fn/cond:0"}, {{"T", DT_FLOAT}},
+            kDevice),
+       NDef("fn/if_false", "Identity", {"fn/switch:0"}, {{"T", DT_FLOAT}},
+            kDevice),
+       NDef("fn/if_true", "Identity", {"fn/switch:1"}, {{"T", DT_FLOAT}},
+            kDevice),
+       NDef("fn/merge", "Merge", {"fn/if_false:0", "fn/if_true:0"},
+            {{"T", DT_FLOAT}, {"N", 2}}, kDevice),
+
+       // Return directly from inlined node.
+       NDef("out", "Identity", {"fn/merge:0"}, {{"T", DT_FLOAT}}, kDevice)},
+
+      // Function library.
+      {no_dead_outputs});
+
+  CompareGraphs(expected, optimized_graph);
+
+  const Tensor one = test::AsScalar<float>(1.0);
+  item.feed.emplace_back("a", one);
+  item.feed.emplace_back("b", test::AsScalar<bool>(false));
+
+  auto tensors_expected = EvaluateFetchNodes(item);
+  ASSERT_EQ(tensors_expected.size(), 1);
+
+  GrapplerItem optimized = item.WithGraph(std::move(optimized_graph));
+  auto tensors = EvaluateFetchNodes(optimized);
+  ASSERT_EQ(tensors.size(), 1);
+
+  test::ExpectTensorEqual<float>(tensors[0], tensors_expected[0]);
+}
+
 TEST_F(FunctionOptimizerTest, SpecializeFunctionXTimesTwo) {
   using test::function::NDef;
 
-- 
GitLab


From 44c657a5323eafc35e01639506b41732ee363698 Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Mon, 14 Jan 2019 16:12:24 -0800
Subject: [PATCH 0689/2345] Automated rollback of commit
 c45a87f7c7080eecf42c430c66adef0e9249f5ac

PiperOrigin-RevId: 229276115
---
 tensorflow/lite/testing/generated_examples_zip_test.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/lite/testing/generated_examples_zip_test.cc b/tensorflow/lite/testing/generated_examples_zip_test.cc
index dfd8eb059a..bade6590a4 100644
--- a/tensorflow/lite/testing/generated_examples_zip_test.cc
+++ b/tensorflow/lite/testing/generated_examples_zip_test.cc
@@ -287,10 +287,10 @@ struct ZipPathParamName {
   }
 };
 
-INSTANTIATE_TEST_SUITE_P(tests, OpsTest,
-                         ::testing::ValuesIn(UnarchiveAndFindTestNames(
-                             *FLAGS_zip_file_path, *FLAGS_tar_file_path)),
-                         ZipPathParamName());
+INSTANTIATE_TEST_CASE_P(tests, OpsTest,
+                        ::testing::ValuesIn(UnarchiveAndFindTestNames(
+                            *FLAGS_zip_file_path, *FLAGS_tar_file_path)),
+                        ZipPathParamName());
 
 }  // namespace testing
 }  // namespace tflite
-- 
GitLab


From b30d3f5ac037c2d84f99130d1e7547d4dc6bbd5a Mon Sep 17 00:00:00 2001
From: Nick Felt <nickfelt@google.com>
Date: Mon, 14 Jan 2019 16:18:48 -0800
Subject: [PATCH 0690/2345] Expose tf.audio.encode_wav

PiperOrigin-RevId: 229277218
---
 .../core/api_def/python_api/api_def_AudioSpectrogram.pbtxt | 4 ++++
 tensorflow/core/api_def/python_api/api_def_DecodeWav.pbtxt | 4 ++++
 tensorflow/core/api_def/python_api/api_def_EncodeWav.pbtxt | 6 ++++++
 tensorflow/core/api_def/python_api/api_def_Mfcc.pbtxt      | 4 ++++
 tensorflow/python/BUILD                                    | 1 +
 tensorflow/python/__init__.py                              | 3 +++
 tensorflow/python/tools/api/generator/api_init_files.bzl   | 1 +
 .../python/tools/api/generator/api_init_files_v1.bzl       | 1 +
 tensorflow/tools/api/golden/v1/tensorflow.audio.pbtxt      | 7 +++++++
 tensorflow/tools/api/golden/v1/tensorflow.pbtxt            | 4 ++++
 tensorflow/tools/api/golden/v2/tensorflow.audio.pbtxt      | 7 +++++++
 tensorflow/tools/api/golden/v2/tensorflow.pbtxt            | 4 ++++
 12 files changed, 46 insertions(+)
 create mode 100644 tensorflow/core/api_def/python_api/api_def_AudioSpectrogram.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_DecodeWav.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_EncodeWav.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_Mfcc.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.audio.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.audio.pbtxt

diff --git a/tensorflow/core/api_def/python_api/api_def_AudioSpectrogram.pbtxt b/tensorflow/core/api_def/python_api/api_def_AudioSpectrogram.pbtxt
new file mode 100644
index 0000000000..38e4b38f3f
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_AudioSpectrogram.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "AudioSpectrogram"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_DecodeWav.pbtxt b/tensorflow/core/api_def/python_api/api_def_DecodeWav.pbtxt
new file mode 100644
index 0000000000..d6fd4691f7
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_DecodeWav.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "DecodeWav"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_EncodeWav.pbtxt b/tensorflow/core/api_def/python_api/api_def_EncodeWav.pbtxt
new file mode 100644
index 0000000000..edcf2bded1
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_EncodeWav.pbtxt
@@ -0,0 +1,6 @@
+op {
+  graph_op_name: "EncodeWav"
+  endpoint {
+    name: "audio.encode_wav"
+  }
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_Mfcc.pbtxt b/tensorflow/core/api_def/python_api/api_def_Mfcc.pbtxt
new file mode 100644
index 0000000000..21a0c8b8b2
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_Mfcc.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "Mfcc"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 905b47ea75..589605e347 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -112,6 +112,7 @@ py_library(
     ],
     deps = [
         ":array_ops",
+        ":audio_ops_gen",
         ":bitwise_ops",
         ":boosted_trees_ops",
         ":check_ops",
diff --git a/tensorflow/python/__init__.py b/tensorflow/python/__init__.py
index 31f9dce8e8..5172dbb836 100644
--- a/tensorflow/python/__init__.py
+++ b/tensorflow/python/__init__.py
@@ -99,6 +99,9 @@ from tensorflow.python.summary import summary
 from tensorflow.python.user_ops import user_ops
 from tensorflow.python.util import compat
 
+# Import audio ops to make sure the ops are registered.
+from tensorflow.python.ops import gen_audio_ops as _
+
 # Import boosted trees ops to make sure the ops are registered (but unused).
 from tensorflow.python.ops import gen_boosted_trees_ops as _gen_boosted_trees_ops
 
diff --git a/tensorflow/python/tools/api/generator/api_init_files.bzl b/tensorflow/python/tools/api/generator/api_init_files.bzl
index f496e90dfa..5182f53beb 100644
--- a/tensorflow/python/tools/api/generator/api_init_files.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files.bzl
@@ -4,6 +4,7 @@
 TENSORFLOW_API_INIT_FILES = [
     # BEGIN GENERATED FILES
     "__init__.py",
+    "audio/__init__.py",
     "autograph/__init__.py",
     "autograph/experimental/__init__.py",
     "bitwise/__init__.py",
diff --git a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
index 2d5898f31c..3e2c00c87f 100644
--- a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
@@ -5,6 +5,7 @@ TENSORFLOW_API_INIT_FILES_V1 = [
     # BEGIN GENERATED FILES
     "__init__.py",
     "app/__init__.py",
+    "audio/__init__.py",
     "autograph/__init__.py",
     "autograph/experimental/__init__.py",
     "bitwise/__init__.py",
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.audio.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.audio.pbtxt
new file mode 100644
index 0000000000..ce29615f72
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.audio.pbtxt
@@ -0,0 +1,7 @@
+path: "tensorflow.audio"
+tf_module {
+  member_method {
+    name: "encode_wav"
+    argspec: "args=[\'audio\', \'sample_rate\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
index deb1d100e7..016ae23890 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
@@ -292,6 +292,10 @@ tf_module {
     name: "app"
     mtype: "<type \'module\'>"
   }
+  member {
+    name: "audio"
+    mtype: "<type \'module\'>"
+  }
   member {
     name: "autograph"
     mtype: "<type \'module\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.audio.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.audio.pbtxt
new file mode 100644
index 0000000000..ce29615f72
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.audio.pbtxt
@@ -0,0 +1,7 @@
+path: "tensorflow.audio"
+tf_module {
+  member_method {
+    name: "encode_wav"
+    argspec: "args=[\'audio\', \'sample_rate\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 7e5f86d7e6..92f4704b49 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -68,6 +68,10 @@ tf_module {
     name: "VariableSynchronization"
     mtype: "<class \'enum.EnumMeta\'>"
   }
+  member {
+    name: "audio"
+    mtype: "<type \'module\'>"
+  }
   member {
     name: "autograph"
     mtype: "<type \'module\'>"
-- 
GitLab


From 681f6a6ac9344425549c8a38fcbfae854e7deddf Mon Sep 17 00:00:00 2001
From: Ayush Dubey <ayushd@google.com>
Date: Mon, 14 Jan 2019 16:19:38 -0800
Subject: [PATCH 0691/2345] Add a CollectiveReduce implementation that uses
 NCCL.

This change adds an implementation of the CollectiveImplementationInterface
that uses NCCL for all-reduce.  The main bits include setting up a NCCL
communicator key during collective param resolution, and calling the relevant
functionality in NcclManager.

PiperOrigin-RevId: 229277364
---
 tensorflow/core/BUILD                         |   2 +-
 .../base_collective_executor.cc               |  33 ++
 .../common_runtime/base_collective_executor.h |  18 +
 .../collective_param_resolver_local.cc        | 159 ++++++---
 .../collective_param_resolver_local.h         |  15 +-
 .../core/common_runtime/collective_util.cc    |  31 ++
 .../core/common_runtime/collective_util.h     |  21 ++
 .../hierarchical_tree_broadcaster.h           |   5 +
 .../core/common_runtime/ring_reducer.cc       |  43 +--
 tensorflow/core/common_runtime/ring_reducer.h |  24 +-
 tensorflow/core/distributed_runtime/BUILD     |   1 +
 .../collective_param_resolver_distributed.cc  |  21 +-
 ...lective_param_resolver_distributed_test.cc |   2 +
 tensorflow/core/framework/collective.cc       |  11 +
 tensorflow/core/framework/collective.h        |  32 +-
 tensorflow/core/kernels/BUILD                 |  24 ++
 .../core/kernels/collective_nccl_reducer.cc   | 204 +++++++++++
 .../core/kernels/collective_nccl_reducer.h    |  49 +++
 .../kernels/collective_nccl_reducer_test.cc   | 331 ++++++++++++++++++
 tensorflow/core/kernels/collective_ops.cc     |   3 +
 tensorflow/core/nccl/BUILD                    |   4 +-
 tensorflow/core/protobuf/worker.proto         |   1 +
 22 files changed, 916 insertions(+), 118 deletions(-)
 create mode 100644 tensorflow/core/kernels/collective_nccl_reducer.cc
 create mode 100644 tensorflow/core/kernels/collective_nccl_reducer.h
 create mode 100644 tensorflow/core/kernels/collective_nccl_reducer_test.cc

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index b655fdcbff..caf7b4cdbc 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -3713,7 +3713,6 @@ tf_cc_tests(
     srcs = [
         "common_runtime/buf_rendezvous_test.cc",
         "common_runtime/collective_executor_mgr_test.cc",
-        "common_runtime/collective_param_resolver_local_test.cc",
         "common_runtime/collective_rma_local_test.cc",
         "common_runtime/device_resolver_local_test.cc",
         "common_runtime/device_set_test.cc",
@@ -3829,6 +3828,7 @@ tf_cc_tests(
     name = "higher_level_tests_needing_kernels",
     size = "small",
     srcs = [
+        "common_runtime/collective_param_resolver_local_test.cc",
         "graph/graph_constructor_test.cc",
     ],
     linkopts = select({
diff --git a/tensorflow/core/common_runtime/base_collective_executor.cc b/tensorflow/core/common_runtime/base_collective_executor.cc
index 92e56df181..03c0e9ce2b 100644
--- a/tensorflow/core/common_runtime/base_collective_executor.cc
+++ b/tensorflow/core/common_runtime/base_collective_executor.cc
@@ -296,4 +296,37 @@ Status BaseCollectiveExecutor::CreateCollective(
   return status;
 }
 
+bool BaseCollectiveExecutor::CheckDependencies(
+    const CollectiveParams& col_params) {
+  for (int32 instance : col_params.instance.impl_details.dependencies) {
+    auto find_iter = launched_.find(instance);
+    if (find_iter == launched_.end() || find_iter->second != 0) {
+      return false;
+    }
+  }
+  return true;
+}
+
+void BaseCollectiveExecutor::WaitForDependencies(
+    const CollectiveParams& col_params) {
+  mutex_lock l(launch_mu_);
+  while (!CheckDependencies(col_params)) {
+    launch_cv_.wait(l);
+  }
+}
+
+void BaseCollectiveExecutor::Launched(const CollectiveParams& col_params) {
+  mutex_lock l(launch_mu_);
+  if (launched_.find(col_params.instance.instance_key) == launched_.end()) {
+    const string& task_name =
+        col_params.instance.task_names[col_params.default_rank];
+    const int32 num_devices =
+        col_params.instance.num_devices_per_task.at(task_name);
+    launched_[col_params.instance.instance_key] = num_devices;
+  }
+  if (--launched_[col_params.instance.instance_key] == 0) {
+    launch_cv_.notify_all();
+  }
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/base_collective_executor.h b/tensorflow/core/common_runtime/base_collective_executor.h
index 09826a8814..b711aa6d50 100644
--- a/tensorflow/core/common_runtime/base_collective_executor.h
+++ b/tensorflow/core/common_runtime/base_collective_executor.h
@@ -135,15 +135,33 @@ class BaseCollectiveExecutor : public CollectiveExecutor {
                                client_locality, done);
   }
 
+  // If we need to enforce an ordering on any portion of collective
+  // implementation, and the ordering is encoded via attribute on the collective
+  // op, this function will block until all dependencies for this collective
+  // have completed.
+  void WaitForDependencies(const CollectiveParams& col_params) override;
+  // Record that this collective has completed the portion of the implementation
+  // that needs to be ordered wrt other collectives, to unblock any of its
+  // dependent ops.
+  void Launched(const CollectiveParams& col_params) override;
+
  protected:
   const int64 step_id_;
   const DeviceMgr* dev_mgr_;  // Not owned.
   std::unique_ptr<PerStepCollectiveRemoteAccess> remote_access_;
   const string* gpu_ring_order_;  // Not owned.
+  mutex launch_mu_;
+  condition_variable launch_cv_;
+  // collective instance key -> number of local devices for which NCCL ops have
+  // been launched.
+  std::unordered_map<int32, int32> launched_ GUARDED_BY(launch_mu_);
 
  private:
   Status CreateCollective(const CollectiveParams& col_params,
                           CollectiveImplementationInterface** col_impl);
+  // Check if all ops on which this collective depends on have launched.
+  bool CheckDependencies(const CollectiveParams& col_params)
+      EXCLUSIVE_LOCKS_REQUIRED(launch_mu_);
 };
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/collective_param_resolver_local.cc b/tensorflow/core/common_runtime/collective_param_resolver_local.cc
index a8e3f4c881..8907f6d56a 100644
--- a/tensorflow/core/common_runtime/collective_param_resolver_local.cc
+++ b/tensorflow/core/common_runtime/collective_param_resolver_local.cc
@@ -26,6 +26,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/gtl/flatmap.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/types.h"
 #include "tensorflow/core/util/device_name_utils.h"
@@ -39,7 +40,10 @@ void CollectiveParamResolverLocal::InstanceRec::WaitForOutMu(mutex_lock& lock) {
 CollectiveParamResolverLocal::CollectiveParamResolverLocal(
     const DeviceMgr* dev_mgr, DeviceResolverInterface* dev_resolver,
     const string& task_name)
-    : dev_mgr_(dev_mgr), dev_resolver_(dev_resolver), task_name_(task_name) {}
+    : nccl_(false),  // (b/111897089): turn on NCCL collectives.
+      dev_mgr_(dev_mgr),
+      dev_resolver_(dev_resolver),
+      task_name_(task_name) {}
 
 void CollectiveParamResolverLocal::CompleteGroupAsync(
     const CompleteGroupRequest* request, CompleteGroupResponse* response,
@@ -316,29 +320,28 @@ GlobalDeviceMap EstablishGlobalRank(
 // cp->same_num_devices_per_task.  Requires cp->instance.task_names
 // be sorted.
 void SetDevPerTask(CollectiveParams* cp) {
-  cp->instance.same_num_devices_per_task = false;
-  if (cp->instance.task_names.empty()) return;
-  int dev_per_task = -1;
-  int count = 0;
+  cp->instance.num_devices_per_task.clear();
   const string* last_task_name = &cp->instance.task_names[0];
+  int count = 0;
   for (const string& task_name : cp->instance.task_names) {
-    if (task_name != *last_task_name) {
-      CHECK_GT(count, 0);
-      if (dev_per_task < 0) {
-        dev_per_task = count;
-      } else {
-        CHECK_GT(dev_per_task, 0);
-        if (count != dev_per_task) return;
-      }
+    if (task_name == *last_task_name) {
+      ++count;
+    } else {
+      cp->instance.num_devices_per_task[*last_task_name] = count;
       count = 1;
       last_task_name = &task_name;
-    } else {
-      ++count;
     }
   }
-  CHECK_GT(count, 0);
-  if ((dev_per_task > 0) && (count != dev_per_task)) {
-    return;
+  cp->instance.num_devices_per_task[*last_task_name] = count;
+
+  cp->instance.same_num_devices_per_task = false;
+  int dev_per_task = -1;
+  for (const auto& task_dev : cp->instance.num_devices_per_task) {
+    if (dev_per_task == -1) {
+      dev_per_task = task_dev.second;
+    } else if (dev_per_task != task_dev.second) {
+      return;
+    }
   }
   cp->instance.same_num_devices_per_task = true;
   CHECK_EQ((cp->group.group_size % cp->group.num_tasks), 0);
@@ -398,7 +401,6 @@ void CollectiveParamResolverLocal::SetDefaultRank(const string& device,
 void CollectiveParamResolverLocal::InitInstanceSharedParams(
     const GroupRec* gr, const CollectiveParams* cp, InstanceRec* ir,
     const StatusCallback& done) {
-  VLOG(1) << "InitInstanceSharedParams " << ir;
   ir->shared.instance = cp->instance;
   {
     mutex_lock gl(gr->mu);
@@ -412,8 +414,8 @@ void CollectiveParamResolverLocal::InitInstanceSharedParams(
   }
   ir->shared.default_rank = -1;
 
-  // Sort devce_names lexicographcally, keeping task_names in
-  // corresponding order.
+  // Sort device_names lexicographically, keeping task_names in corresponding
+  // order.  Also set number of devices per task.
   SortDevicesAndTasks(&ir->shared);
 
   // Get Locality data for all devices.
@@ -605,6 +607,25 @@ void CollectiveParamResolverLocal::CompleteInstanceAsync(
                        "intended only for non-distributed deployment."));
 }
 
+// TODO(b/111897089): we need a better way to pick the collective
+// implementation.  The ideal way would depend upon the topology and link
+// strength before picking a particular implementation.
+void CollectiveParamResolverLocal::AssignCollectiveType(CollectiveParams* cp) {
+  if (cp->instance.type == BROADCAST_COLLECTIVE) {
+    cp->instance.impl_details.collective_name = "HierarchicalTreeBroadcast";
+  } else if (cp->instance.type == REDUCTION_COLLECTIVE) {
+    if (nccl_) {
+      cp->instance.impl_details.collective_name = "NcclReduce";
+    } else {
+      cp->instance.impl_details.collective_name = "RingReduce";
+    }
+  } else {
+    cp->instance.impl_details.collective_name = "undef";
+  }
+  VLOG(1) << "AssignCollectiveType "
+          << cp->instance.impl_details.collective_name;
+}
+
 void CollectiveParamResolverLocal::CompleteInstanceLocal(
     const string& device, const GroupRec* gr, CollectiveParams* cp,
     bool is_source, const StatusCallback& done) {
@@ -641,48 +662,57 @@ void CollectiveParamResolverLocal::CompleteInstanceFromInitializedIRec(
     // custom operator= does a deep copy.
     cp->instance = ir->shared.instance;
   }
-  // Populate the fields common across task, also default_rank.
+  // Populate the fields common across task.
+  AssignCollectiveType(cp);
   SetDefaultRank(device, cp);
   CompleteTaskIsLocal(task_name_, cp);
-  // TODO(b/113171733): we need a better way to pick the collective
-  // implementation.  The ideal way would depend upon the topology and link
-  // strength before picking a particular implementation.
-  cp->instance.impl_details.collective_name =
-      (cp->instance.type == BROADCAST_COLLECTIVE) ? "HierarchicalTreeBroadcast"
-                                                  : "RingReduce";
+
   CollectiveImplementationInterface* col_impl;
-  Status lookup_status = CollectiveRegistry::LookupParamResolverInstance(
+  Status status = CollectiveRegistry::LookupParamResolverInstance(
       cp->instance.impl_details.collective_name, &col_impl);
-  if (!lookup_status.ok()) {
-    done(lookup_status);
+  if (status.ok()) {
+    status = col_impl->InitializeInstanceBeforeGroupDiscovery(cp);
+  }
+  if (!status.ok()) {
+    done(status);
     return;
   }
-  // If broadcast, may need to wait for source discovery.
-  if (cp->instance.type == BROADCAST_COLLECTIVE) {
-    CompleteInstanceSource(ir, cp, is_source,
-                           [col_impl, ir, device, cp, done](InstanceRec* irec) {
-                             CHECK_EQ(ir, irec);
-                             Status s;
-                             {
-                               mutex_lock l(irec->out_mu);
-                               irec->WaitForOutMu(l);
-                               s = irec->status;
-                               cp->source_rank = irec->source_rank;
-                             }
-                             if (s.ok()) {
-                               s = col_impl->InitializeCollectiveParams(cp);
-                             }
-                             done(s);
-                           });
+
+  //  We may need to wait for the group if:
+  //  * this is a broadcast, for source discovery;
+  //  * we are using NCCL with more than 1 worker, for the communicator key from
+  //    rank 0.
+  bool broadcast = cp->instance.type == BROADCAST_COLLECTIVE;
+  bool nccl = cp->instance.type == REDUCTION_COLLECTIVE &&
+              cp->instance.impl_details.collective_name == "NcclReduce" &&
+              cp->group.num_tasks > 1;
+  if (broadcast || nccl) {
+    WaitForGroup(ir, cp, is_source, broadcast, nccl,
+                 [col_impl, ir, device, cp, done](InstanceRec* irec) {
+                   Status s;
+                   if (ir != irec) {
+                     s = errors::Internal("Expected ir ", ir, " and irec ",
+                                          irec, " to be equal");
+                   } else {
+                     mutex_lock l(irec->out_mu);
+                     irec->WaitForOutMu(l);
+                     s = irec->status;
+                     cp->source_rank = irec->source_rank;
+                     cp->instance.communicator_key = irec->communicator_key;
+                   }
+                   if (s.ok()) {
+                     s = col_impl->InitializeCollectiveParams(cp);
+                   }
+                   done(s);
+                 });
   } else {
     done(col_impl->InitializeCollectiveParams(cp));
   }
 }
 
-void CollectiveParamResolverLocal::CompleteInstanceSource(InstanceRec* ir,
-                                                          CollectiveParams* cp,
-                                                          bool is_source,
-                                                          const IRConsumer& f) {
+void CollectiveParamResolverLocal::WaitForGroup(
+    InstanceRec* ir, CollectiveParams* cp, bool is_source, bool init_source,
+    bool init_nccl, const IRConsumer& f) {
   std::vector<IRConsumer> ready_waiters;
   {
     mutex_lock l(ir->out_mu);
@@ -692,7 +722,8 @@ void CollectiveParamResolverLocal::CompleteInstanceSource(InstanceRec* ir,
     if (!ir->known[cp->default_rank]) {
       ir->known[cp->default_rank] = true;
       ++ir->known_count;
-      if (is_source) {
+      if (init_source && is_source) {
+        // Initialize source rank.
         if (ir->source_rank >= 0) {
           ir->status = errors::Internal("Instance ", cp->instance.instance_key,
                                         " already has source ", ir->source_rank,
@@ -702,13 +733,26 @@ void CollectiveParamResolverLocal::CompleteInstanceSource(InstanceRec* ir,
           ir->source_rank = cp->default_rank;
         }
       }
+      if (init_nccl && cp->default_rank == 0) {
+        // Initialize communicator key.
+        if (!ir->communicator_key.empty()) {
+          ir->status =
+              errors::Internal("Instance ", cp->instance.instance_key,
+                               " already has communicator_key ",
+                               str_util::CEscape(ir->communicator_key),
+                               ", received second claim from device ",
+                               cp->instance.device_names[cp->default_rank]);
+        } else {
+          ir->communicator_key = cp->instance.communicator_key;
+        }
+      }
     }
     if (ir->known_count < ir->shared.group.group_size) {
       ir->known_waiters.push_back(f);
       return;
     }
     CHECK_EQ(ir->known_count, ir->shared.group.group_size);
-    if (ir->source_rank < 0) {
+    if (init_source && ir->source_rank < 0) {
       // NOTE(ayushd): changing the error message below would also require
       // updating CompleteParamsBroadcastForgotSend test in
       // CollectiveParamResolverLocalTest.
@@ -718,6 +762,13 @@ void CollectiveParamResolverLocal::CompleteInstanceSource(InstanceRec* ir,
                            "could mean that there were group_size=",
                            ir->known_count, " BcastRecvs but no BcastSend.");
     }
+    if (init_nccl && ir->communicator_key.empty()) {
+      ir->status = errors::Internal(
+          "Instance ", cp->instance.instance_key, " device ",
+          cp->instance.device_names[cp->default_rank],
+          " did not find rank 0 for setting communicator key.  This is an "
+          "internal error in collective param resolution");
+    }
     if (!ir->known_waiters.empty()) {
       ready_waiters = std::move(ir->known_waiters);
     }
diff --git a/tensorflow/core/common_runtime/collective_param_resolver_local.h b/tensorflow/core/common_runtime/collective_param_resolver_local.h
index 365bddc787..fd408e4ef3 100644
--- a/tensorflow/core/common_runtime/collective_param_resolver_local.h
+++ b/tensorflow/core/common_runtime/collective_param_resolver_local.h
@@ -130,8 +130,10 @@ class CollectiveParamResolverLocal : public ParamResolverInterface {
     Status status GUARDED_BY(out_mu);
 
     // These fields are used to count the instances that have called
-    // in and become known while resolving broadcast source identity.
+    // in and become known while resolving broadcast source identity and
+    // communicator key.
     int source_rank GUARDED_BY(out_mu);
+    string communicator_key GUARDED_BY(out_mu);
     int known_count GUARDED_BY(out_mu);
     std::vector<bool> known GUARDED_BY(out_mu);
     std::vector<IRConsumer> known_waiters GUARDED_BY(out_mu);
@@ -197,10 +199,10 @@ class CollectiveParamResolverLocal : public ParamResolverInterface {
                                            const StatusCallback& done)
       LOCKS_EXCLUDED(ir->out_mu);
 
-  // Complete source data for a broadcast instance.
+  // Complete source data and/or nccl communicator key.
   // Precondition: *cp has complete group data and default_rank.
-  void CompleteInstanceSource(InstanceRec* ir, CollectiveParams* cp,
-                              bool is_source, const IRConsumer& f)
+  void WaitForGroup(InstanceRec* ir, CollectiveParams* cp, bool is_source,
+                    bool init_source, bool init_nccl, const IRConsumer& f)
       LOCKS_EXCLUDED(ir->out_mu);
 
   // If cp.device_names contains only devices local to this process
@@ -216,10 +218,15 @@ class CollectiveParamResolverLocal : public ParamResolverInterface {
   // current ordering of cp->instance.device_names.
   void SetDefaultRank(const string& device, CollectiveParams* cp);
 
+  // Sets cp->instance.type based on collective op type, and attempts to assign
+  // best implementation.
+  void AssignCollectiveType(CollectiveParams* cp);
+
   // Helper to grab status under lock, invoke callback out of lock.
   void CallbackWithStatus(const InstanceRecCallback& done, InstanceRec* irec)
       LOCKS_EXCLUDED(irec->out_mu);
 
+  const bool nccl_;
   const DeviceMgr* dev_mgr_;
   DeviceResolverInterface* dev_resolver_;  // Not owned.
   string task_name_;
diff --git a/tensorflow/core/common_runtime/collective_util.cc b/tensorflow/core/common_runtime/collective_util.cc
index 195521a078..bee4a13d18 100644
--- a/tensorflow/core/common_runtime/collective_util.cc
+++ b/tensorflow/core/common_runtime/collective_util.cc
@@ -79,5 +79,36 @@ string SubdivPermDebugString(const CollectiveParams& col_params) {
   return buf;
 }
 
+SubContext::SubContext(OpKernelContext* ctx, OpKernelContext::Params* params,
+                       OpKernel* op, Tensor* output, Tensor* input)
+    : sub_params_(*params),
+      sub_inputs_({output, input}),
+      sub_input_attr_({ctx->input_alloc_attr(0), ctx->input_alloc_attr(0)}),
+      sub_input_dc_(
+          {ctx->input_device_context(0), ctx->input_device_context(0)}) {
+  sub_params_.op_kernel = op;
+  sub_params_.inputs = &sub_inputs_;
+  sub_params_.input_alloc_attrs = &sub_input_attr_;
+  sub_params_.input_device_contexts = &sub_input_dc_;
+  sub_params_.eigen_gpu_device = nullptr;
+  sub_params_.ensure_eigen_gpu_device();
+  sub_params_.forward_from_array = &forward_from_;
+  sub_ctx_.reset(new OpKernelContext(&sub_params_, 1));
+}
+
+Status ComputeBinOp(OpKernelContext* op_ctx, OpKernelContext::Params* params,
+                    Device* device, OpKernel* op, Tensor* output,
+                    Tensor* input) {
+  // Prepare an OpKernelContext that is identical to that of the original Op
+  // (i.e. the collective), except for the input output sizes and identities and
+  // the Op itself.
+  // TODO(ayushd, tucker): Is it possible to cache and reuse these objects?
+  // They're mostly identical inside one device execution.
+  std::unique_ptr<SubContext> sub_ctx(
+      new SubContext(op_ctx, params, op, output, input));
+  device->Compute(op, sub_ctx->sub_ctx_.get());
+  return sub_ctx->sub_ctx_->status();
+}
+
 }  // namespace collective_util
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/collective_util.h b/tensorflow/core/common_runtime/collective_util.h
index ebb5731bec..01fb8b8c81 100644
--- a/tensorflow/core/common_runtime/collective_util.h
+++ b/tensorflow/core/common_runtime/collective_util.h
@@ -32,6 +32,27 @@ Status InitializeDeviceAndLocality(const DeviceMgr* dev_mgr,
                                    DeviceLocality* device_locality);
 string SubdivPermDebugString(const CollectiveParams& col_params);
 
+// Used for executing a sub-operation, e.g. a merge_op instance, with
+// an OpKernelContext based on the one passed into this Op.
+class SubContext {
+ public:
+  OpKernelContext::Params sub_params_;
+  gtl::InlinedVector<TensorValue, 4> sub_inputs_;
+  gtl::InlinedVector<AllocatorAttributes, 4> sub_input_attr_;
+  gtl::InlinedVector<DeviceContext*, 4> sub_input_dc_;
+  // Used only for Binary and Unary Ops for which we require
+  // the calculation to be in-place on the first input.
+  int forward_from_ = 0;
+  std::unique_ptr<OpKernelContext> sub_ctx_;
+  SubContext(OpKernelContext* ctx, OpKernelContext::Params* params,
+             OpKernel* op, Tensor* output, Tensor* input);
+  ~SubContext() = default;
+};
+
+Status ComputeBinOp(OpKernelContext* op_ctx, OpKernelContext::Params* params,
+                    Device* device, OpKernel* op, Tensor* output,
+                    Tensor* input);
+
 }  // namespace collective_util
 }  // namespace tensorflow
 
diff --git a/tensorflow/core/common_runtime/hierarchical_tree_broadcaster.h b/tensorflow/core/common_runtime/hierarchical_tree_broadcaster.h
index ceb9baad30..76392b8e59 100644
--- a/tensorflow/core/common_runtime/hierarchical_tree_broadcaster.h
+++ b/tensorflow/core/common_runtime/hierarchical_tree_broadcaster.h
@@ -41,6 +41,11 @@ class HierarchicalTreeBroadcaster : public CollectiveImplementationInterface {
   // and device_locality.  Also saves the CollectiveContext in this object.
   Status InitializeCollectiveContext(CollectiveContext* col_ctx) override;
 
+  // No-op for hierarchical tree broadcaster.
+  Status InitializeInstanceBeforeGroupDiscovery(CollectiveParams*) override {
+    return Status::OK();
+  }
+
   // Begins async execution of the hierarchical tree broadcast.
   // Must be called in a blockable thread.
   // TODO(b/80529858): remove the previous warning when we have a dedicated
diff --git a/tensorflow/core/common_runtime/ring_reducer.cc b/tensorflow/core/common_runtime/ring_reducer.cc
index 092f15e49e..8ed2fc2f1c 100644
--- a/tensorflow/core/common_runtime/ring_reducer.cc
+++ b/tensorflow/core/common_runtime/ring_reducer.cc
@@ -394,37 +394,6 @@ void RingReducer::Finish(bool ok) {
   done_(s);
 }
 
-RingReducer::SubContext::SubContext(OpKernelContext* ctx,
-                                    OpKernelContext::Params* params,
-                                    OpKernel* op, Tensor* output, Tensor* input)
-    : sub_params_(*params),
-      sub_inputs_({output, input}),
-      sub_input_attr_({ctx->input_alloc_attr(0), ctx->input_alloc_attr(0)}),
-      sub_input_dc_(
-          {ctx->input_device_context(0), ctx->input_device_context(0)}) {
-  sub_params_.op_kernel = op;
-  sub_params_.inputs = &sub_inputs_;
-  sub_params_.input_alloc_attrs = &sub_input_attr_;
-  sub_params_.input_device_contexts = &sub_input_dc_;
-  sub_params_.eigen_gpu_device = nullptr;
-  sub_params_.ensure_eigen_gpu_device();
-  sub_params_.forward_from_array = &forward_from_;
-  sub_ctx_ = new OpKernelContext(&sub_params_, 1);
-}
-
-Status RingReducer::ComputeBinOp(Device* device, OpKernel* op, Tensor* output,
-                                 Tensor* input) {
-  // Prepare an OpKernelContext that is identical to that of the original Op
-  // (i.e. the collective), except for the input output sizes and identities and
-  // the Op itself.
-  // TODO(tucker): Is it possible to cache and reuse these objects?  They're
-  // mostly identical inside one device execution.
-  std::unique_ptr<SubContext> sub_ctx(
-      new SubContext(col_ctx_->op_ctx, col_ctx_->op_params, op, output, input));
-  device->Compute(op, sub_ctx->sub_ctx_);
-  return sub_ctx->sub_ctx_->status();
-}
-
 // At the beginning of the algorithm initialize a RingField struct for
 // every independent field of the tensor.
 void RingReducer::InitRingField(RingField* rf, int chunk_idx, int subdiv_idx,
@@ -632,9 +601,9 @@ bool RingReducer::RunAsyncParts() {
           --recv_pending_count;
           if (!rf->second_pass) {
             rf->action = RF_REDUCE;
-            Status s =
-                ComputeBinOp(col_ctx_->device, col_params_->merge_op.get(),
-                             &rf->chunk, &rf->tmp_chunk);
+            Status s = collective_util::ComputeBinOp(
+                col_ctx_->op_ctx, col_ctx_->op_params, col_ctx_->device,
+                col_params_->merge_op.get(), &rf->chunk, &rf->tmp_chunk);
             if (!s.ok()) {
               aborted = true;
               StartAbort(s);
@@ -647,9 +616,9 @@ bool RingReducer::RunAsyncParts() {
           if (!rf->second_pass && col_params_->final_op.get() && rf->is_final) {
             rf->action = RF_FINALIZE;
             group_size_tensor_ready_.WaitForNotification();
-            Status s =
-                ComputeBinOp(col_ctx_->device, col_params_->final_op.get(),
-                             &rf->chunk, &group_size_tensor_);
+            Status s = collective_util::ComputeBinOp(
+                col_ctx_->op_ctx, col_ctx_->op_params, col_ctx_->device,
+                col_params_->final_op.get(), &rf->chunk, &group_size_tensor_);
             if (!s.ok()) {
               aborted = true;
               StartAbort(s);
diff --git a/tensorflow/core/common_runtime/ring_reducer.h b/tensorflow/core/common_runtime/ring_reducer.h
index 0848e37b52..a5aa8fad70 100644
--- a/tensorflow/core/common_runtime/ring_reducer.h
+++ b/tensorflow/core/common_runtime/ring_reducer.h
@@ -40,6 +40,11 @@ class RingReducer : public CollectiveImplementationInterface {
   // and device_locality.  Also saves the CollectiveContext in this object.
   Status InitializeCollectiveContext(CollectiveContext* col_ctx) override;
 
+  // No-op for ring reducer.
+  Status InitializeInstanceBeforeGroupDiscovery(CollectiveParams*) override {
+    return Status::OK();
+  }
+
   // Begins async execution of the ring reduce algorithm.
   // Must be called in a blockable thread.
   // TODO(b/80529858): remove the previous warning when we have a dedicated
@@ -52,27 +57,8 @@ class RingReducer : public CollectiveImplementationInterface {
   void StartAbort(const Status& s);
   void ContinueAfterInputCopy();
   void Finish(bool ok);
-  Status ComputeBinOp(Device* device, OpKernel* op, Tensor* output,
-                      Tensor* input);
   bool RunAsyncParts();
 
-  // Used for executing a sub-operation, e.g. a merge_op instance, with
-  // an OpKernelContext based on the one passed into this Op.
-  class SubContext {
-   public:
-    OpKernelContext::Params sub_params_;
-    gtl::InlinedVector<TensorValue, 4> sub_inputs_;
-    gtl::InlinedVector<AllocatorAttributes, 4> sub_input_attr_;
-    gtl::InlinedVector<DeviceContext*, 4> sub_input_dc_;
-    // Used only for Binary and Unary Ops for which we require
-    // the calculation to be in-place on the first input.
-    int forward_from_ = 0;
-    OpKernelContext* sub_ctx_;
-    SubContext(OpKernelContext* ctx, OpKernelContext::Params* params,
-               OpKernel* op, Tensor* output, Tensor* input);
-    ~SubContext() { delete sub_ctx_; }
-  };
-
   // Current status of a RingField
   enum RingFieldAction {
     RF_INIT = 0,    // Just initialized for a pass
diff --git a/tensorflow/core/distributed_runtime/BUILD b/tensorflow/core/distributed_runtime/BUILD
index e388d3e6f0..197a3f24d4 100644
--- a/tensorflow/core/distributed_runtime/BUILD
+++ b/tensorflow/core/distributed_runtime/BUILD
@@ -594,6 +594,7 @@ tf_cc_test(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
+        "//tensorflow/core/kernels:collective_ops",
     ],
 )
 
diff --git a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc
index 1dd10d309b..9f94a24fcd 100644
--- a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc
+++ b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc
@@ -181,6 +181,10 @@ void CollectiveParamResolverDistributed::CompleteInstanceAsync(
                           ir->WaitForOutMu(l);
                           response->set_instance_key(cp->instance.instance_key);
                           response->set_source_rank(ir->source_rank);
+                          if (!cp->instance.communicator_key.empty()) {
+                            response->set_communicator_key(
+                                cp->instance.communicator_key);
+                          }
                           done_and_cleanup(fi_status);
                         } else {
                           done_and_cleanup(fi_status);
@@ -283,8 +287,10 @@ void CollectiveParamResolverDistributed::UpdateInstanceCache(
   using InstanceRecPointer = InstanceRec*;
   InstanceRecPointer* irp = new InstanceRecPointer(nullptr);
   int32 source_rank = resp.source_rank();
+  string communicator_key = resp.communicator_key();
 
-  auto continue_with_ir = [this, cp, irp, source_rank, done](const Status& s) {
+  auto continue_with_ir = [cp, irp, source_rank, communicator_key,
+                           done](const Status& s) {
     if (!s.ok()) {
       done(s);
       delete irp;
@@ -306,6 +312,19 @@ void CollectiveParamResolverDistributed::UpdateInstanceCache(
         }
         ir->source_rank = source_rank;
       }
+      if (ir->communicator_key != communicator_key) {
+        if (!ir->communicator_key.empty()) {
+          ir->status = errors::Internal(
+              "UpdateInstanceCache: CompleteInstanceResponse for instance ",
+              cp->instance.instance_key,
+              " gives communicator_key with size =", communicator_key.size(),
+              " but cache already holds communicator_key with size=",
+              ir->communicator_key.size());
+          status = ir->status;
+          break;
+        }
+        ir->communicator_key = communicator_key;
+      }
       if (ir->known_count < cp->group.group_size) {
         ir->known_count = cp->group.group_size;
         if (ir->known.size() != cp->group.group_size) {
diff --git a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc
index 40b18d321a..823d7d5eb9 100644
--- a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc
+++ b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc
@@ -268,6 +268,8 @@ class DeviceResDistTest : public ::testing::Test {
         EXPECT_EQ(cp_[idx].instance.device_names[idx], device_name);
         EXPECT_EQ(cp_[idx].instance.task_names[idx], task_name);
         if (idx > 0) {
+          EXPECT_EQ(cp_[0].instance.communicator_key,
+                    cp_[idx].instance.communicator_key);
           for (int i = 0; i < dev_count; ++i) {
             EXPECT_EQ(cp_[0].instance.device_names[i],
                       cp_[idx].instance.device_names[i]);
diff --git a/tensorflow/core/framework/collective.cc b/tensorflow/core/framework/collective.cc
index 7fa58347f2..b83d183f14 100644
--- a/tensorflow/core/framework/collective.cc
+++ b/tensorflow/core/framework/collective.cc
@@ -17,6 +17,7 @@ limitations under the License.
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/hash/hash.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 
 namespace tensorflow {
@@ -64,7 +65,9 @@ CollInstanceParams& CollInstanceParams::operator=(
     device_names.assign(other.device_names.begin(), other.device_names.end());
     task_names.assign(other.task_names.begin(), other.task_names.end());
     same_num_devices_per_task = other.same_num_devices_per_task;
+    num_devices_per_task = other.num_devices_per_task;
     gpu_ring_order = other.gpu_ring_order;
+    communicator_key = other.communicator_key;
     impl_details.subdiv_offsets.assign(
         other.impl_details.subdiv_offsets.begin(),
         other.impl_details.subdiv_offsets.end());
@@ -76,6 +79,7 @@ CollInstanceParams& CollInstanceParams::operator=(
     impl_details.subdiv_source_rank.assign(
         other.impl_details.subdiv_source_rank.begin(),
         other.impl_details.subdiv_source_rank.end());
+    impl_details.dependencies = other.impl_details.dependencies;
   }
   return *this;
 }
@@ -91,6 +95,13 @@ string CollInstanceParams::ToString() const {
   for (const auto& n : task_names) {
     strings::StrAppend(&v, n, ", ");
   }
+  strings::StrAppend(&v, "} num_devices_per_task={");
+  for (const auto dpt : num_devices_per_task) {
+    strings::StrAppend(&v, dpt.first, ": ", dpt.second, ", ");
+  }
+  strings::StrAppend(&v, "}, collective_name=", impl_details.collective_name,
+                     ", communicator_key=", str_util::CEscape(communicator_key),
+                     ", subdiv_offsets={");
   strings::StrAppend(&v, "}, subdiv_offsets={");
   for (const auto& d : impl_details.subdiv_offsets) {
     strings::StrAppend(&v, d, ",");
diff --git a/tensorflow/core/framework/collective.h b/tensorflow/core/framework/collective.h
index 0321429702..546e3938a8 100644
--- a/tensorflow/core/framework/collective.h
+++ b/tensorflow/core/framework/collective.h
@@ -70,6 +70,8 @@ struct CollImplDetails {
   std::vector<std::vector<int>> subdiv_permutations;
   std::vector<int> subdiv_offsets;
   std::vector<int> subdiv_source_rank;  // rank of source in each subdiv
+  std::vector<int32>
+      dependencies;  // collective instances on which this node depends
 };
 
 // Data common to all members of a collective instance.
@@ -85,9 +87,13 @@ struct CollInstanceParams {
   std::vector<string> task_names;
   // True if every task has the same number of devices.
   bool same_num_devices_per_task = false;
+  // Task -> number of devices on that task.
+  std::unordered_map<string, int32> num_devices_per_task;
   // If passed in to GPUOptions in ConfigProto, defines a good ring order for
   // GPUs.  Assumes same GPU configuration at each worker.
   string gpu_ring_order = "";
+  // Valid when using a communicator-based collective mechanism, e.g. NCCL.
+  string communicator_key;
   CollImplDetails impl_details;
   string ToString() const;
   CollInstanceParams& operator=(const struct CollInstanceParams& other);
@@ -269,6 +275,21 @@ class CollectiveExecutor : public PeerAccessInterface, public core::RefCounted {
 
   virtual PerStepCollectiveRemoteAccess* remote_access() { return nullptr; }
 
+  // `WaitForDependencies` and `Launched` are used for fine-grained control of
+  // execution order between collective instances.  These functions are intended
+  // to be called in `Run` function of collective implementations, and may be
+  // used to make part, or whole, of the collective execution ordered with
+  // respect to other collective instances.
+  //
+  // `WaitForDependencies` will block until it is safe to continue the callee's
+  // execution, where safety is defined as: ordered with respect to the
+  // collective instances defined in the callee's `wait_for` attribute.
+  virtual void WaitForDependencies(const CollectiveParams& col_params) {}
+  // `Launched` unblocks the dependent collective instances by recording that
+  // this callee device has completed the critical portion of the collective
+  // execution.
+  virtual void Launched(const CollectiveParams& col_params) {}
+
   // Used to designate an invalid group or instance key.
   static int64 kInvalidId;
 
@@ -347,7 +368,8 @@ class CollectiveImplementationInterface {
 
   // Initializes the portions of `col_params` specific to this
   // implementation.  Called exactly once for every Collective instance during
-  // the CollectiveParams resolution process when the graph is first executed.
+  // the CollectiveParams resolution process when the graph is first executed,
+  // at the end of `CompleteInstanceLocal()`.
   // NOTE(ayushd): This is effectively a static function because it modifies the
   // `col_params` passed in and should not manipulate any data members.  However
   // because it is virtual and needs to be implemented by every derived class we
@@ -360,6 +382,14 @@ class CollectiveImplementationInterface {
   // object.
   virtual Status InitializeCollectiveContext(CollectiveContext* col_ctx) = 0;
 
+  // Initializes instance params at the beginning of `CompleteInstanceLocal()`,
+  // unlike `InitializeCollectiveParams` which is called at the end.  This
+  // function is called before all devices in the instance are discovered, and
+  // may be used to broadcast data via the shared `InstanceRec` object in
+  // collective param resolution to all devices.
+  virtual Status InitializeInstanceBeforeGroupDiscovery(
+      CollectiveParams* col_params) = 0;
+
   // Processes and moves data according to the logic of this Collective
   // implementation.  Relies on appropriate initialization of op-specific
   // CollectiveParams in InitializeCollectiveParams(), as well as appropriate
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 0792b398f1..4f21b466cd 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -181,12 +181,36 @@ tf_cc_test(
 
 tf_kernel_library(
     name = "collective_ops",
+    srcs = if_cuda([
+        "collective_nccl_reducer.h",
+        "collective_nccl_reducer.cc",
+    ]),
     prefix = "collective_ops",
     deps = [
         "//tensorflow/core:collective_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
+    ] + if_cuda([
+        "@local_config_nccl//:nccl",
+        "//tensorflow/core/nccl:nccl_lib",
+    ]),
+)
+
+tf_cuda_cc_test(
+    name = "collective_nccl_reducer_test",
+    size = "small",
+    srcs = ["collective_nccl_reducer_test.cc"],
+    tags = tf_cuda_tests_tags() + ["no_cuda_on_cpu_tap"],
+    deps = [
+        "//tensorflow/core:all_kernels",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
     ],
 )
 
diff --git a/tensorflow/core/kernels/collective_nccl_reducer.cc b/tensorflow/core/kernels/collective_nccl_reducer.cc
new file mode 100644
index 0000000000..113f1487f0
--- /dev/null
+++ b/tensorflow/core/kernels/collective_nccl_reducer.cc
@@ -0,0 +1,204 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/kernels/collective_nccl_reducer.h"
+
+#ifdef GOOGLE_CUDA
+
+#include "tensorflow/core/common_runtime/collective_util.h"
+#include "tensorflow/core/nccl/nccl_manager.h"
+
+namespace tensorflow {
+namespace {
+string NcclCollectiveKey(const string& exec_key, int step_id) {
+  return strings::StrCat(exec_key, ":", step_id);
+}
+}  // namespace
+
+NcclReducer::NcclReducer() : col_ctx_(nullptr), col_params_(nullptr) {}
+
+Status NcclReducer::InitializeCollectiveParams(CollectiveParams* col_params) {
+  if (col_params->instance.type != REDUCTION_COLLECTIVE ||
+      col_params->instance.impl_details.collective_name != "NcclReduce") {
+    return errors::Internal("Unexpected collective type ",
+                            col_params->instance.type, " expected ",
+                            REDUCTION_COLLECTIVE, "; or collective name ",
+                            col_params->instance.impl_details.collective_name,
+                            " expected NcclReduce");
+  } else {
+    return Status::OK();
+  }
+}
+
+Status NcclReducer::InitializeCollectiveContext(CollectiveContext* col_ctx) {
+  col_ctx_ = col_ctx;
+  col_params_ = &col_ctx->col_params;
+  return collective_util::InitializeDeviceAndLocality(
+      col_ctx->dev_mgr, col_ctx->device_name, &col_ctx->device,
+      &col_ctx->device_locality);
+}
+
+Status NcclReducer::InitializeInstanceBeforeGroupDiscovery(
+    CollectiveParams* col_params) {
+  if (col_params->default_rank == 0 && col_params->group.num_tasks > 1) {
+    col_params->instance.communicator_key =
+        NcclManager::instance()->GenerateCommunicatorKey();
+  }
+  return Status::OK();
+}
+
+Status ReductionOp(const string& merge_op, ncclRedOp_t* reduction_op) {
+  if (merge_op == "Add") {
+    *reduction_op = ncclSum;
+    return Status::OK();
+  } else if (merge_op == "Mul") {
+    *reduction_op = ncclProd;
+    return Status::OK();
+  } else {
+    return errors::Internal("Expected merge_op to be either Add or Mul, found ",
+                            merge_op);
+  }
+}
+
+void NcclReducer::Run(StatusCallback done) {
+  ncclRedOp_t reduction_op;
+  Status s = ReductionOp(col_params_->merge_op->type_string(), &reduction_op);
+  if (!s.ok()) {
+    done(s);
+    return;
+  }
+
+  Tensor group_size;
+  Notification group_size_ready;
+  Status group_size_status;
+  if (col_params_->final_op) {
+    // Create an on-device scalar value from group_size_.
+    // TODO(ayushd, tucker): avoid this copy by either reusing across
+    // invocations or providing the scalar to the kernel in host memory.
+    Tensor group_size_val(col_ctx_->output->dtype(), TensorShape({}));
+    switch (col_ctx_->output->dtype()) {
+      case DT_FLOAT:
+        group_size_val.scalar<float>()() = col_params_->group.group_size;
+        break;
+      case DT_DOUBLE:
+        group_size_val.scalar<double>()() = col_params_->group.group_size;
+        break;
+      case DT_INT32:
+        group_size_val.scalar<int32>()() = col_params_->group.group_size;
+        break;
+      case DT_INT64:
+        group_size_val.scalar<int64>()() = col_params_->group.group_size;
+        break;
+      default:
+        done(errors::Internal("Unsupported type ", col_ctx_->output->dtype()));
+        return;
+    }
+    group_size = Tensor(
+        col_ctx_->device->GetAllocator(col_ctx_->op_ctx->input_alloc_attr(0)),
+        col_ctx_->output->dtype(), TensorShape({}));
+    DeviceContext* op_dev_ctx = col_ctx_->op_ctx->op_device_context();
+    // Enqueue copy on gpu stream.
+    op_dev_ctx->CopyCPUTensorToDevice(
+        &group_size_val, col_ctx_->device, &group_size,
+        [&group_size_ready, &group_size_status](const Status& s) {
+          group_size_status = s;
+          group_size_ready.Notify();
+        });
+  } else {
+    group_size_ready.Notify();
+  }
+
+  Notification nccl_done;
+  Status nccl_status;
+  auto* compute_stream = col_ctx_->op_ctx->op_device_context()->stream();
+  auto* gpu_info = col_ctx_->op_ctx->device()->tensorflow_gpu_device_info();
+  // `AddToAllReduce` performs consistency checks for the NCCL call and enqueues
+  // the `Participant` struct locally.  When all local participants with this
+  // `nccl_collective_key` have called `AddToAllReduce` and
+  // `SignalMultiNodeReady`, all devices at this worker are ready to process
+  // this NCCL op.
+  //
+  // The `NcclManager` uses a dedicated CUDA stream for NCCL kernels.  At this
+  // point, it synchronizes the NCCL stream with the compute stream, and then
+  // enqueues the NCCL kernel on the NCCL stream.
+  const int num_global_devices = col_params_->group.group_size;
+  const int num_local_devices = col_params_->instance.num_devices_per_task.at(
+      col_params_->instance.task_names[col_params_->default_rank]);
+  const string nccl_collective_key =
+      NcclCollectiveKey(col_ctx_->exec_key, col_ctx_->step_id);
+  auto done_callback = [&nccl_done, &nccl_status](const Status& s) {
+    nccl_status = s;
+    nccl_done.Notify();
+  };
+  auto participant = absl::make_unique<NcclManager::Participant>(
+      compute_stream->parent(), compute_stream, gpu_info->event_mgr,
+      gpu_info->gpu_id, col_ctx_->input, col_ctx_->output,
+      col_params_->default_rank, std::move(done_callback));
+  VLOG(1) << "NcclReducer calling NcclManager::AddToAllReduce num_tasks "
+          << col_params_->group.num_tasks << " current task "
+          << col_params_->instance.task_names[col_params_->default_rank]
+          << " num local devices " << num_local_devices
+          << " num global devices " << num_global_devices;
+  NcclManager::instance()->AddToAllReduce(
+      std::move(participant),
+      {nccl_collective_key, num_local_devices, num_global_devices,
+       col_params_->instance.communicator_key},
+      reduction_op);
+
+  // NOTE(ayushd): We need to synchronize NCCL launches across nodes to prevent
+  // deadlocks.  In the current implementation, we define a deterministic
+  // sequential launch order between potentially concurrent collective instances
+  // by introducing control information during static graph analysis in
+  // graph/collective_order.cc.  This can be either in the form of explicit
+  // control edges or via `wait_for` attribute on the collective op.
+  //
+  // The other end of the design spectrum would have a distinguished node
+  // dynamically signal the next collective to launch to all other participants.
+  // This has higher degree of runtime coordination, but it may be able to
+  // achieve better performance if the (arbitrary) static execution order
+  // assigned in the first approach turns out to not be good from a scheduling
+  // perspective.  e.g. consider a graph in which c1, c2, and c3 are three
+  // concurrent collective instances, and the static ordering assigns c1 -> c2
+  // -> c3.  In practice, it could turn out that c3 is always ready to execute
+  // before c1 or c2.
+  //
+  // `WaitForDependencies` may block if the collective instances on which this
+  // op depends have not yet launched.  When this function returns, this op is
+  // ready to go.
+  col_ctx_->col_exec->WaitForDependencies(*col_params_);
+  NcclManager::instance()->SignalMultiNodeReady(nccl_collective_key);
+  // When all devices at this worker have called `SignalMultiNodeReady`, the
+  // `NcclManager` will enqueue the NCCL kernel on the NCCL stream.  Thus the
+  // implementation of `Launched` keeps track of the number of devices that have
+  // launched.
+  col_ctx_->col_exec->Launched(*col_params_);
+
+  // Wait for nccl op and group_size copy to succeed, then do final_op.
+  group_size_ready.WaitForNotification();
+  nccl_done.WaitForNotification();
+  Status final_status =
+      group_size_status.ok() ? nccl_status : group_size_status;
+  if (final_status.ok() && col_params_->final_op) {
+    final_status = collective_util::ComputeBinOp(
+        col_ctx_->op_ctx, col_ctx_->op_params, col_ctx_->device,
+        col_params_->final_op.get(), col_ctx_->output, &group_size);
+  }
+  done(final_status);
+}
+
+REGISTER_COLLECTIVE(NcclReduce, NcclReducer);
+
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/collective_nccl_reducer.h b/tensorflow/core/kernels/collective_nccl_reducer.h
new file mode 100644
index 0000000000..dc70b280c5
--- /dev/null
+++ b/tensorflow/core/kernels/collective_nccl_reducer.h
@@ -0,0 +1,49 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_KERNELS_COLLECTIVE_NCCL_REDUCER_H_
+#define TENSORFLOW_CORE_KERNELS_COLLECTIVE_NCCL_REDUCER_H_
+
+#include "tensorflow/core/framework/collective.h"
+
+namespace tensorflow {
+#ifdef GOOGLE_CUDA
+
+class NcclReducer : public CollectiveImplementationInterface {
+ public:
+  NcclReducer();
+  ~NcclReducer() override = default;
+
+  // No-op for this collective implementation.
+  Status InitializeCollectiveParams(CollectiveParams* col_params) override;
+
+  // Initializes the device objects and device localities.
+  Status InitializeCollectiveContext(CollectiveContext* col_ctx) override;
+
+  // Initialize nccl communicator key.
+  Status InitializeInstanceBeforeGroupDiscovery(
+      CollectiveParams* col_params) override;
+
+  // Hands off all reduce to NcclManager.
+  void Run(StatusCallback done) override;
+
+ private:
+  CollectiveContext* col_ctx_;          // Not owned
+  const CollectiveParams* col_params_;  // Not owned
+};
+
+#endif  // GOOGLE_CUDA
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_KERNELS_COLLECTIVE_NCCL_REDUCER_H_
diff --git a/tensorflow/core/kernels/collective_nccl_reducer_test.cc b/tensorflow/core/kernels/collective_nccl_reducer_test.cc
new file mode 100644
index 0000000000..ff33c1322c
--- /dev/null
+++ b/tensorflow/core/kernels/collective_nccl_reducer_test.cc
@@ -0,0 +1,331 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/kernels/collective_nccl_reducer.h"
+
+#include <algorithm>
+#include "absl/memory/memory.h"
+#include "tensorflow/core/common_runtime/base_collective_executor.h"
+#include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/common_runtime/device_factory.h"
+#include "tensorflow/core/common_runtime/device_mgr.h"
+#include "tensorflow/core/common_runtime/device_resolver_local.h"
+#include "tensorflow/core/common_runtime/process_util.h"
+#include "tensorflow/core/common_runtime/test_collective_executor_mgr.h"
+#include "tensorflow/core/framework/collective.h"
+#include "tensorflow/core/framework/fake_input.h"
+#include "tensorflow/core/framework/node_def_builder.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/lib/core/notification.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/public/session_options.h"
+#include "tensorflow/core/public/version.h"
+
+#ifdef GOOGLE_CUDA
+
+namespace tensorflow {
+static constexpr int kStepId = 10;
+
+std::unique_ptr<OpKernel> GetKernel(const NodeDef& node, DeviceBase* device) {
+  Status status;
+  std::unique_ptr<OpKernel> k = CreateOpKernel(
+      DEVICE_GPU, device, device->GetAllocator(AllocatorAttributes()), node,
+      TF_GRAPH_DEF_VERSION, &status);
+  if (!status.ok()) LOG(FATAL) << status;
+  return k;
+}
+
+std::unique_ptr<OpKernel> GetAdd(DeviceBase* device) {
+  NodeDef node_def;
+  NodeDefBuilder builder("add_node", "Add");
+  TF_CHECK_OK(builder.Attr("T", DT_FLOAT)
+                  .Input(FakeInput(DT_FLOAT))
+                  .Input(FakeInput(DT_FLOAT))
+                  .Finalize(&node_def));
+  return GetKernel(node_def, device);
+}
+
+std::unique_ptr<OpKernel> GetDiv(DeviceBase* device) {
+  NodeDef node_def;
+  NodeDefBuilder builder("add_node", "Div");
+  TF_CHECK_OK(builder.Attr("T", DT_FLOAT)
+                  .Input(FakeInput(DT_FLOAT))
+                  .Input(FakeInput(DT_FLOAT))
+                  .Finalize(&node_def));
+  return GetKernel(node_def, device);
+}
+
+class NcclReducerTest : public ::testing::Test {
+ protected:
+  ~NcclReducerTest() override {
+    if (col_exec_) col_exec_->Unref();
+  }
+
+  void InitGPUDevices() {
+    std::vector<std::unique_ptr<Device>> all_devices;
+    SessionOptions session_options;
+    session_options.config.mutable_gpu_options()
+        ->set_per_process_gpu_memory_fraction(0.1);
+    session_options.env = Env::Default();
+    Status s = DeviceFactory::GetFactory(DEVICE_GPU)
+                   ->AddDevices(session_options, "", &all_devices);
+    TF_CHECK_OK(s);
+    for (std::unique_ptr<Device>& d : all_devices) {
+      if (d->device_type() == "GPU") {
+        gpus_.emplace_back(std::move(d));
+      }
+    }
+  }
+
+  void Init(int num_ranks) {
+    setenv("NCCL_DEBUG", "INFO", 1 /* replace */);
+    setenv("NCCL_LAUNCH_MODE", "PARALLEL", 1 /* replace */);
+    InitGPUDevices();
+    std::vector<std::unique_ptr<Device>> local_devices;
+    std::vector<string> device_names;
+    for (int rank = 0; rank < num_ranks; ++rank) {
+      if (rank < gpus_.size()) {
+        local_devices.emplace_back(std::move(gpus_[rank]));
+      }
+    }
+    int num_gpus = local_devices.size();
+    for (const auto& device : local_devices) {
+      device_names.push_back(device->name());
+      VLOG(2) << device->name();
+    }
+    if (!dev_mgr_) dev_mgr_.reset(new DeviceMgr(std::move(local_devices)));
+    col_exec_ = new BaseCollectiveExecutor(
+        &col_exec_mgr_, /*remote_access=*/nullptr, kStepId, dev_mgr_.get(),
+        /*gpu_ring_order=*/nullptr);
+
+    // Initialize collective params.
+    col_params_.name = "test_nccl_collective_op";
+    const int group_key = 5;
+    col_params_.group.group_key = group_key;
+    col_params_.group.device_type = DEVICE_GPU;
+    col_params_.group.group_size = num_ranks;
+    const int instance_key = 23;
+    col_params_.instance.instance_key = instance_key;
+    col_params_.instance.type = REDUCTION_COLLECTIVE;
+    col_params_.instance.data_type = DT_FLOAT;
+    col_params_.instance.impl_details.collective_name = "NcclReduce";
+    const string task_name = "/job:worker/replica:0/task:0";
+    col_params_.instance.num_devices_per_task[task_name] = num_ranks;
+    for (int rank = 0; rank < num_ranks; ++rank) {
+      col_params_.instance.device_names.push_back(
+          device_names[rank % num_gpus]);
+      col_params_.instance.task_names.push_back(task_name);
+    }
+    for (int rank = 0; rank < num_ranks; ++rank) {
+      instances_.push_back(absl::make_unique<DeviceInstance>(
+          rank, col_params_.instance.device_names[rank], this));
+    }
+  }
+
+  void Reduce() {
+    int done = 0;
+    mutex done_mu;
+    condition_variable done_cv;
+    for (const auto& instance : instances_) {
+      DeviceInstance* di = instance.get();
+      SchedClosure([di, &done, &done_mu, &done_cv] {
+        di->DoReduce();
+        mutex_lock l(done_mu);
+        ++done;
+        done_cv.notify_all();
+      });
+    }
+
+    mutex_lock l(done_mu);
+    while (done < instances_.size()) done_cv.wait(l);
+  }
+
+  void RunTest(int num_ranks, int tensor_length) {
+    Init(num_ranks);
+    std::vector<float> expected(tensor_length, 0.0);
+    for (int rank = 0; rank < num_ranks; ++rank) {
+      DeviceInstance* instance = instances_[rank].get();
+      instance->InitTensor(DT_FLOAT, TensorShape({tensor_length}),
+                           [&expected, rank](Tensor* t) {
+                             for (size_t i = 0; i < t->NumElements(); ++i) {
+                               float value = pow(10, rank) * i;
+                               t->flat<float>()(i) = value;
+                               expected[i] += value;
+                             }
+                           });
+    }
+    Reduce();
+    // Confirm that every rank computed the same correct value.
+    for (int i = 0; i < tensor_length; ++i) {
+      expected[i] /= num_ranks;
+    }
+    for (int rank = 0; rank < instances_.size(); ++rank) {
+      TF_ASSERT_OK(instances_[rank]->status_);
+      Tensor* dev_tensor = &instances_[rank]->tensor_;
+      Tensor actual(DT_FLOAT, TensorShape({tensor_length}));
+      Notification note;
+      Device* dev = instances_[rank]->device_;
+      auto* dev_info = dev->tensorflow_gpu_device_info();
+      dev_info->default_context->CopyDeviceTensorToCPU(
+          dev_tensor, /*tensor_name=*/"", dev, &actual,
+          [&note](const Status&) { note.Notify(); });
+      note.WaitForNotification();
+      for (int i = 0; i < tensor_length; ++i) {
+        EXPECT_FLOAT_EQ(expected[i], actual.template flat<float>()(i))
+            << "Mismatch at rank " << rank << " index " << i;
+      }
+    }
+  }
+
+  std::unique_ptr<OpKernel> GetCollectiveReduce(const CollectiveParams& params,
+                                                Tensor* input,
+                                                DeviceBase* device) {
+    mutex_lock l(mu_);
+    NodeDef node_def;
+    NodeDefBuilder builder(
+        strings::StrCat("collective_reduce_", reduce_counter_++),
+        "CollectiveReduce");
+    TF_CHECK_OK(
+        builder.Attr("T", params.instance.data_type)
+            .Attr("merge_op", "Add")
+            .Attr("final_op", "Div")
+            .Attr("group_size", params.group.group_size)
+            .Attr("group_key", params.group.group_key)
+            .Attr("instance_key", params.instance.instance_key)
+            .Attr("subdiv_offsets", params.instance.impl_details.subdiv_offsets)
+            .Input(FakeInput(params.instance.data_type))
+            .Finalize(&node_def));
+    return GetKernel(node_def, device);
+  }
+
+  class DeviceInstance {
+   public:
+    DeviceInstance(int rank, const string& device_name, NcclReducerTest* parent)
+        : parent_(parent), device_name_(device_name), rank_(rank) {
+      TF_CHECK_OK(parent_->dev_mgr_->LookupDevice(device_name_, &device_))
+          << "Could not find device " << device_name_ << " existing devices "
+          << parent_->dev_mgr_->DebugString();
+      col_params_.name = parent_->col_params_.name;
+      col_params_.default_rank = rank;
+      col_params_.group.group_key = parent_->col_params_.group.group_key;
+      col_params_.group.device_type = parent_->col_params_.group.device_type;
+      col_params_.group.group_size = parent_->col_params_.group.group_size;
+      col_params_.instance = parent->col_params_.instance;
+    }
+
+    void InitTensor(DataType dtype, const TensorShape& shape,
+                    const std::function<void(Tensor*)>& init_f) {
+      tensor_ =
+          Tensor(device_->GetAllocator(AllocatorAttributes()), dtype, shape);
+      Tensor cpu_tensor(dtype, shape);
+      init_f(&cpu_tensor);
+      VLOG(2) << "cpu_tensor " << cpu_tensor.DebugString();
+      auto* dev_info = device_->tensorflow_gpu_device_info();
+      Notification note;
+      dev_info->default_context->CopyCPUTensorToDevice(
+          &cpu_tensor, device_, &tensor_,
+          [&note](const Status&) { note.Notify(); });
+      note.WaitForNotification();
+    }
+
+    void DoReduce() {
+      col_params_.merge_op = GetAdd(device_);
+      col_params_.final_op = GetDiv(device_);
+
+      // Prepare an OpKernelContext.
+      OpKernelContext::Params op_params;
+      op_params.step_id = kStepId;
+      op_params.device = device_;
+      gtl::InlinedVector<TensorValue, 4> inputs;
+      inputs.push_back(TensorValue(&tensor_));
+      op_params.inputs = &inputs;
+      gtl::InlinedVector<AllocatorAttributes, 4> input_aa(
+          {AllocatorAttributes()});
+      op_params.input_alloc_attrs = &input_aa;
+      gtl::InlinedVector<DeviceContext*, 4> input_dc;
+      DeviceContext* dev_ctx = nullptr;
+      auto* dev_info = device_->tensorflow_gpu_device_info();
+      if (dev_info) {
+        dev_ctx = dev_info->default_context;
+        dev_ctx->Ref();
+      } else {
+        dev_ctx = new DeviceContext;
+      }
+      input_dc.push_back(dev_ctx);
+      op_params.input_device_contexts = &input_dc;
+      op_params.op_device_context = dev_ctx;
+      int forward_from = 0;
+      op_params.forward_from_array = &forward_from;
+      AllocatorAttributes generic_alloc_attr;
+      op_params.output_attr_array = &generic_alloc_attr;
+      std::unique_ptr<OpKernel> op =
+          parent_->GetCollectiveReduce(col_params_, &tensor_, device_);
+      op_params.op_kernel = op.get();
+      OpKernelContext ctx(&op_params, 1);
+
+      // We never actually execute the kernel, so we need to do the output
+      // allocation it would do, ourselves.
+      Tensor* output_tensor_ptr = nullptr;
+      TF_CHECK_OK(ctx.forward_input_or_allocate_output({0}, 0, tensor_.shape(),
+                                                       &output_tensor_ptr));
+      CHECK_EQ(output_tensor_ptr, ctx.mutable_output(0));
+
+      // Prepare a NcclReducer instance.
+      string exec_key =
+          strings::StrCat(col_params_.instance.instance_key, ":0:0");
+      NcclReducer reducer;
+      CollectiveContext col_ctx(parent_->col_exec_, parent_->dev_mgr_.get(),
+                                &ctx, &op_params, col_params_, exec_key,
+                                kStepId, &tensor_, &tensor_);
+      TF_CHECK_OK(reducer.InitializeCollectiveContext(&col_ctx));
+
+      // Run the all-reduce.
+      reducer.Run([this](Status s) { status_ = s; });
+      if (status_.ok()) {
+        CHECK(tensor_.CopyFrom(*ctx.mutable_output(0), tensor_.shape()));
+      }
+
+      dev_ctx->Unref();
+    }
+
+    NcclReducerTest* parent_;
+    string device_name_;
+    int rank_;
+    Tensor tensor_;
+    Device* device_;
+    CollectiveParams col_params_;
+    Status status_;
+  };
+
+  std::vector<std::unique_ptr<tensorflow::Device>> gpus_;
+  TestCollectiveExecutorMgr col_exec_mgr_;
+  CollectiveExecutor* col_exec_;
+  std::unique_ptr<DeviceMgr> dev_mgr_;
+  std::vector<std::unique_ptr<DeviceInstance>> instances_;
+  CollectiveParams col_params_;
+  mutex mu_;
+  int32 reduce_counter_ GUARDED_BY(mu_) = 0;
+};
+
+TEST_F(NcclReducerTest, Test2Dev16Len) { RunTest(2, 16); }
+TEST_F(NcclReducerTest, Test4Dev16Len) { RunTest(4, 16); }
+TEST_F(NcclReducerTest, Test8Dev16Len) { RunTest(8, 16); }
+TEST_F(NcclReducerTest, Test8Dev128Len) { RunTest(8, 128); }
+TEST_F(NcclReducerTest, Test8Dev1045991Len) { RunTest(8, 1048576); }
+
+}  // namespace tensorflow
+
+#endif
diff --git a/tensorflow/core/kernels/collective_ops.cc b/tensorflow/core/kernels/collective_ops.cc
index 82e2913b64..ea582836c2 100644
--- a/tensorflow/core/kernels/collective_ops.cc
+++ b/tensorflow/core/kernels/collective_ops.cc
@@ -87,6 +87,9 @@ class CollectiveReduceOpKernel : public CollectiveOpKernel {
                     "final_op must be one of {\"Id\", \"Div\"} but got ",
                     final_op_name));
     OP_REQUIRES_OK(c, c->GetAttr("T", &col_params_.instance.data_type));
+    OP_REQUIRES_OK(c,
+                   c->GetAttr("wait_for",
+                              &col_params_.instance.impl_details.dependencies));
 
     const NodeDef& real_node = c->def();
     col_params_.name = strings::StrCat(real_node.name(), ": Reduce(",
diff --git a/tensorflow/core/nccl/BUILD b/tensorflow/core/nccl/BUILD
index 4be33b2a0c..a19e1af888 100644
--- a/tensorflow/core/nccl/BUILD
+++ b/tensorflow/core/nccl/BUILD
@@ -20,9 +20,11 @@ cc_library(
     name = "nccl_lib",
     srcs = if_cuda([
         "nccl_manager.cc",
-        "nccl_manager.h",
         "nccl_rewrite.cc",
     ]),
+    hdrs = if_cuda([
+        "nccl_manager.h",
+    ]),
     copts = tf_copts(),
     deps = if_cuda([
         "@local_config_nccl//:nccl",
diff --git a/tensorflow/core/protobuf/worker.proto b/tensorflow/core/protobuf/worker.proto
index 74058c8465..4284dd119e 100644
--- a/tensorflow/core/protobuf/worker.proto
+++ b/tensorflow/core/protobuf/worker.proto
@@ -535,6 +535,7 @@ message CompleteInstanceRequest {
 message CompleteInstanceResponse {
   int32 instance_key = 1;
   int32 source_rank = 2;
+  bytes communicator_key = 3;
 }
 
 // Request for next agreed-upon step_id for the specified graph_keys.
-- 
GitLab


From 92e0eea7b892d7169b994a0cddca04d8170f3db0 Mon Sep 17 00:00:00 2001
From: Blake Hechtman <blakehechtman@google.com>
Date: Mon, 14 Jan 2019 16:32:04 -0800
Subject: [PATCH 0692/2345] [XLA] Recurse through copies and reshapes to find
 bitcasts. Layout assignment may create an alignment of a reshape that
 eventually introduces unnecessary copies since there are many valid
 alignments of a reshape. Also platforms now get to entirely control what is
 considered a bitcast.

PiperOrigin-RevId: 229279534
---
 .../xla/service/algebraic_simplifier.cc       |  90 ++++++------
 .../xla/service/algebraic_simplifier.h        |  36 +++--
 .../xla/service/algebraic_simplifier_test.cc  | 132 ++++++++++++------
 .../compiler/xla/service/cpu/cpu_compiler.cc  |   6 +-
 .../xla/service/gpu/nvptx_compiler.cc         |   8 +-
 5 files changed, 163 insertions(+), 109 deletions(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index e28386d1bd..8a02c48146 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -122,23 +122,37 @@ bool TransposeIsBitcast(const HloInstruction* transpose) {
                                        transpose->dimensions());
 }
 
-// Returns true if the given reshape/copy produces a result which is bit-wise
-// identical to its operand and thus may be replaced with a bitcast.
-//
-// This function is conservative -- even if this function returns false, the
-// reshape may still be a bitcast. For example, a reshape from [28x28] to [784].
-bool ReshapeOrCopyIsBitcast(
-    const HloInstruction* instr,
-    const AlgebraicSimplifierOptions::ValidBitcastCallback&
-        valid_bitcast_callback) {
+// Recursive helper for method below.
+HloInstruction* BitcastingOperandOfReshapeOrCopyChainHelper(
+    HloInstruction* instr, HloInstruction* operand,
+    const AlgebraicSimplifierOptions& options) {
+  // Can't replace chain of copies and reshapes with bitcasts if the compiler
+  // used a memory layout which isn't compatible.
+  if (options.ReshapeIsBitcast(operand->shape(), instr->shape())) {
+    return operand;
+  }
+
+  // If the operand is a copy or reshape try to see if the operand's operand
+  // would produce a bitcast with initial instruction.
+  if (HloOpcode::kReshape == operand->opcode() ||
+      HloOpcode::kCopy == operand->opcode()) {
+    return BitcastingOperandOfReshapeOrCopyChainHelper(
+        instr, operand->mutable_operand(0), options);
+  }
+  return nullptr;
+}
+
+// Returns an operand of a chain of reshapes and copies that is bit-wise
+// identical to first reshape or copy in the chain.
+HloInstruction* BitcastingOperandOfReshapeOrCopyChain(
+    HloInstruction* instr, const AlgebraicSimplifierOptions& options) {
+  if (!options.is_layout_sensitive()) {
+    return nullptr;
+  }
   CHECK(HloOpcode::kReshape == instr->opcode() ||
         HloOpcode::kCopy == instr->opcode());
-
-  const HloInstruction* operand = instr->operand(0);
-  // Can't insert bitcasts if the compiler used a memory layout which isn't
-  // compatible.
-  return ShapeUtil::ReshapeIsBitcast(operand->shape(), instr->shape()) &&
-         valid_bitcast_callback(operand->shape(), instr->shape());
+  return BitcastingOperandOfReshapeOrCopyChainHelper(
+      instr, instr->mutable_operand(0), options);
 }
 
 bool IsUnstridedSlice(const HloInstruction* hlo) {
@@ -273,8 +287,11 @@ class AlgebraicSimplifierVisitor : public DfsHloVisitorWithDefault {
         shape, hlo, zero, {dim}, AddReduce_computation));
   }
 
-  // Convenience method for replacing an instruction with a bitcast.
-  void ReplaceWithBitcast(HloInstruction* instruction);
+  // Convenience method for replacing an instruction with a bitcast. If operand
+  // is not null, then the bitcast will use the specified operand instead of the
+  // operand of the instruction.
+  void ReplaceWithBitcast(HloInstruction* instruction,
+                          HloInstruction* operand = nullptr);
 
   // Replace old instruction with new instruction if old and new instructions
   // have the same shape. Updates uses and root instruction. Returns whether a
@@ -403,17 +420,19 @@ bool AlgebraicSimplifierVisitor::SameShape(const HloInstruction* lhs,
   }
 }
 
-void AlgebraicSimplifierVisitor::ReplaceWithBitcast(
-    HloInstruction* instruction) {
+void AlgebraicSimplifierVisitor::ReplaceWithBitcast(HloInstruction* instruction,
+                                                    HloInstruction* operand) {
   CHECK_EQ(1, instruction->operand_count());
+  if (operand == nullptr) {
+    operand = instruction->mutable_operand(0);
+  }
   CHECK_EQ(ShapeUtil::ElementsIn(instruction->shape()),
-           ShapeUtil::ElementsIn(instruction->operand(0)->shape()));
+           ShapeUtil::ElementsIn(operand->shape()));
   CHECK_EQ(ShapeUtil::ByteSizeOf(instruction->shape()),
-           ShapeUtil::ByteSizeOf(instruction->operand(0)->shape()));
+           ShapeUtil::ByteSizeOf(operand->shape()));
 
-  auto bitcast = computation_->AddInstruction(
-      HloInstruction::CreateUnary(instruction->shape(), HloOpcode::kBitcast,
-                                  instruction->mutable_operand(0)));
+  auto bitcast = computation_->AddInstruction(HloInstruction::CreateUnary(
+      instruction->shape(), HloOpcode::kBitcast, operand));
   TF_CHECK_OK(ReplaceInstruction(instruction, bitcast));
 }
 
@@ -574,9 +593,9 @@ Status AlgebraicSimplifierVisitor::HandleCopy(HloInstruction* copy) {
     return Status::OK();
   }
 
-  if (options_.is_layout_sensitive() &&
-      ReshapeOrCopyIsBitcast(copy, options_.valid_bitcast_callback())) {
-    ReplaceWithBitcast(copy);
+  if (HloInstruction* bitcast_operand =
+          BitcastingOperandOfReshapeOrCopyChain(copy, options_)) {
+    ReplaceWithBitcast(copy, bitcast_operand);
   }
 
   return Status::OK();
@@ -2196,12 +2215,10 @@ Status AlgebraicSimplifierVisitor::HandleReshape(HloInstruction* reshape) {
   }
 
   // Make this a bitcast if possible.
-  if (options_.is_layout_sensitive() &&
-      ReshapeOrCopyIsBitcast(reshape, options_.valid_bitcast_callback())) {
-    ReplaceWithBitcast(reshape);
-    return Status::OK();
+  if (HloInstruction* bitcast_operand =
+          BitcastingOperandOfReshapeOrCopyChain(reshape, options_)) {
+    ReplaceWithBitcast(reshape, bitcast_operand);
   }
-
   return Status::OK();
 }
 
@@ -3223,15 +3240,6 @@ StatusOr<bool> AlgebraicSimplifierVisitor::SimplifyConvToDot(
   const Shape dot_output_shape = ShapeUtil::MakeShapeWithDescendingLayout(
       convolution_shape.element_type(), {conv_width, output_channels});
 
-  // We cannot insert bitcasts if the layouts will not be compatible.
-  // TODO(b/33178038): Consider inserting a transpose if a bitcast would be
-  // invalid.
-  if (!options_.valid_bitcast_callback()(input_shape, new_input_shape) ||
-      !options_.valid_bitcast_callback()(filter_shape, new_filter_shape) ||
-      !options_.valid_bitcast_callback()(dot_output_shape, convolution_shape)) {
-    return false;
-  }
-
   auto new_lhs = add_bitcast(new_input_shape, lhs);
   auto new_rhs = add_bitcast(new_filter_shape, rhs);
   DotDimensionNumbers dot_dimension_numbers;
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.h b/tensorflow/compiler/xla/service/algebraic_simplifier.h
index 1acaadaaa0..ff3f638b22 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.h
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.h
@@ -25,21 +25,25 @@ namespace xla {
 
 class AlgebraicSimplifierOptions {
  public:
-  // Given shapes 'from_shape' and 'to_shape', determines if it is valid to
-  // bitcast from 'from_shape' to 'to_shape' after considering platform
-  // dependent effects on layout like alignment restrictions. Precondition: the
-  // two shapes have layouts, the same number of elements and
-  // ShapeUtil::ReshapeIsBitcast returns true.
-  using ValidBitcastCallback =
+  AlgebraicSimplifierOptions() {}
+  // Platform dependent callback to determine if a reshape `from_shape` to
+  // `to_shape` is a bitcast.
+  using ReshapeIsBitcastCallback =
       std::function<bool(const Shape& from_shape, const Shape& to_shape)>;
-
   explicit AlgebraicSimplifierOptions(
-      ValidBitcastCallback valid_bitcast_callback)
-      : valid_bitcast_callback_(std::move(valid_bitcast_callback)) {}
-  // If valid_bitcast_callback returns true, then the pass will replace reshapes
-  // and transposes with bitcasts.
-  const ValidBitcastCallback& valid_bitcast_callback() const {
-    return valid_bitcast_callback_;
+      ReshapeIsBitcastCallback reshape_is_bitcast_callback)
+      : reshape_is_bitcast_callback_(std::move(reshape_is_bitcast_callback)) {}
+
+  // Use the platform specific callback if set. It is not sensible to return
+  // true here if the options are not layout sensitive.
+  bool ReshapeIsBitcast(const Shape& from_shape, const Shape& to_shape) const {
+    if (!is_layout_sensitive_) {
+      return false;
+    }
+    if (!reshape_is_bitcast_callback_) {
+      return ShapeUtil::ReshapeIsBitcast(from_shape, to_shape);
+    }
+    return reshape_is_bitcast_callback_(from_shape, to_shape);
   }
 
   // If is_layout_sensitive is true, then the simplifier preserves layout during
@@ -47,12 +51,14 @@ class AlgebraicSimplifierOptions {
   void set_is_layout_sensitive(bool is_layout_sensitive) {
     is_layout_sensitive_ = is_layout_sensitive;
   }
+
   bool is_layout_sensitive() const { return is_layout_sensitive_; }
 
   // Enable dot simplification on platforms where it is profitable.
   void set_enable_dot_strength_reduction(bool enable_dot_strength_reduction) {
     enable_dot_strength_reduction_ = enable_dot_strength_reduction;
   }
+
   bool enable_dot_strength_reduction() const {
     return enable_dot_strength_reduction_;
   }
@@ -71,6 +77,7 @@ class AlgebraicSimplifierOptions {
       bool enable_permutation_sort_replacement) {
     enable_permutation_sort_replacement_ = enable_permutation_sort_replacement;
   }
+
   bool enable_permutation_sort_replacement() const {
     return enable_permutation_sort_replacement_;
   }
@@ -82,12 +89,13 @@ class AlgebraicSimplifierOptions {
     enable_window_reduce_to_reduce_replacement_ =
         enable_window_reduce_to_reduce_replacement;
   }
+
   bool enable_window_reduce_to_reduce_replacement() const {
     return enable_window_reduce_to_reduce_replacement_;
   }
 
  private:
-  ValidBitcastCallback valid_bitcast_callback_;
+  ReshapeIsBitcastCallback reshape_is_bitcast_callback_;
   bool is_layout_sensitive_{false};
   bool enable_dot_strength_reduction_{true};
   bool enable_conv_simplification_{true};
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index 74a3e8cca8..d2ff0c9f21 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -46,17 +46,9 @@ namespace {
 using ::testing::ElementsAre;
 namespace m = match;
 
-AlgebraicSimplifierOptions::ValidBitcastCallback bitcasting_callback() {
-  return [](const Shape&, const Shape&) { return true; };
-}
-
-AlgebraicSimplifierOptions::ValidBitcastCallback non_bitcasting_callback() {
-  return [](const Shape&, const Shape&) { return false; };
-}
-
 class AlgebraicSimplifierTest : public HloTestBase {
  protected:
-  AlgebraicSimplifierOptions default_options_{non_bitcasting_callback()};
+  AlgebraicSimplifierOptions default_options_;
 };
 
 // Test that A + 0 is simplified to A
@@ -1464,23 +1456,77 @@ TEST_F(AlgebraicSimplifierTest, RemoveCopy) {
   EXPECT_THAT(computation->root_instruction(), param0);
 }
 
-TEST_F(AlgebraicSimplifierTest, CopyEqualsBitcast) {
+TEST_F(AlgebraicSimplifierTest, CopyOfReshapeOfCopyEqualsBitcast) {
   auto m = CreateNewVerifiedModule();
   HloComputation::Builder builder(TestName());
   HloInstruction* param =
       builder.AddInstruction(HloInstruction::CreateParameter(
-          0, ShapeUtil::MakeShape(F32, {1, 14, 14, 64}), "param"));
-  *param->mutable_shape()->mutable_layout() =
-      LayoutUtil::MakeLayout({0, 1, 2, 3});
+          0, ShapeUtil::MakeShapeWithLayout(F32, {1, 14, 14, 64}, {3, 2, 1, 0}),
+          "param"));
   HloInstruction* copy = builder.AddInstruction(HloInstruction::CreateUnary(
-      ShapeUtil::MakeShape(F32, {1, 14, 14, 64}), HloOpcode::kCopy, param));
-  *copy->mutable_shape()->mutable_layout() =
-      LayoutUtil::MakeLayout({1, 2, 0, 3});
+      ShapeUtil::MakeShapeWithLayout(F32, {1, 14, 14, 64}, {0, 1, 2, 3}),
+      HloOpcode::kCopy, param));
+  HloInstruction* reshape =
+      builder.AddInstruction(HloInstruction::CreateReshape(
+          ShapeUtil::MakeShapeWithLayout(F32, {14 * 14, 64}, {0, 1}), copy));
+  builder.AddInstruction(HloInstruction::CreateUnary(
+      ShapeUtil::MakeShapeWithLayout(F32, {14 * 14, 64}, {1, 0}),
+      HloOpcode::kCopy, reshape));
+  auto computation = m->AddEntryComputation(builder.Build());
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Copy(m::Reshape(m::Copy(m::Parameter(0))))));
+
+  AlgebraicSimplifierOptions options;
+  options.set_is_layout_sensitive(true);
+  AlgebraicSimplifier simplifier(options);
+  ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
+  // Verify that the copy of reshape of copy is replaced.
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Bitcast(m::Parameter(0))));
+}
+
+TEST_F(AlgebraicSimplifierTest, ReshapeOfCopyEqualsBitcast) {
+  auto m = CreateNewVerifiedModule();
+  HloComputation::Builder builder(TestName());
+  HloInstruction* param =
+      builder.AddInstruction(HloInstruction::CreateParameter(
+          0, ShapeUtil::MakeShapeWithLayout(F32, {1, 14, 14, 64}, {3, 2, 1, 0}),
+          "param"));
+  HloInstruction* copy = builder.AddInstruction(HloInstruction::CreateUnary(
+      ShapeUtil::MakeShapeWithLayout(F32, {1, 14, 14, 64}, {0, 1, 2, 3}),
+      HloOpcode::kCopy, param));
+  builder.AddInstruction(HloInstruction::CreateReshape(
+      ShapeUtil::MakeShapeWithLayout(F32, {14 * 14, 64}, {1, 0}), copy));
+
+  auto computation = m->AddEntryComputation(builder.Build());
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Reshape(m::Copy(m::Parameter(0)))));
+
+  AlgebraicSimplifierOptions options;
+  options.set_is_layout_sensitive(true);
+  AlgebraicSimplifier simplifier(options);
+  ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
+  // Verify that the copy of reshape of copy is replaced.
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Bitcast(m::Parameter(0))));
+}
+
+TEST_F(AlgebraicSimplifierTest, CopyEqualsBitcast) {
+  auto m = CreateNewVerifiedModule();
+  HloComputation::Builder builder(TestName());
+  HloInstruction* param =
+      builder.AddInstruction(HloInstruction::CreateParameter(
+          0, ShapeUtil::MakeShapeWithLayout(F32, {1, 14, 14, 64}, {0, 1, 2, 3}),
+          "param"));
+  builder.AddInstruction(HloInstruction::CreateUnary(
+      ShapeUtil::MakeShapeWithLayout(F32, {1, 14, 14, 64}, {1, 2, 0, 3}),
+      HloOpcode::kCopy, param));
   auto computation = m->AddEntryComputation(builder.Build());
   EXPECT_THAT(computation->root_instruction(),
               GmockMatch(m::Copy(m::Parameter(0))));
 
-  AlgebraicSimplifierOptions options(non_bitcasting_callback());
+  AlgebraicSimplifierOptions options(
+      [](const Shape&, const Shape&) { return false; });
   options.set_is_layout_sensitive(true);
   AlgebraicSimplifier simplifier1(options);
   ASSERT_FALSE(simplifier1.Run(m.get()).ValueOrDie());
@@ -1488,10 +1534,10 @@ TEST_F(AlgebraicSimplifierTest, CopyEqualsBitcast) {
   EXPECT_THAT(computation->root_instruction(),
               GmockMatch(m::Copy(m::Parameter(0))));
 
-  AlgebraicSimplifierOptions options2(bitcasting_callback());
+  AlgebraicSimplifierOptions options2;
   options2.set_is_layout_sensitive(true);
   AlgebraicSimplifier simplifier2(options2);
-  ASSERT_TRUE(simplifier2.Run(m.get()).ValueOrDie());
+  EXPECT_TRUE(simplifier2.Run(m.get()).ValueOrDie());
   // Verify that the copy is replaced.
   EXPECT_THAT(computation->root_instruction(),
               GmockMatch(m::Bitcast(m::Parameter(0))));
@@ -1744,7 +1790,7 @@ TEST_F(AlgebraicSimplifierTest, CopyWithDifferentLayout) {
   EXPECT_THAT(computation->root_instruction(),
               GmockMatch(m::Copy(m::Parameter(0))));
 
-  AlgebraicSimplifierOptions options(non_bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   options.set_is_layout_sensitive(true);
   AlgebraicSimplifier simplifier(options);
   EXPECT_FALSE(simplifier.Run(m.get()).ValueOrDie());
@@ -1774,7 +1820,7 @@ TEST_F(AlgebraicSimplifierTest, CopyWithSameLayout) {
   EXPECT_THAT(computation->root_instruction(),
               GmockMatch(m::Copy(m::Parameter(0))));
 
-  AlgebraicSimplifierOptions options(non_bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   options.set_is_layout_sensitive(true);
   AlgebraicSimplifier simplifier(options);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
@@ -1804,7 +1850,8 @@ TEST_F(AlgebraicSimplifierTest, NoBitcastAdded) {
   EXPECT_THAT(computation->root_instruction(),
               GmockMatch(m::Reshape(m::Parameter(0))));
 
-  AlgebraicSimplifierOptions options(non_bitcasting_callback());
+  AlgebraicSimplifierOptions options(
+      [](const Shape&, const Shape&) { return false; });
   options.set_is_layout_sensitive(true);
   AlgebraicSimplifier simplifier(options);
   EXPECT_FALSE(simplifier.Run(m.get()).ValueOrDie());
@@ -1835,8 +1882,7 @@ TEST_F(AlgebraicSimplifierTest, ReshapeOfTransposeOfRngToRng) {
 
   auto computation = m->AddEntryComputation(builder.Build());
 
-  AlgebraicSimplifier simplifier(
-      (AlgebraicSimplifierOptions(bitcasting_callback())));
+  AlgebraicSimplifier simplifier(AlgebraicSimplifierOptions{});
   EXPECT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   // Verify that reshape(transpose(rng)) is replace by a single rng of the
@@ -1887,7 +1933,7 @@ TEST_F(AlgebraicSimplifierTest, ReshapeReplacedWithBitcast) {
                                   m::Op().Is(dimensions_wrong_reshape),
                                   m::Op().Is(layout_wrong_reshape))));
 
-  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   options.set_is_layout_sensitive(true);
   AlgebraicSimplifier simplifier(options);
   simplifier.Run(m.get()).ValueOrDie();
@@ -1917,8 +1963,7 @@ TEST_F(AlgebraicSimplifierTest, FailureToSinkReshapeDoesntAffectChangedBit) {
   builder.AddInstruction(
       HloInstruction::CreateReshape(ShapeUtil::MakeShape(F32, {4}), add));
 
-  AlgebraicSimplifier simplifier(
-      (AlgebraicSimplifierOptions(bitcasting_callback())));
+  AlgebraicSimplifier simplifier(AlgebraicSimplifierOptions{});
   m->AddEntryComputation(builder.Build());
   EXPECT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 }
@@ -1942,8 +1987,7 @@ TEST_F(AlgebraicSimplifierTest, FailureToSinkBroadcastDoesntAffectChangedBit) {
       HloInstruction::CreateBroadcast(ShapeUtil::MakeShape(F32, {2, 2, 2}), add,
                                       /*broadcast_dimensions=*/{0, 1}));
 
-  AlgebraicSimplifier simplifier(
-      (AlgebraicSimplifierOptions(bitcasting_callback())));
+  AlgebraicSimplifier simplifier(AlgebraicSimplifierOptions{});
   m->AddEntryComputation(builder.Build());
   EXPECT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 }
@@ -1968,7 +2012,7 @@ TEST_F(AlgebraicSimplifierTest, TransposeEqualsBitcast1) {
   EXPECT_THAT(computation->root_instruction(),
               GmockMatch(m::Transpose(m::Parameter(0))));
 
-  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   options.set_is_layout_sensitive(true);
   AlgebraicSimplifier simplifier(options);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
@@ -1998,7 +2042,7 @@ TEST_F(AlgebraicSimplifierTest, TransposeEqualsBitcast2) {
   EXPECT_THAT(computation->root_instruction(),
               GmockMatch(m::Transpose(m::Parameter(0))));
 
-  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   options.set_is_layout_sensitive(true);
   AlgebraicSimplifier simplifier(options);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
@@ -2055,7 +2099,7 @@ TEST_F(AlgebraicSimplifierTest, CopiesMerged) {
   EXPECT_THAT(computation->root_instruction(),
               GmockMatch(m::Copy(m::Copy(m::Parameter(0)))));
 
-  AlgebraicSimplifierOptions options(non_bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   options.set_is_layout_sensitive(true);
   AlgebraicSimplifier simplifier(options);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
@@ -2651,7 +2695,7 @@ TEST_F(AlgebraicSimplifierTest, ReplacePermutationSortWithScatter) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifierOptions options(non_bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   options.set_enable_permutation_sort_replacement(true);
   AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
@@ -2680,7 +2724,7 @@ TEST_F(AlgebraicSimplifierTest, DontReplacePermutationSortIfNonIntegral) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifierOptions options(non_bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   options.set_enable_permutation_sort_replacement(true);
   AlgebraicSimplifier simplifier(options);
   EXPECT_FALSE(simplifier.Run(module.get()).ValueOrDie());
@@ -2703,7 +2747,7 @@ TEST_F(AlgebraicSimplifierTest, DontReplacePermutationSortWrongDimensions) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifierOptions options(non_bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   options.set_enable_permutation_sort_replacement(true);
   AlgebraicSimplifier simplifier(options);
   EXPECT_FALSE(simplifier.Run(module.get()).ValueOrDie());
@@ -3292,7 +3336,7 @@ TEST_F(AlgebraicSimplifierTest, ConvertConvToMatmul) {
     auto module = CreateNewUnverifiedModule();
     auto* computation = module->AddEntryComputation(b.Build());
 
-    AlgebraicSimplifierOptions simplifier_options(bitcasting_callback());
+    AlgebraicSimplifierOptions simplifier_options;
     simplifier_options.set_is_layout_sensitive(true);
     AlgebraicSimplifier simplifier(simplifier_options);
     if (!simplifier.Run(module.get()).ValueOrDie()) {
@@ -3869,7 +3913,7 @@ TEST_F(AlgebraicSimplifierTest, SliceOfPadLow) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   auto root = module->entry_computation()->root_instruction();
@@ -3890,7 +3934,7 @@ TEST_F(AlgebraicSimplifierTest, SliceOfPadHigh) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   auto root = module->entry_computation()->root_instruction();
@@ -3911,7 +3955,7 @@ TEST_F(AlgebraicSimplifierTest, SliceOfPadMidNonScalar) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   AlgebraicSimplifier simplifier(options);
   EXPECT_FALSE(simplifier.Run(module.get()).ValueOrDie());
 }
@@ -3930,7 +3974,7 @@ TEST_F(AlgebraicSimplifierTest, SliceOfPadMidScalar) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   auto root = module->entry_computation()->root_instruction();
@@ -3952,7 +3996,7 @@ TEST_F(AlgebraicSimplifierTest, SliceOfConcatScalarInput) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   auto root = module->entry_computation()->root_instruction();
@@ -3974,7 +4018,7 @@ TEST_F(AlgebraicSimplifierTest, SliceOfConcatNonScalarInput) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   auto root = module->entry_computation()->root_instruction();
@@ -3996,7 +4040,7 @@ TEST_F(AlgebraicSimplifierTest, NegateNegate) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   auto root = module->entry_computation()->root_instruction();
@@ -4016,7 +4060,7 @@ TEST_F(AlgebraicSimplifierTest, NotNot) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   auto root = module->entry_computation()->root_instruction();
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
index 34a4f008ec..eafda68510 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
@@ -282,8 +282,7 @@ Status CpuCompiler::RunHloPassesThroughLayoutAssn(
         /*rewrite_inference_op=*/true,
         /*rewrite_grad_op=*/true);
     pipeline.AddPass<HloGetDimensionSizeRewriter>();
-    AlgebraicSimplifierOptions options(
-        [](const Shape&, const Shape&) { return false; });
+    AlgebraicSimplifierOptions options;
     options.set_enable_dot_strength_reduction(false);
     pass.AddPass<AlgebraicSimplifier>(options);
     pass.AddPass<SortSimplifier>();
@@ -350,8 +349,7 @@ Status CpuCompiler::RunHloPassesAfterLayoutAssn(
     pass.AddInvariantChecker<HloVerifier>(
         /*layout_sensitive=*/true,
         /*allow_mixed_precision=*/false);
-    AlgebraicSimplifierOptions options(
-        [](const Shape&, const Shape&) { return true; });
+    AlgebraicSimplifierOptions options;
     options.set_is_layout_sensitive(true);
     options.set_enable_dot_strength_reduction(false);
     pass.AddPass<HloPassFix<AlgebraicSimplifier>>(options);
diff --git a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
index 70485c62a9..48f718b514 100644
--- a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
+++ b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
@@ -199,8 +199,7 @@ Status OptimizeHloModule(HloModule* hlo_module, se::StreamExecutor* stream_exec,
       // elimination has to come after that pass.
       pipeline.AddPass<ZeroSizedHloElimination>();
 
-      AlgebraicSimplifierOptions options(
-          [](const Shape&, const Shape&) { return false; });
+      AlgebraicSimplifierOptions options;
       options.set_enable_permutation_sort_replacement(true);
       pass.AddPass<AlgebraicSimplifier>(options);
       pass.AddPass<SortSimplifier>();
@@ -272,10 +271,7 @@ Status OptimizeHloModule(HloModule* hlo_module, se::StreamExecutor* stream_exec,
 
     // The LayoutAssignment pass may leave behind kCopy instructions which are
     // duplicate or NOPs, so remove them with algebraic simplification and CSE.
-    AlgebraicSimplifierOptions options(
-        /*valid_bitcast_callback=*/[](const Shape&, const Shape&) {
-          return true;
-        });
+    AlgebraicSimplifierOptions options;
     options.set_is_layout_sensitive(true);
     options.set_enable_permutation_sort_replacement(true);
     pipeline.AddPass<HloPassFix<AlgebraicSimplifier>>(options);
-- 
GitLab


From 3c922d7df747ce3c25a0ad75a41f23c7e8d1df1e Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Mon, 14 Jan 2019 16:50:14 -0800
Subject: [PATCH 0693/2345] Print out bounded shape in HumanStringWithLayout

PiperOrigin-RevId: 229282536
---
 tensorflow/compiler/xla/shape_util.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc
index 9821790cb3..f1ef202da0 100644
--- a/tensorflow/compiler/xla/shape_util.cc
+++ b/tensorflow/compiler/xla/shape_util.cc
@@ -530,7 +530,8 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
   string result = StrCat(
       primitive_util::LowercasePrimitiveTypeName(shape.element_type()), "[");
   for (int i = 0; i < shape.dimensions().size(); i++) {
-    StrAppend(&result, (i > 0) ? "," : "", shape.dimensions(i));
+    StrAppend(&result, (i > 0) ? "," : "",
+              shape.is_dynamic_dimension(i) ? "<=" : "", shape.dimensions(i));
   }
   result += "]";
   if (!IsScalar(shape) && shape.IsArray()) {
-- 
GitLab


From b0cc10d27608219e44d36f04c97a5aeff9b79832 Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Mon, 14 Jan 2019 17:10:11 -0800
Subject: [PATCH 0694/2345] Set the data type of the IConstantLayer output
 correctly.

PiperOrigin-RevId: 229285591
---
 .../contrib/tensorrt/convert/convert_nodes.cc | 49 ++++++++++---------
 .../contrib/tensorrt/convert/convert_nodes.h  |  5 ++
 .../tensorrt/convert/convert_nodes_test.cc    | 12 +++++
 3 files changed, 44 insertions(+), 22 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index dcb84dfb93..a1c519d908 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -334,6 +334,20 @@ Status Converter::GetTrtBroadcastShape(
   return Status::OK();
 }
 
+nvinfer1::ITensor* Converter::CreateConstantLayer(
+    const TRT_ShapedWeights& weights, const nvinfer1::Dims& dims) {
+  nvinfer1::Weights trt_weights = weights.GetTrtWeights();
+  nvinfer1::IConstantLayer* layer = network()->addConstant(dims, trt_weights);
+  if (!layer) return nullptr;
+  const nvinfer1::DataType trt_dtype = trt_weights.type;
+  // NOTE(laigd): calling layer->setOutputType() does not work. This may be a
+  // bug and we should report to NVIDIA.
+  // layer->setOutputType(0, trt_dtype);
+  nvinfer1::ITensor* trt_tensor = layer->getOutput(0);
+  trt_tensor->setType(trt_dtype);
+  return trt_tensor;
+}
+
 inline bool DimsEqual(const nvinfer1::Dims& dim_l,
                       const nvinfer1::Dims& dim_r) {
   if (dim_l.nbDims != dim_r.nbDims) {
@@ -1104,10 +1118,8 @@ Status Converter::PrepareTensorForShape(const TRT_TensorOrWeights& input,
       *tensor = layer->getOutput(0);
     }
   } else {
-    nvinfer1::IConstantLayer* layer =
-        this->network()->addConstant(dims, input.weights().GetTrtWeights());
-    TFTRT_RETURN_ERROR_IF_NULLPTR(layer, "TF-TRT Internal Reshape");
-    *tensor = layer->getOutput(0);
+    *tensor = CreateConstantLayer(input.weights(), dims);
+    TFTRT_RETURN_ERROR_IF_NULLPTR(*tensor, "TF-TRT Internal Reshape");
     if (precision_mode() == INT8MODE && !use_calibration()) {
       // If we are in int8 mode and not calibrating, we need to explicitly set a
       // quantization range for the output tensor of the IConstantLayer. Here we
@@ -2570,22 +2582,18 @@ tensorflow::Status ConvertRelu6(OpConverterParams* params) {
   auto weights_ptr =
       static_cast<float*>(const_cast<void*>(weights.GetValues()));
   weights_ptr[0] = 6.0f;
-  nvinfer1::IConstantLayer* const6_layer =
-      params->converter->network()->addConstant(dims, weights.GetTrtWeights());
-  TFTRT_RETURN_ERROR_IF_NULLPTR(const6_layer, node_def.name());
-  params->converter->ProvideQuantizationRange(const6_layer->getOutput(0), 0.0f,
-                                              6.0f);
+  nvinfer1::ITensor* const6_tensor =
+      params->converter->CreateConstantLayer(weights, dims);
+  TFTRT_RETURN_ERROR_IF_NULLPTR(const6_tensor, node_def.name());
+  params->converter->ProvideQuantizationRange(const6_tensor, 0.0f, 6.0f);
 
   // ElementWise Min Operation
   // Min op is a nop for INT8 execution path, as the input tensor
   // to this layer will only have values in range [0.f, 6.0f].
-  const nvinfer1::ITensor* tensor_l = relu_layer->getOutput(0);
-  const nvinfer1::ITensor* tensor_r = const6_layer->getOutput(0);
   nvinfer1::IElementWiseLayer* relu6_layer =
       params->converter->network()->addElementWise(
-          *const_cast<nvinfer1::ITensor*>(tensor_l),
-          *const_cast<nvinfer1::ITensor*>(tensor_r),
-          nvinfer1::ElementWiseOperation::kMIN);
+          *const_cast<nvinfer1::ITensor*>(relu_layer->getOutput(0)),
+          *const6_tensor, nvinfer1::ElementWiseOperation::kMIN);
   TFTRT_RETURN_ERROR_IF_NULLPTR(relu6_layer, node_def.name());
   nvinfer1::ITensor* output_tensor = relu6_layer->getOutput(0);
   params->converter->ProvideQuantizationRange(output_tensor, 0.0f, 6.0f);
@@ -2989,18 +2997,15 @@ tensorflow::Status ConvertSquare(OpConverterParams* params) {
   auto weights_ptr =
       static_cast<float*>(const_cast<void*>(weights.GetValues()));
   weights_ptr[0] = 2.f;
-  nvinfer1::IConstantLayer* const2_layer =
-      params->converter->network()->addConstant(dims, weights.GetTrtWeights());
-  TFTRT_RETURN_ERROR_IF_NULLPTR(const2_layer, node_def.name());
+  nvinfer1::ITensor* const2_tensor =
+      params->converter->CreateConstantLayer(weights, dims);
+  TFTRT_RETURN_ERROR_IF_NULLPTR(const2_tensor, node_def.name());
 
   // ElementWise Pow Operation
-  const nvinfer1::ITensor* tensor_l = inputs.at(0).tensor();
-  const nvinfer1::ITensor* tensor_r = const2_layer->getOutput(0);
   nvinfer1::IElementWiseLayer* layer =
       params->converter->network()->addElementWise(
-          *const_cast<nvinfer1::ITensor*>(tensor_l),
-          *const_cast<nvinfer1::ITensor*>(tensor_r),
-          nvinfer1::ElementWiseOperation::kPOW);
+          *const_cast<nvinfer1::ITensor*>(inputs.at(0).tensor()),
+          *const2_tensor, nvinfer1::ElementWiseOperation::kPOW);
   TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name());
   nvinfer1::ITensor* output_tensor = layer->getOutput(0);
 
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.h b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
index 8f2271ee3f..25949b7f07 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
@@ -468,6 +468,11 @@ class Converter {
                               nvinfer1::Dims* operand_l_new_dims,
                               nvinfer1::Dims* operand_r_new_dims) const;
 
+  // Creates an IConstantLayer using 'weights' whose dimensions are specified by
+  // 'dims', and returns the output ITensor.
+  nvinfer1::ITensor* CreateConstantLayer(const TRT_ShapedWeights& weights,
+                                         const nvinfer1::Dims& dims);
+
  private:
   // Verify the provided batch_size is consistent with batch_size_ and update it
   // if necessary.
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index f143f56d20..36dbbe3f6f 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -915,6 +915,18 @@ TEST_F(ConverterTest, GetTrtBroadcastShape) {
                  "(tensor #dims 4 vs broadcast #dims 5)");
 }
 
+TEST_F(ConverterTest, CreateConstantLayer) {
+  for (auto dtype : {DT_FLOAT, DT_INT32}) {
+    TRT_ShapedWeights weights =
+        weight_store_->GetTempWeights(dtype, GetTestDims({2, 3, 5}));
+    nvinfer1::ITensor* tensor =
+        converter_->CreateConstantLayer(weights, GetTestDims({3, 10}));
+    ASSERT_NE(nullptr, tensor);
+    ExpectTrtDimsEqualsArray({3, 10}, tensor->getDimensions());
+    EXPECT_EQ(TfDataTypeToTrt(dtype), tensor->getType());
+  }
+}
+
 // Class to test various op converters, using both a TrtNodeValidator and
 // Converter.
 class OpConverterTest : public ::testing::Test {
-- 
GitLab


From 1f1a4a0a5eb71517b08ebb5e670c1400c944c996 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Mon, 14 Jan 2019 17:32:18 -0800
Subject: [PATCH 0695/2345] Add a revived type for auto-generated list wrappers

Adds a bit of plumbing for the proto and adds a bit of flexibility in _ListWrapper so that the restore process doesn't make it unsaveable (previously any __setitem__ would mark the list as dirty).

PiperOrigin-RevId: 229288388
---
 tensorflow/python/saved_model/load_test.py    | 11 +++++++
 .../checkpointable/data_structures.py         | 29 +++++++++++++++++--
 2 files changed, 37 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index ca9e16d14a..ff6b72053b 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -428,6 +428,17 @@ class LoadTest(test.TestCase):
     self.assertEqual(2., imported.variables["b"].numpy())
     self.assertEqual(set(["a", "b"]), set(imported.variables.keys()))
 
+  def test_list(self):
+    root = tracking.Checkpointable()
+    root.variables = [variables.Variable(1.)]
+    root.variables.append(1)
+    root.variables.append(variables.Variable(3.))
+    imported = self.cycle(root)
+    self.assertEqual(1., imported.variables[0].numpy())
+    self.assertEqual(3., imported.variables[2].numpy())
+    self.assertIs(None, imported.variables[1])
+    self.assertEqual(3, len(imported.variables))
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/training/checkpointable/data_structures.py b/tensorflow/python/training/checkpointable/data_structures.py
index c46af68155..759709aa3d 100644
--- a/tensorflow/python/training/checkpointable/data_structures.py
+++ b/tensorflow/python/training/checkpointable/data_structures.py
@@ -406,8 +406,11 @@ class _ListWrapper(List, collections.MutableSequence,
     del self._storage[key]
 
   def __setitem__(self, key, value):
-    self._non_append_mutation = True
-    self._storage[key] = value
+    self._check_external_modification()
+    if isinstance(self._storage[key], base.CheckpointableBase):
+      self._non_append_mutation = True
+    self._storage[key] = self._track_value(value, self._name_element(key))
+    self._update_snapshot()
 
   def append(self, value):
     """Add a new checkpointable value."""
@@ -734,8 +737,28 @@ revived_types.register_revived_type(
     "checkpointable_dict_wrapper",
     lambda obj: isinstance(obj, _DictWrapper),
     versions=[revived_types.VersionedTypeRegistration(
-        object_factory=lambda _: _DictWrapper({}),
+        # Standard dependencies are enough to reconstruct the checkpointable
+        # items in dictionaries, so we don't need to save any extra information.
+        object_factory=lambda proto: _DictWrapper({}),
         version=1,
         min_producer_version=1,
         min_consumer_version=1,
         setter=operator.setitem)])
+
+
+def _set_list_item(list_object, index_string, value):
+  item_index = int(index_string)
+  if len(list_object) <= item_index:
+    list_object.extend([None] * (1 + item_index - len(list_object)))
+  list_object[item_index] = value
+
+
+revived_types.register_revived_type(
+    "checkpointable_list_wrapper",
+    lambda obj: isinstance(obj, _ListWrapper),
+    versions=[revived_types.VersionedTypeRegistration(
+        object_factory=lambda proto: _ListWrapper([]),
+        version=1,
+        min_producer_version=1,
+        min_consumer_version=1,
+        setter=_set_list_item)])
-- 
GitLab


From 5e347da46abef1ef77356a7b85e2c55ceb44326e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 17:33:29 -0800
Subject: [PATCH 0696/2345] Use `str_util::EndsWith` to find suffixes in
 latency_all_edges.cc.

PiperOrigin-RevId: 229288546
---
 .../core/grappler/optimizers/data/latency_all_edges.cc | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc b/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc
index ff0fdbc131..0b7b2eba86 100644
--- a/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc
+++ b/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc
@@ -24,6 +24,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.h"
 #include "tensorflow/core/grappler/optimizers/data/graph_utils.h"
 #include "tensorflow/core/grappler/utils.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/protobuf.h"
 
@@ -72,10 +73,8 @@ Status LatencyAllEdges::Optimize(Cluster* cluster, const GrapplerItem& item,
   // TODO(shivaniagrawal): Add Op to return Latency for the particular Op than
   // for the edge (e2 - e1?).
   for (const NodeDef& node : item.graph.node()) {
-    if (node.op().rfind("Dataset") != node.op().size() - strlen("Dataset") ||
-        node.attr().empty() ||
-        node.name().rfind("_generated") ==
-            node.name().size() - strlen("_generated")) {
+    if (!str_util::EndsWith(node.op(), "Dataset") || node.attr().empty() ||
+        str_util::EndsWith(node.name(), "_generated")) {
       // TODO(b/111805951): Replace this with non-approximate way to check if
       // node corresponds to a `Dataset` op.
       continue;
@@ -89,8 +88,7 @@ Status LatencyAllEdges::Optimize(Cluster* cluster, const GrapplerItem& item,
     } else {  // fanout will have size 0 for last dataset node in the pipeline.
       if (fanout.size() == 1) {
         NodeDef* output_node = (*(fanout.begin())).node;
-        if (output_node->name().rfind("_generated") ==
-            output_node->name().size() - strlen("_generated")) {
+        if (str_util::EndsWith(output_node->name(), "_generated")) {
           continue;
         }
       }
-- 
GitLab


From 19f14821f261b290d4dc85a50f33900274434ed4 Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Mon, 14 Jan 2019 17:36:26 -0800
Subject: [PATCH 0697/2345] Fix several TFLite tests

PiperOrigin-RevId: 229288943
---
 tensorflow/lite/delegates/nnapi/nnapi_delegate.cc        | 1 +
 .../lite/experimental/micro/examples/micro_speech/BUILD  | 3 +++
 tensorflow/lite/models/smartreply/BUILD                  | 1 -
 tensorflow/lite/profiling/BUILD                          | 5 +++--
 tensorflow/lite/python/convert_test.py                   | 3 ++-
 tensorflow/lite/testing/nnapi_example.cc                 | 9 ++++++---
 6 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
index 8af159e6fb..94fcc8740b 100644
--- a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
+++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
@@ -13,6 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 #include <cstdarg>
+#include <cstring>
 #include <iostream>
 #include <memory>
 #include <vector>
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD b/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD
index 702b893e6f..171d07cf56 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD
@@ -216,6 +216,9 @@ tflite_micro_cc_test(
     srcs = [
         "recognize_commands_test.cc",
     ],
+    tags = [
+        "no_oss",  # TODO(122853023): Resolve issues and re-enable.
+    ],
     deps = [
         ":recognize_commands",
         "//tensorflow/lite/c:c_api_internal",
diff --git a/tensorflow/lite/models/smartreply/BUILD b/tensorflow/lite/models/smartreply/BUILD
index fccbf79eb9..9b2f0da64c 100644
--- a/tensorflow/lite/models/smartreply/BUILD
+++ b/tensorflow/lite/models/smartreply/BUILD
@@ -50,7 +50,6 @@ cc_library(
     ],
 )
 
-# TODO(b/118895218): Make this test compatible with oss.
 tf_cc_test(
     name = "predictor_test",
     srcs = ["predictor_test.cc"],
diff --git a/tensorflow/lite/profiling/BUILD b/tensorflow/lite/profiling/BUILD
index 89ad726782..64723ba385 100644
--- a/tensorflow/lite/profiling/BUILD
+++ b/tensorflow/lite/profiling/BUILD
@@ -2,6 +2,7 @@ package(default_visibility = ["//visibility:public"])
 
 licenses(["notice"])  # Apache 2.0
 
+load("//tensorflow:tensorflow.bzl", "tf_cc_test")
 load("//tensorflow/lite:build_def.bzl", "tflite_copts")
 
 common_copts = [
@@ -65,10 +66,10 @@ cc_library(
     ],
 )
 
-cc_test(
+tf_cc_test(
     name = "profile_summarizer_test",
     srcs = ["profile_summarizer_test.cc"],
-    copts = common_copts,
+    extra_copts = common_copts,
     deps = [
         ":profile_summarizer",
         "//tensorflow/lite:framework",
diff --git a/tensorflow/lite/python/convert_test.py b/tensorflow/lite/python/convert_test.py
index a29d431322..e270abaa5a 100644
--- a/tensorflow/lite/python/convert_test.py
+++ b/tensorflow/lite/python/convert_test.py
@@ -410,7 +410,8 @@ class ConvertTestOpHint(test_util.TensorFlowTestCase):
           _node_name(output_1.name),
           _node_name(output_2.name)
       ]
-      self.assertCountEqual(hinted_outputs_nodes, expected_hinted_output_nodes)
+      self.assertEqual(
+          len(hinted_outputs_nodes), len(expected_hinted_output_nodes))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/lite/testing/nnapi_example.cc b/tensorflow/lite/testing/nnapi_example.cc
index 22df8dbd88..0ff1fbcd41 100644
--- a/tensorflow/lite/testing/nnapi_example.cc
+++ b/tensorflow/lite/testing/nnapi_example.cc
@@ -25,11 +25,14 @@ limitations under the License.
 #include <fstream>
 #include <iostream>
 #include <sstream>
+#include <string>
 #include "tensorflow/lite/nnapi/NeuralNetworksShim.h"
 #include "tensorflow/lite/testing/parse_testdata.h"
 #include "tensorflow/lite/testing/tflite_driver.h"
 
-string dirname(const string& s) { return s.substr(0, s.find_last_of("/")); }
+std::string dirname(const std::string& s) {
+  return s.substr(0, s.find_last_of("/"));
+}
 
 bool Interpret(const char* examples_filename, bool use_nnapi) {
   std::ifstream tflite_stream(examples_filename);
@@ -65,14 +68,14 @@ int main(int argc, char* argv[]) {
     return 1;
   }
 
-  string base_dir = dirname(argv[1]);
+  std::string base_dir = dirname(argv[1]);
   DIR* dir = opendir(base_dir.c_str());
   if (dir == nullptr) {
     fprintf(stderr, "Can't open dir %s\n", base_dir.c_str());
     return 1;
   }
   while (struct dirent* ent = readdir(dir)) {
-    string name = ent->d_name;
+    std::string name = ent->d_name;
     if (name.rfind(".txt") == name.length() - 4) {
       printf("%s: ", name.c_str());
       if (Interpret((base_dir + "/" + name).c_str(), use_nnapi)) {
-- 
GitLab


From dadbcf8c205ec948938e28e21c4a5dd7ebdb672e Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Mon, 14 Jan 2019 17:46:16 -0800
Subject: [PATCH 0698/2345] [XLA] Fix special cases of complex power operation.

(0 + 0j)^x for real x > 0 is zero, not NaN.

PiperOrigin-RevId: 229290163
---
 .../xla/service/elemental_ir_emitter.cc       | 11 +++++++++-
 .../xla/service/hlo_evaluator_typed_visitor.h | 13 +++++++-----
 .../xla/service/llvm_ir/ir_builder_mixin.h    |  5 +++++
 .../xla/tests/array_elementwise_ops_test.cc   | 21 +++++++++++++++++++
 4 files changed, 44 insertions(+), 6 deletions(-)

diff --git a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
index f84c115e0a..727e0bfa52 100644
--- a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
@@ -812,11 +812,14 @@ StatusOr<llvm::Value*> ElementalIrEmitter::EmitComplexBinaryOp(
       auto c = EmitExtractReal(rhs_value);
       auto d = EmitExtractImag(rhs_value);
       auto aa_p_bb = FAdd(FMul(a, a), FMul(b, b));
+      auto zero = llvm::ConstantFP::get(a->getType(), 0);
       auto one_half = llvm::ConstantFP::get(a->getType(), 0.5);
+      auto one = llvm::ConstantFP::get(a->getType(), 1);
       auto half_c = FMul(one_half, c);
 
       TF_ASSIGN_OR_RETURN(auto aa_p_bb_to_half_c,
                           EmitPow(component_type, aa_p_bb, half_c));
+
       auto neg_d = FNeg(d);
       TF_ASSIGN_OR_RETURN(auto arg_lhs, EmitAtan2(component_type, b, a));
       auto neg_d_arg_lhs = FMul(neg_d, arg_lhs);
@@ -828,7 +831,13 @@ StatusOr<llvm::Value*> ElementalIrEmitter::EmitComplexBinaryOp(
       auto q = FAdd(FMul(c, arg_lhs), FMul(half_d, ln_aa_p_bb));
       TF_ASSIGN_OR_RETURN(auto cos_q, EmitCos(component_type, q));
       TF_ASSIGN_OR_RETURN(auto sin_q, EmitSin(component_type, q));
-      return EmitComposeComplex(op, FMul(coeff, cos_q), FMul(coeff, sin_q));
+      // 0^c is 0 if d is 0 and c > 0. 0^0 is defined to be 1.0, see
+      // Branch Cuts for Complex Elementary Functions or Much Ado About
+      // Nothing's Sign Bit, W. Kahan, Section 10.
+      return Select(
+          And(And(FCmpOEQ(aa_p_bb, zero), FCmpOEQ(d, zero)), FCmpOLE(zero, c)),
+          EmitComposeComplex(op, Select(FCmpOEQ(zero, c), one, zero), zero),
+          EmitComposeComplex(op, FMul(coeff, cos_q), FMul(coeff, sin_q)));
     }
     default:
       return Unimplemented("binary complex op '%s'",
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
index 5245c563ec..95a78408b0 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
@@ -675,11 +675,14 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
   }
 
   Status HandlePower(HloInstruction* power) override {
-    TF_ASSIGN_OR_RETURN(parent_->evaluated_[power],
-                        ElementWiseBinaryOp(power, [](ElementwiseT lhs_el,
-                                                      ElementwiseT rhs_el) {
-                          return std::pow(lhs_el, rhs_el);
-                        }));
+    TF_ASSIGN_OR_RETURN(
+        parent_->evaluated_[power],
+        ElementWiseBinaryOp(
+            power, [](ElementwiseT lhs_el, ElementwiseT rhs_el) {
+              return lhs_el == ElementwiseT(0) && rhs_el == ElementwiseT(0)
+                         ? static_cast<ElementwiseT>(1)
+                         : std::pow(lhs_el, rhs_el);
+            }));
     return Status::OK();
   }
 
diff --git a/tensorflow/compiler/xla/service/llvm_ir/ir_builder_mixin.h b/tensorflow/compiler/xla/service/llvm_ir/ir_builder_mixin.h
index abc06fb7b4..cf5083e8c1 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/ir_builder_mixin.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/ir_builder_mixin.h
@@ -254,6 +254,11 @@ class IrBuilderMixin {
     return mixin_builder()->CreateFCmpOLT(std::forward<Args>(args)...);
   }
 
+  template <class... Args>
+  llvm::Value* FCmpOLE(Args&&... args) {
+    return mixin_builder()->CreateFCmpOLE(std::forward<Args>(args)...);
+  }
+
   template <class... Args>
   llvm::Value* FCmpONE(Args&&... args) {
     return mixin_builder()->CreateFCmpONE(std::forward<Args>(args)...);
diff --git a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
index e4cc8b4199..7379fbcc22 100644
--- a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
+++ b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
@@ -1443,6 +1443,27 @@ XLA_TEST_F(ArrayElementwiseOpTest, PowNonIntegerF32s) {
                              error_spec_);
 }
 
+XLA_TEST_F(ArrayElementwiseOpTest, PowC64s) {
+  SetFastMathDisabled(true);
+  XlaBuilder builder(TestName());
+  auto lhs =
+      ConstantR1<complex64>(&builder, {-2.0f, -0.6f, -0.6f, 0.0f, 0.0f, 0.0f});
+  auto rhs =
+      ConstantR1<complex64>(&builder, {0.5f, 0.6f, -0.6f, 0.5f, 0.6f, 0.0f});
+  Pow(lhs, rhs);
+
+  ComputeAndCompareR1<complex64>(&builder,
+                                 {
+                                     {0, 1.41421356},
+                                     {-2.27443288e-01, 0.69999846},
+                                     {-4.19847531e-01, -1.29215783},
+                                     {0, 0},
+                                     {0, 0},
+                                     {1, 0},
+                                 },
+                                 {}, error_spec_);
+}
+
 XLA_TEST_F(ArrayElementwiseOpTest, PowZeroElementF32s) {
   XlaBuilder builder(TestName());
   auto lhs = ConstantR1<float>(&builder, {});
-- 
GitLab


From 7648a9554792d77a1506e39cc3a7bfc361126786 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 18:10:30 -0800
Subject: [PATCH 0699/2345] Fixing what I think a bug introduced by
 cl/228373932, which resulted in some of our eval_test/eval_train jobs to fail
 with: AttributeError: 'NoneType' object has no attribute
 'save_summary_steps'. Here's an example: http://screen/UentDg6uAZ6

I'm not familiar with the code and it's not clear to me how 'config' gets its value. So, feel free to suggest a different fix or rollback cl/228373932.

PiperOrigin-RevId: 229293332
---
 tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 075ecbc52a..79340a9f21 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -2656,7 +2656,11 @@ class TPUEstimator(estimator_lib.Estimator):
         if self._log_every_n_steps is not None:
           examples_hook = ExamplesPerSecondHook(
               ctx.global_batch_size,
-              output_dir=self.model_dir if config.save_summary_steps else None,
+              # pylint:disable=g-long-ternary
+              output_dir=(self.model_dir
+                          if not config or config.save_summary_steps
+                          else None),
+              # pylint:enable=g-long-ternary
               every_n_steps=self._log_every_n_steps)
 
         if ctx.is_running_on_cpu(is_export_mode=is_export_mode):
-- 
GitLab


From c33db572fe31cd48432dab84e17434359786bac8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 18:23:54 -0800
Subject: [PATCH 0700/2345] Fix transpose_conv

Use the proper filter height/width dimensions.

PiperOrigin-RevId: 229294726
---
 tensorflow/lite/kernels/transpose_conv.cc              | 8 ++++----
 tensorflow/lite/testing/generated_examples_zip_test.cc | 2 --
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/tensorflow/lite/kernels/transpose_conv.cc b/tensorflow/lite/kernels/transpose_conv.cc
index 59eee51068..343f2ca59b 100644
--- a/tensorflow/lite/kernels/transpose_conv.cc
+++ b/tensorflow/lite/kernels/transpose_conv.cc
@@ -119,8 +119,8 @@ TfLiteStatus ResizeIm2ColTensor(TfLiteContext* context,
   im2col_shape_array->data[1] = output_shape->data.i32[1];
   im2col_shape_array->data[2] = output_shape->data.i32[2];
   const int input_depth = SizeOfDimension(input, 3);
-  const int filter_width = SizeOfDimension(weights, 1);
-  const int filter_height = SizeOfDimension(weights, 2);
+  const int filter_width = SizeOfDimension(weights, 2);
+  const int filter_height = SizeOfDimension(weights, 1);
   im2col_shape_array->data[3] = input_depth * filter_height * filter_width;
 
   im2col->type = input->type;
@@ -197,8 +197,8 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   // Get height and width of the output image.
   const int width = SizeOfDimension(output, 2);
   const int height = SizeOfDimension(output, 1);
-  const int filter_width = SizeOfDimension(weights, 1);
-  const int filter_height = SizeOfDimension(weights, 2);
+  const int filter_width = SizeOfDimension(weights, 2);
+  const int filter_height = SizeOfDimension(weights, 1);
 
   const int stride_width = params->stride_width;
   const int stride_height = params->stride_height;
diff --git a/tensorflow/lite/testing/generated_examples_zip_test.cc b/tensorflow/lite/testing/generated_examples_zip_test.cc
index bade6590a4..a9a31ad088 100644
--- a/tensorflow/lite/testing/generated_examples_zip_test.cc
+++ b/tensorflow/lite/testing/generated_examples_zip_test.cc
@@ -105,8 +105,6 @@ std::map<string, string> kBrokenTests = {
 
     // Strided Slice chooses the wrong dimension.
     {R"(^\/strided_slice_buggy)", "119786029"},
-
-    {R"(^\/conv2d_transpose_filter_shape=\[1,4,8,3\])", "122735908"},
 };
 
 // Allows test data to be unarchived into a temporary directory and makes
-- 
GitLab


From 5634719232f2aa801adf3f9718cb92eb019da754 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 18:41:19 -0800
Subject: [PATCH 0701/2345] Add stream and stream_executor_pimpl deps to
 host_gpu_executor

PiperOrigin-RevId: 229296366
---
 tensorflow/stream_executor/host/BUILD | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/stream_executor/host/BUILD b/tensorflow/stream_executor/host/BUILD
index 59472b14c1..127452aee9 100644
--- a/tensorflow/stream_executor/host/BUILD
+++ b/tensorflow/stream_executor/host/BUILD
@@ -100,7 +100,9 @@ cc_library(
         "//tensorflow/core:lib",
         "//tensorflow/stream_executor:kernel",
         "//tensorflow/stream_executor:rng",
+        "//tensorflow/stream_executor:stream",
         "//tensorflow/stream_executor:stream_executor_internal",
+        "//tensorflow/stream_executor:stream_executor_pimpl",
         "//tensorflow/stream_executor:timer",
         "//tensorflow/stream_executor/lib",
     ],
-- 
GitLab


From b74605a932fe9b7bc878fa83999d09d12d0e5e55 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Mon, 14 Jan 2019 19:48:23 -0800
Subject: [PATCH 0702/2345] [tf.data] Add counters for tf.data elements,
 autotuning, and optimizations.

This CL:
- adds counters for tf.data elements, autotuning and optimizations
- sets the number of iterations of the `tf_data_meta_optimizer` to one -- the iteration of tf.data optimizations is handled by the tf.data meta optimizer itself
- adds the `alwayslink` attribute to all tf.data optimization BUILD targets to make sure they are always registered (without this, they would not be registered for the Tensorflow server binary I was using for local testing) and further cleans up visibility and dependencies of //third_party/tensorflow/core/grappler/optimizers/data/BUILD
- introduces TFDataOptimizerBase as a base class for tf.data optimizations
- moves TensorFlow metrics into tensorflow::metrics namespace

PiperOrigin-RevId: 229302097
---
 tensorflow/compiler/jit/BUILD                 |   1 -
 .../compiler/jit/build_xla_ops_pass_test.cc   |   1 -
 .../jit/partially_decluster_pass_test.cc      |   1 -
 .../core/common_runtime/direct_session.cc     |   2 +-
 tensorflow/core/common_runtime/metrics.cc     |  25 +-
 tensorflow/core/common_runtime/metrics.h      |  18 ++
 .../core/distributed_runtime/graph_mgr.cc     |   3 +-
 tensorflow/core/framework/model.cc            |   1 +
 tensorflow/core/framework/model.h             |  25 +-
 .../core/grappler/optimizers/data/BUILD       | 183 +++++-------
 .../grappler/optimizers/data/filter_fusion.cc |  11 +-
 .../grappler/optimizers/data/filter_fusion.h  |  13 +-
 .../optimizers/data/function_utils.cc         |   6 +-
 .../grappler/optimizers/data/function_utils.h |   6 +-
 .../grappler/optimizers/data/fusion_utils.cc  |   6 +-
 .../optimizers/data/graph_test_utils.cc       |   6 +-
 .../optimizers/data/graph_test_utils.h        |   6 +-
 .../grappler/optimizers/data/graph_utils.cc   |   6 +-
 .../grappler/optimizers/data/graph_utils.h    |   6 +-
 .../optimizers/data/hoist_random_uniform.cc   |  11 +-
 .../optimizers/data/hoist_random_uniform.h    |  13 +-
 .../optimizers/data/latency_all_edges.cc      |  11 +-
 .../optimizers/data/latency_all_edges.h       |  13 +-
 .../optimizers/data/make_numa_aware.cc        |   7 +-
 .../optimizers/data/make_numa_aware.h         |   9 +-
 .../grappler/optimizers/data/make_sloppy.cc   |   7 +-
 .../grappler/optimizers/data/make_sloppy.h    |   9 +-
 .../optimizers/data/map_and_batch_fusion.cc   |  11 +-
 .../optimizers/data/map_and_batch_fusion.h    |  13 +-
 .../optimizers/data/map_and_filter_fusion.cc  |  11 +-
 .../optimizers/data/map_and_filter_fusion.h   |  13 +-
 .../grappler/optimizers/data/map_fusion.cc    |  11 +-
 .../grappler/optimizers/data/map_fusion.h     |  13 +-
 .../optimizers/data/map_parallelization.cc    |  17 +-
 .../optimizers/data/map_parallelization.h     |  13 +-
 .../optimizers/data/map_vectorization.cc      |  11 +-
 .../optimizers/data/map_vectorization.h       |  13 +-
 .../optimizers/data/meta_optimizer.cc         |   4 +-
 .../grappler/optimizers/data/meta_optimizer.h |   4 +-
 .../optimizers/data/noop_elimination.cc       |  11 +-
 .../optimizers/data/noop_elimination.h        |  13 +-
 .../optimizers/data/optimizer_base.cc}        |  38 +--
 .../grappler/optimizers/data/optimizer_base.h |  47 +++
 .../data/shuffle_and_repeat_fusion.cc         |  11 +-
 .../data/shuffle_and_repeat_fusion.h          |  13 +-
 .../optimizers/data/vectorization_utils.cc    |   8 +-
 .../optimizers/data/vectorization_utils.h     |   6 +-
 tensorflow/core/kernels/data/BUILD            |   9 +-
 .../core/kernels/data/experimental/BUILD      |   1 +
 .../experimental/map_and_batch_dataset_op.cc  |  11 +-
 .../core/kernels/data/model_dataset_op.cc     | 270 +++++++++---------
 .../core/kernels/data/optimize_dataset_op.cc  |   9 +-
 .../data/parallel_interleave_dataset_op.cc    |   9 +-
 .../kernels/data/parallel_map_dataset_op.cc   |   9 +-
 .../core/kernels/data/prefetch_dataset_op.cc  |   9 +-
 55 files changed, 581 insertions(+), 433 deletions(-)
 rename tensorflow/core/{kernels/data/model_dataset_op.h => grappler/optimizers/data/optimizer_base.cc} (51%)
 create mode 100644 tensorflow/core/grappler/optimizers/data/optimizer_base.h

diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index ba9fa93654..1d72d8c2d8 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -638,7 +638,6 @@ tf_cc_test(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
-        "//tensorflow/core/grappler/optimizers/data:graph_utils",
         "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
diff --git a/tensorflow/compiler/jit/build_xla_ops_pass_test.cc b/tensorflow/compiler/jit/build_xla_ops_pass_test.cc
index 48a23a4c17..390ffa694b 100644
--- a/tensorflow/compiler/jit/build_xla_ops_pass_test.cc
+++ b/tensorflow/compiler/jit/build_xla_ops_pass_test.cc
@@ -24,7 +24,6 @@ limitations under the License.
 #include "tensorflow/compiler/jit/node_matchers.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/graph/algorithm.h"
-#include "tensorflow/core/grappler/optimizers/data/graph_utils.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/test.h"
diff --git a/tensorflow/compiler/jit/partially_decluster_pass_test.cc b/tensorflow/compiler/jit/partially_decluster_pass_test.cc
index 38a54cc5ef..1d81a8f4fc 100644
--- a/tensorflow/compiler/jit/partially_decluster_pass_test.cc
+++ b/tensorflow/compiler/jit/partially_decluster_pass_test.cc
@@ -33,7 +33,6 @@ limitations under the License.
 #include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/graph/graph_def_builder.h"
 #include "tensorflow/core/graph/graph_def_builder_util.h"
-#include "tensorflow/core/grappler/optimizers/data/graph_utils.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/platform/test.h"
 
diff --git a/tensorflow/core/common_runtime/direct_session.cc b/tensorflow/core/common_runtime/direct_session.cc
index 8a1c648f61..80b62f273c 100644
--- a/tensorflow/core/common_runtime/direct_session.cc
+++ b/tensorflow/core/common_runtime/direct_session.cc
@@ -719,7 +719,7 @@ Status DirectSession::RunInternal(int64 step_id, const RunOptions& run_options,
       exec_and_lib.graph->ToGraphDef(partition_graph_def);
     }
   }
-  UpdateGraphExecTime(Env::Default()->NowMicros() - start_time_usecs);
+  metrics::UpdateGraphExecTime(Env::Default()->NowMicros() - start_time_usecs);
 
   return Status::OK();
 }
diff --git a/tensorflow/core/common_runtime/metrics.cc b/tensorflow/core/common_runtime/metrics.cc
index f4c94ed7ec..362d3ceca8 100644
--- a/tensorflow/core/common_runtime/metrics.cc
+++ b/tensorflow/core/common_runtime/metrics.cc
@@ -17,7 +17,7 @@ limitations under the License.
 #include "tensorflow/core/lib/monitoring/counter.h"
 
 namespace tensorflow {
-
+namespace metrics {
 namespace {
 
 auto* graph_runs = monitoring::Counter<0>::New(
@@ -28,8 +28,30 @@ auto* graph_runs = monitoring::Counter<0>::New(
 auto* graph_run_time_usecs = monitoring::Counter<0>::New(
     "/tensorflow/core/graph_run_time_usecs",
     "The total time spent on executing graphs in microseconds.");
+
+auto* tf_data_autotune_counter = monitoring::Counter<1>::New(
+    "/tensorflow/data/autotune", "tf.data autotuning", "name");
+
+auto* tf_data_elements_counter = monitoring::Counter<1>::New(
+    "/tensorflow/data/elements", "tf.data elements", "name");
+
+auto* tf_data_optimization_counter = monitoring::Counter<1>::New(
+    "/tensorflow/data/optimization", "tf.data optimization", "name");
+
 }  // namespace
 
+void RecordTFDataAutotune(const string& name) {
+  tf_data_autotune_counter->GetCell(name)->IncrementBy(1);
+}
+
+void RecordTFDataElements(const string& name, int64 num_elements) {
+  tf_data_elements_counter->GetCell(name)->IncrementBy(num_elements);
+}
+
+void RecordTFDataOptimization(const string& name, int64 num_changes) {
+  tf_data_optimization_counter->GetCell(name)->IncrementBy(num_changes);
+}
+
 void UpdateGraphExecTime(const uint64 running_time_usecs) {
   if (running_time_usecs > 0) {
     graph_runs->GetCell()->IncrementBy(1);
@@ -37,4 +59,5 @@ void UpdateGraphExecTime(const uint64 running_time_usecs) {
   }
 }
 
+}  // namespace metrics
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/metrics.h b/tensorflow/core/common_runtime/metrics.h
index d3430c9f03..9641265362 100644
--- a/tensorflow/core/common_runtime/metrics.h
+++ b/tensorflow/core/common_runtime/metrics.h
@@ -19,9 +19,27 @@ limitations under the License.
 #include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
+namespace metrics {
+
+// Records that a tf.data dataset op executed by the program used autotuning.
+//
+// The `name` argument identifies the dataset (e.g. "ParallelMap").
+void RecordTFDataAutotune(const string& name);
+
+// Records the number of elements produced by a tf.data dataset.
+//
+// The `name` argument identifies the dataset (e.g. "Batch" or "Map").
+void RecordTFDataElements(const string& name, int64 num_elements);
+
+// Records the number of independent graph changes resulting from the applicaton
+// of a tf.data optimization.
+//
+// The `name` argument identifies the optimization (e.g. "noop_eliminiation").
+void RecordTFDataOptimization(const string& name, int64 num_changes);
 
 void UpdateGraphExecTime(const uint64 running_time_usecs);
 
+}  // namespace metrics
 }  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_COMMON_RUNTIME_METRICS_H_
diff --git a/tensorflow/core/distributed_runtime/graph_mgr.cc b/tensorflow/core/distributed_runtime/graph_mgr.cc
index ee5823e314..1065f021a1 100644
--- a/tensorflow/core/distributed_runtime/graph_mgr.cc
+++ b/tensorflow/core/distributed_runtime/graph_mgr.cc
@@ -451,7 +451,8 @@ void GraphMgr::ExecuteAsync(const string& handle, const int64 step_id,
       cancellation_manager,
       [item, rendezvous, ce_handle, done, start_time_usecs](const Status& s) {
         done(s);
-        UpdateGraphExecTime(Env::Default()->NowMicros() - start_time_usecs);
+        metrics::UpdateGraphExecTime(Env::Default()->NowMicros() -
+                                     start_time_usecs);
         rendezvous->Unref();
         item->Unref();
         delete ce_handle;
diff --git a/tensorflow/core/framework/model.cc b/tensorflow/core/framework/model.cc
index 96cc7e583a..47c8dbe61b 100644
--- a/tensorflow/core/framework/model.cc
+++ b/tensorflow/core/framework/model.cc
@@ -496,6 +496,7 @@ void Model::RemoveNode(const string& name) {
   if (node && (*node)->output()) {
     (*node)->output()->remove_input(*node);
   }
+  remove_node_hook_(*node);
   lookup_table_.erase(name);
 }
 
diff --git a/tensorflow/core/framework/model.h b/tensorflow/core/framework/model.h
index d9f7185362..7fac1753a6 100644
--- a/tensorflow/core/framework/model.h
+++ b/tensorflow/core/framework/model.h
@@ -359,16 +359,26 @@ std::shared_ptr<Node> MakeUnknownNode(Node::Args args);
 // implementation of `DatasetBase` and `DatasetBaseIterator` respectively.
 class Model {
  public:
-  Model() : collect_resource_usage_(false) {}
+  using NodeHook = std::function<void(std::shared_ptr<Node>)>;
+
+  // Creates a new model.
+  //
+  // The `remove_node_hook` argument can be used to specify functionality that
+  // should be invoked before a node is removed from the model. The hook can be
+  // used for dependency injection -- to allow the model to invoke functionality
+  // from modules that it could not depend on statically.
+  Model(NodeHook remove_node_hook)
+      : collect_resource_usage_(false),
+        remove_node_hook_(std::move(remove_node_hook)) {
+    DCHECK(remove_node_hook_ != nullptr);
+  }
 
   // Indicates whether to collect resource usage.
   bool collect_resource_usage() const { return collect_resource_usage_; }
 
   // Adds a node with the given name and given output.
-  virtual std::shared_ptr<Node> AddNode(Node::Factory factory,
-                                        const string& name,
-                                        const string& output_name)
-      LOCKS_EXCLUDED(mu_);
+  std::shared_ptr<Node> AddNode(Node::Factory factory, const string& name,
+                                const string& output_name) LOCKS_EXCLUDED(mu_);
 
   // Increments the processing time for the given node..
   void AddProcessingTime(const string& name, int64 delta) LOCKS_EXCLUDED(mu_);
@@ -390,7 +400,7 @@ class Model {
   // Removes the given node.
   void RemoveNode(const string& name) LOCKS_EXCLUDED(mu_);
 
- protected:
+ private:
   // Collects tunable parameters in the tree rooted in the given node.
   std::vector<std::shared_ptr<Parameter>> CollectTunableParameters(
       std::shared_ptr<Node> node);
@@ -416,6 +426,9 @@ class Model {
   // tunable parameter (because the information is used for for tuning the value
   // of the parameter) and never stops.
   std::atomic<bool> collect_resource_usage_;
+
+  // A hook invoked immediately before a node is removed from the model.
+  const NodeHook remove_node_hook_;
 };
 
 }  // namespace model
diff --git a/tensorflow/core/grappler/optimizers/data/BUILD b/tensorflow/core/grappler/optimizers/data/BUILD
index d95edc5623..413588b15c 100644
--- a/tensorflow/core/grappler/optimizers/data/BUILD
+++ b/tensorflow/core/grappler/optimizers/data/BUILD
@@ -3,29 +3,29 @@ licenses(["notice"])  # Apache 2.0
 load("//tensorflow:tensorflow.bzl", "tf_cc_test")
 load("//tensorflow/core:platform/default/build_config.bzl", "tf_protos_all")
 
+package(default_visibility = [
+    "//tensorflow/core/grappler/optimizers/data:__subpackages__",
+    "//tensorflow/core/kernels/data:__pkg__",
+])
+
 cc_library(
-    name = "meta_optimizer",
-    srcs = ["meta_optimizer.cc"],
-    hdrs = [
-        "meta_optimizer.h",
-    ],
+    name = "data",
     visibility = ["//visibility:public"],
     deps = [
-        "@com_google_absl//absl/container:flat_hash_map",
-        "//tensorflow/core/grappler:grappler_item",
-        "//tensorflow/core:lib",
-        "//tensorflow/core/grappler/clusters:cluster",
-        "//tensorflow/core/grappler/optimizers:arithmetic_optimizer",
-        "//tensorflow/core/grappler/optimizers:model_pruner",
-        "//tensorflow/core/grappler/optimizers:shape_optimizer",
-        "//tensorflow/core/grappler/optimizers:dependency_optimizer",
-        "//tensorflow/core/grappler/optimizers:function_optimizer",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
-        "//tensorflow/core:lib_internal",
-        "//tensorflow/core:ptr_util",
-    ] + tf_protos_all(),
-    alwayslink = 1,
+        ":filter_fusion",
+        ":hoist_random_uniform",
+        ":latency_all_edges",
+        ":make_numa_aware",
+        ":make_sloppy",
+        ":map_and_batch_fusion",
+        ":map_and_filter_fusion",
+        ":map_fusion",
+        ":map_parallelization",
+        ":map_vectorization",
+        ":meta_optimizer",
+        ":noop_elimination",
+        ":shuffle_and_repeat_fusion",
+    ],
 )
 
 cc_library(
@@ -34,10 +34,10 @@ cc_library(
     hdrs = [
         "filter_fusion.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
         ":fusion_utils",
+        ":optimizer_base",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:grappler_item",
@@ -45,7 +45,6 @@ cc_library(
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/clusters:cluster",
         "//tensorflow/core/grappler/utils:topological_sort",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
         "//tensorflow/core:lib_internal",
     ] + tf_protos_all(),
@@ -55,7 +54,6 @@ cc_library(
 tf_cc_test(
     name = "filter_fusion_test",
     srcs = ["filter_fusion_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":filter_fusion",
         ":graph_test_utils",
@@ -74,7 +72,6 @@ cc_library(
     hdrs = [
         "fusion_utils.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
         ":function_utils",
@@ -94,7 +91,6 @@ cc_library(
 tf_cc_test(
     name = "fusion_utils_test",
     srcs = ["fusion_utils_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":function_utils",
         ":fusion_utils",
@@ -113,7 +109,6 @@ cc_library(
     hdrs = [
         "function_utils.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
         "//tensorflow/core:framework",
@@ -127,7 +122,6 @@ cc_library(
 tf_cc_test(
     name = "function_utils_test",
     srcs = ["function_utils_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":function_utils",
         "//tensorflow/core:framework",
@@ -135,7 +129,6 @@ tf_cc_test(
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
         "//tensorflow/core/grappler:grappler_item",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
         "//tensorflow/core/kernels:cast_op",
         "//tensorflow/tools/graph_transforms:transform_utils",
@@ -148,7 +141,6 @@ cc_library(
     hdrs = [
         "graph_utils.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
@@ -162,7 +154,6 @@ cc_library(
 tf_cc_test(
     name = "graph_utils_test",
     srcs = ["graph_utils_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
         "//tensorflow/core:core_cpu",
@@ -171,7 +162,6 @@ tf_cc_test(
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
         "//tensorflow/core/grappler:grappler_item",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ],
 )
@@ -183,7 +173,6 @@ cc_library(
     hdrs = [
         "graph_test_utils.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
@@ -200,17 +189,16 @@ cc_library(
     hdrs = [
         "hoist_random_uniform.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         ":function_utils",
         ":graph_utils",
+        ":optimizer_base",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/clusters:cluster",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
         "//tensorflow/core:lib_internal",
     ] + tf_protos_all(),
@@ -220,7 +208,6 @@ cc_library(
 tf_cc_test(
     name = "hoist_random_uniform_test",
     srcs = ["hoist_random_uniform_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_test_utils",
         ":graph_utils",
@@ -239,35 +226,45 @@ cc_library(
     hdrs = [
         "latency_all_edges.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
+        ":optimizer_base",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/clusters:cluster",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
     alwayslink = 1,
 )
 
+tf_cc_test(
+    name = "latency_all_edges_test",
+    srcs = ["latency_all_edges_test.cc"],
+    deps = [
+        ":graph_utils",
+        ":latency_all_edges",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+        "//tensorflow/core/grappler:grappler_item",
+    ],
+)
+
 cc_library(
     name = "make_numa_aware",
     srcs = ["make_numa_aware.cc"],
     hdrs = ["make_numa_aware.h"],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
+        ":optimizer_base",
         "//tensorflow/core/grappler:mutable_graph_view",
-        "//tensorflow/core:lib",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
-        "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/clusters:cluster",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
     alwayslink = 1,
@@ -276,7 +273,6 @@ cc_library(
 tf_cc_test(
     name = "make_numa_aware_test",
     srcs = ["make_numa_aware_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_test_utils",
         ":graph_utils",
@@ -293,13 +289,12 @@ cc_library(
     name = "make_sloppy",
     srcs = ["make_sloppy.cc"],
     hdrs = ["make_sloppy.h"],
-    visibility = ["//visibility:public"],
     deps = [
+        ":optimizer_base",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler/clusters:cluster",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
     alwayslink = 1,
@@ -308,7 +303,6 @@ cc_library(
 tf_cc_test(
     name = "make_sloppy_test",
     srcs = ["make_sloppy_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_test_utils",
         ":graph_utils",
@@ -327,16 +321,15 @@ cc_library(
     hdrs = [
         "map_and_batch_fusion.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
+        ":optimizer_base",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/clusters:cluster",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
     alwayslink = 1,
@@ -345,7 +338,6 @@ cc_library(
 tf_cc_test(
     name = "map_and_batch_fusion_test",
     srcs = ["map_and_batch_fusion_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
         ":map_and_batch_fusion",
@@ -362,17 +354,16 @@ cc_library(
     hdrs = [
         "map_and_filter_fusion.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
         ":fusion_utils",
+        ":optimizer_base",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/clusters:cluster",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/utils:topological_sort",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
         "//tensorflow/core:lib_internal",
@@ -383,7 +374,6 @@ cc_library(
 tf_cc_test(
     name = "map_and_filter_fusion_test",
     srcs = ["map_and_filter_fusion_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_test_utils",
         ":graph_utils",
@@ -402,10 +392,10 @@ cc_library(
     hdrs = [
         "map_fusion.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
         ":fusion_utils",
+        ":optimizer_base",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:grappler_item",
@@ -413,7 +403,6 @@ cc_library(
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/clusters:cluster",
         "//tensorflow/core/grappler/utils:topological_sort",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
         "//tensorflow/core:lib_internal",
     ] + tf_protos_all(),
@@ -423,7 +412,6 @@ cc_library(
 tf_cc_test(
     name = "map_fusion_test",
     srcs = ["map_fusion_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_test_utils",
         ":graph_utils",
@@ -443,15 +431,14 @@ cc_library(
     hdrs = [
         "map_parallelization.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
+        ":optimizer_base",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/clusters:cluster",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
     alwayslink = 1,
@@ -460,7 +447,6 @@ cc_library(
 tf_cc_test(
     name = "map_parallelization_test",
     srcs = ["map_parallelization_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_test_utils",
         ":graph_utils",
@@ -479,10 +465,10 @@ cc_library(
     hdrs = [
         "map_vectorization.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         ":function_utils",
         ":graph_utils",
+        ":optimizer_base",
         ":vectorization_utils",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:mutable_graph_view",
@@ -490,7 +476,6 @@ cc_library(
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/clusters:cluster",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
         "//tensorflow/core:lib_internal",
     ] + tf_protos_all(),
@@ -500,7 +485,6 @@ cc_library(
 tf_cc_test(
     name = "map_vectorization_test",
     srcs = ["map_vectorization_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
         ":map_vectorization",
@@ -512,22 +496,42 @@ tf_cc_test(
     ],
 )
 
+cc_library(
+    name = "meta_optimizer",
+    srcs = ["meta_optimizer.cc"],
+    hdrs = ["meta_optimizer.h"],
+    deps = [
+        "@com_google_absl//absl/container:flat_hash_map",
+        "//tensorflow/core/grappler/clusters:cluster",
+        "//tensorflow/core/grappler/optimizers:arithmetic_optimizer",
+        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
+        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
+        "//tensorflow/core/grappler/optimizers:dependency_optimizer",
+        "//tensorflow/core/grappler/optimizers:function_optimizer",
+        "//tensorflow/core/grappler/optimizers:model_pruner",
+        "//tensorflow/core/grappler/optimizers:shape_optimizer",
+        "//tensorflow/core/grappler:grappler_item",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core:ptr_util",
+    ] + tf_protos_all(),
+    alwayslink = 1,
+)
+
 cc_library(
     name = "noop_elimination",
     srcs = ["noop_elimination.cc"],
     hdrs = [
         "noop_elimination.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
+        ":optimizer_base",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/clusters:cluster",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
     alwayslink = 1,
@@ -536,7 +540,6 @@ cc_library(
 tf_cc_test(
     name = "noop_elimination_test",
     srcs = ["noop_elimination_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
         ":noop_elimination",
@@ -547,22 +550,33 @@ tf_cc_test(
     ],
 )
 
+cc_library(
+    name = "optimizer_base",
+    srcs = ["optimizer_base.cc"],
+    hdrs = [
+        "optimizer_base.h",
+    ],
+    deps = [
+        "//tensorflow/core:metrics",
+        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
+    ],
+)
+
 cc_library(
     name = "shuffle_and_repeat_fusion",
     srcs = ["shuffle_and_repeat_fusion.cc"],
     hdrs = [
         "shuffle_and_repeat_fusion.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
+        ":optimizer_base",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/clusters:cluster",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
     alwayslink = 1,
@@ -571,7 +585,6 @@ cc_library(
 tf_cc_test(
     name = "shuffle_and_repeat_fusion_test",
     srcs = ["shuffle_and_repeat_fusion_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
         ":shuffle_and_repeat_fusion",
@@ -582,48 +595,12 @@ tf_cc_test(
     ],
 )
 
-cc_library(
-    name = "data",
-    visibility = ["//visibility:public"],
-    deps = [
-        ":filter_fusion",
-        ":hoist_random_uniform",
-        ":latency_all_edges",
-        ":make_numa_aware",
-        ":make_sloppy",
-        ":map_and_batch_fusion",
-        ":map_and_filter_fusion",
-        ":map_fusion",
-        ":map_parallelization",
-        ":map_vectorization",
-        ":meta_optimizer",
-        ":noop_elimination",
-        ":shuffle_and_repeat_fusion",
-    ],
-    alwayslink = 1,
-)
-
-tf_cc_test(
-    name = "latency_all_edges_test",
-    srcs = ["latency_all_edges_test.cc"],
-    deps = [
-        ":graph_utils",
-        ":latency_all_edges",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-        "//tensorflow/core:testlib",
-        "//tensorflow/core/grappler:grappler_item",
-    ],
-)
-
 cc_library(
     name = "vectorization_utils",
     srcs = ["vectorization_utils.cc"],
     hdrs = [
         "vectorization_utils.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         ":function_utils",
         ":graph_utils",
@@ -644,7 +621,6 @@ cc_library(
 tf_cc_test(
     name = "vectorization_utils_test",
     srcs = ["vectorization_utils_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
         ":function_utils",
@@ -656,7 +632,6 @@ tf_cc_test(
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
         "//tensorflow/core/grappler:grappler_item",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
         # For ops we need registered
         "//tensorflow/core/kernels/data:dataset_ops",
diff --git a/tensorflow/core/grappler/optimizers/data/filter_fusion.cc b/tensorflow/core/grappler/optimizers/data/filter_fusion.cc
index e3f0a7e6c5..25dc10806b 100644
--- a/tensorflow/core/grappler/optimizers/data/filter_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/filter_fusion.cc
@@ -57,8 +57,10 @@ NodeDef MakeFusedFilterNode(const NodeDef& first_filter_node,
 
 }  // namespace
 
-Status FilterFusion::Optimize(Cluster* cluster, const GrapplerItem& item,
-                              GraphDef* output) {
+Status FilterFusion::OptimizeAndCollectStats(Cluster* cluster,
+                                             const GrapplerItem& item,
+                                             GraphDef* output,
+                                             OptimizationStats* stats) {
   GraphDef sorted_old_graph = item.graph;
   TF_RETURN_IF_ERROR(TopologicalSort(&sorted_old_graph));
   *output = sorted_old_graph;
@@ -120,6 +122,7 @@ Status FilterFusion::Optimize(Cluster* cluster, const GrapplerItem& item,
     // they are not used anymore.
     nodes_to_delete.insert(first_filter_node->name());
     nodes_to_delete.insert(second_filter_node->name());
+    stats->num_changes++;
   }
 
   graph.DeleteNodes(nodes_to_delete);
@@ -133,5 +136,5 @@ void FilterFusion::Feedback(Cluster* cluster, const GrapplerItem& item,
 
 REGISTER_GRAPH_OPTIMIZER_AS(FilterFusion, "filter_fusion");
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/filter_fusion.h b/tensorflow/core/grappler/optimizers/data/filter_fusion.h
index 91a0364a46..ac0326c0ec 100644
--- a/tensorflow/core/grappler/optimizers/data/filter_fusion.h
+++ b/tensorflow/core/grappler/optimizers/data/filter_fusion.h
@@ -16,13 +16,13 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_FILTER_FUSION_H_
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_FILTER_FUSION_H_
 
-#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h"
+#include "tensorflow/core/grappler/optimizers/data/optimizer_base.h"
 
 namespace tensorflow {
 namespace grappler {
 
 // This optimization fuses filter transformations.
-class FilterFusion : public CustomGraphOptimizer {
+class FilterFusion : public TFDataOptimizerBase {
  public:
   FilterFusion() = default;
   ~FilterFusion() override = default;
@@ -34,14 +34,15 @@ class FilterFusion : public CustomGraphOptimizer {
     return Status::OK();
   }
 
-  Status Optimize(Cluster* cluster, const GrapplerItem& item,
-                  GraphDef* output) override;
+  Status OptimizeAndCollectStats(Cluster* cluster, const GrapplerItem& item,
+                                 GraphDef* output,
+                                 OptimizationStats* stats) override;
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimize_output, double result) override;
 };
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_FILTER_FUSION_H_
diff --git a/tensorflow/core/grappler/optimizers/data/function_utils.cc b/tensorflow/core/grappler/optimizers/data/function_utils.cc
index 311df15bc2..351209f92e 100644
--- a/tensorflow/core/grappler/optimizers/data/function_utils.cc
+++ b/tensorflow/core/grappler/optimizers/data/function_utils.cc
@@ -171,6 +171,6 @@ void SetUniqueFunctionNodeName(StringPiece prefix, FunctionDef* function,
   node->set_name(std::move(name));
 }
 
-}  // end namespace function_utils
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace function_utils
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/function_utils.h b/tensorflow/core/grappler/optimizers/data/function_utils.h
index d4ce824652..ec08c11dce 100644
--- a/tensorflow/core/grappler/optimizers/data/function_utils.h
+++ b/tensorflow/core/grappler/optimizers/data/function_utils.h
@@ -101,8 +101,8 @@ int FindFunctionNodeWithOp(StringPiece op, const FunctionDef& function);
 void SetUniqueFunctionNodeName(StringPiece prefix, FunctionDef* function,
                                NodeDef* node);
 
-}  // end namespace function_utils
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace function_utils
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_FUNCTION_UTILS_H_
diff --git a/tensorflow/core/grappler/optimizers/data/fusion_utils.cc b/tensorflow/core/grappler/optimizers/data/fusion_utils.cc
index b3bfee138f..d5308ad31a 100644
--- a/tensorflow/core/grappler/optimizers/data/fusion_utils.cc
+++ b/tensorflow/core/grappler/optimizers/data/fusion_utils.cc
@@ -471,6 +471,6 @@ FunctionDef* FuseFunctions(
   return fused_function;
 }
 
-}  // end namespace fusion_utils
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace fusion_utils
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/graph_test_utils.cc b/tensorflow/core/grappler/optimizers/data/graph_test_utils.cc
index 202dfb5ac8..82ca0146b9 100644
--- a/tensorflow/core/grappler/optimizers/data/graph_test_utils.cc
+++ b/tensorflow/core/grappler/optimizers/data/graph_test_utils.cc
@@ -107,6 +107,6 @@ NodeDef MakeParseExampleNode(StringPiece name, StringPiece input_node_name,
       });
 }
 
-}  // end namespace graph_tests_utils
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace graph_tests_utils
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/graph_test_utils.h b/tensorflow/core/grappler/optimizers/data/graph_test_utils.h
index a2707ee7b7..3750e2d5cc 100644
--- a/tensorflow/core/grappler/optimizers/data/graph_test_utils.h
+++ b/tensorflow/core/grappler/optimizers/data/graph_test_utils.h
@@ -56,8 +56,8 @@ NodeDef MakeParseExampleNode(StringPiece name, StringPiece input_node_name,
                              StringPiece num_parallel_calls_node_name,
                              bool sloppy);
 
-}  // end namespace graph_tests_utils
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace graph_tests_utils
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_GRAPH_TEST_UTILS_H_
diff --git a/tensorflow/core/grappler/optimizers/data/graph_utils.cc b/tensorflow/core/grappler/optimizers/data/graph_utils.cc
index 90208c1fba..cbafb9dc82 100644
--- a/tensorflow/core/grappler/optimizers/data/graph_utils.cc
+++ b/tensorflow/core/grappler/optimizers/data/graph_utils.cc
@@ -293,6 +293,6 @@ Status EnsureNodeNamesUnique(Graph* g) {
 
   return Status::OK();
 }
-}  // end namespace graph_utils
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace graph_utils
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/graph_utils.h b/tensorflow/core/grappler/optimizers/data/graph_utils.h
index d130fee204..8f2872c146 100644
--- a/tensorflow/core/grappler/optimizers/data/graph_utils.h
+++ b/tensorflow/core/grappler/optimizers/data/graph_utils.h
@@ -140,8 +140,8 @@ void ConcatAttributeList(const string& attribute_name, const NodeDef& first,
 // and renaming nodes does not mutate any edges.
 Status EnsureNodeNamesUnique(Graph* g);
 
-}  // end namespace graph_utils
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace graph_utils
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_GRAPH_UTILS_H_
diff --git a/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc b/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc
index 6aa450e123..237e4b63e3 100644
--- a/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc
+++ b/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc
@@ -220,8 +220,10 @@ int NumberOfPlaceholders(const NodeDef& map_node) {
 
 }  // namespace
 
-Status HoistRandomUniform::Optimize(Cluster* cluster, const GrapplerItem& item,
-                                    GraphDef* output) {
+Status HoistRandomUniform::OptimizeAndCollectStats(Cluster* cluster,
+                                                   const GrapplerItem& item,
+                                                   GraphDef* output,
+                                                   OptimizationStats* stats) {
   *output = item.graph;
 
   MutableGraphView graph(output);
@@ -272,6 +274,7 @@ Status HoistRandomUniform::Optimize(Cluster* cluster, const GrapplerItem& item,
     // TODO(b/116285210): we could also remove map functions from library if
     // they are not used anymore.
     nodes_to_delete.insert(map_node->name());
+    stats->num_changes++;
   }
 
   graph.DeleteNodes(nodes_to_delete);
@@ -286,5 +289,5 @@ void HoistRandomUniform::Feedback(Cluster* cluster, const GrapplerItem& item,
 
 REGISTER_GRAPH_OPTIMIZER_AS(HoistRandomUniform, "hoist_random_uniform");
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.h b/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.h
index d1bcf6782d..94db9f72a4 100644
--- a/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.h
+++ b/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.h
@@ -16,7 +16,7 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_HOIST_RANDOM_UNIFORM_H_
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_HOIST_RANDOM_UNIFORM_H_
 
-#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h"
+#include "tensorflow/core/grappler/optimizers/data/optimizer_base.h"
 
 namespace tensorflow {
 namespace grappler {
@@ -30,7 +30,7 @@ namespace grappler {
 // `stateless_random_uniform`.
 // TODO(prazek): for now only `RandomUniform` is handled, but we could handle
 // `RandomUniformInt` similarly.
-class HoistRandomUniform : public CustomGraphOptimizer {
+class HoistRandomUniform : public TFDataOptimizerBase {
  public:
   HoistRandomUniform() = default;
   ~HoistRandomUniform() override = default;
@@ -42,14 +42,15 @@ class HoistRandomUniform : public CustomGraphOptimizer {
     return Status::OK();
   }
 
-  Status Optimize(Cluster* cluster, const GrapplerItem& item,
-                  GraphDef* output) override;
+  Status OptimizeAndCollectStats(Cluster* cluster, const GrapplerItem& item,
+                                 GraphDef* output,
+                                 OptimizationStats* stats) override;
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimize_output, double result) override;
 };
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_HOIST_RANDOM_UNIFORM_H_
diff --git a/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc b/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc
index 0b7b2eba86..4529b89bd4 100644
--- a/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc
+++ b/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc
@@ -64,8 +64,10 @@ NodeDef MakeLatencyNode(const NodeDef& node, MutableGraphView* graph) {
 
 }  // namespace
 
-Status LatencyAllEdges::Optimize(Cluster* cluster, const GrapplerItem& item,
-                                 GraphDef* output) {
+Status LatencyAllEdges::OptimizeAndCollectStats(Cluster* cluster,
+                                                const GrapplerItem& item,
+                                                GraphDef* output,
+                                                OptimizationStats* stats) {
   *output = item.graph;
   MutableGraphView graph(output);
 
@@ -96,6 +98,7 @@ Status LatencyAllEdges::Optimize(Cluster* cluster, const GrapplerItem& item,
 
     NodeDef* latency_node = graph.AddNode(MakeLatencyNode(node, &graph));
     TF_RETURN_IF_ERROR(graph.UpdateFanouts(node.name(), latency_node->name()));
+    stats->num_changes++;
   }
   return Status::OK();
 }
@@ -107,5 +110,5 @@ void LatencyAllEdges::Feedback(Cluster* cluster, const GrapplerItem& item,
 
 REGISTER_GRAPH_OPTIMIZER_AS(LatencyAllEdges, "latency_all_edges");
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/latency_all_edges.h b/tensorflow/core/grappler/optimizers/data/latency_all_edges.h
index f6c71a9ec7..313d108286 100644
--- a/tensorflow/core/grappler/optimizers/data/latency_all_edges.h
+++ b/tensorflow/core/grappler/optimizers/data/latency_all_edges.h
@@ -16,12 +16,12 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_LATENCY_ALL_EDGES_H_
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_LATENCY_ALL_EDGES_H_
 
-#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h"
+#include "tensorflow/core/grappler/optimizers/data/optimizer_base.h"
 
 namespace tensorflow {
 namespace grappler {
 
-class LatencyAllEdges : public CustomGraphOptimizer {
+class LatencyAllEdges : public TFDataOptimizerBase {
  public:
   LatencyAllEdges() = default;
   ~LatencyAllEdges() override = default;
@@ -33,14 +33,15 @@ class LatencyAllEdges : public CustomGraphOptimizer {
     return Status::OK();
   }
 
-  Status Optimize(Cluster* cluster, const GrapplerItem& item,
-                  GraphDef* output) override;
+  Status OptimizeAndCollectStats(Cluster* cluster, const GrapplerItem& item,
+                                 GraphDef* output,
+                                 OptimizationStats* stats) override;
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimize_output, double result) override;
 };
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_LATENCY_ALL_EDGES_H_
diff --git a/tensorflow/core/grappler/optimizers/data/make_numa_aware.cc b/tensorflow/core/grappler/optimizers/data/make_numa_aware.cc
index 2940fbecd4..d8ea9adfb7 100644
--- a/tensorflow/core/grappler/optimizers/data/make_numa_aware.cc
+++ b/tensorflow/core/grappler/optimizers/data/make_numa_aware.cc
@@ -37,8 +37,10 @@ NodeDef MakeNumaAwareNode(const NodeDef& node, MutableGraphView* graph) {
 
 }  // namespace
 
-Status MakeNumaAware::Optimize(Cluster* cluster, const GrapplerItem& item,
-                               GraphDef* output) {
+Status MakeNumaAware::OptimizeAndCollectStats(Cluster* cluster,
+                                              const GrapplerItem& item,
+                                              GraphDef* output,
+                                              OptimizationStats* stats) {
   *output = item.graph;
   MutableGraphView graph(output);
   std::set<string> nodes_to_delete;
@@ -49,6 +51,7 @@ Status MakeNumaAware::Optimize(Cluster* cluster, const GrapplerItem& item,
     auto* numa_node = graph.AddNode(MakeNumaAwareNode(node, &graph));
     TF_RETURN_IF_ERROR(graph.UpdateFanouts(node.name(), numa_node->name()));
     nodes_to_delete.insert(node.name());
+    stats->num_changes++;
   }
   graph.DeleteNodes(nodes_to_delete);
   return Status::OK();
diff --git a/tensorflow/core/grappler/optimizers/data/make_numa_aware.h b/tensorflow/core/grappler/optimizers/data/make_numa_aware.h
index 48a7d8145f..81dbb31e6d 100644
--- a/tensorflow/core/grappler/optimizers/data/make_numa_aware.h
+++ b/tensorflow/core/grappler/optimizers/data/make_numa_aware.h
@@ -16,12 +16,12 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAKE_NUMA_AWARE_H_
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAKE_NUMA_AWARE_H_
 
-#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h"
+#include "tensorflow/core/grappler/optimizers/data/optimizer_base.h"
 
 namespace tensorflow {
 namespace grappler {
 
-class MakeNumaAware : public CustomGraphOptimizer {
+class MakeNumaAware : public TFDataOptimizerBase {
  public:
   MakeNumaAware() = default;
   ~MakeNumaAware() override = default;
@@ -33,8 +33,9 @@ class MakeNumaAware : public CustomGraphOptimizer {
     return Status::OK();
   }
 
-  Status Optimize(Cluster* cluster, const GrapplerItem& item,
-                  GraphDef* output) override;
+  Status OptimizeAndCollectStats(Cluster* cluster, const GrapplerItem& item,
+                                 GraphDef* output,
+                                 OptimizationStats* stats) override;
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimize_output, double result) override {}
diff --git a/tensorflow/core/grappler/optimizers/data/make_sloppy.cc b/tensorflow/core/grappler/optimizers/data/make_sloppy.cc
index 1cfaef3ffb..1de0c46427 100644
--- a/tensorflow/core/grappler/optimizers/data/make_sloppy.cc
+++ b/tensorflow/core/grappler/optimizers/data/make_sloppy.cc
@@ -25,8 +25,10 @@ limitations under the License.
 namespace tensorflow {
 namespace grappler {
 
-Status MakeSloppy::Optimize(Cluster* cluster, const GrapplerItem& item,
-                            GraphDef* output) {
+Status MakeSloppy::OptimizeAndCollectStats(Cluster* cluster,
+                                           const GrapplerItem& item,
+                                           GraphDef* output,
+                                           OptimizationStats* stats) {
   *output = item.graph;
   MutableGraphView graph(output);
 
@@ -35,6 +37,7 @@ Status MakeSloppy::Optimize(Cluster* cluster, const GrapplerItem& item,
         node.op() == "ParallelMapDataset" ||
         node.op() == "ParseExampleDataset") {
       (*node.mutable_attr())["sloppy"].set_b(true);
+      stats->num_changes++;
     }
   }
   return Status::OK();
diff --git a/tensorflow/core/grappler/optimizers/data/make_sloppy.h b/tensorflow/core/grappler/optimizers/data/make_sloppy.h
index 9dcab1038d..cf42e84198 100644
--- a/tensorflow/core/grappler/optimizers/data/make_sloppy.h
+++ b/tensorflow/core/grappler/optimizers/data/make_sloppy.h
@@ -16,12 +16,12 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAKE_SLOPPY_H_
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAKE_SLOPPY_H_
 
-#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h"
+#include "tensorflow/core/grappler/optimizers/data/optimizer_base.h"
 
 namespace tensorflow {
 namespace grappler {
 
-class MakeSloppy : public CustomGraphOptimizer {
+class MakeSloppy : public TFDataOptimizerBase {
  public:
   MakeSloppy() = default;
   ~MakeSloppy() override = default;
@@ -33,8 +33,9 @@ class MakeSloppy : public CustomGraphOptimizer {
     return Status::OK();
   }
 
-  Status Optimize(Cluster* cluster, const GrapplerItem& item,
-                  GraphDef* output) override;
+  Status OptimizeAndCollectStats(Cluster* cluster, const GrapplerItem& item,
+                                 GraphDef* output,
+                                 OptimizationStats* stats) override;
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimize_output, double result) override {}
diff --git a/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc b/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc
index 4ff749c57b..3aeac021e6 100644
--- a/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc
@@ -98,8 +98,10 @@ NodeDef MakeMapAndBatchNode(const NodeDef& map_node, const NodeDef& batch_node,
 
 }  // namespace
 
-Status MapAndBatchFusion::Optimize(Cluster* cluster, const GrapplerItem& item,
-                                   GraphDef* output) {
+Status MapAndBatchFusion::OptimizeAndCollectStats(Cluster* cluster,
+                                                  const GrapplerItem& item,
+                                                  GraphDef* output,
+                                                  OptimizationStats* stats) {
   *output = item.graph;
   MutableGraphView graph(output);
   std::set<string> nodes_to_delete;
@@ -126,6 +128,7 @@ Status MapAndBatchFusion::Optimize(Cluster* cluster, const GrapplerItem& item,
     // Mark the `Map` and `Batch` nodes for removal.
     nodes_to_delete.insert(map_node->name());
     nodes_to_delete.insert(batch_node.name());
+    stats->num_changes++;
   }
 
   graph.DeleteNodes(nodes_to_delete);
@@ -140,5 +143,5 @@ void MapAndBatchFusion::Feedback(Cluster* cluster, const GrapplerItem& item,
 
 REGISTER_GRAPH_OPTIMIZER_AS(MapAndBatchFusion, "map_and_batch_fusion");
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.h b/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.h
index 2c64831105..ef3a218bf3 100644
--- a/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.h
+++ b/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.h
@@ -16,12 +16,12 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_AND_BATCH_FUSION_H_
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_AND_BATCH_FUSION_H_
 
-#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h"
+#include "tensorflow/core/grappler/optimizers/data/optimizer_base.h"
 
 namespace tensorflow {
 namespace grappler {
 
-class MapAndBatchFusion : public CustomGraphOptimizer {
+class MapAndBatchFusion : public TFDataOptimizerBase {
  public:
   MapAndBatchFusion() = default;
   ~MapAndBatchFusion() override = default;
@@ -33,14 +33,15 @@ class MapAndBatchFusion : public CustomGraphOptimizer {
     return Status::OK();
   }
 
-  Status Optimize(Cluster* cluster, const GrapplerItem& item,
-                  GraphDef* output) override;
+  Status OptimizeAndCollectStats(Cluster* cluster, const GrapplerItem& item,
+                                 GraphDef* output,
+                                 OptimizationStats* stats) override;
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimize_output, double result) override;
 };
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_AND_BATCH_FUSION_H_
diff --git a/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.cc b/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.cc
index 128a2fefa2..1ed4eae4ba 100644
--- a/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.cc
@@ -92,8 +92,10 @@ NodeDef MakeFilterByLastComponentNode(const NodeDef& fused_map_node,
 
 }  // namespace
 
-Status MapAndFilterFusion::Optimize(Cluster* cluster, const GrapplerItem& item,
-                                    GraphDef* output) {
+Status MapAndFilterFusion::OptimizeAndCollectStats(Cluster* cluster,
+                                                   const GrapplerItem& item,
+                                                   GraphDef* output,
+                                                   OptimizationStats* stats) {
   GraphDef sorted_old_graph = item.graph;
   TF_RETURN_IF_ERROR(TopologicalSort(&sorted_old_graph));
   // TODO(prazek): We might have some problems with performance if we copy
@@ -163,6 +165,7 @@ Status MapAndFilterFusion::Optimize(Cluster* cluster, const GrapplerItem& item,
     // used anymore.
     nodes_to_delete.insert(map_node->name());
     nodes_to_delete.insert(filter_node->name());
+    stats->num_changes++;
   }
 
   graph.DeleteNodes(nodes_to_delete);
@@ -177,5 +180,5 @@ void MapAndFilterFusion::Feedback(Cluster* cluster, const GrapplerItem& item,
 
 REGISTER_GRAPH_OPTIMIZER_AS(MapAndFilterFusion, "map_and_filter_fusion");
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.h b/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.h
index ba25ca0591..8b3c95d37c 100644
--- a/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.h
+++ b/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.h
@@ -16,7 +16,7 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_AND_FILTER_FUSION_H_
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_AND_FILTER_FUSION_H_
 
-#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h"
+#include "tensorflow/core/grappler/optimizers/data/optimizer_base.h"
 
 namespace tensorflow {
 namespace grappler {
@@ -26,7 +26,7 @@ namespace grappler {
 // component. The FilterDataset is transformed to FilterByLastComponent - a
 // custom kernel that filters elements based on a value of the boolean
 // component.
-class MapAndFilterFusion : public CustomGraphOptimizer {
+class MapAndFilterFusion : public TFDataOptimizerBase {
  public:
   MapAndFilterFusion() = default;
   ~MapAndFilterFusion() override = default;
@@ -38,14 +38,15 @@ class MapAndFilterFusion : public CustomGraphOptimizer {
     return Status::OK();
   }
 
-  Status Optimize(Cluster* cluster, const GrapplerItem& item,
-                  GraphDef* output) override;
+  Status OptimizeAndCollectStats(Cluster* cluster, const GrapplerItem& item,
+                                 GraphDef* output,
+                                 OptimizationStats* stats) override;
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimize_output, double result) override;
 };
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_AND_FILTER_FUSION_H_
diff --git a/tensorflow/core/grappler/optimizers/data/map_fusion.cc b/tensorflow/core/grappler/optimizers/data/map_fusion.cc
index 504fcc1f5e..66f8404c05 100644
--- a/tensorflow/core/grappler/optimizers/data/map_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_fusion.cc
@@ -77,8 +77,10 @@ NodeDef MakeFusedNode(const NodeDef& parent_map_node, const NodeDef& map_node,
 
 }  // namespace
 
-Status MapFusion::Optimize(Cluster* cluster, const GrapplerItem& item,
-                           GraphDef* output) {
+Status MapFusion::OptimizeAndCollectStats(Cluster* cluster,
+                                          const GrapplerItem& item,
+                                          GraphDef* output,
+                                          OptimizationStats* stats) {
   GraphDef sorted_old_graph = item.graph;
   TF_RETURN_IF_ERROR(TopologicalSort(&sorted_old_graph));
   *output = sorted_old_graph;
@@ -142,6 +144,7 @@ Status MapFusion::Optimize(Cluster* cluster, const GrapplerItem& item,
     // they are not used anymore.
     nodes_to_delete.insert(parent_map_node->name());
     nodes_to_delete.insert(map_node->name());
+    stats->num_changes++;
   }
 
   graph.DeleteNodes(nodes_to_delete);
@@ -155,5 +158,5 @@ void MapFusion::Feedback(Cluster* cluster, const GrapplerItem& item,
 
 REGISTER_GRAPH_OPTIMIZER_AS(MapFusion, "map_fusion");
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/map_fusion.h b/tensorflow/core/grappler/optimizers/data/map_fusion.h
index a6a06592b8..c9960c7217 100644
--- a/tensorflow/core/grappler/optimizers/data/map_fusion.h
+++ b/tensorflow/core/grappler/optimizers/data/map_fusion.h
@@ -16,13 +16,13 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_FUSION_H_
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_FUSION_H_
 
-#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h"
+#include "tensorflow/core/grappler/optimizers/data/optimizer_base.h"
 
 namespace tensorflow {
 namespace grappler {
 
 // This optimization fuses map transformations by merging their map functions.
-class MapFusion : public CustomGraphOptimizer {
+class MapFusion : public TFDataOptimizerBase {
  public:
   MapFusion() = default;
   ~MapFusion() override = default;
@@ -34,14 +34,15 @@ class MapFusion : public CustomGraphOptimizer {
     return Status::OK();
   }
 
-  Status Optimize(Cluster* cluster, const GrapplerItem& item,
-                  GraphDef* output) override;
+  Status OptimizeAndCollectStats(Cluster* cluster, const GrapplerItem& item,
+                                 GraphDef* output,
+                                 OptimizationStats* stats) override;
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimize_output, double result) override;
 };
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_FUSION_H_
diff --git a/tensorflow/core/grappler/optimizers/data/map_parallelization.cc b/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
index 1274af7160..d8f01d720a 100644
--- a/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
@@ -28,9 +28,9 @@ namespace tensorflow {
 namespace grappler {
 namespace {
 
-static const char* kMapDataset = "MapDataset";
-static const char* kParallelMapDataset = "ParallelMapDataset";
-static constexpr int kAutotune = -1;
+constexpr char kMapDataset[] = "MapDataset";
+constexpr char kParallelMapDataset[] = "ParallelMapDataset";
+constexpr int kAutotune = -1;
 
 bool CanParallelize(const FunctionDef& function,
                     const FunctionLibraryDefinition& library) {
@@ -67,8 +67,10 @@ NodeDef MakeParallelMap(const string& name, MutableGraphView* graph) {
 
 }  // namespace
 
-Status MapParallelization::Optimize(Cluster* cluster, const GrapplerItem& item,
-                                    GraphDef* output) {
+Status MapParallelization::OptimizeAndCollectStats(Cluster* cluster,
+                                                   const GrapplerItem& item,
+                                                   GraphDef* output,
+                                                   OptimizationStats* stats) {
   *output = item.graph;
   MutableGraphView graph(output);
   std::set<string> nodes_to_delete;
@@ -92,6 +94,7 @@ Status MapParallelization::Optimize(Cluster* cluster, const GrapplerItem& item,
     TF_RETURN_IF_ERROR(
         graph.UpdateFanouts(map_node->name(), parallel_map->name()));
     nodes_to_delete.insert(map_node->name());
+    stats->num_changes++;
   }
 
   graph.DeleteNodes(nodes_to_delete);
@@ -106,5 +109,5 @@ void MapParallelization::Feedback(Cluster* cluster, const GrapplerItem& item,
 
 REGISTER_GRAPH_OPTIMIZER_AS(MapParallelization, "map_parallelization");
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/map_parallelization.h b/tensorflow/core/grappler/optimizers/data/map_parallelization.h
index ac9cf7e12a..8e71dadcb8 100644
--- a/tensorflow/core/grappler/optimizers/data/map_parallelization.h
+++ b/tensorflow/core/grappler/optimizers/data/map_parallelization.h
@@ -16,13 +16,13 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_PARALLELIZATION_H_
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_PARALLELIZATION_H_
 
-#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h"
+#include "tensorflow/core/grappler/optimizers/data/optimizer_base.h"
 
 namespace tensorflow {
 namespace grappler {
 
 // This optimization parallelizes MapDataset when function is stateless.
-class MapParallelization : public CustomGraphOptimizer {
+class MapParallelization : public TFDataOptimizerBase {
  public:
   MapParallelization() = default;
   ~MapParallelization() override = default;
@@ -34,14 +34,15 @@ class MapParallelization : public CustomGraphOptimizer {
     return Status::OK();
   }
 
-  Status Optimize(Cluster* cluster, const GrapplerItem& item,
-                  GraphDef* output) override;
+  Status OptimizeAndCollectStats(Cluster* cluster, const GrapplerItem& item,
+                                 GraphDef* output,
+                                 OptimizationStats* stats) override;
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimize_output, double result) override;
 };
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_PARALLELIZATION_H_
diff --git a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
index bc1d6d0ea6..848642041b 100644
--- a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
@@ -212,8 +212,10 @@ NodeDef MakeNewMapNode(const NodeDef& old_map_node,
 
 }  // namespace
 
-Status MapVectorization::Optimize(Cluster* cluster, const GrapplerItem& item,
-                                  GraphDef* output) {
+Status MapVectorization::OptimizeAndCollectStats(Cluster* cluster,
+                                                 const GrapplerItem& item,
+                                                 GraphDef* output,
+                                                 OptimizationStats* stats) {
   *output = item.graph;
   MutableGraphView graph(output);
   std::set<string> nodes_to_delete;
@@ -270,6 +272,7 @@ Status MapVectorization::Optimize(Cluster* cluster, const GrapplerItem& item,
     // Mark the `Map` and `Batch` nodes for removal.
     nodes_to_delete.insert(map_node->name());
     nodes_to_delete.insert(batch_node.name());
+    stats->num_changes++;
   }
   graph.DeleteNodes(nodes_to_delete);
   return Status::OK();
@@ -283,5 +286,5 @@ void MapVectorization::Feedback(Cluster* cluster, const GrapplerItem& item,
 
 REGISTER_GRAPH_OPTIMIZER_AS(MapVectorization, "map_vectorization");
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/map_vectorization.h b/tensorflow/core/grappler/optimizers/data/map_vectorization.h
index cc56a8ee5e..cde6ef6839 100644
--- a/tensorflow/core/grappler/optimizers/data/map_vectorization.h
+++ b/tensorflow/core/grappler/optimizers/data/map_vectorization.h
@@ -16,12 +16,12 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_VECTORIZATION_H_
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_VECTORIZATION_H_
 
-#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h"
+#include "tensorflow/core/grappler/optimizers/data/optimizer_base.h"
 
 namespace tensorflow {
 namespace grappler {
 
-class MapVectorization : public CustomGraphOptimizer {
+class MapVectorization : public TFDataOptimizerBase {
  public:
   MapVectorization() = default;
   ~MapVectorization() override = default;
@@ -33,14 +33,15 @@ class MapVectorization : public CustomGraphOptimizer {
     return Status::OK();
   }
 
-  Status Optimize(Cluster* cluster, const GrapplerItem& item,
-                  GraphDef* output) override;
+  Status OptimizeAndCollectStats(Cluster* cluster, const GrapplerItem& item,
+                                 GraphDef* output,
+                                 OptimizationStats* stats) override;
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimize_output, double result) override;
 };
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_VECTORIZATION_H_
diff --git a/tensorflow/core/grappler/optimizers/data/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/data/meta_optimizer.cc
index 0fc0cf43c6..584759f85d 100644
--- a/tensorflow/core/grappler/optimizers/data/meta_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/data/meta_optimizer.cc
@@ -121,5 +121,5 @@ void TFDataMetaOptimizer::Feedback(Cluster* cluster, const GrapplerItem& item,
 
 REGISTER_GRAPH_OPTIMIZER_AS(TFDataMetaOptimizer, "tf_data_meta_optimizer");
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/meta_optimizer.h b/tensorflow/core/grappler/optimizers/data/meta_optimizer.h
index c39ddda4cb..b65e702777 100644
--- a/tensorflow/core/grappler/optimizers/data/meta_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/data/meta_optimizer.h
@@ -50,7 +50,7 @@ class TFDataMetaOptimizer : public CustomGraphOptimizer {
                            GrapplerItem* item) const;
 };
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_META_OPTIMIZER_H_
diff --git a/tensorflow/core/grappler/optimizers/data/noop_elimination.cc b/tensorflow/core/grappler/optimizers/data/noop_elimination.cc
index b71e9e5420..e474fddcc0 100644
--- a/tensorflow/core/grappler/optimizers/data/noop_elimination.cc
+++ b/tensorflow/core/grappler/optimizers/data/noop_elimination.cc
@@ -70,8 +70,10 @@ bool IsNoOp(const NodeDef& node, const MutableGraphView& graph) {
 
 }  // namespace
 
-Status NoOpElimination::Optimize(Cluster* cluster, const GrapplerItem& item,
-                                 GraphDef* output) {
+Status NoOpElimination::OptimizeAndCollectStats(Cluster* cluster,
+                                                const GrapplerItem& item,
+                                                GraphDef* output,
+                                                OptimizationStats* stats) {
   *output = item.graph;
   MutableGraphView graph(output);
   std::set<string> nodes_to_delete;
@@ -82,6 +84,7 @@ Status NoOpElimination::Optimize(Cluster* cluster, const GrapplerItem& item,
     TF_RETURN_IF_ERROR(graph.UpdateFanouts(node.name(), parent->name()));
 
     nodes_to_delete.insert(node.name());
+    stats->num_changes++;
   }
 
   graph.DeleteNodes(nodes_to_delete);
@@ -95,5 +98,5 @@ void NoOpElimination::Feedback(Cluster* cluster, const GrapplerItem& item,
 
 REGISTER_GRAPH_OPTIMIZER_AS(NoOpElimination, "noop_elimination");
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/noop_elimination.h b/tensorflow/core/grappler/optimizers/data/noop_elimination.h
index a65fccd882..11d86ad2a3 100644
--- a/tensorflow/core/grappler/optimizers/data/noop_elimination.h
+++ b/tensorflow/core/grappler/optimizers/data/noop_elimination.h
@@ -16,14 +16,14 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_NOOP_ELIMINATION_H_
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_NOOP_ELIMINATION_H_
 
-#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h"
+#include "tensorflow/core/grappler/optimizers/data/optimizer_base.h"
 
 namespace tensorflow {
 namespace grappler {
 
 // This class eliminates tf.data transformations such as `take(n)` (for n < 0),
 // `skip(0)`, `repeat(1)`, or `prefetch(0)`.
-class NoOpElimination : public CustomGraphOptimizer {
+class NoOpElimination : public TFDataOptimizerBase {
  public:
   NoOpElimination() = default;
   ~NoOpElimination() override = default;
@@ -35,14 +35,15 @@ class NoOpElimination : public CustomGraphOptimizer {
     return Status::OK();
   }
 
-  Status Optimize(Cluster* cluster, const GrapplerItem& item,
-                  GraphDef* output) override;
+  Status OptimizeAndCollectStats(Cluster* cluster, const GrapplerItem& item,
+                                 GraphDef* output,
+                                 OptimizationStats* stats) override;
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimize_output, double result) override;
 };
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_NOOP_ELIMINATION_H_
diff --git a/tensorflow/core/kernels/data/model_dataset_op.h b/tensorflow/core/grappler/optimizers/data/optimizer_base.cc
similarity index 51%
rename from tensorflow/core/kernels/data/model_dataset_op.h
rename to tensorflow/core/grappler/optimizers/data/optimizer_base.cc
index 38d3fcb234..7fc0da3579 100644
--- a/tensorflow/core/kernels/data/model_dataset_op.h
+++ b/tensorflow/core/grappler/optimizers/data/optimizer_base.cc
@@ -13,30 +13,22 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CORE_KERNELS_DATA_MODEL_DATASET_OP_H_
-#define TENSORFLOW_CORE_KERNELS_DATA_MODEL_DATASET_OP_H_
+#include "tensorflow/core/grappler/optimizers/data/optimizer_base.h"
 
-#include "tensorflow/core/framework/dataset.h"
-#include "tensorflow/core/framework/model.h"
+#include "tensorflow/core/common_runtime/metrics.h"
 
 namespace tensorflow {
-namespace data {
-
-class ModelDatasetOp : public UnaryDatasetOpKernel {
- public:
-  using ModelFactory = std::function<std::shared_ptr<model::Model>()>;
-
-  explicit ModelDatasetOp(OpKernelConstruction* ctx);
-
-  void MakeDataset(OpKernelContext* ctx, DatasetBase* input,
-                   DatasetBase** output) override;
-
-  virtual ModelFactory CreateModelFactory() const;
-
- private:
-  class Dataset;
-};
-
-}  // namespace data
+namespace grappler {
+
+Status TFDataOptimizerBase::Optimize(Cluster* cluster, const GrapplerItem& item,
+                                     GraphDef* output) {
+  OptimizationStats stats;
+  Status s = OptimizeAndCollectStats(cluster, item, output, &stats);
+  if (s.ok() && stats.num_changes > 0) {
+    metrics::RecordTFDataOptimization(name(), stats.num_changes);
+  }
+  return s;
+}
+
+}  // namespace grappler
 }  // namespace tensorflow
-#endif  // TENSORFLOW_CORE_KERNELS_DATA_MODEL_DATASET_OP_H_
diff --git a/tensorflow/core/grappler/optimizers/data/optimizer_base.h b/tensorflow/core/grappler/optimizers/data/optimizer_base.h
new file mode 100644
index 0000000000..45af5a4b7d
--- /dev/null
+++ b/tensorflow/core/grappler/optimizers/data/optimizer_base.h
@@ -0,0 +1,47 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_OPTIMIZER_BASE_H_
+#define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_OPTIMIZER_BASE_H_
+
+#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h"
+
+namespace tensorflow {
+namespace grappler {
+
+// A base class for tf.data optimizers.
+class TFDataOptimizerBase : public CustomGraphOptimizer {
+ public:
+  struct OptimizationStats {
+    // Identifies the number of independent graph changes for an optimization.
+    int64 num_changes = 0;
+  };
+
+  TFDataOptimizerBase() = default;
+  ~TFDataOptimizerBase() override = default;
+
+  Status Optimize(Cluster* cluster, const GrapplerItem& item,
+                  GraphDef* output) final;
+
+  virtual Status OptimizeAndCollectStats(Cluster* cluster,
+                                         const GrapplerItem& item,
+                                         GraphDef* output,
+                                         OptimizationStats* stats) = 0;
+};
+
+}  // namespace grappler
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_OPTIMIZER_BASE_H_
diff --git a/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.cc b/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.cc
index f03648e7e9..f30b2230ff 100644
--- a/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.cc
@@ -34,9 +34,9 @@ constexpr char kFusedOpName[] = "ShuffleAndRepeatDataset";
 
 }  // namespace
 
-Status ShuffleAndRepeatFusion::Optimize(Cluster* cluster,
-                                        const GrapplerItem& item,
-                                        GraphDef* output) {
+Status ShuffleAndRepeatFusion::OptimizeAndCollectStats(
+    Cluster* cluster, const GrapplerItem& item, GraphDef* output,
+    OptimizationStats* stats) {
   *output = item.graph;
   MutableGraphView graph(output);
   std::set<string> nodes_to_delete;
@@ -92,6 +92,7 @@ Status ShuffleAndRepeatFusion::Optimize(Cluster* cluster,
     // Mark the `Shuffle` and `Repeat` nodes for removal.
     nodes_to_delete.insert(shuffle_node.name());
     nodes_to_delete.insert(repeat_node.name());
+    stats->num_changes++;
   }
 
   graph.DeleteNodes(nodes_to_delete);
@@ -108,5 +109,5 @@ void ShuffleAndRepeatFusion::Feedback(Cluster* cluster,
 REGISTER_GRAPH_OPTIMIZER_AS(ShuffleAndRepeatFusion,
                             "shuffle_and_repeat_fusion");
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.h b/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.h
index c8fa53edce..3738d141c3 100644
--- a/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.h
+++ b/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.h
@@ -16,12 +16,12 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_SHUFFLE_AND_REPEAT_FUSION_H_
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_SHUFFLE_AND_REPEAT_FUSION_H_
 
-#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h"
+#include "tensorflow/core/grappler/optimizers/data/optimizer_base.h"
 
 namespace tensorflow {
 namespace grappler {
 
-class ShuffleAndRepeatFusion : public CustomGraphOptimizer {
+class ShuffleAndRepeatFusion : public TFDataOptimizerBase {
  public:
   ShuffleAndRepeatFusion() = default;
   ~ShuffleAndRepeatFusion() override = default;
@@ -33,14 +33,15 @@ class ShuffleAndRepeatFusion : public CustomGraphOptimizer {
     return Status::OK();
   }
 
-  Status Optimize(Cluster* cluster, const GrapplerItem& item,
-                  GraphDef* output) override;
+  Status OptimizeAndCollectStats(Cluster* cluster, const GrapplerItem& item,
+                                 GraphDef* output,
+                                 OptimizationStats* stats) override;
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimize_output, double result) override;
 };
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_SHUFFLE_AND_REPEAT_FUSION_H_
diff --git a/tensorflow/core/grappler/optimizers/data/vectorization_utils.cc b/tensorflow/core/grappler/optimizers/data/vectorization_utils.cc
index 60c557d557..c57a7b1256 100644
--- a/tensorflow/core/grappler/optimizers/data/vectorization_utils.cc
+++ b/tensorflow/core/grappler/optimizers/data/vectorization_utils.cc
@@ -46,7 +46,7 @@ namespace {
 // Describes a tensor with its operation Node and output position
 typedef std::pair<Node*, int> TensorDesc;
 
-const char* const kRetValOp = "_Retval";
+constexpr char kRetValOp[] = "_Retval";
 
 void ReplaceEdgeSources(const TensorDesc& old_src, const TensorDesc& new_src,
                         Graph* graph) {
@@ -643,6 +643,6 @@ Status VectorizeMapDefun(const FunctionDef& outer_scope,
   return Vectorization(lib).Vectorize(outer_scope, map_defun_node, result);
 }
 
-}  // end namespace vectorization_utils
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace vectorization_utils
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/vectorization_utils.h b/tensorflow/core/grappler/optimizers/data/vectorization_utils.h
index bd7d390900..f5183fd4ff 100644
--- a/tensorflow/core/grappler/optimizers/data/vectorization_utils.h
+++ b/tensorflow/core/grappler/optimizers/data/vectorization_utils.h
@@ -90,8 +90,8 @@ Status VectorizeMapDefun(const FunctionDef& outer_scope,
                          const NodeDef& map_defun_node, FunctionDefLibrary* lib,
                          FunctionDef** result);
 
-}  // end namespace vectorization_utils
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace vectorization_utils
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_VECTORIZATION_UTILS_H_
diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD
index 663717ffb0..fab1ec8b6e 100644
--- a/tensorflow/core/kernels/data/BUILD
+++ b/tensorflow/core/kernels/data/BUILD
@@ -54,7 +54,6 @@ cc_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/kernels:variable_ops",
     ],
 )
@@ -189,7 +188,6 @@ cc_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:protos_all_cc",
     ],
 )
 
@@ -205,6 +203,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:metrics",
         "//tensorflow/core:protos_all_cc",
     ],
 )
@@ -262,6 +261,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:metrics",
     ],
 )
 
@@ -295,6 +295,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:metrics",
         "//tensorflow/core:protos_all_cc",
     ],
 )
@@ -488,6 +489,7 @@ tf_kernel_library(
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:dataset_ops_op_lib",
         "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/grappler:graph_view",
@@ -504,7 +506,6 @@ tf_kernel_library(
 tf_kernel_library(
     name = "model_dataset_op",
     srcs = ["model_dataset_op.cc"],
-    hdrs = ["model_dataset_op.h"],
     deps = [
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:dataset_ops_op_lib",
@@ -512,6 +513,8 @@ tf_kernel_library(
         "//tensorflow/core:framework_internal",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:metrics",
+        "@com_google_absl//absl/memory",
     ],
 )
 
diff --git a/tensorflow/core/kernels/data/experimental/BUILD b/tensorflow/core/kernels/data/experimental/BUILD
index 7433303f77..583fce6d89 100644
--- a/tensorflow/core/kernels/data/experimental/BUILD
+++ b/tensorflow/core/kernels/data/experimental/BUILD
@@ -124,6 +124,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:metrics",
         "//tensorflow/core/kernels:inplace_ops",
         "//tensorflow/core/kernels/data:captured_function",
         "//tensorflow/core/kernels/data:dataset_utils",
diff --git a/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc b/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
index 5d8e7cfabe..0ef56915eb 100644
--- a/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include <utility>
 
 #include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/common_runtime/metrics.h"
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/stats_aggregator.h"
@@ -37,8 +38,10 @@ namespace tensorflow {
 namespace data {
 namespace {
 
+constexpr char kDatasetName[] = "MapAndBatch";
+
 // Maximum number of batch results to buffer.
-const int64 kMaxBatchResults = 16;
+constexpr int64 kMaxBatchResults = 16;
 
 // See documentation in ../../ops/dataset_ops.cc for a high-level
 // description of the following op.
@@ -126,6 +129,10 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
       };
     }
 
+    if (num_parallel_calls == model::kAutoTune) {
+      metrics::RecordTFDataAutotune(kDatasetName);
+    }
+
     *output = new Dataset(ctx, input, func_, batch_size, num_parallel_calls,
                           drop_remainder, output_types_, output_shapes_,
                           std::move(captured_func), &ctx->eigen_cpu_device(),
@@ -163,7 +170,7 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
       return absl::make_unique<Iterator>(
-          Iterator::Params{this, strings::StrCat(prefix, "::MapAndBatch")},
+          Iterator::Params{this, strings::StrCat(prefix, "::", kDatasetName)},
           map_func_);
     }
 
diff --git a/tensorflow/core/kernels/data/model_dataset_op.cc b/tensorflow/core/kernels/data/model_dataset_op.cc
index e1a0d94f5f..20254234e9 100644
--- a/tensorflow/core/kernels/data/model_dataset_op.cc
+++ b/tensorflow/core/kernels/data/model_dataset_op.cc
@@ -13,180 +13,182 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/core/kernels/data/model_dataset_op.h"
-
+#include "absl/memory/memory.h"
+#include "tensorflow/core/common_runtime/metrics.h"
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/lib/random/random.h"
 #include "tensorflow/core/platform/cpu_info.h"
+#include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
 namespace data {
+namespace {
 
-const int kOptimizationPeriodThresholdMs = 60 * EnvTime::kSecondsToMicros;
+constexpr int kOptimizationPeriodThresholdMs = 60 * EnvTime::kSecondsToMicros;
 
-class ModelDatasetOp::Dataset : public DatasetBase {
+class ModelDatasetOp : public UnaryDatasetOpKernel {
  public:
-  explicit Dataset(OpKernelContext* ctx, const DatasetBase* input,
-                   ModelFactory model_factory)
-      : DatasetBase(DatasetContext(ctx)),
-        input_(input),
-        model_factory_(std::move(model_factory)) {
-    input_->Ref();
-  }
-
-  ~Dataset() override { input_->Unref(); }
+  explicit ModelDatasetOp(OpKernelConstruction* ctx)
+      : UnaryDatasetOpKernel(ctx) {}
 
-  std::unique_ptr<IteratorBase> MakeIteratorInternal(
-      const string& prefix) const override {
-    return std::unique_ptr<IteratorBase>(new Iterator(
-        {this, strings::StrCat(prefix, "::Model")}, model_factory_));
-  }
-
-  const DataTypeVector& output_dtypes() const override {
-    return input_->output_dtypes();
-  }
-  const std::vector<PartialTensorShape>& output_shapes() const override {
-    return input_->output_shapes();
-  }
-
-  string DebugString() const override { return "ModelDatasetOp::Dataset"; }
-
-  int64 Cardinality() const override { return input_->Cardinality(); }
-
- protected:
-  Status AsGraphDefInternal(SerializationContext* ctx,
-                            DatasetGraphDefBuilder* b,
-                            Node** output) const override {
-    Node* input_graph_node = nullptr;
-    TF_RETURN_IF_ERROR(b->AddInputDataset(ctx, input_, &input_graph_node));
-    TF_RETURN_IF_ERROR(b->AddDataset(this, {input_graph_node}, output));
-    return Status::OK();
+  void MakeDataset(OpKernelContext* ctx, DatasetBase* input,
+                   DatasetBase** output) override {
+    *output = new Dataset(ctx, input);
   }
 
  private:
-  class Iterator : public DatasetIterator<Dataset> {
+  class Dataset : public DatasetBase {
    public:
-    explicit Iterator(const Params& params, ModelFactory model_factory)
-        : DatasetIterator<Dataset>(params), model_(model_factory()) {}
-
-    ~Iterator() override {
-      // Signal the optimize thread to terminate it. We will then join that
-      // thread when we delete `this->optimize_thread_`.
-      mutex_lock l(mu_);
-      cancelled_ = true;
-      cond_var_.notify_all();
+    Dataset(OpKernelContext* ctx, const DatasetBase* input)
+        : DatasetBase(DatasetContext(ctx)), input_(input) {
+      input_->Ref();
     }
 
-    Status Initialize(IteratorContext* ctx) override {
-      IteratorContext::Params params(ctx);
-      params.model = model_;
-      return dataset()->input_->MakeIterator(IteratorContext(std::move(params)),
-                                             prefix(), &input_impl_);
-    }
+    ~Dataset() override { input_->Unref(); }
 
-    Status GetNextInternal(IteratorContext* ctx,
-                           std::vector<Tensor>* out_tensors,
-                           bool* end_of_sequence) override {
-      IteratorContext::Params params(ctx);
-      {
-        mutex_lock l(mu_);
-        TF_RETURN_IF_ERROR(EnsureOptimizeThreadStarted(ctx));
-        params.model = model_;
-      }
-      return input_impl_->GetNext(IteratorContext(std::move(params)),
-                                  out_tensors, end_of_sequence);
+    std::unique_ptr<IteratorBase> MakeIteratorInternal(
+        const string& prefix) const override {
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::Model")});
     }
 
-   protected:
-    std::shared_ptr<model::Node> CreateNode(
-        IteratorContext* ctx, model::Node::Args args) const override {
-      return model::MakeKnownRatioNode(std::move(args),
-                                       /*ratio=*/1);
+    const DataTypeVector& output_dtypes() const override {
+      return input_->output_dtypes();
     }
-
-    Status SaveInternal(IteratorStateWriter* writer) override {
-      mutex_lock l(mu_);
-      TF_RETURN_IF_ERROR(SaveInput(writer, input_impl_));
-      return Status::OK();
+    const std::vector<PartialTensorShape>& output_shapes() const override {
+      return input_->output_shapes();
     }
 
-    Status RestoreInternal(IteratorContext* ctx,
-                           IteratorStateReader* reader) override {
-      mutex_lock l(mu_);
-      TF_RETURN_IF_ERROR(RestoreInput(ctx, reader, input_impl_));
+    string DebugString() const override { return "ModelDatasetOp::Dataset"; }
+
+    int64 Cardinality() const override { return input_->Cardinality(); }
+
+   protected:
+    Status AsGraphDefInternal(SerializationContext* ctx,
+                              DatasetGraphDefBuilder* b,
+                              Node** output) const override {
+      Node* input_graph_node = nullptr;
+      TF_RETURN_IF_ERROR(b->AddInputDataset(ctx, input_, &input_graph_node));
+      TF_RETURN_IF_ERROR(b->AddDataset(this, {input_graph_node}, output));
       return Status::OK();
     }
 
    private:
-    Status EnsureOptimizeThreadStarted(IteratorContext* ctx)
-        EXCLUSIVE_LOCKS_REQUIRED(mu_) {
-      if (!optimize_thread_) {
-        std::shared_ptr<IteratorContext> new_ctx(new IteratorContext(*ctx));
-        optimize_thread_.reset(ctx->env()->StartThread(
-            {}, "tf_data_model",
-            [this, new_ctx]() { OptimizeThread(new_ctx); }));
+    class Iterator : public DatasetIterator<Dataset> {
+     public:
+      explicit Iterator(const Params& params)
+          : DatasetIterator<Dataset>(params) {
+        auto remove_node_hook = [](std::shared_ptr<model::Node> node) {
+          metrics::RecordTFDataElements(node->name(), node->num_elements());
+        };
+        model_ = std::make_shared<model::Model>(std::move(remove_node_hook));
+      }
+
+      ~Iterator() override {
+        // Signal the optimize thread to terminate it. We will then join that
+        // thread when we delete `this->optimize_thread_`.
+        mutex_lock l(mu_);
+        cancelled_ = true;
+        cond_var_.notify_all();
+      }
+
+      Status Initialize(IteratorContext* ctx) override {
+        IteratorContext::Params params(ctx);
+        params.model = model_;
+        return dataset()->input_->MakeIterator(
+            IteratorContext(std::move(params)), prefix(), &input_impl_);
       }
-      return Status::OK();
-    }
 
-    void OptimizeThread(const std::shared_ptr<IteratorContext>& ctx) {
-      int64 last_optimization_ms = 0;
-      int64 optimization_period_ms = 10;
-      while (true) {
+      Status GetNextInternal(IteratorContext* ctx,
+                             std::vector<Tensor>* out_tensors,
+                             bool* end_of_sequence) override {
+        IteratorContext::Params params(ctx);
         {
           mutex_lock l(mu_);
-          while (!cancelled_ &&
-                 last_optimization_ms + optimization_period_ms >=
-                     ctx->env()->NowMicros() / EnvTime::kMillisToMicros) {
-            cond_var_.wait_for(
-                l, std::chrono::milliseconds(
-                       last_optimization_ms + optimization_period_ms -
-                       ctx->env()->NowMicros() / EnvTime::kMillisToMicros));
-          }
-          if (cancelled_) return;
-        }
-        model_->Optimize(port::NumSchedulableCPUs());
-        // Exponentially increase the period of running the optimization
-        // until a threshold is reached.
-        if (optimization_period_ms < kOptimizationPeriodThresholdMs) {
-          if (optimization_period_ms << 1 < kOptimizationPeriodThresholdMs) {
-            optimization_period_ms <<= 1;
-          } else {
-            optimization_period_ms = kOptimizationPeriodThresholdMs;
-          }
+          TF_RETURN_IF_ERROR(EnsureOptimizeThreadStarted(ctx));
+          params.model = model_;
         }
-        last_optimization_ms =
-            ctx->env()->NowMicros() / EnvTime::kMillisToMicros;
+        return input_impl_->GetNext(IteratorContext(std::move(params)),
+                                    out_tensors, end_of_sequence);
       }
-    }
 
-    mutex mu_;
-    condition_variable cond_var_;
-    std::shared_ptr<model::Model> model_;
-    std::unique_ptr<Thread> optimize_thread_ GUARDED_BY(mu_);
-    bool cancelled_ GUARDED_BY(mu_) = false;
-    std::unique_ptr<IteratorBase> input_impl_;
-  };
+     protected:
+      std::shared_ptr<model::Node> CreateNode(
+          IteratorContext* ctx, model::Node::Args args) const override {
+        return model::MakeKnownRatioNode(std::move(args),
+                                         /*ratio=*/1);
+      }
 
-  const DatasetBase* input_;
-  const ModelFactory model_factory_;
-};
+      Status SaveInternal(IteratorStateWriter* writer) override {
+        mutex_lock l(mu_);
+        TF_RETURN_IF_ERROR(SaveInput(writer, input_impl_));
+        return Status::OK();
+      }
 
-ModelDatasetOp::ModelDatasetOp(OpKernelConstruction* ctx)
-    : UnaryDatasetOpKernel(ctx) {}
+      Status RestoreInternal(IteratorContext* ctx,
+                             IteratorStateReader* reader) override {
+        mutex_lock l(mu_);
+        TF_RETURN_IF_ERROR(RestoreInput(ctx, reader, input_impl_));
+        return Status::OK();
+      }
+
+     private:
+      Status EnsureOptimizeThreadStarted(IteratorContext* ctx)
+          EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+        if (!optimize_thread_) {
+          std::shared_ptr<IteratorContext> new_ctx =
+              std::make_shared<IteratorContext>(*ctx);
+          optimize_thread_.reset(ctx->env()->StartThread(
+              {}, "tf_data_model",
+              [this, new_ctx]() { OptimizeThread(new_ctx); }));
+        }
+        return Status::OK();
+      }
 
-void ModelDatasetOp::MakeDataset(OpKernelContext* ctx, DatasetBase* input,
-                                 DatasetBase** output) {
-  *output = new ModelDatasetOp::Dataset(ctx, input, CreateModelFactory());
-}
+      void OptimizeThread(const std::shared_ptr<IteratorContext>& ctx) {
+        int64 last_optimization_ms = 0;
+        int64 optimization_period_ms = 10;
+        while (true) {
+          {
+            mutex_lock l(mu_);
+            while (!cancelled_ &&
+                   last_optimization_ms + optimization_period_ms >=
+                       ctx->env()->NowMicros() / EnvTime::kMillisToMicros) {
+              cond_var_.wait_for(
+                  l, std::chrono::milliseconds(
+                         last_optimization_ms + optimization_period_ms -
+                         ctx->env()->NowMicros() / EnvTime::kMillisToMicros));
+            }
+            if (cancelled_) return;
+          }
+          model_->Optimize(port::NumSchedulableCPUs());
+          // Exponentially increase the period of running the optimization
+          // until a threshold is reached.
+          if (optimization_period_ms < kOptimizationPeriodThresholdMs) {
+            if (optimization_period_ms << 1 < kOptimizationPeriodThresholdMs) {
+              optimization_period_ms <<= 1;
+            } else {
+              optimization_period_ms = kOptimizationPeriodThresholdMs;
+            }
+          }
+          last_optimization_ms =
+              ctx->env()->NowMicros() / EnvTime::kMillisToMicros;
+        }
+      }
 
-ModelDatasetOp::ModelFactory ModelDatasetOp::CreateModelFactory() const {
-  return []() { return std::make_shared<model::Model>(); };
-}
+      mutex mu_;
+      condition_variable cond_var_;
+      std::shared_ptr<model::Model> model_;
+      std::unique_ptr<Thread> optimize_thread_ GUARDED_BY(mu_);
+      bool cancelled_ GUARDED_BY(mu_) = false;
+      std::unique_ptr<IteratorBase> input_impl_;
+    };
 
-namespace {
+    const DatasetBase* input_;
+  };
+};
 
 REGISTER_KERNEL_BUILDER(Name("ModelDataset").Device(DEVICE_CPU),
                         ModelDatasetOp);
diff --git a/tensorflow/core/kernels/data/optimize_dataset_op.cc b/tensorflow/core/kernels/data/optimize_dataset_op.cc
index 04cc48a0be..6047dc5f3f 100644
--- a/tensorflow/core/kernels/data/optimize_dataset_op.cc
+++ b/tensorflow/core/kernels/data/optimize_dataset_op.cc
@@ -40,7 +40,7 @@ namespace tensorflow {
 namespace data {
 namespace {
 
-static const char* const kOptimizerName = "tf_data_meta_optimizer";
+constexpr char kOptimizerName[] = "tf_data_meta_optimizer";
 
 // See documentation in ../../ops/dataset_ops.cc for a high-level
 // description of the following op.
@@ -99,7 +99,7 @@ class OptimizeDatasetOp : public UnaryDatasetOpKernel {
       // prefix is used to identify checkpoint elements and since the
       // optimization dataset is excluded from the checkpoint, adding a token
       // here would result in invalid checkpoint identifiers.
-      return std::unique_ptr<IteratorBase>(new Iterator({this, prefix}));
+      return absl::make_unique<Iterator>(Iterator::Params{this, prefix});
     }
 
     Status Optimize(OpKernelContext* ctx) {
@@ -129,7 +129,7 @@ class OptimizeDatasetOp : public UnaryDatasetOpKernel {
           ctx->function_library()->Clone(&flib_def_, &pflr_, &lib_));
 
       // Create a FunctionHandleCache.
-      function_handle_cache_.reset(new FunctionHandleCache(lib_));
+      function_handle_cache_ = absl::make_unique<FunctionHandleCache>(lib_);
 
       // Some functions may have been modified without having their names
       // changed (for example, nested dataset graphs from FlatMap or
@@ -301,7 +301,8 @@ class OptimizeDatasetOp : public UnaryDatasetOpKernel {
       RewriterConfig& rewriter_config =
           *config.mutable_graph_options()->mutable_rewrite_options();
       rewriter_config.add_optimizers(kOptimizerName);
-
+      rewriter_config.set_meta_optimizer_iterations(
+          RewriterConfig_NumIterationsType_ONE);
       auto custom_optimizer = rewriter_config.add_custom_optimizers();
       custom_optimizer->set_name(kOptimizerName);
       auto* custom_optimizations_list =
diff --git a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
index add123f34a..ddd81d4596 100644
--- a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
+++ b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include <utility>
 
 #include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/common_runtime/metrics.h"
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/stats_aggregator.h"
@@ -34,6 +35,8 @@ namespace tensorflow {
 namespace data {
 namespace {
 
+constexpr char kDatasetName[] = "ParallelInterleaveV2";
+
 // See documentation in ../../ops/dataset_ops.cc for a high-level
 // description of the following op.
 
@@ -87,6 +90,10 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
         ctx, CapturedFunction::Create(interleave_func_, ctx, "other_arguments",
                                       &captured_func));
 
+    if (num_parallel_calls == model::kAutoTune) {
+      metrics::RecordTFDataAutotune(kDatasetName);
+    }
+
     *output =
         new Dataset(ctx, input, interleave_func_, std::move(captured_func),
                     cycle_length, block_length, num_parallel_calls, sloppy_,
@@ -121,7 +128,7 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
         const string& prefix) const override {
       return absl::make_unique<ParallelInterleaveIterator>(
           ParallelInterleaveIterator::Params{
-              this, strings::StrCat(prefix, "::ParallelInterleaveV2")},
+              this, strings::StrCat(prefix, "::", kDatasetName)},
           sloppy_);
     }
 
diff --git a/tensorflow/core/kernels/data/parallel_map_dataset_op.cc b/tensorflow/core/kernels/data/parallel_map_dataset_op.cc
index e2c9057cb2..34f341d1d1 100644
--- a/tensorflow/core/kernels/data/parallel_map_dataset_op.cc
+++ b/tensorflow/core/kernels/data/parallel_map_dataset_op.cc
@@ -15,6 +15,7 @@ limitations under the License.
 #include <deque>
 
 #include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/common_runtime/metrics.h"
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/tensor.h"
@@ -31,6 +32,8 @@ namespace {
 // See documentation in ../../ops/dataset_ops.cc for a high-level
 // description of the following op.
 
+constexpr char kDatasetName[] = "ParallelMap";
+
 class ParallelMapDatasetOp : public UnaryDatasetOpKernel {
  public:
   explicit ParallelMapDatasetOp(OpKernelConstruction* ctx)
@@ -64,6 +67,10 @@ class ParallelMapDatasetOp : public UnaryDatasetOpKernel {
     std::vector<int> indices;
     OP_REQUIRES_OK(ctx, ComputeShortCircuitIndices(ctx, func_, &indices));
 
+    if (num_parallel_calls == model::kAutoTune) {
+      metrics::RecordTFDataAutotune(kDatasetName);
+    }
+
     *output =
         new Dataset(ctx, input, func_, num_parallel_calls, output_types_,
                     output_shapes_, use_inter_op_parallelism_, sloppy_,
@@ -108,7 +115,7 @@ class ParallelMapDatasetOp : public UnaryDatasetOpKernel {
         parallel_map_functor = absl::make_unique<ShortCircuitFunctor>(this);
       }
       return NewParallelMapIterator(
-          {this, strings::StrCat(prefix, "::ParallelMap")}, input_,
+          {this, strings::StrCat(prefix, "::", kDatasetName)}, input_,
           std::move(parallel_map_functor), num_parallel_calls_, sloppy_,
           preserve_cardinality_);
     }
diff --git a/tensorflow/core/kernels/data/prefetch_dataset_op.cc b/tensorflow/core/kernels/data/prefetch_dataset_op.cc
index 1f2dd6ca95..f0e835a27c 100644
--- a/tensorflow/core/kernels/data/prefetch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/prefetch_dataset_op.cc
@@ -16,6 +16,7 @@ limitations under the License.
 
 #include <deque>
 
+#include "tensorflow/core/common_runtime/metrics.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/stats_aggregator.h"
 #include "tensorflow/core/framework/tensor.h"
@@ -29,6 +30,8 @@ namespace data {
 // See documentation in ../../ops/dataset_ops.cc for a high-level
 // description of the following op.
 
+constexpr char kDatasetName[] = "Prefetch";
+
 class PrefetchDatasetOp::Dataset : public DatasetBase {
  public:
   Dataset(OpKernelContext* ctx, const DatasetBase* input, int64 buffer_size)
@@ -43,7 +46,7 @@ class PrefetchDatasetOp::Dataset : public DatasetBase {
   std::unique_ptr<IteratorBase> MakeIteratorInternal(
       const string& prefix) const override {
     return absl::make_unique<Iterator>(
-        Iterator::Params{this, strings::StrCat(prefix, "::Prefetch")});
+        Iterator::Params{this, strings::StrCat(prefix, "::", kDatasetName)});
   }
 
   const DataTypeVector& output_dtypes() const override {
@@ -392,6 +395,10 @@ void PrefetchDatasetOp::MakeDataset(OpKernelContext* ctx, DatasetBase* input,
               buffer_size >= 0 || buffer_size == PrefetchAutotuner::kAutoTune,
               errors::InvalidArgument("buffer_size must be >= 0"));
 
+  if (buffer_size == PrefetchAutotuner::kAutoTune) {
+    metrics::RecordTFDataAutotune(kDatasetName);
+  }
+
   *output = new Dataset(ctx, input, buffer_size);
 }
 
-- 
GitLab


From a6de72009482036ef3c367e59a267ca30d0903ee Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 20:21:26 -0800
Subject: [PATCH 0703/2345] Fix bug in the default tf.distribute.Strategy where
 the return value of make_input_fn_iterator() did not have an `initialize()`
 method.

PiperOrigin-RevId: 229305002
---
 tensorflow/python/distribute/distribute_lib.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/distribute/distribute_lib.py b/tensorflow/python/distribute/distribute_lib.py
index 66112dceda..421b2a6520 100644
--- a/tensorflow/python/distribute/distribute_lib.py
+++ b/tensorflow/python/distribute/distribute_lib.py
@@ -1685,7 +1685,8 @@ class _DefaultDistributionExtended(DistributionStrategyExtended):
   def _make_input_fn_iterator(self,
                               input_fn,
                               replication_mode=InputReplicationMode.PER_WORKER):
-    return input_fn(InputContext()).make_initializable_iterator()
+    dataset = input_fn(InputContext())
+    return _DefaultDistributionExtended.DefaultInputIterator(dataset)
 
   def _experimental_make_numpy_dataset(self, numpy_input, session):
     numpy_flat = nest.flatten(numpy_input)
-- 
GitLab


From 1573389a17eaeff49b7af29fad2d1deb52187e1b Mon Sep 17 00:00:00 2001
From: Albin Joy <albin.joy@huawei.com>
Date: Tue, 15 Jan 2019 10:06:51 +0530
Subject: [PATCH 0704/2345] Typo error fixed execution_callbacks.py

---
 tensorflow/python/eager/execution_callbacks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/eager/execution_callbacks.py b/tensorflow/python/eager/execution_callbacks.py
index af1afa3454..34fa3da39a 100644
--- a/tensorflow/python/eager/execution_callbacks.py
+++ b/tensorflow/python/eager/execution_callbacks.py
@@ -64,7 +64,7 @@ class InfOrNanError(Exception):
     """Constructor of InfOrNanError.
 
     Args:
-      op_type: Type name of the op that generated the tensor that generated the
+      op_type: Type name of the op that generated the tensor with
         `inf`(s) or `nan`(s) (e.g., `Div`).
       op_name: Name of the op that generated the tensor with `inf`(s) or
         `nan`(s). This name is set by client and can be `None` if it is unset.
-- 
GitLab


From dfce752a5a2c9d69186401b4f5bdc4e03413a916 Mon Sep 17 00:00:00 2001
From: James Ring <sjr@google.com>
Date: Mon, 14 Jan 2019 21:45:54 -0800
Subject: [PATCH 0705/2345] Add some helper functions to C API.

This change adds:
  * TF_OpKernel{Context,Construction}_Failure - used by C kernels to
    report failures in their compute or construction functions

  * TF_OpKernel{Context,Construction_GetAttrType - used by C kernels to
    read TF_DataType attributes from their compute or construction
    context. A macro is used for the implementation, as there will be a
    handful of these methods in subsequent CLs and their implementation is
    boilerplate.

PiperOrigin-RevId: 229311735
---
 tensorflow/c/c_test.c        |  5 +++++
 tensorflow/c/kernels.cc      | 32 ++++++++++++++++++++++++++++++++
 tensorflow/c/kernels.h       | 27 +++++++++++++++++++++++++++
 tensorflow/c/kernels_test.cc | 22 ++++++++++++++++++++--
 4 files changed, 84 insertions(+), 2 deletions(-)

diff --git a/tensorflow/c/c_test.c b/tensorflow/c/c_test.c
index 100aefd58c..b86d8eb8e3 100644
--- a/tensorflow/c/c_test.c
+++ b/tensorflow/c/c_test.c
@@ -32,7 +32,12 @@ void compute(void* kernel, TF_OpKernelContext* ctx) {
   TF_Status* s = TF_NewStatus();
   TF_GetInput(ctx, 0, &input, s);
   TF_DeleteTensor(input);
+
+  TF_DataType type;
+  TF_OpKernelContext_GetAttrType(ctx, "foobar", &type, s);
+
   TF_DeleteStatus(s);
+
 }
 
 // Exercises tensorflow's C API.
diff --git a/tensorflow/c/kernels.cc b/tensorflow/c/kernels.cc
index 673aed558a..c45efd7fae 100644
--- a/tensorflow/c/kernels.cc
+++ b/tensorflow/c/kernels.cc
@@ -159,6 +159,38 @@ void TF_SetOutput(TF_OpKernelContext* ctx, int i, const TF_Tensor* tensor,
   }
 }
 
+void TF_OpKernelConstruction_Failure(TF_OpKernelConstruction* ctx,
+                                     TF_Status* status) {
+  auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelConstruction*>(ctx);
+  ::tensorflow::Status s(::tensorflow::StatusFromTF_Status(status));
+  cc_ctx->CtxFailure(s);
+}
+
+void TF_OpKernelContext_Failure(TF_OpKernelContext* ctx, TF_Status* status) {
+  auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelContext*>(ctx);
+  ::tensorflow::Status s(::tensorflow::StatusFromTF_Status(status));
+  cc_ctx->CtxFailure(s);
+}
+
+#define DEFINE_TF_GETATTR_(struct_name, func, c_type, cc_type)                 \
+  void struct_name##_GetAttr##func(struct_name* ctx, const char* attr_name,    \
+                                   c_type* val, TF_Status* status) {           \
+    TF_SetStatus(status, TF_OK, "");                                           \
+    cc_type v;                                                                 \
+    auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelConstruction*>(ctx); \
+    ::tensorflow::Status s = cc_ctx->GetAttr(attr_name, &v);                   \
+    ::tensorflow::Set_TF_Status_from_Status(status, s);                        \
+    if (s.ok()) {                                                              \
+      *val = static_cast<c_type>(v);                                           \
+    }                                                                          \
+  }
+
+#define DEFINE_TF_GETATTR(func, c_type, cc_type)                     \
+  DEFINE_TF_GETATTR_(TF_OpKernelConstruction, func, c_type, cc_type) \
+  DEFINE_TF_GETATTR_(TF_OpKernelContext, func, c_type, cc_type)
+
+DEFINE_TF_GETATTR(Type, TF_DataType, tensorflow::DataType)
+
 TF_DataType TF_ExpectedOutputDataType(TF_OpKernelContext* ctx, int i) {
   auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelContext*>(ctx);
   return static_cast<TF_DataType>(cc_ctx->expected_output_dtype(i));
diff --git a/tensorflow/c/kernels.h b/tensorflow/c/kernels.h
index 721a4aca0b..b015d01039 100644
--- a/tensorflow/c/kernels.h
+++ b/tensorflow/c/kernels.h
@@ -111,6 +111,15 @@ TF_CAPI_EXPORT extern void TF_SetOutput(TF_OpKernelContext* ctx, int i,
                                         const TF_Tensor* tensor,
                                         TF_Status* status);
 
+// Notifies the given OpKernelConstruction that kernel construction has failed.
+TF_CAPI_EXPORT extern void TF_OpKernelConstruction_Failure(
+    TF_OpKernelConstruction* ctx, TF_Status* status);
+
+// Notifies the given OpKernelContext that the kernel's compute function has
+// failed.
+TF_CAPI_EXPORT extern void TF_OpKernelContext_Failure(TF_OpKernelContext* ctx,
+                                                      TF_Status* status);
+
 // Returns the expected output data type of the ith output. If i < 0 or
 // i >= TF_NumOutputs(ctx), the program aborts.
 TF_CAPI_EXPORT extern TF_DataType TF_ExpectedOutputDataType(
@@ -119,6 +128,24 @@ TF_CAPI_EXPORT extern TF_DataType TF_ExpectedOutputDataType(
 // Returns the step ID of the given context.
 TF_CAPI_EXPORT extern int64_t TF_StepId(TF_OpKernelContext* ctx);
 
+// Interprets the named kernel construction attribute as a TF_DataType and
+// places it into *val. *status is set to TF_OK.
+//
+// If the attribute could not be found or could not be interpreted as
+// TF_DataType, *status is populated with an error.
+TF_CAPI_EXPORT extern void TF_OpKernelConstruction_GetAttrType(
+    TF_OpKernelConstruction* ctx, const char* attr_name, TF_DataType* val,
+    TF_Status* status);
+
+// Interprets the named kernel context attribute as a TF_DataType and places it
+// into *val. *status is set to TF_OK.
+//
+// If the attribute could not be found or could not be interpreted as
+// TF_DataType, *status is populated with an error.
+TF_CAPI_EXPORT extern void TF_OpKernelContext_GetAttrType(
+    TF_OpKernelContext* ctx, const char* attr_name, TF_DataType* val,
+    TF_Status* status);
+
 #ifdef __cplusplus
 } /* end extern "C" */
 #endif
diff --git a/tensorflow/c/kernels_test.cc b/tensorflow/c/kernels_test.cc
index fdeb04c84c..531eb598fa 100644
--- a/tensorflow/c/kernels_test.cc
+++ b/tensorflow/c/kernels_test.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/c/kernels.h"
 
 #include "tensorflow/c/c_api.h"
+#include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/kernel_def.pb.h"
 #include "tensorflow/core/framework/node_def.pb_text.h"
 #include "tensorflow/core/framework/op.h"
@@ -42,7 +43,17 @@ static void MyComputeFunc(void* kernel, TF_OpKernelContext* ctx) {
   struct MyCustomKernel* s = static_cast<struct MyCustomKernel*>(kernel);
   s->compute_called = true;
   if (ctx != nullptr) {
+    TF_Status* status = TF_NewStatus();
+
     EXPECT_EQ(43, TF_StepId(ctx));
+
+    // Exercise attribute reads.
+    TF_DataType type;
+    TF_OpKernelContext_GetAttrType(ctx, "SomeDataTypeAttr", &type, status);
+    EXPECT_EQ(TF_OK, TF_GetCode(status));
+    EXPECT_EQ(TF_FLOAT, type);
+
+    TF_DeleteStatus(status);
   }
 }
 
@@ -64,6 +75,11 @@ static std::unique_ptr<OpKernel> GetFakeKernel(const char* device_name,
   def.set_device(device_name);
   def.add_input("input1");
   def.add_input("input2");
+
+  AttrValue v;
+  v.set_type(DataType::DT_FLOAT);
+  (*def.mutable_attr())["SomeDataTypeAttr"] = v;
+
   return CreateOpKernel(DeviceType(device_name), nullptr, nullptr, def, 1,
                         status);
 }
@@ -78,7 +94,8 @@ TEST(TestKernel, TestRegisterKernelBuilder) {
   REGISTER_OP(op_name)
       .Input("input1: double")
       .Input("input2: uint8")
-      .Output("output1: uint8");
+      .Output("output1: uint8")
+      .Attr("SomeDataTypeAttr: type");
 
   TF_KernelBuilder* builder = TF_NewKernelBuilder(
       op_name, device_name, &MyCreateFunc, &MyComputeFunc, &MyDeleteFunc);
@@ -129,7 +146,8 @@ TEST(TestKernel, TestInputAndOutputCount) {
   REGISTER_OP(op_name)
       .Input("input1: double")
       .Input("input2: uint8")
-      .Output("output1: uint8");
+      .Output("output1: uint8")
+      .Attr("SomeDataTypeAttr: type");
 
   static int num_inputs = 0;
   static int num_outputs = 0;
-- 
GitLab


From e3aa49701d60a006d09786f5a240530ee5f47e25 Mon Sep 17 00:00:00 2001
From: Yifei Feng <yifeif@google.com>
Date: Mon, 14 Jan 2019 22:18:17 -0800
Subject: [PATCH 0706/2345] Remove cuda dependencies from dummy targets.

PiperOrigin-RevId: 229314246
---
 tensorflow/core/platform/default/build_config/BUILD | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/platform/default/build_config/BUILD b/tensorflow/core/platform/default/build_config/BUILD
index c0c1022c16..664218d11b 100644
--- a/tensorflow/core/platform/default/build_config/BUILD
+++ b/tensorflow/core/platform/default/build_config/BUILD
@@ -85,17 +85,17 @@ cc_library(
 # Dummy stream executor cuda plugins.
 cc_library(
     name = "cublas_plugin",
-    deps = if_static(["//tensorflow/stream_executor/cuda:cublas_plugin"]),
+    srcs = [],
 )
 
 cc_library(
     name = "cufft_plugin",
-    deps = if_static(["//tensorflow/stream_executor/cuda:cufft_plugin"]),
+    srcs = [],
 )
 
 cc_library(
     name = "cudnn_plugin",
-    deps = if_static(["//tensorflow/stream_executor/cuda:cudnn_plugin"]),
+    srcs = [],
 )
 
 # OSX framework for device driver access
-- 
GitLab


From 7225c69b4cc3affa65dd0cb61b12cb6ab644f4ca Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Tue, 15 Jan 2019 13:31:35 +0530
Subject: [PATCH 0707/2345] Update ragged_array_ops.py

typo
---
 tensorflow/python/ops/ragged/ragged_array_ops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/ragged/ragged_array_ops.py b/tensorflow/python/ops/ragged/ragged_array_ops.py
index a6b2442f09..c088afeca9 100644
--- a/tensorflow/python/ops/ragged/ragged_array_ops.py
+++ b/tensorflow/python/ops/ragged/ragged_array_ops.py
@@ -448,7 +448,7 @@ def boolean_mask(data, mask, keepdims=False, name=None):
 
     # Get static rank of mask.
     if mask.shape.ndims is None:
-      raise ValueError('mask.shape.ndims must be kown statically.')
+      raise ValueError('mask.shape.ndims must be known statically.')
     elif mask.shape.ndims == 0:
       raise ValueError('mask cannot be scalar.')
 
-- 
GitLab


From e8e58aef6f8d8a3278d92718b5202e2cacb9a1c4 Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Tue, 15 Jan 2019 13:33:19 +0530
Subject: [PATCH 0708/2345] Update ragged_boolean_mask_op_test.py

typo
---
 tensorflow/python/ops/ragged/ragged_boolean_mask_op_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/ragged/ragged_boolean_mask_op_test.py b/tensorflow/python/ops/ragged/ragged_boolean_mask_op_test.py
index 19f7d216d2..6f5fad13fb 100644
--- a/tensorflow/python/ops/ragged/ragged_boolean_mask_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_boolean_mask_op_test.py
@@ -316,7 +316,7 @@ class RaggedBooleanMaskOpTest(ragged_test_util.RaggedTensorTestCase,
   def testErrors(self):
     if not context.executing_eagerly():
       self.assertRaisesRegexp(ValueError,
-                              r'mask\.shape\.ndims must be kown statically',
+                              r'mask\.shape\.ndims must be known statically',
                               ragged_array_ops.boolean_mask, [[1, 2]],
                               array_ops.placeholder(dtypes.bool))
 
-- 
GitLab


From 52d0e458875714cd11d5cd37ad1b00981546f492 Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Tue, 15 Jan 2019 13:46:28 +0530
Subject: [PATCH 0709/2345] Update tf_ops_compatibility.md

---
 tensorflow/lite/g3doc/tf_ops_compatibility.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/g3doc/tf_ops_compatibility.md b/tensorflow/lite/g3doc/tf_ops_compatibility.md
index dcfda72137..8b85c08a32 100644
--- a/tensorflow/lite/g3doc/tf_ops_compatibility.md
+++ b/tensorflow/lite/g3doc/tf_ops_compatibility.md
@@ -185,7 +185,7 @@ Inputs {
   1: a tensor
 }
 Outputs {
-  0: A tensor of indices of minium values.
+  0: A tensor of indices of minimum values.
 }
 ```
 
-- 
GitLab


From 82e9fd50c1fd44a96b322e6317b645db338e9f25 Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Tue, 15 Jan 2019 00:14:55 -0800
Subject: [PATCH 0710/2345] Add documentation to help with TensorFlow Lite
 Micro porting

PiperOrigin-RevId: 229322903
---
 tensorflow/lite/experimental/micro/README.md | 128 ++++++++++++++++++-
 1 file changed, 124 insertions(+), 4 deletions(-)

diff --git a/tensorflow/lite/experimental/micro/README.md b/tensorflow/lite/experimental/micro/README.md
index 931c8d9d31..4c3eab38fa 100644
--- a/tensorflow/lite/experimental/micro/README.md
+++ b/tensorflow/lite/experimental/micro/README.md
@@ -296,10 +296,6 @@ Building micro_speech_test binary
 
 Lauching Renode to test the binary, currently this set up is not automated.
 
--   Until https://github.com/renode/renode/pull/30 is in the Docker image, patch
-    the change manully: `sed -E -i 's/"rv32g"/"rv32imac"/g'
-    /opt/renode/platforms/cpus/sifive-fe310.repl`
-
 -   Execute the binary on Renode: `renode -P 5000 --disable-xwt -e 's
     @/workspace/tensorflow/lite/experimental/micro/testing/sifive_fe310.resc'`
 
@@ -389,3 +385,127 @@ This will create a folder in
 that contains the source and header files, some Mbed configuration files, and a
 README. You should then be able to copy this directory to another machine, and
 use it just like any other Mbed project.
+
+## How to Port TensorFlow Lite Micro to a New Platform
+
+Are you a hardware or operating system provider looking to run machine learning
+on your platform? We're keen to help, and we've had experience helping other
+teams do the same thing, so here are our recommendations.
+
+### Requirements
+
+Since the core neural network operations are pure arithmetic, and don't require
+any I/O or other system-specific functionality, the code doesn't have to have
+many dependencies. We've tried to enforce this, so that it's as easy as possible
+to get TensorFlow Lite Micro running even on 'bare metal' systems without an OS.
+Here are the core requirements that a platform needs to run the framework:
+
+-   C/C++ compiler capable of C++11 compatibility. This is probably the most
+    restrictive of the requirements, since C++11 is not as widely adopted in the
+    embedded world as it is elsewhere. We made the decision to require it since
+    one of the main goals of TFL Micro is to share as much code as possible with
+    the wider TensorFlow codebase, and since that relies on C++11 features, we
+    need compatibility to achieve it. We only use a small, sane, subset of C++
+    though, so don't worry about having to deal with template metaprogramming or
+    similar challenges!
+
+-   Debug logging. The core network operations don't need any I/O functions, but
+    to be able to run tests and tell if they've worked as expected, the
+    framework needs some way to write out a string to some kind of debug
+    console. This will vary from system to system, for example on Linux it could
+    just be `fprintf(stderr, debug_string)` whereas an embedded device might
+    write the string out to a specified UART. As long as there's some mechanism
+    for outputting debug strings, you should be able to use TFL Micro on that
+    platform.
+
+-   Math library. The C standard `libm.a` library is needed to handle some of
+    the mathematical operations used to calculate neural network results.
+
+-   Global variable initialization. We do use a pattern of relying on global
+    variables being set before `main()` is run in some places, so you'll need to
+    make sure your compiler toolchain
+
+And that's it! You may be wondering about some other common requirements that
+are needed by a lot of non-embedded software, so here's a brief list of things
+that aren't necessary to get started with TFL Micro on a new platform:
+
+-   Operating system. Since the only platform-specific function we need is
+    `DebugLog()`, there's no requirement for any kind of Posix or similar
+    functionality around files, processes, or threads.
+
+-   C or C++ standard libraries. The framework tries to avoid relying on any
+    standard library functions that require linker-time support. This includes
+    things like string functions, but still allows us to use headers like
+    `stdtypes.h` which typically just define constants and typedefs.
+    Unfortunately this distinction isn't officially defined by any standard, so
+    it's possible that different toolchains may decide to require linked code
+    even for the subset we use, but in practice we've found it's usually a
+    pretty obvious decision and stable over platforms and toolchains.
+
+-   Dynamic memory allocation. All the TFL Micro code avoids dynamic memory
+    allocation, instead relying on local variables on the stack in most cases,
+    or global variables for a few situations. These are all fixed-size, which
+    can mean some compile-time configuration to ensure there's enough space for
+    particular networks, but does avoid any need for a heap and the
+    implementation of `malloc\new` on a platform.
+
+-   Floating point. Eight-bit integer arithmetic is enough for inference on many
+    networks, so if a model sticks to these kind of quantized operations, no
+    floating point instructions should be required or executed by the framework.
+
+### Getting Started
+
+We recommend that you start trying to compile and run one of the simplest tests
+in the framework as your first step. The full TensorFlow codebase can seem
+overwhelming to work with at first, so instead you can begin with a collection
+of self-contained project folders that only include the source files needed for
+a particular test or executable. You can find a set of pre-generated projects
+[here](https://drive.google.com/open?id=1cawEQAkqquK_SO4crReDYqf_v7yAwOY8).
+
+As mentioned above, the one function you will need to implement for a completely
+new platform is debug logging. If your device is just a variation on an existing
+platform you may be able to reuse code that's already been written. To
+understand what's available, begin with the default reference implementation at
+[tensorflow/lite/experimental/micro/debug_log.cc](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/experimental/micro/debug_log.cc]),
+which uses fprintf and stderr. If your platform has this level of support for
+the C standard library in its toolchain, then you can just reuse this.
+Otherwise, you'll need to do some research into how your platform and device can
+communicate logging statements to the outside world. As another example, take a
+look at
+[the Mbed version of `DebugLog()`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/experimental/micro/mbed/debug_log.cc),
+which creates a UART object and uses it to output strings to the host's console
+if it's connected.
+
+Begin by navigating to the micro_error_reporter_test folder in the pregenerated
+projects you downloaded. Inside here, you'll see a set of folders containing all
+the source code you need. If you look through them, you should find a total of
+around 60 C or C++ files that compiled together will create the test executable.
+There's an example makefile in the directory that lists all of the source files
+and include paths for the headers. If you're building on a Linux or MacOS host
+system, you may just be able to reuse that same makefile to cross-compile for
+your system, as long as you swap out the `CC` and `CXX` variables from their
+defaults, to point to your cross compiler instead (for example
+`arm-none-eabi-gcc` or `riscv64-unknown-elf-gcc`). Otherwise, set up a project
+in the build system you are using. It should hopefully be fairly
+straightforward, since all of the source files in the folder need to be
+compiled, so on many IDEs you can just drag the whole lot in. Then you need to
+make sure that C++11 compatibility is turned on, and that the right include
+paths (as mentioned in the makefile) have been added.
+
+You'll see the default `DebugLog()` implementation in
+'tensorflow/lite/experimental/micro/debug_log.cc' inside the
+micro_error_reporter_test folder. Modify that file to add the right
+implementation for your platform, and then you should be able to build the set
+of files into an executable. Transfer that executable to your target device (for
+example by flashing it), and then try running it. You should see output that
+looks something like this:
+
+```
+Number: 42
+Badly-formed format string
+Another  badly-formed  format string
+~~ALL TESTS PASSED~~~
+```
+
+If not, you'll need to debug what went wrong, but hopefully with this small
+starting project it should be manageable.
-- 
GitLab


From 5596b6196d07d59540e62ecfca5a5b70291d9dee Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Tue, 15 Jan 2019 13:56:33 +0530
Subject: [PATCH 0711/2345] Update README.md

---
 tensorflow/contrib/quantize/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/quantize/README.md b/tensorflow/contrib/quantize/README.md
index 9085d9fa71..5b8da92491 100644
--- a/tensorflow/contrib/quantize/README.md
+++ b/tensorflow/contrib/quantize/README.md
@@ -110,7 +110,7 @@ See the documentation for `tf.contrib.quantize` and [TensorFlow Lite](../lite/).
 
 ## Quantized accuracy results
 
-The following are results of trainiing some popular CNN models (Mobilenet-v1,
+The following are results of training some popular CNN models (Mobilenet-v1,
 Mobilenet-v2, and Inception-v3) using this tool:
 
 <figure>
-- 
GitLab


From d292c630911d6ba75fd0219792ef1cb47f531579 Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Tue, 15 Jan 2019 13:59:24 +0530
Subject: [PATCH 0712/2345] Update options.md

typo
---
 tensorflow/core/profiler/g3doc/options.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/profiler/g3doc/options.md b/tensorflow/core/profiler/g3doc/options.md
index 7f2cd3f698..19f33253fb 100644
--- a/tensorflow/core/profiler/g3doc/options.md
+++ b/tensorflow/core/profiler/g3doc/options.md
@@ -57,7 +57,7 @@ cpu_micros: This is the cpu times.
 Tensor memory are usually ref-counted. The memory is released when there is
 no more reference to it. It will be difficult to track the release of memory.
 Currently, profiler only tracks the allocation of memory. As a result, the
-accumulated memory request is uaually larger than the peak memory of the overall
+accumulated memory request is usually larger than the peak memory of the overall
 model.
 
 It's recommended to generate timeline to see the allocator memory usage over
-- 
GitLab


From dc07d3662531af7b3f049c6f3d3b010d4252df50 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 15 Jan 2019 01:02:19 -0800
Subject: [PATCH 0713/2345] compat: Update forward compatibility horizon to
 2019-01-15

PiperOrigin-RevId: 229327462
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index c9f2397fa5..83f31bd533 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 14)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 15)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From b329214d7bbf2e451e8631fee86ae40694869dd2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 15 Jan 2019 07:00:53 -0800
Subject: [PATCH 0714/2345] Add a ragged size op and register it to the op
 dispatcher

PiperOrigin-RevId: 229364271
---
 tensorflow/python/ops/array_ops.py            |  2 +
 tensorflow/python/ops/ragged/BUILD            | 15 ++++++
 .../python/ops/ragged/ragged_array_ops.py     | 30 ++++++++++++
 .../python/ops/ragged/ragged_dispatch.py      |  7 +++
 .../python/ops/ragged/ragged_dispatch_test.py |  8 ++++
 .../python/ops/ragged/ragged_size_op_test.py  | 48 +++++++++++++++++++
 6 files changed, 110 insertions(+)
 create mode 100644 tensorflow/python/ops/ragged/ragged_size_op_test.py

diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 81558db04f..014fdd25cc 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -357,12 +357,14 @@ def shape_n(input, out_type=dtypes.int32, name=None):
 
 
 @tf_export("size", v1=[])
+@dispatch.add_dispatch_support
 def size_v2(input, out_type=dtypes.int32, name=None):
   # pylint: disable=redefined-builtin
   return size(input, name, out_type)
 
 
 @tf_export(v1=["size"])
+@dispatch.add_dispatch_support
 def size(input, name=None, out_type=dtypes.int32):
   # pylint: disable=redefined-builtin
   """Returns the size of a tensor.
diff --git a/tensorflow/python/ops/ragged/BUILD b/tensorflow/python/ops/ragged/BUILD
index 46f7fa62a3..972b14955f 100644
--- a/tensorflow/python/ops/ragged/BUILD
+++ b/tensorflow/python/ops/ragged/BUILD
@@ -875,3 +875,18 @@ py_test(
         "@absl_py//absl/testing:parameterized",
     ],
 )
+
+py_test(
+    name = "ragged_size_op_test",
+    srcs = ["ragged_size_op_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":ragged_array_ops",
+        ":ragged_factory_ops",
+        ":ragged_test_util",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:platform_test",
+        "@absl_py//absl/testing:parameterized",
+    ],
+)
diff --git a/tensorflow/python/ops/ragged/ragged_array_ops.py b/tensorflow/python/ops/ragged/ragged_array_ops.py
index a6b2442f09..399ddbe597 100644
--- a/tensorflow/python/ops/ragged/ragged_array_ops.py
+++ b/tensorflow/python/ops/ragged/ragged_array_ops.py
@@ -1194,6 +1194,36 @@ def _coordinate_where(condition):
                           axis=1)
 
 
+#===============================================================================
+# RaggedTensor Size
+#===============================================================================
+
+
+def size(input, out_type=dtypes.int32, name=None):  # pylint: disable=redefined-builtin
+  """Returns the size of a potentially ragged tensor.
+
+  The size of a ragged tensor is the size of its inner values.
+
+  Args:
+    input: A potentially ragged `Tensor`.
+    out_type: The numeric output type for the operation.
+    name: A name for the operation (optional).
+
+  Returns:
+    A Tensor of type `out_type`.
+
+  #### Example:
+    ```python
+    >>> tf.size(tf.ragged.constant([[1, 2], [3]]))
+    3
+    ```
+  """
+  if ragged_tensor.is_ragged(input):
+    return array_ops.size(input.flat_values, out_type=out_type, name=name)
+  else:
+    return array_ops.size(input, out_type=out_type, name=name)
+
+
 #===============================================================================
 # Internal Helper Functions
 #===============================================================================
diff --git a/tensorflow/python/ops/ragged/ragged_dispatch.py b/tensorflow/python/ops/ragged/ragged_dispatch.py
index 970d88663a..5284750198 100644
--- a/tensorflow/python/ops/ragged/ragged_dispatch.py
+++ b/tensorflow/python/ops/ragged/ragged_dispatch.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import collections
 import numpy as np
 
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.ops import array_ops
@@ -415,6 +416,10 @@ def _ragged_expand_dims_v1(input, axis=None, name=None, dim=None):  # pylint: di
   return ragged_array_ops.expand_dims(input=input, axis=axis, name=name)
 
 
+def _ragged_size_v1(input, name=None, out_type=dtypes.int32):  # pylint: disable=redefined-builtin
+  return ragged_array_ops.size(input=input, out_type=out_type, name=name)
+
+
 # (original_op, ragged_op, ragged_args)
 _RAGGED_DISPATCH_OPS = [
     (array_ops.batch_gather, ragged_array_ops.batch_gather,
@@ -426,6 +431,8 @@ _RAGGED_DISPATCH_OPS = [
     (array_ops.gather_v2, ragged_array_ops.gather, ['params', 'indices']),
     (array_ops.gather_nd, ragged_array_ops.gather_nd, ['params', 'indices']),
     (array_ops.rank, ragged_array_ops.rank, ['input']),
+    (array_ops.size, _ragged_size_v1, ['input']),
+    (array_ops.size_v2, ragged_array_ops.size, ['input']),
     (array_ops.stack, ragged_array_ops.stack, ['[values]']),
     (array_ops.tile, ragged_array_ops.tile, ['input']),
     (array_ops.where, ragged_array_ops.where, ['condition', 'x', 'y']),
diff --git a/tensorflow/python/ops/ragged/ragged_dispatch_test.py b/tensorflow/python/ops/ragged/ragged_dispatch_test.py
index 1aa612633e..04ef0d7cd6 100644
--- a/tensorflow/python/ops/ragged/ragged_dispatch_test.py
+++ b/tensorflow/python/ops/ragged/ragged_dispatch_test.py
@@ -687,6 +687,14 @@ class RaggedElementwiseOpsTest(ragged_test_util.RaggedTensorTestCase,
           op=array_ops.rank,
           kwargs={'input': ragged_factory_ops.constant_value([[8, 3], [5]])},
           expected=2),
+      dict(
+          op=array_ops.size,
+          kwargs={'input': ragged_factory_ops.constant_value([[8, 3], [5]])},
+          expected=3),
+      dict(
+          op=array_ops.size_v2,
+          kwargs={'input': ragged_factory_ops.constant_value([[8, 3], [5]])},
+          expected=3),
   ])
   def testRaggedDispatch(self, op, expected, args=(), kwargs=None):
     if kwargs is None: kwargs = {}
diff --git a/tensorflow/python/ops/ragged/ragged_size_op_test.py b/tensorflow/python/ops/ragged/ragged_size_op_test.py
new file mode 100644
index 0000000000..6ffed11b13
--- /dev/null
+++ b/tensorflow/python/ops/ragged/ragged_size_op_test.py
@@ -0,0 +1,48 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for ragged.size."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops.ragged import ragged_array_ops
+from tensorflow.python.ops.ragged import ragged_factory_ops
+from tensorflow.python.ops.ragged import ragged_test_util
+from tensorflow.python.platform import googletest
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedSizeOpTest(ragged_test_util.RaggedTensorTestCase,
+                       parameterized.TestCase):
+
+  @parameterized.parameters([
+      {'size': 1, 'test_input': 1},
+      {'size': 0, 'test_input': []},
+      {'size': 0, 'test_input': [], 'ragged_rank': 1},
+      {'size': 3, 'test_input': [1, 1, 1]},
+      {'size': 3, 'test_input': [[1, 1], [1]]},
+      {'size': 5, 'test_input': [[[1, 1, 1], [1]], [[1]]]},
+      {'size': 6, 'test_input': [[[1, 1], [1, 1]], [[1, 1]]], 'ragged_rank': 1},
+  ])
+  def testRaggedSize(self, test_input, size, ragged_rank=None):
+    input_rt = ragged_factory_ops.constant(test_input, ragged_rank=ragged_rank)
+    self.assertAllEqual(ragged_array_ops.size(input_rt), size)
+
+if __name__ == '__main__':
+  googletest.main()
-- 
GitLab


From 9825f1d7534cb6d0516813346750fe9a59df42ce Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 15 Jan 2019 08:43:07 -0800
Subject: [PATCH 0715/2345] Split NeuralNetworksShim.h: - Type definitions go
 into NeuralNetworksTypes.h - Function definitions stay into
 NeuralNetworkShim.h

PiperOrigin-RevId: 229377846
---
 tensorflow/lite/nnapi/BUILD                 |   1 +
 tensorflow/lite/nnapi/NeuralNetworksShim.h  | 330 +-----------------
 tensorflow/lite/nnapi/NeuralNetworksTypes.h | 352 ++++++++++++++++++++
 3 files changed, 355 insertions(+), 328 deletions(-)
 create mode 100644 tensorflow/lite/nnapi/NeuralNetworksTypes.h

diff --git a/tensorflow/lite/nnapi/BUILD b/tensorflow/lite/nnapi/BUILD
index 467a2b7a7b..bd3a8a69af 100644
--- a/tensorflow/lite/nnapi/BUILD
+++ b/tensorflow/lite/nnapi/BUILD
@@ -8,6 +8,7 @@ cc_library(
     name = "nnapi_lib",
     hdrs = [
         "NeuralNetworksShim.h",
+        "NeuralNetworksTypes.h",
     ],
     linkopts = ["-ldl"],
 )
diff --git a/tensorflow/lite/nnapi/NeuralNetworksShim.h b/tensorflow/lite/nnapi/NeuralNetworksShim.h
index c39502f4ac..2ce6e50de6 100644
--- a/tensorflow/lite/nnapi/NeuralNetworksShim.h
+++ b/tensorflow/lite/nnapi/NeuralNetworksShim.h
@@ -20,6 +20,8 @@ limitations under the License.
 #include <stdio.h>
 #include <stdlib.h>
 
+#include "tensorflow/lite/nnapi/NeuralNetworksTypes.h"
+
 // helpers
 
 #define NNAPI_LOG(format, ...) fprintf(stderr, format "\n", __VA_ARGS__);
@@ -44,8 +46,6 @@ inline void* loadLibrary(const char* name) {
   return handle;
 }
 
-typedef int (*ASharedMemory_create_fn)(const char* name, size_t size);
-
 // ASharedMemory_create was added in Android 8.0, so safe to use with NNAPI
 // which was added in 8.1.
 inline int ASharedMemory_create(const char* name, size_t size) {
@@ -81,332 +81,6 @@ inline bool NNAPIExists() {
 // NN api types based on NNAPI header file
 // https://developer.android.com/ndk/reference/group/neural-networks
 
-/**
- * Operand types.
- *
- * The type of operands that can be added to a model.
- *
- * Although we define many types, most operators accept just a few
- * types.  Most used are ANEURALNETWORKS_TENSOR_FLOAT32,
- * ANEURALNETWORKS_TENSOR_QUANT8_ASYMM, and ANEURALNETWORKS_INT32.
- */
-enum {
-  ANEURALNETWORKS_FLOAT32 = 0,
-  ANEURALNETWORKS_INT32 = 1,
-  ANEURALNETWORKS_UINT32 = 2,
-  ANEURALNETWORKS_TENSOR_FLOAT32 = 3,
-  ANEURALNETWORKS_TENSOR_INT32 = 4,
-  ANEURALNETWORKS_TENSOR_QUANT8_ASYMM = 5,
-};
-
-/**
- * Operation types.
- *
- * The type of operations that can be added to a model.
- */
-enum {
-  ANEURALNETWORKS_ADD = 0,
-  ANEURALNETWORKS_AVERAGE_POOL_2D = 1,
-  ANEURALNETWORKS_CONCATENATION = 2,
-  ANEURALNETWORKS_CONV_2D = 3,
-  ANEURALNETWORKS_DEPTHWISE_CONV_2D = 4,
-  ANEURALNETWORKS_DEPTH_TO_SPACE = 5,
-  ANEURALNETWORKS_DEQUANTIZE = 6,
-  ANEURALNETWORKS_EMBEDDING_LOOKUP = 7,
-  ANEURALNETWORKS_FLOOR = 8,
-  ANEURALNETWORKS_FULLY_CONNECTED = 9,
-  ANEURALNETWORKS_HASHTABLE_LOOKUP = 10,
-  ANEURALNETWORKS_L2_NORMALIZATION = 11,
-  ANEURALNETWORKS_L2_POOL_2D = 12,
-  ANEURALNETWORKS_LOCAL_RESPONSE_NORMALIZATION = 13,
-  ANEURALNETWORKS_LOGISTIC = 14,
-  ANEURALNETWORKS_LSH_PROJECTION = 15,
-  ANEURALNETWORKS_LSTM = 16,
-  ANEURALNETWORKS_MAX_POOL_2D = 17,
-  ANEURALNETWORKS_MUL = 18,
-  ANEURALNETWORKS_RELU = 19,
-  ANEURALNETWORKS_RELU1 = 20,
-  ANEURALNETWORKS_RELU6 = 21,
-  ANEURALNETWORKS_RESHAPE = 22,
-  ANEURALNETWORKS_RESIZE_BILINEAR = 23,
-  ANEURALNETWORKS_RNN = 24,
-  ANEURALNETWORKS_SOFTMAX = 25,
-  ANEURALNETWORKS_SPACE_TO_DEPTH = 26,
-  ANEURALNETWORKS_SVDF = 27,
-  ANEURALNETWORKS_TANH = 28,
-  ANEURALNETWORKS_BATCH_TO_SPACE_ND = 29,
-  ANEURALNETWORKS_DIV = 30,
-  ANEURALNETWORKS_MEAN = 31,
-  ANEURALNETWORKS_PAD = 32,
-  ANEURALNETWORKS_SPACE_TO_BATCH_ND = 33,
-  ANEURALNETWORKS_SQUEEZE = 34,
-  ANEURALNETWORKS_STRIDED_SLICE = 35,
-  ANEURALNETWORKS_SUB = 36,
-  ANEURALNETWORKS_TRANSPOSE = 37,
-};
-
-/**
- * Fused activation function types.
- *
- */
-enum {
-  ANEURALNETWORKS_FUSED_NONE = 0,
-  ANEURALNETWORKS_FUSED_RELU = 1,
-  ANEURALNETWORKS_FUSED_RELU1 = 2,
-  ANEURALNETWORKS_FUSED_RELU6 = 3,
-};
-
-/**
- * Execution preferences.
- */
-enum {
-  ANEURALNETWORKS_PREFER_LOW_POWER = 0,
-  ANEURALNETWORKS_PREFER_FAST_SINGLE_ANSWER = 1,
-  ANEURALNETWORKS_PREFER_SUSTAINED_SPEED = 2,
-};
-
-/**
- * Result codes.
- */
-enum {
-  ANEURALNETWORKS_NO_ERROR = 0,
-  ANEURALNETWORKS_OUT_OF_MEMORY = 1,
-  ANEURALNETWORKS_INCOMPLETE = 2,
-  ANEURALNETWORKS_UNEXPECTED_NULL = 3,
-  ANEURALNETWORKS_BAD_DATA = 4,
-  ANEURALNETWORKS_OP_FAILED = 5,
-  ANEURALNETWORKS_UNMAPPABLE = 5,
-  ANEURALNETWORKS_BAD_STATE = 6,
-};
-
-/**
- * Implicit padding algorithms.
- */
-enum {
-  ANEURALNETWORKS_PADDING_SAME = 1,
-  ANEURALNETWORKS_PADDING_VALID = 2,
-};
-
-/**
- * ANeuralNetworksMemory is an opaque type that represents memory.
- *
- * This type is used to represent shared memory, memory mapped files,
- * and similar memories.
- *
- * By using shared memory, a program can efficiently communicate to the
- * runtime and drivers the tensors that define a model. See
- * {@link ANeuralNetworksModel_setOperandValueFromMemory}. An application
- * should typically create one shared memory object that contains every tensor
- * needed to define a model. {@link ANeuralNetworksMemory_createFromFd} can be
- * used to create shared memory from a file handle. {@link
- * ANeuralNetworksMemory_createShared} can be used to directly created shared
- * memory.
- *
- * Memory objects can also be used to specify the input and output arguments of
- * an execution. See {@link ANeuralNetworksExecution_setInputFromMemory}
- * and {@link ANeuralNetworksExecution_setOutputFromMemory}.
- */
-typedef struct ANeuralNetworksMemory ANeuralNetworksMemory;
-
-/**
- * ANeuralNetworksModel is an opaque type that contains a description of the
- * mathematical operations that constitute the model.
- *
- * <p>The model will be built by calling<ul>
- * <li>{@link ANeuralNetworksModel_create},</li>
- * <li>{@link ANeuralNetworksModel_addOperation},</li>
- * <li>{@link ANeuralNetworksModel_addOperand},</li>
- * </ul>
- *
- * A model is completed by calling {@link ANeuralNetworksModel_finish}.
- * A model is destroyed by calling {@link ANeuralNetworksModel_free}.
- *
- * <p>It is the application's responsibility to make sure that only one thread
- * modifies a model at a given time. It is however safe for more than one
- * thread to use the model once {@link ANeuralNetworksModel_finish} has
- * returned.</p>
- *
- * <p>It is also the application's responsibility to ensure that there are no
- * other uses of the model after calling {@link ANeuralNetworksModel_free}. This
- * includes any compilation or execution object created using the model.</p>
- */
-typedef struct ANeuralNetworksModel ANeuralNetworksModel;
-
-/**
- * ANeuralNetworksCompilation is an opaque type that can be used to compile
- * a machine learning model.
- *
- * <p>To use:<ul>
- *    <li>Create a new compilation instance by calling the
- *        {@link ANeuralNetworksCompilation_create} function.</li>
- *    <li>Perform the compilation with {@link
- * ANeuralNetworksCompilation_start}.</li> <li>Wait for the compilation to
- * complete with {@link ANeuralNetworksCompilation_wait}.</li> <li>Use the
- * compilation as many times as needed with {@link
- * ANeuralNetworksExecution_create}.</li> <li>Destroy the compilation with
- * {@link ANeuralNetworksCompilation_free} once all executions using the
- * compilation have completed.</li></ul></p>
- *
- * <p>A compilation cannot be modified once {@link
- * ANeuralNetworksCompilation_start} has been called on it.</p>
- *
- * <p>It is the application's responsibility to make sure that only one thread
- * modifies a compilation at a given time. It is however safe for more than one
- * thread to use {@link ANeuralNetworksCompilation_wait} at the same time.
- * It is also safe for multiple threads to use a compilation object once
- * {@link ANeuralNetworksCompilation_wait} has completed.</p>
- *
- * <p>It is also the application's responsibility to ensure that there are no
- * other uses of the compilation after calling {@link
- * ANeuralNetworksCompilation_free}. This includes any execution object created
- * using the compilation.</p>
- */
-typedef struct ANeuralNetworksCompilation ANeuralNetworksCompilation;
-
-/**
- * ANeuralNetworksExecution is an opaque type that can be used to apply a
- * machine learning model to a set of inputs.
- *
- * <p>To use:<ul>
- *    <li>Create a new execution instance by calling the
- *        {@link ANeuralNetworksExecution_create} function.</li>
- *    <li>Associate data to the model inputs with
- *        {@link ANeuralNetworksExecution_setInput} or
- *        {@link ANeuralNetworksExecution_setInputFromMemory}.</li>
- *    <li>Associate output buffers to the model outputs with
- *        {@link ANeuralNetworksExecution_setOutput} or
- *        {@link ANeuralNetworksExecution_setOutputFromMemory}.</li>
- *    <li>Apply the model with {@link
- * ANeuralNetworksExecution_startCompute}.</li> <li>Wait for the execution to
- * complete with {@link ANeuralNetworksExecution_wait}.</li> <li>Destroy the
- * execution with
- *        {@link ANeuralNetworksExecution_free}.</li></ul></p>
- *
- * <p>An execution cannot be modified once {@link
- * ANeuralNetworksExecution_start} has been called on it.</p>
- *
- * <p>An execution can be applied to a model with
- * {@link ANeuralNetworksExecution_startCompute} only once. Create new
- * executions to do new evaluations of the model.</p>
- *
- * <p>It is the application's responsibility to make sure that only one thread
- * modifies an execution at a given time. It is however safe for more than one
- * thread to use {@link ANeuralNetworksExecution_wait} at the same time.</p>
- *
- * <p>It is also the application's responsibility to ensure that there are no
- * other uses of the request after calling {@link
- * ANeuralNetworksRequest_free}.</p>
- */
-typedef struct ANeuralNetworksExecution ANeuralNetworksExecution;
-
-/**
- * ANeuralNetworksOperandType describes the type of an operand.
- * This structure is used to describe both scalars and tensors.
- */
-typedef struct ANeuralNetworksOperandType {
-  /** The data type, e.g ANEURALNETWORKS_INT8. */
-  int32_t type;
-  /** The number of dimensions. It should be 0 for scalars. */
-  uint32_t dimensionCount;
-  /** The dimensions of the tensor. It should be nullptr for scalars. */
-  const uint32_t* dimensions;
-  /** These two fields are only used for quantized tensors.
-   * They should be zero for scalars and non-fixed point tensors.
-   * The dequantized value of each entry is (value - offset) * scale.
-   */
-  float scale;
-  int32_t zeroPoint;
-} ANeuralNetworksOperandType;
-
-/**
- * ANeuralNetworksEvent is an opaque type that represents an event
- * that will be signaled once an execution completes.
- */
-typedef struct ANeuralNetworksEvent ANeuralNetworksEvent;
-
-typedef int32_t ANeuralNetworksOperationType;
-
-// nn api function types
-
-typedef int (*ANeuralNetworksMemory_createFromFd_fn)(
-    size_t size, int protect, int fd, size_t offset,
-    ANeuralNetworksMemory** memory);
-
-typedef void (*ANeuralNetworksMemory_free_fn)(ANeuralNetworksMemory* memory);
-
-typedef int (*ANeuralNetworksModel_create_fn)(ANeuralNetworksModel** model);
-
-typedef int (*ANeuralNetworksModel_finish_fn)(ANeuralNetworksModel* model);
-
-typedef void (*ANeuralNetworksModel_free_fn)(ANeuralNetworksModel* model);
-
-typedef int (*ANeuralNetworksCompilation_create_fn)(
-    ANeuralNetworksModel* model, ANeuralNetworksCompilation** compilation);
-
-typedef void (*ANeuralNetworksCompilation_free_fn)(
-    ANeuralNetworksCompilation* compilation);
-
-typedef int (*ANeuralNetworksCompilation_setPreference_fn)(
-    ANeuralNetworksCompilation* compilation, int32_t preference);
-
-typedef int (*ANeuralNetworksCompilation_finish_fn)(
-    ANeuralNetworksCompilation* compilation);
-
-typedef int (*ANeuralNetworksModel_addOperand_fn)(
-    ANeuralNetworksModel* model, const ANeuralNetworksOperandType* type);
-
-typedef int (*ANeuralNetworksModel_setOperandValue_fn)(
-    ANeuralNetworksModel* model, int32_t index, const void* buffer,
-    size_t length);
-
-typedef int (*ANeuralNetworksModel_setOperandValueFromMemory_fn)(
-    ANeuralNetworksModel* model, int32_t index,
-    const ANeuralNetworksMemory* memory, size_t offset, size_t length);
-
-typedef int (*ANeuralNetworksModel_addOperation_fn)(
-    ANeuralNetworksModel* model, ANeuralNetworksOperationType type,
-    uint32_t inputCount, const uint32_t* inputs, uint32_t outputCount,
-    const uint32_t* outputs);
-
-typedef int (*ANeuralNetworksModel_identifyInputsAndOutputs_fn)(
-    ANeuralNetworksModel* model, uint32_t inputCount, const uint32_t* inputs,
-    uint32_t outputCount, const uint32_t* outputs);
-
-typedef int (*ANeuralNetworksModel_relaxComputationFloat32toFloat16_fn)(
-    ANeuralNetworksModel* model, bool allow);
-
-typedef int (*ANeuralNetworksExecution_create_fn)(
-    ANeuralNetworksCompilation* compilation,
-    ANeuralNetworksExecution** execution);
-
-typedef void (*ANeuralNetworksExecution_free_fn)(
-    ANeuralNetworksExecution* execution);
-
-typedef int (*ANeuralNetworksExecution_setInput_fn)(
-    ANeuralNetworksExecution* execution, int32_t index,
-    const ANeuralNetworksOperandType* type, const void* buffer, size_t length);
-
-typedef int (*ANeuralNetworksExecution_setInputFromMemory_fn)(
-    ANeuralNetworksExecution* execution, int32_t index,
-    const ANeuralNetworksOperandType* type, const ANeuralNetworksMemory* memory,
-    size_t offset, size_t length);
-
-typedef int (*ANeuralNetworksExecution_setOutput_fn)(
-    ANeuralNetworksExecution* execution, int32_t index,
-    const ANeuralNetworksOperandType* type, void* buffer, size_t length);
-
-typedef int (*ANeuralNetworksExecution_setOutputFromMemory_fn)(
-    ANeuralNetworksExecution* execution, int32_t index,
-    const ANeuralNetworksOperandType* type, const ANeuralNetworksMemory* memory,
-    size_t offset, size_t length);
-
-typedef int (*ANeuralNetworksExecution_startCompute_fn)(
-    ANeuralNetworksExecution* execution, ANeuralNetworksEvent** event);
-
-typedef int (*ANeuralNetworksEvent_wait_fn)(ANeuralNetworksEvent* event);
-
-typedef void (*ANeuralNetworksEvent_free_fn)(ANeuralNetworksEvent* event);
-
 /**
  * Creates a shared memory object from a file descriptor.
  *
diff --git a/tensorflow/lite/nnapi/NeuralNetworksTypes.h b/tensorflow/lite/nnapi/NeuralNetworksTypes.h
new file mode 100644
index 0000000000..de8b84a823
--- /dev/null
+++ b/tensorflow/lite/nnapi/NeuralNetworksTypes.h
@@ -0,0 +1,352 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_NNAPI_NEURALNETWORKSTYPES_H_
+#define TENSORFLOW_LITE_NNAPI_NEURALNETWORKSTYPES_H_
+
+#include <stdint.h>
+#include <stdio.h>
+
+// NN api types based on NNAPI header file
+// https://developer.android.com/ndk/reference/group/neural-networks
+
+/**
+ * Operand types.
+ *
+ * The type of operands that can be added to a model.
+ *
+ * Although we define many types, most operators accept just a few
+ * types.  Most used are ANEURALNETWORKS_TENSOR_FLOAT32,
+ * ANEURALNETWORKS_TENSOR_QUANT8_ASYMM, and ANEURALNETWORKS_INT32.
+ */
+enum {
+  ANEURALNETWORKS_FLOAT32 = 0,
+  ANEURALNETWORKS_INT32 = 1,
+  ANEURALNETWORKS_UINT32 = 2,
+  ANEURALNETWORKS_TENSOR_FLOAT32 = 3,
+  ANEURALNETWORKS_TENSOR_INT32 = 4,
+  ANEURALNETWORKS_TENSOR_QUANT8_ASYMM = 5,
+};
+
+/**
+ * Operation types.
+ *
+ * The type of operations that can be added to a model.
+ */
+enum {
+  ANEURALNETWORKS_ADD = 0,
+  ANEURALNETWORKS_AVERAGE_POOL_2D = 1,
+  ANEURALNETWORKS_CONCATENATION = 2,
+  ANEURALNETWORKS_CONV_2D = 3,
+  ANEURALNETWORKS_DEPTHWISE_CONV_2D = 4,
+  ANEURALNETWORKS_DEPTH_TO_SPACE = 5,
+  ANEURALNETWORKS_DEQUANTIZE = 6,
+  ANEURALNETWORKS_EMBEDDING_LOOKUP = 7,
+  ANEURALNETWORKS_FLOOR = 8,
+  ANEURALNETWORKS_FULLY_CONNECTED = 9,
+  ANEURALNETWORKS_HASHTABLE_LOOKUP = 10,
+  ANEURALNETWORKS_L2_NORMALIZATION = 11,
+  ANEURALNETWORKS_L2_POOL_2D = 12,
+  ANEURALNETWORKS_LOCAL_RESPONSE_NORMALIZATION = 13,
+  ANEURALNETWORKS_LOGISTIC = 14,
+  ANEURALNETWORKS_LSH_PROJECTION = 15,
+  ANEURALNETWORKS_LSTM = 16,
+  ANEURALNETWORKS_MAX_POOL_2D = 17,
+  ANEURALNETWORKS_MUL = 18,
+  ANEURALNETWORKS_RELU = 19,
+  ANEURALNETWORKS_RELU1 = 20,
+  ANEURALNETWORKS_RELU6 = 21,
+  ANEURALNETWORKS_RESHAPE = 22,
+  ANEURALNETWORKS_RESIZE_BILINEAR = 23,
+  ANEURALNETWORKS_RNN = 24,
+  ANEURALNETWORKS_SOFTMAX = 25,
+  ANEURALNETWORKS_SPACE_TO_DEPTH = 26,
+  ANEURALNETWORKS_SVDF = 27,
+  ANEURALNETWORKS_TANH = 28,
+  ANEURALNETWORKS_BATCH_TO_SPACE_ND = 29,
+  ANEURALNETWORKS_DIV = 30,
+  ANEURALNETWORKS_MEAN = 31,
+  ANEURALNETWORKS_PAD = 32,
+  ANEURALNETWORKS_SPACE_TO_BATCH_ND = 33,
+  ANEURALNETWORKS_SQUEEZE = 34,
+  ANEURALNETWORKS_STRIDED_SLICE = 35,
+  ANEURALNETWORKS_SUB = 36,
+  ANEURALNETWORKS_TRANSPOSE = 37,
+};
+
+/**
+ * Fused activation function types.
+ *
+ */
+enum {
+  ANEURALNETWORKS_FUSED_NONE = 0,
+  ANEURALNETWORKS_FUSED_RELU = 1,
+  ANEURALNETWORKS_FUSED_RELU1 = 2,
+  ANEURALNETWORKS_FUSED_RELU6 = 3,
+};
+
+/**
+ * Execution preferences.
+ */
+enum {
+  ANEURALNETWORKS_PREFER_LOW_POWER = 0,
+  ANEURALNETWORKS_PREFER_FAST_SINGLE_ANSWER = 1,
+  ANEURALNETWORKS_PREFER_SUSTAINED_SPEED = 2,
+};
+
+/**
+ * Result codes.
+ */
+enum {
+  ANEURALNETWORKS_NO_ERROR = 0,
+  ANEURALNETWORKS_OUT_OF_MEMORY = 1,
+  ANEURALNETWORKS_INCOMPLETE = 2,
+  ANEURALNETWORKS_UNEXPECTED_NULL = 3,
+  ANEURALNETWORKS_BAD_DATA = 4,
+  ANEURALNETWORKS_OP_FAILED = 5,
+  ANEURALNETWORKS_UNMAPPABLE = 5,
+  ANEURALNETWORKS_BAD_STATE = 6,
+};
+
+/**
+ * Implicit padding algorithms.
+ */
+enum {
+  ANEURALNETWORKS_PADDING_SAME = 1,
+  ANEURALNETWORKS_PADDING_VALID = 2,
+};
+
+/**
+ * ANeuralNetworksMemory is an opaque type that represents memory.
+ *
+ * This type is used to represent shared memory, memory mapped files,
+ * and similar memories.
+ *
+ * By using shared memory, a program can efficiently communicate to the
+ * runtime and drivers the tensors that define a model. See
+ * {@link ANeuralNetworksModel_setOperandValueFromMemory}. An application
+ * should typically create one shared memory object that contains every tensor
+ * needed to define a model. {@link ANeuralNetworksMemory_createFromFd} can be
+ * used to create shared memory from a file handle. {@link
+ * ANeuralNetworksMemory_createShared} can be used to directly created shared
+ * memory.
+ *
+ * Memory objects can also be used to specify the input and output arguments of
+ * an execution. See {@link ANeuralNetworksExecution_setInputFromMemory}
+ * and {@link ANeuralNetworksExecution_setOutputFromMemory}.
+ */
+typedef struct ANeuralNetworksMemory ANeuralNetworksMemory;
+
+/**
+ * ANeuralNetworksModel is an opaque type that contains a description of the
+ * mathematical operations that constitute the model.
+ *
+ * <p>The model will be built by calling<ul>
+ * <li>{@link ANeuralNetworksModel_create},</li>
+ * <li>{@link ANeuralNetworksModel_addOperation},</li>
+ * <li>{@link ANeuralNetworksModel_addOperand},</li>
+ * </ul>
+ *
+ * A model is completed by calling {@link ANeuralNetworksModel_finish}.
+ * A model is destroyed by calling {@link ANeuralNetworksModel_free}.
+ *
+ * <p>It is the application's responsibility to make sure that only one thread
+ * modifies a model at a given time. It is however safe for more than one
+ * thread to use the model once {@link ANeuralNetworksModel_finish} has
+ * returned.</p>
+ *
+ * <p>It is also the application's responsibility to ensure that there are no
+ * other uses of the model after calling {@link ANeuralNetworksModel_free}. This
+ * includes any compilation or execution object created using the model.</p>
+ */
+typedef struct ANeuralNetworksModel ANeuralNetworksModel;
+
+/**
+ * ANeuralNetworksCompilation is an opaque type that can be used to compile
+ * a machine learning model.
+ *
+ * <p>To use:<ul>
+ *    <li>Create a new compilation instance by calling the
+ *        {@link ANeuralNetworksCompilation_create} function.</li>
+ *    <li>Perform the compilation with {@link
+ * ANeuralNetworksCompilation_start}.</li> <li>Wait for the compilation to
+ * complete with {@link ANeuralNetworksCompilation_wait}.</li> <li>Use the
+ * compilation as many times as needed with {@link
+ * ANeuralNetworksExecution_create}.</li> <li>Destroy the compilation with
+ * {@link ANeuralNetworksCompilation_free} once all executions using the
+ * compilation have completed.</li></ul></p>
+ *
+ * <p>A compilation cannot be modified once {@link
+ * ANeuralNetworksCompilation_start} has been called on it.</p>
+ *
+ * <p>It is the application's responsibility to make sure that only one thread
+ * modifies a compilation at a given time. It is however safe for more than one
+ * thread to use {@link ANeuralNetworksCompilation_wait} at the same time.
+ * It is also safe for multiple threads to use a compilation object once
+ * {@link ANeuralNetworksCompilation_wait} has completed.</p>
+ *
+ * <p>It is also the application's responsibility to ensure that there are no
+ * other uses of the compilation after calling {@link
+ * ANeuralNetworksCompilation_free}. This includes any execution object created
+ * using the compilation.</p>
+ */
+typedef struct ANeuralNetworksCompilation ANeuralNetworksCompilation;
+
+/**
+ * ANeuralNetworksExecution is an opaque type that can be used to apply a
+ * machine learning model to a set of inputs.
+ *
+ * <p>To use:<ul>
+ *    <li>Create a new execution instance by calling the
+ *        {@link ANeuralNetworksExecution_create} function.</li>
+ *    <li>Associate data to the model inputs with
+ *        {@link ANeuralNetworksExecution_setInput} or
+ *        {@link ANeuralNetworksExecution_setInputFromMemory}.</li>
+ *    <li>Associate output buffers to the model outputs with
+ *        {@link ANeuralNetworksExecution_setOutput} or
+ *        {@link ANeuralNetworksExecution_setOutputFromMemory}.</li>
+ *    <li>Apply the model with {@link
+ * ANeuralNetworksExecution_startCompute}.</li> <li>Wait for the execution to
+ * complete with {@link ANeuralNetworksExecution_wait}.</li> <li>Destroy the
+ * execution with
+ *        {@link ANeuralNetworksExecution_free}.</li></ul></p>
+ *
+ * <p>An execution cannot be modified once {@link
+ * ANeuralNetworksExecution_start} has been called on it.</p>
+ *
+ * <p>An execution can be applied to a model with
+ * {@link ANeuralNetworksExecution_startCompute} only once. Create new
+ * executions to do new evaluations of the model.</p>
+ *
+ * <p>It is the application's responsibility to make sure that only one thread
+ * modifies an execution at a given time. It is however safe for more than one
+ * thread to use {@link ANeuralNetworksExecution_wait} at the same time.</p>
+ *
+ * <p>It is also the application's responsibility to ensure that there are no
+ * other uses of the request after calling {@link
+ * ANeuralNetworksRequest_free}.</p>
+ */
+typedef struct ANeuralNetworksExecution ANeuralNetworksExecution;
+
+/**
+ * ANeuralNetworksOperandType describes the type of an operand.
+ * This structure is used to describe both scalars and tensors.
+ */
+typedef struct ANeuralNetworksOperandType {
+  /** The data type, e.g ANEURALNETWORKS_INT8. */
+  int32_t type;
+  /** The number of dimensions. It should be 0 for scalars. */
+  uint32_t dimensionCount;
+  /** The dimensions of the tensor. It should be nullptr for scalars. */
+  const uint32_t* dimensions;
+  /** These two fields are only used for quantized tensors.
+   * They should be zero for scalars and non-fixed point tensors.
+   * The dequantized value of each entry is (value - offset) * scale.
+   */
+  float scale;
+  int32_t zeroPoint;
+} ANeuralNetworksOperandType;
+
+/**
+ * ANeuralNetworksEvent is an opaque type that represents an event
+ * that will be signaled once an execution completes.
+ */
+typedef struct ANeuralNetworksEvent ANeuralNetworksEvent;
+
+typedef int32_t ANeuralNetworksOperationType;
+
+// nn api function types
+
+typedef int (*ANeuralNetworksMemory_createFromFd_fn)(
+    size_t size, int protect, int fd, size_t offset,
+    ANeuralNetworksMemory** memory);
+
+typedef void (*ANeuralNetworksMemory_free_fn)(ANeuralNetworksMemory* memory);
+
+typedef int (*ANeuralNetworksModel_create_fn)(ANeuralNetworksModel** model);
+
+typedef int (*ANeuralNetworksModel_finish_fn)(ANeuralNetworksModel* model);
+
+typedef void (*ANeuralNetworksModel_free_fn)(ANeuralNetworksModel* model);
+
+typedef int (*ANeuralNetworksCompilation_create_fn)(
+    ANeuralNetworksModel* model, ANeuralNetworksCompilation** compilation);
+
+typedef void (*ANeuralNetworksCompilation_free_fn)(
+    ANeuralNetworksCompilation* compilation);
+
+typedef int (*ANeuralNetworksCompilation_setPreference_fn)(
+    ANeuralNetworksCompilation* compilation, int32_t preference);
+
+typedef int (*ANeuralNetworksCompilation_finish_fn)(
+    ANeuralNetworksCompilation* compilation);
+
+typedef int (*ANeuralNetworksModel_addOperand_fn)(
+    ANeuralNetworksModel* model, const ANeuralNetworksOperandType* type);
+
+typedef int (*ANeuralNetworksModel_setOperandValue_fn)(
+    ANeuralNetworksModel* model, int32_t index, const void* buffer,
+    size_t length);
+
+typedef int (*ANeuralNetworksModel_setOperandValueFromMemory_fn)(
+    ANeuralNetworksModel* model, int32_t index,
+    const ANeuralNetworksMemory* memory, size_t offset, size_t length);
+
+typedef int (*ANeuralNetworksModel_addOperation_fn)(
+    ANeuralNetworksModel* model, ANeuralNetworksOperationType type,
+    uint32_t inputCount, const uint32_t* inputs, uint32_t outputCount,
+    const uint32_t* outputs);
+
+typedef int (*ANeuralNetworksModel_identifyInputsAndOutputs_fn)(
+    ANeuralNetworksModel* model, uint32_t inputCount, const uint32_t* inputs,
+    uint32_t outputCount, const uint32_t* outputs);
+
+typedef int (*ANeuralNetworksModel_relaxComputationFloat32toFloat16_fn)(
+    ANeuralNetworksModel* model, bool allow);
+
+typedef int (*ANeuralNetworksExecution_create_fn)(
+    ANeuralNetworksCompilation* compilation,
+    ANeuralNetworksExecution** execution);
+
+typedef void (*ANeuralNetworksExecution_free_fn)(
+    ANeuralNetworksExecution* execution);
+
+typedef int (*ANeuralNetworksExecution_setInput_fn)(
+    ANeuralNetworksExecution* execution, int32_t index,
+    const ANeuralNetworksOperandType* type, const void* buffer, size_t length);
+
+typedef int (*ANeuralNetworksExecution_setInputFromMemory_fn)(
+    ANeuralNetworksExecution* execution, int32_t index,
+    const ANeuralNetworksOperandType* type, const ANeuralNetworksMemory* memory,
+    size_t offset, size_t length);
+
+typedef int (*ANeuralNetworksExecution_setOutput_fn)(
+    ANeuralNetworksExecution* execution, int32_t index,
+    const ANeuralNetworksOperandType* type, void* buffer, size_t length);
+
+typedef int (*ANeuralNetworksExecution_setOutputFromMemory_fn)(
+    ANeuralNetworksExecution* execution, int32_t index,
+    const ANeuralNetworksOperandType* type, const ANeuralNetworksMemory* memory,
+    size_t offset, size_t length);
+
+typedef int (*ANeuralNetworksExecution_startCompute_fn)(
+    ANeuralNetworksExecution* execution, ANeuralNetworksEvent** event);
+
+typedef int (*ANeuralNetworksEvent_wait_fn)(ANeuralNetworksEvent* event);
+
+typedef void (*ANeuralNetworksEvent_free_fn)(ANeuralNetworksEvent* event);
+
+typedef int (*ASharedMemory_create_fn)(const char* name, size_t size);
+
+#endif  // TENSORFLOW_LITE_NNAPI_NEURALNETWORKSTYPES_H_
-- 
GitLab


From a6b1f39d10bf7312d7e3c43dcb7dea74b24c42e3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 15 Jan 2019 08:44:29 -0800
Subject: [PATCH 0716/2345] Replace calls to deprecated googletest macros
 *TEST_CASE() with *TEST_SUITE()

PiperOrigin-RevId: 229378037
---
 .../compiler/xla/client/lib/cholesky_test.cc  | 10 +++----
 .../xla/client/lib/triangular_solve_test.cc   |  6 ++--
 tensorflow/compiler/xla/literal_test.cc       |  2 +-
 .../xla/service/algebraic_simplifier_test.cc  | 18 ++++++------
 .../cpu/cpu_instruction_fusion_test.cc        |  7 +++--
 .../xla/service/cpu/cpu_runtime_test.cc       | 11 ++++----
 .../cpu/tests/cpu_eigen_dot_operation_test.cc |  8 +++---
 .../service/cpu/tests/cpu_intrinsic_test.cc   |  8 +++---
 .../xla/service/hlo_dataflow_analysis_test.cc |  6 ++--
 .../xla/service/hlo_evaluator_test.cc         |  4 +--
 .../compiler/xla/service/hlo_parser_test.cc   | 28 +++++++++----------
 .../xla/service/hlo_rematerialization_test.cc |  4 +--
 12 files changed, 57 insertions(+), 55 deletions(-)

diff --git a/tensorflow/compiler/xla/client/lib/cholesky_test.cc b/tensorflow/compiler/xla/client/lib/cholesky_test.cc
index ba9580a3d3..095dd4fbf8 100644
--- a/tensorflow/compiler/xla/client/lib/cholesky_test.cc
+++ b/tensorflow/compiler/xla/client/lib/cholesky_test.cc
@@ -157,10 +157,10 @@ XLA_TEST_P(RandomCholeskyTest, Random) {
                              xla::ErrorSpec(1e-4, 1e-4));
 }
 
-INSTANTIATE_TEST_CASE_P(RandomCholeskyTestInstance, RandomCholeskyTest,
-                        ::testing::Values(CholeskyTestCase{1, 1},
-                                          CholeskyTestCase{1, 2},
-                                          CholeskyTestCase{10, 5},
-                                          CholeskyTestCase{2, 20}));
+INSTANTIATE_TEST_SUITE_P(RandomCholeskyTestInstance, RandomCholeskyTest,
+                         ::testing::Values(CholeskyTestCase{1, 1},
+                                           CholeskyTestCase{1, 2},
+                                           CholeskyTestCase{10, 5},
+                                           CholeskyTestCase{2, 20}));
 
 }  // namespace
diff --git a/tensorflow/compiler/xla/client/lib/triangular_solve_test.cc b/tensorflow/compiler/xla/client/lib/triangular_solve_test.cc
index 703227c949..284a2e9d18 100644
--- a/tensorflow/compiler/xla/client/lib/triangular_solve_test.cc
+++ b/tensorflow/compiler/xla/client/lib/triangular_solve_test.cc
@@ -439,9 +439,9 @@ std::vector<TriangularSolveTestSpec> TriangularSolveTests() {
   return specs;
 }
 
-INSTANTIATE_TEST_CASE_P(TriangularSolveParametricTestInstantiation,
-                        TriangularSolveParametricTest,
-                        ::testing::ValuesIn(TriangularSolveTests()));
+INSTANTIATE_TEST_SUITE_P(TriangularSolveParametricTestInstantiation,
+                         TriangularSolveParametricTest,
+                         ::testing::ValuesIn(TriangularSolveTests()));
 
 }  // namespace
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/literal_test.cc b/tensorflow/compiler/xla/literal_test.cc
index e67bb5c32f..b54a71ae68 100644
--- a/tensorflow/compiler/xla/literal_test.cc
+++ b/tensorflow/compiler/xla/literal_test.cc
@@ -641,7 +641,7 @@ template <typename T>
 class LiteralUtilTestTemplated : public ::testing::Test {};
 
 using TestedTypes = ::testing::Types<float, int32, uint32, complex64>;
-TYPED_TEST_CASE(LiteralUtilTestTemplated, TestedTypes);
+TYPED_TEST_SUITE(LiteralUtilTestTemplated, TestedTypes);
 
 TYPED_TEST(LiteralUtilTestTemplated, Relayout2x2) {
   // Make a non-integer for floating point types.
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index d2ff0c9f21..0dea498456 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -2989,7 +2989,7 @@ class ConvInputPaddingTest
     : public AlgebraicSimplifierTest,
       public ::testing::WithParamInterface<ConvPaddingTestcase> {};
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     ConvInputPaddingTestCases, ConvInputPaddingTest,
     ::testing::ValuesIn(std::vector<ConvPaddingTestcase>{
         // Merge this edge padding into the conv.
@@ -3097,7 +3097,7 @@ class ConvFilterPaddingTest
     : public AlgebraicSimplifierTest,
       public ::testing::WithParamInterface<ConvPaddingTestcase> {};
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     ConvFilterPaddingTestCases, ConvFilterPaddingTest,
     ::testing::ValuesIn(std::vector<ConvPaddingTestcase>{
         // Can only merge interior padding on the filter's spatial dimensions;
@@ -4197,7 +4197,7 @@ PadReduceWindowEffectiveBroadcastCases() {
   return *cases;
 }
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     PadReduceWindowEffectiveBroadcastInstantiation,
     PadReduceWindowEffectiveBroadcastTest,
     ::testing::ValuesIn(PadReduceWindowEffectiveBroadcastCases()));
@@ -4248,7 +4248,7 @@ TEST_P(BatchDotStrengthReductionTest, BatchDotStrengthReduction) {
   EXPECT_EQ(has_no_dot, dot_should_be_transformed);
 }
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     BatchDotStrengthReductionTestInstantiation, BatchDotStrengthReductionTest,
     ::testing::Combine(::testing::Values(1, 2), ::testing::Values(1, 2),
                        ::testing::Values(1, 2), ::testing::Values(F32, BF16)));
@@ -4305,7 +4305,7 @@ TEST_P(DotStrengthReductionTest, DotStrengthReduction) {
   EXPECT_EQ(has_no_dot, dot_should_be_transformed);
 }
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     DotStrengthReductionTestInstantiation, DotStrengthReductionTest,
     ::testing::Combine(::testing::Values(1, 2), ::testing::Values(1, 2),
                        ::testing::Values(1, 2), ::testing::Bool(),
@@ -4479,9 +4479,9 @@ TEST_F(AlgebraicSimplifierTest, DynamicUpdateSliceZeroUpdate) {
   EXPECT_THAT(computation->root_instruction(), operand);
 }
 
-INSTANTIATE_TEST_CASE_P(DotOfConcatSimplificationTestInstantiation,
-                        DotOfConcatSimplificationTest,
-                        ::testing::ValuesIn(kDotOfConcatTestSpecs));
+INSTANTIATE_TEST_SUITE_P(DotOfConcatSimplificationTestInstantiation,
+                         DotOfConcatSimplificationTest,
+                         ::testing::ValuesIn(kDotOfConcatTestSpecs));
 
 struct DotOfGatherTestSpec {
   int64 m;
@@ -4687,7 +4687,7 @@ std::vector<DotOfGatherTestSpec> DotOfGatherPositiveNegativeTests() {
   return all;
 }
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     DotOfGatherSimplificationTestInstantiation, DotOfGatherSimplificationTest,
     ::testing::ValuesIn(DotOfGatherPositiveNegativeTests()));
 
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc b/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc
index 46c1d4c38e..86cc72dd7c 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc
@@ -943,9 +943,10 @@ ENTRY main {
   return result;
 }
 
-INSTANTIATE_TEST_CASE_P(GatherLoopFusionTestInstantiation, GatherLoopFusionTest,
-                        ::testing::ValuesIn(GetGatherLoopFusionTestSpecs()),
-                        GatherLoopFusionTestSpec::Name);
+INSTANTIATE_TEST_SUITE_P(GatherLoopFusionTestInstantiation,
+                         GatherLoopFusionTest,
+                         ::testing::ValuesIn(GetGatherLoopFusionTestSpecs()),
+                         GatherLoopFusionTestSpec::Name);
 }  // namespace
 }  // namespace cpu
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_runtime_test.cc b/tensorflow/compiler/xla/service/cpu/cpu_runtime_test.cc
index 1ae3aa5711..4e8c986783 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_runtime_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_runtime_test.cc
@@ -162,11 +162,12 @@ TEST_P(EigenMatMulTest, DoIt) {
   CheckMatrixMultiply(*a, *b, *c);
 }
 
-INSTANTIATE_TEST_CASE_P(EigenMatMulTestInstantiaion, EigenMatMulTest,
-                        ::testing::Combine(::testing::ValuesIn(MatMulShapes),
-                                           ::testing::Bool(), ::testing::Bool(),
-                                           ::testing::Bool()),
-                        EigenMatMulTest::Name);
+INSTANTIATE_TEST_SUITE_P(EigenMatMulTestInstantiaion, EigenMatMulTest,
+                         ::testing::Combine(::testing::ValuesIn(MatMulShapes),
+                                            ::testing::Bool(),
+                                            ::testing::Bool(),
+                                            ::testing::Bool()),
+                         EigenMatMulTest::Name);
 
 #ifdef INTEL_MKL
 class MKLMatMulTest : public CpuRuntimeTest,
diff --git a/tensorflow/compiler/xla/service/cpu/tests/cpu_eigen_dot_operation_test.cc b/tensorflow/compiler/xla/service/cpu/tests/cpu_eigen_dot_operation_test.cc
index f8f5f392da..0b4ac9dc29 100644
--- a/tensorflow/compiler/xla/service/cpu/tests/cpu_eigen_dot_operation_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/tests/cpu_eigen_dot_operation_test.cc
@@ -102,10 +102,10 @@ std::vector<DotTestSpec> GetDotTestCases() {
   return result;
 }
 
-INSTANTIATE_TEST_CASE_P(CpuEigenDotOperationTestInstantiation,
-                        CpuEigenDotOperationTest,
-                        ::testing::ValuesIn(GetDotTestCases()),
-                        DotTestSpecToString);
+INSTANTIATE_TEST_SUITE_P(CpuEigenDotOperationTestInstantiation,
+                         CpuEigenDotOperationTest,
+                         ::testing::ValuesIn(GetDotTestCases()),
+                         DotTestSpecToString);
 
 }  // namespace
 }  // namespace cpu
diff --git a/tensorflow/compiler/xla/service/cpu/tests/cpu_intrinsic_test.cc b/tensorflow/compiler/xla/service/cpu/tests/cpu_intrinsic_test.cc
index 9b10c49f4f..3fb0e3cd91 100644
--- a/tensorflow/compiler/xla/service/cpu/tests/cpu_intrinsic_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/tests/cpu_intrinsic_test.cc
@@ -140,10 +140,10 @@ IntrinsicTestSpec CpuUnaryIntrinsicTestCases[] = {
         HloOpcode::kLog, kTriple_android_arm, "",
         R"(CHECK: fadd fast <4 x float> <float 0x3FBDE4A340000000, float 0x3FBDE4A340000000, float 0x3FBDE4A340000000, float 0x3FBDE4A340000000>)"}};
 
-INSTANTIATE_TEST_CASE_P(CpuUnaryIntrinsicTestInstantiation,
-                        CpuUnaryIntrinsicTest,
-                        ::testing::ValuesIn(CpuUnaryIntrinsicTestCases),
-                        CpuUnaryIntrinsicTest::Name);
+INSTANTIATE_TEST_SUITE_P(CpuUnaryIntrinsicTestInstantiation,
+                         CpuUnaryIntrinsicTest,
+                         ::testing::ValuesIn(CpuUnaryIntrinsicTestCases),
+                         CpuUnaryIntrinsicTest::Name);
 
 }  // namespace
 }  // namespace cpu
diff --git a/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc b/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc
index 888886865b..4a7c4963b7 100644
--- a/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc
@@ -1901,9 +1901,9 @@ ENTRY %AddDependency (p: f32[3]) -> f32[3] {
   EXPECT_FALSE(analysis->ValueIsDefinedAt(root));
 }
 
-INSTANTIATE_TEST_CASE_P(HloDataflowAnalysisInstantiation,
-                        HloDataflowAnalysisTest,
-                        ::testing::Values(false, true));
+INSTANTIATE_TEST_SUITE_P(HloDataflowAnalysisInstantiation,
+                         HloDataflowAnalysisTest,
+                         ::testing::Values(false, true));
 
 class HloDataflowAnalysisTestBase : public HloTestBase {
  protected:
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
index 674df0016a..d34fa48efb 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
@@ -126,8 +126,8 @@ class HloEvaluatorBf16Test : public ::testing::WithParamInterface<bool>,
   HloEvaluatorBf16Test() : HloEvaluatorTest(/*use_bfloat16=*/GetParam()) {}
 };
 
-INSTANTIATE_TEST_CASE_P(HloEvaluatorTest_Instantiation, HloEvaluatorBf16Test,
-                        ::testing::ValuesIn(use_bf16_params));
+INSTANTIATE_TEST_SUITE_P(HloEvaluatorTest_Instantiation, HloEvaluatorBf16Test,
+                         ::testing::ValuesIn(use_bf16_params));
 
 // Verifies that HloEvaluator evaluates a HLO instruction that performs clamp
 // with 3 operands.
diff --git a/tensorflow/compiler/xla/service/hlo_parser_test.cc b/tensorflow/compiler/xla/service/hlo_parser_test.cc
index 74ef8a2fba..6ba16cc82a 100644
--- a/tensorflow/compiler/xla/service/hlo_parser_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser_test.cc
@@ -1371,20 +1371,20 @@ TEST_P(HloParserTestLongProto, Run) { ExpectEqual(); }
 TEST_P(HloParserTestShort, Run) { ExpectEqual(); }
 TEST_P(HloParserTestShortProto, Run) { ExpectEqual(); }
 
-INSTANTIATE_TEST_CASE_P(HloParserTestSuccessInstantiation, HloParserTestLong,
-                        ::testing::ValuesIn(CreateTestCases()),
-                        TestDataToString);
-INSTANTIATE_TEST_CASE_P(HloParserTestSuccessInstantiation,
-                        HloParserTestLongProto,
-                        ::testing::ValuesIn(CreateTestCases()),
-                        TestDataToString);
-INSTANTIATE_TEST_CASE_P(HloParserTestSuccessInstantiation, HloParserTestShort,
-                        ::testing::ValuesIn(CreateShortTestCases()),
-                        TestDataToString);
-INSTANTIATE_TEST_CASE_P(HloParserTestSuccessInstantiation,
-                        HloParserTestShortProto,
-                        ::testing::ValuesIn(CreateShortTestCases()),
-                        TestDataToString);
+INSTANTIATE_TEST_SUITE_P(HloParserTestSuccessInstantiation, HloParserTestLong,
+                         ::testing::ValuesIn(CreateTestCases()),
+                         TestDataToString);
+INSTANTIATE_TEST_SUITE_P(HloParserTestSuccessInstantiation,
+                         HloParserTestLongProto,
+                         ::testing::ValuesIn(CreateTestCases()),
+                         TestDataToString);
+INSTANTIATE_TEST_SUITE_P(HloParserTestSuccessInstantiation, HloParserTestShort,
+                         ::testing::ValuesIn(CreateShortTestCases()),
+                         TestDataToString);
+INSTANTIATE_TEST_SUITE_P(HloParserTestSuccessInstantiation,
+                         HloParserTestShortProto,
+                         ::testing::ValuesIn(CreateShortTestCases()),
+                         TestDataToString);
 
 class HloParserTest : public ::testing::Test {
  protected:
diff --git a/tensorflow/compiler/xla/service/hlo_rematerialization_test.cc b/tensorflow/compiler/xla/service/hlo_rematerialization_test.cc
index 22c3c40a93..c4bb475619 100644
--- a/tensorflow/compiler/xla/service/hlo_rematerialization_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_rematerialization_test.cc
@@ -588,8 +588,8 @@ TEST_P(IndirectUseTest, IndirectUseNotRematerialized) {
   }
 }
 
-INSTANTIATE_TEST_CASE_P(IndirectUseTestInstantiation, IndirectUseTest,
-                        ::testing::Values(true, false));
+INSTANTIATE_TEST_SUITE_P(IndirectUseTestInstantiation, IndirectUseTest,
+                         ::testing::Values(true, false));
 
 }  // namespace
 
-- 
GitLab


From e0c7bf29e7486f1b057abf50baf5cd4058e45eb0 Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Tue, 15 Jan 2019 08:53:33 -0800
Subject: [PATCH 0717/2345] [Grappler] Don't allow creation of self loops on
 mutation in MutableGraphView.

PiperOrigin-RevId: 229379324
---
 .../core/grappler/mutable_graph_view.cc       |  34 ++++
 .../core/grappler/mutable_graph_view_test.cc  | 186 +++++++++++++++++-
 2 files changed, 218 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/grappler/mutable_graph_view.cc b/tensorflow/core/grappler/mutable_graph_view.cc
index 13a530181c..2ba53a685d 100644
--- a/tensorflow/core/grappler/mutable_graph_view.cc
+++ b/tensorflow/core/grappler/mutable_graph_view.cc
@@ -388,6 +388,10 @@ Status MutableGraphView::AddRegularFanin(absl::string_view node_name,
         "Can't add$0 fanin '$1' as regular fanin to$2 node '$3'.", fanin_err,
         fanin.ToString(), node_err, node_name));
   }
+  if (node_name == fanin.node()) {
+    return errors::Internal(absl::Substitute(
+        "Can't add fanin '$0' as regular fanin to self.", fanin.ToString()));
+  }
 
   AddFaninInternal(node, {fanin_node, fanin.index()});
   return Status::OK();
@@ -406,6 +410,10 @@ Status MutableGraphView::AddControllingFanin(absl::string_view node_name,
         absl::Substitute("Can't add$0 controlling fanin '$1' to$2 node '$3'.",
                          fanin_err, fanin.ToString(), node_err, node_name));
   }
+  if (node_name == fanin.node()) {
+    return errors::Internal(absl::Substitute(
+        "Can't add controlling fanin '$0' to self.", fanin.ToString()));
+  }
 
   if (!IsSwitch(*fanin_node)) {
     AddFaninInternal(node, {fanin_node, Graph::kControlSlot});
@@ -426,6 +434,12 @@ Status MutableGraphView::AddControllingFanin(absl::string_view node_name,
     for (auto fanout : fanouts) {
       if (IsIdentity(*fanout.node) || IsIdentityNSingleInput(*fanout.node)) {
         if (ParseTensorName(fanout.node->input(0)) == fanin) {
+          if (fanout.node->name() == node_name) {
+            return errors::Internal(absl::Substitute(
+                "Can't add found controlling fanin '$0' from fanin '$1' to "
+                "self.",
+                AsControlDependency(fanout.node->name()), fanin.ToString()));
+          }
           AddFaninInternal(node, {fanout.node, Graph::kControlSlot});
           return Status::OK();
         }
@@ -435,6 +449,11 @@ Status MutableGraphView::AddControllingFanin(absl::string_view node_name,
     // dependency: add a new identity node.
     string ctrl_dep_name = AddPrefixToNodeName(
         absl::StrCat(fanin.node(), "_", fanin.index()), kMutableGraphViewCtrl);
+    if (node_name == ctrl_dep_name) {
+      return errors::Internal(absl::Substitute(
+          "Can't add generated controlling fanin '$0' from fanin '$1' to self.",
+          AsControlDependency(ctrl_dep_name), fanin.ToString()));
+    }
 
     // Reuse a previously created node, if possible.
     NodeDef* ctrl_dep_node = GetNode(ctrl_dep_name);
@@ -515,6 +534,11 @@ Status MutableGraphView::RemoveRegularFanin(absl::string_view node_name,
         "Can't remove$0 fanin '$1' as regular fanin from$2 node '$3'.",
         fanin_err, fanin.ToString(), node_err, node_name));
   }
+  if (node_name == fanin.node()) {
+    return errors::Internal(
+        absl::Substitute("Can't remove fanin '$0' as regular fanin from self.",
+                         fanin.ToString()));
+  }
 
   RemoveRegularFaninInternal(node, {fanin_node, fanin.index()});
   return Status::OK();
@@ -550,6 +574,11 @@ Status MutableGraphView::RemoveControllingFanin(
         "Can't remove$0 controlling fanin '$1' from$2 node '$3'.", fanin_err,
         AsControlDependency(string(fanin_node_name)), node_err, node_name));
   }
+  if (node_name == fanin_node_name) {
+    return errors::Internal(
+        absl::Substitute("Can't remove controlling fanin '$0' from self.",
+                         AsControlDependency(string(fanin_node_name))));
+  }
 
   RemoveControllingFaninInternal(node, fanin_node);
   return Status::OK();
@@ -615,6 +644,11 @@ Status MutableGraphView::UpdateFanin(absl::string_view node_name,
         "Switch control dependency.",
         from_fanin.ToString(), to_fanin.ToString(), node_name));
   }
+  if (node_name == from_fanin.node() || node_name == to_fanin.node()) {
+    return errors::Internal(absl::Substitute(
+        "Can't update fanin '$0' to fanin '$1' in self '$2'.",
+        from_fanin.ToString(), to_fanin.ToString(), node_name));
+  }
 
   if (from_fanin == to_fanin) {
     return Status::OK();
diff --git a/tensorflow/core/grappler/mutable_graph_view_test.cc b/tensorflow/core/grappler/mutable_graph_view_test.cc
index 99a69ef263..597bf8a7ee 100644
--- a/tensorflow/core/grappler/mutable_graph_view_test.cc
+++ b/tensorflow/core/grappler/mutable_graph_view_test.cc
@@ -400,6 +400,12 @@ TEST(MutableGraphViewTest, AddRegularFanin) {
       "missing node 'foo_missing'.";
   TestAddRegularFanin("foo_missing", {"bar_missing", Graph::kControlSlot},
                       /*success=*/false, error_msg, /*expected_node=*/nullptr);
+
+  // Add self to create cycle.
+  error_msg = "Can't add fanin 'foo_6:2' as regular fanin to self.";
+  expected_node = NDef("", "", {"^a", "^b"});
+  TestAddRegularFanin("foo_6", {"foo_6", 2}, /*success=*/false, error_msg,
+                      &expected_node);
 }
 
 void CheckFanout(const MutableGraphView& graph, const TensorId& fanin,
@@ -540,6 +546,12 @@ TEST(MutableGraphViewTest, RemoveRegularFanin) {
   TestRemoveRegularFanin("foo_missing", {"bar_missing", Graph::kControlSlot},
                          /*success=*/false, error_msg,
                          /*expected_node=*/nullptr);
+
+  // Remove self.
+  error_msg = "Can't remove fanin 'foo_6:2' as regular fanin from self.";
+  expected_node = NDef("", "", {"^a", "^b"});
+  TestRemoveRegularFanin("foo_6", {"foo_6", 2}, /*success=*/false, error_msg,
+                         &expected_node);
 }
 
 void TestRemoveAllFanins(absl::string_view node_name,
@@ -728,6 +740,22 @@ TEST(MutableGraphViewTest, UpdateFanin) {
   TestUpdateFanin("foo_missing", {"from_bar_missing", -2},
                   {"to_bar_missing", -3},
                   /*success=*/false, error_msg, /*expected_node=*/nullptr);
+
+  // Update to self to create cycle.
+  expected_node = NDef("", "", {"a", "b:2", "b:2", "^c", "^d"});
+  error_msg = "Can't update fanin 'b:2' to fanin 'foo_4:3' in self 'foo_4'.";
+  TestUpdateFanin("foo_4", {"b", 2}, {"foo_4", 3}, /*success=*/false, error_msg,
+                  &expected_node);
+  error_msg = "Can't update fanin 'b:2' to fanin '^foo_4' in self 'foo_4'.";
+  TestUpdateFanin("foo_4", {"b", 2}, {"foo_4", Graph::kControlSlot},
+                  /*success=*/false, error_msg, &expected_node);
+  error_msg = "Can't update fanin '^c' to fanin 'foo_4:4' in self 'foo_4'.";
+  TestUpdateFanin("foo_4", {"c", Graph::kControlSlot}, {"foo_4", 4},
+                  /*success=*/false, error_msg, &expected_node);
+  error_msg = "Can't update fanin '^c' to fanin '^foo_4' in self 'foo_4'.";
+  TestUpdateFanin("foo_4", {"c", Graph::kControlSlot},
+                  {"foo_4", Graph::kControlSlot}, /*success=*/false, error_msg,
+                  &expected_node);
 }
 
 void TestUpdateFaninFromFaninToNodeAsSwitchControl(const TensorId& fanin) {
@@ -1125,6 +1153,30 @@ TEST(MutableGraphViewTest, AddControllingFaninNotSwitch) {
   CompareNodeInputs(graph, &expected, b);
 }
 
+TEST(MutableGraphViewTest, AddControllingFaninSwitch) {
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "Switch", {}, {})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  Status s = graph.AddControllingFanin("a", {"b", Graph::kControlSlot});
+  EXPECT_FALSE(s.ok());
+  string expected_msg =
+      "Can't add Switch as controlling fanin '^b' to node 'a'.";
+  EXPECT_EQ(s.error_message(), expected_msg);
+
+  ASSERT_EQ(graph.graph()->node_size(), 2);
+  NodeDef* a = graph.GetNode("a");
+  ASSERT_NE(a, nullptr);
+  NodeDef expected = NDef("", "", {});
+  CompareNodeInputs(graph, &expected, a);
+
+  NodeDef* b = graph.GetNode("b");
+  ASSERT_NE(b, nullptr);
+  CompareNodeInputs(graph, &expected, b);
+}
+
 TEST(MutableGraphViewTest, AddControllingFaninSwitchWithIdentity) {
   GraphDef graph_def = test::function::GDef(
       {NDef("a", "NotImportant", {}, {}), NDef("switch", "Switch", {}, {}),
@@ -1190,6 +1242,115 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithExistingAddedIdentity) {
   CompareNodeInputs(graph, &expected, a);
 }
 
+void TestAddControllingFaninSelfLoops(absl::string_view node_name,
+                                      const TensorId& fanin,
+                                      const string& error_msg) {
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}),
+       NDef("b", "Switch", {}, {{"T", DT_FLOAT}}),
+       NDef("c", "Identity", {"b:0"}), NDef("d", "Identity", {"b:1"}),
+       NDef("e", "NotImportant", {"^a"})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  Status s = graph.AddControllingFanin(node_name, fanin);
+  EXPECT_FALSE(s.ok());
+  EXPECT_EQ(s.error_message(), error_msg);
+
+  EXPECT_EQ(graph.graph()->node_size(), 5);
+  NodeDef expected;
+  NodeDef* a = graph.GetNode("a");
+  ASSERT_NE(a, nullptr);
+  expected = NDef("", "", {}, {});
+  CompareNodeInputs(graph, &expected, a);
+
+  NodeDef* b = graph.GetNode("b");
+  ASSERT_NE(b, nullptr);
+  CompareNodeInputs(graph, &expected, b);
+
+  NodeDef* c = graph.GetNode("c");
+  ASSERT_NE(c, nullptr);
+  expected = NDef("", "", {"b:0"});
+  CompareNodeInputs(graph, &expected, c);
+
+  NodeDef* d = graph.GetNode("d");
+  ASSERT_NE(d, nullptr);
+  expected = NDef("", "", {"b:1"});
+  CompareNodeInputs(graph, &expected, d);
+
+  NodeDef* e = graph.GetNode("e");
+  ASSERT_NE(e, nullptr);
+  expected = NDef("", "", {"^a"});
+  CompareNodeInputs(graph, &expected, e);
+}
+
+TEST(MutableGraphViewTest, AddControllingFaninSelfLoops) {
+  string error_msg = "Can't add controlling fanin '^a' to self.";
+  TestAddControllingFaninSelfLoops("a", {"a", Graph::kControlSlot}, error_msg);
+
+  // Adding Switch control dependency to Identity consumer. Node `c` is
+  // consuming `b:0`, so adding `b:0` as a control dependency, because it is a
+  // Switch, should trigger a lookup of outputs. As `c` is a consumer and an
+  // Identity, this will introduce a self loop, so no control dependency should
+  // be added.
+  error_msg =
+      "Can't add found controlling fanin '^c' from fanin 'b:0' to self.";
+  TestAddControllingFaninSelfLoops("c", {"b", 0}, error_msg);
+
+  // Adding Switch control dependency to Identity consumer. Node `d` is
+  // consuming `b:1`, so adding `b:1` as a control dependency, because it is a
+  // Switch, should trigger a lookup of outputs. As `d` is a consumer and an
+  // Identity, this will introduce a self loop, so no control dependency should
+  // be added.
+  error_msg =
+      "Can't add found controlling fanin '^d' from fanin 'b:1' to self.";
+  TestAddControllingFaninSelfLoops("d", {"b", 1}, error_msg);
+}
+
+TEST(MutableGraphViewTest, AddControllingFaninSelfLoopsGeneratedIdentity) {
+  GraphDef graph_def =
+      test::function::GDef({NDef("a", "NotImportant", {}, {}),
+                            NDef("b", "Switch", {}, {{"T", DT_FLOAT}}),
+                            NDef("c", "NotImportant", {}),
+                            NDef("ConstantFoldingCtrl/b_1", "Identity", {})},
+                           /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  // Adding Switch control dependency to Identity node of the same name as a
+  // generated Identity node for pinning the control dependency. Because there
+  // are no consumers of `b:1`, there will be an attempt to generate an Identity
+  // node, with name `ConstantFoldingCtrl/b_1`. As the input node is of the same
+  // name, we will introduce a self loop, so no control dependency should be
+  // added.
+  Status s = graph.AddControllingFanin("ConstantFoldingCtrl/b_1", {"b", 1});
+  EXPECT_FALSE(s.ok());
+  string expected_msg =
+      "Can't add generated controlling fanin '^ConstantFoldingCtrl/b_1' from "
+      "fanin 'b:1' to self.";
+  EXPECT_EQ(s.error_message(), expected_msg);
+
+  EXPECT_EQ(graph.graph()->node_size(), 4);
+  NodeDef expected;
+  NodeDef* a = graph.GetNode("a");
+  ASSERT_NE(a, nullptr);
+  expected = NDef("", "", {}, {});
+  CompareNodeInputs(graph, &expected, a);
+
+  NodeDef* b = graph.GetNode("b");
+  ASSERT_NE(b, nullptr);
+  CompareNodeInputs(graph, &expected, b);
+
+  NodeDef* c = graph.GetNode("c");
+  ASSERT_NE(c, nullptr);
+  CompareNodeInputs(graph, &expected, c);
+
+  NodeDef* d = graph.GetNode("ConstantFoldingCtrl/b_1");
+  ASSERT_NE(d, nullptr);
+  CompareNodeInputs(graph, &expected, d);
+}
+
 TEST(MutableGraphViewTest, RemoveControllingFaninMissing) {
   // Actual node.op() is not important in this test.
   GraphDef graph_def = test::function::GDef(
@@ -1233,7 +1394,7 @@ TEST(MutableGraphViewTest, RemoveControllingFaninOnRegularFanin) {
   // Actual node.op() is not important in this test.
   GraphDef graph_def = test::function::GDef(
       {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {"a"}),
-       NDef("c", "NotImportant", {"a", "b", "^c"})},
+       NDef("c", "NotImportant", {"a", "b"})},
       /*funcs=*/{});
 
   MutableGraphView graph(&graph_def);
@@ -1244,7 +1405,28 @@ TEST(MutableGraphViewTest, RemoveControllingFaninOnRegularFanin) {
   ASSERT_EQ(graph.graph()->node_size(), 3);
   NodeDef* c = graph.GetNode("c");
   ASSERT_NE(c, nullptr);
-  NodeDef expected = NDef("", "", {"a", "b", "^c"});
+  NodeDef expected = NDef("", "", {"a", "b"});
+  CompareNodeInputs(graph, &expected, c);
+}
+
+TEST(MutableGraphViewTest, RemoveControllingFaninSelfLoop) {
+  // Actual node.op() is not important in this test.
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {"a"}),
+       NDef("c", "NotImportant", {"a", "b"})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  Status s = graph.RemoveControllingFanin("c", "c");
+  EXPECT_FALSE(s.ok());
+  string expected_msg = "Can't remove controlling fanin '^c' from self.";
+  EXPECT_EQ(s.error_message(), expected_msg);
+
+  ASSERT_EQ(graph.graph()->node_size(), 3);
+  NodeDef* c = graph.GetNode("c");
+  ASSERT_NE(c, nullptr);
+  NodeDef expected = NDef("", "", {"a", "b"});
   CompareNodeInputs(graph, &expected, c);
 }
 
-- 
GitLab


From 39ebdee5e7f5fb7d7f0626977d363426ed22e308 Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Tue, 15 Jan 2019 08:53:37 -0800
Subject: [PATCH 0718/2345] Fixed handling of np.longlong in as_dtype

PiperOrigin-RevId: 229379334
---
 tensorflow/python/framework/dtypes.py      | 11 +++++++++++
 tensorflow/python/framework/dtypes_test.py |  3 +++
 2 files changed, 14 insertions(+)

diff --git a/tensorflow/python/framework/dtypes.py b/tensorflow/python/framework/dtypes.py
index 9386cca379..465ea4229b 100644
--- a/tensorflow/python/framework/dtypes.py
+++ b/tensorflow/python/framework/dtypes.py
@@ -560,6 +560,17 @@ _NP_TO_TF = {
     _np_qint32: qint32,
     _np_bfloat16: bfloat16,
 }
+
+# On Python 2.X `np.longlong` could be a distinct type used for long
+# integers e.g. 42L. See numpy/numpy#9799.
+for pdt in [
+    np.longlong,
+    np.ulonglong,
+]:
+  if pdt not in _NP_TO_TF:
+    _NP_TO_TF[pdt] = next(
+        _NP_TO_TF[dt] for dt in _NP_TO_TF if dt == pdt().dtype)
+
 _TF_TO_NP = {
     types_pb2.DT_HALF:
         np.float16,
diff --git a/tensorflow/python/framework/dtypes_test.py b/tensorflow/python/framework/dtypes_test.py
index 719fdc0953..7dd2a792d1 100644
--- a/tensorflow/python/framework/dtypes_test.py
+++ b/tensorflow/python/framework/dtypes_test.py
@@ -295,6 +295,9 @@ class TypesTest(test_util.TensorFlowTestCase):
     self.assertNotEqual(dtypes.int32, int)
     self.assertNotEqual(dtypes.float64, 2.1)
 
+  def testPythonLongConversion(self):
+    self.assertIs(dtypes.int64, dtypes.as_dtype(np.array(2**32).dtype))
+
   def testPythonTypesConversion(self):
     self.assertIs(dtypes.float32, dtypes.as_dtype(float))
     self.assertIs(dtypes.bool, dtypes.as_dtype(bool))
-- 
GitLab


From 4ff0c987c3358df220632320869d48bc8b51262f Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Tue, 15 Jan 2019 08:58:32 -0800
Subject: [PATCH 0719/2345] Account for the possibility that fully qualified
 name objects may not be normalized. Improve the consistency between function
 and classes in namer.

PiperOrigin-RevId: 229379978
---
 tensorflow/python/autograph/core/naming.py    | 65 +++++++++++++++----
 .../python/autograph/core/naming_test.py      | 16 +++++
 tensorflow/python/autograph/utils/misc.py     |  9 ++-
 .../python/autograph/utils/misc_test.py       | 15 ++++-
 4 files changed, 88 insertions(+), 17 deletions(-)

diff --git a/tensorflow/python/autograph/core/naming.py b/tensorflow/python/autograph/core/naming.py
index b8d79daeba..245795c3d2 100644
--- a/tensorflow/python/autograph/core/naming.py
+++ b/tensorflow/python/autograph/core/naming.py
@@ -18,8 +18,16 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import enum
+
 from tensorflow.python.autograph.pyct import inspect_utils
 from tensorflow.python.autograph.pyct import qual_names
+from tensorflow.python.autograph.utils import misc
+
+
+class _NamingStyle(enum.Enum):
+  SNAKE = 1
+  CAMEL = 2
 
 
 class Namer(object):
@@ -46,17 +54,52 @@ class Namer(object):
 
     self.generated_names = set()
 
+  def _as_symbol_name(self, fqn, style=_NamingStyle.SNAKE):
+    """Returns a symbol name that matches a fully-qualified name.
+
+    The returned name is safe to use for Python symbols. Any special characters
+    present in fqn are replaced according to the style argument.
+
+    Examples:
+
+      self._as_symbol_name('foo.bar', style=_NamingStyle.CAMEL) == 'FooBar'
+      self._as_symbol_name('foo.bar', style=_NamingStyle.SNAKE) == 'foo_bar'
+
+    See the unit tests for more examples.
+
+    Args:
+      fqn: Union[Text, Tuple[Text]] a fully-qualified symbol name. The qualifier
+        may include module, class names, attributes, etc.
+      style: _NamingStyle
+    Returns:
+      Text
+    """
+    assert style in _NamingStyle
+
+    if isinstance(fqn, tuple):
+      cn = '.'.join(fqn)
+    else:
+      cn = fqn
+
+    # Until we clean up the whole FQN mechanism, `fqn` may not be
+    # canonical, that is, in can appear as ('foo.bar', 'baz')
+    # This replaces any characters that might remain because of that.
+    pieces = cn.split('.')
+
+    if style == _NamingStyle.CAMEL:
+      pieces = tuple(misc.capitalize_initial(p) for p in pieces)
+      return ''.join(pieces)
+    elif style == _NamingStyle.SNAKE:
+      return '_'.join(pieces)
+
   def compiled_class_name(self, original_fqn, live_entity=None):
     """See call_trees.FunctionNamer.compiled_class_name."""
     if live_entity is not None and live_entity in self.renamed_calls:
       return self.renamed_calls[live_entity]
 
-    if isinstance(original_fqn, tuple):
-      original_name = '__'.join(original_fqn)
-    else:
-      original_name = original_fqn
-
-    new_name_root = 'Tf%s' % original_name
+    canonical_name = self._as_symbol_name(
+        original_fqn, style=_NamingStyle.CAMEL)
+    new_name_root = 'Tf%s' % canonical_name
     new_name = new_name_root
     n = 0
     while new_name in self.global_namespace:
@@ -73,7 +116,6 @@ class Namer(object):
                              live_entity=None,
                              owner_type=None):
     """See call_trees.FunctionNamer.compiled_function_name."""
-
     if not self.recursive:
       return None, False
 
@@ -84,15 +126,12 @@ class Namer(object):
       # Members are not renamed when part of an entire converted class.
       return None, False
 
-    if isinstance(original_fqn, tuple):
-      original_name = '__'.join(original_fqn)
-    else:
-      original_name = original_fqn
-
     if live_entity is not None and live_entity in self.renamed_calls:
       return self.renamed_calls[live_entity], True
 
-    new_name_root = 'tf__%s' % original_name
+    canonical_name = self._as_symbol_name(
+        original_fqn, style=_NamingStyle.SNAKE)
+    new_name_root = 'tf__%s' % canonical_name
     new_name = new_name_root
     n = 0
     while new_name in self.global_namespace:
diff --git a/tensorflow/python/autograph/core/naming_test.py b/tensorflow/python/autograph/core/naming_test.py
index 2db98836d1..cc8c4314a7 100644
--- a/tensorflow/python/autograph/core/naming_test.py
+++ b/tensorflow/python/autograph/core/naming_test.py
@@ -45,6 +45,22 @@ class NamerTest(test.TestCase):
     self.assertEqual(('tf__foo', True), namer.compiled_function_name(
         'foo', foo))
 
+  def test_compiled_function_name_unsanitized_fqn(self):
+    namer = naming.Namer({}, True, None, ())
+    self.assertEqual(('tf__foo_bar', True),
+                     namer.compiled_function_name('foo.bar'))
+    self.assertEqual(('tf__foo_bar_baz', True), namer.compiled_function_name(
+        ('foo.bar', 'baz')))
+
+  def test_compiled_class_name_basic(self):
+    namer = naming.Namer({}, True, None, ())
+    self.assertEqual('TfFooBar', namer.compiled_class_name(('foo', 'Bar')))
+
+  def test_compiled_class_name_unsanitized_fqn(self):
+    namer = naming.Namer({}, True, None, ())
+    self.assertEqual('TfFooBarBaz',
+                     namer.compiled_class_name(('foo.bar', 'Baz')))
+
   def test_compiled_function_name_avoids_global_conflicts(self):
     def foo():
       pass
diff --git a/tensorflow/python/autograph/utils/misc.py b/tensorflow/python/autograph/utils/misc.py
index 1b06caf0bd..046e6cf97d 100644
--- a/tensorflow/python/autograph/utils/misc.py
+++ b/tensorflow/python/autograph/utils/misc.py
@@ -23,7 +23,7 @@ from tensorflow.python.ops import array_ops
 
 
 def alias_tensors(*args):
-  """Wrap any Tensor arguments with an identity op.
+  """Wraps any Tensor arguments with an identity op.
 
   Any other argument, including Variables, is returned unchanged.
 
@@ -48,3 +48,10 @@ def alias_tensors(*args):
     return alias_if_tensor(args[0])
 
   raise ValueError('at least one argument required')
+
+
+def capitalize_initial(s):
+  """Capitalizes the initial of a string only."""
+  if s:
+    return s[0].upper() + s[1:]
+  return s
diff --git a/tensorflow/python/autograph/utils/misc_test.py b/tensorflow/python/autograph/utils/misc_test.py
index c78df48d62..24b5753a91 100644
--- a/tensorflow/python/autograph/utils/misc_test.py
+++ b/tensorflow/python/autograph/utils/misc_test.py
@@ -18,7 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.python.autograph.utils.misc import alias_tensors
+from tensorflow.python.autograph.utils import misc
 from tensorflow.python.framework import test_util
 from tensorflow.python.framework.constant_op import constant
 from tensorflow.python.ops.variables import Variable
@@ -27,11 +27,20 @@ from tensorflow.python.platform import test
 
 class MiscTest(test.TestCase):
 
+  def test_capitalize_initial(self):
+    self.assertEqual('', misc.capitalize_initial(''))
+    self.assertEqual('A', misc.capitalize_initial('A'))
+    self.assertEqual('Ab', misc.capitalize_initial('Ab'))
+    self.assertEqual('AbC', misc.capitalize_initial('AbC'))
+    self.assertEqual('A', misc.capitalize_initial('a'))
+    self.assertEqual('Ab', misc.capitalize_initial('ab'))
+    self.assertEqual('AbC', misc.capitalize_initial('abC'))
+
   @test_util.run_deprecated_v1
   def test_alias_single_tensor(self):
     a = constant(1)
 
-    new_a = alias_tensors(a)
+    new_a = misc.alias_tensors(a)
     self.assertFalse(new_a is a)
     with self.cached_session() as sess:
       self.assertEqual(1, self.evaluate(new_a))
@@ -43,7 +52,7 @@ class MiscTest(test.TestCase):
     s = 'a'
     l = [1, 2, 3]
 
-    new_a, new_v, new_s, new_l = alias_tensors(a, v, s, l)
+    new_a, new_v, new_s, new_l = misc.alias_tensors(a, v, s, l)
 
     self.assertFalse(new_a is a)
     self.assertTrue(new_v is v)
-- 
GitLab


From da47d5a9a6cb6b10b166d4f3169e50bdafec2683 Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Tue, 15 Jan 2019 09:47:53 -0800
Subject: [PATCH 0720/2345] Refactoring: store structured inputs on FuncGraph,
 same as structured outputs. PiperOrigin-RevId: 229387670

---
 tensorflow/python/eager/def_function.py       |  6 ++-
 tensorflow/python/eager/function.py           | 43 +++----------------
 tensorflow/python/framework/func_graph.py     | 40 +++++++++++++++++
 .../saved_model/function_serialization.py     |  4 +-
 4 files changed, 51 insertions(+), 42 deletions(-)

diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py
index bd70b9dad0..977f548b61 100644
--- a/tensorflow/python/eager/def_function.py
+++ b/tensorflow/python/eager/def_function.py
@@ -25,6 +25,7 @@ import weakref
 from tensorflow.python.eager import context
 from tensorflow.python.eager import function as function_lib
 from tensorflow.python.eager import lift_to_graph
+from tensorflow.python.framework import func_graph as func_graph_module
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
@@ -459,7 +460,8 @@ class PolymorphicFunction(object):
     for signature in self._cached_input_signatures:
       flattened = nest.flatten(signature)
       if any(
-          isinstance(arg, function_lib.UnknownArgument) for arg in flattened):
+          isinstance(arg, func_graph_module.UnknownArgument)
+          for arg in flattened):
         logging.info("Unsupported signature for serialization: %s.", signature)
         continue
       concrete_function = self.get_concrete_function(*signature)
@@ -482,7 +484,7 @@ class PolymorphicFunction(object):
     if self._stateless_fn:
       concrete_functions.extend(self._stateless_fn._function_cache.values())
     for concrete_function in concrete_functions:
-      signature = concrete_function._python_call_signature
+      signature, _ = concrete_function.structured_input_signature
       equal_to_signature = functools.partial(
           function_lib.is_same_structure, signature, check_values=True)
       if not any(equal_to_signature(s) for s in seen):
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index b5bda35916..b0140e5c98 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -568,6 +568,11 @@ class Function(object):
     """Returns tensors in `self.graph` corresponding to arguments."""
     return self._func_graph.inputs
 
+  @property
+  def structured_input_signature(self):
+    """Returns structured signature of the original function."""
+    return self._func_graph.structured_input_signature
+
   @property
   def outputs(self):
     """Returns tensors in `self.graph` corresponding to returned tensors."""
@@ -824,37 +829,6 @@ class Function(object):
     return ret
 
 
-class UnknownArgument(object):
-  """Signifies an argument which is not currently handled."""
-  pass
-
-
-def convert_structure_to_signature(structure):
-  """Convert a potentially nested structure to a signature.
-
-  Args:
-    structure: Structure to convert.
-
-  Returns:
-    Identical structure that has TensorSpec objects instead of Tensors and
-    UknownArgument instead of any unsupported types.
-  """
-
-  def encode_arg(arg, name=None):
-    """A representation for this argument, for converting into signatures."""
-    if isinstance(arg, ops.Tensor):
-      return tensor_spec.TensorSpec(arg.shape, arg.dtype, name)
-    if isinstance(arg, (int, float, bool, tensor_spec.TensorSpec)):
-      return arg
-    return UnknownArgument()
-
-  # We are using the flattened paths to name the TensorSpecs. We need an
-  # explicit name for them downstream.
-  flattened_with_paths = nest.flatten_with_joined_string_paths(structure)
-  mapped = [encode_arg(arg, path) for path, arg in flattened_with_paths]
-  return nest.pack_sequence_as(structure, mapped)
-
-
 pywrap_tensorflow.RegisterType("Tensor", ops.Tensor)
 pywrap_tensorflow.RegisterType("IndexedSlices", ops.IndexedSlices)
 
@@ -1347,14 +1321,7 @@ class PolymorphicFunction(object):
                 autograph=self._autograph,
                 arg_names=arg_names),
             self._function_attributes)
-        if self._input_signature:
-          python_call_signature = self._input_signature
-        else:
-          python_call_signature = tuple(convert_structure_to_signature(args))
         # pylint: disable=protected-access
-        # Save information about non-Tensor arguments with the concrete
-        # function. Used to serialize PolymorphicFunctions.
-        graph_function._python_call_signature = python_call_signature
         # Tell the Function to clean up its graph once it goes out of
         # scope. Function does not do this in its constructor since it gets used
         # in some places (like Keras) where the FuncGraph lives longer than the
diff --git a/tensorflow/python/framework/func_graph.py b/tensorflow/python/framework/func_graph.py
index 9603c1536d..5202e9b8f9 100644
--- a/tensorflow/python/framework/func_graph.py
+++ b/tensorflow/python/framework/func_graph.py
@@ -58,6 +58,37 @@ WHITELIST_COLLECTIONS = [
 ]
 
 
+class UnknownArgument(object):
+  """Signifies an argument which is not currently handled."""
+  pass
+
+
+def convert_structure_to_signature(structure):
+  """Convert a potentially nested structure to a signature.
+
+  Args:
+    structure: Structure to convert.
+
+  Returns:
+    Identical structure that has TensorSpec objects instead of Tensors and
+    UknownArgument instead of any unsupported types.
+  """
+
+  def encode_arg(arg, name=None):
+    """A representation for this argument, for converting into signatures."""
+    if isinstance(arg, ops.Tensor):
+      return tensor_spec.TensorSpec(arg.shape, arg.dtype, name)
+    if isinstance(arg, (int, float, bool, tensor_spec.TensorSpec)):
+      return arg
+    return UnknownArgument()
+
+  # We are using the flattened paths to name the TensorSpecs. We need an
+  # explicit name for them downstream.
+  flattened_with_paths = nest.flatten_with_joined_string_paths(structure)
+  mapped = [encode_arg(arg, path) for path, arg in flattened_with_paths]
+  return nest.pack_sequence_as(structure, mapped)
+
+
 class FuncGraph(ops.Graph):
   """Graph representing a function body.
 
@@ -69,6 +100,9 @@ class FuncGraph(ops.Graph):
       inputs coming first.
     outputs: Tensors that will be returned by this function. The tensors are in
       this FuncGraph.
+    structured_input_signature: A possibly-nested python object that was
+      received by this function. Note that this structure might contain Python
+      `None`s.
     structured_outputs: A possibly-nested python object which will be returned
       by this function. The Tensors in this structure are the same as those of
       self.outputs. Note that this structure might contain Python `None`s.
@@ -101,6 +135,7 @@ class FuncGraph(ops.Graph):
     self.name = name
     self.inputs = []
     self.outputs = []
+    self.structured_input_signature = None
     self.structured_outputs = None
     self._weak_variables = []
     self.outer_graph = ops.get_default_graph()
@@ -407,6 +442,11 @@ def func_graph_from_py_func(name,
     func_args = _get_defun_inputs_from_args(args, arg_names)
     func_kwargs = _get_defun_inputs_from_kwargs(kwargs)
 
+    # Convert all Tensors into TensorSpecs before saving the structured inputs.
+    func_graph.structured_input_signature = (
+        convert_structure_to_signature(func_args),
+        convert_structure_to_signature(func_kwargs))
+
     # Note: `nest.flatten` sorts by keys, as does `_deterministic_dict_values`.
     # Variables to help check whether mutation happens in calling the function
     # Copy the recursive list, tuple and map structure, but not base objects
diff --git a/tensorflow/python/saved_model/function_serialization.py b/tensorflow/python/saved_model/function_serialization.py
index e5d3cc4bd2..8e8e3bc856 100644
--- a/tensorflow/python/saved_model/function_serialization.py
+++ b/tensorflow/python/saved_model/function_serialization.py
@@ -18,7 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.python.eager import function as defun_lib
+from tensorflow.python.framework import func_graph as func_graph_module
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.saved_model import nested_structure_coder
 from tensorflow.python.saved_model import saved_object_graph_pb2
@@ -64,7 +64,7 @@ def serialize_polymorphic_function(polymorphic_function, node_ids):
     function_proto.concrete_function = concrete_function.name
     function_proto.canonicalized_input_signature.CopyFrom(
         coder.encode_structure(signature))
-    structured_outputs = defun_lib.convert_structure_to_signature(
+    structured_outputs = func_graph_module.convert_structure_to_signature(
         concrete_function.structured_outputs)
     function_proto.output_signature.CopyFrom(
         coder.encode_structure(structured_outputs))
-- 
GitLab


From c223fbd116f2266ba7129eb8257c8c8a9eceb499 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 15 Jan 2019 10:15:20 -0800
Subject: [PATCH 0721/2345] Upgrade bazel-toolchain to the latest release

Also removed bazel-toolchain support for Bazel version prior to 0.19.0 and updated check_bazel_version_at_least in WORKSPACE file

To fix https://github.com/bazelbuild/bazel/issues/7113

PiperOrigin-RevId: 229392830
---
 WORKSPACE                                     |  2 +-
 .../preconfig/generate/archives.bzl           | 32 ++++++-------------
 2 files changed, 10 insertions(+), 24 deletions(-)

diff --git a/WORKSPACE b/WORKSPACE
index 1c59686f16..957b8d8528 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -73,7 +73,7 @@ swift_rules_dependencies()
 # files, in case the parsing of those build files depends on the bazel
 # version we require here.
 load("//tensorflow:version_check.bzl", "check_bazel_version_at_least")
-check_bazel_version_at_least("0.18.0")
+check_bazel_version_at_least("0.19.0")
 
 load("//tensorflow:workspace.bzl", "tf_workspace")
 
diff --git a/third_party/toolchains/preconfig/generate/archives.bzl b/third_party/toolchains/preconfig/generate/archives.bzl
index 0850893589..bafc7d4943 100644
--- a/third_party/toolchains/preconfig/generate/archives.bzl
+++ b/third_party/toolchains/preconfig/generate/archives.bzl
@@ -2,26 +2,12 @@ load("//tensorflow:version_check.bzl", "parse_bazel_version")
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
 def bazel_toolchains_archive():
-    # Not all bazel versions have set native.bazel_version - if it is not set,
-    # fall back to the more compatible version of the toolchains archive.
-    if native.bazel_version and parse_bazel_version(native.bazel_version) >= parse_bazel_version("0.19"):
-        # This version of the toolchains repo is incompatible with older bazel
-        # versions - we can remove this once TensorFlow drops support for bazel
-        # before 0.19.
-        http_archive(
-            name = "bazel_toolchains",
-            sha256 = "41c48a189be489e2d15dec40e0057ea15b95ee5b39cc2a7e6cf663e31432c75e",
-            strip_prefix = "bazel-toolchains-3f8c58fe530fedc446de04673bc1e32985887dea",
-            urls = [
-                "https://github.com/nlopezgi/bazel-toolchains/archive/3f8c58fe530fedc446de04673bc1e32985887dea.tar.gz",
-            ],
-        )
-    else:
-        http_archive(
-            name = "bazel_toolchains",
-            sha256 = "15b5858b1b5541ec44df31b94c3b8672815b31d71215a98398761ea9f4c4eedb",
-            strip_prefix = "bazel-toolchains-6200b238c9c2d137c0d9a7262c80cc71d98e692b",
-            urls = [
-                "https://github.com/bazelbuild/bazel-toolchains/archive/6200b238c9c2d137c0d9a7262c80cc71d98e692b.tar.gz",
-            ],
-        )
+    http_archive(
+      name = "bazel_toolchains",
+      sha256 = "ee854b5de299138c1f4a2edb5573d22b21d975acfc7aa938f36d30b49ef97498",
+      strip_prefix = "bazel-toolchains-37419a124bdb9af2fec5b99a973d359b6b899b61",
+      urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/37419a124bdb9af2fec5b99a973d359b6b899b61.tar.gz",
+        "https://github.com/bazelbuild/bazel-toolchains/archive/37419a124bdb9af2fec5b99a973d359b6b899b61.tar.gz",
+      ],
+    )
-- 
GitLab


From 9d1b0f9ce7f31d9ed217c06e6c5163abf5f2944d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 15 Jan 2019 10:21:55 -0800
Subject: [PATCH 0722/2345] Automated fixes for core/kernels/BUILD: Remove an
 unreferenced import of the tf_proto_library rule. Remove full package
 qualifier from deps on targets in the same package.

PiperOrigin-RevId: 229394137
---
 tensorflow/core/kernels/BUILD | 35 +++++++++++++++++------------------
 1 file changed, 17 insertions(+), 18 deletions(-)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 4f21b466cd..1be749289b 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -48,7 +48,6 @@ load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_tests")
 load(
     "//tensorflow/core:platform/default/build_config.bzl",
     "tf_kernel_tests_linkstatic",
-    "tf_proto_library",
 )
 load(
     "//tensorflow/core:platform/default/build_config_root.bzl",
@@ -509,12 +508,12 @@ cc_library(
     name = "batch_kernels",
     srcs = ["batch_kernels.cc"],
     deps = [
+        ":concat_lib_hdrs",
+        ":ops_util_hdrs",
+        ":split_lib_hdrs",
         "//tensorflow/core:batch_ops_op_lib",
         "//tensorflow/core:framework_headers_lib",
         "//tensorflow/core:protos_all_cc",
-        "//tensorflow/core/kernels:concat_lib_hdrs",
-        "//tensorflow/core/kernels:ops_util_hdrs",
-        "//tensorflow/core/kernels:split_lib_hdrs",
         "//tensorflow/core/kernels/batching_util:periodic_function_dynamic",
         "//tensorflow/core/kernels/batching_util:shared_batch_scheduler_hdrs",
     ],
@@ -1147,13 +1146,13 @@ tf_cc_test(
     size = "small",
     srcs = ["ragged_gather_op_test.cc"],
     deps = [
+        ":ops_testutil",
         ":ragged_gather_op",
         "//tensorflow/core:framework",
         "//tensorflow/core:ragged_array_ops_op_lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
-        "//tensorflow/core/kernels:ops_testutil",
     ],
 )
 
@@ -1170,13 +1169,13 @@ tf_cc_test(
     name = "ragged_range_op_test",
     srcs = ["ragged_range_op_test.cc"],
     deps = [
+        ":ops_testutil",
         ":ragged_range_op",
         "//tensorflow/core:framework",
         "//tensorflow/core:ragged_math_ops_op_lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
-        "//tensorflow/core/kernels:ops_testutil",
     ],
 )
 
@@ -1194,6 +1193,7 @@ tf_cc_test(
     size = "small",
     srcs = ["ragged_tensor_to_sparse_kernel_test.cc"],
     deps = [
+        ":ops_testutil",
         ":ragged_tensor_to_sparse_kernel",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
@@ -1201,7 +1201,6 @@ tf_cc_test(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
-        "//tensorflow/core/kernels:ops_testutil",
     ],
 )
 
@@ -1210,13 +1209,13 @@ tf_kernel_library(
     srcs = ["cudnn_rnn_ops.cc"],
     visibility = ["//visibility:public"],
     deps = [
+        ":bounds_check_lib",
         ":gpu_util_hdrs",
         "//tensorflow/core:cudnn_rnn_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:stream_executor",
-        "//tensorflow/core/kernels:bounds_check_lib",
         "//third_party/eigen3",
         "@farmhash_archive//:farmhash",
     ],
@@ -4857,6 +4856,8 @@ tf_cc_test(
     size = "small",
     srcs = ["string_format_op_test.cc"],
     deps = [
+        ":ops_testutil",
+        ":ops_util",
         ":string_format_op",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
@@ -4865,8 +4866,6 @@ tf_cc_test(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
-        "//tensorflow/core/kernels:ops_testutil",
-        "//tensorflow/core/kernels:ops_util",
     ],
 )
 
@@ -4899,6 +4898,8 @@ tf_cc_test(
     size = "small",
     srcs = ["regex_replace_op_test.cc"],
     deps = [
+        ":ops_testutil",
+        ":ops_util",
         ":regex_replace_op",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
@@ -4907,8 +4908,6 @@ tf_cc_test(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
-        "//tensorflow/core/kernels:ops_testutil",
-        "//tensorflow/core/kernels:ops_util",
     ],
 )
 
@@ -4923,6 +4922,8 @@ tf_cc_test(
     size = "small",
     srcs = ["string_split_op_test.cc"],
     deps = [
+        ":ops_testutil",
+        ":ops_util",
         ":string_split_op",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
@@ -4931,8 +4932,6 @@ tf_cc_test(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
-        "//tensorflow/core/kernels:ops_testutil",
-        "//tensorflow/core/kernels:ops_util",
     ],
 )
 
@@ -4953,6 +4952,8 @@ tf_cc_test(
     size = "small",
     srcs = ["substr_op_test.cc"],
     deps = [
+        ":ops_testutil",
+        ":ops_util",
         ":substr_op",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
@@ -4962,8 +4963,6 @@ tf_cc_test(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
-        "//tensorflow/core/kernels:ops_testutil",
-        "//tensorflow/core/kernels:ops_util",
     ],
 )
 
@@ -6618,6 +6617,7 @@ cc_library(
     srcs = ["remote_fused_graph_execute_op_test_utils.cc"],
     hdrs = ["remote_fused_graph_execute_op_test_utils.h"],
     deps = [
+        ":cwise_op",
         ":remote_fused_graph_execute_utils",
         "//tensorflow/cc:cc_ops",
         "//tensorflow/cc:ops",
@@ -6625,7 +6625,6 @@ cc_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:testlib",
-        "//tensorflow/core/kernels:cwise_op",
     ],
 )
 
@@ -6636,6 +6635,7 @@ tf_cc_test(
         "remote_fused_graph_execute_utils_test.cc",
     ],
     deps = [
+        ":cwise_op",
         ":remote_fused_graph_execute_op_test_utils",
         ":remote_fused_graph_execute_utils",
         "//tensorflow/cc:cc_ops",
@@ -6651,7 +6651,6 @@ tf_cc_test(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
-        "//tensorflow/core/kernels:cwise_op",
     ],
 )
 
-- 
GitLab


From e0cf2939f71c7742a111de968d9008fa3f880570 Mon Sep 17 00:00:00 2001
From: Guangda Lai <31743510+aaroey@users.noreply.github.com>
Date: Tue, 15 Jan 2019 10:33:10 -0800
Subject: [PATCH 0723/2345] Add comments and fix build.

---
 tensorflow/contrib/tensorrt/BUILD             |  3 --
 .../contrib/tensorrt/kernels/trt_engine_op.cc |  9 +++-
 .../contrib/tensorrt/kernels/trt_engine_op.h  | 13 ------
 .../tensorrt/resources/trt_lru_cache.h        | 44 +++++++++----------
 .../test/dynamic_input_shapes_test.py         |  6 +++
 .../test/tf_trt_integration_test_base.py      | 33 +++++++++++---
 6 files changed, 61 insertions(+), 47 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 3df56271de..17717c8403 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -511,11 +511,8 @@ cuda_py_tests(
         "test/conv2d_test.py",
         "test/dynamic_input_shapes_test.py",
         "test/identity_output_test.py",
-<<<<<<< ab254f392694eb2f7231526c64fc301ef10a1384
         "test/int32_test.py",
-=======
         "test/lru_cache_test.py",
->>>>>>> initial commit (add lru_cache class, enable trt_engine_op to use it)
         "test/manual_test.py",
         "test/memory_alignment_test.py",
         "test/multi_connection_neighbor_engine_test.py",
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
index 06a62fed37..45f3b8d0e5 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
@@ -249,6 +249,10 @@ bool TRTEngineOp::GetCompatibleCachedEngine(
   *engine_input_shapes = actual_input_shapes;
   for (const int cached_batch_size : cached_engine_batch_sizes_) {
     // Check if compatible: batch <= cached batch.
+    //
+    // TODO(laigd): here it only compare the first dim a.k.a the batch size,
+    // we'll need to to support non-batch dimensions as well. This will be done
+    // as part of the offline conversion implementation.
     if (batch_size <= cached_batch_size) {
       // First case: first compatible engine found
       // Second case: smaller batch size engine found
@@ -466,7 +470,8 @@ EngineContext* TRTEngineOp::GetEngine(
       // TODO(tmorris): will all inputs have batch size as first dimension??
       engine_input_shapes[i].set_dim(0, max_batch_size);
     }
-    // Assume engine_input_shapes matches the input shapes of the engine.
+    // TODO(laigd): here we assume engine_input_shapes matches the actual input
+    // shapes of the engine, we should verify that.
     cache.emplace(engine_input_shapes,
                   absl::make_unique<EngineContext>(
                       std::move(static_engine),
@@ -484,7 +489,7 @@ EngineContext* TRTEngineOp::GetEngine(
   // See if there is a compatible engine cached. The batch size should be <= the
   // cached batch size.
   std::vector<tensorflow::TensorShape> engine_input_shapes;
-  bool matched_successfully =
+  const bool matched_successfully =
       GetCompatibleCachedEngine(input_shapes, &engine_input_shapes);
   // If matched, use that engine. Otherwise, we will look in cache for that
   // exact shape and possibly create a new engine if it is not in cache.
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
index 4f39a4c548..c9b325cfd2 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
@@ -53,19 +53,6 @@ class TRTEngineOp : public AsyncOpKernel {
 
  private:
   // TODO(samikama): context should go to a resource manager!
-  struct EngineContext {
-    EngineContext() {}  // Creates an empty context.
-    EngineContext(
-        TrtUniquePtrType<nvinfer1::ICudaEngine>&& input_cuda_engine,
-        TrtUniquePtrType<nvinfer1::IExecutionContext>&& input_execution_context)
-        : cuda_engine(std::move(input_cuda_engine)),
-          execution_context(std::move(input_execution_context)) {}
-
-    mutex mu;
-    TrtUniquePtrType<nvinfer1::ICudaEngine> cuda_engine;
-    TrtUniquePtrType<nvinfer1::IExecutionContext> execution_context
-        GUARDED_BY(mu);
-  };
 
   // Execute calibration
   void ExecuteCalibration(OpKernelContext* ctx, AsyncHelper* helper);
diff --git a/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h b/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
index 57267e5b71..c9648420d0 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
@@ -22,8 +22,9 @@ class LRUCache {
   typedef Value value_type;
   typedef Key key_type;
   typedef HashFunction hasher;
-  typedef typename std::unordered_map<key_type, value_type, hasher>::iterator
-      map_iterator;
+  typedef typename std::unordered_map<key_type, value_type, hasher> map_type;
+  typedef typename map_type::iterator iterator;
+  typedef typename map_type::const_iterator const_iterator;
 
   LRUCache() : capacity_(0) {}
   explicit LRUCache(size_t capacity) : capacity_(capacity) {}
@@ -39,31 +40,26 @@ class LRUCache {
 
   size_t count(const key_type& k) const { return objects_.count(k); }
 
-  value_type& at(key_type k, tensorflow::Status* status_ptr = nullptr) {
-    tensorflow::Status status = Touch(k);
-    if (!status.ok()) {
-      if (status_ptr) {
-        *status_ptr = status;
-      }
-      return not_found_value_;
-    }
-    return objects_.at(k);
+  value_type& at(key_type k) {
+    return Touch(k);
   }
 
-  map_iterator begin() { return objects_.begin(); }
+  const_iterator begin() const { return objects_.begin(); }
+  const_iterator end() const { return objects_.end(); }
 
-  map_iterator end() { return objects_.end(); }
+  iterator begin() { return objects_.begin(); }
+  iterator end() { return objects_.end(); }
 
   template <typename... Args>
-  std::pair<map_iterator, bool> emplace(Args&&... args) {
+  std::pair<iterator, bool> emplace(Args&&... args) {
     DiscardOld(1);
-    std::pair<map_iterator, bool> result =
+    std::pair<iterator, bool> result =
         objects_.emplace(std::forward<Args>(args)...);
     key_type key = result.first->first;
     if (result.second) {
       keys_.push_front(key);
     } else {
-      Touch(key);
+      TouchNoCheck(key);  // The key must exist in this case.
     }
     return result;
   }
@@ -74,16 +70,20 @@ class LRUCache {
   size_t capacity_;
   value_type not_found_value_;
 
-  tensorflow::Status Touch(const key_type& key) {
-    if (!count(key)) {
-      return tensorflow::errors::NotFound("Key not found in cache.");
-    }
+  value_type& Touch(const key_type& key) {
+    // Check that the key exists, and let it return std::out_of_range error if
+    // not.
+    value_type& value = objects_.at(k);
+    TouchNoCheck(key);
+    return value;
+  }
+
+  void TouchNoCheck(const key_type& key) {
     auto rank = std::find(keys_.begin(), keys_.end(), key);
     if (rank != keys_.begin()) {
       keys_.erase(rank);
       keys_.push_front(key);
     }
-    return tensorflow::Status::OK();
   }
 
   // Creates n free positions in cache
@@ -149,7 +149,7 @@ class TRTEngineCacheResource : public tensorflow::ResourceBase {
     oss << "TRTBaseAllocator = " << hex << allocator_.get() << dec << ", ";
     oss << "LRUCache = " << hex << &cache_ << dec << endl;
     oss << "Containing " << cache_.size() << " entries: " << endl;
-    for (auto& item : cache_) {
+    for (const auto& item : cache_) {
       oss << TensorShapeUtils::ShapeListString(item.first) << ": " << hex
           << "ICudaEngine: " << item.second.get()->cuda_engine.get() << ", "
           << "IExecutionContext: " << item.second.get()->execution_context.get()
diff --git a/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py b/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py
index 9985561e1e..1c6d411d11 100644
--- a/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py
+++ b/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py
@@ -34,6 +34,12 @@ import numpy as np
 class DynamicInputShapesTest(trt_test.TfTrtIntegrationTestBase):
 
   def GetParams(self):
+    # TODO(laigd): we should test the following cases:
+    # - batch size is not changed, other dims are changing
+    # - batch size is decreasing, other dims are identical
+    # - batch size is decreasing, other dims are changing
+    # - batch size is increasing, other dims are identical
+    # - batch size is increasing, other dims are changing
     input_dims = [[[1, 5, 5, 1]], [[10, 5, 5, 1]], [[3, 5, 5, 1]],
                   [[1, 5, 5, 1]], [[1, 3, 1, 1]], [[2, 9, 9, 1]],
                   [[1, 224, 224, 1]], [[1, 128, 224, 1]]]
diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
index a105981298..db6be3828c 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
@@ -40,7 +40,15 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import tf_logging as logging
 
 TfTrtIntegrationTestParams = namedtuple("TfTrtIntegrationTestParams", [
-    "gdef", "input_names", "input_dims", "output_names", "expected_output_dims"
+    "gdef",
+    # A list of names of the input placeholder nodes.
+    "input_names",
+    # A list of list of output shapes of the input placeholder nodes.
+    "input_dims",
+    # A list of names of the output identity nodes.
+    "output_names",
+    # A list of list of expected output shapes of the output identity nodes.
+    "expected_output_dims"
 ])
 
 RunParams = namedtuple("RunParams", [
@@ -159,10 +167,19 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
 
   def GetConversionParams(self, run_params):
     """Return a ConversionParams for test."""
+    batch_list
+    for dims_list in self._GetParamsCached().input_dims:
+      if not len(dims_list):
+        continue
+      # Each list of shapes should have same batch size.
+      input_batches = [dims[0] for dims in dims_list]
+      assert max(input_batches) == min(input_batches)
+      batch_list.append(input_batches[0])
     return ConversionParams(
-        max_batch_size=max([
-            dims[0] for inp in self._GetParamsCached().input_dims if len(inp) for dims in inp
-        ]),
+        # We use the minimum of all the batch sizes, so when multiple different
+        # input shapes are provided it'll always create new engines in the
+        # cache, and we can therefore test the cache behavior.
+        max_batch_size=min(batch_list),
         max_workspace_size_bytes=1 << 25,
         precision_mode=run_params.precision_mode,
         minimum_segment_size=2,
@@ -317,8 +334,9 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
             output_len = len(params.expected_output_dims[ind])
             self.assertEqual(output_len, len(new_val))
             for i in range(output_len):
-              self.assertEqual(list(params.expected_output_dims[ind][i]),
-                               list(new_val[i].shape))
+              self.assertEqual(
+                  list(params.expected_output_dims[ind][i]),
+                  list(new_val[i].shape))
             if val is not None:
               self.assertAllClose(val, new_val, atol=1.e-06, rtol=1.e-06)
             val = new_val
@@ -497,7 +515,8 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
         # seq = np.arange(np.prod(dims))
         # seq.resize(dims)
         # input_data.append(scale * seq.astype(dtype))
-        current_input_data.append((scale * np.random.random_sample(dims)).astype(dtype))
+        current_input_data.append(
+            (scale * np.random.random_sample(dims)).astype(dtype))
       inputs_data.append(current_input_data)
 
     self._VerifyGraphDef(run_params, input_gdef, GraphState.ORIGINAL)
-- 
GitLab


From 431165fc97f12d19d6311dfb8b882e3c36d1ca54 Mon Sep 17 00:00:00 2001
From: Guangda Lai <31743510+aaroey@users.noreply.github.com>
Date: Tue, 15 Jan 2019 10:40:05 -0800
Subject: [PATCH 0724/2345] Fix a bug: when building static engine, it should
 use engine input shapes as key instead of actual input shapes to lookup the
 cache. Also fix build.

---
 tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc  | 6 ++----
 tensorflow/contrib/tensorrt/resources/trt_lru_cache.h | 8 ++++----
 2 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
index 45f3b8d0e5..f089f2c1cc 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
@@ -482,7 +482,7 @@ EngineContext* TRTEngineOp::GetEngine(
     if (max_batch_size < batch_size) {
       return &empty_context;
     }
-    return cache.at(input_shapes).get();
+    return cache.at(engine_input_shapes).get();
   }  // static_engine_
 
   // Handle the dynamic engine case.
@@ -503,10 +503,8 @@ EngineContext* TRTEngineOp::GetEngine(
       cached_engine_batch_sizes_.push_back(batch_size);
     }
   }
-  tensorflow::Status engine_found_status;
-  cache.at(engine_input_shapes, &engine_found_status);
 
-  if (!engine_found_status.ok()) {
+  if (!cache.count(engine_input_shapes)) {
     TrtUniquePtrType<nvinfer1::ICudaEngine> engine;
     bool convert_successfully = false;
     LOG(INFO) << "Building a new TensorRT engine for " << name()
diff --git a/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h b/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
index c9648420d0..ba455b9b35 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
@@ -38,10 +38,10 @@ class LRUCache {
 
   size_t size() const { return objects_.size(); }
 
-  size_t count(const key_type& k) const { return objects_.count(k); }
+  size_t count(const key_type& key) const { return objects_.count(key); }
 
-  value_type& at(key_type k) {
-    return Touch(k);
+  value_type& at(const key_type& key) {
+    return Touch(key);
   }
 
   const_iterator begin() const { return objects_.begin(); }
@@ -73,7 +73,7 @@ class LRUCache {
   value_type& Touch(const key_type& key) {
     // Check that the key exists, and let it return std::out_of_range error if
     // not.
-    value_type& value = objects_.at(k);
+    value_type& value = objects_.at(key);
     TouchNoCheck(key);
     return value;
   }
-- 
GitLab


From 347bcbd4ffda686e0b4df95b451e0101c29f9a72 Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Tue, 15 Jan 2019 10:37:54 -0800
Subject: [PATCH 0725/2345] Add quantization for some activation only ops that
 don't have weights. Ops included: - Squeeze - Softmax - Avg and Max pool.

PiperOrigin-RevId: 229397364
---
 .../tools/optimize/quantize_model_test.cc     |   2 +-
 .../lite/tools/optimize/subgraph_quantizer.cc |  90 +++++++++-
 .../lite/tools/optimize/subgraph_quantizer.h  |  14 ++
 .../tools/optimize/subgraph_quantizer_test.cc | 154 ++++++++++++++++--
 tensorflow/lite/tools/optimize/test_util.cc   |  10 +-
 tensorflow/lite/tools/optimize/test_util.h    |  12 +-
 .../lite/tools/optimize/testdata/README.md    |  23 +++
 7 files changed, 285 insertions(+), 20 deletions(-)
 create mode 100644 tensorflow/lite/tools/optimize/testdata/README.md

diff --git a/tensorflow/lite/tools/optimize/quantize_model_test.cc b/tensorflow/lite/tools/optimize/quantize_model_test.cc
index a25f2cc91f..52ed16c0b8 100644
--- a/tensorflow/lite/tools/optimize/quantize_model_test.cc
+++ b/tensorflow/lite/tools/optimize/quantize_model_test.cc
@@ -38,7 +38,7 @@ namespace {
 
 std::unique_ptr<FlatBufferModel> ReadTestModel() {
   auto model_path = tensorflow::io::JoinPath(
-      *g_test_model_dir, internal::kModelWith0Plus10Weights);
+      *g_test_model_dir, internal::kConvModelWith0Plus10Weights);
   return FlatBufferModel::BuildFromFile(model_path.c_str());
 }
 
diff --git a/tensorflow/lite/tools/optimize/subgraph_quantizer.cc b/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
index 27aee83cfd..56f2e6cf76 100644
--- a/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
+++ b/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 #include "tensorflow/lite/tools/optimize/subgraph_quantizer.h"
 
+#include <algorithm>
 #include <limits>
 
 #include "flatbuffers/flexbuffers.h"
@@ -33,6 +34,8 @@ namespace optimize {
 namespace internal {
 
 namespace {
+const int8_t kMinQuantizedValue = -127;
+const int8_t kMaxQuantizedValue = 127;
 
 TfLiteStatus AddQuantizationParams(const SymmetricPerChannelParams& params,
                                    const uint8_t* buffer_data,
@@ -131,9 +134,7 @@ TfLiteStatus SymmetricPerChannelQuantizeTensor(ModelT* model, TensorT* tensor,
   // Calculate scales per channel
   std::vector<float> scales(channel_dim_size);
   std::vector<float> scale_invs(channel_dim_size);
-  const int8_t kMinValue = -127;
-  const int8_t kMaxValue = 127;
-  const float half_scale = kMaxValue;
+  const float half_scale = kMaxQuantizedValue;
   for (size_t channel_idx = 0; channel_idx < channel_dim_size; channel_idx++) {
     const float half_range = std::max(std::abs(min_vals[channel_idx]),
                                       std::abs(max_vals[channel_idx]));
@@ -159,7 +160,8 @@ TfLiteStatus SymmetricPerChannelQuantizeTensor(ModelT* model, TensorT* tensor,
           const int32_t quantized_value =
               static_cast<int32_t>(TfLiteRound(val * scale_invs[channel_idx]));
           final_buffer[index] = std::min<int8_t>(
-              kMaxValue, std::max<int8_t>(kMinValue, quantized_value));
+              kMaxQuantizedValue,
+              std::max<int8_t>(kMinQuantizedValue, quantized_value));
         }
       }
     }
@@ -288,7 +290,9 @@ TfLiteStatus SubgraphQuantizer::QuantizeOpWithBias(BuiltinOperator op_code,
     return kTfLiteError;
   }
   auto input_tensor_idx = op->inputs[op_tensor_info->activation_input_index];
-  TF_LITE_ENSURE_STATUS(AsymmetricQuantizeTensor(op_code, input_tensor_idx));
+  if (IsSubgraphInput(input_tensor_idx)) {
+    TF_LITE_ENSURE_STATUS(AsymmetricQuantizeTensor(op_code, input_tensor_idx));
+  }
   auto weights_tensor_idx = op->inputs[op_tensor_info->weights_input_index];
 
   TensorT* weights_tensor = subgraph_->tensors[weights_tensor_idx].get();
@@ -317,6 +321,72 @@ TfLiteStatus SubgraphQuantizer::QuantizeOpWithBias(BuiltinOperator op_code,
   return kTfLiteOk;
 }
 
+TfLiteStatus SubgraphQuantizer::PropagateMinMaxForAvgAndMaxPool(
+    BuiltinOperator op_code, OperatorT* op) {
+  TF_LITE_ENSURE_EQ(this->error_reporter_, op->inputs.size(), 1);
+
+  if (IsSubgraphInput(op->inputs[0])) {
+    TF_LITE_ENSURE_STATUS(AsymmetricQuantizeTensor(op_code, op->inputs[0]));
+  }
+
+  auto output_tensor = subgraph_->tensors[op->outputs[0]].get();
+  if (output_tensor->type != TensorType_FLOAT32) {
+    return kTfLiteOk;
+  }
+  auto input_tensor = subgraph_->tensors[op->inputs[0]].get();
+  if (!input_tensor->quantization) {
+    error_reporter_->Report(
+        "Missing required min/max information for input of operation: %s",
+        EnumNameBuiltinOperator(op_code));
+    return kTfLiteError;
+  }
+  if (input_tensor->quantization->min.size() != 1 ||
+      input_tensor->quantization->max.size() != 1 ||
+      input_tensor->quantization->scale.size() != 1 ||
+      input_tensor->quantization->zero_point.size() != 1) {
+    error_reporter_->Report(
+        "Invalid quantization information for Op: %s, tensor: %s",
+        EnumNameBuiltinOperator(op_code), input_tensor->name.c_str());
+    return kTfLiteError;
+  }
+  auto quant_params = absl::make_unique<QuantizationParametersT>();
+  // Nudge min, max to include the floating point zero.
+  const float min = std::min(0.f, input_tensor->quantization->min[0]);
+  const float max = std::max(0.f, input_tensor->quantization->max[0]);
+  quant_params->min.push_back(min);
+  quant_params->max.push_back(max);
+  quant_params->scale.push_back(input_tensor->quantization->scale[0]);
+  quant_params->zero_point.push_back(input_tensor->quantization->zero_point[0]);
+  // TODO(shashishekhar): Log a warning here if overriding existing
+  // min/max/scales differ from input scales.
+  output_tensor->quantization = std::move(quant_params);
+  output_tensor->type = TensorType_INT8;
+  return kTfLiteOk;
+}
+
+TfLiteStatus SubgraphQuantizer::AsymmetricQuantizeSingleInputOutputOp(
+    BuiltinOperator op_code, OperatorT* op) {
+  TF_LITE_ENSURE_EQ(this->error_reporter_, op->inputs.size(), 1);
+  TF_LITE_ENSURE_EQ(this->error_reporter_, op->outputs.size(), 1);
+
+  if (IsSubgraphInput(op->inputs[0])) {
+    TF_LITE_ENSURE_STATUS(AsymmetricQuantizeTensor(op_code, op->inputs[0]));
+  }
+
+  auto output_tensor = subgraph_->tensors[op->outputs[0]].get();
+  if (output_tensor->type != TensorType_FLOAT32) {
+    return kTfLiteOk;
+  }
+  auto quant_params = absl::make_unique<QuantizationParametersT>();
+  TF_LITE_ENSURE_STATUS(AsymmetricQuantizeTensor(op_code, op->outputs[0]));
+  return kTfLiteOk;
+}
+
+bool SubgraphQuantizer::IsSubgraphInput(int32_t tensor_idx) const {
+  return std::find(subgraph_->inputs.begin(), subgraph_->inputs.end(),
+                   tensor_idx) != subgraph_->inputs.end();
+}
+
 TfLiteStatus SubgraphQuantizer::QuantizeOperator(int op_idx) {
   OperatorT* op = subgraph_->operators[op_idx].get();
   const BuiltinOperator op_code =
@@ -324,6 +394,16 @@ TfLiteStatus SubgraphQuantizer::QuantizeOperator(int op_idx) {
   if (OpHasOptionalBiasTensor(op_code)) {
     return QuantizeOpWithBias(op_code, op);
   }
+  switch (op_code) {
+    case BuiltinOperator_AVERAGE_POOL_2D:
+    case BuiltinOperator_MAX_POOL_2D:
+      return PropagateMinMaxForAvgAndMaxPool(op_code, op);
+    case BuiltinOperator_SQUEEZE:
+    case BuiltinOperator_SOFTMAX:
+      return AsymmetricQuantizeSingleInputOutputOp(op_code, op);
+    default:
+      return kTfLiteError;
+  }
 
   return kTfLiteError;
 }
diff --git a/tensorflow/lite/tools/optimize/subgraph_quantizer.h b/tensorflow/lite/tools/optimize/subgraph_quantizer.h
index 2543dccea9..9d6ca7fad5 100644
--- a/tensorflow/lite/tools/optimize/subgraph_quantizer.h
+++ b/tensorflow/lite/tools/optimize/subgraph_quantizer.h
@@ -40,9 +40,23 @@ class SubgraphQuantizer {
  private:
   // Quantizes ops with bias tensors.
   TfLiteStatus QuantizeOpWithBias(BuiltinOperator op_code, OperatorT* op);
+
+  // Average and Max pool need special treatement. The scales are propagated
+  // from inputs to outputs.
+  TfLiteStatus PropagateMinMaxForAvgAndMaxPool(BuiltinOperator op_code,
+                                               OperatorT* op);
+
+  // Asymmetric quantizes inputs and outputs of an Op that has single input and
+  // single output. E.g. Squeeze.
+  TfLiteStatus AsymmetricQuantizeSingleInputOutputOp(BuiltinOperator op_code,
+                                                     OperatorT* op);
+
   TfLiteStatus AsymmetricQuantizeTensor(BuiltinOperator op_code,
                                         int32_t tensor_idx);
 
+  // Returns true if |tensor_idx| is one of the inputs in the subgraph.
+  bool IsSubgraphInput(int32_t tensor_idx) const;
+
   ModelT* model_;
   SubGraphT* subgraph_;
   ErrorReporter* error_reporter_;
diff --git a/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc b/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
index 7652429e32..96ba20d4d9 100644
--- a/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
+++ b/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
@@ -12,13 +12,15 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tensorflow/lite/tools/optimize/subgraph_quantizer.h"
+#include <algorithm>
+
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/util/command_line_flags.h"
 #include "tensorflow/lite/schema/schema_generated.h"
+#include "tensorflow/lite/tools/optimize/subgraph_quantizer.h"
 #include "tensorflow/lite/tools/optimize/symmetric_per_channel_params.h"
 #include "tensorflow/lite/tools/optimize/test_util.h"
 
@@ -31,22 +33,31 @@ namespace optimize {
 namespace internal {
 namespace {
 
-std::unique_ptr<FlatBufferModel> ReadTestModel1() {
-  auto model_path = tensorflow::io::JoinPath(*g_test_model_dir,
-                                             kModelWithMinus128Plus127Weights);
+std::unique_ptr<FlatBufferModel> ReadModel(const char* model) {
+  auto model_path = tensorflow::io::JoinPath(*g_test_model_dir, model);
   return FlatBufferModel::BuildFromFile(model_path.c_str());
 }
 
-std::unique_ptr<FlatBufferModel> ReadTestModel2() {
-  auto model_path =
-      tensorflow::io::JoinPath(*g_test_model_dir, kModelWith0Plus10Weights);
-  return FlatBufferModel::BuildFromFile(model_path.c_str());
+std::unique_ptr<FlatBufferModel> ReadConvModel1() {
+  return ReadModel(kConvModelWithMinus128Plus127Weights);
+}
+
+std::unique_ptr<FlatBufferModel> ReadConvModel2() {
+  return ReadModel(kConvModelWith0Plus10Weights);
+}
+
+std::unique_ptr<FlatBufferModel> ReadSoftmaxModel() {
+  return ReadModel(kSingleSoftmaxModelMinMinus5MaxPlus5);
+}
+
+std::unique_ptr<FlatBufferModel> ReadAvgPoolModel() {
+  return ReadModel(kSingleAvgPoolModelMinMinus5MaxPlus5);
 }
 
 TEST(SubgraphQuantizerTest, VerifyConvQuantizationWithUnitScale) {
   ASSERT_TRUE(g_test_model_dir);
   ASSERT_FALSE(g_test_model_dir->empty());
-  auto test_model = ReadTestModel1();
+  auto test_model = ReadConvModel1();
   ASSERT_TRUE(test_model);
   auto readonly_model = test_model->GetModel();
   ASSERT_TRUE(readonly_model);
@@ -145,7 +156,7 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantizationWithUnitScale) {
 TEST(SubgraphQuantizerTest, VerifyConvQuantization) {
   ASSERT_TRUE(g_test_model_dir);
   ASSERT_FALSE(g_test_model_dir->empty());
-  auto test_model = ReadTestModel2();
+  auto test_model = ReadConvModel2();
   ASSERT_TRUE(test_model);
   auto readonly_model = test_model->GetModel();
   ASSERT_TRUE(readonly_model);
@@ -237,6 +248,129 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantization) {
   }
 }
 
+void VerifyAsymmetricQuantizationScale(
+    const QuantizationParameters& float_quant_params,
+    const QuantizationParametersT& quantized_quant_params) {
+  const float eps = 1e-7;
+  ASSERT_EQ(float_quant_params.min()->size(), 1);
+  ASSERT_EQ(float_quant_params.max()->size(), 1);
+  float float_min = std::min(0.f, float_quant_params.min()->Get(0));
+  float float_max = std::max(0.f, float_quant_params.max()->Get(0));
+
+  ASSERT_EQ(quantized_quant_params.scale.size(), 1);
+  ASSERT_EQ(quantized_quant_params.zero_point.size(), 1);
+
+  float scale = (float_max - float_min) / 255;
+  EXPECT_NEAR(scale, quantized_quant_params.scale[0], eps);
+}
+
+TEST(SubgraphQuantizerTest, VerifySoftmaxQuantization) {
+  ASSERT_TRUE(g_test_model_dir);
+  ASSERT_FALSE(g_test_model_dir->empty());
+  auto test_model = ReadSoftmaxModel();
+  ASSERT_TRUE(test_model);
+  auto readonly_model = test_model->GetModel();
+  ASSERT_TRUE(readonly_model);
+  ASSERT_TRUE(readonly_model->subgraphs());
+  ASSERT_GE(readonly_model->subgraphs()->size(), 1);
+  tflite::ModelT model;
+  readonly_model->UnPackTo(&model);
+  auto subgraph = model.subgraphs[0].get();
+  FailOnErrorReporter error_reporter;
+  SubgraphQuantizer quantizer(&model, subgraph, &error_reporter);
+  auto status = quantizer.QuantizeOperator(0);
+  ASSERT_EQ(kTfLiteOk, status);
+
+  auto op = subgraph->operators[0].get();
+  // Model has a single softmax op.
+  ASSERT_EQ(op->opcode_index, 0);
+  ASSERT_EQ(model.operator_codes[0].get()->builtin_code,
+            BuiltinOperator_SOFTMAX);
+
+  ASSERT_EQ(op->inputs.size(), 1);
+  ASSERT_EQ(op->outputs.size(), 1);
+  auto float_graph = readonly_model->subgraphs()->Get(0);
+
+  ASSERT_EQ(float_graph->tensors()->Get(op->inputs[0])->type(),
+            TensorType_FLOAT32);
+  ASSERT_EQ(float_graph->tensors()->Get(op->outputs[0])->type(),
+            TensorType_FLOAT32);
+
+  EXPECT_EQ(subgraph->tensors[op->inputs[0]].get()->type, TensorType_INT8);
+  EXPECT_EQ(subgraph->tensors[op->outputs[0]].get()->type, TensorType_INT8);
+
+  auto float_input_quant_params =
+      float_graph->tensors()->Get(op->inputs[0])->quantization();
+  auto input_quant_params =
+      subgraph->tensors[op->inputs[0]]->quantization.get();
+  VerifyAsymmetricQuantizationScale(*float_input_quant_params,
+                                    *input_quant_params);
+
+  auto float_output_quant_params =
+      float_graph->tensors()->Get(op->outputs[0])->quantization();
+  auto output_quant_params =
+      subgraph->tensors[op->outputs[0]]->quantization.get();
+  VerifyAsymmetricQuantizationScale(*float_output_quant_params,
+                                    *output_quant_params);
+}
+
+TEST(SubgraphQuantizerTest, VerifyAvgPoolQuantization) {
+  ASSERT_TRUE(g_test_model_dir);
+  ASSERT_FALSE(g_test_model_dir->empty());
+  auto test_model = ReadAvgPoolModel();
+  ASSERT_TRUE(test_model);
+  auto readonly_model = test_model->GetModel();
+  ASSERT_TRUE(readonly_model);
+  ASSERT_TRUE(readonly_model->subgraphs());
+  ASSERT_GE(readonly_model->subgraphs()->size(), 1);
+  tflite::ModelT model;
+  readonly_model->UnPackTo(&model);
+  auto subgraph = model.subgraphs[0].get();
+  FailOnErrorReporter error_reporter;
+  SubgraphQuantizer quantizer(&model, subgraph, &error_reporter);
+  auto status = quantizer.QuantizeOperator(0);
+  ASSERT_EQ(kTfLiteOk, status);
+
+  auto op = subgraph->operators[0].get();
+  // Model has a single AveragePool op.
+  ASSERT_EQ(op->opcode_index, 0);
+  ASSERT_EQ(model.operator_codes[0].get()->builtin_code,
+            BuiltinOperator_AVERAGE_POOL_2D);
+
+  ASSERT_EQ(op->inputs.size(), 1);
+  ASSERT_EQ(op->outputs.size(), 1);
+
+  auto float_graph = readonly_model->subgraphs()->Get(0);
+  ASSERT_EQ(float_graph->tensors()->Get(op->inputs[0])->type(),
+            TensorType_FLOAT32);
+  ASSERT_EQ(float_graph->tensors()->Get(op->outputs[0])->type(),
+            TensorType_FLOAT32);
+
+  EXPECT_EQ(subgraph->tensors[op->inputs[0]].get()->type, TensorType_INT8);
+  EXPECT_EQ(subgraph->tensors[op->outputs[0]].get()->type, TensorType_INT8);
+
+  auto float_input_quant_params =
+      float_graph->tensors()->Get(op->inputs[0])->quantization();
+  auto input_quant_params =
+      subgraph->tensors[op->inputs[0]]->quantization.get();
+  VerifyAsymmetricQuantizationScale(*float_input_quant_params,
+                                    *input_quant_params);
+
+  auto float_output_quant_params =
+      float_graph->tensors()->Get(op->outputs[0])->quantization();
+  auto output_quant_params =
+      subgraph->tensors[op->outputs[0]]->quantization.get();
+  ASSERT_EQ(float_output_quant_params->min()->size(), 1);
+  ASSERT_EQ(float_output_quant_params->max()->size(), 1);
+  ASSERT_EQ(output_quant_params->min.size(), 1);
+  ASSERT_EQ(output_quant_params->max.size(), 1);
+
+  // Make sure the input min/maxes are propagated to outputs.
+  EXPECT_EQ(input_quant_params->min[0], output_quant_params->min[0]);
+  EXPECT_EQ(input_quant_params->max[0], output_quant_params->max[0]);
+  EXPECT_EQ(input_quant_params->scale[0], output_quant_params->scale[0]);
+}
+
 }  // namespace
 }  // namespace internal
 }  // namespace optimize
diff --git a/tensorflow/lite/tools/optimize/test_util.cc b/tensorflow/lite/tools/optimize/test_util.cc
index 83b9f0dcd3..3506142901 100644
--- a/tensorflow/lite/tools/optimize/test_util.cc
+++ b/tensorflow/lite/tools/optimize/test_util.cc
@@ -19,12 +19,18 @@ limitations under the License.
 namespace tflite {
 namespace optimize {
 namespace internal {
-const char* kModelWithMinus128Plus127Weights =
+const char* kConvModelWithMinus128Plus127Weights =
     "single_conv_weights_min_minus_127_max_plus_127.tflite";
 
-const char* kModelWith0Plus10Weights =
+const char* kConvModelWith0Plus10Weights =
     "single_conv_weights_min_0_max_plus_10.tflite";
 
+const char* kSingleSoftmaxModelMinMinus5MaxPlus5 =
+    "single_softmax_min_minus_5_max_plus_5.tflite";
+
+const char* kSingleAvgPoolModelMinMinus5MaxPlus5 =
+    "single_avg_pool_min_minus_5_max_plus_5.tflite";
+
 int FailOnErrorReporter::Report(const char* format, va_list args) {
   char buf[1024];
   vsnprintf(buf, sizeof(buf), format, args);
diff --git a/tensorflow/lite/tools/optimize/test_util.h b/tensorflow/lite/tools/optimize/test_util.h
index 26da41e9cc..1981068acc 100644
--- a/tensorflow/lite/tools/optimize/test_util.h
+++ b/tensorflow/lite/tools/optimize/test_util.h
@@ -26,13 +26,21 @@ namespace internal {
 // channel has at least one weight as -127 and one weight as 127.
 // The activations are all in range: [-128, 127]
 // This means all bias computations should result in 1.0 scale.
-extern const char* kModelWithMinus128Plus127Weights;
+extern const char* kConvModelWithMinus128Plus127Weights;
 
 // Test model with single convolution where all weights are integers between
 // [0, 10] weights are randomly distributed. It is not guaranteed that min max
 // for weights are going to appear in each channel.
 // Activations have min = 0, max = 10.
-extern const char* kModelWith0Plus10Weights;
+extern const char* kConvModelWith0Plus10Weights;
+
+// A floating point model with a single softmax. The input tensor has min
+// and max in range [-5, 5], not necessarily -5 or +5.
+extern const char* kSingleSoftmaxModelMinMinus5MaxPlus5;
+
+// A floating point model with a single average pool. The input tensor has min
+// and max in range [-5, 5], not necessarily -5 or +5.
+extern const char* kSingleAvgPoolModelMinMinus5MaxPlus5;
 
 // An error reporter that fails on testing.
 class FailOnErrorReporter : public ErrorReporter {
diff --git a/tensorflow/lite/tools/optimize/testdata/README.md b/tensorflow/lite/tools/optimize/testdata/README.md
new file mode 100644
index 0000000000..b6b331b828
--- /dev/null
+++ b/tensorflow/lite/tools/optimize/testdata/README.md
@@ -0,0 +1,23 @@
+# Test models for testing quantization
+
+This directory contains test models for testing quantization.
+
+## Models
+
+* `single_conv_weights_min_0_max_plus_10.tflite` \
+   A floating point model with single convolution where all weights are
+   integers between [0, 10] weights are randomly distributed. It is not
+   guaranteed that min max for weights are going to appear in each channel.
+   All activations have min maxes and activations are in range [0,10].
+* `single_conv_weights_min_minus_127_max_plus_127.tflite` \
+   A floating point model with a single convolution where weights of the model
+   are all integers that lie in range[-127, 127]. The weights have been put in
+   such a way that each channel has at least one weight as -127 and one weight
+   as 127. The activations are all in range: [-128, 127].
+   This means all bias computations should result in 1.0 scale.
+* `single_softmax_min_minus_5_max_5.tflite` \
+   A floating point model with a single softmax. The input tensor has min
+   and max in range [-5, 5], not necessarily -5 or +5.
+* `single_avg_pool_input_min_minus_5_max_5.tflite` \
+   A floating point model with a single average pool. The input tensor has min
+   and max in range [-5, 5], not necessarily -5 or +5.
-- 
GitLab


From 1cbfa21e2a3f18b0396369e7194e3dbb4bedb08c Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Tue, 15 Jan 2019 10:39:11 -0800
Subject: [PATCH 0726/2345] Support Python 2 slicing on ListWrapper.

"Deprecated since version 2.0: Support slice objects as parameters to the
__getitem__() method. (However, built-in types in CPython currently still
implement __getslice__(). Therefore, you have to override it in derived classes
when implementing slicing.)"

PiperOrigin-RevId: 229397602
---
 .../python/training/checkpointable/data_structures.py       | 3 +++
 .../python/training/checkpointable/data_structures_test.py  | 6 ++++++
 2 files changed, 9 insertions(+)

diff --git a/tensorflow/python/training/checkpointable/data_structures.py b/tensorflow/python/training/checkpointable/data_structures.py
index 759709aa3d..d1e617ed46 100644
--- a/tensorflow/python/training/checkpointable/data_structures.py
+++ b/tensorflow/python/training/checkpointable/data_structures.py
@@ -304,6 +304,9 @@ class List(CheckpointableDataStructure, collections.Sequence):
   def __getitem__(self, key):
     return self._storage[key]
 
+  def __getslice__(self, i, j):
+    return self._storage[slice(i, j)]
+
   def __len__(self):
     return len(self._storage)
 
diff --git a/tensorflow/python/training/checkpointable/data_structures_test.py b/tensorflow/python/training/checkpointable/data_structures_test.py
index bcec6e0100..79191f3448 100644
--- a/tensorflow/python/training/checkpointable/data_structures_test.py
+++ b/tensorflow/python/training/checkpointable/data_structures_test.py
@@ -271,6 +271,12 @@ class ListTests(test.TestCase):
     with self.assertRaises(TypeError):
       has_sequences.add(data_structures._ListWrapper([]))
 
+  def testSlicing(self):
+    l = data_structures._ListWrapper([1, 2, 3, 4])
+    self.assertEqual(l[1:], [2, 3, 4])
+    self.assertEqual(l[1:-1], [2, 3])
+    self.assertEqual(l[:-1], [1, 2, 3])
+
 
 class HasMapping(training.Model):
 
-- 
GitLab


From 2e5ee61e2e0d876f91a66a1fe6279105b14d7534 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 15 Jan 2019 10:45:54 -0800
Subject: [PATCH 0727/2345] Delete deprecated and unused methods from
 distribute_lib. These methods all had the
 `@doc_controls.do_not_generate_docs` decorator since we knew they were going
 away as soon as references to them were updated to point to the official APIs
 decided in the accepted RFC:
 https://github.com/tensorflow/community/blob/master/rfcs/20181016-replicator.md

PiperOrigin-RevId: 229398974
---
 .../python/distribute/distribute_lib.py       | 161 +-----------------
 .../python/distribute/mirrored_strategy.py    |   4 +-
 ...orflow.distribute.-mirrored-strategy.pbtxt |  56 ------
 ...nsorflow.distribute.-replica-context.pbtxt |   4 -
 .../v1/tensorflow.distribute.-strategy.pbtxt  |  56 ------
 ...orflow.distribute.-mirrored-strategy.pbtxt |  56 ------
 ...nsorflow.distribute.-replica-context.pbtxt |   4 -
 .../v2/tensorflow.distribute.-strategy.pbtxt  |  56 ------
 8 files changed, 4 insertions(+), 393 deletions(-)

diff --git a/tensorflow/python/distribute/distribute_lib.py b/tensorflow/python/distribute/distribute_lib.py
index 421b2a6520..f112787ba2 100644
--- a/tensorflow/python/distribute/distribute_lib.py
+++ b/tensorflow/python/distribute/distribute_lib.py
@@ -505,33 +505,6 @@ class DistributionStrategy(object):
     """DEPRECATED: use extended.broadcast_to() instead."""
     return self._extended.broadcast_to(tensor, destinations)
 
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def run_steps_on_dataset(self, fn, iterator, iterations=1,
-                           initial_loop_values=None):
-    """DEPRECATED: use extended.experimental_run_steps_on_iterator() instead."""
-    return self._extended.experimental_run_steps_on_iterator(
-        fn, iterator, iterations, initial_loop_values)
-
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def call_for_each_replica(self, fn, *args, **kwargs):
-    """DEPRECATED: use extended.call_for_each_replica() instead."""
-    # Handle old *args, **kwargs, and new args=(...), kwargs={...}, to
-    # allow transition.
-    a = kwargs.pop("args", None)
-    if a is not None:
-      if args:
-        raise ValueError(
-            "Can't pass *args and args=... to call_for_each_replica")
-      args = a
-    k = kwargs.pop("kwargs", None)
-    if k is not None:
-      if kwargs:
-        raise ValueError(
-            "Can't pass **kwargs and kwargs=... to call_for_each_replica")
-      kwargs = k
-    kwargs.pop("run_concurrently", None)  # Ignore old option.
-    return self._extended.call_for_each_replica(fn, args, kwargs)
-
   def reduce(self, reduce_op, value):
     """Reduce `value` across replicas.
 
@@ -546,58 +519,6 @@ class DistributionStrategy(object):
     _require_cross_replica_context_extended(self._extended)
     return self._extended._reduce(reduce_op, value)  # pylint: disable=protected-access
 
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def batch_reduce(self, aggregation, value_destination_pairs):
-    """DEPRECATED: use extended.batch_reduce_to() instead."""
-    return self._extended.batch_reduce_to(aggregation, value_destination_pairs)
-
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def update(self, var, fn, *args, **kwargs):
-    """DEPRECATED: use extended.update() instead."""
-    group = kwargs.pop("group", True)
-    # We temporarily support "grouped" in addition to "group" for backward-
-    # compatibility.
-    group = kwargs.pop("grouped", True) and group
-    # Handle old *args, **kwargs, and new args=(...), kwargs={...}, to
-    # allow transition.
-    a = kwargs.pop("args", None)
-    if a is not None:
-      if args:
-        raise ValueError(
-            "Can't pass *args and args=... to update")
-      args = a
-    k = kwargs.pop("kwargs", None)
-    if k is not None:
-      if kwargs:
-        raise ValueError(
-            "Can't pass **kwargs and kwargs=... to update")
-      kwargs = k
-    return self._extended.update(var, fn, args, kwargs, group)
-
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def update_non_slot(self, colocate_with, fn, *args, **kwargs):
-    """DEPRECATED: use extended.update_non_slot() instead."""
-    group = kwargs.pop("group", True)
-    # We temporarily support "grouped" in addition to "group" for backward-
-    # compatibility.
-    group = kwargs.pop("grouped", True) and group
-    # Handle old *args, **kwargs, and new args=(...), kwargs={...}, to
-    # allow transition.
-    a = kwargs.pop("args", None)
-    if a is not None:
-      if args:
-        raise ValueError(
-            "Can't pass *args and args=... to update_non_slot")
-      args = a
-    k = kwargs.pop("kwargs", None)
-    if k is not None:
-      if kwargs:
-        raise ValueError(
-            "Can't pass **kwargs and kwargs=... to update_non_slot")
-      kwargs = k
-    return self._extended.update_non_slot(
-        colocate_with, fn, args, kwargs, group)
-
   @doc_controls.do_not_generate_docs  # DEPRECATED, -> `DistributedValues`
   def unwrap(self, value):
     """Returns the list of all per-replica values contained in `value`.
@@ -612,50 +533,16 @@ class DistributionStrategy(object):
     """
     return self._extended._unwrap(value)  # pylint: disable=protected-access
 
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def value_container(self, value):
-    """DEPRECATED: use extended.value_container() instead."""
-    return self._extended.value_container(value)
-
   @doc_controls.do_not_generate_docs  # DEPRECATED, -> `DistributedValues`
   def group(self, value, name=None):
     """Shortcut for `tf.group(self.unwrap(value))`."""
     return self._extended._group(value, name)  # pylint: disable=protected-access
 
-  @property
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def require_static_shapes(self):
-    """DEPRECATED: use extended.require_static_shapes instead."""
-    return self._extended.experimental_require_static_shapes
-
   @property
   def num_replicas_in_sync(self):
     """Returns number of replicas over which gradients are aggregated."""
     return self._extended._num_replicas_in_sync  # pylint: disable=protected-access
 
-  @property
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def worker_devices(self):
-    """DEPRECATED: use extended.worker_devices instead."""
-    return self._extended.worker_devices
-
-  @property
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def parameter_devices(self):
-    """DEPRECATED: use extended.parameter_devices instead."""
-    return self._extended.parameter_devices
-
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def non_slot_devices(self, var_list):
-    """DEPRECATED: use extended.non_slot_devices instead."""
-    return self._extended.non_slot_devices(var_list)
-
-  @property
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def between_graph(self):
-    """DEPRECATED: use extended.experimental_between_graph instead."""
-    return self._extended.experimental_between_graph
-
   @doc_controls.do_not_generate_docs  # DEPRECATED, being replaced by a new API.
   def configure(self,
                 session_config=None,
@@ -691,24 +578,6 @@ class DistributionStrategy(object):
     """
     return self._extended._update_config_proto(config_proto)  # pylint: disable=protected-access
 
-  @property
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def should_init(self):
-    """DEPRECATED: use extended.should_init instead."""
-    return self._extended.experimental_should_init
-
-  @property
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def should_checkpoint(self):
-    """DEPRECATED: use extended.should_checkpoint instead."""
-    return self._extended.should_checkpoint
-
-  @property
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def should_save_summary(self):
-    """DEPRECATED: use extended.should_save_summary instead."""
-    return self._extended.should_save_summary
-
   def __deepcopy__(self, memo):
     # First do a regular deepcopy of `self`.
     cls = self.__class__
@@ -1241,9 +1110,6 @@ class DistributionStrategyExtended(object):
 
     Args:
       reduce_op: Reduction type, an instance of `tf.distribute.ReduceOp` enum.
-        DEPRECATED but still accepted values:
-        `tf.VariableAggregation.SUM`,
-        `tf.VariableAggregation.MEAN`,
       value: A per-replica value with one value per replica.
       destinations: A mirrored variable, a per-replica tensor, or a device
         string. The return value will be copied to all destination devices (or
@@ -1256,14 +1122,7 @@ class DistributionStrategyExtended(object):
     # TODO(josh11b): More docstring
     _require_cross_replica_context_extended(self)
     assert not isinstance(destinations, (list, tuple))
-
-    # TODO(priyag): Remove this when all callers have been updated.
-    if isinstance(reduce_op, variable_scope.VariableAggregation):
-      assert reduce_op in (
-          variable_scope.VariableAggregation.SUM,
-          variable_scope.VariableAggregation.MEAN,
-      )
-      reduce_op = reduce_util.ReduceOp.from_variable_aggregation(reduce_op)
+    assert not isinstance(reduce_op, variable_scope.VariableAggregation)
     assert (reduce_op == reduce_util.ReduceOp.SUM or
             reduce_op == reduce_util.ReduceOp.MEAN)
     return self._reduce_to(reduce_op, value, destinations)
@@ -1276,9 +1135,6 @@ class DistributionStrategyExtended(object):
 
     Args:
       reduce_op: Reduction type, an instance of `tf.distribute.ReduceOp` enum.
-        DEPRECATED but still accepted values:
-        `tf.VariableAggregation.SUM`,
-        `tf.VariableAggregation.MEAN`,
       value_destination_pairs: A sequence of (value, destinations)
         pairs. See `reduce_to()` for a description.
 
@@ -1287,14 +1143,7 @@ class DistributionStrategyExtended(object):
     """
     # TODO(josh11b): More docstring
     _require_cross_replica_context_extended(self)
-
-    # TODO(priyag): Remove this when all callers have been updated.
-    if isinstance(reduce_op, variable_scope.VariableAggregation):
-      assert reduce_op in [
-          variable_scope.VariableAggregation.SUM,
-          variable_scope.VariableAggregation.MEAN,
-      ]
-      reduce_op = reduce_util.ReduceOp.from_variable_aggregation(reduce_op)
+    assert not isinstance(reduce_op, variable_scope.VariableAggregation)
     return self._batch_reduce_to(reduce_op, value_destination_pairs)
 
   def _batch_reduce_to(self, reduce_op, value_destination_pairs):
@@ -1565,12 +1414,6 @@ class ReplicaContext(object):
     require_replica_context(self)
     return self._replica_id_in_sync_group
 
-  @property
-  @doc_controls.do_not_generate_docs  # DEPRECATED, use `strategy`
-  def distribution_strategy(self):
-    """DEPRECATED: use `self.strategy` instead."""
-    return self._strategy
-
   @property
   def strategy(self):
     """The current `tf.distribute.Strategy` object."""
diff --git a/tensorflow/python/distribute/mirrored_strategy.py b/tensorflow/python/distribute/mirrored_strategy.py
index 1ed7eef1ed..f279c3ba1b 100644
--- a/tensorflow/python/distribute/mirrored_strategy.py
+++ b/tensorflow/python/distribute/mirrored_strategy.py
@@ -880,13 +880,13 @@ class _MirroredReplicaThread(threading.Thread):
 
 
 class MirroredReplicaContext(distribute_lib.ReplicaContext):
-  """ReplicaContext used in MirroredStrategy.call_for_each_replica().
+  """ReplicaContext used in MirroredStrategy.extended.call_for_each_replica().
 
   Opened in `_MirroredReplicaThread`, to allow the user to invoke
   `MirroredStrategy`'s specific implementation of `merge_call()`,
   which works by delegating the function and its arguments to
   the main thread (the one that invoked
-  `MirroredStrategy.call_for_each_replica()`).
+  `MirroredStrategy.extended.call_for_each_replica()`).
   """
 
   def _merge_call(self, fn, args, kwargs):
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
index 863ee9c210..c554df489a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
@@ -3,10 +3,6 @@ tf_class {
   is_instance: "<class \'tensorflow.python.distribute.mirrored_strategy.MirroredStrategy\'>"
   is_instance: "<class \'tensorflow.python.distribute.distribute_lib.DistributionStrategy\'>"
   is_instance: "<type \'object\'>"
-  member {
-    name: "between_graph"
-    mtype: "<type \'property\'>"
-  }
   member {
     name: "extended"
     mtype: "<type \'property\'>"
@@ -15,46 +11,14 @@ tf_class {
     name: "num_replicas_in_sync"
     mtype: "<type \'property\'>"
   }
-  member {
-    name: "parameter_devices"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "require_static_shapes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "should_checkpoint"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "should_init"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "should_save_summary"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "worker_devices"
-    mtype: "<type \'property\'>"
-  }
   member_method {
     name: "__init__"
     argspec: "args=[\'self\', \'devices\', \'cross_device_ops\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
-  member_method {
-    name: "batch_reduce"
-    argspec: "args=[\'self\', \'aggregation\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "broadcast"
     argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "call_for_each_replica"
-    argspec: "args=[\'self\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
-  }
   member_method {
     name: "colocate_vars_with"
     argspec: "args=[\'self\', \'colocate_with_variable\'], varargs=None, keywords=None, defaults=None"
@@ -87,18 +51,10 @@ tf_class {
     name: "make_input_fn_iterator"
     argspec: "args=[\'self\', \'input_fn\', \'replication_mode\'], varargs=None, keywords=None, defaults=[\'InputReplicationMode.PER_WORKER\'], "
   }
-  member_method {
-    name: "non_slot_devices"
-    argspec: "args=[\'self\', \'var_list\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "reduce"
     argspec: "args=[\'self\', \'reduce_op\', \'value\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "run_steps_on_dataset"
-    argspec: "args=[\'self\', \'fn\', \'iterator\', \'iterations\', \'initial_loop_values\'], varargs=None, keywords=None, defaults=[\'1\', \'None\'], "
-  }
   member_method {
     name: "scope"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
@@ -107,20 +63,8 @@ tf_class {
     name: "unwrap"
     argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "update"
-    argspec: "args=[\'self\', \'var\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
-  }
   member_method {
     name: "update_config_proto"
     argspec: "args=[\'self\', \'config_proto\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "update_non_slot"
-    argspec: "args=[\'self\', \'colocate_with\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "value_container"
-    argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
-  }
 }
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-replica-context.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-replica-context.pbtxt
index 22f8160c96..c3b7991175 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-replica-context.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-replica-context.pbtxt
@@ -6,10 +6,6 @@ tf_class {
     name: "devices"
     mtype: "<type \'property\'>"
   }
-  member {
-    name: "distribution_strategy"
-    mtype: "<type \'property\'>"
-  }
   member {
     name: "num_replicas_in_sync"
     mtype: "<type \'property\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
index 1c06f7873e..cce83be3d0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
@@ -2,10 +2,6 @@ path: "tensorflow.distribute.Strategy"
 tf_class {
   is_instance: "<class \'tensorflow.python.distribute.distribute_lib.DistributionStrategy\'>"
   is_instance: "<type \'object\'>"
-  member {
-    name: "between_graph"
-    mtype: "<type \'property\'>"
-  }
   member {
     name: "extended"
     mtype: "<type \'property\'>"
@@ -14,46 +10,14 @@ tf_class {
     name: "num_replicas_in_sync"
     mtype: "<type \'property\'>"
   }
-  member {
-    name: "parameter_devices"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "require_static_shapes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "should_checkpoint"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "should_init"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "should_save_summary"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "worker_devices"
-    mtype: "<type \'property\'>"
-  }
   member_method {
     name: "__init__"
     argspec: "args=[\'self\', \'extended\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "batch_reduce"
-    argspec: "args=[\'self\', \'aggregation\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "broadcast"
     argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "call_for_each_replica"
-    argspec: "args=[\'self\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
-  }
   member_method {
     name: "colocate_vars_with"
     argspec: "args=[\'self\', \'colocate_with_variable\'], varargs=None, keywords=None, defaults=None"
@@ -86,18 +50,10 @@ tf_class {
     name: "make_input_fn_iterator"
     argspec: "args=[\'self\', \'input_fn\', \'replication_mode\'], varargs=None, keywords=None, defaults=[\'InputReplicationMode.PER_WORKER\'], "
   }
-  member_method {
-    name: "non_slot_devices"
-    argspec: "args=[\'self\', \'var_list\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "reduce"
     argspec: "args=[\'self\', \'reduce_op\', \'value\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "run_steps_on_dataset"
-    argspec: "args=[\'self\', \'fn\', \'iterator\', \'iterations\', \'initial_loop_values\'], varargs=None, keywords=None, defaults=[\'1\', \'None\'], "
-  }
   member_method {
     name: "scope"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
@@ -106,20 +62,8 @@ tf_class {
     name: "unwrap"
     argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "update"
-    argspec: "args=[\'self\', \'var\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
-  }
   member_method {
     name: "update_config_proto"
     argspec: "args=[\'self\', \'config_proto\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "update_non_slot"
-    argspec: "args=[\'self\', \'colocate_with\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "value_container"
-    argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
-  }
 }
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
index 863ee9c210..c554df489a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
@@ -3,10 +3,6 @@ tf_class {
   is_instance: "<class \'tensorflow.python.distribute.mirrored_strategy.MirroredStrategy\'>"
   is_instance: "<class \'tensorflow.python.distribute.distribute_lib.DistributionStrategy\'>"
   is_instance: "<type \'object\'>"
-  member {
-    name: "between_graph"
-    mtype: "<type \'property\'>"
-  }
   member {
     name: "extended"
     mtype: "<type \'property\'>"
@@ -15,46 +11,14 @@ tf_class {
     name: "num_replicas_in_sync"
     mtype: "<type \'property\'>"
   }
-  member {
-    name: "parameter_devices"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "require_static_shapes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "should_checkpoint"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "should_init"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "should_save_summary"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "worker_devices"
-    mtype: "<type \'property\'>"
-  }
   member_method {
     name: "__init__"
     argspec: "args=[\'self\', \'devices\', \'cross_device_ops\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
-  member_method {
-    name: "batch_reduce"
-    argspec: "args=[\'self\', \'aggregation\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "broadcast"
     argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "call_for_each_replica"
-    argspec: "args=[\'self\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
-  }
   member_method {
     name: "colocate_vars_with"
     argspec: "args=[\'self\', \'colocate_with_variable\'], varargs=None, keywords=None, defaults=None"
@@ -87,18 +51,10 @@ tf_class {
     name: "make_input_fn_iterator"
     argspec: "args=[\'self\', \'input_fn\', \'replication_mode\'], varargs=None, keywords=None, defaults=[\'InputReplicationMode.PER_WORKER\'], "
   }
-  member_method {
-    name: "non_slot_devices"
-    argspec: "args=[\'self\', \'var_list\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "reduce"
     argspec: "args=[\'self\', \'reduce_op\', \'value\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "run_steps_on_dataset"
-    argspec: "args=[\'self\', \'fn\', \'iterator\', \'iterations\', \'initial_loop_values\'], varargs=None, keywords=None, defaults=[\'1\', \'None\'], "
-  }
   member_method {
     name: "scope"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
@@ -107,20 +63,8 @@ tf_class {
     name: "unwrap"
     argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "update"
-    argspec: "args=[\'self\', \'var\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
-  }
   member_method {
     name: "update_config_proto"
     argspec: "args=[\'self\', \'config_proto\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "update_non_slot"
-    argspec: "args=[\'self\', \'colocate_with\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "value_container"
-    argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
-  }
 }
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-replica-context.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-replica-context.pbtxt
index 22f8160c96..c3b7991175 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-replica-context.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-replica-context.pbtxt
@@ -6,10 +6,6 @@ tf_class {
     name: "devices"
     mtype: "<type \'property\'>"
   }
-  member {
-    name: "distribution_strategy"
-    mtype: "<type \'property\'>"
-  }
   member {
     name: "num_replicas_in_sync"
     mtype: "<type \'property\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
index 1c06f7873e..cce83be3d0 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
@@ -2,10 +2,6 @@ path: "tensorflow.distribute.Strategy"
 tf_class {
   is_instance: "<class \'tensorflow.python.distribute.distribute_lib.DistributionStrategy\'>"
   is_instance: "<type \'object\'>"
-  member {
-    name: "between_graph"
-    mtype: "<type \'property\'>"
-  }
   member {
     name: "extended"
     mtype: "<type \'property\'>"
@@ -14,46 +10,14 @@ tf_class {
     name: "num_replicas_in_sync"
     mtype: "<type \'property\'>"
   }
-  member {
-    name: "parameter_devices"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "require_static_shapes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "should_checkpoint"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "should_init"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "should_save_summary"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "worker_devices"
-    mtype: "<type \'property\'>"
-  }
   member_method {
     name: "__init__"
     argspec: "args=[\'self\', \'extended\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "batch_reduce"
-    argspec: "args=[\'self\', \'aggregation\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "broadcast"
     argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "call_for_each_replica"
-    argspec: "args=[\'self\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
-  }
   member_method {
     name: "colocate_vars_with"
     argspec: "args=[\'self\', \'colocate_with_variable\'], varargs=None, keywords=None, defaults=None"
@@ -86,18 +50,10 @@ tf_class {
     name: "make_input_fn_iterator"
     argspec: "args=[\'self\', \'input_fn\', \'replication_mode\'], varargs=None, keywords=None, defaults=[\'InputReplicationMode.PER_WORKER\'], "
   }
-  member_method {
-    name: "non_slot_devices"
-    argspec: "args=[\'self\', \'var_list\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "reduce"
     argspec: "args=[\'self\', \'reduce_op\', \'value\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "run_steps_on_dataset"
-    argspec: "args=[\'self\', \'fn\', \'iterator\', \'iterations\', \'initial_loop_values\'], varargs=None, keywords=None, defaults=[\'1\', \'None\'], "
-  }
   member_method {
     name: "scope"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
@@ -106,20 +62,8 @@ tf_class {
     name: "unwrap"
     argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "update"
-    argspec: "args=[\'self\', \'var\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
-  }
   member_method {
     name: "update_config_proto"
     argspec: "args=[\'self\', \'config_proto\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "update_non_slot"
-    argspec: "args=[\'self\', \'colocate_with\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "value_container"
-    argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
-  }
 }
-- 
GitLab


From 0bdf7e66fcf04707e3094b1ac4648c06bceb2db5 Mon Sep 17 00:00:00 2001
From: Yunlu Li <yunluli@google.com>
Date: Tue, 15 Jan 2019 10:47:16 -0800
Subject: [PATCH 0728/2345] Update test data multi_add.json.

PiperOrigin-RevId: 229399229
---
 tensorflow/lite/testdata/multi_add.json | 119 ++++++++++++++++++++----
 1 file changed, 102 insertions(+), 17 deletions(-)

diff --git a/tensorflow/lite/testdata/multi_add.json b/tensorflow/lite/testdata/multi_add.json
index 97b931dba8..ae559255a8 100644
--- a/tensorflow/lite/testdata/multi_add.json
+++ b/tensorflow/lite/testdata/multi_add.json
@@ -1,46 +1,131 @@
 {
-  "version": 1,
+  "version": 3,
   "operator_codes": [
     {
-      "builtin_code": "ADD"
     }
   ],
   "subgraphs": [
     {
       "tensors": [
-        { "shape": [ 1, 8, 8, 3 ], "name": "a" },
-        { "shape": [ 1, 8, 8, 3 ], "name": "b" },
-        { "shape": [ 1, 8, 8, 3 ], "name": "c" },
-        { "shape": [ 1, 8, 8, 3 ], "name": "d" },
-        { "shape": [ 1, 8, 8, 3 ], "name": "i" },
-        { "shape": [ 1, 8, 8, 3 ], "name": "x" },
-        { "shape": [ 1, 8, 8, 3 ], "name": "y" }
+        {
+          "shape": [
+            1,
+            8,
+            8,
+            3
+          ],
+          "name": "a"
+        },
+        {
+          "shape": [
+            1,
+            8,
+            8,
+            3
+          ],
+          "name": "b"
+        },
+        {
+          "shape": [
+            1,
+            8,
+            8,
+            3
+          ],
+          "name": "c"
+        },
+        {
+          "shape": [
+            1,
+            8,
+            8,
+            3
+          ],
+          "name": "d"
+        },
+        {
+          "shape": [
+            1,
+            8,
+            8,
+            3
+          ],
+          "name": "i"
+        },
+        {
+          "shape": [
+            1,
+            8,
+            8,
+            3
+          ],
+          "name": "x"
+        },
+        {
+          "shape": [
+            1,
+            8,
+            8,
+            3
+          ],
+          "name": "y"
+        }
+      ],
+      "inputs": [
+        0,
+        1,
+        2,
+        3
+      ],
+      "outputs": [
+        5,
+        6
       ],
-      "inputs": [ 0, 1, 2, 3 ],
-      "outputs": [ 5, 6 ],
       "operators": [
         {
-          "inputs": [ 1, 2 ],
-          "outputs": [ 4 ],
+          "inputs": [
+            1,
+            2
+          ],
+          "outputs": [
+            4
+          ],
           "builtin_options_type": "AddOptions",
           "builtin_options": {
           }
         },
         {
-          "inputs": [ 0, 4 ],
-          "outputs": [ 5 ],
+          "inputs": [
+            0,
+            4
+          ],
+          "outputs": [
+            5
+          ],
           "builtin_options_type": "AddOptions",
           "builtin_options": {
           }
         },
         {
-          "inputs": [ 3, 4 ],
-          "outputs": [ 6 ],
+          "inputs": [
+            3,
+            4
+          ],
+          "outputs": [
+            6
+          ],
           "builtin_options_type": "AddOptions",
           "builtin_options": {
           }
         }
       ]
     }
+  ],
+  "buffers": [
+    {
+      "data": [
+
+      ]
+    }
   ]
 }
-- 
GitLab


From 158af766b665509ff5e47a092f64279848549d5c Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Tue, 15 Jan 2019 10:47:23 -0800
Subject: [PATCH 0729/2345] Use the correct nest in autograph

PiperOrigin-RevId: 229399245
---
 tensorflow/python/autograph/impl/api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/autograph/impl/api.py b/tensorflow/python/autograph/impl/api.py
index b1c16b1169..6633441e9f 100644
--- a/tensorflow/python/autograph/impl/api.py
+++ b/tensorflow/python/autograph/impl/api.py
@@ -35,7 +35,7 @@ from tensorflow.python.autograph.operators import py_builtins
 from tensorflow.python.autograph.pyct import compiler
 from tensorflow.python.autograph.pyct import inspect_utils
 from tensorflow.python.autograph.utils import py_func
-from tensorflow.python.data.util import nest
+from tensorflow.python.util import nest
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import tf_decorator
-- 
GitLab


From e7af439def1647a7b8da9e4a5aebfc2b80f998ce Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Tue, 15 Jan 2019 10:54:12 -0800
Subject: [PATCH 0730/2345] Use `strategy.make_input_fn_iterator` in estimator
 instead of the deprecated `strategy.distribute_dataset`.

PiperOrigin-RevId: 229400527
---
 .../contrib/distribute/python/tpu_strategy.py      | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 518c704b89..b9d3924a24 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -303,6 +303,20 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
         functools.partial(self._call_dataset_fn, dataset_fn),
         self._input_workers)
 
+  def _make_input_fn_iterator(
+      self,
+      input_fn,
+      replication_mode=distribute_lib.InputReplicationMode.PER_WORKER):
+    input_contexts = []
+    num_workers = self._input_workers.num_workers
+    for i in range(num_workers):
+      input_contexts.append(distribute_lib.InputContext(
+          num_input_pipelines=num_workers,
+          input_pipeline_id=i,
+          num_replicas_in_sync=self._num_replicas_in_sync))
+    return input_lib.InputFunctionIterator(
+        input_fn, self._input_workers, input_contexts)
+
   def _experimental_make_numpy_dataset(self, numpy_input, session):
     return numpy_dataset.one_host_numpy_dataset(
         numpy_input, numpy_dataset.SingleDevice(self.get_host_cpu_device(0)),
-- 
GitLab


From 556ac5a846e5b03859eda02e17349704387c465a Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Tue, 15 Jan 2019 10:55:40 -0800
Subject: [PATCH 0731/2345] Add helper to get mode strategy and replica context
 together.

PiperOrigin-RevId: 229400832
---
 .../distribution_strategy_context.py          |  5 +++
 tensorflow/python/eager/function.py           |  4 +--
 tensorflow/python/eager/tape.py               | 33 ++++++++++++++++---
 3 files changed, 35 insertions(+), 7 deletions(-)

diff --git a/tensorflow/python/distribute/distribution_strategy_context.py b/tensorflow/python/distribute/distribution_strategy_context.py
index e6648bf7c4..6c1e250f96 100644
--- a/tensorflow/python/distribute/distribution_strategy_context.py
+++ b/tensorflow/python/distribute/distribution_strategy_context.py
@@ -201,6 +201,11 @@ def has_strategy():
   return get_strategy() is not _get_default_strategy()
 
 
+def get_strategy_and_replica_context():
+  per_thread_mode = _get_per_thread_mode()
+  return (per_thread_mode.strategy, per_thread_mode.replica_context)
+
+
 # ------------------------------------------------------------------------------
 # Defaults that are used when no tf.distribute.Strategy is explicitly created.
 # We create them lazily in a function so that we can workaround the circular
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index b0140e5c98..0ebc7c915d 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -471,9 +471,7 @@ class Function(object):
     """
     ctx = context.context()
 
-    for v in self._func_graph.variables:
-      if v.trainable:
-        tape.variable_accessed(v)
+    tape.variables_accessed(self._func_graph.variables)
 
     tensor_inputs = []
     for i, arg in enumerate(args):
diff --git a/tensorflow/python/eager/tape.py b/tensorflow/python/eager/tape.py
index 56b68b9eea..56d696bf92 100644
--- a/tensorflow/python/eager/tape.py
+++ b/tensorflow/python/eager/tape.py
@@ -61,8 +61,9 @@ def watch(tape, tensor):
 
 def watch_variable(tape, variable):
   """Marks this variable to be watched by the given tape."""
-  strategy = distribution_strategy_context.get_strategy()
-  if distribution_strategy_context.get_replica_context():
+  strategy, context = (
+      distribution_strategy_context.get_strategy_and_replica_context())
+  if context:
     variables = [strategy.extended.value_container(variable)]
   else:
     variables = strategy.unwrap(variable)
@@ -76,8 +77,9 @@ def variable_accessed(variable):
   Args:
     variable: variable to be watched.
   """
-  strategy = distribution_strategy_context.get_strategy()
-  if distribution_strategy_context.get_replica_context():
+  strategy, context = (
+      distribution_strategy_context.get_strategy_and_replica_context())
+  if context:
     variables = [strategy.extended.value_container(variable)]
   else:
     variables = strategy.unwrap(variable)
@@ -85,6 +87,29 @@ def variable_accessed(variable):
     pywrap_tensorflow.TFE_Py_TapeVariableAccessed(var)
 
 
+def variables_accessed(variables):
+  """Notifies all tapes in the stack that variables have been accessed.
+
+  Only trainable variables are marked as accessed.
+
+  Args:
+    variables: iterable of variables to mark as accessed.
+  """
+  strategy, context = (
+      distribution_strategy_context.get_strategy_and_replica_context())
+  accessed = []
+  if context:
+    accessed = [strategy.extended.value_container(variable)
+                for variable in variables if variable.trainable]
+  else:
+    for variable in variables:
+      if variable.trainable:
+        accessed.extend(strategy.unwrap(variable))
+
+  for var in accessed:
+    pywrap_tensorflow.TFE_Py_TapeVariableAccessed(var)
+
+
 def pop_tape(tape):
   """Pops the top tape in the stack, if any."""
   pywrap_tensorflow.TFE_Py_TapeSetRemove(tape._tape)  # pylint: disable=protected-access
-- 
GitLab


From b0ce2166c0807e538ef346dad1d647d1983e0d4c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 15 Jan 2019 11:07:45 -0800
Subject: [PATCH 0732/2345] Remove unnecessary dependencies from
 core/kernels:resource_variable_ops to :mutex_ops and :state.

PiperOrigin-RevId: 229403710
---
 tensorflow/core/BUILD         | 1 +
 tensorflow/core/kernels/BUILD | 2 --
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index caf7b4cdbc..1fc917a5f2 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1406,6 +1406,7 @@ cc_library(
         "//tensorflow/core/kernels:manip",
         "//tensorflow/core/kernels:math",
         "//tensorflow/core/kernels:multinomial_op",
+        "//tensorflow/core/kernels:mutex_ops",
         "//tensorflow/core/kernels:nn",
         "//tensorflow/core/kernels:parameterized_truncated_normal_op",
         "//tensorflow/core/kernels:parsing",
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 1be749289b..ea6343f24d 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -2282,9 +2282,7 @@ tf_kernel_library(
         ":bounds_check",
         ":dense_update_functor",
         ":gather_functor",
-        ":mutex_ops",
         ":scatter_functor",
-        ":state",
         ":training_op_helpers",
         ":variable_ops",
         "//tensorflow/core:core_cpu_lib",
-- 
GitLab


From 3e0268dc76f8cc36e72a5cff980d8d0f27c609b9 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Tue, 15 Jan 2019 11:08:09 -0800
Subject: [PATCH 0733/2345] Adds v2 Mean IOU metric.

PiperOrigin-RevId: 229403786
---
 tensorflow/python/keras/metrics.py            | 129 +++++++++++++++++-
 tensorflow/python/keras/metrics_test.py       |  96 +++++++++++++
 .../tensorflow.keras.metrics.-accuracy.pbtxt  |   2 +-
 ...rflow.keras.metrics.-binary-accuracy.pbtxt |   2 +-
 ....keras.metrics.-categorical-accuracy.pbtxt |   2 +-
 ...low.keras.metrics.-categorical-hinge.pbtxt |   2 +-
 ...flow.keras.metrics.-cosine-proximity.pbtxt |   2 +-
 ...rflow.keras.metrics.-false-negatives.pbtxt |   2 +-
 ...rflow.keras.metrics.-false-positives.pbtxt |   2 +-
 .../v1/tensorflow.keras.metrics.-hinge.pbtxt  |   2 +-
 ...w.keras.metrics.-mean-absolute-error.pbtxt |   2 +-
 ...rics.-mean-absolute-percentage-error.pbtxt |   2 +-
 ...ow.keras.metrics.-mean-squared-error.pbtxt |   2 +-
 ...rics.-mean-squared-logarithmic-error.pbtxt |   2 +-
 .../v1/tensorflow.keras.metrics.-mean.pbtxt   |   2 +-
 .../tensorflow.keras.metrics.-precision.pbtxt |   2 +-
 .../v1/tensorflow.keras.metrics.-recall.pbtxt |   2 +-
 ....metrics.-sensitivity-at-specificity.pbtxt |   2 +-
 ...metrics.-sparse-categorical-accuracy.pbtxt |   2 +-
 ....metrics.-specificity-at-sensitivity.pbtxt |   2 +-
 ...sorflow.keras.metrics.-squared-hinge.pbtxt |   2 +-
 ...orflow.keras.metrics.-true-negatives.pbtxt |   2 +-
 ...orflow.keras.metrics.-true-positives.pbtxt |   2 +-
 .../tensorflow.keras.metrics.-accuracy.pbtxt  |   2 +-
 ...rflow.keras.metrics.-binary-accuracy.pbtxt |   2 +-
 ....keras.metrics.-categorical-accuracy.pbtxt |   2 +-
 ...low.keras.metrics.-categorical-hinge.pbtxt |   2 +-
 ...flow.keras.metrics.-cosine-proximity.pbtxt |   2 +-
 ...rflow.keras.metrics.-false-negatives.pbtxt |   2 +-
 ...rflow.keras.metrics.-false-positives.pbtxt |   2 +-
 .../v2/tensorflow.keras.metrics.-hinge.pbtxt  |   2 +-
 ...w.keras.metrics.-mean-absolute-error.pbtxt |   2 +-
 ...rics.-mean-absolute-percentage-error.pbtxt |   2 +-
 ...ow.keras.metrics.-mean-squared-error.pbtxt |   2 +-
 ...rics.-mean-squared-logarithmic-error.pbtxt |   2 +-
 .../v2/tensorflow.keras.metrics.-mean.pbtxt   |   2 +-
 .../tensorflow.keras.metrics.-precision.pbtxt |   2 +-
 .../v2/tensorflow.keras.metrics.-recall.pbtxt |   2 +-
 ....metrics.-sensitivity-at-specificity.pbtxt |   2 +-
 ...metrics.-sparse-categorical-accuracy.pbtxt |   2 +-
 ....metrics.-specificity-at-sensitivity.pbtxt |   2 +-
 ...sorflow.keras.metrics.-squared-hinge.pbtxt |   2 +-
 ...orflow.keras.metrics.-true-negatives.pbtxt |   2 +-
 ...orflow.keras.metrics.-true-positives.pbtxt |   2 +-
 44 files changed, 265 insertions(+), 44 deletions(-)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index ec09ad9ba8..245dffe99e 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -241,12 +241,13 @@ class Metric(Layer):
                  shape=(),
                  aggregation=tf_variables.VariableAggregation.SUM,
                  synchronization=tf_variables.VariableSynchronization.ON_READ,
-                 initializer=None):
+                 initializer=None,
+                 dtype=None):
     """Adds state variable. Only for use by subclasses."""
     return super(Metric, self).add_weight(
         name=name,
         shape=shape,
-        dtype=self._dtype,
+        dtype=self._dtype if dtype is None else dtype,
         trainable=False,
         initializer=initializer,
         collections=[],
@@ -2175,6 +2176,130 @@ class KullbackLeiblerDivergence(MeanMetricWrapper):
     return super(KullbackLeiblerDivergence, cls).from_config(config)
 
 
+class MeanIoU(Metric):
+  """Computes the mean Intersection-Over-Union metric.
+
+  Mean Intersection-Over-Union is a common evaluation metric for semantic image
+  segmentation, which first computes the IOU for each semantic class and then
+  computes the average over classes. IOU is defined as follows:
+    IOU = true_positive / (true_positive + false_positive + false_negative).
+  The predictions are accumulated in a confusion matrix, weighted by
+  `sample_weight` and the metric is then calculated from it.
+
+  If `sample_weight` is `None`, weights default to 1.
+  Use `sample_weight` of 0 to mask values.
+
+  Usage:
+
+  ```python
+  m = tf.keras.metrics.MeanIoU(num_classes=2)
+  m.update_state([0, 0, 1, 1], [0, 1, 0, 1])
+
+    # cm = [[1, 1],
+            [1, 1]]
+    # sum_row = [2, 2], sum_col = [2, 2], true_positives = [1, 1]
+    # iou = true_positives / (sum_row + sum_col - true_positives))
+    # result = (1 / (2 + 2 - 1) + 1 / (2 + 2 - 1)) / 2 = 0.33
+  print('Final result: ', m.result().numpy())  # Final result: 0.33
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile(
+    'sgd',
+    loss='mse',
+    metrics=[tf.keras.metrics.MeanIoU(num_classes=2)])
+  ```
+  """
+
+  def __init__(self, num_classes, name=None, dtype=None):
+    """Creates a `MeanIoU` instance.
+
+    Args:
+      num_classes: The possible number of labels the prediction task can have.
+        This value must be provided, since a confusion matrix of dimension =
+        [num_classes, num_classes] will be allocated.
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+    """
+    super(MeanIoU, self).__init__(name=name, dtype=dtype)
+    self.num_classes = num_classes
+
+    # Variable to accumulate the predictions in the confusion matrix. Setting
+    # the type to be `float64` as required by confusion_matrix_ops.
+    self.total_cm = self.add_weight(
+        'total_confusion_matrix',
+        shape=(num_classes, num_classes),
+        initializer=init_ops.zeros_initializer,
+        dtype=dtypes.float64)
+
+  def update_state(self, y_true, y_pred, sample_weight=None):
+    """Accumulates the confusion matrix statistics.
+
+    Args:
+      y_true: The ground truth values.
+      y_pred: The predicted values.
+      sample_weight: Optional weighting of each example. Defaults to 1. Can be a
+        `Tensor` whose rank is either 0, or the same rank as `y_true`, and must
+        be broadcastable to `y_true`.
+
+    Returns:
+      Update op.
+    """
+    # Flatten the input if its rank > 1.
+    if y_pred.shape.ndims > 1:
+      y_pred = array_ops.reshape(y_pred, [-1])
+
+    if y_true.shape.ndims > 1:
+      y_true = array_ops.reshape(y_true, [-1])
+
+    if sample_weight is not None and sample_weight.shape.ndims > 1:
+      sample_weight = array_ops.reshape(sample_weight, [-1])
+
+    # Accumulate the prediction to current confusion matrix.
+    current_cm = confusion_matrix.confusion_matrix(
+        y_true,
+        y_pred,
+        self.num_classes,
+        weights=sample_weight,
+        dtype=dtypes.float64)
+    return state_ops.assign_add(self.total_cm, current_cm)
+
+  def result(self):
+    """Compute the mean intersection-over-union via the confusion matrix."""
+    sum_over_row = math_ops.cast(
+        math_ops.reduce_sum(self.total_cm, axis=0), dtype=self._dtype)
+    sum_over_col = math_ops.cast(
+        math_ops.reduce_sum(self.total_cm, axis=1), dtype=self._dtype)
+    true_positives = math_ops.cast(
+        array_ops.diag_part(self.total_cm), dtype=self._dtype)
+
+    # sum_over_row + sum_over_col =
+    #     2 * true_positives + false_positives + false_negatives.
+    denominator = sum_over_row + sum_over_col - true_positives
+
+    # The mean is only computed over classes that appear in the
+    # label or prediction tensor. If the denominator is 0, we need to
+    # ignore the class.
+    num_valid_entries = math_ops.reduce_sum(
+        math_ops.cast(math_ops.not_equal(denominator, 0), dtype=self._dtype))
+
+    iou = math_ops.div_no_nan(true_positives, denominator)
+
+    return math_ops.div_no_nan(
+        math_ops.reduce_sum(iou, name='mean_iou'), num_valid_entries)
+
+  def reset_states(self):
+    K.set_value(self.total_cm, np.zeros((self.num_classes, self.num_classes)))
+
+  def get_config(self):
+    config = {'num_classes': self.num_classes}
+    base_config = super(MeanIoU, self).get_config()
+    return dict(list(base_config.items()) + list(config.items()))
+
+
 def accuracy(y_true, y_pred):
   y_pred.get_shape().assert_is_compatible_with(y_true.get_shape())
   if y_true.dtype != y_pred.dtype:
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index d28a615c02..bf2f4d88fa 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -967,6 +967,89 @@ class MeanRelativeErrorTest(test.TestCase):
     self.assertEqual(self.evaluate(result), 0)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class MeanIoUTest(test.TestCase):
+
+  def test_config(self):
+    m_obj = metrics.MeanIoU(num_classes=2, name='mean_iou')
+    self.assertEqual(m_obj.name, 'mean_iou')
+    self.assertEqual(m_obj.num_classes, 2)
+
+    m_obj2 = metrics.MeanIoU.from_config(m_obj.get_config())
+    self.assertEqual(m_obj2.name, 'mean_iou')
+    self.assertEqual(m_obj2.num_classes, 2)
+
+  def test_unweighted(self):
+    y_pred = constant_op.constant([0, 1, 0, 1], dtype=dtypes.float32)
+    y_true = constant_op.constant([0, 0, 1, 1])
+
+    m_obj = metrics.MeanIoU(num_classes=2)
+    self.evaluate(variables.variables_initializer(m_obj.variables))
+
+    result = m_obj(y_true, y_pred)
+
+    # cm = [[1, 1],
+    #       [1, 1]]
+    # sum_row = [2, 2], sum_col = [2, 2], true_positives = [1, 1]
+    # iou = true_positives / (sum_row + sum_col - true_positives))
+    expected_result = (1 / (2 + 2 - 1) + 1 / (2 + 2 - 1)) / 2
+    self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)
+
+  def test_weighted(self):
+    y_pred = constant_op.constant([0, 1, 0, 1], dtype=dtypes.float32)
+    y_true = constant_op.constant([0, 0, 1, 1])
+    sample_weight = constant_op.constant([0.2, 0.3, 0.4, 0.1])
+
+    m_obj = metrics.MeanIoU(num_classes=2)
+    self.evaluate(variables.variables_initializer(m_obj.variables))
+
+    result = m_obj(y_true, y_pred, sample_weight=sample_weight)
+
+    # cm = [[0.2, 0.3],
+    #       [0.4, 0.1]]
+    # sum_row = [0.6, 0.4], sum_col = [0.5, 0.5], true_positives = [0.2, 0.1]
+    # iou = true_positives / (sum_row + sum_col - true_positives))
+    expected_result = (0.2 / (0.6 + 0.5 - 0.2) + 0.1 / (0.4 + 0.5 - 0.1)) / 2
+    self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)
+
+  def test_multi_dim_input(self):
+    y_pred = constant_op.constant([[0, 1], [0, 1]], dtype=dtypes.float32)
+    y_true = constant_op.constant([[0, 0], [1, 1]])
+    sample_weight = constant_op.constant([[0.2, 0.3], [0.4, 0.1]])
+
+    m_obj = metrics.MeanIoU(num_classes=2)
+    self.evaluate(variables.variables_initializer(m_obj.variables))
+
+    result = m_obj(y_true, y_pred, sample_weight=sample_weight)
+
+    # cm = [[0.2, 0.3],
+    #       [0.4, 0.1]]
+    # sum_row = [0.6, 0.4], sum_col = [0.5, 0.5], true_positives = [0.2, 0.1]
+    # iou = true_positives / (sum_row + sum_col - true_positives))
+    expected_result = (0.2 / (0.6 + 0.5 - 0.2) + 0.1 / (0.4 + 0.5 - 0.1)) / 2
+    self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)
+
+  def test_zero_valid_entries(self):
+    m_obj = metrics.MeanIoU(num_classes=2)
+    self.evaluate(variables.variables_initializer(m_obj.variables))
+    self.assertAllClose(self.evaluate(m_obj.result()), 0, atol=1e-3)
+
+  def test_zero_and_non_zero_entries(self):
+    y_pred = constant_op.constant([1], dtype=dtypes.float32)
+    y_true = constant_op.constant([1])
+
+    m_obj = metrics.MeanIoU(num_classes=2)
+    self.evaluate(variables.variables_initializer(m_obj.variables))
+    result = m_obj(y_true, y_pred)
+
+    # cm = [[0, 0],
+    #       [0, 1]]
+    # sum_row = [0, 1], sum_col = [0, 1], true_positives = [0, 1]
+    # iou = true_positives / (sum_row + sum_col - true_positives))
+    expected_result = (0 + 1 / (1 + 1 - 1)) / 1
+    self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)
+
+
 def _get_model(compile_metrics):
   model_layers = [
       layers.Dense(3, activation='relu', kernel_initializer='ones'),
@@ -1094,6 +1177,19 @@ class ResetStatesTest(keras_parameterized.TestCase):
       self.assertEqual(self.evaluate(auc_obj.false_negatives[1]), 25.)
       self.assertEqual(self.evaluate(auc_obj.true_negatives[1]), 25.)
 
+  def test_reset_states_mean_iou(self):
+    m_obj = metrics.MeanIoU(num_classes=2)
+    model = _get_model([m_obj])
+    x = np.asarray([[0, 0, 0, 0], [1, 1, 1, 1], [1, 0, 1, 0], [0, 1, 0, 1]],
+                   dtype=np.float32)
+    y = np.asarray([[0], [1], [1], [1]], dtype=np.float32)
+    model.evaluate(x, y)
+    self.assertArrayNear(self.evaluate(m_obj.total_cm)[0], [1, 0], 1e-1)
+    self.assertArrayNear(self.evaluate(m_obj.total_cm)[1], [3, 0], 1e-1)
+    model.evaluate(x, y)
+    self.assertArrayNear(self.evaluate(m_obj.total_cm)[0], [1, 0], 1e-1)
+    self.assertArrayNear(self.evaluate(m_obj.total_cm)[1], [3, 0], 1e-1)
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-accuracy.pbtxt
index bad488f59b..e3e8dc00fe 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-accuracy.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-accuracy.pbtxt
index a1e7601a51..60b4f992c0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-accuracy.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
index 5f2c2f9807..deab5a7a70 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-hinge.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-hinge.pbtxt
index d173fc879f..10477e6662 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-hinge.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-cosine-proximity.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-cosine-proximity.pbtxt
index c50638740f..cdb3e1a79a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-cosine-proximity.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-cosine-proximity.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-negatives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-negatives.pbtxt
index c153e9cf4d..f4d067243d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-negatives.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-negatives.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-positives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-positives.pbtxt
index aae2bd9988..f898e00eea 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-positives.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-positives.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-hinge.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-hinge.pbtxt
index 7f13c75583..393ba964c4 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-hinge.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
index 896320004c..a86b26e618 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
index 37ab1a409b..5e8f321a06 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-error.pbtxt
index 9025abbb0a..a2eb10c5f6 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-error.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-error.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
index 808853994d..7df45a8caf 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean.pbtxt
index 904a2fa9ca..899502cfad 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt
index 794da447d7..3bdf3a4167 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt
index cae610b9ea..1ac2aacca0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
index b70ef32bca..5c4f26929f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
index 2e693269bf..48ac5c2fdc 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
index e62a2df056..702f711722 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-squared-hinge.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-squared-hinge.pbtxt
index 38f94048cc..6c3a9748c7 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-squared-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-squared-hinge.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-negatives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-negatives.pbtxt
index 1a524d73c0..3f1a99b83b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-negatives.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-negatives.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-positives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-positives.pbtxt
index b9b4f565c5..e5a4207a32 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-positives.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-positives.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-accuracy.pbtxt
index bad488f59b..e3e8dc00fe 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-accuracy.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-accuracy.pbtxt
index a1e7601a51..60b4f992c0 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-accuracy.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
index 5f2c2f9807..deab5a7a70 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-hinge.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-hinge.pbtxt
index d173fc879f..10477e6662 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-hinge.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-cosine-proximity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-cosine-proximity.pbtxt
index c50638740f..cdb3e1a79a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-cosine-proximity.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-cosine-proximity.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-negatives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-negatives.pbtxt
index c153e9cf4d..f4d067243d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-negatives.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-negatives.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-positives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-positives.pbtxt
index aae2bd9988..f898e00eea 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-positives.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-positives.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-hinge.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-hinge.pbtxt
index 7f13c75583..393ba964c4 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-hinge.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
index 896320004c..a86b26e618 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
index 37ab1a409b..5e8f321a06 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-error.pbtxt
index 9025abbb0a..a2eb10c5f6 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-error.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-error.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
index 808853994d..7df45a8caf 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean.pbtxt
index 904a2fa9ca..899502cfad 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt
index 794da447d7..3bdf3a4167 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt
index cae610b9ea..1ac2aacca0 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
index b70ef32bca..5c4f26929f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
index 2e693269bf..48ac5c2fdc 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
index e62a2df056..702f711722 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-squared-hinge.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-squared-hinge.pbtxt
index 38f94048cc..6c3a9748c7 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-squared-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-squared-hinge.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-negatives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-negatives.pbtxt
index 1a524d73c0..3f1a99b83b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-negatives.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-negatives.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-positives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-positives.pbtxt
index b9b4f565c5..e5a4207a32 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-positives.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-positives.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
-- 
GitLab


From a082de29448daa8d91f5cc24a3b37a296f5aec4b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 15 Jan 2019 11:09:26 -0800
Subject: [PATCH 0734/2345] Improve batch_group_count documentation in XLA
 operation semantics

PiperOrigin-RevId: 229404046
---
 .../compiler/xla/g3doc/operation_semantics.md      | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/xla/g3doc/operation_semantics.md b/tensorflow/compiler/xla/g3doc/operation_semantics.md
index 59ba9bb658..c5f9377f98 100644
--- a/tensorflow/compiler/xla/g3doc/operation_semantics.md
+++ b/tensorflow/compiler/xla/g3doc/operation_semantics.md
@@ -636,11 +636,15 @@ details, see `tf.nn.depthwise_conv2d`.
 
 The `batch_group_count` (default value 1) argument can be used for depthwise
 filters during backpropagation. `batch_group_count` needs to be a divisor of the
-size of the `lhs` batch dimension. If `batch_group_count` is greater than 1, it
-means that conceptually the output batch dimension is split evenely in
-`batch_group_count` groups, such that each group consists of a consecutive
-subsequence of batches. Each output batch element is the reduced value of the
-batch group size.
+size of the `lhs` (input) batch dimension. If `batch_group_count` is greater
+than 1, it means that the output batch dimension should be of size
+`batch_group_size` where `batch_group_size = input batch / batch_group_count`.
+For convolutions with `batch_group_count` greater than 1, the input batch size
+must evenly divide into batch_group_size and output feature size, which implies
+that the output feature size must be equal to batch_group_count. Conceptually,
+this can be achieved by performing the usual convolution, and then scraping
+`batch_group_size` number of elements on the diagonal of the matrix formed by
+output batch and output feature.
 
 The output shape has these dimensions, in this order:
 
-- 
GitLab


From 8154ec0b4a893657e48b838c06e1f988aeac4115 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Tue, 15 Jan 2019 11:48:17 -0800
Subject: [PATCH 0735/2345] [Grappler] Lower V2 control flow in inlined
 function body.

PiperOrigin-RevId: 229411946
---
 .../process_function_library_runtime.cc       |  10 +-
 tensorflow/core/grappler/optimizers/BUILD     |   1 +
 .../grappler/optimizers/function_optimizer.cc |  74 ++++++-----
 .../optimizers/function_optimizer_test.cc     | 122 ++++++++++++++++++
 4 files changed, 169 insertions(+), 38 deletions(-)

diff --git a/tensorflow/core/common_runtime/process_function_library_runtime.cc b/tensorflow/core/common_runtime/process_function_library_runtime.cc
index b236343a0f..76c75ad3d2 100644
--- a/tensorflow/core/common_runtime/process_function_library_runtime.cc
+++ b/tensorflow/core/common_runtime/process_function_library_runtime.cc
@@ -568,15 +568,12 @@ Status ProcessFunctionLibraryRuntime::InstantiateMultiDevice(
   DumpGraph("Before running POST_PLACEMENT passes", graph.get());
   TF_RETURN_IF_ERROR(OptimizationPassRegistry::Global()->RunGrouping(
       OptimizationPassRegistry::POST_PLACEMENT, optimization_options));
-  DumpGraph("Before running POST_REWRITE_FOR_EXEC passes", graph.get());
-  TF_RETURN_IF_ERROR(OptimizationPassRegistry::Global()->RunGrouping(
-      OptimizationPassRegistry::POST_REWRITE_FOR_EXEC, optimization_options));
-  DumpGraph("After all optimization passes", graph.get());
 
   Device* cpu_device;
   TF_RETURN_IF_ERROR(device_mgr_->LookupDevice("CPU:0", &cpu_device));
 
   if (options.optimize_graph_fn) {
+    DumpGraph("Before running graph optimization fn", graph.get());
     Status status = options.optimize_graph_fn(std::move(ret_node_names),
                                               &data->overlay_lib_, device_set,
                                               cpu_device, &graph);
@@ -587,6 +584,11 @@ Status ProcessFunctionLibraryRuntime::InstantiateMultiDevice(
     DumpGraph("After optimization", graph.get());
   }
 
+  DumpGraph("Before running POST_REWRITE_FOR_EXEC passes", graph.get());
+  TF_RETURN_IF_ERROR(OptimizationPassRegistry::Global()->RunGrouping(
+      OptimizationPassRegistry::POST_REWRITE_FOR_EXEC, optimization_options));
+  DumpGraph("After all optimization passes", graph.get());
+
   std::unordered_map<string, std::unique_ptr<Graph>> subgraphs;
   TF_RETURN_IF_ERROR(
       PartitionFunctionGraph(device_set, std::move(graph), &subgraphs));
diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index ee216f80f9..70e0f226fc 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -181,6 +181,7 @@ tf_cuda_cc_test(
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/utils:grappler_test",
+        "@com_google_absl//absl/algorithm:container",
     ],
 )
 
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index 8095d6f31d..1394aa5047 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -27,6 +27,8 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/device_mgr.h"
 #include "tensorflow/core/common_runtime/device_set.h"
 #include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/common_runtime/lower_if_while.h"
+#include "tensorflow/core/common_runtime/optimization_registry.h"
 #include "tensorflow/core/common_runtime/placer.h"
 #include "tensorflow/core/common_runtime/process_function_library_runtime.h"
 #include "tensorflow/core/framework/attr_value_util.h"
@@ -1365,20 +1367,6 @@ Status IsInlinableIndirectFunctionCall(const FunctionOptimizerContext& ctx,
         SummarizeNodeDef(func_node));
   }
 
-  // TODO(b/120991525, b/120986912): We need to lower `If` and `While` nodes to
-  // `Switch` nodes after function inlining (one more PRE_PLACEMENT pass?), but
-  // because of the reason described above we are not sure that it's safe, for
-  // now just disable inlining functions with functional control flow.
-  const auto is_functional_ctrl_flow_op = [](const NodeDef& node) {
-    return IsIf(node) || IsWhile(node);
-  };
-  if (absl::c_any_of(func.node_def(), is_functional_ctrl_flow_op)) {
-    return errors::FailedPrecondition(
-        "Can't inline function with `If` or `While` nodes in the function "
-        "body: ",
-        SummarizeNodeDef(func_node));
-  }
-
   return Status::OK();
 }
 
@@ -1488,22 +1476,46 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
 
   const string prefix = strings::StrCat(func_node.name(), "/");
 
+  // ------------------------------------------------------------------------ //
+  // Grappler receives the graph after PRE_PLACEMENT, Placer, and POST_PLACEMENT
+  // passes, so each node has a valid device assignment. Also V2 control
+  // flow ops (functional If and While) lowered to V1 control flow (Switch and
+  // Merge nodes). To keep graph valid for execution we must assign device to
+  // every inlined graph node, and also lower the control flow.
+
+  // Control flow lowering and Placer works with a Graph object.
+  std::unique_ptr<Graph> func_body_graph =
+      absl::make_unique<Graph>(ctx->function_library());
+
+  GraphConstructorOptions opts;
+  TF_RETURN_IF_ERROR(
+      ConvertGraphDefToGraph(opts, item.graph, func_body_graph.get()));
+
+  GraphOptimizationPassOptions opt_options;
+  opt_options.graph = &func_body_graph;
+  opt_options.flib_def = ctx->mutable_function_library();
+
+  // TODO(ezhulenev): Should we run full PRE_PLACEMENT pass here? And
+  // POST_PLACEMENT after placer?
+  LowerIfWhilePass pass;
+  TF_RETURN_IF_ERROR(pass.Run(opt_options));
+
   // ------------------------------------------------------------------------ //
   // Before placing the function body nodes we pin input placeholders to the
   // same device as their corresponding input nodes.
 
-  for (NodeDef& func_body_node : *item.graph.mutable_node()) {
+  for (Node* func_body_node : func_body_graph->nodes()) {
     const auto input_placeholder_idx =
-        input_placeholders_idx.find(func_body_node.name());
+        input_placeholders_idx.find(func_body_node->name());
 
     if (input_placeholder_idx != input_placeholders_idx.end()) {
       const int input_idx = input_placeholder_idx->second;
       const GraphView::OutputPort output_port =
           ctx->graph_view().GetRegularFanin({&func_node, input_idx});
 
-      VLOG(3) << "Pin inlined function input node '" << func_body_node.name()
+      VLOG(3) << "Pin inlined function input node '" << func_body_node->name()
               << "' to the '" << output_port.node->device() << "' device.";
-      func_body_node.set_device(output_port.node->device());
+      func_body_node->set_requested_device(output_port.node->device());
     }
   }
 
@@ -1511,8 +1523,6 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
   // After placing nodes corresponding to the function inputs, we need to assign
   // device placements to all other function body nodes.
 
-  GraphDef placed_graph_def;
-
   const DeviceSet* devices = ctx->devices();
 
   if (devices->devices().empty()) {
@@ -1522,9 +1532,8 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
     // of a batch job for graph analysis/optimization.
     VLOG(3) << "Assign function call node device to all function body nodes. "
             << "Device: " << func_node.device();
-    placed_graph_def = item.mutable_function_body();
-    for (NodeDef& node : *placed_graph_def.mutable_node()) {
-      node.set_device(func_node.device());
+    for (Node* func_body_node : func_body_graph->nodes()) {
+      func_body_node->set_requested_device(func_node.device());
     }
   } else {
     // If we are running in an active runtime session, Grappler will get the
@@ -1536,24 +1545,21 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
                    [](string* out, const Device* d) { out->append(d->name()); })
             << "]";
 
-    // Construct a Graph object from the instantiated function body.
-    GraphConstructorOptions opts;
-    Graph graph(ctx->function_library());
-    TF_RETURN_IF_ERROR(
-        ConvertGraphDefToGraph(opts, item.function_body(), &graph));
-
     // Use function caller node device as a default for placer.
     const Device* default_device =
         devices->FindDeviceByName(func_node.device());
 
-    Placer placer(&graph, devices, nullptr, /* No session options */
-                  default_device);
+    Placer placer(func_body_graph.get(), devices,
+                  nullptr /* No session options */, default_device);
     TF_RETURN_IF_ERROR(placer.Run());
-
-    // Convert Graph back to the GraphDef.
-    graph.ToGraphDef(&placed_graph_def);
   }
 
+  // TODO(ezhulenev): Should we run POST_PLACEMENT runtime pass here?
+
+  // All following transformations will work with GraphDef.
+  GraphDef placed_graph_def;
+  func_body_graph->ToGraphDef(&placed_graph_def);
+
   // ------------------------------------------------------------------------ //
   // After all nodes placed we need to prepare them for inlining into the
   // optimized graph: turn placeholders into identities, update nodes
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
index 827b0658c5..f4752bd584 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
@@ -14,6 +14,8 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/grappler/optimizers/function_optimizer.h"
+
+#include "absl/algorithm/container.h"
 #include "tensorflow/cc/ops/functional_ops.h"
 #include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/core/framework/function_testlib.h"
@@ -1182,6 +1184,126 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithMergedDeadTensors) {
   test::ExpectTensorEqual<float>(tensors[0], tensors_expected[0]);
 }
 
+TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithFunctionalControlFlow) {
+  using test::function::NDef;
+  using FDH = FunctionDefHelper;
+
+  FunctionOptimizer optimizer(RewriterConfig::AGGRESSIVE);
+
+  FunctionDef add_func = FunctionDefHelper::Create(
+      "MyAdd", {"x:T", "y:T"}, {"z:T"}, {"T: {float, double}"},
+      {{{"add"}, "Add", {"x", "y"}, {{"T", "$T"}}}},
+      /* Mapping between function returns and function node outputs. */
+      {{"z", "add:z:0"}});
+
+  FunctionDef mul_func = FunctionDefHelper::Create(
+      "MyMul", {"x:T", "y:T"}, {"z:T"}, {"T: {float, double}"},
+      {{{"mul"}, "Mul", {"x", "y"}, {{"T", "$T"}}}},
+      /* Mapping between function returns and function node outputs. */
+      {{"z", "mul:z:0"}});
+
+  // Compute: return cond ? a + b : a * b
+  FunctionDef add_or_mul_func = FunctionDefHelper::Create(
+      "AddOrMul", {"cond:bool", "x:float", "y:float"}, {"z:float"}, {},
+      {
+          {{"if_node"},
+           "If",
+           {"cond", "x", "y"},
+           {
+               {"Tcond", DT_BOOL},
+               {"Tin", DataTypeSlice{DT_FLOAT, DT_FLOAT}},
+               {"Tout", DataTypeSlice{DT_FLOAT}},
+               {"then_branch", FDH::FunctionRef("MyAdd", {{"T", DT_FLOAT}})},
+               {"else_branch", FDH::FunctionRef("MyMul", {{"T", DT_FLOAT}})},
+               {"_lower_using_switch_merge", true},
+           }},
+      },
+      /* Mapping between function returns and function node outputs. */
+      {{"z", "if_node:output:0"}});
+
+  // Build a computation graph for:
+  //   is_add: bool
+  //   a: float
+  //   b: float
+  //   c = AddOrMul(is_add, a, b)  # is_add ? a + b : a * b
+  //   d = Identity(c)
+  //   return d
+
+  // c = MyMul(a, b)
+  GrapplerItem item;
+  item.fetch = {"d"};
+  item.graph = test::function::GDef(
+      {NDef("is_add", "Placeholder", {}, {{"dtype", DT_BOOL}}, kDevice),
+       NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("b", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+
+       NDef("c", "PartitionedCall", {"is_add", "a", "b"},
+            {{"Tin", DataTypeSlice{DT_BOOL, DT_FLOAT, DT_FLOAT}},
+             {"Tout", DataTypeSlice{DT_FLOAT}},
+             {"f", FDH::FunctionRef("AddOrMul")}},
+            kDevice),
+
+       NDef("d", "Identity", {"c"}, {{"T", DT_FLOAT}}, kDevice)},
+      // Function library.
+      {add_or_mul_func, add_func, mul_func});
+
+  GraphDef optimized_graph;
+  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &optimized_graph));
+
+  const auto count_nodes_with_op = [&](const string& op) {
+    return absl::c_count_if(optimized_graph.node(), [&](const NodeDef& node) {
+      return node.op() == op;
+    });
+  };
+
+  // All `PartitionedCall` nodes in the optimized graph must be inlined, and
+  // `If` node must be lowered to `Switch` and `Merge` nodes.
+  EXPECT_EQ(count_nodes_with_op("PartitionedCallOp"), 0);
+  EXPECT_EQ(count_nodes_with_op("If"), 0);
+  EXPECT_EQ(count_nodes_with_op("Switch"), 3);
+  EXPECT_EQ(count_nodes_with_op("Merge"), 1);
+
+  GrapplerItem optimized = item.WithGraph(std::move(optimized_graph));
+
+  Tensor one = test::AsScalar<float>(1.0);
+  Tensor two = test::AsScalar<float>(2.0);
+  Tensor three = test::AsScalar<float>(3.0);
+
+  const auto feed_args = [&](bool is_add) {
+    std::vector<std::pair<string, Tensor>> feed;
+    feed.emplace_back("a", one);
+    feed.emplace_back("b", two);
+    feed.emplace_back("is_add", test::AsScalar<bool>(is_add));
+    return feed;
+  };
+
+  {  // Check 'is_add == true': a + b
+    item.feed = feed_args(true);
+    optimized.feed = feed_args(true);
+
+    auto tensors_expected = EvaluateFetchNodes(item);
+    ASSERT_EQ(tensors_expected.size(), 1);
+    test::ExpectTensorEqual<float>(tensors_expected[0], three);
+
+    auto tensors = EvaluateFetchNodes(optimized);
+    ASSERT_EQ(tensors.size(), tensors_expected.size());
+    test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
+  }
+
+  {  // Check 'is_add == false': a * b
+    item.feed = feed_args(false);
+    optimized.feed = feed_args(false);
+
+    auto tensors_expected = EvaluateFetchNodes(item);
+    ASSERT_EQ(tensors_expected.size(), 1);
+    test::ExpectTensorEqual<float>(tensors_expected[0], two);
+
+    auto tensors = EvaluateFetchNodes(optimized);
+    ASSERT_EQ(tensors.size(), tensors_expected.size());
+    test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
+  }
+}
+
 TEST_F(FunctionOptimizerTest, SpecializeFunctionXTimesTwo) {
   using test::function::NDef;
 
-- 
GitLab


From 7b8464b172fbe96464e7a800f89883e5796b1458 Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Tue, 15 Jan 2019 11:49:02 -0800
Subject: [PATCH 0736/2345] Adds Multi-Worker Independent Worker Mode to Keras.

PiperOrigin-RevId: 229412056
---
 .../distribute/distribute_coordinator.py      | 15 +--
 tensorflow/python/keras/BUILD                 |  1 +
 tensorflow/python/keras/engine/training.py    | 91 +++++++++++++++----
 .../python/keras/engine/training_utils.py     | 10 ++
 4 files changed, 91 insertions(+), 26 deletions(-)

diff --git a/tensorflow/python/distribute/distribute_coordinator.py b/tensorflow/python/distribute/distribute_coordinator.py
index 78c995a578..89bd8537b4 100644
--- a/tensorflow/python/distribute/distribute_coordinator.py
+++ b/tensorflow/python/distribute/distribute_coordinator.py
@@ -721,7 +721,8 @@ def run_distribute_coordinator(worker_fn,
 
   Returns:
     In the client job, return the value returned by `worker_fn` if
-    it is in-graph replication; return None otherwise.
+    it is in-graph replication or INDEPENDENT_WORKER mode; return None
+    otherwise.
   """
   tf_config = json.loads(os.environ.get("TF_CONFIG", "{}"))
   if not cluster_spec:
@@ -819,19 +820,19 @@ def run_distribute_coordinator(worker_fn,
     if task_type in [_TaskType.CHIEF, _TaskType.WORKER]:
       if strategy.extended.experimental_between_graph:
         # All jobs run `worker_fn` if between-graph.
-        _run_single_worker(worker_fn, strategy, cluster_spec, task_type,
-                           task_id, session_config, rpc_layer)
+        return _run_single_worker(worker_fn, strategy, cluster_spec, task_type,
+                                  task_id, session_config, rpc_layer)
       else:
         # Only one node runs `worker_fn` if in-graph.
         context = _WorkerContext(strategy, cluster_spec, task_type, task_id)
         if context.is_chief:
-          _run_single_worker(worker_fn, strategy, cluster_spec, None, None,
-                             session_config, rpc_layer)
+          return _run_single_worker(worker_fn, strategy, cluster_spec, None,
+                                    None, session_config, rpc_layer)
         else:
           server.join()
     elif task_type == _TaskType.EVALUATOR:
-      _run_single_worker(eval_fn, eval_strategy, cluster_spec, task_type,
-                         task_id, session_config, rpc_layer)
+      return _run_single_worker(eval_fn, eval_strategy, cluster_spec, task_type,
+                                task_id, session_config, rpc_layer)
     else:
       if task_type != _TaskType.PS:
         raise ValueError("Unexpected task_type: %r" % task_type)
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index d1ec5f0ac3..9e08c766db 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -152,6 +152,7 @@ py_library(
         ":regularizers",
         ":saving",
         "//tensorflow/python/data",
+        "//tensorflow/python/distribute:distribute_coordinator",
         "//tensorflow/python/distribute:reduce_util",
         "//tensorflow/python/training/checkpointable:data_structures",
         "//tensorflow/tools/docs:doc_controls",
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 9f544ceff4..d690df2986 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -24,6 +24,8 @@ import numpy as np
 from tensorflow.python import tf2
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
+from tensorflow.python.distribute import distribute_coordinator as dc
+from tensorflow.python.distribute import distribute_coordinator_context as dc_context
 from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
@@ -128,6 +130,7 @@ class Model(Network):
     # passing distribution strategy to compile rather than creating the model
     # under distribution strategy scope.
     self._compile_distribution = False
+    self._distributed_session_is_configured = False
 
     self.run_eagerly = None
 
@@ -216,6 +219,7 @@ class Model(Network):
                       'create the model under the distribution strategy scope.')
       self._distribution_strategy = distribute
       self._compile_distribution = True
+      self._distributed_session_is_configured = False
     else:
       if distribution_strategy_context.has_strategy():
         # When the user builds the model in the DS scope and cross replica
@@ -273,9 +277,6 @@ class Model(Network):
 
     # Set DistributionStrategy specific parameters.
     self._distributed_model = None
-    if self._distribution_strategy is not None:
-      distributed_training_utils.configure_and_create_session(
-          self._distribution_strategy)
     # Initialize model metric attributes.
     self._init_metric_attributes()
     if not self.built:
@@ -781,22 +782,52 @@ class Model(Network):
 
     # Case 1: distribution strategy.
     if self._distribution_strategy:
-      return training_distributed.fit_distributed(
-          self,
-          x=x,
-          y=y,
-          batch_size=batch_size,
-          epochs=epochs,
-          verbose=verbose,
-          callbacks=callbacks,
-          validation_split=validation_split,
-          validation_data=validation_data,
-          shuffle=shuffle,
-          class_weight=class_weight,
-          sample_weight=sample_weight,
-          initial_epoch=initial_epoch,
-          steps_per_epoch=steps_per_epoch,
-          validation_steps=validation_steps)
+      if training_utils.should_run_multi_worker():
+        # Multi-Worker mode runs the Keras training loop on multiple
+        # servers via the Distribute Coordinator.
+        def _worker_fn(_):
+          """Run training inside the distributed coordinator."""
+          self._configure_distributed_session()
+          return training_distributed.fit_distributed(
+              self,
+              x=x,
+              y=y,
+              batch_size=batch_size,
+              epochs=epochs,
+              verbose=verbose,
+              callbacks=callbacks,
+              validation_split=validation_split,
+              validation_data=validation_data,
+              shuffle=shuffle,
+              class_weight=class_weight,
+              sample_weight=sample_weight,
+              initial_epoch=initial_epoch,
+              steps_per_epoch=steps_per_epoch,
+              validation_steps=validation_steps)
+
+        # Independent worker only for now.
+        return dc.run_distribute_coordinator(
+            _worker_fn,
+            self._distribution_strategy,
+            mode=dc.CoordinatorMode.INDEPENDENT_WORKER)
+      else:
+        self._configure_distributed_session()
+        return training_distributed.fit_distributed(
+            self,
+            x=x,
+            y=y,
+            batch_size=batch_size,
+            epochs=epochs,
+            verbose=verbose,
+            callbacks=callbacks,
+            validation_split=validation_split,
+            validation_data=validation_data,
+            shuffle=shuffle,
+            class_weight=class_weight,
+            sample_weight=sample_weight,
+            initial_epoch=initial_epoch,
+            steps_per_epoch=steps_per_epoch,
+            validation_steps=validation_steps)
 
     batch_size = self._validate_or_infer_batch_size(
         batch_size, steps_per_epoch, x)
@@ -1007,6 +1038,7 @@ class Model(Network):
     """
     # Case 1: distribution strategy.
     if self._distribution_strategy:
+      self._configure_distributed_session()
       return training_distributed.evaluate_distributed(
           self,
           x=x,
@@ -1134,6 +1166,7 @@ class Model(Network):
     """
     # Case 1: distribution strategy.
     if self._distribution_strategy:
+      self._configure_distributed_session()
       return training_distributed.predict_distributed(self,
                                                       x=x,
                                                       batch_size=batch_size,
@@ -2644,6 +2677,26 @@ class Model(Network):
     self.output_names = training_utils.generic_output_names(outputs)
     self.built = True
 
+  def _configure_distributed_session(self):
+    """Configure a Session for use with Distribution Strategies.
+
+    Raises:
+      ValueError: If a non-distributed Session has already been created.
+    """
+    if not self._distributed_session_is_configured:
+      if (dc_context.get_current_worker_context() is not None and
+          getattr(K._SESSION, 'session', None) is not None):  # pylint: disable=protected-access
+        raise ValueError('Session was created before `fit`, `evaluate`, '
+                         'or `predict` was called. With Multi-Worker '
+                         'mode, this is not allowed. Please avoid '
+                         'creating a Session outside of these methods. '
+                         'The Session may have been created by a call '
+                         'to `keras.backend.get_session()` or '
+                         'functions that use Sessions, like `load_weights`.')
+      distributed_training_utils.configure_and_create_session(
+          self._distribution_strategy)
+      self._distributed_session_is_configured = True
+
 
 class DistributedCallbackModel(Model):
   """Model that is used for callbacks with DistributionStrategy."""
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index f7cbf4101a..c36b637829 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -21,6 +21,8 @@ from __future__ import print_function
 import abc
 from collections import OrderedDict
 import copy
+import json
+import os
 
 import numpy as np
 import six
@@ -46,6 +48,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import weights_broadcast_ops
 from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.training import server_lib
 from tensorflow.python.util import nest
 
 
@@ -1492,3 +1495,10 @@ def convert_eager_tensors_to_numpy(structure):
     return element
 
   return nest.map_structure(_convert, structure)
+
+
+def should_run_multi_worker():
+  """Whether a model should be run using DistributedCoordinator."""
+  tf_config = json.loads(os.environ.get('TF_CONFIG', '{}'))
+  cluster_spec = server_lib.ClusterSpec(tf_config.get('cluster', {}))
+  return tf_config and 'master' not in cluster_spec.jobs
-- 
GitLab


From b58039978dda1ddc4fadb2ee528cfdb9878e000a Mon Sep 17 00:00:00 2001
From: Ayush Dubey <ayushd@google.com>
Date: Tue, 15 Jan 2019 12:06:18 -0800
Subject: [PATCH 0737/2345] Automated rollback of commit
 681f6a6ac9344425549c8a38fcbfae854e7deddf

PiperOrigin-RevId: 229415244
---
 tensorflow/core/BUILD                         |   2 +-
 .../base_collective_executor.cc               |  33 --
 .../common_runtime/base_collective_executor.h |  18 -
 .../collective_param_resolver_local.cc        | 159 +++------
 .../collective_param_resolver_local.h         |  15 +-
 .../core/common_runtime/collective_util.cc    |  31 --
 .../core/common_runtime/collective_util.h     |  21 --
 .../hierarchical_tree_broadcaster.h           |   5 -
 .../core/common_runtime/ring_reducer.cc       |  43 ++-
 tensorflow/core/common_runtime/ring_reducer.h |  24 +-
 tensorflow/core/distributed_runtime/BUILD     |   1 -
 .../collective_param_resolver_distributed.cc  |  21 +-
 ...lective_param_resolver_distributed_test.cc |   2 -
 tensorflow/core/framework/collective.cc       |  11 -
 tensorflow/core/framework/collective.h        |  32 +-
 tensorflow/core/kernels/BUILD                 |  24 --
 .../core/kernels/collective_nccl_reducer.cc   | 204 -----------
 .../core/kernels/collective_nccl_reducer.h    |  49 ---
 .../kernels/collective_nccl_reducer_test.cc   | 331 ------------------
 tensorflow/core/kernels/collective_ops.cc     |   3 -
 tensorflow/core/nccl/BUILD                    |   4 +-
 tensorflow/core/protobuf/worker.proto         |   1 -
 22 files changed, 118 insertions(+), 916 deletions(-)
 delete mode 100644 tensorflow/core/kernels/collective_nccl_reducer.cc
 delete mode 100644 tensorflow/core/kernels/collective_nccl_reducer.h
 delete mode 100644 tensorflow/core/kernels/collective_nccl_reducer_test.cc

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 1fc917a5f2..bedc78ac11 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -3714,6 +3714,7 @@ tf_cc_tests(
     srcs = [
         "common_runtime/buf_rendezvous_test.cc",
         "common_runtime/collective_executor_mgr_test.cc",
+        "common_runtime/collective_param_resolver_local_test.cc",
         "common_runtime/collective_rma_local_test.cc",
         "common_runtime/device_resolver_local_test.cc",
         "common_runtime/device_set_test.cc",
@@ -3829,7 +3830,6 @@ tf_cc_tests(
     name = "higher_level_tests_needing_kernels",
     size = "small",
     srcs = [
-        "common_runtime/collective_param_resolver_local_test.cc",
         "graph/graph_constructor_test.cc",
     ],
     linkopts = select({
diff --git a/tensorflow/core/common_runtime/base_collective_executor.cc b/tensorflow/core/common_runtime/base_collective_executor.cc
index 03c0e9ce2b..92e56df181 100644
--- a/tensorflow/core/common_runtime/base_collective_executor.cc
+++ b/tensorflow/core/common_runtime/base_collective_executor.cc
@@ -296,37 +296,4 @@ Status BaseCollectiveExecutor::CreateCollective(
   return status;
 }
 
-bool BaseCollectiveExecutor::CheckDependencies(
-    const CollectiveParams& col_params) {
-  for (int32 instance : col_params.instance.impl_details.dependencies) {
-    auto find_iter = launched_.find(instance);
-    if (find_iter == launched_.end() || find_iter->second != 0) {
-      return false;
-    }
-  }
-  return true;
-}
-
-void BaseCollectiveExecutor::WaitForDependencies(
-    const CollectiveParams& col_params) {
-  mutex_lock l(launch_mu_);
-  while (!CheckDependencies(col_params)) {
-    launch_cv_.wait(l);
-  }
-}
-
-void BaseCollectiveExecutor::Launched(const CollectiveParams& col_params) {
-  mutex_lock l(launch_mu_);
-  if (launched_.find(col_params.instance.instance_key) == launched_.end()) {
-    const string& task_name =
-        col_params.instance.task_names[col_params.default_rank];
-    const int32 num_devices =
-        col_params.instance.num_devices_per_task.at(task_name);
-    launched_[col_params.instance.instance_key] = num_devices;
-  }
-  if (--launched_[col_params.instance.instance_key] == 0) {
-    launch_cv_.notify_all();
-  }
-}
-
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/base_collective_executor.h b/tensorflow/core/common_runtime/base_collective_executor.h
index b711aa6d50..09826a8814 100644
--- a/tensorflow/core/common_runtime/base_collective_executor.h
+++ b/tensorflow/core/common_runtime/base_collective_executor.h
@@ -135,33 +135,15 @@ class BaseCollectiveExecutor : public CollectiveExecutor {
                                client_locality, done);
   }
 
-  // If we need to enforce an ordering on any portion of collective
-  // implementation, and the ordering is encoded via attribute on the collective
-  // op, this function will block until all dependencies for this collective
-  // have completed.
-  void WaitForDependencies(const CollectiveParams& col_params) override;
-  // Record that this collective has completed the portion of the implementation
-  // that needs to be ordered wrt other collectives, to unblock any of its
-  // dependent ops.
-  void Launched(const CollectiveParams& col_params) override;
-
  protected:
   const int64 step_id_;
   const DeviceMgr* dev_mgr_;  // Not owned.
   std::unique_ptr<PerStepCollectiveRemoteAccess> remote_access_;
   const string* gpu_ring_order_;  // Not owned.
-  mutex launch_mu_;
-  condition_variable launch_cv_;
-  // collective instance key -> number of local devices for which NCCL ops have
-  // been launched.
-  std::unordered_map<int32, int32> launched_ GUARDED_BY(launch_mu_);
 
  private:
   Status CreateCollective(const CollectiveParams& col_params,
                           CollectiveImplementationInterface** col_impl);
-  // Check if all ops on which this collective depends on have launched.
-  bool CheckDependencies(const CollectiveParams& col_params)
-      EXCLUSIVE_LOCKS_REQUIRED(launch_mu_);
 };
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/collective_param_resolver_local.cc b/tensorflow/core/common_runtime/collective_param_resolver_local.cc
index 8907f6d56a..a8e3f4c881 100644
--- a/tensorflow/core/common_runtime/collective_param_resolver_local.cc
+++ b/tensorflow/core/common_runtime/collective_param_resolver_local.cc
@@ -26,7 +26,6 @@ limitations under the License.
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/gtl/flatmap.h"
-#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/types.h"
 #include "tensorflow/core/util/device_name_utils.h"
@@ -40,10 +39,7 @@ void CollectiveParamResolverLocal::InstanceRec::WaitForOutMu(mutex_lock& lock) {
 CollectiveParamResolverLocal::CollectiveParamResolverLocal(
     const DeviceMgr* dev_mgr, DeviceResolverInterface* dev_resolver,
     const string& task_name)
-    : nccl_(false),  // (b/111897089): turn on NCCL collectives.
-      dev_mgr_(dev_mgr),
-      dev_resolver_(dev_resolver),
-      task_name_(task_name) {}
+    : dev_mgr_(dev_mgr), dev_resolver_(dev_resolver), task_name_(task_name) {}
 
 void CollectiveParamResolverLocal::CompleteGroupAsync(
     const CompleteGroupRequest* request, CompleteGroupResponse* response,
@@ -320,28 +316,29 @@ GlobalDeviceMap EstablishGlobalRank(
 // cp->same_num_devices_per_task.  Requires cp->instance.task_names
 // be sorted.
 void SetDevPerTask(CollectiveParams* cp) {
-  cp->instance.num_devices_per_task.clear();
-  const string* last_task_name = &cp->instance.task_names[0];
+  cp->instance.same_num_devices_per_task = false;
+  if (cp->instance.task_names.empty()) return;
+  int dev_per_task = -1;
   int count = 0;
+  const string* last_task_name = &cp->instance.task_names[0];
   for (const string& task_name : cp->instance.task_names) {
-    if (task_name == *last_task_name) {
-      ++count;
-    } else {
-      cp->instance.num_devices_per_task[*last_task_name] = count;
+    if (task_name != *last_task_name) {
+      CHECK_GT(count, 0);
+      if (dev_per_task < 0) {
+        dev_per_task = count;
+      } else {
+        CHECK_GT(dev_per_task, 0);
+        if (count != dev_per_task) return;
+      }
       count = 1;
       last_task_name = &task_name;
+    } else {
+      ++count;
     }
   }
-  cp->instance.num_devices_per_task[*last_task_name] = count;
-
-  cp->instance.same_num_devices_per_task = false;
-  int dev_per_task = -1;
-  for (const auto& task_dev : cp->instance.num_devices_per_task) {
-    if (dev_per_task == -1) {
-      dev_per_task = task_dev.second;
-    } else if (dev_per_task != task_dev.second) {
-      return;
-    }
+  CHECK_GT(count, 0);
+  if ((dev_per_task > 0) && (count != dev_per_task)) {
+    return;
   }
   cp->instance.same_num_devices_per_task = true;
   CHECK_EQ((cp->group.group_size % cp->group.num_tasks), 0);
@@ -401,6 +398,7 @@ void CollectiveParamResolverLocal::SetDefaultRank(const string& device,
 void CollectiveParamResolverLocal::InitInstanceSharedParams(
     const GroupRec* gr, const CollectiveParams* cp, InstanceRec* ir,
     const StatusCallback& done) {
+  VLOG(1) << "InitInstanceSharedParams " << ir;
   ir->shared.instance = cp->instance;
   {
     mutex_lock gl(gr->mu);
@@ -414,8 +412,8 @@ void CollectiveParamResolverLocal::InitInstanceSharedParams(
   }
   ir->shared.default_rank = -1;
 
-  // Sort device_names lexicographically, keeping task_names in corresponding
-  // order.  Also set number of devices per task.
+  // Sort devce_names lexicographcally, keeping task_names in
+  // corresponding order.
   SortDevicesAndTasks(&ir->shared);
 
   // Get Locality data for all devices.
@@ -607,25 +605,6 @@ void CollectiveParamResolverLocal::CompleteInstanceAsync(
                        "intended only for non-distributed deployment."));
 }
 
-// TODO(b/111897089): we need a better way to pick the collective
-// implementation.  The ideal way would depend upon the topology and link
-// strength before picking a particular implementation.
-void CollectiveParamResolverLocal::AssignCollectiveType(CollectiveParams* cp) {
-  if (cp->instance.type == BROADCAST_COLLECTIVE) {
-    cp->instance.impl_details.collective_name = "HierarchicalTreeBroadcast";
-  } else if (cp->instance.type == REDUCTION_COLLECTIVE) {
-    if (nccl_) {
-      cp->instance.impl_details.collective_name = "NcclReduce";
-    } else {
-      cp->instance.impl_details.collective_name = "RingReduce";
-    }
-  } else {
-    cp->instance.impl_details.collective_name = "undef";
-  }
-  VLOG(1) << "AssignCollectiveType "
-          << cp->instance.impl_details.collective_name;
-}
-
 void CollectiveParamResolverLocal::CompleteInstanceLocal(
     const string& device, const GroupRec* gr, CollectiveParams* cp,
     bool is_source, const StatusCallback& done) {
@@ -662,57 +641,48 @@ void CollectiveParamResolverLocal::CompleteInstanceFromInitializedIRec(
     // custom operator= does a deep copy.
     cp->instance = ir->shared.instance;
   }
-  // Populate the fields common across task.
-  AssignCollectiveType(cp);
+  // Populate the fields common across task, also default_rank.
   SetDefaultRank(device, cp);
   CompleteTaskIsLocal(task_name_, cp);
-
+  // TODO(b/113171733): we need a better way to pick the collective
+  // implementation.  The ideal way would depend upon the topology and link
+  // strength before picking a particular implementation.
+  cp->instance.impl_details.collective_name =
+      (cp->instance.type == BROADCAST_COLLECTIVE) ? "HierarchicalTreeBroadcast"
+                                                  : "RingReduce";
   CollectiveImplementationInterface* col_impl;
-  Status status = CollectiveRegistry::LookupParamResolverInstance(
+  Status lookup_status = CollectiveRegistry::LookupParamResolverInstance(
       cp->instance.impl_details.collective_name, &col_impl);
-  if (status.ok()) {
-    status = col_impl->InitializeInstanceBeforeGroupDiscovery(cp);
-  }
-  if (!status.ok()) {
-    done(status);
+  if (!lookup_status.ok()) {
+    done(lookup_status);
     return;
   }
-
-  //  We may need to wait for the group if:
-  //  * this is a broadcast, for source discovery;
-  //  * we are using NCCL with more than 1 worker, for the communicator key from
-  //    rank 0.
-  bool broadcast = cp->instance.type == BROADCAST_COLLECTIVE;
-  bool nccl = cp->instance.type == REDUCTION_COLLECTIVE &&
-              cp->instance.impl_details.collective_name == "NcclReduce" &&
-              cp->group.num_tasks > 1;
-  if (broadcast || nccl) {
-    WaitForGroup(ir, cp, is_source, broadcast, nccl,
-                 [col_impl, ir, device, cp, done](InstanceRec* irec) {
-                   Status s;
-                   if (ir != irec) {
-                     s = errors::Internal("Expected ir ", ir, " and irec ",
-                                          irec, " to be equal");
-                   } else {
-                     mutex_lock l(irec->out_mu);
-                     irec->WaitForOutMu(l);
-                     s = irec->status;
-                     cp->source_rank = irec->source_rank;
-                     cp->instance.communicator_key = irec->communicator_key;
-                   }
-                   if (s.ok()) {
-                     s = col_impl->InitializeCollectiveParams(cp);
-                   }
-                   done(s);
-                 });
+  // If broadcast, may need to wait for source discovery.
+  if (cp->instance.type == BROADCAST_COLLECTIVE) {
+    CompleteInstanceSource(ir, cp, is_source,
+                           [col_impl, ir, device, cp, done](InstanceRec* irec) {
+                             CHECK_EQ(ir, irec);
+                             Status s;
+                             {
+                               mutex_lock l(irec->out_mu);
+                               irec->WaitForOutMu(l);
+                               s = irec->status;
+                               cp->source_rank = irec->source_rank;
+                             }
+                             if (s.ok()) {
+                               s = col_impl->InitializeCollectiveParams(cp);
+                             }
+                             done(s);
+                           });
   } else {
     done(col_impl->InitializeCollectiveParams(cp));
   }
 }
 
-void CollectiveParamResolverLocal::WaitForGroup(
-    InstanceRec* ir, CollectiveParams* cp, bool is_source, bool init_source,
-    bool init_nccl, const IRConsumer& f) {
+void CollectiveParamResolverLocal::CompleteInstanceSource(InstanceRec* ir,
+                                                          CollectiveParams* cp,
+                                                          bool is_source,
+                                                          const IRConsumer& f) {
   std::vector<IRConsumer> ready_waiters;
   {
     mutex_lock l(ir->out_mu);
@@ -722,8 +692,7 @@ void CollectiveParamResolverLocal::WaitForGroup(
     if (!ir->known[cp->default_rank]) {
       ir->known[cp->default_rank] = true;
       ++ir->known_count;
-      if (init_source && is_source) {
-        // Initialize source rank.
+      if (is_source) {
         if (ir->source_rank >= 0) {
           ir->status = errors::Internal("Instance ", cp->instance.instance_key,
                                         " already has source ", ir->source_rank,
@@ -733,26 +702,13 @@ void CollectiveParamResolverLocal::WaitForGroup(
           ir->source_rank = cp->default_rank;
         }
       }
-      if (init_nccl && cp->default_rank == 0) {
-        // Initialize communicator key.
-        if (!ir->communicator_key.empty()) {
-          ir->status =
-              errors::Internal("Instance ", cp->instance.instance_key,
-                               " already has communicator_key ",
-                               str_util::CEscape(ir->communicator_key),
-                               ", received second claim from device ",
-                               cp->instance.device_names[cp->default_rank]);
-        } else {
-          ir->communicator_key = cp->instance.communicator_key;
-        }
-      }
     }
     if (ir->known_count < ir->shared.group.group_size) {
       ir->known_waiters.push_back(f);
       return;
     }
     CHECK_EQ(ir->known_count, ir->shared.group.group_size);
-    if (init_source && ir->source_rank < 0) {
+    if (ir->source_rank < 0) {
       // NOTE(ayushd): changing the error message below would also require
       // updating CompleteParamsBroadcastForgotSend test in
       // CollectiveParamResolverLocalTest.
@@ -762,13 +718,6 @@ void CollectiveParamResolverLocal::WaitForGroup(
                            "could mean that there were group_size=",
                            ir->known_count, " BcastRecvs but no BcastSend.");
     }
-    if (init_nccl && ir->communicator_key.empty()) {
-      ir->status = errors::Internal(
-          "Instance ", cp->instance.instance_key, " device ",
-          cp->instance.device_names[cp->default_rank],
-          " did not find rank 0 for setting communicator key.  This is an "
-          "internal error in collective param resolution");
-    }
     if (!ir->known_waiters.empty()) {
       ready_waiters = std::move(ir->known_waiters);
     }
diff --git a/tensorflow/core/common_runtime/collective_param_resolver_local.h b/tensorflow/core/common_runtime/collective_param_resolver_local.h
index fd408e4ef3..365bddc787 100644
--- a/tensorflow/core/common_runtime/collective_param_resolver_local.h
+++ b/tensorflow/core/common_runtime/collective_param_resolver_local.h
@@ -130,10 +130,8 @@ class CollectiveParamResolverLocal : public ParamResolverInterface {
     Status status GUARDED_BY(out_mu);
 
     // These fields are used to count the instances that have called
-    // in and become known while resolving broadcast source identity and
-    // communicator key.
+    // in and become known while resolving broadcast source identity.
     int source_rank GUARDED_BY(out_mu);
-    string communicator_key GUARDED_BY(out_mu);
     int known_count GUARDED_BY(out_mu);
     std::vector<bool> known GUARDED_BY(out_mu);
     std::vector<IRConsumer> known_waiters GUARDED_BY(out_mu);
@@ -199,10 +197,10 @@ class CollectiveParamResolverLocal : public ParamResolverInterface {
                                            const StatusCallback& done)
       LOCKS_EXCLUDED(ir->out_mu);
 
-  // Complete source data and/or nccl communicator key.
+  // Complete source data for a broadcast instance.
   // Precondition: *cp has complete group data and default_rank.
-  void WaitForGroup(InstanceRec* ir, CollectiveParams* cp, bool is_source,
-                    bool init_source, bool init_nccl, const IRConsumer& f)
+  void CompleteInstanceSource(InstanceRec* ir, CollectiveParams* cp,
+                              bool is_source, const IRConsumer& f)
       LOCKS_EXCLUDED(ir->out_mu);
 
   // If cp.device_names contains only devices local to this process
@@ -218,15 +216,10 @@ class CollectiveParamResolverLocal : public ParamResolverInterface {
   // current ordering of cp->instance.device_names.
   void SetDefaultRank(const string& device, CollectiveParams* cp);
 
-  // Sets cp->instance.type based on collective op type, and attempts to assign
-  // best implementation.
-  void AssignCollectiveType(CollectiveParams* cp);
-
   // Helper to grab status under lock, invoke callback out of lock.
   void CallbackWithStatus(const InstanceRecCallback& done, InstanceRec* irec)
       LOCKS_EXCLUDED(irec->out_mu);
 
-  const bool nccl_;
   const DeviceMgr* dev_mgr_;
   DeviceResolverInterface* dev_resolver_;  // Not owned.
   string task_name_;
diff --git a/tensorflow/core/common_runtime/collective_util.cc b/tensorflow/core/common_runtime/collective_util.cc
index bee4a13d18..195521a078 100644
--- a/tensorflow/core/common_runtime/collective_util.cc
+++ b/tensorflow/core/common_runtime/collective_util.cc
@@ -79,36 +79,5 @@ string SubdivPermDebugString(const CollectiveParams& col_params) {
   return buf;
 }
 
-SubContext::SubContext(OpKernelContext* ctx, OpKernelContext::Params* params,
-                       OpKernel* op, Tensor* output, Tensor* input)
-    : sub_params_(*params),
-      sub_inputs_({output, input}),
-      sub_input_attr_({ctx->input_alloc_attr(0), ctx->input_alloc_attr(0)}),
-      sub_input_dc_(
-          {ctx->input_device_context(0), ctx->input_device_context(0)}) {
-  sub_params_.op_kernel = op;
-  sub_params_.inputs = &sub_inputs_;
-  sub_params_.input_alloc_attrs = &sub_input_attr_;
-  sub_params_.input_device_contexts = &sub_input_dc_;
-  sub_params_.eigen_gpu_device = nullptr;
-  sub_params_.ensure_eigen_gpu_device();
-  sub_params_.forward_from_array = &forward_from_;
-  sub_ctx_.reset(new OpKernelContext(&sub_params_, 1));
-}
-
-Status ComputeBinOp(OpKernelContext* op_ctx, OpKernelContext::Params* params,
-                    Device* device, OpKernel* op, Tensor* output,
-                    Tensor* input) {
-  // Prepare an OpKernelContext that is identical to that of the original Op
-  // (i.e. the collective), except for the input output sizes and identities and
-  // the Op itself.
-  // TODO(ayushd, tucker): Is it possible to cache and reuse these objects?
-  // They're mostly identical inside one device execution.
-  std::unique_ptr<SubContext> sub_ctx(
-      new SubContext(op_ctx, params, op, output, input));
-  device->Compute(op, sub_ctx->sub_ctx_.get());
-  return sub_ctx->sub_ctx_->status();
-}
-
 }  // namespace collective_util
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/collective_util.h b/tensorflow/core/common_runtime/collective_util.h
index 01fb8b8c81..ebb5731bec 100644
--- a/tensorflow/core/common_runtime/collective_util.h
+++ b/tensorflow/core/common_runtime/collective_util.h
@@ -32,27 +32,6 @@ Status InitializeDeviceAndLocality(const DeviceMgr* dev_mgr,
                                    DeviceLocality* device_locality);
 string SubdivPermDebugString(const CollectiveParams& col_params);
 
-// Used for executing a sub-operation, e.g. a merge_op instance, with
-// an OpKernelContext based on the one passed into this Op.
-class SubContext {
- public:
-  OpKernelContext::Params sub_params_;
-  gtl::InlinedVector<TensorValue, 4> sub_inputs_;
-  gtl::InlinedVector<AllocatorAttributes, 4> sub_input_attr_;
-  gtl::InlinedVector<DeviceContext*, 4> sub_input_dc_;
-  // Used only for Binary and Unary Ops for which we require
-  // the calculation to be in-place on the first input.
-  int forward_from_ = 0;
-  std::unique_ptr<OpKernelContext> sub_ctx_;
-  SubContext(OpKernelContext* ctx, OpKernelContext::Params* params,
-             OpKernel* op, Tensor* output, Tensor* input);
-  ~SubContext() = default;
-};
-
-Status ComputeBinOp(OpKernelContext* op_ctx, OpKernelContext::Params* params,
-                    Device* device, OpKernel* op, Tensor* output,
-                    Tensor* input);
-
 }  // namespace collective_util
 }  // namespace tensorflow
 
diff --git a/tensorflow/core/common_runtime/hierarchical_tree_broadcaster.h b/tensorflow/core/common_runtime/hierarchical_tree_broadcaster.h
index 76392b8e59..ceb9baad30 100644
--- a/tensorflow/core/common_runtime/hierarchical_tree_broadcaster.h
+++ b/tensorflow/core/common_runtime/hierarchical_tree_broadcaster.h
@@ -41,11 +41,6 @@ class HierarchicalTreeBroadcaster : public CollectiveImplementationInterface {
   // and device_locality.  Also saves the CollectiveContext in this object.
   Status InitializeCollectiveContext(CollectiveContext* col_ctx) override;
 
-  // No-op for hierarchical tree broadcaster.
-  Status InitializeInstanceBeforeGroupDiscovery(CollectiveParams*) override {
-    return Status::OK();
-  }
-
   // Begins async execution of the hierarchical tree broadcast.
   // Must be called in a blockable thread.
   // TODO(b/80529858): remove the previous warning when we have a dedicated
diff --git a/tensorflow/core/common_runtime/ring_reducer.cc b/tensorflow/core/common_runtime/ring_reducer.cc
index 8ed2fc2f1c..092f15e49e 100644
--- a/tensorflow/core/common_runtime/ring_reducer.cc
+++ b/tensorflow/core/common_runtime/ring_reducer.cc
@@ -394,6 +394,37 @@ void RingReducer::Finish(bool ok) {
   done_(s);
 }
 
+RingReducer::SubContext::SubContext(OpKernelContext* ctx,
+                                    OpKernelContext::Params* params,
+                                    OpKernel* op, Tensor* output, Tensor* input)
+    : sub_params_(*params),
+      sub_inputs_({output, input}),
+      sub_input_attr_({ctx->input_alloc_attr(0), ctx->input_alloc_attr(0)}),
+      sub_input_dc_(
+          {ctx->input_device_context(0), ctx->input_device_context(0)}) {
+  sub_params_.op_kernel = op;
+  sub_params_.inputs = &sub_inputs_;
+  sub_params_.input_alloc_attrs = &sub_input_attr_;
+  sub_params_.input_device_contexts = &sub_input_dc_;
+  sub_params_.eigen_gpu_device = nullptr;
+  sub_params_.ensure_eigen_gpu_device();
+  sub_params_.forward_from_array = &forward_from_;
+  sub_ctx_ = new OpKernelContext(&sub_params_, 1);
+}
+
+Status RingReducer::ComputeBinOp(Device* device, OpKernel* op, Tensor* output,
+                                 Tensor* input) {
+  // Prepare an OpKernelContext that is identical to that of the original Op
+  // (i.e. the collective), except for the input output sizes and identities and
+  // the Op itself.
+  // TODO(tucker): Is it possible to cache and reuse these objects?  They're
+  // mostly identical inside one device execution.
+  std::unique_ptr<SubContext> sub_ctx(
+      new SubContext(col_ctx_->op_ctx, col_ctx_->op_params, op, output, input));
+  device->Compute(op, sub_ctx->sub_ctx_);
+  return sub_ctx->sub_ctx_->status();
+}
+
 // At the beginning of the algorithm initialize a RingField struct for
 // every independent field of the tensor.
 void RingReducer::InitRingField(RingField* rf, int chunk_idx, int subdiv_idx,
@@ -601,9 +632,9 @@ bool RingReducer::RunAsyncParts() {
           --recv_pending_count;
           if (!rf->second_pass) {
             rf->action = RF_REDUCE;
-            Status s = collective_util::ComputeBinOp(
-                col_ctx_->op_ctx, col_ctx_->op_params, col_ctx_->device,
-                col_params_->merge_op.get(), &rf->chunk, &rf->tmp_chunk);
+            Status s =
+                ComputeBinOp(col_ctx_->device, col_params_->merge_op.get(),
+                             &rf->chunk, &rf->tmp_chunk);
             if (!s.ok()) {
               aborted = true;
               StartAbort(s);
@@ -616,9 +647,9 @@ bool RingReducer::RunAsyncParts() {
           if (!rf->second_pass && col_params_->final_op.get() && rf->is_final) {
             rf->action = RF_FINALIZE;
             group_size_tensor_ready_.WaitForNotification();
-            Status s = collective_util::ComputeBinOp(
-                col_ctx_->op_ctx, col_ctx_->op_params, col_ctx_->device,
-                col_params_->final_op.get(), &rf->chunk, &group_size_tensor_);
+            Status s =
+                ComputeBinOp(col_ctx_->device, col_params_->final_op.get(),
+                             &rf->chunk, &group_size_tensor_);
             if (!s.ok()) {
               aborted = true;
               StartAbort(s);
diff --git a/tensorflow/core/common_runtime/ring_reducer.h b/tensorflow/core/common_runtime/ring_reducer.h
index a5aa8fad70..0848e37b52 100644
--- a/tensorflow/core/common_runtime/ring_reducer.h
+++ b/tensorflow/core/common_runtime/ring_reducer.h
@@ -40,11 +40,6 @@ class RingReducer : public CollectiveImplementationInterface {
   // and device_locality.  Also saves the CollectiveContext in this object.
   Status InitializeCollectiveContext(CollectiveContext* col_ctx) override;
 
-  // No-op for ring reducer.
-  Status InitializeInstanceBeforeGroupDiscovery(CollectiveParams*) override {
-    return Status::OK();
-  }
-
   // Begins async execution of the ring reduce algorithm.
   // Must be called in a blockable thread.
   // TODO(b/80529858): remove the previous warning when we have a dedicated
@@ -57,8 +52,27 @@ class RingReducer : public CollectiveImplementationInterface {
   void StartAbort(const Status& s);
   void ContinueAfterInputCopy();
   void Finish(bool ok);
+  Status ComputeBinOp(Device* device, OpKernel* op, Tensor* output,
+                      Tensor* input);
   bool RunAsyncParts();
 
+  // Used for executing a sub-operation, e.g. a merge_op instance, with
+  // an OpKernelContext based on the one passed into this Op.
+  class SubContext {
+   public:
+    OpKernelContext::Params sub_params_;
+    gtl::InlinedVector<TensorValue, 4> sub_inputs_;
+    gtl::InlinedVector<AllocatorAttributes, 4> sub_input_attr_;
+    gtl::InlinedVector<DeviceContext*, 4> sub_input_dc_;
+    // Used only for Binary and Unary Ops for which we require
+    // the calculation to be in-place on the first input.
+    int forward_from_ = 0;
+    OpKernelContext* sub_ctx_;
+    SubContext(OpKernelContext* ctx, OpKernelContext::Params* params,
+               OpKernel* op, Tensor* output, Tensor* input);
+    ~SubContext() { delete sub_ctx_; }
+  };
+
   // Current status of a RingField
   enum RingFieldAction {
     RF_INIT = 0,    // Just initialized for a pass
diff --git a/tensorflow/core/distributed_runtime/BUILD b/tensorflow/core/distributed_runtime/BUILD
index 197a3f24d4..e388d3e6f0 100644
--- a/tensorflow/core/distributed_runtime/BUILD
+++ b/tensorflow/core/distributed_runtime/BUILD
@@ -594,7 +594,6 @@ tf_cc_test(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
-        "//tensorflow/core/kernels:collective_ops",
     ],
 )
 
diff --git a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc
index 9f94a24fcd..1dd10d309b 100644
--- a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc
+++ b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc
@@ -181,10 +181,6 @@ void CollectiveParamResolverDistributed::CompleteInstanceAsync(
                           ir->WaitForOutMu(l);
                           response->set_instance_key(cp->instance.instance_key);
                           response->set_source_rank(ir->source_rank);
-                          if (!cp->instance.communicator_key.empty()) {
-                            response->set_communicator_key(
-                                cp->instance.communicator_key);
-                          }
                           done_and_cleanup(fi_status);
                         } else {
                           done_and_cleanup(fi_status);
@@ -287,10 +283,8 @@ void CollectiveParamResolverDistributed::UpdateInstanceCache(
   using InstanceRecPointer = InstanceRec*;
   InstanceRecPointer* irp = new InstanceRecPointer(nullptr);
   int32 source_rank = resp.source_rank();
-  string communicator_key = resp.communicator_key();
 
-  auto continue_with_ir = [cp, irp, source_rank, communicator_key,
-                           done](const Status& s) {
+  auto continue_with_ir = [this, cp, irp, source_rank, done](const Status& s) {
     if (!s.ok()) {
       done(s);
       delete irp;
@@ -312,19 +306,6 @@ void CollectiveParamResolverDistributed::UpdateInstanceCache(
         }
         ir->source_rank = source_rank;
       }
-      if (ir->communicator_key != communicator_key) {
-        if (!ir->communicator_key.empty()) {
-          ir->status = errors::Internal(
-              "UpdateInstanceCache: CompleteInstanceResponse for instance ",
-              cp->instance.instance_key,
-              " gives communicator_key with size =", communicator_key.size(),
-              " but cache already holds communicator_key with size=",
-              ir->communicator_key.size());
-          status = ir->status;
-          break;
-        }
-        ir->communicator_key = communicator_key;
-      }
       if (ir->known_count < cp->group.group_size) {
         ir->known_count = cp->group.group_size;
         if (ir->known.size() != cp->group.group_size) {
diff --git a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc
index 823d7d5eb9..40b18d321a 100644
--- a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc
+++ b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc
@@ -268,8 +268,6 @@ class DeviceResDistTest : public ::testing::Test {
         EXPECT_EQ(cp_[idx].instance.device_names[idx], device_name);
         EXPECT_EQ(cp_[idx].instance.task_names[idx], task_name);
         if (idx > 0) {
-          EXPECT_EQ(cp_[0].instance.communicator_key,
-                    cp_[idx].instance.communicator_key);
           for (int i = 0; i < dev_count; ++i) {
             EXPECT_EQ(cp_[0].instance.device_names[i],
                       cp_[idx].instance.device_names[i]);
diff --git a/tensorflow/core/framework/collective.cc b/tensorflow/core/framework/collective.cc
index b83d183f14..7fa58347f2 100644
--- a/tensorflow/core/framework/collective.cc
+++ b/tensorflow/core/framework/collective.cc
@@ -17,7 +17,6 @@ limitations under the License.
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/hash/hash.h"
-#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 
 namespace tensorflow {
@@ -65,9 +64,7 @@ CollInstanceParams& CollInstanceParams::operator=(
     device_names.assign(other.device_names.begin(), other.device_names.end());
     task_names.assign(other.task_names.begin(), other.task_names.end());
     same_num_devices_per_task = other.same_num_devices_per_task;
-    num_devices_per_task = other.num_devices_per_task;
     gpu_ring_order = other.gpu_ring_order;
-    communicator_key = other.communicator_key;
     impl_details.subdiv_offsets.assign(
         other.impl_details.subdiv_offsets.begin(),
         other.impl_details.subdiv_offsets.end());
@@ -79,7 +76,6 @@ CollInstanceParams& CollInstanceParams::operator=(
     impl_details.subdiv_source_rank.assign(
         other.impl_details.subdiv_source_rank.begin(),
         other.impl_details.subdiv_source_rank.end());
-    impl_details.dependencies = other.impl_details.dependencies;
   }
   return *this;
 }
@@ -95,13 +91,6 @@ string CollInstanceParams::ToString() const {
   for (const auto& n : task_names) {
     strings::StrAppend(&v, n, ", ");
   }
-  strings::StrAppend(&v, "} num_devices_per_task={");
-  for (const auto dpt : num_devices_per_task) {
-    strings::StrAppend(&v, dpt.first, ": ", dpt.second, ", ");
-  }
-  strings::StrAppend(&v, "}, collective_name=", impl_details.collective_name,
-                     ", communicator_key=", str_util::CEscape(communicator_key),
-                     ", subdiv_offsets={");
   strings::StrAppend(&v, "}, subdiv_offsets={");
   for (const auto& d : impl_details.subdiv_offsets) {
     strings::StrAppend(&v, d, ",");
diff --git a/tensorflow/core/framework/collective.h b/tensorflow/core/framework/collective.h
index 546e3938a8..0321429702 100644
--- a/tensorflow/core/framework/collective.h
+++ b/tensorflow/core/framework/collective.h
@@ -70,8 +70,6 @@ struct CollImplDetails {
   std::vector<std::vector<int>> subdiv_permutations;
   std::vector<int> subdiv_offsets;
   std::vector<int> subdiv_source_rank;  // rank of source in each subdiv
-  std::vector<int32>
-      dependencies;  // collective instances on which this node depends
 };
 
 // Data common to all members of a collective instance.
@@ -87,13 +85,9 @@ struct CollInstanceParams {
   std::vector<string> task_names;
   // True if every task has the same number of devices.
   bool same_num_devices_per_task = false;
-  // Task -> number of devices on that task.
-  std::unordered_map<string, int32> num_devices_per_task;
   // If passed in to GPUOptions in ConfigProto, defines a good ring order for
   // GPUs.  Assumes same GPU configuration at each worker.
   string gpu_ring_order = "";
-  // Valid when using a communicator-based collective mechanism, e.g. NCCL.
-  string communicator_key;
   CollImplDetails impl_details;
   string ToString() const;
   CollInstanceParams& operator=(const struct CollInstanceParams& other);
@@ -275,21 +269,6 @@ class CollectiveExecutor : public PeerAccessInterface, public core::RefCounted {
 
   virtual PerStepCollectiveRemoteAccess* remote_access() { return nullptr; }
 
-  // `WaitForDependencies` and `Launched` are used for fine-grained control of
-  // execution order between collective instances.  These functions are intended
-  // to be called in `Run` function of collective implementations, and may be
-  // used to make part, or whole, of the collective execution ordered with
-  // respect to other collective instances.
-  //
-  // `WaitForDependencies` will block until it is safe to continue the callee's
-  // execution, where safety is defined as: ordered with respect to the
-  // collective instances defined in the callee's `wait_for` attribute.
-  virtual void WaitForDependencies(const CollectiveParams& col_params) {}
-  // `Launched` unblocks the dependent collective instances by recording that
-  // this callee device has completed the critical portion of the collective
-  // execution.
-  virtual void Launched(const CollectiveParams& col_params) {}
-
   // Used to designate an invalid group or instance key.
   static int64 kInvalidId;
 
@@ -368,8 +347,7 @@ class CollectiveImplementationInterface {
 
   // Initializes the portions of `col_params` specific to this
   // implementation.  Called exactly once for every Collective instance during
-  // the CollectiveParams resolution process when the graph is first executed,
-  // at the end of `CompleteInstanceLocal()`.
+  // the CollectiveParams resolution process when the graph is first executed.
   // NOTE(ayushd): This is effectively a static function because it modifies the
   // `col_params` passed in and should not manipulate any data members.  However
   // because it is virtual and needs to be implemented by every derived class we
@@ -382,14 +360,6 @@ class CollectiveImplementationInterface {
   // object.
   virtual Status InitializeCollectiveContext(CollectiveContext* col_ctx) = 0;
 
-  // Initializes instance params at the beginning of `CompleteInstanceLocal()`,
-  // unlike `InitializeCollectiveParams` which is called at the end.  This
-  // function is called before all devices in the instance are discovered, and
-  // may be used to broadcast data via the shared `InstanceRec` object in
-  // collective param resolution to all devices.
-  virtual Status InitializeInstanceBeforeGroupDiscovery(
-      CollectiveParams* col_params) = 0;
-
   // Processes and moves data according to the logic of this Collective
   // implementation.  Relies on appropriate initialization of op-specific
   // CollectiveParams in InitializeCollectiveParams(), as well as appropriate
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index ea6343f24d..358d3d656e 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -180,36 +180,12 @@ tf_cc_test(
 
 tf_kernel_library(
     name = "collective_ops",
-    srcs = if_cuda([
-        "collective_nccl_reducer.h",
-        "collective_nccl_reducer.cc",
-    ]),
     prefix = "collective_ops",
     deps = [
         "//tensorflow/core:collective_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
-    ] + if_cuda([
-        "@local_config_nccl//:nccl",
-        "//tensorflow/core/nccl:nccl_lib",
-    ]),
-)
-
-tf_cuda_cc_test(
-    name = "collective_nccl_reducer_test",
-    size = "small",
-    srcs = ["collective_nccl_reducer_test.cc"],
-    tags = tf_cuda_tests_tags() + ["no_cuda_on_cpu_tap"],
-    deps = [
-        "//tensorflow/core:all_kernels",
-        "//tensorflow/core:core_cpu",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:protos_all_cc",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-        "//tensorflow/core:testlib",
     ],
 )
 
diff --git a/tensorflow/core/kernels/collective_nccl_reducer.cc b/tensorflow/core/kernels/collective_nccl_reducer.cc
deleted file mode 100644
index 113f1487f0..0000000000
--- a/tensorflow/core/kernels/collective_nccl_reducer.cc
+++ /dev/null
@@ -1,204 +0,0 @@
-/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include "tensorflow/core/kernels/collective_nccl_reducer.h"
-
-#ifdef GOOGLE_CUDA
-
-#include "tensorflow/core/common_runtime/collective_util.h"
-#include "tensorflow/core/nccl/nccl_manager.h"
-
-namespace tensorflow {
-namespace {
-string NcclCollectiveKey(const string& exec_key, int step_id) {
-  return strings::StrCat(exec_key, ":", step_id);
-}
-}  // namespace
-
-NcclReducer::NcclReducer() : col_ctx_(nullptr), col_params_(nullptr) {}
-
-Status NcclReducer::InitializeCollectiveParams(CollectiveParams* col_params) {
-  if (col_params->instance.type != REDUCTION_COLLECTIVE ||
-      col_params->instance.impl_details.collective_name != "NcclReduce") {
-    return errors::Internal("Unexpected collective type ",
-                            col_params->instance.type, " expected ",
-                            REDUCTION_COLLECTIVE, "; or collective name ",
-                            col_params->instance.impl_details.collective_name,
-                            " expected NcclReduce");
-  } else {
-    return Status::OK();
-  }
-}
-
-Status NcclReducer::InitializeCollectiveContext(CollectiveContext* col_ctx) {
-  col_ctx_ = col_ctx;
-  col_params_ = &col_ctx->col_params;
-  return collective_util::InitializeDeviceAndLocality(
-      col_ctx->dev_mgr, col_ctx->device_name, &col_ctx->device,
-      &col_ctx->device_locality);
-}
-
-Status NcclReducer::InitializeInstanceBeforeGroupDiscovery(
-    CollectiveParams* col_params) {
-  if (col_params->default_rank == 0 && col_params->group.num_tasks > 1) {
-    col_params->instance.communicator_key =
-        NcclManager::instance()->GenerateCommunicatorKey();
-  }
-  return Status::OK();
-}
-
-Status ReductionOp(const string& merge_op, ncclRedOp_t* reduction_op) {
-  if (merge_op == "Add") {
-    *reduction_op = ncclSum;
-    return Status::OK();
-  } else if (merge_op == "Mul") {
-    *reduction_op = ncclProd;
-    return Status::OK();
-  } else {
-    return errors::Internal("Expected merge_op to be either Add or Mul, found ",
-                            merge_op);
-  }
-}
-
-void NcclReducer::Run(StatusCallback done) {
-  ncclRedOp_t reduction_op;
-  Status s = ReductionOp(col_params_->merge_op->type_string(), &reduction_op);
-  if (!s.ok()) {
-    done(s);
-    return;
-  }
-
-  Tensor group_size;
-  Notification group_size_ready;
-  Status group_size_status;
-  if (col_params_->final_op) {
-    // Create an on-device scalar value from group_size_.
-    // TODO(ayushd, tucker): avoid this copy by either reusing across
-    // invocations or providing the scalar to the kernel in host memory.
-    Tensor group_size_val(col_ctx_->output->dtype(), TensorShape({}));
-    switch (col_ctx_->output->dtype()) {
-      case DT_FLOAT:
-        group_size_val.scalar<float>()() = col_params_->group.group_size;
-        break;
-      case DT_DOUBLE:
-        group_size_val.scalar<double>()() = col_params_->group.group_size;
-        break;
-      case DT_INT32:
-        group_size_val.scalar<int32>()() = col_params_->group.group_size;
-        break;
-      case DT_INT64:
-        group_size_val.scalar<int64>()() = col_params_->group.group_size;
-        break;
-      default:
-        done(errors::Internal("Unsupported type ", col_ctx_->output->dtype()));
-        return;
-    }
-    group_size = Tensor(
-        col_ctx_->device->GetAllocator(col_ctx_->op_ctx->input_alloc_attr(0)),
-        col_ctx_->output->dtype(), TensorShape({}));
-    DeviceContext* op_dev_ctx = col_ctx_->op_ctx->op_device_context();
-    // Enqueue copy on gpu stream.
-    op_dev_ctx->CopyCPUTensorToDevice(
-        &group_size_val, col_ctx_->device, &group_size,
-        [&group_size_ready, &group_size_status](const Status& s) {
-          group_size_status = s;
-          group_size_ready.Notify();
-        });
-  } else {
-    group_size_ready.Notify();
-  }
-
-  Notification nccl_done;
-  Status nccl_status;
-  auto* compute_stream = col_ctx_->op_ctx->op_device_context()->stream();
-  auto* gpu_info = col_ctx_->op_ctx->device()->tensorflow_gpu_device_info();
-  // `AddToAllReduce` performs consistency checks for the NCCL call and enqueues
-  // the `Participant` struct locally.  When all local participants with this
-  // `nccl_collective_key` have called `AddToAllReduce` and
-  // `SignalMultiNodeReady`, all devices at this worker are ready to process
-  // this NCCL op.
-  //
-  // The `NcclManager` uses a dedicated CUDA stream for NCCL kernels.  At this
-  // point, it synchronizes the NCCL stream with the compute stream, and then
-  // enqueues the NCCL kernel on the NCCL stream.
-  const int num_global_devices = col_params_->group.group_size;
-  const int num_local_devices = col_params_->instance.num_devices_per_task.at(
-      col_params_->instance.task_names[col_params_->default_rank]);
-  const string nccl_collective_key =
-      NcclCollectiveKey(col_ctx_->exec_key, col_ctx_->step_id);
-  auto done_callback = [&nccl_done, &nccl_status](const Status& s) {
-    nccl_status = s;
-    nccl_done.Notify();
-  };
-  auto participant = absl::make_unique<NcclManager::Participant>(
-      compute_stream->parent(), compute_stream, gpu_info->event_mgr,
-      gpu_info->gpu_id, col_ctx_->input, col_ctx_->output,
-      col_params_->default_rank, std::move(done_callback));
-  VLOG(1) << "NcclReducer calling NcclManager::AddToAllReduce num_tasks "
-          << col_params_->group.num_tasks << " current task "
-          << col_params_->instance.task_names[col_params_->default_rank]
-          << " num local devices " << num_local_devices
-          << " num global devices " << num_global_devices;
-  NcclManager::instance()->AddToAllReduce(
-      std::move(participant),
-      {nccl_collective_key, num_local_devices, num_global_devices,
-       col_params_->instance.communicator_key},
-      reduction_op);
-
-  // NOTE(ayushd): We need to synchronize NCCL launches across nodes to prevent
-  // deadlocks.  In the current implementation, we define a deterministic
-  // sequential launch order between potentially concurrent collective instances
-  // by introducing control information during static graph analysis in
-  // graph/collective_order.cc.  This can be either in the form of explicit
-  // control edges or via `wait_for` attribute on the collective op.
-  //
-  // The other end of the design spectrum would have a distinguished node
-  // dynamically signal the next collective to launch to all other participants.
-  // This has higher degree of runtime coordination, but it may be able to
-  // achieve better performance if the (arbitrary) static execution order
-  // assigned in the first approach turns out to not be good from a scheduling
-  // perspective.  e.g. consider a graph in which c1, c2, and c3 are three
-  // concurrent collective instances, and the static ordering assigns c1 -> c2
-  // -> c3.  In practice, it could turn out that c3 is always ready to execute
-  // before c1 or c2.
-  //
-  // `WaitForDependencies` may block if the collective instances on which this
-  // op depends have not yet launched.  When this function returns, this op is
-  // ready to go.
-  col_ctx_->col_exec->WaitForDependencies(*col_params_);
-  NcclManager::instance()->SignalMultiNodeReady(nccl_collective_key);
-  // When all devices at this worker have called `SignalMultiNodeReady`, the
-  // `NcclManager` will enqueue the NCCL kernel on the NCCL stream.  Thus the
-  // implementation of `Launched` keeps track of the number of devices that have
-  // launched.
-  col_ctx_->col_exec->Launched(*col_params_);
-
-  // Wait for nccl op and group_size copy to succeed, then do final_op.
-  group_size_ready.WaitForNotification();
-  nccl_done.WaitForNotification();
-  Status final_status =
-      group_size_status.ok() ? nccl_status : group_size_status;
-  if (final_status.ok() && col_params_->final_op) {
-    final_status = collective_util::ComputeBinOp(
-        col_ctx_->op_ctx, col_ctx_->op_params, col_ctx_->device,
-        col_params_->final_op.get(), col_ctx_->output, &group_size);
-  }
-  done(final_status);
-}
-
-REGISTER_COLLECTIVE(NcclReduce, NcclReducer);
-
-}  // namespace tensorflow
-
-#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/collective_nccl_reducer.h b/tensorflow/core/kernels/collective_nccl_reducer.h
deleted file mode 100644
index dc70b280c5..0000000000
--- a/tensorflow/core/kernels/collective_nccl_reducer.h
+++ /dev/null
@@ -1,49 +0,0 @@
-/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#ifndef TENSORFLOW_CORE_KERNELS_COLLECTIVE_NCCL_REDUCER_H_
-#define TENSORFLOW_CORE_KERNELS_COLLECTIVE_NCCL_REDUCER_H_
-
-#include "tensorflow/core/framework/collective.h"
-
-namespace tensorflow {
-#ifdef GOOGLE_CUDA
-
-class NcclReducer : public CollectiveImplementationInterface {
- public:
-  NcclReducer();
-  ~NcclReducer() override = default;
-
-  // No-op for this collective implementation.
-  Status InitializeCollectiveParams(CollectiveParams* col_params) override;
-
-  // Initializes the device objects and device localities.
-  Status InitializeCollectiveContext(CollectiveContext* col_ctx) override;
-
-  // Initialize nccl communicator key.
-  Status InitializeInstanceBeforeGroupDiscovery(
-      CollectiveParams* col_params) override;
-
-  // Hands off all reduce to NcclManager.
-  void Run(StatusCallback done) override;
-
- private:
-  CollectiveContext* col_ctx_;          // Not owned
-  const CollectiveParams* col_params_;  // Not owned
-};
-
-#endif  // GOOGLE_CUDA
-}  // namespace tensorflow
-
-#endif  // TENSORFLOW_CORE_KERNELS_COLLECTIVE_NCCL_REDUCER_H_
diff --git a/tensorflow/core/kernels/collective_nccl_reducer_test.cc b/tensorflow/core/kernels/collective_nccl_reducer_test.cc
deleted file mode 100644
index ff33c1322c..0000000000
--- a/tensorflow/core/kernels/collective_nccl_reducer_test.cc
+++ /dev/null
@@ -1,331 +0,0 @@
-/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include "tensorflow/core/kernels/collective_nccl_reducer.h"
-
-#include <algorithm>
-#include "absl/memory/memory.h"
-#include "tensorflow/core/common_runtime/base_collective_executor.h"
-#include "tensorflow/core/common_runtime/device.h"
-#include "tensorflow/core/common_runtime/device_factory.h"
-#include "tensorflow/core/common_runtime/device_mgr.h"
-#include "tensorflow/core/common_runtime/device_resolver_local.h"
-#include "tensorflow/core/common_runtime/process_util.h"
-#include "tensorflow/core/common_runtime/test_collective_executor_mgr.h"
-#include "tensorflow/core/framework/collective.h"
-#include "tensorflow/core/framework/fake_input.h"
-#include "tensorflow/core/framework/node_def_builder.h"
-#include "tensorflow/core/framework/op_kernel.h"
-#include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/lib/core/notification.h"
-#include "tensorflow/core/lib/core/status_test_util.h"
-#include "tensorflow/core/platform/test.h"
-#include "tensorflow/core/public/session_options.h"
-#include "tensorflow/core/public/version.h"
-
-#ifdef GOOGLE_CUDA
-
-namespace tensorflow {
-static constexpr int kStepId = 10;
-
-std::unique_ptr<OpKernel> GetKernel(const NodeDef& node, DeviceBase* device) {
-  Status status;
-  std::unique_ptr<OpKernel> k = CreateOpKernel(
-      DEVICE_GPU, device, device->GetAllocator(AllocatorAttributes()), node,
-      TF_GRAPH_DEF_VERSION, &status);
-  if (!status.ok()) LOG(FATAL) << status;
-  return k;
-}
-
-std::unique_ptr<OpKernel> GetAdd(DeviceBase* device) {
-  NodeDef node_def;
-  NodeDefBuilder builder("add_node", "Add");
-  TF_CHECK_OK(builder.Attr("T", DT_FLOAT)
-                  .Input(FakeInput(DT_FLOAT))
-                  .Input(FakeInput(DT_FLOAT))
-                  .Finalize(&node_def));
-  return GetKernel(node_def, device);
-}
-
-std::unique_ptr<OpKernel> GetDiv(DeviceBase* device) {
-  NodeDef node_def;
-  NodeDefBuilder builder("add_node", "Div");
-  TF_CHECK_OK(builder.Attr("T", DT_FLOAT)
-                  .Input(FakeInput(DT_FLOAT))
-                  .Input(FakeInput(DT_FLOAT))
-                  .Finalize(&node_def));
-  return GetKernel(node_def, device);
-}
-
-class NcclReducerTest : public ::testing::Test {
- protected:
-  ~NcclReducerTest() override {
-    if (col_exec_) col_exec_->Unref();
-  }
-
-  void InitGPUDevices() {
-    std::vector<std::unique_ptr<Device>> all_devices;
-    SessionOptions session_options;
-    session_options.config.mutable_gpu_options()
-        ->set_per_process_gpu_memory_fraction(0.1);
-    session_options.env = Env::Default();
-    Status s = DeviceFactory::GetFactory(DEVICE_GPU)
-                   ->AddDevices(session_options, "", &all_devices);
-    TF_CHECK_OK(s);
-    for (std::unique_ptr<Device>& d : all_devices) {
-      if (d->device_type() == "GPU") {
-        gpus_.emplace_back(std::move(d));
-      }
-    }
-  }
-
-  void Init(int num_ranks) {
-    setenv("NCCL_DEBUG", "INFO", 1 /* replace */);
-    setenv("NCCL_LAUNCH_MODE", "PARALLEL", 1 /* replace */);
-    InitGPUDevices();
-    std::vector<std::unique_ptr<Device>> local_devices;
-    std::vector<string> device_names;
-    for (int rank = 0; rank < num_ranks; ++rank) {
-      if (rank < gpus_.size()) {
-        local_devices.emplace_back(std::move(gpus_[rank]));
-      }
-    }
-    int num_gpus = local_devices.size();
-    for (const auto& device : local_devices) {
-      device_names.push_back(device->name());
-      VLOG(2) << device->name();
-    }
-    if (!dev_mgr_) dev_mgr_.reset(new DeviceMgr(std::move(local_devices)));
-    col_exec_ = new BaseCollectiveExecutor(
-        &col_exec_mgr_, /*remote_access=*/nullptr, kStepId, dev_mgr_.get(),
-        /*gpu_ring_order=*/nullptr);
-
-    // Initialize collective params.
-    col_params_.name = "test_nccl_collective_op";
-    const int group_key = 5;
-    col_params_.group.group_key = group_key;
-    col_params_.group.device_type = DEVICE_GPU;
-    col_params_.group.group_size = num_ranks;
-    const int instance_key = 23;
-    col_params_.instance.instance_key = instance_key;
-    col_params_.instance.type = REDUCTION_COLLECTIVE;
-    col_params_.instance.data_type = DT_FLOAT;
-    col_params_.instance.impl_details.collective_name = "NcclReduce";
-    const string task_name = "/job:worker/replica:0/task:0";
-    col_params_.instance.num_devices_per_task[task_name] = num_ranks;
-    for (int rank = 0; rank < num_ranks; ++rank) {
-      col_params_.instance.device_names.push_back(
-          device_names[rank % num_gpus]);
-      col_params_.instance.task_names.push_back(task_name);
-    }
-    for (int rank = 0; rank < num_ranks; ++rank) {
-      instances_.push_back(absl::make_unique<DeviceInstance>(
-          rank, col_params_.instance.device_names[rank], this));
-    }
-  }
-
-  void Reduce() {
-    int done = 0;
-    mutex done_mu;
-    condition_variable done_cv;
-    for (const auto& instance : instances_) {
-      DeviceInstance* di = instance.get();
-      SchedClosure([di, &done, &done_mu, &done_cv] {
-        di->DoReduce();
-        mutex_lock l(done_mu);
-        ++done;
-        done_cv.notify_all();
-      });
-    }
-
-    mutex_lock l(done_mu);
-    while (done < instances_.size()) done_cv.wait(l);
-  }
-
-  void RunTest(int num_ranks, int tensor_length) {
-    Init(num_ranks);
-    std::vector<float> expected(tensor_length, 0.0);
-    for (int rank = 0; rank < num_ranks; ++rank) {
-      DeviceInstance* instance = instances_[rank].get();
-      instance->InitTensor(DT_FLOAT, TensorShape({tensor_length}),
-                           [&expected, rank](Tensor* t) {
-                             for (size_t i = 0; i < t->NumElements(); ++i) {
-                               float value = pow(10, rank) * i;
-                               t->flat<float>()(i) = value;
-                               expected[i] += value;
-                             }
-                           });
-    }
-    Reduce();
-    // Confirm that every rank computed the same correct value.
-    for (int i = 0; i < tensor_length; ++i) {
-      expected[i] /= num_ranks;
-    }
-    for (int rank = 0; rank < instances_.size(); ++rank) {
-      TF_ASSERT_OK(instances_[rank]->status_);
-      Tensor* dev_tensor = &instances_[rank]->tensor_;
-      Tensor actual(DT_FLOAT, TensorShape({tensor_length}));
-      Notification note;
-      Device* dev = instances_[rank]->device_;
-      auto* dev_info = dev->tensorflow_gpu_device_info();
-      dev_info->default_context->CopyDeviceTensorToCPU(
-          dev_tensor, /*tensor_name=*/"", dev, &actual,
-          [&note](const Status&) { note.Notify(); });
-      note.WaitForNotification();
-      for (int i = 0; i < tensor_length; ++i) {
-        EXPECT_FLOAT_EQ(expected[i], actual.template flat<float>()(i))
-            << "Mismatch at rank " << rank << " index " << i;
-      }
-    }
-  }
-
-  std::unique_ptr<OpKernel> GetCollectiveReduce(const CollectiveParams& params,
-                                                Tensor* input,
-                                                DeviceBase* device) {
-    mutex_lock l(mu_);
-    NodeDef node_def;
-    NodeDefBuilder builder(
-        strings::StrCat("collective_reduce_", reduce_counter_++),
-        "CollectiveReduce");
-    TF_CHECK_OK(
-        builder.Attr("T", params.instance.data_type)
-            .Attr("merge_op", "Add")
-            .Attr("final_op", "Div")
-            .Attr("group_size", params.group.group_size)
-            .Attr("group_key", params.group.group_key)
-            .Attr("instance_key", params.instance.instance_key)
-            .Attr("subdiv_offsets", params.instance.impl_details.subdiv_offsets)
-            .Input(FakeInput(params.instance.data_type))
-            .Finalize(&node_def));
-    return GetKernel(node_def, device);
-  }
-
-  class DeviceInstance {
-   public:
-    DeviceInstance(int rank, const string& device_name, NcclReducerTest* parent)
-        : parent_(parent), device_name_(device_name), rank_(rank) {
-      TF_CHECK_OK(parent_->dev_mgr_->LookupDevice(device_name_, &device_))
-          << "Could not find device " << device_name_ << " existing devices "
-          << parent_->dev_mgr_->DebugString();
-      col_params_.name = parent_->col_params_.name;
-      col_params_.default_rank = rank;
-      col_params_.group.group_key = parent_->col_params_.group.group_key;
-      col_params_.group.device_type = parent_->col_params_.group.device_type;
-      col_params_.group.group_size = parent_->col_params_.group.group_size;
-      col_params_.instance = parent->col_params_.instance;
-    }
-
-    void InitTensor(DataType dtype, const TensorShape& shape,
-                    const std::function<void(Tensor*)>& init_f) {
-      tensor_ =
-          Tensor(device_->GetAllocator(AllocatorAttributes()), dtype, shape);
-      Tensor cpu_tensor(dtype, shape);
-      init_f(&cpu_tensor);
-      VLOG(2) << "cpu_tensor " << cpu_tensor.DebugString();
-      auto* dev_info = device_->tensorflow_gpu_device_info();
-      Notification note;
-      dev_info->default_context->CopyCPUTensorToDevice(
-          &cpu_tensor, device_, &tensor_,
-          [&note](const Status&) { note.Notify(); });
-      note.WaitForNotification();
-    }
-
-    void DoReduce() {
-      col_params_.merge_op = GetAdd(device_);
-      col_params_.final_op = GetDiv(device_);
-
-      // Prepare an OpKernelContext.
-      OpKernelContext::Params op_params;
-      op_params.step_id = kStepId;
-      op_params.device = device_;
-      gtl::InlinedVector<TensorValue, 4> inputs;
-      inputs.push_back(TensorValue(&tensor_));
-      op_params.inputs = &inputs;
-      gtl::InlinedVector<AllocatorAttributes, 4> input_aa(
-          {AllocatorAttributes()});
-      op_params.input_alloc_attrs = &input_aa;
-      gtl::InlinedVector<DeviceContext*, 4> input_dc;
-      DeviceContext* dev_ctx = nullptr;
-      auto* dev_info = device_->tensorflow_gpu_device_info();
-      if (dev_info) {
-        dev_ctx = dev_info->default_context;
-        dev_ctx->Ref();
-      } else {
-        dev_ctx = new DeviceContext;
-      }
-      input_dc.push_back(dev_ctx);
-      op_params.input_device_contexts = &input_dc;
-      op_params.op_device_context = dev_ctx;
-      int forward_from = 0;
-      op_params.forward_from_array = &forward_from;
-      AllocatorAttributes generic_alloc_attr;
-      op_params.output_attr_array = &generic_alloc_attr;
-      std::unique_ptr<OpKernel> op =
-          parent_->GetCollectiveReduce(col_params_, &tensor_, device_);
-      op_params.op_kernel = op.get();
-      OpKernelContext ctx(&op_params, 1);
-
-      // We never actually execute the kernel, so we need to do the output
-      // allocation it would do, ourselves.
-      Tensor* output_tensor_ptr = nullptr;
-      TF_CHECK_OK(ctx.forward_input_or_allocate_output({0}, 0, tensor_.shape(),
-                                                       &output_tensor_ptr));
-      CHECK_EQ(output_tensor_ptr, ctx.mutable_output(0));
-
-      // Prepare a NcclReducer instance.
-      string exec_key =
-          strings::StrCat(col_params_.instance.instance_key, ":0:0");
-      NcclReducer reducer;
-      CollectiveContext col_ctx(parent_->col_exec_, parent_->dev_mgr_.get(),
-                                &ctx, &op_params, col_params_, exec_key,
-                                kStepId, &tensor_, &tensor_);
-      TF_CHECK_OK(reducer.InitializeCollectiveContext(&col_ctx));
-
-      // Run the all-reduce.
-      reducer.Run([this](Status s) { status_ = s; });
-      if (status_.ok()) {
-        CHECK(tensor_.CopyFrom(*ctx.mutable_output(0), tensor_.shape()));
-      }
-
-      dev_ctx->Unref();
-    }
-
-    NcclReducerTest* parent_;
-    string device_name_;
-    int rank_;
-    Tensor tensor_;
-    Device* device_;
-    CollectiveParams col_params_;
-    Status status_;
-  };
-
-  std::vector<std::unique_ptr<tensorflow::Device>> gpus_;
-  TestCollectiveExecutorMgr col_exec_mgr_;
-  CollectiveExecutor* col_exec_;
-  std::unique_ptr<DeviceMgr> dev_mgr_;
-  std::vector<std::unique_ptr<DeviceInstance>> instances_;
-  CollectiveParams col_params_;
-  mutex mu_;
-  int32 reduce_counter_ GUARDED_BY(mu_) = 0;
-};
-
-TEST_F(NcclReducerTest, Test2Dev16Len) { RunTest(2, 16); }
-TEST_F(NcclReducerTest, Test4Dev16Len) { RunTest(4, 16); }
-TEST_F(NcclReducerTest, Test8Dev16Len) { RunTest(8, 16); }
-TEST_F(NcclReducerTest, Test8Dev128Len) { RunTest(8, 128); }
-TEST_F(NcclReducerTest, Test8Dev1045991Len) { RunTest(8, 1048576); }
-
-}  // namespace tensorflow
-
-#endif
diff --git a/tensorflow/core/kernels/collective_ops.cc b/tensorflow/core/kernels/collective_ops.cc
index ea582836c2..82e2913b64 100644
--- a/tensorflow/core/kernels/collective_ops.cc
+++ b/tensorflow/core/kernels/collective_ops.cc
@@ -87,9 +87,6 @@ class CollectiveReduceOpKernel : public CollectiveOpKernel {
                     "final_op must be one of {\"Id\", \"Div\"} but got ",
                     final_op_name));
     OP_REQUIRES_OK(c, c->GetAttr("T", &col_params_.instance.data_type));
-    OP_REQUIRES_OK(c,
-                   c->GetAttr("wait_for",
-                              &col_params_.instance.impl_details.dependencies));
 
     const NodeDef& real_node = c->def();
     col_params_.name = strings::StrCat(real_node.name(), ": Reduce(",
diff --git a/tensorflow/core/nccl/BUILD b/tensorflow/core/nccl/BUILD
index a19e1af888..4be33b2a0c 100644
--- a/tensorflow/core/nccl/BUILD
+++ b/tensorflow/core/nccl/BUILD
@@ -20,10 +20,8 @@ cc_library(
     name = "nccl_lib",
     srcs = if_cuda([
         "nccl_manager.cc",
-        "nccl_rewrite.cc",
-    ]),
-    hdrs = if_cuda([
         "nccl_manager.h",
+        "nccl_rewrite.cc",
     ]),
     copts = tf_copts(),
     deps = if_cuda([
diff --git a/tensorflow/core/protobuf/worker.proto b/tensorflow/core/protobuf/worker.proto
index 4284dd119e..74058c8465 100644
--- a/tensorflow/core/protobuf/worker.proto
+++ b/tensorflow/core/protobuf/worker.proto
@@ -535,7 +535,6 @@ message CompleteInstanceRequest {
 message CompleteInstanceResponse {
   int32 instance_key = 1;
   int32 source_rank = 2;
-  bytes communicator_key = 3;
 }
 
 // Request for next agreed-upon step_id for the specified graph_keys.
-- 
GitLab


From bdfdaea03edaa8c91cb9d9606e283afed38eaf23 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Tue, 15 Jan 2019 12:46:18 -0800
Subject: [PATCH 0738/2345] Fix buildifier check

---
 tensorflow/contrib/tensorrt/BUILD | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 17717c8403..58c188b314 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -221,8 +221,8 @@ tf_cuda_library(
     ],
     hdrs = [
         "resources/trt_int8_calibrator.h",
-        "resources/trt_resource_manager.h",
         "resources/trt_lru_cache.h",
+        "resources/trt_resource_manager.h",
         "resources/trt_resources.h",
     ],
     deps = [
@@ -568,7 +568,7 @@ cc_library(
     copts = tf_copts(),
     deps = [
         "//tensorflow/core:lib",
-        "//tensorflow/core:framework"
+        "//tensorflow/core:framework",
     ],
 )
 
-- 
GitLab


From cee860e07b3b3daaefb7628f8b31748aed3aaf05 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Tue, 15 Jan 2019 12:21:35 -0800
Subject: [PATCH 0739/2345] Remove the old mnist_keras example

PiperOrigin-RevId: 229417867
---
 tensorflow/contrib/tpu/BUILD | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index 0fe9d0cdfc..d898f05405 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -263,7 +263,6 @@ py_library(
         "//learning/brain:__subpackages__",
         "//tensorflow:__subpackages__",
         "//third_party/cloud_tpu/models/keras_colab:__subpackages__",
-        "//third_party/cloud_tpu/models/mnist_keras:__subpackages__",
         "//third_party/cloud_tpu/models/resnet50_keras:__subpackages__",
     ],
     deps = [
-- 
GitLab


From 97aa5daaa5e2b7728d8b2a19695a39884fce0147 Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Tue, 15 Jan 2019 12:43:06 -0800
Subject: [PATCH 0740/2345] Increase version of google_pasta to 0.1.1 (for Byte
 string fix).

PiperOrigin-RevId: 229421387
---
 tensorflow/tools/pip_package/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index 55b7046e30..6e182a5e5c 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -51,7 +51,7 @@ REQUIRED_PACKAGES = [
     'absl-py >= 0.1.6',
     'astor >= 0.6.0',
     'gast >= 0.2.0',
-    'google_pasta >= 0.1.0',
+    'google_pasta >= 0.1.1',
     'keras_applications >= 1.0.6',
     'keras_preprocessing >= 1.0.5',
     'numpy >= 1.13.3',
-- 
GitLab


From 8d8fee60bd2215858f574b27dcfd07aac4473860 Mon Sep 17 00:00:00 2001
From: Haoyu Zhang <haoyuzhang@google.com>
Date: Tue, 15 Jan 2019 12:48:04 -0800
Subject: [PATCH 0741/2345] Add fp16 support to color distortion ops for better
 performance on GPU. These ops are typically used for image preprocessing in
 data input pipeline. When the input pipeline gets complicated and becomes the
 bottleneck (for example, in the SSD (Single Shot multibox Detector) model),
 offloading partial image preprocessing to accelerators can be useful to
 alleviate CPU workload. Adding fp16 support can further accelerate the
 processing on GPU and allow the model to have larger batch sizes.

The ops that are affected in this CL are:
  tf.image.random_brightness
  tf.image.random_contrast
  tf.image.random_saturation
  tf.image.random_hue
  tf.image.adjust_brightness
  tf.image.adjust_contrast
  tf.image.adjust_saturation
  tf.image.adjust_hue

PiperOrigin-RevId: 229422273
---
 .../compiler/tf2xla/kernels/image_ops.cc      | 54 ++++++-----
 tensorflow/core/kernels/adjust_contrast_op.cc | 59 +++++++----
 tensorflow/core/kernels/adjust_contrast_op.h  | 24 +++--
 .../core/kernels/adjust_contrast_op_gpu.cu.cc |  3 +-
 tensorflow/core/kernels/adjust_hsv_gpu.cu.h   | 17 ++--
 tensorflow/core/kernels/adjust_hue_op.cc      | 32 +++---
 tensorflow/core/kernels/adjust_hue_op.h       |  5 +-
 .../core/kernels/adjust_hue_op_gpu.cu.cc      | 14 ++-
 .../core/kernels/adjust_saturation_op.cc      | 32 +++---
 .../core/kernels/adjust_saturation_op.h       |  5 +-
 .../kernels/adjust_saturation_op_gpu.cu.cc    | 17 ++--
 tensorflow/core/ops/image_ops.cc              | 15 +--
 tensorflow/python/ops/image_ops_impl.py       | 97 +++++++++++--------
 tensorflow/python/ops/image_ops_test.py       | 56 +++--------
 14 files changed, 240 insertions(+), 190 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/image_ops.cc b/tensorflow/compiler/tf2xla/kernels/image_ops.cc
index a31b5a2cfd..92b20fe0ba 100644
--- a/tensorflow/compiler/tf2xla/kernels/image_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/image_ops.cc
@@ -26,6 +26,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/framework/types.pb.h"
 
 namespace tensorflow {
 namespace {
@@ -186,19 +187,20 @@ class AdjustContrastOpV2 : public XlaOpKernel {
                                         factor_shape.DebugString()));
 
     xla::XlaBuilder* b = context->builder();
-    xla::XlaOp input = context->Input(0);
-    xla::XlaOp factor = context->Input(1);
-
     DataType type = context->input_type(0);
 
+    xla::XlaOp input = context->Input(0);
+    xla::XlaOp factor = XlaHelpers::ConvertElementType(context->Input(1), type);
+
     const DataType accumulation_type = XlaHelpers::SumAccumulationType(type);
     auto converted = XlaHelpers::ConvertElementType(input, accumulation_type);
     auto reduce = xla::Reduce(converted, XlaHelpers::Zero(b, accumulation_type),
                               *context->GetOrCreateAdd(accumulation_type),
                               {height_dim, width_dim});
-    auto output = XlaHelpers::ConvertElementType(reduce, type);
-    output =
-        xla::Div(output, XlaHelpers::FloatLiteral(b, type, height * width));
+
+    auto output = xla::Div(
+        reduce, XlaHelpers::FloatLiteral(b, accumulation_type, height * width));
+    output = XlaHelpers::ConvertElementType(output, type);
 
     std::vector<int64> broadcast_dims(input_shape.dims() - 2);
     std::iota(broadcast_dims.begin(), broadcast_dims.end(), 0);
@@ -234,8 +236,10 @@ class AdjustSaturationOp : public XlaOpKernel {
                                 channels, " channels."));
 
     xla::XlaBuilder* b = context->builder();
-    xla::XlaOp input = context->Input(0);
-    xla::XlaOp scale = context->Input(1);
+    xla::XlaOp input =
+        XlaHelpers::ConvertElementType(context->Input(0), DT_FLOAT);
+    xla::XlaOp scale =
+        XlaHelpers::ConvertElementType(context->Input(1), DT_FLOAT);
 
     DataType type = context->input_type(0);
 
@@ -250,15 +254,17 @@ class AdjustSaturationOp : public XlaOpKernel {
                                       /*dimno=*/channel_dim);
     TensorShape channel_shape = input_shape;
     channel_shape.set_dim(channel_dim, 1);
-    auto hsv = RGBToHSV(context, b, {red, green, blue}, context->input_type(0),
-                        channel_shape);
+    auto hsv =
+        RGBToHSV(context, b, {red, green, blue}, DT_FLOAT, channel_shape);
 
-    hsv[1] = xla::Clamp(XlaHelpers::Zero(b, type), xla::Mul(hsv[1], scale),
-                        XlaHelpers::One(b, type));
+    hsv[1] = xla::Clamp(XlaHelpers::Zero(b, DT_FLOAT), xla::Mul(hsv[1], scale),
+                        XlaHelpers::One(b, DT_FLOAT));
 
-    auto rgb = HSVToRGB(context->builder(), hsv, context->input_type(0));
+    auto rgb = HSVToRGB(context->builder(), hsv, DT_FLOAT);
 
-    context->SetOutput(0, xla::ConcatInDim(b, rgb, channel_dim));
+    auto output = XlaHelpers::ConvertElementType(
+        xla::ConcatInDim(b, rgb, channel_dim), type);
+    context->SetOutput(0, output);
   }
 };
 REGISTER_XLA_OP(Name("AdjustSaturation"), AdjustSaturationOp);
@@ -284,8 +290,10 @@ class AdjustHueOp : public XlaOpKernel {
                                 channels, " channels."));
 
     xla::XlaBuilder* b = context->builder();
-    xla::XlaOp input = context->Input(0);
-    xla::XlaOp delta = context->Input(1);
+    xla::XlaOp input =
+        XlaHelpers::ConvertElementType(context->Input(0), DT_FLOAT);
+    xla::XlaOp delta =
+        XlaHelpers::ConvertElementType(context->Input(1), DT_FLOAT);
 
     DataType type = context->input_type(0);
 
@@ -300,20 +308,22 @@ class AdjustHueOp : public XlaOpKernel {
                                       /*dimno=*/channel_dim);
     TensorShape channel_shape = input_shape;
     channel_shape.set_dim(channel_dim, 1);
-    auto hsv = RGBToHSV(context, b, {red, green, blue}, context->input_type(0),
-                        channel_shape);
+    auto hsv =
+        RGBToHSV(context, b, {red, green, blue}, DT_FLOAT, channel_shape);
 
-    auto zero = XlaHelpers::Zero(b, type);
-    auto one = XlaHelpers::One(b, type);
+    auto zero = XlaHelpers::Zero(b, DT_FLOAT);
+    auto one = XlaHelpers::One(b, DT_FLOAT);
 
     auto& hue = hsv[0];
     hue = xla::Rem(xla::Add(hsv[0], delta), one);
     hue =
         xla::Select(xla::Lt(hue, zero), xla::Rem(xla::Add(one, hue), one), hue);
 
-    auto rgb = HSVToRGB(context->builder(), hsv, context->input_type(0));
+    auto rgb = HSVToRGB(context->builder(), hsv, DT_FLOAT);
 
-    context->SetOutput(0, xla::ConcatInDim(b, rgb, channel_dim));
+    auto output = XlaHelpers::ConvertElementType(
+        xla::ConcatInDim(b, rgb, channel_dim), type);
+    context->SetOutput(0, output);
   }
 };
 REGISTER_XLA_OP(Name("AdjustHue"), AdjustHueOp);
diff --git a/tensorflow/core/kernels/adjust_contrast_op.cc b/tensorflow/core/kernels/adjust_contrast_op.cc
index 47e10f56df..1aef0060b0 100644
--- a/tensorflow/core/kernels/adjust_contrast_op.cc
+++ b/tensorflow/core/kernels/adjust_contrast_op.cc
@@ -189,11 +189,11 @@ class AdjustContrastOpV2Base : public OpKernel {
                          const ComputeOptions& options) = 0;
 };
 
-template <typename Device>
+template <typename Device, typename T>
 class AdjustContrastOpv2;
 
 template <>
-class AdjustContrastOpv2<CPUDevice> : public AdjustContrastOpV2Base {
+class AdjustContrastOpv2<CPUDevice, float> : public AdjustContrastOpV2Base {
  public:
   explicit AdjustContrastOpv2(OpKernelConstruction* context)
       : AdjustContrastOpV2Base(context) {}
@@ -378,23 +378,32 @@ class AdjustContrastOpv2<CPUDevice> : public AdjustContrastOpV2Base {
   }
 };
 
-REGISTER_KERNEL_BUILDER(Name("AdjustContrastv2").Device(DEVICE_CPU),
-                        AdjustContrastOpv2<CPUDevice>);
+REGISTER_KERNEL_BUILDER(
+    Name("AdjustContrastv2").Device(DEVICE_CPU).TypeConstraint<float>("T"),
+    AdjustContrastOpv2<CPUDevice, float>);
 
 #if GOOGLE_CUDA
 // Forward declarations of the function specializations for GPU (to prevent
 // building the GPU versions here, they will be built compiling _gpu.cu.cc).
 namespace functor {
-template <>
-void AdjustContrastv2<GPUDevice>::operator()(
-    const GPUDevice& d, typename TTypes<float, 4>::ConstTensor input,
-    typename TTypes<float>::ConstScalar contrast_factor,
-    typename TTypes<float, 4>::Tensor output);
-extern template struct AdjustContrastv2<GPUDevice>;
+
+#define DECLARE_GPU_SPEC(T)                                         \
+  template <>                                                       \
+  void AdjustContrastv2<GPUDevice, T>::operator()(                  \
+      const GPUDevice& d, typename TTypes<T, 4>::ConstTensor input, \
+      typename TTypes<float>::ConstScalar contrast_factor,          \
+      typename TTypes<T, 4>::Tensor output);                        \
+  extern template struct AdjustContrastv2<GPUDevice, T>;
+
+DECLARE_GPU_SPEC(float);
+DECLARE_GPU_SPEC(Eigen::half);
+
+#undef DECLARE_GPU_SPEC
+
 }  // namespace functor
 
-template <>
-class AdjustContrastOpv2<GPUDevice> : public AdjustContrastOpV2Base {
+template <typename T>
+class AdjustContrastOpv2<GPUDevice, T> : public AdjustContrastOpV2Base {
  public:
   explicit AdjustContrastOpv2(OpKernelConstruction* context)
       : AdjustContrastOpV2Base(context) {}
@@ -403,20 +412,27 @@ class AdjustContrastOpv2<GPUDevice> : public AdjustContrastOpV2Base {
                  const ComputeOptions& options) override {
     const int64 shape[4] = {options.batch, options.height, options.width,
                             options.channels};
-    functor::AdjustContrastv2<GPUDevice>()(
-        context->eigen_device<GPUDevice>(),
-        options.input->shaped<float, 4>(shape), options.factor->scalar<float>(),
-        options.output->shaped<float, 4>(shape));
+    functor::AdjustContrastv2<GPUDevice, T>()(
+        context->eigen_device<GPUDevice>(), options.input->shaped<T, 4>(shape),
+        options.factor->scalar<float>(), options.output->shaped<T, 4>(shape));
   }
 };
 
-REGISTER_KERNEL_BUILDER(Name("AdjustContrastv2").Device(DEVICE_GPU),
-                        AdjustContrastOpv2<GPUDevice>);
+#define REGISTER_GPU(T)                                                   \
+  REGISTER_KERNEL_BUILDER(                                                \
+      Name("AdjustContrastv2").Device(DEVICE_GPU).TypeConstraint<T>("T"), \
+      AdjustContrastOpv2<GPUDevice, T>);
+
+REGISTER_GPU(float)
+REGISTER_GPU(Eigen::half)
+
+#undef REGISTER_GPU
+
 #endif  // GOOGLE_CUDA
 
 #ifdef TENSORFLOW_USE_SYCL
 template <>
-class AdjustContrastOpv2<SYCLDevice> : public AdjustContrastOpV2Base {
+class AdjustContrastOpv2<SYCLDevice, float> : public AdjustContrastOpV2Base {
  public:
   explicit AdjustContrastOpv2(OpKernelConstruction* context)
       : AdjustContrastOpV2Base(context) {}
@@ -431,8 +447,9 @@ class AdjustContrastOpv2<SYCLDevice> : public AdjustContrastOpV2Base {
         options.output->shaped<float, 4>(shape));
   }
 };
-REGISTER_KERNEL_BUILDER(Name("AdjustContrastv2").Device(DEVICE_SYCL),
-                        AdjustContrastOpv2<SYCLDevice>);
+REGISTER_KERNEL_BUILDER(
+    Name("AdjustContrastv2").Device(DEVICE_SYCL).TypeConstraint<float>("T"),
+    AdjustContrastOpv2<SYCLDevice, float>);
 #endif  // TENSORFLOW_USE_SYCL
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/adjust_contrast_op.h b/tensorflow/core/kernels/adjust_contrast_op.h
index f4a53c2ef9..3e501bccee 100644
--- a/tensorflow/core/kernels/adjust_contrast_op.h
+++ b/tensorflow/core/kernels/adjust_contrast_op.h
@@ -87,11 +87,11 @@ struct AdjustContrast {
 };
 
 // Functor used by AdjustContrastOpv2 to do the computations.
-template <typename Device>
+template <typename Device, typename T>
 struct AdjustContrastv2 {
-  void operator()(const Device& d, typename TTypes<float, 4>::ConstTensor input,
+  void operator()(const Device& d, typename TTypes<T, 4>::ConstTensor input,
                   typename TTypes<float>::ConstScalar contrast_factor,
-                  typename TTypes<float, 4>::Tensor output) {
+                  typename TTypes<T, 4>::Tensor output) {
     const int batch = input.dimension(0);
     const int height = input.dimension(1);
     const int width = input.dimension(2);
@@ -138,15 +138,19 @@ struct AdjustContrastv2 {
 #endif
     Eigen::Sizes<1, 1, 1, 1> scalar;
     float num_reduced_coeffs = height * width;
-    output.device(d) =
-        (input.shuffle(reduced_dims_first).sum(reduction_axis).eval() /
-         num_reduced_coeffs)
-            .reshape(reshape_dims)
-            .broadcast(broadcast_dims);
+    output.device(d) = (input.template cast<float>()
+                            .shuffle(reduced_dims_first)
+                            .sum(reduction_axis)
+                            .eval() /
+                        num_reduced_coeffs)
+                           .template cast<T>()
+                           .reshape(reshape_dims)
+                           .broadcast(broadcast_dims);
     auto contrast_factor_tensor =
         contrast_factor.reshape(scalar).broadcast(scalar_broadcast);
-    auto adjusted = (input - output) * contrast_factor_tensor;
-    output.device(d) += adjusted;
+    auto adjusted =
+        (input - output).template cast<float>() * contrast_factor_tensor;
+    output.device(d) += adjusted.template cast<T>();
   }
 };
 
diff --git a/tensorflow/core/kernels/adjust_contrast_op_gpu.cu.cc b/tensorflow/core/kernels/adjust_contrast_op_gpu.cu.cc
index a451bfe29c..1a1c2a4e1e 100644
--- a/tensorflow/core/kernels/adjust_contrast_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/adjust_contrast_op_gpu.cu.cc
@@ -26,7 +26,8 @@ namespace tensorflow {
 typedef Eigen::GpuDevice GPUDevice;
 
 // this is for v2
-template struct functor::AdjustContrastv2<GPUDevice>;
+template struct functor::AdjustContrastv2<GPUDevice, float>;
+template struct functor::AdjustContrastv2<GPUDevice, Eigen::half>;
 
 // these are for v1
 template struct functor::AdjustContrast<GPUDevice, uint8>;
diff --git a/tensorflow/core/kernels/adjust_hsv_gpu.cu.h b/tensorflow/core/kernels/adjust_hsv_gpu.cu.h
index 49df5ae296..dede7d249d 100644
--- a/tensorflow/core/kernels/adjust_hsv_gpu.cu.h
+++ b/tensorflow/core/kernels/adjust_hsv_gpu.cu.h
@@ -91,11 +91,10 @@ inline __device__ RgbTuple hsv2rgb_cuda(const float h, const float s,
   return tuple;
 }
 
-template <bool AdjustHue, bool AdjustSaturation, bool AdjustV>
+template <bool AdjustHue, bool AdjustSaturation, bool AdjustV, typename T>
 __global__ void adjust_hsv_nhwc(const int64 number_elements,
-                                const float* const __restrict__ input,
-                                float* const output,
-                                const float* const hue_delta,
+                                const T* const __restrict__ input,
+                                T* const output, const float* const hue_delta,
                                 const float* const saturation_scale,
                                 const float* const value_scale) {
   // multiply by 3 since we're dealing with contiguous RGB bytes for each pixel
@@ -111,7 +110,9 @@ __global__ void adjust_hsv_nhwc(const int64 number_elements,
     output[idx + 2] = input[idx + 2];
     return;
   }
-  const HsvTuple hsv = rgb2hsv_cuda(input[idx], input[idx + 1], input[idx + 2]);
+  const HsvTuple hsv = rgb2hsv_cuda(static_cast<float>(input[idx]),
+                                    static_cast<float>(input[idx + 1]),
+                                    static_cast<float>(input[idx + 2]));
   float new_h = hsv.h;
   float new_s = hsv.s;
   float new_v = hsv.v;
@@ -134,9 +135,9 @@ __global__ void adjust_hsv_nhwc(const int64 number_elements,
     new_v = hsv.v * scale;
   }
   const RgbTuple rgb = hsv2rgb_cuda(new_h, new_s, new_v);
-  output[idx] = rgb.r;
-  output[idx + 1] = rgb.g;
-  output[idx + 2] = rgb.b;
+  output[idx] = static_cast<T>(rgb.r);
+  output[idx + 1] = static_cast<T>(rgb.g);
+  output[idx + 2] = static_cast<T>(rgb.b);
 }
 
 }  // namespace internal
diff --git a/tensorflow/core/kernels/adjust_hue_op.cc b/tensorflow/core/kernels/adjust_hue_op.cc
index 52dec94305..06de5ea3fb 100644
--- a/tensorflow/core/kernels/adjust_hue_op.cc
+++ b/tensorflow/core/kernels/adjust_hue_op.cc
@@ -82,7 +82,7 @@ class AdjustHueOpBase : public OpKernel {
   }
 };
 
-template <class Device>
+template <class Device, typename T>
 class AdjustHueOp;
 
 namespace internal {
@@ -196,7 +196,7 @@ static void hv_range_to_rgb(float h, float v_min, float v_max, float* r,
 }  // namespace internal
 
 template <>
-class AdjustHueOp<CPUDevice> : public AdjustHueOpBase {
+class AdjustHueOp<CPUDevice, float> : public AdjustHueOpBase {
  public:
   explicit AdjustHueOp(OpKernelConstruction* context)
       : AdjustHueOpBase(context) {}
@@ -245,12 +245,13 @@ class AdjustHueOp<CPUDevice> : public AdjustHueOpBase {
   }
 };
 
-REGISTER_KERNEL_BUILDER(Name("AdjustHue").Device(DEVICE_CPU),
-                        AdjustHueOp<CPUDevice>);
+REGISTER_KERNEL_BUILDER(
+    Name("AdjustHue").Device(DEVICE_CPU).TypeConstraint<float>("T"),
+    AdjustHueOp<CPUDevice, float>);
 
 #if GOOGLE_CUDA
-template <>
-class AdjustHueOp<GPUDevice> : public AdjustHueOpBase {
+template <typename T>
+class AdjustHueOp<GPUDevice, T> : public AdjustHueOpBase {
  public:
   explicit AdjustHueOp(OpKernelConstruction* context)
       : AdjustHueOpBase(context) {}
@@ -265,17 +266,24 @@ class AdjustHueOp<GPUDevice> : public AdjustHueOpBase {
     const auto stream = device.stream();
     OP_REQUIRES(context, stream, errors::Internal("No GPU stream available."));
     if (number_of_elements > 0) {
-      const float* input_data = input->flat<float>().data();
+      const T* input_data = input->flat<T>().data();
       const float* delta_h = delta->flat<float>().data();
-      float* const output_data = output->flat<float>().data();
-      functor::AdjustHueGPU()(&device, number_of_elements, input_data, delta_h,
-                              output_data);
+      T* const output_data = output->flat<T>().data();
+      functor::AdjustHueGPU<T>()(&device, number_of_elements, input_data,
+                                 delta_h, output_data);
     }
   }
 };
 
-REGISTER_KERNEL_BUILDER(Name("AdjustHue").Device(DEVICE_GPU),
-                        AdjustHueOp<GPUDevice>);
+#define REGISTER_GPU(T)                                            \
+  REGISTER_KERNEL_BUILDER(                                         \
+      Name("AdjustHue").Device(DEVICE_GPU).TypeConstraint<T>("T"), \
+      AdjustHueOp<GPUDevice, T>);
+
+REGISTER_GPU(float)
+REGISTER_GPU(Eigen::half)
+
+#undef REGISTER_GPU
 
 #endif
 
diff --git a/tensorflow/core/kernels/adjust_hue_op.h b/tensorflow/core/kernels/adjust_hue_op.h
index 983a4072bf..6d6699de3f 100644
--- a/tensorflow/core/kernels/adjust_hue_op.h
+++ b/tensorflow/core/kernels/adjust_hue_op.h
@@ -27,10 +27,11 @@ typedef Eigen::GpuDevice GPUDevice;
 
 namespace functor {
 
+template <typename T>
 struct AdjustHueGPU {
   void operator()(GPUDevice* device, const int64 number_of_elements,
-                  const float* const input, const float* const delta,
-                  float* const output);
+                  const T* const input, const float* const delta,
+                  T* const output);
 };
 
 }  // namespace functor
diff --git a/tensorflow/core/kernels/adjust_hue_op_gpu.cu.cc b/tensorflow/core/kernels/adjust_hue_op_gpu.cu.cc
index a4fe5f755c..c30085269c 100644
--- a/tensorflow/core/kernels/adjust_hue_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/adjust_hue_op_gpu.cu.cc
@@ -24,19 +24,25 @@ namespace tensorflow {
 
 namespace functor {
 
-void AdjustHueGPU::operator()(GPUDevice* device, const int64 number_of_elements,
-                              const float* const input,
-                              const float* const delta, float* const output) {
+template <typename T>
+void AdjustHueGPU<T>::operator()(GPUDevice* device,
+                                 const int64 number_of_elements,
+                                 const T* const input, const float* const delta,
+                                 T* const output) {
   const auto stream = device->stream();
   const CudaLaunchConfig config =
       GetCudaLaunchConfig(number_of_elements, *device);
   const int threads_per_block = config.thread_per_block;
   const int block_count =
       (number_of_elements + threads_per_block - 1) / threads_per_block;
-  internal::adjust_hsv_nhwc<true, false, false>
+  internal::adjust_hsv_nhwc<true, false, false, T>
       <<<block_count, threads_per_block, 0, stream>>>(
           number_of_elements, input, output, delta, nullptr, nullptr);
 }
+
+template struct AdjustHueGPU<float>;
+template struct AdjustHueGPU<Eigen::half>;
+
 }  // namespace functor
 }  // namespace tensorflow
 #endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/adjust_saturation_op.cc b/tensorflow/core/kernels/adjust_saturation_op.cc
index f0c6ae499d..87d34fcfcc 100644
--- a/tensorflow/core/kernels/adjust_saturation_op.cc
+++ b/tensorflow/core/kernels/adjust_saturation_op.cc
@@ -81,7 +81,7 @@ class AdjustSaturationOpBase : public OpKernel {
   }
 };
 
-template <class Device>
+template <class Device, typename T>
 class AdjustSaturationOp;
 
 namespace internal {
@@ -173,7 +173,7 @@ static void hsv_to_rgb(float h, float s, float v, float* r, float* g,
 }  // namespace internal
 
 template <>
-class AdjustSaturationOp<CPUDevice> : public AdjustSaturationOpBase {
+class AdjustSaturationOp<CPUDevice, float> : public AdjustSaturationOpBase {
  public:
   explicit AdjustSaturationOp(OpKernelConstruction* context)
       : AdjustSaturationOpBase(context) {}
@@ -211,12 +211,13 @@ class AdjustSaturationOp<CPUDevice> : public AdjustSaturationOpBase {
   }
 };
 
-REGISTER_KERNEL_BUILDER(Name("AdjustSaturation").Device(DEVICE_CPU),
-                        AdjustSaturationOp<CPUDevice>);
+REGISTER_KERNEL_BUILDER(
+    Name("AdjustSaturation").Device(DEVICE_CPU).TypeConstraint<float>("T"),
+    AdjustSaturationOp<CPUDevice, float>);
 
 #if GOOGLE_CUDA
-template <>
-class AdjustSaturationOp<GPUDevice> : public AdjustSaturationOpBase {
+template <typename T>
+class AdjustSaturationOp<GPUDevice, T> : public AdjustSaturationOpBase {
  public:
   explicit AdjustSaturationOp(OpKernelConstruction* context)
       : AdjustSaturationOpBase(context) {}
@@ -231,17 +232,24 @@ class AdjustSaturationOp<GPUDevice> : public AdjustSaturationOpBase {
     const auto stream = device.stream();
     OP_REQUIRES(context, stream, errors::Internal("No GPU stream available."));
     if (number_of_elements > 0) {
-      const float* input_data = input->flat<float>().data();
+      const T* input_data = input->flat<T>().data();
       const float* scale_data = scale->flat<float>().data();
-      float* const output_data = output->flat<float>().data();
-      functor::AdjustSaturationGPU()(&device, number_of_elements, input_data,
-                                     scale_data, output_data);
+      T* const output_data = output->flat<T>().data();
+      functor::AdjustSaturationGPU<T>()(&device, number_of_elements, input_data,
+                                        scale_data, output_data);
     }
   }
 };
 
-REGISTER_KERNEL_BUILDER(Name("AdjustSaturation").Device(DEVICE_GPU),
-                        AdjustSaturationOp<GPUDevice>);
+#define REGISTER_GPU(T)                                                   \
+  REGISTER_KERNEL_BUILDER(                                                \
+      Name("AdjustSaturation").Device(DEVICE_GPU).TypeConstraint<T>("T"), \
+      AdjustSaturationOp<GPUDevice, T>);
+
+REGISTER_GPU(float)
+REGISTER_GPU(Eigen::half)
+
+#undef REGISTER_GPU
 
 #endif
 
diff --git a/tensorflow/core/kernels/adjust_saturation_op.h b/tensorflow/core/kernels/adjust_saturation_op.h
index fd28ba536f..c21ce4e360 100644
--- a/tensorflow/core/kernels/adjust_saturation_op.h
+++ b/tensorflow/core/kernels/adjust_saturation_op.h
@@ -27,10 +27,11 @@ typedef Eigen::GpuDevice GPUDevice;
 
 namespace functor {
 
+template <typename T>
 struct AdjustSaturationGPU {
   void operator()(GPUDevice* device, const int64 number_of_elements,
-                  const float* const input, const float* const scale,
-                  float* const output);
+                  const T* const input, const float* const scale,
+                  T* const output);
 };
 
 }  // namespace functor
diff --git a/tensorflow/core/kernels/adjust_saturation_op_gpu.cu.cc b/tensorflow/core/kernels/adjust_saturation_op_gpu.cu.cc
index 37cfb26a47..6c70490d46 100644
--- a/tensorflow/core/kernels/adjust_saturation_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/adjust_saturation_op_gpu.cu.cc
@@ -24,21 +24,26 @@ namespace tensorflow {
 
 namespace functor {
 
-void AdjustSaturationGPU::operator()(GPUDevice* device,
-                                     const int64 number_of_elements,
-                                     const float* const input,
-                                     const float* const scale,
-                                     float* const output) {
+template <typename T>
+void AdjustSaturationGPU<T>::operator()(GPUDevice* device,
+                                        const int64 number_of_elements,
+                                        const T* const input,
+                                        const float* const scale,
+                                        T* const output) {
   const auto stream = device->stream();
   const CudaLaunchConfig config =
       GetCudaLaunchConfig(number_of_elements, *device);
   const int threads_per_block = config.thread_per_block;
   const int block_count =
       (number_of_elements + threads_per_block - 1) / threads_per_block;
-  internal::adjust_hsv_nhwc<false, true, false>
+  internal::adjust_hsv_nhwc<false, true, false, T>
       <<<block_count, threads_per_block, 0, stream>>>(
           number_of_elements, input, output, nullptr, scale, nullptr);
 }
+
+template struct AdjustSaturationGPU<float>;
+template struct AdjustSaturationGPU<Eigen::half>;
+
 }  // namespace functor
 }  // namespace tensorflow
 #endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/ops/image_ops.cc b/tensorflow/core/ops/image_ops.cc
index ee8b1e58d6..0f1555f49c 100644
--- a/tensorflow/core/ops/image_ops.cc
+++ b/tensorflow/core/ops/image_ops.cc
@@ -408,9 +408,10 @@ REGISTER_OP("AdjustContrast")
 
 // --------------------------------------------------------------------------
 REGISTER_OP("AdjustContrastv2")
-    .Input("images: float")
+    .Input("images: T")
     .Input("contrast_factor: float")
-    .Output("output: float")
+    .Output("output: T")
+    .Attr("T: {half, float} = DT_FLOAT")
     .SetShapeFn([](InferenceContext* c) {
       // The contrast_factor should be scalar only.
       ShapeHandle unused;
@@ -420,18 +421,20 @@ REGISTER_OP("AdjustContrastv2")
 
 // --------------------------------------------------------------------------
 REGISTER_OP("AdjustHue")
-    .Input("images: float")
+    .Input("images: T")
     .Input("delta: float")
-    .Output("output: float")
+    .Output("output: T")
+    .Attr("T: {half, float} = DT_FLOAT")
     .SetShapeFn([](InferenceContext* c) {
       return shape_inference::UnchangedShapeWithRankAtLeast(c, 3);
     });
 
 // --------------------------------------------------------------------------
 REGISTER_OP("AdjustSaturation")
-    .Input("images: float")
+    .Input("images: T")
     .Input("scale: float")
-    .Output("output: float")
+    .Output("output: T")
+    .Attr("T: {half, float} = DT_FLOAT")
     .SetShapeFn([](InferenceContext* c) {
       return shape_inference::UnchangedShapeWithRankAtLeast(c, 3);
     });
diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index 24d049b726..8047743cfa 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -1250,14 +1250,14 @@ def random_brightness(image, max_delta, seed=None):
   interval `[-max_delta, max_delta)`.
 
   Args:
-    image: An image.
+    image: An image or images to adjust.
     max_delta: float, must be non-negative.
     seed: A Python integer. Used to create a random seed. See
       `tf.set_random_seed`
       for behavior.
 
   Returns:
-    The brightness-adjusted image.
+    The brightness-adjusted image(s).
 
   Raises:
     ValueError: if `max_delta` is negative.
@@ -1271,7 +1271,7 @@ def random_brightness(image, max_delta, seed=None):
 
 @tf_export('image.random_contrast')
 def random_contrast(image, lower, upper, seed=None):
-  """Adjust the contrast of an image by a random factor.
+  """Adjust the contrast of an image or images by a random factor.
 
   Equivalent to `adjust_contrast()` but uses a `contrast_factor` randomly
   picked in the interval `[lower, upper]`.
@@ -1281,11 +1281,10 @@ def random_contrast(image, lower, upper, seed=None):
     lower: float.  Lower bound for the random contrast factor.
     upper: float.  Upper bound for the random contrast factor.
     seed: A Python integer. Used to create a random seed. See
-      `tf.set_random_seed`
-      for behavior.
+      `tf.set_random_seed` for behavior.
 
   Returns:
-    The contrast-adjusted tensor.
+    The contrast-adjusted image(s).
 
   Raises:
     ValueError: if `upper <= lower` or if `lower < 0`.
@@ -1305,19 +1304,19 @@ def random_contrast(image, lower, upper, seed=None):
 def adjust_brightness(image, delta):
   """Adjust the brightness of RGB or Grayscale images.
 
-  This is a convenience method that converts an RGB image to float
-  representation, adjusts its brightness, and then converts it back to the
-  original data type. If several adjustments are chained it is advisable to
+  This is a convenience method that converts RGB images to float
+  representation, adjusts their brightness, and then converts them back to the
+  original data type. If several adjustments are chained, it is advisable to
   minimize the number of redundant conversions.
 
-  The value `delta` is added to all components of the tensor `image`. Both
-  `image` and `delta` are converted to `float` before adding (and `image` is
-  scaled appropriately if it is in fixed-point representation). For regular
+  The value `delta` is added to all components of the tensor `image`. `image` is
+  converted to `float` and scaled appropriately if it is in fixed-point
+  representation, and `delta` is converted to the same data type. For regular
   images, `delta` should be in the range `[0,1)`, as it is added to the image in
   floating point representation, where pixel values are in the `[0,1)` range.
 
   Args:
-    image: A tensor.
+    image: RGB image or images to adjust.
     delta: A scalar. Amount to add to the pixel values.
 
   Returns:
@@ -1327,10 +1326,14 @@ def adjust_brightness(image, delta):
     image = ops.convert_to_tensor(image, name='image')
     # Remember original dtype to so we can convert back if needed
     orig_dtype = image.dtype
-    flt_image = convert_image_dtype(image, dtypes.float32)
+
+    if orig_dtype in [dtypes.float16, dtypes.float32]:
+      flt_image = image
+    else:
+      flt_image = convert_image_dtype(image, dtypes.float32)
 
     adjusted = math_ops.add(
-        flt_image, math_ops.cast(delta, dtypes.float32), name=name)
+        flt_image, math_ops.cast(delta, flt_image.dtype), name=name)
 
     return convert_image_dtype(adjusted, orig_dtype, saturate=True)
 
@@ -1339,9 +1342,9 @@ def adjust_brightness(image, delta):
 def adjust_contrast(images, contrast_factor):
   """Adjust contrast of RGB or grayscale images.
 
-  This is a convenience method that converts an RGB image to float
-  representation, adjusts its contrast, and then converts it back to the
-  original data type. If several adjustments are chained it is advisable to
+  This is a convenience method that converts RGB images to float
+  representation, adjusts their contrast, and then converts them back to the
+  original data type. If several adjustments are chained, it is advisable to
   minimize the number of redundant conversions.
 
   `images` is a tensor of at least 3 dimensions.  The last 3 dimensions are
@@ -1366,7 +1369,11 @@ def adjust_contrast(images, contrast_factor):
     images = ops.convert_to_tensor(images, name='images')
     # Remember original dtype to so we can convert back if needed
     orig_dtype = images.dtype
-    flt_images = convert_image_dtype(images, dtypes.float32)
+
+    if orig_dtype in (dtypes.float16, dtypes.float32):
+      flt_images = images
+    else:
+      flt_images = convert_image_dtype(images, dtypes.float32)
 
     adjusted = gen_image_ops.adjust_contrastv2(
         flt_images, contrast_factor=contrast_factor, name=name)
@@ -1560,7 +1567,7 @@ def grayscale_to_rgb(images, name=None):
 # pylint: disable=invalid-name
 @tf_export('image.random_hue')
 def random_hue(image, max_delta, seed=None):
-  """Adjust the hue of an RGB image by a random factor.
+  """Adjust the hue of RGB images by a random factor.
 
   Equivalent to `adjust_hue()` but uses a `delta` randomly
   picked in the interval `[-max_delta, max_delta]`.
@@ -1570,10 +1577,10 @@ def random_hue(image, max_delta, seed=None):
   Args:
     image: RGB image or images. Size of the last dimension must be 3.
     max_delta: float.  Maximum value for the random delta.
-    seed: An operation-specific seed. It will be used in conjunction
-      with the graph-level seed to determine the real seeds that will be
-      used in this operation. Please see the documentation of
-      set_random_seed for its interaction with the graph-level random seed.
+    seed: An operation-specific seed. It will be used in conjunction with the
+      graph-level seed to determine the real seeds that will be used in this
+      operation. Please see the documentation of set_random_seed for its
+      interaction with the graph-level random seed.
 
   Returns:
     Adjusted image(s), same shape and DType as `image`.
@@ -1593,7 +1600,7 @@ def random_hue(image, max_delta, seed=None):
 
 @tf_export('image.adjust_hue')
 def adjust_hue(image, delta, name=None):
-  """Adjust hue of an RGB image.
+  """Adjust hue of RGB images.
 
   This is a convenience method that converts an RGB image to float
   representation, converts it to HSV, add an offset to the hue channel, converts
@@ -1601,7 +1608,7 @@ def adjust_hue(image, delta, name=None):
   are chained it is advisable to minimize the number of redundant conversions.
 
   `image` is an RGB image.  The image hue is adjusted by converting the
-  image to HSV and rotating the hue channel (H) by
+  image(s) to HSV and rotating the hue channel (H) by
   `delta`.  The image is then converted back to RGB.
 
   `delta` must be in the interval `[-1, 1]`.
@@ -1618,7 +1625,10 @@ def adjust_hue(image, delta, name=None):
     image = ops.convert_to_tensor(image, name='image')
     # Remember original dtype to so we can convert back if needed
     orig_dtype = image.dtype
-    flt_image = convert_image_dtype(image, dtypes.float32)
+    if orig_dtype in (dtypes.float16, dtypes.float32):
+      flt_image = image
+    else:
+      flt_image = convert_image_dtype(image, dtypes.float32)
 
     rgb_altered = gen_image_ops.adjust_hue(flt_image, delta)
 
@@ -1696,7 +1706,7 @@ def adjust_jpeg_quality(image, jpeg_quality, name=None):
 
 @tf_export('image.random_saturation')
 def random_saturation(image, lower, upper, seed=None):
-  """Adjust the saturation of an RGB image by a random factor.
+  """Adjust the saturation of RGB images by a random factor.
 
   Equivalent to `adjust_saturation()` but uses a `saturation_factor` randomly
   picked in the interval `[lower, upper]`.
@@ -1705,10 +1715,10 @@ def random_saturation(image, lower, upper, seed=None):
     image: RGB image or images. Size of the last dimension must be 3.
     lower: float.  Lower bound for the random saturation factor.
     upper: float.  Upper bound for the random saturation factor.
-    seed: An operation-specific seed. It will be used in conjunction
-      with the graph-level seed to determine the real seeds that will be
-      used in this operation. Please see the documentation of
-      set_random_seed for its interaction with the graph-level random seed.
+    seed: An operation-specific seed. It will be used in conjunction with the
+      graph-level seed to determine the real seeds that will be used in this
+      operation. Please see the documentation of set_random_seed for its
+      interaction with the graph-level random seed.
 
   Returns:
     Adjusted image(s), same shape and DType as `image`.
@@ -1729,17 +1739,17 @@ def random_saturation(image, lower, upper, seed=None):
 
 @tf_export('image.adjust_saturation')
 def adjust_saturation(image, saturation_factor, name=None):
-  """Adjust saturation of an RGB image.
+  """Adjust saturation of RGB images.
 
-  This is a convenience method that converts an RGB image to float
-  representation, converts it to HSV, add an offset to the saturation channel,
+  This is a convenience method that converts RGB images to float
+  representation, converts them to HSV, add an offset to the saturation channel,
   converts back to RGB and then back to the original data type. If several
   adjustments are chained it is advisable to minimize the number of redundant
   conversions.
 
-  `image` is an RGB image.  The image saturation is adjusted by converting the
-  image to HSV and multiplying the saturation (S) channel by
-  `saturation_factor` and clipping. The image is then converted back to RGB.
+  `image` is an RGB image or images.  The image saturation is adjusted by
+  converting the images to HSV and multiplying the saturation (S) channel by
+  `saturation_factor` and clipping. The images are then converted back to RGB.
 
   Args:
     image: RGB image or images. Size of the last dimension must be 3.
@@ -1753,11 +1763,14 @@ def adjust_saturation(image, saturation_factor, name=None):
     image = ops.convert_to_tensor(image, name='image')
     # Remember original dtype to so we can convert back if needed
     orig_dtype = image.dtype
-    flt_image = convert_image_dtype(image, dtypes.float32)
+    if orig_dtype in (dtypes.float16, dtypes.float32):
+      flt_image = image
+    else:
+      flt_image = convert_image_dtype(image, dtypes.float32)
+
+    adjusted = gen_image_ops.adjust_saturation(flt_image, saturation_factor)
 
-    return convert_image_dtype(
-        gen_image_ops.adjust_saturation(flt_image, saturation_factor),
-        orig_dtype)
+    return convert_image_dtype(adjusted, orig_dtype)
 
 
 @tf_export('io.is_jpeg', 'image.is_jpeg', v1=['io.is_jpeg', 'image.is_jpeg'])
diff --git a/tensorflow/python/ops/image_ops_test.py b/tensorflow/python/ops/image_ops_test.py
index e7249333bd..361befabce 100644
--- a/tensorflow/python/ops/image_ops_test.py
+++ b/tensorflow/python/ops/image_ops_test.py
@@ -886,44 +886,6 @@ class AdjustSaturationTest(test_util.TensorFlowTestCase):
       y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, y_np)
 
-  def _adjust_saturation(self, image, saturation_factor):
-    image = ops.convert_to_tensor(image, name="image")
-    orig_dtype = image.dtype
-    flt_image = image_ops.convert_image_dtype(image, dtypes.float32)
-    saturation_adjusted_image = gen_image_ops.adjust_saturation(
-        flt_image, saturation_factor)
-    return image_ops.convert_image_dtype(saturation_adjusted_image, orig_dtype)
-
-  def testHalfSaturationFused(self):
-    x_shape = [2, 2, 3]
-    x_rgb_data = [0, 5, 13, 54, 135, 226, 37, 8, 234, 90, 255, 1]
-    x_np = np.array(x_rgb_data, dtype=np.uint8).reshape(x_shape)
-
-    saturation_factor = 0.5
-    y_rgb_data = [6, 9, 13, 140, 180, 226, 135, 121, 234, 172, 255, 128]
-    y_np = np.array(y_rgb_data, dtype=np.uint8).reshape(x_shape)
-
-    with self.test_session(use_gpu=True):
-      x = constant_op.constant(x_np, shape=x_shape)
-      y = self._adjust_saturation(x, saturation_factor)
-      y_tf = self.evaluate(y)
-      self.assertAllEqual(y_tf, y_np)
-
-  def testTwiceSaturationFused(self):
-    x_shape = [2, 2, 3]
-    x_data = [0, 5, 13, 54, 135, 226, 37, 8, 234, 90, 255, 1]
-    x_np = np.array(x_data, dtype=np.uint8).reshape(x_shape)
-
-    saturation_factor = 2.0
-    y_data = [0, 5, 13, 0, 106, 226, 30, 0, 234, 89, 255, 0]
-    y_np = np.array(y_data, dtype=np.uint8).reshape(x_shape)
-
-    with self.test_session(use_gpu=True):
-      x = constant_op.constant(x_np, shape=x_shape)
-      y = self._adjust_saturation(x, saturation_factor)
-      y_tf = self.evaluate(y)
-      self.assertAllEqual(y_tf, y_np)
-
   def _adjustSaturationNp(self, x_np, scale):
     self.assertEqual(x_np.shape[-1], 3)
     x_v = x_np.reshape([-1, 3])
@@ -977,7 +939,7 @@ class AdjustSaturationTest(test_util.TensorFlowTestCase):
           else:
             raise AssertionError("Invalid test style: %s" % (test_style))
           y_baseline = self._adjustSaturationNp(x_np, scale)
-          y_fused = self._adjust_saturation(x_np, scale).eval()
+          y_fused = image_ops.adjust_saturation(x_np, scale).eval()
           self.assertAllClose(y_fused, y_baseline, rtol=2e-5, atol=1e-5)
 
 
@@ -1438,12 +1400,12 @@ class AdjustContrastTest(test_util.TensorFlowTestCase):
 
 class AdjustBrightnessTest(test_util.TensorFlowTestCase):
 
-  def _testBrightness(self, x_np, y_np, delta):
+  def _testBrightness(self, x_np, y_np, delta, tol=1e-6):
     with self.test_session(use_gpu=True):
       x = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.adjust_brightness(x, delta)
       y_tf = self.evaluate(y)
-      self.assertAllClose(y_tf, y_np, 1e-6)
+      self.assertAllClose(y_tf, y_np, tol)
 
   def testPositiveDeltaUint8(self):
     x_shape = [2, 2, 3]
@@ -1455,7 +1417,7 @@ class AdjustBrightnessTest(test_util.TensorFlowTestCase):
 
     self._testBrightness(x_np, y_np, delta=10. / 255.)
 
-  def testPositiveDeltaFloat(self):
+  def testPositiveDeltaFloat32(self):
     x_shape = [2, 2, 3]
     x_data = [0, 5, 13, 54, 135, 226, 37, 8, 234, 90, 255, 1]
     x_np = np.array(x_data, dtype=np.float32).reshape(x_shape) / 255.
@@ -1465,6 +1427,16 @@ class AdjustBrightnessTest(test_util.TensorFlowTestCase):
 
     self._testBrightness(x_np, y_np, delta=10. / 255.)
 
+  def testPositiveDeltaFloat16(self):
+    x_shape = [2, 2, 3]
+    x_data = [0, 5, 13, 54, 135, 226, 37, 8, 234, 90, 255, 1]
+    x_np = np.array(x_data, dtype=np.float16).reshape(x_shape) / 255.
+
+    y_data = [10, 15, 23, 64, 145, 236, 47, 18, 244, 100, 265, 11]
+    y_np = np.array(y_data, dtype=np.float16).reshape(x_shape) / 255.
+
+    self._testBrightness(x_np, y_np, delta=10. / 255., tol=1e-3)
+
   def testNegativeDelta(self):
     x_shape = [2, 2, 3]
     x_data = [0, 5, 13, 54, 135, 226, 37, 8, 234, 90, 255, 1]
-- 
GitLab


From 43bb4d1f6d3d91a2f0019d187204f665f52c3955 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Tue, 15 Jan 2019 12:49:03 -0800
Subject: [PATCH 0742/2345] Avoid using a temp directory with a space to
 workaround bazel errors.

PiperOrigin-RevId: 229422438
---
 tensorflow/tools/ci_build/windows/libtensorflow_gpu.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/windows/libtensorflow_gpu.sh b/tensorflow/tools/ci_build/windows/libtensorflow_gpu.sh
index 7dfee8f371..9c05db974b 100644
--- a/tensorflow/tools/ci_build/windows/libtensorflow_gpu.sh
+++ b/tensorflow/tools/ci_build/windows/libtensorflow_gpu.sh
@@ -41,7 +41,7 @@ run_configure_for_gpu_build
 # build_libtensorflow_tarball in ../builds/libtensorflow.sh
 # cannot be used on Windows since it relies on pkg_tar rules.
 # So we do something special here
-bazel build -c opt --copt=/arch:AVX --announce_rc \
+bazel --output_user_root=${TMPDIR} build -c opt --copt=/arch:AVX --announce_rc \
   tensorflow:libtensorflow.so \
   tensorflow/tools/lib_package:clicenses_generate \
   tensorflow/java:libtensorflow_jni.so \
-- 
GitLab


From 0336900e0c9b9047444027a31bdbf5bae4fa3695 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 15 Jan 2019 12:59:27 -0800
Subject: [PATCH 0743/2345] Make XlaCompiledCpuFunction::StaticData
 inaccessible; NFC

This CL will prevent tfcompile users from accidentally depending on StaticData
internals.

PiperOrigin-RevId: 229424173
---
 tensorflow/compiler/aot/codegen.cc            |  28 +++--
 tensorflow/compiler/aot/codegen_test_h.golden |  23 ++--
 .../tf2xla/xla_compiled_cpu_function.h        | 116 ++++++++++++------
 .../tf2xla/xla_jit_compiled_cpu_function.cc   |  34 +++--
 4 files changed, 126 insertions(+), 75 deletions(-)

diff --git a/tensorflow/compiler/aot/codegen.cc b/tensorflow/compiler/aot/codegen.cc
index 347a365dcb..d016632da2 100644
--- a/tensorflow/compiler/aot/codegen.cc
+++ b/tensorflow/compiler/aot/codegen.cc
@@ -384,8 +384,9 @@ Status GenerateHeader(const CodegenOpts& opts, const tf2xla::Config& config,
   // calling HloProfilePrinter::profile_counters_size.
   const string assign_profile_counters_size =
       opts.gen_hlo_profile_printer_data
-          ? "data->set_profile_counters_size("
-            "data->hlo_profile_printer_data()->profile_counters_size());"
+          ? "set_static_data_profile_counters_size(data, "
+            "get_static_data_hlo_profile_printer_data(data)->"
+            "profile_counters_size());"
           : "";
 
   // Use a poor-man's text templating mechanism; first populate the full header
@@ -449,7 +450,7 @@ extern "C" void {{ENTRY}}(
 //   arg bytes aligned:  {{ARG_BYTES_ALIGNED}}
 //   temp bytes total:   {{TEMP_BYTES_TOTAL}}
 //   temp bytes aligned: {{TEMP_BYTES_ALIGNED}}
-class {{CLASS}} : public tensorflow::XlaCompiledCpuFunction {
+class {{CLASS}} final : public tensorflow::XlaCompiledCpuFunction {
  public:
   // Number of input arguments for the compiled computation.
   static constexpr size_t kNumArgs = {{ARG_NUM}};
@@ -464,16 +465,17 @@ class {{CLASS}} : public tensorflow::XlaCompiledCpuFunction {
     static XlaCompiledCpuFunction::StaticData* kStaticData = [](){
       XlaCompiledCpuFunction::StaticData* data =
         new XlaCompiledCpuFunction::StaticData;
-      data->set_raw_function({{ENTRY}});
-      data->set_buffer_infos(BufferInfos());
-      data->set_num_buffers(kNumBuffers);
-      data->set_arg_index_table(ArgIndexToBufferIndex());
-      data->set_num_args(kNumArgs);
-      data->set_result_index(kResultIndex);
-      data->set_arg_names(StaticArgNames());
-      data->set_result_names(StaticResultNames());
-      data->set_program_shape(StaticProgramShape());
-      data->set_hlo_profile_printer_data(StaticHloProfilePrinterData());
+      set_static_data_raw_function(data, {{ENTRY}});
+      set_static_data_buffer_infos(data, BufferInfos());
+      set_static_data_num_buffers(data, kNumBuffers);
+      set_static_data_arg_index_table(data, ArgIndexToBufferIndex());
+      set_static_data_num_args(data, kNumArgs);
+      set_static_data_result_index(data, kResultIndex);
+      set_static_data_arg_names(data, StaticArgNames());
+      set_static_data_result_names(data, StaticResultNames());
+      set_static_data_program_shape(data, StaticProgramShape());
+      set_static_data_hlo_profile_printer_data(
+          data, StaticHloProfilePrinterData());
 {{ASSIGN_PROFILE_COUNTERS_SIZE}}
       return data;
     }();
diff --git a/tensorflow/compiler/aot/codegen_test_h.golden b/tensorflow/compiler/aot/codegen_test_h.golden
index 7fbf83af33..35994fc785 100644
--- a/tensorflow/compiler/aot/codegen_test_h.golden
+++ b/tensorflow/compiler/aot/codegen_test_h.golden
@@ -59,7 +59,7 @@ namespace bar {
 //   arg bytes aligned:  192
 //   temp bytes total:   126
 //   temp bytes aligned: 320
-class MyClass : public tensorflow::XlaCompiledCpuFunction {
+class MyClass final : public tensorflow::XlaCompiledCpuFunction {
  public:
   // Number of input arguments for the compiled computation.
   static constexpr size_t kNumArgs = 2;
@@ -74,16 +74,17 @@ class MyClass : public tensorflow::XlaCompiledCpuFunction {
     static XlaCompiledCpuFunction::StaticData* kStaticData = [](){
       XlaCompiledCpuFunction::StaticData* data =
         new XlaCompiledCpuFunction::StaticData;
-      data->set_raw_function(entry_point);
-      data->set_buffer_infos(BufferInfos());
-      data->set_num_buffers(kNumBuffers);
-      data->set_arg_index_table(ArgIndexToBufferIndex());
-      data->set_num_args(kNumArgs);
-      data->set_result_index(kResultIndex);
-      data->set_arg_names(StaticArgNames());
-      data->set_result_names(StaticResultNames());
-      data->set_program_shape(StaticProgramShape());
-      data->set_hlo_profile_printer_data(StaticHloProfilePrinterData());
+      set_static_data_raw_function(data, entry_point);
+      set_static_data_buffer_infos(data, BufferInfos());
+      set_static_data_num_buffers(data, kNumBuffers);
+      set_static_data_arg_index_table(data, ArgIndexToBufferIndex());
+      set_static_data_num_args(data, kNumArgs);
+      set_static_data_result_index(data, kResultIndex);
+      set_static_data_arg_names(data, StaticArgNames());
+      set_static_data_result_names(data, StaticResultNames());
+      set_static_data_program_shape(data, StaticProgramShape());
+      set_static_data_hlo_profile_printer_data(
+          data, StaticHloProfilePrinterData());
 
       return data;
     }();
diff --git a/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h b/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h
index c7341cf8b9..e1da19cf1d 100644
--- a/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h
+++ b/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h
@@ -59,45 +59,8 @@ class XlaCompiledCpuFunction {
   // AOT this is backed by data compiled into the object file.
   //
   // The contents of StaticData are XLA-internal implementation details and
-  // should not be relied on by clients.
-  //
-  // TODO(sanjoy): Come up with a cleaner way to express the contraint we want
-  // here: generated XlaCompiledCpuFunction subclasses should be able to create
-  // instances of StaticData but only XlaCompiledCpuFunction should be able to
-  // read from StaticData instances.
+  // should not be relied on by clients (and therefore are private).
   class StaticData {
-   public:
-    void set_raw_function(RawFunction raw_function) {
-      raw_function_ = raw_function;
-    }
-    void set_buffer_infos(
-        const cpu_function_runtime::BufferInfo* buffer_infos) {
-      buffer_infos_ = buffer_infos;
-    }
-    void set_num_buffers(size_t num_buffers) { num_buffers_ = num_buffers; }
-    void set_arg_index_table(const int32* arg_index_table) {
-      arg_index_table_ = arg_index_table;
-    }
-    void set_num_args(int64 num_args) { num_args_ = num_args; }
-    void set_result_index(size_t result_index) { result_index_ = result_index; }
-    void set_arg_names(const char** arg_names) { arg_names_ = arg_names; }
-    void set_result_names(const char** result_names) {
-      result_names_ = result_names;
-    }
-    void set_program_shape(const xla::ProgramShapeProto* program_shape) {
-      program_shape_ = program_shape;
-    }
-    const xla::HloProfilePrinterData* hlo_profile_printer_data() const {
-      return hlo_profile_printer_data_;
-    }
-    void set_hlo_profile_printer_data(
-        const xla::HloProfilePrinterData* hlo_profile_printer_data) {
-      hlo_profile_printer_data_ = hlo_profile_printer_data;
-    }
-    void set_profile_counters_size(int64 profile_counters_size) {
-      profile_counters_size_ = profile_counters_size;
-    }
-
    private:
     // The raw function to call.
     RawFunction raw_function_;
@@ -134,7 +97,8 @@ class XlaCompiledCpuFunction {
     // declared so we don't have access to that information here.
     int64 profile_counters_size_ = 0;
 
-    // Only XlaCompiledCpuFunction is allowed to read the above fields.
+    // Only XlaCompiledCpuFunction is allowed to read and write the above
+    // fields.
     friend class XlaCompiledCpuFunction;
   };
 
@@ -280,6 +244,76 @@ class XlaCompiledCpuFunction {
     return *hlo_profile_printer_data_;
   }
 
+ protected:
+  // ---------------------------------------------------------------------------
+  // Accessors for reading from and writing to instances of `StaticData`.
+  //
+  // Classes generated by tfcompile can call these because the generated classes
+  // inherit from `XlaCompiledCpuFunction`.  `XlaJitCompiledCpuFunction` can
+  // call these because it is explicitly added as a friend.
+
+  static void set_static_data_raw_function(StaticData* static_data,
+                                           RawFunction raw_function) {
+    static_data->raw_function_ = raw_function;
+  }
+
+  static void set_static_data_buffer_infos(
+      StaticData* static_data,
+      const cpu_function_runtime::BufferInfo* buffer_infos) {
+    static_data->buffer_infos_ = buffer_infos;
+  }
+
+  static void set_static_data_num_buffers(StaticData* static_data,
+                                          size_t num_buffers) {
+    static_data->num_buffers_ = num_buffers;
+  }
+
+  static void set_static_data_arg_index_table(StaticData* static_data,
+                                              const int32* arg_index_table) {
+    static_data->arg_index_table_ = arg_index_table;
+  }
+
+  static void set_static_data_num_args(StaticData* static_data,
+                                       int64 num_args) {
+    static_data->num_args_ = num_args;
+  }
+
+  static void set_static_data_result_index(StaticData* static_data,
+                                           size_t result_index) {
+    static_data->result_index_ = result_index;
+  }
+
+  static void set_static_data_arg_names(StaticData* static_data,
+                                        const char** arg_names) {
+    static_data->arg_names_ = arg_names;
+  }
+
+  static void set_static_data_result_names(StaticData* static_data,
+                                           const char** result_names) {
+    static_data->result_names_ = result_names;
+  }
+
+  static void set_static_data_program_shape(
+      StaticData* static_data, const xla::ProgramShapeProto* program_shape) {
+    static_data->program_shape_ = program_shape;
+  }
+
+  static void set_static_data_hlo_profile_printer_data(
+      StaticData* static_data,
+      const xla::HloProfilePrinterData* hlo_profile_printer_data) {
+    static_data->hlo_profile_printer_data_ = hlo_profile_printer_data;
+  }
+
+  static const xla::HloProfilePrinterData*
+  get_static_data_hlo_profile_printer_data(StaticData* static_data) {
+    return static_data->hlo_profile_printer_data_;
+  }
+
+  static void set_static_data_profile_counters_size(
+      StaticData* static_data, int64 profile_counters_size) {
+    static_data->profile_counters_size_ = profile_counters_size;
+  }
+
  private:
   const RawFunction raw_function_;
   const size_t result_index_;
@@ -313,6 +347,10 @@ class XlaCompiledCpuFunction {
   const char** result_names_ = nullptr;
   const xla::ProgramShapeProto* program_shape_ = nullptr;
   const xla::HloProfilePrinterData* hlo_profile_printer_data_ = nullptr;
+
+  // Add `XlaJitCompiledCpuFunction` as a friend so that it can access the
+  // `set_static_data_*` static methods above.
+  friend class XlaJitCompiledCpuFunction;
 };
 
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function.cc b/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function.cc
index fabbcd04fe..884dc45cb1 100644
--- a/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function.cc
+++ b/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function.cc
@@ -135,24 +135,34 @@ XlaJitCompiledCpuFunction::Compile(
   jit->arg_index_table_ = std::move(arg_index_table);
   jit->program_shape_ =
       absl::make_unique<xla::ProgramShapeProto>(program_shape->ToProto());
-  jit->static_data_.set_raw_function(raw_function);
-  jit->static_data_.set_buffer_infos(jit->buffer_infos_.data());
-  jit->static_data_.set_num_buffers(jit->buffer_infos_.size());
-  jit->static_data_.set_arg_index_table(jit->arg_index_table_.data());
-  jit->static_data_.set_num_args(jit->arg_index_table_.size());
-  jit->static_data_.set_result_index(result_index);
+  XlaCompiledCpuFunction::set_static_data_raw_function(&jit->static_data_,
+                                                       raw_function);
+  XlaCompiledCpuFunction::set_static_data_buffer_infos(
+      &jit->static_data_, jit->buffer_infos_.data());
+  XlaCompiledCpuFunction::set_static_data_num_buffers(
+      &jit->static_data_, jit->buffer_infos_.size());
+  XlaCompiledCpuFunction::set_static_data_arg_index_table(
+      &jit->static_data_, jit->arg_index_table_.data());
+  XlaCompiledCpuFunction::set_static_data_num_args(
+      &jit->static_data_, jit->arg_index_table_.size());
+  XlaCompiledCpuFunction::set_static_data_result_index(&jit->static_data_,
+                                                       result_index);
   // Optional metadata is collected and set below.
   CollectNames(config.feed(), &jit->nonempty_arg_names_, &jit->arg_names_);
   CollectNames(config.fetch(), &jit->nonempty_result_names_,
                &jit->result_names_);
-  jit->static_data_.set_arg_names(jit->arg_names_.data());
-  jit->static_data_.set_result_names(jit->result_names_.data());
-  jit->static_data_.set_program_shape(jit->program_shape_.get());
+  XlaCompiledCpuFunction::set_static_data_arg_names(&jit->static_data_,
+                                                    jit->arg_names_.data());
+  XlaCompiledCpuFunction::set_static_data_result_names(
+      &jit->static_data_, jit->result_names_.data());
+  XlaCompiledCpuFunction::set_static_data_program_shape(
+      &jit->static_data_, jit->program_shape_.get());
 
   if (cpu_executable->hlo_profiling_enabled()) {
-    jit->static_data_.set_hlo_profile_printer_data(
-        &cpu_executable->hlo_profile_printer_data());
-    jit->static_data_.set_profile_counters_size(
+    XlaCompiledCpuFunction::set_static_data_hlo_profile_printer_data(
+        &jit->static_data_, &cpu_executable->hlo_profile_printer_data());
+    XlaCompiledCpuFunction::set_static_data_profile_counters_size(
+        &jit->static_data_,
         cpu_executable->hlo_profile_printer_data().profile_counters_size());
   }
 
-- 
GitLab


From eadd8aca53dedb40f5664d616288db665dd95652 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Tue, 15 Jan 2019 13:06:12 -0800
Subject: [PATCH 0744/2345] Making sure dataset "output" ops are not pruned
 from function graphs as they might have side effects even though the ops are
 not marked as stateful.

PiperOrigin-RevId: 229425577
---
 tensorflow/core/common_runtime/function.cc | 2 +-
 tensorflow/core/graph/graph.cc             | 4 ++++
 tensorflow/core/graph/graph.h              | 3 +++
 3 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/common_runtime/function.cc b/tensorflow/core/common_runtime/function.cc
index 48f32df275..3df09eac29 100644
--- a/tensorflow/core/common_runtime/function.cc
+++ b/tensorflow/core/common_runtime/function.cc
@@ -812,7 +812,7 @@ void PruneFunctionBody(Graph* g) {
     // TODO(mrry): Investigate whether the `n->IsControlFlow()` test is
     // still needed. It would be preferable to prune entire loops and/or
     // conditionals if they are not used in the graph.
-    if (n->IsControlFlow() ||
+    if (n->IsControlFlow() || n->IsDataset() ||
         (n->op_def().is_stateful() && n->type_string() != kArgOp)) {
       nodes.insert(n);
     }
diff --git a/tensorflow/core/graph/graph.cc b/tensorflow/core/graph/graph.cc
index 3ea222c13c..00d3549312 100644
--- a/tensorflow/core/graph/graph.cc
+++ b/tensorflow/core/graph/graph.cc
@@ -85,6 +85,10 @@ const std::unordered_map<string, Node::NodeClass>& Node::kNodeClassTable =
         {"CollectiveBcastSend", NC_COLLECTIVE},
         {"CollectiveBcastRecv", NC_COLLECTIVE},
         {"FakeParam", NC_FAKE_PARAM},
+        {"IteratorGetNext", NC_DATASET},
+        {"IteratorGetNextSync", NC_DATASET},
+        {"DatasetToSingleElement", NC_DATASET},
+        {"ReduceDataset", NC_DATASET},
     });
 
 #undef REF_CLASS
diff --git a/tensorflow/core/graph/graph.h b/tensorflow/core/graph/graph.h
index 289a3d2a23..f65e4b921e 100644
--- a/tensorflow/core/graph/graph.h
+++ b/tensorflow/core/graph/graph.h
@@ -174,6 +174,8 @@ class Node {
   bool IsMetadata() const { return class_ == NC_METADATA; }
   bool IsFakeParam() const { return class_ == NC_FAKE_PARAM; }
 
+  bool IsDataset() const { return class_ == NC_DATASET; }
+
   template <typename T>
   void AddAttr(const string& name, const T& val) {
     SetAttrValue(val, AddAttrHelper(name));
@@ -254,6 +256,7 @@ class Node {
     NC_SCOPED_ALLOCATOR,
     NC_COLLECTIVE,
     NC_FAKE_PARAM,
+    NC_DATASET,
     NC_OTHER  // Not a special kind of node
   };
 
-- 
GitLab


From 75d9981d6c79b178340c0e011921f4b407f59341 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 15 Jan 2019 13:09:06 -0800
Subject: [PATCH 0745/2345] Add LSTM and embedding models to keras correctness
 test.

PiperOrigin-RevId: 229426039
---
 tensorflow/contrib/distribute/python/BUILD    | 40 ++++++++++
 .../contrib/distribute/python/combinations.py | 14 +++-
 .../python/keras_correctness_test_base.py     | 75 +++++++++++++++++--
 .../python/keras_dnn_correctness_test.py      | 20 +++--
 .../keras_embedding_model_correctness_test.py | 75 +++++++++++++++++++
 .../keras_lstm_model_correctness_test.py      | 65 ++++++++++++++++
 6 files changed, 273 insertions(+), 16 deletions(-)
 create mode 100644 tensorflow/contrib/distribute/python/keras_embedding_model_correctness_test.py
 create mode 100644 tensorflow/contrib/distribute/python/keras_lstm_model_correctness_test.py

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 4c25f49f16..1a54cd167e 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -660,7 +660,9 @@ py_library(
     srcs = [
         "keras_correctness_test_base.py",
         "keras_dnn_correctness_test.py",
+        "keras_embedding_model_correctness_test.py",
         "keras_image_model_correctness_test.py",
+        "keras_lstm_model_correctness_test.py",
     ],
     deps = [
         ":combinations",
@@ -714,6 +716,44 @@ cuda_py_test(
     ],
 )
 
+cuda_py_test(
+    name = "keras_embedding_model_correctness_test",
+    size = "medium",
+    srcs = ["keras_embedding_model_correctness_test.py"],
+    additional_deps = [
+        ":keras_correctness_test_lib",
+    ],
+    # Shard count is set to an odd number to distribute tasks across
+    # shards more evenly.
+    shard_count = 31,
+    tags = [
+        "multi_and_single_gpu",
+        "no_oss",  # TODO(b/117919883): Fix python error.
+        "no_pip",
+        "no_windows_gpu",
+        "notsan",
+    ],
+)
+
+cuda_py_test(
+    name = "keras_lstm_model_correctness_test",
+    size = "medium",
+    srcs = ["keras_lstm_model_correctness_test.py"],
+    additional_deps = [
+        ":keras_correctness_test_lib",
+    ],
+    # Shard count is set to an odd number to distribute tasks across
+    # shards more evenly.
+    shard_count = 31,
+    tags = [
+        "multi_and_single_gpu",
+        "no_oss",  # TODO(b/117919883): Fix python error.
+        "no_pip",
+        "no_windows_gpu",
+        "notsan",
+    ],
+)
+
 py_library(
     name = "metrics_v1_test_lib",
     testonly = 1,
diff --git a/tensorflow/contrib/distribute/python/combinations.py b/tensorflow/contrib/distribute/python/combinations.py
index f6c4291659..304762e69e 100644
--- a/tensorflow/contrib/distribute/python/combinations.py
+++ b/tensorflow/contrib/distribute/python/combinations.py
@@ -321,11 +321,12 @@ class NamedDistribution(object):
     return self._required_tpu
 
 
-def _get_tpu_strategy_creator(steps_per_run):
+def _get_tpu_strategy_creator(steps_per_run, **kwargs):
   def _create_tpu_strategy():
     resolver = cluster_resolver.TPUClusterResolver("")
     tpu_lib.initialize_tpu_system(resolver)
-    strategy = tpu_lib.TPUStrategy(resolver, steps_per_run=steps_per_run)
+    strategy = tpu_lib.TPUStrategy(resolver, steps_per_run=steps_per_run,
+                                   **kwargs)
     return strategy
   return _create_tpu_strategy
 
@@ -344,6 +345,15 @@ tpu_strategy = NamedDistribution(
 tpu_strategy_one_step = NamedDistribution(
     "TPUOneStep", _get_tpu_strategy_creator(steps_per_run=1),
     required_tpu=True)
+# TODO(b/122327153): Remove below two NamedDistributions.
+tpu_strategy_loop_on_device = NamedDistribution(
+    "TPULoopOnDevice", _get_tpu_strategy_creator(
+        steps_per_run=2, _disable_training_loop_on_host=True),
+    required_tpu=True)
+tpu_strategy_one_step_loop_on_device = NamedDistribution(
+    "TPUOneStepLoopOnDevice", _get_tpu_strategy_creator(
+        steps_per_run=1, _disable_training_loop_on_host=True),
+    required_tpu=True)
 
 mirrored_strategy_with_one_cpu = NamedDistribution(
     "Mirrored1CPU",
diff --git a/tensorflow/contrib/distribute/python/keras_correctness_test_base.py b/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
index 9f3cab49a8..08ed933f29 100644
--- a/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
+++ b/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
@@ -51,22 +51,46 @@ all_strategies = [
 ]
 
 
-def all_strategy_combinations_with_eager_and_graph_modes():
-  return combinations.combine(distribution=all_strategies,
-                              mode=['graph', 'eager'])
+def eager_mode_test_configuration():
+  return combinations.combine(mode='eager',
+                              use_numpy=False,
+                              use_validation_data=False)
 
 
-def all_strategy_combinations_with_graph_mode():
-  return combinations.combine(distribution=all_strategies, mode=['graph'])
+def graph_mode_test_configuration():
+  return combinations.combine(mode='graph',
+                              use_numpy=[True, False],
+                              use_validation_data=[True, False])
 
 
 def all_strategy_and_input_config_combinations():
   return (
       combinations.times(
           combinations.combine(distribution=all_strategies),
-          combinations.combine(mode=['graph', 'eager'],
-                               use_numpy=[True, False],
-                               use_validation_data=[True, False])))
+          eager_mode_test_configuration() + graph_mode_test_configuration()))
+
+
+def strategies_for_embedding_models():
+  """Returns distribution strategies to test for embedding models.
+
+  Since embedding models take longer to train, we disregard OneDeviceStrategy
+  and DefaultStrategy in order to prevent testing timeouts.
+  """
+
+  strategies = [s for s in all_strategies
+                if not s.required_tpu and s.required_gpus is not None]
+  strategies.append(combinations.tpu_strategy_loop_on_device)
+  strategies.append(combinations.tpu_strategy_one_step_loop_on_device)
+  return strategies
+
+
+def test_combinations_for_embedding_model():
+  return (
+      combinations.times(
+          combinations.combine(distribution=
+                               strategies_for_embedding_models()),
+          (graph_mode_test_configuration() +
+           eager_mode_test_configuration())))
 
 
 class MaybeDistributionScope(object):
@@ -393,5 +417,40 @@ class TestDistributionStrategyCorrectnessBase(test.TestCase,
                       testcase=self)
 
 
+class TestDistributionStrategyEmbeddingModelCorrectnessBase(
+    TestDistributionStrategyCorrectnessBase):
+  """Base class to test correctness of Keras models with embedding layers."""
+
+  def get_data(self,
+               count=(_GLOBAL_BATCH_SIZE * _EVAL_STEPS),
+               min_words=5,
+               max_words=10,
+               max_word_id=19,
+               num_classes=2):
+    distribution = []
+    for _ in range(num_classes):
+      dist = np.abs(np.random.randn(max_word_id))
+      dist /= np.sum(dist)
+      distribution.append(dist)
+
+    features = []
+    labels = []
+    for _ in range(count):
+      label = np.random.randint(0, num_classes, size=1)[0]
+      num_words = np.random.randint(min_words, max_words, size=1)[0]
+      word_ids = np.random.choice(
+          max_word_id, size=num_words, replace=True, p=distribution[label])
+      word_ids = word_ids
+      labels.append(label)
+      features.append(word_ids)
+
+    features = keras.preprocessing.sequence.pad_sequences(
+        features, maxlen=max_words)
+    x_train = np.asarray(features, dtype=np.float32)
+    y_train = np.asarray(labels, dtype=np.int32).reshape((count, 1))
+    x_predict = x_train
+    return x_train, y_train, x_predict
+
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py b/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
index f21b1c6752..7afacab0dd 100644
--- a/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
+++ b/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
@@ -28,6 +28,17 @@ from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_de
 from tensorflow.python.training import gradient_descent
 
 
+def all_strategy_combinations_with_eager_and_graph_modes():
+  return combinations.combine(distribution=keras_correctness_test_base.
+                              all_strategies,
+                              mode=['graph', 'eager'])
+
+
+def all_strategy_combinations_with_graph_mode():
+  return combinations.combine(distribution=keras_correctness_test_base.
+                              all_strategies, mode=['graph'])
+
+
 class TestDistributionStrategyDnnCorrectness(
     keras_correctness_test_base.TestDistributionStrategyCorrectnessBase):
 
@@ -65,8 +76,7 @@ class TestDistributionStrategyDnnCorrectness(
   def test_dnn_correctness(self, distribution, use_numpy, use_validation_data):
     self.run_correctness_test(distribution, use_numpy, use_validation_data)
 
-  @combinations.generate(keras_correctness_test_base.
-                         all_strategy_combinations_with_graph_mode())
+  @combinations.generate(all_strategy_combinations_with_graph_mode())
   def test_dnn_with_dynamic_learning_rate(self, distribution):
     self.run_dynamic_lr_test(distribution)
 
@@ -104,8 +114,7 @@ class TestDistributionStrategyDnnMetricCorrectness(
       history = model.fit(x=train_dataset, epochs=2, steps_per_epoch=10)
       self.assertEqual(history.history['binary_accuracy'], [1.0, 1.0])
 
-  @combinations.generate(keras_correctness_test_base.
-                         all_strategy_combinations_with_eager_and_graph_modes())
+  @combinations.generate(all_strategy_combinations_with_eager_and_graph_modes())
   def test_simple_dnn_metric_correctness(self, distribution):
     self.run_metric_correctness_test(distribution)
 
@@ -153,8 +162,7 @@ class TestDistributionStrategyDnnMetricEvalCorrectness(
       self.assertEqual(outs[1], 0.)
       self.assertEqual(outs[2], 0.)
 
-  @combinations.generate(keras_correctness_test_base.
-                         all_strategy_combinations_with_eager_and_graph_modes())
+  @combinations.generate(all_strategy_combinations_with_eager_and_graph_modes())
   def test_identity_model_metric_eval_correctness(self, distribution):
     self.run_eval_metrics_correctness_test(distribution)
 
diff --git a/tensorflow/contrib/distribute/python/keras_embedding_model_correctness_test.py b/tensorflow/contrib/distribute/python/keras_embedding_model_correctness_test.py
new file mode 100644
index 0000000000..3913f9bc0c
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/keras_embedding_model_correctness_test.py
@@ -0,0 +1,75 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Correctness test for tf.keras Embedding models using DistributionStrategy."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.distribute.python import combinations
+from tensorflow.contrib.distribute.python import keras_correctness_test_base
+from tensorflow.python import keras
+from tensorflow.python.eager import test
+from tensorflow.python.training import gradient_descent
+
+
+class DistributionStrategyEmbeddingModelCorrectnessTest(
+    keras_correctness_test_base.
+    TestDistributionStrategyEmbeddingModelCorrectnessBase):
+
+  def get_model(self, max_words=10, initial_weights=None, distribution=None):
+    with keras_correctness_test_base.MaybeDistributionScope(distribution):
+      word_ids = keras.layers.Input(
+          shape=(max_words,), dtype=np.int32, name='words')
+      word_embed = keras.layers.Embedding(input_dim=20,
+                                          output_dim=10)(word_ids)
+      if self.use_distributed_dense:
+        word_embed = keras.layers.TimeDistributed(keras.layers.Dense(4))(
+            word_embed)
+      avg = keras.layers.GlobalAveragePooling1D()(word_embed)
+      preds = keras.layers.Dense(2, activation='softmax')(avg)
+      model = keras.Model(inputs=[word_ids], outputs=[preds])
+
+      if initial_weights:
+        model.set_weights(initial_weights)
+
+      model.compile(
+          optimizer=gradient_descent.GradientDescentOptimizer(
+              learning_rate=0.1),
+          loss='sparse_categorical_crossentropy',
+          metrics=['sparse_categorical_accuracy'])
+    return model
+
+  @combinations.generate(keras_correctness_test_base.
+                         test_combinations_for_embedding_model())
+  def test_embedding_model_correctness(self, distribution, use_numpy,
+                                       use_validation_data):
+
+    self.use_distributed_dense = False
+    self.run_correctness_test(distribution, use_numpy, use_validation_data)
+
+  @combinations.generate(keras_correctness_test_base.
+                         test_combinations_for_embedding_model())
+  def test_embedding_time_distributed_model_correctness(self,
+                                                        distribution,
+                                                        use_numpy,
+                                                        use_validation_data):
+    self.use_distributed_dense = True
+    self.run_correctness_test(distribution, use_numpy, use_validation_data)
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/contrib/distribute/python/keras_lstm_model_correctness_test.py b/tensorflow/contrib/distribute/python/keras_lstm_model_correctness_test.py
new file mode 100644
index 0000000000..9ed2dfa206
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/keras_lstm_model_correctness_test.py
@@ -0,0 +1,65 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Correctness tests for tf.keras LSTM model using DistributionStrategy."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.distribute.python import combinations
+from tensorflow.contrib.distribute.python import keras_correctness_test_base
+from tensorflow.python import keras
+from tensorflow.python.eager import test
+from tensorflow.python.training import gradient_descent
+
+
+class DistributionStrategyLstmModelCorrectnessTest(
+    keras_correctness_test_base.
+    TestDistributionStrategyEmbeddingModelCorrectnessBase):
+
+  def get_model(self, max_words=10, initial_weights=None, distribution=None):
+    with keras_correctness_test_base.MaybeDistributionScope(distribution):
+      word_ids = keras.layers.Input(
+          shape=(max_words,), dtype=np.int32, name='words')
+      word_embed = keras.layers.Embedding(input_dim=20,
+                                          output_dim=10)(word_ids)
+      lstm_embed = keras.layers.LSTM(units=4,
+                                     return_sequences=False)(word_embed)
+
+      preds = keras.layers.Dense(2, activation='softmax')(lstm_embed)
+      model = keras.Model(inputs=[word_ids], outputs=[preds])
+
+      if initial_weights:
+        model.set_weights(initial_weights)
+
+      model.compile(
+          optimizer=gradient_descent.GradientDescentOptimizer(
+              learning_rate=0.1),
+          loss='sparse_categorical_crossentropy',
+          metrics=['sparse_categorical_accuracy'])
+    return model
+
+  @combinations.generate(keras_correctness_test_base.
+                         test_combinations_for_embedding_model())
+  def test_lstm_model_correctness(self,
+                                  distribution,
+                                  use_numpy,
+                                  use_validation_data):
+    self.run_correctness_test(distribution, use_numpy, use_validation_data)
+
+
+if __name__ == '__main__':
+  test.main()
-- 
GitLab


From dae187c176cd0d59f597e694ae193ff6a5dfbdd2 Mon Sep 17 00:00:00 2001
From: Peng Wang <wangpeng@google.com>
Date: Tue, 15 Jan 2019 13:26:33 -0800
Subject: [PATCH 0746/2345] A partial implementation of RFC "Random numbers in
 TensorFlow 2.0" (https://github.com/tensorflow/community/pull/38).

PiperOrigin-RevId: 229428968
---
 tensorflow/core/BUILD                         |   3 +
 .../api_def_StatefulStandardNormal.pbtxt      |  31 ++
 .../api_def_StatefulStandardNormal.pbtxt      |   4 +
 tensorflow/core/kernels/BUILD                 |  25 ++
 .../core/kernels/stateful_random_ops.cc       | 265 ++++++++++++++++++
 tensorflow/core/lib/random/philox_random.h    |   7 +
 tensorflow/core/ops/stateful_random_ops.cc    |  36 +++
 tensorflow/python/BUILD                       |  32 +++
 tensorflow/python/ops/stateful_random_ops.py  | 241 ++++++++++++++++
 .../python/ops/stateful_random_ops_test.py    | 156 +++++++++++
 10 files changed, 800 insertions(+)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_StatefulStandardNormal.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_StatefulStandardNormal.pbtxt
 create mode 100644 tensorflow/core/kernels/stateful_random_ops.cc
 create mode 100644 tensorflow/core/ops/stateful_random_ops.cc
 create mode 100644 tensorflow/python/ops/stateful_random_ops.py
 create mode 100644 tensorflow/python/ops/stateful_random_ops_test.py

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index bedc78ac11..7ba9afb5a8 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1101,6 +1101,7 @@ tf_gen_op_libs(
         "parsing_ops",
         "random_grad",
         "random_ops",
+        "stateful_random_ops",
         "remote_fused_graph_ops",
         "rpc_ops",
         "scoped_allocator_ops",
@@ -1256,6 +1257,7 @@ cc_library(
         ":parsing_ops_op_lib",
         ":ragged_ops",
         ":random_ops_op_lib",
+        ":stateful_random_ops_op_lib",
         ":remote_fused_graph_ops_op_lib",
         ":resource_variable_ops_op_lib",
         ":rpc_ops_op_lib",
@@ -1414,6 +1416,7 @@ cc_library(
         "//tensorflow/core/kernels:pooling_ops",
         "//tensorflow/core/kernels:ragged_ops",
         "//tensorflow/core/kernels:random_ops",
+        "//tensorflow/core/kernels:stateful_random_ops",
         "//tensorflow/core/kernels:random_poisson_op",
         "//tensorflow/core/kernels:remote_fused_graph_ops",
         "//tensorflow/core/kernels:required",
diff --git a/tensorflow/core/api_def/base_api/api_def_StatefulStandardNormal.pbtxt b/tensorflow/core/api_def/base_api/api_def_StatefulStandardNormal.pbtxt
new file mode 100644
index 0000000000..d963c55278
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_StatefulStandardNormal.pbtxt
@@ -0,0 +1,31 @@
+op {
+  graph_op_name: "StatefulStandardNormal"
+  in_arg {
+    name: "resource"
+    description: <<END
+The handle of the resource variable that stores the state of the RNG.
+END
+  }
+  in_arg {
+    name: "shape"
+    description: <<END
+The shape of the output tensor.
+END
+  }
+  out_arg {
+    name: "output"
+    description: <<END
+A tensor of the specified shape filled with random normal values.
+END
+  }
+  attr {
+    name: "dtype"
+    description: <<END
+The type of the output.
+END
+  }
+  summary: "Outputs random values from a normal distribution."
+  description: <<END
+The generated values will have mean 0 and standard deviation 1.
+END
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_StatefulStandardNormal.pbtxt b/tensorflow/core/api_def/python_api/api_def_StatefulStandardNormal.pbtxt
new file mode 100644
index 0000000000..fb3c3dbdb1
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_StatefulStandardNormal.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "StatefulStandardNormal"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 358d3d656e..374aa88d2d 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -4293,6 +4293,31 @@ tf_cuda_cc_test(
     ],
 )
 
+tf_kernel_library(
+    name = "stateful_random_ops",
+    prefix = "stateful_random_ops",
+    deps = [
+        ":bounds_check",
+        ":dense_update_functor",
+        ":gather_functor",
+        ":mutex_ops",
+        ":random_op",
+        ":resource_variable_ops",
+        ":scatter_functor",
+        ":state",
+        ":training_op_helpers",
+        ":variable_ops",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:core_cpu_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core:stateful_random_ops_op_lib",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:variant",
+    ],
+)
+
 tf_kernel_library(
     name = "stateless_random_ops",
     prefix = "stateless_random_ops",
diff --git a/tensorflow/core/kernels/stateful_random_ops.cc b/tensorflow/core/kernels/stateful_random_ops.cc
new file mode 100644
index 0000000000..e5c1c25864
--- /dev/null
+++ b/tensorflow/core/kernels/stateful_random_ops.cc
@@ -0,0 +1,265 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#define EIGEN_USE_THREADS
+
+#include <algorithm>
+#include <cmath>
+#include <memory>
+
+#include "absl/strings/str_join.h"
+#include "absl/types/variant.h"
+#include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/register_types.h"
+#include "tensorflow/core/framework/resource_mgr.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/framework/variant_op_registry.h"
+#include "tensorflow/core/kernels/bounds_check.h"
+#include "tensorflow/core/kernels/dense_update_functor.h"
+#include "tensorflow/core/kernels/gather_functor.h"
+#include "tensorflow/core/kernels/random_op.h"
+#include "tensorflow/core/kernels/resource_variable_ops.h"
+#include "tensorflow/core/kernels/scatter_functor.h"
+#include "tensorflow/core/kernels/training_op_helpers.h"
+#include "tensorflow/core/kernels/variable_ops.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/core/refcount.h"
+#include "tensorflow/core/lib/random/random_distributions.h"
+#include "tensorflow/core/lib/random/simple_philox.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/mem.h"
+#include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/platform/types.h"
+#include "tensorflow/core/util/util.h"
+#include "tensorflow/core/util/work_sharder.h"
+
+namespace tensorflow {
+
+using CPUDevice = Eigen::ThreadPoolDevice;
+
+using random::PhiloxRandom;
+
+namespace {
+
+// 'Variable' doesn't support uint32 or uint64 yet (due to reasons explained
+// in b/111604096 and cl/171681867), so I use signed int here. I choose int64
+// instead of int32 because `VarHandleOp` doesn't support int32 on GPU.
+using StateElementType = int64;
+static constexpr DataType STATE_ELEMENT_DTYPE = DT_INT64;
+
+using Algorithm = StateElementType;
+static constexpr Algorithm RNG_ALG_PHILOX = 1;
+
+using SkippableRNG = absl::variant<PhiloxRandom>;
+
+// This function is for hiding the implementation detail about the
+// `absl::variant` index of each algorithm.
+Algorithm GetAlgorithm(SkippableRNG const& rng) {
+  auto idx = rng.index();
+  if (idx == 0) {
+    return RNG_ALG_PHILOX;
+  }
+  // unreachable
+  return RNG_ALG_PHILOX;
+}
+
+// Fills a buffer with random numbers sampled from a given distribution.
+template <class Device, class Distribution>
+Status FillRandom(OpKernelContext* ctx, const Device& device,
+                  SkippableRNG const& gen, int64 size, Distribution dist,
+                  typename Distribution::ResultElementType* data) {
+  auto algorithm = GetAlgorithm(gen);
+  if (algorithm == RNG_ALG_PHILOX) {
+    auto philox = absl::get<PhiloxRandom>(gen);
+    functor::FillPhiloxRandom<Device, Distribution>()(ctx, device, philox, data,
+                                                      size, dist);
+    return Status::OK();
+  } else {
+    // return errors::InvalidArgument("Unsupported algorithm id: ", algorithm);
+    return Status::OK();
+  }
+}
+
+// The following two functions use the contract "lower 32 bits for the first
+// uint32, higher 32 bits for the second". Note that this is endian-neutral,
+// unlike a direct memory copy `memcpy(output, &input, 8)`.
+void Int64ToUint32s(int64 input, uint32* output1, uint32* output2) {
+  auto u64 = static_cast<uint64>(input);
+  *output1 = static_cast<uint32>(u64);
+  *output2 = static_cast<uint32>(u64 >> 32);
+}
+
+int64 Uint32sToInt64(uint32 input1, uint32 input2) {
+  auto u64_1 = static_cast<uint64>(input1);
+  auto u64_2 = static_cast<uint64>(input2);
+  return static_cast<int64>(u64_1 | (u64_2 << 32));
+}
+
+void GetPhiloxStateFromTensor(Tensor const& tensor,
+                              PhiloxRandom::ResultType* counter,
+                              PhiloxRandom::Key* key) {
+  auto tensor_flat = tensor.flat<StateElementType>();
+  auto tensor_ptr = tensor_flat.data();
+  // tensor_ptr's index is added by 1 to skip the algorithm tag.
+  Int64ToUint32s(tensor_ptr[1], &(*counter)[0], &(*counter)[1]);
+  Int64ToUint32s(tensor_ptr[2], &(*counter)[2], &(*counter)[3]);
+  Int64ToUint32s(tensor_ptr[3], &(*key)[0], &(*key)[1]);
+}
+
+void WritePhiloxStateToTensor(PhiloxRandom::ResultType const& counter,
+                              PhiloxRandom::Key const& key, Tensor* tensor) {
+  auto tensor_flat = tensor->flat<StateElementType>();
+  auto tensor_ptr = tensor_flat.data();
+  // tensor_ptr's index is added by 1 to skip the algorithm tag.
+  tensor_ptr[1] = Uint32sToInt64(counter[0], counter[1]);
+  tensor_ptr[2] = Uint32sToInt64(counter[2], counter[3]);
+  tensor_ptr[3] = Uint32sToInt64(key[0], key[1]);
+}
+
+// A helper function that does the actual work for
+// 'MakeRNGCopyAndUpdateVariable'.
+template <typename Device>
+Status GetRNGCopyAndUpdateTensor(Tensor* tensor, int64 delta,
+                                 SkippableRNG* rng_copy);
+
+template <>
+Status GetRNGCopyAndUpdateTensor<CPUDevice>(Tensor* tensor, int64 delta,
+                                            SkippableRNG* rng_copy) {
+  // The dtype of `tensor` should be `StateElementType` and the first element
+  // is the algorithm.
+  if (tensor->dims() != 1) {
+    return errors::InvalidArgument(
+        "RNG state must have one and only one dimension, not ", tensor->dims());
+  }
+  auto tensor_flat = tensor->flat<StateElementType>();
+  if (tensor_flat.size() < 1) {
+    return errors::InvalidArgument("Size of tensor must be at least 1");
+  }
+  auto algorithm = tensor_flat.data()[0];
+  if (algorithm == RNG_ALG_PHILOX) {
+    // Delegates to PhiloxRandom to do the actual increasing.
+    static_assert(std::is_same<StateElementType, int64>::value,
+                  "StateElementType must be int64");
+    static_assert(std::is_same<PhiloxRandom::ResultElementType, uint32>::value,
+                  "PhiloxRandom::ResultElementType must be uint32");
+    auto counter_size = PhiloxRandom::ResultType::kElementCount;
+    auto key_size = PhiloxRandom::Key::kElementCount;
+    auto min_tensor_size = 1 + (counter_size + key_size) / 2;
+    if (tensor_flat.size() < min_tensor_size) {
+      return errors::InvalidArgument(
+          "For Philox algorithm, the size of state"
+          " must be at least ",
+          min_tensor_size, "; got ", tensor_flat.size());
+    }
+    PhiloxRandom::ResultType counter;
+    PhiloxRandom::Key key;
+    GetPhiloxStateFromTensor(*tensor, &counter, &key);
+    PhiloxRandom philox(counter, key);
+    auto old_philox = philox;
+    philox.Skip(delta);  // do the actual increasing
+    WritePhiloxStateToTensor(philox.counter(), philox.key(), tensor);
+    *rng_copy = SkippableRNG(old_philox);
+    return Status::OK();
+  } else {
+    // return errors::InvalidArgument("Unsupported algorithm id: ", algorithm);
+    *rng_copy = SkippableRNG(PhiloxRandom());
+    return Status::OK();
+  }
+}
+
+// Gets a copy of the RNG and updates the variable. The copy can be used to
+// generate upto 'samples' random numbers, and the variable is updated as if
+// 'samples' random numbers have been generated (e.g. if the variable is a
+// counnter, the counter is increased by 'samples').
+template <class Device>
+Status MakeRNGCopyAndUpdateVariable(OpKernelContext* ctx, int input_idx,
+                                    int64 samples, SkippableRNG* rng_copy) {
+  Var* var = nullptr;
+  TF_RETURN_IF_ERROR(
+      LookupResource(ctx, HandleFromInput(ctx, input_idx), &var));
+  core::ScopedUnref s(var);
+  mutex_lock ml(*var->mu());
+  Tensor* var_tensor = var->tensor();
+  if (var_tensor->dtype() != STATE_ELEMENT_DTYPE) {
+    return errors::InvalidArgument("dtype of RNG state variable must be ",
+                                   DataTypeString(STATE_ELEMENT_DTYPE),
+                                   ", not ",
+                                   DataTypeString(var_tensor->dtype()));
+  }
+  TF_RETURN_IF_ERROR(PrepareToUpdateVariable<Device, StateElementType>(
+      ctx, var_tensor, var->copy_on_read_mode.load()));
+  TF_RETURN_IF_ERROR(
+      GetRNGCopyAndUpdateTensor<Device>(var_tensor, samples, rng_copy));
+  return Status::OK();
+}
+
+template <typename Device, class Distribution>
+class StatefulRandomOp : public OpKernel {
+ public:
+  using T = typename Distribution::ResultElementType;
+  explicit StatefulRandomOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}
+
+  // Assumes that input(0) is an existing resource.
+  void Compute(OpKernelContext* ctx) override {
+    const Tensor& shape_t = ctx->input(1);
+    Tensor* output;
+    TensorShape shape;
+    OP_REQUIRES_OK(ctx, MakeShape(shape_t, &shape));
+    OP_REQUIRES_OK(ctx, ctx->allocate_output(0, shape, &output));
+    if (shape.num_elements() == 0) return;
+
+    auto output_flat = output->flat<T>();
+    SkippableRNG rng;
+    // Multiplier 256 is the same as in FillPhiloxRandomTask; do not change
+    // it just here.
+    OP_REQUIRES_OK(ctx, MakeRNGCopyAndUpdateVariable<Device>(
+                            ctx, 0, output_flat.size() * 256, &rng));
+    // Fill in the random numbers
+    OP_REQUIRES_OK(ctx, FillRandom(ctx, ctx->eigen_device<Device>(), rng,
+                                   output_flat.size(), Distribution(),
+                                   output_flat.data()));
+  }
+};
+
+}  // namespace
+
+// So far the 'Distribution' type parameter is only used when the algorithm is
+// philox, so 'NormalDistribution<PhiloxRandom, ...>' is fine for now.
+#define REGISTER(DEVICE, TYPE)            \
+  REGISTER_KERNEL_BUILDER(                \
+      Name("StatefulStandardNormal")      \
+          .Device(DEVICE_##DEVICE)        \
+          .HostMemory("resource")         \
+          .HostMemory("shape")            \
+          .TypeConstraint<TYPE>("dtype"), \
+      StatefulRandomOp<DEVICE##Device,    \
+                       random::NormalDistribution<PhiloxRandom, TYPE> >);
+
+#define REGISTER_CPU(TYPE) REGISTER(CPU, TYPE)
+
+TF_CALL_half(REGISTER_CPU);
+TF_CALL_bfloat16(REGISTER_CPU);
+TF_CALL_float(REGISTER_CPU);
+TF_CALL_double(REGISTER_CPU);
+
+#undef REGISTER_CPU
+#undef REGISTER
+
+// TODO(wangpeng): Add RNG ops for other distributions.
+// TODO(wangpeng): Add support for GPU and XLA.
+
+}  // end namespace tensorflow
diff --git a/tensorflow/core/lib/random/philox_random.h b/tensorflow/core/lib/random/philox_random.h
index 058ed95ffb..f4bbc689d4 100644
--- a/tensorflow/core/lib/random/philox_random.h
+++ b/tensorflow/core/lib/random/philox_random.h
@@ -49,6 +49,7 @@ namespace random {
 template <typename T, int ElementCount>
 class Array {
  public:
+  static const int kElementCount = ElementCount;
   PHILOX_DEVICE_INLINE Array() {
     for (int i = 0; i < ElementCount; ++i) {
       data_[i] = T(0);
@@ -131,6 +132,12 @@ class PhiloxRandom {
   PHILOX_DEVICE_INLINE
   PhiloxRandom(ResultType counter, Key key) : counter_(counter), key_(key) {}
 
+  PHILOX_DEVICE_INLINE
+  ResultType const& counter() const { return counter_; }
+
+  PHILOX_DEVICE_INLINE
+  Key const& key() const { return key_; }
+
   // Skip the specified number of samples of 128-bits in the current stream.
   PHILOX_DEVICE_INLINE
   void Skip(uint64 count) {
diff --git a/tensorflow/core/ops/stateful_random_ops.cc b/tensorflow/core/ops/stateful_random_ops.cc
new file mode 100644
index 0000000000..4c9277eda5
--- /dev/null
+++ b/tensorflow/core/ops/stateful_random_ops.cc
@@ -0,0 +1,36 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/shape_inference.h"
+
+namespace tensorflow {
+
+Status StatefulRandomShape(shape_inference::InferenceContext* c) {
+  shape_inference::ShapeHandle out;
+  TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(1, &out));
+  c->set_output(0, out);
+  return Status::OK();
+}
+
+REGISTER_OP("StatefulStandardNormal")
+    .Input("resource: resource")
+    .Input("shape: shape_dtype")
+    .Output("output: dtype")
+    .Attr("dtype: {half,bfloat16,float,double} = DT_FLOAT")
+    .Attr("shape_dtype: {int32, int64} = DT_INT64")
+    .SetShapeFn(StatefulRandomShape);
+
+}  // namespace tensorflow
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 589605e347..adba0b49fa 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -1833,6 +1833,11 @@ tf_gen_op_wrapper_private_py(
     visibility = ["//learning/brain/python/ops:__pkg__"],
 )
 
+tf_gen_op_wrapper_private_py(
+    name = "stateful_random_ops_gen",
+    visibility = ["//learning/brain/python/ops:__pkg__"],
+)
+
 tf_gen_op_wrapper_private_py(
     name = "resource_variable_ops_gen",
     visibility = [
@@ -2768,6 +2773,32 @@ py_library(
     ],
 )
 
+py_library(
+    name = "stateful_random_ops",
+    srcs = ["ops/stateful_random_ops.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":dtypes",
+        ":framework_ops",
+        ":math_ops",
+        ":stateful_random_ops_gen",
+        ":variables",
+        "//third_party/py/numpy",
+    ],
+)
+
+cuda_py_test(
+    name = "stateful_random_ops_test",
+    size = "medium",
+    srcs = ["ops/stateful_random_ops_test.py"],
+    additional_deps = [
+        ":stateful_random_ops",
+        ":client_testlib",
+        ":logging_ops",
+        ":random_ops_gen",
+    ],
+)
+
 py_library(
     name = "stateless_random_ops",
     srcs = ["ops/stateless_random_ops.py"],
@@ -3046,6 +3077,7 @@ py_library(
         ":special_math_ops",
         ":state_grad",
         ":state_ops",
+        ":stateful_random_ops",
         ":stateless_random_ops",
         ":string_ops",
         ":template",
diff --git a/tensorflow/python/ops/stateful_random_ops.py b/tensorflow/python/ops/stateful_random_ops.py
new file mode 100644
index 0000000000..c7a14f2015
--- /dev/null
+++ b/tensorflow/python/ops/stateful_random_ops.py
@@ -0,0 +1,241 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Operations for generating random numbers."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import sys
+
+import numpy as np
+
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import gen_stateful_random_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.training.checkpointable import \
+tracking
+from tensorflow.python.util.tf_export import tf_export
+
+# A seed for random ops (stateful and stateless) will always be 1024
+# bits, all of which will be sent to the C++ code. The actual C++
+# implementation of some algorithms may only use a lower part of the bits.
+
+MAX_INT64 = 2**63 - 1
+MIN_INT64 = -(2**63)
+UINT64_SPAN = 2**64
+# 'Variable' doesn't support uint32 or uint64 yet (due to reasons explained in
+# b/111604096 and cl/171681867), so I use signed int here. I choose int64
+# instead of int32 here because `VarHandleOp` doesn't support int32 on GPU.
+SEED_TYPE = "int64"
+SEED_MIN = MIN_INT64
+SEED_MAX = MAX_INT64
+SEED_UINT_SPAN = UINT64_SPAN
+SEED_TYPE_BITS = 64
+SEED_BIT_MASK = 0xFFFFFFFFFFFFFFFF
+SEED_SIZE = 16  # in units of SEED_TYPE
+
+
+STATE_TYPE = SEED_TYPE
+RNG_ALG_PHILOX = 1
+DEFAULT_ALGORITHM = RNG_ALG_PHILOX
+
+
+def non_deterministic_seed():
+  """Makes a non-deterministic seed.
+
+  The implementation will be changed soon from pure Python to an op.
+
+  Returns:
+    a 1-D tensor.
+  """
+  return np.random.randint(
+      low=SEED_MIN, high=SEED_MAX + 1, size=SEED_SIZE,
+      dtype=SEED_TYPE)
+
+
+def _uint_to_int(n):
+  if n > SEED_MAX:
+    n = n - SEED_UINT_SPAN
+  return n
+
+
+PHILOX_STATE_SIZE = 3
+
+
+def _make_philox_state(seed):
+  """Makes a RNG state for Philox algorithm.
+
+  Args:
+    seed: an integer or 1-D tensor.
+
+  Returns:
+    a 1-D tensor.
+  """
+  int_types = (int,) if sys.version_info >= (3, 0) else (int, long)
+  if isinstance(seed, int_types):
+    # chop the Python integer (infinite precision) into chunks of SEED_TYPE
+    ls = []
+    for _ in range(PHILOX_STATE_SIZE):
+      ls.append(seed & SEED_BIT_MASK)
+      seed >>= SEED_TYPE_BITS
+    seed = ls
+  # to avoid overflow error from np.asarray
+  seed = list(map(_uint_to_int, seed))
+  seed = np.asarray(seed, dtype=STATE_TYPE)
+  if len(seed.shape) != 1:
+    raise ValueError(
+        "seed should only have one dimension; got shape: %s" % seed.shape)
+  seed = seed[0:PHILOX_STATE_SIZE]
+  # Padding with zeros on the right if too short
+  seed_size = seed.shape[0]
+  if seed_size < PHILOX_STATE_SIZE:
+    seed = np.pad(
+        seed, [(0, PHILOX_STATE_SIZE - seed_size)],
+        mode="constant",
+        constant_values=0)
+  assert seed.shape == (PHILOX_STATE_SIZE,), "Wrong seed.shape: %s" % seed.shape
+  return seed
+
+
+def _make_state_from_seed(seed, algorithm):
+  if algorithm == RNG_ALG_PHILOX:
+    return _make_philox_state(seed)
+  else:
+    raise ValueError("Unsupported algorithm id: %s" % algorithm)
+
+
+def create_rng_state(seed, algorithm=None):
+  """Creates a RNG state.
+
+  Args:
+    seed: an integer or 1-D tensor.
+    algorithm: (optional) an integer representing the RNG algorithm. If None, an
+      algorithm will be auto-selected.
+
+  Returns:
+    a 1-D tensor "rng_state" with:
+    * rng_state[0] is a value that identifies the RNG algorithm;
+    * rng_state[1:] holds the RNG state itself (size dependent on the
+        algorithm).
+  """
+  if algorithm is None:
+    # TODO(wangpeng): more sophisticated algorithm selection
+    algorithm = DEFAULT_ALGORITHM
+  state = _make_state_from_seed(seed, algorithm)
+  return np.concatenate((np.array([algorithm], dtype=STATE_TYPE), state),
+                        axis=None)
+
+
+def _shape_tensor(shape):
+  """Convert to an int32 or int64 tensor, defaulting to int64 if empty."""
+  if isinstance(shape, (tuple, list)) and not shape:
+    dtype = dtypes.int64
+  else:
+    dtype = None
+  return ops.convert_to_tensor(shape, dtype=dtype, name="shape")
+
+
+@tf_export("random.experimental.Generator")
+class Generator(tracking.Checkpointable):
+  """Random-number generator.
+
+  It uses Variable to manage its internal state.
+  """
+
+  def __init__(self, copy_from=None, seed=None, algorithm=None):
+    if copy_from is None:
+      if seed is None:
+        seed = non_deterministic_seed()
+      state = create_rng_state(seed, algorithm)
+      self._state_var = variables.Variable(state, dtype=STATE_TYPE)
+    else:
+      assert seed is None
+      state = copy_from.state
+      self._state_var = variables.Variable(state, dtype=STATE_TYPE)
+
+  def reset(self, seed):
+    algorithm = int(self.algorithm)
+    state = create_rng_state(seed, algorithm)
+    self._state_var.assign(state)
+
+  @property
+  def state(self):
+    return self._state_var
+
+  @property
+  def algorithm(self):
+    return self._state_var[0]
+
+  # The following functions return a tensor and as a side effect update
+  # self._state_var.
+  def standard_normal(self, shape, dtype=dtypes.float32):
+    return gen_stateful_random_ops.stateful_standard_normal(
+        self.state.handle, shape, dtype)
+
+  def normal(self, shape, mean=0.0, stddev=1.0, dtype=dtypes.float32,
+             name=None):
+    with ops.name_scope(name, "stateful_normal", [shape, mean, stddev]) as name:
+      shape = _shape_tensor(shape)
+      mean = ops.convert_to_tensor(mean, dtype=dtype, name="mean")
+      stddev = ops.convert_to_tensor(stddev, dtype=dtype, name="stddev")
+      rnd = self.standard_normal(shape, dtype)
+      return math_ops.add(rnd * stddev, mean, name=name)
+
+  # TODO(wangpeng): implement other distributions (`uniform`,
+  #   `truncated_normal`, etc.)
+  # TODO(wangpeng): implement `make_seeds`
+  # TODO(wangpeng): implement `make_generators`
+
+
+# It's not safe to create TF ops before `init_google` is called, so this is
+# initialized to None and get a value the first time `get_global_generator` is
+# called.
+global_generator = None
+
+
+@tf_export("random.experimental.get_global_generator")
+def get_global_generator():
+  global global_generator
+  if global_generator is None:
+    global_generator = Generator()
+  return global_generator
+
+
+@tf_export("random.experimental.set_global_generator")
+def set_global_generator(generator):
+  global global_generator
+  global_generator = generator
+
+
+# This function creates a new Generator object (and the Variable object within),
+# which does not work well with tf.function because (1) tf.function puts
+# restrictions on Variable creation thus reset_global_generator can't be freely
+# used inside tf.function; (2) redirecting a global variable to
+# a new object is problematic with tf.function because the old object may be
+# captured by a 'tf.function'ed function and still be used by it.
+# A 'tf.function'ed function only keeps weak references to variables,
+# so deleting a variable and then calling that function again may raise an
+# error, as demonstrated by
+# random_test.py/RandomTest.testResetGlobalGeneratorBadWithDefun .
+# The function 'set_global_generator' below also has this problem.
+@tf_export("random.experimental.reset_global_generator")
+def reset_global_generator(seed, algorithm=None):
+  global global_generator
+  if algorithm is None:
+    algorithm = int(global_generator.algorithm)  # preserve the old algorithm
+  global_generator = Generator(seed=seed, algorithm=algorithm)
diff --git a/tensorflow/python/ops/stateful_random_ops_test.py b/tensorflow/python/ops/stateful_random_ops_test.py
new file mode 100644
index 0000000000..92419a0770
--- /dev/null
+++ b/tensorflow/python/ops/stateful_random_ops_test.py
@@ -0,0 +1,156 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for stateful_random_ops.py."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.eager import def_function
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors_impl
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import gen_random_ops
+from tensorflow.python.ops import logging_ops
+from tensorflow.python.ops import stateful_random_ops as \
+random
+from tensorflow.python.platform import test
+
+
+class StatefulRandomOpsTest(test.TestCase):
+
+  def testCreateRNGStateIntSeed(self):
+    """Tests `create_rng_state` when `seed` is int."""
+    # using leading 'F' to test overflow tolerance
+    state = random.create_rng_state(0xFFFF222233334444FFAA666677778888,
+                                    random.RNG_ALG_PHILOX)
+    self.assertAllEqual(
+        list(map(random._uint_to_int,
+                 [random.RNG_ALG_PHILOX, 0xFFAA666677778888,
+                  0xFFFF222233334444] + [0] * (random.PHILOX_STATE_SIZE - 2))),
+        state)
+
+  @test_util.run_v2_only
+  @test_util.also_run_as_tf_function
+  def testEagerAndDefun(self):
+    """A simple test to make sure the op works in eager and defunned mode."""
+    random.get_global_generator().normal((3,))
+
+  @test_util.run_v2_only
+  def testOpSeedSelectionAfterSetSeed(self):
+    """Tests that op-seed selection is reset after reseting global generator.
+
+    Fixing GitHub issue 9171:
+    https://github.com/tensorflow/tensorflow/issues/9171
+    """
+    shape = (3,)
+    random.get_global_generator().reset(1)
+    a = random.get_global_generator().normal(shape)
+    random.get_global_generator().reset(1)
+    b = random.get_global_generator().normal(shape)
+    self.assertAllEqual(a, b)
+
+    # Now do the above again using accelerated ('defun'ed) computation
+    @def_function.function
+    def f():
+      return random.get_global_generator().normal(shape)
+
+    random.get_global_generator().reset(1)
+    c = f()
+    random.get_global_generator().reset(1)
+    d = f()
+    self.assertAllEqual(c, d)
+    self.assertAllEqual(a, c)
+
+  @test_util.run_v2_only
+  def testOpSeedSelectionNotSensitive(self):
+    """Test that op-seed selection is not sensitive to trivial changes.
+
+    Test that op-seed selection is not sensitive to trivial computation
+    (i.e. graph) changes.
+
+    Fixing b/32087099
+    """
+    def f(include_print):
+      shape = constant_op.constant([5])
+      if include_print:
+        shape = logging_ops.Print(shape, [shape])
+      return random.get_global_generator().normal(shape)
+
+    def compare(fst_includes_print, snd_includes_print):
+      random.get_global_generator().reset(50)
+      fst = f(fst_includes_print)
+      random.get_global_generator().reset(50)
+      snd = f(snd_includes_print)
+      self.assertAllEqual(fst, snd)
+      # Now do the above again using accelerated (defunned) 'f'.
+      # Running 'f' with two different Boolean arguments should cause
+      # two different graphs to be generated, hence demonstrating the
+      # insensitivity to graph changes.
+      f_acc = def_function.function(f)
+      random.get_global_generator().reset(50)
+      fst = f_acc(fst_includes_print)
+      random.get_global_generator().reset(50)
+      snd = f_acc(snd_includes_print)
+      self.assertAllEqual(fst, snd)
+
+    compare(False, False)
+    compare(True, True)
+    compare(True, False)
+
+  @test_util.run_v2_only
+  def testSameAsOldRandomOps(self):
+    """Tests that the generated numbers are the same as the old random_ops.py .
+    """
+    seed1, seed2 = 50, 60
+    # note how the two seeds for the old op correspond to the seed for the new
+    # op
+    random.get_global_generator().reset([0, seed2, seed1])
+    shape = constant_op.constant([2, 3])
+    dtype = dtypes.float32
+    # create a graph for the old op in order to call it many times
+    @def_function.function
+    def old():
+      return gen_random_ops.random_standard_normal(
+          shape, dtype=dtype, seed=seed1, seed2=seed2)
+
+    def new():
+      return random.get_global_generator().standard_normal(shape, dtype=dtype)
+
+    for _ in range(100):
+      self.assertAllEqual(old(), new())
+
+  @test_util.run_v2_only
+  def testResetGlobalGeneratorBadWithDefun(self):
+    """Demonstrates that reset_global_generator don't work properly with defun.
+    """
+    shape = (3,)
+
+    @def_function.function
+    def f():
+      return random.get_global_generator().normal(shape)
+
+    random.reset_global_generator(50)
+    with self.assertRaisesWithPredicateMatch(
+        errors_impl.NotFoundError, "Resource .+ does not exist"):
+      a = f()
+      random.reset_global_generator(50)
+      b = f()
+      self.assertAllEqual(a, b)
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From bcde9269af0b177e1047ca9b250234b57c5522e2 Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Tue, 15 Jan 2019 13:44:39 -0800
Subject: [PATCH 0747/2345] Fix arena_planner_test

Use proper tf_cc_test rule.

PiperOrigin-RevId: 229432697
---
 tensorflow/lite/BUILD | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/tensorflow/lite/BUILD b/tensorflow/lite/BUILD
index 8fca01624c..18fd1c1012 100644
--- a/tensorflow/lite/BUILD
+++ b/tensorflow/lite/BUILD
@@ -60,16 +60,13 @@ cc_library(
     ],
 )
 
-cc_test(
+tf_cc_test(
     name = "arena_planner_test",
     size = "small",
     srcs = ["arena_planner_test.cc"],
-    tags = [
-        "tflite_not_portable",
-    ],
     deps = [
         ":arena_planner",
-        "//tensorflow/core:lib",
+        "//tensorflow/core:tflite_portable_logging",
         "//tensorflow/lite/testing:util",
         "@com_google_googletest//:gtest",
     ],
-- 
GitLab


From d9c5cfc2ac1465146c8c6e19d1c189d07ab0c55b Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Tue, 15 Jan 2019 13:49:49 -0800
Subject: [PATCH 0748/2345] Internal change.

PiperOrigin-RevId: 229433602
---
 tensorflow/lite/g3doc/performance/gpu.md      | 230 +++++++++++++++
 .../lite/g3doc/performance/gpu_advanced.md    | 279 ++++++++++++++++++
 .../performance/images/android_gpu_demo.gif   | Bin 0 -> 1475626 bytes
 .../g3doc/performance/images/iosdebug.png     | Bin 0 -> 17762 bytes
 .../g3doc/performance/images/iosmetal.png     | Bin 0 -> 36053 bytes
 .../g3doc/performance/images/iosrelease.png   | Bin 0 -> 75374 bytes
 6 files changed, 509 insertions(+)
 create mode 100644 tensorflow/lite/g3doc/performance/gpu.md
 create mode 100644 tensorflow/lite/g3doc/performance/gpu_advanced.md
 create mode 100644 tensorflow/lite/g3doc/performance/images/android_gpu_demo.gif
 create mode 100644 tensorflow/lite/g3doc/performance/images/iosdebug.png
 create mode 100644 tensorflow/lite/g3doc/performance/images/iosmetal.png
 create mode 100644 tensorflow/lite/g3doc/performance/images/iosrelease.png

diff --git a/tensorflow/lite/g3doc/performance/gpu.md b/tensorflow/lite/g3doc/performance/gpu.md
new file mode 100644
index 0000000000..14ca54a158
--- /dev/null
+++ b/tensorflow/lite/g3doc/performance/gpu.md
@@ -0,0 +1,230 @@
+# TensorFlow Lite GPU Delegate Tutorial
+
+[TensorFlow Lite](https://www.tensorflow.org/lite) supports several hardware
+accelerators. This document describes how to preview the experimental GPU backend using the
+TensorFlow Lite delegate APIs on Android and iOS.
+
+GPUs are designed to have high throughput for massively parallelizable
+workloads. Thus, they are well-suited for deep neural nets, which consist of a
+huge number of operators, each working on some input tensor(s) that can be
+easily divided into smaller workloads and carried out in parallel, typically
+resulting in lower latency. In the best scenario, inference on the GPU may now
+run fast enough for previously not available real-time applications.
+
+Unlike CPUs, GPUs compute with 16-bit or 32-bit floating point numbers and do
+not require quantization for optimal performance.
+
+Another benefit with GPU inference is its power efficiency. GPUs carry out the
+computations in a very efficient and optimized manner, so that they consume less
+power and generate less heat than when the same task is run on CPUs.
+
+## Demo App Tutorials
+
+The easiest way to try out the experimental GPU delegate is to follow the below tutorials, which go through building our classification demo applications with GPU support. The GPU code is only binary for now; it will be open-sourced soon. Once you understand how to get our demos working, you can try this out on your own custom models.
+
+### Android (with Android Studio)
+
+Note: This requires OpenGL ES 3.1 or higher.
+
+#### Step 1. Clone the TensorFlow source code and open it in Android Studio
+
+```
+git clone https://github.com/tensorflow/tensorflow
+```
+
+#### Step 2. Edit `app/build.gradle` to use the experimental GPU AAR
+
+Replace the existing `tensorflow-lite` package in the existing `dependencies`
+block.
+
+```
+dependencies {
+    ...
+    // implementation 'org.tensorflow:tensorflow-lite:0.0.0-nightly'
+    implementation 'org.tensorflow:tensorflow-lite:0.0.0-gpu-experimental'
+}
+```
+
+#### Step 3. Build and run
+
+Run → Run ‘app’.  When you run the application you will see a button for
+enabling the GPU. Change from quantized to a float model and then click GPU to
+run on the GPU.
+
+![running android gpu demo and switch to gpu](images/android_gpu_demo.gif)
+
+### iOS (with XCode)
+
+Note: This requires XCode v10.1 or later.
+
+#### Step 1. Get the demo source code and make sure it compiles.
+
+Follow our iOS Demo App [tutorial](https://www.tensorflow.org/lite/demo_ios).
+This will get you to a point where the unmodified iOS camera demo is working
+on your phone.
+
+
+#### Step 2. Modify the Podfile to use the TensorFlow Lite GPU CocoaPod
+
+We have built a binary CocoaPod that includes the GPU delegate. To switch the
+project to use it, modify the
+`tensorflow/tensorflow/lite/examples/ios/camera/Podfile` file to use
+the `TensorFlowLiteGpuExperimental` pod instead of `TensorFlowLite`.
+
+```
+target 'YourProjectName'
+  # pod 'TensorFlowLite', '1.12.0'
+  pod 'TensorFlowLiteGpuExperimental'
+```
+
+#### Step 3. Enable the GPU Delegate
+
+You will need to change two `#define` flags in `CameraExampleViewController.h`
+to enable the GPU delegate. First, change `TFLITE_USE_CONTRIB_LITE` from 1 to 0
+since TensorFlow Lite has moved from TensorFlow contrib into core.
+
+```c
+#define TFLITE_USE_CONTRIB_LITE 0
+```
+
+Next, change `TFLITE_USE_GPU_DELEGATE` from 0 to 1, to enable the code that will
+use the GPU delegate.
+
+```c
+#define TFLITE_USE_GPU_DELEGATE 1
+```
+
+#### Step 4. Build and run the demo app
+
+After following the previous step, you should be able to run the app.
+
+
+#### Step 5. Release mode.
+
+While in Step 4 you ran in debug mode, to get better performance, you should
+change to a release build with the appropriate optimal Metal settings. In
+particular, To edit these settings go to the `Product > Scheme > Edit
+Scheme...`. Select `Run`. On the `Info` tab, change `Build Configuration`, from
+`Debug` to `Release`, uncheck `Debug executable`.
+
+![setting up release](images/iosdebug.png)
+
+Then
+click the `Options` tab and change `GPU Frame Capture` to `Disabled` and
+`Metal API Validation` to `Disabled`.
+
+![setting up metal options](images/iosmetal.png)
+
+Lastly make sure Release only builds on 64-bit architecture. Under `Project
+navigator -> tflite_camera_example -> PROJECT -> tflite_camera_example -> Build
+Settings` set `Build Active Architecture Only > Release` to Yes.
+
+![setting up release options](images/iosrelease.png)
+
+## Trying the GPU Delegate on your own model
+
+### Android
+
+Look at the demo to see how to add the
+delegate. In your application, add the AAR as above, import
+`org.tensorflow.lite.experimental.GpuDelegate` module, and use the`addDelegate`
+function to register the GPU delegate to the interpreter:
+
+```java
+import org.tensorflow.lite.Interpreter;
+import org.tensorflow.lite.experimental.GpuDelegate;
+
+// Initialize interpreter with GPU delegate
+GpuDelegate delegate = new GpuDelegate();
+Interpreter.Options options = (new Interpreter.Options()).addDelegate(delegate);
+Interpreter interpreter = new Interpreter(model, options);
+
+// Run inference
+while (true) {
+  writeToInput(input);
+  interpreter.run(input, output);
+  readFromOutput(output);
+}
+
+// Clean up
+delegate.close();
+```
+
+### iOS
+
+In your application code, include the GPU delegate header and call the
+`Interpreter::ModifyGraphWithDelegate` function to register the GPU delegate to
+the interpreter:
+
+```cpp
+#import "tensorflow/lite/delegates/gpu/metal_delegate.h"
+
+// Initialize interpreter with GPU delegate
+std::unique_ptr<Interpreter> interpreter;
+InterpreterBuilder(*model, resolver)(&interpreter);
+auto* delegate = NewGpuDelegate(nullptr);  // default config
+if (interpreter->ModifyGraphWithDelegate(delegate) != kTfLiteOk) return false;
+
+// Run inference
+while (true) {
+  WriteToInputTensor(interpreter->typed_input_tensor<float>(0));
+  if (interpreter->Invoke() != kTfLiteOk) return false;
+  ReadFromOutputTensor(interpreter->typed_output_tensor<float>(0));
+}
+
+// Clean up
+interpreter = nullptr;
+DeleteGpuDelegate(delegate);
+```
+
+## Supported Models and Ops
+
+With the release of the GPU delegate, we included a handful of models that can
+be run on the backend:
+
+* [MobileNet v1 (224x224) image classification](https://ai.googleblog.com/2017/06/mobilenets-open-source-models-for.html) [[download]](https://storage.googleapis.com/download.tensorflow.org/models/tflite/gpu/mobilenet_v1_1.0_224.tflite)
+<br /><i>(image classification model designed for mobile and embedded based vision applications)</i>
+* [DeepLab segmentation (257x257)](https://ai.googleblog.com/2018/03/semantic-image-segmentation-with.html) [[download]](https://storage.googleapis.com/download.tensorflow.org/models/tflite/gpu/deeplabv3_257_mv_gpu.tflite)
+<br /><i>(image segmentation model that assigns semantic labels (e.g., dog, cat, car) to every pixel in the input image)</i>
+* [MobileNet SSD object detection](https://ai.googleblog.com/2018/07/accelerated-training-and-inference-with.html) [[download]](https://storage.googleapis.com/download.tensorflow.org/models/tflite/gpu/mobile_ssd_v2_float_coco.tflite)
+<br /><i>(image classification model that detects multiple objects with bounding boxes)</i>
+* [PoseNet for pose estimation](https://github.com/tensorflow/tfjs-models/tree/master/posenet) [[download]](https://storage.googleapis.com/download.tensorflow.org/models/tflite/gpu/multi_person_mobilenet_v1_075_float.tflite)
+<br /><i>(vision model that estimates the poses of a person(s) in image or video)</i>
+
+To see a full list of supported ops, please see the [advanced documentation](gpu_advanced.md).
+
+## Non-supported models and ops
+
+If some of the ops are not supported by the GPU delegate, the framework will
+only run a part of the graph on the GPU and the remaining part on the CPU.  Due
+to the high cost of CPU/GPU synchronization, a split execution mode like this
+will often result in a performance slower than when the whole network is run on
+the CPU alone.  In this case, the user will get a warning like:
+
+```
+WARNING: op code #42 cannot be handled by this delegate.
+```
+
+We did not provide a callback for this failure, as this is not a true run-time
+failure, but something that the developer can observe while trying to get the
+network to run on the delegate.
+
+## Tips for optimization
+
+Some operations that are trivial on the CPU may have a high cost for the GPU.
+One class of such operation is various forms of reshape operations, including
+`BATCH_TO_SPACE`, `SPACE_TO_BATCH`, `SPACE_TO_DEPTH`, and so forth. If those ops
+are inserted into the network just for the network architect's logical thinking,
+it is worth removing them for performance.
+
+On GPU, tensor data is sliced into 4-channels. Thus, a computation on a tensor
+of shape `[B,H,W,5]` will perform about the same on a tensor of shape
+`[B,H,W,8]` but significantly worse than `[B,H,W,4]`.
+
+In that sense, if the camera hardware supports image frames in RGBA, feeding
+that 4-channel input is significantly faster as a memory copy (from 3-channel
+RGB to 4-channel RGBX) can be avoided.
+
+For best performance, do not hesitate to retrain your classifier with a mobile-
+optimized network architecture. That is a significant part of optimization for
+on-device inference.
diff --git a/tensorflow/lite/g3doc/performance/gpu_advanced.md b/tensorflow/lite/g3doc/performance/gpu_advanced.md
new file mode 100644
index 0000000000..d0ae123613
--- /dev/null
+++ b/tensorflow/lite/g3doc/performance/gpu_advanced.md
@@ -0,0 +1,279 @@
+# TensorFlow Lite on GPU
+
+[TensorFlow Lite](https://www.tensorflow.org/mobile/tflite/) supports several
+hardware accelerators.  This document describes how to use the GPU backend using
+the TensorFlow Lite delegate APIs on Android (requires OpenGL ES 3.1 or higher)
+and iOS (requires iOS 8 or later).
+
+GPUs are designed to have high throughput for massively parallelizable
+workloads.  Thus, they are well-suited for deep neural nets which consists of a
+huge number of operators, each working on some input tensor(s) that can be
+easily divided into smaller workloads and carried out in parallel, typically
+resulting in lower latency.  In the best scenario, inference on the GPU may now
+run fast enough and now become suitable for real-time applications if it was not
+before.
+
+GPUs do their computation with 16-bit or 32-bit floating point numbers and do
+not require quantization for optimal performance unlike the CPUs.  If
+quantization of your neural network was not an option due to lower accuracy
+caused by lost precision, such concern can be discarded when running deep neural
+net models on the GPU.
+
+Another benefit that comes with GPU inference is its power efficiency.  GPUs
+carry out the computations in a very efficient and optimized way, so that they
+consume less power and generate less heat than when the same task is run on the
+CPUs.
+
+TensorFlow Lite on GPU supports the following ops in 16-bit and 32-bit float
+precision:
+
+* `ADD v1`
+* `AVERAGE_POOL_2D v1`
+* `CONCATENATION v1`
+* `CONV_2D v1`
+* `DEPTHWISE_CONV_2D v1-2`
+* `FULLY_CONNECTED v1`
+* `LOGISTIC v1`
+* `MAX_POOL_2D v1`
+* `MUL v1`
+* `PAD v1`
+* `PRELU v1`
+* `RELU v1`
+* `RELU6 v1`
+* `RESHAPE v1`
+* `RESIZE_BILINEAR v1`
+* `SOFTMAX v1`
+* `STRIDED_SLICE v1`
+* `SUB v1`
+* `TRANSPOSE_CONV v1`
+
+## Basic Usage
+
+### Android
+
+Using TensorFlow Lite on GPU is achieved via `TfLiteDelegate`.  In Java, you can
+specify the GpuDelegate through `Interpreter.Options`.
+
+```java
+// NEW: Prepare GPU delegate.
+GpuDelegate delegate = new GpuDelegate();
+Interpreter.Options options = (new Interpreter.Options()).addDelegate(delegate);
+
+// Set up interpreter.
+Interpreter interpreter = new Interpreter(model, options);
+
+// Run inference.
+writeToInputTensor(inputTensor);
+interpreter.run(inputTensor, outputTensor);
+readFromOutputTensor(outputTensor);
+
+// Clean up.
+delegate.close();
+```
+
+### iOS
+
+Using TensorFlow Lite on GPU is as simple as getting the GPU delegate via
+`NewGpuDelegate()` and then passing it to
+`Interpreter::ModifyGraphWithDelegate()` instead of calling
+`Interpreter::AllocateTensors()`:
+
+```c++
+// Set up interpreter.
+auto model = FlatBufferModel::BuildFromFile(model_path);
+if (!model) return false;
+tflite::ops::builtin::BuiltinOpResolver op_resolver;
+std::unique_ptr<Interpreter> interpreter;
+InterpreterBuilder(*model, op_resolver)(&interpreter);
+
+// NEW: Prepare GPU delegate.
+auto* delegate = NewGpuDelegate(nullptr);  // default config
+if (interpreter->ModifyGraphWithDelegate(delegate) != kTfLiteOk) return false;
+
+// Run inference.
+WriteToInputTensor(interpreter->typed_input_tensor<float>(0));
+if (interpreter->Invoke() != kTfLiteOk) return false;
+ReadFromOutputTensor(interpreter->typed_output_tensor<float>(0));
+
+// Clean up.
+DeleteGpuDelegate(delegate);
+```
+
+*IMPORTANT:* When calling `Interpreter::ModifyGraphWithDelegate()` or
+`Interpreter::Invoke()`, the caller must have a `EGLContext` in the current
+thread and `Interpreter::Invoke()` must be called from the same `EGLContext`.
+If such `EGLContext` does not exist, the delegate will internally create one,
+but then the developer must ensure that `Interpreter::Invoke()` is always called
+from the same thread `Interpreter::ModifyGraphWithDelegate()` was called.
+
+## Advanced: Delegate Options for iOS
+
+There are a couple of GPU options that can be set and passed on to
+`NewGpuDelegate()`:
+
+```c++
+struct GpuDelegateOptions {
+  // Allows to quantify tensors, downcast values, process in float16 etc.
+  bool allow_precision_loss;
+
+  enum class WaitType {
+    // waitUntilCompleted
+    kPassive,
+    // Minimize latency. It uses active spinning instead of mutex and consumes
+    // additional CPU resources.
+    kActive,
+    // Useful when the output is used with GPU pipeline then or if external
+    // command encoder is set
+    kDoNotWait,
+  };
+  WaitType wait_type;
+};
+```
+
+When option is set to `nullptr` as shown in the Basic Usage, it translates to:
+
+```c++
+const GpuDelegateOptions options = {
+  .allow_precision_loss = false,
+  .wait_type = kGpuDelegateOptions::WaitType::Passive,
+};
+```
+
+While it is convenient to just supply `nullptr`, it is recommended to explicitly
+set the options to avoid any unexpected artifacts in case default values are
+changed.
+
+## Advanced: Input/Output Buffers
+
+To do computation on the GPU, data must be made available to the GPU which often
+translates to performing a memory copy.  It is desirable not to cross the
+CPU/GPU memory boundary if possible, as this can take up a significant amount of
+time.  Usually, such crossing is inevitable, but in some special cases, one or
+the other can be omitted.
+
+If the network's input is an image already loaded in the GPU memory, e.g. a GPU
+texture containing the camera feed, it can stay in the GPU memory without ever
+entering the CPU memory.  Similarly, if the network's output is in the form of a
+renderable image, e.g.
+[image style transfer](https://www.cv-foundation.org/openaccess/content_cvpr_2016/papers/Gatys_Image_Style_Transfer_CVPR_2016_paper.pdf),
+it can be directly displayed on the screen.
+
+To let users achieve best performance, TensorFlow Lite makes it possible for
+them to directly read from/write to the TensorFlow hardware buffer and bypass
+avoidable memory copies.
+
+### Android
+
+Assuming the image input is in the GPU memory, it must be first converted to a
+OpenGL Shader Storage Buffer Object (SSBO).  One can associate a TfLiteTensor
+with user-prepared SSBO with `Interpreter.bindGlBufferToTensor()`.
+
+*IMPORTANT:* `Interpreter.bindGlBufferToTensor()` must be called before
+`Interpreter.modifyGraphWithDelegate()`.
+
+```java
+// Ensure a valid EGL rendering context.
+EGLContext eglContext = eglGetCurrentContext();
+if (eglContext.equals(EGL_NO_CONTEXT)) return false;
+
+// Create a SSBO.
+int[] id = new int[1];
+glGenBuffers(id.length, id, 0);
+glBindBuffer(GL_SHADER_STORAGE_BUFFER, id[0]);
+glBufferData(GL_SHADER_STORAGE_BUFFER, inputSize, null, GL_STREAM_COPY);
+glBindBuffer(GL_SHADER_STORAGE_BUFFER, 0);  // unbind
+int inputSsboId = id[0];
+
+// Create interpreter.
+Interpreter interpreter = new Interpreter(tfliteModel);
+Tensor inputTensor = interpreter.getInputTensor(0);
+GpuDelegate gpuDelegate = new GpuDelegate();
+// The buffer must be bound before the delegate is installed.
+gpuDelegate.bindGlBufferToTensor(inputTensor, inputSsboId);
+interpreter.modifyGraphWithDelegate(gpuDelegate);
+
+// Run inference; the null input argument indicates use of the bound buffer for input.
+fillSsboWithCameraImageTexture(inputSsboId);
+float[] outputArray = new float[outputSize];
+interpreter.runInference(null, outputArray);
+```
+
+Similar approach can be applied to the output tensor.  In that case,
+`Interpreter.Options.setAllowBufferHandleOutput(true)` should be passed on, to
+disable the default copying the network's output from GPU memory to CPU memory.
+
+```java
+// Ensure a valid EGL rendering context.
+EGLContext eglContext = eglGetCurrentContext();
+if (eglContext.equals(EGL_NO_CONTEXT)) return false;
+
+// Create a SSBO.
+int[] id = new int[1];
+glGenBuffers(id.length, id, 0);
+glBindBuffer(GL_SHADER_STORAGE_BUFFER, id[0]);
+glBufferData(GL_SHADER_STORAGE_BUFFER, outputSize, null, GL_STREAM_COPY);
+glBindBuffer(GL_SHADER_STORAGE_BUFFER, 0);  // unbind
+int outputSsboId = id[0];
+
+// Create interpreter.
+Interpreter.Options options = (new Interpreter.Options()).setAllowBufferHandleOutput(true);
+Interpreter interpreter = new Interpreter(tfliteModel, options);
+Tensor outputTensor = interpreter.getOutputTensor(0);
+GpuDelegate gpuDelegate = new GpuDelegate();
+// The buffer must be bound before the delegate is installed.
+gpuDelegate.bindGlBufferToTensor(outputTensor, outputSsboId);
+interpreter.modifyGraphWithDelegate(gpuDelegate);
+
+// Run inference; the null output argument indicates use of the bound buffer for output.
+ByteBuffer input = getCameraImageByteBuffer();
+interpreter.runInference(input, null);
+renderOutputSsbo(outputSsboId);
+```
+
+### iOS
+
+Assuming the image input is in the GPU memory, it must be first converted to a
+`MTLBuffer` object for Metal.  One can associate a TfLiteTensor with a
+user-prepared `MTLBuffer` with `BindMetalBufferToTensor()`.
+
+*IMPORTANT:* `BindMetalBufferToTensor()` must be called before
+`Interpreter::ModifyGraphWithDelegate()`.
+
+*IMPORTANT:* By default, the inference output is copied from GPU memory to CPU
+memory implicitly by the framework.  This behavior can be turned off by calling
+`Interpreter::SetAllowBufferHandleOutput(true)` during initialization.  To copy
+the inference output from GPU memory to CPU memory, explicit
+`Interpreter::EnsureTensorDataIsReadable()` calls are required for each output
+tensor.
+
+```c++
+// Prepare GPU delegate.
+auto* delegate = NewGpuDelegate(nullptr);
+interpreter->SetAllowBufferHandleOutput(true);  // disable default gpu->cpu copy
+if (!BindMetalBufferToTensor(delegate, interpreter->inputs()[0], user_provided_input_buffer)) return false;
+if (!BindMetalBufferToTensor(delegate, interpreter->outputs()[0], user_provided_output_buffer)) return false;
+if (interpreter->ModifyGraphWithDelegate(delegate) != kTfLiteOk) return false;
+
+// Run inference.
+if (interpreter->Invoke() != kTfLiteOk) return false;
+```
+
+## Tips and Tricks
+
+* Some operations that are trivial on CPU side may be high cost in GPU land.
+  One class of such operation is various forms of reshape operations (including
+  `BATCH_TO_SPACE`, `SPACE_TO_BATCH`, `SPACE_TO_DEPTH`, etc.).  If those ops
+  are inserted into the network just for the network architect's logical
+  thinking, it is worth removing them for performance.
+
+* On GPU, tensor data is sliced into 4-channels.  Thus, a computation on a
+  tensor of shape `[B, H, W, 5]` will perform about the same on a tensor of
+  shape `[B, H, W, 8]`, but significantly worse than `[B, H, W, 4]`.
+
+* In that sense, if the camera hardware supports image frames in RGBA, feeding
+  that 4-channel input is significantly faster as a memory copy (from 3-channel
+  RGB to 4-channel RGBX) can be avoided.
+
+* For best performance, do not hesitate to re-train your classifier with
+  mobile-optimized network architecture.  That is a significant part of
+  optimization for on-device inference.
diff --git a/tensorflow/lite/g3doc/performance/images/android_gpu_demo.gif b/tensorflow/lite/g3doc/performance/images/android_gpu_demo.gif
new file mode 100644
index 0000000000000000000000000000000000000000..31aee24c34b6451727abffd95bb7f2f0d40f55af
GIT binary patch
literal 1475626
zcmZ?wbhEHbY+!O={4TG_z`&rwz~I5aki)<*fq`KQ1H%mlhCg6Y6%_^#6^0xYh6ySR
zTT~css4)BiD^>Ae@bF;B@nD$X!LY@H;f4ppAFzIv90rdZhMXLR2{{Z~au{ypF#G`<
zsxpDWV**3Y1cnI{7`9AcxG{m@57;J^Eesx87;?5SOxVJ(WedZNEewCa)~noL@VLQ{
zbAw^R4Tdc@7;fBP_ycy6${z-gKMXm47$*E-*z$+r#vg`1AcuykFob$Agyt}WPGAV#
z!Vr3cA@mPJD1%C<ib|-5N@$Ks=meF}Eh?cmR6_r#gfe)9s(6HYc!cJ7gii1X-Qp2?
z!z1*MM<_#1s7g+#M^0!?PUwW3&@DNkH*!M%<b*Oz2vwO7>M<cSXF}+N387migx;7C
z`e#BY!<JB$EukJ;LUXo+PS_H<WlQLdEunw5gfiR+Rk;!BaU(S6M(Bhap<8Z*-nbF^
z=SC>QpHP)Qp&oxibN+-*_!GM2Pw0(5p@05_GBB)CVOZtCuquaP)dYrBTNqZ|U|98s
zVHJbQDixJg9xAJHR8~z;S+zxF)eV(Ze^gd6c&t+KSmoidD#v5h1dml)JXYQCSoOza
z6+_M{m7G-`IjeGVR!zuRwIyfOjht0~a#k@+Sfw&ymB)ltITKb*n6PTggjF{ttok!y
z6~mTQDqB{0Y+04FWz~c&tF~-ebz{q_KU-EY+*qY@W0l8^RXI0SO}MdY%Z*hxZmjxq
zV->@nRVsg0dHh+G^JmqBKdZL<S#{&jsy}~LF);jBVfgRC@IQy){{)8rTNwV|VEF%s
z;Xi}Qe-)Mg9xDHHRQ^v;`M*Ww{|%M@e^mZ6c>GuK`0wHIKgZ+$1dsn)JpSMC`2WY_
zKSR!cm7M<`IsbEV{!hsHza{7Yjhz2~a{e<+_^&eIzsH3CITQX*nDBqgg#R}t{QooI
zKf{*)DqH@0Z26zF<^O~&|F>-Ue`Cx4KU@AY-1x6@<G;s^|2a4QPq^`a%Z>jxZv6jq
z<3Gcn|0;j}d;IyI^XLDBKmWJ<`G4cj|382JEB@#9a}5c0b_{Se(lcOYWME)W{K>*1
z!XVF}!@$760Lmr|9RC^aZYg|x%2%?9O;Aq9V#2~h?R?yNJ~z6OW*bR(7ic=S9XZw)
zbD~4Uv&-k?WVPT$I+j{b)r=M&+jB=nIPL5-eeWV8&COHyMy<`-n(bOLWm@R!YpSuU
zwrE~n?zh-Zc5Buht-w(CMA_w~Q!D}(ZE5;?>#KHHVbtcly}LwK2d(nDyQ%l~wsjWU
z(!!tK<^HA>5%}oHA6XrVoW#I(Vg9%`TlMd6u8@7~yL<Ykr)#H6xBp!oTJU^-7q9WU
zo=V+2q058yLw&!$4O;U0SVorVmbn4Z>!J!D9jZ2ee?O*%{lD%Wi;s_`Yo`b1*;G#N
zT{>-g=ed<W%Z)B9bw3~X*XsMr)Ljk#_sh+-`SIe#`PJt6^RARk&W+ltc(v<l&7xed
zty)J@Pk*?#XX3&+)%TaCFQ{Ty>v-VEqju#%GoKsB!xkYo54JLiyd{ZkGE*JBq;*?8
zx;hjOxg7J3p5x)xrLpS7qb{wsD@vN%wgf2$Y~5hwrM5<6E0^};4WAZ=Ixh+n4RQ-g
z6%F-~^ArsW*d+RNN>G^R)2U%;olmDlm7RP#rMq$J$@p0lEU%@-*1mi?D{EV3>a5tQ
zlGD@*54~JA+kex{ry5pWj$8qz*Iu0R`zLzq{=$EZ$D<b=ustr_F73wka>=AHua`@w
zrFA8ip3HrfxqN}wcg5Nz+k&)KYRgGoS~a7pYGJi_*#GZY?N<|@%v&o_Ib(G-2X9EA
zCP!$<gbf^`DLETY$a%kUSo)+qaGgTo5<$t;eOo4Z-H3};^1ZXlVsh8@X<Iy{zuqts
z33QfJy}B!4>*;mDVMkX_oBL%^=e1aGInDK<ULSu?+5i5P>WA0~cU?amV4BZ5O)Jq%
z=T;(LyQkPxHuatl2St;63Nouw+tktpon;F5`n}37+VA~#chsqO>lSfJZhg3YcKspE
ztj}jHm@_u#_r1Eqs(wCFJ+{;(Tm4JvUD>r?N`3S*<~0TAb9}7{slTD4*)A2nxq<1`
ztGFu(-g^^TMW*QnI5E$(eYkm6-j7wDIa^%3?igH*Rq`{~Hseuu_2K^>8ao<qo$%^g
z^WlebsH<Gxl)i4|Je8on<f-ZN%9m}OIWNYq@8`4fxT7251HOF~f3YO~-p`kFHfLJy
zE$VZ6KBp{i-OeXDXX8|#ZvDQJbAe0lLu>J18L2(>2S11HS+y$Y_lx`qY<J66pLe%k
zSs@)LFV`k<RZG_Gs&{(rimSa@uYUYGJ15JdGs!)7wRql?+I8>p`}h74f7CTm`iS2W
zdzW<gEC2uV{=DxPHF34klm|{1yTUx3gx2L)2E>?Editnvtm9ggB5b$w;?AtZDe|mu
znsuASPro=cb@P;^I?0aQ$JZ!{*E8z0_y@YjPAQsx?ab4*qQcY9&n)8F%+O}ApmRpt
ze;H?&6A~XiEVoF+^0_%}o3bI9t9wzJi?i#IoMXmw3{M}K5MeZB=do_LUr7?jImE*{
zlDOC0INw>fy>qtdbNhlC<@2*P%iR0(Ch*itmkVEZ2)=JvZgu77;|dl@*^g6>XC`sq
zRutDX`C`#+73qAgp5@C5)~4s#oo7;RoH*vWwRF?u3y&|XNH9_g>Ql;FD&wlPck1@A
zPrj~73l(~EM5Zj*u(-fhC`iwC#p%*7LY{W71SfTD*;KRe-Hh$4=Ne_UTc=oE_FCk9
z*LZp7&Z@}Cvu0%ry)~M?Bs8N@MAh)hhX(g^k0rYTdy?eD=ZJf~+_`z~<W+qY>x|F8
zz8IV#v2(f2Qs)15{+;V1yS=6^tlMn&^5`n(4U^8TNlUeleW(>7u$E(^+N?EQe5=EM
zoX&79Dm}ob7*KmPYk`NXh2zXCPoh<`9(dnM)LV36nb;DGE(1r8V2Lh)_{lr3b6#I$
zWN_`va<kTjJee7Wg*AFf9<q-Fwbc$wajjXox<M<%Sj}MdgDnZJv3JuZyfWBWvnJ&e
z(_`*Nt*P=8lelwp9$OY4yS8nYsHHu3)I|%^ElVYPZ977isC{2`eb(NsQF9M1iDO?a
z=eO+bWOmCn>WW`3zUWM`E8w1Z_l4Q^+?|(P4XOflqAgE5@b-zedUNU4?CiV9b#lvU
zhqVU5(r2!j@p;~!vh2-`CXqG&uZV5iv3G4==4r9e4`SN;wyw=tUvlN4)bv^LU2|V<
z*`;>dvV5jnm+t%-OO(R|{3~wkeOKE1UAkB=zw$%P@(TUby`Pd!RNdJf_vg`a+qbdm
zdnYlel<oR0t#!k&_7zX+Z>ueFKYp6;W$?>!5mplX=d04d@ofHjy+>^?3ce`IJ?wA~
za+<^*6_KYl>p^E;kT6GF_EISoFWno8ZGk;zn#R6{k<G8xuU1pl)p@drckZr7QtY9c
z&atieovZrJf3@hcjgjF0f5}<?_m4I|-$lF|m7G<TE88r5yae|~9W_ogY^|+K+E;T+
zPyCR8S$FdB&K(P`{VQOYTc5jH=B<bOvj>9z`z5%ZlyTfVVYQrnea{}_`0rhwiyrnc
zTW^-U&UIzs3y!k$TuwT-MD|9o+h*@SFD<I&nR3c2f6)}(Q(sc!H{BHJRVuJZa@RId
zjJnqM?aH#Wx0$skPlv7l=HX%Ztnhlvl-+N_+7ixv%`STzZSXs+ZQow+f{#YoPsD!T
z2v1t~GVS-B-6k(@-c?a6UHNGDgUH=;ZtVNERs3k!n+?T#9`HT2Gj|CuD)zAXePy|A
z@z=PzUAOP7K9qNEMQA|9wAo=CXTLj@rUrJ%y}o~9QidDjx;1R#8y5feU9eThCTPlo
zBTGb<D70;LWs{!UZMIfnqehWuRA2F*V+!I6_grhbJWuZAF^&KHm8XkrLeF<9od~$~
zkgu~=d?i;-r~lH*kaE4lGS){|hMbi+ykl4BgrzIJ)ch@XpV@WHh+m}grH^s+-&dP0
z*^b|Jb&iVt-1B0cuvv~r^WrNr&BJ4BYtN?(8?^A9{^n}_%Q4qr#V<RR?I%}l+BDI0
z;<Df07Ul2d$e+I~weM9hx7E~dAC|lAc@&s^>#+VgcXdCRO}n>5KfD!n+@sS*cY?^z
z$S=Q4=eY~5GmNji!*~0}iupB97c9xKefo0S`LsJF@vC>p-wbnKyuxhT`s&=blP2%p
zvr&J$>6g0&()CrJU(fq=zV_ZTefzyH{;mJ>qU!wwne)33FE8KmT{_|I(f47DEEfM=
z|B4?-)%>CON%OF)fLvm$@q~vT_9w7B?>Fa9=04(hbHOqHnjdl>|Nr+j4AxQ?)n!Zd
zV=G~v(cpDm&+=GG&d20H)nXsBV!n!ofUsiz8I3yIMJ%VKct33HUl`K7P%<E-DY~qP
z>xNN{Qd*{{XT~ws^lxDy;_j1DN>6@FGnddRe&$}3mR=c}{%L7?x_L`!h;hR)Y4;A}
zGaK#RDaOwz$%sF$lX*NNWQAESL*|>uiW8@LJuu0fnijc!OZkQ+N)`vL7lu_NZ>Y$c
zQIWGEYt7e+0*UO4ORGYIvOS6_?KTzfG~>2A)b8=HeKkWz(~R~O4xdHsmF$zNx;)yu
zPx##Z&ywq+RNcdDxyyuW=XTax({g9ssDARL#QbsYaq%u0r`m^4HJ?q*o7W`RYEt_{
zOzX8s&COPgRZKxkHMDmf3gDaCy~nBGpIO1a6Ws?m0{Ny1%0CR0+giZOUSD1uWS*#`
zoT~hlK}6y>U$O#M@AG<D_o(YT8dx%nv>vD4nbGu6L)3PAv153MbhClQx5gJ8jqf;$
zq&-4HO2v2{hpGsx-q+}RH_1EVamoIVO`9Z2lf#<}zDk!rk(nD*7Fix%-IgBreL~=2
zP3B6O)aP;m91|ZHw&pL-=yEkHWcL@#EU#=YuV^=~Ymb~|#-_1TX-_iu{LqTU8`_Lc
zwyjxKdC<k@(*L6!#Sv9DoE=*lvh8lR`zB7V=a{lsJ?6mD$-N!XfkjnbJJnAu@!j&>
zVsb=mdvIs+g6f$wI`>@f-2Z**4fFU#7Sm2Hk4yV$^XzIo<FUA%m*W0~YJNEy`_0tv
zsB?Ewcu&^M?vKt1`y%St+5#D-C-6<IlMJ+fyIfGlG5_R_8Is2mEiWdDwdjhei|AeD
z+jNlYjY1To#gy4A`;$@|VnmXSw`D(mG4lh*Ebq2if!kb+uQ!U{&@(yKxXCkQ<w~a|
z$+Hcg<!A?Ycun_yQ=pN`*37^u(<#(^;2@``#{{cj_tP)aEKkgdXjin6NOxPT@JunD
z&$H#0rR;%;E%zid&U=R2{@*e8VqojDoe_VQ#w$!pjL}e<dUc+*WZT*;ky@ROP2wWW
z%@RvfvpS|{t@t*7LU>1!hiK=IDeg<urnBc%dCaQc!M9nUgO6)rdxr>D)B^9yg<`_0
zVp$7LPSNjk_OXBD*e$$hi-A(q&Q1-l>ATgtyFTSE^_aHqhvuWQX=@~Ee_ok(<)__y
z;l)Re$40-LdMUV;t5uPGYyR#Ev;C6-_gzY8c-fs75a`cU@G9IPrJ>;U=fIwsORKK;
z2(s6&N#MTtB52D)tJM=`&W)V)MoC{U%uvcp()|0fjI3FVpZk2K^nI={v}!NTS=Cn&
zH7j81?8qR)q>x$mhng}(C;UILs7q;r@c+s=WlvT-tSns;VZzxtF~)t~v|sLbBW><?
z8keR{oIYjc)14ES-15v>JWt&*a#dCN>Q_Q*ROagk&0n{R)#YKN<srwYgd{na$*bFX
zXPQs(*p%&I*)H+R$41j&`?Tm2zoKQL)?B#ib9UF{{=?B7#+?VfxUZg4SO2wG>s7TD
zSIvHRwnORD^`q7qWKGM|ti9L1`0ny``J9@Xzo!2D>i6<!?dQd|H9PJ9Enfe#s*csH
zCsiPkjU`dOvFGFRo-fa5Xc`uBd$Tnk4idl4x5gnW?UV?UMBl$fDt;xBUan0`7K#~d
zn`K@iRj4KEm*y16vT57TjoRGF2G3j!c8j@Rad{vSqW(X+NmqOG?#vDambAdenK>Ic
zGrn5i*~WT|v-EIdT0YCl$yd^H+L|w%lqpi3du68VO-qlSmbnqys|0qg^f<Bd!A=>U
z3=_v66P{OY6P`5h(&u^aU(SnL;Z-g?zvy(PM&#-ya|^eHt9NhNzD~8hjywDAlQm7!
zYn!8Y$Z+KbDovfTWA)0TTw4^j+>u(Zb*l66krkh_*3GD1_ra{|&9$AYPON*daPdXX
z=@YAM7QL9Z<AnBsB7xPXmoPu+K2oH^D6-U}%Is{AgT%xFNvC|#WlNR67Ti}V<S}P6
z5n49Ya^vEFjjGv3UP1clP9^70OUN(InZ~^NQ<2n0&)NQM#S5ADdZ-$O{7)-!E%%kF
zwqzF$)8KFlbIoyQSz+bv6Xdu_BP{BT1NWDN{okUwFJ{E<+Z^_9^@_tcS9EPkJNr^|
z%C6E&3rlbRG>Pe6x$hujFvG!gwu2rsjOS;S&FtDHcxmDb&iEOJT9j|D%JGg|qtd1+
zsjQc@-MndgR6=ji&lt;>K3A;OlP_AC)u^pa-x055Rcol0ps2p+!Qn*=N1SxjmnvHs
z*N9%7U3H<$cgs@mWnuf@1n_^kz#kfO^y>xwuMhYm-tb4C;YwCu-EX`z?&Q=|o%|EY
zyG{q^&zik+;TPL;>a`y)=jCbcDvg;oDQMSz^=VIcO}o$}*dChja(dwU-S*P0b$_)4
z&;LIZ_%)b`f7*=q&v&a#-5|s=lV`V({#Hk!)ts3gZq}<eu9~PHHBD@q#ok2$z4Jdx
z@;6J&Qjkcxa4IaswQJ(3FqhMrNvV!1Qr{#_?~#$(E90^*$928TnInHr?}|BnY>w;2
z-L5-M#;mkFWfgrYQgQ#6?)_f^_`hD@j=aNv=gir?7tTIdbN1exvwt6Of8F5rPjF60
z(G~_zV-M@*4?J7XNp3B;<{sjm5ymao_N2L|WNvi#N|zVu*UT6Nd72Zf4}6tb)t0tx
z>8%TWZLQ~p?N(gbR=8MsnMr0`#-Z6}LbJ-|Z_kRIb-+UWl-0W6OQ}VdoNbOI8K}+N
za7imxO}F=wk?oOn|D7+dZB|>Kte)0%$$riyW8TY}XD>O*9MRLgqI>p|YwQ*0yQh*r
zT{f}ZXOeTuy?tTd=J{I=@O>#b8n)+Z<ejTgf3C*$9!)rVHD&M7wFlUO8#>e19FM)Z
zHg0EF*P~-8noHiREcvds>+lNO!#l1o@a~$Zw6nUV#%xyi)5?hUoEv%WCxR<(q%iBU
zmpBL?W9^tZgMIbV(;9m;j~1*=*z>|@<Ib$k1+k}|a9`~TJagpEsTF@uMK#=7efQR^
z2e&r-y|vQz_F<pfYhrHisJ*@2_x7IN+k4is&SbcAVDF_ve{UVky|XX&_UgHJHpHA+
z9dqlD%w46$NBVZm49`8g)q(%f9sc_~=kERgd-uWHySMM$y<>ClPVBkoI$H{QSInES
z;;{L|k22fxmQCP{xgTDwz<2V%^jin`H7^MGTo8)6a8h#KqD6AzlBQB`F1%8Y>klno
zqZP?NuY5U6n|9?S{f)x9PTqQ+6$vt|s=U?;{~jvUT{87mSLwUtyZ4H=?U{qMR}SBO
zbV&Bj#1D`3_dW8cz2e#X*xc`mdER5Yb&u`kp6L4B(YtraIPQsy+~Xs2S-sXhcKrLq
z$@a2l-Q}yZqXK=7Mz6j4_rcM-d(NilUQ76QHF4h2<bO|-?_P~N!xd$4Y@@`^d$*ot
z$V^+WaXpXc_@&9~q?_W_WE|hwB*&({E9d3GBg>DMTE<oVt8Mu2BPbSlqO0~qpPoZ+
zo#1Qrn<)$#R~`w<CB9S)y~(jUaUS1H%a0CQE(X20#5H@u>h&?bE7#pVeCE}fzqdEZ
z-riJq=V<KfEq0HO-MzJC-|JoXUhkB9y?-97f!&(}x^Ir&dvt6b=gEC<PThNR=--=D
za-8S&-d^N;d+pvE!*x$K&AmPEqKb3n{x1{m-h6xZf$zO%cK2S`oqKie?#pxUUhlj6
zq3_+l3;YRLuX3I=c|^>)(Z1!5*nN`~#=g=DpKU%AZgY1uPG5ba`RASU(W~$OsV(ET
zVHGi8W$<Ga;b}hmQ)B4`Q;z$koPFD#T6q;zUo7apsO<kxYHg(U%7;H@DklrDZc>zX
z{;%_S#otRd>n{1uee8Me$-%QvjOAFPK74ju&*~8W#ijm>+x#!?`d@tZfAQo0;>-Wl
z-~LO$d{)=|uO9th!uEd+n7|q<&l>aoYtsF%@%rBq;=d)#{}y%Nv)R8Zo|`Je=U$DB
zyOtjJJ<j)8(!B2#>#vsQKdX?xRww@>I)?w-zh^CcH>P&yirqQZ^ZwX7)?II}<yGE&
zzRY5mB$G{uz(M<}otHk_?TEP1dAH2lN?^(Q8!~S)Lb_f`tpBAfcyi9X9>qgFGv4-m
z-+t4ysc@SimmtHNP3L~^d;j`o-y5TOtj2n86Y~Fr$+Mo4=R9@)&*}Gn&Yu5szWz_3
z!rN>9e{R(Oz2*Ps>i_t^H_!h)qyP7A{ol*;|6IQRH$Z^(B|qz%_5WV*zdfoek?gSP
zy6n4$_pUyXz4z<=(_i=hGhFn4;Lya%E9Wv}L;U)>eXp)cZA}%P-YLAjVCRg9r`>wR
z&FhZ*xR~rQLDhGj$;p!`;!|Tz6=*#cdUj^IX=0gHb8!01Aj`^MZzM%toS$dWDyBN?
z%Zi|7-gC|V-kfxJ3TMc2pSfL^(tLyLciYvzTGkV>Df#R)*Xpp`TU!k87AZasdUx9>
z`ox4R<5zmK1DAMwyt}$w<iWuv&dE1jj^#Z%+9hc1=ksb~T-dg#$aP1xKF?5zGRi!;
zNL47~;u81vzD25Ev#zd<+<$J9)wWkR1OI2aZ|t=;6}hvq`tvqko9==~dyDza|3zDh
zJULTo-T$bj;`6h>2`<~^=2{isUToZX(!}e^s*kHP)-HBkKmS+N-&@x|f4AHB`{#$7
z>&xfgulx7=@6Z4F=l3%(9?5rL=5`4<!1A^D{PYgJSxQSd6sJt+=Fs1=!HGxniNWDs
zr6(DSXSlR@uiNormbR-%c*sUqvG|g}qk3U;E-p2?vrBN9O#YFL%Vg`nY*gF1W5FRO
zEdv)J6^WO-F1pFw-msB(Q`F2&r#7yT@jj)!wq)^CvrTh8otF00n8>Vm!o*n3=ww2$
zk@@?k&%TzAmjrJx6q0`GXY}bvvA@;RoW-)yw=$PbjpC41{m<pOB_U+;B^lnc7q1x2
z?7iffd|FA!NBL!M@Ex10Z!dF{NQ-U{T5Mq&`amgSYiMxnQQojO3ldDNH0Cukgon@X
z*1N7@a`lU`?-7C48-d3nkJ?mSTHQJ?>Q*#IR2$E%Fon#8XZ3G2uDctyAa29cu;{|g
zf44;!o|ZL_DLVMo>`s1eqN`GuQ_#yrzGfFP4y|!|GF_!!KWpWZ21l+P5BhngdAj)R
zn$Y=hy39M4M@DIHIG4@SSoiYL{GOuKkLDXZlSuAf$kd%Ox%$@%t*PsiDwU0=K6&EO
zU;XRVldXAa-05>~du?oyNX~kEb{~s!YsTX5I<wX<lFwd~xom#g8^a*BqdxzQ!y;b3
zF}=R++OF*D2E3*@cPpNLwc5Pd+B$d3b-UT-Tkoe?=WSZ<eb(C0L@<A9V1!}++^QEZ
zpW6Lk>tFxjprp+o+au!PHic}WAu$C^%JDIuj>)I*DLVTwJGH0J$j7MUg1flw$@A{N
zIueDZ|Jhl3Exz3L+pM%Jh9_=izkhk`PVRF-wVC&nY_)mpo93>&wOeU%&$>GkHdSug
z;&Qt3#oXhQDmQs)u4XZ+Q4>CEyuZKuto`x%oBvpSzY@IOj#XKHW7sRnEuv1h8#6+5
zRdjQ=Tzuu{mvgt~b3B9dQmwG7TwDSUPK@(|53ndPHtuYCu$Ir_V9ecDjly$6^EIAo
z`1Uyd7g;QERO{M7t}=tukzXCAJFRdPDBXBEiv6G%cVfo698r}vlaNHGycH8<g_cAy
zb@C@YGo2_fS5L)FhEphT(~(BqjVd~c65cxuCTr;RsF|-k=~vb?MVGHNIsOZ$gtwX7
z?1>wb6lR_}wPBJ_YoTjp>L1T@Pj(nvbuHyN>3y*?rbT?ui>LavKa&&wFwJlfJLVtg
z{3KaObLI-)wPMpWL((DySv4!pPTD%Nx1cXrX8Tdzy!l_2s4u)=w(G2#$^C=3E52RZ
z9`(c5_RUIv9Zq%mC?&sXUsk3*%DcHE_S>hT-#h0#6)RC-6)i4eF?v=Vv9VM4iMR8T
z=CqeuR~B+so^W3EUplBeOx-$9Z2jzY*+K6so-DaFF?f!HQMyIDim!mCXIaIb#r+Gy
zZS`|c&Wq)`Z2R)3s`;Bfp1Y5mX6{+8wqwTm1#g9=etnp-@5|3jzgNl43tk0k{M_0k
zGBu}n*{;i)eR(G4%qpQC@mIBDECrg`U7A=mXIz^6>s9@fUp^Z{X6gvd+7S89G2D-R
zQgHd#EwT2t)BT&dLgopDu5gvj3Yxg<%3`ytEB?K3*LE}1Z)iQDBysaV%cic0YjPTy
zJ#CcS?b;0*WzVdUR?PN}TV&Acc}CTu@Z=e*X@>fFXV!U4TrKf1<Jexer^;c0$qx3P
zjJ6)rSy!U^SVlDZ`0Tw$)7L+K{9mDc*_Oq;yqT?$QA%^CoiIGG(lA(f%EC2fo;^{M
zvSXN~m+fFa5YY6=>iojl>F>(7?^!!Z>`lfsUsiSR!w-#$_y5T)Xta)b_-eUbk8w;<
zqqa|?(Dj{1d7l}Ix_4Q(PIY?IlUC-Nez$(6iW=YVW#&hv<E|fUDH9eBs!E)+=kdoC
z+U8Di6Lu|H#LeknaqZ2r_fM+NzqIYwz4ZBg#D?_q(uMvT?lG_U!M7`+<yWVF@`K6E
zvR3<!v}{}SCD}DwOjKusr21BQ){QqrG7j)?u`k;!%G%-@aXFQdul3TYg*=yCoyD(h
zIHtd*kiXQ%x!EDxPXCDEE*Ya0>5^BYE$<ZeC;#V3k-oIU@vXokmsdyc_T(&_CBDgM
z>ee?ddgVXY+?eu6w6@7r_tw_#)MMGPKX*ObH>prfk9ke#nJ=3r=d`AoMd!?u%yzFh
zb<Wp|`-!}&cA}%WP}T&;l5}O`nY$y{j2Gp$ig<NL-3*B`-5>b2J$f?hhYMj-YZkYr
z75w&!`L<1d-<P#|#XG$pue(*WBCK>(;$82Eb+e{@%uIcF>mXOC_VlY$y7M+f@7<ZG
zlu>Z{2#4glKR$2Fs!Qf)t9MNCRmz%NzUO*bZ108FLb>JDt`Au*y*>BySixeBlHi$5
zFT8?m3g)NI`}s!5{-vi_c)`l5r5Xo&|NXRW<TBICPAWKX^7w`St?Sm;cJ4gm!t1(%
zXSo`8<c5%Czt}%|ImT=|JZa^<jHpAxX0M~IY&T7u`s&KWZkd&@jVif=^@^r^Up)1F
z@~eeeiTw394vQ|`&}u95h`;_v)3a|ISpSsxeV!z~_o)x-tBdkRp_-RpcSPQE3Y)$2
z>Wb{pc*%vo!d;DyPBsx-oBs6I^`rZ)wM?~D7bwmSu`3Jzaw;?>=lP6)N9Vpx+&J^B
z@j<USs~>GT*cZFMi1+k7R(tmwFF*Fy37T9~Nxpf~$Zm7Ty_`k1){?6_w~4h*V#zbp
zjrJ?Q@L3_HHTU9T$*L>O(>I5_+mQS><{J0+9c%6P>gMv^z5aij-ByKGTbc6g9~b`L
z5fy8_zB!9;eOSk?ACB|+ix=*$xpBFDSDE3W%}eLUW>_0`6*1~>+nS#%I`ewP?XvE=
z=bP@CtG(a*RNeg7dHsKaSHovrGqazuBxG-S^T~hLcK?s`Qe0-lu;qR>(|of`gT`u&
zRwwyyiSnOU8Ys=4AC}Or`(lAW6l<7-V&Cbt?G+nEoQ&84jnqzWT=M$>Uyc!f$tKm?
zE5B!`Mm<oJNpe)P-dd&5TGp^CE0LqEpd;+U&dyhyzXW&vczLk%&{}EZgNk<!WN<E8
zwtCfz%|<I{H|b>0%<8gIb&}Oz?6m%|l{VM@wLfHyxRs4xJDL1GWG3vqyP500=VFWK
z#p@QYo~0@I-?sSh+>?v1B~D(z$>Ydn;-IWzlOlgMXo++366Ycl=Z$;zI_`Cvvh`-M
z3}ffgltXiFMDLB?yf=Q)*05Erg0l|rAClgGu|-Fq)p?5TeXFfe#r;tR{RcO;ubjCw
zh<Vq_g_h+E`|f>a52~17|6^x;!j8vHOP_g}UVVHtCd(xzxhExh+e_tRJPViYzhyOD
zLrCwyGQop(?{3?@{cM(3%$@ssU!jzFq4xHo*T&qQ+kblrW$zZO-fUjkyuDJ`piYaK
z`Stey&D<X)4^)_FG#MXYSu%OX#GaP!B|kN1DM_yqH(n*RhQ0mAG)Wc9CE8XkAt#!7
zriVTd@q5VYf0D`9qyM1*^Zy?j9CMpiiMT8b*dr}EM=Y>`!FNS#+l&=u1sxNNog`d5
z>pSMltg)PTV&@!d5uvJSttLB#V|Fgu%vvU}NacxDwV|7u$|>%JyB2S;TEnERt)kt;
zXx*Z?dxefp`^43Ur6gDX={$VNYe%y6&c(Y8MVxn99A0(8dLOs7wKtbp&*Fp0r#CqW
z&Ns9XeYp6v)2yS>QpXSL><*DXVQsQ}V$U+oDXVtcbUZNiNZC6@=0r%&zN0RB$71##
z)7k3c$fSE?_Qh<~%dgvg8k&3z^vV)geRvisNwan=ZauiLmHAZ5w-4-97noEcSPyBo
zeqU)DG3DsJ&8#77EUPtG!v#bFh3$@}?EC*9=FE=>wWA!XlU&ZF8z0*c<u--E_)+7r
z7d>v`iTm$w^t+e1zM^aW+be#Bv$&`IbSu5RO!cH+&943bqPSmZGMDObm+EYI?`ZHr
zQ?PQ$d8RXyf9Ggao6K*q7WujIWT1xUv)^L#KeI*~1XyRVS{npJD_oG#5kIQvD9Ga|
zo$bg~dN6ONB1dW~vlOeW5VOh&rmqnl7gPGqKbYcEaW?7%3&#tWMhRAyt|q=x%R3S5
zzc%a)TESYs;$W);$Nb6i@h>c^e_T}9s`3Ad<-FY&R8$Wso-w>Ld%9-uBDFb9YZe7*
z-d&|RS762DU1?2DT3;^d&Gt$=;xy|3r=HGcW8T9i$(#T0R6cCx+chmha?_r!DL1-y
zHFudG3Et`;Xxnvq*A8wglO@(HDz;PBINm;e%!9`+W{WGYkK-Ah4h7b#7i(Q?T;eO1
zoIY^HfoIp;Y8RI|rv9v<{v|Tui(2`GTEk8;sYtMT_L>In5EcEx9@x<rsu1ei!56lm
zo%d{0X-7Z5X!Df>_WFeOhZ?@I#SSYT>CVy?iHNy2>vMZqhi}vu)9~LeNvpyh2ie6X
zu?98RrCrwJE_71=6PCGX0>9(2-iTv*9^o%kCcc?`u6*sWilxUYS?#mm2&#K7m(yHc
zap{=CPl1x)?T;FlKUuiFX|8>-On238p_X3r%?~D)K4F@l>AsU;$NyhjZnSi5=$X6W
zlVtO$fF1wtm@-T?ocF1<O~Q(GYjag`^VCx2@Ey%n#jLER0rR~T^C~-1Kb{e6x-g6N
z;M^q#zgzI{<y?44h9mMXpO+->ov6UBl4)Yrn|OPf;+D3D?Dc57;o*DZ%q0<KjU67M
zy|+Ri+^V_|Dk!_D=zwFhOLYB)6*7Ar=Pb6I9&>HZ>YeF3PcFXXDQ7x;e(1^8p8hb0
zLn&9Riq~3cujV>5_0plEL2JBDt>V^RZo_Hyc+-|7?akV2cJA`p+N^yh*V^=P%-&1d
z=DC;mcU`t@joHj>vp(Xq&Dr35E5U`HQpdHo>{7X6`{wkmzuwC-uROTSBan3Un9iL4
zQ5!-|I|Nuq+%z(1-dxZ;-Sxr)<vB4&T|O-6ebB+XNOs+o)mslmP7^Q+6<Wk!<uT=U
zM^k9R)j*GxACBsl74-98wdobQe<+gu^@FXKdjg*>YJDxinkdUBRJG&l1tyIjXEZLb
zJT7t#n-P{g)$xbIlrRmMxycqYg|~W&p5rK6_oKn_@t$>stg_m|*Ppz(R3_E=?8&D0
zP1gne)_;ti_$p+*j?MbdyZk<Num9GyUp8|2-fAcQN<on}&OgTfuV>o7Ub$V4&ETsD
z_dnweKUf@CcSm&BZumXLz&=a<?1K4hJU1Cw#ieCD7Hnpn(i%C<wE3~C{$tbDX}LGe
zGFa!lVvll&`romckK+W3sI;Qs(Um-V*M3px-!0i(y5Ld#R*M-`zW-AD3O)FvJVe%o
z@&`IFXuL>nst9~`U`Jp=d)C5MkA#bH3GI=0H5(VOzm8~aEx4NaFsiX4zT-~H>bLFF
zzwCT?`rynIb{UpS^HNxURi;c{63z2xp?6%6tdV2J&VyYVoGO<RbWHC|{rNO;+NHFt
zWyk(rN)g+=fqQfN%|iwzmvy{?&wV{LDKU5hOYA0TZ%bjF8IEhLbG^5TOuk{b$4+XC
zlXZ{dS7lq9C3Y?$_ARsAWPMf_rf@7%jouORXe+a#S#zCHv)+&A4RYK;8b-f&tUVpm
z8WMZem1l3@i57k@{_h=6{(qgi_O->;stDHZV`s#=`+Lqc`PLn*n{eiOO6%(g*2qsU
z_*m6qegy<NEc9q-VpyiSBJ^cc!wC%m))lYVqggLbR+|$#MP8}dgte}Nx$Ih*!Coe}
zy%Uy&#5K){y>c|Jnj<LSTpD+{&fzScdwTsd%!<EesYYJU5`Ory$E|wl^~@#Qxz6f^
zLD}_B!*g2ncP#6+@HWqQrPef6sP*XbJ>MQS`9{2UJYH87(eP=%k>&~&H3y50lkF07
zxVk4VkUsO1(^k}E)g14e(?uUoefwO`tl7-q!XiGYr;3kv9aWs2c5!|Z`>g_zC9=$(
zXH&Y(7Ow5R6(`XeotfJlaB(8fh1`oQ|5+3~%XWxPI5zFO>ywG1923v9Oj`4Hh3?7K
zXKyt!+>S|XZFG37ae^fx){}kC)5tvW<}Wv!x1{b7&0ppeJ(=Ya+uW075v$ccap;<Q
zY1`iUF>lvV+fy5>yryW}-EcX7h0dC7Su@stnPDVrJpaHclg}~Be#GR275LqFHr4R-
zDOpMDx13i+pW9W-8Z9_{WOD4DCElfVdrmNaJeka6#COD<*On=6tCJ+RxkB8g7wLab
z?+j+!752mSiM7Db1YZ?_=HC(R4+~PhuV5E?`}ko;%caBju9duu+Hh4!HtKrbS=oHk
zz!_&!ANKnB%!%3|sy*HI)7hqkuW@dCds~k69@^6HBFOqbte4+W@0wG{E3@>^=F4q8
zYl@eaaa32R@Ms4WKTRmmP1C%Z^>od?y3FO1ckJ7!_Ru-};rrR#Vtx<B&K-KwvtPn(
z|En)<t2ey<X!7v>&DXxmg%re+8D5vj-rN5*=J?yj^TF}L6+Ul5%MCb+OIcJ*j`j)f
znmhmhoIGjmo3~ZE%nr;8dXXf2<BeGXE7Q>vQk%r)mF`@+^wG5|Ocu-7+fyE2N+}cM
z^)%Bv=_SMLRmo>~VC}s=mQ!*Zs@LDHy!ugE>3g>i$A;df=;JSZr+5TD*xFQZtH*{z
z>Pg^US@9MRqm~T^+dh1sWcyy(HGhHf57wnZ`jK6=lOxnWJ++HJ^uy2mzuKOxWHY7I
zZ9g{WaB3En=sc>(;5*D59?TN{At#Q@;_B{cg<gi-d;TX?ZtGDmD&w>}<y^YW+2MA9
z^V+*EUk{rl<b)l#QaMd!zF%1QgNb!Q$qj5TbT&4-d!6x|>-+b|g|y`fcP_M4eem(v
zDIIh7<=Gt5Pgk1~Z0Dp3w*C`s7F<=Nkk#K3cJ<o@pPAp*`fPZ&<eaMN{n_`l7VZ<g
z>Oa*q@XZzHIoE=!9j*M&xS#HMw0XsX#z`MGcK<XH%)M1?ub-y5<CXpUYeid^W&G~g
za84~<<@E=n><`f;QWLL>XJ*&PZm)S8Ub9tzFM!EXR`x}WY#Z;_%Ezyp|GuvNq;Ay7
z`QNyc``&l+F!q08`a2@(C%Y=YXpgY?&!Z@=b)x;l8HGKqVJ{l=7`{#0+Z>*8bIxJU
zT*D_C44y^X^Ue!22PRBweDLYFM4(1TOSC-4T(x)CL&|**Ts<$-mnPn^_9_!cDgUR4
ztMA!5>i$-J{jB~$!Lf1gE6tRO4T?t^`GmcCc3gPi+WGI#&sER67w_%g=X7hc>!l^S
z(Rop8bAnH$N*bk}n(`p1jhj=?wD{VamBH!a^JN=;Jvs1n#l^)j+jO2DD0w+$sq~_}
zn?^@pwX6=_lyo(#^t9S)@6|;&w@l$yUz#~}lkcf%-O*d~``76Pe}5Oa+x&FNrRY=Z
z@9i`DU!>)H!yx|uk=p-yel;^TCLO6)_g?4obJNo^GYsYLzOvMA*IYR9)}Gzsn`axX
z3}2mh_Se?eH#U5~wZ`}NHs7FJkBO33XN$ZKTvEfWestN7!Urdt`QQ7dT`o3?UgX~%
z7i%T?;QI3Io38E+)eYNvW24m5e#KvPU(fyCeEeSWNxifutIa+in<^c&_S*9I>;Ku;
zuU43N)arjK6RXJLHNpJ0@8@1WuMz0r=w0>q0+XxAY6pu`T3a?apVms^+0kYHWy2&_
zv4ABrT-$h!ww%&h%&_8K^PjUP<~Lisn(?^T=GKqLy(X_TQu<u4C3{<MZ8+|wy-U*F
ztt2KvU3Kb?$jE44^VFY{r$t;l<{I)pvB`4TjMPb-&%)zYYEGM(X7u%0WJvGHkdTmq
z#OE<yu{)RN)_&TVo><3Lb$h`!(OuUUE{%-5;Zt9_aJo;ct5mS$k^rrhab0ygr-}r3
zq%E7j@YPc{=_Tt@vR1BKl%-X-YT7BGH7n0P3HICis6<;bDC(5-`kOy*X>ZuiC7n}{
zoMUuM<Ajy_oehWIwWnxbR6DuQnLX6{qQl=mm%7AbX9$D@tlE2p%aQx6#)NvNwapK-
z_<EZX8#yipcRQ~+@H@c%jfVx7ZL-TGS2c!o!A^<SDk0}94A*^1-1;Nx^@n-^dmY20
z{DCQjj(<zG6nZFhmIQe!E@sJ=UV9)#Q_^nbpJ4z0j)F^teO(%(51rAuy{G83sif%j
zz=B^k>9aDctiD``$t=Bg-hbg#^~=82drib+mxM~F2fcRvdbKe#HX^<^ciGJ3&Qe*)
zl=#`a-)?3~PkS8_x%02)`Ha+9iPQ}@b!TTSePla(#nz`)!c%sqZjY%vrC7e>VW;%*
zHKjLV*<&AHkYukcxvwdCuVPJyQb6zA-{SXc7FMs0DtBh;jO*Q^AlO>Tb5WU1B&y?)
zqVA%Og`BPb)*M#SUU5CRUoGi`#e1!%8#gWJb`px5@cz{Q7dt;+u+QK1#ZBtwL|yl$
z?J1Le-(6!<5Ad7$`hMuQ-rLi(%R4`xeY^R-YkK-N&HaCpJU@mm{~vzWe*cVuWB$v7
zUYRWWvG=K3<;BZCg?^~B{(2c`R``JFG{eEqhFSW?D;M~xnF`O2DNw7PImIVu@l;M7
zMeQ%i36i%Qc3s`#*s2@(kSqSF)3gcaT1`A9C3_~Vp7X@3JtRwPV}r&K1)<{+dkq)m
ztElVTXIU=xZAXlC)I^oAo<yavEgrUYiTyuk9N>vH@sZB?C|Df2q$kB>`o0Ah^s;-E
zgadzE-)~~r#`blwn8Bj<qjm25Ms9DCcn>QnToj#X7Z<49_U43}+oThHy(uahf-`O3
zIBc5!=V)`J*G>PnMVn?!ZJ0c#BvQRtY~372*U1asG-*bsoeG_{X{uE)7yE}NBLA0a
z@cM-ao)+s_c6!lTO&uRY?%->iOqL$pG-FBTEQ9MxTlYLzYVzUqtz-u0?HhbI8(ve1
zzOhcycm2Y(M$>|3-?9*#wSDi~8OtPj65cI4w|C=aUd5gTQ!GB(K9Fdiw?A^uqZ<xf
zVGlOX`%`jGVukXB6N$&?_5SWFH*l8=oRMASsMRU%lrDei@1YXMSLZ8VP2|))v7+*c
zB&+s~kJhh$J}<G&n&%wMwLofZNtKE%zf%7cm3NjaMFSr7PFa?0S5dZddCafOUwgLL
zE#SIZ>y{cH@bb9*(n*>b->YAVeLKF)ST)!0%jV#adrbS&l_o6LSh65oaO;YVSz5tq
zvLaFXu9{Qk{tsO{i|O+16(3eI7M1$1PV!t8^Fm8(R#)5ll_@?ssUp!gy=ok7-UtZK
zUUO8bFtV#QY?6}nJguZRF)1>f;=3QP1y5Nc_Aevjh|_QPlB-E^O)|Piv|83@`{rz2
zS?lrKWSiX0t`=oE<xXY)NkUV7gwz_2F44#oyE9WTe8+s5JpaDvJHcGjcj{-aJLOP$
z<r{BmdcjUT-HFTYE@AD>xObN8;wmSx%{FWIJX_iwbD`VRHL*AQiq9Dn&BbRo)aK@V
zW=%K$SiS8-;lkYcAJ0TDV>=tTv3M?9nJ-6nkln36xmGDRcnfX?%&FE}eyHPhdeNi~
z*`_%6!@DjTltet7+tSr8`hU#@7Uw0`FV)?6EWeCn?v6VSdnbkQFEv%WRP|xT;}<t~
z9Ji@k*|%Z)-VY`3J#&6M^xMyMet}ZyF5kLMlNa+mot1OdkKxVy(CJcYe;(#@1%yrx
z&6PZ0m|S`$@bF~D{UNh=Y3)2~)B1qr@F_>_+z)2&uRLJ?8kBBxe0@vsf&&-WDum7M
zw}r0$xLP}HMTW#KMtg&h#K63#&T{_*Hcs=)+0ex?HzoI$<-w1l!t*!FYiw(heLPdy
zjjQt5*B^^Lrz$yb5eV~e^2@o}xMizkXztMvX_ko!kxz~n{q~sbu}O8xmxMD+tA&j|
zu<SVA8d<cM_iZNk_nq^7o)$a)%$b$A`_?_Z|IFGu-+G0lMNeJ#gh_jC_mt&lHl-i#
z364AwG*z(Apk8>x_fRv#spZ_Cb@P6HkNGt>ZpF>knP={J$1jWYJ)$w^A>aM9G^xuU
zHeK94`~DgIW15#Gi>lI0st$2q<I^cB3-UVoYh`+|fcu=9xEqJgJ^bKc&NJtQN8S<d
zQ*+d^4({pRc12z@vsK$`;@<Bc_AGI=w{lD8c_LH#q?2=L=EV5klmlONSJ-W}3S1u-
zvS#)9TA3GC=l0bZZ}h+UI<-Fj&z3V+j4Vx81%BN${cfF!WoP7D_k+Thx4lYZRXeq&
z=6U9QX8&v3ngoKT2EATl=IZ1>Gr)(dg!xJ6RCDL|q4~>_wy6G}S*v|0FD`t>zduXu
zV_nh=xi>COX!e<^ax{4J;jO7lw&v)oow&Za>>K}Xr7APC=sr`S?RM-pH{=H9OrFWJ
z!F*29sjO$Y(-^1!w0Nv?dK!1^Gp+btOTKI?UR?e6O}hQB3%cKnd!u6CoLjnT|CH?&
zS6#MLZ;t26|11*8Qg`nsyM*p(o!&dQxM%!2BtHA&+hw(#zDH7J7++0Klwm#I*p|U^
z$GA17(e^>(!(&Z9dvh&LK9D)bD&@!aF8H(|-`U(7>_O++UD<_gWuJAhzgrU0D=0hR
z#SO2W3a?(=;Cjcwo7Dd;hHvs58EG4dDNjOv`YZ@HlnngY;vd^7FmLV@RYf7a{|kl8
z9?po3oE8}R_XYoSThTk3j(47(XPDSHr{v*W9s!9*p5iH@e`GE&o|Kf*3Yha&*8lFz
zhoJ&&E3eL*d9_~i!OS`5dDglux*CwZW5Qz7TMK1pah*No&vUKxYrxXIQ7;<j31r?n
zZFz0E>!acYaw{9ztveXC@61}4skB~JXtn93+?g#KrnYF`V5<8v?c-AyAD3C*0&j1t
zoV2C&(x$I1&!#%)GKK5j>(V{;Xxl{TZ>6*Ki|&8pRsPW_{ViACK<V+0*!!g`rDIY=
zCGJf!X>u)3bp7Yyu-t>u?4GpQp?HJ2tp_&FHk^3>vPtm4hw+iBcMf}Aw(^TH+nW48
z@Xmipg}T`Pcj|hBWz#}~W+cb=%&|YlW;4-0&!p2%^SE7N#A(@-Rz~)hNAHxmxI6ep
zET7nPc5Z@hhSU2E54t4hdS6uVOzr1<w%~gN7c18i2R(u7xpTd;WWE21<R;wpUYBz1
z?30@lV}<Qz`Wh|tt2#MjrR@JLb4AOz#Iz2w@I2l7@KV{@yRy@+FW9IUu`nRYGBtW?
zXq2d2bfQ<N(7m_>(Z{@JGg79;?reJ+87IGNLe3PX_|$uPUQy4V9tkvfq!cIhQq$YH
zEoy^<yS0P2$k{6r6PolDj=Za6aj$4O_F!6KpOV*-Bbw8C@+$@SHZJ;Y^X$wXC4Ia5
zpA<v&KDC_9P+R)sksr_hHt!ED2B*##iY+z}oBsW7&zIh`OuzfzcRtH9n_hZWE&317
z-OlTKB3sJ~yQU{D&d*x>JGS-z1^Gg@IR`ghzr%3shR&TE0e4JU6-u}s*oY~Vwh1#m
zQ_k4X+F<#>zHbiuS5ASjlVTxV4-`~R&s|>miS10F-*V4ByIKWTzL^yb$JJ7nxZhLv
zI43f3?ws!)p8T^?W-nC{+Lq$OcXgUb2GdS~sB1HGcN~&^;6DG&l9qcat$r_BzAXt#
zni=j9YPN4~fLCL{vAN<*0W+u0oH0d`kyDCk?!u658B&)bSq?5_PxNAY8rgeKBe#K*
zRUnl!apBAbGjm^DUBsv)FZ6Py$jkhd{~wh;ZQ*};!FQ^9jN8f=ih;?kvtoa7#i@me
zczWk6r0Fg>vQk9e$K;gWJtbYUXq~LvUk@(Q4ojPT?$KEjdEG}or<PpsKI6i7^NbIJ
zQr(<KO&eY2uj9|2CuE?Oysb1^Uu;$8tlRo>uXeOX8)O9;A8OTk`B?X_(5w}mda=p{
zWy%H9f}^fn+LbE)lQHJ+Lh)7e7RQ<_xXR;YvQv3Y(&Qs2wbou!X1t}$m^WvB;T%iZ
zm~tnDriJ_!K?#*|?v)V|RZ7}Uc8{EeLWIuBJneCE<+~}7bMWL_Hy?%<{B9X5mZW%B
zY5Ts)+!1wQ`>GV~n~Zfk8fE{mZa14H+Y+$&<{|5;{}1lkwc4j??v$rX{9a^peptP~
zE!LMQr?o5FSE1G4?CLCsP&2=S0Y*B9#g;B_k=!GzWAZ3=-kuc;dE$<7bsg&xJs$Kl
z;v`FSug;^))DWo)VIMLcM^2r8M>Kx#sq6!9o*bQ_`j_Q-q1?|SOQ)Efj=8pU`9~ee
zCr_t&1#X$pC?e6iB<|Uwu=%Z0qR$i4Ccn*G^m5VXo&MSjR<nP7nf^{`0TW-8-&!A&
zS&1FGTbHI~2gPSs&HtkN*6`S}+lSQdELvOKE1z4p@cfed;V;s{D<sX!R^2aKJVzk?
z&V_iRr}0Iv<cqb^|GTQEm^`<>lw9hjmw1PhvruM1V8hY0v+Jv#TL=DM!1C(UcY}!=
zixxZ*@vGV;({WVhsZHPdk{M@jGEFXF*Yt6?TFEJ>wy;qu^xdsUpL=>fzdD<bwX87E
z$X&2n`byjGvUk2Fv$Xe2*1p5En@LH0i=(E|3qih|%W3-G^70QZyL)S+r)brR!)7`^
zkI6DHPWY}D+SR2vK~7-dPK`hJj!Ah5^qiad_uPx}NAG1Gt@xxB*Pq6^;O@iRCkxL=
ztvtEHZMDowM(bs1Qj3_Y=bdvixKMOUuyTRQH?Fv_MIw^A>=w6YZR!tP(UYzxJ?qd)
zom-32Rw+d>Wo0cWU|U)+M=O(EIP>HV#@%H>i#?+yS+rtm)=tZ~cVpKfUy0T)LF;+t
z|8LY@kUeXe(B)~-!qUpwTH)%FufAwzEp{?ol2-WR;<IxBNB2Elw(MGT{;vD?wGNuO
zxvp?JyW$Y*?vs}+j`==Zw#qW;?CL|wvexNq?bbbcS7>pQ)vhQ;=WTM_7v&BAvK)Rb
zn5=T{rKU=e!6wH~lQ;cqYctj6V4EW3+Ba#2LdAqbY(43}r)X>RuhuA^r2XJ{)3(M*
zJCAwuv}w-s63V}jB`|gGyQ80l-hW|xpIPB9p!~5}PQR%s`#|5^)=#U2{%L>jdM*A>
z<u<G8omP!Ntws@>&C@+~M1piA;&T3YJw2JX`MBTGlcyd&>|DbhyEOXe-P}KsoJ%(^
zuX!rY_VV6?t54Vd?|Q<Tbhhr<*|1yZyb6|uZb+VbPw1ytIZy7E8D}mgO-jqszq0MT
zqyB8c&Du#0oT`!^HQ%vvGq18{Oi~rh<d~f+#BJ!O@ZsYr14)6ESLE{5&vJQOZn<+z
z*L(k3@9ic&>rFEEeYxFsKj6Y!<4Zvo8aCdZ@G<}27D@K>Lb=NUn^`3rVxmQlt+SY8
zW%l(NPr~C@hEM!TpDbf7`!_QqpfTEJvem?;is~1(O-iqtwQke2Qg);4GrmSE0^PUV
zh>4!;6#A!VyHHM`;)<yy?;48^dR4f4_n5Q<z1LNkrCTvUbJ3@HRTp$Cwp@|g=B@Nz
z`|>B<14Ueg0&d>=b5C}?4O{V5_P?=H@wab$uCr1#W*Hd&bTd98^z>iG^^Ptx6Q{>r
zX`j2#t)Ju=F!$=V-dF4AJ&lyvwQxm3Xdu%EA(x!FFXl!{y=-DiVtl`lNqz(G^py!4
zRh6D?F7bc4Ec9j6{ihN<H>X{5vfk1tQqXwJRGMf1rKo+Ev~KBGJzS}Ej)hx2YTmm?
z*VnNuRoUo!@%5E=o8oU9@PAz(IK5Fztx&VYVD_Ve`JXbU`Iy8!DVLnVxMQc1=>9K0
z##znbg)Q3iMHUBrjodiP?v+n=#jIJ<e&?Uf_Hhl5Nd8@z9PnQ)e2HQAcGle+XWL0l
z{^NKyP=5KUU0MkXCRe}QA-nmpY`9FQv);OYM~~M2pWOHC@l(AAfe~k)*)6b^e7NDA
ztCBvSih0pgb#0G_mrt$yT6IWWdP92j>E)9mSTFo6O8)QJ6lk`mZtcfIMh(l|^mnId
zPBGGY-u3a8i*E22*^r=w%f}R^IcooYK6kYnxA@g82f2^8iZtT$cY3WbTzzKoW}hYB
zk1TngZQyLRZ`#7$8|VIud3bo*dXweba{Y9&w||~gE}CPny0Ap&gq9ck)?d==rmhi>
z3(L#ttMam%EU@6@)Z>L;pY*2M{QtF5<ilL8+qT9l>(kh$yegjZ>rIm5BZ1WZSG*<K
zIS1K(6tHPVOkJS5eLmaGNe8b>ZPV;ud-B1~n*A-BtNS&jkMf;1_fY?Dp>XPm%kq-l
z+Yi;7n={R3Jnay#@$q-y*CZcy8((glgZwsGVm3ajbz9pjjQtwxCkjh`|03CZ^wOD%
z!1M9F!Ms0P_#erwJFYKt<e}kFR!jNjvq8eg<o}B=I4Tq^vg!14w&ePExfT*Fy`}6@
z=cekvezd@}x^BVLO<SMxNXGa(p5*u_IGyWLSjmFm_p%)M7t?j?Y$E1PbxPRH<bB4n
z<(`}5k;-)uW!et!Gr#=Qz0b()U3MlqpXuq=4bSq@x7&Wt+jd*|OWN}lyO*vH&2cE0
z^dmIkVW~*lj8Nm?g)6$`0wR`M9s753@sWrBvi9aZ*PlOCr|%mt`-(O5SvNcu{Vu=l
zzrbQc^Wy)B8=d8sT52wx=&$MX{hXB&>+jD~-IU6UzdN`G>K*;AnY4DNaHV8S?q{V9
z*O?xfe_UB(f5SyxbI!GD|BSV9*0XfKeXmlM_W!Y+Gyb~T&X=7076-(qdny0XaNY53
z_p@*Ee^RuRzsGO<F8uh5dvsgB^d-Ml9pxK?TiOEe+}_Zz_|N_yQk!d5E?OnE-nDn}
z<6e0ghXT#uRXq+(bJ(Ui-tbIxYGIdkYw?(nbf`=G^aQUJg30dP^0q~1G&U-TYY9*D
znXzcnQxU!pi-Lzu7kp>VGS7W>WM^>t`6hc-HrA*KJq|%@J+>+y?Gl+C=yOvi*QzVz
z^3ug~CLQ`!@;~e9nppRVH(k=YBQ~X-?vqX4_VUJt*-=|JU0o#>8&-PKBU5?0!gRBB
zIe~M$r*umnNI7*vH`t``!TgSsTQp5)bZdu~d2QWz^Hgtuo+{VwmoXof9XOHNR;9;L
zRT^+@e(do)TWPbm*B2C?Zks#%`@38F^MAjyjrozzx1(QNy>HIW4^K|dx4*AjTk+-9
z>Dt@VPO2_D^X?97-pmcGnwJ)(?96#+lv-Q;?fu>3>HA}!{anx!`rLnSe0}}D|E*3Q
z3XIZHAJmxyopcUP6p86M*d}EXaOldl*$$jj0u~)o{juW3BDJ%3G9If5<|=(WEL3>O
z$M}kaawE5RVvXU787CB$9+S@fe@5`QZ0Vbc$K^|TUh5jQs0eu|7q3}5Z`Y%&qRE^d
zhHH3dE-E?7dt{AHtJ}F4$Fm!>1%C^9n``{q#KV`fDd_C13hl)#=6C<B=@xnxlDFMz
zWAR27qYaCTEzN(n^s*T7Wb<yf6#Qy@(N#2e%0+kCUXe?ls%xiQ^42{oa@kk==_2M2
zcD<e<{+_DIUAmK!p1S$H$ewEVHKpn5)yUY?(gVLDp0@PFNIRXl9(Ow7#r0MxA+sAv
z(n=kQ{WDlk-b`Cq8+I#b*Mp{|Q^HP#Eq!10=;ZCZTBozde@=*Mtz5OHscY4`8>L>c
z>#zOwx~Ft})pwokXQaGW)!dxmy>9o5EY<bLe*M>CHfTDv;i%UUuFXLU!rnYsBXU^d
zcK4Y}ZCc5$=c_KCOMXyN&8snyRjAxvBsXcsmeVHPz2-I_L-d~6y<PG%VD?iPzO*^<
z*JeJS)%Z`+n(vF1mS*fu1Jk|Vt~8&|oP29?rrj^EPv^53HtvhczEWotnYu6}Phcyn
z_dBUW$G`oqS}`?fmw{@|?bih^0?~{YUCtGKGTOXnXP=>(_<y$ddy4zNT>NmTU!;g<
zyO_C=j<o!K!%vHEZCpB6O<$e$l>ScTD-Rp@#eDW@yr%Qi_exUL<>%)gh0OgryST5`
z|4!8Q%dTSf%FgE<?(@{1x*7T3$XNUIj@Xi0re81nx=k;S{pynXKiwq#g>-K0uP5qn
z8!o^7J7?eT_tE+Ly8T~X*;nIo@_wY>=hrPknx9KjLv8&mCWwadJ^TI6I%@moQni`8
z60}=Sq%;UCU2vJyx~5&Fppkn`2lH}Mp{5R%sEb0Jte%gw|7>k(4qm_@%JD;2uu0pM
zM>#U<T$0EVmsU-$#;%Zv8+sCPM`I%z6+_#mOsjr-A<mQ2JF3rU`c4DSgdhdxBV1;N
zlfR~TZIO(uYx=xVL(IKPvDnwoPr~Nnl1vk)pL3qqEjr>9Xqe$(9CBer@|@Qe>AibA
zlR4Kcb~8J7u{U>)pY&sn<Ik>4><W6&bkWPPZL>*kroVAYjRVJ_u+-j((VzeGH!^LJ
zXOZ=oG$ltM$wX-zyKcs$RgEcG76po|2^pLsGn(3N2`>^{yD3^p=;DzbA;RlUauhsR
zsoFcqZO7raLWO%;<~%-T(3POHa=K~O(Md0gc82Y6DsFwr{i$a1E~9nts$Uhyo!xP5
znc+4Mv*aY%cM;BegnZ73ntNaN-^60ol)_p4_Q^f#yORDD=Yr>beG(@b;?#EL$&32g
z5m9Tl728LC>|gNLc!f`esZN6Cl!eEY#8|W?4tDs3ERq-OTx)gXApfKZi<PIS9zNwN
zG4a>an9xMmJpo+$lb3zb(%Q>4aZb{bjV}s3Y}W_|e{y9D<1kDp&1@FD%4YQT$+Mn=
zidx72$FOZ$ylY|JDo@D(v$DxycNU6vPh1*s$|1BQ&a@{sDl)w#eWrB>cbU`f$&nYM
zHqSiyZq1>{*WYj5+MIdLbnOw%!(I=HM3yXUpU`5X!m(T|f+=r#R<Fq#W&UilmKi)b
z8OqYv#a7>lv^rtZIdxX6L5U*&yh*kSx1@t#s$IQMRA?Nd?ED~Lwu$6J56z=rmS(HE
zFFD4!*<-Ty9YfuA%cVW-=@qMFP3DH%I6md`HT9hPu0?qHO%EMKwbfZEUg|c+yezZW
zCO@?-cfKXGq&VqyjkD$6>oy<VylPG<udiHLZu|At<h@^>O!+GHbni!7t=f;a^A~lr
zt-5y0qwfFa`Q@L=|L=eMhJPXZ-}p;8q7F<-YaJwSUiIMla)HNh#v;*m86IYn5_@z$
zY`XR7gPZNGzyjNi8j&?;eqWmUNW$%?wt3D+&+HfLR?ZR8HQgq@@5|L&TYFNM|6A4V
z|M!RK=CF^+b51K|@49Mm#%DWE*#oK2+YGVmn=W?EF}@M8_X?}N;Co(HZm-N`5#hV~
zcCM>5`7C!{$K7_7<=O1A;@J=PxZm(uknwEh%S(T><ZU;zZ&;`0vGq-Iq~q)7li8Lm
z3Dl}Q$)>L{ng0?;lKPXGTswJHgKy*{x~%aMT)(1hG1r~9ESe{-_epKD3%avXw%uaK
zmE?|7Ijq`d8(NP(Nh?q54RnkN5efL;%w{<0WvNc=#UrLGIZI-j-c9|d^hR6swqvBz
z<X4Bjs|HK3nYdlkn{(lIT<-4j+jpOR+pc@m&Ua!0m(Ye&tNxs}|H`vv!B(RU3=i)L
z^qPb$JeG7~Q(tOgZTyV;7e7t)G<x;WhyRb7qpW6KU6G!6)Kaar4w}J!SwSTyy|o1A
ze+)^P6)bz}b)m)Fm421K9`9g@G`>-|DC<g_#O#j3q}%|rmw6k_*QRaNZ!2t_qIfvu
z3Qrr)x)rM&-+hQ|kCa?(A?Y<&%Jb~@Ni$z2P0Kc2DS3TIlyOG>)~d+q|BU%m7P|d@
zs(tyJOxZ!U)D5ZaUlx2{{B&(^?<VJqDK{VR-8`vx@qf-g!yOJc+HP)Jc|?Cn>Z%NL
z=86khJOMk@*7X!v?vQd_dZpJsCO7r;juSV(O`m<kJMZzLGZF7oy)LaOJ((B2%&5@3
zBz~3Fbe6~!TDhX8N0QAfg(vx4Pk$MF%{I=g{<qa#`}IK|mR_lN&Qf!aTR!+HZ=H3o
z{P`!%)%#9rzyI|lI^M&zzb0nF0kJ*jwCcZ!FSv6u{@<<Gg!21cf4Ddknt$F-U@^N8
z$-O6G|3mpe9>0LYJRv98KF_Of;<q`~ZhLxXoCbq_-qAddSe^oHz6%~^=Z+P16|}X8
zIO!!sIdW}VAY#C5`fL4rH|2bt4^x-CE}oEKsFddQ?~th6vu9GyG5>Xv4Rs!SX|S0p
zw0Y|ZT=r;Qv*1KmhtG;<t5!@pwPMPums1}1-dn-^hU09{gtJfHaNK><_o9RIz7OZ)
z6+BB4nB5pPZ#jgYO1gU_k9&FB-E;4_|1X{_r87xD*Qw2k=i-EHuC~d1xfi86xr36X
zXykIPaG2`Cl3(W3Q{-gi(2^~t?DnT^TFC0x9%}`p`9BE+UcI%TXwpj0s6_5%5A?nC
zCs-^NjdAg^S^j^M%GaLUo;BHq3Ge4zdQ$Fj-fKsSp@shJviEDQdHI($moMY1oN@kk
zMxj{lMzOuVck1$)dpz79J)hqY$lYrA=hoSJp{#`y1QkslEqb(m(X;iE-`7U=q+0(s
zIC3~7>A#zY>o4UcA==7Qk{;^#9Xm8p`&i-92ds|*v>uCnIF@qi@eL)WnJL^_0f{jM
zFYMZ%u^l?elF7pBG|lVaYALrHZx&qNpp&1mA!W%aF$wLLb&oq4mTX_N&VSvxwWqi^
zroK~J%>Me+E6Wcp%HK1T=J&non(*>e0{6P}uQz>oeM9*?U*?-d%c9Qky**HneMpT*
z{j`Ia(YqJxHYs(h-}ckE!l$9eyV<0h$u0ZCzdU1`?z}$&=6>8C7hJLvTkkH>uT{HJ
zOE!N-R`!g#lUEYF^roy7o6^bsMakQP;hW9|?>|Y2HWMp69?Y>>ZoF-^+Pz7)yx4ZM
ztS-;<7hBJ%|3CimWuM2o2TyMQVHDVM{(HrFgUxRZ#6R!3ciM2ZyLjP8G4s}19pTzV
z{Iv$3eLpdU7PN40%g@}wfBwU9YnGn+o{tMe*8E{vA$;zA__h8+Z~FhudMdeCQ0jHR
z<ll5vlhsli1CRVzd}P<fn0<w9EE5=*1sKzPEK}FCiA-R8+qx{uyIo;|pzAxq<qC;)
zenQI0GfmtK*_;&jb160}G5=IPlhT}j;qw!lH2y^AsTR&AYc$rjWwD<;@v=#PtFW14
z;s*Qq7wjMNID8Fu;4C@I)g&ICI^jaH>dt`kSDYulSggV3t#(n5m-p=CC-0o@1~^~7
zz;klL!F$Y`pA;}nDpLOtp#J}Z0KZ_Ge;%iejc%q3uQqR?o3Q#-p^e(*DLPuqWsh7E
zWKT<cb?Ju6MUlliJLagrHY)nna9p&#Uq5j1?gKApx8>gS`k2<&Z=`PQy|Ki5Ye=BV
z=OC^}zsi>E|MN2R1CK=K#05d0!(N<S=+^XC;6>w)vu1^#&9XO#ryQEH>Z6qN8vo)+
z(vpjHlomg_z^Xap(8F^Zm#$-y<t+$gpAc8n_4rKE@l}U(T9)Xwc`&Vgs<4;ksmvz(
z3!hh96MpBm_0}dAlU*DqoAOT2I$_eYhTS-5t&&mL!&OQjFNfXqQZ6nH&z>4S(L|;2
zYxTX3b(dBcE32z+o!}U4WXIu|^`~b;+9ap{wpZWzEqiC=cA+WgqL#NpQ;@07Tc-o1
zOnpN9oyUD9Y|7*4pB=PXK}di4vPteKS3U}`3#NC^P;39#<)ZgWQSV5R{--A9A1q?U
z%B38|-xDT!mh0CvtTwySvV!Yd$*M1<HZHnrSL=6m&n<FGTrW~1Rh%q&IoWz!mBQt{
z`?jP!e`zH9)O)*cYUbC(k1i9|EEUgB(fPGH?KS%WALI0ci$njk+vTK&7o;BKcyTc$
zSn_a&nedCh0v&$JT#G)f^%G{gZ{iWX;i<X1hN+KE=BYy&6M~QST+Z&_%rJd2V`+wL
zXNr}&sNB;g5fz4&MFm}V*74uJX1pe+bDdYH<~7kg$DRLqlJbvlRycD&?F_@|cbj`o
zC9LHu56chba4+JVyupEaO25i=&bbCfpF4^khH)MF@_5452~r|<v%bz(`C_+d>xCR4
z=fLkg+M1>ITS{*mY`Ss5S+m7lH-*b*%2YkI5AQy>q`h=yEELEzsp!@eROEMWe$t_{
z+<o?wc-C#N=BNj(@t$zvq)$>x8Pn6Uc_GGMPV%++-rUezvhH=LZ>Zj77lVgKe0ELo
zzG_~3^ZWNVuT$HZtC&+uTvNHt7TlS1z&N)h%`w;{I#uHGl=O7f@Uo`|rLrWIAB77@
zG-h62^lH7*46&@YTmIe%eelOb`oD?&x#aArN?JTu1NSS*EM*D|&XM`Ac<yMCMo{r9
z-33g?A1MXb`xI}wY}fhDaL4!63X3~S7V*wmqoDHmeaCZsksxD{XZEKa?B3N@Fj*qe
z;&{yR-ZgcmVyRP=qQeRujn>&r>RQ+rw$v|t`;PE~H)bu8uysAEs@A~lrr9f+(97(=
zD5StRMPd1tYsHb99G%6(xUwylmi8~68m(!far0cZl#+U@P|Jo^7iRs5-XgIlk8Jii
zEt-7j!;P!4rJU}MLMwO$<JI)H6&-n<;}ZWfbY@)^Gt)KE^5^BpGkcuYC;e}#UaWKT
zn$Q*>L&Fm;yfb-g+}SJd9P#pT)ZhNfc$tVF({<je%AXctNv_i>-ON%t`}e3^p7DSC
z@s!)%vqX*eobpNzb$|L#$mZSmI$am0DG9xz3Uh=V7$+U*J^Qow^3FN;BjV3E{pQoT
zyKBA4&c`O@*V6CnW}Mdl+c7PJJMCVVnYpJ^rZ(4oEvEYdQB9srQk#!vx$|7=Vmi9(
zTg&cki;ixy{JwNq*@~Q>OSP^&YLDBo{6&+9!pGqFWLJr0dH>gm<}t?Z6lYa;HrO^<
z*Zz1nn~>={2VLgtJavZ+?j7JQnpJa?MKO1So%J!L<uALMOC#97*ypMhl{eeVhFdI|
zC7RCsazYpLxe6vV2PQRzUe$)Vq6>OYR;{@hx#r@pHRr3i?fQys%Oj_Ua*KQR|Gu%o
zM|`7?`TGCi;ibI2Zwrc4{#2D2YL|ZwSe570^MS3rPTasfJkGE(F4tsJe7D}TBUf)F
zPO)?{5?Unh?LMDNRrvT4pR}lH9qr5)HU*fd-Pn+=clSx>>@7S@-@8{n|9NZsw(H-w
zZ9Z$@n?3vVjoIN&#YZFO9Q`%ty2YH{>N#gEcb>1@Id%8W>DG*s6c~FtWFPMQeeTPC
zCW-k+-`C&z((p7_Ma&}ep!H#i8;$Be_9%NeXsKB1&sdx%<PfyNg1@wd^J~k+z(?<X
zYn&*IeIwl-5)o9m=5@@C<8Kz}ZFA87bJQSg>q&{3#(lho(=?a9Eo2tASzdeN>!nZc
zJ9e?H4P~F-kyatllX|TCzw4$hUz;AqpzfnHjzr$+kKJSXfxX@=fyqpvcYjr{>Vn?W
zk;l(Utu4OOdsynk<y*&ZTArw#bDS}uSagfdl&@>If9OjtvJhOiZsnYQVeSbR-PEmP
zHaIScvDbEG<xdn--s-?SJ>y8hDx<R(_Xz0!bQf`-Y&lb)GjsVj)6cV9Z2oN&&aCpX
z+3oH0ar4^MqQ^9sJ30$R_xpZQ`;ioF_)t6X?CFaWC4=m?c~0GVY4XoYvn!LuTW1B9
zU6v0nE83Y*_?$m9JEZ3K?87^ET#V$r@^j8<%Q-WDUk$IlmMpvT^v$c+EqCV2?kcw3
zRr|NscJi*j;R{N>&FAoG*k0``+GN-vcJKd;zq@&vG9QNAVwF-Bl4@KRyU_NL;-@Kl
zWpBMGd2DZSBjdowZ0%Xv2EQ!kGcmTdh%6GeQoq{V-D1P>=eXCy1G<M<4qrZhbaF|a
zRML*Lle69w%cS}5=$cTvAtZ4syJX#wtrEpobynqneJH=3Wy!Ci-zvS&u0?J6wR(RP
zlgNP+S9hK8v|H1A=ZWxxC+#&)TIW0s|M%2AZkKu0gadr*wr>b^yM9W()9LH1Qq7go
z$3K+T?^E2uqu8~Nw|Nowjl~;wyYlz?iPnEF7tKE_yZcqA`PDT=?(Y_DlHJIAWU<iR
zz;9mu7dEcGv5x!J8}EHHZf?6NxBVsOl~TzHp7k0wXY01Q7XR<JzIt)y&ceCx=SuJF
zzx|$d|IU*?-{<e$$shlr(stMVAJ-@EiE|Ln_<M4GTj09<Yr8hj-kpA^Ve-b$qFf6c
z|L;D$dyn#u1zTemKF_t?7ps1(wWM81_59Q3-?IZ9EA|UtZOM+;{ukXUzk&CK>XP-r
z2~!prPV>y)zIiv_y-hU&%F~mN)?V#iDsfcv-;sJzmAU`BZEei9?0URcYRzfOCzUbB
zUFSWCwtIT={-4wFPcQ%fb5;ItV%(a_KYz|!t+`)uGIQVhc`@hbgl=6F^PF)Jk9Z>s
zyOd7HhXsdPc*N!2I0-B|+`)2s%EOe23y(IYXqi2^w5(@#n4<eM6-}Y19_Fb|(;oc)
zG>KVUC?;jn%9Bsk+I3SrzNS2$EOKGKW3`ZJl!@l0rA|}DWL?=_En5*%>%?ni@<JkL
z>#M6-p{rw8Z(O3nqjPoH6zTB16BlYC7jfS)l1(_Z^|s@?yF#hYjy?Syo}X}_=r@<%
zpX&w>4|ehMm+@2<J~`PhuALY2vnXkQ*<YtO)&+}QXIX83Gsl2+D~sxC(H(va(T+l_
zoFW<vb)~<*y}3R6{kFNkzvtZB+0LxqC$l5}?wRcKWpdV@UtV5VeR|nE2dg)_G0XB^
z`Lzj)zq^0({ga?#u3w)`cRqIEkGHA%8S|vwz5e4f*FO^HU#YuJ@MmW12-Fmx>X;zO
zCgyRWiA(Oq{{@W<ZV_(k{6-lI8pYCnJZR%D;&{}q)FzQsuQc~WP?yHK6-m-udqXDF
zS$^EY+LSxtl2`DW7aphjteL(()^Oz0oHAkY2E~+#Yh-4sOj=Um&gLKN>3KCId@GY^
zNc4t_UAnPNtZI@%69T)mvr08@b+aAhxI8P5izR$^!8S?tnCe0X@p&a>+tTOO{(G6e
zpyAufbk0_$UDxC4&snC=={cnt=p^G~_;QJ`M!`xJE{}j{ra6zUy<9%8YS+sZbLM4b
ztyp<UEo=4aWv{eWtyyKIy<%%wmiD?`&rW48=VaKx+8a2@Fm=k~!i900PUx*(v+2&t
z_IcGGp1R~ZU-N5uyY*Ju>HllDs>!7!?O^HLEwGbaZc12`T4zg`m*UdfL3@}ky1e(X
zxR?^`x8%-=kT8#*uSJ5H*mFcfBc)=l9`Gm?y%xbY-84Y^HCI%6N;8{R<guxlXFeX2
zE9cpiBFa21&`maQb#RsR60=s8pdC5~3vQm(i9IXmB4~Wp>iV46LOW^pFXxZ{)QKr@
zy5D9}uzlyX+1Y;k)3PrFiC-^1d;Me2SF!uKzB(I^7qNf4e%#Ny>}Ja4_OjZaiATOQ
z=I-xJxWjO+R^cwE*N+8vmEXlG-LIScqx?aI`ii87I_rMqZS6XJ!otJC?p^XEZ#Tau
zPi(#VET6hgDzelJD&<Xm5&Es=>5KTf(Er@p@mm+NNUEzJNplo=e{;p0ymNL%^Sd6+
zTQPUX<(t;?>tFt}?kSIz7G5gsqi}HP%xht<)@<st(#|n-S8$aMa+%N~66i6p<#aO3
zZ<$r=?p4`+eRX`7-P*T{*T4GmvU`;S!}f1ljei&aTh$<Pm`hvdt4((#OUboO7npUE
z1G(+HZck(0&3cgC%OHAU*OUyc6@u(hDzO<aA9#pdVVksq)n&;+&r^OKM#~xAXs`rJ
zghl-jSsbC67;bqgblV|=4IQaEYbG%GXL)d4Dq+!8nJ6LoDu`+OlOv@zfnLJS7Gf5O
z%ZwGAA0>Vg64y?9mfZ7EHmQYO>OcvLiCCkcWQvl&%>Q%LO!bq`UYf#ueA!#IG@%(>
zn#VTS&URH~4D4_cQd+>NyCI1uP=eVy!qCHaOOxo@701OkOq=~wBHeOIPjYd?;Vs@y
zyy1S?yvriCF<x#-W1h-0t8`xR+~Ba@v{IF6*HUiGP3}By`$lrM&zZ1oTh6xIXH7nT
zt8V6k-HsVEBm~u(`zjZv1a&y3OD=1lcT@48Yq0ZWk-$d2OA~XpHS*l9x!!rNbkpn(
z;he*>F8<GBSyKBoW3GMDbF0gpOUi^a=XtGKKG`LFP0gf4o+a86{(kJs)^Bm*ik3JU
zu_vYb`qoEc@ta&V^OBC`^%=Bq@NzlrJ~HjLESH;ft0(&>mLMnARsR=n{Ve;ICE40+
z<&>zYvl!T?wXd9!m9^~S?J2%z*|q)8x^eGwPziUt=q{hRNImL;ldj3e#kw*^T!E6V
zwnZu3o2Hzvy;ObmPaNy2h(s^WGtrWDJ5H{fc!P0qS;0}=BO(mH-e#@co28vI>4K@2
z=gSBtodvP8S>-biy3b*oc#-A!*Xta9Pof)b#ho~;1$tbxHu5;WX$%uP_4R>D{JPhk
zl2%F!zRpPF*1qkTa`+ghc$Zk);?Nrh{kjb<JvyOnT_T~qWJ71zmcwFiUv%k4TojJJ
zq3rIKDa?7|0$25gGM!k(b>U&r-OigHxy3z6IuUTnRjZ3ftya-l=&Z(sDJN4>SN$(s
zd`WNSmJfC2nJ0hpR;Dg)t(x$#f3by@_N#+jWmgU@Sghb`z2(BaZ+T7qKen=3A7c<*
zJ@ZnAuk+K(7C%j{dn~97=6LF7VQ@@fU5oBIjn%E%JrB2L+z`Fh-FG1%(`#02Zu$mK
zhYe<iZ0mA$rzK_=#uait&g@J-_F^ah9j9oY?N@#Ctj`tpeSM~|>uc`JQzgqx7~7w4
zY2I{J*tGI<QrE1hxd$WvJBy_(EqJQ!FBxgO^I~7om#3vkp2gD^Kc8gb?r*Sr^O>)I
zW$YZycO5US)MD_Rdzj-^*SvQp#Qg6vxgNa2bzQbaBreM}XZI7~updenVrJ&r?g<Hc
z6w4EM)OzKAcbOojr+Z5({r748S(UZ#!&8yv^R8vSTGY_;<=A%lRmrVlCI{2QL^jDL
z2gn9R241*wHA(nW@%-i9OILCxL^iD3{l@cJ5_h=D?lmX(Icw|udBf#iz2-ujnda43
z*VeRo)qh<bm)pR4V24z81M`vW*oCDBGC5ls>qK)B@AAIipH!gGoqA%AYKDpb*^9jf
zlbbgD|7%pw9&_Vw+0Fg4PaLY>Yw*zEOxNM)3IlD%8yco8N=J)t7_?_Pa39?Id2#cj
z4YIo$Mf7)iO1He8thaKyu&n>j-jvr%pLuQP4!fe1Y9c&KrLRqSdx+%eUD=+`4cIN$
za^Bz9n^tmDr1qsd|E$XH|H6#xTojoWZ#b%U&wJ{0!K(c?xu;oeZ!m7!wq{a*_lE3$
zcM}4q<s3?l^}cP)^L_j2xrg%af4Fn^0INmQjl-+foIA7Nu<`#~r)<r-)XG=JX3>`>
zP21Vi|F}3wKK`7AJu{d8oe7>PO46H8%rG}<wK<}Y5mfxeOKtB9Z+{=w-d(jz;<8ty
zpOR{_d-wF!or%-$a-LUvX?G@T+H%|1+^RdvdgmM0$OPE1$LKHQyRd2d+nnP@3Uz|B
z4(FbF8}Y{`x_cXwhWOV>-AcME++_3rXwSV|HD!C_Ge^s+T}kn;Uvg*%?yvv)xqkm2
zSKqef|E^Rg)~PriWH)}O-0=Q+iqM15tB3ykrhdwiC}ls@mU!AMSM(X1m|3HcN3LiF
zyTpn{krRy)*VxuK=H{q2-DG31`(l4MNGCBluY9XcqO;je(R{-ndBz&erZ*xu7MiRL
zG;vinj&M$LNN!pc8l`nbgg-rsqd7`U++d?=^xCHe(FY3S8a>`PM4KJTP=6j!8YrHc
zz|H5X7V9C9Gf`1HQNe^gwmLAjK~QRXX{73w*g0Q|b4}u$4>uP_#Id-w-!E-{bhN#A
zM|0f_>3@eggWcRWKKGpQ#botEqY0{B>n7_7IVYC2CyJ|eN~d*B<S3siQ9h%hv+O#T
zutG_&a+m$*=KAf7&Fx*9&rI9Rb@e2Y4VHO#wEs_@)}Cy2&5X0Cds{{2ZjF?~M-&T^
z6(tfC4|h}@si?Z((sN}>k9Szl8H=7T(}VAY*>4nN*tjuQbs9@FgXQfT)iaANQ=au&
zD)qf-u{`lT&A6rSuZXC}q#DMf9tjIsqbH=l-Jp^<F(CCqKzx%x%7fbUhe~hRnnfJ*
zc@L_oIB1?aY%_P6efUz%7zNe^Y`tH_Ctl!~D3odU=6T(h^oAsJxnGQ7+8Y~As!f`#
zk$W|4lFG}RtJ@mIe&k5J;L`j#NoQuG__YS(o0F2CvL%<>-=3zuG)?>J3;SpTJ9P<%
zlA}$`Ee`r=0z5|?OkPYe<CyCI(!uL{36qDZ?T_-vm9i>o|NC7|IB8!I(MxIZ+~_1Y
zJ<7y|ZAYlc#;2lIPYl*>V7;jvb1IELFF_%FVPbivtI0$+wnn$2q}V#m8S_Nj4G!nq
z&M=NRY<n{>;Cire)CIoRr7}M4B~xaWc&d9=ddQrfXfpH3%y}%cic+*^Z!%fYD8;|6
zP3d5#N@e-%7iA^vv)60RUgtS`ZRYHap0iUJOuYh3cPx}$$ueiZX2Ojb44I}C>S-0L
zXH@82lS{o^+}~cYo27Ekl!$5KKDNiY8Jp%hUQ@8X=5we*;oypt`#%+!73VEspZ9WQ
zionL6SDA`WZgL89)_*yidv%4jDx36Gj_}RL=i7bq%RZ+3WlHtF@U-_={-+fz^FO$}
z@9WaqQ{n;90q$!9xP+uKt}_KTxiZ~(7_@R>-Fe}r1!@fki?qIr8wWXLFq+#XgmVX(
zE?S~izwFbZui}fXTpK>KXZbzLXa5<Rs=z3-pdoo{4&RBKHJX!i3mV0*HELBZG0$u?
zvTXFRn(QmJG<esfkf_O`YHF`0=PI9Ax?e~~eMXbp&L-^_UDaDH%w9wUu3Q$>xh&Pd
zWO2t-JB{Vm%8ai}TFPHdT@qU8F~cC`r%3LFmV8UG6b057hBihY#Tp`u-d~ivBU;e*
zGtoVG=1mKa;OSX$hNXcYvZ4%1Jyhere#rWGmFvxdkhKD=A)O_aH`;4wc&=Wh$?5iA
zYkgI)W!0=@f{8-RogS~atPN+Ew$I*ot8?|uSqHh+%>1!>$I8x=T(k3++g3d{UMS%i
zZk$xCCEME76_s4ENyY5aqVV-MDoo7g<~Vn6S<$^CVx6^{InShZyEx_@xncHhRm$5}
zJr6zCf6D6l{A-=K;{2$n`Bz@7-<mjI@3Md5v-vE$({`6{h=0~Q&Db)pZNcKtmPKU-
zEzkOnur9n)R5zO~b>pereQBYqk83^L%(^dtKe>S=<T<~pG;3LzhnuO#k_Y@Atnq6S
z_}NSYerHV-Ia#%F*TjV;I(NQIT>F6Q)UgI9t4^L1k#1JG<~z5jdFHBDE^*JE?4`9-
zZ{`wx&ZQC7|Cg%%Tp}4US+8;_?~I(72OIYU&EIix<A=}Mc3+l0bW?Wo%qwf1lDs+3
z`(}7{*Y=!M+l?Y-7PFWf`5CrUWBIKU(TOjY$Io2O)xLwL-Jr~*P&iuj-zLtS4>gkN
z(XW@9E_CT&{J!S0)(*jf(x4B`F2*Z+GWky)U3t<pKAB-vYY4wz>ZX^W+50}kFWw-!
zO<~uuMw5jvO}vXc@<YnFz8Ud|rDk0h%2ntTOXJ>EHG6Lsw~7K|Zo!(pR(rQb?me5m
z_sZ(MH=}3oiuBs&DSOaT)`+>Hr*f@ETbItVeHRx+PAWCKu+sbc#E88qb8krc{1S{}
zUNqNkTX*?PpGP-2V-<R06aLR*iBV*eIlvlo;GO1y$CU?|ZF+d;9N@1x@XWK`c-PVe
zP4jOyrvH8Ex1HUe?Nv1s*Mhz23;dPxeobkyJk-a_CAN0kqR*U*B`xPKI%X?4fgvbi
zQQwqa6Qx~7Zx+pIvfXYP|M6?w*A<+B3pm}US}{E1-=-bEsyS)N2M^0?tpj4KXEdcA
z{uZiQsOFFr#;36@;CGJK>n%RkTVtcygb(xwcrQ8g`&fR=(&9f`J*|$H^&Ah4KCbe!
zQS!`ENgf7E8P(kARIL+D6`{*gH*fErv%U9E_^h9aP1+|f9Sm8+Ad=9ze8!#vSB}CO
zuAS}CB8^%rUY8nM?OD0ay438md;OA?GrRuBGc04hS>f^a0r#Q_aos5yN^Hwza`@M5
z;M{eT%iHIu%@k!pFQE{hfMBMTOOndCwrMh5kBt5heJsr7<nGR+yY|kqSaZUPRpr3m
zQ(0%%zg&H^Ywy#Vy;r5zNPX+PU}gG-XJ7s6=3|{}pG$kUKkqv8a_xs``G*J3Z>>mv
z!*$_*&D{Sr$-HMc*>n%)C13a|wcaYNC$?ZcyYGcO<B69t=3nO6a7{v*V*&#QL-gZ}
z4Qf1ntiO9zV>e84?z1oI6S!?!6sB^m-D*{H#)hd@r_uvdy#r?L;+(gcb$eMz(-f<{
zYeLo?+qL?^rtU8u*MuiNRAvoSWqrH!@V413A3NL`ZrJKO|6jB$s6o0R^$=6oaj{8>
zGD`$1j;YO@6zhE~_3g2%+ZwDHkNbO1mJr*<rx6x$ODJ4wYijHDBHhN~y<6*KCs&=l
z-sF2D@55GuO!nXnlQXin_3&=6&79xksoe5sTcPzbx0zG8Q?@g{JTdp|iTSnyE45@A
zR!=?Rxwa{1NAk)Y3)bA4trL~Txzp=~et_VLwW2fc8;kC>DK*d*Tfh6r*4Z)dpT@O6
z5&g;P@p&n0xl4wQR>HxJUP{)CK4-NSaoTbnlRDz9X=>;qRJLZyiM`ME?0xX}uH;t6
z=Q^|3E>N&qF19X!EjoZn!c6m4&b`O7_deIo-t}|YKF)o$>iZhXWnUHUYg_)`wBxw!
z<}3T&oL=!uXRg^c`5fo{pKqHz%bfe)@ZqoS%CFY@J5vu#YCj-!=OLHRL1CE#%zF-q
z$~@vYBX6?mk^HaY-&Y2={I<v{tNu6B&xCJ*>A(52lozmTS%o~3Ef*2hi7AX<SYr?)
z<{-dm!y4lGA#QO($eRZtYZ_SmLXQO4#K#6^yYj@#s9#PIsCWGHG)gz;;+><KieV|L
zOHx9QT}eB}ZLu|G_mZ-NW3k@PD`TIpsXT6U@}7{wIhD@xKqcYqgNuy<ndTg2EpuQK
zE_fk&j8*zTf2H5jN|~3nv0KBv8>?krW-xVi+csrdH*IFjbMc&#RCQv8t!~ZHo3|z{
zTX6ROt9iD9uI^pSbzfUE?|Pxq()s$<&VRS&)l73#tPQ!S>UY9<=Wf<-4yQtwa|$UO
z{akuy;!Wi)6|p2m){}dcQx>r5DK9$9yL`=~{5eO3kE$i^lsbOfX7+=eHD7BxpXjXl
z_V&5xwqvOmX5}7;R5xafJ|NWfj%`~*xp0Hnl%~Yg3unL7zGGhh?&h9Z&)%^`FStKh
zWo@;&=PsY5lQ!qqP1&d2cK+v|efMvj|NF=LU(CA4r}i_i?UstWkkfoYt^X4*?*Xpb
z3&L~Of2g_;qi}$UXVbn(7tQ)FTFT#)**jlhZ|^_NOZM|GsqNjsahmNVhwA*rU(B0o
zBAuS-_gF1aVC8sV6e#-N_@_CW&pocKZd`YjJ!}uH3<|ia-^{{LvB_%bm9^~ldY{ud
zre4c@`z&AndBMFcMe*M~q#E;OZ)nW?zDBY!Pk}L2VXOX0)~>k8Vv2@(rAqow*>Vs3
z(2rzWShBTU_j+XY%Zhi`+iY*lioY?F|9ay$yWK_G`gMOvz0qv6xtVu+Vw=x0-3kZu
zgHvX|eYMu_w^_%kLa$rVmak2Nmv7s*W98nH36|3iuNR42)V0yKkkkG4isaj}Vs?#T
zYg=X=E|hF&{c&fbU;FAO=ca_*d!hH?5%0efr{3M;zqfV+V~oSz3#VImMQLUoe<2*e
zR-V9}`{4aO*?&*$|2^(~Vd!k?#4f-o@?XVMVZwrgP0Ye_ArTW48gyrG^O>@vv*(Cz
zj9KBy9j%8&^b{S79!{9J#FI}kwCLfIO{piQrzGvuxjZS@ccw-Dq*Eu0)g|>(E7(*+
zgEJ%-d1SZ6NKH{&uIFg&y4JIFRmketyfUMztv%rz!lt%8T{Km5Ig8LF50Benj*g9m
z_s>n`{;qqci9^UE({+`;c2KSdqnb>`hWm$`xS7+6cNjiC+$!&`b~H_7b)fjkoQpXZ
zlb)SV(F;Gvvny0PdTrLqw7skjj#~vUt($u`Z0%c>{QUFRe5=2|-?r^8o4@waUTy2O
zZxq7csr~%){M>B!<#w@_#n$I0CSRX7_gC4A2S?l0`Ty_pnaP`btUR56uWdzt?y37q
z#)+ptS+Bh_W#i#gCUNt1yAMw_Q+u$0$9%_)2JTPyKJ!_!E0^RuacI^U>}E7vQ{WsO
zQ|Qcf_tm0o7eQ~8kLy&-f+rexTixp6o{-d1=q8!^CS&P@h9oTwxv8%{y304S6nQ9i
zt9<fMp6pWOsX95tp|f$3O2#P`zEz*Rv^F0RI;|Ta*uyjL5KFO-;c1o6KE{`Gc+Qw=
z3vV{L_^8GBtp3rIrPHeXE`K)BTos~W?(ly~(FF%?)h_|gysbuSgQi)(3*=XBy%OlD
z-TLK{hvrg|%a6^@e!1*-T=+b%k=?9@o)1o;p;tnuD<^XK+1^QN{_hs6nasr(qv_Zr
z<l?$Yh3`a$=#{r^y`sVXr!SX;1O}I?Zsp~fbm_d4_$IT{_S<XLF6vN{I=s2V<Wa`4
z33)c0?j;wucB+=;PI?*ZSF|hh-n^c!@5?LSUNyg8d43U>R>R^;UaQx0J<HN=-n~le
z!O8_$I}%!+i~d;JxKlyzNDGt43B6v{R}qc<U%B$S_jxVLF+9TXTRU}Hc-hYlQw^1K
zOin-g;$wUvN6gnC=+2QhTdssf>zZC~;QsjH;SHy#mrEWe{d(Em9PD8GVqG<#^_un9
zZe^`@J!|m#?I-Q`ub-T{v51#3I<wiHf2YIkUb9;h8hFA_yp!6$;qbFtd|v`Y^AG$t
z;(7DpK=dgNvHT+&&s{jU>$BgV>=Ux@*L^%LyZ(3Hsrj|jxK~V{rrog9;rkDl<@~cU
z7K!RR{XHg?Sz@a^A>sT*71>IbRi0|a$7{>)RzLsyeZis@p@3}JCU5V@D^2XUjG{8v
z_`W#YGtc+s>g950U*5GbHlFA7*zTOsZJFS$Wm%c&=fk!w_~!ZWxlzb12f?oR3wH!2
zg)Mk^)4y9JeY)wk8(x*K0)J()atg3G{O?^Z`rOh_dRxN$I7=@_j#&zIQHk?C1+^PF
zOAd>8G&p!pUBr^IKx(zakNvz^0j(N;LRL*l?tXHEscpr?MBSV~?u9GYil)4{{AY^8
z+DEY`>v$Cx{hzv|U{_a(T8!*X1`msm6DLej=~DB&?iF`PZ^n}oYQB+Ndpj;ozWtL$
zJ-(zXsO`|v<$q4cN8I!Z6F7EkeG02~!56NhZz{wd8T6J^PwAfeU=iyPjbrI;jx!a)
z%_L7+xJ}fU-IMAuar5krD~yYne5Chn=UlK+tv%&LvU=;|odUI+G8g8|D#%&fU{v59
zd8x9m^o_#&*D5Po`X2SvIVj0L*`nwms}w4If<a`}+$YM<Z(ePjcGx>+>ch}gMgiJ+
ziVJyTIczT7JW#u6=UhJC7h*R=LfgZ3T6~|t>(tGhZgN-Q9JeOB=^Y<m>%h<F!YVW;
z)a?}3ys}-|u1#;g8dC#{#sAOeuN*yBx>otc>s1^!Ex-I0a;Cm;YIIqltD5f}wdlpv
zB?s8TQv7Q&l`gwo{1Vt#w8bR3A=vZemFdQJHT|F8zN~g_x`xY9k!8kHGpC3wU|@*U
zYOv~^D(m%eK9A7`(W^NMo*qhTe&sCWNm0ns6x*^*ZBeeH<5AIdM?Oy!Qu1#7JNYWx
zjH3s|nkKAH7R}y#jQMc-NdZ%{5MI8E$GdvM)D{<{vTNUHRJZxMNN(?ocKu(8-GMF%
zUIMCdpQM_W9uC?PE&6lXE2B+<H$x|Og)Yh2y+K2G+5zr3tA{y(Kep;uYCheZIwc~K
ziOrx^HYf4-rt3#IzHMljdUOlZ<hKVdY})d_|7lsW<n7!TE~Oi~TX@nAhe{T<M#nt)
zm1<%neWoar?WE^3pIyCgZfY21a)+f%nmkuJwS4FA*yXL$r!3EzXa1edj5~}sug^$8
zJTJV9MRrBA%!PjUJ7(HK^H$}ubH#Y&wJR`BJv95aW;2T)tL3CAjB{slpKsb9_P}rM
z_JdWy)1E4YTk@E%bH4p%g7sZZ2?6D0DdkSSwl#@KBEJ{Q|69@C6}#t=*zIOf&k4P9
zzAE=F^0t>$d(KtyH(%Gp7OARSDcE0DV*TA}%R=w4qRE0>F26YgPlc_@n5NR;`8w18
zwA9(6Zw{{l-^K>{CI6E9ZZqrdr;S@gpDVK_y)Iwz+x0(ZU6Edr&qwnW*4!!FIy?c)
zrx(m)n|4b1P}WiRrmtE;B2`=~kGu-7t(f%L=c?%Xt}kL<AD%NS&B_(@>y*w}xJvxf
z&lZk9FPFK`bqdUWb=kxwjW>VBVcSHJb(`8=&wb~`zh~PH`I^v;@u6p4$F8@yv0uz=
z{p-Tn2aQ~A9@SRb%r4DRulFtb{-qNxCYM~ME^xUW7&6^^&s^8<c29L4ADbTV>Z|bb
zJMT0X$!-t67BSWGnBl#YiRF7wCQZ9}Oj;{w=SMsDFNS;esJVV#v!pOS?86LRqlps|
zEixte?utFkIWzl(<av+Do(ogMbJ-13uN>g^df&GFv8;c8>oqZv{(?5{|Le2v)p)K@
z2s!I@aOr~sEJ9ltjqW^b<d}6R#L)J>*`3CfD}+oqr$~MI{G=o&F8aKd?v=R_UJ6r1
zmsOeXlyrVp@?-8hpFZs?X6GxeH67XJ@Ub(prYv#s-J`qgs=IH#E8Dt!@6jlMH5$2Z
z-q+36d&DreW5QoM<>IUB%6!!JOy-TRP!FwK)E)k3ib{fnL3N~WBwOZFXREZymrnZJ
zTzfq0+8XsBj%~|6FV1}Fe&0^Rkuz-mwkyncpUE?5?&7si6=+=3b=CiTR%7=zS5C29
z?HEbZsVXN{>NBo2`!TB^^5n0q&pWwZv4+O}Ix+JhZ}YRT^Vg$(Z(a8~*7|>K!v2Uq
zhu-~X|9LjP?#uG{zpvx>$Df>#qaft}?~C)5kL$Ly=6g!VNJw7y3J8j1IHGjAXu9sM
z>1Q8Jzwk=DYNvQ@;dY}Ut|rU199!0M&e84M=}@^^g7?n$TEXtyUfdTMXI^-y+pp+2
zLD7-(LHFaRgU!(g4|PaBpCu`&!aH}T<jYBukG<eN9UvK2pdL}YX9Dxie(pnlT&k=$
z8jK_ixL56Do7Ld5nR#Y$bK0ZXS+DxDvYIE~=%4ancEUqdqli@uD|d09mf0O?Wvp_z
zkcppj^D556vw1JiiN3L`yr}t&5zl1K-5b3PcU14*`r29Iv-8%+hwWvKIIy^EQ*PCG
zp|@*7gTN8PcTp}3|1B1sm0;ed%-r~E_qEP72ZK46SZq4{+vN!J(N-=ct<CnGTCPWL
zZ92JIE;Qw6U)0gC8sWJ!g+Ejtof2jI`J%8{aQBbJ#y`4^+fK|k|E+i_dcI}zq}h|2
zRx*qJf589c#%9miO>u_1rf6_}HJF_xBF2BQKR9|}o{wq1%htRn44N6uuMY1!m)!gI
zwp~;Ld(;8b54W{HE1OlxY)jC#PdI%%>GtueE63}exK}?htzXgV(z5^1BKdUh6Hi`>
z{kh$8Y}4}NQq$F)Iwww0TKvdNW~Wl`l|^N*7a1CDFA`QREIx2m%F~3av#Zg3>JtaP
zni<u%x9hD?w%laCT417^jN|M#Yc>CWUOv@u$q@rR$0&<a9H(3t@wn{b=?I=U`{TTM
znXQ2@BrhJY@O>nF<Aa6!7Oxe8OFw>E^1(`!d!y*e&jK+M`-2=sw{mjj=BTc@thRZU
z-SX^R+vY4=uC1QANFaIze~<_7!5xR+?3!$y$-c?hdWW~Q)1AY1GH0B0e4Jx^j;fqF
zqI0B5g8BZ1jdiQfxQcXKpU~vVvO-K!<GjeG1CNiWaJya-IeO`j?6KLKuH@Lr-Ch{c
zV-vw*eEP7Qv7u(vAGyO0!Z9JnwvJ}j)_b!wr-)2pNMiB(7Ra#6aaxeUf~N@!=5kG4
zby@4>AGcRJ{9>yNUg)egyKp9~;&{wy?P?u+(dhjj|EJi0R`LI)?f=zf|0kRCUrPMH
zJ>D8vq1iHLf2#JHt9y+8?&j#*U{?HcKLg8w(wUuSj2zf(0xCTN^sNqD)6(s4@x0O$
zz_rEUcGTLog$_5Wbgw<)TDhWgwvMBK?LmR7x@UAc9vIDhlI1Ac+jB7@a7y7R$s;{d
zbCz8A(IdlSydz7!_wydRjb4g-PKkuf-+rS>QzJ-G_0aN%OrNa|8=aBjJUoZ>_N-Nb
zVrMd)**42)%Q~;UIQzjbcE^@8$NyZa67V-nJ!4>c*(mpnajaqW3RX>tL$ygRLXESu
zZuVS_6wKaz<h;p>i7)p1ZIN}b^}Q?;;;73U?4$EL*;o5^%k!5;?*CI=^_s7^=N>(?
z`l!+3dA(7_(E-NOZZ)M8&u?W2P52_LI-x;x2FKkSOt%9<e||jr?cn?v!Q(tk?on&c
z73{eh<*Udp#a#bDyT)dJHBVUWpU@=bYYi+O^)}b)W5QB%ucg=;>KdF#lU|g%IPCXj
zi9AoGf6Y2&H@6k<^w9og#;6i7sfNqomy*8K$)dL&lXI?@d3#QqqRhL*e1in{Mvkta
z4QGUIO6+=}yH$c&FnQ))ksA+RaWBf*amXT~HG0XMi7L`Z_AGnSc}b#Y!AvjnwWlWT
zKDBlWuj+%Q*+EQlQXF~=g6qyi)c^AORB=gjuhqvB%UDlos<tfSPL!E)G-&qK2{Zq1
zJuQ2Q$zh7MNv`!r=QAd;!HY_LOl70CdGGdpA-wov_@3R}3_F75s|9_&ocSNZxxitQ
zlWNF1+p|~yM7i!YI+$_AZEDCZliQ~k+iX3$$8fQ5*bTWD8{T_gLVlU-z4Uqc?TQ6=
zF5I5wwfFHIhP*jf1BLIrcycWIXa5>4?Lr^LO;L77RmD6uvIbpXiWT<%vFFa|SpPG3
zW4=9!IscWtKB2j3js5R8uFWwvnkVerV`7g>O_!dzhJiP{K>7evD(5+;0OqS}m~t-k
z>m1;4@#KhA=9&_~u_S=ASE8eF#*8Z${C>#H5ea;u8u&t&<1Eh&5mSpdy%*ouN<LmS
zbIBe@V`rX=|0`4sPAWzO%geh4ePp`n`mjUu!Wmadr=@`nB4^I&T%Duw*I4Kge|(Yt
zZzWN_8Pcr5ho}19+_*AGW2*t{xeaSGn>ALj&bN&+QN28WY67=f)RMn9!#cut7ti8X
zox3}`o8`sw%Qw%6GVZZG>r(VF_~Mr%Yil3+FX=k$9dgxVUhtBKnw;x4mQMDt<=OH!
zQDNU6@2NL#@=P{k@VRHx`+aqE;Mb!8t@E{mrBpK(=`slZp40uu@!0HH4N-<-OjT2Y
z7Dyd_*dG*W*}p+m#b*Mi!-QW3TaO+Mjb=UmJlI(`fi+$@?2C8UY1VDF=aTKyuBGs%
zq{yZ?s6E;DOWSva(AF7Z+5dB&WS*U-5}}iSlOykB$Fv2Dy8hVrOu1Y5^hAO3_Ith;
zN`9vVg@p&pdDeKRhU9r(Keheo(#12EgfHLPeg0wi9*sV0o*OZGdd*McT(rCv`b4y?
zo#`>jqHA`<k%}e9Ud;4uHN8@>QmH5HgVD_(t()C5XQj`avng_?PKj2w)&`>pg+xP{
z*^4bBHcmBldmhqtb9>H&vT3KkI;iLNolbV*)l_J;7rC{A`Icd;b%o6N(;iVgW|ukm
zZq2`%ezatFU_|%Hnp=@}j%S~o5uP@8m8ixIUnB8%QN3-6ZA{Vq>o)0eKH9{)Qf`~8
z?rh)NYmR=Xylr&T_e*BG>bB^|Q!;0|xpDq~@yLItD4z!FX2v@|o%((`h#r2}By_{l
z{DSR1My=&VvyX4Rzo95*`;zp{MXx$OXvc@{kN<5~lhPg<(H<(%8k)eqFNrm90c)Z(
zb6`Wl`Pl4))YoU;YKKKUaWRYy*qiio_auf3=X0L=d$C<8dbxi}M{MC;59YnGcdy>P
zb#~FAZ?WaC#V;puhPJ)2UZopucdyb{VqJk}^TjTq#O`}tal%Vd=WU4-Q^^&-n)*0N
z?_|Nc$R&Xn6PhnL^vvR%yin0_<LmCe-xuZICfr+>7VNb2v?BkE+iFG=8W?_du+5t7
zG`nFd<8#mCyp2A7i@8KM1}<P;wySv2hW#52G+ZC7+SuCpUn98cKu@SZ>$-RCRS~T|
z39K>EtcDV;brI}_5v`kx^UGE|Xx#UJ=UV;+)e}<Pvs`%&@|WFv=+Us^ueukHhJtsZ
z$h%-AW6mWGMr%cna`L-M9+Q<kSK!pU;^^+gYgc@x@ja5C7b3Vk#K^i(sqN@>H(o#6
zUb7z|mx|>jmtWg8ZKb;06xYQ0x1y(pzZ5ln#((3m(*iH8M~8R*t#D%E-1_{@G0VEg
zF?&O;H?W#bVA{%et!_bkpag5wi#?$d%vBxvx;x(2tzZq5Xs%ko>KoCT;9MMLkm9KK
z#L4X1YPqoVRzv@(*)1hcvU1HHUGzD&sw1o0!+V>D2GjMDn<s9nUZ4EGB<GCMQ@8vt
z!D->1@z;;9`@+i-*SNE5%9`tX3r_mw$F<M9C+poH?D{sQZf3A~>7}q6CoPsd>k@9C
zx?@sTL@M9pw6xyKD_DaTr2S+K%Dxxj@8oy(qEykpsT)cbOmg7b@^JRnNc}&x-;Px7
znsxNuign+n<<7}{WN>9hOqjt7<LfM81-rHN?alJv>+CPuBIaLJk>C6*|F}qolTJoO
zetA{?#sgU|+RJX;QFti#_<YyB;8TxY^oFhAet)aqu0rc(i>zFxUh<EVw>55<ynLG&
zedmqFG&N&02A`dGC%;Wenl&%;@t51Pn~pt=_RIUn@BZ-Ei=O1@t!2xn*@bUbl<{&G
z|Nq`QD!El-LH?%{cEbqvFUwj(U%ZNoC=WdGX~Vaxfdb4`0ft85#oOzuR+OKQ``jKT
zarc3W&YJzdbS9=aynlWDtl7bL+g#S)Wj&(9{5Iui+ZU!O;<vYQhW2sk9NNBP{u|yo
z0X#=@_MCsSe#O&WAKnP^=H8Il5v%w1wQi00oqNtrlb7b?b{7cC%J3RYJazlt*Dr5N
z?}pu9%W~({zFF%Fn|J<{o?05DCHq%9_0sIC2H_7jq)$9t#<bh&%vT}dGlmyfoY%CP
z{4Xlk`K5pTufr17&vC2^-e>)}moK(Bp<Msp<NFUz*0?avzkIH2)|oAry+R(Y66GxU
zd6|R%%AJ)T<R8{G{=Z(ZgrWZOk;G5mKXg{@sbCU5Fd?K}zVLhaQR&j7BHKem`fH36
z3%@M%YBM|L9%j}MFu}o5sF8(N%%|hS0;g6l7Ox76qrKW)Wm>y-Wn2uMX(VNw^hd);
z<z%m#<2;$nO;dFzY>V4e6L~spZP?nZO^Hj5m;24#a^vEn@YVM&)Pasgy?#DCXtAez
z=E~(FD=)@<4eLw_T_3;Ka8btB=%p*KU0mjty*7HQmd2%(X|uPLMv1NqUX}X%(O&KF
z^>_D`{{Qtk-8;CYjZ;WYZAL=Eq@!Ju>Ulhpg$$arx5b?EdG<+6Z}yIoXNOX|GWZty
zP8VA{i(CEbTG#z)y0gx_Hd<?W^S{{K*{Abvtt-9G$5&$^et%DVx!^g~of*$gTb5SJ
z+5Y_EcX{FGZS!oaB;VfM8GZb2rb1=&)05VA`ErR&nz<fZoAmAN{{H;-`g*$SE8Bm6
zmwaJ<5#R84<41YToSscL)0X5ce4xgyX2GTT>1FtWx~*H1Py3p04C2;|Qc-Xe5^9N1
z>J*shzNkxc)`>@5`r9~^dxSSUX_H+WQ?gjs?9q!;vg@`?QTA|pcH?os%Pk8P4|gq3
zl?m(ad|J{LeCrBV{lAOX6WevJKJE^Q*LvC?;@N1qY)0ZYXK%5>xidv<*K}+;r4eya
zNz{7Hh32#wX`fcA&#c?VnGsygl=W;uyHMAPMd|-<9X{5fC>HXl>Q$iC%O&lNW^T>1
zCtZ}xoZly=wPJD8vZ##J69Z-PS5G^*YW3PfzgDf;c*$z@+O3bOR<GOnsp@s&elszh
z4Trg+bv8uH%*bgxtu<Rm@|;ujgUy%RyiaYp=2We_`HbD|+?IP~-MQP>>=D<Ke4Egb
zIMvKkP|}&*@LQhqr%TrB_kOWk(!lg?;g$7X|E94RxH3-;St7eF=F6oiTd#x!PhIn5
z%Nq^bjh})8oFi4I&v)Cob$YN@;hyRCUosBzolyE7%pLIM#?y_b^$*KzK4X0OPtkGr
z(vw{sUr$NT+})vR63N7<vVk?ty?$EB%GI4^T9^HuWoKX6boZIG`mz6ky3#q3`L(z6
z<I2~XUOi{XE?cxcP<`&r>w)dDx9^uGMHg*5C}plbRmeB9s_&`pg(RVji}5wTzPj&d
zY!04b@u-8pV#njQ%irHNbNa1Qc$(7Fq)^K*ckWW#)(uIQ#TZVUQ0iLlJ?mHJs@dm$
zb!qqo9MiSEmr*czi+<;CSD$}3g5C#gVp;NBDy(w;G}D`xmVWAFyY+njdGqsoKL1TQ
z$n~Yo$bQA=i}Ct@=gEBLtoznf<*ye0_d;`>{+kBB(@{~|lv+F1Z?=2+>e<?P|C1Ob
za!Z!~FFzK%NJ?P^i-y!C5p|Q5mwH!KJw3qoPOFK3&VyAulC`;`CNyn{I<sC(c_v@e
z{}0W_vU)_03G7OX)o#5Sq_{PzLsxv$ht?Zi{X2zHbfuH{VvI$mZ_Nwp*V?++OF?R)
zvl!D%WjnSw&nZ1gZ@wC{+ALA{spWj&pXnqSMIAM(CeKe&X_GHF_9R*D6#m^2#J~DT
zt7cwga%A2!akFEpeI<>_!ABIjcJ7#{SNZX@T%ek}%_rCX+&kVePHEGlI9lZ#BcGfS
zo9VIS#!>I&f{ybOP8{*JVw>zV+3CN@=b7hfE`=<0RLI(KpmDixZuTmZzQU@{Qfu}a
zO*?BTT^Kdf>-n2B^Av08@;lE?M>eU?xGkAhu9fQJT{xlHbHbt8S(-c2UOl&Mo1OM(
zNv1;9wd1q?bDVnEylbX2r)!2&^XWVFZjF1+=Vp|A?hbtYrQJ$m?`79FQv}7Ewk%Qm
z`NETP@}&<2UtAC7b|%JI1<yb5MZH<bB;c0zd?(3Wwkvy*cy2V`a}qSY*rwOD=*i_3
zKhJseS8d?e5EXvuFL1N_OYYM}Vqe)cZ<X#(J|r?zK=JOfKigLZP1IV>(lRScOyl7E
zEP)kHyjp>^rcOzUUstH4IBRdas<S*RyVa&8aa+oYwZhi~V(veBxMRgtUHMhnvC$T1
z&b><wn>W{V?T;oU&RH!6GcUBL@iH%#(A>6ZVM5M?j88XCnuKlM&s3J?k$5zMDLZ1{
zuAKEPHp+oQKZV04d`#*z`TtVEd0E8l+B5Q7JD8#*9~m#uHC)+kIqRa)<C#IM0aJV*
zYi5T(-nugO?bevnQaPEwXRkZ>X<Nu;RsE~ew%z}=DfbE||J?=7cXkvmw5ff(;Qp1%
z+n@Gbn|bNzv&SOgcaO$iFMRxDS@VwSoy@&Tb~!g#8q5-fc@_0b)+#W+ydio1`!)T2
z@22E4h-_fh3Vdn%tMxo*wBUs(9+8For5cU9)9sjx^$#+)$ys|{v-udye@G(q!}l}C
zjlOZVE_v^qVjFO3DrbDjL6LRG#pF!m{{+5T`XQH}fwANA+*cxv3!~Pql8CqxwRBae
zW#5Coe4CKS)DOou1gTB5)reZ^l^~pZU{&M)9jOZ>oFZ3wml=glUn;!LErZA6h$4Tc
zhMxSbBZ+1=St6gCN~&fRb+UCPo$q<`OhC?!J8{X3l#3S2oX!<a$iGrN@97M8E2dfJ
z>-ka^JPzXajW~b4@r(Anml2}Jl!PXV1usc%O>%RP+H`)Fo=F1RNs;)TMFzIcXJ%fO
zS$To4C1d8np7e;bl2@k{WiC&N%86MW{d9xT)TGI&g%!Epk3LMZ*gZ#eL%rtQ<nn{o
zr^M0>Q>B8{iUM5DSE+5CU8#B`HhTBfJ<A%`bneKl+Uz)$`MLS=OvTSt=ekW5nU{F6
z$Nybvag|FUs`jPrmM;rDT-B~QePXhC)G|$TA%|F@qo&RO4<8Fl4oQkmo@V~)L-Qj3
z+KGIcx^7aL{OmPLx$pltY;C7ey+MhywK3ht``o-mg<4My;wl-_|7?-twtQwcZ|9lv
z(kHW(Bkjbhw9c>Waaw65y2?moS6FF=Ue46lrz?0L?eYkGc5aeX%gGp@dmTJcTh*2r
z1n#_<xclALS#x#X#P{FSIAXAD!ot0Jy`@WzD*g62w#_H4c4~Ff>%NE$`b*`43|<R2
zFMU?Q=^HU8W@~0}_K&4o&9b)7Tbpw@Ej#p<*!HbUm)_jBE&Ccn_>L2N>k23TmHoKk
zZQc*Pi!W3eVlR1nab2xbD~;y&eZ5Q0e$wZ4Pxo~y94gG)^PKlZ+bw~&V*e#u?tPZK
z^&!}{N@Tj;PeZARVr*3(o=e;OV$m;XYTY#LrSpEdH4(vH%JUv}?Oy*#Yj=e$ON>TK
zv&7#U)9o7nKIGQzxpU-gj%?S%)eH;;n}W@13wz454oLnES#5Bk@8F!iwWs<FFEE_)
zV!Rd;m>9r%^3Dc{nHzi*&gC-hG?`gr<K4<IH#D?+s%z-f$_3$fCtNIy+);6J$Cg9<
zH(4$pJ$~6UaAK*ztkg@68zKcgC#IXWbWE5kbmZpEkCF<Sx90IaT*i4zP-x+zEv&X{
z8Z|l&YKSN<PmF3_>wG)5N7C_9n8HE@*N2M)Z;6X3E|W|Y-_~kB;b6yxo4je4q@skR
z%`X1mC@}BhoUrsQ$1>-zX>Ogi?&ZUrH;PY|2Ce&;s37OQQSi2c9ecr-NgG(%Z_C^+
z7nr<hX3SC-FLuuZVr%A=#3reEO*ixP+4O?_*@Qz+D=zP7e5CP#{rLu`ilx%;w>s@U
zds$m6UgyxGPiOr_Cb)i?5&vBw#C2kShc4IrD@-C%r`mUV26DL0SrKyFH$*q3DU9RJ
z&%zj!syilL$);9wLN*+_w=-JhmjCMw5vNMI{%^d)uvF~w5-!GnB34`rR?Sgnxp>Fg
zY{6Mg(Q}GZMJ6;({SsrxcK5<dW&5s#5ZTzULeWcwp)Uklx$7?eNszr__{8bmf)g8=
zuJ^{?*?C6trtGN`uK$lMIiqEx7}3CEy~gR+g~JjXoKGaMMjq+8u{6cj=HZQ<Pohii
z-nn}>dc~4O5z`X3ct?d^;+Yrazb`SF#bJ^n?`qk&K(#nIy?OF-Paho>%Z}*1YH|EV
zKwQYZwmT2nZl37PXTARHY;V~f-#aCTia(x>I5qp@LibLC$!bn~Zz99%HmH^zz1I@q
z{q(|9b6;=uZB4venRYD{Zx88A(YU|+?EHjNi}rash^$fg>KpGR=w-WuKRlterPW`l
z@XXvZCp8wF*Gh5n*ygFP@OWWmO3;bUlL`|0dHz8ihcwpQ&(czJp1JIoqw??A^C}bX
zbn-5%DHO=KF-2xl8^hQ6JtreF);#`Su&RwwN}WaZ!77ahg@=-hnIu_+rt+M7T(Yg*
zSEQpplH<sSm??jg*^8F5&suP?^ri!!>x`WX&b6H2l@gq_*L$Vz5vOH}jxI+MT^KZ1
zOcYmJD6q^Y?8U{o60SF&dY;k{`YmzPs^_5Al0_0T&UmbAI<-Z`w&S1{gV3A{O^x>=
zXMIqYPMsw(?Wkv;qIlDa<45MU)oCoB`9d)7g<w-uVvQK@HpK@!S0oAe{+|%GT2z&N
zf{>==*{7PR{hwa+|5IG2$*!mAt|$6(!lg@v29NS}Z!5n#8>x1=Dn`grf%|nLr&`iW
zRkn`psR<1sC*Dkaq}HO;)U)#3)XS9%Cm(*2l6X$(|7@{koq`iPCO91sjp<<Ivh86?
zTQp@x3~RW6m{*+I+?ZCiD<>DOZAw;>3OM5VOJVW6jQMI$R?gdScuqjFejBIzGf$(e
zIYwcxmPI{Y9=0m2b58ZrzRi}m|9<Fy*t$5sFZuMz2!By^7DI(IKSS1@%CfOaU$>9T
zHYk0A%FK$dkL`9%IDd4)dCBZh&J>4F0hbm&a9r12uc@;4NVl_@$`#Ee9WDpdemEsu
z=((ZUx@773x-Q2koh<t`%@Rw_?_9RJX;sg(t4zzfPHJ>G>CbtsVIdS=!5Wcq#-r}&
z!Zqi0_bmy!z_e&@%B`QFZii%JLLMH!5PR#S%9$%q4rlq;W;90aeE$Dv->vl;2Ltxa
z3Y>SUPf1lv^=ZG|y{D#6Prh?|5}CRoHdX&DZ(hdJNqJ98R`{0R(1~DjV0w~s?j1k-
z$2I4Q0<J#o-Ez}q+O6YJV&|gM#LY^R9{Y5}PI?qJQM2a7(<4h}$8S1)f6Gdnv*LSp
zp1Zz}L(p@|mzo>yohd(3wRDf1Om$LQ$knPQ^XfoT>z%CoX-Y4SD6Dc{_AG2kTD!uE
zkQ1GjUpY^&xt}d3)6&QE_2Z*QU7Ww(aO(Q_%t<-g<d&AX=)V53XPH`IGja}EmA=ib
zd-syzz`llKtJ=6$f4l!*F@WJ%R*~HD*WacT>{4ELY2CVMj2Fz@&Y7_>{o#85Z57Lh
z>;Fru7L^q#gjPnDE4}AvQlG}EGX3I&&q7Ci6`rsySzhB7=&~mA8{ZM<0~t<JuI@ha
z{F~E}uWa5dBGcL~9oAan>T;HIXH@?!xh(q)hj+}<-lcWa)9Hm(k59{~^<o@)N^jS<
zv>n}(x8&EOlOY?<h_OX=O<E||c}PNH;X$cZv7n=~SzaWVz6vaOV&xI#vXsAJUIue}
zL8o4;{RStIFLR}5-Kt8Ow0@nG#>qsDT*kg>$BtjRb>f@ighv|`{za|lTr^Q=k)l|5
z*r#cyi&B-FmMWJWKgXmiY+Weq{PCPy!z;<PYGKlsq;ySLIcC<Kep*tf$@QL1jzjy)
zQt_kwk-2MU?^^W#`Ld6na__oaJe0P{C+iuf^^#@wYn;P0Tx}I@ExzVyUtu6GpJjXF
zb=ZodM?{Juc#fXXDLRyywcP2`f~aZVYHm#b(=sFA#dLwNjBn0abw_6$*=YRh<6<Tq
zm1eCKY{w)w8~&Y<b#mf~#25E{7v`;ACALc8Ipd>Gh1(vVKD+MiHNACe3_=WR4L<7$
zs5BUI7oX!YHd1GJ&s8iGp|pW3d&9Tnjd|*(lNZF!d$5TwMSB6aQ<vYIRq9*soff&H
zEPi15o`lsRSD3c9-Tkw!Ij|x__s!Z{S*?GxGXCCL-MFqPwC9jjgsXD#(W5a-j+p7V
zm1UnbiD;<e`O~It>m8!+z*_&P_`l8WlM-u~TPJnOva-(c5V4!C;$!s2r|WJ<pNX&0
zn~p~*_dT<PH>Q98)w7CMRbye8Z}Xd=Um1MzOLOL~xf;;xznC{H?CC9z(!$Jpw-3C@
z*Ja-n=`LjbRoJ;y#M|`cjPUwIO|k6E=(w-1YolYU3?;lduKLYxnP6zudNX3H;omFI
z99G0>|6$UscsskRV~*$fg_e)CCM=oJ(K<ivsN($Ay-cmH{XHpKdO1;gUnKRu8!UGJ
zcJfc!A&rz(dwyx&Nn;H>uvYDU;IDJq_is&%VmZ3~&{}2x$I9y88g3qz<Vwq$uJw0m
zmd<=VUFPLkt&58;nH3&-mviJ@(Z5al*S#kA3jBX3S8Tv<%=AL<g4AXs=C!s>6F<Z)
zFAL&kSG%kJt6f5Pb@{B9#m4!ZS>G@I6fa-g{e|gb#8lC5Ve5a)G8f6z7x~*OlGC8Q
zrIBAKusSK9Z_&aUFGrVicRgJ-*j{HxHGDeNvXE)H31|0)2IGRxpf_3*G;D8Ql-R)2
zu&%%95a+xMNtLOx0R<`c{2y$$3wjF`G|K<rn7=~GGPZ`Dch%q6o?o(@H#2%q8Su`Y
z)OWAI@BN1UX*%n71_>~46ySD0{b`3F`)x&~LW55C5b2*Guh)o$e4G?*I63~Mm~xTk
zHMyOQ!4lJjC38F_@3Ta$E}ZV&*jDmWN<a8fSV+W`lP5}U{C^szl^MTSbNA0<Uk|5>
zckN+&d?csn)EC1)hjQmVczJ7?($Z%eo0PkvSYFJQwmP0U?avAMS6Ak1uV(kXX_B*E
zCi|uq^TnK^u&Um--<Cb+I-3<xAYHtIz5HBPl@a%L1MbbQGM`V0H8%?`-oX*O=6{S?
z{^m_-#x*G%7VEEfZBO^MUc$1aN!!D5>$3$yE-oxC2c>p=ZjjgUTAt$Ox#94<FK1kG
z?w*a^I(v(a@CF|zi#Y2<o6RaBNgD3S8tvbkq6Cgc$(-^@61(U5d$)FCaD`{*$1_`2
zS#WQ5-%zk7uPVp)@s<-ze|q#&uO1Lnh<kc{O-^9^Qvvxs{n}qIUw9LK@6CT9<)0Jn
z3olyij9#T0uCE*Iw6i>SvbcY4+`XsuU!I=KKV-HkbKlVpp&d@zJ`Pc{J?3;dWv9hI
zliHT^>W`-TeXTu(i;isEmvl6>+$g0!J8RaNxgx(<cBftL-nM+(&ZoZ)+n4C&ttwmm
zod4pD<o^>K^S!KA$V|vN{Vw-%*;=iD*UyZWAAiiett<aL7sCXrJMx?FFf$g)Z@xRb
zDp}#+dbag5%gXO3r*E!^{HT@Bp_O0S{=CduU%R6G@QwuQ1!jDUzE{qd`tJ9yW^sA*
z0rRTs+kaduua&MA>d6%d6Kg!&>MB<8YWBa*wjYAW3?ogXTDU&^TA?5G$3-Ua!z=zD
zOt0(yOR2X0{<}|#u|9$0|F*nr>4Ir9S61*ITETy)B%r9Rd+{-z(>pZZX7DA~_kPw;
z_*L-z{g3bbrAk^ae+U;&dMC8G(p@v6=BRU_&>#27hwG17@2qQCWOlt?eEQ4JUfXxo
z^2IQ_FmW<*@|i>kJala3l+-JkaU?-VGpgq6=XAlxuDxG>U3?v$zu10)g6^a{C$g@%
z_ew^tE4eB3%zLI`=D97LljcneHU7PJRqCW<&)KG(e5O$%F9Rp;dv#Ok>4XJMD>#>U
z$wo}g3R@kyUd@9c;k1}h^o|L04?WF^+LC>Bp6c&yI^sKvUrw4C{p?-L?xOFPo+gPK
zi0^4;6<4ddQIK?``@g7pTF-Xw=_e*CTAT4Oa4;}%nB}t_vSggFKHgE;GeJseW!Rcq
z6SB^>iR9een0bEN*;S#rcej;)fA)8_$jk?aTgBDu=J0AiK6HD_s;ld!etxm`mWn5X
zVQ15u+grn*&YP=Mu6p;Vw#5IrHQzr!QPr3@!KM3T-Cl#bC0p~Zt}<Nx;``@#`NeTB
zes<bLxd<L!{$KA?z9Yl`;(&u3l2Zf@u<0*4e~4W(C*bh^s*5Q+9O`olmI@{seN+`O
zj|o(+Pbs>nBoRAjqFa5<#*1>%vsOK~4C}a^!mWAFo8R`!q0f9LmG|$~W7Isw_1bgC
zlAhD+wbt$tI<52fQ46=;&MBu)>plPUe^QUZ-ZziD4J|dBg-vdj6rZ)YS!3*Hp}yJ3
zXy>~(i-gUsH5UsrxCu|W;K-}HB|x(%I%KPxWbKuUZnZ5_E_q1Ky>iLN@ax3hz!M)z
zCi|b{xOzFrT2#+G#8a_VG<r$#R<Y;vvO2<6<g69F-f?fTLRVBv(n>Mk2+4pcfo`u>
zh?TvP?3@-DaB97LncKw{u?0@QLYHK3K6))GFLjdt?Yvl_?ej9X+nOsEWCw}gE!e8t
zb+=-3?y9*_5g#W`^*weWsas=9hf|lzq62>6ZQHCWR;?3x)|JS~Fo8jD`;A#U*6vWT
z^VaKGDbu}v&yz!|A5ZwNw0cATXQeaij_U-^d^%NC`+v`->Dtvkn=hC+D;PUv|MLl$
z=kKi0X}BV|SM)}h@vd!)I-m6hUvJB{3JuH5_O+02oqhI|+iJT!OaJE2Q6Z0}n5~;y
zHMd!BSJ|xzkF4I_TAgLH>X8}$yIq@SSMS|()%nxz?VNeXU3s3*&X+a3_OVnnux72Q
z{U4Wwtp#BfBB?xmT+7`*a2oSZ_{?oBf9LZBNBP(fm+Z}BOAh{(RNdEQbjRlF<<$Kr
zN~H}~$(Bl|@44uDuk^e6aoOd+t9TUDg@U@Cl;6$s==ieFdav@*<<(WM4}JEsezW$r
z-I{mH?=&$_vOHi>qkC|R(b+E-<tP9Bd{=++di{sx^1inFQ$9cb{dM{OHRr8ZUw^JI
z-*lfr_3|=?b(#U~K?;lbR2zLmxSD2mJz!CNVZ!xmh3OK8scf#HT6}#?%~6x3a*|lJ
zU#@XuW^MFX7g3=tzREx;sKZJ7(o(^)HA=#(6WsleIq=uNNN@j|(E8}lX1=I3Nl`Kj
zS+_<VTDQa|P&2@VOV{Y(YK=I-F26t5f-0WOsqSs!{q<=2eg-wcH!N>#GfW<TDpQ>x
zy6s~tx3ff;SlNWNCY?Oui*LwjFEgAJc(SJSVo-d@G^6QTc_y~|_?$^<^Ej8mHfiFQ
zGiR-ujW2jK@?6Xek)CqR<U!3P^Mx!=uezKx4c2qd*<i>g9hJzv#aG=b=kU3EQ(XR=
ze_6BK!{t({-puFoR9qTFRk;MLd7jT#bkZvfQ+dXB{o>)cHBLO%OWGye9%|oM>*1>P
z$g_3BN=3fX;+Uv}<L16!7Vu15p=z2n@qNY<?&=-E?&l-VuIk#(`dIUP<>_MSgH<=@
zJ==TM_Oaj!y+TRb*tqpoY_UNMGB0d3YV{`wPJQufRf^iT)|0hKR`Ij<wWwKhwk<AR
zHTl2IuT3W<3j8%p6&5fYb(pSK5D>LOfi=@A*faChvI)0BavV6^f_h(N*|RR>DBi3k
ze2i=5+8a&Mk8e0FKQ?jxnh#<2O{$^wYn(-_C68(x({%B@(j>RILZ)uT(TRVSEa`kV
z>($N^OT(7_@6cBJ^wM2SW&6=3TTIt}{ouMcB9J?*H;HE!$EE!#5(bB7ecX_hqHKTj
z#Fdsk6I(eZCm)hfyLqbZc+$+bkB^)&67Q`o)2m<NH1*U6_eq6s^yRdB6eD*>&CoQO
zxN+*u9Lb%V=HA_wZq3|#-eq&^3kly$_vl*%t+S;c<mH;JQc}&av6Oj`;yZOY%Wci%
z%Uv%PoytjVUK+H1^XeUYeHAJ;cU2sot@5hWJ+HcW-ffm!^Ei5x3g)h6H~qWWW#zS`
z)qRs!HH2;KOALK&|J6bEWt#b}sXP(~S^Wy$Cq1!dzn;vqxS*iNAjO*PdtUkO>hup)
zCHg^Ef`x@V=1sVxyyUCVw*Pxo#5R`Sy!}Y{_u2eyJsa;`icbDAuRQO;&ceR8)Anw+
z5t)#$IX~u~#6^>fuC<b`MZZm~?0fAjPkv?AWc+BJIw2@+>D&|Y(^lRI)PK#{pxgS=
zTT;Sg_BD~S>C>X}3`#ZU)OCq0ymf^ArO)gT%WgB(DSdkvZEkJ16}=FkD<QKnc;|!(
zOXMSude(~{=~MpqB)076qa{l-M0B}QRg^fVN$Bl$-TP^)=s6~xj$<A-Oy@=QcBSeh
z%X+_Xbl$dd<FTt@Z(fA3vM_kfzWOz4*^-T&UR(4VC%)Z~7@c#><dWgEH<Ia7WM^Ic
zaAuOn1cr+ioU^Z6^qOC1d%Dg0WbVBMLU)$`mx<0jw=6ayZ_)03KK_mikDrdc_37@l
z<VoS%#2bCDFuA>0((3+4Sfr>Zc*Zl9^cIb^BBgSB=E}64yV0<OQ`bY0DQLsd_a(=t
zJU<a^*^s;AQ;P^|T<x9d+GZQ=nZ1f8PnUVB&c4F_x^%!UTmE|n>EC;MU5Y=ynlasH
zQj@b{)~)#ecTQyO(wH)HPD-!W&N}6<E9SUaE!P(Ru%x=F%0t-WnPIaU*LSba*RH-v
zoBL?W+4`ps{9CKGE_7SFV#$Tgx&_LzMq8{lZ4>igSS7^p=H^_jisi`__cpisEH7PK
zP^ukwGwbC+Z>!6v9|`8|v*4<g?cb0Z^7uf?sq{ehGVZ93n*RsbrX35`7yGiN%=8Ys
zY{upN6Ru6!`sNj{ngD}_gYMkBTji{0Z@lnn>4~aY-t%(&Q|@&0=(nl#oaD(gN|L{y
zx^?c%#IopF>E`hP9t^%hCJuKVuw6fQo&DXn_G<m?<oUnvE%&`TMJ)Dn{aTC5yylX}
zYPVMHt+%=$>B{8vFZR<s@#k?fxBvTj|Non*=6mDbOy9zAU##bVi+k~om$P^-l>S;+
zCb!|B5zpZ&_lHr9vX}C?W%w#9RG9+K>@G`vqL9SAaWV5vjVEsJSnM}E`SkMWtMf+_
z-q<8uuu0j#di*pSqrjd6-Bl$Y{=L)xzic_XFvInRC&ycSYEB$~zV6A18967l{&y|v
z&|BQ|?lf0N!;&?p?x{3v5(wDPb9%)EZtqiEtMkqr(_z}u@cP__*IFyiUh~O{Vpe^y
zCdA64*-Evghx>fbvZ>$JyuHWwbxYH`FX!I<6R4I_yKs-g`iSPG4NKd!dIftAxeEDT
zzHu-vK<V5uhEJzXeC9j-*-Y?inC{hc8CRX&Uv=Afm1pA@zlmRh0<VRA{Nl6T)1<pU
z>0*E0daojV@0`F1QEoqCxPMj!PMnZyFemq>i?d19v70{3jDC-;TG}SJWSh+~G&|*Q
zKC64@l8d*!ICmcFw>aSTcSoD)9Bqp^i{&2WSM2JpJh1rxkMmVId{u0#?%va7yPa*V
zq&K6DX)f3Qb!s{~ZfYAI6{yS;^PAex5^pBJ>ga!z@6r>eiLAaKn8i2!6Pol?N$BYf
zZl&OJilGHh*;ef}xpXwCZ-L#XX;nSPcPyTEM0<LdfnF8gi?B<aa|IU9Tf@0<!V8O2
zb#Gc0|8ZGj$HlYP=hd`xuj-5#1su5C7<n%i@G8I1xEk=rVQKTMNz21p&$=Dt4OQmv
z*Rl$kC=eN_EwA0%)cIlGrw<JIaVBfm@6(9?mbyWWDM97Du&v<}5uq<37e!`vC3p#n
zK3pNXJxKqv(K|7vudh753MFuV$@pfGty}fpfH707w%?Fnc&kI3_~%7Gl!AZuoHgoy
zyLFnr#B|?)Ne6x|T5qEFzw>XG_wNqv-#h$w9Pu~x{`_Y~w^_(%)1@1;FZG+veJwp*
zIroWXZqZ?ud;|8k>+`bSnAZwhoL<eUtnm6vU_rt2vRR)GX`DNl=HgzGVjA{1_*Lc+
zmvi&o`mDEKw)Q?_?fStw{Ed?AB&DO5ZL$MwGbh+S{>XZF^HZZuM|+Ab3O87kZ(wg0
zKK{knKDTF&Py$!O8IIFQCyu%6wfT5Y3phDr!wV@BE?JS6lh&MC6Hvd6r;#mr=`sVS
z-8>p=`8c;J@OB%yxPN*5VuGgaiRL#w%ZnD9E;_8W{BYH`Ah+Vyga@kyMGJMZ55|3X
zt;7E5q1k$2%a6jBPv~`*eCaFE6IJq@{(p+d)GvC89s21ZUW>1MVG0pjza>e^n3J(n
zui&Cri|*DAUJ3olKGDl2P6;&eQT{dMqRFiile(9UQ<mM5dir|{@9lSAB-a{Bt^a7Y
zBlqu$)fRche~w(dvq(8i%1D_pi|x2Gqs^xL2f@eA73POc2tRQ^?uX#RJeOeY2@l_B
zdTa@{Iatgb+@}z(^VBc&X+UUb@YJJXlM>`61(|=cNer0J@`3gD%DWMzO3{ADEQ<C@
zO+98~#PG3yT3L_1>!!f#zKmsls>_!cMov;)dciT(=+wS9YAYwabkR}|T~)i}jhB_k
zvPll1E?=516)ZpW?)9NQA%Bq-2QpT?Fj#(Vjn;DIu&Doz9xF<`a#@ugaI46DyioQf
z?#t7UbtSr{8-+uCMP3`}eGFm}IiMgT7O!j;uR1MBeH;JZO<xxzF>wfcf9EiCTx}?B
zZ0O&eqHsAS^mU5%W~17`)DSb{R5gv6E+%V|eY2-|1}&EKo@^2robFw0y7T?59&htC
z#s2Od(%Hs22Yds%mCfThbJZS%<`(RIGudMGG`q&)ti%si+Zg`cTG(t8D&#P=ch}^Q
z!;@{!Z?ks38ghM`Lh@Fn`@(tGh3!&5WIdji_x_sg>#+Q%m!EDj%YQ9gaI%}d|EfJ_
zxqV>(ml=cVLH83U+*6v@scwJ5x%8{!%XF^(bxvz}>h->y`jhUo^Z!+DL-Ckctw*aS
zyZB$%$jw^rvqiI-D{kU4-i}LwpA1&MnwqN>mc;wfz4^;$k+3fhE_uwn;<50HN9Xm5
z_d?gQR4XOJ^mN6_=Wbv5@6*aj30xN247i0CWqo|k#OxEe%<x!=Pvkdo-&b2Tnti7h
zR-Y3$%`4-USnvAui*#@I$4zS|FH<v1b@sRS^nYuzbDLo1%I>Tq%S3m1m@iIdKE$n@
zdNuK`aK!V?;nGvt<SVABSt#8O3)im*HI`^Lk7)j^*(@&FVt?bQUul@)RomZXPYb@z
zzh~6w`R-|4gtDk<e(Th>>~%J6Qv(w%BAd4!5iwV-zY)1Gv_mnxNV)vvy3lCj|0U6r
zcc^6+Io>_~YEodp5{oqXW7`A7yC+YIQC+(H*49}6r>5yyag)1t&--5a@KSlQuBUWT
zB{Of*!s|T`Bl;fC2-8`8P3?5+6a|ljk5$I2zX=JaFz)mU$W9IjGxBDeI6Xi0^c0_D
zx9?}nCD$rXx@exZHo11O!sL6?)6H-GvamF@URgJ>?av|pDZj-tcN+#UTLheTGcPUX
z=5XUu*kQqZJu7o&)=D$Ga8ujc!hyP%o^X9qI(WG`bE{3NK<jg{nQz6iKZea*T4wR|
z+D!ilyZoEe*tZv~pP45WF?;jO*^{r%mKJMTexQBn5r>1v3;C~juWn)Psff~?9(8+W
zgUKr{<Ns5*|Et%(yUD%rLUi&ZVYll60uIZzMJ(93!)15Gif12UWm(UQwU&K9V>zv>
z@A-^Mbt|vi5pL7VdsVF#nMW;7^{rBj(01CD=pMzW=FtCpM}KPn^{^`M!i!>@nm*~-
zKlV+Zcyzi!)uknQVaqR1Up~`?dsedPJ@fSF=E){EO`nuxecFC|TGj0o;Q(LFtVP)k
z`tPPjHS&j-Hj0-n5xp^EchoHNORepaYwnuOu&ii4dTWhoh0@6=C1;C3_lT!mKXs$u
z%*ZY+xF0yH+Os`%YRlcg_U)GK#W!ZN&)&dYy+NH}qj2?H@!8xS4Ll0M9m>_-?K3uQ
zbaA{d`D$68-?TnItN+p^<&(Z|O6fYjAwX$oExXj#shgICE!FgVS}JZ8q_#9vYSwv$
zU5hQHw&zE0FW#MWU!vST>g&B7OWLcMysDUvrO)+tFgTT9aOwc>jN3cqe{WP?mTbOy
z^S_9GyKmp!(sn1_O4(;U@pIC|+*?Mh9!75723nEJ)hlaXzin)5GwHQjvDSZI^)@Nn
zol_#0{`vHK*M;-@59>C}7yMUtdDZsg^Eb_!l|D1@{;Zk8G1GDlR{t%t+q<jf<*Y-$
ze?8eEwJx?``gF^g{gQdF%?{0<c|>&2wEt!^Rqs6AB(<UZ#)kMiI|OGlt!QA*n!|0?
z!0okwr)19Y@;k?U4;-)FbNnNd<NYM&JO3BW;hkc5z0!Ne9sN0T9Op}HUOuPj`A+VQ
z1>E!V=6$SGU%bcrkENt?r3vS(Gr_Y1Ber+PUN5~_5PYoSyvIT9CpQ*VM(aP1xv)d%
zyzaGgFZT$@CyCbnPLyRzQk=0vm0{yV>*LdJ^PEb!7|XCr_AZZC!*SKx<K72YbYd^<
zesHOA56cOMo!+Z=InVtTVYPXSxOak-@yY0`y3EUWCoGTOyF7TYRpjB?yVGy2ID0KV
zxh~Vz(nfR2`;d&ukIn01Z!CT7ALweirEnFyjnxSS>yR_Ho2rgJ4ZdYuv)~l>;l;6R
zt83S)Zfd=H>-NFeJC|?WD!sb)xz%*dqQj0BcVn*}dVI0f{eMKn%DuA_?`)7Z+o;~a
z)Hj=ZlJ$xDKlfX09<awf;GVa0cgQB;eVZ13-88kJcFv5=QzQJW%sY#VUYUt_OLTO3
zSDOedC@EZg#<{LLeEOR+=iY3YkrDHB#jmWeCpSD#Jc=uie)hBGS@u4~7V8VFhZ4*8
z?Wn$Yp}(4IQncZM**m=#a98|!aryvD-@QxPcUi0)F3q=lwNUP|`P`SXvag%}Y-D9#
z`u)W&*2vw5?N02vJ>j5rYPQr{!No@Mkw({Md)}Ds8Sr?^L+iTN`}TdDJ^5&U`pc00
zlat>cUB7>_*wpr9Kh`}{FD|os@jl1*&Oz~rBX5$JjwnrYymPQ{=aI#dpI?g|dHsLZ
z=giW(=JOA!%FJ4L^OLR3=k0%UBW%8g$8Siqx!*r$WAD8O>GL0C&;Q1|@5RF3mw(sl
z_02i4@A&t#KOH6-ELyOqC{bu$|NJ6Nk@?$WieJurWc<DOv1RRnKPF}8&bZDyW2t)f
zrH*T?@!LBE+p@#Mp8YtXuD~cF{x^5t-^_b||JyvzZQbE*^`bHEMd!So;<XnW=j^nU
zRq&eeBFHIWW|JBpmrllo7RQ!Y8MQePZHs0$v+SJm;83E|aZWA=FOLNZ4xX%v%L@We
z7A-wJ%^-f)okOi^zEjP!cRg7&S>1P@L-nMGMq4zd8@W#|^Xw9Nd1;x?VyC%UTeYsP
z4x3sO`Ku(m>Awo+gc@1yZEqqr2A$ZFb+jvY_O><mcC~7o>&5QKiFkT<_4F%o`)hx0
z`slri_u;_~X8Ew586uL$C&+vE@yvYt_}JQ8S1;Ykdwa{wQ^aH9f|e~WFL8NjYxG8$
zzP`M{JO8G@udS1=ZY;cg?d<H*{L3rzZ<opL{_)}QvB}Er^Ny5!zPo<@>oPsZUGHC?
zs$PFjuckU{%7eq*|Lq(PwR{#ft9Y^^@bSU#s;71|_}lOM+x7MH{rl_p_tgJ)X5D&F
zM7XC+I)YI}z=46mfa5^3fZmD&BGEUNOm4ORx+F0`rtHPTc7-;Lr1m)l2R$X#m)&UX
z(phvvL|uE=j72>rt9FR?Sm=FK{LjFtFoB`pj_GG|zq8r~R?W2vD^n)<=)Fvt9AKxJ
zIwi!<lS@0y&9f_h@dP6->BSR1rcRH^vy7P3oH%p3?%EBJ-rD)wf=+pqs0BQqTlTE;
z`MgR$hlTSRK51qwXyMDsSlA&oD{4`X+^dYm6W;B7F}GW5LHg4Bo&rH0LOdr9ipb;`
zh6JZ<acry4{bL?qv-;HX04D)Ql|XhOK@|bVA4l1>>%?bXxaTHYyJh84Rm~sD&Bs68
zSg`4YR`r^R7AHl5y)B;}ne26)sr9YjowOZq{Zsz5i1<HNbJYuYYBg0f?9IB@dAna7
z^HzzlYnsXsnJzhVdbG!}tM6ylempI9faQPohyMq{za8~Dz_Wbihr@i<HaCw<3H%e5
zoWAYm%_!;ZO0G=nS<_c$mEBV-I;lL{=T6RI&xH*o>Z`>*=V~mrG4@h<UbEng)#E*z
zPaXOwQ<Cj`*>cPI%ZcurvkboX6kT>u*8OzlUZz!C#jK{5dsQ-jy0q&PzL(!$mpJ|J
zy!wxS%e5q9Kbs^p`bh77uu0$=ughJ5dw&<)V|X#Q;hu2V3Sqy6pT4YoD4FJwbo)(X
zgi@!RU7k|Fq;Nk=f9*-XExKlv*NODFmwpOS*PB!qqN%q??pN@F;yUGL{p(-6dE;yq
z{azySS=`fiyS|^Ro|082x$Hwy73Z>9;aWRWCxvf%sivKO?a-<J$FJIE?VTHRWlN(-
zw2#!w@7LS;|NW>obGQ3>&wJI%t179xGArl#IXDGL|6Ts7e)plM2~OWcQ(n7lpXA{x
zbxGjQkL96Xv;8-mGuD}Gv7wA7YZ^<Y`@vh00#0H+hs2htuH_5LR#WKd^lw<SLx5F7
z)6FtGJnY9hu|ua@4RW~l9Z(Y6d?SEqaY8rWTchc6pFXrV@4S&Dux-PUCrdk8KZ@`F
z*W|Kq18ZgT!WkmkVKc%KcsvUp^`u|>wDFS7bCW%SH;X!grI<tWEiVfy7tEM8`>{%I
z&#BJj6Hh#j8{P`+vnZZ_b=|b%YfInuzcZYzaP*q|rl-g2I#dq-ar$b;y!TBU=YI|-
zmXB^H_l16%RI_`DTK$&=EGhiX;#zZB-CRxu6?qgnIzD}>@O_u`(HAcBR||F}x$F|Y
zx`}7|oIt(!lT&3H^@?K_#7^iqvbg%A2Uq&6py-<8XYQMR_B@>v^x;T|jb7)=ij&U4
zpEi6^Ge5XeckK-)h9mChK2QBT=UGn3uQw(u%U0<JypEB#e)+`jQ-PQ5d@Y$^h9~mo
zch1}GxOBfpO=x2D%U7ZQy)&HHq9!(!F=`%o)Ud&2(k%9K>mtSXD7a0%dNQ?e(&AH2
z3?BAgLAM-FH*5S&*s^VFQ0`8XXk`_y9T!|L7q1M^{@At9FaF1&btg8oedapNvwsuo
zjv6-o2$P=w2c*gj_9}=b2Bc<%^kx{S2u<t?UUFlfg76WYs%6PhKe|}Xcx;)XnA;b)
zJMwHxTG&>rwdqRDQu#T@uB|+H(Y#D|)}?b?($k-s_CJn{_qZ_GU+>I%mK2F4Nt`=x
zY`Mu_`d=|od2X_f=O+!0xnE|deV!J1?%mnV)BfnF{&c(LHBD?u<K~kMSw%Z^W%QOU
z>shN+nPhouT~X1tv$_Vc$8VqAx~_ED;nY`c)mBoqeWmlF1NF5tb}K46FhyN>Xgt^R
z?#n21-@~7SK5p`s&@o(Zz}bDz{OIo1l0~2QJoC-}@#y8b%CxL`A4JzAJe)4q|7S^E
z$DIE9s1Ni1-dM%L>3V{J{lC&`CZiI&e?A`T<yJZL8w6ylY|L<*Y5kBZ_{QQhqANY*
zmn1Tj{BW0jx5C4C$H6JHEYvImU$%NgoDzP;u{?TZmw&le^1NLIE6nCRl(_ck@UAx-
z^c?n`4w=4leS_{YJ;yEHf)j5^DE?FJis=0mwP4Q1vqv(!pXQzuziV+!Gv|HpiNNH@
z?_JkS4h5bm_4!=7<Feb#kAm4x9Q&8wNH%jeo3^pVW%ggcqjGLSCnj=qJyDSrQB+`I
z;HdXDStNe;;Uy+U#^qO2)J#2JdPUE2Tp4SqktluJuiC0eF=$<{k@?~v?&&4lT}mYb
z%9X-SU%ImINvc7o=j+64-du}Uw3e<u{rZ3GcfE=<CE*!Odu>z`Cwt~LT{gdN)aJ3V
ze2w%H^{VA-<0`zY_kEp`Z>_m%YjqV%$Xmy&^?zn+)xSEmaAQ`+8kS3qrNTEByfS;>
zSa)<HZ_N>#sa^q1dR2+szjxnf*q`7&+cb$sYR_U0`IYT+OD~Dc-Vt>t>7}>wy@h%K
zjYo~%2JFh>dMtVS$79pF@PNju^)kwvWxYa`4o|qVeu{X`D&Nw|V9swLD~)tk2ktu)
zwd2)vv#XC+&%4<Y;Vvw*Sh07V`|sCh@9f&LOuF`E!CGy}rDml|!{25-wd`N6a;lY&
zWs+UlvCnf4FH7G#wK3Q3rraGT;fxI*GB)SG&$(l}#@yrd|G)X=?Yr+j-}n7O+qU<U
z#n#?emaW)-`>kWkG=|okprq?n+Acl97jp`|f1bR)?bC+SHQMQwcg|<N(_B~fRoB`7
zPU!S^uIvB&d9Ic5T1nRP`rf|1>r>-Zbp&Vcz9HkYaqsFSd41Q7Q#MTgxgu75Vn{U8
z3HyJW@2<}MxoPw5$p2;6g);2xe|=WkTgA`7>9aaqF{I($vHrTJa~%?H-ah?9?xg3p
zzwf`X|M~mwe|bB1-NB#wcmA>R8?stksqhwk_^&SJ6jM-m|5K*ir=(YhL=}(gZ7>n#
z-O&)%m~0`&82v!f?W?nWS)<*QqSB29`45xJ0#lT3=rb=)X*?L>H?27E8r%P{BgHur
zTxV}|U2DLV)Ec&-G0io-IdxlePHTAZa+8EDX`aW8!#BG}IY-2p*CuUmNp_d-*pkj@
zBwa6No_0{DVsm-f_l$nA@~9n&<%cs$jwiN@x6R+0;e4qzdPZx+kJf3|+VXz1O`npv
zz{HC!qP;Ama#nMdWl_wAvZ_^XRx1`~ZTpf{6`^dSsL~OU9dF2XQN(xi_l_;ots@R`
z9SZAAn`Cp6MJ-@b&ZDhj2i5(Ll-E4C9JgRb?z_Xe&7wLFxB36O;-6X|s3#C`%e<Sl
zFya34ybss3J{Jn+O%Qx`UGSA-;A?Z0H|+JtUex^*4?4c1K*7!NWJD0_^a54Y-ZK>i
z|GCTy&Mjh5WouyIC=_B37MWh?!YGnbP-wWd(CliV07ucAisY|Mo|a)^79}BHZg3SP
z_B$;r@@NzP6VdeVMacgjMP}*~0@{ivH%eH4muOokX80o1+Pvg-aF~~RY39)|kMuBc
z%ZZ{t(;_#UJ`gkedALRCq<rdhkH;1Y%tcZujXG_|%B#Z54Q^%_dA4O+Og8nLZ0Olm
zVP0+>IXNn0atnKuqh^J3r}^^I4E{wGO~zhpm-^cZR;o_%S}-kohWpg%<x^*EkKVtu
z>X2#2TC<pV$!W{fr>#0%c_=hy_4EAikCX#HG)??5Z9+!ufiSh0CYwF&s$Q;K`>uE9
z1R5>;G5!CU<2B1O{M|eJZ#dOHZnIBvc3@6q+7c9S+dN^NlOV@4t<(qIG7l5(ZmT=G
zspsSL#CejlPE^$Ai;D>c^q%af7ys6?Vr9?Sh#<+Oy>}!Uw9<5$cO;c@i0ChCcza`Z
z*vE!_mZCxu{Xz~+QZxD;w>AE~k$gHb<@XC`rH&B!6De92uKGJBOgPvSrq=9P<d*GX
zax5{esx&-rnbCud?hUP_&m1GZeJf3#tgS3L|NF-l4?~eUv#<~jN8Y52?y`*f#fh0K
zT06g$O<2}uSvi@tYjSPy0{fK<qF+QxyS0^0ZtXg@Kv01*Y5`}m0he@_YS4kU0*OUR
zQJ#(yqP$mT$_Y*VzjNoJpq0`4L$Vfq^V&2e>o9A_vUV#Au4ye9S)0>*wme&0Cd4|A
z(YHXdx-Vi$kkRzRZqp}wNLZP0?b$ANPsMg#WA1ww&2X=-x28s6UNbI<=Vd?cVppmA
z;h69-ZQ1*2%ebxvh&|N$_ECGmBf%eLjDpJ;{(PHR!Bz0(dC$6=v(~>{-mq(V#VSr~
z726Q4lFh=s*LSSAd}775h#-ln48a$IHlK97XHj@xL&Vrs&%Uw2<fzlZoqdb6oIiFX
zAGJ(<P||-gvtM`ms&&64|4-_dd_1Sbu-GtsHAnD-t1l%^mq^-%bK5_ku+BjJtYe9Y
zXQ}AUi7z>YG&Y*nDFrrMX}RRp{r_*^q^~#AgCZ34KYMT{MOyz{YstB;X!-(M&hq~+
zCppYq$F*x6N7aINR+Ar`S`c-BZ;iwHHxAt2ytq6KCO4~3Szy}cmKnv=Z9Z=cW3h(U
z`lnNFJem^nQ*rwBMaxa5hTe>xpRT;a#oNlN%F3(C%xl{6_GubHj@h0n9s-O(3b7uu
zHWfxzm+V}Uc_}vXQ0J*<l8(au(-+yjoV@wM_oYwEwbCCi`>@fT=V2{V$QHJvGg4=5
z5f7}JX&I;?m{>G<d64$35bLe=R$H59t#CTUcd2>(76#iT3am9z^|iBRu`OT0bba}s
z^2nfom0L80`L^rotSVf*EBJt<cTnN>4@Ps0LmK~Ef7yPVv&n8sV}tgNA2S*qrny{P
z>HFtXV@u+MkdnE3ZiODWH6i;U*IEO{qrc|ne@k;;Q5y4E`;+Ca<|!WCQ`R=H^&d+r
z(0MtZkt<WQtBuEMcZvJ%&E0L>TDxN-+8U<so^y3UeR<Sg>nLXFg$qp=uyO77NLv3j
zYW>;;oX4Uol&$todFH8Z)xKk=V)5?18y`ojB~7z%>eyLQ6<4|V+3dxQ8T)EyC>;$^
zIp?ZU7Sz~#Lglk}(>HEa_p1HgllE)vww`ul(@mEq%PlHK)v0oq*L<5|`flsypIcbl
zxB9PC%FAfZ+wo!s$HlxiWd~ojEla7&o4!*}RUvTJ%zX90t!g^;|5rzD)B3Y~&CIQO
zdk!^n9TsED4A8RWPF@kgwBpK+!+oa?_j0X#e8e{IcJIT8BR4)JIR>)L&eAh13tsr^
zNU8RmrL&R)Hza>uv1)zQj;Nla7c^I|-4&u^o?<Y4^=>P}y<BlA47LYvP1tsUuVk0I
zn@2>+vw2IRkAKdb<ea!cGgD#JX=Tp?lPbke*z9cmXm#T8uXPrj3mA4S@MvI2x_DyO
z?h|jiPP|>UZmP}6rkwJYto3UX*6(6Va+N&w&uhx!JL`|9Y?!qzYTeci+_No0Ju3yc
zH}Y3UZ)Np4^|W$s`Nq}4R!^*_?K`t^S^8qx*QZUgym_Vvs;aZjW~lzNx>5W0nZ3)F
z{GXaJ-Ozfeu{EpgOn;e%8Of{Ce?PQ$j!x*6ba)~&)Ah9Wj7X-!U5vAp7=^-m!m1B_
zvPo1DWenWFDz%F<c>!zGg%h9m^sI@rj{H4SX(?+Jr?{^b*FT^0208WhT1i~$7sRh^
z%eS6A-O5Qky-%C#sLWXr-X9nBg)S+XAN?^&%qFd|q5IPDmCow9A-a2=-B~7_;k;}l
zd+b-ovD7o6d!;0@1No=zHe@~|pJ=|eJZ<e`39b2$)PJtbKk4N8ylUO&@2#9xSJ!(l
zVA++i<@c4eAG>z%ZtIdc>9K%SW)-Jo)HNw7*6r4(%zu_o71{eo>r_*?XTaT7sV$s8
z*03%XWEGsi^1s%7Z=G&tN4bw^(IQ>fcCB4|=ZBqEWy&_n%Bqi899x-nux8(sx%<{F
z^H${c?BrMyRB++T@0%`F`_J;+(um&wH(T}69@QxlOLGkmoJczxWF?)NkT<z<i&)G#
zMz;JZH(9x#$?s!2<a_&E>8kuPsq-IgF8r_ATB~(Yp^YVa1E=IG)z!Y&0u8t_?p{dd
zUcqzr(2cWq7kh5I7u#$1XT=lSd)d*}7tD@mu=dTIwenen-n?0T((J1O_cZWaoBe|+
zL~XfP_?)9LdyfA5yUHP?xcjuTuEd11KX>ewa!o(;VC{p;d^06w6j)v6rl&WTnLbp%
zb*ik0`9zz|3FT#xit{3s|2_O)9llO=-PI3L>!#d!WW>p`W7@*Ar}wO0z&g=@El^<3
zv^S4t+<82`=Fu~)tB(J!E#<kstmcVx<c99$r#Mfal2G5cu_k(x&*_kTmHUobZ85Rh
z>r*9XeP)_&OuSvk>cjiQS~Av7XPJBU%#ywPT~F=5pwrn>SgrqilZN+!Yj=EO3>e)L
z1NvXi5V<&GPm`d))l!iQf+7bTUa2RZ-238~@75EsiL(;!p54v0Wdq+DhnI^FoR1bb
zT>0xz@4uG|U%s4^cPL2Vh^Vfxa>@;VIg$2W%Uzzq0fl<9XB)y7Zdd5t&UZquCR<ER
zxB0=8OZtCbYwNzz^4%e>u|wiSzxOp~%iLqr7~}r`esp>6MZ>iYY$9=M{W(N5I?wJa
z)OWn_@b}AIT`{}2-o9GOt{5lV7Eyb(_-;mb&bw{eyPx`9{Wx!TyzkX-cJDn5*iz2!
z`FD;h#kV<!W6%3r3+&%z9Ik%C(EZNo-y`cxPlc?C|G%zt+kX^X)GO+wc;!;mou(U0
zTiPcyD{r`bWAl?6bs9HcMBhwU_o*pDdDG&X2KGMlrZ_J<`)pzCttGa%%>O?#_Q)yy
z`T5>fpFpk6+Zg<|Ij~jbY2|YrDm=yb{;|VRrt|;*ya+!3qHo?^F_&u-1J-ZZz{P#}
zD|ho_uD^+k&)r#k@6g?~cf;S;3GlvpyjJ05?W?!FD@)D`Ka;rc)6@6AGXDPF%-7RT
z_U*P4+go=@_wW5S`3GM(CbU~W;IKQ!>pXYD>(%`q-&7j@6!>@fTHc0&$3`;u*2?Lv
zTl(|ig8C2bYt}uD`epZTlBVB@(AX2p=l?Q#dFQX)<1PNzoQ^+U!vA|${p7{-Cws&_
z5vYpd=fB=r=B#Uc!`si|)c#k3r@hYp{}Yipb+yaJojfs7_nz+G^XZVyP0jZkcgAq7
znVzwI8*50^XG?$8yuN?SV`II#Is>>i?>@{`&bLkIfN~K3s~`DafA0UE6`#l;m>M$0
z@O8%8MX3`+*9JzJWimD}I5c&Nss<g2FnsLRE304jL_+9^$3#`nc_yA8lf8N!gj6O@
zaQWXgMb{)V@1SK`x1Uk;4xgWrA}<0Ky4Et;o^oAzeu?)~yH=^tRhNwXSC>5u3e65%
z7rnQP^>=8_jSX|Q%sSh(ttV<zQv5z&rtrK;hHs<p?mf-5{$5z_mw$JqXXI;#)v*e@
z@%&hyc(ha4xNXf2UQM$lCnAp3L>fOkmGdP?@64&<)n!}X+_Sx_#mO*ztuTY*1t|v?
zh0V$FExy*@vZroUf7X{Pz2p6ZEv@p`*Ghl;^x(`~@%*~qKMP(8tqx9K_s8n>?PHsr
z%jY><mhg;<*qGElZ>pN+>|<-zy`61Z^kMUJ>G{4f_J9BW+<994oE>BQ<MVb5ye<Ov
zjN&N;N-EDLIdpOS-+m#b+gInv2Pa<RCj#noCU9wK&R?dq(M8z5r)%=ckPt6dzg-1|
zx)RqHUDT1ze6yrSru@pr<3Xo>7<wqQi)`{xs$FB~>8Iwzuvksqi(xT~(we7wK?1u!
zZ4O&^rpQ}oKTD6d!QLxDK89OeE_)kaRhfJ?&|~47vlfvvCvVcbo^y0s*s3+3)<t<t
zWKwmER5DpK(a!0Ll541yNCO9tZpl(dbzKu>Psyhm7dcciJ-U6|B=x!4k2xp>`Y5`7
zy%ZuI`!ysqeyK>PcVuYkwh7CooLn|*+XbfO^D2t+yW?B$c1FxC`80LqqK>3(8cQ3h
zzG-GOcWt{hZ;H}kt=yHjrfKK>-?(d)ZsBny^EG-~tghcFx;}UH-5rxIG)&dvea)?3
z_0o0Cz1{DgcHgi4{WN>ag@1d!6Po`E&DePA#ho>Y*0O3nJ?HhZm&(}%Z`KZ&Ubp$G
zIg=&Z&*X`=@7CM>@o}r%@n66ri92Y^k9AM}+3EE?yOZ-S!r~5>P2{u3GrA8QGQ6W4
z)qS<<ndQ?ZwQjpI=Qyj)v}1g_S-+8ec8&QV&g!F{hZnYft8(6GsWK~vr*_wbmAVri
zCAef+PUt!+5?<q#w{zjH-FbVX??}6?idbT}TK?>*i)#+Ho6I?Oxc<z=6UX{%{(L;b
zYXACVuTH*DC&!WluYJs}$80{MZ_EF;`0{%j+fu#%FH;_$4U9^kJLSep6P*&<pLdG=
z!xUDTT)S6quX*vtq6v@oM43H2TmG~$z3xJ9e794G=9UhnrrEYpcE6u(pZ;~bM)D-5
zU$OZ@w{FaBvfUOj?+Sm}jfL0kx78-C_gxkF+xf|%+|;S8g;6V37zl<<Q94j4S`eEm
z$Q^xd(}6829~$Bcr|(vA=(2mtsxQ7LA=&7~jAJ}&nq^YXBwI-wIy&ch;!MYdhc(os
z^g`QAmMQx3ykE5O`hzrc^Ju=sQ<f?GjZk;sj0&jPB`o_u!BOE=BD>^-4My*;sHa`M
z$XC|pF<0Z)gie<&k4k2Jo<CK0xkFseqq+kgN9KFBIBLv}YW{Rs`v3G4Uni57S8qJY
z*k;-FHoI)fW7(G#X8dpU>+4+t6<8w|?fIyi7gKSudk4c}$-6JyY_wwf6)IJfLnlp|
zt}<tmL()zSQ_k?@8^W(Hshl#Kcxg(qlkCfYon99Xa+Z}wNxXdVWU0;5bz(IeS+%z<
zuytJ7RXuSQN9+^RGf_QFs{5X(en^<Q(#`Pl%p29qRxYsA%=Fr<vSZfCA7!a_MmdYr
z4>K+Q)pd1p%EA8GkH4syU;gQvxVE_N$gh*PrlpGf@jX57(xLRNI~WeQewy46`Y_C4
zC(}AtsS6AEL$2?*)EdBH_aauXr&}yVV#Yomwgi^d8i`8I8yZuuG;=veaB@!GFrn}N
zmliFT#8i!IU8hT)c`dyud@O1q>&qS<OEcEUvo>X0mcPBQ{K!QX=^GcjzRYQt&G9(o
zU;NBOrtD0c&)vw&Q?x>t`CeV+v-^Clzz2nPHwLx*C&@Psn1$I)3hhtQjgET1BW%mQ
z^Rr!EKfACibK%asChpAB@_theF>c$LJ3)QZ?Rx^6Hx5V@ZRlo+Nm?m%(nF8W-NS3n
z1f_CMMOU#xTcJ5_M$;SAcB*(4dN!Q?@?=KHw4$ktQ<lEEp<;DmWnNiX(p`<XZdLI#
zPyg+km+DR4c{^T?i!*t_0|U;_>YN|0?bz%3uCUo%dqTyv)ekQ>t=lop?$x5tv)?+)
zTQ#z-d0n)9?*Hqo@5_(gKKo%(`JUG$&u!jc*GaWJs;RW-n%)0jYa8XR?BkK09`)t5
z1GCeX*dT@GqXskddHB)}i2YGWFw+QPEuPTK#Qe}R*y5pR*fIUdioLoiS<`)E(!2b8
z&$VB_VJz=-F(oQ=rdaBuv(tsPib{RnAU@yXvF5!EM@;+*dy|)KHe7Wk;Dn>0xMz*g
z?->)#!m2yNx3V3#y*9BgT{XMy&Z%pf!fQ{3Z!4O%QRcc%@ytDCTeIc=-njR3+RZb?
z#!K$!TsfiQJ89!2mr1HmeQqjS7U}xznmR>8iBb2<O2g$YxeI(x-?}=>@51sJS)J7A
z<&kx1)3$9@jVrQbd#d(5<hJ+!CNUMp<@T(nE=}#bYP3AkH>|VeieHuIk2Im!xTh|=
zpW93mJLBmWvEO*kX1V31=cD7ke7bdgx17Gsa)t9>FMXc(%ciX6uX;d3)#;mO`NZp%
z*52PRCGl3&MDA+}TC7})*y6>vCb%7ayS_hei$S>5_MPUh0$We^9TE=H+@)Qk%6Qmd
z*;<FV;9{%IE8beKvWk_eUn7z9cgZg&&#UGK7H+#D_3znA-u4~)r1Lh-3%mNrd-sXm
zN&>AMYpwdya<fl3+7)e0lgugNl|J>7D}37NQW3rOo#yd+pJz-?d={7YGg`MKd~cmb
zeQtG^zXjuVW8q18?It?if7)*B|H~Vb?-z6Be|5@DOZ#(^CYS0wG?;E<8~;`{`b6+_
zyN(x*tG8`UH{X)!=x$wp+-O?Xp$&7lJrTe2Xq{I|h1?4JTA#h&Yu_(^w{)T2(|`{z
zU)9{sU2b`wJ?FF8zUw{pdmh*8R-W*;`!s#KjrG;tKY#BPpD_RX>);=rN=(!CEx5Ss
z+b)qC4lMJs_pjlf6CT$8w*8u26T`aWjm9_Hc-v+l;4I^5s(+vG!26l;N4J%Sb#hd@
z7DgX7^x@UKF~e0hJig1$%&3AbulAMy@vQBdSNk~gl60SK_+!3qq1fsjbw<C`?ZpdU
zc&i-v`10bBli%}=rhQ$X`fu@_|0eaT{}*fPNgtTqygKlQ(RA<s4l|YYS!!16Hy>c1
za$xtb#fGa7EIRDqsI}x^)aES-#@5YN7N?h3Dmz*|p6C-bNyE|9rdjCRBOSZX!VcQj
zk~+FlHftqqWVg*$wY)jW-MU{cMmFq%f!)OEQA(_*KkmM}X_?BO-D){gr%l)&b90&D
z1k)3<P0J5${hGM^O!a~HuMTmrtk<7%z+j4vVT*%4i?gxG;n-sJv)bE!XL6;UR<{h{
zs{gb+uhA@5QKM<swkxl@TA5rnMyzQ5q<{OgR8YmfZZF<iPxGEn)7@HJii~IISV~NG
z+E<>fc}7FRt7heGOJ0ddcAd&IR~*uuvujnuVy)L4``>qW1Swj~%{(epJfXe#ziUVF
z(RT^1vv1llG}^0pH}ytuQWxGiiP>_y(oV)Rma|vyoSl3uHN=4-#4RJoEwjWeYs;~W
zEj!t+xUsdk89cUJc7wtG5z~UhvmI|8*srzbaFF{_Z=C{0nPami9;?z>sqJ{|;F9Vb
z9WkCxwT*Mkgw~p9?la>&VIVnoOSbMs#Vsz2CtvScx+8e<Eva=;n|AN+;8xq>p?<{U
zejz7Ih2G(4n^Vl5Up}pyaAwabW6w!8o@cyIPI5UpE#zc&<>9lN9loDB&?L2B`2jAw
zFNf_?6mGsgc|CcB(;nLms}=5A8+2QFbxGMhl=gU3ZQ$dhY5j2T$qzg{rzc-Ja7tp5
z-TTa5>DB)?hRf{#czff<C;JS#Hoo@Qzd5Dv!)>jtcly3(yT-G)ZP;YQ;^J_6!|9wk
zmRj07|2iMjHa?Jb#_fB?u{|jcyJed9xSTn9<@B+V0|#5q9G`L~x5wwy9G|?FGlnvU
zkI!*1-0fhr`s4+ZT_)Ztk{dT)p0n%9lZjPJHm{m}T&%@cwCAjbV-tsB^ButxhKeVg
zcROy{?0Cg%i*BQEScHiJ%OU$plS3<Z-;rF~b;jeJqgv`FkN76enF90Q7j17nbM8%v
ztl5Dx3_g24^k{!Pvd+ZB-_+*(mn%MBQ_g=aIsdKY{8yL5Cxp$;%~nsDeOOSjIkQkY
z<EQ$ZBhI#K^yl3<nRS^f`}BV<mc3@8TlSuB(%`(nVia+t*3#hK;uZPDE^Nvxd1N&w
zep=y~GNZhEd&O<>Q!l!w8}U{%yK+{{3yj%Oo4ljt@IIv=*Y&F`=52KS&}=Upy(pyT
z=#Ss4a+<V#MR;2t9ueTZ=$#{x{pEC?$eGL|ZYOJwX>2`nq$fzb*7E2T9<gknGbzV%
zw(#iBy`;a_&1h+mdaX9s5$*Z6nTr0{@Y?JWQQTGJV|7SM;iwn)^3z@Bw=Y-jS+geF
z_k^U-n#)~BJ15-?IjHebN8G33`6P{HC8zG{SbMH@-0ZDuoaj+LyXShBb;p}+Pu~bX
z-*e(=rle-Yxu->wo(suG-e$2dJmKWL+ok)z>f0RIAcynQWe)#TIUKk9YV6n0Sk^ED
zo@+6pVbfDQW9IJpwekGFpUVGutS+88nXVnc)Vp^bgV*MQz4Nw&u?u-Q*KqgUZt1*a
z%j|uM@$v!>)`iPLc&2;pTd^hJbVSeUmVMk>>z=G!`TDb0O{)F-<USwctIuy-uTkB9
z_l4H4%S~K~8!zrq*qsouX9IVU?}pDGxX*XUP7ey&Q*tK5=gfCaAFZpG`p-s85WP8}
z)JI44=0w+<J+@~~`P`g3*XQt=GbVq|%w+YwIA=Yt&vA)BmCGSQ_O&V&n|C>+p0)Ho
z=sfl8V$&%0wddkK3g0V<lBv0MX`^H0RN=QKe&xGw>8_k~TkHR!=&Fsut$qv%f!@kC
znkiS^KHu^<t*WGQXxY`X_s$%AFlBPo-ZMY8-2Q&#_U@~<_dLD5_v`J1wdZHFtWUXo
z{=(*yD^574S=%zY+GfqYwk{xm^{;b@;`F@Q3omc7aPyk;n1=K9x)dmHFXg_wKjZpY
zg^R+e%{^J{(HkUY&e9Y=>l%0`u)TPPnUbr_uGr8kcPFeo8pIg1_T|*wIhuP5VnzQR
zdHhxQ`rH+quBXCgxJ<U<FFVWg&^Pi(%uR#Po0C${=v=)w`R%1QSA+Dgx;@-`@1yPI
z&!)k~s;%mCgG3oO&*MJME#v-I@@%Pyj?l?S>$R6{p5EX3LwYU8!NU`eAK|>UaPR-)
zM;|V!Ejf5DDPVn<-m<kz@35TM{#B+aMD?D{i5+J{WT#jMWi$r{OkO+HBiblb?aXeq
zDV&?%s)e7O)LHV-qxzvlTjE{M=;b|E&qsRPtdhOH@ywSip+9x5?GsJ<yk<{a>R|=D
zYbti0slA?yI-C#NZni18qv^Ilt8Z_1t-|TA*WzoGIR&OwXU8p;2{(-MDv>?Wts5(R
zSEzI62HVV7i9@GicZF{;v6r^CZ}8pUDQx5MaFxnp`(2lgwAL!N`Ce?xy-|Jol#5=9
zMO=z=S%gvBl?xql))V7ISH-=TjqKU`#P`?}zh_UTeNFki_V)C%_s*Ts=59%vZF>Co
z)!8mZ?k<o1-@NoEQY6QCf#|JhGa)9oE3t03`j*U_xufUyPu~@)&l11+%HKJ++Skum
zF!9bVSzU%`uY+n9n|8W&d)z5}CV4JR@YyrLdujP;e$U?Ax>FSWkjYOb?8Lj4o4aZq
z%GjLGEjgo@m%e{$deye)Re8>8(>yJE4yQhD$`M?lx5N49QvEBJJ+<6ktl>Dqa`ut#
zwMVL_)woXWHT-5<bob7)r*|%j_T;aQSkbe6rAc7x+SY5nahDXP%9eKCTYB_vs%uP)
ztNAPeHk}|zqkEI~ZJ)jF<r<eKBE2i!<TB?UJ2mTA=2Ov>V&=ee9*VDWeR@)}3`(<>
z=4CBCcY4J&9^Iu`Z<fAVS@-|ds%Kemu4b)xmuM*Z`h?77wkg`*{~p`(YIVBnaf_wF
zzwh2Z9A)+YZM@CW*Eg82><OD=_4{)4vg3Y5jy8|oLq4mh%zidg;L`3Bs>W{N8=J2>
zIiCoRwZ5?^N1?<+;Mt_SeQRwT=Xsr;qhWA+VdBGwQt6UrZzZq2DY*CM>OQY)*9@+&
zTY9$d;X2i28-wSz-b@VQxId%Tzhd2U`F-gsW=RkA4nK0cHZ{h1M$EO!uEQ%c%tSn2
zyto&pJ@1|NKhM0<7jN3S&(7>reZ2X$j)8|>IFod8X|0z@o4IM*b+dJ>)fp+50v4^E
zwNLW1>(6LYhq#sQ*D_x`&U90I@}KP$1OF>V@&6x0Y9BM}fB0XQ^3m_Q=ej2^-=_H9
zd$peLssFkU+}l6!yib`f`ZU)kJEZRNB5U{UX;1%#8oMP~T?zSkWshKWjIX2VgTosI
zYkd;p*xqO=%FQ}a7{;7<$42#T<Lwo)>sE$7dvGMG+}ZEUwOcpBa+94pU!76=y?N(@
z!_Nw)J<LB=q|KgwGp*>xw@-TP#d`nKb}xQg^zBv)i_@c?wWmbxpXJFmOZUJ3?5*YW
zV$0*tAME?Qr&YbS?Qqp8{~XQ>vsGR&vUWGu#k^D#Kbf1MyD#Q_-QsUMe5-@Kt{8ar
zu*LG%x_C}8c>1_Uc(3o_4cy(8$!)64y}y$W$;8caySH##=6|{WPoC&LaqKH)a{q93
z?brEwy;140mv59tKmU5|?bo>a{JGOgW-7%A8fkj_yjpkf)#|)&>FlpF(#tZ>zv68F
zmU;iu(z<VE>e+eIgYx1p&tu+YwlrSk<oz9bYnC==?@T+sY<0Hn)rtIj)-b-yuDbJD
zXvuLFHeK7x$6c15eed`tewu(LPxwBI;$wADUahxJmRalF4B2^^Ti{<$+P5?-%gNER
zJOW(b_MZQw|NVzv{M~C}Kd;OEoS^=5viZ*``*o+Xf4+IF;(_(<8<%ot&#pYY=BNJV
zP@DeZx#x=?sa<<)7B)HNU6Qi#;lJnCJ`i)U|FU#{-VJGSx9jiL7k&B3eeC+e61MgK
zUwGQ{lrHsJ5n}hud4_FaK9hQNxccvK{oh;s3u5A*1e$-?{=GEtf9B5RzbD@NT9t5j
zkNDT!&wuX<uU@q7%jaV*S^K|boloI$ui*{<b0q!dvGnKs%Oihojhv%fXwv+8sdb@i
z(h^bkvjR0APgWIrPug|pWqF-@)MfKqF^)N}r!c+NvElwGyywSc1^u*J$FFFwyrt~^
zOfKY;e{N-g-`fZ5{~o62cK!eHnE&$)v&yICxzp@_J`pdzqE}oTX8ky3%e9S`?^`|x
z))l{F|6alN|NWKQR?jQvrrYpuJb&=-=X$w`N2XqDO8fPr|6Oa)mu(Hd^5#}?KHTxD
zvD2QdBQV8dYu5j(%M=cUy^Y%FbYny8>^3%4Ih_-24_w+=mGg2=yn1xBQOUDa<!ILx
z2Q#+7O^<G@3$u;>nzePZ=j1fUIr`$;E>&)Rer~pYI^WXF%^ALn+^1{hW?jj=wA>)~
z*qq4HRiVp*7RU9z3f1siU|Ff<b7F(CvzeUZq8x8G1+N{-_q(QKZ`ZrGqw?h>Ddi9E
z@9nN)|E$aNLoeY_k@-c@m>C-#PNb*=FFP%f`ZO?ED|w#H%+t$sqaE@ka%XXi^DPeS
zf7bHLH0%0?<o(mm9yNP+YggvycS^U-Um5LB<>vRhW2*izb;p#KOYT&D3OYa4yWj8c
zuF{vcPKP}2n`_;`!f`~M!O5=Xe-q1#tIPHM>;C-s;JM^Nzfx+Q2<PX?FZSQJ?-C2@
z*?82I=jtIZUE9i%1&wlZuL?M_SZX|s6mh!Y)uJ9-BGe``%O$8~ZBLV88|ST{VC`*7
zb}Z`B-Sp#8r{1=XMLi~)Je2#4U(HDBvw0`6xZjb<GNs?;Uqs48Tg{mYley$hENB*Q
zo1y5rc*3XaQ}a_fZ>U9AT}w@g?>ZTgni6HEHY;sa=QFXKT|3ie7wpqan^STwGi`2V
z)w1WA!37W17u0h}t(d=9^wiO;&3oOBEn2U8{Keukp1Wk0PL7JwTsA#xmge%=d9&^;
zU67=;YQ?;16CH$BteT~@diAbbT5DE&DI8t1_0s=Y@j`+hrfaX+nziw0YL@7W1Sf9K
zlxrK0t4+~~R-S4SB(0NZ5#Y4-MzNAl0_)bb+wQWthHQ&z^%mV3I#v7f&hVML^L9q;
zJgv9q#i7@F+rPGjPK%UsJ$Y?&-cHr)`x`meNW`_9mTfr5Dc)mnn9I6mLma>N9fPAW
z^H!=YlAOQoV|J(3FT)et->Pj|wDaOI&H{Du<E#Z5{N=NYPVSU3K6}B`JhsRz|N7kG
zb8p|omS6gLC#K@#)^L;Rmut6wx#)X7{C@4fx2xo@?(u4HsoM2QYO6?eNvwLxk{XXk
zH$<W$6mP0;Oj_!hlkbt*uI!bOq-%7jV*B07$GnSs?eEPfzwh+#|Jm)jrT=qx-0$RG
zx4qL_uX8~Y?;_pQ_T21>h-ot`#nhh9u3l&PLcCrt?Rje7KDAewtL9m~UcI~T`Mi`Z
zH=iwR;hR<XcE|o_&vQ<*KGzVRz;wTL<<au0Js(dh@89!j^1fd+Yc4wXuUZ@8y>K;4
zV8+>1>n?leo7Qi+r2g+m5R1osu~`*!|B3C-5>oi{tXqEn_04&XURzSOxLgQ`s(fnC
zuu$m0+AZDMQ7^q1Z!gtjH8RlVo5IC(kagYkj-^dMw7fT6+9W1$RyBd;_ku(=u|plb
z8(R7-7jgu8NSsu7X2@~-%uKKEViIjDjT3LNWH=~Gxx6jlUgda}KX|f6|HL&NYR3N`
zo$#5qY3jtB&f|uBA~Qd`JS%CcFyG84ws-Q;1+nQF_K%ZFdeq#$^$2EuP*koy)I9Is
z8?UUO$qr0kgd15hdiz!`QoeSqdFzK6?)^^=O39tPe4ORU#0m8xPbzO1pKwx3RLH%-
za?__}iE?0&$K)+3s=X~IeX}-Ap8h38jX8b0+C-zGKqYA{`y*|EYjUSXSE)`@n!Hp0
zY|u3GlDoRm%THhbSTud*W#Qxo&D}RxC(k&nqh{28bNTI??*dNSoXwGFmCE$;syMSd
z?em6}XLGzJ1~0Xm_x;dM+xJ(y?03AnXK`8cTxp+h_B#`mYf-IReV_C@*(OfkA)~Rr
ze5rQq&8P>={~xOcR0@_XVAIub;$O9{pz^eOisI=_aw<A}zg9Ohd7p_1*<m2G?L(r>
zS_uzBtBn)ewk%;YmgRFkY?#ruW~tuXvx3R9j?CQjCROiqZ$W|9=2;hY(rsC#0=8e?
zyg+Db_tJz}7pF|~SZXBN>*0Ls%C}%s%Ugwgf(%bWH@97_NHevKdAh1HXzPh@JVg`y
zeN%$oAMet2Iqgu*t@_}Rmxa%@x0C0~$Y|7tTt9R)V}0FQmALQMEQB53c`mC7(6T&o
zN-H*VDi8Nu9%XB*lZ!%1rk-8;%HZ|sxT0;P3Eg*%W>;D}XXcd4&VH6vqrUP!=lNNa
zHYu5J*Svk>lGvPAdvvYim;RRzS!ld{$KSQ99%Y^g;oQDs-6W<ha)<nvyG)4VjnOM%
z+~wckw!QIk?YffZSMRJdD2Yz~ndhi*YGO}K%ObI>8dbq7;slmW-z!qA&uM9V^JiLH
zTkpb!wUUpeF24!rSpHQ%+bK=@#)&EB9wr}7FWE7Bs^vwCusM>aR&*HaR<&EYeLZ(!
zlk*jx;2k2VIkTjMB_Dm6^Knfb&&P@-r{=HP6cHL{yF@-Y$3s$l)3L5ShVqvyl9zfa
z8p%ykiLuPQuJC;83Fl?2CwA6^i<)+x?5h@P^7FI|-Bcv9u4n1A4L*`V3-YF=JTjVP
zu=?h?qHmi|w#=P#DM#IO@y{7~=Y)+P&#}Jre_cr6m6VWd)2Fv?e%iV1{+qNFdo*7-
z{N8rqz|*M<h41=R82y^tCzr=G!Mp4C+^6$6kDe+O?aFLb+j4o%zb(t1=f>9iZCEIF
zCfw!p?KNGSsuGoA^Eu*mUq@V?wM6*$inYG0vL%<9w3n(q@^1FM%q%|ffYNE+qLRru
zQ{Qg6X}|TRpZ=eti-bZCU$tBvH2Kw)Wu~IHGi3N>FHXK1n;B{&XtB8WuSS?l(%Lm`
zCFLE(GVgPl9hf#R7JG6*G~y^P=X%RqtK)iEKI{|Pqt?OwV?Xz~@Mi5l>v__D?7P28
zMSb$74cxN=Zmrm|ZP%4g2@{s)^Em1~Gibi!lN@*X<jFlZr%D(9FWV)oJD0Up>Rzf&
z#p;F2vWn$wY_qwx)&8r@?Y+)=Q)}0YcC%k!TC~rXvroVHx-NZHkj|{%TesGqd7JLA
zabM=z?ClGp&+oExmo4+Jm)-SWyEIU%Lcb$ymhw~EPFL+)al(hH-j(gwxNE=ayU-?s
zClik9r+l)WDaW?)gELQAW9Qt2NfQsI_{@29N`j{{C^9fzm`!=>Qs1|_S+{S0{`Wz0
z^ItjVu6xIH)+xVCU-QeUuhiy4S^u6y{oj?VFV>xCozR)2l%o2%?)yIfPw)GAY7Xqm
z*x=u1G)2H#mA6w@Z^Id*F9*L^>d#;IOK1PSQ}0vGm~D@oaWCI8BVO;h#eb=kiT@A%
zJlDAH--7JV=T9+y`8~b;%i_!R|6bRa{F8oL60-LEy!h`{D_9q-;9bBV_F!qP)LxS*
z<*nkYL|fclSUw+}Ta`G4J$1_Jz2Y8%XV$)yTK9Gqd(;`v$ZP3S<h?m>d#m1SsO90}
zlP>zv=J$8XW=rYwF~_%DO?Z~bez`<?d+uz$R}b31CR}1!_<Z5jZM70_U)~U0G~?~V
z=yxx9HF)nFe%WzYS4Pw?(Z}VXNLu3I+{8nrilRl664rKnloXqya(~K%6G^c%lUo&1
zf6kTOrntUS=45Wzsl@rR(`Fh?Y|MDO@ZE+hXBp4GXL*zzDtvzJe8pqeF3j{&ns!iW
zmRI$Iu&OKfE&jVM`l~BjXxm>HyGYF`O6?TrMpZWlf$NM5N;~f|3SHSMk-#`D_WIgx
z4X%KdD`oxH#NN&g*dL)#^K`D>y;!|v(+}+scZ+S~KDh8frrU!;HlB%2e7R1U3a8ja
zyacYwuyfqzQj*|a<aR10xydt5#P!}8i}set_SV?r7E#gScOR7H1P0G!?`)hQAoMU&
zcHwKmhYE88ZIv$BJd3c~)!s8xwD0AN$}_VgT^+7-oM>Ir@s4-CxKrYL+l5YMOFp<R
z5$<|8*|KwrYS`2*lcpv}KAXb#)l+_UWc;_*Co^;BdrnjF+7<b|G2W;)KBIU3+X>Sa
zF-Xr7O!Ade^4lc;$G7YE|F27OLzM!&k}Nc11F9CWI0!lYnXe_J5UjT}*i7~2)<sNh
zNB2HCy>H5NmaK%tkOy~j6V}x}Wp7jUtd)8FBP#Rf!}^&EHby3&jCp)g;o>IQ=+jR^
zcn&>ydQrjZ+0*k0fqeTGzfint-E{B8SJgzlP(it6or-EUdTLApDGfc29SSMEA&TNb
zYGPr}M1vC3{)KjBJWQ>7CN30{-nA_C(`=2sCnTa`{1RlI>RdY(dDCs(LYYZVrKipD
zxR&_s;WOE)YjZwy{`i=9=Az8myD?dekFo?G6&(nhr@Gwl*^`eZk3OonOBS3e$>XY&
zkyYh-dQtGH+BNxdty^n67*#x50&g@2{a>WywDb|fs^^Xy)pzZ99(+t)Gj1-+-6K9H
zl$<umy52}yow@Yjm!!LnXY~9Y7*xsJYfNaor1qTQ9#1PD-$!wiS#Bmwv$=LHeH59|
zWE%0oC(gX-vFKeN%d#7mw-#7V%V?K8`@DBvf?Ar5l45e*?8Hqk?Asz7<d*F|`&eRO
z^pZ+mw}h5`wkne<AHFkOlJ2$efb7Z%S~7<mRUQYZNX9++l&X?rci*il=2^_k6Bi$P
z?CP62F>-ocy!WOjvpY5COcX0PcJs{CuoEe#ip-YJex$OJ=hljVuHT04fmzSziY+ZT
z*`9atRE3m>&jIz2Q|b%(RxM<Eu)AdX%23r!1}rSI!s}OgDX_AwsE<>Kh+7qrmUZ8j
z=atUmNF^_mRjJ;g$}!WlHp$-JDB6Bz&w|a4w_A9x25|PqozoI8WxA}uE5MYN`0N$m
zyH)X@9<2ZMYVEES+f*I@Pvmg=bx0+0;i25LRI!(Fmf9IkuQ!Li-Xe7WuEhMwfp?Dj
z@@Xwycw1pb)48<VKJ8PvFY^{@%ZXjyyEny$tIOxqoHKW47H-lk+NZfsN~y%?_&y`0
zr6KCeSy&hpf3mQOFfcLbFfcGMG%z_ZGI0E7VC4|Gk+s0FIgd@urouqcxpAEwmxi-j
zLT7U-OVPv=feI@d7h4tm(O94~Gi=)}$u(E=dep*{lI>)U8!R~zD&yKH;t{NNe4ah;
ztDcpw5-%>ao2<2!GvtxQY)g5iP_NPl0fiQ8)#hrIE^i23yv0HE!m7Et)j4hx6<ytQ
zZ*R|jd}yxrcAmTa<&TRlX*cLD-zWN;@6V6vYu$}n*{2D;DctaQlFIdMm8ZKzq=TmF
zZ%!)RDViQM@8*Q3Hc?v~F3)G(kQLlj`k+yg_0QbDJ665Dy@U7Iw4E1~PflcVpC~o^
z%U21;*<x~QmcF#`nfZuCVEPHu?O$Kt*qnVoFZ#-tSGNxS{x8y6Jat*%nk>$M`Ip32
zZ(DO^(e|F3n`(?^3%tI+J6z%IzfG)c3H86Gewkj+EUOXS#G~}$K@-2$3<=Gp6O1CY
z7o>JPY?CTGk?44;^oXKlG)I7vvyj}Di}8C+Peg<Yx{55yHTJsV<&>wxHk(15bIa_~
z{VPkQqgk&iH+ZeRa;VSyj^q^)=clgMpG=Wmx#)6$E9aD{K92%dPAR|SX?ZF@vGnCq
zm)9~=mN{m3i3s_ctUdIk>g|S*ZriO-mM*WWd%1PKXZ6?22lM}h9$~FtZ~J9_QV-XX
z#SHyMU!>!j_L|;IopG)GW%|_AW15*c2V8`*C#$|naNDRW_HyR+x8EL5*_--5<(1=p
z$t|3r+xKmA?O6LIs#_!JqS3ixrL|300@f&)#)d}coqV-9-ceh3vmx)(Hx*GoSwt(I
zOf>J3SiE7nir2iG+pnb-J`E45t6nMnz=3I>n*N?ITqdg>xsH7a7gt)lBOysR?8m}(
zmU@qAOKNj>T<=`J_t%9y(W|Edq#J$J0;G)+?@T-<p}JN2*t!CpOM63gy{~$@%UzkI
z7A<70>(BhPCD`M7)y-uAGAln%IL92jJ9Qe<3Z=f8`pjRagy%idnaki3?p~7j!1m`=
zt!KRl`MmetN<KZWN$khF>&(Zj1G-ZVpZ3h(Vyh9L&a~m~#M`;MGcGLYi}bpmHmg=l
zDx>QA|GPJ{#JplI^{t$?@WCp5Z|}mOm=u+wy+SkRgmj)XoN{mb;tN4K&PLw43<3U|
zwVpaIR+kGtyKMQnyBmGB>jhk1=Pf>M{o`9n-nxf2PQ5SoW8%Tq_iBIbDh?~#yKM5;
zbbFDskLkzjKl^J#D-sW0$*(9lB$*^r<GP}3%AY<vu4xyO)+_Nnb`UfSdF=es_=&-f
zTQg0TBz#@r9$@`!;km3?^ZGY;JzsVD?AZk(2j+ING_2&F>o1@&p>)lm#sgmd)~AA|
zO*!$@zuPPD)`IOCTQiqbD&z>ZJo=^)YRVN9eCYJf7tM-iHwY9|Y!-N+7QAZz6phNZ
z(~9am?M|OX3%uSx_7M30^4i2`&m$eomKPteY|?K2l4ZMCqWNTu*xCgw?%wuGPBBdV
zwE4{o{qAjwe6LiFFU@&+EFyx1TYJ%guEMKLG0~ULzpn@vvYO^tQ9ei9`3uJ(ey*1I
zTZ~J(j;RTY>2?^@&b)U(EYfz{l#X)sU5i(%IPKY=a%7Ugrif&g1)j%j`V`;Jp5bxr
zyG6J~i?S`#e2&RVJC=C%iRf%>b-Z<uYuZNUo*gaM#P`qep84UELedA%ms>0*=oTGm
z<Xdt}Ys&&ou0=<5);?I+xWL@^sjSNcw#cXQ>`iAg^_~?lH%i}8G4i{8AUEjN=?nL-
zS<d3#x~NTU67Q`yth4{!aGj-5a)oob=>H>k<^*YOx|MP%Z?jjqq>61(&;!Tn%n1|r
zyi`2a<2kF#$fxnc8^!CZG@K859J{@tPw}FpcbMD4W6usuijmNAuB$s9^k6DWfAj0O
zA2mgLnhp3J@9xRuuspP<XUgSx_DONdk|!5d>@%sk>%Ho`+NK4bbt=mKt`AjCKA#-W
zrQ+uGFv32)YHmMArh<ovM?~h;=}}kmr)`xvs_W`>G}E<Yv8Ut%*QG7O3o@TC%IrGl
z%DO19`9;opc1NzT6`!{3%VM9&@AF2>V<D%}mc_gmzi^wnuW&ro^<-Jhi?ePUeu@iM
znayVxOuut#r<p?ND*c(iW-q_BVODwF64(2)d6NRR9zOT~o4}TWqrK8kmWy<5-DGxR
z<JtTd8Um5G`bCoiG-j(T+PUpoir1uU;aA%{xR=CCN#87SLWkw_#Q105cE|K|`kva{
zm9nx>B--QLArF>4y(WD98=Le$?#L3b3wk<<>4lo}sb|Nt`vl^7K8Y(ayI*OVn!*)R
zv84Y&_o`!trX7=ysHoL{o)y^3WqvAhzN1B2Xt0%*S*pj<yb~L)E-iUtp>)t&`<3G9
z#tWZp-W2Fb@N(Z<wdbo(?y1$1s!TVQIHc+<3|^Nju$wPTrP%n#!kjeS*9X<^XPfJ;
zT4^WxMNn98>&b<pof(2`Grd2)%{S3Gar4}^Q=5H{oay*@kx$);OK#et|2*@rE}v5z
zJfZ&6X02O`PvuWNw4rf}gqp#{Bc&$l0+P)ondf(HblCJ-*zNd}U8a2|t9$OO`LyWx
zr8PHVZM=_MdiHZ&Wo6FGW_REFYA-{S{yfdB6Q6R~u4zIgi`*+Qp)+v;+rlTkKl)WA
z_msjPx1N;`wI90O=bSxj+Kj0@jvs@vTu$e2Y`!2<xNc#tTBdY(wC~Y1mtGerr7*gv
z`*v;ft(=hg<ZOuA$?c2Iex7pa;0&2p&9N2&cJ{lqwoSc!uTcB#%nOnhCw@h*naTTO
zQfTd}TZbgiZQuH7Yq$I5Zq6?Xn>U>itzC5F`ij;Fg`i#r7Tp6gY>w}~_VSy<hgaUu
z^vWMj{4ZwdR=rF$ETPIcu;s}z+qDz@_m>~EnBUR-@}z*{uRoD1%}$)(e$u9%^HRL1
zW{;9}ijvU9s%rxM>(>?Y?(b{mpZsw1hQs<{tNo+qYL%IJ{EA4P5Hjh2yGY};e!;&2
z7dBKVyvjVf-6T&SgWcec#(J;*!o_`SUi2JWBJ4T4ZD)BigPre#ml;8uf}VUjc0=FW
zETt>V<U@|t%?f_bU#rr8`CcfiJKb``D6>%dTziAu+&NnUWLfJp8r3;+r5%0VJiYsj
zKm6wB%J08wzt^Zq-+FDSt5>4i%X2ZunB_flNX@GTwoiMk=PUE=<?}lgd+*b$yk}ft
z8>-89{hekNu`=Pxvj3~CFJ+znUVgXp&4kP8XaBw1UG?h2eai+-%^C6i{VRovXSDGf
zDa-c>ZAsmb{q<<T)X75Mm|deb@SP2HE%~m)^QGn*V@2q~z{4ty*8~GkeYd>hnCE(o
zPhwIE*Q2`Mp&nhA1aGP2haR_?A71t2oA{5m{9ngppKpm@6W+Ye+-#Ps$L~V_DMi(X
zzSVFptmZ!)64Yj}?4W>`N|@&tfuA3xCB>8V8_Q)AeI#dim#GRLTT(P>S-@HGqDN^0
zA&-kb8nX3^NZguU`EiRNk8nJ<S-iK2hrLP3UNQ3@swH<ss@5NqIAEMT^JsfaLxT0x
zwuNRL_t|q~1rxt8h32X&g($Wd{Z~wy_)Ty&LnO~cbxsf0*bHg@A4X9-tQ5Wm`V}_v
z8~Ow)3$<OfpQ#aOUYPRc0Nd;daqGh>U$oc#EzMHvXn*%PahFHc3OA1&<vRP0YCFa%
zcA<ilX^}Ds%_k!&&n}MKpPbfq*mf(sW}aG)#`6xDgNEOQ3i+3d{A1EzyG3%QL2kyO
zvLhQi?}xSuaOjH7kWD($@N@^)7J=m14M{h*xA?et%$e49N8IjCa);?Q{ldfgInEP&
zj+)tRtz4~ESs7u*^t?S_dcwQqkssAP88ij!CzdpS7TkYL;`la0&x3}N+Y%#FgipBz
z&pyDG=BUmT$h}5E?8yz4r-GR;I2skq#rXekbznMRAA8W!c4v{7BR^9B|K0%pBKB^1
zlZoq=RbD?B|L2gtq+ouLu=zHL=B1`pLJ#YA8cLhpn7U@UpqYk0+al+(i3NKqa&Kq|
zs%}fqJXmJ9(8Y0+*q`N6`j%Y1;_@DgbzgIY&vZ~-`NR2Cka5216#k3+e;@FFZH^OZ
zZ=ZF%&84HV@P~js`_v7IY#b-Eueo@5X%=uyi=6wjUFk(!%L{9<$Smg<{nEnKUWuW*
zp4Pi|$nH@W{@EzRWhj;OLFm>&@3%L+LnB%b9ubr&7npUSGjFG*vtmTFk?3W^p!%B)
zx}GyaFY+@f%-yTN&~cM%eMgr1jwwtNdv<sfJTtG`^WQN2#Abgb!Ja5j(U_GzjN5zS
zJuAJZMQ$ojyYo41`*-Ps9C3LcCh}kOoULN`EmYL=g@pD?7p4WQ=Vn^H+L3%+a86~X
z@d6D2zFmdEm!=0gF0?CCJ2F$c<tG2vh8b%%u&NuedO9xp`NQ$I#l&)#nSWMVS*cE7
z+Ud($*}lMhv3!chyQkJdoe4I(7Au{|6xFO(wP=v!w2Zm7$Z+Dc&~m{QLJ}JTr&b*k
zOb&JFOKm79_ZGet@W@m^edlzgg{@7;3eTv_VP43|QMHV{OM3sxWgf<AEG{+73oY3_
zR4<(rkbAZ8>Pv37pIsg%3;vom)t{2pSRozi?zi1_9)IJ!POauo|DQ~KcXR65jAqtF
z&b5Kl>?#Ca&U8(Do~M5(RmhQRQipWOX3lTMdb*j*w6-tcxtJ?vr5af!%Q4Ga_|__+
ziA&|TPrh(d@SlOeZY$lLT5}b$Lf8@)9Xhr0`E}Ni3B}r6_K(<gcPq9Rd8`Oe@3?zq
z&7E@hdEaK*u3BqX74qJ7)=sr0LW&*&&IP)+?AAoCSTj>ub)sN;__{bDN!f)J8p$fJ
z12d8ZSYux;y(zd_=HdDsf}CuO8+fyl)Y8kpMa^XjT+V)Q1FOqirUlD$4Z@UM%+wDt
zY8;xoVX8EfhVbJMiw7p@3&e$gX37>Y&p&Zvrp$r%cQ5Botm;Wgvye^{Z~U0H;eX(!
zom(w-I=amLC9(0OAhVQE-K|Np6<EDKt`f6a_D4#XrCNZgS|Cb*l}TZ1fPp|z_0}+L
zhEQ#RXl=pB2_ok=s&Pe&UYNO>IdMH>ii(pb|A8oe1H&!b47Ye$W*L30VX|CYv3i2{
zR~w64J-4}L)!$y6Ah~W=*~A&gCvsn!nAIukw8F5At@ExXgG$2MkH#t*!RlNWg@U9V
zqCRY4`?%HU;ZA18T{j$e#oZR1=&&nLVHeYZU6Hp1m;JU{^JoKi`*Oi(e$hgu-x4d1
zotpc`Y{{;TX@Ng&c@%RB(|Xww?M_*%>L?Y+7_e&Z-gM%})QMUJDchR2mPT<+Uuplv
zBKoS*{{NxcHorHY2(#SE=^D6U=fY}fUu~hc+`D&f-1^>HH*mEHzu|JP#%&qd0>5`}
zIcT+r-*FAwLYE_}Tvl$Ywc4KWgv;LJm(GQ%_L$aqp))DXPbSoH?~st$u|TXqG;>Gw
z5kbFMA}@E<Gd~ni6l$?~bZ`Ra>;^&MV5t~YM&ZA=`Ib!Bx7lhd^TS=1IszQ4_IW-$
zyo2!wlj4!<f{e@?w+1L2nYduztJ$nux-!;m2y^a~-r_BqEG-<$6JGVoAf<Vu!j^vK
zED!Tnn;vye+_}S7P{-v#^5!cU(a)Hdaxs~AS*72}Zqlqieqr^dx)<`tW<(m@wm9@V
zZLVqlGtM~Gu<fBAXKDT~6WF~;VA}?P9nMECH?hcE+#0)DB-Z<6Ot$QlH7D(KPL|!;
zu;a>p9>-H5iKn(FZj*huP0eW$lgxJh!~=>~SQkZhJov5?5GDGjX#482nbRe61k@$O
z?$k=goZ&cDH}6=`hvkPD%=CXw_n$p$&uzg5qeP8Mo7T&I6mzy*Zqjp@c_QaVZGnx_
z=kD34y6G^?syP?K&B)fs^1O%TjgG*pnj<`d=YzF3h)MaJ`sTp&A*W!uY~&e1x298u
zg-4kKr|=XWjm$aq>$VC{^pt%C2D_ryWON*8ywNW<cjMDe!OaUpCaVO0Nt<@e+Nn5a
zTE{Helx;ngfA*Y@ZeAbWoa{C&UG38S|6AvWY%G4VyTd0#di$jlx}4=p7O>8GBm7Tl
z^?aVK5vK*F-%)Y7b7VnHX56AvjJCS}7Mx<cxSA>PD$iUImbb#+cCG&Q_!RTUQ|ew9
zQ*}=tk2=j=cKZG01D7`)xNLRse)CjO87D^Z9Y^ojI7FRp?LNVLqPT5Ehpgb)ZGvSR
z8F!vv<fOOf@{$d#@`?Ofhq;0T7~N)XxnOkO^N+x{9D#2&Cj$)53uy`bes$hE`WCa{
zRlAL*1v{p2img0*B2#p?!nulzQnu@Od@Po8=FHtLv3UEQ2(_~%F(+<ZoM&UjXmjgQ
zOv4@hyHyP~o4*}7_D%E7EU80Ref%38H4POFS4o|oqQTJr-$!g(hU~_wlTLfC@F(6|
zZ+(Qx;T)Ud(U8VlMvYeu74NTUx_Zg->ZiR|<DBk$CSDC@IFQmSyXj5bmOTe~k6pVu
zO=<U;kQpU2L!Lg!{~?vyy!cqm!?QEaJh*lJ*QSXwe)Ia1?V^r`ygX&7<SZy&SjO}4
z=&DM!%Z+DKSiRFr?I+2yCcWCZGF$fL?0x)<N1SxntR{*E{<+MQc!9O{RwU1T!$cMC
zT7lSFwJI%v4$i0C(x;p|FYs62a@J$mdx8Jm!nWPJZ&jVGUAuSY-yIM4-&L^ulE9|E
zC+wug@!!wdvwBO{$nKx9>3sGbo!T?zH>Mixel}&-Ug@g`<}6sUPf^P9tU&GmT^&cB
z-YpPc<XIpcAH6^GlwcskhJ%%vvld$>SKoiHE94P3<Iiou`Rim?oVkCw^3}Is{;v$)
z*E*l_TwER2>%G>2?f+eOM!N$)GOiu!x^_1Bf%)=c_PX1qYVDItW(pm9aH21Dw(7Hr
z-s?YqtZm6Y{bhP-|G~sJ$+D_>vz7(7=+|gZVY<2b3fC+F#wdmj{xt$iyw52ry0(7H
zamrl(qubm4pMc9d)y3<cel0eA@b8t|BB6f|xW8TCdRHJ8{6|1H?bSEG_v{;23--z@
zy~<*<d3Nb``})|AiV~HZI%YbRE97acJ;Sr<%8$ftOXC-a9$zYcN27mbch#kL*V-%W
zSdaZ}Kk@(V)S052FZ{T3wPUmCH?5X+)Ae-(ryM?^@%Zk9o*3s?-CxDt8|5-1dp?9X
z-al|kmghD9mM=1X_D7?Fx%MV;zgh6AL$dt)Ev{_Qr%T_g4qq$9|G%9{;o9~*+4LEU
zPwBjsi(PT!<08+*?-pB?`Ok%fxQ2Gz_TqUNXLSFK#GEPH-F9r{O=!{;wAJ@gi9E#I
zc!R0&(Vo?^TOU2TImf`bXD-JswhtN%W#!Uy-d$C?_+;7TtEE{pvb2SHy0^_g_jExb
zKTq{??(BxB%Y011>R%Z?JkS-`ex6Iq@G1Z7<*$t{ruMc7aZVKVKa-+Yx6yRTC70;T
zc_BH6ZLaz|nXL6!<bA>QKi%p$$L9zCKN-u<H_)B`XQFd1+l!rFl#i>eJ33R^{`?Ph
zFQci7Tzt_-!}W!Jo`15-OO?UV|A0dy(=T3^837LuRQ+Yyw79ikss`h%jEm8%sUZhj
z|FOx%MEEy&G;{pDn7sAX0{iN<9#cbZte>~PY;Dxv&Ckz=t}grUa!c6ZRG3q>L*&NK
z7N3+)F5&jIYL%M$*oSeAS?<k4siz$Dn76xy?qFSbqcP>M(wkFLdm{8wcT5m`u+8Vr
zwyekZWG&3~Blgw){pV{j{qkMEa&~UB9`T0{T4QCo3)X~wnsRcgX7STY*EYtStvXmW
z?{|>HY%}gy#=fX1ht;j~gb#(z5Hrm(TKxY<UXhknhIaK<kzaDmVk%c2a@L-8n-^pG
zpzh5Uo}NVp!P82&nkZginUu2s5qmY~lIOFl*9RtDQ+zW$(EDJ$W3kY7*RJjQ)!V$z
zzE58rs@d|TR{m?QWgCl0-qzJ-6MY#rAGwvq&1Z00Yct2rgl?x3I%jk?t+P1k({NBo
zrFr9Z0j@%QVbR&%>-`+0SBUJ5(aBN#C)O3XP|9^pp&Mh#6+=Zi!B>JCrhA+$=;G{g
z3DV}%c>Ktb*YXU{DI<%X?kSq(Jz5@2>BY?34jfp;t$6p>rL)^wat`yF-Z}OqX#Xmf
z1a}5wTP?l~%eEac-f8vUs>3>JL4#6PsGFyz5+lp1^#39%TO7I;R)s~TDg^dPbx!lx
z6yVt~c}v8UsS|%)Wa;v8Smm$b>ny~eaWNvRb!vQC*2|?~Z9)rVqdp(mI&Jl)Q(4;a
zg}xp-iJ~oQZXCCI{brrW)YNOH3G=@e7&7kaDAW75X76j&9dTT~4Y@~k%og&qF39Op
z-L7%n=%mq^9^-P+$z=-;?$ce&d$A<t?fT0h#mtIVSzeScxOD1+&_Z^0CC7%Ay|Nv=
z8LrK%AGDr1(`y=)E3`l?$W4(`ZGT<UtFV~wE-R85J^!tk-tz0sQpKuG4o*CiFTPF}
z;ZW-IPzy8aR1#ub)~v*R`;}mFVB@buRxXwW0?l$WEqME^<6mhm{Xf^5HDKrTI-8`Y
zwtp*(PfRa6wPU?pbi=3P@>3S}a;Z$smQ3YlDOcU5AE$ZdLtdiwln+j2{h=N%f=6Xn
z9NEG4IOUXxaC%MT1)=zjt~x)p-?;zNn{V@3Ve6OfV#b9rpFA#}H7!51<zHR*<c?7P
zJMKC+{qz#{CA>c6eX&G3_{5#lMOwaXg8DLAVG$QyEc^SUPxQ4inaF*6>i4T-u19#}
z@_9Zkxfu(re=lU^baYQr?b0wh8Ck`+=)^fo6`QZ_Q#3XSFmO%%C1`)Cv1pS2lBQpO
zH0`2)iJx+O70ea>g+-%wrsOJ<qf>MzGHIvm=n0#Wys$tw#WC6Obd<tlu^j~y{_DxE
z>5iFEs+4ZYZ@FnAZ}+#N8CkC0aUZ#44$B3vD_(lKP3X_;-G{zNwwzho{6Fig15=sQ
z@)PGYmum0|%$>BM+4fjV?Z>&nCpK_)|7{aVUceRcC(&r)&PRO93lB?+EZVbBfj>L@
z@d>L8#z|WZm#4_T4Au=$YB20c5`D9x)kE&<zDIlYg;%dQF0rS4UgLv{3VOl}LLo`A
z9259$`?;=IWnDWlF@uqdDdPAKJ*9meD|%F|Ke-0IGBKUE<dk}9#&L%i$=qREnpD16
zihN!)=|H4as&(K-cIzDm=G;H|w0Una81ODL=gNJMC;rHx^$3&de%EYgKf##rbtOCY
z7__-w`5!f{Q)&BR?aFhl4!@Q?VLLuu-t*j?^~wV69_G8012a5lT1rgW!LU>>Ak%BZ
z#!0VJzPK<}vUThzzLAsXzHxs}lgO@{ERrRY^(_Sbrx`YK>ar%V8-F>?<G1nf8k^D+
zvs7JpA}=(ZXJca3U9(6teD!59{?IvX8<H0@NlaPzDv-a6#g`$HW0u;HWNDQYM-jn=
z?6$Y2d;H*4;l5cIBC=q@-06yq3uXn$AAdEmRiog|vNtc<f150nTX^rh{C7s~f1g4{
zf&>J^oHJeLupQ!hR{13B&lNdqnU3vo6SJE{zntFC_Ef?sgKf4VPvT}b4*ea+HW+Sp
z>`C9}s?R9W=6b8^;s59i3)YgAt7GaeFXTuUpOQ0|BfcwfMW%pIl;XZ+EVZ5!IJ1Qf
zSJ;>;&ek}4lr4z6Ea77>PvA_g*UX~kJ033L{F&<%p)A~Pb9}|b1}A?_reNnfmTa}+
zj^!DS9=aV#qVCG)e`+bU&F)$x_IE|=ZzV%6o1~WXnF*|ySQr@JuR9ZLUfn0LX7}8x
z0x#~bD_qrX%#NPe5We5a_x$fW(>vc^QW9oz$g2)d;t!wUHI>QYwnv1MCgX%jvTIHz
z-(Z_`sz=mJYu=jUBITapNxa^UQ$!sM&N_vQ2~VD}f?4L_za(G3gw4l#cXOMuo<4i3
z@$;M+n`Ny2KRR2?^jh^*%5#yX|1-~|l%2>AJT<{q^7zfWl@^L_e@=_-QQ3HhMZ|Pn
zsE5ejm>pW{-IjAT#?=cXbzl18T~<}-%vpYgElfqVx^2<%`3n@drrBkQ&Js8r#`%)}
zd5n-^QU~AV4N{uR#g@pmt(h6bxN>jR1*eraE&G)&$}D#~?NZHlGit>q1>r!QRe^<1
zr{wx2Et_$cWzxy5^F4!pUTWF!HH7ifW5&OmwrxM>wv_u}N26AB_~shctr?DH;Z3cM
z_cUkg=AHD8THq^wym09)l`TdgL95>=teddmpOkw2F-w8FE@_*l*@}tpny~PU(^2nE
z*%`qh7E+b&OGW=%t(_x!=;<N0<C-GE7u96cHvR9p`t1tG$6iTMrz4Fs7!sDR^tUy5
zru%bCbLGy|4xa=yW3zW}(nTx%ME1PCzqC`)Z|XWtvHQO-S0%)#xt?}%jQaLz-ZRUC
zQPLgPKApZdM}5iuu2W25*BdVzzR?ey*duqQB2S>ILvhmG$_%HdXB*erE>)Hb_LE-Q
zS6W|j$wB!XS9?pV9xLnSH)2n^nk%+4p2%r@yK<|5v2lvdewWX!8m|)X?LTr>Y;VS_
zhc4<*H-s#f`r2CjG(}q7seQ@CoyJc7ZAGt^c>4Z((RSio_vf6l6E|!1TSSt-?U`d{
zHrrG&MTD!AOYZjXu+4Kb^bhJbOGka=ZuxUabW7Ip!n(bi+G76e3oG4RwDe;6#1~!u
z!e2A0{FC3_Ur>A8?9pU>ZIzn>fyVi&?{=%$TBvp<<ZGqHzCEF_XoKCm8*Q1Pn*JAl
z%=s9Yp~&6Vq_t!V-x@pDrPn`i4?gCxQEX4Fn191<vwJ%gwi!&@cT3@b*Te&>w&ncV
zmh-s#-~utp?uF%#b&fTydGK)BjU5v-9~>y(wea+!rZW-S%d{63{+iZ$c=tXF-fIeL
z<vuCy<2caHDR-k|(FzS=qYYE<8|-ME(7E#BZZq$R-IHvedWt?Nl(yjFoYigF{DL(w
zLnUT)>%!NXYl@thnmcE4N-bZ>9wxx5JEQOOMV2s*C7Mrc9lMy;TxwB|nHjf(@4t2l
z*OEpijt#waFIdYStkd~0)5m~6l1ZO&0qcuTmUasz_bJR&ZsF?r&E2|Np0R*cX_tAC
zv|;i^ZC1tI!rI;ID|WBRuola<UZY^hnJB)i`{4c+%NsS0?w&0qyL*yo<DAf(HQ`g{
z$W4|x^K#?i&xVTDi`1E9k2z0Loo#)h+2~r}9IcZwM@~&PyRd!53U1XP4g+C^3*H^g
zP90J?W{e-^Uascc<Y3G3_}I-^Q_R<FkkoA6nZar;alG}whSlC3ai7}*I+;FRuv1yU
z5hcMIc%o&(rgakoMBF0XKQ=Q(W$!GFXnkp6@b#gcm+-zvo+81!C%Y>;e=*=}<TyO(
z#v=E_Oa8C8JjM2<X>^pkZQ}`l<s*G}PFz&lo}z8|@AcGww}nKSdQLysmK9@|<hU?%
zvFCxVZQ0(A>{EI<zHrY=@f2e^b!5lx3`<8LL$S3F97=9?ok<W9V&UeJQEHvR-Db_3
zUdXD`-DPB<+orgtweY~c1rjYy+)b;e8g7txi81~Y#mlj|z2PO3umQ87l2iBwyMi80
zS4-#S6YG~6u3FpF@m65J#{yG}KXPj#CA}ve*65M{+%UN@XUQDSGha?wzT0qkev^ir
zi^7H(yH1BFthsGEA*3Vz1G}q_sF4&$Taj3uwBW9a2|EjTHWu?F8ZO)9Y`D4DdijYX
zcXD?7giNSBB(QJc?*FYDb_ahE$lSC{&dP7UbWd*4#DfCnF+REnr8y3H^Bvi}M>kMb
z&tuckX7w{2djvA4Uwf$b{K(NmS<F{^xNduPH2kvJ|3Tzi$qXar?iMZK?2wL{pZ&p!
zds~07hhON{Gibh1+-$wWJnX{W^F3SYuZUD>n11;<#rCGt`_8R1CmedcsMT{NU&jTO
zkAjE4JkWa^WPkt0q7S>)Z7-S?94X1<sJVQ{tl!EL9t+G2i|SQ0R6PG;|H%`=3?fHU
zt#ue$%t~+bq(mNI`{NbF!gJ)tCN2}N6NhAZr|2wEkv-kCr_6Ybs_of1D#zJUEG{lG
zZ<6e&xvf(dwf5<OgEytthDfn)IpXs_^izAZ1}nb+L;LJ$Mh302F9V)O1uqTI@BPu?
zagpPf)s!7w^F4oac;7VM;kcp2Ld%e4tF8^d6X&*xiUN~wo@k!1f?fISZRQ!5bDFai
zXQVvgp6A1NeaaEB%>ru<bO{S{i#i*YJUr|B=7`^w*|Mcdd#W5Vj9j0{xXsaNK6GMY
z_?N2(xouiFt|>CBpZU2_{nP}NXd|^kAqnn@0-DU)3&YMiUDIW}dhYb8mWX3d7nrt6
z2;W)YR({&;vUJD71)|$FE%xs9;QW04m;v*@2L@L@v~!)c@IAq_aOS*5k=DuG9P@WM
zRY|aZ$<mxLY4T!^mF`6jlO#{>D_G4N+|he#x|VVGe<e{JvEL3GRzygA;X2--aB9bb
z=`0f!dbctAIK*AG?w%`lK;Y^W-!)TYPjQFzJa6G%_Jm2*;}m}n&-M&)#ZvPbQ@vU`
zPBpIZa+8T_EtpvBGo$je*!qy0jUTpOP!^1yd7U@8)nw~rv)W*Ni4F$c=E#etS02pM
zNLbIqy#Cu|J(ksa9}BM7Ke4rY(jD#8_b}kHO)FQKEuZU_*4xpJOTGrRT1YUyXlTv2
z*`g7XT6^~T7nit&zPFBWi5Bm^r6RUXn&YmHTIAW;@jZqq(sQ;fnDl6lnABmj>#til
zQqJWy9o?lY5bYE8WQWbNJLfd6PSD^!&$mhF(-xZ@r}ik{-P33w{6Aw(?~jsWD?C;h
zZ*+@1bUyG#hbXHr=Wf=P1=G5<TBA8`Kig_wCDH8BAz~F75Y-`KYIRZMtd+^*?#W4k
zB3~ummEQQ5vG?i<R!Ls2t1E6^wYjwX?Y7+qA~cUqO}==ufHN{m^k}KHBdg4soYdKY
zHj0^nI*WPacv2+P-mVch+Aj6UL8jJd`J3+4#2rB#64y=yEqcp+^}?zB4iD0A+OPYw
zX@LauwFx_n_eS+RIZ;<|d1r&Sxx=D42khmp@SWciE%)z)ZlL757biZs-se*83b`5m
zbfWE&v)uYOW3GHS8F2N$^cJx|*%*crLoUbAQ(ZCPOLbI#Cy8!8!YaA@sOi<6+KK<K
z2+YZfnpPzhl=<_jc$A+s<F)8L=aOZ^cD!1BRC=yzrFufowQU)5vi4rHRukCk;kNe%
zPgV4Jn@s=TF&y`I9?O;CZrdfg<0i+71lwr}N({Wbn^Yz|oUy+z@%kYvxyhCHFRQRt
zf1GBtBI3)Ic=niS(pPVuHrlrFN0*n4*~yHP(pO!$w5BZ$;n32EJiEg)t2lB&t<pT3
zI}A2=rt``P|5>&`hF45!&N>fq%@dK_YSZV{ieL8*Qi|nR^Q5~$k(s$Kb^i`WF}IG8
ztafp+X!D74#C)FL+;Qci2*WLJj_3!^SLd{<?whl}v@6i&j=EG#>x-u5FuuUMr*3AP
zRr@$Q{@K+3iJN`HQm-n1d=$FWWwZCjyGLS<&K9^Vc=W*2Ybn2Djy*V;nRIJvu8t_<
z@^u<KHw6T5c09f^H8ZKSYnGeMVe5sZk6i8Jy!blyXl72o{dC#%0>LG9N=0o4cWUs4
zRXn*Sr17)nRO(`-<tti4C78EX-Dum&9<QKubNAG?v-YJWa@&PESJevFXFkw>xP5Aw
zZTH-5Eg3H~x4qu&^dfFuYF=5_e^%k-oeQT-dC|z0Fv<0qV(NjvtUZO@T*tm`Q<?g#
za$}HwV%nA&J<2Z~q(9AO{Fk;u!u-k}?kcevTT5hBc6IMs_v%p6WRZP~rW;7k+Y$BS
zbE4SRE4xIKT36hb-{<t&x9fkeiSng8YcGA;C=z(Kb>-Dl(~Kj|WoxyqZrXNrznxib
zwZK9<F>&Xs@lMm<{7ZWKEH7TMn@w}$YrEttmG5p!Y!423IW>09j%AOf6O)c`UpsX8
zW#pAPF)gwxkB=&UOg=Gr`Sul)GF7EFX}X>9K0kk}W4^BE{iVIr7tGLIu}DI;IsdEa
zhK?)eOcwSgrOtGHsO0*VXI*-om*9k{JyXs^B%e-=X*($#ZWbr^QEb1on&wNcOM$yA
z(gc$nN{)#>PkR^hv~bVnw4)7g7ToO)O!4Ad!oas@A@|8ODHD~hB)DCAwpKVOPk5E`
zidL>Wj(cZTKF{BBW9Gs1<qqN2H-h%ZF1i-*;(wL#CE=G%ZFw2Xb6?+lQLsPYg{6O=
z@pEqLcn&N1j)SG$=|`mluFQUO>|IFdRpHqUyBteg_MhFvZM~<_H(-i&vf|6jhMDKr
zF7p;o7cG3L)Xnv3j^BX==bZMr3O{N}@Y+<?#Zx(H(lyS4=X0)oYMG`veY2TF+~ciZ
z^&aoh+3dC>fOq4LtVuDewbbJ`=XJ^Sbv=I3+wGZJ-^eTD{+VBWf<^yl9>2%xN-q_%
zKJML?x@Go}Ufs<8wLAyU#j^=AB#EWTn&uoi=Xq2uaq^cOiKYVvwFw2Cg@+D3Q+;@#
z>{5=Z_@#TX9rcW%%f!}Qv~zEL*qz<J`YKzGsU&mI3FT*r2mhBo-FruhZ;R$5ra+Db
zFVfAck6hRMp|2W~GTY{`s*ddS15?_#%*wmgsxS*pFl%P76uQv;`A)&KHd&kYu4$#u
z(ldo5PRG37_wKFOuQz&zdHeH>>q_S9=RXmM4e>d*Z2o(RLxMKTCVoGbtg!n1A)(#D
z>ay=s#Q8Gk98pY(us`->Mv)DtMwjGuPrfcT`5k#Y-=eNR;mW?hmFfDm3&&=;^I0~(
zo+=-@zd&q_N^_lHamM=e>mucX>r;2HlGYRuJDzT){`ZDhvqJnmvuP?0lT4o-zPE6n
z`JZ(8QyFYU`w}LeGZj+3kbm~v!n*4^PX&JD=6*ll&vko+MM=txJ+{}M2;TZ%pVlik
zvwQ6mru8iU(#<Oq3V9!=J-F!3A!PCVfWup%4DPlh-f#<lql}*{8xvF2E@irN*uMWb
zL0w?Z^ST$l9efQh&b0d8xHC2O)VtLS)~q%t@y@xr#zXj<y!9!E32xW_2fV4A%)+6^
z#K!eRWx@l8W~N;QQZE7=B-;g*`Q8|95OM7hm**0=u~PYXqr9^mPv)k?6G42TvrJA-
zQr4WVwzFWS(Lpsou3!tFmmZV59raRpinM}7l#aFOM;qn-ny@IiPcWLdN7DG!)fG%0
zsaJQUDo8Tsx>?@2QmUZMxb{Sbq{q~(*&Ae@axLKgy7Jnds_#lV#o^hKMtLkZ1Rorm
zI{W`_;nQ~<c?=h=He)%xr6}~-hLnAk-dTC4Za#e8C}U;4<wh#kbY0Wn&2De2w!YR~
z!CJFcH}!PxtzCj#Z~jVcef3~X==m}aSBLYDIr_h|-JIPM&=I5NwC3M*lM`oee2-eA
z!SM9j<f_bDAx5ds^XJ>2_Q^m0Pp_hHrtGdF4auU*zpqD6y5YA=J2om;NKSL@m0)i7
zwG)gSOaG=GpXhMca_jt-e~}S~O4_CdH8JohT@ezWv~!`5cc-I9lBP_`3%1b9Ta6-I
zl&5VIa2E<&F+J17tn_i;+*Y$-_LvnvgneJF+;*~HP3|Vv$=-g7(o>$>-VFBN%DEvW
z*C%UbRDgq^=ac6Dk4;7IJGj-l%a;1|9=SM6YpZ6%?1FVi1UrlJN|v`SxmY4R-{Pam
zlj$asrWt`@t<%%Je^<H%hNs&eVT+G()MAMA<+TuqOktccEt+GMYa0urmq}Jg*Q}IP
z!Az~LZnI_;7T!oun`n4HIQ34@{bVMKu+_SKj^z=xsZ)}?wcqS)ifE8g+NHC(%JoTf
z_C+D_+#K7NZPMFz)_xV(x?$}$R>?z*+iy9aa@1fbPF+$hP@S6gLvH0&)7P?lYdn7_
zhX`tZjg9@&^-3d{mos&;h_FtfTn@`t_Pf>ZGFw;I?($OI7@g$1W-i0zbE~Ju7=~7@
zaGsfW)ysODMN5J-_q3Gh!T)XSG_ryXzTZ|{t^YPkt-Z0=Y(jI4hBq6h<SVT;9jPlU
zCUh1}b(q+ry=h5KTiFJeiK$vw**7NEPhF@Jr?_xdID6&Y7n_`8wu^l0xzqP8*qyEL
z<gD9qax0fkj@I#%KDa5;`oW#r;>M+S8aL~v<XhLzh@Hq)TRy=l?bPoI7qdlECOW6s
zZFx3ZsdIY3Q|bKUoc^`nB2yMPPIWxBVD5~aA)7b8D-{UkW9*rGtSZ<0b_D;#gPBda
z9)Xvx^Ur*1>6p8v$o-94QqPyo&7Nzgt!vhNeSJlpDch}$>!y3Q$?o3ws!LW;DC}9n
zHWqt<>xY+HO|G|(I+0iE$z}fk{wMW6D!Ko+zwDpQu=AqY+P^QJOSK1ZS{`HUEuM4P
zpkm?6!XuC8qzhE+m{fBtHHA6S=S9$jx`{9KOfAp*o;xg@!FkkHZBO?%mBmiKR63Hx
z9(R08aht^M*2*(oDBuK7NO#$r3%cC5n%Xm7h<yEdCadtJxZ51J`8y^w=<SdaOx+c(
zw%YYbfx^ZUt4*#<>?rAXR%Mvgt#V9j)s^<x*acFqdOKvfex0!Y?j=28hrkkt%1({1
zn-@Dy*s{d-(pt0U6P*~>mn{ADa^i&YC5wMA5|TRok-Le<QrJc9yMWQkC5rMtJ?0o`
zF1LvFQ{zvZE0B7W_2R=PYFbQJ{4EScoCH^CKEExo)=Bfk{|R>lg1Iimh6{vrhizG8
zq4rTJ_)7S;r?z_O(=El4{ut@>Oj%gHP9gByk#swiV@qq!cb$;cU=m&TlQrx~u}^}}
z6Qw7eu35Kc_avsQ^h@(iG4qrPFE}(+s?x99BH)(1-F8v`Uya@|7o|Lk_$&^VZB6fv
z*}%c9xZO{n^jR<O#*PDju6lGfwQ9~^=;3R&fTh{@fDQ8-jx4_y2FqHPh;HOLX}Pyd
zd>2RY!s4Gd;ye<aPtI7Av^IftvW$}R+;2K65>+S8MTCjZUs&MGm#I{nqB7&e2CweG
z)luvJIkBzgZDT%u<=*C&>H5zUO=mg^hkfZ;rTk^8r&7l6s3$5KdlG;BzoOzeOYmmh
zD$%c{jJZq#X%luiShZGgdRDA?Ec^D`c00RFp<sq5m28`i?=$+gewE;*<Z`zieG+<w
zuAfdm5-JMxoVInMWaM3!*@ryMEK__tuf%v<Q#mqy!t%vSQUrY4QmihkKAk!%Fm7JL
zsVQqrUMVhF5gV7MF4o`pRLitiHgsD6Thi94YtHS}{^PLdti;h*uKhdXtYkYB^|D%w
z&Zs)CXZyT5S@){Z?d;|s?<TmMx}&r@&toOCcjN8IgJ-s#-K)L1PDJW=)jNfVHyrlQ
zcXk%soN$1dO?=Xunf^6GD)ZRYHZYt_J>am2yYcK!gTroFvMz>`79Lq6_&e#ACC57c
zCI4mT9umrPV$EKgW~*MZAoy9a-=7ekbibD?y>}b2&3%_DY$EGnEpv(M<^qk3zc0_9
zTQPm6QBA0`R#%|rGO4|<dOS`&Sd!JS#_H)}#qREDm8<5g(=j};gE1qq@XV!x>cyAY
z3Y&zBH0Nl~&<dIMZbJQEZTX)ToO>CAxa!uOP5R3f@o>Xi?d4Com8FBGxNgr1klHwz
z!ASh2XZ)l~YAa2$Y&p{>?GaPw@>UIyICP5d_wqBHvl83FehA!aZMn)<xHW&f@q#ry
zO+3d+IX1TKX`R8+9df0(<Mj%0$(IMbX6aZy`kr;mbOq;Yv7<qG+H>v-=KD#{IWE(2
z<=y11?K7l~oxl2@r{4Uoo#L{s+^bfewhO&_o=JJe0sl?UCx58Ck@GRX?u*Lx?O90&
zB>e=e*h;U5S-;LL%zk<(D^}9=dC|JLN-kdO<~I55`MQ*qOSkpHY4103M)Mow`}8&|
z&kIe7@Sa@J!?QME+wyBhtsNTG2BFKeI%C$kEDTHy6LscsTd_-U<Fwv0Zq^=27f!GW
zIiyug@|^D}qq5m~dR7S2$yX0q&V5|AI#4@weMju^Q)?Sj_F7k{p1Aex)@&OlvwpKR
zi|<}Kcl@@*w(~+SK386<Q`xbRZPP~9>W4QvBF|{~6)x`zUm`zK#G+BmW{I5cvoB2Z
zj-*RXiTYXL?GS$I<bTa23+9~*{C{deltplhuzG9FYWYRCHw%>O&${HhHovwocK%J#
zHA>DVw;R@II=w&h=&#&01*fiGHxhf!h<1CeOWu1Uv$(tAr}fVIo03cFf7fQ!-u--3
zq_p9(M#Gns@<#SQ2|9Nt-}^PYN$8Kwd(FS^@7(k1-+Jz<jI;KKSm}9I2jzEM?(j46
zdCsKS#akvfSK!DerPk{!cM5rVvK;=%)7g1j;)U3<2{9)oIG+4)!Ff%qh=b%|0k+V9
zz>gUmPdR*RU&y5L_)6Heq&m13=fryKICFhM<IOc({l4=<?y$Bkxq00n_RrM^*G>k`
zy&HLx@#xK`GjF+a=E+Q(zxVF#yDXEh@c*qjnHTXt<csUQj?&{#QsjaHT9z!Gw)ASJ
znrhr5-Fdu4eBv)$mUp`GZ4=r&=gLtVKFN+QCNo!`1G8H12t>|keKGN%#*!G8O>W<K
zLN`jDH*s{)zTtSbH*$vPvGtDp8b`f8ykY;7d;EMvh@#f9P}d6&V`O<RF5tc=AO9je
zGWF`$rP56wAAA$Nu{+UwPwMGC2W69<-rn)2t!3%_oiAdUJddj}%@YWG^l8eG-U(3=
zHEgkOl0xRNi7$!h%Dv*(5Yine!*I#7XYQ4tEg~HrH?1U3Ng1(9PnmaeW(&)#nC~W$
zZ)dht>hjh)%)IGyByuJT*QG@VO3ywnIhf(oU##iYsdDk#6b8}6{|UjoEtRq{e|YAs
zj0l()GvRc^q8MA3Ylb0Vg6AGQh>Yhwdf#A)++T&sTkeW^9J<Z9@V4SL!DGC>E@3HO
zT{!JdI2!Oh?wOH(^mOcsKGncJ)!180GWi~vCVC%x@@(mmIW_^=OK&_eJfV5fCBUF%
z!33$y2iIC!+9cmii|aWQ+7Rb)=bS{yVuh<wvX5plyb#h{z-FY_^1UE>hvKQQb1u?*
z#5*GVHMcM(|7mq_Iab2s9afqkzex1c*>1I4D&f2Ce^rfe-ZuN&&giTJ?#VhczRO<w
zagx7i&is9hQydyuid6#gH?TYL%J4F#sur>QEfnw&i}BrY%}thP>KZmtj#J%=|4)VA
zn!&iO<)O%fV-{Ww9X`QVZk=SEIYG2rq^GUNgN0|=;`#&e$=6QTpFFL1jKj6!Vthbg
zhJY-`ue36W=aJJQB_G6mDM=N#d(M05CLdRn;?D*AVqyu47R~o{IB3%ut2D`BgS<}L
z64gElZC#gVo{E{NQA>|{R{PDWK9!Og86|$qC7>bE&uLCo4X3kB&}NbO3|unHExM$i
zvA*BX6x7z!#20Za<ml9tJI*%_HGOHxG>eVn2wuG+d1g#oq?6z76Q@Hph4WfHj2e%e
z4Uz7-!R02?UwXIe+mz+HdLc1wLVsM2teO)Y%N6lW_Tlcr#`sgWzu!Hk#U$`caDttX
zi};@Zi*~PAT%6}~_2DARSxbVyD8=76VsNj;uVTsn$}}UVjC7va$)~z@cq;o89FYn;
zWqR$fg93;C&vR@W7UV5@e4vo+)(VAlOH>>Bo?X&S=P7Kcnaaeu$7SOLL2h66D~-vu
zD;{)j5VQ3<6B*`iv{R<^L}Z-F%N?03=0s`j-#eATBYpY=*_*m2Rd}xD>TupZJ6q*X
z<iVIHl{p7)+l1XMbVzBOyH%!lcB=dR%HApGlpf}`1O@QTvyiJk_k4rggH0c%y|HLr
z)Y><-tB)bzKwTB*v#ov&Ed3X9*k7t<d;Ga(qWAFam0MvotdTj4CwW>n-E`7C(Y9{g
z>rDavie+ch4K9D(@}E&**TL^MUMJ4*Pk6yqwNYF{*UfC<iiLTBol{n<pV_tSnu1Q1
zgz&wrRIlSMDIC6cqMJG&B^aKZQZfJ7qD46?lD6{*v>Hx1I5lB$ZiI$mkmrIo&swfE
zyST8mw(py;<YC}4*QGO<Z0`7ekryv{mYyXu&*kXG(2#bA=TqXmTFn$fcPyK*^W_CA
z@$@Y-4?PtBb%OQEw`CDm&K%X8Izf?l>eD+HPF!cT(wgG9YOm6T%_(l3Ilet5EQ^kv
z%CqyU6k2;HDXF@q(<v(b*1|=uUGqiu+!eUTwkGR|KMQ+qgy5=)Y$ARxRns_U8q8aI
zcF8Q4yk%Dt!dP6FCuT0c7I`NsqoL@3_sSi&A|1RMjgG#sVr@=4){xUQecj6mYv-J7
z)BId}%yGig6?-~*nUZ2l`cE&N{E<aAWsA4uMw!(L$4VOfrS83U-6H;V2BTWu?Y2*?
z><3f4e<&Z+)seAv>4|!O|6unn&KZ+9BqmQd*gr3+Y+lljUzfKuteN;pI;qTMlHSX8
zZJ8<8PJcPJ$cb}pMS$kNSNa_eF==*pZtZcaesT5s6S?3eFV6Hl3pC9=qBCV-N2a&W
zsWrKdQ8y0H%6rT9=jBAtBNtS=K03Ihl|&w!crk}<E?d?_!N}f^f=Oi$+qwDA#{NC}
z#`3~Xo4F-_=knezm{asL_mt+cZydXmjyuR5nJjp6Iotnh&PnRG3!f*3K3Ub*7i-1-
zqC_RtvSqE^8>NX$Tnes<3Ex})D{D!h;%TwfHEB{QZpxn9#0;&p*#jDv>vackP5yO)
zHD-%TlUpm(q(gnYEd?`=NVKr7p13+>mUgI5;~UN8>r>gQWRD$vm}k7zW8=m-@pB}r
zK5C0UNfcST;tEH!sjRmO7q`;v0;St(n&obX@_HV|J?ark=`_{%S@!AhBF@-Vm+sUi
zZT+Kq{FVQ{pTgR0YdmG#lcZ#h?0dIKB}!X+(%*@lXB=3knz^+8^GLt<#_jH!v_65X
z4>?YA{nO@Lp2Rp=$@Tc1XRd2EI_a)>>AdCe1Yd^NH8&N@Pb>;z>vs6Bwa4M&BD1U0
z9VP9WbN4HFb13_s<8_lZ2;T2-w7%6%nQK9BhV1j!&&6>wx*U!_WfYr~lA~#B=$CUq
zB*$UF0WJ6VX%bO4F4jDmQFA<CO_!Eu;w+1}rMZbdQ4hWB*9CK|w$i+nI*naZvArZO
z&wIxlo{!=!aYuDCwcV%W&CGI(NIAA+-MbJk1z*j}x=&<wap`n_SsgV?V_(zdk1WUb
z=538SaAF&$uZr2x4+?8OP5pEyNWUY~t;cEp;Z0wI_i)GD*4TCSa(+p?<|d^bXTENc
zNY;NJ-O77JyX?5*M(#xO36a`sq6~|<)e4N&H*8dDVY|7$)6>{pw#Tqc&Fr_%b*KNc
zeD({hE-47E@NmgEb5XDR%$q4J`9~#=wAfa3O~`Fx;o-KHZcB*hnZmp3nvi~b567aS
zo7Feg8BXN7VxncY?22or>14w<s{b}PJy#1}^K~|-!qT`5f$rOd@<*Zt`KlSX%JUCu
zzGX>yn-ZJA;j-QL?$K2XKeD6>78+>#r1?&HS`g#4o^Qjnjd2$5&Rkm3{LcJ@f%nt0
zz9%0~O5J90I4b6E<ejO%J-8=sX4AcV&1s%B*^vyNj^`X?=RUC0=<7xswv8O8%SC6K
z|7?{_xq9W#v*=}A9eXas=$`t#Z}p!nw$FF-#ScA_w#;$dlwT+NvfW4^=KUi#zcv0X
zto`n9tOfr+bN^ZDvE_Sfs<ENHKuXs7lwCz<bj<#D>A3IsX!1CxweZ=wD3^3&*K}vy
zV3DPFn0i8|6njo=V)<~(=C|uH!{re%!f^qQ7S6Qji+%5;)4NDO)LkX3SgiMNs5!r}
z$-?fFC*Gxa{J(OdD0YF%>owc*pJ}C~FL3cZefm+=?PrNnnU;D_wjG(i{_-pdm(6?@
z^KG=xS8ZvXVEOyZyQ8_;Q9Be17P{7RA80glmDf+$e}SvH?TFsQMxg_8VY=J+)3av0
zt6VpQtL}@`_Lhp!)2$m0hA(v99`%@m*VFp$G@~eA_CCJMU$3Hn?Nnk~dwT!LN3Z7@
zEqc%$<@)VkQ01RV3x0Y2zmS?YWzY0qU0Wl>6_V`Dk9NK~S0|FtRPMIMz94YBOT&x8
zkX_%RSL`fG)W2227<;#3zqLn?(dPVZkK-%mo|jY?>&^YxC45zdZMoJL&ShUCBCRub
zq%J*G82P!?Sm{)+^K_QdSPzq$8yk+_m|O17-RZvSyZfuTskUXxo+rX$*;6)c5p83s
z*mGq1eZwWSlQwZ}t$%5AZrY@k+YML!J}#Kq{(R+*u$rFhvyVt!x}{U;8o^?kdx7=T
z%qc7{N(`o+-O803B60a^KtRtOZ<jrepXX?NisE0F!m{Q`=IO^4zuru5Wl8A}Jg;?p
z<`>nj-#1>wN$=dg{``qoqE0*SEA86)e9r%kV!>(2-Cd_P1r{$j#+2}^^X2mG8js^P
z@Lez{@UY<UI#RW8vP`JPY<|xJ;ZwG~S#&<+&$bDb0T<SUmaYl!{rK&@9Lpl1H_|hl
zB3ZM(pV_=wxpmzP#@m~N--Q1^k*ptc(W7q$TU$kkzR{8gMJ1_P?jG#FPFW?EnmlXo
zYG1VRiC0nL<Ettc|7~CKnbXfod&@S%=Y?)Mm6{G4gbuhlI5IBg;tJ`wkZ`DtpLK%K
z4$dT3#uSz<lQIO7ougT!w@%#D>Lc0B<YT6jeCe#Sp4!_5I>oC)4;o3tb7cfCTRMAc
z49kv9mrqU8HjlX@A$s9cpYL?F_cL5pU3qY2ns)5YC`pn3sR8D(L1J%`3fA11oOaPp
zr+QlM;jOv%!&14oDBRt~^l0AGfQ`;a_f%-#Dm^W>CStOhH^W_}gi~$Df_4-rZf@m%
zsyTIM%(XK&9!xn<^qOmd(gW4IMoov?_U@kbHP%nvI_`<Z@6s&Bt<NV-la<PNqA5Iu
zL0qnetNrwn+gr?Td`g@tw7PMc4y$2Vbd$h(`!jJ97cz9^tQNl$viaKKsI_W7fxgcw
z4?nN=+^oaA|3S&xrB&Osn&bD^nw$0>I@SAf+N)Q6K^_L4+MBOQd29ak(m336Sk9=c
zTj)>m;pO{w6=kjA_y6Us`t)2-@0xG7j4o;l$v+CP6#x4wK}YoDFCCNRC;lrqn(^&;
zkg)OovIz=1nnVN~I@?4VdW6qRTe9Zli5_L$Q*yS#4el38P8XeQisQbhAbk4PC$na&
zmyfm@vnov3s>k}`h=S?Wl+C)P(oLUzx0!W$=;>+v$nE6($6~a17fbTS1!h)G7Z=?)
z<kh$Jom5kaY15}Y-Aj73dfzHHM`gNp@!r_wxzy-gf$F{>JFi!Z%{sU`yUlc3uJ%nm
zwc+s8xqKy>D;J;2csgyqm5#=td8aCz4^KClvPxn84DK@5Eh4FFd-l7ioLMh*ev@~!
z4)<9x@9r0i&+Gk?QYt^6!6>Y7>$1e-1g3KCSq_Yx&wL*!R!eg+Ec&80o5%R95%-PF
zTJQhA^4)&VZ}9~)XURMdzvde?yo=j+uJL$uQ>82SQB$ndj=T1v0(wtWZH(7N3r*h5
zdACa<_eN^7R%*qRuv<4pmiSp}pIl((<>_O(wn%Ej_5YWibn!&oRcL1Zy^Cp<>RhfE
z&lgLVF`rtYcV@w<C7a6b?pnQJTdT)I{rG9y-fX@Y=66IW?M~O;yH#&CF1Wvh;oJgc
zZZC&?#y7r)6!!NYnZ9=7v>GF$Q+(@v8jdO$S-fdWf70UgR7tMb%3NQ4bB5XS*L#$#
zE-(9IrniFO$`^;rvmF??&XtMmI-0!t<f1T%TY8t;M7nM&-3`CIV&Mm=Rh>(g7;Q3r
z`DBLiN;SPrS%+Uusr|2;sv14bT}pLP!`2Cw|JUrxS~;c0D*I`A<`<*2S>ev}6Tkkq
z+PQG@U2~DUhy1*GP90NY2>r^*lE1pXLuG=#Qm9IU(2S=29!`>%7<dwjU-$Hu7|*(N
z<47$xx9s6VOnVYCdvbOh^mU9XTdXs;MQ^5LU%@noMpcgqy33k4LKn0N$@Z>LG`{j6
zM{mZqz9*S!Og}s-Ew;95D)?7@yLrA|=o;7in~qgpE&^^kM}&+@92|YEy*QbkEaDVh
zqQ6RWeiXyyed`!n-L9-UAzrLu-@Jy$t)}z7^DW{1hi#s#Ssmi&uX!@#il$-%lg6a`
zT#Hz0e!i_w-za{MWkL%JgOF#>n*SY4P0o_)iE4$>1~>bcxNP`-vNb8$sx$YL6PNMb
zWjcAUg1lC3vz&f&rCoUwhj#ZhuSmO=@;%kPE;Y(B_DlUH>v`YW>>)A#ng^GXO$&#%
zO3=#JCC{zXZ!Rq7b873op`*O^#v!qQ(5+8yH1X_aN(>TUJ-1Tlbi10Ob3@a%IVM)!
zrjMKFpWbD<HhZgEt&)vo?Jp)J!A&dbj#ggJXHn3v`ub7PbrXkUc=LrVnh$4j{F#1$
z<=cVH2IrC|8J@j<W2awt$t2%Bw^tl(N)@^=^T=Y^;yVX@B<4(DRc}>ioa^fKaM#^8
zsWnRkeaiAyT%2^TPw{{<>*^JezIq;vnXXT|=Kk+lWISJ`BUofky4z*ZtNRyhw3OQV
zDwO^30s|&BRl!{+eGP(+?|2#*z~qoIk?FYkwI97}%cHZEw3$x)JNZ@RThM{jX<In;
zY%&ce#oUQs%_84?^Go!+8jq(8!eJ{msrly}in(Yb9(m%}q)o+#R=J!Lk&Md|TcE$X
zk9BA1ie-o1q%v7@8|5Cqao_iHJClaLZi~e3H8+w}1AKyioTxCXT-NXU^~g1=`y9tM
z|M;et7<eU`@AITbI_1p56Q;lV^TM6;!RZB`4`m(ZSaqSv=ey1yuLsWJ5`j%$Hz|nR
zotCM0>fmcJ@%H&PI@b0#rvAv&Y2J15!M0@vt@ay3rWa?4xy)Gnf3sv-_WDJ(-dXEU
zooNnuu9$LzMLLUZV%P*nuB(g_JS{|~>FxNuayHA_g|{R^rB?Cg9^nn!`smTtTCVI~
zBgID7k2_k=dO3Qqxh_#=UwQ4CLXl}}3a?V-9|8A!s*|cyFFg_|m-Mb`^V+32qmt8f
zSJ%O|9A2f<hphtJuKT2)j#~F-Zk$)7=~K>WssHZG^qjKXSTenT&ex^lt6yE|4sl<`
z=4eo!v2uCRBT1#ROEMe<4moE=EfGt(aX?Hav}?_u^1ZEBSL~E3^Kb4reevDNS6R0{
zPdif5$X#0)KL4AzQK0jY1s6jd6us|Cz132`nxnbE(QEIkwiCY;g;snh7v%_K`(Ko{
zi0$R-<9bSxJ$~<2$+X_sxI8DKUGesYWB+7NTV30+>rdL!^Qtplg?!JY@vhjbE@_Zy
zB9S-OZk6I~Nmj`Yr9`gcMu-15PfBiFl6Z}0+r_WX-flj|G^IxSl|g)Kv>daSfz<4+
z>zucJkL3Ah-unElqGMji!SBl+H_dZlnsRY<+}~T_Xa4Ne3Saa+OYWMgkw>0+cWXy-
z|J<D~JzO?z$Vr`{TYdL>z{`?7DG~l9*QBIQzRTGa+~GLkq(SngX;rKnSvA+)e5)TM
z>T0qmYs-73W1{olZ-1{7o#Z!V+WOdxogXGOILMzAYgS9xC3?JrWzvP?DlQIOOcE9h
z#W!9}*Hrk=uy$JY+S)bx9V!bKPEt8sSR!k{?$oj<;2B3%q06>r``C|#WqYO`z4jyB
zzqVxw=eei8$2YebSlw0-)O{RYlfJEE-oYCiIShAxKGU7k#Pe35;dQh6B;6FA78y-W
z&2NV{e`|PqbkoBhu4b3mVz|tYs3%H`>jq8Z6Im5?M&C~2anZtCt5u=`qKaPKI(MA0
zYi_>h1twRsg{>NYbeTLl97M!kertA7<Vsdx)piqPY}w$f%;-}ZtvjjU|J|imIRhFT
z*|ky{RF3U9t-zALP9pE&1Q$=GW^RLXCsZ=MR3;?Iu`3B=2QFFrB!O*e%u79^<qDrv
zSUj_*T{+_Gv4Qz)j_Lm^YqU(bOPQW3%B1GVHAOUdGB4D%tmt#i>J$m!XOUmT{&7v2
z5|g6aLYt;lVk(as`qVU>9?g!?(3lr6BkxdP9hb_S<1QK8ugo}=G~T{C@YJ|z^{O}N
z2@j4cd}H_QI1<OQ+~eIk#S`ncMkw;JHoWCa;$GsQ@Z`L8*21?w-J&K7#cnl<b~tgd
zY3}CW<9*n7h3C=LZRf9?=zHVzEO?r>(Xt8ZB5Tz;4rwe~;E~bm!n49i=9E!M?{$xh
zN_D=54jwm-Jv;2TXtLk2oA<PDTD@GXGr{$dHc##=a|3@)o5Pd5PVAFn`u^a^9FFx0
z3U9x!NxpNdiR1LzgB^?H<Q4^QN%;0Zi%}|ez1*z^;S0T0KHVZEDux%<O64w;wMv!A
zJSO(aan=Pv;jhWu&h51i&Ss~nEYV|HQ|6Mlp<nRwDuoaH>wauhYIvjaA#GyBLb)6H
zJg*XY{sk~_6^Yz9$bUMSTj-PG^8%GDwV4jBx)PjgA9(RD?taMk;s3w$*VGo|NgWnH
z`5?n!qq?7xTFfQU6AO67-!bs6yP}pbNhj~|gg4i9PR?`V5Z5m-bMrg;=nRYeB9?<W
zn^aC~sJ@;W@If<OGk1rMcxh(9g4Z5)OW!&q8_KXNPq-+u-J$ZF6U%X<a-Jr|3O#no
zNg~r;22NuBcf&!cYs1#M6I)fBxQ|R!jBe$*U^MgpBxYqVhKsKbihC?Hi0>~uSE#_k
z^z94t7X|+#8(MY<DPPukWTLb<wrg=e%c`qi_^ur~#~XTff=Yr&jw08&^<rTQ6}T6P
zxh3-4SSWRLy_{R4=#504Ee)b(jUraA=36%j^JX@j(i4(p7KwkF@qyVs?nJ<j%xfGO
zg0{xI+h0#`__+H}ccW3W$jR61zhxR_8MN@N+is;}u5n)FQMXO*1=SM6QZ>B^en|;m
zPVmSx?UG7T(Q5kQcgQZ%<={JGiRkxV<(g!~pG&enFnzsH%^)c_Ynkl)W}an-<!(JV
z_~zjYwzYDPjFgQIiM~<HjB#<B=k=2($$4wv(M>7eUvQ|e$=CnyCA2H*?86&PuGfoA
zW3G67YJ2}Ck$Vrnl;>tM&qkg_4ocAnO(Gi)op2CPJ(OQo!hPY~TCcYZFQ5LeczL0s
zC9J|pw^vV%=ct#ild!+i!yQw2&uXd5PcTt3V4A7pwWft>v5$9Hv#8^R(@T~oH6$%J
zS?>4cjmilgEn9)JM|k`nw*=ha@rZeLHN)4=JLu4>;&QhQe%gn&)O;wbTB~U0sBlZ+
z!k)!yD>_Bb9e&Fa$gp6en97vG2kT|FFX9n+tHl1+=Gfw`73Vk-^<3qi_X<2N=s6r~
z=3xB%=q{;`)>9Av7rJmQr&06<L-1Ay?hBKYCN=OJIsbdhC66sdC$1PUs5>zIZ=0+V
zw}_|WnSI+LrI>?hU58#Q`k=BxB!)XlqhO-TFFs?wV^-$M*3wL;b{KhgY-GFslIi8t
zn+b(Cvkc>tH{N{GeN)QG*vFmm=H{F@-brGIOWz3F@6J_t!xI>pVqLaHZGoYbxg&D~
zqpYNn{?eEFs!GnhO)GO6!($rNf-Z%<x)?Ht<Mo|FIm1V}N-k;vO72a%$G&KIu&rz0
zILQ!w#k~HE{Nt_@+pes=^lo|CmbVNmRVIItU+Jxq+pFR~$=$=jDq^YbhAT=Nwq)IG
zx_IoW!n!g`9hXP%4jtcf<?Xfyk7NoSU-)*t>Y{tY+olfQUMnyEGjBFVtn>ZSqBGSf
zKl*C%f4>h;zfH;iye(;33wMs5N#o}FCw%cL3JhB70<{m7hX|->wFc@Q+A?F~`5ljS
z%SA5ed{tbL&a*^8|M3Q~EXN;F-sam&dy5W<c{TD(c;stp&@9KkbKeq1Lu0Rf@50VM
zy29jPS8AGDlk;Ausa$cRN-Kwm_oED*i>)ckHSHFKgdI-dJ`@r#O`Bnf(wzfdo6D;f
zZ*<^Uq`AY1A!jM?6TXEa-;J|*jS9{dUYV+QW9tTq_Sgk)%dek2;&#UR?C0)Jt2eiO
zcWw1ac$~$p@x5tPa>Mz{g}0k@e{NPw$TvM%D6Fy|+4zu{Q&}<ZAq_9Zn4Cw_bCWNf
zXm<%rTC;6I8n@~1|8L&w-N@rUc3$Czp5A0J!P`!pUcT=te0J}+vUXmvY1fheWfGHK
zv`+u<LN&?Db<4E6%NZI^iaq7urRz5Jdnm1!Sw3k|%E_LMK6fX2zeo~OK0o=IQg^Ac
zUrf3GYcs9)PZGo4X}rDW*E30T{~5n1zx>HXJ^7FQrJpf~UQ}Rc$X8h~Q{t1}4F(Qn
z@e3)>6jVfm4x1{sgeXNl)qlTenTAK*-J`GPafjP~j&yHw6;@#CyE<!<1J|u{ll8B4
zOt+|b|0A;7Tw6brStLQV;+SaB2F*N2h6CFbSD4qxZ(k|#mUF+U<<9TBwv}>Qd=zM%
zX?b4BQ&Ow(k7}&@$J*|Xv5!0D{!1>4ncAzHV5sr;>#Wp^aT_aViNB~#o{}iE;Lw)G
z<xh(ylur>^_3>*1M=+Cys;`s#i{v)FEXVY+7HQ@M6Q=d|u}tLiWR&fE`@?8r)DGF4
zDSW+QTv3x{YZ65kCFn&v3g1{DC&IO`t!LsF{`SA&Zr^YAAG_9HQL(^B$V+vx%9_y5
zDJvY_H1Mc7lx-;cd?hnGJo)yHX}VFrRJwK~ZsS!;6?&BQN%?G+%9&|OKfKgWcN;H}
zjNe(A?QyQ@f_h%GX4<l%GoRk{%-)nCw|c7F>)x$WecONaG@muRb?5=th5Anx`TH(8
z{d}XcD_OZoukp4}v0{=42U8LMqT<&=6OZmR`tSUsF2gj1$*aK1c=7um0>9I^ZyglZ
zVB$RH!0;zgSYVf!L8(Z`RMTijGxvky5}V{6oEPhvmY6c>(Y()*&t0SJS!YfakLFdH
z#;r7Me)#m;4os&WL}%Yz8m$~tF1faF_MCmw=fq@8(`BmK`F-2gv#P0)OC7DPO0TUs
zS8m6D;yuIrokzVFd0hMu6uACr=#=ZnbLJ@RbK|iQ$(S+wa=LEQl%H)Y8cOG=v)YJ#
z)9Wx;uJve+wzN{q`8P+d7|JZpG`DI$ZvFPq`KxlA`AQDLEC=MW9EFt}6c&8s_e|W>
zJwbF)f@0)Bxh$rC0-wH5dho6$*kciQzDrTuj~B0<Zi#<?-0^?U_iecgmu(g8+{tis
z<L!?8rPKG#><sQ*C>OAv@7VflX2*Ls=0v4U$$MhNUhdQ2B<;KSw3a&S`bvq{hka+t
z6n|6pJ$2yZW~o!r?@so<I(53?X@IBJi${z83qE9<oiD~IdH8CBV1=o~)v^cux?<8n
zs)tJ0m4cd=&zcpmcye{av1#EK1=8IQiJ2+3ia%c@=~=jUz1j^&hJOWmf<-(N9JpUN
zDtr@A{A-{%fm8pyRNH^|E%UW4mV7nxO6K~h9DQ{9H1pJL3*Dw~{o2%gh_7q$^x|qZ
zMWq#S%a*0z++Myraf3+B%w&xhhXrgmDtt**xfVY2`c@IkHod5A+IMbuY_+)gf5NR<
zn=Nnsyn1sozd~`x{?}_HO|`5=zUhePq<5{{bUm`@;=4cR&QIRozu=h`6N>`ZH9f8^
z4hMXLg^aGriCnT@+i6-B!IhE5bBj@c_xD1<NC8vD2hoh2($|}sRrZT+JUrX%NW6|}
zWNT=I(6aPfk5Yf-<Q%og`}JsV`IXGwN?E&>b?<l*e`{H0)RUv(8U{^!gik784Vb8r
z$93vkj<(mN77y!#8$LyCbjdoRH|?+AB+bB=CFk4j<u7$Vbz9}s)Jvy!&RRWpU-6=S
z8EyYo)c!hmQ;Bg=g2!zoUH^XF0R1=nZTA)jta}+$rf@qrZM~=n(@eWVlP~6RZ(%6e
zo{$!0%=KSnr^D@avK1HPqCVd*E#|&q8~%CyTbovqA9h?1_@}rh9^_7n2#w&<iAwJ_
zPw|*-q5UyxXINRvUp=3&9l1M|7Hn6&a;Y__ipk-<P7>1&@7BnD*RO0pdo6e8<LRwO
z-#cy7k5Ea<6|6neS8dAhW~s{Z%eBn;2}^dz?pN)(dD1#srl{_<;dQ5bafj;;@4mM0
zPkprV+qk=5cHHkbUA4T)Q0atng^3gQk^_Hh7RsKPFY+eQAu^cz-(IuWY*Cg5xg!jU
z0(VShZMaOLxfz<yXoP(|;Ss2yu+XWMgH^+Gg~G{cx}4@RQ<M@Hy0vhLcpT73JR~wv
zX2%qrV6LUoy<E9B6aRA>FFP~SBs;6*s+*JKY_8awg-@4-NHgX}{ngodiFYP%O5CR!
zF4HU@^R;FQJ`V~Kn^gRF>V+63a!>b-(NGi7PH2wU7|fFM@WQv;=xH%51*^DAm%Hs@
z`k|%LJl)4BsEYN4@5&84>WoX1<hGrTD135MQ*-)D5y|3H$23hB_ni6pIqkv{cjZ2x
ztW6D9jE&D7IrUlj%0e!8&#NbPl{tm^v#N>Q(fY1-kSm(g;NGgxDQlx&ZFCY!oNOPI
z%yQ*YNcR*G-+!+-6H4b!ttk6vvN6frZL)rtNb~Jea|O1!{q&UD<fFj2zlk%*!|&lC
zSILwufo=gIve$$Cx9ylTLD$gm#DAe&CI^iJUOYiF1X?B3c3kkXiH&$DyVmEybtikx
zt-+G1Hv+lDg$ooI*o7wqbV&>61RT?K@N~G78(^t=K-Rc&hQI`)pHC)7a;84HmA#fz
zp;1V9YH0Y>|FI{JPGvfyI(bs4)ieoz)pJ`U{8O_^o;hV++LGqfUCJmqC)erOGN<Bu
zKSRTKesTyG@8LYaHecuK4uOTH2h}I%a<Q%o$Xc=`$X{1VRh?0Y(<6~PU+Ai_n^4Y`
z&92sKR2I4}Sv<j!%XMo5`zp1QTMwO*2;?Yur6zgOeX<bOgX{72!7E&uBoB$Xh6Wih
z{+u+?NOiTqL|Lw?+urBwUK|rK(E1%LnLp9-zf+HC-<HYRsS&3ih&bO2yei`2XFYk7
ztK4sqO<as8^V(7szFctfe4uq8%;4(s2{rP$j$)Cub6uZCMekx&-jd~X%(GupYFS`n
zYvardM|Udj3P_%&dL%N9sif%U3Dxp2sbsBonOkX#*76je)`*n}cH=PBbYQS~duu(*
z>@z8MeAZ}qtzJ-H=WgO{f0!jE%X{+2<;{+pFLjBW*{Yn$68&S7r^w<PkFv5ndmK$Q
zm$+;^x-u$fq6<rgyQ1rDCYKG*&v8z$n9{!1;>32R@|+!sPLB7VZ18D5UTx9N&B-}A
zaNUU+7w)?<eNEbtTX3;WTg79-PFcO8hd)`GeohJ7Vft6(|IYi4%$kpOyGprRZIRey
z!8k?Abmsj@8=mI<p7hsgW{XJhgvKzZNV)Z?pG-c6PLDMCo{{nJlD|H~9L~siTb~>G
z?+f;Qv#eQ};2x}B+kf?}%ZkNow>(ZLJQ-M`xm0N5@vE0UsJC2vAO3gU#Be4C9>H}s
zEp|LVJ6cY(J5<;nRlV|=Q&Me$1>=o{JXw*`Sz=Wh@>VY8-KKH4dxA@o<^`Qbl^n5v
zcgwZce^{-}#dtbuU(58Z2U-(jf;<lQr0>~vXQ`i5W84fQHHJqMoI0u$&+u!f%`~m_
zQJxy;D?B@6hDz5N6ZPnuC)d50IiXUm(|GZj+^`j!cNMZ~3u*E#NI10O{ePV<wa3j|
z$15hydT@m;i`Qte?ve!pni`sU{*z8}B^S@<oboXLt@jB=PWDASz8wXZyA+zTg4Xl<
z1~_alxDilr$g+3NiKyvYTBYwac=GXNxH9cH8lX18(maRJy=Stgw)YB$-aSE0LOHJe
zGnNR<xvS<StT=D#gy~ClStqNDEIcv2Nn)Acp`w{Nn}oMKo1!h0m8+P!L``bB_}+sa
zs#S|4;uX7#_y6X2kk%TVxbNRn0agvinJR0h@2R`8Cbjdl$~CP!UK!U#eC_-u2wk}&
z)+ID8(>Fj-Eiqjs^ON6<hzY9fbG!V#7M}k5RlniW&wFpb?BGmRdefQ{84>N%<h+OB
z#DBHiNfJitVQk9l(iE#aCm)r$x?*ackD~t+o}+q;CFUfxsm^MQ^!L0rbLPY|2DZ6A
zPBzR}<~RweJqhuUs64)Zh6@8@^vP!r*vz-w)p(T=8TvR+<dU16F0Xkb*DZ}RZCak5
zXRWT?bYkL|A@KjJ-#XKaPknTx3^)rGPPiYkWWhSG$p-^7N*#aCmgQ)fK7Y=MLUrCp
zY-R_nIL>Zza-TIpwxVRs+>U?<`7O-q%Yzg+4NpuADs0{zV;C8A$z>C_W=`9k2KU(x
zrR4^hLiMMVX2gGE_0C<X=(Fd^w0#U}$rU#{J!f{tOiR$;xlcI#;UopdAel)mSEqPg
zN?4?EFl4ew<NsH$3t5^)IWL$hK6iOgVZP{`w2?_1cjAIhJrR*ItuI?0nV;@(d9GNP
z+sNXua=EO{OtDK|Pc@<o)SGt*tT|*dWv5M{Sk1%~rN=j#^1rTVE{$5a<bS4x8pE1S
z233V=^Ci#dhWt9OWq)hKvx7@q(zRX+&fhLEbzWOi3Dcu2KINHrOahNhvh)>hmy(<?
zmC?^mE^>mhbIR;*630D`YO3#AVKnbgp!UZ`551)YCQoKeRL$gB?Yr!?!R8|ua~3BY
zUKw<mZ`zF{VQriDZx^&)pYfnwsZm)*MRj>X$<&2<YbONO@k*;K*uv=A%+lp?@&TJ(
zj)S@z=OxXlheQ<r<`&P?mieD08r#KslqFVQaMJD&>C<6Wt+A6nN4Evt_By8eX>a$D
z3A*2wJw3?7dYF4-w%+P9!Y{m5R<JXtIn`KCPWrY>|Cno<*?cBnHv8h6I-X`fRg_|9
zE!nusc%j*rzPpFb_*!Z?J&U&=e65nYy<m>{ACnM=m0o-%R*Kr1%mFM62XiFP-*wjH
zj4^YFTo%LUzUt0_U8)`jpFMrsw8;2k?k8(K1L>H^uS!CDpPg3c(7dLuH}gXaYeILF
z$11Ut3vW3^9Ax|0u}AF93Rl~zK>m0at!*YR4Z@TT3D#bfS{!pD@sQV<4Vzia-43ih
znY2Jt!pLUlMvYoS{=&;El&<|q^w6m~Y4Cq@u<~S{S3%(?u9VI)T=BZ(b-28ib*0)0
zgSP8B4Sy|s_MEx8w&`ncy8nq%VWVqucAw5<irzVP@UT>L_iR=3;OQYIEmn(pTw9JF
zJ86Hk)vNGtdvue|VV<K?CobHf*79yy$%L@wFC;myHMh>pIjk&W$v;VPD_eL>c5J$E
z@%C+(xg$-FCH{T2@WQjeekT436;+0*+DGJ>KAdxUb#d<JySsMo-7TB*rrG+1*Fqhx
zpXc)?Zr_lgt#bJAQAHn}ZQtK0m!Hxp{9L=pp)D*zy7BTqZvT6IB`JQ@9o5O6+A2?^
z7*E~&UDx6}TRkgv1BZq4rK;~wzWFdQCS@dE6qi20>ZSPKs_cV9N1J1HoB8S#k);kw
z+Kx)c1KHd@I?T9SrQN8^cHQ#Q<@mc!w)3Bfh!-hsX|dulNz5vhS$m*RNJvA;Ty9H1
z(ZnGBzX}X{H&k;y<mPH@HoC~aZ-K^r=X#@u_C-&PqXi6W8^z9)C-EeU8E+9+DlIW-
ztZfZaGjo;cUg&)8i-DG_gyA)f<E$yE9_rrHB!Wa-<2uxIu5!kHNSU!w&B3kaoM`AD
zqtu8-`L0JE?w3=upQK(nTviaS!7(W`&6#WKf!yO83vUUAm$K?sxeE&^FlsN%{1eW`
zQe>FVXtnl%<K0gV;vQ^0)AF>GJSLPk>IvzbNbweK$eZ=tQPuIk%}NpF<R(4CK+mqt
z3FjYK$V^By67-z+Oi@+BauTaj=M#k&kGPtZW4EwtuRc^LdA)hBd$0YA=Ia@~`_vh(
z%@DXXL*QFO^`@=fc^g@eB-`vURXZXQ6BAIozcBa~d;dobjfci|kIUVjhUZ?I=5TAP
zz@1c?dtoxU38p)@mCFTk-B@IK&57;%w9e=N>#L9Qw?`y!{}5J8kzliM6!vgr3Shbu
zq_DJEZ;^>&-U220lX(k{39b6B^ZZynW7(wFp^+Y+gvG+SC65=%eAoKUQDkHxr<Pu%
zalQHS3c)?cg<Lp=-8!cjEo{EKkhRp<&qp*gbXohK$D&*g#e0thhHMhpU-91{@MCdV
zQczQ1j)RGz&O#s8kZC^Y!Ra%6M4dyb9F3g6m-s(#D|nfrdYCg-pfuDYbw`?TJqrU@
zpj%V0Q|#8D<A%Wjg5^vfOuCFj)_l~^IVde5=Jx4^y3hg!lf@BTY*x7p;@_4EH@D~Q
zzr<!KsBo1@+3XPGUsVZxMu~=pit;zDrp?Sc7Qr@khmNC@_oT;MJ6~|E|CY7=M_sd^
z%y|yEx8hUoX3R8Xtg&Z~%@;GMc|K+9Hi6Ub0q39BTuc`}6I^>fxz23coQIO48v<K)
zZ(;rIZuiYYxcP)hzlp@d=dmXRqs^R!3$F?MzEoQg8u!>ORc@khS2$Cf*u+Kc4Q&(u
zH|j9ys9h9!^l75&#!0-3COzUT5WXNG@?A(cEz#^{)7u59hC3!dFO-umFS6tib~@>E
zM?=){2LBoZ?I)t?Za0cvsWAQzR6Dy_?8mjK?94f_o#NJ8N}4}8r#^CS|G7B!$K1pj
zhK6QKt_G`J``l(y7BWGwv?wFRF5Jjun?&``#o0I1{Zi{q4+?&o$dK2Z%lWZA?U%@_
zN)6Y`(%T%l_6TaE9d6=2SQx#)*dt=Tk>mWPCTlJQiP}<j$s1PdI)sg#B2@zV7CTR3
zyTm5GYBryd_dQeL(y(;?q$-n#I+NH$f<7$YU{;;$P`w~)j+A+?=JQ^uiM=L;t5mkE
zy3Nt@@4$a9-<^q%RBZ)6+D99R?>rj4T`}j)jJ^P^g$J`{UJ%Xo)>@<UvFuNTMl<sS
z8LJ6bk^^6*qy{YvH%P3wdnvE?*@BNHR(H6vzE5$u<Yf2#30Ib2k<xL#Ee`y4mW!-r
z2<%~BZ=Es4G;xZCmZ+zt@HZ`n&t0oHFY;d(4D|DCslA!Voj6rwVzf+Z?fz-SaVOiJ
zsh7O-n3i`^*wQumL!x2i3N>Yu65-b+fwLsSx3|S?hzb0)<lL0$fs(@dS?%G+gfqU2
zG<ukHcE*}Mls^7&S(=!M-i@eeff-j0Yx^jvR!p(4Wl;?e6kl1|y(wg+&BiTLpVsvn
zuM0SBA(q6Rv9S7Q)&KR!wL}l3^QU~6QyNjTBRE>-o1npynhVD_TuQfjmLajfsx?M?
zd%r~Zo$_|mhw8=L!Q2~X6iv5ym|pa5yU9oMwPuC$U$x{%PjHepU=7O@)E75+{?xYa
zx}Jc0WO2090vDUD5BUG=68IH0#l3T<*+uTZ5BT3)Q27=)?}bRXu~!nu=Bb)$t+y)Y
zMoS4FKWdWwY3g4NNoSE{%Vn+p2W_txhZGbU26w52M@gu9tDl)|yN!EUfM;sem(b<M
zr6bwG;-)V9<q`H`d2!)&<@U?I63^>W3YK4Hj^(pVmu`>`b`aD#9U?s;Q^ZJD>QQX7
zYtvs7&$Sb}>yD~(f2-90scbj#zgF&*Rr9rl?>&@V5beayqsyhk@ZdyI6<4~a0c-I`
z?<gkGy|<=1e(zh%eQ1MAe?Ve?YL(kbRSEVZb$iA8nl5kWTkIE8ARzQG%p_-f%tQNK
zO03Txt!0q$=f3K{o>^R_VcwxrI+K+rtZ|saqhsrnBRuc+dQOp+1&8dPrpoJNZ|6}|
z^@$J*J~VZIskH7h=VTwlBu(KP(|sj(Ctbh1(f3wTxW@Fztc|%V1gk1-gR_DQZm`x&
zlo!3a$wY7l=LECm(={F)lg{~m$ni+K<B{0e3W;gy-uBwIKN*um3b*NfIcmDNz3IyS
z{uW(bMT@tOg`2~a`BW5cEmF!g5c+?m$~f?U)_RkOU~B!pFS8|{?X_b%syJu;zYUxv
zkKK>BZ_^PBj5t`7x%%K1?ql5DKHIBZcD*t9Bsxz%@ledQc~8WJx7%pm6x(s_VQx{1
zb$!ZwBf-PD6ZfSWGTu`zezZjUwTl0W%l_<5!pu#=TPqBHyD;^sJFN>~3RY+;=0320
zCu?#7zhU6|>nobCbu_=*)%^a|8Saf!Hr!goR^6iWB!+9kTw%w))Pp;CrIoYSoZ)cU
zywb2F%}2fEQI6_v;bW(lOn<oK*y^Q)GfE#s?#>AkFR<wU#<=tQN|i<rk$0TlSH8K`
za@(f1&0sP#IT`37{rv3e-y&f*?I+!kDmS)zXL(AqDWQGWe{=7`2FaF;)g=x26P5e6
zvkHd@xn0_#<G9~+VwS<iYf_9y4!;oIwuE(GLv!if^~<BK|C`YJ&w=0KhV0)BTpH!Q
zd@;gny>1vhJi@VO{aS_9`EA}8CNb;|vVP7{Yh9Il_RfWwvZ8j$GFe_{Go*x7|IB^T
ztkuq9I$=@AyUFDNiQ6M}L|<K*H7iBtx2nXmAU&BSr&g~NIcRi*^JU)4GN(`6-n%7q
zk2`TKO<Ht)#dWT`{C_WSe{pEu<H&99!JoW=uk<Wm(g8lcSf?LWQ)(7n-?xBs-yT7~
zouWxr@-ol#V^<m+T2>tMbG4nhaOuQjGjflKMVB1eEl_ai{`uGU?@U&5X8+$_E;(Iy
zsbI*dB@+^tRQYU*S+S{}Q+TQ*11Ce>iPF7>6Q#4%E%INLz1S?k+&Clc)rppyQEMCK
zC;h&4Sp8h(E#sI4%H|V=*RIU)-RNZ*#Fh2X(uXl~f$?J{Bj>dz7*iEi9-Uz&Gbw9B
zv(B2Q*WLx~SKJimafvPPfydd}UCxKO7CCVL-o>{_f$O?9*Xj$`<Nf$=SIDj|xG^Jp
z*SW^Shwe<@BYY;HsAhYaredpV_HN&E!uL5@&em*tu9I5wq2pfNO8p>#+^XLqp3K_G
zGdfmtY4`Jb|F*nX_tKuLKzG&?#|6pUzhf$Y+UUL7c773y`0CD6qKlL^1|Q+SwboDC
zo3r%4@cLz}OD0TNTD>dDfa}#RuGoA0TNZGx-q3q4QSP21*Ce5uMGG#>+S~hD@deMp
zEBogNFg}oykd4VN6f+JHkC}Ax0{0Rl-SX9nC7M^?I)@xP;AN-lYpD13Qe^qNfU2dZ
zmO9&38Jtr~|7cfyrsUxQU10~=<u`oJ9#piud!>%EvHIjj{nLDNpXP3<f4S@5@4Q7<
z*!Xsa&k7auXpGPj5}C71@bB%D>2IF|WDCE3RH^Cdeey(o^<viK4XbMQa?MhBJ<*B3
z#+Gf-1%72Kfqw>EvG=O?n6o9H6Y5+m(8epgcGfEE`=77x5cv0iJILW)UtV^Au3x?=
zgVfS({tH#!`5rqa_g}R3+}6*F3yd0buRZ&^_-)@jU*Q#-VplZXntX_>k8g|Jq{EL}
z7H8}gHV}HXWr@)A)t;Z5zOS-V;rLqK6<R3PUMTgvX}ytY)PhgHUp4>N;!4`E{_BEu
zm+$f~R#?5*fa`U@u3hJ(tIl%Gn(%ZdkM2ER+tS=!e?J`kaQaPM%3Kw-oa=g`ySS%a
z&8cfuoThAhDRoba?V~-_IqIi(UkaRZiRs<3-TiycZadC%?m=1NWxjtR6Kmzoh23^D
zC0&*7H#j}-gqA?zk~)QZ7NN=phopoSn27dzIi^kQ*>Abh#qZHgnT!?Jubp;^TI2p?
z?@EsiYI8PS?^*n+Y0~=7uJ2`UFbe$5`rjM1v@^^$Y*xg@*WvyR4E0}FI#z|QpS~hc
zNIvh$jf;<uxBqkK`=T-F$?>Xx9-JOdK~7EW`pI=u7=+XqID>a^JXjRG{J5t2_5u~-
z5QdqP`A@6d-7#fJ@Up<gPD^ivyb4_tDD0tl`IX}Ai5{EnQd&hdXKzYoUGrpy*qd8n
zU$Zz@T+K^f&1*1K<Hw2b3*#9Zu9mf&+?Xn~pHpqirs7My;!Aqj{n=i6ZDO#V7HhlZ
z@zrT-qqaTSw(9Ds*Wrw7A{M3|T~~UvYcI=*Hjmx%y3K07-##DrJEO(;x9Vq{c?^$r
z-=^Oeqm{En9n1b&U7E+YadFAyYfG!|>@%#s^ya?#x7W}AdoP%BsB!YPFEiv)Cb8)D
za@{zzwC;kgOcU3G4~5SU|6d}sdNJcfrRK?7`D$J`EVqyhSi5xB4G*Uk7F_`iMpg?i
zteU$jXq9mPt_LaVf}SoD7Bggfowr@X<(*)6%FpZd;_YU$J}$g$rj$B`wNfQhWWydQ
zfwg^ml9q~oaa*N%OhPqaLGM|!+Zx7<OZM<Av9{QBNm<jn=<%uEO<z`Qj5g@rWEjn|
zVd+#;%dF-pVIGy6MZ>aIZWfD|EAsGP7PQKpN4{Bl{ru-F3sR?sx>&wgwB_5K6H09|
zulQ8lEsM5X+GtRlamn3&(m8)!v#6kuAoa-#4QnUdTDg4Qv8IX3S^Ezag#Blpx#eqE
zOmXU~qh@(3E8Y4UmKG`}Hc!2JETS>Bdu6I+lkk$THmR-3-%6%>aWYPs@W64BRB^J3
zf#u<xnEb74CK;c-!lHY2r-Sxc6RX=nzNTdv3)#)O(>~|Le>D5+q4G&j>abPclIXsv
zrE#}Cw$-iOx;1gq2JXJ?pUgTNG!L336)w(V&emi&@aA^Xik}|K`WOE@W>vW4Ki88K
z%_RwC0oS6~dp<mk)SnWd$*A(B(rJPGt)0&&YOxfm2v$l(H7sU*kn&vAlPh8IO6yr>
zo#C?2G>)&hq|`b!ZE;ff+Az1DyH4!Px|x)`b}~~=i2CUln@;z?oA8Nm-Eo_>8&6AZ
z{=M*j^b5aY-|(Uq;d9EbRo0%X`Xu6TdE4KW%N0IZ6>$`uC~lpx#M}Do_s<h5ygSX?
zUc7u7*Lg#ecjb>sv$kECEaIWixQg+K=8C^FxISnsUiHZA+8V7KY3+x$p9(4_Mff;1
z`35=LtxUSz6{i*dFh@PN?^a~=hCWLso#aUe-h0&atK9gzDt~o}@?G7lp_}KK-d9~a
zb$i(C@_SY9r>>UsTg+6kCtA>pTl1h-#SEvplX%(^uYR2FQRTQXF2Y6cdlg4f%w+rO
zgKiA_65M@N8fNrrEQq=*-OaK6)BImKiOl>b&)0p~Jn`qB=l0sxC*O0Fh|Dp3p=hZy
zE41hPL~h5_Oxc?k{<rm%Z{gwHsBB^*H+i2r1E<TuHkFT4-9KFuOFH>NkICSU`=M=u
z3WXCzQXU9uc6<@sdiGqY$waO@n-(?9?Z}KtzBOydIn|{HH@-B<t>~D;=f3>Xmeqv{
zhBKB26wN(e={NDOu!v&zM9!!Wn%X@nDvC>;JnI8|icUxJeUmHF2w1hQWO=rS61N&d
z^$+Ec)6Gh6?)*4*^Px-FnnSW-K4s4H4!(&v|2S2D?Nk=c6EDLq&Ys86!Ya0LO^WHc
zS0}`0zjVF4WTnTN&rd^dEN;+U@?J4rYR)dF;BJRQ-PzxFC+L0=pIxhJU_EuqwP!k-
zmbz;W&E1)lUhY!l|LMu+MnRKw71sX&>#HQWntwSjSnzJ<Ik&u1^8XZ^MdF?derny2
z{Gi$2+pO`><U*D5$Vf@v#^-^vPI{$nm3rxR<WTF}hiiEJH+OZ^9J@6Cs7Z#(!I_th
zo^0uBSRt%^x5ICGl1bpkUIYHb=&LFUt*c`;dTc)xJ;x*QSUU@sik*|!oFhrww3e-{
zR9y3H9m|9U(Rta**-ER!dBoKn?rHk&dl3`9+e^-up<VekOTzqhLUI>PTPA-OTFP+E
zvD>P1(v&}^WhRSxacORNUdo@F`qNK|ks;@3+1gtx0<4lmjQ+S+uu6Fg`*OK#k2<k7
zr`VrqO2U-XV=KC3HJ^D1m`_^!?^O8WjH6%WlBWN^9kkit#-oma|4x&#K2A&ampS=n
z!mOj$9`uW?x;;OKi)+_}p4N$v64QI6+)m#7lFYNH;`zRL)<T`BMw10|IxL@mI>~iQ
zyP0J!?|*@lbD8>D6927mon)qTM~pF?VbhrxC*OUZAZ~Z0HIPv=(BZW4?5->An|~=b
zZCrF@@(-R)sZSh2l06HVR;NaBRw(%_nXq+@$ya5|Pode{cTAJ*^bHHsRhIN9ytnO2
zlIqKsH?*ZsADeli<#6l6?)ZRKt*>8Z9OgeLrI}Nxe%z67w_Ij+tiGn`naFo4Jv~PG
zEDCJB4w4PC&a8g<V&n4A7NHGW1#*`7%nb0BHPUMBo$-Y2*^QY8{@=_pku0id+oUyr
z2UnQx@0^Ukf1a;i;<b|L*3yju%zG#M=m!MxhGqS}(N^YBoL%!p*X8PAA;F?ItDWb{
z`do>~403hg+8Uv8@Ytu=xO1PUaL(~j%Va#+*TJ6fYs!Mkmx3ps@Lk#SGb*9vD(Ah8
zE3A_CNgO?@BWw3<OJ2=%J|BU&6#0zihyykc8@801Pn!Ahje|<*g)`e^FGU>nIp~!A
z(mln>gk!$SVSC$&AM$3!_lNZAZ{GYZi@kU1RU_`4oP;gz+oD!)wRz&J_bf>^*OBYk
zv6SOdi*MDj8kj2B+qm7i?X<5wJa+BR&NCAGwQV11#@V~3tWWu;qV2G=tM~KyGyik!
zif37-d;D-JH`ue$U3bxfhh|D@4CVXoCX_WrX3OtTKCWCfUw@4l$MWAQN13;*FcVa{
zc5l_HfVUk-H(Q9UoF~}Bv_ZFL=RLOH+l&^?+ve$BF}on`zv~VK5mRk1UaLg+Gbg?4
z^6z?nig~y}IWWSH>-gr|+`c`GrT=5{vaeM8eLr>Qx{u+4BN1mS+NUm;i)!GX=r}1e
zt=r$gS#6cb0$%M1)>dvtF^6|ea-!#yb}SQDTY0hj-GkLCvSBaOlrEn8p7-SA4TIn%
zXL@Qvr`@;KcP~72@#<o&23Kx{ZM8mXpM<|8nHH`!iC%GcQ~z1-84R26M?4IfqS~3J
zu3foKxaR*9PZd$Yi5HS&nJ%<kUw_H|WUmwVvo-EZyiR^v{oudKuYBpUnnc6X%A(Sj
zH!ws@n!jMp#)zqGuh+2tUOC5d*Rt*dUlZoZJXC&SxG_mfOwMy(zXG#+G`CrTu(p%X
z42Ruuht~hL(lXQ7x?}dHi8pw6-JWx5<_wQ0*0vi{qJFGC70LSj75jaMzKCL@Wg9jz
zM(t_Nnj!Yanp0$omll70Li23t&i9OpXBLaS-?^oWX@bb7{!7doPCZ)D|G~`g&bB;d
z=LeQzdK?0=pLA|#9{&GBaw($#r=noL#eCUY94whqJU2D%e7G)s(q-+?(cExg;|}vp
z5<K17+;zLVo+aD8IK8<2|AxtzA9D!jNWWdYyiRfM&JXRO5gZ?wEI)U11U#A_D8Sr$
zp`&9)N1Fs|;0?aUfQ}UhEIb#s>O8Q2^3mGiGke&AJu6lm`S4Tg{O-A7Gq=Tf?p>Je
zkWyl>tXuGwvU$47taKSEvm;zWi;n3EEL*EMdrO3F-VrXI6d`^QF{2G@`!d)!c=K}#
z=#}nS?;Gs?u~TZ%B(8lOiwswYysC0^IkhA-g1vfDU*HN>C&P|6yX?b%YnMs11}<QI
z7||XW(eColF-m~-pf_Kc1oKtFR>_Gn4hLP2eYQ}D*{W--!mPDIg=O0eZ`0@tEf&cZ
zzS?@ze0ERcXqvTwUt-ci@lA(&4|s0=?{VyEGgmST=d$EoX)SJxWKIbvo|+S6yFRJ&
zwY1ivS#uXRO>64rvil_JR3iM)c%P)CU3%uUy6Oprk2;K!n$st4@#vX$Yo(iJPghom
z_wp%Bx(58tla9WA!Twxf!Gn$t-x${JGfO@<@%f1`yH@svEonEDX#M5g(VV~<cw_Qs
zWsWDmJw6rm>2wQzI>8k7;KcXa_O48x(IqoJuWk>t;F;ba>t*75IAHOdHA~n0aN@9Z
zWA8c3`dRg6wA1VfN|!mhU0$nh2rzkWy-#}Asl@^ZdpB@jj*!-HI+&FrC{nU$Zv{t=
zXOohN>7)s!4{r#c^b~3Nsot=pHSolljx%SX7M$_;FJyD)#F@GXcAdk$_Z=jpzZ`tL
zqpy)cMB=ncn1kgj7mfcf*&SYP?f>BK9cA-w!HhZ^*-2OYe;S!Gf0|!gIOkIWtMDGR
zl^*;z1vjO8E6x?!JNu3PR*elgzXMnwcJQ3`ocrXEiO*rPEw-X>nWiuah}g>Wr%KJa
z<lN^m%a@t6a`JY8B0)wDwdk8((TcNI882IsA?H@T%AjghS?9hjiL8rfthgb`ee9E~
zMi<j}?zzV&&hcHO@JZlkZ-+{~#?cm+6M-KbpLcLHePI78bH+u{_<4oQ9fRea8^q;x
z^q)U)uNUlSxiaP7&oi$-um@)NOqS_9bV|E*Qulr)KDj5o&wqHR{eOJseX-^4OTwR<
zIc6JglRiCl_KwY`ihL)0xFTeF?D}a*y@+GcE6*;Pqq%sGD9aCyiai(aO==Qw5i>kA
zHQ!}kLC&;wE7lg}wE6~^^;XVa{a|fn_R>QVipzKoh_omrZ0dervEXTh-UE;BpeODx
z14ODWoNMS=5*NX0xPbe)Ku5vR-kuJZLRP*XDeVsvj+!cP9160pFVg;<v)`v_&z>)Q
z4=+rwF5>vZ)A#euk*~p1Za272$l1j{Nhb4Q$BRF@ZKYnXl~+}AyGgt{B*wJ2SHgh7
zJJ2cEZSn7uQc)Wg@<fI{?G#&jSYYAB_8A+bbOS4d8^Zps5`U@skfrm1w%w)zwMqXo
zrn%a1-3pq#aZz`e0&@WO+P;ZQ+JD8X5?sG(ceJi(-@S&t)uVGsu4~wf(~Voj!*V)Y
zJXPLlaX*+b$4Hb<u14hY!6}D6T~0`D{qdx)%5;Wr6Gx{8_oYg{Hxf5`TW-4~N*vZX
zbNGr+e9`Tv%$(BFcfzV<Sv4FIQ+%UeIf*$5F4?nT(PZD7TYTf3ntB-+7&pY6uH{e=
zXgGH?=28uZN=L)9#MYUzYAX#4*ZlBe6gaoeLuL1YwHB6P(?cT4zI43kIprzS`J$&m
z`eyG(mop`2{kj&g8%EeaoN%J5!$W!wx6bSKK#3WbEjcE8NL+2~sQI}y!lsiov^!jf
zo4IJ~>;GH*8+|(-S1kCr=6L7B)1?y3*{W0WdljB-?ReoDJ;7qd&R^nx|MIElv}#C1
zarSm;rg-t>##~H}VYwH3oVBy>kn@~f&MY5SFgOO9t6UVR4dmKpE2$(UHrIJ>i?d+t
zA_2*ZCp-A>y`I(25Y+#{HcIx=#2a>fH&~jAr%7vi&$nH_AT(@^!2IbG8gzf~gco=(
z>sWMEDejYn|Gfm)Hir{dm3)=6qK>^ea$IregAc*Kr(RhiuwVSo<hQeZ4yH)t*FG{l
z(PC2C5hfzL%T=VTz|`oUU(XWulZo~XZ+uL%EnRm#vd?9YJ9FjW2Zhy!>n=&VvG{d`
z|8%-19a{KXbMBUf!vE8_Le^e<zA@&)Uk;Uun`eE)rcG$raPwHNMeG&}HTUc%&M}E$
z5B6?95IMnuo5_&ZcV_J2j2jaJyagY1>po~Pt>k32u$Fb(&GA66{)Op74cC=_dmkj6
z@i^t^650BqqGQwAj*}m5Pu987TDi&7^Im?bOf9d6L!igAt!ZsB?NJ}X+IIBr?(sO-
z&2jL^^alogseN1_o5gN>ob2$rR{n<j#T409lA^zFF#J*2yCxx$+1fdo=i(9xfjKg#
z7zA%xm_&-Urc6A#*XqiZ$B$H3Cv&Zf<lMGHYm$O;qx9){vX{ahDDM||w9$isQ-d{n
zMuu$O$$k$eqYEM@6I(Z^PH7fk`1`+dN>G<zbj!07FWJLRw7mRu?aF~`Yv#%XMqGa%
z!f`L6-J!OlDI(<56#t$rkK(*It1ASAW(8eo*?e=Q*wN7L=uaIk+ql@zEM=b)b80RZ
zM-SJTrybtUI?mPhFFYZARba>Sz}Uyt4}0@snG)wbIwH7j#<bR}U8jSYG-fuKDQ&9s
zVY>9TwQiz}g!P3Nuj2al=)EgFcsyHShVY??WOm&P90~K{U8kOSAn^RD#Ef1Qk(|<J
zE9>~a3v;Hs1;or;{_cf)e8YwBKaNg2H%)uW+!=dS=lDoAsBK^`OX%-Pc&zts&XYsj
z>kbPaO_;A4$(I)>VD@ghm(+w$Gv{u+C~>g!z*aG#|I!;L3mB#uZP>u*J(ErO{ELqA
zk25Z(2u)UNjh@g{8P~6?5prsStM0j}YT26@_V!hsXsVdIddXA476I1aCutGg9n0gA
z{9B@{H@>uc`eu6d70*Q<K7L@Y_IT;@F4^AZj_!xf{~to;b7#Z^8Mum>2QKXi&6N(8
z?VMhE%A1YJ+pXcmp|dN$bOiTt?MZWaK2t9JP~?(=*sy0EI~NGs|5J!Kd*#`lLl)Vc
zVKW%Et(>O${vyYW{m0W5cpi#BmAa&SpS7M}pRb_AKgH*dJ2<}I?Tco9>$URj|0!MX
z4_V}F<E=@#&ATh>_SWZZcO?WKy{&HRe|#(b&x=D0l3jNW-gtN~d*lCQJabH5Onl4n
za@k%^Juj}}V_yDTQ$o2JK3v>#O-WgDcKeS!*`L1aBRK1#-z#Odp1M3s#UUqnqMv)L
zitgF{d-iPg<6p4n2*>8qtzk7HU$$gwZvGIxU*+Sb_WB61tzRBrQn?d+l9MIptYh4l
zjdso_YY%L!`{?JMol_n;spEB+Yr~Fjcm6VcUpwQfjQWwwrY^VtoJv>ZvY6O?<LJaN
zgM(`MpKQ{-CZ%x&Dx1C#ntRAwV$YxHHHY{Xr?ob`kJCGU|IbqozeM%ZVgWU}9jrqA
z`uv_9=@;Iz+Fy9MqdH2X|2u0nL*>O4+*Xysi#L>Ohg^FXdHg`fInh%A`YocYe9r9u
z_c_njf5#U0Ap3HJb<4Y($~P@rj)~Q}E#LW0_E6qau~{znr_Riu{UPq*M2*sw0n3`0
zxi>g|U8{JoMqamXB2)aqb2+{qE(<w246aO=z^&QF%Mvc)^Dt9&O8C~^uO53J`yBjm
z;xb=Aj?B&#lh;eK8eM47z0joF!KAD3XWb)%B~~RX6+W`+F`vDqdhU}fPvn!hpk0f%
zh<*Q1*8ZA9fq{WRA$FBczFpAMRZ5puz5dw$LNxj1#|;PFCU2T27_;!`tyfQNI42)7
zoD|T+(ZRH{*R*Kc!MxhI#ttoMpC#|F?6#eI|KIJLXib%Vhxn#ix!WiBZolAT_g(R}
zA}9XGj|uk+e*Dj5U#--td!t470*mehrZ4rXnjaL7c1?Vqa^w5`ZpGp<Mvm3L!v8E$
zc(`Qw$~6M14FU=idK?&8Sc^1QZ18q$W0F)8IiQ*3%*eUBKx;-|qATNEi=ZbOMN3Yw
zczR6qSy`ldx@m3HzM7L>4^}kIX1TNH<ixWpX0CNxm}2^B>4~VdS)QI3iZp|#d2xGr
zC@oxeqVB6(r{K{pIbGi^zrG4>?FwBRxcJ7;#o^v>H8y6REqZgkMDOk{mCIcVv&z>y
z?DT&6?X&ofdG?3C=4@Sk)JtQfZt|9#iHlUb8Z?5UygY*fHaneT678A7P`V|fb*bgN
zun7s>2Q+I~H-t%iTyXGKYR=>T`)X%DcsFs2=)F4KJ&PM29G%Ui(ZSH+z#=o3J3%i<
z%Zo9GMPO^(hO-5`uGjbPiJKlWdnW7230a}*ryVHM+ioKo5w&^J)Ol~O`L9&q%-{?>
z_Cq-+aZQI2m+jvT%EzoYx(@a+{Jo$QW}mwD&^fJH8;ll4wDgKBaF8ovQSz*_-7%>{
zweCcqzRyqAX&T8K0uMU1Ce8@5-^XxllD+k(7v1*zZah9+)v4>oCAm0fqh|_J%(5ey
ztd^P+(*2&5&&mi7`uQa2XKh)$x7VBH2U5j09*N2+%{>x0E4$1*XqLjknHOABj6&24
zvvzf=yG!s+KQJSgD?!e^iEH{?j|ZF!u6yYJ7knbDy-kNpegRX1TF_G8qv{PTR-7LE
zF;bg?{KK~#dGS)?Z;uh1_1`r{$26F;HgVZb-V~zT%{npSyoV{5T7%lQWlOZ<8$`p_
zByQcXSR#34me9?VshwZaz2{{<OWitq;*zCaw-%Yr%H;HU7?thQ$t~@^aiU<L<f~QD
z(ld{4)SghNB^jRY`1#VV=N>;!-Ht6%Qk-yP=G{(rA?-(kS`!xPtqmy%eo?qwplYqk
zvI*5|6||OmO?PWu>+m)3^~R;Fp<Zt#9K%Ar8(S)O`RQ-&bK<sL-SJ6OcV*uuFMpX$
zht75Fdk}TKtM)4E77w?@63LDV-@LxG|J$(m=o%HaXJ6LI{(tu(`C2>gO)l#kiOwaH
zW*MAZQh$L<#PzrWH^Z{+rG?o`kEs-P%Da0tYO-cacbqoQk(r{c_kgKTL1=cH@z-ZT
zTO{41j~(cJGJ!!rGVl2uZr^q5-FZI#mSULwY<GAWxAvS#6<vHw1VZg5HU;@8$*MZ&
z&bE1)`a)H7%1RHRjKuz{hOe}|><{NlEC{gT{;goU_r<I~@3)t;=ftQA?T!<B)BB|6
zLL+aQ&ZbjLTTdpR3v!F}s$X<}?%NCLr>C%RdIadjuDj?j^mxa|vY7nd&G+s1`ki&v
zWMXNUf8s*dn+=MIOdT^E&K!1N`<1U==GkA#VPtr7>vDy+3w-OH%vCis|Eu^E3wL;K
zQCP$9l*4hg0#6Iy5|d}VzL}RMU%X^hV!A8ft*OngzzJ^>yVr?!^31-#v@qq$VuixB
zVsA58wJ#X6sb)HIPv6<|lchj++tT%#+g?j7y_4oT(KlV?_3GPOj&L}&7I6Ax&2f0(
z_Gnh)LQmx)k&Z_pO6tEa7WP#Maqi!tu=AYdk^Z_fb6kRgjvLjq==(c1iLDZLoH{{I
zl{=f&%j2QJj4Q_i^$yCF1uWV<wP1BEQ^l#@T2mXmMO7HQ0v#CKH0NtvQ3~hE47@Op
zeeV-h|Ef!u<u2MBQ9dl<X81t4RqdE~>q(zL20@2I8Vx(;!VCg6{AHR`RDzFga$T+P
zKT;;~gh|=jb2df#h1Jn_9GLgppUhXyRq@=@cX@sJp>FfHBAy=aQvEC1f{P|Jr$1+D
zYH4C%wE8b|Qb~9L>zke<6W1?X;&9;FfhLPJR^2=MoTV$~&a81);L_!ICE_%%W$AL!
z36D6AP5N;1!U91rr8hzno*qV%`viIeCpuZDP1&+cLAQ<T;!Ce2i!H0}8mPLnT;iW9
z;j65f*%1_>wPlIvBty<Z?g<l{SSCpZ=Pq+Gez{uc<SXaU#(=A<7qY0o^DvVQ@#0(?
zZ}K{O&xz1J4F{9oo%h*9F1cAO3ub6FPMIh0Jbao=i*TPcf1+TKzu%Y7v67K%-1N+M
z`n@n#?RfHka)FN#8*5vz3g_gLiv+Y7yw@!Woc6|3J=2-%bB{^cwt^?WOrA!C&zsb;
zB1APea%I@A1NvM`9zXqJRJ~Qi?nH!>(6b~r!O(2G*Ry;JS9xt)IpNHf5R0{6bxQ*$
zw-oGU>l55~dDpL{v(J20ixu-ZnecPR`d8~zHkN-*_hRc}P|3+k6u6U7we9kT-hFM0
zv<~KbIn7$pykxnoa;S%P-ed-&psZSdPlo2hCnPytGF4(8@o24{(!w#rz3R>8P`;^Y
zN7j7Yu)>=sDM<P7m6W2)+4Y~YZb#`%`lWEAC#BS|_vERyw=Y>7yUeQ`oq5%_??}Zl
z`E0|UnJn&%Pee~vuS{0_|6gnNv^AYPJ!cnB-l-i~?$;+AEAhnnhU85(Wub{RiU~p|
z8g9CCac$00<IJ$V^~_b0bJ3CqDUNB6PWA?#+CED$Rpgw_>8Tk%!s-NjHa1mGc{FL^
zj0u*@Qa79w%)HsR(<bj_fBK$u$#l`Zv+8oUm$e7|)JS03;Q1-;gad2k*3G*=IGL`t
zbZS#x&NuJ-<LAr1YaaVi^T}@OWQO|CN9mI`{akoR@6CGAPp{(^t2eSTM@zeK?G96M
zopbMzfBC26NgO36uG^MO^xAUFx&Ft5Z1>((lV-fi@OIyROsMWl@YbhCpP%XtEA)OW
zdHvNX#mBStWL_8Uu{=9zW7V$QrJbuqZgBi}-`~4h(<iPX{_UiVpOSu_j+m#MdUDCr
zrZb+8YFgGf2xW>hxHw#T6a6`;>+Hq2>pLg+<;c!8jNlIHh`AgW#3gj>!0r7$>#L5f
zVviJ9oAJqWVa>Uxh9xt%JT`my%hdh<N6-42FH!B6S8r@A4c=ls;Vs`MGak?IHNi#h
zHAb#`|4p{orzJag;vZp7wM5?Xb%*DE(s(kdF=z(!+u0XB{`i$}Yo>QV5W88|M)%I8
z+0I)mof1p--#9vTRdklXoLT2w5`-PHXL;3dCM{x`DC*%mY4d?wL92y!9(6vi<~?tp
zb(Q$eZl%oydtZh<zU1;_>qc>lefp`tZC1}-`(1OU<l3VmoB#I<m`uMG@rw13TZ{OT
zx~kgb*)z@vbzS*fAjG*w&a)}v!=;bAW4BG7tC1Jca^>Zt*oHO7UpFyu2FM0X^eIj^
zJ-y*lV9@8H^6Lu(cD=C_Z#lR2hLiTfJb8)0^WQ{n8SkAFu%YRPkEl`N)EbK@qs}|O
ze#{Hvng2g9fps6-frmHKB_vlGvdS#v-nAskx^V95iQ$PC=kL8B^`%3Wi|fGtEejs5
zbYXP8^!7}Gn&fTgv-dZAoV)3&WTl1l8;?_74`!U;S?HZPu|_6@|L+C<-sn!<(8+WB
zy)H!W^L@C>bovWj1!lGw@0LlAXUfGoED%kbCAiEbN7q$w*%6DktxOB19^_>Bf9T*s
z2c9zzUQC^AI>*UEKGtE*k%T+8zA~Ou=07QLv@N{z{c7;D0>7WE0?G?zu3S}kv!O*f
zu=VwvF#b))8#r46j~o-vnstxk;*W^Q!3+7*dluCuNJ$7yVf}SObJ=XIL%m9&qMtq<
zoq6ZtbxXh9Caz8y^6n*m`gTFz|31`<>rUgjB3^mYL2^nuCkxA27HyjgcMkIXUfMFB
zafXMAfTv4H@P`EzKNOM}7J5FJml=}SvUK9^UYD4M(k&P7S*hJi%?Q2Ob8lxN8>`$r
zv0u`jHWJ=X+L<iw*%n>gwQ!yl-%P1Y1#jP|nt<8bff65X-j+y|s(r$@&F6v0r^g?C
z8)tHy`yX5A@JsVfwnx02=D8`EE=Og<rk_0R_TsXFr=Qop)6;V0m7YE1=4#>o*X?Q3
zr~NBR{bam5=f#t6=awae)LxZ8_CO`R=7NFLV$EryJV{=GR!@V!&DOa4IQ@oPOXBrE
zTaK{D#jf7?U}wqQ@L38+N>zV*$UIMoG^}dlex$rUm7OJ3O!#Q*->G~meeE-Q`&2tc
z^tUp-b#R?}=i1Vih5NR~9q@EqniG0+&(vi-$s(^7+dPxrvoz!kXXL(uMNTHN+ChnX
zRb?gTg?QhIo9~js^Xx|Yl*g|767P4gxJw1O7~ENMXNgzFo!t>G9#$@jN%Nn+;q+>n
zu9VazqH`?Ri1j4{>wnIB57HO1lxN&iJ#@}7<k>P-wO})qw0Eq|QE6I}6bq8lEspYP
z?z}d2Mri$kex@s-vuAYfys{|A>3&cEOGQeLe%SP>mDB(Eq$@9t@za}N6t}q6@NxEp
zJ6<Wb{k9z16nUH{NXTN>y=ikMN-axHE{l4UnpW1<_V$Bv=DWVjl|66YKDJvXvb=EO
z?#RxN8lGtho*b+*MQ!gzHL|_8P0P3yT^g1cyFpRf%b~~5q4(Mgk(?)SH$y)O`u*kT
z`(K;#)l|~@>%z6VN2l{DDW*z(f6ID8R;1<C1CN9$`2{Zb1!uTT^z%{Rl5)zNUZr`Z
zG_73YMNbshpE;9eZhFYla%<K#jsFh{7{sSE%qa<%+`=fn;#S-eb=#TTe>o4V^!0O5
zS?M~ZJ8YJMeA(UXK3V-H@##0C4clI7s2-Hq;a<_w&h&Ia%)ceGOczC8ik&LvqI^sB
zujB>(gMG@LrzaPz6fk0yx+gaM&r{25(NmAi=#xz8K6S%B=*eQHnI&eb@eIpK*k)=z
z^;;TqD(&8E+jp<J4u$p@t?s?2!S0|fzGtP=lmlK<93+ou_7@$gC}NR)r+G~FkxR*O
z#apg9ld_XF`*OWvY}Y=SyTzqQ<@nFLEhil{_h#Mm;&{42cEYhGS?lXwmMYy<j!i3@
zH)Wk(YH3$m&ffDqt1=IJ9bofmn0-i7Z(1X#i2HxZ9SsaK7`GMW6o#yJm~r5El#ApF
zwb&)<*>jd>$=$5_l(}%%n*X)oXKuCfY;y^4@hLr!y76kPvuEb{DCMP9S%<eUl?xmx
z-_sH|>5#qN3fm~p&VAP&YQD(OQVUMWh|Y}K|46RAPT+BEu;V%21HFMER&OUyc$|LB
z{i9_rN1#{C#TA>~)@&(qWIyoIBSL3EOtz$qOXaS>{#o89fB8?i^-zIHYp#Hv>yc;P
z$2@lz-9Osd5;f__Ez=~{ORG2jUGDC<*n{WjX4wg83A)P+*4+A+{i5w%uB7%!4uOhQ
z#|~}EJ$!4$k$tDjcREVWV0<#;seG>sdq>V)E(gvj4V)zh-tBVue>7>$+<B|5GGA|B
z)sof4Ev~bE$DJpKMORLed$(l6g2<-18*iP<`mv^AmAtpf#HZWh7reN8cdEkTImdlo
zsA@mZTCp}O<5TjV4bOgeD84u(+I{P6^Q2t1Ywsls&orApU*b8vv`?|ORzT7!C8I}o
z^S0dCht}TF%1rBgbNtn@yGh#O0xk<qE&6!sVOG%0>%A@?A7y^ow%T*oYIl(%?kX;d
zs|<3ceeiHN&OJ$T(Z6itbu4L;UbTrzWik`2Pt6vZa8SfSLegW|3!%bN$?UGQwKL;9
zE6j3M_%a+}^4oPp^PW_@bjF&z9Gw&I)~zi)c)ZNv?m2JHKYGqJYmUxp_%QF|v1tzf
z#n*VxndWfhTE>b@u0I-$_oec!RFBNO@W^Az^G|sPEBmfjl**o7^+@l~<d>n`>u<X9
z_VwLuypsNQxd)4D^fpe%8SforCd6%Px}=t^zS(864}-YRf!c%zxvsh4x!eU|YrQUH
zeROPX`6QjWr`Jn#=H+E8n5@@q-uF`Pc&@l{>xo-$#p1PDr!(H;(i6{Nyzk=>@z$N`
zLfC&s2NQM&fg}Ag%O=ZJr>^ykpP;t#LDUh2%FlBO7#}Moxw9;i?_zOlng8TSJ9k;y
z3C-kSpLO5a)Pn`Q=D6l9_|tMI<=rBO*$2CVnS30K#6JcxY`K0ZpH)of_o6n2lgBkT
z>K<;pb@<VX|6&sw=D4{?PdJzx@$t>9!h$)4$97%b>gV?^DEnDRw)mB`Uga}C<gMZK
z$(d8cAY;BUYr`bt)h_?frZ5Qv$m$=ps$G4*>A^iNEvFkB=he;sB6R!<*YPh#Dp65_
zwMEzb{gM~%xgX<l$vbA1SHz|jlUg^txX&gtH)`TayG8fet{nGVs{2wa^_p6m#ido7
z?%lSR+pu=gql35d<IaB);ZhMYe%)h~^ZHOO_jVS8>#i*C-?4}n8|Hs8RR1iZV{C8s
z*_gY*@3o0{gqgq!p;*zD`+q*NrK>#52%pEq#UY#-)W0EU^Rukqf{#~iuuFAOGTh3r
zXM>l4d(naGfzB&@=3JW{s`<apQ8+YU(!!o|_U(NR`%epqe+g#V!cdvc;b@Yu`Q>4^
zH)1}oj~q+iDz^D(UU|)?4-&;LlHb=cq~u6(=NtNeJaW$Dfk~m}#cAHLWvR=Bf@_2r
zzi!EUpK(Z(<y2Kc);seYh0TezvAGjoSIqoY?bcVKlq@14u>9S0j{^3<KWS%^9^Z@1
zoSG!U8*Ccf@+x7a`x1wg=A;FgGo~8(Pt?rM`1dUOxzg%Y>s)T-&9d4h%51ki`|hWB
z>%!hVhC6<*-%4!r2+5w1|M95ax1`CAyLc9LI^RgKF4ngE>C@LzQqc04t7VFI%hby%
zGmrASseP?Cman_bx>L@=mN(C0vgfhJ&;KWF+vYss^XIuc=kV<C+VOx}I^^jy<APc3
z`Pm_NuCkaI`tV+J66iZqWPUXLUF$lzk^|=q6O2F3aw>G?U$iQ4Bg2t*xmz0a))^+T
zzxQhXD4AX8_~o5^<eO#5I?~}eKA$Uky&7iiGWRP!GWqy+hlp))r_HCBw--Iwwf6m&
zT4jB2hOctwuAh}!J}>gFU%Z?{-!yUZ(i3a^4V4^EY;-KU^RrejdBe{)F*i26%&|y_
z-QnnPxhi1-8|SY5N;%s$XsuBy>Cb(!QBET!I=ywPMd-YFB5lc5Suc9sOZvjU&yL%Y
z<Mcsn^~b#;#hL+zl6x{IIhz^J4$)>kxJJ9>@aNX4;w=BC>bKAFc4uO6GztG)-+kmk
z#P$bA+WOY9Ozy4ZSX{QLyLeh`%QV#^E?a7HcP%+y+LY`XcJi^+%tp(qv!7#L{Y;O2
z6>AijrRlOjT4UjLsYUC<UoK0k@Y?S3x}uanY1^%rj$0b6b+`R<DG>j^&f(9x!U=_U
z`?kM}D*Ui-%bPGA>DA`G>sR`+GibYB@O-(&*?NnU#k)K5+9fx>-{NFvm%3)kZhCz3
zH&)>v59Zq#EHi8BzU({U-0iol=I7=*9h7zPyih98nm^m!Y+BSlvnlhr4=mYr#L+`-
z6;ovI>guLtKXletznX3LWx~FW>zf=_>FzUMz4_6Lgx{O;6~7sXTP90y{=dcUo00xb
zPhA@m_u`Mt4h%B?KW=^z9Psm|=gukS_6s~}raJF5VQ!f5;_Z%v&kLdyIOI3atp99&
zf1=fTOWE9+R^@Z9_wHAe`y3-6(0u*DIq{PD`(2*IDaB8-I}r8Ef!lC5a|=TwGpCqI
zg+QWHE4L(<$Au3I9a))FMb>CIG9KgPGAo;NAd!n*Pj$x>P0wXL4My6mJ0c^Gxq3Aj
zg>t>PwBg8sIk82@LL@KyND3==c0DmjY-;J15M}WYNKm-a7QB+r#Xx}JT5}|8k*GrG
zfzVb#Sw0nR28X6Cp4&Jwb{CvFu!HIOuC-bTdee7RzTT&5{6XJIkC~lYN9Vx@-^Q;0
zx%WzaCKx7f^5j#JoZ-T>GK~GUPKxu^&yHdJoPA=t4>l(oEn%6Fx3{WTmvIyOHZGlj
zP6t+D))1K!x6AW2g_n4ANZ3>yy0fd^Tzo-7<-r#$n%~td?p7AP*`XO_BT-ZS<AcY^
z_B#rj4xeaN^9VkxX^`GICH2XE7tc2Df4b7k1g0-=X;PUd<lxY8=GQ94ja*9Jj!jZm
zy0q4FOe#=R;Q7R*yG3P!(}Hb3inE*f9ECDmcCl4;GdJ@I_iPXmjtX==#AThcK}vA<
z#SPNCAE@k?-7V2GS)P@nXoH`Kagc&ayUd$qa_m_`$JcUg3fK{5md$@!NBY*8txax%
z9j1*w$^TO~Ut!_!U7(%1x^)VR+#_G_rb(ASEqp7d%)PK(<M3vM2SPhEl>3;za;Qx7
zFyahkjLFDUnD(st{ethi68sgs^0gZG3YmyD*b1NGICXh}*Na0clPoOqyLcy@(r8>Z
zt4sLWagmx1|K=0BzB)8z=$^jDebuW$clV<^*$P@8w63k&cB{+#rf{;zi}eQ=sje|p
zlitmJOeT46=j~MuwXeOJFQlxPs>f<{dBNFZvMY?QTm9C#Q+i+a#GRE1#^J^-Io0p1
zRxNt`LE+JZE9<Pgra9*+JbCcvob^ktZSU+m=W+8Z_nvsnkgVs@^>y73mJd7jO@DK^
zzx(emZ9h&{mYA7g3=5q8do!%tR28tGecdk0w8jRHgy(Y)fBJcF^(Lt(Z63udJDvI_
z-_vqtT@tWu9bf<5q$6ol_emZ9lXJytzx3MDNzqERPh1-O+GK(ZH<rd!8y?wm?z01{
z;Dm(ks8_Ks^45G6o4i0jTDn!#$dmK4LbT83R?|%nVs7lTQ%~S4s#8vvxfaF0>Z(Oo
z)YBLKjuW+}XfOt7tyG<M=Yr-2SM9$uf{#5AUeMfV_2NsAa79qbz8PUfnW~d~<~m$m
zTE_k0>R!o1Thmr$=w(#uPG4TIW-+se_F9&06}z;OmfqW^kj(k`w87i2P2XIzw)Rfi
zcx0Z*z6D?OtL+!EEK*oJw?J6Q=4?~M=KuQ@|9o}ZuDCk$>`u4I`nLp@$j*J(J6Fzl
zc{}F;zl#g+B=eOS)O9XTlG!=qn46G<hhR&PSM!wP50p1eopMO!2P0?wzt8gKf*e}Y
zoaNTvm~JC|>4IZ$<2=V%%gwbES)6?*A8P+*ETAOxF@i~BVmHU6wHg;s)I>EU_2sy@
z9{JG0ZT(7W;)O63Mah%h4njixt5hc_+-&Lbdy;rgNTD_G#@e>RHC)+$p6&lLb7c%$
zWbh-E@`HA=DNQV#Pn^Wsom&-n0;E39Vd*>6xUJ=#&qSNpbg`?uZ?AL-R!E-ftB~3x
zSeB*W*xBh_;y!;-OG>D8pu_48_Gx<~*R>ZtQDhKs`0u?s=AFZ|*#YZ{%AD7qDbU{U
z#k6_DlD)}CS!W*K!fiP7<(ej$w5ZEXLAQ4;H8Bla8okbXp2&#<`&%lE6<-H+SSVXg
zHJiZclWD}uAX&8j{2t?YrVMw-H7c!Ud@5EvHXU;tKFc!vRWoNal_-(1*myT(f{nDv
z6}8l#Rug6xq(~XhtPeV7w8ElQd-o&B`327u)+sz_XW`5YU%?X8cx98eLGcCt%|@5z
znR0Nby<K`H!EEykB}PUMhA@#u3JldsDNA_w3Pde?zD3vP;fnmsG>?v-Vu_qzBO_l+
zUC%PPs<UgYXQaq&$?_62<%zl;3B1wU50r(jdh}^Y7~?cCkHQ0&|4W?j3r}{9VL!yS
zzmU^syJJU=!1v(JzhC>`9`yKQ(jL&~;Skz)q~FtJ`lNGn^i=HxJACX`2)Fi3m^>q>
zXL;Qxq5HioQ)L3Up2w61t~fMl`tikF2C|dIG$$kpuVIkUWV+s^p~JsDVsc{0is#E-
zTYl9$R-#qF&182(VtdMkPp5?!|JeF<EAOs=*!#XEnc58#jr|rgs^90TShs7JTK;pT
zn2P0_?3)fOlg$%#&=z`e{mz0njhu_S@<07FSjLiN&N<h?!(xWq-q33bGk9+{xTjC)
z6jSHak`54vshl3m$*|mE_tLDT-C<HjFQ#d+G6?-~^DoO$T+z>280w}ofq`M+{}uku
zs~GH#B&|H#+UZ%Ml9c3gCwm^hVoLT2k-cVDwZCz6M~OV1usdR#zMN%`tJ>p9i>6%L
zB9yBey&${#MD)YP3xVsuB#ApkEWBr%*XS=SXsoYpk@oD1@x<m!2@AJ5G)X;8v0EZ^
zD#dw)C&T%jwLG#M)eTA_=O?YwkoSC=7x8lyTeRfGt8=>+?QUZ5&4^^!lHjguc5(HK
zy8@Smw;5z7tNA4Hy;eBs^K8?!nL5GhDzjc_D^FSFDW$$oYIf*Eho$Dp_HUNGRtTHr
zb??lPwmox;8xFehT7@2#3fs^qvhbphc8=ruuUQ9+bpNOyl2c3jC=<3{{-MGm*<Xgg
zI^Ni+{C~_;ud(Vuy9KM3<B6H4Pc<#y@+8QqXIfCiv$dP=e7j_rq^Y@=Rp9c1CXsaM
zUd{d2ZG+c0gm5oPWY?}b^|rA`LO07$D=2)<k(HrqUHHN;_N;pKczbQkYWq{q!um_L
zZJ2iUiRSe}v$Iu?*R`lJdA@k&xNOaYkNZp&Lyrl?Zq`w|#<R|Z-9#onnlUZQ`CTE`
z>yGDkLQ#thwv}i;oV!S^dhNGXftDF}b(X*2nf&v{IvyL&FM-p7UWAD~xxjn$!_vn)
zY$`ubx_qXXLwB)&UeuZ<vCWFEhnm%5CBI#(x>4;D|KyPmV}*WAt9{-haeb?%9LK0L
zXD^zpUFX{y%xXOK-2MONvcl`$Djm6gPxQo&nBd~$OVaA!+iXj_bup)DMK-tJ?8~+9
zX9p}UIJlqn;IDZb-JJ5BO<EuOT@^8V;$*99@pjjwtNW(Dw^{PDx_x^4ZQkpCA0{tT
z^;wsB^pI&_?aSpN41SOK3!T4&1Zf-U3GKb8oZR#HnVidUp2j56y=}*&CI+7Hy7y?}
z<dziW$!BxUWb}OVUoGLA@uZ!n<lS$N$Ddga^j$dOb8Ep`mIGhe7K!C7E*8<$6zbmZ
z!DRTX`IW~@)mJ@sYOifGoD`DOjjpMeS+EA%9X;>$bV^?mGs{tnE_O$;34YJ?7$-Pz
zEpp&qq`+jN_wJrvl|vSf&W6*|3V7K5pEH#6c{gjlt<Bq&Z2ljocwX7C;flxEtaGp3
zmK}A|(4W1o?f9t)$p*^nZ1hc}0&YkxGR}IdV7J)#p26IJ#U?KKH$Gg5o75|+5^(d{
zYNq&pkr&UtG4#CdaX-*-Uggu_i6?kdMSL7C=qe?swaFYx&v|zDm#$J>hp>EKn$bjI
zYp%=>y6r0t-ES-4aZ}{nzTna>{`(B)v{IfaB)oc<W$2i;@$QS>^c=xcRhr9|G>D4z
zZS!d42?^xNTXtkkPR6CA^bO0ox!?auJM(Z^$KxHxa;^AQd+_zT^sV+gCmqW7^bFgY
z3u(J{WH_c7GA(Mju_;@T<M@{aT=maRF+RR{I_<ET%Krqu(nGI*^cY$l5a3QyFH#D=
z*DP;)G`Ol=!%Ja{TQ<`fJ@)Jq9D=>m8XUOH5_uW|geP5Ce#}AG;)$b@kkZ3*hLaq)
zCMj@Dc%br|w_}NZ+p6c=J)C#&oL898tu}Ra=Ze&*bMCb_jz2ZfSG*9Qcw>TM(Btbe
zi@h(nFzq~J807CW!FS%gO$sUnOian&H#}+Hx3>J9vx--n1&i82r9g%&xr%(}6bfEt
zcnOLd_bR?S`)f;z(&7Y>Eegus$s#6>+-8gklRkb*^Kd*dNu;-}cG3E+4GNFuTs-Qq
z#QD^Dmnj)LGTwZY5sv12bnHw)_LeTsCZ6soIX#!2b+aAH*~7t*|Nn?g?DJJ~cn&pr
za(y}>t9X`i<)gfN8y7M)ubXp;&s{&B>7v}}1f_j^=OUg5J!n?g$roa?TICpDMb3n&
zazX5Cc~o{QahW7?D<wAEXcTixez@d;UiB;G_eTzC_2?EJ(_j2SM${?qoysY{T&+z3
zwp$CX`1D3j;x@duDCGq|PiQVrc)@X%V#Szs>XJ+vTi2W2n3(^_;5k>xmussPIwsAP
zP`X+2R{u|d@vB6IXPJB10()!xRed^mj`ZH?;CEm-;@jk)yDur+kB4jFhb2{Mx<Aiy
z-riJn>v5&bV{vyyA(KlAHxfk_Jy1$kJ@O^UM!qk7-JvwDre!IHX$#sjM3UNT{+H;i
zR(dq+qVBxFtXZyJU)ioDJ;|+lvu2J<_J{r_#{$<l3HuxteCib3v+T9|#X#BCNv598
zj9=THuk%qz^WGaZndx$i*%}w704IeSCk+qRH$JRWvkEk$I?X4&=S*$2o*sNzfNANQ
zPrWxB`yT~ycVx4#S9^D1lZ~&iF5_ia&yPCJQ-95P7S41&NK{FUmFc6@WQD9vACIZq
z@hYi@>hAgw-1TL%J@+A_E1!&v9y`=e^_72o?bu7cyBl~{FA}v%HVsOirx7Bcy@<yw
zWRb(7I~vS7!b$A`*=;|9Dle6IC>Mn}l>W-g48AMGsFLU|nDlT_0=J3cv>m+REg_fh
ziRya(Pi|)k_Usp7;Fv6_oR_MSpl9<$VOqxr6F1kaH7CCquln}+@3v$A-kdk;eADx<
zKj~<p!jUT{COvVR(xUL_eXu_lqy73XPp&9DdS$xsmDiu;hlNT`uj^SKm%le^p@rLP
zw)1^~`O}h(zB(;R5Po7*a3n>PCE?P(CFwVh|GuHV-6c@Oa;vp;>53}u4@^r`mNsO~
zf0nte!Rzv8uFV%*u5Dg9=jopZmlKuGCq}L{jM5D|bK$SWvGByQ&r%m+d>(t<n-amx
z)X((g>x*k2WhU^hX%TrMV5)Rw?X#7lDhvkhSA+tmcm;%{XFR=Mvn<Zh)a_)k&D^jM
zVWuM2R|+$}=$n6cn()6t^zI4;F+DxwqVUj7_uiP6s=ryxc5ca>2}|Sjmi9f1OaHEC
z#8Pg|!WhA3+{J7y+^49u_R<D3FaAZ5i`-VM3-Mkr_+(Yr)ooqehfJQHX{*|Ms%rK2
zs<S4$mKfBfyz}GTQvc?P>B>jXc$D^JoPPExeO=08Z?z3KXX*N<nq3r2+U~YBeAecL
zFRmVR_<qqOQIvtb;7yQlQAoY)*36sdqQ#nY!%KRDUWGI0Kl2LWUh_?rJ4rb2@XDMm
zy2lu@?>`FZ&d%oL|M)AZM&W#DVrb51rcW;EtuOP{*RHdzc%!zwKuals`-XPJS@Y`$
z!s7*)G^3}bdM>^`>FSMhN4@5J{VMdoPy5WJlQS&5+oqVT(zf}nyZlmMy7Y~rLe6w9
z3AfZXBe#XIPlNQfJBUj2rLSHTV_WLwp;dlSY1uWVh_3hLtP6rtgA_ZiE0`1sw`Etb
zGId5>)pc|EyX08H+>@>uOa6ZP9GQ{gR>ESuo#O<@h7<c%`RHGpbe7}kx|dImlor=^
zi}JAN_nFRJ#bb6kynXu<pEkp}Cpyi(rW+R+r!hY3c`1G`i!<U_LcyGhY4VR3*LZPT
z6p83u(mOk;yZq*%Jh$?NVKbJ9acJ%cnx3s-a^9^qb=LhwvpAJFj~`HqwAgVYc&6Cp
z*}DT;d7bU{E^B>#K5=`v;*|0>;jp($7VtjP%)9-6%4adAIc;CR7D>cid#Sf8(>QN&
zKI;OxYb=d(v^*|vDm=}!<WcGU2^WiFuWKp%(#iZNBEQzD{esx(tusu+6)mpo$~Ki3
zo?bk+WO~uEB~4ySSIlsIdd%sT#mwBw)JuiVYBTx-lV*uO3MsDAYAN^PZQ{PTx$VZx
zs5KiOcT`F1eqM1=&1Ca*AKkEI!<$Ku&nI6LI=mx^Bc6L&?@v9qZG7BLa)Mt@#m!`j
zPV`;#q<JGt{o@p&VlQsD#JpVZdhs363s=2AG_^=q*LUL#6O*W!mzfxqW~gkO+{Bu|
z^XkX4+*upg7i}=#p<wf4!}bGirb3%UqZLg0dA2>+^iZkg;LVjB|KA^#;BWQc(Yky=
z&dP@4I{e;0g8L>t*s$@)=NGMW8A?9aZJi_Mod0^&ws?Q1%c<NG6qcUf`2Isw`0CYl
z+Tz864(@wriGDJA_lvJfFlxPhSZ0b>oc{Wl-@48}L?4tri?7`t`*KF8^zE4+EELb}
z(vpzgD;%w$w0je)!{@H>j~v$~9JXFnban3C$Xvy+6QA^rcXIS--AtPMD`(y=g=M=$
z_Gs-WUgl_~(tbPHw7fUPb!LoDWPQ}(x<{I^8osMvn(JN(3`$>Yy6cUE`J5wf&AHm$
zbaQU5XPli?@BM*KX_M~M=IhcGmKL*xoAy3_(PX&WOZZUs{+o|(ZJxPf`JCfobNm0_
z*|%9`|82{#3kIuST524OI`#fS%o-NH%`UrtRBG8Bp6`16M$K9=HmT01ojhez3lj}1
zzfE2I;_a^2y}Kf>h;@E>sKQXNrr7Ch^sdKq7SAl}<!$;Yq*5l?<h3lK%s249O;-HH
zq}`3I5h=T}L>FxpbXvZ9!X;k*^P9ysex9)@KW6qxhl=?hv(NkgTCs7(TCLUJq>XoH
z`ApxnlKswAP9+BRM+f>Vb@og*^NO7I;+w2_@~IsvK6){09*X;im+5b9iA~Nn+4W|x
zsHnGD)Q-xN>)tOq>npP~!*k}uhNJAeH!PNnnkGA&U&&zGPFuCxx=OKqI@QPjGo5e#
z!*%%f-N_uSfB!q#J-ut-v%9C{lS07lSqV$6{5~rvIdI?5_TA%o{aw?8V%;zAmYwS3
zz23A#k>R`J?+adKv*(>q;QSVP{;}!$%o&QCn*444rB?+o+$a?hD&VQHR$ON5oyI2q
z@vGSSW6MPT_Db4vCiyWfT96elb5Y{A;Z5tk%PaeDN58oHTjAdA+s{Q5|2e(Xuc)wH
zGSfx&*o)l<ge^}l_Hg@fw<|z)lB!v**|n;rR+DXo|CM+g)D15=oND^_c}-!TqDWoM
z+_T<qw(*{PvMz6PLBZd&?4`n!9k_q(J(4*yd@BFp36V$ZTX!$LHdFcci(H$HmUB1#
zzgKZ-?(wxBj%(L*Kir!*anHY`{}GAvGe6tjS6H^|<aUL<O_%C!d^lC?HzzMzamySn
zp}H>;`$XBKj;LRG^z>`=^4m^-bX{$~KV`ie=>Dsyag|8yyv)8WOY93JZ&xfVUG;2n
zg7b<n?@eNtR~Kk`$-Qj7cjvnFv86K=j>K-<F8N}!<e%a_3O9c1Tm8K}Kg*>>X2oRj
zaQk;s8(-yYHGFDNc{SShT5_x+pNxVPr-a9b1&+-OY;GM+UJYxTS*15Md2C$N(8(+v
zbtgjb@xd?_+bR_e!6yg8mPBo>$~c{(%ed9U^MS|aWxh?Ed9UuA3|4RIwda2(BO$`T
z#K1X0%Xfl<laOZM7LS`op{K=~QrPmeERLmJ|KAwDz2&dfhgav;+Fs@|*gZ`>sw4m6
zq?;~j-gk9#H2oBJlqbY9)_)e-b8EWy!^6xwMNCeob|)Eyh?EO1urNG$DoZoz9*<!0
zvtG_ETUNPmnCm%N(Z2l2jb~3>8P`Sy=d@1bb-XQ>yZxALVCvp>woo-sk1GiQO~?Di
z|5w(mV0g~Du-WY29<>Pz9rC=k&)i#eu;t8Z;p3e#3>69tmlm6Iu^jxW!a2!ff>C_;
zvaLC+)7P$;zrSH!_(aoVYK;57&U9O)>M@~#+brXPM(jkxE`}08iyLt*YmYLvN%LKK
zSZ7=4G@&E2aG69}Y{4Z~am}KGi@Lf>jcz(<b16J_V4A4W@V~>d?}c(73nxp8CcEmE
zxfM)*T8<P8<+Mh*ZE#c(^b9F#dEg=K+oCWfLg>pA-!(QfgIu>wP-<<;Shn)nERLDe
zugvyP6gU{{?;9H0z*f5Sc|CX6Q-#?LY+5T^OKyg)@U2$my0NG$>BNggl4%@D>C@(V
z+*~?|(?EIlyx3<i9cR4M$ZGQB*s#<$KIx~bYwV)V+isD(N8DztnJ`gEdvhP##LU1R
z^8lv1p(S$*4~m88xZKq&f8%hPYeJ5zyB2%YhP$$wbE{7+V?R)JW1sl_n$Nk84|Y7_
z^SBuLb8G9mU2i6(PH1F3nw8MPc8>Mh{BKess~LWWnLKp(AM`|TA#-qq!GBjqbsd8P
z#b=MQ?qoIn^<r0Zk;}_Yy`X7@%e4K>7I!hfjF`}2^DblYNhZ!pBX<`rmy{?E*Thx2
zYY&BN&RWi7B`9qv?lOCZ!p4*U-8FL}4gKAVRNk&{Qrmka$oJ&*mXPQS!?p-fuMGXz
zC|Qy0T?*SozttyPW_f>+QQmf2bb;WOsWGkhj$Vu};otURX^rF)iKHop*RJm?wQc+U
zK!eZg<pb7St`{@c?g_b-BYjeAPPV!y-$eI~vMt(<leRmqUL)|e`u6kSwwD*&*Lpb0
z2p#_!6<26!xIJakk*Mr@#V;P#U2(L(xzA+lQO>DPHk_2*J#VwS>VaL`J#|9nt3B;o
zWcpt-lCe+p0E;h2qU*K8t;=uQ%GMZ!U$5phcgdW6YU0k!ZvKlu-XH&;@J-MuLa~l1
z=fJ~%Qs2HN`p-4;U=U27$t3hHNVh65)ncNd;L!{>E$b628bpkm_MA~rijMq{vE796
zOrA#22`}#}WlOK{JV}cX>de+Dm+qd?Cn{m+Y0btYGdt5U*NU+rSgFDJlfaYAf}neI
z<~W(IzL_ebe6fQo*H|zt!=GWXLhF}B8he}P^s4=Rsd&5W(tMp;Pr5j}X2+hHbadCx
zr-6NwB9^>6rgtpiRPZ#dJ1J~4B(3VW`UFlYtgs7SrF-L8VZyFESqg`obZ=>S@iKaP
z)iF<Qn&HOf_+N6G<E+xjX`<O0ZkilIvs74Be!QA=M#-ZrEq#KjD0lRm!v0T>gy*+<
zBx*h0G<9v-*IB~XCZ*fVR{dJ0>0S48Q<aUNtjNjldLJ@3)`+}}^gZyy^+WEV*zlBP
zhs36A-M-elKBHyI4VF}QX0>Ckg?A*n1vr%ZLnoxDX-isZdpPP&3StP_vCuiJ)F=8~
zii*a>&|iB>4(VT9HgQE}jP{-g|BgLIj+~52K|%)0!-H7$g_2yo@>nO;f7_hS8@R~r
z?N2H7c^~2>Za)9R#icQE(ier;D<^NiIuz6;d88+&XG-EHH%a4jTb6Zd`lOh+wl>MU
zE3kdi(_wbix9dipa{902|E^q-f>rmPgswMA)a6f_y&_K~Jc_H+D~sbqSd5|d@uXPS
zB~#`2kL6X!sdT7MR=KvtNZXE;vwyQorFGs$KFtrAa<vJTY%eD&h5uO+GezV2?CVFC
zb<W$(aXw*3gl?tEbf$(iQ}`b3+m?G?;nW%N2QyACuU^=%#yc}0<0`xA8(Edw=!Nr|
z_9b1au*|5ATFlLSdy)N~gktU+A_x9|I}ttk?UEJcIx@W{ZXS?2xx(mJ>w^60t2c4H
zJ7Re<bMpEM3xO3J4%}g)OI(+UOzcW9@eY3aB0>Aps>=&likUOn752JZI4{8&z$Ln<
zW|6~G#)(Ob#I|y-t=o40-#;F+;8QDpP5Hla=ZS3wGtGA<)i27N{H;hu@0gGGO}|%W
z(~hlGEA>u}Hd(WRM{`yF)T+fxb&QW&ugh#Wa8`N20%iXWi&Ev4uU3SFZl1mRmG(p%
zW6v440-di{d|h#6rKexpnKe%o7$b^VwF7Tmmb)hV^W@qdrycWldP^Kywy0O8<FSu$
z#MBSLs?YXrn!fSPEVWZRIs^W$o>loQDsk1ybS2e-1p+SVT(fl#`KLsfYHnGlYmpV(
zxKbkEA&1|dPT^#^>WjzCmRgjo`kZp^uIG}=lP1qRFj?Jvv$4q2m<b}6mWjQ#_0+yo
zbLPjXmzrPZb}dx-6~E}>1+~{lITT{&rp|6`$=KfJEBwED;{O^ZiOn*e3*9Vi&#aS-
zHeI{cbCY{ij&J(2T#>9B65AyWXKnhZ&~Y=`?|RYYy#Kk&mF*TVwyr*USo7SuY1|n{
z<7DReq?>Qr(zGz=(3VM2^IJr2Gs<o0d8+eNk!9+Epwzgz?B0bhgYH@-njc!P!*IDJ
z@=T+w&R*Wx5z(8H?&X~GShoB0M-yFZ$4a(u&Fv>2aONI;a(10WWKyUrC+~^4iknB+
zP8HqeFZJkGebwVyCv~SQ<k=20uRLp2i6x8p@4k?#nHyDKVZCf&=Z%$u1&z^D4}CZl
zvHPHz=rgrLkNYPqYPR*v5SZB1qIm1&)5Rw?&I~nvy-;w;+YOD?kEMPJExP)DuIM_m
z&vm7}sqy?bT4nA_W-iVU4vP8sSn!#LSKuwP^z>uhg}V)WjrJtwWmojJ7V@MX(0z2S
zc-fImuZ@pqroB9xaWj3UV=vdm)9*U^!_|cjrrPg*H*5L=l?Nt{soMn@p6>D6xqjQJ
z_tSU&nRt>@?CYwTiB;ar&on}>oEEVEH*@{E*(X*>ZmAbHxZ?i)uGYkFGuoefp8lS*
zaL<>Sw@d_W8TYO%b{5}d^vEQAk@lrKPsLnP#pjr~c3d%fm}{^u*GOP#Q0xrREZK+U
zchXDM1=Z@W`d6ekY*mmEP0Zp=RTpS=5lI!Wy_UJWT!3wdpnQaYyhT%fc$2I~w#E<t
z`HQpvM=cfn>5{W*xm}J!1?P+eX+vMr6+su(+)5kk(!VqtmYN$c&2@Q^dpj&xeN*rP
z^*o=7yyGr;7AYodh0cMkf<8+X{1+u>PZH9)*wD7MvCv#_<q;bebv?H2-fx!G9Q$6i
zD6B?wVcV(gaRomt@~$@;2)4(Baq}&2zrbG0;~o(h6rrwSDVgrQs9CynTJz1~X6cQ!
zXIm<=4@L1R+lmHBiY-!Pxu_U@fRXQ5OY<^A)5JX6=fR4T4QG9FU|AUBUS2P-u--%5
z@uFz=)j}C&55b^}*#8qfc9=;kW~eH)mp)>xSkW%HL?Mt{gJFrIz=4k*E-c=>&VjK^
zZ6<<QrRn+qvmE`o+j|AY1Nw~1%hHX@%hUe5rj;)9@l*-yh=^ikbnBZTHpjGTu7<`X
zwcL(LzP|%of)ga1SM*Pu*k3E`K5w}uhsp$2Q%94;2^QgoWrk`wCj`0KEo;B0K3*y$
zEpF9$g{{6cV(o$ClRpH%Ov_%oJWt7T;<|EytP_I88NIyQdw8!;dbu^Vba|Y<rSmJ-
z$yGNd8K=aZf8K7mteo{@UTCR6=Zl=T5fzKogHA8Az3SY_o9z4MlAB$Iutr+aTlKtU
z8h&%r{SI#xv^`|RRNyF9==kURR8z(>nT4!I$pQh4fvO@sYp)9jT<_C3C_O)|;nEJ_
zy)znay=YU(Y$|Cl|8KT9b=l=?!xsPLP6>Jyf{R@fC(ID*jcgXUW*TGMug4lZr9F7Y
z&i<Ok{T?@F#x7{#V3}pJeX5G7gWn1hfr%zoh27UvwR0JSj<^fgo@msmFkbE`{5z<1
z@3U6xZG!5Vf;Ty4msCtX9iCm;sb=(Y(y5=b1sEsix3|}xm>kC>v^TKO_**U8#J*Q6
zItnZ!WiCeb-xNMn61m~%6rajYqj0ghWr{C@LgpQ0d?P;pb@|lkjP;8>COqEaFi%j~
z#4s~~Q}AbcNs4HSZ@KmRC$h|eRi=uf#f?J$ztl8aM$}~pWS7qsjTBh>ae~xyRrAAh
zc+(s5JGJzV%{?J5Q~JDDW<~jb!I!m>K|Q<`3q_n38D_cmhWpI=))y6MJMnt6m}8|-
zv!e0Teuq_+b8bp_a!l~}<+kMH+=YvhSXEQm91Efx6fS+9$aj6xp~yv1UJKW^ORb+I
z8t~Nn)n%^G(!#wrC(5o=E8J0Y#<O?BwK;W}?d#J7bHyi@WlgqxnJmDbCKa`isi2kZ
zTJ7!9DbYV7Gny86ci3_YNBizHO-V6-UM{>?)4?`8W2)19spR=nCT2W8mhomwjO3H~
zDL?14D^BBbioK+y8{{Fd+Dl;CHJi_&E|X7W#dKu%UUu1$mUcz5d{>6QXlM_ETF31d
zt-H$RG;Nxm{i*Q;*Bs?n)30a<iK*6baLoDNSs9<GwfJ?0utmq(<7~N>vrI1^6PuQq
zJ1MDU?#o>DNByo{4zF&tto*T#vtpLY(gjQg!dJEm3Mr%pe(;J)5PfW<@br^UovL?e
zTicFDi&$2hep|eOQ#vx%!&}SH`CC9h<}NkS-NJISCb3PNQ~n|~{8i1m?W<=R3p8j|
z+zg#^M@wVkPPOS*BcFDb`DS+Zzq0>VEIPSr^Wu(`Q>^mJpRDxrQvP$(k8`EtvMAvt
zS%UV4a_1ge9d+~+HEb|;TyJ9{!gJLo)Y_%wkw}n&Y4qx?zkh8F5Zso~ofUO@Ynrui
zaHG&~r5a7`#dji_uFfcbB(c!^W79T~(63C|6FswXwHN=NymRqiMZ@$jiyd#}_H8ux
z;L7Woy@X-sk~`*G=Kos4e$e^5)?)8XDoZD^L~pQ+d}t}HxqNx?^vsV!(i*)nUzeFW
zHZ6GFSWw+w-Zh!$dQFW~ZDYk;0l_J~9;zDM#xp<0s{9gr`B}fm%GLK|`imy<0LlK)
z&doNr_k?#EKG@P?>NG)N^HhuJMibHmk~6n3cnGH5-1}srtao(m?<~Pjj)EJ!1hz&9
zoIPxvC(7vkNak$u0vW^Dpat6&ea$?hReOC$ufl4v{hZ~;IFtXaUU)owxkg15W2@?^
z6d|uAGomgDbaXj;tX$hUtLV<jgsWFpJj}{%`mJHHG<W98b+5ScROYPv-}}1rgI7>0
zgP^WYrj6r%k;KfvfSrBY_n8##+a{<jsLde!dZF#ajk}$epPVJRdD+Iuu*pU$3oljH
z?7o#5b~K{#R^jgMwa%Uz2ez#hzau>7Xq17s`16-$+e3>RpUpe%J(YEfB%{en?~au`
zOC$`0O5TRlvsG>p39R2N(Xi$5zWa#+>8$&F3k58E1io9TbF7->?krQ_YWx4tw%XHz
zt5%!kNB1hNSY-I6D#*O&mU@M$)SmNUi<^xOc3Jc{ygpdD*loh1nZItVbJt3mb9{$a
zmcXY~V!@TdlNopR>uAhey>HJM!P!$x>$#*~nQk;u+{K=F#wcq|0oP>Hm2+}WEI+?p
z>c3`4q2tfI1IudftyB=LiMr`?ROgoSqM9i0n=78Q?|E*X__BQd{2P^u*EAnalPI1t
zUtnX#2P-Yr$6bk>3;ebpx0^WY-<d7eE+^j14&za6xMF(3l1GsFmgD0_hL0D+4Xd;_
zJmk)K<e%e_ru51~Yv;vM?TJ#qRG55DHs17?d1cHiBXgjuqvG{;v7D%#GK=fIXWC?)
zYR;}Svpr-vb%#dHx<!@qU7l9x>o7RlUeT@zGCHHiq1%wRR9OCq{&|%wq2Fzdi_UNg
zom6_UrogNHoX?yZiL{us1EI?;cXTJ{&)uC>sP$grT%58!clJE~#|BZ3$NnapPhKT?
zpG7OX=Ga;93I7*M9Cr{u-q7o4zew1XTka3fJ|17;$B8$rCNf-{s&Kzs$8D-`Sfbc}
zmLvMPR|~l`IK;Q-r=P4^wfpGy)jlRiwN9_i>UB2E)etkgeD&Mnn=Lz(a+WZ1p5E|g
z2kX*0W#b#Exx(&y)!f6xk{#wsM{*zW5j=6{cBW!)%-u6*-y|hptGV=ixthtf{Skd!
zn|4R#2r@RGT`n{)N2soQ*TE;^i=XfDoqJ-B-STr8G4rK_&woCC-7jN;?WL)ifm>Gm
ztoOJq{9(1gk}0y6k~~X9y=1@KxbWyk>cpt|)0X-ky{S;)Gf`&Z&5ElFyt8*!^{Bp_
z`<!*n_tsmwy$5T#bG2F5eeTHpzud-fR%J_@w&ePnby_(NcBij5Z4-Pgb7k+G3;(8z
zdCryO_<MWFm&e}jg1&zPw!b`la_`OJDy!o;!tUpGl}TKRc>AO@V$PAB8{K8f<+Cs5
zM$BE~B)#-+_bJ;Iw==JuSteZL9jW(sQ^KF3*&hsQ+={0k?=0b+|2q8n);Gbkem#2E
z_1t>V`~n*#d1rTv_6fgi1oVYt+j4I(#L0d1w!VJoe9_S>j{B~72R)cPMXk$5<V3c&
zvCCEI+yjfdcLn=BT-^7>=HrBUZ!LCj%jTVFyx%He&Y!dCx<M;z(@m;g_wTtiMRKa0
z)NR+Q9b0m5=Z7R+ioTMXJHh1R?Hjiq3n(5wb<X5J-&57YGS65;mm9BaH~)FZ&TMzC
z$Lzcvme<zy&D42zD`tf$$Gpy2qVYE3IeGW`tM0{xTz_+R^Lv@o_gf6#&t2JUyjR}%
zM&Gd*^~K5+F@isMMJ`W#e6i5_!qN+D`_FgmN%~l*7IHCA@WmM+%d1w6H(3ieUiJ{K
zl)A+$Sspm=fNHMyoM%mXUaDKyxjsC8Rxdw2^RJ$a&gDm5mo@m7%&wG}Gf`A?&AoCy
z4b8Ck7VCtq<uo|XzG46PX4aROY(r7K455S!ezp@IW&8z0`2|DQ-}IMz%ItH*zwgMI
zGc_AlKD~D@<bmFl>ub#(=&tCFNbd+gdSr<(50h&~_;*nyz5IP*|4q}yiuOsCth-mX
z_g=Mc-ZIPRTE64<;mUj>C4VfnzU{sJGv@{K#S2d#T5&4wV+znZ_wfVUybljf=i3YF
zED?FRkpE?|s24NiliD@;Gyls7?PpzeZ_>PX6V7VQ-s<lkHD66JYLSN4rJ~d2+w!+}
zWu8sl`$@&?)LPrz`oLUM-rN6w{PDbXY@^NTvTf(=&)>dMXR>w170ZVK##4k|m43Zh
zZ{j}Z?ag;*7)^Hzb4}cum~iBN&Z5>@j>8=p@&EgMUjK8ld*yepZ<i>;M%M<1CT3nS
zkqH5bPI_F@MkNx0Nr&5o#JN_?_^2q}C1Y6i<VI_-2V<;Ik&z~Ys(P!A^-7TsL5tMR
z%+!yZ*E93;qJ|)@>Npn#h68?!xP?VrXN9~BVsxKr7b;=0@bWToMi0-9sZ$w(wnp7t
zrM^Jy%0}zcPPJ9rG9tF*&fFHnA)a?=C+GQ9qN_Jt3){Hnm6NLLbdmUVjqW>L=TtmQ
zJlZR}_-=_|(IMXymI*8Ouo_FAWfJ!b<gi-!qA|uH-|%d}mdvhTjifCz49zZ1QJs3N
zu4l`mLw7d|PunRWz@Ye(g%vc4tHZ#+z|g?tz{tSypMjM_BqA+gcH>eWr4WmP1*_Y5
zSXFdF0}?p(m^^t@lqXDK=o4`6%Gr^*#9@N0;<TL9)p4uCn6ftt8E0MH6{=SJD#pBE
zPQz?F)(2N4gQjROE<PeFx}apqjz;g9|5|=*QS@(o+;&f7N2tQN^)Uxm2}MoKys|X4
z(<xSJ>k5ZLXT?5KYlC$smhnDbr_~+q5WR%iomEW1ZNZ8|i>Gmk=}g#oet(vtQD4m^
z<);(Zu%)FbClseM&aq7wbI2-w*|*q#a$xPOEicckj-1GKR_n`~b6X3G^QO)Y&AT^?
z_0XO*)2^I0TQXttLzYS0ZqH*Z9H+mVQzib2X+!a=ra*<|jng{2|L?!k<}>r%(UUzv
z{!u9vjk8bWNZQ9g5R2B4PP-Ojyg6)x$6BMA!2!Heu2padl_oUtG<}__(X_T`TDVi_
zi3hEZEniIY>`v@RXrIq}g)eYgZ<3^@+{UCwIg3v`TU2PUP3Uot(J77Dy(Z5(9``MM
z`69X3kc;8+1UV&*Wa<00TUu*U3JwNN_P6>eSaqH=;OW$`w9eE?mbN073Y@y0EcHA*
zH8D+6b%Dx(N~x7U<q9X~8XfasUZt``_w5y<MGRXvD7UGZiE3U74Hi5loXjA`aWPQ9
zOzUb>aH~s(pG~M%gVl7tloxTWw~o9FtFC38-r5zECbGDWGfSb>CsRT(esbggh*cGx
zs@qoimliF3xo(o1RAyR&n#8>gIcKgVZ=55lExqZOUiO+z=gp*bx10%*Uc32>-tC*Y
z^Himk@_yBF%V-~-Xe}_~K&6YIW)@$GYRN@kb$z#6vD5GF)m)i%$#KfogGVc;sBV|b
z5LlRz?)-ISYmjHnvvBVDYpyIhDY1G((`Db+7mPGlD=OZ)5Lm{$Ouv1p+tTYNT;`rB
zJR!4q&H7WNjv7~jbh>#YPHp77wum9B$0Vs+?r3^ay7sD$MHiPco_*YFtH-N+Ilx|b
z>*WyhzsVC!+a%YV)?Bz;#m#@$>%8)FnVR2<8UA;hRmDt_40O#pc=y7D+&PsKZ!k?!
zm@sAa*8lFW9e9}iK26u%Ya(Es%&_ZR#uBcq41fE-*LMU@*n2d)sh_)L<x`v7xE1SX
zl(DX2NN$a(^eFB$34bP#o3rzobFWlo?3TwO4=s8+m9y8a6x){J%<Ok#^|~X!vqE<W
zy#4vsL(bGc>i!!k>F6)#PMQ7vav|=%^0(Xaw`;#%(u$7GeEfOa?fe(>_3!Ky7uyo!
z9FpZ3yXWfJ*|9forM{G{h*7SPJ225OVz<X($;m2jjM#Hrt#bs|9bV&<Y`?IT!9?xg
zgqy9U{fAEmo7}LUI3q`E<?e`9GY!$xOZ8G!-z?x_UApkZ2R~iQY@URqip`QvF2Yqp
z9L%f<{cQzpB8wB4FZ`ddOi7$~qq{!SB%$;pTU7Nnb_VaBCA`!_dF{eAJTa^MZ*2Q9
zQPcEzP6U^-#O5!uRi-IEOpeJju1%l3cB!F{@Wk5=@6?v+@d@}b%?xgen6$jCL9=j8
z!CkJ>qr&r3mTJYH?0U-bdCGE@)Cn_w%lJ!4YPs+|X~`Ds(E3*-vhR)8gbADO1RIEY
ztq+)_Be-fIPqEO{sFj~4J`!BwAt64w?TWL}+s=DgWh+m|ty>m#?{wIYJ?wVbiP=%t
zrP2&1dh8RwsAe*2;Zg@9Wnam;-g$A&@_Qbb?$15fUHAQ0pGEOZ;W|f`K1)Y+moqCT
zY+3o#S|{to28qwh8f7jt#QK~Ri#zoHK>NuhVv83@=(Z_2HU?~{5EkKNQJ-NMHKAZ?
z%<pb>>tfNUX`9k27s?(t;5^ya#*!u>dXiiF`0mL|KP@hmJR4B7)Mf!^g0O>1(#5nY
zlhv+kv&^?<iWHx^tl{@@xs%a3DgBTxm4zxfFB~~<tZ^|jj5|3u+xYe@g+*?SGcNZ!
z=XyBpGnvq|WsySKE@?@VNfUdYY@WA@$@7BKCf1-Ehvf7n{j4Jv?r?afvSi|sQz15K
zLJ`YO8OSZY@>EB3mw)7hl9Yu4Q5QC?y1cS=%8JvS&uzjL{U$!uXmXsI9Ga<pXRA=r
z#v^A!<-)w>PH|MddieBn&B>_Of|IK@T%DH_BYgY)|AW_NTzWJw|H$#^q;G1qmwS4O
zmTcZWF<@oR7w<gtx6|#nO%+y(4o$4Q<0)b!!_v{Hso=6D+y7CGdLtkAJB=*{U9E0z
zO)q>k{Ioo@K2Li(aj6LZs(DI#SRS$jE6nZUC@sBz;Fb2ii6`?<NBj(*>vK%qqH;;R
zu+OT+J<k$<9PHoh5jaVdb7P0Y?t<<<4iyc%i_1g)eDHVnNmA-$_xxnpSsT(5V!g{E
zC%Gi6=WLi;jVR~pV(wtcS7CytvQv3h9lgL^s5$B978i!PGr{R1;ww%~(hB^wDMV56
zdAR$sQ|b)g)<@}xPTLjnsa$bkK3BZCz<EZA;Ea2rvr1c^-RR`{uTwfHG}&#Ub@a&#
z6J+mjrGHs+AnL|pu{BSZE?~aveVt2q#_=uJ7EbQWNM-wc@j{WBkUg_+W)`<)YQprA
zW0I489PD0skEitSwB0YOXKu_@ENu3BaO~B(S1ohHVuPo8luTO4x~AbfPmIT{t>-^X
zDici>Xsy%?zHpK^!m0W5gyh!taJ}<Q8nb7+E?p9vwlm|WlZ@+xhX3CT7b-|EHr>3M
zNB+d-z$=SHpZ6KWguAqHo&J(x;v3~(xa;!jv(d|ymL&-{XC=+8aIw2Pv;R87I+Y1I
zi7P)^dj_ZLByWx6>}Cn<wtQ$-IB`LDZG5cotOIV>R&J_jEzdbI=U!FbrE{f8lK)pv
zs8`tZmRoRDk$d2WsoZ;RX7{(0#?F&Xc)7`Ij|_*=>^)av3_^Drhu_wkwE5)fX>vmQ
zUwz$Nm6{kKW^P#aqVqg=PWyuE&MzF-U)y?`r8=)Rd-lDy+Ba9F@8_LN|CToO{q|R<
zuNO=YHk&I`o`2%$GxO)mW&b?%uQw|zbxe@Wp7N8s;iaMAd$urxb6cuy;<BCVk{-=Z
z;J>ZnJo9Kk!KWPFgE?nd!=|P!H4-yi&U#A9x7K;y-<(E|*<UTT?|kDrUn?Xx<FhZ{
zI)-D0Qckt0qO%mjPeumuJM2!gHQ-HZGuLrZh?_C<=dp`>-Onh$n*QV0-@Fr!Gq$Lx
zpOvZ$v#R(nF8J?JP1UT2FV+jK3g?(?{53+wwr^fL%i?T_)l0>WeqWrnO76te^>;q|
zFVpJXqP5JtI4rDqv$*8s5F=UU;@#gHQa3j61~v#+ln6LRUR6t-e@SbxkXA~Vy5Vv!
zz0aZ&$-<2;aS_Ya^Gl;*uQ%=4l5s$!`NWir6H78KDP^X&N9!#ryE93e)wz83hw_ON
z3l?qjtvMc(BI4I5F7(PpPHACQ>ve&IslLjIv2UCV%$sB19*pf}6xyX`@gXrx+e}`k
zF;C#KWM^^Z{X-V3H&m8hY-7GyWo(?F9TNZ5w7pYR<l;8V)eVhpO6@ZzHHbfIn0Pg?
zK2Z3~w7`Rcnyg<%#r_)yl$f#!2Db^SDIaUBdBvD-D4OreQv2gu=XbF#1-JZ*>aLNY
z1?=T@+H8U{j&&^IF<NcrcN9YtrkHJeVY*hVaAG1Gb0U`@qZ+HDpwxCjS$4VF!=_nF
z!z>+((mGN+6xq6st+W?c@hExk4o|mQY#nhia;|GJt5b>Cid4lNz4;TZyHh2epAhML
zDAAiLHkZY+EJACZv&O>D4Ia~rUD~Yu)q}2hNUFThRP8WqG~qgU-JQ#&XxEX_=4DZ9
zLUnc>ZO)x;yS_F07|X;Q_cDo{W!dR=$399=o0$1=lD(y43wOHek<^yDaQiQlCM`1)
zczr#v$T{|(n}ar!9BWIM>wh7ORFBHug`B@keMKMV{$p(C3lerr6!)JPd2fXiM})%S
z?;^>9qSL;JzH*q_!8&!TcvX{!OpAx8;MA$@J2aS-6s#3R{(g6O?Z6*)oUg>JrXq>^
z`8LjLlWR_GO4L}CD8Z(CH9g<*QR4OI)31rouzDoRpk5d7sCFZVAg_7dTs5KYoilQU
zd*sYR`5A?$z7()xR4Xh|D_df|`<tMJrQDbF1htKvYbVq@e=@#s!|Cq}qc2+}mM@BQ
zdnmqql7w5Jc5jmQ=cQcCKl&og<Rmn_wwBK}N}j#ZbN0rQ8Yep=x4zUk>p5rF%Bido
zV(uH$vK|VzZ>pZtq<y_3G9fKzM!Svh|0AF~mYzn=+v>o-H-P`|g?aC8av!+n9=_dS
z?S@kBmrWC<%{SRGzt3%A&$lRx6F%3PWR5P9kv~|j9Ne;KTg>unljbbbYEfqp+~xNy
zQI1R5K<1<Kl!;NRIoj?nO<J*S^1-LlGX)$Y0}Tr<wk`UgklT^d@gi5aNo(@PND~u9
zFU~-dBU5=YsuNB&Si7q?FqY10h;)w9GMTd2Y_fvs^(miHmoz&DNKQ6<6TtuF0)J@L
z(!U4hg+}rJJ1{S5*3yJ1{w)hQ^&d{-PSLr+8tl2aGyB#uPnNE)Ze0aYrn1+j-(Xj6
zjhw-IQAK^BtF$3&b@Gg0hq?_M6<w8PFT!I&Bw{W9Uu&89GHY#tz?+hgYAe;c#ZFnD
zT0dOmsz0XA%oy;bwQ50X|B|XDi?t-?Jr(aToZT5H;r4J<yQ9R$U#qrou3nwB`b4M5
z=9l7kB#O5!vfg*gbMLLW?`DW>s9e2u)|xH5)@;7DW=GcSQ@7Tfc{%sc;@($3`aVjO
zD*dola7u4rm7G~P@0ln6%bW9FXRdp5bKc9Ub$<`6OSrY}^{S=G4qTc$9KJq?+P!2#
z%eJUJB4t_M*Z-T@ye+jX-+cpH^~Br*T&ow#%sR%({(aJst23AmGBP`Eywy^EzHrhN
z$8x_*<=Q_tdd<-PS1QnvE_CXK>pVvX=7X*5&N4EB7I~&JOOEm_nZTO-f9E0#ZAGio
z8Yj%E>}F5>b3?H^a0!bhmvhyYqzRmE+FV|%IqNrVaXBp?xpK<x%egMoa-6cAU0;j4
zpWfnqdrREvElIPtByw*}TfKFq)7B`?z~&hWx<X5(ocUjME{%A#G?8mrN%qo;>h1Br
zmX>la(>3ax%8{XQiK}b5-04F#(rt-zcT8Vuk<?JRyh(G$waqHixXm~x7qn?Hd=8XP
zy3(aLZAQnh8Oc}7dbnbPO$DvYLgS|Hn!GbP?NOlyW7xv#l}>DJd5eT+Utr@-arF>f
zCE%2P$a>F?$UQqN_Z-XKvn5L6jJJf}>pk_0Sp8OWUW(p(`M3Iw-+RvP-rI6<@9k=d
z2miI%mMiUhBE9dS_Ufmn_p~qEd;Rvlo4@y6k=}pl_r43S_uj1*Sy#3H7x(JL>}yY*
z);MN4`(U<n&@rwjmg`>ZT>5V30iH7lUcNd|zIxp|O$OdS>jdA-`)<Wmt2Y02*?c*l
z493@GW;ZevWhM$&9y)zQhsV2o_P0#Vu8pocWCAvAyijQW*u-z)OohH(ew)HJC1v=}
zY!lHi)Zq#inw7Ol-b?C-q4_>R)=ou5|2K2W%#OMnF6m-abeX-)-I^=m_R;v$TRgQH
zW=_}=WpHd}z_FB^W1imI{BCba$vzfuvm{REc;TC^B{j#(<{Vem+LrXjI_=JJ8{=b4
ziYK!79QV;a)~>Uy^uNvVN}nxW-P=5(#6?7w7Omc1|NB7PEB@$PORH~hue!aRxzL;W
zBL89^ex?T}%T^y)IcMp=1JeYRcch-+$~wh%QaxCLE#FP0tIvA*9-EzE!OJf`S79&T
z8NXPC>-$2@ZC!k;lV*6YXb>-YzI^BZP38r{Isq1H7HVo72lWq0gvorDkzp|Ne`K@j
zkVJC95hZV-+yMT6OXM%zUbRRd=Mne$r@vSE?cVpK`uvUV^FQwFyY~Bhy3bz5*kf0%
zFQnI8_^-2{DfYre>%A;%FL1_Q5aPY~Df@!3?}eEYSlQQJl!!ehGxs8!?A~OZ3-wA2
ze72V|WcEMF-oHgkp-Hg!(X4q-xz<1XA9e6e)#Z;}>#TfFZ4)`gv~ix|MQ(wOmo4^Q
ze$N%K=D@j6ku_YiC&>Rf$h3N*nB>*%r4xf>H!xIB)VgCMGdo64Mku#gPK?_=xwl*-
z`tX;-8?*Lql$$MEGwYmes$57!tL(@89>F5ngPS-$sTpdkEKyYnykN{S)57idmdnvM
zJhYGJ@41m@v#sy$#mEJ$6Bn>H+TNV%dvj9n&1rLQPOrT=vG&;fwXBf=Cp=eka@5|E
z<-N61_ZExn&B<?X&62%6SNCS50BfWG=hm|~x8&YlfA`jE+gt3i$CPu8OfM>BXl{R6
zb#l#_r9wH|&(&_PtoFWCd&(;Dlw#tkYq~<$a!<~`b9dGMGuyQuO*<0KUDT=cdSQiB
z+w`vAN|(*M>?Q^8dvn^-mFv)(4c}~4lT6&0j_>sJF^fu?5t&q9VZ4iFU0sff;2ULu
z<*OgWUS(rvR7(!xnp`EQa)4EIouI_K+L~F1IfX2Wgd|lj?k+S{TC;$4Qh<b@??uhJ
zi(30GtgyXV?Q?sF?c?pSk4@?xZ`XY+$9qZY?qlh{kL7qD+s%6{*?U`I?_=A%CuaMe
zSlB%d6yWr)yB&Dob|Bx=ZD${Iyq#>P$&ef%%v5>6XzjeO57vpsJmbE@&Hd*9kB)$i
zuD~LmD|zoO7ur3~ay<3c^I#g^LEaeteGL|ZlT&`QtNZuP*I(gdBe6i`|DXAp;tP8I
z%|CGD5ZAA(eKQp@WV3X5vNF%g$o{tbaKzpqbK%rqGnY)vOf^WE{;~DDh>+YxGeI+7
zXC+qK$;W2a9^1?NcqZ@dE$41+UH5v1?2Uc*UhkA;WqR;>tL>YkdACo<y*cIg=8WIX
z)9>D#?R#@!9cw_q?Q7@WT<3cm8Sr)+FKb}H+k1TP?%Tb4#P@EeENAT*dF`w_A};(P
zUQ6enIT<tS<m#SNYt|fy)_T4?XX)>}Q*TykEm|XXyZ8OvH+SdWS?a-i`p0wbi<h~a
zCU=(Y+>y7+WY3z@d(TYYU6X(3`38pY`w!a%V-@OR9aw8;seW$}<YZnEX0$>+{>&oo
zx>l{TTmEY>BtPN`n32pL7wW9~kc%%Yne|+3%0`E2LNaxd&*lg(lzjAeiOrjao7{iD
z$jg3ps(X5E-o-t&x07Ptgy}sEv3nY3_f*i}X_Ec7l=yF{|G#GFf6ILTHS7Mjdv@Ql
z|37)m_cY)B-GmQsnKpcF-gCa>S{vJ(13%KQn9aRx68qzw)Ot(C=T~$Yoc2zwh`Z9}
z|HERfl61`k*)w{-tX{OPi&o0HnsI%;_{oX?q9*Rxy5Wsbi~EX63y+1gN3CFfrzaLY
z>0{A?gu~a)ZM^nl^Fwb7kN?Kk&lJZV5SCCdRM1#>T{_CZ`_0Nj3t59cygqmT^@aa`
zPU^iqVfW^m{og}+-yY8Ud;5Ps%j0){@A3V+FVFhu{@*8d|8C3wd%phPOaFiG-v9lu
z{~r?rEAzr*uk;l9EtWo!x*L1zKSSX8`3I{0a!9#E1SC4OePyvI(QtU=QYEY#)l(6;
z*rh{UyJ*e~LuHSNiu!$T)EN}_w*8IJYngdLes$oQtgX8uSI=D=mf~?yWpn2h4Kvfu
zKB=fFX@NmYZv0WXs`V;lW$4Pdty)vBUR@i#dDE$`(42@(OHSlio=)wK+Lq0_q4sp^
z>F8Z0&+`^`hv(nxth#+jRaxP~vCfoda@%H0e2igSs<y4FX5*P7CnuOj<=I@^*l<#H
zM}^u(!!HRzlR4+R)fVz(1u;cM-%&|O-#vTU|H$6XMXHk<JoZ$kcTQ3jDtHjs#Lu4>
z^RwvL*}2-{0<I1%F9TP^w+q=imAwwwR=-{DuJsg+$EWMf`B|H)z6D%Sj@*CmZ`EU=
z%_)}_dG@R@Tpac{zFo8|W8v~?RXlH&%D0H5Y&gg&t8?NYyYv)|rISo{eNdUK_e$0I
zkeipTvw-6!4<}(o6T?G%qHC6_-HJ20AhtQ}%Stz?{4E!cEiWtKTD2~sGHbO$waBL9
zij_P<I$>QYlRVV>OSn#4JM*Ze+xN+%MP54FpKLnay=RT~)?GV#x_NYWf0-m~eErE0
zQ}d%O#(w8^7_8mU>oIwS#v!)q&F8z5E^}Pyy_MplG%sgk$A3{q6GK1l&LzzzR-Ti)
z^}0KCw)!{doVXlv$yd>J(H;#a)&47itQ?}8A^yIlp`pRDQ$r);drd<%qH;~GeTz@)
z3LBb89$lfDnYdhTTAb901u2|T8IkFtRbi1C?o)yp>eW6ySDtQGqj98n#oBMqmRoC2
zE}eQ#cSTHT>Z<l-a|*JKPAiLAxpHOT35QoJ7G~TMUpfEJ+bpg1JA`&5tmSEY^<dK>
zsT-@>j#_E!Zn^fzDQVl2S&ugK+`kpUu!oDaa*>new2tnc>$hh3_x>>PSv1YtZ_N`c
z&ckPv!p(L?E}Em7<-929l)_>jJqv*|=Pq`YecoEYDec-27#1D5=-Mo`od0HXbq?v;
zKVUi0vg2V{sGiWvuI_dFew|u&&f?XU%cZ+JOGFiRf8W;3V7L3RbCTVUd)@wd94e3e
zcKrTydj0-?EFu#em;`$^9ApuV**O1%dicho6QOf7RP$VWHj7)@Ugs#`GZ)(Ha><`x
z){jj$d_w6d{rz|3%Z??uFApnN)xKP&aL@HM_dedetFE>*#wbB!w#~eXr;}6URz2&U
z@=EhM&uO94#_QdSU+C&`Np9SsqV#L+hxTO>X~(vh1na$e>J;zIcqe@G*`?2?=cG*4
zSL)?8|9XD1>A#oTJ+fI;8*VGE5@TA?y2wni_09>-;JKSZSkzVJULHByROEcsC2p!?
z=ZaPTn+qmM7i^ipYM7EFGAp80<DYYPslaNnr?bK~&oFhC+?Eo>`|_G=&?e`pbKHu>
zcZy{--Ei*ZF>G%!RN8ug)q$sSb^D7X4xZc_6B@5139nrtyC)}*Thg>O&Y5xXz9T0@
z5`UiP;n9vbz_;vJ%hrpD$&JZjS4<4MOEr@vF5gI8F~Pm_&C-53<|VGn9vDvko2r{(
z>6;)Dx$&LmGQG(weX<lh9ZqdYG@5<!go>kb!v{_UllV`<!Vg`xc51ve+P1=7`<dy)
zHLgzF@xp#fn}WA-`7N1Nlr{VQvCG@{|2;eHn50bIm&J$vuTd&wwVL3hRIZeEY)L6k
z<+7$Du8U-(REn?vzjA`#H+*eXg<i)j?YU;Imgh33b<BM3GO<O=oNe|VkEz=>xJzez
zv6i_S?C$@Ib3#|T&2!0>^KG<*96m}0)GK9rT|FH4O=OnsjH9hS;*$5>_>9G0S(Yq(
zb6RNjz3qWT$2$FbcHTdALgD;@58ZXk-gM7-@_Zwcn;XMygV{6WGA42y;?3K}ljx_-
z$}0HFOE}qK;f4rDo#dIX71&?8Z|PeVvdquh=aEa2n_be8pRwJmoaUTh?6)}E=&@wA
zLyVHC?x%^DH42qqdx?uIopEi3!fG|Mjh=g+bm=ePcpCrr(CWZhPLmsW&#dS8z538C
zi5mx*%=(&U&WO4cw9#5jsJ~xV<^ORLwo~?DVGPYj)|}*cDK_0RB&BC!_X(Hk88dxT
z(#2m!rl~z*o4arsPwEGsSi8<Gi+a~A7RtPJIih&hwFggJg)9~FHZAB}+v%$mYu$TG
zWviKg<*MvMclsY@2`yk0?U0kLn)@VXd+d$O?Ym`_9&OhuiT)IIMEu&WRjyfq)?d6>
z=fxeJW?z%YU%zDg9x+W>v*69n@2alvzj;>AH)N&P+$)FVj7|yKHm9Gj<9lD}CnVP*
z?PH@9_@lyYl91>6DHmQX>|~DkwIRl7jjdqu`Gqwrb=d8eC4FqwG!R*PDca)Bz6+~P
zZu@@riTmRx!j)dW41Z4N{(oZPV8N<p7LxsvQTYE06ShTeP0JM@`Zq2;G;2*$>y`~C
zH}+`hN|ya>Gh{U83tV*gLzwp7TPm;J7G&w>oXy`G#;P>?y7k1$t;rn)`>s_!>`DGR
zt+ZJE<|n6XQZF>Qs(6-ET=xy|)UUg%9iI}?6)YF4Z}!c!w{>gDva>%nv>r4}YBy1d
zmfAhd*32w<`7@`L+?F1Dj&N?C-mByK_DbXo%^-vBLX~70VbANkrc4q#$fH&tW+YUQ
zxJ_uL?!17m$G;zDDpxo@T^}npN9WFSyY5rx*Dv~Q@;_jDM7Q@dql=yAj!nI-pk1}d
z%fekLw{7Z~!YE_@?JE_pG#D#;rm}0lu-?V;^ROJ#kL|WGfnEQ@b||}j`_XR8W}{!`
za7;Y=S+S+D`q{$pJ&syQ0Zmnf`sJ6@%GX2`oREAY+&pd10`>g`r~Y^P%UxaJC9^N%
z%*8K}CR=A6o%3aJr?qymUgj24?Ou!2J=>SteUi|0oZhh@s&<y{7Lgr0&Yss=b%sm(
z_L7LGHqYRtOV@1Gni~6I>Gz6mi)$OZZO;{MTJ`LkO&HV1%8soHtcwDa_3Ljh66q^s
ztxkRXg}HgDlHg9`2QR0mS#rj$ZBN%;cw0-i{AlP!_D$J~KW+_gc)I#xjcsxwt9QgX
z>9Ws-caAI<<<1IDUioE$?%q{_WfPZ1r@UP6_H|9B_G_oTYf^VCo4j8464y1a|4Y7`
z-1g9{`WAJ8r+3?A&oYnv)zSB?a+TL+F3%BK${AI%c|XIxeXm25EPD#F>jc6-=f?kV
z3R{yb`dEUw{)S>hDbIAV2Xo5*1gkStad7NglhLlf;wpdq7U%m6&p8_|@Bf+A8nl7+
z=PmQ=39RMKlm8f5{|S<1{4nSJDLbw$4ku^Mz4g)}M9K1g<c60smsh_!Aedt?ZMNVO
zO~<Lt^CV}^<J~;(jpRn%PDg>y1_IHBU4h&pj~T*lH#qK^d~wqXzT1<&%;eVMT-w1s
z>--9_%>q0@mpIgQmR|1Yz2?n3C2Lp3NiEl%yk#pE9`)WMv1s9$i$>EtwpadE{Gh%2
zQ<U+lRqX#Caq-Pl;-A;m6uO{2)S*cyp*7pHb1Sp;>ndx}M^4!b*#jGzLN|z2Z)jf;
zp;X<l`}-uRkWbrPN?2PKuP^(twcgoz$!EKNS)GpDoBi+X%DdE}>Z#y*yHk^qt6p*O
zNo%fsMqM7Ox18C$#k6Sgnd-%Ule^3~_8wu{8W*!Q-bOM(#4SYmSeww~aBG$B4O>(9
zm_{=@#VqEzx{JeS6Gzq)o=g*7Z=qSfN>=HI4TE+~cd&H&Y9-)3hhtyXobKki>X+vH
zEINK?V|%%c{)a53LnmA^wQN3~T4v|XTXmr+AW$)XF?-a6j?x1Dum}6v6BY&vuo|x5
zY+ulB*ui3`usSfJ|G!}eN0h+q(ihXJc38)MV83|LqOZW>=c;z24aWbnd6>5xv2vc^
zlj5LW<iK-tVWIc&s~dNSNOIlYDSIox;-28n1ui=mnrPGo9h|H!RQbv<XoHZVjqto?
zuQsij0k^w#Z#as+R=K!|dtUbZX_>+cx_3O>EPv%jlhFi|;3%`G)nZyMy%%P8m>qCp
zyx?>vs&)ON^&cN87M?QJUDf*elCjXwRT~bA9sF#k#$~f!$;V84v(QBrjT0?Z3l=UZ
z;`k)Mnwiud=)tF3;T#xYf2M`Wb?3Ce6SL265i7ef<vX|1@fRGuKek^J;9#5}7%tFJ
zvB$bIgS~pe!lnrvhk`jbU7BY1`0Rh5GYfNWDsI`NxYpa|u}H`5f;~GPb?!;(+I(g8
z%RPIl61VtX-t*?op7X+MR^4#zWIUfB;vZ;i`^jZyNU=_0jO{iX-mqY|wJs{jGBe#4
zO}>BGwtK_*tDk0GS#`{lt2bG>N9m8-a!F<-?gf_@NHK4mTfC;TMP%|F!G@okSfe*E
zM;9=MZ(!y*dVzB*Yq`&X<y8)L-P;A0+8>Q<{}O$`aEEi%hxR3t1gcIjsR%H4{5h^8
z!7AJvsJlYe=?cq(Q!E@OS~M2e?R=ssc6CmpwL(X*#f?4FLQdI#DQaEO5%A^GezCcp
z@`nt6>^^Y(=X$Owb9xT%5KCk(KP1bNe6VWPyk(M>|5tQuP|Un6R~alaC73h#RCAGI
z&!NkDk{fN8ItsoH{v^2Z(?xDkZNvVX4R)^_4H;)j2G3t765yM|EUUd~Vpf;@VcqT@
z^EIw;EN|#_(AqSG$!w>yg6qW7fgaP2%IyAZH6?6Av)P7bqXOozsjHo5Ub0rUu3FK~
z$SIJV6lme8AH7t|W^#w>LL(;5MWGkY2r2c+9rf9{XB|^wd&`Sfp8}ue4Tm)(SoNY?
zB}7}9I{AW4&+05-bz6BR@C0A##<O83oI_Z*DF>ZZJT>#?nnf8`U9NDPbNqZHKxd2J
zlxYc-)17yARwcTIwQTW>=JGo1=eyX|w_3vNm5JF8zxXdlKdK!2pSa~l*V+gLA6}QF
zV^P|(#IxN#Z{%K>8QyzfNy6eK4=0;3{;-{@%V_kWXU!KC>q4`q!m6tpXL;WZITUaq
zbW*QaWGjav&-st1nHLA}mNqXjZ5GmCHZEx8-pkDUl#_We$I9FN@fUd?2K4zjocM6~
zwA?E-o<zl!ulqw2+F2?)LKW6M6qwx^W3S}OpdrvKCN-fhf%R*_(sd_$_g8ZmMqG@$
zz{1hd^1WGxRgjgXqv?Az>pZ1PlBul^BG~tzy=L2Tc16eWDv1Nvrv$ik2h>mCTFYfo
z`Zuy|wOuENdhL?!mpVOVPn|5eWbroZBIBNz310##oKz(xn{Qvd%k-+#^selGff$3y
z!f{JvG=y_3wK7jF?>Y7KtelLGTj$#)E~);rxDUN~dFqGZ<SNlwmsU-_RK#0(NqAM_
z%%2CR_ZkE+o`_^*TC!tHM~!jl-8<9Fvc>ee#Cd#gEolwCv1sz2y#dx4t!4_LQ&+G)
zRp{GuFib1C)#S2FZj^rVLe@Y5)&#>NQ&t?#I^CPq%ye!_$5GjJM<iS8uQcsSX%D;@
zm~MDBpm3??W#3K(7mXEm7tb*1m1u>o^sW+MUY;nDy^`Hpr*D@G=ZPQN+YfB6pFO>_
zmg7V~IP*$tmM-x!j-)!9)j7Y;a(z6@{>1l}v(3@ZM*6SW`=i$f#Y}4A<ht9-)wFxA
zX6C{FCf^)9l`TWJ_`NmJIrG@h!cM}>Ld8hp{O2o=oo+q;cx18Nz6iU#CF#}vb_+GO
zdvLGavo(Fq<o~*nj260#56)kEHS_W>TMw?4A5(cu`gY8zwKJ?#pMO-hXO08s9FB@#
ztko+n@c7<ZQOaywk!sxVG|b?%(T|yW8qC2HmNPTWEnB%RDx&q%ffI*JIc6nzwCmih
zTfypfu>bij5snKj%)4&$#Lf9t9UW!B7g)h@!ZJ};V}g$ZgYN;))PI4mg4OQ&OxT^e
zzb>NvYXs}bD>B_ZE$d_xziS*>(sXf;WK7K#U!xnHMOQc(Efu0y%+2%CV&O7xd|;PU
zcA=oNLz{Q{jmFsj{|?8nEV)}@vw_)K;Ql1_&1VEn7Ya=mekr{&^}gxL`)k!3l@40&
zH4yTdzjcMsjJAWi3nlEn2HP%u>2P$Cvr>q|)>F<_A-y{cU5^P#TKBY6&6m1-Z(&;R
zJjt%k1+y5+W*uxe<YIOIibePF3E~fwLc;`DPw-shSbLvmEAyXH=6|iB;X7EvH&~xD
zVAd;Op7uv;b<m5b0{+6d4v|-)h82ec69VgbMAxNqmPOprSaBx7jcNb0r0$Z$lR4AG
z)F#v~@zDPqonJStQzB7?<5rbM|H0h*^-Jt+uROD!l31T1rLy6T=(-rj6|$>Wx6f7R
zR7eu*xsmik_wdr4kFMO<d}oT{|G&*=^x3!@9lI|4K6hcYgmkp9)xkKw8*$Rw%+H?j
z{EQSn7$E#AY{}DY6PHa`9+ShRK2iNpm2&IWqboVMx*1bKto?gJ{Xd1g|Mt(dSM=4c
z2sf8+NBd(_?qsjE2)tS5dh=|-&BF{A{Q0cru!{Ki@%q(iuE=BMxT<w)2XE`KW&ezs
zgLhouY7dP(R<NUGc{xwU><xCgRm-|2Tx>P)-MMxy)5kNCv1teI?z`qF>b<g6EN)6*
z#zGFkvl<;?>^q}5BHHUNu&|tHiF%OEXmO`@p1^}H77l^dl4%_t2GO%RF6z7}%v{8p
z#aon_C1_ZYF5Vw-=4;W^j6m(s<NC7CXO&F&|1Nm@Ar--@CC9an-r+c!!9Vpv&$hc^
z1*~^-o9AotN*#PRJz04A=NC`Ey^ySY_uydO>vzQyx4koEb$l>);=@)UkNFe39}3B&
z#66oDvFeVF(^AW2e8LyHcD}xPL}kr1xy>QLtFjn&Wd)Q<?#Yk}<2ca$AxkyQH~PR+
zJ&V^d?N{{-nyps0#!m?U_32ZZqVw5E4sG_=jxM=+N7BQ7h%~h%N`{F!i8Nd)o%OKe
zu)sU^;=ew(%Vq^e9>`d(Wu3A=jZs_R-POX_sb|j|z8-QS)L?>f%d|6%9k<NN75Ua1
zzwte*c-YtFv&y3EL{@9F*Zb#iynA8D-4|MMxMN0tQia>Cxc^VhdOGU!y2A6i*Z;6w
zIr(F#cc1&LAC>bDW=x*%JuB#gjnz8W4&QDYv6N}cZ+y|Qb?|%d@XawfrSx3J)-`Tx
z_TTt<i)rQ24{;uLy$jCg&-*C3K#+&Qbzh|S>8h(&mWwYio56d=V}aogy{mN{pI>DJ
zHF$R#H0!-!h^xQFJC$Ps%ZAGzqjlb~_bz$%>+-X^3}VwHSdX}#IHCAmGhOieT@lUe
z9vN<D?|r!~@vS|y<Mz5<jxxRe@_);+rCN0aSQX<Z{`%a{c+<{ozr|(N;L9`ToVvd+
z_Zdg+{o{W&h3?;y(Y@^R<?XEDJBoLlDPCacEyN*Z^uyrrjK7B?RFr()ePMk0zjOH{
zhwrsc@Bcn9pL}2bUj|R*<Mwxtp09Xgdgxht-TaI%?)q_)H&5(-`SSJlk~!Bhg(h=9
zzx(p-eaEdg7#2LsTu|4oYZ0<FVdJ7c!@jJILctnRv$;3y(ch?3y3T%AuScn2=B!v2
z4n`(+A)OOTdk^R`N8Zd*-5|EQaTd#!&*}^AyE6WJ!&#tuAW=JdTi#Ty%ST;in&-W1
z5lL7rd$jIr%B3?A0SnYL0y9FFENAtc=9ARYm2}i^+5W244ePwD3PMFxR%-?MzUsew
zKJZJ#%QGoku4x$tZ0ebG()5(Za;e`}PpK@(v~3jQ{I<A#OV-u8FzvKE-Ga)ltIgOi
zpXIn=@bG_QJOBBsFK#T-@7)^b)N-Ti%G1-i9uprrrFlQUFwcGZ!=PVVUS3%dzB=#h
zE}rGrH~OA$`+I9^gVE0N*ZW@1R#@J6kd;q7V2{Phpd~CS>RT<1HN!40_nuxCX*Jt-
z{!X8V@Ag_(ypP^Dng4$6o=UwJr#Jh*k5dq5{K%~p@*;BKpO3$;em*a6Z&LG#Q^P@X
zw`I!&2M)0(6Wy6CzeMMyOcA(WV?I&wLeXi*9g6KNy-F7(m|8d%wz1}3^0MW<>Lg&T
zc*-R}pXsY{vi{dAFS>2><~(U-NZWC6f%IC7FG(?-nq?O?JX{JzqGu{Fw2Hmn;B?;A
zIBKV#NPthNgziKK1unP$t2YFD+itzUEGm+~b-+t4&}E{lN`c$W(>-5$0<ZM2O!efS
z+kQ7Cb)v~xi^~h@!e*woSDi~;+{@(h%-go#>J@+A)LY3bb-$%#b!*RIyXmH#J0-+T
zGNr^!kuzq=tXC~-SDk8Iy<*>~gBgoBUEB)hJlM5*-QKoq+Bth3MX_maD-?^~c$De&
z>rID_9a~elq4eXM4F|Xka`HB&YCEWJF#Mj|C|P$}QZr{lw%)F%tFrah{dv1xUR<W<
zIP2c8x2pB`{e1L$T~+jl-|HEoa}*x1eZF>EvGv!z)12aaFU`(y-0P?lpD7@``(|rS
zO|`J5s-Mn*c23_tchnx6eVM?pb;IQ@jsJ#!+pfF)+mSR$l4-_;R>{;IfnL&ryx*oL
zg$S{T3a$CDwaYKzmFeWzmP<^mBAKF^4AzkfipOT=XRT$OvS`B05c}6_lul_Zy5Pir
z!}jjc8E(eDZ*Mr{nX83|?%NwNVWGL{Gyi+l{3+euYp*C-XkW+-%U-)+>9w1=y^qxl
zvwJ3+S!PZ0erNG)oo!xJ-W+b0o%urkHFFB?vpttdTN!PBw{*r!u3MFlHg1=DyZN-7
z^}9bi(|=bpcgZN!&1bFI{qBEn`P;3cVLt?$c<W3qG&8>xvvoW--Ed)z3P*$OCzgp1
z?;L+9Y~prOdfNVv4)WVR3w_ny6#O{fBuw(A_S_PM36lR+O9I)>H<@b%^jKWsU+1tu
zr&P#W?2H?KRLfEgriJce3k=wtY+XcB425;C6)q3#N%At;G--j8g)rln%z(n(9=$xG
z2Y7o8446O8j?q{uDEX@?X!<6_KC`8o6K^Pa%zqT=c_LsH=hUvy4OU6)eJ`}4j;`YN
z2}u&)63+DgZWoV?$Gv)qvb7ImJnVLSQaWrSe5|Z<V%$NeYa9Ap*O(aj^`4AY-nwn;
z?DL-M`%1n{li3?O>4C@!3#ONk9gM!%KC5|ebU%5z!pp7meNS}9_4Z6Z`0VQm&s*A)
zzDn^Q<|&)UoAb%0S0qCHs<FXWn@@>RCMQ%>HRY>&H%@)TEuY0=<ocg)sbBKxixMYT
zgeK};y5ubWD0PwD7sWpY&s8;|Co`BNnshVxx^*1vIFsS#&hF@CQ2F_6=prMoeH<ww
z`BL52Tn_eqS(_y5!Fen8+avM3UAeh6KfSH{T?EskmdtbW65)CHY=*n9TS1HQoR#lB
zOJ0oeD_*MjNRWS)lu}-QaqyM1I@^yccK$6=5ZCH`Dwckscil3xRUb2+l_&?@yHdEM
z^HcK6j>$ofr@V7oX&m#)Z1bGAYtqZja{1q%2o9ce&%}m#p@8EX;SCG@5-VQbJg=p#
zwczIqHpSF{3l4MK_dd6DF=rLhba}i~i!tN9Q}`aywm8i&!Q2zctby_m!mf0ct@xkA
z!?s6l?xQg0Nq5e$*crJ>=RE9oioHCuWTC5?P(=6PE6UBQC0s2t163sjxdM%t6jPK}
zOPu6-lKc5c`fY_^L75<~_UXUY@KxUElJDvYS*o?DCGG2~2*-_`YhN|}OX9w*$J4US
zJn)`>@$Xe3rEDuil5S{*O3i=JanT~BL~L(GsoPT{?}#Lkux%AvgC5lQMJv@#%iR>(
zmRH&oeNQ)_oLeQ1X=3<<C%@#59dYF<-}pU#dd>PnBG2Of{^*|a<?nSdtC{ito@hHT
zAN1FJ9IbEEcQwi3ZJgt-6#^d^gBf<{7&OI4ZsM7%ld375&c5^J!mcws>1h?s=b}Dx
z?=dMh_+KpDaq88=#8M%x+@8`?S5`2cU3Sssu&P+LUGX3D=^MVzH!SU2$s{FpG%;lH
zc1Mlu<f{Q4UM7ozYl<eTYMt(!6k`7IBCmeE<<1^4%Ox^@iym942c4T)(w8Quo!gvz
zd+NMC%TtXj6^pkATMFyjY&{+8U%4v%1J4tioqy*B_Ik<vk0`O5E7%~N_lRqSQ%_aI
z)T^Rf*{{u)6FM|)-j_C~Abv%`7SY8XibsAWatD<#rdQ~)b_LzI{=+w{jzMM0D#yP!
zM77ddKdjba(3V#2e)FqU|4$CD_S2gex>Vd3^NI@iYbz?Qo5r)IXq!;t)mD{h52`Mu
ziRLW*d+XM{{8LOb{(tj5FgLm|`eyn5z<Z+JlQ-P|cg@LD^<81~%sXLWt9yGci{?J7
z%{TZ|-94l2PJ!Dg``d3OI#k(M$ZyXsu&bJ`@I>nRjuRU?e{uQPvu@U}34ZC}t!4Ch
zmPo3GHs7a-&6ncmO!05f=agr-ojz&xk?=kGdlxpVt?HgpSa4$B)X(pFnPx0D&_0nS
zmDr)%KEuW@+ePV+qonwgBSjnCwR2v#34|B7te$py;<d8LWxLL<|8(Vf{+mqyy+3Cu
z{0W)ZZ+@67xR9-o<*$)bVDppKch43EmVI(qz>}KqEjg`I>SgOc<po{){=SW6Gk&zU
z=Ev2LogY)53A_x^s+{IGuVjVsle+((YOc)5syjb#duGEg;}8e&ugjX<<C?jrEHLvq
zZ{8!;;V$q=;o5YCt+#$>HSK0EnfY5SMfp*LUj1gfZHJy{&NqAL`Y1``kW&lq8in9B
zb9c^|eL!FD)PAdVo-?+`IcwdJV%(Ol+qyXV&4u9OY@wN+N8empqjk%(Z2z{q^P}F?
z-ZtO&_3mHoEomF|@>hPlp8H_$90s`?xf`z(tQ9KQ%3WaUbJ(Jr*(`js%mc=-xKh{d
z(nRsHsV!``+_oFcsrblW**l$Mn)ngUAhoU6)whaQzvZs^c)j}J_1dT6>R(Tb{OIPL
zc1NV(Qk`Vap+DCHB+s0F=q+KW;Po-CC8S68>AmL-|Ih16CYqZ{)GqN?;B+|Ia^}I3
ztF4V^Bo_3hI-a=a!`6AX<${H5TB2+VLt9^BY>ElXt*eLn=T5sVDfv`o-ZO(KcOS~$
ze|YzvtK@ylhuXbUK61=cSah$p;^MTEiH@#(viI7a1n|tXeKe!-oJnMqpqkgShgar^
z_Rp=AQ!ktMQgCY3j|aYQ`f5tpKiT+v5^(vlVxEJ_E#DoBJ`1>bp1BzI_gHvmw|w21
zbH0h@jUCGhZ#h-;CYPRwcL}q<AeC@omiV;0Jhoivo$?u(UCT<kv}?FF@y-!&l;(7p
zy~c+_IQ91L(A(Lnk@q8~EcqBDxui=->d28d@p@bi+eKYm{!aNe<^M5JSspt(9%ioT
zj8g5IXQTJ>EZj5oVzFS#9@XnwUzNpK=d5#?UUu}D9IuRvkjx>;m<r7p&lYiS#z|g`
zmjs-IG%F_^ec67jG39u!qo(Jiqpf#%nuJcwjNyH>fY0wBOUIH1!Qk2ghw>xd0!9jZ
zcU-dG#<<zBXXS}0%~M77f4P<1Iie?YH=wm=0>in;Ox7DUcW-F2T@QWQn|L|7<Y{Qv
z+{G+@sUqk4StDh-jy>9X?2e{L^dYaiPi14CEzCdj*!tAtBU2dP1>W>+=$m>rygtYM
zlgK?qtwoO%<)Y3#%(js8bmcx0vS?PIT#DPh+|)(tWs6d)9@TyIdm-!ZI7Lp4>A&d1
zN;c1pN{%+7@$cH2VnpBGyyx)Xp6K66-d9zF`}jZnWz+0&^w@GZ%9UR`gGqPBgs-;O
z=D&z?O<)PNSiHee*`zJZqve#Tn707$MZvA{7RTmWoVsrzcQOCM(*Jko*FT)Uq)@cx
z%Hqta=|{iFJ65DS?s&X&ZF=tC`)xT>^It00K9euKd*7WcWgmx%gVh;-vjq-i>M2oI
zHr;Sz<BM@o(_q)jC@7eIuvejF(slWbA$7l&wys*BdN#c7;1Zr|34B#6_)a}(KbKOw
zc6JSG&z?UCFJ5vNTz#79=#ejQ`RU)72u0x|HW9f_Jr@HN`{#LhuQ}1x;BoszjO|C(
zB{vSGeEB~kb;8n}TN>BR;L31W`S76ZRHJ*cpB~;ls9HWR^0BYdYu~xr&)UxII3zpe
zQ2!0b{EogHtHhmx{+dB2K63agE^AY^Qkt%)Ht*icxz}Dk>X<oyUR#o1(vJ=5|M$L}
zo0&8>cG;pTjq*3sMRHyQu9cb7wTNNDqVgE-^ks|k1NybSd{*RmlvvH4`O>YzFLCFZ
z%tZyqeopcDsnMdFBV|x`#N`L`TD9lAH?#OWmH9&-|5S}OGvYJPnrbe0(R`oS*2w6b
zo%g@y@IN^->BX5#vr=ceUVOZE&f_gznU()STnZ1&_!VF}tt<0G*0!$mo6kLUIHK&h
zV0HcqZLPJJY-c^$y=ivMoc}_FnoAl!Jg<ND+EHkU|F>+3zc=?*P5dJA!lkUc$xr1_
zk+*Bs0@q2?-RyK)9=ty0tFvcS%CX8zeqHX~d28CQrL@<*X@3^(TNLEGDcpZm%KU{H
zfiqQhzsy#RniVX#Apg&$CPf#Q8Qq5+J(}OnJKqs{@ncZPueRuwtx{}#T@#k36|ha=
zyE&!mW|Z2LXg|rxhBITnJ(RZ-6?wOG=B`HBn(R!zBmHO2Z2jZ0h*Qv8g~MRwOW&NA
z$5-f1KPMCXmTl3ozE-}gbGV*8G<`cg^-7{&rjlJ>l2=l@huTBG>-iVn+|gV0sL4Ie
zY1QMZjI>u%s(!UCohN6oQLf2A%E@7>bk6_07aby7W=2=+d$#q>yqtG3KYPy1n!x#+
zqt7AZ_)pvAn{+R(R+;!#Y2qfen>nU}?F*JS2q|PR>4<7xeK4_M<~bkTpuoJ4$7eqH
zm`*z+H?46;)v=FeqMs+}e)=W9(2wWu(c|C#-G9R8VK*zI{$vj0v-@?gmY-ggU&E<U
znz*1bFZ|CjH}+Hdf77OKFA#6C%i#Ey-?AyGT#$YJ9q+cJ6^y&ZE6zOV&@)gs{m|a`
zhUt3B$xjLXkGfASS`l#c`u=^&LUnJKJUpJJz|s59?R?oSrveZ0x%_@XSvxb@+gRQj
zc6Nvutd+4kcW0+p^u2}AuVkmaQ*_K(IcwR<2a`@+;r*C2>3`<R8&9W9%x$P>5fDAn
zFOei&()H`-h2X7+WOZ&OEqWEvc||o(Y3@v=@`{PICss-ZpQwDMt8{9iqM6d%Ya1N@
zYR>G<5^_56-Ab{w<>RAIA&-spo?LsdnCX+4&8x-tW-dt0I#snyV)i44*wxy7V&K(r
z_rB>!-(R_MNv6lOBEut3WHk4@`Ydon&Er6pr}S3e*_w~vwahVkXZgDNoQ$JKl&+iL
zGSAl)T{l0^OAeP^A8_q)*v9z0o>To>9BV?7r7zt7bjwkDiI=1W%O|7ankh}PCFfcv
z==x9d+VxSZiPun0Z}o?JEW4G?_#b_JIx^?zzHr92dwc&S@B5|gAoXPbqyO3Vb?UiG
z1YPgt8$8a>D>-e~J*6pbiCbL)gSUnwvq9xtKk?cW&sQlPlVpxtnRLu+KYl6Yc$mq_
zuq`a*?CMj5YA$pstEObHd$U_D<#e4O`{L!dn%Wpz41Tf5x0eK*__AjC%S*yt=g;>x
z9%1E?-s0l8g-Oz*LHXVW7vJ3b4-+kmK0dz7CA|CX71a(8Q&Y(w7czDp(w)F1zC&ql
zQ?BVH&6&S4wad8lre|t%846c#`0*pJ`d_SP?Z$bstLivBBv*fqJXARG>)RN4sYfP!
z9xIQ{XM5*sp*EY<jCI-PwE@rMzF&E)woSqNAe+ZQ5f4WHgY6RiNlU|4y*Z+(n6@_0
z;(!0MwoU2vt4i+8%fFNSvS^dYubhr6OdoAKoF^P&dGjH#d3|2$`EsrIR@2mO=6zC$
z<EZQs`TVJW1&8WhpN-NS7ek(#-ONhJGZx(B#qs8Ehpo~qcOUie+;3MxXPbKG7pZS=
zcyVmj+L`YkiP_d0CqBNmddmaPZ3_Ht`qsD9ySXH49w$2*aBsZwoh>Q+?<4c(Q|teI
zGynf@{eQL}6`kJe1PZF(E_kQ#L%#U9kJ%2_;1V}?i<ZkvL@M4?oj=a~X?mB`+(S7V
z<Y&FVt;Hf~b9|PGD(7!E&iMu>mKe03$dI0L=BJ7JgkL<88V9#}@bLfM5ZC5+B*!q!
z)HKn%{~S-{gctvh_1`(Pu{5{vvw!Q~4&8{EUv4pJPdlWWqF=-CD*XPzfL%<p<{b;3
z=_Z*{7+n9lKR;J+|JqF9rE<GXKQ1`_;^F+e^EN8ax|ndy^6?ZyspmiMw@EHY+WGv#
z$Jk9GFIbK*SX^0kk85wpvPItw#aTWn<gZw4ZKh!VjfJIX_VbTRgVxPvQu{b-D|`97
z^zS>i^2WY9+pe~}(s9W<jU{>d``#U!aPk3*W}C+WF_DW7CpPKb|D~%uBd@$qXYxAz
z<+pV{`7d$|)vUSt)o<&R$&Q)@UP7OZeoR^G-J!d=yhS?as>pM;)g~g}3$$-e;`)5*
zxJaq4;L5lev!p%0&cDspal2}>knsPO9ap#I?alTI%Go>Lc-Mcu1AOAYz1G+qH@5yi
zCHG?6Mu`_w3z>2~y|<~<s2lug`gieQ>=Smsn*V+;Dtt3~gEI0pO}yKJP6TZ^5ms`d
zeNW4Do7Qxb>bVKWKR$6^zJ_6m#_7$M_B>UZ_FwkZjY(bGcb(^$toO0TC)uN`o!{sA
zmgbNdTIUMP)>-%by8HZvk6F{onQWUkdvBNBkhDC;V{?60UICYr#m*V6Eo?=9LwC1?
znj25zciF6eYpQ7Bl{!C>-L=mfG_TBJIa}KCV3Ae-<i<tS^Ja6q1pc($^?2v~XJ2Fc
z-K6!EX0X0`a4o(tCNzcZTdRPTx2s*HeA(h(8<Xt+TP0hUwHDVc4c;YWWA^-H>ArlW
zG~s#a-aO4~OkP*Jytd@Hl|J>`60f*8ZM|9BI<22-teKkC?2_$&KuqBKuJijUgzP`g
z4x7OEUB}4!o7w-u?zLspnXH1|8%%Z*VV^S5d;crJ%_}2*3%~bqQH`>V)BhZ1%UPnm
z#=~ci%Ae9*T{@Z<zaFXIcm3mneMWQozyH(=UA5vn<9B-pjt84syBV6<dBs9BCOmX%
zjZtu$a-uc(NSCm5+LI$66eRsp?b=u@Ck0QRpys(u<YuPI>7dlOV=b1Or!>qm&wX|z
z^74ao%{GEuD=G>VrfWto%dwbMvNCA7kGjKLp{cJzR)=o%TDU6oe;3Qz;4Yy#KMa%(
z>dIKKiCTPH7}b({XJ2phvV=PeQnpP|WSu^R!7T6pEztv~9%!_*T%B}N<;R0*&DE?E
z9B-ToTOGxerI@GM%JVdg?X(P!t<|OmqtsgyEZ;8cnH`iDbF=d8DPQgIwN8h(g>K%e
z9lelsOJV8iDX*f}-MaGEH@snW^SqbSCa&40Hr;HBzxOiT9ldkI*4&-(jH%{N1*7UI
z`6o?wyDGQ8b2Z);9bQ-a`(y7j_TBOI_5Tt$nGEckc#3kmws2WZuxp>8-Ot0Wxa9Rg
z&daMe?Bzav_=D5O24h{j3y)qoY4AIStafg(nv%2lf>}@?gW&z&=j|6ySX8)QYWn|M
zPFzb$o&<Jjmi-CTR&AK$(8<|ma>+xc`<H(M!|!hHQ@S4(<?njH({y^l2bPSby%*+O
z4)T6pCAF64%O$4{9wDoyH~Y+AOD7xt<}uEg9QJDGn@!KvZZMhYd1pRW)w7tPYhskv
zyt?PcgxPD&_KL0L3AB!3)C+Xl%AFs0UEu7bfCU@=c%D9SYJ+owedgg@+Y1+jx3IKu
zS%uhs$#~mz%{aVvD@*kC*B_4Th^rN}7rNVIs3F+vdtw1!LT&!dw4DjUN_<DOx&k({
zEh&rIxMgoyRG*A;$W2z`pfcTsI$q*Ri}Zrd-)&hY{a$H;0+XjvmGA8n&Vr7Qt{<Lh
z*L6~D2HXFAD|`G(s@^PbtxmF7-o|^YV}<S%CoQe+>AQZc>fQLLV%7QyJd!Ev7D=s$
z&R!YIty8!^w`VbHXVd0=PhN1Dt%%>O@^tflf!C+g!+4}yr&tL}vt}^%sLjfl`POBZ
z{!x~Qo++nzZf!m->%B46Iy~w3n~*mRpL@-{4l6f{tYcMb=DESr^xFJni}JgJMjF+f
zJZpD~Jm7o9!nJ<WZJ({Fd#?NCzBS={os@rd*4>~xA6qWiyjPfgb<rQ^q^E}?4Ypi)
zB(v~D^Qvr<oK&4!x69t*4-Q-n&6i!2rF^P`o9p>J*+cA#i#90o8o!zQ{l!1td0Q5I
zQN6eJ(M0{VNBdYRs$2vo{}1Obm9EUmI=;Bh=T((#w}{Y5rA`~88r7)@LSI{ccwb(?
zyiwP8(|6~iHa~QpvcKChYq9Rz;_tk#yz7_E3{&6QbI`6lt6lfjM)mc%3EYP&uf$~t
z3vWw&<^Q<z!JS=g2lbw^Szdf4u+VRdW_Uy&i~6GjnM_B8_13oYsAfE@O7K`6voSzn
z%I=`eJWXrF-+Dz@eOdUYNWzMNFVNsg*EE+6Q;%%W`KaSn^2q6#vbCmFijvF34pCL5
z-7j07)Xw>E!sFA^iTcmHRW&D`^k#AsZu)sdUBFu+;Kq+*%^quXL^pQ@Sxs~APjyYH
zebXhhN2yU+x3%+{(UT}YxA}#MEJ`jV|4(O#NKV(?#%R>C@kzq0a{2!|Sq%$kPLBO{
zNu1H|tkH~0zH3|DjExqYwMv>jn?Z|N?bew-o+)0pSNf$oK5BSo953as{cgdbYYlCt
zJb}G0+=6`~wzk`HCdTj<hswwL`W3NlocZ+4V#`F!NwXESm|yydpL@yKF~_%RQs9O5
zeQsS(=dcQLzdIGYzT<-HyiaTHzOQ+1&G)vr!eNX5rWx#uM0-12<)mIX-eppCdX`ar
z^KsmSUF=IV1u{JzDBpKIYTjD-Y`J1W;Qh%_;*v}U`(5HLr3E~Ap?!G*qwadA(^LJP
zY|cBSlfh!s!NA%fFz3-pv1KM6+EELY*~N}GEsOGH|F3#@+vOu}3@RHQEWY-*Yp*O{
z(TYmWbseQj%P+mMF=RP6e_hsD@gTpL%*M=Te#{pXRvnGV+B;L=`6RKeWeqW9iPv@u
zqzG)ud0m<oc*^0Qmmve|v<(bal8p{p4Ckg^Rot}r#OZ5G^}0N!#qPY_QJrS4uYc-w
zoG9nwOpnG%{WI8ljTPT|#5EZ-uAP>U*RuF<-0Ne$mp+#IUwoYFzD?AgOUScHWC;@k
z=ZVF0Qd93fI&ezpkkQkkl#P$Od_L}2xx&NfSk9C?GZTusOn!>2oqX%BO4-C~TtN<J
z4VJ8ta?&u{8fmwpsbul3oOcUWeVfOn_v~j-5zD1{f2O$?zVD2wKO6erGK=emUbb<}
zJ-$b6)-xC*nA<LUrl0b?Wa2qT^xlh~5!<awxLO<{mn?c)BrbUC?B%i_CL8TMnfKJ)
zQ?F7kn%M31c+b<b5tYR~-E|W-HNJTn;eP)y*AJzri#>dji}&pP(6-nl_IS^fdQP!3
zou8I1onFCV>-Y7})Vu04ecg80-At8|*0`*?V3j*p_ou#Kb5*&AIWMixWj@TvxO!;g
zrhvBHmqgdUvbnpZWtqY;Ep0yemr~aa+#~l2niS2-lDd`C-kyE;A^*)^58R7xMsgLt
za+wsdSU0CiVb-%R%UHi0$tsP|?KLfRmO3dGy8f4vTw=$OH?D3I&+O{S{KMg$cUS1j
z|5b*4>yLUKuKH^*MK4FgZRgw2B5$MA?lanZdDrDnWr$%lSUTaslMN>m#H|&VMV)uH
zS(dQQWNv`elSxnbymo5VPVn!WG$}<QHMOcsAUem9sr+|eWml%$OAmKj-bb+;5?0(#
z)jE=<IZeh)Smu!Wbv7HDmjSH4kK6Xh?g;v0TfwtBbAI0a<7)-wWLx`XintdS+pluo
z)O6wHxzZx(Bf9PCwx27{*UXccWIBJoeSfJy^3sk4A00CsEB%tUzH(oD$C}Hvm*t5{
zYROAivH2__3XLpN-kf;tUDV(zmAAFvrL}$O>9SdempU&zUd*;r(W&NGo9^=4x{l4a
zYL{)E!Tf8>f8|s?*SWj3oOiwJVt&2x>XWKc?Hx(QRw2Byv;R%sf9c3Fk#1L;t~l1&
zFZQsRhZ|i9_I7N#zUPQ#`S;y(b49D<n6|9FZBl!@bwyBvlf-(9l2^@sW)b`OrhRKz
zTPwWrY`9aU$wq~})870#_tw>|OeaF+Uy3-(q(6Dr;^wfv756lprMY~^mdoys5B&Nx
z>CO7Uz5`-q()-tM_;;|2yW-D`yz}dG?)|%X^W4AOf~B%=l{w9n+H}`7_5EI(X}akO
z<J^@cPaoHQI^_H3k-`mz<$u_hce(98<(=r!xO+yM$gyn0S1k8@GW|~^Fnu{|6ZdrH
zF0OhG9Ss4!3vq`(7^xr3I9%}G#d)TKa`p<YHy$a|UND&jJ@)rnTJX&~JV~QaCa1o`
zW4774;yJ1nZhlXC0_OPW9Tri#y`qEjVW*RU%d?EUXJ!{3{NhYsaQc|iv141-9NT4Z
z%;retnLL*(iz-6SRqsi!xl*ubl3$I%suPR!I7Ji~_Z+I35TKC45+lT2_ki8ht0ltW
zsI%SS@Fj;=-eA!^rFZdO`r8}l?9Qz_y^EzWEAfnjGnYlexhjql0jGAR4JIP}F;&cJ
z517urds;Wk%T&hIO6I77*3&Hv>n)WQvp8+EbJ4KL;S6`s7Y}^TE-kq3T9VG0N0%j5
zTy9&P^dkRqQ{a_pFG?41L@ndE8m22XF<<BZC6}H(9Y5C@sxEr3yzOGosTIF=oKI-V
zl@UJnJ8AXrmPd|f&i7Y!YcsvP`9Sd20^NU){FB*KqP_CCR-~?<F)?7*TFG>-F1e3-
zztThUvUXdX)HTagXnr)=u2bX+U*@u85gh|#o<m|Dj!ardX36BRsGea{_A}*sV||N3
zujO#aAw7$v#|0_MzOyuE%s8AK@MQIy<sF}R-dA0`cq8wS$SRK7PaHp$9VQvf{ZQ~U
z?fKk$okw12&QogP%IurJ>cg{$_w@;9o^M+9oTV#b)udy~7JFU#z<bv~r&4=I56i1l
zLKB1%_?GhRcvDc-b!jofBrh4|1p+;yuP^ECQL!j6<WP3L^Z%HE`fWd@{jcqw_zB&9
zxV*rbYsZ1lHGvz24iyXN*Lx)R+OX)VeKuiXRa6rWjXB(SB4I_sqrfOfvkN*GHB_xK
zlFy~AP!oT<;>?2ZfTJJIuvnx7ioacRULslCm)}()S*y73yiarF3#A`3CMKnw|2E;X
z1o!C_p6=*5!QX|nQzg8u;(MR2>e#fQxa`5xO)pwEA4}e|Bk)(**=s)B2@GioW=_Ak
zmdPAGzct~t?cHK-%S74!mcRcz<et9&`z83_3&UHJ9GKQ9S$^){!KSqHlmE`3G>&wy
z2O+O2G)^j%YUzm?Wb~<JG$p-td-ztOxuJzk$-I$=BSTniqtO?ZX&#M@|0ggzRbF<v
zlHwip*fivEh{`)tuT8E8*YT9C3~4xYc2;xRn#EId98KRXdvxe9uMVG0(1p5By_SB>
z%);&ab<WK)csGl|_-?~z^^GDL6Ak2@wRFzCyfwq%NLKohWgC3H`c~xi8BSWxz29K+
zs-R<sCTPXx`29ChtUJOh`hfSd$b?0G{<W+97y2<hG0<ff-C^=dee#6GuaZvYb@I3J
zPJhI%$Rudumh3B{e^KuI8?Gk7)mvukOfh=JxI$wiug*v7ckdK#JTVIN_!ZOa+j3Y%
z`l(ey>4kldjbFUUzIE)Z8EfmT<^(BLG5d`0)QP5fOCy|}*JgeGJb~5M#U-+IUHiPP
z!T)!k>@Zk+Yr;l};w9fWIKNK`l$^QozmKof(W$yl?mhFe^EL?na#Q+sLp11sq0Md6
zUsoo}ybSGobmI4;MOJ;<Df9UL{n={IIL-d=X1UE<b^E<bzrE6_+Zgq$|1`&1yA@B2
z<u`a6ac1l~t=H$KyXQ^P+oZdCB^k?-x!p`O42mxASmu#9rJ04--0!K|`vjA+$NQ?9
zvg{UHNiEqcw)9ig)uwk2Dw6AXkDT+6Y&svp<Mr`*A|wC%9a*zxe0wi-R(YnIdFkZL
zA8Y1p`I4o*M&0y{!-os0A@8zRFL|1fm*ag#^K6-O*R^N=CWx&SdU4Baa=_!~RepKD
zAG!22@gDv3lEIYkzi31L->`t^0gOxzy6cy7T-B@<E9JL*q^GWY>POotkwZrRdwf-t
z!*~Lsl&56obr=b~FwmS9vVFnXT`g?(#!XI}gS;M|+0nqp{@kkSi}C3vt0sktC4O9a
zZb^aFfr6gWvO>Z0dEs7_(^tPawAQQt%OcUJ>)RsU9X8L`d9EbhS;T*3(q#$m?90<W
zOPK0zVEeLRS!!z6A;YWfIv1AyxUpyx*P~4_XKh|Bz50yRMzPgq@p)0jr&|tf_PJy%
z^YC)q?c)0Lh60v}@yi)@d^1kqoMHd8Afc5zxhTh;>$NA}6#350dYcLyMLmo8QW=hR
zryFQ8IePxN!mBTltf#tx*Qmtrf5WQfURxZNJxDq3?4Ijn_9E%*wPTiS>E=IlExaC@
z%}{PqIp-mnZt9bKNQ%X`>UiOtFB##kUY13jt{Xa?1+wN^bkzo^%X*#{y69}>kiEI2
zdt=#RE)~T?syV8UI6cq!oV9SiFs1Tw(t@l<zK=zCZeOW<HO<evS@3;W{(q4IPSXPR
z(0<Vc0hf(*n-}E2{dh=7w2+%cUnVrD+St2+SubYADIs3L$4Pe#!VPj>`MM?YL>v@*
z<#>i~5wFIwF!>bL;LjzN8Qyc4eRdo<>vh7X%p)?rJhJekPtMV4*}-3=R$uscJf3L{
z&lv^QiQB*GSw5EvRD7Y;<ofpV1i>rowpRV`QQBbV8@*(E^z^{!d4lT?mB#2UicwXI
zb90G1r4gsNRN8fs^y6~1C!zXZuFHJ=9@pM0b9}v$-LeM8Zw)U4`M&<_7k1#=&ot|6
zq5UU+d26ljn>@I8XWkd_e0}ASqoJo-vs0$r;&h9$=JRL&zj1!cp`79`VJxtCc?pYI
z%t4!^%r@VnysuuHq%hmeni6zXtatji4*LQ}k>$=SwlJ^kEaLNd)Z}FG?ttaLb#_03
z?7sBa-RaWro@MCbUc0=dd+x68_Z#Qz`c^8zr0MK4?;E$tscp7PKIAO^#Kq{b^hufP
zt%Nnaue8{+c+3~+e`)2}kj}(pS#Y_i;BnPD=IQHLzpwlMeixtfyMo`zdY9QYFskOv
zd#rGPZ;|NHpjfUut5%=nGB8}xy^P1nPp#Td<F>BaChx3;#rBJ*Z%bg($ZwBa+;V1t
z$QgyTU8U^hsqZ=T!sm#su6SDEsPbV_g|KhrC!x37c-IzX7alG9{6%J^^D;B@*U4X<
z(k&KlpEdL6w-wXBt(m?vzkk;D>;FpLPmnpZ`GRWFrs7R3OuKJaSzlgO`&@1A#YmgW
z9(4yd`n;KLo6KEb-&_CryUfcD%}<fHzD3UeJ#+s1=>0z{Czw22zv8U~@0J}2>bsS7
zOzlq1o^gCvN?Wk`NvDf?i=53`<eOU@MGOOHygT8*$;ZBV!jHmvoNZHfto}bQqN{VI
zr|pM{O`M+F{bq!7UVL;|G;Xuk%8+fjHjkw?W;woFSt*k}TS`xwX+vk_<8wbW*G}k|
zmfm-}!td#hzPU1e53l(yjp5;E(&<xK$8LW5bJXdNvrccfS$~>+Rr8(iYEQhmr1efb
zDQ-TLeP&OQuwm%z9(9RLGmL-wa-5jVx<FQ;O4!6`QnbeVPkiTRecj))!sLKehmLqx
zz~&k2ST?tXed1DTnxQUo_whTY=Fj&U488R)T{^q=%8ZVA({<-`%$!}hqvFWS;v+LB
zZoE8cVyE5O3BM-B_~%}mA9!2MqV{cTcgEW}`!`;H7&`a*=D6mwb02TEt9e$+RFUvq
zdjDjF{r_kGz4_bn*8Et;e>*i<Zh4AmUzs-XKcCF=Kb@1#=-x3d|HJ3Z^gDerhx48*
zN(MHZQ_OZbSXDJwck!qwuq&pwWvx&SzcTUe0g(>V*>Bo&xZDFzJxHxxDw5}DbFX#7
z>1MX*X{L3*_#y&U&P-b|b$iFEh=ZCw-3xa;-fXr?lu6G$>k0RwBHJf%U(%U(-|G4P
zl3zL`t>PX3Y9sBt+wu<H<@aU2vwMSDSG&TNL+ke1oECO?(fsc;GlS=?PbGh(P86r>
zSuc65@ohPeklv@Fgs@fbjZC*kPYLEMS|%rWNcq-{`d0>D{}dYwJ$7M!U=n=Ms>L@T
z@*>+>&%66XT$KZE@f`d=Q|pb^IT7*eQwuqLBeG_=RIj{qfalewi+M4Qlc!&9_bB!{
z|A@6=oy%Nqv&Ji$_a*#gKX3fHbN+0J<>I?$8h<HVFR#2~_tA2x;Io@|+`bwlcYSB@
z<_llrPR*`6ZM^riaoxqwzdu~JeYmq;RxRH2sz%-Fo6HKgy66Aswf{NWmSa-=1d$oH
zn4~OO9jXOa9#Ef=wK$|MPh-kD@5wed4%$}y5pzBvyy}qhV!sl#!)x1kL>^3Kxc9(b
z#q8Zt>mX%gcJ`%-kHt0}W|LfJ$#7_@v>wx)JDSQ44xD<a9BMkn8x~53Gwqn7Fgd_s
zWpp0Pl#P#<B_3*J)t>fplA-dUDLRqUGX8TGFF8EJB)F)da&xNRG(JJ4C!1JTT+nph
zDPWsrqIqel*KDKOS0-8^%N#q|_G*b}i#Nq+SN+))q<VNG@99al*=D*ow%TRyN<FEW
ze5a+*aFS5N#&r$5%l3b|+bv!Y-%?-cHFJWCBg4@iW|fFH7M~uUoFH!;X5@Nm+Dczx
zr?op=9bU|ojOH|u-JHA9ELC9AgHKiJod?VfCn*L$%X_eWPxbG%m9te>hJDT2>NI!4
zlos986MN>(71~wIxcM|kN!&7D?YF|Wr_8#s(Pw4P{n*Ow@0N*Fz1ktT&C8!D*{Pk|
zdx1zoTax65RIfM7R|I(GwS>ou>;FujU%OxT#_d`jyBYs4g$a3`@C<XvZA@&F%Ck6{
zCRcVtu}!h4;$i6Q$d1r1C*KoE-Fo|WOa+~nbR+Cjp%Ysl&%+a*lD7X|B<DF!5>lDq
z#`aQhLc&*%C$TPSCUH}wd^J<2hQw*API>avOfJ^Vb3qFer;nHmQ>s^3+N_jWC)1MJ
zeHov#Da^BIog-JJy1Z4-X@*eV659}A&D=R^4BEOc(=wbo(?ssX=CFjXV94Q^eo2G7
zFG)jkTHYl#?QI)Ex*2jkHiSm3{psT7vC1x`y(;Fzb2sg6Tf(|@*LJ7~IIg?VC@oo*
zqkcelomu;8&x7-B1$yc8YBXpnPTe{`LVadR;8yOrA@_Z59oq8$ZH;8<>#5sy|IVDy
zELeA1ch{q3A=A1-MZdlaE;&~%GB@b%Y032W@3ciS%!H1zN;+1GT!~1T@Ki90Yx&lK
zLw1HwHh2krzB#F;@g?hvSdN!V3#AKF%dcliF>f<W5cM^5Wa^DnlVq5@^NqxrDVl1n
zX|tYYU#SU*4Gmlsz!w#~>A<G4i%y#z-RClFU3%roVqM=`vyavuQYrPG9dO6XdfS#{
zsjMYF2VyEW9+Q%<c)GQ5w!_+$a|6nEJvw@@{;Y4vw7M57)hD>WSGAewuF4~Et%gG@
z<v}f%R!U$!n^H(c6C>Ay1v@1@?*&Cn^q13n_B4yn(qFgqfxwFi-Fgi#VkhwZx9S#U
z+iE2~JK;rD+xl#YMRTKX^i{tNtE|`^TKSFd%6l0Rn?GSk^Fs}~jlxV4Tca)Q7x$h!
z&AxG?nx(vovy0J7VNHpjKlXl?v(<d^<L>Vza#OTdichZ!+hw0x5bL)x_3xMa^1%h7
zevHeQJbk{|AD`3d%M>%2fq6m;=K>y=vQ?Ac*OVJQRP9$Nekgp)<&yL>4lU(?WKVX@
zQjG@&tCDuFJSibzsu7g)%49xQ=nA1rT$%+7!`}b&cRw&$FnmTkk5x_|pKnlCnBoyW
zPGJZB(+aDv?a>N1*Xda4b7)7~9+u|&lO9FH1qn`4@atkcl;m^btF(SqUd+oV@f`<V
z%vS%;#iNm35*|41lj7o<6LA)+mG?bpRkB^1;hN1eLtc{YL@TG!gj`LPqe(#rT~4J=
ztj$z8$-?Y;w_<tA73at#E#;ZhR1c)isky24clu0=qGu`JwIq04j|x3u=v7_)v+LrN
zZ`0I6-%b$N6ymh*o5G_Trz|uXXWuq(+WvjUvZ}{|9QR(7nZ>rM<dsam{mAK3<5AfY
z4xPaPk2%uMo>YBNbmvWsR{)b=e%It_`3q$%>pw5z$~{#2CPuw2ZTrHCOM1@AQWI?~
zOU~Q)3Ap`f3G7>vzWC>p73o%|0{dD@mdL5TT)^p?F;Qlg%EtpwY9m%<OnYa(zsD}4
zRG8!P3?olTrnZv*4c(QKnanRAnBwQiz-Qw1@mr_c{Vj<)rW5@$PF!evD3BFs`cRYI
zmn}rX=~N)+MpfT}nSwqKpJs)B4)N|&KCEK4gjeJtQ|Q|ZQj8pip*t^~;N8FYh-2py
zb)|)whn97S^-FLRvaa%X+oRz&<G|Oo&uqjc&%T&uyh^*+@aXGZZ)VL}FSDk%r8K$Y
zNyPdiN@DRPXCwn%nvLrsRW;vp&FGqtzF1dgjpA(CCFz`<8#cM{1T6V`%F^4y)S*zv
zK=E|uACpH>6ByO>b{TRdg^3yaKif8WCojW{2PMZBzgVL=RhM(2k&?-bJExTYHu8$+
zJ(9X|Xi}a0pG>DT&AazPzU!+0f74sGt!++APTQ)Nj*Dbt6a~9DmaO97^sM!3RnRMG
zD7d(V|HYf#JANfiE~wLvd48yJS3%x7Rfd1-8_lm@KY2EM&T?e~pEEZf6sF}JFcrM(
zFXXhVAaUZFGr!gQZ>&6bV{dSoirhz=bn)GMoo*&dhXUR;eRo(OlYi`F^4#tP*KHcu
za;ge1UUMjq(UN_dws{H1r|$(SH$Jw$uaWbd?^g3A<?Bj)uh-03tWUSA{?6QD{)^em
zUDV)!0?UF1c9qRDnI3FX@pzD?p8JxiS4sV%M?h)&mPgSnZfy^9E-W``6tUd9owG4j
z#Y9)}OB|bP+v><u!7C~sW?D^I6&8@9v`xYLfAqPcwYp)ayJAYR#diPL5uok8E?@Vn
zhez+^Q*V!`O=VWz$@KkNXu8Wa&V^s&x-)LY%wEOKUwtFNR664<*X~|1Cc&i2<D#PM
z<u}WgoRZ7Fv*@5y&aDM&zolQ!@`%u1(WLp%OqsR8@LJdHiLH|6D(kCbr?hfTo}E*!
zUitbYi_e3!Gje)C6>P$}%xkmPICfn0Xg2A2qjG-v(xrjSqE~tfyt0qWZV@^0!0yWq
zrD*P*`HxrJyw>eqQC#}f_qc4#hY6qAuB~$uce-^r#=bDjN#;S*91)J<MIjyTY<~+n
z;@dy#KCyYOc5SCke7b)X*WDY^m!8Z0u1x0AjIF%)$m{IE|JxV0zw20NXL)JTsqn9B
zW=~rb#wD4sqtAX~^X=2G%@%lEZCS>$;-_zMiGr7GhB{-;fq0+i!lIc;voCOXEq|!%
z>lSl*lIzw*f%^j$%Q6>*m?=yZ+2+;3?a#b>W8+nw$U9qHZ$!Ofp6dO1d0wZOdW_qJ
z|6i{yk>4ACbmoK8bLBQPy!8xo)sA}X+sG{Zpn*xPpqkag;k(x3y-XqnN(s+vHyqS^
z6JS$f9C%JUuG@6m=b64YU-*k9oZmLFMJi~Oeb)S_xm&k=SH3pwlHJEGVyC%P57rtM
z_CHU4A%FhFt@eLC6BtvqlDD6^$9uqZcV6$8I=f&G%ShJZilrO9I3CI-x&CL<4X>*(
zx8vLxtFy7e|FJZ)2Ez^$ws2>z<OIz+HBHTpoH+{GyydY$4dN>{3Fv+ovQ&(h+?ep-
zlDe+AbZnvCs_lh~ofET<O3Q{9?fs}OeoTnFP3ZQaMAeF-vmd3{jcpDK7k@U(7g~_^
z|8q;0dTYRnGQkC`eml~H7O+MeaIrnLHJh$%C2no~yjYgWKRqHTbw%4=<q};soBR`u
z493ZSSZqsolw4#^VN?!ax|$sDT-h-#C3tztv&$(yYI2T;?EJ6Uy;c?qJ7l7qV&5`d
zMk&Sqr?Pm;cZuY(GKM8(*~^Mq%Dv?;rB!Zn&|sC4-r#@lS?nSsvCSJBwkg(Eng6fo
zy41+I(W}<X$<L|NFj3lLs!-c8A<LysJ)2B*6S?OIh%}mnZxN|`5mL2iy8@?}@V0AJ
zuQp{jT<YNvY~OO6QK+GM$M$NW1xB0-eMv!mPgbP8(CB-;qVI(VSM-Ax2_=K%4arB+
z`wyin>0VPb6RtUKCi>-4&4uSV(?e@@lyZ44*Ip^hy(b=(qgZCJP<-`Mj~8KiPtx)P
zjp{WI8@&??@_JykbU}LabuNXK{fs+BzNIN@8hYOUSibtASNjYD-R&mk7ezPh5a0Q6
z!hS|m_V3c4J~l>O=Ic{qc<aEX^1`;-Jf(`G@Rp*`_rey&;whD7a_&E4qc+D_dU{_v
zAtZk6|5TR>t@ujmIujfHa*3b|_KBaW6NS_|CrV_mWXRh&O~^oI%>k~(4o!jrtWg52
z6)UF;Zs3e+&<r$Sdvky*MmQtzXKQ6zz-yPblogUGH;O-$_)q4vmMl^X`xxZjUSiOc
zEWF7kc4PAO&5>d3cDzSzm#-{+akxYGB4_xb?#VAYB#x?b7m5FV+@gH6({lrBQ<2Qx
zo3rmN?q7Vup}@UdxJi0Z(VT4^UaL(s#E%Gs2S!{Ml{z#{di#%#$rVx^i(+;)%H$lF
zymE{9{g+~$+bi~{F!Y;ONEFFzOX6Oaz`gyU_NFB6`9@i{9&&9`%KUM2-ns*tn+|ck
zUYR?`xtVo&_%ikXRU6pF1e^tKTQJ_AQT^~n^>dAdPdpaBInl>7fq_$zkyDX%+X1fV
z1)N(Ca8=#pS`)ynb%ZNffOV?@SMy5N|B|dR7Hn0|l$SIp&rVp}xomOcvBi>y`cE#G
z%+qiWWt?#Cdc;{V(Wz5Kq8Sa?C)eJ*9vQG|0#9V_gXP)6!37VuH3-hEQ%$bxeG=6t
zGEp*8^2c-ri6E}yhB8^VW}mQ7@A@jD6Dks|sBYwGQ|FShed3gZ#n$;6QZ>8krCkg~
zUN~Mmn(C=4bM2(l*8&;MgKV=DwZ9%<o0r6&lqxqds&bvgN>#3=UUlua8^pII@T>k3
zV0qZQxGhoVhihf1+oH>&+8_T{tt}LcG!^sQ;lRbB_IGi$+}CMImsV>DPcL|BpRdW#
zIZ+_1Y>nhZ7F|Wg<ASW50*e+OVEcN2>&<~hix2Q6GpthF#kuwXUqz<$+66NzCeFBb
zxpnb}&fcA@lQyvSzbt;$)s}X`T8^pY3rq6%U93l)z0ObV^89X7uAx|P(fISn_F(hU
zJH{OzOp{xW*zt6C+<KNG$lV)VW<Tji`g9A{q@uKmjcGyx8T&Xl9lyxta6L`6T11`I
zX2wy@UDK;&O&NtYFq#XuR~~UN6W**bwMJ`NaICQA^h?35pJs7BZ_H~9NuCf~nkl^`
zK}Pq8i9{0jQAe(}B<`gNhPeV-i93a&FZ^fwn!xS8(^P(zY4yZ()n88A7fqEncS|nm
z%{iF8>s!p`@M?|AoA$Xcd}h%n7PYXld;3wXzM9!<B)N7Rl@gYoWg67L62-7;$pik*
zMT?jMWC9gfA4x6B)-q7~*}Ak~@sb0JXU)!-!|ng(*Wzdc&Z(T%`M1gzU6JHVbZ790
zVSTxzE~WOGx-zHd()p=N8Gk7x99bH?xa&pw#QoBqH&de(rluwTo_{=gQA)Pdvf28F
zzv}H>#gaaA^+oT!kr~sjW>34Bz4!L-f?XFK`DaA&YtB=AvUhi)=Gp-680J+nOC;5z
z_&Gmn@?I<o4&vwiD5cL>xM)^JZUTSo?DR#mg1=|Z|MuWNTS|e2=?>imKWBzlX<gY=
zRQ9XrmD=8npQ{gZtvP&Zn#7&eIhs4X6d2^*9L(Ri=46!cX{$pEe@*xLZPjBXGdrQR
z`FTs{wzN($DLHH9>#Z}oGS_vMHNJmUJZr-`g;$4PStWf?k(yNL&ug-tW#>#^8Rb|(
zag*?Vd#6&bPXSz|N4dI>M$B2TM0B>uQdRMzjS4P)zaQHxE=`-WK=!2crmS1*47W>i
zsi<pxT`s(2dwI|H=Py=Qtxnths`Kuu;F@JIb>BlCXl_w6u4pUwHuyMqO@VkUN5<>Q
zd6VbNo4DFkO-noE_nffbn(W02QL9ZK8PCsMu=TU2$`4PCM9J{X+rrr&{|l_Yezd^5
zn)lTL;a$h~n=gEmvAwK&q0}saCo=?VwHc&m2}*kT9kDrcd>8AokMn0Mu<q*AJhvuo
z!tK6@j=qEHQ<p3_tF`6q;sa-GP90t^yz37~PVS9$c2@2#TKXQljJ;p&GH9*k);zy~
z-Efg=9skdzdloKj5?uD@k9bT}v<9;z7Xt&=hM;22N&j~)YYv~X@XIo_na01p1dhe5
z;VM`o#nYN@qZV@F;@lI{1Rd5r@HREL=y>C5`HN%H`!pDAb~*-aURk?*Ui4y3rUd>y
zLh`{uE3X>G%Luq^D{x$=pp)7h$Fopwc~jGskjluTF`IA2oV{37a?9GMVsG}d)d3u8
z$&3D9l-_gjZ}OTt?H$Lv&KM?MjE@yQVRMbi;F{tYp^LKxvTV;}X-fH?SiNVK#O`0!
z!U_zZ+q$F|w6~W{xiu%fVei>li!`0r6h|$Pbl_QUCq6T?<H+HcNrvCH9douR-!-%J
zg=p=Hn++KooP<g}KMBP2^yUa|e5}4vcVpf9(9Q`QA)lL5SH}u}KD<OKT21ccY^B$;
z|7)(hX`Nq?w5d?%rvKc8#JPfExpy+&N@T3Pvwh9%s%t{sN|$&Rc}Z^6u(O<}{kCD(
z#PUfxQx++*ZauJdSFO|(_0Vt@Rn|R`TIIQCo=z~|*t_|8Z!!16%A9Fc)~v2GkBi(o
zeR9vjBfi2SbA=ez{a<5HcxZ3owdA+=A7^ae$+g*0OKN*z!=WSde$13>-o80-_u0NR
z5AU5>EN;Q(u&4FUiFMHz4off{kr3MTTf+U!^LjtU<z-7m^*`;}>izg@%=vJsqAnNl
zJ%$sXeG|8T_jva~*0h%r>9^K$&E0dtTIR!wv>6jOyU!ANB>ME|+%@Vk4;pVvH)yj?
z?h;geE7nq#epA%j&`?uv?!K3&g+6F#%unLpcu7D^ef6iUP3DD7)4xl_-;}wpH2M5e
zlkHV|_x`HRv1}D&Xl;pW^>lcl|K?D~xfjU+{4HsRWX}jqymwXL&^5`2*Rp-jq|X&P
z{`TeRyZ3K)cNyG}yT>TEyXn>QrLR`@{eQJ_ku3X;vm5nZ2kzvw{bS|O^V(_84VN=F
z_MY3LEhjod^^M;ho0++97Do05ikF7@%vvHDc)o9U)Hc=lIjNf@kFD9F5EL}q?V<hY
zM#0=Qw=K7_=4A$L-{nx8BU&%DVSnPClWQdy=LwvgD`afdZTnj6Yu|grhi6-r=G|F&
zwtLMb*HVqc7h5M?TB{hbO_}4N%c6VNCxtB$jWRs6Kw9DccJYPc`yT{tV0^ivHKm}n
z=Uq!n-;Lw|?i_}*r7KyZ9&mr~cpC9jEb;71gNHAxaz$U}zA9Zb-{I}N-Iusl*Tt1z
z_`K$ywsZffOsPrk_7&3q&d%X}<hZA`>+QowHRl{=ox2qNe+kpgU02i_t{?w;XSu<a
z-4h;!+gi-ae&Vv?+4H6QR)2dxz3%6kr6MLzjAbTy^gVH%*7xj0KmXgDe?iysY}J_d
z-Jd*9NN#qpUvT#x!Mm-eZngbV(48wZ@6d`~F6Y?_j1gx9yhNV6y>MT&u!Va{Yu3rD
zk#i50yj`95(|-BK7ae+>Yd7qeu$!}Q&GaaT8E+nND`%}$o3-|<1KVk-L+Nkt&zvW)
z>-x3Xet!<e-<e(a=ivV8jJ;_^bJjBkzdB?hxGIl(wI7pk!K>wUahLbUxolB5#PYgv
z?!!&%G__J*JI#4*QU0&*=If8Y%HH1(%AcYB$Lr>?%Sv;9t~V@tWB<+Hnn@sL$Nwo0
zRVFMr*u=!a6>wt%k8?YprEQJI0ujj$ZtqYX)dpdgHfif79?rrgu9Gz-Ck1tGdg48u
zDZ)s{B9(ik+1jFQmri!2O9!Pa(eQ9GaTMYVQoAu{Q;B!*>}azb!HB8co{XLwnnE_1
zIyS5oP*qWMUUl_MSoGzjR8!%U=(b#iL#ij)mfYSD;og{%TqfeixVYk$P(b&R07hof
z=?@Pja<y~HsQ3gpL^d^@78269F~`9;ozrVug_>>H=a-i_CA<U@q7F<8`yY|%XtZjc
zeNdFgRjH$^?#>9@x+Nzu+<UTq*_w!}w?eu_Uhm-vO%A;~H8K1R(;Ds>ew%iE;}kOU
zdAR9+>lp{5T(6y(smrJ8hL^=0+;43EuiE_6*Zu0o3w!?kxFfK@mhp{-fqff`kGI2w
zn8pm<ww#M6b+~R@t=0{zx;$Y@pvTRWDZa-pq;x-YyOY!-5PZj}OEfxV;?Wg55?*Oa
zo!lJgCX;`~&|PNbr;Az&tCMsLkDW_?bK>YEmm*J9_O6aaFBYA7t-NN%9WG_<%{D%2
zj0Q<3PwVYu5#C^MR+wQ$SWS@e+089hYge3|G~?0H<tb~5k84UiVzN?Yc_F;Se(7W*
zea?bI*%x*R@CILe!6$k^??6P<IyuH$i5j{KCJ8xoEpd`Qy+WmrsdUX^FW#?L7H`W)
z?^aRjaB9%#Yx2<kzjn_fzpk)Xg??S0kr!LcBeFfuUDBwLcD}CX8Mw7~(UcA6BHMmV
z5mP#|Znuwhj^X8LTm7b1tuZt@rJ<_+?M}hb7>3Ke?_<i&mX^mdK5Aak%jr=Nz`(ZM
zXu`tUk5|og=9b**5NYBzyYV0}z@@O;SE6i3qNrkB1-GZny`)FoGiNoe{5Vf)z0HLB
zKZYKYyv0r|J`%L=)#6SI!%I)62`fKd+^80`BDHD3nV)GIlB+nD^tnCqXz&Z*nw2r7
zOYddIjJ~p6stcm6UM`tg#x-O4d^1koD?w8uUS+R1!1nXW<i&S*7QH$kb;TscGqLp|
z<HcDo->&_ZbRtjB)9ST%o7u{0?f;v1q$E#1$W`6l6X410%XuY$S$8YPqzQ{cPcCIR
zH{ExM!W|vWu-ja_qBcLeI_<%<*S%8>Cv`Vjgv}B@uszBuzE4r}#0@9ocRLoD?OLZc
z`BC!c3)zc#%&%p?w++v8yl4C4PVn`gzn?tb?!YwneE$K)<+~OiTr2ZMZc+8))D`<G
zwyWzL`W{u-s?McfvU~e?%P$w4jqL6$sS|m<eE-Almnk0~$)DwLD?23I&sCwkYJ$50
z*O7@yJcn479XELTEPB8q`e?~KQ<l7oFPj#f_~O`+kQtcxi-W`aMw`&09##27&);Vr
z32)xeA3Fb*55vAi460{Vx|;10?q7GvT2<vi&dy|y|1TVpTqfi+iq^j5o@u6`y4ygd
zE1HYDYsZuA&JR;scviNqQ`j<zV}nLwy(Fu2gkWoc;=$1Do+dGqg{>Mc4#I0Z`qUN)
z$dqk4B&W2A)%wJNF5R3~jm=TMYrj+o$6s+aXS(3BH0oksSi!40$BR-y+YA`}cvFpQ
zPx)<CF_2v0)5<Ks6E1n;YFGJ5_SiQ;Pj`j=^r~!BiRZDJzUYp^)F?Ya?QJi@_is2P
z_jSw4gw)LLgC{O5+<!+uQ8P&5h?v^O?vLjaWxBnNFS}s4QpHW{$<3zZQ;*Lc3s`Cx
zDycewOG#kX!kkni>*tPHZ2}ehKBw88_OLC!)~Q#sL|EpchrRv(OA{C6sw{|p*%8gd
zHL=-@cOiGHr&Fqu`0B`BZf8v|*ZoRb>sxuX<boC0KQ+0$j5ywQPg2t}OSSRr(x<rs
zi=yM69GWoANl+<Rh4bJavAMIo3=48@a6i14ykqACC9e7{t?S$_N?)q<;NBB-PNsH}
z%(6{R;gePz%~=%CADEG7xn@J_wO!7#PTzDVMZE|Rk}%u3^qhcZc5U$EEuwM{Bu`Fx
zZ75g!MtT0TyI(8L-G2RbN%?%nuSL@Y0t3FwsqE|VP>65db^YhHO}6G!605gsICGv-
zSjgC^U%O7@Xr#?=J08X2y6?A-{BvkgO1*l%jIa6PQ3o^S**qT_*p|G_k@PV7Khx8o
zN%YBD*3xB;4_*l=MXkCNwuWWWV?J}An&cBRj(oWzcIwIIoUGQ_4^$c%G!%B<<~@5X
zD<oRxqvo2XU$5A*w;x$>W=hu~R!$B>CMQ={^OmY?cU^^(0^YXjTtO17FGCJ<RnIwU
z8NH%?eV^)ssyi$PnKTNdxz}*cn{-fagVy>fJ6c83Emv!BJ>cQ9oFn3+m&(i}vDE2^
zt6tbT@7UEW0y@E-lMd>Ai@!MW<CkqgZ2x~0Y5z0|kUV61ZhF~f2d<e+oJ(3xH83vK
z@#f=F4fVd&X?iBrcisG>Ywj3iUF%qUfA6y11Dq;_vv{_LD{ej|q&#7!s@i<H$juDb
zWQ=G1_|KDW^Z2HnWYTk<e>$o@&bJ);k4<WNcQd2e&hyGNvB?`kc(d2voaGuQ+OpWg
zVX@M)XE)X{U3zVhvgFp?u3Lu|Nowv8$SdmBcm6zWn*LAS<;`BFt82X^4u_cPuC=(j
z(W9}&`d0GF(-&D@F519Zzri_ZztdcX65r$3Zz`F$emwcMZHD;k6>6q41<$|V;o0_O
z$?=MH8~o$m-rDoc#sAI);{g4Y%OCG);dgpCS!nms=Xs1*j`Ip>Fz<PKmQ|>xJ4X0M
zLz2Q|;S&Zcc!CYTH>;F<__B4;<|P`ehrNG)P&a;aGW8*6`t2u+WrYgbW7yW`R!%vv
z${=9QbHmL^9V!R!efrk)fB(|-(>H&A+SBntQ8VXVjlEH^P*G>=rBcP!Ryl{?W`*x|
zn&ZHieEYC$u4>2)r9}mWTaUQAp6m>2{~7zQOx!qWY4_F#oGz=h7)|q<4AxCDSbgKe
z>?#F6?yt2A^WIcX(_N!CGsoEXk<r`b%lz99|CL+1<CMc5k=Mt?Hsqu}4DQjBFjko!
znr^e`jxnR&l^F)pef%Y5g4@C@+!(g)VD8)gfKBho41>z1@+kq8%T)M#PWp(6SKs-(
zU`|5yR<nuER99F2{>0$_{SJ#d`;nK`jZ=cpypFkf^4!d4<!blmau*i6t@v_#PVSTW
zq6eSPy`Gc4NH=Bvf1l1v3#N&0_c?pzVg7$lr_)S<K|WJ7+%_hzxK$r`EOW}@P{HL2
z<?O|*)>>93r6RtftW)+}+^=y{bLI^uvmcs=891Z7MdwYB*?5`dUl!}yRVum)y1EWs
zW>Kq8NOp64=n4Dia3@j3sDWQ7M!aBivzf>AZzuX0F7h*W^6!jrtV<NIt~hw4oBQL+
zjYo>NU&_{bcd+YHwt#g7PsL)Lrs7s>3Gtm1xQsg1tX{F*Xu<ZM0u0L8%^x`DOPpEr
zMN*)}dB@kEhrYZ#^!VVKFM&p%Z?s%55?u1y!fcN21ZJU0+QxThDOfF8px(*xG;5EB
zr=F(Z!f#HJf<=1!7VdnSHQ92d;zDk#1;V>dO<;OIOV{bYk&<GQ<rJ%KH7UjalNYU+
zymI>C*=|3bS+7j}&BPhBa;}$?=C0&9ON~X>Sx)`BS|cFRe%WLp))nT5tEZPcu3`Bo
z&iK-`-ccmJr?2kf!3`G-i>24ZiU^e4=IZA<rl`%VWMZTIc=zfKewl8YLmYemH*}=(
za7$|MP@F835M-Ni$%uhT;l`s){f-I`q-Q={&1Gz@KGoCM?}YkeXGyI}<E9hKYmZ3I
zUdUl{S>e5~$AlLOlQd>EI(az0=6F{v)q04_L3zuk$3oL?v@FU#%F<x+{q%{RBMSwZ
zyC!VpnslS@^^Gl#FXz6L=(e}&=4}#~c4f9p@`~v-Cl3lXgf?^dyJ&>`zkTFO=T`QK
zO&;02=LNi^jChY)o?3K;Z%K{W3k8vl51OI_HZo0|UgRxMviYDq%lwktr(W!s{`TT2
zZ%O$OrRJJyP1YyV^$&4<>NID6*+0K>P2=I|nmsz!3e77En0GczcX-rf_Mqvb!Ss(0
zk8SB#S`obGI=4G}WOuEn&(;Y7&ocX4wV8Xeb(dv0pPsXpb<zG!nmd(N9=3|Hyil@p
z{zYTkE2_d$3mvxXZ1Y$+H+kmuh>7PNj=bHq(DjVo`obeji@J7eD>)jj{Ma?g$y3-j
zWsgtG&b=P%H=MAVpdh2~<*-rW)S3yd{FnJ6TBd!i-j=brkLz}q-Op8mZ;napoSklM
zTk!wO(t;)%+Z<+14aN%Qy_^g^W-ru2b}cJ#wlP1@QCKAYv1aEZ#SX1*E{0nyIj<xb
zZ|uGE*p`KNU!T;zDW2+;IqDp}>eX|WTQ5<6!Mx7w^1iMW$5$U%&S%=W@L-pdH|MA9
z6|G!b+Sh3IYxIj&9__y6;jvk<H-us7hiOZHEc<nN?puYEzm*rIf1b1CgU%!a?+FH{
zAL;nHd3raWTIs;VxrbAxXMyy(EY|rg91C2!l(M|;NG^K!AZW|St~YNkYTRskW+`y!
zlw<6jqgz7u9R4Nu-csJY*8JCvJ@P6%878WSvYOS7OxLa8_}1Z6z4G9hQ(jK2^Q|*j
zPyF^;v!g4oipS`G0sk=ufe(cTf8F4plyD|ums8kfSD&+Mj;J`FW9kZ8z;$!iWYaGe
zA+@^R&BrZm`tN@7nY#PJUl)-_PBSgMeXqr=n{)W8Zr1wVLNQkvs|C9?-t9UYymw3F
z&2wumsNP`cQl6{t;u!R`t8eA$*{dIOtJkWm^Y#w=$tQRBV&u^+nSl+w3TKjfgo;<6
zdU?>M^#Q{rp5uy-4d;I}+&SUQQjNv}p1t2c@z-g#?*8NYPsK@ES*PoOP3c6{qs>c;
zOKm1;_*b|_OfZ=6HPB)K$Aw}SciqKml9BHDRXh_b-6y78Ws|-6Xtuh=;p25akygn&
zA1vZ*UgO?jaAE_)iWv(cKML=6{$DLMSJtFsj-=$y6TMR|c6SB#xp3t(%1u1NRS+8G
z|Jfu*viqusrt3`4zs1~pHqJJU<y^g5e)gWQ8GjaNK3$|Dv}oBG>${n|Cf6+5G?(eG
zhR=s7lYS-e9MCzlZ%ugJS&ltD;<j&3f7;-5L0Z*lM$`VohC(YiPP|f+pUogPyTLf{
zN=NSY!;736_Fnn5L7RDH$N{BJ=c)6yq?~EDH9G5bx{;OhvmnD$MIQZ<*}oYTn0EPG
zzdO-fr+>=D$rds~M_({S#PVD?v$<;4RduhkvrdP`+4{!$_Vlh)yrmNMw0H6ni)*hv
z=5QU}qHGnt)?$6w=h$PLl{6Ph9}wvI^=j9*s9kUVpO|~$V%N(Tm$)8ZylZp&C2J?k
zjE2k65w07Vb7R(o8QjPg5jZBHTJFnpBAMsQ8~!gdx*onXKVmGfvw(;1#FcvPL-$+b
zKh&tcX_aQ)$QoDG&%LcR>cOtur=6-nd|#Lzh6eN{KVs5Jp27H3jrr~}=Fk9fx#bF)
zh80te8&wB9?q1%!;KIc>ciS>Hzy2+GCw24G%9{##`(|yuI%UJ8jw3f`L?i_-mI_|X
zd7PtnAuq$iy-C4>v(4|w%7upccE`OoSp98gx1#Ko%F0F4W+|!IPEk9{y{1;=hbA}Q
zvug{!@O_o_bk&UB=pe$m!O6I-WBr~(TYILT5EA&&;l#Su+<XR;(f<O06U7sb8!XNK
zW&OW{U$NV%-%4m_N6O2caz-<FR_qb~ReJp(v-ggj)6HHyW;ob&bc<n9NsNMvxv%Ud
zu6@&^7X%xY^qV*eJ10(lTEXwNvfbrV`|TaQhd8=U`GlsXIp4K0HtKc`KO1ZP<StLu
zB6%qhJC~#<r*=lS>gBPy6r2m4G-Jj28D|r1(-#W&GEM69{Nnqqi8p9Nw`P<6l|7R<
z<s9}M=w>$QSr^TH<R{;bk4ZD6!>gp-qUKGH{B6C{V1m#Dt|MESWrSl+3*QObak}f6
z_<7?^*8@0C%#PkUgKK}bQ+NXpUj#p^iO~^nfeOpZ$Nw&ETRQJMi+e}w^;DA=jsJE1
zxwk!-w<L()Pxz^UvVdRTqYI*o-nCY}U|;f%y*?mo**Df@?^^3TmN2f~+jDo5UT}i`
z=ZFW)k()GkPTGCEswJTRgk*h+-2{c~xd-id{wmJYJz?F<@&5C^C$}PZO*^q`-D|y-
zocz}5eNVC%%knSTn>OjombWEQYZ{_9EfAQY^my};t~I5eTPh~Z>6z_4vvXES&>WUI
zb6GgoJyx23!zL)>d3D;9w+Xz_DVHmhy=!(a(^Y89o5ir(cH;?yrB$ACyHi5SW1sx%
z<=LkqH+RMKZw%9ai!gi&==dfv??iL>nb^~(U7hl5PRo_V1WsepOW3|OB*gWt_?vI1
z_Bh<t4VnM_{{jJ%#)T#;SMmC(eA&mlH0=G-y!>x<@4tyPEN$cPcr^d|-uGU~3apnV
zXHQ!&x%%pq%dwHUH!t?}&1Z~@m5sY<bGGoA$P?4_#J!4(qn+Ogy!iGs*?H%Y$ffBy
zN4v7Fa!>l;VstTV0oTH%yREMp#7BSZ)8}37800nS#W}I~Uia4RLLEK58C`t`4F9ww
zcx>wsG#1Zz(r`xj&cu04+EaY`KdqRfz&No_M|%pt$nVwL_J+Jv)TvZFrk-`o(a`+R
ziU&N`GS_XETEO>^f0=UBg7&qm*;ll^T3YtGuAz8^+bhfd&&%JjuX(}Bc%az!d2#GV
z<<6aZyEiT?-E&dz_p-iUd++~OmVBsuQ&iWEqbXpM-0SC$6T4<~yxq*fyL8ef%|}Yh
z_9=YZ*Q>$#?sR0~L=Sa2i76HuB2OndNSDr^%cOPek-~;Yi>7Xo&b=~u=i|E~nqSu%
zYzls}r$Io;_VI=z2Op|*&e&uXym_vpAdkhf*#{Q{&9Lge{KEIx<xA%sMOZVQ6rG;l
zc$?{`0%K?y^D$YuYZIpPD$L7Hy24v=<=78?j$6m}vhFFGJl*K*YAF@-^24Vb|CUYL
z7ZQB!%Fhqpw$}1irt<PG7T+V}<K*6d`j^Tk{Vb_0zdQc@x3sLJzW28Gi*3|7y%IYn
zv~$=T|2(0*@4(UgsqP9gfs$@>-8284VbuSi`TykMf-t@3LeFRS8_(N*wR2VjyLi|_
zbE{jM3chijs}OVGe;=B)v@%C^GV>Nzk6BxH><Ef0|HYloQP{rStiz4*#K!KlenDwf
z<}%jnYK%3^dx|o}-)E{GNcp`}^+|Q-#t(BcV=ozg-EwHoy_dP6lg$_&8o3vn>SujR
zd$q*xmAQS*(eOWg`ah4ouV^dyv*Aa@iT6MBvShWrkLzvDI#AcM=2+la6Up`K65Z1!
zw|qNp^W=E_p2M>57Hqqx5LjNiTW(8;`K;jmuWz=0ouwcuEu^I5)}_atvsH4JWdEc$
zhrTgzTskl_^h6)uiFuu~a~{vJIk&e{y61saOFVZF!>RKJ;tu~06X8=XIxaQ8F*qjt
zfMHo~E|cY?|2I~buDtVP;i(8N^XEH7N|(xAJX-(Pea)M5Q4InL7y1q~F~vyQ%m_$W
z$r<xzOV^FID@QulMqE8~<5ZY6<JX$2pT&(*>?i*7nq<=XX^B0f4(qq(od$_MGtJT;
z&G|V|bfs?evOiClO1dsAa_N`y?JCK<sO!3uLzJPkE5s;zRnf#KHSOtZ<90XcDj#@#
zV-wTSdAc2D>bJM$@lFyv;OP+4So-qRQS0zGcXpRwU$=Mmgl-W|P7xLl0S3dzN4=Vi
z&dyx&;K_FG?OT$ZyRQ0}O)IX7)3{-@vT5<Hd14QCzSA&!o3%B2>W1DaEK@}}{`(jN
ze?N4Jv+}cEOa#Mwl^wUWCKw#;)i!6#^|XDlEOhp>Ycp?eoHuu8N%(C+t^;y%e;k_6
z+AsFczbTe`W{ttM)uHRJ*G$^$^=)_kCi%L*KY#wee#zJHtCEASrR<CZN3MG+0gXIT
z5dt1tdyI~$*(F+BVUsG-xY8DrbC5AmQRrxEN5I?+r4Vf{mZ>3nMNG*hhDDBSy{QF4
z;(eJPGp0q(S)&>t9?5yZ)zM&4C|A@R2~7ue-?B}~nykK&vnB<{X*Ns>o7R{r8B-;>
zY+8KZ$)z)sFCFdd5YkvUBR5ZNLW7X7kE42ryin)mL?+gNfcZOR(*sh4axQUcZaS$J
zu!!~l*1!vkdi7XVGITssS?M?_%k5O=>>Z)A7A@H@QA%>jl}!^FwsIVq$q=oQC{f79
zs&&w3)43%FSPpY492MK;=lD9hG4BGK3yb8Gxw;&c&+oe3<D77J!vo9Lr#3znyq)7z
zW}zOv?V;$CIS%)Ve!um3Amx&`k!fQ3J)gA=FS_m3dXEI{(R`?$;F<q%`Q;{)vn2~0
z|GnESaDeIRV?k|kK32&TFT*oa4)a)lUDRz_<mz@*q<(5)MC?@6X~$nR&bb`J)H~<8
zlj`@W;W0~(PTh2xbs^8@GeK7?ZzP#(PiSD-V{~U){wq&Mx3pD)ZzD4lA8);I(L&ZX
zDtc+9)`cv#txVC2y8q`TMlUe_duH04lDvz{grq$bFZl1-xzxomD5QOI1$(afyq%A_
zUduR^iI<e$yVGQCzWwU1+O-c#mb*{Tjmb!qZ}{`JJm06^ox}3UG~P4s8yMYhJV<Am
zcZyGPR!-qUe@_365B;SVom<$Uu9UhcSo@Hib$I33c~@hKH&*OEY4NG<_oQso$S2+M
z>@OEPCP_-xOo}q<PyKw}T0d>t<#NuwUoUefER8kUJaaRX=aH2SA|;o8E({Le*=n|C
zE0=<!P_&FxX>@nc-gSSvZq3iiV$=OQbK#nvM|#@~rZPk*O20@sYY@#mbA6a8<J})F
z=^-~KJ4mfq&N+e6iZfDVix8{e={x@yDk$!8mG(Hm`n<(J@phEQE|wzp4|fhWS1yjP
zjES(%-pgu}I)SOgugRq5?}Z{Ko<#>e3`G`dyh-%Rkr4J)VOW*&RLgYIqcELqqH7D5
zb|p%3AFm3rQEIxVpn3bA#Men%jDIav$gGh(t2RMqj_QLjZs$|^8p1ATQl8Z%)^uMk
zD4O**G^3vF<CQBc1+#8&EKz1Vc`LW*@HAFE7jvhLXKxA#IC6bbOJ{YI&RY_|miAYm
zfIIVO$+@8A!tV^HzS`)lx#dFFUzH^ex)W0!Sxj8c>|9#3>rVL7f=j2*98DLAW{^Ep
zleFtn;foSkC$$bg7bTIY9%V(1f~L9QzS1X8xc_em<Tg+Lv}7W$r-#d`WigYs*gP-H
z)bk2T|GJ`F?Um>Hxu0s%-?tbssv5A(f4ur~;F;xK-)r+j+78{<P&c)2w#!nRkrd(>
zqRQaGIw80&h9h0dmCN6i#cD+pH$%+QeG_EWn;t1U-*j0uuYC(sG@Gvnhv^DE*PRP_
zay6QzZxyr~Mf9))KRCd-b3-F1$C0IX7A10Doaz)VQyTL;szc*p+q#(z?#)37tOt^U
zHnfVoSvBLgYE@aK<u(TnCl>EeL(l6+EX@qpDodr8=q=m0J*g-8i19qF%fUu-j;u*d
zi<x=mi`S;TvvfA7&ORg}GQr={Zp}K;p807HCQYz2TB}>T=fC&=v_<Z7d|V7dSG&6?
z1x@hH@SQ5X$#)$K)0GR`TyvQM892`|>oqa-P2uO|;gnmYBv_flBolB+%*?@S{xXp$
zuT<}?UDlJAJ!9GVbk<2Rw};LK+m1|2V3iDCSglnhqf>HgDc8%snO5)Lne1^YRO>kP
z+v>*F(j=ozcYALAzP_JHbKk*iA@NDqH~!8u_Af3ClC)}E_S})TDy8Z9jCCsKE8P4K
zu-y}y>5(|KS}VctkBYpJd5FLfi6@KMHa;l)_GDl8zkZWxk}smgT%C{4Qaj4_@QQ}h
z8p(4hR|30~n$MZ6k&7?2IBwcEHM(GPYjW*}iqsoTM=b=z=O0_jtJJ0G_y4W~8(+*)
z9_brSWw*lT8hI?R(o}iD#;)Wn$^D>JPe<#-CDmsJhcDzLU(k}sbnI_Rm@eCOc6!ot
zW{3G7-gH!%aYapkkuH|V+8wbaDTec{;m)$a*<GPeGvcPX87&Izous$ep~1*bRp6U?
zMy>bts?7qN2HP6aL;O?4|IT#MS|utQxFp?O@D_)&k-B`UaH>bQ|6>XF!-cD+vWRS8
z3^*6yqN>sCm+jrAIc0%m^b7~hq^u`GU#3}weB_ff$Y{?hJ{Yv@o>`1pAot62Epnd(
zJ3Jyb@)fU75?!&XRl}_?G^!&>l;hCTu%Dq<E4g{p4_<lKs<!auJ+;a6Pkjlncc1cq
zQS<bU<wB2{dR5GPvz=<n4Ta<Sr??wEk!<ns&8n#RzuT|h`{bNAEWwT5eh+kK{yCue
zU1<6(i-*&HKQZ1IH=iM@b*6J;<Q4NA4X5y50g|WPZZip1P2Zk=qG^FYgHW%Of0SX@
zr3$rU(xw`c2cDm3iuiCjw=lG=(5SfC=Ro8+#(wECsWi>to6GqZo@Pn1N=@Ismq#XJ
zxArN6M*m%kFHe_tZ{tcz6Vod#IN)|(x9SkrM3I)WTO^KZJ=)v#`-+qB;s|!#Fhee1
zksQ%k4WBh!9Mh}Xx892vT@+D#=;o%Zs`K@6wNb|Ru5HT_T$g`TKQ^@T`nEg&_BI{m
zJ{_{{_vHUGKd!Ii)j8YZ6d|aaGkK-REf=p`$>)z*H!1GEpPH&8!7R3BGpk~2Zpzov
zuQl>1!XBABM81}Nd$Z+T$iugHZwoBa$U1p)zKTa_N9x?xA7}F$%f0`#-xiDJxgaJZ
zw_|CS#s;UFHHRhEFxPe*)VSmBlX1|B<A7BMqwSfX))tn%4_O!=ML(-}u)oGlKjC`o
z0=f1ajvF6>UujJ4sPU+9l$^5mL}lU8BYWcBK5^~qxX62|=Uu>@4-%Zm&R#Y9=rN^-
zRWRzjlSXH~?v3dsp2B($n>25H6`3I%r1<RuM{D9kkw2VkauzZ?aJv>8@%v4uhzQrY
zhetX!r*#TWTFwv<>oMhj*UA}bC)`#@Ou8Z|@UNl&oJQD!BT^E1H^i^W^ZvXdmUFbt
zr#<4sl`sQ$pRJ7>Sgv!2Ms9LgwCTV#$!h|9rrhj;;@n3k+}C7zz{yZ@r)i7FX}2wh
z#5$PRxg2&!CU{<O-nT{C`U|slhMevdX5EY?-DRg%hD?9<;}QF+OQBbp{a-{1DGI(;
zTpZ-$-hCiWO!U^V39OM7f#!wMm!?dNTN}7U@~$Ln+mc-6vjW!)(^`9P&d8sU?8L%*
z&N1H9>%O^Ei-IbP;zicWD`zoiT>8d1McisqR^+koHzqoMNOqEOpUrWqZL7cLi?c<A
z2O_?RIkmOtSa$2WbZ_oy|NGK+YUtztg_$Z0^SBuJ0(2WvuBIHT-f*Zpvq@ruq|XKC
z=U1KVuP|9}XxhbjKr7|Ylcmw#-<WMX4l;4ESa&d5-w4v*)*4>9K<gX}<1u#Y9!A5z
zw+$CQ2xglz*FgAyV{qsyj;g18mW6lt`+5YvEePLq^jPdfK{rn8nNMfTd3us}#;KL6
zO>1Lj9#B11JEwEcoYtkslfJO!1qA(h(BizX-Puxvci}=uiz!Pc98{A@^V_57>=?Un
z#;GfsUUEgU(LWTgMLv_$bog61lc7h+B_#T<>S>o<2VIm}wud~*d^KPF9dC5bO(zF;
zZ%=QHX^$8#s+CFzxXN&8Z)s%AJhHNZMLWgO`%}<=`;vLAmkvKNec+?E-1~vMkHI}}
zvwO8XOWBwnzX+WkkS5yN<7jB*Gk@jP@}ldjI+pWVEq;?5D3G=!_Uw@(KjY%IrhnMG
z{N2Xq6J3tkPhFVi5Wmtf-r6qAQRb`_m&=uq`xkm|EjiNuEiqPkYiq9}yPzt&q)N(m
zlkRd2&Y!wUn?t<+3;I1!3~ztd;(qOrdsj;OL>1{NDxRB|tar?;Il*3gr@h{wPn(M&
zK+VPb!XY0i51)*tJr5sgnLOY7<&agxJ*_p)-rHPCC7(L6sFwblz;RFVkj+FR&CBJX
z5mT3)d({!%F=x8SMTJVe<m#i>`|kAh3f!M4!BQJ}r|$0cj|MIOEqhxhZP9G9d?=FC
zKgUjSZdl?xr-dOFtVSni+5O@yc;j}KQ^alQbH|GT$$J+^pGa&tB%z%VlJLyyd+Wo6
z=T7-dXxwCQRawouvSmWi(<3wI-c*-z@$MBWxsawYOR2=e<xd2QTgDNFN*0d;+}>Bb
zy&k-(OJNFq$g-#Rkj9244Uq}<7o2ykJkr$R;v=(qX;Yy6mBTt|;^ke7U-_;s*SqNS
zgfk+9Rdnj}%2QJpedXA^=*Ah9BXK|C`v0_=XFcA<cRzM&xNM`(ng`GFs@T%M^vA!O
z_+{pcgk|j)E4*S>%)091wP?es?=APQ?)7r|knC{I#bMFeH8XwhY+>EJ<e8e$)&KVx
z6kJkrI@(fJ#vE~-@uIp#$8%Fg&6VT3e>i#W@vi@)QE&60UZrEdiGt40Ztp`&x+b-_
z*Tm_vo~VBkt;gxmn3JF@s2hDlXh-NdZ@tuDwac4$SwsG{gm^9eIfI{P)`@P7J8w+x
zblb?hoxwA+>2<hR%h62Ni8uIemhsG4^z>HUoS1d0r}w5tAJdICdMGr>NjR#rcY<c#
zhllAe+J2unbxwk7znhoe1MT~USJIxb)E`@%<8U`~#-l5WUb$ByPK2mkDv@G8lIC<I
zE!*t$ojZ&>mt45BCrxA3sx`A79XO|*7w6(OrAxQQ<;}IMy=z?Fu4>u)LC8RhgDr2t
z+k4%fKPLTOyUhK?-#5B{7dfU}eSD#rBh6*Yvbe2k$zR-BZ+57QUP<8PJ1II>{q4q=
z;!A_xO<no!X~OP9>eaLIe@=S+Q)_;z!HSy(*BZVp&k%XB<Ji#@U6;H?w=QzB*r>Vu
zlUi~7694Lo`SW<PRF+QqedUgDA-h^rmg+TuRZS~&Tb6W8JC)b;rnBwM*=cLKzd4^;
zwq~svABR3)gG}a=86QfM__Q;aQn-A2v^oOX1lirDdyDAuT6<i$=B_KVyz5gz*RGpg
zNz)CcF5p_GVC0%RbzQ==Y1czHc?ie9f1{)P<CWh#*7Gy_)_$ltV<L3k$~18{TgJBs
znKNFzl(2s|XWB=};Dz5d#QYZ*GT5UfaP6*)l#t$LJub-)?=PQuC_nRsdtSh`7Z0;E
z#U!{s`LK9T_!$!@+4r*R(aUvOt}oK^>T>e0F?n3bT6b%m>iu^gSFwMTYVl`E_44c6
z@kr?1x6_-uZhzY3>=q)Yo8Y3Q|6bcp`_rjzk1b4xWFCL^(%$N&f3_xbo7v(8g%=}#
z7@0OL)i|E-^ud+CPkZ*W1oOPdi)BBjmqamfyIgo8=wC2%?>`yqya}r^*dFsuy0W6}
z4@=JZo2);6wB;GR`NgH;H%Hlyv*oZF%i2GNw>M-}ro3_U*q~W9EjP}f>(z&@SvjYE
z9cQ0haDLyKdJpx<*FNxCm+TKvKj0!4%DMLcJF_o=*FH>&YSTK_zT@tr!gmI`A2v7`
zbDW;n;>6t1eTi9f!h`(_7X;ho`tZwyUixy-^P-cL%z{kSi4FPDXT%NbmmcGEi8NE3
zuqF4z)|V%aKD|AwQ1sl|xtG>{TVeQl#ql*;Ppvv~UPe5`@#L|Gvx?7iE<68LDOyD>
ziEmQQ5!Vcdtg8)@8Gk=6jXL&JY)RDwr`$z(th=+qj%T@_c(<qHgU)Z$V!LpsB~SFl
zcn-*zIX5SK6gnfZ<k1UVvqbj5u*#Nm`xfn3nW4S;llQ9)99G$>m-RR<pN@ar&Tgme
z|6_LKOQVWS-;_GjFI8%q)V)3)@^O}wan*|V?_X}K{w61VH2uF;W@D$6@4jtE>)oQF
z+&}+2l$gGuDPhZv><-gd3HAk5yq8XFR(|O=C*%9N)N&yX-nu-tle2QV`^?_%`oN*j
zr*o3!(mvNo&swV6c?uUTaoq9!qR?BO^BWvRIBH)MIJ9r+YAUI}&aS(#{P7a=s@WTS
zrpVQ6r+QEBn-Z1TlFu>C?D9_?!;U9kz8XDld!O**o6+?;&9}M1^)n|o?#&cm%zDK@
zamE8($=yHZ9-2IF)*KPPlUW--Iz4#W_Ob2v0shX7tO?s%!{5(|ez1BK%Y}3fkywV;
zHq%aBS=hDEC+&C4v*-}Dw#vk$n4NhFZM_%0R$nQ7b7__B9X+*VqhhUZiw>Iqf9mor
zi9fk7Z))x2=w^%8k7pQZUuJr`vB{&`Bt8CHI=9)($=g0GyA*o0sre(9p7i4^Z~f36
za_!gqzJ&!ET%R|y?X_H`{-r-(oKmWs7Q8i!UA(KQ*vaQ}i7?lRM`;pMXPKq@ulW5u
z)$)ey;l+QpG_F+eS`>2kevtW^5VrgipV#g6D$6gIE8<Q4kh|K$k;UcU!-ZcwmzQkT
z|M86H{l1Wrn%f2JO$#=COJNN=w?%n<J^PolzX}?zyZg>Cahm>{`BbG#_dIr?_xmyr
zmPrPOoZ520H}&eizN?S62=@mcDU|VF?VcN5=yxYK>eIPE?X3qFrv_@*2<iQ4H2o@h
z*&<rZ;Q#L{QNMlK%0H`K;`r&4V<Y7zdtNdqDeb&;(WxXg%PSX;Mn#8BQP)xUyw=XF
zEy=)bU3A)mYcmob?2%(ycXW5Zh{g1`Sw$M|+n$7^Z}Gpr_oj>fgEbeU{q!>qF@|l^
zeE!5cuPp8U&$Ppz(&ft*{C=myZM$Xl&eW8vEioSRzDB;9Tl>W>&?%->#CGG+gg`Ox
zS$DcOiL8HqmpT64^z6tbKA9@TM&?o1Stp7#cs=YkUMYAs<3qZXXZ+u`&*IK?PA1!%
z-{xChzOeRkO<vCTvh9Dee77HbB=w@Z?%Dax3<|#`*63`{FRD)d;>ND))_nUS-(wAv
z1lPj`(N~%!I9_`+fBDZ}z2&WZ35WcbO5wewVt-3S^IvXk)qk*o%e!%158o~6LP2L$
zLH`}o_PrL)w(oen_r!_(o6A-%5<GvvI8pH3iM!ig-rcqK*L7V{H%=+(8^8BgSjV1k
zeEOsH>5ut){^Zvz6k%}D4qD41Vp1WH=orSvsaB$q@bF+8zpR^x!?Q<fU99e$3P;lx
zFqkbCOIl+ZxX_)G!Ax$-1c%0k>0Cl*)^G-=H8mS@f69prXmZqC{O9WLN|Be2i#=t(
zY2V1dda3=(tE<gk*;gd0{+WF%)_S0QXhVW)=3DpkFRxeq`z4zCdfVGuJMy0f$($C+
zzrL(`>aV2ZUZ<vMHtpn@7c^r-;*s|MqQ-iEZWN|;Me`I1I?UvjJRNf?LhOQM^7E!t
zmJ-8`!jgN7mxNlU^nNgPy*!`0Xwl!9Ti)K;Qab;euEX<VDs0X6cZ>`c9c@~n&y}%s
zkHV>GIkU9YzfQKi_gZtic+TZh(~3Vj39E44D&EmB<2COl@dYi*h1y>xv-$T2?DP40
zgtcem^VjRHMLk-zU=d$YmqW7vi?<`AkVt~8Vc1I7m7y;L7t9NFd--%_YX)b4LkG)>
zrMm0A-aI`bB3u)&b+?7hhJ(wKM7S0C9QPEuEze3!<@RL}&S5^RCNRn4V4KOM4GKGN
zIC-k^-)?Gd+R?E|fzRggAtS#HA`Yi@Y&Wmn_EaY6|GTX>miTCFx$U9ZJlW`)PKNJo
ztHj`jpch*{TUqBG-t1>~`od>_`?qt1FE}!96~5@q|8}#8o2=`Wa{`ySwG0}Ua9>>1
zxny^V4dX3=d__OrOD7h*b$h92)$gM8bXU|Z&5lED9upTW``u@y@3O4za+9vq)J?Ay
z1u_$tF9;3$cy+<jUlY6JqQVker*00rm?B!XFDh(G@@v6X(Z#of6r!1nO*>=_Gb&%-
zjyufsQE_Dg&&0chlZ{r*i*mcEWN^6d%Jhv#6n>i<HOOmcaJ6e3_GvgP<QK8o#IyRw
z0;8vMBTNl;nw)slQ<=SP`|C1p;V0di)0%q1Zv8l+@akBq-tPMvhyJffH5A;n&N_rQ
ziB;v<KJN9RFEe}(*>;8Q*0*Pzbirxe7MZBR%;p`3x(+e9*Mv31Z7SWcpz7tr4b%7E
zxOl;d@x{dn(@Kv;EElgjvcf8sG2`Jh*|qnItZy?E{#hyJyU5q$<p!ZIyEB(WoY9Vx
z<W2wZp<J?L)%2>UkA+<)8dZ!=DC*8%RCH#!gwY8F%L{XuHRdn*q|kc)$feEGa-w6O
zT-lYe*|NiPvrpMAxwgCK&lj{$-fC`JWPHK&W5|^YF1`BA0gtBI%UyDt+kd5UjsVwz
z<t=ecTrX7mHitO85EA;T@nO1h>EhQ0N*PODuXt@~!<BtAkDH<3@5*1(3ij?d)yyjV
zAAT?NNJM(%$yKb@J2r6z&d3l+kai7S=GvY1a;@B!3(<})hF#pL*}8v(8yNQ{DOB}5
zYVeqNYL`?<zhOsGLDz53hb&7Rk1%@c*PIq~@{y2`ey43{`Q|zwtJsDnQ60r8njHe-
zZIWkuju|f~oOvWj*+l&wTUY9gUXj@yvoG#x^AW6m<hHHA=u2gnh=k_JRY}t}EveAg
zOgb~wVa8#$4;{<X*)BR&syN6m;W-}0HnG3*LGT1SuLb?@8PB;XDKKbmSsd<LEx5_x
zi0LlT*O6g{x5NHypUSauu~=5LtG&_|QQ@YgF$Kx3TYnk0306K0m+%SDza}UucC*7x
zg{NEpfYE<0!wi9GL6?_0ip`uLvdW<8)S=U{1tsPwhU?__NgQSSRJ!D>=*yWioiCY~
zZeOCts+l-{bx>-{j-@S?nkl-8N+Pn4B&XFr^UW)gc%LJ@-0tm5zotp)v$<OLIH>AZ
zo{7FU|JRjH_q&-BSUnvWo)rtsox)z7EYWe6#btu8qDshIr4!48LItL(E>V_AT(N4J
zF;~-uMaqdXf!249cRjaB=id6rKj>Nb1e?4i7HqC33Km`H+M%NtDe;TTmQ6xzjl_ni
zB;nTE`<8J=U1=7b<q#P3-CQ@!raRi>Dof|DiMp#&x+C6Ltg;VF^omk(Sv}9DBRb}W
z-o6Bll{zU;0)H&On*M)c%aq$zuOhcsOxei57H@w_obTt+>vG=$80>vc9TZz;P;9om
z>+zcGqcTYwn-B0QPN>y6rZHRKcv#p1#YqR89E&ctR9s8t->H$2swUKu(6Z&M=HgXm
zt0wqw-6*=wiqn~81E<xh_6(7ekG)xEYCA}sRsQF(@cMymeapKNMMRQZL*8yZ5XDof
zn{_)t{cqvPx*J8ZzfY&wpK?9Gt9D&8)>pwgV2T<0uT3tST$UP!M+lT<Bx;L1<hs1$
z&_b5C{EG42`HT;rt=Vjv$0KQ~#+kZg%VEJEtsH(b)*noloGA1wo_YGtQxzHCv%c?&
zZ`E|X+_TeQ$p${Nbw7F1bHbBuzJ2$<OyGCy>`Ob<9x}eK_<3i!U8YU)3rjBs9Ze1<
z9;bk5EDDS|j1Ciwvpw|n#55l*_Q)1A66#yjuVRucukGa}V02>r35Ud$jvua1E_7WO
zt(l`>J4tHWIw9>wzE|xm-K7reGnkx({!Ezj`{Ny!rZ3J9PJM1Qs+e|cA@_wy?SNPT
zBa3h6PAF_W;WE$h$-2!4vP9!+x!4R<wgm=spO=<lX51Tekh|+`Ygh8a20>frmLE*<
z<}zpEq!%APx@VUmgE5DIdhpCsG9QhdOoGK$-;gr)eRDl<&Lfe?;AJ}Bj&<7Tb@Z_=
zc+-^8C>rPEmwkH1t*@HPZ|O;|?O5@6Z;#TG+bjMjEq2LCoMO@UYMb=R`#!9ha*bXI
zTqy!cK6ToY7j0ExTFelzf;GXLeWpNecHb&S`zQVH;=*}m=!iF~l!%2LDQ{SOC+q69
zPa>roSzRUUC&p%%oY~yBR8DBs`iL*Tr|yXL$er@8Uqj=_!tIUPT3c$Ox69cs;@!3O
zu@TR;DB<bi^*xSUj`JOi7ryja9Q<zP6`4z^8#EhuWtPmo`$fni`nu4wC3>ao6C*NQ
zyk?oGift_960v=5S+@DQ#M|Fr(my}ix%`?zv|7*uxk<rGpS&;C3iUg7+TFQI_S?-l
z3L;9*zOPPkhTk|O=K6ST0hdu**JXp187sBgc;XzJ`;$ac5C8eUX&QG>^1>DJ7gt#<
zc*^vmq;|D~YF5CWD_ZL}?J#hjCBeGU&E#NQhLgyhT^D9@PC1a)nY8ZjzjbZjp7*z|
zc{IJ`dY@R{jhcOS^W*EM@&tEyL|lHjtL$LdKe4}(YL_$KB#Zz4x1%9p!YBC@i|6JY
z-+ZKUr(E6IyKw8T%!!9CT=zN>y0&7fvu)74uOe^nHM~|VI<2rRfjjD;oLa(14JXdJ
zYzwo6;&Mlx2=(dmbhymp)|vBYWfRxbQ>V3jSaf{OyIgv$$*0>oBW-#}$GRu4pH+D}
z{#mZGM)iqXVx+@~&$C#S<{WC7^t!oWeO4RS<Qr|e^Ik{HU^@6K?W+*aa*=lJL;scX
zRUEk5-W4hIaG9oatrg(5RzFm?;w9GyqplOz>YS464sAGhGO>W2cdy^pO9BBU>J?Mk
z-(2$JOYfOKOFigVs)kPMNr9%|UmH~4<Rw`piJWU!);VQ)Z^bDWr>dk^>brdG_ncAA
zbL5I!v~<P$k9<#dZtIG$(ae3ZsQJ|*mj_B;-3ot4J>u}rRSsj^C+F1Eb2MVxQKxm@
z3^x|a330Sc&~Hk4zuTds(d3|(%g3f;i@sL%NhEeWH%Q@Gr0`;y!_`k&S2>!m#;m={
z#HP7QpJ%#q=Om7eFZA_~Y2MhPuetA|!ZMH7TR3hW2<(nx;=HEMc<#I%OM>k6i77s9
zN-U4k9R6qD;hDHt;n1x<CKWR#(|vChqO$LVw4FM`tLmY6_lmN;mZ2ikg)oLro;Qiy
zo7lrEoT|PZR%KZ*A!{M0Fgwd{#m7b)_Fdxs8uCO!{t#=o;K>-4Y`!Kjv9(!3PP=ww
z^O&$a-=`Iob+CTLQn{*w^_#M@Pqnf=Zo3??QlVm1L5ij(PtRjHr<p1WTnjD~EpeEo
z^XA36PcPJ#8_en}5m0)yMCG*-w`P#~jWb1^TVA}*U^YF}aa7Lj&8yBg&pyq2HR<ga
z&iVIV&cCMTa!Q|ZTRMO1F(tOoX<WsM9Ku!}ii@+Bos?)2ad;(f%Ox26$Wln*u0&zX
zjtAn~8q@@sjG43k&imghT6Hv9q($$U8Vi4miR_`)wv8fBQ`vaeu8MiN(nvAp2IJ=&
zOj>>`w5F|YzNN%*RBhrhudh85-npG{%yQ&vINdhmq~t<=t6B~tCT~-Y)4!yS9a)gE
zHLke(SiqmC=bH|3ZkAc}=gNA8M;{fG5`=FZSmN_q{-KNf(^jh>?b~um*53macnz(6
zUWLY~hI%aQ`{!@Ubm3mx3Q?^~it<hOqzpKf9Jt(q&5tsDjVt2R7AX)p^LWOgeN5>y
z&b`VPS)#M<Xl7pDsuzhoUzVNoc;qsxHO|Au+}Fsd(^De6V7lFw=Mfr?87mbsR%&Q4
zbFDPun*WlkJ;bAX%b_VOCCeOUPx&9B<NaL2fk)Bj`L$yy(%jxYmsq@W*^eGM$IF=N
zcY<&Jn@N1#&l4}KoA1<ed{P3BkCwqlrrrxL6c#Wqc_gxUmvWk!qIH0p(uWf3e|mz^
ziku~nszQ_<d6Sq#8iZI=G}??6U$6#?F*#3E3YtIZeuR){W=53C1%;ke?ORRDA7w{!
zEcxPQ8tF0F!R_tipwJ6prY?UPJEkPOnDNo_dso=sEUEvh`fCJVO;~Z=hDU$T#3Ye!
z%`|4k6C1;y9=*xK#3`iUt9oE@L0cM^@FJ}yzpZC`R$c88JEZVDII3RBUUqY86r-YK
zlW3Zgvh=089cPxx1Tj81e0R<9r4OzyYUz$W^}kIt>ZMcAC53xw-^4uKcNYJyaTL{`
z*tX};J{KchHzSt)O@ZQ>s@5iof37P03r*U{#>&6w&Z)O5^O||ywVqZp^k~mJwBeG+
z<Z~(geOApUt{u8`_|Rm{V?w?$ZP#y1O1mmh;j?BQ*CFrIA@i=Cn|H#*=k>|-_HTU7
zPaK^WiEtXLPAiL&Pf`>qt`TywbXs-)!~;YAjkP}}gmL6=+z}So>7;dM>F1v8rUp)p
zk371&R^>2G%N6Z)sX3~v^G>DzqUp}dfom7VEl3q{OXAtW5I@78d&)(dPaI)O8zkp$
zmi($J^;_%Ov7^6grq1R$?UlI9GGW@B8`Csn%v>Jm9cSZYDEa?p0bi2r)^-uUHe1&x
zQNFJ?`@Q~a5dK5x(7#nzmb_XjTlMUY8AFDgQMlL#LE$C_-jyF;gl3f-xqQjZ>D24&
zn2WU*Uvn2RX&hY;?CBe%Tb2}AP?h>5@wis@7KWd3FO&~IN_?(8ZOwG2t=rw^rtPb7
zthsz$zUH8u(Y3+_sy&;Mgts|x&v@^?yhw|Ed*3hf^7D%s9mBY~r+9R3IW$erb7QCH
z`6p7JnO?8^*ck7Ymh4`hz*csG;pje-qb3!8cFBHrKhi~2eH>@B$g{QBR+#MacTjTR
z_VbEkn3y?rTjuh`Ucyew>zYHN5(VZu8*MIgm&wXzzgk*)TEV0-aK2OS(f?_mt}%UH
z<mzm+N=`3D*1?0>z2!pDtIkuef<g`2B#x-J?~q9BZ!<`7FJoCUX;uDq1uiD@f~5`x
z$_@38Z`A+3p1pQ?LF(49`GNiOKZj-NhKGwu?=?tl;ZRnRD7tLeC9qRwK8Gy#p$K8g
zH&IHFvP#RYU0ix>L3)8&%`VZ>tyhC(l+tJUOiuo$V0GI3-DHK?j~5*&-KL_nlp*u3
z%p;pU+X5ypQhfi`rmrnQ`0TWl4=qycJ9)O*b1}tlJoIsg_QQY_<xS?{Ga{d}rKr7c
z(yrU-dd|!J_I8JLk+W}#Xa74aw`FzN({i3y<-!c*Ld%PU&L-E_U$MDV*~ibskQ~{6
z@Bd@>#TVx3KR?)IGJoqOF8#+dgSIKmd&k|Gp&WCqsCa45Hoi%=JIaEb%bslVFWL6S
z;)cJ~jW>DD{!di-CyMo~{Hng9c_GWk(sZ_3wnvehzoqeAF-qUL*j$P0-nPJ*r?d_D
zb=R!YSf<)#p5MG|)wu_2IHIFWw@55@@LeI^9ap26f1iEHv7PxhBMO!#6fE4;x5SEb
zMO7bXen6{*#g<#f=0ZE(-O!r5UQ)a~z92M?@0O?T<5xAIYn^?L>XmoIhHW}#9A2@;
zqUp`Lr3ZR7PAT{;Tcpvn{y3{&=yd693C$IUvs9awWPRVVY58^*flcyF4y>Jm>ekz(
zHZVS3H(|oFj=TT6nHDnraQOXj)_3*YTuOba``TES?Y`l5@_Nnb?H3{fYR+GGymH-9
zrNhY~rS@X@j+XAK^4qhG&E`ZZ_MEW1>R?uXQ*y#Hi_NT=mQS5OK9l_GB)oma?pR@w
zU9WdGaL(AhJ(}Clhhbk)c>Ffi3ssvAUFOL)v)-J)*Z9Ufom;$jH?-Vd(jvL*_XKs9
ze?<%Do!~b-`by$d&4VMG%XTP69286H-j<`dG|wqRX45qG3kq?&;(E3AxHLAL=ut7!
zZ$6?nSwmmfrSSPKizB@s!=+4Dqz68UC_N%;n!RgTk9@I!sbcTa1-+|(ty~<{S8StE
zwx`_j^E&pI$D8jQmoAJ=4CVarKVtoan*QGr;m$1!_>-)4zt2&UT-O*C(f_scO7Jb2
zMZU>1V!S1lewA1YM60+(PE~G-{#vZO@2Yjg!erG&ap&He{OGvLvE_7wn*FWN(+!X7
zHqAcW=B&<i)reJgp+br3@wUrs9y9koQuhpaY{IAGb}4bSxxSd@&nK^E>0aA*d%DA&
zaE5!^<xIBywGw-ABVoJc-r0A<le51odnC@;zM!s|Z|C{Qo$7`SzsoJ#ne7%t?^^o%
zpuCNK(N3X!+Z5+*yZh%JGCMw>-)X_zyp98tte5XOwK+rg;KYc7?@C!5S~X|sy}!Na
z;qT5#2eRG@t}sv9;8`ghxYU1k@*<u)&(hQa9sd_^yy48ag8jsbbbG7M3J2bNG|9-j
zkeV21ETS9wv+LHH7fa*LNE}O-DVX|kr77delM<6x@EkjNz&)d;H@{|8Va?;`6$Rh-
zly9zDZEvh-H~GnSkL$^%Yd){CoZDZ>`IuX?&8R`6l<TIKuv5~F-)m3T{&u>qcx~gK
zGt8DZF0X&M+0gswq6>EvnX0=r%=HDL?|8ftINr>|6dJpCmf(F+2If;nyYIc;zt2X(
zP(y6XgimL;{p5|k^eFnuycRnlx0ugK3oG~4?B1TeyY|2GmDjDZH#+Lxiq}t+-fSDE
zwY*9=ZthQp_?@~!A7{_L&i!cT$Hmt_H`x_PDwf^3E3>h$fhl(7f9vTVlXtS-xUs!@
zH}|2UIeFZ(=X!B;?~$CLv}bZ@7`v&3Vw%z8DcdKS#6ACZQ02CFa=FkU6SEzQd@X*w
zwEm&)9r&DY#rfJc-7EpuDurITjU0EBd1MYMHYwyVI{5$7xW;_q3iJKuH04U0z~lcP
z|4u$^DblTd_Q?F1-P)n?JDZN4jL45%_#t)9F~fVk3umobYju37lmd%~zzi3!gUxKb
zULKn^C_1+>DRa%3k?_c&TbMO$$&A9q4E>_cb#G>DTyk=<M(`?|%8O5qsxVFQ2{3X!
zC8lYb|7?q;Yx;Rjhn;`s>_}aDvB$NOYlhK8?;s<sh4ZFpF=R<HhO4L9p4z(l`v0Ej
z{b?))f*pY>(ury}53S<WWL#skZ>~i6np3-SU-`A%>Jquyne%MZCF}C_3})KgvvZDi
zozXZV9KCh5YD3XUGcOg7zy_mSS5{6ZwU04@oavm)Bxi}usCw}7nwOiGk5J-d{rbIo
zz8TN@y<~mozdCLXzL<)F`osC^?sYK@w$Wb~T+cdw!*Hf=^p-nU&iqYu*5oxdW?65Q
z=5x;NxcMB3*?TL$z3@FAm%C|i<qW0^!Z!T!YPGxd7G4SRWKO$ssqqKzlR%$4o`wfS
zge^E0w(y5>JZu#+`uc!PfNO(myLWDb5WA0NsuH_z>DO>KZK-ETo-sM1Vi6{@jvV!h
zJ&+OfUp=-U$W1-U_la_Ugy7HQ2y>>42Wqunmp+;7wzX-}l!;4Cyt*|e8g(%&@!*^q
zw>2PBQoGiZ;gaW^rIHsNa<WpM&E`B1sSwW1HQ|WFSB3OTdBPt#mqwhps@$-k=~~d~
zge4n{PHEioT-dre&_nR36PK*ZOXsOpE(xB}GcyC1&y_mya`}>?C4!#rS2I_6uG)0#
z=KKxQp2WA9<~B_B-G1xWLtpu#9ohaHjE;r`9=0l79ISbdLo_t}WXILbXYHg@)?5j*
z78H_9j22>IO6b<DV{&nHi_YD<R3brgt6=J`(wC`g!=KMO!lvC=C>g0eCD1cfI!I;0
zOqWN#OB{r_)|h?xe_+MY=@)C5o*$RX$~D@t%tg?3fvaxSE6s+2n8G=N#fMvYjEeW&
zFwW5ZdgYK`>EfQoxr-khy`ymC(Z!t2)f<1N%N>3<H6{MMT4Q#6PSBe__b$5g&K2CT
ziHk?C?$^TQ)2?_YZxL|w(EoGqstEIp$TNa%y$^4QO|xZfPvt(K*zUdX#6hwAYK`(h
z&xKQ#c+HtHrTm@~*MaXLp@oj2eM|+8JGOM>m#$&3l&dQjFL$<1**tg3!^IQa<MMt^
zto@%mT_m>c-#V`ly=j3jSEMfqfB7OR{F2(b<wwM)ueqV5Hk0Xwlh=Ojt(MPUXg*xd
zYAv-|<%QSr`Z*UrCd!@iS~R<7;nDvR6S4vWmT*+2ul)8}HbY^>3ciYGt28@J_T{`-
zU-fUrf$jV@K76NCTGv`LKE3?c-c@gVxT@aNiy}@R1vahvG-bsG4(%pqg=s-X%HKPW
zM!H|T<|Xt&m(gRP=-P%JX4VHPeDnMaI}&+Po;B^-HN~&lEzqYl#A4>DIW6VPNw)$o
zES_(1^i@_+M}vsLH^avo>B-96Gi|-tWM3v+(ATZLS(c(^{MBNkwRoV^`M$LI%jdeM
z`+k=6`nOnq>YtpF*fS>%tTGdnoK~0_qAc$<@0<0yo1HzHALmsbFjL%o^P^MF>=Uha
zCXD=Zdo`3MMR$6bOx#h(=-RqiY}$%LYot{9RHOe}&be!}$X>zsh{sgzpni+*%7-fb
zJUMSPy)df~x;FQm_Odf4rmZp(ydk?mW%F#=q(dp(Q*sWrEfMja%XNsW{zs<LN#!dG
z-ZbhPIr>-aX_~S`sA*+d+^d;98z0X0bM^HP-5i>AChgY)4zI;?@@v-z#p@mSG>JOP
zAHK^<+P_Y93cu>gdM}}A|GTD6IqK+Ie{ok==z^oC3b(Cq)c6>-oo&`pbEbu6J(_cO
zU6?w((<FqUa^j?2YF=u>zl;q|Tb<MjHd=o`fz2>C!&TR?!*PiUgHrVGN1p#wH~u=a
z(pGSG|Mx`_dS)*=JH9Gt3ay_!*Zr(dn(6ObXChX)>aGgOcJMy=KQNq2L7R<Z<=2~!
zq&B8aGPvcL#U6ZGnzQWKx_yBTafORlxG6as=S?={T_w<Z;#ipASHax+n%DBT&DQL=
z=&SSV_3XI6(<bi!*gJ7e&cug3hgy?Gw{5U=(s}$&Y^QQ+koMHsb5dq6-RWPY%9&!Z
z$lf=6>A9k7&00GzO$}(8A?o>3clp{gZ@tRBH|B-uL?yU>k@~8k>%vt%^_1@SJ;sKX
zyIzJobB-^NW;=IX>idopyz^MkR@%?`(xU#BMZxBEnEARd{_~<a5+2SAIxzV~aN~KV
zHRZBDMLn7LoBUTLH0^R(Uon$K-~ZKu=od~-Q_hHQ(Q<s~dFS-OunCQ=TetXl=KbfC
zV}6;wGKuGTxa_o$DKa~jJ`l`GixN(JCs4X7)-c;Kyy(iD*opSM&cQz~EYZ4dH}$Z@
zvdK-+M;}?ViLTK(X)wEG*EQ>?7tX?tv*mxEaJOaKwCkTRgChUVNqZjg7I>IF*dpP6
zQqBKK!K5>@AO1VGcu(Y}^>dGKshiC_DgQB);f_^T<e{8{Eo#%Fgj!E~nI#JCuLzj7
ziq|0aTa$OryqT)^I)hDavh2&A=CwmP)A08@n}Z_C;TKxU?$?}|=@<9+N%*@txBtB<
ziv54Lbhl<zpxgf0{zp`AY0r(@G-=+pR%Npg{g#QRoj$HU_<Zh?IN_Pyb>2*p<#S#1
zUqvM(nng|ezhj4_fzn(aPvcFUlNTL3%4zp;*#s+v8A=ivL06;>FT0Vr(Z)I~ZOf`n
z{Y+bp`Zi`W3C%k2@A8HpE^Ak_pFV$OM^$LP`t3Eh+O~OZ@Ho2Sl!@7#M$IVMkmPMq
z8A`QlCNQwv)82Ygc=u$sBCno3+x0(heW<!|O`_2+C9h{+H?-^CNf52xe%4caQ{=~q
zC(O-vDlwKsODY;OsT^3Mx7TcHx#6i-4~)2UTg9fyx#YTKymYFa$T<DuzG4Y`;TMa#
zTa?}PcUf$;eB^Ka_vQR)4*QagCP!Ir-}#Q^TCD!vsa~%QPfUFLPUT>98%y1G?H9V7
zQ;d`DssHEP%b>cMMK!}!Gv$9+%eFju&eyl6UV0jPHg_7gm)vB>palg+9q-?0TuYv}
zK!5JR3HGb%&S>Q=jeC2G_nv>~%v)PDvo>Z3GuB?MOUpA%sw+7=?Ukp=ng@>S&wq@6
zxhQ{n?4)AuH%eRwYGqz#B=hj@akhF_*zatjv+dy8dRN0u_D64@cS*bT^Md-FV>9Qd
zXr&*S6nQR2(re>R?c&zx2k)neIbC;L5!`(?Y*i2k|K)hg%d-^cY~1{Ff2US>X7IoC
z(2rgb`Tzd?wm)JrMV`xmk=deQXOrLSFBy}zmBtqqO#9YwWok-FLfM}!{&T*0C0#6g
z{?+5kB36kh;;Y+)b~tAXOx62%NP_QZ{D(=Z|Ff1gX-P=teH1^arm8BM{cxest!4RQ
z4=avu%MJNj@0zN+jlGg(l7gFr$=2fuTBg-+6Z4!WtNBW_KAz^5^(lAn<|<E#P=jSA
z?*dy-P7@MVU=%iJ3qHWuUeczLWcHb*;NQ3EqOW0D$|>yb9x=-c;vR<Ael4g{wpQP0
zVKzynb!nl*^1@Ej+T7>i(+;QhU9ELrSmbTYwwR@59&2%FU~y<;oidBXO3~~>H}N%}
z1>Q4Ndo9Xcq|v#oqO0DiE5pe-Q>cJz1Ea8mQd&gW7769l3vAmux}6;DU!|0OD=xKA
z%v9SX6gRyfA>7>aP)%yOXxelK&oYtz$0<I~%C1=S{^x(<@xnEY^{9!Ccwqf>39m_3
z^FoC*I^xw`qMjehmI<xsx#D!W&}+$oiqNH9iqlMAT+{saF(I}vH)cia{>iPelNDUA
zHrrY_$L~m(^gJQ%qSoIP{Vp7#&EG`jCQjJCz3c0YYKF|}WQ(@68xvz3m|_B$QWYkt
zSWZ%loTTdMnHFL8cbXZKc#Z6&HoLHz=9Zcm#lmT>Y}JQrY%I$z9FNGE?r~{)M9*PP
z^Q1V-NwqVkOqp?|gE>^t^Mb11r=}Ga>A${;7k?BhP*YV@3DOHuUE2`ksougKId#Q}
z`fusek`}6YMzpg0m?p%~tr8TWI-xtYV0vBvTjfh8wVO=Jkxc*90@&0TW-tXXsTv5j
zRnAbI5FvY^`%jqgmLjvL?G1-+l>TRK)HZCac5Iw0ru#Tm{7m>%A%&?=C5pF&iEl{L
zpA*rPxolR9pgy;$#L8<mae;aPfl)t{G!CaJd<p9hP+~oHh_&`hzY<%Xy+xSakIb9j
z)bmVCXHPeEDOK>5Xt8JxVSPNC{bp|Di`F}wl`jO_q$Rs$653=1CaG7>)2N)MSvaHO
zsff<aYO}NgrA=n35%ZfuX2v#7uDanrM<MR-Pfwxk1&uFinlmC=Ga@t+Q|p9_Zfy_$
z$}S`*P~{sm)xf>)Q;^WK8A3r#SrY@jA4ep-2<|#+nGk)!oq0-q)=Sa<N?fJR9@B2@
z6k0PQf!U&<P_mmrM0uM>cU9%|rjv_nG$+|>G1zm>h)tNmbb+yTreJ&L68|)&j+aYn
zBxl%HO*fnryFI9MN0R+-;hw`8vD0rZ-EbwAWy{PnJNy@RHd<^?iFOpd7||=WLgi+M
z(5)RNOUts{KL)H8YbyKLRM_Zr|3(`37NKpPeUlU!-WKSKDLQ}oF+n-X<o_ndA8CR=
zXUK7B3OGkNTT9r5EzMnd-PE9TrTEQ~Ov#B~-B*1oTQ!k=VtmEQz{zS{2fCOA=gE1_
zbM2b9ZPuj3hfCBKuD&jza*=70u*ZDP&!$<a?s*k!4zVju@~p8uCSpFR=8{BV<%<RX
zg*qB-S7u(4=<P1naf_UCHM}<KyN-|LI@e30H-yxEjG8pFL^dC53cbX(c!O1(W<tm%
z{g3Q6W>=zqY+v_hmUmxxD=WvmjS}4!s}|erT3jwUBkjWCX2}`6(KB4Kmc$y&P<^;$
z&#Ki;noG_;Vu~=FWDzx8#$bcALfg*i3|s;X>a!Ub1Q-}5Flb1tn*^F{FB0}T*uW5!
zIW=vwo|47M84k0L`KxYR_T;O2{;x9sq;i#|%VQTV&%3x~&I$=%YjJ+p?0iA-ZD|!s
z5#o(i;<bgdLmM46H_x`bHv9ZdH}{2p9L<%RzbA#b&UvQZb*rq!cCox};oQ`h+nzo%
zxchUiv}X7J^cnLSs#mM0>;80M6aHbAA)Ltkc$1zqL*MRA6Sx^BS?|>G7Au<+CQ`KK
zgXT;brS3yc?UUJ5D>w>kOP85f7B<e<wJLi-v{^)6dK}NH)S1SD#a_ErHi|8Bo~pTU
zcP+b3>q7~q2VPQK;?Ih#<g&7(8Ki>3rx`@8^mb(1@o3Km_EnF!tkBL{rNL$M{MfXr
z+48kd=CW`UFk3JdMYi1x-LU<{;>w*Hf~}US1uPMi-tV(*NoyvPtiwj8hW&p+*lH@L
zM+r`9H`yTlbCp{6CXL&hIM!^^I=zXj=Kxc}&ZmphR|E<>Bsv~W>Ph&u)N$KEvq+^|
zu52e(i26V5Ia|I~wWsmF)9#e)?=`Nwg$lw~X+CRQTaZ=vDq#0-QLodY>(WGC6*=u+
z7FaIW+@ht?-W4AmAh(EJ?uDY=xoDx!n_0IdIc_<=B7|qc)wbrykO?Ptnns_IT%_J`
zCc0XVQIGGX;1z3yB{y6Rqb6?pwf&joj>hhd;<E+!a4k0pVoPZ_uJBT%GuohAnt@Y*
zg~w;7gThYEfD_u?Czt|G@ZS+kV@%UzKRB&A=-9D*#_}~*(@x4BayNOY*ixdTy>Pzz
z#L3sZA|yDboM)fno;f9W+o>+r$QZYE_Dlyg59;1G(tCP+{q|#ZQJ%B&1=kB3a)ku#
zk^MDwM>6+z5%oLKx<#ATFYlOkx25$z+s{Q@nbmU`r^|S5NDZG}QL};l^}hN}(N4~d
zV%0N*v(NEbZ@l_qb>+>C$~ReW*La8sY<dv6NqzRF3AYdYnsXq<@w8`n%mf=})<Qp1
z>2|>r_Srv5#n)`MI<#4M`~Hs09yg*2-7X&TX9{TEUA1^-z^hfWL~n_0Hw%;r(OZ2@
zT=;1D`>WaNh4F{i%!>B5$_vY0E3j=@$&pzf=lqih6E4dU7dsZSu=!-?<;afhOGMmG
z%Pd|$>x!4ILCxuj89$CSN$*$^vLki-@c<h|PGhMCo)a84C#H6vU|_h`%EQv0v$KDV
zR?6QK-ETy<*a$KTuy9S-$yDxcm8ST`Gkn_@l|^eN|JVC8pL17x+2mcm6ARl~3tMX=
zvaegX&g__eS-EwZpr8P2$|IHfSs7FJ&RV!t=T(M&@XO`z!d0U`sOf6$U7aP;J!_)f
z+C}GHpHb$Tup#M6=$~oK>5KD?8`)Oxv&)*K7qI$$^^Co~OT@aDG-%#&w-S7BEhzIk
zQ)c&u(wlb}ea^FTNNJwF8h7{VpEa9$-)Q~+vq1aUU17$9a)JzV7D<ZoRF!s!yyz7=
z8qpJTTuJ%Qe#fLk%c>9QH=VMVoRY_t<*a=+e(Oc!oUNvRdZkSCl`QsH<t*GC<JG7t
zwlhp3rK!T?gwleJxn6&oc~1&4p2%7Gsrj1pvBkM_<{VF$yKB#yz+3;Pow`!>dzFUW
zZRtw0g3sFnb63xsCRFNk-0h*xT_3iz7xN<KO7&=;m^k~y<n9xU4o`Z{=;r#?Z`=KZ
z@$QqY+-X_1V)2e^n9~aw*=r*Hwb@T}pD$zX#J1L<tvyNczFp@1w0Rz@-mJ}fcjL#~
zTF;kHeC3`BJ!qV`;!;5PLe@TsC@0YlLGgOUGlh(2ekEBmE%Oo-P+wCkB)ow&g~30h
zW^d@r*2>vJccNe3^IrTkyX{WTzT5+g8!M-GXD+FieAP1Z+%moylCL+k-&}n9u8zvj
zjWHJ(Kl3QEW~hBXbM^O}*MDl#&h5F|AM3WoX{N)qd+DnWmfdLQuWh`T)VQ>!OeRC<
zzf{IW_l1p;dxer49K*5~R>)PnbP^XSjptjsPD%GtnO)Pfb5_&$>cvS*9tvx2^J06P
z*dLN3FJ$!oO47EIZO8sC>GGO;<*xNU-FFiUb#@rX>CWvGZL9VOmsO1PJ?XgNz+~x<
ziL%#R?Vcp|GGzTdAw2&sf6mTHH9`yly94)KOHC9}-Y=B<_io-@LEW2*$Fi=^$)5jh
z=0OIfXZ**n&wX5A=D)VN_gT}6&uwdWx!klkbzQmLx&BVmI?kiJcg2XVX4#_}DDg~4
z;%(8MHNyR;d~WG;so$G<>&l##<?b)l?tQ(o=M4Kw`RQjSKI5LWwPX4Y-AQ|OXYAqK
z|Ni%ew#qwiW-j^v)BE_(EBnI>brh;=F6fqSU&rvr=ESafcXf6P`SWS}zZ2}+p*1mI
zXtIBrfKQrS;pJz&2j}Dqy`Fn-q4cs3zGZ8kyh*XUxMk+rOFIr-JMrt%yrd7s<^SJ0
z7H(YlY{v3$p}#F<nx0ScntC?=_P(3b14V=S9k)5i{$PFoZ&HFn)X`<r4J7wTu6z4+
zOSz!y<mOdXNhQ1YYVsX3k(%eX_T@II83p;1p41mr<{#fT>v+nB<1rI<n0$V$6TQ>v
z-lmS2Ybk|Kie?r4@fTxgbU5I^z|6{Fk|FT$U<(_!S&Bu$Ll*@h(XgBjhGeH+0alwY
z8VU^!ES_wB6O2+%u`o)9an1N}>A&h}U47OybBcnOGw@kf=4GBZs;s_{$%41l<Jf{r
zjVXOx2~t9<jxS<riQ78st5%?1YzNneQ%hHebi^*<a!E4i6bo7#!KvUH_AX{u>D#Iv
z?unr>LHX?4q;7;WBrvYYdD*J1*vO*FD=JZAV_<ZIvseB8T<f2OOpMbN%fq;TeayJH
zRJA*7LBrFm8NyZjo&~jea~h@Q+&62C-rgy+vvRh;f!!|R_n7P1%{gKmX5P^|HAUm`
zoim?rY~DHH#(~5`Y@EU(8eswoH9uGwg%~X6%l)nTI^(5Zh{S!RBdzaFUfbR+uXLhC
zSyW@DQ&(eyQa}R}`<8t5xiwiF8fzA4d1)^Fe`-e8A<pGlSv$E5pM22U%=T-;LIL+L
z8ZKP^TfDo>A_`Y;n7|=@#!f<7X=!iEr%e-kC4@H_s_bWZ5y&8Wx44@ps3c_7)Uuq+
zR5yjC0S;W8LZ5opoRlhlW39UE%ZgKH_ALowx)dbc>vKWP!Pu8?M?|s3i+eSCeijdX
zKIn;@Ug4`_7Cx!yw2GIOzO4dN1V6)<1`mddTeOlw7!+7uaRg2hdgQO*Z2#zdfG_u@
zFUnr}ORrq+khsKQ*K9V6`)crOm5wzGlV$`hy;{(f85X^~;N;SIC3hF8GuQimSlQO6
zR2H$AN6Snlk$2TAE!(-ZVg9S<on3XSWd*Bt<Uc7d?M}o0w~`F?cOOaJ8dE%x=gs<U
zeRl#IqNJ+K4bSaaQlzKn{D@~!jbpRN=JS5tJ*t;wqJ2%TFN+8^YfaWZ+<RMQ-nqmV
zb7udreB$+@cSleC>jMhC(pDFDK09UL+#s}0i!sq8^tYeQ#~aO&wllYzmWKrAPUC<6
z(`_yH{@Sn7U(@^Z_F0D1N()seXmYV|IWBLM?QmSlo3tTw#loyvJ&RJLPaImh@0`J#
zwna+O>(XX^wa68?%63|r>+Pfj;q9|{E-jE(RJxugUMR9Xe{bWT?sf9-`7~GUoU-ZS
ziI!P5JR2mJdL*2_%+eD5a>6^`7jeh_UA=Rvey_EsuUY!!%_T32tZn^x|1Z^Ac*lDB
z>B-G;KV3W<RdV=EIGlt{r+jzTXKi}+L%_-DMQ#83omb~NMscV-WOZ1y_{GGI14}j+
z`FXsUJNbGz=adxF4@;vCr)R2tDn4{oJ-#w%{+iU2TC=OSupe6WZ~cxP|NpIV@;R|)
z)wOeSUnC}pol|M#c~~^*kI0NYfgA}U)s5T_FS$w_7I<b*cKgsCn?;WGm2D;rvqSdV
z8855<*d4#xQX(g%*yy&yakFz9d0%;LpY!<226MI7yv37*Wn`J$EmgYv7?v?l6Ww&u
zRQvO}iWspKGfY3)aIZPN?-`T4$J3rJtJxt;i#D+8_BaYJu29hQNVI*jlRYEo$UM=L
zObq`yIWjWYs+2D;NZV><=%K;4YT7lu5=D`ZfzGLu1#VcWoXB2ywm#>v{B5C6R=YeH
zVsyO{VwE@-du&tGch2Hat4x_7xAAp;&WY}aFM?EpTwFcHeLSphF<3I@__g#Iay&bb
za_FLur_VCa$uB${+AL?Dn|`5jkz-uD`pu4|>RVq4zZ06!ps-~5%yyxWMKYnQ+$VA=
zhu)dD!mdehf=ZXMYw?Mt7EcBJ8o7db1(&Zlv(oROYoq9!is#mujQ(W-c?zp`r5oH0
zo)EtEVwi!Tyib&q_MA6diR`8m*7K#vCz^^x^2>HOPc1e3m8lvfeso2|vaj}QD|&=G
z@7&zC#K2r^;(yl|p^yg;L~b~2;5;^|JNbJ5qo)&j?ir~t9)9WhhjsECt|>+quEIGH
z71FnNEHu0LaVh7n2E(OBTbX_S&WJ4f;e6kKedXmw#nv^&MsG8YTYLI2RGxY4vMDQg
ze&C%2ju&Q}{k@_v-No2L#4V6J>_U@h$<o%{Zz}gTX7G0x)MlTXlRn|Dsv}on%VL@N
zZpT}EbhAQ?Ig%xJ&skfT*e|Df`}VWr3s<?cF1q$hr<38vwu@!G>57To-9kndzU#Kd
z+TD7RE!`Bf|03%YPC?bkk}XOBuQ&{HGh83X%{~|wwXCIRqU!!LQ=~-LN=x>(tkZwK
zNajq&x7>}wr^V7How1*pKJ)*Ji?S8f?HXU^7*{x-K9Q;0>dF|ld99SE!$+qT=MQT6
z&psr4Pr~NYG{YsVd)K`3aQk36y>OX2=Y!Rp-Urh6u|9sHX1KG{TWHn3IU6{YgluKL
z+ePp1%R72F?59`6Ej5vg+j#9+QaqTH3@)i{?An&D8!pj)<WeKg$|X;BWm`K1U7kCy
zW%+{F8yy2qtZoTizJk|zRc=(mF6n>IEZ1`Tz3#ZoC?7P_qFd&KKx@WEr83dZ`<$y!
zpK!i5)AnPfftzvsf3H;!*WOgRF(=o<O|bE@T(3^u+t)SAUrd}>^Ta0i4pY)oi^wob
zGmXkQUJ85FwmFztU2Hm|+s>pZvUI|YDAjrYyVIU-Vii01!d=&|n5jNnK!Ww8+v0!T
zp1L=q?oQezx?S{ALep!#j+}?~+pqX!S$&qc_C&}m_gCtMi+`-vN>29gNjzi3m^zQ)
zO<uqg(J70UoYlxwdNoUYr;_^};|C^}o-5rC$#)6(I>C42<g~e3>brWkY59n{aB0Z$
zJQY}^m3KS!0JGiZ<vdsYHG9tAf5cp|wqTO>7u}57@WVbE?}-O?$Z;#lye&BL+FiFN
zZ^|R7wO!%wbQ)A#9JtpqT(5h}oWEzu%{-H7pC2XhE{-}A^)w}-C#&VoEMA$w6&o#A
z<(|3FDsk+jrOBIz!N$)*CUY)G76`m+5X-P=vT1Cpbiz!b|Hos}jPkYbt4-$;SS8N*
z|JF{$)GM278^nYTY%*8$n4qcs=Ma|@yTOKh{-|?*xVHF6S~xV<Jx-E-E$A%0>ur?r
z+ced`8E4rxEt=Z)CCz+Ig^Srez37ZdUa_Z77Jb#2w6V=&g2~M7Z6%j%y%G;cXNgEW
z;3{`plJS_W>TArUuR(2lTR5gjD^*)_3OhwjzH7L%+xm0M^qG%K%{~YOO!&MmP%pV&
zZqC-d^F0|GW_xrrscl=(X_%ArE9k|pm;-wIe`Rg1yz91nh1<g&2i~;uL`9fe+<uk&
z?efu|ESz#xziKBpFi%l4lC<#4QhNBsw?g6dV}>=WSF}6NW`Ak0?!bJ0@Ba=P9S;2U
zoV_Yxo`~>*qYZlsCUWnSuw+!>K3P0byio8!MR(K%!!wf@7G*WxULawaIai&TZ&{a-
zd83j1ETatpe8P;&ZYL<}y<pIKp*4-MUsXzb)ojW33l1jLA}TZX8D3!0oxs8vF-yOT
z>1p%EO{<muG;H43-T!$4)2qq*G$u5ubTrvl%58{V{qs?evCrzStBq$cZm|&Bl3--Q
zEHiukM=6IY6ZgZ14<>IpB7I<+A@>9?ffLMIEu=ZbD!ctZcbxW~uw7w);{#n8Mj7{%
zRzIidyFTb1NSN*|a*#!-mGi+;$xj@&ie+z1>@}SzJ0(Mz^T*_ONsB%gn59=Q>QdO(
zUh#iQmX>JAAr2F#!$vXM-&bw!c(Jc##rDn3^K@^pe9=Bqn$Q~Pu-~x5Y14|M`Mb=w
zDXAB&T3>kR*n|o>XUB#YD_x&6yKlbLQK)k$;O3M`6B>T~HgD|ioT;GET*dok;@lIf
zgn7I>TDdth8^t;|@8~+MQG3Lq<FZ-H3OUXjOuK?~S5|D?^<a;cqvoVy!7yjbv)zLC
zqhvy~dhX8>l5NtOY}mM*t8cNg?qiFI6E6x(Nu2z~Ve&hVUIT4@))l)H_v|VXaWI)>
zr+a~AYw+%<2KBTlO@SNaYCN5dOIBMgJhW&E2ZO}Er83JG>g?Vbyn02;ZsSEFGd-D@
zJUinzN;xfFys!EHn&V8TS1UGebGBNU(51Uzi@W#f^PhB%y*}Jiyl3K)ZrzM_M-|gk
z$)cxspZ0D#5~gi=<}*i_vA3e&UX{-72dmh3Z)x>YncFCEbd$+iVad6sH}z6}Pfis&
z{F!0jeak5iXD^FC(*Lg8A=i4d(H!o}Jo}4wZevL}Hm7BIOT+Tg4!bXhow^--A1B)x
zGuiK&+&$A~-@=2-jaa0fNUvv!kg{W(|8e5fFPZM_#eST#W&i)?T+QLI@__sAh&^48
zHq?GGV0&|p#l}JCcUPlC=fOuj|GQ>S`z<ZCvA?awV@<WffiDhjB@?Gv_ssm&&8fin
zY{K5{7dI)rV&JRjohm52lKaq%oc~_4LN46j!_DHs6f{A*%%CGwfPt%2{LAH*SRMDT
z-uhW<&c<K4U?Uv3=!=z6%2c(i!;a27tfB-?Bu#KSwB}&(7OOpnrAoIfsn~Jn(uzw-
zqK-K(av!;aCoh?>LVAyn%gPlh2To25I>hs!iTi@^q_7pORRRpM*+)2UG~HgHbLaB9
z<Y1<I+*8uJB{L=;&HR1F+}OH3!YeUlUyY)<*;)0?hkE$;o?W=ps-~+U@Q3H5*>;SF
zW>!viV^Tg;=DfUAiDUKYW5(PYt{mw}oMJX{h8yE*w|!AZL*8tkqM%WB=E|RJzgS0&
zmRaX^A8@bEu}FKU(e>J(huN^9M~o#TG;FKb{!{;_bKGE3^E`2K^~_sM-CtXDj|j|Z
ztWcOXN#S4v_kj&3b}!jlUO1=jHs7>44F<y6iX}?FwT@f;y})|cVy6RB`jahOrX8gM
z3>95Wj3)wC-mqKAvs=(~`eGls4X@QoN_Vf94iRV-JsdTwx>sje;o&(EE7tw-V*Pk(
z!<@tCnb%sooL=u}bEMjPyUq<eos9yVrxxT)*!6|mD@tL~<2}=}SWhQ!?d1*Pyd-$=
zims2(r2R93cH4BOojT)aDH5S#cfP0N#YFy@OOKU_Zm3Z_{^P^h3RyolQ-8~~lU+R-
zUKT6#uV78Pa`}OzTjK-Qil=fvG&-{X>KE&J3BR2wvN}X_Vw38V|4PYUqFyscCyO*}
zXyx<?-O!MHZcFKLc4v9cWcgP)vtI2v&U(tP^Ygjgu?P0J9@zVLgEN!MvbhJ$vV<l)
zz8d>AbXS4DMw7?>1smB89oWrs@cJy%KRxD|wMsch@0{7&-L=4OXO=?Q2JNs5EV?&Z
z3_l#L5Ov^t8@ZlaA(m&iRjj#Y6wl#J*BxHp(v@h|y>R!=)4P}MG8t*aGOpNnhxOh@
z8SmSr&PSsz?$X%tZ0SnTw`<ONMMy8b=h1EA(8J}EEb1O~KF;-8Ak+O`*DzO;OD!3?
z5r=!!EQKy6H>)kYw2Nm>*o~$~CvUpmJ!{$`vb4qR2eY~6l#cAl-S(b~zpv)5S^PgZ
zYi_u9QApyF&4v?L=6qQ++hl@DfnCdm`|fKGWzROxO$l(by;5YQ-ZjI^ZR35G#9N^;
z8_g~?E#<r|wr=10qdmS7w>N%`x_dU{x3WgLh()=|j*351#n(y(*>1^vsqvP_f6vos
zl|HfVPx2ki=8OxB)pPFe^pxY9)7`db;-R&|hk52mN%u^eHSy$L@8}Cnr?1A8mfqUc
z!K9nO8s?-uxjA-sLGa@>`vM;v3Y_rpg{J;OnZVUkuiNd5H(la&ZSIxzvtvX591`7j
z_x{mXH?@0ia^8Ql1U$v!`Y-4&-I3zama^6-uIf*?0^jL|r@MDaoL+Wf)*6Y^TaV0=
zzUq7~>Bfbw|EJxg?2oUU-SxFYNpz?9!UZ>yT5rF4x@TjA^3l}Y30(z@dk@bEx@k1w
z(ZZV(9Hw3kHFh|6ZAbLJb<Rw>68p<u=m$qWC}zFWd;4ZlVK95FA>)J={T}@puB-F?
z!fPkp`gZt1v%?gxUMA6NP3v+KlWQJcnwltS#^J@5(ZO-Aalw<82~QfIr6;q*-Ssv2
zeK@h)$6%|j#`8Oz@f$a^ICm-L-DtXdPUTv(?ZexlU(X3jbFVVt6y1GFOgZMdl8lbo
zo$1G}>ZFNI?=zOQTsbwH<B-G7gFDVg89z2=+t4^+_3mA(EpI0;Xj}9-!RxDpor%k<
z?=x<N&WzQJcwGJVR`oJFU%mhL?ti`eYBG2IyoY>@uUkGmZ&G`Gul7lh-x~88sWS!l
zj#p(fE?_#S%&li}BV=CJmNSQc%slM2S!ct~*PAq^Ph92wN%{W!qgPK$KRdilI>s&1
z{mlVKsmKY<L6Oz*#}#>uJeWKtMn;Nm(EcR(U#nxQS3qUj{;KAEz8&hEx;$Ld<ZG<8
zGt9KJE_L`cKJVV(Yi|73Bx=1QuR-bODV2Ji94pe-{=K#S?(@g*p6@ouZu<B9d0R(A
z*}IqPo;-Y;{<`jA<E1Na?_Ke;)o7jN&e$@Iy)twy$IE`P%nxU7*YS9moHK5W=nmWv
zI(c#C$`$Vm4@ytj6&mz9bC(oH^o^GJey>)u{x@K`(ezO{Yrg`kro}5E*;BhalE1H;
zDZm<7k{gxzFZqncyeiu@u4?yuHR1%sKMLr_tr3*I!<3!E5*NJh$ws@3-4fY97z^FZ
z;@q~~J#y*PiBFvMyifhVJqcX5b=e2bt!mAhJGyqe743a?xc8sVoi6W=rQJ&<?<DrE
zWntW!(3ivi=FvfkXG)?sd3A&oEL|8ETw=MDv%K%-yxK=5y8HfgWO1fMT}@+GefdZu
z=h>{U56s^m%k6wz&UUsS%8vK_8|(b%rs}&Nos4o|j()Ay9GFp}T=^xasKhh>i}$*`
z7jiGW`~+UE6ZS5D_lh|#;Q8~7cNsQc&71llq_eNHqi+M(=Kn7$MNaHI+Ox}6+UvCZ
zgkz<u*S>Dp5#8H;>f*B<0zxr9Etz{GJ-*r8F+6@%KJ&zHyE|vJ<1fx$JI}A<=#++n
zx~zVQ>xLcY8!N<hGd|>qe`mX^z~b;^)ib4l=dT3RS>5lql(EMJ34AQ$=W%cE7L+es
z!BAK`eS$=N-sZlpXMD?Q@4srges^nF;9lmUhTOoqvxNb1_m*vZt-M~|E$q|Yu+w|`
z0ykZV+o9MxP1y9vxu-hrL9IPGf5R+K-1{UvolB*24|CK+m7Q<gY(5u!-=)>U;hD)%
zBXu@|ZT~*4c}pu5WG}OnP3va0`{h*k_J8VjhTX2q_p5m}R`s87b&-Gf=-B`CTDC&>
z`6V9uOV@w@wSoQj#`Z7ko>y^7vi=SF?VUb#t9ZqaiQL<tr@M)NY<u=@_x+IOU7W2=
z8yDTUxL@C)yGbnk?AQKlUs?Ygj)>p)s&UWJlpGW5Z#ygeRaB>o-Oe;T=6~hzx5E4V
zE+3teQwqX53s%{#=2pLW`Mw>?3b|F+m`dVjl^!n_y8r7@>Pol#F1D)gJ1WZWY_F9}
z)2zDxYh!Gop!mPW8{Z#^&)%+oNYcFG@zW>L_fN@AuakdYSof6a_^yBJ&(y6xA@zEH
zw(@Z)uitk=#Z;owZ~W^#Z8RmKSwgeAL-6BSn+~6#RJ)(CU$?wJTJ&Ro{q0NL2J9VS
zOgAQE{SRF~&4H0ibdu*J22LghCIv5%h6F{o9&zD_5RO2G<2~H^T_GMu$tNaiSo_6r
zs60B%;*b(&(`l%Be3oP`ljh6f#G{imeal=fY*s!u*LdPTk(mb)CvkYHc&uFIdU~2~
z<nfHVN2Y4e3`{uMr{t`qb8A!j)kU${+jL{L<(%JDsvYKV!;tGe*N)lZ`EGk9-A#4x
zOj*xxFv<LB>Yf=r*L0(o`3M+P9-6{CWqHiGJ(9+0e6v|A!Ze~7#QCC)iVR~mnP%VE
z5U8J%D{VUcc8tZ-dv|A-=iOV;_?z#ab-DP1lbu~QGg+3j_->23nzZ%8E{``%9;sK>
z-7WQer*t%3CI6o3f6Y%^!dw}tbFC}Gwy|81(pS{@@#oj~W_~R}2c{n%%@rD1-?g`d
z2&n{K6${E~oEjeOVAj^s6Y9~{(lBx1!nPj6w(ISQUtPHp7f-Nwq>(dY#-lEcMJpb4
z>S}H2Y&5;)q1++iv}|!Jm+6nisgA#PT*z`RQn)0-t-3N`lJ6w9lnJhBAu8h0H_}wI
z!m@7OYLhlPk~lqqX<2|cS7G3@=_@KNZ|9hKIh?9_^Y-|>Ig(d*##KMqsyJaz?u{99
z9nv;7&UM_n<*>S2ki^#s4DN>PlN$SyR37+A$hy9Ca_+q%ylj#cYvy8e%`H<_c>aCz
za>b&ytU&*HMYl410s>Q~Hpiy^-x?YoU%7Qc++MYB+3V{jP8Hi&vNu&{({Z`ovYShC
zm+RCVnHHK9pZQPa#*9Y6uu0P;a=R}|+)i3OAx^lebluJgp4VokEvWsrZugALWivCD
zuif@!e%iIPwEVpnmqq9O|MpwIaF1ZR%%OdSlVpkx*=lKd3dT%1BJt^IW_l^pp6TyP
zf3Dnqp^~dMCSvY8!|p)s&sUdkblNsSiM2uI^q-ZDW@qQDU*^kedbNqqs^Vd*T#&`W
zmgt_{0o&v1G#c7{oS&KW_F4MQNH*AXWBOICq`gW#OG?(7cB?e)y|A6-(T@8!e=X(x
z+UM|EW9E5>b!tyj>>gEbpHyl5Qzfe+{bayIiCO<WpNfPQ1y)Rp>FZ3L5pU)vJ@fY~
z>z`+tJRIGuFF(E6HD77#qzliv?^-^1p^<#W@>Fie^_>--`bXS68{^L0b-dEKZh{EE
z<kd;;(wq<CJ@;uZdFi?Dw$GH6TD)H;t`sWF5@Go9)a8}Wid`W={vIu>YQL?{TVm@y
zRjJ`fuz#TI)wR$2wkm5THy+JSIPk7DcjuvN@_BKWkA1xsb6)6~4#U|`IzQ(4G%)^|
z-=U?xd9miP+P=bsFtY^7QydFo=XG9ZwKSf^J5SBj(~I}%tt0aFtZ(8ETb|<-4%_f}
z#e`NKYvBTx1Dvy8x>So_jPqFWX=hl$acik)NjEb!<F`F;Qtlf5Px4HdKL7gyzvR!B
zO4{25j9G&=c7^@%WD-$n)yO#L7{%ayq3nsG;kyjCKH*?V%d%~`M_2UfBpsE0yh6$1
z+{Nt039Yg(Piz%C*^wC@l5m?TQ<3MaZeE~O%H0V|wyRsNoalH+{O*_1rJhkM3!=Sb
zq>O*=$h+#+GU4|Ub#{@tVKS>6<_e!E&udp_G`Z-oI6F3RN{4tDM~gyTkgR6z#lCMR
zJcSK^o;d9Abk>B3^XfAeiMSW3*?!^-4%L>FGnRZhaSp4$;-$%<p;@ike^gIL{F==D
zf63Ho33vVL>@`ovv0a|d(#6qm)@T9aY*l}zOU3iPJvsM(RjYX6)ZU!uBAx618*SPq
zX&4amNjp60iu#t;b#BJN4dO0_8IBh-ju|bN6l%P&Wqsq<)r)s%_T@%tT<TjVkf@*2
zmo26;SC;8whvMT1Em5J%^F<7mqb}dr_MpS4aYB>mEl<hXrYc#@pwInnjPmteRjw>9
z44i=-hs2~!+VsOZToM%pWiBUgUGr^9l#x}S)l#)<%T3p2Dy+J={Sw!<^)og4%dWhV
z-Q9KEs%Yxkq}R#zKaB$xTj};R%PdLJd8}~IFD=j8OXa>$x4HVy7X@iiK~v?dcO1XP
zR@iPV^T^<c>M=vTXer)>PgpLn-dK=ne?HSoWLv<5?w>5yxj$Au&tlotw|3I>jc+w|
zT$TQxo;&Mk%9hps-svKfv;0!ZTNip7udNhb($=rB+3M{lF;<^<AN$HL9pU`p!Nf4z
zb4g8LCsXK_1)L`z<m+9C3iIU_)@9jrz5eQ~Yt^rxpFgytPw}no1DnT^YpXtqr~DI@
z+<a@df6+(LwR|}|O<Ro)o{IRe@Y5}x-DVw<r$eqZ2XHM-w25>wJK2_?!;<Og_~+rI
zc_nQ=2Oqw-YPxcH5x?KtnVJoo7vGyNP-yQevdF3Ix3Jw__A;BC^TNVRE=Chp9r$(7
zE4V9CnZ>uYJMKzg%*PXlHN{!B2dB*}P2pVpQ9)4IOJ}0A>$kO=CtBJEeLCrIc+c_O
zheG>{kLJog|8dObe;e<a<hT=YUyIYU9(hc#`DJ{k&LraSIbE+{i%Ch6D>K}UZW*;P
zKb@vidDBZO&wK9OHIH_PY%rME$UVnIc$S^@X0hwbOSYQcU77E$K7Yn7zcs#FvlEuP
zL~S}WZOxL@XSW1nmYsB&W#=ik!I*XB5|v<;C&_s~<363q<SMQd_EZZ|Vu;g>N!2c{
zG<M`@Ixf@QqtN6Pw$09V!M}A!>u1XZN1dE{eeu~JHc$4c?_vCt$5gdi<@K=>ao$Hx
z?)euS{p$_4lkAGsC7G(t5;hhtudgoY7E_zTH$CQn#jT5-)l(j^Tc111tClpCr^9iv
zhG|c)OtN%!W%Qd&)zej7MzQ;<{@bN^Gz9%GiDFVM?)r66?!1Ib>%v?mTbn6GQ&-lc
z{*Rh@@AkDB6;A7dr@bt*xo3FMVS(NK`s>#5Fa4gEtrIh|&TC(n`}J4M;y+(E{-1r{
z=yvwJSm!jOU!oW6AN4fzX_xQTy_DZwmh*5@h^M>rw)azyzPYa%YvS{`b#n{*TX#+K
zs1IjklzWeRuq=77I8cGBbZwum_Uxe7hZgWXU7>gWG0$hqrj|bInZmE$lsuo<nz26k
zs_fMDil64FczjoLj|^GWw`g~&5!dV&m%E}o;<KZ6AGbZUzi(Ui-NR3Wm%J``r>T9W
z>3DwiOC9Eo`?l9B<@LO|zVxbp!c9(|g637bZLa<|I{)2uS$xI!@+bPL33iNg@{Vly
z9nP>e@b5v>{p_FY4<=mf60x|kn_~fMDO;vbL}tW{R!#@@m24V&+4j3A<bCMQ<!H}e
zGNGe4q+?<P&&(Si{<!egJ`gLNn8M+5tVf1x(j1>e$;%h7_55w=>ts1Kv1z_aOU;y7
zd6q7sd%JoqLeJZ9r2L3+6uAFfKz^;^+zYI8g@Po1AB~>AbpijuzS5RECsS_9y3Vcm
zAQ<u2<yND6+)BxqyHjVMjhu6I>YTX?#qQ0N?3~m1M|8o@$OWlxcO=E!-u5b)N-cgk
zH~H<{2eD!Iekv+N-CE4Mi1Q^|oXefIR<X9foh&*nVsp+uZFJZAA1f#NZpQST$J)-!
zs@@|ucaEIX9X`zyM>ZQqySWG(+uq)&dU<2$bWO3_n;nk^yU1<LWsQAvVc{Jm&8LqR
zzL~J9^_-xeA~(}wgRC<P1Rj6ljo+p@e@!K?v6;N_uOr_sh6rDZ-oNwMc9Ds@eHWYU
zlD~K|II)L&e`%s=*j)dX(+OAlC1fRbRdl5IG>9n39E!Z)B0KAz;w8n@8A?G9+yiCo
zzb&@UQ#dBd-ssZv)Z!7>z9@l>ij6B`c)HG2*SMc@N%4HKVB*s`=X;J$xhPbc^Q1(f
zMR!A!`z^kCA?=)(q&%BcKJq5Ity|)y^|0(u#MHehze69M_;KO6s)}5ys92G#<^MR@
ztG=?9-=5U8K8zMsm3g}4hGOcC*1NH-^4Bz<Oc8OrGU1}cPTmC3Fb2L`cWx#wDonk1
zH+J%yg%PjrE}6P?N#wLt$+(At_fpp`jm|rfqjOJPbBXV^1s-oMH7tmUJ#@{fLaw6c
zUY4%Qx`zrUJQ#{M#(7w@e4CTBD&a^HSHt_v;7Fc$y`bZIQm39LEc4(JjZ2ctUOVg9
zjRpF3eCmrXsa=XP<dZk_d$`7Q{wK-gX)EXWbh>{tP2t%$-*i%M(jJAX9arja3Y)X3
z7tEV)H#7N9$Nayl%m3Y#FOKW>uwg5ayOHd2EOn2vi{ye9(~~W^2VG4cu$v}uC`!$i
z@jMXGne*Re4r^P){+O6UYo2$Q`Wz@|Dmt4UxaG)49TsuU2C*9)X1A99Tq^#(m0f6I
zUvA&=wi^k$Q=|l@&T>xRdhF^r*_I=mV_|iOyGLAxOI^}xTfzDS_hSoET4ya06pQzs
zw4zh*;hw+G>T;ql@v@wb=<N<T9sFqa-m4ndauua--t4|3C^>7P<fChWmn3^0U2S+X
z(`nbj(0z)DsY_W@g_3lyEppYAdzBdRZ{bw7q{z5g(R@nNUbszXJCji=@igIx(kFLi
zGgi%=3E4;cX8n5373HoHr&KSJ$p6vvl*|z)g(J^B9KIQxN;>AH<9A!RDEMRTIbA8P
zMXzQp;%Zp;bmihCmV*DON17)2yinlMe<!dkEo-4zmO<8;Y}UuwKSQ#FR$Pyf?>3X?
z`E>mF1or(i=Qb2Yb{$(~{%hGD-gM6^@_#RAR7|<K&PuguYHRWvjtxOicLlVSdQC|`
z@i<Lm&Y_8J<?GzC-`qUx%XjqcV*5DlZM&8qJ9r_lM&WEre~{-qCznQs39sWgSlXwt
z%$c{~%#oFOPp4?9E;-fMr<d6`*Ta#;qs8@KN6V2=Hzk%frZsQh+;Uoy?X)EO;6KiS
zN)6qM?uX|+@f4HSo%-U)tQSX4J*hgt?z3t|&&>FFPgE{l6}`E0mAGH*>T54<bS8#x
zof)$=XTqkb!T(Z3B{S>R#D)evl}_up`+twDte?Q$%G7%ynwbGFFHBl`k?EfNx5TuS
zOVhrdRY??*j1be9cujuh>zQdSEH!}%$1>B)be(n`)4Z#4>5QkgiTra7muC+&mo#yh
z^LnKwEn75cQkqcCo@p%$IhHvI=-fTl=kVrHRfFHsZ)<h=@(vfRa+{Wxv584tOE23m
z@p;ZX!)G(9GT2xBI+9zY6H=yZeoiHiD?NZo=1+rO(X*`LN$avF<duAjf29_Y=s4qb
zl3RL$I#Za4?V`5wTkqIEc2{&QUpzPc{iNm9Sr4kY)_a_s;3jg6_iDPY$I&`17xAg9
zn|^d;6&}}aU6vT~$Z?<6v#xnBrmQ;Ipptm&WJync+yA^J2fA1e7HJ$h$NpO_=eO92
zxePf0u3CQ=Eor^=zT?^Yqn{##@^WmKD$Kqnb|NcUE^%S_-sv+$+x+zkn5w5rr}Fks
zdl+Q&Sc31R*e{l|X?oigG-VDYz80D_;fKzJM~V?*x(accT=NT-t}Re3TdANX$Emnz
zhTVfRGqOE9RzFQ)m5gvt-!*C1w#@9%yfu3xt_v>IU9tAAR@TbgwA_pD|5#-$dG>bk
zHD}KN!-|MPw>Oi-p1;*Q8<twPaar5PWzX`L^C@TVd-2*Pd1chPZm-49zb*4$a(z|S
z;x%c?&xJhxEsXT+cr5kv^0oy>olK7%;C;PQB)dgXn=SClp_g;_wis0W&j{J(wn-&9
z+p|(z>?#MJ_7Q%+^&;7PT039PPk4XH^U+b4C42Aj97<~WHO)m!`_1A-_f9n|Ijr~Q
z(6p3}d!IUj-t@=jRA=azGQC*6XwvCrIae4}MYq24bu6gr&<WDp+`sAO^Q;Xak0fsw
ze2^&IkW$dbVf`|!EcIgD=kT`A5uaoqXy#gOSG6jC8Tn5(VZ&3FeZ@1=jy%rXlzroa
zzX!w4N!nXWkMfoK@hhu54@kVRPE-47SRKQ&1wV>)6%2npTRB0q<(Au*Q!+>N5;uy<
z7oLtOIdt#L-KeyAB_Hp^J=3fE=+SSoY~PIxwvy@^t;a%(eRis?%9=eZPwweHzUhn~
zg#X`K;Sn;Q>(VO!@;txg2{SLs|5R1qeKG#Uyu26d^n%y(Tc;Vn*|(0(^=tE4J>`1!
zeF2ScvrK1xS~rv7{rg$VvyOadU#rgUBU7EWAnJ6wo5sv#CDXlJTo+AQov6WKS1uTF
zWQoV7<K<gK^7hUCsX9Sq+sgivSJYpz2kdOyeJGz{cU100&H~;wDfjhtn^&ZI>!iJY
zpJn(^Hd<-!0e8P?t3;y`dnWj1#>*PK7Va}MkjRsj(o5}s6*wce-Qv=k-04ShuY0gZ
zZ@%ptu<)W{@>O0={_ULq6-8GRa4p}U;1#uGZt--k9>smeGx=0RU85(T?J_fDi&atP
zUZlCW?)%)DtZn~mI;Y(XoBQs9ptRSns1i|qmg-=`n)DVoZU@_hEpFS3oV!`-^tI~i
zw+kKD)stE!!T-ba_oL&}3|HLxVk8%P`DB#z@r~cUsax&~Y4~R(8*h<tepQh9^28_e
zEza>@%k8lGC*K+Wd9BLogSl<2J5SrpV@pnD32(nvGCSSlt8%ZuQD1Y&hp0{dchpzx
z`RadTvxiH=3K!P%H@-IMaD*K7y_0bCywB8>A2TnA@K#n{?5`0F6XD$!kblr?iAnML
z^e)|bHeXh+J(K2Urh9(V;ai_lmp}Q}b-e%UC;#@HCl0Kfs`~U)QljDc7xU-5DbAZ2
zyLL@<r%sHb0dxIM`Gu0VALiWtFDe%%cFipR>14LI7vA+=klM}_9CayeS0tO9)%wks
zw{6Z)^qZ-Y^rU0X1bvo?*<2H?S#|d9DnI&bwYRHI)b@}IR%@Q`D$hMwJ%Q2R@RieL
z^NKgRqKSrX64i+ny0>OoZtni_AnMuX$tBCOQZi;O^IBcv?YwSV_`BmrLMqGW76er^
zE-{liU%4Yc-(GO*i**l+q`t1@^I%@Nu4L8YT{(?uGM_@8?|!3a^zPekrcZiX`bx8A
zq_cFH_9iE6n`i17dFMk}_R@})29fitQd%08Sk-&922b#3G6`ie5dEp(;A3#rr=sR~
zvCWD4n)Qu;d>Brg`YzSr!N}jr+NAd_`MU0+|ItZj*SF7n{x)Ux&enC6v(wGe^3SC$
z*Em<c-ly$b%-$E<A~y8R>*<Lr2wqcm?dS@LQ-LhMSLD3TkiTnF|K^)?(&h!VAMOZl
zzbS5BwEXYXS(}%YR~7Lu;^`K9a%A&~*tcippS^K>=JaGQ!=BFVqU+Y(bUuAx)7^hA
z1#fjXZZ4`Tc(J7D<EcO2?|#~z=4IU|Tyb`GVR?0y)aE|_i_@Cx|6R#m?|)?ev#Okz
z*R$IGzb`nqq?nJRuYk4a)$tt{PRR4WE}3R=@NfLaL&;s?rnZ;7-(JgGsOi%4zlG=e
zkKXv~PtSkYKXVJ<@ZW4()AjZ55{-T*m-n{Tahh3PXEOi4s&rZXe{%n%uRokT8n*|=
z@%QFGx@@^dwC&!ms2{T)q_bUJ75VDfmiD%1_VW#PpFeHna3$%gSA#<nTco0khJ(ZE
zMqc(wLIM*LwL66*lTvmRDm!+IdTz{;FnXd9rs}^*QZ;ZXLr{*zu_>05(|l)XdG;Rq
zS)_V?@|vo5cQg~8uIP1>RNA_8N@nnKCNoFX&6!!DtD;vW?VUCCj96RrlAM{3LUnFx
z#-Cqyw_=)NG-KBC0#EJlx_5V!KYiA_+ciIKU-f6M57rU|F$Y_Bs+gT|HJGj!B4OO4
zQ;|7oxs0y4uR!pjV;s{RkGpIt*%Hhm<Tlas>?+>FWm{iycF4|p_-4j7%SZQC{Ga{p
zQkiu9Znn7B<*TE3d$(s#t*O+0dV0QfzMbu^<Ik8myK=6&{XWupNBt1b`M~7Ey=x*4
zis&cot>}GkKJC4&eR9q3uOD~sS#Q7p(#+&ZOa+Tqun258(92^oL78>Fmx047H%VqE
z&d|jMPOHoo7EIPPyOYwqaoG$87r~=ShDS8*Oafi_{A#*g8Mjp!9^1U5Vxyau#^cp)
zvWY%D%yOkEIvW+sRW7+J_nS--R@oV0WT?LT0ZZe|WpgGeDx{tAVbEqhFzNK!4No{4
z&!@avV?6VOOs}_|_u{jsd#|-@K0RZZkCW*NyQHIHO)p+wKEKSSOY=h4na7(KKF%^+
zr{F5=YVv<!lTg<c#TJ=WB@4W@bww8U>peQZ%)wSkQ#H`D$#cmx6|Jrni^F`qmd%NK
z%Apw%TY6F>v{}n*=@g$!CspR>wT6baas0WgJ+tqxvD&(AS=&~xVrkg6di#kcLAM<%
zj)vXdzT@gP{XGZ%hQ;jpp*4L&;YCp%BPHqCYq(EN>79P2Vs=)sNws-au($4hrNs_!
z<}{xbyO&|rcdt3wXvczH5w&v<uPdneHFV4j*4y=J+3ASKPdP+86(>4;(|ghzwfN-Y
z-Yd!ePnSLEp1N$B?OTz3ey=Crw40T@&oX^Rc2dEC<y~h#I4~S{IF-J1!l&y_t4*d{
z$mCIdB5;h~e$FG8Re^hc{dW;`Ju>Skf2>dUQTg0CnwyTO2p64Dn3;3??e>RxLT_i7
zC9lY9vGh7{sxsN@(Pm@T9o6S{FK21+(_%H4qoyZ%Kge(Qb_MP8w%nUs)mkkN&3QP}
zBz?uhv$o<TiG6%RcLSUpbwj>hu{D};v6Z#x%Yt^ErjjymZB^Cpopw^1YV$Z%ZI?`}
zQ<@qcRFO0_?*Ekj%&>1gk2otAOc2+){whCLE`Lw+wpWonGr!tJtx@97UXy+C)VAAS
zZoj&=?)!75?du9&K7XYD`@Ahr{gHRq<Li&MZCO`-b}jehoA-i$lsdF+IjPAh<IAKg
zEGDtl$!7h`vj@_gHq4CBGm&BGWzS@Cn)RP0J&CRP#=<V=IZsziu)L6Y?&z#$nZCsK
z@|&3)6*)WR_HT7B5%oOpQg_06`i^w_4}l87Ju^1{QQ2s3+!@fY;Di9Xr=Vl#l+N>2
zX-Bz3Gn@imKMI+5OmTyc16Nq^#f}RnOilK=y5uixxN(VTn<VEkH~UW$``(<8uwm=e
zpCTBPY;j<T!?Cluep#2Mn-y(8E21~CZ?@1h4s-Q3AI}0NiL*-b>E`D(iqxj2B=|~Q
zpKaC1s=~8y4qN|<1D8vW*sREYs?ED}g|jYKU{6)Z8<kXRpSMP(2UpJwc1jiO7Re9U
zV->fy!)f)WGhLM@diXCHG5JQT8b|D7;_|DRDx8xv`Twjfo?%+Eh3wYW^v`)L!z?*P
zfmL(DMjeHq``aGu-1qCOaO}ff@;g72@2~FSHtKHlvA=q32Gc(7Ijn1ag1=2wZO!s*
z+9P~2`%R-`v!6-HLgy>lTtbGuc0H|^Hs0Vi7rWH6@l%STw50D3lTB0BilplOzU!0r
zE$D%6V7l4nmCKV#FTWMd*5p#UaFM0Oq<$q^kL0JM!0uODu6Q^yy^eW)ZPPW=6{`Y!
z<)x%$4o|z9_0rHi<MHEL2WClZ6P&cBrG!h(Qc}vL$kT_X(_zjhu}1GxrtyuFRW$h3
zCm%aFRWWLUljheeT+uS=Q(Jz;=(tbcu)O4*W=E;&>#*<@F0!jir~aS2)HTuF`s_Ww
zApV<k-#c^%PuWy7<H+T?E9zWrU#jL;pUs*2d6K5>i;XVf+Ku<Jm;Q|EVtSi?j_JO|
zaZ5XfiH$20H5V5ious={<@($g-S_`2@b-T>&wKurt?!Z!J>7qRC9NTSv9E^tqyu-2
zR2Q0Pu4g!>J(*)(r^thi3v#%&InKVJyzlc$_GC+?L(3CNk`8-5ERMJl$Z$9yx%CvE
z;|}A)((~RV`*S7<-26PLXGeR7$=UN2uGtTHE+5;<zUWcA?Vddk^Clkg+B%`l;>y%z
zehYVG=?F^2&sk);ErYA{NLFvg7iFt=7gv{m*%)#7RAbr0LJygU>k5z4Jd_HV)=t}^
z{a@8QX@}Rbmy@~YoK%%ff9jPr$vaWVH#YT?=Lf$}>r%cM&78czPrmBhM&roTMB&+%
z=g!_#`;ofT=YfsJrFWLo)~$XzU-79%j9=IE8BI@5&tO`0d0xtlB}vXJ>yG+nsQHFn
ztO+&Fyt>LKeDmzBSC=ifyqm~#xlQY0kyydeo1&#nsa#$P%H6j${hyr@uKT?D`P${(
z{{K3&=ihwFl=5eVUZa(p5sT}K6`~sVetHCO?@EraRq1-uRQGPt)q5I8nA+a<3q>qZ
zuH9TN6%#aT3!ClpL``pV9%HpvK8HO^w%6r#ZLT;x``#BpsgOC3HdnV-IR2?q3hMo}
zWjW)Xm8{`SOjG_}xUwqE_gKhPsT&VIDQ&HIsw2`qvG1hO_H|4<*QB@>9XH)KWs4c>
z#FS|#r_Fy8=6fo|EA*dmv^kUKF0KVD<6hpKXTb5i>)y`xhN8_$pZ^qFI!}8sF;*m1
zU;Ff#Q|HQdg>bmoI0;^OwdeW5OMISEce^~6f2%TP?aeOcE}L`5Y<5b%=hC^FxyxKy
z9qw6w-F&QaYH0h@2aFRMgc54)HtKxW+q}=aB>QjqubaE-uHOCn@cZ<NJsILRGMleI
zzFOyy>L0godg7{VuWxa8Bzh9QOznK0%~W;ay!(E^sqcL}W>h_>=u6T%QpU~76V!Kf
zgA)6>h2<qDCQLfna*)sPD!;AT!~cqYJMMmt+&=$a?ZfGd{%UX<%@R#+I$H5}_YwEE
z0te4tzrv!D_?v6{t<E@^I;ri3+4&O}_gs({m$@AHxAINk#0UNM_C<3%JhuJ6%`@}A
zqT$Vx^R{SDmp*wa<<XgWO}_uuuQOZht8nt<A)Yf59~i~k^W+3L!*W&lniP_nUzPct
z(`!E`vYOd(gW9b%XFZ-@_`(zNXN?lW+CV1VAhxv^j9kx8dmpTGfl2&qg~D5v0`G`v
zCoe^~U8z&Ac6xicqkv;-)ABSX&Q8@&_ttUFcbz*yn^T2>K}dl?M1eEmY|WI{J0CDz
z-(zt7!qJ!`A7WmyFKBuj%aLW&(W}^_`{~XEyML$ttI90mW#PVcuRzo7y#ET_#cO&O
zuRFUq<-FA{-?k*DJBN6e>Zu-7$XTlL=Au>(+sWp;Dj$DbTD$B;?!9Mg9W#_NmUIaJ
ze7k&se|1w=1|Q3V0G=9;#vNg6c5GWOI@y77p4X}+s~#@udblAgTBpxY%a5VJh4=n?
z_cN_)j&VIX!S&4Gq{#=ir{9)&IW(_7r=@l%<yn?qlz~zpoA#zRZ}`juSd>@4h)Cbm
zv$!;*pg3vy%dB)oy`$|xUG`P&Y=OsbZsL0#@LGMB-nL`empTqrg)QAy;4MA9;BDKh
zw<iv19Y}h&?1AV?1BO)wj6w-aANX9p^>H)v*X`S&RI2-thhKR+H{b4-BL-|4%6C+a
zIJ6WP7!-f9u!=A+G3YQbFfcSQIWRJC{AXb05Q)fIaL|a2eVbHAzyqhoToJFF&}B(8
zg;H2jL{7LW%<Pk5o#f@QX-R`GlUuJ~$D}776FJSdt_t^dSX;(aFzJ#}*Hx>@4##<A
zOhXRr2z2T1GP25g8MsJu;WCq5SyxsSu0FChs=PEie4W!lt{*=qF9~Q&KhCG4?Qq9o
zd(KhaT<h=yL5sXyUv51e{%TcYB<nv}9lxm!p|9A~?bb})`0$|W|4A)fqHCY*t9@=J
znj9|j{H&%ew@7ZltBhkV>zA#&^VJ}{asAb0zS$GLIou8o&f3`ddEwfslsG?8zAZZr
z_A1))xzyd+QT*(jYg)?ZO}v^d2TvAUI@iiw{(jD`;{TlY=l=QeF7?E6_x1CFCG}Fh
zmiGI{@B91f$NSgf3G?g!G_ndliD|ef%5kOoy{mgyY08Xg(M~~IQx+8Ia5{A9@?SmV
zrTb>Xq)z`e45E(`*DO&DjjOrZniMN6_FT+UUrl7Or{TF9k9r+q*>0w?ee8JL&m^e2
zL~XHv=aUd;uAd7M+D&&Zp6sw~>ypOOk6Ru(t-TQ>tQoUuBd4TB@0H+8#Z8G9y!oe^
zBspa-o4L#}d)Ak<DXNV>FV4O9v{Rv`>eJ5W^Xu5SG@6*E6v|3|nR>CUMl?1&(2=88
zBy-`PK(5TCsXGca8=028>~c}5jk+_3Y1^eXSJjf1jJXDa*OdY_a{q4<3Q*p;i&OgT
zjZ1#odu|=+nws<1Vs>CKXKK#IDT38HVG)f_MZ+)Htrm;Sx@bAUiYeiBSa1H@rPqUs
zI92*W+D^_8O=_7tdq&usMYr>I3*X{csNu`{Q#Wu<t#nk}y=5GVlX`gDriMjC9$mA=
zzBM$rJ4bctY_+v-tS0Dw+v0S}jV~&C(oB`YH6Q!a+(agp-DypqSNUk^nop;mZfsvr
zZNU9tUd{EF)9;`C{dIZqQl`3`80TY(T=R>gS&k=otaJLV;1%t1T~M9LL|3S7o5~e0
zwdjjZYZ<bCx#(PX6e|r6Tt4-0{*B2K|1JoTH1FJYJ6rY1;`4#$e9b3#E-P6wDS!6=
zCCUj6#*sp|kKNQ=q!u+H==AK+=98*3#Ew;de^Q?RJoe)B`Y-dI)fV;5Td;uN%48vj
z(yKUU(Ma+6uO@5Fnz(3ER@90&OHb$}yyax%-BO(SE>6g4=@bW{VwEkcHhPKfd%LPY
z<LRl}4sSjzDJ`6&-g5hvkEPF!fS}gWs}c8|x$U;*u)a3lmd4$pyQ5|&>pP84>$Zyi
zh$@_VBRlS(SwzmC&l2^8n-yjLyi=08AW@({SMYkwq0)7SH?Y;mD0W3``?j?;_Pk|)
zq_djGM(sHqN94Bcm{D-6O}SrFS5f`TjpC*>Q<Isi0+vXeFW*-@Z`#d()VGQKd=DSb
z>pQ`2_eoRa?EeBcm$`q_(^<TfJe7iE)Gp~NacWKAkTzMWF!zIsE>mY~kki(%D=p4z
zFSuMgbKLY~NpJQY<{6X1p6H8kZcxcn^pfdRJ-)KQJLc%5+lRQ0C_P@#QPl8f;v}|}
z$4)rB^lTDdQn)UCmOy8saQEvg4zDhDMfX1OHsBOG?v|()RIo{9iPQ;6*`x&lvr?uw
z{tmim(7rs3t*dxZcj^Kb1r0C76uS<qDGP(byMhz50$Z8VmQ}6P@Zb~<w$bie!Ls2@
z>Z~I(g%U$tgIv9Z3L0lg-t;Kgw!(w4XQArmB2IJmrb#-UIf_z3%^G{uPL(lbMxN?=
zzL96rhRmCjYrgZOG;UM<uWED7Sn-EQ{1&!fYl{D}MB6uV%PbX9HeyPRx!<L+|I4K*
zoqnk~D>Hq<UtOQt%xf}#!$N6omyN7tcUZicGVX93zo{h3cf{wy#~D8HUcrrLwm6uJ
z9$2&?d6U$yDX9*;S$vf@S})$bWNV<aC@42<!m_YaE9YoU5T2VMu<GBMRW(Kzn%oY`
ztn0emwJ}TZ%93M8bvRXA6qPPnEptEis<BrzpfTGktWbQ{uhUw|-nVc6Q<@Umb3#As
z_iT;xMz)L>m<*<`?C$k55fC-~wlPh6x*V%g<jGgU`B};f6%{9Y9G-e?LRQi9c$pN@
zy9I02LKn|ybvxAMTTx`e$~93_-&@H1f1}qVxs!A9VqaR?)xO)lko(<&oRbrJKN#8N
zt&lXxR$uw_RNjm~mG6q?O3z*K&4(c`;e^A2p91k|#+HY?J=VTZS(5LRp?q37-LGWo
z<T(WyJhK+CTC)^I{Nh`(FM-=D?W&p1<~wThfAFkq$kbLo@$Iu?re^#d7At$U!>R^c
zVfVM5I;z{sSK(GBJDn+5XkAyFiOKxXUgp)4?ALo3)PzdL-(kyH8~8ybiEZJHi*ir2
z9($ZyYJRA&WWwR(xmn?D>aY2D1J&e|*tfR5eyCWrMO)z4nVgjYd9MT~XWo7EW1U54
zUzUqSDBHrf6<cGj>TvNkc`~P}tT8jx{_@7CZR@nw|8jh}>QOt>M4Qh}?On9twfxUd
zH#h8TU0=ykBKu!xzrFkqcSEO%zi*v*VfRR|zfIGmvHS50Pib|xyR+X!o_&7ZgsoF_
zmc-#Un-$vi87>|Zey8NQUGiMnBffpR>(fZrkW(+V94tynNS#+PTV1jHx$?e$b9CB$
z??m4+ntojO*~)plk0!AR2hEC_vSsSs(^`jbZCk=~wX9L-h+cSbcdHU}Rlm7t<jGf?
zT`IR;EfGE0cp#iXHYvF7;kFb14{hH6DaN4TyKK65wD&>A(2tIrOV>Jkch**{F_HUu
za$m0L(}!QWERM+Y8LX0cqs{tvg@YrP$B#{mWKA8_IrOhRm*G}e{_Fo0L!VlkP4n1(
zo-pXIJk_&yx6{El)6CPergZ<fGEctaq+Q-4hh?wk*lv8~BerN!bJVB13vaAE<08%R
zSuA{OOUJSeS0<m)`V;kB<nh;YWfwQ!@(FT3@J+O!txh^QQ)R}^jJ@;kTld`lyy*F^
z&4L@|s{TmTdHsHp#+~$*;OYkt(k#jjomAP<sGWYxc0#OUYu9SouD<kK!{#&J+%}o@
z<rbg)srzHs_Sv6v*XwWJ^qs@Hq*ZF}t9(n7s7HI0jGFf@j!^e`X*RX&;<~$U^Zi$<
zW&hiDL4>2+pS5~vVvGIFknKvLP0tQ`?$vl^vvRj7<9oraU)^r64ww4hncwjF_x=n<
z`~QvaPdwLRDmi+&X+w$_vz>(QghSkWcv2TAP5*AWH16wi?j2$~4y3$%e?;9)jQ3cf
zpqb?YGd49<L#6=!Z3nn+i<p^)$>=RjmOjWe`MHqov7jX;vd-?B&S5DE70z0R8FieS
zEVekCKCeEutjWnl-s*eP&xi6sY@rNEsr#MPL$0zp6{o6gRESYZli~;qTpY@*Xu2n;
zbl)VQqz7ru>27tbX0eIsrPne-cBF4P>bCr#9Y?#w|F5N$ZsqYA<t3_R5kCTI%iE4#
zG;0wtOK-23ZI*sD+0N-v(%OU|&8b15#$0Q@scIOiDKUyFACwXNQ7!x^Z2ESY)l;(P
zZ_i%Lp5y-iS*4b9j!AgVO7l*W=Ng?wKF79nE;q_O^rZ8MO70{Un`x7LmbfTnCgpN%
z3>Nv6%M@tiY2IPMq`jG`^n7@7@&wf>OB&v8;7Z!UXLm62OPIp<Y3_a#-M1C$<Sgv@
zCK8mVD02Qp0DGBZ$svE9@`4*2y{uwER~7~63rX~Eb0||(iOSHgRo444y-(0RP^esA
zh&}M%p&qI4wYBWwQ3hF=pQLXV_Dy*lrnWG7#S=s2#e#b3#cY)mUJ6#bo636}oxsQ0
zEFjrz;?dlAq}lI^fYl6z2Vv?wPFy()+JoGS99P5%Ifbz~hWRp^hAmFp{48nl0@G-7
ziKy=_(cwy(isAA9+r<;Nr}s^@%g7KYO_V5iZ?le^5}z@J?OVD1PeU(B*~WI;%IO{|
zf#S;y%~KYdeo|#{dE}b8NcbRQ_ylLJe;yqvKb03wQ7b>>HK9X&nR-?3&1o?PtlG|O
zMUtJ59If203RHS_u2HWRofxyn#CkJ(?%A!?e;&r{`L29=O4H8A@~uq*tBh<E3tewa
z7kV~T;Gs&~tD{`|6>FI`vhCOCVVhL@J$#mrqyC2#B8v+|{;8Tz-Pl{QG0~^7&tc}2
zZJHv#r_UDss{3lo>_q|fa$>W;KB@_EoV|3TIP;N2iBK~u$=T;R=RCY5x#DD>2y4Go
zm?P)3B%W{L_KoG$CKW0w@sIv{@^5kA-|8^$%LV?g7r4J(;QyN-Zah(KYk;BBN<#~=
z#!V-jxF-t!UKuQYIoLcU#eS-hnnjbUhTN7GncXcR>n~5xnmNI&+*>fy#j@BXc$tg0
zWM1GB1zw?!rsZX47J8*Ei<^6BlGGyU*D0b=7fUTO!>v|Mt~i#lb!WWftMaGq>0dAP
z-Cx8h(kglSWdDq#=A!1AzAMAig`=Arr)D4J`!Zpk-z|RsTl0ce%?t2a`ZXadd4a_~
zMX$vPvP&+sCvVS6SBkbMnr3mSGE*|A=S4@~3Zr@7m+5StCL8EI?Zou5UsaM17fx#O
zX^rf<VwQVxS<bSE=9L^Jp{3CWoNI0f3Pt^A5xRLr@aIo0^^HO^SK6ptG}#}xXpu<P
z#=xT6&h8$Dg;u-zgqDiGQVINb$h}{5v9N04rJH?^Jyz{|DY?gUb<zUP?W_EDWv$*;
zwU}vwe#s?wo1Lq2LsqR{r7x(u>dlHiQ4zDVmJN(%^PWa>n_Qgtu5#^-S!-`(tquD%
zk1dcphP7yiX%Rn1v~FthH_u?ni$%;;6W(!!9NsKzutfH=;6n5C;xDs`t&fM;nopd3
zxY;HmMv7C!)tT*DVsLtuvIe86>_HQE3xW72Nms<%5?-u6<Eg(eP=){aVmGHinP`<o
z7d9$!Z>kbjxuPzh&b`TI7H9dv#Z{L#8T?L66W*+1&872y_a;ScE}hw&NgFm>{oZW;
zd$Y{!%`+EFwal(_HImYc-t2vQi%IszfS(E>KNDM6TKG3{d1=Le{jel3Yw16Or7v&J
zON?In^?^))m(Vh{@b2a)%~bxa52B<RJ&RHmQnb_;xTy1Ql<oAWs#)F9tFio1;PgVt
z9gi40nGdRO2+67LoL*a*<141QlG$`^@rHBDsxOA-o^P$btyWV$J6C{l!(1h!yJdo0
zzh+c+nLIgG6Y(YV{B!~HhcjcEE$+GKuUa|FL$EMepm49}>e&Yu&ud(LVD{=Azjn`A
zICc9f)&tplG^_>mR!=<>y=O<)reeXp=YFrgcv|v?^`4(jdpel+ocO)>-v8O0_qF$R
zIPZ%(u=#HF>QlS-sh{3==Jy_z!o6>!g$_jRS(LbH?#8{pX7B&ATe39K?M=YE%USbY
z-sIm~u$FzzTF#sU*G|oQcXHm*i`>T)52_yI<ERieTqdaWGOM+s=yagWp}_esUL9QT
z?9K2hm`N&xb=RRs2M^tK^geqrMP>T}fn9QD*9BjGw7MQ>{5xx*u|~6<`HC%9Q{B{O
zrY9N7a&;_iH1uND-f6ao>7q%r16SlE#f;K@C9Z|%x>Tx@RkT)bvX$PH*t6PN`j}1h
zu?!uC^fkw9S07_OxS1(%OVOQUDQh<8?m3p8b38-mSar=Y+uNIGDeQYRoAvo>))t);
ztu-f}|K~nYaOPNH%%1(b_Am$Tsd#g&vSzcJcdg9qlZvZ1Sz2##|Fk%zY3pkTezl`p
z?^-Qg<+C)_TPW3=pPBJAlj3Pr$J3j27&iPlHF-+|(?#yt2e@J)OH+3roS<Cl#Um^e
zti3R$V_|6Y!qV)y%a_&tURR*CW5NofEnl*yRxI!GwyJr#V8d~Zvu!?eR^%MM(s@KA
zckzwt>tZEu-t6rDxpTV<+sa8=*GkT>DpEh?9yhrv&vl{7na^uY9=bAJ<Xf;!dgrgz
zM|NL0q`l|C?+bUiFAB+C6wKYnv_YDw@gnC&X_?xK;=UJ!V^2J_-lx)gLTxT<Q_m%>
zxt!W}FX`l7YUQ~suDh@Be~y6R-@Wp(mqga?yCQw^hV;qet)&MzdKi5UJojA7w`Q%^
z+ABd?4EJ`OUU%m-Ti`afk5`yJ&I{RlI&!XnKn#Cq<3ZtI!x+((7bga5O`o5g<fM2o
z`0cNCdA$pYql9i3%9@)W{-k9rr?{T~=3#w~Bmb(}TxAYRO|;~0@|=6c*EwU7!@@~i
zjfzYQ6{U>W0v9;$tlFGhbF=Br=I)*o69Z1n^*u3vE$c$tTTue6i*ipa=e;E(%PD#G
zmiXCQtJdCnbo=J!-dkI0Z*84>dwK4yJ+ZeU58U1_duJIh>;AX558S<dT=&*dS=OU>
zZ?4~a>w@e_4%^F9cusmsx7l74UJ^4ediU0p|G&4s<GdGcb?=exy#U$Mdu#;!Yqw2K
zJN-geAe<{a+{u*b0r%>wTq~yKr5sjG4zEmo>34DB_KSD6?=L&su-m9<_JbQT({I|a
zw%<<a-d%P3&Dq{+Z;nDMu}M~EMXlR1cgn5XIp_5m)x^-S%`V~xS+5;oY`)udD^XEp
zU(NKFf^kzzW*u+f>ev)mu*vH0g=e=PtE}aWbhv$V?rq1qeU9snxu3hVCU(m@T?X%Y
zPu%qw0{%S-Y<RLpmeY0J6Rp~(j`yCPk>w2h@HA15H9_uKs^7EZxMz-X&$9KNMXY-k
z^Y58^-&3Xo&!Wy<l3Tm4gk5({z!i7dZJxbrxqX)M{5in8=W4)z-h-TpSM3hJFn{!d
zN#WpwUxk<44HZMy&1?xi_b}|p#74!9MH6;%=FUB&cZPLo+ya(chh!E$QcKjZ>M5>V
ztHr<Sy0gi`z-QP07sgH&yK&MmZPnw6POlD%D0M4EDV%T&3;lB=!<5Y<;Y3!?t>%~$
z%kJJh5qs-e+}j&<Z?DHaFa3A>j-Eg$-#cc(clYDoozZ=ICidyedv9;Nd;3!FPNc(~
zGrX*U4)4G8u>>}}{}%WDXWaXb@7{ltd(Y&5hee+uAmIZ?{k!vh`+Ob4*EDSPImLI+
zcH85%^WwGdt(<dePtOa#w^wBHWm2U-S$z7aH2<pV`l~PJev--0ir>4GJHoSHS@F_;
zLqEkip2bbMTg<eqmTtd1$LKmw2wU8am<N?boXZQi4l_S|kiSYsHK$X^@8MClbIX*g
zS9o`wW3xVQW_?wyONm*8J5a#lK3kqc*S_yNm=exw=rOGMckr>>wZ)NnB0_1=B8^6l
z!e)UN9!N2U>YbR@^Q7p_+lY1V4%R+Pj{DJ8|D$96kFNbcdhY+|*#ACIfH6Sf=T|<)
z$^5L-&cFY??`N{#dsE+iucqua^t^I;*Sz3eSG>=zT~)vK!`yo!cYbY{fA1CNr=t(&
zsWQHtkSC;4e^1F?VEujW)dp+|<&PzT)`k7<Kj$cuR@BI`d&TpKhw^JDY}xYdsH4&C
z!s4E@H`o|WZ~XU{x3O7hv?IiEP1k>^>wEc3Zf;@aVVZO%Ug*$81?h!1jO<$s4w}q;
zbfY|8IOW2dS8=x(6cxH0nwWXT&LkNuJfIOMWL9%y!$JoY=FJX<5rGPBU2K+a7decQ
zMS4WNCy7*UT6A)XZsa-<&fv6D-ApVQol8_FOZYB!Su1rkOLT4N*B4){qt?&ARP}Gx
zx8hwUT9=oG&gP2HS}@OdUBvD>R_%s)XX|3O-?EVw%e}GLFLzfccbU$OEonE8#m?S-
zZCTjbtgVlt&WmXpE%xA4P;L;>3^U`mZu|4qFycUkE;qxSH13&aqjwZM{4<kV^kT5r
z{C_!CUo+3I+PPz1(&w!W*S7rMJ?&wT8$;gRJ>|{+=HB@J;m-ck6EqL6?0I*07ytCk
z&wG3|8JFJiu#d^ADmA~gws>ui<r?jEPd?5Pzdwg1Lr;cf!i!lq)YOkoe_JoRnR89e
zp;-0XPrNsLVOfzng+peO=HynTNP$HjDqW8bv?(QY9OBfTa^et&_8I}MMhzBj7e=KI
z1xJKDw{W^LDmXYXXefO6;Hr|4a?w>Rb55X}MqG~|(`==Un%zEIF1YXn9+OC&S{jkq
zE><;X&C+=qzh5pFU%KLm(dkvI_FVGPs^&cHHT%dO{WGUuX&L*PUXQuF(lBD>(z)w$
zR<?^CQv0oc_L#=)&HFYydECW&q)fP>+jqwUm;YN_T$8s1UbLt%xpZD(uE`~*ySqco
z__l4}4E|O2dOc6jjvKlce|m^MxcnqXYw?sPQKF(@;ptPihDGJ)Zrv5La&ez;{6uHb
zfUPGKly@EG5>60V_vuNO{W>?rWZ`qCT&|kUj@anx$Mijgl~Z@og+u#vSR}qH6kEU2
z;GB46s%L}P{ZjL|irlP;aqFJ#4Zm0Ov2}Y~qmY+}dZ3Bq<K+w5nkLIGUXrycak*UC
zj8z?LFRe;ick0j$?M+XFxb^GL?de`;blK^0_#UrqGtMsHSaU{<kuADM^l(nlw~fh<
zv(}uSbNWhT!!d^LX~qj3Kff`))Z?}`@J7VrLz}O;yw)<CowWGY|EPx?Z_n;{P^8^w
zac@VRsO3|k>UDN^V{{kqHd`>+Z}*nF@6Pq^sFI!VZruyF{?%LiqtDy^xZ};U@5l*@
zH-?G6er7lSuuPbimOe9ek0>K!_R_gb{5;|b?Pkxj7B!mKc(`!6%Wo{X<k`c(c_p-+
z>8nI+%iK5_&pn05WD>tzaBq_dDmbC!Dye!>b#~39c^YfqxOzX`Iz{5NwxOHw(@oc$
zBA={Z_P^`QYv$b*XEtnbx2f6c@vQD_UCO#WVjpuhE}#1E2*398ux}}k4IPDd$2a^r
z>9M8W{XcsG1M82OPK8UOm=1_A?LTDyuQRJ)zEs#|i;E6HN4Ubpw}i4)rmlWfBJjU%
znk8Qz1FOa}9xua1S<Q)4o~j>NBTy9IIIYftb(*ACTj4gvR$Z;d`=T~9i3Uz!+HtBx
zZ+1_Ml50+&=c818HN8OJ^~~Zs$}R~vbGB-1weSj3*yg&-=V*5<%j47o%QwEAnyKfZ
zcv|wj#lG!ZmTDIpMt;>%c4s`msu#cTvFzz5{ADap_00+;=SB&N9C~p)y{}O099zod
z4@)>R=5q30v~zA-y7t(l^`5sb&6_55^T;x@(8*G(cbT2Kw}8uhkqDn=FY_9KOYY|D
zdd?Ls+dSv>2_x&f5Bb(VS+;${mV;_#kGslpF0sX5TWg-uAX0OsY1N@xgTloTG2uKX
z51lAT70&se(SG{ZCY{a+uN|L!n)7_uriJGVU0s!aoqC<~X;JB`zPa*;PBbs^P*GO&
z^z<p3H2Ia{QoT}5@4tV99Co@aU6=7*C;85eS^+6>gT|!`Z~i`Y{MHK}Q`;$vvNmdj
z8&%C`+PFw=Dt8Yb=l3-m9%M$wz6=gO#4O?Xwq1MI!}!YrOEqTw>koQ6QN&Ms>*a45
zr<PCe@~vL-O!HyOsSrckt4zLAWghNW)g*pJlx?Y0=&Cpqy?E_|;a(b&fwJC_7qY@k
z#Wl`mu3gz?dFI(R3C)QGrKvGbmBM2lTCGz`I<juV>*-NUZ0mG76or|iT}7`ua+h6k
zO7u*W5Es2|&=#0eE!mv&|N92@i$-o&R>l03NZlPY?aZ}}eQ!S8NPX+<ZquieQWZ2U
zK1xmXn1$Y&gqxR>-#l49RZqxf`%;-Cg_kBXKb1|M5!0L5(tYxwnE2#YqqA377`)cC
zT$;R6Csp(0fq728<-6h!>)kkDw(Zx1wU!ypq9u2>$$ngyYrQQsG`!35%=#Tk)3s7V
zCA|)`e%1Kw-;t^2x_{S>Or^A06E6Fk-DbR$%epaUw)J(#W0^B`3{I4P5ow(mX87=-
zW9R(nsJOQI+Y@z4`>UfJBcqJirWq=gEUt*mnBnE7ewNFj+V1o++dmiif=-;26Z_oC
zP?Oj%dwA)>C#O%R`sK`T)e||Px!HTd<o|!f%6b-VV1C9mmqUGx_^(xUUlm&yd{Wn#
zp_jFcndO16r0TQ(q8pa8KJj9*G-&nB)v^<I;hDqVF3vXXRcO%Fgn7$PX<s~c+&oM3
z;o}<y#%3Pbtp;<7Hz#H2{`PURx6ElhU2#KVTT$z}fC}wpJ1<>R;7sUBe&HF=bcpLv
zlAva9uY`Eu-=k|z1Zv*@Y!G)XV`k*~mkT<Q-77Y*+gaQ^BKPgY!u0!1)1=GGHs`At
z$~@%QWHrrab8e(r<gG`?KGm}Jox7JFpd<eBgHD@>!{z%6SD#!n-*(x=Z<Q&Rl_WhM
zY}-BQon3F?WNYasm)Jtj<-W@e(L8kjMoLu7?sa9GPfP#*sdGFh*MaYLl&JT%w{3fF
z|LpVq6y}q-ZM)5OR&UK`vsWJ3k-l1O&t&6piL;Y}ero^f{L*mva94h+BDdAYEes}%
ziOQ_`hkmHYiLEtR+W75`x83uC^S^CML}u$>(^M1C%u3{MikepYZ-e_bufRSH*UaNL
zvs~A)%RH75zH8)cCfxb3-R`vO&f53ZB~uSyx^%s1j{OhmkS!^%Qg^jmE;m2=`>y5H
zd3$%ye7_~qIM+<z)Dpj)AxoUJA~<$vusvJWbgFRGJXQXyk(Ga!viBscTBNM8UM;&N
zm*1P`bd=h!Ii}HikG+!9-fdpUvvk8T=QU#L(_Y=2KTA_|wReiLb;*+D|5t84mRDvQ
zws4==>eTDge%4yoa7=F%sz0zqgLT`Di~TF=cgtFCYkxW=FY>gq-jU_5WiF{~%ct#K
z)7slP&G$xK(9tDxXI<aqljr>S_PuShP5dXD7k-^z|4{Gx|1+WMR&DIu%e|v9x&Nin
zW+oAf#?70L1n@n#=o60O`=BNL%0jw*^^WO>ES!EWjl1L!_h|OKpv@-CJ5S%z|2siU
zq-M?rjtx1T!c5MSo_NmVDVAFu!KJ^FUBA$<m&x#s@MJljO)pk%TPAJzp_^OTdD9fl
z?y3{&WeKf?3|#XhSk=z7XhgI(2`Tee8y;T7dS0q0Yy-Q@8?N{l?FR$o-LzVl-rg9W
z(f&VZhn?)>jaGqHT8g@6Mphd)&%1Q$u<Gx5n_sNhw7Tb&^vdpEHucgXGps}mf?eDd
zjTugfn7kJ8yV+!#y;)Pj#(eSucBwhP6{d0q_4)TqT|R41;GU^`Ov_F`l5+0h_ET(q
z5nz9zc<pJ8o-eBxu3NG2KNGj?B$KUNT{l0oAK_YTx^nAj4ZF!Vw?;E}C&g%-UDO_0
zz~R$;tl?ryRYJRtLsMvmgwZ9pwHj;8qgZ`kIMr)#2X-*|&X~8kdPc5D17mV){f>?M
zGT6&otiJDLmAuVWc0|ecm4uPk#*EvCJ5O=f9_h+v_Gt8R>T~Sw=bUFBwbpTx<oV*B
zDNNjxjm%3f>;5lYy#Icc`DF!O-$N>YZts7($b8)j#hT!Cl8?*<nJgA*F4$|ml-1+_
zM^0ae;SR2#9kZ`Y-F}ld#!GsuhT4k{4*W^$U;Q}QYrXk*l;i4G`llP_c>LD**fIIR
z%!BtXA70(PYxSLtGOw*pYiu%#Tp_&JOGigI@P?o+2iJvDJL^`odN?oy{t%j-ag4)y
z(!<3wbr_CcsTTBI!Mb7vyRJi%;faIG7HqUyHEF{SC%NjG`Hv0jC3=Di*w<{9(frsx
z-+12#M(ZOlz04ZBN+p<?0xj1T%yjfw;ug%^>9byM_U`{5H@Tg(o_A>XqMK7ZU+%uy
z<8u4Ml7rFfI2HDUIPUTH+~c%z+W!Yn&fb}!aD4Z=6NfiHd9vV?_0jHMQjsM*-z80g
z1h!iqp8PdiO)_Zm;T1>s-#9AgCDz}?RmyXy|Kh2r3r9UJZ~S^jM}cSUqtC}r{ZTlT
zW%%pM%&v&mqrY7Di!fJpa0Tw@V9Va=lhL!ahiUtruD}MS-UWx-BwE{7+KIiMcgD$G
z@-R=>ht_2o@_q%Z%crd9d}-I^H1o$6{-_HrYmz;z0(%ZmnE7A8eB0-B(~^3+TI}6~
z`hTw8-!o-p-<|z(Pxntq(PQlC4p2J4x<xNYajKGS5NnU;JE_^>k$kgzrfR5)aI5Gq
z<eA@p+2PIyX;v?V&@=kNDKh^)na8kjPdPoUl&Al{<7~$*KMsDncxv?)=I|N8<_j)c
zZ)J{7kY{u_Gv#L2V{eU(TMipW9#ZZxwR^**b4Y2XU~`~>&V&fo6%(%LuN3liXz;jk
zx=w;sLX!2yYbTD5rci-~&;urA1zd&>Obv#iWeS{O3l7T^I~)1PZTQeyrob%My=z~@
znW$XRZzsL#1&%M%o-b~EXp!=HqfHtw4Rv2{oMaear4jA^O;Y8x%bfGOrJN&O7_(el
zxAg9x>@sN!uV;y|x5{<>-p%Zr_*G0dpRW*ox<VmP#P#Wv@L-i)&rKBh_JkkxSm<(?
zds}vojpzQQN4O^+;`#bSan{R_=9bfm#{PMmo5MCRn-?^j&A9x(-Piw$@V0Kr?!_6C
z)FN4DS{~k^aV*i>Epl<I%Fj-Zr(IDmWOWa0(GXy6oyoEI1?!`U+d@}#gtB(FJUv}?
zfu-k$ddS>EbqNzVg&6`XCWU<UU%JBX<j)yOPc9!!*!b!)*9z?jgDvg#JGv&k?V31u
z+xCEM4tI}lls%UC=T^rmQxmIenWxu02-I~5I)BJvzx!*+EUB~k$`L;=2VTB#!sn8`
z+|$4bI(ieA%&I<Z{?sYvf3fE`4^O5$yYpNROzk;&M`On+-6^w7cC=XU_%f-lo0<E|
z3F-5HxxzE~7PHLZcUfW(>Sa}I|EJ(o?k3Ks#UY7DmA~$^iarr+y_fm>q==udPX0Hq
zXuZ4j@~Uc6#t8O60oJ(}IHyzy_{@&o<e;Hdd8^@$MAd|uO_?G6kGRSTZfYb5^qgP`
z{Og+C()58<_r~Yb8Yfu%Ki{*>>?%zV41X)8{l`-4ry@s1K%0WjS5fZiygD2P?s^X|
zhiR}ze`x1n4P&uOkoe4+u$J9u$3E%HhGj9wEfR&ka-Oc=<7_*{U-fm=nL}%s*{(6i
z`NVH!H@nevs`0LdBu}E?_DNoo{~QcEA8|J4>PqQ6UzvB-{-3PPXCKMm>$~cr?@bMx
zU`apbS0XC0dpx5=*o(rso02M2k3N;L3ghbTua-$Uc|GD!*EFWpM!x%hSD4(6Sa|AS
z#0Q>v|Ch=xK6lA0Z!@cvue5PNvynlwo<_5tLGzZECtox0bS`3N<(=UX5wM$uYvGOM
zA5|h}zleA!&Fagb6TG#vv>;OPBGdHKEqYh3bPF^uVpyxy!j$u-U8g}XYQ_woh=9h6
zEujWiS1(}gskjix!1>XKJ8ZApQibC_609l$&C4QM8P=w9tmRu?s>~J2vwZKy{AOXE
zy@m&Gn2D}koai(8hswpIUheeMH|~Gd`J6ISZWZgE+*aKiOiQh|Ge%3kdeil_*;VYQ
z?njU9%LQX*KDyoi<znTt{r@+oPxiU4YI}hB&$YUBdNW!)*WBdY#Hpxh`eK&Kluea9
zyFRMMpW<Ou;BOaW_#fZ(;-alD<BavcesLYtbll1<pK7?oz|1mDl6iT^<txF{_>^bf
zUK{bfNZ+#UrCC91L{Oh!#Ezf@=KU)sZK@DrTIj@bqQ&Q^(BZ6<c^Ov<*0dWk#Cv^v
zqO-%P>VvnghK^?Cq-6`(v)`Q8S;^}AK`_Yes!l+%l}vM21pB<xLK_)4*Dh$)$T+=y
z&opt#)+dQKB|5XWim-}Fay)0emghHr?P)8un}&zI<cw6})_TYl&N-zsGjxrG(R!P0
zzZs0QGasB(d${IE=(mG8;V&+jWwb_CT^CSl?pe0}`~)VW58RfWKG6xSj!Raop16C_
z#)mibj7z^hm*=~F`<cip$vwO)T>r0YP&n$GR2mw7?DLT_-lLk+!Xsa94r=n7m2*@-
z?qXqcp4I2|(!2U?JQJHkSGQkTTyo{8_0$`dWz3fEdaL%Zo?XUw{4kH0cHFT=>>*c~
zOs5I4Hr`as?C`$VHUFzvY9N>Aw`m)9bZAUSEiB;*D@c90SE#(PDey)6{N5(bOzsFb
zrtnnLr2<cUBD!MEF|o~;n}3ueNbiGeBU5DD>$U}~`<`%3-Q22vT8wkqymw_YT3D2O
zKDuwM&57imnV!A)+mk}s$wK=RT7MSGTX*s$NzPJEk9o^waql1Z*Nci`_gKVwivp7`
z&ir}3C{EJtSAW^G=M~?ID$-`@%dfk-g4g8m^Z&;7G5@6x<b|G`{>Qwa^v-GDPcx3p
zI@K%s<U-8s6pQej9czp9J9lfvd9C05vezi0|K%r(tsFZSUgn=rrul92yK^Cd`#WN1
zS{B|qYZ$G;YHZMIUXf{Tz-lZ|8aO{A`bVghp-%s#gDlSv)(NmK5X+j@%E0tBQ<ybt
zjYPJFMEjAj3mzXd!cMTrb51whap<2~r$_DScoxnBM{a7&*tgk3j=8<zkB*Pgs`mBY
z&VN6~;cIX&aK#~y18ztBvI-1C^(M{?wcygbs8f1Bvyfk->6Or$8FDR)ls|BK=lUf`
zu6}cBpVygZX67&6gsl*@X`k{_t76Wq^)2t4_#QL~y<lohulV2IUvYOv(;K^7<GKg>
zbq_yuiWSUDbZg1Iu}$;FnU#jY;l2+aDv0g5J;mjY%JuaINxxpWFgYnaoYp(@cqLm=
zIM*&?4Zpl+cc$7o?+D(Nm&|ECeUYDo<MY24{NC>2Nqd;%>76z!bDPo^_HYeWn`_A*
zmnGYj@mM(~-0JRH{DJGu#Sfu&w`46puGn#?tKuo^)Tc``o@!JCXiBmg{)n6O;!0Q>
z6U$nzwGTMze=LljmzvE}p0#@M?zX0_`EJv@-FziJtgV=oo^E$+k=e%nkJ9%gq^|F^
zJ@xGT`$DriNrm@{-wWp}sZTq5<GJ$Om^(A{uRou4{rHPV6aHRTf4eW^)8)Lx|Hpo4
zy_$0_@W<c#{$ls-m&`qQQGTg?|DTIhVwdjv-1)6=sow6=?QKW%{yx7Z{$dU1Z0qO$
zxSaa9)?8ZHk`WiBFTFHlZP5C<#p}OwX!pr!_Pfnlzv<l%gW!`d&a7$<o6p&`S<dV#
zTZhq-6H~G-dpT$d$69d|JU+EhH=2p%%j)azAGvmmsDGRNqEMtoV~rlyg(j75hN#sn
zDW@iR`kbm;Gr>W!d}FvR<CjGWlBtHtx@BuzTsw`r=FOP=#)E5xs+NjtRG>n~u9q9b
zR5+Nb48K}ToT_H%-N+Styen>PPpsyuuN{W!$9Kr?3|q51kRhh8a<AW+X#VBvtM;+{
z`1Jhry@&sgbco;Hbp6IO?dY{UL3*LbjAk08cojVKS!A4XZjp7lX5g$ZFRm;MFMfAm
zR(JO0c{@EaZmko2SGM)dRw>nnEBTE38io1oHd++yle5gddFbfQV!cbtgV*Q%wJOs)
z^)pZK!rtE^?`8H?f8UpQYMSQr<N4w1{_bV?@#*pL>3R*q;(jS#inSKcpW?t6W%eMI
zRj6V?Bipv^3eBgc9z5F2%jIyaFn7YCw*1?S?hnJc)yywyq~rw6YdNlY+)I1X)k`AU
zOdE_&Y5lwM_<StWkLDTLOiON@(%rt}lB;CwjKpKI>+Vcc($;=-mETs@@Md7Ov7~CU
z{Va#Z7O}t$ERVZHe;(%Z|GzqdCEPa9Vc|(vlV6wnd%mVHxvG5KFgZkQ(S@XGYLO0w
zSEj4IRC(-T(kH|j7m_z!cxu5~&Bu$9k1ZANJ#~>oHGWFetFY#*b2lz7Ypim4oUnCE
zkoP2wNtd`Z#U^chC6-d)=(bv5pVsT<t(U4^H!psAYbL{-sTZ|3sIo}wtUn|bt+SEI
z<Dl!L#a7yqnv3^H=t?ej=~mQS;$zLV?N-!k-5qOlKFNp&PpMwF^O@E1cRMf5eEqKE
z&A#q;JL6i-FYaY}vU|Pz&s(nrE7zB)8`KJyrW+i1>L~HLdEKq3*DeBF0oiE+VQV7S
zrn3BaePlmVc$YM<+>=mgZZVUIslvq{BWwQ4_-Hho7TG0pr(N~dnd$k9dkmH0uU36g
zJfCb5+P}hUoAMox?44J+UF)xY?zVQE^`MnAt;J@dW2WX=g&7`eCSBC<aF}pXE8}lS
zxc$}{np}Q<(^^!c3^oOG+ivMuv|PehDKtdI)pg-XFY&9JPHDWk@Kp4o{noqTOUrVv
z_}`weon1U!?y2j`rir&aX7sixKKq&2+xl)s@<iJ!UDA^UXGluU-+78LW5f0~skf?A
z6mPwqd-q;;=H7RD((jLb%Ztk4{upQTX@RQ!?Mk+-MYmV<@tW95Uj83u`|Vor`r4CE
z`lRnxTxvak^vC01_T8l~!r7~jylvgiS{*$n{a*R!)9dg3{V$h!oU`^vX1l@9L(coY
z?LBVy?{<F)i_=1eug8rTiV~UxR1Th2c#<YIr(><kk%b|tuT$j{1*Cr*>f)|!ac18w
zSrGM9(Ckfy(h?yCYd1qKx2qaKmNPH*yisIO<jhoZ(rT9WYjr-8s;3&z;1rs6q)+Ed
zi(K!k<3VXI{F6J9Iz%=GiM(6Fs*$3|RrP97zJ%+RPys)N*DKihGBw<`i*mKcDJsQ1
z30mA`v&yC5&YUG?%Pj*q9b7exu5#Qo5NO=FYQp9siFJ8TwJ-cR6@DP=$&m(A!9?Fn
z%2Q;fZ8TWvc`oEr#Jo3G|L-wo3|jFba?vdpg|$a@LzA>!)=askwYgR6f5b%X*e{!w
z>MjgE9sB6hw1qko_e+WRtmokg`=ZF-%qAx9Y{wI>_514KUy~&ne1nr27YP)cN>Jqa
z^fY{Fhl+{qr_~cHRoow)nVZhy(DXLvL(@B!i|1@MadH)&a54StrQ1<9`?d+Io8-!g
zZjXLUj=yo3C)VJg%bA2(Mh~9p{|`AAXXC`L*5z2ZxWh^1$OgWs4auUjE*!F&5ytK7
z;2<3Jc#8YyYur^El4K7>vRY>pnyD}3RW^HiB3kFT*s?nYeW!Mc#P2w?<p9sJ6-_^5
z5+`LzoVRT&nK04k)vX4F$X>Uy4S{iQor<L{mL#@sdU=;`uhYqOlR_(Inq+_2bSl(;
z-~YDN8+WeCO1f<I<$;>{`OQZ!>b1;ylls2u$jW)~t}c2nTKuP9>hvoUikAJ=Vl?HP
zqeSH^@yO3BmT!7$xiI`0vvL2Ko|?}y{c7Wm%CoV(Xyi0{%M$$ioKUQf(^{qGRV)pP
zb1Ij*it}8wG1NYPH;ltY?&ig=xD#FC7biIS74b}HNonhxxlm-gfRB{%)af1t7xn%|
zum<jvVAaiVTJ|qtrN*==;(IGIeUf~p+!mUuvCl;$SZJ4W^R*`{f99PNfBPX>o8ja^
zr(;}UZ7wOsxm!IF9Y0NGnf63YxF(F%xZ)sRo<XzvAx6yy$HdMaTCKb6`=YGgg6PYi
z!nSQw_4xm4)Aavp_nz*0)cUA+>5c@UN8<ao6|evDrQ7<n%Z#%en%WGPU(Gzf<m%B?
zD}5PwBa?4mSr)#|&9As>Ny^)qJf~LuGMZlKQ?K^vL6+*2&EI~ku@HVZv!G0Q)kIm}
zbo-4r6M3SxDyE*5%kG-fGNWy}VM-ED-z_iOcU!oon;l(da<s$$C6ndRL>6_cyIt;D
z>=!#vSO(93G{@dSJ@cr^*OkkSJ(`Sy+s+ppTa_Ox7qk7=)dxQ(C?@Kft}4&SVw`cS
zLO9cOjpIpyN3!d$9a#KzZEt&atbU8h>DQ&%DSzcSCoB@XP<<~iZKv%%z1=sP?z$Au
zJIq)*^?mF!=KG~r%I#9O@h86D{9k^@e%-vlMeKDmw;D0ryexPA-tmS@jtVVq+zUD#
z1sqrOD(u@o^C|1rg-Lu#>g?Xr=J0I&Y%ZBKVcQgArMuhITbUS&x!tcSEmWD<E4;9M
z$sHS&r~4RFw*GJ}dDwm2f8o(ism4i_!G6d0t&u4%D4czwzDv{e<;G_w*XM*6oU^o_
z#1*<C=;|sy%he3(n`W}4Jy90ildv)G`J~yd2A9=OHnQvQIB;;5Rm;l0uR=b1jicF9
z)f-*eG$$O4biFOu9{+PuSW?E@^=HI%gKHkeO`9=Y=UeKoFI)2y)ZVn5&<Q*+BeJB^
z^>gA?n?I)gdwdz6oR}T{xoaQmgB01u%i0%m?lSzJy4mloNX`2wv2%x=`=7pTs1#YN
zAoY|#h}(GmoeSM+J+^l)bSlKEWjD7apLzf2`RTo^7pznl#ZP(s%i-~a6&F;sg2Wd~
zcv+h&tq7hQ!hK|yYu&P65x%`ELX)M|^W|TudAX`=&;5vFljb|Fv)f;kcQz~GY}D#a
znqirb;`b!)o%(ORW88Xs-c#pu61dOZUOpqxzr#YBG3oqwkJ);HVN6~9NvdK${|d#o
zw5NuK{h#YzY~s!u^nkNgEIIS|8IK2BcJlmg5G(!ApP@Tp&(vuRPo%l!I-cjcO9zCS
zPZc;+AW)FNZT)mXc*}&Mj*b@-Pw-c{u6i(|Zv{v94vvTg7nf-KXFqngl`rxb|I$9)
zmY8QI4`n4!N#A8X_Lg<>m5WUpr%v3xv-qO;iWhB1o=!i%)T?8|srt7wq=jTJ%0{eH
zk-e&U_pHTS)wg$5uPVx?&6WT5u=(X_jmQ}*Jlb^U1ZkyuZ|}IVy~O*@!@2rPXQ<qp
zyXV5`<d?BMn(RG^Ni0%ldY+zUTO=*8&HaJs<qMX=GEqE7Q+iq@BqwfFnq+#@WkFxj
zgy8BKlaCbyJ-fId{7rk|mCM{(?&ppKpXXtJQ9ILSZd`%v*^3(vOPF}Rv%MuUiA(y$
zvA3>|Wld-K-{k2v;FWj1mC&$wc4ePMpWh3Ymd`s6rL5@tGWEchqkicV4lciXE^Oid
z%hfl|MFa*(Zi%`w@#>KW4_8+nSykDx`r}att|dit?yr(vvT9+}y0cR29hbOFy0wn;
zyo*r(`kPTMx1uC{X7pXTYWFboYfl8D(1Qe5=|3w@S9b9K*pl)qpxv7-;B>*Yy)Dvg
zUAKLoJrVIa@cWF<_R<FjY82GU#D4$quiG>G|BvHLye{<&9htVN+|Qy7$|kQV5S|{&
z!I|3;E+Es<643qOL0H{{eBGGN4DQwjNo|j|zC9BHG7e2TGqJjNVXwn=sX!Ouw1sE3
z9$#|j>Eh0X%P-7X-jXaoGx7Y~u1Kc2^_;O<%N8li&PaZisJb-1lR+i^%|q+Rw3J(k
zX;I5I*2pR({l9drby;&~+WHA<*RMWw`68b;=dQ@DLkxKl%7=osZ<*9_;nJPRBo#g}
zl}AGAQ5OupstMkjVf1%ywr0-*-L#9ImrZrmO=Bk)M%;VcDI7oLjAMwTiRq<NZ<6?L
zT@EoiF^%)UxeLLEKOR-PbJcy9fVR!!SqH?$A5EG6^WJODDRU0G1_~YvQax6yC7)~}
zpW-6=C9y3{Lg-Dwt!x2}cWXlb)K2{nalvZmjh$QD?SEY;Yq`HNbVZAuhGQOkB=3_p
zzqAcGDhg%un=UQ>mU}hp%>4BWkG1$TvP^M`eR@nJiDUEG6`N%rur5kToY-Be5XS7r
zexI{}H7u-(<z=OaW>tW~Kb!vv>|z479S=l09!!dJ|No{v=v({LRSSaMmS&ZzvmHF%
zG-HCpmd33ot{qH`?7ner`cDPHCdnDR5BVnrb;&4p?N}<x7`pV%(HR>A<K?6zPd$|r
zWJ|R2mST#P{4#yvHpL>h6aO71UA0Y9Yn4rnn^fwxJk(7nElh2C+Nx$nZuz2D@^!D$
ziyB2zdT%c6Xge$1ad487*&~N7r*fYOsjiDqJr%o?VY!+Xd+%K>fnTzQk&=aGC)lJe
zOY_AoVvF3rbJ4`iC~1qC+7^<QM;AT0`Y3;D&#?_>7(yit{5oc*tuo+ZH)ML;QE`)h
z+HzZ+#qMV~t@lJ$9~Gb1c&bOj?fC=s&%OW8CBNyYPn(t2!S?Jdo7=b7cCXGgGUcqT
z6lHs<yfkWs#<k1(ftR+fy4+IG(daR?EpEbh#U(`sPkzO|DbrbUv`J%qWXcDX74I9~
ztczW8d{)lynQyFo8N?f&WPLiu@GCSyRAGyw`+r3RZ$EdRS1-0cUE_01g5}-m334l0
z@A+4Kcydur)1OI^^OpNF-V0lN9cu1$pZ7b?82i>G;rtPUrCir$Pt)R<9>>G-?ab*2
zj#U+@oefXVKA2`$D_5{(rQ#-!fGtgG8=9o2o|w<H>@08E#XlDp@VQ)2)Y_;Oq;zuO
zQaL@9)U@)jnNJfZNH_ExGl|svlAf}I@o|>gv!G{}JE!_P{a;p8vhK>#`)B5aUSI0=
zvguN0)3fwtdN2Eyz2@US?r~6T3Rm_vCg}y29F9EOdg9sk8_)7yJ!3le>ix63`TH)X
zu%@qfy2e_2d^e|TzpB<+jr3wa{bOgMAN!?0Ir_-4=JJUrN*Oh5Q@Bn)6Ybeiq@A04
zX#1@*YFiH0ew@tRsIILM8uU{AbLgzkKhwXMKDR#C_BAs;S~cU3;FHC18u2~zt2-XC
zPIO2Uad~2vVZQL3vx|X=V*#tJi)KM6V|{#^+=lRq`Rwc|%_T3sZ=KcDmgCyBrm0KA
zm8pR1`K|D)@~_gKtetp}t>;zuJvZBk8)q(Q#9w_={6M2W&RyXBigRLy6({~jWC`g`
zl3F?OQ;P6=l`4UN3{TA|c?{_XeVFGxX!*cCH7+xFT|$=U(?`E<|Gy%<&Cey6Yfg7X
zKwC?|wz$aVD{3<gW#S7i%s!TT-q+>sz1$U7oHVaCC7ug<@G3&u?)?M48(Ai+o_oF8
zpt0_`ncicjz$boL7QQ~OEL<k;I4J%4gzSsksdt>T|8zNLY&z!PR^!5PWv2f2d7{}1
z)@gSi({5g+^F48)pF`!eS0dkzafF^$+IP401^e1dv1|Mmeb|?^HczSN=iQ#zu=~A9
z8cOe*Or{+$(N~fz+PF@sC8Ou)pSFp)$DK8foBJQRI)}$8X5QDvb!TMQ6Fx5b$)`MH
z<LBZ}|6NU2ow#ViTJh_CeU$Js+m6smDVwDDzemp<m=|wncGo{H^@vHvYn3lYbV_P0
zK3M%dWwrI7bIvT4ze4N`fv(C&G$tQBQ(+?6^Qk1j#7W?YM*U^y9JixIZ<-peHYMmB
ziwamXbq&w_H(bstC(GndWj*nGf6HoOW7UnTE4Jt-OikfWR!EGLPD-48ewx3J-|QcI
zymthOrK+wCb^cgs@zJm9c$hlR?uiRdd)!X*T<dR?P@1q-EZKjLB~OEifJA-n6V_y&
zMVpjo8?x6vogw$waGxRP#cNXB>|38qJKGcRqE%b&_U3XE_oS%L4-}2cjIy5DeG19y
zdFuD7rRP(VL7s>Ge-|_N<rX^+>HhIDyWe=~18aXo+YynIn*QswZ#2H#aqy5=S#v|4
zR=eJ^bK7z^Z~FhPO#ASsS8=iFpBAR8%~&x>E8FjU&Zl)sS>6WDkC-lH6-#gKyqUG$
zFRc8^hmVi44!LZedXl|_L+{HW#YchqW~MABR3x9a2A>E~e|l1>tZ35VxS%r`s~uJp
zRkQG(|4?ev9QS))#x<`-rsVieqq)CgwV2Aw*`nWAr@FcC_WA0|^J>Z=E8iVQEO;8X
zn4QVk@-%5?TlF=K{9F397Qbsgr_M29bv^UVGee`+fN`@$yPpo@b_?(RsFQO1-1;nB
z^>2E&f9Tu%!8iEHuc;r_3(P)!@4xt&Pa8JheNp^6ElIajP+siABJL3Bw?g4!{&AKf
z$vdYx8#bA*EeT+p;&mgfb3u}>@c)93V&#TY(=sLUU#5hs+Gw-0pInhM@vYkX2V5I3
z+nhDb-}};R<pmd+Z(TPIiEYq1^5fgXMQOS2-}bi(tDa9#b)OX4^v%5Y?U6I3&J2RO
zMMW&XIdZy-RQy?%ruX#EDbkqgIMJa;yI`AhOWE3nZD#s&YhNeweEycVzi8u=U-|E?
znhSe`);w)pICsmFJf%1f<+ukQ_%?6y{{A5;Y2C8pVkg#>dmQ{7@Ow+Rb(&<s!HnPO
zXT3{I&TMGguC?i$mX@~ZL?27nOOrGuH#-?UYyAKI^q(yP>prjfEoL<7Z5?wj!<O|M
zzbf6-E$Ol^IZ|>>)zQS+-=*1Zt&@JuwV;`-|Gm;nGhNpEs02o82AuGGed4hlhj>Ge
zw#^BR`R7u1l+|wkcKom#&wByRk{{_)8d*2@yLVTe$Y6Usaobyidym`pJT02U{yBM7
zse!ljlVf=~N6k{4FO}?Pnte&R-)-NAe&PN<AGch&Zt)?5$KURJ)}swBJx>^vEWOsL
z{JF6v$bM&VxbD`KPi{{4o}i>FqW*R&_kr8-J4H_Xz3uM9=zE-_ZR?+$b?n#Lw|O3C
zyxOk6?B}&jrE4p6Lv~dfq+OW4&u2#2g&fXA+230WulN+PlnEI9=SbxKrXjM(R-04O
z<G_K8cTE%aJ#YN=xR58Ou;<nDJ<IC<sBbs>e(T@~ha*hi)oa#Ng+Ak$bAF+_Mfm2j
zvJcI>EKb|0g<GXQ=Xg?QwcbWnxmY_lf1PS?-rCJ>FWZiN?LPMMm)UYY^VeoGzE}TO
ze*0wQg}kmi$HX`;z1tP5ur+HFcd@F`?mvsU4@G<|5BPoO@qd;V_W$zD9XK8+C3G`1
zv-9$WXiRwMq!+xSKv8+(qa#K!dTB>?1SY%trJhLP@#IR8oTPl3!(-;91nH^z(Xug`
z3q*WpYweuyAV}%CqY%3#hl27H5y>EqiE5#{rmQ>|$Zl?ETP2csC2ZTRqdB6%|5r>|
zU@zG8RnuY3!TISYwOp&Gd0$v)fBJ~3@f4qxW=l)96+T@yb@twMG1tDH4qt!YYzf!&
zSsDrXtBscaiSs$bxY#jVy#3bBJ)3<xTi6ZXnMR})Nb)WDKJDd&r9CT~m+F1e*=Y1G
z=VmO+gJXM3zQ1bTJ(0;><Ina>_jY-gc<!^>`TW_bh4%S+3A?_$xV~1u-)}GX=XZCk
z3(Ei5{w`w-I*}84f7RTzVQvq_r^?Siqox=A;r{jgt%s-m{`;N#!hS|poeTRlw>4yR
z@e0gIYGRdb`EZa^d5!}Em+l;gE>3-s6$?5Xcmy5(v2bKDGy3s#F>7gXUv%|TQfe07
zmA0l(Niy}niKqJwkEXN7k2n0;sJ0{~Yn5i-45?EKcJc`~h?u!qcJVG<mg1^uG|wu-
zYpL1F^Byg3iOs8Y-B&)<YF;C9KrHA(w(`2h6sIoLr9o0z=Pt$xo#M5=tI%t5B4%Ur
z(t}P~Pp4hEmGN}ij2$0d8#R8}#k$QXhcQR_*d30y(<bqWzEIk4#(Gje`jOIEUh@^+
zR~Oi^T3kKf6Loa0k-z0G6UE+?;L;uXm2anp-z?)2yOG$cJMBiy#JinVsdK%z-b|m{
z`z<nUsqKq{VG%c1iG^jYJU%V3Cn;4p$RjaTBy66_b`4hftm${l?_T9!#QeDR`n@XQ
z8jbsG=Vv!5X#Sb&z{$({e_qnzMt;2)3aye^I}$rsns(gnNdA(q+^z9yMOd%vD~ZJu
z?5+iRwshHOEK_=%<kK*TSF2Th!oDnR|AiULPHMy##%XyoE^^}XR<Aef&|;jIb=-$>
zlhA_{%^feg16yp2H8k42c{=qJLYvm;?}^II*|7hel+VU}F@;4NgdQ$g*>j9#QpZL!
zC+Dxn!c4kP>-)Q2?qs>b_e*=*l`dnAUFR%z`!2Z2^QhD4HruYrig$~&{kDJLSv1Gu
zpr*Ul=cQ)$Cq8|cc6+7uMQerFFSi5KWA|KL9o$#?DJGt8&&|~JvSrVUW;rzlT{-<q
ztNO-^qJ0k^-d3^yrz1D9xl*!+<G{8_|NZCveEB}x?huQ7$_K|cs%t(tv758hAL2F7
ziMAJ?&!45hCE+#E<$3HACbv!t!4AD&y0ZiqFS2_ictWmu%VOE5Yc!>PPMYL#k?RPD
zg70&y6BkZARcUdUpd~nE!G!}?Rof&rn_`V_&dgG55U}F3Uw7lQ)1xIq(|%-z)+(M2
zW!h@$F1jjc#Ujt856TX2;yog|E#Tq82WcWQMF(5TW^lW$dn92rBcPS(g3q#33s#ul
zJsVb=B${cVq&-jNY3jkQvyO3XQ*@;?vvz4IuU~d-iiuuoZ%j-Wr$vEI^4Y+?(v)c%
zSF!5aT50!Pd8sV3Fm;;BlJw{_qsh7vjjAP@Mn12&{?FU<O7vv4Z_uQvHQ`MLHx+!C
zdTQ9TZya#YG-y<PYB#^o!&$>}{qH*&$!nj?&^G)ual(n96sxG4j$USyH*ICnG;BP(
z=}Vi)#vQ3zEGB6|W!H=sDK4Gixzo#b%CV_oTO7HhGg>+2n^zkBNiov8*`37jl<md^
zUIX3T(vn=w+mtlbOv)IRr%5PDoiaGa=fpVW)`@3qlUTB*FSX=dW)Uo%s@hku)Ol{D
zg81$hx4ebCrk9@fN$xno>9^>>&O2KQ(%pOa8TQM{_Lszk%Uy^~Un04bPf}$=>(>p-
zMZd0K{g~jz_D7&&+3&=FeOa@DJ@Otd6YvV~Hw;l+$Tl_MiT10&|F$Cb;!`_Zw6rb;
zho%@V)pBw3o;5KbYsupN2}|mkSQboZ3UN@rd$J*3izBH^i9thA#n&rS+i9DL$=0x=
z-fTA&LijFu%&W{4j{d0VbZyFlz}n3nWxEdRwFWuM-C{C}@eo;QR5aByR7-ivt%m_3
ztHtGywRQ?A9v0R<>~c)yqmq^i^Xh`2C>2>gKgqisn^v)!8dsTY+tu|VxA$dK!qI12
z*S>j_|FXh;@}aW+qf6h)o!{KZvTf0X@XYp|XO4Q5bb2m(yGVR*P^RbGTiNk5=P)%f
zu*LanCGKJ2ep>l=sz9sbq$H8kdX0Kw_ruSLPgDIlgCV6;ecHoMrj?r(#)j<pe{|j-
z6%B=-CwnjKQn~Z%mF8lJW+VPj#yjHzn$DO;UGi&_caT+64e@!Nl%z4^^u5ix%WWUt
z3Ro&!warOg<C~yEpwF}&do#@<ltnmCED$~_ny2pkvhdNRZL1q=(ia_^sNt#`m7<~@
zSu|0)=%qujWrx<%o|)E*CpxGzG%wOs)jj8#&l@vy`oxE+a|2JlIJMxy!&#D%S<7VK
zc>Q`dQAolz`eUrOVCCi+Vn-`1WhZ{kJ9WA8@64*Ex)a~li7US4oqF-H^(qmSh7BIx
z440T1Cx$G3ZqsHll}X^C)8_@|p^`^e)NSnP@YD&K@^|XO)eL{K7}~cyxU%enms4=U
z)LBAHSN>n%ox$(3npNMUi|5+ZaQ{yW9RHk(5nS~$zGvf6!C0o7K|F<e3u15U25H|s
zp~t17*77Lb|IP|ChmWBWo}L>VlDE(L@YYv{@v}qjZ*8sSiT65JZ<pU`vDGNfvM~2Z
zdC;F;w*@Lk3lA)PfAIC}4+eUFR4;gSrpvv%X#Z(rB+JYLVt0}~o?Tlcwu|px$@x$b
zrqt<AZr^>UwtqLHU)G7q+UoBsT~{&$YB}x|-n{dO*Sb?(Z)6^R{(ItBMx{{nBTmIW
zWh>v9N0Zu5_Z;wd<zSkv@_J6{%Y$6os*ERJS}TxsRAKfE2hCS%6ARj0OcV}YNzE!c
zboq{iX|0xHR?6f}J6@apk6_!I{Bn`Sj2QxY;UURhw=PXfX=hSD;U;CYewohKHj6h?
z8uY#AJv!}m=D~Yq`}D6*J%vu6-k+j2H~&ZB=M!sgbl%%xqj<RK+T^L<t}uHiO?vR5
zFMR=n=8EMOYipY&ru=?a*uSDTZ%d%!+7?gl_#Z42R|Q>VzbfRLSa~-6Ux>2N{<e^-
zZd0xcdTyw%ecNn$ZF?*ym%CWqEZJuU&W%DW&%F-lDyDixJbHa-%BLxdOEk4=eU&!N
z^kvhu*>-o`gpf?%6PbI~8nG<h^lQncDS2TR-i9%(-!d)r<-QjM0*9u)%hf#7*ZN}Z
zqeo}&^*rA8<E^pC`tWBdYL|2$R-I!B6bSntvcV|q!sj<@zhreYf9v15sdeW%Cp)o|
zk6q5~stA}`bK%JjZ^vEU97nzP@GaZZ_aPxI@0p6jvlFZLHe7h3@#u=tGA{oQ4;+@M
zgd}O?tO#{`yno6f{U<Fv8qP<vj-@|I{_AwXL#iiz5>w-hmyHwN9@?>v*UWJzN9tR@
z46`3eCZ`VYaHPMS<a%5v>39xv&x5x;O+ELwX)0<kt2DUk6zIH<S)sb;Szy!SVgvWc
zn#I;E`o&huPfXJJFeB%roBFAk<wYJV6BoIq=`A`CW2m&@mD{uSt3hwYzN@?M@LF9l
zv0m@|m5?Ukg66Aki|f*kF<#D*h*dxP=uDYiR^+laZj}b%{}HFo?Q(5wNUNx5<Nl@N
zptNY&twoLkYb$wlFK#+6Q`^7dP<z4=^&c#1mss2wJNy}!97yUrUum=U=OICPWu_}p
zDlX5`ue^AEg(;{$Aa`}oqk54`^%6nX3XXcDba@J8th<+`spb>gvaCr<L%IF!x=Btm
z1r~d6Qn~4r(XvZPh3~?hH>dAbbZ>9ad8G7#F}<7l=!x)CC%QbGHN76^$T4q{VekLu
z8=+Njuj(nYa{)`{+0QWnYC#(`*I&?#obY)`kY+HeTIOq;9SzYtj<Vf#z9Hgq<4DTH
z4cc7#np4{X;!Rd^gy!wJAv9$f!!x#HoazTSXMb=!ap?(L%T$9j{e_<%{I}3oDLSTF
z!x_x<=Jc#Q(T4?xu1(OKbn=y2$Dyi|OD7#O@<?Q|V=KMqvBQu3uGj{}Yb!o5_%HH0
z`odvC+O58vE*H-j9#!+j2EPu8+&OFI^~y?1>y(qgBBAvkrJRoL%Tb!rrT8LAEH}l1
zU&wtM*XIewzqQ_f+LUEEyI8fS@bipdtGSc4l>Pc!x_LtM0>e&4Uz$+y#(DXX%~iYF
zol5dAo-vlKKYy_y|HqwTxv36v*?M(9nt#5z{4;FH&k62V99Z_hS+3M3>|SH|dlUbf
zeO(RTw71?lW;j);QS5}*8CNeA4_?K|e=CmuU35tK@dT|OUA*t|Z%%VzGGN=bkE2{=
zg4%!k2`1myYTkLkcjA=<|D!n}kMbTp$>DkQKWPiky91^cSrJCU+j=xVhHVVz>epOx
z$<N2wbLtZPnO`)YXJ3md&{{f$C0%X8HU}n;#ON)pqB#fUq7sFdWyc7;xa7w_=|-x~
zX`{&(ihTEL>uy)r!*$H@gf{1N0ln8-^jiI&2{l}*`k?o|_4%p^`g5nSa$bG2t$^7x
zZLZK2!-7RTlUp=;lK#4-8P0T?xyj|wqYcw<IcHj2_0-c7(#+C1lVq|;?RW~4ij;ef
z9+OT=+s6n~4W7k3@0wH<npGFE83k#ZE)op(Qq-7qj-S0Lr%1W<l(J>TQjhM%9t(3s
zJDH1|J{#|HsQrIS$wenu!HswI8c{c%Lkgk1SBnG{4}MiRx;47?qQY6zdYh$Ui-V&h
zb0w6Q^H?QHSSPAJ3RQhIRkDz&WQOYTS8K}utdgC>B7Jb_>m3^%oOtcIdL8#^uh_wE
zd$~|nvCK&&H_4Pm?y<1l(?GlF{4Pt{<=;<s{bVTrcd>$J@0B;lWBhw#g3qoND{2s1
zvqo*I--T0N1-%(hveK)3`Qp!eiWnbyaXwstIi~E`PM=MxGt$)Bi`;t8xs`4S+g;#y
zhedJMgfyO~v$HZ%i!R+;!NK63teo@ZCEtcnzoi08pG7?K(C9UKz|^OaeW{m2&u>kz
zh8DBu$}QRKrph|&J-?de{SPhhN<Vf_>6}6cU-YciVz->_q8MY2ZP|6<ijVHg<%hH<
ztC;V(wrO(T1>Lt-{GJ9es`VCs4w3tQ)L)r%pW6D8?_w{ed{Pc;oqq4z3kIJBXAF9q
zChH0H1XV8ORm^ez%%vE0s$l0i&!AN-C)TJh$~kZ;$za)6odf3#4{7Q#_!xY8X>?&_
z&M`ehxn}(r=Zuw-!$O+h?|N1hu4WRZYRbc9ZFbqp>tb2|oA)e_<QMAy^19~MBJ92<
z-?IE;ouR(_tjiWZ!#__mmRK&_@PB*k>SD|1TN_piMWt|ghp)2Tq!jx?*(TNBrqRo`
zn%`L|jMMZ+(tYDgM&FX`Es{eyFWZRO{}*+!pYOi%nY$SKqPkW6^_#uyqOM&%#I-%F
z+ppt@<Lv{D%a)|;ZgRZK<#UgJ=8ThG_xineDz)5MEIPyO%##$a+1phAv;?><)%W2E
z^->NrbIs(su<^{>{}m^KMZTo+sHz|C%lvtW_gJnYJF8%2_l~M5<~b3X6SjJ!FDjIH
zbSI9<^Jhi>%ujj#;uGeZc>NaB>_0cbe-qp6Y~LQI#-2lb<*WRqug3-H73uEueZD1O
zib3(~DV&a+{<8y^COI%#lur?Tu-j$&q-6p7))-94zY&le(luErtMa68%XiUQwL?;D
zzprc$uS(-7k?fe;lyT#5`xWO5y>m*_RP?xfvX5kH{J+F`blJ)LW9#(poSYT8COcy%
z>-R&YO3fw=foEMF=S^W|w#+vZbB}OiFFZHh!sWTSlVOPDsTp;XPR`3wULxwUCA7|~
zOkrcGSo1`&d_l$ir7@+23vvy`lQru(HZ5$Nc$MQ)Y<Hn*9cP^W`-Y8dad%B^3?4ge
zYp}k4*!wb9#w4$%ho_gVV3*aeSfLc=cXoABXa0{T7n=>mT5^9ivmUtYqLQ++f$PmB
zyWfE+KPBtfH?3fuRQJ++;qj@d(x0~7Vo0fCos~XKseNWg{fzW-F`sEJ%?;cBY;*9O
zbfVp7Qs;iF-ra1WLPxvSovggeta7U=m5J40<J;gekJNn@s>^cU|BpFnc<Mob-XWfV
zqjzR;JPPs-+oz$n``dP|jayHA^A6jhp;_wZx6&`8E$^?GKg-Rj#<xBHifO*Srohl9
zBG6ed`Gt3WF1x$l)WAN!aJ~eC?UT*TpWU0NXKOy?^Oa`L$|=GQj7$zpMS5JiZl6qM
zGp#gM75X9(xGZH#Ud7pftO<vrE-#&avtZ_yUxwA&*6(r6WSDkSq9r_5=e-nnaGR28
zKo3t~^SXr`;nq3E0XnJY^vwRKWc}!S-zFaM<!Xeru}NUZ$1d~!U^kDYvo*b**K*3Z
zWvN8}7H|71Zpj@{FIrR2d?n`cRK=JbI|P3y8&t(T7d12av&eSOr>4&~i!{IfzjN4g
zk>9s{feuw(-}NMZ-1xl0&T`qq$;+I7EX&<<tk^0(Xpf!zp_bY|a-1?Nzq+4DjQAb*
z)}deXWzN%=lOC<UxH4wm6NNKNH7~TLZFOn7wMuDi%#F4g%^N<ZyGpnCusJS!>-%Pw
zf%750AG_AtpYA$qrJARh#qC^kmUpYsp$(#k7&BY9Dj5h$&Q`m-;?c4St<n$^p7#s0
zg6GY@`aY<eOXSeS_(M@sw`}@5W0QIH4*k8#3$_F-{WKx-?7EOF?W}@M!SYz8pX_IY
z9QS2&8_k>?TdS8|y-hV?yXEw2>9LI8O176<GkDD0UKndvZ@oM0W(jNPX6wBk%T}gN
z*`4aQ)!@JAbDo)d3PqBaEVMnayKh$`TY9P4`dbGYw=*f;`;yzm^yb35*@<gH=S+87
zx#F15;t;X+g03F!Q`ZYxyf4oE-T5?9O>ODvHGD_^Jz_ZfJ9`d~sQmL2CKpdkQaY)@
zHQ(__?c?;;?=w^W&y?uhr63rUDqgkF@y->Ce=Dw<#&R0PYF2%IvX$H5&!>dNt3&QB
z+MM>egU@hT=F&|Y?LIj-1^>BpVzSN7^hD|2q`MboJN4!AYhGmAw76T^_0N+NU6lO&
zqn+}S<E>S#jGN}$&bv}@F}iZ1V(hmW4rx6iXN;ZhT$9_f>iVl~7xdF#IyviZ`g81n
zw@Y<!x0&Oos5ugcMV+$e{GaaHYR8rrvRZZO(f2=d7l?gemMqNbO)<FhfR|_Q3%=~V
zflCj6;&i(!_F#(A0}lQRkC!Z;9(SpE@@H*pEq8&-uJ)HFE}5`AsI=R+@mll5i>r3k
zZCDocclo~8Mb?qUseiXU-aMr~_ImmL@1?h|SK4wE+;s}7EJ;=gaJSE2czH*Dr*<Ca
z{|QcW6h5Awv#NK&Z54?tx5~e3R;i_JmeMq9<lR$i6y3;ufjRuRc{oc}#FWMFp2W`W
zG&Rn>{GQK$UZe8yAImg(e$QSfF3jlEz@V4QQuHw3RNrbN7V&9CkFypW=oHb8+S6h9
z*hP=ovfw9&(UKDrRoIIjooJoxV5GHkgUtUEMylRMdX^h?l8vAH%(cnnvgiy-@}4gf
zT6E~7z{`t4saBe`RbN(IUg<H@syFIbR`8PW@JXj`Eeq8yU+Xc^aCfWstu4woS3NaK
zJw1iDc)8Dp?(gg3b_c0xO#EPO@ZeyhsD@$Bj}4EGbWGML;Q6`n$%%<meI#~->9%VY
zm{-sHbaQj`B<195OLi7N%Uly5ov!<Ns-|Y|7FMGv8Vx+TH*!5T3SG#$l4B-ZpPiz5
z=h2(a)6>_>xTZ~Jcq!)YyTxu*PKnVq&zq}Ky&ZHLleT*(M7TLfcI53a(0OS0ue$le
z>*6^P^7|Wverr#Q<WH#D7ORrrn!DrCvHFxH$LCA)tup^F=)(NB++9=TuX;l(?@mXy
z>eoAqjyJQWT{-Ha8db80L35eM#afZlgRbIpcukZgGoO|vOIki-TbTFqW!mhTj~Ay*
zncymynKH>$=;fqIeoCB6t6zD}yfWowXysEE*1XJ%VoVb*&DLGZvEk^9g1tv0qcZ2M
zd^WSN@SB==kWb=tp^{&%%Tueo7Bs{+{Ji-*p37rG15=w=R0ac=%B~emU3RZBmP{~O
z#o|@4tMld3S@Sf7oPFx7o=jep^ipoZlbwM9+N_}_X)Uj&GRkwZb2>1s6}YzQcI4i*
zB@<U}{TnsARJicF>PDuCPIDNpd0y|*ecN$0D=%om#{d_mEsk#NF;g!5cdfmf7o8w3
z%P~c14f8zihh8c*FWj{AI35T(%Pg^8TJ?6@$M~&pTv=A{{xC_~!DVU17Qy<DS&_M6
zzc(#k=x{4cMQ`WhRTdi=gg0j-9n(F<J;`0Zk|)GVq1ETp37Ml)ZU;EbJhsXA9b>ET
zM1$Qum))2B{UUfyzdq#iIsNddQ-WWW1|?O+dX#<%^Dbola>-kD?Y+P>*`;xs>!uuH
zjmocm5_VmFx##ql>fX82Q-cJKMI?5wy&G|RimJeoWKNA)NA46kPje6wI@3BcW>(`_
zNzvJV|9;F+s^3!d$n5+X<FjcyzfZlq{wQ0H+xwj_?r(^j6eL!-f1_fK>%w}?3opda
zw*BArVrP}7Y;MB2ZLTHpR?>00$8H|#WQo4BZ9(1C|8*1F!}V@FtP<n;u&%0n$LlWH
zmphBRHCZ)H6rYI}h8)xVA$26c<^87g+f(!IG%D`;A7!%M;br)if4_d*71!&Xk*0R>
zq|nn_$^WAJPSsBdG<v!J-&cO8@I^vbCpbAg_}gN+_LNVW8LOlD$<(HrnNIO<mMpjV
zm}$@~B$B|R%6MGFt7Vdw;u61%Lqckee&0J6a2uy?R@3!tW8!%*k>kd?P7xJvF-y69
zrySTKE0=_A72;S|@kRB<1iwa~Gbdiugi62MayW8rMcC>S3ax7dCvtDvb<b^IhPG1M
z#W3MT*Bv|@+L-?v`t));O<2Y9NT7qk(E0zw#Z&GCaQ(31Nf7svbN_Lbu}pCV5C7w)
zZ3Zc}4?@=zsdn`r_h3Hh(R4(m)8qtG_oU577A{ja_4GXNWHe!puaSnaH&@t<Cebx6
zx|wqlR4)gu+_F62sn&YS1zu5$XRm#EDVwjQ`^G1xGaF+Ca&xrPZrw4H+{wzAcZP?-
zczWdaQ+HCC&66%aaA-c8&F42sabf34zs*uh->{fHnR52YtQGTEO;0MZ8inRpeU^U0
z!BX;BL`>V`q0Hle6p{JDVufkVb8hx@T5rmh&9!2eeRE||<RiiJGdvdVmd|<_{zoz3
zlhPLFlB*jl?KU0$c*JDFh6wKej&~k=Xy#3n(`#5?!?bdR<hul}jy(r$U9B%w`Au2M
zv`9EY@6}32J0oSYxtzx8y94zUTWsZ)mP9Q46cYTC{lN0D^;KqT7EWB#5*qZhWYM$7
zT5dT_$|jv(TW8IBFz1y?E6=K{`U*N%7TnQdbT#Y>)L656@tbF7+#g;IZ1{LvgvB9D
zVjZW7pRNw?>aK`2K}VuQ6W6S|ba;wp=|{U|mwbQae6s3R$zCpgQ)ffR(>ckN(`?Q$
z@wwhoVCCD9oe(@r{Cz`&%S%SX-gOzx)pxFH<%BMY`qOxRkBHI&$7SsO+*RRWEJ<z8
z_nzkA5DJ%k_n@n5pGUN+hI7$m@7Rc3@0i>F_uh~*eP+-(_rz{t#rNXjDihu6)+E1?
zJ*gZhv}wW=9|f(2`aUA<p7X!1N$vifrtx{T@!YRGn(q=%ola;GW)arSvOlS*mCgB(
z-TQ~Ba2Tgn_fh33GA+6?5#H@rox}z9cjl;=-JajmnRHS^F~2-G?cTF*N$Gl*XO=ED
zG?rCn-Fczar}?&1fBhLry@1aJF30rizginBx%SOrUB_kBU9vE)QnPoCaKm%0MgHfe
zy|cPLajsjGdDp!ON@a&u9Tky2HAhC9C7x4CVAZXu3mz(Yc!>t}yI<X`#ds%af{<6{
z)ELGk9XGCtUi0Gr`KCv~e9uF#Zn+N03E#CC@3PxXpMCy+>$X(Sgq`_HZy$egwVG?$
z9N@UYiMJ@gq~_IcMMLv2R~_EK$*+QUS6y4{r@*8g6dx0>a{YRCaKb~EgVtv~{WH%B
z&tp2OesY09URmz9*w0+a8PC7oEdQvT;Q9BE+QF*ZxAMNOxU^M^yZ6Kvu}Sw<Cz)#Z
z-eE5dxMC#iyL^sn+>*$*5gA(oPPc|>q{P1rR1Ow6GNpaWA$5*db6dic76jS+vj4Vn
zUE%xBZGRd5ByuxeFI~JxKB6@HvDPE8?N<9f_AeGs_K8WUoPFb7^O8BUb<W>(P14=;
z<Jgf@UG=l|h07+-Pkee{l}|xP45xQgR;w1{@(mZe<qE9iR=fV^?QDPe-$i`a1dAE9
zD<{vtQg%jAO*sEz%}1k!oR8O=f1YW`=ODDK)3^D6=j?2GozowsKVNymqUBNOn>}yM
zxw7yhZo5_$O*1~!cERZRdy&qtX~MhaUaQnp{*iK6tz#?a#*R#z$ATR(9CP=CEfBFd
zH~EAE^R;<jZZv6T?zz^U^Lios)DVWWj?G>>m)!Uf;=haMR#;6{_se_l)W3^2^5+JH
zoD^>i{~@;V-ipSqw+prGS2&uk3M_0EIvmCy;=wf4Job>-qC0Ni?k)SuyH%?C#%}BI
z{ol0bZcA9sEmggC8t0USK`-89o|a#Kaozs7<!f2?A98(rGCufQ{NIqkr?ugxN+%CT
zuqpi)n8@nl{6&ANt6<tjnX2b``<rExH~25#thMm6&Q}JRsp<kj8yL@C;$lgzVw@}@
zc8QD2SWQeRh9gOr^AO{yRIXPOV;={_eqC62=Mtl|d#vJhk4vm^k5vt26!la$DOIP$
zi4_+;d}{DL(9pA^<tb~8bf86iaI17#>*r5~=H_0Do)qnAlXjZu`S+@4hqI8XvXE7=
z1@kvik82hj*Sy%v5>~5-h8rdNEff`XODr*w7Jt^xZBj5-#LHI9tfEl8GO$#5LPzx?
z^|p_W-47+&9nH&vlMT(vrgn5DvM0}6p;7VC+=k6U$<5-EYg>D9$}+=*zR4oqYKcBu
zoHsoW-IMfRIsTG#n>s_wb1Q3B*)`5;;tMMp-MKD&tVlf+ASWRgt-$&wFkyRu={5y}
z1?ACY5A=>57T}zukk%lyFI{z;x%}M|y)(=NL{qD`D#kv2Vg2`E=8F}5l8jl;ANRez
z(HD2M{&R3v{zdPv&He7I!i*fDciFNHTl(*sH86WNu$8E?r?^Q?%;vTfOxPi)=i(@B
zpe`PglP#(t+>j%CMdM?-)5UK&-`FR8f3BgkMDve$gu=nx4Q>-UStD;Z7cMgmba)aS
z<D&n1vi8iaS;8DPHyj0%6RcJ?3rc6CB^#;bZeX3R*t=;_50|i@aRi%OVC<{H=C+jT
zsx9*O7rEb_Dz6a!KhSN4p~~{f;)z8*5rWApTf`ElZC$AQdTOi8GgDJ`PYJ>7ugud+
zJ13bOmRfejLO4l;)2+Q^yNSnlsfrz<O_?*?x3@o4%P)DU79{S|u|mA3qVzaR=OjUM
zW3#fP_L;L+b|zdeE&VK>|H$0#TlkGDlDW&h3p(2j#f|lgh20kxKT)YTkQ5!N)HrRq
z*wz>2`#(u<z8NX(=)a~(w$ojDVY5!Zx!|PhTvN;iPe_O@WY3Het-TysJ#||3^_A5(
zSN7fNtUmmFt|(*6k>{>9p_X<>-6a$i_Ph0GAF4@GwRaH>lMK_pv%U7JxV(*X#LrCQ
zxTf06*K6-?@8w{rixr)w8}vVHPGMMGqT}o13*H_NYjbis>sH^z#9$D%@Lb74xtBhi
z0#4mYj_DV<W-q8ONSbs~eFDRdN&ZdgRxWZ;MtMB#LbKB%r8inOidXDO65PHs`0cUD
z31K?t7}Z1#+14a*e>G5(lUO9=U~^?@kyXW1i;U(FPs7_&<3t4K>AJSWX2uCRPs@so
z^UIhQ_`<-+VyRT(v~(*4yQQ&V6#|aieC69(3s<#n-qgBSB>uzK;DsrkOG3KJv)WV~
z4VL^^&N8j5H+e;#M*I3eQ-{qfCK)HXiq8;S=CaArU93&mGbDbRQE9UB%&w0eQ*N4R
zyO}d8BroAIUna$}V$~|MW2HhHI1?5Bi(TBv8ZD5t<ySaUz-q;7%O<3x?5$eS`fApM
zV&Q#}(gD*ZOck|`{@}MgBlW<`H5Vs!e+&~`oT{^b6K7+)%n_#^h1Rs43zuwXU&rGl
zKY5${%V4HwS?hjGv|}w=cc5tL(_fh%yX@XYS-yYO_v%*>bCK`o74xQFQk!|uR3p&W
zw|%~G%6zSt{fk5FIc6`~JS{ZD$swdzPsm%4Z>NJ;;zqI8*-4Y@xMn$u&z#txBq<=J
zdFI;0OVb!6yB2Dm-lUan9u&Y8x=glkljbL3$BmOW@J2gJ1_{_k=h@C~$W@3oQPg%-
zSwCaz42!4*`<e1Z1)~<HHf{~z|8{MWz*md(D=MzP|LbS4v^W|Hx}J7T<ldGb-R2#!
zAT!ePeyH>{<)ufzyDtCcA*;6CKfIM;_AJ$dg0GviGIwmRIX$E7W>LUHU-gSZ@`Wb6
zY_q4|DqikVUfwaIopWc&k{Odvw@VgnWZJ#5c43|7v7M6_b{d(PGfr4l>AGrB_o}%w
z&7%)kuDQUq#+xf^Ve-w8SqmSQHE9WHE3de2Ufg0;UYWU3>SLGP#w-URYZ1p(my>Hm
z6#XNfZ3){ZbDDeYkCN_7t7Q+pNZlJDdps!h$Kp9>GzC_;s;qeya=1zNvD*Sp6UC#+
zF%pe&Gu!;8vh4r4DsRg~kBFq~HQUzz<#Nt)Qe%qlKf!+BJEIp*<A2+qCsnRC&v%(r
z%gr;vf9Yl&PgMc!I$@b^z7<=ZdMcMEnepzbYx}fdVv<G?YrQ;Yu4L9Bv8;u=g2l9X
zWRl%B&q&(r^tJv<dBlHC)%8lY_R<9p)wXOCba|~JIJ<4JSd-4)LoQZQI;WBv&leta
zJf$6&z2t5dcbK%;1}@!QOmbfp)GFD81w3@$Z<(`pgY4I9I^PP?(<57Av$kb)O^f%M
zmQ*!O>~V`vx~I3QLdO<+xrN6|4b9|^iS63uyY*La!LMa`Gg}+FPGqiFo@^ns`lN5h
zj`*JD6$hlexhfp0eyo@jRkE?LWK#6X`ES~%S0$=5&J0s$Z95n?H`=E_*}QAh|Cy_+
zRxQ;!t-p*VWr5b}+iYvJ7v&2jl`=+b<yw9}rORPq+diw4t9FQdTIlD%vc-1ynH%b1
zCvSDf30cn(Ub4f&f6mdP{68A@?5Ge;42WMOJ55|nVe1^;t6V_^3Q32~B^|On`{~@`
z=h{jq&VA8oN>^0VuCdv#bmYB<^oJRJM_+}0vsRE**e$HUD6#TFzNn_mEp>g7!@pjy
z|IO{psO=Rq!}x2bkmk${^}8=#$vnK3t<KVQBd?|cx25~JgA=|Ty(Iqn(oQyYnKuiJ
z|JJo9YN(x_sIhyK=&8$Jk6#W>IP0Rhc^{jz;9gUizXuizl@>otHhv?NQXn?VarHJY
z-%#7Dn^*ncwR_TJ?`X@bU4jmki$xW;K4#QCuEaR!g7lG#Tv2nRm42vhP2hicgLTs-
zu5St4A48U8tv#9+m>+zbZ|_6h7~B2TT7vdY4*wZl{I<r;UMQZvS)${T#)MjlDDlp#
zZ2EOR7QVUtvw1hjFS}VNCG?(k#{!dE6Isfa-?dow#A{=wcZqsQKlkkkvu7N4-TBn*
z_B8J!SsXKaSDac9Wj?)g*W&D5n__k?*zFU2fGz1D*Ume;5*1hPxl{Hkc=x0yC+?SY
z?Ts$CczA((ZHM>W(*=qX1DEf7$lGKoCLW}4&S>_Yv#SNq7Fs>oD3f|nX0zj*=xKY8
zf2w3Xxa&?;x64e|fL@ozi2vulOp}g&B%35yDU%_#<^lKH1{v)?(k)?Ai#iYHop!fu
zRg6}>D`UVYZNPZvZ(Yl#iQP(%WV$q3HaV)FTgY@F+(q}|AFqpVIE9Sl4*aT|Z=M%n
zbNIl8p2to~4T`zW{Hq^+F*@jA<nEQN^_Z>As!Ht*y9U2h_<Qw3mbMG+dJkF8HCNcR
z%gHuCKW6in6OTE5KjvQ}*IOhp@rOje24kPEME1L9>CDgb^%$?sIV+xX<iMuL#ZwnY
zZ_{Dgc;whm)!C_9$+~iH9pu&;Fz#l)y7?mCydbWiXxHqwY)c+QT~Fc;_R-yzz|V7$
zt&CUgor2oB1a7&f(L&D0(^mGz94+XW`(GqAK)dzujhaTYHlv%90yVbwy-r*Ax=&G{
zRY|}zL9kP=Yul`wJEgjg^u5V{wX98Nx%T(v+pF6y-aB#WpXU$Plaubun%<S-tGu!=
z%J2TZlbdsIKkmCdZ_aJ+?K1KgZ>Y1anp!DQ_*h~x$F9vbyEf*$pP|8c;38MffnBH1
z?%FQ1dn5O~de>Pl({^9WJF{X&%7ox@P6Y<FzDEN2k5u?>3iUs#GS;Xv)-Bn0Z`Rk@
zXA%WhGEKhTB-mlzb9Cmtt6t~t&N`P_D8`h)zxIO4(;k19Ray@`xV8t#u^;OB5feRU
z!5r>Dm0jW&ST3=KHwDK(krY0_$az=YKmO4~*_(p?Uxobte{4|__&M*Z@WQVmjBi9f
zK1$?&Y#x2l?0W0MMY~lFZtygE=I}4~TlIwKRuOKcjamz(%l12z-=9#OA76F<62B#@
z!tW<$xvy1QE`KP0s=NA!EiVI?0c#AO%o^#Rie8RQN9$$Z9&mCBHPc~;U(c}r-_K}u
zbIuP*^CN#P)n`2+6eYIiY}hfbBMYB@KU?9sJ1f-1wawi(AoluXJ?)wAxRMlKaXu=%
ze#wUQp5EGh4Y!@xzA0+&OSk7PQtkBB-h3&sZ)L}k67!|+J1(mHn(|R%?*G3B^!^DO
z{u5is@}ys|+W12d@9xYZ-(z+kuJ+HGb@yieNne)eTm3p`GA#V`m<3WS{&VUWPw+n2
z#B@^TNy~<Z&Mk8VrfE4}OmgiJH%)u8BCz3TzoK`iishvg55^Rhf|*8+T`a0xN#~|S
zPF6ds!Zk^S<Iv^w^9(l3EWAc04uK0eJJcpPEen+nTf4}w_0}rUnZD7>a%LQwq}{HW
z5MHDheeF%8U;4>qPd9DQx!sb>?zBYvz`MJ<xjwzST5Z09fwi7l-9y2_OFO6|@?q+p
z8|#vfm3eyh-Qjw_;`nsW-a}nq-%UHwBRo0iWZLA0ux(LSg#xs-vNWSlY$+^VedUl5
zJL`(DTGMh##sjA({H(oD@$rs!z3_G37}d@V!WKRY_BkGGdCjGg;lDtkvE|+U-P$)N
z{b%{}lO@Gt;di;HhL-QGlHXH#?(5b5ddFJw^!%mzdWQeP1`bT@CI$y||9EfE*88Td
zeQ?{UqSF?eSQ-iraTY6c@8mGGdFRaM%u?tgIFWmurAWK5-_i#WJ}>pAT5$5YF{ZGb
zTPG2}h1)_>ee%V1@+W5`ajB{WBw4=^xuB=6%KSlZgZkVlLZ=oi3DII?Sa-%~h3-a`
zobHP{lfBLy3D~W*jp?WK>qDEu4y|>z@K%|4L~`;4e}<p$P8n~p<z^P%X#KG$FhJ^k
zit-j`$-_BQ4E;898(&foZJ6RxDzkKIS;Xq4vb%L|J9HUE6+U+1+o5sricyelX{d~6
z#>bEc9u^);|8rOhG&Jk(n6*IlZBmfYvX2~-6V#K04fMH^l@-DwlecE1bE_zIICW_#
zNgNf^ED<{3ci_#FvfIDkoZ4(%`0<qgoqfmNdUNR&z0Km@aO<hpJ<V+e)8p3PoBKVk
zLb7LicYUF%_Jd6<Y!z#^3&nXSZk00Y*40x;^H$iTk(S}SgQsh?LQmo48B^|SSamFV
zppliKJVDcIXU|mMRWBA#cbv3SWrlxI<)(>MInz~V<b1MRHmBj5=gNh1eO75KY7^3W
zxuA?IOW171EUm27tCU=|*FL-ZDmF&2Z--N-Hea;f#?$+LC|r^V>P+3e@2Ix=fk%8g
z0X8359GY|mHopltA-1_E@BabT2hksPYo#VnIm>dRh3A4&@t1WMPdLaJhgD?ru8(nV
z*?Mqym-fQq-M3zeui;cVz_<JD9_IHaKSa3aN9e^VP2&mM_Ct?Tb5aKbr{)rgWRYsO
z&II9=Sr=1!qE0dM-#IXwKgP3~e~R7Nm#dfmt74Sx3;4a+prv>r&+{q8H6Ks^&vkn!
zrL;~_A%2d|ahcog`;UpI&beRr>-PR`RpsFCre0dC1zyZ5Qq%XO9MCD%<XpmWIr~f7
zETgSW!V$aY%~txN_9^v3&90x*Zhr`xVavKsb8P~1rrKj0kw-6Fo4d{|diTj<&zuPX
zOgloXb1w3&jyS3poxzpa@%iwRJFmhRZqNGfolxYmK=O2i^J&hFC7)d8*lg^)T{KPC
zY$Ah7#zMJ&2YS>Z4)TTdIC0Hh!7B0kAeZlyiK1UKj&5DYVg6U6v65+ma@;GA<9lyB
z?fVfZH1ESU<7(9talckwzqaGh)HR|)6Wi`Y?S6Ds-<>Nc;X{*vLisUUpR}OKSu-Ql
z4ZlhEUOcKOSL~tb%DHJ;smoO9#MCnxX`90*r5G)V^z_dAWIX@r67}7Cf`WQJonHF0
zC8B=TsTp0KQ<vu?hPUfH*{`+bq;Hjw5|5$kgD*xiPP9xgjNTD4Q=xcy{=KKOja=rW
z$P_!BkU6Kf?BkiaDH4{4I2RPH&ODo~(_+qTS)?tz;oQqz{}0Pv-KuBvVxxFfi|K{N
zY1($TCpC!VB=$#zWQ+W>NEV6N%xcv4sQXIMS&dhL+`U{0JZ}|_zWtCi|7Xjyn1@N3
zcRm$aUEY}Ocx~~@SgsK7SxNd1DTx#6U9NB7cGQ?}zce7QxcMM&R=`RHtBD0sTNX?3
zvaf2Lw2EtOl>M4d8E$5a;{Oy(S)gpn*}yEKFqLbC>K3U9-W8XlXDCTbJ+fhuSh(}#
zFrF<!7bLz`AFC7&nx3`j*OOHB(%FkSt_C}a?K<w-tvjVoNZQ}~qJfDK4{vwJ^p#f&
zUg;%UOxrYL>F%V~u+Gy$Yu~vg+GJ)ratCG1cIs5s`^*!@H8YD{YJY^=f89RCs5wdP
zscE;nOtvNSEV;;%G%w@04bw5ZCroE{ZcCnVNmn>7M!`|?RzR0o$10JA9=ZMBwlZ>a
zGE988>AKQ>Z;#qX+><7#JUXC|vLR@t>b~@u>my&Cw(YRxNltq_<JO;Ufh~<e2d*qh
zb9M;Z)Ty##>PnZ$voboHXY9O`s(w&1ZpS6#Sv~HIjv7%Hn>x4fYwK!SOD2X!IBrdf
zQq@}9ad%zWRFBHgO^&~ws@yp9ZR_T*+ZAFHPW071aolGRzVuUI((QjC%gjy$pPb0K
zd?A<kES25q#&a`cDwg(0oHQ&5nwH!<DOdb<;@7_R=O;bO%Zhe$s5nGLUvR2irn)og
z%Kv1Z_DJ8iK^baBDP6M36IQ!Meza!HP|9DokyUfc1J|GpNn1A>@^gGG@txkLZLPO)
zbxz%lHlrPirt3B|Yv(-dvaYywd9B7_pWR89=PZj_zG~S@h0~XUH=l^meQ^68>*T7}
z`)BdjzDT~`*AQ}b_n#H34<Eb#TkWY`%6b1b`$zL5PX5Yaj9cjy=cN!edBHrxonISo
z*L4cDrG&GD_7rciTeY@$?!kQ(ul4Q~m2jy#*mya9J7;yte<{OrCr$Tf0n>7uc?)Ma
zUn!V&KK_rAWL30oitED*Q2{&L+z)=Zeaez+jX>U0iKA{_nFk}zO=i%HaAvFCqQvEQ
zFnYzU1D%Pv2jc$Uo#uIdAG3~XF1sD?)V!4*4l-}1IbF^xU_EdBx7hD{%nKX!T^DXU
zh=r@$PH0X^Q<{Cw<MTdm$EkHH5e23vz0+c^U3jwVo0{6f^|O5yi!@Uk8RrNDE}NUm
zb?n3`i8~Y7-ESywSW>Oyf5Lttr|)#>cT2ZaYbFWCcSZzttPb(XIb#@^YU^0MQEBPD
zQs1<m<JN5+1#+Lvie0Pns!lD7op#;XfpyhM?=+tmu9r1WEs}HZKJ4?r&G~Tc`?h&&
zHu~tA2{!6(XcCRNbLhxT2~pi1ktJ;NS7&c|@S%!ByeD$aT9Jr}5<=5<E}gmm{8mHb
zB25(quBV3AHr!C2cRkjfYxk%BFY}Goeh#~lczL@(>2mwE|K=~BfB94U|3B8b;^q??
z3#~6}?)cA~_dnW{(P!(-S(@{zCUmV-;rKXV;im~jlZE#mGUYUx_}NlEe)g*6yH?5U
z6kJmzxc1U2UYnhKXXgJ^n6#<MOxyDybJRiI%I(`unMt&0AJO2y`f(eR;;Iif1uQNe
zEKb(qdopd|LE*fK{0h5;4`nlZEmS?aN=I$aD#=CLFBB-cPn@$mV{dXXkC-H@*#^bc
z2NbV(bS6$RzN^SuFR;@17E825tA+$?yn*um%yyg2wkt0%nXHkun6oso(d69j#a3CC
zyuZ~|IbGgXa5Bzlc)4rm!xQV?Xjv_IZ8Bfj%KbmjF83N2#s^z=ONjp2eZ((u&WoEY
z4pQqsYpt(RT)*~Wmw2|yd#CjwKI^|_xi%H9Z`7P2HQQCjnS0~sC9R6H|8uDZypq`R
zQ)SD?8Cw}8nY}kmj<lYdxqDmpZq8)4^f@zU|K!;8V~YAoPW3{m`3bWGF1zLAbcq@p
zig@pNqtu*}sl+O=f7*jtERwUMrp(xwaJ=W>jFa8RAG~PE*`-z#x$@lV{=*Wz1zkqR
z4CbguFR$UAbMht6u0|0ZZS}7YHl4m9+xofHw}VL~LRr^?X{Cgwkpolzrk=9``!26)
z*S)|}xrt}0gr3<0#g7m6ocJ-#MpKCMhWMQsyk9PF@#o(9J$wJR|K9t}8urgyvj2;>
zJ>v#u^9{`lWu|^{_OfO;p;FxHwWZrXoB5{4t~pE&|2Q1Nq#f2ykqdXWdbd%5`<3Iy
zIR_r!FkNnSI!@T^PvB`mooy-BW>XE!w%wWkaq}@@lhv8d3R_N1%D!y2zs7ND%_5cs
zTbzOf3KyFm=@EFUaQfp#v9i_DxlW1(j)#Q0dX96=IqjpUeZ}X>4n^ai(q<BB)e6lA
z4^88U=(#Mxy6lAUY6W+LgIrVJ$liQur*3rIV>h>@)g)!fnXWRs7G%44{_(xv<1+7X
zcc7&oqr*9$kaJIXcztsG{&>&cEU?5u>gdNs3S3FcR(PvOC?EB@E$&@py7KVqHUFD!
zK0n#DF}P!kGRLk2|6n84?>0-fI;*Do980O$9oDpAXZO-wtC!9@cx>^{-Ag4d<m8B5
zNoYN6t#(AYmD9len78{UYyWl0-FgpL<W^s()akIjc=)!8&i50Ocb}YXbdqO}gzfhR
zp3ByXmv?vX=R9$=aiK${?ZM2w(wkb3Z|HIAW?<=HXpiDoIkETCffcRHYwzv!xMdUc
zkz>XC-IM<WFwCpAU(jQ(xz*m=#OsH!u(s%_%F9O_b~X#&+`saP{p*dVZYW+}YqNEY
z$poe^nup(Ln)^D)_5|;{w3<WabaA$pjqPckCk_dp4;)Kcv(?2+K*ntKqxtVDwrwt&
zq!$^ICB0bu{~zz|N@s0sLR1xf=6{-(J}H=8r~A%~J(eG@2&~*!rrdMQYoVZ(jw+8x
zb++8GOWVb&7HZj;ol4;OIzdNIgh4-M`SnfWNs`l4g$^mCC?%S@@2#FRW5R6yB7RN{
z)&;X&yiNRe7kIr$>3QCAO_x>1N2Pa5#PPW&xn8NL1U@-eV&ZyS*!8Q``r@Zj^?}zL
zCAph-F8ST%`g4)1)aIjXoyTRkRTFuR#eO;3E7|ej!nLi=8<JyYChJ7ZI4HR-aAwQd
zj+S24-4dLq3}&_(2=QA>9k_f<Fqz9tVc`y@<IHQjR^Mo{;gQmaQd@aizb#XE>e-`Q
zs@>C{HqGe0*%Z(xKe7MY{~c%dC-g9I^d7X@B<8#6sKj2equq-{ms?v;(Ttv>Row6D
z*mF3NXU&S%D?4OVlUX@;%zod|KhxOQC|d3S!~R1VCiYAVOCt4ZV>DScn15@ZTI|BR
zxaQRJHP<dBU(9?vWqD6e;KXaYAKcD(d)xo@ndLhc7>J%{HVii2yV`_Rfc@qv)2UX6
z@9sKmvVHHS;A2-zw_FKs>EKB`%)PDW^!5_(3tv~AJG*>_!@+|WyxrcOcF(<g=yuDU
zm2Nlp_Wer`E}R|H@=!o7#V5DX=aAN+12>sgIOrTtauTarbEYKJ^<sbIq^qHCef$_L
zcAv1b*mU;sBqfi>7v5auuT9~xo5G-1a3u5p(%T|kw|C2WeL3s0+aP|I;VCnP=B&NF
zJAcgD^1&tlsKy#el{Y%BE3~h#-7vv1OMH!>ewnYpo~#(wgI26A4xFufI1g^*IVq7i
zM{UdR@b4zpTbns12fF>*az50_ZO3K*e^ZuD*s)vlL*KsBr}<y+R+-_$dexRS)JCc#
zV7}w=sGNi-9%=5a0W<F=rtIyX+k1vz^^kJ4ozUJejwYF_&b-=p&XhjlIagroaPiRY
z1yRyV_ue(=xg9C@?A0Up-IEI!O8+U~XH{svZxi*s;MC%;(fV(rk7YCmZoIgswBJmu
zA>ehi`H4A!hF<0yOppD1{PEBJ)iNp59PSv$-jU=sUm2ZZ_}|uHy~--zyRm|iKBxCy
z=JJ^syW2rMXN~$D2dRfzqKD+q^Uj^V;EIS(tXN2wW`p_l+}Qh3vg(2>K08dlz1Fnl
zq~qR8j{bL3A2WpB&5%AM@TfK<tU|l@igVKGCr{M2I7!$B9+{+Ae_O<G<~`M>*;9Xs
zSxQ9*eGO2&wk-GCh5U=%x-(o3B*yFc`kB~SJ)gnEcjdO<$#uImSR;4td~k!IF6;S@
z1v7p<W%|v#<o8*w*I(`jvm9;k3ollEP?i}}vR6F3X8m`q8ST@;+cP6&Uh8*nl#pef
zS@Si#ciV&iUbDAaNAw81<kfrGx;C=yYeMT<gVwdyyP|I-E3@vA<`Gz(x%7YGta4|=
zx$~q}X14ljL~3xoQsV56+4Ios<E%w;vu7OLIP>X4qYcN^cWy3;-kx~SEJz?KMa_0k
zMO>x!oaUN~>zl4#T0F(qLh+F&Z`C20wXD%Lp5jLDE^G<a+}X#Y$Kkc8Bzb?W*Xak*
z%ip}7`k=umBYAnqMC%7loC?hPrOhD^o6J0n8WyhG5Fs67xhv@9V@{6vz-GydjPlD^
zFEegD_4jVP%dwTtWiiYz<v2e+KD{>On2GmhpVJq9#O#nUOJ4nU8;{wVO)G`_%v_h6
z=~x~QYr7)3XD)-$W+@q=_QLKdD+HEYOPyK!&aXBvo=?zSIIVp5E0qw=C%hu!E2lhJ
zy8X%jQbqMX#oY`ZVxGM(bZ?nI3OKOw9l!57>y7=I3cEg+@_*iX>C0ZxGO>o&hmOS0
zV-k8{sds}xsYq040t27I)(rv-LKFIo8u<7Myk1J|dNL*c>yr4oWB1qWxW80;$=|<b
z{OiRT&fUN8tstmpms8krw`*21eaj*zC;az1zq7%gYhK}2<zwGfGPgNPw)?&mn6_JR
z-R{U$H!eT5X7cS4oYXaU8rNN8n-|K*XDxUboiu%2_B+njo3h6OBzG>Gd-vuAhm{44
z{6Al~A9c7@nY}n-?$rz2i%ssonj-PuTC;fHPLVCgqNTUa{oL?=O_}+z8x27mclPhJ
zx1M&*T43sa-*fVx-pBq=oO4Q#Ic8(CQNYeW!8;!^bT3ve_{PkAIxYRR)G4zIB>}?1
z<`-sMmgqPqcK(H?Pv~)}D87$JRG(bR+iE6y`Cmaw44?e9bERjFrYt{npe)XbS7n<Z
z&(pO{PM4p)70pR}DE3ZsMu^73D=JH4+3wj#ubbfUE;Xk=FIRnH0$cL)kCU%=o1e*X
z`k?kbY%h=N!b4Tr_xx^(=+rHYx1HXb%=2ANTq)TzudgS^jOUiXm#?*4LDG_u4;jQJ
zdNHwchif)(5D-1QZQX`~R<nqTR`m~46c}0gKLpNXcyns17-xaCM8UCXk7W%NzrN|=
zJsw}pnDIwfMZn&btIF`Ns{YLQ43Yowu5IhA+S^<kI$u;RHS9TdbpHMIz30O3Fl5&9
zZkXt4&?K9=Onv4rmWR!?5%T$&E7?EIjC|R@V5+bh&ti>$&mF5(-2+w`2&=GUC5dLS
z-r&l+tD?O;W9hU#Z4Fsf*NfHmcHgK_*tL;I*wmwG&t#S8sAGcJC$oAgtkqWs@SbL|
z&wg}oN>|T2-YJ|MZ_?ksHk!cK;8<|%oT$+T=Ckg-FIS}Nacni3(9recPnbZrM}N)h
zM@&b}j`%4zb1vvztlX&UA?nV;_fz=JWbckmbr<cw9bsjBTh+<?==nk+^As+pClTxK
zewee>A~N^I`O@ot^W)VI240iD#dlY5%IQ1n1pob4T(DK>;)Al-r!V>Vgf$Mfa6WnG
z^?bfz_Vjm~b_D#G{{N$*likjG<&rL;E?e_*qXq$m0tLsW6fPwnkAf)-Z5*DQ4IF_E
zvm4o2H>?U>Kb4`6*;CbHk;4IxR_4hwd{$msBGi-`v$N6i0^dx|m^)i;8MrpAH4^sN
z^2lImuda2B+LSEgR%dN<(~~=HZgOmpW(<Bi<>7`_=IPV@Sa)o?ylRo`jI`6owm4s1
z<1i=HV`8KBbRKo1+1u`%?Jj?RS2Ow#(>$gh`VSAbaVwjx`TYKorqxb{XPZj9o}7x_
zUU0I<()jtgdG^iE4r!L8?Mm5kLHDd>X}Di%!0x_Sw{35(ZN7i+|FXHi%U0f9SIw+1
zJVzw&(OyM4@c=K&&rc6Fn)knp+4Z3#U)4Wgk^;l`37p$DBwqEhS^eqh@x|izvuq_b
zxm*s_>&Kt0VXZTttfs3_|Kj4DU4M?VHf-jcaKM30OyqzUtLjEp&9yc=7PM$@bh_Xo
zsCeX<MoNyx1vPEQh=pxTYty^kWoCM~F$t})2=Ej>q%na->#W9N2D6(&)7+V+DjxJM
z%LzRuo9grQ3Y&SVX{)rT=Y*MIhEo*<eAr*^NQq3nt9W4Qx(3y2)55)8E@g<^HAzH$
z$%LR&YD?zq?2JuG*v2}i-qCG_Nc5eV>hsDjNzR<#dDZjg{3fR`jfL$}vosd9{7-U=
zOzWFB?Pm6J-&>;TQ}<0<xpY?1u9eH@HvL+;VkVcvs+F-*XKD2tMqSZTXOj7nwPw{J
zt%f!0uKju?7JWgyfr&vy!Bv{S>-&O@0!r0ys)9GHXjso_v6^Aad9&3Fb64bwX^Jzg
z-1gS-_C7Vu^w=0L)}8C;T4(ieh^?5wGJAs~m-N<6vo5)^p6$M{m)ojkp@R^oNWSBv
zt-;>z?CvU!9x4-C6n4K6`m)GL$X+MFgLC1PAWxMR)yWPKH$^5hSnphPh0T}qfa9qZ
zw>HK|=IShe8fmmpwKyuRGv~8IX5iK0P?nvpb7C@%rHaSJyqNlB-eeK8gSCF|ceOI?
z)>FG$+xtJX_v_VY?{71%#s-_soIE?z)GTvxYTLGq1q;tgWpAF^W_D+7pw{;6*n_U-
z_h#O+{hqt@+c)X_W@+;s4^AY`nDZ!KGyHB*M|{VQ;vRjz>%Cu8CpHMBY!SLw@v+Wt
z-t*+O?p=r8x3=u^n4#@3Ur5Menpt{Qxtn$RovO6mZ<ub?=BaZ!)-3RRKUYM26L*-$
zg+58XDes#^U%r0U&f#l+L1WE?po^`l+mbdMR7sPwIn3nw=Ftb%y%BalwQ~!fo=@0m
z$jF|1!NpB7X)4z;>7{?(c=eV4Sf6akbeh5Xo1u9HXO+myj;LKN0(yJ<UAB36&APBf
zjeC+)Re@8~l|PG?TUY)s@UODs;ouft;V`oxuq$dIS9HeA^K;bNrq5Db>Xv#q;Ngch
zt22pHe3q^XS7hm66;yJ}D_t0^xG_w?@`O;XsenRJqaEjiLmOAkSfee~&GYf2sQ6-5
z-HB^FI7<%ktl2bm=HEFV%bwg4+j+}EkMW>kti>Af-c1?}m2X!oR9*>R&9cdT@}G-U
zyq+goXI*z^%xF=5{j=kcpZoD92@j@QniGWPY-7~)S)nTRm?64q*)+2)i_#1QC8N`p
z?eBYfrjes_ftS>Zr88AjGF)eK$4pr!zU;0mSMY(<B#z6|kC-f*-pAN;O(FQq`q*c!
zg^&1+XWx{}-MdzA!@}<TGH3I<FOHf0zp-Xn&b-g&cQo{@P6+Z@E;N=s>AAe{Sqk5Q
zZ;z+HN$nSXXxRVaMDx6`E$*eCI?um~x^A`Ha`McKlx6h{CJSzK_%m=#5@^a&(Mjf1
zsg$UE;_S0&Ps@fRXIUYKudBi*Bo?Z~%V)KA<`iyt@7d^Tx@d)i)RXCD4?UTJ9x!=D
zn6et|EK*o_MbPiC(6YIFE#7j1!P829d6+X8tl(74eCd_AhQ}$i`BL{S_U|HxR<#+d
z4%_;Q^Y4*E>sM_EoXQ!!&?H#OxQyG>SR|Y``@~_{Ltoe0x?EjZ<*{00V@cTeUEx!D
zJl5|&#}zjFfKuYx9}9Hn<V9VqnkKHQd3jTwfN92`|I->)Oqi+n#+@N{!<w|?sW&a6
zy!z%Fc{+&(1vgq`M0~XJP*DGJ{Mx6B{Y)>#o!t!E_(dlvF#XQfewDbv)$L2k-;x<#
za@PVheHb<}O=u{ZY0a3?yIz$o>tv?45#xe=K`Z4xzdT;{?C#VCp$wf^&(OA~Atr*n
zzT55|$^E%W-P7Zt-~KsAMBcgFpZljMyrJ=y$dA|YNqN17G0xADcLbO*Pkw8>D|&X*
zO)+!No4kjF^3JxT2b*(z<~{WMlfR@@p)~()wa_ObGM(Md=L-Gag-*@BV-dLKoOru&
zntUhsnI3ijq{co`(a7@~woS17p`yU?D`oEMN!GiS-|Lkxkj!}dU$5J&xooL0^RKxr
zUWe9QR!ejfy5O{lL+Rp_kQa-@dtbPSu84MaJFlnpGW^>M(}-nUPpV}nIx$6UnKW_A
zdspKuSxJNAtlxb$CcaFb@oSoP<Lt)NmGZ$U<-&6(ME7JWM1JDreD~chtjN==OU+}M
z?Al=AorfYMU4_4S7;>%1Tevq$Kyluq(^+ditz4xnq85?;_Rxkfx3!tA%c2ZLug9Mj
zy)j$JaO2yo>xybE9sXyoZeo2DuBJ8d=DDz>#MQjhb+os2$NpKwBEoRN;^YP64PMGS
zw;yFaw@$EcdaE-3@n`|fo<yBbvrfdxSL#2PbWyEdqpSE=Re<&0>h1egC-&@%_@B`+
zRZQ_r?vr)XcDHgKaaWur(r`>ApljCJO;gvGq=p`0+U(Q&RKuGy_56`;fj;piT5EDn
zyI&9H;#erYSElHFtjR{!AIHM?RnOd7_g`to{y*X`ngV$mSaZH4dM(R2X_D<Qd)<?}
z2iNLd>iLzc5i>h0YxAo+cdR<fA15A|Dm>>Z;|xn#@o&$U6kBDbKaYOYq*!rG+h$&{
zzw{mEC-Ten?AA5kW&7~>%O8WC2lrUN$YBaF^pLxgq#QUy#I4ZAXm_>a&Z!zLYtpk!
zt3xK7HNMs%$CB<MH0NAd+4jayKj*ex4vEr^o!=X=zQOFnmqsQ{EvHi-jE&~Bv6u@o
zaONCpz4yO-DeouYN&dyFUixXje`4LFaEK-7(xuiHD<$T<T4(p-)x7(eS*laMu4Xh{
zvF?`DpQNe5+dc&U+tPMAC{MI(!)^OpyQXo4uAcQjHasZ*=BaOCn_`u|e0k;4X)$%{
z!sBWeZYD`8B(LpL;dEQ)>znCwEq&(f28&ZWW5ssWitTE-#=Et<B_YzugYo&1xldi@
zwHy#jh&<7w+VYtt=AeXV*2}qDB6>b3xHw(Cd8jbj>uShr6_(P#J3l-aBopqq@Ac7S
zXbm=Sb(iTZDma?$!w@2K)b#~R#ETZ`8%*bW9(aiytvV6?W$LY!y-n3O=cql2sd;&H
zu1Ax8<W-i2fD2!**NMjdmzI-tP?Qy~INQkCI=O%?yrxY~YQ_V@8}e=vtq0v&Sr=XV
z>f7++Z2b<m>$P_~ER*u5cwJ{br!3|7c*TsHTP3<PA3gFp_atRe%#T2urxDU3+(#Dr
z2~@PN|1-m0^d^hi%_d7(DchSmUR_iEvaQw;X_n|{wwdP@aF0pw&VJu1P7h{#y*V+v
zWQwQ3k>Hmi+A})7^R(!8$aTB){M3-?)tL8atw;BoI4_L@-XV#Lb{<;tx24gOK|{9n
z)Jgx~8~pD)83KMB>GyEeywMag<)Cg!k8DEJsiTv$O)s31<q5Wx7fU?9Vy;y6)j1rQ
zjq^-oR>y{ZKYJ^&;_BK~`E|ONjSOS|r|-D$wkh6SN_wLwN1W)zO+V)g%v9Oz810qR
z@jq0+Lo#gs+`es+3nr}f5L%JAFeYW)UB1kMX;PdX={qOJ#6H};;+pK%Q}UvZ9YvJ?
z-JBg5$Fl$IHTNI01C5p*C_VD&#|&jge%F)k-leGvMO}{GWhvh~DafEz+U8-M&w;-?
zn!*m85t`(9e`lkA&(jkPuACNZ(Q#2HeNV(~x)c9t>E}#E@lDQ-hmKa|I!dTLbMiVU
zS;tXhnYL!7;)Rz&GEvXY^G<Gf*>deqYrq>WKdsC6JJaeFZsy2baqK*HE%$O(*kzTn
z$q&B-bR2wURXN#w;pN8x$8SAc=CVW9W};HRZ{w>9_WxR+gtfI8ihkbB_$NC>QY2S{
zrDWqhr@)>bmBx=Q98NVeclXY0nKLV)@QOpqz5NY+dn}GrOu6EzD3sJI7vaZIU6t&#
zMnwEb+#*3v@h2YAKRUF(bc8=qty^`bx1iyz=|hW0^CoOLnEjzg`(m79%B?28t3Ete
z`&=}bIa4;e$cxJ@iS21T8yc$T_@w=u6h~?3{0SPaK~H{64eiQv`?XPnxAIB%w)j7_
zDt|judT-4ap8DYLLY2NxFIGFW^zTyHRL~?M5w}ft!7ImZAsvM-zl6y}ZIjcErRQFg
z%oN*x<i!t_J53%9vJVAA;}-6=O?7;7&E<^u^ncQxTMvgu9ntcd{-0ri%e1iMBPo5|
z5+^bRUA!Zbc@|yMsGXAh!QZ#Q@w7#66rXE!9wX-jM$R>?kp<$>BCL^hnms8CwR040
zMb$JD_}e9XC9)dN`u3`29F*+Js-3t>Pf@%mQcZf*Dru?U8&?)CzN6SQ(^uXucI_%(
zg-vYDj2wzWNo_aX!lxX{jGCJ{txHMjwu;j02Mu>s#r!sJy3v`d^wc9;&F}br4^@pI
z7tIe|_j1`^tw^d3nUp-^{+cp2`Dv@=FC}fvlGyNx&Gl-CvZv#w88;o?+;9o?a7|I)
z$Cv!sah|KG{DHZ%uGDyZy>LyTP}9XC`Fq2>&$&*#eQ|T#vU=v6Venkx|Kj0)>q#Q-
z0$bjGYzav8WT-vqSINnwIDh$4my>Q2J_fd)dUfLS)G4-|QD-(WRc9V*yC>s*Q|J53
zH(^sJxGW2BWzt>zLM5Xh<>H|ie4^1E7w@;sd9uYaxBp#+(8?$O0&f>F+?sewMZz;^
ze$Ul#-OF_<>FKH0wiR@HE4<~%YwSBan~N!Et;SPDL;m1pE;E%__J2LiY3BcF&4Nx2
zhgs`hhQE5r#+7%3k7eGs1r8dVTDf<QO=Fyx!+8GAnQ3><XcTt(teB&|lEr~3#cM@Q
z*~}@r4eNBTG`U@IvhHEG-FQfAgHvMMs+GNms(KldnpUY8q@A0(N^07wgb(6rW^JmG
z&(hMC{a?UwR9265joR~!ds>s`o}Kvf_!|Ma?6k?5$Jo+5^{>}P@T+Ry5It8gXYS3H
zX^)o7wTNB4pCf#GTHnKsn%Ybubu#%57ay%=n3#2rtt4;3k0&>aHl7w!>2^s?-|u#>
z?40(&*n0<>)D4}keEs9CJK@Tqvi_GjkLNwreCV{sY0c6Z0xp3+ul!-SFttPHWTod*
zy(o_K>C0TDLUl5ly=E<$nq$IsKv(BOrQLn?gO^29S*))(@rpnB`l5iX_t2~#1#Ayb
zI_WLpxGLT;O@piUz6+D?WIN4YYo{*VFm**o_r;)(8-7mcz4W5@*fh>(^Evl*c}!E;
z{8vNrOZc|Z)jn!3nI=vB&)q3{Iq;gpgxs{LZ)HAr?7ERz==*_vn%{vxQ(0b5R6KjS
zENfoKBVGCX+!{BXW-(otUHfR4&GHW(6K8KbEqnQ8<CPPyOD~-~c*Jk%B>%7d@m?FE
zW=zp&)(h6$z%)6j^2l79*#621>+-KU>wjss|Iw`T+u8n$lP!mX?V3F2H3v1;9I{%|
z^y=1z{Edg(+E)3rzLGc<Qu{D()vkr_b}hWX8L_&lce2F$HM3r8Yvq69Q!?j%w9e_S
zBA-Y8pDd+c=h_;*|NL`T30kibmh`}7_2y}^TfJU?cj8m@S55Jlw7shRpy~pVFLPxl
z`8w*qe|O`Q<X$fs?dVNX>~hXQPQC{>N<9AW@MqEVo7s*9E&Jw49yF9%%-Z9|TXffr
zrR7PW((Ea7ZZ<Km4sjBhRo!=9bdsZJnI3bmXM}=4LRf3dr=<a`%YsCY{QuyxByi1X
zv(}9(uJNS1xJ?ML{m~SBz1iO7tIqjPA>Lm@qf0}wOYJutl32qO_P!`$I(s``;k+Z^
zE3KY{{an+@rkgQIw0_Bap3sjmo8vcL?D8_v-DJD=$}j17d+94yx95C0HL+;6#Kx@`
zW?gihF<ZvCOh%n$MbCl_C4GBlu-x7!wy$Dk&dZEDR!dt%f+|8%tyMP7`j#tgvgwG#
zI{D>Cj!f-(H0#c}9cK1sQt`i(bz0x$&i+(&;&R^Uxl1oG{kPq-N#{3H=>4L?;8J^q
z@*?H!I{M!W&lfpeX;NFm6w2*l&*7lY;b1Retl98N;@KRZxuvOmvC>w)m6N|$eir}m
z>c|JK_dC*-6kW@jTxe`&tAB0fvrNDDU&OR??v<|O@csVp`?s?9--RklI9hrb&-6RB
z*-OSu`X_v8>&8pGZ$Ej-##&75-#2&vR*5ZQcUpd&D%v`wH13M)>TL?$FWYu-I<`D_
z4~Z+6YhkiiKDe=og{8sk;L5DkEaDFqN2!E5e%aylz~5`nhY9>k_VAvbD9qhs{xvwc
zv@m_^War9I_nr13<&H0kye=GmZB^+Mvh`q&qsWx4t7m3?&kQ{;_c8H$)PL#aFIX0d
zitM(GJ^b_c$@v{;cY18o4Su!azoDwcXN`G>*E(_>dz};gx<FUDIc?hlK^bi;S?jX2
zTJF758vnZ7n<nQ!GwNJWTG`$Nt7Y{^Haxh0_uFj6jn`xsYG!h?%+>9A;1~YTY0a-Q
zD^*TEVB2Tqnd0M_Gl}sihh{hHi+8_Xl=Z)S_08t4gzZ(Pg2P36mp>KU-=zE4DAf4(
zW^3!xXmh=A^D<?Yvf!N#DZ3eRs<-twNpKt};BMI}?xH8nWAr$trt$mApk)_kZm;B?
z>v!T|vO~q1Tf4O%CGMN7wySLO<d3Tq)E@1hotB*=@%32#)Uq32^|UQB-`hC9E}U}N
zCO_%FmFy1(g&i~8yy|zJYp#!$mD&<nX`FqdtM$3>X5ZAewqJO5nK&)kC9=TtDP!{9
zJF8v!&op&Np3OeNP_LfFE3`lLqK1Y4%Vk@p=Q{e9DZf5ivU>Y+-=`kF?56$`rUZOk
zo3~rVL)Wy&*>rNY>6HAvQ_BB_+LsrGTRO)4beK|mv;S{XjHPx8Q_=lRHxK{4nI0nW
zJSMU^LvXg`tk-`R79abV<;~X_Hs|op*z+0n))(X4*KYZ0>FKm~-s4hcrRm!>4HrF8
zk(_&Mk@>e9ejbajbE^hhr568^|6I6l;)AA!ZMVA;Lfp=sJbgKSqvOii>l~J7u6i8#
zB+w{4iZ!@%)B5>E|G)RR+O90B+>lpryr@XKx=4C^k+%IO^Xj7X@1OmDe@-s{qTN!O
zzk8edk+13Y))|bpcZ|NK|97}@$gq6-uIlYl)4s3VS1o<Yx?$yagN@~vbdO#-`2Cj2
z`Zss%6^zzf+_tx3W?62;#K|Xf!qq|CkBgT};zYoL!%eKJ(=<cdMBIACjf=MY_?YA|
zL5y{ZR^_D>uPICsvvN)rsrt;Av%}|Rg0OnKu2pH#%}0~H0~R`2PJHOJOhcV<35!PR
zuacEMEA^Mp3OQiJ;LXXIz_!Ec>uLthps0yj)}d<JGqRR%bNVfo7sHr$vnR<bZF0k&
zBGw)&YcE~N&1-YEt~xqt>TO+y|J%F-1g7Pu%;;MlRHqZ_$gpmvut(wLQ=!`COKwdt
ztKKELI;t)E=c?zow(hU{_v+-b+}+>a+}U1wTTCIksNiA2--9~mK2_|Gnwjx5?Iyok
z3FFr#LE-`3S0pr}7I{qE^t8I-L+}!xKqt9B(=<)5UN5m<`nT%m`<LhE%a_>JE&O@%
z&yyfG4Tff+5Br%IZ7v*O5sV2q$R-(cVc~xzox=;cA{Tvdnqjczg45CzC1y2&NTxtd
zK~@97qXMBig-3;F7fp0!Y}mxVV*a)l{E7m-PTYz!1zrWZ?XZ#YYfwornxwie!fDb{
z-6&1ntt)RdWO@lKR|#5X);4<*@3NRT+-FW~x$=La>Y-aRPwUPw^HkvEoD$&Rxn;@(
zPe!%KS(820rusCTSNW0h`Mk=`Uzh!rJrh|2%OVn&Ppguex_0`_UmsmB)v5G~1bP4N
zyb|PRzBOc3T!K)j6{DB!xsWXZx}x4I16)hBZXJ<ey&O<)?57pa@}gBP%<ZC6*Tx3f
z?me7K46fbKTsDJYU+V1CFx!l|YhS9(Uv>9U6yK-Uo!opapUO1!GPts03OlW)bGJ4H
zO~1o<JoWp6|6IE~7WnZ<eP8(B>ZMa0*XbMEmD=BkcHHadxOYOKUqnt*b&^Gt=8_qH
zi?SYd?R8mwC-1Q7_Z5W`h0L{<PiT7is5^Dn4b_!Ca__X&%r*aO`|8{jN_aLix$md$
z)^#Edr)ROQ=$;bz;T8X~z+YTaXD&y~nzS<Nmgbuc{LB*;`0iAg&9iK^(e0?s>x4g?
zT76t-hm@XX)<*Sh_o`O+zP)|x-P>y~g0}Lh?TTEhXYuOAw9NM*s*9xywzwG1GTZVi
z&GY@sUxg(i9~yl9_k2<b@Bftccr|~ql<E09w$kQiS$7#^W&S!|^3|99%Jui@1bv?N
znp!o9<aExmTd{kzwS@BT$Clq+Zq=pwpqWQbO`%cc(~n2dwccDt^IEcg3SN99v{vM&
zkdM}4?>?=yTKZ<2eP&);BemA{zs#2!J5%rZ{wv?`c)vbADd53b6(s?O?y^50|FZ}C
z{{4Tyer3}+hNo&XgJd@}6iz;MH>q{?%ML9)9+f_Zu!WsbC)yhKu4rFj62V<{p-EOj
z<RQE0QqF|R<JL#5u5Fy7s+)F~rOl!<k!P#x;jeFw*gM?3!B@HCaM9m@m^Y6kj=u{%
z(rJ^T=pM32aHhtj)>M`*p2<l^LXw1*g=|bR^1SK2Va22!HmNB~BqD<rF-?8DF(oyF
zr%y6fX!&Ul1#Qnur=%xuV7=m^Z)6eeU63TUWufOW{Us~9&#?v1*t(;YhoM0@LT>R)
zx4K^A-W1P2_cn>HRXJc7z0)gsLzoJq%frB}KhNgOYjzKwSERKy;;7bVHQ@-!cG08+
z_lGYY{y!MiGC_1p1gmaHuy@smtisC^Im=FLkWHR++~&lezNb%w<<s-{X3g2Xu=<Pe
z6q&9(hnH4&%>s}6GW&AOS$+C~h2<8-;;8rDFJ08F&aPZ2_EoQ3Ym(f@I(7fjTgI;9
z(m^SoT()h`Sjf9pfJb{zi0aA=&8jLB!6YT;Wd;YArWgtfCcb(oHfLg!Z(pPEUzP)+
z_6L{L=AXJXcT3p*A6GLc<<~r(y+bL;T(eVj3RB<xj9*^Ni(H$`YegSPS%}83brQ+m
zz{eu~LTi>1yV}+ZM<iGrx|jXh6nM#U&QhbPD-*h$xMw*eZU{S}le+UmWMc2MnpLgB
z^E0$HuD-gqZQs>v1^*d0#(pZg9&?%Xdc|GQ_{yc(`~I;`)Z^$;6xF<Oh_7rzpTe6B
zyumr1GS3X&h3ceEbUiI8?;Ux163^zWD>^R5zi)(Pns7EX-qgI?6tHr~(X@h#lU$!4
zGM@GKjGoi&(;Yp>gcfJ6*V%U~Qkx@LV&w$J$z`jSM0Cvd^yz!Mqu8app!qP@-3PtP
z^DXZLz7C7s7g(;fbdxk|xFi3{D?)|cLJYRQ9Qy-1npTx_$CR{MidLQJGG{80&d*U^
z`RiAk>Q;v<YTGW%{WK?i=G5x6T0L=J=Hzq5*X~4^OTIbFW*_0SdUAowGNr`ZzOAm3
ze$QK%3Mt=g<S4&%_-@Dhmx%(O_5ROOa{GA7uO`V<uf%h{>DhCO+#k;meLqjWsoOcV
zD0-?d>%#lRw-PTs5$ieUzqDt~<w?G?QbLk+nHC>VSix~4m~~(3@}rhL_EoO^bCteM
zJ$lio;<PoJZ^`y4YXn+@GZnjLeoYcdy1^^geVIG3CoyR1rO#7LTZ3PfZ77^2!M*X6
z@?kNlUuya;UqrGz!rY9l+ONNPxa4rE%Muxt&gV)ec4bV;taN<h;1#H}qW{Ij9*HpJ
zK%1PT<tJ|FIq_Uue@yA<7LLif(+Ybh+osOh@kk;rS=BFoB9rEnw&OExXKt>jd=Zi*
zz0Zt+O>K{<igSJ2oxV$)H!m6oWz7BjBx=p$ysiHuK9`yl=6s(PKD}3J{rW3+Vhg*I
zL@!(RRk~m75A8X$duO7nTGq!qJ32P)y>NH?$`{sNHc`vvJB*f<YluHc+qitD;Phul
zGQUjDeAdn#!Vzn>nKQB9`bWt_o#oaiEB&g2JT#?^CZ77}`Ke^noFgBm%xm+H>ALiR
zZ=vo!OUb;v&)RNBpE(;z7C$@`V?8a^YgzB5-sx^jO>W&-e))ERbFQ}d<fWcH9hoB8
zo?+_dQ@`9Qb~+t?<lR0qKh~29q7FBg9-6B0y+34G&*t)ASLv{d?|a>v`xaemd{)(S
z@a$`bKZy;uWK35tJ$r8LRE8UdY-L}T25(Mm7fai@^2CiVrzU^j^#5t3)0H=BA9UBO
zYG1!ifiuZ6d(BGszpoB@t(9I>y4LaGwKsRIM0N-nuK#mLJNDsI!@YG;>kis?91sfo
zb13V{5C1<WM5T8}Xjz3_+qmuP7xQYPSN3M9)7S3Pl$8`$t=(eir&YEsR$Hd{v`U)8
z<h7e3CrwXt{~cf+*1csXU(ou~g1XZVmU(T?U2eF6cUOkj?hLI$4G-arhs|?N2Q64D
zf9_3KSfXr3qDWYytW<&Mie%9v3in(bnN>gB+mXYwgeUJ;lE|stDJjijd>d}JWbV4A
z{a~FkcZOrogF~#bAMStY7TKa;J7Z;$!Z~(+9u7<OgBrX~?9Op`x|dE0nDZlPe$xNr
z<#8<YOwN=oDmc1o!_h~Y&o*4((tbT}(}W`*G`Qw0H`&GVYQ?(Hgr4ImO;wjZ{I%j%
zYsy&^bujSO!$m9F>MUOQUGw8V>942LoTYT2=^J~<nl+zaEey%)6jhp_dd-pNfiHv8
zhQPGcLZ^Z~Qi6<)U84W|-XHOu;d5b{S*nOkfAAzN5p({KR|_o-JVL`ZHfA_;haEJ3
z#JAE;UoY%w*sC?+KfHy?3@#QpU92m-nCBFI;nl?y{YxDJlDX$&CUIQu$&fTX+wmvs
zGS{N`SE)Y*o^>ob(J684>aLAl3;cgA@|RJ3{VPjXFRzz>xBuHaFK#V!;$rgbKcj!^
ziPBb|PM((k%Xn{pI2O^Z8Mb;mpYwhlFU}uFCb@O}Z*%_7BV_z2>Grk-pW;^TxsWR=
zAb3x1{mKWqQ$r?*89ikB%P9OivEW2@fsd=OT0zSnMU|a7B1?K?!zS&HXfzYb|D3{P
zRlpqP7O;!w_%4<OD)q^`)@55LO|n|BINZxfL^kh$oS!Cl&m*;SkG?r;n5=WK$kXb+
zpw)jNUMc5^m*4SaO><-NoOcD4MVvdb;DXNkr00tk@#N*{Uc97h&+{UWPpgLS`U@f1
z`d<Ad3|=fRw_a<iia508LVo`emsbp*x1TAnxa9v@Yt`12MTRb3{5LQD?s}(So0*f6
zbo-Bhnvw%of>6++=6ezUAKug2boSr*bN}+K7}g7g9}Eh*u+wY9{RS<wyKkE3Ih*}m
zD8_I&sH!!sZsmEijwxapr(#x~?|L=WC3X3P&w{28&6SRZW*Ie_c&KG0$}VbCy_bFG
z)9Z6Jp4QtEgr~TEJ+MmrxL5RyqhHrN{kknRT17YcLTX2z|9Ugs9~R5Lf6G#Ad2h$-
zX3u^){)h2TgQkr$&i|ZsUi#p~u0tPxsdOiaOp(bv%PW7y(UhBkCBaF1ij(jHC&MpJ
zk5_oU>)?N?B=@_>Ai7&G>2?1XCfzOR`cv0!m)g8_iNfL!`Pb!=xfOyf-YhZxm>uGE
zY4<h8dtHm0j%BO<&fz(6n(=Ke&$WZ{kN&S=3O{+e%2nhS6XVXD(>#Ud_t^Ho(cYc5
zYF81LcbFr$+4_Q%N7DuJM2#l0L<$?GEzOHqsB)3p>ea(NE4oC>Hn3M3aD2X^S(>5I
zbI9M7U3+~{fNoEKDpPXWb61NsoR&Un5iI30qI!D^xbJ`BQr#M)K2>#s1A~AAgNZ1I
zRcWweXz=n!sS%01r_ay3w}|IxP-yVfmp>NWywp+sK)~?Hh0F@=v=wgQxm(_T3wV3u
zoT24H-ZzZL9xz25Jyf^ofHAwrDu(sqe-v4LS*?ydm@fBm`X|3;hWx1t8|5#Z4J>;a
z)bM#<T0m6xr{G_2_We0?YTp~N^=BGqby;d1wRmxMWzhd%5e|Q069ENwb@L-@cOOd-
zohN!O<>j<9Iou@yku@JLR85Q&(QTWND)E$gZGne|imv42S8Ll=#9H-UZqoS9(G-6r
z^~#Ek9b9Q~J@2K&k9EfIstA<G#E1P-GU>LP;-NN0HD{XBbnh)^k}91{bZ2SneVNE>
z_E7GozTPu^t*D2o|2|rIh`3G8ezU=a@yjHUyo26CljhravM3s<zFcei`0BYwYxVgS
z%ge2wrZdTGh1Zmt$NR2{J=o>z&iqb2I#Vl3K<&jt(R}4yZ_WtidQbba@Z*EtuosLg
zp7hEwOlx8gW9^-E_XoG8YM!Rzc1`{@vsQ4rubSi_;qskrYO&%Ihe`hf+-`Nwd7z|o
zaZ!o(n<rO3xt=fzik|-b-Xs%+t0sOfdY3;vPYG8uDRr92z*uIU9{DJEx(kzsvf+<S
z-ltl0dAh2cO46Agr~549>Ha#8$IIo5p`OTQABoGEi*lI_IE|h^F=~H&z3-^#tpiJr
z@YOw0Vp{az|F`3xH8RdlNoJ~VE->#8mtt+QTe|AY^!mBNF~&<9?p>c^aDM8&tqtiP
z&wT1(IKR3e@Z#AihXvpJGOfHCbY}|F)uyPOIblk!A$LmH4{F&QJr!BCQSHp-HDyk=
z)f{$ri(5Zlwkv)A{R3CqTjA8d#rD|}($Q~!vQ25{F1aQp(vkXAw<Y4bS@))C?K;Q)
z_ow}I(ye0Cy^@r+>`1sw*0#De-U6Dps!of|5>rc<QggzeD`%mI6tljju;&s7;a9JB
zuPQ0rBx3P~yDzsWcM`MrGc)!7X_Zf=Sf1RPA*27XCgc={_+F0f`g{du=eP~deL3@D
zq1?F~^IMG~a^VMNWEN@fe6*so{hvv(_4LW|KAe&6Mp^1cVqbL(r*f?q2+|Loy0u{H
zeu0XoPQl4Lolh?^VJhx?+T^pOJY7sHtnlrEH!DSNJq+8K$Ng-%Z-eN}uOA|6ON|^B
zpX>?MnsUHUNrCg&0l8PNLza{-Td^u;{^9yb&5?>h4G)}}_LPW)3tIhbioGuB^PI=(
zZU~$D4i(-1`pvHwHkoYLleYA}($f!a??0Lx6_@fC)?$~C4ol?SB`$nxO_yM6_L3xh
zvE;@Rm!}1GSn0;U_?$ekqth!_CE)FqAhC@ORnqRaQX{`PsGjal3roxYc~Z&Y&$*_h
zu3yrlm+Ezu>4!~CtGMPhO)P7jvLVX>v26_so!uUPUV2olUd&}=v5mL>(X{+K6&lCC
z%qgF=^sR+aR)PBR)zO<D1iovNt-BIvQPBUe(rm#N%LRv57A)@NEzGT1Sa>AaR_;6d
z%pm{8FE!@}9W)4^%`|m=cfb>ua&1S+(!@vWRg_(W*11M}-=M8B_35S_nc2N*<z@aq
zb!Vi6*lPNpSRlLLNJ#Jh9lFcb@oe1Ty+r)jse&{g@2X&tzc&uKJP`;#uj0}-^UbY|
zM%JEMe=hhhNnp0hsyiTVbi~g%QHg8I0<$0PR+nzZW(w-YJP!PDVve$N!$y}m6Pfen
zFHODGW18wbb=T^GqUOj{ui$Uu@0KW=d!LVRs<3gN@m20@i?2m=(2nRo&CxX+v9C6-
ztKeUE<LJKf^-}ha>~nS`7#&h%*z<$WTa#l?BG>75@jpxanNqE^e_M5TtY$jYEzxOf
zvh48ke|c$gk*5z&XOf?~wyW`tiI2*Hs#{j94l7o>tvvaD$A)d2mNHG5ldO^Ys4i6b
z%azbO7eXxN&DLKdrO}?ew@OiXj`u3T|Iepgk?7;}vpN=<zu37~&^`05h`;(zzn8Zr
zwF*owU$K7opAwcsjt<jH7Pn71z2^v@=wY9kQ?K6%3hvRb2@mc~ar!!GyN_s<Pj>pP
z2Y)i$&IfWX-&@qtEwOd6?~L;-;qFVePnA^r(6a5bz*`&5><K9+=lJ+t%bPV>p?*=!
z8Mk~U_swy<72$Rqdx|y-e*ZJ)=$hC-+0zd1V*_2o_k>&(Js~!cYad%-_!BqLpFXo+
z>-BBq4u6xkAd+Jt|JeoIe-~V>Zj;MRO5|T;`qp-SVe)+2MR!%NnC<%Ea%eHP&DC{*
z($>F}-=r>Cz9O}J$CBV*?UEw051Sl+D&75=$=xNs?CF2Q8Wm-i+A_V0vU@9^xTLJk
z2wA3gs^vzGwqez*v^$5y3|<}0-+6TNHIGFLME~T@5W9KL<)D(P5aVa<iK(BP#Uw3q
z0`xZgT6W`f-fOefo7-3Jn6mfBjr#%fuKxFBP_R6ZR5$5)RMBgxg9eh14bIt{-PB-6
zDc&pd;$WwCyWNKN#T)pGOPZZGdjBjfZ`U+=^D<TaRVtra!8hG~{`wWKE7E`IUebS^
z<I`n*yYl;;1xuE{+3Z)<wRL)L#mlJMfjrFH1Z#En<oPKaUa@%Ew!`5@(tBSh9Weg8
z{j<cyvnviysD60w+Qa4RL>c${@*ME>%XE#i&fobnApDWiuGYmCcedQO`!Bql<HwW|
z)dltzYeb@Ku1ZT(Z@hSpYv1$T-(FPTikDw}v338)R}LI83U;@?<^Qn%FMQ|!)7!`A
zI-I<#!e<k>lxb<B#9iSnhr~l}pJ2FWTqE~+?qPdt@0`~ztNk(zUUzNoTX3U6lq<PA
zc#g;kiDL`r>rWA7IC-T>dxETqK!LNsTHg)w0{?w1H<|qi?RB@i-SFpeGqb{t%*5yG
z;~W_|gw@_G->~p-y8!DIZ;Odn#5Bc?{cdjBsC>LXct?qdr_qv=lQjHS`E*X=7T3~C
zKJ~;&ElpA{cE_DFJA;>>pY2f3#p_kFB5(oIDV5k&BALO<0_MBTc{L%}X?2Kvl4!zI
zp78&Sah{E$3TAI^Y>Gcy_13FQCwe0XgNBQOf?g~m!;4G46I_^9hSf)I6g<!t@^DSA
z65qU6rmG)s>=o_mogtC<B<yX@R^bb)LJye*3297R;NtM|GN-VO-;5_4n=bMBdkYv?
zAJT~b_s1~A!~e{RxW5^lQCIt?s)aFe{d#xY@64{cy{?Sn3SCnV-sRe;{pI@vYp3@o
zSq+Zx?W_6z`I%(Xf4O~OYGHp%Hs8ND;ZNnaJH_qi_urpx8!hH(cu<5{%;P~Ln_9<%
zCgwbY)@B(W!G#{ec?VltjZ;@7w#_+m<zbsb(Z@u|ITJD#`N>XOv8YpfTLhb?+98d_
zolEybxb<3Gs(Adr&+^#~&VGk)6^j{`3Y=U#(N)d!$s{kclTRkQT8XGm39-t2n#JX+
z;Htfb;hNYqK}B}8^pMK72{So28gfZy=Q&-~NMgJup}Ay3lDGC&jtS~c3~g$TOq+W<
zmog=9yEcm<S;8c+c2Q4~fbHU*NBq{k--7vdkH=Qsv0B69-f#b9gOQi+$HbL$7YOLS
znz1U<H>6);QO9<<RjZPYuX5qIdZ=sNqDh-P-GY1?1>K|txhHwZC}sz3(l$MP$xE#5
z$)qhRJ6j(5>LtDo-ey)BePz3Q>+jqkE7#C<!QP5XrtIQe+Z`azdGNH}s${oi7ozf>
zcE8`3`Sr9|G>59f^$?FcQ;+-?o6Bhy>BOzCla`t?&okl(Z)Mtg=Or%NX3XH=oi-y}
zoZEmaDJ|}rpKisr$1~FxbgCU(7%cc|!Wq7UG8>#sHj4#{2xeUBw%^xt-P<Yo$SZ%{
zUakwLtroA@)WxuRhZnc?nhU|+wg+BLE4AOJV)7+<xl&8)b&gtHhMSL6Tedl6Y+w7W
zJcW7g>^n-`g|SY*RK?Bns-7P=zwqrxy0`7t6=%4#m-Kvmtrq2>*!Z|s_2aC@V3`*Q
z&C}!~ET3`3<xOf<YTS3UC1%1o#x|KFQ-1l#cfEM{T5qdf^)}^ee!t&pFoh;}n!cJ+
z*lD2l^>^Q%*Aag{9(n&Gx%V{l&6-Khk20404>eT^QTghhbicqUxa_>`)FjcJdw=lg
zTP|ba_`lU)@0(&C!C3;GH<vNYZt0nj_$t2V)7)yMBd5d~m_&ZF&6vufkjQ;~=F}5m
zA6Hgwc$B$x(b08kLi2y9Y-swqQ<^dVs9k8`20PxusY|mC_64RSW@pasT3zF~BUW~u
zZ}NphVv82EYIrDen%6|~DSzK&KhZ7Xz$FfkHj7D0K_?<TYQ(%2l`d2}w?ysEqt5L|
zTwMzy3pJipb%`xLG(~8uwx;T&>!B<PZY?>kis?(dq|UCGYItw8u6Xq0@EaZaX0KQk
znP+>;H7YP@PFQH1WO(L?)ikx%4N8VO(khJVY4*n~Qc}A93kIj$y1;d|V3{!|r*FI)
z^X!%>YUwL8JI)GxGGgr6WXc)fs}+9oREV0Rso(`C-J->^@2)h-9eCNUH#u1A?u$#)
zMgJ_-EC1=ebm@m&KF(-o5ed1LBWvw{UgW>~q)qyu;JJcU#l!1O7PjVHx}b1{=T7~r
zg_3#`6_ao8=@3!)zC-R)!Q>?`A7+@XWH{OD#=vz-z4_b8ovf}LB|?)fyl_fV<gqk#
z;ouGII`KiVt*c|9Y2k~mB6b_G$SbY}t1>>Ou_!Al`g<twPV!i|e2JgV$tA8%FDI|L
zF7_$*<x<_XlYEwMkz&blto;zJvZ_dH<GdiBI)!3k6=5Tz!wx}y--S>A|M%gF>VL)y
z?N+%Q3q=GNt_GMgGVWN#u*FJL*Ce2Ew`NEAyJ-%*uiVz03Q-KT`yuhp=g8U}LjK&9
zqAPuS9JzW|FBHk?IK1y(!TR>qu5+&~>C`<@Ft_IJ!le5z&&e^Z(r0X0#m}4}rNHOi
zaZF2Tv)Sw;=Qg!;UheALvS5eq{3}noR;*o=wf2N-`Oa_MpS`Z{miu&T1(%jlVNP<M
zeDU-q#T4P(7n2g+eRtDY$eGkFc;L8*rNnxLbVJF-H>T@Mp5T_J@2kqKlJ;Y{Q7gj+
zea=9Jvmu`+R7HKAx<=%H*45vq)-KvUZ&}U>o5!4Sx@UvzZr${?U$%HlP_4UlTC+#g
z-v8SKQkZ!2>bwi?e=+l$GP78u%lM7!t&@JodzT&hn35hZ<35}F`-Hilj~b}I_Kd%D
zAe>Wuo>o_M^P{!X+#FaYH9fak?$2D=r~H>k)=@jf>WXB@!VSf;T``X?C`!+C*<!u+
zZO!ojX~Ul<^*?p>PTI8lN$Ag0$Aafgn_0eO!n<=G8mfXbmb*XGpIsSyCOYWHnd5uK
zbv>)yxZ=NXeRV(Z;mWp(ea1yMT)Ip(Q&vpc7@;_Qm%3rEB13)G&9LpWuC6khwJJFF
zY>t|2B%|N0fT)C{ksFQYF>U8M-(M)YE=ofy?8es(rdP6`h~2)iZ(CLS+I?I2!hN%k
z-<oCi#Mfk<=Sr{tUw`eoe!$FEKtEbXN$U3I6Jbh@vm*kOE~jk0;J0?m(W_A_9z^yn
znAat<{ZOv?xsSJ>v~mc&ayoT@WjpT=;RA|OyO(~Q>*<qIFh5t?b83ZodD4_EDd`us
zaPB*@dGgv22Q5*jd)HZB&0AvVYa740;N;EXhu#0SXELd>`kvQabN0&@qwmez`yQ+3
z|C#yh{tvTrMRI&|qko9T$sRA_bZI>Fzinr0^0|{Oi_@meo#K1eGIQ3CMbGAO>GvM4
z*4yc7F7t@n|G-5-B?XD=Gv)^zmuX>=Qj{~VzG?5bVnU?*6aG>~*ZB9lI`nR(h|k>n
zM%=gB)q2%p`M)bF?4(xSdHn91(Ut#qm*j~%+<QH5o#x*YRb}DEKUnyK)f+RtV~fhS
z9WQx)Y_+ssLgD2InWB7s)lmPbDXV5iIkKJNn{a8t^4=GoolBiP9fL$;YFu(uC$4+h
zwQgD8L)VbM;o-OEUF|u$H>Fti-M!W37xx}3ebM;;^&|G%-&cw3Sf78aCSE&%>EJG%
zjfT}Z=d^zRx}e0Fak5}-USamF2mNKDn=^86Uc2@uQgiXE>+c0V^;u6@=EEQ_ShV3t
zwc1Y3${WrX-U@V>s_c?Uu531J5H60H^iVX$$>WNu(bLWKH?N9mPFC0c9R06NV5wl9
zk+X){wM3~KNzxpG3!Y@py&|wwg;h|1HJQO=WxLOR$?fJG+Y*%0`I!QWz8+BRSnMmf
zt;k?|qU5z^xo^#Hj}$B4C|>?iibK&@x2$FHmJF`+MvosYI&ASiH|mUo3sxywhOdYV
z7R?j9UUDHlIgq{7;+q0{k>Vm{t7Xc%%NLh!7Pj6XT;`^h;@Xy?u_(o1qG4%8%A5r$
z%SB@Y!vaH=DK|#gG~Tc|8*I~_q0$hpl<%Ha|GXnZE$wP>+Qc0lT^^k`gwwQIRbL!d
zop!wL&G(o(H+#WGdyiIujwzKpgJLI46`gEWwZNnP<5teu%L*fuvWmYt?Nh3*m>RLQ
zta|?ozYSuRA6dJ1mG$h~o_$bUBZi}X;g+7W?gji4drpV<{=ekWdfr?|dO`1{<AQun
zF85yuo;2&bRUW`08u&=O?p25Di%qV45B0LPNZk<Y&Iu9CbrF4>W?Wk4(YVF%=kxjy
z#tHFB4WG>_BaC96PZ5~9MN%tSt?6mL!}BEf6%3r_RV_@?iZ?vXgf$+DO1rHP(8%E5
z`O)WM$fRo}rt><Aw60qiiMQzOEdJzDZ06o#yIjCIWAZ0cV?T+wb|+)In@M(_^_tHL
zf;vjtHkAasPH|aTVjEfFda@*5qRoG~KVQ()loNuX;sN59lB1>vXl^Q9TimDVq*Sv)
zDYa55Yx=Z=D<W}MMDo+i3r?mw6_poMmRE^a^m|PA{5b7dx@hIf^8dUOgOZ=AXoUvt
z2^C+Sp8mx{|I@SJ{3X8pjOtA#YOkARyqD(9KF&4a=)`VIwWtQ^O5yPE#o7&m)kZGP
z`z*3IE~{Q+Su^Em^|^|gqt|Org!f*ln0;Ad_FT<S;RP(yQX}OPbq+;Z9$yss<VECy
zP+dQlsLRK6mw(N9y`t}ly66k>=;!5iyCu85CF093y60Zt`*VFpxB9$~Z4L7UW3Np0
zePt&3$}@jT`vmS{BXwuBG$*d!ub$^5n)nX(tX?=@a3L$Nrn<nkNqwJ-6uu=I?o3n(
zD>hDVwEA2;FJ!WZhp6_lg1{Y<jf)q$-<T*pYj)7n7Tcc-WvgZzEiPdUUbM1t(SNRk
zMe#36{M4rglqaWX2r(~8j$YZ8)nOI;vNV@tT9UgkujmqctM;vj+p{Bctv5~gZA@*>
zNLBmP5r3_ta;IMKuIcgR(;M0adMnbdD0enkc3yAp%y}lkSrFVcbM6AQt_;Tz-sG;X
zEtTbywD~vB%y>03P=TRl>8t}QmiK;*uya{{;zV}SuI%d^YCAkzJ8lVF;Sg+Jwc^~4
zx|<b(w>4H?E}wcdZROF<+F4aXCuf!#ew^c|=xW~Bw|G|HvRi#uzx8biU2b(X_bZ2>
zL9zz#M$X^c@&p<^GM%_O110u&N&H|o{Bu0-jq-#;SrhmdR=oOVIQgTyMy2OZ@yX_%
z#zGDg|MOW*++?BlNHx@4GTw$|@;$9Wfmy|ZT~d5r>r8(x_#(7gHmf99BGdi(6ba5L
zZsk+_R!;HMH2x|z<=v@uuAQy&nJte_EsEn<9K6#%?4{*@mr|y!ZLud69j`4mQe2W}
zvC)++(4ng=ZTd95q9y*fHs+q}$cvn=$i2xcs$8LZQ$gjX@{{Gf6U$r6gAAkv?|O=R
z2Bg0_ys9EhY|?dsNuDuFS5BBAzFhFJ<Qo@_voGdbWP5GfHnU-s*8%sMO_E!;d5VTT
zU2%5CmWbQ)FMOYEB&KuJX--_VU_!Lzsi19cC3Bv7=w5kLyL74U(qG#yRIXZWI`?_d
z+_LJqfvo+W)|)vm&i&8%P&`H`W`)MQNedfR3$A(fEzgeCkaI;|Z>QS#YZA__Ytk33
zIeS&|^G>5QrnUd0R)kw`?TlLC;VpgpWYWi@i7wY$Y?n)U&L~!Y=II~NBGHxPdSaKq
z<U%E>-u_*?|LzRkyS>#tV^LJaqAilMRjc;MXzk&;D8(c_mG$?Y6bpsq7nUYo0U0wE
z>ut0W{Jr50_maEXDJ2s7yw&!_Ft}ao6kX4%@3nAWiiqgk1^eo!Zz{-Ks<OKL!|Q3^
zS1<V*of_~eDt1-pbd4Dav)lz7#XT0NmWg>pT03M}+4mnY`2Jl|d2v^j)BJ^wYq@=v
zUE|m#b7#f;?%4~rtl0B<#X{>t^G~n%FXwegD`tg`kN)}PEV^1cw@+*}{-Y2l-8XZU
zkZ{wso9&8Aq_)41ICB13&c&I1mphkin0oNbrrhrl#sZB7^C~=gD<%Fd7T7y$$F@^3
zQ+^9P>RdDV;*NKbJN^aF=eo&d>M@^lBjblHYb9??bmVMOoUyj`klnAy<4;SEue&&3
z;^w+PR_ni?T4!;2-KN*;T+_XxIpSN-v?^+>jy%!QyJnH_^*#Drd#CL2=d><0mlB?L
zYr~v7i*ru|#4w(6kUCY(Ay&OZDd9`0+9r`VzeV2sKBa2CKWC+D&CQLmD*N+SPH*58
ztPrnY`eUnhRiKVrP{?6*=ky(3F@~u@b`y7Rb`6y%3{wC9vs|q_iTlz~zmszg>Gm8_
zopbiyoP+v0hcB8RKCE;u`L^JZ%sFYhS6&y7a(;8}^vlEZZ`GcUJYV!$<oTrY71?t=
z_nhy0c%EzF_SfI*p0EklO)+Tlj_I0`+w$6ea-qeumjW40A&GYc{#5uWT$XGp_RQGC
zdZc6i+pb0#DM6*Z>u%pUc<t5hYf%d|(pnGxz7%a5r@6MqaPPXCr!H;m-fek##k|$C
zUwbV&qh2sUXV3m_FT1@5cUfQJa!k~XI_X!dAQ-gRco*yS*?Y5IY_MLXIL~H@ci6_T
zMx{Sz&c8XWmwfir7wyv@XJ7lU`&!toYZ<bea`&FTCYg3aSvBsJgvSC_i?3(@e_5nf
zo2Ku2)Sy*xhr8*)Kfwn%86DQ2yv|Z`=*{yRchB62oZWk9(^lW~bFtAk&F=_4ym3=@
z<KZ`Nbk3&f`fk}48!57&>inCC?a?lC8Se^IX<zuSbCb>2vr$`(WvAVNs5O^kJpO$*
zc<r`^Yvb*|<@O56Yfk%!Nhco@vS?&_aM3vPlHuiJJGzZ0{5@8)X7a>klg(robsf8V
zbMEe!UF&~kE#i5e<iUQTPv(SI#*`;#C%3(cTl4YW(H)^Oor_kqoZwYF`P1Yi3-{vS
z=~E+5=13_qJgt5(XOE)LYpdmNgkl~(i2p0PIHo_z&20{Yeti4Gx2x;c-kG-j%tra$
zr(Sn&+NrbYzs6~ilFaFPw>P~KI#c{nz|(-Ou`qp$t!UCgwy(xJd44RbXWTL;@T_Q$
z(fPa1Zq}_6-bw^ra0wC!m7kNlIQnsb-|@YjH=o2jy{Nt|zWdyx9p_G_J@C1*a#8Ao
z(<ir`xvBfjCU>Q+-ilkd{@!_(Si3!s@A<~96}Mk*KPnRw<8-u>+y0R31g>0<uiK7_
zem;6p)F^sSo`mhOnXlI*IQDqgP4xG>H1kO93um@|zRNOu_MF=>*C=+6V{LCy&Rwh6
zOGR%cn_QlJ<L*lh-#xNDdn`R?JHLH(X!WZ%v6r{KTX$0{F4#0vFG}Ix-z#iN48aO-
zu0*#5pWVQ-TTv$ALDIX$!mr=_k2`zS-m86)jHKwh^Y4`0UPtS#f8+YI``U(_@>PHK
zc}2Z@ly~h>-2UDh<&z{jOWz8VyVRFXIuQ6yqRzI9=W(8~9oGrQ<rPggL|#;G+uF;q
zkuiG0%ddX=?sb<UHhtjiHx~X-!&R^$&f?s~^qUg-AE#cvc|LdL!oQI$c>!nFDK7bX
z*okMmtB;8LnP;1NwRGj5PvYIdIeCr1!qo{ulE3z?ZV|lGu<yd_X))~+^CmE=eVrL|
zMs~rP9~RHMjz2ux^1W9egxCK{?Q4sh>vqb;Np`$4v;DgC+!vpWSC?`xhw>l)VKuSC
zCNzBidg)olwwzNOW`5KB`&Dwq>s{y8@8tU~WWe~p%ARrMWGmGJtf~QQjrKoc9Fhan
z-*o)H&sx3VcwWG49bt}(C!@~pP1Jli?felzMYm5)!qWHlZSFa;Y0tFx(vM0iAFYx(
zt#G<LZ|5e?i>D9ToZ%=G=a7-$|F33l*Og~9b9(u*HkK!qo7BJUyivw;PtZ_dzxs!*
z^Zz6)dNqIcE48=l1lNC;Iq<!~UL>{P@Ada=P5b{||Nq_mAfxxbC;KXIUR-`NdG^i7
zbAl<+mZuLLUb^-qpWd_7bGqklZga?q`qBUFiBjsE{lW~23wWmMG4XO)%-F!=)XF2S
zRC6Lg$)#OXGj7g|z{hUA9#&p=F0B$>Ju$~aF-7^K%JC^0lG`dz3jSAf4&z8yin-a9
zxO%d-=_Dm(g%>XKrP7&1Cj>27TgK#Kq%v8>QHU{!MMGc#i}s<lGV8N^6cSFewFHT1
zOeip26V;Y&xvTW_hSd!_Bt@_MJvBiqd~eLdqPM$B-`!ioGgV=VfW(?R+lmEug`N@L
z@MLw<(|L1i#9lw+6jH01cc8KRg;1*3e6zW?T(YjLjod7@A@Nkp&7chb4Y9w=^6qR;
z|Nl+)&-R>K(RIZ$m1dt7;|t!=bMMofW(K`0^NXv_vHt%0=JwWd_VckeGv7X$Kl{1U
zyedgQj;qyIBLo)yX=6QI61C%kU0vg!_wU&^>}U8Yyutn(%LD#{9dd`(F9_nR*&xjL
z-|Hp^BcqLwLz8dCBhQ9K510*I1l_N6^alEIt<nhc`E+5)#3;$7YW&l)mWnavq;!aI
z3QbVxlr=5tad|#zauT<kr;CxN>hzRNo+`~#JPZ%21w7dN%;H89iyoU+;%TEh8p=N6
zhjfxnjZc}ZZMo_bu$kM;_eH~5A*&geSr{L@nPkP6^SHRn?*Eaq=N*2l6m>mWX5eS`
zc$V|Ji;A+XUoO6w)2+Abjh?OkUR_h!4jx}y*3w{q>$@ci!QQT4LqabLXLvWiG1jo}
zRhrZhrk=gmG+gt9fTv2>tqFmPW}EGLslk{vsW4(r6jvC-jHn_pb^hEcPdBkvr!JSd
zQ9h?umi;ftNsnGQDZqh?lW%s0Udck9iSDZBQg!rC-ng5+`MBET@Xa-keZ^0l)|h<#
zzKZPTaIZ5W7rKnjimhAo;D`fvu8CgE;vZYPwDXD-9-Vo*+5Bz;bEL&Y=Xtu@oYyaz
z66h12R+;)<uhw$U`y;{$yJxN3ExGVpNm|D9-d!vFl$uyHJnX`6UhWr;{2VN0zdZ5m
z`pJ58rfuzBHc{-wtJS<qqMVisvdnhkRNwSrBgcdbO;e&gBs~|W6}MI&ndPIBJY`{+
zOlnk6)+2*kxse-}ibm(D%vmlg?A@)``28!_y2BSwy;y(j_9C0SI@LqoAFCPL_)nc+
ziu?1qY2}upE^W)Nn|k#AA7a_O#n9V$VYkq&IkNtykN4~?yOY}=+hF!qB*U~VWqn@K
z<@Df1=Bg~Q_qIJ#ulQG~$#Xa1my_U8Im^de>bLy%n4Vn{^kHuGxl4`ntp5iGn!E1*
z5$f(+`qwYIRU?CGW%E?KUt(KUDE1Vb<@j%qq`hP6Tc=u0Z|z%*Ym$Anqb!R$gQvLa
z&i>#V`N-+g?KzKhvLC8wINv<cF4rM4LuaX`|He)cp$~ei6;u)<f5yb!P@P!5ms|G*
zQ^Ye570a_<4T_syo{D;=<Z{~MnZXOMIoyKEyH5Xot?j_#6Q!Ugb;;*UQuv-5qQXXt
zgudw8&tCLMtSVWo$WxUe*l`Anh1s$HceiXbW8#p^cW|5IEc!g)bRb{xiFVbO5wC(4
z9h_3mqPf|zZ@JBF%Ydp4$|Ca{H8O-Q&KK97w4X`vh5ghiw^kcSZvFe@x#LEs6YA<E
z^8>wBh_FrRXw{k0qPyYj)M?GClY^YNc0bk4my4R{73b15`3y@zlX6U7NXL$|x_Vy6
zdQy$1FFCAkJJImlD(?gjb;g?o3CvMJqBFNhs&$<`XHqih?)yhimRVfnoRhyRSp0p+
zveILHvscf)k>T`trk1J$Bjb*ACWZz6bxaBjzwe|ooI7gc=zCu4l6HpE{}9{9CSNLU
ze~xq6|J?TD)cn+Wr$qiZZN2#8V1DTP+f!=WwwWwy{C`P%+oY=u?k*0gW|Nc!Z%kOP
z)-fT)R=`Cm=K*u%!8J3Lp5!0>)X8?YSw-ZYOQTQWsclgg**4F3-61`*JJrlU>_Xv^
zjybD&4m7pQtceY6`x7)hOtj2$z0<}7?r4dZ4$iLHX&2W<`1l@kh-5RW=<3>au`A?^
z1jD{x!VLASISGHYZq$0II%mI$NV=(Ue4=TYMY+wrB8gDVms`$kn$|nBLNC<#?Vjgp
z{a?DgoxjXk#JhfpYp~SCpxY;R{&L?Mk$kImLfib+rM}x@rBz#%LVUI>C@^_G5N)q%
z3QpDHZ2mBF_T83EIx$|W^Djtw>av6@GxGGQyLp}Ic-AyqcZ-1M|Cz~Z`4exdoNv53
zL-@d}&?{9-FXS9t9Iq7o`jpCUBj22|hdr@xKX@;{^L0B5>q{kTL8d)FOHLkG#_c=x
zLXv;YypKI~SD)MewoGgYia5j|z|bbTyg|i7vGQcG+~+Ckj|99IR<*raw{XYyq|R*;
zL87@=ich$+d0*U7`(kxr8JpAGfQhG$uZZsd+u|em%OPlzmg}bfQ*;if>U-Kr-0EPt
zu`Ohl)r3Z~)-b>IF>7X2EM2azxGqrrlcNeFx0>U{PR_|%f})!~taj^B6j{4VQ1OuZ
z?$F-U$cdfTHV1HO2TzG$zg2bg>H)rWkugI5Uu0ePeE2Lz=2-#LIf<5fHU^G{z5ln^
z8_#X<VP0@~_CJ$0K?atV1wl%VGrFX%o;aqt=2x;@v+2nUXLl>9cC!<K=VaYCyL&Tk
zN^PAJxuG%FIB9<S4GUr2P9;nE%ZIv7%s%p@!ALWj*>U!*>*t~*%c8k>zh7|j=x7ir
z<CK3SW}dg%rK3o3$D}D6npF0@@hoZVRGyP?z3tgn?*n|#r#-nC>1o^|EzJGQr<#*t
zFZ=uhY)a2o>J=8xPTDkM)+v>b=2It^3;)-j&9-J)8P@}!)eqBbb{}cHP?2`<`<ds>
z`ZbAM+xHyO_n)*<{nA5z|Ao&?*+rZ<mukFmELmh<Zn*S-d*tb&D=%JM&Tuzv-MoNN
zw}CUV%=7>Jc_{)Lrz}<3W!35QNonD&DH_5#)BJ7)OqqDVB{KY6$IZD{g_Aedwxoy&
zaxJuuoxjhe)lZk_Ky%3j_su&y6}9sB{oFKjt=qGN8n>FM3TlRzWNv-TzuRH;aPyX3
zMsgOFbGw$P#%3N8GP^RJ&y+#n!*!m?Rmu&i9BqO6N(xJ6C9iUOla%~6;`pUsDHD5j
zF6!^DKKpLo=KeK)n><-++C<)4o=GdqT<dme>Dscb>sPJKS?quNw%7h&A=Lt{EC~*w
zN@^<a%M{#C<>slYPCv6J#{9k+$K54teC}6{>K&MEY5B{zugFJc^`HGtR?XTD50@K@
z8eLE`XtcPoaQ&ZG`Ya*;9sVRa^sW}QmK1I@*ifNlIZ0WzI?3Q{^9J4+%Z8+#(`RpZ
zyIMw2$F%K%ov@2z%ZqKoI*y_)9P&R6_?zW~W;;HYR2JDR*Pdk9@oDFRW)aELe4jcE
zKX~$gUTOH*QMT`v;m^YPii_v}FwpI}F<&`*lS=X??Ilh+HI98Zrp7t$)(Y;NlckU?
zxN366>;y+=uB>T7lcov$T#$2;L(pjRQzPqxnOzA^9ljn7*MpVBC)xjfeYineNybWT
zOVuJxBU!!`i@G$oT+eVhahc=f<Ri+$4xN>Ky)P$v%r-gqTig5f5ueXTYPoFv51U*y
z-kay_+WW$+)5BG~<Y;GxYs4Kpl^g$^Z$z6O4bq9cd^ECJCvMMH>r0DIMM=g_mYkue
zl5D*%iF;qlo0(~6I4;cM&p0C4Qn6&k<|UJMEy-T8BxjD<{||1Nua{iy6v(x942tH~
z43OTI>HI1AcxOZBtQE6+c8GOe=uEiC`I1YfWr4ZTgNYk7ENl*QotV%q|Ixkj=fVpc
z?KMw!H?I~t&%t}SQ+?N^WtTW6yXly8EuMS-;tBJUCwf1h=(Z4E6nRJ}z@=)@q$3VH
z*^75hWjQ$Q%5olI!I`H;<yLhIzuC9dfl0H$QS#3QsV|<d6IaZ$p5k&)&h6vD#ed{Z
zDOpK7?@|eo4|-|1{Ey+(MJn274*ff5wPw$$mCiz%3ICbn1$XNeAJ2HWX^Sz}VvEB`
zKc{YuHs9*Ad5Hs;0F&TE4^E>Ks;0LU+AcURZxWaK)TN@;)8nAzbfu$Ia!=dE>CKH2
z+KGD{t5+ZIS-pl^aMGa_v5R}%FRx~~;IseHo_Zx$U+trQzqKz&n_g}9^bbCGE_m;O
zQyZ?T*j~!^y<Fqk*Kst@*tU&f@5+uQjSEb^63u}U&At~rqigo|-8}kOM)GFBIp;{f
zldnwgHJ^KbfG^>6-}zY`DZ+NW8uKz)1Xl;zJ)bT0)LM7xLzdqf{%bed-M%&J%btB=
zJ_o~I&Hj*m{HM>l$tzB8e6#dx_JZmqyA2-;HvQcFu}fU5aM?fR6Rjct_gv205!9&f
zY%MHqalqu^+`pIRa^3YX4^(H8o;N|-vcs`^bH;|6PX`23PYR_9iK!eEaSfb#WRBpT
zsVp}Hwk5Vk%?O+nuyF1X%NIKxojeU)7*ENVOi_I3xMazS51ob^E&7#n<}c|!rO-TI
zf!k<-B*(<6ZRtAuG^Xs*x$30Kyg=(qkQ%q(j4bAvQIkwcj+?}++8()T@`cW+8s6n8
zmp4vUSaCpXXG_1~)y@STorbQf_8jR9IW4fJLh(>ahl-cXdnFgk$699<=4#FC?Z~vb
zzQN;a#hx8a{hlE!|0po)Zd~mgXsUYQ%*qF=Sr(WCPBBohxVrz*mi=01FQ#1e5pM7f
z{(n(qPUv?N-z&SXUb%er=AF<PPp<Yv9F6l;ZDD9KOkfYQxR&C2%`iabQHjaDhOk?g
zU2lJOeREQI>O<vjBe%0!Dqm*Iyzk8a<$=yT#p@Rr9b2%{O~b`b_vo>-%PQWlj;)w@
z{c7j+tlis!m86d*E-imOp>&S_r)JLIHWQ;S&F-<#`&VLrXro}A%bd5jxR~a0eBlac
zOE~eh%H!e#A$?6o#w{}ne(C>RHTkCGa&yhe6PL~{eq>=Tz##r%hfzZ0)tL_8vaU?=
z*(p?da`s#G___WfTj%WJ;Fim}$aRrLYH4fe0@g5#DCy3I+9L6JTXxP5S>ZS_YQdIk
z?_N4C_-(mV#!6|)e?zt0Q;R=~$ax<MYn1ZU>Ud=lz3lKMO&?9YtEYUvUeXH{;(a-F
z)*}v)PgWaMrtO;CF>97W(2v`de?$+&PTL_f(Xe(}jMU|*9f#vGFPp4V{QX<}khjm*
zsMQX|XS`Z{b}q7UJM41Ccrk~tYhBjfOR~0p(xz8Lk6uyTdad{BrJl1lYJ59)_zIr6
z8k2kWijVKrId^ZchK08D-x3MCGo?c#fjugry(ghnBOr9I(z)ms(_>D4lNN?O^O0QJ
z>o;BV*oO^$Z(EMN*%FksC+MT<{mh<SizKB!{_%TnbAIJa-AY&gSDSV3cm*<cG=wTK
zPg>E*FSzQTtDdUlI<<)tg<p!TulVmRywY0gqWh$RY5!Ypupi^;TzX^6ga)n@KDMjN
z3^fn@oHe(9t}Ne+1OYz>!M4d=&gvXz_<7BGA89hOJZ50MJ#k9OPW~;CT$dx&A4W#W
zUh8oPd{}y3W}UqdXIO$OQ;KcW6J9Z0fxWf6xFlNUg?7s|X&J8g8Ypslp7y?|B|Utv
zR%k5ylBD9ZOO@xguVA!B?IGFA=RTgjR1$mr^W9VWwaIGAJ)ILai7(vLDKOpIPS0RY
z$J|G^?b@cz+>yL3<Q~_d<jNN3^&XsN2?@JJQleM1Z$C5fxTQo3gNbX5%h{_>E_s-A
zUl7sB<a^q<<-*(5p{y(XLqY?*pN7lsjkszM+57*hP|Z`>Wio-6L!<jrlS)%}ec-sz
z>>G0}^7@oedA?_Hf1f_|ous&|TOzUDri{z>j!D!P!6Um|!@J|mCxty+pd7P!O7_>q
zai5j*Y~5U)j-{GBFFZT5bAsEukjL*ruIu!wq*q_hd&5+fs{3t8xPR43jTxKfJ>W=D
z++MG{nayb8R0+K>h23T$s}7tzKG%UeYuAkqpSf2GcJTdE<Np^?uuAw}*IXerb=EzT
zC-4b(L_F+S>oA$+;;f@Lr*4aEWD^W!V$#$Q@|9rKz0hR2?PS!|OtDy=kGrPNUn;X;
zpLj6ys|_0_=*|c%+aM_Yu+z|?>GD+>^|*PnXFIOCw$ov!+`P;G9hLrC>OH&l#$uPe
z^)92h`G$PApL*GC_09g88~!mSeD!LMXU3Y_Qzu!uEsTG;OQrI%m53Ohkl3NOulLP!
zH!$Yh(Y-r;qx&(@hJr}rpc$)rThE-Bc4n1uhu53M=dZd1z4h^Z>=O88OQ5vv1-7Oz
z)wdV7#a=Yd4Heq^Hu~*5?zh*rOh{hGeci^FH{12zotAr7a_?Vh;jr+%pL+J(>AMf^
z#ir~_d>odRT6a|A$JqyWn(j<{sgiN^ii@(?uaFzn*KW?f_hirBnAvM`&fbgX-u57Q
z*JJJbMWrfRpZPt~zF%4vzL-(!sdC4s7Q369p1<6^?{?97o{a*9pG7&koYgrWl%Dzj
zI)+p1w@bF+6v5IZ?(?%UmX&dF-k2n&(0o}Tk;~Vic$P4?s+9H1j5Y)2w8ajUuXyZ}
z4J6e9r-gk~efV<j*$xc}R^J!CVI6`GdYYbJYx;HdV|dKLxZJ{9E&U&w`mZeQo^E~j
zs#w;pg6YMHpXR+esmW%z=H0~wDTZrGH|pNIrQ~x-HPva8++$UiV)Z$1S{3Y!=6765
z+V(r~y2@;))g{I29gd5iyzMNOFl)v%(=7_jVq#ylC*~Qx>CNa|zTvd1m`wGAC(@a%
zoHv+Uc3Yji&Ea~w@4W5w0Nz**?W+MMPkqjtTyV<`yOw%;ZfHVWV3et8v{`9Dejv~6
z+};4yXVd<FEOfXQVISr@p((JV!S8pjocXiZ>tEyeCq8*r8o{$SEY_8G^*i^?c><fC
zW!%l1uyJdipJd!=z5>1_h2O$b|Lo0)Us6)`((Ijl_7S(Ldn(mt0)=n(b#iQv`<T97
zV!uFU`KQ(Xa||-LZYiXfb=BMvTgKOGl)*glZ355RjK1arrN7qq$a2gKHJBCZDY`zf
z!RlRJ`;Q>zCfx}i0~KyPcD<F{+jKnP(V3qgQ%_yr@9}AV*+oV670TDuxi2a<ewwIS
zUbKQ$r1?<R+I>md=Wne&Yq;U>t-1ZD0v2{Gcrbs9?{yES+n2Lm?`(Q(uD){DyiK;-
zG#&L$Z~YRyQvOMjWJ;Fh|J^eTR&6c$b;z$k;O8mh&z({$*^kwB<yPi+FXedl`^m-?
zyl2kXu)lWj^A5dYclV;)-HX?%FWj?>X!D#=_Euy|F0b`>)w<j;(StYQa*v$fuu1aI
zBfDopCEVBN+|6sgC>ee&MU<iJ0@E|Mrt?ew+!f7>PxTFB`4&IbHJ<<8mFCd6+@oi-
zl~1G=J-+fTXWPA}E|MpOtlsdxFTA(rl})}%;`^_8&mXm?FOGlxZRz_DduG+-%~~{Z
z)~&8N_g8i8dK=DhE1u!~<M|irj8(Z9)4!JlPt>aRmUwvk<sm_~yw)cT>suo(h^bj}
z1#IB7C}e-EJ~ie?+xbYfbrVvUI9Qn2ghc);Y*0AV!X<7LGULEPCl=2WE|N#PuCD6d
za!v4xNNTVM0|$eRPDKC%Lz8;=s+<>}RDGtavYxTIxgmvt)l=n7h|$62Q|1RZX-Qf+
zo|>ZRJ~_|9D)sbb&6p)YbF;Q;g{_X>QgriFYftzFrX5orCT-J^+!TLxnXdJ=Gt;)m
zJh>#=aP8flmJpG5sxzj0i|=M)nl7nh@xkFpld!Ok&4rJTPE1fYuhVP{S$$^htqoo`
z)YLK<*WBQdxf{vwym1M8UE15Cug{v6+?b%Ge4xzvQclFyUAEN;tE0YFUC&~VIUuGP
z{fKR<l4IvN7Dgc!PKQlao(c?`qWu|E9Lql3+urZLvR2{$geFblb?%;<Q=J>xFH4u?
z3)?wO)BO5;dw+qw%ff|I4yZ+W9OAO(P}|}vlB}|EuEXuE<_oJu_)->DztL<_Y<tys
zEXYYgYRfza#tTzin5-?urv&NvEfDF^KP92uBYlA5YIpF>8&~^c3KXXH$IU#-s_rZm
zxg^p<D03Qv$E3!T$v#e&sc~}}rinX6G%b^u&Qj_*BQdqa>Bfw*shVmtQ@SGEG~e2`
zxYPzFT;occJ5lMtT&AvN>Q2i)zPyw^ccY=#yd`%$-R7<3yb$f!vn}cLVx}uW7sR-p
zn8<q0=yDfYKD+9c=86Y@EwxrIomCa=<B+lN<f^S3HVHa%nyomwjQQmM_W3Jy?pj^&
zG1k4JJ^#S1DESOqPS+`c9;VZ;1R1D(RSpUFoGrRF_H62t359E?ifzBW>1%FOa_jDC
zG205YXXH*`En6F(ymNI#Vk@iJ`^2WB(vgl6o~{;1T^z@zxtQ~(XTSk&&0A91T~cc<
zq;m;BDLBmf*i(AZj=gFO(lhUxNgbPcHF!=M%f!Z{itC-<Wlt$9G8{R@v?P6Ax!(FO
zpVr=<*&^imzcb}?>wm*7tJ`hA-??|;)mrxwFPC(SHD_%epSgG1Gk!+NWzU?2og#}i
z249d^(<jtec1u=w;T4O?wqGy0a=+c&maG4E(&Zg`OTOG<VHI%c5;9R=+--91$M*Zd
zH~w4fco6*O$KtIHNxLl`d+Wt5nN;j(IqeBcq~(RD28lZ>cQX~Od>ZlP+-Xno<XQJv
z)l-+$MZB83=4F*Sr>a1sgK+Ax=y}V{o;*Lm@-!~seNA1NdS(Y-NxJlsl5dgPdjxr8
zmQ0pQKdm-D>io;wyPiu4)_p&E-S)?ls`C@pvs^V15Q@zfJgG6e=g=yV<Q%1==ag&h
zb^aZBrN8Lg`-xHeZr=%+z+#iG5v;+rX_~6VauZd}lb)wS7Ku*H47?B<6DafaK+?3r
zokydS6t9PGIeKu<4p#ksM%jIPj_8+9onZKBm3VTJ#>`VTZEePD7J92};5>Wen5o^H
z`A57qFL&desKCkmzwfZXrP(*HJTo(V=$pPsV^-9S=D=f$9nC>1S}F}4-9I*7sB95C
z_U5W@z4>l=M}~$L4iOH{5b66r#F*UYcXrf2;}*DmL78E7uTx$4?L%r$)EhTAuIRCQ
z93UmU)67M$$a(3j3pXlSS#%e9xK>R*wCUS5r2~aV1!h$j)mLs>tgCR&-QiKg1UYjR
z)r~=3>!#e%UVLSLV5w(YyixG1f|cGTrHQ_K-yHFjo5a!-@iwshm+ZVrnnzN#f*tCd
zmc3h`sbSN5$ZUCI+UJfP8Xh;Df~Ksxs&iO!a%^O;$W{ecmLn_Az2#Kkik{$7w8df5
z#D?Mshv4Tw7&-)MPi?f@&ZYM2ztV=uDhWZ{rz~HHZP_rVRqwebd#sn5q7@^ffvRTf
zVhz^SO9NN;xE4Blon~8ax-PHxO7zoPVsW=VO%*w{LE%f1`0f{_>&4Yv41arh?2ph=
z(A=<4_H1&<hlE2bnx`%^wRPz|5aT@SipQH3tcqGkXJu@>lhMQDzQtGDE={JFHzkE*
z#*zzf$`yYKyj5KE>HZ~+FB7J$DYZ>^^=uLOX}08|i|~>JckL8bhH0F<nVFYwUSZ<C
zv9Bv-gO|2^>nu0Z$s2mQlWwnWStNFH^F&`K$xS)RoZ1n~U5i}0->kW;7I~7x<>91L
zH(3;dyKh}xwQcK)*o<H`t=k7mH*GW3y3=2l^50@*PHR!`Z`}gP>ni1`=e&+>o#WJX
z+p?UurTA>@yW&JCE77iV1}m!P`b2-<Wt!TT8+4;Zx1fC09mmZxxjA1somegKRc*4#
zo6tp{XZ>99Kxebq)x-C!4okX!$xIYkmvztbCHp+fHjSFiXVm3>XRiKL*fp<2+o0vQ
zaH8Nx<B1N!VQ(v%PChA!SQyQ4TEcEmt3dz3!>VyBirn5L@*PiA;4@vYiEoAuyA`XJ
zz?=te=2p|zYVdiSxVeC%*++aU&(7wDMv9ZXSEPtco8VnLlV@^9mPGWbfHup$n|RxI
z3Y2H5neuohWmJ5>^4MUF@odL@L2kF%PiDUEy!od$W0Tfi?f>7>Qld=w9bX7LxGTAw
zp7~ecJKrURuq#TA6}}Th!j2j5+SOuTSX<)F@MPiNPp5hcj2GWHwK#+E=vLb)Z6_Cf
z{yf*qYr&QoQ<~WPUPsxpP1?FH?#1fqn>n8dEIE22v(UMJ(d&ITuWeg6$E%O)`qZGf
z)9$NmwmqI?yF}xl(6kxe@1}}09lCJk(ynGp;cb_f+>Kx5=fl5KbPmTN^Ro+f?g_dx
z*ZW9MUK&@1shf@5r*BtgH{H$LcX!IYk17rK|9q0RjaD!@P~SDj@9XXdRa*1ywAU=J
z-}Yo4v!aHh;H)H}dW#9lVuq4BdiFxuO-I!BSsZ0uaG|m1&k-h$!WfGlnRYva|BFs#
zg>B9W3FJ)M@+>$bFLc|Zsq)PemMEF7WUtp-H+A0Wq~K5w@qeGDp9=Zj`eo54pZpcU
zwXM(MKc{&J2!0E(|N5-ME5w6;_AZU`-+g9_47IYe7Jpl;uacba$D4mEa86W#XfnT0
z=e)^m0n>9=tj({Tl^(z7`SQYV2R$zwQd=|YdS!TjYE7t0`zf^uBdK27?z?O5NNJj$
z{wKP*X4#)qcbBcF-c7!;>{^pk?B0)I>+fw{SN_#0M^@_Q^(uukH%r%NzCL;T)whBr
zB0*Zs+J97zZrXjqx8tL<(c88N&XR_cr}xZ@zk6}v$+)yvW^Z=97HXVyaK=rOih$F<
zCw<p?VsiDr&AgfO!@u7-TOKs|Rj;nc!uR{ve?G{0ZqL%n(jUG*GI?0s7aw%J!C)%I
z_^iL_ZGX&77AB4<ZLW=u8zYXnCOq<8z|!Q((iPFsD8XhG*wGoo9w$>&^FU(G9I>7U
zP9+acUX(gDG2xWwogT&0OqLT|Z%TxE2%LR7W6OsdY^ebeEv+7%7x=rnC$t8HZ@n?8
zRFqToVe*z66LlqLgh-3r61e(s!WCad=M2X%(KDG1GgoHF8nX&!T#-I_f$ivlBUvAg
z39US~sK%$@!pyt1S0nFUeqb0Bb+F~o&BQ3vur38T$qQXKcJwSMoK<gdNhT^vLF$q$
ztFN49lC#EH$DU~`b0c2;XT0=kqNBX(J=MO0s!@yn6g*<dlW{u2y1MsK*2F~`mu`7q
zxYu+euG@0Ty4*?ce%{Juy{+-}M1K#fR-(7IpuE1C^i&h+&Ait)vnKe4-kcH{{Z&<Y
z+tKdVG4p5M>G@tGzT{1(h~ng(qJi%g-26C0+$lvlW5M)80;jGsy4P%!H@|q@;mG`k
zr#yz=ZlAm}vtr9)o4(*@ijE?0T9}nWIMw2gMk?6HDLDLF>=5-p>mHl1T#BIDlan{2
zx@RVE@wGT_i+p=i&ZX~$)3$^nlL@CoMK(u>Y;iftduz^?k||qX%-Ooe)uZ*wg`Y8=
zkr%usbqmkrzS!v#CnH(>)AiET374;S`}~ibFB<jG_vIAtoP#xuil#;`B3m5~l*|=$
zJaa%`D!*lD;KZAYG(?3r&UJcoZ1&oli?%fDO0@g)wiK#Fo%(V44v#`5&$NpS-3|)Z
z;u+&!7`8lK%M*LaC#o?l{i>o;Sx3y%RH=(+Z-h)s*fcfc>DiSF@2%Q6FDWfXT`MiQ
zEJ^<EF%5+yYkbobViz%_EK1IM_Nr9v_1rsad{wmUQn-v%)-Ju1sdn_<hbWzGY8r3F
z9=*A?f%X1}t7!&m@=h|-i(dA9zWd}^&$%y=vuCBme_Px4>IC0LmjhCsj^FpnhpJ7r
zvYq^XPqfn%@oL}6SsLdTC9oZw!=^2B+050;v1Pj0pZhldnG{yO@v*t|xN_6-$Zs>-
zUM4%~vL}C2X1$fp8nmD;O(vsd8N(0t2CXF#0Z*K?o@C5YTRLNbtCLD&+k&^f54ehM
zIG&B*5j)p=@kVc}3SX2!zUXYWg((~>RRq7?5#W2V^Ul$UOIHXTQYn6_5_iTmQiE+;
zi1f6;#DHg0<{P%$+j20B^D2L+YS7V^#R5Ik_qN<*^mm-FFr>4`JHuz*8aG=P{|KQL
z5fujUhcjCq%1eAqJ*1j6?c15!HOC)3ja0lT&r}ulZ>h@;&I7ZYSPst(jNc_XXTq(n
zJ@YiTNvUp|rEa9A=Cn+$D9O|}&5=XPQQ(!6KvqiHvtx;Bi%*@ex6(@cU#I5sE8VeW
znfz3>Wqk`Y7KyDrDyH*nUhYGWhZmIcrwXzExHh4HVUz4hEz$Uovhxj|sclh|U%6w2
z&s5KXX??Ta2yI(?wVG$L!O#982A;H-#dD{~Z<;j!TgGFPMD<c8%?W9bEg#K(J2SmH
zPJ7#-%z2V7%y~14X3RLsn!?7HVn0pW!74CZmu0DtOM}^wHLCX<d>+*QQr>%v)&5-e
z&PS^=n%bI*&N^vngm|oQ*(<{w8Nw`jj`!Fyk9#tnWf}Zstmi+<OlFbwuDXyLku!Vc
z3Lmw$lE~TIAD8%tEfiTbS!71=i5AI>64_f*kA3f*DU=%g%kWI7UZ&HI*<qWQ7OC)O
z{@>CRo_BS^z3|fu+D}!ibn3}$*zh>gY!Rc^jG5bbFJ5RrywhdJT9(wQQ(wO1O<FZC
z>DkNIPgkZpWpMQ^P@Q!;h3(aiS-Q9S)}}1ecAk=(wr}n2V{6s4vQn#_Ew);g*7T_U
zoo?nY?WGA?S$gg{nL;aG#c2qhdl<mF@#?Y<u6<MPJlQOJ-Qbv3-lJuoeyTl*3b@qC
zm%y>SOmMmWUcaTidX8sa%H=X@mIxO%%>B`Oy(CY$L`k)zYw_-joYxtrT3!<0y6uoy
zME15%w`JE|W^$S%vP0M@QKpXXNo`m5QOgvKO>cAeXxAUp-YvH3D<8|ExO``qgfC^H
zdp<p{)6%b3d!O-town<xxKm%VfBnlo#kywGmOBUS0$NYyvs_i+3+v{awdB&rJM6t8
z0$hp=@j1J$ED=(Q6plMB$aU<J<_502+@jZ31e|K|KGJ`u@M>nk)aj`kcJ0i2zeUeU
zNAxUL=z)s4VU0dZr?KBp)VzPNU3}+RZw8K8Q7fn3JMMYmti;5ZcAB|&ZD!rL<KDGp
za^^uPg|{!&#0r!7HqNVCc7toxy=|)(c|8*<zxwl`o9WeKDN!3m9(P_mxXxQ*)z_$J
zx38_0xU4Igo}{*`WpR*xMxK`R`j1X~m1K4=dwA$XKg;Zu^MY0}O@I99QSPo+PonfP
z-Yk12l6?7L)(jox^<A9|TTUwRxmf*QI{98ne1+&lp_S81*NK~-IySH3L!g)X;Q*r`
z)2o>>nV)@Do0ncKS9ts0?vB`#7h3NpI*7h!UKUX|=@84UqPlC@4nj{pn`t!lJ@amP
z_T`hV;cxYYQlG!A+L$Zj<@0X&&PlKJZoh6|GG3{nabRCWPp-<Lyv@Bg9{er3;t|#D
z?X+fFk8#nL{0I9|E(*r_im@>Mj^TE^zk5T#j4Q={tg8E6H-$?*-Q0AyQdTu9=0owW
zH-#~>&Jz@O?P3YbWz#e{v&g1PT8Le<qP={}nFp-vf7Y;9bDVa)%n~v2xMS_?|28*f
zZ1PEb=$}|9Cc_sjG(qfImeahxx3|ug99^>3cmCR2|Jw>Bi_c1Im)gB9Zr7x>B9FiL
zHb1|W^~vm6pSZfqw^O+{&*jS7eht2!8=|kJvFzhrrxi<@mWBCeohsDoTa)(il+wc+
z9q*^AJYH71(NcU3*C%g=BBu?eUrO`VajEOozAKx`G+SZ0lfi<kQ*xw^UAOoqt^VKm
zjoU<vOP+h5a?Y)}U(UqtQhvGOktfrd=i0);e>Vg>E;;TXs?U<NNL7kUTcGRHgl)Q$
z4{7jlWw|HZN>aLAm;dA0rY~HZg{?n&r|5sn($CA+&rMg${$G~EZoXG<{T{Oy_kR_=
zUHI&0SHf?<3^VBq-LBjJu)Q%)XZ3uRztL1YYu%LHOXfu6l>Zl!75;C=CM;wiG%w(V
zNCA_#<Q@s--dh)(b`<fg<&Zpa_wbQ5_UxKI1q%<TOblEU>)igq`%3JDcg81J&bY*{
zJhR8ZX-nqIO`0u-PtMwwdD_R~>bVaOG^OsnZJ&2><-HYQzTs!ibXjy>E)ngvaC%b~
zeE99{XWyJp<)*&NUA*nJ0rx7^tdFAiEyV7h6y2ZuTCcR5{b$$i+@AiAJ<|)NlDF<w
zbD329YU!_wMFM)+eP4SIbo5Pmr5Wg_H>))~=clKhpMl=(DFWha_I+LR@1(A)V)Bzq
zvu?D;9rx3m5&G>!WuMrUT!jkb*iXVwf4)vip08{=|3BX>!}E)JjFN?;bno2O)aN?+
z|HAxtC%(^**yeO2`G~4}a$DI(v$AE{{KB=%m!EHQU1FBGJAA>SZCTT;R<)O{O3v4P
zt-jA|!9KC?Ci3Q~Vd{B{&ANrX?gXjjaTNP(sxsy;H{t)i_nvmq_kP!1>knvo{}Iy(
zDfw)szs4gfAiuNNbDr8Z_PmPlHy2co?1;Fq<B5vT&KKO8ktf{lRzEMkcE@tdBAwko
zH(%FH4V5x!<S#oElj@{@#OZa;!q0EQcg1fx7tdY3B_`LbWwF0!!_`Z2f7)2qu7B{j
z?d40KdUeS@w&mI@Crz{HGR{1vR&v}qU;Onq@#j`2cdzYMFP`_9t#YM}8viP%6+ceA
zuL{v#6>@ox+JC22&T5r6qO>xb-iGGuhHt(m#*-%#Z291cSdN-)RPoLz?(3UGTk@vu
zyz-heM&BjY{r)pKBjddAHw#34jx5($mgoGTxGpo<oa^F(6@N@cx2R9wRIx~T@%718
zrn)s@RewU}EX>$-FMrpv`seq4?=3sFPDIh{Bgd-@@qcq=_N|y*wP0S=$1UG(G<~ac
z%dZPtFLnLb$-_0zj<@XmQf3;eR&aXT2KI8p_p4vZ|88tcc;J<B+VOjd@r2+4$KHoC
zwuB!o%{G7cenv)9dFQt}hS?duS*HbsBqypyK0l@rx5a#_ZGlDXB(>9bMK`)#vpw^f
zW9p&_d-w2vF%0zy6LNXG?U3aEZEojR>3#4xpt6;{_6U#flJ-N3r~cCSy|;1Y)OQmT
zy&PWs)f6`E51SX6c7e@dOVZJ{DGp6*Jp@xsCtPu2<d>f$G+~3HOOr6?B*6)V%9_10
zH#sg?1TAryC}3*mQyFw%#S}gHLphv<SI*2b&)p^?Jo%YlhhX@!FC0@|9GP#@ul96j
ziQ4(V<bu3QnqM`q8isB3yQ=kI)uqLD`<?deYJIJ?Df*1pO{3JNDV#ZXH%<LL?On`n
z=_f@ety%S_>&h%Ec*qoC(5~Cc&o8E9u`!;pTb^6&fTvNa-em3YwiBM3&)esz=Zn3a
zS|Z-DEO36<-wi(Mm)FLZYkl4j!o<j-aBJJ#|1J%=40{#TCMvjgPg(pn=WmMZttS_s
zF6sB0A}5<vy#B1`R>2Q4i@t?1?Q2i}YWu{waa+_v_WuTzt2H<o4z%0ZYfiW=ILV_k
zsOGeo(QFUR6$)WlnMV&q9rG4g$jX{>;o$sb-4`@CU*6_h^wi+ai9<ZjhYCc(+!7xj
z7I0T=brE(onb;!~dn@Ircy!Ihqmutu2d<K2Juq?St5t_rF6*obd3Ahh-y}~Jsfj9w
z?Zu~uY&xd4@Jyg*$BC*Hr?@u$$?TrHE#S1S=CU6}T>B1)Og^*qLsa*M12?XmK5O=H
z&zZBw<*ZK|YOg%=S%>k&qeGtdPgR%$964o80xVy$d@*#C*KYpbttTRS;Ns1o*=K`1
z4fkHzqGPG5+x4)gTH{Llt;qL^?>4wxJv4uxasJ-0cvI7BVOwtZDfD%OXsIyYmRNQ5
z%d3^H42fs6UAYv59Gn<Z-+Fu9oImBMZYPJGs-N(d6AMlp$i1Dp^mhK<r)AO1$Cbks
z8qMyk>*X<wDYp(&k;+)usPpPYSl7F=;k&x6PL(a1*mi2#l16`}A4{iC;0k&;r{K_~
zT|et|f|8agg-K|&>F;})w0d5Z*P8iDtTNR%tkasgdXt-5rSRnAMyGXl+*;+GwmWTI
zCD(xuOg$U+^Q|u4IKPoED&vHJ?yPh39h`SvUh4mN3Af*=f~bZo3ckHteT=8B{V%*e
zr<%ijef^>J7B;=o%_51pfu>4wGxMsChfZy}y5~8Qj@aHOa!O$#Kc0N%;{5aF@%jCM
zVk#33e2LNw_;`drUgjY4Y1a#fD!A9zIGk2@(kOmu!k)ittD||A_2eDewy^@XF@7%m
zs$PZ4Vu@=EkKIgN|ItITI7BOD`|V|SUGL}e*qwa%|McI5cgw%OI`#DPk>c~ZZ{Msx
zV{rIQk<aVp2kMv&Hb4J&_Wk3W&xRl0@0oo5+4=H$=j@h$nOysgKi8!0yZBz8px?5o
zy#L+b*Y|RJ|1aO~efC)v>yx?RN(!t2vaU@3JY!b&1+AVx#VfC{+Cy4v#*_~ZEMZ%k
z0@kyLZTR`WLR;llVYqdc;F`pfx!j%}4cyB`Sv3XRxuXj_ollACF_*q>WPg;XTPNWn
zFVzt$^W;JKsjad~)3RgtWQn)kU~ttq_F9Q2vcqXjhkk#6O0?udC2?;i1H-nB2}XvG
z4{EfDEOTJDc1pZ`WK+n-meig;sml>zs}@bQdG@h_pOO3W1ovqhzo_e$bb6o3n08|3
z!>77cFHeifdCoY<;%4ahxM7aU;u$Amjv4Wv?D=c-dFI(0{3cU=a;iU1-ap^wt>Wxi
zv(jTEx31osmNlb`S37&MbnGiPhHnp5UD_@$T^w?{jomBeMcY#?j?Q;ZOCFwkKBZ~y
zk`;cMLU$4>4;=2=t+?*LNBGIGhceSG3KuPRcp{Q+d4A?OrkP&(Z*^pM2`zD9J^ML^
zSIWQn!^s6_-%fTqqayqA($$66SH5#*mVMC~!(`pMio@ZE^Q0+xDSIBdswd^=oI3o#
zLjIwprl;>B!71}pRG&T6)C{*$n0hVARx@XPRceyv%wtoQy*VpVW2zcIyK0%&=ZnuP
zpTE}p^GZBqG4IsCjzhPWu-w?cn602;hE?XYHsw`I&IE9$2{W*|&$z0T7ckTP<AOC2
z7qw$TJHnL$1&W$loj9Ca);g4|j@rw5q@dP&-GeM`F6}LT%1zJ1A1f`2XROv!IrO>Y
z-K^=mo|S5A8@a^y-d!Z_EOE1;^Fo5s|3*$viD|;iGq!1-iBU6Z6}hFLnwltPyw>kc
zM^uMO{?<C1$e<vzO;gwUr0N)R&)+7nS^s9w_O2=2=WZ<7yg#QX<EUw#nyRp5$BIuY
zmuRMHPiBjKvFKQ4K=UkJ)umgnZDI4~oN&N@pIXBo0e6Nd4Skm{#)tKOzAng&=6`B%
z>(2I#=T=|t>G0$!*}hILb@q!d`j7r~oBwliv7K9#esA7qi^HwYi@0{(X@0%Z<(zMY
z(?6T5_WCuecukiqSaI90Gw%AXmSq|)`+xI0r%p+_FX_K8&->%sq$d}8`TCEZ2=4N{
zxO}Pf>yzq#U&eW_KW_Ub(_c3ys@pgB)ycywYRf~l?*I3=ES~!8>y)}syZQ5NDxZEc
zd{y`B^Zfq|9M6`-ooZmoDGXZlYN`f@L!<E?u{!w-m$Js?VOxTtZ&!U?ue&OxWi5v%
z=k(^V=p|n@zMhyZ&>tC}$~19pd)b!_UalFu@4l~hd7>SEc(bl5-?ejoE^8c@WN2&k
z{pzv^=@8rfBO=s&S`Slpsn3NQLM=-_o>};Gip<;~ciT&X{joQb!%r=;-p;y7=kLq@
zJi(jMsh?Bzt-Di_c$y_LxpOD?8=m<!<&9^6k@@Kvf~s2k-@d&tCC${mGgVh}>fPwH
z=?y#ooZa33NczE_W!VQL^QWflcs8Fy-TrrAcfHRei}g02bH(`PmTvifd6!ONu)+%+
zgXNQi3mHA;JQCi%qrq_A^%Sn|$I|lGnl@MJRox4I^3|l$?qg&kr=zh`@x0Hkcy?K}
zOsa6zipuCo=LwuEfAX4XUT*87)sy<{I$!u`S-CGivP9<kEp6L(le{9cmQDAYyDZeb
zXpYd-(z#ms$5p<*y#Bpprd?|78yD7)f77@Qcp83Pk!x;z`J&CLC2K0r9_nPOYZLYn
z4_tC_$*T~ygIcdPs<F-%|B_I-%{0Spve$tNkv6v`{haKmee?qV$~_Ev_Kr&vL@zgf
ztw|`B@SmQ4Vng7T(4?&|B5IdLZCa~;V|DzODvMKvTjusgTo87f)s<GZdHGw3V^(3C
z{y(T*v$@tL@>I-g;akVdQeqe8_RZ0c-cq7@b6;#(?1xF4XB$RNa>zbY*eo{FF7;DJ
z!F+z5`O$Y@hJDX`yZzl|9aht|j_P|nIrA&t_r~3O5xwXBREGT@`|dNj>#oWwc*QnT
zMg5ZmyUn%E*<uq+1d=RwXw7a(RXwL{yGiBojphRu1<bY=9!*;|SD`6_<H=)-JR7@p
z4~1L~>KuRK(f5R<Wxk0zn<h(~QBH~8nZqRlM`GHKxTLt&2~@OPDC<&R<Z$dyU6O}W
z+OZh{$K+1^xWjgQPu{5mehj<Zm$vzw*z%$3*pU;5*iUX+cl%WT&IN0DF1m9aFW5P4
z^@>~UZ|_W~zsC1}<r9P88|Tiv5#W?pI;+t1{~N#kk=Lysm<19w1m5~FtxC1%YPLV~
zC@LoHSJ*PGBz7TpM%}1mg0dS^Vvd|QS<*S-upHZ2u?fzaYWa_Nc-OA;Vs`$>Y#Q)b
zLg3<)RqbUSF$~&I&MdvmroBe!$d@Nizqt5b@p>yB__liwhl*Nu56iQ*B>!s(z1O<@
zd!H%q(>kVJc3wN9|9aWmzOuE~tKKHJ^h!^C)F1b{V$~6ynl-=Ho&U9M{lpn(^;J5V
zl@7};=apk;zV++LEs4iGo6o*}(L8yQx3P<+rcJV$p7XRhUO`Kq-O0$j8@2de-DAa9
zYws~VFl%@md4${IMo!y{bxM`Z{~KSuRa~aZ_(7S;;ZmzxkP4@v%5qiCMGg1EKK)<f
zz>s)JHTaTB`X}Z8@71<Hu+|BfeMwo=l;7&wEA{@z4Gj7h&ECxrP|@Ud*NiN9;>CB!
z=D1@<0b6LEQ_cp(f1V30jyNUwJ*)ZfEQ#Hrpzm105}n$<=aUW{yKqTzZ_<l9MimZT
zFQ@paEa2hyKIOe&ozE?$WxE2Z*SzE2n7888`zaPnmoF1gTE-y1*~ft2HQ>|pdIe)X
z<;{F|`IPRxYcMOad*a9N;vD~SR|X{p3%O0^2`^fGHk$-~HVHL0jSM!Om|&V{%(+cr
z&6%$CR)r#le3l9tk6tbOsKcLM6)fU;(fsqV|L@J#g<N^HN<!E2tyjC<drRm34Q~A6
z{_0&`Z{r@gehtd+j_S9LU%!bX?ORd*w_}^1FPr%N)Ma<0V`?RDzu(zx>v>H2py9R!
zAKX0Bwk^otcFaj;_XU}Y7t=QN-`tY#`=llP#zzAl5l8OF>HG9~`kQVqV0Q9szP-qM
z%9Zz1)tVLVuK%0G`EQN(zXNJ6x<xK~nap45^MB`e{Vbs5^jRZL*d*S>#NC)FPM9%6
zL?B*RCB8VS=!;vW31iL{H(`cIfdr4nDfwMrOqVh!>If9d>1j3z@h>*<b}6v-`@pjQ
zabZxwysU)>z231#f0&n2@HFoNTcH5EwT`c=&yj`+t~U<+Kk^{tXvYQDYm>I#5_&hw
zz+YCYyyt_CCo?bO*4H(sZhR=J+V<giT9(h3M<*CQS06i`airii%T*r!%dbOTdTrAS
z`1OSMDf=0v;xjKctUPr_Pu9p-{?R*!%dL-mr)TAw$Of6nHz_TAs;+WCIiX27`=D@j
zv6rW^1$XZ!g-4%^^6jp1t)C)n^MFfi?&S7qp&RB*?VRMnG&@6lWvKY-7g4^v+V5Mx
zwiI5CS}W13a_!6s$wJ+}s-~nRhJ9;3+FGhf-<{gu@K)oEh>UgXjSG*K{`o8;*?IGd
z;Up$!h6jZ^xEA|8*_iOU@Xef@+n+Ahh=lE&v?$R!*(Bnm0=tpIMc!#T!vF7@d{ckY
zo7}IY^paOeDo9EGkX=^@!$1DB5nHmIg5Bn_DrIeP4gabbb)|LQ(NOPgP6f)GENUrw
z0wSH0?Yg$UKewj&TNmTm=EoC?G-rPEo;z*c9+9VCI@wlF^7)_ng=Lkt`xEWlb=lD@
zN55^+QD4QOdde@7N7dw2#RNUQho4@YaMD}8BxrF@Na5?_M^cvVNY|gKyY%0uR|hVv
zV4J-B%Mz}yg?0v^3@aR73oTtC!y2UWxlnWSO1;T>k9pgUoKCIpEb=@YGBrtA^}vy$
z7iQJQpTi18Ro_3d5?~4ZuKuO1wEy$kgz4*MOpTm>`ST*-PCxZ>e($(N9vim=TkpJj
zwPpK%iT%OfPJFJ&?T<M<SyI|9$&1_WEr-@Ck1Eq=Kh~7n^{@Z=rTFKe!mU1P_TN9>
zD0@BO-{tL0A_n)EcIYW5+?>8c?D6k=-_uv|7F?VVta3$L!^D|+)2>BryOzD4`hefr
zby`a1HEp+Pp_7wR9y6;r25b6Ea<RRpm@1~^H?2YYMypSmhw7&W#YK?<7My!Bg^xVW
z`@;6&mK)1v6Ys7|O-lK#lP<Ar4>_1p!Maz(N72XUXo;4lkHfl4KD7}_m!|Biy5M`)
z#IJpVU$?v3nQJ-cb^L9;{L9bz`)dX8f36H#<rC6XB``H#{3`eQb6n3|432*A7Y#n8
z7RswE+8esaU%~Z%-4pk>cBOnrb3zs_R-SpRZ|Wfv=^vqY-NH5>>0fv_MEOAKF^9NE
zStX8vqNe7h>$}!=9F0i)D5_dnmRcB{SAKaGNBP^HbMrdHB@c;<zrM0<YjpGUivGZg
z>4B1yr(e4?wen_A<@F4khnr*Td8O-prR(3RzxXO`c{J|0dHm_}_^*$2TPzJ<r!aGT
zlbvcQ_cHLcjPsjMhP=lkdf$s1Yivo1JUqo@OJ<a2@_M!6%ENumV&)IthL$JaOPJPF
z?QEV?(tmencAQB6!PUWUoExV$H6L^{>s?#$t}R@AW~-=gn3YE+=Pj0*+cf82%gm0@
zT0Tu{?KUlUckP{%w6{;|*t_ljvtyG)4sGK&S;qO+B+v8k;>k*?N2XQGyYb?|w_L|v
zdehexMEg{QWa*!6$xrRdXKW6f>{F<^W!B~;b^INM!mLH|;vueE75>jOG5Ws3Hb}Yo
zv4+v-`)QwJ0zAT-zcKqy&%4dJK56^9xWaXRy2`S}H@Nth3%-uEnz?b)i}L1;)`5~X
zkxHDGCvVPLAM<k7X0z4buNiN>VqA6K_{TbPnThY$&+NBb{Jvt!`mK3Sbn0f-oG+aC
z<WUXdG`}0+j^-;3Uwo`x?d$md;!^FF<;vgWQ(ro>Rv4`~ot4$u=r<+%(+m?o5p#y0
z9*&!ulD}j}nJ|8sVQw=c_`}ZW95a&V`*Hu6`G5Ae*BKA<m||0z%ui$-U1=dXp>UlE
zYjOqa(%Y<uLy8M3*f!4gaeUrhY|vq!;~UTJ7VxH{kezFKTj%s+ZVF!h#eTZhtzG?0
z`wGu=9dzMr$XT{-2DfBr;6s+>C*5Z;m(SwezWnqI1M$sMpFA;akK&uT>R@+O(Zr~g
z9|a~@x0)|;V7|g6&R;2F_<ohGp_lDPMJE~WsB)I6i%Pdz9lfhHDR$zd<cpsTc%!DD
zxV&QP7bo2*n@z1fD~}f*o#N^-_4w7PucczIub5i8UOngL^(i(d3oUQ_II*=NeY$hi
zR*q5&soOPAlXQQ5jlW%<u(!rw%1r+^7v<LXoT;rcQ2m;r$NB$&Z|0pny$ZjtD=hUs
z=f8WJl~~i+%B_b_&wG2L|D@&I$yet1UzslU(>!Lj{j?H?Uo$gQV^*{L%9v<$Y5r`@
zxtlJD&RQ7UmnFaJ=?0-}`!m`Hh02Z;El%?Bx>Clu-et+<q9s=+dAL|Dy=!(Yc-PYB
zVTX3Bzsjgumg2Q+*Ny!D)@M~^c!j?!3Yiy52Cq;T?t8vrh4kK=*4tMuWxb^x6!K%S
z!Zy`#`@)F1caQM*K7OZF_W0D%xyEabR&}1370Z+DyheEK)vB*MKHrTzyS6c=(p!7;
z1oLUFckX@NtG0q^>$2@y9MwN}nEwn|_JIBOPnLVt&1WCvd!LrvT~qB{oA0)%w(Wm{
zpvf(X!}=z2j?cf(oUHBNBYEJm%#GV=NymGg1LtLy-TwD%@x6aGjk22*f4kXrg*<b=
z*6e(3p0dPw)l1Wr7|ewhSfpv}$X(dt$-LU){KdUaJLPS6rnk<Ga$Rqgcgee;E%d`H
z&Xk4u7Q0n*d=zKxc4v3C@agFMq1>~h?3|q6e6GD4pSazaty6q+?|GYj=Ogk~ZdktV
zLqPi)?ypVz|K84DY55^D_V_e5UiZ44A~pxVA1{=xIis<8_Vl*m?RRsHHh-F3wbF3E
z((N2>DVA2Z?GxF37~Jm2O-VXZ?{s_ZtlQB)KDL#$PT9ORqNXEmPvmh{R#xv$7KO>W
zOD?b2{C(}z|5plPj<*+HeQ@`<Hunj^W%p)JU$=DQ&*VcVPK&Pls8OExGj{47ts{$m
z?h&+lWIVk;Kt|loJ?YoOucoKJFVbgWn0eE1=KehiPmi4W%ra}T(i3Mnr8|KiuJ5_+
z(dl<<&t!%sVR6P-3r>X$frkh6SXifcp7@Z^)Xc;Xu%+U{BbQF5296mso+Kab7O~A!
z@eEwz-ls0Oq~_(Pr42Jg)RWd&8mkL+>BYXA^CQ?nNHb`gN9NI0qAR1;=3L#ZYP_nU
zENJzz98uBLK5O+iR*8O?y81>G(~L=`(c7*@hGl1O^yLy?d1pt-)KjXd;_L72Vfymz
zuQyjgd|0EWpq)=wFGE<jyW#(|#}y(eL6h7A)sibko}Hd;G5O{fOOqGAOm5TdYA2Mu
zYEId9BImD_)|;yfVz2AnwfZ7<cUR$Uu?JCUAEqA;7TexsW9hp1Ot;m1qtwGwbT6;=
zUVd(A)wj37JMzU10@5m;J~^$v+^KLuwqELvl0<ufy5B!P$!;p|sIRN*N<H)YKYv2Q
zsa6%?Mt#B15bd(9!KZ`$IVGmr3Y>De=9oV9s6yLTgA)(iq_-Vf*rd?qkz}nhuVYcT
zO4HVe?w-9nBBb><eNvJ(*>uP)$wKW|a-ZEd4=zbZKFySgj$8pMle{KveB!t`;7iJs
z04{~psli?^pG=9&^ISSDHj4Av^w>#erzHNTPV;;wF}p{xVOGw*o6lwy?%TO+&isv)
z6J{4(TB+t*9dspqcEhS|N5zCa3=fKQ>aEIH++%lZMad*Lb5{1LW$Fy<4{{g2beLOo
z@Ss>skC&GC>qho@D-W!lm02j$V=hoNhmZYr3%hE`1qW8cE!s}Iiw<e8v-vCT=w))!
z<+981N##MI=7lV8z14lYgSR-ywmb+dFiiDsO}!Z<8vdlyH8A4orWK;W2`hiUZOz*n
z8W!|X>FKqYZ)&aL`^xWLogVa8?AiMG*1uaL5?JM3Bo4J%sxhSWXNE;4<TS7urY+2L
z`*>7bdHF4G+2vmhkJtUuyp_JAH|<m2l2RL)(}A+(0%!iO5!!y|wDED9;)Rzz4UZOE
zJG;wXuqj_2d-0y2to*8-%hKnZeX!4j)iv;R%e_k<x3bG!`*d~r{5tlp-%acN`+du9
zp72z34E&k2bzyKoq{qeE-}dNEZ3(-n>tK7b^0axV#Iv`FQF5z}Dm>(>%h=JRrlt4j
zk<uw`@i0%%bqji?UhnvsWR@j&qtE2o56cObT$Wb-ZbmaxUWpibs=l68$f-8Z-%8VZ
z%BK5#)o*uR_Is9@(dU^ue?Px|&4R{0nWD;yQ&%=-9XEe3Rod?4wRiC~^<9k&ooU-^
zOJ<jy%3L+SjceYrr|aGKRe7z?zj|ewmuzUwGyUGx4%_*@2>j!;6)|)EpYZs(eV0Ur
zl38b0V^8a3CM|Cj$4D{nmPrRD%CxARU*y5;x=3)M&V&XPMowl{50S+a7~>NPZ#({J
zlPT==ZDmhX+;#5jRIxA*-KNzKcR%{NF869``}a-P<LXXKZ{!R#aI*0^DDrtjdtjQ8
zz;2&3oi@YsYtE!BGd!C$ValG_8`Ziu{p`(62$sHgC#`4m$(ffk(#$@IoU%Nd$yc&2
zSoT#!mo<~aEZO~=x0p=Q%?fnpubOm>)o4e0_LrBp>i$ii^j|Tkzb9u&eU)0HZ^$}L
zzEAx3f{Hx5Qi_~JFGaW8y<EsAsprz;JyFx&(o*bT##Ftci9IIe%JOetxNngOdg=Gy
zeTnkIb)LJA_)Om<vMeF)<@6(SVh#7E_-KFi^V;hjJmrjt&2!FE724LmbMH)%vU-|1
z>+I#UIooEcXZ|`Fa%uNG%hlnwFIBvP=bpOqJt$Qxs52{Ub=SRBSK2q*WPMb9#ys`+
zBxAYNZ}=m)9v*S#Dhg;Wd!(rA$m!E|^#oU%i;|?tdEdNE6O$CvRRt9SZn@0R&%634
zH)3Je_LvtZ*6q0v6Bjr=DvB$3$Ifou&$F&a-`I9IYK^X5_Uo5Pe_Wj!S$Pa58A}`#
zI%W_Su`JO$^--Eu^2Q!d(^dPnC}r80ZPJc2?LK>Ym8rj)RCe3e)jwSpOST4Y-o%%_
zHr++L;)aEZ&SDPUj0=iVC7YV(S}XB}GWxEpTo&fu6_Q)~F4DjHkNCWGcbD6DiA3LE
z5fETd{K>*9!obX+!@$76(7@!t$iVTRfqjRRLqURL6LXZJiAKS~!_6Fm(<B4579C+s
zF_?7HBa30BnV54|kHx7Z&!)vjN^c}4P0_91rnBjiN!wk9*~}4&S3D=AA1UQ%6k^$x
zcXbD&Yd>E^X;Fhli$AAZX;(?g)rF3(TzgNIKDoB`)r56I<_>pPEldv=jrA(iJhWLb
zRM13vlg7>U%<tG_PhZWuzb9$qD_8A2(b#<~M{RX~mSiQb3R>8eW3@Bu^6C7oX-7}}
ze6=dnrgoe2jXqzOX@a$VCbAbxw8PdUeowJIu=UNHZN=(S)&`Wnzq^f7OXM6^%IqLv
z^+^w>6#iuJW8UGzV_AFPh~Bk{#ahjC6h3v`*DU(K+~eNvtDj#uH~IMAef5h`L0hWb
zziwaUx98Q9lU%#b{`>d;=X1gADI1S-YRXRC!YX}OFy(Z?;tr?N48ma^imk6NOj2yy
z(!eH?;XYO55re8|M|{<c){?O3nY9sDBlV8G5Q{9DoAJ2U;#o(0ly~Q*uKvJ-qBHw6
zCS|C2aTrcpGEsRN%cGz*Q(g#bFXm0~p1QL4(!|U^l}Ccze?L_TY8IT>Ibr6{-j&Px
zPES<4J3G}Q{qCHToguPS7ksCvSElX=^pZ}MNxxgzurEE(^^0Q5iuo)%4IAe7IsJOE
zXsOGu39dYbA`_QQpLTPCOL>%wo8$_?ohh9uOG1QZgiQ?cwq0`d|D?+riythxyf*1k
zmt34nscA!yQ`Wc1D~}{yeZ4VMlvN~_h4ZCOWa8PWqFa<&MXzm2T_`TL_0ly~iRk2m
zN0ma=qNXnL>{eV7vT0vgn9e5klUlDUx7<9j)aS>yEtmJLe7a;-L3-ECgyvk4=>oN-
z+fK@qaG0h)FT0kjUMMM?|6R8HaijaZV;$4Nb2h$jt=X`cWv_eDqJAZnA`h*|sT1d{
zeY-n##$nZrCu(93dp>98Zs00#S?sE*=p-x@@}*U3)1-%1Uh02UG8CsuUkov`njN|M
z{x**bI!>}{6T`wLi|RO;*5!P=`F!cq<o<}|=4MqhrdBNRNVk+s^)GwQ`&47if&Zp#
z+J>!HxbBN>6<X%5&A4XrLuU3rpAYt%%k6wTAzZKWDTm?l*4cT-JOk#OEEEhZT@v3C
z_ws>bROTzuuP3V(PdVf#xOlZOSNw6o&{;+<^S{l@Tt5BZJ6Vsb69od@Qm1Sbyrm*r
zKg(m?En|nNTYJpBe9d3Fa9iK%uKcq9ZBc1RQ0Q9gf9nsmo&U3L_Qu<<6B;{@By9=1
z=x^Ws_*rOjM2K?B{{OWbZ*7*DJma5huI6J4Po~-BQ%WB$&CUBLR=4jES6Ppfu+_}#
z49|{Eu2MYGthkU@yW;TV>lf6__9-TwIa0Dg<W`#9vCP&YgF`YwJ6>8|)V%xL$6WnQ
zfR5$d8GU8{k1)^csa%un5hGuBX2FWP7u^;6X7xLTwQc%Q)5D|ilehlQX4fW$72TGf
zZZd3Z*m*$lU54UMsrCuSO!ys7))ZTZ|G#8v+8x20?Vu^$b7zz2#4|||5jP)49@?~s
zaf)wehl_N7MT@DN=UKO+?j;kSh)r7Dup-~<aEi1s=T_Comy?+%+|y8SSrpyaX}{u+
z|8%}GOMlyS5gD82tS~ulX>qXAZwF843Z5yJuMJL>HvbA<(DHJ*lde{(`sOPq%OZ?Y
ztS$%o@Nh2wVC;L}^mFmqz<FkSJ9Z>(R0(DA6ty^dV^Ww0zv3pT7~gH?3*YUW+-l}@
zjrsER<1Xv?Baa2eZ~dnEUpG5d+mPW%|KD5b#}9QJ_GDMRxI<&7%8>>7O1x*hY|pdp
zi&0GBd2vT!O;nTe<vXj@7p~;BS-xem;5)bY$+Km6{<jE9Y2{d+x+&u*aqaZXx;|mM
zmK*%KVQmU+&zHMBxpC}G=HkhcUMi=T^q&kl7BbEB&B}*w)Vyq&7OX5=Y2V*-;+)MT
z<8@3t-iI&B`MEtZ?N;ekn6@YSv(Hxv7AMX6{NTU|0-uhrEq%Mr?e_&8j>@glk{2dS
zQe1WOorGiXT<^T(%eU8jxVAnhYKmA5^VxKvrlp@-UV2}Mj#;TE?tL!k+~)N>oVw=a
z-WMX>{gz&SIA_Mrs6Q`GPJi(7)vG?|-KqayoePro&Z_h&i&pnCu9zYm_ukAjMovv%
zd1-{)tsIL<%cf+hK06}Qc_cx*sUSaom(_Wj>sMp^|G92Ro1GrR#C0R@@8><tU(;RQ
zg)jc!>LUBwTmG-lBe52z0v)cNjtNRF`${=8wF~zOdal({Gs@af)%EyUr;Ws%$>*LO
z+)?=<yW_F+#913!x39ij9I|PHj8wS_k8*5B=`GowM0U%$3X8f97t2@6eYN>Kk^fZ6
zoimz7qFo%5-+4q`+tmN$N%!A}YBN|wQe+i53S}ARu$4C*+<1>ePtQtWl9Q49iHMzR
z8v-v?{5JRabET9a-c32KrNm>u7u)FvzMp$fD0!;;Ui$CR(t2y{jBUdFA5$jh&It(f
zOtesrou<0t&&H6_m>uUg75Q3b31>dOvh(t_mlt>Tm@Fyhd=%%mr1a<?VZ-d>mps0T
zl^u0?wtW7}=qZBJ%MY1OEt*t5@yW)POXg0!tBz=<tj<})`+Sa^W0oVAVU9<9#*^@>
zSCRgLE3@;m{K~GKIF$2=tK{Z|f3H10nP1$pk+tc5mGoBKL$NN8UfsTN_GE<lu9Uue
z94%^VS9qott-k)^iS4|(=XuQ6>|AkA!2QC^h$CgQ=5+ooNp_T7b!>|5q@|m#t?+R5
zFuFQ<#k99SY}aTSd8JKDS+%KgVf5J_Uc0h_r-zhP8Lxa$`ERL|ca)?(%YPZ6&<%fP
zEZbtULA`zFsq4G9t~@i{Lv#P9Q>lIhAEuqo%JZ8T)o=1;nS*Yo_hq@c)-D1)Rvc3{
zFkPO=T@snNd(#EYsj1f-t8Yaa{qnVR$<2-qla;^Z`DRt>c^9iqyKZjm+cs-m?ww%s
zm6KlfbqOwf)NQCbDeKqX9YT)x=6Wz@E30SD(Na`*G82qlaMy)<@r(<RT2)FGW!9d7
z5A&INbKlHa-|ZT#cskIjK4;6k+*lSF_YjfKBG;a~eqNMe8+ziC$iKPb^_J_Er(V?3
zE8V%L;ojSIp?dS*<cEvjag#dhrnx*pY3hfca&rt?7WOB;yj@|-{n+VSqFLFBF0<lA
zi*Nk*ogDn=YMHyu1xLQyjOR9feV}JsmA}gMTb|ah`#ZLtv*vfdxA`c;&v^Bi7OC^4
za(?N82}#^fkMXTtz&Tl5=frgV-PeoLgH!t((;lei3a!X_nUv%3T&!TBNzIek{A5MZ
zLwvV``M(74e-%*W5EFcUL^$e@`Er$vRZF}WFGqG<^E#t0wDpSRHWsU-<57X8nLAs`
zeweARRn}Oy*s5VtW}r&eJh9gESE7S7H2aJMB5nvoRtQ9A_&%R#v#K=4`bj~?WFe1D
zTFqikS2jCJAK{v?T$r&<UeHu>k$|FVP<us2d*zPyvWkG37wzk}xBDH87jPAISFGZn
z+94<!c=lRO&3}n%KgQ0D>TDAws;5<SR!CG=?dYto=nPWqoFJjtE7&#DT`8nkazce1
z>nG)d$5rwQJMO9FZE@h=Y|*{`68G1J?!Ozl_q^c$(qNloq5thc;Mwv#v1458FDk4O
z3<^%I)k-$_^ekx3g4)vuSr<7AEJ-!{b1Ya?x$c>I!2^vD%cS~@&E?$JHMovDFL+e@
z!_nBf(Bi`pA&Ko3Uw+syRj9UZZdlA9rM@t%d~<)1uu5o}-ttA9s!0>(E#b5*kI48O
zE+OHrvS@;z;zV&3E-ufB5|*MxPhu^XPrO&EC%K~9(7ZTnsiN9WzAp~^e<w6)-JGO-
zlmF`j{;v#^jUp!-OHMZb|B}DvGM{0ho6eQ=yDa^8U4k5TmRe?HEWFCL_-XS(VXG69
zm6v_ZaMNh9E)&dAFE4mr)+AJ3dVJ~@=jhNUKG&bfP2W`h{-D4vQ9;!YQ|52>^*R#6
z#^HA~LvZoI$t;FaZyY+8vQG?7>)^W5u2Co?=+xOP*_pIp!Xk&x$r&>`R?g_moH1eN
zjFuT)i!xNY1-l+O%ve$}bNb5}^E^e{I|XJrbS|7ZW3eSe`_38uA7`%mIb-e087nwv
zPEec~FtKv!jp-~4r@s*tUEU!SIMLzdr}*Nq?k@q{TUJc^yP*5z$?m-ey3ehgbHQ@*
zNlxv*4g&uggg!Y5e0?xEjL}Izqt^as&5{3tmRe5|)*c8FcI>^p!}9gXWGkayv#t4`
zIt_1rbbdZPX3^q0rs;-^<w|LqLjPwr>OY%j!z`aD)WG;c)!~?{+cr7QrUtQv4JI4=
zb(R-vg?EXUOpvUaD0Xw9_{>={Qi~Q%;8fOPQ_)%^bhC)#=0ZWq3A(=)O>CLSesiHp
z)MByBMa&a8Emkd_IbpGN)*@x8C1zcV<f4`+MopY$z##sz$op5CLZswbVe=E$Vk@U9
z>inFfRXN#Y<<d)>ljBY;HLzsZV6-erO7ME8fOP4yK31>vL+QcOyjTvZE`MU2Cgh#A
zIKx)MYH4cA%g*{^Uz=SyGB=9(9J|so=Na2N=JHb^)&c)L<WD_u3A-VnFtL@HF@O3c
zg+rp=oXN6l#6nxP+ihN^<uR>|*I4GNvi_R~C0jPFTJ5E@<<YA0741n3o$a2p7FVur
zU&*?C)#^Q6t9Soez4g`VZC<lBdal_TwdO?Dn&m6k9K5y0^VjM<vpAy^)?Tq%do611
zrB$4}q-GuJnsw;c+GDTQ9+@@!gp|;(O1Z~Uv;R74XDt*HP3b;5v-{JlIbWjIe|xob
zPscKspt)Bq8TKmVcz@Gpo*{Q{XWq5%HQmg11}e^%R@Cxk_g-J&^7EVX*PXI0%j-l|
zJ3DRN__tH}$&2|9z8cy52)V?V!C9CRSG4K;QsHQ)RV*7r&$U`e1}bgaP$T$%;*#ve
z3k|v^NOvvqI<?4T*6b@;i!OU@v5DH^e0#A?)*|~`ix=NqEO>CSrPU())muV-ZwcPL
z#q0JKyRKPQRt&D(+Y+QXqYSo139v>vY`f&OEpo!PYh2shS8t8c-pciIOL+D6u<Y$s
zzc+h(O)Lvhnx8o7O#t_sh9>=)lm9AA-o0Xn#m}YDUXyP{E_*(4*_7<L$y$@A&ECn}
zuv774p;%$z3d4k-n<A|xqTDoP&TM40d(oWu-O}i(<?^FZ8@#QqOH5s%D!celZEL!E
zjZ%5u_uaYX(e0;Ju$k9)2-TkrSvhIfp8lXcViRq$I|Z(LFfDR!oBzu&VUf@|)#-ZL
zGN#f($^QjdlON1lX1RK6)V_VY*52B+FZJ~5q~F`Fty-I~dhI1Ifv>mseUV=KYxe#>
z()<6-Ui*Lde#V^rsk2$VvJYII#X2$J0N0rV+;cd2c@FZ+9OUR(D=Kr~@9M>0Rxfe&
z-oJCzL9eQH%m!?~SLnR2nzNZ>@`cLvXHL$!Bsuq&)XsmqYRsyY{+^OE&k>sXRclgr
zBzKwGmFe2d&b92bU3sE=Z!fRCII;e|_<Tjdjn_HmiwM_=cN^aEn180l<xiwbj7^Aw
z^|TY#^@^H8g122bsx1C*U+~{>sd8Y$iiv`=1Gt|q6HrZ@u)S)r{I7-TTua<<Z@YAB
zTh8lk8Q$BPYqmAjoM`De(e{7NiAJ7n4Qp8YV@^z5bD}lo<dm9ikpisD0<1IdoSgIL
z<h(ho0RpEM{5jcOb86b16K_{95nDK0@RPF7Me`=>?zYoQ&3^9aoqf7TdYMI~a*EW>
z{W?1j`7E36eP*Wg<Xb0qUay?H^@7N}?jWNKFRoc$t5qzPW{YkN;aW0vxx<MJSC6S{
zT+6DgcGp-<HQS^vYv|*_5&fE{HO{kja#Qrjm~u^H0WL#<@D~D+9m$iUEc=XP@-8UE
zB-ls<GA#-^<|VS~vB|3QkN1}E%vN4=^3(0rKcX+b*WPA1>!O!6!^8;}W%nMCUwc4C
zmsS4kMH$^o3U?34>K^2qvsh%#C4;q>jASpH{Fh}lm1Q;Sy==Ajvd!JgcCwsiwwE2}
zUSgiW`fc{?&X>xnHHVI6&e?Hej*iULb2pcM*|nZ|!eNfat1K7S@1AirZtqpG#;bp%
zu7&U2Q5>9iw_AD9?4A=APPucnyBCM>TKB%@+*ma$O@ukPC4Hm4OU3+Gy<JLm;>I^*
zPs_<?o3*d)U3-j^@05^9aB1+>h$vHr=&zf+o&45*C@fjSn!0*h-tKJ^-fYWVeKN!Q
zR{HLfD|~OQ+<R+D@8#J!SJuoux&G|Q#Xh$-{XMx^_te~&+q+^<?R|T4CogND!6}ag
zCl}e=2@JRsV8C$5_SEb<cTVu$JvEp0^4>eHxpxH_mpBORkTlv6{(sie9_f3RH1F-u
zxhEEQINbNztkq{Sy9DNF?|ifO-qhVYKg-_#qI>^ZW@%1`{Av@`8NE{?mY2pZ-<AKU
z`I^VMP0_ozzOLO+oqtbfs&A*a`lqPIN#~+}PMyM3-gPST#f^uFoX32Y?opbjk$2&|
z(qVzafmSTFHl5<j(<bi?agSHN$YmF)`Y?K%WbGB#y_Yue9-PH<#lG({*PqLR8&2BD
zGTQP!v3q;j<J{$m7oNE1J+bC}DyVQLFyN_G?$Zc9R)Gt5V$a<XXkZCA@bt9rv*deE
z*XiB~+`V>@!=blXOSeVvuiw;t>gSx!-QD_kmR9;bzalBPLqjMk_v*haWpkMqe}AoC
z_P^;x!rm8#J!T9U^HfXDu1}QrjJbZH@xn{S`8O<r1Kt<~^=xd(m0iJTXz@f=!udw*
zoum}bM?A-0ExlH}VVTg3udef}7c37v`*LHbyyNn@-b%~Y-a62Ci)G=-7l~Kgd~apR
z-95SY&C%F9o({K;?mc<x?3>eicP`(1bEWR>&3$ig+<SY|j=_WBonqpf)-?y$%S7Hz
zp3|N^*>vU(i<9pxCErI!O`bjbUPA2LEGwa`UGLxBeSca<=x3c0<NW)*d>`24h5q<`
zc)uc>U)SsAQPsr?%L}rat!Ak8J9FKh^YQMU2Md|cZT$VwyySY-sVRNmnxFNk>rOq_
z!R#~LZH2eQ`JSj&ga7*Hr(J6Oc1Pf6&H1l220zxc9(`)Sk!W)&S#U|8$D}0{ii{RJ
zzd2gmy{U2cq{F<EYh~Z8>lJYI69~6wh>U*{lm9j9|JMk9*3h`8iTA&z>VHeM`xev3
zl6dc1&iQZI_n#K=e^2@Un4@O3%9(XO$xX*AyF+}Rf8)9us5|$i><h7lFTTzD(QbDw
zMpm%(-_<Vu7n9_L+U$OI?Eevech1c7KYHTk>MHAR_;lvjj~8|i^Z1tO8(i6NE#kV8
ze}Tc#>!Ejg`@Zc7?Rh2pyYAGq8_6@v*QJUsF>?K2u|Ye#L3v@})@7k(R$R-^%58|f
z!jX9Aq};npc5fcW{e9#oaJKI6qrAU2{_p?Gq4@U}|G!K8tdI4do$(booqH;hVF}NQ
zE3%W`wPv4wZ}<O$-@SMHmU8XC`eN;epZD%D7&<&~sAuN=wc31vyi+3&yM@k;2?>YV
zMU=y)WCSR?HHz7qZLu&^KHe?h?6oH%dx3868jp=a7F^F7rZcBs%Q0n0J2!`^I_r;Q
ziGrkF$~FPlQ%iZomwB^V$ufj$O<x|(B2?>Tsuj8_j9uaGu8=iBo081u)kdl5o?Vga
zwY4_-)Z2!jyeHT8W?x@>Zx{0&jx)EvzmIQhVc#WnA&nuav4>qtjN@lP(w6x7d<ue7
zebTn3uuNJd6ZPfl!ujkyQXEpc+S8<OdUP(zOjx49{C^YAirVVOZ*MJH=pyjL+TsJF
z2b)qS11nRJ2J<ZK^nErzi#RSUm(I5{@S37Cb62+@Yu%L<p{p}4TF0-R_p9!2)z$3#
z*$aPv`}+8BdH?^ve}Az3=B(j&=&F+BUHhNw$_s~9wx9sV{}L?)i`kT80u<OaYc4pk
zaPJ9lZoc+9gQ41NO;0bQj|j)nnLJsk%jVcL_XrzVBzAF2&3u%?u=d=kz%8=%G8yj8
zTNW9v3Oe;*Nssbmm7~U6*|drnBpHJ@c?GTN@jlsjkLirU$qSQo&K&np*?eY&i?*}r
zj2ji5B4*cfgk8<ECw<<+WbV0^_d-Z^t9oP3$xad7GcsSy?lu+Y{a^5=%le$pmU*X@
zxO9yKohKWbNcM})Jd?a!uBvL`^7&N)LZXVn?x4e}19eTWvg*$XIM}4ON5i={OO<1B
zp!}n)3no>_zPz4LeYSMLgo2}`i&Glc=0v93%ako)y!EqKA)YPE)pxF4@ygp~nMEg;
zGyeILdHd8Pr|mkMRy3{RUc2DZN1gQ>es!&`*uBf^?#6GSug)+{7SV9udQVD1Z^ySo
z-unALMeW$o!ut4<UbDbyPQ#-baXm&ovY(AN73ymzHcT*<-f41L=I#WZD`9EgTTgjx
zOg+LF64WVrGe>x>`CBbzzs2|F74tPQwus%@d28EgrAhC$TwvK_=vFyPv}}oA{{OvI
z-&UWO|GjOSzumtVk8e44Y5iWG!*zCE!Qzc)#eDZMuCoi+*YsrF@##Ar%bj@3$Z_Pu
zo8BMaCOGd`wb6ddC9Gpp&>SMvQ8<M)s`&O?JF)M}CDPY?bX)eWcF*UA%PfCSskndl
zQCODd614V8e~PU4^~Z}&f7Z9Xo>68r|4X^Dk%;zPlQ|hhJ5_s|jt4Jz#BNb{zW0`f
z`Bq1E*Vh3r3&k|LUCnph`{l89f=RcRyC7>#*qO&$#HwC3>R$~M52_Chip;HlbJ)6M
ziX6-Qy9MEsYp(uZ$b5GCwY7FmLK6*Et1zc<{BA0plxBEA<&@I{2Hs0jjRC7wS12C!
zKa?t*o7(-~b<t1GpuSCdLVl{+>UU11vOSv0c*fh~qtVQLDH&oc57+U`^PCjr5ZtzN
zieplR<BZrkjU)TcJWbrTxaWAx!?y0s9s@Dy%R>6!Tu$AXU_8SzIBnl0>syj3{#w#<
z;(^U#GkVW9t-0XDGR17dZJ$#C6IaN|m>!b8UEv-yD`D2<6T$OJFEq)9CiCCfa&7B|
zN?rT-!%`0}Eb{m@v7@fdI3Po$a$5G$o}7OP?*4xgyLoOsj$-|}#J(nxzrI5#qTfg6
zVTk4a5T(4SiyKv%vJ{O^%w$Vd%L>ehQ&B#fd9Fl?aZ>-ePlxI^uJ-g1@}8{YEP5vL
zm}j78(CZ~DRj=iq^j`E|aPskK*InHozDw;2<e3xmNzI)1h|A+8m$$n)c0FFGT3O|E
zFWoAQJ8NSSk4M8iw|V9ax+#f4Q+Hj_&{6FQ&FK<bv%+<`o#?6`JJ&CKS*)=xKvF7R
z^mN18Jtyk5x`e`oyd!r0%5cqg<f@j*SiEOdJFDxIweg}UE?<03c*?i$kB&51Vs<oh
zi{7b=r+P9(EF&*klx|XAcqpa$k<+W4U7JD{<*aji_h`=E9Sm3F{{^j@vzcpO$4;k3
zZ?rv)U4%_Sr>mTQGeMX2abiNuM*}53wxqnj-ANBt8SYQfQGXyKE^Pcwyr=SwdY`1n
z36I6DMpN`E?p)!JaA$OTDtoPa+MLM$DHFwK+jKq3^w;jcvcir3>x$>LZ9KQHIjold
z({ak~nAPcqYitUK7xFsgyv}I*QnrJqGpk^W=-I|wmj%9UY>L{T+F_d%b~0$sQn9x)
zp4z{;*i~k5edmd*vz(%CoAbX7E4(x3oz~{==#xri&U+rGKFbn0W0YIET1G?Q)w(VB
zOca;8TM1oyag1wMrk8P9OqGn#-rb7%QcLz%T6KGS2-?ixk#N0TB>9o8XwQ?4smf=i
zPb?M6vJ2R{Ds)xS{gpaA`@dVd>`sZg=6LSY#Zv|Wk%~emep+Pj|F~9p*CI~C&3s9h
z9&9a9RnB^sbj$Ie*wHic-+s6krz-`syA?Dtx@_u^{ogT7so7<DZ}#)~Ptv6p-`X^7
zW#NbBjiyr;^=w!fWSSUy+M!i}a|t(Fq9XTdF6Uk!MUU9IiIF~Ko9?Xi(_E_E%zLwG
z`ic)n&o^>DI&>@Qh*F!|dheW`xUH5mkL4^$`Fr(I=)c#NR~tU2oRT>qaek7caaxt=
z+sOHM_$*(!ObW}MtRuWCFfI1=u1!~YX05ox=FnZbAxSowgI{j%;?B<*j9ke#LYvZ-
zM(+`2X{e6ousyYSQj@`SRmFAe#qveE&-yk^S;uhJc8zY#+!)<S*ICWWkM50)Ug~>w
zqpD1n%yL0r_V));jV7j4IW3M$>pPqMjH&8-^?RfHEB1P21_gFqvHmY@Sgf$Bo2&YD
z_tM`{pLZumm6(11E1}o+%v16ThvoUmbBoS#&VB6SJH`CaCd*XGLldN1+oVrTJa9BZ
zIe6Vlzu0}j{Y_sMsjt=8#9`E>qmWoVMe2o?g*2x|hEtfP-L~~F7HYP-u=Fp=_{9*`
z=>I#>yY8fxtN)I-U4auf`M(Qzw<T-h<T8(vBkA><TwQCD<_LQ~vrM(z5OA5Rif@Um
z(dh-V4LH`Bv0hbs_Df=?U7E-t{i`bjUH618th%~JNiIHX-5HLbraz4<mpz*9Sn=vq
zmgHuk9=+TT3u7*6UEkFbx2{xF>D7I!nO&+A*VQato%C+j#^!KSV-?|>V%*-7Jum-X
zcS+~=?qw%$a7B5EnS8j|p(MP?jH$P1%C-i(FSajF7Jbj%wXQeyLCosEVOboL)&%ff
zv8*=T@z<UE2Tz%8gd&%DP?qf07wozjo4Xn}D06vqFj+IGaB*ZD$`xGM8X3sp8nrHH
zLu+>iYy1b%oMP5;3%#!bg1axSeYbn9+p9U9PTLE1&u+ZX61IS2*2x{`3@0)NcS~qm
z$Vjg5yuIo~wj$q$uFDUXetV=^WhlG+mW<}aNmn|TT~D;Q(I}gDN#BU`(B>EG?-|S9
zVeA#P+`y!*KL0@H{Xm1qksEwg_e{PmHn)1V3iHya53IXBJ4HWmN^o3|w$VYtdG{>m
zZt>e1lf5?nXWTHqw1Y__q4jH_eBcefAH~krDN-K<nKe3iKTEQz2ryS&kP35P(r9RM
ztdTZ!XbR;pE0<vSt9@AK&xDSQ_D8F%gM#K?nPjOv*>EfKQRUw*970E#xp=vfCfZh-
zABg5Xb#qgVVb|rA6BP<q=@l5SQrO&k;;7MWj!#~jMWr~*tq)m9b2mLcYHY|Yb%{sj
z$wJvzik+U?yC%$NV^rGa)zk5C(#{2|U1l#*d*ya3n&(hP>yAqtcdEP2D)0tLIGq>V
zxB7sBw=*~YoN31zWfC^)Ufdz}d^5ZC2G+U~=I{;8s!S*9rtEuYWc}{;>_aan6l-tq
zI3V3z!Jl?npe&-*PO9~Py%WEM5UaD7%F55qWey$Hr<lq(mV_?g4K$FCFLtSyX#FSL
z>><(mwS$GDU}N2e+0iF<J+kIt*wYcj;gF-aFL(1ce(w{vSJ}0^S@HIk1Am|?`^E!h
zcQ!83n%I6=Tkl4b@R3uiU%A#+%Y3@EO0(c##mDUp&O+Z`xvsr2ag)!Es|)qBioE|E
z(pPM;xITNw2B&W6ARfKW$t7B=gjaGFEbK8&l+?V_@nog&{1v+yH?aPd77uK&oWrDg
zG^e#;3$u9vvwT*wbwO)%0jmS!;q=uVJ01J&SDAj|koS>P4&J$Q)@jRxNJIaJ8c`Cg
zh9_D)C7FMDZ~T0lZ`nap|C{Pj5v>1x7qT`tI9s~Re;dH@)$oW$1naaTyi;E|+jmWg
z((vQBW29rk5%!~<OKSStKa<yFw+B|uJ}NVN+vJtTTQ+W8Z22|ES2xC}&q_1wM$6+>
z(=P1b+}Av3(q`40+%padFFd?kt6Fh0gYTY-ja~6J0SPj^CzX#a(&)W+a?b__w~e=K
zwiIz+*nQz*x0_?~>fHyli;id?Y1n&B`eIM!qHB-O?-1$mIL&kVwvyFq?`;ulB&S}~
zvF3hc&0QwZ9PrVO^`v{C(7HO76RMkhj5V6|P8b_YG?-Vk)^BMJn{Z;r2fn3>yPQL2
z-MYeZXNulp6M6eXhwmlrzqQ6UP~zNn!;=~eR=Nv#|HsXi^mS-(aWPwevE^9@)2#=6
zhQIiuIt~ZEVE^6R<iDA}enI=Z$CsmKTn@Z&S%+h4U`MZE1Cy?YJnI}WnXf8=TB_b_
zmj)S$d*52}?9-Jn1<(BAt#5?Q-%MJ&$1${VgEVW2op<Whm(_L;vs!g$a0>YJhL%jJ
zxGY#DtY7_dj;M+7mq!hnHyBpW68!EoS7V{;x(i(5R}Wrz=qR7FL$5Q?Skg(w$Z6Ju
zp2J)s4-41q=iy~LY-Hr*di0FS6c;<e?v7n6S<kbaP}+Ouv<!pFTPEebKGqYsuNiQB
zP+ak$)7dg)W00d&>m-gSj`^Qf_`d64S=c=1?TMDAC{~|DpAV;*{vVf_y0m+WPe7+b
zieFITe1{g9&k^k<qE{R{Pex_*P4NzIT*2#mK`N@DN#_EKNp64C4L*AlE3-BJCUYdi
zc8G~JUTV2;<l~vCix;~HHH%I9;L@R?u;lE{{|lV|2cGR>nzC^<YuJftIf~6{6PeCC
zxK0f^T4pn`_s6utpQn8l_N;!hNxEm1p2P*l8%HmEaIMT-xXX0{n{2mY<Hb5pw|%k4
zR$N%Pt>D;>hGU9KF%F*t>{h!Kg(zJ&=E?6_<ZR7d62g6Gl|_u}8bc$UV^X|kKMpi{
ztT0SyuQut({@Ge9bE&?D`G^6tc|g$C4XnE>WS^bvys4z?7IGyxo4-mx!@P^-zhTAx
z{~4b1H;a6_(DI7G%(=C}XF+Sg;r^%-E&rPL2M6!hXkf6*J*lyvHSjgNTQ5u4jQ(F6
zCe#&N_7Rx+oKuwb<7LB!TXrrk374gQRbTdBE&A1Y#j{f?f42H@^k|+JQkgx6KQQ?O
zmrOiUn6%*Uivd|z1Fi1PD16ZL<q)^+BpYpJp&grr9yAGQF!O0}8EIV83DKOpWz+48
z6IZV>XR_s9_rz;TqfKLmjrcu=4S$wOTZS#&At<wE)mp1-TkjmyUa7xp!Ll0*!xYsX
zWv#tt^6<R*NAHqFoV~wK?-lVmuH<9g$r;ls*&Lu*5_+BSg;Vs4Gv%ds4eIV5{lOaj
zqxIY$$N$kB+xlJ@thjtHB$7Rzncw&2;V#jRmCl#9D~jH^aPRT~d!tm1V95}b4$shq
z?VqKOJk2?Gxbdb&z|G*B;qy1o58crl_#kj@f{QN0$##pBKi)e<PN#&exSzRjx!}*p
z3{QL8dq*zBB-^I>Zr{9fac)#Z-r1;oQ?oZltqFB6)mgJxW6jbiE{n(FJolzgwMf{U
zJCir)*(7DH(kT-c2i%TO30T{5JL}%jOtId07Xkh3faERW(n@X_kM<O93plSVp?K>;
z=AH|!jLWayp3&pTbG*T}vTxCOYYV-uEe|8z?=McW_S|qvqSefUgPY0m;@YX3B$w_r
zVBIdo5F8Ms8p~t7BIJMA)>-Fo=_ak_?~}bNvRSU0#dL9GdtJlf%QvKEG&C7XHT$f%
z{NACrZbKx8Kx?>kgO3D*ug3}tmXH-km)OQedfD)DT)6iwg8l7A9p4Rdx*RXN`;uqx
zd$CmdF55)@-GZiTFWz12>9OwPzL2H59z`d1KAakAsv1@r8a{U)JDZ5P#PP6#ZmpBX
zMiE>*dX0JwuX)ZT?A-Qx$FbMDw`DgTczsCdn1rsySxNnP)1=;vhoWj>Nv2_vf)<<K
zxXSJ1`Wdt%gEuL2t+&ddH#@h?tzBnp+t@S9V%1@$>!OTj90YGybafW~4KujH@z?pv
z@g+hkUlq3oT&mJ-HviBW|6*51l)(R##1KoH2`ewBw7g(#=@t2voig>RM%jv#kd>{U
z4k+8F-1ED}<fOv!%#Qa<X5WmuofalmGqhdiM?bNPZN2qno(}8DlDn~&o3?)cIH5gi
zn&%U}e~z6zucz`vcf?(sAMK{6aWpDcZaJS{SArUMV&CSSZP^n~KDfg@lP$fVSL6D%
z3nmYhbNP}ctGPvI*vW3y-7KDF{OQu|?XjE1%--;77DzU2UcJN1>h<lUuw&9%yy-G`
z?38DupE-8eb%xCQ?(8U&;}dueFmW|=%sjQ1>)Z`n(XNZ-e|XN>zV&*|`KxD5U_v`%
z9{c$`*6<DO<{GTRFPMY{Jgg-c>Ut!ZUiR}jO8@_```-T-tbq+oGv@d%KGzcXVsZCc
zUc0^SFQ(r6z-l_XllOs0hwN5?s)QpRIvuwKn12e28a`OQs@TghcH3G*`SV=&xmNc7
zm5BefLO1C(N5sad!M{XzYpSyHly0iK`h4;}b=PdpeTrQB9`L9!@0j*_SKDi&8DFEO
zXGhr^N0}QRwAy^Ys3T7N)8>XxrP)rJ>v*<q@LD61!(i*(_#h$OoqLOmar*UVqTfC=
zK6r3;VdAl4bHtfGo=BUnxp7jVy^ws}=}g@up_#Isk8?~I!|qP(nf=3}e{uP~;G_d7
z%?Dj_doAV}O32OfICI3N*YZx1;7|4!i>-~zT9rjR%nO*$%838B{t&r8@m8b6EBmD!
zHqX?*xXlk<**<OSx~3B;zd2Jv1W#Uky3<`%!{(EvT`T|13oY}+UOuq#Fmn?PEtB6G
z=(|`|dNIq<Ij`qm`?I9%?fcbdZ%tQVHhWOEj45r8DtFSd?5z@??o6(VfByPT@s`a3
zI&1cGty{97=&X?`<J)6d&FOtdSG!ba=Gz=Pc9ii)_Uc1gb+483=Is%_etXN2=#54l
zT)nbeUj3Fk6(B9YWoMJ^;f%f4y3-4MYW~>>AFTR%QnmNaP4lN0?{LcV8g+P8)kxQy
z7n|}Y%>L3IEx{qI!7SX6vE^ILom>03Chljv;n67Z(vfHFXW0g4{e(b=1`mt=|L+z1
zZ?55;Qj%G=;Ovr#k(%Q7J#ymSYOnRPU+ca7RgkPX!$<zLKbQOwdmY3cR9o^u;p?N2
z{oPuH+1r<8v&h%)J(s=bSSjzk>nl258(VJvKK;*E^UYt|>wnDuA3Oj5_x%62iZ`&c
zbTDpX<56(^u;5@bm$=ZKBdx;IJA~Bv9t2HPJk}#6T6Dwcz+$I<1??h>h@!(&y2W&h
zln%G@xO7SzA8LtgyL{GYNy#+LqnFh$%wulg)3y4naj7+zPfzuR$x9z2?`Yl(*RHD!
zVd0ylRF0~>c3cy+G0HXK!|MiLiK{{~OFz792;7#HD06gG+_Ac^f8M@*oqr(6nAxYq
z@bXb#&Hv`iHL_2<FBm%RVgARrV0mhTM%6@imy<55^X<#l-1zzWy1(!gok{%bnii$T
z+6E>mWu9E5dU-`?mG|$(;r+3?WxId#-MHPKzrXq9&!D&)dwx2dtXi0)^LK~gx7R)F
zx3WS{Tdxj#xObA`=A)`rdZ|}7$G_ia`{%prmGYVKNeBKKef{`;|9g4+hVPmx3eK?s
z?y{2Xh1+L1#dem4I<bmgVbgpz^Y~FQ39l)WT;(ENCb)*guqcJdXH5}uRPNm}vHkVV
zCrZ_#svJsP%7=c)^}TA0IM!n$7vd(($&@KDVJS}lTax>>#FQi-rEO0p`yM(fGIbq8
z*@Ps&(3i(Tm{lwmPyeqXW7#kxjw|y?W2P31nq;2qi2$dREk+j@-fFaj>U*kQKd-yA
z#9=~0>X$2S+WUT7KApv-dOW~>Zw8a2TFKW-{JM)=1Y0E*_bl`j+u<8_USnp5<M~C*
zU!2Zsy}lCEAGh^_^J#|F)68Bn^v)`Ky=IbPR#f4stFxX5)-y`qEav&;d^5gBHd|x!
zKZES^`sZ#XpWd?W#-lFjwJEz({B5|b?pLW47{;?dz7`%Ysk&fw9_P!x*Ll1Dy)9p|
zi+N7<gvQhx-4h#G^_J*6GW_>W*!xn>#lZ2iR!g2I%SBg#xmkP+f-aIp2OC`^CVxqE
z5iXCJ*r_I`^67-4EKAT~?YfeH-v0_^GLw(%AGC2eEI!#{(;30LUmp3n|Jo89;KG?J
zaxufJ#>6eyX{m51OXAUEVXT>~U#|#w`3AH^FFbn9DQ5Rov1_qMQ@iK#?40@{{=(H=
z+Z_D&CbmhZ-bgyFU!54P<MyS)s6lLrE88oEnOD@>G}c_WJZJTJ+2`Vq&j@bc_V|or
zsMgcTmg$wZrp0gH`HVd`+`Ih#RU^69oGIUhUcG#}<JXQijotJ6elo7mKg1$9<I3BB
zr*a?q_Zn~6J*(u_;R_Klp1xeIVre_}v`Gq0nbfZ3sjt+jR8|rAMKkHjrddi$zbCy?
z>E}@Tv1YgVvVC?!Qx-VBsn$%K=p_}IBJDA4lStD4pZE1sL&KwXPMOMN!JsYJq7`;3
z=>N5lWz8F2)(Sb@YP_{&-}<9Z8>KSckGyaaUt_bTXj_tW@~;ws9Xl8CZZim(_l0Q<
z|EyIZT#?5mLbk4mzIr%x^{gvvvr1*+G@pkpOT5Zo+aW0aO)Jn@#bcht<L)|>)t%?6
z7U{ncbj`Y(6m|8~wQWtCgpDe?dA>D<zrInbXShi__Pfyq$=9X&wO_O2Ig6F|e_(a5
z-&h!*yJxY?#To9_pFSpJge;b2eX>|`t8=rP57$wZB*U&XEGfM_k}ST<7WJ6)qy^t#
zII%=r*sU*LS!iYFnVe0+3re*!oDzcqdzJ}=x)mvVH7}?Va+>;o=>*p7quEm7CnimJ
zl=)OAA=@L?VA6yrn;gZ}63uCwF7JH(<8c(Tz!~WYChEIa_9|YDnUiUBTqdWfE#gYZ
ziN_YlozJPV=1-2CVmVn=pt0h(^^y%;p%qQLq>nD2_nI;M^_OJ%8-JJ2|ND~Pt-*Xj
z9p{#*Y_=MDs?F0Fbwm_TR&#z{wm`Adt;Jb2d*WBGu)VDv?_G47PkAVC*?lN2V$gUr
z<JFs$%HO|eIL@h<KS5)Oiqc7wI$tTTNlTUlA7v;|V~;uUpRL28NoT@}*U<`+6W9;7
z$t`u_DVgl1WTM_AVicCJ()Fpt(mC;4lxx3dyK2AMsBQc6RM;}fX?)VGE-teFUBcUA
z{L0=en|stE+vrl%G}{lq)@Vi*=3PIOq&@A3uHGjP+Z2ItPCjcr{e7>yQ)}MXzZBij
z6lfOD+MK-aPDJF%qH8^K@9NasW#2qGsmy2EQRRjl_X}75t?T6!o^VJk=xT@J)1{)-
z9_|t=4QHNtldOCx!ZUx_-qs~=7O4bA_E!8h>O8YXS;dgkd)n%GGv4s1FMcDYIkV){
zqQsV|?0@y#zO;I;iRsNs{KyrtZrQ!wg0*|uC7l`Xhn#8Mki=7W`PuRh+#8R0To$X_
zImc<$$~k#jS!xAW^`8l?*_QM$eC|SKew&Hz^On^4FMYOp`5U8}4ZOGe4$H}1aC|;_
z%2xxIg#QZ1JlwTkTDLg_?O+c!e6O(WT%$po$a@j#zSNR=Jf2gI$4toRb=kDZoljx0
znCip{s+@6eD_0zP?9DmFw`x;YjD-6O<rCi09+!XWde(id7W(_<`_tdN4?MXSb}_DR
zdB!+x=2<3{2k%cziJFmg_K0`h1xGhUN1Od8n}S16tcuv!86}g|=5l3WQBB}sr)$Zb
z)u&!LiM3v9X3!4IusxS#u=a?$9(PyJlrWK>UTXuRy`vj>a{@TJ{eQP?wCiro*>kKz
z@3y2)oRiR#8TTUGZGXM)-E&Mw{;Nh#!tQNx3%{CfH2<BQqJQK@qs%wS`F9fyZ=Jr;
zsk<O_@v4cwZby<NKL20qBX@J78=JZ2<dB&vbF+oq{g&L_XL?e^Ytf^!>NTpmmO_37
zpEd_gE7G{_y0rZL8WF9tN6v9E&gh$yW)t0{>XgKI`f&crt?|oLXB51e<fN2#za-DC
zxZ<XF-2dtqyGvsZxLipqdQ#<gGS$zzwdF&_nn`;u_g*<<79T5~VZM7V^ZEs{^O8G_
zAKjAR-t)rEpG!^ngg}gyuAlY4k1p!F1QeZ1YIOCscV6I)>u@`w8k_Dh!Tp(#R`d@K
zZQDuHrixttdo^UkpY9;Phjjuuf+{Nt_*PBhFbZLlt2rtZq~$n6tH8`r^n8T3p!SNb
zUwk(tJ?ZoJ<GO0~sfXiq(WzCd<NiD9u8sdv8db--=JwCCu1DKm$99C~?2*!bT^8E%
zb+T-3`d_`@U+3=V+ZE@L=I^b4ZP~TW3%}Z?{@qc1mGA%e?U(m{jC_65>Q`MnoASRo
z+2+rM?brM9Sx;!ot2WGUUXiBtQ{soEn^A|xlEsHrJuU_Eyzk#qJ=tKg=hy8A7)2Pv
zcbUKZwYPH7`}r@9+`q;8!%>2Frs`2`+1O6c)=Rd&R~$I5KDoh^)f)OGX#P%dgTOgA
zLR@cNxGu4VsYx-S<UnY{)9FpU3C)HcYiwpzoRInCarET@kJiwcXZQBC{f+rjJ7X3{
z&%v7t7iC%boC5fg<aIJdu5muRI#b?R=Y-RO>5&Zo-Mnq)_A|~r!7)>WPf~Oe$M=l6
z2Lk6RuRY`8GE<x{(rf1Vsj3fuZ#m!6GBaV}!@oEB|2cRj?U0<U+j@~hAoN8ygCL7m
z@4P_CsWZ-S%rKb3xbNh=&<m_wlOF|!G8)PW2hLh_rfXTGT&(HE<!`5ol-}D~a(vr@
zsO=nF;;n(N=g6(L6q7hAB{@^@_0s;&vD4-X&UCHmWbQpFZ!}>+#p(W<snQAF2XuQf
zS*72doOprpuC(dx3$oJsPSRf(CmAjZ(`QrGSGp~~aN6dtr`amn=Z0JrTE{AG(Xw5)
zzxXDPa_?;crL&&0F26R)dn}pXwqv%H-^@Rb=l5*9t$s7wHzvI7>;Kyq7!<S|j|Pb-
zT>6k4;(IjAB_UL2!6D5#dM|@&CW;@e^qr^c@~|<~t>>VI!HlMb!A)mHdlLh!d;MC@
z-uXVq{l@`?*8wwcUi5p@)A_|S{{GsO_Y0KX?|3pRL2w%fkK~<2vMO!bcMkH)oiJz$
z_kX2i{BMaqleDpvLeK?IT?049UMHuJ4q+!*k-ifuiZicrCc3rUy*NYOqon6(Oh@OH
z6XKK3dtC_rIc3THjh++ET=ChtFhQ|@#@j3Zb}&wV5qtM+?6eEAv#+Yky<0N<#5p0o
zDGzjA64H(&ScN)=aj;6pIbQH&yVbdj=lrwWd8+4nZErYg?>Mxw(0kR#jvWQd)R+B_
z)3my_>r7Op?flGDD$WfjKOaq6$9i}FgOqIyJbaTFCOvqhdrVDyOQZIU#+i4P9e8pt
zfa&?c+-sj2rxhyAR}kbWQi|8&oBMX@qt~MGAO7C?d^KUKpvat<`8|;W&KdD}C*{A}
zPXEFYF6ig8Q!}z_NxWs)kubgkEHjQMxw33;eEjQZbkWW;E(city*%UhMqcypa@&8=
z401<<KP<N2HA(Q_>4|$dFPm~y+A17g8(vd-)-fPK((LSYfm1VipQtIMmbdV|^oeL~
z@%Oqa^ZLQ$ngvU`E-7SPQuylW<9~!Vp6S?}1n=H=bMHlV-+Z}@W9mWOgzk_nhjd%S
zC-%+wF+rkx<^T9edYbN<nUj@#-Bn*qPHS;3IbIfaRHJA5^bF;YGR-dqiZhZPPATdS
zH1ofHlBK9N<i^B|n+uctc7#YCTe&YM^6!?rQp*<J<_weh*3WQKbsNitn0E>>6%*%(
ziuy3l3$$9oVwn&kyQpl^A-0(5;=OXK6|be&sqNVzCCNE^Pt3IqhWDok&CR+b`Bt`b
z!G}KWX|E>UQOmXpko>c9<-29t3}<>KIB4EDxMm;AECqr5d&_QaU1r4gT8?w60b6?Z
z&qbRVpE?{3Tra2^#G`EVEpSsGXV(>{(6i#s1)84?{C})HvfEN~>bVw=Gxx<}gw7_g
z^z2=3weEHMnfrehy11{9wz2!q^6zHL-j6J{N#gq-CTpp_3Y1yt7H}e1CPj}|<sjFS
z!-1>^7qPNNW+W-GL>a`sFqk4T#r@}<Ij<Ky@q86y&+5VZE5_j53a0{2riTyKUOWHh
z=`8IIfdmFlw;L+e8pm`c?&zL-bLZ*Ppc_t_I}V+CwP4e~qyVYxz*U?ZKP}lG#4hfl
zB_ziZ8_?Rj^`Ke=hsZaHE3HR*G7`V6ICFKTkGAW}@0Cl#>q5Tv>VBJW_vk$STicfX
z+OsG`;)pMAXVfizzq>F0ZMiw`lIony&btp+<eZ9}A=9#u>6s&sT8O~AMS82;0t7Sz
zy0@muXzo$(x0tr`$MMX4VVVCp)~)St5XgG7_P_t(o>i;NCY*Yt<x}OIaFIv+=aH4`
zR^`4sF~9C~%F`$7&7>bpYm~ilC}@iNNx!Q<c`mGc6<5flzJf`=uqmuqNdAMadb+Co
zC%4BR4CjAV)9+4tZpd|iPUVpr*-IxcXq)FHySx$Gkr{HLLq5po;f-}1U2D{THKzYI
zyng)5{3$opy{4RTInq++H+@1#N9ofR{~zjQKjZg5N>q{5;1|ohbl_yOnt`+9imFK8
zYXKRoVKO)NT)rfm80x|BBO~H?nNDd#PN(9@S4yt@&sONxct(Bd^Pe(v<-=DKAFR@T
z@MOX{jlL+)15b4wj}-WR%2C&9^Gj<vF)hAx#mQ@OZ+984><sxY)FbF3RO}L`b>i@g
zORY8wr_bxDY<YPqrDcZLT-PZrJJ!Db*70`A+9Nk_U6v?YK1b~6oMXE4F1?-cHDnfN
zfd9dd^RiCQGC0B{`Ks~W&Fcz!Azve#TSOP!Joz}U@nEIy<ZFg+mulTU6f*Nk!5s#F
z<?fR@T&JYMSo66=^<H)7vH55}$P)Xh6)SZ0Nt&44r?mS`C$?%D1*Duz)MQ!nZ{Frr
zC%5qBZvHgyv)t^htReyDicYDqnmoF=_rklciP??}XL55_&%7k9Jx^UQ?xXnf4)=sh
zfA|hm?NfJ75ZbL(JaOmh+M>_fH)i`jnZ5U!%f4%iFK}dNKU{Iwt(E_>%X9^uBf0-u
zkJz3#$hPF#4%fqGDn~V0SOZ;ey7*{&9IVV~VP8LE_Oy?V(lQ+1{d}7~1gCJXi+UCx
z>lu3FtnRBf0Uy>RU0LJ5qJ6@%EROdlb|kp)PoFt;&Z-!t2}v=hWzVGi+K?!H;>8}0
z><gES4DOkfY9y<CO}Y3_=i)!VCJ)9VCNG%GeO+AV_GkKEbqo@dz7n$NhG(G2mD1a7
z1&m)kByZk(9&s+XW`^p;nKQDs$@=!1`dxe}t#$IOALqTF{Z`!(rCy)p&vDG+$a|2;
z^3bT|f$Ir5=c$jlg0*?hJCz(~6EM<D$%~&h?S<Z^l|~a({-0Pm?b0NXsTFsh&7YKa
zH+5p3Rj}rn|Gv_DS{ze~R!%&)mdWk<-A4f(-oX-`;+$U6JG)~q-L;+S=pDRnK??WY
z7v9Nh&350B`yi+PMM(dnTY90J`e(84g<k4!c$a^8W>n<AzWDH`FS;(e4Bvj`91^Q|
z|E+fYcDL(g$9qEaT!gxL)GAv{CoirvK2lo0`M1{Q-Mg08`Z?^;+*<YgOiARCb2=+D
zDn<YQjb|vH&M^IDcbhn?^pjAVoGVMFGn?OSR%&r_+Q2fIsfKq)ujG;Wb8m{|96Yxu
z$9i|##5RK|9+Oj**v{|TurJ4qvC!mSgm-7WclzfJ<EP8EmTXzTvvN95i+043Q_DU~
zyyG}il(YABtY`5Q*Zqm_&;5Vr(e-7O;o>QBuKs0L=F4mnaK0eauI=n&x#Q87_J5Lb
z3j8iUYp2(p5NWfV(e!ju=g)5?nxXzjLML0!?2--dPIS7yQS#i*=^b1DR7C#Sp%Z(|
zaJHsfz(>njrbm~sRm}WyCRRG(;osO0Mx)E~&WOEu=BpR*?!v2Ac}bEpq^A7IywkBw
zJ={z$%4ovARjJ=t^2@emyuSPQO`7JO6AUvxMQ-<>cH?3HkzEUewk2&jtvl=By3pw4
zkC#2(2k%+Zy&*oVVMa%>T&u&!lI&Y1TV8z%F#5T9`N@yA-*rD5c^~t-x#rZBnVP@#
z&Wn|O+}^+ZO9#)NzTBPt+pB{<J5DLD49+w={hvj&z+umAb=Dipr>tH5_g=KcbCzFk
z=4<|~u793nFC^m{%+42NFzaNojdIFCxqb(`nkx@B9}N|_W^=UXj3?WpAg>ua7RcXm
zVrHxV*x@@X%KP=2j?1$BQk*@n-~7DeBzx({snZs7erC+9<z!U#IRDJQ=XH0|jmD}i
z2W4OESea*{J}KhZzrNQ;I@X8JKAv^$xK{Q>t?F49wrmdCd)4^0QC5qe{1q{uH|>1<
znp2!yU1deA_-E`p@-tOdV!P!f55_M${)&1Ztgh+^33d$a-|IBNvWj2QHZpQz|AtkE
z*3NsrbMAe9UsfNV>q`%O5t+7PYw(PS<msN0BBF~fZFP&VOHBL!Twn1Wb9#2fMBW8k
zo7{6YdEQUcEMV|^rk7JRaSc<?nn_Q#POy6SN{hL$A!}dOe~Gg_M}B4BR=IU|>C5}V
za%$<xqV^e^1asaln)~o#v61?f)oVW|=H=^~f7RZ<m9f0oJO8uc{(SfSADC``GdI`M
z*8j3A{c=L~|J*|f$tTL6O*`B3e8<uK*Ea5T&t%!X-r9}lTg`6if`j@FJ~ItknMJvR
z*;IT^v_53;<2h;bL}=pa2BSGAB4j3P<XPFB>eVO`ndRUyNlkQ;&x)Xjr>5v89-Cr0
z`PmsWV^2+$qh5;sGpvi>`Dm0dTx7Hp&I?*$@=&AI&vR>*<dtcMR)ue$vL<W!|0%PL
z%42p)UO0Aj?X<Od6IGIrPhH&^6T@5Z*(@=1Q|4(Eo!eUP8McU>JTlk%z*X6OV*j7%
z1|Rd)439mXCgZ)#$L&<iucUPu6TLZSq*{f)S+n8Di&@6D^QX<+<vml6Id1j5oio+m
z-Z04D{zPGO^O?Jb^^e0O_LQHwb2wF<M{bX5?p=eacfLe#k3Gp4y~F3^zS!#Tj2yyg
zqV;oYu0464+~>c;@sx<B@U8v%H*2k(f8R+JTJv*S&$E9E7_azjoMmxyg$8r+DUIbu
zw>&;Luz5CJSY+n4(^G3N$Lw=Xt&d&>EZ%4KY0ah<k;OecC$(JskA9vr>y^W-8!Pe}
zqEdV`lsD!4&wRR{@dlIUnqI#@6F2R!I6YyrkzS0?iQc-H9>bldG!)k`dNEAwx#H9%
zw0XjpP0hwaR=YoW>3R!nJ|*C}MPqBrE{o5m0-vjb)ht988T$%6^$GUx4SO?r*>#3i
z-wO_$vR5uR3EGBSbm|R$VKKw2!X(I@H8z9cd6wh>vFb;Un0EWy@|Jerog@7AQt*4@
z(5ug<NH;KlFbmMQ)}j^>$kn>VJ2bq_;=!Y<@tj=W%rBPB^;p!c<TPz^|Fo|slqb7w
zdZ+R_X4SRx+1ZnNxL@{t^416|`ZR5MSkEKQm?dZ4ev8>ZC##TQ8C${i*q|k|uTPVT
zKGij4)ib8G>$*-Qy*xDg$p1~rN;_{&I@5dT(=Bb)jd`a|YabO8UTbueCB$P*E2s3D
z6Bm-AiqFncOP&1WcEGYLPo`?;T0Wg6R~ni6FrqFpZJKBI+9w{iw<FVMH`?jGaLb-2
zxonNmTs=kE?F!9yZ#R5ir}p^-H~;OGx4mCnV>An3+Q+#kEMvj16X!~I|16lAY4e{Y
zNBV%(!!Qpc=I<#52ic|Pd{8kjdewDUzVnU5;maNkCChe-CMkbBDjZbfdOTL8wMR_7
zK2Uw8S?1!otD;U#;n6%*aFlN!!?#VF<aIyoUUK36f=OOVpRFc)9rKA?b4n|B{@yLO
zt(A?v?pFJs;9WWA&fjuV``6m*5B>b(b3x<j|DHJImoLvuzVvGLak*bFC!YWJ^Zorf
zCVua>%h&%lJM2C`{VU_%y=N`%rkt%c-^qU^<#+zRohg40>nn#HoouFDZIQw~`vy~4
zO{)W6O%^l9$Ei!j9<B7&oe(ZCX94R$XVYgY>DkOL5}Y^|Y5RTY3J;l~;Kb7;AgZOJ
zePk2At3k71a-mDn*<T(Wzt|>So$Jzmd6Ss;I<X^G6C0!N?79$9r6i&(<f@UAkmPi3
zs&KjLQ_WDB))0a1>suaVD+^yp+ArX?y=B8v-L{Dl;m?jq+{$<sw7}E%m`lsd)+ug^
z=R8kHGm9*k*3;vwkj0!Vk<4~wWm<jhr|ao&GA6q3bW`h;`R`De!t8m=VUo#F!JGM2
zZcF*T^f1pkvqU}i`exs@pviOgD61KIp0tjd6lxH-w5#Ze$-9<UnfC*uD_cB-R~=mv
z#ua#oq3-3@c(251JQw*ycQp9CZTMjGr*QUdI}zU*A}drSH3Q<_JzKyU=#ekQ_^54K
z+Yx29b)qX4KW3QG?q_(4S4FZhM(<+q&IK|WF(#Qu_a=SXdEm~+QWdUybIZQ(o7b9F
zzN@-7vm{JVU~7s)V3tNs)iPD}n;v;8O=k{GE5Epo=S147&YM1Kj$g3!++3;T%D-65
zbcN@}{94`Am)@~b!7q+ZsnzriS@ceQv&+}kEXI;nM#t6z0j5hrOI<{{|F2ipx;25z
zdif?P{oJRH30_xoj&AaI+@Q54VdC-#p_l!s2Agy?Y8`7kFLg3<`P8t?wW9NW_H@pD
zB{KD8kGIB?McT0oBf|IgdB@&g)w!ocX>Qd!?s!k-qbwq~*lvZKnO`|2S=u~!mUEm}
z%4)^4sk)mkOjK)e;@;exW}Mw_R?(`<!rlEzF8b26nkyN`(oz57q@%B&D@(O|9XWeN
zOZ2>_-?r^leX3q69drNPv>k_@zAfxNJ!_`pq^(}ruFG-)xuX2cv(7Q{`>d<JYiDU>
z6FK|5emc{=jWc`qe2aWmdF|N^n@J+;3}!}4$xhn4$574bPRQJeMl9$0?jCV=ztfX;
z`R4ygvkSP3-lxd;@W@PNIWMgB=})K6oJ~6|Js-Lp+iWmfCui23uqiGzZ{mtx?2fE!
zTW6Re8JFsEWX9#Tyb=|S*6BS@9zM>ks9yJ4Zq5;PyAKoBr&b%SoVDV+jjZ^o8k^ZK
zC-%-sY1Es!OxM+D^87a`Gmm}QA)tF}s`Z6^D_IPJQ#n@EirhHFwvOHAa98K@Bvr?>
zQ>RuMyj+@P_rj5rrTE-^&Z#aHrJ*UixLOZbNo|y25?pV0MCr;IZr;hFlb9X`s8~Kx
z^j7#Ql)Y7SOUIiNks^n58U0yf-4h?hPL|N0ae#Zn;s>14Tq>^TGEZqOxp6@3jzr|e
zkTTcqlR5Ln)WpvIuWDTwU&z|P)ogL-j-tvr#ThYPh1}8qcU-Dp^mgkkZ{t~=Z_>+6
z@;Kc%V*-+*r`-LQtG<#YFR#7&_RVwKb{v&GRa{+t=SEwGmj7Y~qdT9SwoQ<^vMu=T
z3f76c-L^ic+Ge(zanH@^TZCpai8;x?(I}p#U9_yq*lfYBqb;*OhE6ayoHw1rwj)ZX
zv%9Wy`oaT@hH9;!c{u%+ESRmbW3B1R=OQm}PGY$qtmv8TF1_SHqjslAeTmeB&Kegs
zp%oU_Q$3!uy*3G)utD|u{%t2RUrSh-YVtSP-x9JGcV;toxLfr!_+p`!?6)12`HwPp
zX}a8I51w`|<gIVs@7s6f%6FWv{lC;~?G=Y5NySDvNh@l%^Oq%mUG;Z@dRU~O#lgEh
z^Z3dFIE&|fzOkli{u-v!=XHB*#MyioZ7^D`JhxqJvHktSTPM|A+B&T><pGzvlGc-s
zGSTa=W8eQOJzBT^vZAZ4XY5g*<0&Gm|D9ws<z8gUdxH09|Lpax<+tpYz3h*hDbeXR
z{q@EpOSSV1dOkGWx2jjQ{;}P{XPvjYp`E8pn$bkviwVEqd=Bs0v2f3~-|gJ>PfpqI
z{M`Tl@7wwRf4w)e-+D^0MBeHCCtYg=kKLD+nf0A&%2?c*!>MNX;_QKCYAO$`KCIo-
z)5&n%XL<mu$v@^<C-lFr>O8!sUHQSYyh|(v7Ekj3e@U9K#`VP}_OCk4n+4t+;Cgc)
z#ZN6uKrLb2ytYI0YVzHVIdPdSnAexSu*|u#>WG8Z;UiiuB`f-MZY6lk$}7=fTC9-#
zvXAAJgInbzC&Lz&@+c*pLyJTL*1Z&K>)Vq4=8QXUxel+j&XRr4*QP8xan0}KGxbx8
zd}==xoLbPszqIGw6}{6+t^&Io1Xu+RUs#@cs)1Q;!!t9d6$*l9ZuM{<%i%iKlbxj0
zJb`Ofit_Fh<~=G0gj%!0xtdn6HE^+<Z<+Nt=uPqk7G9-CTo;3$x9Rb=m%R|H?T9Lw
z_~k+8pSnXG6ExOWDCIXKNlZQ=v2aE7C$~(t2Hsn&#U=VNQ<P%<>$N<a)F!$uflJMM
ztD4qNvqvSTUg~Qc5p!`@-f>9Di#su<k;h_L?g^#M-(LEqG~KZAHtgWdV>lUa!2H$Z
z%zX(hvj-{`3`P?rGE=!W1xc(lWe{2#cWm2)(|_BvZ!3J-t}^NFttG}0tEOmdx??fv
zj?JgZ7jkbO$-R51cYah7(<UdaB}^eAek(t*?w;gqGOcqrkAdQW*ERxribX|SQJ*rU
z*3DG#+j}FxOzWH@*ONzje6vlIV(k2!pFGlx_Ma1QxRj}9QA5Ym2)=m|Zw^*mD16oV
zOlXl<Uk3Lc4i$|hyitx;dRaWB8l^>RIoB>0^=a?8v5EU6!=;j5qfIW?8J_X|S82QM
zwCaV5w}094@`~jK%38CoZFqg`-Eo=Ch94)snUlWtg5NuxV19jLW5z&Z%VJ~Ui2<h=
z6qh);P2ma&db+}Fabrj^*QP?#Xr(Zgb^bPjijUr?B|0&-6o{z4U-hHs(--E@64eWH
zG?j8#lueW(=PdnvM(axelWLvEr6SK83X^RF+G7rJUdiCM&hGy<?b){|rVU9=A|lT_
z)-4u2VI03~!RA9v_f{F`+z2c&VzT2rEaua)*=%vV*Fr6~6deQQ0xo7FJ8!)|Lgsr8
z@hG_VeR<&_#^ey?tgNzHUOm{T`plmz>z&s3)PKnT|03Vnc*?&`z0P0#x2tWsYg4kL
z?sVvX5e8l3rJ@m&b{QDh^97hjb(r4PiO*{_Td%6l&hW5i(nE=Nds4mxSs7^D%(lAw
z%47Pa>UEiFhnn_J$neTI#43J@^{y8C?1^j}w-j#r;#ImqL*|U@q09O?3*~m?=xj@1
zJ~VOju?t6HmL~eCZ~n1Kzk%b1S{l#K6a&`Hhppa}=e7Is7*{Bi^uJ0=`E<myqi=Cu
zj)v;yy>omwUrMht*v$VrAn5%CzK;gRH_!1Ke|^hQSj!_=Yga1px`08YL3qg`Etl3N
z38zMp59a5ln6fr8Wv5OLTcVa6lJn?|8T-0bULD~@Pt5}+Dr8*!q;*KNmsKovqjE-v
z*x6N)g@Ip`)_(cFZt49yr<I;?UAEAUIiMPyoFv)Jr1_~UdUm0R$HCa5_mU^KYP$4F
zzFxE_YGJV0;opz8-f(I!cGA0fY(o5w1Dic2c;}t9Q}pgPQSQ6<Vh+pHO-&jT=J+SO
zEZ1gP`A4thuh=Cg*-d}#UT$~!l4SbL#nkNXja(J$O)O49%C~b<T*FiXzbQLw-kq1F
zn*Q?i1vloai?c2q6AE9Uw#{vK-X$~MzUdt&)f=Zpoqw}>UgqAKi4HtV9%=P0>YMa<
zlja;Vm8|2B+}~*%uQF1+bkgWWYT>3!Y<st<iS?*GJDGhvA^uIH+qooRwFgIkBz?7L
zd3i0^^O=X%f-4J)miwPp$(QX)|Ic%I;hHll!ON;%r1N?%dAXub`Cia3j~;KgWjCuX
zdMF+>o}Cl?@mjgbRX$boV(a4K>v~~%hwe?u3KyEt%5px9eNk9I`Gc&aR-OwC`Kj3g
z`f5@a&n|0V>SCQ)^K?epVe_enTT4`x_MNh5EML<RZrMBCa<S{$s3!4nm+Ki!TMU||
zW=^zD?Ektgc>ONp>U1WXZl#0U4%qOsmxN7trsVecM^J^)q72<MlRFuGVpsZd4sTzx
zrm*ttrmD3s4jif!PI`UMq3?#$&#v_jn!<mrUfG=M&ljBW&!VhRTYIw4q(;MS&Z-tJ
z%0;(doKN=b3^Qvij=Ex!Xu^CsjeApDRz|7a|DvzL+<rMy`47S*W>gee*Og3b2w}cn
z?(uiA$7w##``&>MQ?`Un*nfwiqcK8%$5O3U|G9gn#2q^7HM7Edc7*rtDeU_0UE9~l
zr99d&O;yHnnTXMwvWDeB&F<UXoR4{^cxf~Th6{dO&}Sqlnz&~Xix;C#&>Ak$DPP-`
zUw>BMROsU`KDAx(RnSbI?LSH;C~KM4sh0~>J~#4S$g^`I>nQ_PuX3JMLBA(#XPQ}T
zZMxlxL0S1|=Dz7Mzx>QQPgx#&6S`TjG+C4>IqBm*=c%$ARMiT~(!ZN@Xoh-dl-GV0
z@!mc$;$lSNiwkQ^BSn2xrm=phS|PSj)GE~Ja!AW7msbn_=e2w_@?E#aSYl@2{*(>T
z)lG#Do?5T%V2@_0Nj#*dp>ssatc=ZU>n-s_`5%p$zR41cf6Vz=xtddFRl&_u4|q*Z
z^K|f<w6OS{T69UT*^sTI;j;fk9#zE|-WGYu3q6y2o@ONLo0S=?e#)_O(fyxEu}hOJ
ze<>UONI99?BR+$9$pM>_4h)H6Zu!@;?l9YhXk<TdHY*8H3%dS5YU4+d10Rd5ILk`*
zr2JT(e5tMT3-e^>OwpvwqZ*p=E(V{posTH#y%5{q5INT;?|IS&l^Hj^E-ubKx~+@N
z+stF37#j!2*Cr8_1nvu){4QL$RraRtntOmq){-3s6;jDA-!2AdFXs7wI`b&k;oiA?
zo-5XI*R^b&=VHipSY*`_!yiX@7oA-s?OG&Xv`A3Ik^5}L<j<QXT3Q-=?-jXZn&iI2
zMEspa<rZO!1v+nP^<0@=u%2JD%B|3|RZ62YTPj`D$&Sa*J$+WlMdcX{oS(W{KUEd4
z)^Tl#D{MQ`rEHeJ#%9r)3tVg6^2HYxe_zMSk~+cW^{J|lT;JAN&)GG3{f^tSubSG-
zeUWi#mW^fmAv2~~PREYjsNU<y^Hbxo=5jG3u3ax?yz>k?q;JUPx39_HDs}FeL)u&N
zjq?_+pRZqUv&vuUlM?%?gDY7Ut&HCK<ibW#Q%k7|7sZH0w|{S{YG$6)!1L~GuwT@X
z|N2W?q(2-n4Qz>y;uK)YNs7wOYI-1`x4qP8TD|6SVFg8%hUJBKn(M4Ga!qX&yjN&i
zAG$bW(S(?u7f)uf@$7%HThpD-rD<`&x6cP>eC*t5a(bi8>H0-Qe$GZsi$st8QV;l)
z#IcIw_&R~R*H=zE9wecxyZPKAo7dOs-rDXxbVSPS`XAA<H7S$xSF7GzQ1+>^%s`7r
z{20%#vR7=|C)}CYf6=C0c5BeYZvUI#8!a|?^cl}+-WfST_=v9bO!2o$Vd__(hy<Qq
zAbf9$@GqvT!cJG?ov#}9O#J%(jCrK5M5M`qCH8Tu+gP1AMIKzAs=%nh6B4meaRc+p
zc4kE%%dq6=(*H}jjxC%kTpF!DIXQi~)(nvY*-cBs3nixq7Al_6Y3ka#qyFZFDFH=m
zLJiw)irxG>W2*SBjjCLsDcOfTRS&mDZxii2^Yq?Q%h`!e8$_1JOn3V3lH0Xsvqs3K
z`-OenW?geHGS8QOe9lv)e(m|0zki3k)6^GYPS9zR5j}mz+^uhm{|4FXy!RG(Fs$sc
z{k_qrxwmN9Rwc8AqDxnlJv+#4d`;um!j4<VHU5~$n0MIp?6o}{T$jqE=D^tZ?}h4v
z>&y;}LJo|x_Pv~M$8fHSA$ZozrkE3<F*BAmY_FVlE7ZI!-}rd4(($u(3`KL67tcA`
zD6_)<@xv?M7tY(cB6U&bOfkp*25U8i<)Ytf_%P}0yUXn<lKx@qADbOkmMbit9s8z!
zWKL``i>kO<l1Sy|%||vIJ#_S)gprlEyUG7We7Oe8{jPtgex7hf)Nh~4wjHH5haHT&
zUw>Pn-@{%huzA@P_Vh577q$QTqr0?XpL4hhRG2HRX$y}uT-0VKyLsaO%hOe>`*v5%
z{eQ*3mch#L*UJ4$d|YO`8o13W%UKuw?*DiFp#7ivzb}v9e|h%)^+wy<n{7*)FZXS7
zx~cuKBzUK4Uh0e`tNt}+mSx`YUvu|5qf#f6s_?b~!xOEir?ki{GpVs)P-^es+{v?|
zqmW^FONzxr$qO5kCH<7WC)q@{g`6^48n#X6zbV5r?`ekFd>cAHFFQBarZP$<VA=8u
zj7y?;4XQSau3X~XZ)SSxl;-6XUdyePdTD6|2Klo0T>AMnEn-p5thy^VPl@?&UAv?}
zGj;XU*+F?t_jHPnPnF*5_;H)ehU>gD511!1m8G5z^)=5c`&W{D@#@?Kxp|U&ITx=;
zpIjz&z4UUb!waLb#(O-qlf60hQbU%<o!!j+hEX-+wq9e@mUDM@mVdgZ`{nz~*;~We
zP0!6T4S2ft_pE)ZH*fj!;>tq)w~<xPOI}~!T^?@X==g(^Nrc5<y+dOQ3)hYL<@fFE
ze*gTWI>&s6z><TyDLj00FUdAU^DsPMVUf*v;493b+Q?+?S@3^u^L5n)!mZ|!77k3p
zJGKV2bJe~`th>%TLq_tXY5AjKt-R}ZBNbjgS=6obX-R~%xv5!0+?}~!1v!`fj9`;Y
zpP2O6FNdS_iLWQ$hLp)xmNzd?vMIfK)i?Cp6)#)!tA{R&MQ>R+Bc^JrlaprI4kyX@
z1TL}QWF1jSHE#>2@L4&NZazDvE&fpBcI=L+4pO<bOkNtfjwj1B0(J<RzsOJ3`L$yH
z;?U(Q=38BLo3nH(llk2x9b(sA*|m19nCL2d_sd0h%}$rDRcj}GdF8cX+LB9ay!U#A
z#BbcUL(6kd(WTb)wWUEq8(Vu#9Mo+Jg(k<IiaT<Y#alunzQ${b`Gl>vHkJR^-F{7N
z!4bztYAF}o#NQry;~SmB@Id5^(2fTp!8$u0Fa+JuZWLmUS?#iq@0T@0%e}i(jxewW
zM||*;;4*1oViMdUATASfMZlRmFtE^N<;#?buF{1U1D%vR&uEvf+jJ}3`^>&8YkW)&
zZkgg^C297d)b(0T_*A#IH6iES?)F?^FwcHkeBQ^g^l8NG7hASO#0j=uk?`EK=$daR
zf5K^z=mS^1vMsf?U3Ya<yXGcYxHR*4WYgQ!_SmvJ744d96S*SN#ij;^=cbk<PS286
zW{)mfB&j~zb;ZTCvlq<NJ16|e`77I<0tZLEpF1ZT+kNkxwRpv<%yYHxes<X|-K}%(
zPGjHyT`yNHH(OuzWpAABn@z{X53q2*Ro8pBYqi~jwqThXAND=e)JSX=kee^pCUH;X
zz$>ez+g&>p{N~^Hw{7gP?Yd<wUiIqcT^FleP7}|)aesL#b#Y+V)ftQZtlou4PuTv^
z#ZA*puG4<<ujl+r{{{Z~ufj0vUD!QI*J&k}_*QRCa>!2NJ^SL))|V}uKIIHt_e6?U
z?!NmXaN$dZ#s5@h^EC;y$lJ*Jr#+DsTN$8nvry)-Px%V_l^MxPnLFKcB!v232&8U(
z^w2AW>4-en$8a@GC9l4WBa=$>YfKZTZJfk<R9%nV=Fg!p(e<hmbke$QqeMKIgcMly
z6H|JUx7=Z1+p@0g|IuuQFRxDST{mNGHtTsuAt|pz6+$d?pG>S|+s!G<s>FKxDW8kX
zff>#+iwx%mJd6?53_b2svPzc8wcYE@=`}e=C;4phNt?5#^UR#@i{{t(Ys7fSN3{rT
zS+YP?+ika(cIP3HWd*D{K8u%5JvGTo`(~53)yiADJPNlRzM*JTzMwnBBU$ikt(wNE
zMxWwo&5Iu0Syr%UrgUz|WsQKwRV|&)a|(pm&*j%$HlN7+{Bcw8xhr2>i`BAxtsU9U
zR@fEKyA!>TbLV2&S2gSAO_5U7h)xk`+Of{|wc?8!u2~E%T3gEgT%5Zj@4Mokj%*k6
ziB64QL_N%2X(yEU^s}%SU1%`%|6lq4%VIwk&An6VURb}=x%liy<#|cI))0eLnw#Gw
zomzOq!#!1-Q8G<f<>$|iwr|1<{5sN>Flc$MWOy@wO4%hXZC}lm8$_a*ggxvRe0XYk
zY$5kOxmI7NpKtwFJzQXRP|MeCX>qWOi0sn{4|9*58`f^S84|0!b;q_xqU+0JL*h?n
zPTw1;r`X3MdBRRiNQOyu)#|7>Clj6?j_{los<Au8tEp5Y(f3lzIW94gxg9!FO|w%M
z>gMiRk#RIPw{UAjiG;+?4HJucy<;}6<#0MzajfEOcdz=}&V3Iojz5^WrQcffhU|BZ
zDc0XqIJYHVchpkb*d5x#CA~u~QLsC9;?iuZx&N(iCV4B}@?Cah@)KPj&*qd}jD=0<
zt5Viq@T(MFz^kC;e7UoxIP)~4$=l5Mm-^vBKSdLguB!j!oWlBdnq_tzOM$)O9Pa1G
zckGaR7c_m=hN#!KCKpZ%4l#dNU~zh;^y|*a$yOgflwRApx7^97TL136@4M8kYUW8C
z(`SqIJeqE$rDRfOv&^62$c+n*ULlJ#Wxd|(NS(N^<)O}7bVL5}W7p3-3o4f#p3pq)
z`UaH~-BmGSf&Ly}{KP*Sm#$Wx?{_9=-;xI}($~h^4`|DhR^U8Y)b{ZHzjr=YC)tTk
z3zZ5A{rqOCS<F`TlnG4}!kkz5PZhmrZ+mtT-}5;Fv)o!PRIdKtv+?jD>F-Yt&Jpuk
zUVO&lwV+tk%8xf(i#lG(1_fQ6D7q*$`|9e5!%-0joF0{B#(Ju0c}G56%DJ(kcg=H~
z?qtry@NJiZk7|?~+6kym&z=AFP0O7=hYyc(_qKgHR(sKKN9mK!LQR%;Ja<`ku4<l9
zteWM~nUSN^vqvvco5@r9Uh6d_$CE0pHzv*4`zH0t+a;g(JwHEp!b2~cgYUIg78QhR
zv1(gi<YKHgnzm8rXoD@In*Q3htxIzr&g5QJ5M}u1<Bl|`BP#3XEWMcdY3s5*MNeXW
zw#Vk}ec!P*S$C~v_hFv1A{T{^F6>@u|5kI?++eTE3qnr3S35ATabMnXPKHm;^Z(mD
z(QAI8#<Zxc$!wOwyoxWjFAC33+3%OyTB`mgWLnXIDFU`Dz4JBnwpyN>D}4Ft(G|<4
zAG<UsDM?xW>kC&W!5~fVj}vFV-MP}IR4bT~w{X#gE!H^>r;a|>ZtZ+q`>93tN5JnX
zD;G4ia961}--|qxH*K?MaJ6b}iO+h?qc20vOH}us5H($SW%s^=QhRRa=v1swXsz2I
z_t>M1|Ml<PcMDY3HJwYA)_(oiIBe6jH+S|t*R8Q)RNZ6jd`-Cetx>|{Yn@gvUBqp^
z30^7d-Wm|r;5@f?cSg8Q?}NhYd&*8vuD+YHn2Gycf%vuK&r}r|89f{h2??;Mq#7{o
zsApNska*;OT{J`N>+KCI4GmkrRxjTaseLG7-eM!|LY*Z`^M5@vnjaF>UGB6dP-OWL
z<4;Sy)PKaiOKo~zD$Ma*_P1!Eha*?2qsTUcguO0YahqgQ1XOYv9M?VI%3UnGXo-x|
zB{mxlk>-nBWeVc9*M;S#Nz1Xd?rM}uRnVGHT(qcMc%gW)N|}?6idW5H^F>R9`2%fD
z+S}$_F4H~Mc1zhmSuMCUJ=w&qeen^Oy9ae96t_z?8pjBDcpXoE$P~!unv&&idofr+
zY-`}}O)0jFihgBs6FJJ_kJ~(KOSK6s<5ljIS`?&}>bB!(Qj(d!ABR_UM%Q^q@nwn1
zc^6u46^aRe6coPr-%#I3<$H1Z&&?q_GeZ6!?zSja4KWhQxxg1{obdJnTiZ0T4#S?W
z0^-#ZCAKB-Z%^P}n%MH~GS~GRY?+Dtg@y%N6Zo%ZaKAgCHqWi(iAHejiLyG^B&ibd
zB@g>NUMLG6X3R{nmUhy;zDSP6P~pNP!CsfFGfe&yrsxUHa6V%0`E{C$<B^(GV%e*w
zX>Bh{lQI-qoS1FlsH2jo<N780Noj51lZlI)wd9j?3XbTr8b)&zPI3w}d>`Jg^Rv!o
zVaookxz_^?7&9jSVQ;YT%nNZ#eK5`a_i|U3jhuev*4jS>xFigPCkx7No+9Yk?NO5d
z%}x8qwfsNFJayYz+&v_1e*70^zbGHKvp@Y}faXN+RLdr7K@;xlURgU;0xlZ692WHy
zD*S0$n6pfHNdR~Ma<OdzTya9;^^x5BE|{iVVB0=PqAO5pjfD8tgu-nMKDQ%8ZddeP
z=V-cXn6&7*W7G<(>dWFa%*oLbf_#RI!U~L!7s`31%d<TV<SNKp?xvKty-n*{+w9M6
zHtgkwZL@D5l-X_B&KTZjzASkKXW(JZ(zY9lnd<tV7O{LvQiwd>VRKQTZi?Jl<xbno
zx#B^J=8O!s7fa=*>c<3#TuPgxEn2+sW|Cir(*l+z`HPd@DD`tFR<3o@lir}0=`#OI
zrN9?WHy5XZj^*8L-{-fQi>wt&2rm6Eo3fzS_9ENsM-d^({4<l4aufK=9JnsEEZ7#n
z?{<*w%n2@L_TGSE;fSMZRZpvK1TI?K?yt@j*7c~*&Q(h9Rpo4RLEeLUUvKFDZPLBf
zlyhqm%fC+sH<arC>}ZX>S?!t?;ocSd_KTsiaU)x2&E%&Mfn7Dsp5?w(*_zC83tJ=o
zk1SovniD@uxQ0WxIzx5kAy(6_e&)vc7tNwYHp^>gT3<}IKJ#*t*(%}vTNyr^$2c#n
zw~}P|E6!5S)u8O!`N*u{{<X!guX3drOmTg=LhN(QhsY&;PdpB)DXS$*t!rAUaxq)_
zhK^yf@ba4qXCr3?J(wrEi81ksUSrCtRkKz_3;cIWJksQP*x0c+e)BHj?NO^gY!&vJ
z9G`Sh)Vz55)`I4+%hQ7&_$?0L-o3=~<_a+xb(5G&THX^EMi)33pWq6e*rP8}wQzZ&
ziHLN$V@s`I#GTIqi#6q5O3B%V$+J%lShrd3<E#L&O>*T?^L8AS&(N5?$YS=Ntl0%S
zoJ_;o=l`5-k(~D6@sws&{SQ_hPTxxAR&=~GXZ0*AIdFKcnydSpiSAb{J8x!oN-{BU
zZkT&(rSKI?fu#XlUXE-6(J8MEdHmzrz#8pbBjIj(TK{L$W^-*;p$(gDPj9xrz1b}}
z-P?3aMVV|_+m_uo=JQWq9Z_VOv2lU4!hE4uGdkG(%_LOnLjT)mU1Xc1u+V+u)^8hv
zZ+A#kIR<b1-a6x(<+~HxPJWzcYNfwsR+Qx{-IHmX{jTZOh8T2b#8jQ`kl4EHz1j}h
z&w0<8CiH}43201E{5e4?aKaQ3ZN5q&%O|qoQd1+V8pR{oW>x3UKI%F9c7DJW&&9vf
zy%#$7Y3E8ix#e29d8`z8!?ayjb9X71jge*ybMRzK$%Y246)E4B2d8z~YcjeP_AXf<
zaKh80{fmH5uv_mJ!<vo_U9L4d(wsLXdCpuS^mCc#!fc7pTI=syuivtG*1gqowyRja
zWENyrhOD1bka}`;gi-u%iv{jU)AgI;*B(%1z9f^W=<h8kzNWF#saZsB)_;jL6U8pS
zYOYu=bNArP9qt>f7VcTxxo%me&AV5DVlw;gS+8%avf(_MB0py?<7XSOqV>|uY5hyq
zKjYl2H?7#Ie74RbrEA4vdnEmfG&ZxHF1PqRRorn-V0g#zn~KqCfu}4tp6yg%Sy(FW
z<UTWEMdI^~irkx+1vV*bAFaD$6SP2N^34^GBW>5uJfvgO^|CU#c)R}B=`nuCT;rca
zMLwy|_EAh}2>Gae+{V$wuc`ar<h5%qu&oggZ7!E(bYo*(W$c&D@3hfGeqzrx7LB|F
z?wZaF#VseQ9!eE6s$?9kOirz~`JrJWv3TvmDHlT*ubH)(?{nb4@Kc@wj1HQN(H{bz
z9RDwO!_(mWO5szcr=1gzZIH29_Nqblb3^ZwJ@3p`3Tm7=$X>%<Il=r(&0?ozH#WwK
z%~<MZA|?5<*1WKGfp*-Ir$$l|%lJ)<%Q?Nz9GWuc!^y`+{TGhu70sHndG$Gg<Rhll
zyPZBxj^AFtXSQNv)TX<PQ?6Of?0%*Cu3SK*!0`FDhE$Qpix<zRnW)U&yz+*1)ch#n
z1+Q|q`WS5Q*;wo%|IMlJt!tC(;)~l(t$w+CUx2Adq9fPerQOjRIoaB^gbWHUz2G{v
z(40AeUDq(I@8SBGownA7{W1oOypFS$c^;JF*>`WHqVvoE+gSn86HM8PZItd@v2eb)
zs#!5_qww#JXtg*0XDu%)&fan8Z`C}e*&3JFuU+iUG%fB@=)88}bbIDE*$FMyM^0|K
zcv$a@<q@`^jcz^H&mYc}*}ZXwN01WtjoY3_Yo-Kf#_rKtZE9mPZ+ok4;<93v*R!t|
zUNha&QPp*RS#ys|b+=pemV+<sLyL|FZQ4>_R^j<^q3uRa;RDS@jbRh++<JAw{Dg?g
zi>;L{ty8}4*t~pZRM~C)pb4i|>afmUz`DZcj-UYR{<WMvnRlWVa7GERMknygU(7wX
zxnpYW@=Y?6>%A2|bLgrV?QnW^x_S4M`?WjHN8eM_oM<lMDYW9+s^X<{qI7m{+xba1
zXU1)#d0X}_@!qxX$JxFuyHZl-mumfgu&MgUzaZBgv+H-AuAlN__x{_$|JdC)cAfVs
z-1A0U>4K}<zqfn(R^NLtCxSoH^ShYK&u=Sv?t1?Hso<2T=qGdNYn7YEx_z&-Mb<32
zRP;wA;3$`I#wEYQk5de}q722}?4GgZ#-YOpr8fCmTI<e|Ua;;)Q^^e_Yv;WOT{mv_
zRA625fPaYstLFyR*#>Mu0<3`tM7K^@7<l0L)&qQN7O-wJ;1Xmwq%`MI-;Za3vGr+p
za#v-i#8v1e9D5#RTm1V}TZ3NL*3Q|^D=xgSEGm3q6FDPv#i8~qJFcme?Jx-u)T!PR
z6@1;}+8oce>+U|fLYt0ADy62BpKsI3Q>s>M&OI8<@ZZOE;#${@f>FsY<rW<>`E%&6
z%4WTw=bkc$j)-3L*~WQDYQKNzaVDt+8au35yl#E)KE8afaE~VAnMFeKf}tBV45Jlx
zTU*!NWqN3<#kz!N%FR!Uc^<#EjSAfQfPYQFos)ClF&nUDp5xoPfHOIOf6an5Zx*nA
zZFrXA$M)5MjZ<0i*Q9`}vBwnF$knMBu-`W*TX&uDv4F~<yJCzVTW;?;ptIx|?->uT
zGm6HZw?u1eJu(C=Yd^jXb1*!c_*eYwq{i>MaeB<F4rIg`=3nb#ulX=@$|kwp%yJ%|
z51nH-65wWtIazvsxAlszJ}Y$Axie?1JvL+Q+6ld8Mdvs*E(it-9zOG5ihb_vUsF~p
z^WVSy{DO*~vC!N{X*HdT-d$mw%);D0ZOg8US|?T?dKV|R_L6~z_u>sAzD|#SEEQXl
zP$&E7^3s{>e7BVZ_Fb{r?cg$3areC|{<ZFFAMk&5;7W_*-}m5MrW|Y30j{kRekS^H
zeO<u0FW|ks9mA~mkCy$LeOXs-ZPy-Ir@#!BuJ!&(JzJ8@u3z19fBl5Hhm&kK?1~Sn
zTybr3Z`%HP*;tG2{x63GUt5cKZ=Cn;<)ewq4qd(O@&Cc{yjLsEZOn`pxw-%RUqRc|
z=TkK*e-~FAzB^Gr^-px*TbskTxuQOXcuL>$VEWIvE8X*O`re9=dp|0|J_YVydn%AY
zfOFD+4iyUnPnKp@K{p8n*Hbg9QWj-Mx~5HOVdP}Xk(sbW*?ofa(k-iwuev*<Y|V|U
zw?eu@Sv0ranAmxv=zv%dbHvu9*3G>ltX^y!s#a4LH!*rlRZCsv&@_cJV0q4sLyg>=
zMyXy~GoLP7z1nQaj+}=^4b72Tn0KrR-F@wC^p3*E+jL)-F{p1V`#A0Cl4(BoHz~66
ztL@pSzuc(9JJWa0j?)dtIs~2DY<3ndKf>&ld2P+q&uJItI`{vJu_}2uVXnoqJFm-5
zO<TE9-DhU(>~C*ZY@O<JV{J;~qy+~XnbqH1OrF5#D8zC>LZRWo^OL4(0dg)(n)B9f
zd4J_z*BP}N8_azk8vSo&uH0Gh{hz9FL+#UV_2y<}`j($+gHk`ee?F&pbNxAsPaIoW
zCQMpzU@@CT!D8n5S*q%NQJotMI9XR4EMeEb<Ke`o+%jQtpo7RnWiz**1&hthF7l~F
zN3Be42%A)p(cOOMc8jQ{iiHNJ?9vYzN}5)e9=Uy(bfal%RMf|G5#6Ybts>JTo^U8@
zvY+-8o|JR($Pz;*8HZ(ti!OAkhpF9qvRil31*PM?CSC=~OC&-Pnk-M=ZsO59rLefy
z=#Iuw-ted`N2f+JseTDCzowMm#V8?~!`;UetbJ*=+T9g;TTiX<4z{s8`#IR(`RStI
z0QbLJY<-_|>s*~+(W-Mk{B*}u<^Pe-7mJ$46z?rv{4M9}RolL*rEe~Is+xWaay_lh
zzwOhq8LQ0iEov=`$`qX;Y>~~}QM~vCN3rqZ9)^tII}=1CGS%N)h`ngLNi6hQto3!F
z(BP`{zgw>*em)u=URU~8Yzd=$?owU}_JqX`nj<COJm{6Ub~2$uT5Y0oE9>fr4Z`dn
ziW>MmGao<dU9iP{LzKrvhoe<FE1x@ty^?em)?G9qNyN-!p~upw88WVFGq*%|PS*A6
z%1jMwY5nZ`sE4VnTUX9zW{=E!mDFaHGqtZYS%oGrSgc)J>b1fueeo%+zgs?W1z5*;
z-sX$GmKU8HexXk~&p}u=f6eAAR{r@OJ2!ac|KGVLNMC>3w_U3Xj{Nlcvwo{Y_#K13
z4?;dg6C~EJ*?3fI_jdhLa?C4v&X`5<Y&j$LI(zeF-{84hE;E(bUgx*``FhW%wBl=K
zca~1tcI#owXWN$z^QAj1pBQm1x_yxCmItTNuHQR$&vlY}{Z4nq{@d?&T)SLnckbg^
zpZn3NSH1R~v9^xge~#fz_=bkrNnTCM40U!m|9ZZC|3B`J^EWs*ao<*XB2alU(KPv}
z#<kXkVLmS0y4sVE{khl|wZujETStf5tAl(~uOxd*<h0wHJe;J_(J5-K?AqP5WOajQ
zB6na#m&lf5i^b(U7!9sWWLc*=PjkXUzsgsa=SL;7tIK8uv`U5iztxqoV(Y=T9SS>-
zG6;QL_vs6FQ1B8F(cfV@Ha}B?mn{lkxywRde;Sv6q>5PT5q9m5GXr}weIEaMXC!g1
zb&3AlnUAFv-Q5$Ox`dUkQTBM`vBo}*E6!pLQ^*tp)~vVBqDmh1@>V`J&Czs-KA|kI
zW0|pprx&C3%Ts<|EH-g!9P*B|R=FzfxYg~$){DJ6UU5(DIyA9<Z4lR4;WGzwEH_3!
z6Pm7ADB;0ovGHm|XYy8-YgrzfCW^ZTs7rJ<sLfNEV0a^$M>*5|!YPrnlL~oMEt6Yq
z3zE9KZY&m!$U4q&%du^h;Zg&agB}HYwryQ&>XH+>(<|)Mg|@Xdu6dqM)bC9Czr(+t
z?*O0QlMZP%;{e~lRGs9T9`_uY_I_~*Su`UkCYFPJy1Cryp4ByrYi(xDh?9L*{b+N)
zq$so16eTv}8Hsmaxjeo!jVsM~(P7c-m}cuc4~|W^bnDjpnp?Yj71ArN>`d9NaZ~NF
zN&;8R%M^9xmSxv<y2?^!W<0D=-TmuM>Jp{DFWgPjt@;+~FUyTmoqF$zz?&2K&RMD2
ziE>-QxOH=qrElIi<`KAP!jmU%Qch|;ixej4Zc%m<ywSq0ykYaS2~KJHK@r+w5&ZKq
z8<lh}x6GI^Z5i9cF0ZKIg>#LzF0bBpP`g=atpNYlZSuc@x0W~DX1JnuK|U{I!MdZN
z3)n8O{SVYz71-Q(#jWb}1tX^Ig-2F-i}_AnsjDQub4k*>m2+mV@?Yr{n!Y6OM?$H7
z%-ihRABPri*i*iGU%~g@N4)E0oRcoP-pw<&_I>hh9joVh-=Im8)}$~o6r3zP=vk5S
zAf;Yj&C1Bdbjso>;^KQPv_tE@hs%ieTx^-NI%1I<SIkWTZ6UUe++T~P?`yfCulg<V
z$`chANp8>o4oaI+ic~y>9GqCTTu^GuIVkr+<*?@-;|a!77m3cUI3;Gbuu(?R#hq!U
z;>4-$VM}LTotipB+W!8rZFS3h^MdPiJ$Ozf96IwY^!j6l(9h4z-F}`uUHY;phJ`(F
z=Y^*QqOrxZw@%+-8vXy=*~W!3&-Zw9g}n7Dez1A!51-?18~WZYT`agdoy9lf($<n?
z>!QCb;@mv_^8=HpEjP~!HGMj9FZI^)TS144PI$>K<~!!g_K~yPD)SMCuD01_=l<xZ
z&(8v1Mjotb^LZAawd<1TubQ>5#hV;9iO80+s4JYj|F>uD{$pZ1>yEiiZ08bo=*{dA
zDYxBt?8brX96JMi%WY5S$Syjhae+h8bcrI*aare*^lS0bGZ`egQeCbx8J=P)J?!u{
z;evcz<+gPS8vA))<=={{H29-tk;{3t%jn6s*xjWe>so$CYln3$iqKiN^4O0UX7lin
zW4hOh{3G?v76na>m>2WJ=iXiOZSMX5HRaV#y`CXF<#U$7jG)tt%E~=5bc3GTZVhXE
z^1Jv~*R4s?isJ4}1)F%jPFELMJ*UCd^OQkd;FDl=OPT56Uk}cClYC^#4gLI^rEz*Z
zGXx)p`<=1#*(AMLPGz-7bHgN&2O+-ijF;?od{y^-=kEOI=E{PpcbvChzcqV7li`YK
zj*C(c7qfiqO8?^~rvIby)r2bUwWmaO{|VR{uyxh!;~BHwHfbIxsd+T1=yCO<;Q2oi
zzUqJ9dB@W*dH!`{H`5ljfJKSFB%SZ?EA4&8vEF*uG>_*d=6io;eyk3cT_m=C5>uAP
z=LOG0Qp+>4HZHt3^XVozd#+{NcOM@~ewQqHBc81|)%yRIv|H*kSht>ZP;i_uY4U;l
zJA|Tdex7c>?WO&~AD2yU{d#VH{KKkie353k*%MwW99T0^ajoVAmNT<@5**^S#m~-~
zr1`K(?bi0*9n2FRm><00a7&8en$ZsKFWdRQ9N^h=;NTC3rp>KKD|Q8BP0`5OagU3$
zW4DOM$2HqlDDB^|=+P_b7q@o2IAo!F%JIpfb^YC@6L#xG%{G1eO8h;S*oQ?upSqkh
zmh?=W>{RoU{~U*HQl>axLzn6f%Xzbe*Uwy^b8^;CCaY%+ODAm5SMl!p9xQH=vWwSK
zXhou!*b$4B+?J|SOl_5WRF7=>e7mEBsq^)N&gO{|xE~u#V34%ru@*=^())jh1n=$r
zd!l&F5?tOySl@GESQ6C5<T-1}1&)9ehX9u84u3cfzOZ&&BE<TDf$zzRBgK1~g;w-t
zNV93Ph8>nZtK7Y6$CSMvrQ><r9(*=RV%c|Kqee=I@!?1}i)1%5W)-X5t2RyO|E1AA
zanaOwk__*E8yj<NiLsXduVHfA+QeD=c;Ok_dy6$oQ%okWcGEjB_uh}*d6!IlH;X;s
zXi~a7ZDG(pMhmTimmQo3`rcmXd!ESJd}M-1W!KFE3lBzi?>)hzvc&EEDYth!r~BP-
zvzOjqXrmo?+uHGq$M((pe?_iN;y#e9Ezz`c{uGmKU5(rJNHu3z@1Bq;5w?Mqdt<Av
z0Eg*+7o*L;+~=p*PbgVD&!tOqgQtLwW25%LrrR9h32U7UEqF?F%@vrNlh!(RdhzOr
zELS<Wl4k>Bu*Kb^DQ&YYy>4ne$lC3-Q%P22Z@00Y|IQuT*X&T}vD$UTdDUXmuuWDr
zw-#%*$V5h39$=aFscGjl>1F$uI2}2%`17JI88>%&bg9(195%LbKDc8^{zR1@o>pgj
zj=1gNlzge9vFbGQX7Al1J@#96O*(9KsAK+?PrLZ4ePg?PZ^&5lEAz~hQ4`wj!mX)3
zL1TGwmi&Ye*Q+t|ss}oFFE|{W?0T@nHENGT%W8pYZB{;k`DK%j9(mklbmH8NOH=f;
zjpMGko_c+*<Mq+j|5@Ee(cE!1=UX4yq||VmF4kD><u+@x$geZ2PDbuMdBA@{mhCTw
zmHnssCtJ;rl{UF!?e2Pc?>+AecQ1?XIl%j9vHRo2)-`vIKVj~C5wNdu#?p5gTK-CA
zZyRPUn9y*0N9XI=-R_t7y_?MQ{;}5k-E(e8%N}dE=o-9<o#o`W?EQ`*f`4Y)PrP$6
z>9QzC<?639SO1UhHCxd!>j}r~nu&}zSj;;&JbZJK<I8Tw84Qim2L<08(7EcM)7rzC
z!K_!Rv}(^<&V)_T4}^mmrx?v(5R#ZGBQ>M!r<bMZ(q0Mg%{r&gEOxa0s%v8^w%uo)
zyiW5zLp>+fkYFuU7H4IZIi^>04t@H+NmN^Az50{WhnIvLHaW9lhLbmI=k*^>u30LE
zUuH3BI-hiL_W$kK@KFD(%k=+Qp`0ftGI=)Llu)(N+z|9@$A^<wZI9^1ot-}SiN_U-
zqf0IX%(XbeIrGTDfEn>kE}NXg5-;t){bokCv6MulHE*S030LPnVXi=zt4n@{v4t!T
zY}s&jR>KXa-kY0_>IiEX&1kSW$;J1OVU<oR?*pdwFE$A&Ft5&NE_*vwkDL3$nmvhU
zj=tJB`Qwhx=CxeSTPJU3u=}jzW@3GQ|Ia-;3yxVP8+X@kK9wPJ^5*$}XWZ;hZ;m=>
zeBwm^q_d{8n2tL-AD?P^;Znr$k`(tDz85|RiTHG1*zup6Z|CB^8WoO5FQ)|xcR2Y>
zZ(`~ECn>S{vf#rFym}R;eWj+CvPHY@Y<~Y~U-z1=JvN$>wX#=_aLm#0{CQD+qwUF!
zs-6;xtYHEhC#+aKVW%%Mm*irePIXWFFQ=_Irv#}^cG!P9TB}vq!1k2dp#wYi>NaO7
zowzW?%v8gCMuU%Gznx+K4n_VA2K}1~?wDx=Z{2vTWp?nnvup2K>3d1;?R%`-*Wq}v
zbe+@O)7{CZ4WtgOn7!-LkzE$Ax{p`vJd$E*C**T??G^9Xoo9C}zyGx7u!z`~ET4d<
zJ5TCdeKK{)(+xh0q#M)%r~7hodK_7^>f$WtBJu9k-Z$Rfd+%jnYNUVSzoeg}&4QaO
z);F)H^8MbmMcO*i%KC?^wScum$nQzL8SG{P*GxmC7F*aqJfc!psmiI*dVDs&ZD#PA
z7k(EOaM>>8T0f&f*I-49GUMhMOwG3#)U{e~CiK{4wpOuD`SF0M^_O2`@8!_o8=pLG
zyh@Jv<iNeBGU8j$!;Y%+dlvZbTXFtWpslQ*?cXUqS8w-k`nYN>m-gD;z{?3YKUrVc
z*cExGz+{%~9Ni2Rj;mXC-oDA18}}r*ug9YA*=<h!wtezn`4-IB{cuKTr;$v|N15C=
z`%ItBJAPx|+ui-*y@6I`k1Tcu#&|Z=SMC@787L>qljE(eD0;Fl(##_zx^L%dLF1?$
z|DT@S_eetXZS<#91J(qiz6U`@HyU_7Rv4W<XXLs=pI1b1*(uHk4PiT4jTUryYBJAR
z!c%o>iv3L{p$P)ED|hZ%aQWQdQ|Eh?r1|cOsjagK<<`>h-uB?2<J~EazG;eo9Cxg-
z2(~&j;k9x6+!Xr~r<u)HH1C}D*?Q%}sVi$a&Qvo!@2OJW6rl3t>NKM-JD>jD`846)
ziy3^%H<!HeSba_>c4@3bpiFnv(ZfnEq9$HWc5kj;{<F0|Wq!=fX-<1|xvM39<%Yc~
z%}Cv=`+e!z&T02&&N=DSb|fsLT}-gY%)&KL#kG&&`qP`!B&E70tG!(8_%i?DL`}u!
zh%K$ArAN8NR)qXt*UhPr5bhw#Z?;%dp>ggq9-$jd2BpnLC!}jv8O5k@Z}#v%9~H4|
z<GfbYh}S3gw6Pv*Jo>7ml>2q@iB8`+%UX^1CUi@?JU&`^qVKF*h+<@9WcHM+{1cBJ
z+i+;>=S=?mlA9`Zk7nG>-uv!#sY%bRnMwC&TG!^pai#VtKNf7%Ing{NW5*@Fpc_qd
z=H>|UX1BN83Uqv2wq<elosPGkb8@!aS~GR;xgXgHv6%}*T(&QMzT?y5jkRw#x4zxF
zOCo5->TSL$n<r232{lxeWtcrDx?YE8U#P&NXVKH|Meo)XH>x<=>yWz2LpoCPRZt%D
z9joq^-MZE<)-K$)*80RbvjYz6|8MY8%{t}0lexwCVGNshY}u6LolhBs_r|fMwRP?2
zj6Qw2^zP1@XV-gUvz|?N*rjYeDWp*C3}btbPw39ud(#|eH3jGj)P$rzO-+9`qu_Ut
z;@LTI=e7tKX`G9&b+AaB_SW_abNA~Ts=lj!zh}_z3BGcFwuiWDNybf=`1WZZCGzhl
zbj8OeGWTtZ@6%W=d`UF#>a~B*;_qwOM9FBFH^f-KxW473k?CG8J&ndIR?^FQj1D|j
zZ>-YwU1+11F12FWzNVuKKT7Z$+9$sMa_)mlqGVk0&ovKI?r8j;;&Joi!(Tu4DWviI
zi%~vea8sdAM#U{awJ!3iBk!ddk@ie>Y~pX`{Fm*}QPX;)xQb`53*S@ie>XdPckF&B
z(RJ$Fye}uFnx{QB&wFedckyGg=9F8vf-X#BnW*?R+d2O5#WPC;<mY}(oopY?AN^C&
zu&2}WP{(c6UKd@DyhmZa)8o2!xxQWUpo!~X(EA5WD$BjL3hFv1uH7Q|zFawf_L6Tp
z@)|rovnO=S4*co&Rqp$_(u47OVqdpCwd)mGHt+qhg<2^G4+cs;bC~LV@yy}+d!c(|
zJ|(>>Pb>`i&>8aoMq2*8G|$=)@0D{6uZVHPr<d1N90?1pe0RZZ%?!<g56m)G*A|M&
zZVK)BoSr%DS;jTD*u5fdzVjCN$Q|ks=8Zk+9FyT9V%^#KLj1pa&#g5-v$UVMu30K8
z&m+#nxlm$)h33)K+Eo@4n|NPr(dtbS4XT=A_RB2Lh+B+jc8GLzi&10MCiQOVWmBYe
zl$)cTx-OIK-l!iD*8NIf{qv0jid&9Li`6CWxSrM7`fzRD3BTnLJ+|4EH|N>zHWqqb
zobWR)M*C)*;?13A^Z$9sGqFx6&?%W&TeHvX%dDj)^SiE=$*nr9*RftTO5bg3mt~Y*
z#FqyV^G=oBI=wIF$^#kGe>s!Bzj?Rw)|5%M$0P#O-M*Q38uXl~y;i2Z>F)e{?YVy$
zC*JzEB!hcF=HnUMffE0u?%X?f(nGH7g?ye<$U@x(ldiA0ETQ5ObF<-n+Ns0;o25UN
zt*zdjQaO3;GagPJo|vU2x*_i>3jOnce2*#L`1I%W;EUN0f7V!ZdwF*@=QljK)Owns
ziS2ZO;|c@C!>t@Ac{GFz7ai$h*7G`2QK)>pPtLyS&xy>(9{tLS^JFF%sjlqQRoo|Y
zA?T6MOy-1pAv-5>&(sv2qQN0hvf{!bmrgZ}Ss{m(g)N!m#Jh9iRR&F=Ng5st7_yl-
zz4qkIJk*+gcvFg$+Y`^iwXU79Hx6ypDAkMY=UTEM^J*Hy?A}`ae^0e73hp0j=9l(+
zQ@-(GpKf$q%^tx|3}MyQ+cudl2-_VcW3D=3)slBdFEYQ&n`q>~IbD-$O3T{ps>f6I
zmYLoAUuJxucXjvn`p0qwvlr*xm8s9xOOCcEe!#Ia=iwnvt1r42*Cz94*B+j-;m-c@
z`AqY6f7fK*mb2rWtQGsl=v$lj`_-=b^CJ2E<MZqP*E#<E%i9+^p+Wg$h>)aesfmz!
zs?H6C=F|zF7BmZcWn65OI(bwub?J=|t`_D338fC@u82n+x-(BKbXSX8lH{(JRdJ0a
zXwSB(y-utX9r`TU7A)@fRGGjMoU`GQ*F^VCK~EUg>fC%X$uBllIbgA`Wn;wepYBny
zu?2@xCFj;ewzVhpX)c?glBd!bCcbv%GNzn;ng=uGR5u<Co{@XxiEEYBP4)Tpep8oE
z^ZB&%qDC8Efre(ze~qXYzDqKWO-!GaD;VWol=U(&`~|OqW68o?!-l2BdvC5}5}r_a
znxXv5wyf1<TdW+`NN=nXTs`Zq)dAg&-@@FgbGHPpUcGHnk(lJC&Z#-sr?qD1M69)y
zmfL*U%=_&Y)=+J+ZETslb4qT-vBd6RY2AI9Aw;BQLIeB6uK|&K6ITaB<sRJ{=q8*w
zW#zsg#}Yv=%b(fneNFCOIp)LW-m<|(SyMdtz+*4N#xS?QIUxaJ%p0bNtg*SeF?edN
z%T>3Of~U_`EI+iZb>sVKt$}-(_O}L{VLqZG5Y4%^^<uQ#+dcUn^R`{`ig)AFjgVd=
za4m3>XRxS5g1qJ48<)+FR%E11{vTQ<={~pg$xVg@OJ^QoNMd&V=H@UpOzT#T*2T+V
z(LE}z*^YH8X0vCMX}L<SZ)2I`UMK#_O)_=DK}V;x6CMbQ=ebzT*~?nA(eueQ-@dz5
zFG|~gmOY&KT=@CPIJsTb3&p?Oo61m>AkY4HY2Ex+zY3Eo>MlQB9@OyXVqUe=<<=t#
zE<%6LoqTWUXi?L~FK%6w!gFfM#CBy{ex=VcPWFFi$}Q?p>sH>j<Kj1iD*1hFYyX;v
zOt4UUn%r-rrobxcF63$QM@fEfiuBqEj>=)F*MCeE3iEnyx5ZkZZpkHMy^lG{`JIw8
z#GWkdm~zrtahF`C+gCx$+*Y0`2PZG)xT@mN`2Rz*(w7JG+%yikOgxk&WpzbE!I;Z-
zqEm`=&y^FqjKuex<jfbpn7DjRvgWDCrP@wj!OOdT2<qK&43{Z%6$({dqw=aPw0XC)
z_Nfi5LN3dB${#pY2^?zV+_6w}^<!S%)KbCd2?ryzoTqKw#Hwq4Xk%|t&*NQ>Zmcbx
z`p$-lYpUb7hzV17a>sF7b}PJgU0;3obh?70fV+g}B=@)!)&4)?ep;Er-p5oXTNch}
zSoUDn;t(HATdyXzg^xB`pG_4Gnh_CpPh|Q3o#{F)DKniP7;R>~^d_ynrQ5Gc%+vMX
z8%?Fh9+#H2NgR{;oY|G>V`Q2ld@`p%Z|271?jC|Faz!q=|C4`ru$nt52iCG`N8Fq)
z<Z;0KiAxJhiUOnfx2B_J8EIU}HZhV<BqwT4I4HXPCPT4V09VA%WtM`1_ez#2@AOSz
zUwCt)U&V&Y7tT*rUvz2Jc?P%Xmand~TQO(Mt=aJ8Vt>+0dnMI?CMLFt+?E*&M5SId
zhW%Q=F?ETqa%aHzfG2wxOPs3JUrq1}zH)i~x)-i@ihF!gmhA6dt6ys?wUni$(nQ(u
zt4E;erC?7(*1}sSazZzqf3rztl8^3&#zdvbGgTtOgx0=PDtwi;{L(G8J`cqVImQsq
z*RRwaCoXF)|0|IivO>{qBGXi6hZ`{`6kHO70@voAT^P12$+<t_a!2%m|Dx-;k}O(s
z8o0MTNLpoP)!4m*G5u|)M_TT~E+dZxyvut}tg$HSem*r(__&pr+e+r+8HU<XyS6@-
znZ8vw_3yUruen0yejBfwEAzVljaT^YS=t_RPHOMEa8$fhxbsGBU{Iwd=Z!LRPNkNI
zOA_ztwjNrsZ1UV6DpNc{9gaP!@>+4PGvKhy&G42(r~D?KoxCYTQdcr*UFVfI(=rPq
zT^Dq3t_d`s{&Q2bhlI^^UXINzuCqT(n!vVckzldm=hj(QK7@#$+H%CGc&)4LDoZV&
zo4&cXe4;bYsV!Yw@${(km1WI^D;jt|oHbv2LMLDARO4m8gL7DCHn=#2a+xn^_x$vR
zSMq;ADm!0pV*YW<$G#>N3QTe)3!~1(Jhw_;x}oM~@zch>7bkK8jW?&4ERE(De{ebN
z>RvzlP1&DopUEt^BWYf~C0W4nN6q9JdK^unjXWQeoVsS%Zm2jNy=7)p%HCIw{Js;W
z&GW5&wTL&-*X8c*F9ESEc7ELX+)u6N|7sFhqRa8t!{2q*s?fhn%yzJOMEKqgKO!u9
zi&dbAr7*=^;K%9s`QN>Hc!F6@SM4?anG<5_RvP<4YlVQ*?H0K?W$p~Coy63*S|=2W
zEdFJ}Ahj*2V}kD*!LL&f7Zs~)l4X4Cn)bAfv*4iUdfR^4Qw;e6g$i4j{#cZHFrsVo
zleEIUMb6SP>Hm-A?_R{qtJIt$op;+T%RP$!sBXTt-kp0)+xyu%&$)K2cDGyiTIbB%
zr#kbqKUF?`vr9`~b3<iq+}xD#$uqv>Rh*4p*O$+zAtLR?<Rm(E{{HVRKWx3CL?4L$
zowW3cZt~2A!k1rOzWY?sduHirL2tI5RW63dbe1Jdx%o9&Z}ZFu=8b19gY5e@9Lk=X
zc|7IHSwZ&o&y+Vm<e2*C-lN^OZQo1JbkLT0zT@iD1cuJvA1jS!TM684EBJPKxwgc2
z=~}MiVhpK%4sQdFCJBn1p6tBBNmlCP`R+&?;rSX*Hzmt`Zh2*O{!mP|9{c>hbL!s|
z=GHQruA9vCmHUF|%nP2a%Y&x=4?5v!(0ZYH!*v@c>s!miPtRN0xGLDseE#K<Oipz(
zhs7*Q^xd6qwHyq~U83R}y2>?+Rs4tgxvO9J8-7S8ybU|IbVC)VPl~Db--}xgZ&`LS
zC@<?Z-(Jm#K8t9EM^oDc7ckrkb&r|7t@L(6qszjIvy(b}dt;~Hp4i5B)Kg@umr$(B
zqdBhCBJD*}8lTP-6P<Z&U5~<(lU!?7C<tFX)unM~s;SIj+dJDE?n&3q6L}+c?B#-^
zAFpi_n7%_IypWYcekHfF#@v&8xlf(sZc?0dU`@=jlJNaLLT56kmv>$lZ4y5;;rc<B
z%lk!ILOU+SvP{#Qn0Ud_^#Y6F`JF;_o(*?yu>aq4MyUAY!-l1<4ObTm$wf3Zx(IJN
z>1q+Na!;(;($3!oovnhtlfN9j_{H^d$c+mTGp-pZskHfQRZLpuBFwDit9|7#vmYzl
z&xbd2F6VS8X*{}nt4C?Bhbv!ef~0L+f1%)Hk9l@CneuJq)?8Rru;rRiZp#zacI`jW
z8%4TvB(FTnWppZSH>{fXqA{|fMY3{(oO7<9iJfw}$i0~}&qeHsd%Z`}tVnd)5y^LV
z&du#*o&WZn#k<G_KmAINihLB{_ccA=bYaR&iwU1UBz_6#|Kc*MOXfC9=&Ua_{Xa^i
z9I9qHSh4-oaQwByUs-F(AA==v8lF=Mj!G?>FO)Kk;iS;Ihd2K>uy`m><aWR0<M|+H
z#+hWl-cEa_C&ITl8R}fNY<v<sRbZ>-DetOirXXK#U9J-`ETLNta=-MHbWB)zKz8TS
z+pN9{9vTf<xi0w{PuF=&JaF*TT7!F!GusZ<M(@7q#kOq$+qNd{1UXHUIpL=sY=3g?
zMPpp#rw4qCmP+TIF37!e?BPU5hM;=IM8&t4gwMHNt)1%W!_wp-<bFox;1!v6jrN3X
zjO`u^MIOyz%3&4X6P_&Obs@KL0*j+I>jAfpvnyjdFRkS{nR&;dVwpq3Gp3t7dowuW
zoi3>}2?r<Ky_tKrRhDggX~dO`yV+ec3~evx#yxV)=)IitC@qF-S1()h%-;X*EX(hm
zlr<=2mHZLwzGeXz+wuD>>;)d`PXm`f&S(pcm}y`XV_2mAT=&Y0$WHZ0&WahSM$?u)
zvk@`%yYl*|TzIO;+q3uTTa<U+nd!CU@l@SA^S6pvNlAS?nA{q0;{C~x#eYLS`(9|A
zwWRS@yo1+$&#EJh8OMWP98K7G>wD+@kca03&)#zJYOv{LUHA5QOv#eCf46=wd~Boj
z;+w`a^<~okN?8<FEs2#(VB|~iDVpFNch)uG_9Bh0ZK9{PZEW(7d#?3YE$N}a&On7-
zy*$E+sSdGEpTwpHxxHNV;-v7?B%zA|oNT@cxl9pdnTG@Wj%2d)m9fuUc5KGA1xI?H
z$^^#zkNS3v{~gQm+$REk$;C$(3Z7cpI5Dl%a-p2kjMz!X61C1BesaVoRPi9sv+}Dh
zzCN;Fr%p<C(vr+`nzCV1wCvrN1}!>w-35zQIr6X+-&`2o5IX;pq6f=Uv)m&qJRFaQ
zv?=d;<s_A~Cg{dx1J27O5ep=gGGiXac_jXsa>LCgVa1+FD<sl2*F|iMO1n2VX;UAY
zLQ$ja4;7Cm5v6wO`gOMpJm%?Utme4Ytb3(t+chWcf@9xBPiv&8OZGUGuX=6h7E@8x
zS7DUA$1B9RNYbS<YVV?(#=B;kz0;mksQmWjOpAB7TxTgC<P#Cz_V_)IysL^-j;nvn
zjm3U87nG--SgiWu|HiM2S#)1GILSM*P37jBW^mNM_1LSwQ{uz+Mr1MwxYbR0w<E`G
zn#LK6sP76Mex*6hQcv9(Qu&TvZ8MGbIPj#GZEeVhld4IxeK$GsB%YlY(7L0Hcj7Xo
zNnB5_#7$=^aL|6^<sKqdJmZ?=R<G;3m~QkXgl4trrX0I@k3Y2Z%&M8W+FJs+YgbOM
zd(Jy=<;+J9?oM02*=EY@c}t^yO?$gjHRe|4ymP8iE^`lf$S!!*omutpwC5z%jV^uy
z%M$uH614oi1&$=Yag{8}t68{k_M4dGtq*H*p4E2DIVSor{Z!AxUz(YFS~K-}Vi!q;
z2Cm4vbl&aF(VVK-Yl^`)Tze<Bu>8MSdnB*OCBKNxy`_uqY;3dUJKo&JLXImXR+-zf
zR^JTE*KNuE#^yVzApiR&ChIFqwi6=suQcm_Y5w`E+5QKUbpg+=MXx1#nEv!NmRCJj
z$!Yvsb!m51-_#4*FG?j{G?N+J?^Jcf?JEk&-I-qVN4utKipNy$MjH<ArBmMjN&nz#
z(C{uUw#dV;k#qUPCml}~e|z)z=)c8HY*Ux9zRzuY^LNdy6NkiI*f>uyxgIG=KK*TC
z;oT3)tQwhKZJ!KZ1ZZCCI(PD);A;6t30wFQdLF&#F)Ez*V~vka!nUm~S4(dD`ng?Z
z)AR`8itdg6>=u@i>)?~7d9&{J%~@-;U$h2u2LIPx5}P(9;^(zy`y0(N%h%f<DYU<_
z(f&c9%<_Ee4Tm)TYz*fr;?8U;{lYTq(c8Ozj!N@0XWDho>{@wm-$j-0Yx%mCMzlSS
zd^26<m-sxfbtR$^8p~r=nysI9hdXIn$eU@e_A6wyPZ96ZxO2l+_>jvh&54Ww8z$BA
zwld9d&;8g^t0Ep*P_*3W`n+qcBHw&gMuof2c{PW{P%*CU#09+%hhD8VQd(Ol+xGRO
zDL30&H{QUfD{h_*S88K9`DK3WG+*~MXEQlEHQQQ5wm5G0oBYj9bfS*$qz(E%@{(*P
z<m*4!`0JnXuWyGPUT?G)H<A0^?D+jKgU%PH>(0OT=^kFVdY72df2W5FR9cpYdNA%O
zDtXa*)BW^n_uH$Dr_J0K=e}QW!`#{x^-uUL_|~j5<oD(cng27+r+byJ`kQ*LsB<?K
zADx!s??3Ap*VeBoZ$AGuI4)<JnUg2Qel~b3^O}V^6Ea+<-k4Q$DUxyPmIL?Zi7=hF
z3F}|{R4qt(YwVSyJ8yETSIsuzT)V~gw$S2O51Cx9*t1H0Y?^OeCn;>yZSWN=FWf2n
z(o$S=<rz+m)?+Md-rkb)+>zI&f1prKd5fI2S<3s3DZyq=#$Qt7U7VA@xn5yXU@qFZ
z>_S-<V|kRjlK!VmYa`{!saeZ4@9)~SDW>S{-qe>x&(zfuOFjm@H(7uBV_96>rT_W%
z@ACCmiSqSSW(t`|`aE~y40|)Dw({uKIqk3SoHIC``g&T(<4@fM?2FF391y$Xe=VY|
z<74MF8{W$1xfe_3w!VCqc%ntA?(U^&vhR*9Q7bBQpTMJV?Cz&UY8qz6?g{tnnJXHS
z<>NPV#=ZIcW8V8@?TS?IZw}TLPWmNj?Jf2>EUnEft=E};SAF}#CU*Gj`dwa*9rL%j
zbw79e?Q*0=ZqF|5|6L}P%bwrc6vw##c(qYM@8{!n3qq`dKCphi=OV#h-<8599_`X`
z^$1_-_YMwEsoU(z8C`5DT>V?8d9X37ZI5{x6ZY~%_d*M^dlfa$Hq<RjZt19OWGY<N
z;}ylp`v3pd&^a8>#cV!$UDKFzI%AH!!UZR{?LV)kFo?x-=ufQBcAt@}8!?&n@78H+
z!u#hO3*tTM#_(w3Do;MugU9|HPQGuL8eX-OpCP5$rNunoUgczZ@sHbmhtzr)`KFcK
zowrR$r|9moW2<hzolzB}JX=+Dd#38)ZQCojb2*Eh7(P1+C_a3pDc5Cm=wGJh`Z-$R
z3m05tIXvOj0Sz8q=iSftU4Q;;lhmqR(zV_4r9}#kvK|~-o4tiLDeG)ryefoYsluvf
zUS6BmgefUAomwrIv|6vNH>lWj<%UA$w${YY41ZoR&ni}U_5H>i%ZBc*kj<VMON_0L
z-}tvjsAp<#xXXcK3}5~`%5Q0jchS#bITl~h8o#YI{Z~xIp5yT?`YFGhj|*hXUcEEQ
zq`Ym1TYH7H-$9k0A3X<N8<l<OdY=EE&A>By@#~f6rhdrgT`ymEX6D15kb{T5SY9&m
z72e(d{kh-v*27W!{{H9Pa}4Z~YLA%h4D8-|eA>>Cm|H0szS;#$JlU-w``H-IR(O4O
zR}nnY%FwfL>lEvv4&Ownb9I0DwS~@SO*nN{BX-M@&5B&x{!YC#>#lE#;nc0N;_AD!
zL!Tu$*ZsHMmF!#hR9w)E^SN1X)3SV)<=Nk!=WlCi-WQUw?L~gs%km>?o0^0F7#?3T
zA-vB~vA;~oyHV)MzSY}*xfySI_k7>~HILWMdbf6Me9X%;?edLt9^5MXBt5BgvWCIT
zWg#~8g~wf<9FsYlx!pxV?P|mHGW)h*37@3_Y8M#;WKO!@G5;GLx?eSuW#{LWZZ)&c
zuX%O+&bmB4eN~}+!=E|tjM}R9n|m;@tYl<n<ukF^;Jdn+IeM#fM7H8=73L`#0ZxSr
z9a%li%AWkVn0%sB(sP=LMr-eW#=jxoj8_OfYnY+Mf5{{=_}S?>#^sZGUUti_Ua&Yq
zP}Mc<$_(FVliZ0atXUzPUMe0pl~TK=tcqM-GS|qFD`I2(Nk0wiROYCyncICbZ=2=D
zY|q-6v$Fd7J2Ae>_eCFnix(suZ2r%uaVC=?=?L?Gado*Ht-?=?CR+Nc1@8==8I;QA
zbL~%M@$++2-HYe#*>Jhx%F>YS=jK!@WZz^?yS}Wpy3Fm`#=`qw&RQos8U-&a^C`Hq
z=q7K~=Xa_+KRz8eJ%@dQR`7*{cef9Szx#Lh_U;x&u6_DbE_H?eZ03@bO0j?N_xGfa
zcmJy$O%pj8xa8gQ{rWLG7_%BrC`3oCY!%@Y5?HUodNNqx5G#8?fx<_LITsdiUp<{6
zQXf;4=`3LW$L~n9i%wxfRfJ&cR58!YjT&KQ6AO8C*X;`A)GYF0S|z<UNJ(3M{Ua{z
z)m=Q7bXQNx>F(Y+E2XGQZE4S>Q<`hn7<ujHd7QLh*RCf;r;T<z`S0X&_>{_IZlen#
z!9J!pV>TNbJvQ;((zs%E@evz$mmcFICko!2v;WDm#p)uPZHP(ZsSM{px7Ec{OdDC8
zbk3@%dpI!q2phaUwR@9FGPmhc=F0~6Sxm1iI3|#s8|oCjb#uTuk-aZ>^cdKBa2Z7f
zcJThV^m%Le7l|a*h@{5W{D!1z-}ToUn|#G?rtxMSR!Ex|%dtDhIH_4QYi{c|1txYW
zPGuh6RT&GqPW;tvEYeguf47^@%jvL+Qb!U`eU8+LWwo`JP9B-V_Nra3qbjm}TF;-4
zM-@cmliJ0~vQ{pi7I;LWRjeyIC4Fk|agQvKWm_t|R%t9)^=j3s1x(tj*B;{f|5|&-
z*;$o(n{RE3Ts!-@*J@9{mzT7?^ei}TSuSMUd^+RMwkMN>_X`#8$~0PBSH*B*$CfIS
zh__qTZMu+Qox3^ImixEyk%Zv4Th1IYlYX=9XrJ%q^|_DF?cTL{_iSrhUZEHIJ1fMb
zUM~9I^~-3#o^WZ<{Wr^`^Cyel?^CG|&oWomberg(Kgl58|MM?1_b|n<4bmF>|4uTs
zRW|bY80OH<wqfZ37UT0T4&BPV8Z2BNeycX&o_JA3!-5wd=gt#jUywfKkJPf|^Q+hv
zwTec$MW#;WU;aK~N>Ap+7SZQ>*DrNlq*p(2g)iHUL+-jhW}mcVn{9;Z)-_B?V*WO@
zCR<O#`sn|n|9;-Gmlfh}Ijx+KCSfXD;LBNanMssG-K?|Cs`uh-mh|lcp>J~;mj9Yu
z!nO1u-!ZP1&6lRFF_O@B&lPB92vKB;oS`QAkH0PU*SQxTt}G9^dO@LFDUr2U)auKE
z<-B*L7+yz|Pf+sHR$_a3RcPuB7d4}g(U%S_SmJf4?W`tC*;mO$B4SGGD`p<)uKe^!
zLUYkV#*BlNwhyO!{<*rIag9bI`^%#rSX8=REnT6pT7m1Riy~XgS|^@2FIW|B8cyi{
zx=1c5qP0C`(WJNn9gWoz4##`KMV5B>a7CR+2{sWF_0M9Rl2j>q)}vBv|4gHFd+Dgj
zVVo8R3m#Z*6ghJ_=EZ*#NjVq8<%vBdT9>`ge-SE*YhG~cRKd1O4hGq!Uc9#}HqY8|
z^H|=k#xw)%9g13huL|Z^EHK+CDSK4JOX1ef-f})CzrBIyRHhVPXi=NgqC4SH@#mK@
z)h{Y_t1m5|_hU<XRacV0|2J#H|0Fs)vhCW=sM}&2#nB*{IbCs*!Ic=nt#hlEOgQ?9
zAzwAK=JD~2ja?lN3q{UGw`$CC?|ySJx%#ejrtPnb{JtKOMD@R}aF4V)S!Gb9P%}m4
z$ANU4#UiaDvn86<vPzV{q+ap}n0e^WG%b6jz3a7qR0~c0a@>CPFMqH9(hsKXW6?U?
z`_hM}L^wkFgu~?%oqkcPpGAo$9d`IH$Q9PybajnDsd|&blE8;0R|GzHIf}@&gy@Ut
zcv?7X?R=yewB@7NYAfftMswWS`_3l1UHdA{nUfTL{g+JDyrpY0WV^$U{}M~T&!Ky%
z?c0`v2Oq>5e_E}v?U2|y;p+@b8vO20_^2i>d9wM5=~Tuo+C8FC9TgW&7|c1M;>xCZ
z<I{@mT%EiJmLxaa&^mTt&yo!#vd3Sy9~CfbFkG`({<Q$J?VpExCTwz^D6o+Av)atd
z{JaOmwVWipi=w!v?3*Z?&2TfIU5hQS=FFtIQ%*G;44OLg<QZ*?;0;kzoTsUM^qCr&
zm3lGZ<#aA}-)Y{9pSCSdGhG?GHgoYrpNx_jKC<iH{?AUE+;i(fo8+qvGma+yo@M+z
zOyij6=EDBu*S_mG{cLAWE_95`URxLaq%5#cd5UUtcKMl|6-pu93$!^!zXw|dY2<g6
zFP-L)_0X!DFNK@?>>ZcqTd#GxEEIbm>!GW1ZR5;!FT!8Xkej{IeZ!Nf+vV;APt;JD
zB-gRVN%*yD#OFVEf&zOERbEl{`gZBYMU$=<%XBm+wr4mQc5ZAsJM&xIhloUFe<RT=
zk%^MOKZR%g@%*}KrCd$|dxXQmu2ZcXZ=51C|2kORmGbvpw_v`^=F|3^PeN4wlrHkp
za7vmiaI<>r#>HhB$?5S0(=XopxadvE5y6|KXFjmpU4P>+SK`c;sJ;KK7Hv5pE12vm
z81wR<*tUI}cg7u^rm{8lIfGVqbk79#?XgQ@3oe)H)u(re=9Q!bDX85G^vPKwEZwY8
zb+O+uY`a)lil-8L!G_1I`W1f@*c-S{?&({tlEj-o@8FBq3#V-hyp$co=~t;#Ff${E
zhk4zpGme=L0_QEAVyS*w^Rm&F&{s$GYbHLuz_>VZUESxVeQ$2**UywVuVB1(=e!)l
z`F*)-_qydhc*&c;Xk}^a>sf8_;=dgj3>X+x7QD%ncVM`XH-Ul2g)!Bhf3o=!|8q}^
z=Y8+_RLvrE;c_Ck{h!Ef+qicLFgvDjGc;(-lzbnsV?KXvpZvT0MHlu?>M?Pxd^hjE
z{fgZ})|0|rnHC-QTDbc8+tf8;s}ok*H(fHAZg5_8qMzW>RQ=33*H|hf{`%QQE#&ll
z<+0N8kk^XdFZ;INIq!LLeb)a?EC=**uTHxdpEx~1_8QB%TWg9h?ThlZDf_xW;13(;
z|CqqmobajjA1XrsFZy-rACJ~<h3eFAe|9?Oo|xEZ)vwLEB!d0s$xpNVRk9fb4k(<;
zQ{X62m~cQLC!2eZuI-J-w;vsj@Oa%jE0syVAkyGN#EchP)Glm!@lIh&>sBA8Z7fGv
zc$0*M8e(#~&+s`pmt-plIdGjiAe*;PL?uB*Ez9wT?~YZ6#RHDWS1H=(b;%t|5S7YP
zh*&89gLA3d|3u!1Ri7gotMa^DXS651c>UzpL-uH=OHzGmyC&@WvDBriQRRez@|gm~
z3m0}gIIW<S#-rlMbj>~0!Sjk6r*;|NbBQ%+Uj+0vwWK#3JG8Ck(7#8A7#0i6JeJ9K
zSl^~+T1ltzt^-HZxWkUM38gOnZIyV$$NAVfw_|3_$2R4$eAPehW|+U{yofo=35E1I
zU)0{JBybzO7WwR=m=<`7&#7dU@&^xf7I*a$r+^q8uCfyjACKvU#2ge=%5rf_;xSuW
z?$p1u>R{iTcG+di_XsW1xVC!F7VYH{C%*-)lin#P<A0>QN?o~Ved4)~izcm>wesbz
zaQQJStIqB0k9X`p{d}*6{MT(u=m=qJ;9`1FA9Z3)*4i}-`V@3MY&A6e89uZv@l|X;
z`={ruqWy^*7R{y_pKd%`ALuke<q?19tpoqsR@~y~R@jpo`0c!k?IzxD=U>%0H;EW9
z@-CVyc1X39RZ-?bbd+L;hjvFBtKurwPClnDFKvzeM{hY?QaPme{=iZe@k5G12Ffd%
z*!d52Yk4a#IUpg~!d|T%Tk~4MERpxR$K$@^&Mk*jpJ|-+Iaj%B<I+tRRDI5Ui#Z?P
zP@p*Bwe_tIm1`H&J{YM#(*Al&o$J|zXE}b)*@|_!g7tnC=-2fzvrK%^=69Iy@XG~j
zulq62P{?>`bXeG)*(~quQ7h)xJtvJV{tKGVTWP*w@iCW;BFTZK;l*p0INUzwH$SQP
z{gxM|6PM2a#VkJC<HLm3C=sWdTu&T~Je=~9KB}B&HD4-{)7Y(*!EK_IkkZeUxai5g
zK((Cr%NUlan0U#Y{8;XBNanPl>~TXIV<%UulPMJ}TK5X;+Lrw2T7P9(*Uw+-@;evE
z^D_TBa3qtB!}m<fnjgjr=eZoV^S2u%sg}Rl@Xt%-<oY9Ns_P0mnkMP~yVd{ySJwY!
zuIu$eHh$4E`{92==(S4VX5)^IbxZivgc&$|oEXnLE3!1*cG$qYdTB)4(aiy?U8g@~
zzP)&}-Rek}3pXbe*(RiK<8We?cKH-l5R)LJn3Kp{^xuF@{DKFslW^8U?&p^tt`g>y
zUAL6&*|%MjO>2{R_kYq0eW%uQT0TNic*_Bsr4RJ~8%@c})A905|7GC!Qt1%W#X}r|
z{_&ay7Cc$MLbBeZ81CY4*C>2x_1Tb3>DZ-bM-?_64bpjS_0cNtu!-lySyo%TwKYV&
zm4ZE)T+F{(3q6>tchx*2(9CnI6_?Y3xF;n+TdnRLj;;|Y^GLKf-p3u~$fe?*zop@w
zpNh=!VwZcbt!tgndbrq@ytlSvb4>E%dckTtf0c(Dug)7myZ0BpzFds@{!!seu>AQa
z^+#BKXZ0!+`mXg8RIEO$Ao=Nr(~29HN&~cnTJDI(=@%&lZ&G*uUt;cIq#78c>Uu@R
zOGvf-L_D)u{M+}+svA{eiWHMB#Tl#?->b{nxFT9ftfQw$vFuX#ila%_bvrNk-Z^#H
zUHnj@;3=^=#SFI?9dEhXJZROD(^oFIs&e9Kn0nLw8ton(6}6=fUiphuHx&0)IQ7k2
zonGCf*L>}1fm32dSVBpYUTHh;(JK?<ytqSCM6RgwW#*-(9P|CS;b4WIZX4SnG2tA^
z&4$UIhnKOvyrVK*<FMhBb0!{(U+GMK^<Z1uowAoY7lp!|UmwX7DLfP8dNqId6o$zS
zYB#x9o^rnD3xA(8(Y_#9;ahWIa*10T$7!x3EYXcxpIH{oxpMD|OZeenmu3G?U))h_
zEuJLh&d7UovNf|?iS6{#gO^Kdk4K0vl5=wu(R$nK;mCa}aAjAE?d+~A%SwN)F!f?f
zs+hYzWZx0_-xIyRZ2ooPl|B2X#vSkTy_5cUtyp*N{Mvv1an2!5yQf$0FaK{~Qe!-&
z=6HFHda3ivw2fk`3-9FD*ncY0FL!#dQatyf`s52q$yXSYO*qrPFsFwwXNq*06tQG>
zERZ-P_J&cZZlY)o_uLmtJ<MHVgTfZc^lM4Fa53d6d@<V7vtcn)W^;KvkG5{oiA{2D
zUc0VbP}p}xAtb`*`iGM6h_?70t=by;j3+IB7G*u1aKNckXIIj}XC(*adc4$p1enze
z*#18j_$eZ|G|g&9rqPZCMzVr4kD0dbVH3U^W};x2XZksCK4;FZ<gF_@^R=(`l=cLz
z))c*N(R=l9aA={6Qo_4s<;Pu?M|vHPjC*2dp%Rhx`Sb(jlP`AI=sf@C!cy7x;Y*QM
z)w+w7Zv4qB)OWk(tG1bnC5M;Ri^>Y8Z8KZzTBENu=T+(i7Oy6AyVXyk{BL^i;5zd!
z{Th#l!hG@Ti~Qc4nmR4=Qq}dVO2t3TN_NJ&D#w-noL(Lo`~KA5F3b4C;>nyM$=X-q
zK7Uo;`&jv7X*@SmBEJ)(;Dmb#n;2VJ)qkFhRtZaT{B&C)b-}U7j+n1=>X&MGC(T{`
z(&3e2cg%A3>;IRQtzf!e^+Bo1d%m{H{VzA27F<f>Ws0Bit;O_U|C`hOE?mi7?XFsR
zJHuQi#Ux2i5A*L5v!3~F$E0bQ=25*b9J#qN^^8yPUf$w+|BFt^vnkuQ8Pu)|IWW!g
zA5-SBX$A{+WZsyTyZPAkL#8oXEvCit7<Zq1y@gM6ci)QlZuzY~6DC)kF<9L+Z_PX&
z@l*3=vAjQ3SZ*RNZ(uuP*St*{7TJlGQ-7HkvD+*U6p2_R7T1-uqg|EnrmEE{u_ro<
z!%~u>#LHe*mAG$Ld3S5w%PQ3!?&-ltc<pDdeD}d^#i<SJTc+?XuIT<=Aszke&qnVk
zN#A2!(=MLc#2iw!_J_!7^BY_Kn^*H|S4(d>lF}S^`T(!iC0-4Ye*vvLY9ckUw`*>i
z*Pgz<^{)7~y{xnIk1P0XX}@<O>9%A_>Xq=G4s*xTwl)UynQlb1eM<IxG&}pCoDox*
z#uKFroviX)yQNpL%g?Z2SK^vwynem6s$E;ck%UQSH^}*Tak4+S@ao}28;jnU9hu@y
zt*MLlJ^#IrpUKC0#(wc*-u#Q?_&n1k)%K;dWM%UmY^mVCd?@`)Qieq;gV_N)@ymxM
z{P5GSS;l_cUwWdEN%B#r&ACgb2i)E^LnV2J#z$fE!$Q$VtuJg3IJF~K)AU$6yG!!N
zUSoESEe!K!Nw3Lovi;TUULIT$BU)m{UY7H}RJm$S%r(`?bp_=Ly?Zx*`MmA)&U-7)
zx%rE*IrV38CA~Ni9$bE0Co<xya^<BU8_#QUIgKJoe%vkVH@q#J(ki|>Kyp*6M|Awz
zRxRt_IljwTLj%6L7>QoGE_1l~j_CzgWmAQZRn3+&)Pt0+Z`~Sxd`7tXrK|51Sj<#D
zn<`)TpY?SsFVBTHPItHYi7;xoyT+G_wVv3Xdr3|I+SO)_nTBRbok3T<7rDNknw0uc
zU9>5MH;G$e2fMlp;|3w^)2)tew(~Q^low2z|8ZuT?hys02R-XH%dF)7@+G73x?}$w
zZqCcQ6YfZJ2ftf1G0f+`)Z+iCdYOmzhx)}QIK{t_R!jP?l|Hvj{a$TiMZRzbpN_;G
zgO3xk{WWuTvg8=9(NC$GrZ72E?(p;zZZ8?OzVdK3%9-^lW=)=|sEtSQEF-&o>E(0o
z>^N>ePlaWASIOCxi*G&go~I~Q?#{ko)oQ7EmzYoMx!djYxff<!Y|8XpwBfDLgSbm;
zRF<fFtE%a4-e{%yR@%PhQl8V9srw?1%bY3Ja+PRXZguRt!>)CVcN8A{)Yy__`}(8p
z<n>clo38Gees#ij?^6wxzl%407p~HecKCZIfjcDMBKprOyUiEhzjMAl{r#~&F1z3T
zeX>ne^+RTK4ZHm|?)vTg8&nz?QuW>EM!%ljc_L9+{!{zzddJmny7S#y|Igle{Kk`R
zja^S~vTRSVP<5YMu+?Jsi<|u>oPBG)_CNmG@BA&gK1OAM?4wPs{dG@w6@K4aaOu#M
zeH$nHysp0cyVWP&>F(>jPuUL0)hM3TZ2KP5!ZR&NxXxeu#uVN~zwfzkJAHv!bJ={`
z!V5~vl5~E^E<KsVuw|N;!k?cSF&Qj^|FrKIo-I54^8aDCX777uujDrRafvFNnxb{s
zGhrqpvsbC3#fD&~CVp?*CsQ^kIR$ZO^BvLHxcESikYtmL=B6b_Crbt{i^()va%!4N
z<f%C?H>LSZH!gNdysEYQ{CtP>Z#+^{uB>2O>@hLPm_b=OXbszzlP5$HSBGs&xpixj
z0_Vm5>zPGbc%>B_Z*ERI+BNm1Xrd5jVAj{kmfPp=-&Ousam(rQr*~%7AAH8cvn|f;
z@YhY(l0vy(X&B8umT^<b)muc<)4==0pJ|_;om%78d&y{7(bLm&P1OS@I5@q&=C#aY
zp-<(3DZ1N>A0IoL>h=EqzGmkCwhi7NA0L~XZZfZsd%Et4?&+R(d#%KcV%B7zKNtJE
zq4~l7beoO|dmi+NbOxyyE7Vqh`4AZt^Fr>S)9=Uk<)`e3w{iORKlRAHDqr3Ma&GL3
zH`v_RyN(Ao@|s1sN&gX@v7k{*uVZ1;pTHS{3>=ITCw4HaFtl_st1M*HT<daFSUY!*
zMQD%VHi=|6JwAoU{~63q?Rea4b4X)qNFZ;()Cro5H-kfU5~o}a;V`_}mB_JmvXJEJ
zlPZlu9JxD#qB&F*p3O*^G;^X*+`gqPOj&u`p3ctTQ()AtIy>b$L)BHyh<O!hN0!gi
zeEE{w>7Pi~@`d%^UiRyMGsyC`-MZ!Squ>d-3p>SvJSMzcmbd6<*s_c^uIIDo@46|u
zGH2PYTdP*C(>%%&d_e29i1d*x?KN8;Wo573>a_hJgA7A;%m$epK^^fEYOCLDI-_=5
zZu5Id_rNj{(G*?B8*!`OI>lymudR3>%JR<PN!^veM*exL^ITtMnOxebbF)M);?us8
zpuJztS+5s!e);<Sey7+cR|35S&n)@h?V4tCDl}3t#anwTPr_+6&Ob}SB2z-wI51>C
zRcV~Fw(-ra6H1$(ghsEqGF329d;1>oSfjw5lg?<erWN~r{c-7B($*d!DalzmhBHNi
zTo$sX&sdrE>e6zvzaK9vS?hj1v3}}{Syx4mo88XYUbr{9U=Qc?+XYGL&!bB?48vtg
zj;^eg6}=RwUM_z3YV6(en-68@Rn{=~C<Ze8)3pewojzUOwN|<AhgT!}yr4(-Gc6R|
za|Jvsn<cd>Dxb{upEu!&)HJ44KM$1#hpyO(ikWT!N|sCd9Mn!)Ik|o_nflhz=<1@-
zH80|JPu%kR-0v69-}3!Q)N$=u$`rKZOW}+})rE!s-J-ipwPtaJ=s%xzJNkU;QJv5`
zHq+1cFS@=!XQ6$=uf~<{?H0b!=d%A(`75aYZ||+-)8{5la$?g=zHvm%tWk4O=BtLK
zPQokPwmP&FXzIjNJz!ArIJk9<W>e0r1*}(vrm<)w_{zluN1K?gj=7i>zU@%&x`d^N
zZYw;=*<a<yv$H@)YTMd~6OR}+TQsgo_gvj>^TcWU1Xls2Fm9#$p64CKf_6+1Nl_4a
z>9SqT$+gI~MLG4rqvkCYM>}?C$C+~~`}i<TshSzs^W7=g>zsvKy=Q{(+JZ^<pIIDV
zrQ_-@8F(r<qVTvzYD=QdK`za8--K5LzMY_0Ehj7KZrrf(O{(#K$1@BHTaD*gr18ws
z;9aIOdCQfGPsa1VoQYoHZu?>C3h!ks_pfRlI~DB2pR(e}Eba7_8#!}&&32tk%|GO}
zEskrsMS5_bLI34Bj1K8DN)Mkao0hr#T|;-@yUt}kuY~7+S7@1aea*9)d(QG7EuJXc
zmvOMz-g$OD&k4sDdjeV%j`*EXTrpR=>v`+5P9dSCEUs~mhZvSPw9VsL>i&0=V1HDg
zvLfS^1qND|dfHr+l_Fb9xq4r`EjyyJ;Y*9HPolr}8k2?&-iF0F6ArPbDrp6#>Nn-T
zy5TiXT;0O=L*~L+4}6q1bjCRdDZW}g`OmrOE9`Ib`}Y)ys6O^+3uKvK#P%fQ|FT;l
zmzd2=n~YjlvB*k<Zs-wdvOAgZCCF6lTEW(2d6kE2k6Eo2ofi<kXVKIR9ja@ds~nFw
zmpLuw>8ryTuGwAJy7+}IPF`E^vowZn<Dy-gUaTvvJhGwH_l@MktwM7+*fvfG&5@Gq
z);GQMvBcv7liQOWC(iv-NwjI|n#gl)O4G@Sq6?PFh8cY_=%2Kx(kO6c`roasZcJ+G
zRbD*SV!~2EE@~N@ck*OedEN;yIy+&*6ydMeoK8fRt(m%hNvfNV<>U~*a09Oms=0fP
zZH;YuyZat*{>_F=vlOAvdyY3smp^hdPupdlUv507CY^iAv8U~-&vy3LA6aa1^z3(q
zf3gjASN~5F`fHb0$Z$-+RpnCrKQ6t40@wOmq-7rVG;sQGTwEUh!}Y?7k{Rv}s{|c0
z!!KOC`>u4oaYk3=@kdi7@`JQJa~tcDY?i&yGt};kkT#pT$hb69ZPQgDr;t@AZcH*!
z4*KxouLx_%U*4_d|D=zKE!(qliPDEt-leOit(+yKSLxa!wdB?G^GmMs+8LaPiQ_C?
zFKst-Ns+ExkKWHCQP(!k`fyHjizxfD1nI{|3b^anEnJXqaebGSc88sg($+VNT~zOV
zpUI}_p{~H>+C1UVrPvmw-A2AQdMg4qJTnTHx_#nU;OwecrG;<A{d<oEO}Ele@?w%)
zbSU!Ity}6|nlt3TYTWuiC1i>yZ*-@w$HLa=^JUy`BpPhhEKf$g5!>qdQ#aMZ@=n02
zXSe*k*G=ED*6Z~lp<S!zq?Ww@IoY&-_Svkm&AWE5iPg+GkiGLy)QyVBZB@d@_dGLQ
zwy$4u?oQLodnfQ!RImh}YsxMzxaMPF$*S|Puxq(fS8+#Uc+k0-9>%kupV@Ipc1__K
zNxcKF*L<B`^+t?y!-_Jytc`t9M?STzvvJp(^`bXsUO=PSl?ropq27ujn}a=ln(AVw
zenc!fvB>_y1l``t{r8_tUOc%-_3Y>Q3@d9KmX^J$is)Y%k~%AGn$Og}47cKoyZ<~c
z<+t<+Z!Pw^rZjC^OK4+hpLOK(zVwzq|6hggi+X*l!}aivb6?+|voaCjKi8?rbtC%G
zt!PKRzejYUetn$&Olbwb_RSMf`#u>)DnHXd|MTqnU7J$Q{P9=XFuCrmN}^HU#*M!J
zCpW&lnBez{Gb?ktdqVLPcfrl9+B=dqRxZ4mAI$x)(B1QdQg!sM%ieaoPRhm>OaK0<
zQ@C9urTO%${<ptAZ~LzQUiM7FI^EneFB5OEEx2d&Av&)x_*v<k%I)9YPcP^<^Y@cc
zSD1OiUisJB4b8J>#Q&*JeWb(K@Y}uc_tN_3ES4NEgnvB|i#?>Yi9Jd1sB*=ZMCCSt
zjh}^CTb21sRrWl#7Jl03pjZ@UD3Wu*T1X-%U^>Hwr^)}*%jFWL3d%Moim^58{0Q<d
zwT;;>6m4D-U@9E{z0~=7>g{Qc31(tx>0XKHDQCCWyeO;ji%1Fis<pn!skALUE!C+y
zO?v;O^wWtt<|WRBt<DYC&Fn+kN?S8*ruce>$XyT0oXQr_Tb`NMp<Arc-uFDSLL%#J
zQ^d++S!xdjW<3|;+o;F=O-{(L<I$JQ$xqvReq`AdT5%{aMsKiQ$zJ=#RAhFk;BjXz
z?x0TRM}b101CBkZ(fL?CZAZ}Bk79Slyzd-qIrqHfx~kam<29E`^Okee-YoV0(B{2&
zk@v0V-ruj~-&6O=ajyH-mVbL;9mDgw>g0ND!-DtA<t~T#`mxCWzdObE!4uzGE86Cz
z7Tl>Ql%DQ;SEV65$=67uId7qr@DG0-aTUg=eWv35rcNqMtk#Z7#rj4zc0~dD%k^_F
zaG9B@*e;X^E2`e}TsXj^`P9*%bBkkApHJ`?pJ){ky8VY}{B&VYi<CRgMf^7>#9nIF
zTvTRM;i#=BCjK%xF44==J-t}HT(8q9%7S6PkdE=o*7J&ORxB0u;w~`<Cbx^prv&Ia
z-VE;ypYlMkvhR3>Cue23M1)_Y-dypi^S0}CXb5$1baZM=jhY$Z!dPXg$hu^Mgp^0M
z&cjYoNs+*VY;2Q7yalV3Jgg5clUP_WLA7$4i(>b3mhNpUCP>_@)}0~ze@Vvlrj@2w
zkH>yui@VW2<FI2~&&k^J6U_=Pn>}ckxjeD%_1FA2?D4D;^>Zxa|2fq&{>b=weI`%4
z{6fp<UsGrOOP{4}IP3P2S>MgQ#m$=kJn^sp;;#5C(LAV8qpVRguz&gYM!jbNMu|3+
zEF#`QDq$D-G$K?NzexQ(WkUVTiEa^d6B6^d4wR{#lycrSH{EUS{go4LS~lxils&PW
zsC#0rY)D?3d-prb*h`ke&mui;y0)gLrt8j3-<2eN^hJ90_f|`hwv&@4pE|_J!sWsd
z)z;-E;QZ45(IGi@#)T4-TzR?VihsyXEz9I_%$$2{VT4O1@5jtkgN}u4ixxc-438B2
zuPL>t{f9uuj)W=GrW$1l>vjorL}W$0oSG-b=(&M4UvoxxOJ_}si0?s>J!#VtPew^D
zv<VIFa^RZrwsP8eQIYjOmRv}mbY;8g`)#p1E_TNUYajU0{aCc;$d3FIjXJNE_ssYi
zclJj-lX!vs!(J8*`O80gOQrf80}IQedP}N$D^4vhUe%!E7`&`plvy;uCUM1{Vo@)p
zM%JrEs@wWGO_l9GDVaSl3Q?5EacFWrB*HRNNLY9V2j|3qj1vBxxgSqXw_LGmO_kiG
zpR2qw=Ej<?u8*AVKi!t^i_MOxGQ$;RxzCnlrcJu)nNnobs$aS0XQuSFgJC9|I%b_Z
zb;~DPUzz;>N|%mJif+keIp@xb+ND#jFPh>Zxvp=zOI_;}>5mlwnG5|RBh*)QsC6ys
z+`7o<mB5!-g639Jf2_(fI2a*WXr=bq-m<aENO42T%Bu942J6F@upX+?*=Vmhv6}6Y
z*Okn!15zR)uOr2)H#(#(mEjiK88Ksb;*5@&wTWCky*tfL3f5_L%N7co=VX~zF=u`-
z-F#Lee)h|m2@7Yf(QJ6aVPN}O*y{GIf8q&`JsO0?6P|Y_s(tt0v2wN_cVoco08x*|
zohuvHE$%=0QrI}$h9Of$ZAT#AZIx$blO%snU9n0(^>wM~kEY1dl-t5{rLE^Wn{~gb
zl)K`(QEK)Cb<;JmNo)T9>|CQ3;#9-toGqNeaI4(XWWnXEwVbmu979~LCCPY*EWH2F
zerolUk{LU@-P=XG7S6I3oR}t{;<d=YYmu7OF1=a1m=mXJ-P$$lo9v3yi<DpO`mD8R
zqHES7bB2GbcFPMf1|`hdmNxDD^iJ`ED%Dejj_g)gaAQucfmhARZO;u;Q#P=s98i_I
zz46BEy>pi`2rn>=uJVq(y!2z4)0ePKNnQEpChj|RXw&QE%k+2qR2{C%zqRaqMOW3T
zy_a7sf3RaeySqsAf#UY86-;a92vvw~`x)@|L{a#`;<pz4`qLY~?@;csT4`M*H#;lI
zZBvx|)`PO99(ES1mV4#Je9#tl;JO>V`oG}MRr{x}`h2pqOQp<7>`?A$$1KBnCN}dd
zZx{-RG`l`4NoJg6=+W%(W@7c~601KZb=_;ca<ro?SkG!X?_&(l+gg56Yi*IJi(}_9
zr<0tHnZB_(3NdHaCcUYcy?due<<8VUQ<SxKEqc8u%O;ce<FU?&^(8jP%VLgyvDzK7
zQt11xsV1kUR%v$fXzyXE-ry3blC9}3A?V)PBd~yD70-=rMcs#VEg07x^v}?q5!|(Z
zPR{CiXZFfl*G3z&K8!v!#qm_Ua@@1*O)0NrCfzi96FqbK&wbY$%>}$RpHH#4aH2k5
zaQ1S|Epu&z9j(t832tfK+#u7L_$otm_e>SJZGHZ~6aNQwD+d@QZBtGbzPd#x@a&Px
z#SR-+1|~KAkKi^D*7Q9%A@+Io2hCa^@zsT=w@>0ZzdCCBCN1^M*?Ae;TOQq9wMQ!Y
z8E06I`09kEt4n9jy?J==#z{x&yXS9kI<@1I)4oSX8otf9WNGWHzIeK5L0ZSr{?iw4
zsqE}iTiB+2$<4AobIp`l+jcF~-o4b@b@H}_z2dv8eB^YpIur~q8|@X~2@|yHy}atR
z-uJHEMv6+JlP4woJmC;{;?VCUb8k#@)rvaKtvb8adV);O+6Q|k>zvP4pMP|s<|3&x
z3BCI-aHj|#*n3C2^}U+({nOpMHg@Zk_&k@s{$lo~ziBhKypc7v-amWi>Hn83qQBqn
zjsG}{XM4dTi!IH*y>^PM|0O&YSp;;hTJbMr1;ZPqw>yg7X;d?~_D^0F^~ob}R#sK;
z#+3^)&dIV)bTFK6yQeIO;plIcL#jIu`98mO?(yNLl85tu-*QYl9NT(0GH9d7&!C-E
zW!Jnf@QGY-R%6_B)G>Ye9U%k8bG+JCB6ni~m}=YCoZ59kTV?XC+Vb0L%a0Z61o5m5
zyQ6cxs_l4TVQ;ZZ49~IKzjR%GhQI4Qrv3Kbq&D5jW%sjfGQalT|1<Y~`Jek=vlf@d
zT#lG26fyI7?VR;9*q7RJ8Kzop5Qy90o!hy0nFz;35%G^_R$dq0dSC<Z=BDf3C%>L?
zI?Md%|M}DBFL=+5wvE=5Te|$lqtmhH8F-t86&SY$=~%hkP2KP~Wy9mV4UbblJa(va
z%Dij3!lrgv&yyW0aZi7DKiOIv=yQFNWccORo9-;EE3@6)=)8Z=n`Lowy>=I$nrBb9
z=RUKv^4XC+n}y7ZCjN1oc<Ag!o5r(e&iH#*Gj5%&)b^agC+Un&lHiHiVrN!5up4gM
zT9V?v>Q_^caP=e6>hs!h=jZ)-k(_II?5&qpYm0{8olUFe-d#E2{M$QzHjI~InNl5I
zEsJAQ3wX6+-K*vEUM=YpsNQyDlbrS~*~zDWIn{|BtuC9~zWU<TU#~N6?2yToiLXhB
zKX+Q}mfTaWh2mKPW~W^K>+e0bfm`m^*<-~XyX@y4|9+8`ecdi&Te<IsyT8wUyG46_
zU&nc70oK1Q?m~9@y?<(^-sp7O6nQ*)LuJVUFRs#!7ON+(yQBV3>O^dcqTID4+4HfX
z*G%^@E_x?@IoI4_-(%Gatk&~hiNwEBb@(Wr|55D%t6IUw%wQ(YT<<L1Q?C1#y7}$X
zxZl0{&6AX_O^xEGOOv1Gth#=JZ+Y@N^L=klv+NfVO5bwl%jf!Dfv647m4CQT<h`k9
zc0gPG0QaAwnXxxDuiu=xw|HT!%_ie>Z5xffSKZagxt;j%y=7tAAD$E|p5_-mC%vX0
zn(TKnJ-XRs#+_ZUho|OU*mCN9?4iBqdmsNdF}b_u-uLDIzSr6dt0_oWnY`Zg&*}Hd
zF!OcgrjgEhruX*CRWu*JcmIT3yv&qo^}e;G6|?>Cf5@Hk#B1F}y<gwv-d}eA{$s!6
zkN(|1A8}l#@xeE*-E}!x*I7GfB^`J^v1*t7G{)@g!X@skn-scZ=6Ijlup>Oj$l}9R
zm6salUQFP7{E6r9)u-pqo&TeBuR2v$j6?s`qI(~187exwV_FsWQ97PU{66cm_<wTu
zKdN0|eDnUFs=~jO|72I{{ZCc+cmE;Nor}^-VxBN;bUh`a$;2r&rzs%uAY+V>U*Qhc
zM=tH6nsHAq2`qBcla)@pb7`ZGyPmT2q>~YoxKB;g3EmbGnfCvH$4ut5TYEG&Kl7b!
zUH<3^XUGcwIWE;=u2mr~f|fDwGN`>(k{PnxVrAUcS*2N*R4jIL-My665HUL`vRC!?
zs;f5{b8qljWS8mO+3E0jm+Wn}bm@(2Km6Kz+G}m}#;U?en=)sJ&Svac%9XlUY<cp=
zuMO7@?Nse#SvDuPN{+$GxLvoOU1LM1)bgx_YkcRYotm|E($%f;%lrOHE@!#EGIV#C
zfJ|w@{nhF+?&li2KMAGIvM-mD{q^OAjqjbhHO9|<g;dOae(E`PHmPt;@tI-d$W`_A
z&GqH&_6c1z-#)yqcmE&zk8QT}{`5cj419Kv7#2NsQqpu}wfxwzgi+$mi^c5!Z6C3y
zv#~2IcH)`LwMxye=U0LAqO4a>G-vwg6fW~C@!HrUny$69hd*9N`51rln-m5a_K1%w
zWwhUHTrN|7r*MU0>=nV2K@~nl9%{KoMV=aklY-o5GaTS-WNbLV+1Zp;nZ43r<K!$K
zqw_gl`e7c4%!X#3iQQ8UwY*xbvmoc-$!WSm8#>QF%=#^Ce)U$hkD1BF*J}fegj?5c
zx0lZ~F`uI*@n8qj%<h#-jLb|I^9Y+wUDxv5>qp2Hp<M!3SwcMD9%2deec^ERb(-qd
z9>3TPq05Z|d|zIAEb_p&k)g$xp&_Z8mx19kM^OLGMy{%|<_vbG^$Xfne)%bNZ#%jo
zI`6-B){2D_1Y{O$=~=;Ht~%wK@5*kUOs&ji6<Zf?Km2w}^egqtJzQKGwfw9S)~j+Z
z1+3uOXce*g`Vxi3vb(OV+Pr4VuSsY8IyA-BJld&{maW@&TIkHOzUP-tKb~-XlkkJh
zhn{3NR2i&3`j9c{nvcm!m&LkUE=+lFHtoiRC%)zj>&3XXEGU<g(pWsZXv)?FjjdC5
zTvnTKAc${DvY*x3mD-`N8{!vF=b2)qb;0(dh=)QW!xg>-xeSw7AH3c1yl?kgmWnw2
z_ut+;-fg%4cVE8U2NwMgyALgwC~%l05VE20xV_iPg9YmBs~&LKABj1}W6J;0@tp0H
zjrW9Czf$@<Q;*+vx#a)Ylv>y8(dz}jT~B{5=y|>HlwtXu?D9?J+QoB<JRh!=sr#{X
zm5A4AmTd~G#&hm9{i}TGnx<9tYW;HVYU9oOb<c!(IQAGGVQVN}^RZEBwT{j7OPA9Z
z2+wG^xT<jR{0!&U?62w;ecqn?|KIED_ZNsbto#wlRT%w(Z;#niwgAHomOHZgICmth
z3O#r#bd^^qJ6izLoB{)_`Lj6JIYqFs?Q#rOeIfd&PPLJR<8*6w^z1tmQgtLReTXoT
zVVD*+xusd-U?lrYF@e?@0@<!9O1G*`Y~SN5*f+(^NnWwpH|UbU<P|Ml`jMSpi#&uY
zU#7%;3_7*vO{;#j=F*<@C)fA$nHv1Ryy2o+MCIZ`4<e>o%-X0ac){iLF(L6OOV9V$
zAMBK8{<*<n(Hh-C&LxQq#?yDdQBCPO;+?GWQ{uQun@-Qcl+;;HZD&i;%*`eqx@4!^
zx}@<^?OMkv>N6)9FV*a-3$pHIS!Kq^ccI8q_|B~5B{#RO3w&-UvxU<j-*}OZoS#j)
zcl51mVlBHE7tSp7ZZPguJ)3dv>`Z2d2EMH+lXK!S88~nFXng%PZ|_v$g8JS44}LVy
zV?3$uB&{iMlcPo9(uA{xB8Sr56igIr(^Q;oF9t|isstGK^(`=W415_;zWBn$6K-o>
zJH3f}u6$ebrFx*_C7*dtDyoK=p6;JwKX!aq(N6qt?9J@Z6n^N@o)>F#s_xvr_idk9
zu=UYQ-&_s(fC8-(4!4}rUQ0LVnmydi;MP?i_<LDK7ng&>!b^b@ow7nWEe>!><S}c6
zC~$7=(q?<KDtP{`1;uf$t9XyCYTEeeLFtSjw)IxWT{K)2IYV|dGBRaHl<$4WJ9%oj
z^f%M?+C!Z4<u;#PrsBwT{#W8X4yCn6yu6i|4qn?jX@`qyo_6d@t=^q%4eK=1KB`2v
zbTZu+om@RNtYq2K3Ht-8Z`JTlS(Z_x94sVuyoK?-W9Y`*V3iG>TXJ}GMVR%XjMQd!
zIlkHca&p9xj@!}wxjqK{F_FdmPbH>rTWXlT(Z_mD`sA{LOX>gDyz$9DusX!^=6N%B
zWBuUw>T@phXjyK5!=Sg)_<l^&hpo;##XUWm=Lhtub7m#azWqSWJX_2n?-I)aQ{gkd
zdb+Od^+zu?m;QLLpUrHJkVJa*iAq1d#l{7{9Pd03QrFWncsEadpZb%$)#us&PEIf@
z{FrMPzDpyoPuPqtL)J2ISIylYB@9<RpM`DNb@i-Ynft!E@1~qtq7rVhefR21?(SMa
z6S-VB+;@4&u*iY!$)PVxuYKP1ZIyYN;dDvs`_4Rmi`(x^{5nN*n?ftgzDf6f%bC^%
zOY<%LkTcmj{FNz_$^sTM2XE$!E~{0~h_c+2l8xDUignqp1kM-(hVYv%Yv)P*cUWw^
z_OzRRRj@{m2WQBM#$8?y)*b137?-^wYwx3iL+Ve{5)yMSMO?S4Ty}9`BKsrfBTn8O
z`_vwZH2J!yh|S(4@$f|I(icbAN{_^b?^)!%M0dSz-NsvSYfg%@dbzEVxpI3YYx&$(
zBljZh9QCJlJuFYY%+OOdO0AqIq4YFXyo%e%;8OGxC4SAA*$J0SrlvTb(CQSM$+>j*
zToEo!-=8O2s@0~f;Cy<BUEpJ9_ofV=rSe5Nm0Q<boIZVL&N-heS|W)|OP^R6HfKL<
z-TLLrbekuP1p<>yA20-5es*EXvuPVz8rZxg)a}H-pVzEW<!Tl_EOBi^^^|*w%bvT>
zZ4s7f_<v;S4rWt7OKzo;c8XIL_7%uFOMTkay?92lN!N+3(JL446|!2qbvK9W!tRM(
zB2r6aZ6!T+X9Z0%%epwB>dK-+uQR4_TYEjev~<^<YTg<ArfM_iY1Za$diirrw#Sog
zOe@N}jQ^d|Uh$80)yh)S{MhAHGKVh~acdmZlAN{qz}jtgp-HUTk@`Cv6T)U)a(ZbJ
zE`C>YjbDM~mNika@f;7Nw#`~!V=I!_T=(?4M!ja6E&ux2OCw(S&)*o7@aBP%^vzR7
zf-!UV=5#8vO!~0AJ9Z)K+YHz4+$(aeJzuq6+&nE;GV9p1Ue4-M!CBqWmopSoq@&k8
zvC@02wT{81`^lvl-Io6y7`DvlF|_H|-19PtYmWu@%$s@=+WY}t7XK4P{_Nbm;%6^Y
zmA=oy^&Ct)W4Lx*)?zl=u}jxL`ahEa<Ludt-UiI89e15nWL>=>^6=IjKX=O4^enU%
zQCwqSXJ}REWwkeRm+GI{e<c|HJ?LkA*x%zZM|Xv~$l*gZvnGf~cb)Du)Zb$$Z_OoK
zxv^=bOta?P)iVq)8Oln|w7wB2s4`jhUZSAN<GI#THrdSCWV>Y3B&}HmPLc-R@&?LB
z^o?EKZ0vUC*}UvU_nbxCrpEJkNJwr<Xgxi9=7Li;w=WpmKQ?B3b>!UR&0Z55qLf^<
zdpH9U_p&=pb*)zL6JG5uEGq7>n$h9EcErsEp1b?*L~q$v)ZFpOHt5gPAC9xdowiCE
z9ouJR$ECa`b&hG;7Sr@Qj)#<-ela^~Z{~b4o9SisG|mfdO()#Gp0I13(4SbnM)}jU
zl;3t89Gry{7k%s&&@!6-+1ox-dF`|pi>iC}e}6n7FIxNO?`=Png?~45St^~_;yj}+
zu}e3j{hrhODVlrilDe3-NVNAXarHS;<7`v&+1zVZ&(w?ElYZ{!R8hTkLh|bap05v#
zyOjA7MO-f}Ub=dw!qwI5U;MBT?on7Kux#>az9kp<${w!Y{BwP3km;OewVAV*?R7H9
ze6*9}x3+GgSALUMRJU8qYPSW^U2NPlGUvF>=4i;@^?$=2$6Y+i%XvQ=@Gs{7@Phwe
z!E*IK9hzGXRc>0Y^Mq;b>s@PC3y9qoka#>{<81p=mF^2VbQdW&3*R){*?stM%3O;W
z*~5FBm2dA>t~_&p;u)JaEA~yE#C~~_N=L(m8%<FetOg&ArVIIYha7P-+5CGF_otnj
z?sJ4sIm^%A#JQl!SbdhrN{%g#yG8E*HVMe=TX$+|kfip59^1%HQLoigBM+KPynWR5
z^(w{(wo^Y$W#Z}!ZeEzO#c%&DuayP+4jS?5e)OB%bnNYvQ`*UPg~I+zA4<M_eeBik
zW8Xx~jtA+yoxJAV>NTIVr<d(nQ*lPAGDUoPgnjPglUIc<GzaO5zVXiD`R{QlanhgF
zT`h)(mv%H63M}DRGnq4X-2){Dh7=Db7QLQ@7anpg`Jc>sct@|75cjc;C1DxtHidHS
zneAo)JKQc>1U+f`p><GriGG&I+@xDg^IIHKXIV;d1<lc(wv>fw-f1=Ety{U0Pwi|p
z*wq*$vxRd_iNS&kiYgrq8XXP)A318TJ$7B;kU?tKn>UmHADX>b_;ityx7w50?+Y%g
zyPVb#nXochMtHI}d*OuLdvx{gT(<jqXub6A(+4~E&(UP_J!~doWNBh_D>B5j)p^n;
z-P2DFPdsFF;M1hQi1~WEJr-uz9MwKM-RsEZ6!X~<@^iGhKXFd(vy=~8(aOE0^}LQa
ztHyuB|5v8O-kh?ou#qciOL)s(J`d5yTNL=T&Us%x8hz3}V7F8h*Rhu}`y_Uqd!5sz
zZGHY-&ateuK?j(+awjzzUT6ut!IF2iXX%7v`dhEv76^`3KD6$QuzXO{;YV(1mi=D?
z&dYOc`{BZ&eCT@e?`djXGJjkIj5^%^_L%7^MN~ggYGz$ryKpfB>xq4T+}mq;OQIt>
zr$#KhVbp%{3QNuQT?rmtZz9;1Y;W6RzQ4k}cD2V!1`DggB^+Bix!(lN+~OH~>f(Hf
zoedv3jb89=l9b7FQ7h<jT%vL+lj~O2p-I`NuCa0ZFVpZZTzg8s=9KK0gU213G=7Bn
zUSL{%G%BxGO<(qZFiS3%;u|&X-(DuX-lnlu#YWygCUAVIP-h4{{=dR|(?a!~PnPex
zx~pJk^vVa``)b^sYI%w-Psrn5A*MO;YUc_?Z=dbPKHHP6cg|k1r+eawCoA?XzLRrw
z)A|7OXo<6DpN1TccDelK&hZQzqm{xQK9g;4-SE=cVRLW-XGXJ8#odQrL~bfiy)|{}
z?b%arzuw}%Q}o`P1u1J)Uz%)9s|`yz;<tY3vCPz4i?252_qGH|u!beDhe@;=b_D1~
za4D||S*hSZOUOTCi_ym)Ze_LMm9p^_uJLacPkUc+{^ynJ#eepFIKAdKi+!nLL}kl#
z-CKOdkG#JN2mCh;xF&VM&}#8-tN$L(u>q{Ln_9AGIG(z|c{Sj^)q(aY5^Q?{_@{dC
znJ#gXy6E|9yQ5dn$>@vTM>yLxdFQUUxqhLz+*AF$7R%Hm>EwwAHP$Xuwhdx%2+FYv
zn(VbfCd^L0XT$QXK}NM3G9C%fpS*#SJ4r$8ks@0cheYe@srNMJ#g$#yWH_Pat;*$w
zg_o?Xoi>T8oA^30W$oO=BJ-zWm-*CPg2MLJdoR~@$?P$a;9i*Q$jY^Us;<>EF1gzq
z57}HXDY?@slyYe9?myZw?zva)tn@iC_lnl$&M8t2XQwdvq}m9a^SG#LeCbGvaresJ
z-Xq&2=D+{=#JcdR_m%rGuIFM(j`|BKKA)ka#qi(lrOdHcQywPk+>>VNS~~Sq!_T;-
zOEu4|IkrUfdaf^1SVCyo0@jo`_NWBb*Rrf0|85oWT76V;`ozOv>Ab6~*Dmi#T4gSO
z?iGW?;Et0{O21sTb#ZVqf4p9BaZSVDHI1TcvnHpv%1*4+@#x5n_+cGTvU^4KvKz;D
zMD)#_G3o4$e%TDZ((My`m-Mo#xS#627~AF5r6jaf_;7`!&@mStMGd0}$7$6E#o5$%
z8+t8LeL3;m&dK+7W?xGEaMw{^)IZj4@;h0p)k|J3>pi8-*ra=*MPtRIr3+dEJDBD=
z-dnhBgFfHm#d41~nj{;$C2#%0U%F)13O@nsrQR0XZcpCez4yxhS9`A9uKRPl+DyaA
z&1YlSolR;n>XS7(4>|8kPMI*tN9VDS-N8G)5%VuDnK$iI%sE|`bFN2x{A^Yh95GDn
z;oZg^y@6-p0b!#F409{GL>@If;1+wOvhSqXy%d#Kr;p`4-Mf`#!z~Suw5Sc8r4kLH
z8#phqy?S4hX>{>T#@%~QYkSXj*lBj``}j2NQ>_2j1Jk}v)#=ojP*ivSz4Y`cGw%P4
z<@~$kYQ;4rCcXs5qg-rfpZ}Fjs0~wQR`XyxD=1&d(-s*ku<rubwFGv(bxyMq_-xnR
zmwMkA=dfwP3;w$84cgtJTMx`EoMFNJ_@l?{|6ENhC-S8h&6adJm2`8`ekP`aM*nYJ
zll!pDr9geHG{@|q=bKMumqsnnC{W*LI>+Jz!~CFy0ZEIZAI;9=dehq!IDyMoq9L%O
zVJ6!QUA5~x%QY9XB^&w$oB2j-^jx+Ko%0}1#yn3(^zvo7^~v^EAD?^~f10ysg+kHt
z50CB2oK^lOtA$^%w|nB^mU2x|sQ=S$iGxL6x>70=8gx&DT<~!@^5Kf_z9$#{#sq{O
zG5kGmhsON5QCx=kH@RiFpL3o{E(}Xwn-jkI<#n&z>)Y<Aeob}wUfj^=me6;qkhSE<
zfqPL7&t^J4(~xU0Jn-;#L~i8aH*Z6)XSd~kd~h@OTiR#VZLI?7Sra)DPhbD#bNt*Q
z-$!gWCUg9csB+m~xy}8J0hgYDN3)5!=Bm=Vns;rXYa5SzO{g!r9B@Kt+KDe>-K|eF
z`*J<{a%X<f>uy_T?)pmER_V?46-)X<-rHW{_Rfh6vk+4;cqHxDFy+xCS+|u=_lutk
zr>)E>U*=HGw*HHAW9H;naqQKmi#``{xHEW4G&5dc5@hev_!lO-Zgbo*zLxDqiccmp
z7+g+%r7z*W%V^n;rFM_E9O2x`GhMnrYt^+wtM+BB6SJQ4B7RxL@f%$c^?WgAJ&Rk4
zxHh#_PH)#Zk(%AdWTS4m^8K4GmIDS}Q{?uZd)>3<$zkqLWA?j-^F6Lkkvn!n?oi)M
zYtAMA4=t?SU-D$`t=Mb-&#lVusw^+hF`A|kP@?-FOm{-lGuIxKw?R2)tKOZ7(}~W^
zH0#}Bx$Mi7x8-wlzxk|r>C9hP*HN*HW555`Prpw;+~*&5|NP<W@!x`#YgeW-#6`S)
z@O4#0o{`oCCO(NN=ehno-hcc8lW*uvPO%pg&L%wSdcd?~#;18R7;I{{N?bguFj*t(
ztV7fLX}T6+9SzE@Df12&IG(TkFy~9u;fH$Pel0w$CbIu!@%?j*(^-rRK0bLUq_(Yq
zU){+i-+*bq(X6mX3g;BpigYRWHeHfv&KIrM^=Mj}lk~>o`g+HxtIwhqd;53j-{x6e
zVK$Y2&W!$^bZ@g`A6Hi=r!sdiGFkE}{P!qekZfiWFN%B-kZ`zNNZxIZ#e(jmtrDzd
z53hWDe6(B6w(ZXi&g7YXn&GQ*PJVjYU}TtFCKYgM*;ym=@<)4i3LHG7BDiFW(2Z5o
zoWq#A)M9@HXkP9KV%1t|wRPo@Rm@wgTD`ii1O)|%Z#vl#qI4<DbKR!g*@7!vcI2rm
zT{K!Hs=ZZA_)p2{@CWyI=8H4Uh=^Dav$KOyfk8mAbBadPB$b3S6%GlV?X2g|p0P}R
zz{04aGI7C?EiXB})I0*u-YR{4b>s3Qb-JmirsxK-`8=Go`+NR<-d27EKbc2jPq-&L
zt4yBbJ@a|W4j+AI+h?V(Q_{SqXkPgBz4cn4-5rTZi~k?{^5WuYllOD2|9t;+^z!_S
zdwUN5es$hkH8nd?+lMt{N&~A<#uR5Jri_FTUO&+V&3>Lcrn;ntdR$}36=9v`uw=u<
zQ}yY#DvZe<C$`>9bWkvxn56S<t5Og17Kz2q^Hjx-O1#;!NZC3lhV4eWV_TX;qH9*#
z6X$kD1_l9dr3+6c1<dPE>B>8+vAEk%?ODK7W*yB7)8pi(F{Cc_Y`isd?b1x?*};3B
zrDf-xI(eHV_`|jdbGdU5Twtg!+Q`a&wD90Tu^67H7YhzcMqOAm$0RGmiOE$;WvST9
z?Q-?M60aQf6fZrc={0xR6<6`amqfB8C8kcgRBpQVmDcL@kDjbb=XU*nVM<WcQ?G@g
zVQ*Kxb~;ex`f%OePhM{}?q|~u49v}Ws@Q(c>~)B)jFrkF=l@$bMK8@&ytp<YD5FVy
zXQk)C>r)@Bn!Pl^A@o{A^3=WEk{MsFSw+MM^Li>z<IDXiAi$~OFiV8hd*ST7;5o}$
zL{@7q+>j+;|7Jrr|M_Lh=1M;>V`V>BsS}fbwA1EJ=t*1iPbF6!w|^=#+Szl#>9(g-
zyodgNnYikgxwGQVy}4>L`K-g?=NGD7;!9sTnZE7W60&_;(D{}?RSt!#oO~P)gP7Ci
zmIm@V{n&b)C(ia;(3&Z!51lg>zg@FAr(%X;2lv%WeGVR;f*ZPwp4lF_*TQ)5QChUp
z{|1)?hriuV)7+b8THI&ybnw!C`$pobm|9eb#A8>Zd3o`_+1@Oj(cQi8$@Hb|H!Wr?
zo?K@YUH#oRWq+0T*C%i4h2zp#0zD=?6Wg1;sa0$*_kPX~hnDZ${h0a9KI`l=t8b>w
zy?8qE`P@n`(ToN4LRlK$+mvqoU35p>WS`?iEtRW|M#XMLu418A?A%ovP5yg)?l#Ht
zmfibA=>NvL?OOg7N)u*ZaZwO#IGxP4L5#(5sn~%@QCFuuI*{U<5)ly@7H056WM$7$
zZLf)9>5C4nEj|0xOYMqy_o4$_CwQ}ke|O1p-afkSFl(Rg_bl&+H=gQi>8cnn{kc$#
zPtD~)uVO{C#{WehdMw(SS^GQrZ=TJSaNDREwx(-S=Zx9!T9)fp^mHHNlsu`BB>1dj
zjb(Izq}0aCo8D|F?0vP_<6_%1&ADd_ZETpO#P0;l&uU%XUvos{64Ohu=nH2ZqOF%)
zV+%RKtjLia>nxzWUQN-}FV|^u$u#FH(*${W8FVTeyL8_eGVJ19BI|L}DN$$ouF$4;
zT5Xd(?(vmdoD*sD5xT(=(Pd<ARHNNyRHHaOrp5BcR-t&wlQD++%a_Ory(`=1Gg++e
zy7i;LV(zPJo|&v&>1$Lp+47(;hsgBuWtV$`=bWvMn{!XZw(-Wwxni7CBsQCCIOe_&
zIQp<*k<=rN+O9K!9G>xK{_`CYYn)^-fqmzBNAE=!0t~-|n0Sh=_7W8iJ*pLIS$LJf
zpewNZPL|@^ALlnUr0T>ddPhi~S>V|ls>9`{qBY}8z!f%wgY5~~4F7ko6w7<zT~Mg!
z`O(YCBs`)!@Xo5`)~X8w%QXb&ZhJb_IOrn3vam}_W)e^OotEMg4OdsEl?sRbeI>Ye
z*BnNX{xt`!vp-(2a?w!BNn8;*Lsa#f)V+ipzu*fq#2Y`me(Qas#XFI&l&SmJ)SWz5
zhLe*|w@kaSfVJ1APc-VB1b5`DyE&WcdKTBdSg|hf%voKYSx=NXznnO)*O3)<dzSLY
zLt9y9<}ze#K5<jViOVRYm3LW-l<!He|2xiqRAsDOEo!v!i^{FPPf8QK!e6o+Uh<HQ
z$7uEiCLyEaR$*38O6PN@#q{i+u&0!>WEr=bQL?7>-h-^g&qRYC-3eWGSJY?r6&dC*
z_l2Q94wdhV+pAwIpJcS@)AXJ^#l>R3tQ#2Q&#yi1to~(Ldbn;>ihuQ?xI!+~#F{4m
zn%{5a=F0@x`JL!laQD*&mj~*TI#o>S<R`{^J}IB{Z{v*6Kf7P7nbaoSF7iQ<-AAri
z&ST+XlMT(&dLG1tt-8dOwZT4l<&sFTqTrcaM_S#VzU+RLI-{I1A#SQy*7hfpoa;PU
zXW0HJdb~(WQcbLXb%gEau**HqOjn;@l`PvZyI(`V^1pQo&#%_c=5Oc7sF+<$$zIBv
z84xU`6?ZWy=TYz1cfU41YS_5`{pk;LetOFu5OcTI307WqP<Z~E13uO_G<Cd`7G-wY
zOuHhWc)t3>=HowO!X`A_;lI2@iPh+avrDJRgeN?e*=|v1H5WcDRaWFYd0CI8MO^5F
zH|wW`$)&ne7Vi<8!>KMHYT6lh!*p5DVx80H_iXa0Pv)5t_GS8mb*Ez^oThKpG*vws
zI^E~|ow?O_v*V3=7VZ1mw|?(&h5J>?DI&}tzK1;6o&M<H%>Am>n-5l-r$5uEVQ_L0
zJ|g(XhSfiT;fnVbXSuh_b$?{eYg_ev@fXP=w?Os<#@0V48!yTVPS*OL?ycRqB8l(f
z<SAa6Yj^EjbHVw5#PlZp`@0^#>tedJVok^rX`NkcQ&SHn|I~V^?pmaMgy}Me?~-Nw
zg+-?}+f7f*+WY#?t8HBZA0$?;+ghIaY}L%GYrd^Ll_ShuxI5$lpPsdM%e$g7<9XkE
zuGw)1ng&U1Il@$Lc<sS5za1;%jyoK?e)PNO!}`*aQ!J-9c`v@Bm_KJtN^_dY$$d7T
zr!75SqZKU|p)F#&D(+`UUL^OOZC{S_DV1%I-Fa)>TdC|7U%zH@vmE3;x=m+}N^9G#
zgl&tj%sM%3xex#P?1}roF1>v;cjD*M-A|MA%x?!Pes}KZwt8iMWc%Kw(<kldyLTax
zt$f#i;~O8=c-}X@9ewug{qJQjbnENi|Gxi!+4sK>{p<g5vpo1F{+fY7;m*P95}Y^c
zWEU`goBm<K)Vz+#%t{hRefy8@oc#5L=X@T97D2CX9ucjfGZu8n*z+Dq=a|wy^??(I
zNc>h-&RI7)I*(kcIMVg-{J%2`yr0ClU18yq5@nijQtXHO<*yM}&Rps94G5IE>HOy2
zW!<~1x(~1XiF!SiH@b%@xbkkOPhxsan`m0(l+gGP-iv~njiGbz-s)AHeQ<7=Z_AyZ
z^PPXpm(pXAu4}8GIQ6cg%-pLB&3{VDO`E&$tzzzhV{A4)Vh`K$KOBoQy?3IeN5b*e
z<F!6ZdD$yiu9bJps`&4*Nc!v%DN(T%XOAp)y(agQXVqDjOB-&<pS`$xs*>id0FfNW
zI~7x|Xq;&D@XLI9OZ(WB4Tj$DK1#h6mDb+Uo1NIP$&h=qXTV2aUP+yR$6qdmWLy_J
z6cC@|HzieZ|Aup4JXwvbj%*Kz{;KHrhNo{gr>lqv&z{)HzdDuc9!l@~8vW;^^xl`V
z|9qYO_v`$>dlk<tkok8utbNN(CxHihIawIZW<KRraCj$K&pC%FZ}Q2M$%h_Z6|w05
zydv02Q(|7k`FSdfPUOTKKPh~o@d5j!J3n|_TpA+W<BmQJjdPqR=YB-S`Rbg1D~`CW
zIkIZXsnsu9T;I-FbLB}FLu;9$<GY-8-TysdEV8T@1tmQ-PxDU`VHH&MW|iI~Iy-pJ
z6al51&NrAeOGHGAqC=O(+~}1}nQ&8Z;arJLvJqvWf6gwpS8BO4_v!70iBZ#18T?vo
zCNcfp$Z8(QReWP+?7h3@x(ecNpUzM0i5HSP_Qlis?b5iHQx}^`{c$-`_c!#3sC%;2
zqtc$9<-YC}Dq`mn=BfFKRhk?*x8u(F8l^PbmKU<VX$#pM_sDp79ep>UQ8SW7({^F{
z-9`SF=J{(qeUo}tFYQS7*Cd~(u7Mqse6~K@ytehzRSuoA9N%L7c7@)bWODhH0bj6Y
zd`Xey3_-U}o&($0T>t4Byl3j;U$x82#boXt?Xh}w|Ih!q%PW^Hm!6~&9nrn(%i`LP
zi|;&5X3Cn(DAa$5v%k(OrpdGaOUQZa*o2}VoNR74=9eh^$axa8W&uZ5gh$tc_Etex
zhg%^fE_FInx(ZLNxFW;5>&ZVG7lyA#-jvK|v5*No(wp@$;;xN=;5&({CIalP6NGlX
z@Q6IZCAo0J#%V4(H=TT*h8DRA<un@p%W!jGOX-mmoT?ZzNl|j!)0$nalCu`t>{FiV
z_j1ZHfv8PO?;Ks|GNb42(VMY#q4S=m=1mBVXL=?kaVz)9v0Mw5f+trC8Ya%MlXC6q
zDY>&UnU6iyP0gue*8DiJ<)Mp~pJe&dvh3oHTbi#X%CDT`srIUl<!H!%3-N5BNx^%T
z=_pN7yqKi)cS*y87wb5sgC(T%`(EYgMGIeJD)2iJwL-`!ESZzj#nkD(*&(eYhp7{y
zj`Ymv=!#J;6ADkXnaVV=lgaadxff&kz3Y|drqyU_OP&d<Svs?(kZphH$|YCit+cY$
zuP)y9OL|+`>c2OW-FBq+?|NGGP5$oN#ZrCGE7qmXWlfoE$bL{#rS4gx^Rmm%zm{8X
zyzC;Du_Q93B~it;*59shiM^p$v)v0$p_i?*k`xv$ads8)j+x?fh|BG5i{F_WzQ_DB
zcHAz?aD4JbWx`I4Af?pHyh|@neX~RCsY}PI+-EDU=*-mJz|^Us6Mdw6^}6l=nU^8|
z8y|*FbgI#Psll-J!aM1hGL8c)&wFHPPFc0|Qdv+8Q_Mv+-7A;kuY~Dd{T6>UX|2nS
zxsqzfXKsBO6V$cq(9OG=b3$kO#C^M)`{(H`p?k4gS2}4ed8D{(@zjZTxg<{NJWGC-
z`DE$Dr(ahl*WFs0IZtM<RK~f$^nX1XQmdAxOgn3<x733%Z04nGro|xvvAOq>-rQ9)
z2%4LhbmFD~Yi{jX1_t$@IqROdIBMVZPG;at_;AhNfG_op@1vZXpa1fHzN_+iV&4pv
z18;d&Z_GNl@!Itq(frMF>p!aLe=57=yeCn*tRq3@?dMPO|E|hM=w#a(t<N#kk0{AM
zwQl*=j@3KXCEEO-rf=iNa_Qdm$!rPsLJHSs`RqCNp7E7>o{!AzJF8ikUc^Q&Xco<I
z&Uw&Sq|nsW-?GU6kldPPtrjNboGd$*4ntnYT(7oCTxU0O`KE-lUwd-2ZH0e8`mQ@(
z4UGM5U!FvYB!=sBcg@<+buFXkoQ80k%N5C3ZK<Oh_UMHE=(y(l_~{LfwPlKjzP!=c
za8TpUi|~%40@5GDH#l8T+IZ#fqm!&3FGd}iY~?a7sB7vp?XC0jw=>+y>(h_irW(!0
zIlbxSoSipkyt{dDhUTFanev+qXFobN?}_Lfqo_I8HY!TZxvRIR*y35Si~BscB;~Y-
zCy}=ncqu6)ySJTnUs?OCPA5<O|27qs`gVnh#~1xN>g1v4e@aOw%(o?G)v~ZjDi@z=
z<Z7u4%L-*(%ARS^uz_ino{MdXm__dV)$g<pB&7FVC@NJGew8ub#bWZzmc^!u@3~B0
z8+5yV+{OFJ?DhA$!srUa^p^FmCwzW<y1zXyds|Vqsk8pXxR;@8vfZ}m9~4w^JheJ*
zmFHeJ{(t|T|NC@o-@Dbv{$?MjyK~^3uuF!D<Bei_*$vE>9|ta+x6EeEVW%8krx2g4
zf)2h5e2hoeAM?|nnz7`i+vZ6Hp=U${uQds)r3Pwsb862GEc$Zl*RAFV)(!*K{z;)l
zP2EOm^TW1XKX_#NZGploM@^ob+2V8jgU|C*0{_1&`AyDt%V`gK^DN!Ru)57q%zy1q
zw_Ka3+|P>?+s~-CZ+IBIT=n`p!|Q2xr%%eAF=_4mt%dU2-rmacd8jz+*ax;mF{Q_%
z224S>A{Y0}+h%&-OHNUG+H=)usahe)cLP_gofRE%<-#7BCrK{7p<H_!k7}Q1eRgQW
zvqR5ZpJ%8w=IOV+3)|3`Z^e3L!@Q5Vjqi8mt-e)cocXQnuj{*S1~Z?WdzZmzs<{64
z7DnIA6SCG%xV>ka;rDqi1s;(-Pa`h9adz13`l2kaUf-m8{r9}}X4(3M%;h4VOs0M-
zIWtG~Zm8*;!nGFN%8{Pi&1M?2Jks9Br13xR`~M_`1srb}gbU>UE9)@5{>(M6f0xOJ
z7c=K^7aeMA^72k7=w_VYQ?#L@%)s~%N2u44kezouyf3&gCvt_GgmFA{InCSG77)?X
z`kYNQ`Sz8f7X?cu9lK#DxMf{{@8y!XHDztbOSbRWkic=%<%(Ip{Sir_t*$otV(Tr;
zo4qgW3hNG!6ZQV+a^zv%#I-Wh3X2*RhLt}J;|>hJ>~?6++lL#4Bll~{s{brhNW1ZA
z<7EF?bJ>iP{eI4~OH$-sHGkbj#q3!%Tlg1QeM;5UiU0i7Y39Mx3+4D$zEhpwDK_C?
zV8FU8#UPz;Qp>n*d%wN4v@UDIk!iyAF1JHk&xkyHb=9|k?|^pl-H-!Eqt+ZP`#-Pj
zW0UVXSH2bxzqo*^29Enn`Mt9Z=LUH3=H6Z$<gwY`xY$H~+M!S1O!&=rRBbdCF0aoi
z?B93sOP#Uk>E{RT{wg_~|Eto}D%t#~<j#G1EH<GkHm8l-B5$0yxOtO(7ssbyjxv=?
zS+a|ox|{>6-C9q%?s0JppK?@gi<5KKm(u$TOBNmEomk)ehV`ce%S@5o?N?YTO_n!*
z*%iI+!^I4f{d@r_>IWk%!ZLTbepflrZ^Fr}8fOtO=kkXzm%BTfegtOj(tq{r^R%Pk
zJlkDj=KV4;mC)aMdEN3W|DrGE)f_*Z9P{m4PWIa4v-jB7y}kbL#f>`Aux!)WYCYlX
zI~N@+wEmyj^XTHnM5*Jl-4`FvJril4c7NN~gvj=br?<wGOQi)pIo;}hx&B`2wpQ1-
zKE5Z8#D6}Xt`pD@YgC;ms@*clmHXIw*=<kd7V7G$>YIJjkKzcPa`Y<CuWYAY&52G6
zSo00^UoUY#cjwLCCjol*U)SEZm%egQ$t}92`Sq9N@{U%A?$zOc5^tKFaoMfE)oS%`
zv+r^H3faw{<<~un-{%rz_^a*T{kXZ4cjeb{%2+NdeIHZK@@x6T@C57lgarZ_cCF@&
z?dR{lE3S0T@X-5kuxUvZ{}h(j{%vtAD>+<x&z9Hbf2i%&*qw52EytO>oWBjc62Eer
zf+DtjQ?_U*|Ifw0(BpsP+>e!E9^t&&Jl;>6raZau#pD0(1N-;?X0Q1(y*_Tq@5^sR
z+I8k$OugVyXz0MBVA{}ofRUYD%x1@j2Wmz<if&UZK0eUs6w^#=>6plSj8VpxuVP2R
z<Rg=$0`=}Jn)Gy<p?2&c8RLYWX;byL`+$!soolQ9;fm&$=OTK+Y!X*jmOKsAW1iz=
zs`d5dp;ZBE({imAXwF=gI<M#>SLvFF%?T$99IYAN9Nd<3^iXX?n&RD#vWL55FQh8o
zZ?3z!Yzl{jLF~SYk4$1RhK@pAK`IX4EW3PHN4<^ODs>@m;j_+pl1*|CEQ^m^V)txL
zyV@1%*vcZlFYZK^$*b0_uWn|g|6hGC?cVhI?Yr+}nWsHGuAjfIwA$g*vpK3w)BZ^+
z<#yy6McHdqD>%KqFIkrE{3`L_M5*)h<7Mh<tG~Z_xPSeQ34acKQNMiK-}2d$24D04
zi!~4E{=41ez|48ap=*J}oFpc8#hL&oj)`7b&CSbpd{E%qv}*#lp2dwv&U|({f~)xb
zYr2F*!b3J{`H4jtDoL_81gf;<v}m|bS$D{AkzS69&WXKzyH-s;+?TR?ve6H%r9FOZ
z1wvlx-kqCzw3eo5G%^ZEFm$%=VmaV#z~807X{_4e(Z#rE!lJVpTP8H^znHUejcByM
zsyv2UTdX_I-(>h}yvUyQEXVE(e|pwgc9c1<{eRB={hnfmyC2qkS>VRMO60Pq<yn!X
zcHifm+css(YrlP+8P+07H#LmDF1Gm`^4{&gM3BBpi_R(0>*0n<uGgcVC!biHxSRXr
z)wUBFCl^lj%N0|ap3(Jf=}ZBcRm-!*pJ?9JG+JY@a!wyp_{tS0j%uyxoblA_Zt+di
z^4RjjOx|%@&v>1VtGYbN++f!a-kybrzo}l@(7<}QTWMdy6YqzN3t}V^)A_S>7N5|V
z*5hEr)|9vTo6GAnj|2TzuYDYv+<kUelh@`P#d{I+Ow4<y9X5XE%Dy7~*-Xi(b7p6k
zDX-lYD9$W;<#od1T$}4`MxlF>zgoXo_(!3wBlFjaK99`iNz1JMXGd&&XLWwfK{JmV
z;o_68zFej{%k!=8kJ-Tv|M~v-zhmBc%gK>VyeHs2=j2znqdWBW7#tDs77;umB<OBf
zDDw2Q;IZS$J%1FAt6slo+ur%6iF?1<{ngwmyCZ_WsL$`YRHEhYKle<>E}4YWtuoRI
zOe|8-!i!90vyC}9%lw_V=0EwOaVhGJ%v!NqI?n$p9#6b49PmoRsqWSM?<`x*J3sVj
z{Px><|K9JZ2i-lN_lLheV|7j7ZgJ&0<6Tz2p3ij;RSFXp+FiTiVjlmE6`WG43ln{r
zCjMJ<T=`{E2V-QBz)Xz~sxg7mO|Q~)bH6UpaJ}i!dg!RgN|RKrC$qdJ^38Bq^S@z(
zwu+2m<UXPGo7V_v<t_~8Uj0c-^4Ns7_Lp}K{0kD7eHNfErFe#~^T^>9o;eKLw3A%Q
z#98ls<nx`dvD4XOp}^UMV@4q|=R~}nkDm2l?~Zs9?6~E#)b#_;O!*()oV)J%%J)By
zrR~m4E_m?c*qt+mY1be7X6)*iHKFsA$+^oupWDh-yzelyTgT*o^;qz{NKcmfKgB9X
z`b_sux$-=RNs-wtXX}K&u1oA0AMrOhWG-NmT;aI%&g^@C(iX01xZoW9;6jI=%Z|eb
zy$f}c=IDloSRQ}4$t_~xohMq~=gBv%(`Wa*G`ZvChD)iOrDswkoMkpI)!Un=cqK>H
zWx|98mL-}0)ump}6{!+$%>AZkWNr9dY|>Nh)*bo*=Y9u&dbHj6RbqR(60_*}oW%a#
z*epe<&*}Dzr~Dh=O<tfWdH;t>(7u4bIT=znXZ^j$JkP1ot4FTz>QyPr0~5a-uitxW
z>iV*)7h_phN2D;X^!OV-agq{u<SDN(25Bduna93NyjY~*yP#G0K_a(?-ODgPy^MuR
zEmC7b3}5YZSRpuLMfOUrlIvlIiu6}r%WnO7Xgb%=(!;fxq6vSUe0<rN!#V^El?5$4
z-kh0oO~s^bZFXsP@~nuBCysQk3%bzc`|j6~9l4I{E7rb#|3fNbk5Goc#%G_RH-+xg
zulA%C=e$X-7LGiTc50*k|J{%6MN5xWwfPtbwaTB|7QCsnudM24<t?SNYVw|&{IjB7
z-*SF@c<buc-mSCOg#P>Tc}w7&urT9~{rOW}wYRaBI}}SVZQhi6ic?oyvp$)loynZx
z;7%c}btZQn2~1b&{L-Pc|4pi1qvXA~a~Xz?w<aF4Xxu5I$1~F_=-CUGbX6VO&0az~
z)l8K?sovb1S9WU(!yL8+ty7k)^kGhT^lp=-U!j}A!K|qc&5CP$+2_CXxw)o*DTAGX
z$KByD$BTTXGfP;5D%`ARz2tS8DJ&==pK??r)9_48%<QKh4Ce_*-gLPkx#j%6<#mD&
zb?ewFR{gyhrYw7LiCfcaIi+BgSl>L)|IEon!R+f&H0)<S2-*?qxG?2HbK=VccK;j7
zENZ4_q8uO2Jj>)P5jFL!e#vK^yG4g3^0pc{n9QiD({S0iD{pOJM2Dbi(zVWj&1+lh
zUN7XGo8hwl>Y>)WlEt~{$1X;wZDPKVo-fp$c{J}<-`r(umcE~n8L(Tz>hR8x<3BWJ
z*&pJsT=h#wadnz~kc^Px(}hozj4z$|_V#Cpmo#UN`p?(iQ9_fq9#dg(-IUVtBg|nz
z4u_B1%$O%e({D*6b+m1Gr1&!I`&5Olr4Kz=rWVY8;o@2p7+P_2k<`W$)8CwXy8F8A
zV)4Hj9!_gE$4@$9P-FaZ?~>V<j2P{fcI?daVYb&k`0;=A*;bVq<;)#-T@IWSaCF{$
zKu|FK;Axg56@wWEh3;9r-SqO|1a$@u2|ouj)0m<O>JC1RdGqY9FE$htH(Aj0=)R-m
zrdgjm&m@~q`+7dodsSJRS)ia{(T0@ns5X{LB{|zo4ijv*PX6?2N$7tK-@l)O+714O
z#5~{SR+D{T!+Dh$>%E(rc6?RHi=7aCe-;z_G1fHw?cOVWOpABT+jM1ZiR!A5+|MB=
zg~ZM}rhfSHDeC~=|4{c&OB0XY5scukbzyc-e6%O;+TnwKa;Id<lxA!*Nw%$@;Pc$L
zx>I!T(rw2Dj;5`;GN+MG+;sNLd7QCzLV;5+cDYT^P)f0JTi%hWD<A#;`lj}|mSwp&
z8hYj27V*}fJ%7KT-aF=@Q}lc1gVI~Sn>BDoL@Ql5%GJ(Re)n}!c;T~EsS&68^Ad`#
z@9r<}|6M)v-oX>|_vSoo+5g+tN-OrI*D_&?DIDh>+)DO7;!>uR=KUwzR^!Y=9l!kv
z?n(yl8C9E}1T`@zCve|rcx{nvX=ZRVBQ4O%fW>|SvwJ~uz?tVo$C%Ueo-Jv<meHK)
za`;D=zIEA}Z;vu<!nC&+=>C}ZPN_=gnnu%NJ*6LWc%qj1oJwlqZcx|I=+Kx`@OFo*
z=A=cZ`&ONCd7-tTV5(on$_*QI_5`$E@zMRE$HT6#|Ke!d7p2uA=i_e}sD9Gvco6X4
zuxb6;Gb!HxPCu$J*e~YCWXSh<Myl|whV+8uuO@2+9DOhQ@eA@7UXhqsoztfnb>4VO
zhw--#w#({DF=zcn7MT|G*u@lNifC@F;n{AM#PQkSn#RfNP3uBi&ME~J-bj0TEy2)s
zO7G1TOK%o@oV??a{U%kT4e!Nw_Me%zXzH%D5(jzz-a6`XqIdc!-@9`r-aW$Y_F(Pb
z63?Pj2i*2`%_z{gzr*mskN5WvTr3fD@)4NeeImtsk>^7>J=x2JPo#8I)$|^6_&(Iq
zV_2`L)WD#`C7|@~V@Ft$!Yub16{hE57ZzwV9aBrP<m<YY<71GP{bSbA&0MV4RG1de
z(^sA2m$fMG{|U20uj~$mHu&sF>k8YDr?iJ}*}cOj=X7arY31C$G<t!6rreRY9a--l
zJZ+ihs&vdJc8=HkH|NxMGp%BJuQqw(hi9p6Li`^aUW8nEEn>Pz=JiF*uL7L`O&bCP
zp4pxM{Do6{`y|2LleEQ|bi&!JN}SXCl9N;XzNzV&>J{3?6ew|=%qTmbt&pYEuznIp
zhGE0og=rJ?Ul!?C8>r5CpX1kUF!f!g4Ab=+YRmi%<<45QqwLYIPtU|>Yflwpx_#jN
z?^WI0$NK+l@|HZGzuP6)V8`mhe~0#5;PzN2=ppuOuT#3>gYJ@V`uiSbuQ_ALz#3o|
z*Zx$6$6U7WfX2HAFD{mTn)rW~T;AhP7j3^8%CG0F*mfXf8HbWbftBwDE7u7u>|C4e
z)D+n#Szb%8ZD3<Px=eJJGFvpOsP4vN9<DY#O`#{)47r7qER6K8KYr~NbTcTZB_+vz
z;soQhJkeWn-c~qiEh|{Dujk#N9QDhsTwgXjhEGzz_$IdN2zTqJvl$xaa!#~enUHX+
z!0^Y}do0RKcb#2~l^*)Fy2zhoY;<5UNo3xlV9cj#zU$KGj4RsCYrhnH`BHMlO>yGU
zl)U6m9L7>cHiAnRiYW=NP8Oaz#kO(7LLp8w7H5&qAFf|8ez|6uSLDWPJ!>bK@atz@
z^!{k9UK%L=&~3^ZpPhc0YTCi-TM8tSU;g)XU!v);q#!Hh&ZYBr!Z^7vo^_RYbZ1kO
zTiHtg$GLkNj_7XKsK+OIU*zhHkYdkW52W{B^kT`g*qXvHN%)~bpQ7GX3r;^5-@K<{
zht{k(=@a9A!N)}PTcJ`=sf=op?}E?FteX@#H?eSEvaR8J9@Z1^D`S$vq-_evHrMET
z`EC-KxGww{+p9Hf5zF3%SG3kzot)@5Y4VTfjYWs%EMa)Nu7@-6)7wYyG#=<lb_%I~
z@8MRRqWXz1?h;d6%nJ=(|J9Q|om*lQ_jGFgB^AFD3+Bm8RNQBzW6!U%JLt==squ+t
z+%I0T+_u8l&sq2Oq^gX7@Lc`?qmLOS{i1h{+s@2q&wlk^ajnP;i)-P%{y*IeQ}R{p
zDzj63uVrr2d%cAJwZi0;ld_F2i#P>oI_niH8cjA%^_*s7T)4$sa&1A2gnP`ySLyfr
z?}beM_wCU?BlY4w7PmeXvxQ>+mC{1JzbG^mGcYY+$}eX4HZe<d9fSMUQlHW?j(6VY
zS`@=Vee_ISUOJ~cG+S90JbNUzARs62=gXtpiVcgV9{tsFtS7`u#q(7|K|sXY+JI(}
zSIg^io-U2ZiYsOHYM+|Q{Mq)9iMp|q<3s_+EvgR2PdH~TT5-em-Ijm=SB)0A5Y_D)
zd_&8fv`b<SerSF0D0cd%iBBdd1(d`orpYQ6*j`fceYe$Y%cd_ZW&ahO`jw7Ma+eM?
z&+>aV#k^l)T9v@1g4+kISTfC~l+<yp+*+b9!a7-`e`?O0RJPP^ui0wsu}ekg9ut-3
zym?PJ;@u@X@k6;K9~}f-qP;oOHuB%z&}$rh(s>W>v_~d>)z|*acslLd7LPEeyQgv+
zc_sYkZoIqbqh^Ck(?-KUX66L3uQTVEl}zaT_{Z0bf02A-JOAEoGV*=FrwhIJrN~J-
zwVeGpYuA&Y>)Uc_i%ZXZ^xnnNew^jVq=v|r_KHl?j`Y$FTY-H2K$}$2uwp&avcB;B
zZF;hrp;snP2z#Z-=Hyj)CPF;f&RcM)gIARCqe$-}mFang56TUDE;yXwi{3i*|Jxf^
z@=vkGv~M%{d0gFpdr35tw$_!H8==Jq4)tGk(tIdd&be~}`<srs?+S5g6@R~eX80D*
zylrB?x$>Wr373TuZx&fk+!4^_XWPFerNi$#pK6y~Sjxs5wq7^&|9<kCoHtcgQ^lh!
zrTlb?zUTI&pU2&b8or!!ShjY$X7TS|%hFiW8H_$p?Wi<9nlI@*Io3(I<$$Tuj@L6y
znf}$NWb!xv+hA#G9^q+Wk>z^4H8#Rv$;o{=!OADhT5^MP*w1BmPYyC%H~03>S!cx5
zA{46~7MrizY&AK;x_Zi_`QjODX3u|o+Z^@5W?}e`AWPfcEsE>gLIpl6``y%(7gs%M
z5qEya{~mdhI!O-aHB*<bdAfL|PxFd`KKF9l0P**yM3=UH<hs4(gIh+M$(3^{yCT_c
zPFQ@5@w0%%$px~@wx|6vz0xMG)MBpGw|zxu|9snHT`eWYYMym0d_Lcu+4J|3D#yt0
zvr6h-9kx+$x^7TZWL=ni^!lVNA!|-rtx;ZdW8OROK<>3u`SkBAJ3MGxxhCJ}<M%0l
z(&|I6*LQ#PJ$m+z#FUx*J0!oF`2XD^*|=>(ZsybsmcXw}&H6t!-qzmK&&lxh>|TXO
z57%Y3wF;YGzSttlCwo_N?)z@W&>d~JXU;0wA@9ChHv0zu#&z@94~S2y*yjD=2-B^O
z&t<kEm)J`AOw0H>>(~9CviMk%Boq5ewZ(a-MfTrV^2#`Jc4*Pz89hs@dsfa?k@ZwP
z{m#ccT;t9UjY*H99nE7lUx*C&9us)IL~)9u#FzC)qhpyP<EvIU=T_}|dAq?zc|Oy(
ziL#cHeo9aJYrS9f(fk;GT~05L!^Vl)h2g?p{w1ZW6w5-NU7A|YpRD-u#{qFwmg!Sp
zf123kyZzGjX_=MNvUe(FRW4HAuD-J=ddb`A+MCnNQ*Ph+lD6`7TK`Yu={p-2&RoiP
z`;XX-KQAnl11`_H%$*r&eBa~yOdaW&WlLIeZ?taS7F0Yr`($L<p`CL$^R`@+oO|5z
z^m)nIuO^06&(LOBV?8B&{=b!ZicIq_&-wrCij7j8$kn9KPVq?#&bKRWEkDe@-S*8L
zMRxN@x0+pZTf6s*FV5DAV&zcfl#Nv|>0No$Qr#o^f*+STM~SPsR+yE&ez>y4mrIID
z-2ZR2_7oNK^gCalu+MkzgyT7VN|h@VnY05BU2V2nF|~ZbkCk1UG_JOZteXFQUYc|0
zl3(YF*Z4aKZr>rLs#RVyG4@(Z?wZhz$<zOy$a`!1ws~!0pmN_`l{vca<HDR?ML9JU
z-agUe?6BhYmt7u5WL<9DZjm$DP^q$^a8KiW!99=G20ScCDA4HGvC;IqlBWnWgJtPv
ze^ZN}g%<uhf^#Dtlzvwzxb;A^j$hiYU4FwB={SqCTK{9c7SD_B>Bx+pICV#urP}u7
zHDQgWbr0?=)UjdzdS`Osn+TV}hJb&!&fkjEvbxB)cc18qH&t4vryc5T?uyRdw6}lG
zgz}=x><urT{^-9Q8b9gNm6`J<)HZ3K3f})c^wsz3S3hUJQvWH)DPw7uytyJz*FW{V
z`l0K)okLZ^6sqz?{ihwO*4?r3%j&y!3w!U1ES2@){p?-8^hmq!5kpRU6RD$#Nw<@)
z-8NR*Ww7~J_8;d-k|yV^wH+p(nG>0D)O6*&Sc8D#6SBuL|GQq{T<;O_SxfKS1C<9M
zd%vA*KViGhtW1TW?R3jm)(c<rg)Mql7&Y8DV;^w#xO#7ie23+}@3mH8%6sH?+-0^A
z?MnQ;ZF1sKo6Y6Pri&J~w0TNqDhN$3)VzMHdO~gPtqbXP(}R_Inddi6S-`pTsYXfN
zAHx?8cUvznUiQ&=?q%D#AL7JcCQ9kO(%CmxRe^y)@h1zb2m>>N4g&)NLj#ioBLl~O
z26he^kAelOn>d8UY&<rcIS|Aw<0cYuY~hin9JVx(9fc0Zo0gpDkrF6U(VWD^c}*%~
z^}SP3i;wUuFm1T%Q)ZU=Y|Tky)q|zXja;%D@~&Dea+~-|gd=3Zj->&M)pESHXoaqN
zHNj4ImDfv#($zEAbTzUv!&Wk#uY0@8?CllC=<v?y(2|CW6N^6D$r=c~ZP--6*~et_
zK*Q;9=us}7*Wquh!e$-giMY8b%r^cl%dIaz4HH+NWr^k$(fv?-{*vZx1s2~4Q(rNz
zjpNT?zO^hPY;DrvzSvd1D~*;k@IPpoG}+mRQ(sKw-j1E|#>{tJN|T>2DCxMe_OP5n
zY-#zsi`%{bi|bZD$bQ0dkze8RGZoF7PtTsd@6%;^(3e@2+eOai(aFG9f+^gdvwBP!
zY(sck5(2Ewez~EwhJkfqOY6s{bDUZPkKJ&HY@NEme>0EBqYj~SP7gcFSj!%E>1>;E
zsV(v7j72@t>mtMw{WDiQaxie)aWlDl<0D1wH9dg|8O%3~6+G9pY>~~p&Fj*Y5wa^|
zY4=*6pZ7dp8*X1TT||`SVonOz6F1FEOF8Gw%Hd;(oz3F8{fcX<!}2>*HDy_z)(Kzq
z6e>#X*{L+W#!E9)`y0=LNe<noT!ej@Y^^RY;k7cq641rFB|NF3jO*&su+~|k0a>Z3
zTI~z#R=n!t$laysw1Dfs>-3n-eaRwgAN5?B-nyHs<gr6)tLY5~<=PjTA!eD0@wVG;
zC`(DlD6vPSum1Su=GGmlr-dhO*g0)&?#(#wWm%7|eVMiGMcB@_>A6RaMOPo<O}SVl
zJdrWYaW2mUeV3nd*-K{?sy&gNd4*Rd{@|yRniFK1xz`9R_~2_f!T0UWu=trwu|B$W
zOmDq196$D^d0S8T$@+2$50~rdKm(!OB8e=iH)ce!tUMalF=rj~%5%OmJw$Y6&s|dN
zcTqlajluo*YLgJ%-&eFl4F0p`bahYM8qv#ksVH=bK-M8$$FF}bspKpch+3L!Tf67W
zt*Eq_Oa05O#JA5>IQ?DVZuxG{2@4)7{$KLp{y!s;t9w2%t%|AVJ9$lTtDNGKc^>t%
zS}p|zY;&35qq(uTXbE%fyU80_bEUSfU$O4)%WE_9Dps9gl094H6nRwjTJWn4s(o6i
zPiIV69V#0-<^A?svlhj5zuo#!V|^5_)f%=<CvGLbk?@SNWll*u@71PvRWPGEOYBzK
zU2PY=AE~#d2e>WEb$_>8pjAYu@cn<wfQC<8DjN5hAMBrUXf9Kqc?Hu%-&w1a<qxgx
zOkkIpQRZSiuk8b;rN9#L`diKdTuD8i;%oR9TBIo+U&v;$L_y>mi;=>G3(AV>HQmiB
zOHCC+Rdw7eW6vyWn-KrhFKg{@uB#?T4~uucXxE8($nd{r8<*0n3nd;Ug4b3()0`IC
zQ6hIMaamBwCaqUFq9&5fzDCLu%6>3c=S%lW1Ues6DLa)LEp=jVpQ}u}!;#)XDdE`{
zSPn3MTA%;Tc(TmP6Z?PvJ>Mg&d1BAAud}%~X+%v5mJShHx*}?W6VK_@i$o%~?BSaF
zRHK7&iP~bOUA~E{+LwBJT*;U|!Kdw1n7^ZH@3W-oF`j{G{=c7Gn${sOr=qti)@PFL
zq3uda50^#1G(Nk1SE#U7qkfE;=k&Ce&b3LK_$>|<#!L2GR_DByKTAZ>Z{HTpWxkI*
z6W1rsko_p3Zau?u4zq&7p?4kAJ{4$spWJfoXrA!W*qg@Hd@|=83OxTGD!a36lIF#z
zQ*4i1)S`MzIF2s7%CNcXBI~hIW=|=;@WL&dALMk*d=$B)%59;H+3|&ZUzbVFPrI<e
z<?Ir+&<+RLEf?3EUE(3*a(2;!mlMlR3FnGGTKe*qTUXGW7U3_>C+>tjS#JAGzv0eH
zUb(;1g_aqarf6PV=@_wOmV@aL|LVzHkA;$iPV7wAlzF1`?Z!e@{#I2dmdu&YEpFO0
z9x2G$x$8-vlh=iSyj39!mq^ZAk>qu0uj#_|x4WF?3Qgg4yt4R_fNLZ}l<3XOr3O8M
zaw&ILO^98)D1wzyHhFQ=*0T>>6y8W2NZ&3HRmXRrh;wFG)<q`OPk%P`+;h%yjM+Tv
z>;Ic!D^(c#_WnB}^DA*{-LFX;|9+KTv021Ar|j?yhUs2@?VO=KpOwOkc-iC*GOlS#
z<XSalNtff*0G-@qXAd9VtHHe~Z?-UfdM<K3O2G4Ni>bjA#n^@00$NWMWbIeFyS^vf
zr00>(dSjPIHLkDDE{^!t>F<(VFDP_Gp=+AxDJ_r9UL4X+doLea5h&icc-2xf22-W%
z7^%Wt6Xrbn^|nFXujAjjB}wxStW>xYd(rXTm*|c!%S3K%y;Q%bNa&T}YE!W+uHG__
z(@R5)*2%V<PM9{w`bp#_r<H32)Eq0r1GI8a<;5Ds=N(Bo%6IJaB8h8tzdwbqY@Hcl
z?zK57Q{(=#6RH0<^;CvBFWshnL_(s|`QC(|c}8s$e?$a&R$cqKNJ%$ea`McFOlsQ?
zO<!BAxNFAUtZ?lr;W_jD@+=x&T{2Ppk(d)^top}l)xTNHp>IxIo4^t1TjLeG-QLZ8
z&YKo5$rA>z9WH6xPF{S;ru=Vg_uHKca^9ONMLZ9Mtw|B-DO?yUv6(AvagN$|olW~4
zUd_4w*Gsf#r`ha_Md$l+rTv-j6*RE#o*AcEeg2uVSwe=BchrW(Vs93(+x{|qU-jeg
zbk>iHGFf70#eV;N|52Nv|5wgCQ_keB*raDz9C7B(f6J{q`SL4dXKGm-6g=|i?V0%n
zteSS8m0Aj~I6Yaq_pQlIo{X0I|A7UU)n*E<dc&1=V)Ds^6g@6Ojcw1&msdvalX*5<
z*+0DVtZYIP-*nU0d>eAMeVcro@r%i6myQ1Gf5tE2`?~V-pI6cMEax&BuhIR@WsvP>
z^l6LTmy3pTH~A=TUdM6ydf(o4w^<{1re-$_pS{>DR=(U__to9*sxOP>_r7Si|EKpu
zzt72JieSein|l_02K&r<mHVCIc1zycQugUXXKvmt<4hwPzPcBML3X)+ZyU^vP&l%2
zqtxfJsufD#ugsi&>UDJPi8HIE8ndKsbPM*qc2eZrcFI?8%c;4O61;PsFh6{BBJ9%h
z(`H|OE-I+|vi$v@>HOO^?Rhj+tz;=@V9x&EH?N2O|8cZ#X@T|GGk5*>e|KN_?_2f%
zzcoMYesAR~|9HFp&m(?|+IrUSvtHx`efg00*{tDjLVjI(L-TX~Z)z3XD;oGL7`P<j
zu5Ok+7nJC`NG`FYQc|oS<B51(lEc4CZbzr8)tIPV+ZwWXnZV*{O)ICxggZ&C5%c&m
zS#a*gP&-GBP8HEt8&wvlvdy}XBWfg6Zq_2^)UxYRs--!D(Z&|98!bLBngdpFdTX=>
zeQybIFHK8JGcPX=-rj1qy){9iE$K&z--?!?iZ(k7hKwC;fj?SOJ=zK^1ZG@rVG?XN
zUXm8~y;Se9xUyq$_VJX*=7u(n<t;19zg{SBbg$^z(b03Gqve0QK>c(6*B{e$F7lm7
zZ01UAmVW4cqbTFV7t@Ysos1>PyG-r(rpm}~i;kU?+~nf>tH}I!k@>3QO?;OP0t>6}
z3sxU2uU>ww>Hp&HJ>l*g3*`j1WSMPM|7;>9-WGhLJWg$)#qOjirj0^T3-am>(=ShN
zz1Uvz#Jv9F_PSToIp3@I1$p!>n!uS=(f2Z=?}<hI{q4mMp7*}vsDJ*W?m>8ey<%TY
zV&6lHzQ+~}FFN|}icd&#n83z4f%RoS^Ul8iKWbk_C@`;_@Ik%*_V)HC?E)&QIXBF?
zoEDi@xaYHSOk#^@;5*U4d82`SMT5%AWUdtge;)|z8FFpCz?ZPZTjL^E?f;8W|H2aY
zvwLJ*;YwH(yWeyQYq~1mMYp^Hp^T<z3FB%XQ3Fl0Le7r``!uEsnp+433Mx1X1+W%1
zmu1F^<~(=rJZa8$)q*QoAT49%G@ohHjJKzneV<mZD4faJ%DjLzy<@uF3f8oa>2)hP
z>nf)=Sk9=joKaobmUW{oJ7PxH%(ftn8Nm`W+bn0cu4J8=FmndyjOjaP&Nw-<W#;q+
zl{031p3$i(u;k~A%#2wRZ?>cgmKIBt-fOd5q#(JuDaV!3wsl8&<Mr}-@rvf<9i1E%
zE#h+;%?0#l3e~31Ii)FZ+>+sJ<(z#1{7VF)UYJfkyGTmfF=u&T^U{ThvjQt;AJpCQ
z|7rIwLDkvenco&gUzw7*?7HgPmzl2&Lj(kqPCj(^Uf6xQbZS^<*6+;jT^U)dhShw*
zY%voAj_i=Tbkt&hn)#9gY^j^IWmF_PkEq_d>2I)*HT_tfw_3f-s)hH{RlfX~v9`0%
z>%`3IGg%pK&WxJC%Gx<IYX)o0&6%?%aI$tzV5{si^;-OB#o}o*Cs=hYww|@vE^30Y
zmayTkMeb6I4Wu}u1X!aEEb-%7>VIoV@Tr*wyJqUHT5Ps!Y1FU9QBjM&p6LHpQQv)V
zVnmDUYlZx3_DNr#HE?KDsC70Jt<vDv=$xb>P{!5xjjge!YdLe_@|s!8Stc%T@LDc2
z)4To_*9k#4_5UBGqL^y#U1>bS*i>BPyX3f=aG_A%l4{isfprS*Yzyt@zcSZ;k;EBT
zSkY3byJ9L+AQN*S6YHz$zX!X{eGC;p<oeT$A<$rY&W+W*o-><fE-AMZsNtNzoVaM3
zrohpvndK+f7;?=xxN2to%$aqaYp$(YbUkbBv00oNx7IRWSUYh7>qDz`kD}H+ky>|W
z*UZQXYj50Icd2ToK;$f=_MB~!itpOWtC!Dif6?*hRL8-e<+_*VuoTYO%OJ|4y`lNn
zq`GM}W|^udQhb&c`Upxp@Xz*{^jv!BiOg?{Hm<x8T_o)BWAT(V=H1K76}P8X`&!1H
zR9&!T`uv0CSsTryH_j3;j$ZKo|Hi1hkF&)rq_<T~RDQlfzH#+;Mb?Q67e~&T@HAt(
zcgIqn-HU>`7VmzwB=Gi<U0z#*qPGTlFOA&2)qnNY;8R-@UT;m5W{nhB8Wp|Gw`*I9
z^w!kbtN|C+=6i1|e7!B#dVBWlZ9%KIRdH|iyS=8^LgMQ6iB`@H>^ml@SME?d*`O`i
zxkIIsb%#G!;EFSx8BO1p3utU8@7fS1)VN@x<l6?xU2Q_**Yd@fS5CgJWOKC3<>x$B
z@%aIUn*$t^5*JM=Vp?UOU1-Q%_}g<T<1Tx?$6WUf88&2RGdnQFERgFyTygSs#Zhs~
zA5Vl&nM*oFuf4*(_R8%wr+2M6ZY41Bz+Pqn&Y=GrR!<aQedN6_$Y9@7?`<!%*FE{Y
zH_%|+ljwCXPw#tmd*AEo{U5vcf8D+Rz4yMq-uqts-p|yt@7?SD0Sv7Fz4w3Gy-veQ
zAmm1kE2Cmry5zKrvk!AFpQtfMY|p_%FXza`2(W4kDDrfk{&`UC&m7gBLz8zOVqJJh
z&1S=~o7^%#)2}6SuUQa#x>{;NyFm3M=?9XcTdr1qE?)We<u1;LsvdJ<zilpL30x?k
zxLfsgjK1~JJ*x%$Esv^4S8Kf%K5X3+q!hb}Q8nP|RHlQD0#--NSNmrzu89%Zm-KsU
zfzHzC)vHTnwwBrm?A8*f^f_KN=lI*x$9LU2kh1!C?(S`^cedT{I*|QedTajgwaq@;
z^QBLg%svs=a9~=@$r&{#r}wY~9yqyc)k$Us)`c>sczqU6+*o@tec8J({x27}bG4Rt
zyk546=XCX~<!3op?8w<!r?sN-*6Eg4rw{x&eTe7G?mNqm%{krdW$Pb&SoE5l`ozX5
zn@uuYx{pK`G^eaw%`N&RQb6UJ+djvpIa;!dXN9P|SgCa*DX?)B)52BE1p@DELSNrG
z=kBV$PHWFvFE&+&Y~dpnD|Sykcd+@M@cxwUzPq>g-if-9`g?EVoBiBx_w)U|ARv2D
z$o8TL??u4}`^94~3d){hi#fTNhgDGFl#=Z!j>dH}XHFCF5#L$aQRiN<zw@BM-Syos
zlxF>3UEdpc+4Sw4rsI4m0Z~&A>U3VJ2%Na*#pljv#g(s$eBNG^6Ku&`zC%8n<*2xk
zZ;8?-rPWo(RI}EktDb&3|M8=yPY*XG?cG!%(j0WKvgw4k$<}Kpe;5{|C+?9D&C5An
zQhU7e&Vl<@0u_6Ym+G?4yK}PY%*m#^CtG?CG@m``8E~WP?8&y?lkL7Y`ekoUuw|I%
zbF;x$Ah>#|-HsHQlM~l4<gd}0q-uFfL-O>7IXfy|9lCTwX4dS}Ww&lw<j$FN`w*w#
zZEc&|)4K&)t2&H*oR)u0_PwMc8Mw<P@{XZe$3|9B*@|rEGrP{ko;`lBaOro0jD<2f
z+ot>!&c3@bNjK=|%KyLa{?a^G9>SFRV3p{^^J`|=f1G%jW#ZxJP@x|o3%ksFdnO6;
z>F#Gev+usuK4t~hiZ{mt1sItXA2ir7u*3-nPGH$p^+2$Jh3Vo0rh5;j_}&zJ_h5Gy
zi>TZsfrf`38y-&C`%voLgMbGQnHd;ooIUVCYWkN`T(dss>0P{3YhJNWbB+<;a^t$k
z;ySmP4Gyt9oUI)*=giEVnmqzG?+&@e9Wpp`NN<jSgWqj0yC?qZPVc-ce#6;T^u(Uc
zUn~B^=ziJa^H{QS-9}aE+jo|juXKGWJ||$oJhzoUrc_CbcVDULPLaL0D|%``<EH<S
z0$XNfnTh&u61*ef9Q-uU?0e$GcMIog{}<eQ;=07Ywa3eR1^C`Q;Iezkyx_qEyN65<
zAIz+Kz})z7rk}u^eJ?o_FJ+uw5|yCRu=*6!j%8bNZf*YaierVqYTFe-eYcOEIlZIq
z%vQapTjsq!Y;$IN-p>8&UQhacx~}TYaUYd83>VgZ;EwrX_HN^ythbtxj}ke*$0Q~>
z$p6T^Q+LGerSz&}Q||Xvf2%B96t$|7={$Ggs`5C2XSNJqV*~|DHk(wa+TPxraj;qY
zAXm_Yi#fZGSIiMwxK4oCUqC_kl+wS8()&Kh+C5S__dzZ1`jg+&&Dam>^sU!FH(QT)
zx&GV+!?PW_yc_)NK50(8tRy2k=>_+(HJ@zl1Z>`Y@{udp_J6OEE1&S%1O8{u2+B8i
zI=!`hqqNcPPN!GzJbAZ*)$IbeeL_}f$f#7uwHVD`&+&Zjq@yLm*MCUL>2Ozn=QO{1
zRxV}HWUJHHtY+tl-PC?(!Vujcz?{VW>8<clsSl^LPB9r?>Wll{Yj`Op`_#&N6X&1h
z`#ND#zSpfyGRxQBJDpKD=fs@bUG^0#*WTXR_ha|HAG_ZD*wy=En*Pt8c!8PoH(32U
zG|B&Uz`38Z<_igWaGh?KI#*P2`{S8Y9MQS+?{G&YI(XhazjxP0sUDVDZ_QuGoJn-t
zH}UQ_W(&R7ch{wz+y1+I$L*L_5y7uL=lPREo!;Jyzxi2Z`NV)iAxGQET}+9R|L0!k
z`1q}U&uZ3qf%-GY>vg{Gs=BY)SA0(WphUjp?wt+$0xBeX9`D`RpvYO_87Ht~${Yqm
zhX)7h{;>&}SS+}Iu!Wmdtj0s&;o%l$o?n|h0vE4u`d4%F^V5xSM_ZKH&1_}_Ep?u%
z8@tJdW3#IFEFtB4KAM~3*E23*yCoEHKFqptZPcdBldDp-mooQk3%u2}@_-rhM5A2i
zRXzTGDQwd+Im1#V{g_Yha?zg9ePDCKUcN7y7t&X*Wq!f+MrGQ&n^TrvTD3;&!r21{
zn}Wl1Z4^Ex9cuqC$MeSG&8_1_%E7W3BE~UGCq_>U>Y8$8S>qDczNXv3o3A#zux{IY
zG|M}6df<PyjK(5^?gMuMH@RG7lBp2WopmHbN;&I_M%bCit4ZPAar#w%-QvERZ(8u}
z&E1Xl-~A?;-Rj~oo+X|U@YnO(OXGFz=j&{%W;4A#x=jDx-+xmcJwMHEVE6r>^oP9+
z{5k>qKkKYoE_NkO_{6l|Y&*I{%pOI(UBHvseR}EnTMxK(H?lFTKC<4!<>C<|jo^<i
zjbSDmmu|Wz<*%^5hUxTr##LF1ryLZx9cZW+TI78ENQ@-M$t_cSIz5l^2zPj@EcYp5
zP}{Gxm_=x_%3_}Nc{<L=rL;e<JZyA4=G0OTwyq=F?%tEp==RujV#$t2M+8)N8oEWM
ziiX|z8N4g%)yt*x>iBdu{s-0z*_K?4lW;Y;<gVPi(rA{<q)asv^P?+O=kcsMy2w>N
z>cyg}sFlwb*U#B>T)v-;LsKO@wRWo6ys$?m;jyv5w#+k}vMOES#v4PfmCZfAoHvt=
zlT%kIum^QbT{Z7gmx$31uEOiFdslfY?BCbgy++7V@}gdGd9Tjh#tI%Gk1})VO@<X$
zRfF!Ovw!N^bbOcTtIaF;n9i<j_$&J3{yes-f`*pbstsC8^kjT*GB$MO_U!MHX107f
zCHb7?izSQC?Rw+#Y~7O&2bl96O-`@wx0`-@^9jYjTfF_#W@jI>O#Amo_=V+MYuOX?
z7~d(gYP30IEuXgJc3{Dk`JR)0E}1P}67^qk_Nr-7s%tJMSw5fiCy{U4BDNJzr=Mqe
zc4j5xqC*=F$Jl2-U8Da|c18Dw7?(l@vG3cekBG+K*&QR5`Q*y#O{Y$rTBjuXUF_Zw
zzN$NMC*IZ?#huVNZlkKpdTUS5x%e$Trc+hKI3h2{ZZdn=dU=&=#-`@1RAF5=&%B*>
z|9Pg&b7a^3a=~5J_sjRW0;2oA&*i!6`8<JLqeNlytn)9Ib_PYw`!$ots$`K{qAkDD
zceb~;e)rU+=U2CGcz;6ag-Xr}m2NwsZBxpkJfmLxe%Ai>$}ht#N!6JD+ovpJxMBN0
zk~i&Z;fcjECJhO!-5TFGL?61iD`xl>_^dtQ6}u~Fan459|M6)PC+@Sen6UNCS-nz0
z;d=)znF~#2jhVgc%&rs;RYk^=g}t2*cgrk3EaBB=mNG*uO6hP{gKA!`yQOcYz_Ja^
z6A$I)NjGHonR0zt_$DQMOVX*5t6TrcT<HFDlK;#jXNATP_wqLjSZ&!3^aXY(IqDWh
zT@{<8VGzYqzxtHFw$oOv=qXNGf0Spn{7dY2ywxPKu<Ee{?-zMS-JqWSJCZ6Z)jX%A
zwp%(2PSbTV?K&T`LPMfSOyp|VV*b{Q0*yyuU2F#qiEZU5QGLXu<GI4zs9@nVgG<xA
z_dF6m@^Xdy8z~<H)o|6RF_|+0vpA=+IWJ8zYx9!WBDinMlckw%mz!q)SDNc}&P2{7
ztC;sz=9EQBrsoQNDTOK>Jh66>`ULlht5gr3n&P`@yPrwud&ASw9nD5%GFB%wxh5vG
zu3xZ#wMrw%;q+<`{R=<lRjqLn_WjtdnEi28w4;hj_U2Z5wwGODDJCl04Z3XW9!~sy
z<+1;(E23WPi-O9kPA%1}&Ge}`IOi4XYr9z|pKWq;Kd5ZBRO3>Vc<c@)r<|vs^p!)r
zy#gxMiI-+F+)<jY#m}}*q-a(9+e2AT^rjz{yw@EU_x4ragjv%R#e8yeoL5J3UK6^i
zd(lupSoP>Nv#xhJ?v92V(^X#YaP_oXmN{j~6Q>&!SyldQwpe`ONYTF~8#G)N&R)ET
z{lD&_!+nhxnnf2aXgwG9arPOjRU0{lX9X=&(^<*$N&cNdS7z(AuF#v#8W&H#zPwEA
z_|0qAvbXK+l-WCN>-<(j+k;*=Zm3-mR$BFJO-69Gm)R1b9W#ycm8>|r_iZvi5IWuE
z%GRejIhuE;cm%Ahc>ONC=Z5mZB@_01Ty<ySze)F!7j0LKSH0M=DP{4w%9VOcI?lW-
zi`vEcY)WFDZA4VdwFL{VEV5iVD_#6<^Tde8CwAK!3|gdX0;fuT)qmA8@9T{4#HnUM
zwKE$U)G`|PJo@Ua8Mt80RXu4L-4rEme$^`)ixzZ7S-9|ca1@#=3$R92xGY|LM9xd;
zm2vkCr~W(LynkQ(H($|zW|dmOn-E5A(*ygD1XoE}$}<Q@2vq&)YU~j4vXu*)IY+Nm
z^w^}Zt!rkA<tR=SR{HWlhy6pE<?0rrU%Rex?%z6f?UGMZRW^k^es<`Jp{kmZ{i&>K
zW%hW<PZP}Qe=po+wOP-&_LF8tRBVi%aLjaxE;AKIPu(7ySAEwGi7zW#qEs%n)x+ph
zx9B}>+1)cHoY-{ODoLV2Vn_1zi5rx;JU+bCTJRw$?1!gtWWymXsVSWr3(^$#H#%*u
zUM93C^R2={(Gv>Z+49c27Rfb#abNu=Gl0eSQ}6S#C0noN=9PW!QTUtC*JjcwIDbv)
zl!YqmG%g37YMyf`zm!@3z}o0_|6jg3`ye~=bk3IQ(}qVibj_Uv_0O~VeEH>Jm}De$
zj6LY;loi(v&g4uoUMq4<=uCO>#gZ)lLnWcN=Bzv8>%+S|=#0@cn=a*zpN{J*<*<j}
zIVhU<qVmF9frtGjk%|J5A(c`Z8r+Mk?RlGYe_mU0BalU%JH*9T&82lk+Z81t0g0nd
z5rVB80frSue%m_RV*{!#WQywPX)I^|(EoGmsZjoh^H;xmw<`XM?d>B?XYFU@ObA=r
zvQ#>+$~&~^$KmTM4zn4h7YBWESX@!=V|Uam*eGM|qB4VzQy(%Na;_J!JHPbH<EYI$
zmi%&B95*%gRnhgm89zd=x0{ClSgUZb#C}7=(*H|$c(T|DKP^%`+jA)Q?i>m0o~9aB
z{S)E>?=sHceb8nUApgQ{yR*CXtgR<DHHpa<uh8fb=G5(QQu95mo09pkwk(7{WLAP<
zfy<c%p(18n6;Z6B(or|s_1uM5TXa<IFpQOE@Riu66*Z-`p~+(fd%R;);EMJ$O%7od
z4pkKb(>PSFU*uoJsN#G|qaa#$!2@Nb<gLpe>fYM5R%FSni-&ZXZ_T=A<@ml(G;G5R
z4$ZD$>DgB;W<OTadm<(MOsRFpNAu_^Jr852SG(3VIv+BM*wN~}N`1}5dgi(NKg|9d
zCDT3IqC1*LIdIFg6?2?V%Y47pvr1r2<}7{bMvHkt&VQp0|NrMCYvj<I%Vn@d!r<BI
znL5b^lYX&QOYE}V&}v=KZ0;arY`}b8liB#k&Uw|W<`<MFoahMs&}-Joq;X)QhXtPv
zr^aW+1yd&I+(}#^s%$TQnq{d$M`*_6BT=n}3EMm*nkPoEb2u<H3C*kfV5)M%(DEcx
zActtr3wB?J1|JTwn9EGFJf{RYH0dO;*MHcmv11KmMC+RayrDe}QzVvXRLSpdmX9!8
znQ(DMRYr4P(}F|4XZ&o`Wezgeo-Ow&gMY>k?Y*;9dS-~qYg<*Stn8m4z?s>s>o7Gh
zOZHF3VbjYim8E8{z9DFOTR|YHTOf$js@kc|#KJne^I|Y}=ayxAEf)S)Ix>A*g;u^I
zckx9FiK_MMlJpO+*n9o3)?|}aJ7$<1uvo3IXZq7Bo+}ehGI00!ddSRgnxI^*!|cf)
zZDAT+z&dYp^GjpqSJDi|3WCNjn3n3exRo%68O)k@a(|L^gHMN9*b2Tk3epadtS5~W
zq73++D)8NJU^0wgZ(Y!OtZGGRMqj9hVN{1%pvIIpE7${P9BF7e%_z`1d1Y%x^@?w+
z`xs2zbUoazbTIiZ<_PTACiR+q_m|!UAKK44H8F1BEt64o@0n00Al2ibQdra1nKhO1
zx2~Sj*^t-T^K#6u=$xHZXv#KQV$;RN&oibe9MR9d*`%u>#Va|hj<MG$^5oV)&;Jju
zI5s>qSMOOSTD4B^0n_fpb5l0VwsPLlyn1P~u|=zN$ISrlw%g*NN;{@msLELCcL}Q<
z7Fc$a)AG}nNj=i%Cps_tmU8}k$)t(SbLuWnnq+*qmq*s*<g)3LyQf^(-uy!0iMQL~
z*X{i~SgSuc&;PtL!D`Y%<4tm$lc%npQWzz|;Bst<wMbdO(Yjf}XM{RRe{7YEUK4tu
zDS6A*zy^jik>xD{tQ;p=I=!swIi?CoGgST9YA7)O%?)0eqZb$%9YkfVH&k2c?%;j9
zfK_1jnHeuz#e9Q!c(-qjH2M=L^<iRQMc34ji+p_rL6MKA##Y)?wAh3%be}W(xb2qX
zzjtW%CAs|P&}}wbs-~yOsMm4XLX|Ok7L(BpyXTTz>nBM3<MCJ}u*$~!#G+q&w(Kyz
z$Zf0lfT31#dY0zic|U~CYjYPZyt4AsiDk`uC+C>x^9UW=uu!UNy4<Hp?#xPCZ|+_8
zVDFYod(Uu4BxkOEuvlRF9OLbU3s%G^2+K$<FJKMd!TPeed6CO0#)@W*n}_3fw8xdU
z20hS-HsE~D$?q}8XSY(I#)HZ863ygZx9@9d5`W4k;L5~sW({Kpi`~S&N~r@)Q5~;W
z*r&W{_h?}9{js+F!mh=y`(`wpPTRaBQMi?DiL0T621ANK*a_Z>)CtOSW!J7`zrL}5
zL85|TtIs<Qu4`x3sz)#Xzq{+;-JpY$5)5v<(74$$`<6=cguClRcP@=Eij=UtIa#$?
zcLI}i$ZXD)+qZR2yC*HtsN(m>%0o%zrqU9gry)+St2|!j99qz7+<3sxPtl^?nOjOz
z{L?2vy(P92b9%nAI7_XXb8=(UQ?Bl_mWMO6S|ySWZzw%H?T3EuFYZ=VE_SEhvO`AI
z8bZbft%p7c1{*Xk7U78&urb^l^e}P3Oo0f+tsGSy$0kOIOzvXhxN&`(G=q%jyr>Oc
z=0=)<KYHtx%$yppR~{AFu<(qac3`Zre4~;^`dx$i4Hwz(^ldj@lg!LlwqnJb8D32p
z6JE{;(EZxsa`DnbM!D7*A~P~<-f!IWf7!>46>~%<NCdO*KK{FabwSCgu#SLTFNDrd
z41S@@`10!<y&3m}Hr#u)_WYYz)pL>!FJmY2S{N-}Xk*Ibu~C<cXIHnSvW(N*%`a9N
z|7i8F){a_qRhsXD^r?;zyJAKA>Q#K7q}K{;>0LeX*6xLs6L0w}wV0gKywIonoe%e&
z-7?c>Y^i&>ibpg0;pg*eTs)@PoXWux^R@V|Yc?-kv(IP)b8vu{utu}F1kd6Fm$ZY9
zoBlFwh@E|?%I$9Coq!nDsuk^bFSJyi>N9j;GStv;4qW`SU}K84Ql>X&j5j|s%Q2Z!
z<>OA>aaT^eWjg%NR$jAa*874mJ>COVPZv4O-Na<r|G)bLM~{Lj^Y0sLI3g$cMow9p
zJh}I>=+0zI*NK^%?lrFPxBnNnRfAdc1J}+jrTPrk^%`O|&g(LEZZ|)7GvrhJ6B~>9
zvFGNMcE0HGdn?7@x6RLauG5RGr466YHLrAve|4qXc}p+1+SffhK25rv7BwkF%2F*e
z_>!^n?^lyd{siY#@mT&je??-$iZu#{g{Lk|;QBRjg0%*#@Cznkjw9h8c5(>MdgwT1
zYF1~IMa)hcUXBx6SFIKfeDU!9N_M5ITe}*}#n*1secD-n=#<derc#M1QnJk%(e2KP
zkGOj~>Kcx4f4w{F=ZqN#10H+snxXNc{B+>znaZmlcD8<7YufgrLgn&*!>R7o78k0w
zu+EDP{@NR0o*Vr8%%m4b@4Ys4nI!k}{nmTWas<V=QaRktiO#ymd*S%?gG(M}Iotp3
za;U!VS$n_5<gw2@Rg2q9d=jttrNsZ0b}zlFuwvJG@4uPTTCxspX!eioSvgH{t&;6&
z=eaUxdX$a0LVd4Rzn0{l=^63WztfxZ`a+588C^m$vmAeQ*#2QpHezPYWerbY{kO8;
zNbC-KjNH-&CXE2TlS(s}7$4Blm|C(m;8f6JWwqGE(@p7DE+n3A;gwynL7PKpso|=X
zrs8k<UnlrZPBiVvJ$`S|4NdL-%li&D-nbFfaW-Y0uf@Ab6PLcV+V(a=X`-IS#OFGR
z|LyM0v9o(Q*}41qo47`WcMfdbyeH0{`Oy*fV97n64eQ<XUjE}zU)8n!!cA3|LwXCC
z9>2T!KCg43uSHbTp~az{x(T;-Tl&4c^s3HU@B_>Fp4SuQKdN?f`~T)UJaNk$J(<}-
z0`G(@Eg!Ni3VLvP<KN@V^_DwlJ2M7zo_sA=#wB-Kwjo-Af&W9(;*>;}6qk^T*;S`F
z*IsUEdzXDk<T6J%NB7C5mkk|t41pR4ybM<eKMuINI{NvE){l=Qo(C=LlS;n*S#n;<
zmDr9SB2`cC`doDRd*Whvp;6~LcUu+Bp0*8dqA#ys+HLmreeAyESF)LJpMHAqEKV#*
zb*0uj!*WgocftS0-=*K!7N7e#$@*+?o_C5$)sE$#9!x2S-?D(G-Q`5b91lzB)qdBO
z_Fh?Fm%Y$-`D}5)2g0u=^s8)P`p3;&_a)>Q58FKvg-M~3n;x+3Pq^*O-96Lu@#F+<
zPJ?QuhtG<`KI~{e67g{9mx(-@Zzl<~MuaGLOSInqkmMZMZs*A_`Pi`9>-34V!Xr(E
zAB(+p{j8i1@-2C6+Q#tqpTqH(NZ<5+mA!IY%Egaw`J`CfTQ_O%+q~(QBVIkecP`hI
zKjppY_x%ES4*&L8K4qF&7MW)~<L>Vz8)x3!9Qi6&{m{!J=he47VSJHiH2r?$CO?0+
zlEtN*e`-BA)b_rfCH?<z_m^W6|F8Aha^rKz^ni-*J?E=3)l?>~|Iu_g_MXQb=lIWi
z)&BkJ_L}U>sQgShNVBvkW2%Ff)0R)yCQdcXJhO`DdX8$~{RxHJje{!Ye!V|wO=OOM
zQ=*8{@e3^0R!j3|9(}z?QJmv?Nb;QIA|p}F$F7$(@>@02gZuwwOgtAiS?}e>{rB{r
zpL<uTYIxsoy4us{M(<4PtGnY>Kg?zNXv?v5qQH^^o_vbUd=sxMfA{%?kJiEe#SFgr
zn*^>L-!V<dvt(70R$unsb&pmtyKy^vcOK&J{x9RVZD#pKyG5%GoD5H0WU)tJqld&L
z2Cj{UQcD9C3)LL_X3}h~amvWzLVQM;#VL{NvKs$8v^X42PCl||QKY=&<c3+hjOy0%
zJl^$4>YmV}bpN+KR~@#Bnc7z6-a6A|wqfbbXWuNJ)!n}KwlMwN+tAN;^Qx!ZyH{-A
zVIj-?D6PIx`tx&<ci;Mde=o0p^o>iiV_{Z4YtxmIS8N@OYdxk)I<#@mWaK)X6M5pO
z;t|Hw6&i1x3LB1g@f+raya-G@K0)5rOy;HF!sAWcwrV=fm!6(#(e~{N(JW-p?95H!
z(^z_;_dH`B+ZN7SZW@98?)6nNr$QbEE%9I4R}vMR)Y&e3dP+uSfa=P{Icz<l5!2Qj
z-<)-M)6$v6BIhPr249j%IG?;XO782g;2Y<Y_wQq7;{Ra!VEXF+Ld<7o6g@uGYa9RV
z&)>hV!#^+_oF@8f@%FB}X{Yx7n)oiX-{|F~#^1ecI#$N7uPipNe{=iw<FrG~zonkd
zy*XdM{*WqvKhK{XPvZaO+}xddUaTZKFUpraY70-<x|9v+Pwy>dx&Hp)!RZ3=afWW6
zVmGYY9=Uzd@vk4aPc65eoc6c>t^2k2^_TZoOZ+ugsD1PI`6QN@8<WDVnXe>I*Uep+
zI?a=%(S$)=I(5Z`Hi>=ZlRW3ttPo(AE=*h$CO`AXMHc0cA#9p)nl5hb%o!GoJIqgp
zcu6Y#G70xr=K2{I5+*uRB*aT|rpn}S!K)W11lC?THHj(IQ+1kL;LD5A%Fn($4gBw@
zx#gtC+Z#@*+RPV}Q{tQMKK2TB4y?RYJ!9+4o2kofUkIkwvX#wB{rkf5(mdx+k>Po2
zUpb}I+s(3O=x&`8>8-tWgHeEVqM*w$N4?Tht=<cptX8d9q!kuWnOYMrShac?yZjoa
zW3S@tHgWBcTleQ~`szBah2q)kng5(xy-;5B+FTKS*A-9ozb0@#iHWp}KBmTbvL&D^
z>f@J3t>V2Zms<HvpX7QnPpN*_absOdup_H%$%P*MyCz9amd8RT_@|024c5$kaa%vg
znYVPx0pE)(>cIs{HU|!s>aH<3Y<u&|>y<H{DY=uf12bMkzIQyG8sn6=bB$9gSK4!r
z#b3XATmMh}a3f!PErZvs1e5w1(OGIsr)@qbcX!!^^BX?8MdTmnZI`)pP?L9#?S7l@
zB_;iV*H4w&ByGQ27E@R_ukwDSuUvRyGw;1U=3BGNZl>2C(Yuv(`RF!J#&zBcYW<eQ
zn%^lq?=F1%Q{(Z2BJR4k9~7@ux$?0^{^*H??MJQtRctf!nx{ME>8=a`b=#<H{k?N8
z+{k;z@>J#7<E8Z?Q>S>W%1a3f+c-~X$``rTWeyjX?Of`#<c{5?cU%|k7-HmE`E5?f
zFK(@F*=BO$l*St#wi%0No4)PRPW9QoAn|;5fBNFP>D!juSKo?}p5J)snQhvW;<8UC
z%O7&CTr%~T57#ooy!H!!|1X`iYyZDZtEyJ5W~x{dI#u0n-z$%`4XyI=KhwjwOB7gg
zuR3r_88iv+Nnq_&71YZ~&^x{8Qt<;Pd)=4kyw)t3|D5NNK+BXC9^D+p9c&NQ{>WLv
zZF;4vCaFWO_00tpy%LXmYqlPemr7C2lNP#Nv}pZKhNtR(wOUITtgyNwnWMF?`^Iax
zO$Mtyj>M)a9A0^iZPUaIox;>VCnSrX3ksfkT5|Lfx9$#g_bHYuEVx%4&HOcK#px?O
zeH}sa7wdwzEcHywJmtxEdCsvpuDc%j<SvNP-jr<mE=4``@y;ie2ALZ-Ze87wx4Qq%
zy~ziXT8*X%@`&9pNfqL`wlu$dw|~W@XP*BVFS6UD%wUq#GHu|vddTP9VcACoOv<81
z`-+uUUF>!GqARdcBUa|Y!ktl(<`a(i6(v2H<W*s^YvP1+CBaKLMT6Z9uA08oc>L!?
znT&heiU)mjyFTBINlRa<lFR9#CV6kVhoIQeiAln1GbT7mxu}-)DEanTpH2BPWuKKL
zmw$QImASW4B&X<xDDLgNza;5cm(SJr-R}=g`{#6Oe_-kH?ElGE*Kld*^l#B>-nmG0
zqn(O;@S!csBm%FljhS^&aA*25&Lb;*%1zIv{hGiWbVJBNTx4O(gLCCuPhaT?lkhFP
z`6B9r5l7%!O=kgDJy-cnGM@LCo!8G<8>=~K)i#Gzjbyd|E}h?RO`pVOHc8pkFv)LA
z(vfX5t_$+Isy<27OxV~t%_Og<r#ba>L{sF#3A`^YnJ;uZWd&a7{FcMuxY{QtVDs_?
zwy$biJ*6_57t1Wy6w10#*ebi^+lf`F?AehoPi1evX8N_@)A|U_?AsUEyzfryRli;%
zT~MffC;XOf{(jv6&5veR?jG~qndhBXyx7}&x=(`MgthF&E0)}7J>ub_6eVc9m(z0H
zBy$z%n}>|I{t~Q~{H&rL^!{m-(ykpHD`!9Y8}Q0*lI!2T3{7{#)YxyG*0*?fv@kcS
zH$AUBKaq!jen9AA&ofU#S`R#}sdK#V&UH25RE2-_za6Jj?xainNZlg#|I6hoezzDd
zP3$sR600%QC9G8|VlMkqp`1+LxHU22iOX|@Q!<%DrGg_lXHFH_vcPZRnkUM}H&6KW
z@oy4xGHU#>NOXI&Oyl!U&J%eGJ2rHSZ|IS*6wTVIY-?leD-|T<zxC~!)V^hXS5BI1
zvOet!{^&7xbxQixn<`pn8&6GJvPDoxCndrwC`VFQcj4Qzlk<91mpeAF1nl+LR=w-v
zn}SasbCyWDZ|Q8gRV?+g%iXViYvk*`Yf0Yo)-IKry!q;q^;R>tZ&sY3wlB4F?d`AY
z_3Hwa`hJ#vy}_2?yz|rcW6M&tmhLuB)&FkGc<a2^H|1QWvzPKyZa8>WtQ0st>EMDq
zmH*5$^-`2Y0)y|owpH`B_#T(h*21h^vc30#Z9{qy<KFDpW#N-MH%fY@?6Y6gS?fC~
zUg_n}+22)9t!+*5&x~F9(6=b7PMXIis3Tzhr<(qmY*t+lO+#vxKk_ay%G1>5Us5;c
zlSp^4-lcaJRD0Ggibx25wYqurldLmu7bU-Y&=tMopk}JaTHQ6<KhHbNwZi$^OD7?X
zCz(ofkGc7s^q-wqGW(EvxQE=+Nq@^G>ppQg6t>A-+pX|H)3tS?8-qW49O%-mI2xc5
zQu{mPq0F`B{j1M@aN6y8?Ob%pcG1<`hR^rE%h}L$b?Sbu%o*+qWhxz}p7FEqs7NUH
zzph<YxLe5SapM2GLeupPZ>vnGeWP*Wg4#v1wO`J4MwPDkcH~VA@3u8zTPu&K{odcS
zHaA6jokXkeoyT#)+a(V3mCAjdu`~7g^$_Q`EBN!<ySE+`Ek6;SSN#6Y3t{o?PJ1m&
z-haCz8*abjq~7g~=}dBZ4~nK&-&A#2X>sQLwBIgI>?(F0(!M%*ippLyr~I9(ZtnQo
zp1tbValuH>Bd1wB4+zdV&=@>bCc!fF<bkCwhqhj_(!9~0a&xLE_i;gf>5YL~jtKkC
z>2%o8q!uvi&y0wZtw;aXcy+Io`Tt@@<VvsIECN$*&N#F7%7zy=d<*9n{yidl<xY|1
z{p1~u=XN&UF}bh%fkWKplI{Ud$pimATVz~?e(`Jj-eoq8oMb4em3UVqDKQ{5R@Ck$
zTcY&IIh{eSv$j~w+Br8i^x%b_3UA(pS7*<Zj&KZq>EL;Hr%Uyj`$mk{&&KlY-^vp5
z&^s`q-RVZS@KedSnMdZyo;>WC5H8`Oe2Zl{&mzl<EQvEOKGhX#wY|02biq&8Fzp*m
znm!CG8ZK(=yH>vE-s`jTR$mO$sTIo4d^lmF_t}HKRy%tH$^yN1bURL9ko+U1^5~Y+
z3z_v(ow9Zw+^2a=x4~uo!fU5@H0mx1Gn#bidu8yqPQeDB<juUd{a*5JihaCgXHbQK
zOZSJM<^=H@wHy_dSEY|laKGXaw)S!Bp6I`4llQFrf0HfZ^8dH-Z8L5@>2c}1z+v?@
zx}Sw}!WHp(oSdS%3cdlolX&i^e46sOXTswX!Pc3blL9)Y9B};*Aj9T&$F7jmy{@Ij
z)kE9k;K`*i<r7YMot5xj_k>64)|p&o9joYI8@K+mE|M?iNS54pw^<ml<)CiZ(Q}FI
zQFjkb+t3@R=i=BSCC<|tXg1OJS;%EsNx!{UCW=O0@=aX7XNpLiENkT45GjH1sHfc%
zQ^Sv4*{IMW!g0?1iqq7mM;bkkh@D`$=Nc?CuO+l}YS>Oi`)jIyOoe5OSZZb-`4c(c
zsJ3JA-)>9Bo&yeo$y}`>JDWsAd*T%1VtH7PDD-%p;8_20wiN4s*CfTTM*$wHLSoNn
zE}T$v@44=2=Y~MX4N|&iFBQDJS2xF(A@BaTkF(!DeROq-qc(@~zZJgwzsyi!Y}MtO
z?{tH4A4l?*sScWV7(E)=4@?m)<KU5;&=6%Sf7n5N{oiHUZ&LD}s`&@aPds<)USa%?
z()ksy<f<PnsQ$$(?!%yNBKlD0w3}muy_09>A62VK*SF^|HrTc7zAD{#>$Y>kZJBu-
z|Gq9>o7fk_GuhuIV*<}37o~(rK1mjVtx8fdrf%IVQA-vcS*n({=))Sp=^GBRy4`S}
zkaG9H6CN+&&82C@CtsWwTyC^$xtQp}^&3yG=WOM_btw7EqR*a!hyP3uuzA36=fBh0
zpDn4YjvQLya!@6?l<xpv*hA4z>Z0?4#PeQi6=qJW)6|}OO615xb-RumhUeVxsP6W?
zx@|*Cn8A_i7Z>hS20uL@>5~(1Yw68f3yyrPJ+W5vqWm|OR~Bgsb{7x(rX5g;J1CnJ
z6yf7zwQ8}{kusC3?HW;uinF8^s@3_OI8^$mnd6G>r6=tS5@N3wiJv`@_Df0k)FNHC
zNy)o>jI7)Y4|)2%iTOH(X@Xne=bMcEE(+_Dj$SjluKi<<(;EfOkV}F}@eijmvPbmy
zJE?~ytgHx4Hhaaj^_ymonD{2w+jkvq*Z=W-aKl;ORjiEl){F@qJ9$0r7CrvG_xg;3
zk#q0ex0U<<V#n6S6Lx5u-n_qJ-(`g~;bcjbR<#AKMH#OOCLYjJIwTnVj%P+wDyOSe
z%wesQoTVzALj86Vo1#iQQ!ciMoLL$$bNOSLi7Tqoyf_w~405-6I%kDyqOjV_WQU1s
zwOoV_aSBQ;bnwtz;qbE8ZMv@&>!BxivRCB@-Ag(*D@Wb-NJvy_PpfKV+tTndhRbem
zFIfphXsx~MmU}r$_sX$T*P<3W=NwQ|=?}J?W^?iBoJDj0t)08er7z^_(_Qzj-~J^Q
zS{oXgIMr;Q``(19(W<O%Og#(2SW~Q86>m)y3l)5r^lb4tnVhAQeFElXM7^p`$dWF+
zx2*3(tU-s<9M2Wsw89eq^K8G7_WYNu_YRgaJr~_i9Sr|w38o243M~F`^3@03#4OHL
zD{dz3S7R|2YdQC|#klO^+OIRth4x5t%znouU#{1p-{f8zHc9JRT=g{dzDer&jcc|g
z-TxB!{!YYuxv<x}rW~9xPhQUOc!Wdfj;GJ-ytET0h5x-PpV;TJg71Lsy~TT$O8YK&
z$n{dbSLIGk;k*{F7Y7R$N$Vc{RQRH9<%}1frnvOE#2Bsd40zREF}qW5TKgjwb+z*%
zUsxtcX?{8P?7rgq$Bsu&uX<DB@c10-(F<HF16N5126;;Iyj^kNr0x=7fvHX0>Fapx
zGh1xtC#$48GALg3yx=D!8>ppPu`H%$#{W~dr=8Q#-8lWwBJrxVOu9FxdTsP8Yzz!M
zImNZ@E?4LDp8?#BN2VFRkUlkc*0b1(r^<q{qCP5x4n^F185X|~YgsJGI?-#9%C(IP
zyaGdBTzx2+clgI?#`{m6WL=-&u})Qt`)S^zjyr;$Eay%aaV^YMTfJ#sGE;OZhy40&
zdF$BX4XavYJ=%S4Fi!2@)cml>m}}7ypH)-7DW&Z4DXMBQh~htQxL)=5i#>Oe>Nh;f
z{u$bMAz{fciC<bDAIE6_C|u8Ycdenacx=UL-w4^#iHDx^GCbU1pgHHdwN%%?x5+y@
zrT48{-<5Ol_dVr?bJ~AS-Q6X6y85R4-a_M9r(7SW3Ag<FzuH_?r9DpLf#0Hoe4-7t
zfv=`09!^>^HALm`&dGLxI__Bttk+#<IZV58%V`_y%9E#D6z-`AY}#<nHdR|CQBck`
zYbVn?siod)3Ra|iS<(7wkBiWX2_+M^bO$rZpWepwb<U9hmb-oPSX`g4@#^XFWC-xu
zcgJl{OhV<^t4eRzhCIEhwbm;oH&EL&IC*Wb`PV83-4OnUklS9N&rgL`yM!I*S}`M~
zB~()K2*<3Ib3+>o{~wHv{U%`bTs=Cv_+O=(^zzJaR({n>T>^icj%8jX>&z-A?Ajv7
zI_HV;L!)_%zTMc+aPjB|spTpx2Re=>wyLGXD?OU=>{;){jvgoJ7`>FO|C{3`G>W9?
zso5Ky;d71nFZJ%+yRs|mqLzm7pFdG%ar7i-UZTJHO`Zdvc7A$o#_F%2{z~ipV_EOd
zZN6(3vcG$oXjne)o2vQ8oy$)=diE}Kb;YN+`A+Zi)~0_s#9NVbKqTN~nzKjjj`<Ro
zGynEQhhLq&lOw}Y+vU%+VvEh@f2I}QoBP>4;K=Q?K9?sP5h2@_v2B+2J$fLrwVLPl
ze5(Qn{|!vdFD6)gVC}llyj58GYU|Sw{(vX19==@RGNDBNc89mleD~cg!KXe1ChZ8G
z`sEYr+<di%K{`vjzDB4XKhpJ{UqCbY&59*Y1OjF}GwNkMz0qQ>&$o){|9*t?N!DjS
zy_ov{VMoMU;nNa-CLNm67@%E|GjG!ofsdKfU8g>IuyD?)hkLb-e4limtM{eIgm}}n
zlG1!>b2dqC<2u>>hQ&NrY2htHE&GWvoN`L%R^%;e&$fH9-}9U5ESF5S8y!)3&$?9n
z?`&&TNp!5)6fBamv1@jyfYcVQxdQi8)(hP_xP<r0E}m5h$K!)DOSaw8;q=zcTtC4)
z`b6I23Bij^l*;+;i5n)bUm5T7Gc)>G&`Gfd@#cNgt?Ry<t-oz~I(bhc%bv#1apzwC
z7P@@<ReY6i<Gz3MSN@v&G2&Z1Q-QNoLi41c<wkex*-f_lb8crYiRp3jNJ?;w*O7`%
zFmb-N)$pRmGz<TL|5|fQQYS6bnaZ_k(yz3MdPf+lH?-Yf;&-|tIA+V|z~i&`y`FF-
zfj2Gns_2A(^tE1wHixa>t`Aun=)W~I&Gc%k>ap&p!Q399`8G8*dybX=IhtMSr|A;P
z92S<b{YDgLNrvUFlAG>VWADc8JiGd<%sbT?`3KK6?~K08Qv3N=#p36Czuvv|<f}8^
z^H}4Ahf+CdK}TaaPjB0E&1;2sM2=@lyIN+{qZIvFEU(LBJ0`aUMxDOHl6|;{`TYkU
zht(Ud6y_e6D&I3tU${KidVTKHsrwI1E8DZ^=9xt$g2l6zr{}6|{_^i{sl46F<?3tH
z_ZiNws_uB4{ML0RpKWFP_Lmv0yM_L*5WT;v@m6#-|4G*U=4Q7~@YHr&F)ip@GU3S^
z{aW9QRmQf5oNgV}&~Y!_?J8tplj_-|TQI*^rt8s>_u3NDZ_3|Ik$-T{(fdWrI<JZA
zzJ?}AUljPiyKQx-i-(^vzi9WwH+v%b*HsprzngT{awBJiuGZB56Sa^0=uZ9BoGS9&
zLYR5y&&~T^{+Xe%u<EaM-_=r9NexHGEjA82vP>IJH!w!?sF`#KD2f~L$_w?pxR7wL
zl~XV-WXHs0*QQ{jCKZiTZV%01hedBPHuM~ws&BhXCV)-N$4J`qY|iB31AcwELc26s
zuP9tFb8c2sxn-Og*k@vX&%~px#l3mSv`sfptopAU-W<hSCCgBj<G#t_;vtoz$L=*$
zZDYG4#UTEQ!F*lXG>yk;4xD<a43~8lPd5<X%bdq&#`Cq{{=rz0X$BmXMTd4x7Y<Up
zQQ@$1CfCAgyR2tDl?``l<-4$EXIl34^@+!)-QA^^ae90D<#!!3m!_VqtY&Bb7nvx2
zKxo?o(~Q;k7=mK9op^Ghwd!cooK?}P5r&JMUa(mjJaH&EUGYNt#NzN3aSx}oyyI;;
zv+v^5KV8CXJ<H!u<WX-8s?=&;-p6+4t?A-~=&Y49mu${l-pC`xafnf6$pxprBdZFW
zcrB$=75UvCY985Ik)(WN)1#=ob*s&8Eb29ymbFr~Ddo>;w`n>5|Gd?oR`+`K;`<7V
z*SJ4w`*Tq*+-LJ)-hEBlhfU1CS@rhp)@jx{aVF&Sq#fG5kAt=v?Rg?}M(6O6Pp3{b
z$prhDY<|<=qkCT`_#ES%FOz&vys9xiZ~XX((Rtg4PbT{|-&wO}hh6KRY_2a4noh55
z&Pwu7>JXU}64cAH=Dl*u4QU51f7`i_g1_tCUAKTO_OyoqbNH^z1B|w>PZ~>1Eb7u{
zX*|@ezo%krc0(L5(>60B7460dqgP6gmo2>J<)OW4-rW~E>8G2vMfGvz<lbVuRd`xq
z=bT3kH$vI9uPYp?`NLyeWF8f|<5r$9@A{@CyLeX_O*rBmcW}WXp$$QA3YMD1o&Eo!
z_il6f;ocpsym{ewTcq2jo43zzs(9G>Q1slk^2F0WRQn^}uHfElF(FBFXO801%TxW<
z@?8z~?VY)+P2s81o~C2c{Ck@&yxwX0y+N+w%5|Npom>C0ec+FNKI7cCRTH9vHz==M
zUcHvbNNh^)YR>panTPqzKNaZm>z+Azgz-e2=aT(yW+oR-v^bY3Yb<>7<8;gWP0Aaq
zU$dRvV)jq#saOST{!-Ni?;ONdHNDrU@Yb>1BwQrSC)@DpWVrDOmQ!Ze=X^e|I9<2+
zwBz<YpYEu%R~21NOj^5?b4P}WlhFh3(nYs3k8RogF6+{i?yau3lCJRDg|7I~XY{J6
zYr^Z+UH6{N{NEO+HNR?A`HBeMU7;(N2_4GTR1-cl??y?wm6$>5=>z+g<+2NSD`s0J
z%|9@UtwvjMXTzfdo2!hMdm0~Db!5`zKiMYCPtN{~FcOKHxIo0;>TG73(Sf@R0ZreH
zJT&=~H`JRh%g-;KekelR>1KmZi}Y@ZD>l=dT4R@a9A-Q_!HrE~f<uR;_eK9YDG|A*
z5O(WXO8kx`8JpW!0_~%3L^|DgJZUOtu5$2<11si5Y+j}6Vr92vk#Cw47Xzb*!`zpT
zE?X?xC!n;VCD^g2c*isWzO@&fm?OEQl}i?GnxK@cAS9^i@#Gj|E>q0iD8W`QA(?Ay
zQcTY7jyljVY5M||w!B=2|7U#|o^>^F@@Q<2nDl_<+lCgOli~9_r?M^;(Mj2PXwv?y
zm=}Di%M2nrr%n#Mto=M?j`hlyzU4-jZ{L}=(}>Y)dPmk}{pwD23)xrxTfba(V|zK#
zNzu5(Z1N!onN$UD$E;a1PMux3tijh$l0QTykV#mdX>*fKe&xo8vYfGtw=9`!sJ`wO
zr@z3Gy^B>Yg&+8J)wcMmr|CWplgGx>QkaUTe%bpz>SEWm@O7;Lt&2JqW@g^<+B8j|
z+GVoS+6WQN1ws0=?i`sgagP*3`=a$`9-o?&OPR09Rfu0+((mQmC@`5JgPlj}Si+>p
z%%BfX+qPM5FrM%EGTHBhVN0S;qR`@qfdBnbhP{mitzycnf+j}^i%t_P)t$(onRAhE
z=ak0s+l`(jE0q0a%CcCQcb|(qJ8LuJ4<0ke)pJEY2p_$<AWPYN4wt$`mxN!A$a>3<
z+LsfQ+^)G!(X_wGA8S5SAZUqUuguR`PEx;xHe4~xIcGV$bx+fA-vV#NKB*HT9ZK70
ze(25%kX(5$EvY?hN7EEr&BMC?BDkV?o^1Ur^-^}x6E<naR<-8cx7PodWT^MysENN8
zZ~VVa;&q`cK8+LkQjeLeh<+B7a<zBPbA$G~uP6EKp6mVWRlv+x=hAhhx0Y(XiaR8=
ze~KfQ`9h7PUAJc4O>Eh>>7#5--SG*NbzZ3)w~ljCoABTFt;CW!Q#6-nYcTD-W22w=
zp(TAy-=w{-m%5p>ZZ-I^@A|s?JZV~?h8^uwFLC|cx?Vd?(=g{Bms?ZW63eeET&-oc
zwsKBc#Isi8foJ5bAgS*6dVjX?wAf`$k2pDn#Y8ZIx!HMpkkpeY%L<wwe>(JVH$$6o
zlcEm~XYjhiSDw~C|GMewxo*EDuh>?p)vRh0-t<sz<;-NqOQ+c$e7^Vg$MJSom88>t
zt}dTu9PfDf(KspoQ`o|X%V+<1nCN7YcwMt<Ybd98kC^D9E%#Ub*wdn_>~rML{GJev
zzML;xWb9>co@(OAtw=eXq&Zi~-YiQ&&3RI9bRqXGyG*Zj8(4eT40cTXfA3e;E~#~r
z#cStS|5&xXq4O)VX~*7&R_hW1js^CnrATzI`R<mR{H1(h^<ATzll&JSR(;xV#VNo4
zwpZ05RfCsT)T4{c{kbyDcWHmlOH9j~c~m9D{odZYCEQ(V!UtB#th_0>JuhY7H`&!@
zcTX`c5L`cd>+D13--S0n?KL+4m^!!q(<{GOXI3q*o*CMbDr(pF#mn-?p69IXrZ0=9
z`fs$h%y3$nx^}wsmIVg$=U<jPI<ZsAW%b*FqiNBJ+>Ljin0%62n;!EpnU&F?=J2w>
zf_o2U_`W(CfAo@3dwcL~oy|8B|7{jOY-cf9x;bApxc61i>2n6%Gbg<DJ^JO=%&xMn
zr|<rMqk3jWoX3TGSF6<1v^Dwb4sy-iu;}ZW<y!npYK~pI!SXh^!1P2tbB_GJ@D{E~
z<-VV<tPE!?6<loj@yzq;U#YDDyq6o6oo|l&bmPkEiC_CnPiQQeWR??arfvCgf_!L-
z+qFv^FWn2LxU!{BcyRrj-Ud}`-?r`=V*^=(<xORdJ<PR|Ut4o^B+Vy;CAB%83VO4y
zqWSyp(pjo+T?3BU-2HiUwd$vJ^(E7#-+SHa|M$IpUCEQs`rnWLPpaWJ-t+XTdCAU(
z3a@qhl#)Ij&<tMZk+J2avKCWDi@WN#)oKcP3{w_l2u%ppSkxoYlU(FfRitOCx59YK
z@#+|7%LR)RHsmq<UzOyxse?z%`$&Sb_MPPm<C?VHa_Zue7tLB+ZNX?Sa5DVO17oH(
zuO)puT@F89@#fWq)YnV+PH#AOYllvW+OgYNkB{9^+x}!}UYB02*Mr?I{-s8`@Akbp
zbH-=YopY@Z)L!q<V?EolTE(HK>P6S10_6uhHs5&Glr1}dF4<b7-*(!Ghxd{%bg8df
z;=sh#aDlIot3>1Tv&RS4IDJZSTAcUz;;&9s8~zPuOw}x>9hN-1^nma3hBW1UYFBQp
z5b+GW;_=pHol^fiw&y}hv$A|%w7d&vU^no}cAIeYXHLqms6+Z@$+s5d?>&)zcE#!3
zh$PPwe3$1P@l@b2@!)KnRmkA8c*_5fmv@f!{(HgyyX&IVY29ggnfJcw%5-ze9RE1u
z#oOFd6L+mUa_`dF+*i7a_X1~JI-@v6Az5c#P=eE%EzWC>XsUkFR-Lip<D<nDS?x^1
zOphLUWn{63${2=9xwR}<K6g>NlC{7S9iuRhck_&llrNu|U$ex$E<3V_Z()YtGl^9y
z*?9{^vQ=XZvtP05J$ZCA$c>5V`NU62DI3=0?)qS`AxC1m)|)F&S&nYd(_xB#*YG$(
zDf!zY(Y|-Tbxs(S9Wkjn#j{CZ)wwrX6JIsoXnkjAxGG3}@0kr}ju^b%7m)IYscgrw
zFG+_KMAol4$7NfRw)RO;=PhM{yIKd>&UVVY?-aTu@Spc$u-So6TNW$UrXM)xaZqRR
zXSrAF9r(YT;yB!Nh)+*fGL~5@pm$>w{{y39)ewE16KR{ma&jKM(9Y4{pwjSyErDZZ
zo_J=L+X8)Q-lg1*9!ArSnA~g7U`=+Pr?+6&@+}Kk3<|j8q?W5n9B;~C+Ro9W-;^!B
z^yA4V{r`IerH`-vw_@||5Bh$4PVYXmgnO&7y1Bbq^AhXO!~eY4t*+{)>{)BN?xK;$
zqOe6w^2Q(JlAR9xC=A}DsVZln8t{nmZ@02QA(KFnveNqlYc$M+ax{{ih2B0hzVh;*
zOF->`<xey|KRNK+d_|M{1lLFf?u8MJq6ZE+d%iKvDbRL0mdSBi?EkGpN6nhls=JPz
zNwqyIeA_@bp6yiJ1cAs+0Zl>1OL;mhf;3(kJrgZCrLZqwEBWb<U7K`Dncc#b@2Dx?
z`LjG?O|I9`wYyH8=bm};&8u_f_gEMFXyxf>SF?I*^{x5bj>8HclslhX>bjB~_~@|S
z%oCjrYF%?)2z=ipz;CGdPw2oM<<D#n4lqo;xTLv!fx4l{B8O8Bm-s@TI2rUfsb$-I
zw3S{P8<6X4_&F%@y@=`LzI9Dk;?7^uyKL!FtgNw$DRMHaa|h3D2QAJMC-NRP#)VpT
z=b3zK+Ii-IM(pLp7{*g=`5wJK8VTAOIh!Y6yC`M6*lYX8l37=zyQg}4E$}-~tp2~S
zNZq<2s(WdaJ6rVp(5SVgQJX_e=Y5U2bLMP?X3X0OGaeY0eA3+&QLy6CVg;21;RTCU
zvrXOis5`ArJ>H%v{e!V;hkJ-qtHS-q6#|<QgoOk(1DLb|6i;ySc&~6gf1*ca6SHQJ
z`jjN$rz@U#=`GmNz>@JQt4DS5jg`C!1<#WBo&^*g&B}`o;t}R<*<itxs?&TdYIA)Y
z6Kkf*0yC}SL5Edc4vYPJ<`I`-GQn@_hojvy_)h3Bd96ERw(q^(qIJK2G`&*D_<3Ne
z;RZf!@e*x+4{cLT?R;jVJ@330l|;!mN1wlzv)%7>QIOZgZ4tkma(?<}Up*K5Q`I2q
zkWGun!~aZgIz_@5r#LXOm~(DZP=9inAvMWHpiJ+n(S=7Q`ky!DKg=rnY2^R6h(T|<
zvWmiNfz^q&-@9}^*ejKlh@Ljmy__07WtC2EwGOjktoT>AV6%|?vS|MPSx=n{%0o6g
z<s4v&u~K?rlV<ASJ?%x>mXK4&6g0yow!B=T6v6#SvYvI@y0)<PwPNP0Vs$)!Px$EB
zewugM#ljQkO^ZK!`-Q4+Km9l9%B~Z^#ahpco~T&woofB~TJGcR+y{KsOM{}fUai|(
z9QFKreYsG>+w!`<(<Kz$(>I^x6YXPGOJEW?&?p;GAgaKq7Qx7r(4^YXB&^UZ`XJb7
zk)upgkiyYfdk)M0Z)8`MIH$sJZT5vPv!^gPL`yjQ{v<Hffk|#h+x^95Or`5iC@??o
ztGvdkExSg;&fju!6HBae%5ip!DXz{FnKbM|*c`Vwr%qsd+#V(7Rkvwrs&@-p_QIoa
zJgHHPwjE{$7G3H6AzoQIr=GN?v??8U37GUPeajNT<|d`c45fq_ZyLL{h%DO1ePBDw
zv?FF!lhoJe<feZ1T)dg*Y2T_R*V1l(%TcXVSGSy``E!!4<z)WKNwUn7r(cN4N?x49
z*8X5kzUYH|)g4XD)0-wcOpTq{#H+-}!_@p&MDa|C{P``)V&54UKl*+xy6|R-0Pn^8
z&sP+_mH1mXspz??Nqt)FGCleKHjf1XdMxp+fzJM=VddsspE4SpuK0YMZNK*Nfk@+v
z%jPg<yUa<>j(fjpPpaz!Ca1pMt?w3y3UyAImQu<1SnS8j2_l|~UGKvcI^HZ<ytPV8
za`NlUvkMX!G(5KW$1pGDt~Qj|eZ=#ism}r7dykgf5wonFEgHC}G-h{c^yBjSom=a?
ztD-q)L|ZcCMOvM`q+>JL)96l0qk2Q5;*TkXyPEVPnzbXEjb}91Mm4iLOfw9hW^OS}
zy`n`q&)Vy$>aoMNuRho@pR{F}q@aEwA@fH<{ExO=35VQ>#Qjo=tY_ZUNn{!HWvUb=
z**7O?3n}~iDL;&gXy9GY(HiG6Ei&wo%KyNi_2Ji^iET|aU&Nbc;JQOqZK8(8lcO1`
znF;}38Xb#7@{|%74)<27_IIuoIUpXhw2<Ma(&nkB^#y*qnpYmSXxgHEEN}n072Ce?
z?EI#_nPtVLYm={rO;(TOoO*%N{r05w?~D(1G3OjhdlJ&dY+ew&b4P4sLA!K;=mCX&
znw&hBb}p=DShRbmu+y}~($f-|)1)3*ON2j=D}V51Q;~l89>eW>CPpf*t5S}uShIG;
zC%f&+BE1KBzACQ%DWo4Ld_1GNh%YTsS;Vr~*w}behPd;Z9|4*C-Rvwe8eS>|EIYD%
zw>KESEb7fR;;f#x(Jl9K*25=8Z6?3w>HeD4o!fHp*Z+6odHaJsj)le?e7p2+xZ<1j
z$~VsMSylWmsMDJJ#*&ZMb02>@QfkvsYjga6_~zC>$}gALWbJB{{?Rx!rb+Y04*eff
z=F6<G{4u4uYK3vd3eiWWx^K<+y5;ohYKCdMLwed;INyD|#jL6nkioL8HCdv4@2s}{
zQp?p7n*DF?Tq$$r|0X8xPyX6|O2>jqMM~ECb0p1Vs!=O(N)StF^wrwH#ulB}<v7LJ
z-^j(&q|5US+jO<7UuG+pZRnhlay;+U%dD;DZ3W4lAB>N+sy+C&b@7|5>t#>vn7ORo
zJWp;~^Izs)p3(Nwy_3piC+8dK2V4*rW$q|DvAuS8{@fL9owHANS5N&vReGmt#m&jl
zw-(1XU948_uu`0)aNGRQPV=ND^#{`=&l;>v?l^Hr@1xP4!_xkOA7@0~Xw?)bczn0@
zX)ROsaev9MfXKFzaHhop=j=0h%xxCWif2xCH!?ePLCEr~shHgy*N+d?pY#Q7yz=K%
z?Y~tUlYD-#6>RCtm5gr6>nre>ti9-H_$|4rMY~<R^`iWie2sXl>$k;l@yxCM6`C^h
zr%XS!Am+VF4EGM^3Y(>#vrZ(=t(kU<;ia?S#CiGB6|0piPO@%4nJdw(YH_Z9)~V{K
zg2r9T8?8>C-F@14`yNyAmD)+nvP@|_DNJW$G8EoDOtSFb`)f+J#Jk*xT?b`OPrtqM
zf3MW(k98}Q{aYG7?UCqN>mL4?!8`f+vg#K*V%2h4O|Mvbn7LSoL~pQAJ+Be9>Cm%(
zTpRAIUozd>pQ6O-xSNM*<}a?~>y6B@a%r(a^<UMWl|;?|dT`U*6K|4>l)WndZu6Hq
z&aE6ias6_xrpGeIH5U9EEVR~~Q{M44DDmf{qKWhMr|!KqS^C!Ec*dwm#Wl0veV?bi
z=dZ%j=zCX<<oSOWJa#UcUf*^;dM~fgy_b8ZGEe{edM=YPe;UVU|D6f#$7OGP@hUMt
zZ~td*CPUL@h9GA4DMlS19yGM_NL$t1*ud-D$$!^G=P>J{V~nXDTW|d+eB3ZWS#na2
zhtbo+fqapAD*uljOLdvXB7UtpLtA;eS*qBypF1aTA6qENK1HBwi_df+?&(scQA}CW
z8H1Obm1Hf_3~dYBp7nRuRPC!95_U(aW<OeUt2OiHyuGKy-W^`YbNf*2E!FIhW!3YI
zLQidYe!bO)vs$$ChN5IUlc`>qO~aXKJ(J|!(^eFpnx-i|#V6oHq$3k2r-+7vuGH5T
zmpFxX%PAx}Ts*x=_4+-f*$+jo@0Ql8w0OAv`NM<F$K@6<SY|%qiMB7%t4b`nzau{X
zNbKG8gxBkIqs4q?KmImLgnfrpK-{Cs_6MgYC?4Ff;pvGbx3(l1PZF&Q`z86v-jbn?
zccYMk3)8-^%M3=UF9It0t_J_-c2SvdQ8?aKH`Fcuwr}aeiW0{Zv2rCfwtKDuQ+M2T
z*67-@s6xxC<zbxUu^+*0rvHwzvTu57!O9-&^C5YH%oB;34z7tu0+O4iUYh9WYvg&s
zF;MJ@lBV}0#R*gO4W}qfX0PoGaGO(Lc(Xr<C4Cl?*OHxSnb~R+?=YM*oWe4<gOh#Y
zjGe4sC-if_w1_U_mu0`rAndX60!ypVsSEks4}Y#;V)O`?az5F)>={GBI!jjZd7pk>
z$Z>h+sg%MzRZ(EFzHX<MQ>@dHfF!;sbLk|Btp^u)$gkY8+Eeu_*Ba-&c`Fk(@=bIS
zXkK%mWsNWQ%}WBBI}|fR)~*W;m52?LJ@!8^G`dn+X-9jb_KY2}J+H5Ae_D4sFET0f
z>Dz?z#AlO38;%}dD0!pn_>}&Cejg$aEZ~(XSgve#MIw2+>YNV;rA>8&B<Ids*}}4H
zqg&K5iSSb|76}}6o3&^`kz<>m>h)<87bvBgIpiyOPhYWc-pQPeEc@q%J)cvYSF&>H
znNGGPTiGYSS#-FQbBn|!*2?1-l32bTS>W}Yb+>@zr=8)YmnA1Uxz!p(w%>0|6t^h5
znIgX9LgR6<DX$xjZ!5a&%Y5Y$-`%D{U$=Wz&t)ZByHl?OKhX2L@x5B*b!^+J+gro8
zJZPHT@#yg*OS{UaQ?lE4KJ7NITlCc7D#x$+%r8zH6tiQh{J-LC&bobRN9TQg@pAcb
zvy~I4#O!(%JY|b^%okUI>+1^CH%NsTy;{5aq}8XBS5NL)R%REXzGBHMPg`d#{tb-Z
zIz!LQ={OYoYZ+64lDMt&WHbKf%N+gH_O7V3dm?-I)>f|xla6efE8x6Q&gQk#-|dF-
zM=aLzu3?zq7P3izWtvK(K%hb+i^}ATqbj*iIv03sIzE|a+Cko?OlQrUhh5PcErnBl
zbFwpsvst%X5G`Zq3O{jZZTUfsSmq7fJolZPrL$PtuW5#JYP;`}^KD>eP`+`%r+L=p
z9G*_D6X%kcG?X;jmMaFC%n}to+$yhIXu9&?4X3he8(TdT81`mL<$093-TuFzkl}iE
zho8abmTA1NOj^14iqrzsRZp$7<nIjr{*S}hewAjo)8oDc64Nag8|r3v`b_*UwA{Y4
z!eElmFHXr45lJ5_7Q3A6qqUzl%WwJmypj8*|1B@cQ|%LilvdVwth=>rqS=;a#myl-
z+Bu2*F|UNw)~;Y}dDW=RKXIYRw@lZ&RX6!!w=K%vvBp)qfYaez?8PwY&Y0(SBqR=A
z=}@+9^grpby_J{cWLk#N*;!|HT9vE{t~{~&#!R*cY-)v%@`H6Qtz=rfyT;0Men8PB
zC%#Eb8B<n<e%iBqGSe!-4n`5BPruVY#!UAs+xF6rIlzgjjCtd$kC{GJnqC&3p>B6A
zQvUlk)O0MCdb;OB<f@vd&OJ;PmsbXgZkk}bg!g*fr@0}VU&<1GE&o@yYvK|WR$G^;
zDy>~kYGz?-Hd7UyD|vpbX8XnE5Yqi>L)@VSfk#?IbR=0f-EinibzP!ZR4pc&dyy;L
z#&|k=B5PE%BUk#ihaM>f&AN3DJ-1&wES7h6srxsp?Gq!8ML*}dnDA)pqE#(RwQN1P
z6Lx)wP;tB#*YmbXvX1HIy<Wc#7m*w3S<8;-m2FI0Y}uP>yv*f7+MCi8rOU#vGw0ra
zYtpc3?ORdqn*mj-CK~o!y4fzjqn|hQC?5AbAhs&_sPU%4oi{F?kUpw;y!VKjFK=dV
zDZ2nm?-!p5_WxhMTfmcMqF0bHOKGC~WsahIr)5}5=NR5iNebVzuGV<Y!<Lt9FRJf9
zmMd@k`?<f~ztU@S=USB?iBE4{nYQNk!ATC%jbe9|zNoJ&%ng^#Y_?_l;Jrm`nR<5X
zYlfneTt+oV<aV#R?W_04<m|cr;z`+poaWnoF9g0&cafgdyZykuD{GB?g?#T>26oMP
z>?ktVi7n{cy@)9~iJe}{y}~97%}Fr$`q{g4@!iSmb5yl?rwGgLj9x9Z$$r5z|5F@{
zo1OQC7!|JaJMmT|ep!Uqa*LhoYj?iPa4&XlJn}|u$-1BSK3-C{dv2<}`(~|28S{h&
z1(pSVwvRsNtYDaEW|n&?;@89fw`#1GYJQz!z3i(_hROy;&5p^;=|ZI|ooiPFtd8P}
zp1>q-bLauH*VIP-hgX^MeTCSf3lvReCNyL*g$Lbo%DeFCs))g*t&9t%rYw;#i^>k<
zj-KK%jhW-4lgFOU^j}SlrazBXrM%D+oat!u>zC(FWAB~yg$_*f_<S>htD`E?dCacs
zPMO28b<WiVW%rlrMg)mB#eB+J(#52|!@+LL?c*sMjZL{33PKmpYV$d6#-JX0G}G(3
zj#&y<$0NxcwWY>(|0eD>kUp?vkG6Wd@~wxSGnnT0ww-8lZpmayjOj8g^}D*$(|48k
zp;@X5tg{(wUpVqtrP%yD8TRG5^1?s<jUNAh3M}8euz@vb*OmNQ>u{%*S5gb5y;g0T
z5w^f0KWRJb^<SqPS2<jgo3!Pqu3b=9^n&NjJ9!c}#x^T0{NXaYhM&tV=L~D%qIFlC
z&i}NWxV^Y^)w!T`t23PY4%@cB@OgeJSaxFXn+Z;3OD=ttJ~1i3YE_7B=9gW<wJ-Ur
zF6b5STrhWmykmKk?|iQQGaFacPLfcvX_@QCC73Jx=<$&~7LuIbtdH`#mzTIXD2A%k
z{0mHS>YJm`vEKA@yob=7;@aLhF@Bso_6haP-g{>M`=>J}C;7`USLrUSTNw3y<?qum
zcWqx)|C2f_BL4FI=ZjZPDz96q#k{^Mbdy^)=cUkNixkd2`mfCrBv+mu7V+ID?YqR<
z58UUT2yRozogwVR?#Q<N0oS?>!Uf@pM~<k(d=PI8jLa%m>|0!JyD-oGP@bDh)GE>Z
zKg$e1o5h%W#C&`%bu_g3*R@Qp2kH|A3)ZU^yJocP-YT$jYV-{ek;oL8eXND66=myG
zgw8&d%2#pkxFntGQg?h~?d|7<x3>ytsmms0*iB+8k@zmfQr_aZ!ytb~oK&IA!SdSD
z8?|@MYs(_qeO(-~n&p<IDDWvJgkI-jdE%+Wl&Tcu%@=0=)<GzBgOmHhGR2a_Z3*n^
zflSg*xFjZa+82r}C>BvoFf}{myD(T;WwNqmQ*zoQE78ezLKE!nn*T4U(qNpkfpvD0
zao=~1zVe`bf^OYS>gu9BJ0&VN6jrR=oR-C;(&uOr5tJEoEH&y{YGj+%D)ZjZWBw7>
zQofh;UgGF&+}yG~J#F*%;FNCyiQnotCacU-s>pfP_vo|Hu2kC@6GP@ZsV(^|ReepM
z>r*$6V+LnJ+az|ubB-D3gl!)&m;5%B+~LSNV@pl8N9Odv%u7=gUZq=a`&{=zqu|0r
zff-F1o)dC91heDavv;&fa2*j~y&<qyEbItVxV)m^HipD<c3r^}jm4&6^H>~IFUvWZ
ziqxr#xPP6z%h6?SsLR_hsZVb8s})(lJ}mY>>9*|v=MT4-LZ_+ChcX2cr=E2DU;pV_
zZP*EqHxE5>BeZ5-R&u=|YN#&W@zG7#m@#^Z)Xbz-k?k?u8>O!sr|(@@_=H_hsW|BC
zGC}KRnfQ#NR?8XRzfBQpkd9%FyCWEM-*MVxv1!lJgJhX&uRWLHESM=YA=lI}A*{VL
z)Z8n2yQ+|z5L=o9r<zh-Mpy~6uEY^Rl^cpzn@sPd%$}R!`OeX5l}1Hckn!=3;)63;
zf)rQ<C$I<#utphhZO@o<;-=Jzm8`P@xZgC)NjBh0Ucee1!2emW@~DJcNlE^t!l@55
zJ&rTZJJ#WH;HT9r7JI>+l`@6F?{B7lI9b^k9PmAJKFiZ88Abokom#O6QVpL-eP3z6
ze)@j{shQR3YJGb)E5{_rytgo3U{rJbxj@6Gz~W_sXP0Z}3-|LE7F}DK!9F>oX$edB
zL#|7gG;bPeEmO9AB_Wq}GQ`3(;QyDZi%f#+C-pVnl(B7=xX4_RVlZhvV`!-9lHjcl
zEGI(QPE0ycH0$cMoUJKg<<I4uZ;E;(D%D@gkS>^FFk_0zjwzCp>!m+9?l7FQ<^%r{
z1=hfXxl0zXW}cGY7s4HMfUEC@_v`@f4Joy*9<5Q8`F0#p`pr2TD=jaX_-+dge&`w(
zTv(8#7=3PPr5{J@(us@KUKO~>IPHCNO3aSJj*Sa%rOPsM3NlQ{@a+uCX=zn5S}`%R
zaKb{9q$$lwSN;bz{FD-UAo%29!p1PU0QC;`YYLNgt~T$om%bR!`!T_6migHevkk6x
zIk9JKjA-B7(SG)p->w&{Hcn&+3gG@1z&*==;nGX~H3F<_0{GV);M-a-_p1S$@<qO6
z2F0hl=Jsi@o)hd5o!Wg;QRXXGrCqVH(T&*YLB{Eq;%1dh%UdDwIc3B8uRYz2eX1h8
zcW<QJjp)7fqW9_v&QL`G#$8G~UDFO7-pG7mBm2Y!xe>yd$EuySvC61gUk{w#a7f_c
z^7M;G7lx&@sf$l&4V!UYo%Q}k&QnW0AFxckwQ$2!;VsiIS6XPLZ7c8Ny%==MHGkbK
znaiea4(_4qFSS)yI)q1W?R@w@)L^56B4du=k%nvAC086<%Kv@oFZS-POzznUb26=^
z<SzGLQ(wOC0^b9x<(Dnl-W=faT)_JDRamAKSN3b~H3x*EA9%`!7pa;mHHjE3P-9(s
zIYjab*K`gi84ah@0NE>_CO7Wd*<!UJt<u$yeetTSh@{F0YgRcAH~o2wBt@B}*qlAM
zUkXf<(%QthdzPW0Ui$RHErMl71e6}TH&wb`OO4?ykrTSSsC`wEn}(~&bEQK+WQs2Z
zRsRxbJ1yXSJYm7flI?C@w_opFq@u7mGU07`rjJ?!GsDtTYU;;N?mNL*a_na9htvB$
zMz^21Ip^d`&PQHsS*Q34O1sw<PW|@GIH~=A;-gdY*}nxZcJBL}$e+A{^Tq4^vjf;1
zx3BN?o;S^@b!l44vcs&xlPtbHWLqb2s4Ss#T>-ml4Yz7e<Hsv|dN-!%XW1`Uz2Re}
z<foOX3%Az^M)xi?ILx(L#n5J&oobbHPu1Gzi#~{N>S)~Zq5O#W>lVYKtQy(<TPJN=
zqLf*t8N|1dq3x5@UX6?^vn6hq*X|Y+D)^c&`EUZ?O@TL>)4G?`@if)F*dEOKwazU&
zts_t~;f#iksQS$^SC_2NLrF_GT^f!)3f<!-kg!$9_f;ZaVR=e``s@b>T5g-J6<|GU
zIp?J1oT!GmK@(V#8M0qmag`qC`{%%w9AGZ@@jzvl!nOtfSs!KbufM|=ov{4rEw1DR
z%l@=4JF=5CY60uv&K<R1jD;DpcD#<V4rH@=$aX&Ekao(>vu7I13%oD*oR!JoW^C!2
zv%5)qg=Oy-srlXI2Fq4$U%rCrR6x7pE`I0!iH&Y+w@#Q6HN9_^>+Frwc@FNbd8sis
zOZ||xP(sd(XP0YT1%=ibsBOrW&9OMgsV?+bdG(!%o8<%LdDJ`|r8*ef%>J1sFno(Q
zTQzHANcM$89Y=3X+4qG1u@zTA^4#PFoa<w_*CfoneDhMR_qvkJtWUK#qXKd#87x1x
zWxurH`ePbNYumKDJlR~`)iNIWwH{A)VwYy%R&joGP)XvD@)@U4-$UVN|2wI!af;5p
zs$;?s!+T9>kA%)EDZQL)p~cr^4E-m6?#;>V70T}A<?i*r62QQ<VAAGb&R42(r>`rV
zzAo$CCpGiJcJ-qhlW$y0y0Lnf;H*=-=k7XM`>euSdy(CA1I3GM+g40}b6K4^{P48T
zho2uePz{>M>R9c*da>Q-3i-<ule(90owhY1dXh?HZshDqf$qZW6??Z|=1SNt_{fXv
zuqD@hs~xi+aIe|Gy7&Qi)oH$c6ZSv7#rG{?IkUqYU0Z=Q8#psck1}iC-9BU6&*$5I
zKl5AYc0T&RWWTMBXL8P-TXXNF?pduX_s-_9Ghev(>F?DqviINXp1r7ZNJ45hSGCG*
zn-wqr{8w6F{a~JTrmoqI<<=5C%J#MkTOKLeJAHO}Z?m#Ia<};NB0Y81U9Y9~X}eXQ
zI=@2qk#u8*R{+~?ro<PGyZ20dAozH7_m%F;;*Wf+cdH9#-;7qM^+<5}D&T)reg)^g
z^vR`9wNFU#Ug|r;b@u0hB@J^@3KD}7eBN-KOiti0-=*_rl|mBxsjF&b@{VkquW&Uz
z(%6}i-}h-t8@u$IP%Gz4sn30{zFT{>Q0|)6muKgb1fLxgxF2?(Tk!s?-&ezW4{dwE
zRy-}m$yn{%kr2H%9!&QvRw>Ol^*NkTdv5CCBgJdxn`jBR^;EH}ULZO1fuD@^(#M*M
z<{9NovAwx=Wz356Og#Su)l)ZjX$zS}x2NPdOj6V^I(lpa`%DMz#gjE>nx9cvWc$Qh
zr_yFu^LnmHYhw!nL~rvg%x?P=s>c}SqotEu{AAXp+LyPuE*`$KBy0I%0md7zI{q|X
zE)90r<HKexU2<T~w)mFgAujix$@-sO<1}^my(@F>hw?gI+VlQ(A^X?A@3}v|FFG!e
zC-8wMk)6%|K4boevvc0dSiD%>ZDiBg^-*<&^H*n%e`ifoGj;96k1Ut%=Gyflb)T!S
z!v+<twO2K@*B3sTZX_ue8^f_TW{!s7sZ%$mN*r6am_d^<MMr1rfx|VLb*Eo{%-i8{
z_plAGt<n}LAtB$3n`BE58BLO`y|^aw|18z}tN8=D#h(di?u%I?Al)y>s`RY#_{_~Y
zAtf&C-Fes6WakHIg!XVLuuiDBIxXXSY5&#YcUNomlxo%s*6{zR{`Wkw_E58(O7)^P
z4-eNaKKtyyhpxW3CXu~ih12!$km?Ed<!1I(*Pp$fme%K5Z_Orc{XX@E;|yCpZoyv}
z)90?z=-+EAx|HFP`h<6OZO@D~Z$7O}k=GHp`$<l<U$1L}tC9rcuGf1vn{tV*Nq#2v
z@6MJlV(K!%cD^UytjpcHKKJj{`)?waF1dJ4)lo}OrSPqJeXxT$+v|5HH%KsMtgc@<
zamKy3cY|IC6wcO2J<#~>?$vMg_ZimzXL|phhf$!M?LUu-g~Njb*6RecOgs!8I<+vr
zktj($9_rM+J!a=arwQCMb(yW&o&+sRKH10YGs)4@?Wy}z!_zzkmc`Ew%r-0(I&*XK
z^Ye{*+<dB0S6&1zVcz}c%@Sj^<qPe&oqS(yxx%nQ)0OS%q13MM4e`rGY%GuFoM23z
zs3mFPc#@?_n1v@Gpfn+tmGK>u!isW*`wW#!CnR}n5e-}G>b&Z1=Jk({7dmA!EuF36
zc<{8#WQn!W1<WfCGe_{s+D;IazPOxyOVHV>1nH)A;nU?hc9}U|$@RLe7P{s8f&0wP
z{98>o#2$RaJjo+s-yWlTDqM^BCoFK;#q|30X8sL++pE8xySuYo&FlZZb!$6a?(*o&
zsY$5%F4Vb5eMa2B-@jbniqFYluRZ$z-G-&SYq?ZbX|LgUq_UJXbCH(g)D@F-)K99-
z$yqjwW73)hypvC5HUyn&&~jN}wNvv*qmN9-5yn|n8&<Af7xmFlQgL&k){$M0nwU45
zNn~^F=t|1z+|ne&pr<Sz@p_%YyHz@y7yrpJYGBcsoYe5RMSI26uv<4$m?v&~kg!0u
zf6}Ry3nmG5P2qQMDOe$uEu+QwDKBN60_Qe~cWz6*#prHZ#3wvOA*@QebxI(c#gqm2
zZYZRh+{nvro#Mf%rDyj_Xo5oUGnES^mcHz}c)XfyR5`qY-uiedgh^MeSolb8R|doX
zUZGW2uQKM${CX{#eTT=w$Wq-G3f}rzN0p-MpJp&j+Qzl==Hv*W%%#%;j9x~~+_cp*
zdil98-_)lYZJBUhdSTS6l}qJn0yIyh?K-uby}Ee%#f4T)TI$TZg|v0nvvGC0Z|+LU
zUeI{k>XXN|S6N0^7k|(?p?_wDkHW?#x9p|UJhx{|i|MnL<ItL3w>gKQ?c~&^o(mDp
zYl9Z$NS}IoJH{<H?M_zT%oX>F+`P<Zu&+2euQ4mrJ)?1_vPb8uK0OVa#oTGHUoKjx
zmyz)4m`KL{KOKjp?ANL~y$||*=-e;6C-<~lnT_tO$jF^+aiMVa<ZYjJXc?(&sa{^7
z`enzvxh^`&NB#@=J~^`NAa~J5*QQys4BZ4cCtd!yQ~7_yaos(i3YiodO}aKVKMeY`
z@r}$nu1AbtxE6VFR!NtX@?Cu7<YT0%wkl81_{gdEhg^?j#<D(KV!q?J|JupB7MY38
zt7uv_IZ4=_*L&4Z`>j*#s$R9<zI@@9kZNnMtMB6S2KVl+^-qFMtDpL@@WS!U73Vl^
zB`u1UQtD7lieFp5;zzXVir!9(pxsxScPQ0w*6NTxb)n7Z#zK*l84b6uESqSwz%@@;
zXTlacp|z7|q(m7+p7hQ-<T!K6lGrJd*IO$l&EOM!>RcSnwL9j9?x~tLn(mse(oMoL
zGK#9&1<Hr_E!)7lu_u+=i{r-sz&nRzRb>kqxNn^K(3Uh${g<jf3!?)QQ}I&Ut4(H0
zH@fEUOPVg~)t4;h6`H=xc=;-g2Pt)0yx9TCd#~|0q;0ND&dfM$Y{l1*YVo-@u<h}t
z`MgJLPjZIFPI@M@H1Sx5OJS6f_)m{T6S#b)2}xgnF?qF@S4!j387HouOPkE=p=O!q
z;II8JO-Sg(id<_YXXz6U790KPawz!hq3fEoAo`f!bY3Yg{rArnn3x{Z)ST0ktdk-v
zG<TAM?iEk<T>*<`Ypg3Q^pbg!R^n?|rju@UO6J*~lRLMFyi8!T_Bz;NFi-s!Yf_j`
zgV2tIy;ob7$;JrrY3hhCGvrP25&aavqx^rf(%pp|F1vP56rPwq-B)!=q{W-yCkxp3
zdi;H6-nmHQhs@O#e_iwar+=E6*5zdPb*6splT&A^SSHzj+mIE$JB!2X@%1*&Gf$QX
zc}7RK2r21JNZ>ZkJS=h{jBWn5gS_#Y&C=U8G_rK9<durNwz<nlC@StkWN>fv))iMB
zb8@{`-+3j?vt9H|Qkt;Qp5(M$YzNL5xGgO-esy7Y*Q3c&jXI9%oZXQI>EW6NX<UBJ
zmk!!Z-@th0s2iKC=v1Z!(=Mu*1$=zMB9N*ccH-8gl%y#PpSyxIZ^^WZR<D^d|5=6f
z1*O)7)3-+2Hf>vKc%gOTgq&N-(w+wHPHQJ_Qte>Mv2OeyCUAPv--Z1Re?H%4Tywlo
z_=cvWM?gqxfn%=pjYCxxU$#G=mc$&hfrE3|g|@JSbv&sGjN-Q*aLLt{M~3#k(mgrb
z+h}I^rl(BWj-4mXCGC#o=bV}w#VvGf*?|>22D6?;{gvKx;@nzu?puN8yCVgZbQi8)
z5X@)AtF@}pVU6eV&1cm99ogl0beU+(1g4k+Un2Mlm12yZ+<oxsws6L$B`y<~3x11q
zs#~cp^-7V*b<Srw|MH>5;w2pCG<Kv1zHQ9!Uj4A}v%%9D6Xv})^mOa`cxQ3+!!^hG
z*xzt^++myj)n|Kd%7*3{7Mjb#b}Vc(bevtl`s38BX+Im*?l`Jl-sPR<`TyEV`OIs_
zR;+TIl2gzuzonC7KiA=eUzX8_SYFpM=ZI|i_ax5!VQf|7XQ?c^Dh;!mvpp*oSx<NI
z)^VQW(BApuma>*&C-d(+&5Dy}9ta8K+O=@rcb>~#x*_ew@i(0~dna9U-kow$#d+%S
zAa!Y{o6C9&*CflvUv{&c#b&zrrR3(RbKgkcUZe0<Lglxb_4X#-6yuzm+e&(tF?=aq
zrdQO&-?-~Eqv6{ZD>!uym)tJcrdu<8=Mot$_L#)G@7u~1l3xaCUTMpVXWYuNZicG%
z)n{!|Z(pb|gzxaUBX+g@_}Pm^&H}7&k1n`(bbHGmosuApFySDNXB#h?lstA5xxf<Z
zI_v+$Z|@h{ev{6t+kDxr>GZOmkS!~D{TY3&LgU$tmTHB1`hQvSytP$%h3C~jUMY^J
zW=;sYCNj~cjX!JS)_fLKO@`Y~%Np<H3aGwx&egBcTU=seJxhB_`s9;0r)aKya{b-2
zX^qUnKlFUEl^Wk~vz$^UaXM)ICp$fbuY8UE3%orZ30c=H;Vm~%b!BTQh&2>lrBD{H
zJ2xtwYg+iK+55FPw%@wCw$5%%z247j)s@%G-%7RREq&(m<m<Z`J3`|gXm|A_<j2Pz
zKC7K~b9MBThzYER!xAS5&du^ZevTpJn}65Z85u2`9hS&?atXaz_44f@>lpVu?!B45
znGtK<FY`SAt<C(u-myw7YS!70fBG+;m5VN%Qzej_v35ewue9iM3q96uU4PeJC0Bd<
zUkQQvyExdin2*giXM1E6&af(?OHU|(Gx~zod#$B2ET;7`2&DX!ViXkUow4*w){a$O
z7A%$0LNW)tR$KJ{Tx$MNqG{FK`3iF`9A0+drT)~#j?yVIuZ6XlChy!XF=N(huD{0n
z=38^k3+`JGt-Sc~hQ-;;njW*11G_!7y0`+xmp*6_YwEgl^3c9O!TT$BefDgabaAId
zv5@m6;l4@@C&no<!G~XU%DtJW{Pw4zE!TWq!A*UR^S5rcvJqKnyGEC<(E9GA-QOb}
zY;SciN(lEA%;s>Cj6SiF`;gwU|4Ey-t`uebHMM2Ok@$-ozH40g=O}Rga<QDOV=>!i
zf&eG?CI#Lq&N%`LGbK&<s*dspDXu@|xMK3rxe;4jfAe}-FSvf$H9n+slXO@75yywC
z7D_0t*!ijJ-s^)3mEGqqS{^spx+h_|O2u*$6|P$utkE}EG#_kx^}^&h*XmU#_MNbr
ze1^f<?qtv8O!gZW49nlxJv^Z^J%KrCwWN8~wlhMy_Dyn^7nwD!Xk8g7RiG@s&133W
z5BWTAxy6l>c4g=}YAI})vFYRvbH}KDp^T+lH*)m8(d~Pp+q;Hi)6WzAEel$;Pp(U_
zn6gBBQpb*t(>CtT0&ztrt2)=eF76Ry5r43wbHaaTuEt$lU5OJ{7fe37aoPV<2dCe*
zRe0(6#-r<@=82CSO%Eqdx+Ey770GwibLJFHjyI1cDOyQ!zSyu>V`08#hggx?RD;uh
zwY*bwIHrAeI&R_d<>o$5!&Aq1$hj(ZIImohQoPNogfp7Kqwl2i@fl2-HyAz|bT!Wr
zUhA;3LAXov^^&cgVkg%eN%<(6c+zA0<5^7$e8M~Su(G%wU$Ae<jy)F5t1KTcT(wIq
zJM!#>&wFNUkTrRzbvb*>mDh7Ccb+}A)Az<<-<!&({#H)EUA=v3=H6+S_b!>k!1}?q
z$?@D0O}pJ1`!bgZF$6SB2w0<dTS+goRn11@oq^D$&+1Oamba$NvHAaS($qxOn;G`6
zUasjWGPMoj-KU~^IiT0BsOL=8;!Nf<i!;v6yKHuClJ1#Kr)P)e=tk_iVb!(KV$*#l
zX|}&>dr~+i`gE<8<eK<L)JEEI`$8GswR28g6y&${+@-XBmDEAbB>hDa=L>rTSTC;9
z`C~krVP4Pcxl$=IQhN_RT5`HCa>9}$d~#>EnRNBNtT0%xS!jykE}6q#eg%h?GJ3mp
z2WpvYjEnN#Y9oGRqT}KdJZi5GH*=ZKdSP|a(fPR{*S&<+E-veLFL|YtR_qj!f5@=>
zlSh{=i~E)nlXZ9S^xnMmXys&Hjn*HMT_?_nGR>OG`BsHXwZrbL&##3Sd_)w2na*1L
z|15s=fiS0(>uF<GErX*xr>^kd3JHC4C1lFc_eU-+3*K^-`54d3qca>rZM!CJTD(Q-
z6>qXkN0N(0Wa^v;7sUB0x}UIY-_>FFM8xS+;iN?q{0v0St~w#{cE)61M@NV3uoDXI
zvqePPbhy;57N4BN;Ag>;sNI!lyzOSk@#MqjlYUQ5Yc?wuJ+|t=I<JH0+gVOT8R*_{
zns@rc#R^j{brojK1AOaOOuBB#<zjWl>++eOnq7x9dM1c0@ms`iZlKq^W6AFy{Oq9$
z3mF`zyt&Y2>)E|`so}whRhMjL`f$$Rxp{J-=i&uUlX(62F!B`!Movu-ZkQz);k&cJ
z$x@4DPUEUT?f*yBwtewkEVJSCM!p5L8<t0Uy>9SYED@yrYisyyIon0{%)Q}nW-#0o
zJS{ls=+YUTt6gpH-;A6ayXmh^N9gCEKwqvIC%qpC?7qMx|Kg#>*26u%f!yxBZo4{G
z{^%5SICEzA)u}T#iFPHN5f#)knz8%sr`xL|uUK<T+;P_`x%SFRtFx0|F!`qX+8<U(
z{KR>&IxH}D%e|7lA#Y>OUG;r%=gwV`bD^xqq_nhdd*7A3B>tp?oAX7_^}l92duOmP
zFg%qu`o4tsf=0k2!+9?}8fFW-f9N?Ixb~!QZ{W#^-WP7H`IH>@Dck*h@x8w%-8W4&
z+r7f@&JE#Nl3fce`8XG?ZF+f?`G4vXxk-Jz4_KZZy8f?uZH}>=rUo<XS1E7D-QJcQ
zt!o!1OA6@i@F>w{4xZH%Af)$q#t~MYi!qljtb1_bdZlbhaspp&0;}D@I@v(xX*)j6
z@^!ndb<1+2d-u(SErHs$$E0K*{yP=fS9p5wO{VQHE-p>EzicO;f-Kj9#T!^SqL*Er
z^w7aSh~wVU!eeT-_kL6@yS?$yW~E2zNorD$ZmO;3((ygBX7N1+=F4Vp7Ar9Kb<c7(
zU23bNDXk*B@~dQr3U?0+!y=|o4TEowm<_xBY@El@()+`(^J|1l>ue)=M+Ko73|lps
zcUWxtdGiQ=VMy}Q5YM$X9&;_2Eo~+pFn0Ch%>4iIikq29jpV&c(zj2^D#pB-7JDWo
z;%bV($|)WTmq}>4h5K3>y)?M}DD<|`yg1=Q=O!o}7P4A=;`ouQ)*yveZO=177e%5j
zb0tZ6r%z2;EMaLc5~f&UxNF9~3kh3g&KW-2>;6%CTZPDVn_cb~1-5;?oRm{1`S!2y
zHeHU3G7_PBTrqxfA1;Wj`g!;DitFZcXR*JT<#J}_+@Ma)hWN#|8VqjhRi0bUxTl$E
z5A*CDvo@aW;E3^zeRRY5PFJsO1{>dG{$|;<lgo_?cTA13$Phdl>vXWYdX?d!lO~$Y
zbHBV0o*CI-wR8Gk{fTlbCKn6O(e)B$GZ5+1k&S)0AT^$K#`WF*C7#b;eC*0Wp>H{+
z*G`;^SRD3#!mYmBbH$?+Rjx(NnxeLJ-6iSXHBSYj7f*S$#q_pCjfmGz-fI`nx7%z=
z*t)CVOT$<4^}iIaB^rHt+z$@C<}m4&XA7FmJImTUqt$RD$83cij2^1-j0#bKVrg3i
zPu<?-ebh$qXh{Fk&M7ZA9cMq;z9>Z@@6v>sJ@-0t?#!`0ZFctt^Rdw0Da=W>XW|rB
z+z1TW*Q4a4Ze+4%&k5n@&5vHYTvNF`<w?n9>5VauN_o;YO=pN+>~T})o}JkX+w(7f
z+O2((d+pqcc?Xs49M6A~>8L9SueZ6NkfSR;O?S~z?ps0Ws*Ag8q`H`M^cZU#*#1xH
zIPlxLT3MjlY*o#n30E&@3cc8M|C8t4&lgq%Jurw&sF|fu()^&#_C@rpjGk*YLcBTC
zcXJ9$Z{+=FGk2#Vhozuca;9|dOO?9JT`u=)w({Lx^iE=)L6e27)V4%@-Cb`&;@`-<
zI=VJ@Q;gab;YT9+Nt;r#>^L=@os`vX@5^b^DUZ?lp0iK>ww!V7lmwP3habG!^81$Y
zxjZF#P38O<TV@0pM)yVw9MN<(w?4KzY=hqP3oCa$HtN*S>N+6Bu<FQ`2Wm|d-p=$%
zc0RZ4=voDrWvhawX!*~*Gi7bcl+qmImA5W>PdI<H_|{|BggZ~JD|bbFHJ@`sIp-Z$
z@VV|_v)sr3FL#Gzt0z4QyLjT<+Z+Q9=Iz^mC^3Fdb#eZ<Xoc(2d7Sqe7iB9;28tg#
zbx7~k!Pw{0TvIE}EV|Od{mq^%^Z)fk*)v=6#mD213a%%{=}zK$?h)h>7`c1<iM3g~
z&F4Q_$G9Qgn&Zx2Q$4?^12cCVU_GkB_H`|z+611~Rcwj}7cWw{{`1@Q%olBLo<@xs
zCHCLHM4j}`-7%dda>CUN_wJK7cbX)u+b8w<<OH@7w(TZQRK2FNRY<M0-MC5p;<La@
zAGay4@mk3I@!7*kIdeoFFASI4E99o8c*^6I_@|P^SAtq^O;K9<xz8|IPT1Px;KIWj
z6#uR<n|AP}p4}FS{n25DODFv=;Iw(?bK`-J*Akb#Z(lR)Ix>yx*OBN1R<+_~n;(2$
zRD4;-ciHkgSN2p~EVi0h&vWnO-V@7mZXN!0`dv=d%N!P|*atEt+q2I1{ZL%O$+KmD
z(3bPuTQuI?DScbGujgsUTc7+lubLHm^&_tw-nU#$eCgHRDbL@2n;duF&HtX?`Lu8Q
zo~nKMF2Z~4zueiV;M*767W_P}bJG6$q<5RXmCiZ5Ur>C0ua8aI@dH~wPUR4dIKH9v
z$IlDf6*(8NbN8>9vsU0jv)_xNa@nmGog8g1IL#cd`&U^^p5uL4I(il73JHV6Ir_6o
z3)bl-N|&CVI`7w{uUsEG`1VSqORY2L{q@14JnGv2Rk}+OdDaRTirRAdKmQ^AM4!{X
zXo*7q`X#T7j}#WYG*sBU;xDKCYsTKqv*Yc|iW(*uBrI@fVrCby$tX}f)XXW&7qGw}
zX?F8AkzKnUznZ3Pp2C*3Wpd$Thww7CJCX(2iw;fU3g1@}+QvOoHzv>TVN&ramwC)P
z{_t1{Xk46c$<LI!vgG9fU7tub*#{*WK}$^QCwW?ZetwP7*q%?PIN;6YO=;)nslG1N
zx!ss>y^U9E+LfqX#jmS!lf&OP1d0BZ(-E!M5Z~Hl$@uI@>M0ggE+(6476C;KEE6VA
zNGuKb>~N0RCR^=lR(Fevj;Kn&0*9taE1B=D>b+a$AiibG^lSg+X8hjfbYz>l*SsA|
zMei@26}i>w#;M{@ou_Z|MA&_({K9zQ_J$_Ey`tYAGym6GVjZw2=sZ{S)lKCauQ7aM
zJ|Z5n$>3mqE32?gh=%;bx@*jnj+6^XYR)}!$+P^1(xgV_*dq@bdHEO;n|L;r1WL}C
zuyBU2)IF7{A<{QYrus><xF&@t|J%~q>37qFf!(k0pkshOlWKCW!6k{u0WtxCAwjZY
zUoUs2GA_O0uupT!!e}PHFViM7Pc3>_o#M)=IyH!k?P-E{D(8%Z*ruINBV(_Hs|A-7
zUYfz=88OX4fSvWCTEpeU?{`bi&b`j18GRysRuGqlVWWScuB6x`_R@#9^Sb~0O^H?)
zwq3P!N!q3<@s5WBXRVApJ9D`}$@`6tT`Lwgd2yvL(iE4i6mzwhATsOFsY&y_JGV~q
z*6=+N=(W9RiO{-jk7fyNFmTNXTCIK4Nx03Sb;k_9ZeD|oP}jiMi?;?Iou%0=cj!gp
zR+ri@6C)lUn!0wUuxwSpE`P;ug0b%&eO(g!UiQno>f9+8+Y)}teG%VZ))c<}fVrk3
zW6D%kg98UA?p!vZe8FAMV|D+1TjcB94{n>5^Q6coy7sl!(y(no6B^aiMRnWfp1St+
zwAWnq-90g<MMF#Go@T#kBUp4Kb=C9oSzD93ZMm*q@{9cB`{&9L4}OREOYFL~43~d#
zsx-RX_&<wtp_9<Is)@Ss&3{XEw_Qp0)=rr)DX}dk$0=lU{%eu94l3tt89G(0TFrxW
z>~1f*ZDKb2k?;9q8J8b8g?WpHIv%Q8u|uM$XyJ~6iA9N$Q#ZS=o8Fl#_iXyyX1QlO
z7hiWjm9%tgX8}{rq%*Or{0pz2QuAlzdJ)p`bJ>&Zy*H<Yh{wDsyRdj7pTnPg0n3yy
zq2LKg+_7^f?Gu?-G41(=(!~v4?<=m&d=*oAKiJ~p8sVqM1Acy;D}T{ZNRvTi19OVV
znxG@!ssuM_?fmpr*YR`yw$D|sR8un#Ur@^mnw<VLMg4+MSAX9loy7u28+ne*dNjl7
z;PM^EOr4tqBWAQtU0L)0sJ6+>-W(25+sz$obTm^sBLspE6>Z%RbNln@xED@`_uL4G
zZ?No;-t=X<G+zo=aKPC^TFo=h<*0Rf?72Me*zyg{4sMK1UXzZ^NI7(?Bh_4d&3Q5Q
z0*zlAR7_JR_J~AX?6dOZ@7?yWr({~kv9~GeRtq;y4mz<}Yi{K$(-(<eS1+Vb`)9-2
z^Gw>Kep%20PTnUI)PF{`w6K|d?O7|g)q%fo!erU71fj`WC#a}b2lB^XVb0A+mKL|z
z(y4nRrA+ab1Jg@0R$*C}=CW+Aj({b?(knGwoi*?1wx%eHzs+#JJ89{JvdlAFn^elf
zV!C%6Q5M^fu#PA6l{e?|%(Dj-I6eQ*)4umi%~j1%l5LLH&u5%d54G}oJTzSG?ZC9;
zgN>CHd&X%)Wyv_z)h9*HsQaC|yGvrJ^aMpO#km}EA5YBGe4?wu89gJ}GC6f#@WpJA
z#X_Q*9!q7;%~)<3mmTVyoGSHgqf5lTC{9o1DKh_D(<f<ePSK4p^xV`aH09rh<34{A
zFIU~TvP?XYRo%n!0JE94x>M;i7G;l+#e83*gtuJdG>({>tNSiWwmQ+>K5SY}T&eEw
zPm2z`D-y3bczU&!$mxB@q&6^KSQ8)cQAvDB!iM7>3le=ACCWXLrGD+vO!Rc%UEpJJ
zRqL2*aqprlc}G62*zV&MEOf)Ac#`tw^D6At+JFD|q~tH#H2Yw2YNqSs)Zn%-@2vhH
zJLl~5OU#PjA7o|)HQ&k#J#BJTXj9O7)f1ORHic_$QOudr6I*W7=~<RzwBzdwSNoca
z{d1JQ?>P1Kc+vFi%jbCta&u?v-44i%ooB(iyXbRg_Pb+iMHU+6HE~}I*v7D9pD63}
z!rh!b4J;~4_0~Bb-O_tYseR$~JJ*Y(4zJ>=bzXXb@kcS!yJye1b~zV@nTDAr_A*rj
zd74EORb37|S2RaxPN#)hfZxOXWgVY(&%UBo;r?jVP602w)-8Qi-aa29-$hId==Ab;
zsJ`2xzA3Ov^XL&Pzk3S)!V^#ZnNd3-a6Ze42V56R(k<VuboKdf-+6iVvyz<_w(q9=
zHDq`eRmt#I!ohZnjbL!+ss>gbK}n&CD@)uAqSE6wGxsk_Tc)?QDK?}lcy^GeWwl33
zq2P6nzpo6%ZoBFJ+tbUvQ_A^fLa6SiUwsunik$b?tZc7+ndo|lLA8FZz|?OxE5C-F
zaeKGNYXYaVN0y%1M8UR#*3&X2wOrnh!i`hcFWL9OVNQoYNNt<y{9kXD-QX~^{HNM?
z<L1|r8O6))TX0&&+I3Xhh%9IeS#@o-sqg$h$x>#G6YbvQ1Yf?<nJe?W-R|T~mEzv)
z;NsuRcc1Cy6`!u1p!0pPIQzu!4mE}oO771V74nT{6g7^0J#W&U8>y*VIL`f#J?5jB
zA9V6*N7&3oqK-DMJ4`0_<s~W0{(a&q`9ADXi_-n&FZj&<sVYAE<D-$C^wO)N^U|sR
z-B-{1c2%%_Q)^OJvT~@O$y2B3ry2A8^)WYhf6hqZqu*vFmYm(Gqx4Qdz3#a0J=JSw
z)_vbxPW_0OGuzWfORitm`6$<}j<(*bg>&6RA4x^eW?eEdQ1;N3bB~{d@;dxroZuAs
zkzZrG_4~NAlhc02&41c5lP6j1593*zka;ry&O{4n^$5jZ$zC|Kk#}l#h`Ytk{bgrb
z)LA4QrXH#+J!z2Amgc?d#H!Fe9b4DfwQS+Ls3mda8h2{)f4MKqg1L+?)GAHz>=kLB
zB)a{7L|yr;>mkQW4^MNSdGCw&LQ}Tb@MM)^J)0g(u9LQvj`Th?FD>c6NSZRw>Zdg+
z*IzQmThDqszsYm<mAm)Me&5#jSY>(aMRwAMxYy?&G%d7ey6CxX>gGM)H-EW5b?<D3
zU-SRpyuJ5lv+f3F$sVSMosycYcf?#2v{Ypt<%(-_da>=BbXR3gh>DDs>CtU3czb4q
zcb+-1b;kAEDKde3!XKUBSb9Lt`^xk>PO;FnbEbO;o7$dgIC-b>qRg3}31=mzI3~&l
zU*O4q>$C1IXW<Rui=NSKo(;1Pa&d>TzH5jJ-FrW1$}t9mFu_Gfzc}!Jc_S^Iz#r8p
zIq!tr%~t=G8&iXLB>#VYD&Z8-!}6eAW)Yk0vh%i<7dOpyz3<y1|7)QWLsH}oXWbS_
z&5oIjtfI4Z<+gBKEi+AG-E@;VFllzJ_Z-iA`B!eT{FM`5A{Db>)`zb%#s9T+Y`pN|
zBj2LddCx4Pq(!25pB`Om8nwh!>a4Gel%3d-4matpS%((5CD%+>_dEK2X0U_c$t`!>
zj>k^OpK)+{h>O^&coV-i=PiChf#)MHUOc<yVtT>-7|Z)+#{^5~xG#HgbaKH>hAH>w
z&zX_q$<qAM-P(#xF?Pmf6W;=nBWpM&)$H*S;dxNZ+afRbfPrO+6VDSzmJ~*%1&&Gy
zwl6pz_BOF_&0`4@*YbVvNcWzI;++2inF|)gJYnA?q;=6(Pjim0;T@gM=|?^1=(7eN
zbDcW(q2v|S&WUSy>qJkvthjYDwsUHP=jpo-P9L20fk9Q^-kmeDH_lFcv~j0gxks3)
zYkS;>PGPTOaZB4od>DfMoQO(H(k$&rU|^jk^YnIZqRzux;b~km3S~=trfl?`k(_(i
z?VzIL6-7BS7x|eHp{$F7J}gwY)h2(<+5X8>hHcIJ6qS^O?um)adlb6BH7Y^C)b)_(
z!AH5*XRe(2I8;e>UX)svw?Ud~Sj{7!l4n|zmO81(hFx`g-W!oC;Xhg9NJMK`%$d_+
zE92rs*v>xbj$gs5xqv6{+`XuK^J-EazGLj#lEeL<_2_~~1HXwJ3uj2Et~oOGeC&NM
z$K_{^@MV;`rN&N6iM?|A&zY=MqEdphb5)MJUpeFOB=zEk<L+COw;UB~_~L%$$hGQq
zV%{#i|2)tBy!e!%=ir5uNQPOg^+F1jYzwA-WwPDiuG`a8DY8I+L-Rh)dlEJeG#yuz
z1};b{d*Y@vr?u#bcIbk8HA~vG=9KrU7+gKUY1geU$;U3h+7;)`-PiZ@hB)uL>pNWd
zPG3!kF7eEgy3z1b<3+~FzBaX`GKtax?1gh=FJ6^j;3Jqe>+%DG%-X+8x=XLDux*iW
zWSO17=KL_wIaf9~XJuu`6$!2O6%GDUY`m-2EEL-(J8RL=D|avdXYgoL`?E0j&r^H8
z15K9>JLW{%ZeX(JVX$4}B+2KZE08hWbJ9wW5T$LsOosOs<sD46S|(t)Y<AQV5eKbD
zx^60dX)0M-vmzhO)zl2mQSudV4bN#+UnI3{hB8lQT<FGIt3I`@nzx8kQ7rz>(T=NG
zg$?ZMo(1qZ&MG=~??$KF`P@Sy500<+nUuxi5US@oVT*@vR6sxt%e5!mI@g{p5mfX}
zNRI63XX$ww|AEJ5?ZwE-X+o1;e>*v`%t3yF+3|HV1+Bz*%a7gqb?5lcDd*>?i0svC
zwasm;i;Hx;c6;y5+dEaCOpTP(?qJgWGk>b$8-|`k5(2O0Z#ty4#%cQ6!&(VJ{}1Oi
zH9MuW%)8VUwqQYqN~lrWi6o6fuHJ2K-tJCm?90}88r?a)jzceY&N0z*T!(J<i*Rmx
zl4BIR&?(2~Sq4w9-{kO~7=flUy-cYVZ(o=xuGRhbV#XFH%^Ziek5fag^_9u8oQ;d|
z)^eLD@Yp|(|J=sTD!=4R5f{namzNhEy*}~bs)oDQPtGuP61$nhyI^7L%o%MW8NK&E
zzIFWabgtYHjUFalnH6azH#L4NJy<pApx6oxnd2H;L^Nfz68YvOo4tB!m-ft$F*GRa
zd0t*<lK9f7gd`=o)=BS{1}zIxy&L)E#P*zJD~gV;Op=RKa?2EodVFWy+rAm+FL@t3
zb2@uo-2bgw@#`i{+xMXRquBfXHzw}85}f%VRM4|Q#7F5E$Nf!8%Izl~WKBB$D%bNh
zYvRnDgAqozDv$Vljh$N>rv9x~?36?Q*TgGlCMciVaK_Ro`ln>?>J4s=YuakIiMgCv
zUjM=UpB?Y7tLj@Ws@vb2?eycmL&c;$oLag%(fS(>&tG@QY6JVBKBmTXhb8X3)UwF2
z&oQw5a!8}&pq0xTtqsjA#Ra+>4ojHm9P>)xS$s)FL+wQ08<qk+zSjkO@frNn*RbV<
zA3S@>qif0bB??uthLy7C9-f(=G0oGg@JzSR+U`Ru7M^(V^n&*cfv3;kC^YKkHdJoB
zRV2t3r2Ag8L8QT>FGlA7WIe+orMB=@Yg1)dBozA<OqWY^U#nQ~^sB+Gp5Dhr1--Qw
z*gs!Qoh4<o=8Tufl|!*|EV>(*9{$tRjac)TujR<6mGhO>&5xTIpT(5k;S_(VNligV
zyJXX}i_Y=an$)f|`F5UqBK0geOX^wDyJvolLRX5CYu4!&zk6o(>X}@Nzr!kav#e8A
ztCH5;JX`fDQsA<`#oU$0@**-=KQM@81c}ONw`e|Fx4P}^i?^@JEs}M=wXSP3w$OjR
zB&2V}w$S7ip93@l9<+3SHR|)-)@_!3o?p~GVap=pFrH14XA3K`=hkE|&rSa_QEX?N
zh<DE7+?wT)952&c+b?=t3R2<S_f6vemYdn14kXI0n`Z0DIR96Ab%zt%^+RmynWw+J
zt;VTW7%9pA{ji458he?%xz|?OZ!xewu!Z}*CfDIFQWYiMFK2V_{vejo#1o(L{9VD9
zM+Vs{rF@fhPX783%xrkbx7B@?f~Q-I{>CL|<B|*|PkKAeog%RI<Uy?qw-ik8K6rlT
zL2yj4_-50DEn6=ab(-e;-{Rrx%6jTs@-}vT@AWF(NjC%|nGP_pd`sXiOAKdY$cU~m
znH`hwwXW97wQ=V4&?%1nvf3APSDSE{%;P>e!*AP`LrmI7v^t#Zzc_1cK2-Qu(|XGn
ztqaUP+IhAe-<aj!FzDn|XRuA(Xi%LX=@%>M$C~z^Igd#*<KR=NWlyX0ma=u}s&BV?
zZKU_U=-IhV`@CEY#W$;{&3l!!J@<>p)5PszAs63piM{*u=H0vr9TU}}b3{LA$tSG&
z9`=>{`!2b&B5oG{yB96=E`R)8RY|?*Q(i>H`<m%f>wiy;Jow!~;UI&|k4EVqjv7B4
zLzLHlluD}&D~Vcezxw^D1&Ke;@qB9(D`i^w$M5)Gwyf$7i%T3IHj5V2=pD)Nb<F#F
zNOo^nO^9#xuFULLs?!gCvH!(<=#rDYfS&e+=6!R^?AJKWb<@#)AfkTIBr{pE=;oEe
zulIMp%w#rOqn2?n*=>bL#?K)4_nnXBgN(QGIe(oIt)V^hC!3bcf5Foyy02Uk+$=7=
zz0ljtZ;?l5t>paqYbTgJe9*Vel)osLpRYffSt2=|`^H7?SsI<;TE7<Fwpb|s%|XB{
z>H1v7pl@A<+ooSTE!Q8mjp@^cf-B2c^S^XSJh|mi^uJ?H{w#%Gn>9o#=X>cCUu`<N
zj%jCL<p0o{Yttpyz23pZ{_3dGBQc)uEq2TFu5S`i*&g&*UQgXz_jQqe_PXbO<uAA{
zKGl0=82xO5!sM@2rf*)&<Vc)&tm2W$>RY~#49${1FTAL=_R{GsRi7;`3kp2Un&u%j
z<4pB}#;yHRvi32vO6IWc-`+jth}*VXol6t5`7%Atzh`MiL?!UE2q;(tn^+{Xn*4WQ
z-ct1E^HtN-(>=<2<h}&YU@uwlD|myWgZnh6jvd~c!Xg!>?7jAImdT8pD}G!)<CQDO
z^6krg2R)6-OH2=bE!|^lIc2f*w2iZ;tu)#9HJgcl%|&~04Z|1Cd|NMbZ=IH1`EOU~
z!6lccN88Tm{&~&3l+9`@+qyT8YL|HOZ9IK!jaO7mm)7ICs;=SR*#a6$b2PX(8`Baz
zJk^)n;a#RySzdU0rYrxZBlqgpFP?ogKHqqX(7(t8w<3kQN8#?vJ+@sh>`%YIIxps%
zR9<7+Y;PtX$C>90*W5{6``xTKQ}X5Oxl<<AeG$>p7MY;-#?vKj-NwyrZ?qdXnzu3B
z-&AC6&R4(gf8g(b&$&LoT=s73VM%dC<*hHQEBenp-{^hb$}p;AUe1lvU)b{le`RqU
zeC7}^>ri=mO?=dqiPm?gGw$DhbPd;!mlG#<{p8MGCAW{~AA`l8Ci5SS%pB`#7nh_=
zIrBuTJ=mg@$wfMhDVjGWwBP_k3n#Bw&fyP<PMyM<VLcujeRp?my>(UULExfEEkbHq
zik1!sPco{gE)mK6w3KC{kouY_8pQ|B%n>d0nz(^=$$mzs_HUoB>pu)!=%s2jcZZ4Q
z#f6@;g<4<vzq-71->Z#Mrq5n4Gn{oSYof~0<^D5_L^U#RhdJHaA-u$AhLd=H?A{9Y
z8(*u%H{8_?HsjHVspw8T+WLQ|M&6mk1CLL{SQKom*;$x&no~Obi_OH~LwXB^+n$;1
zI^AHjN_AbF?Jl3?H<(kl^WE)KddaxUoA*Vp^bVIhd$rl#`~ER@eR6W9dHFw?s*;zN
zSB8JrSsJSS`u5K1G}DH&E<yMAPEs;Ws5$icm2lQ})=Sfl{16f?@z-lS{p#7*x!FD)
zCJU?jq!!F$oWL}};XD`51}m0@!mH$}BNAQLaBaWQ#G|?Gf&r)IqzMb0C)<exF5{ds
z<Ixe}r4EH%qM;@mT_wKGN^0O`f1K1MT|9@$UDmkUXVJMC-aVTYdVO^4RZhRiIoUGp
zN`AwYIXP$dY1vLTveXIA<~e=Usrl0BmRkz{-!3w{mi>5#p{mvAGbWwIk1Z^OCkgK~
zVP9ZurD3~!#m=X4Mb1hV|8-OsJ+2An^S%FL2}i?|NjIFV+eCyiF1?Rh_~p`jHjQ8<
zKg~;DHn$3K<ue5MZtd&!n4F!yk1<NtLHC!<<Nifc^1L+EwwHLlIQ-isNPFG#MP8?-
zcm2@}TanZr^zFuys>I{Ut7puec4KYF-m*xJY|d<M-)$19-M+i7GU+DVl?rU+f9ln}
zZpWRwtJi5hyQOV#fK6KAcnim54Z~v{Q+m2}6*lv1I-$iJbmE+lp07uz#Y-P^t>tqf
z9tEXswlL|LyJxNcExxO~+gsW`-+0tgH#x&{N~qb*Q`2Je{#OPXTF?FYw98~o>z)@5
zb2FaLn)|yi<l`Be&`aN9zUnUwJNaSeA{Lbbsa-0w=DnETDfTD-Q0SglM|U!<|CTi+
zYF*l`*SRMZyt_9ZyOG4Bo8#i`!tXf6iE~%L5^Fc{#5LBglG#~>%)I3?Js%EMJ`qe*
zUMxI`OS!(rW|P|I?n$Nw+S7k*?pS?!&!P((mYgfra*nP#^*Jp1eZVyHx4B!b!slsR
zv^Jmmcj2vYeUY<wtaJDJ+*9|OdBNoLiP{RMdiyJ-S0^5~{c$UJ+KP)^6C@ITX`Bc!
z-TiGtc*pNf87qz5&1P4(D7@v!;`rOX&;5nMm)*NV7Jazk@JsoBoaxDB6F;2)-?{Y9
z$NZNn*(XK4SuBKRKVS*ppwzjn<ItM8HA$iuqK-<u*f_DOV6DcNgrimw2m9hmTt$*U
zEC}(i5M2|JtrZg473$L=y6(Xl?bM4cVtYcJi>97d51i1zciCxO`QHHj_oq(BcX%jE
zz2uJPwCr)=VwP0rOGub<uqSO<#1gHE%e+Ih4(IATpVcM7)pO$Wj1xyBj!kUUu{@uA
zOX}tgxA~ckB8y%{UYyr#WgeCy@@S!l_NC@|rkU;i?>(ccA6;f!{<0;ke2$--{o$_A
zo<(wBSG1~SOcdblJEXluL%>??i-QkegWPI^3C*`66gUJ|h&}$<wcVs;nnnf3G}%iR
z)HYrI-{m{Oo7-!_?R!2g(|28Ubu}}J>E~O%xKn41rOT>`j~cF6=y|+!UzK?2$-ENF
zuDv>@@mYP-wsr0_aO^Buqd7P0%oer#7dYqU3Qe{OZTGuAl_}sk&phkEw%yi}{sAV&
zakhp|yJtQ%p5L|Wxn=)!BNmmED&fw13{p#5%~q_koiS;}QWd8CrD0w^)3heL&3>%e
zT#}wH;LEZ4%Zv#Tsx4uiQ?9OEHf;h+h`YKmXL|6CLtKG82b#Y~9Jb<M5!-lk%cKQs
zTv=T=cGcNTaEM&k#Wnrtm9&`~RtD-m3_Hc9>sQCRI{SjumRAqEmtJX-O~0)u(0F5A
zh3H59ADg@rg%+J@{qHKcZjNSziMg@lITIc8m_v(PHdUNacsR9a(MgGurwZmA`jytr
z=d5$Ot$5=BJN3#_flpGtUy=1cdrryD`;f$z;(3-q*HWKx@@v28kh`R$()VehN6j;4
z(<u$tQs4aWxBsQs_5O;pFxw}0Te;3Dw@;OcY+1YHM#Ahlt5r^9&ui>b@ib)H(vmT~
z=JV~YFGe$XCDs*~Gffg*dqqM;_j6&&gF~0*xtu-y@}%XQ+r8-}m*c+W^(oz~$+wuE
zW@RttpLRy`^5-0OjoVMl%eI}K`;l#pwd+pLO_@rp^IwVVt~xaNz;B*;4uYl(3O-L3
zh^4J))9-2F=uxVkcxc{_J;yhH+tU9(h)dD1lA~$jx499uFP%S@1ub3G-f~>*n@5Mc
z1=FOdXIxaZ9>z0;C#|0Q$3o>7w*uqjnAP!fU5@kq@z-}tymGfssoyHKJ$(DtuBr=%
zwoUre)u3h3wZKYr)p6TvVOMgT)-MZKm6!K<PWz)n>+S@#+f3n%eAybNtTwT;FYt^o
zquPd(A!)`TM|g~9KE8M9&9Uz5XXZ?H6LT}}o0yX1qWr;Tv74!w_?3n4{6vDL&N(mi
zwr~F0j>-eh6F-00XuD$a)_(0vUHnT!u57L9yS}ekr2bBcMrfMr^p_9&y{(FzyjCW#
zt}NcVQS*fHl9ay9s-JQezPKT`E#qDBcb>^{Oq*S$|8tq|D^g*t`Kn>m@mMcm`)Qe!
zZMS#(&04l+dCJZhYxVA1BE9C<vv+<L@S6L9r{ax}>BB~+3cton&tKPtCp7b~dNBR%
z7F$-XfSPsd9`VjU;XQvci-l53wUnr(o3ZOBk;z`xw$dK1VR5S#9PK%#xHV*T+)-^-
zrpTolbFZG8scfqBou%>5#~m(HOc_!-g|eAnwAvrI<rpUNJ3@TVg1`+<rSU&zMP^?6
zRd_Mx(9$f~*4(g%v#;Ast-7)4c0r=}?srAkHed6e>D975dE0@oQ!-KOCPbIJDlEJ4
z<VJKce>PA4wbh2b3qMx$ZoF}Vt=i_9!n}tQ_isO+@NUj2e=+M-W?%m=SG#Zfbwi@f
z5(e7~0q%C47jA_vKQF)T?u*aMuDUZaE(m2=6j0pxWwEyR#9cuwDI)i^pML)@8Ss3r
zbNc$iqhf14QrjF@9P+=vb-l7Ry0@%F`Go7{ySD?=lJ>3so>e9I=MjtVX&KwO7pC5~
ze7;gYJEDBWLs|a)YlCP0Veh)ha6A6M+^z!YuH*HOrfz&N&E0;^@q+?u4)RQ>v|(n{
z%V99eIV_fS*v$J#(7F(jBt^A=YMrM{;XSo(YmR!Q*4{c^qs4X9L^Y#inbM)d$1ReS
zTGl8vya}7ZbMn*kh6>eKAqMv10QORY*s2L^vlnTtT6fNmLwm|1j>h*Kuh+c3)bnQB
z|8tx#dAK&KD%Kp7TVZhAVo{ApBhL}nm?uh{RROVgj;dTs?YO76wr_=!!s5;a9lY~1
z5_ohkswwkH>RR6E7C!NS&vGG;M`Cx3Gk@q~{=e&#gtC=-ob4}ge2O@BxuaiDy3lHg
zLe2}0t7;dmRh&M@xrLgfikwUSBBEsL!egs+v@gg1x!1?7T1{W=n7=VSSemq6G&O%|
zSB^^1Bj>7AEddsXIfuUI<^Ndbd`shzc<ot-2XCgdI7zfl)SBcysV&#Zz*Far8`l;E
zt}P3kvo`)*^U;})({<Nb7q2JN|2%V9aejsn`|lgMR(HB)&da=Kq5MbV*u4bp`w`B2
zg_P#(TY3N8f5YS#oDX05``vo!`_J$phn51PpO46!11y(PBzO+k9iGROHGdBCe7$u`
z1x_(0rq*e#V)E`|+I~DkBkPHo(NUGY#X5oq^xhjcJ!A3pSA532=$UiYnYSm?oR;Pl
zFcfW2EBv5VXb_uaq;h0}ZUU2@M@s#o1x)i7z1Y<EV#Cvw(<VjDDR{Gk?@dRb(uT8Q
zuMUdcQsmxl5M#!odyFfhMu~^@h5AjWtaD3m*A%_=IHZ5?-7dRk!=5L)+-+RXver$@
zeE)_^pLJq!+++S&=43AIjV5k$s+yC1)C}~W32gSan4_{u>|?jpSwYvcDz`Zpw>5|q
zs2S~4{$h1;%bOQWlbg2&{4ZP@X2_V*FB-4?JgGo+&auo*A2WZkTe++fT>nUZTDL>Z
zLghB!soOT>nz=|NXXa%E=B+Bsi`x7%tXRe3U1ruN)BLqRV-87IYyLWv>pqF2aGk&5
z>g3-t?J~2E%sP^<^d(ttvGd+zn~OEv?|%uBJL>=M*5$u<{Qd51{`+k5Ux|1AOg#a=
z^#2JhwdLgd&+(=#Z~6aO?G{{b%4Vr63T{z2s>0atU_o3KV|>?RDFekV3P-fEj%;FA
zR`hZ<7ZUNjP~+EHBc=586njljkJXKaBL?0-OWr8ITD~@|KuJV_G2#7$b3r<+8XD(6
zsWs><vr}dYIR1LcrIl-%I66OY&Y!@w_J6>n4ME&n17gxLVrz<cE(*DqKT_@E@t(U_
zw`h^~W|8n8IUNkiozpzt{aVU<dCCQzUYX@5WL|IJ`@KQ`Yk=8~j-D>(R32|Lv$M%P
zi+dEbOBqa!*|(;tOi)qXC=|)Zx;CZHua&9&n6RagNwSIQPv(X>YN`vKnZ8{dR9WOZ
ziQ8)4TYHT|5{rYhc4?UhpVU5=#=YQU+bThejfEEL3(toIv)N8nk2VZ14-HojajN{f
zH7d|*|Jl4#xiW=IJno%cKdCeRQX#{fl{`}FUe_+l8meD;B<S_zbkUa1=<CKl0)789
zn0hy;2%c5&I~O|dL}%%#mn9;tzD!sB?mW2vPuub0f0uwqp@O18hmO4PlerLY>&*1%
z#3C~zWp1-YR+FmjHfb0esbr?rs4WdPVGZ^(ungFAL}x=t)<IYM!>s8dR=qiHfnAS<
zI434e46ECsWVZ0dl7e%yCY%#m#I<>?@-4<MCmN#jR>st=@G8IN_5Hbf`BKX}O(y~Z
z)pnYQ{ZCInUb1$UsoVWc9``QAKRT5#;Yq@~B4fEUt9uj8_=JnylCzE5vc<hE(;oX7
zp8fdXYpK#}W4qpf+h&#<oXz;RTIvaIbGbOjWqrR#fnvD!SGEKmwgX>F^h}H2=C3}x
zV~gMMJ<qfXmx-DcPObSoHLlY_eR^I~ao(e4;nRdI&BKeQPd07*|1~_;OR1)`WOj3j
zt9gp66Jy8LsqdZ^YzU5g>l}4*qD=5((*wfgcgyx3`YflJ_V-S)S3$CA%Xi!J(=5)E
z{`>LS_HB3NYhwpi9s5rAa-}4P<<lMh>Q{Y#U-_1Qp-E1a5LbcXQ)9zxOpKW|p*=O(
zoi(SYD=N9xXt=T*S9iI4z50b&ozI%!C`VUzg+lfP%5GgeS5&PMCa5!*Sy@d}YFgBK
zEypu;n}WybwCzP+PZxQq--xZ-pfZU$qc*J|@|8%SaQKhKc1e$pf9rYsZ;=o4S)ZE)
zyt6s@SeN}4dt3O5YoA~9dauL$%5U{wf5}$v^*{e2K*+GSWO|3DH<OM@p5}kS!09tS
zOKk1)Ixnp2n$X+U^<Fu}^|FYyXm_b@w{5bM<PoifPZi8QN-A!T(GZu`pC0Dk(r_z6
z?Ea7D*N?-iMN68umN=VVar<$bX?el9w;dC%M!q+)PM-3<Po>D{u=U=}QR=UYZIYvl
zuSZvx+sgCqUFNd4#l>qvIl~!+lwW~yCp|09D%k$b^?P=83jdliznKc^Ia6iN#m3K^
z`tpg=ga_TS6BySo@6s2W;CJRTM;Np1rs+G6FOs{q$nMlpg=>~NV#=X8nI>fu)RY=#
zuTib&>wbKrVb+;h@e`Cj95;$Cb$+x+Tk}TjAp@Q1JU&}RQWI4(mhMambW-y-6Z1Ep
z{rvyWZtv|%`z)UKyb15>So`MEe03(@s|H37KD=9YZNbZ|53g3emwRh-bMnHkkM()q
z8|>^zR(PKx7hE}kxl}6nQ`e2fbDg<7678nk_#AVrZe=oysp=P>?Ea9V+Tit}eoCTS
zcg&i3%%W9z`Spr0(@@K5E6emk^#WfNwUEBOH|D7qJ?{B6ulVisb*ua1r<Z-ro3Nd^
zY`25#{`Y&<Y+YFVc<;8VKN3+T_q?+YTSezrc-=Z*p*T79$>oLYg$yTN@B1-1cIkAx
z+oi!NzOlvabMN?9b~<M*d%k|XvT}>0bNSAi$7TH{BDLRFEaBX(aoRoT2>T)*1FMjd
zO}yV-BR9z2Qs5H#f8fp;Un{d6th^Rt3!TH)nRo|R_u9?24!UR+BW)_G@Z3dup79S>
z--@;0md-D)Hms`Vy<4zgZimsf2Sy#gjh+M;Klozvb;8c4Chd$LcOKLiIQ=a}M>qGg
zNUnX<?wyYVA9XB|_RRZOo_KI}-;%|FSC!MxJys0)l@x#Ls=$xt^%e81XIQ@Y-CUaF
zbo*vs{niz(+gJRbt<ET8l@YZsareGr-+oqi_ZpSElT0Tpzh!yPSN5gvteo;9rRUc7
zH%{95c;dm%`@62n=|*mJ+Ozt&$CU76ydBdkTw@NOoaN=Se2zh%!dt6je_yOS`EpAB
z9EH+9iZUDK<u^@B`Rp{kvQ|@Tk==hIr48weeE4QKPU;f8ZLG1at8;aJ{1-(gkGZ+C
zyO-}25sWyo<;Usy%v)@kGtTC=>`p#2S$SgGt(2)!;ZLirIp!?YnX22nV*d8Y6Q1*B
zv0G(5UFGr5HIb>doPDkdhv0|2>qR-{UR--~r02$OmcC?ZwfJgP@0D$fRc!-G%{dEp
zZgD9|^v~VnT)Ro5;LADX{FmFOM>|dk(YCw6bv@zziidl8e@TbGlqhB1bM5XOOPAXs
z+jm=s99jAQjn!q|vYE>a=Wr%YJF;ryE?Lc_!?nwzrytw9<nP+QE9BmHdv$F8TcF(i
zqWF^e&UO5%raSH?zxnuQ&flZ2g)1VN&Fik;+IV;0q5rn`)RhV<le$z(r+=26X&th$
zSdyjNE#!;X#*OMhd22T9K0ZU(<?EmDxslP=>iJTe*H}HceSI<WwXIj`U1scay`A2?
z>+E)kYv<>j>-Ko3rPLZ6v7qsGXM#n7#ep4yZVRQ)z8A{&J9>Gixb3ckmv`NN>3`Tm
z(0FgI;oawV%bc`>lpctk@(g-<$U1O##FhDRd-Mx!?LJ<fn!j;pd!5He%Nu*Hd*6P&
zTkg&B!~5jz_NgZ^rG0w6-R^B~*p;yDc|BhXEFSm&uXxAu)7O?+*7EQw{+WlA*RECk
znR>5qnM%{X4~CPrc79abdB@c9sBqKA$o9#h%AcZZrkwq?(dy0z!}&+A{!dxUJAcyB
z=f+3Gou^BR*oOphd^_RUmsoF^ICIZ*!=rigolo(HJycJ!yTtT({Y<4pbJ&`7)XLAP
zpSah2y6!vwjcu)G+&9lTbN%1r=QH2z{CD7S^fKSDmei*$7eCCOIZf%;&V+^|@xSgQ
z$lS@2JZ3B=HEU~C`FFYO{bCyrh~?D(yZv{Cx!JE>Q`2)b?^?C&`=?y{|Gm5d8;^$S
zh6M+knFXa>3It9tv<ax@WL?c-(C%i{SGqDo@Ud&3f^}Yq$0n7NW~?_ST-xy|LAqCz
zd)uba(+k!zns4U_37D|lalWmu9E;YL6&DwK^fHASHDp{`9<bO;*2?kJ6iw-6IWvtK
zR)_sxAG54L(qNm;&CS7{eOFgrUURc4^#WhV>}?9_ySd(TU9oOh@3^1+<QG|mh=iB}
zh1R?;n9LJnno`d=l)Z`g6mz<V)hKLF<wu5dtj3R1c`Dw=oGN#{Y@0jd)02~H)wRQB
zJXs)OWXS&H-rC;@NADX|bF<95{ovgUky3T}IDw}HVItl3sh6df7T<l#b4_Ml@F|A(
zPO1_qRt_}*%|U88Hx-Vqbowq-BlI=1Zux;<zP^vv&z4sV{QS<8bxO4t&yRTytdovR
zZv5uxdXK@&L!q@r*e5VCP+DWkLI=sT69*aO_&zLjj43$i*r_3AGA$%e@yLy?qEg1i
z4lGxHL^`oeoHXHouZ`N0<bIZm5djm}1y9BYy78@CGQo+d^2uZ_)qnsdL6wE9n&zD<
zBJP%2N0v^D=MtIZnXGk1NLzR7rAx(Uc-3Y7SYMhr<nnSIO{izpbUY?jJxTC^|MQ2H
z&li-rNop+kW?8DfD8TI3lc3Ph9jzkK6HkanPH$T}txKm!>q1Y)RK@g|){W0r1TK}D
zlI6IRtL%DK`%EeAHO;DJGm|Dh%xYWLb5<&9?f!kgva@HIe$(8rmec!<;}$J#?U0}?
zm(F?Tg=J4FxzsxIPTslDl0fY%MZa@*Jmg!jxKA}ueZsD@oazC#OsSR!%I-S;ke$c+
zc6q#`aDic?kn{vY#wPY(;T!%322J~(KF|GVdA>jQY?q9=jA1nem0T~o4d?N*od~HC
zYuodXp{Ob1!U>IPtt3bJz8yx9GFw!|d$ce0tlBtx(=|`Y@~MYDJJ@r7S-jI*$X8%j
zDo?LopO5g%l8gScMM^IRgx^)U%%*x$<?8ZjEk_uF>gFDhh_7pzI74n}?6#TFN}|g=
zHDZ?td5bS(nKRwwCX2dckk+axbA?qH6mB%?b%h-9kIebd)L!<sVllVh0*BD>+|-@n
zv7%F6M(Q?2iHpu@{};A?`O&Sn9xq#JcO$TTXIJ=|WfPyx+_3WQj~kgg?rBt~@8-W&
z{dVi*BjO+CH7=9r+49UUAbs1Xy5vA1-RG(Q^N;NOeS6y#=__l$o?<=ztT1MuCBtsl
zK=wPOm$K9YtheP%GN=f&wUBEO$~ov-{Zp%A-Z8e1_kYw%Hed69ve<&*-`Rwp%(H9G
zDBCkw>ID7@YN$~2FktZ$ND<w2;p7A(qvn!~RAr-!%ah!u33Z-pag~&d`<G<pHzhIM
z`t!|m&mK8Rs`cvmy_$HbX3JtZD^A7W*-_WJ%#O(X`q1IVvS?CHi&d+Gi$drjU)_6c
zW?KS3=CEr{7F+JFru&mkIqTQ8S*s#s!fq&8vrZJh|EBf8VMQ0FV1YBttf4!aeR<+b
zbCegqt8jOFyRp;LP~h<^mW|6oT5YE-zOq!y)u=J$)kLHJn<GQK!i;A3c5ycu%AWI6
z7m4Cb4GqY=s^-2^=c<WX`RS<Mj4(@`^CGXOuCw9L_72uIzVOcEkkitY8H?qEV%GZ}
zFzItx_jp~xwnxzsF3<8$Z9Svk$UQSv`M%?gqk8vv+H2peK3?{$_4pqjjYer>E}8Ex
zM&EdrCj4s-V*HzNU<*%1frzm`Q;m^-SL~bmw`clUI4kCH{<u6p_`v}-zmSD%G7O5P
zZYi{jd|Jpe^@TIPZX!pz*^F6lQ(X^Bb4z}?<g{x?LAveZNXeQo=ZTX<G8F#?UTRnt
zq^P^r+x_*!YfTwjmCuD}Y_|09EACp-^kmAiXz546^CY%4F*rC~+_>oftxI7?b*xHN
z+;=L4bRU|yd+HLUPnV(-**x9<O7>`So#^uWclBzX+PirHnhO30y3Q|B=(BISvOGr8
zJ>1#PkLll&hKL;jOnOreu20Qymb??d7WlGMJL#|2QiJU~H+7_@>aOCBJ}2a9lpr*D
z>1myf8Jc_cREQ~V;aM9H@?&>|k-y@iz`OA&FLyVy^lv@-XpMhi`K;>~!W4D}&kH=J
zu=kCkd(E*YXNx+TWgh=As=lJl_bte1E@O;)?1e^Qr<{+MCY{jN6zZ73qFi#f;2Zy(
zublb&c4^18Rc;S>X8XMI>S{xFtEIbrXC}l*FKO8E%cbc5oCBSHSFQ_9Ik=xeubK1T
zf9(V&_8lzDClXHhT{RNOQ%sB5F#qqJq(dUI*;^O9?e>VB5jpYFhQ;#tBsKF^1kL}M
z({AXy@nV&Q$t<alM?HEBg;R5aR~@W4)?a*rLFvGR_c@Ey4&<(KT(RTq+nkLn-Tl(s
zw%t5xAhub-N9?ks=OHf5(lvn}CpJ8|*L7{u8sq-dx5`y#SA`kydldaG<eDYi6aJTN
z%K8RfW=SsZ8&<+MSj+Vc=J^^l*nXe6o-tHssnp7y+cH>#=U3%KsAx`KZt!bUTja%f
z-{;}26-Co6n4i3EYJC}`#Jz1<uBvvT<*Nv(C0AD3s_KR3YR7L)GfSHh?78GX+lz?J
z)dzSs6}JDG@;|+)vwF`p4%3;t_1;^GtG%h7?vNGKy);&`?!v!!Q+R&(wwVT8&a?5b
zG7o$El`UZJgJt^m(o^np6+XD5_GoqE_1XtazsnE2*W;Mq|0$f6zehzj<b|V~nOOU}
zb{+nC1st9`rd^o2L0`hubEU`8#EE@BBG)cF-SOV2aLT4Dj|}H>cZF94-(m7KG_RZI
zCwl0z(%lp3j=G#D{QX?k?TpD*d1?6Yg~@W=gqOLBHy7R9wPV{{?Z^{3!bN$Ho(VnC
zQaNS5(^ET3Sx#@ixwh<==tX;SjvH2s-ie?0tL9_m^>x<^i^}gd%>Ma0+~RWO239j?
z$q#Rugbv((%ha};)9XPK<C*`p2a4DKW)R5vZuh<NE%S=cF&h?%YASrXFx}f`napj=
zi=L^QdiUq3aW0NbOSIP13VF*FoBb**YoEr-wx=uWS!bR5w(iRo+qZt%+EX}uSbFbG
zmfM!HDCQ|k<+tNMm#!_G8)E*l>c*9Ye1@fq1*2~`$8Gi4w(mi@$kz#pT(_r{Ri3yU
z_e){bUcsOUYtv;6pXz2$h@Y=5%Wm1o`sDka-=XEZnR)Wd*WWIjWpa0p_ZHS_eG4s3
z?!eP>I*S^P^BhWTRAM@j;juB$Cb?lwLc<&*v7Utua^D(w)lFCpldMbhY($F8HWm48
zDhi+MqOTrodQ4gQYopT_70zM-PX*V|tttNxC#J-F2~nL`Y%n7wH7O^F#rL3Ks@TNP
zf<-0w9ObgF$$u{s$S||K$*!t?!cT2t%T2K~w!oHbvDUW936l?}B`pt636GfER#vvG
zta@A7f7kNDbf(yX^0MRQ#pVpM8_IK!r^o+j%V<w;y&B(ejPVz1nXmu@*MfFN1;xom
ztW%6KR4<E03yO75&1gGTq4n5OEzpx&B6EU;*Q_aCOG+x|g^RB!>s-_xHOC{%|Dn#>
zWm!9w{Pww2U%SM%Bi-kkQOuzhvt7#s4m3933$i=%OfTny;H6Enw^&lwB*>llZg8MO
zjwd8Oe~O%{U@gbR+}myP*S6PQVDg`ISU4~u@5S`I|8J(JzjsS`yS?|*GJ(%!0{1fn
zYQF2#r$+uxRTK_V;M~y1DZubnf`NgdpFyDC;X$8BlIZvLfL|4w+*kY6igo@y%NICW
zZ(!_nJjgitAg9KbN*Px1s*MG_H>KEq6nGj}3T+R3_Q_a--NdBHS-Uidg|SHgOQA%0
zk+S5Z&8?H<+=VnWojDISsy>|LDO9C)z0g#$nBTCtMVR&C(h$3JGy9u@4ktsrMRSZ_
zxO!yfWF(vWy9wQ9tS-33b?;(HprT`lr$}g_La;?$>e7~qC#9vXtwmE4k|J9xP0M61
zgikh0kKt%*Ia-!9qdeK7EnP#P@_KpAx86$5wu*1lqCDFF%L)YiV<{6l&|ZC`kCTC+
zbLNZ>J7#qKoY7~=P%F_fxu_z?v7&qG%q~@>u4fgKzINmpW)=r#E|RR5y_UILDN1sh
z*s7D2YaUDXJha&uDYWUS4co>lZOtN;g^c@BLf3NS>|^adSZIEGa@W<xQ_gJk*&2~^
z#?tOwsGxUn&9?=1+b(eZWb4^~!tdrygTTNzA>q7VTNFxEdvm9Szql5ED<a|T_PUod
zr$4yS%VgN+nmDWITln(`(VI&dJSQ;xmSp&++5cIhpGj%~OVk2pgMRi^$qWo;o(28i
zrd!T9?5OtH(dwb%A9FEfp#tWN#9p`A%@ZqEZ%$m@*r0IC^N3TUq^8M#?bbysj)l%f
zK?b{ux)`1HE0qm6lf#oI>pd%Szo;B=DOoT;?&76rhv!{ZC$g6-g&K6|?JR8eFB5d{
zDhc@3%%s>FS`~ksqwaUeQeTnQj-;g_76v~|%hJNj($mxHUQTCjOaFItSzdWtZbw3%
zdwG6Wd4h#_ZRPU+p<=2EGh`atKObrD&|J{NIkW%fjJB!;OodH85kiwKH7Bg{xSwd*
z7m^X5*paj_qrWs!q*Y{^i&(vA=9Fcg-0j9ocj-*C5MAmfv|1}`u0&R8qPNO-oegT)
zTpL;622|%7#T?z*wN|70@RQlckEr>I$#}h*b2d|E=PKm~O|rXA<P;cnsBE13|3#zw
z)o%)?e$-xiKKE1MQl_H#$<DpkPUOAY-ur50?}tc%Pdm$BW(ib@SM&zXe<^1C<EGr_
zEAu%8SUyNFI2iQ*ky`NAs-KaYfq8YZ`i=!0jT<ayZs5|I$vaE#$BzYqyZV2Ksdr}>
z37Sn{E}t-e(gd|76Xd)l6i#W)ye2VO-HRzuVoAe9{`NwtpPLj#H|uF{Hqn}_q`p{l
z<!sZaCX3%Ds@E6mE_dNlG~1+RVISsWF@1{5$`HGs*^Zf0d|wy4PR~B^Rqb}5Or+6N
z4#Sq1sHth!{2$!f8r2n_GBcdfG$8HNv<#{C9PR13+tW+Jrx#UEX9_GY^A<1fPS09C
zEj6+Izf428kjRSmmwj!oX0%1`_?V%wVzNQM*UY9Jj?<YJc9gE{7peTiSUHV#)e_Cj
z<(8c*f9+h)RkdNI^}1857EH_9;aRmJQ+h|B_@m<L)2SuPG-9p=$6P<GdMPC5%9mYR
z4dw<+%ehpgFE=?Zjx+w-7U4%T_b>(J{W4v5{YPt=)ZV9E)32<a&b4tJhhTlmhWM}P
zima1Zes#|HXSv{s_XdXO1uUypFfH8h>h``jvo~<9PUa01Vf?URqL2|6+rnSk3+0#8
zD_p4;aII&`C<rv%bkn<1^;UtPMB$o?n@mz91aAbISRJ%CUTl53Na{zU=JiH}2p1E!
z5dTRmK?TLKhTX1RuK)KOlg>5{J!<Icwy`PxNte;bAW^}k?2R>6j4gi-=_hJWvy*AD
zJ+qa|s3rYpeHvFp_UdH`JC+r)Z!eTuUOm07SZck`8L5_3iP#I_FU)6L)Y#F|y&|D!
zN5`st>KesqIU#;6$6iMZFdF1`zdFXDEYQo{QSV`NuP{R-dSy#lM)$S~@otg6ADRof
zGN-6doEy@~mbl1$(Fy;Ht7iW=u|eu&pNy#L^{kECb}=ck<}ys(BULRrarJ&vhMR@i
z3*SgzyK?eC=qV?|9JYme`@aeqGV1TH5Z<a_m%UWr!ZC;F-R@5;8K2(V^T&1F+viMa
z3CnNJ=q<as?%s+sJZtu<2Pv^l?fd`mb>A<qeGE|x7^~0zowXoaM<D&p*{nPJ+&=ER
zyGKC9@f_E}cHt&RiIA0lew-847EnE$FVVeGT+7MevY5t}3F2E%sI4v(Ub%@$*F-{7
zyJ}*T%r)(~7g*(8#kp(6jgL%J**Vc{ca<5pi+bfGEp}A{^IV?55S3<Cb)9DCpIf|B
z%{E(1@%fz{yj0h<e9plVfq<E+vlBd0t1l;YUAFq$lB&%Z`EYyc@@cu5>6NdJ6jzoP
zN0ocbZL3Y+&OUK_BYQ>agrhMR7@r?I`t|A2=IR|CRXaL8k9AcG7^R<k5OdAa@)~>N
zxs+NJ^&7Pf935{Qk3T4!`%Y%(l<b{7FIM&)XZ>$9Y10hX`BTd?7if#F-5IqxQ)or{
zuC2ziYYyJnzo=@9>J7=wtS^goJwEO_(V2a$%zUG^jN>sb4Z)hN8n<3Q(rt1STybOf
zt%*5rr%ZhrC?6xbZ2z1(*$GEqZ=3hB^31Eqy{CHwuFXENJTp>I?Cj6Z_3=I{x;)SJ
zMD{aP@4KUOj!XJ3*TQSe8_setuq5m5S6O&>{%i(8rj6gL^Lv)|`%kP-f4E;ztU*1<
ziTOt-SNj1a?L{BnCW=Yk;N?u>krm(UX~o|Wt<SR9VbR6you;N&FNQu|Y%@#DQ$uyi
zT-CJ?4)L!O6tH`^iNozyqmQnHkZ;$V<G+eI7OEMp&5qEHed0Flf6{CJr1I8$wJX`L
zTPJQjV({)rfz|Q~_s99C)|I7ORhWs_?c83gJ>z0TeA~=Btx-=ps&^#(J;u>^?dhEr
zfd>Vy<O*Hd+u~8!l)FdR>%+B1jqCp`G-Ed^wuW2Q*gWg>I8d1@(C8yJ^S9VSt68ld
zuM3DjKTxaHb>_t4a*>%YlxE$ioRA|lGm&lYChhG_C;u7?F4H-+mA!g#&x?h7UhLjw
zw>xZiyo&6C99_?or;Z%!I-zp<LX>dPm(#I@O0o;;cJj9E%3W9d@iu$oUe&^r%mUfV
z9vf>c3KyLe`*UaCFR!z|XPsr7&9GN)!}Z<`pSoE%C$j{{3Ha2BJn30*+m_+TJeL1~
z8|L5WeKW1&+^6mh_uejedq-5<=-vk%5#`CxB(wsRRn%Y4RZ{3KSe1OBHK%e)#))S|
zo7OlL&MC?K^;c)+#tX8Vlm5g_vh6--Z9VDeqs^PSwpeuuxT>mh9GJiM_GBZUt(q}Y
z<oueQB0m~+EIHI-=XQOyhD|K1qh|DsBY7{UCH>gu%d<Sa<I2fB-g!UDU2VlfVwu8o
z+kdD&sort5?bcQIe={ESSQ$Nk^5D<sEB{;s6<8N<;7s1Y>As&eaKQ(siEq+kMFQln
zz4ova;O>Z=dGg;3qsgl)s;-LG-l=G~<B_-V*|gC6ODkucwYkyYGT9_^SG(=5HCyg)
zX1%kz+*(D@_J7X&o4zM)kFC79<ZMi1eYa$w;L5kVPrE%*y}*`RweDs|`TLXdUn(QF
z@YS*|>U;BJeqzj%J@3|^dHb5B`mVsjWd9|{-gLj^<?eqd^HlZVFZM1LH+!~O3s^VD
z|K23Oq&i;Y&&szGCnaz{dpm2g$lq)uUbA=7QHk7@&$t3FtZv>Uvn{acVZ-_RJC{Co
zx+ZT@D>!?O!*`p>_pkFi|9@L3_GhzWw`rMNlfn;YmVaLYt|ar^KXjkzp?IEcxb;K%
zcOimthxBed^b9-1VAyb#p=m9fgo#JM!$V=)M66UYy00+k<(-Ns2~=C$DI{&p_rPL7
zPp1)!bCZsx!eh5yX0Iul2bcb5Y3CER@R_kdVR;J!R~3_ofC6K{BE}9r*QzNm7#O(c
zTDg7*d3Bj3c&XdgQz40=O^hN*t*=5(uP~ey(VKL1)wLCYQO9y788PTMu8VwoFP3B5
zyF)CVY8o?@R0|&HHnEGSMO2(>;4NIPFmIEi;`HOe8u<$|DnF&|p21l((}&Gi+-QO8
zEIsF6B`2?}an<tqIV<7K4esp2$84>)=j_?Lv@DQg&Vmn&M|#=a-`&oWG@k2XR)6m0
z&k)YV3PL&8wH6+F!^2(@o?-C2?Bi|zTfQs)?*7&MgnOBP{?y(-O<}JE*F4?QD<Y)3
zc~$_2L(`dn4u|Wf6qHT*c05Sv;#@Ytfs^(Bq!|V)Segv97!?>8vJUfEUYT&1(MD$D
z%9YP{y;{BY%_?oJ4L(!2+?J<o;qF+oDC4wFS;ozc#~Zg@NYQ^9)KhqZanB-cPR=DW
z9J*9z&q-)tUp!~hf}>)agA{exZ%i_5wUKUUKC5Ew<I{EKNmKVljbA(_em>D2R!vV-
zlDAux9-eG|%R{kw@}yTjC(qyQSe(0kQiJy5$$Ed@sHnO=^(kox%9~um@Z4)-$R!U~
zro2nu%5$eE_{^>D3fW^I8mbtgExSNVWsX-<PuQo`NB4XdIdy(rW7^%&1)tfJG87U;
zrNkoQd8N!E;(K+=v|89Q9`8v%tUN7hxq?zEH{YFrm&#55V**yH_RR3s+&Uxgkwep*
zq96sv`S(P<RxWuK>ZPsx{-}BU{eT)SZJEUpOWmW`7PN}Wxpy{A(cmZwykFD$CTlf|
z63?2GU5^h{H0%&Nbz>v1*6)gLZ9`>CAG3#F);^i!nEdm}wH+28Os|*BzNzW;b;`}e
zx#e|UyB^o*>Ah&}Fur=FwP)KZtLU7kHzTv>CjAOqc{jOZ@A7BL9IqL-oU>Z^^NRF~
zeT*`zmd^e2?J>jE8Ijt(2X*c8j1H9vFNoIIJ@JqL2k*pB7c>OTr*JcfH##`vxB5H@
zII>J&{={Q-;+rnUiHdgF=pVK`qV@4`?dqCUCuUztnyhg{G2=mNM520$?*DnO_PE`b
z317EfHNAY2r)K{8qEp&CpVXe#s}|mzYHBIrd)DN#jikOMdqkbzv$>C!_`i7lW^&a`
z*0-B`p3T2+m+@fV^OQiBX1fi66Qx?eWVCwR-BSD6acb7pR)eMYSB5@)zgBBQlIr?t
zagAT^J2IWnW_*{`!2Tzm@nf<>s^n?~#*k|vlfH_ud#$w-TQcp&y2k<&169<XAL8^A
zQD9^}a9Uk#QaIOSB}-vPA3eU}#5Zc!O_~n8(=&@X@k-2XSE)s-?14*8JZpodi&^rx
zxjgZ1S#m(-pn)f6;H8L!H7y%8ZQKkd7V@O6(%5v-My+!GPVO0EO2-ZhEHT;a*qaqq
zEcM@k_kmxSX2`rp8JnL<c%>gd$tCyENoMs7M~e%IeR*?)Jyv=o2w(r%7kVUkt4kHT
zO{6A&&ArD{+=F<VgD=jyw&<(r>aC6uYXzc}KR@pB)i~fOv?y9gM}%chYR6QEg8}VZ
zgjm)xoYQJ}JU#TlBJukQ9}7b!b_rx=EZNADAhJ_QVx7shGfKR{>9w8?Z|8+rYd!Sy
z;yZQeT}`>T(MC_BtY2!)cFueBH=Uf5@aCS=y%m1<MOO9tclOFXx0rKX;Z)7EuE$H3
z7;E{~a%IkWrM{ZQO7xFlNGQASRqpt$6I5~-BeyoaVD?|`kz&;vxocC#R;?Rr(manU
zT>Q1d$)RfXTZ#XtZTK9Y1U_=rpQgv^W>J#nle%h7%sh|j9ulo;$AY-aK3rLDDRtPX
zHRAFH4q^SBB9mAqZE)$f4D5<|6(O>Cm!WR$8{=7mai0RN%{EP)Y7(!@yC7kjv9VWb
zpZVeNg;{37PhVVI>!*3^SDOlB`=+m&fyGiP^WC<-J)=IE>!bFTh=XitGoRaiZSb(>
zaPs06edpvX8JMD-Zozx;Xu%}q6Db>$lm!(N3g@Y2#bvN8zUKI}*mx~hig3Gy;K!W8
z?rx{ZuALLM?J(q8cvaFTeTsdUmglT1As(j1%g^5Xx@`8Oi9Iu#mao;7SikeQ_H(Cg
zR#Tt)hHI@_d2h}&P4(A&xqkm2amRl?wtVka!6kvueM71)?vDGq>Dhb^(HDQ&xb8Ql
zv1-@eJj5cO;qWi4yJ62N(XD#F;yIK)gkSAiuzrf`0WPQBsAnr5xJZ2x;F_FRID7TN
z*lJ7XC2YkjrRoy@rEpDA*^tUJiQ!VGbNhyveP80-d7ibs+_Wgnb;Zj1Mxl`98KSH7
zj&x5y=@V{w_^STYi6)CJA7@_BxVCc6)xM%N%2Sq}nZ0|#*0m=Eqo#fSF~^>7lEdYh
zT_U#<_;+iVX+Nk`@s!P+=Mh(6Gr4AS+(OgmYg*YO*bYi*KPXN&ey+apScr#^&F?AK
zijGcOTQxVhrt^g)zh+Uhn-h!KMXn%@lPi8z{ht_GB7O5%>WmB0UWXXW4B0kKNEK%G
zXnJPz)<^hlQQM4Mlh?nJPG$N9Cx>j371^jc^}grQ*$aL?yRETEx2q#~^$G25Mb|Q~
z-Sv6e>!<g6%1;kF*SY8Cbr;ERdvB@kRmvZ7e_zP5s^IvaMz`*;SS~5o(>Rc+t8zd(
zw7`v>;b4XNjC<ls6ZXi)%uoMnoh4k?<?OLBQ-EJsn^)EKX4FEjJu$m7cbq-FdcBf`
zkhdaNP=M;u#+gyOpIo|nK%{U@?1~0;re#k=AIDDFc12~k%QB;wBdfyJ&lWXU`z9q_
zBrf8m#O5_0=A1gyrhefiN790en_onIi(A$p9RB&68&|Z!>~sG`vbFbUU0NKze|4zZ
zCes5&b2l_Ty;Ut!AsZFCE3}2%`&rVKjEMB5TlCT*^i3VP%C;<e_hteEcjgz%<BSis
zv^NJn)QNUJy7rge(%j!pf96^6>+ain&gIWyb?NVId(UZe1~9$ND9=_4n_PbL%A}$R
z*_)os;xRecoOk-%%h^vap6p#8zwOH$@x3n&-t)h``|rXHIdeX+KL2)uZ~eC|Z&y6u
zn7`Mg`uz^=(EZiAChsfPYcPLo7D$mRxOYANL8Ed;0+W$S_z59}b-!jmsP%t)@Izh)
zo93CW)=j2Nhe{T5+vnx6C@7Y!4d7;|miZXrfArzrMhm{f1`*0iUD90)J=V&G|5qt7
zSr)J6mMom!X}NRK;f?B>)Z}h2($?acxRdAf0j;wPiv^bKRBWECyqM!6NALL$mbWX6
z=FHqwDP)yV>{vU~tZLK2`wmtv9~U!z+_dI}p>ej#YKB8=yf?d6b(wb?e+%fh2;TmS
zp}(bL+Uk!Qw!$krFBl)))nBF{ebi|8!k{e@)`BV>_EOf{`(_I&Txfa|;Qa1JlRNju
z_QZ+H6n8W|-f#YAe_QZUm5zq{QIm9Uw5W3Ooc*n<amrNt;?_sEmgxj`JXUhlmz-Ca
zB$+bV#oVy>UFEj4;12II!oD?o#qZeaELOiO?D}b;nSYI3gVp?oPgA@crJik;{}t8o
zEx2RP?*Ee`Uv6jkY!|&|#or`zmS&9)!ON;n?<<|Gu=myW+TZ5U97{JQY6#4BX4o@v
z%ZHV(SF@hq;cRth`jN*=B|jhfw|jbakA<q>an)M~bPbLdUtub_b6|P$@uNSwH8m`o
z0xbI?m#tj2XhNkW|Cd$8uhy*$kW}W@-q2*bNov>J(~^>FjP9(E+&b%E;ws1G47|ZV
z=M^Ve6-qlA9p=l?*<p5Lm(^#Fb)T0^tm1DmFy5rhvH7-+*rV<Pnu`uSoX+Pc`1hxi
zQ?s=Q=bojvTC4-tlu0xniQdZ4A}iTBN9>}-HYc0T#WL3`6=zS`YSYu85g}A$a@_aB
z;Z;jctzL6#<(;MymdU}N|Mwj0kj^aPdU(r}U2)yrgWVT{Pa7nzd;GEI-K*mzLXNj5
z%iU_0)61}ZHqo{tgu_yBzE1R%haXPgW7c0ef$zD)nVSjv&Yb2omFtVN4<EU)E0$;f
zo7Z+#k6k~!-j}Rtz`1(gKc-`;oc#=02HsibwYx>5PfX>t>Ik^(k>BF`%lVX)Q<utx
zrk$@n45plAV{xx-lx3MRhgC;t(WJRcjr@vgcIhfK&q?vS7~%K)PLpBBxvMkJ9sbaB
zwUhgeOhbuENAJoRXBInfw<yWJ_FI|NQ+>rjhjG)_C&vx6j8vk9&pRBv+TrN)#`5<Z
zNA1i0f3_IiuUNhN$6^Lqf!!}QwlEx0eDc3<&n|8g?+eOD>{&KTc`CUza#*)Xtg(E(
z`h$d1YEMU4N5gW5rk}zW7gs56YhJkigTV19E#C<Z={;u@FZnL!I>mTq3(FOg-3Js!
zr_Q~_IH`D#myv;~LM^wV?AFya+<~t>S1T+&GNJ2gB9GdoJ!iP}G!uo`S6*U&ILX=P
z$a03s@x?BQ+Kaa@ak=a;&-`F_>gg_Xr^|jNT&91INY|Xu-7Hca<n!#NNX09WAA+Jk
zD)oaKeO4Fva!zn9>t50E>S*N5?f(*Iz0kIIaWyE{K6c?qcbRub;|br!4Xi9j`&z4g
zYj!%X3s@-r_DT(--_ht(<}IQ<uMaPOeCm0MUr@!>h|>R8pC9Q~spWci$B$|4@#tVC
zO^);XC%CT&vV6bAefr}`Y=WB}92CExA!%ULtCD=ehSN$cM*C0BCb5)*fv$WHHmqC9
zE%p3E*dB|msUnAFuG)3a;=0C{U8QsTGXzhr4dicj(0ud5j&sAr@QFuHX&nwSwaMUF
zdv2oV=|ayf#a-L8=UyrjlnmTG(?sUrV}pY?bq{CH<>o$TGlgMA%%YMbrvjx}6~5l+
zW4+lg8aW}9W0EYZ!{n8-v=_e<Idb#h+;h7R<i6UvCgDiFk?rj4$!j*~y<D*O&7^e}
zmagw^dasOFURN^j(Sg1DMfm2eJ$S^#&ie1p-z}@dOOF1UC|~#U0^@`Yr+ofjc~#7n
zvZ?i>^nzr`vmdpi|L@wK=yG;%)hu^kgQC=9Q4tckp$GC+4)1>9Yt_TGBYBHQ^eT-D
zOl2m=BXiIBHiRDfdSi}FWK%@Ul+rsZElkb3V#<^@85Ufcbz9=d4T<=_-8)t|$P_D0
z)Aq0WvPk#WM)_#l?<qloE;FxO;PrdsZ`ONv_Dp5lUDq;oPuwWwdSq~GIl~g!J&t=a
zG<46Lc*J|C)#3shuanxHE54?OzIODxblp?jGHYFQhcDN(^@*&dUsaj{u6L|x-&EKk
zc6W{d>&2g&HNV&f<{w$OXX?U^+uh%!uSlEP7?zw`8ntM-@g*hM$o{93`~KeVzZ#_=
z`{1bR{|Enj%~;+Y>CA{RQ{{-3J$O}iv6A#+jnYV`BM)x>-1BG?w_8e(k)U$XqzB6l
zZr!lH?Bnt<<MerRS1XIVv7dj<Tkd;w{_f!VRl$E|nuX7iKf!6z^y9*sw?|rj*hLkF
zMB7|pFn*+T?C7?qJ~flKtLAN3^LYDirzPzc0aJ`-|6P1r_gPZi#)+K=@0WgYetyRD
zsZR2d+N;NUV}dg798G;ZIi$n<+AWK+$5zw2PR@;4*^sb&!nt`<{LZ}aob=G|Y-mq#
z)A8~UCT+p<nx$O2(|YRP+&vc&xo^td<}HgqMIEeOc`fsFij~g6mAShLauZkIa9(#Y
z#OqPgo(&=8so{SQ1Q<X5za&f{?%J7tGns~On-v3MPYN1tVSMuN>s&65E56dQ7o|LP
z8(3}DR7?%n93gi%!oH?ELoX=P@6JEl$VsVpnC2y0*xkIN(%kuAviI8)h0E^v<s@G<
zoiu4;)3J=ml^^!Z_;kth*z{YMPy0N%`RZZhyO+J@UBNq3<d&x1dN1|#z@O!<XS#lQ
ziLQ>_*Q(blsoLRjSweS)=%&Ie>(^$iYYjQAV{jqm%2|`!zd9ay3uT^~V}7cIL)|wu
z)+@7S<HSOdm~~V9OzQlMr)Apc-JG2kv&J%g?cK~NXMN4<x&kL8ug*wV`Ju<Qjw_lq
zMSW|`q?31gE=s6xGW@V)lFPLO-fIWW%>AD-)mM^h+H;j^E_2ftzCA{JI@A|dd)(R@
zHcv2M-pW44hTBg=<5+d=-x*x^oN(`}!969O?C@)GZ~w+=sN8!W;P%nBM=Y!BTqajX
zp-i3O{f51_7j-pp98h&BHeGzF`#7KB=dFP&H>euMJd=HUxVP5NF7VECo}hoK+~!fa
zim7kssXXv`^Ng9zgyorVp4@}UdnVsL_cr%iZo#!(cl&a8cieIKmc<!&Nye&YL(hbp
zC->~w@an?njk_hzFK2kzDv(&Yjmz67apI0#b6KIPZC&ku&IgJe&DzXu(YZR*bKlRb
z9nsUc?%!%Us%zk4dc{T7_f@rm;xvuzd*7>HJDL!}{bj;`F*Uv&yt2|qZ!vvm-DrD<
z>AKF2SxyZcKW-n;Jw8M4&SBF#hvsI6m(DPaOSTq!m0kB((0rMYd!bPHM`88CV|RD4
zO?$lX+H}>g$xRYZN_^apcPu&+JV|iTD;Kt+gx);YIV+!(DW_EL&Di>6-@HvNM`Gu%
z4%<5I+L{}3_gwipr>qXzIbBzQl`l4P-p|$hKizA7;50#qzr#W2$gX{+zFw}(Tt+b_
zY;(>g%WY8=GHH4n?j*c+-({}fz6?F8$N!4nEclo9U-!!d-UkyxOI}5OaSOk^NG<)r
zv@e$PzIZMF;=R4ZV%rz*tB(VNJEWdp`rh+EsX6FFPM+pd(;CUW^GzqGrT*W$Vc+vx
zw@>@eJ8iOb-ExK(`&#B5_+lHko!cWb|EuSV*1TQI9WrLjdf(Boq$5h|kAQu;`)!t8
zJ`9gu?A~Peeu33Wg_l}kWr4{B={^i~uAu@;3woT-9kk6|sKy#*aHlV~ye$3kQ<*I0
zWx8|ZCfS5_oU~0}$oHx<fw}H~c~HdnJiZ@m87oTd3!AqeI?DT_vHs)yd!G((2wn28
zB<ASV<@-!k4NJLHI$YP~=__{R6-i8;>c2T?v5Nnv$?WgF%#OAl*|d4utrb&$iu1+I
zV!!wCn0U{_PmBjn6y41(_1@gGMP*^T@l#jhWjg~jzbsk5aC6*3gAbzjt9A&7X6?T=
zGk*Dm|MyRva{jR|e3!r~31-2_l8t+NT|*XsP`$|1D;wmL{@aQF!S?@Y_s@0Zt$TaR
z?!j*J2YbvH-)t+{qmXXF_UC?I^2#682lZ>rV%2NnJ09sR-mXx|W#-Nq$7AR6HIZrC
zl~OJP#mj3yr5}0!Zubu1k8zcES&#1a*k{qo`|!f$9}~N}T&Aw#Fun2qAtOUl>xFoG
z;iTWkL}PghGTzI_eX;vvrm*Vt)IY^tcWjPv9t~M}B4q9FvfY<b{|9SzZ!XxLQLd2M
zT6+E8o9F-D%73gkS2KKWwL9anM10{qzM8<(-@n##)uw;^p8g}ZuYS(Fey&v)?rHr0
zT+VegoX?)ELzkJY=jMN<f&~YfnYVdJUI;iPqA5JZC(!7?BZnSl^Sn6+3Ku)*$-7T-
z;V62@Fj4v>kHX=lJmS+*!?wv7Y<_ZPrcv&vJwGq1yU(|*=6ew|>B)r!?$hf`vPu?Q
zW}Itgr|UKKWsq*rmYgip17g=2-yWKz>%Hx)n;G+!qLkeiUo(W|KH$q>FLSWnu<g~u
z)X6CflNjrn`8_rmG721)=45s=5a3KY&?V{2W+cSW(8$4H;m2U<*`y-0WWv2oCru=S
zR^D8pcJ$ZQ*Vk4@&(`(Mdi?eda|!>Ix7FtF9_(#p{%`xI;=^t4sm$)@=2Ut<PCK3>
zW%2BBX&;yHq^JK1wy^Z^vinH}?ECh=tzTx_;wKtGs=<EJPx+VFAFvDi!yMW7<h`A$
z=eL9I@-;CJo=>U!=4*PPDb>UxNR!#c;zCPGYS|Q)zRa{44pEX%rs@`4WeV?L-eQ!b
z8=q_P$X%VUL`hp?Q&N(y@h*?Wy{3<DEb1%S>pH1ld(xCgflFG#r6-9OEqUm$NLA5Q
zL@v+qm|9BDAui2`ZBD1A#TNZsIz2jT<(zP{Ni$RPjro?S&B|K$R3>^|uT$5Yl2xwH
z8H)QhU7W}Kb*B3K#$S;_(zRk)&*x;HjZ|OUu2mJiklE7eW!`pOsm!_kMPHuJSKatL
zvy3ZK@q)x+A(l%O?7iXh+@&=yzG4ubV;)e&arQr#!lbvoA;+ee9ewiJjg@bSu<!4W
zCT}*L<k~72aAs0S=sNGvP|+G?-`!Ir{bx3&ut;aBI<DW=bM)el(%g_d&$mBbOx>lt
z^|sQU_tTDaMSbPFz_wRyRi}Q`q)S&%#g^r!blWy@KD+6_nXw`wQATs8;A9@<1j&p=
zo`$nBqkASrWvxAN)Jt2j`3{$~!sXLXPs*)cx@lj|)6)}Y8!i`p>v3K%-T3U0t2M^w
zt$+U!Jm=$hJiY9OtgGzG<#U(FI&sNvWnE)wY<r<<YhI!$J9B4=fVfiOqDggMJ$)zD
zt4_837Wz4Iw^_)EZF?^^$%jgqd(M<hyAfZW|K&Qv-M%9MS&ydumsm2vWnP6vcl&G$
zfo|pnI~G0C*ZY_M$c3rWGPd`4>Y^R2v$dz`u92~vkZNiFbBRcFhNJ4-*jYVFJ#we{
zjAq36#ogYocI%z!``yJ$Vx)H_S|)!unz&PXT9)ekrP;>Zv(z=KUukB1Y52A>X3@lb
zoGTX|*!fg!!EJu4x*rd$|EDduUhXyNa!PAe>_u1pgjKINr<%(}3inKp`~AN6?f(B6
zhbGPCxe&?n<mAnoO~RXWyft%Ip9-i+nv&+Jt5vxAn!k$5*0zj91_^0b?Kg+Sw@jR%
z@<NIuYTtLkI^8_;lm*__Sr0Au@+^^Cd*FVu*yoA+KNOfmU0}%yG2SF_O{vX|=YJ%}
z#1%(W+j0ucX5D@B;i;i!-I-+Rz{@?armol7cQS60R^qxQGn0EJ%X^%E-@LKJ*=^Rz
zeub-&C#3%`Tm7xJ#WvtIM`cmi>fR;kwyz^O#J648x~|dC;kzOy+gYXg8){W1#y&Wr
zEL}N;T~#$wx-+0P>+q6E8fuI@+Mch%j&to5;L~2Xc;b_|lpSY(Y80+{BdC2aVzJ22
zM=jZtoYHh2&p6Pb=CyFiq-hBziv$FP6OQRTGY~XiEb%nMH@s$<@#4;@cb0By_4#>y
zX55u!W>Y_wpT8*gWkSk|H@%aqC+!Ncd|Kl7Zewt8-b}$Id$#ZMwbhSe=e+8%&Uw|h
zk^;r=zdPLiTYXwJaSnTPr%Mn+|01CoGkVn=7B0R$>8U)c)$3f}l`jGpw2Jm7r7k>d
zbU5VHhP4?Va~1xo_`cfpX{zwl)k}MnukTFKnkHs;ap}8BnkMd#M7J&ph%f#*aTkNx
zVL_oaqER0fI+{&WRS}!m;U+RYxO>O+6;^&8`dig@XWep9nU%5nhtb9G=#N*cZ))g;
zncO_}>F6f?cVCqAf)sBxeY2i4=VFi5;@7d$o5P=YtnIxf=_xaH*?B9yx53gAJ=RQX
zIy~u3pyJDev#ve+A;`$OdP1V*?2QwSTwL1nRxS3!n%hPz9ap~e^;CU&BJ;Q9{<R*7
zJlieR9=SYGx*@r)Xw?->cdwuS7jO7*ZdUl#WjhM2)<rU%(L1WTtdO<*&SRt2wv$r?
za}&2Hr|T6LUtOzrtLe(^xA*dV5A0T&wMRQ{pIm$JRqqgs)6@5U&h)SB`n71mf$MuG
z9uTd)WS#$G6?=riG5?UV8V|X}-%P$XY^?vM`Ji#Gs3S`q%VwX232f81Ca^Ed%_(7?
z;Gu9r$GBr!w7jHj#kQcOw|kCuSZvifd*=A%-Y?yvDFu^bxu#9uQ<HkEdS_wWye#p?
ziy!nt_ZIdR@=OyEop{1<&Pw0><9Ah`e{;RfI&<HHq)j`%bgO7>5uX;gCw%v$z;$1*
z7@dE0>e#Csrn<ruc`t=sCQ}rqsdT&gE3?j=VE?~q;zSuv>3WNa51hMlRmz?jFL>)b
zUr*wT=KQUk>M~0*X2(cOE|9*h&MwsGQEkBW_O_D<n-c34#y;;2TkiyVRW3Gq8j^PD
zYOZPE+Y}}K1LFFw;fA$u9#3NFQQj;!qswFFB7wJ2g%h2h_IbRun0ZiV`O$B0^Cs$A
zX1(&8eSPb@XC6)Wyac;~cW3Hb-nLwslb&_^*0OEi-@IC(KRJJ1V&G?|Kf5Ld#Y!Du
zDxK>5?Z%BOy}WnX6%RW~aOuRWN~jrq)Cl{wMyJQ=N0dj_g0w=_sf`U+)Rfnn2Roe+
z?REUAeb`q#CTiA4wa4F-x0Sr^{?w{(cHrqzKQ)OswuJt@JT3p9YOgQU{Snk3d%8y7
zaQn}mkB2@<-miJ4q_ZhB|J28$tfkuvGL<6YMZz@Kb>0lix%ups_x;bFzCD$1mtEM_
zBf2_YRy%n{`k!yrTFX{S-+fl-u{zInnyr!VJ&u#N+^*E6Z+*&hK1Dms*C_ks&U6v&
zxib>fPkp$#Uw*X~my&1uQ!}AGxkvJt4RqaYGAHq^EmIJx2ollT|MJqBb^a%NN^|7h
zZ?C%a!tLywWs`P$SJf8g=_E*UudrAD-GBLB%kxQFk5=!wB;P7mm{z%M_L6J2`{YwA
z4h!D<c(#4-&Nlw4OTQm|YQ4PoMSt}#`IYa#pQ*n8d^P_WH{~~%<czc#FMC~9{eSY7
zWA@>>-*+5Q{TsMpy;#qKJ%{&C5;%79p82c4v4u0##qZTGOO4WZb<qELA+zU3#9Z%*
zk{6l80%LU+?3^R8>%_U)8zzJY2=1?F%{m}(s`8xMht?j>nY~YCIGJvqoBM!+Npdys
z&55=NlMaS>KI!8zlZ^VnHziPzEAe7rOr-eR`13bJD<@8vk$Y3%8rz>9S(#M!nTEXO
z1?<5NzO$!ZNal68wjjQ8io1u4*e#u)1-eJ)-IPpwd2{~Pn}I4Fi&*c4ELix!^@?h*
zSfs(3)*ipZH7!gU$0n*Yu%z5_+9T;{!O564Z}Nn;_MTHyLQbk};rpE9S;%sH<x7vA
z1Kmn$w|D(tb4z(wROrEZJC|^%?Ys3VGf6#d(S#hOcUxoDadNI-DfB+$+={}MKAzbd
zU8Qy6{8p((o61JdI^tSk=sM>|Pvzd})>9|z=iRnqRkn0(|Iu*y*NoX)C;BJYCfdpN
zuFPO}abS_FY`I-{WX+kb4O3d3{&;!qi1)s8+o|{QhCf~VueLN79&vdn>$*`<$@A=A
z*`tRek209d@o2qb<8&)i_L|#^h4*t<7~9+&)DAlErO3HCJd5S=`Z>X=&ihqHQ~R7_
zD{`8|dc2&qR5BBIvaj-&*0!*pnsI999FL|49&IX~O%L4G-0@sB=iJXZXJh9)J(99Z
zK-PB?x9`Wk@QpKc+2l9>KlteMg8r$jsksHQ6Ep5!o#`4Ys<?CN(V3O9UMr+ME=+X!
zkQjO5E`!fP<7F*g4Zcc-mn6)l+*_)A@8|KfsV<RB94x*^7aF!^sK&LX2u+?6;(7Gx
zy&e|Eq6-xPOWAXeJe6%r@R4HlyY-Y&&GAuUB3l}h!m=cxCGtkg)Lj`;6dKYsd>qu&
zl7Ah)%qp(UHrMB0SbE)&^r}Zby=t119%<YQdl~Adb?&sWVDy$V3I4mTmOo^hec<|q
z60a|{Ek<6_^Pi*@&ycQO%39@n{>zlxpLHJJXo;=nx%MPbc@4*mP`9vOGmrjCoKebj
z!F*G^?~B>Zx;I)BUH`SJw@S)zO14J*mzz{+71MI&$p3}D89Qb^oyf9O_UPRZpNu=p
zYoDpty<1-QO}+l!@^@2wdzPwri}f^$X?XJ;Y1}8%RHxFUm2rCM1CCs^iCYvh>tsF}
zN<K7*=AAUPe}`H?+RUFn6fXQxnACGIaGFM!rh3B06`nrxl3x1XnH$U5$?_|6X_#Mn
zokURGlgN2Tk1KLUiED*yRk;*$;&^Le<HFWyY?e|jF+!6|js<3TuxOk-6mvq}jx+ul
z7vsOD>0h2^mTK}#`b_pXmK+fElELkejN8is4w+V-)s=Ttm&vI0vBqD?^r>7l@7%pP
z$L_h`nYH%s4J|&|r&p46eyL14^2}teXQZ#}q8psH{{<gg9u+rGdc35M-{{tvxhY;%
z6aAN%$rl(sHU1asx#9Lom6qZr7PlM*dA~-dGhH`NP88FUVe5QccO$DpsQVtPjCGiV
z`<$h&FXe-C#Hy`gJSyYszCCt-q1ZX;s9oQY%#Fu9N)jS0G+N@an<lleuX}Pp;Od!`
z9&c)w&9RD&Rgl%a<hsA(>b?&uy8C?gRZP@-#uF%%x|uCeLh|V@*|WP?r|mtMA@Z+(
z`GLFFGR~T@U0QQVcEQxv10S{eXQf6k+zb9Q->ontdYfYFg0R>k7Y3Q5n<r_eR85l6
zlXGuzkJn?puW{_8liTEqOX`is)X%CPS@F{0M(;75mS~FyY5PRt|1UXlZkf#ax`fcL
zYcA9&sq3}s&P{u{^x68O%g(UGpF5EDYT*Q7#{R3O{P|w-`O~8F_?CZt5Z<-z=<*`@
zy3%-~NiM;sR{egW{pZePdqbAhYb1N5t{t2-VMEE{QZMy&Tjo2<ww4RI-WQA6e9Aqf
z$K}qJ<<mZ<%eCd~oist@L;Kq+tfw7cG!`9slD2wlR9>@PhQ~F5zNc#*Z&}cJP$55&
zZ^D^`_C>DU8xQT%S-0oW>aJ-oq*7&H7qOJikStAT6>bW;by0)!;9P~LFX~zH4)JA9
zP+Bu#lHz^a*6TI_i&*bPi5<N!c`JO{k!YhHS+~qHRTu26jvXpsNeo%}q;^J4A4~H8
zk~fcPC#ANmPg%C|aDuDzgW08mCtI#fcysYR=UM#=_wxTfd--n?%cWHwr+a5<{X6lJ
zQ##n<?Td4Z7A8%8qEK))XI0JIi(682m-QXVV@qDL?RrqmhCI3FL4Le%pS=#caq;HL
znRlM%tyvWDe}%H_+{NoI-S?fL{8!Pt{MRbp_wgH#E&d*<e_vzWv_Q7tDenX4uyN?}
zZLNCX5wlJxd$s$r6wjiT?Ph*E*yidSo8l`s*Kg@+vD->vds)R^KIq_54G?<Be@VM*
z=Eleww{xEb&v#Ij$#pU2lkiv~`?K+4iJH)zT3$)H_bE?%BA)S7PJ7yF+sbGE@@O5)
zV_na0lUk+}y=VA;;pu5HHl~tpetkC<4>gCrBPzdBBW!qrH8>es+F$vci>-KY{-4g@
zMMbAB=4<_4qy0Gl&0dAIj33^eRoWR;czK`FSqI&Aw|3_ET=w55uT|w%WyTx6z8)lV
z>RY6LenI@N1}%f>6W-4Ax|riw5@qbd$?{)Ari*utdGTlSEZ+JfESB?=z5cAdFLadU
ztbyCvcZUo+LM7H&AAa+wOec9mzk|p}X1^Cg>W0F<jkj58cogY9fAqeqVcmQ?UEN~~
z{NfUhu6(C2eCkZg#@rKnw_3C+OC_)E{Cs`pO@<>MyI-B^<~kxWA--f5>+RMrcdpI+
zD5fvIZ;Q9G&V9lBV1r5SjuC$UeU3)&dU(_-N;BboP{pRSDM!^*d+x@3O8(}nH2sLZ
zlbYv}^+_V7o*Z92diqY<CY`#gdBJbRnYBi7KbSb0O2byqi75Zt@=mjjNlSNf5Ci*%
zb*UL!wkA1Ec5!LH<+OO-X4$ObEx`uix7p*fKDXxD*sd~|&Qhe5exu4Da?iw~dAT0O
zEbedge$R1#_bQ`2<l34%<%SlPKgU?oPZwDJo1O7_cEzV$>wex1MlyDCtF5=2ah_c2
z;Bi2@mFLt{7yINq2VsqEY8yTJj?CTnB){^RPgm4VJ=RN|qTN@wG95oEHkH+NaiHY=
z?AK*;S&ffs{Fu6QgW<|aE;m%qawIQyzp_#8|K7RF1>PLYdur_`c7uUKemzUA%#AOn
zwjBPKmAfWU)8fQpNh5#bleYFo$J2CXvQ<1$+!R0Wjf~WfSzb|1=O4N)(fs|ZJdI&%
zJ$qYXz>kbK$Bir(*miu?+;4PY)28s^ukE{w3glg*9^cZ)%D8k+EW2!p$<zxxUT;5K
zoEnt1w%7P`p51(d*A|s`PcE?doTsB~^tf%pd|ge8ZOO_fmw#Y4-M6;#fS1?w`&^fK
zYHs9;9n<wnQ2+h6>)tkb%l>aS*KFSSr!aGRK-IjT*1NaOVy-xT>ZylGOHO*slQcaC
z^TN5odX0K3gq;gJs`KWwt(!AxS)u#d{G&U>t-oA;S!&e3an^tBrHkuzmo(pe8?>2Y
zedN1&NA}MviCEI!>*w1Tu`RVS^`nyhglR5|=I&yd>hi#FR>+f*pf_9m%{3joOvC;x
zuIK+6SM*6Htz$`Pka}r;v-DSGb%z$=-Sy5}9|!(Ov@liI-F-A>V*i8nTPAOf4E`OF
zU$Hg$cUbcywOOnSCUNZ0FY!N?;-dZC^seb9*3Ow0!X+z$7tfe(RhhN!jQF*GN+zdw
z7e1>_a5p}ibMUvo6z3QhIaZrHQPa}oZPG4hRhO9kX?whuk9*y#Xq!ptisF~HE-^cM
z;q#ks+fRN|oZWqVwv=J>GqaPAx1GFi*0N}c<MZ`VzjsDv$cU9sy>zzb(#)fm|8L|G
zPdr+@Ur%cNPQyhPukVeInB*%RUS(&v?b3=bv)kKyk`_mXoL+k%RykMoYQM|GX$w`(
zn#gZ2KNnK-@Da;_yW9Pu-!yc;X>#vbH(eyu{P!%Yw%OiielQu{;V(_FXnXwh?2m6B
zD)t1#2zcsVxzy2Z(ZO9epWp66SN4BB)_00;doB1L`V_8I%u-xw5t|U6cjV-`*Sih>
zKdVgifBwE=xmWaa=AV1tS5LUkGwJZgSK^m$_Q-4Wwp@8XKQYqMVspEB`M1)fvQ>}g
zTfdW^_1osd<OyC|@8l|P%)4#*>G94da{nK=zgv4$<ptZX7g^I^9rQL0mAw4$!QK|v
zXW~IY|4MuR@6Y=n&wJ^*p3td@vtoOGX8sED*y~a7t4t!tDYNWDw^jPx<32W*d^v9Y
zdvEW+^TH&-VP+FEw@`@1f+>gExRrH$HUvC8!YC*ZpfY2k0z;F8c9zV6iHnXh%DHBl
zSO%$hPE_Dj6YO+J6ygjLnY8GJU&>y_Z#9Bzo}X5~u*jue$#mA0=NFbaYF?8%5S-<;
zdYj2Mr3~-a4C|tIYw=i@F=#i1vR^TpU6dQSI_%`W*o<wWnwxVuCn+6Gn-aH=>-W2<
z)-xVyOl)KS<7?xvJn29`r;?n)Ors}<+I79vd<xEpbwwW&_~KKs<(MYR&H$giH?uOY
zb9t{1^ZmHx*u@l^>2eMKs~uilO|APMmT_Uhu}5nn%L7-;QTWVwZen+g<lJ8gGp}xZ
z9CKsURNd+b*IU6F^Avv0e|&Mee|_9vr;m1f56LUcPv0B$cEh6D?y&z(^H~1h*3{X!
zO*!L0>#c~@yj_o!Iz-LZG)++8oOoGN|Bb<w6Nh*#FQ3`OZ7Y+|bIQd<XUjo9l?>s#
zK}i!A+`2a_VS}!jLvQnxu*Xi@WCJgEOqNdn9JsBufoGEN#J(?qi}faj@LBZ=2x%-e
znxUP%AZSgB#_9GTqnB1)8IJ;%=%>zl;XP@eO2#4s_5~+Zj75u#oAht0d{%DF+vKzQ
z-;>wDiuU4Jug-6ICLN;T^z^g;ItI-{r~U`7=Q7zaZQT#$FFM;qc4-G2246ZA%p_aV
zZ5YOq{ASY}lULeC2h8T22|fI)X{tzMW~u1alWDc4`VqNjr-qA~-f>Y$G*>QTNM3Dy
zG2*S*s)-7cxxpD8e$makyRH|#&Sf|s^f)Y9Z`X`zN{@u9CPk@Sv<<&odNbCe>GmCN
z|9d=VyTkWXbK8bos5;<Xa>-kN?JC*75~^<wu(7MWKESd&X{qQQv&9=?d3~>*d=&k3
zSH<Bl-?=jmi-=1l3s1CMea+Qxq3nUk)`l}-$8}4tY&4&FQ*`=P70$yq-HR&UE#aBV
zexOppZ;j3`g{Dn5QHqmdlsuSTdN`fbT4p-wfYNGbsR#dGF1_EB$uN0|o2F9lo_&5y
zecRH?Ca;fj)C%b8>zciC=k!(5D{Ea}^&5Opi0yd4o+m}%3Zs<IYTb_KLYyD{*!AZ)
z_KDxpHaKo$w`OCj4bR1EXFZ?TY(5{nR4gTPZBZH5^(74#A76J--uqQ9@%pyXeE)YJ
zx8LZgK395Id-;hw_Zs=<Eh^_!YF1zS;p&M!Pp>EMv;DSU{q#lOd1OWGe}C8<qPN#n
zD`NFy&L?>vKDx#hTt0R^>8WF&=AwW{XU)X!X`SFaT~mC<kh^yKIn(W4DPoc44Vo+b
z8cxg%>)o-Tb(&zN;--VK`EJg}oLrjjqC8Q%)-ahII&n<lf@RQ@CzDJb{r|z^9p2R$
zb16hvWK)~U<((eW_n9^5&usD4)Vw_B-I7DXeEEt;eoe1j!nC4>H?Sk$`ida8sLR{Z
zeyjdR(hm=@yZV1^uNJYC`7PGI(4SRL=b82W)&oBjPb(gC+IT2*?U^5|jed3Yrl<t2
z`{FK`qIo2ubjf2`^<~ntGgFvWZWil*@_kn>@22Qmr>?!>ndu=XI*H3$sI%ysgomf!
zrr7A)ERwr|H%#&9kG(W`-uxe;^VNU3Yb}jYY7k=;|K!?LkQ~{qy>*kqaa|Ut_uO%R
z^h!2vzo1_y{VM)L9p{>fg4!l3!V{gGgdMUASGk$)y4V}Ud^YD`j*080uKWzO-P!N*
zR!o|cAXM1#%y_Bf@pNb5M9Xfmr7Ru3ipkP)?n(uw-A6X7UMjqGRjg@S<+cgg`XQG~
z3sx-afAD!8t84>Pn9*|Gj7FX%8mZ+XOrk2Uvh&(E&Av13knK+o*U7Vg-VQw$baVon
zpJJA!x7fxR3z-e|LNqgd5)OIn|K)nTuJDQci+9(pQ>W!bFAnZ_#i1dU#v3$Yb@Vc`
z@*}gI>;h(Q^!5xeJ#_G)&(&l`Mh3;7EUY37%nUjV3=9knOb(0;9RC^EIb=LQ2c>Xu
z?~;13>g)j{7DYY{#SK1^MyXzsD#jC%or2iSRc$=lLbR({Utg6{O;cE1x|(f+lt6Ks
zPm?k0GnveSyAJc2YPtSa7Th@y$lTNAWtH`E$1+cL3)zM#%R}cmu&3NzwKY3nQ9S1=
z;jJMukt<V9Zu6O4wpPP1<?JF`?eYeTmCVz*VpjQuUYKq0{}Ruht_=@otUk8ZDeKCE
z;K0Rl{j=Vb7zH1j#u~OXOZ1~v*er`{LO0G$vyEPB6L<Hd)xyKg3haBP>2A3Crg0nl
zxqY^q%MXUFb$EB>rBKD}pkt<d4_-{1>~NLSI(DvK?UEx+d2A`=kF$O|nQxBf{Li9)
z;oPCy%t!j~%!~V7@iz6ALrdI5%OfAQT-|>C|16Q&flG3>9Svk|+`=iDViD5TD7V!(
zAahN_Hep?JL6?P&TX?1<wiy>%FzC)-`$@<*w(~`>kK4i%GXrP84ZRR#@JRG=k5Q4^
zB8QINpoRH1r&KO?sy2B{p0HZLGo_O`G(xt~<7MjONdaz}s*Z+1k&lBsQg_}+3RF~i
zlrp>L+XIH>TffKW?0RedbXJa@W?Vr*(-F_&D}{j*=7yhKu26cDGv$~=y<Ev$mvtK9
z8D-pCBV)^#?kS6P<&QLR$jMp5DKsl1F7%S;wkyUB&2QJJuvs%tc$M70_NSR@xU=u|
z8(n3t|Bp(fN4kAowJ31Yt*Zf%E1RcVtU7Sw+J+-yw?!h4Py3p^hUxCr%MAPeu?qV<
z64~ap^-5Iq+9bDwUseU`#JNpPot&C^GkEK^jH?CndyU#wGG9%e<uKLG++i>K!)VQY
zFF3EXdA^lAazFni@AFBHf900Q7rfk>7`=GOl#ojd=XgJ49A&nAKBq+D@hih6Y#Z4(
z$=2>toG8n@@K0Rj^}W}Y?|=6*dsB@pZ^Ls}(Za&Hbyp`gE!(9MnPaHA)J12)jF&eL
z@n!6I_T^^3zM$nJs~~RG(yldERz=3e?2dRkb#=nYr&{KnFBZ3^Gpnjx&5EvFG%<hw
z+i;&^=a*0J#@5$_w$=#$-y(TI|Jh&j#^~!^OM4@Q&)xKywSg(?((;Xm*ffp*rf;}6
zLt83cy0|VU{rFW?osu)mGZX_Cgxk*gxhU^LOq0v{>uc^XoVsW(w9keyF2FN)K}nSM
z+-pu=-b){SDOtH!xAxm73tQ=F;TMX(+BmL%^fw}8s^!YxyRu(t*?zgmXA%~9bYH7j
zOxD@Z#jzz1pFaBi?!A<@&%U75SGB%YyM6n6*UnNjjX^|wSIW{IEgG2y`Zt?)JxS$X
zqR0|IeX{2=HP+1&zjUYX$&8q#6)~Yo@{9_<^|>ayAcfnUEG2<KcV`8(Tdm&Xqg3u8
zSzYEV-DJUHtM4u1d?&c;%@^-diH&y;{r?m^;iiwC1GD4=2bWFOw+f!br_AVV_Gp&k
zJm~Greq>^(4U1CPj1%4ZjL98`GL&ZZc)0pLlIp!yu~cg5N4Mox61S|rEaVSd>ru`V
zbf=9)MZx!FLEGZ1uh>?)nXg;ln{;_%pVVP4w_Qhaq87HQ9W#)*^25pY;6ksZ70V2q
zsy_Li=;W2OayRs!<X58QbK*_KBIjF@OQtRh>5!IUHF9+HDhZgSP-<2xRoS>oY>hy(
zzsFLWkfVX2XJ+%Qe{y}*>W0%{0XyYVmKjZrK3e3v@6*-TcS>CeQ{T@nmJD0-p>0BH
zUFyusyXFUQ9X`{tCoAmyoT)aux2y{4U066TsI&Rhm;Vm0eDaEzF8k(0O`Kfss@cS>
z%BS6vJiGm@dXn4e8#MtlHa4#LH0x^O`O|S}6YRgf<g>DoWZyXXRLjcaDNc*e*W}C)
z@H=!Y{c7bR2R&26C5-1Cd%e{Pd4vwG*y*coe>7QQqg#maGLOR93nx0B;56GCqT$RT
z7!+W3WShskLeb4D<aKLAh1aE(GSmrrM67MvZhkDkKu-E~g!YzeOmZx1{iWvIc;+>A
z&DAGros&B!?eX9)S5ytzYIxS`$Yj&lq$v~fjXLMND7n!+X<e%H9xv|7Ll+xE!W1@K
zkuLu=<&;O@hPLlqvm9TgM(kH~Q4)A7JM$6inQ3p9sIfYxWlu{v7yEzX=9!U|>ett)
zOBOn`8r?X3`_;bi=to6s_0lp=|C}~^-?LY#A}4a^JJcMV-n?M)%<X6Ma(%T0?zVMr
z-*78#Ql5y!Pu<)}!7>eAY!A9MUmb6;K6uwdkNtwut2DJ1+00ynNSQ|sAKI2mC^{OX
zItfp^q#$zkvU5)4>2pEf6!br9WGFsz{$7*belw8&Bu|pu??cOZWg=J34G}(atKpiW
zvx#BX`=`bS7=4#rpE+YWThpwt45n>yS3JeGI=OS(uI)B_sod#Ta%{zdiIL%jlcr9q
z5?E@)ZCEC9qHR&nac<6=`Z2DPHeYjD9GLN_pkdFmX`DsV*YPbm6KnW-61U~z4S`Sp
zt3@3W>~nWA)z4~5fAny9zt4%2a<bd3W-H$+TKCg(p=P?Z?UladSW8)U|MZLO7aykH
zydt~WgGEWQu%jkFeS){=N|hXk&OIK%++`V#d$??u8Ex$L%U*gpI`-$Zv}HG^L|y)M
zZJSzn$gwla!u-8=ujAYHciWt4$t%~Z<~e?2=$Jlz^M<9@LOi|y9PmoNz3%g^w|Cw}
zv;K^Y*qe3yl*;3k!AVjdb8gN5_wtDcN7U@bMVd`#rIVchw`TfVw~I6jNzHR>f6(j@
z<<)nrFI9fcu9HG3i{-BIdM$rEVbauxseQXHFUV|`^V?-&JH`9^WS`SV1m5$h@P*9k
zHJ@xM$C&D3|KH0i-|q9w#fu`lqkN`IZr%2A;(@QvuHN*ypTKvVi!ry}R{nj-B(14F
z2Tr_*KX)Xq=bWU$?yJex{8r~5c8JSfy!tAr(D+*Fq@_oCm5wWMS-1YsDl7Y9>f~X3
zR^Mpm$>uQjdwJnAk~G~O+21w@-psk<uHo+oQ#tRvXNmM!>MORplfy-M#y$@>1KSD3
z--~@$OpTZqDeV*ZB%<$hbG75I?S4lOxVYcjB=$mB`PUJJqhHRh>e4v2Vfr+aqhFs@
z*KSE%|7(>^YgO0l`0XD5E3<js>i5f^d0Fc{xllt=|D@Z$>C^Yt7-l#robjvsb%$NF
zS>d}{mtfEPt0(`K*y(HE>DB%pyvthha{s)X{qw}Mr=^uuT{!gj%L-Q8szCl<{AI_s
z-dL`iGD(=(d|&6(%QnWQQS6S+Gt!gR3OX^01xg&|<bEV5I8{K%(3&$$;QzLSfTn_m
z>*f8ejhxTR%|qm7nieQ#MD~Q2vm~lpeQspD-1I2XMb}Kg$ArN+!kFukpi!Ijia_qk
zUsOv&MOD9-KE5FBtRbv3DV&uh)k}k`<6w*5iWU{7RJMny%nLZ<%38x_w1#VV6l_)V
zmM}~B(UK(5mf#_r<WUl*#udM<ZBau>){nNl7j0oDT20#nn-}Zln6&w<@aPUKay`WN
z<pAHOaQ-hB_**0R|0)O=n=y3W==k?PfL}03;Ozr`jYPHit_IcavW1^ymX!F$8ZcZc
zbYlL{$s8`AbtF)Hg}K%eXXPDg+!whv9}yNy6qMT-)c3vK=A&`x55bfL^-HeteP^-V
z{787ywOD3Gz34-_OUxx-Y?65Dq<7vU`{J_{KcU()?K!g=dUX$0c^u3M_GmW>w|HQo
zvokE`8Aoosv%ocu+Vj(U-@d2`<>-I+qUID^ZElNzOS#CB=atcu>h@gsD>~L*<6-t9
zT;jDm_gnS)tqX*@BLw~h$o{>+Jz+^lN|Th4YeRo~r*4L*c)I}0(YynKPV>~f)LSz7
zG)z(h3^g?ac`FhYWHja;s$i=~TIM0$b37>E|D^JXrkSE`a_biw={Z}?GcD|5j|!Yr
z_;r({sYmmKBKbuvBEQWUey4LUOtRbEQOtVLJh8ay@|LN75!qoK*@>R56F%02Bu-;*
zoR)PmHgbpQti);25v>>0L{lQCpAVl_QaQbBW@+-x=_NaJz0C!@(nU{)x9Ux5+npg`
z&6XEt)E2*7IVn7~=>%U(M#tX^6PO<IcUmyanmKd!&6#sN1!i{&^q-J?IW;}d)6(jw
z@T}v(Rf{9$Gz<KjZn?yw%V4G877-(XZRN@{oLN_N@&qQbI0~+tQX%0ey}e^%S5wqv
zC$>s6so&G<4|15S5f9%ME;`eY;m!{6yB59$AFEDHwJZ4FnDfkd9`nOq=7p@x3+BDq
zIgfqeJSN8ZEE9W~8mFCgvw3D*n?JGlx8=M)p8bC%tHUf8yohZ5u#z)sK?%pMc^tDi
zqc$w~5HUYslkJvv)no?Ar%Af3KPA(qPx!JRj_u=wnLAY$&lF<6IHB_f_umg4Q&I$;
znyJbND&{l<s7;fzVJ_#MnzbWQTDV1%<zXRvp|R=74CambEQZ!AJtn`uSkSOdZs`nx
z+D(bSS`E0=3N)4m{Yf+AbaHVMiBxG3@cOct`&&c3tAMoy*Qc2&m5RkVR_&Q5+TQ+X
z%Z_NxdQnr_Rr9fV-rJS)&R?G=bSgDwv)IfHVyDAOk`hXqyH>Q{YU}?0sxLod#mC^@
z_b=ytnz^hwv!7#OTgj@{<eyXhS9sL8nl^dxPmS#8(GXbtbEc}5!0N75idF(jQnS`q
ziFW)bWlfaY=ir#zF0i&TVz!5uXoc^B;MwiRmnh$u-FK~vxp2+IZEI#uP82oM*f6=G
z|I*US6Xz&<$_g^uR86aA7GS-6Eu}qWYU`?;XHo0EdfL3rnDH%hM*S(_w_Ve?W=&(>
zxbEAjb?>?as!uK8ytSV1)`EY(7X00{;G?BLWPt#?w*aTMz|0LBm>(>7{c}CPSO4c9
z^N*TU$^5E&zkI^i_L&>1R?V~!T6=2MoSXcell|{y@ypH>n7wS4&Bcl0%jLMY2Jm0z
z;{GooC}$C<t=aXrZ4!rImwu*zXO^gOr1y@HCQk7t<C$Ip-xGGQ$O|7Awt6h6eR`tZ
zCwb0`N~;nB1UGZ8RZt85xiw?e+{sV+z8}f)He~ZT-dDYFTDw=j*Yp+q)hmjnw=)ZH
z@>g%K?B33PdOLgJN+!m2mm9Y$3vxzn*ig$oziswBro<hQ3t0QOcebx)4K&y}arcf%
zvo~~A?_jkSXqwIXK#FyN_s+%HJNT-0^;Wf3ORVCE%vxGCQ}Nf#4YO7$du>`5wP|zJ
zZdS$Jmjzd|FwWZjn*Y?|<~OQp`AiD(!BIg%GOo>*Rqg^uw7S{Fy5oK=nK{G2=I3m&
zBeIi{^xM=0j{H9<lVY$%=XNIRr;-g1=d4zp`s)bSgI&|u82A6YweJ0@1)Nz6zG-i)
z=w8rvdOzo@4O6Wd*jLYMI=zEQk(Di4fX`ZpS9&A+?*mLS2L=8dm^guzH~OIXpAC@>
z2c>fkO0PLsQN6O_^ua~9H>l<uWXj(7R9#7S)y8F9i*~=B5N^0haka)Q%LtZ-niDq%
zZ%h>xzOq{Sr<7r}jG1@2;N?z>Y=NI?nPH8FA{l`$ryHeKXlypjwCbFE)jN`Ld%pII
zS!!WhnOa@EO@sti$k}aXol~;R`_gu1#~oF-cd$P^#u>PB@~xdc+VeSj4m3p{U_E%e
z@%P#WnO*a|cQNVg5~?||VD^c&|279XC$a`SI59zI*Q7ZM@};LGXw3YSK2u@!?%s-3
zb9;8PZJep9b!w5Ptg`pvvhN*{j~ym2S9?A!Nn&f~X~8w7I*pz^OLkexPEHd&HS2Wn
zquCoI1TB&cwK5D{Jq12()emp9o@dr@dE;8q&ic0o!hsGO8oYOYlst5kXJ`58d7?Qx
zMEA^7K694IXNP=_K%UO=phw38gAO#w><avFuE9qj;KSNL2G;L$&VQ6Sf8S=q<IaU_
zFBfuq*6;nWk-zf7re7Pk{z+%utETm9!d?xv;47M6g7(eXCY<oc!BTOvl1Ss?kih0|
z7bAD6?eJXE&9=oMdZ`YFvE59ya35*WZKv05obC6Yt0MfAu*Aiq(lfo+JdnS=cx&^q
ztvL_2Zk@0?SW-)2A`8dH17$wP*$fW|_MCm4dE!RQ&V|uegA`8G$DDs>BjEAi>XpFr
zOp4c-9j>wZT>B!!>M`MbcJKK-*^^VePfq(&T7P3#`-<I$IRbLhs~6n7zUtM?ZLfDP
z@;Sv`cxvnH3kPnms;w2Mi9L1M@l^i~p?h9ZDU((!2+EwQp7Q*)nwsNj?@dRPePtQN
z_o}X4vc&kP(c05BPM7PKpYHgsA=9WX^08r4;MS9!0yF1!3kP@3pH#nAfmQIqyt+46
z-bo&h^_6{Nb3S$N@kX5;8NLD!r0%5Z-VG37{E;JYHTHZwFRQ?Wy8{0&T)VsX{2!U~
z!q%R3Te;qB-}rgD|LR?<INt8&+Iw=p^!;_WZtTpy-?8_C9P|Bkuec|8Xmv+OY3-hM
znNikSQO1=!|5#(b`y4|JiLLD4_AD>0Fz|dR7WYtW$=(H1Z`m|DU$ef<G10)JMUG$n
z;o&omKNiXVnxwDbSiEeiDo2oGdXL<SL~i4cSG^}KQ)XP}_4jyV_W|YwSDfUYxY%7g
zS^MO2Ba7gMYd(CeKI^Xive`9tcB}ZUcGp8(GyfdoaNJ$-)^LT)sfN2xcYJzU8T%~o
z(EWfQ{(l0UX;J+DFL2Fnsr+fG>d+(IwRQ77Ymdz{o7f!#&oK(t`&s*VSx?;>*mw1G
z`2WTixkYQ%CL2vUB=bNzNvKh9XJ_|m8=+l4&g_<*%KYGVl)#-&GUv;31Wxka<=l9D
zk=!f64d<`zeZrD>H=&p1zU}Kxaj)0L-TnRM;QwE97nD!<czgoO-jnZnPjOtlaOL8S
zHrx9tbvNep1RnDfIBzF#QBUBq9mCaqYUVavYd3UQ?WzB=P-^=L!Ba68ZO^PWJF@uO
zr;D<|<uZ<I=K0G054)uK*6YdX0&k-i#=U#Bgda(2Oi?!6YB2etZtIlWlQpBQgL5}<
z?)tKI%?4HrJE2dOcUb>?@RDV9uzT|5jBv`|t1|pg{r^4bw>cSH=Rf&j`_gGoRd;p7
z>^pf<@BX&zMaqWB8UOh{r^VgaD1GC!ozMa6-QUlC<~#ZH$KKD0e4Co)&YWQ3xb^|}
z9$D!hOU;v*%FE{^RG&M-B)E60%2sco+W~Lnl>R-OVSF^=&Xy3brBlC2X}Qi}c3?X9
zXN&9=<N0hte?+tvF5*0>dv#IX`P`gWflAi|46fbJz5DR&-5)Zn*L3g6>TG;w%NBpU
zj_c=ZgP7f`v=(Wf>d4}IwnFRYAKRZht?%2;7qHBKt7ZQ=>E2Jyvp<b$ZY=7zbo9SJ
zeLvTti!Xn%Kes8&GdBFXgmF^rt_OU2d#?l<#tN+IZ_4LrJni1~@cX(;(iTf2c^<9p
z-R!}=bsPT&Yu)(bd0VCk_xWfFTwN*k=zpNlk^+TK%S@6#e3&=;fUoS1H#^UNK65Rg
z@4Q0&`5?LLkq)0$|87^Wzn<{#q@KUW{P~>>5AzQkVEp>$s`8A0hfZz0vUV~T_>~+R
zg>~aXDkfZ85yg3%cSCx2f_BFmwh1nsg$<r<Ydv<#9MDvhoN6jQtx}^{VLGEt=&2)-
zSqZZlUGDM}GzTw~UdR{gXjAp^<)M|XoI+b)zRHlC<K?ZPb8sn-<R)e>!PMkphg*#~
z!jpom8+cYmDGKtfxVrAzG7jcNKYBlz7aZE(!YuFBW8pnTXQov4MHRJAD-0KhtxY=G
zC${(O)>mIwsc&3l-?)^WL&W#vmsgF;#J`<du}kUjP455M)_eXMY(LT=QaO9x+x7Du
z&M)ELwuya{QvQT}QOxBhY~Gc;lK9QL-n{MZfj-vUo6g=7mfdcB!Z|C2`Fo1@<TW1K
zGOxCY`2Mm=Hrf)uS5eY0HH9nJOK}HNOozkYHm6go7H?3}&{{2&WZ={$Gp9p?P5hRY
zX3vpLuMX|-aLgC!Fq7=CTm396S$NhQ<rT8gQHso+Et?cF40R>CAGt{vKMCa4J$4~Q
zBrr^}NF*$5Ve8bW8y|BNUe0Kqr0|ksm-o^Kb+ad#sz+z9IMunQh2u=?rXG(4RY!bQ
z$f@!Qb%;c#8J|$7<_qfZ(LA{&*!*hF=^%@?4LQ1}E`KVyB(Kc=YsG)j=$IT;-AQ^?
zUzDavRB^7;{if`(@@0+G0`9hqB#+BGP8?Xp^OG%!>kudZl0K)ceV4Qp{SPv2T^=QA
zvR-%g73rz#CJ3BeEf%D7!cbUuQ-cq8vhw96n@;KN)-jHnvF5n(jU7H77L7HMdP<I`
z?`+!s;M*Istu1HXip3PH-_5)24#zGtD~-U}9os8}cWs*=*Y<h2XxP0qo3~fhY;1o}
zKPk9`VWNj}MMM84Ro>e-Ud_F+s8Ds@+uNP@&Tf+rbMg6XJ+VYK<I*dixXcA%QJT%+
zJC7)@;GUZ%dtiT~OK3;%>|I$L4zsi6R){QRND;ltCi}@iF??#BK*nsHBLbRpq=mZw
zJ5Rl?T&df1r|7klfM=q!%LGoPz?@iR*#vGQk5})Gx<*c1w5h+}r|4wWi<hl(z2|jz
z<?k;&>=)gv(pAh*Rh6A_N=v-_gu<HLA}8iq<W_oK4C8pWT~u7~SBI$B<3peQHQoPP
z7uq_?NUFxm^<7cfBI>pB$*h>BmzR9<_n*w4v%D_bYR&^8QT4@re!m>TdA6)rwdPKS
zvgW#mbyIuR@2lGRGElrp`*+;L&8dZn$A9?vyeRv;Ir{mr-s|V`-#(dP_WgCu^gq_W
zIrod5dvP*{;mog<uEM%}?lQT*rs(B!vYY(=)M}q|a7jn&YI)8zNB4_F^oLD(WX<`<
z)jH~7U*n&{|K(Y-Stm;r>38^a#)&IVaq_U3Aj<YZanHl~6D^(_Ot{e``Z&-ds%WV|
zK<fE;&O#CGOU&{=UDlUee#laPX3+$OmCw_89OGiT*iG4Gk2(f8PpF&rO@Q~KUdVz&
zk4j<=ac$_$Ra(4&Gw{+1C6`_m{k0cZW=UPLvt{K<u9mvu`e%u1-_ih8j?T+kMurS2
z4_~-N3Aoz33C?(@kYdrOJZIaNuoZ{bI+<LTY>;qsw6DLBocdoP*s<@kV0FVL(ejr|
zyx#?Sh28p4>b`|@u9ld=oi+N=Sw%OioS2LzEqJ52evZe|OJ>3|p1w=lv6<ss)>MIc
zqMNN80;L*ajxNykTQY0?|5bjUBaVscw>?cw(Ks9_^JrSCQ;Y7-H78?sEH_Bl$Xvs#
zBKzpuQ}vdb4GAYb`<(2KZ-4s6&FimD!oowP?xlhs=C3;Vgu$a??&}g3#()b;?%bF-
zGpd4zD=?vbn@4beW^9|7->c(RTM|w>Zd5XuEyHKKg`u@vE1;Q``|M7k9U4z291|0h
zVPIn~I-d~Q?aQ+E<k2lt)_y;`sl~OoulU3VgZ9!V)+U`HYt>9w%Kj6m?n|0^`V{BH
zKLSfuv%gYwys%Ov$j3M1TjwIzuIaloqn>nyzZ5XKo@E<SIZ;t2Nx)$93ezdBFS3~p
zZaa9G2pBt+DlBn$6Z64ng1GKVQ5nM*|EKd^@{BK8V$Hg;PbP7u+L?U;(KjSng|=*%
ztK~XLHqxQVv*mJsXvac#ODWcj2*y^OloE?Y5)B!-(cK4TxcE&eR-77ia#C~cCzF$k
zt=eYSH`H8HDbWzQoAz_svgRvEIi0WOdB~K`J07d6xUreztxl;=lAN%*$>Yc>*-Jj@
zLM`fcGjD~}Za=ataH9kNO0T+^o}wKZ{R;n%zW3bLB>XGDUom+ix8xDY>04%W*z7cU
z>=KYNT};>0db{I%7r6}o_#21#<m3*t+xyHupJ}1of8|<$ce#^@@Q2L{7Ru{hJIJ-S
zmx*cOixWFqg1dbb6ej*_@J-mVc}ZZ364$H`JrWWBrIt(od^-0{hm)F?*R1+IQ%sp0
z6*A*y9f;1kDZk&Mv%Bcj(vGd?`?D(bbe9{e3VWx_`>N2Kzw(Rhq#5nm&sy$!M>tKH
zcJjO3rO-~H#Tp`WOIn4EKED#Wm?^A%skGv7w#ocPH{b2+xRj+n6`vEh6Z>j~dJf;k
zSEsuoru;nMddBxJN6KT**_NVb#I_Z%u6cif%Ua;cRn9~~1y=K$7h2lnyrY5&*t^t@
z@#by1^srPnZ4cM)shx|@GU@cWEmTu+X%uCz6q-9hfvtIi6SoJ$%?BBbtQsAM<d^^D
z{$F(Emc|7qUjf@WbKX2$*2wo{`;r;K>cMUT!SA+AUhrnk2G(`*|D7%d9$mfdF}LeU
zt|^^Q#0?}Hs(hBo?Pc3sHls<kMmTAA;;B~d(<Vt5{iaS}&}_ZbAji?bzMyF8WEbtT
z!cmr|9hzp>vL1Zk^SV*2SFrRS*XF9s<E^1TPIRbDNZ7yOc>DW=ZtIMrqG=H)bZ7m0
zb^2Yl=W2=836q}AIN*}B<+82#H2IzSVs=Ky-t0KTFvFy*tmEK`9?1qVh10h*6cm|e
zMI3ce-Xwdm&&jCc%*K5=J&&u_JX$0f`eMPch8CSaQ6lbPWg;#cm*`|PiS_rt&3v=c
zQrGafJbN<hY$FdLG0lS1uL6>BH!gNxJQBQu&3tp0ugwLMW)r2)SHv$~PL)|)|35%s
zv4gax`>{z;Zv{H;xY`Li_s(Bb=#<X$@IdU-ps={7MQhmK#g?wBFj-tWUr~-}|5D*C
z_xGl4<eeGE5$Ap=^ZMJ{iv7QCEkC+zLg0a;fuC=>Bt0w=pQyvNVc!zH<_)c<noO+=
zTCHz%F?VyB&)7OuLb5r5UG}74k+5!{z@*a;mRRf*s$bA<{IXSYGf$tvwrihuIXS8L
zNww-UG_85P;(&uupVD@n7kqD4v}-glXjq7)O18eW;LSSSlpVc9V}-^UqYl?5rt;Sv
z8YiqYHY};T(8BmZUL%3^P+(t>#nJ}_bNoKB@4BJ+++yjwmkO<!tO?S_rp&vIT`XF;
zmv%c${-4L#ci`fIM#sHT0-d@iR9#Olzs|UR4da3LLdy<Dbwqs>ne%|t*Kz#}M>XG%
zi@rPUY}vs7VBz{t%9iiDk1X1$w(^I*c|mKCqXCmN$L!TJH*L^<{CmpUgK}FnOwAl@
z-)FYVmh?Ib@-^*b<!9cwteQFG;*l5|gW!h#3%}>6-t0MRut!5-rP0Odou{mn7qWX^
zYV!E8+p()fqoXC$db>u0M%GNRC<#7|=uHlZtsV^wfjgwHR4r#()t)b{o4u1eZ>5_?
zN5{=WE9#Th9v0@S;OQw9;EOuJ!jWJjm*W0JNRsPtcjLzPJqtKae_DC(b8DkvU-%mH
z4h6-klQS5yjCqr%{};Qxn8k{>^^0xO#}gAcI>U=unM<0(61oZ=sm+?fG)qGAVK)1$
z86rP-ZT@7X^OSq1TZ`%(OP4@Nfv|}Jp_XdlnOj%`IXo{f+Ev6XT-mWoP|oji|8b_R
zGZPjE1~2}bJ@KoBNb3dDE5aw4VjNZ!GyCpojXp7Di2*bBujxlVwwG>jn|6RN>ce~)
zM==?h)>m9TxtmqaRoH00VUJp|ZCA;hYZ`m%{w%NJSbOUgyYl6h7B9nFUK$TqO`H7a
zK)!Hms*H@>V%_qSd@3up$y9GYb7zv}MTOImQ}#qmd4FK>Ga;?Brxq_1nqs@DL*&q;
z^N+TkY|@NgaM)1B`TGhHGli+I9xnVpSA$vaL6fe7^rRnCzs;WFv2sVN!1_LI^_hn|
zobPb{_{3>cv9IO96z@Rkts7PcbXsgKVE(FbYGFo~mX-gz)h0eZmTdt$r)ZzQ>&YP`
z(EU@RYtHPx;zfNs6_|HR%xp|)4poq^Pv|nQXk8X<sa?foe8RPwVY8G~r=iS@6I>!u
z8;;$mT9Q9y&$$KdhC5aiGW1SewYKrWw118bM{V58bEcL4(8w2LWtp(4FS7NX6UW6}
z+;^h){J*Iz5p^P8y7{uj+UzazVHa4E3^~g>9XgD*y*bemYGE<|(c(+DmtUQ6skidP
z5yjm%wO3qBm~sB|-uNFz?4b(myeGr8Pfp5d-YhWd|MQ95OHcT{k?x4pWL)>4iMK-4
z>hi1>28W+XJ369gPVDCVBHgk6*XA#;rEhB<iD_E@C*a6-2GNBNME}HeXbRac`)Ic?
z=KLa`2@fT^mSi-)FIn%qd)D^?j>$8$SrnBIM41E$NbfMvF*>pTW%6=&&9iHktg}p%
z^iRBC<k7X*pxLN`nJt%fiS>*mU5S%Hngy>N)oqN*ID11K)?S>%acsq;W)5B1*v=ha
zxDQC55$sjzUuniD+S)C3$b-u%_sQfVElo#d_`)QbudlM%{)V^hHlNm?9&K04n?BCT
zCdXoDPqv8kxRbMH>O<e37W?y+?3V{dm`^=Fd*Mt*j&7p~P27P~{zrXaosr<Tyk)M<
zZi@#YUM@8p&T}~Y6eC+7SXkWE@yO{iF_rT0j+oQKy|$y@XW~2~1B2O%`wKn^1f7v8
zGIC{9?BHnT_$Ol5^r5G}Q2fYEgHxH3N=BXwHJF7HE(n~S_StfW)x-I6ms@IIvRCc6
zkd@dT_+h?}0CU=84r$%qhZ1|lLN(4sv>OJT$&_T6*5vx{A`8caO_fJ9s&@9>>|#)l
z<?VdaUjEcfYa#2J%{+odCk{Mad}GF?8=HF#f3DG$UD3BC#=zBT$!^vOHKq~PmwyP%
zG7aVWnHBQQE8?DE?1aBZjBYUKRfuT(6;(c@7?iW&_Z#CGf*~_h{l0gddbRlE|2Boy
zs*|oTH_kHPQeJRHz*A;F3y+~+t$@Go#D%ilD>D72F4)n*JUiHj!$irH#ZgCUmz4O#
zg%z*FBCO|y|89*gXx1}m)~jHS*vWeO=bRsjeHMbvxtq`Q?O0*>!D>%*tFhqH<K0bJ
zZ%=3KHggwJ?Xy~0cjL6|lKKA@2G-BGcxBe~9lsrG{+cpq%`y&le$F)|d9!aE_l-Mi
zS`F8TH2?MOt_WV9c=>r?m!3wmo&lr(%&8J>96u%nZ=BIIS=8C!$<jB~9M=VxP2PKQ
z_63ViK{wjJa_Y@!61sVgx3k+VlyjPHYsk$j5BG+AeIq@sd38jjw(FAf(*o}?U-W;w
zwc}@j;QvOOTQ{<7e<@zP#K5bk&ALLmVWnvRvkGTJQL@dhy~j_p<UVn@GHK5lhRLrb
z+~nq{m+d&HIY+G2$7$adJz2wP8E0B9d=?HAV0Q8Ah~ItP^p8UKMBdpK#MMuV*WI<%
zdumX9D8}e--&s?K^QN8`Ts_UUO0BkxO<}s{!r~wHvdQvkgJuRZ-=^)mDg<w8_RlZ1
zx7&RFMe07s)})ROyO*~+=XCB96`S=SB`v*e-U7{#{R>09Le6_HSuJq*isuxm2TNu>
zyknA}6#71De_(9Cp_cLiuBZi9r6x686}s6HwL_=1aAN<(gp)z>O$#0`C}`fYWzxYb
zO!t!>9Qd)~q^`)eDNg_IKhRZ8dEk7C$?$_!(v^fif~*P~d+OwNm|1JRNSex3W>znn
zcxCe(p=tcnEvDW{jd&<>PI;Q>)ul6~_GPw~a?A~jjX5{P>aEt3Ev(UZ^%#1lOrORn
z*Cy#-6Y=)W%eDnf_iFd4=Otx6%k1&EYi#Dv?Rnm<dznrf$CJwQ|BKH5liJ{u82nj_
z`^SlAZ5uY*@Gg5fgMSg5nCZigYteUJKbY1o&7eOaN_azL@Wg=284If=`$8KU%07$j
z_`@wFeW_n;`+*478wFvhKjTw{C#5HDJ1}KZwe0-WwYE386<#P9FG`Lv`n<PyOAHTJ
z@DgFki7F=}{DQv*aY(%ri8WK3a`x%}?g>c?9Ph@)-DsO}+2{t-<Cu<j2Nid)oxi=8
z->EgS{Q<+=hYWq%TV`FHms`d$^98?g-#bsyciX$9JZ0p9AIkpRlv=)w@0rN0pVB%J
zysNh!*k9+yDJ|97z&F{ii1pjsCS!wU^9P~f3f@N+-aj;Ftq#MTrWYN~uLJ{Us8=Zn
zZBym&TQ;fIRJPjNxp}e2YX#ZI36llbZdl4h%us7>J{x<F_tmVITvo10pQZ99Y<-y0
z^y)=l#G@%qk8}DywCv~-PU=|P-|rkTRpQ+28Lt{OKK*@QA-6AU^4GUPIUl<j7PEQ!
zPm@j0o5wfR`sz}PS-x+$m|u7L>$pT~e$=|&Z^U>0|H7m5wtCubJNP;(V{co5czwtH
z{nKul-*5>xusXj^+$U(7=R;nO?L21~6nLVY7c|+ncTE3uv!F3(y8kl|1KVBKx3!<W
z8xzr-Y7!bOI{(>(){gt%zPx8EZJKd;{S1a33Y-g=JKYmIS30HrEO@s+@?l08N96T$
z=BZg4T=!x>D(SRoe~?RNv9$E9?U>)Qdg+6!!s{iYuO05ad$_#s`P3QDH!@1zH`Q6g
zFxyw+aEpa+`P%UAA8W13?*BjL-0)jDUvME;xOeT*@aRJy%C38r^!IJyeHa$)QNJN#
zn%u|f3Zmh2Ib65Aywevvr>$do->ZiEU-av<?(EBK`d;2%%aLzo@V_ZMu~T=s>&xkH
zZ>?*6QTTO2Y{|a2Gd_0ear~U$mL=y`)U@O4vuPjiw<QH~?th!zp=&koV;M)`F@5bv
zU8!1(vlLwB_%0555ScQsBj#gl$A?)K%@>wAaVS?7ADi2|&#%u<gE`pYa%1PZ#_SlQ
z;EiHlyu}BVB~D$scj1!dBaiEGsb}YYIrlYY=GwQ}?_3^C`Jv2o^+DOkzqhjPvAt5i
z{(H~)SM3TvEblgGc8E0h#O}N%x^CgV)TVa<+dI_aiuNpjZQSn?H+@l3xPb2a@3X>Z
z?e^dQ^6HHp7o3kg5_M=Xu${_na_HN8E6(5VoGbpXwM>hT`zs#e<b3OO!8H}FPg@KA
z-z_gU{}ZmuQ`ngMvPt)&(Eo^4g8ehgD%Le57EC`c5$-Zy@5gjqsesf!Z6`8k-)`dF
zak#Z*e_MmS{PVZ<$6qSzf4U#{u=n?;?uzxo@}E-d)l%xSwfrxeY)$&Shhv7j<=Tkg
zR#pxpRW<>ch_*8fMjW>Va$bB;JklW$>8ax3rF5u$$%ze{QqPC&ZTp(loAmVj^uu+6
zp{u@_FNl>6TI=?!SaV~zYr9^w(3<6!67{Dsw|x72{8F0jZ04MKIloT7^sL@*cWd|8
zi(+BsYqxCEyt<VqqABH+z>`VE4%6E*IVZK=`1bJj_H_0Ww$bH!_x4nM|MWJx>5Yid
zT7_F#?Gt<t{cr2wG}768QBl1;joqa!V&|hL`<RXV&&@GkoVKx(-Nr6j>r0F|Z>Vre
z>_i>sW4wvy^G<BoID7YuOk)|J3)|l^9%yuyv1<_3G!u8Xm2vH=bZ!wzR=>}3=UT&|
zt!zDs|AN==tq)@V<Gj{f<>32Q?R>7U&flx9_k8=)D{B3|wP8QAw`QqM4)MuVFnJJ@
zTNv0D$$rZ<Ks<Gc>$NDpRaXPjvRzXWa|NbO5Npq{G`Wz*kyQ4ug~#uPVwd`>XNoS_
zxhoXgSUrz8y2MO5aFi|D#^i?k63+06^@}DRJlZfvaf)P%W#u&`7b~lu7baKmdS2_g
zXzP+NRaCKLVodB&GY04Xt9GhAV@TQcY=ZRindQ^8nRhITicV=%yXH7sbIS#W?x#D`
z{a9X=3NEP0EeY(`-|KQoi+AUeDc%~oyF{v4eg=gs;eV(yCGf52*39ac3!iugFX(!8
zK6vRQEzwZ9n~zSdirM-oRg-<I>w-v*NQuX@xHnDB?p%M7HQXhHY4_`Ok!cn=8%{`h
z>%^7ry?QjBBX5Dc<6EA9e)r~=F9n?XQblg2?rC#VZZ&=9x?p}y>F;dk<tsJyT;h*$
ztL@!%Hv7GE`nIrVn?&TU+@5vpubPY|>v>nU66VQ{T-uAbIL;~9w&C)IxYtX?1*=3?
zid^z{(~X)^`)P0ZeEqFI7RjIb`%^q&*Z<<KttpX(2bU_u<~M3wKP$hhG$$%8@y9gD
z!~f>Ian7E}pnu6z+jg2rspvHMq+9=fNNX0UElGBo+!=B0Tu$J`$*s}+4W@fss;=p8
zPnVC8p3byw_xe37d!wJtT6eTpf7aTq6VoD6c~}-@l-!Lxw%Rke<wlbD%cq+nmaLfd
z(PME>QILxxZ>nYBT%i=<+O9UPt0BRaDpq0JZhU+;J9;ZqnB>O3sfy{x-?eIYzqzGX
ze7)qa?X>&5Kb_k(D?{7X=Ht<3-fJu={9>2fyYjumuXY7r3)mqT>VM(u?3vnvXVP_V
z?tR;1{(8cfV*U^7ejdBtU6Qx+rR(QkN9USK{IwQ3(^mZ7(s6t0UWPm+TgNTI++iI}
z`mZ8IYJ@a4O}P8qZtIEjb-$Pu7#6N}a7h;b)8)1%`s*o0ZKjF62cEBIo_er0#Zo~0
z)`d1}fg_^j24U^bnig$aqS2PYD9G-6L_um3N4s2;STn<OXW41p*`lu!c3ODsp6B_*
zEp^7BEoC9|8GDpG{$?|)Uotg&tD4~<z<5?jHQ42?U<%iVOA|SA*_Y~`%F_?^nx9$p
zNwxOvX~pfFe%#WZ)YiA8nkas{9kIiKN$GEwV!LJcJcr31_Icb5wU2^JI9lgs%XWz*
zdz@l$Tk07!SF2=&mTXu|BKy_CquLu^E&g$7h235TzKGnP-dlb>oB4l#<5S<&nU^DC
zcDS40_|znqlkC@{YMd_X9kD+p=v`l@PWz1ykvo5#W?1%NLtDga<+T<n88U~Dh3@O`
zNq)j)adO3}_Kt!%hfa&;J}dRxWq7);A}9Vq*<ytT!SnX4x38(KxFK_sCoS=NPls;A
z!7kq)PK}=ro@b6eIG=G{PE+3L4yQfo;&+dDI`J9>HaApQ&wJVvxu7yux*&Zwv*Hq8
z-4x$BA|>7CQwtcjTRioOJ-BP?tL(XVOr*>`r(9-~XyHD&`M$f?rp3o6H0HQ%ogAyy
ztZ3ZEpB*Qn?0DkYgvIArvyaJ~owQ`v+4CaBRX>|NG#_l4weitw@8ToMamSvh8D0Ia
zW-O`L7-7WZUSZq6I(`c0whO`CWjqDkYb8=O_g``^dm7Li>-&t=_1D2Rj)}Tkwn%P{
zNEg`jMW|U$NAcB(BKDcPOm%cZIrJZ<TrycJy3yKJH_KFbt>%wVi`zF>y9-=q>uShU
zw(2@#AwOyF-hxBMPh}QgO4zm}@zh(z;;AR@^2`?BcePDt)z3@OlXgq3KHIwF)9;|a
zPs}&GHEXFm`SIedZQL3kKQ0#y>R6%Azgmay<AjdZ(CD{XN=JA;nWU-ato(jp(@nDr
z({*DWJmoe&d^-BdQ-^gsjwbMhbWLqLu~aS9fU7z*;?_-$oUD%vc{@2Ka$V0mRGt~Y
zZqnhy+xkQCzpBQ^b|)ES(efMnWhO8cekqm@b6LCX;~g!_y(_oAa%1Ve=Q1l;SXpQ1
zpKJX>rIORn9hX>NQyNj?((8Ho)b{NuOIX&wUO9KhnF+hJ)C?az{&Xqjli=QM%J!D4
zCMQk2AwF9#xNJ*{YT8fl)A4UVx^XHxZci!oio10%e@5@*cU~Lgg{FnMT}a97h}p4g
z!@DAe*{;#EnrGhl5j5*;<-)lijMk)iW$#FfnZUr1eQ-~dO`F$)Qy=^NXDbwjZB^QB
z8g8-sxsjITq}i8P!=&E`EPL=~SKrrDXV;$WTwv|gv%6UHTYuA(IW3kc_1|t^a+~Hp
z;q8;<qR|Q}iR~56^X_@w7EJoT%<-PJ(``{RZw@JgSc}W%ER9ENm;cN!lX;t9KI3R>
zk>&SBe-AM>?`5R|Y~0N!4|u<MV10m@<JPkD<(I{tPbphDb)oQi>Gy3j)`*IH^w&J&
zbX9j&>&g}PZf#ckd}WcD%j%uElfEB0t}E~SB>UOPW&NiFW^Y(!lP?gM?H41Q+Ww|y
z&9<K2AIF^99QAne`Hy?Jh+XTF+q><|S_|Vd%DW3vwT$)F&hgdt{O$e9N^nA=WcRE~
zUkom3ariA4Onr5#VV4QxjU~G*w`8$N-uiXT=h>&SMH<PWT-H-G!*;n;SoiDwIOzUF
z-?Azqk>~2-RvWcBPg*2*J}a7eGPu8*x6_9Ge}vzJGx=>X?zTUr7N0U*ze{KKbj|pT
zGruN8+l#OL6|~p-=hJ<&zRxZ^QReZN#cXTJe-F#8=jPi97Yn`$2sF^2nNs#;bF1!!
zdp6<It9nlqSygS8ZY@+y{C#Mb=+sSF3^n^*&t_yL%xX$Kn6LCO?e1~gvqfSLa<!_P
zqq4X+&6xkI&u)X>-SCJ*UM#udsTGGpTio8>U)DO&Wq|~f8B5pB>-@Y7-5YP5&pF6u
zA=ID4o*8icm}}_7$dl%o=L1s}*0V&u?7i2)aQT*p|LqIj;Xh{0?3sG)rcC@78JCc2
z35L@gj)=xiVVdE}@+jm8Ul+Gz#PJ!4>^WPR#95AA`^z=e^nbYY&zVzS`qkb&aeqR<
zeG9?8Ek{?kgvvGEy|ePzsx5wHA9xqvbX;?1e)?0lxjQcG-grOFM&5s`2=~_6x_^&K
zpLyt+Dz?F9=DM6K>yCKbc(X8*^TCy?eoG2(t(Q<xe8t-ShgFh=ZL3OLnFL$xgRqkm
z6|&aeddlnYELBcC=5|_)v__s>|J9CTxoj3suZ8YCTgdBQ<Qnd(a{b<tq`I5_es|(r
zB--m0j`It#lw7!65!=FAHeJ_aLFA3tI2-x15%aH2n6f6IQ<`JW_uSS8J3JR%yb>>a
z<JE*l?Hke&bNCwH&er}g{r6V?eLtfuCJ8ie<ZIa2SG2WNFE^~=@AcY+CqwW4zn$~v
z{(r;Ehb-?zJ_zZY!C?PNz=2I6o@dDeNfyZ*hMtG>RyBBv33i4}y?8_Bp-a@g7rGC{
zCniWW&eJmN>)R3Yf^}Y+hM$iCTZX|+?l#%@2QsF=X65Z*)Nc(b`NNy+tE_nUqTa0I
z{6^B-7hFBG#qn5(cl`v%%N;@5Pk7GUko2DzaD5}6c$@N#txLq$#m;^+@zz{EiCauk
z^JFDg-Q6FX>{)hyip0_{TRBtj9+axO?`$(Ga?c@YjcL*|9P<?J$miXV>k@hR^y%S;
zKFvQuG!xqM6S{4`2!Bg)d-701Tt@BN8`d8wY*LzO>X$mw6MWU3rn<bj##W=$7s__#
z?8%qB|L0|Mc)y-IwRmMhHU~e~T`61H<GSzU-o3r|P{PN>pnY>JOTiJBi>#?fCNO<o
z>($}YCaT8vZRu=jgCqCu$~T<7{bS>mhP@{vwjA=>$-8>bxp231$s;0Myff}y6uIFz
zqpV3eTZ6;qkjkRF;;~*+Vzp$}?MP&t7vZ9_pl61=<N<ZZ3+b*m)ZgBC-tn}hPVDSE
z9*4^YPZ<6*<xiO7JY(YV&UsB@7v~)D(@bOYkm5g+kg#f@%nRF~yBmb>Zdo8VNp$vs
zlbnA~@!sp{SaAIg%Upi7W1rV#tmE;L=1KJUFg1}y<9_4A)jw`VHM)8lUjO3x_{-FX
z;z})H)0|u)Q$0(UO3M9z8T9Gq`Xh;w_ZZ7KG`ZJal`)!?QMw}T8LR6`aiypW(_SV2
z-x|5GW5x`rq~&upm+s_{RN*gpbCtoQKRi`ad@Ad;3BIX$foV=G>@h3mMJ2hNi*mXW
zm35T2#Pvv|@AQd`;T;nWu6=ng`&nzg!I@1@S^j+pp7+Op!M;p}ziql7)Wp|tH3_nm
z3;C5gJw4EPw0>&xs(n)|yOc}jEw;RLyP@&u>NbzRSEGHl3N)Qe{Qc&v`^4z=D|~so
zPH%Yj*xv8hMFE#hi!!J82{IbJRxo<ttfSo>@Vdn{yY<)6MhC%M&c!~iDXtYs6|ugY
zY1&n<W=wNA-nUR}l43aL!j!XHxMef{E1q8{S~PL$j5$IZGI}rgc1|<Aq~*!)FSW{5
zr{`m>%S_&e+4IEzRd#+qdFAHPPVt$3lJm}`O*pW6)9rgRm(HHB@DA_FjvWI2oVR{I
zebKh%pd7Ez!wlBb--NUjl^s7R&J2pJa-6^ZgyuOrnJrTec28`yW_dQdY*nfi%gzrg
zzn8QnKXXYdOPa^$a9M9fUg*7~ZF3gIb-IM`{+!cqw)RmRi*yvHw|13A^osah$9nc&
zdh{pc$kDI!4!jfAS335=kY}&RT-%(iBR)@7&3L+M+Qa=b=Qpf;wT5rDRNvyJPWj&_
zvntO$z47XH?b4HGQg>QS!kCLLG=6w|p+lwS>Er4DxVCEj_hpKc+V<^vUDE5A8_Qo<
zJ*YNI*!A~Hc3g01V$v?N4Am|<jz>pV9n^a>Z*^M4tw;-<3eCvLxf*-3H1^%_eZNV^
znyqi@(MfY%W-DeMQ#|X+|BZE?37_PY$b~LR(jIe;%6VwNnLc5j|HOL-U&SalO*wGv
z((QPk_?Ze)u@@E;%{}VSa_yF9l<%?StWWu7KTJJ)Z;cRF+|&>2Jbvk4N;r{~`b;Hs
zvi6h9-`|K_nx%Alrt9T2HCF3f7aQF;k+<LGxbE1}y5QNGi%+@hOug%OK8q|_?c(60
z=eO!=*Rc<RKGIKu=P@k4`^sHY=f2s!rV9&a@3(FFv6Hhgm+jOs<=6>pjeq{v+#+;w
z*Pf#~hSSbJm>^tw&|%jJzLVl-B$hZQ6n?jP9hXq(@X2+vU$(P^ikNZQnI_FFiQd+l
z^SuR)A}6|ejr%6PX${Vdd*hHBcW0eT9$R*00=N3Aol3EvzqZ~EX}Q)Z`)kG24|iti
zu5?Sjsm$l+d*?)$Uv0q~)>j9uTq<)J*=zED=N|nzl_9h(EL!4Y><Xd(xAb>=Bu-)p
z6!_WewpVdZ<Ay3u1Ks;Z+PbPM0w>R&a5T}*;OSGBe8;Je7rvG)kLycSyP05cDZrt%
zNn1f|x^ux>?M0yr<2GGC#d<SLeD~LNe=S!1Rhj<YwB<`jf9@$3OKz=GIR`~17$u*1
zJfrFVI?idQcf~zv3Yp)$=LAEZhh4gey)$>aOz*bF=!X}VOMAWh@xxW({&FQRmrb+Y
z%`%v1GdVMAj$YT!6OA4RC*3%-+P3p%$`R(*g;kFt)@_*Px%o5OjFNfJ-Z{IxIQ;Rv
z{>qcGckjIF%=JE|&@r#bDIwD3XWn#67r$4o%762AojUE7pf7kTL)t^ZRJ{Lu$gHUw
z{p3wMG%w`m9gqyO4tt-lRoH6b7N&#;xp^~YxyQb0jd-8u%>F3i`1`2t)903bN${B@
zC>40sYi*j$=3}ZH>e&m_%vg-)A7v{@ajoI~;d1wXSj$h7(n~7h=W>hd4718z%RYK!
zEq%m#z(wodl_u6x|4ZvO6gT%y`ToaO=lArvfgeAA4fHyx7<DWD*H)3;j@`WsUwDsh
z%gFD@y=>;d{N!`zZH5xPe{~)XzqVa^GJ&&Up8Yqo8IwL1roR?nY#6dpy}m8aks(kr
z=O7Ei1Z{`nof&aAAANl@>*IxK6P?^H%Q;^aUO4NmlISIoO@Z6pe(j0-wMQjxhfXw)
zZ0u(5%q>hw3k;Mxxw+1@2FmN?Xy$0Ad}q0!vpFN={fl!=k6(oCF3Jv2)s9xY|57k^
zMq_9Yr|+#Y-&<xa2QHlxJJQ5oP`tWdH$AhH|6b&FVP(IADLa~V4+OfjU0xmejQMwF
z(zK(OydK0=%;{(UCUN!0&NFP9KL3xcUO6f0zPGRX!s}fhc5=n8OA(K&2zTbmnR@uk
z6%ij%#w(_>KPJ9eEPPhz+s1Qgf77JhBKEC3X0DYyQ@%rUkBH-v`FdL)DqZ`@`_p*d
z_szNQH!r)pzH{mg$M7Q;C$tu1pJ2>7_S~+;b$MQ`mcsfe7Tvw?!e1vdF<tD{SmG&q
zt~`6HJAdc$=dES!ZC=(bA`BUo#d9vr*}=_xI`hK)+bJ1xo=0ZC@jdHs`Gs!&**hoq
zE6F<Q1o<<4^?&af8@XvJi$%Z6*Ern|rRv>ROSg(}x$Lz~p5riib9KOln62|O!&Xn;
zbn9d9i9?fg%D29Iqq8IZtD=|0uEi^P&KhUk{3O2o|Bf@e0{^eF+OBwlWy&Elu3S0x
zO|kPH%@O;&>LN#PNCE5P*5gyHgp!ZBZQ8Qrq|ygIt@smjHkOD@aB<mQroE~0GgtUG
z=O@A?F)pd!C;wN7t`ktrcpuQR^!Bv{4+}nJtqQvD+R6EAr)>0+_=nSSG<OHMF{aoZ
z-<Egy+r0Z_7nr{|yeP|;FP9A0mFzdXv{|U&)cecME|-)qZ_?X)#l59Sd}5H)BB|B>
zdAdpw8jfx|^jxdl+$v(0FTSX-_}F)q43ocG5BATyz5F}F;x@x3&l{@WikF^Wn6={i
zrSq{S7M((!pWmIF{^h|#={n~R<?Elj-u{&2CUkrO%i2SIE;A<{V)^jj@A^$GYcn@b
z*NU?dOBYXNVxQ6|dB7-w=g1VT6VfL(HBPlmTli3Js^`qf)!SEA?>eue_uh_Uy+yS8
z>2qrR+B=x{=H5&8yZU(MQG@e&hS@0>wbU-g6~rn0VVUyFuzyROhFWRlC)YV`3l^QP
z+8y-g!WUyPkESKT2VNXaIKq<47=JU(QZs^m`s7BIJ?HdJillNHn&-9NpQ!IJMYH(y
z)B}v`?i6SqPHk{%Wxm5R!D7J~+0GaxwUWoL{4_htdGbmgKl7C|kM;6!xtYq~JzY0?
zTEa@BXJ=*^gid>*5U75>(^mZ#&#TKvE=+M|Ut%LwqJC+a^Zb83T~oBqcDl0rOqKZh
zf5qA5X3e)mqNl#{UXpy)?~diS*8x*xy)@=*S~z8GSjw^qQ+K3l#P6&A{YunkO5(#d
z=6`HcjS@aKa5_%qIWfDUAocWg)k%_GT}c<rgzdfm$W$G8-m%!*zbC^|=}^#oUbd??
z8+lyyX5HQ*b+*eydey#(O!8L0&4bQ06|$?kwr#eJ{;F~<E4_OPOOR5u?3UnFJ=)gH
zC#KwrXghO^@sZ%9mvS4+j_=s&Jtb1>Vz1`sPnNqT?T}j-x_t(#u=mR8VUslXonu(D
zW*f`Ir~(c4=B5eTPODs5?X*cRMZkHLk<RKhKXy!5wEMipfgb(CdP+TtonAa$-MSKw
z61s&XZzV8DGyXrLz5bY7a=}r?6%0j-6m+9E9zJX0Gr@%6?uuTEgJP2mtd~U0%s9ok
zj%Sru=e9LnB4HjM-|jfIYL3wv-%WE)tTS^dIV#r3R<yL&=*<QHg}Yd^o=&^|Wr@bQ
z!@rC<m$Wvm%3m@$D(lmRze+Bzj&Gjg#kF+)jtQD881Fl;4D#ijBci?9&NOjZfcxH6
ztK28f;=j6N#jNwoLUm;uLOSh^YOaV{d1&iW|Hq#Kt_8*#J#?RTZIP)?)bUL|4!6~A
zG+CWo@b{zpDK)P%w^YQ>ep|F>)6tEF$0ocATrb~wWZFS>b(?j)H*bCL*kfobm9>>C
zTv>Zd&yL9f`_8Ccdb90%&}Uui`_ul<@z}8-D)H%qg<^IUXHHFvS!Wr&OnSkzuoaun
zO=}YQ)Mwqf!?|B+;+M~QnthMf_V4QmbDf|wTgK+o)6RfN`S&LZ8Z|GU#XDtvT5%&^
z)Qg3keg<i5{eE40b_(R|Ox-V4W%#Ny>dB&`jO+X^9+TR-)yKVHxxjT-k(Ewc@2EuU
z-MVv2Mkm5_X6%}?xd)&58SD<TsIF-6-LoZVk-mFmgwpP)iAiR5oL(;$SPM9H<WBU7
z3JA29R6P^+pUwH|!lkD?ymidaR3tB%U?F9zG}Fp`?*Y%oS^jTXwlZmHZ(UL4wfc0a
z?X@eXnQQLaC*`tsr`>DZ-pj0c>w<K27UPs}2K&tZ|M>O#sMO<WI?n>P-ur&g<g#_<
z#Yef<ji<XVUUhHPChjYqAGGEdG(2^%HupR@)m_W-rdr#%M=!lHxAFc}IW+Ojlao9f
ze+K3TsP5Su@>HXsIKGCDW#X~9{;FvON2T99mii^qqQtr>^1Q<j+xway>_c~6;MAOQ
zh;8Ku$9Kx}xdV4No%UG4y>;dRX9g4HyuBV-H?uCrdx;#8<_%P`@Vd~qz;*8ej|8sF
z;wy=#L{*e53#-?!4wn`wJ?gYpzxc^Y>Bvhe6SZSkddKd%J>9~X>-X2!wag!UXa2j^
z=6hyYQggQJ!)*)XzIse(O8xw}h2!h)zgGm>EgMgG{0Y{a>Y~{oZS#LVlkz9Fwt^#e
z9cwy$Tv|^~?cma!Gv#Wi$cvK!h8dcBlSKVjRIC)$b)BMj$@^Wx%TpqWc@^_ruIbi?
zo{nN%T|Qx#hvck?88-_QbI%-}_H0>!(h-v<*^?e#6Iu0L_r#x#U7uESsjD@ez2c)%
zvno^GWt*oMLz8hs-U;!RKdyp80WQWV3e9<5#>)&fb2MVPx*qVDO))qXtiNNUdNJFB
zIWMm~PAZ#uK;(pTQ=hM&qj2SwrliLUW;$F*GYR(6mbk9{^A2Z=#hJ+-F%lx(4}8+u
z7jn27hAzljH)W|qcjCpkT@KElTohtD;<Q#DVwBCzb*uilP%yW|bdJ)|s_uz#{r{IJ
zFdp}DS$X=8pjq^^>6;_hWpkt`N7@AK|0U{czd@2KGvRPRq|mWO4&feK*2#09<a+j|
zyo}4*bXX>==h0jbMVZS{780ynOFXwS-jyy2Tj*uEM3Gxb$K}Se<&7J3yb6QnNIQHw
z+ds8rqP9?M;%)^k<CJaOv(GNk^^BOx_vBp6#@%-la>7N|%t+n3{nXX480$S(te)B*
z&GOz<lX|gpQ=aNv#T3bH1<tEf+@ewvI=-c8N*tDOTORH4ta$PCBMTz<cRq}I<=8H=
zb{Z2$?3|Uo?kXbdKNozSt(L1&yW$d;`3iAAr<?xO(ud}h*KO4Lu_xW`b@!BQ-!At)
zO!3p6qx^qjna6Rxi`tBZYkuv$bwmAN-;$2<FU)bL4!)c-sk@;1#|e#(8gCuM?@afQ
zn>w*;*7?H%Yht`M2wgsXKxfHq-Sw?r9yWZr?=!Y?zFiv88f5WS!ms&h!_`CEW-Od^
z)Xl8vUSdne5hXdnD36JUlGPM^mztHXJmvc?;NsL9NBmmmzmr;DvHiR2hD*m){7XcX
zxy_D!@nc!bne^J_huTDo`_qjpgn45(2Onp=aa1h&2E(qp%BMSKu(xyl^w}qI{kqZP
zo<e2T<*rIe?2~sq65E&=^;}|RMk~*f24;<f=M1Y?aC;r$=@E`iG5@LUyJ$zq%B)ho
zT+Y|0CVaTMWG$C|`zG(0|3yZdwG?%oyI8XWFWp*r>{e)0-K58Eg%y_rrQSGM8L;yC
zBrtQ$XkbxG$j$avVSFU@?CU&nkw;?IJJr3^j8!)?YWh69^JW@{39GQ=@e{MPyedLY
zFS~VNA>%BqS4<`HkK3nIWp1<=iAi59;x+H4k7kmPU|(&!af6niyK&l)^+i8!vsI<F
zi7;$CI!|}o^RgAvc08URsuiL%r~Lg<|K;b8CHsRl?5dWCmcBYC8|669Na2>5kR~Tf
z<*`YD@+rzzn;-1)xOUuV%9mq$Wxp<m?|H<g)yg9x>QVDxiF>P1u$z@_#uU{B4V@ja
zN-6SPOd<+T4o_H{dW)w^aPIaervk$N?{b{R6SUGNwe!W_yGK;ix{q~DS!ow?-=lE%
zW#7w>w#Ob*OuZ|2??B0hze+ESE4=UhJ`u1qily~@T^NVtvlM|tUF;LN6~8#<HM)9<
zFX55*_o&`+xvg)(b_Gep%E@LynT;xaiqo!4k#q{TS`vNr$!yi~RZ&ylL>W!cjIhiN
zEVvqVgY(&zEzNJg{#V{}@sL=>-S?iWS}#RUPuSHa9ebj>MkhDZU*Y!KaEBX5^S&`I
z*!w1N{{MS3fBjqfsoLyHQ&d7w%iDSGf0wMTQd_i3rRkoUQ_d=#{|ovuvJO|<IPW=-
zlgGf9cqa4sfyWve=Qd1oe!ODQ*$%cxzdU1qEqy#^vD|+RH>)MhJT56-4yV2dDAgKW
zur6BnjEm`uLt5b<mL&@5&u=8ZFw1E+TWx)>;fh1L^Od$6(|9vQ-nXV?99v}I$dU51
zM1B1iCrvvJzYXWo)}4F5OK-=aG%ghf&Nu7dyz9TM)^ty4F)#Z8Ug3f__X-$<68MxK
zf6!9Sl`wE$pn77#8UFce7?<ZT{NTIO#<lZPZts_;`4dvuBVRc&<eAQD*_UxxvB<gS
zQdY*5HKO0LKm9oR@CJMS6|W?=1+o_W$xYonX3LB<)|%ZpV&>3ufn%*%UgKPcoab7-
zN*qZ_7D-A@Ywg{h6|Q-8)hF|`Myg|$#x<9t3I*KP^H{IPET7EceEq+JYIqj!l#-XB
z5-MA~G}RN*B&V-yt2=$XA?KDC%c(Q#_=FqYJn`q#T`YUj>HV&^W=DAMyeicAq*Qh0
z?456iR`3Prt2x|b3Q}U)r102)acURi#_koHUNc^Qadg6?PYO=4L2Wv(Qs+b^$MN|n
z@%4qe`0w*d_ByuWtb>5}FYiY!$z~7SADvRSkoahHqru$o!sC#DdW~bOw-1QjV75Jz
zX>(<<h)&Y;rcENRQvBMK><iR5*!h%Z>s2h7P_A(4qfC+3-PPJ_CI#*>(B3WZx6a3H
zpI*+C1?~?f@~uqL+33o7W)t7(pjp#8b^fi>+L6lt@RR&uH>F9Z4BrX8VVSs6M_K3p
z@@Kk|MjFeyJT^URW#}~yaNYBT?M%b61P#HB66d>D2z}btb?%q<T9rN#bN<8?$;=*&
z3bz`#|E2q92qwH@3CMZq$HDZRk9A{P{)7`}9a~(bd|zlLySAt;(eTRrU2x`FhVk{D
zCQYUmsqE~ZT}M@QmTp&B8oWlxa;~uYxqfb|1lOh)hGGp0C!H9t1X-U9l6~B-zuZTc
z-SGX7ApK)S0@FQ|m>71-J^jCGsqS;3oH^QeXK}{N%F$TxX4ksY`piYA&V2a$WbMi$
zew;m>T84{_BtPyx!R^4+SG4M!r;@6wh@b5PPbJfHhKx%RPc*TtWK!{LwyZfU_GY1o
zxxbY}Gtd7!EDi4*)a7+vsvH+dICN>+dQC-x?N&!5dJ7H21huUnFD(dm-m=EMRm45e
zE3@YHk^>jAri3hC^!)W10p0l`Z$%h%FZOvYH~F{Vi`V8YUi*vrbQkj}D|0fq_^e*U
zH`#$vz@Yfh+T|-ab}l(?_gUE1u$gzI(hQw;#T5?%1p?jWmE7xD9Ak6Nh@Jn)b<*fi
zhw9E2M}x^CU()tJdSm?Rjdu|9f{+QutCP8}ofegPF1AjkNTYofSGGs1Nr*nXWbab5
zq?0#RUF161YtgyvTEf=JalGnnypa)WgL|GjL@3Fy9{Mp!SJk3{t3)fJJ@oI61)WVV
zBl<&^Py2dup3iNS1lj-h^JRq<qPBjHT0J#tt!ebu;^@Vt8dr>LyW67r^9|OqhJ5^Z
zcV4IN$KrcJLI3xaMm_guIua7c8ECWf?Si&54$BLiK3s^oq8;z&Wv0Z^IAbwSiL=Vz
zHm^Ty7TcEe+&Hcp_(^5wYSEw~OSV^!&H9sWv~d4uNpVV!59K+Q))Bwl$a&qTC0izO
z%RV~Zv`G8v`vto$z5mx|eCJ*K@obB{X5l}6Jp8M8n1j5Ri@Z5xv~tlD?}=h^hfOu6
zh4JbqTd&^gpzS<U&FQ2_vqCS|?pKoxKMDWeQ#R|-1>S`vd?!o1Hg1kFWjUzyu2w;1
z>I2tY7SlQJT(u-U_4z_`HG+M1<T(HL{ba1xXKeQQ@rEOM+Y-2o7W);v*Ypmpn6z>0
z0jIPjM&{Nh%o{fw<t_iU!SGr_(iSyUwTz@lji$*rmK|SYY-RbN<a++UJL~_>dNG5~
zIscwk*=-LczW(-E8vUO*Y+eLLE`1xhSvRV2di3h40*o#zcTK5iX7XlztY6jPDDvU%
z3laOjn^yiSW;n6MT76OGRl#cEg7}|Tt5+(;9b2h&ER(HQXu_&<Q}1Y+1}s)RteJSF
z&G{r#ot;SXd}WO#heiMNh?K26?vc>1GKEK^%b72QTeGY7n}(-S_%b=c2@5u8P4(Tl
zt?2j;hlRT?oj(?!zx0aN!Y{A?^zm%yG>jKo-FE2oe~ZH{w>9$9w)bxEY~9?oXAjS+
zj+LHE9Wq|_>m1seId|K=HvxM6{L3?1Iwm+Nd^ED(EmpbuQRU$+Z_b7Iod5Fg$$3A?
zq&usW1g}r4Eh;ha`@#4;Y|Wib8%i#$xzls@+L^qKDIyZ38do~jrIq(>Sn^nHs<GBp
zFAnjAd5<?NV)<fnIJnY#D$~jhX1THZG{a2G!=~nku2qs^T{)rLx7^O#eEMb{7jc=h
z3vP-e$#^FGUB)YWSnu|x?a@oGMy>Vb%v<HLLi59_&PVG`xjuVj@48c<HBFY6-7D-#
zgfENX0|O17B8Amwcx09T?VFmp`MOQ(O~Xr9Ql@@rj1N*$WMaF@t@;0oxA~728!^|1
zM#*M>S}pYg6OX>Ic<{Yfo6kg!ZN7=~S&p<8#;LWhrt=t0Td-`gY}CQ?PHN^aewyBK
zS&-wXuI*}f$1~}aLs9^<3cGu=SVUtnx3PjsYH<&D<r{ACN5^;M=`jTQu_imLH`R}8
z7WpFR_RFm${D#r497mDQo>3K!D;Ei`;9OchL8R(hs^#Oh)<~aHiT2a$ZZ|Qd7hZGa
zDvn?26T>y}-Yr4h_2wMMf*4P2<CR}*z<RiA_Ju|lso2(KJkL`6&S-x8!1{ejlr5L?
zge_+jPaG&X{F3o}2k!-m#`O^if7*DiWEj8vp*QEKSRUiL+=t>Ziq=1J&Of;I=f6ww
z`FF=uq7~PzY!*A3A?lbtp<roonWuWe(NeMBlLLH$4xKOMVcB$i{pM5rm!;Ors2_N}
z!S}X}B=62%*}H*!n@uZDPv91KWbs4Sa|35=kyNfQOZwwHji}$XQ42*(wu-4ML?0Ek
zIp3$%Q_gisbKkoek6K>dWpiFRIehxh&-Uy=Z&IpeG`2Z3r)|CJmg^bj+LOII-?{&n
zd5TEijI<Wt%xmx28ie_le`0#KTCdqeM{hqvOWoAP>zH(`ZXFagOJtt+)7YiNT13xh
znX>gaOB<Id1qRFHikw|?t6d)Mlx3Oe-dGYcRq}Yvp(AV7Eo60Ct$QP@<b`(POdZz8
z%a2a={=YDUX<m+=%%k^l(fT0~`g`6uc+Y4Ftzt-%S(-iP*m>*Ow><W2O6JO`P+E80
zr*V$XdU4*#2eLLUby*^&X*b95(fZDI8O>AQe3mk}ZecfAuI3STbQdeL;rDBT#n)oi
z`QBA&j?Dk=x<gujTIeFx!#i^-jb@1_JhjQU-FRQ@Tdl=mk@iD+F$IbYT**5Ye_Z9_
zr=e<I_fXH{VV(QKKD%|^8fnRUy292LUN7LC<lcEz<Is<a(w90v|FoE<-ORsz<f);{
z@9CF6TseDAXEjsB@~xAXCfI2<pZ@t;Sb5#H%)gA0O*s*NA~y3**;%~Ix#r)N_D3te
z2`@hy^SXHFVbgzQ)p7qDYYn=h_<Ut*7Wz%SeS8%U|K2lt4rdKk`q`X1U@Nfm+pbqe
zr_2<}@1}WgT5|SMirK0b<6SB9Zk#cHa^2$H^ZXk>E$1aCzh=yrJ3RN6!hw^I9gd}#
zJvuzs<%;N!>y`qs2jd_2y_prvy}np;pLBIe(|W(xQPU3F-kE-I>YHnVTb`G0IjL6_
zxOZiml!Lj&TeF1I3t!!qbD15o$!JSg%fu$8$ZDm$7^O5D4Ux}#W>0B87;13f%a!9^
z*TtXePMxiDW2(-JlFo}pF`ZRg)=t-9N_anEYu2J)T}vm;Kk&yWXzr8C0e-Vr#ogba
zFx7!8uJ&Bhvpj8OYbAeuHU722p?!Q)&;RF5@2@*3=49OO>};&2>J|0y!hg&8xsAdC
z-!4erl{IsF6u(H*i&JM(@MeqE;hda*A1#Vlp3A(!*Z#TxzkSAEP78Idw99ClewDe%
zPTB7o<C1N$@y91hYSquEO|hFfuP17w=>L$pd{bA=4YM?uc4z8}2Q%E~Yt6H@o@g(-
zyZ49pw5506`CB_P*;Z*B`BpKJ$$R-09^XH6H>u3kZRUOD^GJxz*n85wr^|o8l&+6X
z{H*dI-sK9f$3kxV#bO(O?7ee9Y#Z~}FN<wt+7B@-I@7=y>&0Wx@gZTRP^y$w&5a4%
zl6o>4c{dGGo2E1}Pw@y?#E>#+0<&9@jb>x=sj2GGWg-6!7?ycYH%i_WQncyWfeDJL
z9yc#c;&#^)VxM5xRkG~T<fYThWN&<V$q;0`GN|-s=i-o+k?du%6TZH_v{830pTaD`
zB~f#kC!Y!N4twFblX=@5Sq%}5fKANic`^o#YVO;WybpyK7;!vebd)I0sWECuKFKV^
z;;|s*kVs$1xl>!M1x)Tvo72Z<ayIZm>{8$LepgRTUmraqFM0VA&z+l^ScE3!$ldNr
z{%|8Xl2Jb4&nu0-*%p09fA6SjhAnQ}zU69HFN;x1%*|a#YaJx@nCf>1)f)V52+CWs
z=$Q59(-rMcUY}p?QOMQKDWl@^WBmaZMme(yEPGO?Iiyalbz#f!zRL1n(OvMJ%0r(V
zPP2d}Nj-}NX-8^3WU}5jdL(svl^zuAQY)Gm*ueRwY<3Wrhk%2Z^1ZLO1CI2XF^GEm
zOkCV!#;X#UCl)qCc+y18f=C7*Ed|zoH9ya%S&x>srRG@hR;GA*hD~!2;`F%ark%@E
z@XTk;hKoVc>lCM|x#yg-oKRTG=b}D4weZU`PxrZ*Qkp4SlDch~uNXUMZ{3p2ttrg$
z^px(ph{!hS=ei~Vjunb3SDg4KUJT$|(4-|;CzG=<a@9=U4;S4HzKSjL-+omrjm6VL
zfK_r$#>AGjdp$3BX-i*y_S7M|OhnoJc-_rQn}R=?EUxqPJ=xh=%XcL`Th}!7f3D`L
z(nYhjU!7;77x~o4by|peXX!16IRZzO{1r46Cbv2FNnQz%Inz2lR-|x?xVz&qu^SVQ
zgk|I(lF8Iai)M}~n>1r)?2(K6eumAwGW%hh^?{wnQ+HpTazxH&TARPxLxp2n>RmFL
zRa#qR+*h!EPcJo2ynd`~s+sG(h)<pA@fRNZ9yury^I^JRf~=5)aHH`jwWDG|5-)@$
zm*f;=U)%5I+od1ma<*$h5X*1zlUmnP^fzZ|pU}!@l<K|Gt9@SOX^`CR?vHn6>+ekJ
z5|6f=HYLVGaY5tmvmy=$FQ2=6>A;btzH=_S+WGACH4sWOmF#QQFn?TC-80K!?!Ijz
ze$hsM|0k}Q&Ej1(M}4o)ozpqnHmOB!sQ<BiZg8Y+gYCA0EgEG)Ie~1NsSR8col~X+
zWVtR+n)2$3^vt@2o7ddCvPvlBhH?%|U{7Md{w$XToZ81cg^p@$-4NVu8?AHPPy1^`
z?&&Mn_dmVlE_AG7_RCXByVo4Pklnx{v%q~xg5eJxK5wSXge)7ksadhvwi^OZv1}}z
zWK?jdQ9JnhHiKzXvpheTE>Sqewv2819F2rxFTpNu|4r*z>M}T)1Lr)Nc4>x!!#Y)~
zH9q|2PnJ&XNMTx9;L5q(L0s<!YbWzdpL8|_=S@AwRbSkc-W5}PQ0U;1ICk%8-a;Fk
z4L+%bUrUsUFL^%c!T*3(Uc>C^B^QnunYXU(Vg4ao*5W$lchB;S-#7akSWHx^-llXL
zcr{$P(Wc@Uv_f^i)KPn$sVtfs4#|I&Xco&!+*fwRNqFmx(D$o?`~7Yui7Y9Q7Z9{m
z)Qi=qUsE-UCB||=c$SN?&a93=mP-rDTF>?Dd6Rly;<<U<-U3aXSq<(LpBK%LxTNt<
zXd)B)0jKFFR6UNVa7Qbot1P|Ytv)4D$erPX!Tf?JvHNtEPko|hHFZUB<T|SxTH<Zn
zqIr+_9DV5V_SA{4#S0AQ>&{+2rzLpz?TL9h?h7x!Tk&Xj0AE62%H(+#`K*(E7oLpU
zkk!4m=4e-#iy~v-gsDMmzOPnkc>ljV`oa+g2~)1HE6w4_5+1e@f~~q&nnjizSaS8)
z2F~b^(^VRYOSsZa4uoIw4*PfVgos(g{KuwCeDWk3Wamg}M(=P8uzJ*OTy8GR%NJsE
zca=in7Oiz(7pTwBED9HInIyWa##Jlp>51@bzZA<J=B``+kmbv>qm$2GSQ>Lx$ZKm*
z*z{UeK^N~GiV4RKiJf>e!J>B}&#8GqZMr#%?xozlL8~|{G@V>mpSS7Ew&4&JIC;6-
zIP%Tfe@TAAt-ZeDd$>X$iO4+OaxhKZ_N44mhGX#`*;0d@pLvz-p5)<X#046UbZeB=
z&0w@#D{v}lYSC(4lT~6}vrc)tC-(bhFz!72KePR3%|@A^l1-wMR9O>5rzqV2CAMM{
zkHEt(n_D~2q^IX)91WiOX}2|xL(rt6=71R$B4?JB9SBv*=6-K<X4NI%gMVgqK3kIJ
zQ>9sG!l@=0>#%v|*`<?|xedZPO<4R2cDWq!-{c)~%AoagpXw|YHSwTTTTgQboiEyW
zv2#YivSY?~*YRe^c+V-hBXnzLpQz2Y<&K5zYIixk_UyCnRJ*;9Ra54AR^sYh5k0RM
zPY*?|+p^7SivsMvz4a1an{j-h+0ES>CMH(NI$jH^x@>Ls<VrxPN-Im&t-i1r)#5VS
zx16_Q9GLb6O?!X$fS8sO(-Xr|g~?JU-pibw=)QZ&O6HBt|Fv~=Cuq!DW-?dw;tr;q
zDTRg8OhO-Co@Zk%%4-#1D*Z-h<#kub3+GgHbdN=|bkCf5N-wq3P{!fFs+5RLJ6Mg>
zF9tAZOg?)eg(Fq>;vA->SC#u#RA{*ToX~K8$0qKHVlMiRe2dxcB-bx93^XZ?Jagj0
zoZ{ry{yNErOFSkni*oqY7j>hl;`PIBMw#p4CncRljH0^Vt<!(7B%%H1pNW^e)Et+V
zO*B_?Ox`wgPT2XNh`ZO#Hnys`{G5<?an`kKzuh?#Tsl83U0>T?+->Dn^>l*Tx!Q$Q
z-C-`<^zx=et_?5C)smbsT{O<1tEFg)_^DP`hjgXxH6c^Z98>iOUZ9xd|Nqj%X)3#K
zsCY40WPf>^AQ^bwe17z~(4ZMwot|9HQXaEnlGSS-tt`{KcsVqqO(tZ!%Yw)$l6yLH
zzDHd-EXEZpQfjQ%=)G;Rc@)Reir0tuqBFM3?(2xZbNxD3^@hNYf-7FCm6&QA{m<`d
z{l)WOfW?c-lUZ(;jJGTlp8LYcaO*{_?MG#E3gbPkw?5S>?_8qq``azLis4?N>+>B^
zpLx<=dU@^C`p9Z}=&YB3ZPZe&s<N98$`@YsDsY`1{_fD`b;4UrF0D7)`q63neHQ8D
zz~|mg+eDVRPEwtn?BqLhnNei8v0p(X*UUSb61OJtwyCa(neprORq4yiYSjg0s*Xh&
z3ICVfEi9XpB(t$q&BRn7O)RseDO&2dMz|X53>DT{5BNJT@kK8yXWk(5v^e&k16T3^
zt{Gc2riti03Dw!SFfQb<w34`~n{a%}1vZNtTwhhIe?M(zo>=5?qR4n*{2p_GUrXb+
zEvS~+SnaYSe&QotrV3RVlLqDpUF&eA$cN^okIglLbeTT3CVWh)QMT}SqTsEP>!lcB
za$K<Pqhd~@@Ou{}rD?*B2CPdSa9wlcI(xKK{h&+e3Ug-0wj<^C%MYst3#pxU@lQK$
z)YG9Z|0s{CRA^VT+oX)pj7hSU&#i?jg1RRa3L2PcFD?AKAaHN9%zk0U(kbarwpvw|
zH3b$|iZK6Iee@)8>x#N{hosxp1Q#5Vv$oJ&lkUD%y(e^q$h0!{iI1uSQk2{s*`$wT
z>K*KfIa;$&tVUu>4Ub6H*$k=18TwnMi3SR0CsovL`qq0SB5Nt5{=MZTUL``0&4qQI
z^d(;`eD=Iv=3xZqhun}YoQ((j>O6{rEUXTxcN}?W_#i^{_BJ!!i3S}XtJ6gCyaO99
z8!~L#>>yF1cJN`|YEywv*Q&ZUu_h|9nwm1KjNlGYWMwpGTk=3Wsv%GEx;VG9kzl&?
zMWLSklZzZJdYm(&<u5fCH@Em#a0gbD@TC<qXHIF+n4oLcE79C4?<8*TE}-(svSzb)
zbE=5hiiGE^5pyU0SN2W!{a@^>GAU=;M#aP=DLxO4r4zip%msAP`br9O*9jReUFyqK
zsr^$!#@Nw)*))xu?`qa=osZHS!(2rx%B89!+__5A))t6xCYfqG8p?(06o?6L4HRCo
zRp+*v@XVDg)r!>{-Nkm4%gQkJIBR4s|5>CyaaK=Q&xsw=RGDf;JEt6;UVN-sH>AV6
z<;DcPi{bJr;g33M4{&B*Pw!iDtyXNI^8ZhY%#TanddRc{PESr$+3qA-T`3i0Auwy3
zh`Cd!`$@y*nf3Bb6HL>LPCXG8ikwqrIk7OjW=V>_PO|#llbxk58nZvde9|avUO1oc
zbK<{`A-0wbt`ZAm8Yf$8{%>Kon#IvI*}hUpb9ua@gr@Pq_&|^1prjhlne%Q`Ofgq4
z(G6rvyCA|OxQOdn>r7SAno{w^iYYctV$PBl%npp+52qO`vb6uu(R-*O{jlMBpi{x*
zg(^bR1)rKEiEC%5rxaZAUT55S;$X77n9_%B9c~-jtDmQAyTIqJDEvvpC6}#C*HP!V
zlhMCo;SN!SYbV7SR*CGNu|VZe)%37g8`!IT3o}k-xSh4|oBd4P)wB1KM(_C(y+=&M
zo_2`p9@JygoMUe#c3d)I|3>}&+me_s%xMp;i`umG2%D1Ck(~FMvXh@G%wIV7?W0=n
zOp#iP+J}-+ZB}BBSJwSHu640>MW)LCiEF013RqYlDlF#~P3e8K^yf}LyQ}s4ti*1s
zFbX+jhA1u24_~nE!-7**)l-kPoUal(uAzD96<1zCM(&0+WeNi6C)(C`id@d9S){cp
zxG4Ah_5_(<-to_q6fOy9Xw_G(a&URRn5Qg4At~Z&i4apm$5#&(p$Wm6;d7Tb$KCk3
z%CtRl_R0wz70Jm0OS70{_;=3R^0>^)P(7zTmnC#Y;iJ4y#c4aFbb5cLy;hkl@p<W+
zDUwSS798qWW|gT~RxoQ%dS-f3rpf_U->)IA6`32Bui3SHbM6I!>27TQ9httdwl<1N
z2Yp&zW~IBaGFO=?>*@|=v+&5rO5#a|u8mg0o5KEY*v_Ue!RVYUxRTj$Wz9n!o1@}K
z%*t{mq#FyW^D0HgZxwkO6(+rFoqWnF<Ae2Hi$vHq39D^%y%)BDWyieeQ2%QW)r2}V
zgjcL?j8=3gRhtyHK&7as?1Hg-MP^I@(>K%Qmv=SiCS-04V6&8nOufLRxm};viD_z)
zV2Z)QRgG;KFZ#{Q1<yo?T=dw!{k8WD){>f!%R?rKzA%iiRnLy9Ry|z3yyU97p33?^
zpR$c#%}KUq_?4w3u9TFJlFt|Ic5a!-$D=#;+}xo(xg*8hy&`?btS;THzdBw<`x`%8
zvN6##$HD)M#nKw}{DYVI(u?LBbl4mIRJ`(7u;}u#smqqxy#Bw5^$yqaBu%C_o2NN3
z1z%tb5-kx;)O0;8;2O-fF5#e&Ijc&6TyOx}{fRx#eq^QyTFq&bdX<q_(Yz|<<Lv!6
z_sU<Y{WZ}f<zd@iua$pRiUdDjTJ9z@`O&tLS&2uEIDRv1-NhE()KwpR$=B>waLLVF
z7H=_u--`T_6OSEq3&_qA=X9@nrnbYY;l{_+x0kIFQf<%w;pHy4bK2_#*1OkCzRmVE
zHM7jY&^ef`+<{Fd;CRdi#?m+0%n#x{RF6wvVA~$R)-Xr#TOiv5MiIx(q%U0ZeacD;
z9&V|Un*GG<B<t#R&wrVheinJF7#6W4VUgOYvuz2p4lBthuvSeybxu3bTk-#1C#QaX
zX4N+*r^|G%-nL+x<};zsQafHA(fPV5QPsGtbI}s-g()?ebFY0glF}}JR-7eqI!5N9
z+WX}*<TP|Lg$rX<=Eb{2#h<cQP!yTJa*b8C=CmR;@i&_|)DBA6FscM_sV<P0{c}*&
zfKmL8dx`^7^nrt}kJ*Am&-4213QywBT_7KFab@6b!?2Yl?`ADGH9tA$a>8murG%?B
zMVF@}-3UAK^N<3QzME)HU+}SHFO_bSP2VbntX#^x1ZT|1aG8}T-uJ6|#?+JTw{#bq
z%~27$w6QQbz^CVMq8O)zz{QVI>l1fWdx(CXsd2LPaN}>UpiP^<7iyhhmT=ZSUU4Tx
zH1R)o)tO!E9GHYJgmk~jjw#@-dUHH~&WWlyCvpq4W~8;PydD34YTLA>Nw(#Q7k;cQ
zKf9&JxivIm(WNJ0o5iLoFh#C;7#dV4depgXUZROZQ`Cx5p)~E%&`lAEN%@<;PvGs4
z&1O@ny|Fh^@XTti=5kN#wCOV9o+*0D<r0<Zxk^V@-z{tIwr&#IFTAt8$7$)@8R<K&
zo|=%`tx%+suxtC3&8&ad6j}a0UMI7~wEJLiKzu_D_qD8+-~gr#%a6YiJfBx^_lM(V
zwu@{r9}e8Q#kN1<oPqeE*?TS?-QII0!)EQa+4a5VcRaG6^D12mH<2q6GR;xi6Kgqf
zuCuBl<6h~MG^78lTQbC#C%LMxSy*H!A1rb4`g6mNu}XeMmkcLLpFCp!YD%<c!;C#C
z%T$F{f7qoSW#jMm_GrJG#uL|e-Hl%2ehF-w7?)NHo$z21`fwo6fbp*3wdx%9Uxp`A
zH|Uo+u(hR$Wb?Yl6tMSbia+pWJGyI@Cij$khfdW$-p94>!Qtl@)Qm1mo@#CQ<uk$b
zM&xd}qr0Rc-*vpLQa#13k}25Fn&cO@U3mINLG8Pq(wXX4QeE~3HqUQdc;v?Quoe}y
zkM&+fisoX<nMvxd&u)EZ56F3>wrpbA!#4k_q>U}F6PhLk7BMZ|yIJFN$b3$w^q;?*
zigya%Kl7~HL|N%>nt~#$+3Eiog*7IOuecg=xMTkwY%D!~&-Ox0LzTCke^TB8t<y!n
zQZ*B7uJ^?V*7d&mT)T&N@0KZNZq_J@KfT0Rx7je*Gu+&F&!<>1m%#gaiS{WUZ>!lp
z6iL#blDKk;?u-(X`3H+U8a_(rYtL5w^v3FvNZsQ@T8j;)YWl5>og<Mcwo!1yzmQaq
z+D9o14i_soulg0UX^E76c~5G<`K!AaoxM-2v=+!~Xioi5w7PRL-zh<UtCka13ohRh
zv1B^B;dg(I##0Wpi)ZJ}R{d0Z@y0#o9mV3YO6w-SeLO2Pq)=pUr`WNU^)au6CUd<y
zQdqikYJYg*=`D8}ayJMXEM#$bq+lH1>h$oxx#e6d>sYs=Je@Pv?TgX34!^i$!(YW|
zd>TQkw$*H(xN1ekyni=V?yJ<2b1Ay%7qCd{$VK~rpx2oq7p_VjFr7JhPso9wP_E^R
zGK&h|*lz#y{@0<yNA7h`ep^?!T}t-nt#i9wCThtDO2}?|mhUff^NK{~Y4K}<I%U^{
z<b+g|7F{S>sI>T?;{zo*#V@)&k9}7iIr3_YwCMHADS=0ucc+_eKU4eCKX20)VGlOV
zdq)n;vJYUkWjC_W^HmBtyz3ChCt3dm=FVa~uUhm9H=pp>8FFatw68t|Z-xA?tPQc7
z@Nib>^qikwAD8D9m<N}BTyXSj+3X`FzeQqsYMOt1Ud<%H%<^Arrb591rbbR4u8<8E
z79MWnXJ_%4(BiYYbt})cB-Mb&4Po0nas_9&96F%Mw0(l%iJ(W`GYnI?bdoomX7J;l
z>~ba3;H1+$er>f0MS%&kXL=f*dc#q~AU(mC-6NING}Ajs$8b~5WYNX8joWTDimbfw
z{N!pOPQQmnu8aqDg&uhQuw;09h$UkxPrxDvM#*R)=H*jgcrB1^F3VAL)45y_VZH9o
z6e)*KiqmJr7;xW-xtZ|%oRK5<nVDCXIvXwJ)L)}HA@#YDrT5mpl~!Ah__1lOe|Gow
zcW1v`^&NM!Kdd{mi}}p27zu}j2`y@#mbE)Yl2@0dERl)N___VyEOYiJVjlnNY+koM
z+i%=*GT8B+%DwC7+dDUEuYSq4wol18>*1HBY!0hLgR&$~D@JqZbna_WNzh)uXvzeI
zLkr@9H1t_jCiHJOn6m5e)Myr_i&M>Z6dYb4z`84$;Ru`bMZRf4Nd;<WJU;PlzF1JR
zLhV*Y^a~R?c8j-LB)jK$nqFsnk+Yq#EW6oLgZI?=<E>@ZA61u#Ica%+Sa!Hax2a2b
z#R9{(Q5?IjDlOt*;9WWUl)_4uS1YE5nVI<cD1W}PS&7q(aiwS&!`m#a)jU!m`&Voc
zxe(B~qb68s1=rrLo;6IOK_2UKRxWpDK5cYzN&5$r49iXtkEG@u9FJPoafbQksd0)v
zc%b_KFq83{YlmHIayA}R>ORAJl8xaSub$e2MZJ8gNoT~&Jsv(dc7747ud(XNtAXz4
z7KH|#S4!^ojc)e(Y`FZ7j8^!9mos$y&GT;m?c?TKHe<p1uRg7uyHq-o9$T3|HF;sh
zq+5M;!ITMmx=!ikd9T0ArR#G-J@{nD<-7MiUU~%mXF0Xwg3jz+c~cm~y*5m3Onv01
z#BuUO0H@Kmg2=?lpJy%F*zO?n;Ba)z&j8nOf!`PUjP58b?lpUtv6M$|!m|TMj67yE
zv={}IoOsSywC(ZfuWwp7Z{_Un<IJ43Lt>Naruapda=KnhTklt2%5puRkX0*nliEwM
zuy<;#i)L>Mny~5jk^ef2Wf(Faa%6cNVtnSRZ5gyah@qjYBWA_Q{-}o?=ZsuTU#wO6
zY-6qVk&R>SwhQjLb8N3nz1-;^G%e|9pojb^{gigAX&rCt|F3<ua*M{*F41W>K4dXe
z^ZdCk7Um)PIdqF(__@6*Y8~DRngMgm7w-1c^J8K)6No%|RE7H*Z`x@Gu?(Ro##zip
z9ygkpPj0)RkYQw|z-;)N_tl1QvDl5ScOxxROjINnOk%Q_^=?nEvBB=HgD*a>&DC{D
zTvhmh;asZYhnCgmue_v>?lD<kX{r;#BN=gW(PmaFPL&jG83*fI7r4T69;F<cD9CKQ
zV{vY4dch)X|CcHi^DeA(S2BKb_J6fn%1XYvc84Dmr|hgSHxw1~Fwm4~;C!ikXo-%x
zbI^`Sjs?%UjMk(uoR7LMrWD-9xy8{arNH^}r59X6K8-bbGnhD4j9E|Gs-$n%V3kaH
zGE?PT3m2PY$~J)r;b{t!a*~zLJZGv@nQ<k>D!bUDU*xFX9!`(g=v6{$>#uB|{=~)j
zQ&&eE+l2}9V=~liJRKN*u`G8#Sfr>Z**SBEvbe*uj}zl|ck(tKnq6~rdFi!Ck+*WJ
zyHB3v3*eu4<+V(bsEUx9cHl;@Wp56yEK^z0rG9Z<)TB?7pQyNp+~yYju6ChIRCZbV
zHVtRl=!<MWkMh@*Yc&7uabq%4$YuJj9j0mb-^DH5X0_VOnhxC}wiEX&j=SA3O4%x)
zaUix$D0gl(XDZLJgC$RUo4Etex+&i}_~689{iI8G^fz|sFG}f**E(}|f0Sd}-iV8i
z`ihg7B~P%*dmWi>d}vXu&!VmU=YppB1zuQM_vMJIEtiABY$Z1>p~;d$hW?ck4|H0I
z%(N~zSZ%3%nB~p|E<PUxqfYJS#7h-hIH!pAX6=dkm3MKUZb@KP$4N<>W-Y<@B0`Je
zf<15i(OPo0!PmsR=Rs`!XUl04JzhJkxsrb_GuM9TTQI#wCv%%=tEPmWQg89SXDp9o
zx*}8@0=}GZdtt)5CFF#P>f2t`!WEa7w5TyIzkRYPz--n3ge=$2ZPlGiV;*gsW>;M9
z+tc~}-<60?!S$_joSxbjlv>+8Om}UI?d9x#EY6wpg>PFyl%&|=(677)s&iM&?>usv
zRp>x>Z>k2fo66=X@ppA1nRc>n<|#VRdgOG&chv`s%oc9zZyKzB@F*&AtE(gLr(?^e
zPU-OWaFmX<TGAx6mq+b<@(rclPr8DUa`9DG3@=*BxW!G<xYs%>80}@zt5Mad%9wO;
z&L@NA)^j^c{5z+gl@dH0zO(P>siX6I7ILrTR}AQMS8F}@E<#Z<h#@zKWs%Vl?@Xb9
z#gk4<*phs4NpBg)dbvOr;W?H?$1^Xk?p+dk<E%%YsvXaz89R?odU@eLo3Z6jnW<{O
zli2K43WNl&YdF7{u2TIiaqXN#te)DUR$m46FSf2rRhlDq{hh(Ki=P$>)O$+UIf^R_
zpXLu?QGK#xQMdOI*DC>EFTY$iAxiD?lF7}JcdzVC+3aU$n(7mgJz-Lp@O7h!6Iu*}
zC8Bu`h`DJy+U<TdNt-uq_o23vHHRNfnAiK_BHu<HakDL~*~w{P*Ur6Xvc7yjTQ{Il
zr%z*&z{>;Lhg*y!V{bTXX(!IvaWl5{_6LVmCj?9+RgT;~<Js`%?K0u%jJs@a-s!sN
zdqdC1qQqG2gzu5b(%#E??HrDNSXlX8ev_kWh0f80iOVy(HCb+X$k$24?Km#9(z7@H
zf6CsOJ{e}hP3tmTEq7n?(%6^0S5M{nhT4;7jg{YDwJFIdY1^LKEB9V&Yi{0qv1t$9
zOxwQu>XPpH51rzL7BD=IVP)X@v3Q{`@1&?4rd=*)1fpx~-h0N+day!ys+jMi<>mX9
z#K_LHHkv6E^XGtP#b%jxCR;a6Sr@QM?7EFW<SoZZQTdlG4*V$yFgrabI89^S`DHd|
zZ@r!oBl*P8II-uy#^clftMaprFY2F>lajj9Kilnqr-3i0^vmlf)|zhq`EWNI_ZOvB
zla?h#=Uoa`->zegZ!Hn3t^UBG(&_R`%S5fG_kHbS@7YT<RL$O$EZE@^$rkxrHj{aI
zugoOfjy=tu=l**$nK{(zU%uYGm-FJ8=zYg;CP;tZe{pqOHp|8KTPn>-^Y*`z?ki~*
zSTt)v&#V)iPr8lU6%Oo?Jg|A@niEY{vjkM_i@4Xkwq01Ddb5yYm4sc;jlStWI9Vn5
zp8cL>b6U9nRd0gS#;KFKc1Ej}+_W~ox!dC7mXO^$+!prrb*LSB(6vf|^`YRrum;oq
z38vEmI`TG|2R`JCl~$}ycKY@}hI8WLzZd1TdN|WB?`eIp=bW(F(G9Eyi>6h~-Z1aB
z7}F)qpEr7LD=4%!PQPVrFrlhr<4&=So>CT%jkzCdFyuJfys_`noVJgHfz@Gh8)p;C
zj3)6SjxWL;Wf`oJS%=yK7+(DM+C1$9r=iH)OC5&uj63dF9F(-|Tiq=1aI*R9WX&&^
zRl_p6mTi!56`48V2Fu6G!a9$1-rZ~nnz8yw#Qqrzq-L&`n!9sqv(!Xereoz?=5~9!
zeld$bE8cP-N#S3y@r=kFPqH^UE6(O#CCg&c;or1t<?R(KC-1e(=H_HLvZlD>f>GD%
z4>Kle%=|9Ev9n;N`)cDBZ6W!f=>=UKReRQIDRGG~b-uIik~yNlsJQm3$B`S1ogSXt
z4}V%e$&!t@J^#%Dxue#q&yzbOtM*K~FmK)=JxvEOmdcJ}5z99|64l$nI!Whb_9|B2
zm8=a5CkN+vx(D*w8*Z3#gQYb{s8!4E;%c+T{~nxXA`AZiTAy)q!f$Sa|G#%Gn%#Nh
z^x?%!C$23vn9;!NyJbVi>m^AooIwok`%g@NK7->Lqs7#sDO0cTonY?Tx}(=cOWI^M
zzv3*f$(o|=jr^K7Sf0GxU;J=-_eF=^g<SG)rtj6!h^pFW7_7Q^LIcZ<<pP{NGd?RN
zd^vj5N8tFA1M3(>`T{(sZfTABz*_yGy((aW<>D2q&PX*kh@H7%x%k%^pBm0HJ2*eD
zSirr?`Lx%XOIuXb8J8{=m{8m}H*X0i-=z8WDXQU0o%L^Yw=bOhlJU&bh6#dGI?p#A
zyM0@1(d<<<nq1}^t^1r^N`7u}Z***2p{DusV0zHPcfqDJA8;M`|Jkwa$`mmp0cOSK
zgeaMpyS5fS;%<*{`gw=@$V2X(GbYUH=$Z7!e~Qh?s0$nonVw9ktYs@&s~j%GtzeIr
zV2J<F&MwPPKOvyGAz;_$1y@#VIBRkE-GP(Oo^0FO)S13g>|DgGMQ3bzd3F3U+&Dru
z+)Fs^W8^#YK@0B$rr^w_EDA?nJ{C%uHMzd9HR!|6?Kc=ak91xs==5^&l`GO{f6XJ|
zE9_$|dOG3YKWEno3amOuXS|hZ-RbRGlG&9rOLgLh-p)CCWm~%UI0WZOoe4hGt^2|0
zI8Wya1`#V>k+2zOPNxL0iFR;4z0CN6ovXK#^RCaU6P+`B{ECW=FY#QwX5@S6|HO`q
ziaKq#c#RvJmmP8Tl$vwkq?=!l8^@E`Iv+Q$)^>l`!a4N{->!hUM?ZHzJg}L~+1uyJ
z{w+IrXLtr3XzApxv|jY;c);!@SzAInGrHb=Hcy-@W47V)+ZbN2g_E<QuK98X_g<Ky
z)2R1MQmM<Q<Lo5v;EUI;R`H)}w7WV}w{Y&-kX0SJ8W*Ax*qL1eSiZ74d=0kC4d8lv
znQLk|N9*<W53DOUtn?Kyn5r;&^XqN<9`!OTvRKFHY&%EL&1hBMFW&y511tX6m%r|f
zpSpe8O!4B_?K29dSSB8Md_jhN=2S(^-dR;07hdvY-0696py$pDp6IW=yI$BT3+YP!
z6_>x=KjVMGqM*+QEhKyPzv0aby8KD`Oz9sTnWY?sIUGxMcLpxx=9T2J<-Ovtiv7{b
z*1!|i%XoV2tpbhK+?3aHUXT>HKK9a<fCKTm;#>Ak*}}SY+eH;u#yQd|YtFCUwRyJ0
zE@i)EcTerP!R>x~p_zAg#N4YHD-{fDj=q?{z#wTi<Ah^M)?B$O)<F%jeG@0^-t_Jf
zn5S#$Dm`b)iQe6Bb7V78&8FtCs_Yhy`NdgfD0+3RZgpT^VeQ?URr{_@^)E>6i0f|9
zkYL_0BcO`+O7+xR(-&O7#~W91)#uNFprs-m&a#Iu9MQgUYTNk>o{cxQwMyMM#c=9!
zh^>pQm7m9wccPQt+s>Z;|L>k<dz{NEx2wxWXFh5=vDR;u$HE6oCYwp@__-zarRTi`
z*%n!a7N=jGnZCEn=;$@>#v{&m9L#6$jy!Z}-=A|SUoVD)96c1QIB~)KWv0Sy47zKg
zjT}Tw+BqHs28J%b%JQJJquiHkRd1(oTW8gbfHkHZ_MIo6iQY+C<GCp0+V$*>?p$|-
zVmc&}C(SB|wAOLhdUUgouNGg`ft`krcHQi~R$*{6LQ`%vtDnT~poufjajv=Kvd_iW
z+O2fa!JTd$8l2TNozlJiFH<z-bx&Ni5P9XZY{{$Lt1s-ZcjW5LmQCcotT-orkwxF}
z-4}zqZeC%WYRu~6_|KtTLX)$DN!KE=@_%VSjc8o;+6xb-UT|BN(phl5|3iP~BkgT3
zc0Sr1H~+N#0<DO~y<Af7lFXO%vmZ{JVIXmPLWiHP!{^@4h-Zh3G+RZyT&xt&@kMt`
zoWy^KHS9sa%@#)Ss~++5ltd5Z2sa(s%MiF#L3?YG(Rs!n9gkNEAC|T6^kCAwz?9wA
zDKxLY?n8Tl8HdBtTU~!G*?ujSTD35w?m6G3Sxld=$eOMB&w3!dY@^311=f&r*AJ=q
zHAU|zy{e|y9ICb{r7GU)W?y4rpTCOm#+_42s;~aZ)o6Ql=6cGhl4rdOXY6}?>qgSq
zJEz~8SZ`DCdnEL`WAbTU4}aq+cW*k%$8<f&jQ^5i_U8YAwD@b?Z3ibGDRy$NnL77b
zH-`t4u0*%zH{K$)xT0rw3*{Q(1Ud?qT{nEtpX=%MRF=V>an;(X9Pc&lP8ZCa!H~^n
zw3$UK+u)3Ss;j{@=KGgD_O7nje!R<KM_<HO(@DBFpRW91ZWSqdt*U$Gt+-W}c}#c?
zyPtaIzl8fm(YD@u8|A#ySM2a+|99zSN5ZBVTdS3K9!hhc+|lHslfJ?%UFh1GI**5i
zegR<;xlt0WaT3p$O}oIU$|_M77aZ5IHn-cz=LXl}8Jn2{yDmMtk&=4YDSP>u&dI&z
zigvr&ysVrx#o2F3Ki?*#?fLxXLC1(gPZe}i7HIfzwK;pf>B&^~I`jX8y69K!Z&iM_
ztMA`?k#VnTvTt!$($nK{u@CN9n3(mco@;$S^B9LOXHQ1#kq^lkZ%#%Zl6c&{z<<&q
zf6kkIogdp5+r4Vd2w>ZOtM5fHTX;BIYOCRb_ND9CqZYJBy=ae$Xf0YO^175`){<>=
zRc5q)c*SSh;a^rbyXIrpxg!mC56fFx%<K@ldE#T_ve#w5ugIzQM2Q`}HRb;63dN0U
zxHo&KdiZR7YQ6c&0x>V1RjXIW%KU!(eMMpVKmNI?YAzf5wdcAS)~+n{k=8zRHMgtj
z@;}~dX6a{RTJ6`iFZatTd%-Snoh3>ly)1(DzXWU4f?GWk=VW_wI>|RUeeVhWHgDE{
zm1A2{N)G(v+mzP1nMdgUzjysy=FgP#{+!L-G&kb&&ds+ALnHeF_v~EoQtfDB7W0);
zx4A-?x-JWP)_*Dp*17f~J#hXcrH7M!Po#>U@teMFL2~S}Z+quXtDEBQoOxR&Y&WaO
z$C^%gpM^8;O`r8F|8(KeT?dvPo$T@IRf1ue%ZuITS5FXl*>T}sYF2Um^`-CHS!Ee4
z)t?7SFqg`<hF&PBYWU76%3SuLy(*raC6+bNAs{S*b$Z9!vmYj}o!akYU*a5J(H>Cp
zvFgXVwvQdtKmP6#>x!><xNL^#)85-3R%rHLj7Ssu?C@-tU)Sz=ZUtP|w@dU1)|TcA
z&wg)u>0C^X^8b^&79F)R%DX(%)cyCmQ%eJmw(pS8YhC(n+k!pa2?^Djtro>v_T1eI
zZV9F3p1pD=ZbAFSd1plZF07Vq2#sKUe6*Nfyft(|RA9pQI*n?+>n-c}E5a7ESAF>G
zAU;X_SyNr;kLGqg&4hH(_k7zj(ihHQVk&xGc5;8`GNIT{xp%6stk%d2zdF&KH!a}s
z!+6yL%MTx0oiO{RCTq9^C+E*2Tr5WpUHM)1EpN+_{ke=u-wr*R@j@wuOG?-I{puH|
z7EO2jwr|Pu1wYNU{artKa#}m5*UjlEfB)t*ZT*+#VmGt@u}x>2h2*mPTW{ZOVO#s;
zQrz#*8Lt>`6bGIt=A2$!*6=;<|B5=!x9nj*zHg6jXO(5Wy6$^mLU>igg_UWo<<np6
z^e>#nQd8!)kyFCLXyflqeq0SVj1nF+=t{E{C`JS*I_k-=S+(paSm4kmrqB1pB_P?Y
zo71pNF~UvdB;!)H3>(V;)q{TGma|k&2r2u{Gz-6WW@WMZ`FX)=8i4{Po=pr+>OPt$
z91~ph<b~B#nhg}yjW|6gHhOkFxYih3-IOYkaNx#9X5k5{8D@@5oIz|GxU|dl&WDM)
zYyPpCaF$_r?w)N*p|2OM4GZE_TUfm@_x6#Ji)s@mr1;o2{bl>|Ieh&zwXiR*dZmt@
zlegXTFQW5X`Oae7s99O5aZ{_Vtg8CwcJ%+(?OtZDb})w>UF+(#{(x`L_C3B<g%6_J
zg72BhUx;}yZOJvwxNyG!%aEhRyM?D%6(yW#Y~Q<h`Ok@pJ{t_~E4x@O`17aDu>7Cw
zjQRur{(jf}6)vQiI>-H3Ma%?4R`DPPRYxW+4TIJuuDpY;tsh;NUaoqb8Im9wTWcX#
zFSb%-foq<@l!zpTm?siRoGOYFVrw-`lKsv2u5f8vZ2A<Qz+5tQew@R<P!-MGHz&B_
z6DBr>YMX6(xJ36G&jmimw;b$?U4?iKsQSm1t!!aQ%Dd>MDeNL9>BwbpM9tZYXMzh$
z)Rjl)8OrXx<j+j*xyGMW`c3Fo+M<MQ&#jhp8gl8r`QKycrC(=tOUiaH&x6xxU#<j2
zE}hOBz-=AWYj{B|I_J@yQemEkS1deE+qG6PXOzcQiRT7PY~|NHa#UnTllc8L`n(}*
z?#fd?25vmaSCFvb?ZV?i-g-MjE_<H)7qxb4oaNDjBGNh03xn@B{nl0YOPxBkCMtHe
zo-cz6Lj#M^DrtSkGS+9(6L>Z@-e#ydnmRpy35(eb*Vc)QHyQHI-MS{W?Z&6>vZAFc
zU2XptSWU=cE}3@IYUS0<T^h4^8>9;sf4R(`v4&wg*YdAx_5>eYv%t-3<poF4*pdxq
z-?LV6-QD9@bR#!>V$s#WmJ5ep$3{=AJz19$y1~m>Wzilh&tAFOFaI@LOsmB_d%w6w
zht>v3<^@D^b*XK=D0=f3@5xIIvU1#0FEfjDTx^p#mi}PFhlx+*w_Z}KF1yOS!6M)x
zvw}mwBWA{l4I(KIPE4&U4CK4MS2=3l<EIk3p22nhj7*ZBSec%5)2{avRh16Amh8Qp
z`NpMl{aHdR&mB0DTduh#dl|S&esjpR_C0i1)0HKNqi4DPOx_DC=56V4>Yn>%L(=WE
zE1R8PI*2~+oVRpskI*ZL*(TwuN@KK+D>%n(TXo^^hP>sM9l6f(n>ubYI->3S&G<>_
zf9cAU%)sFDu59LOXa8|my!dOrtJsT4GcT|-1<zjKbf$Ic>WB-O5f@K{U3e<Wwd{X)
ztV2cjfj6cC!c7Hw#!9T(+l;!_Jz$Nm{m8qQ#aKXFaEU=#Wxy_(FYDqz<g4dR>Jbar
zvA1qblNj^Oo@Hzoj;I>7RBv8&CN1KJlWvZqkWc10ktq{ILTUx;m;<M}E-;?((~zat
z%Q<;n&<ug4tc4AlF8)qirmbty3CL0XIjio|rQ@@|v@0}zx$U}S)6`w1%dMq6m(;8{
zv~|*zw;s*Hf*VyN7EGMcq2O_XQ%LB-$rleaivBH3?8_0E!SGhyD@OAYJLjTPwVEoP
zH81D*Eplm-y?9o8f;7X_tdJQNxt$8ht7HwYtNR#gHu8K^a8AlOzBHZdveMDwZo##x
zu00PpjsM@;a%1kU$dy%3x_K5$g*v*awiZ`;R$OdZ=2RlK|IGpaFfmIGY4(QIvkrW$
z^HNNRn6XsAug68}xg@7@<|4rvG6oAdViq{@bqMzwlrZr7Jdn}unz%kkb@8O^rLIqp
zu44%~5_eT3)7>!0`Tm{_`Z`l&UMPth9I|<-tG6rm(7n=b#g{F<yC(ODe7JNZyNz{)
zL+A-%&Y%$Mz?EHV4En^DIJ3Pd3FPu(IK{9`YgywBJ`Jm4(eoNf#_Pp;Gk)bPcDb;4
zu7%NuiCMl6uj(Zo-*U^hLBg`r-oE8P_dh{pB`rZ`qs}#3RJ~P>8wG^0Y3uBgG>c?%
zeWU!<XvcwnsWu70|2rO-nKLL{P%-Qd_Smhmp;Nu}=oGG(G0*>0P4rRHTt6#WNYkg?
zD)v@u7t`TMv1wL5v#Qh?g%yteS;DYvm6))yP>0isnuRt^4?O)(#J)3|T`l=6^Ssi{
zkP`tjU%PzhU9U7-qFLo!r(epWzKX5N%4<TtPHwb$6Z%QU!S?0$IaSyE*YP;6w#^Q@
zZoBWQ(o@Ol{{<>0bzFK9A;c2q&c^g$(z7F5IM$s^*4*h{xs}^(w@sqoz9juQVOQd{
zW!I)Au2c?Y;#sz~f_qL{RM1^shNH1vJ!(3O1+HeO9dkYEanvYyW~fVf*WD#TXSQ`8
zEmUH+44S$s<<iRqtCkh_q)y?R{QvBYIn4p?4LxtxCvEwBY*t5lfB#=T+1JA1?w5{c
zSIv37KA_WbLrAUzo79AG4v|G>{Va=jh8Q~=_$=PcyI|8vDSp+TIv&1EQHpBsQd0IQ
z-7>D{+p=-V#6{<0l-+}OtY7YXDL!MR@EVnu20^c$CS+agi&uM=#`SZpvFIYd?gxH)
z=T!8jPKnl#zMLehA>DlHOG<Y4OHJeRp}x9dhHa%BYt&BGeso>-`bgA<B%WC}n$)KF
zuIj4Te&PYI*GIR##zwQkPredeTz&Dzy<dD!6u)mdu|u=s{=_HY3oq>I@cwgBVAr=N
z2e{{%@70!ixP8HgL*gsdB7W#jpPY71Tlr{af*bq)-7~U06<7Kx9x4b=(fx5qY1g_|
zjfjJNQ5TMxnZ0ATowH)r=>=yVElG4uxAL2=7Iw@}vwV^H#JyK^GZJ}%8Z3(>oRS_K
zI@#?tAxnI-a_g6c=5;>LKE<q1h~Ffqy`fN*KX%)k$?vw9HrOoOv|dg}-ttrWfrF=1
z=II{_pSeI|LaCov!s8gW&C?ZgXB^S4`)4*`-SeCMk<;f~vFa2&!>3Z{(mC;s%k})W
zBdOYEt75d{RtoI@mFBtFxT1CGrznMuiSC>BChA6QWW7C4YJ2V}&O=A4r#0UBw*8P#
z(~&)q8uedh`OlohoAc{n-`zDyvRhub>)$YB@M%47_T|Ckm#Tg9p8S7wGI&McHZhZh
zt&gScZv{%^O<XL{sF9H<9Q9$XB<H59pQlY%%R1w^^JipfQ^|hy*M$c5{Lx(>OPc<?
zWmD*6(Re@YiuFI+LM#6l^=ZE^x5mEH3SIJ)Uu=I)M0@YGuo(dd?@n+MY17SdWJ<Yr
z@I?IQtDYirk7;Eb(%Rr`&(USSM@~C~E$a)@=@PE59ZWe_oWGp8!Lo9W@J%Lb7B*>}
z*&-Q^A|3}nsWysSd2lB8V4uL5Ge4(^uk>H@roXnrcYTca`a5SC9?p;XB7Rj>@$%ox
zeik=ou->T8n46V3GwDZY^Vy?ut?uon4{tjzbX{>Hspe9pL6o1tJiiYd4J>l;A+!G9
zIoR5FCdl8UGsLFzabj!2jpo-eo^N9AUAWrvBKC;DTR(r5nKK;bPrY+%?gcsP4#)W$
z4sj?=T>OG_amxAEv2#?j+!nJ)ySdCMi4->zjQ$?MB7E?S=Ako3uO3$CYq3?hEs??a
zIaFRDF8ab*)>TIWT?-flk9l8;JrZ2tSoinT@4we$f8IA#T3o(?n<;})^M=#YD=ck4
zXUBw0ZjC(h;Nx+xy<+<XT)1PCI}{%rym?5?rJ+-I4)Y?ZDiayU1+Lb|P8O6tTKAPX
z{X$djf@2wP*laV{zO1>&vCZw+9k$#VXJSIGf4ksfwp1bL?n(QC$wv*IIDNZRntSl6
zr)cO^CA*WS{!fi$sa<+p=+)G#E4!O_&K9nU^whh{(5v!0ccJ(pS^pMUzaAIOC+#A~
z<{B1V@tM&v=c>wsH&1F8obcGP#5G5ZX@j@AsVLt^LH`#i=I6Mxe3K6x4U$T7dHJDz
z(iX7;3_UrCp35cLx^MKb2t1m_ljO(YAF*ihhc63CL?ZKM%$xT{&RU26=*B}feTSq1
z`u9yZCiQ^r>x>xv1g}!jCw>YkeiG8oB5H{STweuS@?WwDT<ZGPDDTp8-!tg0`#RB0
zH;y-`9QnCa#qWWOp_Ig*%4hW!0e&HgMseLu507X5ohjNT5Pc`e|HjNeOV0*x;_bcT
zy;mlIZ)=n81SggrMgKX<|NU8>TWdA&a|S1eOwRh^lzxZPmVrI(2j{7ju0UQDk%ei|
zn`BB&r_cRy*uG}D?Uq9;p3K?isJ5m?n#1?zcg1c=twjGFvv*hedXy|<*gI7$>xg*L
z-8vRk{{>-OzF~e3BE2^qNoG9oP=oKDCJPe-W3!h7&%-p|A5%lTmb$WZE@AEH+p{p|
z*22V;<S8j#uS=w+ZDBUPxM=GKKBJ{CW^CwWQfZvM!L{euia9-p)YF8QSF{D%c6G-d
zOEg>J_hZF(0T%7Pq<Jqo3vb>k|8-dZN3;H$hjQDTrE{91?_5iYW3jo|VlC)WwQb_X
zk4f%aiWOJVosYISwLVKf6mPg{rOU>b|9_`GpM5B^YooWw10nZ=4|Aekut)~-3c7f)
zc9^TlB^a>CZ)++jxVP}pHRG+W({)}=l=Wc{X=QSF)hy9!?R2bQ;z7<=Ez6`r-?t|D
zUzuTM8}8S#YHo_#q*t9^Gum`>j`eN0Q0A+`<HkPw0-y5>pP(x-j(>Vq96c?l=X+^q
z`Za|c(R;5}W%xQxh)+1!vi?-$tw!HEi`VjJkKE*$aobbM?aY(-Jx}Cc9`#9j?Z>mi
zXPryKfmk7?gAWB%dpsHzK5fa3I+wF;%F{V~jA`edpWzO1;*@1N^D-s*VC}86GpAp=
z9PBJR`D;Z+fDF%}u*|NdGkS!MHmkT5hDC36bz3QM`+wrE=5Gf5C9Ikpn&FeRRu(oc
zpKPVM`q;Vl#G_@oO>8w+!*_NVherD*+;`2$w0?N`;=`Pjw%cBtR=O5Ai3hX>ON4kQ
z@UqNJo1?Ti>r(5bb6x+=a`x_cJ^R<u8;RO`<M=F;Qu}@^6Jc4t=$8P0W9O1C-6d8J
z4lQ(DH8bL!$1yP<AEUli>vY4{z2vNDT5f-%IW58~Q9=4ghp>*4JI}d<E(7<iYr0Bo
zj(5uOR<G)ws`)O}FLtZWYrhNbKL3<r0@wN#$U4qwd=|$t&ozv(@kz&$-WxmbHayI%
zVo`0{#_elzE+vBV#hS1;Y;Rl}!Wb`gd%9{oh<(SIyCTT!<m5Ax{)gUTmEYI5eCt7-
zw$A1)i+hf(=+<K3kBr{h(K+YB$$qhyF#<0qh)EaSl$vzw)(Nwha@+de>{w-Cdz-;T
zRACi|!AGrw&#w3-L{EFyGbyvfNn-BwX;-(FFgh?!xis-*#;3E_N?cq-7tdCD%Xs{y
zQB~N}+$l*tA#J*<4@$G$UY>aUW76I^%3Y$%51n-_%rbc4dQj{}oMs$LfN1E7e^a`b
zzPh8fa@97!g@?|p{<U_E+3^Fn=5#+WNP1Clgz2(X-8?^&$zcnhdOpifJ;nMsiA5;9
zw{BUxfsur>%a!TNMB8Flcm}+_Rih^MJEihV`tx-xVzW6V>a-hN7H9Wm`^QbW5qspn
znx@~wJGWa}U9a4GZJNfHS?c=aV#kA6zDXw(bs}9hygNPdO!kx?+N;^3pT3<S=^K2e
zEA^n+C%-o#Cbpkm>~z%@Njv_@<$GJsiCfMmxB1yF$hqHl%7sraVoQF>0@WEFZVJ!(
z_>x6m@UHu{L!a?#j?m?ni*H;0T{!Z@&DU$j{OdQ5q<wo+-=k@u{GrKES8{q}#0Kx4
zk}jFU3tb(It3(t+;<6QvCqCN{5F-+rA9c@R#|bw5h-4ALqZO+h_pLRnOrLb^?cJz;
z{RB>J4)&}sx?&#_c(~S`<@D#vmb2Jh<hijkbY<6Dn=`+61lCAwsSA}Aj*qoJxZF$W
z^aP8$ihHB~Cxou9m)sna@wy|Uuu~@YcVme}a|ui87uO9OX|jB2xk2isLjRwxV~#HN
zmc3<aKg+1UW2x!AeL|cGHBycxQW=(fu^g>297(<_R)`#N{kG0{RbBp%s!hvZddT(}
zxnF9T%=jq0(e>86<Qt0?NARgzsLxc6{`%+Hk?j|ScQi^r4r&w;SKP>QtwQEPI7>l>
z)Q60^FNTl5%DkN4;<z};BbnL$X|UUdXAe#kyouh<bS#X6XJMFJ=zjy>iVapxaVb4a
zcg?o0-CJzK&ZgPJB)R8-*og%^zdnbShiWn}dd?_rZ*O#{<MSTP23MhtPs<|zMX~4=
zGwLl~Cdhul!PLa<%Jcsh-MS?=9NFN~Eyg`*W>(<@md=@an?=j!dL<MFJet|5_##r_
zz^;8;7XNIk_fUOgnEL-J%l=pEbbcB-GZanP(ikh#>GhQTz^-$Xd(s2$-#ObdFW_*I
zL8$2@TR(N@l^>ri{F%piI_l*lMg2&=AFc^oqx>eTIG;%hIbE1^)!0|KS>yP(t4C~A
zSyn0VUj6xM&4u~b4Y;p52Gr_Z$bbH|GyZhk{?>z$$??f8GSQ3o@O1VvOlu7*Fc&|@
z#5CD&%eJJNuM&2CYaVUpoxdsh#shn9DQl+S!bQ2^^`>3js&n>ilh~T?`cOZwS<ksy
zla+DPU)6`PmrZ6KOmXdhcBfN&tK7Z+g{NJlm(1NUX=h!>YXwIsY2lWpcd}O`=6K)D
zmTH^i=U}Y$)uS>wQqIov)``T9oqZkOmc7YodbcH?IqAsc{-~9;oteKsN9<l7nqB3u
z^22Y%r>zP;f%cz5mv&4qnb)$JKj_9Ao5K6wT_v4jIxbu?i)Ao!c~Lg=(XU3!5a;#L
zvCco&o9CwIGH!}J9sm9QOS=sw!rQd(8n)ch4F7W?DSK_ZmC*|SPZ3%3ma<J>nftr%
zxl79Df=R^%Rzk}sO}E~A^^;qWKznkUqHJ>e>Ej*--fQ0<S^NAwqu_^HhH10szx2y#
z_4!vVG5hQmmZjUbE!!+%?mG3S!-bz|vV03YBkue9{eSbVeQ&^vMYh$F$$tAH`JSFp
z^k1;O>i9wh`Pu&8)ihONzFcZq@|tDkTbCyOkCO_m&vPW5`Ob3M`bTm!_i6F1E4Fhk
zInD2OKydww{!0gb3tw`%8?o$C;WEz|^BFk}M3{eHsTO&-!8Lg51cpm2T#;+9?ceb(
zX^&}e#Fe7OdDm|Cr@VC#=sD=}>+J5m&whS5HQPGH+st=iME5z{tIbEJ6iYWfEOYwv
z_tJ6s+G!<vPv0kdy4E;t`QOI+SM_!6q$Bq`wnzmlK0eO!^qlOXjmw-aSkB$79Jt5z
z)_a%5ThkWHO1(V&uKadMmMv%H;}XS*HV3|N9QgCvp(i6@$)B^8?C16y{ht#4bH{1c
zmMbh@xb{z8-#J^sXKh(^OR;qq<KC~inGsWNx2a58bK5#{rn_Dzhrac@3%jIeysrIW
z+;aGQtuglz<6Cv6?^rx5+c)OhCbBQ^Y*cpJJH<65PWns5hveVSCR#f)xHK^`bBf6<
zIFN9tiHTh)WW|Mt4xBuqK|K;56<xddjf=kg5M1J<8O`P=>0y*|YDx^pCLNEbijqyK
zA=kD<1}r;1hdHC@QRW3z_xUFBzn(-a6Lwsj>Q!rErSS5=a_4lu8LK{Nu58TRBy`1U
zLGQJo#JzkH+S`0jv3RQFNL}bthzX0%m&#fCA@9%@#pir)c9-g{-cs$nN%88lb=MAb
z{Lj6)I&?#*?pjY9FNKbeMQLYQb-hE39&GkHBOom%Q@de{%Y+$A7nt3!+<LH=C0+ea
z#jnRl&uwSFCO3h>>`kMvkI<a}a}9Sr88$U}2H_0ve*e{cODb)bUepU>w^;AJ;pKr3
z_fJkYub12F$ndJ3KjOc%Lz70zC%zxAuO~cUjJN;vJpRM}{(n**7Vdtv*jq8^o%90H
zAgML1dd)eROGO(sx_O;1C4D}-<&44M+|BI{I{b=!yi$gHMb0~OC{LLZ{84S%q{*IY
zZ`L$#w4Pn|%2RQEnUsdZ%q_nzu9<&E`IN`3NlueIjb|lYTJ?hM*1KbkEhZeEAJ06_
z-_t)S!n>Q{wblO#3PzG=x~DRqb?I5adXUL`gAr?TYwv`ND_w^gqc~SjEz4PXd>W%d
z%Z7uD8riz4BJ4p<Om5D<HB266Rc%?|QRp&7!P~lJ;`SC5R^BUpRjWON9_3lR+tD~9
zxli?3$k~YslL8(ca+O)RThLl9-#25=vRync!)01!r$tP?XUx&aGQopk$2FnN-HTc8
zZpn=Lw~TdzudSp`v54W@&DU15i_ZF>*XFZ_b4S|t2=l{VOm4loTjnhE;*LP~-tFri
zXX(CNGGWi|@5i6~J6g^#Gf&k>wPotB6XE7*uSylnYg)4uj2EpswNvNyr5!8egpC$2
zNH{3{;?OstUq+p}*ILf(*>h{c|1gKvWmo&pF~0ISxgpo{WT%MPyCNMYw(Om&pPXOv
zg6HyeyR48V%@a48xP#0JUIfd?29>znkXdA6Au2548(1xTNF(REoz}VqoISxOcRr3N
z+P$FmV%x{~*4dYhQkrK;MJP^mRg?IjRLN_z<f}lzdUF-GRjJQdK2Nrcp7H$CHRZn_
z4rfPC;8?xhu~1kkoQu`#-jz2SZU3wdoTjsQ>$=NItF|wzI;}S6+mxPb<qaWATc#FF
zS|FVe|1!(`L|*f{uFeJ86HDGbQnzfXSkNs$sjmFpw$GiX?nW-k^4cq|*P-@eF;D$H
z{iO=Z8Pl?Ng+>3qv;2d;(Q9wv{kz{;7o4-ofA-%)?XbY^)%8u8uE|VaXE<(6Vs4jm
zxOAb|MAfH(dsi^WQkTWEs#A4?L}G5O2w7y>RB<6i_{^28n8gj<Do+}Vy*q9RDLFRF
zv|Ad;G^~Eaw&&T0wQA=cvK(jIAgh<7?`3y0bfVre(Nn!V`AyRIgl7Jja9H$-EoX;6
z(>8}L&Ykaz*f;}H60RND_vESeZ*Px<n_eie_i7sjHQX(~WXf13s2!Kqoh+Q|ENxZA
zQZ{WS!_ux`8NDAD`}Ms71Ll0TZq(hFcQSeQ$9>-g<#_d~SWd=OR{WUo^y!L8Q*};!
z{Lpb+KeNeTrt#$1Gbzg~T)DgZnK{qIESoe(RZYG6*G&f|*Z+zfxeZEEBAn@V5t_?F
zPc_NRl$DjNn797sv#hy+2lC#mP@JuyU~=Kh854E|M&Y>2Ez6`BxQn-MYur(wdi%|F
z=O*<%vr_!sTDEw73_3D@ox`NgN2jH2vXzWpac0~8lsWKllIZ*?u52E^3uByi&CqRH
zHpzi8_P&Ji@>A`1-uKM!J|pHbefHTo2ByM=%96`Av}!g?E;(E@_lAX%K+mGICDOaD
z?|vbAsETp<ES*^gj6@BBXI~3plAdu$-jPe_*_6q7=M_)9H^@|(@F%;Ye2dWgch^EZ
zzpZWOI(hFaPm^C)slIjS-c`QuG_S6$+N7%`x%5wvr9I1D&SD;|46&zQj12$R9jUKW
zI-L08%Vwr+%>ip~G%-n9SYOjAV49KDs~7%6a8sg2wwz>6j?UsU0kLc?GB2f8_+)HL
ztm2qGbM65qwF{BwixMnjG#Z^O7C0sG>TG#&bLw5SYf+86u5JCY=Jfp+9=BUcLRSX7
z@fCUeSZCqc)G3OYN5#1h*?n1Xasivx2@SSS*S=jn-t(1HM3;R>t7#jH%k{Hw-M8!(
zn0jX8%z3vumZ&KSKG6BR$9U^9U-b=*oKv<j#~k20Arp3=(LyKb`&mJ8fv?ARGm6Hy
ziriCKY-Lk+OVjFA311`krU$Qb!W9)(M}+94iE+A5vX;E2a`M2HeQjzbJ5oOt%sU_y
zGrQ-~{~c$yb|%)iEu6zq!0>HD>-;Y!$*=nYHgQ@?IT^S*t`gj4Br6hjl_$w$HRI-4
zk~tnrjIy1w(}J1i{mseZG`iI^<AL>QLtT%ZKPSwSYH4G3nB9Hl<dq2ynh8w2UH)Ey
zY-YQ6dgRotb+9cuGNpjC^T4Uk8;+z$*T|kO_$|XAutbCL{<0lW6SyZN1$3@$PW!0e
zBUbqIR*U)xmk_6`Ycm!p?DpB(a$$pLW^6@zy5P+R5m!DvQdkk&wQI(@>C4Ph^D`|!
z#Qp4=rk=VYUoxg3KJND}H8qK~Q#Z!6$%`FP$<XHWsFzAGtUluAwlq2D^`|MO;y=_q
z7itM`Z<^a3D82gG|0P;&i>C-=3#WK`sdP+U;yZ8a8rB2y4`sBT*LwX{yi$9>HhXs5
zvgMY%<vV@_tN3voO8RlO)V^b7YY%6UgOP^8mGy~JL`rTw{lu_y7S~2KPkrf+$CV^Q
z`n#U7o8EY$<{f!z0qfeS><2z+^D~_0@7(FT!RF1h_~SR?_jMi>eiVOC_IB87lj5lr
zo19m=?AD1gS|oNgR^`^Mo7Hj?mpkox^}Ib|Z<)W!iK-c_44OHI#Co;EmLyI|>U+Yv
zYp(NRT{A|dhXK8c*X|q;I}vauaB@e)Vnrjzia6gHQNkbMSHFCap8Hzp&MuzV1t-^O
zAA9}!%Ptm^jhnb*@=|B>x7?Un?)YEQ=fRJ2`S*BIE%t2uZ_)aw!bFiF&C~MnTp?$#
z%)`v>1zr^vSC_ZVKB#!I_FeIbtDM1imBrU}{JfwZ)4Xca{fyc#?V(9!Y`n`JaNYEb
zc=0&NW@Do$_xz9Zb8hyvPd|R4(RSiO_0?Mx>i0aJrkeTFc<a`+4}YATwO8%b@-vKl
z8|FOSdu-D`UZc;`)l;6O@%_(R$K$e|fq#PLsRo`N-Yx$+ESS7YT+|tZJkCVDSN!#E
zHiy!gDnGA=bv6-6>RBx!CT#~}oD$1aI)x5J-}CHzlevxQI4ftGSwn-uq}HxAD{SWY
zFYH^t<A>5zIj`tfi~ZIey>#k`<Czqv97k;-2Ilbp%`;c9DK64k8nCLwgLk)=)*cf-
z?ynk;`JAqC^nbW_`a@ps7N`D?HEXZ4ExLYbf<lY0q8k6_D;rog_}nn-|73QaS4RJf
zPXo{N$9tC)Fzr)g5K&-YTOh#u(Pc_P-lW{dnymeDfe{Hz?P=$g)dW@AI&SUpT`iPk
zZDr_{!@qqAb4Skl+lyAMb~5~ybGYW-qB4&)YbJHCb?Q`@#Q)&ep`BdE6nGZ@ic(Bx
zSs}`>RPJ6<z^#S63hVTGcv_F-2c1cNV&u%JcPL1zbvncOC!0F|<?wEa>*Etml4e(8
zNL<Hz*ibPi?*@;-CjkN3oeey78Qfa$B<FwhW>R2eS|IRxqV2Z-c?x`jkNuX;=4xV&
z;?LUU>=x4ER&Y_v#Ldm(;Q=--?Nf_1_prX^Qqugh<`s{U$ojmL3~i;;&D&?N-eXgE
z)8ghY_qb5#6#G?mRf*#ZkGxa6@a~eECg=1IGfkM8AHR#caentMgU3zD9};}89htyv
zmhk0^w(cA4{d+!M4{G`p)u-9Hme>0LpRB%?@_AO5(^}bUujOSaaR>_XJ`y<VT;lUu
z(Q1|Tkxdag6FfxJz6DJbvYa5C->+P8-sVZs-4!hw3sUZW+Ek--K(|S)wyxk3hqGAX
z#rtY&6>dDS`F2D<&rq>sVqMZn1G(g5YzyR6Hp_K={5$QU&9`$U7EUK8`IxTyZ#etV
zhRq>-*B=#Zi8;XYE#H*g!PNH9|09Ajy9H&G&zqI<%JNU(k=h`;Tw#NhE7Q}7a=agH
znHE@d2P^ies(Wj3usF@0WuWTtD%a1c-0Z@ckn?kv_<yT-;qDf|+Ih&jvrl=!JGEI!
z9cG(!Z*mk`ctl=nvU}B}y6~JzwSfIKt(SEZd{}~6rMn!Jx>QcPa60zxVOsvjV)JyB
zre!roJeg;&_3&DsGvL2ELBq_Sm$QkNal?NTe=SRYe%JF_{QTP1*&jq41a=B>&(z>N
z^XZF(VHi(Ryv|4K6jq_1ZMsK|Z8vwVI_7(450?sujz(Gj>P^p;<aJp6({xoDME*^V
zDEU9Zo_F$s8%K)HElCUMd9dY0{esu^4TTCk9TN4;jV!CV0zFjioSzsSb(nkRiP(Z=
z0q0CkPL$5fGkg3+dT*iG*^3MpC-PjI7^Zcuv8h<@wIIVQL5?>Q6*3GId8R5e<SJ~@
z3$$G%pe|r_k~#E%k0M8=!d$J@Tx@sNCAaq^De`_&{L}pMLr>K)<~tH@s(;i%XTM8o
zn-Y<8;etz%w>yu)%~yq80!395b`(twpJ@`!<?5@XRXb^dXkJdluGep`aPx4V|L1W)
zOEzTx8*Qh;rFSP?G7a3U`E-Mms)^R_t=xS4E^pUwUh-j9i?O`uBklc0eHu-H9}5Gs
z1>)|TDsFkF-T!~7Sx|`%Z<7wU)ACu$6Z}?Jtq4v2aA}3zR^7FyRyKUmZFfyn?NM8?
z<j|Z~A-qpLCV$bNdqug4yFTLLnjU}sw8EY^wIfbVzYcjA{$Npgs3Z|=rhD)FyCYkI
z76m*pGXBSxWa2(~ub1(boXMKs`)!ienX)Tn94_4Av%s|1?9-<W44*&9h<<2SI${3X
zL{3<NgTd7DZs>M4Cri;qv6qvxxvvJ+E4N-r<B~X`Q~f?v<PcNhQ7h|X@oa2~E1bjm
zf^0gNl41_!ecqNAd?9c9E0NcSLNDsNUI>n3?KKln2=mQ}Sm)~ba<j_EvthHREMz&g
zrY|7iO_I}^)y5pF9e(=F@%#UTJ5~2oLlIY)-@bb){N-z}f4acOyQow?iI+R+liK-@
z5#r|7=Xv&h)$$H^N$(P<FSj^vs(5MY?x3|-gBE4;PPdsfT_LDMC1*u^km}8V^(qC&
z<=gWjn$p$vL_?fEsHuH(ljf;5f4Q<mB-mL=#IsP?n=5tOgQUaZX-+Te{5-!+%hKk1
z%<)sj_r-Jnw5b6atGH!ZuU2(RC7x9eJh%6azVu8ZzTiVoQzrYpERj~1cy>zRi<7>2
zZ?3HOYsUK<7ik=h**(c(<2P#oQ^il53_%ugwvVzuJ`KF-#AWay$KI(t=0scei@Xjl
z+jY7JMLubiUGO+@c~aV;7afa2Et{&m{%?I*_q5}Ar_g*2ri4;%sqjKeC(o~MwL>PQ
zs+?UY;?<ok+M{Y7BB>Ja%PGQeyXXmqWfd=trkIv2)>+>0DN@`Z>Z`ZY>PH{fG&{w9
zZfrlx`&@u;x2XANgD(fBa$7m@Y%LNHk8FAE6njxLJNn1vADodMKX&&VRi5O)HP2kd
z?v(54Nh&|s+;W*zPaSa0I-2lsyWW-U?(a3c6{a0BdgQ$*fk$n+Xl~`2C%VOx{EK1>
z9ltHn&j|Ftr1ZefNlogA!LOEcKaTik^h~?8Q1sS1(?tb)kA0Qe_|fEWplPJAnWd5$
z^9ED%^`B>mnO`n!w$I|4@l0~2Drei~EPr1`fzkl!|DoCI&E~!TsTd@|!N9;FkTC!H
z<W_s9N#SeNm>Mo5SOiH2s%Fhl4q7UE+;e5wvEa$+!Hq|5L<z9o74lxERKOgrwD8p8
zkFAr~nK<j5Blo@Qviim@{XOo$(~BOL-b~TfeU@bKQN^KA<;aQ(rz;gq4=ft*zy5b(
zgV(asDE%v@mTG0aw|Eot%2;ofaaWda-lQ%*Q@e9|=PvInH$)i(6zoLQxt1t!y17^0
z4^`-Ett@0|i*<Q!S6&tQMR7-Yd;2*{x5%lXPc&;p^zP3p-=+0kZ_-^8HIe&oYwnrX
zJbwP;QTh&!<a$4W`p-ukzkS#H^}R^(h^U7n)3yV?+EaM9e{9_Uf1=sejb_=Coo9<N
z@}H0VKheK&veBc90xwnV9yBk#m2CE<Q`x6N@rmU8EoSroNHU6~3m;1m2}ly&roi=Q
zrcl30TW9OS;79W7!?)&XC}u2ja}ZU@nzF0DNp%6sxm&zHLPEVc%BvRn?YY>zolS{p
znkwg^lwBfDo||tJR!`_o+p6=9dD%?91wxOyZFSb}vRI&Hek4rHA$lQ?TEa)2DH=-e
zd3$Fy{;xRLA2s3VP3ME68l3B0IKOIN6)xy|dPw=^r0}x^+pIW`IS6la;C6{QvfV-0
z&5>K_f!s^U1qWA8esw~CH<!EW&1|RU)+xPnzD-rWW~P#s+9s?t_4TY1ir*Bk{GYB@
z-~5Bax*$d<swzN_!THoht{L5t#Z_+F4-TdLPWi31d8T%B?z1n-5$m4K5;C7`W-e~9
z#loWPx}^p0M*(vg!KXId55E>9{HgHIU2r~X!CTAhuaXQC5_rrUxm^wx-Eic|h!mDc
zTHt@<;>Vk6OrmX<)#87qE_7aGb$<F8C#IV9(kgqxD-KLf-(szIdV2Y$6wM>%lV&Lu
zM8-xOsE%V@yX0mhS3yA7)$|O5ZFK@&OTTa4@#^>9+bt>wkNTcDXaDU`kDj=Z+7e!l
z8mH9he9V&OA2&G1I(%GbG|_#^s+V(4TufflyIpxn$Lj44N8KOV^LB{N$QGIO?Dpoc
z%Ab2CCI62-#=IbKilIu>;mJpRmVZ0y8u3){<*uOUB-yv$)!X*mPCQa0-Zpik?jGAD
z>sc<k(_&Ue2&o+Fa$++UPZvBbdprG&vi_UKX)H(mqY{};Fnro1+H4$Q?#Z83FDy56
zsV$dbK)rGW@5BqwcCNYKJV|)LeECQ7uR2>c?<=@Ad0Xlqo2ZDu?R^(QVua<I#BL-q
zm?ZLe?V1<N^iok)#V0~elp*lSvDXvd-qf+RRWm(fw)aedi*C2yu6m_#nUfw@Hd;;6
zx)mV%!y)O_l5^E2MG-jx+-`|`pZ-31vU%xMm;JX}de+@gUh(7H35jmG-_y98V&?o^
zc{;LObVqq6SH<dC3OoMW9!q4o=h}KT&*`<>{`0*{w|h2DTUe*C!0z{rg<>X-+-8kR
z>-N1mE;TuyY1W-t@gXaY85?{(@u}ZQ(w!so!BxThjV&JXJqi5x)F#Ok^vv0<vQOdO
zcDsAiPe!}#eCGR;ZS$#Yw#VPy#W&a3ziOHJKV{dSC2}#0joTcW7ltjrBKgcQqUHWg
zg%|SkUf+2!{qyH=i^`2s3TzxQ8Ul<0O&r2nF((Qb7#bNE7(5Lex|W48NARW`?RzZY
znLNov#&gn>!)**qECCB#4mEIcB=XHLnxN)5TU9Vm;zXk2^m)wNd7ey~pyA}ld6Fk&
zR!HK^Mt@ejRI3Kfl`Vm*o$lT;SncqCF~@F&vokk@%=BZvI4SmY*@Kge9Be8c6I@QI
zMKc>HU9sK}s@qs|TkFs6jRj7<mZnN<cg)u(w}eLuik(<>HMDhG)YeH#j*TrWTw!by
z231=gTwW2pI&Q9ko=a<f?CGi82OdcHMu}yogm;Us)`&7&+p8U|aJ2QPyIj6UAm_Qp
zpas+CY)U*O_Uc+>zsRpS3ob2xz+O?hYVV#`dR!NuhuAZ$arpM(@~r8zRUAc(-adQn
zzjkNnY{vf^4+=<!PrT%oDiPZ%sGa*Ho!@cmmd9S&LVH}JoY^_2JY-<c^|<B9$gn}C
zeO>Ad1_6H8Ct;52N2g6_Wloil7T9#~$D&?VMVFZW?du+9#C0)ve4H3izx7g}`z#N`
zF0tQ*D<-sO&z#o6a*>y1vX8#*71>DZw<;5+I}5s8neld^nV56B-cz}-D9@vup&_O%
z_bQ6JwhKg67=@g2sNQz<IYZXRn<|d#t_ByUL_hd?!=p~KMbK?;>J^F9zE4Z1yG{2s
zlk}Xev?SVT;>}GGn$vhE$UDu<HPgu6)N3&zXTv_NMSWabttPBKrrDys)^gvsM;_aL
zX)O|&-uvkG#vQpvY(gj2{>aW=e{I$4wJN?ZV@ka4T$vzZl6xSx%=qaRwMZx3uc33+
zEL=Tz*Xm>6?|Ho8jbf|%XciJL;-sl{VWldonZW)WS&Nm+`({NI{7+rO;UUn-q_yWt
z=-Vv|A2hDrlI(5EoM6+S`}WGEQ)05uyppysD+sQxJh}1eqBcR#kU$od^V4RnwL7#m
z>@3gM84YU8Ih$J5?AdKki7DOWxqVS}=bH--%XD*Prntph1PE~z9Gl={Q99k!GlA*q
zg;`Ivrlds|%+eOv%{{fYYz9lG>xJM_zLIA*VrIGuM5V1XT^~}l`1|u!mxPY!)P_je
zG#qDA^n3ZRL9}zmqwXlH0_TpOvjdj2d#<`r-_7~uKErfTH!aP%vt~u=hBB?Q+RCS@
zpd=JELGR8h%eP(vg^ip2UW?71u!&Q0wvJ|NyZUD>W2fD>_C+xX#c0oUxTz*0`+qyr
zQoDfC#LIq@g6`k6ocC?Z_1#htZ!YegCaHWiTwL=PS6XG|+}-PEKI4m?I^oD?^*syX
z_=T<%T1RgXbnQ!C^!RRGrn=J!N0tcb$Chi;v^LFjRFh~mlvyX6qA|;H&J%~ouUwUU
z8z(x(s7uUIRB`28$URjhK%O^OKy_8-#QugS+Pq6Q*y(P(H+55sz}mUXyqQ1D3fLni
zqL<RI`%IYQZpSCv&9$vQFC@bw-k2~f>B-HDd+8FgMMFSu8LLf<4R;t*s95Wpm-!mP
z9xMU|(>YWeHQ!2itnet(Jf@<j_+GgqUL@GPDI=wBy0phjp3M`O&Q2^@(luY^Z_$yT
z6DFAM{+~6q!=;6x+rrIgNkil*uT3X?C%IIfFN(TiV|i-HRD(1Y&$xG8;d3sEysZ=s
z^q9Jfd6`^SW@!q??H$JsuTxufZhE$$d{#@Oknxpn+XH(NS>A0DsQcKhe=daMzzHK?
zYbBLg4Vlex8q+pR^5}6`cB#8wYqES#W>4}gM{&)CUmcxnm$WWmNz_P87GO|G<QC8l
zvW}S8(ypUE@2uBCo4H3<-4XOESF`c`$Y<fdeZdMd-PsE|3SLZVNjN!e#mdwGHU_oc
zX_`5`Q)cg&!(sJ4V52WbW|GlKCA%#ON0&CNQd$t!C9vT^uW81a$SYiv7*x)%&hMJk
zBWHeztHp6upzEi{|Kp5;LTrpw4CW_!ED?Dba-Op#q+NPyDhKn1GdGgT9(eO=WH+-|
zB(=y&u4L6*BYv$dXOhy6=Ikw#HV522xixZQQYw2P`)UzIhDG(Ia}#c!*t5%JVS(Yv
zurC}%flV6Q)=K~NnwhnWRr8X_rWvUnA}>uFG)^t==5(Cz;BtD-FONo^uMIJ2H-)b5
z6$wq48+@vs$%Tv2^Ub83AB)Y{6ZrRNysA$zxcPuZ>%!wRUMT^}E7+ZHoSW0(A-s^;
z$2v-|RpPFn8$0W@oUAWmQk|D}t@M1+J3;N#_BA|B4qY#t|DTGR|I36kFDCKT0WFyg
zT&Y!hD>Hn=4qfZ{^kl!SV(<)>|2C6Uiq>m++RP5*+hfKi+#(#f@lo}X6)uYoupjd3
zSy_>=YwCv|lX^H9H#+<)adYhbk;a^OErvm%(Zw?_uu1S$9NWz+#~F$gdzEG=tX#Fz
zsp;K>21VJlsKo9U>%L7s<=7EWGdV1bBXurYkK`0jqo*@wPG4Z5CD)hrOzX3giPD?7
zDNHvyT7F2CaR=RK61x(!&aKZ;BU&T;)vnDqCYfwgQ#i}ls=j;H+NC0jAv@QE>?jJF
zBry3xr0183myIVlb4Mw>y3I6EI_T^qpY4kMNmrh%>wGM|pMTC2#^+0K8u*1-yyz3`
z3SgT$?XIv^xtYNtj-IAcn}$V2%b9-0rmp(p|GznFyKZP+2JhWY6YiTUQnyyFa5?ZN
zV3&pTzQ74hB3mY;&0nj!gm-7Y>nCp8h-;N9(ODkXC747s|1D80ah6`dy>^0wwV=*4
z*-kI(>Px4qQgzz({QYt*7|geQw3EKP*n)GWxMc7t{h4`l6vS0lp3>N}DkyZ7(q|8W
zHF{YkF>kxWd-r7gl~Lim(iVBVbe8hoYobx>9%;u!J==Alt8$Uf6th{6a}LIAJKDsg
zd3_c;Lr|lEX6a?`6C(R+x4yDH?|q}u`25XdB?mcLeMHL2qvdRi4lHy(vMO+4z_0ID
zRy|j_%#q!dsIttr$Ka^F#LKn+Ev6k5-^5YAQf6CWp{Zcp+5a2%oZA!-^T%@Sd$uWi
zw7l{jNJ$)&T6E$4p<SPPC!DQF=?|#=IJazTr8(OMvnBg%SKByNt&`7xIbFwuv)0$I
zIDMO7#d-dJw{}VE+$;>dq2czh<+hwn^~6iRV{4|pxqCw-m_u6iKDXw_eU_cy8F#ek
zMM<<SDtc3p&9+IgWnxc~jOmF<d{<{pJ0ug$C9?8#&ey(folnoL?`7Y5Idv|p>wm|m
zQmfnlh@KG;T{B~O$aItClIQZnI-asS-G6Js@|>BmA$dbF&%rP=M`7U$Ld+L!&Oc7S
zqAU>nz<K9#*%j$QmLD^^%t|}lGV+t<4zbi|s@kPBh^0D2ohtNaWVe{wmi+&}k-_;T
zmh&fz^RO#0Uog;kYPof(!oMIxNoK{Brm=sI<a4zoZ3@fYAy%$*+3|(BlA5u`NyRwR
zBq4!?ak`u14t>q}bksU(h0Y;EF6M)U%nw>E72E!Zx7kk4<+<o_L(O}miBLqd>B_*W
zcZGSv79!b`5=u|xt^C-y=3zz-V^eBNDa*v*0`YQYmfRm3Yiz$2^f1?XO||3*tnG1@
zs0>NwI+(0%Xw@AgDYo2Q`g(kvc-OCq;o9P9%YGD<D6(EmtIwKN->|Jd^kV(S74^J~
zHmMzLJdUDbPr7&xs<=&$;5p&!#$FO^;Ju(j)-_RX(RACaW~sd<@*&Hu=6uzAoYeGx
zcewQjRo0vdQ6kT47Tu7T)!50L*vuYOW;(;S?qal*vVizzN%is;x62NTro;%S1ZXZU
zKc(9L<3`rnWfh(7GIytXNjoX`E|2SX7Z)sum~=eNJG_#?LoCHZAVp$gnudVLO@V|A
zvkfzfy^l{+Ep!Ppu4CHhY`sNF?rZ$|>Ft~wT^hEz#2gUO&ouKi5uMzc$QfAezhqJZ
zW1`YF*NZ<Q6_+Ji7S<J9bSqWNTwY|=w8ikOyF2USDN_{vE~-zxtX??bN63=N3XX0P
zbDdIFZOX`BYU^?`z-xxmr}Ak@VQFoyxyHvQ&RL!^v9xXHF_8@`^0JFVr*D^!{xIQR
zdm2Zh?*Frs8=TE2g)wzm3g+bs<|S+vkn9LLyF==;S;mg#?e{Z$dasGSN*1@Rkh+m#
z@z+W0u7uxTMFpwEW-rF9Euwy=86grI`zBk&20f5E%-YYb7^iw%;NA6{qvFQpsm8XU
zvvodaSr&!A4xJ--F__&_b&g_X)P}TcVr?ar-L}diQ^WPnTQbag-WqEmATo0z7h_vO
z#l)2dr^`!Dw=`!w{j<{QNPKSLB-7A5yTC^C8+mg!q@P`(7jt1ojf9W7s?UKf<|-T2
zn=+ElKB_zRIJk8y>#6NcE>}CeU+S-7O|E=s;qIDzc6vcsilu~Lv&Aw6m(;?`Po`$?
zh-$jpwz{=z#R<3nrGfRv+tZX!Nj*&}-s~Z;X@?kJu$=Twu~SWRZC*{YZ5Ixj(4f4M
zC1?SQ>Q5mq1EHXTNsq6ooUdrSw^_cyLL;a}ESSslCP&D+lVUup<YTp(If@r^B}Q*G
zwSRC$e(MJX*~Nx}kNS)b>)1__(NuRh!mcfBn7jUolR;61)#KQ$D|6T^1>Y}q;*w0C
zvVCrxgjLFk)|uh){3lyywa=a4wV0`~SU7Ux{SE<s&3TzO#J(TRt(WxXS|Pjic(L1*
z$t+<)DF$MXjl@<*u@x%^|31jN<yOX~;{r#%1Q{(gUtl^@J<QE-hS#Z1|B1m?XVv@W
zEXs_punY;EnjNSWl^N2<A@P6a$3^bn{Mv4cX>?7~I<n~d@v8pF#im(nSA4YQP+a!L
zi7Ul}OG;3Ev4X-2rDcg6>z|(xm=iT`)~$(Lis@I4S9#77aG9R_U{Md3;N;kijjKOR
zR$aJZ`ph14%P6kI-m8n8GTp4AcZtQnve%s<pg%+S@>h-jAMK_c6q%va9H`m6wqRqf
zmdqp5**iRy-aiTRbGFelvDhRj{$_g|n`NMFYwPKqtuwl&uNGc;de-7=;#*cfo-6UQ
zKFNY@PL|`7UlZqD&t<%=vWsQ@i!!OZ#nV+D%K09V^S5$kHB+DKur0-~#<9}t%@&!D
ztTLk6TC+cCm|U}H46L2HLVDT`3pHuW?g*j(vobVYmx?RSk}NwZ6*OVFv9rb<tu)RD
zZmUjAiwdYb;LNtTK$PXuI=?S7I2$XAZ{}@I?*6>eYDuKTr>ONytov4I3-E2;WhuCo
z<>2%ePV)pf8g&g-=Oxzsw_>=cyoAwu!>)2Ei;1g}t|xwVo4G|;uE$gEuuJ~D(8k73
zdzuzHF)v^gjpklCb!CajjuTb_De23sW+eYh-n&*oM%pOGEZR$kWwY{Ql{1f)7l)Q#
z5A{DO7V)mNm1`xdL-M*8K|3atuH;snH~m#BQ_o(}>B3AaWNp8%oO4mAKegr(v&A8A
zv8>N(mW}OvV%0v1i7U4s+<83mvc=>_7g<eOx0N$Zx%8<2ztY7WrGd6wzr-C~J;c2V
zbSgR*Sb7Ar7RtVoxOcrK#(nMD1N-}y?6~daY5R0Xo^<*Y@0|%Ag}VZGn!Y&V#C>$7
zA(!Wxxejxryk-l0*V=MXbl#oF?q3gg%@$w9BQqn+sOnx&Pr@SaUydS%QEI=l)n{=Z
zt3Ht-aHiyl%0^~|jqeVLoePWA*~7^s<bMA6#!D&^lE3voKbE(3Q^;7{Tz5sD^J8<C
zX}^4k%ohts9pnA$o%h>vSAL(pKiPpRWkPd^qJfIzspqqH_3ey`+#|OB&(Q^+<~9jh
z-II*nwPPo@;_e?(BAGIicpGQf9Sjui*y_7{QjGQS)yD-UM(*KXEq^~b`2VLw^Crcl
zy)QTII%MOuj5Xze_$0;c6)A_O1X((X?wI&yf%zTrnj;<8+oxKc2|2rcX`Cn5K{Fv|
zorw3UjP8G|_b2PHUfTZqpk=FI^U~c0Mr#Cuc&x&0+EgyCF{tVLXuNC9>X0O_(DgBk
zS1VP8KANZUv7w<jL2g<%hxg1A)pAFEZ<VQ#s;uc1t|(ohxc8T8v6GnMyp<c<=16t+
z)P9&Fe0TT4``!}Iju?surzjT+sC^bNQCh)au;P`os?m4rC1=*%<~}Ur)yx)qb#tkt
z`Kqf6y@S@&q;z>5n6OZ4^WoFaW=b9C30!&6W8rbeLvy4mDnvF#nXO)RCL^hOla+}4
zoYnu;)<`u)Xhd!k=<629(aiMT86rJ7*<-6zzp;ch+l7UZL7YwY?UT!@Cu%7*I(_FU
z){)v7kSpnG>ap*u#>Bl_s-%-=ZxdVdU_ZBF-`1^SNf8$|*KA?cIX$6q=gb{z{&!0y
z-Rb6>X#2M`{mRGg3xauj@0K*O<xhHfmG8D1Tep<2i(QjraKj3*X)CQbcM60!u}ux+
zZ&@R2sk`v^rW1;{ea*Wh=0C0t7v0pbq)bv+agh+C&FL#4m*(sVKD9HoujzE(?@xDQ
zr~BUNz8C!V?3TR(+f#Ra$-2O+c)NMiQJ%!pJ8f?#=iU;?Y)rI~l{hD6UASuVYTaG#
zE|Ox8e1-B$Vub#G+No7PDgRXSjtR@$j<KJ()T|#Ju&d&t=G)-yy>l%d+}4y+KF51q
z>_XJZscXf>yZy4u>JK<`b-Hw}jJ?e5c%m_})7SIBQp;uWcaI946m#RYU*;_&`uBp+
zJr(vf0xTzr_etm-@e#XeC0{5sdC!)-SI0K2vYP*I<(|?ta>^(6Opnx`yWn_A;{;_#
z)u%zXrj(udtz^#;e3j>*!r#K^46jU4=gXQeX3bI(czwfA+~x9&KKF(#%^yBn_T+@J
zOuYAet`yI^(v5qx{;!epO1x&9evRkjwTyXE3h%DHEqqjdS1dbd_4a*ditcW8chujm
zWv+R1b*}J&T3@lo+?w?*43FJ&|J$)Y<}ti=p?qr9ktgl4o#(s`Z|TvTX=ZP$b1%p7
zMYf?k@7*UGc%HDI)wp;3Du3PL0G+m1N^R@syt+O4sZ;cW__G&w>{y>>+t9n^nVOVr
zzU-a@+|xr7MT+wB%I?iLP`5!qU!6Dmc%tgwl!x6mFBZ&cJKQc~KR0tzPmdti=UZJ*
zYA5$@I=f}&o}-3;&i36n@~M5Fq5S+ct2ahJKNqEJcy-lk*)?w;2+Vu`+DeG;UGh&o
zi4AfM-W#75N(Nm~p0sMCu))EH@<+vX>=RST+jFLOwdz8-AJ-y9U+g)2v7GsX*KtFo
zU?%S?%epuiWh|e+Xf%=hm+NlMv+wdQcZYWiCtj5P|3L5ELW%G9I;5=ruj$x(%4xp9
z@;a$@d!g2DJwe;&TTV#r?)Z`%qjx^gJJkBMscYleyS8tN=AYxSIIHu#=a?Kb*P4h|
zLT!>Im*NEVypAsUw2_V3;C|mHh5t!<8Vgs=5oB?Cwe@L1-#W=_r=wK`-!VD<Ipi;P
zjr-|iz8|>@U#yz=Bk8V+y}y}8>6fXu7OM!FF|B{dA($|^{u;O7^|O1$GAA8861U-@
zzWU*P=?xJ{$MU>R2)7<9m&se=)@r1BX&vj;P}Qt{0S3jy0|yv`wn^M^N)yozS}V0_
z73;>SYs0px#VyG&e0-owMwf3z#Kfx&6BRh;sd!G9a<Wa${+9DiuK!ODl&M+rt<e1Z
z%&*Bb{T~b0m1(|VR?Tv*vqCb17W;Ba`JPG>Ik=MfhQOX(A}_s`r1TZZGJIPeu~uVs
z)?K5t2h(OIZ`mNrApU|ym1U|==kLjDV|J$Y7d1w=ElAkQytj%~YsP8+L(Ee1b9!!k
z^gpS-`^VqPpDUg<_FD@+>DignvLr~wMY5r)!C}Gjr5+cP3NBCcP4T#StTx)**~~R^
ztL81Qiq&OnYphciTYvv_+dp+u7W1aX8L#Zy`S$o3SO=`|nZHPf)8<+EnHX6<os}lq
zUpK#<IM0@I|F1tYQ`m3)uPhTj^_TfK^V8G&XPPa@U%*)x)w5t<fR})#A6JZ)QPY1d
z=^UqDr(QhSbet{g!lrX}zt`+i;qZE}TxhaW!U7IA8HH_E8YVR@kl;)bTq-qLz=KPy
z^o${wW{ru)iRLzuO($;D)##e3CJGldM0q40-QNEyaMkX`S&HYEF>SZzQu<QSByi}<
zff%nfUk|w$uP~f0)vEpDj!&@ef@44O)%HxAl*FX@F2umsPS@wuYF<{^1}83I@6Cr?
zY-Kc#G@eo52r~AHoN`3gzsvBL28&R~7C&}{wLxblaA<Y1xaFu?zSLSIv?9z^mO*Vs
zz@twpI?bDcwoF{mmAAmy|B=?NFSicpuZXX2ooYKZA$H|W6YbmU&n*^n$<${GJT+xk
zHs_;+Wqki%{w`a_Zkb|mXY#SRFE9MGxcDYok7M_lKoiD*iT4_VSiY)M3m^FI67W%$
zp;IWbI#=Opl=I7A&Pg0o6&l&fW?WeyCD-t9<{McCjwwMbN{uWL%1^Z($9+m&79D71
zd2>OQZ2p2Ng|7OGQbXtUE3RF&i_3mfQJDUkdDEl(wK(q1o~Dqw@@m<!rBWv0EBBpP
zu+3<vP)$#!i?#Erjl6qRS7ct?#-`<BY$84T*(I)|vk&KMnC-qY-^aIU?GqQhcULE;
zt>t?^`HkC2xoM46lFt@Ly)<2Mh%@QwnFLn$_sdPwown&LVxKYRSyJNEe5cH2C##jr
zTmNR>Ub;(l`|DR1V!MP(|1-OudOdTOL#(!!|EyZg`gEolRxCmeO1aX$Pee4kgcD}x
z-b-H;@o0g^UD*zfEgk2~9=Av@m^Uf$>&mK&MXDMFOO$q9J-uyrNzCb)a{Ki^eXMu$
z{CoEE?)(KSSM8c(b@Ga<Pv#W6`GWcvedRe`t`J_Zm4UUsPSA%*n|0fT3qA%@v<@%1
zW<FW6c2#1|>;<3v79H%G+oj8uc(@?zp!ux8m!UPEJ9S@uG*>TVSG{?-XDV0uj>#fV
z)%SLuV!1fQlEd<Jq?UEh<w={SpXYU{*D8+P#3#J_($+l9SPpUVNk3*D4cXZE`Q+TL
zKdI|E-n=ej)(|}A$FxB>u^@N#$?3QESqS{^7IaBkC?$C8h=fRAr4EzfrCy(9C3}}n
z2&iU$ko^7H^Szq{gu-4r-sh`roqC4ztd>XN6VJNRoi3jO+SMCRznb)ExzfB8X5&YP
z=1$U>WgmLlV5SbYxXDbtwI*%G;W7+5lS8*LI=l*;&f+C<X2uHXOxH>&o2e{W!O{~%
z*xOhxN<04Hn3D4AY5v;;#o8U=Vp2@0zOSP1KhcR?nLCZu{;p5g95w^C0x{m;88^eM
z0+k%j73MO8Wk1r0+&LxG?^)>r&#pc1POgYJ^+`i@)kROY5Tz!*gua|Vou3$vEefdR
zS?FLW`jJJd(NyWZMr6#XcRs4DR=p1uuWXLHy=lA8E0_PmsZAy-dhG53QeCPkTCB6;
zADD>E3OcvU{ipx)i1JCNWF{(9J(}#N?RsF3#Yc@vf<CsqD>P=PvF7-;YOzlhI&<@i
zTT4LfGTk5xhRJNI)yKb_^PClQ_S?ku+6{+JEjn=0dG6UWI(3@MqhBift#WwLvD7D|
zNlZDUfKkBFCU=oF`vI2)CpKjUykY8SdZckw>8Xp*hj3TJtbipxVhKXt4HC_st4^q?
zGD&@wSn2k~yGiECM#;}J*zX&0*}hVl5Vi9hS9a6Vn^{|1H>bud)tMAzq#6|B=Qw$0
z!L!Xj?v^mj?p6u7S1$4+#&u-?t4RDut;Gv|Y-v~`@+k1f6qisVQKSD;98YO~jMR*r
z=4kBSnpd^8Xq8y5BX^vlpLX0;pZya;_J(FTuPX^UyYbi3f*VI-Vhx3_8=W-O$klxp
zeRxaETAy%}H-Eetc23>(``zXBb&78svbM(wH*QQ~*}2Bm^ULIuY*%{K{U(N7?`U{Z
zIiphJ;*89fqR9fE&oytFdC!F*^S;;I50g4fB7<tI9XF|EFXKGcqxN?5*{$b_)oNV3
zEUr3*vx_g$57o<R^s?M)XUc0>eQ^ns`4R^w-WFr7oL5Q5QcrUo4q26&_&$1xL)*g>
z;Y+`rJr}Tu^V)o$%{qRHGAfs=CMmH$-KoWVI&z<;)P_GBR<h`IzgKWs@PFlZM@hT?
zchizqPC5N!vcj)9uN-Q&AB!z2jQ=)?VV}|k{i?TmuIs}tO5eAcAXI3Upw4!++jN>6
ze{<ngy|BRjENzPgPaaEM_tY@phmElOYLl8hS0@#`+ODyfN3t;S)E%#LSC+YR?XdgA
z+~wWRtuxC}YI15Icl3@6_STg~CCfy3r)=JBwaSp!@UE+Vz>}SJ#x-j9uNWHoFz;It
ze7DeJb(ERU=b9;1b9iofPxL$zJ?;MDHO~ti_I;Z^$!7kk-t`v~oz5H=_Tkzk)Uxi`
zI=*SoohEp(e^U|g6Iph4nv1{qG=-M6ciY(aecs-=Y<j`J4}oR2Nec}0ne@fv4nI<p
zj%G>M*!N=9ssDfGh_r9Nkm~ib!`;H@>$Zjdj~+Y!ta<fBZPV$kT2n2pmvpWYwOqRN
zbonCgpbwLl+wwLzd~)Lo+}X2b@0ZT(4>jomyv5g?9}8W{b^GJ#H<dwc;zEl<p5Iiz
zrrq9am}lkgX>nw8%9>+8Pu*(O%5K>ID2zq>X!{Y@lUsyMQztL}dUCc+fV$!p-Aivn
z4|Vc;HK)`~IcD<WzQ@fE3`Ks6PX1%rq@cI9%R}Y(Obvwx+Cp0&)hi_nKJGhmqW;C<
zj_EBoE~LHo`_#%AcxlxN`KMN2=5pq7hfG>^|JA3ROV2+Kms<Ag;hKq_i66pveB|ZY
z_Rd`p&sDm5x8{oO1wWQ-y&%7*`v2aFT^sgCIj}hj7_HbItkpd$TiT3!a^vB>CXst~
z7<4Q$P)m`T|MQh}O435XAhifay|o-$TtC?yIk;oW4<Uxd`~N!cHTh_-Vl8r&dCm8L
zu74UkUs(xw&hF1ulx3}1KEGjSZTE(QMF*Gyd)W3^F6FR`^IE~^A;`OgzdmrD&`G5T
z>BVu2w^rYj(Ai^Z7$exfag(GV-)*P;^E>u5b{w9}wOV1fRmN|tS2t}GqvdC6m|Scy
zkvnC-;DDN(%=RB1;vUuWw?=m<++n(HEU?5vpw!!GMzhwEm5WqtI1eOih2PpT^T%F;
zpJLp19KOCfpcu*d@Py6OS$md5xX$trSw3O^mahMfl52MS7TUSsg+&DCp;sTcELRF=
zM9!MOh+kQzLG{X!lsgV;JnQ{!rk7W(y1RRy;TAiCLo;-hb_E6RoB3h7-XS|<nT}>F
zJCk2-4U99g=eTv<I@a&mblsw3YK4&Cn%3)HQ_O8R?^GTAxqxH;1?N`=N2E?!M8|YZ
zT(MvEvRNwQ`q_^prCe;>WYm{(OxK*ieKBySYoeR_qsa@5w7z_DS);iAqt?+0OH6}5
zI&jYDdU(Kg|3RyZ6+3<?ANisoc0b8uV$1v`4C)(S9G#oB<+tRK!c+EzQ9Mi&_bxR!
zrY@tZIg69eL%_S{P<oD&x(r9=l`XzYysEhb>y7ju{x-R-XtiJC|Gs(E8ZTGtNGYD)
zz_a7TsujkcBse*Qel&H=+Q43QMAi9}yW_2SwmPm~8z;rCIVtbcr963#+?>wuClAhl
zang;Y%lhN|BWpIibaF~n;m~S0z;e*5deO<#CbN$UPx|-5ea0c_>Mf#{U;0)#bFI7F
z<?>5^jmMlxGd#tQ9DJqF?Q+;hee=pd2cD!|^7~&b=~}|S=fsinM@Md++3S0mr{tu;
zjmA^_j$MT|oguFcrWh=aUwq8yiD{kSR$<4}o4&Xi#q`#BdgsN=Ziw=h;S~BjV^V~p
z+D;$;GacR^7YpsuQQ7U%Wm9dxEaKpAEB@I_WR7g`Es)-@&&PJP#+d_iIvJn#FZPn~
z*xMm^Z_oM_g*$yFb-$c+CSukF^-BvI7#I|PvapIUFf-^dFfcGQFgY+XaQtUr=aBK(
zu)x8bO;}5(V?*zuAWl!t5{(IG9lF^J^<*r(5@$B0sNC9YdLV2!<JXAHsH1&i)?v&s
zho(q=PCdhD%zjA3BYW|AMp@QAX@d}vi;QmVTruS(ceR%JE##YU>dW%0t0S)R#+Y7M
zv7_-(2xrLgQihw1{yUe6PB+WCy**&Q-`w3L33pbxzvj}l&|9%%!Q-AjA$6|}2M)4b
z?(?!Xc@?s=@9U+br}BE1g|WQmm8dLKNb9yX_X?4)Qd~V(A~G|DIr!_*xih%8WvOnQ
z>ejlE_nDkSp@MkR95xRgJ%OacH7C|8#n?^Qd9Xjtt5%2kV&Cds4)&DqH+O$8ZQQNS
z_5Y9WggpoPSq$6t-bOxn(Kwr}Pg7zF&wIvHkBPS~c@<YhujvuIlEQqkdAd{R*CWAA
z=3Cn)`!ScLTxs!8RTl`4T5H4|ELG%j+RC=ILRfe87n9Q+=5tp(V$hy7<3eZZLf41k
zZk!>DTg4YGP3~i^S$d&?sg^aR-=WZDiKE@4AHglrqBk!Ftln^F;gmHlJN>3kmfaz$
zySV40T*ba0?$SQr62<OSc`aM1)_3e{+Wguu*6)Fuxl=B>&1Uw!KBugPFJ+EXno0T`
z?Rvhf6^^%Acd7NQ+wt+a|D}i1FXWymHCf`Xn48vJud#9qm#tpwtsrmHk4O0Y`K?y0
z^2^dHowB+#@W1KwHJkLZUOUA<opmE9%hGjP^npXFI%^|ug-!@zduTdkwQ!$O*X9d$
z+PM>>P1$a4EkD~8(~}yzl5-Bz+>jvcZ<#FXT&Hk-dB!02R4hu{dM=xs<A#gbxAyG1
zY3)#9aj^USp0yS&_ZVL5v0OaJY*V$m^^9fusUm)Ukrj(rb~;X!70E4+cP#&6HbM5q
z&Bhmw(!wly`K<qcU!L(ME-NMdo2AHSwT+$D4%S;)Zd_c+dhzk(CUaelMGlq=D?_GC
zTc<SRlIX1&0^41RRKq$0YAvp|OV8p^>gHthz0k?lI9JEf=X8W(>&e3CYg$PQuYPk1
zc>Y85N&w#%vm*0*KNe4vb^L$VT)Vz_h4NKnS4rWxtcy%P)V2zgT=G~vLsKT~jc&|O
zr|FZQ1#51Z>UihmiCUG3bJ?x6<}Fzvo^tE}i{$ZLg;M+_cgr{F*JLmLzEE9u2lM2Q
z6P;gfnl136J5B1HgGIlP$O>IUdxjN`*F#FaI7sJO$K9}ZTOBh=_sLg>r25?75wWtj
zbJxGv{q=Qp&d*hQw@pg5-<ERb(^H+m@8742OmMS2n!T;hQLORH*99#%Qh02#WxQ$<
z`|GE^c6{N*m#r4Gvt%Wk>B5O8SU;EUnzYo*oN@7GhZ6@l^$MC=o@gc(C^*e&(#WX^
zzsdJn!ib?O&_{6#xBr`@?E(|Gx?7hS-E{kJb4=@k3X3scVylalki5^^?u0`(ourF|
zrK2}ES90kDgx07DFsrIKCb)=R*tdLI;hH2v@5KVQxR|!`oL%aiu~91AW{asHuWl8~
z&v{N?b|`PT@FFKG==1|N$;B-yo*wSrDR)`S%_V*<$#<Q#xNljBs`l9>eat@t{8yDo
zP2F%@?AwR-^l3A8@>_kH!ZnfOxyME8&l;Rci-h}~({9Rq5hzqP&$(NCVY0@THF-&?
z99MV(BL(h6rutoZ>HA??=7jG%aujD~DJ*{#$o;f8BV^Gjwew~twU~s0qZy8fS+M``
zjZR%(yE=h8@|yUXgpI;=kMDF$Gg~xuVa-wv_sXD&38w%3lCHi|Upj|NTgzDG?2YDl
z)=1uGCfnCnRH`jFxiTgvf^q!~7M+BZ%;FzoW^Q0CIpqKJ4<~=N#>z(C%}b;Y&u|D?
zCEK2Hrky!K)A5}_(4@Q1r>Sc<zps<HI8B$uLyU2*vxnfkuqQbpvkE#Le^?x=7R=3l
zz4DY6SESjpg&Q*?dZXnHTc$<snCL9sm3);?eZq&*x2_MYC&;8v+|Io=FkiZHmO|y0
z(AJEp+PyhXrtn+}TmDmYb?s5Dn@XG(TM{@kn@@LM;9I_6EoTWsa)Q5Y-9^3`7uqtm
zH)%Nd-?Z#3a9r$Vl<EF5WY?rpbwv^DOugK!saL&DZGF0AuA{^AUE=?ZiWTEGP59`M
zG--BdFwc!Oo*f>plb0ox)vS{Wn3N>qm$28r;q#uZ8%Nl-yj;b(?q79})x~zd&?Tw0
z86QKV=iNS^6q(`BSbDQTh4Vm#)RC&=Pj5QJE#eFdFpgQEa+0OcWzVEkK2!0F3XXZ3
zwb)sC_HEudc{Yoa%;$#g#j;nQ7VellVe-5w3LDlII0;Q%ZtGj2z;}|DUyv#I8lS^9
zSvK94bPdZH5*Hm`9y}=|oTT~P!z<yCB&*~K4FT;{DlUmN^P;{cnZA;gJXRh2c$So>
zz+S&it6y78R0}?0`g+kRJv}2q8Cgw_tsRR_H*fKpG;7V`J$8|+St_*^Ztps_-%05I
znb^8hs>O%g%_ebl_uck+e8sry`<%|@im{VtdR%N1HmXQ0dZqF5U5kK|<PtxrZC#7~
zW2~DOL?}$3wnf?Xg|MyY^Jy1Ec(=OsYA!M}Fz#i}Gm~yP?jW6_;mkEn^0uYUi369f
zMih!h|5-6DcKPP(`}xGI<|ZABJj^<0+rQ<l)1EE~uid!k$g6wRyhW?B_!rz-BXCsW
z$kGE<Eh~ML-}ty5@ttrdN+a&;uCDzWzt-KHbKF*i<?ZaKj0sMzKeQIGef8J7bE|1i
z$bGpliN2yq?GxDom!BvMFN*qjrbS|wLhg+tefiI3&RO~7)~q8>rZyj2z*;Gz9-7!1
zv?+GqYPM_Y<^OrRZq0HDUs<|UCsB3vneLpe8qJ~Jty?C<?AK9V`g70omNnX8?@kpR
z*mvedyXO6M|GsTM^7XBAs8Zk+$tw}IMo&Y0Ym&4-{Z_EM{i^7hTzqCmaMy<0@05=e
ziY)eA{Y|{-)>a1fXC9f;Gk5)FF`ige{wiUm-n~5q2f__^J9W5UnKb2ziiL8km{+Rk
z>$fVqK0WT9_~EC?(lv|pGY<ZFy7TmmbL;_gJ{pI;aJQGN(@OQIGwcbxwDZ^gnwwvz
zmtT8keBa!-ZF|p)$5XdTJ-&BB_(<qi?_cqpO*@`9{o8oF_o@SX)!W8POd)oQ-BUJq
zOgej^EvV0bsZ-VA$2|-ccOGP({hxQw`J%wFzZV{E<KlModdN5Lo2A^hg!ahQw|=U(
zbW~Zr%&Xye)!%cYSzub7*5y*xBf*|36^Fj(O*gd^cCWQt#>wPrG0CJcU!<`sC1S%g
z|FhH73cjVvy(pId(dbyN=#wO18rHP<T0z*wf|!fuiX2S_KbrI<N-Vb<m_`WXu$hZi
zG%0tahA3-&b>QFX!2j2wh4}*itcfj5ie3g2UAF~r?<<JhXvj5vapaC|0$VP!y?4y-
zo073%Ny5?#W$cOhAIe+PIx>G76`XfuGh}wH-L7`7MYJ`%mN}4ni2_^55C4BGxhLFm
z`X^Qvvxd8W5BD-tJkKOzqh8(oI6G~NMDPC{m3<x66C-lcO{%B;5U-um;CDIAi`_%0
zNu*U#w^dMgiH7d}Wy-aWvwJ$KkEP{Q{m7}F*f=X9E?Q7D<dV&K^Omgv{Cf{{f4#tO
zr|8AR*t72f|J5t<y_>sdJ!+e*WOI31&4m{p|CkIc%mm_<@?W?+iMJVWG%Ig;A~-j#
z{*j>dytbf4p@tC~W%mXKi(OWiy=E;CF88Q0AfmCVd4`d&BUcK8uvu}%+6kQ9fe|4c
zO==Y-THz(u?%{{0r_DW)wkf^KNTNB;Y@);WVzHCOUXPokG?l+q6frS2Ixsf6DNgce
z74Zn|l9ZgNlrhn1yWp#@ChpH0JrpO}|7dpdXtDU;*>c!I;6R4J-wTtu9`f6A_WYY5
z^yZ_*m&OKzl-6?hki!S9m#aj@%`hmRWWXy}pH<PeW@_ewuyRht=*Y_afRef*kK_nP
z<2HAJtq1s`6u5$dIx_@2`#3~W4X1A@pZ@Ynf#bu8oPiRtifm>|YzsAJv?|Uxmo#Hh
z!;Fs1NoOzVm~EO~eN)u*(ewow%9Ax`Hs0*C;F#gTIHT2a#)8TjmzBEOXOuKGcItLa
zpY^@N@1o$^nPQ%To+UqLZR4oj>A}CZL+F6V6lTR9&Wqd*o&pC?PUigB{r3WQo3g;}
z8y5Q%^cG%>pQ$V;bdWD4Q%d5BFI(``=oNY~D+InQvu3{Fr2F5c?Z>u6g=tp(W%J&A
z7k-}}wdZ2r#?QU17w5@t5t#5T;p<6Oj&hexA6*QW$r+~!d`#3_{CNJMh5bSg8{S^v
zic(MzmYgWMa#HrO8PkGhEm0GgX}};kYa#bS^H~pO&Qjn~v|6Op6{5=3sBFbBt6-9L
z*CN%Kiwv|DX=yE1{58=uXtCBT0qIo>=X5Tz^b)YhTC7|(>5`DHNYoM!t%=4RX~Im+
zl^#lZj{I*7T3jQR?$?;(-l2SW=F+U7DOX);quDL_nDYK-E{NhRT$9okBjLT|u}1E8
znJ1i@Vv{m0wy2d<)lU&?Tevuwe~G}s%kgO|n^+w)m;(7#7S36-fK_{;P3(W8S)UVU
zwOg<)+>urn*)eD43}%KIvvwKGnYnW2uZ8oq*yOoZE!wqm*{hYytX3_xTBWsXmF=v>
z3Y@F0v{q}ZTCru-igmMQ?DCqid)4asm1a|`G+lQ|9MMX@!BTtbM0ZS-z&=jrXe)-j
z4WbU7Q@#pBvL>$h#t<kdGS_FN$~05g@=5ZCF2*nT+QyQcaamC2)hvO+E}1Wu^Pg8v
z-L_elxjZOOp`gOb;FG6dY0zBe1@k{GSNUS0d3R;&%uD^-5<+GzV7<noD;hLOu~I@?
zYGZcTVqvC9Mz2<~D6+|{TF5<d;Y0(5$N&N9QyXofHqBJvnt5QO&hAZmw>RllZ!&tl
z$vAtH$^Yuj%mSMvUTw7Ay;;$E6N}&`z16JxuQw^J+T=cali2M=Jd(P0<`aUS3pq=)
zgnG?kdpO(S=9;i-?aQww$L?Meb8GT`k5=Xd4vD|GCr{qi#}Y3lBz^Z~mdw<QRbQ4Z
zX>EJS=^WQ78^3dVs)uu(b@YnQf+`=RGheOeo~YOKYA)YK&8$kHKm++%T`T8W?PPY`
zX&)u9QED~U#>LXJSY=kRI?ZNv;@&0oYNwRfmZh^fU3Tx{UcGDG?p^Cw?^?Th#R_YN
z4b{7(yLPRs-o?DJg6-k%+13J1r+3?&-raqCrexQmrpy_!n<Tc~usQuw{$kfsd&9je
zzN|Uo((^YU;$I+l|7S_-|Al&DlcE=HT)=-+OL~diwH^9V7SZ#Cvz}>1J>ZmK{<ux(
zS>4Cu`7d_V+BvR&JAHqIp+We{{5*ygoQxY77SBuL$}mcs@TqX2bn}9?a94}X74oxK
z4R&vm=-IUP)IsIh3*~h-F*9yCv|HnVwm|pvL)&ie)}C`{;(|k3J%@Dv9MbMN6qs;0
zs^GAd4TE`(fc2llHgg05AM6e=U~tvh6gYv^`OhJ@-HVoawAx!NwXAIMdNG;%p?p+z
zi<70moI{hHcWzDCt#)Iz%D)dn?l1XwGYWfkM&8|}@#K>rTTSc!MERQIWo2)c9epBP
zmcD(;vE$Wq)XG<>T))W1@ljBf;kd%YI-~!OdjhU3@A>1w_ec2G4~spa+uj<mnZH?l
z^7TeLuEyDCcFLXJIgfkGOou~@P9IjTKE-tLuw#z)qUyt|a!x6)*}Q6x+8UlCEP||o
z46M7>oL;28YmN2{KC94hroC65bYI*xn|Z>~^sKhz8d2v+U+cm%vkr0f>9{nw>n}OL
zw)o;PzNNxD1=n5Dba~;J|FNx3%~@vN=5uo%Sjwu-`+9oaYtHpwD}z70+W&i5;`^-i
z|8LKGzp9-3VZv9P4JR(nRTLE3%PPki%{}YDDVD-R5sVkv91pQLUSujf64`Lk{m!Wa
zvoEfQIpWE4Nlx~X{NIaSYgm=`UgX+%QBzkyjQ4av!6j##Bb)wfuiC_ZG^l#ARpbiJ
zk7vRz9*wHndTExxzLRH4b{xI1XL3T#R!u{$wB7m(mK5a1G;oXdPRd!AC3IXYNiL&Z
z{;hOd&hx$>HOtzfmzR4xuUoL+=}6n0ZE6RNJM10V-fqzB$v)Y?YlDQ>sVdt;+oDe?
zy}7iR=k(eb0lu{=+wUA!ID2?k&XHzWR>6WJf)`Frh!qfU5C~vkRFJ*7cg~TC4|eWY
z#mS_kaGHIu(~It~Kf-Q%w;ssY>$&y{>%ucD?w$$zxRsf4&Y~^3CoKeK3pt)RxNZBB
zy#GuE4C(UEPxMHA-S@n4UFG(=7k8W(f0VzgycTI$@V9;W$BMfzb_y`%)PL$U3j4os
zevE_5^KwguE<xr`D?S*mOIy`1>sY(blxyNz6|Fg^lz6sy&N(8!_mp+a5f1}~zjyDm
zExdFh@WyJN8*Cr%2R5+mp7UUz%@K(^izabd1Rm!KKE2eT_EzGXTUV;r+P)PQ|98vo
z;pOFD)~MuZe+}UGi{19)sN}Rz0o`)}i`taAS6tn0<ddJhKOn?P>B+JxtIXSmnJgcV
z+Y6qkT%+dF*LUAh@NfuI<)_FTUYj!#*H@SnE?+ih`kmc#?_OW>`+C^HQ?u^gU>3N~
zcJ77~4{N`zfX9a$A7hWC@!ibT6PVh2v*_Nl*>7*kugdLqJaaDm)?SIVYxbULh`Yl4
zfP4SRHLHBDaR2}KeD&Oit@mzOD0!*&J(RY4xy`A^ZRMQQ?11@(HggvSWL#SNMZ)0v
zjT3MG6#SQZ>gRH2={)&IR_EA0ZhM<`_gm(9y;Av)k{iC+^u4P{{4egz^(W8d-}>K`
z+RW7pBgOMlE(mgDhb&fL6+C;rq4$)>oKp&O4=4LQyRlA%Yu$rAYaSTSJskDn!L)@o
zLg%=GW-qn2wUd8$d5i4J9=+Qi=E*qRJo-;TpCwT!V@}Oosktu?PGMMn?P=DXZ_BTJ
z+>;;5c`f^R#;u2CAI+a|C+|qCtZ$9!V+~{~_A$6pC>qr(m@RXC${TsLM8UuXoXXx?
zRQ*qHesgJ}!-ETTr<nv9t^VBD^<U>1^TGQ?^PWvVd%ExJ&4WC*&RXBPID7UbYk`)1
zXAVeC;hy;Jdi9I?dk>HAoP6W=nlnDPPdI)(*zuz8-n)dH9@ecrVX+Md7rWd%BPaIt
z^^b-9ojHo{=6w6nxNglBr$@JXzq7smd}dx+ep2~->ubG%7T4}ZJpBIN%qP#`!@iH!
ze_#5xzn}lU@Xhxh3L=VvCXeqG=jI)_ebLVF{^fZ$)~vb2eeU$U^+$|k-<kbQwXj_p
zv}SUsPrHKNON&nryC1#Vru@k9>@Nwu-|N<X-4--uIpZ=Om#nv&4y-OJV~Th@@tZK?
zirV5!>#fhJpEFEccKP^)`P*IP{_?Ires%qJ$GG5_15a4~H_Bz+iqE(G)9)kDX&pV|
zpyY|eTtNbiK^rPm*Y9FlxVb6zBKN|>dwG6&#+));n_FQQ%5bntR<~*^*R|K7>+dtP
z@%%NdIk7?fNSm;9+>sfD51soY4ejn&C@8tMaof)OVo|j4<OGS@sWp<9=kKrDYQh)M
zIr-t~R<S@UpP#`m0v3sOznXI6ytua6SGFrT+c(S3WXzEY+B!=x+bxJWLNxcYsIquZ
zh~%V)Dc_WqFV2>n_K;;e&&r*tUc4_(8N6rM#`BwRhqOe&?VemV2{)SqzRrhvq|J4T
zXJkA%(GnOY_o{8xt@$wq+c&-0D5`l$&OPzYlC7_og#}7B$t7gJXVB&SpKZLS*U<a0
zPT-aw=N74UOZXmf?~H1#5m_C&CE{vQcz@jfmwxrLTxQAsKKfQ}?czUud#f*g)Hpr=
zyoJzW&slNoH$JDYpL*k~)BedkbN8?K)%N$D`5*n=%jd016`3u?bXG3<j&_&bqBLvH
z%{?kMUb^QL4u4oX`9xC3;nN?O&h!S|(r`T@@IGnH!lM#V6IUGn(tL4~sOFv(JVI;}
z1deXgOVL<8SwW<$hb8ytI=QeV9Z3!M*iNLF-^sY_(*4LqF|Q|ZO-!n6Sj58A1<hHG
z84WE}MypoDT{wJVdG|RJ<71Prn5^0}``MFo)2pIZ9vA=TwdmxsnRS`$Cbs&&+z(cX
z#eDd$^0A|7^B&u+ul0i#B-E5FuR4~Het<En^OW4`XV%^t?15Dm4~pA<3ec5e|L{SH
z+qUVo?yeIXvezB_mb7|x82jZw_d{~tZx$tl9?{qo-1?Q{#>oPmYiBo2D9vHGkZ`!e
zE^Fmg?Zua8xvp7#HR|BBg;5bVmAV2=B%P-`4vSo8sW*S|aZL-;wf%nS?@PCNFz9E!
zS6}|`u%5&FKZ;KsraaO%u?yd2bw%@sS?<vX%ul!In>6m69I|{)L6pd`sy~r^iq|{7
zET8Hxz3JNGz^~rDH^UQ;yh%+n-Ewt3quw3U#k-YWui_E%6uG$JXhRr}t1x@P(eSny
z&fb?jbJRrRmmOBs{=aFJ>7JQAf~$p29q-=rFY~tS?x27t%cCSyr}}0)X`Gqw_9#EF
z>vM64%5HC~RZACKdvb1m-Jfr}=hq1_1%HT|)g6-e`GfGB#rwVU(gSu!emW{=puK(R
zTq7mBD{>Cen~%!18Kpl>cI#BS9Jlbo8uPP@)jm}>T+X_+<+6+IE`gj4@w+|NBuk(A
znYNwn1ylT%3#+zH*;Vpo+WLb<tbqmx`-032Qy1q7-VBXA^5Xl#x_raxcek|`UoH5|
zymrgNsAsY+*EVcjDjI!5v31|h&qu#6>fZagqqN8L6T5B3My{})M;6H!S|%Mcko8S*
z6lq#~-P6Z>!T!ImH5)&3+3IPm-Tt%rzvza*0L>ZS`0M{D2i(w_qC0Q0{h<>t>m-XO
z+uztI)9m@Wc;UkHr{-*3^~i8Z`+2TVfmU}(nM$`bIj*?gImeA<{;qMk%slO4&==tq
zbB|OnWR%}{Y`Mge&iu5at9Ue56dje@xIrUDLE0@#Xp>jv6Q7O6UG{NFY8>0n^(6@j
z)FpTr{x3<76KRaKzaqx!^h?d_s9}KZmIf9>i`0U8={Zgcm-t^?Q1Yphc2>Lctb6}P
zWv0Llo%3=6`=ZXUH1KD-`n-9_b^VNs*|r_*&s@rOUi-=4ws7Oqy&+0!UV81h%o38?
z+lqwbUk0?tDLxL|Rz7Ke#S8s8D`rW&T(l?im{)kkf5pS=mY7SlY3N1<PvO}l^tdf-
z&MAXqN!OC2O7^BDzSW3JxoBjZbY$`ze-oKA9ia^+g5MfX7$n!o8@ONZD-;c|mp^^k
zH7modE$UcPO_Zz11c$?~HXM<k)!-z?q<J`0L|u510IROcAwFFPN1di_CYA*Wp8Gd4
zaqj%sZXNZRpK}_IKaZi}ybP(}f|KgSE?F;hL)*<4Ic2SQV9Dvr6>(TG_w+oTpQnP$
zIaYHec4(#C<k{TeF>}X_H%gvIIHFe^ns(xiPR69pM@fBUTPBADc1w8Px^(ZA+KRbZ
zS>2k<ttSpG6xxwhoNBZ9Ls_{XM`y1_Mh$;JdZhhj-zC<HiNgQg>lQ9x4ZX-GE;v^v
zJ2Rx_>5}Bh!mB*OoP_^hStPI2dLZ*#i>FFTqg>tVqZ(C<SpyXc7y<<jm$)%D`phtE
z2yAd-F-g1?*TKZJsA92wP9T@#!;SWfbX+y2Mt89~ZWnpWaktu|$bSAG!xcI;9sU}}
z`wVxm%}7uZ3SBYPdfnd^-X#f#!@ntAeJ^LGcg07p<Hn}=eKVF;p1Pf8zP~+;$=y5h
zwa;_$i$5iloK{!AXGu@KrEs3J@v_lFsoq}(NfFOYpXi0W^pbWcR&d#Pc!BHB@R}OO
zr0qs$)wokVDrABjXKhKZe-as6@1gDa=0Zxtxky*FB`y<OSC!fP>v-Yy|Gyd6gLO{L
zT#`qX#3TCe^(<Pdf3;=jt!9=mohQkk5?OsGYt0Y4uqEJu(B>wOjSEb!Uq35fc=<s@
zvEWbHuIvpDH>~KHuGsVELTHWMX|Z#M7Jdp`S`!`0E!5nya9z)eiz-%v7Y{u@yG>>C
z+@Heo866qZ_Hmt`Tf(_~Gu!P6Hvd^RrZNZUN?5yJKN#r0X5SlGwhwIPI}Y;jR<b(l
zN$(2pzNWzTQb_yS!F7*297W<iIr@x09rI}N44%W2P}E`Eko>%J(yRstkxdFIzCDW+
zW;Qgau_#RlO<AR~rNfCO%yF`-nUT`J7p{|bpW({gU&ioiovYfS1AR5_2kxXV%G7od
zPyWwv_K0|+RD*`#KH-a7gZgrkzb+A*KJSR*sf`OmL;ip8Fzn9ZQ7_ocp7izmyo(N^
zsp{uX7{34Gdb;GHe#^s#fDKa9v-~zICaHKwDNNRm4xAh=w=JoqwNIhA&N%9Y442^M
z9j>Zc$I@+faS7Br#n&ixY*=z*TI4UuRfb(34Q2}jxf!YNwn<}IxNigNqL{??qcaZj
zPTg7R98<`BA?4KRzXGjdN`<#|92j>*cC-{6SzyMQx8WFTr{Ddw7lDQuLCxO(dZ)Nu
zRMF^Y6g3fK7FuKw<9R8CXV&=@_ZK_-n5b7I`n8~I=a$R;J`Rq;j54f3b7F!FHZpOA
zz3R<mU-Ilt-~SqhR;Gy_S7bgma(}UWbEEG-ici62^>ux^BCF<1xo_X6a78}(flo-*
zL+*gU)9zP-cBbkr(c*DEc0S(ZON5?M#zV<l#_ykJ^_;MpvB|73`?1cgu04|BOHR(M
zn%X#v!DR89liGgGqDB&n+b+ni<nFEIn$%<{(kj(pxS_A_WBbWjYi@Lyu4r6tw8PZA
zfK^a<$186>ZN=TwquS$(mt3;y=zHBR8P#_=L+HMT<3Ud&Lyy^k24?rWG>QW`S`=1J
zKg9XCi)mt4hib5<Objd2tiDMnwm<gj*tgiBE>OkomqZ!E9>Wj3Ekev15)G$jFW}1R
zoo--JJwbJ;(cI}7y`Oi@{=bz`c>W6x3r_vs-}7yD?O1%O^TyBFiyn#ZzPRSpL7D0u
ztfw}yhAXHGZsyjz!6aC1cw11XM?z=j#;!o^uIbgAf3M=+bzthhQ#-#*=AM09&Fi<?
zYej>yUvm?0t_s+##P*xhT4UE@WuDCvV(E&!KOSv}Ub1<1=FAP7Ie&KYmO0F}v@$yH
zImJ4{dXH2?v^KLq4u`Qp^P@;NK`+%spZb@-XxBO2^8SUe)=3sa4<TQJLsb$gcdWK)
zU$hD2Sm1hzsaU&n|3Y@n%L*MG8hXZ)t|c9>FI=e;(O&h@Msd%I9xH*CRqRex3sfhs
zIGZh_V{^QACTCE@;WmlsQ68!nXS$hoiTuB)xo_K!jSDLlK3jd*cFo4k)*?SUB?<~X
zf-;y5AN8#1>fvpez~U%<uh8MNg2~s@{42U8zezL9-NU6<u;2MJkNH6!wIY+_H>3_U
znCNNsNc@y#+%vsYNRaFGma_+XQmQt~3-0KV5a&wS+P7Iayr9**f;BSOYsV&5!`aGV
z0tamvIsbWdoGI3AHxR15+@kYx1!D)xRz^vV4Nf;!vhPS~f7H~p_wja5Cvy#n_J0<W
zqb{(pTxbywSs6G{(P>ftJ13_AE8V6PO)sHVjT0@Cs}9}P?tIkMv`1%htAfThM&Wd3
zuCrOr*L^JdJ9e1m2+w(8Vcj&lrMts%r^ER`f%`tY|2wHHca{-+dU7@2N*>qK9eNs!
z(leR1Mx3>o+4b$ymaLx>@6J3qe}l_<C(ev6*IfsW>{-Dx_c6EL1A+gmc;+)3ED-MU
z7Uuf6(Tkhgbk;$Rlr>w$A~(#Glv*)+`6?Gqp5Oj_OuctcTJ>`nuK%rm;YWXm;e>Yu
zGoH@cu&Km)yLPjAgj@8FR>l(nZ46UbZ(f)s#Jd0U<TEDIdmZ-bXdkKyvHiBdylM?=
z;EzN1D%bi<+^dvmuCm}j)CX(6#j;n8t(Tr!;j@x`;;AMb1BIKfdM_<hh<f41ZYk6B
zp`#~dMK8;YO}EAOd|dO2+ta#p{twCN7bSY9<S_s9*;Sd<A^(Vp??%&qtyP;@1NC2>
zah-g6Vb6>1g)UP+C0jlU-0<!3<+cl+j>_B)tX?}$$O~+?|1CW0u*1apf(EXpJpUf=
z?=Cp?Z}Mudr6FxbE~-lVd1fyAA);jD+WCej^hU;v&r0hnLim?%XfM7r(I=H-JF}zp
zj=5);xa@>f&L+1XxVhG6gKw3_f)mm0aS80F-tc=#HC%ZmE1SZhv4ZzliBI!K>8K7C
z4F|od2~L++?knS-(0ky}#ly2TvM1%quzFwelDjRfJ=LP~LTBuw!*8<|-?_ti_sZn>
z3CfnA&N}kUb=t!0tfS7bdg@HZtEnpue(mYd%kaF=(RD?|(!y4U)#c=h*ArIvxXN5S
z^55r)%NB$4dp~hIcXRvj7@S@)FGy#@rV@2A&E+BsH+-)--_Uzw!IE8zEav}8<cN8)
zJVvG2_XpRJO`LC5?Ph8;?1~T&)IJg&A(&p%%xG{dx}(?XjK#zW+qr~Vr*q9b#=KI`
zaE|QKLrSmbmujqJU9?@J;Ud?n_H#N;2Q&8`f6|xz#YenrZPDS5iiPcQGx}mCMLgT=
z(e+sP;c7!;8BcxFlLl|OHb3xxdgU^w=KSp(rw*Am+S*>`O!S*Sr73x3YkUUlip9G=
z-}E#8x$v;7{kPyFd^4NomQ2+>vCMh#RGX_@4!@0g6}3IO-4=B&(&n1KY~uO(4a?s<
zx^?XB5IV8yy2s7`e>AtwuZUQi(;k@6zV3~Li>F0(Mf2_tO*@>U7WGOPui>z?bgFW`
zcF=HLkJI!*g$}vRVHyTH6^FMytXk6l*hk|*Xkkyw+0#NSpUzzVyhmfffnH19_^53c
zYi64}h97#Nr*l>!Sg5o8f$z)}b55QLXN|P3>piu}>Rd-_LRe<ZjGub<?lR~xOlN*6
zGCM&(DagTJc=4H5!S_7chxT^N_<Wh~<H^3Y+_SPdGNK;FcJeqg`ybfA{q4!J`36UC
zsYVt^&0^Q)N{-~&w%VU3vuFF|yT5*My}!6&Ps02)(yi+z&;9PShE-P}YRCLXpJldR
znh;jd{J~qR^@_LhN9%Ko6}Vot|G(#On|s%k=~^5vS_)wq3LpQ9M|BA0D!Uhnaxzxg
zbc@XR&G_WPFP;0RH~#MEcNROT*E@sX>g1F!Ce!{3gn9Zs;Ov#tVm6ukK+1Bzjc&8%
zji$g(nNAtj3kE$a4skG)&R@te@6pW|PQ4rJg%*iiJ9**nDo5X#DF-~QY7-y6oH_sF
z%(a25g#ukQt7gST&iea#f6tECzXhRFUToQXLF({=BN4~uzHicRD&}g+V)s#e(7aLF
zb|<Us%ci4ewcTWP+{_7|v!XTARc~Uzl2eaW|GyBj|LJYO!g=s!Xp@72E=TySpiA8k
z9-H%Um0jJ@_k!bvwC=}-@aYq-S1k?a`E#}=_QC&$Z>~q3T+QBh;}PR(g)p`L8&6Vy
zzGzgP_v}f7W(IS)hsf=fcLJghtgjT46MXpI=^<l4uxV|g-h!R}a~{tAc(Uj0joF^u
zV(V_W{k9BTv*o8uB=h4$_GqsPA$iU~?W}tOZoQWoY6UJST;SDK4!YK@B`4eVVEY8e
zx!k9=rhHK}jyPtuTH=|vyNUqo>j?`D6FPP+5xO5R+jhy?CzEBvI*v0w*xvLaJGI%)
z_?OO`3JKBElWi1F{+P*sz}J&y$}5h#)Ky6x>HEY_vu){(n>yvIz%!jiybnAxPlm5n
zo<D8l<)<xAm4sgEJ(wV`)NJ3H;K&^t#WwF!tNoYDcYRZLh9`FaU(|C<xO0oEY1X0)
zK|Q@k8I@9!k$0!x+v1kny?#d8G8G-80*TJdc3sDGSH|?`rp&<`wwgr<M16hcDLLoU
zR?ht?t$VCh9#5UolyP{@3}wb;XWKgR-<){2Zpq}Gjp46)uQOe{aL?tzmf(a9B8Dz!
zw4YkWhU>)~yV`$w-EH>st2J*hT<Yt!-kG*_fs53;^(tx0V(qvNeHEC!Ec2Uorf6ID
zCDyI84n~B%<mT@b?TVc@@ASimai``-h=%BRr{2t&s(V>><}QiG-l&|8SR3BMx4P#{
zUB)r#1k=y-ML#4K`5t>@*leD`T6H$>cfhLT&$h27^kt>o($1BwE!=!uN`SGg|G)9}
z$)dkkPGXrpWlfAbUxbR)o$twFhj?G|f6rjO9%1bJ_HJ&6(1X=;c0QTZI@MS1%lb!5
znjGTMH|zu$d8aZ@o4$C$+<O_bLkfBrLOAb<-kO*DVB^#Ks$bp;Y74B~A8+RMooCmQ
zC(As1pLv_+J<j7^c5~*dl0w$@H%B`14*YuH`RBpxmSDpwjmEC7zz^$eW_1?0y?&>*
zT(D$a@m<wRt9D%b;%r-$zwpG_nFTTjW`3LB(b-n_f}7<-ho7~x$&p8Y!h7cB9J=vw
z(T?JYe&7E@mS36recOf<y#mjtq8g`HJ~MjMvrfVP-=lNv_762Ty!89hvtVAg%je9n
zLbblDFSEE@{~Ig0M)-XFR<qiyz*T{7d8*sgKYPS#8s90YZ<Wt)z9O*rih|(=y$QKD
zPe^=juP&JNNONLW&Ml5bhpjjD^!cP$HO_W!>(_sLX=2PLej`sq-WZ$kEqhi;1uW0u
zi;S^<p5ayYtZ!P?-xXhEHJLScO!)C|-#0-n&X1n++ET*Vs^u4@>MdaEPq077aW`_^
z+j({-Yj1roOZxE2BxL@}r|qT8i>^uDo~vE+u%bWX*}@$Y116@49XdC0`;Xb&)|#^4
z=9}_cZ%i!coKbGgwf&8BZ=r1I)0%$cls_H$6MuYT{^wg$%KGrjqdL`z-Nz=f_<soP
z6J)P1zbg9UR=VOdoo3z{P5;9R*!ijg5?AfI(mQjTK_b6=_sn;n&mZj&O7GYlpD5yQ
zugEp;apT7iTM`$UrM;5e74SQ8nVjpc<rPfXf{PQYs;szH&j^11Jd53vcY`<Y;UMlr
zC*>dyUhQMMrm=r!Twv|C|I>S>jw)t0J`s-shy9F9yh>lpFU+&=_*QfEb9n!vwPBLk
zTeDnGB})h8p5S><@<HWPW6Y@)HW5vyPwMF<E}AlP(gNQ(%qh=gZnQpSn8n=4r!cEX
z_0&{@>1?K}Ht_^6^K2Fp(K1oJvMS1d+DpqR&;6$8m`|{}nboj2YBArYLs>?uo{Lm!
zgKS?+&AX(@yzR|i&w%y!8mqQX2;}-|5c9vOxLeHil+cUlqpIu^bfVQ#W}jYqizj3A
z&jd|#7R%~qHVZ@?uQ11~w5upvpxv}FQMhLBE&X?4i{#~1I5+dWU$|84vE7MfI|`>{
zv+M1z+*5Adx{dEz^468p?Bg|s7hSPCApfIr?>1%i7X=Frf4(1Vb>*GiUj6$0Y(F-A
zKRr+P&$gVMn->4Bmt4xiAyeSWSUT13spbxa<`b5!x0`!RH4=R^@47zp75D3y?c+8z
zfK4)Z!orITTV2BYb>B|-BI9_xFw8Ad!*!ctukO+#*TrOf4mx_r{P18?S7EfA-tQ<{
zImvU*l!Ypj4Xiq&B3En;Q}$X?TDCM}S>Vov8O(yu7exM--qso&ozo-q*mriwPEN_x
z*znU1mw7Mv>8|NH$mhgubmW5Lou`okjx~A`aV}!6ErGg{ny0utlsZ+OOtbvTa>bt|
zuu34n#dFq-h=WgC5_~zYg*sIV8fCniw(`=CpaAI)Po<(RPc0Rj#k|$<aZ<e730BdH
zxm#w0Hu5UGaa_MlMMu0<v@N1wO6FFHkV4D0#C6l!N^WhPn=5u}TjVygrE4GXU3lvG
z<Wk1E9QCu`R(rEdoRHHKwXEvSbe5}KbM{P`n|j{!>lP!fIi_ky0_>mvti0gz<>2&t
zwcnqXch~*id9h1&UB$Duh9X8+wvN)uJu4$3aywS_I!@F|<SbmOvuV|T-Q9nT&X`a9
zb%{;N)jhrB+l^<c%@fb=n0nlf*_NT7SycAQYIix$2^;79d0~>8&v=@(-#eJ~Nxq|)
zYQhwQd147!Kl;LA_H7e0H$3??GjiMJ6N;^uBCXpkFU&|;d^6^DUF7v=aYg!8Zg)?-
zm0aOeFST!;tB7jfBDb{#PXZS!@AY}Pw5(M|EP{Jp&826)g+6x@Q+F8t+Oa^4{gY={
zYU$NRm#zOqPK)fT`{|hf_-bgxv}+}rzU)a^l}mSOhqZ~sFMHvwtupuMqik=@*E&mr
zBTW_Ko4=;Maj^Jxbb8QfztfZQ&s}=5C-Jh7N=fRuX|8N(x6;CH?a)}+Ryh69y6XB}
z|97nAe(k$%g13TW*t4aRCnoAGO%P;aKcI9+CcRPjmVoCc(?t=k%N5q%IFij*HsSjb
zb-VkSGi(*uHvXNk!T$fpGYt~SA~UXNr|J2ed}gO2=`+>QoTZT2dqU90#ybb=4KH6g
zp(A3=%V`+mHbu6r#C`YupIQy61y|3pG|8t=)%Da55p62kvRF>kGi#z@>?<~*C2E?T
zB5}eKrucn6v~_8V@r$ZNKJgI!oxUf`wn+-#<5;A&e@mi>(;<a9w-nW_Zm!g2_vUxe
zDiRb?JyqEEyJNA(X1jT!=~XhHZUk&fvYbC>4fpKTb3Sxz5}7t*VTvasAKznf_j!s>
z#WrR-^Opy6{SV)@aO&ohR|1$Sj`$wxQJcH_QDD~#Cx4?howS7=xk)LLR)=J3&ssBA
zNZQ$2FG9bX+bJxb=kko5IgPD%dAt(1E{hz$aZZ1ow8NBjTi14bcIKv4KcC9xw61jG
z>A5M=(ivP=q`fC*CF^VqI&-RILiOFv=BWq!6<9CzaZh5j|53<VuV!X=zQ%Wc{^lhg
z^hy?T2nISX`}Nc>X~m?O8;>M8eZ0VsB4K|q!KYJo#!_v*W~u+J&&*h3&dMHPJu~;y
zmgUSp1th!787u`=lykZGniixfFa4(zdhF0V&A=ob`GYK34pv?lS8Nkqp{uEvCc|<4
z$`9R*58HZooV>InD$&0p>%XQ`s`t{S2}~Dm38)m9o)S}?>bzZh?ouNOo0BC*Mhu!U
zt6P_HO$`y7V)^i5YH+t{`rCq+W*6@?8SlNU8gl1BSG0h~=JEnAag)<Gm+?FhOAqYT
zwqUz#e(apnl+T_mNw2kgPnne}<R<A&&(K_dpk;f{RnFHnGlTSYoYGN|afp5)A*ys$
zwaQues-1t3?aO;BCY^dEmptv0uU<xf_4U*+A@#}ozLdJD%qo1mqr__e#-`AE=}l+<
zo!K<|&McmqO5??T!IvETHfHZvKe;$w#-nZ9gX0Qnn^-l~CR`6$dUp9VCIQvd7yFMp
zq^2=HIn;h9f%{_0j$q~&iNRiir(^b-O#2^qNrx|T)8QYDKh${p)bs<VpDunitutoj
z#bm|TQ_>ncix1p%I?LgtE^@Qk<Zxj@^RC%e3tyb@@P4V(+oH5YZKmM<1n*=i^CQP<
zB0HCA_ulvQxPEK1)pm=_&95C^M97&e&r~?>si&#baqzl;sdDZFrnZbVP7+0WOJ)B?
zm>5PyZ=7&&^KzMRKKa9~XIl8$r}SoK#C~iusxi7$9>AHdHQVacqBK>5+~(!cQI~$&
zZBLbOEouumt1wC6nyIGID~7UZI#Zu;t=j);>hUjYPFCHVwd<M)&*e>$x|u<0navB;
zF7GLqxGlLfN^NIPszy|B=ImwB3ExiJd@i3o<D!VO`tSd~_K_XBGdA9g<C*lpkvV$N
zLbV-pcGP~)f648;v1ay?)hBrvf4<fe+q9%z;nt1oIT8!>zPdZe&+C;nJ2UBmO-IPT
zd$+RgILzVS`%rYH$Xso`vyD@2+Lk(mn4WO9Qxx!I@J*lis3gqt{nX1#R!&}cG5GtT
z*eL}K)rWV@o9xY`wff4Pj0+`Eo9;0wub=m^CoAWD)0P&!FQ-1YiFMzPN$)wY@{fOh
zwylYcddRzqYg=^~z6mwNJ97o~H1Td*nXo2Wk@4`M-EK;*C$~oyabHt<k$XVVbGc^r
z;f|s;JX?5jGizr^My;AQ?cyJ^RN>fLe&Q!CuQLhImfJtcsry_|!k;pO|5IPQo8NUO
zbw%-;d_k4T!P5^+)}F|;BI+Y|Im=P=QvbW9^N)X$^jms;Wz@US9o=s<-|W-6|5q;0
z$0)8uY^llf+<f2YS@$;R*55v7AMW+&weu}$jqR)=o|SW6A9zx*-;KNHvm<}egED5D
zT@PKAyo)wH-gCA!G)19tx`~^@l@+3<t86<m-UNtr{p8;#RIV&_H0W4HgjV61_NUkV
zL+|DFNVOew3*T@=THuKE4kyjOAvpmL_!R?h&x~x|bK`jGl|;vmq?2caeH(LUc&y<$
z{<<}$Jo0FnX!peztf6NU3io)O4{g2b;v3U*u_0k*#N6Aqu6L)#%w1c;nLW4n=fh*q
zj{cAG%8_mU(;ffd>}<ot=q-%1=dj*d8mTSP&v@;|&#CSqJNz;jSywQ&*Rvd594qA~
z*`a;Z&HLy1MY?eTTO6N6@-P=g1z$<jWjua=BTGym>+zG5Gg4#<4zMUbyY%A01j&k{
z%R^XK&Tvrna}7Bv`uyPBfD)0_F_#z|9yt^&5=>nb|7Yf^JuZ?t2U+f%j!bbecbr<j
zHR<(J-wl7I3XjCZN^oz?;qa*lf8TRnOX+q=rvD}u<)jm{e=g<UB07It0+01Y|D8R@
z{@l6Hv_sS+NjiK_yw`#I7Xt5lS}0yuJYKwj(^1iV_7x^4Sr<2-17Zti)J{B@Tk5xN
zLc^(wsKyP-l1=|_p8LQqwMyv!+cvi!{tIt5$vSl2IVzHsBXKV?%;8(K&&HNRijS8r
zymWBkr7t&ITUI`H3`yuY81Xz;#XZYUazaCg#U#tBGsjjQS@}|C>W(|>a~T{(uCM=k
z)N_i;g}pMqLcLxlHz&P}x$w7bvqh(8$j$36Col3ooigL9r-bU%Ese4hZuPG1&is?O
za>Ly_Ih}ic2<GXqr|a++z4Z1fart;fQCL-Mv#nd2k56F7v62Rf2exsVN7!^j@98eM
zc>iId%&aipotL^NI3B&h^n1#+$TcpqO&ncYg?6q<Jtlicj_<&dM4@F*Zsagdes9z2
zR(WNH<Mr^0#67$Y>~F5L=BWAP{8x+hkz@C0ynp0Gs79nj;-xc&99g@1&RY84e!yY+
zRnClavDw}k6@v2(rya8jR5q5A4a(!p?)5vCk@Dy0{HWZM-wrCB+2X1FfjePq)9MOI
zhLa+~Rk!wjecJU@-uG#|HcLk*tH;q3K}Ry5m!A`<^plAC7Alz{<GAl)i>6AlkBX9%
zdQpeciIwqhmhjg3b=2i4D7P(2J#;(8Oe|!|osvCnpY}X+yS3n}&D3{^{h?<P_%6*6
zyBZS~_~L3$;B1D@$x~Q6>U#OpB7^u6D>sPUUa-PuDes0q!RC37n^xRxT5&S!&Bekq
zUUM`j$R{o=TzD^XN0WBI8HOM1D^%m`ZarP`^?&RUQC3NpD^btp#!nKP<Q5Za6!&1}
z!+^iXv+gu8+TBa@jj${0d?9dA_0tW{3Z;A-g*b;t%J*71@7%Z@xJc#KQBS|5-iS*l
z8`4y#`OGzXc6x@~#~M%VGqZ%2b#1sI=aV#X!_8aozRK~G^_vMQ*S}5MQhGZ5kHRHQ
z`G2|`J4;^KsxFVepxpZ4x?@4xryUoHs+7I{h(0tqSHhS1|K;2YySoQ2ia4wF^ZkA5
z&~&!VagKda1Cv<C{)NwfOD1WZy|V46w9kSzyPG#^ySmmDK1lX^-J6)^9+!Q(L+PwU
zx9tul&4PnHGQNw1T6VDA|2apSXVH;_oCCT_t0w<E;j>HT>Vy9tlYOU7l;FH3(t3^O
zuD?|5^$php_GL`<nR{c$ip{@R-R=pj>X3f3rDxX3vymH^G<5`K8?amH9xMDZHE)~i
zfh%pgIS0k2IJtz*x0lL{txK2eQJ>tW6B6Q+b#iX%$+_#|G{r44tG9^+Etz!p;f1(&
zTywdEUo}k3?oB#V6PM+7>gAG_d14u@JT3E<MJ4NUteA38B*$@W3&#qDtV?Gucz#=X
z`iY!E*JM-1V`n`>Ur+6yDR#Uf_H^Y|-G7DM(>IDRnZ?&Ebh#UMTfyw^&)8|d<{T^9
z)Dg1n%=OI23_VWTI!7E1c)Ascr?aFmuDnsXN=LG#&3=`{5>1t)AGcR5z4C0*|1Rkn
zPFE{=R;-wk<M7mN+5Cogm)aWc37)*ske1dUwYJGXS3mU$ccoBB&C`;IhgrWgS7n|T
z*}>)A_u6*{S8I@H-V2Y(xfgefadKy#oiIf(fMMyuzY3jMGY`n6eiLHNUg&k}(#!6f
z7n^eC_Ph&>)AZjH;1->dxW?E0M382OMdq@VqA^ZVPDU$T4;^`Nqvc%N(FKW9RChW`
zTg+M}!D=XbWOh`9^{RF8do(ZHXi--(aNM&p*rH(JyiApQE-dHospNg|=L}chwQerg
z+c|DG*_Z2u>{xQHcJg{b{WDYBcQ0GbeLve=)%#3f<Kk(#uUt;&)+HJW-8anRPF^E!
z9XMs%h5voar)h*Q;Ib}D+!XVQ+0bLvtLaR<NA3z7dBB$QaM`2IBb{A7jnW&RA3V6|
zpYF=%$K061R=>aY?x2}&il-JwlJ<pt>twEPZ2sG#d1?N;)QjiyJa&d%58n7h?9QWa
zPfmP1rOp08iq$}WXV}%tKRm)kUFUe}N9cBbZQ{M8$#MHd<jQS<%{Hoe^ZEsr&5S>h
z5nK9nsz=VlSAt#>Ttxk3@0a8R3WiVqB@o~v?>i%}Vxob}{4TxJxU_GPyIs_3k6w%A
zk!rpZ)_?Y0lZ|eRjKSMS5@#l4?%C*aibd3Wi<7x*mQK(m?y8fkCpfOp<mWlI;R=(|
zr3(vo-IKXF>#fGZwM`5D^9xANm+Cy1UJ_ZgP&8`kF@;6fO!7{AF_qz7dS>?%Q9-*4
zBI*yPxm<9NT~{K&;?8pQ$*$o1R#yGrpT0Mp_nvbmxyEC&k@41Th0QL-ulMEEhpD~c
zy1Y3^jd}SxDd)G|>$${z4lJr&ZOLpbc5#sc`+EkS_3=NxeEhWO(>sn$EL-^|1r{)*
zcJyg{-r6%&=8JF7CcWnTZ(1Fihk0W!ayhmp_Z;E=ahB8T!_uayQmr0ciyUA7n)p?-
zlP{lTk#yTT{pKFd8GIb;GYd67vlJV|z0cgepoO(sDdBVEvs9%-_6xBhX7TYSTql3^
z+-~B-e&J=S&Q-3p7WxuXZ>+wV*T1XizyH-H;n1rEn*UmRIk*+)uif-^ri|~g)FZ67
zw_WKt#J}}yZiw)kD`$m|iEu1k8j^otzLeyH7{}7y1(9OfY}z&Q={K4jw=^lN>=2pI
z@b_kVbE%MqBx~c0LtU@>Hrxx)+~Boc@C8Sk$u_fXCeF)bcW<n(>*&+*Jm0w1^ID46
ziYKnxGj$)e`Fo3pGRiikT>Zqy?e|DwU(wVHwO0eatoCdX?PSuCSB>RR`(f?-_0&%T
zf6q6)d@VmG_0Fm(RhX$PDH6_InH%~eaj8#~b=0jL7nV1!Seka!O|$UYbEy+gT^HBh
zl2Dib7W;ae{^~E~^N(CQXSnrLp22eq>D|4R>bc1a{x7!pT=a(XsQKrQ-gUZ;J*6K$
z$-Te2;WMAZ`Y&8Jw-uKzd@Cf@{^6yX!1d2NY)!A9JY98_e@({8yWjYQy*O@5E|~nL
zXWE8XCf~vb>!Qq$-Ru;Y+Ii=u{O!yW9a%XB3HBEQszg0}Zo0JHG}^T2`E*`Ong5X*
z%TLXJsP4JDWJ3D01%;bVD;W3Wceuaj_{8;-JM-#SoozyL`CcNtqCBScrxyhBckZiN
zv5$>?ih%06-2nxwG-_?1?zr=_UHtgrMW5mlO*ON>`s%oO=(t6fWJGds?QYbu@0}u)
z)|&t0q1TRs>5Ru^rv$y9(ik<ZQM2b-@PR2)lJ0Dqevswbq}E6Sjb;Cj#Gk(%$Gl6G
zX=l8D0o&o~dZumqUw?Dn{?W3aDy!+y()eBVQk@+i=J7;N=H35jny|j^re!4$Z+dFn
z5b>}K{W$OCgl(la(`G%gJinoAOJnlNRc(UI=0z4ucfB!Qa^pHfE#Jq+SDNb8qQNJ>
zO{_`sxG+!N#rnrj71P2%fq=H!C4mV&Ept_6?&ajYZ;>yPkzuP9pEBv<^eZ(?7v%k_
zD%1B~yI$zO=g6_U3KEu|xTf8@)We_05b3}D$>hgw2{|&YMTg`MvmEz+r2P7I{u7Ol
zkzS4BvKb~@Z(T5p{#$%4bn82Y>cc{Iy?Zuo^nc@d=k$lY!E;t}28xDiZpc1fa;KB~
zzsC~Km?McXr_NrwcE{;ZU6A+@q2o_O!Wb?rSRC=<u-o?i5p#CU(0q8#|3=GenawBu
z2zf_*Fn;`F_d`DB#a`;2T@{bpjIFn^{I^K@*=l_KZp!*a8TVZbPRSLQeV&%V@9ozp
z((~0p!*OGR4Y#t#wD3z7y3%?7&U~2o=i|Zt?K6D;R!G{f_Ab1$mN!$ydYQ<Q1KoRf
zc=RUdIPeHq6eP@M%;8WA@tAOip_%74=N8iqVY|(O&8A2_@LC`po_DIE#Ce&Iw)MJG
z86_4Eo-`OqJ9g;=Cn=t_UUz2Ck&{QyHq29v<N719WwGQ0=G{6WUL`LNO!tq~>eVXM
z3SAO5GfH(v=zoRltE2XoU42!O7}OefSkK3L+S>-b)Exy6jHYu(?+Sa(wWhlK-L#fq
zmS3tn5*Z$DV_JS8bdI^AXrs(BhiPldKgKl9*;Ul!<z~FnXol@%sTC8yyg0mkX;tlv
z4-YRMjtSGw-8$jwfm>YZ+-K$%lxbgFBA}(R*;C<A+b;GAl1pnGj<jrx%DwsM)#Afp
z+oP5qv1@pK{9RcM-;Sr7uB;78J$2{Fz0J=Z^eoNyt^9Xs>+QuZlIlx5CtJT-<}&{g
zlkJ2Huk#uHeaSklBg|oxt;5c4lz5OsUBttQTloyPBa84U{{=!OG8{+vU3n(166DwU
zc$jN<!h|EO5}zlmlDNBwQ%{fKzi>{2Zo!?2iu-pcFnfF#yL8cB_U|nxu0|f|1g6F*
z5+0qpLXQm%CD|ExHa2c~GoeR;Rm5P`))wg(J65=8FYbNJleJjHJml3&#i>5U9;)V|
zUAYUsMcmd{Xe_Ses1S5QW9ig8dEOcqADyx3xWFj(x47Y^)UCH0+P<hLFREh8TE56^
z$>&a!iC&q>`-1YcUh++M`sBCd`zr0k70ZM^aUEKJC#ze7eK*6Zmbm#^heam(JnY-o
zZ}G|hX6nq`Yd6yux~>i1^`y$jfN$N^je34M!5xMwznLdBAK<7eiYc18B<p00*$SQg
z#q13|r}=p2bZ_2eC;ZytJlomtHWy8pEEKI?{;vr;KmCh}#=|~_qpN$`)^7T8^Ok^+
z?@Ht5=uQzQ^OZ}dJ>zKV^Lpi`#krg5=%RNP(H2#k4=lcSd+NPa1)*9iXBW;jd(O{l
za+=pTqIUIjqcb}W^Y-UCuHS9magW<!>w+>v#g#{`PCwgJ7S<8I_Q&q6YOD6tZAe>v
zh%b62AJ3YdEn5F)ZTBvk{N>__3}K<1lcnq(mpt`WuUV73t)a!c!0>Oj$CfsO<g-Z@
zw{=1lyWJ-n+t`JBhiRRTe#!Zi|KyE+ZXp(%bt;Ww%e0cO@L28D_4z;NtXz{;R_bI<
zFUIwYU*(l2-<baCmW}4jwaa#g^`*^PnKiw}RWfqf_t#!_3r_$4@xmrNPssDfl?_e5
z3^t$ao946YB+u=zO>9!zZ!0B+tl7W!&f})z6_snckLw<j-G1`&yK~X?r^FU#ZPGcP
zI`dzlo5GyT6X^mkCY_tYSj431o4IbgPIEwCO|oJP3+JZUE??6aq@_KW**9=r+n8sT
zBjL)y&$^q@p`FLOYBF<vqUEGB8QRA`OSN+ZO}}sZRCjyTX{WH%H3mCX)D^T2zwdFA
zpU=Y^w%W=6>!r;$uUc8#C6c(Wx};3y_Fd}~ym*2re~MD0j)!xS%f+@+KNcAZdD=@o
znio=c)J?>b)%)j0pXraLEtZR1na4cycIK;y?SE8Ln5rHN#jaCPKAw~5R`B!xqzA`J
zR=OoVQ=Rfg;iXR7a`WCU1p%1@b02M4-ul5)aa~2nN0CDdcU3G)U|TsQBqUtgM>lI4
z$Hu3QD~b-aPG}UFSrof`mhx_O)>iIKS^T|Q+O$KKX@rLfa=v5vGAWu>Td}-C_+#JK
z$?~U`u&m&EEO=2({-CFR70V~icLp>3O4m)Ed#33^%&Mu*@<+K13I%(9?`ZZ<DS96B
z{f^UQj?eDfp77^;&2Uh9en^-9>Qe9gX!+E4Q|BB>Rn?d`bHxD#_YGE!`Kn8}`wwM(
z^=$K8nkZf}?Z}Ns(pNQAlGCd?GRzt;iEdlen!+$8yQxT9R_SQRUZGin-8<bD?pji1
z$9d}T{~bcUGd`?SHxKj=PcU1wXzp_B?K8w;&OGQ^a`0)WQKixat%^n4JbE47ZM<)8
z(3rY5veiCAI$3n)%J~b8Hs^SFW;bssy592T)Y`v46OI4QGCQcnV$gc_lHA`HI{Xg{
z=P@0W4iyrPlqo!D*q8Ytf=OJ>Lyz0I&HJ3{{ICgAAFNy~Z>n4Ea#E~si%#OvH+LAN
zRcD1w$(y^JS<$mxQ>xl0Md|EDR!ud5UFRods_lP#uCC|%>L;bVE@w^Wx!cVYYML-f
zVV%WEW7bKlQsiFjT)(2<yX)tjpF1A$7EW3em#v}ucCMQ4{ce@bcdGY%d1tz$nD_OK
zH?tQ#vt+g9|1D&<Rp`I@FJ8^Mcg6;Pgi`h&`q5$VJ=uHuhi^BUcb#?)SrMPgc*5@M
z)z*eN3J+#D7&i!TpJ`OeT^_W8d)oO;tV!x_efc&X0<l4==F-xxT!qeToHs&p^nPj^
zx37p?-g=y8mTy<sQR4~LXD%F()pX|hEi81oYm04~>@9{cQQ^KBU*^c|bZ&ehEVM}W
zMenMR$1>tg_d>E*XJ#Dfmg3x`DEhGUT)|x>$9FkmEP=e~zaCx{JNv<UgBGuzs)Pai
zye}%6G8XgST})1_PKhv!D!5c#d3!S31{Gc3hZ{o{UP^h+G{ZG5Gp^^0&<q91``r_d
zPMUTmMa@)8Yhv4_NH$CV@Krm`Jo*12$bZVLDMyuDCro`bDe&;Lm0=Z{b8hYF3uZ24
zE%1EE7QcPQ?qf!AUcb8Mr3h+hd@Ae`Tx#=r`K>&KvK`AT&5y6@li^rns&!8~PI10D
z!&+{+g;Bd==j7MUuUK2K?wgN?(PV>@u6oil4mr+OVO;6)KyP{^N7|&653(mu&wKZJ
zdE&pBW({@`+B5FWW)WH(@bzf+mFZKT$GthK%WAYFenBEv8q>Cm5trX4EX@_}I`8tf
zrQp%C-)`p?9BZ0z>rET4;nbHoOC8u$q)%)9xaB1|<JeW*??>*`Y_hsAiPclxishuw
zL#Jzd?`r8rN883$96n$0?eH8Hsc@%x`58AaX>U9B^}k8&TkX02m)~z*8u~6PG&<&~
z&mzHxJhruZC5DfMW^yW2F$v9RV3Pa)gHa_xfir;NDf^;^GujUiRQ?R>=SYfZxBFt?
zIg4qkieJDXp(7Kj4wu{)a%R;@4C`nPHqGNN_dLHW^_jM#ZkDZ<$m)WWZC-+p50}Xn
z&%S@{`RdDSo$93T+<VsMAhl8EW86}$DKm~tnVqxg=?R`G`NdDwBh=)43dDt~nlHY$
zi+Nv@IkkKZ>-p!8t<SuDcxrwQzl~m}`wkH<&ChHOpH2Edacua+5}+{2!E1@i4we<V
z|1^zM3fylTuMzV(D04<jMOX1%8WV$oqKMx^wKK7Di))k)v)oQM{LkaD$bz{sChv&P
z;_tK0JP$f^GK}ZuiH2=_k8j*aO4eD{nC7EU;_08Ym^bZlve63BN$W%-+?FiM)9Cq-
zGEc|))gxt(N4_%(l%t+bzVUcPm51FO2cLa9yIW52o>5XiGvVd4^==E&>?bu998vk7
z#_x8{r{|H**?n$(TTbwaFX7icl6~S7|I$OQzw#J-7~%^w9JtON63b`ONMKUp|74QS
zAUr`qEl|s#UqQrAQLXUeuSYfKmgoAsdvfVn^bTk5DJlj_61dFPUS6^2Yh3=OH7%P=
z1Zo3#uV%PK7Wo}1`gpa;|C+~P-zQ5F>I!upJlXqi35&-f#fqM7WzNlPhs>jR6#v&S
zTekUXHBC@#XudJ2StRRWDcb_2l4en}28(CTHUUklCQN2MUA7)d#Q{wTD#vfG=(g`k
z^3Mv``($<Xi}gSMT(qB|^x>bg<VEI^eJs%m%n$keIL!hzjV3A>eN>#C%E+1|Bx_Kp
zmaDiS)t!&)YnobIz~T)y2HrM&mzOM1lnzpmDq>>m;kIzpdyqNNBH>Dp{`@AFg-Wkf
zzb#$#iS@;l_mjNNOjhyG`r=Zgrr{sbdwSOTmNiM@?e8{grJ2o9mYQ0?J!8Tu8z$9t
zuep*vtlOGi6-`hLd#x3-%uXgn|3`D}i##2H7L9@xYj_Mzxjt4f9Lf2_&(FA;`N|p1
zXGTu{Zyb78#A*=S;&{nuYR~I-Hg*?*ppPs{7ey3U^bHxbiWuaB6?iTP&1O{<e<bwZ
z$-`!S15@%hvk83auNOX0y=)<17}c*U<jbJ$y-CP9vL$Ep6jo)&&6`X3x14ZVeC3Is
z+v+U`&THkY)vj>=)7Q*$PQ@d|+f2!0*}6dOj82iX!(y|1m7)@vw;kYBb8=X*JXPq>
z6c47z4chueht_;})%#Cy!@GRmi^971{2kXVI$!1UAz<Zxj!ngr7XROH^ns0ms%4?E
z=0sh+iAvwo891gmh+c3IXL1w_P+aD+Fd$i(KhH@zN5LpSn`bpM;|FHpJO-x=&Whev
zJZt{5IjOn6t^fbS-Ow(FwbbZ{Q;&j!#<P}BNA@hT{S^3o+k<ank0cj=G^uG$;%Ks%
zmaJfOOhKiMw{L=Z+F^Y)r&k^?TZ4G+vuUrnb;gC|oYS*_bw55DJn)J<=P{4ry#1~z
z>+UoeL@$|9a!lvU=MT>`9x4SgDV+bLqtBq{&!qF-P?ujxThPU{P~lIo>R*$9|H1*B
zp^@vJSg>%#_~c#|(r(|<!X0x_D@Nao=ai?!r3Df~UO_^i5;mLoE9a#JYeudKbl4mc
zyNFlexVS9SHIH_Qxt>#eI-j0mD2Ypv+^KxiOWVIgL@V)%RQXf|A=c2+JRP~C+oBq8
zPBQTB<I*U4W&4?X!vnte{}-87U%Gts6;}?^drgi@6@AJ29FG^MFa(Px#7uqgNn}Bc
z?)sRMr7`EH#-9Efd-rP0#Zn>VO|h4+#^`Q#RQHd+sl;g^?6t%x|JGB+gCWd6i}UBH
zFuNBjntfFHzRZwys{h&~7WWc0`Q$G>`O!grPpvZ4Ihmd{e43QDW$vBLrfs1XE~WRg
zdXn=6BGjKw>Pa?RRji=a<erni?u?J_)FhwdQ(P{b*4>=3-|3Xwqe)sn-mTf7q0KT$
z;Aod)!?DcYdT;eyX5CS8Xy{@Q;lF64FLXB4Z}+zBw_E(4Uhz9wmLt3%=jgVai{El?
zhUMP=mU}u(^Iyrs7u)=Ix_JD`;FeO-|Nn<6`F5KD!(ju4WKrcaSA-5J30@0&xMjkF
z5GJt?hKp5%E!PMC71NjV++1|g-ayE_Sl_oKSjQ$@Iqr~o{W<?t*@`DLGuRyhBBm=c
zsNUEj*gP*-JMU$4-_-3ThsAC!Y@OsRS<4z>$MRs)GnX#)>(>O{zgTAbm)X|eSz)5~
z?LXa7N*9=a8wc~6Fiv8q7CjI@`9R_J@amo7HQTqxUKfpfSQ>kAdd+Roxc67%ua?$c
z-5zuIdJO*thVLZ`d?k*;hbHKRJhhvuAhO2P)=wnx^2T2!dCb!oHBIx+i9XrTs+aOU
zc8g1#*vZc!Ay0Wcj72&cKe?L3eQ8e9ZSh_cwVzw(e?l|+@h11_ts)v}$2txt^(BkA
zIG6Idn@liBIniJ+;e51Pi^`|Rs=J=?S+nSA`L4OaufO$5_q8`2KPP6Tv}9fvy0A>s
zaOVPMA^wZUjv77;QB^j|c{nY1>EYayVhn$`=6>8#`2I@w)f+iSxAoq<)_Yx|@8*wQ
zWuv*v4j2owxD_qAB9<hoD{A0Vq@ZM)^wresLkYuMk%^zTD!rW|s_y+Q>2tkvsC#*5
zNVHi>b!n*oR$aR!v!tv3PDy4Lnt51U6})~HTZo0{CQohjQ^;$6p*M5sj>6`IaE_Ht
zrFI#KS&hpk^##<-oW8H+delMDO^<!teui;#%vgTNI4#?J!x7v6cUML>wCg_Uao+N^
zy`k%Y(uD_yzB+IJQNXFhspP=5EkSt7k;0AUOq;e>Z#L&rN#MTqK;^5Mr9o2Av=A$o
zFRG8(t?wRx@T1siwRzm-@3m*c7rgzt;Hsz}YZMb(W;}nh<LTwmC1LZ9sCu4z`ap~)
zX{pm?mLCRT!U<2OH$D&NjQH3ntS@rYCpmwTWcZsvt7}iacv6BxpDk*ga>!S!z$-}V
z*hIgu`w@*lb1YM)Z|112WJn8B@^IvSFq3oUEzuk=6SXGJ+(v(gBffW9lpieN<ZqQ-
z=wrj7+c}jf>!Xi?)q+`bO|`xirQgq5v5xEG11BS?jf<H};x+x13_lB<H{<$$Yk^$Z
z&+SVT7*9odY-<oXroh=SbJ?0rAt9!UR^QnK(-m?)n-;cIDV|>-!_Q!tBr1D;f!yaF
z>)jKi_>FXw*_{`Obll`!_CUyVvR0V2!G{@=&)s!@YHV8Z$a5J(F!z+NCbJS{ixip~
z)_woF-LarZ)q2v~M@7NCc?FY;9V@=OWR*mhPIfH1D)z`G$19yREBXY3RP0Q%8B2My
zW(Ec{^F|z;%I3PvPW9!jMyu?FaybX(0=|~XKi$jyHZjJ%JUVgj?S-Ous>|z7&MTKt
zcvu`c>-ell)-yl)9~dw#u$@w`Y3Q#xdHeV6{@2B;RU1Td9;%o>+^+H<VABPTzl(T<
z+L!+So*J+B(tJsiglEok4V|YmHXcoXJJaToO2(aCTTY4Yc)&2X_46O`0=0Gd=6Mag
z5((FrI=9}5JXX%|+<fAUIrFtv*S%OiPvw`I<(b20T#Pr!XgNr<upA3*J^q-3IXbRJ
z`%#;MQmA*;t>=eshlDTYb!$w@iBQaH6g@T5D)-^BJF`V6ToKQBbu8wf*nu9NI|<wQ
zSPxr1JyymY#F4apPmIq)4ySKDXP7SP-tf!dTYq7bn9zKdPjc6spOzRo{<Z(Vb-QkG
z&-pUO+(d4H2ZwSK4@WK(jgy`C{ANSO5i{??=a*I{YIsU)oRFGy;`;wT#~S~NmAp|D
ztk97um>>4O*5ho!|Lsq<IZAJrHOY4g$d@Xr7CyA@cvEn<^p{&oK6>V7!_y}fUA$KN
zNun_@Kt1qtU(LatM;`zC^58^dOMt{=FY{87y=985E9wQ8DyU1Vw!Uac)?z&r{4JX6
zSOe$JTYFzO^6t4Pw`8JdPU0$Sb&nYW`>H!HZvQP7=We3&=+Rq+nsZW@Hft>DVy`^F
zttQ-WZ@#6_uy~#O->0IMiX2~r)=vLwCRp^^WZ_hX4Y`fN61O<wzilqtc!b^cPT-#K
z`o%}WUwXTodAU}mWu?;Mc`J{-O<0xtXIGH)q1O?b3cogN3ehTZi8F{=Ao3$daz>=s
z1P9g2TIWg+Es~k{NupMYSKV+&;QyYu6_p?3U9y(BSFi;xel07tBY9q;;C1G+_nv2+
zdB6K(VcfQ3liscf?>M)8!QQ`(4|5&QM>d|nt*|UdG4sx1UbTH673IwA;;pKiQ*##b
zcI<F{mHqC9LHMO}-ySsy3g$8^2qu2~^@*+C=zr+#)PDvn|GRzf`mHsxYKY8FuPs*m
zt^EALoyv)kGv>G5P@fX{OfBh!TEkrKJ>PO1zc{NG=b84qOC5XJobpv`dq2~KuR29n
zoG(n^5c#q7k<j_Cm#wZISC8B|`~ODU(>ocH8V;B(Wt$=yP~aflD#T`zqY=<}fKf<Y
ztJ6a8@evi~Ng*LhiKiRFgd=s&%=ozE&{U1UO^W|bgH)v(mxOF{v3&K+#YoL~l2Y*b
zuz<$5Y+JHbQ++O$aYx_W?0QNw#At2aP3^-=k1!Z5U9(Fu_}I0zVKHxa{Ia@q_GVLT
z3D*^_@`h`jsjsFnMo;IK)RWov<*q?$lL)7RY}^$K1;?YELgL&t7J^1D?RoOy=U!MW
z5@MZgUHp$H=v0elv`gOIQ=L9p^O;UwkYNx`^S&vZ8Q<3XyY$U1b73W!124<mwq~7`
z+Me3Z{o*{6rIr7k!sL?1;BS6BdNmCvG;f*v%rRit6Fp&`d2mi;g~P8N8FjTCcjFpg
zXvS<SSa`m*+hL#5-*@s0;_Ipk7s(f_-FoDtV`{>7g}mJt!~Rdm*Y%t+*WrC+$E74u
z-?W>_;)|9<2(1g%kkPg}n3CkFu~q1{k43MCu#aA!%VlroE0=<F*G@@09UQpz5qIR0
zFB#t2+jczi5@%<;(KeC2@S&Tw`qaeB^-No@UYo>RvM^NAtEe%Qv&Q!z<4oZS_IoV7
zD>WOL9z|+2ibSWlE%KAU_bqYWbX5h$`E}cFE@!IQ<tbkqo%=9qk;<Wt2braIT$zmw
z?_9#CxZiukwJcXdA!9*ZSyW=}B)?sgJ*vJ=mo1m{UBI%sC-6vS`^&xCZ_J%~ac1Je
z?{8PFDoFU(CE63^bW5mb9qaO^8+Jau)|L~x%XJ3R+s24MCUL*nEi4+7O8!T7F)#6$
z)O~LeOY+ovMXQ7T@^-ep^A6!n@OQI%x1({Qf9F##>4d34G5eylGAC>*Qq8^)mO3Sf
zDNcqXLM@?*Q*3#ad}+nKYC#pjMv;_^#6l7A{=oin^P98oR!yH-BkaQEu;fOXZuza7
z88Qo&&Cg;_OEKQLapFelZ1r1zE@yB4nx@dY_}&%HjYm>nb<LVz_{u2#LgI}})!$~8
z-7R(Z4_R>HVdnQ(@1O-KammjouDw@sp-`k{%O%tDU;)M!bJm2fQ)k{kv*g|q`9|hf
z%ina^v!+Kk+b(vIZV?XjSW_+O81}Hb#!E2tacN(*@GhpaDYwI3aoJwd(BH@u9H=3>
z^M9*I>Yo>F2fV#brF!Rz_JqFuIWcpYL*}}ig$v#ZZVQ~V(tqzZ7SEiET-rBJSvJhs
z*5R`0>5<zrAAadO{PX$zwU^poEbKDU5-8K?^sl?t8@8=<?#t!(gBm7#X=O_0eR$oa
z8(h&<|7~Az+f1gPjr?}|?y_Z{)lTr%Qd+^HrN*IhDD=x6W7gQ0Sq%)gy5EYZ*gljw
z9k{&lq@&Pr$F7AbQIeGl=1yABy5ftA`mE+Do%2d3OX_-gJ<&Q9tQOS0xJFl*)44OY
zu3~z^$LIQuoM#f)f>(b&!e#uX^RoArhK(8<4#}l_m?5%d0jH*mBh%spR_mODT^c=$
z#NHgZDE7*c?fHL)mKJuK=&FEeW>XxV7))B2oFa8u%}-Fx&!~ESLsps8JWjsCJs%~P
zSbbk`R8m8E-@<@PYG0&}T|Fr()O@Qb?wR29ttu)%6~oloAIPqcP0M*=IQ!<EPcmjk
z6B;EGQ}0|co)ycPk)QnOu?}a;toN+xrkzT28XY($H)-fOb>-Y?SNWu9Ht(&GXk;84
zYtznCTLKjBD6zh**xY$7C0KDWL%g<|;%vi59I94I&Br849ouGhGUsvezc;(-H@9F#
zR<`p!`-Dj?hCC1C*IYQHR`Nx+{@0;dUstq><y_<o+j3ZJcEc%Sf6hgxzMOQ<4Z6&t
zvVl>P;hDqg%a^`0{q*r_E&0Es`gF&E03rKzD@7a{A6_{n@_FS9wyv0OCpoUIV>4?M
z*kE*L;UOU-qt=Z#H5-e$8<@Io9q2ieJ99?hAu*K-`FPHaWhz%YH}bd`@b7YUR_U5&
zBwS!K<-i8jH6O$@)07S~o;fjFcxIr9_K|hl@>*3sM!o1|dvIa3ZH|HX#V4zUl2>zA
z>TpHBP(NniCTQmO+CX%o)3L0F{Lwv1TuWBCI+_M{*CkzZ`P$JW@|tnenSCkTK9Or&
z1wLCe?)kW7Rq0Ca$104RJ#J|=jKP|G%ce2zIK+2iWxV#gpi9hG*A(wN(zoFVyVG5r
z)ka5LPfW`57H_;`X2Gf%|6`Mz&r7jlhW`I0+&{m4+Hf}F(j@(8rTKr-7R<eumc4gZ
z=gBjNT^^m)<>Z{P{>dZf`>&r)&-(JU!+nRk_nwcg>b1Y+^_V#>J=I(1D8ltD-$C7T
z(z)V!F>kM!t-fV3$;ViE{%INCh>H(<L^53}vw~f`Juc>$x_2|(b(zC5Q!X`%zjY16
zV}^exxr~~6L_IDz^|>eV`~J8g7nR}a6xk>u*83qeM)%|VuxD!KnNxf^n1m&7DcCD-
zJ;axHaLV-aPHp{Kt5!Wa?J?6(M(ZKVa^38@`$~bkf{Z4(ndqO5Uv9Tmbnh0H&?}M-
z)!NEJ?^mo^zw7jgDk-Nci}@H<s;~12^gXGr>XBHz=T_+dg1{`t1W}%|*Z00-^Ii48
z*R8Go`t9ITCfB$ot$3nhn%EifG-{F4rdvfet1s=&(^wy<@}R5e(h&v|$F}pXhD^*I
z4kAkwICb3|gp(zXO31v<E<4c8GsD))dfSAx&wH%;`X@F8Pf-$?c*99}%?ftutT&IO
zmRq?tiFCTXJ>g>OG_&9JPDA{Gkm*4tCwZ0Tb?AQJ`oLJQQ}wuE_q4Q?!aQYD92YPz
zYtAcrX3XFFkjZjo`PuEUNom{tJa4P!v9K6^)H`l0^HYVj@s@>?e+r{d>UoB+$oniT
ziqEWO?sP~y=kVO)wQ*3#s&$`=P5181pEmc*%&eHr3+C+oX~uW0LdEibtYS>5%ihqt
zA9BxBT`M`sG3!OvQMI7A-IrDUyWF%gBJ-EDGG;h(NinNG^|{iOzWmmvtm_{&{ycos
z<ktG?O7KSANkRQ#9USv*UO&HaV_&O=N4v8ByI@yN)d#mu_Qyv*yQa%l@bgTv-Ge#j
zPdqg)YX7xU>C3hi-oD+eU$2<=)tb+oTM#pU;==v@=W?1d7sRb<c=aLDI_1XSm;Y;K
zYR0&%Sl1+WN91sZNH8yhSkk{Q(^h44sU&c-G_6{7<NInK?(WOsYb#3j-WO^-7-(3p
z`_S;_WJ4*(&qkYG{&JbL{a@n-rM29JXLsJynjr9+=U>)b?L9qCzHbA#mu$#xJsKcs
z{NI%+ko(*-e(oK~$szo9&s?W%P1Z=s3v#f^P2l%fXm(}_i?o1@*Mz`NA0^%|bywJE
zv}j@Acd@j@Ybw9X#54-og$<hzUlcOz5SnsD=!9ay-cs&u1wzbi360Lecg0OkJ8`u&
zOK*3TZn<3B^CWQgwJN5IoEy^9gfGU1tnhd=xiz9fGWJHnhK=IiC%P*=H&W;*Fk_Xe
zRTN0QBzxtd?3#A7wrP?<>H^<H8H~1v6eZhUF&2z>=PG+F{=?9?Qn>m+VnTy(>@lap
z*=(`rTx0W>S=^dhRi@!oGs%CIyN;Kd^xVUidoGIn{qD<fRp?|hi#&6bhl6F}a@MaO
zvMbcx{x$qhQEQ5PbHR1r0?w7!ayCUoSTTt@vC8L7v~4L2Y|3y9cdIN*P?@bRFyCD8
z+>0J7!-6vvR_1}tOy&#=zo&8s%Fn8h6nftKmRU1whh24XftG2>_l=Be4{2^&UcIO(
zSofgpUZ$G=2W1~QdKE5Fdm3&l+!*5R>d(o*FzIoP2IGXzqsG=jnXN^_%O+)pEUh}Y
z(J`D=cpanDgU<p5DV8^lxK=Jxk~0%{@2+!ls$k|+*{x=J+rQ<1a%zyc9I#7_H79{z
zNPyLH3u{P;&AJ=DOB94=wg#M?oTW3VLTUr&-gKLMWv5lHf{PRLxi=fAY|uAY5&bW$
z`Tw>_bH)2u+Ix>Z{LdwJJu~^4GwWh0Zl`H7R}^MXYw0X$U4O8EX|e*RlXduuLP0Y-
zjf)-u+ogAh$Tu-caesEt>=61JsK&ckD_=>*eyf~7sieX5cAk(B2F{t?YBLWW=3IHL
zx@m{@zl&|lE_og_RPbn*O1arFJ<Md!!x9mv>5-ffg+Z19>7}JkY`4F(s5%N6Tx5$k
z_jPp=N?<iwa$w^BWx8t%0%kvm+@LrmYdh=tX=O}`tZx&Y4ozebc3?YoLugicW%x?r
zi!buc4SVMtOT9HibLoX@j<%{*nU-;tl0`4X)}^<mr?s+_No^8Wc;%t7_q)BRs+N^w
z|MtWgUN6OTj!SSGm@_@-|I+ebk?Cd)`;i(&A<-!+>D-c`Z;e9}7Ipf27@P=_RNLXR
z<4|Y^W0*99ut|yK%|g9Rg}$ar;c6#K<Zjm6FzaitbauUz>s4mc_@nXmLB-ZW-7SkE
zCOfLPUToJk4%^;1S9oJ||DxvLoz3QfOKxZg>`9gBNiI=(D&|sQ+*j1n%q-2btt9%D
zcj&L$`_~Ka|B#-<)Ymg*>h2Wty2GqFr<Q%XA^Vv_Zqby5DZ%YG4b9D#x3fk`$_vfe
z`ZJ`=SnZc$$>c}os~9~jl!N99+no=cbt2S*|C`1s6aPiqJWn36`H?<PyU3#Lh~q*F
z;rmxqW-3<*Gxp5B<tC;yclz<vaL(S_k^krVyb`<HDRjAF>Sd4BR*dt?3IyJ6tzFQz
z>S4vQlv8Pu%=W1hOlp(+E-noX=TLTEEPj!z|FTrS$PrnkW}`Mwbs@$D=H@eNxnw!C
z1kRPK=RVHe7g)<JIYX31N2F_>N@{FGs)Cqi>-NOXzc1ugEXy+Am?eC1{msHW`I|at
zKM8RgE}8mp(dX-nrhXM!R}t=-WW&oCrflNA=4ftP+Qwt%a=A`yzEzT*UP=nlis3hx
z>`@gk_3+%iRcyj0Z})bod&>(7Ivo!zR4|fmD^c8>@40MCaa!yQ6Z_Y#Hz&?o#V)+^
z;IjJ8Ey9Y+<0q@#{1|+-c>Se~Ui#)UCst0-E4R^E^<Q9ifYj1L^E-#z7>(NWv$rH?
z3ViI^oL4yOw&)rKrm*OO#N5+Px)Xixia2Q(mfbfJT5@5Q_NQ{;*F_=1L4nbxLI+rH
z&2*G|z43a5^p07*>5Qr-6=KoPq^h<Bg=w|Ek6LK{arMoQ^GY98ow}7-s@d1QDQH{D
z{BMo&dp4zeot~rdam)6LYXu9phTqQMJQd=t${63CB6W>zNn-8dNwu@ow$Gd`G&gu<
zbf}{Kv)$`vDyiQpeY<>v)~ULv)nNu7MV2)yb4=ZHXhziK;FU&UPCB2%=PWWeKbY_J
zN-W&d)Ad$#MXFftg65rBQ}>kircK<p?Ezbi0i#G2`w7Q+OJ)}w6aBx8C0LTZvBa}t
z8C#9rfm55AOk%U5XC19BtTb!mX+BulZ0#GJ;c2>i>O-%4Rx2+qT(4B6(Rj+XX_rCA
z5A&~8+s%tx0wg*Qdv*32OP+ka_|$U2MAw6of_v3IbqZfh{GugX{abJ8kL``KtL}Cu
zO;ToDb!pY5X3e$6y}@T#KmFeAeaOVPL$iECYOcf4#os#`oSB3UFs3rh%lnZ2F8N?0
z=jKAmy2zCpT^5^*v^~?^)_?gWHUC&_w9$IDMQfa=?-DpQVV$+{cZuC=gA{e#)hoNy
zk2o8-g)aVMF0}50@PX(|hM-h_&xL}Vg?TXsqLNuEQ+1UR52;>UZ_MPJdqH?s)PIll
zKQ;7jab=Y5z5RN_?cK@~75C}3M-|W5_^CvAj??l9la?%YXOj+KyDU+qI^oRH3;Xw6
zwQ}5&n)+c%%m&7o1hHC2fm%V;Z){6<n{H8YI`C|=u{YCngWuB@SRQ{dS=KmOz+rXk
zhdak(G*(`{dic!ILmt{%Ar3QSYNWNUpYE2Fty~de^+s&ROd*p(d5s!#h9Ym#Bi_AH
zX}m&(63b)ns@j$R)DU*f@>wG+bWz}#*n#bgrw;{Qx}svTNpO3(t5Cu7`en*V(I4hW
zd0)1eBip6O7`SHYRKcoS9hI@1av@3uyIIB34H;DmmKX=KMLRG>m@%qe;QCW`EZ2Zh
z=7Ch*#Q$gN0!3#&lG<f1xGYhYEl}a0(lycaoyWNA0+!F4)txAjr8@u40jq-w5?RY4
zKd<4dwQkAD{2no5ZIQ&^n?mw6!rQVh{8*LIyu8lDN$BeLkePGjg->0}ICkJvP@0jO
zlep1k-z(ciAL-_c`o4A2GZ6~k5~HrT%kuN_4fdYC9HvT6rH7kNpDqfKyj)aQQ0yRV
z;LDuO{r~seU(#2d?^>w@80@iP{C-Sj-GQ?wcvaUHoGlGhD=j>`$CW9#A@y4rn~J0T
z<yf`{bAmr`rZ3KpoojuQbxzvvjiyiEXt*)UY`z>j_rZ$rALk^GZ~LC3Fz3Pa=BRvE
zmKzegCu$@X>x<81yY*k@;LF2TR8;3MNGx4==x5=I-W578jg<s-9>#EaO6khT&6WL_
zSS?X;>ENwP6K@F_FItr2AoN(|@=Vz^;+NF43VXL-V2fxIWqT|h?I4qqB=CjtSjr`~
zatAi<WcIp^j~(8zh1MSBn)o;+;mnCSGTg#Cxdq&Nqr|5#j7$7$TeYkp<Lx0uR{JH9
z2hX=GEYDeCzA#j!a?&J@M~Um?SDn_b2-f)iV*b(g^;V7xN{ueheUy8x`;yASUA=#c
z`FE~-@Z#FSH6B$;4`M?Xv)T#&e<ig3n$VgzV&|NK6t#o{14a0L*RA^~m)$bOYmIP$
zO0ra^V8bJ+qi3R19n?w%B|bT`UCR8=zPj(&wg9F(jdx7e?N52cb@?vaHiNstmaNxn
z#j`dV<bBAG77#nfDt2?nZEfxC8Py7D)w|BuN$$N8C_Vk@V(VR=%AS7i_g}6&$hY|Z
zue&c_U0mC;`(=ptjT$a<uYx1&dpJ4kOn=o}Oa1vE-TPv!)v|;iC0Fi+${l*I9BkNn
zy8O1ERZVrW;yFiyB0Uwx%lzIAn!;yhc^Ksu351_Ky80aJLUFkxhK#pk*@PE}ZNKpN
zM#ZtU`wX@foJnE$l>X+C_g?YETJ|?zEHTJm5?2=e)k16Ub*Tj2Yst3P(md~F{$0mW
zcD*oW&${CW3ipbhZ4!8SJ2U;JLSS4W=fN-hVvqML{Z|kYag0%56;!Da|NSA|`1#Vb
zth~YrQ<|mM8u-Pgb8ioon$aUaL3WLsd@QTcl53CNha`XRoG7__+q*l9ELA$|7fRHx
zGduN<D_bQ<c=I0NKtqxLW_}?ovhF`M<e6S=@O`JPtf~ET|NhgAHWB)#B;NQxx)b2!
z86x(EZ<j#ep?qVDuSW5!ZGN1cajAFTCygCfo`z`MWBPJ!`Y{czH!;B-La`jyj<=6T
zMI64Zcb#wV!TWa6jSuIG9b7B9&g}P5ulI%T*0tD3Zv61dU1+bl)UAipdHKG~SuOm|
zR`9*$!-*LWox2~ZC~drae8ug556v(7KG^a?ai^s!U&BAK|B9>voNWJjWEM0$aBSh^
z^zv{>J0RP#{Zzq?q~oEIoT)K4Zys8BcCJzCwh0d&JxkO!&%Lu{)8Y$H4@PrE^A(h?
zPFronyq)L9&B+SW8AI3@?s8sv5wOI8eZs>>2FsmSFc<kv^)h(wy@q8m)6yza#pP=f
zk1Czv49mRg7wX+>`=Zq0)+#Pf-CE9RK9YJBr+pq8r5+YJAiTt9qLJfC5mgpmz8IST
zr-Mz7Y%DX*lr=cCGKy(y9`0K3oN+D3HK`k?LN$$ow|$s<Xv)faJEo;xU(0oE_2HeC
z&y(h@jYwqD%yCiw!ejO317j|Sl6=CP1x3>@RP!sS##ns~dz*7r+QX_^T5}J(ilWDV
zxzz`bOpxUFS>w_;={Uz_{u%4%?QNR7P2g71;nu?^g<Kx<TZU&SdF*fe68z!7%ZyDY
z7Fp}v%53;AwM9UCgI<V+vyp}7VdtPdi@Fpxr+9O(W4v~$;3&6e;;QwW%Q_0(B(s;i
zcRRt8rK9~>i`z-zBikgOg}PCRe3s8Rlvv&N?I@VU?KkI5!GhH`X9^fLta)?Dd8N^p
zRo%^dWHNM(g7-{NJTAt?ZOl38#vx0w$2_^`cHUT!H)Xn;V)D9YFTSjq+w8HqRYbGu
z_QwS*DjsVtxU_4PbbYN0_F&|jRFly#Ey>FxxXGf`Q!QX)mZsC}oTAqYI44TI&<P8*
z?O54(VupbJf32xjR~5C^Cau02Q95<AxnOOtuA=Da2a|V++*T0Ywe^U}<<rUiypz2)
ze|WWe{edY0XAJUQ2m2gnDOvJnL1FA8-4jQJ?xt9r?}=HrCCnk4RYx<dSZMwAPeR|q
zr#J02y1vt^QSZcQ4k3Qyk7YhiO}@{bT(_{B+q}cXZsO+RZDvt{=M>F6GMCClGuW<>
z{m|g8VYg9gvO^c|wg*mK-+y&UUTT*5T((ftr-)ODO?b<M^qX%s990TFUvx5~b%I+*
zc(~^guJB!9S6?k}o-uEi?xsgwLMybrcdm+T3b5gD6+NpmEq)98qb1IKSxY8HOWy9j
zV&}=eNT-v9$s^$e;}5$IoByAdC?s2W-|iG&_%P|ofs<>O_f*}OexSzt)AW7+E}YqP
z`kFxMIZ@&DXAfRZyBM@<ft`oR<&QC0dY8Usr*b#xhb(x?DKsO2`-bX)S~FHPjl-9x
zTd>vzH>D|Q95VBhRZ#0_klU&jK0&=pU5V4H^jBE_F1wX+#b>W-p4b@lFYA-*{I_1W
zH-u#WopWm1wdyr@j=$QqYT2&S&gzlIJR9?N+L|0!(NKt)qWB<3h=JkQj!eA*F773K
zmuD?|;u^1PH2>C|wypJR`{OwyZp2JD%yrSBy^%ezugI!w;|7N{EnJz*+5sn~z1oqj
zpw1#vJ}r622OgFJn;?%`5rx+2XW!WFPWJyFaqshCGX)MsRw?hsMdcF~-h9O8e0YlP
ztqX#Tybhh5H$6m@r%dLO<<<6Dz2MirOS5^r)_Om_E#P=y+QBnBH}(|A%&Nb$ttD^o
zafAPdPw}-**}gaQnX1r-2q%yB37k9$ss34BkC(ZMrKfWE<b>P~u?lSSUbfUn%;~11
zmFM<GhRjsH#gB7bl%?!Uef;$kkN)=fc#5%ur)&M?g@Wo+C%6<UmU~~Ersp_w>b;4L
z!N;sQF4&n0%yv1UWU{l_>(qidbH21Q3oAHFw}tO%cX%&mv2oVdJuBV94cBUh{!SCm
zDfL@1Q%^gllJkm2q}TKv!L@H!pZ#_1)4nGzr+sQB`Pqy9f5a9vB_PT&vpIH+==Oh^
z-qGEj(|53}XkZd6FblGXak|IuWM}CTXt*n!C6U>_;@C1Bzu>rjF0oTlX|2t|Nqm2W
zl$>_TJZa@fY~7=@Y=U6hQL8}i6=e=)4c8?3%D7(#9{Qy*|N9a(#@<U$#kf+}wMe9-
zE;qU-;@-NlS0p2Y>9&WanENENm=rD<j+0_$4MKW34?=uwIxcjS2^PIl$qbpSZL$01
zB#l~E&CDsjQnKdz1S`*mD@t+L-Of1p<7A<aV(P_fX-h8HYK6GFyK3I6E194$MO%@t
zP_xiuN9*@BIm|tih1$O@6*$G~QZ$!=>uFQ#Ns)tPlhdj><!w1b|DQOb&iOUzO$p!1
zUDLIeBs=(a=teoV-N^Ldn*XNPK}3O3FGjfX>Ymfie}dbrw=^l0Iq*u}acDbwb0N>-
zze!5^MvRjSygJWbR`Yu@ON%!*zshytt-v?4S9|g)Y>oUa_vFdV+Z$tE)_&1xmVUHC
z`rE{Sgp(KVIQ-$9+aTxSdY!W~^bFJDd%9QZ+dL=tsW{K%=l*2FD*Ql%XZGdh3posi
zMZGNX!uyOql}1^5a?2;DN4GE>S3F~Q?g$&x@`k1l3Rge&B(}eK@cc|_qp|Gi2RdSh
z8E*vG-@D0@Kf`*v+`}8uos2OxH7}%_4!AR>1SmBrnKWolNKgvCz`=RrjQszE|5>`J
z1;*?#7u!s{C-UrL?tI^}&4oSmWA~Fi>tiQBQh8^asgvTexYg>5V${mF7bBbE!@0Ue
zRJE`5=yyK8$*^Xj##N6%_m=+ZJH;WBnNF%c3p}uk=jQV18mhL7YbPn|GONWXht*D*
z=B0M}!B!(viFsTFmzVboUlz}En!jT8jit|(PBY7z?>y#qqWGb=K=_VCCY6j<mqXrd
zyd0NBRZm1pYdmT*UEZP;mE-L8!?^J%Z_AA7N>i&}YE8>dPOWZxR=%c1X=-tozv6bG
zYmxU(-be`KTrlCZLeuA&O3d2c>D_O)Oq!&cxYFm2*7Vb-MOHA}<ndN`!lu~opmJFH
z?4ppT|GO5hNRKy~v)rz2vE0-L2bU)opOTbO{LiHjU2)j`ZmN!nE{~pH;#^TX!=ESk
z^ZhobCok_6oyKc-w=W|raYtam2gC4fPgBK`gPxhaT9WMFA!KGCptj9r&*D<%`eRkk
zP1386G8Z_zpQ<X$jPp7h>@)A!M$Ij`PP#o;CTm{`a=p*VYq##9562?O{-t}@ecmv;
zb>qe>p0Y85+p1^nSC781{AE}2oLjq_CZ_oMX36<h+tfx+UCQ@z(ifkGExm8<J&tj*
zykoxhdho|tYMI(O7X_oYyGrd%ow4UsON`G6udKPRCq&n*P|ITaTo&NIhx=XEmL~4o
za;Mnt6g=M(vY99A{r@#iJ(ZKM1XzduPqV4*bz05wO-t+3_4fyoiz{EWr!EmIIT2OB
zywh^=+UGMis6;CYC2eRC5Ikh1@jp;xqA|}z<!w!tme;%`_kA=B<KkU#!zwp(QcWZ8
z?9*L;U#+fMwEXBK;kDTkA-{N3gcUEom?6$%)Lp#u+$Z%n6@82WyEfdItj;Z4Z6W(s
zfmQqTrs+2}eM;zV_3TiI;b(mztA5CA;Y2k(g~q-c9KDrlyOOo%Z|u<(n&r6Le5vG~
zMJoiRTkcu-L6vKw)Y614pI0u}y0Yv4s`&zpg2^}4&lE~+ylL{Fd-2Uv_BUsAzl~-J
zl0Fjb+!0dTsp#1dqRBTa$#G4Bq}%^ZTSG0mu0CoJ+~Tq|P)hrtiXY21>nC&BGI><0
z_TE#P8{D)s+;JMCL~C;9(n}tF%d+i`I2zVHwAwg<f%C)eZ9=@^1;(*g7R}?F^*C}~
z&T09UfQ?&DnjeamGVtm0sCHXAVQK`Yg~VzT`z!W|lA2~&%(YG&c|Ow`cJkFJb#!Xm
zYdy5kyRbp6W?ez%p((=L4nlL}ojdYu=IVCvKcBTz?DvWjjMHjQ?v3%3mo(PAqjW+h
zSmBW60XG$nrK=BKNKkk-LG0JUqthF99iO>NWA>)T741<oxEHOOx#q&!T_+7|_N=|C
zy=d0Nh9CpAnKL@V5>6(xC~Y&?9DQM?Me?yV9E-XB@ARBk!}%?G&+c2RPn<qtoTdJo
zS@U0!;K~a&JeA#ZFPrQM>bTIoGFxThCFLW^uXav;BhBZuQlU}b;-_#>3O}dBYPU&3
ztq1HIbiA(KvgLod*MADHjPZ;WPE$@xsCK33@K<e}?%-O>wCQ2PwhIEi94oxkL@b})
zSTrM~BYTqGXJ?L}lboW#$5wsm{H?Xc#rafBCD(1^Gu(@K*GL+q6k4SGw6BUXU{`if
z*=<pr!|iaVTl2--(wY^FO&asPm$be)EmO>+x?%dP%hO+;FwItRl)AjYH+kYzrMYvG
zn4c?HMQjniB)CHC6AwrA(e9rL))yTY_e?Jl>Xnz_)m3UeXXx4ZU%{i<#!2<gDY?i)
zs(-|$hHz}Vq&36EdgFyd+YYHcp5*2wqdZ-bYyO4K|2NK;dOOdN>8M!jay;6`%1eFM
zib?#PTpqKzEx4pxKI|2?Id{L=>-?MEEnR0XTkmDivOM!caK9u!=L41?6IbVw9yQIi
ze1Yn2UK|fYX6fD(@_RBVez)_3%@cNpIB80-My5>M)1p|l#5F9&O68+Oa89d}rT@i^
zuDfD5SU+$rD4drl!?9rJGXF-7XN<>KF7m(FskGGL7_aNpw2)(*6WtnCPpz|DYvjst
zT}!@dkHYp7yyj6myt&O%J<m>8n%o&}!Fgyy`=y>+S9<P#Su$<*!J7`ksuz~_{bzFA
z7IE6_Y*&J3_us<-Q;RrTFK+CLv1)z8YZ1s7-g5MEq}@i3<s8n6MhUG^5BQ|L1TJRy
z%d9-@>^ph-4;k5}OWzvSy`0h6yTIS)qvYQW?9m!wNo!l*u4oMlxVBJRaObW`Mo&67
zOLt`YO8vNWGFy4^%ZuARLl*COwY@oD&vKEKIZyNtcC9u1DYf#>7Hz@h=jWW_uDv|B
zc+!cg6%3bn_FSC3KhpM~=Ojg4>t8!$#aaDUHO|^RA*AWV);TklczcE4Z@xYyvHK*K
zD(j@{mo;@RFzAJbbg0;>C|7D7PMK^NEw}N(CGSSg7YjLO8|Xa_3VYl&J?w)-SnAcV
z4XjsQbpHLn<;Jckw>Oz?xp&X}x){0dmksx$S*tb#|37@;wuJApxdu#A`dWe~#<!TL
z7xp`~YOIkJ50&oMJ+Y<RXjfx^;i(Eeg<rm0ySBEhwO@15XVF#Bj@XH<fd~D2`DAkV
z3a4laB)PP0S@~wlG=ZXp_Z~QgIb4lW2)n-WTI5v$b>05|hAYorn7b`__oK)ONo((<
z#D<+oonEn0Gc2J!>O)N5n)bLA?C~$y<0Tp{e`T-FX!UIn(#x3fJ!{Y0-mNAjR}81z
zJ?6Y6FY54wrm0(U!n1DP*idns?PlQIt*104wicYczL!zu<PF7e=Sktkd>0S+PMKnR
zeXG#r&cM6Pm$$5S$!gr<{hw=_MdPI}K2uaU=EnaFzxhL9<Bl0=!F@BPDjr$5^ypjN
zbA}x(7TU8J+`?DfnZ^6y;FOycfnm(+S}%WP4R__JmuRSyXke>*(72(UZQ8N_DjY^@
zeVP*XZ@#g8nW6fwCO5X)%lek5Za7F;t42M&xxemH@TcDEjRqSe(iEE~36`I^wdU+>
zMu!E>GiC@^A3c)Fao~n~6YHAyqTI8!A8u21uaNQlePP#kL)WK^Z(rCT{MbOTW|d#i
zCX4dK_&vFbp{ZBHd$;LM2vuS+zA9~fuc5ywV#XJNqn~56x|C+Nr<|LWbS{kHZdmTs
zOK)47Cp@@Zn$SAoL1RWj(+YMYiPr0(T*m+BtlrGrmHCzXf6Wul!bt}g&&;~(>FZ{*
z<Uxw5*M*%z-T9_#Hw4VG>^Z%8;>NTiT$?Ti_zQDCo~yj<2E&Yl;m=u@$Yr`doqVsH
z@y3SIyPKEX-6(Nz)dZRKjNLc8p56+Yywh^!N1mmDQr<5nPxqGQ2=z))7B+gNvn0Yv
zCc0s6{7oxYF7cu~9@%TxJ8h?*zQGVwaC4Sx!n+OZ@jni-nLUZ0_+W-A>y1(llPeSE
zux|LdXx_`8o}rDrA0GO2t1jlb<-a6WesL+sZh^R61-DWPZwXzRW83OFH*&*Dj)}Vq
zCOy5xGqG(2vz0=d<F3WRw!3n!=>9mssWkti;xmDi@FLy=w*UVueWqpRzUM}-=~c-|
zk+LO<GncyC{Oeh=TH?Zma}_~BGe1cj&fPab^D!es<P;v`)!%yMVplDV3UrlHQSD&R
z%&?2!@VrGK<%%i?r`g>`gV$>&JaBC6(#X6!SNCv^skV<=$L7svjJCwySt>B6bY_?)
z*WItGuD{bSHf~8gc(Ei_$83&RXxh_K+qm<)z4l(5Eq`(Lg9!@-v!=7ybbJ%Iar5=P
z=-<=-6mh4BIo#Z-T5)Bru3qr>tt%QWf_}Y>2wu~w+tKjbRrrjB$n7(NPjvWoKk#h3
zcfxt!gok1&VppDP-pn>S`(SsKjn$_I&yU?rw@EZN-PYVZYtH4Vm(`XBTZ%3CFOk^o
z?E7ijV%EtIgD&i_G#2}&)}>p(Kk?Yp(7P|!yUNIVi64BiqQ>*+K0A)2ye;0TiuQL;
zneN<JclYHkS0QN`?u!gDQ_eZQO66W5xo%0@hyKn@&w|5@E_h}1T?#nVGimaH`JW3G
zS9N}!leOX8%DNB9Puw`1!m>H?vNy@`1~K%uoz!;QHjncm_r?V&^5VHgx1JQ-I+<Fs
zKcA^6aGmXSmihmZ-v0=^9x~4+Rd2S<l$<pa0%is>#oXgm{&}pjhV%IorcV=Q$1$a8
z>@<Jy_1^8<)7KR~jemLZqk-YkDc2t!SpI*u`^yP&<$^xCOzVEX)iv{E(!FrQe>Hn}
zjWf4d7tjA`cTYc&D=_kzZ}4ep#mkv&8%*TGeM)a1x9tA^djIjpj9oLjvrZ)iwYsK8
zDs~=rZ2I)#^iJ*r5(_jPE?Oo|f3oiD&H%$N5|22FcugOj310c-WcCgoQ^(DjzBeNz
zG&L5sgiLz9;F;JFNAZ{i^UoD$oIm)y$3P|c)s~Q;?kA23D!h)F?Y#3}`9&QHer&bz
zb=I#7S2y0U(v)DXv&>_&<EUTJI;%)f{O2{lcmDG}sCqQ6-}7x*gFe@t&TC5^l>aXY
z34Pho^8T8!NY0r*qN@s)o6q5r`8sLygVpV=skfXaYPsiXi+8_m?)N=0qvvIjvf-ph
z7h*bQHpn&hi-m4syD9YkKVx^l`TTp=LW&j5PW}1B^T$M)^-=l6`|T?0{@&o-q-}Ta
zhOX1}@>YrDE#KPN<&szGKdD;4TK3|h)cy3GPRHiI?G9ZybDsv69Q*Z9y$fs~6Be`_
z^L)DA$n;Ue5~=z|nI(cV3V0+EF9wUBdm;Jf*aU`d0mEZKGww%zue@QWw~JAUUxzDV
z;sUku>hsPVr9uYm((V)D(_)x5?~wJ@pP$roEAv@Emh$1-lBaGjk6y6i@v%1?A`7xu
z%{rP`c1W(v`@T^_YfVRkh6HofyziT)eXKiBC075SZbfV9g4T!ft)UP2>OZiSWh`~M
zCtSzYeb`uP^QF%0)z{{pc>U`7QulwIUjOg?>ub=Dz4Rnh@Kf&7n*m~{PVIb`z&QVO
zb&lrr;Mm}ov%|0a(hu&uGGj{qLOr*FhK#M5434T?Q*s_Gn!v5z$|7r~qEV<M*_o<S
zrgb>eL9<WD+K)%GNaf@tMlZeqg}_5znki8Wdo+vHd?y;*_E>YG>w(5Zi}FJ%rVP){
zF}`)VB^0$et!bj~bUz;@N8yl}fB&qNI(jPJzUkkqtBb#zzHX@cS2iihwCl=&Ugi@y
z4~<+^#5+sn&&^dT_m10H_w|)*W%+>ztJ*xT`T9J2ceGncw!Nih$FkcKT2;f=S^Zda
zp<7n<w(NzQj~V8$oYoM18?1Eh+S2&hb-Gffxl7l29Q=RAcY^2{M%UW=_nZtoPjOZ~
zR+3kk!yVHks@>EVS($wF$~os{=l)GhcynVir`m*7SFE#x?p^fTvo<>EqS@Zdo0F6$
z^1lDE^6}<6-Fi*Qg=@bFw?tcS;|b_?*|z0yn+T_eAwwav#{w2dK`)huUaX-fCV5E3
zX<Tep^^|B8`+DXHzh}(CimQ=}cpUh(-(0!mufOl4#VL)njmB)&YbH3I(n#47I<bI-
z*Yb&rw1&FiBt_pwl}U#jceYM+vl9_dj4p7P5gQRVbB5FDLdLYDC?T^OsVb>$H$Cz~
zQ)ZSdsk|cSu}bhtpnBn|$g6XT>#i@Kf1~qSl=i!=K`$13-FhfmJLSL0j!PmkN48E_
z;vHKOuDN36vFGy*ZA~Y#aIO$p<uccfHEVH6+n3wRRxewA#V_~V^!vWfybke^9uqey
ztX^&Qb&5!kM8u`gG~U}+8N$B?pVqRzXw+c;rbmfOdo_=Om+sdMhm<-OHB8mjwO+fG
zGulB*{8?PcsaEm$(vMg55?=c;uAjN^r|2!`Z||&xM3?gl#O@bktcl$>xwP@Ui+C<e
z<iX6aDQb?HRmTFQZ=E!oxJ;u|Wy+j&2Q#JfoIkP^<tuJ?N^PlcmAo|}^$x4{{(#Io
zZHL+Ci_PA@Y-&^R^0rp<B>`)WY?HrW$jG7KXs~mRe6{yp_emFp|Gt@9X)v)OVd1X-
zei?}#TAfEqJ)%y3S=cHhP`*keCA)A+r&`rjZriOpc5vzH-8$i?e=_UoZH?7eg1a@6
zXSnuDtp0MiPa^5*i#H6-{0I0X?X|XZPKeG4Qkfd;V{lQVn0G?>bLl6hPZ|9C)}>7H
zXzTnXwra7R!!(w?=Um^c7uFP<`t6*xgUhbVEU%0IA9{6TX=&c}i*qe`%|4$gma}*G
zeD?iL*^kV@`hjKZL~pe%x+U$!^*mKtTy06zJkON@*IG*qzcTO~u=DcQ6mVeT^e_nc
z`n`C0NU3)>i%6NZtg4FVPT7AMn>KU0F3e<+yryOHQ8z53Dau%?HR4;^wA|R!ZpUA>
zYy5EI_Ko>}I$tqGu;-GHp29&@pPH9<xGb+*irw<#ik@(YEk$Cz^uMRAU%1Z9cx9^g
z=j#NY-k;*G_dcyx6x*QkbhXdBiuMgQANzt<bxNeKF&A3;<8_*-ac{u4@_n~u-X_12
z<nrNhvVY33tmaQspJ2L?@zT(ojtK{^eDQFa*Qt=8$y3#tuU6!~;!V2-%fkL|4<>6Y
zUdg4FW6)o>%4q8X&J$vx#sW>ZHY^BkZE@DQc;SZIp{<)e9rf5hinLg%2&sA~a0xlY
zhyS|RAIR`dyZLd~_Fsl=fnN^Et4SSZNMqSicOjW)^#@mjn-@jHI+kjmZW7d%GiN&)
zsyEAdR(FA!Im2U#=dJ(e9lm46$+Sy#>bf#Nt-BVjmv=oa@T`fFlAkFyNu@dTz0yJB
zIFZ*c#d;5X_A6@g`mEEoy!M@Or{|Ww1$UA!Xlf{MvY(V$lA%6%Nv}&S)7_Bs9*znQ
zzE}KLF->7g7d^(*xkPNujYFQFrZ#X!T#>7f@zRP6+?rguRBp9EtG4Bi$?0$3>g-`j
z3)^SJu<FHHxl<+r2EDEVLSKEg)z`=rg>@XuY&((@JYk`TnYpvrBZIh>hj-$(?UIpW
zdlpw~bZO7D!<(kNrYW_440mG>FxcqybZy?#yL*JpCaH&QG<mFeuY&29zxj{Q;Kz>h
zE6<4^yR+qxXp!Smp(mo^a#1G@c)9<d{Kq6B+%@qSuhPkzajKJ!Pu#%I`F1kL?@0$1
zOcYsa?kM@hqvhK1lN`B8cN)+7efVrL<x4wnYu4h*zE77HPkpl`J-YMSmS<Z#D}1x{
zHfs3TwJ0{plsgB#>~y*Dj6Z&j(zLG^5;iS)dWvyPV!lC9m~F<z?Nw(yRC0`8G1LV1
z-?lKG6_cP9zov6%_9OAxEC=U^Y&dx~r>J<A9?NqR?wDOyVqeBZ3f(VCE7#xj=(S$?
z&ATodbCz2&tx53k(RTk}xrZ-*iE@5R(yU*SLN7e^Yc;23L}wQ5UE9PL#PzUYiHMM(
zkx7fCvY+pls6XlqXY#pUG~UWyxGC_D=h?0mp{Hv9Ur67oYx30joN3RL?H9PB8*<gG
zm8BnCxh<L<n;JQJiMvOgwWRP~^O(m;T9S{hE-(-OJS$)B<m}~E&gENQuQV^Y9(QSm
z+ozwLUhl75DiUL!J8dgZLQd|-4#D6_FDF#^{+H^VeEUN3`xX8EGR?~3p_g6#&$n^w
zNS7v^Tjge?X}EUQr=F$OITMX86mHwp`Z#&b=bJM;ZaRKdKAGqJ$^BicN}xpSt%+|V
zHr`|P*lw}%gi^HOf~uG;vRgCS9sHz)`Ife*2~B9yjd3(vVdCNB*>OOv=HXN$_SMr3
z4Yw!WSiQsK{->E!W`9>RGY`%AX1>2@o<sfbxA*JIi`5>>{IFs?`~Sf%of~tx%1<ou
z`<v0|+9vF=$NQkxx(p?|HHW1R3w!#^`DksMfAoLL<o?y|H+u^XUC?y5d>_$wb5c0}
z3C{=0UU7V%C!U$bd*!b0+9}gKf|gV(ZHovA-cxqq{hTkSX2^&x7dUZx){bbEG&`-3
z8)4k0-g<HUdy7M@m$&Rp(JN*V{JqcGJ#_W05377{tvCBQ=jxFU0#_$1W*58cl?hqz
zbA>T`v4eQ+`raw$9|jy*D;&r=De|;g`fOE8N0Z;-`>Ikl{Oq<#{piUSJlpEfhnh1Y
zCog9m{XJ1cd}40g_REnM+FZ6cD7<BGdA;vid-_!O=u`Q(n2+C<+x@INC3jVYywIKh
z&$nIQs??gtp1{9Yf<tYqf@=lahA)1396N;{MLBJMJCD~RBPZCcGsGbxwsXU7m51rK
zUDmyx`q5wI)zkOVn)5XCCBx30{WGtAzp_hNUH<HU4D1{De!oAo^>D!35YE>NZuLy<
zY!;MyXEJf`of(H&{C$4J9*FTuHSqV@5OFLq!8SBOHuk~Ui#%sEBv&O&n_T%|6@vqd
ziO0r|b37&}oH0FnPV}bBhZCZbtU&^@|1MnkX4u-yF>}L#DZiH9_u%2a6A~MFgSo%M
z`{|!k^G;s4u6dBfCQ-z%fvM%LX{~IX??K@(QQotWizl3${3cRjkx#_Qx$`%iUhsE<
zfb2ynKanM=3je)0<YGc^Jn)T@jk@+_OO(eNk#h%+WoLxRt#jk(ire3D@2Te;#Z@Pt
zcD5+(b4}jDDdgH8ztK(d0n3`uMVBlj+hUKrS}W(cXI@6<G!F&G^Dh(wR)k%iv52RO
z<=xr1s*qL%%SoFgFDh+3*dR0C#piae%-qC;*_ys?-*Tfjtm$=8=)Qg7P~U`(-5bS!
z79PF&(btDzq5hWUs6F@h1d9Hf%bV<a!p!IzgU5;4FYnjioo+T!`Sw#+%ZZcUwoHB<
z)A{Pejp+t4Hq#bLce;r0NVI)=Bs#?{@rAI++hBX9DJ~6jK65N_4{1Bz8^fvf#OdDo
z*Ng|&3Et#7mf*7@f#cmNZm!n<=e91=a`L?p=_@T0u-rwwL^bfDXG_P51mi4LZHIu(
z6-SM@uHF{uh`iCEF+<$Mi=}hMiH}PqqYLi(TPcd%lGh4U40L)J`093u<@~v(r$RN|
zgKq`Q?@bgq7%Jn}^5R5d@t&in3M7stN<=xj{yZV-dEzP)7hA}k<kTZ)UOx1BUV3cJ
z+K6nKxMMX6(cidLyW~{=oeSboipmu9%((XA@985w_Y`Kii8i+VW@t?Ea?#i{PyL<X
z+N&-XJ)V5{>8ALw{k`vzcZE*^5|-6$NXp*pdg6?j{M|=ybmKoK3T;chS)4jMyH0tV
zYIukxhi8uTyPr<IYnW^qF6;bs)_-%NPi6A|O?yuGTx7a;$HQa7e4hwe?K_uvG8@$j
z?rtkgZjoszHM&-sCcob@-lFYp?aT1zDVOUcXILzJ*tqr_<EnHft_4wZRqc|FwG_tr
z`YdpdVPjn;;b?L`q2<o;g;I6b1fAIyH8{1NeG|*_iAyMl@xou8M!#h~2OjWjTHx;G
zvd|^c<IN+FZ=HOTT!cIrd4!b9Tu<M*(!6EHQLQzHtRzloZD{uYxU9JEP<RTrz@;Mw
zLDD*bq5)owW?74wau)X+T?=%3xHnf(q$zf>hN}3#CG)R#S4XA#%vreL?}Yaf6J-+T
z^e^?4vbr_-+|*9rX{mvNs}5WY|20ego1((0V-J4@$8TEsKV=1flHbG`bw{2rb=S1G
zcks_FS01HYfko-O4swhKUN2-3P-zf5Bj&3iz<Z7*{gZ$GmIn)^+G-TeW>09`a`Uij
z;}nmp&w{q7{nKFNvb(!;qPXr24%;tIwj7hIdk<RO={4pGm7l1kZ+lu|f@DZbyocpE
zwG2iL&qkGibhD)AjvT8UKfGA~YX1J8(RH1R3u`8aRZf0sll@x3bDEBXwbYzLnQ~Ew
z9v=NW*=-7=Q`eCW-X->blO5SsG%w2DCDnGuaf*-4tMWao=Y*-H+O5hwaO7O=8IL#D
z{!|2AE<MZF#J<jxF<|P=hznOjO9Jj^FlifH&|nDmyLi}V?yCEHJp8oy{(G*N?<Y0?
z<4vZ%EJu+)r^GB2Z{{ox`LV+3r^bpoZvyV!zW#N(m{jK6Ld9DTB@=9J)R_j#@UgXs
z-kPApc4Y3={U=15o}G4^vUJ*ss~5vm58M!qc&6;RMzy?wCBgNWm#?nes?23MQnL;%
zb+MS2pedQSX~BKpT=r@031Vsc733W6%<|w^m9BVDi!<IwWtGb>e$Oj%u58hYkzs)W
z&r(B=Ecw-zdFuf~iC1B0YmQe-Th<Fr&lVovyBiZ+b#6{Nyy%t25ssBcZ@NAm$=h|^
z@ZORt&P3mY(;9DHt#@hAn%f&9(c&bydc&@FrA?DIHeLGjG`#-f^!ksioBLQATisqO
zJoqoEmpk*sLYE8YT24Nn>UAW-kL8O6oBAHnuCEau7bg4Y_}8t{b{1ND`P}kP>&|Ch
z%}xL3wrAT_?nOuH{+y{^;eI#H_w!pN&pq!`zHoW{;aaxtzW>n<`!~H26J`dr>J=Q~
z?frMs;MZaM4=LI`hj=_2+r3zsC+qz6U|pdh!cv@knk!RGN>l7q&e=zjQa=S&&q?H%
z+%k32%T*3ipLeOpNb2r&k(&AI^uH<hmL7E}pRzJLN7pyw)E7Vfvo%uvsw!SG&l2C=
zZS-?l&avuu2^W)!BmXIu%y0f1R*J2EsQcbjW39tRVg1%6pYD2JIkdFRMb~8de3QKW
zX}+b~@|<$BDm)zjKY1l&{pQ(5ndGXoPwsN8sJ`m7?p+p3&+N>Pg%v*v<-L!J`J8(r
z^l6)Ip6D;}7d^{88ZKL;p4_rbc$$mX%njYsdEN(U^uNn!u@z$Ba@Kv^=~BPy#!jJr
zm9+Poi;Xf}4~Vgx{_*L&%Ow}ru=8$;2E6BT_i;_Q^zrp?gV+O$j(Fq={pHwvaj6SK
z0q=?n{vuZn6-}FY^pcDAU9PLT6N>s+jOWij=66K*AM?+zy!n4Rgl^3gYSH=gcAD)U
zCryK{0Jnz$=j6|XO`W~9n>R5u-1Mc*C6<=EYZ(5m`II*!o-5;LLc}KzDQ-Wx7gLY1
z2EApU*5cyplbs{;;wjJkJ1=iNTmC=!OX@7Md7px|#JRH1Z#X8$msJt6YD475#6Rw-
zi>_&ZpCfwIJvUUNx<zg3rm%Vyuh}NT+9iff0xYUq53?P4U#!V+J?HVw0QnXP7q1$R
zm1p|@oatxzoV|SB+58CO-Cqx{^(z(#`dl9I`Dj~9sPW<z&r~F&T^A~Ne4F63D<vpL
zh{fXF?M?sYyD&U>xkWiK!12SebCq3FBVX}-+PBbtH|Lctp0#nOj?J7gcSn@(f(Whc
zH{^5k-z{`W{_n}e&>+Ha_wNa#)oa*wgiXXAv;DrtG)1u8!|i^HLx({<*Yy+rhTUCE
zEl;vfxoB-Y6n6hkcyj5_*f+OwHtYRtwykNc=AHY$YQiCf;<*7`+ge1Pig0C~`E)zv
z>IBQ1eiHX&r=Mf7%RaeJrL8UWrIgdRa+6#2UbhbI;Cv9cUhwON6S;xB{YT{a8mjzD
z-c`zFr|2H*Sm0XcqP^$k2Hm&~u|6d=Vhme!e=}G#-&pOncv_~&Y^K<4``n(SE;@ff
zVRp>~#}i-L^Dnq-$7pr32uMtJyjjz`qQfOvTF*#bL~1jaR<E@7omDx)kH1en5|+O;
zbbg1+lQPEm>9#)(wN5WJW;G8kOy5!YxMTmbz{x3|JmPPw(#>;)51JZThrN#N<~(fM
z(h}M#9X97k;I!&hsv7fWxGSXa{ojz(T>0(PoPwB;jXRCJ!d?I0J7!cn{YzfJbhlTE
z1$)ENU$546xanVgRzm;Xyvr@~UPlS-2>B`z`b3KRr{spsXMTi#*|w$9Slgp-O2AZ+
zhBxiTcmJ^Itn6VaT+veKywF_k>1m_yVtT4J`?=3;I<+rmQv_@7g<qxJoN{fcnUZ#a
z@yQ7}DkXAOa?y8Ry40+l#(#}h++^+v)g^6fzyG={%e6c4n2S@f{-3TSCLWfpcke9t
zArlj(veiDs?$Vb<PgpL*zG{&1u%6)9R;BDIBffLG%RFnnwg`@$k)}Q0Idd=X&_A=g
zxBmAQ-!qTT%w9OlciD979IyNRhtD54e&oiHW!8_QD~cwY{d~7qXJ^qKzUczT{{P^}
zXXJbTF5yq+oy#m{$J}DJ9M#WFn{sf@o#dY{3T8}yP-gx+aYfDVjx9`g+Y=|~H(Cer
zsJ)u~{5a1$F~2qXo+_-_%dOAkMhcb8%xyN!UH>Cjo3Ho%wngQeJ?<Xc;jvug&8wGP
zEbUbuo`1fiWK`{xFW)biIq6np*xl|D{nj{PH}L|-vdJ4ZUT=AD_RLDrGjm1Gq|bSJ
zW{UaQBj)SBcUJ8_Ca)z|$#YT8%0!`T!F25{J};`~zWK$lW?u57Bd<E$Jyj~?H|9?a
zwUfWqQm=AKajMI1w+dfj`3nrPTYOu#bnbRqYwFDzve>0T>_&d4#pyl259?pqp<7eE
zQ0LMUkCXq)ZRYQs+|tTmS=7iC^x3jw!Z$U=noUeHFI=)cF8F;hww$uWr^Nkd%7QuD
zy=1<4#kfW6Y(JQOOjhOiLC%xHDyJ6jPf~7BY=6HhW#1k-x4<X7b6#ufOqzI}`{=db
zmXCjBUuJY^V)Wd0McUxOeaF`SZ~lHheqp|C{a3YLS>fHG+CfXqdM|ErJvB{(d69?Z
zj3*03R5e9srK}WCJJrmUpe0iA;Tgj$)5>pOUKTGu$EYpp6OeS}g@~r-(wqrKsaz*?
zgLn8eo^(~bHa~j3)}2S2xu-U#ZkeQN+>j&LnsvOWG<x#e+j_QFi=3h_>z&!bzV({Q
z(hu(&%$D-;y_;Fi@Q^X>|Ftr!7i|Tahr5Keb)G(Zf6t7|T)nJ#Lcs&GC05?gJhSv)
zoHJln59^M6n=x-S_kPzPVTLm-ERlO<FT6^4e{X*xllr^g&p$BMw_eXV`Yd1jmi_rd
zD#ClMv|k*^y}IUVU(cEOj@4cbJ13=|`)s57Cd5Jb?*)#aoWGx+^6%HL-u9+8Uv=)3
z*V`llx*20uY1+-;3lZ4a{3!U*-i1v{`Wj!WPMvg`lThTpkezo*f~uxUgx8U!%Vs<{
zB4Yl6bG4|YM1rf{wup}kQt3}#yS5&YYUth<affT;R<;*SlZ^s4ubyHwAw~P-o@c9g
zmMPRP>QO(nCP(13&el7VPV4QxW5mh1FX+FX#gh{u9y^WwjW6py=y~1AJHPF9&y@dc
zD_Yf~|D6)Bw3B`^*=DxNipe5jZi=bhhE9{5x+l8*dg-8f<a6ew`C*?vpZ2M3&Tm@O
z_bX&2?;@$vRjW2wg$B>tXZ1pRW5Cnb3pQT~T5Yv`$(>Wj*YH(quTy{DRl1WgYpv;4
zZO7mf8<Vdty0rajnD<)sixI1HH=JD-t!H-SN|vv|(TwD2TeIt1y%&1%@8VTc5t`7{
zQ?|im-PC*czIg6h`>fUTPV>9c;vE*XyWiQrFuQ$jL9^bj&2!H(J<WQ!a8u5V)dIXF
z721=<N-CE280K|84lxgTv6r*7tKi8*^(8M;=6S5kR98L2{Qu)&A$OOFX`+cC9mi%T
zo(fzjZu#St_L^-kHoD98`h7jAWI6Hcb>(cYGKLlWPCO@1aZI*)HA|`a(&Ke||4F^^
z5%x&vesRdIBie+qds^<6>+e=h4P4f``^<xPp&sWM4c->LpT)o$`Tl_auakEgTULoA
z9gM77{eIv1ds$9Prdeq%30$r9b@{#`*LCYd`@dza+0+tx;`)vSwSUxizq<5QXHC}8
zsjJWKbkm76K46;DaQ>KU_XeYHLcBkYq!_=w_2Bdb-eqfx?Z187GW{)Q%zcKAgvnQ*
z?cJ?rdhyJ5%k5j{OYgeB?DO_z*IP}kcfDoXuypRb9`6XQPd>L!+I+kAy6EAr^hy6O
z9PGRAH9=cETY$lw)uQcTiaMvqgX1FVQ$Ciwa8{f(vnywt()migblpc6d)YR$$`n3q
zp8t7Hn91?U(Jx&+axZ#cl_@se@1UQ}esC6#?%}Ch7V@=QUP<&9SRlagrj2Lqfy)g0
zu5X)X^4#Z^_ids3-<+OvJkvc;D3)Gj>@wrVBGKRjQy%ABTCSJH(Zjo@vx1A=Qok+G
zy!*D|E1Rn;m-sk_1W#OiF+g><&)XHvyR;t0rfuD=H@CpM?4@_i2Bj0?v;0e!T%MZz
zjL9_ls6ZyO!<qPVm!y&-y$zN)-cIXT;<VjyV$@8Xdl7jNy6dJCDhN%Oo!?a<Q)@fX
zxTN{1c4g9knaZojbkjR8mobFQu-baw`hVwqhMg`4OyxGKH*(E9<*;qT5!K7)0$%6C
z984B*F7gPux#worj+MswaxLlhzi0MXd2F#d#O1BN#e3p~Lw+J@Yg?zze0VEng-yym
z;rZ5Q&N^0EoR#-F=y&|!z3EY%OQnmR1~4Za)_U!B#GJQXbg$+O$6d`wbvaZDWA1Rx
zx8OSa*Y-?9Vs?bzyqV{>FL=`%w6H7m#ShDpU0WQLuS|715g<1E+r$Vp+3T+!O$zqk
zXX0hQNOAq7aEYTc4u+eFYWdhpuJKpmFYl?84tX9XwJU<H+~(!-`<I+2nEz#u<3B5+
z{psoK^{0fYpFBQU+R?l(?EjU+V#1~o+S?TN^DcC+pY}^5ux7FS{$EFCuy2X?|L|(s
zj{=VkXL1yh{!Tl<DizSk!K0a|7_n$pXIE&<$CzHTjK$(#Uo_jCR<`Os)_FFtS3M_D
zJ=$vFlC@t#R(e{Az0Io7nR`!cd+e=OjtvG3EFKFSzH4b{2Va$X{Og&1)?8J+`e4_2
zR~Ct;91xi9@hb3WY02W4m(iZJ!FRcyisiVjoFfy?wL0AJtGD<WW$~<AaTmV|?n`HW
zGC{@j{<bA+CoN{a=E(bU#`C#(E9@9I_y?#ib?Q4hVI}XiZSS0BHV0i=DOgt^yjOJZ
zJGVrhT&qXX=bkKkbM@1-{XbW=i#}S`pf&x!`jcesf;+GDb{twFyY@y4lS<kC|AEEc
zt)e>R>a(@vj6Wow66rcI<)zQ5x+zJ`r87G>aJo+`6IQ$XCe32MCBsFoUiW7@QUSs0
zo~jcQGhc^XnsL}?_3j_5_PM6Y73uKAncpv49W8poPPl<Vvxljhed8m4zE@ixnswA@
z=$)I_c|*GVDoghz-%ZOhCN+F3@#=}3r8h}B?2?!B<l|C*FZE06PFdDdzDH+mS7r3C
zAZxP~Em1EI`CdHK`lG7oz)CA`rP}0Ozdn`drmct#Z_BbS(sV1|=`9vr6FyD&T<}3I
zx3&3xe?2!{5MFi3={m!*T1Cwb4=-$qc~`*3C7dUIeCn<LFFzM~$2>ZF<h6p<GQSe5
z!;iwYyg%yP^HJ^3a>LdqS9j@7J|J?yr|m4C*!)UY{$B1KOXc0mHoh#>NzuPjFt26B
z$s0#*9@(yRS!1)-7RSp|k8Vx6wX#&^pyK^ZuIVMyUs<g6+@1AG`Rbaj=KE#Ze}{PF
z`)``PN${5U@l?Lq*N(WXJJY?;B_u;9{j>NS^<N4<!gL<!v;^+;bYS}D7AUE=?4^m`
zUEMkRtj=6NvC`jPwQsRf>AG(#G#6~$(wCN5TJyWZdH0!JYRl`aeP7<2c8`-Y>)+Zl
z4gFlgZcat}qJLWZ&$9iL$!qcc#4pnsE2e8cJoG6l#W$%arqWz<pQ_(372p4RQ-akW
zoG4{5RAW6T$s+P(O$wKg#{rf%Nu?ZiHEK%L3fCCB6)wx~Op(9(>r_(wlL=wBPI#7I
zIIR(GEb>z&UHtjUDQoW-Ut&8uXJ(Z83{HEMGNZFw^SF7h>tvd)E0p~c!~SUQvtv{3
zmBVvmi!W<UwT^K<%;MiE=1~(<|J$8^#Xt48hD*QK-~I4jAXwwL=93jgdhB~w^`G9d
z<c+|F@Doo9+m`M-^L*>8cX~pchAHVcFZE<+v1e^a(&IXBK8Z{Jg|}B&de$F}2_i}#
zG*Sc74x1<S#NE>Ra^URLKIc1jJeIc{tVNtIWThtD(6!jce2;B`+!_yMvEF2c6RDf>
zlKTEPHOpi$OnJ9$NdVXRXERrEpEOufr<VKjU)IZ>^Jzw>&HwSvnf2aPgTXza^|9IV
z#|{@he9>MJwYD?m`Hq_Ar&+K6E1Z0vaAMoNMQ7#I%~_mc8MrLY@mrRi%blU&+p@0u
zhfc&k^|e_aRz@rq>suGMNPu&5_N;fgU)zq^_%Q8Wu)%R(w(w4_-e3H7JF>d>Wqd2h
z*cEp%Kg9oQ#-h4?eM?RJc<kHL&vm{zCQx$hU2#s9-mQ1uDaU#LpV+50A?D4Z5)se$
z@7^gqQ#xqGzW3idrh2Bc3Y^(FXVuqSWXyh1xvN{@+KV+;mI-#QlR1CZ;#@ELMCXJz
zDt?ca@-`fnd$7Lt-~UAgOx)jp9Zy}6<o3YBTkq|*A6&wc9<wxbf0(&^yLWho4a>I;
zDk(~vG}IcD4o*nZSf@5A|IvaoJl3DmUwEAM$XFJ{dO#+5?yJ}Oenp37y;>mmlK*qT
zytq$Kt5&qj<c6}X>Ufu`TD-3F5}VNMj`Ib2>d^=9tynGknSW1959h=)OI(hvmszW^
z=u)p*1M`YW++GK+ZrY?}(GarekwTuJnv1T4yO6gI-?FlltGALQ_8aD?d|;?Pq?_A&
zZIYq%_CV%lEoni6oF_%)oBE}@eOVJ%h)68HRg)BZCuqT`jlTPQ|0V_0%*tMStv71k
z<LMn6rhT~hjV(h{nN@Acgv(p>HnRM`bZn1`hK;bO`{wTf&uq0eDXpHw?C?U1eE~~K
zuwqQZhaEyZ#cs-$-R7%#!jf8=Z9E>E<S|~<WL|&DYMP6s_GA4NqeC@GcUH0RFFtO$
zcbV1x_3AR2R>dpYtGorirkO8k5ZUD{v~}&mDIPEO=>AoCqg%Gge9>j~6YgduLeIan
zYIt7WwQ0lue{KIW_?OHG>UA-6NHi2Ne)aMlukcxC#s!yp)h21j3u~WDGK*iVc4q-k
z&SfY5LuR?cx*H~GAJ*%0(|-4b<L$j5M(>b_nBM-dDc)fh)MKXTFdKU>Xp{VUA!CV(
z^m$IT8ddjfC4Enkuz&9)>lcNeQc#oj@xHc{chmp1-dmO*7TJ8~im*|4ay-l8+ZrEq
z{PN@)m93u@GQ1MJ*uXsXm-|Mylq9>)-{amN<2u*K(x)jK^q*@|-K>|1DnaTIla(wT
z{LgorrC(-Yd*J_niaLvEfM`<yXQ^4`5hec??J{HK{A1}>M*T6YDxMxH_m%Vy^qe`C
z<GQmjd(D!`+ujGypKQ7DvhKSNIa`dq<34%({!mzyw7ur^Gmpj#&yEU6y9gSy+ATJ*
zy~w8gXOW$&j|)cz)3$YL3;f*r4EZaQnDU&yo_nEw<#DQvE|;i>R<{$=rxGp&6YbwG
zjGKH{-Qd$@G1U{AYQ}#kCOgTgy-jQ0A?<KhtqPYbM<!iA_TPvx@%WWp387c|BzAt}
z&0ifh%TP)*Gok*}k||pyg-`A;S*xG#y2|fEJf}O;3tmg6RW=Jt)>TYpTe@Z1EtNPH
z-MhCgu4H>=dw;E%gZ~k;1Ru$?`)Um{;ufjJiJiC*;?1`1zm>6SoSMmW&6J4CrfY?q
zet(JK{_4$lHK4!D%zFXP6fxbJ<vd=F+CeE=D(|!ngUk(!#1*eLUzy-;Cv^JV*T)KF
ztqHyncQThvW6M3pW9pzBGV{pRNlPr}Y8(<ctSahn6+4xG{xR=$n?rX#vzoZLy=ZYe
zmoQ_4N5?)EyQzntUrBqemh)*xlGQntnsY5-FOP~nYWL~=l>CC<+d%a))Bh{pt3E`S
zFL6{?h+bI~y*|kG?AK`D<7VlXSr@35B{~IoHw3VY1_+lrpFZG}e4S^SnYQf|t?5pT
z4lG(BSqy8N;--pAOcnLsa40mZ%RMVA?8fTgz=HOKFDy$pX4i@2xKHsfb)CE8kl_ZU
zk4MaICTPgqRq`qOT6pA?p=-|(yEDGa7KgF$TAfph`x0)=7rx+H`??=4@-saf&K%Zc
zG1=4d{_7qC$Aq?qJqBFe;;D-S{8twFH=D&yd|Xzq6i}~}*}5%r?lzti4LWhc=Kd^B
z3Mn~E4Xw-x3=t8GGjv?D(ikF6nJ?i9n$53mcluNtTT9K3jJTtf{sw1$<(6D=>Nxqo
zENorA@3l26wyEX0Z{^o~T;qH7_nS<&*mZf|1#W)$P?#jD_U~lU)z4Lnn4j-DtKRx<
z!Mf!uw><IM+#dFFa@>nok=&R1d<E0HMH^!cm&~~uA-cn6*2)sq1m=M0(VAZjr@4Fl
zxf-ANsQL0$7NM&d$G<wAeo($qXtHgG*5V>DW(P(l1;&^kt)fm-+$GhS7+g34rsmG5
zd^+`XfuY+(=VeXD?pHjy`DDY@4f?IqzIs=DN!WSl+Mm}qzdcI)I!%}N^6I)pw-W+x
z--wv`yd~NDO>d?_+_Il0Hzx9)IOTP}%SP~-{SF3IJ{Nh*%}IIs4zA|juh`d@<eX6Y
zaq+_y)13>Lcl~$B{9B@LX5oFNw7K_MjBsd6ZF@`NW~~ghx?^gbPL_;K3#Qb*Q1Gf;
zsQ+Za%r%;aF0|ENo^A9ZErrFRI3;`cwCwYzgH$=695K^BqjT+*o`X+bmfySF*WCuI
z)cD*ds6We`m!e*^%VPevAKpn*)kGg{_hCvFRf>4MAzbsiVCkd~;fbFrOy#Z@=e`iG
zjyfpzDvO~<Y5NX?2{TJe;-5u*zA_<nx|62Kjbl+|{nuEHc9>oK8go9%nO&jGcLC3X
zUz64c`EQ@9rMam>^u*-Yh$*tOrr5Ggo$8<^=DIZN#)in7S``~z+IOg5@!oj9+qH71
z>&w;4#BaMNTUvYjZhn3!eBJ-TWu2>@9+}9qm#sSQ>Ppu$HcKU^wCSasbx)YJ+WW7$
z&*8)M0m2S{i~IXaITa&*d~r9uooxEB(%-vbUi&rO4>Ozl!`9Y5^8fpDe)rjybO|TE
zs#al#XNPvEuZynwrNWssV?pMQsrePNi!6fdv$H3go^Gl*E%7_ds_)l>3pEy3w_Y>5
z8D$jVCb}&9grSgs=+l$GKYa7v5b^Pwb^gcBa|{k#W{T(cIIpn&*4FZAMSRE6o)w+j
zQ{v9F2`%0wwqrpb>+-&fJJr7YSnMG4pz8O%B}UP*CZ)<-oJ4P~t<Rb?`L}{n<+19f
zn1{dCK8`AF-L55F;%IL<<>#~Y+nt_u-FYbT|9Go=%7*EG#10yX|2(AaD7E?iA)TAv
z+DpQhHoBhP7=Ga2`RSF9POseP-6o+OwD`>TMN3|qc`jx!oEegAE*5VkmUpGaOh<o3
zB2$XH`}Cxj_H4%z{K6FZp4lH#Zo7Kri<ryH1$>Uy#hi>g>l$`9eHC%)o?UkL7EjMK
zXVF9R%V%cj&&}a3HIx5lJnwdS+-7%<D8^mC7dXyrjo)!qylVcRsMQiyyZ0VB5g+kv
zSM;LHAKLxv7ftndF_f~f&2Dz;)<4mi(HFbqyjfV@QmYv^uBEW%o?Wvq`R&r3O8X0=
zKE}M5v1j_(ZI?BeH>zDpjXh?(s!+q=omy|?CdUWiFEW^G8U82ttk3ZGiAndKqy6r0
zM4*?}d#ib>A&O0J4R1UuuT;9+e|utjOnLmC^2+NEYuP7jE;zpU(`4<sh4Q-=`bOp`
z^@`@teSFtRA!z0X^(rlqrb<uMcZ=?9+WF+oxpjTFvfqCBQzbd`^31bKwuGOlZrc3P
z+g&vCd8+&EujiO1hD5Hhuw8fd>_N4c(|<h=*nLN?f9riG*_#Qn{M%=1a@p+K{biQH
z$qUX$L`@u3e_ZsNuEc$_rS?g4q4>^=Qr<oXF6VwO^W3|v&MhQ>#U)5~VYKaI!?@rr
z);)jgdSfbLW2Svb|M7%Z+H&2uEalSnwQt&;i%$2o?M)SFo#7U|e*5W776&r^NA2G}
zSNl!pf$tx@9OUnmcAdL$^jXBiSNH$ytV%lH@_2PrvY<3qUdOIK8UF=V&oq4|oByqa
z>+{_E%P*OlH?VALU`aBbxc}<WMoCNd&`IKT&UbG;)D@ZR=<z7>v+b8#k4*Qe|CslF
z`?@K`haQK2^tP#cy{!A#v^g6hceZ)|)4qG3!97={d$RV26)jEArfvL_aphf{zmx(S
zhm6OD1qYisgtbCWq$W&n<rVju<FWBfLtCnj+!K|6#|-@(lT{`dEx7A2nJHS=W#$Cc
zLuFdrvr<eaJZqR?p1kdnXTWmbu3WZlH*a?F`p&Xuy(S!?of^QZ!s#WkAmPB3Rm|!c
za}P9nU;i%@>&2rH==R24m^p^W!CO%A_Qvdoi(FqH;)ybsHGZY)+|+%aGg!?da6(H#
z69aRTf?tmX14C1Hvx>%!8AZ2S`nl8O8XOcF&dwEQvfNQ=@~oM|hh0E3)hj`>fiZuE
zQNy;kjayisne9k?Fs)IB?Mt3T%n#S~OcUc{>TD{HbjVB&Rx!S?*!7i|=`^j&6Q;NY
zo!%?I#7?2==jXY*3ViZ61WWeaW}PN=A@jhuU!V1Z&#aGE{Qv*2pvlcNPo60*ES@<B
zy@J^vng}#A{eAMlL94Mnpw(RTi?E{@U&6x{Q_Ul6tsE04iL`s}RGFZ=#KUkVlkfqK
z#VpbVN3V+2PCd~2-?f6XWxBK!qvize)D^B5Vpt_tPKfk+<f<~kWtO0rxNO0J#xNtV
z2@On)AxBdi62wGqIZgIm7BG|9r;ULjTaBS*<_cbuWsU`LX3Impd=!t0ReNnbIFH$9
zfwbe^T*IR|ZE98r@_9{n%~>qi`twE6eo+_MGWk@lNlWIGRozSFzxrLkaiUvRz!LtY
zuP!hwHVO||@>MWp?iv=x9TysBSFF-@RbVo`(7`g%{6NQ%Wk=dtb2luSq|0n@I>b8n
zz&6+LSWcB`k*tcDxlXqZwZ4tcsO6q6DgScm^!T_!!`TT!8jcaOm_;luBr#Z+9Z2J@
zy(YPD9_N!~X}h0Q>kF(@6=?l0)}7T<FqhH4C4$L|L#;4D<a*l2+})~IW;;krUym+O
z;Ai->r9Y_g-O2M`)8*6;O><{EJtsx{ZZ%u#^vnj$@{qe{o;`G*Uu%Ec>O##Q!I1bh
z?25747o514lpN;r3`k%K3SV_0kXd7QfVQkqi%5%2?T^cWYjb7<U60J3QC9!p;+4`M
z0lpM(?KZZrj~UAOcdm4qJu9{8KJ!Y~C7iC)Oj9|-Zb>X=iY~jkG&P|4<c9roK4q&h
z<YjF<snN}6(Dr<GPUDf($;>$y=CZ0TRG+_Qdzirdtq1K|uQC~0{dV0x`&HITp-5g$
z$*GeB1<S3twco6nsLr3Zdiu+KHI8Ce#cTgB^9Z-UaGCR=)b>x=<@yu4?wij@^wa$4
z^5`dbp!pTQlWbE<wY`^K3DI4<#j(h-jC0C<hr4o?0y_#0az~y@)zjnXXKv@}**#?f
zgR!ZaF|!q~(~nkxD-%5xMPwQ>4;|4r>&+91x^P6o!o1H=BS&P><pl=}JeGH=C~yi*
zaN_xDVt63tO&j;ih~rJlvM+hsxek8h=9j%H<M(5!g4>nzAK4mKK9Y2?k(1&Ou{04-
zHdE=e-E(vGlh5;vj<9EZN)(l7Y}>n}^P<9;?wE@joAfJAo{sliz}K9#Y40Y(497>t
z9_v*$9cLF!QQ!K5^VS7*f$y;yPM)3ej?cEWa(;QC;PwBP^#2(mJNg_}@}y~;?%QM3
zs(m!0u<!S!Jv-dJx)!V~kvy}&A?WjLflW=a%!%yUB5H?PBR1GCH4YY9azR~co0=A<
z<&{Nw>DF$#>K&7d;@-0{X+GodS6H=a<~c7<U(c<t6rWA>bH3y;Axv{~kL~3}m)ovQ
z{27tb%ifr{Zq?2CcCVJ_Xh{a1p7CQfBTFmKgO%}iw-Xjs8EEHSxaE|;OH?mkG9>u<
zBfhP1m-UV<eQEHIt6{I)WXWn?2E%^I3(0046V5aoGm(_JoUXTd=ES15xy%bS81At+
z`Es;~NeQ)zxEYFyI{8jzYEsN8)7t2e$kQ(JQX;=c*eInakz0NCl>e)59h$gIDrMfm
zMaw_(WN20?xr9da&J>eZGS0k~ZfkFS>nj)20@G=QnHjE`+-sQ*vY4(`XIsg7Ym1rS
zX4xlwx4AF+UNBSKSn1ziCEC&Hr*dNxufqCb<>m$dri<+n)thY9m7~5>X)$L`r&j-A
zuT-tW=Pn68nO3ywj>ChHOWUS|Ipmy*63N(P8+F2nE4hO!gSY5b_0p0p>@)A^Rqwof
z)ipJF^Y6C{N)~JAJ<02i40-l?#?KW-Azc+wQ<hm=^}5?qaCz4}ug723Fi(1T_Dz`J
zji${aOlzm)yj6Me_12G+pzEJzey@2Je8T4XcHtlf??|=CF0TZp$-X`dL~?KNNuK(D
zKzu#Nl!fnvPA1uS&oVfe=~IwoAkA!IIj3yPL6L<I_PH5%T|M+=DSwhXN9OU1x4*qU
zk{IMWt3=AV^Prl+uE;mJZP^=Tez94ck?kwuc{Asej9|CJIwh0U&QE3uN?I>b@SnuG
zIz`jTedp!9uU5TSEckuR8M|9QZ(8)4g*7jj9PDeHx`$`V<@*-is%4?kEhUq#ckT#Q
zY-X9P$a(0z@Ww~ZUWOh?t;HIgM?9FgK1_P77j@TZbFs0IUPt=PDJ(&2$`lM*Gj-#v
z**w>LTC%+Pdb;!OYo}QnrZ;_Ds^7dy+U4xKGB;nf8MAJ#NYA^Jw!v)6n)5RRH25WZ
zPv~-7TDN+_|Glj$Oi?RZJr9^?f0a@>%j>kwVL_Jv=V!w6-fn2MmuC?^$Em^~u=It4
zNO4q6V~NR(-5Zt#SFb;}ps01*)t*+%$)`f|1mdT?ez8mWcu+XsgTqV{pE9IrHC~+Z
zV;NVbPh!f6;Ca8UwAug7Y^eHdIMLFk<As&qIhCT+8M!GpnRiYJnf+_i>x{2TS8hHl
z3~F_K!KpG^_U1+R16tQ~Cr_QDXaCyqnukfO{*%)QlcHt`q@<<(|G0sbarWk!MW%bC
zq}3CHI}&q0J!|6GR(&dI?YwBmwqre;Zp;cR$tzDicyrf<BazcRa)UV;&bAgXyqWQ3
zN#w;#h6)Rf<MPeBlzXC2N&ZhUWX!lO^684yKK*YB-Kw4i$>u)Js_Ny(pWby^^G!P0
zt3B|-Q@^s1y16~#mZwgAE<L=bQXs_lX7H@*C(33nu4L|IsoS0POY!{e)po0v)IHv~
z$H8ri&Y^WP`yBK;V}w7<*v_3&!p7*7TRJtPMWrf!Mz!+tOzmgQ4E0L?*@8VMGMr)T
zX%bm%uuv>u#h)gZ`6i#;3{$kKE;eqT@xZp%w}*Y}>~G30D=fY}ENnZ$=yrAgaod)2
zM^|U1bSA8NEO0R9vPrDFOZN5PRFloJ{Dtx<F7mMg+OYy!d&Jy~p357UyXAgNU~OdK
zUBbHLfKHi+;=83f9_l45g66_b;rZ9N{^zoVa|wEgA961;wQF1IVQ6T+HlQX#y*ArP
zXvqYYxapa9m=kxjs!cg;@GPvHL4j3I)oIoS)>#Hz3z=I^U(YNMk8*jT87pL1cfGtm
zTtG#z?YCLmp(BP%%d*xi3lTY3!R)}g>YB!#g@rSOvj1*xK3}XQd{FRuh=A~mB2h<e
zJr&lJ1s)HBb5=c1GfxaZ<5vB<MMq62_wqKSuLn#$BFejt1!gsMPP<|7L9B*jf<j=s
z!2(BtL+yEsJuI_@q@ILjunSu)e5~|xnfAl-B*_r98_QS>r*yNbYwUSZ#}%H;wa{5E
zS;k$|<#JO<U1N{Ci_4s*qOh$pQr|TXC28H9V*LNrLcy2~?J-BSqK<~XyD0E3(blj$
zd%9qnSV!@tb}69`(h5yY4PT3Eo|Z^m(k?XisC=qm_^|lYGNHzn5);F?qRqm0)7y`2
z?<{JvoT1EWwo&2g^5*nS+I*7ThsDdLRj7-(L``%qFZ&+wmMN=F-0*6;K$X!XCdG=H
zM^eW^eX>@_ce!gSe2+Qo9J;|Rl(|Ck44W2vU=?>^?6za=^W6nwR8?cQ$t+g4%wwJK
zWUBk?){^H_r?mbkZh0cI#vuLCV!NLruGwiq-wXu4D&%|#4i-|Gm{2Zo$b16V#8#EB
zoyMZVZ!U1X{UGgF;h5f;$)>5Prka<;R91M>NyDscTZ`2H+@%aVI_h4CYp{zaZ~NH#
z&RpO_xb+c@0CC2dM`y@f6)If6$okv~^J$Ml&Xi7O+|+LMxc6QJTkHeZ`9W-{B7(I)
z1b;NnlCThbrlIL+Az*l<$#-JEOSy3Im$*(3`#q7u3Wu7it~Bkxsi4r5)7Ij#(_OJJ
z%r1@7^t`0Op6^ae63n<a`BhtHyGe9UY0jwRjLMoMAk`Utb7n?HQR~JZ6RXV`E14<+
z7pgNWwsEFtEKAeKyfOK-t3dDatguuE!<#NM+$S)t(t5Na^+{5Z<xMdI6RvFzIZRwr
zB`2kMCVCt&(h=(_44NXG%bd%+L8Rr!)cj43N>&2P6oX_~ql7hU|IghK6eN)T)`87q
zL9SV9)Zd`!l?%HnPR`wJ)NQrOraUux{zCnnNj3UkoOV?#VKEoolQF|vE19F*S<*O!
z$Gj(4%1A@F@QjDxGeN=Whm0y_xUzgKyjmf6_k;}VjNZgma$6?JKJ1vqQQ^Al*)03W
zMsstw7aE$J0#bqp8r?OUPIIcoGz%QzRQx()w%&||77vTP$|I76SL7aRQvA6fs(jiN
zb<QOWavMq_78le+Z=cBcc<v+j#dE`~V$G*56rK0Cspe_hgvwbA9t{dX8=^N{R%c$o
zy7^R{{LLj^3Ig#QK8Y`+>K{oRHDx_JCFb;#8OLVHoc|(d(Aj(9bI-}kT2?1*|6gaF
z#ik+1-sLGfacz{Cf}jHEOkLfIj@^^eepd)QEfs!um`gNE;H?+e>*RRrgX^?5PR-w{
z^HzcJ&!g&vw=@NB$}jg2_<DfLQ=$Bq;k@Z)f^4%FFYA)Le|-9u89|4RtFbsQVHaXG
z+`<~QVdK<Itg_Yu9LfTUucZ&pT4IzU8W}jr_m`A|=+d5_n@;SKIqYckBB{RjQ?mEd
zWcBYMo2CoQUSxgZbK%Zqz0s;}VjFuGAClv<=(CJm8*yzGhvV{%lIkpsHG<o`w@el|
zG*hwVX@AyZVeihV2TzLlRE8fbTTxP$V{l^58&&&>S}R}f$d{Y3HKKfCWVx5f&WUB#
z(^pmn{a<A<cgE*c1}dw!?P8suEHHoaj<riR{w`acq|v68ZTM#AJi*DE*ah3VoaQeN
zi^=Mk&-Ku{Kgs5cNcLr}?N_F_RcW~0*dma+W!ACjEKvuz&OKVN?Dmum?gF1obiS!J
zeb^PQ9AuWeVCrL2`;yP4vl=I!IX+kN>f--qYQ>)xtAAb0abUS%Nv2$DqJ`H)i>(>E
zKW!+nbadPml<TyY{g8LifsN}d*k&)-$24)r>7#9&2it!CluqnevNJ-!H)}?~E&1T6
zrF`Wf3_VNJRtbo@%ZaIn{$1&-J#!|9ploJK=*fjzmeR|fHqN?UQDjnVoOz`v@>#0S
zWP$Ri3Db5g&r~wK(m3V+dyW;2S?R|;mC8!pO&Jf@-SW$2o)f;(ecOj^Tcca&O!A)g
zWBn=T^39xO3l6So<X*jLm-<vAwy@29LWMi#84LLA(dd-kl$@|yDOxj-(Yl*yr`%1w
zHz()qnI(FlL_<xp;(+OzV=TK8x&#)S)(D<jr1^ZNP4~hX>bo3%POkFtnsZ$&O+eep
zlA%7vSZGdT&;_nXr)*QZJ8Hhf=`dEm{iW@l79nUP5}7Pwevs|!DV@W=r|mZvIIlkK
z<mwIO-dmT>*}!?xO=`<t$%yGCmEOx1Habbv%yV?q7F{CzdX;O}#%HrPzMNgVGb#72
z_E9FrGt3+P-bf#1S-5Zi!81Q^T3(Pj`~O#@^tb9w-Jv^Q9<ST6@vKRjOxlmL$3<p-
z`7RS)vaDd&=0D$iE^_qdF7#QtT3-0YET_}EWEr=vmvp->dMI6V!@@fbLKC#Ne(>K`
zzAdJEo5-WtA6VzG&NcnMVv2E%K*-CIY;D)aS==SN4@=xVRiQ2T;^eA)CsWTeR=-!e
zNQ&D?K0jjn`wZ*BBVDU^Y;8W;XSrJaWNYG?)?JL!TN!tL<ZRoTn5Fr%jcG@l`LZ<y
z9--b}&h4Fcaen36GaM&wT#n^qbC&nm?X}kZK(*upM&Z4)O%GZ#L|f*@-Q>#)H<-+&
zU$J}c^uq?Hweyzl$XtFn&_=6l8PAgWqP44M?0v^pt{}Sazhl+nnl6Q19Id_E_w5Y2
z!7X?uSMdys;+dY@+T%56UcKJ;S^Mn#+m>q8OEhnH`&%Xde!c1a^Ma#-2Tohu3OIH^
zulc}|poZPTS0+Bn?laQLnB~T5p*1Jq#PyV0GH*<Ztcn%>95hv5wJ1itW*4jckL3lM
zx3bA99_gHVvRAQPOVN18i5Hvumo!{>AAGkug<B&w!@=hCKdtGhZ}&*D?f7T2BkF?s
znq~pNJ$Jbd-eZ1n&-QeV`ObUhX*;}o)&%^qJlk`$?fOxZmTr^SogX46S;q=|xG9yU
zv!)~6M|Wk6_+5blnaRa9m4Ts?r(eG%vTySI=es1&X<qBfZaU*T@%p5{AJ_hmzQA=@
zx><A1ohv1~&n^=xIiu((=x@02WZc=4zpgGYc(qbO%`sPZn%$Ylx?8TfoRL&Mby%`&
zmFUguqR~s9aNl?qy<@Ji6Z6E{FHiQp?Y{JLuJ(h*8(n(^;^WRFCY=fDIkWJxr84hX
zIqA(`t!F$pKQ^njpsBrJ_S<6y(&r{u3Vz=n^5l<Z-Mxp2O%I>E5R{iwm~~ukH}9j}
zx#yqE2n+l@Az7&Qlab_}l0%$^=DQ_J;)LVdr>;0v*|bNpG;pUvS!suShhpAh0iC2>
zGcMnqG`Y_$=3bBh>)OQ|1AK1sGTWTq=9iLiqb*lpkDtIh+c(GWygtpgzx(>7d7Ga}
z-LA;hWBniFv9mf|AihI-zt#L3dGj6K?!35X{y)wlqm`ObEyp<pAJ1*N!q_!=TkTp!
zy_7r2OQH%69dWywwX{k>=<y$u$7>XH@+uz}U7o7%)hhKxKXa0#bI+-NL5n=@iDhwi
zZx^jq4cnN~d1__g;u_t3`AR}M+tjyt-FP1>u-HLPas8PE#jlwJ-#poP_e1Pco%I5}
zxdMrHf`;qQ$i?57TJ&tP?Ec?jXLYCDWINui&lZxKxYT2550BkJk0(ZPSqB15dr#kx
zsW~xI?B0tPHQ8Iw#;jVqt4?D1uCuc)o5=e&o9wSWudl8tIH6@j^@)qmmKO_$pLl!W
zt!WAW+=W`_9<evxjnEAL|K&0FgUzqDMwbN%Fdkbif9~`ujt8qYc)gafKdOHJQ)1H_
z@w-R*ZRKVcXzr0avp4VOwe&YWB|mrn`#JZ2$W;9|lkdL`iLnki)Aq~Or}wn?MY#uw
zG8!hb8vCTyaPHf=qxeg1jlhXGt->FI5Bmfz?{W#Y7i{Hw#4+ogl;77Si|iK*ow&f-
zVcYU9YPn^~hHtx_-nVz9?OWvT9=PyIqR5d;ior95QWAux<?wrG=B5NFC^ZHP>=#&g
z$7vyBP5CEDj!zOR7yh5^VZdOhaFC&ixj}&;L4a`ri-(k&hmc~Ih-TE5oSP<T!3P6b
zZR=Q87CN{$$(r{Gd2V`e(va0v$R={ae-V#aTDe6_W(Ge$H&N02P>f~5<!OvkomDng
z4_-`L>SgXRQ^B<<WTn?KFA0N%2_fr(xA4ti+LjP8FNSN@U55$W>|13fshOTW`G&zy
z;iSkO?+@?p8p)r1CTmd6kZ_=uvrMxi;_b~Nz2cu`7VMauG_Urwf?nvd%MC%yX7bB!
zRxW=rmxGH>WyXic2Us{a@J%?hZ0g#%YdE$^1*}$h*T1&pvs%V&W$k+h3eSJJ!L{RE
z|C$-x@_aKx);=~`c{wM&`%3nSyN+rM3XC1!=PV1XW@m`Aoy4(ja=;y>)|GBX(&kFb
zGvi-)9gweo_P2BSk_nGa+A+L6l-)4%U6jE=wy!}28(97;E(tiq)vEMZ_Y=phMN19)
zeoa_0t)okSALo=APMyN40f9%?1e6%M-bkC0#HE=wC#hM}gJ+HVL9-AokIm0+WgF=I
zkUFqQrBbU$?Qla&k5T`;mZ0TlHJp7mXxpyd<lQ9H!KB@N;>0@Hc?HIoy-nFYCac}M
zrP0C5?YOB|#3DM;_1x`(&Fu=&uQo0{6SgR#L*cSbQm;Z=$nOlBMjI*p&d&~3S1vLN
zF+I58%^ty_*eX-xajE0bniq=}GQ}8&Dx@FkUl68T5~@BmB(yZFKTb%~A*AxjMCbV0
z)>T?!?V(8xNzHGwnl?-m^<KW^lGAGUZEZX~+<G0GzPX<`v7&qOp8u~V6|BqI`}^A(
z#kH)Qv2(Zh_6Q$Y;uF64#FM{0r&TMJc#ST7`pb9GS@_5Fy&4>soVbiy7WXRLoxz-U
zxNYWcT}xBxWyQ;1%Bi{ToRgFI)GaFbYRTm<Zfzb--=9p*@BVc{qO~`0f$=)?Szlxt
z(kvt2<@^0}i<Ddu5_opy^SvIYcBapYySwB>|A*u#^#!H69*6lXb39kB>ehO-NWpVY
zpxx><UQI_uW3Rkgy^&*Ap?gzV$n0)SmgI8v9Y#DodCEVnKHb}-rQP8*?*VJ)rX9;m
zHoe!_EiAL-wB0|AH%&UXl9t((E}HT2?jDsswHG31x<zK+e`5Tw>0gd(m$T$q&Qh2E
z6E!-%x^pu*ZEY5EVp`blq&riq$3Urh%DtNvrD=1Ew$&V-ow4<5(x*x7rM^>Qnr<ys
z=<TvP5kK=&Y4?hCo4g`cZeohRxqgP|gpFI)@Sj;L*Y6arH{pP4kJ^q*CZ7%JozI`t
z9dq!$zVXB#{+LN>$_M&u(h6rPa_s-sy1wZ{x6Qh2UPq-5a{VXTRkmG_n9@}|x8v!u
zB7tTJ4=0U{da*iA#)W)&ZZqbF<v5leJsiR$(Zn#>Q<01Hc9YnP%bOmZ<T0NuEa#Z<
zOwf|m-_b_fGj4{t;(|sO`xQa6TI5t*wTxDD@y!bUvrIyp<z=eF)!(6htIW0vB=%;j
zF5<YMvVh_L#;qEz>RKJgk8NVz_A<jG|8wB<GfivdbUQ1AekF@@KR4G%e3{=G9OHNI
z>qJ+fC2?(wPo|wpU_0i>q7rq(MS9cXgQlV9tR-85r5j#oSw8f1EYCW6FfV!8$*YUx
zSXa91KH(7J5IUkLck+tkF=nj}2NkAWr#&v&n4da*k;Tu+#ZTHuOj|iDvqSjR!s)Xl
zeY5IV!VjvP@IG`y`?aU9PRLHKO$#207MW$MpO<2=TKsfcX70nNn^x1e9dMoAbmp@1
zw=bu6X^EW3I;%gwaQ5llcaE|i*z0dR-SUO9{Uyn^qBa{olPlAm+=Q=gIA&{NI3w5W
z=ANcMX$)IFE}4BoIB4e7{{;+^Dt^Z<un6(gO;lHLb~rhy!CCi8yNG1;vqM(D0v`xW
z?zWn#HlgO$+^{=emI&^;=|1Vz#9uu}GA8`=5BdIS{v?ML8Iem?1uv95I#K8AayxFW
z2u0^bhH<_wQPRydQ!Z`&apQ%inN{G;F41X)^YYgQS47{w@@<CQyOcV2&BG5kE{WH@
zd(!WJDW2iytMmWgE?Lm><i6jT1v5^*J8v*~$`VPDSz6w8F5H2egJ)|TxA5-@Q=e_E
zCf*%2!RGF=@}o;UbtZ2*?fo?<L}+4@TE+9-sq;!&wi@WB+!EaU&LMrpdM7^V2*Le&
zqSuyOZ8`HX<>}JM*tfI(MRs;v*nZZas%72(;D=lx*VM!|&G@Wb7W!Olfhof_fhBvd
ztP+cT_Gn*jXy?wK&C3%0orvg{FxtfG(v!C90&6AXIfI!ueVq3)U+njp67)BVzuYEh
zL(|@G%F1UvISMBqeAM%8#m<$!enHhHy{Tu+G+92n8(6S?Yk8YuxrE<OIxOl$$2n!0
zKL-9yTzC6>EOuQun<chgN>P_9a)M>{t?=!)j=U{nck%C&THoPxeY+E@>x17OeN!eW
z?+xPKC;e#R^phVJuXb|V-+blv!yj25s=Q(i6LeIjOWizvl-FUir;)Sf83WILg#zEX
zf7e`Y^Dw;3W`117#$<vS&zy9-&r{TMTZ_Jjbf2tc2{<kLe_2t~MZRYYUPm)8@#f~(
z?BXt-rO@_IBHw7b@T}g7<MN%0iYski&(+ze7P`rQ-I*v`-cs}G`#kwKXSmwEUocrW
zGqF4RUZ-y9*HGUdZ{!_6ywv@9w0vXblyvbsPX!BkIlWo#`KSx6ntaA&%0b?iGYgdt
zO#Zy)xA!8hk_nG`r}V9vrM5*<k<D8x$VEI~P2%v!mE29QGP?4vbY9~;61Muu9G<UR
z{;2gAx^GOL6Ka~7&^zyfUS8;)m^~iak7pjY<?~qe?fA8ZFMeMxuq&<FdMEta)mzcK
z%9Ny+YUrd%cgH?#a^KE)<xINd=hm32g@3<x%<r&$vi16(Xa5iF+<0%{lgt0@i_*^s
zsl2-QCWrO$j?#rTpSN2oR_y<Nceb$c)-<;*ld~s#ZnWEbNp9}GkAAKm!aVn_&80<i
z@7{Dd8?Ik7-&DDO&EDp3rVrhB$r(O+SQr%AKgVW`@7IoXCMJufHg9YykX^o^FfJ}V
z(k|+HRerVn<$XUdvD{kuEBWuLv)BKHdB(BC1kHWgnsDySzn9!9=a<+CH88Rys=sbK
zk^b1$yFcKA*}0z?U5l9?J$jINxcXJWqFQCccIK}u_PT!5&g7l6;rB{wp3BycQZMZZ
zcjip9?R&R+#_Oe5+~>V`FF)_=WA%zzU!;uW4oUq=+P8IGXr#9Jbt}Eaiy2e&b1o|@
za_&0b@qf|&22<tBGy5CX{A!wD`FavZ$ddn>8i!8mu<zQ{uPJ##!EUL*vK`vXcrJCm
z6W+nsw8g@5_s8Vc<WF-ZpAvo&!1rv$mN^&fyt~)(dd`sXaop>C(0HX#*hJw6nR@JZ
z_JvH;J8nF0Uc#QaJ7(o*b*bOt-}q>%+HI*D9NP+&*6Tgt{NlA-&T*T5l9~SRj?1Sx
zO_r!v-r~;s%w7LT|Ja9}%NIKo3Y!;H+a4-h!J@ofE3>~TLFAq=hnpiW=Y=T&g8NJ>
zIc-+5iYTvqkf2$eta~p(R5p6nYv~5z%QIwn4$U=K`@H&y9;5w<85>o+olY#6`RaAQ
zT9Zxh;t4Y+tU7o?Yx0Bt7AKUgR4;KyUljdyc*llSdsc3g%AO^+_T@4uM@ebJX=`s!
z+_F)7`$J{@M(zd7I)#OrhpKw^bvXYpwsw59ZIN&fPm3}463+ix9lS@3`){m~JGJXw
zMXyG|cE*V9D;s)sFRbm+=#tP`YNN>K``F!VrDH(1gmGY>c*~X`CL0^eWnms3+{Ook
z?>N4*ws`uQuSr?+F_)nS3+L=g2hKYJ!Cy{%{Jru>@w}tIC8FLOeX((!&KJ|yH=RC8
zuK)CC&zuXEpFPFDN=~<%rEZ$zP}I0=LFC?jr`)VkR2Z4uk6S3*_Eu<LC3LrY#j=>*
zCWC(c!rtSG9i1=uEDv#U&Ysvbqod2=zof}2#Y*8W{z%EGo>H4rx@3a9mtI)7<<A8E
ziM_4KE_M$)cYR(Gz;je&&WxwdJ5+4!AAB%W%(8j1K~$yagz}w5KMo5l=veiaL*Vz~
z&M8~w@t;(C<9Ryuh|LZyj@_p$w<$UpvrNj6nRLKNhjXLbiBpoQnVU@`RSw?bEG*IC
z*&=@S)@GaC;->|+t6Vw7C^2JIOHaAY8I^_}mJ2L4hE5*EOHFqkY?o<xdc-l%dxul-
zRC%W@2iDBIrgiAatG=(+KJOFcjxuwq&C#1sv2|6YQtyGO{X3kdD@|O^v8_nT?DM3P
zXL+1AXQ(S?nnj)Tocm!~o}yf7=j=);apMx-3m3LAO#Z*kW(_y<sk6;m=IvJI^L~gg
zv*G6q?zNX;;OzEV<sz-0c<TKOg(d^Z@@|eOC*Et49NsU*Ih>dlEjYLH&*aF@2m4<f
znOo7fXNt!@75}J%7eC&a(64P7x#noZ8mFa+M-}!?crsH<A<<~+>#ds}E#thrj?v`U
ziVHp)EavaMWt5XCoi43?o_m*aq2}&|llE+MP|4iS<+v+@+3SC$&hlX6n{T=qx_9kM
z2$=BVz=JpE-WAM!IHk$VV`)9}m8ODo!B_S*&%UVeX5Ygo;odEFzXG+?0w?$L_`fLH
z^U2#X{txG$)5{h%a2l~J-jd@v!({o4CH~((9W(Hew@Z>QoM?7=_44^|{vVrrrZesF
z(H)HLX18@J95rrLUsw_&eD!C5!c2t+Ej$nJa9)`q^3dg4Q%8r8Zdk>iNsJj`9uGO&
z7c5*{wN_+P>$byvrZWR4aD+?U<!n|yDk`!nYn6xC6OOOmi)^2I<~`BaSae2d>7j!S
zKJPVFFG{d9Ipw2MeNyjD(C?hfOsPC}i<(~DTx{dC^ODY_t9Op|URi!)iGu~}Bwa?{
z;u@hNJ0{uKoVoYpY~9VvX>TuQxlU<(xYky(p|E$E-Aa?`t*Wv<cG99dq}EFGr08xG
z_OW%;t7z4mSmUJevSXpG)Py<LPi%4j;&UtZ;nslAu0YWndV$NCT*NQ5=>0Eow$uq|
z&i%jluOYY2$@y%d-OQ>Q?p_*qS}trkYrBHwNRz-3Jx0GPhG8cvdK+I{wiCU1|8=lv
z?(I(u{G2zM%pS~~64WOl8ve;cSh{+tWNgeX@2MR<7W3coxt|VEk=e=JsHC_wbTyaJ
zskMjXiY}eH!kHj5Vf%qI37p-$Ggj+}@-W4C>g&$Ww&vQsGIY<3SYy**g+!ejEj^~a
zn@`LNwwb&{PI1G|2eDDr$L~HdoXjfJ(J|+5g<aE%Erq&Qcvs%*|8Q;QR}Og|dxedW
z*NQ^YH|f`F-+Cl?J|t80<dGe#H@f<N-MTb^YenyQt)F|gM)YLJ*oH4%9{%+H->>l=
zYbUJL-PQEVZSI8sJq*$cH(a9Qb(!x3pZfG*%H;_Qw&#XDt3A~iAbYuCK}&~bg^tTJ
z)_u!MwsTa?bvL}pG9$XbU<1cDj%%F!&4H34i~91uvdSgjlwqCpVe`frEA39oHe7l2
zu>5P{Q68Ha76NLUu2(%>cfI$_G;QUjAHuI3HDU~zy-+z~U2klwsL)0M9i@}AcX=9f
zuq4d2h<owq#(&9s)suE9{_7C2bKP^bGvMKb`6~`eKifEc>dmQFlbasI?OZ%{;iMza
z60S?!5jfwgc8L4rQ$<gWl}F~hi1c}T#beo_`5%_OX6*hRBBtNMb55rtg`20mOW%&m
z>+cfvxhCRfZ|_IGy=AV=T*I}!+3WwLT?Gln)er8cOcAY9<Vk(F<LzysYf~mDUKG#`
z<dkK&F8NT<jN?XNn(IOhvAzdiUd&qc@S5b-cBwSKmtR@8pMExV>$78L<BG2G8o!Ja
zNRm-F)UOaH+%03{DZ777*FmGtlF=%&7!nWldke&=@K5EAozSpp`+@z7Klmi)XwN7)
zHsgwRpx&-d*ZB+P810@Bd%)=Iov)r|yk4g?l8<>lR-HM2QA%>wrU$%@3ubzqUi|9f
zyB%>Fh1Mr@=b0sjoDFpRz{}_S%)9$Y`WM~!PrG00MIJg-eYEUaNc@~bZ=_=R7A{T^
zTp!fh^QlSt>#rx@SM90!b-$(U>f1BxL;l_^?CD{V`>$|*LeI_3drL|ZO3rXjzM7D3
z`ufU@uvt59Uz?Z^ZIpdkz)wWaj&str%MtGBZ9UOx(LSDOJLkFonbFI*;n};Gg?b0g
z<(i+oJQMk1!ppbTNl^#WYp-#uoeo!<c4+&V{a-EJt}IQG)y&#7LyY;4QAUVoao44P
z2KRP*=61}TtZsOCw?X#<+oP^h_imMKj(D44yHs`Sxz0(3Q(Jue+V|Xh6`I-lF*$VO
ziA~%ovdM4eq?(GY$qaRT;+<llzU5t1bjo5=R|UuOnpaGh{WbL#Tb`o0O@GaKgD+1~
zo7LO)-dgcZdxf3z@mpp(lDS7V9X=>@d&|<N9JOAZubyr@cj2=`m~+nm*Pc5s@aiTu
z{t$PoT-fi?A@X<9#~C8adNh;mdSl)_*#B(Cp{e(7G{xlJ>-3UdSSb2@uQLBsKKs4K
z>)cA-WjQ_Pwd7RIy02=QG<RoA)!uZq%aVuRcuJh_pU<{>;p6AEUq#dJyxDSK^^XHV
zn+l}2zA*ZB?2$p&CsnuH(?Ui$DLe-^XLK(W?OqqGw@~QPmd$t6KEL~r%lkv8@lD*x
zt;xdwC8nKod2Y6J-xj|)a!Fpcw`~01yzz6AJFZuCH{^S+QG;GqisEt~x%E2#_ULlh
zGD+@DKm7LHmsri4S!?^=hkp&#3*5l&cl;P*r$Fhc_;*j!Y+dGcWY4vD=&RV#U-qog
z;s1y0k{!Y7kuHu88z=IMXx)jDcystlZ@5;@-Vnne?YuchZ;F0T^xxEbOz-o@zju~!
zZc1Z+^=q@urwKj*%YwA0Hf>+DT9uvK>e046o1)Zbmwi<!XOeuo%W_gl`DTqnpP1CF
zR3E<RU)k+ym3(AH_l}5fHShV3A6#-bqt$HrR+*HT_m4RPxDQQID|1xiPvu`Ew|$Q7
z{fiy%`KD&1yt-Dbf75l!pQ8bH<qErQy)!UdmHNeh+q_k0&j*`4<^4I4d2ar~9INV&
zJQ3S$qu9b`)XQ=@@S3mFVEf<aWW!%--RrFQVavaOr=bj@=WgHglik*Oe1aqI)=Pbd
za@perWF3Cq<7NC`p<#Hv-TTMPXH}mXeq=CB=!%w#F8OgP*LC9Kb1HY9FIOu{c(Q++
z)5OK+CT^K{{y;_1wbPt`bjx*GVl%~q^)GQhUg{PZn`+CGd@W^{%cbwfmUAAT*|lfJ
zzTiFg?9?jW1<zQmzfNp>=1U9LXNG&^xT-t(|IV+H({%Y8$MVJR?DfSQVjr6PUF_9o
z?V0v@<NdEM)wdS-UJQPE&;RMlv^l!&0smKhnUnGK)??>?-%2KINC<NmNl%l#tnt-r
zQ5a`mDMS7>m&$+h&vQ<m`S^ao!r0|EcPjsX|M_0;MNVn&yjDTJxlGmjd#;rA{x_}6
zIeO-f(%H`5;{9s9Wjh%=7@64lZu~d8pm4B-OIS(g#L@**bc579W*QY9n4-zC+(*D7
zwfPt$zr2vj&JQZ5rf@A^U}6!Z;xm=$q|Or0;%5y;M&+Mm4rD$&KhLRMNH^=tivub`
zEMAhz8$z|01PNc0j`*5l8y)PwP5M;UgYc%n&^p=OUv;7ym$0pgU0t@|c1z*yLu-GB
zJ@s8Oo2BpO?sA9teLUPkZWbRuu=L3pay>OjVD>rQ!dkTQkCATLLdROQCn3i^H|Q<o
zWE0ih;2khG)oN|t-Ao6~8=I0Z&+`>%Jix&s{N&i&g3Nb~d&Gaqt=q8UJtKpdl=!@w
zh41bh7FO{0j7?~I#yN@O&ZecJ+AEk>-MT#M=Bxh>)%(^h3XkYs@pE!w_Qopx33fOB
zFumqqBl>+$;F_LA0@CV#O%JG5@=kfcz{ORn;8;IVQL&kispC>3Z|aWAVt&5YWL?BO
zRg~Oxcv*x}6=TJO19=6XJaXhd$f4X}Gtq_3NsKQ=%FAlu(ZxP?aTY2QT{M@ry70~n
zed6?O!I62B-)M$}hJU@mo)DfWdG%PUm6)mcWT(ICmwglXo+vtIzcqa@tC4Zph4@-i
z6Xjr$n<poB_N+{~<UAo%O;TJYKrp#io=+vxDdxhB%V{-yTV{sXaA`itTv+NPr0(!+
z%H`~(Yt(LMvs^tE6XaT`eqkl^!)<e%*6>}J<8?>(`Ko{oi~gUQTqrn4>moxyZECl!
zUt4FWQ@EIDd<}Qz&#N_2d<kzhXj-lo(oB{zRqz+ki<lCBTdU<Q!-K5bN(-GTEuK4j
zx`kY1kmY>x&RuaQ%jCs}^{OuIdATVg!1=3?%F7I^Of7+>iB3|whkVowV_cT(x$YIt
zDI6ktR4X>sJ4|?fi1!h3J<9`Iv<fvh`ds5n-E{K)t)tgYCFPYai1$1BXPKklNADLd
z6>7&K=4m;y%}jRWe-M${68bFYqKa_mtPG!KHWhJS{x6XS=Wh~w<+NBWWlq^%O%=z&
zLpxtZmmKGHS?zFuH{ct?dCuhmw-c-92C(EMZ@<0nKCj4peJ0PF+a0#KB+hwr&*kdo
zxQ%=Hu83~oR6HHLnMHE<nz~c(L^k=%@aE3*&DzLleERV_6|0wKopsY*Jn}MKp1?Y{
zJ8jqNJ`L8CD!vpYaji`mm%_epTN3bY%G{Rf^V}0dj(z0xwJM!x(<LUrp!k!8RfK_w
zL5G2XfuVuPfsujZKLaaxg||aEo3NHi#HrpxMgrnqb0P{vT#baY1-EDv-gW5b&QulA
zoFMAjsK8qBCBrcF)MO!FDHYDm&s?T6UtRfUht~p&*$&N%d}eHVzH1)SF-6gYjq1k>
znZg}qca$WCb{Vmz-1RD$d(!YN$F)_@wbwL+F&lEFC|^;~o|&7nNmL>{Vf7m3`)WsK
zPuFvtoWZ^=aK_fD56zaGnxN$D<?wh5lXS7*0nY;|jZ<0Umfb1a;Ltvwdzz+2#TKUv
zOaa0!8Rm%zO=WTH+YaA4=VHG2>bBmMp$f+vXS0RfyHa5ib(iUk)Rq+!7528AXT4PY
z^WuY&#x2)6-|W1*f?>JO<ZJ)`WC~{eoD{Y7`#QTVw!TeDk5@0B8T(*O)A#q+zt@$1
zKRM5SrP2X^TgT7`<sqHN?>jIFsdUu~+wKUg;QqMuLN)8gO{X+wnV3(|{dU9XLN?b)
zwuhY>|3WV$ubpsdQRWQJ=@(K=rX6hUo%YZoGS$2&$W1)8#OT%pmZ?P(CNk}4QgL0e
za+*N4Cfn1Q0mZr@sZ*J4#g^s^Z)~1B{nbHpfwmiK#YOzIb2XRE%3V5hnV;3)C6oP`
zH?2&Y^>(e2pTG3C%z1M~zokuTU7T^S&+Y70<px_7u4~t)Ho1kRE?&#HOJHgG*P~of
zb8cOA)9kXGmg+QT!bauvsY_2uwghj}I*_og?|;}V2k&W*Ryl9ra*3W)u2RB2C4f)l
zb_erSv!@Pu3g#Emm=9LyY}A}27}L2=L}ihKoR-L3hTB%=fwgRz?a#~Cn3!J3-|uN6
zs;!^udVj`;B=@-t4=$PK?`1x#w9ab_$BvNZ-JTH_f*6z{gyZZ5T{bK|-D=*|`sLfS
zxvihYo=jB}_<!hBa#UKB=4=6Bw#4JLu4&<#`etFPd`+H3JU*?*&2X_-(DHRghK2r<
z36uSd-V_H+=sLA*(f!=$jU_4<KkHRxyPf%RzHyr7*2}EAzNRyo?xe@>Gk7R+WsY_0
z>3K`zy30eF0)0=Yw#0PiY-EV*Yns-vb&K=%unNuF8B<T3{NIw<rt0sx=)N*jnp#46
zWlh*4fBn80Nj;8nk}K5wx7}DdLw~j9>Jw#o7dEr-wq<9{yPX#8xRr6WwiDl3u^cCF
zE*9k;_0p_IQE6c&6E~&%T2Cn2ovXdaN#UWi{N6a-7Z(qSEquM9f@y7bTG{db*lpLZ
zZxt`Q@%3E0P1EV<Sp{=hwa+~GmRcSfTd%RK`dvJ~d)8N<P<{W?8p$6-#V$r>Wj?we
zz<j4O;>n^TDQTe&nKw`8962C;m&3zu(##1dZU-9`PV#QF_So3fs*(`9LD7>b@jTNC
ziy0Gq7;+8@`P?a2dRmgLaQ@@1-F@mp>#i7-tK@L|vWaR=zAzyx>TrMk{~#u_rWa2#
zxE+-a-`O&GCC3BD{R{cHn-4k5ndRwA2zM+xlhl3S18*xc=dHzAOa~asUg^va@>!vj
zEHzcM#eK@dLn}jEkLy0w3STHWS1#Ulm-RP8Po|%t3fYS%Xs+~Ywzxg@dCGO23r_=-
zW*<1A-=MA()^W)BaC7HXGs|VQOP8xW^SXVjWch@ZSJZu8C0@TWt8FXy(bv@{tW|eh
zP-AG=`ZWHzFvo!#(-kMM2v$A0&G}iw#i!L`ed4h%3&Z!B>%BdBLFFK)cSIE9qU^72
zyf3wq8E%*DXrBAV^=#JZG?5jH*=9{nvzQg>Qo^=S@ZFn<=BFRHgm?&4i5;0I&Ui{<
z3G;vRyn`2V*FSl3#p#*P>%Tq=!lh4YH~%=@$8)5U?^fLPW77_A`uogLfc04($17+3
zOn%kx(#+OtECt*qGR<e=dF&z)E^fSaN@1fRPkdgurjTJ?;o6EhCs`cYPGw{oPUw8x
z_e3M)Tu-pYvX?IDVlSs({;+Xn%^hV&875WZCC_-*9DMa^n^wc+($I*7oM$*Hxk9&3
zV2e@}z8Ygu<m4{vuC2l>x9!hK|IR%VG=dk$9m!j9inX_2;fdn2c;O_4cZI(88(PlS
zbg@r*W7)pZUZiv1f%XLl4;kk^+a(v*zDeQ0q)9X0W#8L*Ds)onrqf;PYi2v$*vO^p
zuqLWO(RJs%RO>DO<H9u7I^0aYQgncM(;1@!r4KC6zEpnrt;6GWNlJ8FyNAP};Mr`R
zKG6p=1g7+5Eq747wbf(e`TD9(S>rq2QHzfpNelY2fNO8C@X^o0>P}D9Uk2xvPI<BU
zlh+mnNl`D+%e(c}cAd}bIiwi<!zkg^v=zD+*3NyzQ#@m?n_klvV-EEWt3<sL#=g=e
z$t{waOPdzBmDC(je);Xkxs7Fchq*R3r-$SSh{<!8qz3453K|RkyV;wu_D|TyPfRzz
z_~gW|PH1h6a8!u$o2cYnD1V^KTs^YO+wN1Bk<RMna<!%B6C(D#&7UMQPhs_W+aA^H
zpHdE;Zjq@}l({GnIL&O`)H9R*$Ia@PZ!-BY|KT+!w&ga?n0NV<uV`YqOPi)bYQ=N;
zBB}Bn{3#2p#Lp(KFucBVnv+M<)udT0(U}=c59GCvJh*7|(_nAl;e|C9Q?~K0WYJ6)
zej2zy$+h|3wgv8;rmKD~nex8laHYFnX}HhXN$u}kHp`28-TdTzrj<2%PhDEOk5#YG
z%nFyA53Y8p8KiQDB|I^0Y4h~S&k~q9O;k_Y)YCd;^U66=$2=UK_DI*Sntj?w!dRrQ
z{FzDeEDo-Ord~J0uH~~PT>R<UZ~1qzbkxNqJ}cA0LvlYVdRGWZq*fbjRpnU5blE0I
zCOCuX;@N}`N=i=|^!Hr7{ralU?lYaNm1iFQ|9x4bSR?AgoH?8H`+wH8cz#xTy`+XM
zasI}KFZU?6zkfdE?~}%JXH+jr?JbOZcAD?(_mr{=j|0A}V7L8c_;qFWMMIUPX${^?
zSKQ_bs#SmUzY#t8wpi+&?SVlHP7Bz^$TPll6KUyK^X=}P*4uNN&+;l{r~DDHTXx~V
zSC*QR_cBuiR%VD=s=cqN*PR+Vr`P|z^{x!nNqS-3$3J~L`)9rR{L16=qpo?a+&WL?
zzfGfW#;2nu9YN=U<yNi{*P3zv&&I|{YU)OTN}&%Yhwov!cki-O<`)TX{a1#6;zX9n
zvwm!6_;Zl2W?Qpvjd;xi^{u|Q4R;*u3k;J~kU7^gIkYckg~9(H$L-x&)LJ}5X2v=d
zJyj9df4TpS`dr_zKQpZxmWA({b=JwoSaMhRhoE(V0@KuLg~S^;Hp&0K82-IERWCi|
zeX=O$Cay&X)E5bO2?aJrPHQxEEb(yj=2mVL@o4mIY!tHK5;AJi_#rD4l=k*>!xJIV
zC9M_7(*oZV7YbW6F20~{<iWkyoj>%rZpidZrp6k!g*D8MEq@Q#yF_p=4luu|#Bj}|
zL4I-&!_)M;UrcJRDWz^Vx;MRS*5We0EoFzcW^{$M*(FI`+%9GMF=OFo$I9<21r-T*
zpVxCG#(q~3*<&1Aan!}_Vb#hq5#58?buZL?4f$F*TpuyG`nNQ8YX~s?|J<=`x&XgW
zXN;04(`S*^j?VjuP1~;7%(1BAxMZ_kOs;E(*@PDzLP}gKSD39-S6m#?+4e(r(f9VI
z4xtrg@f(g+%Dt#qeW82z_v-!TV#$szOb$J6J8F(rD7n1g{;Ck<w8B;4Q|VRJhD+Nu
z>zMOc4fDzzgIfbF{)Xh07fQ$;ww}3A`Yl_&b84b!S>mT{mJ;9VU%5+u{89hsT3hPV
z#6Zzd#_KA}o5fZzDyz8(EdH3tBw+n-nsg?I#U+o{$foS;q9R)taO!a=P6(f<Wg#GH
z(W$m$Vxii^SqfZooRgRoCQ7UnnE60@@{TUaofFM}xcDFG)RvsAV>#I<a<a+KNs9kH
zC-qBAV){5yrL$^c#zd!;jgEyW3O@`Sw>9roFLkge{W_uL`1GD59W_Tz*fT%mKe9vm
zRJ!8Vfb_M23BeudkG~qv335m~-kLF8>5NcJ(vI5P<7EPcZO)q#S<?lSkF=?@NO36o
z_og|1TM^V~Ud!T9A#jK<=|ZZ{!_LJUW^~V-vG3pvJ<lq!js9W_yH>8~($nN>ubjbj
za3<4(ne#jaW?Kp@%oLbBW9H1xnag+1TCsChQo_ucKWA-PIdhBV>{$u3lL}_f-#L3R
z=d9hF!d)6O&j*)e2v6C2qI*9_+yV8PBjqieg*|MEHUB0~b-c*Gq_k9il0dzAuY+Si
zN=om;8NpBgw@LIQ)}2r_zN0P>TOj!$UGj-|-DCEc(k%j?H6|36&wqY%`kN;q=O1Z(
zyJ1|~*>AxtC6U<Y=o%L7#Kk2gqMkA}xpAV<4ng4)BI-Xnq!lOjL^Rs3ob3N)qO4Wt
zuAd4?1(RkPESl@NNbT3`C7zSGHcnQsnxw$F$mrI>z84D(e=W3-T0B#M%X-#K)n7Ab
zDoj+1TBK1mNg{Kxj+cPjE9t_ph1$;*+&P=PPBsULxA<q4x_ZpL!8zAy;?zIMQ<)g&
zx)k@MR88$VB);`PPc?Ic@Wu2KWvvCfG}9s-Sh@TYBHGNI*easfs=EAMl(a2vY%>*{
zuwrvv_6`qj_Z8WZ0yqDcMZcUTaM!gndBMyjFFW@AbeZkBa{8~?yDTR$Kb$yk6=&3f
zCCV#Vy>~6K{kd}StCehwt8}<lt@ILDziZW^Ta&j`tyb4uHFE*$u2(B}uUfr&)sll+
z0*g<r-jXG7RBQF-%%;hwx*gea4@lH}{jeZBa_x!ZOB<Bto|JGn_MAKJTTAQ_39Tn{
z*EROCRu~J3OUw=uGB=&~l+9AhIpb@jfXS!451Q_xVIe=Gv}R9~xa;2ke^uR!?;+`>
z{oi)<bG%mBq$E|pWd8i(I`vHx1gC4RZ}gRYY@PFTI_E;cqOYqAUQLuSVzc91Y^yoR
zu5yM#=O!1cMLOP#m?p3?6-+Xa=45u<%=ABTGgHH6uExz~t66Pk3z%4MjxyNn5WU%X
z_U0o}oJUt}VS2FH`}Y>#(_8$cx32cu9B8oBE?OXb^%fVY#lBTjq@1|d1aQAmDBT-S
z8n9BsYscE0=&9#51XHD!nJrxUcfr#9*-Bpv>^nXwwuw)B?zNHU`m%c>u~8PwZBrMh
zlq_$5rd)ff>^Q4nr(p8>Bq_$Iop})}@@FXZZ<}5lxtwL8>Zg{V+bacfm$pkP3d-GH
zHD7Ag(pM9Y-dgRud-bO5Ra<ti_LAl_n7viMd$)PEKxD)21HX44%HF(RTVVg|%}2TS
zERNc<+j`GA={*;I@3|zs*Mxh|zT0~?SI=H#z}EA!V$Fg%TmLJpJ)FKhwQKG9pIO_N
zuHEOrf6haoD~Ug0Wzgc&bCZ5<zj7n@!OL}-TO?G39sgd}$^1EwNvDpfX2&X{*v~WO
z|B&=!KJ6h`rk%93|Dt;6C-s8vE5@9i3yuZuzx73m+e>iY=k@ugT@NyDK4vZ8;=N_B
zw}6_Dz`?35zSaVVwGJ~0vhI{-J<@epzxz;tfuL2*-lz|Ut>&<r-Pz(QbHt_Rh^r2R
zYtA8ynyvcQN0=KA8@xW^S+h0v_QdW;y?n=Q54}qRW-3Kx?JJa?>il9(!ELcJYr#~X
zW5GM8Mw<wvEEE?>-ydN(Ev>axHF!sBG?(_}6>N^}yAv~Pwk~fk%SicAFZrKGq+GI{
zyRdxfBB@?3<z?Obx`PA~vR3cRTD2zXuzC04WwUnAoUq%vXUlHyqb!1)PBura{v2j`
zc*^$8Nw$rr=II;}{BR_2!s$Q(#()W@1p^N6*I}@gIlbr2X|pw_BNcY<{5k8+>Am-V
z?|pcB&f)1b$0Kyp*9fHg%q`n}_GI{4A;*39dU{+X&OPfsd#Ylt(~o^pje;y{S*nTt
z2B`vbpXK@p_q`Pme>5Yyd7DJci48y9)=x`Z|9Z#z2kGk{l<)YPHD89U{FB56KHkvH
zO36lPNgvw#|5b$f6*)G=2zW3#2plh%amn!A=Yub_7!H0kU$;m4<g7JIa`w2#obuN>
z<70D_`F|q|v*2l?yJClJ1TB08_WrpX$iNzIvt{Q=Md?FFZ!wl8)@%>ld8{Z#cxKKq
zZo#X0+`_?~$AWnk1J?@p?XAf-JfCjp*_&j+>Ji-Jy>6Z1a@|W3woi?_xnkG~m-no8
ztBQ*0JMB=UX*|Vyx=-rKtPF9!jrkH2r8Uz0lT_Stl$5Te2<?&DJm<`!-LqIv>^>aY
zb0})Jfacp%nz5%%?w(#RbB2|X(M*?Nf6T2Vx(tiXo<7RMx_!=RPn}h_PVe3O<IDr=
zwP#nZecIh~D*W8tHEYv*ZVN9w`<z=icaOlnv)lIg3LIZsb6S_-c<tHCCtKD`IC-^A
zQ0q&1hU|{5llMQ&=*|5vkP&+$`nLE9=8LTHFXl1pxWCps@V@fEOL657Zx3+S%wx9R
z!1yblxBKGXY5kU;@AY|I3Y(;NH1K5Brb})*rvqzla?~j;)@ASzVCBDen!j$VNZu;f
z-sTvaZK>VcLVTT~=L*R42`j$4-tRKE>ypIV2Z~o!%x~qYD;_@HR=t3Ww_nLdS;#SD
zmg3{Y8#`W=oG4i7HtF?zmfbP6b9esc%v|3*o#mi4XJ)&-^IBsYf!$KObz=qgcb}Tg
zE9@_GW^K(aQ{6Kwd07Q6oZg*tc=4ar>vzs@kG*~9`JJwy?GJqf_cpX>Fr6!^e(v<~
zPGR3wmcV1V6Ym813dG($oBNQvs_|t0|8DWeRqgNA2qYHvzU(@0>AF0_Gvk@>z23BY
zU!yL3DqH^~Jki!Qq&INayAv0_xjp{BXM@l}sVU3_tsxuQxGuJ`oG_A`CcJw>@F6j=
z?+x;Qf;Y^QNt$@qFh_WG*5QplM|A%@jr#pSQ0~f!xQCGi58Zn!+KV)DW(zd?J@V<D
zyE}2aY@WcKKW77E?<oAc@<{Hk&XOCOrfUBFYRFwFv_iE(Pic8;^n$-N(OK;>y2jVb
z-afXQcl~vX^nAyi;x>X_|4y`f&3~35)sm*2P;!@zO*4Lq0E?es;E5wbm6LYtJ~Acu
zrf$qlbsvHKJ%^S1&zzXE_e_sHKl9p0x95cHd|tHscFq5N0{Zn=jriZ4JaPMM@AK=Y
zjxj&v{u`ist>?~PhPnS7f|v}x#N7Jq-2c3(@9nX<0#)k-g89E*-C1+?g!os1-UiKt
z=9@24wp*khy!+VlWp1zJ9#`wkodR!8-pl*{EqCuXp*F4e9``@*J+S(o<s#<-=XV<4
z&)(m4=|0cyiw=f`)6E2LUdhd0bxGsT5og_-;j39M^-bC0@b+OefBdSev0YbZ^51Qi
zySm`Lv8#M(RyMyU|6Tw7&$skMA~L_$)?Gc(A^!U_chHTwHEM-fzROcwUmM%?KVLEL
zori$CD_isd9o@r^pKfumi@PphsLi)pjs5@i8Q#4$FVDwvn7GbVTKizv`u{SUPuQJG
zo5wnd_x7Gyv(L_HjyuS8QLD$FQ=}~V>!}^mRrB6f*1bJ-o!=u~>8-+-u19<fiU;N~
zH2wYa^K+?yqGKDctkIY60S-snMby1=GA=5)^+>CJoBSfs!9Hv;$F-siZwBekH5Qwm
zyx1V>ShhCe?2(hJ&Ylg+_1LQ9+9vGmmm*PSSyi%vVF}A_o!k?xisF7TYvPu6iE6Vm
zMy>Wz_3%;(S?Rj8>uHGEny76&w^ylhggZ#L7pZ=G;xT<eTzkpOb0x;%4hj2tzV6y6
zy*c~vOcrV5IX}Y>Bp>fO+E$t&vMi}di!<vaQ&hK@;jCjbu9<jUSrw=nxpC3aRbsoV
znEqGUu8A#Bai1`!_CK3k%#IJY_BL)`oU<p<wtJt~&Q0I@6LPEny{gQ6x^rPs+nrMW
z=X0VPe%yM>nJ4mo?sn%grWmuF^z-+UPfbX@#cu!edEEjBPN`fDhlfwR&M0mWWA@f!
z6-=3MaN!A!=57x4DI1))^rr|o1)X|Ou#QQvTXVzxn8c1FGki)oj<&~e^5_|P8F(F7
zA(xc7`K;8gi3??9cj?%-Wo5fJ_-@H~Yg(4Eko{!TtH@2$tMY>1+jQn#=GddP^a}41
zjosXt+uzpooRDwkk?K6zpR;?#!rMtgo($0ufhWb=HJmaOy1X=(PA!|}<FPbs-RZR(
z&1$13U(n?LTKIo&dyn%=Tl1MLmsT7%DL$__O(yrn-tRAVuMD1@b8BU9Sc8yP!=f&w
z6<RA6WOZDfv8Bm0EOMjL;^S>coK9ak>Zi+mJ+5B#!P<m&Q8UAR-<bT49DK9DJMs{l
zvCpF9si8b;n1qk!EN#z8J-p<M#o;T8kB_mg+pclaS0q;d=u{7zo-2?0ET6qt**@=P
zL}aUI*`LIvCLg8<U-o|eOVPjKn}L*uM>%W24f*POSNu->+8~-2@IXiJyL;50)ESA<
za>x1~pVXMvX+N1c#p6Ks>?AGCCcc)+RV>0H2Q)Tp3n<`P$q;kl**b$iH=P#hKPhnD
zzR2~PV%EXO*_j{aOk2S+Y5B5W|IfTX)V*SQ-I={bcl&B?8!4}syd;%eCcP?EMAzh^
z(qtW_lS*CN!vzyWH8w41SFk)UU7TM(=gzG~H!~8~EeJJ<R5h=ku=(=BJNH~x8k<Z0
zN)4&56MP_FZ<p6PS9Sk74yB9h#jHzop1;}j>hlZ9#}DLLi%O19EL<b9_y%WEt7<^~
ztC!1zT9^c`{#Qxbx$se}>4(*8lAfMg^=xOwJuSw9)Q(>ZJtp2;W3)xw`_STU-KA??
z?k;!}d2WH8PwBZ=+}n-X-4tC@7$0o+`D^+5qifvex6gHQTrG|GZMP^iXe?P~x;Bz^
z(ld1i4VOHtzZOCJA~r6o+TbMo*z&yArhwg5UzGpnEjsYPA;MActI8XtcQfWX<OFtA
z$E-8E$$4pa!{!Y$E?g*l`)m4DJ0_v&E`Evo5?a^3X%_y|u-slvIrMDHbLk}}8})R9
zxleb68|xkMsIm(dI9IaFc+TD&!#J5!7mjo+<XpPcS>_XW%z|e+Z4SP!H>#8!Jwm#6
zEI6YfTjZhnW0LaCs~$>qPcD85*}nB|OJ=OXO=)K#CH3eEHNW2Oj`B0h=B;bg^Zg$!
z@`}lQSyW7b@YWaZN{WZLG+Yv86*l&<KHlba<Z7B|pa6p|ml2zaf`ag-FYLCDo~@{7
zS!p#rhwrgc5%+1qC*p=F67$)L=6M!r2DVzo)_ps!`fkA_6}A73A;;e|3Yk6ebKm+~
zO+oV0tipZIg6I9}^s8du6Fl`*h|9`bYo2JXUg)sY{mSY`;Wu{L)(O3?{wH$H`I5Te
zXUEl%xh^X%am)%>QTW6`c`<*%t#HkomMOx969ZExSjx(#9ZB5xVY2Qf<H=D^^y`&A
z#^*Dc$^6%J^SO4j!_n%-K@O{vDoH1#-AX!HcJ;RBMP6Xh?NjN}aX6^*cgF&aElWBz
zPWTE(uReO$AeB2v=X2K1l}9=BovwD?IVmc=SLOeaQ}%V0+5Rk<?*CpSukMr-?EKxB
ze(K3D0ZqT9de@95#Kwmm)hlvLkNvqizWkZ1zLtiQBe#@fdzgs{pLS-GqQ`${VJ=<E
zLmOQ<N{?=oaB7mYZRzWpx#Dfe^8@W)uFTB&b<`mwsoQ$TqOJKAtqggI?QT{_JXs|t
zU*p)eBueFEF>9u9`2tPBc!?(pdn1GGs~q`jUvf`LOlp@1a}L=#^N7m!l)j?uVgbf0
zO2PsHuKGVhPnGhNHz%)P*Hsd1{r190SZvK~`)i5~{#*(9YFm1Dt#B52`5|2CuWd)@
z#?(1$7V;?{etXMgzsj8^mF*KUSMz(hyxKQwuZv-o;`4WIPlbax^E)L=xjxOibTiCW
z#PQ7TQ=cswj9+BBD%omze_ON3*2sfHm&Gb7=Fsy9x%kzJIW;+nKZGQgI9@x!$0*pb
z|G&l5Y{7|!0(<)%43+G2?|K<-%~<XI_O-+HE)9dcg0-DFjY8!=n`KoaTQ&TY+kAK?
zG(-igD$H52bmD=KbB`4{bya@se0kwfQ&G3m?TCr9=Q%m@aO8+M3aE2!<y6R5SlKSY
z(hywVU?B80LLzwa2TP3w;nR{8JVcHh;M={>T)AXR;zNm<mQ7P0CI3zB(4CRYZ#hvc
zZoTr|%+s6qPn)H8eZOa^K<cNBTLOQa6#RQuaBA&LYb`gu(%z(?dAwpvbTT$rPoB4R
zr^b!k<RiB@g5I=VQd+phK_en{L3fNp%9GOnKQ>Pajh9}*D!%*2jI!_7rpX`e%)hGH
z<J%R}q5M{%;s3R57kP3Fn<Wp_w1?j~=)$RZLSq)!cAu78CW|AES{3Xx?aO8f^3ysg
z5#`LUbv0QuFWdN(u({mi6i1P>qOKxVXXlijh~fScz-PUvp+zG=<{aM}b`E1kuc%#>
z)@8}-7jG3{z2?%G+<1=Vvg{_G=nI~T0`vW&IFqivzECy$1m~rB+kP+gS;n1{%<NgT
zHuAY)=8*{;jl1>r%)j?vwbj^hFoCNu{As?>k$%0xPTxD9SC(|GI{b!b-nyWrt3!KT
zEk2iQ{2Y-ZTF&)p-m;?Sq4#D^+mdr4<0w<ci^4ksXX94$mQCo6o8da4QJ(k2lI`1*
zUQJHD*>9|N?^aJkS;}FPkN@rJ^FH}ZXaCG*%(I_|JzS%D&hKa@p>@_f??lY{oT?Ya
z@~dg0Krh1<hwWVVd`0%IJwEgEwZA$&NimCWDBNB2swDl|qLq$EEzezkJV|GV%cBi{
z`s{s}SR;*8E(8>am_Od%*1^~Mll?EF$lC&imnT+5aR}8nn6?QR9DQNhcR<4Fl;VR|
zS~FJlvE;DotZx1Lp;J($RrdnZrCEIO3asvhBDbRFZ}(dCaN<F^>P5SXx)K(eWiM>9
z>XI~ip=89dCik||!BrdfXz3i0T5Q-PVEjw8O55mc*P2$TJ_{iRk&RA!R~t+{$f2=i
zZx<JzVi1ph4aZ~&k?I|+)fyI49VRZ_+5Z1(R%`sl#r#W5U0?OpZIEtO<hYyEzLa^@
zj)nSR44ut|mZ~=ow@tA4p1}8RR!7_i6%igQ4hgHO8<twD*>z`F_Fb@KFmVac;;0f}
zR@t$$dxrv3k?dp#rac#>HFt7ucrCGUcklh*+dq3OdARDpSC4hkOEwp-oaEZHyK}<s
z_!X0{WG(40w6M-#{Z!ce;h^&IU1nQXY~(F4{qbPV$pcfoGdrh!P~&|)_prApW1vxc
z#-8?wsWS{kS!as!d2C$QAg33#am@)a$6fQJX3I%-8%qD4x%Gsh)Eecj54!7m#CB)c
zUTR*`U}L0mxTEYd?^z|Kl<rM4UI=Pcb1*U<o^rtSQ5Ng}aE{H^3&bAn);X2Aq&r9Z
z-Gm*CGW>tLXNNB6RY~NSvVc`%!TLKddcP=leC=YISlxTOYm4XUm2C^8brab6o0^v2
z;#<<hq<g?2tH}P_YU$}BtST!NqE0bg<}%&cDf2XGw#DaJzkjvAm~kL(_tqI5;xdP<
z`%X=|n_<mjvNB3QbK)tj1rH32GMJ4fc%PH7eZO+1(F7(2$7N+7+QV+}9ME2LD#7D;
zgJarigY-G8r>$HpU9u>ndz-4Vd-iFwISYOCE-2**uhISCmTkR8g=c$Fw$nBTbAHdo
z3c-s{HknVlxVR?9hx@bjzSqj!!iQaMp5CLpVS^Nph+*ra;*M&Gx%Y#Q{J*%9Bj{3Z
zjF#3eLE$DxD}hT(y>mGHclB<$*z~pNq>lt^l_0A}%#^@}1`mZf{wBNLF_?y!Z18en
zGSuMber4IzG5hUfzUvxuY-)sV@9JO?p6elW&gT=K5$B8;lhaeXI0Cz@>luz+F$yrQ
znUQGXmU3mWY%+JTw$sBIi>~E#>t<-b?eMKVZQix3vxG_1maD65mHM&n)f<hDM+P1j
z6>iYYIIgYb^vaR@!AbSAi}#f8;*nvt{m8kuc22jfum@wt-pk1ySD7ze>2XTAc(EnN
zL%gKp!Y9F96DD%5Q9O~s%%{n`Jg_@VVn_S|c}-dAO(rvR9Gdnl-XHpc{h1J}nRf4g
z$Jgu;ObfqXJo(hZ*!RYQzzqKBGOR+UmijL-5x8a2m(=(7_5KM9f|tJPE1j?}uwY-*
zA6~b&D=sVT-u!Y^r^D&j0jK%@9N<$uxT7b4QSrFy<ZiCavuy^Anwz}0-)Q0$SQHkZ
z{UO2eZ-m+Z6Nip`Ii`MyV~K^1Oi#CL%S^omO*)JA>=SZ2edNskNgXo`*QD^8b2kcb
zChC;jo^bJtyQX!}sa1#id<2gcDC@XLGCo+-rqO)$iBrMP-p-P<PbY9(H=O$?^YrTp
zx=kAUZb$`%cg*-Rr6a1N<$Iy0MfPlGXCbajO`m)GC+yIR334&4T<RssZ1Z&Oi(RvS
zZaN%%XL|Pp;s0S7#-+6#Hyt?lSK3*LNC$q3kk=H+2og7%84}-Oo$%*qd*aQ+l!?i2
zI)gmiv}Hr52l}W<txjNcTD;;q=VR-|1)SM;b}wAI;?U{I0->9N1l=D<xbOYdwZg)^
zG$!zQ;&Hahi$@R4lU(b*aKb!+w^Ls>axY$ZLFx9P#EWav9iy|au1SvEdtkxh8{M4M
z*$3N1&iqce<gjaM;oEf*P6t<Bp8ev4`=e^+>r9v4NboU6Ex7B@6eQIeSRnp*#{!=h
z3w#6{ei?7RCnZt!Lv#9tn4K5*UEna;^>E>(7q{R2I{D9Wcd4d?!CI-$5gVMjCi89S
z+mdoRP|z{x!4;R6OgnDGKHeGmU$~m%L95T+u1=u`O}san+BfVzusHg5a?c@|c@maq
zUWy9Dlya!AQjYNrV~d=y?#1f&s@G1*98&yZw)>IK`7dX-C`>bM>aNq7ptUwEb)nO$
ziL*BHE;+wxV!<!5TY}4F4X=OgG_^SEtY@gw^Wf|wZws-RJ$D=oOcbqs8yW&jObn+^
zdUE7I9mCdu7LGE|?dc|rC&GO{h`*6$<+?TF@&?cH!bxoFRAi0#Ca|7fEh)S$`(enF
zqb(X2c6zThE{RO(xoH;QroA?9_F4WNtl^y!wsM>ADX+V^>jo3|-i1GZMDDx2`;VyE
zouf(fJ0>107JcF=GUH&^p;^oIlP(&SZIoJV^MCV$sECW|n-wmWe_5{V$8$ltPppzd
zX|9e!>ZAK7f*#JeJ)?sE*F=uVJ65!7q;S5G*fJ@kN1#7CK<r+QX?VibcxDC9$y=Hn
z_V4=XSslrG&7g1Dqa#fVS}!b^ZJfO2O2y8(F4zAQg#1(7U|`u1*{u0z<CR;sO7mAp
zYN&2x4&0Qg)4c3Q>Q)2h7~f0AMcg*C0*;y_rGJiXZxGxwN8(vVYyOjt1JMV!cil@|
znAWp_JNx6a9XIaH;5_u<tYngG*eBbt&%CoFS1;mPxiM9@S@p^B4A$G<Yn?W^`gYEU
z&#`q&^*Lj8YDMS6?r_(c1v7nrC$>Li;$yMfVPp^^5yqn-xa<Gh8_wrN8KxMUe)r&=
z6y0}k*U^)q{n0AYE(^~ynq@xK@|L`I`QpQMEf?m*t6Vs}v)%YfVpVPAC&na&Ws!Xw
z4!%~Eo9+9^*6-26G~W*qX}h?laSCu7b1F_{h<eVtJbbOF83&inw76xu7YkV27cDqm
z#M1pD(7kw0)C<phrEj{PWr|kvbgd1QTa%Go=`(dhYqZQ;?%ZB6vm&V`o=x>~eY-{Z
zP6?zqYK4E@$u3%W+t_FApAEAEA5T)rymK|8^}ONqnA?+1*)4I`J#AeaoAqeYqlVOW
z#@x!2^S2(k5`FA$u9?7&2?7b*#HI<}OAF&sKXh;Q*I4O4smpe}y_>t}_PV$KXJ38$
z`sv&D3A2n0uBoqP-#?EtyDxO(1)rT++n!&0mKd3I|Cn~cJ0`sydGBH`u3UTOo!ax$
zGKb`AL|GgkuYTcEp|-t(P3+>6<KL!Ut63NJ^&t1|9j`dfirFoC^ygVm^Rte8mYH3D
zq(2xmK02-QJBmYcmsFRBP~Wxl`fFzee7X@+v$AtW;+-vtzl{oH)Go{rJI1n=HN3z=
z{@hE2bCJI-v*d0+UbQgWa@$p*jtQC%xY>m?y%tWnz0qyevBK5w9tr7w+`yJBzn^Pm
zZii9W#hl+trg_;XPRqqEluNsI(bDZvTgKk&&SK^n+nXmHy3fG<jq{Asr)&$hwdGGF
zHYlmHf1m%~sDgRNif|vHzOZ#3U5e8;CNJZik{b1`b)wM9K*P6xpMLH+bTzRpJfUvp
z1F?o*veO=(JHB9F=x^7Li~hZI>yO^<@$AnEv!!A8+B17Qgmo8iDEYN(H+jVUPV?Qf
zYjOUytHJi~3;Mbe_nryU4?FXuXXeL)lF1u6D$`R%uYRcuo1d}f%I@dKvpBTwr0<^A
z@yqV&wplAu&c9!9_5J_1yFYB0l>F<~RS}K|m%H~DOmel;W!JuWY2gi%ox6D3Uox-V
z-OgAzV{YVDg~H!+OBR=vY_$3)H@$puL(;ybd^0YH<R%_mwd_^Y#U%Z>Jq_V|*7}KV
zxbVtA>v+?8_Z6yPs}nYgg<kw0GPg<c=FH`}kJlzVmU<hthV{0~_lmkHVh?779b8bf
zw>e&UQ}l{5BaP~gV*=d=;ubiI<@`M$+i_>Nd+~<}2TnZCXf^&aalMW3*9Zx#(<<lt
zmfV>8^v_MTOUI11?}-hppPGF#)G0s7gI_H8Y37{Px4S0g-pcC`dhm;B=G(h+)%QZv
z9M@N^DqnL`jhSykQ^qs#yOEa;Em^!|>D8CH-M`+QdA;u)$HQvYiN{$vON-TN!rAj9
z-p2`5&y;GO`pGT7?vj9ZZ`ehj#f@3BY>tKXNPNp?y{@u0zoyQ&uKBF}p^nq9-)ow>
z#CxlJonRWLlfbU`Qva%}?#zPt>^_5sM`hH?|MRC$oG$yZfLG(nv~s(;lIXXEi_*S`
z)R#ZGAGrGaMvjdoKLj|~lfGT&i3)mAuJcZGsi=0t`PBw|aqaw{X5W0L8DY52!tkAM
z(OL)h$VBzi&Hpvd2d4Gqw^-e7(phoP;^TF*ZMT>?7?~nCbuu<gbqHb;?-M%V)^I3{
zmwVf$?Hd*x>Sq0&Y4RdJ*?oedbCF0y&;ieS`OsgRuV0F{ow4uL%~h$ZL)V6_^>~}J
zctQ`u0tbCQ(F02#1U9B}>4lyOahTo|xLB$u>R8X!<$k8qJa2_4gf4MA+7z3W^ziz`
zfXl0LvnRau*unH<*Hx=c-km#xy2RFeT`lX_*vQUxBXfp`_WyZZCvyaMxhf=|=x_ED
zv$~!azBI1C=;WQBhSN`YvgSNAYW043d1c`2G+WmLFRyQO|E%kwraAr2^5W_5WNwvT
ze{kR=i~6~hJVik#YRy!yE_+bic#+lT{R_9n+FNh3#%%B1lRGnrja_+7^u7nq%`bV=
zWOnY$)n(lBit9z_g?$I&Wc<G!-v3|Oi-C(_fj85SN9P;a|2;b2wExfNZd-v{Mhk7X
z?MPZEy*A}WTg6g^>mr2{GDXB{RolK?Ezg*^RiHCw<`1PF^<x&wy|TJnk5;nXJj507
zn$cB2dI^hXTk)xhL9?Ugs6+~=dzxxanB=a>sXBS5WUDm$-igzqr^WcWNoD?*-m^_9
z-O+UB?U^ypj<K?5ud7sZ**<mQgd)zptutd~3nOGpYvWAr7B|E_VT&~la+%>^+13(}
zRK~Tzvs&DCORyvF)#AV!;j2gGYk#w9Oz_g%DH3dJd^W|~+Wc-vchcGk4}+zrDM|*-
zOo$C(3snu6dMzZ#@$2f~y|=0a#Z`OK0@6>kMyyzWqD^Gl25l>AoegS}G{m>*^)0=+
zamS_^Z?~Q8+j^9ZYwP00wjWikyb{*#@_sozUMeIq;OnWXn_>GTuZD{M`ezn4v)@)N
zI(zflX;L}M4sJ6zpm_IKWajF%n(76j+~Ri%k7zFcSafBoU`BAR$yM3%8@f|uPu}hg
ziCy@A>()cdtC?)R$W^u7GP|_2cBA!!hQB`^Z}yb^nWWI<s(nJCN$^*)cdMx1i!SMa
z{V`na@^Nh8-DcT$1LL&r{Yva|nCiGm_D9~^tvA!}@0@y5<Izd8%^A(@+cu>5G(K5s
zseEPe8K+;eOZ-zbEO*=ri8ER1HfOD7z@CJxn?J;4_NcAj?69-a^4U!8y4egZmv?UK
z^4_Uib&GZ4n&mI0=FVFlbv&+Q-lFxZk7~F~<XVt<_N`aBK;VwWDwi7cSBhL-VRSJ?
z*w0FnEj#e6y58GhU8Qu<Fs7LaSHojBtJ=P}rR6G9<X$U!?FQ40g_|5=V#>SQmi3=s
zefy!3bp7q}UWx4?|I-%s*2k_3woZ;ZF=1)<4JQWHIaeI+Or3FfOQW#!$}=ee$}>5G
zjv1wV<YBXXENRfYtdJ$x_n=7I2E*J;AF(ylJGP~W2)uii_;1$?Hf!fODN`R>$OoT0
zE~d2N?$y*%CPm4U0*h8eDz4Ue-gwiA%~)>hg*63RC5}2vEis9AIg#mOegCJ?6E^<B
zjXa86UcpwM_aC~+6Rojn*7pNnr@eO3buQo4s_miPU~#XjwD6;x<Yx_Y+fBK8F*gpl
z>ZSJYRpc^skLsDovVFqE4KH0gyyKrAV^a6<yW|xnbb>+oy7bA3YLk<=ggM1IH>fAB
z^UgUWGe_ozWUInDKY@kvf9GwU@_*G=&44<tlO^k#k4PPT7qIuwQa6dE69RAWDIL3V
zqBvsV${%yQ72YZwRV!MgHFwV$%k{rFYyYrsU(>1`eop9d>YhVp8Ku0D$98#K|IswZ
zddI=`3_<=YUV#qMoGz&!L~hz#-fs6V(|h*aLyvEJU3XtwqFDETjq=)e>=(BFSwAOU
zu~}qQ@w%`dO72T4u7B!tnW^jQ)5N6BvG?6pk%fQKn&&)>Su3+VsPW9QrCgKG9!juU
zHbv@)(VRj}4&7zJ5><tMws&Xv7i$<z_$%P{BAEN?H8*jkRqo2KXU;s6b<O458I7Qy
zHAy>N!e?LnFu9+>#gFsJiNmw(QkSq69+f@f+*Ty@Uu8|I@)iF4i)K$s+-*EPUpr0G
zYKY#kxrcG?ZP|z4!mZ<c7qrcd4rdKuI<1&8@2rl3WQ5?7lmFIwxCEs<l+!$QbZJPT
zY{*M@=hqg7ydNs(?VY1}dy1urrjggNo+~e2x$laMOPe0Gu{k$)_Kj^{CY5iJKItmX
zv~oqzR7IwQXRaJhZHo+y{C$7+Nj+zpzU;};?25oQrTnjLPYZ@Ils8DJ{Oy?Xpyy2T
zb;gN~d5ZIP9@uPseyw^i*9!0O0~ei&_8zTf-hI3_cF6)J3+~zlS49~AU(dcCwW5B#
zx}Keys($JWrl(xW4UT3l>TeEZXc`$!Iq6x@D$>)lw_~Z~;sslH{x5i_@Fha(vOwL0
zL_Id&V%IyfuAX^jzVoDle`>@W*~b$som%-OJoHwc8oONd&z3vkRnk5umAnoG<YozX
zTP{wU#W77eYQ>#Hr^2-JQ#Z0k=1U(b-~6?0Yu3}*+yb$Q?VlUMU2CQ)$IX7;7Cq&x
zW<d_u)oZ_FzYCRZkNa!9;Jr}TN!R5wx9$kBZrCKOq{>#%ek4cb@azk%0vESVoAyN2
zxc1S>+DS2U{+VbT{xU<|_q4_m8`bG$i=)h!PAXn=#G^PVHe&giOU|`Pr;AH8Cs;8D
zwr|yuESHN?O88V3*sk?Xtt)zSt3|rX`jx&BTUMIAUD(?D_q0Nki`xnV_M>wPw){8y
zy(sMMA1Rkq?@Z(Hqd{%fX9_hR=p5bt(&)j~!p{eq&WiutHQjDU!{)m0-^};@nwxK3
zbU7^WCQp#rgg31QXT|DnpJcjw^}vaeEswNqAMR~?q(5bm$Wg&-9X|2DE={PBY5n?$
zM>OJ$l<rZ3;}WkkkJX&on9cWf+BVsffv=UWiKMwHRF!HKihoL3C~WJtIVk+xC(aY^
zb}iMoxOdjKEjruGw5}B$`xWr{kTS!nt@*FnZ00*~X?VZSJI=c;;`6*KJrlm(o4Jg;
zvh})tF?-8`K!rE=ewFT<ecx8RZ?^yYe21Mj*<1hA)-Tnb@?d_Ga<fd;A?rlu{y)zq
zE1cy#5~nNt?7#H1pGRA_UFQD$MQoYEvWx9TmnI&w$y#|~S!i-zaZs1a+*!{48_!KE
z-RpMuZDVZ6+g+}m2bc7}&sKk$v@Km@-)@hWv%~$i39+93eA}nsj{4k<y=z6Tz55pT
za<^=>2y0e)n-63DN7w40T{$IJ!sVZKrByU<uX%m<QThJvcw><xJXgxsZ|BGo%ZTj%
z-|)ZQongk3v;5m7t^~!*P3pXu^ELW4%dM#%j&H6U)_>EI<Z>r5;$BvdOU50a@Q$-=
zMRVBZ-stqab-XpE`(%jYgn+d>q-U;~aO!Q_><2ge&Ib6ujM(%;qWD6)_1gr4kCBoA
zvv2&7tF>X7z!EE{7ud@ZZ^!wcVG*Z;HA7HEMc_+|{*DD6R<^DJP5%Cs97TKLD_xG?
zTFZasf_QLByLQ0|*Bti~Z$c|q$d~U(>{xMM-0*1CLRZNhhoq-O&zgJg#*8M;ACd<w
z7ep`N*eCF)_^5x>gxUKzlbHM%{O5REUpcfhgd@o#EcLC+)68iP&u}~nnKUJ0($oW-
z`90n%Dy1}9@3C~;6Y*il?1(#QI{i_B!>Ns(s}=@n?3(r9%t>v_Q+ofR?|k4q8N!~&
zC=%QmAuD&=Nh5M2r?7sOi=pdrBi$oAr?lDJ4Pt(GyWoq6qySe;pu6^+BQhti-ih>>
z+v#W5n7C$+sMM=_c2gOh<~$5r!BbZmJ6rYtO|iSv=T*up?P}C4P!3Kxy3e;;>D2lA
zKf|}OBsuVf$ZwT#{wBlow)^Ohi7!fy9rRt|`PSR@-R*<D;oUB$*X5`v&zve8)S`Rk
za3ageM4eeDR|?Dsi0U|b%JZ+=@!WX~0gRf{Vtm#{<UC1nS79wb8F6Z<jNeu^-*;1v
z9+}hkb%xWE3-4}RDtU58=AwHUPoM<rQOlnx?Ka0lY_Dz-jlEF#^qR_DrvSd^A#+Q5
zLn2%s#$<T(m9)=#8Fw?`gt^#--*e}#jr4GI2n)P<@O*A}Oenkft-FSkc;aH+uDrP^
zk#tn{+f^l_q=Q#EbUOJ;biKvh62)VerKY8&I*H%lky$JHU+gXGz1uBQXDGPTcqAR(
z$>krRlzKJ5`%=4FTT6Cq=d(zm0}YD~c=)}TD3`w~NsB2+>l)X?j-y#?<yNho#w^%Z
zl*E$sG4AD)dD1+Oo$k!F7FCYakbJ|jurnrP%H4Ajk}du#j*8WAng_|}UX9$zd89V;
zMA?e#R$>9cH4m-B+{-37GTuBC{^U-vNPtY`#j+0fN|At@2LekLga*Hn44<l=B)CZ9
z?QQ-p!Ox;f3}LNNHfpsCI+)^o&Cap%`aSELtJ+cL(RNL)vuRmpoti>l>dqG@%!87=
zwslQX;PJ~#IKMUJbI37I$4eY--Db6&oNe<mTjU-$gb2!d_bi>m&G&z4-i>G6lBv9Q
zFL`y_wQu^InsZfVE}!VN?sgv6K!t!R14*Zfl`WiZ7bdQZ_L#ZeFxv6S8INtv6QWvv
z?s+P<a{0U1BXdjd8%naw7iEc`#1L@es-YB1eB0Hyk56YTdak=CQo1+s%7y#<o$hxs
zCbx!3_V{GUH3p|wcr43X=wI1URFNf8mzCC~l{PIa?bfWQWv?En#wFkDy36BgyFktN
z%Hb!WVGnv{{V!3<@e<lpacydW+iVF|_83pKLZKIHm0A>6&sd_gnae>dY1TsnA&pZ>
zt5$Yy-F7Qb<A~PGxN0BG)vq#7a&+nb?KC~tQL45uswCUk>(rsC9lB3uPEhr-S@d7v
zyQFK`mF$ye9$U+C{haD^PuEpjV8!{`+m1V$zP_Cr)WH)haiv06$K_tv(q$13IpmHW
zyKvZR$>g-;Ge>0tO5ga`zBzMs!s*mCr#+Xr{0je9d73N8JI{l$+pj}kLg>iWu8CYc
zNA^7Po^>bp%oE>9Nj-m7<kVcr){#}oU9eV$)q`)Tx3ACSzBMP(79YPhDd2C!o^7qC
zR`^u1To74w;<Q0vc+=DHRlcPfEv_LPOM6fH*09E1Tj_LR>D`@2OBO6=*mFqwilW@1
z%hE>z#kX9nZuF>FqkLa>Nw}-`YN@cUN1hqSrbP&{^H%AsY<sn`EbGZEt*3oiB4vl3
zrmbsF$a@<0KdocatK!0gENjG9EqJED$(bD$w&`Jt`;ldA_tetYiXBmDT|QB6v(oFg
zn-;w@>v^fjs<Df6t6ciC6+tSx&!3<0xYH48aEbkl!LzeBdW_T_Z;YKX;mOKfmwc9J
zu*_~d5?CnTmL*%x)B59Q>du$i2Ny-!9ugJ55x@1s(SSMqf3`-3UYP&lO;_8b7aehD
zLnW4!xn?k#z4?^c(yZIU@aw(h&W_7k3z(vp@E)4OEZy#(`-Zt&hedzGNjG`6*(<J2
z(O~D7TRQb%oJ!ZKEzYicS0ozraa>%-akInUNucv>M$gnJp5r1D)Lx5u929Yo;Nw2P
zrg89z$XcG=36oaIXfW|?GyR`yrnYp-Iaj{A<*!y8{WPy#RBMUh(-R3Bm?m4ggsV+I
zT+*`3Ds;2qa)+FJ#~p8O>bAtxx-*u}Ek7u{yOVG6EWfe^Yp(b7hy2h_%Mn!kH8(@c
zRb-jRvY@%eTuKT_t6b$yYIUxe=%BYE?A_zAbt}78t&nSP57=}f@7<F$Belg^t5&ft
zv~yEg#^oEr(ZAu)BF$sjO~*DQ3HeQmQuoSMnR`vcOlZ0U>&~8|>WbQ0kM#7-U*|u1
zrS$Mvu9x3dx#d}js|{1LcWem?KRPM4XLS$D&8;6Vy%&t0`AjoZ^s&&rp0_Hjb$N5O
z7lc|Izsw&v|FF^(g-aaEKE3F;8~?-P%#9WQXG%W^UG_<Ip-cS3S2EK??+L0K_{$y-
zY+9q&w&u9dm(!(e4)$-4s+bfQd^xSi{_Nb*LXj&Smp(Fmy4^hWw8=`YR>KKSQ;toz
zrQ^M};MbcqK2mQ3YpzV#)p~oW0`KZay#5FHPJa>cILPv&U}dN4rF{t;7uB9|whNkn
zIqkT?(O=iZSv5rHURZ=mq^nDGj_QTXjUlUfrF|ClZDE#az8e@ZU$f#go4!OuzhJFE
zRuszyXGXDKdRe_i?l!t=;#()SE$K)P(h~i=xi~SWK;wAfi>>~;n^dxm5}b_^?28h_
zw<XM8Rr1p-{Eh1>m!vh9TozfnKQR@USk~e-`^2-8>!$Mj*G_-DiNT|9#+#yd-!^TY
zW~ShIdwp7;j@0ruYUw)S@3T8M+I5GnTY7H6jcs@D&WW0{de66Q8+%*c#klOw+xV5~
zda_22yGwMa(42J~EsLd={15GJ2u%5RK_L1*qigbpqf;F36)D#`hTCRq-FG{?eWB=Y
ztE>vq0!NJ-^?|Jnd{?U1m%rpOVEVP$Ma`r=YigbHQs&R5MQ==5PtU1){+(5y+bR3|
z(L3er-pfunO=q*;5$JP3ET?2uK++Lb@%FVPoEw*Tb+<j`O-S9I!gIdh!aO#<*9Cm{
zw{&SY`fPsWwef*lnPb4GFPS?Y_($hVc6u_|Xx)@c*UDYEN*<L>Ejtq6@jvnN&zMC9
zl48P%A#>dYw>c%qBs$6@CX}Q{w-mY`(Q>-d>=E>$WgF{O(<w=fywQ_;)-{O!Xt>C~
zsN-Jg!k`zA_iZhm`MYpt@x`Al$LGef%E_&(TWI<zZPlC=L2BPjmA&;6g|{XAFIAQ{
zODN8hwD^^nou^j(F8xu5pUkY6zgVUSdj_9b;W$NXa!BX<f2%gE+N34CTGxBllO>BX
zz8k%}WwPdd-gAr2C8<)g&m?`=Bsc5MBBSrSa{M)I|3(M@yT#X<J$I9dGMC~krz0V<
z!J9uTS$v%Du6JXmdcwbrC!zwj+;#M|p33^=C)<44)^8{0OW%mM-r2~uR{BcEdEFU@
z4zK!O9lqsYxw~)uvCppp-q<l;iEI(vKY7h$HIoDKB@N<R8kncD2yb;_|I?^nbBMd<
zfcO;``(N!YTaJi}Jd3hfcO@>MfRUwc@$u;?t_xEp7KCU`7WMB_&DhJKbM41Q744im
zAJ`rktnp{;zW&FzJ!oyhy&mmcf8&+UGVj^@e>nCa;n2NFpR<m?G*HN5UJ>SXRlsd)
zcpmSZd&}o9pXiWrbB6G%C`Q+H5x$xmX3OPg)^tfP$!hs}bh6}~TzSzc7W%Eydk$5`
z`bN*+d)qk6mT#K=H%W)Lcbu+Yc*d|J-ua!9e$^6jyT#>#i{$q`;{Nu<#X@sRUb6X}
zi5r|AsLXzy#qxU6|D*5YJoDGw3FOgM5Y)ZcXZCXcuCCiA-OVNb|K80CSp912zo$Fo
za^j<M^mbp3W;c3hCc3&c?|0e8xhtJz;)6f-xy;;KqA)WezSgnC*CXli?$!BeD}GO%
z#@J@U8b66A`h{U_VWRZY{JX+kk_Tr0?e*Omzx~_7-M8oewLKqoZG*X7woA?H+xv`L
z_unq}7yR+p|HHxe5oaFSJ5112UcfWGDOEtoWJSRP$7XK+qAZPxiB2u4DoSTMHu`Ax
zaahNyXdF1>!KfB}(`)CY@V#M6;<i~;7M=E)ZI<mPxZv}%v$Jd)i)Lz0IW4ltjaB5Y
zW&-#5WzkD~CK@@OY*`t)F=*+n&HufF7-Ls(u%1?36S&#+qT1a<t}i?nd2%08mHM72
z*~s?(maa2bqV(jN^j%ywmw7K7QfC!byAi-3*`$)g$8xJnboS&mew$Y6Fl<geHIFr7
z+s()W&o3=x{lU6x)>q%^b>-8_f?uunT+e)RiogQtM9xMI-f3CNo2RDST6afbd92&^
zh9gWiyLLSeJh1kW@$D&_9*g~QFyF>Cz3}U?s}9DT{#EY#_AU8*n8SCc;bps`?BCz3
zy|%55|9^t7l=o3f55vg`9aEUTdtFfO6no=wkp1T;4JQuSITsFb8BV$2$R?X}VKM({
z4$dQj);bf92>becJj~R-`RLrRTN{tcO+BHow3pfZf8r}Ozp%#-)a4tax-|IiC{60t
z?GZ^}-a0ME`{Zo38A_)#m#(?AN^|ujC#E$j60cO`CTDf)D(`%tu*T^8lZLfMHy^d=
zGO~!ga<JU{t4J;Al{de}KNbrf``@=-ygxrVMM}i}QG~XLzfHB2tI3s|qBl#TvIG~*
zogKCEr07AOS50~?cQQ2eX7Xq~p7!BRbmxNT3EIj#t@VQ5PiuSU(ARUAC+q37D=fQD
z@Xqu~+`8ZiSMlLqa~&<yD*^Vh84JJM6566JoOAl}zGrEVjP1AEv0hl1r~J#zVdo)U
zgP2AYr||V9mO<C=lv{oZQ>r}Ld;MPJ`@I?}O-@cHlzYGAuDCCd{@-i+(WyC}5^6J8
z)?}_-`f5q0+Pt);8yB|sIlb1|ToUzZ<%(rpT&p!Vo$_j&xncw7;|-@S-FUkB)sY+8
zJ^PPt*DL!e#9?stRKVE_2L-gxKF|4jW}E#jy~8=Co{`mCOqbXDg*05xD_Xj2N|5#!
zg(DqH*NR%7+P&5PA;;Og^=2`Qvu}h|D_1SB=9rXc`2ClRzrmA7-!7kbT$;KzKl}ab
z=;%KJpS~z=Gtdes?3D1oaDe6KECH93p=)BCd6oSq7Kzwyxl?@3K76CKpmo;KFA{He
z8d@6__1sF7t&ZwmyEV=2(rVSh`cF$#s~24=*L2<&<gL0*M^5wF5glV6<Kr=xE4PXK
z7w>X^`A{U`qV?jq-1oEpd@9bEy^UYy<<%+2N|rhKzg}7M$zLjUS>T71!?ka6cTE%t
z(u!TXVscL8d9!H$B=)~kUYV?0HM=zI>grU+<vjCN^=*DJZH^V^2W{>z!JgZWNgR8T
znO*G6AFylF?7JlkSz=!{aXNYEo@^0_{+{{JC**`|US)UN*HsI*OB6ZpHE4-t37qaw
zw8MGt71a)bgAccNX-w)4c%rQMszcJkWJB+l(>y^w{QDA`HiTVSA?nv8Az8<CY)OY<
zwbtiJL6e$9mKOM^2QuChc`!qD?UbCpkcBg%4^2AOvo$9<)3JLpgQanS?Gi)wo!(Uv
zrHl97P^+o^^*=aqU%GJ4$F|Isp7QBt$-<{%*i4NWd(+L5XPsHo7I<^=$?R7L?Wb$J
z%<lYnGm%e)A$pEVgwdSnGOlFyS3A;eo0O&{aa@x7+wwuJX)}M-g~v<o8nn0YUi9Uh
zvH76Uzc=};j&o)+m7HaK%j#Ie$>(m9B(SgGh0}EHKugUOzmlSkxza`IFCIwls#w%f
zawyYP`p&~H4pENEtG_SJ_FFVrH^*UtRghe8kI9AGu07>u3nl#P%$KfYIIk7G$m`>j
z1q_-e1biA?F3-3$Wtr*9PVO5mmmAhR7qzimIYrm>ibIXqW!s;+)sJVVNs4~mH~Y+c
z6+377g>_PznkQ1Gsh#XA{r^bG+xs3<<{GUt%j%X2%{!YG`P0xjeBRWn2k)}1PH(y9
zyCZ;!;ci;wJf($^-ZvaXlvrB7SbFkoSM^-*A#GXHm4~sp5&E;IYRB8>gvWi{8ouY1
z!3J?ob$6Ss`sz(f8oNyo?-!GflvCZdG2%|Qmsg^)f#KRo#)6Mu>wGhwxppFRa0Zv-
zLyrK)nAVgs8A;jIiyFF5U0Yb8!gV@sTC{0V`j#?P@jc7h1zof*PO>|DulP&JiOEme
zGM?Xl;>*rz`r=;7G}o!BGtwd_EVO-|w;+mtqsnKOF9OB8t|y)I?C^~HAnCj2(N9&A
z`I5drPMNR!ow=^IyKVmAtt*#YjK1^8;?jQ$Mr*zziBmE!E?8Qg-MP7NZuWDVr;>^1
zO4;Y7X}hMs74}#-ql|0S*Puy7)2*+)XgRV`@#9ntsXeO`O1!yFWQ4w3vFVB6)5ReH
zmwV?wk7xYI;cw&j(laLN-q&*})<<nTl_L+E%;vIO(ejE@bILQjd0v)v_Q%fs^_pq_
z`ddkGcVkM`ieK9o@VFWY94y${p*H2S&=SkPmL01f<h1#<ZVy}kXiBl)Q62WW)V1m+
z+4EPL>MGtY+EA%m!aLc+Me!MHhx3^Qg3?7Up4V3^`Ra(eb8HgyRbJA?n4BYN<tyD;
zk#%|I<s8Q&Ct5?+-%4EBH_J$S+OfF>iHmbvltY#%F{u9cSlYMYYFAFDdNiv?!eaHz
znQ}i@L~q&RbJ)1pOmU&C_|s#oiWRRU6RMhG7FaG<_lVO=@nd>-YzkZEA{+T7=fqF>
zExK)#lW?V2IdgK*CbfGv<3bj=eL9$2wbC?ts!FYUP*;Fydy8@A=cFgk=0rIf@5!rR
zJv909M62(UW`1s6&!OM<tnR(_e&#n#pEpD<$nKNbar?_MYyW#Y^X)9=>dv2FvAt8D
zeSh#F)AkP?8Zx^I_hnXH^_H8%P^B_kpoA&m>GpR?#gm1m@3^-5=%uNfS^~wl|2V4G
z7ZhzZ%jQ|_79Y2lRiEdG+rC~{vXpi4{HdF_?|ol;Q~!mq@tTds8P0!8{@?p7_dP@J
zr$&6tE0vo5l!&X`-#b#b`f0D}jYvy6<eu)`)Z+5qb&;y1d5NOs?nnLFX{I`?Np-fn
zwS1!fuDP~(+tziz71wk5+Y9aL`ntW@<jZURSmV9t?7p>%<p?Y8&MXczyWDp5dXmCz
z&IkP|2`WqydqU>^Vbqo1V#T%g-@Mq{eA9oZ9IacizA^Ur8-DA>cb;>07SDcTcj%4o
z#rL{r^TIq9zTlZCb7$Ma4{DQk&rWG~NNkNi6`ucl4Zp@Od6ECQi{!La>sc0fztb+@
z+o0Hxk?W*-|AbGcP?F+<Wy%YbHCtAiA5pXETg)iz)bvO7k=MeoFHU)nxM%qtdbDWe
zm;Y;y-v8vCYWJ?PDChVh^#h+$4@jlY;W%@}!iin~$pNl4g=dbk9n9p&KljXTIa8)G
zXK%st70YkVQ}@YvuF{pqb^67^oJB9@^m%urb4RXuwI^Wlq9yA0QkI-+(JA0zyMDyC
zLd9>@@#E``zgu?lb|23S9Zl0CK0NO~+^Jf0av7&_MC0-c>uRfBolI+D_-e49%kMGA
zB1TEgT&YK)JS&<nE#~cbabnuyPeO-3JvhUwa42NLVFiPJvpGkEA2WT9IVy7XvG$Es
z+ULB@13DC#d>N7o#d4Ke6HbU#FWM{j=rLE5cyhnuB#n!5yx%5dNQkce_NukfZf&Pn
zv!wbWrq=f=E$c=8uhN*|!qQda|G+NA%xV#@S#P6U((I0=s|m|xl64m(t<|tPrpdy<
zJ-N@c>g}Q}1>06QF}CS4z7ga;zbN%b;msoN{xcoh&-m+ce44<=X>n_Px{PLq#HK7Z
zXEhb)JJ0e>eb{S_JTosGH#v3O#As5@lGo`n$r&GxtLZJ3>*CV?wD#<V)Yb#tXIq>e
zWN@DEIvo11b-K-ky&u#bMJ+tXvMyXr)Arsn5uXns-+EN09NxP}$FS}tOY{eZxu2G?
zotX6_Tg#iPY6GXThhAX<-wwV<8}@9{ZZB|>OV<`|(OLgVM{s#@m%wwoloAb{=hrTA
zc%~e$Sf`i0$miWA&$L6bo0s4CzhT1-zob`jJ+C-EbMqSWv^%|a;#zVnrE;0jizV%E
zRxLk%hVR5VKKnRN*4G!zSorz%J>T&>b<1lKvAbmG#cAxa`TaKm4eQH_IZdH|loV%W
zsYM(W^E3RsYO!eA;(Di6^OFmz{RG1V7KJ@|qi|+bqtc5;fk(Vft=w<Uth{rm$!A5&
zy)M(RNf}8R))&3^=WrBG_-NfJXmd39+c8(=EM}jU?kifa6`Yzj*}dIZb3XpjWa+2E
zu1_w<zH$8-w|u_Y8rj8*l9JxYMm|o-QQB6wwxYq8X~%gvRRvBF2KmVe-m{Xf9rM4b
zRibo0U%~jwdIslNTi)7zIF?b6oVVvm)w`4Y|6f0!^yhv0pG9|1aWH&7u43A$`9&$L
zp{OQ<xnshH2o`1=fn;lq2?}Ncic1z-Tw>c~=fbjcfy%uEbDzeuJnPg?Hrwnw#N%Xg
zf5}SC#m8n?`56~;@E#PIy(;MH5tAp6R0>-Rd@}{u4cpmr3x1w4dFG{c;K3xO$s7D0
z8~H!IRD7&+{xue^>n&wcS+k}jsCBqAaEJtoUOg&t)j;}d;NAeHW&X!9dUPZv`ySY`
z|HPE}*EaA5zp7fGZ5*EL%E@k$&*hrkBET$Iea<H}Z|T%ODGGDE42~@iHE<P<WYu#D
zw3)HW@PK!2#w7Q`&F5Y%^K@?#70<}-3Rc{=O6g9QX!v8%|E?Yvgw|W#=Mw)|uz!ir
z)kB4fM?OZY^V+l;#`Fs6?RcuGovXa(Xu-bL*w;)6S0+oXd?B^IQ2Omw<sE_2AHO<_
ztaSXubnBd<0x#DhuB-aIs)2%GOdmeVNs7n|J0wb)aL9!)G9@GlEAaFt>HhNg6Z7oG
zF&~Aw8=|lH|5fn+_u_-|+2dwWCk$DRdQRhcALV@4;pD$LJX7q7LX}gE=jz-xO6C5u
zsJ?BP$glK&WiKtXCS<!MS@{&?O*kC<hAsa1W6?VYM6Ni`xaU0M$FeUG0fMJHyjQzw
z9XH82ZNj^DQqJa1hjZWJ)^)Bw7xHw2k<rrSdM8C*yX75F(CLzCDS4IY|Nm==Zr~~3
z6!s%ZY7UH>1HzRTrzr(88yCu&E|$4cDEYNe)S)PG&6aydUT-^Bv_G%dGt1Y7FRxZ_
z>478jJ<Lzc+7uyXR2rx{>B)r<$Iq&DJbdoOY77QbS9GhF9aT$vxpJDf!JP&kA?6C%
zl{_V>hOc-;boC=<O^%o#dhSqmb}rMp9v!jzNt&##)-#Jte<-(X3$K#ySog$fP27dz
zlj1QuO>HikOK=+9dZ2gd!t@Q(I@af<ebZQ7r`#=-`Dx<|VY{coS5|zicVzY4e9^J*
zyy@{b8f8vfdCHkY#5H)XtL<9i$ay7Ey*x=<OkQ<cl4%5^(TpV92+moDoLqnCWT+?o
zcL;OdyXg6|CijwMiW_=O1HR>LJFfR7CAB&!BjAhCEd~Zl%`A_rb~hf(i%33wYoVqP
zv;UUnfNjdwf1IZKJ>u(P&XkytJx|>BPM0>DE<YoC?B25My<w>b%z`fl_#Ca!D$ViZ
z=H__lvg4A7pP<TyV->k?r{%xgk^g#!6PrpvYm>)?iKbjrxdpE?h=&ILwJ6fQ;VAfX
zg1R}QP-Kz1S8=*;Wx|Hy?dNj*yQ-|N9>4R!Xt7)Vuly7LmKeu=|Icu->dAAvXJ^Yw
z&DD-SQ%PezZ{})MG$rCnPUDBuEv6bR>#`J>)^ZzUw0z;!IBVMW!mD+O>Wqs%Rcp(w
zQ!m;rF%|p2*|qiC>7Y>GvSa1Z*I8!oy)3?+Q-ZzU|4zBh(Vd-n`LVs9=jgwcei$5|
zSrpHFWA0bed7r1+J)imX&_=&28OaY;^owsdZ96OH&m=Es%D}=PuYaT2++v|%)k5vC
zZCSZ3t~1I4x-RtbB|Lw$Xm+Xomm`y-oi$6#%vD(Lob&EBG@9a5l`gD0`O@3TPjpSH
zo8IfVuJCfsU9#Awsd?3mnc0pW*_P){yyC8Uf4aN*mUrnc*26n`9QFEEUh;Wqc6pPK
z?oOlJg}-v|m-(!WdM3x>62X^mo0R|d$LjA>IBzC-{)qTx@J)p!a)RJC(@6>js^Sxc
z7bHzqSSLAi-Sen*!VL2N6LZu}%!>`buRnOM*td)KxysTxDJAaVWu~9ow~A<}xOThG
zyl27e|KPcw*Ug%c=JvNLrY!Hw>@!;bEvwD+vSvTteY#=GM&Y(+UV`4cXKvr&z4H14
zG3QybOe+tT)a-1wpY*=AvU<+Ptgb5QjV-%vWwn3U^wv+5uK!tX#?)$ec3C~A#jfJ@
z+m7cu-PKqiSD~g95h(g4MeT=UdeZ{Ysy&js_DD<hDeQ7eJoduR!nwM;wccm(&feXd
z%69q%D9l)W=8M5*PS;G6!}=1x7tep@s;oIRL~<EtE93FyTcxXa?VHiv`Rkx`^r7un
zwi%lm>fXt*->&u2X!ezoxLppsSEm*JKNA*wf5!m__I&3<tKUnkiM`|Kmi5xdXVHp)
z3H8g?N>@&N{%f7E0!P)K<IE3^*Zx^|&>&gaGT6{OSp0af^;9Fx%SOvHU72GN;>|Yk
zE}inMJuvB{-HNobw-=_h{?v}^co0+g<<epn!*ybtuUolRE}J4*uwZ7vuf?rbUaN7u
zWAS{k$o}1ls~(FEv-B)zui0CCVXLUa5yKs|vnBqU?zlNMcIw8<4sS0#`#SgO&p2(_
zd7n$?-c@kYIV0Wv!sA-UwFezX#D4ViZfIoRwIJ~Bg0t4^G_PH4j%iMmy(a3g_p<aA
z?-`3Ucm0|3c7lxl$}f*C>$(=R+C*!0vOGBV|C+mrmz2r>tZ$Pv4jZN{3XQf(oBbwe
zSBzV&o6(2a4^J`NJgHzl<>jMW%gnFk-{Dg!%36GI^;drlnN6=>Z;ezI6b@{z=<e~B
zS@|)x=lD0nnO3=npRAfc>&dY%#akyGIFq?Lvv2j>7e{65{GTt~;*+CZbKHLWY|~p+
zrnjP!D)&5oJ8NC7&524I4kiT#m9MWRuxzjtFFt%^68r2oKUTgmyM0RJwpzj_roTxi
z+m`WaoxEp%?bK%X)6rFRCl~|-mp)THHlgBQblqi3uJpSz|DBz2w4Eh6Jgw6zwmM**
z=U0wLEpCUZmxs2nJZ;@JLu_qn%c`qunmDf<*tjj7{kr4dw@>Zfy_Dk(Ty^1p?u*rT
zsuglGV;H$>zrXT+VZNzj_lL*EJ1yVFujra9ZhuB9zTx-u&er5zHxKEQHKlB*=JI88
zWOdkM{r_|0p*``j*OKS1^Zx(k;_Yjp|L^ciw9dY8v#Gnab+xm~{X=z6zqQ`TIkeYs
z%6h}H`a5w&A(oHt+|ql;cE|8eZ`$%so>^;ndG735lg*u=<W@GVV!4}JMO~)gBBkhs
zK|bzoM$Bp~2@G!&&1!!Axq9dB)4bO=?O7LVhIvi%&CB_<Vo9;_pMUS`?;jU2a^ACq
z!HHqHUJOgoLzM{&99!n>u*i9_VF5!U)9Hdu5p5yIjFwtd?TP4G#Ly$_U1iewY00T6
zI^nx={&Rj_dS;4%xzEf3<L4}z!Yp5=7!)oNVs!3iv$ZnpXi(v7GVv97urhdg;BqhD
z15>h?R4rI@9)9||+F6UwvV%3+Y>~5(z|BodPfSqKWGsHIw70v2E5a!HM1khvqxuhw
zQf~F=TyEn@*jIh@Q|%q`^-oSt(T<*HA#NOGRuMfhOO;^@Lv?v><2|EW;j5#z)op!L
z;oY$IX6mUD$5*ee<%BUsY~G}LAUxpV*09NfAIhJ&bWU%cH6!9#-rL3Jz3=R+-rLgf
zT>0!By~lg2r{8M+KTF@R^)Sm<;al1-EFb=LVHH-_=&T4^@BBCBOWjlc1eZUSViV)}
z7{dgb9A(>Mn%xbz|DV%*R`#eus(@KWT%}anjE5EVl^*w8xBd-{R^K`^QK>?0+9Nmj
z)Ga3jdvvC$B=?%!lThw6KDI(IN$T&?>mt%Wj)YG9CcCnQ<vp)!d@-ZOhm<J+zLKF+
z1Dr}zr*Nq>1lPy0{Y;&a@JdxN=!|GfT0rK!PQ?T!jcEyzLOiOiEH+QK9284iI(1@v
z<t>&K6QUWcG_u?~+eGpgYF4dS)L~a8ve;44N-(`5_1c_7o=MlvxvXv2c2evs$8j#r
ztv4PY^H@6JVwdhWl`yB0i>)HJX3s72;wtF47Ifalz^OgF<OXZ@%rarg>M3hBUfcGj
zqoGhhklpFtk>I(W_pYi0d6`)LpE4n<wpw6EW~a8^mM3k$16%|acI!p+Nrnixd^mSI
z%=yI4+0TkTc9}>z2{*b-^3p!%QsBcT-t*z0=G*q*!*-kuA^tADF;kd41*X2O@#1U<
zWeD|^Y&q`ldQ{|OL}IFXtXSjItGiPQ6>}qhy<Hi$_x&kr@#24<b{j97q2wBWfr;bq
zy=tL1EhSq$^A(PY1sU|-njq}w`!%+0YF1{zk{+k&srKH>SeFWyrp#mbdgIc?Qpc;0
zyjKce4L+^0cFSXb?KK;MxV7isO7>oLT<iGVvU8zg78^I+N@J_o+IYg1(_p>&lZ&5@
znnxaUj_-SW<h^HOSVUaazxWvon>lw)=bPL5|KV;yPKOJnF+0|!y%D@uns_xQXxrx^
zX}+?H@#4MTt=%3keIZz(Cl+thqrYoK&1d6nH;#R}xc#Bh*UQs|Y`=6Q?zgS{UFe`P
z=~3}kmE*!*k*ZT)+0Q&OO-RL%RekN0M_$@(ot^(@#O>3RoSmJ-;b5{eGrnu*iJA#a
zQ%;7w(eU&Nnk%Qlpy_kOpv}^YRlM}z;*_&S^D;vMM7ADWT=Q1XesjojkK*Y856{Z&
z+2p$3&56xu%AwG_j5!<BnC7fgJQ~WBAb9`Bwe8Y=9@Z`*4r`j6=BZ3lZ)<FnJX3OL
za<;5MvHQk5B8p-n3seNUqkR<W&#@je=ql`a9u@9<u0o*V|ITjVcV6N<&fHjV=b&r0
z)e@gW?^YPtyfW-JDfBp8@o9r)U{4a)jceO(ZQr;cXN{)PWv>%<&qS7a1TjX>xV_vi
zw9U6-X=?IACefs4M~v3B_WthTc6#$^{elx8jlxXsi>|B?(e+c|bTACLs;0b+|Juf|
z@`t?n9)^e3ry6L8);tVsFlN)dlBBeF$LyS_z#02@JQ9~Zcv{zC^6Hxvr%dt}-TNc7
zcurHvL6-Z9&pp;C&lB!#FXX6NRAr&C^T!F{j42yqn5GF$=}vt;PxxWF-G9&dzo(qD
zpMUuN3p?>C7ZlrlerLZ3kN9p>>-nX0>+YqlxyzRwwBUA8(7M<e)Uv4m+OhwY%2y}$
z7P;)5{4QguR_ndKIZp+YPl|Ykd`S<8>QrU$@mwDGP;;utg3Br=Hq<S1oXzm1J>=rT
zqelNG2Lyk9s&@29kdd=sSIUo@b|*80Om<G;St_H-aPCHK6H^9rzTVHP{B0BZrcD$H
zx;61|*%hS_rXQ}6(MPvWnDAWe`wB&&+NWImIS=G-EuK@<a&=9bV4HzT$F&6|My-~f
z+Zx3-2Va=MWaxV?DEj{$9j4V?{^#4uwnV<U#&#!S+U6BzTYpqekH}!#SJ)cVlNK^P
z_wEh1s+-eFxAtB?d&Wg@?#8L{?kDfc^`CZsA{VYaXF^!`lno5dNo$hYwnS_z_~qdi
zwsfM>|H(^O?3%B6Z{smF*4EF?F55i&=hAFz?GrJ#R-bzK?@dzq!uPJtO`=9SzV4g)
z@>Q(OV>6*mipJThM-7@Utz>OwTeoe8Yn-l!z2c*#i`u=D#AVneju$c(Nf-3nbh}xy
zYH&{bG1Vcy&B;5;t3dCYZC7pdotqc;SsXKdc1c7cc-IBlka}THEyu6s0VXToEnFQv
zH~43F%E>j8()Zk++i}Xl=8<fFrA1U+_qv>?)8i`LZa$X#nw4z(rttjITfuT}OvbI6
z5}BS;cF*wT_b~ahEdLzmg_ZtmRa!r+TPC_r!S&xipJiOJ&+1$F5(<7^J>_`MWOeXl
zhU+fI*E%;GTElch|NkP5*vvgV7TIZ~X^|6SocukHiOqVu@Kcw^5$@24&aT9y<g<@r
zvTmw3a%BXqHGF=6<4h1!iKv5!(gc=RcLlD%2~66r7Pe|+I0`3EXc4(}k&AO$nbM}3
z`yp$4U9-+hB#FQN_M7G8&eDn5TP^P`(VgggtfXBqe(T=rx%*q+<-g@o{P^n7B=-)H
z8hKTflW(^d{hb?QpT98i!rFBQxjR36Qxwu<6kt$Uc<65b#5cRE{$2TN^L+k{O1+*u
z!>2O$Pv83y^!9AU%}3fF7rj>bJjv*HYGuF;gRZp)_XO>}a%RbgNr!ibctqTLy|Xgq
z+CyF$-gD~5yKQ~%bm?Xk3NO3qB(&xKg4Pr-MXux(Q}t)HZNJyDOf>3+ncdwU|N48E
zUni*lzJ5OMn@Wyi$#2Kg{df6Z?3lc6A<x>~4Hv!y96OO`)O2t6EiJ1ToR62jKVa44
z#{d7xk?xX8%Pbr3{3$_(+eJS0_t!j0E)hJ+a?*}lWB%IQv*%wcxz?87d+8@SA${*!
zx36w@KhC#Z{@|32Y1z*%<^QvkJ;bBclIIJ4O}(^h#s5u*f_A1%xoffc+OsV&W|O@N
zT&)&K1pPRwzoBi>+-D)#j%p8THC79Yy;Q!LnLfMmJBzRG76C@j4Tt1~FEUwYU0fNM
zvGCI3h-C~n9`cvT{Lb4t;n0S)vFcx1a~V4P8DC@{DEr?a{y95PfpNcB4&RLiwvC(_
zig_EpM)5B#tX`yUT-^9PRd37JycexeW|Q^ai8<_D8vRiz<Fi`Pk*iHxE;;{KYi2NS
zmOE&`W**BC9_yH3DD}AD7HiARsj-u`l+I_$b+fRKo2YJdxWzvt#q6{G>#4>cS&LSh
zSnGMXc5V~lbcuHo_Wq;dJ$p$}vWIwtamCDIl6wq&VoW8MTrdgV!0L2J_~Vn-XU_7R
z7gWBpwB4SbzQWP_wtJCLnfU#${Ck~B(}VfXhHy_(V2Dv>_-DW+xPWuQ3GLn!T(cdx
zz8bK3DzJXJ+UfnVGex{>@r~TMKkSyB=sdM7$M>>H?Zo6F_HMgx|EpMC10>qZ{};=&
zw2SQWXxqZ>z_GCV_`xP=Rhxh`>0=Sn!C^tCXQW=6oSLFm8L}-kHm&mf;neFHY3360
zO)l~_9qrSTU0-ardmN#<r_6IzxXjUu=_v}`Hf$AtA9rs)uI4S~eLbi>I5~LV6URx@
zGJA|N_okRGF-Vx<RxDN$e^@-MJke*yN4o>hCTNw3M0_kgpf>Sf+QbDKtjPiVYZv4!
zHb_vv?&@KfbNnLL*8}wxk2_9I>zsLlYnB7ow+*IS7u3HsV4Hn`Yj#5ew?z)$%LeO}
zlkGKGgBVIrZ)2Tpz_xfn{F)DS1;u7>8<HOd<*iVh`s|uzXr<NKrFoG*^Ii-6&tG+U
zs_QoWeW70R;qJe}B>!A<&h%`~j%?1kDZuO=$zERIyjVoY+{lr`NL`tAO;BNmYC-7C
z!gNNx7iNO_4-4Bp1?M)jp1qi)W-i%RDI%0ozqcYjDOhoiqR*s<9dSoHeT7TiG|m5|
zPh9Tdz4}IH&<4&m23*kstkDTt3l4HceURRpIBi#Y{OkwZXTNkTHsI=9!C85k>z6v~
zk`Hwii#dZ9aIUgsJ1RD1&WkzAXC?)kmqs1v%64a)@k6rWvewrJ{C7^y{dQsQKZSXZ
zPC9K{k(^W%vT?<{{PKBa+XFI$GTh5rw=2)ra-T1HNXGhmcx$IiySq|{yIjb#iVGGC
z)KdR1xXRJXA=NA2sPLrOaO*b}XW@n4Ed;}y(jQ4Mtb57zqNA_+8W-=yz5;X8FP@8<
z+$Eb>l9j_vb<;EZCP_{#t2y|+h;wHCe)Fs|-yQqHI;BracV<kHjns0jte+WHKhY>$
zIFNf*z@+(;YB*V<LPNc0W#pWgHpk-Sq$L~X98qJPG$F^vGUtd{POy2->1{dgl~c+O
z*ZaKW{<~q>qnF%|Z!Y^Muzc?Y{(lFS{Y&7k|2gg9qNvk?VrGnD4Zk817`<9=H5wGJ
zke6uMeN^uUlf&mu#W!IBxl^J)9dc7SV*F|8%5ui$Ki8(szm*?9&3#US@VCHhtB!(-
zjz~%67KO_m!H52fm{qn+YiX(ZCbe)=qtA@htCID<6s3GoW9T~B6jtG#c*1MfbcW|{
zGyfcwKFK~|x|8NP_S{p?CM~=;@t;DtV<NxWO3tqdlYT6pUF_WXW<rjtCd0~;re6~@
zZo5VOa48j9(iQb;&L=5W&jm|Yag-)6;M~Jm{6{@}@u#`}80I~ST+U)Wue57J*{Thk
zvo}1_tX*$m^-^+?@a}m(nq{_CY}}ENaolias!%n5QJ|`q<em<ZgP-SjP48hm)pPEJ
z&E+3G$!WdOYMb@7Hyaz-8yqrNzDt$sK=Xqan|+(?0+hCtEMuv8xp4lWs<li~1rm$4
zUzcfR+uF9wL1J;yEY_N-WeLAi|A&P;`}8f+5SfyBzA8}qhz56LCg&2)smroM-PtOa
zb*f3STC^-nKEO9EgFnT9tI~@teKTjyhKU!{a(9}C-YRRb_sse3wQO<0vhvMS_8ijA
z{<SRs)v^Msc~f_9n0|W0jNKb1Me|>m)XQM8dZxB3dwStN!S<FYtIe%qtF&iCaptdl
zJoVA1sXMk>O}BC{WGo7eC=P9&Ht*E5{oJdJ7#A(8D&R^t^c57jojN^C(DQI);pQsO
zub1TuD|Rgrj%E!~*vqv>&V1UDDoIVQnZlP;oVKp<IMx;`TwG#Oyj3V6ee29~%S+uG
zrDQs1{oBA9G-3Pf3Fd+<_32aR6f{qYN|?N);D2ZE_A<X_5zVf$vPJ9DZ~0Vgp7QYI
z+^NwU9%|0JGjpee%)HW72RUya<obP3hUZZ6*D2d1RD_~8ifDJg<(#kkXP)F~`<z1?
ztHTvda{^VPs*Q6FZ`nNmRAR+{&LdjCgBE!O^|uGzkm%)Fb;M(jgt=mpKhwg(X9dAI
z3nP+LZCb15GtF7O{HW}#k}J2j2rF$0dcE-$)66%oyfSS%Dhd;VH`nYs(qH%{KIE`u
zi~8CR>>`2-Snrq!mP@myJeV?HX#4B~)0ZTuyH?i9Fh@BnH$;ZSw`emQeztv|!NEeU
zokd<d^P)CP>)DWdi+}!_oik4_<M3WS<MqkHRogSREYG)$Ykc)z-rHiuMx7N6uUfXo
ztdNm#i|^DIGCY=jDY|!6T;PhPcdFvNi+0-;Yi@tC``~H!MNtNeUY#+xD1W58pr^)P
z>~aCq1CRAtMuCA_wtB68$dkL|fr*7+`{pz93?VD`hvp~VZ2F=hxuwP7QMZGT$?>P&
z$ICa)G&yxXN?B^M<jh^mSmz6F4?4gmG-aZrX72oqz&BCJ@y|2ZxeqP$xwt6iqIm8_
zN!~-#Eov9uPS#j+NK@yMC+lJF&<rabMIIpuiO}lgGlvD4*0e55+4LzTXxXN_lQ&uX
zP1WCh#Ps#$i#t-+9Y~ehSmtT8P?)QaSA2_jm*~1obN{mo53H=pm7bE2vr%#P|MFI`
z%kxXc8!JM;rfSR=(wz6HzbrW{P51aej%~s%`TsnM&s9mzDiTS%E&aAZ>cR8c3dM#L
z@qM+ilE=InM0zd?-QCa}dr+$P#&qj>g`3@C3>w$iOr5y5EROM5LXL3QT6KZwsq1G<
z-LzbPU9}t2#<*?WGj?>JpCi?LVD+v4VrLsRGG1-<uw5atxpsQr7m<y(rtjR#IjL3s
za?viAZI|9V&A3y&`)I9rm!{;yYVZ0P4*4~=_P1-sC{M83a(?$}jStq6#?LO8q)DFo
zdm_%9Yx?SgihB+|vc0I(bMNWg8;@)6alPKK<bd0{6MeeItY3I9HH7U|UVC5Pr+oVl
zwaW1MhyMQWo5#E9|Ka(vw=cPuNpDt9w7*kq$CPH}bYx1pvhCa>9(#^(91_0oQQ{bU
z)c;Yil`5;`#iOd{j;?j$3U=_0`PlZ8S1rY35wnZ`B*T57SNt0@T%)wNZ5PQ*eEqnf
z#(UZ!uH%L?EzJc_b%`%fJujRby6C8b;?CAZOP>fc<|HTZPqIEGq`SO*?aA1EPovJA
zjJ>y^_}0l7zYRt=3;WJ4e{OY?eND^PzY)*RY<Xs*yDsPX&aUT8UdF4rqj$WyB*eP=
z#nU|141xcP;*vCP9ZD@u-MMDVp)>t@X)+gYsWC3Ho*imqef0L9f=MBs(@j>-?+dd$
zA!@1k-1V8lv{`W-b^`Bw{;xT#6&0Hy`M|pFyl>mDYisQH6;0dZbEEf(q1)@*wT4j#
zcDDJxkzBLk3&SZnn;Z9T&MjoVf1Oj(G4TH7y8DW`Z-31VNxXQf%`M~orU&1CuJ|gc
zn7QfdX7hKG8Eu|iRF<B2`NZO$OPg(DpKa0;P2-f>d{N_ycCqmN7p+l;t^{tZHS64>
zHqZR!56y{3?|4l}3wtZClD+6a=z}|ahTSshQ4#*m!S}nBq~fGA%Yz;Ncf2;La_qTu
zjOFI}5asJCo#G$VGrE05{9Y|_{J5m|+l-hgNl^v|kG{LN@Qn5hZ|*k^%U2t4?M>Lg
zbDR5X!57>6{Cgev*KFWh{h{tn!}b(~+Kegxy(aqJ4D#H&MEiM>oW=~>xR;&JSw*Mn
zTz=jBY**IB!p?VMD@(jezkFNa^4f9Znbo=9t4#LX4qABYPxG1C(|tCw<}OHIV7GdS
z!`Zeg2A4Of^Cl@Aoa?pk>yM*v3xyZXm}>u8YkHD{-S@MKQU|V_pFMNmjLPEai6>^;
zo7U*EZcEtkX70UH_kR7o;3MzDZ)?xB+JV397WW$m?uuakzZ<yLeE7Y!zy82}F14S(
z{}q_ekhq_;!}Ibzd7<dHS6@H6VRz}<-hk5w11hJNryR8U`tpNKOtnJwyC0J80%t$S
z5t8L(lHK|-<^N6P+z0=9uiH*-fA}T;{kMB;IVHkX_VWMu1h|;~t8gq}U^vL)d16bJ
zYgeLf7{BtiO-DC~h^q=2iG?gEJT*l(ct?kb<^dJS$(o_-QckKa^`30Nx?{;nBTjEd
zPmvi{W)`XYO`XlMg(>RFioh_n=#5FnAr8Tl1N!}Pl^3i!74|jg+G^DWt36r0!n#a7
z%bxnq*1R}tO2Xz`ziCpwZXS!4&egB4tp57>`g!sDdumvpzn%SE|6$c?x2uc7SKqrA
z^(O0U^41;kYXeu`2wgVyhEZDB*ND!ft(74eXHq;SK0MMox#3!LLc7R@)aIPq!CCnX
zmZ?tf?<+R@KisQoKK=2@O6B;xYVIOFrR^@y9|f6<^KA;<H}B`B-LGFgNDrHT^Z%|r
z-q#GGcRz7y+gr`J<k9jce6#;u`TgVTavjwLAv6Cou@pEka;toBU>0^MILL52AizOS
zE#T4erb!MO94$IB8l0RmNgWHz?05uS+E`{N>4=oh3h2CF=iImK>!sxWJzo<RPdug<
z^J<Cp<ha{`Y76dZd3kUOF?j9!J1OLn#-at2`bEN?NV_gw(zQxRJ*X?D$b0d%PeSW7
zq9k+99Qv{9_<ZB@Cc!3AH(owJ6KJ$LM>OnDWV%S%8<|$Q`bR0MMn`{je)g$)bfsAI
z*eg$CuWNUdop(<unS8}X*nQQO?Gv1&JcB2>ecBQnU@vMK5~Qoz8FY7E@Ykd7j}@=%
z4$Ee`r13wpa8hR2l(=O5u-|20r(TbGUA?wvO2^;+9wQ6Q<eUGesisD<FU&Q&Wh9fc
zn8(~}W7zGWO{Ev2b9RfGN9P|GEx&EFN_e$G8LJS-!X}1{?3MiLt6CS@NAKJ|)pyAW
zkL4^P7gj2(+FdH)o)QpLbZtgM?pJla{;F<`$q|{#i@K)oib(FZUbgAfvKs{&OILr`
z9kEI!@YqSQskbtnxwf(xWIvmdam@3?%8r<qYiD~LTz%%yjz#U$N+TAwPYpGBy?WCn
zk=Z#WQF^D>$VIs<JUV~X;yBmImu}TWO^N#RGWAu|q<e4t>uVC-O|QKP<}`nt<-FD6
z(W|_R=6mWN_r7~nWIS>E|NYh3de%FGr%G>Mc__H%<A<13n>{T%GrpLfdb|3)=l18_
z6`vQ&rF<wp@62BN<-gJ#g9i<UG91o4x>J%C@+VKX{ibU)E5KDmeTvij=W^}3?jO>g
zEaLVndULgiu`I-{@>%&6{&dE&Ebb&O)`pqIZ)TQgX_~n?2C2Re{x$DWS7g%6wSO15
z+gDG`D~fN>aN^!&P$iOGoW0mF_=|~?=;!5|Yy#axzG__2`V;6-zrop%rRU85DQ>3B
zpPuIQ8S-d)T$Gz(<CUjo8Om<%D&*HH(4s68%IRHt$Uk<K*}N)0Pa9KVtvy<8_cCvI
z?kLfg;&Mo|dUDbGlIdjmYZnr?3OW22sLl{d5-p63i3r;?>xWLB?rGkDo@qN6rmWR2
zS{)N%6E41eg{#3bZe=cxh}~>P6Pi^Tx%oOhq!pDWwi~!`2|WmqJQ<<Fs^Q=yU%8NX
zdyBK8(MGlOpC_Xh9iKcy@sir6$Sww*W0TyiinIa}yMm`o5(<maRDGg6Uu|lL=oE*g
zvP>7ePRFrL`}XsRt{W5g`HE#i9V?Gc)eMx6dZEz}_-BoL^-1nDH^&u*PdrV!S00zR
z#yE3jD6i>i3Fhz<Wm5aDyf>M2^Jy~6;nQ0<pBeniII3~U+49K^4)cc3()U^>o0oh_
zvp9It*W-tp#Y+?SdDj)^Ff2-*RK*rl_)O7%&58fY^S->;;AR@t>l3Fe-`w?H<7?vl
zx?jpx9h3YPv93yTWj(^l+1jeW6%gwrbV+gPgVmzSQY$-tEEQ~A>akRE;-TE6q<K<@
zN)}io^?$TVysUTN)iSG7aXPFQ#5g&YE_6=q3~>>YpQ^ev#YNFedf)L$VIGZBUGC0Q
zI^Q(;yXwN34#t-=er@!=G{@7hW>T>JHOJ*FzdZdi7P-z9S*m7obgEDB-1xX@n>gJ!
zrUX^HhAuPs8oul2)s|-0#6*47RiRrqgmpiC$oKz-*3>^<;cI8*_-fsrd-~lfcczBW
z+BDZSvW?q1&q@XCaMRYk64o7kE$HpGbzhHOpLJUF(JWoDtE!CujrVCY2<?#Dd*{b;
zGrJ`s(JPc?PgcZSd0NO6lpuO|(vgT4az*NPSI>t;K8o;GdZ{(BM&<a1Ak`0bJd>x3
z%nwv{vzt7{M>W<lKQgOLcy_O=s*UiBGtW*7FZdU-e@}0N*dewp`_?4$)M%x#XbYP;
zi1y0u6}X+egDHI1xvvI-dOo)<9pf_MU|ptm(6{7{rsl4@zDHijd@T4Dww-_ekHd}H
zcOJ&b+?-*vu5jPtj}=l|SD$(+Q8{C=R;co?$DM4JU2IB6Jpy(qw(Q~4-aI>2a><0<
zQ||Gd(h=f(yzj8Z%ttGVEd1vA8n@fNn)FE9Hcw+pNYj${s!J6gAAS<-TeL6C;lGxL
zxiQzPph*l?UJuk?d8dYLX9(xGe8}kk;swpm)793U3cc81_hpfA?Xo>L7K^Q(*z2RP
zRGQE2l*`OXOv`!ITK(&zTI5wq^EqZa@wa(x?3&xjJ^yl+(l>$K?u;q?$+CA7{}yz5
zt+~3+eD`a;KL$69ICX_)PIXO*z8J|+q_vs(i0h3i!JhlGwu!1fpKX-Jq57QpUC=#^
zpst*Q@yE6%@9baQVkh@;=RL`kHIpJdBFZvX1>f~#dNL<&QspG?(5(EQ$94;SednIa
zCA~Dv>tx)5n$Bt5=Q+=XnJ`X%uygM9>h((bKQF6%`M&?A${D@Ci+m@1Pd%i}&ZBR8
zPki!nn<xL1Z>61nebHAh!(z@iAJGQ^J?5p8BA%<V`JF%ivv^N$%bX{BmfLFYb(;RA
zPt9Pa;pP2@&(91yr^4FGTeCt>KJnp^$}`HJCpG*jRh+!8i$SWh#b^to`Q(tdTQ<i|
zxw~)C29E?T84Lf_Oi}^TS3}%Q-frsKv&MK@!1~wC*XK>1_2}6r&!vh#wy8YyJruL^
z<1}vTgC;STnmoQ|F7y+5_Ghu}{mKjKR{C!fSP#96t=yLuRj}Zm?Ca^)t<%b+R5pFj
ze*6E_%3$$rEvuWp`ox{R@#djHgtN`O)XEt*9~Y`e&e`g}Azt{+TJhUAmMs03*}pa_
zGj{csi_h#NvZK4+FJH}bL+pPCi-hI92W7LB)@mG;VVriUVXu6t+{C`(%(;i7?a!#U
zt7PrhymqX2BUex1lJ1pUUsg?8x`~UGXXd*{+dCAQeS`F?E#%h=FhmQiTHK+(IfDE4
zLJ`ItyjHgux}}Bwb}_gs^|)}`in{P{ML8bOm^|&U!NP_qe6#yrZjhFFVkvvaQtpcM
zoZkocE##Fh*|?;5&h!a}lAAa3221el+PLbn;hNd~zdG!W-=4F9*;<EXuKP(1O%4n9
zNprW}W|?wGcCN-@&B?nZU-M0wAbK*g|AFE>na0BpZ)#fR$jw=>?(YV@Hy5|dIZt}G
zalzCXla4j7S{xu@yJYi@9f!S7bF$88iVW_o{NLQYOQO{+*~DeHi~C_0x5ZMcJtv>{
zcD(qx^(q6`mCIXh9oW);gF)=(+@?(18^YVer|jhyGWT13bia{k%#<m?(Obo<7P_z8
z>fR&qP}y$!jjfJGi;_glVxy0RCmvIC>VFz7rt*nDWw)LBDlx4^i_)7H>7P={`s|ju
zcw5HLj_VWo7kPB$1@Es_WPKzgamQ+1lfm`|$L*6mW=!EYb|ld}j?-q|f^{qwN1R=D
ze{|B<pTlK$q?1`?{nQucsuQ_dH@1J><iL|NfBOp#myph$4vQx@IJZm|xW2*W=cXOA
zJDLm?dU987n9S0{sVzL!S@>AS<Z}mj3?;Tqcx*8{$8stQPxSu@2WQ@1u=C)-IoT8M
ze(YTsvrg8=tN53r@aml*3;Eux*z`uB|K7sgCNWkLj{N3lHkmCE{E^YW^7SdXQ@gaz
z%+t~FW|Q2_rYYe5lD{i))eXg!{2S%AD<7W9)qOu=;qq*WuQ$9Ot~~rma^C(o*6VhA
z*le-UO|)o}Ij!orM`dMKwO~WwE}KPFTjx0Rxv$*gvP)~;1SYQ-NB=)sRX?;&Enu}O
z)LwCuXT!rI=NMeYb$F^jv<6*J;IroPDPDf<wzAmCv)7A{Ui)p}zuWTm;k_ZJ_a1z-
zcUHvV>Xi9Wm%SC7_B~&$qad{8X|hu6Ouvtd&lwf$dsDJVDM)PE3;ygi`(7^=`}F_H
zw1UF5pJUd3IWz73?|nZsn2V37-2b$;Hj(4TVdJ?P9SRQX9y3YwCMf>C%>TS=UBznS
zH<{}h&&)78wB&780HetPZO7xxd${algi3O_I8$b>&tQ*s*b}))f0_aJ_8H1iyW~15
zyL)mh_;zw?b}&rZ!Y;DLV0sQuS;wqRowK%Zojse;<0v_M?IQMMr^$v9ty7n5IPk)P
z!TI3aGaGmp?S4~nNt(w?X!qGrlT&hgE(xBt6kRQ?Q7d@rNUyk~)9R2ppI7wFtm-#Q
zIkfGf;ZYTv%{7;`c`VerWrJog>@soQG1*8f+sbUt#%mh$zADUn)!}?VB}D)BuCs}%
z%AO6H0$t7jb1ruXy3|irSy17l(-`t+2d8qSysrb#0?S#KJ!S<~2>;*7<9i_R<P9F*
zf}V>j+Se=b?wcXCpwZx3$d-c*Ch-}p$9I|UU9?+lWzb!ou*E615p(ti7>A{Z>I7dt
zDr~jJFL7&J&DQuQ3mIzHI0%{5wrosW()s$2qT?f-{8Ts9Q>HUIjy*Nz{Ls>S`O>uX
zgLY-Ey4qdg8LPLwOWvlW#JYSI|4$CR+L;{D9cSws-5YhcD>)vXB(SJ;qiAi4g!7|G
z908kJbtG5rkZ7&q`s;C~K6}Z+jti=d2Ud8Ti)uK*Wzy+6TTUoO{_iA7k*Axxm_1vK
z&rH0+6jSSXN}_eeOZK$?RXu?}E+$wlNO;9s_TuJ(r<3a>ngc}_TsYvVviIa1lb&V1
zv#S!CeJ5<*d6{pQ@x&Jw4u)Roc=||pq0Om9PY%wBHr(&oS8dhzD!?i{GTMIcskLt=
zN;gKE^!9JNxn|{KnT;(W7P{VBSmv(EXtvB{4KJ7}JkiKz?&&)-8QxClbQj@w6rFcp
zGvwr3qlkwx(UaEgK6*uWp@fDa>%`7U%XUm!rp49O>9Zyy;_T5?GbT2j^w_e|Y4XJv
ztw#h{Z?0|K&@^lDOP4blv0)Lu>o^uf<(?Ja(jGQJ=E%X$)mzoQ4887NQ5GwVyK!6j
z-W`VP$+oY0;{tZurm3C_dU*8LZ`)HPdrtn(>U^T?b|%a3%Td2qE=sA9_myt-O}Sx}
z%6a~Z*4j^drv1)cd(~)qQQ=zkli}?jCYaCS2tTmm`|b00Zk>N}Y4NYe3o}LBKI;ZB
zS{$i;yu?>YlF>)M=c8`dSqa812^@dTk9Di)Eu7J?_`v!MR>g_)=6C7$?9ms9o!?Vp
z@xy8Uwgud~FDx|*R7=0LYQa+B%@gllIx*kG&{9&iNBQj@m1!3xOnKHx9Q_y&$9I5_
z@6@fir+s@*Bx!9966B6vvS+sLR?FFor-XV&+lgMTb$VRuG_U!T%(0j=2Yv3g?{czw
zb~}vW&dtJB3*$RejGXT@?w*zuz5Mm%H8NNBv&?(wnPTx};=%tZH&uSRw^m%aV$0<}
zsY_eaW0_S~SU^+DjFbg8LJTtR`pWPwNo)<Csu0C-_QI7TQCp8h{&fl87kX&MfiR8c
z$6rr+FSfoJ;QDN;0Ov*H__U)DGSB>!!;-~Jg_6S3?;edSSy;srpCGd;D%<?opKE2e
z!wQ~WORL$Jdd4(g?)uap{eBDr#xtgs9aSoydp#xidBL6gs~Y^5XK_@faqxF)*>t&=
zXNouJ&S3ktSZ-C!r>AQY&+O-1w*Bvc!<`osrk%LH=taWR7YS1%Is4{XnK>MAyu*L>
z$BoGs#3%Z4^7)?WYIzv(Xmix9{Q|x!I~!(NZ9HjI*b~Qa$tLmT)ZnOPrBSp0+g+M<
zE_2o~?)VMt%d=*`?C5-Qwomx=qjg6gY3X_`mAkfnZ_ona<Vg}TE9Q7PZacLrT4u%W
zi9AJ}?-N&?IeB}}+x|0uH+kN>ZSmLez|qHog{?XtA5Xb;{`=)%lagHyRl7cLgzQ<i
zQt!q}z19449AhP>`LupXnO5<*>4>W5z5b{PTK;S~zO2GOAMUAASoUCT&cm%a7eBLC
z|6s4)(Q2*d^mJ`+jaTQBrS~`|FvMT;z1L#<CO7xq64yry_NG3KyJdFnU4!VCfWy+e
z>~mu~AO3#(K4M>@bNut*MTL5cWHsMCGV4qdK7Z+!-DjQiZ!X6d6zZ0zU2p41H%scc
zndQ!UF1_zR)B7L4&sR;^D#Mh~x;Cx-<u0Ext}j9>dCKy;Wn)|Q=6hvL>e8Ia@<gWi
z$c3(#T+<dhoDey9rY3QtN1*t&^&+{6Q+2sTQn@!NOwu^8bbe&gEYnA)-5yPeiBijZ
z#PvLro4Ih_JcsHH%~5fOS1)~}S$65dI%&<+tUW(D-h91vdrhI9-}2ek8_PdkDhP@;
z{rAy$tGn61tS2{hRz`EWsOBs%$W~LIu;rhCda&kpyF2Z_ZQt#fDsP_RyjIgKZk~$&
z5fwxGIUa9TCioal&+xiB&&0dfEWXpwyYutH2Ul_<zTeb}I_eUcmTI)~v(J{RZq<$+
z-K$MjvO2M^W1N>RT;3e^zkoR<@Z*AGsnc#e_51CR{LLgF`>1AyYlP0T2$`*^qUUb0
z1*FeQGgAJ#t8d+`1G#a@b~*pT5`?d(7Rnh*h?T}p`?>XgX63e+`1IK4r3G`8(zp}8
zDRWti-nerpy{|y<-s)^crVItij5x*&n-tXpW?Yw_s&v^N?>=XKOxeSC(fgnK9`N1M
z``hut+OKo5SLZqhi?;^dE29byFmrY1e2|FicHW~pBk>}~4VHO%w<iDIG5g-j<$nt%
z>r@>6&>rtG`?^YD^on-%`XAPP71jl<o!eQXH!w%<XzscHqqDr(NP+Rq3j;py!d=?0
z_9WWgyTs&sgK3Y@Vauw@8;d_mvs=mkk6B)^>*uQiuetL4tNtusV|ICWU$peb=s72L
z$tI>OInS?je~xC)ecjBW75AqZHd!0bIleu(VWCRtYO#KEo69Fs<<`4Jy3gY>5=vBM
z=c=_7TD*g+$28XBK9l9~E7N69u02(7=3cR*yKvAh@s5hcZ_iEd;;0GV(0Wk5CVEHf
zA@`c2<*Y~cGas|BIabcv?=LPiA?cDIr=ZQ-D{*>JcUpWlwU*5=F`B@1DgEusv-ckC
zIePQj-gmirckeCCI(xif+t${*?r}49o?L$RLS+A|C(oYfH5Zn}KigRP`fw@B%lOZ0
z-o1#cd%aDprj#T1XNQed*@t`nO%E5RpL-h~Xny~g{iFY%UmZH*?4ddFTh{4segS^3
zBzjNY`TGAhd+dcjWnHT}UUYxWU%aBVtYJ!pdt{^$S3`i3V*`UKiz<i8j0p<P%}i{3
zA_}Ta)1v<A{d(OlxLChe+Wyz07w@O<pRn)E+%2i+!*rw9-rCc{DZETNye#L|rpcGD
z$<DK`e#O&e{K9XMRkf1ttjkkp8>Q|jXxP<ycnWW@TFlw5($gAdYqG8y-#i~~8#a53
z$HWO`Z&@^RHy%2hWvUmpx@@)ILa)#RVtTbZ3j`S|3=$5u@GCz%#AzVum$Jh|&3VFP
zhEvnDllQHWELLZnt1tZKOo7SEOUrC$i`7<{9t=v2Ib5e}ZT9x&w(Q6M^^_lc%Vi9o
zs^KA2v*W`ft_JQiyIT_yp7F3t`|YtXSgv=~adFz(xus1PPv;7zq(=QIyR`PyV)25y
zJq$r_{MXCJt~UAcZnfx=^ZoVJLX3YWXB_l$W4>m*z>Qt1Be0SCR&c-f)-6HY+G}e}
z6SegcD;DZX+$!qUmadX`)V}V{!>$gwX&p+P+V?Ik?9@3lL#apZ-V--X->r_vf~|`J
z*)%8J40_<|^eegF{_7Q|QxZWrimj7_Js49b2m4%nA{J^CxpZ1gjwS2#_$o=Y8Oc?e
z5%FnkZpUVxd-{FO>>RFdYMG4}CDZ4YmK_Y3S7N6cF+cW?WJW?m&@++LmUm%p3wQm0
z8#XgI_dwd?><RnSUiMB*;##@XEX&Jl=~4$NtrhdU+*At~H5yi}TB$X`VKqnW%j?rK
zxE$C-&mQGWUsv`i>UrLdU)$v~zpZt9Q#ac{WPZg*&I`vj<qCWW+OpVDT7EOrhKF$#
zOcTDxZN0n9y{&rfoa1hrm&9m0SU>kHyW_ap)g(&jqtTW5PRlGsWSo8}ZC~#oEEqD$
zLuKxiOI{ZzSx@#^yW`?1U$>uAt_0c#+I;kN_*qhTRI<3oFu*z0=g~2F?ym*f(OZNr
zxysf)@z;H`!N_Z}->R=+Q=%UFOrI7}m^n2*gpqAaNQW!y+>0%`W|GOC;n(IQU(uW<
z={%*@Z0ps@{|%SSx)xd7TbdD@E?zd9<>R#3H&Ykan&oEsG0a(gfRp`u;qj|aQ;T<q
z9m^K?5Z@+obw=&ks6B_~x$W4);%T<+QTKW_UF{_s47nViHQm;6yyX{dnK?(|+>eJm
zj$Z<pgnI0L?crb8H~$rrN<#z7Vi`}d7NMXUT)SS>l`ecIbIWK^``-WmlHMB${ZIO+
zr6j-j<7xH!J)cf?Yb19_vqdO0N$IXw(6S`MsCdeK@0m-dM#V|4p7DJ0%ze{q*V?aM
z|C4dGMe4c0<F7Nylfw4@>-yyx+5i7De*nW17fFVzolMgf8dqwCRL@b8UcO@)&lD!n
z6)GMF10n-Lay6Kj&kA+)i~a8vvh@+S_?<_qLKkM0aAkD=`tW_9&D&G0duGkg?ASW@
z+(b97$3gzPwk@4^u_N7SO~bU!987C325=;Liu6~XNM4tbd9K($g(Je_a^FvtbBgbq
zPB`s~n9%eiJCA8%bYaBw2IEDBy_qU}Ys9W?nB|a@B+<P%=tahHt4AOE%D+U&IxahI
zlXpTa`9{+zvquwqH}H6RU1B+VW0I)WOC7!Pos&YpY_eP6`&7TZlP}Sta>mP=qRiUK
zx6-^iXYJ3`Em#tGHY0*_+qnamikSkUZ)JJTj=XirY|2mG!Zv1^?LLkAG6p<<ZDvZW
zJX)0VC2EdHn6PVPk$2$?XMW9TM<*yxde-^>>jHjFkMda+jwvOJxYJ)q2sON0!ZmkA
z5(l5>W_iw>#&Y>0!4Ea&iyp=1RQ{Ba`y6*=kErN+mmeYX9Dcvt({Qe%Qg7E$g|*Wa
z+O4j5xH(NaGv~;X?!J7l2&4HwqdM%qZ0z?-t&(3SWbSn2uD_q=)v2>RwABBI7W*?z
zSjq7B$nx1r1p$n2gs+ucmgK5?6*z<I)zXfvm7IB6K}$HMF86AbjGOl%)N4V|FP31n
zlMi}Le;BU}&WxB<CiJ9-Wn*}R<fH}_9e3-yRm%iV1|P}JT-R~7>(P#B7Or1Z`eGIr
z`ddp}x-KtwQBi1v`2KIsVb^9FGCp@nF37Q%)tckJf$PEl13_M4H&5H~Y<9k>6Zj*;
zu#fZTX2sdb^7d-CF8AqdwaRpMjS3U$|H4$AFjM!8n<1O;0#{w%EN+#?$n9%WTD@&f
zpH}UbHrdVlIitjDRuUWYspsExcm1+fHDXMUl}h@ab<H>R#*Wu_zgSG2_UP=>ie%*m
z-X%wKOQPq8vP`~n=uW0pucYd_Wi#apmMsrGlys?P!A!Yr+g7$uG3L+QwRCOnv}60;
z**v$3RAtyO>Dh*tiegr(2CQ}ej1Ql`{lSs<)k7ZrDTnoom?h;d3-gvsRLUkD6^n43
zru?-@PN?c;&&@z?$;*r7Z!Uf@X-!F>24iH6{8x>M?Ps*t7#&%%tTR6I|0XVVNi)m(
zAh|0%xl-!?HCOpFyb)X|P~qV6_Sni@f}dIBI~;k6w3@S)U74j_k!Bu$!F21QNlYHg
zr+xYQ)MrxQr^(S>k|#5`XWuYcR4=GKQ^l)WQ8?;s<+aV%idvUnEZ#e9^CM>6^&*8%
zT<7=&H=54cr!+(1bK|s!->%C47Vz7!s!qI4KxkL$zF#J6frr?l9HSQh5S&o|XU&n4
zx7m|4RW&85PZ?M$Y&!So<F$%4hsBOv;ETEuam-_tyZ#}gY0grgl=O|cqb@M1&h`w_
zt5}#aqsHyd{LD);IwQXu`tF?hYv=6jkN?(bZJ&1LotV`0r>gq<&zId>%`lm{GVR>1
zg}eXHbgCAPn|J$5+~>7BJ~P}XYKiN$^||cauJ0G|Lvk)l)}lo<i+&td$_xBrcSg9?
z?>*myOy@7R0w%|AIiy*h5FzcM(6G2;&0*Hui4OjC-BA+~h4sr`eC3$3V3GDDk*FC$
zKk~x%8I+cs2wK1D_cal9mCYu$K79-SRy{41IA`K#`uDyMYiQZrMY4_EQ6Uw#AIL4Z
zr*X#9G>hf=xq}Pa4BjiSC_1;zd2TY-WNGBxs%(izS1z_(=ViJ0BSLm}#pWt@wXD$d
z(UVofUEPBp=@)7rx|TJ|!T<BR7Z+D&3h>NxGdKRqVc8+JRYRxRv*Bav+`r<Uo2$wn
zxhrUIDGjvfzIpNgw6bK~rsOmIcQ;;{wnDAI#)ebonXph!<I`Q5&Vfvat2-7>yKcRa
zRdZW{O5@ry$~x7#KTiLT+H@p`dscwAvf|Fo-+v!XkDjpqPitCzX|>Rlnr5leZ`VvJ
zqJ7gD7F;iIW84+AcW)4r(t_tQayQ@HX1;ffW$MgI!8_HK(=`H*CufKs*6pj+nOx=(
z@zh1;Sg&u*jP7j)bG9-}WN48RG8N`LCDdwrqd9PX&mESw-WaFc314<NN#;2EPC3Z3
zrLpsejNmON=^KqKg%dh+1X6MwCHEZpQ^<Jku9Gy!9N7ts>{A%ee_bH3%cJXQg8znT
zx+1ZXa(5+_I40TjiBw3tIo|y*Efy)Wh=1*s&Pj!gH!_c36TIhLIrsmIlk&e5r(Hd{
zFeI{H#%r4CsV}NYN?kW~ER$F|y=E<yobB1gYT2q%cVX$9*n7DGOL~sYn|tL!W$fP%
zHy>tBV3(YzIZ0vXhNQ(;`zkB$2^b0r{S?@GW@^Qb*yx*!mIR(ykjXxOqj#nShv?q`
zXAZs>3XgYdJm%r*6WiG4FrjhN+bFraP0~KD=bj$%zQH1y!yx&>C3pfu%8e*V7gr0e
z7+;6`0<YXSzdXom2$L*Hw%Fx(q#)UZDd-#P?Y22S+BHWMe=58EQErN1$(hQU7b#K@
z$-2{5C@#RYT5$RS&N~TzTrX~jc5M?%kdXPWnfW9&#Miy6x9rFz#zPM>Tdsd`eBgOY
zNbl;Y%Pz;#cCfRbx@2W5*Rs^l$m!1Wlarn&-tjfNv+c`-tvlS0PCa&#rNMe88{fqz
zyt1m+z7BIW9VB)5?HlKu4}99Wha>a>$Jd)prrQn*`Z0dJc`#&xqx6&3)4mNGSKeWp
z(kRQpz}}-8;-YNhrYtzEL)PIai^J0|I~e2Y6hp;Ay5`6PU05pU)!6W-SyD(=O6%E#
zEl#efA~I#2GD49ak)g99=1!i-CiiOL{LZBGe~OL`bA2Yt&im7OW9Ho84z|S{4U#2o
z3oS1Exp6}0Y;54gG}T*obMhW7&){M`s<tBI>ea8EDLP*Nvt=$WJ+UnR-Q1@d^I8o1
z;y$Kqa#-&7?9sEAi<S${5Z1i$H1_hQ8Qm`yhGf2xd^y$a=-;bVOQyOQxK!z?zS<C6
z-RBl%bInm9)ltLYo$U-3pG3P;g8p}$(jLZ0zTjYg$|CM_K&&O%{FbA(kAum!Lu>^H
z&-gNkA7BY}c*vI1=r`qH_%5baqt=FP2SY66kL_R-U$OM~4kn)m$zAVUx(kk;Dros}
zaK+DF`O`78zin{7@b=p8SWc&D(f!|g3LROCe4ivlEZCL!g!SIT5|gtBW5XH563SKv
zX=^I%o7(1~=f`|%rsp^5O5X=d4z%sFO`4%5;e0@)ienaoNNQwh`|5xh|6g$&ciy8}
z6?ezO>W*(3clMc=k2fqhD)}_ru<!BK&Q~U?;yDpQW~>d9B);^jo+(srC_2D)g+<ci
zfIyonyTKWj2@SqE4$F2mvbebR9cYoAv$XSoviKb)OFKo$DGZVo9sGT&c3TgA6j4lQ
zQe5lyRBVD{(yWK;?5;S6-kzx9+3$OIgV!^eqJwK?ZeD$RYSPoh>xs`cx6Lj7@lc^H
zR^eOXP1yxmT93B53FzIO8+_(cURl!JiLX^BWe4Bj*mdY;-ZJg{T{EjaCNvp#^~Fqn
z+S0{dsiajk;n7Z~J3E^)Cx{j^<!qWTkIRpHv80ah&e*Lv+)osE8n?19)0L|7nOMG#
z?SDm2o3_H~>b&FZo7Ah%&04LRTxW8*^_iggzy4OIL{1+@&Kpi`*AiAHDaL(cIen<5
z_1+Q7ycfdzSR}6;vCLBy-;vX_?p1a~&qu}ECThAFE2rstw7D`Y$@a{R$Z6^MnS5qL
z(}W*QXL1f*zIrycLD$)CNnvWxg*CG$Zkl~RN9B45@3oM#X&0ok?#!Or#5-5&UPgwd
z*R<&~ZoSPciCQkm!ldC>&T@de>p^u&+@o*0H6n{EGrD)ioZk9HJaY$&q{Xy1hRM(R
z+TPrhoSg9P8Q*E1Rp;1v(rj53L)w-`g*_C#_V|M@qvRZoxJewH>r^{#sY-4+v~m^a
z7srRP9v8arxcoo0@J<5Pvv{^;@qB9Ht7fd-c0#{IUR&mwq}sDBKbdWh_}C}#Yp-eQ
zOLejDIHb|T6v6OXZN<qS1)V>CxNY=X_PggwXrXMzJH@Jwq$!EZGG?jWnj8Di$~8g4
zYtfpy|E(U}<#Nau+OT_8c0g@*?xR>n`Q<yLp7-vVt1qU$rR2g9#pOq04Ol-fW8lzB
zTHDs5cratb!Cg7+(UF%oM+9%#bCoM!{pnVNopTC=P6`<$ov!3~no$t{cB#7YBk|X|
zkNLD2PjNKp<{WT3@SHzcug>S7&8La)%aS`M91z(Mv(%{dEnA|=G^a+njJi{gJFV^?
ze$djDbzeeWUb6e+$E}WuP4fSLesw-`;ZWm+CXEeF_E(xTIvjs|bJkyTNFrjk{f5I5
zSHg4?&P&_~`<~(Hly<c5(b0mL(SH@iizL}xqZB48v6ZO4)DV?h=diLgK%9}yP5sCH
zJARtwx_XZ!<L>3X)UD0Dz4Z7@zd1(^Zqhzpv{x)j_+g4}p8L#qDUaRe95j2EnR1bb
z>D7%j0^)Oy>zf1zRc&}-H$!0Boe#PzIwxpI{yEg<#~_fms_~h6XX&FTDgX8}S{BQ1
ztm%qbtES&QVe==u^Cu6Sb5QHZ?PSsw;jr#Gq$P7wE8~z>kCW{cXBqBK6OZ`xy!_m7
zBkX#b=Px$llvVm07oEKS)$3luGq0_jTfW`=UlRB_BdLwC)R!riSLxGvC9Or0Ez_0n
z-OXF_`tHirp7%XGH-Aw{d&IW&^<J%cJNPzK&QN0A$~5g<ke~LvoK@}n<4)}ta^a}w
z%M{qkAW+J}llpj*693xc{(7bftQWO|PA}Rk8Y=#HQRJJlh*_og3`8d$s#_9Q$q`n$
zrO`I$zzqqOx;3uhN%76kmh<o08r*S4y4CP#$F)oUCT{vzy6LCsH`BG7ZXOb=n4kUC
zSw^_%+BGNJ=Ns*!-)mf8vh6u&72~3RWpmF>G3%5gS|<u!6^m}P>4up+xe+yKjgQK$
zs3)#w;eQi&rvzQUU+^Sb`hmvvT+PjGe&TX@EiXMjZ&Tj3>3`;rN7}a^X`kNoykx@)
z*>@ajc_NzS#s4Ol%1V{^@n!qy9cg;^II;ZMIrSG44(ZQ%{#<YAf=ybM(pn#c9)4Ny
zy>fo>`J1Z}iXNVm&96Rg?jWfDY1j9f{pF#*CST5C>{a8itv{-<q3LT^Q{TlVwFXw*
zE1PA+Ic$5J?6@s{WPQ`<I2c}%>*Rj-r_k%R&FeiXcg(%RE3dcu(UJA@Z@%|^n(zD8
zExXR-{~LXV@9VR~?lGK>%(-OYXz+pgc?F}jLH}#nBDpyK%RAVFD>-<7PRxI_UrSD4
z1{;IR3<iUrnP0BXDJnI&t5O=Yyj1O0X36Pk$(O^}<tl_E=6Bu_$kua|`S^eDib`wA
zma1!Awi6mfmfQ+jXZm8x(F<&4mtU36n{7CM`p5b6KT5uC@y=kpx@*O@lZLX_H|Jh-
z{@T@;wtb`Q<Bd(}y5ID+CAH^Amb_I{Fe<35nqgp6&A9Tf?~}~sx|7yzEA<FZZTU23
z2HVwwqT5l^r|2!nJuQ^?yKLR#U6IV@y624we^(W5_hi4_zG;ek@$D*qx$4Z2?WM-X
zCJ)`~j|i{((r&akIM_|&WNV`3;yD(@zdvbBS<_X$W3u&*+16IK8$~`W{_@KBThzB^
z?jl=*3b!jCwM?we2*mbC%C>Syx@(%W7h9y>^Gy5^e=SjF%c+1v)1Uo0@k3Ea>2ULw
zrw!Tv7k??a^RVL1%^#k(J0usB9N(_3mUUsq+(nNj{^9?a?fYH(<Z)U5=i8QXIGp<b
z<z)MZz;=Vl?mrJi$xOBWeMEE(=eJ+F%ezCPpXbMOzY}6P79^<66Lw#({lFr3qu^_D
zu4m#OfAu$L{phKD_5IfQ?7yx|E_|QEs3~&E?S-Ly_&2RJO=;6vZ7;l7KWV=8?rHjW
zGN#phQ+j+fwqx6kuuD6qt_|5CuDbkN*77R1C~d}FxoHJPS`TMyxMyy%-uGyZ=7WD$
znJnfy)qdqV?kx{KJ;|-hbeZF)^{#pi`?HYC{7Vx;qUW?|7MHzluWr+OaqVX7!|mNS
z8c%a(d|-Ii*-&hHdh7qH)gB*}^6NI2pA+2w>D>2P<M(fj*9)3SPjr~lBGY*>vHy$K
zvoG;26HDT!W&HT&u6AI1fJb1Oc#nwT+pe^4xrgfK-u%nO@Sg4P?)pU&eE%HZ-#giD
zLU7HahyM#-{qGn4*~{F*V7~Mghe}2OgHt1?q*#gafrkeh1#eH8l`(OFD+}jj6$T{+
z&ciJX5_wNFj1sjcY6Y$fxw(p0n{jQ%xfG7zWvAyD3V%9s(^&oR0t3ZuE+^X*B_~=&
zcIBvF3e)D)UD`2c)`wLMYuvVK2`o6WDv&kqV3V)Mx7XKs*S)*3>9N>5*>II@6CP|z
z?4G@&`u45Ho3C<b>q_sKlCb-GZhUmte@W%IH#-WGPE1w}ugi%voT<C*PwBVo>o@CP
znCDzACR_C-vwok?^0L_MD`J|_%Y0@y-8Or7du#b^F@w~@B6<f}^B*eAIbEoCa-x3x
zImd?2PcJSmF3PL5EYrItoFx80*5<Guk7eNdD@*_U_;{Cfg}T0+UHH$>JEtwIm*0P}
z>8I;E{acszGe}EqP-psSyh<y&X_Dp&=J(kb4z?)lIjP3zF>}HcrrBB(oEB%T<kDGx
z&Pc0~BT#7KLZLvP6-Na3201Ws-rLox8=f&`br0vMjHA4dqjnx$YjjFsaj$%=U=NSp
zn@x**Wjd$4c9YKTnRH5XZObPwY4OWKr*v0-De}@kdgT8kAKm<u8>ZGfM1S6FcC+U4
zS&N5rE^n@iy6I$g>x)R<_J@C-d_M2MC>s)Bzw*OmD?OoH&x<Wmt}ieCR!a4_=(V!A
z<l-lt(2&dT7illH`R1~<e<2U+2L7wTl0rs`LE=>>>>^rqR2MeNNv#NDJX7&Cv?p(=
z=;D}G-Ny@K{_ZYS>yBD9bzx)F!4skxtO_fa^amJuE}5`umBG@PS+@fAElXLIxi3u3
zWV^z=G^^>N1)r6h54KErv@M!((Jh6AtG1oZ*B1UL9LU6~mf>o6^3IO#ZnI~%7V$>&
zJ!|0E<xy~yPqA{(>K^MT#W!pFe(rj+ahsH|<&#OC$9jx**u+IXofet*-?=NyWA@Bv
zvvTEf)8+)PZtwLqo6)=dK@s<?9S?fc-tBnM7o69%bn;}rS1XoI<Fo!UZS$@@A2xj}
z$^O14kN?ej;i&WXH=VVUe*2GSkN<%!+YLe)IK)L(g)DwH?Lyu^wV2N6@<68Y_tSD#
zo?5N)FEzvUS3pdR&JostLMB$JWr7|%-)S_raGp3J@i^Mx-_fZu(%X_hcc@F$Hg#&-
zpZR1M`TWz@tFCReWjABj*LvTHeqU5B9V&Iy*uwg9q+k6j&3_dZlk+!k4m+`f_l0fy
z;@m2ywfbMG9QZ~38ua+y{=4L@n>r=vpP6p`AJ^Ru`+pm0mHrR;;m;dd7ZAPl{uMKi
zjsMpx{L54g3tg*qcwG{!PDCe5!xAl5#%xu=)Q4V*0y@*0)iw!qCAg>_&AM2#=BU!7
zoI|Q5)?9a9JQiM@lMz>8-R1GgTyMz_Ulq+e{Idck2}j>p?ez6V!^5aa7MdOhbi#is
zhBs*lZC=w+=q1tpc#1+}kK-~CX$9wKFN0R28%-?11&4L@)Lk#-E)a=VX_k#S*r+R^
zW~AW#SjzsG+myY5y?Wh`qjNU48s2D5TcdeaF8s38l@EdawJb|MZ!<7){>C=Z>G9;;
zw!<Ee3JzKv4fHG8_F>KoA7AU4d(M>{IDFErl-D+ad+PFIljVY%a&ta9rrwIEGz;G<
zq?+5kwDHg5|LavI6yz5)M?ZLR*jw~jubtLK`G#xH7Fcz?a8=Kmb5rQJs3<3c8`q9U
zuNqiX%zX|RY}a5;%yVSxxad)`aI?}gljGAm3LW)BAI+U|YKdRjp>uOTo|$BGJ<`|s
zW)n}81@GJ+kv3L0dHRnpKB@LaWbfTS&&+QGXM7Xu(r9<nwSTEBzwBJOU6<afMe`mt
z{dL<a*EdbkdA4!rDvwpGCQT~|*Svbwsl~;CVb{Z@`fsPMH4xHnoX{0|_L1l&mIZ6l
zZJ#V>>bU0IrWL{%!5uAa=*b?uM0AJgQ&o|sRUwTtC#|^FJ)>~5*8MG~1ePrMVCS_m
z!0|xV6t|bB<NrzpdSB@hQux1bYiHq92Jwh#GkPyx70aE$#3|Ee(7AD`sOpUu@m6kz
z6Fl2eI!z;`#jQ749{ZTmp8BRICy_fS<6(u_WgU6X!&2vO9Txw#!PWZARj!~3jUveh
zCX1*Yh`M*<m|52p-HS=lCQD3txBD4CF4nR%y`MLUJB7htV|!T4y1AF8Ec+vMGvQ{;
zbGz+1Tfc-eC8?S34V|jJZ;F3F+05AB%y2vZ%#&ggNz+oDE*IL`=s(`2)X2heW-p_&
zC)0@p?o*v7`ARniN}n)r?V7sKGjGDB>F;*?oj&-vY~A9trt0+FdUwSSrA;!J;E;ZP
z#TG@btrBr_{(R&=U$MjZ|AAzO-sKCTwq*YoS#slu#Fh!mH9Q_3^_<`&q@wWnro<Gp
z{R`vfSL}H#Yd&S+)F<<twKW;e+IX!IK6-xfj6Ey5(@GY1y>j~dBK+y^tyxRL#DXe|
ztPXrUvubI4*-A;Pi<cNyWht<3(PZvCy2eEP!&KoZ37b_G1xKb`aPhdoQF;8-eBO;m
zV!l`@ww7(0@^i&%1uaeO@GQ%#)2=wHc&44$x8;z))>+@%pG|$etZ_xdcdf#GoC050
z2pfp8<{V7!)eK6MoqcSV-L|Wm#mZ~cv+p0j)6~69O<D3T!(>*r?|KI=dd>QGP>R3(
zoA_$MwR1Bb?YS(p<*5CXZ7j=l!=>`07=26D=83G7&c7@+@%aD3C70NgJ)gY!mKkuF
z?`p$tS^uI}lW$*qQSMYJ=Q>y2Hg?DM<t|6_U;K-<TW>P)K=e}iD7iV#nXC)F9)0z8
zHji1ob4}vS1Kpv{@>2Fc_IN#%IUbj~ZF{FdRsPcRLJcf4WS*X^i3xU}!oauZv-j6I
z=`(Iwex9{G_@L$SpUt_s{!*^2%l?$kPn%>m_2|Lr$2=9BZF3HODVw+`eOBT|h9wJH
zC9YQ-GreZjdw+Jx+H%`{G3-g%^~=;5-)J7+JyFj2CSQt&l}X^`#wn36=LP!QwGEc|
zdga%e8m%1<^F0)f8cp-xr8T+g%92%5n;(VGde%B);o03%98XSfaG1Qw%VWi%KmVPi
zZd~Rpd;Qr>@l~LCMaZS)FO+S3I+m>W3SI3cT2TLb)%GZ-b<=vZRp0#5sypK4+;M>S
zQhj!Ni8^oi<Kv->o)Lm8XX<D9+sp6m(q7N9<;JF_lOM0HJofLK?&Mm_O&fKm?(WfM
z`SVO=X5hrNOLx_9whL`|v$`|=_~fu<wz*Q_vu_>0ZomBJ|FG-V%C;qN-(i|yk)xdX
zqvg&8-n(_Dx9fS{-NtPDrT5+@Hzv!4_xikdx^O-)o4{;+A&0?%%VWj#b*GsRw~F|*
zwnUwG^mwRsCVTBKS4JPcSw8EoFdevjE~#0<{ppj#+6wLow-y{KYJZlbUS85yCc$%1
zY0a#hx8ha*&s{TFc0EDsL(7|VH3udocJ6qUq6^+G8v+=%ec&!Fczx``>r)1A9=+b1
z&~nU$@8yHnd+!PGKJ?~{()M3){LQ6v$A75s-_<)IlXarYM{^d>d$pxC^LpO#e5|?V
ze#&lD;2U=jsfnlUw4ArSIi>M2<Oq-OVI9S?rJt2D8kNo~9P$y~n<K)hA>zNW>B*V1
z=guticw!gS|H`Um^&!@21?R=m&p%8vY%ROEddDJBG2Lk)=Q|==eolH{=%UhbPhZ#L
z9jhzTOaXQF{Yq|b?bjln{d#cXw*YraL(3+u=07#L9u^Ig5?24c#<jQZg!YBPzn3Qd
zz2(($!%KV1+5gW1<z8R>Vf6p)51X}@?Q(S<crkDaC>-=Pm*QmnnKFs*Q^2IdjN+H>
zZd!bI$6`h4PuBN(?^(Dqee`Ai-MiCkg0l4`=Etm$o+alw1+@e?vHX3ilC4=E(?31y
zVaqn9!bKBj`l!rgS;n*C<g)}l^`rgj`DZyb+mkAm@V4mfSn+{teE@g1(5!6%2~##{
zPB+qAIEj1tCidsvudjUI*(mg8n-Sk-gMBvy_zoNKJrCeI_0H_vJDrPuZ`V9Oerw&^
zMUSiR<tZ{xw7RCQ_(n)@u1?LYH#JsA^?Q9jSUf$&5^ONlX?o8YC$4oLjMkqCJ7s54
zxHQDesC21_!h1!9^E_IY4Lt=vTU{10^*`tL-!-yoRjAu$4Gun4CnZHEr>{Kjrm5#2
ztyuLnWJA8+iEldRF4lFbo_OVOfkm^;^FmnDr9Vq&R(#mgqvW#UZC6^arK0k6zqX`(
z8$aD~mpPlkw6DOWr%&smxAn}4+7G%V?`z3zRQk6mN$Xs&-8X;R-~F}^wQk=D?lgNV
zAKz`qx?$U!&32W^nLL{^dlqk(5O6T$+_6E8X{Q2H@9ONZ&K$Rs%F<UH?F|?gJy6MB
zKRxD><F{vd1sheEk8cT5V*TvAN9a*oT9QR<;9kEkkAtpAO0&D|NZ^#`dG_Y@JdHKa
zmmJZ*lb~U~T%-FOm$G_7%@@ttM%=<oJTotO>^9KK{^YUre~@SS2Gyu1+KWSUmS5q0
zJc0M%ChgTCZc`Gt^4{>hEzn)<?o-*r&z)MW^^||^<7$zO0w4WUbiCfPO{_KYa*t*`
zC9qjg-u1N1#50eUhyDr)KX%x}c(UT1GXZDc7#e;IIin!-m_5LFv#9W)fY#Q}Jn!vV
zE`ISlv@*reTzB2HHYWxr5wmo`kd0n3Y3pNaJln!D)UrJH#C&Q?SZ0xYku}id*Mt)?
z`KEjQT4Y5V*6%8m_FlGO(FReb{z*sHnz^}jo=EC+N(sNHBzyj%4dXG5C#$tLFePl7
zYO`^w&Be)UFDNOVT6FnImiK+*Csw)MPdCU<FO|P-YWKI0p)#54K&~DCf6s|+$`%SQ
z70#cvT=1B2aoXJm?;RSOvaC7{j4$1H%bdbztf=<IQU02;=OV2GM-u}@cV~Q@zGZb1
zd++XNheOO-roT&Z-MPpj{cFx5!<pL}+;nv`n*H1>Lp&CDa&MlXxp=)tsll%_Gtce8
znr8*m=T1s5-e%#Q@?_H#ul{Qplf!ga+r0Oh=w3Inxbn&3)+V3z?k9FU@cHU@>VSI9
zy^?qDL~^!Hv$8rGbfPRr@UuSixuEbYVM!;WnXE>}!JkS}8zlM-%_l}`tq(nw^Eu|#
zk^?#)6*e!~bX3FOYQdxg?u3lpDTQK;yx*EO$K@HD?d<x|(<Cn3r#P?arP&eDA8&Hz
z{Vxo67gOsOo^@b_QrnY1r`B#=Gl6^h3bpAc(r2#NbK!L4+^b=iPupno+LnKhDK(cn
zDqL|oxT5^J^<h)pBU^1RPmQ^LIp%8Ux_|oiAA_r27H^xvw*AZ2Obty3#w%94zAKt@
zI!@$H?Bz~6eZ6M=Q7yAXm(R@)#lC-Y`%wG1{E^u8Em5EHQ=CHb4;9Z|o$@!;bM+?A
zrA9oRBA%QxJd(pzy!7|3E7po#(%AYfQdy!=v7#wG)NAUurio$A`-_-nZR4}Is4DQ`
zci)l4{WNQF@`+bQH6JF~JqYo;%<h+Lp?QNncuG^<moKLx3T=OxL}+YYE*EO7-ftA_
z6}UG?xO)dr*Z&`Zwpq3c97ctz%eA(u#sz<!(6B3_-^tpgFxRPYdxUdxam*Ff?+t-*
zPm=8wRDS4OiTf6A*QO!nzO6ogdy}qiXNGuF?spl@i2m0%9#@tA{iIsH^nKLDk2Z-<
zbj<X`cyC56Tsfhz+%>yAR(WU4>8bLXH)FpBcK-NSxqLdq^6hcrD-|||PuXgIm*tA0
z|4v2rYf2AuvkP|I7x`uz)s%RC`n37oQ7i{k?x|^c38_B6K3!W+Jd#6v?+?2@5tqWY
zr1u8*6;DfP7MmH)m(h5waq-QWsume5BWG<do@MRV?=EI<CNambqIvrk@3TVj`$BYA
zUeooK2sl#q^;xt2_aeWe|F6A!_E!J5Q11C_jusc~9&9sW_b^u9oEx~*aJli+sHV=_
z%NLx^skP~f^q4Fv?5|=FYA3|d^69!b``Q=@@uL4c=JMigCn}~1`8qM_D!QE3)XA2V
zUDEGa-lu+J_THUKi%m<irTXt`+0=?_7tN}u-epr-UQr*uqVl^<^RG*1Oeg(ZUpcXV
z<?*X4?}y&Dv8-AyW#6A>?|RdId-!T&&m9IARtt+v`<$GdJ#)=V=IA`hXtk&{ep74x
zJ$gMS<QaVF)|?`$m!jsAdQoDUs==?h;Z-yFRr*3hehD2<k>8dgS+VT<jakCCA3tOE
znr_y7G;-7B4b8hnTDZA3d+s=Xt#Ypae~IkvVkdVf&0`Im7ax%`xkvACS<MQa_Mf?(
z{Sw=lMe~z?Z%zHZAdoX)az%dm?T+Wmg-UmCf9u?6I(6-_SzDI9+F=^CQ@-WVqqOh8
zqUs{QYQAw~P*^LQcVbfmhohL}1YO^f_sk{1BCg9ymp@;;c8X=caC+qBrl`}md+%1*
z@kC}^^_uwf_CCS%$De*r_#VCS=GFa%mXm%49=P~*lJ?F27Bg3^G><#J*5Z@tq<N3;
z{F|(JlXt4=Nr!`rS6j0xTPk`cSsEU_by#xC8uvMNPK#W;rd(K+wdUioT?&0Fu14#A
zvdxgZ{(ZqC`wXix=G%>rtDh!SZJhctJyN7;?v$Cm!n0TZ-&tQiVPp2KoI^KHs{eMM
zeQl2O46{XIS?04dZ$vKaS`+ky&v5bL!=bb19iL;%cTM2zw}jK*Y^9{^-XtsE-4hUc
zbbe;F&5f9CH}`P=5;=eH*f~)VyALtjf6uX><iMR)BgQ|sd#bhh-PVh$*Dsu&vE-^7
z!;0C82~$}$cb%xTvFF%*V(OmD_H$2!=9NsJ8R<GbYH#R@a;dA0*DrN>{BOIpqSkyu
zYV6(xQY$B}kC`sMYOZ+f=Czdz!&f!i+V?-pXiLtN+cT*un{n$bM%@4d!4m5h?ZdXJ
z&c~{zh0i&%*;Y}>L3_)A*hi}04&9xuziw*HOVu}3-Ts@L6IY04iB4m9Hgnd0$vrdw
z6kJH#(HgU?{?AhJ%9HWCW-pXHweiczy^}Vb3^H5x;Ehv7oMXj9^XSuSukl|D34C!W
za97Qi+g~1ObJZ>KnRnm)vF$xS+d1}U4(Q2?>X+|$5-3;mYR;2zIn_6Np7Ki>9ZYL*
zTI&CLo9Rns&SMP8|Bqcf!TO^4rD~T-(chy>PJ1npe0$MlOJe`_Wq~sLPX3<Y%qMqt
zw*8%n_Q#^5vupS4{@s87hjn$wKEav$=LYVVyqSLT-~Fkk2RQ%iSHJmI)$*OWWn}B!
zcL#anoqryj*}Q$;<Lx>&O1x!C4|_kb-*>XIaTa;HMZ~^FEbPhFSUa;%+sr4ZT=-R}
z!q8$g>yG*V`$?6V$E?kgKRVyFUgF508<PEgS#`{W>|cAAIT#;#b??3Hee=1q%m3W(
z{By70W_IbEdzJQ1ZQnkz%Qdsg?OW8@oWJG`%Y!+mg)-LE_|KWYdF!8mM?tMi&p0GX
zJboT|W_$e^%XtF3#Z>p5`EVp&<;1dce{0U4kN+E6cmDk(;g3dtZ(nm<cVlP9=9^1i
zMGMI;ab6MqPF{h9L*zu7!okKA9=;sa243yPT$Wp#Q#VXq8zwYKBV^G91@}fStFAjv
zX{XtOQkHN8Xa*&EOkh%NnIkFu%wZ;T{G%<J2OoG(SE|1!BNgoE&*)X9CA(`&S}>!~
zBqz}sUmgdp44j%Ywaesn$p7UotQr!lQ`SUo;#!p!db(`w?N!N(k4;r_Jk_FE{`t|}
z)!X&&?=Ls^5j>OlValNne*cGyW}Ft&o1pAH&u6BQy54ks=W9KZMv`aepDYkO(77ek
zZ=w5iJ>OGX4qRjM-af~7_qMmUcND)4<7oIc?b=#NyPVM13mqPFPgZunH|b~Y^K+87
z+H`GyIs|RCVl(#$OwgXjq|R~oZs5UbnnFSrYUlUu_1*aH^x^L_H~;%R|Hs?&_vioL
zSNBU-b;7*{Ca#Prjm&Zymz%gc)F(IdN~u0<V)I*YgoP<b;i#B&o&{@raE*p*heBP3
zQu1>{wF5~yM$eME4fbg)X3)FjaWsR;aoXem88%%l%Dv2gHB=_p@<pml^pHE5EV;N;
zanfWq%gAF6sWUEiGK9KmPLT8~OH7@>Tq?PYA$Hb9m6<6;JD*Nvs=1jqGj-F;Ws*!8
zCDBaL92fjtxkJ;VOOI_en!`{R<*u<Xm*v!i`D=AfrMs>Xd2+{bidM)3$E`OmUv#ZJ
zXC_y%ZEETiujqt~+kspQrQ&_p)J@F_P@JkN5W88<HGB2EMO9M*KV7^gU$46|YWBKC
zr&2>Uo;VizCNk}$$u*V>Zrr-;mSWGYZ*53jJw2h^@Y>qM9?oYoc0`M?NwW7BI=)Sr
z<GD;Sb3*CCo4dcf`l08w_@Op~*BVC2+x!0QQrnQjwXrh#zb3Qdor#B-402{4WOLlJ
z%408Qi$kubiphl~Oa*V|I-PF3v+=|&#+o}P)R+4h%@?$MWvJaMw`;l6wiPCE)jvIV
zxX$Bz!+yEK<mYqkn*Eo)ysl<sa-Dzimtk%3<^Qa#TQ4wu`BQw=rF-e?mKf&~3n~)a
zIli&TOgQVx8DkK6>{jZIKsV{sDFJzHvRNk{-VMsIaBWxQD!X^T_o&8>p625}A|_e>
zT2bNdrZjKIqz>n}Tm8C!IaFswaix5i>X&CX;o$V_o0iX(R7)DpO5%E=`aJKM8e@*I
z^sg5S<?G+9c-vbQ9{a9VPfNpj`m1AI$4ttvoMK~J$+BvB;Nc}l*cM;h`saV{ZmBJk
zzdAmyx_o(Cp}fvoZ^7RoKN4e?YKJ~q|8z=JK;e<Pkl0+-pv`Bl>E8>Hn05Bx&BE{T
zPLgU23mBLs4*ZuiaK7cGuuUvO;-E!<;j=8CotHvqa_!PKdV6RATWIsl3u~SkFXv3m
z4ojALz`#?sadq$Epc9AgepzGA(ZS%oX|cegnw$U`=_$E3X0s=(<S}1T;o*{@tkJrl
zAUJqMaK*1=ttFnh3c?DVofEDf(wcNU>x{62@O+LnacjH6C4{bt#eP-bbkxX6UVQD}
zg=J>jcE0V+)o$iyu{h3fEwHa*&SOT)CFc8nZggq*A|TE(^T;8=cN5xH2lkdt+2Qu6
zttjDtRm_1lm6OLz&nZj@xtr%?zN~KvUukAnu*T&QHQz2yulQM$81&L+t2|jLW-NSi
ziG=HHGmrN29X*j}GLD~?%3Bk(P1EDF)S)gHPL*J$74E#&tPAewyjRVPJQ*5yXzC5l
zZnNjfSJ$5K5DByloV{b`v(-_qIx&$?=P-Mo@_Cdb_R+>W>f$Z=tvoO6ZT|W9GyREY
z{A}q^_pW$;wlX7=dV=@?ktvNg9!*Q!_f@xO=IWU4!0=t)%Joj?Y%rd^^Ti&UVEwBd
zY4P&8zWW?BE;S0gSu)=^Ggn1TplxkHvF6*3=LueN({4R$T^+(9#4<ts<%J10inmkU
zYb-Xkx9GHU|J3|HpDR?r{q8x1bDd8eayGKQ-zd<1GdfDVe6sP<%B!oko}71iQ`weZ
z9iny-KB3La7Kv;uJ-ha@)tQ@@rfoCZ>dTNPa6sm+nY;Lu1?%glCMW!jIxaNh!TM%j
zorK?A?&cE~@>rj~-J!z0=;XJ+{5HmzF29r$=ah=vTrUOoo1ENnX;#^!i9Iq|ite*7
zGIb`*5x()%f&G;KtRTh3kC)6iClY<Fw8?KrC)3BU&=t$iom+Xb^3^+@r@LE4HY;xC
zjC0XTz49t$xi!<xVCxx{0v~+xm7|K|^X!^L^vd3Me+vwatbHc<I<&avY4U5|Z9+on
zSKsdwj%e6oaUfo=<UtOz=0S&;{{{(+GZGZo{R-?kU-;L(+B}b+PvN%O9<!4f7xg66
zR<wOA3Sg~@Sjgqu)1m%ihEVi}BVu<l+;&cwB{1p3Q9Zjvj{mNYZ^n2gu~ol3=wcMC
zUDh%+g1IPPXn|0dNPB?S-iJ9Y#Tt)gW}Y<rqpCgWjNFSWd8x;QJN?a%ZdSkd-qtpF
zg73aPt{YxG%S_|Tc0ZE2cgv$wt5VvO)<>isv$E%2moiyf`nA{cJ-d7r>h+wtm$_u<
zXu2F|bA6NecUQ*2dlNTK=v&OatHEr`$L%?Z!46C!0u3xI3cA(YU**-MJ%oBsxF|YJ
zOb>s%=(tXsTf25)mS*Rz$lJFjZaMdLZT8vh|F@U$?bye5Ni%S{n*Q9Co<6H`H#Wvf
zD{XgNRVBr-#d_27r)iN4+d@3H&h*x}_w|z1wS7<dT0K`h=i3-KOUPkT`|WvqW<J^N
zrTpIPsAvAf4L|p++cfXWlccGUM_J~+RbIa0j^;vZd++@prl>ctDm`EpzM;UL=YND(
zr=jh8{`@Z`(zm@1ni+Ml8J$pG5i{vQrawCa10w?yL%_bGDeQ`j;*;jJ@!v{p`RBUu
zIp>+UbI$AX9KXZ}FTKc`xg@D9^Ge*eUm97zOoNs&pLuC2cXq0map}}+Au3jRp8fJ-
z!HhRT6dl+OIh18>wMpFG7UIUYU|sO+#ixQ!<<y@LY_E#BdGWvBbftxg>X(G)=lQ;`
zvM=21aK}bYCn+?m^?Lt9*GJE`OggWSc*fl0)QJT~iww`p++g5*5!Ix(=Ir@Ze~jZ;
zmE8Bvic{b-=U5b|pRiZUtbbD9Von?T>zfj-oMt^USuT8_=-&J}-w&`md}#mwQl+CR
zzgFeNy(#^PPv%P%2!?!3kG#9+_z|f>zpP1pdBrO3NkxTRg=<yz^Plut7Pc%_{#))=
zJL%htR&L+fc|dK^+sAh!Zco2{PIhqu+ua8nrIjwYYUD6TJDUg^J3GI1U@LnRx`k2h
z$y31(pBrM1#${gfalRb)qS=oBMp#O!{o5-FoY&)5Ul-&{XH!dP5@9bCnV#@p`$iM{
zwkm$HW)ZhUIrYFMMc-7%qPb!W>erifzc*`7FLJPGv0qViNWI20K)lqj*z9^s^}>ML
zNfO#l687y$VKb6GZVEKm)@u08G3G~WoOoMkd2-^3x^0aHk=wQ6ugghIG}L93_7as<
zDNcE1CR5_!92nMq+@<{bwv-(kV_aM6L(;g5kGnRS3otEm%~NBPW(X5K5qE!zYx0&f
zA4ieWw8{e3rVhjKtY+2nuWl8g87*e+ZP$bhA_`Y{6!yEhugPFzFbSXLW_F>edbSut
z^Y!W-KdN_F)VQDM-gl$>zzhaY1=c#1tf{4u=co6~I;u2#YvkqcJtm*B)-BCu`dI(}
zYkT%?w_e7}a&uIyPMhVt3Xfh9%(g<ruhm12$1r#E#@xHhtgAM8-}}~n_=?a$VH@Sk
zQh(f~SEV#=UuZKqBmbY6*lSb2N#dQ)F588F%uiMndvj%i0*9`MgPF(%`;<#n9LvJf
z91`>znpn&V#ViXIUN)`h$oTW5>;K}y?8AjQ!jel5C2~z*(T^;$(O|I2=-#Fw;M7s%
z#=+p$!C-xJvi;5?PKGJE;!_T#w>WF|c+E&ODc3dqp6I`$xLQ!JJFxZHB0Z<$%C6g|
zN~`F59Cu9cl=2b}%2_Vu5L7z(acQhVY4*x#LE;81i={I=>P$inVz;;LF%)pPW-K)^
zEaY1If3I)OQEf)~>8a-$`&(|xcu$StGIR+DwSMKn#jGUszoPf-myWH0GpEn!OcoB$
zK9atWqpHEIYt@$mHbw{W$ijA2ftIIL-K^C$qSZb^-P@o0%}<-WTBv8&&+a`tXYb>z
zVOqqpXj;wWqcvyBBZE_v8kB3VlzUubw_H3?;8NzyrK)wi6YK7`2l+8YzdxRAcr=Gi
zaQY5YwSBI0H!6qZm#4m1Y2@bG&*WeuxX^Izg^=S11w<NSH$0i~^r`DE#m2V>MHy!K
zMNOO#5Loa%HU5K(+25;-Q>JxE?3}oUb6)hKrWMzlL{BwIR24FrG|Q?>eL9k;tUh^1
z<z!vOGTA^wPfdaUYFZ4g5mQ`lEYi1X(b_&)yM2nsjwwDf66Kz?)O~d5`53^TniTFK
z8F_N5q^PtMOPiNe@+w9BB^RfuxU|h{p3WP*RIjykr{mJCj493P0)h=JLI+~EPHZpP
zVN@nDV@6WD<J1}X+r1kvm2Fp)O;;D#_Q1eeFnvmg^#2p(u~8GEOr)(HD+QA(g(kRF
zO*1Qh+Evh^+UVA`e0liN%!$i6np;{UCZ@S>a?hOdEdGzGE^GPhIc}3JIy;u!S{{+S
zs>xj1_VVn5?F{ZOR_)<f&7iP)lh%}dlA^~fB7+s?oSkmIXH`w}#2%GHk!Nqrxw_n=
zJy__Xq`|ca*IPf<JZPWm-(>ke>yz5$EUU*m>+W-S<rYb=TO$4HhR|CF@5iUq-)Q9K
z-0FL?GWVBx|1_f+_ZY1|p3n%I5H@3{(A}x)WgpHjx@h-xsqnoojpf&73RGsMUtS;>
z)wISkLFm@nXrW2!GZHj97K(;X%3(JDzu8RwIO`oJaZ8OwW|@;kofk#8Y*NdrQ19;0
zw60LkUgYp>vNrqX&0bSXwWhe8oU9kM*-Cq}T_=Nw0Arv6W1vHi_lYUeWm~MPii14$
z_(TK4JM|_imb^A9k!TH^e=yJ@NZ%uC+Iq)n_D%+AH<u*+UXt^3TE6By?&hVL-qN<q
zmg+L5aA&9s8PwMsEvx@2V{0<QrPM_(Ip+WUZ!@?S`W#y<l=Nl!EoQMNF1A09%`7<D
zG2?jW3XZNt<~x5!By7D{wff~OHnWJ{t2>Kb`lhON3kR%74(#6LzG!pfuCv>AEt8U8
z^K15o+q2ibnjE!&HQ9h`%?8e7hn`(lO8cYc9N1m6Jk9)+q{aCgb1t4xKKW}NhoaC~
zDfwsOdv{KpW*a0m{qwZaou!)#)pm-u-PKmhi|&0^nOb^V=!3<KU*-a=iT$ru?_=Sx
z{u#ml!&BqytNeeR>z{;*GHJzSg;wUfZn&(}#Bq9qtfc&^k|uGjNjzc>d5^n{cWjiE
zUSw^tXpz;bRkK&Ee6?!p#zkxg4@&14iX2{KXmv=%=aAh@hX0ELK*#O9Il#5}fTBzG
z7NawZy;c{Sb2zGeo8xS?#Pd+nbV2Dwf>N@LK`|#BS8ts5>CuwRlbl{TQrR7CT!l+@
zpUy4GbmiV5q*1b+%~7_dql`Il*}ca0sBbB$tDHTKIe*yY+~S$K^#a$KlKfxOma|s5
z&52ymADwykRr*4US!*I^$ttd7+%~H-tZ?eBjEURAH;J47TdKF{_sZq3Gi4X<T6}BQ
z(yCQl8!NUno}9Mk<jP;$LYaFu-CDKY>eL?h)w?oJEjhq<bm5`34qUsXv-V3Bi*2ge
zq_ak8Qr3}EO4>zx&ULQ2xO2~zZ+ot+?7iRKyX#`_5@+e@o6j6|%Sqj&^-ALG{}*1l
zmvhQrN#?dHh2Ct-&1%=Ek9OX?WZmpZu_3I*rGmoSjnYd#uHSL}9A{Sbt4#&sJ0}W<
zORtDrC{TVN?LvWeMpMy~^LY=>^VuX$Ow@f5D3*F~`l4S&p~5WNW-vG=teP|@SmBPK
zbLXn5Ii{kMPpW2J>{&it#Cfw#^=6g1n=Pz28?NSDv*48OnM=E+50?jSasNHVucP<@
zlU~@%ErLs@+W3~(@>+6!och7&h~iSi6^YwEFirE6ndW1&E&S6~yKCEG#jQOiFxWbC
zxw*;IEebhWx;=aORoTLnyK81NY0juU?VWK)sA;#)r>VjjM|NCxGLznr&Lk$dNU}>R
zQd&eh;s2WBv(~ytY<RhoVarO{3oEx;iY@4xz47wN?%GwGyG&Pe-JG=c<SxgX^L0+G
zpM7&dO-qnL>Z&(4w>AhJktp76b^2_%;(qNtwZVJNWvsanp>lounnN~*m!d2^ilUwz
zoO{!H?aW}Sd$0GdJ$&(t`M$5^^}Cpj<qq#VR^q~QQL1{=wcxFsTpP|!|9<Y{pNX0)
zuHW=+;&R+5x$8jaAv5N^ljOCEEIUl~%?~d6b-`i7D)$qMDkDwb+TM3Q=_ThQ_uW=N
z^zkAGhRw1%3_sr{?+rY};`hK{Hp`j|d~43|tv&Ey%>hMk&CAAfwpgVf=1q}MI;P~X
z>+sZxQ=P=GtT3IssaGJnL-N1p+#^1BuEbRyN%Jh#N}0Z8Ve-S*a-0Xe1Pwx6ri5%|
zJeKR_e4;nymFBU@haSHw>wE1f-C-%6GJ#?7Zbq(Ml{qOH3(BweMRnDjEvV_um}xz!
zw{7JNE_b!a>UP8Kb4;ry``nq5vr0euWcAF8Oc#qd8(4w_SfdQKgzmjK`^+<iw-+VW
zKFjaBIXm{$5}Vb{d8ah)9Mjr!%3w2RkU>wG;Attb)2nJvA3Z&1?b*|vD!uEMtvTbJ
z{nqgIb;-5Q+3QZ(yt=90clylQlV^6Z39j4u@YPF+{<@5H?*BB_KA7<)d*8K-XJ;81
zfAi!Lc3@kbKC6DttedivIAa$6(P>tbyeD!j>Az5_Zo$Fn{BJLqbS~O?@W#9xfwpw@
zw)W@Dh4(enFY4VsXd}tWY{)iCfno82cY+u0SFc<2(C&Rg_&fV$(`95fyID@L)ZVPl
z+oIIV673-K_?8Bz2UoO1k*7v6kNss6nbz`&#X6sK{a<f+xk=gn@8P)*FQ2)!=TY?S
zsM%MP)+Z_5=giIgw9Bn+hv9SsIeGD7Lu*wAb3K7%1%}Vv^^e4ky*id6s(9>)wb66$
zeTI|!V{<ePZuYk8lMYV0ek|_1(#EIty*uYfXKs{ymT{Red0u8`?7@tCUqxz9%Im&h
z+Q7MZ0c-RE&czEjD~-5jFJN7=t778a=fc9fOZZ<j^)akl^6kI0(H74S?+%#vtcbm}
z^6u*6FBR*8U#@!_xyp3zWzBtk@4r4|SG%OO*M8C7sU~W-E$3}y-M5p`dqRKx^%I|8
z?@pO$6msd~ozLDK+}-E7?lhe~=PzTG*q_?0Fzdo0jmfr`o3HyF+<E@%(mBE6lNU|<
zn|x#6v)p-8zWCn%-z)ty_MMF;YwCK=)d~z-3<NUYe+y!GuO#?j=fyv4j)zR59x%jD
zIZ=P<!1p3<`G>|nmqp~JwlF$&ZBz<BsqAv<;g|c$J?Hl*#P#a1sW3RU35z!|^9nsN
z+MsYyZ;ljK!3ow!&WubdWla%*kKOv1yh^{U5KKNgQNeefivokgQ5GrXoSdZ-|BIfO
zX^=g$hx7B2V{@2_wyiQ1&Tw7elpn_PYVvZ1Ag}Uwb0UKqu1wBVnQ}8?(IU=XHLo2S
z%wlV1Z%92cNjG>I%k-_P%L-;3a6H(ulT(Cchql6j;C;2_+y1=%u9DE#!kg`LGU##Q
zQ7J~38(UPpe9L6F=2Y!8+{kcfwt03O&m@D4shkWm+zO&hv#zd;+|TyF>RZ;W4Z<ut
zbQL>URD-7Y{Lp3nA@b0x!+(NN>Zxf@PR}+DKbP87^6K*1@Z)v0yN<q%+2*10IhH}P
zTl8qSN`Ksks&7KtyY&CZDc5|HdCC2vNxojP^XK2!|JQRH|Cp38dGf7?n(8c_Oqq_$
zEu0M0*#FDE$y8>S+;d?iS8>*1^;I_n798$!S<|tKJ$ch{F~+{Eqtlk1$Z%Y1<TK&p
z8lzht+Q*Gf+?d#`TjRoK$vR2mlNsYA2AzhzEF4KEmzzniwL9mO-LrR!S@g*z)3&iJ
zI>oZ<3%AdNq{&G=8g2{Q#riT9wolPGF=5G~^JYSu&-K|X4q~v;j^b>%(eOc-!NFfT
z<bngYu1SFN|1ZLgj{9w{oYxmt2;$^ivg6Aoh1S_hC;aT^GK2&=^OiRIZ~qZu6&yPE
z-SL+@9vo0=N|~y$Fp7Oshw#rU!TNjm9nkhy_%70v5tgJp%R?#UV>7>9toA8?uL)j@
z%5GVQI`yoND!LW6JokU(pJ|Q@lB7;8&p$1@J({h&Yo%JLztP0ST+^MprgA-Q&1~9|
zqmr%r?^;tgGv|v%-b}kcI7R4Zil^;JY+pPjc*}AIW(RKHEfTH_Wqp@4G`hD>xN+J>
zaax(#WxsX0sW-#&)}~I0U*_A$Qt>(0;^8FicNTLd2A>NFslTRYy|7trSJ=laS^dm7
z*)%J=<+JO;_Q%fE>s-(9_g2ciW!vMz;|@k;vc@wyJeewDG%Ki!ccXUD#iP926qv8+
z`ndLOSBk!+ov%FmOp&JolfY&JhutiTRNYOv_MFoB8nfp7QQ_#CkD9WNFB_hGuA*yp
zOkH<R!MOu}f()!Budg|yG2xPjZ}0!;i7DN_cHVmy-S9spRD3Hm$fW#k`E%Ke9)D|0
zzB4o}2x0U#+*=v!?<u>r@|pMCLm@Bkcg?Gk7hEK&_;!1L{<XNtVj8x7C%R|bB%I_g
ztBEa~oAI|@rfI+8&#7lu*!8Ru>s0<T``ntvMe}!cUbDJpD9pJ*{c{#qfY1rUc?JcI
z>VgWs_d*4?JY)$rKHSal>5+<h?vm!1?uZ1dLo-fJEl^gmd3epE?a;)hoc=))58Sdg
zZVa5mnNo98_~jZFp%V*z%<M{Lo@_Z`G-IJp5Cg}7th8w?norJjnmwK=V)kLqbB*Ix
zD<<)Yb#HThA8=ghkzxF$JH>JWdT;E4Z+mRsCbstf&j;xa47d2VT-&U`7PZ3BKa;)l
zUBrBz&?T}_zaIxF?GW7jbBU;u(5VisB>ruJN5$SwXn3<FL{KG^g|or2>FucwUB9q(
z1~zl<o=7_GG@)lxYpBqHH%Zf-SyUKPf+SeA&5rejow<~h;&Et(*Ur=aT{<mRKAww3
zo?1Wf?3&T-q-H3a>XSB)&2zV7+%k);douklS6*$7Td_n~i!q>IL8jvB*E`L7qbkp5
zemHpQQq5Ms-yMZEZ>*<nlvp&O?ab#n7mED17#un__rdxom6P%A|0*YOJh?Pg*7||3
zNR&urVNWAZs;lf(R=qiMR`l_In&roHLon=yl9qrKCy%-1rC$>N&!xOkl)N5fwJ|uV
zh==LIp%AlUT{ANrx$~T!CM?^cD$Lb3wMHxHRFhT6Lb0tKyMOi;&RcOs!)&Xi-Q%Q?
z_G?E!Dey+5tGrx)ro!K}Z0oArCmoVMINdXCw<Olu1cbfZDkw0qS5)Pdz)hPJkCg_S
zlHcA4^YP4_ba}y%$xB`?N!X-f>Cfg>*5~CQJaL2INx^TPHKAv`c)nYnQ|y~wD|q?Q
zB!jlVoQFxX1RB(0+}SiUf+ooZKGkb1*m3#Rxxzip&kV}N_uOB~wr%&~DJDx-?(vKK
zTzS3a-WMNMzXC3Xl>3?2zD?=A+SkKh^Im0s@p>!m4-Zz&dp-NP&F%xO-)&tY|BGxC
zb2z%%p7GB5J<nxd9G)@5aBhiioOtZSkPF8SFvXR`WuMk?-fHI2r&6FLrFe8(&B2MY
zZ#XT<a^1kq$?Cmq&b#TFUsO55XI*CB9>U<U;f98rV;8Tx+Vp<WME}2^f{zPaJ>tgu
zNU*-6;SO87MKA}SNb;F0%gxlg!>43j+;>@4u0OChqsGf>_OY0Qe}2qUd&ysafb01G
z6B|#~N;=y<(4N7Wx#U#c6rp;@Sqd{YIIT09sNHd>&BJWM>NEd7Kjt+r^7Ll9wq=)(
z!6PZrRzsI9a%(N_CY5>|+PltR!@`H3tDJtGp1Q?|DNrMW>B}3*<BtrY_s{v@_v*?D
zciya-J%|3Ayq9@8D_t&%DKO*FQWgP8%{w2`cG;Gi*6)7he=(QqE{~|i&X=q5_jy&n
zmwY~{_l(~cUEdRlI$^GlPR3Rp3%()M8Fghw=)Jc_zm@o3w%C16kIQ2Jvgh&IsD=7#
zoLihGEPAgb{wiWi*qj#b#llN8UQM_p{Ku|UL@xAuaE@Rwi+R<O%@Z5c-Yvau@m=+j
zfZjtd>%%*=3x8i!Wd5Sg?EA1~y2R0Kk-SV?TCH~_wqG&0a!4#m>T*!}o-1PMekbzF
zWR_iuNVkplJGJ$O@5>D~$8X%+7`lJexw`9ndX1*<WH@!;*y@)t%K0Kbtkz9ov+H@A
z3>7a<H(gZ1sPJO#+5~3J|1L*P@N7-yu4*ghTCcKm8b@*A6tAUkx%LSkzt&gC;Bm-z
zVvywgDP`wtP6oYbxEZlodRcM&t?oPiMj2P)PH$R%^v(UdJiD&e=}nE-`Y}f@_xA07
zok2T$om3Kcu%2>1_G#VWKTl2PSDp#J_xOzC)@3!^ce1&zSLHkEec!V^=4lp7?fUk-
zH&2y|6SR+n9m(sMV9aFST@hY){*~T`&zb>W#P;8MeD}mZzeTg1f1eV2u_-@Zf@>15
ztm;IWyPiR}j*9CtODujZSKywLxL{H92V;&KP7T}Rl}vBvt>@qT;UK$Q;vth?uPbM%
z{4L*AAM%Z}!ytMiheK0mchd~6ow^dtEG_?~S$X<dd>l?mH}^O@aJFbZwmQJka-dt-
z@<qWSO(j15627MmeNPSaWU_Zo&lc4Wax^HCD!bJguf2cy#jTQaq<#f#(%W%RT1ESt
z#stSztXq_<w3-zr%$lz4Y&`kGVu3)XSHIUgp5l!0Qk`%@MJ{TF43o~j2t`ec*<lA(
z>#b?^4KO)+c>j;)HTxfF>V2Ga<<)M}Jx)g$gwIzvoBvsAd2*81;w{!Tz23W)q)$@Q
zJ+LFNc&_0M7DtgI4qNs(-Pv<?wavNcBd4QXCUo*}_9*$j)D7EwB<%N;yeQGA=BeR^
z>#m+QjhQVK_n1SxL_Pi>m$LM<77gZ#DA&Z-{3(Ik(z^d|ORqMI?cDaK!>r+w>*LMa
z>R;{B{JEj4qH}tHHB+}@3eWUZ8-?W-;(a^zvE33_6tQAuhTGB?`_?fUh)(R>x@yA5
zZpW=oO=?Q547X>5DKPKT-es4(Qc&u!gLRMb6OI#}y;VGmCTq@NQE;$N>Fe@vFuKw5
zfx)AZ$76@)iAI+L%`z^{B?nqnTv$sSI;MC$t~$_Te1Lb)4$UOK<yEE=Ud|L^nI$|&
zORsC^dy84?l{OW~Y$`isIi<vN-OVP2BS!ltoYak;G3&}P6`Mo7E7o_dVCOiv^vDjC
z^EY>E)R@>O>a;a)zVd;4$r{-UiHjPxh#Sli%I!XM(CbvC%9?#iYi<Vo=Q;9e&XgeM
zz6_JnCBml<&OH{r_QnloFU=L+nM$VBE2k?p2d+51dyDsO9uLL`Yxc<;*{|YqK*Yy|
z$LElb%Q<Tw*CjU3W+*+C-0S^k?ZZ!M`Bv+mPMSK?m@}wH@5b+W213(*R80GDu_NRN
z|AT7&d6z|#vgbcYHhW~O->_=i^*v_sOD6rh*sCnap!q|dvBIwE;jvcFE>T7CzMq_n
zU(d?^qo47_&g3)e79p#tA=+wLi^_a9v~us5CS@^i)rM0W-7kOGbxUK;-3jNI-*E6b
z_xIe~(8GD+tBl9DlJnnM{F{6HTUbui$DClB;_>Imd5#hX-535{Qyh9!`g8-d1Y{0Q
z(E0B;vBp$n%~ZMK<$V`y4+<Dbtm)tg)m{^H^k;yPUVzcu*Zh+=T$rmdfgx#nMKZr~
z&iZAyl{GhtukF<C6%fqp<lJ)cBzL3J<Qa$Fnpmvc=+rV>=b-Wq*{|9jn++Tm&f)g%
z+?%9(HezXW^d6UkhmSu#thVUV8EgM#2ZA<da_Oda?Y?qp^T(h)D)M{RcsuG`ZjfO$
z_Px9(B-qL4@<E#;7N$omeb1abyr(-_j<LZfu+TK(^$A~=<@sKFPumzt#?0z<a9t|t
zdp-FIS8Pb9hq|A_*`z?<2p!*)ZvLn%SE5t4{l2(dZ-Va=?iEv|+`PQ@7vG+c5i>D#
zs$$j?&cz9#4^940e|W`Ba#3h$vU%piW6xz&f;v=Quv|+B<*8J;rVz?m{l@RZnYABv
zOifpJd=X*DxVvH5jB6RD=QC58GxuK0RlWSZ=KRMS-oHXTnpDDnrkp4iz24Syq9exV
z6tkt~Ey+0-JsxWv<kfKO-M}|p+H;}d$vMFx@q*XZI!dXmUd3A~HScoA##h}MH=2~T
zY-*S)_V1Esp`!SPo4$@xZf4Fh2PEdI+FC1?+MjBa*>AwR!}|0ZmD6TAhApS}Z@4nq
zoMrN{)3UCUFZnm04(h&~arV;OyFv55-kisJYyR6y`liymb)pQ<o;FIoyfpNdv2E~v
z9iM|HM-D6rK9UoD%ty{vb?=Ea|L46lT=!^|&sF2)H*a&s73<yjJhky|$5%)BH5sm#
z=7dBt>VJQ@cUS7!*eQ!MKfB(v*>}^V^R@`Xv&l`y(OjaM2E38y?j6}z%IVkIr9a_=
z{=A;;n!oM#2MC<7HR!0_yggaMUVEjXjN#P)!-YcUcjR3Av_|2J&hebs*qqY4MJcTj
zO073iTW?N{y|wl3t<>0{8+Y%WJ*{1PK2O%;;n%PqI`@9tc(Cp9XG@9e$_eN7;p=|w
zq4nxSwXY@X)o|Hp#}5gPKManZ$q8|t8!&+*P~cR+rl$e*cRMl!SGQ_TFq>dh^q5=W
zO^5uQC7hNERtRjK#WB0L<6^0D^Q;#FY)%4uKK<|1xM8CCbLH8o(^%`cR*P(0<9(^}
z%rpa&g<D;MoK~*A%60yZ)4a!*ICed`Bxoo7{&ghRCp(d_M4hDyn;za3i+jj*>ynV#
zEd$diX}73zrxKUd1~0D-UeOwScuBBB%$Z|XE@xi4_1rTl$ICKr)x7n*%L|;=N%0&N
z<B1PG@<=-_q=Vz4=T^ONn{DrflGBWRjqLPeTg>*G-kc`k*75VIu0hN_nL`$N8sVma
zk$WSuW3)T91pi0w|FvU<Sa0Zz1?C;Dk<N7wgsmkzla<o0?wEdw<CC%fvVgl@+rsYo
zp1(IY_TJqmD=y#lyZ7YQ-;_YM)Szp3Bcxh$dSgTVQZuhcdiDLkw)+kXpYgpYx$tjO
z{A0wzn?kO$#$0E$(W~se9=<tzg66Xq8{?**zE``qZ?cP2L##wbn_qqI$uFgzUsI(t
zE4H7KwNiZZtWJ9K#>NFqS<EXLX6NU%G9TV`xoZDDj%8~ecN}s&HRbJ6=Vi|YF3mZj
zZBrF<+V}L_RINGNT9)l7x&GPZ*21eddl;jnzTK24GiTs7mdMjCOtW$_dbp7F^5Ux*
zOP;=1^7TbuSd>EFtrcsdj?R&Oc;T|@z2Ijv<qD+qqCR^B?AdCvO5Vj)`F*U>2Uq^D
z2e~e;k@uK-C4C|P)gJjfi=%yPx81mNdnsS&)W4y<9{ZnbZ{Xs5QW<zham|WQ-v53x
zUvRE`=DXIjt1S6YVQ|;|DdOp_DQZcElbjU)2TNYM%6nNiqffzpSKpJ}ZK<vI?%v3a
z4cvEkzgcQfMys#fn_Id$2i4vjG0QnB_GaaY*xR|We5bEv%+2|_=5*lh9FwCtMo-^7
z{QBmxZusArd-W-Cm-wDB_Qt)Ka4-C|+BG-P&j}Y^Kj2$@(^S_k<aXGVUWs`3uiUL~
zJ3iYk=QWMk^p@k>)M*~Kn)<r#Tff`R5qNRAhTj^yiTsgQS7e7W{_y0^Y@MNHBk*d{
zRD(+)T{1T_w;W-b$vH2fcgCTM2X=k4yH>Yt%a3IyECLysn}S65y;%_Y;?KGKxw^OX
zQrEYoB`z!Y|B$IYk!gBH;ku5_z89TwAC@*3u$&K)diMVBzK7iG!94!A7Jto<efN+*
zKiFbvu*;UqT@EifJ(8y8&9kdbn!$I)Pi485pQrY+oqYzbTwg8keY0H9cIDHG<(GPn
zzWJpW)3VJs_pEY$))VRcgA&c<#tCcG0%LNQ%wRAt{w@ESyDMG9@As=|9H)-;N~Pqu
zyf)Ih*}70f_lNndz!WFX2GvPTOm8f9z2V4stIx3dwaC2BpVp*=K6`TD+misck|4V`
zj_z*`nw1>0D{=b%*>75GSX!+2wQF8tmm~jWT)6hM(DZt&TR2<Ib$*#NMvb%;aToHG
zO4)QhS5>7szva$+*ZoQNzw*|vtFr1mS1v4V^V}abVbeSgC(g=<SveBA%(J(z>g)>R
zD$ZkGnBDp&@2dWt2(2lTZ*JVKab<IoZ11^?JiV%O?N=WEaj)#UK!*QhrrDb=F`U<u
zJXYMjEu+}JfVDo6ZTXAx^6wSx-z&=tSlu5kP<^O>^h58z7j^6}`O+^t{&^@r?bhKX
zd)ByqYzhD9t)lnp<`MrX9gd-2=SJ8rN$XD$mQT5TCg$MIW`%7nKla|gz4u$>*^I-k
z%k=C5TQ~pfY?vvya)n%1L8aY4C-H43uYOu{|Lcw$Yt#6((gjvJtPX2<vPSEwK=9{w
zYhLr~4nMD)w$l^bGO=rB#9gcB9lXpNo~Qih^3ikWS`?;}s((NB%?9_Jjo~Hp19CQt
zcinXTz0tjTv%2_J{Wn|9Q$wD8c3=M{XYZT)SF0nR{XXXw9KG(TkAK`HwLjrPmRIK8
zdllN()zQZq8j$u*NlrOW>U{2<b4jxI-^x7a2(Rbj*=yPDp&r4$n(t}N{tFC`g>p0x
zdp>@{tsGNZC74inF5b3FL-bqao3;u1SFAmyH|j}xspJ}8xm9LU6XZGblJ`lQb3T66
z?S7^51<&gfIs5;;*#3Ryit<@S<!|P5>CcYUzg;JA@!yB~@9fj-TK>KNd_1x4{D-g4
zKh%GJDDeIC@~M~ivplqked#}OD=X_2jd`LI7&-K0SU4X1clwav*uu^$l%mn#<J``n
zD70k7g+)h=SS&MDZWKOrYmT+(R5_WdBHlb_*NPO)1(UsJuq75%-CTP5^qjdxLU$?+
zIvH4)7&KHkLKe0xnjN)OIl}wE<wZfFZnhDtuR5*?5Zrc2%R&2ESm^$=&=Xl&w>AZy
zpClUKzc9?m;nJnG(Ni=USSr82dVAVDUr4iYY0gBWG-1gjOgC=1R$R+|yeR$rHb;*x
zQBAYy#*>8(-28m-%z~RepVmlz&OA3GIycX`%k<6l*^%p~$^PE<_Rfy-)9ZSFZhv)V
zaR9%*ZP%2-Cro|n`TJxnC7)efp5N5>!)m5(=XT|5_iA^4fAipY_WhtAd;U++3{qP%
zeN*FxhQ?@?GvWqE?f(4^-tp%B`TzUt|K%P$+Q6^MGSOw?)vtohLA6U3II*c}Bsz&i
zMJ#lRozf`WlIgkdsAE*lAvf!7HLNp3d=-}@wX<)$q2#!B&Q-~%siIe}CdFU<G|g5`
zDaAXzSkYwq1SXNn044{Oh9i?$6djgK^i*OvFxg+NGF6B(Wo8RYP|Xh3q{R~sO6D-#
zsF;wJvp}<9R<C19U{vlQi{~Llt18uJS3cUgd|vIRmCqvUe_3kGW7&FBa?t`st7YlQ
zR$6MBeQsPYOOk@JGMCOw%6cg=r%38q;=(+oSBVL;cFkiDUSid{N_fMsRjW51veIS`
zzVd(8eaFwLHw2=DET;&qKX@%P*q5WwiEU#(R{)!*=%gz;t;!o)E_rFs4Oz=@Bu!h_
z*X-?=wSEulq!}i$xHbp`x~digh;ZaCWR+ffL+D!YXQhz!q0u*`7l-tCn!b<fy=o-c
zlU(_ATF4R^R>inF%QNCk5nRun?!L|IzBsbvTC382*-fQ4!xb)z6{WLmE}B!YB&Mw>
zYs=Sfn@$^^H!F@XIXovicjMDL(I@(I%Rc4n7`xvpTB7J)QoO`<`<+XPX|GJKTq!g+
zEqfR`_pZcK!Irqh@1?%+RqVcn^0mKi9>0I{*Fsr^Tg>mm18NVwn-$>7UfXifQS#>A
z?d~=Duglj5{rw-+*3xphcVU~7<T^z^nJk^DV$l<Vx^2UIzgEQRUwSE;^6%M=b2I(+
zwr=Ql5L3P4=wi51MJUXO^Twnow*yb7w|lOVj7;Bn!%I5n<_k;5e~<hGj%99aojU6T
z`~8J;v!zyjTzFCV{PWM{o0u}@t%=!x<J+zIWq(uecGpLJyZb%!)|XDVT^X5^&2Ig>
zG}+E-|DR9EQT0pbx-I{&vF&^P+!Y5xR{1t`+9qd*t_r)lD)Qv1&~JqgM9d208+|r}
z9S{<WtrHdOeW@GG`%0Q)`qec#OIcL4G(COXk|xhwGNJXsCr<w)jfPf6R)*)lIHQ=9
zMGZ=iifQeN32e}qd0KLfZb9S!o`^{^#QHp+YOb>sbbMntH7w<&KL2a($wCT^2j{q3
zi#PLXv$aiK-qI>+WVKjw`f@wh#<q%_BXgP#If<P+d(mj&Pl+tE$+K+#<)rPq*n74@
z*~5Wvl6gJLIl1;@bDmqIPwWfnD^cQ{>*3|=b7SX{ihYYu`XrsTNpL=2YvwRNux(S$
zjv#3*#wA{if*nq!&L{4E5}V2YPq!tw^1@@iCtf^-iG|X?mOheUcW?1<WQe+ONFpwn
zUw)oPam3Bol!zskO+LoDdnXIVZaKtuUv#~wO^`=$lp>?r#9;5JS3%QRI9BGKn%Gl(
zBryDu<aw)Wt%l!Pyd$laoj;ZEbk>?5v4Q`c$|s!SSQYTR`Q9upuNAHDGOKw*U(9u3
zzW8!yVXs(d$R8C`**~#b!7_92FPpb*f8a!u1{ReEVgf6I*fxbUtt(o&HYsa<w4zIM
zk+sK0zU>D&JXM!VpA_5Fl%>$iA-Y1ba$3uTDBaA&?3KPxr*Xw=tXo-@6wBnn@G4KE
zfn!0Q2Ft{TQ00ae4j09|cfTg}JY(HZW4Sc>x73V~BTS0&g{<oi2+v5~;$*d|qSGMC
zqvzDAmhd{;r5iFultbc@kC`;JZA@Dtm|b*b;Um+w&X|+xc9!a!&f29JDsW2`{L?VL
zcxg>~i;Zrc$7|`!yLd9RmHpCt%(#~5F3&iYcq>P@^?&rz$W4>Pe&=2>=sfvBz}))t
zPpP7Z$_*QGdvh|Y^XeD;mfvo=#O~%sRsYu3?cq`#ySMz&dyqBjU~7p>b8E0n73%?u
zuaAoS1T~f!Ufk3gxMiWN&%)JqpRIiuLyXs%c1@g<lM=t8CRlOX3U}Q%rw!#jC(d#V
zDQJ4UGGW=7;CTT(g3ohWrXK&&ryv_A7d7L@L;mmDZnq~3E;u$_+0u6X(aIvj?z!KS
zE^e7|toD=K>@(?m-+#2I(R{b@yTdkhDdC^JIVtiP50*dqeldTs-@6rmO_pl@5BzDB
z;MJqf*|dtaD6&zo=BilSSBJkI6YRDNb*y8#E;0L+h{~P2dFB02%`N^bcPlEJaB}+J
zm^AA|N~_sHPM-r$EhPL>yTj)`blerC$2r9`{Nq(8L;I8&flET;J}o-s`0Z-g@)KLN
zrXNnTdurvj>(1A>ydx7Cq-=bwUInkSYVumXy(!|zJj05u({xtyI~fSPpWaz?XZbqE
zXR}Ve*w|mm^Y(PZ<ftn~yC<Bji_GfiwOzdN<;30=m2I<y=jP;FFS+`1ZNc)eJHA=Z
zC%0IZYxY~8diE?m_bp$*tq{uvOm8nKa2QQ+evqP2dw<ik-@d0?D@qmTEGggH`MZ3&
zl=H^xB}-*F>ntK$I&Mn4Ue@vHS<$0)IWZ)C>3+?ki^pZ(e3)o|XP@%)4Kms`{&kVt
z{<~^CsyHS#`{U71#YVR-fBmGE70`Ul^O#k-#zGaX5*t>o7fn`1mvm39aNU_->sw~E
zbiutR6M||_G3>FLX1g+oRc%9~PVLL-*`?a+Q>QN%*7ESWBsX#Md0ofjRljH7lhavZ
zb-Lxsq-$GuJP27<rg$Ob@9*s*u~Rr&L_H&2Io>{FzAvyfdnZdL(~`5s%jDDq6Q-$g
z)G6n=NS?LY+;`BlW!0k0rU`vl|1I577rF1{+BNe!1j}=td*w{6)>?7Jdrn2i=0i8{
zRF)Zu2XB+le`b7@OMZg&<eBHX<ZpEc&d#`McPm#kGTK!yH^c96$d#(qwW~U3|2}%P
zsOY5Jc7x5kbF%*zn`=Fe71*?pH?K6=_|M&%9W}ps_uT09`NJ9~5HEIT`hPKpLyQT5
zI`gK<wumedUJ@{4&y~Zqjx$bv6z}s8ekFM^B*87L<y?<J*USLdM&8TOCag`34?-Ol
zL|7z*PnftmH%_K;PP6QJmQQ{?84~t0Ig@LTrZh<PJ$=ya>H7J=jPrjN@Q0o6df7SQ
z=L|tMNuefxp=Bq33fx@S;&w=7rifXro8!&NzOrJs`2UC8^iXYkD<DuOaOB!bE}zP|
z*L!dJga*u7IQP$6uYYIf|M6vYJ#lmP*+{1aE*?)7az4GeZ3_$c*&`ueUHlkk1wIH+
zoaYyya7%F=_u|G5Myt5pCUQ~#9pn-#+|TJudm1utY2>`9+(l|>kM_(F%(4}esFYZf
zI!phJsNGUI)@O066T`T~rmd;H_h|!b%-Kgt9&<h(n7IDx#Fu~F&VPw3P?)q)HTr|4
zw2@u(r>$Wd1>F}!NGrQ4oA@Pv^$l9;GpRI!{X6gViV4!!x3E{Qm~CM-{iaK}<vjV@
zKiKcYoY))KvFT2<wOohIt6sM))A#?Kt@k!w|E;oqXvf1p_m%xv8Mz!pR9KxNu0A%5
znjUb*lT)C~;_j~+eyN2q;WpFKGDKYN`K2FtkebQ!vge#*jY|Da*OOD9oXl-K)q3>w
zS}EIDiQ^|H+8<r=Z3d5%$@z1PbGxrT;s1AM{(m0X**{c-`EH(H$S<-k&{gv0uRZ>Y
zJ?^@vrd+9<G9^@YN9{v~v&SW>W=fvp^113YbLH)8Q)2(DoO|o;lz(q8`5&Bn=joJf
zIxJfh`0l=xy>B};m~o!pm$`llDZ6BDg-PBE_hfZiclwP8+inZF#6_)+TnX{BydCtW
z^LaXoxZFy%Teie?-V)Ka=^2eFS&j)V&r;LU9=jfW60pF-tW7H}?9|UB@1$F=d<CQ5
zKGNQ$w(joKw`-G9dU|r}uBCG-f9wtW_)+cC-7`jVkI(;+`z(9>bLi|`Nv|)e&x`C_
zGFv#MTRIs$(!ViI&u^99+1XyX#l5IE`KM?2ubIkL+k}2kUEcCv<iuW9?!61utK|IH
zOT}wGNza`jt}n@FAJy@nQDwKzlMs<7k26D)-Yreyi%Vh1h|=K^y5K70&@!jz*8Y`A
z9FvxC_-WKf_Hm}Mo-w@`QgO$4p;Sad22at9_jfMyrXB5kDe>LsHs7}gT}+AHi~Iw`
z5?xrkE?j&el-4FB7s8*!)9;iRwZ%U-aG@w)rdV9!l*D+|TE!U!jEsii*Ar8{8)wcq
zDZ;Ai!s>WJx+^^JLcDC!g?kwSa-X<1-;mregXP}O$oZw-vyQbi$Sn#Nbcs`PX<Bx4
zSH{E?p_FzBmFY5%-pHs$8JwJ_rQ~$x+#8u&bNI9z*PK3-<K$=6s_)~P`M)nE(Bpye
z!ey`S&db^~Z&{j;?x{yvUTo(s#;srYEb~@cpMdfv*JxuV_0|8<jHH6(U84)_7U!nU
zHo5i6<dK$2?BeYcopwa}nioyF*3etk@>t_+yGHNh8*_y=FAFL;%l`XovYlVH{XXRy
zza#gqJpO-mwtdj*$XDt?G7lW|&NxbF?6}6d^VtOn3z2t>DjrQL9DOP$&t^wgzR>zN
z>-1FDQwN1RoB}2PzP8uBwvw&?gT&nhCRu?umK^-1a`<103-41tHQAtFD~@d9ce9cV
zmRfntD>cOHteU|}^|i;RmEOJXnmP026|sVb$eXP#Jxl*}UcV!9H0s<!mkjS{wW+Nl
z-E;rvc5gkB8saJ%dbef!67GkpuN>91oa9>OE2V@R=FZofuA-$Ibw?mzN%pQ?nvXnP
zg*vYCFus^5<I)rr%9P=l;(IQ~%XP^z*Wew&E2CDe+#0v+*t6SCM`JHuy!mO?igU|e
z&Wv0A_TCAG#o3Ja^SGWr5X;LKdwg<*&*!Pf3a#!JE2Lj?k^ka*|6)M;<(PNhSg)H^
z-7me^F0{h^hwSPb6Bggtb6az(uf(?d^)^SUZr#4U<&xDd`PbKY=boHYe)C<;w#4o4
z=5OG*yi@8KBh!PxjHx^St#gW58+#@!Tdrl*!4ny~Siek)Yw}#tVi&_X??J=HN6W)x
zB5E>@7pVN`n_-*j%KQJ(g07bjeAym$pS#f))bD>NqxYYRkd=<>)f>XQ1pjRdj16#E
zBH%A-$2HmX>9Hn}{~c4lX=+O7UH2-CJb7qJZOPM_D|4J)oN_pjxM722PeEqQE4_1K
zN9Imt;aEL=>w-*fUL{WES&NqWEh?M(ATf85q|&0LAHoVx&f{Wp`jNTw66->x1H7EB
zO?^`*^SoLpw`$HJ*CQ2dsk!Tq1-$YI(~|x8Hl*U!GQIb<nMXOY*5A4GWL;xi`xf1O
z?{2O7baA6%p1k4X2W;sDazTcEw+m#RKfd>B>rq1)_r>WMx3~MMmu4uJv1J$LF0X3H
zyV3D@7jKb-S+?TzWX0?4trwzO4K}s>-}A0<%IvL&W`3MCcS~H6s=Hf|itD8`qb9Cb
zjo+S@Y*}5mPy0(<!k4fYd;IiGZ)%G@f35v4cV*a<{g)KX-rhRP%HORu#ZOI#|J{;{
zo+5&FI)DA9{F6&v^kluP<kN|%A;;2|1jp$d_uDKU$KJ^C?8GsyDOGpFPEEO)V$e2a
z>AOj)QEQ&wc`39Z;>6Yub2(NEZ1WJc>E?cw(aEK}$SGm!u0Kz?g^$eXyBvPefbYAi
zZTqI#e1_Nit}Z;L+2Z1OZOg>4EzVkASzqk3Rt0Xc4fx`2G@0SYiQqN4-c=tT-%)x|
z8Rh<I-m<>d=TCIc?OepQ>Cp$)$8Eeh4~oJhh2K4t(#!v^H{W=f&ts!^k1t(Qy87{p
zB5&aRX~mB|7KaH1MW;W@dv|rl?CUI>Li%|f14=`uzY9(Ost|nr*WUZTY>gu7w?>rf
zMdlYw|8?8lqVN49d5?nC)6<N4ShmzH(-wR#aKnpX@juZQK8DY1^jrE;TG+}GTZ3CF
z^HW|Vy-R&B(%8z=w_@Ey{pWLD^xs_2_3o2C`@-Hxv%pRHhNm<7j1&XKBL98M2>h`{
zR*+?a-pmPNniIJSt~PSzq{r5-VP9Pk8`d^+^%X9bRz+XWf*KPS*6nwv8MUlGFp=$f
z4*TPWa*u9utj=^=va(9E#eYkIQkjvHTvqtitxh7kN<v2iRahT3ZV&uFhx_`cRKfBh
z!Oz7?*N$Fy;|XXup)tcFL}8oL65S|+LW`wo-u&0Nq70)Jl&s*>xo5}c!Onl8!fn;M
zjZMn4C8ZxrYR5|$-|zalMLBSt%=1^R+U~N<(`MvYTzOHrA&z%v?%AEP;x#qCo0hRE
zP1aKq@9;aQ*RwwP^)bE10`YCPZrjXTy*;kz%%QFQhqsDExSXzhqS>A@u|6fSb%9RQ
z`<E95Ynz3=n@yg-$(6b~%U;*5{p=Eb`Pd_FZE5mv4wac1F4s3wzMCPQderQ5@FRYc
z>=5_6{>S&3T@^VNnwA>oHG!pb?hip1R_znp+2*jOb5CQkvg&@ic}`|9=P$nxGZ+28
zoEmuIYt{Pgac@2^IsCFAPP9}wcPC$d*xAg=VBPDVHY`_vafst+%#2-J?mU4THcX7U
zx<vm~#0m~K<Mpn=TW;kYy_NOFW0UU6ZA!5U-|mV9)>Y10!M9`M=4aX4l6;PDIq6}M
zG4<p7n_Kd|c5e|*zo4<M(CYr9pm*J-lUkWea~FR5qWHB$%e{2o?%>Jog;Rfo2(Nxo
zr~Wc)Yw^iy=PBuXuUAK`*83efz4WPX^}dD97JW5re$G?EE#&*!{_Hq(xwDMh;V}1#
z#FW-r`@Z;QnMwZzOa!+z?Jdf-v)uoryzcU{#Ls^BoVIle<$T(baj~{CHlr-<x_)}|
zx8M6}GJ<o2{@-@(pR_HAE0ckLLw0gv@PGfLJ?){F%BJ7_ImOA~=8v-Z{l*biwf&oQ
zt~hgFZQd~N<HA`?YyD+ecKmtCb-19){ntVzeWw)BD&`jg&ACd;Y)gL^ED>LH`}p6B
zYFcX+dwjlTt|iv0d&6pq^^E3=fhP_t)o=c?^qKGmb_?H2N{#Q$4U~3d^lX$kt|h4~
z{nUELQ+1&%pB+U>(pj&wic0F*+U-7C9N#Wku&dibeWr9#=eo$FzZD;!3RN}|<v8%>
z{J(vZ(@VEbQkr~sU-<NY2b1lE?i}2HqVID06@8me|7RCz?(vwmOvL$4LVi!`#!W8a
zV()(5x_2&6D)y(Cc8g)T<>&tln-6_@@+;^~=Ii`(i|c2_PsvKwxp3U_&*`00mxh>K
z-!^wY*SzCX=3N$Xkh6E-5U`lAz(J3dw<vH$zyYT=ZfUzGms&47c8ObZp19$;n4wSF
zwoT;3#Ki|#7<`l*BorGDaIl1ObqEAIoSDfHy-Q|>up^TSbEeuw%MZ(aW^+tjC-dXd
z(~FH7tp7x`98Rk-uI0G3$^AmOE#tQuRo95~(;POvWw|2Mur2K77Uqq|_F9GK1Z`z{
zpCtOAY<=9W3eI1mJm2%8!kGDmeRL8EAMdJT?#^|6_2J3kH5$fgJ~Km}hb^|r-gZXv
zbH?R4{?py|MwzCCEskkcJIc0&;ntFz>&tBaZ`kH=hso{gQ`?w?g2zWrvfQ5%bD-$y
z$&(iSe6hcZUSFP`KXYB}X)(s=B|bmp>h_d31|O;S*H5VZ-T1C`v-ym9+_k^&Kh0lh
zAFm+!Ip&M-0sCeqmE^8w0ht#H&1|of_OVI_1vs&3_MAV&ZFnZ&Ft4RZ#_Ii6BImR>
zvwb?{a^Qp0Y3(idv~t$&yp_qi=|_wY*V=<DhqR7Ka$j0K)vPE-*-+tY($W)QGRZxn
zW+us#4AsPoHz{aN{N<~lId{(4?K(T}T=Hhz!XkWH|L~SeXByYIX!sf&R5@&Je)|jW
z4zp(ER!i&GEqwk)-`)iKzi|6$e8F+D^x|DFcHB6!n{&yGg#TTfECnJ#p7Lu;n!RrB
zp3<yq(Xc+a(Sl>60>@4c(?i^fAq59nB6}xvcS$vcDz{m@e6+w%gNxH?LWQWNI%C%=
z&Xw`aOFI|D`wB5cE@+9>a-JHZHBDnlQ*73%v~sI!H`nmYetlSoeUqvVquZVAJ4O0?
zJ{~<-@#do5#+Vdwy_T%Zr5mb6oqaZ+{&sgm?9QSVZR6`Thk3kodGGb`UJ_C3-P{^m
zedFOZnYJB|&SY4>+4i_g`KU#IBlAx43r)<QA5V66`)M($Ry*>ErTIsR-EY&*^`*_u
zJ;uLBbIFa!2Huim{vkgFiu`Tf_wiZ1T)LlGK#?K3_tw>wvMm=Jd4uN${6Dm&{oSqC
z>#hG>wP}oC*}YEq;<D{W%dV_yeY@%GxAJ>scX=Zel-3&sDTMua7jE1%gXhk+gU3at
zo@(S$ulb-Ynttby@zW{7MP{n$!rM=4Zl813hLgAC{5eKm*)Qi^_3!R_C)OKM>Z!bT
zN|2ZJ-;%Byma^3f!H=I!xUyE!>4IWsUB7^GZ_(Km;gg)SJeMu$iS1Nd&a&xc_zJG3
zuBADwi~jwbTC^*Ufippf<Iln)T?c+;PZssy5@3DRbw}XlCRf*=_q*!dc79mT9WOF*
zQ|r6^1uO^lUsW<>xpMv9zi9_lT#x0w;hvGYZj$7h53h|jos>vTXg+wuC8t^Vbi$D!
z=dO$Y3k;gYYk2eWCQCmOS&}05=*FC`&4<L!ZLmC3=*P=ky5jYl0xQcoGs{jn%gwJ$
z&)F8u@A$ykM0H&|mzRi#aKFZ!-zG<Fm=`^A<he4RN%TcX?3P>26E^MUdfL~M$`Z}A
zWI^MJy(fAeC<%6+3hn>9wnITLD7?ohMET^J?*5XAfqgYB;)-|I8oD=mC;ebsdQ7)(
zV&~xrGk2+|8M$uqnc{L;_4*7?pRS{Rv({Xm^T9<u@Yl)WCN9~RUY5Hq%;mH1NxJ)5
zt5wc@WBRm5iMJ{pj=BWg?VG+khvQwrbZf(^i=mP;Z>?b|m>B)^m9ku7n){UFIxR7)
zqYg!ec~4WFkiD?~e-vwG=axlHd(&JSX5H<Hi;dc}&!_Hnw5Q;8>9eBqH=PxBW8zuR
z`fa*M<;}Hmr8yTJVxCBF{aWk%>C}A|^W+Wfnrl3Hd}laWTFmbBG+CrPWyV1-4(X|#
zZ_Je+uQ+Wcb!kekh)I~CWYHAn&9l}{X}7uJS9U=7@`o>L(jA$jMG7;$f4&G=?osW3
zq2q@MD=SA<@Z_kbGbgs1=}lcJdFhY_)6I}23%u(y6K99sH51rzwYS$};qnzcT+<mI
zo%dKHAT)`CQ{}j(UqSZNS5e2Wgqf!170uYZ`k6ua&fV8ci|0R)neXMU5yCcYgPqOX
zMPbt;A`X2Dnl)>#uDvP)pGn{*i%YfIZB^fQKgbMI4E0W1+jD*51tFHoODU(m*Il(q
z&DE~<GCa9HL3>}$yf7w{MlYtMrsz35E=+f4d*z=j*O`6in#!vwOp;%wg*N3J)&BON
z`!|b$-o^|4I%x~%M?G4;j?Hevsy~<S?|)@+ZO@a-4R^gxF^L<mXX`U<nkso?A6weN
zTYK0R%Pv{ZCM)Ljw{F7zxjTEOsi{vBQ)_KAG%?^2j7gBz`(de@`m%ZF=ect)e#v+u
z=ppzx`KF3fAcKrrhrj)rg_`e{E3z+=5{(L7sQ=2(iII^(@h1zb2m=#?4g&)NLj#io
zBLl~O238Ikj|~eBHgj-Ji#)NR$I*z%YoelQq0;O|abd0ojR}u6807^gdEGb{cCu`1
zz%}W`onqEuuVW60Y{+}CE0no#lhA_6%Nyo0HU1Lmn7qQR=`mZE(1eoApeAqDuG5uU
z5&~C;ZqAam2u%oK^gik;x;msmYZKFjRl!?BVsAC(u^tmOD9^jTy=dp5yV+%{Zm$p8
z$@HzO;L7pk%+8nYSZ(<jx*_?-%~jJcO|y+=x-sFBO<=;Y`O=Q^DWw~yUTB(S+keJp
z!qr#j*YU5*`&_lPVWM^6_GP+J39Fm;NB>T{y2_&P@zL8d=CN}uKRrKpTjajSL}771
zrZvSU&$W6>zP-P@{r~rMd%Y_^Jl|dZ^4{EFY4co`Bm8;?9{y?k$i1sD`NG-v-<Mvw
z?k4$hs$*MYYR-09_w8GqPUSnjEt$n2kYvIoDW2zaOpGZfNWevTqEoc%>>EMdp4+xG
zMd~i@5xm^J<w%HtF7w48kzS^o6&I2nMUS4+5)?AI>h3MNbLvFzm78YxuQs`PL&s9|
zs*0b5qvzAft0OcUrbb%1UDe9dw7i<oGnMn&%y`A0VY9LqMNXTUR<}eVk=55jpr2)<
z(Q~aRj(|yBsyB;gXRn)RbShnxjdj-C0<%{u!nUbCd9Z{(%<JXSBepX%i)Y1I35ll^
zIJGq~RitM%t(dneYt@={ue4V6Px~Kvs)5%sB%pz@?L=gTtn3%bW;s!jM}b;#ZIis#
zWE|qM)-#R1yt#WV^W`m<eu*pz3{SefBHDLvY4{qphn#tf6Sqo=MZcJp8n*N4FW2k4
zOFx?GITlN{L?k@wIrMDrXQh&-p;0|S+^NT-o++e@w0^sJXb+S7oC2n&*Q4kC?MvI}
zDD>}$OrebR_KnBSHp+ZDakf`uRq31=$K$G=#7=*3>dnzJ^D1AjJae?lC|_NmhG|FQ
zamO-`oY&{Ag{RDyW}NoW&6!<pMe$*#))!H&UouUEB&E`BEX+ym(75jry<mmXf^A!N
zc<CDK%Mj>e+Hls)F(=;o<J>hWt(SuquSrswn=bYL%hXAMXBR!07RSZ1G_d2difU+^
zaomN6eWsS7ldHem>dkIZwY>1)71xoU0d7Tol}Fh(YU*VK#Vrl?u;*KHQX^%xMuYto
z%M)E~0^g5iT<vwUn(#jGwAkw0>;KGNE}NFO>d(hB<-ZaNMH+Vru3R5>|5er6wEE2}
zZ%kensJZpp_0<l1o+?VNvsKT`a6icQ-mYMp@3cz?PHuAY-~4@Zae%KrXNa!vs>NM*
zT8gZC@1&M|wDMw*p1y-ssIQUVr-PZl=)k50Pn(*B-PvwBiSzAxs>i7lymwpEjC@^B
zh2pMLbB;Ydkh8H_b*;jU6rEz}Zk090#my()c|2Zyi-X_Pt|UI|zftpS#k3aDJ&7TX
zK9^@b(|O+M=ffwm(Af3m44sg>2Oph{Fq`-B#+&Y#l$)hV*EWZJXiwj#dCQ^c@ce%Q
zPwc&g11<}&tm&vd>)5sHgx@1$#m>ef0TG99T??p~?R09no$|*Cx<}H4ChX8~GhEc(
z^J1gu_Nyzj?1UZ!9DB0tOr)Q0a9~i(rq&&iSC&k1o$0Fe@r#*{QQwrgU+$?VzqI6D
zr9CBI@bXNHlim|fZ1O1z4mlQeU8(WtIq`1`L=>Ep_P9QMH$9c<-X0rQU+>7t!83kE
zEWM?m6%e>S<iVDC7F*rVg*skV*cx@hu|Ra<?Ki9X1XezF=J;usxzDif_@eonW*q(R
zKV77y-i*;}Yg^~8V;QR(p67%x{6DI4Xkt(dhpPu`<w-~0dEK%1nx^j&Iwf(~AXVXy
zSC@zAt$;Q=H{s*}&s4q3T+)v&7_XVQ{P2tj^#{BK+C4&D@`l@1^q0O_9{5H=)YI)s
z*puZ}$u^Q_ZY`eo=}AttH|yiz`L~WOczY%+cgy0sTNUkpZY=EeSbhGI#G|>)57p;w
zU&xzVHfhD07~Sk9lN0V97fpY!Da_G4<a6^_!L}`)MY)^ybhQY47TJ4tQvTw{G7ls=
zmalf;E9z}p^H}ck(x#rin0zZ8)}NcY)=8Kd*!=9`>`b|7v^GHb@deKPZjTo)>PZs*
ztUP<cnoQ}(|6Q}nLpMGT%Q<}e{!g)>{cD#MOn+{ZBXr&I4$HO6C2f1(#`afV@C_)Q
zJ$u#5Szb0WO9NaEm1}=oc)sJ6G)FUs(s^#lCEGbPr+GdL?@w9VBzJnz(Z39*>>IgE
z8q7mOm}SeS2&H|?7p;Edpu+8V!g0!eDb8*U5uK?U3zjaDJL3KP-qgD-YpzYNv3~1*
zaOwW}bAlLtNwc=!ur^@4;cU#e$|GaS3xmaiDjnS1N{&H?yKV&NXqfc*=IM#e)$+TT
zd?d5IXvvH@lcwr~U%#kSK4aI>WmYp!nn^29Uh0$j_KDAG*21IbZd7b(eDk~~u5Gb1
zhsp7ed?&U~n-v{;!+^^n>&5@G%{nO^kA(gCkFJ}zM&huB0_S3*t(SZx7f-4`88YpZ
z;I3)$lB!|b4xMWLBdOgV7~E>Q$UFAg(@XA_JZ;9^5iJ`emV^c_N_buVyz0l|V^fog
zm&iRili;^2ezo8`)AYG#v*pTF7R|cnxFY>-QCRt$>#cg#o3gKzD5-=i?^3*Sq<X@~
zOTiJL2cK8n{=KViUxwPHQ)T`VRq8DL*-K{4ThU_w-TOG-jq2c+Vf$PbuAk7+dUw&<
z7c<>nOsm>2<=64kz$E?CWO1&Z1m?wxze3I|=w2{&&l<^dckI^2yECdTUwcg=V4`x!
ze7C|C=O_9VvGo~g`CF!i&0BTC-LL4qjL_*{|1G`3-amM1xci|)ira-ICuXI44a?T<
zyvq?|`{mL7i_5a2HT>dI*&i8pG@UoJQuHgneyHhMuSTHZnk2ny6~E|nx)*K*YVj*2
zaPS@VZ2rCG>ie}fkFI)TSZ=5|*Q(p%XvPk${Px+iY-L1h>rYNxv*OH+8pnlR6O|Hk
zR9jbH_W1GnpV+Oo_0Jz(wEpgKvTDZ_n|EuiP3}GAO0k{ZzpmK)e)czy@XwECW|eVX
znZUq#<jY$V+uSnKBVQM8sBP={>-dPhAj_Ri%`nP!o9NZae}C;O2vy99Jho7OSxbsc
zvZVJ%UyssF?E4jtuyaOwct;%y`QOCqxjXL1vcJW*Vih_9&-`zdTstxN(F;}Yg*`m_
zhr7BKtbfS!-!pT<i`goCff^?J>dy<mdwJI+CQ$NwGE+sNgIkJQsFC6(g<Bg_Ld8PE
z#EQJHHUzUZ`l}j#Ol*u+6L9$~_dd|Ay`=F`vUakWJBPS`R;cEOa*tXTsa4Yyn_8?h
z+KS4KdA$7~tn*QM!WOQzCW_V<g(V;Bvu@>TohowGyzFME<-D!RlSQ5CCracMWwaIw
z6smjY7^w-Cs?XUX@@SLO;%4)t>E`Fn0tzm1+bbGqFlwk>Q{capmc1m_#-f~4(RUYX
zOlNyVzk7B?Mf<!-vh3*sOOyn-nkwfkuiUpxD^kQb`hZ;U2b29%OdkGE*5+upJDFBs
zay&<MM-J;kiDC_bE!*QX47s8f<eC=>pZgx@l2NnIq}1QT+l?bDMZM<Lw1hP;JhPli
zbIQd%4MlhT(Ayo+_279+Vugsjn<x84es)LeeIJ~(LWP(TBsI(mxIQaRo1t!c&8J~o
ziI{VM!`5DT=R($qLi@G`(<_Ct;YH6X`d*x1l{wInb;X#qF;w?@U*Qsy<ZEncsr_t0
zo%(K?Ii}@3pTZ=B!e`D1i+e1vF-=KBN#KnE*C`_|`$g(52d(nNMJj@$gkPE$G6~HJ
znCQN7q7-Lq=<{xkClh%jMY*4Oi*io7C+yHVRXA%qR}oXs>}jI*5dvQw1eX0To6PKJ
z&2(@w>%+<aF7WStFyZR~57$iJs!dXB%xvsMs(Zdq=~lPdCRQ0@IW?@)clE;@=EO>-
zhRRsUj#<}Z7Mf2x_|53vmyqrq9qQ@~QP=u76b&Ow1-3BReP!%iF<o1Bqof3*`kT)=
zJ?t{m1FLm}x(rrUJMFMIzbx@LQ`RiC?l?s@X2zL;3ppofGR$HSWh$H*q!Abu;dI<x
zBzB?xNp_((+cKRbCby(anwqILD|04?W9ihNqWR*YhrWv*N*B1lJ!AH_veQgSQHed#
z3nxoCT34AFY`xIqI@9O=RD*3FB_B_=Hen6s6_*h#axrVu_|cf`+LC-oB*dQGc9~%c
z<Nu1nN1F34D)!x4n8N=fB*~@m`o=z^$MfE;?0dsF-^#gPQNvEpEX|#<GdWE@ZRvd0
zB8A&i(u9K)Yg(jorztKupjc2Qt$4|_L``0*DBbi1-x2}Nszv59QI;v&Cr-aPYht9;
ztevwO7S3F>VCLJfAfuTAN~<QN2rXKtxmanHMA8IK=ElWL3l|3zPEz;s5VzEqYMZq$
z#NmjAfYr;%?22>DHRo_VTw-yQE4@SM>j!T8Kuxo-3RB7Y*&aESKbBT;EQ{pq*zG0|
zdtqvPq?YMs+w3mm3s0AE8fh(FBo{n&TFioJ&!_lJ@DNy`-ucqGljFMfxwfzk+v}w+
zavgir**w#{?SG<#zo5|84mH<J3F@BR(~<>NzqDExDNw5<F!z^+oL9!oECG`$fhnB=
zMzdzl-nEM9g8I5!t2X>vxpe6&C9au^PnoN&66~>>pnWBgsc;Sp;~e{$qK8feZaJ)E
zm@?;NCO5Nq-2P2D=Qu-Bz8eUo7v8B@>v*;IR#(GEmp-8xeGj<SecRdh<Z<fF%<!kH
z1X&l&Pf;yC>o#FBQ@gvNP>$GyNa3k#uCdv;vuPD6B|lbQBf$DrjjO6SO3P$*!o!K_
zO^dm>)wjRWSUYni*Xj*6Tw<cy3^NT@nO6x&RtrpBxp<q^qIkvC8&-)LzM4_1yh!5l
z1Xjj1_L*5yfoqr)Hg7KO`TO5#Gv~tzuT`VhZs4q+KIPCsYx8BXzL_iPC8u_=*C%pS
zhR?K_9K<!{)0_t`%QB<1&XjJ=;;Kr0SsCL!%_w<!*{*4kqSJ5Q?8|-G_ms<yCy*)1
zlS$f<ZEm6+PqNz6)-5kZXRMvzzhnXH>;pn>>THVyW>4tapk*yEH);om<0jU^jUBgV
z&04Ud|IlLb)q;K5YICi2u8$Jv$ljs6O0eqN2A0A(DIRTXiffkap1k6>hC`&la_cz-
zO+5~a>WowP_g&x<o|u%}RnKWye`=u=r(^x6?^2EncRMypU-6vl+~zVXY2NX`dFK!B
zxf?O>>}{D?X2I(R8(!Z`37EF`K3CD3$fC#pcXg<>r@pWf`13O9>8bfjE3`g$#WcBZ
zIkYjjIz=&0%)>y!rr0gK{m6RbmbPVrtWga?a+Yk<cP;kbxY1(P0g2rQxb`e&`?zv~
zHbdY9QQ<YBqBSCs1_I(W2SX1Ye11heQh-%*&cUb&2l?i3HXEw*bqX_>3LM~EYT>!&
zFVo@Bhx{fydVdcnP5r3;akAaIh3@BE3_ngUF7cS`t-Z|MM|0t#sVpDc(|;Yw_;n=5
z>S(st@-Q9a;;7~2yN*`8(n#s{<370U=Z*GXl1c}R+CN+|h+5g{zLB%T%2`8^Ve0{|
zpbtCBWDZ0s?&O)XxW-3da<#xj1%`<Uht^u1U=rjMwmI1HU*|-mzzLxkwQd`MiEBg!
za|EQ<9Gs!U&=Pa7aSms@wBDsdIu@S0Oja+^i`i^AJ7D$g$&nX#Jx)42F>#LS#5p^>
z3#v7PuVhG>x=lV*B69tc!8NHfH@eo|+aYsBTJu_TpPcx*Gd+E;f1I&uZY&GgClRsl
z-%dfJ=?d?qOzn?nHLI`x`l?CWBs@pVFu7~O|66=(84gSgI23i^T)T}x-=7nmb54Hi
z5s*8xlKtb#i982I*RcMtIT%nNAd_>5`R*$AxvT*TPH@&<U=?H)K69R5_axK93u|W`
zY}7gM>(D|g31Qh~lg&FfixwYd+jwe4&7})2hYfTtwJI)kPM6#tpuzX4oZ){(^nr_Q
zW!YQGw0#{Kr}nqo*spC5Se?W6aH^ZFK)Tn~Q19h8oLR$jrUm>BNm1Arv*+mDiA8xS
z0tJ-<$ze>{r&)8j{Zbv66pRG-ES&DMMzq#)a{Kc6KLV#1xk#>IxKQ$UN7Wq8-kyz$
zbFPc*Jus{1MEjeA0%vxz2r{xK-jHg%!4!D#XUs+SgEyEL-ss<RL-_0s85!2Tp5xOs
zxvI=mtt#xNESdZ_q36KKa^vh>%nwiLoIS<z@D$U-TTBjZhq8J$&D~|BBVdtzYDev<
zV=Hd6ZoH-bc5>CCHQGnl<Xosbr`5|AC}p8L{X@CS1JON~Vw0^Go;%x^<iX;4Q`Ys>
z9+~t1gB#qg?l~VVd(&}Q;IX~e61NJz*z2D<|MlN{UuSl_(zws{vF};5!0TP>UTV9q
zog|QQ;q2X>;!nS%#UG}9{cU~tktx#w#f2C5_qr$sDzGxxJbZsfV)B}kzk43eUv*+~
zFUvQZgK~a1nFJp(IXq%9yr5Ah!2I^1TAhH(zeg;Bhvv>=O=<|zU=c|_%(vBnf3L$O
zwu?(VW(C;UZT61paX+}kao#EWxF<`c=df;^u*RYJ+k!dUoG(XJ8TJ_JJeXAepz}yv
z-I3ZAQ*Te+YASp*jqhl#)zPzSIznZ)z17%~`bj!V`dG?=ZHapY(vAwQT_Cw~VUCYZ
zaA|Zq$41Uvff-$=_)7o#Uazn{(X4Z*SMHF2?8`4UH>czsj8wd-tjEBfdqW`L;#@m{
z*>x8L4m?_{Cm@(`QE1JHxjF(3w{JQqojPT;>#u|LzPGz}-+f|~eQLAqnk{|1jJ$8T
z$*oyl^ZMYqT`u=-9o=_(f8CqMhm_VVsAJl&Mpw1sy?8GN`-&?bYj3>h&sI@XHa;U}
z9`Zn9?{OKenhx2AUkbDLy?b_o@l$W?@14`G>by_YR(SK<<Xqd7dv7!z%6R1T>}OOm
z6cDukzv^+2!ow2Y1GTk}Wa=KW9DLX$`)JdvgM4=%u{FM$*n4orJk~nz4cb23mCNnF
zT;R8ieC<E$ji=mOcg8!R2S4q2`{|6{8`1w^YZlqQF^m`WvAUx^QAzFCn$Wn(J&e!y
zPJEENR@CnA)1$3ly!Np2Fm-yyZcW`M8|%OtaAs<zm2vEB;}G5Z!Dr>(^KJVO_dZhf
zEayb|EkSHaJ>QJ43l>yM3$4<2>FxZN+xcIw+S*ldTA*an9X`(WtQE57>vaWs=bR8&
zdve0w8}0Egr4%pBy7zMVy_XZ>1v>U#2ozux^*K~_=M~>ip)*BG4&S@IkN5M&zo*Rf
ze{nwKU#9E5Y`>`5e~q2FxBq=`vf5WS`Jt5Am1~)MU4E=xy4PFE?uhec-n(b5|2*8e
z=X`UL#Pwvh8*Bl$xvqB0?p$`|{;qjTC*8ZufB%`qd>Ms(-~LygeKAGg`Ri-1Z{Po1
zV)t>Eu)`lo&R0n<>ZQd${cCjbNUQOfVDZ6K(t7IxPJy`}*bEObC^{@Spu^0~XVMX%
zct|5)5r>PCf|5(8sHmLC0@nmJ!&%2VqD;H4YDKNdy13eX{d|UcZt-;`mI7%$vrMzk
zxpZ89<~Pr_R&UMs%V`(pxv)x@zS?|mMcCV{t(&f%3kwJejh>if>ZPE)&}=7{imF1+
zjaeyzajxE4Z=<&6-rQFz{ashxi0RceU1bIRLwZa*IUZb6-O#YFl~dmAj`)Uz12H?M
z6uznOPB}GMHMq~Fl56_e*|NJ!j5qY2y}<20-LZEU%gn2)a*O3W+Crp*a{g!CRxQ1r
z_vUUX>#uivqZeMA|25}kchu?3-G+f%QZ7BR37V!6sbnZ3@ieiFk$-PF`+3>jKN@cf
zW(g^0ojDM?n&r}>@Y5P+rs-C%TXR0B{#@hF_)k|n7yNk>oUv>g)0KXQ7Lhya?c!OY
zR?d(6wQz&@-?U8y%Q(2dbUHG!UU=y+U*JuaV*M-0W|5y+f)4v0uiNBnc2|HYZ{e<p
zgA=EQ?F(8lwQ&d2B=?qmmo}b|?bq4Ft32Ih)5)+|S3Y{G&9B)s#i;19OM}kl5DsJg
z{Vc{lhQ~!FGoP0^oo&+i$t&2T^_Iwr-b+_}-Wpmn7I*U8y7cIjyJGCGVr|A760JQ>
z=7wATyKPwarEvY-zV1_ED|{}wa6Q+%%c=ZkQ-jmxMiZwMiw#XASyd)(Th%&u@~@Tb
z>T^Aoa9;b*;^n0lohf@!l_fQlLvwn~-wC4CFJ2$t7v()i^ya#bOCQ2g3R_l9U7m9>
ze{JBIUnkw?pE<NmZO6W;+A&*7M8frUwcO@4YWXMBVR-1q-__h&B8NBYmq}-zId@`3
z#~Yt|<JZ%<&sr$|FmAab!tn4?hEv3rD>2Qzca3UxXK%ZCV3TiTNv*P<Vvek=hgS8J
z=!rYM=U+azt7%R3iC{)0M$^dwSEZNk{T7kfE*AQFV!kT3?Us{HjdCpo3X4p`I5JWa
zkM<vKZ7Dk`U*8uU@ZV`h&C2DPjBkD(pT{5hDP48(o?}<D1iJTmhR>eWn|XL1$CJ>^
z`mDsR^yNLydgg124)d;Qyivee&{%dyGdYY${7$}H<ju`fk4>%lRH!D|FQluvvPY=s
z`072Ac=WcWeE;0I<AKViOS0B;zw8fVfAd8zsQqrq&A|UgrDx8R^e|t!er0vJneeie
zXU)7XuI9B?44(dQ2g}pKEk`q%=1g&3s?HqPa%@ts*`s>l1T9bH+(HrW#me%EeKtZ1
zu7vui8eN@S6_{!NufVo!!LNzO{=fNs<=V@-^zM26-t$@}RB3c*eETr#x`*T1#dA(H
zM!$};-t+80(!*71|4z#{)NMVu=tkQAsGyG%ZKDo~tPyz?QqL5`R?DO(Hf?olj^ez$
z50`dJ>Z%wtd-5ce+*o&U?K|&jf)N7d#v4wGOiY;jkZC`UT0^Vil1{sgL6Tk{HthfL
ztXx>xSu;a<nglDmMc1P^iHl2hR!XE>o>FXecQb68wU#NJ=|p5rMHoMKYKPOO?voAY
zuC1AWlO@CMa(vSV3Dd2PXYCvRTu@AAG8GU!F|p%~zl!*y?v*QFI0$n^*ViQSd+l2x
zqO#guo%O;L(XA)sg1B$Dy!)}A`=g7?Z|`fD{%A17_cZaI7Hl}7qB>h*qHCb(72iiw
zOr#_tb&@$ZOXN<m6x-P2HZw}cReOttbViV;?&SZ?L1w*MEN@kuD`pmq^*PZZX>&EN
zY?bu$84VtOZ95rL4m1b9uUJ0)vF$RosxI&OECL$y1-hy_w5Hr&C$1i!cc}WJ*EtQh
zgM49@0?qD?OIbxtgcF{wuiKLpz5dPdgHv~`Z+I!~s@5HPpi9r&{%yu7dxa9Ej!Rvx
zT3cB^E(mg{|KX$?BkXX!W})&Vv!GUc*IUl@CR~#gLmI<4SA0H@Bvv=iwN<|)F=S!Y
zblrx)B%#BT;~$jHsJ`1B7yCAGAD7uR#Yv$Z>8^`x>zfR=@f~pw4C#%^{4#sjl&0x>
zF5VDly|l4$*Gl)}j7y6)UGi9+dMU$m@zg6Dt|@SeUdTzxp8a3t=W$^-Hr??5E00<^
zeVX{3LqYgTqO0wZ1{Pflv#AD+QCZz)K|eWG$VGDMdM8D7Glyz8%{A$?-=)?NWDu~u
zs<7QlbLA!3jU1EHW~b_0U$$+#Vz1-A1;-^qO1Ed;Y&M(w_%_dTO@FCZN4+?I=Sd&>
zeb;Ehb+6}pm8z~r$GL@GF3c*7ne?D~$?2o!MeqMUd+wz^;crvl{<==tw`JjeOsm(G
zK40SVM&~f2f?mbJbf>$bwaX`5_Nh?ZzgYd#JJHG8-gN5Jd)@KdAtdJf_p*71ly~U2
zmI;9}A~nBdSZ!raUHVz-n&}WaEx=OZ#MBMS+#U<g6<aN`EBoOrBGBQqTI#>Wr6~u_
zFWeHr+AXz{OE;rY%yD6W+9C&;`i=?Of{P4YTyJ!(TY04}_SswAA}s-dAnhETr&s+R
zy;(DfFZDP}Otfz47uyvj{N~|1g+96|XWV1zuc)*vStyvB;WoWdxV>&moYLDD+M!lD
zky758I&7hS+fx-nLp$<!IJ8}zC}q)gP?4|nMd-S1QpRpN*=w3PUMpX-GWt@q$ba4r
zwW|{p*KPbcH>YB^>)w+FS1P3zi+|S=S<&;RXZ=;LjZ5yD{M&NGY08~j`V*45^YU{i
z-(RQO`tFC;v#bj)Yu3C`mgig@<-)`?YsaU+s*e|^tzqH}OkhvUH0*l*W%ddop`*@O
zif{iLDljrF`XOm+@m2KhHs&U#fTb-gPq?Q{dMkFT>Vp2QK(2BX_WUU|n|B=Ly&^q1
zI+<hBwm(sy6ui=wYECM0)sA_{7e3+8rHY5MTT0(qf8KQ6%Z$BhspZ4sMeDD>a1~!)
z@qV?hx-6&n{Wo4#A63KTml^xKW|0t=;8|wzbzP#K<BY|J^2B-7`_*QLowlpr@@>9=
zRv-iG;<K*70hz8`T7CxuC%R`v%q!R=w(W;?NJI$tn#8w_=3!osbU8xiU36B6ayX>%
z!Zzre3nS;21)|zAlUgVAI9UgFtSxiOno_-9DQCuYr-|G%%on|tRa90!YG3!dYYy8}
zIi}8^AxpmTUrzcjd3^u%8_v;+(%H+NdR^c8^MdNxWee?>{hU%}_i3Je?aOfcU(7d8
zU+6Ud;w?X4WY#;A9V@bw%O+O&MzF4qciUd~b#;8)nqnXE6knI7i}Ib{*W1sU@gltS
z&8$N=p0B<7W9zPj=~o51-+Y{Sdxy56Q-^@_HiJj28-L7b%`odft-4Eb&Euf83l}b(
zwt>}pL#s81<E_Peqc<=!GC1ZfRynM_W@59&--XH@lg)g*^p+QjEo|frv+SIwDOSH>
zLTChgm_Y0E2Ycc!u&`WUnaFK9Z9%I>Lqp&Ofv_EY^@}G&S!~kSu_?5n>Cz|mR^f(c
z7dYx8SQ`)b$Df$BV8X29RRZ_2S^sakxTDcyLEt9Vsh8Hyy4BBdQXqBaX4Z*oCKq>B
z2J5gp_N_Unzu>dXw@o|eC+ojY64t)B&uxi`d&+wEH!Yqi>%F!J`RFkFUOBS(v&;O+
zYD=rvpEx-uW0Ur-3+umF&;9mTsHEuVU$6a4+y?qLcb&dq#eRA&`)OGY<+*Q*ro^)x
z;GVp^Q`7p|MnV2)Lq|c8rBQAhnm4*wiac1swNZfEO|Vb5V(0F~(?Sk*Npg4nPPEJ`
zZjCNzwJu;TFH&dJa1Z|>cZgf{au<JC!=^P6^8+3DXByg0Dq2x=*qCV*duZXFd%_HX
z92SboT9dRa-d<n|5@h`;!Kxt5sv&*iLXkpMgU$bw!VJCz9gWdOj3=7PU$a+9tPNek
z9@W6axPV=EicR%o8}>&_SQp8&Ixb-SG^;vcLF0>AGnA#LJ(zjqm&THt&b?84W>jb_
zteOy|Fn!eo-q(w_JW+OAzGUBv$zCrOpIWiz)T`G_?$$jzFS*x=H2B@%U2u7lwjlS1
z$*XqUkPBGc8I<h(PP0eVsHZ}61JiGwhJ*WeuQA=@qkWu#?|<~_Krdd+Po~#aC?%Yc
z+bh^{#Lz4uXxq-D138T%yDhnPEo?Or*|~7#%2}Ug@$(!MtZ6PbX+98g+`OXIx`5Ss
z!n9y5vF<s2FEctOxA1)tV2zrv&AfZb6l<=z2PMTgrNbmo)HV10zbUgau%kue14p1h
z^ScWy0+E~?3c6)81bn1eHFj)&wS&+1f`f0x_R3}(!yg(Gt54_@*pziN`7D^N``Ssj
zqb2l03kS!_TOx|>tND*s*{+S;!)YZLbV$`|l39F*!1jfl3o4fsD+h!xTIyjXHQj;l
ziNneT#R1;hRxUjOUp~ot-Er|+ePrS33*K`s@cli)?`rjZ(WJ8zmz}$~er|-)s>fER
zUMT8slr^|Gcg^gr3YT`BJ1|#2vZMIVu1YU=%a>O7R;q7)Y{36n*g=r1zU!FtM+sfU
z6&5#J#6GgN9I}?F+&r&p@4L>RXp6I#S$xe6n#~i0jbAif;NdmTU^P!*ZGYANf0khD
z#0yT3T~1bIOznTgT6RG2-Vz(%7khLrFtv9yg-+O5b>WasM(|pJ)>bRQv!|N^5Ae^s
z$nrKpu-f_j-q-9=JIsAgbWNJjuT|@zTVT`M(`0R=d%l}V>?M2P2`1kc!Cf(}O&9vK
zt2YNrDHvNX4Hn{9XUG|Sc4=)@m+z}h!ZW*$p3-=r&9%dE*76mtdq1izFY$`6KC-Yl
zY@xE->Y9CzS6)jxa!o<j>ZnHZx)xqJrqe+N%sw*LKS`Z_n>=On9xkIBO!Izpm&uxb
zN#4I^QTR@l)e1~K^^y`Qx32H`5+1ijeZLIH|L7STvY8Lgk+{BrTPQ(Xh@t!N6QzBH
z2afn``#<Bt>MxQVeFvw>zLjHBm0T{+Df#Anz^3--7e3Yz?)rbv{CzE#SFJx^y4UwD
z`~NipjR8AOR38la<>VVM^L0c!*Tr*w7cbXVg=k1LFF4UsF1=&I><Lj9{NBW@^b)mY
zlyJTvd{b|W&G!WpRHsOt?`Ao3*@iKK{j`nX32oi~OInw(8tr}Es;sPA_2K-6mnvly
zmQ#u>7H4wK>bfn)sd|d*!f7X+77M9Gjy9~J98W$NKdqQFTln;bAC~u4Z8;w5ve)B?
z$Cf+kTU|V-9O1nhbNa0dKWnT&sD5C`uFNJYwYk?ze|Cq;%#Cb08a5}Kx!84WcaQX1
z*E=_vZp$j&xodUf<Hi5~gn7+>?BZf}yYgXc)gsQCU0h4GF10@3>Qh~|wfWeql@mpn
zt#WTJFPnPl;O5IUUyiN)Ij^vU*_z?3`Hvvu2f6}Rw#9E~?JjN&>aaX)ymrot_I@9y
zsL0?~D<sd%KIdvXu}RwF&kkje*e>r>)~N=%^3F`(3)cRxy&YB1adGvfstbI984tLa
z1ik+TPkD1C^g#cVo(Ec6%{flAxaB%M7I_eFx_HHg_9ZL&qd2?f-kGpK#p~GZ6$@QD
zW~WSCEios(=F)zHZsEV{|F7!SiHu#g<XTSXBZaMDYxZ2&x57o^-L>^Cyc=|wjT=^f
zUX=8xFreTh_x+6PWv(5XKTH@ok`EZ{|9_W3yE@YT$L!NrUxx4fbKO#Jb#%4XUJjq6
zITQCPH1FQ9`pv0z$JX3<!poP^I3<0tbl3{t%^%z2Gg?0tF$Whghj*mtn=*bmy4N!1
z;H+P6GbOe!(~Q29B=)_d<;Eg`?mgR;q*+yhpG>&CM?>QF(hEj!A8;@(I9pbI=&_gT
zTa)v@T$>Cdo=kfY{JupXQ<|eHgM&@X^R30X7ZF#M%4!;(VCi|{IrpFq<G}<*%{!-l
z&2mthz@9jvM=Bs>*M!B4Qq!z;S{J1J(>Bpr>E>K><W#83mr3ayQ)3oYU*O!mXyM~@
zuh;3kM*~i$x?JSF+hn$DN%4{SD^81S-L>+x*6PVT_jHy1-<jFOyWygy!DQ*LlNv=k
zie+x>6}b1mTm4E!&*P=ncY3L<kx<<l>^AKmr|_aU)!1#f6uNbVI@GTo+bz82`<#u@
zy_#1X+`}uF83lrka+$Bh#LudFXnp#G-$bK}QYRW$w0>K`S5d92V7zy4aKfY|qSm3I
zQG9EgK8J99VlQjhv-mB$uD8?1O5=c|2iX^iz7k-av4DM@0P}N+TWd|zSk$iVn`UhB
zq_f}ho{3%N>>CDZv3EC`JX*z~e&7bTTHK=(ZEsF4+p<Px*Paze_S^{5+V|*;UU=cr
zBf56ii?7bxVK8sE!sD7)Z)3M)L$w1pV)tJ!+h266r-oO;@>p*D(VlWAvHzx)H{xDz
z4fxEvVDn48SrPLr^7>9jaOOBnw|LU;ddjpwhj-7_<qIZN?O?6e$TBWyzGUK9_GDY|
zC&%P%olU~$*$tgoPPD9fvE4azYEmTo+k}MGD_fWBy}x7Wgh@I6z8<2dC)@Z4ove;#
z{~yzTNvPwoRI^{9&9s7pF1;R$Pv*<HaO6#sI{m71@yyPHXO&MrTGIHVHCnWTznt?{
zwQwa9-`1ejQk6MoC%sNh4VYcsbM9!2<o4$$wtke{f9K8OQ+^e5xsMjgsJ~D!f5E|<
z#3#X_T;5r2`zAa__Qus!-+}{AUAP-7BGJ9>lje=4u46AWXEfX`y}0ewu8TKJ`_m?u
z{7L$sv`1Luzy6l&xao3}{#O?-)!LX5!?9L^`%NzEoPTlxUQ^e6IA-v83G3ev@dnLr
z<+3jCVV;@I`m^=*2`&qrmNoA>TAZ)WY%9{?xVMQ-OCdyz`S@aXhZxSP7guT?Z|}*t
zta!)CO?O)0gUi=e`Id2TGXFfuy~wy&_|&vRU!}s<`TY~v<ay!U!MN!U-mbIwcrj3X
zCzIRpV|IEMy*xj=rnP@F-FQlE*_*88r_|hf&+H30Gw;#azNB-<9$nxmcUhfuBjCt_
zV7o(VkIQ^>TN}d5OifdhlfPEynzudP{Al`}Yvp&-)Tg=5vY98bY4*gf&KvhRo*b1)
zk-jOBy0X=gXU)Wf5C11{3ohQsUcT!|_w&5)jOHs-eiS~QP*vQqF}3q$3sX<bljLV2
zCx81Zt2~@6Fm;w>>+X9-w_dTANr+DU$nj#zwvbNqXF4%<Isps%r`qxSnlJFNF~B%0
zalvJmj<-L<8fPdm9NEf~H2L5fyLZ{A*8ETBNDW&%{X%71jMV+Ax_2KpeoyBU{j^2?
zlm8u|g)hFVAHOJ)y8lGSr_(p?p4(8Qd;XK&_Kfa#pY)zvS+LKOy=!`3Ez@NE^&1M8
z_GZ3ZW^{>d&oq^#_ZoM7p5-=W&bJ+cS)!YcJ&gX#9<0Hvukp%Im)W?tOKa=dnC(*y
zTbkCtXfJDK`uFT)T)|?8r3n^aJ#~Hd{$J{G?p;OStea8CSN#lioGo<GaKD_2!1Af=
zfmN9k#fpAjOei_iBHyvn-@f$cs^ezgzB$Y7PY@RQ|4;P1!F&1o)c5~B$IG4!tIs}v
z>UzTDb8fvwPrqsJ%PIJtQ*`#t#eMbOGWBoSJ2aWt@&pgG@`yL5a>@BvTzKHz9xXJf
zQDeb^gPm;4Rh}FwR60IE&b>?LWns7H<e0E^I?4;q9-d*4d`aZNtA}U9Qp=`kI;Jf@
z->AYiX{ATo6o#N2xlN8<#V;LJFf9pMnpyH-MSCQ>)mq8T(`=*P=3HH^x<G8V;nzQ3
zUtjHCdws^gJ0*t6$6D^rU3=hL)a`G%`*#>-I(BZFs;m3oH`1t6(9~_CTG-!TSEZec
zH`z0;xxxC*bc2rVd8T_@0&n#{J6u`Jy7}8z1B2{?tCIQus2<qHP`#by4p;W+Y1-y%
zTefS4&RO2DfO+Q*kqb`~iy1E_`}O~w^R@K3;A~_4Q*%TP89lI^`p~7#rt0JK)0@?=
z?mP4S>r3yA%iqtlQ)m2f`i=kjl~3!>u%3`B{w*2drv69ML%x~Y>V`rKzn6q!D~DFc
zyac|rDh>@YRw9BfF@MqpJ7t<O1jJ<)oH*#BS$D+IL->dVtC!I&i=%A8Cw2(2YqM%7
z`54=(q)fQ5_eq=g#kWUfCaYd@@sbXhbm`QTI8I9e?Jc%bRHYYB*m!dK|CAinWeky;
zp{YJ8TavtWlWtz>VtBj7$V)eW?N+huZ+{+lNtVw1*d}@C>$mfB|J_-sY8$DraVkUG
zH=|P;vpObnE@Ardl;8I2h2)h>H}kUG&Yn~EEYx<+G^<z3;~KlNR%~gMy0LQS%vZM}
z^Z7zfPYKxfMYuSsYHDoJ0Xr3)vcpwZ13MIp*ylT)*1avb+4tKjfh||fyx(rU>@{6}
z@`uXXx;yT7Nv~_TH|ct7tF`Kjpr-R%wGZt6IIZ#BuCxos-}!xbmvU^ESe|v>7DXkI
zIA;#Y5J6V~#W{kTWI2x%c&T{G6nbdPHDT=$>pl`Be(TpMDSy?1hm%iSn)-1vgPW*o
zYS4f0P?wNXhWr0qIFos*=JV;Ex;tJGsauo9qOvxcnj|dfQftuvw<m}zE{$dTGTVJ3
zu_cT7E3GapPA)K$N|^ceYIwrT&c7e8hHhRKb<<Ea*KBRnqqP1tdw5e$JLGXBY_B^$
z(<XqES@5^v1Ljci9SO~s|Gcg2oW0$m`o+oX7Ek`U^6jjCH!=PGlNY56E!AgtN60nI
z_l{G1DLgCA>eX`X83}b;Hf~Auk=@v$^l<&>4#RhHCucnF-TV5-?{_jc`KqNWggqxe
z^yHGu^VJs@Rh=4=*10tN^5qoUZxNl6%cdtJUH#-7V#o4qR`S&N)8|WWS(f-Nc=hts
zp0p_+-9E1U6Qq&(fAvPGY@a{zZmXvKnXYx{XiDgopX_%Yv~8){(dJviEEQBzx@pGI
zzO=Z`bHzeUdhG_?R)#Mpd!KP|c&_4aRd0N$r0S_|x@UTnR!!ohm>p}xj%{c&+4GP~
zm&u^$?S}S;LE`?`mg?8+d=k^N+i#yx2v^<5_D=7RMZ%UQ`rqEP*myh5*|JDfxB2KB
zXW5G!;s%c<cJ~OX#4I>;<dTPBudkv??wux)TNXlUTV1>>gn4`_n5Mjw;7ng?;T4)?
zCc553RZVZR*Qq^UY~y>I(vw!Uo^LwVcX>yvZtmwU`<z1ge_Imuza@C9{(rQ3@lGwX
z4KsPyUP?2YrD2e_wbMJ|Nt*Eg*&N67s+#@o7ewmB|9zV;A(JSdS>$x!YJtOl$Maqy
z;qyMo7#Gb}o=_#DzTmj%3tOh34#UYaeHmNV&6*w@`|1$qxx!_9GdW)dwD&Dt*io~h
zls}Q9Cr|yn?h}u4<4>}7_1jy;Iy+R~$xKRkH{<wpkK-v%K6$wovD%slPN|9%5}LzS
zq}ouFDYE2++)T5;WxZ|&i|#2iYs+NVeYx7?H{B{^CCd!yXq}BbhZMUPCansJUuhD?
z=JV1yElX?tmQ5j?J=}M5O2uCBzEp2o;#)T1srjL&lhk~_Dll&Tyq!O^!182MpZL}n
zIvKg5(UDJHZROIk5KHs!PI#&qRm)+Zz3l&Kr!TXTcHhZTDnDr2X|r>B-Md}utG7;F
zU!&?D#qAclx^AO##+?(+EKY`|M$_CZvL3&`rLkDkW8u6*(VoZIq9<E#S|*x%;j!Nd
zfhnt(<fOWHUJ9CJw0SkhrmbQvlG*lV7tO9T>!{dzN8L~~-IkCR%9b-l|5VVbGd|09
z<Z+%^9cFnzP)WRC+rhKWWzOb|7tV^7HB3%_seJ93#&WebX0?jLt7YT<or*B6<f}UC
zeJD@XxR5>b{i7+F^NKvz?K@pEr}k~N-^HJs=i6s3t5uv5UMMtU4x3(jrNW`LH!m(p
z^vf#nVcFDL(Zu2=S*`QFVfn+Fdnd|X&ROAdw9Dqn)<(<!;aA>F7L&TDyHR3~$3waA
z7PnM8t)6P7R9;rD7nA%W=@qic@qw1((#zjEr!U{UC~x+L)skAGoN4UE9%t>gM1A%8
znymM50o&=<=7OKzDylk&`?^@@ui!es5+|qF<tihtdT>Ikc1&W$9UW(xZ9mTzZ`;hf
za_O}LU8lQz);toK*{rvX%gWQErcadB{fLH>U>8r3(vsh<89KA3YRKH|I4t8Pe3UC;
zWA++}-bWg(oEnasyk;C&;k|d`nQvT2c=yf@&ojC<uVosG$i~uBZAl^;e*-l&KbMBg
z__WdK-Wngv-x12$+A)*1CG#w1P)+D7^Yp3HnR;sJH|Kk10`V#aE0}lwKY6($S!T~S
zm6bbPKI(V#oQZo@G~;=w`JRo&>;LTwpZ)FI#l4QleSf{$eg8JgT$8_deQT@xxLMzS
zZ=7P0a9Q)BBd@-}5&k_^EO&3agzdF>!?S2nH~*7FgI8Np%Ud4uGUs2BHrv_z<7fQ)
zu8M7}q4VSpeoc^aU+<;8OwMPG_@mk3`C4K}SDdq$CDMP$XNiTMytd!1<U=pNZ2zBp
z)%nhr>0w)%L@z35xb7>xdO*`uO;vf#WES2<nW+MYp8tJU<aMo8fA#hVzI#gzZr82Z
zI6F2n=IxBIy}#CMnrHex^35uq-J-i~h4;l+Kg^Cf`B~pU-SgwA$x5*=UkUBGcbeCG
z&!4#ePnRz^{93f5`Cg*0)a{oS?R&oTh9>vj+QAsOc-E#R^A4>{*8g-p^_thV^_+8l
z+!DU{DXu_6`o@kxYte+qJx%LP_nl3$b~5Wxon%-1EpPX={*ZNVr^s*S-M(|}+npDS
z6mHh;nSOm=Ncr*Ie_B^({kZdC%jBsI*Zf<n_pF|CZ|T0-{~pKc@BQ8R{_pSb|NmHb
zJTLHj#&Cz>-1Pl3z5lVEcxDq^rE=o1XUq{-1(sEOZ7p(LJYsG+w_GD`yg$IfY7_RM
z;Ks>?vw9cYNxkXU(HqLO?G9`2=`*~Z>tg1f+d3od$T6O2CtVLp%t&dSX~Nno*s65*
z#w?#$&xuY8WxQf4y8NI14`Jp_)P2)>)v|TMk_(+j=H5O!m(8_ZG9#?GL;ik7!0t$Q
z(S5S4I~%3%I2KrRdw*=xUhq(NM&iPS-r_S3l+KvR^~*PS;apKGcZsM4^5<e%w?@id
z4V)vGwEl?Xyo)n;XecQ^3wR(pFY2lH0^dh$fs3|>C_R?FC15%6=}m9-X-!W*wo6~(
zZf;T1n5NV^gE3*J!m^oxUOjQgRj)}@N=diQO4&H?&C^G3*0M-R%J`nRwVE;6=ftga
z#-sjUTEsb7UFU?cth}bcDxEDLZK8DPn@&*B6&J-`CTWe?xqp3ZJ3KFC%(r*A?)>9e
z-kG2sFI&ov@)%Xgp6F2gb5gd_wEy9k|Gl=0F5GRI{rh6n-H7ml9ZYvCyrsoNeC{}E
zcli8PP5FCQJ}7hMPg(97l?5)J?$_VFeUS0SfxojEv!1l&O#a8JU?S_GsUuTsGRM;|
z#&gjV7O#XOp$m@bva<&?9!orOT-4>IW>{aztez_z9Ft_)=iU<WIIwc*te!6&?ROup
ziM?<-clw!)ZBt@o&;LBd8#Lwo(*@HFx`HP(1s~w_*&{LO;SB*P=}R+MOV@TXBuo$A
z7ju2#&A$sHf;ij6n{LW+9s9j;PKm<FvI(*!3ls|$C{F6}mfmqlw?{?t49lDej0HCs
zPrYfA?&;y&82k2wvNMxktXg|>?6SM6!rl_BlX-lc5Bz68DA1$4OX2g2o8fzBc3c&h
z|M7yB)jh3E*OfLM4?3mxWNG)_6a9Nr;v}Cu%iDJ4+23b-pQt_CIVFBBPnI70(F2RZ
zHNuh?vUshClkSz$J9O{$%S9qRF8cE>wsR!s+_~t{BzE#ZSg}a!_l05=KW<H#IE~eJ
z=48?Pfe$B!n#A`l^q0;#82I6c;j>BJAvc}fL@S;y4E}m#=Fyg4cLSZ8;x%tL7F;;)
z&~>44S4;e&8y?Rd=kI9Dc$=JhPrg1&V`}Z0wo~&rF1gOe+uuI%MgN|2hCk!G!k#?(
zF#Q<^2lJ`+7YYk3S7y|E_Az^%U|yEe^5%utKMilSh!;nnuoX$Yn{bak?f(o8y_J0)
z>MIw{3b1?0xv$UXg3HG%ybdc93{S<(;*DLG!a2uh>5NMW-wIq$y-Dbq=@}H_G9|*_
zwLvvP?WkSjyuYgMbF<uIHI`26h%arBm%nwt?Ow09DZ9VQ5y^Ek9w^TDz0+i1wye;o
z^VeOc;)aOH9BKxB856cNm2f`0n=|3W8Sa!x0rj^uXEH8woOAoe%>eH$98Nq+va&2)
zE>BiBrLBJut9(sz-JuxnnQAJ_M0rcO)#4Uym~}z$rkbW%;9ip`;f0Gdj9zP+X>YN6
zU1t*EsgeHD^@dJdkNY0Ml!F(gA3mD=YSEimb@v5<`dr)wT$~0)VkHuf<z)FJUqn51
z`R{qU;ka>Dr!kw5sg}^g35_+=j>c_qmiAy|pTcx<N_3l^%7Y2<F<&NGPx5g3p?qKF
zvEPdOJ}RD*7WCB3S^Dc{<3X+ivNmc)(;TG_`R{p_JpE^jw@8L&&I#Y1C8c-IdQ3Yc
zd&4PNEhWZF?Qq|!!(6ih7rk(~rQyo<)+{!~P44YJK^gZ$FU)H%dwyEc>hSh5ljc>n
zmo5929P^ZT>FaR(WQU)X*yJLKQ@p8Dv>d1W>-%rXQoe5CRG}w9W$o7rV|;8<r}kv}
z`lQB$ObLw>@%^&EDKkV#a&Dy15iuLBnia94bt@_~PdYzXn)>X*xo3-$yxgTfa6K-2
zGi!&_(|ax_nEoHnGdR7VPrYWveOZC57n~}K?T%c1_Bdgk&T_wtw_NqkA9D*+Ri2z8
z(D`g#+Pl2;Ls|!4$W9l!x4ub!_SaQWTc1VwPRzb`HQF~$pj0mZURwS;(XT?w|FkF-
zmd$G^m^X0(%S+wmFQ*>S?OR{6D{kdcxzbmQnD)%`n4#UZQ9Jj-EW<;uC(jMjG-*98
z5Win3<l|4S4#(cvTMkEFQ)l17-X_)eMrp&mdx?!)FQy-zaPrKVCokLYPS7d1aOv0H
zp5xORmmQM2x982NrZ?qs8NWqE&K^3y<xWe3*YW8aby)dZ4w|*!ouKz;hx~$fn(cZj
z9ZfHIyH}Kz96A4LodADfC!g+r#{arTrF?E`8~0V7xbW=ys~6q<zgAogTPqsQv0X)s
zy=jhrS>X+<m6K<!mEc~=@_@tMa0bH-$BwB1B{6HB`kk&8OXD#Tx%W*bJK}uz!k2uf
zW{4O~nX!O>PL;gZlo=1J0_C3-++|%gjYq4rfN@et!Q=)VhoV)j_a;6)Db;L{mF%|4
zZ$WhP(}2fZ&$>=VvHw+ly^Be!eqHCq56a)Y=3BJS+<D}B<Pjb5E7xx>TwkKOa@RQ?
z)yt_Lg;#IV>YCu#9TT~J$0|eh&BZcF>+WfNT(|7wyzUKRvp5siEc$zMe&9qWhKs_1
z>$l8Ovars!a9^#-^*rLNQuda><tGJG0;jE<x&CL{|J63#tC<*<9!a^w#Wf||Wr|$w
zt3z{g*2%V{3(n`~V%r#cL{W0mqD|=+j<)T;Cvo7IXKh|X^^@l(6qHvjlZ{*$F)jA^
z^{3v~A}>75%{g<+CCSS@G3dH?gVNm#?%FFhKE2vg7$M$#U`zR#Nq08g-Fstk!=_IT
zR;w8HIL_d8S@0ln=Rswq)9+R!Ej+=pQBd44B>YJ*^V<(PYY(Wt?T9JezwunFLWf9a
zN_hCjh<HQRu;c8nmN2xe6<;hP;hg#A>gIrJo0(?JX+PK^eWTIg;n~N9*UH5{aaX-`
zidE*l*0Or%EXBVuJ+_`KwjsQACnLS?mTYB`^l6NpFQK8HlM^G^+3^37|MW(w>%7G!
zA5OZw@|eV=R=hzgoISIA!mBK!;tr#1ZayD_XW91~B^O;^Ze^q~FR>ux(GSO#n7$h^
ze$SlC9v(jCI*rvk*i`hL*_p)Fz{K1WZ+y0wsCU14p!e?9gp6NVn}2AUw4ctN<^Ad0
zvd60TKAYu7fAua{w>>XOKfg|?W`0slb&_|aT%1;%c)=~_6&*f72dw<3^-YOdwN$@x
z@%kNp2Y$_3%jDr8v>-}KP{5$AC$Lv+X5pooE5xck<yS9z?({uBry;_l^=QN&AL|{5
z)N&j&8QK?rk_&jVsXXyjfyePz%X++huG?Qb>vQ`^#$}U}t3ICG)AQ_}ueY?i^sE2v
zzWnDL-nGcycq95Wxp7xh!1L}06FzWQzOi|?@Xe(8kN%atshc_dxZY~PjWhZ!XL7Ht
z7mZ;Mi}}Ct1>+?4@MMmN{;hK+mp1O4>{DepZ~B6H+e_aZ-70kA>wIR_?tqu8cg9TR
z@?+g!D0*&fmtly{luTZs&?WxA?)bIJg(siRVg14*q@8k9;L(il6B|W*7IrokNk)k-
zzt_@q?(C~+Gk(40z31~{b^52f7Ml_-wJRUqP<LUQ_Z($OmsRY|29hmr;<qVIa_IIJ
z@h^F?lwr@Oxk9IHS1stcB`JUHq4Nxe(33^mrq7&~Eh)>#BKysENBPdVRXdB4?|8*b
z+qL<~uI1Ld)&B<@s9!c%%<)d|xM8-3vHGKoI&+mr0vl)bY-hM7%2{(j`t?2y!Eb$^
z&AW{cPu^@{Juhu#$I6Kd8qRM%Dqo@Jb)e~gWm9%fnV6N|k*FU}LQjN*mTXv=!Yr*B
zabEFh_L(hvHgHY~*^vIALBz#=VMG9@>;7N%S}(X9H~F@_>3c4^X5$pcGX`gVS*$V2
zVn6uBd&@$<hBxcX-uXqm;!*$EvS^{NOJ(Dm-bGuNRvGFAd8IkWRr+X@RvcA5x2|@2
zd1J8q*3M`0p|f{}T(;dS`F7vS`InS*8TmNEo!^?D-TC11&Q}&J*Yz|5yjDJ%towMf
zR1CY-g2P*rd3SL!PFZJmGl=j1`vlHQGVybF?TniK<bm3g1fMSnyoXu>GeUkIS()O^
zc+zN<cEPvjN@)cyMUI_iPE!u$O}Vk|dX?7WU#m}lIPK-$qW5ajnV>8CLn5p6ik@&^
zc)sO6&$*~;E^WJ>1|6QwpKgEqXx`TUFISrJc>A<S_Wt+iurK*vlo)9)5o!MWboDXQ
zGd#BQ-e%YB|30%~`sqhY&HX)!W4>}blxP_TC>(4|_2M-NKIJ26&LitobK}Fq1MT9f
zMprrtl2-P~3zumgc6IP%3}*2OFmmnmnx>VwZHi=Z6VnW{=tol)I5Y{d+6(`gW6;VS
zxJYfHQf~64sVBpjqGWY~o4h3%RZm3N&Y1e2M=~haV<V@wgU;+Fswd}&PWYx1y*1<d
zK3z|*l~K{(a<;Bs`dxD+=aQV8Mb6h3GgvQW*|AGCLFI}y)3jZgQC-a9=1g+COe0V6
zrRm9f?LByB=@Z%TJhojzv4<3&tqhwu$z{%ksV^t0?s{<SY(!rE{i4Lr|L)HIQn2&r
zWF_~$kew&*1kZonc5YTx-s_N^DcASStuB9m|6n`+c~9$~Z=PJ79$r6BMf~%-yVD<D
zX`Nj2>(fW>HS;IfJN^I5_{aYM1E<V|15AQ80S8&QpB!1D+14d+XxfqmnQAkuUU@RA
zntVBVnAh>o2B$?^v@=(3_G9sDSQU}Ovu^V~rKO^hlar3BNPGW36v!Z3TJw6P%t@(M
z9z}D{Y>)Lf1cZ#F+C>sXR^()=o{E~6wUk%0NqS|E{Mw?WE7#~~XKss*$y~h3NUXS7
z#l&{<n`w<6t8^6&B{#p?B>Fn%yvii6qNfJ>Cqsh0Ctk_QS!Md&XN!v6T<w!RlZw1o
zwXCjKthK{-t&!sn57V<(RE$>%IRr0RGs`5{$=x=4r)TNzb=<;hnnlecm9OSr^!#|o
z$$C-A)2$kMRjl8xCv3KUbzQXAcbl!y?gfjEyxDbOYWJ2+(NhBLcJH{hDkm}9|Fqb_
zzhQR@PF^iq@N12exP9{j9p@#+XHBp3Ogt5FLg}B@Cy&K7KaXC&-}>w6iTK99|C2W^
z(VVs+;30SND^JyS%PpFn^Hn$;8W#vjyt=+P=#TDcGd|f4mPsa?Pie2e<<KJ{ebnjH
znk`3~EcJH8`JGy`&F9jY9XqmC9-mhA?UpygevgT7HcV8l*4TJ%%DT0hcK0=o22Cz7
zJ$Y_fLbb^D2L-7mJC^u6_i(@R$Q23N%Avb5`k0#UFVkq&h`i@JpHI8J!{c$9>8`!2
zUsvxkvU;F&`_z`pQ-u%hbgr>|cklMPk4I&{zuS9qvvTvN%(I(qtbdxB_kUhxQ@+9O
z@1NWJ`&)d{3l3fl&7b@AdaV23B^-upVwdt-zcDy+*ZxiI;e}=jf!Yt(sQnAtdiZTG
z&xW)dxm6FV{`+KU%H_&Mop^n7(QD6LFPe1L&0d<LeoABEE+bu|tvab2C*AS!F<hg&
zizkrLVd?H4n^&!qTDhwK{O_Dg5~6c9N}f^7GdP=iW~#)MBWGiGADo^4%iZSp(ya6)
z8x!t)N^@k-3uu--6J5QcY5AKMT`h-i-Qes^a*AIY5OJz#+C~=*-4AOxqN^SWOKvIM
zU-Y&hCTf-cj=B!{V^`c&ST&mkS(&WLWeYn!Y&@LUm|Xw6uqcb%obJA2g?>*+U_8TV
z4!4*_EAhQ5>gI;YN92x8?nv!W7v}Dg7JYDRo{wUy+RoshcM;o~6I5H0t0zvp6vVY*
zN~=zCWnfTAnaB1QXFQCh|N9;H3F6+v;H$5id6Y$`c-Hk8w@f8N*1s*ArGKyF&0Xx{
zB`M{(@?OJRqbrskmu!qASKF|e<<IWYvMt*mJnd87o+I}|f24W8+)*@R-KE|Khm@T^
z>P((__RqP;H$Lw;&B<Upf3bhftI57=C+L-J_?%{b@TYI#DLyCRQ*+;6IB8v4myu*A
zc~NXx!c_N}85XGwH+R%1E7w}_YhGl&WM96;cv|9m5C2<nr=NXV+~BlQ?D6U#?IjLu
zj@!Bo<Gj1B7c7<ed$ete>dRS+$|_g8DynAIL@CYx@?5}on#k0sRlQ+v3Z#rYmT4NU
zo$RqDSSVuV+eKYQQ}s??3*;2j58cG{Y|a0^s<}&!^@eS^G@J8S+xBlMtGRV}!niiB
zIwbZkD>Bw~T6lfvnnRh#Bkydy7J0qMxjSr2yG==U^kY{2(u3XWyqAQ>@E0xXa(NQR
za(ViJZ8uyGvb|X!CG%#^rA@6|Q$MO_Er|%Y<*~8zZ^|aF4G{r)*^vu180V?XT>fQ-
zP}GSx8?{m-Zbo`+5DQXBnefYpTib(i^NmSQtJPLYI%T&Rua~)0xAudiYV|d)u$%<$
z6<?&;sw>TYho<_;?L8H8=a9yeBc7QPq&U{Kg|FYB*fLGyZBTN`OZD&qm&r#>mgc`H
zn{)rg+5Dq9`385ZHKg*~+rG-Kj&MJ*<7h0O-Q9=p>%U9?Px!|3qnLHwvKK4|&R*8N
zQL<9{dGle(M2_Fft3O<cwKC*cRCX-zux#E*kq4F2-ppc|6m?MjoY1cGpMMoE%$j-c
zt4E>jWSK41{)wkGQl^NvJFzZ`X1JUi@N3%ECXt4USH5Z|Q)WMr+*xsACQn*`@dTTh
zr%m1j_e=>7Y159r$(!<F(gKzbGb$rrH9ikL?N(Zq_RK_UdG50Wwuf)h_@5nXyD6p3
zy?IvpTDI=<8ae&z!dNmA1-&EgEPS<Po~n~fmq2iS>6RTgR=KElxy1UJbO@cldZbw*
zuxswV@C3))8v)Hny5o*`oYFcpd(o1Kho|hF!PN92%i>(r=8ebJr0ITpXz^e5SYK8q
z*UEL?ZFa|0wV7X@3SVO~ZIgu1#4OJz1yVnA9(#OBcigN}*!z3imsyodmTd6~oukHZ
z#Dmq0pHVV<#-#OMziOT?bu!QNJ3Qmion?MpZ-jLZ+en_=kZX0UGog6O>D;Y?!81Pe
z_HB#4dhc6<dz8!u{aWs~KArwCMNIGBALZ}y{gYALvGntf7Du~>6Z-GipV$@KrarBR
zZ}ps~+T}kx0$d(Bty~++=6p)2{hE!GrpSW7doG@5JoTtiZsmfT-a?v(oE9&=(8?3^
zY37twA>Fz!ykbhq-u?TqZ4Sq;ww;Z$eCI|L9;jKJc|RmX^BT*B8OOGCm}GR7C*4a~
z-W(G7zt(^Ai5u)q!O@vnK_VgE%e%e@FHG~)Ta=TvrYBN6+%k0Kwx-)5o=!?@Uqzm;
z-L@)h|F6(LZBsv|-aS|2*q@QTcJIZ4+`uJUkAz)mUmd$PUOX-~{(9BPx~`VHtKY8I
zFbMhodV|Er*NfDy+&IH0fBoooiI4s3Z}xa5a?dE-c;=<1>x*e!DQ@phT{vLmvEqzb
zTI-p4=XTA%omOstOSN)RzQW7fS+AGM1!!`)^SRF5SR1XoFlkHZvAtp?^Vzbey%PF0
zM_<px^Yj}9t!>^~NuB1u@*kL-$W%FO;?mj3RIq1WcMgl*k{4=w!=A4Ar?$`L!UK`K
zeebj%N%^&Tv@ol@fB3(p)#eTNtOJKx6E8es=<#2mT4J@l_!7H)nD;%go+mW{PmXD(
z9_eKD?>k(h^i=3T(j*1W#~Pf%37q#e7?XIpdJ8nBCB3dVtYvk~$M22qCjX;)1?fp|
zJW_J1+m@wf9P#W?sgdcbS>{)(bt>pj#)>s<+_w@m&#5d+T6LQ3QNyVOes+$jUS4<d
z8twBGwaU((_EQegT4eU>h}o`$`v;x|-dP~DKlk-Dt@;|~vkDo{CLDTyspqVL$GJyu
z&U#!t>v8DSx}=9*9OwV^eERlARQ-KK(!|yS%<W>nUvf0Wg`Zxm35;pdkf_bL%;yx_
z(b3uEad}aP#Pp4l%U67J(Nzjclls5;#2uHnQu0bY22Hb0DEyf8Ub<Ib=AfWV_@e87
zbZ_u6=|**#_o%pWbWS)IpqIDoX3ye3ZkZxciIYA#W!!4yS+aolP}=&S^QUh0=JF)(
zN#ayo;H3DNcUsIzt$PoypE{T8wLm08lkxYHcTubMoLMmSP{0Ev4HeeD2OgW41=l_B
zD9U%)VEtxwOU$ItHLf!?^3<kl+G?oVF5-NAr`fI`<8e*U>^cLD#sH1vbl0v8Pp27h
z%)7ujb(7}IAg<{~4D$+{&nfj+D7$98JpShWvkOU;-#q76ozuMH^x}%s!kiY-4ebja
zT~wK(m(FrPOp5E(odV_;i!^`mpLw^yAm{K~Hz%Y2IS1!?9TbxiG7@1n=5h|Wcl4gb
zVu7_xwPkOo-pkX@<}{nIIPP5135Ih!fBEH>b>%VS*SEbgzQ4$#>{X~h|4O0v9|AUq
zh6R7Tpz~?lsZS|_V*2{4Q-ar|OunG9e$6R|D5v#NTar~8gqKugeBGiH5_JB{r!33l
z^%66Mzb&|Y>CMs&1?Sf@Wq<eMmHx_lRVKyN;>opRg4bsRX!YmY9RHl8cA)wO`>!(3
zERXD5pVPXh(^Dt$>7*QUN$XDAv3}dT%@c}Jw!Qf{@tjjlL%05)z|%)~6_dL5#ArFP
z8z}C7ry1rojc2k}%*s<nEBzk5UA|$0T&PQ^iQvvQWn*7YmA`NI{twu~>=@v>I$8DV
zVpVqCiUwBJMprj;f7Zef_2Lq?(xL-ZTlSl@A8=AXePqSsBOMy;MxKs-9ITlMvqBVq
zHqUBWu0ErHbLt1prI)-G25C*5<UKcllOw-k&sx1*A9TNTRv!4IT>bcHTMEzle%+%?
z+g7EBWVs!GvPs1vk?T^@o;M#(Wprup^AQYx!pQ!}T~Ap*_Q~lZZZT2KAv&*`mYqJa
zYpLNZ=7wEuA0!ql8m(yj$2Y~uYsI=e3zy5m=bD;yR+#u2hCcTR{d;_IR>p;GU1sT*
z&#%i0OEqn6dfIx3xm`>B;;m-!uPICFCbl=NaP9jTF}L{A1VQ^r6EB(QO5SYk{69yq
zOYBhBJdO?P1g*CgU)j~pt;J%~d!F-TaLmcl7+tq5OalEvCH+0REi0C4pHA31>8#E)
z-i22}w#&RUSMX%0%a*_G$nZ<hTQ0=t#OlmNJpDhuY%Xdu)m!SUZxF<u@AKL1l*Xsr
zkf(8n17?_P+v_)J1|Og6?3b$18#gc0Q%f}6A5pZ%OCvS&(@Z8F^@oSmb_SNz8O7y_
zc@~CwmWny9$h=yU#I^8}=I#>j&a#Y?ljlwNHbv}2%Bc|DeV^QSJvWay^ZbyIM_-@E
ztdxvHOvfj^)4cWI&m|_k+XZ@N3Vu62oO*L4K>uRmWN)U<<tP5Bueh}A?X}5`tD55E
z4DQ9KaQ}asb;iXt=!#BK^<kIZ<GWm*h6t923HTQZpEDJ|ZX&z>49i-x<f~!2*F_D(
z#mxObS(F#gW3{j~7jK+;anI2u6F!&p3tw6=`CG(y%joa1B8NJjcquM;@r|u*{ff<=
zzf-Sl+gyGyDNyT)`o4=cXRmIWBfxOVl<|d|w#2n99%@^?n5sfo^F}%9__f%54BqDD
z^k>`l>KvDCCC841XisJN^6bcQxhA%ZH7bju9x6sS8|i#WdK|W^fotzQreJox6<40=
zoz_kXd8Ks7`>jM1^TV{WOnJ9onLlv5Jl(3y)!xAMtafzRr$Qd}M-{Ga+Zmb-Z?r^B
zVDaB}LFU~)CySy#s~`QZnc{AI+@@-q_0~roqDKxEnYERdIn4NCFC=2<b54DoiTB1Y
zoXid0^*b_xpKzt??Vc4@HSb&HwJ*xKEiVgvjy+4tSaPGfq(!eTEho28#9__as#nhK
z&h_hBOC~<zv+H(^6MpyV=v$UnSGf&u#U2NqNHJWc^InQ$xx?k1UajU1AM@XfPj@jE
z`Oh9IxLqWC>xAg7AFH^+jK7EJZ=WbB8E(15x<d3~LvV?re^2w&lbbf!Z3?(lbs}==
z)ri%8QPV>e60S$io;t<yZPfCqQ$5#8sh^TcjkGT3*Hv(-ID9omn4xlF`1Fa_WhOtm
zKBZ~Pj||_o<=O#iFJD}pFy-tvw@kVJA46}=J72RtLjT`Z*>x?<0h4lFMUMKgm~~_}
zu1{LV(mbW*tm2K*{|>IF<(PMC1)Z$mHJr07ftxQ`zaaVJPL<%I`OBFbjyR`?R=#Rt
zpT|+fWhSveP%2C~N=;Rd&3i|S<-sB`-h(j_ZMs!$?(f=;<}UU=nBH3+)4HwQ@mKq4
zHP?3?A5OU1O<g2X()avHlTLNB&g3H=1wGrAN!ah*R(;4!@3_0qhPAP4_>6bo%+vnp
zcj|#*PukaeOr~NQZ*@*CTgB9<_H`*+so?ITzNdUApHMmN^H9uZVc*@>_fwKXx(>&>
zoDnt@s4vwj^fg}NFI?oi+0=4H;my^{Pb?5kzC7{&*IyH*f0|kPN4?t>ZYy3Ad8Byr
znKl#4q{)-NhJ9ZmV|_U$f1R9lT*`FG=_@49CQLO~Dqnj!zU=+xuda8}B!4Vpn11?l
zQdQ;j{_?H$GnX_(&bS}E@0exP@$0hT%O>pN-?l3}?SQ!3uVT5<Cn=vq|DJR5x)5Ho
zZYS@u2Ig}&oph(2>~a<gKjI{+to3h#(|nD)LQ2|E3ynoG{I2<)^m;mNN6#(^=jL5Y
z1O=~(9MLg}YrL;B!Nt9EcIzr5%WwO9g_;}#R^4j032R|}*xa&=FM`dcSzqD^&n*pu
z2nLUigPe21xolcuC#AU0a0@JBO*J^|zoRHb;^2oVAu&4+8dM$muRqJt#L8>`y2X19
zwIXXWwwiD>S#XL}7aTllzGu_Yw~<-bA9Zdn>ncB8z2}<Bn=Q+a^3K~~An;tEK=?6V
z)8ZSGmib?5(LHfa@GeW&36_*o&sT^G_Fm6ZxcWTMi1W;=w+(aB=6XNe`}e{+i;2Sj
zXY?0xntZ$-tRZY_Zo2OF8WYx;6JFjqyFx9Up=#oHs|!D+#29~xZB3r!o>U@OebI5%
z<Z2HITggi^w@#T~ddW9(s_yN}a|)-<4phi4U$o3Hy1IOt^Iu1=-;3>M#Z*Raof^PY
z_Dpjb|E_CHQ`2gr`f}}FPSuG>yt!@qwr7H~&aeKnWT)N=N&PjJ`ctZzx6SVFlbrb9
zgTd?!i`Kg%{)-M7@Jwb9R(f;Hd#PE}vL!mEQ=Zk^PkSa4U^=gR@~ymE-b>FE{ZamE
zIsf6`yWell=d|8$=D6^q>_K*`gAX(B8}2wDYI&b^-v#BqM?bn-noH7FW<C@=b2xg&
z5ygE+_IEDUw7ZaZ+}lu2Yq|la3d6zbMR`|?r@vYG%;UUIv=uLF*X@EO%T5LuXPn7C
zukU%}o8LmFc{)sU7oRb*NPe@!`rReHoIOXx4)hwPdrcP6(4JGJDHFV9x60`YW@iE(
zPnmg3NHrjFyU<RNvI#$bgmQfs-u>@2=l75(;jlZm)=F+)`%voq{aY(VA|f=mhiR_g
zyus@o>)v;NtyuqSyGONb-oU%;{q8*_M<bq8a+KZ73;4iQrhL0JIb2a?cC1nLN5$Xv
ztBlJXqcfd;e{!F#_rjxnromQ5g}w(TwWAp4MJdf_yg6yfp`ezve*!Y2Cae`*74B9x
zw|Y+5_V`&x_K1J4-YeT4t26n~j=xjAB>w0hIL614>~}{&D~a>!A-Be|fVUy5csD&V
zS>$y4;AuzK=E)kYOrcqETYR#A*;v{c84F5;2v@kQW!vArvhi#5z4^`&hLPd#V*hdP
zIq<z!;{oUYpSB0(_qZ`8aZT5iW)U*kaWvszBlq@#8;4YtT=e*r`JNm)nRKi}+A!<S
z2}7lsy-XgnWHcK@yn<9&IW+zsUg@Jb!zlaQo=ju)b91e#KmB=WtZ`wXQ};cQf-NGK
zmU^*HcxdF<*~Gxv#Iit2<)DZW)2%H<u36neoL*=4lyq)86S-CI=Cp^OzUkiDS?sxW
zQ}*<A_x9HO{*|iQa8@H|@rfRxf*Y#d(_&e-WcAz#vyNV?al7bdR`(RMb#bTmJU!LR
za3S}>v|Xl~MPC`MN-q+8a7s+_+V)d>N`-%IU2VOM^-f`F^vaB_vA6c=-CKGjTkq8O
zePRKJ^h;i<eRC_X+?!vz_x94=Yx1tSh#SWqnXY;NX-(&slQ(DQub10d!T9#&`sMcf
z|9#ncTJhI@jev$24vz;7Y+@Y`8oAU?JZR#RO8KwgBEXdr*eba9>lB6<8&UC?(<isi
zX;W&GP>Pkk^dvFHa4AQ)&&#DRk|gKeP*d(TURIvmYjLY$UZ3eLv&Ff_jG40~ov($o
zO>~v3oGiU`M_}9J+$lz>(t$ypOQ!}U&78y#owM*j2xkDR1C!^3#PHOm6BJunJUtqw
zWz=bgx^br5=u{7nu{^3iD|FH&Htn1VOcK(K6OCRh&{I7%Bd)D#d&c}pRoB^KV{Rt(
zr)=F4>}?z87Z9nPIw8m{Bc$z_WM1Bcg;|nwc3$Ml+yC`iSc%^)uGqr#)a~NYUmUq|
zHXP#f&RJV37tOTkl-lVxo6ec-mfLb_m&>9y$;=&`;y0bP{C_+BUY(_@6RWDpQ4!YO
z?7SUJP1XTU@7lQ6ZRc89eX;((XUatvv3Iir-4*}74%{!e^$6!dHu0VUPd8nkki$Io
zZwijMDXIzwi)puRYzyqH3>6Aqa`RMj^n|4jLb(e=SH|Tntq_YnZ+Vq1uG}^(=WGUJ
z+l<6ELAPUSnw-ai&NN(nGwJ-=A40EE`!@cZ8oaHMZK>{DSIOBEwnv<7UDNN%Io&&E
z@1ITa=YP(cyFR1$`X=vRFORNXcVlI2ak<uaukY*g7MIVfOzqe{uOjWu-FwwFk9*^`
zG(JDI`n2Cq_ha?KOg|Ewn7#b+pYq4uOL)fT#Zl=ZVpqD*HNREPs$G8Gy8ntD3VjyW
zJ9T!+P0uaZdqX+K<y2htJ8Lz|>X<F+l1n7FU1r&n>vyW;&(ssfJ8M4sC%sV#x@d3z
zbXuIBqG%|ihk@&fDO(Sn6wBYoDH%I==arii%g?#34OUsORDEh0=PQZ7aoTNS$!knj
z+0XB{I<|aDL2E|yrC^4o&Rb?DdKpb(RM1p8AQZhK%iLYCA!ET6oy}YOBl2c@vw#2O
zxZ|a2%YLhw@g66{g|aTR)P9_Ld&1N?=_d`MjC&M$C%Y<dy|BW2&W+%0F^AT6=XgiQ
zXI<YTWW-piD%k1O67GC~)m2^Yqk>xsN2^}bdYyNtl^I@lh_j~&87x}TzCO|OU`)}m
zY5zTPdIhqTrSEeXb|tP!k}UQNp0`NSP}9-;?GA47B?}l<Y9!|Ard*8sW-@8!$uns^
zC7cOmVKWY@JWFq35@FE3$~E6tEwew9RZ_*FEvV#}XFzrAjSGil_RbN~?M#-|UdyZ_
z@=4YD=EUe*e=d1>xE>GBy6IPRW!n7jYt-#3rTHKJ`F8rpA1%)#iL+ZQ5+)q;yc{>-
zu}t&5*0ZMq`HJscnPYCg-IcrXoYOk**sbSI?h?{m<EoXFUKQptx%`-_`-e5lt!qUV
z%dm3Jw-K83amMrW4N9Dzep(t+W2P=`&dc<w`86rTW}4cm4W3M#1(TV+O)}cD<&f95
zlauELRPO&#sro;yEl6Y$-&T!(Q%xfNU+!k)_0aU@OlbUi<I1uiB}TtJT44)UPUa22
zabm%|fE6yfT4iY+Au`V{m~B=(JdfprK!#WfQ*PGfh;^s-K95;ld+XJzz57)4)zl0-
zgMGz!-4R`1u~d6@&QkH`D?EL5+lto}zrB9&n8mT?x2%0NrHkW#?>g4axvfv?)MF7D
z;Y~i1T2s`Tr$o)VrMQ2A=fp0PRdEGw;s-V)8Tj37Id_4{#Np{vLq}t&OFEZNhIu5K
ziMw3ww2_dF5^C3tU7Wh^Mev!;TYILL7&=}*V=Q}blfm@0m!}q7F1sW2?bEb_zTFI~
z4&VOC@XqFmrcm#y)4T7?U21XR|C0kOW(yiPTf!NIIPzUjMqm6g?YghfghLKrwzOvM
z`(Ai0X2R|z6VBS-E1%7=Xioip&II;Kz4#@c177TUpjfPBagZlW_SLbr-69wL4((gL
zK{K)4`c|ezS=x<spMvM-pPQaHlMkEJ;a{`oxhH>>PF&&FDTgNM@0>2Oj^n1<k)(?(
zF(Sqpmh!P_OgA?i3{l;D^Ti*zRQqEZUJSpBJ<rIjOqpsRx#gerf`#+C)DpHbZSIPE
zkn-z>CudG#NNUC__tPKjrcX%lHz}DFf8*1><r5k#?R>*-rFeQgxt1H@@HJt>hckDs
z=}vWvspMu~u&Z^WU*g8F*f&4Uq+B$2Rb&jj(DQ$$(u5Z8{ha&f`6lysf0N0p?Ckj0
zxhP`!2A7cO-dBD8T-qUY_ej|0sH<UX7CFng3)aR?jF0~FXuliNv4s2+DzP6*{#`I~
zUvMNv%{ZydcUG+OJO}9w^=r$5UUa<e2|U>s_qZe6YO&<8X*`oJbrz)jRod(tWfsvo
zX-T4h^K=b+w~Y3iDp`WNqk<$i)w26W7TH{xvrh9|_SM;M4ZkzZv(+!xOY8JI{8C76
zi^?AB3$oL4BQu!NW<5=L{?NO8zc%B?u9`isd4;R4|6bO$Y|h@tlbwXN2~6|NKK9h0
z|K>b9TcICLdc`?M*YEoF`}2a^<_`bo#xyXW*~yz*SmI?a`-bO#<l3f<n_bsbbJ`?c
z@hkI_a8>ULe4oVfP3~pzn-p;khlweV8jW76a))L(tqoDtD_ojj66Q5KXkCMX21A%;
z_tg78>y|9D{%Xo*lI*poX!?waB1_w&GQDKwXYAc#@=33>Ys&mr(|exvFEh(+Pu}(*
zazgHvPq{mMmc7}yDsACOf46lZy1SH8x1?}c7;Gs%`)f~L!pjRgC){fCR(l|%$-%I?
ziD$`2)}0ZLCaW?=XSm8oI<{%JC~yU>s1w<;p!H?Hqj1~ALmF8J`*!Cf8kI#(_Ogqe
z^ql9S{o=6fNKxMZp~v^f<mygeG_zveg56Sau79*zzPN@no^WN>Fo>UGz|Qes^UAV@
zYsVV$O(OU$#HEjNs(cPm{cN?)NILM5SodWM=8L)CC)-b%T7CFXq_wcb!teI}Yz{Wh
zBzQ!$`U1IsJQbR!#IR2xaGij(qH*xL0I5PpuAt_qX@T7P1&nkO`E`GYZWFX<UmU$B
zUCBYzDfO{f@QLazkHqr|yktu<m<l5F7}ZNYwsJmbeZDo~Not!&Ytif0P)&EXZwI)d
z4{*Ibz!hk~wq`@>>`?9~1Gcvg8kG^;OBS%sa^RZ%fV(z?>%rvWk4rl~Z!LB<SNNjN
zI)S6pVY#I1a$lGB&i6?rcaQjbrYkTna}_r9;}f&uewMJT!|KYXlGQsB_df30m7KV<
z_`kKNn9bG*VR<*<b75R+(`+=&Z1>+t@;@YWMY+dA(70|(8`py5l&1numgL85WV{z7
zQ2bFfj;Sh!LBnlZs#<BzkH@{m3!7rDY0NlkulY4-!XweDhpF2F_^od6MQz~Rn!vyH
z0N0uT{wWVd%N}@^CGu})koAgSn;FD^+@jwtgWLC5AJYTPMNa8wK6$GJ<@se89}&&y
zRbr4f5RY9Up4cJQ+#(kLLg<vaSg+colh@<ksIqP|;JOvgUmd}H|N5kd$GHqG*mO19
z0~Hu&2Tb0!K<N4O)ab3Di=N3(Di-ibsSh#;n`n{knNq)T+Z4Z<Su4d{w|#SA3lv<S
z(V*bipb+>!=TMjshZ@7&3qnbmIT<Ztr>Avib_!oOHtl?vNG@kY(Uk@(MR&2ox#c@W
z%3Ro1l!dB3o_>IRhL=NVfJ#kJ!^HO1iTkF{khIiNvS>PRDbMC{UYDW9?_`b7Yz`>_
zl3NS%*G_0JSTUjg#LW7LwA2lp_dZK(`P^)pD*iTs|M!VL{hg^_GPt)|3b$@H6KEHc
zW}jGfaCUm3jNW4B(-UQ77ffVgsHs+LOIjgr?Izx9S|e>KrZv6oO6J@*u5)$6xz|o$
zx&EBnU}xl-1?_Jh@UKx|o%LZ-;ST$)3pkS}<f$t9&fD7Ywbb{aQ0I3`R*#iaZagW@
zeLCO%`uyo9I^DzdS+)NAd2R0sn7+VYT}l6=RrvJM*mD2G<)wB;W!9Ul#iGixwyT&J
zsqnN0=m)AQb_Eo)tFnDf@tP=Dv$7}n;hcj^>V{U6-l{Sjdak-iRHHGdU$r#o|749R
z>`P{fY7~E5tXWYxy}fVEf=VWZ{x>T$eP&5UC1`A{?Eh^MsrRt*{|cGZfY}==TeK!N
zXEpY(Dr~WLow!RpW9oOAg>7PHT<X@(=E%Ji($`{8;G9%bxV$QGl1k<zRm(|Jp4D7^
zUSn{4@+?ielnZ<*AJh^X<FyQp@*h<%c{DjK(j>!hiqI#$5A9P{ESn$V8s=M4zdCJ-
z|II01me=27Qh2mTk2i1u!;Gn-kN#&fUY@%CqBBRv>bpOZEe(}3c1|na*>LjNYO{|G
zEmuV<S1K|q<z09wE`D^)nU}4luY?{4FP<`0eA>~}9jnCW9A%ql%4KII(V>*e@<e26
zV*XqehA4(vrlym>YedbfoY3GHy*OyL*JZB#4@6QFOf?_cO0iCO(I_upVxIPK_9aGN
zy~c?TOvMdXCChitu@)1}NHl6*vZh*aBU6Hrc<4sOn-j%eXUJOH$(=4z{Jn9pgX9+r
zHctiC<tv3QeJ|2&-z0Bs_i&QX(g1@`Hyf{n%%3%Je$vGGH=1n&H*Wsh8Sm`f=@h-?
zr|SZb@15S-R=(1ayx$U*XL#{u$#DO&TFcSBp5uSg#vj(QVJh;=3{;LO@4k_<$3shT
zRZ`5PZ3liNT{Z5x(v-#cU{P_Pkw~-==YuuH6Isq_wi#^ibw4I#aXF==P<*;*>Vu5-
zrXbHd6&h7S)t4srm2B)YZCyI+;u5Vu$@MEm+;8+veaN*jReBwRlwYH?-)9y5iH$2t
zWOpq0k}ArWAzowpN>*Ld<FLEfHfxVup_=W{%Xhw3lm0nzpZ4;7tHsORcOTE*bFzEV
zo1>eA1X!gWsMc65exfQWYAW_{>HLco8K#a}Q(mb~lgbMFq8hj|YhG073F)a`p8Gbr
z?eo>lUhg*Lh0@fpN>`DhsZpG(qjm~zKIQt-eYMgggL~HE$ISi<XLL?mHgQ^kWG?p`
zgUe!T1cRr0Ca$^FmLuw;dt%piVQ=v>Hy1HEEI-e+@rUc$*55Pk3C^5&#JGKRV9h6q
zS()5(el{sF7c&*K7>CX-J2-RR^PO9sEAp_48?f3de9?Zz5*oarHDKw+s*Q8148y8k
z$K@*2NJPg~e>^B~TU;t<qX_p#rPoDymyYVcKB{;6s6qGKH?DhsTIzKNrv(QkPZN~h
z7<j<ydzhe*CBv=FUK8d!Sc_?RZ}}G4>B4=YUgtz@&lcva6aU>4#6F62*CYntJ|T1}
zVQq71zs*UZSqsHamF{gmIs47Rze~!}Ck7~asVhj8EALXh_}R4Hgps>j%zE1YgJRig
z-g}PJZj8Fcv99rxM&L1xKBg!yHqD-*z740va~3RV(Vj4O``nyGGj}AiFPo$z87jP1
zSm*Rdspt&7jn-nt4=3gcoz@d%RnTFWA;wrN*gs2P_qo$--xTa>xioLDIa@t(_eIXR
zhrXY^e<$LMb?v#?=Pw;UfBE_OzdRS-E@ccfU}V|Inw=&c$Ebbw%!O?U?0mcz`R`sh
zYolt=DD>pd-hI!F_bm@weQf2k>PgS7Cw(YN_R3r(=Nl>)yK2s>`B`6AevbC_`MruU
zqrtL#b?QwaJ;iBOzJ|t+E}MS3?D%%t{$B_3Ox85DPG`1JD9t{)_Knb*H?lL!7GH@v
z^zoXwYQ+DhD#a+BLUCr3b$St-`U{gc8*Y`p(xTwn%i1mWs8uY$cYBh@a!!HXN68zb
z4hWfLZH$e{JbQf-;}Ndp4V>FPtWbQt=J4)~YRyNBc8OQmidXzSs>gm)Uz_!c`^~6=
z+Nu@pvjyfETTY(2ku``}YN76~#k!{Z8E@t1+}i!|*fs_GqJwqCe=KHloG5>@*-L?S
z+g~-WfX+If6OC_fJ93xI)>z=_ef;a}6N_(bS&`xAKYIc9wmVbKY~5(ly}o1XRbBDz
z$!t3^4w!7pDf-f_)3{BOt4A-a$4gM~=9=yPb2Ed^Ua4F?XX~Hsw_kYd>{<KuY>L=U
z@$0vCOk+P%^H6lM`(d6d|J{9Nt@!yY#A-{m)fc<WW#a9rH%eEnIW9g|F7jxk&!+bp
zwX+^@Ga0azoZ#~@W1AQ-nQQ06<N$l#5dN-Han(MHyL5MNe0{ci?($;8b4(4B4du=;
z$g)~g*3J^>NDh$rdV+86CC$A7{QDm8d%WZKp7+G=F_+NaCw#I`dHy~LwYwM`_w;!4
z{e9`ZOa;jT7oVCh)3aSY|L4uPtKyfm@5L<$@0`40^PH+FZ&vSHt>yaWw#(MCY(Ylf
zDm%eyKfyiMTz|cun)qeEe)DDPwFay<S4=)$-W8F{_vVGgM<JKFIc1$!gy$F?wG!xg
zdr$7oLGIN(5B}`F_-gz6$g65^MBbUroKo_?VmgccY{`}-s%bqo0`1RYg&AtB#oEqE
zzPiA*uXtkI`O2el$^82qxF+ZJ|64G*wq#bZWBb<zV`sZ3jkRx*cr;r*b?)C;ma&jk
zZ(;l54UwC7uKO3j|KuJ2w+H-xC%p4n_wMb!cdzpJ6AN$cXO#M~?$%ekTPx?@I;&&4
z&Oo^Pa$T>)?a7~SPwY5;wf%OYk681W5Bu(Z@c2EyJ?BJGX+l8uR&n{0d|e-fy~;j?
zEd188@Z*w&iu?V4Zz;<%Jf*3BYJtuvwNqu0hlEzzEML3l-d&s1^ZJ%AU%8}V(`S{f
z5;mt6$1Vyg*_culsOj3j<nLc`$<mqMBZB8Jea$J@SI&O!jOPEV1xDv8qaGh*uZv&7
zDwVKj=h<^r(u|E~H5V^ny)f^Q%4$x*gmahVSYMpvQ=2z$@dnlleUs%+Kj!RyY-YE6
z%G<Zkc$TdB@X)8~$=?Uuug*Pr^Y2~H{wMGFetN&-UbBI5t;5sl`A=u||D2t#Wsqnf
zmD~HLJbCf?XOkCfzS{mPUGI`w=db95UrGOtr|5Nt9-Z<_yyWBb`dJpoe=X7vF}U1t
z&ULfapR&@`Ho1Rt6|bB;f527tCBMu;ci)3YPl?ZAn0tZq>fC$coD=euW;iTACw@!H
z(_b_y_L0Qmqw0RMOpL!eq~!Fbu8G^nw>GLF-h3bH;tO1+Tca!%N1f_ud&eu_$?~5^
zWJZd&ZP=2WjjNZQoYr8-6ycd@S`cE(xYfW*Mx!BQO5p4zDw3KG3GP#vmzG>PRJ34P
zhp6?YRjLQXz3Q3XlpeWpe!hRbed{&ps4MD$^F9A6#!79OHpOh+yB(WUuTGU+y?ssC
z+*4Cug@s40-lZ7o<}g#2X<LEk?y^(UG-H;nn7aF@?j7BrPtWEmrFIEvHZn4+MOu8g
zx4laAee9hb`j2)M-d8(j9r-Tx<P6jFXJ?KU-#I_;bj35HC=+p`r8_wSY;Toj-&miz
zewl5-x3`yKV_u)zd%OJoowSDO4}JIWs6XC2ak}NXJ3h;emd={3tGZyy>{r(}Pqw|C
zR$cb~+Q#nxdH-#fda`;7ad_Al>^=EiW3t8zjn)w9u(c5z6;&smtz@v}Dp+9Vp_shj
zA4>-FQkK9+?;VVIL>?SqWKCE+^?%T+<I~D4a+b>WMsy}|2i-W)FB_(HNu^06>Wb53
zU6qMRlUufRI5CvjrMONu6maU4?Tk`%ZBkJQP_k84ec^Gku|ve6J1XMhBH4J4nL#c=
z4lG8KG%{pd8GNmF6)w;SlhSD7I?pV0qOVF)WBHT^J}=c6o0v|CFKBtietyB1Czs{z
zzV9i%u+&W9C|~r3UCI|%OSy`yc9-4Tb<uUBl=rTGmu|VNZG3YpNqDyvYv~pL<Q~Hf
z2UHwFL&K&fOjQZ-Rt?qv@2PenyKCRCAkpib_n1mI#XWpgdgG+ogfkJ_EEs$v&(C;z
zD3Z@ZxmP%I`POU3S>dO{8geZkZc~23_9-l;;Pg~J%a;qh&374H$(<fs{<w8|?8|Fg
zzsFXy7Hp5J`u)^=_lq;D^A@~eG3$TO$F}Z+3Zu}1e1%`Pgd04YX6Y0qX?_tddTe6-
z?Z)G-8xpNk0#8U(I<fpzo-id$L}bC!|2a1|91hF+<k9W><#YSgz%2<&=V@f$oHDi5
zLs2!O!)@i$jFyZ^EKAzlPVyho?U<s#xcaX5OO@X)cNV2LZ!rk&nzqPck?Z7h*`5r0
ztzNBPG>!dMr#hqAs}=Ki+xv0mFa34xbN>JR%>S)d=$_!v);_ZQ(yG-*`1dasU9+WV
zG0&SbN{e+F&348#?7ViV=CbeGSv`iwesP^CIph{38@Brd!_{@iJo^7$J0Z|~X4AQ%
z`#Imw*hH_sbtOpJ_s5N{{k%JGOprfo_B1;C-qz>d-TN%>F1Vaq^<<rTU-aWO^WVij
zv$8zh^Xe4G`9Gh|%lGekcd6XoruDM7KI7LbpMN~;+jgRVD%01;`HufoQWXCD^l#Id
zqA*itLxXJKO-A+%MGhy9RFPK_|Ib}sIP>Zv*|jU&wHHlj5L8s*5|n7wzh%fKkn7Z}
zzv^Urmf@zd4NbD!6po%hrohS9*O;mwe7dHqL?wJ<dXjTW^nVr)Bd0ABS1M;jb&4q&
zPUwhu;$1a)@*I{1^Eo9f0naxo`Tbzqaj-G4Ao)wHeGI#%r)F2+giWqbXE?VRIjj?v
z7HE(ba%JPoSkA-tDNNp2T|4sc#E|9`o>fa0tO}70nz)9mM}>iNmFEN1NnIO7*L_g3
z+VLTaYwxBfB0Z^VPHlK<l&5$$$|q7cWRX#H%z`lOwMRSkwtR|y?8T{j)O}rJ2fx*c
zMnC<JU%lhmF2^6<BfkI1SN$Zl&+%;95yt%AeEKbQRKG;>|NBrb$C;Cutn+x;y*)pD
z8K<0aQcpgt_AB`STTzYz`=bfEQ4jrQ`!4Ya+Hllm!Ug3Zg+`ID3eJ6t{u?YU_~euI
z@5iKm6OD#48_)LFbNH`yiO5gUP|S__{oMY#+0nk6ys9;qlkQ$>Im(zR;-DbAIPl*w
zk);N%OKX){&5klni#AB-R=LsiGtaS&*C(MO_{3sqwF8Y|9)4VHn_V?qLfFn`o+?eL
z=nk^qe&zbfbtQF<S49$*?mDpH+{F2ZPrv!GN#c~sGs7K^V<StaO9uV&Gg)BRo3$&!
zX2&D9b*nACW8Th^*(2t@e*a1B_|?Z{?k{*9{X*G4aclC8k5icHUN_%ws`1$W({zop
zz}pk1p-ydKU-IplZ?17Sy0U=Hki%i2tj8Uhjs=Wf9!(-V6FSA7DDa$GBf9zS0+AQX
z8~^j&DibzQS*YvwQPFs>&cX$=);5|=-LE<`)7Fh^!el<n3BEHtl%)<%o^4enH;M0r
zdVbSn`!(7P?=PKH%XjLmI9h#In{Ac2=g;5;w@$}7acXJQXR(G{yRl;B3e`SU0S5M%
zi){1Ex>lwZv~Id}U<Gf^LEd!*&S^hq>^vj;V&Xb!=5S6+kIN@dOpv~}p)K{Pm49Q`
zs@+m|)yy0`H3d%VE%=fWv#nu`uEXi0XSzOa@(^AdlJq)ijfn5Iq?eL;M~kMteI%#1
zzxDL9JFjPWvOU`~<;{~iUCEijYRgKQuAR}g4^HHgo?s#^w(-<5&1{uh%O<d}_?d)j
zq$q1k&ejZ3>YVaFqn$fT(mMO0pUw0#?$1+Bcx>@(Q+ctW)YCxe*^w=iCT-|V%(UZ_
zoRPGAB9q6G%w=EPk5;^#ly*ygt;U>ivFBYzFTNB^-t4Sxv0y>ij)zB6cCd<@Ikc_Y
zvqL1b@OhvmV`z}g)s-_>Z+NXFbJf%9>gqV%n4s+ES1)R&ttl|f-h6(S^;V{SlOwaU
zx4vH`b<M&5_Vs0P+m|nwd7QHT_PyYHMQ0A%ec9T$;kBK4^4;rp74FmOUN6jP{UF|`
zyCrekpWWx{-Z7_7*%Rhir1l{5|8ye`y9>E_zx`XnUK=)=?{WV2U;>+cNc=BPhLCk$
z8%0DEE;csABz@{$k*axZiNWF{p8s8@II)-o9<y__>K0{s>Gn13vse7bt{LxG1K+e{
zE_i=Ze%pmRweqfPbNyO-%N`tJ`E@(qNMRLEutKA*m}`@!%#EO#M@1y_YF~4oUUg_b
zQ{fHSmmZrkcN`A7@v4Q_g-h>`mj2Y7RU54GR-Zb+Iz?Y@Znyi-@4^50j$5kz(XM>s
zrT6DXf4Bbh9~D1!WA|mB+>>K8b>hzTiAnc9o!YX=*!H|nhFZ=nO?OX*RW}WrMapNV
zpR0V$s<P?d44*CO&mvC$x_qwun$LN@t+$-jxfR-UZ+$%|?jTadvSaVd+)GyS9%Tn#
z);xReTkz$9q;A+N^C`1TmL}g|T6^#k8>fVyc3H-M=CD%=jTY0Q3Ue(=rSIrB@Unc>
z-k#B_-NRM3g6q@ufYR7?jy%?h+y{;_9H=?HcahMf(=W9Wt9foIu1>E$aMyG7X^!3_
z%Dqn?n(-Xry?w&r<;&G~PHek&L;JmC-=l=Sr!)8(J=Yx35;&sOe>g}-yk_kQr_LWM
z4t}=a?47N9Hb}DGnx8q!;tJQHDGIuZb2zUCt>x<4!P&W%H)|&ErKX<W3$vLl85cO^
zFFK^~NuslA-I<lF);}lDUoqL_^`aNIHD14Je}8Ir-vVLFm+ZX~9kX8uFNj#bG@&(A
zqB(GZvxDLk+drM(KOMG~Z2b{!wOWG1dQ)>y$F!3(8&-IWxYx+c`fp__)F9)wk=4Cq
zIaA?|4bn?C-aZnvaO!1475?s3z7LvyZ#c64#xB;KbA&%niQXt0@n!1XX2U2OG4W<Y
z$?Rz%QhQ`4PYZn|rZ8FcIJ3AC_jG0JW1B6+%NFiY&34n6yeU0It^TD=)|%;AXEx~y
z+qV4NJH=w><Q-~jX3nV8YX0*;?Q+rHIoy){8Jlm+UUXhdSI%f{)DGiWl^xS>GOc;I
zx8TXKwJ$jyuAU*yVpsphed~#ROEjEImPk)f;(GIHMw9?+RK~ugFWAdou+NWReVyGE
z7{S`iv&;LoiDp5o+s`dC7OlLPVb8XD<uOV76O7#N4{!dZ+!}OXzoXXff7Mz6je-pS
zLwW*CRM_rx&oOjZW#GV^voi7bs(>bkr6p#8i<%B?Hd}FI-pv~aR@^bm=Qe+Qp?CkK
zd3+*{2PW}7)0kf@?Y*&PdujCiV(a-ICYYaGWPb9Ogwbtzu{9mPEf%!=kZ)+-+$5~f
zlC97wVA1NmgVWLJnv}rhEDOdBtX<540xM_wp0H5(wL@!*PKToH-i?R)7%Ww9o>85|
zr0-%k=kX$wjf>_~IKNuWZntuB^PbsN2c#DYFhnJEG)Ax*PMBr0u-!1CeV#OPS;Nk{
z2Q$xqnLR(a>!C>3jMePUol`t!b{RD+em$Yfw8-V@lyfgsT%w8%d}~zRs)$MiTJ!z(
z+vIF|bjy*C|Cd;FRWIke#N?(m^+<=S|3Q^IiH2#}8v|Bt4AQds6Qde`$IxEN&EBd%
z`jR-~SJhL=ZYNLgsr<O~Y;-{89HX>DZWoHxS~Mk2Z8_6-aamD_x{{AazLGk>uFtL8
z3wKDe-f3QbNvXMJ&#cuC15C6#IzLIiU2L)-l6AtbPVrTpZ3|l8u9{tUVa2Qshof$+
z3q9b`!6Fs-;Do`K&Oiy)s*DZ=QLcH3>;|tqv`qctZ!nqt;5v4ZYm&_z|6q|nMKZd+
zN7sL_`?cD1@yq_}51Y1K=GHZsdXtl9qh-%AgPAv*JZ*YU-n^yepe$rHRqNyo|79_!
z&P{N*@S^FyXYXZ;-tDur1^%~q@$Ko|t90;y65qoQ4ttCauJ7S9H<HLtoG%`;=2PIo
ze+p|}ub5C(?agp;+094%|8HLSd?Vlj(;-Gj1-4lVDsLnX#5m2II5TR+OkO3{6O#_<
zEIITbd6JIGqHa%PwTDh!9Xp$kEPAKpY;<Iz+)gE{HSIkj42Baf)h%cZ6=*1pVAV`z
zH;ibFI&e1dMWBrcLsbN8AjhT9i0gJ=+Qp?VJ-&UoJ%!co&tbbM>#M%9)=9CNC>?oe
za^uw;krv&E)}>q_C9Pp6{NCMR@~sf;y2|8vAj0x&#HN)I5;s|7CoL42s3<HL*~TC>
zW$C)I7kgC#RT4i=^`7M#ox+`xX*g@{#?=2CH>UlZ8}iHb-yX3%PPc0s;)RCXjAxe~
zOEzMzHCndTMkzarEmkdi&MocLQJk!MPucWyxpMwW5SnyBt@w?a*jket$?k<*KH;S^
zr)AEJKYKYox4C`OtdO-PkC;80WIVP>vKsuFG3)ik#s#c_4|Ga$&jxPTS0~Uq-||lA
zg7&%_C)BR8$bM~CWR>!bXf0*9-gC#}>K#jSk=EWn(yx+P`>$-VQEWAwuwO59&eWLL
zV}Y^PMK4cZ#ie(4Z?EC<^ODRn8YY@wjqSSIbSE}wp6A2`D<u~>oPG3_y}HBlY3O;S
z%s9;lJ2py%yi^Sdd~`F@nfK~~Z62ocimG{6^;~&(b>7<ldrqxmkuT%)ex`7GbHM#l
zYx!@v=6Z{_v`cESU+%xY$a{OGuI1+rCPjs&%cmnn&&cn&T6!o{fK@;>W%*GXjoaN$
zKRbOoe;sF>n4sviQ_E&&UI-^=2a^%Qlt(GPy$5FBs0rGA(_>LWyWs(kvKisMH%`op
zz7)5h{nivkv17hhQ(7z5wySR4R}!lq^+TwNL8hSglJ&=(7kQN*<}mA89BH$?VY2Al
z(#hx4+LAu4N&3Vhbu5Qf#Bf630cZVpNe0K9C%Q&jsl?i!6t0$N7MyhGCxcQb=d^vR
zdBlD%5euF+OIB5Cb=;pbhV!p(w6n4ao2BY_Z7Jhh4ySKoiC#~h1*EL_-@0kVS|j$}
zCmeI1{5X5-BD0;6;^qyqx35KSF1DJfsG}?Rbm2DDQ`0kBpM_rB`84H<_iW!6!c!)3
zSU;Ir$$Gu=ZQy~YiAR>+Ig#3~uc)D7`Y`T7OW+ERsuk_m?mV57aA^+5y+cPi7BK2}
z?%C(PI_S;pOTM?8f__{&#&~&Uz}@LtvHG&lqc^lpw+dc#XGW0^hvG$rnUP$Lyt5|f
zb}c^eEPLOx1q!Vb6K?MKyKsp}$g8Ds&U^je#D?T-UhRJO-uu>&n*#Cst6p4kh(Bd$
zzCmV<NXbFbBN;ty{0tipa-CHBvmt)kOUHvH6Sx!)v3>HMV!4^ki-Y6jRTD-9j-=Z>
zRm-RUmz)%0H%UGyRO*b+`nNOY+wtq9NUKRse6ses_Mt>Mv&4FbhuTuDraKasu51_K
z>ym#KUJ<*#(l=`*<D}3CeMv4ZnU%tgZ+FgA_Wdzy{f|u^4pR<q`*;1D)Gl8Q=GERe
zJT@NDIF_v~ci!6C|K_dvlkWD5I8L0rbY0bpoj;qxwwArF(tT`Xm%N=VV#em{8aEQy
z(;l1Wt+Lb;J2$O6Zen+o&$KyrZ`wXHl(o)X&+T^g+O+LL0WNw*H_Y7f1CPa4O~14v
zm$fu1U1u5FUblvyx%c_rt>pE1bZ^#Ko-I70(P3=rrB5U$D@tCQrF1qbdP>Fes+j1I
zum+L4#~)4OZ`8@-{LlAZom0eCu%U8k`>vVc(q|JHPdq&K^u%G^XVtuq6nT?6IUXHO
z<M^`8GXIf=@xT23C0Tk8E*<{3g~|N+28R!?TRbO*rT_hLwNfOa!t9aF&z*fN1w8DF
z7c6)c@-AJ}w<-1Y?w7J3o&N3);hOh0HRE|?#yeX%aeuj7?)kn`GZ;;|KRTK-dYVhv
zaBtB${dFzh_L$TKa+$)Fq34Svqgod<UB1tfbg<=ie6zOzbJY%JJO2MKJD69tvgRF%
z-zqitpM}Ba8L6V1J61VOSuF7^s#E0xld_!O(r<dp+BnKO9<G%O%Ad+9;+UVg_S(fI
zO2+@LD~riENVYaB_<!D#v|#1T+W(*B51LIB^oxAV>U=cp4bS>74*iz#60BkmpR@nl
z5jWBQ+_l|ew{ubx#pXTI{5|EsrFCNApKjV^<sR!Ub*!^FCalJs`b6#Z(sS7Xaq$5U
z`vN$&rW6EConYoJ7xixDh1vdg>SsGcnJm90yZg_udFtF1{q9)+=LVhaU-HC7J2ZZr
ziTL)kz_p`vtH;uhooZL#n{1SR;e1^GR6(}g%0)M32DhDDswr@mOKMid@+g6bdh;Z%
z_S9;=4~_djc#D@j5f!ZaS0EbNw1@jjgIeKPi5E>_y_XpdwuBYBuhs6IAL66ySYy~M
zY_ff`LAU?b7ykm*1c_c1bJ1CQbJq6=lk8LSSO330$Rd<*X;SEROM!h~IOg#u{B`J@
zK20Ik^%LjGe%_Y~cV8FD8wsf#^wD+++Y`F;=K+Zw1wwPv)~O11INx$=iOO+Y$$E3m
z>{%x`Ukhb_S~dCItev)zhs_Pwn^>-A@XS8xm%XEJy}f3#T_U^gi>!yOB}e#f{J2!T
zt9<<-`#;;J=^s`9bHx15F@LUO>{F^PG?|uFnf<c}75pm7{neLA?BtJ}*;^xj*G#+E
zw{70v$lS=c3u?cHr{`X)%e{GRx@+Ik^2n&WeaGC-)vgwOcUS)1%>VDUD0ZApxoXrL
zn(|)a#%uxe<OwYr0y_jJ?AZ7&GrW`Kf;-Q(*^@Vj>KRPlZeh6dzd@&{?B|v0!b*DY
zO``4QyzYEj_q{p4-n?bf<J{u!^Xt9W|Nq7=ve7g3^>HcH1J(^QHn}PsY>ZhZ<I_>F
z@K76%yj9PRR_>!+BBu)k53nk`^-5Xh@o*L`IWb9rb%DvsNhy+kGO=B6f|dnK&NMka
zMYFZ|ncwtSmI9#wHwVpyF1=#1v#$7F4DxBU(@jpx)HLP{;M!3os(pPO*A6em%d13J
zo2@yqO))i$L32tD%ND@{ZCCE>%zS<6?C)>6;X5N<|8kZ5t}7Xo!XjdKXZ!mb>y~8Q
z+<bLi*zPcm+qs@fj+@g|xoor}dxDDN&jzXG+??Rj;J_Ha)8nTO!xk3R*@0gF1t$D9
z)2`q5`zP}~{{saVkM)1owTrPRx_C73m$Rmg#n+eDY9rULv;F<0D(OZp|M^&(pC6x{
z4nHopN51^qi`L!S<>L3+8wD!4+}soCIO+T^f9@$!jx4zbu8pjGB@aY|b&ebq;j^j;
zXyFMondmB>bb?h~G}q%|+nfoENdZbsp-SqpJsC-zb5}@+`08&vBHUy0(j?i>=H-;f
zy;h%gEbg~u`nh<5gIuUc7^BJd2%(?}Nv9*1Y`GK`?I~rhIyGdI(shj`Jx<3oYR|R?
zO_{bgbtc2w8%@h*rE_g?Wtyd`;5a9|>zk-#(1I;T#frAgoI0oeRoE<s)v5=%$|G#u
z)x$ggC%N)#M|y?NEj)j<Ido}lXjNSK!>fl^E<cdDiK}34n)|A<Wm#TwtKM(4dbNDb
zx>v7C)?`d{VVU#y=FR$O55o!Txg}X9ZV=>+&T-`kY|)VvO)Lp?Q>+aM+M>QT<n5NL
zeJYpTZe2SfxIJiR%Vl4~M<RLw7QU)eBz3&5UJ3TwDXky)ewX2kuo%?>{jk4@U#Btz
zZ3(;?vF3woK{U(%trGEzA~RLP3KOpu9BEf=i<p!)vr}Tq+Z#dtx*79sdZqoj>b5C8
zu6J90UUS*hC?>9)!sRT%6BuWm+?*&Hwcx*-L0A5luZLC?cj$@9hHXtb)uplef^bQ}
zE6xYrMQ+@_car}8zpz~H+JT>oORpWO{cC(9X1!sT{(f8AvRk(^=gO6T*}3=IwW92|
zCH7^<)3@LIz`4D=prZM$zwWF%tK`$wezPx_$B}n1p^3$E-vcKeIhUPJL|k{cw!{=9
zBtD;!d}Cpo%%pdUt`3<URc^A=b_jOsEwT{qV!DtL(qr)IiO@TXPkxUl>}S6j;^)Ha
zzvSc5;GI#Y8uPZXPVsj;Ic?G9;QwON<BBv7Ot0LyNNo1B)S782nUm(H@2fwyQva(|
zZlh4vhKU#Je*MzDFS;=0NY&G&VV8ru8UB@cF|0{w{}y(zuWn5e&mxDY5|yoX&$nnb
zJx@F;k)(Q%dzn_V#GMC~uMF8@j{fKL`^EWCr0VKwMp=~zEfX>BPSuGmF&UBPqQty6
zq#l;?<5p9?xX`0UNO|Uu32J6HJN=HdPF}3Rs_EsUBs}wx%c?W0HNFSQPgU8_xiMj*
zPU_4_v74@JI5=TZb=gF33FgVucdO|5q_Ffv^CTN~DW>Z*J9<XHGTe8PqroWVMSt$E
zA2Zi82&t;S?#kV;Y16d_ZVo#HpX8J!dpK(;GR@x5nt$QjoZW)z!AuIFx;GrV>#ih;
zt`>Npm1QUsxZzOp;s{sUD#I==wswmv3oogzPc!A6cYM;7i%TDv1aimDTAsV($%E=g
z%JY6m1P7n=@xSh-*!btctAZU&0!=(F3$pF}9RKqqO=#v)S=d@J*-7SvfTef&LjK-9
zZVj*4uAnbgqI)mT-7;CwHuuTy2aNjjf80_3(6z<xDX*7Ka?_+Kav1?nQaX8~FHK6?
zAY{8^gQri@tB6^4Y1~r^m}(oWG{XNp)wdGV^8fmSebt&{d|_*v#1?y`mPEYduUhkI
z>6QiV_BS?qF|HG7So0y$cFHr8DHB#{hu)Ug_5ZrccWRf_vQG*56S!7?xUksD_H8?B
zd*)S+D1+7eRvxW&uyE>~Hz9m~R3cY&Mwqx#iqeUug*?ti&KU={Dw~IRZdrF!DDUWz
z=&HM7J1v6NDbLc@aLx1<^1j%%M5HtBQ`7Z50ZvEPsCY63Wc_D7rcu4YK#|8M#^+KD
z-{xuiJb1K4ijxy(U2Z$JWus`>mk8}Y=3+)F4}Ddmd(RbZTYvjnzc#~DuiTX?tsEW?
zr>A@0Ry%IMwPbB#BZrKuKFjt?yXSsca(-#6hMlBr#HM7PHwH^@B+Zmr$$2%SdsayK
z?qgdw%$%N=_1RrNVCKEuUv>9Q-6L(A@$lr^KCj*Or}eBayf)o>mUmzM<ot%eY7dyD
zZ{+=FT_Cn#rTpyzw{6=dHZ{qmC!XY0DVlcKz(TrZ&Dq+%VtKx~magfKE^q8B{n!($
zo_0Oy#?BYY`p-JwRh28>e;6^}%5L6=CU+a<g*=wKr#6Ue@VL-ZXmYYUv)k$a`t1Sh
z9CQ{5x(Xk^^2C1I6Ruh>vvYZcr#DSeP2_%XvX5a!_0zdc?o$Hat~`Gu@j%^$WKp~3
zms&}hT#t4bd2dWPH?39c^y;h)a;pW7u3pj|^!-HT($zOIwO%jeoYo{XwL$IK64Mv6
zZ95GvNBSLL@v#VNnj*S(k=^SJSGpFaUi=WLvhj6H_KR0xuMMu<{gk*Psbt0inYIlc
zcTAJ2c}3&Z1>>*WK6K#84Ao}WoRds0o1)HnxOKd2)yq^#mOf~4t4;3tmcVTx$2Bf@
zo!YXPx29%84vVD6x-~iJXI^{l`u%BY+4|~K*L6$EUM!5A5^z0h#hP}n$jI){b<G!<
zUbxO|oc#azf8IS)FMTxD&D|-p`merq#_S(@+UHEG!i^f|GNtJYo#6T{b$!Ye(<y%9
z5&eNO=KIeqwOBCy<h*YoQnvpRnwu44qz_N@;c{8G!F2P@;***y9C(5P%;i(Q<QF*>
zi)pD}_V;-tcy`@f?O9bl$vlgtZci@%bLUsz?s>}fK`$5U8h>*5edl?L-j<UK&IUyt
zP*44|^5vq4dGdG9+U=d7*D0zJnpN^^qe<8w>*ZfA%wrInyR_=cms2ipPyPEAzV)Nr
z^N)39S!di`x#O*_E-QUE<+zn=I>RsKr-7oj48J#?S{fwv#go^>u=X&QW<}`b&%sX1
z?Paq9>{TKb-?Mf0nXA+Ff6LZL#g~UZJYMzc@n)&^weJ?LF5EP6zmvSGyU3{krrj=&
zwrY31@;bHais_1eKS4`Q4sNmA`&=1U6(({E`~Ld>Xz$~a_4<)sJeii&(^_hG-<Dx6
z*IuHK`0q>`Q%y+jRWS!f5yo3$%3oQT-u0aGbj;_uo-g38&O6~!3TMUEONSF#D@s|f
zDs-{Col0s5U(^v^Jmo=UXb{V)1g<jQ>o=P9U5;g4Wzx)Wv~KaW-*9MgV(;-Q;Y&IX
zF6z8AL$kN^;ew)4@g~L_O|i#s$HWL(J-oF?y!J|y*aP;RrEHI89J`Qs<9uO6n`7U_
zTnUM)8;%vi&nx<#Pnhv?MWWTx$**=K{_hfu_jZx}c=24<k%v7OTfDEF=>4d8{pZ8J
zzc>2+MC><dXSAC1nJ46nMbBNykeD;~r<I(XT*x}5#{Y<k=hUhE(Le5oPdvt4I@3|G
zOXSCyg$z>}j@+D{(8_H1NK4PX;)3+^pG?_TnI2qJiahh^_!s$;sqSSTMRYqRX#`x=
z%4n*++Pw0{ZH<7bFMFLrLgQYUCYRR;yeWLN?C){5ma7dDL|1;C#~sMX6X_DoI)97A
z{nZm6f7;99ed3neiKs847k6qL6=alNFUh+5g!0^uxp@N8sk}XrYgm`$%$~ABF#3UO
zOwBZxR?j$({=*ID7rRLRJs7wxa9aEa*5fxWru4X`{+R#&Y$Kc36lr%Q_gx2PXWVJq
zb@KN9Kc^*b9IjcJRPe>0{oBlgk-ZgD6}$=*rk+vp<zjH1I^QlUT6;&M|E?z48%+VX
z4h4uU)y-+VS#U>tqSx_WMqgC{<1lyMP9ZPDUgM;v#zE}XA&F;gZ-o0rJP8Rl%~Xtx
znk+VLsi`POOON7ngS*l5f?uw=+!1gjX4{F_c^=kBJKmkS{O-@g_^P`KW~?`MvZZh`
zOcGf(DaS2QMM>3)O<8ZzAImn0rLogXjz({gRW+Kedxhyv?3Lp=jtm)BlGmQMy(jDC
zaZs(}IJ0Nm6T!2GRM`(X%7xuv_PsUnb|LGMGmBCu_y_KC3cPnHf0xt$0;c5!LeIqx
zOj^TmVZZGu9X_e*D^8`o?%EeaLQ|iX2F<Iu(sb_RJh#kSrgc$fUCYg;MLGS5yTmBJ
zdByV07T&Rv^Q%u?UA49)U`Fy<Tlt+H@n0FmZ20<jUQpg$BELhzHNc}aIq_)n-P7)#
zVq%fkA`-4O$6Y&E(E2NP8iT8k!zICO3t5(Indt2HENQ{C)_tx2DsM;pn7=OOA)}rf
zquPV%4}JdDJnFvI=IQ3hx>CXCkeZ(pdzgUCk+qNP-)My2XwD8iuglROThOX|r!i1(
zWq{dAZJW%=x0oa|4(d-8FDnTOeH30`aG~1r>=Dfy!fX+D1tMnfN!(_LxU=N?tTf;M
zwmFT>U){UTg`M=75g8XDo~JmkP4QBv#ET!AmX@;8Y*}3kCAuyuE?n2QaF^ekH;M1=
zbjW4Rz0UgVdf=--jgz4>)OJr~n>g21>C#25qAMFOvZ^GVzZJSDwdDMdBXei$I2nGH
zbH%)6VmFvH53MS^@{G-lT{Ui|%SqXAiI=WlwY(%&GB{|4-8op`q`65hyFl-C!K6cC
zCR10fkShyX9kz?>Ib(qS7ybvP5pMz)b7(qpc8YQ(Dp%$`-Y`YjB8+8~ZTIFWk9W^n
zY&CDO=ZRSXJ6iUBW3h>gxB2(nMr-1Ri=0Ygv&A?sY_si&%DL#+Bq-%LTf$R1TH;Nd
zLP~PUw8Sg-{}*m>ba~er6R;$uB06=4r~KVH=W0ThSxxGB=c9PBQ(5rl{)~gia*r4O
zaErf^a^*qRq^1{zKUBPSFoi3;+#r`3&!QdnB_~|r<wdTw0a|M(Ni7Z7#<=Cu?aDv=
z1}mSG1gzZ}6m_>GPWOv{?cM3MioQ3lF`u;+j(FED->K8ku{3I)>TRyr+j+vb#bl$s
zPFy<p^e&gyO9$E5ce(**67RoKz4mqC5+=2!TuKudXBH}?E&JBWA|U-m=H~LIu(e7d
z5}|67eUs95#U8ujl)g?Yqib32mAM%co;j}MzxS7$`6Soj+(|jR^1?lO@5oAr7rZ+t
zsJpSu^&;Qf^+`uBaA}8&y#D{?P9uXv=4QQY?F!EdojZrOCY5QaU*i*gR45l(dPz4=
z{6TK`^`NU;)_G}P>3SIO_)kU~laIrWDyf})&)?2n%{}v0Wt7JshWV`=_x~u#$KHIu
zy3%FOyCb!EEVX>C@jH_lPIreeWbEg1_%2%zpToMpVo7(KYidcBz1R8WRf4}_ueFK!
zu*tVeFFnd)EwV+_Ej`m`(w+GUKU#HpGPAcfGpzZb=5eE0a@mdABoD0z-M7vw{pI#(
zYdgp3z`O2&@2xk5Z?fVY&bW%MJ<s(1?I)c{i&%vDp9&?j3bX5O^fJ8YRVeCSC}wUb
z@_dPI0i)Nn>HSV}cb2@YdONA!^x@QRzW=pzb@_|l`V{8Ml*j3B*jOc-IO7-7!x?O1
z=QML$RK@ESigOjst4n-Z5^-s&V#mX^FCvaC5YviP%Tn2&xNu#c`-9bslZtvo*4c3`
zTUu2#_r<hH57;s@4r)wrO%BRa=DM)LYUb1<_oqBq*19UowUbk_WrCaP`KeRhg)>~)
zt8qSUOH=-m6fdFB_48)7ZaOLOI44j@Ka4^5!m>>_dGp@~tu&~-{^V}Z)=#p}GqYvl
z^VKgheY@s-+2j3|q+HtxP22B%vZ|lDyixrZSNFS_t2g=_xpY&vjYEF-q1SfZ$)7%6
z*z@kf|6AwwvKcg<QWjpy?X{u9Nt<igqGy?s(vcAvkrpEVU3@=C#F~ibicMM@=p0pY
zqGZKUyCsDi7+rU>CKdRkL_FEf*}(nUOWv3NwB4nXuiSE{uFH!Iob_eN!AI)~6?BVa
z3IpO#3YxPAb{`Yve*5KyljfeBtM6tWUb^V$N|n=z?6Q{&Z&a<VZOpyim3tw=_;l!M
z{SD4H_kF25qWS1T+pRjq^G9`?eHx!^F})BeEGZTHY=Qbcy`s*IlRGyAUs>s)>Kz-v
z;VNghYQhAjhyOb4d7o*m^O>>m<V4XaPpS&$yQ;N!xjA|EJ7s2k-*9)PL)Y!nb?MiP
z7<!&18LZ1RH&C3mCi$+nhgSdRh%2)!ijBX&d#`5d^<~xjOW%@s^u_WE|HpfLGzsq1
zbtn$!*k0gVoOjz%BqGyuYEq7#eo4aW=at1@>twzat=*RQzGQz_n63W(we!y0WGeSl
zzT3NuWkP<$%I)jtTzI?mz0K_On)O#}!dw2VdtUvhyyn{a8nyV<Ki9wIVf{MQ<Ev(1
zs6d9(^@ohiD;jT|ZtQDodbC)^*55u!EG)!eQ)*a3fb`Z#v$k!4(nsZ&#LV`X{G~0;
zCq{;iE0`^H1E=Pm4JQ^Y<CSI?jXj#zIrm)8lb%<1_|Cr&+u?YbZ9Rj}=1KR;!UZbh
zIkt-y7Y68A3I!XA`xky-vz(;2a?<xN`ONF{pD)qQV9;D)=xZ0(U)Ea7d%@@6N15v%
z|KIVEky`yx@ZN*j^QX_abX?-e@jl_3IUhHs&tRK)^sU0{%-g>z732G!)kH+K-iV!i
z+2ZI^DWz8`-bG!JMpqMC6EcgII%_TYr<$ntZt9wEg%W)Zj%M!HU$sx#FwH<xYZL2|
z1DmESWJ&G#Hgm_0)m9aW)82$Cc)mRLBKhcxEC!*Cg6W&Y%1vg+PWZhl;R?5x$?K`w
zUAh^J{~Esg^cBaq6tCuw@rhFZl2xqNI?rTNae0>1+nt*u_7+z#dGB2RK60nNU9sb;
zkHz(eSATh_zc1;^lFQbO#(_=jKN1$VNJ_9Sm%fy;@Mz?WEvd7w{m$INsa)fpY!c&<
zu{l&nCR1aE%ai|}{TZ(7D&!_Iot#pd^VIwJYM(u>(+V#bggrPZ^5@4Tt+$0Q%;pN1
z24z>iD?0jtYrC+r^@k*v=U2E-*>9XI{Wd8;Gwkrk2UBONbJ}`!SYDivAQh||VO+b-
z`)kFP2W4V?C!8eef3&;B+0H+_HRkx&+s8_;pDwjXJ@;HgHqLx*+~UbEmvAI(=d-!1
z>h*$Snew-UysEWjNfUcrpK$$Vc0Bbot76sRUz*{MW>%>E*mCGc*38|4NjH90XLa9Y
zo5Hx6O?Bh@ZMoOKIf`aa>TU~^&y+MMJkOWD;g;k1Ne3_IF`W0RE702bEZT14Jtp2w
zP5LFevhkmbH|w#~ZV!vw9oV}4|L3N8`8!sI&;0%&U2;eF>#F(rk$=0Y50_V7`u=|H
z@87G^W!{y4J^cLBL2JufeHl-F7wVKWoNL*ovm=oy=dqikT~OF^rO(ZID_Wgbu&V!Y
zO)h9vnw{dkj$`_?qYa*}-XUGgEqkNZW^EO|;KiWL>8auocp#vYiIr&z#|90DM@PGb
zw4&a4d`z0uEpMNtqv_Da(j9#3#-_)Yu6kRqGr6}h<Mc8&Mw8O0Gnxnd&d+dQy;E5E
z@%j0AVs9thbZ~1}yQ1vYl#MUCSj5fsmS!#C-2Ck7QXA846N66AUtjm{(zo3&!W#B3
zEjB%s`Z_;1dQaBPPgj3m&wsF|={KKQjD+EX!<qlhJv%l{NKR3mpzKt0;^U_jP2pt)
z_bya=Wt?BIz2V;($*)r`2YVIg*|we*;oGqD%(I~C)46xImVbYD*3<mM?L)gco+riZ
zXn1luUweOS+hhj4u&MDqaa>iBZ{}{9{;*HhriAf<{xmIr@!F~$DOautjZ|;#pieI+
z@5+c*uKW7?=kj#bhTgz8+yeGN|1NVZWHHS;J}pct=fi?9i6jk1POUfV_kSz7G+}SI
z+OGm8qrAe@Bi0@&t2G$&4o=XRACj|}x8;<=!XBZ}5?9S79!EG8jZSGeYYC}6Xy6gf
z-FUQD)1xDk%cwwQV-N3+8z&bD1*UAAs_S;J`}Ed`JNau3cFX(^K4Wy!=JFYni!s4x
zt8OHza%rk<5!l3~x#Y5+m1Ix?(~m!Q9{FE;AiQOZQ>68k3pYesdoQ}m=89Z$uYIg!
zuD5GL$u6JM7p8C<8BNU#4zRpC<w~G0?^mVOAzL+e&5Mf7Iu#xr`qlkioyY_C_bVRf
zFXC9Duy|@hbmG$9|2@7dZnWrRIjg7LR%qq@%i{4($!yh8{iyCmt|y|>PB%>xF*+nG
zuJz?c0;^&!Ys0E(RXLf<7euj0a2R!{{GMb#Rp-)ld(DO^ovRmo(B)dK*?s8bG_xHA
zW!<X1w@&hk)So%^FwEk>R34?f9YNab<mN5v37a>I>ttIcQ^bR%L4J|S(c2{cH?K8f
zE#O`^bNZ@CThpUGm1lOpv&wmHTq!rpd(qh^miqNKE<Q4>R<Dj~>{xQl=hX`7ERC&e
z952s$z1BCF_s#jV&A!{bELW6V+H!TC$j%3StM#sJpV{5_?p4aw6}vy2xm~?yAA6WS
z<99P5g@UO};t7XzpE;M?{g(MJZTojgYmEaVqs&wLpGNcbW4kYv3Y2hgdU-NhdGpsO
zow({<yLX=5mJBAnHCtGZi>%(0akP7#OBOfZCEHKa!V+Qvy@c;RXll8)FsY0w=JbpO
zJmK^BclO@9y<zdgu)K+{jyyRz<3O);v%U3-nLal0I!(``CjJdOaUsda{Q9cp_P>hf
z{q*^H^UnYHdc)t(Kh;}BPqwXD*!tb)nV)Eo!aRu?>Z)HfO&YYWEKt!{!6Bz*w)CXJ
zViO5XelM+{?F)MsRc&7R{6$*ubgrw*BX&567kPy&)e>FtBiZ2PoeN6Av;0cxOqR+Q
zK5|cENRm90plw{^$rRlYxS^|cLb|F-v{Yj2)&&}trJ;e*lP4#MJU`^hFi|_^brhGV
z@{^t2Hx9|YubRGxk7?!<0~J+KMb6hhScUp~QsQMFa(H~nm{c3-!d3Q!;Sj^KMPD8~
z(Gu)Fm2++C)M*u~Q<`n$*u5^YEuW%TBXeNl(GvwWno}9HW|SPg8I?47`i&H3=SQ7^
zecPAJJ<+1GyIQsC#D@Q@Oj6Hu%{FtKyP`aO#|ssomY3{_B9kSO<cbYXn55YryRhYi
zg9_uciE3#&izU}o`ev_w8I=|GdDe}Zwx9=ZWOH+xXWuSJ$laZJyC5%k&g}_}mW7j-
z7nj|TdDx*IX0e8E^_pV2?iuc-yEl0~VA?R{+l4@euGRfDEDKz(+D_5?zj0da2?H+8
zS1A#5f8S|#D4!7Jp~%E}CHw*3^2w7HwL8~W-xc#oS$gNCO6r-ale()W9e#1QiCa1#
z>DaahZDw=UI-Xp7D(4i#RNW~Hc~xc{3fu5}iJ8gEDvzUqe$~6YgeEq!JY97uutdo_
z^3qb?zzwdm6n1$ha5c#NUBDi-^9Wzq{|!m?OjELBUd}QJ%1HF9jCv3!S)tzX#j~k?
zkpin=XoPdAKuhP}2kgboW_K%21<&qU!xyiyN@5<@W}_BG?)(*prQOm)9NsvUxmml!
z_FfCy<@89crByFUvO6jzWtz+KrNUZ5r<ES;>d2O^nxAAlF`}>g+U6A;ZYwfM^->(B
zORPP*&g2kx+~-r1F25^vzq-+HsdR_u0fEeui{|yRFj?FV_Hj8CHYJf;b-_syuZGDQ
zjH%qWwmSQ<3hY~cM8nPZri0|E=&M3k-1f|MnZ%Hha%y&hhP!R#%^<fWjbVS@r0+Vj
zG*UB1Fn-rkxh)Z^TeCNE@ozqKc=ZZb`z%GSFcsbZ8San0yQ;T6TC`W<s8jN3cg@dd
zT5VX43m^J;WnSCXvkAf{Zu1!3a8owBX>iJ{_iAtQ<(IQd<=pc=l<*bYax<^K#e2H+
z@+JnWg_FY_OF~y~;$WE8)X+U)i=uBzjGOxN0PSbn*8JTl)5tJoKC|nJVsXw`clYTH
zmpvDh{IOiwel8}X+vkbX-<c;w{C+N(W8vDF`k=vU#u86+4+f9@8*YfbIk55VR>rDz
zKkoU=5i<NT>7YCJ<a=9U61n9w`88(bI5+K15{|BZS{B;1)PK#b$xA*Q5ev#_opmLc
zzw+Tzr8gg*?s3`3RTi;BZnc1eewtf<XpZKsXro|;>qp-1+_v&|@_&z}Ij^ob6n{4L
z54zDHvP>bu!F9q0;WrQVSnh~8dat@=i(>P>z#I9ew@ui@`2Rb%X>??RxXtFpbN%>>
zw-yB5<>C}GG=9WlP{`<VM~Np=M);!Yo<BOL?aK39w=I+SbbFG_Q?>nKBH?kLwA{R7
z48@cEPe|w(n{5$0UmLE?bE!`^WX18xFHd@22^K6Z&$0~{wvziR;XcJ@fvDTePM>8<
zRk|&nCZrcTYbS||ntwSb>c7>b_Ur8~zxEupX(cND+k~cVeDUt>%f*ud9_^D4&p0V=
z_jwXSPb`1d=evimq_uXwo><<!Cdq(F;;7cK4Po_Pnm*>un(lg6q%v7e_;c;f?*B&J
zH-GN=-7GSpFfBDt=j@79wy(2ZPM#3yKWqKYV_)a|RLgnf?RR_r=96zqHrZC@uPnVJ
z{NwaO#c8=3+OHTUh8|zeR~~e=rkmr+&%cfh>szkwD6#zVxUfwomwm~aohe(kRXtnr
zLTFY4>$}c^m&YfxO<~aS{l91ZtL3L(t8AEh^I=Ew`S}jsd4l(YZY(l8;gHzTvP^H;
zv`w#6?=<kFIVl&l>+DE>_u|oN9#3_lGi8_DHTSGyG*9Ga%1HCd5q;;w5Y{O7r}3jn
zBa3d5$QEBcn>F$V>UzH##Y`Tq%}D(CK<lu}!q`_`?EBZU-&`a*%Z=;HsypY--+A|>
zQSOBQHQfyi|64hExw(W{o8R<u32U3XWdy054$(XPK;Rs=vGx`&zK%l$?H;aMJhZKi
zSPCC4-a5m>K7o6iQ(x2`*6r&w?B{9Nx2;z9I2iq80{=nPXNNd;PD_fImKys{xwl7m
z7grMZir1MxUgQ@Yc3hUAvZ~u&ZGp%Vy@XfZW*^>b#yPEwa$>yL>2~V5l9HN+&IbE^
zTB&tSsd|q@XF8=`N<PTf!*kB*{e<3`E{Qxh96vmBWph2a=z`bBA9=^z8pTosiua`O
zOlcM4>es!oP&{5oy!TkTf{yO0k5bzUGwj|=)h?8qvFupYnJ<$Pj-Q)&yk^bUYl0`j
zl1-1Xoy>TjGfn6FrDGOX{&(gsSaYh$KX22T`7fUR*x_SY#+pBk>DN5{Uq6x`O<Eyl
zcJ9}$HQTQ|K3l@{TjP0=mim(^t&v=T=WWveSiO2Krg0%6<-#mh2Gd1O8?rS+f*2c)
zc+5+QYT@z>(XBXioKe?6vucsHo60+%CGX07-<7E|SMt8QqOMVzyL}tOx^2E6*Xgpn
zZjuvw&+7TX{>~8=@qV#&zU;3T%DWttFyy)EbB6u+C-&#3w75I&xNzM)*W2jjZhTKS
zK+8jq{q=@~<Nz~{`=2I#^XO>#_IB>Cw=GP@!66*M9C_RaKX3~R8=AU4Je9D8rQwX0
zpK`+kB?CW(MGZW+7IxaaQ<&t+QvKTA=)aM&?jvJgKa<j=u2rnILBh|jxe6TjIB%!E
zy~wZkmSXI!Mjn~Me!gBMqbBpxr2aX4${q{<2Q5F&;;tUVG(k^kLf4Vx9j8POr6<IE
z^focjNG@bb-?%2Gjn|C-Xhfn&*1=nMR;mAD{1g_Tyrft3PM7Qw-?(+W^<NI@PT@7!
z6Z~0^-`-aL^Cm;X6Aug*F??Ot`9<sT7oYsBphw4VZ2Wph&?L?I+kv!i8cXM>iR8|B
zb5ie0ZVCUX8hz*H_nU2&ZeMoTu1Ei;SBZjd(@#GU#hq^yek(I>IiM){s!*Ybf9rei
zjP7k0oE`J>&ReYd<8n&LYn96P#kMC7$en0dVsOZb>wm6CygTEZl<FuhwcL>M8Y64f
z4etti-|1aY*Ee9=x@C2fyZ40@7Wc%pw}K^f7s;`$m%sMtmPz7t4*r|hUfJJxC^_-$
zniE0(r;F<Ev8?~nqI-SP`UwSJZ}{H5qpQdM)cDET4PPz=D!ZP{Yd*<l9DHk~fq1Z?
zv1qWlsr9K3N*ck+O6QDRow!?$hAcf{e7ecxWdWaO@pg}M(aT!-i&yi1EZFYQx1%Tf
z*$btJ6B^NXTm`w6B8q~IzID8aP?@0P)Va+;{fWoLC6A}rEMrRS(n#9qsy88Zhm-P}
zQoS`-nX*4d9h@4y@~dd<v1#iJeKH(HCJC+$JE*PeB<20#<E`MEzy9<2>+{&|C^Xn1
z82i=KaD|}s&#UR5&WV{M%B*ag#TAmsS>mvs%~7=E#I80+`{ty>YI2&a$*&8)o0U0R
zm(9JSIe(i^%G+bsY?G`f_`3#&sB;)8em{I>l7gMoq_a}5e!WZ9R4Y>2+^<()R<t1X
zT-!vYTs@_!PnB<Ke7TToA9AU5MzQMrP{thxMC%^9TxhSI)>IaE>A)R6w-wE`S?slK
zLd>O0H1==Ns9&poG)%oB=-o0`t{le9XIBpEHOV>Tr2Tq$q++Gb^h0$yZD}9U{r^v6
z|9@Qm-$A(}YdM%j3x9fXDxUDa#jE#R=$_K{z~vtT*G|ySWZwHP$iR0+kmgl`{r@L$
zTZWdTt}tBwxIkyKvw<Izh5F)_=MG6%e`^Ru+8OVd!Wvz*s4O?zWV1$hLSIZ-sB)HG
z?5_ZZl5=LyS<N55i1T}-JZb7n6Q_8l&cD+Rz4)=TZ`$OG^FFJ7@p-$9S&hL>Gfz=H
z%%w_-O^kD;_6if*bzkzEjvv(Pe}6+G|MS&D({lBu<$it>Y~QO{|9-Ol|LfAs6$O`@
z8ZKpsbd@@szi8CJ_f=xXE;D7v?P`w75=r68W^y8P6t;Xj*EvVWMBY@yxs*TUC8t$I
zf1ZO$%%WtMspnk+ueC<ba1AW^86jq?5qzU5^{mA5?Q-p2+LP>;q%L~(#yIi2Le(M1
zti(S>|5x8pW4`Fq<-zNvuI8lD7NR_-gn7$z&owOJ=br7Jc1q06rQ2Vnrc0~lP4@l`
z5niXgL=TE+?CxTEzRvrknN~~hjVt1|8EyXGr)1nX<ooM}_=BfwSyDt!I0!R5^=CCL
zyg4aAamU>m8w!QDPJGOpBe0rFJ2GfpLGF@fgY=9N>z%==89b(;JZqUaCVeT<fA?t7
z$`IeF$}5llj;Ra{{TvqfRjEVb$%&5g=}O_ug#!EgjrVuF&p8;op!@l)12T^@D&KO%
z_lXwmk*L}2v~pU?zI8tm7cf<xdZ2n?HH(UGr_iw!lfvqkheWoU#;pDtbu!ZSn5b=0
zk-Ern>76ZJZ@(*j_`ki5_1e6hhVTC+e&&jp(Yp1CRu+#%0JjN)<NCBOeA5#B*C(z%
zCR2GPtBR+IThvs;B1zk7w!)VaNy;aU!jj(_I@dq8xYH@@A~kPGkFd`zdG~Fn8#olg
zqZHmRJH11BnfJESpPsh8Ox~F@E&ZWRQO>mGso(P>4m#ACDI2@1Z+T^Jw@JHC%;=wv
zM{Y=_{zU(*6!#@xR+ea`%uCl^)$X>cojGd>ui3H5Da%Dug1mCq_J()%o)*)19I^Ih
zRZgo=`JP4pPduo9(LVh{)uFs+({+3fy}Kdt(<0&LRo^!U<r}t&#dM3BDK0qfz|CXn
z_bBC#%I>&Jo8p8dCmP=_u~-x!`~P}?eE9~I?{{DE>g(?0GL)R`>QQVxy*QZTfs^<{
zC-H5zmYbg?J(|G47+1#Al4|_?gYoRA5e45v%dbC;>07>&!#?S1RLiXC<)N!v%9UTQ
ziOstF%%qrO;!nZK%aQLpDyt`}teAdbLb2YI&AfSxsXN>~VmMznHO&%k6Ut;-&2c7W
zndPe`Oto(##6C)Ep38{crtqHShEh%9yy;B-FP-N7Om6sWDa~mn$nIpPwQA9}2OGI(
z9a4VWD--t8NM@$KP2<9}M#JVtXPF~{VvE*iezj^|q*!JW_9q}EzxjA)^ZCr6i=0aw
ze=L67DWA;hGM%|~>6xXI^OsK5Kk-xZM7vGf@BhwR9tXv)ttqUjvahK~zr0v!YsSvk
zYku#`55M`KV{$}f?}w60`4`%zbj{wTKC8s`hl|T53ANApQ3k$M5s$i6d%52!FuYgr
zvglQtwcKrod-wCE+Mm_lxhA#Gn?$xPkUh6Vov+6J&7<jWzRSOAD)`AKBdxO0w0)zx
z%%-Id9IXXTOBfn${%(5ia)oj2oj+UwodKJJ=lacSE0nd|AX9nQN%@-k%xg<G6!}L7
zDwUd>WIxD=-j*3XMOUxjOwSSxVNM|ju3HDptk((jtL{*K71n)w+H%%yF;ned2?*4G
z-*Mp7<6TGXBvc}DjU%=&+%&!uIkPfh-FNR347G<XJxyNndHq)`d*_g{rAcj<<lYXI
zg)+xP|7mL#Ez%AW6gzFX|Lo4_%tFn5U*`m?xHl~l-NLYL+r~9=f5llITr5=*6kg=+
zrnq3H=wWS{C-!kqxNpt!l56z0dm%C_OD<|5b4g;T-3ytpMas7pW&c~$%OiJk*&M51
z&0hoMA8MZRn{~?TSBv|#7r|05V|SfkEwUGKSlG1m<ib76v%f4a6?5L=vErbxa{knX
zbN;NoqETYQ{x9fK$&>YGraPUvC>NiSo|T;*RieeHp5U36=r$u%JI{sja8v@D>^lB4
z8oRHo?Y!iDf0o{XZ`aC_dN~Vy&bs*izIEtBnclw}N>V~LoRJJ$7*w}7aLxG7raCEf
z(^7^n$92Cr-i`CJzmk3C%I75|-%>XB23(Do-z=?kO<r<~^7jCJj_;DHCSJd<75^<N
zW#%^D9&D7lsp{^J_}>p2{)Xh-Rw|I@P-@s(vHNktO#2rfrutqkOWwXiS!gG3;PTnk
zx0}t)W*&`N(cvh=vWw+_Sv}uw#<h`CcdE-Iy?CLq%9=&BvRH}j<(~Hs#P;zic<k8q
zc$JT~^)`-Of(h;o&h5Oi_bN3$K6L4lFgCh!V&h`>9zj;0IT9DcJo-|W^n84Ndj9EY
zdWM^7UYwq{wruOIt%V+0XVe&@WxP&WY+1puI7o#@bitO)OUr$yy3Cywk`=m2XieH(
zt*zSE|F7Yi)bLhHEQe`B(o_wL)n<)MTe;3O^{$@w=H_zY4}K@D6W8D0*YKZ<Po|>z
zzHTs2@y$gQ-0hk%I|^<H&2)ZpI>sXM*qoWgDSp!(ixril99~|Y=)0Wl>`4<zy>-#s
z+um*{eQUH`wn+2xN^bRgv1Q-I{(VdoGFlt8G0U~fvY2tn0{fnMhP%GLywn}6zpn7N
za~RWs&7ArPdk)Wi&bsAN#_DNn!`|oGsuY{$)iQE^czyo-{$!!Qd{4i}*GYH^Jc#1T
zn6iL@(=YL~!@4_07o1f~Rped8xh5>smWzFIp;azOM5&!`qDDifM%IyqH3IWI0$c^w
zUcboZ>bZc`=i86Ti~lqH<CqYvyJmwCm!^H@qYGj|852`Pm<%efI0x}8Z4vR}FnC(;
z9%`97HPUMW%hdQz(diNC6JIW6NnKQ#HY;b{%Cy;OvnsF6$-n0Ld~Vq@%jZ$S9*)WL
z>%AIP76g5nDZVh+r!itto8q$<BK?JoH<wHb^Ln{-TAG(+%EZ8DH&avJsI6jH)RvXC
za>G)uSF2WS^Ln*%!?a)0t2Q2*CD<s^c{Qf!q%QkB_x(Xv*eZ^+nd`W%k?{`P#Q)Yr
zr|Rq}k9iE%8y&m0rc9WmE6G$a`(j0|O8C7UOk0w@ZSA)OuVa|4QV<#*mULvPdrTpF
z!a7;a4UO6yU$ZB9NL5NKa%HHl5c+@6fN9Rf1DyJI1U+RPcdv5)9Apwwye8pyaG|&3
zOT!@lCZ0_J{zgl$o{X>jns-X4`f1VW<fVV6ohduYb3HooVa)Y2o;zkw532DLG)Zh0
z_5E_OQ&l%}$&|`nH<kyu-@TbWx6>^uZC&p%smvuapKZ;FZ|D7XBWd=t+1EETo}DXl
zYHRPtE#Jzv+`V(-V~BbAJxh_ClANAJ=9LBh(sG*SN7?V!l-xT0U@v3YlWXp5R`cFJ
z61pL%*uv7AqU1VTy~g`MVfvTK!(tOP9y%}EJxQ@@vE{TG4AKQH*^QEar?;>LdQ7Z7
zshF;LB`A<9^lANPKKb?2m;~x;J{`+dT{io?z5nw6bFzZ^)#rVCx$^JV+vWSy8N9w)
zW-NTtpKtfGUMp(FuTHO9cFR7bOZ{Iqz3>#P=tNfW$Q8evt)$kj<hr_=$>RW{R6z)X
z%~eL$4u?AyEZ)7zaYEY;tbQW1TJ)6AHTMgXH@;xWj+FSAP_~3wPVmy%yj#1bZQErk
z-y5+ZTC%uf=c~fWQES{|mv}Us+LWZqqME47sc@m`gRsl8h#sYhlP5V>L@4N~1~PHY
zIN&IhG^wSf&xxZtLX7>|gyz_klEuj#+G3a5f(p8$H&%BvT3y@Nb@H`hUW&5Y6OT<+
zpZfabl)X+=L`+@$p|DfC`X*oCw@q8ltkE;Go*c!K>9I|tK`;CNMDJ*o&IO*^T+=OH
z&x~kko9_21)xgO(>LSl&@grYcgL*T4mZ%(Fc{1c!@a%&<8Oz*GM@GG!w)$l1tqsRI
zuRAQyPU!Z_Uc@Y&G>g}4ZiH_E%LUGN9tpyGs&1FQn6~{@!tz<=yORpbl;l3mINo|?
z5y!(j%<~VgY_|&txh(M0_=uvBUfkP19P94<oWOE(vSyygiKlT_Ec;KWJIlY~XlLuP
zYESjB%hPa{KJmce`qDrK;U69@vO*WT>{J#_G=3zcX=MFZ;e1&^jNB&~qxBVGnvM#g
z_v!<7?U-uBzw~8K@Yggh^;u5mml-{s=ePHG@Z4S9ZjX&ce*S9F@H5J)ZT6D-pQN(&
z$`8xXx`3teSK==3k5b*xu<nr7>LgPKR;CTBnH-!VW`1|rq_n*0QCBoux9FN}Qr`I&
z9wo5-zozrbfje6Bju`iT*R@;Piq)TcO|M!oxxu%tb;+$eEA|?7x%5qW#25NT++zcW
z;Hjp*lBuO9oGx9~6=FH67+jI;Ho<&TpN8SaWmi2i!)G72uJhTLb!CMVE2r_TTidqj
zcy}xDpPcHPv}vNv<F`u2oyH6nM>U-vt6Z99?X_Rx*-o#}S;@1$ZSnJbyK56qZ&~%p
zyQ@pX%?{{D-#jM0ZA<Oj99hSP0FLc*o+P|cu72$Cz^&qtVAtoo8%y}KJpy$Yf2>vB
zy29__ssEJ=pKMT_`+ebao50f#(%ybP$k)8aLAJ=MCuRxH?4>7M#FroIJ2q_*@4k%o
z=^hFuwo8ueXnNq@dpPOHrY}cRYBfLigojO=_Vw7w?##KHrth8mYsYeje;X&f6>(9S
z>tFLUm}!#m)GKPrp@EaLv_j^4J^dBSRv&X={)%7UX9bi>uI5R35O&b#8SAu4a|uSL
z?*2<$b3-nyebu=6h?&IPpHkgc*Od6@xk}s@`WMuC`_O0SeK*s-BpBE{+r+u6py}{>
z6;(YEO}Tc<LpS)9!xq=R>1b4QU~Br}oxml1SN2ec;_12U=J&GJ1{lpc(qHoC(#N1h
zYyL+py?pGKuA=PP?71GB{zq*$)7<+ldBbmu*HNE8-q_=&lskD*^d^p=9T^*E<}2Pj
zkrBN~r(od{Wm(VMonLxt9==w&Fh$#Ik?N``?S*-}{59`}?z){B^0`NMtMvPovANTK
z?AxA|vU%1z&dWJ}?k&xkRynI@g=Khj#`@cDw0C4)n*Q+mM~~W#l}A682)Df6xS5yt
zyWH0;N>wjo(rwj!1t(g!%=^~${G_A{TlAenPeYb`a8Unb*;d+ZBz&WxQ7Jv}Mp)Eg
zrF$RRq*n`?r=@hL#v1SYrM$q@Ss-3-+DFf1o`b8b|Gv^$t2uAY^JOm?XTEmk`+Z$C
z;f@C9t2d9WuLS%JvHP+xHXv}_raQ}HH2>WE|K4c*$=(0=Xr--9D?PJ%7Vm1d_umh0
zmROS2q89b`&)j3xE^fzP<v%&eoPMtE!pF-8SH}haUAyI5SM;_G?Fq7%x{O(kj+|^O
zEqvs$s_tWUPx6_L6ba#53)f6+KbpKK<8i5r=WVM?uKfGfHRgX2Xn%KgH~YS2Q_tVH
zZ8bk_N2%QwLC@OzR?)Z0PDXC#u8t|t{{H(6Z~e~6%VsRGZOgpct6KK*tmL&TM#eFh
zrT*;aKJ%e(`t<b2F8?_D;<{Ftho24PzO+Q9@Nv4##P}amedPmve?N_{DUN?&A}bT_
zC#NoUiP=>py(avLpYR3+X$7s62@;m<fdX3<gd4O~+oIHVH2qgD*WOU7ud$-hC_={n
zdeG#wWaZ~U+RK~GUiiJ5qHZgeV&7rwq*3O!LXF{pp4+!F$E#%>hwK6}QVNfU1Y4Ac
zro~22YyHFA6eE$A&Q`%YB{Z(WA-YvC?6}D8rm+1+;Zhshjuuv4yW&{Ws-q|(cdbeH
z?k1;|-+UiF(lNZkX5r{;agob%qD+^GXz+2ttc6uB=|<iuCHC!^YnHixQ*xi=X2=sJ
ze!4m0=ri|YWu1G(yZn}@?lSLEbj{i5mOXDr*UYw@H78uwz35mR8htsvcFT{P6h_au
zh259j#9xNg?z9k}G=<^8XOTdmyt~t5=Bw15arFuat6O!&Yr;dTse$?bKd1G+X_xx3
zEq~*|-aA)&Z>^BJ9Z?|Vs33U2Z0{wreUlnQrZ;GODJT=}uVB`>{>WEqqj2M9IcfI<
z$#Cv|u_B2Rfd(rSBzF|KyHC(=FA}@js1O#YVcx7)F+p&ryrEdLV2j}2ixzTj6U8IL
zWREr*JPXn@)mJ*1q?p-ks+KGh+M-k0qIYvr>(&;dNDZ&F5VMtKjZ7i0K5N)Sww9!~
z`ZMQGbPWCfId$Kcau3f6ubESPEDhxy9lOmV6uy}Mb**eV9M*e1-sn=8Oo`l?PhoMD
zat+hMb&ga%QPR7!NanF%m1UrFOGH)MQ_-FsRXfGHPHnGV@FIK7@{ZD((+^Fr`CsZe
z{cw0!?a%1~oR*uvHch+MG5JODM6uclKWp@iYbQp|yreG1_(1Q(R}qg%JrCUT&REo4
z*lIFaNOMY4>+AO3ki)T`k9$8#txsb1`Pp9296nooru?g?dR@!=&L=jo-e@xlif6x3
zAsF6%bY=WQL#_vj>5|_Cex2x-b?*ol6u<J*uYG5u$k8U1iX=J168_4$;Ub+6G$+X3
zY%=edFgba`^Of_o+U1=)60EP;I!4IrYR-RuGWmO?x~(vSvE}68fXOaf%Pd~D1pY{Q
zubk@OsTmX+oAN-@<z;H(kJOZi3XhxR-kxpAt;tS>VG>8QFDn_IUa2F#M5o|!+ntB$
z2bFaH7gS6wpAlYVk@2}O<CURYPJ?h)nls}xndk-4GsR}OexL9tvt)-_wDT{Q7oM4C
zrq7W0ICsK{=|{HPtX;8W<B6<&?A^;=bR02TI{!z<>W-yhzq+Eey7#!x;8B`+$fGv;
zWB*OFnJI}Ldv+}AFIiSNE4SZeZei5&fRg0_PmDc3iFjSAoo$(STEc5#q(q&f=9@CF
zD<{0xU#YwKW5p-&{JSf3|A-YlSmFI>#>!vQ%~-zI^DBliZDL)*pz=z?LczuNl40SS
zj5++*`<7eD{!T4?BILK^dEsZ)xtBJv7E2X9Ju&y`$$2j-{e>)x4xO5CV3q=R*~I%l
z1y6g;`}AaW!^<Y=n<f7ZEfmsrSOgz$*6h^(64GqJQF>j<pLx;zuQM&4a!t6>xsD-H
zUY5gE)`C$eK|;oWaq`6lDH}ZhR+a?{Ey%jQ{)zDdr=uDH-!$waH~5=|{%2k&*5#0~
zd4u?^DV{F{1sxpU3QYYLR2ktJ9zJvGWOtp(?BVsymB~9Rr4r@#E|RlmtXh<=C#vmi
z-PJMs#p1`7L5ixGCfzp6JT`ZP*7#{foeiIHdU|%sucf?foh!|k1!ZN1{AzNaT@|eu
z9ahvmjkESDyXU@$<q@;DR!F%DU)U<6zz`qJl&Y{T#d=$s_BQnn6{!o$y*9Npa)~}L
zuWR+Hn~+pj8m;Nn8GHT3ifHDz|8=KL>wotqH`hONjoWrGP%yzdWkJDvSJ?t(r|+Wi
z6IM5T%JBX4V%3KmzS_H{ZntXW-^dy*=z8_$nmdlWj&7Xz)^gtCofB8x-gPEx9{<gG
zE4?$nMDG5Oxo%tc{GGeky*jlnW;f%PU92h%Y0H`tkIfD^k-bMRVcY53Yzt1e<g2sB
z7nQk$P8MFkQWH2ivpwWa^@0!03)rGHCnnnSb(Q}+oXX?9kpH3<_iKkY+-Z`t1cfHF
zNnD9L#MEARRVRDL{yDCzOSXn3-4xWm<v44*lgPo!jL7|u1G&wsEKU}#&-~?ndRNEA
zRr59;6LdJWxzEzw{(IKM8G({A=F?wv1^(&^s#^NrTe~}c*3!gPOJlXVVon`O(AwgA
zySpfBt9j4X%B<yqDvTd)Z9VXN>yn^tso6}a2}e$AZ*$voBrW@hOU<?&A3g3}pPBxA
z=3})LEnX{%xKm$TJ33jYcj`s2Pi!mStFOEzQU8F$XHUz@hdWj-d${sdD~n)(_{AqO
zX9e3!fAk$cxvJ`pY+cUIn+tb7J~Zdd%bk21Ijf@P9@escb~EtXk=5T-cC%J4=~iAN
z!MRo-b0YK4)rE(56)K<9WZ%s)bM54qBH4#7FK3;4HS5%StzBz*);scSiwR&_x+vgy
z_36{I19rx+oxXi~PYlyuo-_Mp&g^@`l(K=5bHaMJw)I!EO}yInUi-cNe_nYA6Zbw&
zuktIc8=QaI^WQ$}^KwIyhcusJn8d__?&B3QjT>i}A9y`y>Xh#r6C<5iYuu`@>O3vn
zbVbOS=gu_I+vf%T=ownqi0)wxQB~DUo?a0-!(4iEo#k|nw-=;rW}GoUB%-^;b9aQW
z?q>hfMl;fC;;VwJd)D=boK*2xzbQv@OU#ior%&sgJ)&28#Nh8Gqq|1}{v63&5D>kf
zC1>~L2CnV1Elr|z>MoSlU6z<~b?U5|-*sQMU&%Z(YmH|8wv$rFOSC(+8AYCGpW7%S
zwOi(pq}*Q4{vu;1ZLPwY-m<Z=tAjiRPbQwY@Ui(yWs}OclL~AinjahQ2u)yoxvQ!-
z>i@hw*H>kql#=M=zUlYw)?q90BNL2H?=C!j^7f5W+?S8?+&pY^Q%&KF$C*nL-`?=_
zVN+FLP+_oU5KujpQ}*xWng5l0+ZOF*nH3Up<5qy!K3Cb>Vb`pjBloRz)LQw-dskpv
zi+NjOMjH1O!9WK#(b+Ob=U)4_`kI9LqU5&~3R2S~4z~R~vq*Wj)BTbIuYYJizR1Np
zXYtyC^~Rt3Pn1u$t}gL%Q07u#<>l(0u&nx&xr^+b8Jkj<bmzD~jJ&r&%zeX-EWg|L
zeO?zYyV0d<bLhbJ#o@EMqyjUqM`!MtB^YmgI$+Kb9o=oGPapB}xhy{Kp~2dRetRAY
z=j}<#W=l1Aq&zR{|B^3yj#C&M85o3@uzD@r*7WF6+V#uxcJ?^#&GVeR;_?cK&DU21
zY~Jo*H)VyP&sX;+4@~=ZrmS4PFmB!~mSP>o#SW|TFW*Udn_k_sDuS<Y<*UBNn0`LR
z)z@=ZOE#_+cjEe(S+w-_YM~1Y3eJjj)ttPvYso>Y=W}c(E}#8OjAv~_@5x0zy97*J
z^jb?KBbueSqNUaDe>DkIJFC0JN09sBja@xAC%<Jn<n!`~&C98AFLh*J&N+8!q0{Nq
z1B~&f8SQf9xda%N&tvf1@oLq*SDqYuy;oQ-$}OuEQmu4UU6DK4^Qs1O^!95!w*ynJ
zRC-TYQZ(hBOa(vpxxKLt%Z@G-HEI4Iy7F8I=Yh*-!(?Yy>UnJvNtAm#b=u=Y2TYG-
zSYBk)xa@9GeadLIg7k;T{*LOlmy7eBOt<3R!XNuy@U7wJx(Al4b7HLS`)cn$`u%-?
zb+om+HLLzSKE)%`CJN4dE1=gPsHVUmnE%l5?L%YVkN#(lsO<ZwYPV-{U|^`ASgUIf
zW56edf=`SL44U!`4ii3c6nygJU@?$q(dT<@SNqwt?(x|d;x217-RI77d{?)cbN1uV
zC-+zjJl@v@{awj7amOOBRqvi&oqYOgRoK_*<~vh(8!LIPU2B>1Z|b6ERpq*;b8>Q@
z?aW@ICg#|bduquci8>|GA1|*L^<OOhzlJ$~-h|87|Ep;1VY+a$_nu(?KLP&tAEW|r
zivBy@R)1vL+LsgGp7y%Ks;Mja<<{0k-3lxEUagt;Q}g|+IrD$cz5mLrUnST3){_0}
z8CKo$xwd!mqrK~R-)!2upuGI{%7bU`KGND<o0hok_V)d6)(a)`3o>2^xg%%wt!eS1
zzo+l~{k`yy_1%lL{oi_Yu3cig*(Z4Y?ZTfv4JIvmEiVdl6<A;16)7|n`I55v=*PzM
zTUmK#b|}nw;C1`omruJ^vRa-@d+^uqe$uWPpZ6b1?mDz_sR)Cj!vdj3R#p##837Lu
zHL0=8D%3Do)X*WoJ*^_6V3A8#s?QY7hk_}R#>%JXxJ0&fd+|l@;FzN1_`kDB#YC9J
zLqLV0fyIG=f#bpC<rn7Jck`KQi7*B(_3h2NrKI{QL^n|N$dpN6UxlrSS?@Q4DJ&;q
zQ}XdV4ex2pk$S-wiXLtf(_`OJ`nKp{h`Iiqy%i59-Q<>7*RW>}$79=^8NvyNdcLdK
z{W1EG?8m_pA=csI03HGTB{OR(!v!vPX_Zietp~0$1<en88&#HdYjf`Qb!V;2UWakI
zZ&%b^u>HWp!=ksc(pOI{ie96tGGT$juaY({;Sl+V*v5vpH}<N(Ikz|A$7`;`$@=zj
z2TDJ&US9scueQ3n@x#?{_Z0$(b&SFrev3D-G@sB=Ry60)30T0w8eyO)E_G*xV)K84
zpbw7brz$oq5ej>>s9DouMgkM7OO0P6gO5*x`hpOjjjm$xC5CPi=~Fhki7ua{<u0?i
zOKY{}rb&;EuTkR&;!-MfxwKYuTg0Z5+h*iwpVC~iq{vHq^_tU5cb|CB<jokz^x50+
zuu8Fy@oAl41(W?DD;=9v6pk8OKJ8iWcUi1S*Z+Q#aE6IJx7AwSuv-ePy-uEr%z=z2
zJUV)tY>K}aJF>1|-x+mlrR##STN9ibXH2=#rO@mm+7Qy}VS3fCEkL*9>YF&-t)Y>L
zrCWDJaX-pe)J!_trx2ei^x$$(<JZy~A^cJq_JKcd`z!qa!91b4Rc6sQB}Riq2KzWx
zKkc~v{=c7+!MYvG1U%GcPulTv+1#>Uop<NoV?ChBATXo+-m(|5ua+<UFx5P+_IK-d
zby@a%N!;sqIsJIB{*qC5V*B}^O>3v^DDXVJTW{Oy^@sjE;?_SX#w=+t(J{}*@VJ}X
znhpJ&uMcf#GU#^TI-k`kyya3<C2#tr61laej~J$|O@9=}eaqrz%V}M^`Tg%UUkFTy
zK4IKCZMEypsA<aYcD`INJAdJ)B3%!c^z+)^vwv*~zWTcLgjmPhuWz>S-~4~5?DpFq
z2i9Fa%gkzZfu~KVX2QWHu@->>PRU&#R2t269v<ejn9y*9MPvt``kb&WhQ(6m*Y^~k
zd%g9emJ0j6uHz~dEJa+Z|7&!LzGzP4+2pA)cS&uT&iW&BA89<CC8Vc&T<3zfDR))o
zkB0|CI(?t?ihr(ndNU=X+j6=8itQKIh;!_C^<5<4l3nZL&w+M(<>Sx4+yDJ?fU|IH
zJcGMv(iYzH9)e$t87HlNe#yHv+qH{j*Q9gnLQ+*2r*VfYe-yg&mC72XQ)d-;bv%3u
zScTIbYK0!EQrMs(p|eM}MM3(G!lsUv34&GX_6$NDTO=L{a77+rz1k_@&XO=u@>qdl
z-RTp~9O5>N9^AU;XL<TwXceBjhMj{o@zm}~8S9t)Io;{glC*C@#gxS!7rQ>~dL+y^
zfkoGYRV(PG%fU5FOD?IN>2b-5kdpo8{9m;(RX1T`#QP^o>vn%pO|CkaJYDUx_$1wD
zsvSl=i&Q3WIw+!M)LtUVa80=Ja_+;dMK60V@&rjOd#GmS?R+*@D^vPzNs&2cB(FA~
zsg%3pS%n~LzoKc)rj0p_Rx#Cm6>p{qXa?jk@0jFQ9x!9uhbwRG9z@>O=QTRLjj>qq
z{Hz7_Ws+uTR^IlU51Cp6Hf?`3C8dD7aI*WQV-pUjYE<?~-!{+<mk+HySNz;0-Rj~?
zrK+tR30A949cVB)ue9);dvKu8<S#S*Z`Nv*|KB4!HO6SE{>~@fr9UT4-?c@pZQjX+
zys?wlSsCy3$P>;e<(i`Te#^xVsaH&2J5O08p(^0M_kVx=k&Ti|a{@Jewmz==`+L5R
z)6@e=bF-MEZyZ>cB{e_!rdQa?SDF6RPv<W9wI%%QB!{5i%{nbQ2WGM^^!_X|X>Ind
zlZ);~ak$AmGMABZDU8WF$#3r}%*dh`%P7DRb(M={%Q8_8+s4;|_qkY3C$z0A*?N87
zhgnDG|9G;_@5pQ6r4j}eoT>?Ju4{xiz1PooS)kP~nk=#Pfauy89}{vylGCnB7^-ef
zNs?gB%`jhP*x$OQU&HwA@sMnf%Sz93J#X}gd3^M^uw{|ha>H{44#_9)mW7yomcDf^
z?C|D0&vYgq=bf+WDST>0ZT_S~8(pu4-IRH_qbU2L;Q64h!sf|o;nV(abUCkI{Zi-M
zKKB{L!5m!EE3dwa_-bu-WZ46*EefkXtyR$9lK)`Wh8>SG!*e#tq(_~!O!!viK36O2
z`7@z2ek}48a~G_>9^_^|Rk!D1i6zse)>aV})juyhtg<e>{`ONv`zGH4^}b6EuP<5O
z70owb7w!14#B14;!WRoKSO@nuE}t;t<a*J!GbaC-cUr*nU(Shpm*cdzT(|Al+Nt1i
zZH40%@k=g-r<MdeFD-Cuw*R|xi}^a1tIO9ty1VF<JJXXU(OdH(^^`3QnsT@k>zpJU
z-=C4R_KVeGU3B%n%qas!-8Cm#)@(hL`8Sf`O3%hwd72FyCcLt8)aX#UuAJFCLwxNs
z)&B*RcQ<D?COk7@63`KImCXur-x$AKZ|nYFT?__!3$-TQJ*1{0(0WSi%%-3NZw{Y0
z<Q6x%)wAHp%PHFoj$bG{b3?}2EcHi(M;d=n|5>q~&4QVdXH^;|Mk}=HW@X!ZxkaA7
z^SMuT@-peCc}1C91*LLbJWqW{J%8f5_6e_<hfjq?h==T)k~;T7#Tlm0yZuB%5)P<$
z$nI%%pQ#md(n+B9#bHYiMFFjnxoawQGaEPCUz#aXe&tqvpZJY~^7REL?7s_NJb3wP
z@|&do7oIBD?oLklb0_^wrg(tJ^K(h%FPWSae+9_Cmsxc3@_99ls3(2l$8U?@mHSom
z`-yYt%v0T79E<<kEZ*z5RLwGc|E;Al4^ORY;96n7G?gh)>09K*EpHZYnjCQd$d#5S
z&utXF6z6Z{vXI%eU^UmSg{xj=3;d}{znJ*(LH7Box$|PG*0tF05&Zu3<*s|CiutL3
z1bbIZSXvVGiD|-~<>L9_dtYr;uj&<#ckMrVD9dnTi>~df`%_sAx7qb19y!&Ylbdbw
zDr$3<%jxJpGnXcPpSNXk?ArJcvp<)_?6)3HKU1Z`5PdFZciOkvOY%Br8{d1S^XtBM
zZ(-H$zY7d|6<uC0(U?2?iw?8auFuQzv`+{$ynGNno%`j3+55~2e%?vn`}dye&1okU
z_a>@4sx6q%Iiq9WZnZs|SM94_m>2b5X2qwzbr07q(l+q%=HQ&-(0pOf`N$1y-V<3K
zJMgw_;1RJDc)~e(au?U@iLyHlG$ZC#Ju;Xsyzz$Q##@>jXD!|-a(k{Av#>a`u--~T
z+2VOEvo}gCHhiM!BvU=vLek1wao6XQauwBj>daCaf8-W78oe<P2~3ioxQb7{SpIp`
zN<C)zAdd!v#e8p8O*XnbmE%$WhZPGrEvEhWylz9|Zn;U;69PJRYdf$_T=rv=aJz*-
zkHrKQO;)*23pYrte%-M4YlDgF9+S;l9p1l97#*5UIXAKUaGb4{{Jz8Hy$8R~n=L+n
zjs&FanQp0c_Q&e|PF%s&i?pP+R%vd%XuMZ_Qj>w!e~E}@+f%!?u0O!F(Lm_-<*nBX
zmA7@wNSwTQ_U!KNjf-s_&B#bTW?Qr^t;H;LGEbw&k<}}7XAAP$I9jzWSYmZhC4k9&
z(aPT9*}M-fOjdPX>0_w+#KHcN0`rR<=0cJazh_UY_g=QtNo|9pqD71}gQdmegNIwa
zIlk{&->Pt+Y03`PO3vd88@id-yYuLCOgYhcgzwA>7nP1C4UTEL9!-2AJNsJ<CZrg!
zB-l<~-NiL|xrWB#N+luw>gBag+%s1@O0ldk-q>w;ai{d{o$^;a+b{Ls^m6JFIrYP1
z{<6!5CTewf6dJi-l>4)B%CsV0)(=)QKkbURxUxloBPpsQd8d8Li~oJ=UiRAxZ|R@m
z(ey{G?Xwos%Y_Nj3uihUIe3_pF~jB1>^+Bz_Z*4tI^Vo!&z{w<ADrGL)Uh+mCh&_1
z_mtJAcK4jwz2|U@&lzW*GsQl;ynN0~ZVHrWUY^JvmcZU9!K}hz8**m)fv!d68y8)%
zUK63R=5ny<A+Lt;?llqJTPMxl8@<~$K1aeVihJjcz4wH5Q)P5*Zq4Z9^n1u{H-XV_
zhG*023BoyNj2@p5QI_8SSZm$fm%^{Lr9OP_effLIt24YGv$wyu=>0rf?@<PG#HpqK
zciGowtGzASoFvr1`@>zRLrp=4qw$N@!GpX1<g8~hZqz$4t;JzkqEP?r6DEGMPjI#T
zcks6fU_Q#DyJKxs0y{^oA7e)o{}%_rBNw=@Uf^Z1<oMmmS-r-PVbV&EgQ6jGr=1oQ
zZ;=&TJomcdM)9Sdja`%UU&<Q2nESWNu$AjjtI#32*eU;{4n5y!_)CZ5Wu=P3>J=+9
z^<LPhDl*TXzf<m0;UP5<K9%H4Pn3Ffwrp~<lwY}U{uCjG=nd^YH|O6|V4cM=b?ppQ
zjztSv4;ZhnStXJ+@Bb11-N_v}nG2hC?9OS_?+Vz_?kzCqRlk7J>}^5@J6k-C80+ua
z!n2>{sHcwW5uG#JGegcUxWZ|2!H?yr$C^IRwI_VPn)tH%=q9j6o$x6$2z@l6B|OJ9
zBIIazX=teM|Gh#jXQNtt*GMeB`K0UC>a(}+m~Is~W+`;7c~Q63r?WO)+fqy1GHTp1
zbKJ6e8k!4K&L227MXQ(d?Ap8~=RRy)`;qzbqK6v(QtrM|;Xywpzo?m_k<3~q<Nwdl
zU$8O!+Y{CLGrp#p6I6vg4h43wtm$jMvux4}|EVt+v?c_95(;pU^;)>arRz(G-`XuK
zu1sf_u!c>z&?M13;Y~ovsep+!6KC9HVzk&PV0%y|(ynUJa`{5XnC!VXI#1R$U7Xo+
zvrBPuYmkO)uExACJ7pdlH+C?+T)0BrH=@&IzHMTw=H;Lj#vBh82y2yk`MeC9HF0H3
zS6>CM#kNLkz7DPbd&Jmx{@-mE+R?e-vh|-R`?J>_YL%5|a%QbDIo-&=(@E(BkI$Ee
zJv(069P~M}f6AFXYmRJh3OUW}<IWnReChP@Gd4k6_H3VfW{2;cQ^ht%TW=o^_1Rym
zea0k|LzJN`qcu<>X8)?&=R?-)V_I|2$Tx)f?j>7aH7BMUHMUpJUX8I)W_lHRo8{Ua
z6}P*FZuj;qPX6ut(8uiI=h(+A`wkuGnRJ)s>5Q4P1D(`$jdGu!dvJ;O-D11j22!t|
z^gjI*rvCTb%}MJjtC!5MIA8gskF#UW@t+g2A8u~=a`{J=Kz*tH<`Rp1&Bly_$G>D~
z3p?IeuvVBwb;ElN{+_$>eqZ=lLj$;DI@(hIckEzn(p_+aV`+qaW^3P^iM_t+{aXU~
zrV4U?^N3n{_1fg+YdjpIPIrm+bWM?cC|VjRS#wiDberT|StA1t(~FLeJXbstoG0~6
zVD{UCN-CEU6q!|)%DqYDC<(cwzFGK_;v)^0;|_^d*Da#l9lgpwZ*=@9{6HYsK!wk|
zM0ri`vcJ1Htk+!rV!DOZr(@kB#rHEh?7ysHdAZ)~cF$XdfbRxZUOiaX_BF&?^~{+M
zM-HiO2=MiBpX%cIRLS=OlkbEk4F=nTPdfd%V*;;S@h^Si&2p8=?2gBjnbUSnk#FwQ
z%)08EcGa&gG}2XvbHmj@HRed=$m_NeCa;d(XqDJ-z<1k<|91;#b~m4L>`~NBPQG@p
zc3UjpzI0~E?B=!qPnzAnwQcsnW3R4gy!>r;`NG=Vl4q~~_ALqA-V$N-++=A^u<A2~
zzOWgSj53&cG@H3^N*_2V!1*zp@r5;`%Za+R{#`Gcm~KuG`Q&b1cHZLH{pw>^elRp?
zTxbfCWA&9__7!OGz0kD1Y2tRyDCV#m%@VD*4@rhq%<|o^fv3l$Z|e&_*9Mja8zvhc
z>^&o>%aAC#W=&<0=k)HbIg@rum>ACJz1XxWwfV%&6{l~`u3aD}w@dZN^F@~@FETy#
zEs*2E1E;0D{?!$yp69gA=IZPZ$TARnyk-@j-Q6e~5gDTi=QkL+6e}7yxBlPbF+1eQ
zVVhkgK7Mca1oFH-@h7YKMpnR7lXKmB&hB1)c6aoySuQFo&NN(W3Y1`FlwfAuk*(Ws
zW#cxR9bHF`wfY=N^Ql$4^U5G+b6@tE$qjm4ohk_`hrV<kQ85kqB7VWe^eo@rE6F;S
zRecvNxOz1<_L5_2%frx{SJ!NH=)NOjmb6(S>R4>-zt{(F?%r3idypIAHmNDm?%ur}
zYqe8@-mH;h-*@jtZbIXMuqWH>-nR4xB<Vf0)4Z}n>dWS}`q7>2j1F(_?OyvfZ&_Zt
z%WHiRrqe$+JfGwCJ~t|2%ephV9V}rH38zJ$7#vzH;rQZrUBs#63m$bTOy}-1gw6I~
zHsv|?{{V+s0r$Fu9^Z*VHYfM$NlcKPJLzk}m4_4gyaR=gE#FoUC=!;a6KE(kH&Wu;
z#W}(pLhT#n-v-LL7Ruh+$ez?(H9=VZoUV1D<_8w}itZqp^?a(vs+umFu4g38cRZz$
z)_kcz#4z=^)c*Nfe{?L7>txjN=xIn-o1;IqfqRR=0+vghTZ)cY=q+O2rtvxGg=grC
zKhN?#rn&sTcjZ9tmEE!Zfj5{uC)@}&4Gj}`5;%e3?7Ayv`WLM0QoORC_}PB;yOO+R
z-s6@JDdC|derBe1Hy1^v#@^7Lp?4|tR>)hkETM3=u!L&v$iD6yW?^@2!vfVyQ)A8I
zp0OnKUAwn!mZjnUXC6s$Q;ejN@69Y;Tl$czFJDwD&t_@%zTU;g9S@G#y_mJGOKMwz
zUL$*8SxH)1BgdTX3Db_xeBK(cq}MzDo7U&-_<U|3rl5|dz>WYmv#Xt{t@+oRq$+3U
z9>4!JFM{h#$c~dYD!$9qTs$Cj=e41MqqyAMiRZWk+TJMroH)nmWnGt}X5>r0_=loq
zD`Zlp?2jm%f9Im%!%3a@PtM9)so;GnTmIB4W=-dVd{&x8jti3(os;(w>?_-FaZ12X
zdEF!1RJ1*se{Nf3;;yrL&We8JCf$=;_<u8;IsEj>myYTFIiHWWJUJM1=a`Bs%R5ff
z_doaSJ-w^-^|94w^4AxK-1Ye&`TxneHMeJfKfQC&o5Q8i$6c#V=;j<zt={<m)polx
z2U1?ov#;_$_olWnbSXz}ZJ_Oy%ePq`C|wG>YpC!dc-@zfxToInZ|~l=6`lPy>G0cI
z&+gt|oqKsx&l8)uiRX4@_m<9O6Ip!FZ{fjt>QDDQtG#4lc>RT+)r&i&ulLQ;pUI%7
z(8$@~U$Hd(?S``6>pyb#KW7Z<(D+am#8J-S8#?1dbxHa6|8I*o@_ncgD|ub<{ZW27
zo9zeAdly*x1y8!mszgqyx&FHUE|bxO<=nwL>+c&}4ZJ9%mnboJZgYC3n3{%oU1wYW
zL*e?3KbAlIDeg3NMdYvLszwbT<?lUHW#wRGV%;Y7KjTBf!DcpLu8ga0iicadq@{A4
zy%;o&mfT1YIgpm59hMu_nsWHm0^RCux7PaH=<}JxVkmsVN5Q~@fswIEg+oc8!BK&!
zL19Kla6;e`=E+<MRv%YhSmwQ0OZU{47Z;l&*ZV!my0AKADc9UKUG8hI4zMiVl96<D
zotX9Zyy&&D-Xa_f3>+;?8(*4dT|3~--fhZVnsjBe8|U=9x2Ar)u%}w}@zv<cP51Up
z(^O^%FixAq5xB%-qC)CvwJWR4dp*zIDt&!*Me_1%Tb`D@ytAzQ^*dX~Q|~#0TZHBF
zYPvu2p6r)?Zsck?T~}sV!NWs|-lcj>oAS?}YYi^@aP??Ce}3Jc|35#yxIUTj{K~xz
z91Dd#ZltWf`ZsFrm+Mjy22bkOFua)Gz{HiZfPvLW<dT!RV#fl8zp5u5c(S{dJa7=t
z?3n1puNmUq%9-SHsaEx0dmz)il`|d%>+brY)MI#RM?{$MtqSEnJ+ZAvqh=lWs%-uB
zi-(uc+|p@_5-gw1Si)ec^)sbMJ8jydiNWhST55z`7P96sWf&e4OPI8gQ!=_IGA%n{
zm84pB+BVB&GZJU*Jes>bb7BjN*Mu<jyd^W;Zixi3oO+%cY-7$^)EU(NV$nIvTN+FH
zy}a(0-3VPJyNqeV#kmf%^H#+!pOwX>B^fg%Dob+Ns;X71Zu@Rw^;S8lr9DO9?Ei|y
zwagn239gqez4Fjih4V!~>(&)l{B%#a{Rr}@HGTagptV+2Ae4zgHK4a9(fe&c_{Y@H
zq+rF;>pLD#+8WN}!aMCsY=$p)-uB?PU#A9m+-MQo_i5JBcX5ALX*&qnY%*oj%$vBW
zQ*4&kL_LObM%KJtb*5LtciauL*%)zMYf0f2iQ?=TJAd!8-njc=)7Fh?FK%(qVqto-
z@@Q7(J~iXB>rbW`pEE!I>{d=t!Ai&cJ)E*81+L=dUy3}m)n$rK`I^sNa;@~a?A6f8
z&+lG+wsX4NwU>L3pLB@yS~!2r;ieVZw{#zQ3EomvVwqHL7FRN<ZLLVn#9Kl;V=s8A
zaiwr;26_BHnY6C;wMBB^63>sqljbdL-SN=S>GqSyx0PCt<ZR2lH;W-KVC5o5pDw=0
zrwZFGer}m)ZzdQJp{95=gXzjo)!FI0D%ED@b^UxcyXcVR^LK@P3s=5BI@yqG;rj18
z|A;K+T=o2OGgsH_g{4MabBb>mTWNlunB=~4@$JB0FC7*XRn?U;ExjMR@@0BBTUpGK
zu2oJ^Z_Kql->%`^-p&2#{I}aIm-_z4zEI(tIor)*lEa?4Pn0<=Co<Yknlx2qwZh3H
z=8TYOYb!+El4Y9bY>6t_bo}gUO-5k>O$oP+dzNJD^hhSY@>xD(`I!ktt&R-SObpus
zPvjKxFnev?z$$Ud@>r3f(*N%6fN(4EO9qDjjW~pOgc=(&6cd?dN}kKww%J6HzuS7s
z&2!skxazbO_N<VcoSCQ0Xd{@L;mplf&?fA@L^M6KN2ss(g9Yn~$abr1o$|}>Jh%Ih
z({9uDMM~-VV!5wdj;ncQOv+$M;oj_c!d^k@LW{s71qRy}uKb&%)m0a1{})`6C%x--
zutbp=k131&x;^}^9*T_zYXeHR^-L6rILNh0G_zp)pVYMz1X?fL=<YZ+XU>tlP8Vr0
zzfQB6NzcX7Cz#$`H*4o_S+n&k&l*Keo_%|dti^6gKcmd7**gLkf6Cjeestq!^Tw*>
zB4!4*l~P`oT~2&gl$M@<SNJ|?-6g)C|CwH`(%a#+;A-alReZa&LYHPKea+<j=kdut
z;oc=jvA_lE?LJMJIAw{0$VztBsaLuF>h>fFuAV03d|hSLhA&ILOFaq<^Uy8hT@w38
zNnA+9pljl`N4je|7>-_0osuo-spVJrD6+{)^!OBm_#aML5w~28W*^(q;&f*E(Uz*9
z<x-*1{yo<ZD;%CKW@Q)`CVJg*!ONwcJnPpoWoYx?*9zah?P_<~Qc3>E$=4s(7OuDX
zqb;7Tc4PNBo=UsJz0$WmOe`*4wCD}JwQ!!B>`R8CtkBi7Qde%<>a6u?)6<(dHx@}4
zgvS-xv+Y{aD)MNX&#hN(tdVPb11yw6x?Z^Mcoe-RLGwSu0nT-WEy1y;i?*KWR7vF7
z{pF=<@*{8MOA{6uIX&xHF#CvV(+AJPq8#(0TC5o-WD4IoI(PXA&gDgq@+uy@eOGz8
z``*`8+xKoy)~`M(eNO9}CTGke;ROrKS^sJ|R{e9GvS8hIjkh0*=l>RoUv`E)AbjQe
z8oqy1R^B+i@5jchx=*t^L~aD|&wI9hszT)Le|e$NxBb2-lte#R7SU+l%XPqci&BTD
z+6ke*irq^%H%w30C==LIvnJ+RkcY@dX=i1x?<H=!dA;sYj%67S^~}wL`lo9?Z9XHw
z8MpD%Q74ZlmwNfMmnL^j-jI}<W|vxYU6wnpDMP^bgrCB*vnMo6ifk<ZAD=1mtXDh!
z{;ccz=cf7;{#`Ub`tWrJcB^ysvL=i8-(Hthi@MOQW-^E4@6pQ#d^UFazDb^3cHH@8
zV86%=9r^XavV1AiCOp+tS!S`@BXskvY^T2d({GCHH@#JFVEdWk%yYz(Ez!eKuJrcJ
zZ*As$xtA7mSuXAQ!aOnj*eQj*QzsQK)}5e{(V}91D>XPn^KI_+E>AO|o+EA2*TVT*
z<}gXHYPuEXiEd65(W=|N>tgPU67DAnU&Yo}oLv3>%Ovr=f8VK9-RJ($m~Z^x?w#^?
z97`=q_1^n0t(+^QcH#a@R<C*AWekpSuPJ=*s($d_mnkd6-h7<+e{$9utE<7=9(eKX
zm-v4~X<A@MqTR&URFlPWmM5Goe#u8kYA-n{)X>|U6_gjty({9c*wbfA*G)gBz3k<p
zmp;=^t-s)LXopwcaj}l3r*)E76&!RTkLz4Jv?Sa6)!Eh?m(;&~S+ZUvCQ5a8q_5c3
zj>?|b@tRdPH(%PMCH162#_5~p(>W6la_{?4y6p2D-nq|@K90Ot?zd;Th<@b>|9zio
zOidQ6^S|6Hwk<80NouZMZsz?n7dI|DzolRb^IW$?_6he+u1VjWw)J${+Rahkr=(aH
zEseeOD`)GrgpMxO#fiILTxXkjVeLeD<=~<f_tvl|Fx<Une(Bvd(dY|i+jBW}RVFOE
z6}Cle(kA`7sah!|C;!_#J2p$Gan3$Jg(-4zZ4r%mK{tGlmTdfg_m%(u-6!Smeef&a
zy>#}Ui_&f@W-pU?Fr(&rmF5DbZQ|8OZ!*v0WH}ef61ISO+4KVwPg+iNv(lBZ*6bJ2
zIA8bkx+Q1Cx0E@JxiQBZ`F})s&D5CDY#PzbDxwf5(Yp0iu*(6pD~H;i_OIcY(9Mu|
zMv#SZ;tYnp90E*pJ1)%WT6n|3VuJIZc25-!ca8%aw;b6NalcoQvyahd%Z4e(mV{it
zdSSxG*cmewr*gVxcpUsSu~lM{yi}Z|Q18rXQ;%-zm^uAvqKuiRp>3qpGQMev-ep^P
zGC7W9ojKuVlejl?%K8Ks(O+C$j*Hs<XRx>hC_OwV`6zMGlT^8<fpY5CZmP%4(}<GO
z^jffZ?xSaOA4O#xWDA*fc<-YZGue{o&RdufC2M+op^N*9pA+8{wCm53GFTR+wDy*P
zR8XKzP>sT4u?bFfC%A69c&wZ2(WEhZUFoFOA5!amJw60-m>=uy?48ug5U167$!X6e
zZQdhae@=VcBX7NJ@s_#skIacb)-eAyi}=!uoXl0z9VHex*d%>UJhJ;PuRDv1<2!kc
zkMd4pDZ4k`{==HEM{vR3slm%%D3|-DR4l#DvMAgwPN61H!M*AF+Y{oFIS=L<$Q<@$
z{IW*SYgNjTz&n{7GZx*L%(+a#*URhmL>ZnmuAdK7-u%}TIklBzjf2Qq4-rAJrPhy5
zxL%d%4rOul5c#`z8bd=!Z|DVv+(ThsT@71hpR`2B*YNBL;J)nH>Ko&|wc@61UugKH
zy8(B)c?GX}E%g(4IfIwcExau@c9TeW#cA)gcW?dVzOFe};grki8E&Gvjc;CD-MN>;
z-j1`|beX%#B4w?mDN#zPev49-;xyCJ)U(vm^OmJ2&3p87CX3gCL!I+7pRv8Vn6%7u
zS}{k1JZsRKqiL&;s^wO>2e~l3+j!CADZA=NUXw-Fw0b3SOg(PZBzM*(s~XDx?VWMU
z!=v@Vw9Xq$juRMl_a<6*Ei9FL+S?#G-L^IT%HwHsCMS9=aSVB|^MCKMa^HkGg(utv
z<efx(oDZb`J*xgs_KD*o`MOK<cdI`6!}~yEi@Lp1LPOp32B8&=U208LOR5$vZ!*eo
zw_8!AdfrNu>r2amR;M|wLH?Xk{fp)-;9B<J`x=F1TNHRE`8Ss~az}X`4?UiJ>w)7=
zkJ>Fwwj1uOS#d6xci|)l)w8Z?s@ray<vil?;<yOk1<#m;o3^xs3LPykO7xi;`IP0R
zBa37-7lZ6A#g{uSyr>D7Xt`1@=%tH8W>}H8$kpz!Ra0hYw$ya6Ws0QDR(;l>b!4{M
zGKV8C?Y`Z0$mw#jP;0AUnXkqn{O_fMQQ}$MfRwN_)m74SWV6)Dl+?^#rB@}T|Id1~
z*e+`EQ@$6Oj5@zObrz+)l=OOaagnTIdx1~8+}lShj`siJxOC#AoZ-27tFy#*^0s*0
zYX5iU*58N=x7P?|Z53-&X<7fY%fw24(?O55SH<o>aL@BikJp)AW|qC#%1?x&!E4w3
zifb?D__4SydtPapP`>o`$BNZ^=1Mahn>|NQ)k8vM*RAwj>sIg5Qu)90c75CGJ*PDH
zN+~d&i)1WXeP40K_I2U=gC6Y9Q)!WksX2R-ZJA)^k|pyVoP2HY#>;Qcq1c%AJb&M?
zoZ~B3bp4xr>{Cip;+vys?MwHZFydOu&!%npXodN$Te=gSq%Xv*-w@&PL}sl`s*^|R
zraNyKmI{RZ-xbmynVHygH?T)@N9ofkmY$(YUGII(@tvv}F)1?U-%+u5IRR^8WxI~u
zi@aFTwb(I-V~UPi&x4gKTDVR$=FXq?_Uz1D=1LY}yJem{OH%9>*>@#B=sdOLT5kHi
zwRf+*&6YV|d(ks5<5lLTV>-*ymRYu^IQAy)OY{Ghl53WiyR45z=BQNfo4+Q?g5TIb
zEL>CIbfjR_8z&3(f^)rh&xCz0&D$vUI&Y)PSI_hst?BuetGgTc-W+<^YxCsa#HPAO
zc?@Dc&PrMhd`Ifzw5vb8ZaNkHh;@Pcg6L1@dKpjM?|yLh#ijI`RoY+IW!Jrucl`Fw
zHDk)&V=H(4j9gyxeqZ1EAOH0-nr7uU&wKA)_JpnNfoIc#c{*>THa_9Vd*SW3=9|Eb
zqeU{@eJlP8^`Bt;aQv%&x131p!%3%Fuki1a_%cy>qug>KsizkAq(r?IG0Ar9bSu#2
z><GE#WUF%8YeC24(!{V;Hzzk_28CpDsUJ8T+dcK^jUI>003BW9xrr}YUiKb%?#Ls}
zpWeZ5eZQw+t-yUv!S37}m);6C$IScn(tXKVJ=2fto<uDua`C(%zu?ew2R)_h|L)#@
zn7OEK<9*9l4|_i@nzZqe=PJqbl2XZOk1S_tPU6(O(s}<A+XFUU>Ais$D^_LYMXh+G
zRN~Pt-#P2b8h*uQuQ#m9t4=PvF2XuRX|ec!uhk+G7&C8X6};;;=X+i(c3b1&tU$LT
zW^DI!de>Din^b&ln(mcDUpDG_?s(UAMo%TbQQ6o}_wOYg@sIV-<aa+Sa(p%a_P0-W
zBU-v-TNt-Ve~3G>T~1+#TJfH5%6s&l^xG8+A3odZ^SLN&z3J;BC$1%XESKz<^xC<q
zcyD9zJ}Ig4i_sZnkJK8nf317ZIxSenJ7d1in)xPcysUD*+{phLu;yRg3a)!89eFv&
zlDdxnx_tcFf}eXjGAA7DaTQ@UFZjPF=1Iqq+mC$DZN0JP!WxFk4d<Wc9J$eb<(<S^
zy@{OMM`joW>TE9HXp6mgHq~a|(r%B;441~tHwE_d)_VMS;r@Tinw#%TOtc>fT{j7k
zcVSaE6zta(k~R&JHx=65k-Tl~EhoeI@47-hpStyl>vopMouZFo=hrThQ;=S6c(3hi
zxP8`wn>P>ME7Z|qT=v%R;k${^((go0oGZ=AS!J~D@x86-h0j-?&9CtKR6D^Ve_5C+
zw~Oj;twekEkAWQ74?CJ>Tq<yLS^IL5X7iO-W~$ol@lSmGbls(y?#!r(m?+ZV>ALIo
zm!Ko{A;wwrw`KXScid#A;rTxM{PKS%ziBiZwHfH=>95bT&Tlfw|GdqkW5T4&-|oA1
zmFeAo{_p8?hEIC;H1aKWYkqLs{y|OcN7#3(#rkE|E6UV0YVKaoyiwd1x1#m`DpT9?
z2lb+-eBOQjXLZWHy}$aqev_a{r&vMhj*OP09Qi*=C-JKG?UI?2k$2lhX`$WylU^bo
zvKAc5+}a)ulib!$*K_-sD1Kqp!$6Co%@KV4TTjfel?;0{K}1pc{>w;vFH_GgI-WCJ
z&UkI?ntr6q-LQNAj&AqDTVgIIi{IY-^<}a|>GF~zH_DE%p1zr!cZxOq;pdY(COU5w
zi8d6C|0w+0AjNJc|91lcd9A748y9sIN~T**o4-+NyJ5zn&2t{IT{yG;5l3$n``MZN
zz6)5|3tw!MeOV}{drIE<xcuB#D;D-E?)E>)nY{9%+}qr4Uz1G^J{PS7&wQDB&G;Bc
zi;l;}|C(hcK?SO|&rQ8LijH|~v^sIW#Go!>o7KN*uX;-Q5>9T>JttOkvP8VHFwIEN
z?{mTCU8}cSt=V6dBW}KWy;X6Ya`(qFljN%2Zjbfzk2sg)ojO&O8NYes<4sOx@eQ*!
zTkg)>TyI=*T6^!(;-@Q(S^i~j=gK({_u%c3kCvu7(>3~Rrk8VUN||>c_-Jl|-D-3D
z=$wvi=As;5PE_X9Hm})xyCbtf?r-gfHCMC(pPg-G_`#x_6qIx&VDS!@y%OFr3q#C~
z=(xV^)zzKpbkX620k3<FNa`Q4NjHOCR-1&bUbi*K@rQMN^5T>4XKi?M>tuJ!$=Md2
zn{B#Ze+jYQy8ga}|9|$ag3^US@Am|Kw&_`-D{AyH`1Pca*CxX1NxjOo%HKBJDJm5F
ze~K}#a{A<t(~Pt3b(yj(6I`}vYv!ubS(D?U>sR?$^+xYrcJu$mdF+!?Z<@wkUdFz*
zZ<@d<wKr<2$<m)#)m=r(T5fFXeY2|jjr_|#BPS1zkauDpxx2KsA3BsE?^@_n5SXoH
znq2a)ebdY^XO`PSX?ClgN8S1`>-lP{=gbod)_+?wxo$1X{Cw*Z`QaX~&d+)MS?lHU
zZ-t9rf8DqGozU$~_O0K&%o&5eZ;3Z=PTT&AZ@>Gb>$Y}x=7ngqP1=3<oA{A)!3~l9
z?d!H5J@&o&|NF1X5qmFx?`qPmlNWpOe<G*%2kX->`<<;<Hy_YF%_n_d{o(%XFYK%H
zoIM)duOGQw_1vK*K*1<js?)-OBjd7{gEiyYoU5y^<|S&ky_FHmaTQQH+9{+fH0Oku
z((zvQ=t>^WNmq|d5WYR-PDfDcsR??~8#Pa!Qq`WRn19P=h46BQFe}SVA2l6TFf4XI
zDHA#?gkj|pf7S_wUL{(ySA|cGI%^f08Ms)cmg%Wwn1)}F{n}Zs)zh>hf?PdidsmmN
z3)&s?agyZK5dHgsL0_5vy)G$8IMgI4A13o7|FL1O<m8(o6_=--ir%K;so=_#t~uYj
z-A^cr^W}wk*8UR~RUMjgEjq@(@voKH^jpDZJ`WH5Z*9)Ix6d-?=2^!Bd5dGZ#Gh}O
zeY)uBiJ9W*c6%*9KizGqD8O(auZg9sx9px$>Ys|0+p=zEg=YV)VtkX`xZV3g-pTb}
z`0q`ZxBnNg;jg#SQc=dotGxFuj#$aMW+98i`DtZ2nHLnAmUu8Qa2q~(=kQ_2?h7j}
z_FPKp@#S&hP~=pu3D_&@n=;Y$+m+_^wvw4K`i^~hUpyUU3%{H{F4y?y;);p;q%PU@
zesT^vsWLOB=mf7gkCDcFx5VTr3)ZP*X)J8IQ?PIGw2(;$7*EPvK4Ww`<>?yJomV~`
z*y+|ddCJ~LON#v%Umf{;ZrPih9KXH)R4$(1&oZ@V%1JKYDMlwHRHyv!(Us`!vhlF@
zUa<N6k^rZ!i<54IH2GP6-E#Trlo?K3!M=AJY;HzuJsWyAp*HkdMCRPkFs+!Qq1&JA
z5b)dasOIfd%ZFvkW>)%7U;A)tPKzz;4vbp)+_Bwf)(0j1kIvsts~<eQZQrsrS&!Ei
zPW8$>(fP<|r9$)@&l9nWZ{1|-WO>Hg6vxfVz_f>D5=Z|Y-=0ZbC%;Qg3A^=OA;jZW
zdq~TNg{=y8Hx{&WMQQBk<UaM@Q&m6c`jUP}zR0DM)0A#LnUpze<+52RPTd-Fd$h6+
z^3Lo!mA0awFH37q`8Lah-0nYKX?nS{Ix`lgINe&dy~OF4?&4QnyH@UgVqB25^jp-L
z{~Hd?d1bZfumZQW(Z-W<vjc;4yrYe;+?ck&<eZzf@4L%E(Yaf1L}lx4-;;b=cl*76
z^A?#iZj+d^UAJPX-^Zhp`Fjc%Z?S%5z4(KuLQ~yjiSv<ZOF3qUh~Ms>u;AZ;&He{3
zrcXP+FW>x4|AA#}d!kn6+;zUNqCC82(;@E4=ie`%KPEkKCCgMJk9{2Z`2j!g%*_Aj
za$#Y)+_4Mka||7&`EMDzzbSrm&;D)hea#xBlkCD-y(~5wwJJMVqck^sTO@Q`Cvu_v
z=>^;G3fO=787_a;n9)MPlyib^)+3LYH7ceyM|JABkKAICmN>ZBqrj+dQ^I0{h=V+O
zubQM>L>4bGU_12xps&r0TT@m9oY35!`pP(1L$uOQb#~}YHnVN3uJY!p&WRFD4`264
zv7}1E<=Ky8p}Q3qF(^IX$5D|MdCICQ^~46b4jY%)o5u0Kzo^USAN212Cn&z-TFR6u
zQ(D#78#(s8P+nW6%TxJQp;^Ra0jtJ5R-G$WUiGgEr`X(5b>jBykQbd~;J_C&flc{|
zye!-NE)$ooo6#!Aw3-AxS7}d@U-`r{ic#1nD=2mDk<N=f+k}n!jC9;@3k9t>G-cHf
z9_^J^ZiXei^jiPtiFT4<s(-=GEn8QlW)?Jd$J}A}S;2W(|ENljA@AffYgOH5#v4ng
zY?)?uCL|-IM4C7K-Z#yQ7ta1SuIyfTFUeRs?BX)hZJNs;G%(8~$+BBsvOE{K;_;jp
zA?}6OJC`%qm5QiHs4`6s?lN1vz*54@n_;f>MVBNM+wULJ6Ut-mI)}E;mz}!KXT#=O
z+ls!({j%V2I^`Wyziq|FKe8t(B%8yfgRcaNF6>yUT`JfzN8DCYb&A{d;0wO_OKj)7
z3h{C}B>1!Mips*a%oWd)<(vC1sV(JN>1&(yD&W?EUHYA!^^99y{pUHcEI5nXPk2x8
zyaT^NH(ugws^^*=DcO9DPw!Q;e9IN~lgcZ(VqY20x+JoqPJC6s$+IC(8x?aHPOp-B
zG+~3;l&O5JR}aT?az|V+4BOBlu_m51SkwFeleFh=l-z@OjwlLtpV;Y8r28vjO|-J;
z%QtOCdaKuj+09Ws5v_4juUyiHp?b~RK#el@Hj#X1_StXqdD&tfvV|wso2WQ5Ng8%2
zu1VUxHNsPK(vt3kGs&{tH%r~yHcg(mM(qykQ;F?d)2?K$PO~(AfA)fR<op>%nf4b?
zO5D{J`M~xx(?eRVLeI-|mq2jii8E303o^|%*iJr>DlC1=AXxHj%;ed5#%J$ry6L<7
zWAC9_?&Yn;$_ypn(h54Fms>AqPCDLOab)sp86nN(iXx9c&tkedhhd8V>*5kW?!d+{
zpSA6`)aUjkxXsjPc336yMD%5Y*wnj?9nX_DtkAHN70vqp+iBI>NuR!HXyk61dVQ+g
zRJ;DudRYQrmx-sf+?aCa!^abS8;;w~`}d-xv}E(d2Ri${s%D=1y*tU^b^Css-z!R1
zy?%7;Ma<RzytjOqV}$$WnQdkBJL?r!Iyq46?EzcI3yo9nb+cRUIJh-$R$w0M!uYM8
za=x`}A&X9}HghgrB|go)tx@((0&nyU*<TZ`IrVg{?VgdXBp0~%{}-MNgKTk0`$8k8
zRUBK@HhCEMc9fKF^WOaX3|C{RuWyRW1RtYm>$+D2Zp+$}tB{{9Qt<qu!V|vjHJ>v2
z3)Giv+p3Vay?(b_h4}P)0^WH=oNPheo`p(1DN9t8B|p53GT*JH`NPZT|I3q8Y9~ER
z5lq#Tw_7@&bLpO_uBBlqH@$k2ZM?SCROnrH60?d*)y@0)w6=Qh!aaYcKmC{b%xG)d
z3<i0L=EXmHvWy<@-gdV4;LQ|++sg!IFPL0B$JJbY%_+~;J6~3rA667A?0M|B(z!kS
z{KUS}4Slk&Wjw5Z8J)7TS7~5zVVKYSRA$*Cg;tFxlY<p!O$jm33~p^|U+9&XF`H9%
z_n*3@H{~4r^+T){vUi%C(tMsYxp4LmZT)#C^>}N(DC8>{UO#bW>iL&X%Wnmr{ms5Y
zYGra@XH|&$)AY~5Mti6E$zRn|6y3UT|Et%ROI|Dw{e9xd6n~cQCSkT;Gr#XPzp|i?
zU*o?MtJdY*w_g^Y`{ou9eX6CRbl=MFugpUJmWHm&o4O`>ZtYR4X|L|&=WJMYE@GAR
zKaFX*(iJH^mW(P7i&%9tZtXBLN(s*HnmhMVvyJ!0&YOB(>fW7dtV-LStc{(b)Su&h
z=iFI_^KTultYyf5epjJjhsuQs?|gTv9Ao-<xZ;~`@}kwu&aau*bMNNK+r4EqOFYw*
zs-(Ri+S3AFrq?~&lW-xuAuU7UT<WA}sc*Ws78zXp#{Klp@hq+*-x@moekC2?@i~yP
z?f^%QuvbQKNnf6qa^4nAE?J#Jua+Is%E>P{d?>2<@G^(PJna@j%YU!yJ91`2;Uo`D
zD-}ki_BTH?J0~doXIiu%GJzx5p#7~1L&g=oV`k~xPdr@gGJd4!zvuG}WcrvR)FG0U
z=<CO>sNiz?f^xD4`>g|qw`l}^R529hW%_7gc=}rF9i@+glb2TpwVXcm=JccHWviBd
z$~kgo)AKV46Wvojy16B(J>prBx6$*}n{!<JCm*GsZ<zS?kb1XST5ZonfsI`bJZBqf
z9_c=6(U`YUU!k{AO(X4&d(*N)DaN+T2EEeV?|<Aowf>kh6Vn3YWebAeas3L&mA!be
z^_Ny#gzkn6j<5vY-vt~S>onxEc{}#>^K5(HzNqidlt<TZxG`MQd$TB^=*D~JKb=Jr
zT$%WUls*S2UGFo#v@TbxNNI7y|2GPgl&`;geoSF<&sminzV`$I7-csIHD~s+G^oZ-
zy32MUHpz&opL=4C(0!u|$4gF6{KY$oO;4Gj?|~ZU<c>|MJ)c;l_0&C9s>dwTe5<6L
zBA}(sr)9uru)s}g|IsP4E<LX3d{WY>Rm1u8o1enZ?8Zlno+jx%OMCb9#-+eHO_9q^
za@Oa~UJ<}4;=uJh=uk-Vi)|BL9PylEvc{BS`AaVMxjm0xF5{bbYvu2r#a5>-7(VY5
z-lNjFC+~0HqIiYpMK4~pTzSveuf%`wFt2X93I7pmm&x;|JhJ|k`sNuc<DZvtF~##Q
z1r(_`aI>jANmwGraNhA-#)myETZH^oA1n8_P5z(I@p)U9vE@WzOX>F?qq0sb`HO0v
zuZsA%EK2Zm0ZWb7`Q=51UmVsi{IJ-4O`uKSdG~o;$|38!WBe~9oL{x6>zfK+<fqlE
zEBM!(>RR(^ve6Z-9}0&Bc#r;2d%RZ8RmSvikXvr+mseNj>0e!@FUNXXZmYi2$G|^_
z)^9wNzfo(7&cVe8Egn1TJ$LrpY;98F!k@y;)WU1mc6*1e>V93uXc5N562|CH8dr4h
z=4Gor&Sh#peV2oiS^vULwPQ&;GXB5e@mz6wmy-bt_c@lpw$z|ysR?O&_NcI}6xq9g
z$!1T|%Df9~v)#OlE_ff@@~vh=)|G6X<5M{L4Sa7;I&h(__*nD*_;~`RAst+y8GbuY
zhG_ZB`O_Bq&BQ!Gxqzp=U=nlT0_9g?&kfISDD-*bU{Gw`w}@BK)W%Ku?HqoSNFGx|
z(@3eQftEtX_N~hNF7YwJMOwx4Z{*KklHlOsD9Cz5u(q*RZ({eGb4$K>xcM6i^9p?E
z+3ebGtXNk3F^^s7@#53_)`;xYaSIR>Rb4N7`Ha=vsb}H}#jdaME(okyW9m~T*xitp
zRwpREbjqd`r|NG_ZaDGv!mN)^^3Gq}v*^+nZE4-LQd>Vt@4YB}yNlN#)kaH&vFQL`
z<cc8SEq>t{SAMO!`eI{(vWQc1s#D=x`R{@9mAx+fE_}|+^H2F3yM4Bf6aJqxk@d}+
zrex_-lVhq~9zolDF1aRO*%q)ZZ0l0SXd$)OFxQ8??(78<vWzAObh=mf?3}TB;tyU8
z)^q9gLdtFe56xaa2xDWF<$LmJv2RwI=96Zx@RgYhS-$SLG|lZ@)?uNTBSFdUou;-0
zT#V7vojBpy3AYN5g)R3=avO4IPkNwtJ&<#rfqwAgLw_daO?=0l%*?a@gL#*-MVqtL
zf;_`Y*LhD?8ouo;*zV_Ymz_^l-GrOV#b)Uro%A<upG}VHz1g?!?-l<y7GauhR~0pv
z8|`&5WmSF`^)R&3xKhC^b?Rq}*z)kH#lqL0S_ZEwi|&e;o&G7TaAC#w$nsZK%coj5
zKKZXwcF4N-yEVUx_vEbAr-Ca#9zWNX60>ystu0!1<|>!&e3g8wwC2_4)hfKcKR-(g
zss`{q{lR2nu)?3|jVhy0Yy5xHRuPln7hW>yFMjcTtK0bA>105JO7}Ij?(B)IA>W@C
zF-l}ug){K&_|xT^GPyS;^|b?EK<ZniCdmnI>08|Nccgk7rKN6a&itfSYtk~aJIO|1
z632Sa2_~NQ%TmIcJojpA?havHD`>Oql2@)pdYpmQ^zYtN+uC<eX+J2mze)2_$Bs+S
zCi^Jz9+(o~d-aOumIY?V3lEwtKKSZNIYU_Mt!H^Z+_-mEaPPJ#F!Cv|@d>!CX(?bD
zD13E(h)%=S<o{;2-+A9I^1IIO;`-TIDIrc{d-7A|#_z5n!V8RCziaGy*LR)2%dF8S
zO2wAvn7C9+iP*EUQ@J1b*d`rpo|wH=E^e!<VI*Vm)XAJ~B94+vyGo~c8b(euT^LYV
zUOPSV;+m<+8>_cWc3dSY&Z^?@%l(TVcjTfc;(|{k8z)Zp)~;GS-8NZY^7K^6t0~#r
zr%T-pRoM73c6Z?AC6bqqUFZM5%0X3uLr8(a;E46L?Xw?VvDP~hB=)8}<gM)5e7TS1
zb4#5Q_pVQT_%o4v#wH!{lIJs%`746IM)avImStNcc1v}^{i9*N+ZJd_L>gEuhz`5U
z5|w6KaW6xxh4tjllCb+T-u|Dcw@khHTe|&+Nt?P=B9b!t%{1-LJw9eUW7QOWoxJG1
zR~~z<o4&88O?d78XOSK4oVwi`X3qFAvnAc&m|)w35Mu@ghK7U{53j^^{c?Kay3%k-
z+=GzWr*GzM`Jn&S%%FFj!SAyC=EeET`^<$l2W))zYM%4!4TtARel@!I>$RB*@5|Qt
z2A`e9g-qX8#92(|mEWppEL>#f62GH)y}W4S*Ox`c?eo_L+Zshq44v+}_2~ltY2nj9
zmqv@1WnQ(6&8iH^sw~i1xXFEL^>nL7aqH%CYwu+h`#)E>=f_SjZ<F_}JSn_&;Twsi
z+h?Tm+sX50D0OHS@Y{U2T>biU_41oFjko^?+I5H=Yxw1&>O5h0`4sD&Gk2G}9d%^)
zKAhvc>*KV!s#C62usZ)X+<kh-vc1(VKO^QFe*f$8Bw1m}!kbGQB&*zdllI=NUMMTE
zm-Sb3=jpo*r~BpD5}hPGW@~@!n8G~siHE6EtLuhV&kwELY5T&gw?=O83cjV+U~%YR
z$f391PlNcXFHha{LD+xVj>F7njO%^{_x<Yklv(L5<Lo2DA;7>Opm5aC#rU2Y_s$F5
z-RHPX-yG|03V6!Arc$O?a`_*n=SOY@*!cxJD=Bm~#T@-`b1if9+TW`4KeLwXZ7vcK
zil4N?dBReot;&q2lDWkM7oNKv+f_V?J7Cg8_SEx#ZW~qppKMxr*7&B}ojsGwuij}e
zo#H83?t5{{!JNpS%~Ka%J-^gc?5d7+Zs0k;Mbl0msTY}B`)rZ;(o>DsOzWh2y(U<c
zT{*7I<tu6WcZTWThKhiW>!Jzsu6>v{v&q6{R^Z&VlitTAddxnub7x}h&eXlLm%d-m
z^X=CP>+6ZG*Nabt3Y?y+`z7$#$M6<g1;!cu`)@bQmf3LimiMotzQI=-_r6^a9HzQ<
z?%uaesku`Y#!kC^`R&5OE%(%ZrR9pL7ncd?-<oK&E4^7L!}!nx2dRe%N%tm(X)5z+
zWJT{SxX~6HV7KqemwiF(H=DMGNXMT2vRnFv&cSapt_9_szNR@tpe@AJ<e$ZVhAFpJ
zg#LLnwd&}9iN}_9D?R@>t?SHRG3&*eUr#P-uKqnOVEe7ryrqWXp9?nk&C9T}o>6;L
zrgYC#r#<;`yc1cCf2KBE{<Z#XW$)rU8cdQeS#JhTzW4Un;`<Q?H!Nk{$i1xeoyOT(
zp4X-~T_=CuARoP{Ke(*-_{SV`SC5?$yL&b--+Vs%)Tf&A>ZC|7tNtzdJGUI1d~k00
zLGF16?=QcYeA_m)|4GG#tp}a2uuigD*!KMgOPqgf&41$^HxJBm<XxY(_fxX%HG9_Q
zawZ${YOdSv|GZo}f93C8rOe?4OPzM}|2d|AGjjje^ffkTtoOC|e2x6JcW3Ovi_&Q?
zfA7<r;65??|9-=Qbn^=>wgz8<W*l(8(CR%QwZOpVtKHr_i(lXOrcIfCNd3<*ZJYaz
zYWs!cW@OCz^I`MhUW1uR|CX<fIO1FLIMn8G^q!;tZ~VP}|L@FON8IJ|e|`G5RiMD^
z4fn>X9>%8bhDIh%A(0cR3lFvONLxK=N?72+D9FGwMIg|DsZ&}t>WYS8lMAOj>y#%F
zMF~wDlCFzbECZh&XK_%uCez`<(8OV?Jnamp@N$<0QA<Q}kFE;Uj((eSQ#jy!m~Ch9
zZQc~FtJe;!jNa<ht9kMIm5l*M>twUH#ogMPvHY4)!q?p3o!ee`ebI7QeQ%HV&sVu$
zuZbn>Yh=_njo~&-3hT@^{Ll91#zuw{lU0M$&UiUITQXC<+|5cLQ~Uf{k6t_53ni~E
z3P(0iJ&|?gz>UqR+4C&AuH<%Zi=KQcX4ce~_Yb$S>-YVsSRHsMd9u<`we1ZrcHQ*3
zvqU=Q+x0uU`QPx)+uq=t;x<t+U+(YEPp?ldP7e(^CH6B&MP<SSmU<?J-@iZqe;?n#
zz^9Q=|MTMZqax896BNX`{1h2lf-NKxTZQ>l5?e(4G!{0;<UCx+px9UOu$^fl<Dw4D
zMI95$_4aWj_ZXe?c%03#?Xp`?;LXM1A<=t}h59)9nLNq%_bqu6>i^L~bTX5{%PAp&
z?=(_F!oniEr*1U~JwGj|^5#;8$UMtsGyaElYF?Z+b?f3Ovl=#9O3f+IG)<pVqScx{
zZ;|Fyk@;Rvf2gN7Y&yi2-l~);tKD)g?XE-bE{|i`6ZuS5GECj~<Hh2ck;h-=Uv``x
zwQQy5me^k7CiAwQr=96>tKUrgq*cAR=b_e`*&Yh_#XhS}U|Ij`BRdPj0VRet8#q{^
zbr_Cwc?)koVRkxxy_b&x>(*;_9=T0d+;+Sbk#^X@$}UiJ@L<!ES*IHv4OLSFoVW{{
z1om*AJT0JpPxYhJKFOUNQy6|Ul?r>vt~4pw|JSeO{Q>5`+JP>XZ+CC-QZ$sAEVA>~
zp_vJOotmP@UHwA3!}EDXj7~btdvh%^e(%(x(;2&R#AE(fnNAfy>+|+&___Aj)ENn#
zq01tY9r@ozB+n4-y_q{(bFE~?oYrYqQiA3$i^>dH?Do|mcgK$xE{m*wMU@x|syh@N
ziOe;-b!ekwOiuQDv&U+CVqeY8d04A{&!On?+V2%_c3zKraO2=E*TBzbm*21T^k{Tw
zWU+f!@#N3l<wry~(mHmw&Pwn5*(O(%@u)?iD{oPkuF{mGmrG}#s(QU{|2^5JDz}r#
zZ#SIY@nrI@=lfLN?z^t1IwhPhUUgbj+0E48w-a{Od_H4uKOr>VD>wA(6@TC5Unaji
zS^K@BNKNX;8lF`ei@NlpG?riXTw)h7)s1Vv%g6oaOK-ece?Rlz!fQ8fyZqnv*MHR}
z&J7`YujG5)sI6w%wJJ>G(AB1A!i{mB6T&$im^Nk<>F{WAb4ao{)mv~J`YV&&BF7TO
z)HH!X^}(yC`@h1PqF$_Juhn*Xbm@)@*QRyXcf3}#$vNQ@7c!~-t-C7w=1xB$hK<*G
z)~GrZo^<eHa-Hd6s39uTAuL)psk`JzW6+JuXBU`kS~I6pE99fsgqkHrn>ZA7(-V^?
z-v|=hz9v=AXtDR1PetPUUj!ys%k-S*D4ua7!!@H<F!+K}v&Z7lG~-E$J?S&Pop7I}
zrWv~W$yK$<b1${HDb0E^`+9=1tZ(b3hFK?{+-zGsCs@jT>i$GO?bXT-8!JDR-Q78x
z!Q+3W_s13K)8AI|N%k}?`1N6Vp>WF_L)CVle=H7+mnR((5jnEpT;VzenM(of>rN_g
zb#|1fY6ZU9Q@JQ_UdOa2I?FA!w=w>l;ieh$iFL}nPlk#UKe`)w1x;c*WWD|a$9&o9
z^bh|$ZK_jVdPhAHn!&eg&y&5W3p2eYe_Qf=nU!aVpUByHuD?~J%o4ZwH)I`Mz*f4#
zRng11`_-9WNiOzYC-axy`+3XLHgua?qt>$D!VgyDRh^64$2L8|aY1Fm*UMjTa24*H
z)F7nM!NTSBp-IL^Y|Fe~YgSmDj{NKRuyNhgwONVYPR4hf!m~Zr6)yF55jwaqlK+yP
z;Kl5<T#?!l8q?PQzb+D}!FRcnLGGD;-AmKXzezXtH7wfDP}rWxu;E6{i3K7-SCmq0
zxFpZ`xf$)5GI8D~o5iOhzHK^_RJK|BwT^6Vnz7A2pDne2x)nBlGhVdvTE-gRTNh()
zdo0t<HS_J3x@J&m8uCbN^6VT<%~-9|dsq!})1sFa+;WiFw)SjrZ1u%^E!A_G-j(dy
zxl#4OHUW#z8IBA=58l-*xv)C%k_zJ!+hvuLC3iABI9}BL_|CI?ukZb+JLeqoL7*e3
z{r=^deQPYvtgQd<z4%aWqHyJ}yT=&b$liOKckn>=hbeMCdpM-udvmHzcADFs_{{T2
zRq~S$SL|w{ZF0|7SKfN@{ZPBe1gZaCfj_c0{<?L2?v$vkV<pVo1<^d)ZVL+Jv)n0U
zsD7m+FMJ_yyN|+}KA(_pTn_faIiJ|)=Q#0uRqhF%W)!@VCpF%aadlK{VoRDut5ln0
z49}z!O+H80$*rnXGd{^76mN8-BXfxg<8Dr0o2GP$(1}X+8<PY=izajMxhDNf>h!u}
zGpW9CBG2nDlWgk>m#%xVQD@3loz$Jl{PBNQc*dXnB5nUyboGR#TBR!&ul#IeH0N#L
zmfD|-SO2}SIbwd;gw7>ADI%G#-Ck8~+s(+Esn%)g{i*E6+%1_KbCZLfXQWNpaA)K4
zS<Rl0#maA;t5hxd^*p)w)}?7XCU)FmSS}Y^I^EIm%S5M2=8|`xS4r<>4ssAud*G9y
zmZ9+Z*aR1al>F*sW9P-2)f|sxuKdF$Q>yM2>%hRE_>+ZIgn@}chk=2Cp@GSPk%8ks
z1M3z+hYg~RjqL0~CB+vW9BN}f&7l#xENP}t3SW_+L)#g*pcIEmRuu-yj+14)g=AC{
zuFUY&jhAcj+_bF0ck7$0S)twy2g4R0S(Cgq@9Kp`OdaclHf(u$a4E||qgt=4%dW0|
z#L^?I;HB-&_-J37s>V9)8-cOgXI-t@y7u<=g4=njGp^;_oW_1}-I?rf+6N1hcFM)P
zHrKv;Fz33ISWU#*YoVF^Oy_QITF+ouvAfS`&G!7Lz0rTq9sRvs>-L^m5nESZT^DN2
z_${ZCXQtp&&#Bvsg;{2nq`dE0Q^BroF(;(JY%i;Xdh4Ewf(K{n-{<Ku>?|@nwpx9y
z-6YX(ukY@!{?G4kx3c1?=j|!!rgNrNeZG6K`uodyp{sv?i{6^(nilMF{rk>KT#|Eg
zTqZO!DV%6+<eGG()kT0yK+#b!Oygmz;IyVhX;Ir3iIUQ-GlJZfOcA>4k<w!%RHVHs
z;;x3~H}_*=e>pd_hMG@e5$;`k<C3z!%|>IdEa$aO-qC9uj-GP#7TSJllDF<ml_|H^
zCOr)aStlfw(p0)NEGEBns;KuO*P~OWwmB~4iU`!5_ADgmXyjFm)hSO?rzbpnd2M>f
z%_Gkf>t1PQEXWqUr8>PtZ5P+fZnIe$j!`{_Zu!}91^8>vw%N|98PB_2R$DL1{6*Q~
zX<4sU7XECzm&d;HmVoBk1OK(M*KDbJF2Bb9nAPJR>8sO%nw8jQE3697^qA<HI$`3Q
z4$Z<V7jw?+Y7OeD*vTU7ef`i?ubj1|+*2gp2&f*gdfew7GOb|Q>nSt+JX2qV_2o{z
z8ll~(DY56sGi!}~pWn5LdB=4vnwjuF%x#&*f`_f|59Juz%sR~LerD319N#-P5B{$T
zTOY@!EF*b%d6p5I_2g}84Z1RC*%pUhcDwz?@vPx?nKF~}I>u$kZ_X(`Ykm5T@j3I~
zGGEG1H!hDUblLtr?&7<n?st~Isof>J#&>(t{i|W=x!RK7OaI<$%+r-nY7X;sS?G}G
zVKU)XX1qt@ZP7z_+cwGZX)E2BqjBABMNRtuBX0K%eyRWHVSk#DyxREWkL0Cmt{jq^
zxMhv-&xzrtk&~YmI!&z%@$OAl)t+as=@lI3^mX}y_@a|n!u*Ts)TW1(iB^ZaniIME
znaV@HX;Y)-S$>$?^h>p7WlLb0#0S&4>+%;g7+n)ubfH{pudnmH=gI!l<b1i7oBw>K
zwBl~<d+|v=)AA=jojTk9)vwc=U)A}pQ~I}W)tA%zwbpKa=KoqP`9PGHwv^OGu`1I8
z@tIE=5*{BCd6y>Is?egqSEAr46VYZC;1Vfr$mv#OEPa2AkGXuzOrH&{;<L7-3g5oP
znH>|%9h7m<<4M=Tz+<M)OEgq9Ju;_fK1o?7cPQ0xzTo7D|4g4Z2JL*UYcr?Iw@*X-
z$c5K>hd!Tqzv}Ds^%iY9lTIdOZmW<od)wN5ID4MJ>W-tUSuScPs+<+N&$P;8ZGq|A
z4X(BuoTcwvdD5(}-1&5lvdsO8SN*#y{ferDjb8{X?{~lKckYJs7Pmf^v=4^2DwHfw
zF7ws3y<2&H(uy|O)hmxr4!M24_Sa>$zbpbdD`dj7M3*S=xv)5I*fK#US@LvB<BMsB
zcFnHxaZ!3(vEKCn&z*IDl6NHCOHng>G^>A|xJsDAIgi|ha~PI*a;t95^Gcs|$u#=A
zW$D^?mE}Qk^A2gwFyG2uZgu6M&bck;v^f*R!i&`3@40E+V5zT{l6C*%9QXg{>b`Z(
z|L!t%;R#RmWg1<9+MSc;Y;oY4#K@Vt@5a2skSFs4S=_z!SIu9+wdj0bm59&B<g3b_
zP5URk$`3P{bah=u$blHS3rqE$X8i6;6^x&_#7r@eFSjCDddHK=Etxsq+OHD3^L$UW
zx_8dm*;OR6_|Av6mrK%puWeng({?n@=}6WQyDU*Pq0@enmwOIqDTnn)u1W5^nI-P-
zJgGg?P*31=SMnRnuAY~w^Lry-ubgJW=F4y*W!r}KbuNo$-@frBwcS?bf|$3*T+ZB$
zW{+=PXxet-_$?kAOJgY&-EPmfu|kU%T%DD%(JjiKuR|o2!Sh~LlR~K83x~C(+_%-f
zoZ|eyH8(e-dBPnz7xPfLfJo)1(gwy}F=jk|1+C7{ax)g~Nd2~=(6e#Q6P0k`tX2N=
z{40ZA#C+d%q4Ay1!e+To9pC+}W-j&f3VHPG((EJO!dgpxg=8(5qb8f&n!e}vm**eY
zwu%2^67gYXU6FP~dTLYCt75iq-5Ogi>}daXLS2mY<g!Uhdp|_ZsFkRj)n<Gq@yp8o
z)m5wR|9SIVYx1lgOLjeCS#J?^py$h}?YyrJF$tSIW;1kI#<XLj|EDJcd{?(9sXAQP
zQr6{ebMH&{@hxGT_nVv+eLAw@O^1l?jg2A8Z+$ow_;q1*?T={}HGb$s+HyypVUjp1
zkslD9wm!3nJ;5OV+5d#EUB;2md#)QT-`aC7x+i+)j^}#MSmWdKw2!ahn6+zOWli|U
zRe=kXRD35e^=bF`vL_!gvi|mELVJtf8H2qt52o;E_;*WQDDtuo^nX6_QKOD(@LsvF
z;2^uzL48-PEa$B&6bhf8<0{FOGWGhcC5|ga8nknbPDeg6b2`3G^=?6?`R=7*w%Shq
zU2VU&C!FeVJL03~Sd=0(<Hz%qZea)A8yC4=YEA9*EShAtP~?2sa^rcPg6@lL(&KCt
z{paMGO_{r=BhKRB?zyWo(#vK)dUsuNkLJQe;eGX9u6wqfpU<*ZV&1Wo{9nhG2E;JQ
zh48I%``Y-;^IFiuW?fd1Hw9U@*8Fdp*yZ-oT&qaph{5%6kL1`Sk=uKg=(J^Sj=j~I
zrhY-os_X<aYbckz@@o~xvM~LHrJ1%*jic2{POIG#aQE2WCOT0_JXl=!s>-IyN}khN
zQ7qrm`dNG4$XR#C{Jmqho<HlY9JhA(u_}qapWKP*tecN$vTba6eR^$8_>2eY$CkA`
z_71S}xp%@#t-)g1n!Zb`zO?=1xY-?aZIgX*&ePVk%FyU-Y4h&pR42?m`J?1`mZ#*}
zSKh1VuX`KCu)6o>&9wL1Zp~i%?YQ4;*9z`MEMNC+_0Hzp8XlI*_1pL9^{{sjBfjN6
zU$jR3>9XJ_m*)EFq+Z^A!`<NhA#Lj$f;;ZDWzG8^`mX!QHkOK#=WqX2-dxFdZ>xRT
zp2^?e%rJUTz5D5DXQjz%U(SE{^f<itTX;U}R7r*({wjiXOlI*+H#9_^srXFHQ!rQc
zpCrY*&FAX^ncIiCW+w%Tx+RLU*#$iolQXN#nWFx}xcG=sv65S{N@=miH+$7XLHcP5
zAsYm3)0@|P4GJkx2wspYSy=5*Az)QjEwUw9$T3(_MbNWcaPK8Smt(=x#9DPHDk-Z7
zvj~=kZRE^x2r0N?8mPwQwXrm7S!(_Fu=J3$Wb<^UX31=e@*EAW8IIxQ2g{B&r~kO-
zVs<e-DpZVv*|p$BTf>dE+!LaO8m`G=Y2m7AnWe5d>}s|D&20FCGbRd3J}Fnb<0|mP
zN$8CO_ZNe_WCJdpE4q!Q5tgcz9jWeBBC^K~J-mg4>YupZ+34QCMdkiM*%_(b8#z3u
zwixfek+p9}*6Tolt<Pk`owIkQIWjy+DqzgvG|E|9B5*iN*fB$KXL!!3V>wpeJzhC>
zFKm)KdR$4Xyy(c3n%m2APi!$d*J}H1A?th#)h?&}D5LzoOB&2+aT5yrUraBQ3@_yH
zko>8_HH%>aljH=(o2tLB7gRA7{@F3%$rk?~QzTg|Coov5Fm0$8n_17%nej6zjcaB>
zN0}9e#H6QN6NAlCzg*~i6TtuF0>5@7|K9-qe+-isTuPWyWTd%RbI$)qnr9Cd)fhKz
z`aapPJjrRMz4p>#x1GftTfAgcOZZeu!WSkVODYjdjPSfJ7~_x<RN?-8dkRl`Q$=8D
zeW`rZ0?zJf&M_fUwcE>^#SK3e85X9OF@0{2ZImpjl$^ehvvlV4vYGL+$>D{^+pDH`
zWKQo04=rnpoRMEyR$Q(=!%-lqVS3rh=~a~t9ff82GYzGkr34PA=a-5$Oq-}%7;ZYT
zbB%!BY!Ci95!`x_{Cgiv`ukzl2Fclf6&TiQGW=7>6AR^^>?APBEk!$7*tJY_#&(&h
zS1V_UM;?x>-1)*|`-{qX+f5dmcYoaIY3~?yaG~j&K<(IIE*7S0hnLZg9@P*3XI8Io
zQIr=_+CN=EaijU|&%K92V;Y?MPK#&nU6OQGJWs1-x|pfew~o4R%lelpyZyh?@3E|Y
zmf*yh3KM1qOkA{K0q3p-jGPO3qb9N^PGq0urWCm#J*+*eB;L7bfkM;-9<7BcR*QI|
zCdl6sn&lwmICa95hw<e~4HZgt{)Uq;mrl}JIZ5Z|B*T-7S4&O~|2T<7amkj-MwUSS
z)ec-r+l5uO%RG{>SCCjb*U9eJCU^NTEv*dE$Sh&)7s7fLQ(Sizo6b<M)?kPYSQgW;
zEaO(QNtr-Sl|{JcyhoAqicTpCH|j{22)eTgsRT*OO_{2ACD(AGZULik;g^)&<5JNM
z)7t)<aZMJSRT>)3p}exsicL<LP0pAty;W?6RoQf@>G>zx$}`0(R&pl)oDt<9uuO{~
zxpLL&TdS5vP3-qvSf<%gRJpoevtz5)^xaac8Z$&oG#5?KT)ix6)yl3_<u6xl$`Wn6
zG^2h-8|&xb-mLVx>kHGL=>;iSuDIE`>*cJqKWFV_nEg;{cG$y7?pCvJ7B6x35^~GZ
z{I`KG`9kY@#=Hnlr-m%i^)oyyn!6WFlbu~QXOl<qOjgfZiE;+Pa=Tt+?cI^}C9!9_
zS@zFBuZ==ERfkwFt`HQzon68uzTI8uA*bT=m(_beRi9@Qy4#_o9vyQ^(dUMnMdgv)
za1G_f4{9E%<t_U!RsOrZ{Lk`55-01$Zx*o6tQV7-=%UTWpS6W)0;k*U1s<;#xMgpd
zHGwlqVe6WxiC*3UUaz<C-CE$|y@g@t7XDvb6Hkk7e6^LMY7vvB#G+kdzSax5ySDaL
zZuPX@E)XTmsyR{YRZ;H6`tZa+gAm`~Ws9|MPO_7l+{QiGpi*G-&K=@AX7%jeu|;y{
z1nZrXqIaq|a!=YM)m_}ErIe_yZgcUQH|y`YOK#gaSME|!Z05WaRJM4T;-O`kQc4+K
zLbeskN=X(Ri2_220?Z2)x0|=PX((2_($+UixsVjG{Ko``sVP3sgk6s=Q8Q+n^r3TF
z*UD*CdkZ?I-_)MD^w*4gxA#8y|9kJF=rtQ|tqPsJYNge7m4^$0y|;UI?+dZs`r`De
zC(-+#X7B%2z3)r(zTdO=zxcf>lv|VW&4PEox4p04w>oRP$gV{bUhbV38UIhH-#u{F
zoEfw3YR#6=nQeBFd*beOxfj=kGah1BJoHM7VXddYKL`Hq?@~H45nr>q+@ERteyTj_
zIj1jm&Qw+t3!BK9<{PYP4o`QTb7V%<(xY;&IZ;c$m@YltBj>$b_Vp3Li)@OgdpMl9
zE__oueqAZD=V-*4O{PgsZIcBni<L!PH~+MpR%@zK%Q_)j$H=RDI_IuM`G2+)_)HhL
zHDkKtBLCT2XBcj+)!9060c+#})<zlDC;?WcgcBYA*PQ6AIT3l_L|e>>_MUx>JSRR$
zpLm{qa@v_yQ+-ZMsW~}+&B@Q!r=Dw{oXDWL+(%%!%*hox``1)$4g7t)@bp$~E2}Tx
zEhjk}X<F{kzj?ajHUHm)9S3<1?TuN||9XdN%$ZX;OI)JPOlw*v?!hm2=ir+xVW+G>
zheJ!$m4%v0>>hn?eEfLw2WGW~61zj0_7SU=zWQSq*R@M+S(4@UUF(1Evhff|k2;?_
z>patkWnP-g#TV@ks4?4jTG4Tuj>-jX_P2sE1yjQ<gEW)Z3*6P&&splNR@(c0&-z07
zw+3u&F>6GB?*A@*>G$gWi}x&=IcFd9oP8{B7DN`Dnkc{;c;QmugG>GYbuJtJy=2aN
z*-V#V;)P3*6F7C}ZgJ$jvfSs2Q|uMzxmP^)Ua^0B+2!t)h}Tzw?p|hkaCy}n)==N8
zHoRAze1&e^UODkq$nuk&Yk#i&Yp^a|_KZ}F@aCUuyQ<d}%w1P>cikDE>lb*=D$lvD
z`sa-6s_VtIvln~{U&AP^yCt}_biMmF57WTz*~=6EMVT0HmhGvHTzopq@=nCWly2_b
z-Mptgk4kv$2rJnq80~p8>YRtrIf>p?ce9uLJ+kHMkuBnairnI=hh#LpW$$a`YM1AJ
z%Ji`+60&HV-@BPJXu*lM)+d6x?=-zR*?eYIv&@O}wkMbA+`V%4)Py-FZuFiAmA%Yz
z@NVb-o~sXHuLd-nco=&*wD#V!+<P~B@80gcdjIa-cWdvy`FrpE+<VVs@86lr8o+Sx
z_ucys*4~{rXRGkZ`JPVPy_-*4|2n<j=1hC{A+x~i&QaHUq#sVP-uW!+q3F3YU8^5X
z*u73PZm0Y?mE;Qv=Uxj~e%|@z@MK*l0p0Lj3Qr%IK0K$fEXky4O3a0z<XOv@0-85h
z3uNzl{QJ$aeWl7fPEWBGb3E7`y-$0OtWb*MwhM9-FDM@>$xTeTa(HUhYcboT(shn(
z2D+SUXPsiT+3xywUuxe8y|b6I?p^jd`^@g_GXvYpg?7(N_TB5$xnJt{ylUU`ntRXd
z{yl%pd%1kyiB*6073*-#{~r<lylB=|2maK#2PgEc%fI_jHfNow<Dp+JXYy^&K0Wnv
zhTfrGZvmMYZZXyM8xpzy6z&O?Yt;xewx4s@aZlyCb;hSF=gc^IxW9BmPs(Ap9GSpm
z)0>$`R+r0rzM1NvsZ%`b&A%M8Y~!Q36V1c^%+q0-e^^}kc&X3Rh(6`S`R0zA_KTbr
z2i%!_=k9CY7th{_eX13GS9||P@6}tj_d0u6x^><^_I)4Nz`_*po=M^Tk8|&T*1iAD
z_u<#M_kjWLS?k}k=fD4v_kl<M1JnBZ?2Hel{+W?c*kA8$^hIHZUDS?_(~GxpKCG4f
zbhhT9!u>PSat}51KdI(lKjyPTYyKx?`?G5Q=YM)B^;zVZ#XY|pvyB3eFz%B3@p-`{
zyJtP;R3g5_XxTsV5jtjfZcD5{ywzjvQUy!C-KK22?dP56VtKORclIr_d9Ewo?9MJ>
z&D|JNa6zR&v5u?q<nC_=JzYg~OHZ*(D|qrWXaT3y-%G1>URc<^u-SW}N$z>M-@ChO
z?;6N5mgoIw?Z4Y*f1gQ!F`z)8|Nf63egfS5jNJ3zvj{Ry{r@8%;pdEgmcIO-0T+HQ
z@c;EQ@8?4KpX`m7H&pE{F%n5WyLJ`lx_hgxt>--RuxfVA+iQ7eukSnmspjpOX?%Z5
z-d;Z{fBm@qbrb(jlk6U<-uZo!|5I`Ax;GmRHOH1lM0UB0UEKa((c|dBNByyp>t47&
z$@lP15_lqLy1eZ5w7IVjoY=r-oxNbL>F0beVQ+zr<)UKTYQOyj7!Eo%Fz7L{OUay2
zUFgJ^%4R3?!r+l}yQpSd&WWWxZfydJlN7JEiD(DqxJ?vF5K^5vnJHpX%4dUTr+ibk
zO<1`zb-~$otJq1Jn>`)Q`mT+-X?1Ir*y^YyTU;tvxi*CK2inAmCYXAMF$V3Z+A3wH
zJw+qsq*7;-3q#b_EcR`&(&CGv*MvV*>P#?S@2<hWFr|^pbh3o+Vwbg2N3%qA!?xVG
zy7>C~c`5Z@wc@wUxjDb#Y{YBcDK%T~AHKL?<Fs!bQb{ka%x7JYW*cSp^7{J!vFH0@
zrMJDiy~p?bzqQd7c@GY=Ub%F{>tzw+e9uOePT{2ufy};}R^7Yw#38J7@AbF;WV0(6
zmqhuB&yV~2{qx(i`^)9`hiLnm<!U|I#A@%@6#U`$@}_Q0xkXM5O#CJW2N}6#3Jx@K
z$(&f;cq_w$flGf&Ktq_(k_$cEW;%;|jofq^xrKaOCbBU4sWfv>aL{`1I{n&e?cEDk
z><-*(*tco<w6dt58)d_mbSQ8sA2u>tt=u0Hq|@<Dh~bpF{HfJmS_@ZfTCef(75C|V
zx0)C_4Fm-g&KNwk3O=))t!cqYjZG(9xP709I8QNji%b;_yRoo!Uev3f!A6e16}Mcd
z`(rref55#0={%#`Ud3y7w#U3y-FhobctMa)V{yo(Mx~=(!BW?Qd(A#N*j_chtgz~h
zpM2w&i(=L1stc`*HtZ^0-d?nqYw1K5g-#VSFR7I`5?f8D-AL|K)xVK4SGMeCa<G@#
zt)%AE)Tl`fw$rA*iwe|>$v-F?zH>rUCUfi*3(2i_%XYrtSj?rqs64J(RjN?Ac5k-f
zu?eTVGp9s{a#cK-wxRi@N@MXMP51Vjz8M;`dz`pdMolTw+OWdr)+?>mJES_+thsPX
zt8>E@tL(K~SOjK1o%+72LvPKN7~>~8|1L#LHsrF_*?9EE%A@B`m|T7naJuK+o1ohc
z-CN9;&5w)9ypgw9>VMYNIek{!RxSP(lD%Sg-mO<__nxbIvq699a!U_&(ciBp&+y{X
zj6Y}W|8TmT6K7%q|GK<AyOw2tQ~abL^(X(`9a#?BBkW!LcE_JCK5zGz<CEusmOroG
z7oBA<n&7}FlhWbXuX4rTrBaqD@Nl)M$=)e7ryn$Q3;C;f92LuN{kT)I^66H+d*z}D
zJPX|f>q3mCWF4O--!DArg!1MNm(?#=8GLlqS3b#Fud$%V;q<$|O!8bDyKI7Mwtq6(
zbV5Uk(Z|p7{hs3Uw$1-G|2z5XrD~wd=lk<FU#%(>VQ^PI;P04~%j&XVfxwDwEM>29
zcv=!J^Q|!ZV|k&0yF_7`(g`yjzpVdFl5ZX?w|Jo`l*gMRSL2~8bh$5F^}O7+3O}Cu
z<=Rr`PFU47pJ>|=s`G;3T>G7$iW`KsJEz<Yh&b>mar=^~+Y{ej=)8F;VA~mG*8*Pd
znA`xqU0#YpYmU5Dlx0+Qx|BSj$%aMo!>g0!DoX3CDqZ4b0{hqRGM?PvyIi$=vGT!!
zr1^91bd>N3d)0OoO?i^*p}FRQlh7h2(KRJGTKz2^kt&{>CNDiYDQBn3X$7Vk-yBuB
zTO?WM)jX3p&eCh-a+qgP(9gN&w-mJqI8H6}NYz?-bW+PB&8y`Ro3*?J)wB0yh7{CT
zp1Ehx6?n+xTwX=ztjRHNlC}vn=)Stx<?Etu&GJ9^q{g=7#Zx%^Ivxeyd340`+<wz@
zbNY?>)jt)l{<CFr(f@<~4NO}UPAi_6H$PawPP)av<HZq2F;2l(a!!hk>n1u0Dhf)b
zO;-|m*vGT=lIXi*nTsD<`q*Cj63897uv4jUlc#*_rQ;6MY`;4Aa~EE6nfsu9dFh*#
z{$EzJGkiY2-}!3DRI{m3MzTjWZ@vl1)ZppW7O&8Z7Wz74Zd2+?6Vc0atyUTPJni|k
zK=S~X--Q+Ss_Oct8JdM+rMCM7B)E_6Qha*VYf<=1*$?Mt?Z}+E-1<V5*ybm@TvD!<
zt}9&nI`-|VBNHpMyTZPN?Kt47_gP}8Xlg@5nCR<?e_~R51tOFGr}#ggWd5l=UaU*e
zZ0qKmEeBoHn4)jUwJIqtbm2*=zT>n@YulWqIgZIaU%EX~e+#T~FyemySTae(xqJRh
zBje>EPcP~fYy9uMFqQvERGvkJY3Nkn)e+VUi<f$Di>^Dqo>@@HTE%oG`=nzzxs&rs
zPMY0&|4*n@mF2zOYsYR!tIxp(ES#Edn>zP!_T^Vv&v54|pPE~HdH?UP`VTW$9vp1_
zZ2iJ!xuxdu$>nK&_6wRfH0m6;T;rh8%RDK-{_>7pS9~)TOxAVfY;a<`HB&=zTlI?W
z8qvT9R}0$Awj`Eyh0ndj#1nod_+DGhmW!&6`P%mLPR(hFSsrk#Uv^CdyXN2jEE?~E
zuAX$i9FnOgagbYULIcm9O|0@)4`^mrHqX3dwA@6(;a$+o(*lX^+HXG2Qs-A$JiGE?
zjm2aBc$ZhL<=+(kiF{4{(ski{?Y~}KRgSdB^Tb<Ty<M60!jNs*ACAgPJ$>)8eoPcQ
zyu*6?tNhJRx-##&vUeJO{37UlR!2k3u&q?rbmh-qM`nZ+b~UiB@#wk4cD=H_Ctid@
zO1S8CtoxbkJ))20m0czrPjcNPw(@$1MNrO~J?<*SED>^+(Z};9B_%G|;VB>MbZahy
zs*^?Fxva*mQw&`a)7vh-)o5fhsd8`&Skdx5^_5aa?4_dBJ7aWoGF{#;()xV<|F<iZ
zYE!rTZ(kG1@IRv1+RKsa>ED$`o1;!8$RCzG%^+-Ur+e;b>&%#duO&0Lcy7^gtebT^
zL^#9q;Ov^0&n8Wt!lJ}=`<%nCZ($z=Zp>~!UA|AaC*e4!gT49Boz}O1KBy>4oHu{9
zx}EEG|Bi@?`E#2&F5eAMY%0=l<~tbh%E3kHreTPydeTIm+OLnkED?1#d!*KPB)D6A
zQ-HIMN}Tq8li<IqUS_KV6@u^2&3O0d_nu#QsnhqXecJV8(&I@hMGhX+k^Hr{O>T9B
z`_jH$S68d#W-)(swf0Elclqgbe#Oeovt~c$3YyZ``gKFA{e(qnk(>!<H6H%?%jK%P
zKY8o7t@Wwz?z*nrd+>i&<)%fUJnDD#KDnIUZ^)SBGe!T%)8<{XuURR5sSll(dQdd-
z(*0>7%)he=EpPwctj8)TG+R#ppH$D!Yo5Pq=e;?(NnYu+Uj63V3L7TnTub2GZNJs`
zkkFN`vcp;R9TzW6xVlS1MXEJqmBmF1>1U@j89k&QMyS15r1~_=;q#+e@3c(poq0u*
zSOX1u3_B*@?qUjCz;5WkWau&5=(E<99la+67FHeTi<0ONPug+u0*~M;!M14QY6aVj
zPweNaty_ehcqLieowYuBt^ee7=w;9$wVngZ5AyA(T(<avz^_&N4<FQjH^XutWB=^m
z`uaRFbGQxkefSx?`{z0ze(qt<HhaT=vm-L*Qw*w_tlq!i;GB7Q^$eMI16IDrhns{Z
z3TaNf!MJ;Fz{L3xh7X$!y^1$_ou2wob6(>LL*~m<!yNWLt(<x!V$Uk41$&qmcz$HH
z+RPq*!!dA*O|YYuGxMs=FOLWq85=uqF+RM>Lz30AYO+s*@F4-_vKQ>W4h=plSZ#01
ztNotrsj1o#!Ky33Dp|uD)iLjcvD9Vh4)xnT7qZ!}|86Zau&rlsRhTFmcxIz~<EG!4
zM;)czLrjhu8Xph+X!PXOri%{_WJ;~hNMhaW<>I!*UEz^Q!0RowXWZqS7RkL*)V}4e
z`fE!(qp2p7*b4^_tt5|#>aFjC#W(Nn`fO|;w^`@^#{iDfMJGPpX5zch`J-65`eUE@
ztqu#J&YWF*)1o<dU*wy~Ao1_C*}MzezIM-;Rk320g36K$6P6y}J9A^{#NQ`HR?Ud|
z(0chp-(8Kq>e&t)HmBw=i*32U5|z=n&~oY41zl$rus-5EXkFPGb)rQ>p!MPk9$}W#
z&pxz=YH&ww;5ld9@4vF=ZINxf0e|(6_Kc(*U6Z}LRvug2v0XP<p_hBvfjxVoe(g{f
zv8=aVwz}b<Qp#HYh0A_8akgHZt#L;>+uQTh9f4WD#T1QHcr#Cb{xR+RpQ-0^_^-b7
zSzgm&a%bmnhFuE}_b*_Utz4yk`ldRI<^KLJ9LqlUu?2S69@)h+v*Z7&%^g<?=Nd(<
z5EgV1-YqNIEI56F$<d@vX~*4P7aZvkTK!3)x!}(3sujjZ9-j+f@weW<YF)tk$>hA`
zoGE*P^^YVTx0f~vOkA*l#cPq2TbW1qs~sz6ziJO`Xb@boBeY}Q!5PBJou}#|TK8W#
zTDPLrLt?&80=tW4fbNeT=b2N38q}Ol&bR&TxLCvMx^P1l%lS_YQv+CzCd{$D6RdMs
zS$)0Hshbx>!X%naxF@w|uvf*bzCUq=%w5?mNr}|Lolj>g%gc&Q*f34$>T2b)+Pik?
zWqq8Mr#Zclv-6NpXPcz@gqrE^6F9yG@4s4TKXD1iq#p53o&KM%*JNGZ|9g>H3#ab?
zMGQ+ALXON%U}XyFQkHJDI?VI2iDS{4)~iPT;Q@2oHw6E84D^(o^<|a=XU;mISM5%!
zytbJ<fdb9`Oj1m>y{9xp!yLLpGx)+30wmY;yz6N4S<zl4!K@;{dhS9<aWhlk1|HuA
zCSL>ZbJ?vy7CWSQdQFYHnm@~X$#B#cv`6k`-?hhabIKtVlWR$?*HmK;t-HAWamM-#
zt1jE#jZ=@T&suwCV#ZF58vaQPXLG+^%VWKs#CiS4hcjodC>{;+o%fh?!EKq-l^e=>
z4Y)SzbnQ9O_hJsuY2PKU51(`rzal8Oj{C?u;SMp!jnD3IMz1t+ZP_R}X=B%_ji(nK
zkx8DnOrrU}PmHVo%FX^;T6<;v!*?);7c@^WZJtou9A3f9sBk`GlIW^M@>LE-D$>l>
zlAG8=rj+W4&if_ud*`e*Dm<2wtZ%JEs!lL%*>KcSkag=6?&A(TE3b%DaR~c<^<w<s
zdQL$^BIN?}7JrR|*1t>msxC0CUCXZP&=knvR<B?yImO0T<mj@5-SRs88A3<TJa%Ig
zIC_d>%Bn;1i+=6adcC9mvqkWe+aV!pp-)y_U$Nkp!qt}<ijgz-E0+4*xhN}XxcY?c
z)-$VO8lEU7DF&V9^*DR>j%IJr1cumG+?|zA`qZL4BC}6u2E}T3o%p`^q<)lrmG0uI
zqg=gHV()BS!;`(H?iRy;p2=(Uc<)uN)X~1nIWfW_^Rv$D8*8MEl&1E0&J}R*cw@$x
zFnz`u&!rZ9dnK=0f8~q6ad~k;vo*&}qk?9^mrTY59Us454%^WD?m|ys1(%-_lZHc+
z-z4F!U+&#n&1!eiRj}yhlr9$CA4`JQ?ucv=4xBN0|AiL&mF=@7Z?9eIHErj8^Nn2_
z&h%7uFmW`5zG_^0TEkK00#nro_P`BYRTo+`EF8A<u=w>hmEPz%UC<tMVO`?fgY^Yl
z;!L};V!Z1!Si?`8G)Zo?dL6oElhpg515Y3AIHc^eUv;^}Gru_{%ig4RdN0<`_3F`?
zwthN;?rEQ%Y{umDmtls_c9y7~J^$zO|1!hmlBoR)s>998t{=M<RC`FhK1pYdfZ5jv
zzH2P5ul&5BlVM_gsK)wY!KWn?kGn>+#?E=MWJSBj?$sJyUO9?I*0UTvZFcdV?<-}V
zAR1|I!2I>>bUg_U<Ba=e9V?w)Iw-B?+al3AJ<8~q_v~;s<0B`84L7(lrM8|CVBT~=
z@{#Atg_6vhG@NcT1g!ocys74bljD@2rw?8^82vSs+Bh{}y^8SRwJYNu99{X<YXeKq
z+PAI2rjtAtv^rh9u~Ww@u*5dANljwX!9`nIm4#PF-QaQPeC}l?a_DnM(3Mq7MR*r~
z;kY;3ZI^D6Rd?dKA4hhW?r3u6GIy9BcB4s`LFn)=!T(ZQ0yl0BbeR{Ey>hCesCC~u
zQN6C921}QTVlVF~zH}DpoT4VSy-O)ldD`SJ`$`@v)f^3yGQ1bR`L5NJWXpBQbw(?=
z7&Ywq?$tzY`_ggg=jr|4)|ZX0O4}-N{a<`<m&Cm2iI+RHofh3(8er4)N$32huTR#O
zwMKuqY-G@^r_roe&>U3IY;p8e`;DHdj<Zfmuo`mk99Y5Us>!V}L0D$ul9*+zar1b*
zRUdvenzW(kb!fs<p=C2y&2d#Zk;)RH8s5uvrQm|sR-W~?2g8zDH%mM`wV-vAz{C4d
zr$0|vmBA#n?83Uo9&hE>@MIdDmYa8>@!A2^eGVVG^lVtyZ!huZTV?#e+j7aJEqn_L
z1Q*<93Yzh#Yv!`R2+NlT!m@*dr)K0C$X(ADy>8^Tyj7U<r_-8GH}axCibly!H7)Bm
z>r3X*&ab#CV|_09mg}75!5z&^=YF*1pR`(2!+4}gBgIH6LUPyf*0=d>M+M#2tT>(3
z{G@P8PgCID;M-LNtP{RQ_<c(emWbrnY|g7$n5iTd5%?i!&6Klk44bZQjxu=YA<<kK
z!5SzL#aMBHJ?&|{%arra9xRZ&rL%(1!BV9D1?!d`CSP8+`UUO3*RUT6EA(a)`^Z?x
zaH}U&=c<l}blDC`U5^WK3)%w@2nkH*5q4ql)tI7iU+%Emr;7{PJM3(K8#>K8CI5e=
z=WPw|;={UUlv(cF{bsBrrfBi{Lckoy{5{5fTUp!EGNKl5HImpU#Q5d+zW01Db_okk
zcw8yE)M0PfgqZHOFDLK5kYKudQFPzLV_znvO@Dr)<YV_lKlZQ9y&V$@=8FDa-CnXa
zq~C)vq@(|f$G-!<I#=`O)Y-RPFg0nK9(JFBSGeK+SIhYOObtx062-qAXS&?|XLfq=
zPNSpCGO|6Nr&?Z|bn?d3domA|SM|+)9KaVLn%k;(?lWuoXVLF%2`}Oq{I|Q>E^=AA
zdc*mRe|1$SdU?mTFLXUvccSUvAD%N3tb%nHN^hhpPn>Ksk;f<W;q`gXH(e;&!XmaP
z@Z-Xx=g%lC{2$43`|qDy4fa>}d$^tbQ2w9gp~R9M)~uq}*6monYF%2ZH_sM_pE3Rx
z>?ZG)+_p_!c)yO{tYXD4jVEG^!IQph6_Vtb=ikBG@Z#wT38@gvX<N+n+^QD$2Tdw?
zxlEn)O<n(|2#eQyWoK`X`<F1iM`hcZ{Z(tfSG{&M>lb+c{e<YE&-|aC`F_53sl8&?
zymj2O<YaBHskd7AGJbgcW1iL8?2a1AC%^R!#R3f{Y}>fz)!dhBR_y!O=d1CdjeF9+
z-ar@2z;|odCu%TX4}87RtWj@5!R~zKXorlyAyahuS2Z>i2A;SeU;h2s16{+8c`Mei
z8!ni)B{1#Zm-CJDvwe3+GIpH$Uv}Q_?~~d?%lcd_`#es3iWjMSXD}&n#!Aiqr=mof
zOAY=xP5iW5b;{Wt)))Pk?Ma;yWU@@xn3wPPswF2Q8tpQy-81;leikocjb^wR_Tsay
zmY>6Y4yX54zu1rHX)yjdUVroE|3B;<La{8<9t3?*kTeS3HYMxoxrNTrUvErM3JzqD
z?2$4qx)bqnv1>PzT_2C<q$?*T%{j3q=j10<bw=IPOFfyJ)qEM5td{n4P70A^bi6$!
z%ehPGXxN%bda9{cG-moTZSz^{HFedwHBsA(-o6TDn7$$5XxmdGGwo2l=rde1HYF~+
zt?AhODo6Rtx^p`#K2B2%ZaaN{P0mic(pi(vYW$zlA*wF6;YMMynP%yto~%f%Fvg%#
zue&m#>2A?$U9JlSbjPj^-1_Qga;*P>?78b=rfLNr>s=ka?o5fH@Ri=V7Z<<&S@hOF
z{Y_m|GWUYzteq>af6Z`gT+}&LgE4SRM%31Ia@oECi>Bn=optp`Ak(j{VfXs)uF{BH
zlr6;Iy1cS1c|uC$QsZuhS2Lf_lJLlI<j;AgsLo)$xFcv@_SX&0+}00xS19C2ZrT<T
zcGB>BV)&WD9S^D$Cas8R*PO&LTc2sdH80&YSDd`G{X(CJi>Ex<@i^X!?U-0><WvQf
ziB`OoOD3>!7EEyT5?dlH9yCEQMtAYnWNz)0o<m-)vs<2|PX8a@6?xIiZ>pxScSh0{
zp(zPFGsWiQUt*D%WB<=$c|v4i(6M>7mt2(>m~u&JEG+XqdQ>D=u_|Lxw&^a7h)y@I
zrAsm*Z>cSd&s-I}Xzr}k6$}fSq@Ju`x{@X>sdC`t!Bv)OQ<61RbEky5Ex1^<dhO0E
zkp<I|bC%s)CbhNjsq@<_pC;73`4Z%B|M$zLd13!Nw#?H1b|=V-J$6O{TVDANr3US_
zH-cj}pEkSAznN(UcdqU%g$bUS?_C>RL}q3vp3-_V<#e3=Z`XzT*=IKivx+1eM4!((
z{`K*k#$}ugj3P-rn@(v&IZQmNadw+_gW5+c?GF;GR!V=^WVQA-_kN94+70&q&u;3G
z>3V4wz3$A*Lt>k&&FWX&*{n8Oqro<^clG5R-);nL(bbc>>$Ss0Fw`w~>rYoUC!bAG
zPLk}UiBGP1=lv~ZXDhwfcs)@#CE!|;yk*(ssM%-B9M?3N9uyNeSh3XBbk-Jg5AjB~
zG!Kc`Ba<Ta*G5{X$J|)5!>30*FIe30qsU~xDbKeAhuOSbV(F`KG(<XJ!S;(>!Jb=_
zMCNDt9(9ab)BMy*d;`1mnuyNLORFL>4|ZM^i_V#FH9D77|9WiI(@>jFj$E$K=f;a3
z>q=&F(3eb}Dm!gzij}8XZPMZhx2e--CQc2I)F>6ZHFfb!wa3$rb!wDsSvTdP^k1i)
zh0>b;g|=>A==ZMBwO@nh2;Yq>N<xnlS#8f8mV4aOl(43NcUM-RtnZg(*`ycernxD$
zb*J*!NKH9^RB5AX$<{V8rUk7f1}bw|cB;8(ox0Ezwqwz{1>M?Ov+w@i@kudSg4=D)
ziiv?=woQ_KxuT=wli-}TBciH#t({&WlO}d;QPH$A^mI!)J$YG$E2Bpu_mUMt60Oc%
zz7nk>4bh#`uN@IIi|y-;Ul}_4k&3(d(Tl!@jA!P3keFfpsI$Ljjm&(n#Kn$+liUmo
zMHUHJvbySJP3#wn50kn&^-M<Vq|Kkbmfu<=Y~JF<u=Ybrn8nMf{`0y#eS)S6`Wnsb
zi<zWdP_wiuiBso)<f`M^L5kNl$`q|J+>jLiyeW)#^CRy1nA6O0294qzL2Qe*JW6;`
zwd~xEOWc7D3PNfw*ENo9?z`$TL6$#+uS#LU#(v#1Nl6RXzc5T`^;r|lbK6FImj=t;
zmWh{B-!-O({kd+R@ab~>zm{uf8E*MEZd>7MrMZHkE%L%b#<fun%(pHoz2Y+8(zM85
zTXn9_%_ZA2eHr3EzX;owCN{C@%VIU%Oi$KHx=lPwmZn#~bhxD!+`DR(!YZyf?KjW3
z7Ju!{myx`)#!BhR;jJ!(E!tY@O3N*lI=xf#d&~9QLVV4Nsr*?_wR<jiHBDL_v2=CJ
z))ejiO75Fh<msmM2G6cjx7oq+KVRSR(&^}l?Th!bs0BxDoG59(a>kSmN=Cgev%fBR
zZhK<Fq3Vab&n(^bNK!m7vvzC7{lah8_dm&aDwelEKEL8|Tad>R3uZ30jOm-Xs(!Yc
zeu+4G)a2oUz#FTRb)wtF7BTE)<V+CVR?2VdQz6AI+q3P&&I_G3Uk=UZG~c1crN%S)
zwdkXL8g5>aA|VA+PAOmN>^mjRBQkf&UFBSHFMk)Iisce<Dz1fl9@(z1Jh^(Zu1|yT
zqLV8#?paUn-67Z)wNd}=Cl*!fvm196ZMiHvoBh&^l1BfcRWZKnyjD0)JQ&#ZNu)*o
z%nHuptg@MsJfbG1GuEdRP0TyXwbDW7(U!liq5l&M4aEayP6+sWBy9bWtLu{fJc_CG
zx7|O<OV`|V&K#}>ii}dMr%tg3%w4%=#w{b}Nw*(#E&h2}{@Et=GmGx*FLmIlI&0Oh
zvE{%y9}k{ZU29&h-|puozV%N06BX`|e{Ic;IXx$|jRH@)O;f5Dd7gTPQ|cm5R)0K8
z?s>Q0n`P62FLup4wW^_WrPtAK+qPGlT+~RN?S5G7<wUQl9IK5bOY~!3dZn{oPR`c6
z84|T^Q)AS{Wqu~Bb_!(eII4HlD(cD0qAIJk|F(qHG2fZ>WsBRP&~;_08|Mb?6nJw?
z*RN=Lch<8>do5G*j}}=m#)dkxZ@p-G!#BL}u9(FIjT<NRf~Bwh7cOIER^L=0t#i}t
zdsXf6m7E7!??x-g#n|WA?Ag*-dH);Nf^Fh^UwZa$xOjW@w_j`XkHjoHIKNRP&1(~@
zZf0lu$5Ne?yXDU7p73r?t8muL+nC;Y*<gO;O4faB8FRi~aCW*=a=*~zij?@V0O!Q7
zTt%JMG2&j!ExxUAEw0Z{z7pEfVY6v_-E<Gv_usyI7OzRZI9YCu)++P8FKVkUpO<!>
zTj0R8_CnU0h@YRr+IH_R`m`(ZPlo#j35z$$>}QthFBd)i_N0D8m6VdKX5!Avv6JJs
zPEonOyL4^I+`g#Bg0!|vCwFz+&@0|}Ha=ph7N1T(i-clQcc4<TM0M&h!*g9a{|oPU
z#cW%A{J0a(sd;~VjH>^Lt(|dyTY>SeM=X_REhcX|^JLGw_^LlM^=?16|1No<zOQc6
zuRC6Ig}r2~F7M2!DqAtX@!0bYsoayovR_;U`;~jeRH9}6X58{$_1C@l|78W)m9y=n
z)F-E{{{2k-zei+M?bKacuIPt;^?A7d;_9Dsba&{*1mv5B9NrnB?6!68qd&74)I=E{
zwQT&zvPn_lnZ|;UTrXxP=>U(}Pw&XGmR-G9=o&fYXhh0`$ekzO2ONFBVk*NP*4<MV
zEID%Z<B|vMjIkSTa7=gL=lK<yV9S=kInD1%!a-G;^qiwfjmOlkFr7QdaYXS#x~O>3
z&1UTcv41cA^RqmiELe4iyVryFq(FvbBHz(FXZA*l9hzW!rTH*t`<lIXdws-(6a{ZC
zyc(8py}VLfx{0-X??azVL9w$B#dJM|E-sWc@|2Zpk)0-f<?q7FtdA~-%JoFvV|_Gz
z-2_+hhP&5)&JCL|rJmKT@haz!g^AaDWED~uIh~1BE?uNjyNH!dR^{d0H!aaJx-3x^
zmv#rb#?($)k<uDFhwoA3*;Njlk7v64T+;F+Qchn?Y2B3@I*i@RBF^>QJ*)op>KjY$
z%^L1!kMQIy@C!WPVXNbPdcn-uZ|BYa8n^3+%=sNRPk!<LpejDkG-RIO*-x6jmu@s!
zaY?!6P7C$y%@sKkKcn~0$^Y_qCWKt)a9KX_?9YWE-%_KNH1aOZlv*nKWC`Pv?~b#U
ze|+qeHOo0|){2ett0Hf?th=>n?yOZe7v4E??s2QDn?j$vTHpWJc^ykVpMFr<z_`GR
zE8&JEpKpoN6NA}L6>e|3s`BiHitjy@XIGBR`!IJ~VC<qJd@6-cmgYow-#eG^qEFZ9
z<PO7mQGfh`{>kp1_&{Vs^q$z%Pj`;Sgobl7O+3IElbq7>^^NHM%=6O(rX@<o{77j{
zpK&xS)2&(7yXNPG<X$JH-Z@33M=y08(_bJX68Av$ug``j0^K`0#J^1_W<7Of%EG-D
zMK3H&6c*(aIl0iScJ4(}CFP_*nb>Et&panb{8y8$TNe9LT==Kr<;;r;OpC58e5NX?
zlrT*xGGSRx+w2=>1?7)=He5}Uw_J3q(d|xVP<tq=M$y!JjaOOI(v}oljb5`zM1@P8
z^{6_h+=`l@l5Gcdf_x{h=qmd)$HOD;|ITIW?o8Kb<EZ@S)wg%z)F+cRZecdsB{wDU
z@sXu-Y<JEp^gRCh?PN#J`;NTvj)L=@dQTUM^nUsq>9RK2`rNG0g{R^}?z<h0ci$@F
z)#+=WC*SI~qBU&A@}G|vPnG|btMNln#Uqu~`Q75{E8>59ru^2Gx_v>p`;h$Vlm&em
z4|bf)$l*KI%{f*2?2Fkc3jKTv!IBTWUp?7msPIhY0n@%`OrQRDe&S$w?0f0n%N3Dp
zT;HuY+Ecb5(jYFa<pD=pE9a~QT#FvGw|3<koX<EGk+{Hx_tJyJ6A54LJeU>MHvP!U
z^a)DQzZUQx67P{o?&)E)u9+rc^)RKhBV}sGJHb^8)ux}xR4n@8^ee$rf=MvIWBU28
z9?3<jGnOS@NWCsvJKbkzqTD7$d9z!~tFpv6+b(`g6l={|p5(Rik)rI@EZKcpa$HJl
zS1qf`?0S0hMa85g+HbXjE_wD=ExUhr_6DW9a|0H+PnB&@)t0(<dy?L%s0%OG^(2LQ
zyjCrptE?JSwCYia$#bU)ZS7fJGwNpO98;cq;l6Is5-m?T*B4&;yIxoDO*1(6KYN47
z;;L_=x^r%tnkm2ed8Y2&a|6?m2Cm+Dy7T87&euJ2?DJ0LP+wkMBaI_nQ&zA(`JSn9
zKUC+N>7wn8tjjmDE^eH4-Q<>P-<xH&Dko><oK#YAUbe<5Z<gaWosOU-3=t|P7OimY
zi*gA{S-tmMB+u%)qtABx#VK(e^J;Uw$hKg^O+G&b3BOmK6YX@HeEBx7jSWAu<npy+
zaXGRZyITFVRL`$!4W4&9xatObQ5VNH*Xd!ZA6G00J0co;Wp;Aas)~QU0<uTLKS_Vd
zI2p%yXWv)p?;_W}*Yq!4lj_>w-~3QB^MptIMy7(+Fm1tSMOP#X8F<erE<5Ka9&lr}
zpk>;fvj47`e0Q&$eAsCDNcg1I%c|SY{qClO2`*Dwe%&-My-qD^Zq}Pi@8sfiR^M9n
z&MNlBH?2Lt^b)=$xqr!s(0QcvOJ}3e+njUDCOs0o`Bcz1r#J84#BdhZwIK>yrJkQY
z8Mv)Xf6*Ext)_%xuY)Wm%L9J6sh@2vZ3+qyIa)f6p)Butr{Kf~FOnX-xxb9#{kjvA
z!alqYU$VU4l0`emNq-^Fr@!e3jHWc`Z8*3p`A}Q8YhdWYUd^v(r+6QX?`XTynz!Lc
zV$P4+4IZAi+~sb>TxmNpPlIc-2z$E0&%_MR!WZ23PtMe>_`3A&88a1^Wg08q8hD4M
z+&mC}W5cO#A^m=%RJOeT2{}r0KW)<!G&k(mdh5%h>?g2tqSgXmrI%qxx&pp<YL*?{
z;i);*FLO)NT6QDN4>>QRw@g(lTzRW1^IDfg_yyMIYWeHl$Y?%TyFai$fi;X*DS3Zp
z|MVqKdp{<`Rvc4%pedpGPGSvH!A6<xh<C-Eia&K^N&{cXd|uYJF|YjO+cMQa=```!
zgm4Y^*xUEc|IIm?>vP3r=fPsLlYb5<e(jvz{A}TyS$eD6(q78VJ%7(gbMmuy|B5PX
zla}>8IVQIG!l$gBnTuxEr1W`c8uJ$~W9c;dFRprSql(lG6{FXZS;n3ZJ-+Jk-(2dN
z6sUc@a#_2fP>jn-5uHgtr@gnzZxr(R&-k-1Tl=2t!jo67E;$wS>Fj&C;tS^@dVg|N
zZo4!^rgx34i(e&wqt$%Zvr%8BHrXumn!Mn(i+0orz9&c0yx1pSIi~f5J^Yd5HXV=m
z;*uUx*MeeIzqPJ>z?S!fHDSf3(3P96t&m?Cy2K$!=S<s{)2rS%y!vwb-4>7EC486H
zoPDI@Dp$IqFeSp`=?+KZPw$LEl$B(!zPK1x`cZ4`6}#D&Kea@9=!U;{@k!h27o{r}
z+_q9<rH9$MK%=Ew9X(&`JPn#H9n$q^=>nzQjLZE>G=HvpD_x&?dy|4<UrXSFj|b*H
zJ#aU7_P4i5OP(HDo0@)LY4kGB9}0Q%rm;zSIR2c#U+(w+RdN>V>rHAezAap)^x7o-
zl~kR<yG^+=yBjsPd~s>q7EsYG{loQGYggoyCJjyo@8!MHX;bozJtq{T<u^ZT<Sf@c
z{%-c1Rl#cJDRtY@nLmq6Wn^y&52(>o?lUMm-?gsqL;17!9!chrFEy44EqeMT*3@0;
zGfPBi=iD#L7UVurQ`>eiCO}+GUOjEqqVJoo{wU)5X|`O*$HAt1hM2PdnicPLPEGl<
zOyWuD^S|#_RXn=BJ5BBxtGAnsw_fF?mpO-6wn#N!db~1i#j;#^kN$Y&Fb(BLkKNrh
zWH*{jzV|TID`(Y`PsPVhxi~6CY$)lOcF@J~%g%jY(iX1b`(EI*@58s7|2LePrPG<G
zaiiwese1*JC-d5$ThlRV*4K|8mT!!o+8uR+#W3*ICc*D-1kb+^RQ?!J{I;Tdwy?DB
zN4s;Ey0n`_AB5j$5tUyl%AFaR-O94Kv_iS?olMtGUGvARRg&{{!&#Q!TDLK7`umS(
zqWA1HPbfZmXyXjm7tsx85>h@~U79?vT&r!)vCO&e9;9t)R8-z15VdW}joZ)N?q(h<
zT9q|@<+p{~bv19ky{?|F6}z)>J5P3k#(81Qo4PEl2~!U0&N#GYcjt!F=lfLFNEa+!
zd8er~O+@!_kxbsiH;1&=+&0=){w!_UHpA?+Dof6idc89D>}FBEoxgR;olUo^Zw_vk
zc2EAFu$$jF^YeAH^y0>&N2=$qe>R2D|IY4!H@g~pGvm*f-*HP7*WS3TPQPydZR_{t
z*8k0a{`+2k*;wYb@wG3mzM2+p7W4WxbZMRSvj1PaC(F}d*7Bsxqgyx<CGslbORk)5
zi$7jv;NkqpZJ+0v^9ENod~xBdQ8mg4=vkp#bTDuCPn%^$`SL9nx8KTR-X8yE#~qva
zM?ZgTzwy{B_NQ%F?c~=bXP8W{=i8pD`s?Ur>2UAEggZIslJ*9wmrgF**><#g`|iEF
z@)v}zw!AAm`|`e>vk#<%%dfxJ{PmvnOHu0w*`Iz}JDr{P-7>K2?X+1(W74@ILcNyX
zDGl{<40}`hbDDmb-v6qj`A;><TU~AXlakMev&BTV{kw0p?SWO<tO-un)-EmB)1f}G
zkwxL+%1z7CCM{ddv`l#FLG2puyxVMh!q;ma=SoPq(^uZD`R3Pt^JkkcUfp>3ZNiks
z?H12+bgC1k9Mb05k#TBq*~NEFrzXA)tA5+}zo5JRXZwEL`m(jxvP$=#l$b3&HR`3&
z`Tu49al5=tWm<1@-2U<Qi;)9|z=6~R$3_-GF`pd)4-Xh|oG#G1ywpd%QIs{{$rQs!
zhdcPph3;q=ENPgi#=1!4!o<_gMl8X{rf3R3oH8pW$M5E(;^hsRcIG}am0TO9EOhPm
z3ovr+yu_)!@SKkF0^Tb@+TIgRMm7EC=G2SXyYI<?R-Tocm^?Ujyxkb0x8^?Fm-<@l
z-JL0Acl(4EOn-lGrt!^ftI}7;u0GW2>8tR2s^RrE9!2|UJU3t8I4R4rqfm9hl;h`8
zym#)Dx-~VUc^2!Gb3R$GTrbVBuF*X__1hc0S$7LMW0#ezyup0@Q`zQ837O$?z1?QT
zSIau}&dp74`8TKPi=Jd~*}<N_yS}}<!J4yQFTncen@5Jr)z{ZmSA2hav;6&o+I^pY
zeqVY0J725=C!@?iFHW^)4jBWrmK_cs7O=1<892_6u<<y+CcoywA#UL(EslP>Zk%wQ
zV#jlF37@0LfknN3CV@v8B6BVtWw7qv*mKTgPXV{yw*L$Q2YCMpH@M5OG8rA1-|^+-
za?WKAL7tjw370&V>PhLX(u@|qu(%~nDT8~<ErnK5eRt39seuuK4ZQ|ueKOB%x#8g5
zV06=^*w@%xaq&6p;9D+gwkt197PdZ_aa7Eab8V}Zrch1Cw&p*=C-%Fk_LL}iBxilm
z_DpTMVt+O-^XSxzF@d^UqpL2z*<n<1dd<?6R@#TBU5a|dw0+vI?yHw1Jxf>gSexx$
zq19-zTX{=ttYPxi1oPs<y({NszX`fEr`9Cum`v+QL(Y|{BHkwr<`&#$JXpGAf1&)^
z&|Af4RkvTeRwuP0b`IaIOLr?cl^(n8I5gEHp6TVS>-Ra8|B5|eTKm66!zf2qsY9t%
z-fM<(oB5>|52s4GO;zaD?6Xjtw(d;PgV}eEaxR}+Ae6<uxWuSSV}6+3s+H5{&bq|Y
zcxy$`s<5nMt5l8lNOiO{Gt{hHqj%m&d)?xfUcc5Yy`NPjvi*mM!Ygss6-VbqMHy}p
z_4k~Z&}|kK7_xf(vC|v}8n-=u^k!YwU&$R)P9=Ef&N!99x_$MtWxZP-p7QW!i1t!k
zJmu^y1&u9NgC3tTx)v4W8yNo1pxkVe;p49D=4RT-J5s$b#F+Bsy)LzQkv4nPmusI+
z`zL-0o1XvmYr^>#+drk$?=8ETygb(Wc4>O-f}bh=clZ7KroHder#aEqYEM`4eTc8M
z|KIm-ft2B%4afM*Sqh!mtQ99ZH|=uYxZp$R77vBb;Xd-UUuNguQ<BW=S>eR+ed_h&
zQ*Py6KcO^PWz$L3`AQATJRF&rdnTP-RwCGSbX6i-S7)@xMXsQbr9QbymuFupImnf?
z!B_f90^9ts<+==~>Z(><mbh?<tFgl%pjm3l!VTBBG&%aT!n1Ddn6+JVcIPqPGVP=5
zmt9&trRG@MQiafmH<GwRw;mMTeqe^fjAEClqi+=#o;vGn@|mkEV;#@2RbhvH8h4s&
z6odvy3ak=Kvn;TD7Bo3%y7a=O0%oTKjf!15WYt5DpSUxz?z+~&n>>p&iwYOSO!F4c
z<yfhAnRAIpbkYAM3^JX{9$rO8Q>R2btCXB%{>UAF!*RkPo<|4r9P|69`gk0%6Vjeg
z!7_Di%2OAsMS>xPK1!QR1a#Mjcun+eWVUP+*Y8vjQRbbT+EXJLJcnuX869J_AfX1X
znZ~C#Hl`ZJt>lTklq8n>s5P_HG0Obqrvux*sAg3a1}5BclsKH@+Z30)NczMFAuX;2
z>1(o_)Bgp_T#RrtH7U81v24>$3C8TA!aF{iI+JH!Jk!PO*wX#DNki(P%yaE!4{n!w
z1-IR}vD_+lj_Q_0%wBCV?1~3|``z@@l6$x2i}kZi&5DBKK~tagSsV9izP<B#_xB|(
z_O_yOwGB}_<+iS`{9kpU>A!}HaFgs*r>n(U;#nb!R&<?kzBhSQmy3yTch1xmv9*GK
z(nPMYJYC^#pEdE{kL~-FZC`paB~B9Jif~{(C#tn9I_zxD4&9kIy1kctkY-~Iax|H%
z>6bKVvX9=0Wmc^M0optP(mz-<oOW)=IzQ!**t#HRxiC%Ry_IRPlT8JTZX|ZjyCfsI
z;>HfYj|Q7}?mV%oqV(vIo!dI+-;79qpAdM$wlHZ{^bE;MGlWbxZ8YWXd@Osm^$wG#
zcK?ecWslnx#|tdGqpzB-EBTxe+aUUB!ue0@iuJF~lHU7T{jse_<-^o_m21V_Id)E0
z<7Bh_Db}*&Quf5a>9gWP^>UI{-Fff+L;JsB&%!KamUA~6d3!d#_GoF`n6+MCQD+M8
zf;TB@&XHbWVv}b$dNQ2skz{&VwK>vHFVFV1*WNlclb{WXv(s;`HlO|Ntb}&1e!%Hq
zuQNB54koM>YfXI4vM)?(_sW2UG8Qs1QLoLd6{dt~CidR#IXrU*XUHje-*p8#VNG$C
z?5100-Q3g^%w=4#(QIz?lv)k<op;Xmon7$f+^eYa;K_^Hmt}AJ{I=@2wb^?2^X2bk
z{VRW6e*AOF;=|JOf}V5hJv&-H`E#zx;?JTBmpOfP3~pWJxlMhNQ0I|V>Xs*e1u?z)
z+&b$@oaxpn)|Wds{Oc6{lC1JC;qR+|YrlHPsx4Tnnvl)-Uo3#JAoNhKCAayC1=|kl
zCV5X+@o-#KFyUs?-3Oa5v*dCcm9;2^oSybU<lweDD_dJv-MXeX`{p6n)H^FDO0QzO
zDt77FGu4H;-R?H!tAaENBTGX?3Ij}6Uu$`mbb41}oM`HXH)&elO(NG1mR9Np&l2jt
zvMaAl_Fv2!x0**n?K_TcRz2b5$Gxk*<z0x(Wsj}X#3!k$|2+BFb^Y->+D6mY=pTBu
zwLr(#|Cy<Ms-stuLBn;UX$<dnx`Z8i(7&K7X_>NtcHzHM(+n!6EB`*(8X7ib@sB6h
zL&Q(CC4PIoOv6EQXIzS8(J`5GYgpDNZ&tg#=h392YmVmMDfw;wR!Mxv^{4+cYhRwW
zbWfBh%{zPCu`*p#sPw|Fy=6L~h0ak`j|CZTJPhFy;VW!Vo$%z0noY0Zff;{qta)}Q
zWpQuiIW^&+`;#wCUusla{2@$A?&qG8FKf^Hy^Nh7H2vNen`bLG&k60H_wfMxidM6@
z(uVi*zs=cI`(x?O!1-MZUuSkWUrGKKy0R*C?XoAAm#Y2yn$8p^d3D8fW%-}~CtbPr
zzchkjr&;~asTZ>2!ex(y6xlFGewnndY^9Hu&g4z3O3S81T<~fBy8qwf7M>Mn{g$vD
z=bE5(Y0<4+y_>eUsjq8V6n3a<#iIooY_pP@?y<DaoROKotRw7|@6>Cpd9!?`WH|MT
zoDb4@n*6`3d-spz$5wfdUc7jq^z_WS)sH7}?=pM8V~^|pqzR86c|YFQ{n+OH<3&#O
zZ;o>WAF4Z(?R_G9(kz3N1snF>dbC%;S7C)gOkAHs(xRLkzR4CVRrnS)9+{9>;y0CT
zO;X!BWuY~u*ESqr;K{RM+S;Ob=!?2qm|DY$_pi3-&E#6~I?j!0$GN{DFVfZ=DmkHG
zaltRQVVUk0O|=N0cNsp%R3;vKmB;ht$-6&EYJN$=EX@X0J#(WTru_5!Se5Z3q&feJ
zTd|5igHG3rXU#7{*w3a+T;k+)?!lSKS?OY-tCol<nOQOP-a7N0&9^jXqr`un?-K-<
zzTi1uqkeu;p=7_IRH)a5{~GQWSaek_61=at#ckTCFy%w;Bmo)zl#47bQ~t0zotiks
zXvw7;g|c@GU2IaSH+iY1Fx9dhxxC{1uN&;Y_jvtId*Ya;`)A6M!;_9&o43fbh?l?E
zS3ykZZ`IoCOe!}XEW7bY|No2pqqB5xnQUsl<f33PX@iwei$_Nb*M*ixnyq0T43kgX
zPRiik#_(~Shv-t5u!z*gPg;p#6T$)lc2oqh@vd9V^DZL9P;K8Ck+h^8I{NJLd79N5
zG!qN@OEx@O^6AmCOYEDM2Aybq%;CwgQ1w9y)7qro8JDtjdRsVaol=gyK3KAnTlw)*
zj|)GvUT*VJdM>7?lGE}0L(t1B8<f&IP5<}hz4UQc`d5_P;`c%*g{!Gav&WO4f6^bm
zqSw3p7$12tN(31_%$k{1q_rx^m`yNrL8{IgPQ5xc$F^lhd!D_WqV{f&rcm(ZcYl<Q
zYi;7abn>{vMD8UCZU@%bYv_x6o)VWlY@Yu7t5?=Hr~U+&{x324s!q;lmNhAUJGR+{
z(|(z=MDuy;1pj4e`rp<WTPNmAW%hqvVEA=|@Uo&Vn-#%UtBb7)z0}^QtKBk|@qeqk
zb;3n47AHH+pEJ%o&B=GR^8FR_L?vm_uOEdD{}0J$syI$rqHz4ZrqHH8GE)@O7rkP6
zp?K5(hSQt78b%I%x=mZ&{xjh4b29q>>wG@jY2Rlm8~)!**~q}^zw8;qYA4m#Px94X
z&kV`n7GZvHi=(|H<gr|CZ<edW3~#NpBK7p7%@KuOuUw7mSc}3~cW9|{#2a`mUHvd<
zLg0p{F*@hjG6TFma<qT(>Z|*rwN^wsc7pcFXRP@eItM>py)k7*-v*tBB6~Yn;uc*{
zyLHI#?b5i2cU*;mL8sbS1bugVtmHZz@ZeuJPwuA|I%(5bgn8YSc%@52FHLAvQL5t;
zp1JAVVJn^241w!Zj2R`27)2b6)*kwsvFwywlFkmLj07hAEh~MmF&$GAoRe`i^V1=Q
z+3mvVn~!H~kv);n#y4?SS-Yg`sT_erM$gim+Csn79gY&&c)CH^{eSR<%}QAj(&-k*
zoQ&7GoWAF<xMTV9eOIL}zLEUd6O(!V`~t(+-)AH)i^d-P`n|>Q?6RcXU4glEhvFI^
z$L;=Tzhz?FZ_$h`fex=qFMfNPd|{)bMvDF2iI-fO?4_q&HsO``JoHOtnw)l-Jm(hq
z!>8n<*VapkIHf)^V%1Pp+44%`%eEzl?iralsI5p{s;g`-l=i*<rp+f`whOlkzUf#v
zwlX};O!?Bv;?%k$fjQ@BScJ<%F0s66NAKG_)Aunj-f@E0&OdTarb0n;lG5^Ik*tn9
z#!o>?<|ao!G35oCosTx?^ST}sJ?B%->PfjPKjj+x#cr75-5Q{C^^5)qqqQlyPyWx!
z^Hbz8ittlpDqq6U!u>v*{d&=^8%0L%g7l@gzK|=Q)|FS3?s=8r@&ON~X(~>9@{3Y-
z@EJWkWyE^GNL7JRMS;`Gl#_)aBs4t4b?dr>a+7(S>$$S>jxjj|^$4DRBv@K*T5E3l
zoX_;QCqulkaIA}d*;j*Y&%!+uX34BmjLTD79BOeTEx)c~UeeQapY*(fV#{zQbG7&4
zOP53*5-Y6<{4|Ht{KR$9dj~`oHCHTiw!czrwNLo#mdRh^^sRkgS-mf>e|q@atLp`f
zJFK65kLCVR!?^Ov8UZ`+cN66@8x?1`-rMZ{HhY)Mk(Fw>iQgx$`epp<4*TjahRLos
z9DcK@|7Y@e?IzY5YS9`Q@mk?h$ldif7_OewyYz3)8|Qopm+wj&Sr)mZKk`Yq634N@
z^=C`Qj<@N?0@)W1scqY;?)oxIG+(u4scymHb!%p{yQGFsJIU9yB76V#fbH88%^pV_
z`_Wd=tgFC&e;Q}b$?1>t3Zkkuc&qZoK6tVF-i)2gwrK4Y@ZJ?t#noAJc4=>>^MsY%
zT<^B|UcZtjD0%JJr|B{`{a%KoEc&qbhsniH-wW8k{s=LceEdS<83EqVTwcZLNlume
z(pP!8&!|Y0PK`dKb}y^MTYMU8lTkWT2rE+vvjbzCcq5x2UwZS630IF?V{5v=6job)
zEF;u3a&vpGkABn^*Z;3nDyzSS?`#m+lN9QBA&uv_$pQ{@<5zS21jB+<-L_0Mu{o@;
zXotuG<+El>BchzMybowpv((%@prLtuG22=x%Tslw=VPW8+Wz}4$sG0V&(7G%8^0xR
z#fi?k(DOV<Xw~mHUH4O~f2&AMQuwv#>(<y^`O=3cF5b!_BdVKlX`{mhLB(&y3L;{c
z<-(Rm{&H6Pwp6z)!S9irW>|~rr`2X=%k2-h#O_!wA5?moe}yQ+3RVTqs<+Ejg&xGU
zIIFJk2{1`p=zDWPmGaD95sB{G8`i4)>=NGI%X2J2wK1$yIGL;9i_)TPmg}b(PdeIh
zsEudM3e_Hs&2ictHqX0ey~)2IwdlW4TacG?Bwtcw&(We?88#C(MV$!COAqLmW6l+i
z-Z=Ma-y)&ByJecX@7&8vv>mxNOFM6#Xyhwg>G9#)gtyx!{@=FcwzjU~PVT=VlUExQ
z1~w_axU^MqapBvw%3+J9Xik3-7P&p}sQ$&IX;#g})BBc(o2%yRtSzdXndbIJY-Q-S
zCc||<kF>DO?qU<F|Gw+~xm_O@z3UP;i*(UvSt?gCb?z+(E;9v_HXhN|!&T`|HuJGt
zx-{#)amWaE-WIlKVbI~(mXA;Ksmv30FTS(gD&=t?SC`e2<gY&Z2ZLYL{k&c$J?Eg}
zpMyt>zwFq2=-}dpTK&5EF8c|lGV+{HrB+Ixz0~+$bVsB2AN^OQ{h=`lS}M)`t7J8G
z?N2scbz73SnBO6mrS=H>ard>yUk0|AUX%A^bf41XRA04XVoYgoRmpT4#t`wg+-GX(
z72Yx1oUcC7pCh$l%b(`7MLx$=IwHR?BsXR3`jvlaWk5)WNQqM90lisl?Js!VN9}wt
z_rxLH4IXn&H-v2H(2{!2_onN_qDRb~eD_a%=)U#h^NDYf0<&3lr85gJcs;l9$rSLA
zzPo$Y>P!9H?3%YPP4s@qvCfBmW}bZ?i+%JK;h$F|f9m`VyBb-`srvKk{cf(VtCPCd
zHwj+9W*n(#^)tZo5U+lkzN%?Zc%|x}P5A+=9X+vu)`wK<&rbVaS~)GaX_~va+N5eN
zAI0<oeW7kXy&<83_tW&Ew$120Hlwk8X0p$mNypN6ONZ~D%`<8G+*=Fx>^#u!bLfMW
zc4&D=(vM}EndUB9wItA_i~EejJo9CyXL=+UXYOm7Y<;(<{&!{MbDjm<Qn9N0%Gq`;
z`%>PJ>m_A+@7#*dj~mx7Kk;m#tfq#}u01~c7HwB}s=4c_snz1oq83bI&ups_a_p*n
zejPQnxEwX>`F#H%EwdM&1(#`0TUI;g$F22$Q<_>bmw3t*g<S4Gnf~)+ch#%e?Kie;
zbDfaewC+#p-g&F`&uhvnTissKF|+yhf!nPi9J6936@)z%o55VLO2zzz&HV*0s{iXB
zQ_O4T?f#}wTI93$%g#9x9D?sOo=)4f#;9M4<B`NWO=H=Qrm>eF@5t4>ug>myrPUzx
zJl6^F%KV?tCx-5QnU~`4y>j2aFL&HMw~B{vOMRzTm^XFG-oQxatwq*1bMK!GzZsNU
z%;Z*jOY`&hTKgZ(Wj94ScAh;GnHMg2SZQ7uk8*No#X|Yr8)xs1otzcrvFG+f;X*dU
zg>U992`@2PUR<)Q*k*GHM-2PP>t7~r{!((Xp83Y##`iXl;;nZYA4=x`r+Vj~y3Awm
z^M@E0bvHD|dWqO%7$_cU<q|G>dBD}-NCz`}-c2Ld14ny;4YEWg1TwVuMB7c<k|C6G
zYLdEdQO5tsNsEqj38(X|v7Fd_Zmv!Bt0!B8U--{WX%TyR$kXA{Qm@%YzEKx5kNSx&
zbj!8+u=2{9;Pq91xy;tw2ueLMOSgNP$aGENDH<LN6taXE9oJo38^N|dj<ND|!N=F(
zYc82BiJF)cD)`aiNDhle*_<6pnkV|@t5c1;e4hEHSXA#j70Hlj#AH>TclA`*tFWz6
zoEohgeKo_^-kGw?wJbr>nCbs7z8c{l2ZTgB`Str^78UB9o_&AL$vul^@44u0kRBJn
z@bT?+S+y&10=rA!om%ZLvwp79udlCePxe1=TUY+$!+L)E`*Odo&U*R2`72+1#pBNF
zv5nWBs?G>kjNO{h#BCSx|6+4?qsc=N(RD|Uib;5-D7IU0USJg!{OT6$%EYM_+@)%F
ze37r-svC>)^_|3L^q5_fh>)~oT9)i$RkSEs(&5#K>-~<CoR;LZOx>yy=`qQPEt>0y
z=yit>wuDsILd6r)8P+|$>NO#i=LDm8O2NT{zL}F4RW);b)DFyEVrmwUTpZ>il^OUn
zGHQO+H%o!}Reeef3tRX~Rz##Mx#p+)cEPh$>EN7(6wS0LkJ_|_99aTqDQI11by4Wj
za$skgDDkSp_}Gq>T{Ct~(kw_#eRZr;K`KRS-STHruN?RMd!_9tE0FSN!(t)rH|vgQ
z{np%c)~;G>%ltO2oUO;>ymhx7OUwSRz4Zv=>g*ldAHANr=<{xIbd{d4Wu4pGyo>7?
z-tSv2wCBwvlSwYm`KAcYQheH-=ED@QP=B?~vKhi9Hk_?f{Jbo6t{fIDyu=zT>|B!;
zDph=SQK(Vn9>XxDXxGptwb#=gL~1P#Jr$L^S2UbS^rVfzG>z{|#n09JRTGX_B5<s0
z_5{;oH!iq|_m<3Y*N**?vaHh0&2xTX*rkQJ0kyZXR#j$B6tTMXtW7dcW%=FkW3i=W
zA(_=jTP0I{%<qQh?`AY#vn#c#TQci+Swd;*Z_|VatW0_Ge%D@x?|5Lw+hOsjT|Mqc
zZU6M^6;FNy=G`y0oqph8*@x0Rg%^#@em_nwn=kiYv_)oRilV#tw!VdJvS}W>+orz0
z-g@|V>(TjjfnRNcgXF*2{ds%T|IUsNPE2en@eco3RU~UAS1M1sr0kh8Vb?;{T6OnD
ztrMAqI2JfCJPnqddN#LX+4bqUFFe{xRSFF4Im73kR7+$!pt)lHzwi79{&NNd%rp#H
z$fvRW_tIr@eObI&Q5#!1Bi6GdS9Pq?`>~SWRW(XRhyU?Erq$s$mxUfV88R_WRq()#
z+$dhl6Knmh9+LQWO5w6mAg}d})zYs7+O6Lx^4ITh4t`k>c}a`UxliM;#3vW$=o{1B
zpBxqJ_gSSJBIB{|$5+vsi7Be`7n8i|zKcG{%u^To*eTiEB+~ic)<-*M%BhrlJM4S6
zPSjErW{`E?Hg$CbYiW&-!0CS#VtzReT=^TlLo%+ZPSM<?zhBVXg^T&*0^K&FiHf&E
zucgmC;bWw=M&rR{E9C~?mLTH=J;It^fl{j$x|y!Z^u7M4Xql9*d+y(zC!gf(-29Y5
ztyqNTw!@hRUDhnm_0t5>5)xGRUV7ph{7P_w@pff}&n$d4@+#+R_}t}lHn|sVP4Tb!
zm8|f+<g(43IWZ++Pp7^=@_Z(X(47XR%aiO*O?DJqr7jlR?$7mfva|WjyKi!~D?U=?
zaC;Fl(Sh~Us%EM6ZpJT}`qVZp?(jJ8e)@Ibgf^8WN{XH)o`yn^23#teUby_{6f%(i
znG+^ByHRsutP=O1QzrKQ6A#v<YYET3;iA6u?o0p1Px1csulCPjGw|>9nsQPswYyB^
zs_EJccWI>#+x=(w13Es7%+fo)RH1i*<>@(mE<8*=0)-E9&s@3q@Ko2Sxs_S1ESZVx
z%sAFbo(k=dxic+xa_id6T<`Vg9)<CCzPK)Yb84jJlWVaHQ`ZFj%-Z-Uh+}8X)7jHs
z90~r|6TVyRsQ&3Tl{nkv>8{V79#yNl82_v5<jxBz26vw>QRm5C>!-!G@rKg-q|LlD
zLv^+(S+}j}Q+O;PcQojD<vgAVM_5mZB(P0WND0l`EpqPfqEE&iewQZANSRft+j)0s
zs<`2j|1GnwEHd*7?3z?tDYW!j#N%0)DtsC89^8%-@|4os_&s;VDuugo$_<<R^sQWk
zG9_aQyXP`KGh-?{DxDJ2W_{<oUBhGVvb@&$nOuDZ{i{DbRq9DRF6)0-%51r%;rEB!
z{*#MRt<O~~dU>wE$Zv;^*qK*xXRNO6^_#!t_j4D9TNddHHMX;uuJ~BD=b-|7{)x@0
zD?ZgRUo!I0uS(qzH+k#wCu-kMo(x<&Wx@^xR?Y*g-e%qo3@O|8?P>XG`ri0;=_~jB
zooD##FRm4yQuFKCLcW4?MRHddd!?q%`F2I@TgHOvQ4hI-JQ_r{b*2_%84Cy=>YAB*
zVG76OC81jvY1#;!JpKQM#*Fv_p$7Z6X*O#bJ)Q7pUt*+D&BDSbUd~5<6<l@5F<l-1
z_Ep5;Uy)Z_7OhQvG2Q5GOQ*S=r^xb-i6yMFyFT`KdDqoWoUF3D+t7{EOG(LK+uatW
z^fliOO<;<?lf_yYHPa_(kM2~<SKs>LJh?(W(sn)Ab|XbQ`}@@826l`!mG>9}URHlL
zx)`g~#nWQgldZMb^F&V2gK)+l2LvyB2IV}P{Alfoiu+GyEniz?x-KL?GA5AiX3yIh
zLWYS(`3o0^-B{@E>b0RE-MMJqT+zHYf4!SL=Up=oP&bI}h`xF4<=HnYjAxsNZ#3Dg
z=~vG5*Y+Mul_mER`Ni6`vrl}KF}eHy<T;W0Bd!TC=JJXyN2^~Izg~OU!{M*x#G9QR
z$CnEyJDyy(v&pW+O(3_Lt#Er;yVI7%4JW@(n(Fy|&-bN^y&Szg=Kr~<;hS*!>E}&n
zru|+zNn3K+m)$34olQ{-;o@DMKW(y~ujGMw=T?3y`4*fe5GrU>-)iC;yNW?3Rf6-_
ztI*sZ-nm|mQn#mQhNW~&JsoN>H!^XK<p!<8;oMHwHhz5SkQU%^(sAot!?#&KJ6PAm
z&5XaXH)?fs^Hs6L|5it&!|XRyzRlkFLF#YHLoS=cheEA)2RrV+a4u2OE4(c)D6)5h
z>p$TM^&yv@PMx;rwDPwiad(H+f<Kj_mYNDL3w1Me`k$M6Br3xtI!nZmpTkh;vq#WJ
z!(E|`vX@2owdOGgy31-vyh<yrc$lZPSWtg5%hzcPr@qF8Z}h&wB9@aR@wd!VbYtwb
z&{)%sg6pi3OSTk5e=u2WBJfzWaOD%#HAg}k7qL_+*)C1sR&v!AWYn%Nly*ttmUIuR
zOLS8DBL4Ym!qUygJZzT#63e!+*i3Y8lbM#-W^5z3%u)V{;~&GsuLs+zi@h&2$|;!1
zpSoBQYm|8XVoaTevZ6-Emno%M3rqFe6gMbacUB~C-JGbFTsG08)WNOODNOiKu%*d$
z5%Z=_&xZl$H``nnPW9Oq=wDX8Br!GcsY)%QeqN$o{4u-NkN-=pcoNhdZn1M(TFy56
z{7Vwk(kn}^RUVj8$)-|O`YC4Z0f+ui8Vzb0F%H#RoTL;(*w&}Zs(le(79hCpfHeCi
z=_?;Yx3M^{6w2E4tY%YM-{!KummPhtSM;4TS29#lyJQ>~+#029$e43bY|k+l@rjD!
z3k3}s^*>G5d)-w3iOKcHbp_K#!NbRdj<~tLE)rxqQUBGbRQ_V}zU17LB@>r2ip(hx
zk?nNLRhjhgn#l8M?s6wYpNNT-vowWYoYWj4sQkk{=7W*rgVeZ*p=+1thfY_w<M90F
zR>0=&*>5O0EkwhKvtaI(*t?=$4m;FaCweU?iQB5cu<e0r&;oTUBk4Cw|F_QB*6PX>
z$nmM2L#QMDamkJ+zR#D)J^12V{-e0t*)KyfF~`&Ix=2ZRWr=%x@!LSQhZCnid}4iM
zqWGRbm!1$c<%5jE2N<~w7<)ABf|<l8Nam+sS4v<MHi+zqI9_Uet+e}vq7Y+%tdkVW
z);#f<Wy?A(QcfnDET5I9u3R+TDet(bN_kgfVYz*fisv-(`spd&%j(wcP>tDaw@6Us
z=_36!qu`|rh3%Cq_erErJ#Jr{q&|0fr5AIEOKSQu=PIwQJ-$<`&PlZH4e8#;UZY-S
z@o0vw%ahv6+iQ;=a&FjK`;l}0XG^W?;<_JhI<5+fFe{mF7NYfZN0vxn=0V2=T`d1A
z+a(=;+z`_@R?TA+=Uc@fptX=ofmLi{zt5Ape>dv%ngqX^3&?3P$lX$ynmOTEn*QNu
z^_sWpnLVN|EYts+WWd!dyzVIL22+tVB@we%4HCwNlFo+D+T2458}~m|lJ;2Q)-`EY
z>&#sb^J)Y=nq9;L72|guH0}x#n!sZ6Cb`GcId(!}^DWUSZw|-Kb1_}~CDzeH>UD5j
zYhugVlBr7)+5a6ie^)9~8kNM{+BP-AMzO7ZQnF=^MSIqa<<lf))OVFEuavR6tZHQ}
zxU+dimnCDb=E_IR;w|AT4T@JzU6nd}7DE)9acI*DDU;5h;etoEE&0ll+`*CDeIk2`
zhti(r|7DhHs~1&GY{^<Zdq&{atDTcBs=FVv+5c+xL9NtE$JG0uRrXs7be6Ylm9Wn|
zR?#zE{o>K?JfZZ18EX$|IFz?Ja5`0eT_APuWQg>Gp2SDRHLLnoE$VxzrTgmCx;HP@
z{e2nGeM<aOhkf{0q1Q$I!V^+ej;`mLz#{l+J>&27%-*X1Ux~4FZ(zvYz$Lx#!}E@l
z+j0b=C(dYH&lXw#l4;SUG{G66LIN)*$nBmG%pxSb({(zh*y^bhZ5B3|+%hn~WuSag
zL@jfY&%{X;BAXwp8I`jv(fKK=U%9IDn9+L2m@Gk|(_3vry#(tQsfW*6$}V2uDY?wg
zx?th*0-kU$K5>nJ$p3L3u7#m5RmGA5Z7wJHiBI1>+0rDa?fu1oJkJu_@I+^GziU&b
zxtedklrr5!QU2Bx`S>5L`yQ?co-jj3fl=6CN1tUy{==1XSFJR<yz)u2N7SJmObd3*
z@!C0g^^T6{l|RabOqV$(gs+;f9itXKagntDs?9Th9N9VlXXzHJDmA4}scydtsl}Ry
z0=T};+B(~5-i*MlFUwTgXK&eQv&c~L2v_Q&2rlnM!tad*HYrLw7KNIYE;w~^wY{;x
zm0gwBUk5Xp%&nP}9`aPwJTzqC@wKf7=kjmqiEfyBVrQChs@hUU_ZU^J^W2r`pDMC1
zsV$wTCgKwQC$skM@%;`<=9g>@Z`t~P|GS9!H?AkAbSj)=@eVhQ3|_MSmloq+Ej#wv
z44wjv5_4F@&m3eDU=;m$Fxo(}Sdk$}fR%Y6>+FVokA*j|pWdKnvyl7tjxC!xw;t}-
z+SC7k#zH}*MVyls9x|(!)tVq5Sg+9)sleEv{Y#Hwv(l>8s8yj#JQX5(-i!G%ghi7Z
zrIeRM#Vnbf;U2N)sMLw09#Te}o7|fi)w@)h>YpsB59zY}+<tXsrJcqe!-#^UtY+uT
zSj!Ag%ZS(-8?Ge}V(lfQB4@`1p44#jm@2SD(d(OC?t(2_61mp}wCYAIIC(@o?}NI}
z45=w|>NK7e*X^<}50^>IQ0_eCU95Ggzr*+3)&HtL6hlKAW;95zZ2LW9YWL}huP5hh
z;EZxO6!hV=gbeG8+g$24Tt2F-$p^UB9^l$GA$6KIqi4hEOFU<%M=hM)ZLyhI_7;zg
zj^VD|mz5S}&*T$Q6je%I!EL=^@h*#H>MJ5kADk)sz~g4;wqk}x$~T{sov)5gpRv1E
zvg<UnZD3gHo+{f@A!{-&9!qg#OTQ$nw6Ni_ny}O1?(1F`Z+JP}^6Ke&r26#sT!EIU
zYaXt>cZw_LP&(g}wMz}`uWi)eYmqv2B7K{|rO-(JGjF8UUUz!^Yn>&p%L^_si^D5;
za@TY1TF>xW%=PSsC(T!w1@ns!UJ-3P7}U@+>mt{i1+0f_Bo;6D&$`BdEzp7OjRM10
z1Gd=#+!4AAr+6+iCGe*xoMp+rVsPh>%iTk9>>|HTuv*;d7kYhGwQ!^2sq2!zcOEsH
z@R?mG*ueF*(+yXi96rfSK`#0~jCSu+UMzic)8yJ*d&87lZj;QpH_wcnH23UgqwdXe
zD<&m8+^nlMdBegj`vgrl9_~;SwEq1#zy0Hu&AM9yvv0fXEZAx5H9!4?ON7+fJ6rim
z1SdOAT@t_@=DBy<1+}o1lj|MXwkceC*(!C+ck1ysYfs+d5?*Ric=~RYbk^e|32#`G
z)>tY%vAy@4amP#9m6LDH5IQhp;>(p&tn8wmT?tn-E-t)3Nt?mfR>0SnYnuRT;DZ0G
z+YGpp4K6PZ;BG(2r=ECb@<LXRfU9c^E-%``x-;it_vw|gjP<jU73cjd{g#p=x=K(f
zZ`ES1UH9MQh-P+Z=j~ecs#Mlaae-#wHse_vt=dx4A6w1~_*kPX5hAR@WV=!G$$X1@
z0UMP)4FfB-m+yR8F8(z|b<X<lP3r}8SVd>A&*1XB@w)rc$)`nTbK~Es&vAFSxpS>?
z#)+r5FRgH73p&8YbA&H;ZV30?6oU-@JO{38iNQUS?5}o*yj`Gv`b7GyHT(5g)~{!q
zFA<^@FS*`)^NMq}_WU+ag?tV&{9aI?EAD4I^XuLFVNQqsX)&;$e)(JXm6Q+L*AEW@
zFYt9d<bQMEKi3*L)-?-4W(zQOD0(f~z`EuD+pG&AxfAZTs2mpNx&Alm<y5&AlbXZF
z?>(IL`YadIBju<GtelHFbl-k)uMY`1;`L|3ccF&AMNz&!xttb)I^K(e_ble!vD%yQ
zn6c2Yh@MV0&0`C4mwfyu>Son(BvtG}&FQ&O2jA^oP%~HX>v2|l&Z(z&o5XiXu=O90
zo9KCUsbna(X)v?b<E6{$E}ii@Dz*H$)NBP-j=YcFscLqdcf%yT&unB}<M^Qak?Mn|
zYwZt9wO%}_JEL9cS^JVT+Z%7b&~$0rJZHum-TM-CJDCLX3mp$Fns?vT-q@x8i?h8z
zzqUm2M$T;kS0@=<i@u;SyWzh?paN^(qtiJ`tdX@6k<(aCx}6pG+bOzcJ?l=PIkV3(
zZao{X_((NIFyCKzUG1aFv#zJ>j8;k|XYc>E#>;xUv(&aEtM%OH`&TUAzAN$6=GBgJ
zPtF~-*~Qr9b1cO+bj_j&)&IBd|C`0|^X!!cT;Clnq=b4abCWI>t4Zb;&ArK0b=0D2
zc0g6{rHk2qQr=5Lz6$)>YH->7V`$o?lbKokg;%(?Hx#}x6)1ZwzMpA<>=k3XyW4F(
z3VlB%_M7XCTkK1>yH_spe7U*)%B}fViXOh4Ehlzg|JCeuEP)SSUf+LTau2I<ti+lP
ztcm>vS-Olt4f{l-&hmP1m};l``~AJo`~t$u{^#5{)W|Jql<KuL^Xajvht1~f$a$nx
zc-loze0$EB9jRXKjEdViBqWQP8YZjzwz-_#q;hgva1@t_MXJ}SY1-yIa~@q(KR=hL
z{N0xw!p}VPbSAg?oI0u*ygYEJ-%_oMD?=H>wDq1&xtblmA^vEas&$xy`g+CFi&Cqn
zrABX2e7tGO?`iL1_f&qpr@Qn*{{0|D;g*mI4vh_mxR~B7UERX7GHin9#w1V6C8>-v
zOv9henSSNzIVQWwH(hR(WKLV=>?C#}EyTl!DQD}fSZ{~5VG~tr@~&qIuMXQ6WxMUF
zUh(Rnyi-#i9=fwow?#2}>di}T4YQZ!dQA1aF~_^}{X^#_{*(Xr)C7ONzq}x#zW5Le
zr?AM0=?fg3ntpxfel1=wf9c-|lLQ#-898MpEIs%kh{=(SHQDn3gZvZ&r#%XHw7HDj
z6qC7iCrvoy%>Vmv{*m8*4*PR*F1g{*)q5gC&&=pn#$mo&F*DDKh39QFJT9B7n0lDk
z_|c=q-F8Mr9xA;{yw&#1xYB)6ab{7c>!cNT4tdRBSrD{jPn^&JPlNq`j7}-{>ug%t
z&+@{lVd;&O%(D|7q~x62eQQgwrp?kTm(SYnO*XbQpUw5f{|ndO&8ziJy~wh3{`;oS
z+I4zxNW%?_iB~S|`w*ni$-?t^%Vjoxo=o$%H+}>%1Sfk>y%e12@-@Uvqc$}3e`NGV
zozS4L-m6!6l7$~^ozQUfwABrd%*Asb6=g5o{&I=JVKK(2#1mrWJBp9WG3IPJ&KEx8
z^@5#7cV4`<cJr*5XrsrYwXj?0otAUM7oMW56|<Lk1Tu+Cz2Sdf#mrZMlauX9Ny5dD
zN99ht%tR9$8TtDH;&&<7^lWHjZ3t*;6Lr&h%)zQ+(Z%7!6!~NVtHjMmonLJO8xJqJ
z5VWFchk~Nw0hywX=sCFuZ%&<CzVD~vyxmV*cseyUWi0C5cH_wM-YTgK&s`xx#!C!#
zUU{uDd&9oWtZ9CqPQ2DNQ+TPioK?$f*@la0{b#pc-g_c<+szoS3(MAZd1<fisN>S^
zyppt1`~TK&Vm6xl?!H;lz3u|j9qprUSFP6DGAZj${;A7FuRrd77B$Cj7sKz<jz@T$
zbH41=nWAKLrT47%wX<uoZH;djF77tEp6p&K5@e#2+kKhE;qCR~9lpM1DbX8w%`Zme
zxbC_cm3c1W>g^AGmalHNeD{0(Go`<q^TzE5dw(7}-|r&CsJ6j{yT0YaVUb)$ncBd|
z%e5WPJzqaxZ~FWB#{VZwnbF{?w6nuH*TYXc<|3EW!;WaKI~QeR1cj_Q64H6zL|%}1
z(bfD$&m&zccv9Pwi5eSS)FOm~udWC?w0nn7Yj~1k_XZOoiL{m80h2v>y<Z&U6Y}(E
z?(&`eWXf`@r>fOGl8^o`c#-C!SoO5S+ePW&gd3uV0{OcpXPpz#ojO5#TB3DC%1N=U
z5p4C9IR?o~?$6n#F!T5ft+3s*rmm@Y>SnQv%dBw8W9j<R<JUQ|_4^~k3bPXD+}X02
zH|T+blgd$<4>P!9A1I!mB~)rF?%B$m`CES9houZt6N`dpzqx+QY$>azivp)$$012~
zC9jY!AMRNzShW`^G#F-hcz;bedcZ);plf4_<C4`EVr|qG#v7fPzLH1M&^FA7QDL&)
z>X=j=<0D<Z6ILnmaz?T4ES=7tc3JUek+X+fk?exP*XfU0W=Sto7A(nYRT7Vkx>Wbc
zZ}x<@^%gVLuNF8>pO})XW_)wjl_mepoEUjhbCef$+B81eGIv9IZpaKN_FpRIt87#k
zMNQ<g+I)#seVXgy$dxBt4>no!&r&VyDdvvx^13v!Z%g6q*_UI!EEhk|nwGRE_)G#{
z@@0k_U-X1-IHg1!oOQOrZN&!Ztbop{kW(tA5sNi@-+4UV&GU0z#OdZ$Yw}X|+?MDU
zu~|E>@|#9OSz#j=7wh~QwFdd$6Y}jB9ajA-)N)$o%N4N&K22f~5BY;U6uCC0w5okM
z#r5FIVY4*`eAZbDS#NLPPY`30J*0Ew#vX^V<<*5ZwZCQ=1r%lQ%sz3~KR2|y>WA`{
zg<sq4-(2jve3c`eX$Lct>W^F2d(4_Pm%hzWjkx_k>)EvEwR0ahx0MO><_NTVHmP5Y
zS;ivmr&zBxXO??U_8o0a^Idu(C#P*wIXu1MWXnm_;PT5`oK-fyRLXiJTp1s=-`X}L
zfIl_Aa-;hSwO!@`FBh&~IBoL^&%5G5-KV<Z1CD7n-enD`@SLzYYT>c-Pq@OkmxVUp
zKH(O7(kq;&Y_ZOJ37x$h?+sHH_}UhIc28aTLLnd}+L33LeC@`b_{2Zg_is`0>VEmf
zpkVF|edZ|)npY0VujoCXmQ@(#D=<y8P4=k9E5kO0>(1PYiR{|16dB59@Ff4|x^cuq
z-G?E1P0IQdPeXS%FN1PR{WDiwbbj7SIn1?9ZtshB{a=Am=7<0PJiD~E*yGkl#-N_d
zv+q86HZStf^VdIip4Z`&vOF4@5x!%Qh|A^K7hA6QEj8@bNcnwh`Z9qHUPZcDiW8Y8
zS1CD7DcKe0^U7rT!7W?8!gnojl$`Cb?@5;M<R!VehffDT(O$aj!P`7P$2$)!v@9(?
z=H@Hd$~<hF8?;@=ulQuzw<jIFfyZOd6`krcdw%bzcxzALvUW`$$@fW5jhFSUs$M7?
z6wPdXZQ-f1r**QaHyW&$_boA*z2cS3+RZj*V$a<o*Sf|%e{BEpW_xOzVnwH0i9!P_
z$MnSy3_A)u6@UH=;(oJ%U3am=ji?RDJjWcHL?QyYmE9G&+^)4SWc-|^uziv9|E{H{
z56hpMF@5usO@SxYh)uW?*;a3JcAv!BjM{(CtnNQO>Cboag0CJ!{f|SDHqHv?Q+}7+
zcDu_PI9*}>TZ7WOVZyJiC(m2duw>#xDb5zZ?8K7&EiwP^s_vW>wwr6KN$gXtaPR#3
zh2DKh-!0efZ98l$EPCkl5sSIu^*%OA>rQK{3JKfob5|*R#Q(Lz-qKJp?hTjN(i^S@
zd}re4XfNIz(fELK^T*!B>yux4<{!6-vp8JxQ*hI*kH<5frcL<o^i(@@-WlfJz%vW1
zBQ?!0a;bVB44Js_hMN0vPrpYMkxLe&+Ik44hHY}@x+s76<lHr_fhlv0qJH>lxN~j_
zd?YvZA^-nQqe!PT<Nl-*&TP6F)q(uxzm;B=d}F`x`aS=yEj#6N-W`wq_3{6N4C84p
z>?hyu;CiLT!n1{i#f0sT@m_~5&YNdCMQ>T?scsgz+#_46;#0?(m&^Zc5ZC-6?|plD
zRh6mQ%;f?i!Xk5auRGZ}Ex@!{OJm`KPJP3!h||j_%vch=*;H=L?2catS8mYQ#IREF
zkA|OvjFO7k6f4ciPG*y{xYbh*O%~eoTa&d(xPjrpHs(N{Lmd{~oaQGb%=Z{*8S(TQ
z$*ek`(7eou_iiHRl?RKjRX9I0lzaTsBJ2VaPoVu*<<^rD%({+DL6evqdsZ7UFV%Q(
zxbE=bu#8P*AJ}~p>@NO~ko&73$h29)FPSykV^dVY%&HqrlP+6M?pzvQxKv{TlS=f?
zAeM<54FY<>s<SrlG`j43fJry{#m;%!YcH(Y{7d@4V&wyit2g~q-nB=-ZI6Om#H7_(
zNu3;rx>H-MQfrRgn6-An!?ky^bY(g@#5si~3QRJRw4TqPu)}%X=E+QjbEKXf+O5A(
zPrPe)$rYiOyYwWqOgC?yrN(Hw>f<`YmkMS-_4YF_`m$^5=U?uPUyf%>pZFSO;}E&%
zSChVdqDaL_{m(D@+MleizqyA+a)af>hWpkVViKov28(ia8%$^sj7a3<m~3GFOORVw
z%ytns@9&9xzYWqHb$4g6R&Qtx>`=Y^e@2h4#gVdv);Vu@m1bLLe4hA9x_NJqk#B@t
z^i8I-%@W#|o!Awd%LH5&^fVZCw{Ro~?)PGvsJwLj7q1CFH%*wpqPt<|9GTWohBf`c
zo4;4vTXvsb!oBO?YH!9H`x%bx-~FbQLB%2di1+`;rzZ&b95`dYB$-(=gW2V^abeMc
z97U5|4>q&!;#(Ev#=)b$ps-b5Qc77$e@4Wtc^4eKq)q*}CA^&k120QW_BiNud6G!r
zy8jb9ud_G?F>5qsnNE439&+0^eDOi)Ii@okW|vHqUeVAMyIK0l9j+%&Rva)i+hDP+
zn`zsT#%+gZ`n|N_eid@)=tjRcJx)`*wxuq1@?E&{|9=6|fQ?2oA1j{};C)zla?c9c
z3oB%oHu;wrZ!bSBd-=iK%4V&EPTplxrrtT&yV^-E>_$`HX~VDyOez<est$NwkYWzn
z!0x+2`w#OzYZl)76E0|6U~z8=u=~=w)=K{FYgNXbwiAUJR3eUQFdPx_4V?UN)0s;x
zO3bGs?^s6f*&nmVepd?TV&h$le!D9Cy(s@SP(jp9$@Y@cSsvYrX|fBgZZG1t&R~9;
z(QGtB*GQq!s6os;qTj6J*qY7DUS{;=woG!+oUC8DT&r`pfXi;q0H$w|0{+emtJWx%
zE(x}pt0%T<l6K@oo3BDv7kAh2So1UOcI33_ky@nWbwy>>Ru!f_3jc%lG^^PBnzhJn
zq0;&XN<M!Vc?r%CtX+9`gJ{gp6J}Pnku#Mojch}XDu>qkUwgd4o@t|llde{^t>0$W
zuos3Iip^CQv}dz;T|F`3(T3GwRTriR_wruY<9BLlitVzEHGzx{`?s*@M0srf)@=Xx
z^2J}>#te5{cfYx|`^ixT8Hc?lJ_TR*@A+_gzs~hNAKHCYn9rmf*ta9dXoIAl0!Q<W
zh9w28LK8Z3lK7AQk(_^0clJ#_?kmFfM;$jQOE*lK)fy?``)R&Z(7I|*NA<$90(&BE
zl<fY~=_~HTdG|}iW=oC58U6Z^(v$W&8XlDEWI81OMp}i3d%xkK7d$I_wVY<D9#Z{3
zr~BB=n{!lGs!wtHsB?a{Q_mc(&Z7pm0u5ygfAbV;YZ=!>E$@xGxL}IulI@kRMJ<v}
znJ+QFwczj#Wljc<291Wj>3>h|DB2gefpx-7mT1;Bdp1oyp};!pv5xKw&%i^@&8Kf4
zTO^?QgZJxY%L|`&&Z{=Q^lN7k@3DMSxA-O8$tFPyZ*w0Py|j38%rRNF6SB>^3lx?&
zi+?K+tS@Ed<Ct;wE|axFGv@;)_D#)IwPM$b)?O)Xy&kKO`BQh&1Gn3E-9?fnNhAqv
zncc+dIPqV^8o%yIL91`<nXtR;j_S_hg`2!D`%QMQs1XXdy}SC3b?u+sCKr!=<GN)j
zdE(orxSw5D)c!}EI1%m9zDM8dqgKb8DcyTKtR#C1e%ri|Jjw3t$u4~I*V6mJPemhz
zTzQhOCKZPAAJ$5}$)dHG*Y9#~hGJ*Xj-{6rTD1>u)ShXSvex75=OvG|{de8)lAXN&
z(yskGkJy{vKD{et^Ijf@Lq3Vq0v_g{<vw_3zj$9_f$kacZ>^$k52fuM9z2q`>qBec
zjfQ2?%fmigzr9H0%?t*k4a^yw8kv#&%d5qOW}KP*ahbEViR<TGixoM~*K``(l)90_
zdHGF$%}tfMi40plcHB(4aj7Ow$jJBBn(0$d%o6w$aVN)DYR}_|vXKvZq?Z@0kn7><
zxMZsL!Zca?NxbzFWs{VBjoV!Q&%5ci&M)or?73IZy<3wqkJtb7!?-vnfBh*E{4ZA;
zJdwHikmun{|Ke=_8-nwG_iVqX8CsRidt!nrhuDR6M|p4GU}88h<M;#l<$J{W1p_aC
znQ}-%^pAjGQlEVO<?bC>?e#BqD*6Vgl-|i`OUwFpF^lig$+OSW>z-xKORM$>kV#xH
zp@5mQV8w(0g$m8|2T8)w8O+}bI;<<iFD*@4zFBSA<ujf)k8hB+Hkui{McKV3a7{Fm
zgsR5w=W{PN=kb`Dz7Pw#{J&Gt{_O5MXOBCaRaBJJ|6sMJfknwG&}MJLl^;ent#jtN
z1um+YH^pb|9#7A-zlH`Ef(`yo-4Lef8FV+^;=j>V3$MM?_Hc)YB}C{x5Q|>(=(YGF
zx0$^syJL&i{12M3=-<*7>z=hatiPCd?fzq5vi8+c+w1vXUHAEf=ktbdcy@FRU-sS#
z`>+W1+b>S9O5>E~dtKsc6`#P$nX${cTdZ+{&m7Ixxie-OEm)Rk)tM{Az$Xx)xz9)H
zqS9W*MU#G*Y~8%2yY#Vv-P!i1OGG-iJWqA^V?HQisdyqSdHPCmku~bGI5akKb~Rhg
zmRhs0#VRt2dC%0`n~y`zt+jBvwe048Pp4achaA`C&YkKsFSdK~C%^mGQg&HHu}j{%
z^z)EG%=u-x?~F_GR?LkueKU1+?y9Qadoy18uUpHzEJn_VgJJLgv$iQ|0(UQO^AnZ*
z`7kAZlZ)=o)iQUV<(RphnmhT#(X?!}puA^+fAx0NMR@xPFjpDu{C%1G=e4vYw_T1d
zQQ+j7tuk#v3FmZ9f!Q|>39=V8H$9lEdm!j+uiLd%%a(0!-S>iP!)3)y$rF_lcMBC=
zu6LVwR4SwArbx}tV`51MrL^1yoz|UByI1<gMCN>ORnH4yuA+5;Lfv-4wRS}x@7`PB
zThtq--%z9Pxi;jt5&xG`k4~5SEeZG8Ca%$wnd*Ci?`=iT{6+DBy2Zak&5fpKvfth?
zHKlXzw7#&vJ1#3dxTC<T6>YfUO2QYvurF$JQ@0+mUB9tK(4e0?arM1KSvDVr|9|Yo
z!b%P;;TF5cEx9f$$|*aJW$lC$JF?ztM;Ux!%=sez@5`dw4Tb@a=5uzMB^10<=*amP
zKG)HtFp`gR&FlPCkIH#JuAQMMCi6*f%8h>lk1y|8QWwa?{ZCwTM$3|!EH5XuGzcC^
z+b(W6^TzcpomXn!h<uT{x8||+yR$ay7H$=M^We(Chi{S}`JC(O%1sP@l3+Y1o!RWr
z$#WAY&6$&1?mf-y-5i<B;+2+(xAtB*l=y7vkDW7A0?uDid{;bsZgKYZ-5sKTt}OVw
zB`@q^Um^3UjGe184)R*>Uz)@xI<0JVz(#qkFB1X}1nzzFOl9hwU(48;QrH(UyD^`C
zZ_53lY5}Y7e}UG}hSOCq*tgbfQd-P@@l=mt1go6x&%bs*bB>AK3)E%l(|x1Bc<&)o
z`wmu3*9TX#+&Dg|MbFr^YWCW-p9^Z9opJoDP!MCi^>^(3Sc!(b!bR79_zJo-ACtG-
zTj*GGye;i<(>`m3=@|}N;~Z@#IYv%i(y`=WX3?SamG5J1j+om_bgMbKS?L6`$no~+
z>k?l6`WeC+z4&y!!7R7WvzIBG%~~9~GIMU2?Yw>5MjSc<-P7)EC|+b{=eXU_yX4i4
z^!GQX*WM_9C8fO7a07cz>=&_rwfU)E7A3bbd_P?k!CLmBU0J%-Rs5fi_!_m~pSL60
zLs!&oS;nqXY$Ut1QoVcW|NpD^M_p(Mb+{foC$T6t=BF~VuEO%;UyL)?{5n~4CP!&Y
z?k~03h4lrG#F&!KJx;Z5xS^78X_{uR3a5vL!=Xd1UMe;{D-=#1>GBkc@lkR-CDt^1
z8^;8ppi^#b!EV#07&)Fgsi_se?v90H;-Q(+%W?z)3YgB#W6dvmp}8Ra!UC&at_Kq)
zbUQ8c-s#YLDkLk!FLY<bS*?c2PU}o|Hr>@~&|I#UcyN|zb(n+t)}-5qYEPHt-PvXG
z{?Xah=7|j~Od?;N?lxcOrn%Nb)=sA(#3_tR*1S%~(kPX2g0BCun3;yN15<4Bm4vNK
zG8pHHZj+iiq4z-8wwUd!uCAKe5XPD3{oiAO!^6Bid*A<)n`5w}U{;XX&(+V@$A8-M
zwTrtVu78?Z6xZb6$>FQx>;qR%v6nxZ6}tAt;;^+zPv^$%4_tiXYV^`u^E1m%Z;W22
z*EOx^+1ESn>J9Z=w(a4K<+kqfno(Pdy0yz}Zgj~q`B*%Z6$<J|Y!!L7HPK19PUC>A
zOx%lw4C^=o*utYNOj*Sh=BiBc(BmqZ<Y&HZNsEY;+Q!8qwpk)eSVUMYFHQ(5FgYqF
z!#j0?kQv|6NqsVbJC}F|+x%3S%-MJ(wJ@ORq^oxlpX;)2y-hD3MPyAf3=7WJW1TxE
z?^vYz+&sU~wz+<HEk)-gPkp$20aJ-$n0Vu|6-kRK{~Lv}Ez&G{x`JW5<`eO_Y4^6W
zYUV1qaY@gcbm`ghMQvIKRxF!#s*Pbu*REA7*R4ABYSqeJx1Ocv&RRAht8!~$`0U#!
z!|vs5-H<eqX_uy9X8tbKBWpID_4~4BOZ~^Nw*}R%3I`;<-MQ|s|Ci%Nx6=bIi*-95
z+Og|dC#`YjwqA2VSx@_Qj(WWA-wjIhQdj*oKeczuoA2|>Uq3m0s$|iILvtAJtC_c{
zC0}{WUHtEh5!Vq;-STzYIdf(7jz8DynI)2Y<I%C^bvqUYoKj!cQ{<$#`A^Yl<I^$4
zPA2m!BqW#Qd=!l1s8zY3=JS<D$irw-i;5?|F7L#NUJ)%9U4ra)x?YU_zvxKNWl#RC
zK|Y@6uPpToZt?wgBlYu@Qt!;kwU@-)3K`AW6+TLSkMQHn5)G+JE)|XRt1P`98T%*p
zM_53vYsI6KrB}tm=bHQrd(!WIFU)D$dCT;K>5rzmI0@xUj95PNyH@)8>6T@`R<G1v
zqNTZ@Q>@x);n{fs>$kX?-OgUGs-iP-=hLvvXT?cfn!fYo^&8Z_e-&5oz4X=DU-yf~
zmvt43ZNj_No<IFP&;H%T&wn-EbQa0!zFT|gZuytF>iZhLToJajeK%G7Zqc-p*ZC_>
zUOmLucvjZK>Kyk3hRmRD-3bT9*6lca((H)?U+arTp%#U9wG?i?*b7aZ3w^a?9J_*g
z{vXRyP;9=sN=0a4t4gcd7NJR1M;6P=YI+!4>B#F45nA1O((Bm9OH-yjan|&8@OBIC
zxN(GS?lhB3pL8K%;prXA#Cf&4Dpub9z~=0#>1F5_@n^EgPA4&k%D}*QnZ*$brYp77
zT)3<sUs-;uLo0OW$B;Z9kJSZFwXCjb^>4ehST1z)>Vs4K6jKshwQmJ>mlYJ4t(BOx
zaly5ndn)uMF-=hUB;w@h?bfZ-l6Yc*(gYUWFU#f6NT&Y3qbeV^!P)NdXLj3T23%2R
z&c}b1Xl8g9%KiR|v+egwPwPz>^<Uq~7G1PsI>VmJ{hxc2WM3?7-5#`)ucz0@N?|v@
zuM4MM(L>hR|8G5saH?6zsb}&bh0Af`*#xzQnF~ASs7Ecg5-mD-_o`CFW<?c6&yTwn
zv98USwBwf8x0(K)9*hgTq%*t}9+~ty^m+uI?pPYKNJVv{ikF_|3vUgnZPJyR>gE?S
zZs#2m?*GQ35o~bE<8IkvZ6&{ekjR(!{#@JMGKEzqqR{&RTl=&bvs9N`)XKy@ITSqa
zma1Ncj$Z;p@=6At^y1@_ot6s~i(h&A)X1bUD|N-HnX9dq8E;qeeUq?V>(;|QwGB6A
z6|)z!+)+`eIQE-2dabMO{g9870(XdiKDCw0@Q1hEtQ&l#KMr^8)oK?_j&QFI6SQM|
zq94zcFu(T3LaFkFo8(_a{@3}pMw6lZWq;+O7PZ<IVTb1pV*1}C4vUCzD6%zqD5@>;
z%+fuwFj|9U+k2U8ZHv^~zKU)0+H-g^K6rNZhiK0CiCxIKe#v5SF3p`iYaZs!daW{j
zLqKcvi-p%VY+0<}w6eqH&m^Y<Q(TN2uXhH72#dtJx>}27tvDokW!{4;>Nbj5-9DBg
zi#bH^|2E?a>^t>dvi17?|5EL|Yk!1DM|{)_TcUL^_tBI!Pdv0E9}4!*kMfl-{CeE}
zp6Z$koh3Oe4JU5usI27-Ww<iu>6-29M5DC!UE8wg>$UAl{K`QY?MJ358!`r}C%zSI
zF50rSD{f8mwEIjml3OI_ey(jl@pr{h?f<Tx{T~m!j^4Y1{jgY>Z(HxBWAcxmO%q<U
zQD2)SCE=UV@eP@?cs(@c2;B0Hh*f5|ch$^{IgNGhO^0;m%Rggptvp_~tmmoJ^|o^h
zB%Lp~Tt4qytdep3<3l@3r3<RLES_w>FP8A%I`K?RyI1;WjO%2z$!l|6c4t^iuPnbL
zylvxKHRp-9Js&+&TIA@e+3{HEe%9OVhYJtt1Y9(IoG>eH@58FXJ?Auo6^>s{+Ra&X
zblw~%gXeG3oOgE}G@Mu8ZsO_N>1W8rnNql;S2ENwv~*(N+~iHWByRMwg>MPwNEUdk
zRede3eB;WqIZGM@A1n>^Q~&n<!-~0kG`1YFzftL@{qyVp@OLHZR@;5}Cmm*A(A1i5
zC3t>K$t>j$One_Ua|$XP=k!qMFaNVy;{O`^s6>fyjethws^wXk*3y2mY|ngCSvNj<
z+5K1ZxcuE$YYsWYaR)tly*=4u%fiOrhZn1JzV!Zd(YZn7u%hGKHXcJi)rssjEly8%
zMRGkVooV8t&wH_>`J<lD)Eh@OFX!yiF?uii^vLo@d+zQv>8!4PdfIj41jC*=98)BY
zc6{H@zOGB?#*xHFH{NmX?+Phs5lv>XTy6E~Vqf)~nzed15}_AU(p$H%D!sIwt*bFb
ziF50dSiQ=V`#p>*CvG{RVmQ+#=tWG@wHPBk)(v{Uj~qYsx0t8i$SqatzrNXNuIO`V
z?>G7CN!Q1m(!c&`vi^d{*75pZFED66(VZQ3;qtj#`;F(`WnFXs_aXU**MHWQ%uCHX
zC-?Wn{k`dKjpx5-J8{jr;ACf7m{A|+duiS@4kz)dmno^YR6NhV?~rqqoZ^);-R*9N
z?~Gn|&I4|?SKK*WJw;t_%Cg?Q<LkBYO2`_Kxl@kL{&36t<t@L9Q~U(`{4P%MKYP@F
zZGu1N)z}-XQkqkpM5e~698^e+ejIu%B(*K{XF}SC1g8SG-=(2~TtX2CyDAR2ZMYEa
zv0!SHAba=Th-gj+h7D7hST4p!c!@l66_Mcie24#^3;Vto{KvmY)Ny#kp6oof@_M7q
z`TxgrWn}m+NK0|}erb|26-~Y%b9Qg!kspdbpURZ+#GlU%-lgD?Q90-9hC^xsQj$01
z3k_#onkg{Jr}1)R(iKTg*B{cpI^6qYy{|aNUT&RrJuyTxYx?vjt>vmmW=`q6sT+7}
z?a9i!omB_tR-b$%%IGEb*K^?+@tUjR;t#GbQt?{yqEn2kJ>G*Mi6>%t!n~%1Ud^^Q
zjMeU~4v1JC^4R2E&su}FrysjQwsN)~b+f(@ZJp!DU@+;0XLqMAcl6qOP8~;FH5ykv
zWO)^Or2F92q@MZPV}gqsxN8F@A5ZX^-rA$l8+l^FO~<kx$9t_aS|g{csyH((akN|F
zkagy4W}}Yh&HqQP%&@z6eyZ*K9RewF4ojSjRCY8hadLCtCooG}^z5IVDGr$jmF1)y
zIj*>9o?Q}pq$BmnQpV%10>>DbQeRk}b<UYSJw}9CDRt%6-jyqR{Z`z%y~8&^fv@`k
zFMHa9yIW7LW4s-v#1}5r7tR(PsdGDGQ|lv%+c65!4{UBfEpd&$^e8TZOI^}UPVZPW
zSHt#=eEeO~X)KdoFNl6q(kVOhnEAbl=@-H(OKynWiH>?I-B7qF`eKWBL8FJw1F<K^
z*do-DSBdTyWRrb+zTrrsOxv8}ozIS!#$+_bWX(&<nx-b*6?1H&S~lN>!y+6~VvA%h
zJ(M~1FzZuV&L_#H-lhw+E&s(7W?gc;!vC{FRjZMQ(=*CbmRDAJpXc-9IN6ekNjL6x
z+*tVNhU&%g&g<Gn7w5kTY`K}Ld#h9I?%caaJH;92&3+*(aZuf$s<lhx@dpjZ7aIgu
zY?)ZMZ=%V(Q*w!Mat|*)nacH4GGoP$z;$n0w`b1Fu9=~<mFdM(l}^j>{3UU(m(FYD
zbM<?3tVFXx#3xN{qC|J*r4BVmmK-NLq4_gc%759J@!7S%ROEPfT5#M3jZdM%J-0GK
zGM)4<xSMF+H=Wu&LFna#R~g-GD<_I+`reu26?Mk_#(eLrMin2;NjkEVIHoLl`Gg_r
zh~>OwTefz8Ki?OU+%vTfvPAf5o;xy&tKq-zmY2-?G(A4Huy`%-m?&~*P6%6GLP(72
zd_UIv4?M$O%vygp<k6ccoX0{qH!XO)BVq9-)wqOn0dJ3~O^Wjnhzq{j#%DLR;7>y6
z*9Xh9MBh|QWw>xo?&7hgR<_)mPLVH<2pcVn3s`6?6gYjsOBNM|S<jB;v7AqExg_?5
zU22zF&bq{g3(W=}+yCxKlU=mzi=vYBvuB&@WVACH|L4S{i?&Ofz0OP1&f2%^tHmP?
zgGD}GjbZ_vQ~9QF|2@I8a&?jC-PK~}H)?Pv*DcCBk-gW6rL1YLrr6SbW#RkRB~@&F
zbW=A_XVcwEtGLQ-;fJ0jRT=3R{7W*hT4V5T`jKnP>q95~7h4r*>l?A6p=*tTgh|ty
z`a@igO(UMDvY0<|mCm?vuHr`an`=2+m?b$<H_f?W6~|?*)-X$@LCAphMZv+y&Uq6f
zysx?*h?{wTJHryQOrD)KD)SlR^)E1|Z1vXbWr~~1KJ8|rvz&@&P|9_;6vtWJ*Tm#^
zX7HyJq<qv=F>CCZux{-&sZ7VdCmwOf&-(fwjPRLqEahHb_B1{b&o3S#E)F6dj1Rch
zJ@}QYW;H$DN9)zyV>7~Bcv)3DbEBqr_?8I<GAHTHbxZJE8WpllYF^O6h!e9yuicHf
z5%q9FV^~D%`r4(BR8~byQ+;f}_w>pXCy`T;`w||w-j2=C3iMR;HDLL_Eio%r=}lA8
z)!0v`_SGDFeT0weqhM~s?9~zTpKC5lGgB)mNOVeA<`nQO#Y=r@*ebCNO$t$~Glc>q
z{o0d#j(=U#^xxu|w&r9FpGd>p;|Hd`&)Bx?C!6;6SNZ=xy+8IawBgI06A#lE`Hy8M
zz5ahF?S!Crmd>KXJ&c8Y+JzHeKge<vi80uj*QkG%?LpN0`n}nkYu<l9x0)s4;oJw=
zny*^R9<kosnslH}U3l$Wy+wj|&pw)cBbLv6P4%Y_5*L@(99zDKBkqO7jJh(NdZl>}
zr><F(viPLg$BJ1wg)6)iOf{Z2&QLn3b9KYwwz9PJxD{%1AMX`pYn}8`;`W73FGt;u
z`~Nj^!;BYRs+i)w?~eD;HBP#|xrVjJ4|NJz_c;Ez6SqC$TCY+3m!}yYuf6zG*{yLQ
zNQy~wvRVAkUAgafBu?bgoD}v+L7e^Wr<q@OOgj8z?UbfX+LC9=T-wDZWOfFfeq<&-
zu|QLM@=C8IFSWE-W^c;$yHGTH+u1quqC5f<yd{p_PFOJiPVc;TnfKnzVqbOC=is^p
zT~9fLq`AzB7a6@W`93A;-MXr)3m#otpvv3%?Bdg9r#eGiuP(33N@>XwESl<2Gxdq(
zygji39TS^Y|63P*RIB93(m5`ZM0(aA(DX@Md+2A!L9GkNC2p`Mbu#UkGAXmesiNml
zQ3k(pX8-3E4;8fx{%_G+&G`D6YxWnd3?{R?=Nl7(8D8>Vx#ryHac!d>&!Lc*^O_(2
zajnkTw@mTv!?~U8XW!mb%yVM7puMlrK=j=g|MS_L!OJdI$X=VHw13-%{re2Gj~R%z
zJP%$!_Yhawvl|<NHT+&aTEzE%kv?1DJOSnBJZ?s(VvOJQH0wsJtXWfUwK(!VYxMCh
zx!lF;UbLP{lS|&xe7Z~dOxnlyQ)Zl9wmANMwD|eYDusq0mR<aj<}>Xg^X?AriBB)p
z864Mg;F&BbHo4PrkH)4Voox>1g$_kT9;<lkQkR?2A9;L6S!T0O+y-WiUha(Vepfaz
zdl}EH-f^iq=eS14p;(zOHy3TYed_K1zPz<@Kbo9)%;QALLtGuTS8tl}>6oj-I_>Ez
zFFa%M4S824`;>WBuj2*Hvc*x}0S?|as;fKS8GPqluxrY_54Rp32npX*{*XsH;*j3E
zS*n{4hQzIT8l$kLJGaHQlkcO{rno86pV%y3FW?hgb~|%}aiUg*uC8LU{;}^x&+41M
zCU`hH|Icci^0wGwnnQlY7mnb{2WGlQ^6&OFnNDmk@#(iW^UoJ5=-zL_l)ZY>uNbij
zMj39$>*rjm)CtqpHTj@;?R$rl<92uX;xKRJq&zEaH}wKH{s~*Go~M~{??1ZqYu|>i
zyUv!TrN_y975utdn@i``RPm>Vt&Ltkbl+{byD|O$tnUW*e$Jg0KJml5^f}F^vMRjl
zwG4%}`!$Mb$m_Z_h0lA*A(U+KLUjL}y`rAWET?aY(P;}x?938MTpwpGR3=`1Nn%F)
zc9-K4EN9}E@=xw{teV4KZ4;RjeExlHx5i)P35%>sI5x#?vFbB=J2A;9HTTYzZQrJb
z=I*&+X#CYF@B55TS3brYeu_W&;?34VX?~+#wfknVMw6_5-_~mRRxxSgh2*XIA2r>}
zoffVP+?VOQ=cR6QwO;v^9iLrT!yZ)J-ykMx>^(Q{{uYsi3(Sp^k5<oru&_=rz%*-#
zmqolrUYtYhiwmjmn?4_4e|Vt$)yHS2!!-Y_P+G{ht<$>s#`&Al|NWWW;|dbLo%0m&
zy7cKoar#oBLmp+$??0BMOsn5wC=qsX((Wy;#yd|P-;s5zNk8^@!<qAr`7wV~Slpd&
zR_@T(y?cJwmBJ5>pSJBYIlcbR>HUd%TeYV&8>rvX4!Hi>MWA$FootZ&R>75zg4)Gr
zOctNLXZoR2kLKJk2>Z`@^R)PpMSqtx=*LYsXe1_=r+F{dO1U}GZRLg9sLOk!trxev
z-P!cZHg3(DEQuhCzGP{MUoB$(OBuHLhrg<~{dH8#?9-1`KV5})SmYjcIMa2BecH<0
znLoFA)YWZrPO+ESevD)LR=!$u)fU?uOs}sW%63?BYL(94mon2jPwU3|Y;5w^zjO6}
z<sajz7w@HNx18zj&Q?$9J(l}|-{_83{c~m&>y}3?|5P43>DfBH`#SmKl@6Eb5%0?U
z$|tw%e%2HcQ1ULYM(?i1(Tel4X0D!MxPSNV{oBJ%)vwE4`udzX_cik+@7BHJQrykh
z!qB*cMNDVLfd!77OzvBio^A+r>J-)tn{vX9VRetR=C+$2X~HW56&$&4xGYncIgy{W
zLZl<eK|6YF*2N`}UQSJOtXY-bWD34Gzkq47h)z~O_xiej)2?a12uZMwTK(&*@P#ic
z8I}iZEqnUQ^!2qhfyc{KtFPs_Zj8>|y87rV-6QR5qArSR2(Mr38~w`g>veN2ZQrfy
zE(xy*Y)}iU{XgAoscP0$t+2&m7eAl2&<$^%bvkd}2}#|I1%7|8PTCZ_ykQPg%_h4a
zQR#2yaHYKVS)_1)g)w@`hi`kOe@uID*qleyV5vgU)0k=I>HliDr@vrYQoOM4?yj%X
zk2PoczmTmt_~y~Z$<Isv?)mzK>DqSv_4BIzHB&q`uB^XbSNG@p%je<^f~x<wT|U2$
z!PAlR;C~h;p9M_(A_|U=W12SD&AgSVs>Gu|MXB+&O-zRwYuBz`jm2@&fsBh>L>gSg
zV!upq6%3y9QA2ENLeI)o>o&bw8Mh|ujC|v-m#i9c+`kHyl!YH%6mM?wS#eU0b;EJ-
z>YR<6=FQ5=S}dx!_su3zbC!bthefSV>0B1GzT^|E5FR&aky!nVPl4`L$0S<CmfsL;
z)~Y%p;d=ZS%c@q{up1|hkFR3d+ZDLF!$s(ln`-ZrAkWq<C6`ne-+JWCpDOa;a)7OD
zs9VtF7X{o8xi0xCJYl)S8^)M4H}smO+6K|<(fN0ix_Fm8co6=yPq>4*@z7#DYrQSO
zVv(scOSj$pJ});c(r?P!rpbCc4ovILS^HHi`r8rP{yT+7d(T@mMG0#hE$zJZXkmqN
zSHEMk#G363sz2WiR|@mUOcrVKkb0@owB7ln;tUpr70YIBnN!|<E6jPib$PQ?V1q_h
zX4lFycE2N(F1qM(uTk*)d`f(K#-eV$%IB;8doP%oGVP@3%)2?Cr<uLiIKjtwCMJSA
z?BA!;3T0snd5l)Q+PF&WKpWrBRSVZLX`4*4-n{G8n#+qk)&weT$lZEPr);hIt+ej5
zIX9U+9Jm;7-FusRk44BY^uaT|74P=M%5J-wxIE#V#ex5BdRF1{7r)DYwM;qr<LjQM
z>$;ni#50Pzj#dBp^xZ_hzxd55i~b%XD}LLB8y!P)C+~U_{P+H^D5VSq=G6c14>D!W
zzgxzTA1<KWoU`0NaltN=6^Ekk>@rx!Yb_GUq?(Z&{^oj%((J7wtM#j^x(&oSWG!pw
ze0En_FBH%4>Fj=!ed@xCF3r<iKj-IZvwy6Uy$w&AT%Klf@z4L8VvSW|3w&~aMKUm2
zI2d=XOwXObB>iWh$Oq0P;*paUERssz_rWv8<3)3V<GJmQ{Cur7Y%ddjT0|WB$f?;R
znBpv=a$%*UsEZ3nq~Hp_sSJ`gFNAOv#4)IBXyx5=R;gP1-0BonhWLYA;qG&o*5{SZ
zQTxVa8N(n}YM~(WULqy(w37d`J;6IqM7Bp?Et<A<o2%P51MTQ*FI{$=da7H@`NXC`
z$3y14YJ;E1lRl=C$xcoo6UDYouq!I$ILa|opx5Vwl2jn4rr#xxNjZm9Zg+AV*d{F8
zezV0O`oY4SP2nE?(^j|yX0Y~6TV%SSqg7{x&k4=iK>q40k90TY%v3N``QI?}?xwVz
zED!YZ*$&M8nxD?Zxs0JMazf(2{u!(@Pl}m66n8A-4Zg6YBV)<WlNT$N%71ymYRloe
zV&#P<+5C%6LS;^VEiU<0NzS}km!?hT+j-V}??m;xZO;Bx_ju<E30|{fy?*{6qhI08
znSRgLG*4=BdNKL<FTP5-_MK;KOKjdX^4C04ZmihaC@OKW{}0EO`FD2aIc&Hb`%d81
z?pp`nC9qdbD6Dc<G>TZECNU}Vh=;3l*;bC_ZVM-V^$1xdv65rO{G!0l7Pckwo}LUI
zUyn|n5;Z}Cv!(X^=}A71!@Kkq3+jJgmN%-5bTF^ox;lY1d-kEIu#KNCiCx<0zDR7T
z=9m93U#y+dq`g=#R8!@uX|Pyfh^^9%Em0Sk=QP@0NyrqOD#Eu?e%}ojmJ^YegFcqH
zT29H~zM1B~-~!j?t(QXAEvV2=`1#5r_|74bjVA)$Sae5Tsk)ZMc`C}N=yde;qGai_
zuF<#8ZQS-~m$Iz(rfn}-6O@8hbjOH=dDQ4;DE)GJSEMOqz#*EGpi-%Ks7qRKiV2VY
z);qmN&Sm<(SDT)^lx1f4yh%p78+|4eEB2^<cG~>!=A!9rg-6d-x!q5>c0K+3)0=gF
zkKh0EP5i+BzxmoH7O+N3e4Q1=&2=@*GLB6@H}lSuXA2`fp5<x}p8e&8d;B!9xb$zU
zH?An;wYnK<wx#g;|D5fN4J^0R{42KoFgM(GyXSM;_Puv+<p1ND^RQyh+#`}#w#5E<
z=OU72*%9Pk;&IMzlI8IyMjMWmtiQ>0I&AvIYd6yqB__&y{aN~y{pV^&^`moVh+bJ>
zB>tqFr}3nQ&ZSGO6CYL_JAI)w<CM+wpDoKpBd>F-J0*3m_26{tWYC<pE5y=wrRV$2
zlYCc6?Kk-O#wW{Y@{-5V0!%^zml(}n>)lpS^HzVdI!1MMjpg5$anqc*-xeq&`R`B`
z*|+OzvFiMZN7%!aQw^?Zo|)9bd-b(5<1{&2C!HU2ZW_v6ta5+ynN>SVL-~1Qk*md?
z%&#7;ho4riaMmfiy^WR0MR^_99gqJa+taLfyTr({Z&_S=Rb**~Yv&rX6CbB>OgO(n
z&Lpeb_Q$`lz2`mjYTk=&FA~|2_xi-`1q|kj+%J8ry0d*<n4UgtpVQ$kB5BZMdr5&K
ziBr|PYkGTRc+y-GzbQL~m89qKDJHLQ*X|Xc9(jseZ(Gj#vk$oMZ;-k$?dRM0^JRJu
z{(aq^?s$Ivh0b}!3#9id9p2Ntd}3sYImh)CucWz5cW$1RQzop}9>uDA&9QsN|Bk~)
z`+mhP=i3zL(YUexO`g?KciVUCigWemE$@Elw*B6f$;T>wZ?paW(tN*p`thqjZ#&3K
zDZY=pxMgYWcQHx(!1HU>7k!-RKmAqm)4sTSi+BHC@xSqqOxj--{<FK8<2-Nje4HZF
zp{8|Opv_}q?(@6PzFIF`_~s|i;dxc3+Bh%#eOGy2YK~DpSKqeyC0n&#{+lcMbGBUR
z%E|R%o6P3_oZa^C%i&$EffBQtyo$7ASwAg_HqSg<JC!SelRGDh#rj6g*?l)w3oY^3
zDfLffOZKu8c5TP(%D(A7Qa+}#+eTF-;i#W8SKEo(8cLhfS~r`$4x7>uUc(yUl6_3-
zRG}aD)=8{e`*=?5<LFaUk2VXiy`ix^&zq&#yPwamt%lp2<x&46hPX@FaXTLGut?IF
z;?4Hf)scm5SJ8)rq(wUnnnXAj?KNmIIKhyUl&tZOCxj`%?b@-nfd2vyE10-Kla<UA
z^t0O1bl!?Not^fK`_h}!`<T|eR8!(HJ-eSLX$Ftel>-yfcNFlj^6-}Cai3stH&C`y
z$eH~r@z9=ghd7ohnCQH*;W0kJr@Jci#EB-6B?smlI`($W^Aazfs&i~Y$&cqg@ypXN
z%zO84-jB3n-`c__9Txt*ET192pzLvhSD}?kK%&EXf!&KgnjJsMaq8rqKBcM-?+EtA
zRnP6D&N-U-fA;fIc=BBQWRH05@ueJ#zOX!yP<JZ7=i~fDr&8w8ty%4Y&P&>oI=+{@
z4_WcNwM4^pNucSJ2mL=Xltdi3?i>(%_uR+gZNoq92AkY9cP_5B<Gu7CP)RHIXUzYD
zmp$If{XOpcMO|LqmuLR^hGQy<66$_WTr?N4wJN+>|4(bf3I7c&3V%y_Z?wH#m&L1`
z$a_6!?X?MOuX?;tHvZJi#`^XhziWU?&LUsc)gR5)tlfCS?RLY9cO4yS-7C~eT{oY|
z-8{p?a$ZON4PK4Aypbva3lAw<pI8x{!reRV(!E)e?w?r^!@&38iG~)p-h*F0N+&!`
zCn-#bNfXt4_E2NdqY$NC3$#@$Ht;IEw7+rYu~Cb`Ed{~g$5jQ%(%eak3k*!(1u1!~
zD^W>eI;FsU_6Z*o$ILgYdHIW92z`EWXu|U;p3gtM=K1dZLTw%Ik`+94o!nE;&FDFK
zd79r0mIq=U{}WIBQ=0wGV2(!5>pgjUPq80b;`XLT_u#%DGxMb!>TY5;->Kg_XT~t$
z=#m9TxA7ipD$X+yeBX6=)2lS4f;I2|y`0bG{(<9_Rl}JNMt)}9n?x51Tid5=?E1Lq
zUu&7!sl|tcKbxE?-SEcx@MNpshfCe^t$g3B2Ap4?<+;q`;JFeHhIwx{#;w13!0-ao
zTEYLt()~$Ni5(8YPF$xNez1A`5}WdCPQRmgiI0=6tmj!#uLf?j1SXG!)t5Fpo?fN;
zZ1JCS{@Q%6O&eVp_MBJf^L1@~axJ37JuD<J?u*By6t|DB{=UdsSFxmV&#{eJUd!|J
z-Bh&OPIxlhZ42cpP>NXEVens@(bBuKhILC5tHm{64fPGh2JBH<1yNf<m_PgWY$>^)
zrp<D^gk`Cab|LfJj*z+z?xh@>Y%g2mQzkeHsIZ^>q;k$kiQ|y6OVi%8LkdxiyH~U;
zdFW{TFw!`oVDOy#^1T4w{Q(DVeVFlw+tB{9!kq-Jd2cSCI&k@vLdm-YymuDxSTu06
z6gw%0D%~oW>6Vz`(V3C8<sh42cHGtEgi^EP-b#NOMJGMTb~#hRc15sQU#Ogkp~i`G
znZvvuQ48(Bya#OsZLQ`U?pzju@9hGG^L>ub*et@XAI*QtN_B!LC+A0*RVNb)PqJK9
z&q^`Z=r4^pc*=<B^sOyBzBPZYDm-)Of6kd5J!gbmMAg?VQCK25ak;2oi%3|2l;9!z
zrOzvuxjRHBCvFs)@=v%;WC72)i9Z(UEB;FlR1I}{+s@;4Ku(iM;VpyDi6gu_9)x))
zD4kj&YL>u#q=lvF{oXqXTW%O6|14C<Epy6E<UW?LwkXBVBuK^FD5HO^aw4ya`J{}-
zu*~AzOv_8I(Y>-^2Aj&b+E;w6TH*U$Zne~ZeTK+0*^9Rxol=rNWlGLrU3F1`@QO>F
zQ<U#E7<DxiX><nVJ>O>YO1I~bhUV!G&GR08S2AM0=*GO`usfHm^*Tdr>!x;zbzv7>
zAKg0h=#-G;%g3rxi&VY(C*3kymb6H<vb50EFChNEvg;0oz_$l@yWTw)`dq@B^!(HU
z9)Xv97NsGl6PV2s8rVLU+<g%2+|DD=!LzD^+3bMwsRSmA1|E;C>1GYwDPpO`&-kJq
zmLE+qV86^$+4k~}_6!akO@TGVw#-@EuFq;(qH_IYEXUTj=R9lfYI4>TD^B=oQTN%r
zSHoiBVT;bgk&TNZ+LA1p`J*?QE;x1g!=bMqI(!<Aem$X-uc7u{>}YV~abf!lK8@D?
zUur&<g{-pGO>$~z*ITBQkh)6r--*g!m+f+2m1L}6&N0O;?^O%q76)D@MvDhN7d|@5
zZt-n6s@(KWCc50G&NXoE8DG6A+#Uzo`?srlHMYevMx2>=X~q9!<`W5>z5Zz{3k;mM
zInO;W=JC)i?CiCet*$%Q2Nv+UHz&D1e&wDZ;vQ|{mMrowC%0SJo8eePt54B#x2LWv
za{E?K3wiV|d%u`P^CO1oA?mZQgi9GjEc1x?l=<<=H>S-V-5ahj1#)Z3?~LBSTJYC5
z-~Ga*?(V)@s@h8rq&{PfOE=j0$b>C(0$XSLF1@^6F{>2RG?g0)(*Auh$$D&-epBJt
zv4GRp71R`5w<TN>DGi$V&R}T=_pJt=TMKy3xRjh}VNq-FIhDZW<X#~dVtVU<?}WoM
zR2@Rfo0<D=wljQQ&32jBtbtkOfHIpgckA{!v5MTvpQle;Suc{2Y^UQI`(LytdFwoT
z&);9P%>2_t1SW=y7RrSGP`H(89`3_g6Kt7sDYxfqWdGDi`CpNWQNr0hQB#YL>;0N9
z;J$!sRiTKMsOs@8TI>z&OdofgD4w<^rrC*Wk4oH|=EWRLOI)~?ZSN^-`WTn}t2A$x
zc>Qw+w<HalM~mH0N?KMhJH5I2?sMfko+^Q9iOYF!ec)L&T`DBVg-gVN(X8ReyL4mr
zOT33vu0%0*crOq+a%ASj=`1xGJ{NBB`D_z?lfbREV8+ZHsV5)kvw!YrE$2S9z}w`i
z*fh!1=661)61FHMt&Vb>rTn%ef7#0|7p|l^F}Eyhlv2)gzSOd9#_v4e>q;+Nsukz#
z`EQY7zdO@$cBZ?v+PxW-b?dVP7ilbc`(#tu*8CMAn|v8Q->BskxYLnwryywiWRCEL
zr=6G9=HCC7yI8sV#kIwcdlx7~=Fb(<nm%dLTMib^=?{4Nl#~+8tttx-Z&Lknl}$NM
z_w5uN_eEb=+Sua_bbn7#J2l&xDby@~rczJ}*9ny_&KJSa)446S8l9I4ZjCH)xytus
zSNfuv2WH$9J=HL$w?eUG`m9t|$I>$9zM1>$E0Rt->_4h9y^O8=XJFO!f^-w(12%V_
zZ)xC-C>8zrN;@M_M9ulA|CV0YhaOhPsx_C%S-93@N@^@}PimWLUmI-RbM<&MPh|9)
z_Bfw9L&K>3|JO!NTpXdJZTa7?etvSj`L6ooi<`8Qmh>_=Y~0+??!*+-{#9bf;!QdB
zM`G+x$SgT`+5Xa>B_>=8zf83jc<I<8m5`dnbfVLK@5UoRJM?4&Wqes~h0PEw-nmwJ
zt-_qhkn>ecZINZiQ&NvVk-7NG!Xsf{*tS$(vzZTm?eA8u=#MPx-oAeO0^YnCELPjr
z{+(#RId{59h*N%ES7qnS4g6-N9Pa$wO?<4ieQbtpDWXRh4o5hOgsc;HbL3%I{P)t2
z%+ovErmFPp|B?CJOQELGeYUTIEo%~Y$kyhN?Bo^M>9cQt-*G!fdi!4q#*1Z}r@Yua
z#d(W@LDx19@7G~AJ(IRB`mgG}l6jK3!8Sd~`}a>M2s}`bc)e7DZ>inQ9d}IpxE~e1
z7MUu1L|KVR`M{dJ9L)w%iqljb7+rYGb8V`v3Ihr)&xbP0F1_x{Amj9JlTV9tuyM#-
z)`Xdb-#cO#@;=ygKvhiMac}$T7i(5l{xGmH7TtPS{Y$zL?~8i5pwj`#zh8@ed&Ixg
z@9>F7*Nq-!RJ6#sHJ+WbNIuI^*v(OdqggY<QJ5>+kxf~d?Xi18&F^<VclRujD++yU
zVZPw!oK{~RQ>!IMFPTU2OP#piw(y|YqVNAse4m&9%kKTGJ)v8YMY!cpEliHSyEl4m
ziIUdw$zuN_e#8d!A6jIP;(Pn*sW_|CasS(GPRFoq*AXw2I#X1o@Mh~&-}$!nyW*K&
zR;0XgifTOe^YW^tRx5?CePg|1FUoM`?ZcNlE}n~h{Pl;;1<?d1uZ7dM{(2f4!C)0T
zv-y0-=UY$Nqtk@{u93gP=k#^PM?qhYX$qRTJi;LjBD41!<OZFLZ@Bg$jd{g^M=4ip
zT(qw-_}8U~x-8tV<DWl=@{QvUT&6e49%E3ru~sT{^?duPq_C^2j(bgEVBCA7Gx7TB
z`3-dv`roQ<<S%yXle2DO$qamQe6m__o5%q_FCkV>6^+0N4hPT7;%C*U=m>UTV)U$e
zCvr+Stx1KGNo9gk<I2m+0+#wrRBCwTtSMZSR=XqA;rjnI3Cmk#wG+B;XiA^sa<ER#
zy|qc^7FUOMgLjNx+3ROVXB(^E-&MEMM#N`^aNMC%-p8Rj4I7j9^|M;59bRepz;Bx2
zcTolv^(Ky>sEI<lQ4A~3u3+8bd3M&<tSjrIH>VlDGMVX{;&!)g#_XPkJN(fz!}fYQ
zXh$FKm)<{1p^@eJx!?c(Xzn(Z4BD2H+xRbrDeKkBaxIO=r~0&`^UiQ<Oup9piZdnV
z>RpFtev*8`k0xmrFAtS8TD`WcU3>c!_XXO__Y@nN_<1wj{mOQ1Pw*@E`0>D5(2OH7
zizn||Vkx)R4@J%B4UhUAHHuWaBo<u=>bGBQapaQ5EDi%Ndmhe!3;&%Bza2iOv3gEK
zzgjYbC71S>IT77zYpxVu>{M^kcseEc*JJMBJHF57`dc=t$7Ve!Y!_j;=ee+jB`{#>
z)S02ap;u=Id2V!^6Me+;c}VHDtp}r0zwUf4R?9SNR!q&d)E7}PJ`-Oo>eAZ9D&Buh
zR5NJ`Q(J_11Mf89W&DM=rYEdu+%<E>lBldKzqzcUSDZbk+~Cq)zwMxx^pXt=rL?zg
z3Vgk8*EcPP^?TS>ug*Mh>s9ndOHuDr8@K-3_9p*AlZ15s;;q5n+H1EY$83u}Qu1tj
zE$`CH0gtBbz8P>MH*vC$nQcm5w)MosdwZU8J~$>O&2gQJ{nM>WY~r~$lp3`2wmfqC
zU%Bm!_6KK9^*<9DzwoV@S0mzYQ{X2OKWAg3RQaBb#}(?&6dqfZd$7%3J?RO*`)Yxk
zN%mU|Zu;vk>Tx>nu+`&8-=sxrocMJY_c#XHF5VL1?;3C|IK|Z?z2xF$zvv?quc*el
zl)42v_kO)5<N7n&N555f-VObiB2%V2JYAv{V6e6KN^pVuRgpO*%fC$xi+gH&EmHWy
z+w!P<U%~4Sav8NU<}a$NTD7$PW9f~=DqFuBN$rwq4&q&fi4m!D67J1-J||l2*NaVx
z3J2BJPcpmZynNv`tEsa$2F-o5;-Xx&-?oR}X20A1vutku7Ov;g+RHaK$rNpUvEp%=
zVc~aG|5!KvzyHfqZzyGz-}jt<IpE37CWYgxrha_zgWYO}LZfh;M#FcmRq74DKb-z<
z*Yas~{{J?KNA~sq{@Clx{#Cp4zHZdU)<B1o3Vu~5H2Pe)12;T!7hmaq`jumU+m<t~
z{}PT$RIx3QQq2@$@|eWHm!Yg;%i_iF`5-OihPYVR@zza&3nxb?w(8indp|s>n3Zuz
z_4$OATAmY{M79Kou41{Ul^4Yw`r^oPo2jl^{yH)ub0d}?mhJQBSDdnF(+mw4$yMF`
zlTI&EQI)tqr7$Q&W9qVNGgg`%^7G0{31teJDy%tyF?#KVtE=0Na@`kLU3y({T2|dr
zt2D;fR=bj}ISc(<ea)~aqW6EwwD<F#9)GoF*`(_yr|I5mboblV#H0T+Y<G#1tlT}9
zDjS>Y{u`9F@*iH9!X<FsdCG#xyEJm*R!sCi^h$MOdu&F^{y(Y5YhIr2y4pWOV%7XV
zCSv>JtR_#F=2kOW_*3C+&q29;4uNL+hqz{%8n`7_T(mkeL4`BRiAQ0|f{Qv%+>C1$
z$vLcUoywD>+99K>F6!uM7aBBa!jEDVjgy{UEW3ncJsH*BaH$50-J0B1mZ@Ry=oK)f
zKxF4p-n_g`r)GaRCffJv?c!SH(<0|4P20EgXh7rQ?$}#N;tod*3v)u|EYn)L^H2z%
z{*=ZZKh@2bkA2x<G-)EQ&=P}Li@4G)@_knQ4+>VczBR$ge8<J!km6)%0X_B#e#5vT
zma;3pCd;iaWb)oxQsQnDz+QbT`E0<B4|`dD`q%TEK6%K@B>Y;*i*+K4CUksSwQ%1%
zeHY$atK%J8t{<!m;@;*k@w!ul3gdxbkwq7dI3=|?GH&@iS1dE=Qk5hBaVs+~&8Srw
z?yqmW^;l48K4)IW;V*L*2Q2krQds1@XY!FAzoP*~i-bkqt1#UAbgiKARd6<+>m4UU
zuAc6yD@)W;wPtO*7(7o&x8uqU$*`rm8VfU9*Ce@eMPBR*+g0UT-f9}>GATT=TO>@m
z_^Qq~xk-~QZhDetl&Nczw_0{VOP9v><MHK3FXg=B>K9@7Zz@u7)#w0+-F1;S8@$=9
zKd(P2qjsVw$l$xnni)dEktfV@eE6y(4j*{)q0(z%YuJLG7Gq&oi(;WI=ZsW5RU<jB
z_)R&hvDPPR_DU(YUy+kme~maE_PFEPF0~ZJ@UtfZKDRQ|7<OG&Dp5--o*fW2bz8@(
zUhmaK(M${*gu9(o)qHL`&3&=W<=P!y9R=2v_no|gwjEK`ZE5CSc;iRVejc`vN!2k=
zzU2g;vRzVK&o=jk$#scSHGL&(J((mEShW{#%JVuh`CZ29SJwBAZl82W*kQ@`TU}37
z&zmIp>dx3At`;=gs(RI3pM5vN*Yn+6$eXL-Ed6BZnQu>@h;4ty%f7>5-+vax45z=z
zy?^^6Ws~-cHXPe`BmQCg#tVBm8UvF$-ppwctnzq!Q{99kJH<m)V%xFX5Bz+0L?-u#
zEihY??E7l>gh>nzfv(#uE#Hb{R9f`j+U*;7#$0r{h3ms#(%lUCQ^eWzJ{U|(?MYT~
zUh*R6xnHi=rK^pZ@d634Djo?YJ=3Nx`zdiRHn%3c@yJz$QdgV!n^_Yls<@s!p1pdi
z@u&VRvzEMHQu1j|%yQo2AD1xLaGql6C{j086u%J6c{*s)YW*!fyLNk>EMa7kJs4~u
z(R}y2fO$b8b96<q=CX6A_X+syUc2G&KD7;u4SV@(&QD3ZwcOHhYw+fzAJxBaJ@<16
z`thGBRraO2LtxmGhG{9OQV$KU%yXG^Ksrvf_V`DR-%ZI!!`Wwl+u&-Ob+OMSuStzF
zwrWR6^kg0JrTf#oPx<e8{8YnXsoqh|&wJlK3pCjsrxVxs*W-8CCCwQ}mt+abc!!>1
zoL-b*u>X9>-0LrP9Jw>4=)`wT|LN;%<`=zNzG)(pwp8q_%X{9H3QX6}S(Uu{&R4dU
zptY|uTZ(60_4!{h_nP3(-Ir}-z8S0y@-n}*tau&s>lCIxmi^LfDW5lq_wWc`V0+G(
z9rHo{T_Sg?15@;Z0vQFny{4>8NeX$Q5p|&v4XiohH^fDQ8k<TR#Vk~}EnrRgkfSlB
zRxP8+Z&FiVD);|40o)skR2&y`mM&t=WHMVQ7N-8g%l?X3$U?@AtS;-fHZ(D5>?n=)
zPHCwUQWsv(@@JX)KQ@hn*TthkG>%=>IG)-X{UoNNQEIA4;Y!8Ae3e2E1%_k^VTtdt
zn=i#?t|-(gkIS`?$y=dQbVH|}#q<4F>yN6^mmlfOYSF#*sC~f_2A_|*MiUeMi>5wS
zlDqvm{(Vw?*N=_`;i60)`mX}Hnw{Cg+Ut{+H|)Mv&1{@);m*|LZ16*<OrW?-NX%C$
z*-+)N;S`Oe-0fZeE_WUOCLNu?T~Y4fXI$a%EqPCe=^lxmy+2Iym!vQ;R8DfQT>r^>
zwnAE|l6gXNK*C|ORZN}#lbMCX(^3PM^=38qt`cHc?Nr6&P-SY;$GlDFom-l;sp`7v
zX*u8ea>Xn(LoIlpyOpt7{?M@OncP3wS+gtAZU4vqX`d}^j#{=I<7~SorJ5FUt=*b$
zX7kPPqFo!Ur(SFARgp1G*S_9lV;RW3<bo`hLe}T<tassAE5asyc&_*1VAivR3b_sX
z(G1zC3XD%KvPCv3qy&`x4P;}jD2=`#|3k6#{^Fc<Y)Yq(gbT2RuPL0eVxir-Z9bop
zxm7k9UwBs0cfDNqid66O$Q6ZCEvE@<SyWy#s?`!Vi8f$!HgRzbY2GaAVkRN{m}9y{
ziSw7~c|l6v6K{w!Y!C_A$eQ`7`G1svaDAn8=jN)=LUrb2Ey0G2UK2$3AI)#OoS!yP
zIQWEXe96o>7m3Ijt(hVHg)9>{I2Sq;iX<<H?KxI>w0V~N_E|d~d8~XnYnh_wjr8{O
zPP2D>w*D|Rq;X2|+!lA!C%PMt=s7(vak*ZyWJ%?<(m9tWd#!Jg3UZKJZ()#X&{Y<|
z_92m-;e`A@qb^kgR+R@_xejw<803X6B@~#KUuZGR+dePn`n(q>=kZ@;yHuW}HMu<O
za`)|&`OHtJdZ|`UYIi@qi8J{^PbX)u@5dCs^3*LZsb14dZdeEle_demQ7`dQhuEoj
z_lXRS8~UDkR25WOZC6YyUR?D$BftD|U<_mRe?`&i>fq`$#s0QK)w`cp?^`ZjUbgsb
zX+L9UFw@D!cBfb^kJe1QSTpsY)bhnvznW?n%xqq*Shqh=I_PN9<Mxol8z;VO*IgFq
zyu3B+Lh|&R&zCZ86OMVHaKBjb%|=%Kq{;VJE_*(4@}6gtS6`HSFthZlpc3DuM(K`5
zu^r1lZC>v2a=EC7qPwL7Yl%a*s`Q_eQ&$$I%nRk;7r=kQdD?bEwkU&U=P7E-o=;nT
zbz0`+InhGX`4uOrc-GHxpL9tzuvE$A1Di(1BcY9wnoULt;Tv1JSI&q?T)pL$=<cWL
zwV$fPO<Iq(wuV>4WKL<dv?}&q72+^~AvweShLQHB%d_rXjs3sMaczb~+@)7-A0^xG
zShbhWXkWXq(9B$F&r1E9m$>$83cp%avNc4{{ds)y=E{nfdh?cdJp4F!{SSpa1&c$4
z%ibw=?cbiw_*wc-5Zk6UgP`XI7bSf+&nSD&Ih9*_<A#c|-0xv~P3&G5Pg8G8I=CZc
zWnzz9cF#od$|OyxY1>&@v|QM-5;(2AEgik}UT@-^wZPcgo3*W%p{*mgxnu6D-YXTc
zU%FDm!+NVD(>{t#e{`ZJX<462){NN~`vO**XCD(Y*d<ot*itH3WAfCZQq1ztiuBYe
zi~pWTcaYNMkepE0GQqV=%3}j7Pi2VLuZ$w^rPrrR748n%c3IrlbmCrS-TyOOJR~YZ
zZ#UV@ZCkqRsO;yiNh_yKdiULCcH$%(6IPv{+2t453JVRsH+BAH>`E;#NXg3Hcg?Q$
zS2n|w(#22h<e8=lhILCAN9;E<;A)bIS-@&KO-YX>CGlm%z6(wIC%6oEuuj!-$>B81
zeJQj~Yr1@u+T#^_=HFU*E_>4Z@KvRp^$Aw0oxf!n6kLKYPA?F0;a6;_y)C|xOWiv{
zv?p_)*M=D{t5<uzXqh_G_5SaDN3OaadZodUzGlIuf?Srile;82CTMXsSlwzZTy=BS
zEHfFlIR|cd&CXFMni#fLce{?lajQ2|<4oIUpWss3Y_#s!M$d>{GAtp1=|Yp1mm~zV
z&H3*lzM1pdW}kGCvmX!XztNLsFnqZ(c9)7Fm$m`NX9XT7Uy;}I?EcJi;F<5pGj+!_
z`!7L>{~Xy;9!xG?#Hyd9TC-hp!t$QC0>z6PHfd;^rymkYcvce4v{~2sSW?U}{pw>L
z-#3{{FEo>0m?yKv>b0ss>B8V`hth-fvL~}J2DlVY7IrbIVrC1~uwLM6D5@N_s5Hbq
zx=}R9TkHcz|MvEvR%h|zjn&4tEGnMO<PEP$J95&NYl-cz^aVx}W+zL|o~SkR$b^tv
z`_*m~UHB9d@^#7DVCmC_OP41uy<Vhy|Fz7D&{e@klm0AaU{sLvGCXs*W{ZL*qoO6_
z7o$d*rmp9g75<BSo^t6-qsR_P$w!JpiyFli*)P2<^)%67=>xV+Y)c~B#e@tPxfYyT
zs5WD50RK0Jm3v<AUZSwzpm+08?>)b2qUPS(!+q+!p`i#vtuv$SD(gbC)uvgS824VS
zUd6<~;5uc7i;K(MORIJpvvGGwZ#y2iJ!_xWj(uBB?Nge&y86++;zp?hUag;__j9Ok
zTV_-&c2at!;N?HuN^=9-#HU+H+pIO%-nRSDWxKnD_in9qscjdUb8z!S_qyA$Cmkzq
zH|gFs?0Bsu{q$0$LCsZzH+9y#^;%BsNm|43v&egybh+qnpI4q66{aO-$tGo=^%3*7
z7Y;Va4d6-*;PSmwttx81$fe`IgZZ%=ta~=TN=-KkG>N`poPDf4_eQJk=9HdeYZZ^Z
zmAV;yciq=jo7tzO&OL6n<Vo+G*j^6b)Kx)&-Yn4zxuyp-NHcZaQoOarXLaA^SAD_1
zPlQM>p6t`#_PDd@VURi3w!bI(Q>?d5Idrmtal)R!C5E~TBH~h4u1_en-hS41JO9a3
z)zK688%^v_)1E1|<A&3Y>q<6eDtZ&5g}oLEKi1WI)L|R*Ye$dts(IRt5*7A;jdRv=
zpFQhx|J0UU!Yg*Y7CtAsP5$@77?}y@Udl{jS$F2-9My%tR9~1~_?XuGxZ}(MsfT}L
z9)8<i>6oY&GR-z)v+#j4J9yXLo8)$UP2iThN-qB!oLo5dSOXh2akXFETkN{gD*wf8
zwJj~83cQzsr|jFy>iR)?|CFDPXJ{5U*{vy^*gA1h!Qt{uv+Yk}R)nO!*mh^D`}&i!
zSl7&w+b+p{CoZd^=qyv5h)vNfH_x4%>oyjLT&;4Ks|l%i5_(|e9^tO5&zVx+Y}(Tv
z;q71Ur8?)(ztZ~X>j_pqNAk??JN3+yC_eJ3dFQRv4Ji&xx}rfof|L67<|Nh}n{Zdp
z_~8ProMUFMH?_Q7&~o?Xk(Ub$q7znLTwuQYWs1(DY@a!iVP>0HZrxoWto~*PyV~&u
z*E?)~??`zh;`Bv8LvQ+S;}Zd*i^2^TwdX8apSy3@u0?0oElL+W`9Hg{I=}E-(X_3`
zS~auI6qwygpDet@a_iy?L2lWzH6Q;Ia8O{3Zdk%=xxKb<`-aEwE<HW9(I@ob^9fI$
zPngnPxATwmo}RD^)mu3A!n&mE56a9JvX~;VqVd<{T^p>g%|1Bgw{F>!H*CKH8~UBf
zA~v2=?A)>N+k>;*4`0~!o%>wJ;k(KyJC-LlRqOp}r`ex$-hbk{yJNA{=kC*|`fqD>
za>gvZJN-eowB*`?i7QWR{&jI{)EDKws}GdA25~&m^4)(ptM$~<*1FSA%<hYKeOt4U
zY3;qOZI@=X&1!2~X1Ml&sLm##xYwaqQr^#!uu=80^~{-2eCo%QB?^avd2ePdb)FRa
z;eSQudeMEt(e8&2sU%5HlUnL+*I{@4|HbOP+grSMtUHw>d@%F7srSp|Gg4FgzAHGr
z3d_I2@3nc>{TnmmelAYctI6-JtnXc}e{=IavzfAH4%`XzwwFA+D^sAnk4b=$f8Eiu
z5~2GiA6@)bWbNIH?^dL3I-Az;I4vh}>(Q@!|AlGAzTO(cedpr+n4{qm44!xTy+b^*
zWM1>#J(s)vBJcZ2+qN%vls;}W@h-dWt-Ug`F+1*_eZTP7LygQ&ecYc7{I5Rwxbw7|
z{Z$<XhQiZR8Jd_`Zxp32pE}zp@5GdcPHEijn$jvhJ61R}9b=KU<(e>|;ep2l6<;p~
z%}EPRO<;+hR&p}v{}L}olia>NmYbjX&9$xk_hn}B@}~JJnNoLm8m;tP>ND4_)=T77
z$nqf8h@)0PS(8{8SUebayK<e>_1xkjsa<$R%xJdCwpwk6HzKO>FYD$mNa>ogr&`_P
z<LYp|2M2?n$f;%ASeeAwCR<!7Bl#(XamvXMv4WYWPn>13PFAb1{NizOrsw=+Rz@eo
z&DKP1%yRZ(&^FtYd0Wh3)!DZ<w^rZ&b~f5Vx@mj)Z!rfOv*0H?o8#Bj?kvfiv?y`1
zitDneYpd4o;r?*1E2K$9Xv&@thmg)IU$sPcwD(V5aCOO-rJMZUPwAG8E))7D{(x`A
zpU>xeTcb1|D;inM$mr%=_T%*8&;L!L`VUQfm6Wx2mx)M%ZIgisr;FhFS04^>Zw&Hp
zxVmmv{?YGNAqiV|#Y{1DlgK{u(M{I%$a`mL+m?&&Yuo1ZdnkB@<l4=OvGLlsY?_zC
z>RC&Aic~cx?#VKp7^dRSv|))urp|P?OG%S;&)iAj+3u#;>>IRA;>_vwdwM45H@!KM
zy@d0NNWyvczbza5Ikcs=Y-E?deZa6wD|$+|+st1&LFd%gzA-+#<3W?KkN(_q#%Fz`
zdsSFM?lg&f^qZ<1IyptT<LZ^LQm^gd>AHHBX8BXso{cGHUVlAm@8;CqVH~Sm8-lE~
z)-PVr#dPZA(g};Msw@&xI4s)uZo@Uv9KoQDg=YWd9yPUes%^+%Vm<TKL%G!6sdHu7
zLRU?d%Bj64HF=^hz4Tl-?dkQEsq<f5SI_Xx`sBH6md=l>Ey8RTQSIrh3XGaZ7D*Ux
z**B?MxA$REx9>K-+mqMr`*EpT|A5c8?hU+Zc0A{Jqug|!9dj#Mv+?AzhBIk%%J21b
z>sB4($yl(|xYz2uT%Bo?@1EOlx85kboolutG1)bXaly8?nRgSEmu_1T`PJ&`tv!D0
zvNtDlTjy>*c|y-RV0qS3-b=ggt(zNlOVZ3HNJZoQ9^PgZp`%K^M~wC#^4xRDj&)mx
z^1C1*mBSZxh1Lc;ab7m=cM({AYtIoL&m{hmCBfc%w4#>!9CB;=A0tv!klA@;Qpx3R
z6|L2`g57Jr-p*fcd-~SIBY$nB3+ESYy|Pv2-Z72d@Vk3hT%=xmH*Ja78>@T8=ChCK
z?UdPXb@#CwS6LoD9^td>&l}^~FaKqx{ACyR6}hOX5w#`FqMuEw-skiE`u_{GMWdEl
zJ>7q{MyuYMv58^P1*wp^t3s6vLs=9p4zh;{ge~6uz+=@hHnY@Yx~{Vo3fok-C-%BT
zY}H7-ck1xOUYpgCzlyH;CS(|_+H%_e)hq{h6DHwl9!pz&D>*&mj9g}3@z&&7eCkk!
zs<Xz?3!TO_i6WUE&W(oLUCh#Rg$#GNE>>ww3R*BvV6wt8-8HGY1%=*H4})g>-!<Wl
zzH+m2;<uz3J4K!ul+2NId^df@%(p)2^-pe|F9@6I7<4S1_0P;qomMHmb$u#P1->ea
zkD0FYoG|Gyqn}#+X3u*wa&vqh`mXMqthLQm-8j#9mS;?}?AwCl)<-|}X@9%6{Y)r_
z+xN`aHwo&ye%@)fTPHl1sZV8TqLgNdU66mBMak6tOEsqJOQk*UVo}L@q&`z`SDOCZ
z6pMqYo}SiC&-xEVE>7RJZq}O1fp0FnJiA)w<?LOXgRHaD=a<=SwDXzruzSIaa|{)n
zI&q8Uu_d)ldb4n)@9ookv(K>D>6@BVN$kGt@nMNw*FVpIhFii<mu+Zu5?p22J*y*8
zg<Wcr$NyIuQ<)Z|SBVKSS}hG;SQX7=t<-vbL09;yLz=AmH%{oDTAQ`&)ENzr#?_q9
zdNmJmi8+2Mo0r16F`oJMblZ<FbyghVR#WYqE*87veA?61t3M?^5?tyPT0e1XB71hy
zftVMeTis08*tBNH%sn*qV9iC9<Rdr4wtTQP`F2(J{_f=DDGbLa{Z&nEe|z~Pk9NcK
zhlTxWD<e~ij~SnmGMd;Pl9JZR7QC}Bh5O#ZS(jJ2ZC#moHY@S)tUL3LZQYpbn;Vh6
z_D<2_s0}ZVP0oFN@AiTs*%v9tt;-HeR~$NQ{`tzK;sY(R=3ARi$NkeQ*v&Gx`1PSX
z`(Fy}<o~T|@VR)Y+q83~|8H)6_wd<fzxVsrm2VTyud4rP{Y@^fd}na%qYm*sT1Quu
zeT~hlz8QW0t<t;7CzG2TRxZC@?#bwTD_S^4WR`7PTYk;c<`+M;tP*>+^#^=V7Ww-@
z<C3SKKrq9b`R&XS2mE<I)N9LnvK+JMVtW?SBvxa%K4Hz}<wqo}H?9(8Pd(hm|7u6h
zCdGxEo<)a5n%ueGPrSb6#Hvu8S)!S)J|}0SeO<S<C~$TDT&07@PF>Y;zB=#XEUoFQ
zd!A~U?>rrLZ<Adsd&35wGc~KFy(PX^*=}1A*n6M(&EF{u)Al@A+FPr3GtNw5#?fmR
zFSIO-T%^o=Vz%U_3H@q?O8&pLg;?dduF3rGe)&<Gv~QSh_FO){6^?UHuD5g#J7QRx
zJwf@^iw9xb{WKGAo;c7Ibwk<a&{w{~ziD!*MU!{k@y#omX>C!ZlsWT(>$jr));V#v
zg7%&Y%Qs3tzNjRr)$?i@zu7^}Nnc}j-@c`z_Mj)%w@_Md*Q&%k^S8ejs$AV{>l1i4
zXp*jV*7EDB;xnC?rYKBX&k~|HJ7B?2uBr>Bx%mrkc291W-&N50u2X@}|Ndd-)WC0<
zQx><s%2;gDcWH84M6mp`iVF^YnQqg5)U!A``Fe#02R9x+YTjDOX5H-=R^q$DW$wnn
zZmBg1qTJIGva;NFNIlz9_jRSOi<ke0(2|XATVF=m?F#=tzt3QU*Ts_|?%GlPde_!C
zyS@rf-!)CT^T>*dyIuV6FYi=akjPiSsFcrtUtFF$(Khb=dYQ*JBof$`@9}t;-+Yz(
z0l##{l!ZMTjhipu5xwg^*L2q=i{Cd>^{XF+mxpCOTzD&I$+XH#Mf<iKjeJ{lYTB*M
zftR*rY}UOQcY9UX^KF;ZI;B_C+~#`lOxGq=bKU*E?6{<CR_8B2Yd1VR>-}NY_8Xtq
znSYuX5|@x%erwN@+Lv>Fzj?X6{Kx&)vQ1yP<KHbV|9$oQSDV_KF_V1`{CWP|;g{^u
zR-q{i|Gw8h9Bi>b&^(*5r`onB)O16*$;y)tZdmWoFI>UMV!&Z6ZyhPmwfX;suGcGE
zE*i|x)SSq@k#orbVd<Uo9<oeUR$!Ev$`Y_KGTTs6*irhjX86vHlFp8TOQy~eZs23m
z;?v?)_$)d}g7w3Too`=E4Py`%3)-Y%z4^+IUGhFkOPn|BCT-H!UcF9vm-cNVC6&#a
zi=0&UC~2x3TIC?I$YA={$lX5+t+#78>;G|T6KH7buxB)AG&DJ^r@_dm!D#x0=ko;X
z$rjz4CAhn8&akTPwp=}<<CVbv)7|zgd+b&ASS(ULv_jeTlg_aTYg|7scG#rj@=DTE
zdFAv~HY+rGMJF-@-!xgaQfkr;mAHp{9x%8DK2{0(vL&cw?P|lmfULFe995?a88vnE
zMy~$f`Yxk&H)ns?0b7HMds8NDE1GTguygJM=3}qD_a&I9DL3v*71{P$$~>NBPL`2c
z-eIoqJNUm}==!ljJ*#?qUWjp4Prvd`?E>ZPS(AjGUQ$b+GVSB)?S_{7w?^;Zrp@@R
zcz;#S{^b#eYh*ZUW1O2_9G|Gc%=uxGzLbG)!(^t@5)3L9Z+_16t&rT<A>OL37dJ<E
z^2y2FYgTk#KJi_0N_TOOGpD9oqwr)S%~QgLQ?)GrT39kVG;u7@d~YBcn%Fg`N?~S*
z<0^xLGjA`L&AdviWa=uzRYH?^m+U$!bEm0tqPVKbE(tFYy$a(MzqKTiv{n90*O0MV
zeZ))cjg^9r)5g{Roi@xqv{Kr5L-e7Q4Lj@}Hf&Ai{`T3)K*hPOfRWLF*?7unp$zBU
zdrtFNIPW%T)H7%lGHB$BIBcxqv&+W0{Q-l1k#U`JXLsl3PEBQ-HJi7tkg!-duic2>
z{?29x5u1bBHiwP&w6{3eb?F?-b~$dXa>9Da$;$_3DvMrcSQ~M9&pB<~TYLCcFz&sx
zW=UYz+RN6n@7K)Ya<UD-vu5RkwZXf3!d9@y9N8MCz#MgH@5@=T#nNVr4ou-q=++ca
zi~DVx_W9h(gnh4C^wLAjA6AP#l|G-|Y<_L!KE=%QZ;NM~b=>zw!2Z`3|6f;Bdn%4^
z%ihhm!2QSW<3=f5bt#9LR4*`_{=dNT_qb`#em2>|x)WwtTv_b9L2JqlZI<8rnNQoV
zndA|7#<qQt^%if3?~xANhd1;uQf6Cd^LOC^0cOMhGq<>E`7L<q!1-g%jGEpV%~R*}
zY?RY=l=^*9BxS1K6H$pRqD>B4SI<1?Cv!@A3v0ZkSL#ni^@W>OZs_DJ5SaKwPL5|$
zyP|db&!Ba8POUq$NUi12CXvlc9`B88+@$HrxoE?x_Q?&r0p3C{4sF+NHe+x$6lvHI
z<81HiywBt?>)+E(qCN+%cpLj%*{joRtkQAl&FRWH$G>=cTb<?RWN2uN@tItq!TM<R
zi6gs@rp(kW@Y(ap`m71RL&()bN+t&_wrB?M^m+Z?a%9Gus8kpCZWoU;YdjYpxndxt
z$#8H-<JqSlZ0@``;%Rg=@Qq2*Ud=#h)w9xD?`P}{PwBeUu{A8%Ph^+$!>sn_I(pnc
zU89YTr%vuanYe8AgRSv4$3ChoJEG`TIA?CMvHy$LrWrEZQ~v1X__+VN5MIW${G;@;
zriJPizV2r|xBrvs?0zBqC3^q&VCQO)3lj^jFnvAFvi3&z(i=Uo0SzoCj!Pb9JvilL
zqd;8G#OIz33LFg`j7$s(H$TmgFkX4smP^>^)qazWkv5Yx_*4(H-P-z@WB&dF9vzE!
z2&FpoFnjh&pFCV3()YTDlVw3ax55k|-f2xM&jlFjDzru&xHx0Z{{_=O-x54xc%5bG
zlLw*;cJ<7ea+Y1mYrb$pl*Oq9mzS>3UOjgL)AB1_OSV|)U*5GfNo?ybuf3|Lmp<90
zamnk@+Fd(+C00hre@No&a`M)j<KB^Bzx|Jsct(faRPQsoR~%J+4t()G%yM@xPpolH
z$feqlT~lJ*ZywU|>DS<7VC86F>4;_UxVdga?D_<*z)%|_K?dCg#(TUc+b}C{VBidv
zRgU|zoByk{(;uHh5jq!r_Waa5>zcFYc(w_*j>4qo)zjB#o|~<EUV5$PEt{0LHjyDm
zFWi>8qC7k8?yQ?u(r+^MdbQZ@6zqLd&|9^k^}#9E{JW-Q$=l!a=+BCrX?Su@EYtr7
zk7r+JR<o<p)<4fF%b;gnnzHRnPQtsz+kU4AK0S1(_3rtKKmHx2jYe~Bbh!pFsV4S@
zUSKJ`AlldQccILrr(HLFZ%#?QDYY$eny&&QL&Mao59N-<PH2eSdujE(soh+Ew>L34
z%rTwcaVL`fb)>*kVZpN=LQ9igvaXytaSC(P%730KdG9>p`@P{!YS$F+Cc_RDb|L-^
zqL&mxADgK;&YaB^rpP+`w&6mhXknG0txqjwM7);h?9-EVn!pe?(e{)I^OKFjw^f;f
z=WzwQnJt>oaCz|##m|f9TjzQkt+`_~!`ngE$Np=K^WB(zC5QLbgs`h}Fp4HN1vr~d
znX|=a|Fzctu{Yj!-CBCLV`b_@4hEyruE3qzoUTW@J69O)2-uS?`DiNB<G+(0oA%T<
zowL+hGkK=+F((t3?4F3wBS)=$uU>gJPg7-*)=Ae#I&M~+V%+^J1j8bE?kCH3t_Zy3
zt2O)H3eg80=^;y=2On9IseA5~!NTO+**SZUmOTi||9XM5^O&4sc=B$KNlPbi9oyFy
zcHVSpLNV9*gN56wmS!X>ov--1RVdM&_ujfcKNgn1NvLeO@on~v$z}oHZ@YJgzGT&P
z77j}^$q6XPw4Ps?827mAxntz5*vNV766MaNHvM>bMdDSg+KlN-B^sKAx&Cq)D0a+p
z-EgG1OKsV!`ENtEOn$Ub^?zVQtw6s^pqrp38{^K6ZidVLCP}dJu3GDIv7<rb!jZMD
zJJ;9lH1#`p@uX+yuEz^*ZB%OATehv&_S??Ho6@pb3*((GZ@R>}x?uHY={qaeDD8ZE
zdqee`^&xMTsu<78*mu$M!11ZatyvE<Y`wEn<nB4u+;gV6hu$=~|Bac#_t4m+*{CMh
zGxfBW?eyZp*vNTzZ|{vY_D+d==HMq9s&p>$)kEpMM#{YtUp3#^tnpy>#*BBLIOo^j
z^0}}i?`YAznVd5ZSFd@wHttG^%bbNwdK~Zn|D3W|^W62>T$&e-yc4}&a&OPauW{em
z?%!2edns6VUE|u-p7DRfq#v;?TrMH@!h`R6#{c#|fBW7mFsq-lJ@hc+Ez|ku&NobL
zAM`D|UR!IIH01;P{k<G~`;wexKAuhxbL5;DXU+BEd^_uf4%@_9nF}I%iQ@VfMCA^P
zrA?Uq&TKMogXF%4vilNc&pnit>y$C;T=<P^nrN2t|A$iV9xBUo-ZA~88ZWDOEV6;w
zf^Y7HCZ`?CMHca`eZ^kASy1(s!SfUAr@hkQdo1SqS#O`ilq>Qo6Bu+~+%t(&ouM2g
zzckxC?eXqopF=n9G*jf`R9a~BOmtDrrS}qVRE(_^E=4Obd8-RM-D`95aG&*P@uilf
zQ_EilwMXyPJu>A)>25ua!)(3YO&#7IUt{*!Tsh!U>Rfw;FW~=MLz#zXdPBnIUHNIa
z|JJmI>$P_~CMHHNOVudvS~O4e{<pVR6uwQgC~NxgcJYO~?p6+aeqDKgarY77$;o|k
zDQwDHEF?E9;Lfw3sjD$FBTn+L#P`F=Jdsnov#0UA{+pNAx8)7%^YDdzh8<^%!(2<I
z9o;Rq_om9VYa660%F<I?uT_~H4J$GA+H%cLM(lpXx%y|<%sgcISvk{pEp6JjV&e*d
zIOk<eV&}Wf)Jj#=sxz;r)wuQBuj{&3k*PDOm*Hc1<?;R~`~N<LljB|n$@z32{VAIN
za@w;S!P8!P7f$*1arUI%hcf9m8D3;A`TtTW|I@PYvXt^)_r+c%EU#MT@oB~W|5dBg
zxfC>n70rZSt^CAzMncD2c%~(%<GW9Mr8_!>qQ2hr;C}q7-{<0nWxx4<&s_HOk?t`O
zU(f7<xX+7Ub}YSl%Vgcj9cF*5_Z!Y$vvEUiGViNdcMe#ta@eY7yKtLm&XOyN_fB`R
zPvmG%Ubtql(HiSJ2Xtc=HSg1E`m#+uVa+bBPVdv(x@G+5$N2iCUYu5XF7&YfHShgC
zA!aJ6yKL@WuFbu!clU~J<CobCTsumqT`#?JEjRidSAu`aJgu@Q&$7u9qAUStY+YO?
zDcqa3@XX;IXEXvPK9gJgcq&8g|4A?IeS0Rm=jf?-wl|ea%+K0=%Dd9DCq^}Y(k-O~
z){d~X5xhD7RTuDTGja*bg;Y#XbYkR;RP6EixbSF?@NJurjtRUgyQJltrkvQg$SEi#
zWRr^Wfu)CM=(9?+XfmX%F57nN;u6gbQ>U<asqpB&3g|pDgG<J%({suSCsyytesi@P
zvaT!%pPIIGRjAsfwS24VwywIW6}BdPso#xLVGmn&h%7635VYN8x=`wuPrAv65;+er
zG5zPwiTIG%+AhwYCliq<$vE+Jf#Tt(M(w)N%X}s{g*Z-PVRY%{V>n=vrLa;|je&{5
zgNczL<uqFZ1H-$UTMI9*^Q|s>cYjyqag}?wcjVoV7gf`LXJA>v!Y7<_bJNOQrHr$;
zcu#n=cXz|9>&NpW*KM%=Uv>V~<<0r8?9y)kV0u)%&i<g?zu(_keQwIkJ1p|>qdJ2)
zpJ;_-6Q@?if+jYtl9|mMR#T3ME!mQ!=x8!g=|G!U*@=ZNe7Q3wb*N6-F`-jo-U_8|
z<!vjjc5rSqJk@5r>quMIVvdqWZEJ5lTHJ3}C^3bF(_=vcOZ-tahl!Unx2nXfnai#^
z<=IIS#;JZrnX1#7bY?E8OgN>I7F2ll_(7KPLdQtS++8o-#M3w)I3=q+QEiy3Iwir0
zz3S7;baAemni=dZeBZ7wZ2Tp<!a4QHQCD^uhK^?}9!io8Of#}(F)+-jx}{mL#`OA9
zpG%r4s~A=+I;Fa5@uX9$oL3#QdcAVXrT^PnMWQnrS2r;AJe(kQuXX!Ti5m~2;wrg3
zC$?0d;o7jK{6f`omn~Pk3|obQpH#iwdeiG-WWx@&9S0k@d<7NiU*riY?2g&;d)@Aj
zd4l?l!7>^8jX&*POz{0{wPJ#A6i3U2KF+N|f$qGOPbRpj<$l@VDXpfm(Rs~=g905A
z(K>~m(%vjW9@?EY!9iwkZ(Q(V+7j%oz4e1ifOW3N!iIELS<7h*UOuf?f)XFTF%I&5
zx#q%!SW(?+=aWpAhD^#3JR5q>&%O8am2B6oQ$o_tiigiCb3DZwS37gV^mxO!t<P^J
z3#WZcPPe}{BdL#*Epjn`)U=yvb8DwLFfHsAW5{w`^4}~vn~zcP>8h3A(z1Ov8Kq}G
z>Q`sITRe64^H-0~dFly2o1s48^}6e-tM6{U%Q<`Xi)&Mtw@!V#bKktGxwf~jz2SRl
zE}y>R^>q2VKR>s#HvDJJ5N~9C75!)5N%{OgpZ-6a%_7Mnn6j`<q3*?_uh+8Y+kV$P
z_T}Ms&0P|89ST=H9{YwJ5)sn2{}<8CprgH|b)v^L{b@dtc{4*M`i6O?1+7}5s-JnX
zo<-&Aq>SxzR5S~icwB;GcQI@;-MP{?PvbD_%P-=*?>PNsO&r--TbB#|V+~XmS;%pG
znSk+>vpoG;jO<np(@l=LGEO}evdn{dm4BgD@%mp2rz)`@ImWxu+ezpDqoO^<YhE2;
z34hmkZ3D}O)p3TJoJ<}~EHheNb-lEs;zIh%w=C5ysO^q^FzbB7iPkkmGqWAuJnG;5
zAY;AdvF^Sfj}Gm5)~Y}KM|a5INe9&GlmvP!S(15Xdvs_p8M<1nnJ6+*!d*#C$kXnV
z@*8VUXB|zWsS|Q6nRj0F)mbvR?2X3+$Bi=sCrvZ{<M>u1Dlj#2#j>sIY~E@)L~~y|
z@o3sworLU|*SBI%EEC`AdQ7p^F*^Q-lKC->$um!{xt*fYtibSeZQ6pDv-x~4OI_x8
zP;gJ@+(Qxdxi@EM&0)NA_Q|oE(${NJEusoVt>wj6-MJuaapdKlf=@elJ`Tv6|9%Q*
z>;Ds*Wm|LD%RCSHy_$E`N=<Uh!nc7hE-g}#|Mh0wq8~TUq^&r)?!n%Pj&f(OyDPTr
znds!<^m`Z27nY`l+^rflm$fhcIHDf>;Ur77QS3xt|07&MH(sc{IDE0qqG*%g*Aul4
zR+4|`C28(>vEI`sYt=t3(WNW(+#5a4$k&`>cGSAk>YJLdkad3)oA{Kq2Gy^aIR1Y*
zXlvW%-_~^IMo5H@=%P@?%F2aHGCUZi<G9-EA3x+PuLxC<3)Jolj#(?zeIqDR)6&Uh
z>c4eWH^L(I)OM~qmnv(tD<;xVZTG3ribV_7oO9aPedg1-%%ussSG6WaC*8Rwv0hZ}
z<}~$~m$U9m7fb(lx4sq_w{P#`Ju-(@Jk#v2JT|$ZmCsG!so18SvWa<z1ll%CD_g^>
z{3^m*xpR?w+lgguTlCt^PP*4+K22e`ec@EpG2vMgj5OvsUaCI0Y4hBjle9O)s9t>0
zeJpa-QQe@){raz!R`NCY&ZtwI%_%%*8>_XkT7L4gn@YknGQ4&emA&Lo?^?vL@8?>R
zxm-;7eCD$&CheI2n#a&m@%RlE1xG6;mC(e$JLeo$bY$>&Azd89JXiMgjE%N!eiptu
zPfpEqx49uXxq6dE>y_B{F26#KDv4jaG^cMY6S_M0T}ArE(zon~4=D>Y7&r$pOP=3x
z&bYX6{#Rjs-jwRRcXD&?>!i>AU)Jwp*t^ojDacOXVMZcbr|0~e=^;O^Hi>@lUbQsO
zOw`cuS7_MQyZ^SaJYcO;^(~p1FgxPJA?~ySKb4r(b1%I*C^SjHzf{$9CF|8go2Di-
z8O#ac)ecqJ|4Lwk?U6eP#cQ)adn~`+<|`_&M1WP-BZ+60!eNbwg`9qOm$f^d`I;EN
zXU>tYv#fTn;(Hyjoz-a1apOg<*ES|49o5zIJ{i$-NO=Ea*X+{gTQ}C4OXXVYX{|fm
z`|aFXnd;Ng)lWY<9GBMroOJ2<^c=?tO)i3mw$6@qae6Xg@}0cHR*Oz}7EPbnbA-b`
z`qCsnwWYl5#T<S$E~lpdO%Zkq)S9v6*{OND?f>^j?4ETwPHujDSuYE#Uz(xoi!*bT
zpUt$Hzagc`Px4GcM7|ZbZ$?u<_JLjthWj!bAMRRt<MxW+jhhTZ?T%+k-wK&)IkB1d
z#&njllcw#*&q;<Bn256mZD^9sUEX^8X5rE67i{N$-?IGaqm?o-*?gA0ybn@C%>ynq
z8ZP2g6`d;lU(O@Ga#7SXSvPa7O2(<j&-}A{`B5(9l2P{+hoFe!*?V6<n_k!PMe=v?
z%fw$wbF%usu2X0AueTCzz1*VMmLor*uI0)vS<h7&Hs{w#$Eta+?(`MelQ8>)l!W+`
zr7SWc4(u*-uQ^qISlhfP`kbcB_ORmmHkk*nSxlxLzF5-rXWRdGAJ?%<Kh|><3p3~o
z6}gv}v3<D<_kyf%>#qMld@@V*!_}Xm6+4c+)bZXlg*_vaF*7nyL_>0yWp}3Q>C8lL
ziAwHmd!%zXId)yRmfPCC>p{27)7$&LPLq@MC}?G$_4oG7v*-N<&J=r2KbR5j;xN5L
zvxUd0g|~|3sE&(DWX!QI3ddVoj(xn?!E@&%kBjGvma2;inz>A8WjL!Wd@s5rhCaCz
zq~lOOlk+-<#EmDdjVmYk=p4IgaU;AXQDRds!wJuX8IGDh$E4PsYz@79FXnOv$GP@Q
z;f~%Jc{|$79XoGk_VL^DF};W|f9Q7os+XM9qoj}Xr+9egct}tGaQDdtE`k3p47WEj
z-`P6%V1|>zGq(wXGu8jyu+Kd=q14ksaf*-1%ygH)$s75mt8_lv+xMoXdy|QP)ZISU
z$dG#*PS<6)?P$5#`m)<&?gE2l5BB9aE);EP-{9f3A;jp|Ok<<Pb8g<8Q+YnTz@_(1
zWWd*!GS*p68+rmok9dEO`|<bQ42Q_2jse#L+*Y@SeyY8?eCmbBpPbRE5no*A8%~|Y
z$vT%I<^tDDsnimNa9;*i9fzfNE-af7pHO>Bewn<pnTmktY~_DZK_|Qv)S`7GlAP8Y
zI<jR(!qH>Nq8Af=m-PP)cn~{VM3mbl*D>An;-!>}qB~V~77Du9a&H!K-CDsexmQeb
zuHer9XKwFV8@TsjGzZ_(eLrUh7R9JD-tn2hD9LllUqMju<PpAI2}-8ohdW&k``mGz
z!E&rqT%C80dyPzAL(B2EEvGLm@cMc-eUHplkC2#*#2XSb;yl_o8x@mt=AS>=D<Ri5
zn`3V4L*Dd=V4)KW+gTef&27svk(K+ExaN!S)u~;1HqRdA%=Pe75%@C8vas(F%W<1r
z*}@-DeMeogE1o26o&0pk+%&hOH5pFYM`CYZRM2#3kh^yI?$yaf9ZJPFf}hV>)@isb
zm2=7g!;lRr6U9_QTv|^~+mqycR%J<G$h5nU9%o)z;&V?bGVZg?(r2RKo-a>)QCRG<
z;mVN%&&s`cDt;~a_J7ZmxsH#!CY*nDbxP-(5a|W|re-c?o6?;muJ(U%v9MD7;d_8>
z$w{#rb3Sf-yhy<9^HWdupPuvIcrG;1SU5%C>&~e+uk<E2`0`x5@Le>8Dd)zIN-n2+
zx7J!N>F`qNJ*g(Bt0cfF=)!lQYmr8x=fs6aSG?WoBw`@FqUTYPXS7Gt5*N7%(_I)P
zV{S`qcqEa-wB1pJVeQ2holIu4?#YX;`Ien#=@MkUG;Qi4Zt1z)d6q|HY7?fnJ>a;;
z&B>QMOVvO4-;tugr^ijhdA}Xu-L}%v!KFXK#q*COUs&qVl^hkUi9wUZqaJk#icPJO
zJaS+{6AQ!nv!#h9s;%>Q+!mY72>UP5c6$eJ>#N1mX>1K8$5>=!6OCe-E<9YmXogeD
zG6vB{>Rb*CDz0fk=hUi_T;_T8=Bf#~G^#}$m*_bpmeC}#<wk1Pjp7x;8;WB4yVAOU
zzF6&-mijQG*x~$%tut>=Y!Z1Pk#5&+{cu^<GPO-ntNW|^oLw3mQrR7+_KC1`JLfb>
z?d{UG>vmq7U4BGK_mJzm8%pozoUVPc==~R^eNHF#2>3sns%fxDvSosk?uMCP92Oh*
z-7}F}b4cyY?826whK>N!7R9CfIaSH0W7asH;E_z25OHwvak-57EjgSbUMKjxzAU_Y
za?yoVqQ@3bVPxMTWhJ&GL?&geYRZZpjdRa#rT$lHT-&!=g@v0zjqz%u*SV;Z8|P&x
zt&#|ya7eA^P<kfguGloEuHZ|n;?H@tZ~P;_aZgLN#YL}kVVinxi|$<_rg_sx&2viK
zM9H<fd*%j4-v|>v+7i%|5co+<^nu{LrYZM!xio%w>m#xBplxc9NzBoVqer|{51R%Z
zQeAf>RW;0y<-~`V2X?j`zv`kf_35cKJ^nE#SPB>o&!)zyxy2phefes!3FnMj4z)}T
z*`EuKWoo1;JI$Tmb2qhWQMX_~*GIRT6JI^L^UgN(ZQ4Gs;u*n060^lJ4r#1$zP+}|
zDdUiYi=%>DzWTFgHo8kD9cX(rLG6+lyGoGuEmd~2yAi7%9LxECEA7_IxpxcCy??gs
z*QYcMwYSeX;>vjBw3<%7V3?@AY4ty+D=t@)_Sngl+&f-!>-LL;<@chVKNaO~w&^$^
zw#KVhouz&G%ZbaK8<ro;<M)2K+{Ylfa@W0A2i1?XZKw_7(R}cH{y~3j4+hQ!qRoEu
zzq+ni#1U&ht6}v`*SM28*E;UU-(0g<bnWk-8-6J^ZuE5A6scq#vxLDy_Y>FJT#jp3
z49`vGnULZ2@oLape(P&1>yGx#n|yU+hLGUtZMnG{Cmbv-S-W|Iii7Uy49;lrwbP{C
zGg-^Jcb~nsBST?YmL_}N%iR}r^Iu+UW)a`}L|2k!`u$lC?rX6~v*gXWbcAQq%31&0
zPRufz&SP|@<4wYRO`j6QAl2O`j&xl2=1Huqjrk&@czOrV!bw7!J_p1e+-VM7_3i4>
z)F+REH5l(FT$Z_|XwIo*z17XKRG-Olqki}~_g{(M<&wm>Kd%rnT$_}1r}VLg&P=fv
zH`I1<S#F%HB$fRw=%euB_)TS>x4shoownNIagKqpyJjCdgVah9)tREEH~%hL{cn}D
z&MtT6zxuAWa(iDX?f!9G|CZ}hUcs5qyg9{l4j!AyGCRHM(v?n$9Jb;Utd}JZGd(<P
z_vVm-(czE{3!bI_xzMs#MuPkAQTOW~R$mn0dtGprDeUPKV}at67uS_ccR8qWWd7}q
zOm{9aY5TZ{{FjjS+Wlq2w=dtOJ@fb^74UhB;8`zauY(Hx8?7dP+$`g*b?ohpJi(ij
za<@B*Y+tc<_r_ewNCp3(IJxZ;1Kx$*6??kpj&38vC&uMl12-{hY8IVS3i1({bTsnF
zA<Ii8xh}^lH|_5Ue|k>N|EGuT2A+lgj<Dr8N*X*hxyoK8b9eb9wg)z4dY6xxC$7KJ
zdn0+;<=i=5DNI`pvt2Fj>UAaE?J4C<eVkY9(D^tZElntLtJ?8N2ThEVl{Um2&-tch
zADEvTFilMO!dtdQAN1S}Ugzb$lTB~>eD|&Jt-H_6gID*w&AIP#fN4?RTC<r>f4nxZ
zbZ?MdwMiuKpVo8d(&+zI=?wqv<J>N`=KlR0s8rChqO^&p#IfYL_tEXFlhte8Kh!uE
zd|mR~sEp&)*0}1V^EHPS__(BcMXYhV{6NH`f#t-lwo5rbE4!vIi0vpc4t$>D%|G#9
z<;RsOjG70&2x{y5stftBy9yi6zQnfGEIBme`wKDu+pAxgeRx-vdaNQf&{N3eK=0(+
z+g^BW>@k#3FAaKQk+sm!m&<`!YNyocC&HSiFKwAN`LyAb;~ynExu+LpvUh!ztJj=n
z&#{ZesNk!vg0PXo{FMjqnkozH%{ladd*8}gL8>vqscQFEKKYfY&U?mpi__d^Et;FB
zJFHl;yyaNVoka7QT|dP9EIRLA*IaGxY}oaG->WZuoBRI-tW+{y{VQMtUm#PC(}pA4
z?s&GSiKK54TBNXS@`k+60iVsxCVLr%hjst|@;jE#g5N4fA+}0p{kjwSR@O(>Gc1^T
zCo9;UyCNoZ*Ru9qqQ7@V&D7~KXL!s${q5%uEdKVz4WEv<9I!mU-{FMD3pvOA-&-r5
zJEr?rTnhQ-@%WHgvP*GCsc*?Im6pR*L2m4v3n%Vy3ftk`thP72Lcyt6rRR`V%p>3b
zZ%PYP_+5YP75~%yODdz9lj*^ZEf%Kz&4m-SN=tup9sO)|K&Ny*6VHvJFqg`;6K6~N
zxP5)(IWaAl+hy;yDS2PFSDc=5`r1yV(|2Bp?a8prpZj&j|5aO~`W>UC?>s0wS)O^c
zB(~(}9G6F*9`HKJJbCu@SZ}!e+%v~aQgyF5`>p6v$aTC?>h(6PY<*~cmy~Ju_V@C)
z%r@MV$W?G;Iigi?T;krzBU$2KSElSOiuSyHo=s4C{b#-_Vh>hb_@<?(^gW+L+wy0T
z!lY+Q>>s7YXW#jKEKjc8!u@@(Vl2yw<7?ad9!%^@tNi+cVd><rRY!wsCQMqL9>5_i
ztG+9*bK~s3f4j0JTfZ0U{f;S#{J?O&@^lMh`0o8m9bz11MN`vj7eB8({{3a3Pi^>l
z7UzOjp2{x^3+A0YVH##Kog?Sn&CJsm&R>zT=_=pYeaY+Nq(d(pDt^UQ{QLjYL|{Sm
z5o`TVS-TFH7cSmkAu^l&hqKu4*;c8gb9>VTndMGzRV|h9w%PQ?${|N`vulF`BQvXt
z$_j%-=XQR@NuE1A7d7}sFG;vF$)(9zlXIyGqh^qU2cxHt*pyEb7dlN-^<Fn;Q_v|9
zRb%x#Ybt{ew9V35ZZbzG<Ul~HOLq~cm22jurM@$Ta<v*xA6mq@JWRwPjaz(GRK(S+
ztII;Q8FM3SWoDP<9NNZoZ(i!{W4gCm%AOuNy4yVW-k$9D-=1nWh)M?KczBx33DA!1
zOnJG<vtmc=>Z)y8hu7Z8Qf6S_Y*1iOVCYD{wAg>K--Hb%&MX3n$Nf5@4!pR&U3#70
zivPDwC$u>z<s=<mb#=up4zJ5@e{U(~-{hZZza=)>;mggdE93j~{_b>E;Iv4)qN2)`
zasI^l-EDj4T77@cIblzn?*84*-!G*0hxti!)_i!NIzj1xuAWvwk1wCk2gTJgNgEch
z>t+=wu`4dxptx2@DtR^U)*l`YlccIv+RbnfTA<ux6{YDQn#py+Ra5Ol0n>!EHHBQ#
z>N^6D%T!(|<d*HerLaQRf}@L@kK+^9YQfacUR|4_6#Y(xc_^OI(O#|6C!)o=LTH`V
z_A{GK8}3ypKC|x(i?EsL(G+9fU6&^5=<a%#)#0~wRnAc{+pjIzi&*TquUM*a>R!3<
zb;6CLDH_>{t^Y+>7<#{KcGg^bCFuQ(&$(t78P?`s@U~^0y0^W?S%GPa(Wwk>BaNu3
zi^4rCr-pWztoeGV+3kvfb4>14(eMDrOD_(yMNVo^N%r>=Th`B!_3dC<aBt@_iNZ^r
zQ4;l^GEc0YqnfcoX?>Mf%<>aUleF|!L<GgIy7Bc?>?#!rR~?OoEY}uVzwg!64SREI
zyQ0bWqp$DR-L}x#*eqJDk!W=4mVlwzsa4CTm9Z??k!Uo@fzQ&xHQQt7&5Y>tTQ5#}
zA+o#0rCN9I%RcL!X1=Cv8eN7rGM7G^#meCO_(IIS_d8!i9P{7xdxo=a%KLTBk}uy}
zx$PHxd4+XV>h#HJzAvvj9Jc!ZeC^b>j?}+4*0=Vn_{AD3Giyp6-LO^A&nI`gXmIpf
zUez@l6jzJgTC%X4@6zr3Ccd74_nkAYoH9rZY?%<mzy8w%HL>`f7>8qtA`9G3>)sFi
zEN3}?k%wY8%k+iiQ&Y<C-tbti)v5dCPS9zMomYgsw{PmX`^_l3Joww){P}ag-%01c
ze6i(O3vYafhhX-3+w8*C(*jj`uiOi0{2S7su_eJo=Ml&0hzpB(x7PjosHs|2^YQBI
zUy2NYS3MLO6J%R^13f0r|9AMEcthP|p-21cj<G$T5V@5_kv-CbasJXBX^I~jDw=zl
zqk|TPWHPF9AN1%9+9Wh#gNt(cZfS2Tm9;bfpEwd_HS?s8fzgurDlD3Anx}$Jh)$Jo
zdKwcnk*jspBfVuCUVDCRh_ycPNPWA>6R)a`mt$Ch#SbbN8dP~M5}wT=5vIV_m45Ql
zVF5PjlQzqAg(_#H889zBo6r;dQIhw(!?!tqPq+myDLirUjJWi#IW0V<*~gFMq;2M4
zV6dIMsBc+CvaRE)=gCW|l(QG8nK}RDv3BfUe0<gASuYm!+ZC70tp63f=>yLRhoWTZ
z+fq%t1B}>wx*MZ!pZKsX#HcM#_|@%u4A0FEy1oeC?-bx%BI<DH>ZEC#E;el_+1x5}
z@R{n-FAAHNENBdw**kOc<d`0>B1^f{b;Y()UVU9nXEixi{#STl6*NgIXxF8y>2uU>
z9hsM4uBsX7IrmMA?8l_?WhSio^Ku?dnyqH+Wj=9HLhPi=ky1)CtsGmtYog{c%(}kx
z#m|W~yt0BqOBkj-X>FCg9<A|9W>#QtUWxgA&xVaBE-UsboCsYr>BRq%$o-Q~t@RO@
zIsacm*8cx*{p<g~7W$u(tT@+h!t%1undNSc7JVhpRV%zh*>|@doRsjSQ2QjOUzQT<
zd=705X^(_ZZYJloZQh!(vf@0Kx9ToA&{h%~=%M}W>IQAmiwpf*I`&WVkl6VvbzRA$
z=?Cq0E^Li`k#uBr#}SQflA9BBvOhW|Un-b<<4dCY1f}97Ckm41o7w3qYW%;+qM4`G
zc%h~}eZk6*<1Wkg{kn79^1<!1m%L_gdiA2c_gkdyjY|?IH!@}vMg2%^<d|}5hb_zU
z16i52yHmU#PW$BaXZ2NG-AM{8y5T#|roJm?Zol(f?fcZ-H5{%i3YYjxRn+(e13MBX
zg=E<J6+JCYd%DE)?Af*3-R+s?KHv6gLeV_EIVnp|Wh`OXSh-WrD@<w8^s4D%4?gnE
zV}2H7xBtoRYPO1lOA4k${xa%%w6Jq6-^t6L)~QU>^L$V(@l?ZoazZq3p;myr*~fNw
z)`|U#SgPGar+TKADn_1_IHndVT^6kAb#lv<gDnSTwR;{v_<got#7WO+V&SYO?;=zi
zeD2-;KUw)kiiY;Oyspc$yLbLRcq7A6^}?r3><7Jtm%Tf(QmrS8t8vmZ5uvB<P7hy&
zXG(|^-xSc+T~j>we(Jd}4l%EirVRp&yb_MLQ!f;<_QZ48&e*Rqt26jWwwPhcay_dk
zqiLMUku0-W&Nr}cS&`YxFkyn6<)LTEt2I?F*R{P``;m>o;B&9Xxf7cA8_sW8&MGQY
z7p(EU=9c0PzmD~S3^|^ks?2BpIj^R?ijztAcc|2*uA;B*3V&JiMA)NL%f5zB+_`bm
z?X^8$I$UP@n}w>}+1+HcHB-{!HJ{(zw`u0P`CYlb=ejV7|IG83KeC2Jsn}t1^PNg&
z7T+&HlOJ47c^|ap|H-q;lAX+7YR}cITqLbKfobj?@8@qGG%dTcw|E`bv2@$_oH3@I
zyS)|Jh3Bw*IB$LPtD?A>hLgH-E8qMX&+^v%JZta0^=^a7BKO+)p5Cjxj|nUZ6?@2a
zp*QZ*g3VfCKIxj?!EzT2IZd8ge4VuT*Q11H%dEN>c_ZJ0`xI)N1KtR#{Jwe6=V=X>
zwyK_C_4M6u9xc(!j`S6sAT&qns<78r^?R>4x37EjY;BgPRz6?&E!h`qW!`<rt63C&
z*r!LWouS8oY4#uIAF4Cl*ZZ!Y8}1XywR!!+Er;Z0tdxj3bBRkcr)TrT)ZQ#d?%xwl
zk7vahe_dHEb~5Jrz0EP%w@&`xnE3zXu6x1ltfp`8KmL8p=&Xs`;<Gs?&35YEndWfq
z+Y)uzpxnKZ>ANm`wcFJz{4ca>_vbZtUdQQf^?z{aSoo`b@9y=@Uik9Us$EUR<t4I{
z6dor;zWH@+dxbDZ)Ryi&4C@%CtL`m$@_@(B;~sa+CDmAkzMvXkrbP{jIgJ))4wcG0
zI<Vr27tb5!+UGtJJS_F@PyY2(aHV?laIJl?QYDS=d4d7k>f^x`e#%=C6xbFWi(@rw
z$(zpBF|F=Gen|mG>eFK-uQ#phKf%#l_vDoK5B3us&$sVWdA-5!P|F&w{x`2vlpB`h
z++Ehgy}#hpDxSBG@|0avl*L$h1>N6W>T@x9QTQS6zxp@6)^ls#|9aON(V?;8z_|x&
z_}}-mCN!V(QFqZ{E(_u}yV2C~<kbbSi|3_2cC30KxH!XrZ|!^y-l>bub(<Az2<n^v
z;E_>4{`mzdU+oU}c&G;3wOP9@?ft|&F~vD`SF>4Ak0sCRA9ZJIo;|)Ub?ADEvx?g3
zYiVbDf2`=u3Y;9*a&t;{Zph2WoAO<%w4HYFFWbg1bx}9BXI=f4w|}-N8Ey+S+#zUa
z^6~Z-k5_*(jaXLQaeH@iQQ+?<@9!Mgu<K6lt$S-_U*}JsqpMtZa;DUwRm=SEn=DYs
zdF}YGkefqDVd2vIvtHhhQ<){@G%L?#FV8ynZvnG48XoFQ(vo_o*w^ykbB_{Jz_NP_
zSeSonKi2b`o3ottZsy!L_X7rfK0bL<j=WX<nfXNRjiSuOL#LRG`BqgPae5}^#nxz~
zQT<lML5nNLV7ph>(hti|%rP{JI#n;@#F5&0Y{evw_75+M-n`tk?$slgb$?o4r96JJ
zh41yYKJDq7>hI-imz`o)?9e`NM)$s+-c`QoF>iMFoI3rakGIn^YKa1$>H+<=uJ0GU
zidymX{hX)KZ0|n^6&s3v*4m>K%OI%qh_&B<_mjxP1sqEcaj?d#ypNxCqU(U*r5%EQ
zgVrl8)6ms>+^3`Vb;$vH3+?OE9#3dtvr3w5VsR|nk6SGHuw{+zgkQ`PiXK~s>89^n
z{NKE_e^OGeM5dsPpAyre135E%w+FmT$nxV#<WicV^CwAbS;^~PTRvL52d_UAEFJ6h
z>VVNJm%Mo=ma|u9%5kReoOC*Wk@sDJAcsw_mFr$Q?%-kAt!lqKe@~Q(!`asihQ}3!
z4U|fs6i;J{>1cVQ(`p^&9(T%#Rr;uo))p4S5*FKaoXzge-T@WcjvNp8P+=pi{=22}
z&LN#`3l3-LviZ7d@IJko_2K9xwW)GGuU9;A)6Dl#Yt~)&?v+4~d#i}<$8?XKxdk~o
zp4nTpGS6_#-C{je;KZsF?Z2$7uNQX|m2ho8!qv#bJ^hl^{f)MVFL0mSGJVb)%QcJi
z99>i;m{xu&`p+wQWJbmzwG1V`+$6r>RJARtXFq+=xT9l~{$lkGt^m^@Ed?beqozkJ
z-W$w582xSO6!Yy|)p1cxFz9bjkp4x31A%4=%XD}a$A3HX<xG+>*P<RP*6SI*KccRP
z?LKTCVdyd;b@PM6wiO%0SYKYXSsgZ&&Ggsd8xwS`_|;5SvKXFmGCF6hxI!=Tq@eB0
zuMxgJCoc#|FAcRS&DIWCDZO#(XS34i{TCCD2}Yh2luaz+UK%L3{&EawyKKB??8-v9
zn}T+Wi+G;s$_u+Qm^dm<Ijius^q0#id+wrNY?tn{T-(Fs=cdE%CMV=r{A89up5~5D
z551y?29q>gmhZdP${p7HU&W?Kbz8$D2mQGY*B-fS+aL0&%*w#ys!Q3WjlO0doZoXk
z4LRr6cEY7}lfOxiafFDQEK{I)lDSvUVjosjC!NqW{Sk+kF!zgH;Cd>s@rwTpvkU*0
z-qF$Vt9m1|_FGs;mQPsM%Y;_Xx$m;Ss~=yL=k@f}{G>AXx!3$(Fdf>m&U?qX;{uO!
zPjB=|bDeg@$Vu#w?%QvUU)1tD{Tc=I3-0$R1TTHt^tAboh~HHK-h-=mhcUfp2v%b&
zE?Srs{fti_k)v38vZ3Yo4@t=mliJVgd=AoDW31ZmGM_p3y_SjP`GPN8%&JOWZ<I{^
z{f)B{xi>BQ8kb*wH9h24^2VsN>tz%Fhq^0s*$CO?d@i5({c6rR%ZQIR=1iVk^g7|f
zA%V5qQ#w5!Uo%^`Z)@aYzsS;WmQ~t$`+{xueyrJT*xj`HPt-}pp3=zUR}XzzsW{It
zO0(5AIr;aqV7rCgBF7s3Z#u2?EH|E6Kz{a7``KLy+#3>-moy57C0_UTC{igAyHYIm
z#c=|Y5w9v!%~@sFQ_0ziF4gWGMpIa%*E|-|X>}D~IlSb_mUr`04wo!uIV|+b>zG3N
z$#mb)vqw_4)P^19N^Eyp_{7P3%VVp}3r&JE-C4Y=PrEy>^pa0HzQ}Lsv@aZMe|WBz
zc(o|x&@_Q{T|YV!mA#8gbk=_`cu;miLCNQ8h0oC~|9y6En6A>I(DZ6Wz#(O!?L7CV
z6pD7JW-x_1E%Nhx(Q?Me|NVxun@;&NUOi_rouB#rhi5(vXPy=>)h~J78t3sv_n(fQ
z)Wr*7>+}>h3eEKB5?GvIvNf7DLtC$H{x78vO*KW2Qeh?4<gU-=N&4nVzGm`FE^f}@
z<uk)39u|}Cw(K=r#PKP2&32=Z$y24<LIU4^ahe(A_|rIWN2<4o_rZ#;Q`ds#Y_&UL
zD82k^RCKw?QOVk^U!~7R3eRV*IUC_|bgSfV_PZB~r0z%7zuXyfb?U7Tl03_UZY_E5
z{B5(szp42gLCPwP@-I!}e|~igN=Xp^u_vjeQU1pRl?tx9l%~d%|Fhd#99@skws2@x
zG*EuZt<<;lxyVu-J&}}$nek3#OC19;8GRndM9fbyTK1Idsar{!v!?p(hYRLR<0;s&
zFz#!B*VB-HKaMs;bQtQa{OM8|7N9dnYV972RlPepvezA&CD!%Pm20WQ!5IeHdv8si
zVBmT9+pmx=&WSx<n{RgKMx}9f^~f$w*zvBl`_1-cLHVri`RpgwwSLI|Yo>H-5|4aJ
zpJ~zxCzh2TA_@i78D9Fi{;?>U`e5aTrA55PjZ7c?&A)H5HZM*+oU_Hj;@yf#H?=w^
z?^tLdEXdtfYU`hv5T1KEX?f`7mG3#K>Mon+uG?1mF*Nq-w#4MkS<xyrza_X^{^!c^
z&-}h7N$r_p$Yw>R*E^VBf6;mNW!AUMrO{htx2B#=%I!S9?gm%S?&R6_YiiEt&$;}4
z@{!wrZkcz@6uoms#pA7b)cLD>lOIdoFRY)-ear0O^nes!_KF3fzcn~-6wdN&JiEG)
zJA{#cosy<R%!Adlk8}lDMA(M4FIk%4eSz~((D_F{y7vbJEcKN*U{mAleWNvWyIsfy
zv8=}B>sxsCJ+8LXVeHs-bXot)J~N#QSu3YYnF(<{%W*s&IiWJ=&hf@Q$D~yi+hi6B
zP28P)W8o~Z<@1@n?Q5n?Dd=8Sp*?L%*ZK(0wKt-cPvMNb<hA^lS9MO$(Hfoo?@r8P
z_PlS%akJuo@98f-SMv(}TV1R&R`OZ*Kd-r;b*Nu@b_-MF`zt&J|F2Atj-1FI`$6jF
z`!il<0;Ud5FBA?b&3wtae7emAy{nUZUtBVjoMk@KV8e<jhYf|zS#;ejUB5orJbU>L
zPsM4;Gp&3!Ud=2FG3nGepH{`2yu;VZSf<HHb}DP<9>t<R*K6-ynR`5`uR7%KlbsW5
zOH}8c)ZaPT`%A@|oi34#v$b1a#caR5Zf@SQXEg^S_Z*%%=lo7#`@4}>f65%cYPIq6
z-$^|4&X&rq-gx)(Ou5Tv^HmxY%nwGKITknDb;11BxR+P&P4g|ZOIpafhavk(BWK<s
z$+AW2R<mWkDX}IjRy0;N7y7R(d(O&Y+LH5GD}6UT4*yk}mG0#up&CAk`Jl>^lx-DP
zm=1c?EdPBZGg6VUZjz_PqGy)dmc+ELoLY4}{9a|vUh9fDhO(L$t@~bd*1ZT1Sebuq
zrC`_fxpgiV#GbE-%Dr5+pl-F#j19{+6s<Wut7~`N>(wqf|6gh^uUcDJRdGON-PL<N
zPrupr=B>Y9_hM4syYdft-S6@~%IQ56y1-huz`iB<A)mhBY+l~mk-uha{QXH^*xrEi
zXOZp`MGo7|hPR7%I~RRxODx$tX?6AWw@+s=Ki*<~KKlEq;Lyl*G1W;Glahk=rlifT
zto>ajVY`FZt;a=hW}v-sQ2&m<Ed2%Y&vZimcPcX7=={?s(Dft#|Bra<ov}9q7FB;%
zt-aLEvHNAxuj`!wd!!>`#P{s2o#QHe=Opjned2XDt?PPC?%tkVw_SMolSrQZ8+QIW
zwo}K6i)B|zmELZ)U-9!(3;$34^~d-=gHr>GCNsB?O@_h3DV*G_J_Qy}AGUOes7I}_
zXk|Le8*ITf<4|H(gRU~`44Y1)R4*1LAr6LuNlOo?1TD$fsChKSk8yU?)=f_@tFI1Y
z%9^MY#r^ccLid?m1zHP4S1u1)=_i|YY2~$5e*Ke@N`-R5rY4<C$vt{1On1RHAEDk^
zDvm;&ChlH0H?^Ko3yP~{TKjdg=!0#Zi^cS+K5ck(L`&NGf1S>PwVo#@YHv@K{+XV3
zpiOxH(>L2bL{Hv&#HlXF>g&s6ivzcXSzb-axyf{cV@8(rca6I#ZZF^dz5QM7!Qs}~
zsgHMm%X=E`zEX~RZfVZSHG9I+pZ<Aq`9@8p*}W%sRVqHJA8)tMx2yg6Rn2o!ef*?9
zv3poD-`y$tuHyOU&--8R`4_~#;|!S0@XfREL8I3dlUAYNAKw+~ls+z>?;*c1<zcH@
z>WPP*vUw}+dxp%IawLoGt;z(?s0oEr{ND;Z4Gvkq^QxD0-o|esA!cn$7f0kagep(4
zw>!$FnLFW7(xk1N1u~PjnwF;Yxol1Lw)GUe*2nNx#gj|gd9TKe?zdMaJ)QBtv(a;z
ze`edtWj&c<t<Sm(_Dxlt6SXWP-Ro;cr0UGtpiI>Tb!<`G3)N<NM$OYmwM-6)I=16h
z#-w)<i-RUU>X?+<H*=}ba=)^wNz3P`rH4)0r+89jmDjpeS*wpPj9Ri>D|6SAT=lbU
zuag(g*`i&sm&@Rk#J*2jIveh>TCZ7uREaxhBa_bSHJhUwws*2r<Zg0}VedU6zfC!@
zdb--5%)-lCqB9cnYRWtv6rA2^rCe}k@nV_iA`xpM=q_BlCEs1RO+-mthR0*l0j<JD
z2PU19B|)8<*Ghtqu(&BYid1a0n33SB+ZuLUMn6?MST_Fav=d6zu2UCpTXZM2Z`;ak
zizEKu`5AgX)!feQrl-{MD;ejll1~|5SgpHil5WVnlA9N2Fh14jX|vST%<S;2ot1T-
zx3~0y#@iW-J!39=Nf&K=`%L=gfvLV-1)1NsZM&J$y;ded|2fCBUIS~htKn7Rk+S!7
zPxoFbxH8Fc(gPd8x*rc(tIh8`>Z%U6c=SL}CaHR=aEEwZ$<;0Kb#s}2#J^a;-x1`y
z>~hL{7H+MDznXb$Id+RkSIoQ5_A0Amx5uvd9o6smRR5@Mi<o~yVs{kJ^(5`0Qxa27
z*{^=GCEH@wvMUo^kF8oYdDAHk?TChNoa!^9Ce3^{BgxA5-}#s8^VNPonQdP;^U-1c
z|5r4h#uTTRzTa6t^?&U<{f57)PivXi-1j_Ha$wEfHJRs|?`1AhX^>xdG@I*B$O<iv
zRY95g%}a|ySM9jsC{S0>taejw?E$vg$Icv6*`1`gKJLXKiDgS#jeR;RnZoB8PH2zn
z*&O!lk<s)yf6`T6Xr0{AXEDF{?Q)ejt2p=EV|8u2klpotmC`<krlb7~Ii1r^OZkYE
z3C&n>#O}vUPmz|2V>?wEV{|@sADh%PWm&;etw6<@vhHb9ZSqv}uXLWhx<cG>?%yMF
zGZGUP>peZ-lBJt*(O8AqFk<FOh1^MjJA2Y*Y@2zi<C(#^$s+2ytS8sCewtu2+t2IR
zwK<P<+If$1_Gw4oJQb?Oo_8bjT+X`x%TBz$^3ke_nLFNOtJATK?)tAED%M;n_W8Ep
zdC@b)xePkb<T+IqBt;2K@Uv6jl@rwAWOdTB$?y3QR?mfwf>O(4)t*@1$vx=wnnkX|
z>_}_*vvr2bTu+VXb}SOH-JGbtllxQCrp?EH^k}&-UOK-)Y3Xal%!O)}3n%A%UaWMq
zC#`Oi@ZV?4)qW|m2Fi%WFAx$vV14stfWqGubM3N%CO$i3G*NNMzn*W)|Nqr&c$Ca@
z_09K_EPf{|Bdk@H?oji3B|E7~<IWXU?}`xau2)TpGK#Bb{ko#Qca`U6D^7#_*3erH
zX%Tu)R&Ct#YxM@b*AW+ju2q~mH90BtwO-VM?wA{@4F5}1KStkmN_zHZrwi|quL?hE
z9tlett?#(9Ixg%8$NrM%6LlGU62zi(*nNcjPt0RUWO8uwig{$fJab8>h2vrojW8wC
ztZyn$PkEQ!TV%LIA<c097l}|iwW&9r>1G>E<~_?LDY3xScjA(m2fa07Tiw2yHLB>Y
z^Z04vIa||whi~$&JjTr@uim*d@0-jDuFb#9wr|bNt#+LKU|YA@vySq-Crf9C9&TE_
z@9kUdty%9M9&z?jbA4CXmL2oROTzNI2Vdc*y)sXCaairTuyodKkqKt}F%K?XeKLFX
zWTw|gBzIrrOepg%QTtHS{_sJj#o>iM9m$I}oL6yiIHD`)z@ibRP^9^Pv)LMnNtrF(
zQ4PmDG`LbHI;|C-t>n72H~DAWKPRpI3j6izT=r#5`{rVz;;gp(+u;YPpPujk=V^H=
z@Y{;tVbv>U$!%Y5-+W8MU+JI&(^o@2*OT82Rva`bzOYiRXVt8$Z)PlF3tqj;Xc|wT
zle5iwxAxwfq4Vad`pD%eDmtWGci`G1zmv)1;*C>xgm`&f_q$zGoU-k@sD7_tUxi9{
z<Q5rm!xKW&I|7*&t38$!?@`z#apHn{r^D*_w-Soe_=^AEJS4Hjpl#0rZT*?2zs62I
z5f=Ym{rdLGZkhTZ<Fgi*PM?X}ws|)1+jQ^Sr$winxu~M)`7|nRd;Q5J7Ya<$J(!9z
z{vR&oURT7bb9cpT&kaxX3h%8oD_bv}R~)RMUm1B}TIqAMU`hVv6Ly|S;j($a!ftWE
zopHeqpU*wlWMrEJ@0_38AbeFMT(<d`r3<IzMyIzwFW5Z2x%<Sscl*D6`C$AoYLP}y
z#Z#4{t*c`n2~9roXOsHOoiQ;+NpqMDpDVFFI$L{h(he5cXF(4pO;2!9b&@-K%Kci=
z<=02POse|gr?K?cm42VElizT@R=Zin^tMoFI#<I4k9C)B%;&j0i!<wOi|5HCtsA$^
zzKCtLk~UVVyfvdxd#hpkzmTE{Tx){na;?hMvza3uoK|wX>zvlY#E5u~u#AlbYtmx*
zH|zC><upE5;4k{Gee_#ZMxo$D`$tLHJ0<qYd%l>eGH2?S_*M(ngHbITes%qITU8OE
zYZj)`W7?TKSMvPKaI+qL&3$J?V>X`8D$|SbZaD3$Qyd;2_N!0O&PyoaX3Ihr4yFXo
z@4lb@PWL#dU9*yFQ=ztGmi44S3!!&&%eel3KRy5NmvWKcwbol~UjA>m^{zfmZF6-0
z6Av@jSXK_n-js69iPGDf%y>_j@!g1Bw<PwVYyP>7Dh<pQ*ROiVZZf|e81lh2{zyu^
zjY(6FvgO0Vrhm^YeuP^75tW})EbcHRYf*9nUwZRP7on>uR*zZ~{9J^2w+Zky6^d>v
zj4iR2os{UgBXNIn151*?@Bd4R7JjX{9-d@yy(PRtC@`T!w@f4CT4@w}Tl@+kuQnmC
zw6;WvprjdX*&b~fGfe%>B!ku3)3%qSABs|Gb%>~tYIo6QOI227_FDe2@yge<Y`2Pl
z)Q-*w$A@A*eeBMqWl{}o&YFsqhZcoR^WaKNv9f2+oU^&m{DgepGhOc~!t2B{*%e)<
zx<$-5)+MshPexEEx<s+dwWU|l?bvjo#ce`-&RMfvg%#YC)}#qtaj!kxA>`3etSZJh
zEmi2u451SlNoRKSSRAkA4DLO2yjCwUl0kuSc7tp#i^{*nIXgvjFMf-@bga(ru*SJk
z^@5ao*MoiBE?&RWV}7|yexDID{r?if8AtlrHB5Zmq9sHccABK`Q;ECJ(Rk!x-m!xd
zst!uC@ARrt^YS_v_x+mpHlq&nmL|64J~EaC4CV@FMG7y5WT;Lm<Y(`aysj7g!B^yY
zg0Q=vx@Qr~VL#KCeo|~j=9!7U6-n#D42vcft1c6|w!Hm(h1ATa2GI>F`wT_&lDNMG
z1S&o)&0W!!+R+wrQzBxyhU-fE^mL&V@%D7~RHh?kYF|C`+k>x&mS1qzc4+n1oGA76
zn?!ATNW=H?I`xWWkJ8$<mp5FG3wKTLx?XYIDeUB@(Dus8*3KChgDTGlRbDAla=2cl
zI#GB&vtG0U=ih^t(|2-Cl`LGQk@;y+*XIA%yTZ5l{hbuSpv(~cBw~-b0h2)Fc8%Gl
z?lmfxY8P+qvGY_o!yc~otY%+j&D5z3w=!Dqw$~nCZgAXk&du~<HAdlw$7{F0kkJ0*
zuger{AE@s3x$k3m{hgh2KYyQlmvi1d%L(6H`lH+W|6TXwT~ZhGa^9uO8db|fg?}`B
z%`_8~@P2UA%zKfv#EUo$!)VzRakY;nzKX{Go*Mr>wCUGY3mz`Ze@C1CE|vE@AkU>P
zZo6WVrlr-Fi3u!D3Xf6>O&J&Qs`>E<D$IAY^!^f|xv)jID)IeHp`#J42398LUi5hE
ztcg~TaaWfp(wLg_V{!Zm`vi+6PQMmASGBoTEveen7WV&VTdH}x*Q-*CtASdH4$+zu
zsw>LtHRcyyYpLBH(%R9HJgcK*rc-*B&gO~f{TdUenVZN}c3xQ&=C#st+2gQDEB)o7
zX5LcidUh#ert{3@9J9DD3WS8rY8CC?b$FKWBERX)-K!PdI2nW&h0S(b>9)7C_=Lxt
zi!+1@5B8Xe*Qp2<pQz})`a@#-i=NMhY;h9}TTA+$Xym@z5xi-Z#LMq>w=7rRnkf>w
zNo>!q)Qa-p-CYv<cGd6h>c1i8Ips^e`Q{)7&&4IzmUchsfA&o5^h8aGO6jc26E?8q
z^Sx}?WRjn~D)eVr{-0;w2bWBgi!6{mxz0^vlKf4*e~k*0yw)2qGEDm4)&1?1|DKg@
zwi4_AhKTETX4zCI=xMe{wQrDL?kD%$+VbUw9iP`2-jZ-x*>09KrSD+(btg98l{Wq}
z%Oc%PLM1nbzijghD~r!8OSoBDViDYE6x@3-<=n4amC(Ei6B~p*-GwbXM7M-A9(S_!
z*2*dC@S7TvwyVNjdxq(i(6gJi%m}aSyg8#MQz&RRXI5zj`^VxUMuzj2D-9W0qE8#>
ze01&a657teV&2{}Z`M}3<J<DL3NO*DZk?(xGO_xfkx=)m>UAfp#f2h|pYXp|KHK<t
zkFj{2YEtc<ipayBvyG-l8o6%_sSvuT$uOzg^um%U8(Zh*Oq~1Z<j#jLcRsP4xA6b%
zogbGA8U9}V)g$J_(|R-JHLoK#FBDnJc4MB+rS#0ndG}An?))8_T5R@<bKT=sb8X`V
zy2a9xnN6RMHmS^LYQMZdaFw1h=R(s6`AZ?|e|Rna-nIAY?7g?6_j(5Sxnx+Ye3$Up
z(V}>LgL3!2$j3d-%8XBxSaSlD{XEOk+Lw4o?f2wb5+t?6J8Q|0?)~{UB))j3GA!92
zklxM`ArbLv-g<4@M7P~9cb8|1ZApq+mc6aR`EkeO3}@C3pAc)^?F&2YDvHy+4^7S%
z;z_PaSaPUWaNAvW7LE^x^m!Nzcn)(a95&-&Q1MysxqzY7YF5$g%(B^n5iAC~omp4i
zT;cq4htYrY*-n*_mrfqqS~+L>)ZPWJj_l4j)NtD1SJ$B(5j*}|6mM*de%&tdD&p8-
zuKra=SNmLE{lQ~R9?$Vyo#QjE^sW6Z)&239N~px~S!=Y9_UAhu5B@C0*x^0ND0a2r
z?l+!!%fE=6PMomiqZjv2FRqh$dsoY>+LEV#EN=Tk@BJ>)yNyI2-CFmWYyHwSdhey=
z7fjA7F5Gte_nJH0r*B2?y=>C*(@~+vYqI&t4SNo=N?bcF|MB#p!&7^?54xZ9PcAvb
zTqrCpE#k=@tYqD`ym9K~I}%r9&VEyo)SA8N-{q+un@e@2OMTVOMIJuT&C-$3c8>p!
zXVO8gH44)sW~48kG~LHZtNFkA!LA$A4@%F7yqezfb4%CD8D~Fj33|PS>8!B)oX+<_
zlfL}gn#6MS+)kmC+ecHm>k@8Xu&7xtzV>4DY=#vPnT6G^8_fk3FY2%Osb9rCd+p3i
ztj&>)(np;u&-l%WOlT2G*s^bX<>l2E58aGd>75gKD?{POLe|7RCbNI<T=aUUN%Zlr
z%dfcY4fb6d%-b?8Y}T&lGh=j?=JKl8_9=?3op_?)=e#qa0sV$%dyGWCcFy0qdhNYa
z*S`GR-Mqql&Y8xew@tHO<u@;G;=dI?dGek=Pn#}exA3hJ4%be|QaP=xeR`wJX~qjy
z7C$;8ocHOSN<8XwlPyCag(1Myc>nj)OMb21pY^|@ZGrEmMZ9Nw8O!1&%QPmQJ!LwL
zsb}iZ&|u~%seKC%Y_j%>P;|_%j7$Bt)W`Mwfxv?zZ>BFl*zxC0`ms+HlTV~Gsdolf
zhrM!aI%*_Z6Dpt2>N5HCh2+(D!((catPcz7Zrk3;VEkvRVa`R%oWmw-4yQ}IOxxDw
zP$Ss6s`|68(AO`s+%y;%87}+C2rn00IdRt!fxAKzql$g!2raCd&1Sdq$19-=JJ%Oo
z)ZI8$EUBjN`}Ao+n~&w|9DloFwamJQySN@n@*U?=S)&?v++y*XRkik-apw|z7hjB;
zw@z`!iYsv$ch}z8wf3%+cYg2M3lp_Bm26qKXq~g&DFw?@%t=<F|E=aQnax@EXN}Rq
z8=Y(Say{595Y)20S#j5zr@MsD?Ac>ImG{||RI3lS&&aL5=^J%6v+k_Vu4jE8Z;6)N
zN)|6$&AT-5X6uwKWhPfkqrOQPd0#zNs?4>=M$1_9$M$*LPv(cVrrB4A)IWcbUv|DO
zb4%CsEn0~)&Inz2-xxMiZ|j_YGcL)lXEt1)W4%-6?^Z$HqaxlLh2I`p_U_e_+Ix~`
zFTUlyBvCl)pON79xa})DFYR3(v73GN37I)NGFoT4?l^K>$g;QF?C$+Fd^^rc9OZFd
zd2M>_;qBcCIo!UDt8Z~$_3yoM@7$FKoL3*~y?dJXa9MTmitMGwSDENc&7HvL@%Wkt
z>;E5DnV#nCI_tEeWq0=7yki@8rzg+9y|kfwy7_@Eo3&k<PD;zGbKE#Dy_fI4Yx0CA
zhC2?gN?ZSR*1o`XAHU3ESU-7T->j#Wnm4^Ao~gz=3Kj^=kQJFFoBI3o&9C0K9Hc%k
z)ctJw|5;(*=ap;MtkIpCcKvK*q=Sax)xL|5F5mKG{L${;81VLwjO33s38!9&+}V8T
z==q0bcaG=IICjLML2Y^L{4mb=trv9fmTDc!eLd$@?xCr-wsPIO&vfsue&W3ie)k@)
zz4vnN#iEOcS7u~7+H5c0o!J?6WHX0>d(EZryqZJq4|3;sAAWa9&+&4%{R1O)VaB>c
z`msHo{BNiFD=mF$IP3px<1MdNXU#d5vFD+Z-kOOYAFh-?euwW-MDio4x?QfDKX;b&
zKULF=k9}Xc{M;VZfMrue7wz$~usdn#_h+IF*OzOLUpzaxU**)Yo>MD)ie5=gdaGzv
zIB{=Rbj>fWgVOsy_T)b0oPAn(<7tIYPY)^=$z9rh#r3D!{g(e1m2w}PVVJTetMAza
z$IsvW|A$LFpQuuHi$~xq3y+G$1u@5HrrWA(uI42iZraKvH04LZBiA0*XwHHef{)!8
z6%03B^4K7%ZN}uWOvLk3l2^Z^`$S2Lt_NpknXo>%vvRV!KciLiF&&St1s5AV)bnm>
zHoRP6=s)*ekJW}%4Xb9aOIyn|b>+47k^fmG?y5MR644Z1;^8rg!J%oJCkK-N0}Dg!
z?&8jOXFayRyUS8`pNo$t!hlJE<K8h-YsUjEntj_{<{3$DdaB8!k=?eY()pRLj8U<m
zZo`)smzMi-cU+jB&~$wxyV3;R2ECj++gP8>^PN!kzHtNlrN4V`D+~1$u`9*<O{ipf
z8ok6L-}Tj!wXIt>+`jc<L$ycWgT3wl?F#ldcRoM5-u(Q$njhbu-d`;~)6n)<--oaI
zE*cLW)c<F)>CoS2wkltL`mvx14vSojE~qp`1g=^T*rT-AX^zLtm4_CXU7XP4>%cN`
zih!-oM<>DbPX&khE$?)SEcIyARGZEAD@#Fu^Olpl9Q*%U7mv%a{s{C~enR4tN7Ia1
zd7dwC6utJm)o`I#Z@<JAqh;GS)dYF#ZLcw6;GE*%y-W5qPtl1^BWVp|Gt=VDz83fQ
z6!}@b-ZI%ybIXi#yTYO(SIlb>5^_**Vpm-e(8%WWCBRKI_ey}fa%;&Y7xh&pYSkR4
zN*4I)OI?`fASx?x!Ark)#oR8I2^m)#qdY_RTYGB>9BRBJ(V-INx69PAC#|PcX<^Nx
z%o}s+_G%nkoR_sqEmeP);OY$ry0UI%%oXa^+%(JP^_s><P2M}2-b^wwWXxc0h&j@x
z&B0K-I;C7e^YmJyxF!zfL%K(Jf_3g6oaeL1@RWkEMZzh&<}(j8T~7Y?*>b6j<?PuD
zS4whY_g)coVCst0Ua>>#;L#In8>4<(?C7yA`>r%$r3qimGZv2~A5Fhi>%`tNT+LH{
zd!acX{r!gEtIy}uKZ^|QI&}HmqJ_<2QZE*LzOB7~$@kM42N-x~eLm3t?U2<WZQ&<d
zL!%B>2PiKwGxAcIQRh|oRv^@6WA3*5(`LIerhO^Zk<6Vm(W!sNCPU8y|HbBbC~&LS
ztm>XNCF!K*lnDz?Hyv1e>4xRrH=l}53kyt+HaZ_7?Az)5T<M&q+w{#}%$S)N6o0a?
ziZC!S=rAxaFf=eZFfwrbXJFkS=x{;wU^7d!tVzd)2M2?gqNWN4Ec4;iV>-ccpkqRa
zTQ9G3m&i^}hm#ZJtj+FRSvGHN85jSgOG*t_Js4N#aY-!Sy!`xx*DNfuySi2cE_9v9
zCt78&;#?s6QCShwty+%r?CY&kcj;wbFLVuW{3|7v9$uJK%q8G1s&#d1i0ejCQ?Dtp
zJ9FQ1NvQH^#jJY#hfPgFBr9NED`%IhmDZaZp}i-);{J$gK5DBoe1B)Rq?Z4F1>tK-
z8Iu#9t+99R=UMwpA^W-@zgAAJu!?5yQsreL8>AESrbc>h6uhaHcxHxR_x%u$RTZ;u
z&)MP8Zx_3(w2ysb^8b}GrcND->RkUboR^&`n<zEg{CV#O6DK20$Gnf{>Uz#h=Z)NY
zX-k%<F5~}++q!($@NitySYxtGA%(}R?W*5|+-I?V0hMjS>0+x&7lw$&{dnM!yH?d9
zMtPdYqfRxsD-T<Io{EI{&A+Iw+-7|3#LZx(vW$g&b5$#pd*3bmam^!X&Xoza+j=fV
zyS-UyAye>0G5ztBkg!bEs^|$5pO#*mcu>_*qScdihEivHRQZ=s7HPLMBhL8AVbivB
zW(DYW3V7>nHI{PRy1`jk+wZB$i&m+)%*R0;T1zt)MOS`b=+Q5AOC&s|_3G87)7o0E
zEnB^yRXDo(uIj55<yTY1<4Zd|riM80I<65M5%_<a=CUnOtEQ}7A6F{izbEgTLYUdv
z>jBnGSClie{SS!+Wpk)%D{o$L(<ElorAgQ3Ze~x^4)zG@OPO4i>*2yBer4k6rKOXU
zQl2@mY^@9^|IikGukymG=NAhuEL$z4xlLw^y7lj^8VY`_YB%N|<S;W^&}66QBjwLF
zscb^OVk+C!hA!D9506RZhph<J?Ap{dL1BB;qD;y76<*GILN49GTX?iYkBKj;5k9W@
z|H$G@XTFzr&)G+<QJTQ=`9;AY`&U=4ov2><bozpz8(Y$yfBs8JcfR{-is`jYrE@3j
zONfu;5|8o<H@o>F&qdJvS)=7LhFBlRfPJ?&YESSeGcGaT&NSt}QJ|~1;Yl+`x!Blb
z&zW!2eq4BaBdCAMnx4i7R%X1jjL(_2+g=KDkKLuQ^zBrhqBQ<p%PoUjbhR?VC$8Q1
z>!^D3sqpCHiB+#+HZzAQ`R{V?`t?3q@!8c6bA{JuygK16@ns#aY1NZ!!C6;M<(>+P
z{d+y0-(33o`$Bo?Z};23%=UOZUsJhg?F7Ntoo|-QPAZd`pt;{K!|T?!%3|{n3rVka
z9SgJMBwO9KxXAZ&besP@b-vDRW8Z&)6>4l*6IjGu9667@a>zV+C#mG=JienR>TMof
z@b?fCUpDcKZdd2jkb*^B{k1+S`cktvs*^0GInSmA8{Pe6w?jPa!vrUn)f4*v|4|oO
zX3CaobLohL*W?2&H&oQ>6pw1ODIfj*;zLQqnMF%HCN9-2IN1ICQIPnDb^)U>Ud1YB
zjxSX~6FVKU8;ZOA`S_DmrrcSg(tBCh>4xIY|2(VOESSALYQ)XX)vhaEkQ}$Z<f!n<
zFIf{$U4FDA>#M|uhSuQuP8}1zB<=RQ)EcUEX=-`gQH@6ryMj8s?)8|cnAUSH==5A2
zv?*=o+Ndm-Xr=|JVMWbzQr1qad7<p9{bjNIp_LcSZ#T^=n#C;mVup6l^^=Qs-}<_)
z*{4_~Rpb1hfKPMGI1HwLX<W>}SK-i7kQDtk^HGM|=J|$41k|~cC!CNnK0oo~=1lL?
z>UN^V|DBf!s<?<sM>(WkF&3WrVS%d83%lRWqHTvZFZ3za@VmE2Z1UYT$<u{T^7=39
z6v+Le#(I`ZU}BqsjMbYZb1X9tYa9!@Y@w?%L2{8t&(_YnAM(BS^Thl0pEURU^+hep
zYWDOc!E6TGa&xb!c!=LS#=0}tCBpsdCC52Za@kQ+74ACvl`8~w3F(~n-Bx*ds(`xi
z?IWN3m@;C1WErvfGOW_c5>#o4>Dnama6-un&DaoyqY_I5+;n3PW+X67UUhJtYH01x
zlvOVl?Ve!PGC`s?Cp0PL<Q9t;NzH=Y@nI5^_8egiSt8`T!mo7tT#;`nyREgI_FUT2
zSNU%1G4FYwi?-dG{y(j!h-m|VQI?}lQe&~#J{{lfOO7x3^HP7!X~B2-Yt!$1`go-;
z<dUlPq@{DhEYf3USEx;2v3T3OBl;@$b(NhCWwTw0DV}w6PrUz$4|A&Te(=2iM@#Zm
z#@0vgSndC9>Qp=8u#ofC<aysjW;QU(Y!dmn?f{eC2Y=JZsXKJcQaEHMatit+3jKQ0
z-*QSusCh*vS80j9?AwC&%qzkoh887G-iEzwR~IgJ6<TItx#%>*6^F75&9Yk;xZA#R
z+&JMzQ`J(#pt+~Kx|BXYi_4DPFiACX&VnYx9_fSe;x$S-g-R)Q5ptdPE@=oo@i~&8
zRTNyfIWp|`tq<;BzH)|KId!gii|K!z>1!`&JhQoBdAssl$-&KYZyTk@mmWJmMK&;&
zZSD=n<Cpa9m}i}pc<p(+^Pyh9*wqD+)-T!3rx{N<SF-fCPQU`z&O?iI*TyQ@u5zDz
zD`e&}L6zDhkDx`LuC7!Ou+A{duDWpO+LsfB`mZ*=uKV%p`o3@Mwpm8O&rfYS&3jPQ
zaY}E_hs5ltxaf7;m-xO(<!&!{`kHrE&?lLDTNa9(Hw@poz{gFy^}6-f)vGGD77P0w
zXy^4}dM=s0bNbI&DPG$;PFm#}ap~%09O!Y)h;=jj&>(&1(xTtXL(NVVNM)5|EdEh&
z<hIbGz!WDD*}Xert!8C5S^v=G;1@mrPAlMQT+brA|JB_`!u>u@d;j#2q<Lg(4cn%x
z-<~X%iq>rU6Pz4%y;<%rLzQjQz5c2n$uIs|R9&2!_hq4Z?Mr9#Sly_H++iO4CdNS?
ze%rTQUZa0A;N81Lr7INX8cv(Ll6~f;+sn9?Y3+_xf4*_{u1!W0OD<<Ov>J!Z`g(nG
z*q7C!4bwf_-`cd<i0{*D7eBiA^`?rW#@jwlIB%S(?i+QY-{a$S{;doy-U8?D*J#a<
zH$8rIL;TVA;lfcCwXb|*t1fN6cV$A~-xc!y)yLBReN+Bqb8>2!=-J)(zAQfX=S6gV
zb;#^#8?5dGp3HRH8F~HJwXG}MzjWWrPWZl9eE+(A-zM&>jnBV-gK5UB|5^2)q+i*F
z_qI)r^IE1pY08Eq^Fy&RB}-NQmo#R_#L0Z0zU%b=*>ip?Jal#Yzg+nG5jQDE{kP#V
z9@{LEj!J$E^m~0&?#YyV#X#=AYDThd;!0wLG8#z|?S;bPN%9ee%Fhz5f|^u*G#Z#U
zE!xnecujbg0)wH2l9EJHQbFO&f@a+n&6YDbXI*Hv?r63-(X1QMqIaUnRH8{)qDfJs
zS>cKRtEjlCMUzWKi{6T6IgcduZv_HhjJ7;0b`WoKWiL&BQQ{Ncmhqw_b4OcRh7QvM
zex?ijEC>0bid|(3rMrsj;<vX?y2O^K9&+J}8IwW>hhxWziK#`#@_t{<*#qOQ{5KIe
z$|xwjP$1q-F!zdWPEuz|LRk1E#p-mQsLT3~rV9oJRP29VwY<G+$@Z!h%d1xY=vtp%
zwTh#AokjN=jpiK^&Bimj_uS}Oy`pO7gsLcko+B1L$0B-;c2pg_(RJuV&$*25^B&!+
zJOq}P3+&41z2VWjXlw7K8C|ol_ikXXZk{H&<7?{<vD&NYwXa@ed!*OC{n3|kqHn*O
zz`iztV@3V!58I9^wX-t{e~ZX{QKEWYP3Xe5oE3sGiU--sCQ0vHIN_p^q0vI`SBi!a
zrgCDDY_Au`KG~Ka5TRNxtR^@^=-egg&t~ST7bluO6jV)gXnZcc_CmgbTb0E1Fp2W!
zb<>-F|99;%-%(}NIXU9TWP|j{mX<x%H=A8nPG)kL!mb!)s1fD0a?0%+ligRcMlN8T
z=rA>GC2Qb>snL?Gkr$>$&72xqIrX5&G}p>$R-V%i|7bQpF>R(ov-yq6{Fk!%Kc`8m
zGgwHpRB1M;lo`fFB-^`}WX&k?I9}3rvn{it&8d7wfVj{f38BMc?MfeK?0YD_L8R_u
zs&drzoG>Au2Z!53FR{g{g!sFaU2+V~UtFf>7#ia0cWmn{CIi-|ZBi#5nopCkNO53F
zTr^wyq3Q`yb<gCm`5pq%56Ul1myez~<=l;)GbejapO|z0=bY1)a}P<(y;?b!MX=|l
zr*Os1xwmf4y{*X*`M;s(@C??82J_-A=LI@Udnw8JI&)s&gQ@Q-=Y8p%_k1Sj<H&hO
zX3YO_a^CNs^N+8Xf4F1%ou6}WW%iu9(KE+9sjE<P%>h1su3D>O*&ixu4;-t#dAwHi
zR_*oceUn%A=k3V;=^?OJK}gk0zQm(l@nLkr;Rz28*O?}&8~h4Awzcj6qb1|a_5~ah
zLX%?SADcg(H1VpT?9GX?%oej5I;3P0I>Hyl6)d;A%cia>ZnfZq{kLbba*eV_PcC|R
zJU=L)>!8K7h{&l)yQYR`E=%N`mVRn#(#d7nzm_F(F3+%9p0{c$lK^YE7DG1Iw78p7
zU*23^FSVk9Yej&<indeBbF5axM=me<U$yL+<noZ4mG@t?FmEf4>qxdzYs<P(;(mRl
z|MrzR6>ST;`hRK6EUH+wtV(!FrbfY!8OM}Vj~tR&QYa<&(aoH(Zo$me<`3H)MLS{~
z%94fFuq&=%QS4~z5R7!2v+-nE(d9KP6V15;eMKW>;=*Kv3k4ewvx*Da?|M1=U0A&l
zljz~d`eU4SYXn$>CM-ymT=2ST-lvmOKW0w-_-bAMs%hbs>+@DEuevqw+0OM}cFz09
zHUEFt^3OZxeazgzK6?Y#?hQP*H}K8gz{<UmEqcS(%K0x=_AXVobY;{KoRz)PY{qMe
zO-k0A-dXf3O9?Hl+B9Y6CYFW$>#8={AKcU*S>kDIs{dbe!uBXT=AD~aPR>-mSZXUO
zed{pS*Tn*Nmsz;~&P^3qA|>g&;3Q+n)g>_j4R>5@Kd`BaaE25)x!XE&rEZw?_{yZ}
zMe@dt%R8#JMeSTsyJ~9us;M7$Z7Z0)EOXVilGn@Ar017+EoWJ{y>i!f_J!L$61FvJ
z&kGP>ZLQuBD6pe-^^VTlJ9?ydOnAMc?e>mt>z(bhSzBLEZOz&?$t%jxRN>t8<R;A-
z?Jrl(JJr^Cvu*k95@pBDg%%}BjO{FnyYnjqwsZ?@souRYd-wl{-J8ATRZD%i659_h
zY`1D$ZO^!6`>ys}1N9VUxAf1eB?Qe6uL@Z?bJo#YA!dhW*&R&fn3xy$f2yyF0>f=B
zd+*(y-m~={-^%-*$RGS%I!J(#<>5RQ>G>aaZkQ;*$i902$CLX5AMF1(djq4){4cln
z|E@mpclP$m-}@`I_jBIaKlAm0|I!D4uihb0utT7MC4hlda?e58GY35c7(E&ox}+Ir
zc5k2PuwZF<I{&W)mS(k=+84g+sJ*v*p=r%w*{;Jfw*(Zl+SR)@8JDWs>u5UXC{M4{
zc^F)`K5%jBw^{Q&Z7(cdZ1Zbo^pY)ajh8$LT5?+?j#+~tRwqvBVMqYCnR}|~`H6Cp
zg7RX)_2FGokyQfG0(!4pETSf`+<CdZ>(+9f*(`wvj(<G4qjmRzrkV}qQYU(=mp5zg
zZ2W)vc=w$hJ#Tg}3GAG-=EM}6lhbr|a!h1hVqU<>tg%IH*Q(Pi7v=1lAJx_qd8*^&
zE_TDsc{^5pnX!uF;VE9l)1gM2Z5>bVt``2fflHIoyWe<D=_ToR?CVS)#XN1^vr9^7
zk5<_Q71={sYfhiq%c5w0=+zo_m9wX<&ffIt?5z+eQfFWj)DJ5ad~Cgs#c<!vl&~W@
z%CEKS`!1((9pw6`J+JD|1}^Rc|4$$I)qS4r&iU_q_6KY@&oYtaled6c&V>L42FAG;
z7-bL1+_@mBb4b$WqDR3&;k_4FCbB--+0(yl<3z?<Lz~*yFZf?YEKHrzC;lt@t;D9i
z(K-g)iyZ%)vO2g)@Bds;6I&sZ+RMgCI#XUA{<c;9*rBBdw#r<&rt@W6%np`{sC4xm
zE#5o4xZY0}c)d{IR&lOz*2KF{wo0z7f48|k;P=(QXw~5ECBHt)Ts@?7<hPvqO<}i1
zlY}@bHn=E6HL#SgxhUx?(AcxHY4`Pho}FB^C;EI&c1xe^^uFOCaFV_72Ft=59eZ!A
zV4OCqXL9~DgDnTRYL#|%OP*S?XV>yOH&@ooSZh7gY|(D7X}j}0Zq>{%^;{%r*0kr;
z>OJ<qCq$~u+Q+%qUv2ieGi9wBLdi~NZ6D6MC3ELykMF(1!MO>7i9+^quD%}+cfL9;
znUL+%DJ+zHK>yusF0<8~pMFoR+r7Q&|L+4#3I|zb@39-+W4ArXQ7gdt_J%^vA^AB6
zxp?o(^jzm!yZqbfi+$JS*@Lq$vLC)Ry+qdQ(t%?OS@I5B);zG^v&oKU_iv4vrjvH_
z1|C_oH$g~`!C7WeNYF!G!I?Ix{Abcs&!sNj_Ug#}$AXu)E*7|HTeCbk!dhAGp?P$!
z?3=|$r9U3^<cd?;D0}FWYk`kIvdp#MBsur3cXoSm2{^hc7fPKI%4as2qdalxC&lCI
zT;_DGy%?r9FJj%rid9cLbavF&-e`V%J)!SLd+amYi5CjBZ+@PluqNT=n-9f%C)}KE
zd#c3m)UrP}OW*D0TeoWUo#(5&Rw*~$YMOW2zVTM1;R~bxwx@;Hy@+f)y`Q^%?*vI5
zFP)tWGu+eEU(eZNH0k96&)fcGddpVsJ#W*o#Ba^j+qW;;_^n*GMxf*DL*>o~K7zH%
zudlGkP4f`kd6M<9_mi`}W*YzQxaII?UY%rgVt<Iyy|2FapUR%7(3$srXH}nhy7cQB
zh2B1G-%B^_HfgWDWNtHKFWV-Ix&7<jY|_8`-0I<}eHV5s{c3YlXV`b4<Rbe^!Mul_
z>X*gmy$G9h+3@TO8=lR&f8R%T3%p)pc}rOEXv!6i|H51^C%ETMu-9IEa+$QCr^V?n
zD&NzO8u9u5HCHwJciV1Vhcv?z<D|6;oFOa+IqyCZ50BpS*wOa$)nj%Kw_5*yRsQg~
zbn<~4m2;O@`R+*DeJ^ssjhgPAQnJ$~?md~^r;sN%qbTlq%gLLw{(edMvC>`q^y)jO
z*804NPk&MP<Jr%zyA>H9p7c>&ZmT-+vcTexLblHy*?G^-p1*a^lt**lxfbP{ZO@gx
zlilGXb<B3%nws^mYWoF>KA$}z`1-+Nfrj8utTW1gHI@3kdBT#&eW>p)3#0Jt28K^B
z@7}fDUp{yKJ`Ju(%5N9EK6L5(My1#XRyhxr^*^{H-hTc1C7yE+mT~<yVtOH|C-hRT
zc3;5zqu2QUJ>cKRApGqC_qPeV_u2nBX#XdQ|3l(>VbOID^Ez}FOW0mxUVMq^>a*i9
zKAUV+?x`I6pR+jLIp&D#QKL(~26}mt{!60g{IfcEl!4=p&AIO~1!oWYs(x;gdXu{K
zW0`Ws+)3Jj@eCVX4>&Zg&Ed@n^>)xLTRefILrI~#!AIz5hLoz-<5huctTrV@bjRse
z{eAV*dZy4*uc^9$vub7vE%TnG=F4}abMyRjRhG5?Sau1o@SF3GMZ(moWWj~S4wHYS
zoDxxA8Rpl1YqRUA(2H~4I!Qg5CA>0dQ|j4yroT<nLzc?bhHYIw;fjCI8jp#R7Spf9
z?I|t}TiZ2FH0n?*ue?}58uv`y7#Z`tKM@U&HJMg$tf-0HoO;e#pMQ;vl<~P!8Q%Ol
zt(T|Ho|DRwll$2C>j9(G|0kwAaBA{CCuH%FOX2D<{YQru7f7A=-Zs^i@hyYC%+jTg
zt-{`9Y+N0xU%kHS&hOgy&-rbBR6WU^_CGFe@6Ru9a{I(L1lRri@#Xn^aR>YQoAYej
zf8Ez#^WjNwwqlh?7r#2g?_e*jMO~|wiw4ygu9_dmvg3nO{Dwytrj?!2SSq@{ud7>D
zFuU-aB|~wP;0iJRYD0(lVj&xj&CgIdy+7`T<m2UK9md_fb8aZOdMLHZaOf*e6=^ux
zIL(B|^F9m5n>DMFRtFtgS=8lg@$kfhmTrT?C2M?+H>mh2G!+C*_Pu0b!Rved)D2$U
zpi>{Rcc`5I#k)hzT#(_yd=^m#%Lk_vME);qE|BtJ2s2sPt~O0=>K3(`dP_@I%zhPp
z%6{9lDg(!WdaZ(x6;6Lw%D-6?eN;K>ldx+~s8?=R$m*I!&(~`Crf!)!Zx7eTxvr0u
zj*3<PvlP=_HD!^gZ~FPHX$CB{XHG7*ySQdkpu*O*VjE)%e|?KQx+6lz$Y`0yW}{Oc
zpI>f1cW-U5W}}O7Z|s6?S-k4kSLvjlGb)*#v*4aZ#-<0(Uu(YYaALlD*<X96w}yJq
zF^iK<U-oRiq0+NcSbsNf?#guMKEo$h9odDCy>!|V;>NSo(CDJlWSvDPlqPB9U29pW
zu^=MVwZ6wHQ**J%t5yfbjL7t;`WHK2gvP%rY!zwslRNQWo>P2j5W||yQF#jSY+1{#
zS|@c0K5Z;XnHV0jQm^~zoI7^a59hz(6!*}S-k0sQW!=@=>U&iF+)iL@$#IQtxzE%U
z&6p&-V2YU+<K!5_(<;U0P0Vr{af}uP-_`vZe#xlwU6H9>aOwITo8_mr-DrqQf4O9`
z`KC`fMI!roO!KB>zs?d$bo|NYrLg~m0bjrdwxEuM?wc2Hshpm5^kug1>0cosua8=N
zUAg>*ThFU1qedo9T?3(3<*;inSBhyY5O}&gDoVC>n%m#R)Sfx5uhO6Sd`V{Q)VXo+
zWWUJw$zdHY^w<74?)f6DtNO-a@i<Qp>lw|q(V@CJX}3Ka7r8pVXk7K*!RO+<w^B<d
zxUw$eZTr+Y?^BCYiwa+i!^unBXU&|K8n_ir*r_OfWvTPIlr)p&f@d?!lqXbd^(_5n
z)sxM(X{y%Qlj?pxK9U8^(oGW`81}y8jXSa7sP)AQ>Mcx<r%h2(;<sHc_LV!bMrX=`
z*<aGt{cm45o|DKPej~AOn?ab&-;btK7G&9L|LW_FnX*9aZHuFX*6f~BO(INJoq7Hx
zANS=DnOBv<$eF}3i{XZ1o5z(mX^{d)S7%J*<mh<h<lW6Aq+rm+)5pa0eX+pxb8Oq!
zr?RR?M4piA4eAQovDotF%3E^RGn7{z(U=$~(sg@}yV$gYZ}mA86?qqKS)Tc8ZGP?l
zLu!dVPbTO7b14iuG-+N7^PJ<Z=M1MBPky#1LacF1FvB|&r8x$dceNZmH_QIuX^W#L
zG?re-E38yzULX`Y?ZJXRFLuWZ33DcI77z8;-TI-`F)ES0dB-6!w#F3<OT@QiWaiJ6
z)17(iT9D9-5O;<}4Nf96!i7J)%~xN1f=T4a2f?sAI$Z0MSVjJc&;NJSgnPBb35f{?
zT|Q~K)eE)m1W$S;&zZ}rDWm(!YN=KzgPfj2^TniVDrM6cPYD#Vd=X20)utx>XoAtp
zSv>r<&6CePREc|Kk}><1h`NegplMoSO!hMuHtk8S-G@Ib6;)j^*picEDeiiqzv2kr
z#|cL@*e5RjFMIbvm-(7wyi@1u$iB_cNfzaByk&Tybk$+5Dv6^ryPupYcbIkNm%`hc
zC9)@4<(9Taq&`X$i9YIl?V;@3kTWX%XPh#%HTPZO%a|f?qQz;8!sU58f|LU<G?jT+
zb{)I+SnRC=Yw?O_{QR5=P1`swvWR9*;r)47PW;gQkbD!5Ek`=4w_AAyo%pagzqhpQ
zz-q<B50^p@WKQ)ueDU(#3zxc%PgV5kT`6OAcUQti(;w;uvRvX`=0{FW$>aXC#~|QE
z=cXGsJ)Jhr<d*!xG-oGArhy9+U)TG{J)6H|9Ld%aGZ9ZxYAX_`-nwRm?wy2oqaBW2
zu^$T4)U^WFa5l)TzR;oO@}F_DM@D0sKbsc2BbV*gn&VcB4dk^K3C&eM?_>2((p78o
zg3iz_Tn(n84Pq<_`#<j0($r{W(tUMtzed1z)skiJtdf3d86KIL8L-07XJ+|zC1zK}
zGlr+O|NFRj^H)LZx;dZN&O2@WKJV1AY0pkYG#r}K_-M%q9lj0o>n3lnO<TJ!b>-~Z
ztKHKIA6~d>Q*tJ)!9wc7AA<^>I`!Q#?%{_HJTqlVI6HNd*zB7whFv>;c8bOtXSMro
zy&S#im4n$uwnR&H^={UF|9x(X?;izjo)X(Dc_)F*_{uWwt%l8-YSuT+Ty*Rf_c*Cr
zce1d!BrRL~!eiYfGp&g!%f((xG;16a*#Cd=m&IZRxtq>d?d)1~)k(x$n7Ncet~LCT
z2$!h-j*1LHR+blACeIXbRuSI9rNNOP`!9l(g~3+g>N74*+gGg`aVNQEPn{+k$Y3e5
zNPFJgKZ|6SX1G_@r}fXzU17QAgi4wCqPez4S6yzr?2}^6^l|k1I%C?i<ja~{zS>%C
zcr71%P3%=!sY0r;dYbqh2fsHfHhG+UdylIxp6&C)`npWvNzTh>hW!;Tf3{?<*uFV}
zFFn`qk!$0YmFp6!R-YsE+jV}Cbl0?hznA#3Ofg_RTN<}(b=kITr*oo3Tyol3?B-MK
zPNY>d?>buhPHzU|3EOE>;S<y*RUIf<@$20A3m?`B1|Ildrm*OOlghL=PoCae6UhHu
z()v~5)9|?^=^K`&_$^+*85%ih#pFfpVn?oTp7bE@t)QZoZcx_qB@#ADSIqxUR$dt8
zyTnLowp51>cL=}RZRKN6L=~L(WXDYrsd`_XucP&tOJ+0cgalU3i2`O9WHmMRw62gA
zlj7iL+4ICeIy8B=*f0D02NX*(dH7mPw=!{YX!GnWnjH0;Gfui|PZw{zbl24g8O>X~
zoZdZBUDk)9mft_g=`6`-prl+Lyv8u0mElZhe_^Y2M6a^U_CjOE!f2W38v@}98+LQ_
zF*)^bHCX47Fh43`XWItJ85?zH3R$*0G*}(xt4e6~Sz&I;tjECB|9}2MmAAp0zdLX~
zu$tiE!0_d<MDt~qCzqV&UG8t);QVSeyM@Tk8%3(~UUt?oXmRXV(sgr3-(rbPC;N96
z^9iqXiT}BHd1wFI6B68#0`EHdFI#o5UTiffP~h#wrHhnzt$(b?{z_oY2959xbMqI|
zqqgwgzQAkskU?*OlHQDl3lkTvI<Q_+$bj+kwB#+z)?J#mN@32y!YOWvZd`vj8I_mY
zAKYD{HTm>Onad|TtuDG92~aP!l#N=fel2n??}1Lv2OACxJ1)4ut+-Qm!3Nobf*S?0
z&4M)MC4Q0F&mgi_Ly?71;c>^RIR$g%wD-!rHY^n0Ui-5(CG$jglhcGq&Uq1>WdX+j
z0|oTzZ!CGcc<&2=`3*ix1233QnPB%gg7xiejy{3T={IKj1n5q6u!}me&->Jg1%(~|
zGv_tV@H9Lzb+XUq&==;njW^jPNvW*ZY&XmJ#jL4fE7|!I`CcXpPW!Rgu4}f}MTIMy
z6ao^v-|kX~R@7&=QD#}vwL_U%Pvi7v6YsJX*{vzwHzqdeCa|tq*~(rrLtL6Qt7hZ#
z3*P?^9pEvUz3G%<iQt)CNz=D|Qru;<DB9ZbkO_~@M^3dYnVSW@LOXajAL+bjv3t*f
z$)S@SWOKI7VVIJot?(prb5rpIUO_&ALgwHK8Tsf#sxlpB1y1W1Pnh1&Z&uYGb%BM&
zW8D`C=DA6S|J&VicA2uTw4%Rm0lW7r!O%)cMi0#wQmt<sPCg2nStp?7`>NIU%~{>%
z28|0w-y=Hio?(gFAke)>va)%{qe=X7o%0=xIA^N#-(ceZn4$CYgn++H|9g#De~$Qn
z&sl$?%d6Lm(^e_qR*mbqO)J6*n3*;oty{?(_oRuh;{xZ@3tX)iG+#0CTn&gj6Tp`{
zw`}&rz1nW+HM?_fsjX4EC}JBZnrbbax_js2<w4rZnYAaMu$WwS$GYOw*0Nhomo6NW
zo2xGThx4d)_l<)WMVK}QNzbW#BWCz&y~9SE=1;p<W;CA{>^QGEEyt2?N%k}*t>&WQ
z<{JjA*Nq$XBs$CwtXN(s_kYF?xw#^({VttV2M#?kIvml-(q&-j?zwbcXJ^!o;8)gN
zpAy*nf1G`3z2mv|s;G>W9y>jced?T}H6?8&`=rZE?y*u)9-Df8>{*o5={vJCD3JfT
z<Ea@Z`gu0V`F|C$Uo}g7p=f{Af}K34nPpDvdwOrtIlcAB)om>SRx=mH?!6E{cagT0
z4daXk-wECu5+xQmOv(`M+4N%7!iK;D5n?QV*6%HuR(I3kq@a($onv`N=WMRfKDuOU
zVXf?(gkHt3>kq8pO*!2w`9_;@gOZ@Ip;GAjMy0Ldzm?1crnNd;GhMuHa^;#<55eS}
ztw9Ei`WBbNSLj$J$<^POp~WNG-rVHz;L`txv+PWz=La6(^Iy{Hp`rTs4ZlSQr@2Yj
z{Xg<|=S1|L?Q`!mx3hK8jppt?!(!(uYTTpJEyy1rBGt_pFyU>ae+;(^Q{>+e<u67m
z3m5Ru-KyZ5vulwxXP8dF2IFb0t%r_l8bqrw$Fu}+9_3y4mT5ifZJxhTe5}!%Om7SQ
zxTWnC%@`4FEZec=Xm`^KuCG<r=c*?)HBP#5VtG@d%$d)(^&-WT5|8alU_GQ6qh%T6
ze(*HY;)~qfJr@gka_<~HbF=sSN4Na1omQNa1?F7hwmyFTZ|A|_<8q<L^Huqd7;_sX
z*r=*@y9Fv5GEF%$N4D+b@dF<74!+><`^x!`duDPa>;G-4mm3S3tr@l@z3k`O*#G~;
z+WD&lf`sPZ6W-~W$o^V;RoxBesjJ$<SGG^I5a`&k@<YkF8HJm4POw~Z>I~%C$!|65
z(y57^BB4_|PAzb})yZ>frT5g9l}n>O2t-T`WjPb&x8drRGgleEh2D7(T4wTq$t@vn
z&my*CKJ0A?I=jwr9`n&W!O0TH@nnZ|&C_LynSpgp7X|#TCrW#76zSuieN`wdabwCu
zp=aKkIVR^$O_XzT+%&`V=3B+0RG;}P61SYtuKCEB)P1H-@LJ4~&ho2^_H8(L;1H*i
zAs-)GSMAzKJaVmnT~y{?@c*!2Lep1<7SRU269I+AbIKm9HRI}9YH{R$s0aU~=B5`i
z0#jZ%H!CmwoyfJYi|KX7{+7>6X0Da^d%5MKWOL~R<Ev4-9I{L<`|yjO^fxWy|NCKz
z)|V+C3|uD}u3Zp#`ChLw^VyV@N4adGcv({~Fs<%ow@u(SP34k}-tsgm;pv5SzNs5s
zqtBRJj(pd!oi%ye-lzJ<I-}0TaLRT>$KAg3;pol|#b*-^Y<fMLEvMHyG0mnhE$y4|
zUV)7neQ6IBgtDK>Dn(lFevlT{5V%^kRi=sO1n+eTle;U6mTOAvDLrRe!V^>y;@+e<
z{kmcL<`s_ifd{U!a;3`6IaO_@zU<OYk15?u51n?}`TP!H&0^io8mGr_`N|?WH^Wo^
z>#r^`U#fd6n*H$)8^#J1-Guv(J0xc=VXX-^GE!bOD`D}p6MGwDLRb9}`60?Z^9TRL
zB(cpNi<=$Rw0Jz!&1#MkeK23`L5%N%g>et!c*8i|UU7tFHO@>}_OFxOtoxqArPM20
z48>C_LmX<(yi(2Gvvc;fT@i^v8m&@uxEcP*Y!++SoF}p6SkOa>*J5JcyC(?joENtA
zLYVBK*9YGu=Gz7di3IJwlz8|+V%1XK1svOMtl+)tuua)-(mpA}v|2IegO05ekInfZ
zlUTU@fWo}Ej%nQ&HR?5P=)GuCHEBMg(Hd2-pvzJ>aF3-%hs#`<)~*hYR_>ikBid(P
za+>BT_l4uY|DUmob<Rb8(3s`-@#c<`5@tbHr&J2Qyt(*?(5@AwN88^_h={qsy7yLe
zi(~Aas7<2L24SfY@ABS-rGET(>)p1epN~C_4SOGNcj;?fbnLgz@5i=oY0X;i#IyR^
znIjy1b&Y9h$GB}At^fSuWtirsxg>3`$AOJko*h(-IrOnROoR30PHVoFJ3Q9uY|6r0
zez>u(Hs#D-e(mMOqLSyD8nHt4>cZ{mXU<OF6mfd9fZW3f>l5F&g>S1fT%LRC@Wa|Y
z-E}?tpU>FnRGGV}VgL0ETSg1!E3Y>uH)g&t4E;V)GVD{og=%Ka<da(?J}s}h@5vbw
z*ec+8+JVWZgUKuLw!`TK*RKBGs`83CteAD44@X`x=d`)M%3dw|#%Y=VVEM9Q>+QvI
zZtnjl2MJy7-n$@8=&}-%dh6zMtDNpnW<1WBAj9SUUj0GC>kVzM*KAxUc`dt4%W*-$
z+^E@GkBa2Xzwtp;sNvE*E`_o)wVg^5S9-OYt}oP>#4{zQwK8X^fY{p&cUNC?inCPV
z-MFq*F1IlGY}kskE3B`YnI3&}H^Tq#k);*=t4^BvMfU%*@pDgm;;N^nW1nK`o4f9|
zm2clW=D#Vgw;c_PyS`cXeMH{Vu(s5){?re3<y%stE9a+f+giT;Y--#$&f4wYxBq?m
z_1<@@kEu&9e6KnF{nNTsgRqJpb@`Rkm}*rt{%0h|)|`r|-B_;Pv`F$xr{9eqTi7N$
zu79Q^_*0oNTQPHcgV1xX>37(<XH+d)c=ce99B*Oiye0v0hVp$sOH{6%Y~WGI{J2SC
zYetCm9ZMcgNxqKh+ZxX8aZt?I&HAbTm{)Z{tCD`Hxk21_*Ke*~oqi`jxlO+RBlo2?
z|5UHpMw<_0a&(0X@9SQ}`pVe;b8++o=69hx^%Km`2Q2k_wParL&4Zu!mQS<&|2y|c
zqJrLQXGso*e{)Z5R@-v=Ober%hy5RsSzk^r@8a8&Eh%<A@5ol=(&-gHzZ^K0xIgaK
zp#|Ye*N(F7n{)rfs*GjFBxjXI{6C|X*Y`Jdi{~EsEeGt=%Vhp*or&yVz3}_(lmMSw
z-HLUK?soP*<otH($g&f=p0R(ou?p~C_b%Yw^fh~h%R=7py^Z=*SN@*u{RY|h70dJM
z)$1$O>+9}6{dSDAt#8BE|D1LF;adX!FHFcdyE64o$8GVpg0A>Ss%#v3F{d~*GCn9Q
z*JI}udUGf<QN4-DOY+T@4@n2xCAlXlrcMZv4$AS^nq<1kceUA^9XXFS9evuSsb}dU
zxS`eAQID-OPjEr-a{mPqrb$OmEPOs~hFNmnOrwL#f*HM7eL8j>d~kU+Q+JVVL|Vf2
zR>j*K38jTo+!-@iOKdW>CERMxaG!WH#rTb5f7$nKS*i;{bs4wb>q~sj|2Xk~&$pVL
zMWLsBwWGJzT>Ts#>0sa7Zk(<+BjP~%>1q01`~FDkXI<IGa-)l9hrZ_og_Bbrx}+&g
zzrCxl{_WY{<(zj8?&o_R^vB|c5EF-%{*9OepI>m#7oS>p(W<2R%2wke9kshZzgxMw
z`#6V5uCx(%?q*f55}TSU?;ppjgnE^&zVY+>WTnqRY7H?A7IFTpq8S0crlLz0Fz^^j
z$Omz|N<0i@=M0^o9W(Li^`Z-s?bE9YFS_5?$l^>0cT_zj8kX{v<=Hf~tzRxjI)0V8
zGPC#V7bS+&Z95_&?f0JPnAyuNy7R(B7rw|8{i#=-C8yZfom_0cPbchYQe;%-Q~hrx
znccSk6S{6jc%^O&^peb&7RV-fkk!4Zuy9vqjJWERo3897yCRp*D}R@{e6H+|l>!Ty
z1Xj(Nt1NZpU`LbKlNAg-Qe2vGlk`$nE^V_r<+^Ml7i-8eF4wN*E1HaMX)d$Qy>j2_
zmFUw|t1N$Rx!`ZfshZ_9OU!G63wvP6L|4{8lQ%9$cd0kHpH%986Tvhma=N#H@5ZCP
zrguvM*WNt0>V+%or7H<O5B&r$2R{ueUANQ!>6c5bA3k2o+MY5&a8l&jHAjza`q8GE
z<oKhiMPE|u)b=-y(O<6Ka!uY~G%KybGICbNjNXc<2$AY*8}!#)y%eR}u<=lQoz~rN
zSM!wrKa)PCzWR-H(f_@rD`bi<T-7vQbdxda!kGuD<#TsGEc|VJ!SV6c)!|O!S5~i>
zB^Mi4Gr#TXJLf>-)$*Q8w|u%(&y=Mbyq!(9V!>s0+Zzf^sromrUXNS7w^iiYwzXNi
zCG@UnuRplam1&PmZnnT0x8hc%@IUJ$BCVN3@&vSJC0u*rYGS76FxlAco@DRB#G0j3
z|0TTSo)%vvsrF*q4kfFU33DseW@k@xkd`c9o1VU)<WOJweAOurpS`U*wa<D%iIw^0
zO9$hlvH}a2|9Y{=Q9ZV1%@(u##LtP=@3((={B^mY<c(sd61P{ms;eu{PrrV_d2WZ-
zMAr<ll#4auf1}>GedPWyVbh^Dk=(8SFP%G*(5TnCUuauFkeQy$Qic0b9SoB<Em&@%
zsvQ`3eJ2;w!8I(dfm;~6&WQ<0Z<=v5{;Off0k@CS7A;w-_mA^*;}@q5&4n2TtdfVd
zB-|EDiakx7UKn%u!z9D%4a;O@nteT-uE|$EUDlx*Eh!YKHb+&cz4K=0%=2Yxu5SX~
zr%Up5pIdigvqsuQ>wAI^97~SvoqzGI(%Z%U_h($2nw*%e$W?jb)3fxwzn>hpkKUc!
zu%>kSmQq#6l+TMVTyk5u%`=gy#U#aVhuN{#AFbhgvM&1<G?^Q4ZKw;?@#<umwy1vt
zhmY*z3-NCj?d*QCLLqR|nQ3Ormxgv}`fL-Js~weS_21-Tk?+37xpOR3)$RUf1~hz{
z{I2Tz>DiL2EXyy2{@9|bom;ErANx||)1s528E#rS%U3-!4Gi^N8asLRE*lT?$;;1n
zr*MVbj!6sD5j{R<i+=UHoc^kNH_dx3bn`AGC+C)Jozt=8yuyp-4u)TvYmx+C*S<Ga
zK3HZXyyoG1jeoD@Kfjo4X7Befnrji~PPq=fH91>)|1Iv%-z9WjcZ#E#+(+-d7bc6D
zWo%MdATvjedD%8!F_wZ;Omhw@HBU6`X-X7V_T_UttbA?9v&qsbuCKa0jPA_PN%iwp
z5-!f%&VO`G&!v?&#g;b<zWewpBQHW&Y_ITky|_(l(rjNlUaFXARQ7*q_$BRImv$7n
zeR!KYH*ZSvr8k+1Mo$F$qkf-{omVEvKGQd_sYd$tvH%ZNjWQ-LgC*C#sZHzNa%f_n
z60_fd6MOq6JXOkCIxjX#%iy3Si$SZkOuLz5+quSb#mQMcFSVjiH$Hi%Ez=>aSA5#~
z7N>Ffp(m;Z>FO(QKRjb4-qmMWo)o>I;j+`7NlUdA4y(<nNma`0c<0#q{EWA%|E#Ob
z@s>y3uW#?#p7}6ax$69GbEYzms+*T*r~g!&_jaY>x;t9QxtBuP>e#Yv8K*8@|A+I!
z8@VO>r7xcJy5TC^XfdVXjnMq<J&)FMdcWuw?ec1t_~Fny%e#qnVw0nT&Qz|ZDLelc
zD=&=yV0viBhiny<n2von7Tscg=-QfTIm0pT3fKRV90P^Ly}3OaPwv`mO)bn)n$*80
zWw-Cg)H@FsiNAl{c6`b~o|v8QrtHsR51-(wlsdyGr`7G~1~)<ds3+HE9CkW+rC_Rd
zPU+_RC5r^+nk;dS-D~Xh*dwmxQ>D$;B=7j&r!F{(i%m$ej#l`dF?qY>rj0$ZZ$#u)
zw=8oBW5}B@+fncCy{881(H50{HsLN&o2O;yD?E9!g8R|QeLs%sFP`Bb*j#+u^6AQJ
z2k$)9XZ?En=2yS#2dq0{|KE{GJJ)rC*>Up5hhcAwT(;kgkkjA1{LJb!VV3;sq7Iwi
zZrk(LZ2Fu>PyVknSy^e5Q&)Xw(xJ<i6T|(AC*M{-&MdmT>w6~SgD2-XCO(#Yu59}4
z%=ysbx$|OpCrx-OShN4P?lf1SfAOuxHHxgND-LTb6@GhIUB|T3;@HjB!k)!$6B>1+
zn`R_UE$rQKSlV<agTtCnQ=~7bo4EN-|2iwI+2r+-Y{hPOr=l#zBR>>u)m@`rI8C4R
zt~+7fsh3$XULi&vjVH96&&`=+vwUU!inOrjd|^9_o*A8A%M)E!9C50Ey?LLKbM{{6
zlEUU(>kUUQ)$J7hKBGr!X8fdg8KzO;54rT44y=0r`1kJG#H;JhU%Q!F>EC;|vG@AX
z`_a+-8`T3AslM1B6z9lfW%BGl=jpY~S5uffYPTdi>t`~0{OXgaTxqY%EwZPQRpTSe
z$-CusN+~<L;#dAVoyjjG6mhv~`=O(2&z#e(ndAA&>B=0@U8fba{@N`QS;OvCdT{%)
zN5SO^H<s#M%sCn7{P4-m=!1FN7oQem650{*$lO`-*jdrtk5oTyJnnJn(fYWV$Ioud
zW#?jG<WcJRad+Q^;P?Mu-krPf<hK?44o7)^FFk+oukeN*_P;SQ>N*mPCs`PC?|eVw
z<!=$O@aB>21@m_G$S_N~w7Ljzt~z~6)A5ui1DD<j*1qGct%BhvygepNdobZzRE<E)
zg`+(SFSu`MT=}=vPxMIlhE^6G=}=oip}ON0IsawdV#IT7k9DpUeb5<OQY$&N@}{g#
zkmj93;TNafOZ2!^>nh!IZq^j1OH-Ya?=Z&N&Ya;oGb~k<r;lAi_K1v?yX-a@xgypt
zt_y1(&aC<n+rH%D!-qF7$;f?4;dXi0Ggo(!%Z7=cZ$t-LD9OCMzgP2K#mS4UKSCQ)
ze6R7GxS?{ZMdp%X<GmNQ)0ig2HM>lk8XMI%;poy*u|I3hy^>*heKP#*-uW>SEDw6b
z;yOIzHDonC9EuWRcP$LwbaVE{#_MLO-t%0dbsI(R-3XG*l4q}(xq0WIEs`ypr!p<R
zamlsh;MS=Ja}#GX%xJo!+kW$=n8-TLX^F}rc~ZYGvhNP{+Fkqqws9hhiRxq9UD19P
z!W%`U9fFb~|K66G7Owiqi}6*si%G)Ig4RPj&)3_|aqRWD5+marso)kC;jhx+ntH7{
zAYj)7g{(OXxTdLa*CpLp85?nB-nqLE_HyuPA8F<I7;}5&(ftxE-Ybt@6rJmF<%!w`
zM**oLSsNS$lO~81-nEqT{hJ%5m?s*n$1WAN(7$G?ed5hzkEKgP6=xJihV5dUYa3nJ
z8Jo(;I<3%ACUK@q%yrorGt3St>+MsGnKwgG@A&E1B!zhrYgByyzg)O5;9$kYh)V*G
zE(xeyIC0-u;7FpCzxWlm_AT9)edhTTD9NSHbD7DT6uV5(GVD~R+{?^m3?BavXtHtC
zSWNOiackLKH`T4KG3VsEXP)SKkf1#$Z1usTEEA@D=Lp#2I&tZPXMPXT%r{N{TiH5m
z4QKwMWNW_`mnlq=F-x}?s<-*f^j~t)WddXS4d2agkGvL;w^DkZwCj*`h1Zt5=ce1L
zmmTaY3yGZj(A7pPCH%yErAzI<|N3n3?f?5X@}HpEg&MK=Td9ExDxY75|14?rn{oaZ
zM~1^q(PX{j&I>x3?Cv-$b^96m<f!S5+(?DiC~tNx*0#h5&ZK)Ru`L}N@3b#nAhcou
z(-e&k-V+(CRN_p$yIB3Q?j-1Ki;ZnLD)HsPg{fhe3@3?95Gk=*D&nQ6SSLC0o7c3N
zj8R4Z&xcQa8r~wg?QWB!qL1W+b0<@lYOi=At-xDY(7r5kmgIuD;p<LH?`WL*ljWE0
z%cNNmg%KCCq|VNIsu*`ivF_}t^qpSYUL1>MN}A2PC|Yi@<B9oB4{l04RCmqtN(y3e
zj9^JRcj~cZiqT(}1(}bo{qcD|vG=*DnBu}D&9Y~jvyudtDP2$T(6;J!oWW#SbSvbb
zr*4(lo2{8EGXtCoUA0>{SO0a5yrMQOQEczTXx*l(>t0Q_&}-LO>A%JCdZ}-8anw=<
z-T1aChrY*3OFCq8C(SOAl`dMBoX6E586iJYDR_YwOW9t(xh5=&uK2zEacS<E$CjzV
z<%QGD_O9Oj>+Hv)|Br{<4QpjvvPeLyI&5O~lW<+l7ncOM0wgj%usr+aqH@IYMWm;O
zTiTq4k`&k8?juc4P8MGA*fi(V)Ra!GgPrp-nq>l}FMFco*-^zJsQ%QQ{jQwG#-^?(
zDUoYLqMvA}NgnB)c0r9*Rb#7D|Exr-Us>%snp0z?a&vAj6=aFn#WY2(clDB!x*1Q!
zy5#ScIJQ+h%{9=J(qqrPdnlsqaZ!x4_=;3<4HpY3-AQt(elGDPNfQbZbg!01UHN!v
zt;n<JW6L8;Md$8ii55A*`Bb+iac$7Qx4}`5?(Ccx|1GtiNiIrv)g!Jat2v&`ewn()
zW8RWyfmsFj+ON7S*1eWycP**%|FZN|362eEN?X07rmeJ}nq7EJC3?rH^)H@hZCkeH
zSAg}IWsNxpCAK{-UE1ytkvdCeb)lAe^o~Q*R<ch!8KkVp{cazBVI<e)m5Or$UKhE%
z{%Yync4N{q%h?unk=hUVmM^@$L_==n5x)fjUF#LS3IylBS&|)iL-aROLc|*NtTXD5
zriR<z^xFIM{+B1W19s$xJIMWKn&B*S_Uj$V#z$uk{L}pMMZ8&UP06(Qa37WJ33pmw
z^#?O}gkEsr5Ie$Ax1wFwrK3%Q%Zu?v(VJuQ9(9Jf2KVIDA4_*{@rm>ayU-)xe#133
zOd@82YwxZreR3`5@){+#F`4<f%!r*X`-SWLrvIKZ1oM-R_$JHvN}A-}%9?+^Az7kK
zHP`cY__~w+OZuWUQ`bvGUbA!$;k|8nj*};kU3inG%%((%yNR545^D~8j5wM3Wak}!
zpN^EbM{*ymUS^Y5UJ|;(b;>5r)&+Wx4rO(@?wPo0@AJB6hV@p@j=HW=5kJ%6>ajFw
z-KorH%cech402U(R?C>DwA3&uTIfhYRfqor-}R?Xzgo5I^}V}#MM2>?ht_OkYcJ?{
zes|sGqSqT_ULUno-hA_5pKSESOKi&}b*H~<xhd*mkrlntPx4*O)Vpgu&mDOC`i0Q?
zm!5@;*YywH|KaDgl0j>!$SI*mi!Iw{+iuI=-gw(`v#~ATJO7yfGV<%6NnXrVIp<#|
z81qd*MOpmRr#BA|DaIW7C+;G|9mP7+^%9Glua0xLtA|rS#F{xSn`OB2)^u_?98W4?
z-OYXLl=#V0ZhI44r0*m|oiK1aB60fI_2YYQXfXK1y(!2H<kk$x%*(kMVpf=MV|Yz%
zrRhDU+!+t8PYK=qvtr>6p0yID*SuJ|*X2%fy+7+2%jBJ%WgVwNZ}!|Qk`0S5tv(b#
zdsq6Kk6hPXuC=;!ta*{kcF#5Mh3=QNrVmyXR7b@<6gxY6=hm8{uVMa+YIWb4^BFPy
zXNmV+*AQ@P%atRyV_q%t4B+UNOBXxBCD-zzZW*U`Ly9@a^&9+WIlEuYT;~7af6E??
z_PjIQXIE{BY`y06A?eMnLvL=aYV>F{z9+W%FRPL8l<VK}!kyOonaCGCJGbWNG(FW*
z)p47Dd<whzQEb;+DXWWL<Gvd+tzR9^e0I;i_Z3^-pa1dx&oSfblg5uvvH#<gDZf5<
zF_TX1&-n`#9vs}(&1j^MHASA~uz$^q_g+3LSj|gVt+#kIWpo6ZbS_)qIOmaTdWm{4
ztJk^$cXPc>Puv}6m}K=xy*YLCo|*;Uasi*}vmz51rz#cC5Ng?I*LF)#Gx`HpaV?j`
zf0v-D6Vtj1mrt#jy!6=2I?3OfinTj-gvsa4KBOvJ%piJ4FyX4f4TdL?*H86rPXF@N
zOToQAcJlvC3BP#b%D-2=UH@l^_>I5(i^Zf9_{)3}R!wb=aLvw2U-y8eq-gs4@)ueT
zf^9`Y8V*bnC!D6N&DzcwQG4|ByH~CuL2HgZU9I;c^1}vx#>C7*#%%u8vrX>Lzrwrs
z>o&bqu{_yd28G53Q<t@#h+TW+Vv^AG&ucip8Fhs&dRnwP+RDD``yw}srIW4Nt*okV
zTU744w)L~M{Y<^wIm+jMsEW!QWQ?dYRH|LO+=c6l^QP}r)*tTf{qAJ_rH0+Qwp-&^
z(i@KKH#4Sdxasfa)US4&lN7q3t$q70cWdr9ttI;(JUMeI#7C|o?oCWDlb_E1W${-s
zHobWv5XZYlb!t(gtB30UuTk62+|-l2cGOcuGrl54wj$!;q)m!ZyBakERHsUINta|R
zuU>vft^A-^<r~xTL)+sdPaNrWR}1*_!%Mi*@tW7evV+GqfBnJuZf(w<6W!Yr^n{AO
z`|PeLjSIE@F2lWEJ87|0asvN15g+f{S8O!3%97=eoNM2C-gz5~N<#|!-kI7C40AGg
z^@{~o_wC&tD_h2r{!u<=)2TU^6C4+?WL?*C_c*xb?WH$={NA&Ef3N9&J%74b<xEqy
z=O&ShB`uz(NiYA}C0{zVNispTM0?MJXMfMzm@B{jn|14QO;mYn)`KcnxqzO<YaYDc
zcBJ>w`S-Po?_%n&JH=^Ne~)bRymQd*|B7|fL$lYlY^dMoY%iAld^?A+S$*-F^hN*b
z*W1nDb`bJ#Xs8zJTEFDvJW<UVTYRn>aN6$ZJY(`IQCgfs`Lx4V120+Wce~GDYL`w~
z_<!x?&4E^-dc2dl4&1QZ7`5+@)b6UD%L~7CPgE#x@6A?vcF>*o^O55-_s?IYSIe?$
zB75FGwab&9->jEZlRx%0&OGZ3^BjkW4`;2#9DF7kIdVES1&c_CY$#}IYGsnKi_vgo
zaP1Zl*6>)QkbI0mYB`6-=Yl7P!jxR6DF!!jb27@Pc&RwIg-8Z*pFDD9=j3$%`PTXO
zzU*A-s~yeccWTO^L{;%=vSLLq54bj5Ucv1>Mf0P;|1`;PWy>jrkF(ayHd5VLFhT3<
z(?grncNe?}+IHo}L|LDmnxUnu8g?yPRaJWVn(oc%A?qhq?!JEI+FF~1UM*5T6;n^!
z8>U~ADc;0zYJztAwV#$Ina8b{`#s$>%XGS~wczm{-NJ8ax9zfDKlxHnerDo+%fDih
zH9Ky%R!_9wb2FOZv!DHUx&5}W#w)MNEb}?BFLw9AM%Jkw9P;}X3AI00T-HD1U;3o8
z`-{J;Ty@h}JtHUSR=h@tQhm@b)(z{Y{$~<-uxt*?jVJ00{gj*!F39oJQZq7hc)8`c
z+@S(R^D`1_mQ1<#!>eIa-R}#%dgqKz>ux-8Dtk##Orh(NO<^jTE|NR{zgW~QaC%1W
znge^ZRF4X}OYx|_%6Xlnt}32nq^7Rg^l9OZB}aJ8UafN3blhm!gGHzHwtooPHa9MM
zg%9I78{=)J%X5sij09c_F2A$OVGXy|tRCT|{Ij=Q-0^+FAJ3C_?PM~yzX(X`Q+KVL
z{Xvn1w@1tR1J9x>L9J?C9ZHO5t1^^XRbGTNGO&DDaV60EsLzS*9y89WF5t;|d~xmv
z?vTxaLNy+aXM3DhiE^HfSfzP#$0r@#iLZn{-7soVS$%WktpFdFgRhicYm3d)T`an{
zE@aab0dcRjC;3iQIIFBmx$<i5nw~S<?u=_>a}0PF-_h}&_ipRgZ8fEj!sF+z@ai_@
zJM`a2`^>6ybBxZexUuHc*;Pf+x@vwZ%PJJz*6yj;<s@r*L2dsG)}I&781!*ooGkD3
zCH}U?v%E{3`;<KxKWB*X?`^#{Swrc&*rS=3=hq&~IsKzz7XQjWVc$A?0%h)GUXPJq
z)Zb(5*(spGx$Qym^35uX3XkPR?^M0J@sjRSjfqFhE{R$f8NE4glou#=@4&gM8>U3Z
zeVM>*{$tt+SMwbcHuD&>@z=cOF<<=s{0x!wKLzu*=$XB&PmEeo)M@u6=(B#Z*AMmc
z%NM`e+90cDyYF(ZuwJRXdi>0t*8^{LKi)f~#!ua)d~w$HIYs$?4EO5mSDc=~?X{w!
zNyXrvQlR;%6SmLK|L1N}4)I(Y#}M@V-pbz6*;Szo(+}|5UAq?jJYQo;->ReGOPZc8
zP*a@zcWu<J6Dz`0Gv>G6?+x3g;vKd3$tPv8i)(p`4+e5B+B|jR6b3Iv<?gUu6Q*vq
zaMKP<e2_dv#Ie5fn6Zq?G=C4{xyKsYYGiq)vj?u2e0<GP(>0p1xpHjImo49DnZDt?
zcp}_rQh}1Xs$yS(SJMhVmW%omC-oikIyUFYf;SzidwiqRROMz~%t;G1>G!XBCa25P
z8_anjiD|?4$-1d&S&t`QD0#!Iz##ci<=@Y{m&KUepUzO}V>&REA#0+@SKT9`OKuzx
z>pG+x{=iqIY=Ug>o=>|M?O<{=%=&-$Ucr>fJ48>4_RE~y>h-Zp@Tx{^%nN}DJt_=5
zy@m#12T$#M@<ilF&Qf>nm`mEGkJs~Zbxqq?b3*6$6mP}{L1xM~?>u7>n6#vk&o`y2
zV^)%d+QbK$(+f{soG8bB_Jrj}vzAYD7Yels-Tvv9ysPlayRxPgSNFWBc*r=F>4Iis
z--1&il93(~szPp+zkHQ8d2#w>Xb8?Y)1qav(0}cU%GH+N+?BQ-5?$Am)E|E}m`ggN
zRolx@IOA@H%Gci$|K>e0kete-_W6;Mbjla|(%z#yewv+KK2_8BG(5FKPRg_`+cb4*
zTIw>XJqy^vtGwClG#9AtIm;2eW7DzL3(Nht#60-VQZ~1F=d&5pCZBjApuDSQmZlc>
z;#IuT&kVYKdD?9cN*otkZq>>0(jwJjN@O>ObHm#uDYsLSqN=$J-UUot7q%uzZ1<0&
z^Hdc$VpT$ge*7|?f5OWzMTDXDjV40_(-rk+v(L%Sz7_Fn0@J4G4N*Qhj$OJDVNPkz
zeY!O(b$kE4(<!^5vWFpj+JVleI@Q+GJ-n>8Ow8a|nP<3EGxPAIuyvOTA_AAXG*q41
zp~P2|HFw#CoMOu}8s;<CfAtO7|HV6z&)k?>I@U1V!ukH2BLBO**___ju6AxTExRdm
zvUtfBY4w+VyI*~k{V?msZoNRRkk>KK-9J?CZ9j0?tflqYyeO~#&eIRL>~!?hDa{q0
zGGF0}*w+=V)?!r=nmLKlCuT|S`eL*_Xv1r5yH(x0)Lw|p=?a?aFIecayCG@Xn~>(+
zDJQwtJ5A;JrLtrD&Yra$UcQT-NtoKoo%OeC4%xav&Bs-5*^_lqXRgP5*SfAcmuESb
z!N*&#R+|ZFA8+coru^3RpwP{;(^)^r7-$Qw+wOgHrl60szVF3#A$Ly9-pO%8;e%*T
z<J{tvObO{{u57dVVtG=slIiAIhFJ@IColGsS-rqo?@P}%jiX{q0z}*ozZQFbYu=uy
zZm}OZw`Oj3)zZ%2ofOKqR!e_fVgdW{9d|B#oA#|KbxP??tH^Cu*S5U#oc8FkO6-5-
z@LdOdK0lW`rlxjy$A+WVgd>*z%vMQ#>D+9jU$R4T<zAJI9glN3tlUj)|D;*(|Lk+s
zN$A6bf@W*AJnvKATzG}wSL&<V_V6WN=Fzl1_c8OC^#)HVnTN@K4{E2d++p?CN!n^4
zaFo@4qc8Km;J{bA9vJ9wZr*WK?BEgYBFnEYdutD!K6zEb*!o+czeS(QkuRz%1E$<4
zX+0T}8n(13axQPM@nSJ^&zz#mF5CPz)m+<LWvO<4W>N9OU)Q$tZBNZ9+w8W9_s0Hf
zk{g!3T77xr_b&&W{0!~ypKmBpT5y)(#XatqyOP`;yPQuSKks9FdHp%FFc0TBu2tHl
z2THt?{NHc;KUHkY11;B13!Yg=b<%EhO8R_k+osIDh4o3a#^J8=hiT8A9Vv||OzeE)
zWa#<nVoc<;YuT?=0~cP84f>Y5=Cy(P;TdgyQk>qqvp=6#<}AD0^R4`NSDLhCSn`4H
zN3}cW{{GWt9s2N_Ekn)!w~IYjh_<LYub%TtXllrp^4PrZub%g2D1UQtbjv*6TwglL
zET{3lV)WH6@wm(4>s4d+Pu;k7cLLjCmo_isqG_@7eAbHC8bmj8*Cd-Aek`Bu_J1a4
z<)w~S|GT(O1Slu8<;;m`UKy9Xt10(n+G3v>tIV8|+S=BdF||EQ@Qsgp|5Z%Ax$Z^#
zERPq*yi;q~%gT~xi#@XZKTYFB3jaY1#<VxD&Q8cX_oV0CyElrzy7rqqo@lfo<4?|l
zvZUT+=QncjYRqtRJjNGR#2T)#Cj6TRLthim5(dAN*O%Y*HF+G;`L~|u-GTT2UMqS$
z(Yfc~+H*L2M}dCU>KLDc?~imXzxImvuE4KH0`Xs1ug9#7zvJR*b^g@?{+S!NcC$3h
ze6_@rWunTwm!~4!`Bv)O6l34FD3|%}iw${?)`ltD*f@PR5Uj3y|NQ_j!-=<L9IKZ`
zE!FmT^rPwc{5012A%Qg!g2oSW72^aq$*HKFc`n+!NN&@6#i&GK6@`L6|EVE9Q?yR}
zeYW0xmr{b8(@!mrzkWybLsX9bIKk`l(ER_LrA0~}4EGL*C^i3^vnVQq!<prR*{^&V
z&JQhTCK@++9sjaapYO5NffR11B*wW}a><LH>Meh|D9h{3#6*>(T?z)P7&iHC>v~@E
zXm!=%j}=U7S~%9Wt!O!-Gbbhg1=A_3We3vcw7)#yq5LZ6+$SB5J!j24-l`WpVt=Wz
zQOKJk_vrz?EMvDuUXO*2Vr#?29&w&@U{c7tQkA{i?FnDqV?Mr@OI|6do_VB`=ji%N
z&-p@Yn}?Rx6Bc(a4YzGeVltlZ_+T)7*~&te4?DiJT-&fI&PD0-svzf8Dl)q7o2Q(d
zzDQeFbfVRS%rEoaw7zNC-JtPRpoc$lnfeA#H6>N`J01G}8z+3bryaDTfoE9)mtNs*
zfz9VW@%)f5W|RnKu(>F_Tia@zyQ&LsvD~6ifh8gzobSm!5*0kt%dlB=E&p$&9&yiy
z?!P#-qqJ2|oYFhNu;kRCNCmw`cBkrD7W3FKFPGCRkn_;v;?R1NR3Mk3?c^fmnPT;e
z-}8jfL8c3xMLAE{mYq1_A}6k-VxvE!kJE6)=Y*_`U45=!6rTA6@ad^O)1AD`d{+W<
z^Yc@Dm()EBSiLWl2Ta)Gw;}M!XUh-)rKbg1DbJLj^{|;=VmtU^?mbWT%BBq$Caa%l
z5N2};3kVV~cXE33SnWZs$$Mqa%#HKR5;J|A)KsVNOgW(KaGLj41J8eBo=Ah0AI>cM
z!FSr=j6t5uBfb=uSrfQEu;l+UYte~$#rH$QeWRhiRm;|Q6Lx34^~ldip62`6OnG$$
zQ)$i%&$~~)ToRfgQJ`yhnE&%ivl*RjJ-thL*abJ5tW#i}z2(@f51sZBsV{c0C>j-p
zyyE#GHd$CTSUqb>ZO|j5{Q`~^Co-m_71<c8v78kxUv%)(3r=1arl8M%U!0HJaiH#5
zyKu_n#U~Ev?fGcdZWJ5w(mX7uA#aJKyOVUQp|%T?)}}{CgVwB`wpr%n^8amGE0mlc
zTjiZ!cO=5gePv8C^8uIei%XoIZL~StV*OR{WQd+k@7AYHy=@tJk)J;b);rlow*UOU
zg-x;biJfVi)~jGEGl9}qmo4QEt(~&r%$yGHf0NHy6*-(2{b93d|0^|b(U+?IY!3h4
z7^@3~i?qxTIBRFKh4ZqXn#LiG9c^w&M?9hqPWY2_>C}M<RZBEPIJLwVUDA6Q`@@aj
zxA#56vUgs*nucZSK5wPVd7peq{UmnYqw(}dFR|x7*R*$kQvb1Gb=_hw%XFWws$Vu~
z^Y@?Ky)Phb{?t7=hXpU2R48rK5j|siLDTSaj-T{rLuFy3moNOaHmmj>t^RUpehZVy
z*GaqBww^9IWSqTFj)mb+(L|9d!GCiMrQ$bFN?M$Ig)PPJbd*a|Y{_NaJg(62oMSm$
zO&(kQqk63VhnWS~_z8x8l{qW)SY@%9)9S4qI*f-?xE^erAu+vh{br}Q#ZEryE2nj}
zZ3wphYj|vx(~Ki7tL|NT8pEz$wb)U*<Z@A%l3dZ+1b6%B4S{aQqGQc1Ym}moZe6b7
z#$|CRMs#wyWe~%)sUOd&21E)vK4Xqk4l!0TsWhL$Yh`kQ?Uh<oV`kE(_{DCV7uZ!7
zuucj%B+AvlwdzPx;HoLhmTahSx^f_Y`YXQLwM;UF+6(N=0#dGqz4^fZw%mI^cd2(1
z*M`OQ>I~LG0lMo$(r24&`mo)5=E)m5t3Po_cxRRFKEOHu!j?d{M`{yz1DQ(C_FOZ3
zJf(TESytY2^Airw_OYw}|JG^HcU{%Axx%oywbV)MaOg&^t8#py;z!H4>r@uIaTx?H
zni{ownUAOxC!d7tnk(x9mj|sm`c14;_uaG;h5y+Ej$fPDY>;Q^?zi^FL|NHoOP$tq
zd-GYBh^8gD+2pXzJ)_L2lEC%vgy{pr@W&U!8$+#Alesm@Hw!#lDHk!n!&HkS&1Uvv
z8*Mh`o2wNH-`R?@e`-51K~e3N%BIdenzr%R&v<nxZPjJ4x-{u0*JRsG%JoW7PM;>o
zKQ;Yh!jT&BdQYF#hs7M4+TkCjiX{s9s=QUa;$9o&$h9NE?p{+(VCM8vZuM%(D|J8L
zo%>#UI4JdM*3&*o<7d;gCZ74UAmr=+4qx@}Dn2`txc+Qc+#q>-(_5jp-g76hr&+CU
zjAk$C=MlKR#P3*!-prW|D}9X<mYXT<s8C+owB6e`T2Orx^ZqLHOxqjHvOA5UPhB~5
zTy<W4&4-i5u@(!$itLxIPcFGAm&2IT@5b=&gb^#-gB#QH`1A5?4==NswfmS+-O-()
z|6NPu-|2`bUuG*_%E{KV*_lg)VcBahwuS6fDFw>sZgiPtm+TUjtNm4IYQExP3u9$e
zA@i}Kj}pEf>}p!m6MCvl7pzK~J0<1W<*(~h=6+{k$!CsPnrzRx>A6Kf>ESnL9$ijp
zJ8eIep~v9nq?=W1nHd<E7|ta$Pbp(^Jk_GYy(II0J<|oBnG2I6Ha^LCb7r}t3X=<W
z^`TXL?<Q!?y}kX&x{pm_@0>opGufcD<f4D;9;1LC1$F<I2IPsY4xIjUO(m0=f})Rg
z<&{q5J(i3?v$w`pZ;iIzX6>|XuVk8OdeO3~uP)|Zy}tF5g-cd@R;Wu<91zk^=U*q%
zV;_^OY}Yh@ZZy{+=dfD~M7FV~FuDB<`smwtxMKO?te0Q?Swq7)Le--f7ezA$MmvUQ
zGWiub8M{|3jdoRa%1}M0x^Lk%iO64FhSFW09yjlNm)R`QuH^A?#->I0oD7ScPAstM
z(KfP4xs`VBOt*>91`V5K&Z<lrGKcqSgqY1aG?DMzHjNCH1q=Uo<!ruFp;$3>QD;<d
zdlggLu2rvotf<^IHTh8S?L(&DCO%!Dwd&`_a<QG$9$omdsMNym{Z)}5zf-%`EI%Br
zI+Jxahx7lZtY?!a-?o~3)s*pAa`um$?BI{8?lae`|Jso8G-J>14XHcq-)x<un#6NN
zI_|q^wQAF}s}&owmCaV}*3;5aY`^o`u1D+O#g9ieZQ3bxVPcT+EaUeLGOk7(YZi1c
zdqgvBlH3|!eQEDbg#@YINa@SYM`tfi$aHe$R@329T5$RQVjrERt=dXgRUav(E$dC(
z)>8EQ^Q|BoU-m68i}V_2ui}(6URV=WQ`7h7lK$T)gXJyUXC7x*?=UouJ+r)e*Z)u^
zzbi_&;x!J|r<?aY`x&$246~}3`IhUY8|9R&G$$o(?s|0omg1t9hs6|r-8mbr=5cU-
zSO2cwN8$JWs$Rb<cBhfOc(3mf@7>#e=sK-_wIDVjrmE-oVI})LXVZ?Q9dlOR(jXeu
zC>qfyvPFT@P4<ei1J@RZE9QyZW{skY9Jp^R6ur?XYSPH_s*xw+pqN?Woey2=8^Rg3
z9A2?9_+;E4tEUm+l~a#?tT>iCPraj4Yep4!+rJh122AIL^%~E8HJf1jvB0)b`Bm`u
z=#QH_V#4iqORcFqRQA1?b>kFavj;bBSG}=+)Fb+MgTNm58@sJ<uk|_gqT=~hj!M^f
zE8(j4zZw6F4plykQ8&xJr~Kso!+9bxm&CL<XC<jP#T<O!dr0;d$J@H4B0t#eKHX?E
zRlZite7a1sL+Wq6dIFQe-7V|a?z_~v^|kjU-~C@S|88BXu-RsxSOnu0<$Yov4^@vV
zaAh@S-b&>0SSU8hfy*S3C#>#!bmNR`i+{)`Jv-=9yvMk&SMy|F<@U*6>K4YIom^US
zg2BJ&$V@Sd&-K#Aj+OTx3(T(hw!QY{>{W5m(o$=+1pS^)X|`wTy%?c;ZT8ZuGx}cF
z_rBb5<fx38Ir9RBrk;jI7H%n<iU5T}tvu}8Hl?o)UE8#Fn}?#a7lWqRTcs=&<q09H
ztMXV1Bu=#YXh(m&F(K=J@cOW0eQQnnwp=|Iy4ozpiswyG(*pnI)Fv*C%>^0L7cyBD
zNWKtxbzaDyHNaMC3r~<y)ae3M=dC>Ad@(z6ZW^RIMS)HR4)_$-D72GH=)&CY>3KqX
z3nzd1YkfUGp~;iwNAB^g1<RrqM_f%l+Gn`BEag^i<eeR{Eh^Dw8&|4NI3vdB#iOQB
z@#}%e<~8OU#Wtj#I&gy}b^ctT>O%*%Y)BQpqj2C&#^wX_nc8h{Cw$y|(4Ad;!=IFb
zm&t+CL=U{Iy1D%3YN01z{{9X&dLXUjVkM9%>Gz6#T2#G2t>zmZ>r%;xdk<=wKE33B
z<08<&dUpGK239eR1(nRHOBPh~7ApL2W%6VZa9|P%;}DM%S|BRTo}GIjtV68PVv)#}
zPL-f$X4NtQX_l>y0luaOUkEtm^f;Z;NXc0=p-FR+p;yl~3zLg3qKxWNZgRPdlNe@S
zNl}oDk=WX*ZTRwpmy5Mm3b$t5wn<4<p|P%0N?0dmUI@Ooa^^k%)H~1Byppy?PHUI(
zdGL^h>&43DEL=Y{8)G?HxL(X}mh*bCpiXg>#=>r|4=*y8i<${6Z4*q3&|c%wctq^$
zghO5S3;H&>iYWUoX<gRcc+1aCXVRwDrL0TMCNMGIRAm)kox4ywd(+QlNkyz1J>L}P
zm~B~BdC$}$YBP(i>7C8Tx|Yg`U)|O|kHMDh^#9!L_l%~**J*vzn6TqkR!N?lN9vX2
zZBN;bw?;E+1SIY)a$I#F_E*}Eu>E;|Z(n26-<uMUabN4jBL_}>krYQs+a5jVwIwPH
zQvxO_va&03CiF|EOjxQbtu1T%bjq!DE;qdeCN7%9ux(4=sR_AjQo4`3>CR3I(Qqxz
zag2>zJ$0*-_sN4|a*LLRMgM(v^vga!bJ-a!RUb{KGfa?Db6gsJIMsA%c=Xx)r7RPJ
zy49j4tUW35^~4TOmX(u~4!X`hvNw6n_g$vbW4GO^x^<7~kEYW4rp?D*=bSUMmdXoW
zpvsVcK=!&tQM>)~IhBhY?dB985%a5fe*0kbn^NAk?tt^G8;}3@6o13Hz^dxmqw4~5
z9v#{l(8T#KU9N#es^h{tUeStzm%$EMi61Lf6*Cr;UW}gpsmDpGJX+$SMDhn-$4Emb
z$&Ck-9cK%C`P(Ei_jS-wImwjMV!{!-S<fnL><>J;mZLZBxAUf$Q@U$AuC{5%PIKhy
zpR$NI#xU~blF}(GNvES!Huuf_Fhz{>L#F>vqma*RsTvN669Ob_;+Duv)nGKyGHu*-
zWr<6N)aMggiiU}<NlOk$%0(X04{A{S;x<Er?^C9;TGFJgCoZ&2=P4-I{YrF0L#V+0
zkaa~Gg7apDeU`~nUELYdljN-RTvo@<+v}iUx5Kk52Uvdf*r#lnCwcydFw6gCD|?b`
z#4A{g<cyxV$9_3^ZG+Tu8-@su_@-OjpM)kf*s~c;5K>b4^o?nHBX3L9om*l~CZUVP
z1YHz)lR27&j3%gGOi^MzIK%DK1FpXl6TDI%Flbn_O`CUTIk$?&Vc8f*-SCPvQ*$|H
z9ou5$wQ7Q<j>pf1;d_j{7IE_FW^~Rr{P}dc^P|(cbu2wCb2Mk9vn|taIVp8*i`q6(
zF_+og8<$<(pxL0|;=u6O$AN#U+1WiuQv-^B3dh{oWT7^(ktIelLHmw^clZs(`M>X^
zPv3Sx+mB%i_iP3Yr-IGWHyG5-cV9f~q?hih#&uvJk0oC*+om}$OrEQhO7UeMdlYEe
z`R>2flIFRd;!V0{w?1k-&}@BmtHhxqk=>6$>*XAw76ryTedT;!7hdyh3TEdN?VTJl
zUr1+*e$yM?31u^<FPQfH$}?X^A7id*-6>ZT-snzJ&`X-=)}-Rxq%%>$@t0s9$0NbD
zx0-l_ZX7z3$LRfw^8oMK0$0YPNmCZznOaw~G%TK1V=enL&n6L_5I#ZI!x4=ETnUqy
z_~cn5-9v+OqO29yW|X>lXO<}#9J#V~Lz}azPeVI{Xm5DJvo@m(SH*tCbZW<)jH-WC
zysfMtH`n#@PL*qFMLaX-xCAmCjbml+Ix#8iMNeytu@!gJ3Ez{2t0!HWB2>DIOKWBQ
z0auY0iz0>F|5Z=a+!EdR`q;9S0b50vvUNHg_TXx5wn_Qj^x}k^?wg*EbDubBMV?I9
z+L*NV#rva1VO%wzT&6Es(iL2?^Hk%6rP@_*PuU!2dbmhYS6A{9x5CnCXB_g>n9kfO
z5VDbJ)NnY~X4=!la_b`3@n3;X*Y}8+ZGPr^zHaTT)|DR})@ZW&T$MQT<d1>n(Vxk3
z?G{G@JDSA)ZDm}%bjdv9!|HawEpMw|Hk<FatZ2&mJ^u3w6s^}4@~+=!KI;RciP^cp
zIe8PrEbj(<37_QLw?SZ2z(yx_%cC=kH_ke7@gQ%Wb&cOT$<v$VS{u|$Q(3;e3X`(A
z&FZndn{C2k$=U^MjOTYxS@!O~;|_zyWnXg@M4e8gTF#xNzVyWE^gSt#d%RvXebm^W
z66@jHuw}#Nt1G;7zpjoyeCYay3w<drvzz8B-B9O#_Il;X>U#@jsI;Ela#d_`!$Ogs
zoJT4djqYz(w062JT<N=Uk<v+)TrTd-_j0SQ9er|7C-r3RJek$fx7_bFi5=UJ71pyz
zY|RQ+pCtxuQ+J51+qg7W``6F+*O+P^FzNqrc%#73vU{P(j?^H*=F`twc10Gc&H9m7
zyLk1(1es2jlZCU)`0iK>U+*fP@_F`hkDU)XmUUlf&6D2DTI_v#j^b`J|Fhyk?(ZAa
zpII+IUvtCM>_$WiQ+460SeBWGilm<?t!W4>SN*?oLw@3sgc%|XtDCPb?&R&9JlRBW
zcf-%a2~ArvJf-7WKPH`UlAAQ4taH}w`8R!z-D(znWHQx0Dlw(Zp-nWc+b89fA={=#
zE)&KpE?GM&^I9S<2jxk6+&yf0PLaLxle1pY-K`1Rw)7_PtZv}ZeUbP+?#f~LzaL!f
z|2*uEn{!V-`NAQ)r99O-pNqfwn|XM$D0nyBsayB^^|@)$#uu8}W}fNHzEfDCEPc1&
z&shQ4)m_J;p7;Da)>YaRe$k3+*PGK@tgjvP3$4qFpRsSxjw|cAeB*C%{@BBAW7_(%
zrs-De@-6=rbieMb)5@LT?N_zp_x2~7OIj1Xt_m<ZNNm}}s<G(5LZ_qcXIFuhZK0+Y
zLzyOInNCu+-lWX?N%^6&K<3d>sf%L%A00fL9cvevpLWi%ba9rrY*YO)Lg7n9kbu@#
z11`@6X8$hmr3afO3rH?b;GQMGy7&S2?1cP;Aa0u={*8wsAC(DAxfIECp~e41OW=t}
z&ks50ujRep#`)=))0edTe+E_YFIp2Vs+P02rM}=_vPme9A@%Q)w%~34JK77_8w-ke
zRI?<qt1_zQ9pLm_l)f*?WlyNUzSN@Kj%i)N9lg%d+=VW^i?w@pNcI1SQQ7WxP)PTP
zP`%D|xm=-Q<8<2x;=ybmgXgVKc+#Y|q{6dgi@?o;Vt!Lgymkl(J_uX$;J*@2MEo`7
zcr787sVs)e#7bEWjh&U0Z!AtavbpT&in6y?<7Q9n30z<n<<L6IK&q`o(=1J3!lA@I
zL&@lX{IAP6*B;;tRA`*fRKc{sEZHFc%>uy>VJ#Eg{od|KwB45Y=u2X0c;(deO1Ej1
z&ocU|zn3q(&cDjNim|eaS#!cN_o}}J6F532Fj!8|yk?(m5iS@hB)qa(G*V!9c)Qq3
zqqPolPaFf}Z>FA5tVv#GF|oNuxOGz2vgi|rtQ@L^8Z!e#rpjzHOrI>AJ|S43a$0&;
zlF6PRF}szrfnRDl5@UaCl>5F>bj4=5j$>BmH+nrQm-`>c)~6;Pd@`e7EaZCVl<sf;
z>-Idh?*3+d*_DgILUD#_1Cv9;{I9-^ErJO`o^vt;qNdt*A5nhzSY_=uj~799(w26y
zk8(mXlx&wo^iJ}*@Hw&JWuDLl%^PA`Q44xwUPQ12I^8cxesrw4dt-B7VxOx8mxdbW
z1Vx570sONUu-;(P3Npw&H={-Fd5fP%Ylwv38?hPF4cTTbX#M)F{4YaWio}Fujkd%S
zvv>cTz3=60_KW<Yi&;|)R0=cXww4O4ZxL8GVLIPI;l^VsP0nl+erQWBcbRilC&+;H
zRhq!2hlOW9b#!uca5zeL9iHoUrQ`6{=&Y6^o{ehS$62Rqm?SNlzh{|44$Jiam2MWz
zB43ZkJ}m#Q$6e@IXc8lQAS7$0l8}Kx!l#mh#tTE+-E23-TLf~y74Tf4DRgyN`DZgO
z7cthL31+htSR+sHrA#QlcS0g+!psKq{51`N7b~XCdsdMWz<+t$!Wq}OG^A8stl(5k
z>x&Zb`y7$-(ZbKOs{cnv<*yqORoDB!CRFicPFQL_foJ8?$Wu#~Y+o8zRmF6HoB3mZ
ziq$fvfMv{y+$S@+&u}gf+gZ2&c&hx*)GdJlsz0^)wu)uD325D%lognk`b^qmOO0M<
zfJ~9}IrrK#j7`^`uehujH1SpK9mgpfodvcyThHzg*raS_ceu{rdQjL;Io6HAZ`0+c
zpQ`Usb9+B=>QBK;=cE6-wTilWGqkrR32a@}Qs^9dUuk;&&DBf?8umsASP6x(D+X#7
zDLWi*WOB$CiqNub5?;E<%t%pqX(soY3Cc!|z1u7+QV&bGJ>zByn5B0`b(vwF_j1;f
z_F2&lEUK$m-)5zF3b20CTJMn{;hNU!rpCGDrbO??-q?&bM(a7u+8bD3^D`A};0WYr
z+OUE5;s&-v{^Sj;&P~<ahAQO_0ut5S8DFD{U6r>7czY<$Wo~R26wJ_6bmo0NPw;v4
zE3tVGe@>Ep-jdqZ(e@&(Ijbn=*t}aW=hc;^PpWX3d^qMvQZ|oB?7yGO)R!uJ&XikZ
z(Rn>|0qd(R%PST<`k480sRBF4e>F#ut*p#zE{m?=NiNlTq;%WJ$Sp`fON~`SjFoeP
z#5Bc4eG6HaZ8Dqa$d%k6@q7nsN1<Txca__2k|!<;EOiu^sNU~v)~^zkGUNM_ClX6L
z6Z8L_n82htfunK)SLe=|r<cyC-pSs%jLUNwTj8AJmIBFMyI2Z$u@>%Pf6cvk17l@|
zKweh6`$EyrEH0Z1g*dLW?MtiP<<9k5aryr1smX48Do#af-4xTREX)!s%u1Z3ra5`{
z#YvAPrDcrLCpZTlI2c%#CUo|z<$dWXp%+(v5ts9d-uJI?%8JWYzBApvHyS@OpPG7L
zl~wB09m)F_bO_i@-M?qC^^|Saf8Ar3T*$0B{6F&<BSY@i)!LUDxaVxq-;~9bI9=#-
zXtkuE$ZLb>28PIB?zJD*MQt#f`$^)nnZRNNt!EZovkTVt3Hl{(=w0hDi`$5;z9o-i
zqtlzRMT;-2j|=5z>aJRR{qWwOv-fBoIq-8f$8Le;zc;W>JhEc-29Alld~M2u0;<^`
zvc)(sd4&n(HmKV@QB*Bd-yZHIJYgbpA@}QWHqG1Zdlqf({4sYk<GcqeS!B`$^JM1j
z-#V$=LfZYxJZov`_cvV*sm6r5%>SgR8*-rNH>cajg+U&-5A^cLegAkO=ZoHw6(@c!
zT+pCWx;DUW<1NpK=O@Lg4AOQll$P?067Ue(Ak?SID)`VWHt4_PxeBwtPgOLnN^_ZK
z99!aZD#Vmmi8aSy=_>WA73mXJm`@1*we<gur6+in&9&Y&KYQ8vm|Z?Ob2ti@E&8o{
zRc6+D?cLoaVljbUc~`kI7>~UcmJwbckhj2&W8%^M*3)hpE<fZRkaA6Wqv75Y+r=JA
z+H#go^82w|&vKrHwSed1^NSf*WG1e-S?wT|Dkc*uJK@#dN8A_hl?O#z)){QI+RH4T
zG^OtQ?u%2sM66lso<EmcbW8s4YlZcvf;Dq9=ccXJ)!h-)pj0$d!l+kZ<+IQm!rS)E
zsJ|67T_PfsN2w+K^qQp71D+S+Jzl6@Vw1L9=A)LB^mdky;Dx6Cqq;A>j<9wAzsgd*
zfphf{mW><Y-(G#T@d&%&wT!H5{x$-sz1Pxg1^y-ErY!L1bQjp(kQrLY{{A9c*#--b
zg9Snh<mzO-m>igdK1@(qz3jERs#wL`x)m;Wf6f*AxT#=Hrpe7?`7y^IX{JwAK5ibp
zxwdEd>z^*#*;eeroeQ0H|E}!Jy>f#6@s_o<`+{D(y$sswdAh`4({xtBZJRUX?uIVp
zUD*9XXtkfapv6Oh8=E;>Qj}h@lxH2kqy6lT1lwKhZ|2VudwX2ZOqo4_>*SeftM9tq
zIdlK)kq5H7&dQwiui3S{`)s<ez@qEiF}X)_eFfh23OqcuVXk5O=4hti0Hz4F8@mkE
zeuc2Te*B-!*@^AhnK>ey@zp=hDOjd`Ub9*E=W@j)pNeGP%d2Z%g_WH=E~a6*SNN&)
zncsWbW-RA5t(7>sH)qpcC0E&72Omwcy0y|nzIoa{@5+lS&hEQ8Rc?Lr%0SQk>=UPM
zc)V(rS7z6@{c)L37|!VG%$;_(<>cnZ2A(^*`?K7vib|7zd&DU*#7JKevk?&9$lA53
zNnGUFv9gHvQpe7w)jPZn&vE5)*faat#%rm21&Zz+xm<IEgRyG$g6GrBq;4>)mIpkD
zQHWld%APVIC67TlaGOQS1H<y58?1rH7)p-SWQg5-slCTFO|41#QRYoA1=dBX62{Uq
zrq|vSW4tLgeO~j6o1gCdf7mBcG{GXqCtI#9#`XJ3wS<<}Zl@hS?7p?PFLucfw{Nai
zd)6v-lqc6~3sld%9F!i)d+Bznx1g{xgN)Vm_<iD2l}z)?OR9xUwZlwH3<Y-UyyGg8
zk~zk9WK9(}=h-B#=f>^#*dJc|(y(E<_kH$g{^Oq9i#KpB+Q3z!R~mJ}{zY;P<Fu!>
z`*zDH%>84clI0}!ck;yT-{q2DN)>AblvZtyHsDl~ym`_2Vf|#q$~$5|c;<b#3Cv%!
zoOR)H_53sey$dfTSG2^WUrIF5aFkVGPCV%vbl-f6+SHZZF#@LBvzJR*zj2p;>Hfv#
zzJOExV+q?!E4wav_{C=V%@qIm#7O=B)8dl?or#UyhHT}Qhi*Jp&27k4zWj7@iH3RE
zw@iJfYb9G-9xVtzEs%DzP{pR|itD{gGP`ol3S`UfsyfHD@-*KT1A#9GxK<z7z~sRH
z``mZl!h1eE+P@Wd@0ONPd5|3+B9=Yl!?~H){~2#8b5Ir(bcuVCI@h@9WZ=ySyvKtT
zqUQ05x2}A-BL3C1xv#8lr<v@2xn|w@MZB-|-|4o+be=U;FDkt?bH^>e)1B8u--xVr
z<2`ti-^yV7+SvHrZ+86MddR9PP34Z$rVScxwu=i+UFLnq>$HP6=?(A0<Yb34<@<lE
zT7B=h<+7OC4O<o73+><&X8h0gu=V}?Yg+_<d|?pS`{IA^C6l&Tdq&R8&Lq`@#KSF2
zZoMm?>n}RmAt;#UlTny_BrNFEj-tbX3@43LPfW=ao%qDteBB$44a&i4XJ@N=tCd_{
z#iMC7XGf0IuGGnaizT=F2pKG0$)Fj$$Zx8`(p4;~5uzG_6I2v0trTA5H&M%U;SEkf
zRgQ?=*D|9vMK3!s*I?<p*yUxyem5s<f6vAk!&1Ouv%?@!FGL_;QD#Q>gX0tBt;KST
z6qk3*&`*E1Vkh@<BPP4{cS;IdQZL4MxxWcn_BESveeC>aUuGI--C;60E*7xrd;a}{
zjnCEcEFGO#WhQHTOo(Z^-VziwF>9vaqJ~#8o5i2|&FN`)&t&E*)B3-v%Gu1peQQ=K
zcZ!SA*?xDQyiNNWe*fHEyk9<W^8S7!t^#|;Uy`lUsw|>fr?R+9w$AH#C77XHmXX*#
zH87%a!n{yEnbd=IRyzI4VJ<6|I&-ix7%YrpQ@OAtYTJWDduQG&e##dvbZFx;+0?95
zHHF+qAD8vK$v8SS=+=Xl9^Vra0*`6#XfRr&JFhEGUvpc)A|2HwH#VJKGiicC<JKp)
zl2#cn^)g<yY{`w)D|Oe3Jn-4MZ`NsFt;J`$yLWz>(tZ9Em$yd1yH$U*7CD4Aujx4@
z;oEWXf>NvgqRT$jAA@{$NBf%G)j6tT^Fhs*<H~!Etv(yPlxO)jgj#wjc!w?f-N*ai
zXZ>jt<)A=mL-Q!dqwA)Y<z%WWeq&L1y7Q?|Rw%=Zkko5N>z`$Bz1gzhwf<s_Eh{`4
z;*?%(+ns0VwL;F&?WOub7S<I{6&StNxZW)d>-_G(7<SR+u#j}p(|c83F-zw~xkN6N
zkJppRSQJ?GaryMRKZ))<Weg(I59BL<N)_fSI%BwKQdZZ=$8&OSX(~;~KEiu)L6g)f
zjYWM{To=vNPoCCVwP}-<_PR+ov>4a_nzBl3<&j@EeP`9Tartk$v`BmXmJh2o={f&y
ziDEdwvU=950}`iq8S3qLw&2tWuGKk43y<wvv+1tJ)ipu4Jhazs^1b<PiTU*;ZRNKu
zW_LwQw+VmJUHt#SDS^Da{mQ>)?>@Xc`dj3-1-FuRz3s?e!TT=fXm8jRDWzl)HQn1^
z_qKQlF6i~WQQUvD<HiNcg-7#UdvA$v-N3q#Mex^&Z%n4A_Rg<+v69{Co@=3iGw*9z
z)kmSn0yZA5HxyBIX=Yh*+>!b8(o|7y;i#uS+w_y#6<GeRto;>r>1u7vqci^Bt)n~c
zSS*~Cm1TcoR)BG2!$jk(_Z~r~GJ@O|IdBH}-E?~XPGsUrpNHPU43~?SD5!DR9WRVr
zv|yHy#By`ZfM)-dlNMSy9oS>z>d)NLb0el*N{d6GRW)VdVjWdYiJ&awyM|XeJDazk
zeX};$_|C%AqOU9DW-V!BnZ)#8GQv^&%-$2>b}5fmxLlRJ=z39QN_5iCtc!g~t?w+e
z8^u;z2<g5%pl6;G<ITddK$k&rNz$d)tOs5M+C{Btb`)E9v|VNk-`h+lMW)s4y6>*?
z+iy~GnKeONTkMK^;Dtr<duMPc^>5d6{gy176ye%!cZs{KWJxt=;*BFZk3A=|bcjg*
z5M1^tedffw{c21bSgRW@87<ktqq#!ze8>`q&8zS5Xh%KtIwzvQx?-YgV~A$d>~)uR
zY&UV$_Z4=C{Z%Bk&!sqX852*UK(a)(+%ZF$hMp8PVKc{tY{rkT@Mby``yITSYo_1h
zAs!Meaa*OOV9!CH>?bFst3N%P_j%538{hv*G7Wt07G^5vY9<-aXIRsf9=)LN#SUT1
zCb9P72a@M2gf?lnt1aQZA}OmWm9(p4kG7Jh1)sQdy8P)FZp$wldptWIi>`NB=sIm7
z`$w500%32u9k(i^G|f<-$R3$e{aR?o;jUeK1OKcO`%-YD{mAw%rmq`gC6ipGZf*?e
zGg+fFdrwDqOgOjZmV@qAw{FM=1c&`?e8=}TQT61FTV9c-p6i?y56?X_MSVp|i+9|k
z?CDA?j13>U`-m(NPF%ccDVG|HkDucL_OGj!FZ=M)U%5x<4BJ#q{;B|jgxSFfbH1!_
zYvt-#yfwtd*IVJt3xRyqHB<N+cibx2waP;1#t}D7ckTaY=Z0<B_4I1q%>(li5<i9?
zoAf~9utm_5RjR>pD_<S55ZYjPVX^q+Gby&m67;GZdR%LCCMrp;+|$78!?5T?Z2PmN
zCxk1%_p(H*9N6}%yR9%(d2URKQ_gm!(;{oEf2}>v<`KC6?~EL^Z5xg@aNn61b!Cw#
zR~~b9)Wx<rRVIg$Z?LrX9@?~IL7S$L@-yA?mEQ5cCM}6`yd0>(v1QT2OIoK|yj?Ca
zO<iKRP1Aez<nF^WHD*-i2E=pom}?rZ*vOe??(}fs3WsE=3saotdTd)`U1~h@%8V4#
zxf^(=zK=Y=`=xRI&cIK17^2MYzfmh(p6ln8>fP2h?^+S(#u%Z_*16v^{^u>a^E1ZH
zv1`|~El!-A3c;@n76(cHiew1PV128k($MUgAHOAFV)K&Y6S@*IME`ln?Ojl%x1y_G
z^mW8V>7~(v0m&(=FA2x5ycqhbXHHeLq~QE7I?4MUEm*CYa$|;;|MhEOyVS4$&VIVy
zW8=k_4707)1bh%(`n;RNIFpA##g(z&Xu4-1qrc2X)fs!99N52ZskYkQ)I{#=gL{>}
z1yn~SOjFpcerno>O|Q#h*{x4Uip{zZbiXrkRYmptb9dA_cAN}#^PD|VvCN%`*Y#1}
z+#N10#ZTI&@17!3=<NFT%Hr#{HS=7iTsWF_(o9+0+;g)<pLBL=i^#qM`5vv?bvxFc
zaQXjN=!RB|U9e@G#^gPZ!@gIjZ~Sp@kAvBB%l9R9#j|c--?!cRe)o~56ALAuPTA$Q
zq3I@bBFE;39l_@{wY9#uai+>XkN$pqm%))G5lwefk4$7?%72*{^C(9<HqknvGE7v{
zhe>m?#qO@vlLSPJH|tja?7p(-r}!F1HN^%`?_{3htw%jh&Q46O$=PZAr0CkS6R&d4
zK1q#`+FQu=dDi)k9F0`}>f)C9OtO9tkL5(!dvm|fnaJnvyD6>4yQFyTW)1g0yg7_t
zx+1pMO>bIwD1Gv)_U#RNe3}ktCsyV3Kbuf?LuTo8&#F5%SGH<z4dcF&wbm<V^$qvZ
zuda6|-njg;@J8W(SM56=vcKFeyRlWFtoqf%?tt|8fA{9^J5rRj#&?3#I-g0~X5KMr
zeSd@L_$lornO!OmRx`IOV4t+$z^RTr<L#y&w;eq-sW(|+%?v~PlMW|0t&!fWc-Be8
zC_+S#MOWdK^d+vnAG{A1aPgj5G39;6!CGdE$EPf`Dh<AH^Dj7I`>0^^-w36qPup8A
z*#6}dv(P@I%b@dU^TLBd+|v*7PWNi!kYKGZ(DU=+;J7*M6QjVdUHjg7F8;Ad-yme-
zg6JhbE-iVqVn)}_B_eAi!cH))KE1MZqtbUKfgiJ+IzP5s%{iq1#nO)D@Q+i@j1zXA
z-Dt40OSSxQM^_hz=Z>YC6I!-@Fqr-S=At<}5A+q!NnhM89Be4L*@AO|+U^g#mkQeS
z8SHUTJgU25*1DU98$KGADobzD<WcHAcx{rRLf7(31=80$mb)}cSFYORzR>8x2lb1;
zdG(n$U;e<Gs3Vj5r#JP=+O$fU>xX$eZuL)FFlX8W$KFePvNEFQR@(A!(g+fe?at!7
znAm(NQQM}Q?}6|Zalvlmg_C##m>AE9bY++_DsC+?-1=^{eW<aiamd7Km-%-B<~zUU
zi@43deiNVWid8=r+t=ot_%(a~TVu{h?+M%=SNtiK{HxsDbVA5;#&OG)lQK8+EO$7-
zZgsM|d1dC|$%&uL4sTju8mPcH;kcdQWV?xL1Sgpb{|_?ff1@MDb?`ugh+cz8nUkaF
z9#NSt$36~=R^w@xx>hJ<wSHM3dg|a*^-W%rfAGokOl{gdwTJ0YN5c-qH+$dSXkvKL
zKV^=m{x6Gb2YC~$T*D?ZWmZq!`Gile>5#9&PNvAkb00g;dVF}Fiu3K0I~P6nUZ}h)
zPjbSMCwzyG^d5V|H%~}KKSiSLp|k#%({m2{9JsUd?n935hW%j)tU)i_7Av?c(Xfu1
zrOG&gW#x+9F}JL@_Vn$V?Yq0#VbMx1wUCVge~#Rh*^~CmH|-Qh;2O2V0ejRR&N_PA
zQ)6>aAy-$?t{$Do9hWM4qD^{F6-)E&QSZx`z5I%_{^Q;TDUCh>{{#Ou_j+eeeY~T8
z>x_QpoE0xEPWe2XlXryoqNL!R+cF<BHbiTCKTz&dapBz=(am|I$#7zWZiQiA#-a4a
zKHZ3|{-@m6F{ws+n@(P__;`!|y;FLcckKA4N*EsDSaf-9)Q1%XQ?$9HnSBKg=>G7~
z_2~MTz?J@b%|w-1%wJZ|JR)o`bHV1*+IyP&@?K9$oOH5#^VySHK_`lAWRlI7ZRz2c
z5m_MPCD=1%;gX}vk6iFB)V<>o^zw~}*cXweS^Sz4tj;91zHhL|+0tXYHE8)04y{k-
z6HaY@W#KgMX4hW<<*hb-8V|X4PHBptWVS`8kK;m9RD^{1OO?EjVy83C^hap5tY{7V
zFVI|?U}D<4H%_3{P{FzI;au+2t=$S5A0wxqmGr&*=fq-#s~r<mLyijVxU70~m-m*>
z8#F#_IF@)mtkWf7lXbuo-+MhB51)kcZIQ~Y>N)yhR_Gfw#cEeYY0<M076FEirw?4}
zSz&aY$xWAe&I=Q_hX;Bt8_VRy9?RV;`r^hSm4p@FxAd&qV}4_`o6yb`=L|bPt=L=k
za79rJe@TeA#pYJs2zMq!O;+BMhBJkYCU`TYx?j1_)N!(x@!;I*n#HR(IXO?-vgFYO
zpNLkGlUk=gtZJ4y(Nwb4`{0eX1v{HnJQ_>3a@KB*KDjbb!Z$VhV*i&DhfZlPyWye}
zxL|UNW}dZ_=KmAZax`TtYFA2!GO>LUnk{o{w$827UFP$qIQsnb)Z=h;E4-z-)?91u
z6>-l~+PTZsH*HT*;kz}fZ>7z`b$h(-746V8vUsC#NJnbwRY$$`38%$R3av}v-nW74
zK-XE;)H&N4oHQk_OsZbD=JhlSoeA74+v|SFS^g4e*mz}imBfqYkR{wp?<~Bv=rr%&
zBzE0|RSba<cQ$aGV?A+>)&7Di-?_O?4T_gLExJT}?~2}K^1aa1vSP*6yBCk$Txzh>
z!LXBwv*T*SRHkN$D`6c?Uv&E7c5%&q*?sp2cY2iCX#v;!U(V(T^(0Q6c+~jXn>UI@
zO3P12^xm4VP<8dWGu>fpOgnG>|6o(IY4&>=nKYU6U+!MZ*X?V0<ruCsA#+b(X35+(
z0ohxddwyNf=sToq_Sm*GvtKu%uTIoXJx8-yYjeHr%Ik?Tnk%%f?3ln)D6k@`o#&#Q
zaAC($S=K-Sj#rk`t1C^OYwv2A5_95A)2D+XJ+{+3K5U8J=)rSwf8dG@BDsNwt0GyN
zxTdzOd?K}Fj%ioe3Xb|8?PUdb4R(36t>LoTxO$dNP;jTY>ffu6JQkj2nyp?cqA~T7
z#INluO%r`)9=-A?==Ig@TNER<F6ayL=znB!{nuN^>jzhKFnFsJ1#gkn`=Fq??SaMA
zthuQlEuKa2&7JA;=R}85v@J_lgT@N2A5QxW|3_HPE#hdqFz*G|5%UrOj=~9D3)r8&
zoc4)BzUh*FgAYf5Td&Z%Lw6#Siy~W_0?&kSPq16W&b_czWTmp(=8l?=K1>Il8zv^O
zI|g36+H@^9R+Ax3a{`NQ#F5R#3vQ)$Yj|+DTx;S=W^a9|V)G{QX@cgH1!Bv89+7z6
zy=#L|V9uqbEi+eLluBI6v*D*&xX-m@-Axy*HZ3qvf75bK<FUvynPb|IdumRpt2*~4
z+1$?yy<cy&a9hUS6PDL~9A8ZC<@hq={QryhFEkooTqx83%Cy>-BU*u#Y2Vbkqq{=|
zC)@27(VTFo-(z~!196vPld>Ip(brl&-JDnDA>ivUK}a!%`Txwtm7RBlPenYr9WhVu
z>B+DC^&4;W?vUe|m};iHghy~*<HaYjiR_JveT<rsTzj?pizYO!NSv;-X0x@!iVLyV
z?=tB+Y}Q$0U!Bm(w&ilqf@!}dxO1dDTKIO&;<pFxL|nS_!(mBl)S@Z37W;VZS9@Ho
zHsR8Y**hNcE>`I~9%wJ^waA$Fwn^?H@kWESES+ULl<s?XlqoRVtm{wPc=_D69dgI|
zySs1y`;fEeG#8`86T=PdCZ}XS?p(*jIb(^`de6F$%(!;L2myA>j#-Bet+f(wdwAvi
zUeSO{nG4vm0>s2cVtrbD?b;?hSh#GRTkRF|O;<`d*Z2PDIQKT@?Yoek8HWo0M{02&
zobdYB(*(9OU5RI|Ik9~b%t@0coMGAJ&~o-~#MF4NdlJrZ%NYIM?T~u3M=d*v=cTXT
zv1q@^JL1kcELYY(sC?U|{MMe^H>6L`?oCLU&Gp|e(MBeP=f$_9Q#PkO=+d5>ai;g@
zUH)Htr*vk8-_hv*^y2;$UeWG%wxyP0Mj!p38l^NT*~#B+*{9m?w{K<f%Ej%nGgxj;
zD6TxwcVpVC&q;T^<d$(Qb?_4RapH(N(b6Wc>iNp)I@9NQGiOQf%1fG^5Og9d{@kmF
zf#MxU*7v@!+NGPYu9xqB_h&UWi(`+3moCjdew$y0Sz4!T;d0YnP1}QdeJc+4?KQY(
z<hVUM^?&E7?3B%s4>{y&a)jAVTb$$g+p<|Fi`j<Ht|@KquK<nEpXWOa7GI5faw1lu
zEP|DD-SHK_#O7Z<bf)F&-Ys)=k6bXlu5a^(WA72?b9YPQH?+rZXYFL3QFddxBkzgO
z28nwISG_o`*VrlE9<d~4`ZFfaZ|dO(1JCWUm|_{JH$N})%<(Pj9GAK5{%dS_vMXk7
zw?ws6UqW1b?ma%2J-I<g<rd%DRJp0F@8BV^l1snh-kms_FeNPa?54V9vkzpt-4WQ_
zz;^wr@^!)R%b6iB6iPBqOYpY~$rfF4{Fb=7JhQXsU-;IvPd%5L<d(~Z+!jBiYT9|l
zuJ+8n(9L#SZ*Tnlkvadr-i##WZ4r`YCpxDFMw}3B-0JX1c7@XV7s~5P7qq{)tu=2^
zgv;%D5A1a3J<IHx@~T_4f8rSt)*B7m)o$(H^m*=?ORtWKRB$_Pa*uMH7`1O&%!%Uk
zvfahLm4eK<2B#)goH*w7j$!KYvYgFpzqD5?B^2^c<T`E3tz-4{;np;n^p!{0#O~>O
zb>#fay}QvT;-1)o$?qfTzn;+F^@#6cfLUkU|C&kMwF%qVqZMj9iw$<xZ(#Qpy82&Y
zriKAi_ELkd9Id6_9?YA^)3$2nEz6z_i}^a2h(G+(u;pvtkJ#x&x8F@@+IKmB&jukD
zUA_Yfeq0CCUWM;@>3hybzi;dNDQx%eo&SGe&)dCg&NMH4t@Hn8z@`sR!g6DS?{PI4
zIBhPP8h`pkpYZ#>HymfpyE8=9`R;s<Z%|NP5T?z@%)V)rtCtf4C!4&J$qNIeqwT`l
zNlPpYI~n@e7GGI4edGN7RsTG?L;^l7+25hdy5x*Tv6}A;{?|XV!nGAwhS}Et%DH;#
z+J!~iVgzkhiM$M0=CVDfigRQ9{!k{ji%IU@S2UUzyIoA~3}Of^+xp_5O{8(!*=;G>
z--=IPdoXPlm)yOp8mXu6Y1VB2@%D6T*TI9W@<k_S{4hvn>`uLwd#2SVNiS6^-fTvs
z!_1z!ruKi%+|+$}X<2Gn<&0fxS%Pw_HZ6*{kl@&uBX{ioKHk9X4ELBK?tZh~Ec)Rd
zKkKYI-i>Di7+G8Grc6@kWKos0^;%OgF)glxQz1`Wz_DRk*}g~LUVja+ja;Q(eN^<o
zzb%gOY)gM_UVkb4^q;@1Hy(G}ZsW<|cXTqGz94(8!E-n5yd6pIErPd{9?FUaX)J6L
z&by%)6rGdE@A~56=2IH0*Bs)q4QX`|t;t&H5?Z6b?`l{~kgsceujyI#iFJ#6u3vPP
zyL&Y@&a345M3!Km2jY|c%s3+^OADwuq%h@t2$;&`^Wnl&8HKhRVhKeDS=pnfR4z-F
z2?<-qFlPtna>t^5H?L(0c9uP#D^PlkRlT<8nx|xa(Xmd3r94#`l1u+<WC<+p3A%DJ
zvrS4RbE&DNN~4J5#6^pj^Kbolp;&UGA(vz<XTzNE@QIf)SKD%4Sn0@g^HQJgqGN7d
zB~Ep>UY9btTs#$Fcy{R|PbI<Vz>WMz&4vA}RF7~K&S6u{b#jwzowChax3qoRJ-MfA
zcihj?UdXVn!n;jmn~8O(=ieDFllCOJn!Mk0+U)kaO=k{Wec$m%YKzXM58s^qJr)<V
z#JYI~O*tY`{grjb5kEJ#8AqpDPBT0vU0szLTbk=5nL53+Y^H;1zuK&fecN1DuJ7OI
zb5YF6uwquhvX5oOtCnmC+`P(SIoGPSR%g{5teHIAV_hl>Q|9_`{(e=m>{{WGQ=Y&6
zulyo#EoJMLueJZa9G`#VAE(B*hOYvy0Zlx+!cN`dEjp3dT3j7bey1$kW1{x+Pq`Cx
zw@nF}7h*1zvcBg<;Mez_?^N@WeSf6#{;Z$8Uc5m6YfDL3*qV&1mbz@4A6=;6nZC?r
z@^bsmU9Ut-Z-z~aZ1PN-nK3JJ+1u^ecOqQ%KF2LTEnpiqbKa`LN9mGHLRK0JG#1ya
z_&mW*e(&;WTn@Hh*q(l!%ODhBZoFom7;oT3*W+rzKSWEtj;SqQ^j><+!eeR^0=8dE
z3iuh#ATTd;$?-n{B5P)_D4%ZP=*xR#zQxsB@p6-Iz=o+J%N+_dw|CqQRFlvvip+4j
zcWQww%LYe&xBn-anO-b#w5n)t-;o&1+TyV%%xRLuT7!i7m#*`?zR2A?vsHA<g@;-h
z7o9g>c-+S7?Emnaa%+1=zmnS_Zr3RWtp_}Gl0;TdN|bRv+QFjlGwI-oshcV!CIqI1
zb>HMUX^}j0itKWwYA#-B$IJUW{=8Exv^gWi8+2S{f*)5(LbOYY>AnS1*DF~m_ZA3T
zkq+W`lD>M@$?Q6_DP<1qO3^d?N|l;qTXi=oFA1JCiKoa@S<{E%+QdUH9tY?DYSEqg
zn=_y>Cy@V{LWd(y(QF+>mW6zYC-`=rd3v|1^7M|m-7U%`t<pT3%#-b&%&6_|Fw!m(
z;hnHZ)mCfKo(z+-=4mY1|1CBtJ}WrUC9<)(c>eF0XRjQc8(g2*BuG08HKhosst77^
zc|0ng&?2iDpy}m5>5}mLDXPsmD@$Wmd59lY+1=c9#&?_cnWRTstSxs1GL#;9<n!x`
z`V-mr#&aWhKS@2caZi0#CAe_j*E>$OA0k{bqZzo&{Ef9{PVowR-Ru$W=GZms(lc|m
z%&82MoG!2wtY5lhr;6JN0f*TNtHW4sp7dM6D&3@VwDI<=RV{r%p(Q($n(kd%#dgy4
zAYZS+8aB&^x-J@Lgr^m#ZT}?gwk08C$-C7ZMoT<=&dpS2R4VPTxRV(6<JRGp3ybBR
zxlHp-adN5NxTIAj(&PV<4;%LT2pTYL_<tis+qzY!h$lI^#5>6$ZRs7~G(#ottCBO@
z-7a&5y6>GW>~dd&HG)eaWaFZa0+CFCdn%Jc_*dp$$a_8K)6H$VYdEGVPkklhsn#&%
zcCeGG;-iTR^3Lyco~7rq=CMg!sY|t|2SfSp)eJUyj=7s^Zdb^wn8z+MciwyCc9r^+
zGvCdOw=UbrD!HxTRNc+5iiUgo>o$vte~r;A{vwfXVQZeyEP1BQ_~P}4eA{=hIp45l
zdf>0UsKGWq)2no)_gx28t$<k#Cl%J8zA<x$f!gs4eC6D+8ZOTw)URDjF>PV${irg_
z#%bxYl#`QBdTJ}JoMfj}<v7oiNkXkrMJ;iu-ORP?|DSk1?ej^M+SksZ^5viR1wJ`u
zd3I&|-?Xa&Ke-loB=QwA>|5Mm)IYOc>0KxD>`xAPrcYNrIHbk?GUqw(CCL^wo`;;R
zoX;*zC>4pen8x@#O21yRV1v1hx9*lhhh@!v={T#r*{?O}hWBEY%X&}pUDzsAds2$j
zgc~GMym<poOFi2ZvS*RF@wUjBnHR6_eC(5Q;zX-V|FOQ@GhgT1PF26k5Pkj7xfA6o
z8#q}`o~k>^H!DY5D8_ZuB8z2?d`i{a6I1k~Yp#{`X)g@k;v4X3*SwyN(p|w%-nLcl
zzWevwuI0(kS`61J`2`<{jawbZ|5wPV;+VMsYrvV}zq4nYFiAUbzx%&alYEStib#da
zwOu}`$I2hD{VdN}nqYeQNc-$r(i_6~Y|gQ4_pqBfIeYIaP5ongJ>1?`D!P60J}`ap
zsv8T}aFlBX6+iQOSYrA8`Q&*PH|u}hQDC}z$v%~Lna<1~3;GO$oh-L~F}IweR2*`1
zUHCD#)xj;?#ZO+O3$O;cUu8KVFKg{m)#K6`pw5}{qcdRPpOm1RTeOW`=WjcGJa~Rk
z3y;Ipcb6{3JY$$+$=Klkc}BL;XTLP9UBViN_uZ~OSa{W!A@!}oQ}-8bcC$@Cr>yme
z<PEweeD77uqW3SI?c^s$Z#l@D`eF}9YSEO9b}C<NbC0sQZ9LS-D%S3E)r<c+<4e*1
zjPh}&mTMxq`Xj2}dZ;nGn7IBq$o1;T=U^_0ukjvlRdge_Z3;VLmyz0g^IiIp&GXK#
zIdQIH)8$1QukS4~JRQ}%z#;8-u;psc0>z|gt>g0MM?~Xw`@~K*6*%p%Wx2XVdN=2R
za+!JR6ZwiGr)$(sXSyNG;*?Mtu%L9RWA#r-qqk?-R>hyW(Ih>Ak==)J#)8oGf6j{a
zG#)FAiLboV*xFJhb8VsM#b(=xmbrIM%Fa5i8}Vr;i|>`@7be$MWZn>9y3)SW_2SV!
zhB=;{3r}9UdrsKuu8+hi5vMEj1Guhg_PZ>Uzv>#Y=)<gN2mej1Hz!P#b-QpmRwlB<
z$9r;U%k7It{>wkT;<xnFl&eR6=QRDux#@W3)IHgR^(m+B$=;ObQp)s^7jL-Au9y^&
z7?;`N|9q#S?*_Sa5#iTUAFRFMxMIq|)SLq>DGHe-N!lyCmnhEou<=4uW)$~ArDbbb
zKQ9c<`r0J+;{5WE<7^Y6j>ue`w&GR_?=j)fi+bl~eLpI7#$(cSmdELlY-v7il9_Hs
zWe?YtDxCHSaeWY2su`)hp>g$*g@qC>(o_5$@3h@Eot=NS%WRUeZ%lIDOt;*Qe(lVg
zTLUlogdCr^Kyi}Jb(?d^Weu~+c@(}qVdX6FIOKZdkfV!wo50J!BWrCsA{<!`CmxB4
z?U3DaXlBW=)(keu9S7Nq82@wrn7u#uAlsHEhgZD^16$@#Y!rLIQgz0C_nil)Hl|d~
zm=ks6xaCU)-<ZiKwuabFa1_bt-y!=T)@8oqpCwnG+&O1@Qh4FHl&$BcU2(Zs8tA@A
z!f7ezi4~mPM?FK;j%8hB5&v{+Zp|^#psx8L{);$HU9SvT9`n@siTq*@E=ebT&x6XB
zdYl4m7Oy`dGBwa8TqpX+2a%A4{PJ1HzcD1098K~rZC7+Vv7dAD;gfPMK5{Qh1CnRO
zMQ&x3yx}B0;XqQ(yyu+)DO~~_(;8iij<Mt%5NqjrQFxF|g-vonn$g4qEL$9p@TR!V
z=r$}B2;9Qxv7r6q&Ix%+_kF}>@$9?!sj}<;I+X=|CcL^~_v0ftQw`I<#a{cSnfN_*
z+M2n?O4_C+_k^X-kzcn^Y~C4F8QUp79LcUP0?)Q|<*{7KyBMFTF=>^?C54H{3UZ^h
zzs$e*LiKygaaWV+Hi5jCe>~q|%lqGeXK$f{<FPZTC$2^9baX3P;@;)eY~jT@;b3af
zf%GW{!~!(L4ca-TIk@aNIAaSFr;nrbnyIoA4n8`&jB{I)n9F5ew;Rz%5_l%vIsNr~
z-Ny&9sl3Y>mwR1nYUNBjlOhqA7cn^_e&V!a9kQ_{lTLOnJt=JUaLvw>zxH;D`%N)O
zTlgs=@pDaLbjMu@FMb!629vt!lWNarvIHsYjIDZfY}To^|0`Trr0*<@b!+iu^>y5G
zP;Eh`To?bY+UJ=$jjdnKXP!D=q`=PZ<6x+(!P&!^*`~%h!BNu3p>>zzl4%E<7A@;s
zl&114dWl<>w$Fu~4zF0JCFGnsvf|m4=+2gX9O?zr7CjD$a#P`s{j1jZC}UsAEeV(D
zLWY9b5k~^-rX@?ZB<MWXTP3Bx?v{9DV5-gY)mL2{Mcu?h+$1y~8|o>ShsAEV%5v>w
zxCzq=x12-S0_}Mrw<jIFwkdG--<;WgoGzPLgDjS5`(C+Ua3<X&Bl4mSi@V1`n~QgR
zO4$8Rxd?sDc71Y`ZO3)TWAl7&TymbmuyoVO^cg4J=cQ>TXi08yl<Zmd|CKL8QAftn
zzGta%Y3<t%iu_o-y=SWSmb7Efc++zjBm?9QA5`;QBN{SiQvNKq8xgav&0#xxFT?H1
zlbSyr))$ZX_etA!O>*H`dQS4`grIBA5{j+{Pn}yLjw#4ap5>X?67kkXbB@WCP(8LV
zzqwM4x~>Tir>$$de$r*p4Bf>NTwhi^bq?rf{B){fLgSVv_hVLNnq9h<v(o?XQ`K!3
z-HJ1$AAMaJ+aswim9fKPUgXY3*(r`k{gRI6Io3`*@G4R#(dUIo2^*)!f$V9GEHMe$
zU1w5F-#uD*I<xE;$FFELqd3V1Psxg`m!WG6YcDKh(0XktV|*|!=U$g?&Y=Z2U-*jk
z{1;H{<~Kd};95(g%fSS$w*iW0SNwf-Tt_|DhjD7k{gshEu@gE=n&fLE(j@{{OU|4&
zEo=41q{+V-+0EQoc7Mw(ICAD+r<ZoZx&MFJN@~0IPnzv=Bmdz`eS5zH?shDkwGS4z
zJ(zKYFH3AP$FwsXLc)m)xZmb7a+Gy;E_&0M^+LxmZ+c9Mq>m%#jzg(O`r6z)BrUY=
z%S>+n;#*a^sIO(wS+n^1ml=$Le%=OAebY{KF5s9pQ`owL(b~a}JNd}_XBq3SG>eHW
zw=+BA^I(pE*4se6wV5KalibpibY4!HHKU7B*)1UPeSo5G#x$A6rD8!W;h$!mdK2+=
z#>VKlQ&Cpu|MxR81Z>;rsK9V(jpAt=rF+{J&fccEYnqqKorMar*F$ZNY<&9A??T(I
ziSG|Jw6i#HxZYuWtv8Run}Kslocpv!*`5W`Q?7PeonXspXz|fv-{SOqp11S_Jx%kR
zB%2drJKpK8Sl4Efr<#0q#|$lY+m;uS8C=uZG;<Cf6%EthJmEv><>)KSx;;)_GcIfO
zG-{MY+Fxl>TXU$3XR{hZfbn$yHNQUPM;Wg@r==~`BKAW=DSn2}3Q0Rlu_HDYk3UfQ
zE}5zSt5vk|OmSJ^>el!f4T2|b+%lSWW^JzK{FrQGw<`~h83uS=Dv0E+y2Pf<apdl+
z5I@mNJ}g4{DUy4FT<&Jg{-1k;f3Mpl*Ee(bac#+r5ufxxSbfFmPp5=Sc^|NN<TxCg
z#?<~SX-nFQ08TY=;S0GAOIN9huef(g?D8Uq6DekI-18pfWO20FJh^)(kMm>Dn^3vh
zfIDVcjM7_NszT23NV6DTKIG7INMeoi^tFfiq>ntEaYCZU>F<Hl;a8YEFEmT^Fzx9m
z^{rdZcP!hE>n&5)q-7~LgzP=PS``}ly}hD!ca}%$>>S-Ib5(nK^IiYk+K{<4tnH&H
z+wF)dk=wHhZ*w_FN0+6BZ`*is>93u!>F!6o7pCTaEu7le75>64hH0G~+iUB66NTPf
z)ZOBAd0Xp@qhdiCi`XrV?)x;@PrG{R-l6}Q$F4DWbG$H`s~ecZarIqG#?&VZxr-;i
znS0bIY-#ApO`@(F?v-BOl*5*yozwVs?&Qv%tsWPgjSipI$~Y8$gjLt0*ek%;dd3Oo
znxk4*n7TTcbZ1=dTyRwJI*WG3!TFXK1Js_UJFZW0eV<~eKGErULd5cKyI8-}#b-P%
zZpmL5E%WVxQX>O{ql?QyIl<%ECeiK3!(4Z0oAp-wKBxZYTfAE`r+VjSyIZUsJg)op
zU1nG{bKRz#n%V^SZHHtp$obuFVk<6Kwc#X(dHJC@m8y%q)pw%<jCbChv*C#2&YGJ}
zu{W4|Ot`f(nxf)M^*fxnxE#Xkk4E%=*Vxd`x7pdg#Pq-Qjq{c3nV--3>bb_L_P9I$
z_XYf~Pgwc}>t;B9aZJc&e<_eRBTiej*1_<StKn6xkVWo_J$q*)O*!_y<Kxu0yW)N6
zu3Oed&x$Q!V0C@{UEbX#M<D!PDc4*shIv-)E9d<BBjXZm*<Nx%xbr@jXK+Qa@v>CE
zgG$kE%-bEM-^~91NK)xj<c9Z>y64w9aS7|~Ze8iO-D`K;sx{n#Ck3o~nlz65d_JSx
zBcU>EhKFuP6T_43mG#2Y>^C!b{MI_6;=P7>>+~aL(ow6+ZY@tSUU$xHza8(T53GrI
zsuNgZCoVHiT64>~U*5qraq0QbVv5RTUDqV0vL#z5v+px3%k?_@=h*+VKPEaoF6`i6
zWnmR9<!G|mmBq>^>B0i_Kid|s$YBVO4HUfY@}-bP^Ufg^%>xN#CGu^XlPfmeyK`x6
zfCr~yc`MK5wm*}0CY&fyE_Xci>|*$l(_K%T)M}$|9CF&?6!KVS`+6ta4NTM49^w=4
z3A-OKr@PU}xOn>d!&+N*YOG;=7Erxs$(@?ZP3JbPs@Zx(;|2c}iH*??ypvQ4+Y344
zK3iwn7#hm8#p~a_>i0Bc*Q`*dyK@9=7H^NZw{)WY`6J@Ra~C?4J+)_9tn-g?l4k9d
zLf?wA=oYVEOT2tfa{Vji+R=9?H)iiDX_o01S+lvP+7>j%Pjj<RoBFI#V8@G_2DMrL
zOJ;kQ)UV>uRoc8g?xXQ!PQ$-1Po@QDt0^1*KM*)CfaSF1)#gx+`03eBx4%2z`t97A
z^3%%VzQl~*8*^MeeoXjy*L_dPuHE)7e`O~AO1+UX)mU$zeS(PWH^b#AvGLaWKNc~!
zFf=lAiisR}n&8wDvrETELttTp5!0>!6Ai(~M>?6zlb%!vrg%(LX7{>rr7KjMG1p_O
zpoiB|h9INZV=<na6lOMB6>>d@oN)HsJl5EMA`>>ntqn6w&hu1q>~vsdT`uMmFm(ab
znxM%XH!h`hHLY-79yWK^(cZP)>)vcARXw14XIn1oInf`}6ykO`|NQpVIwV(HcS()p
zo9o$AB&z=Xf3;CaH7#U))oPY2ua3{SsuA@!Yol<1<d&5BzaIVH+y%b8ylnfsH%9k?
z>8tLIyw`3k6}`E@qN2((N8!Qu7R^JQ{(=cIKR@ngHRs6a+hEDgcxh+O%5$v0552i5
z%I9(-Q=_8!k$#bq^1ek3elK<F-W>9Le4tO3y=+ze|3~jM45H7@*(u{NAxA00VS$_B
zDTl@z7M=}Zd?F3KfzviEof@U(?a2_m%|n5avy3}HMbqX@uF9HiLJEw0Q9E53to3#s
zoofAV(nZ(i6qAmlMglUg+ywrvGE`CE3_2~Qd2EF{g9>Ms&y=$gi(NNY{aLtNOjCdL
za+Ah&CcInDyZv6XK$|`2|E1$9$7MQwj5w`wg@yF3E-F|!YiV;nR-15W$L$_rhV!=E
zpDz11FWb@E?Z`hXNzsMB_l4rbyyg?u>-m(D7aB<{ZCc#%?DCX_jW=fd?+ZBnhMTid
zrJ+;QEG>z3$%4KjPsb($k&dei0)Cwg<>;3>u`G_gX{+PHKfAUr2%8k8wS4&!jTNgI
z>_rr`R7EPj-ArRW@XC4Zl37Y?8)FKa9b`QpG_macw98}d)MXDE_VF-&E?ZK<JxN?;
z@0Ug8D#hC`9GZTkN^pf-)Dn>e;+Occ-#$3c_0MR_X^y;#E!RZadbdpE;{K4tXe4)H
zeqG}49m!X7R<`qG?|yY5YG06?$F0@7qW)hqf6ySE_w2zWVU2)Ee{%d5xJ$RbQ_x#t
z-NEd%G5CFk-z2{(g-<7bzg7>hpH-#uZH?H9$zL`d`?ZK;+l%!F0*@ZcRGWL^hD$@B
z$ItxSnIbJGm&dB7tlD(S>`g&V^Snd5*KcCEwLnBnU#q!UQ|D;{Q^MPV!>$Ke3z~Vn
zZzs&@=2cTz7`yL7)Sl&X-*;)h7h~U)Iz^Xl--^>$Legh=>#=*uo(a6_GPz78_x&2N
zFeB|+9zoVBvy#S10SpW)-h}X3YF3o3TX0*Bb&KV5Ufn4HuClxC>Aw+Qw8dIaN3`R2
z&yo2a%YUpmXHzlB>ucu9(7XFIu17p(@6r%xG|h0I)1~!)+uQ?_{u-Dwyz0Fl!6wRR
z;_a8WsEcXIg|_WCycJYuIPFx}-e`2^VRXuah6|@2u!T=a<W{-xK*Z|-*9Mbzz2{!@
z9X^<@65DlJEN6m~u3zv*)6$p5Y?l`v(O9P1x3{l-{p_>+te>Qm#ROCS1@s<wx~3m}
zW3kuPjN@zF4BLF)WOFMCT@2o#(KtmjA(g4~)G4MLlNoH5DjZySv*1RDjd{}w%ZrKg
zN>(J#U69M(e=Bi{$DC^>_FOEs4-EMgOqt}neU>P_Stzx5otl7a6_<+81OcBv!V5O6
z4G7b=yr99-VAfbrr11ad5g(Z)lg&I<h-PiP5Ezog9aO+Enc>zc77>I0jYbDJJdBu5
zv*bi}8Qu649sPho@|J>0%a(<_vTN>$#ke$aE^&3UWeU>MS6Z^rXY;J<M^5TSEMyJ4
z@T@J{N|rzUxpT4Dk~xXLy5F))+I=UrO?0+Iv)HB;teP2$TtOa9cFGrA>#GD89GE=0
zlg%h#A_GH|u*7MRDHG;sdMI}D@8wwFWMtbN#$P=_ifgfelg7Ks^+9t^g(%;CEI4_Q
z{xlAOd+kdc+Lmr<Qq30@Slu;s)pUs?YQK|@)`ujm-qENZ!Bp6}IwEn~et~X>zpKwQ
zcuixt|8%(v^G30)OM=!m#a&fk)M?huS~5M%BG|P|;V{byg|O(f6TGjh=3G!(ey#cc
zjZb_rqMbWGwXWOR@S;^Qvw31xhQ#3yAL7|Kbq=yk+SseOvBM|ijnLbN+|@lkU4L)B
zyt%EwwIeH#DVpU)@RFmE22M{m&t=sWU;4?~U|N|<s*2}Fy-(71D+4W<cg`$wU@^IE
z#BY5?aMr~y924K`@J<k{oaIq>Xx@s>Q%mYQIXrorT66<8*>Bv#;j3SC(tB4+XvzZ>
z4elUMj)T7HT8^Bpw@zvYRb(g}n5^g5;2tJ%)m1<B27~ymOKtaS0+>=eCf3Y6*~)n#
zeBb#a>p9j|UeCQGT4L*HV8+g*ckybfg2SQ<5+)y)mMkl_%-qFco%@`-Y)`Ss$0Y@}
z?`NK`EeiZUKhEc)eW2N*y{sa~s-yGkgc4-T{~q(JDB_#9q#=1G=aH@U)28HXZ&=|d
zX&ET_md~$d+1_`NT6TeA^Bn`M9#5FVvdmnzW37jw{nRThqNX=L&1mIPn0`lJz5Zr=
z&|mQ@W-5v1Y%-m57G9NJp%IX;;F;j)v816$G$OWO=VATmD_y)bPgl47ejxHJddmi`
zgc#K`(Q)^eKG=Qh^wg*gm-Y*M+o;gx(&-W6yTX}G`nXeB*P9tRqU%?1R(mmSRtj=T
zdvB8+eO>DM_HA1uDr9vt{{D)(#I-GC(I)B13ppIrPjj^=qzT+$T7I0bH2K^=q2nz!
z=~j0S^X0$4p)rXg$hPi3OSxa0b<C5dSGFBG9!l0%C%D9^aHcF+GPU{ms=ss3DhM)Z
zt@e_<60&SkxW422_Y16&o5GH6jX99e_;fepE*}Ti?F!7C*PiYbO%<3~mcbf(=aJ<c
zgB=PDe4WjlOc@iV-&wbPgRAtGgl_-q&wLoVf6kmVY08urc_ND^EEjvz>908>V!osO
z#tY$lZd(8UJfErIW1f|X=rZNsFIoO~&+%x|U3z-k!V|l5=2iGiczROojfM87LZvE=
z9zCwNd^<bN&yvyXFIrU-)3N)CpV#{d3W|*WnVhTcxeJ&~mvs2w8R{SI)b-(5?}vRx
zuQnXI#<g_ar|=+;hP9DlAF_LoM1{HQ{@-9(zjm@Ox2sG7H@Ew9wxW$oMN>6eWv_f%
z8`hAm7t=fOO{Bi~&U3HbXZ_}Ga(})v-<EIL@2vJj;oUA*IS$Bekd$1NpZY`llJT`I
z*Vd$^pS@{)de*hg$Lzi>71iDNr%ESO&@9XP^wck}ZYnUTdPwkdykEoNuTsub)W7&`
zRF}|`e@c<}aut_G|CjE)-Ncru#I)lO?}P*HHg!&n?oIDr@Tqvbn#khBb-?Yzo4lN?
zqv{qXC*L`?JHkno@4beG%T%^SV&}T^uVnB2<HpF8z#9GGlS4jt4x@gRi|#F^<`fpU
z0s}V|1@`14?9~@s#rT?77@B#UJY;%`qgXgS*Ug&oU*J#-i-*kd!mJNp)t2*gb@Hrf
zzF5VRx{q`2q$A=Rjy!viDeT7htLog55T2vgF1+wt!`sdGlFg54l@p(R%7Y2L4_0XX
zJ$IgolWAd<PFmmdh3`@XN_9>&<t&<z&MdR)1efZ`UF<^cO>b3>o_umnNONh;vGh~F
zKKv14;NW|a9k7TuV(IAz&y;qpiC}qeU-a}$UX#j&HKKq0G{o{X+yd{h=}%p@!a?Qy
zW+$fe4QG9MI2tWlB)Tt3#Io_<e;;L*z<xbRPws?Xg#NdA6Q$lSTHV(4eUo6EN4`hG
ziC8I~AGZ!nH(62C*8DHaSNjOlbu-0H0f&BSX?G|1{_5jfP@;SN-T#YnrM!Qv`ac&P
zmiO0g;K=&CBJYo$zRIx^S~~nHSG+le6d39zDliyYywKma<gC)+b?R$)6<@9BTi0xD
z_bKB_UjGk|JLe8F9L~LC@@l!kBO9|r6Rs>)5@@=cv?;i+O+h7r+v~$5i;25D4ymnJ
zonzy9-)W`et!K>5s;Vv%!lF)U|7cVAazWw#dDid(^A(?1FY5bobxl)A;Cge2qt<}q
z@ga8cJOvxRLp=tw&wV<0P3TETpQ{WzSNwY}W%qa2Qkl#gnA8;XMHBe8Z<_XFlK$*G
z7bf8a32Tqk^}V<gVEAwYZ*QM|{F_%kNBKT3<NGP_YSE{ci}WM|4IFs1w5yr?Zv6MR
z<oIYI)pcC1Yhgo<j>s$jO)o6W5+d1D&15gWnRe#3QmR4?TX4uFA^u0z4qCx>P3n8t
zcYac@z2v-<PviZG1+s1*H6jXCp1C{3T-?KQNUf<!J?B;3itc5bPOEEeP!DiYD?0ko
z?ZW$I37-@;X+3-+k=Q9=%kR3v@71E9mrGjp-+n$PlE7@%CvESp=i>fENXKFOMum#?
zI-ZBV9tm{USXv<V(eGUfpW|!0txitbTYl?=$bOzExAXn4cN2dd@c*+f|Bt|?KVBv-
zM_;%sWL5A@StWAGh<B3l{7bH!PFsI`>}EQ!QsL6MlxL1-Ct75&rT=45{@h<##kRe`
zdq&f;jH3VE)*Me2r#;)g?{U9}5|gi1{gPvM_!dpHTB@*wVV6bVStlk9`<3sv_^9<w
z`Y<JlCD2LL%%$;66YKg*?r(aGtdIDWhxkoB*Ua*WZ_Wgc=@a}N5BZ;+r0f2EZ$*Iu
z-&NN;3q&Syo@(j5`0kVbUn3{R%|ZHCgE)hO*(duw)IHX-?&z)zBh|?vQ}v9!OO5r5
z`9B*dE)>#AEeSnnqF9_3ndi(k#bM#3<pS&*-5eL)EJzcS7CgCzUC2F1sC#Smq2nu-
z^h8eP-pR4()Q?3Wam#q8DV<?iug=)N{LmSN4WVaJ1Y^$TsQx@;$=e@Z6u2iR$yvi+
z(v?Zut^0bD?v_)FTwecwqZJyUrM5t%%i!Q2;gT&ARXUDHb06YO$dZvP;g*^5TSLU&
z^h)AILwVOJ4P2|-7B9W#utodNME#&CcGuq<KWACJD2TP4{h!yh*oXP4`#S!*7)yF4
znb%znQ7ZZ;H${oD@YaEMN}X3;_gzu63i90fR$26rTE4OO!L@fzF{_+On_!ZaQJt+a
z`N7Ub+%~DAE-s!bNm>zn{UQ;Lnrkn|-r1u6A|z)$pOX90Mj7^}R_cM*%>tTT^=>!l
zt!`%iRHUET8KfI(z$$E@X_~KWk{>@sD{YIg{^yXDs|wh^7YI);(Dye^4KbFUo>jY1
z<(^7Pweunt2ksv(Jt4xTzQ*Cz|9Qe2lTBND-XvW%?e&Q`-)B}iS>+aA(7ki!u3kzV
zmxKDaSFB)*nlHLj;X}Dy;L8nf-mef2+r;ys<GN^XxJuom<*T$OZwd>S$hRq7YV&$o
zWziNl^(}UiW@*gkVV1!P`bl5qP8@8z5YPGLJ6D*aXJSm(F~{W!POd3VkHwwUORmdZ
zyIzx9{C}d`(yn^%Z(AM+H{AA5S*!YoD=W30x6$ZZ{r~U=X^BP~v-3V;Zo0;9nloJ1
zDb;3$+*Au^GQV!Vr6aBWi^ZGs|1wUuFg~sPrXMs<*|V5isV!~$F5k>WB@F&W8lTb`
zT^A`^aG861F+J7YrL^7Wm`IJq%1K%RzNP<NeYIPvY9A>+5ALuIQz{eCyLm0v<Q%V6
zho0k#0I3pv!>Pev#d<zo%Rg;l^fb88e`;u=XhHR%P|XUJxUIdv7d0)k=wfs^-ou=>
zQ1y64X?UUOYk|N36Y<i1ozjUsk?n^nCxupLZ8tX!JoRF&_5!g`7s1YWrpP+g6)P@>
zvo%E@-5h;*s(!{s^;4xQ!kA*?vlPq}-@owba^04E+0f5JCAGxF?^M&i7pyTCw(}fG
z-QK}m_xZAua>=zv+po9$bSgdTl6JPa(bvw=IOVpecD!UfXGf}YS!4W!6!$Q#^;t>#
zZ>D%oiws&_e|qMFDQm-{%lg!Rmd~@8G~;GSl5R`x|7mGjpVHE^RZNQdHodia$I-vb
z_ifH(g<ly9a&Ii)|De>Pv>?|gBT!}YjW2!GO3}_YSA2M@>hki#mTOj*S)H#m@ysz=
z^f`H-n(K^roAORp1ph7z4*V7L-z-dgQ^@-d<$4mU<jhy`_!V5`E-JTDdG%xFTRn-p
z60S9B9@aA!emYV-`Bs0c&%bRON>1`wa<MAuc%AGCRjIh)Y5OYh5POK#p|WtJDDI?o
z#gh|%m~%M^F7wcgnQQzY?xOxOv->M1D}+2PD`bj1@cHAdg}X0pUSQH+omi>h+&)8c
z+J>jw6OFfQFf^GJ`ucaArPk>c3R_t>FiL#!nP$8`a#rj&C6j9#H~v>L@6&Bs95m;s
zq}ytyW@c-}ja^%kUv@82VlXi`a$a=e*+-e|u$J2^db@75r2h6$d}`V9thM&FS=x>5
zSr*>HQafK;mGd52RVfh^!o|2pB&Vf9{lAjNsb#s(szjSvt<;y~DeU6$yRfl!hgNh|
z&%qt}$=0EZ{Tlhodv*N_cbE4uvuvK;Z2Wr1^2ZSc&bRdA$`p*s=O`DycDUZpXc;hV
z=}jMTOEJM!`HKs*E}E^~RarDyNO|Jw$IBvTi%wqBz`Le2+{1IvzHSZYoy(j!r~Ku!
zQJ32NKz(O>6YsjKQxsID8s8L+^K8tYRkYb@)9QEP)uLO<e6=T4@`alGvDm%Q_W$Sg
zm5-H9P19M{SG2lvQ-4g{Rq1I5EjPG}#e1JVHTh4n<F`*?&s+?C*Xie<S)O#Jy?w5J
zQvcpVGE4cLa;3LxUz)el_y3IJw_a!{T20!%O<9Qlm&)n4HcYV*PKS1KABvDv&3fLu
zpZUDPm1mXRM{{!)WqTGZd%F8{q0MG<{{5VnL$-&%x>dbTR%-wA;66{;zW85PuHIzY
ze|`P^h!x>4mcKox!_mZgU8b0qX;I4N;=eNvEY$V*ex$^Ad;7}eW^;X1Ip+$s2P^J7
zuAZ{JEZtmKy5K_b4t1YJQPYzul=p_7=y7N=uUO7~=Fn%gqN5h9E}`$*JC~lY3VpQ4
zC`>EKwrJ-68p~~msyD9M^wKtEzN^CaM-8)Q_AcDydN@+@^35#|@*|6wPp*`ly;<vi
zu~nb%?%2q`ch&zzojLSKGj5vkzD=Bu%&M5@?|B!P@cHcBIZCf3xBE*dJ3F=3{#J;x
zXxrJmBY9qCL6iElGS&UF6zUGGvR<SVqIK`qav$-(`VXp(@V|f0aL!3bMQz@pmqFI~
z4`MgxTJ00RekJ!--|y%b*R!u|-e=7C+ePhL)SI%P>rYoYF{Le1Ft_eHUUc<9^;}cI
z8ifS0s-n*)GA`e!UbJ{Z=-Vlw?@SDDm2UU(W4*oiT6U=6WX|(PGFMGjw4HLnCv-{i
z?%1WOW;^dVE3HjFx-`deedzzl2Y3GZK5*Iebg!Lq=V{wo?FW{#e&4!h8~o@$(^uQt
zS;o?@k}V6i@A+!E<6Ch3XQj9bm3j9c^7?6gdlYzjv-rMQ<uyT-o<Geqn!Or$ycX~*
zkpFekflGwp#(KVkOBI;Bn9@p=y4~bjG$*<H$K7f>bnD%m`J#uc)+qOHc)nx#mj1qK
zcZpdHoCYdN2~5FFFLusTImM>HGC@!wLF8aF2kRCghgS;Pj9a5_dRYWE2yuF<Xn361
zxVV9Z-Ms3F#;3=Q6O@DJ#dr!VJvB|o_s|^8&C9%7%tN0odAaGy>G=-zyJVz{S6p1I
z@7)!$V2VQUaxZm_308?r?5l$<3N$xw<q7}4K6lTwLgj5|!;EB@d<?Bx&rEA%T9Px-
z$n~U%rtq6bTeTCM?oSVX#>Ju2sxbWk6JN2XP3p!+)1&#^%xVfgrfg?9sUvn^YJz0r
zoSg@r)_mCfym|5N1#Nd$iLSmTcT*;(IAEI-BR}f}wH?vs2gCLT>us4^z0mm{XXbWA
z(}FMO!#3S~5+)JoVBIGnXy^LNsL}tO)_&!?k-`fn^|@r0`D@y#RxZ|?y5)Vvyo6dt
zPp5e?t1ho#`FhK8>HG#3t^*;#>{2Td;zJ{kwuZ?T1UiYUP6>S2DwfvquuUrKhHLeI
zPJzfex!g@hJA*tXJn9l&@?(0uu+PKm2`1~diaDAWZE5_U!v5CdW?xLsLB}M1t7Au#
z=E!79OqyL-G+{D}<I9T~%NmbeOzR9?rs^k@^JrEfZ=?FN8Tu#PPH7ZOOp115*1d8&
zPsmZ_xzk#Yg%19POWoCi-tKL`H?N*))`a;@Vz&f4oBvtPU-&EcN{AC@Ub;f4aL$CQ
zVk$XHS4Jd-zL?56H!9*#tLhAg#FZjt28o=qGqMh?5Ywxgz!IqPMa-$?YG`=e?pvYp
zeTA&i5s7AUD%VA(<z19W>Pzi>>Oav`dhO<OS!Nf)m((uP&D6L2K3ODX%eD*7$L^-R
zb9}0FgiZ6M(3e@ePw<w%^H}r7Y;Jy@){<pY-58I{I0;=97R=e4RwDm@3FE&d7u}?D
z3ZL{zw^{^M_c@)4Z{&>bnd{Cjf5q_F8{QQK72K0FvfUMxtR@{{*|;X*C~KFx{GmsI
zcV3*f&|VPO%$v1TwB4ax;>!hDsjZGeb-db(M4~trw4T!AFL@NB74~)!i`Ri4N%2Nv
zufIAT*k|%&!dbbMOKxTacTbp-ZU53tJ>|gJGWQ5A&q${jl?|-BYsyR3dnFoiWk}Di
z5c!e6LscZuwQKo`O$()ER<<0rd!$mah~bmS3g61Sc_Ghb1huX_t!?Tv@Kmzjb@x@R
z@6)PKnH1HB%VG<o+{DwSFlMzb`&74zbJe0>S|1jF)2ms@G((|*TSP9!Ewimw)%8D9
zYj23HQ&ffZjhIr;iCfoKK5c#c^{Rj4jGqVD(jF&XG`f9bT4b3?<ex9y&m__oR;op1
zgnnBuG;Q(4xR|KRZ7=`yDnId(aXch4=l-5f<s}&kj@McyFbS=Rd75-|KZ}I)ix1tr
zYj>V26-k<-n{vc%&Er1rB^w$O3m;Y)?w-SQ-FTV%RCm>W?noZXNh}lZELZvdGx1f;
z4Z*Ipd74TOBj417sVuS#%m_Flu)X2YexnJE+EJX7mPWOhvT7(YG6-C(pCzUwv8_j;
zRrr9!1&ghsp>LJG%@n*IC9+t8_gLT2Ll?cZwo7c9yVk=cX_5k~!qT0+Dh;c53jLWk
zb%M%-ho-#$E${gASvVVhZR44ka>9{sy8aW56s4^*CNtz1+S)|=`Twh2$Py5F<aFn{
zNsk+sF3f4$6{Yt=vv1L<kWEcfm8Uuc@BiZUuITwnqa6y)yPZW>hb=u5R+?CG-e%tF
zDUnP~4O}5{g<_l)CmA_keDdSsaIiS&;m5#oV2-}xM6pEyGoIb>($frT`d;yMZ|ud4
zn!=slzb;MJNzr|6n7UYgi;K(J2CFvhSvRLSEw@<evdo2Pv4D6%QSy4WOE1mDUFWC?
zZ(4V^;F9^lg-i0)I2>;l1X!Gw<mJd#G$>SjlA-l$&hfNa0@F4YaIHQv>qv~7{FQ@e
z9NbvE&bD>U?br25=JwM1?|*bl$c(dRrm0Sh2uW9M_;2ybA>!<-MzswL^KuV3+Aj_e
zIsWU&0>-7YWG+toYq6nK_gFBar-wrf^MnS5IiKe+oLC+A{nT{>xh0t_j5-N|n_5kT
z$}&S&ud3BKbYtzsjoGDtq5|4jRN7UKC5Cv+zRC1+vXjuK&GSkpDSk{-aT8META$gf
zqPp`y--fVh9*ND4ViIa>i?kJwbxw9ud}J}_t>&tB&M>BFTpG4}H`#9#S@p7K^`S*u
zgi`E$SDoN+5@pk!SeCRiN%Gc(R{zrQ@+8xF74`1-9=R6JTIY4oc&>WG^G?<ocWUOA
zAMp&|HA|Qw!6v=#<LUcFzqaqMRMiMb_`id5#{vzb)`PRYm;BoMfp=NWVW;~E9L6)m
zq>k10Ro=fFWVC;m;Fidr+-J9LbCPnI*QTs?Oxn@pfZl_OmY8ijGuW=T-o6*GK{;FL
zoLaf}rHMHk;<s=naAaTX>%MfXMQ8PSzoS<Syd%AokF>5W*;#D#dBV#b-A_FPoP##M
z%zC~mVcrz;-M2QYMR}}|xi$CQiBr$G9Q{@QS@Sx{Kk?T)b=<nx*FoSN7t7vnmh0=;
z92T(8z0UII=7maEhxx1z!sU)0b&txpm7u!q*qIIv*UqgT9p?^bzB(tUKjCduufk$g
z`&qXPa-QZ)vYE3)qx8rGCYj!L^D`%OYck`4+Sbneu~hH>oj#F-1tLpzw@qLOF7n&S
zWOd%Ht^28TU#HEJg->6)iSK&;??{@xYrwg@vW+o*tlIX>&%$=yp2{_GsnbqxwfA02
zt;;rZ?R^_2_o3?fnXFURm2Y*MtJA-qRt-B^x|-*?#oN{K4FA7BVDq<FY&?AeL(S>T
zLQki}y{6);PJUMU`Ou{KZRK>a(m8r@ua_@n6W+}u9g@3b-ZjU}^0hfTZO*lqXgHk^
z&}+I|X7)L&SEcxM_H`?lIQ#YC*_u`**K>+u)oi=6*o7X4ZP^m*VWxb%YSoR2F-z>!
ze`}U3uQ~SO)Rwku+lsDkt}$8@csD8Gm{!34y9zfA4bSHGS-0CXg)v_FFRS=>chbb9
zaChswk>{W8(O&S8E5rV|DMRDE>_@!kGZbWweALV=aZgxb!ZO)Puk&f<a#`tDCvF*P
zSS!um-15X=ORm@vt&cgkkELu+cNLkmv#<8itp>HY=*+D<Qe!kYjdFGt?F~45HS$BF
zeA)9;^GdFqO+A)3^AD%TfuNkLr^2(stvAk4+;f8G{rjf&xX`VuLJtM6-*a8N;hoxf
z!@tiCgs%JK`-`t;{|A4IL(k90Hz_<&=q)?>_ig|GK97C>wE`txxhUlYX0og41Tect
z9M-PiD7HsQ`G}H4$3|Q8PZG_B*-WJoTfQaEnkX?>Flqk8q);c-ZWj9jMkkgQ2hNKE
z|G5P%ct4u+d=dD*Nyhe>zU?w;rK5u8#{}Mqi0HL>3Y(<3mg%JwIEg80wjL6R4M{zI
zy`@G-sHnsx`*^5Vh>EO(Vuo6Il!mKpV0r0v`=twXXC<2DU3D!C&G8H<X#bwxvAwO0
zouO-5Mhjbey;wWbhWhdffr^fHj*A&}FX|a)2-Jy6eqPLZ@R0LoW63B7)8y|$r6%qR
z&9bx(x-&7TO!ZJ%CZ@aA++)@E$ec@pp^pS5UL@8mj^doy?3dxmVHj<4P4Mlc=xrCc
z7K(UrCvwjY$kBEbv3w|OC=@d-+56(N)crSNu5K%3oRk~9tkf{j@XU+oORUQGx4CRO
zEPC)t?Em|zTH7~kt(v6u<(bwwll=2n7@wxc9iD7+VzJG&5Gk=k`TR+u|J6hvxWxYt
zE%*}N&)LrKzg(o6L!(BcgQ-m*DBL#4fNi@|2h&UzgDaU|L;QJfh%1<9nHd$mT3p0B
zwb3zAlGQZ*#-@N!kzx%t1L4x*omZuIe(gH>G3ulugSAEp!{SNLH_2EY({q|$@|Q`<
z_j=IPMk%)+hGEx@bhfb8HTJ}Qub#Ed`Ludj@6;)Zjiqj`sU9l~CvGZ>a@7pwP>hTS
zi;ggHJUDUTL*rc5#<Ym=Qq_vOW9j)C8R?eoHCr>77%G(~_Sd`!uU{ebb4P*0#SE{-
zCcerN+b&HH7Z&+%aH(?Q40nf(%F|xBr#&{9B@v*n7`5Q*jKxbmCjFFNHp6o*tK&)y
zHGait?FfNAf@%qi0(YsZFHGTHvcW55lg8O+f<6}B*QU8_uv7?A?1^&Lj2F$j{5>v7
zG3@vg>!XKjZ-38w;BIr@z3)+bK6_)@i|c)Fo?Cs?n8x7N|H*vr=koTt*7n|Q{eQO4
ztvoT8<6^(e!nt=jH3G{zTDG!IHPe}<&MNubaQ3l=U6&emDk)neCH^Z(6hAiO>t`W3
zF<sden#yHT%GZjRT2xmhW|lt=%DtecpXgw1p~vOetoy=~XH~P?M*;h1&5mMf8aMd1
z8MOQiYKhy@;@(lpRb0yXQ0ae@P;kQ|VVlmJ?Nhu{!lnuprA2U-M|hSm_>neilIw)0
z<prTRMXA$%9BnJ^$Z-AD<{{Ny;vsPJq-^QUcE`Z!jB0A3x7wRM1fDDwYBSaOA2^RQ
zaCVbqWP-9;6q5>nQ>S-Ol(&;d*<{tZJF5Qtlv$QxF;i3H;K43gN9pa;9Ojn@Y_afK
z&MEX$amwW|L-B>i1`|yU7IvGn2%WMJm0X;A>6^ypn^sYW1$Vh<UKdroaziv_V%{&d
zxYI6i4}Ro5?1)>rYt{NmeNWkKSPE<RPm%chtna=_{CoGgyR!NdQl{UR6!}|GfAH1n
zN17Uktd?>XuCY)o<rHL{<krx5bo!f<^ZwslI3ex$gt~_fCX?1yO;P^0MCVQLL<tL_
z&L4@7c1={-7O?S};KpY`5>LB!r^>t+(ziVxDEK96L1**JMQYP_CUb9TK7Wvl*>RGb
zR(DR3Q-aqNlcP1dpBJTn7xGf;30PSsqP^IQCCt>zB&%$3z>xrs!tx7?1FFhZazfia
z35FMnvix4!R@EBr&`{APwz-y#rLC>Mkb}jiYg(_kn~iiv*~)O4iT!=%Tl&9GbCs?v
z(e&=UruyHbqkV=^t7`zqv<<R~%jT&^wpMl~2L~)U9=Uva<J=cB7C!B?Kj;y2K#${N
z)RdM<b&rcTd==QD?!YT3Gx5ms{R^v?Sgw##R10<6`k%{Nq58JfCdt_nn+3N@_HO#@
zeUN3<eOIx#gEsp-S6yP6bJ%(3(nqx~r-rTCI7h~T^(<%qXVq!nMC<qLTFszlaWt#u
z<f`_@9qKnc)dIT&I*M#3iAnlSPPnQp!6aG8$>=T?C~k2`{OBd8XU38znv{iJbU*B>
zTCCBfx})fq@B&pgb5)I*ZtWtL77H0JcbS*V+{)IU+u3Ydrmz0AOIk|yk&$3Yl1#2Z
zc(}T8aB8X9mnqeUCVQ}mo(&f@YTD?yIPG7z4@1vl$wL8g6$0@m7Kf#n*-K3;ncC)=
z<<}n8F4)uWv1^HtOhR|qX7exgHrDe(q&kW-xA<{&IDFitEVJeR29?U@qk=Un8wI{k
z_bA*NuHE_l$RX8;$mP$r)*n$>cT8~NPW27xB01H|<t8ql{l&bDTfac4n`L8l)2v{I
zMe1fpW}W*{%2~XUd(+B}?%A`GtZ!^dJ1U(2u`Di$$M<Q*u@jy7JA>;sY305C-e>i=
z?vur?WzoByK3VPZNBz{S_O3Dkj|nWE0*sa^YqS{W*%z*9o3n_+knO5*g!-pLQMcv=
zmzUm(IvKNgt$^mllS=C})YWEPVx1+lm%+-FnbCposH2;?tc1(V<Bi!Lxz>xa=x=?f
z$78jgJ0<Y{Fa5tt8-8C>dn+L5mm`~-px*l<dS&zez{=1QJZU!0jPtV7nD%V^A9=D>
z{&uV4@1nHBVYTAZjxO8u=V*Gz^)^?n&1T)3pT(Ses?*+NcId~93`Vu--`Te)FbbKy
zUM(wASzmEJ=+>^>PKI-<IFk*ymN;;&Il#B3z;@&7X}@chR_biE547NzmT)I<_5_b*
zEtMkC(PqoGAD-*8t?1X`qAr2+N|PHF3tcum!u2G1EzgmC51Y8BI7m)izoT*c<_NCK
zn~h&88cyGJx#jfbqA!<CzlvY^TJuh$HulYq>~A}6rN{Zy?wtDj%B7Mx_E$_NlYJJy
z(%JfJXT;OGAFjvCVvhehvrF}kVCNaxN0HameXcneaIFd8UVrggzAal$=Wzv}Ye}^t
z8>Q!!&lULJy!8;%0Y3qCrGkyS-~7Bj``Nq%<vk^e^KWV;9<Oe+ShVL%&_wC5Z898t
z7s@0}?G<4yshO*LdT#8(?~{_*U6PrytHq>F%eLwjP6~{g#qfT1s{0I~BY!S$=yVCb
z7|O$B?c37&*E{Upv9xtr<!mu`)Q{=f{!(!06fe8vT)8d%2kYIcuna+;yCOZC`Ew3R
z$hdWtt+`WsJ)QU7-@<x@hvD7Rc4f*QQrvT%yPaWHz=aRF7v2PLFTSw)Pt6v&jrYFW
z3T5owrTp-EL#S<_f&Gl|GY*McMRl{p-wIl8l4xJKOl7x9_swN%pIyug){v0exBP_c
zl$NfkTe4*)>TTocKKAg)|BW6w+!|N3{4R1#T-kS3++}ZF48y5|PmU%v`fj*%WtEl9
z(nEOxy>ZVZRv$|#7dh0If2WQ~-7n$q)&H*7dGEG-KC`QO4QqifH{0E7ZyMyE<XRp6
zA=I`;{Y9^(&P4gj%7{tJmNv^qY!zaiWgh(X;M#(HHymOU1uK=UohF{)mh+1?liF#o
zoH6mwCJD8$(|1o7YrZ|TY0o|uWyj5h`s;cvdgfW|T~o4$$6&vX`g0>zt_MmHJ>K*B
zLTwJ;&Nn~1<e!j&b~@9<vjQ{3<3yH}Y)${ldd}qcCReV_H*`1euzQ`mv*6jC^EH27
zAL`n?Kd;_n!|TKM=1vg5Ry6Y($HX^k8{7Yfa&2jvasAC&q1v{n$ai}S7hVYaC4A-H
zn-g7b{H~$@PpolI+}ad*c!7JsLUo3e<j7e^FRpl=Rm^-z?A?s)&rMnr9UU(|{C7%Y
z#SFC*jgtgZ^wu0WGDA=Dpt}3KXPFxwD|Fo!&0P??Fzc##P~7C-v#0)^eW-KQ+Ffa<
zoPFH#R-cpH^_6w558t(KF`f_K&VAIot98y62SL`w0o<!U2y79sn3Zrydwxt3Q*N2B
zz~n%QZjY>%+9>u$4ff=+=7ZuqE8KNA*&cbMeA-F!*y{O@UmaTfW4`>4#)dqR`LXL4
z-F+`(r+?>kQ@fnBd!WPVg)+L!_I=*OX|>qpd`)ZkiH#NZ2mUYg5LiA>U|FG9<d><s
zmtNJY-*t4jd&SpJaLtdVWq0{=+V|zXQHhyey{G@KuYg7L8xv_(K>^ml2A0~MpK=%P
zANDw}vgiJ<x%Ym*J)UgfByoi=xq;>WzF*7Z-x~HGx+m%OjBkS6M%H*QwHJ98O=Zk%
zA7%WWdAMOq<mzY3Jc=GRT{bV%@|5cnxUBEJ<)eUlslIaEb~&yU0vEYbKFD7+_gU<v
z^(;Q_tD4Q%ZU2tE?0p=ediL{5*#&jN4vg<RRu}vD#qE7^`Q;M^MTZ9tO{~0fE;BAD
z2rvd7%c%NMFhxw$L4ae&ih##%eX`bRIx`%Cbs5)sO%gKjYM92tAjHGsDe(V^ceA1C
zC6R{U2i|k6xZ_wV1DhN)nYVJ>3EHrtVF}yf92u>Grb(JiY92F{o@QNL8_Mpt!YbAI
z#zv-{92T?1-bQVeJFgacy0s@}7t=|eJ>DPI9NL}AGE3*>#H$Ama<Gf%h0geJ%&cCP
zJBO!Y!IMS}&!=B+tP+);!lbl)^VPm3&koF1o1)nmRho5eb?kOO-2?fDgxLSydp2n{
zvp8SyI<J653Z6`ljxcX=;d7|`#NxofklA-YwdB<$!H6Sue}8>`Az>Kxg-_&9@kd8i
zoAsOr_As<@UKH<P2%J3a=hv+{mzr1_4*X>|UM0U`Kf`%pj$L#3Lf$R<%OY|?`5&Lq
zhP{9P3*AWR{<&m=QYS}S@jAzEY#}F{*{!qIFA(zfS+ufwTZERDW{8N!DnZ^^2eek#
z92V4JJXBP`%vfX7-MqEsNmG|nv9plpmZIctWk0s6O^Y?Gg>#OztWfbVY}ydxqoB(h
zw8mJES^A91WF_enVh22wI(RrwX&i3R^U^rj!sqqkfS=XZGoQ~{JzwL?Ak0#rc<y#e
z@@f`6s|O9uoN{v=7&%1(6dJs=Q_rdT8b2)w_Mfc$!BN{?b?RabGfCyHK)1x!(4dH!
z>kkRlf8t*jlzI1LSYwWm>6O`6n8S8WTN2z69<z7%)l)N;NG#USO}OK9bg@9xE6v6!
z9a@}{tN~67H2<%-C3r__`HUvlqV=~X9xd7XsOfGI`>kx<%>qG&_qJb9VqRDCHY;mg
zW5Sz`o@0EId^YKOyS{wT6qWh-^kKfBH5+!{kl4uErvJG|q}$op*w;@@;V{e9H7%Fd
z8J)P{YcA6mtYGn|i6ddxjM>qA9W$+$RPLHp9=6{3+mYAnZIi2Y!@>o)g$fns^q7XJ
zw<%0Y;VFxlx$D)zEfT4k@srP*DnFmVYMr@``Axy@dBQE3*BY;gnW}9pH?6YW@N`<{
zg4~Hp9&VhI1QLYztS${WCbjg`#sb;OD<_UuHvSP@tK0G9t&ZP}kfNfK*JstdX^Qal
zX*hW7jnP5_#cOv?>9dP9JTm(4t;DN((rNQk1@ng{3;i6kUo!<Vo|&<hM^B`5i@CdU
z@0TF&{`V$7{im~+9=tTa-A<8Zl7e$+SXN@YSXjo)S)yT!ZZs`@ar}NpMvsZ;wadFW
zAFT}AVf|_47nVs9hZm&xZu+gDxoF4l1$o?3(?d5TDX$Y)62pF{DdOH#haD!K`qF`c
zEHgJvV>@<KVMb@T)3yf5)-~%?x)z>EaZ*2Ypy8xry3!0!lQu)XtR<x)-9Fk$>1vla
z8bnN+E-x(DJo{cls+NwnA5)(&gA!L^+Nqg*MGGdosAaWVv7F+on6pFs&4lOH4l|=G
zHvO<`s(G6cwdQu|ju|G&69Y9DZ<u%W#>1umO~q6>d6EQLPGzj_-gCt2x#OaCmoH}8
zw#WOUKMUO6_U-7R!a%h~msyUFEF61%UTZQuj_Os(+;QmbiAi!F1s^L+SMiH7XlCyW
za&)S6YE(H>BE=}{v|`GsQ)fE76;(PsHNp%cX0`BeC0t@@Ti)Q|`JiHkRiAg0rpr^&
z$(83G&-&>lYX2!xlQTob^?SxfzW15=s<qXN{in?I={FEo%zu5dV@BD80;wf)j(<{f
z+%mQ0Z>A!b$jbSF3sfg87Ez62-57Rno#}){I<B$SQVRXM7Oi7`w<4Cg@cN!Ntom*L
zPA~l2v|^7Qhv(ie+;M-MWZgx|CiLV@DVeYBDZS&1s+Q&d9%GT>2}fP3riW$tw5iH;
zXnQEiJQOhIFxz|c?1bXr=^wUOdaaS%sIy64ze+HN<>Q1q6|8c4=a$U&YJRn1Qr4EQ
z4_Da=9_UwFQ6>;{A)VXf#<J#nW}&WGE&a!tIvZYGJ38w~?;N>I0U;rU>kl)$xr8bL
zTfgjT-TUgQg2ArWroszM83nF+HE+JCQtRRAUFE9yHET}Zv?Gi=0>pH88+JdLVRGtH
z;kxD(IjNfKQx0D+6D}1=R!y*Z?Z|i_PhqhpXJKEaPcq-8ncI)%zSK)nTyrtWP}g^=
zS5(lxJI}gy+>V(QeS6mG5~Y+mtd30#*KVE@4>gZUak$E6K5@NaiumsT$v2$J4@&zp
z_H5tN6T^A`%W?ljAB3In%}NMbr4-L}%R}p)lsA*~?E9KqRyXm@5fqh}C2~UV%X&W5
z5BpC9IQLe3NsQyVd)SICvB@^An3Z|5N4QplnAVcg-p<5~tc!dHwj6E~T5!%kc+R7j
z9ABNi-}p)VZg%}_`87Qw^>z8GsClhA*SEQ|3fwu;VKTji!-bb?(*>4Ei>Br<c`x2@
z!8xN%N9DXn2Ah*s;{=V+%~dWFEX@R4%ABtM%;i`wu}L8Q>YeNF*4;c?TWcK8>2z`7
zm6`c<&iN}<X5Nt2E!)_;H^sfhWr=RV!V7bss5G2>9(k#+%WK1v6I$*TnOxO=itAPW
zFHBGHnmK)wromc?b2$@Vu67kGcN4!kf91|&XP&n$*&{m1f0pUQtsD%ecd%=znCIsv
zf3I*^8R+|D?M?I8*ZW>XO7~1FJ2-oNM&tKg$DiqKnk^fbqCZ=?#7xKW*U5Wzv$Xed
z><x=J;ySfRbm@ZU*Yu-gv;&#Tm)vDH^KcU0l+fZ(Wz=DK=JOlh%=3XfOF4g<r-b|v
zpMGnR+?wvCEHOfNnbzvAW%bIge_&x3_Wy{(<=ai)H@$tljB)l%{f`%p_s`Pl-XI*h
zSU`V@Ws6zyy(5qH&liPuMqi8V<WOJgck9ooGxj{5j@;*)j8}cR^=!fCsV}nacHIv!
z75H?hT6y+`+jIYa-cz_XGdJo*sZLVujdU5-s=FM5m-Z-_m+dgf^qeW`@XYPVO%boC
zEmQwZ^0e8d8ag{*i_TS(wx@9uqV|L+s7U&Xv_+Q(dmpUcns8KNgU^p$w=b%`sYw@E
z)L`dpXjk>@;G?<?5ffBg9HZZFSy#5vy6WQef4fh){e9`Fe~<UkQibw^JZ~&*s@X&5
zYF>Dy?piXx`t>651jae1YIi2R+cc%x{pZ(f^|vnxGCp40cs@VFQ6=Ji?eR$^*Y<B;
zDx3CF=#h2n{MsImw@e>eW7phYa=|BqCE*dLx5K)8Qx)eKyCzBszjc&-KWox;wph&#
ztE$)DXqmss{>ST$dO|y+xz&vi{Z}@eyyGu7|ML!|PXT@F4_eN)-Xxtoe~y8`rdguv
zFZQv!tYR|}oO@Yd-r^2(>E=KQThTk~PT$a5D7=2r>(%=fIxe&BShm|ydd+;<Esl)`
z9QS03NlLDf+GCY#-BERF^PipiM^~xHIm)!2P+^oXIc~T`C34c11u~sdvo$XE>}+1C
zvr#Cl%KF{yIhQ_hOcAj6-E7cwV#WRrS^wYM2d!791g;OXbZA*H|Mf!|Tbb7H*?Om?
zCq9YNOVCtHxNSH+S}lHZC-1^_!Zm^ys}EkhttU3+h*!@M50kwggg9&_@8iC;?z(e_
zzs&wX5!YLfJ9?E3<|lK7hD^VAcy<N1c%imJk>dXqQD3@BKCgdKy`8<vwJceMFKK$S
zg+`|mv*}EOmf*wsArkLbZ{dpGGWk^xAG2$B=E~g{3?!{~gf$!v3)r#qk?AjCWBJ*F
z6K->~bubwvw1;J|zh5{jOrrJK4*`Que4D&FSk}0&;MzIyc1PD{zB!uG&MY3S7RwWN
z@2tFXe8LLtJ%_aKGF$CaYCav{C|=`X)Zx)J=S0_$6Wv=p_<}iuWv0q+ne=NX_XHLF
zJHiZ-JJo$JG-NMQ|MT+bOKy|3-a5ys4y~`&h?m)Y^3b8VKD*C0O-h-lcp*}Dp^h^L
zv!=nGEf=MncL~qEBH3|y)rRJe){G6#ni4h+HFBCaSi%%ok2wFg(f4FZI_Pysd(+yV
zhj%$Mxc#wl&)McZ#U&`<U|@rH*Bo#4?$f)Dc-u6eUV4S&X13zZ$K2OHdk0T(TX%9>
zh|8IK-Uh7?xC%M9JXW?-chPv7J?ZJ?4wlUdGkiAu+_dEA9EH*~`y@MOn4Ym){Kw?L
z3yEt=ieVBGXCo93=ycT2mN>sd!Yo1JmWZLy;vQWO!<gtjirid(SBrj`u!yx~)$0w)
zy#iv3JR8byc{Lgs203ipz_o4dOi}BoJ;hQd<fS)hEjD_}Ah_|*M#<n!|1-2=Hcz!N
z(%W}YV0YAswid^Qn@{Yxa^m!c^PPLn|2cBrE!e9!;Gmoi$0wl+GA0}o=1g36`oGlf
ziB^9mE?Df~#6DNy_^Qp5pBSrsF+S-t=ftH8zD_2aFCV(3*eY4Xqn~puSn9NZm0MHg
z;TcL@t7W`@GB)t;<X@N4x<SXgQ0jEpVO_^7?kWjKG#reRrI~djw!Lj~@pcK?ePrFv
zDc(D`oZeeAHK@yWe;}vdlk-h4PV{!zUf1T(UV1QS&A~%+CN5D}vyfp1OR}KJU$bL8
z=BpXRf30#YzP$clleksyir77#C$bjwPS}3@kGN!#eTq&;-z%Z&$zJI(5-xwYv}vB=
zb>=ve!E<}fF`-F{)f$`@DP8JK+D~mf*fZzz{8_f8aM80P6EqidZo6pX_g3orLaq3p
zrZby4+dltyh<dp7iH?*|55v5}6NGuJ7Ja_DaI*iD*)H8JCw?8d)Og}jisv;ZQ;sR0
zJ<^wYr0iALb6~TkZAYI-uzb(@1411fH(bv9bS?Jin%Z(w;-vh+pIlzQHmmD!aP9D{
zu9Vr#vHQf%L-YQqY}4-Wza-BVthz2s^+KU_(8og;b55CUF{qyH<-fUAx8S1qRT+tB
zx%SeC)U5NZJhqn(M<}1YQFAlma<@x=&NYwQ9SyfTQm5VsJjl=X*5&zU+x`hhe@)Wg
zI3f6l()0}re7HYx*t+W1_FkCR8tgr(!B=AAtLS4|niu(BO}f^z^n%IR44aPn-{xm;
zY`C_DwOZm9!_x_~euprv`M*QYTHvF~0ShMej_yTUvv&T`UanuP*C68XZRUPOW4#Tt
zq(8(+$Cw^;ciHvhjb8Va+doWp?Gs)9+37;^R-4JborhI9{N@}vFz4E_T3dZlj>D`M
z_EfB2T5;s|?`;Kt%{X#BRptbgYz;VeNQKw8yW(wttM*(G&Wi^QTsdH{bg`=MC&q|l
z38zHXZogs35zTQx+BWjV&nfrzithRtnB;xg-RIKd($i0#Mr>aq-v8pnxq!|<g~%m3
z9D0GWO(%k0KJ{pt5qCmWv`a}|V&&nfxi|X?)=Y2JFE-p)&S+9<++Ch2Zd1Ce?D0*@
z=!rJ0yv?1l=M^IA&TcvLqGR!zd#7#~Wd8YYE>P@y{$da7>RaBmdmlPX6ONq!P2qS@
zrs=CEJ9LtFY_*O4QX{`pw1M{-$1H(Fb`$<D3s!UN(4IfJ#eDYpUn<um&UyS=;*wDs
zmezYs?#sOsU&GhLTzm6&UE__!_O%myOC~N<STp~0jH+12)SO3+1xZ|2r4D}VwR^nl
zTPR2Q-r&nmc`HQ?R4sS&e?8jq$aS~n@y6R6jLYJ}v{qhHy0~ahP;&;4z^UE7k59L_
z-nwh+{Xll2<&)%drza@z&R44S7E8J0yzlhx8poi3xTC#tC(b@>>9}_|RaEZZ!zl^P
zZ<eMUd75%^?ZMw~IX>j78EIVZHTH3MEGW;(y+(lhPmvR+(tn{VTZPW(aGYGCKl_TA
zk-|n3Gu{4A@s|{PnXYkMS?Jc&8FE&D_byLQnF2H4#m*Ox?lN0m-Sw1X1Fv$(*9UQ(
z&z#R3*w(u0+e6M56SNEeY3}UOYn!dz&OD)K#p<>V5ldEk{GF2iXUX%*Y3cubmU=8Z
zf1-8o|2rpo-<-Ffb}e;nnBLtZ`y>N0pC~NNUMR1Vp_peC#1LOt>)HPyhD)}~cJ~GD
zqmNoT4k;`=q;0cBD@5qD&$ONc0+Tah>uL>H<)jT|CYt+Q`9GKE&R3xW60K3UlN+C9
ze!DqAzVwKaYHRq!)8U4%rakgn;%2{iS>PhMS78Uer)Xqdsk*mHPu2I?!#k(`bE`8)
zHlBDgt<HMqnO9*SU!}Nmu5#1NNj>N<_vrYpwTl0iAM!esveT-VdHR>sr%_*~?R}wN
z8EyCT>Vnx@;+G3QU63j)GWTWAAp>shTc4VFUVY`5lC-dK!r2=ozD5sZj91>;clb)0
zU;27mEoRLVj8o=q;Y{Q_u{@_Pw5da{O(St>=ejkr&jtQXxIS&&&xz||TsMDM<p29g
zPx}+E2iwj+P;0$^&i}F6yT4CPr#uvzIw32Q)zx!aUjL0tfhs=D>)x$9cJJ`gON^l$
z%zM`^sW?&bD9lak(KL|;F}1FLiH{D<dbEDV$=<Z<y&TrZPkP>6p%Z6f(t06$UhSmm
zH)bzMn0#IJ|6#+HmknEaj6(`e^gX`lBz)=3obW@jxl2x6n{+OB+6|Tj!C*C;GNm}B
ztLv8Ay;_>LQb_%yu=&R&X|F`{UmfbWbXX&AncmWd%TGMFJ@L)^a7i*rPjd20U)jm6
zw_co0NpbDmJ;R$XIU}$vyE)^vp0C))T(!omHH^2o7@eM4eqF!sn@XgdJdbWis|m-l
zTDhj)rN`o;q!-P|k-c*Ip7XVp?F+B+97t$37xJ6Ep}0Rd`iqMnzuf(w2Uj0FY$H4Q
zxPjID#*W;VlhzrZUM+IO|C~pU`~Aa59-L}>$g{5Hm{<QtZ~eUGU$T~eG52Htdb{c6
zykmI>d-7Umr1<;YX)bu87XSaNVCF8RDeHb`a2;ZjICs^Iop;Uo*7p}|W$bM^j%Lcp
z&pf0xM`y{6d;_tDXOE@abhvDih0l3P7(U<Lr*JCT@P@ui6leI-yf>K)FXXN6%obL;
zBo^|~-Q7OZy<9lGT%`QF(D^MFLVUe%TQB)GM?Ai#U2^40HC=i2`R&)@mc8+i($Cfx
zd|luCO1^!==6Bx{WA8M#$#n%4eQH%{IJo-b#_ZzHR*Z{WEqS|j&L918_2{L>HIkBD
zi!)lAlh^M3ykURCmbbxIsveu4Za8aJ={0@3|Je;^&n_!?C(9@JO)Br0ses#wiaYkV
z^&S?>>1FgCf7|f(_O{jYi#--EbGzG8e8J`aGmH05E6=`Z^W<5d-WthOz2kTHMt`1-
z;d3{BpZEIf_Zv%cUIdn|>-_zC+q*a0s+;$Py;yco%a`+mSW?<sj{W!ZZZ__UUGMC-
zW>@CJYe&nIzTYcV-J-7@cfD8q*u?qQH*{a;?-ftG&&|&(Gwlw`<P9$cCBI%$D!6RI
zA$L68tmiSix?-Nb;-n3&-1lDwZhY&oW>1^9_X+Rdw^rr3EOn+YtxXl{9z4&<J@ZB=
zyxdK>)PQkritep#+kZA_-rUm>7smBzlN`^s?Iun9Yt@dfn=--W;5MP~>oWTvet4nD
z^<V<msta7+uekmtDX?7nbV}s$qolIt9Y55vd#<r^Ffy?V>HN?5pl~RRDS|WSXut#K
z7A6s)l%s7T(u^r8LMfM5iLUI;jq1$_T_3hPN`&3Zk->#wQllZOlBdJvWe#(gN?uKw
zsgQPdfqkNoYg59r11g-8JSJQ!Ijy!L#c9&7jIDRChG}jry83Es_VxaVqkb|wf)-xw
z<h#15Xh&I6G-K|qOR?V5AKq@|Fy=b*qWnR8UlV)3P|Oa4gNK;}_@C{#_#nBL**s|8
zor;eRN2gonN!^&aIlVENtFO$)tBFCENkt`5)1fQ7F(`z4$=s;2T+I!EpY1Fh54`6L
zZV{fZbim@%V-9nUgmVjbHatJSlVicT)NY11(d&v|=I#CE@M<=@mxF>r=l{GL>m_eR
z-3(aY5XJjczURcne-3|`JbTSg+^>yH<2)emxOPjqfMyWGh2Z+xj0%EGf+~s&#R78{
z9+j9q!H`Qk^lGYtw$@hlByE*h83~;l^DGuIY&&vYDQfEj)yFO-T1Nt0ET3I}kicN(
zrjW*6Xd#!(^x^1ZM=7OkiEe*PH7{uV+1PYria_AZ_$h8dGgYNy_%tR=O37HEIKyL>
zra;;BrINF%m=<n(Hk0k3S*uX7mf{5#FNx{P3!^;@kBY=7sJ{?v^b1_E;C?6jJcsUY
z?nfQ64@qh+Wp{N92u_i>;g=jC`s1FX$j;@;F02nlv*Q1>s?2nlxYh8&YB|d-7u`*7
z{a2Q+W$~QIAn@#|iGZKs-Yt_W*It|=vYu&Y$c2``Gisutd7WPcy`n;+A4SCZcI$3U
zUZ@hVF74v<OAhyEExi!Iv_yKr@~lRyi>p685?vo-_3Hby+HbOzktsGtVM`Kca_)&t
z`5SfZ)xkAdIRz7DXqtJoEtLEcnaSjGC*o*flucJzrkVn42y>?6Y_T(Y-DZnOooF-4
zTEFmG+UeE2A~FSsZ|cY_IF#Ekr;usp8I6iFf#$N}=R?!KEV^$h6}Obtw<}X<Tc+Ys
zi6vXg7aaTbcgNyu7E&(nSEqA&2u|SLDWTvPt&?$K!(T;3#q3=_Esu#wmvIPf)$Hqd
zbhT#r3ZVy8zyH5o)csJd#LV%nKwQFuhUw84`zB=D>~#EXmL`8-KIb*b=S(?HUe7y4
zZ60XEhVe-xbjW7znz&N`nyTjF9EsgCq^tanUG9B4&)R9v_vLeQ_66xZ&nlPFZxE_s
z`kr3gZq@(db7Rx;7ay*!UnZBYFgcTZ`G&CdTNB<fb_lw0&DRU?3_1Ec(6&|mUYFQO
zxoVw&=Qr=?1V_0nUHB_E@@weU14U~Y7O@n>C@L&J!Md(V<b&%2rSNqHUl$efE~yr^
z>0rIDysiA|DTSn9k3A6|R|y_k9&)Yw@_v?Y8`@@eCQe$(b0|h)gQ%HDar4DFysApO
z#id%3Z~ea7%9S+R?Cvs!+)Mw@W~Yd4o*}4WVP3tcV2ZK?mqWXi)9phg+k)r5OE_*F
zy(F$#NWo^AfT4Nt#Pc^pQU#>1HaRFb-cERPb^fM{-3p1nMGo+=tcqEw%g8U4e8+)p
z)2`ZeTA@E@cM0v>?#h+tT9bY8AzRZ9Pp?grm=-=!)zv&HGWDqK7fr#HEW)P(<{1<%
z+jWL7vcsUhGS6`9<w;ZLOj)XF$?3hdXX&&hM^yD%HGNWlJ$+J>E|BPz!tFRmifLAn
z)2#*1P0NK&7{9!-+}^c6pzYP8h0BhcG+SiZ?0zgL964!|@0CD3wv7pF++QYa-!M5u
zx@5W~yOV;1*pX1)pC>}HLKl5o7|ihXLrn1hd7C;7PxRINmtL@4=+@fSqSXtZO;T`F
ziLmgS>ryCMyTE>bXkc{Cmc_3gX?QuzI>^D-7W_hs+sgcgqlH`x?*r!eIVmS5ihTQ^
zU=q4C!EO7gR#l(n9Em>Z|4KJ@IQA%5vOIGznikT;XsmUUab`{q(^M1Bvu0Y=XG3qB
z7~R~{V-siUQFth8bK{qu*|*p5iEZ!LvQ;n3_t8tg%X~?~Y@7;Ae!F|~ma$Fk?^%(<
z<it^T#X?Sj^_l|D!f>~mBLe2L-o~a`rabti7P)`oiu|dvb7mSudX*NYOiEW=`Os|k
z)?Wp@rJqy+rUcFOx-f;aeJxL{cJK?;{p-$cS^N65|4-xp);DYWwkKsxUBc&dW=p48
zZot)E-vw1pH#`>lc}-~eIK{lto>M5Gbywb(Y16hmDoWYW#JM7P%Ldl{LF=;rZuI#1
z!gIoj6JE;gOTvE3oH(#n#6b1(Ew8W_yG2Vq1s3(ce$S%FnJ|ItiOSpG%N_r%e9)*P
zaq!=nC#teF=eWNGE4=hMal>P`<a8|+Wdqj^pTLY@(MMB?^w#wFGqWviWAsVZJ+?B4
z+ih9X4+G!blV<jQJS8|q)zEf#;3Ag{wQG+&J{n!RxU78J;iLLl#WNm%3}t!2wcL9z
zn{Cq_r$jH7#jSfb>OQ$z;P^)?qO<1bw))!sDK4c7^0AXPb&1SLd%VWe^1t(p-5yag
z*EX2RNzF9ylV2929e2n1)Q4+6t@jqbUGPQa#?ws~dzPgxRy_Mc{pz{x4pV2$RV}*m
zh9lfIIC?@+`|&$+(yMneO?*~3<GK24wdYq_R{2FXUOOAKt<>1S*h7Tr)tbK(AD*wB
zH@#Bz(6eqHkG<ayK3B|bwK!)~5PWS?KFe2+pozQsZ++aTA!R))_m{$q#q1Lc91rZB
znR>dsxBqZL|Do@%*;sEI)$@Jy5A}K6c-DHk?cVQx=^6%IfjKKZjXHu{op(0cl`@~%
z+R5S-CVeZTNmavnW|iC9DJr{Ou4WZ95;-lE-8AKD#skG$QPQm@ZiNg%e`mh-V=>S%
z<Wl(WwEZAo@yZ%GO#@cjlrO>|9v|lm&;8_<_)o}j-x@b2*>B!#%mOzZQWbv2%vjo_
zSYAJCUS^5^4#SFR9vREeDO-NqsoW`fBRTB$sm`_Ai|16INk6nt@K$POS>HFIJpcBn
zZI}I&lMl^J*mL`CeKt?$`pnMQulM9Tt;H)Z%xw}`mMzn)EcfyFP06x_>9Q|1eP0)H
zJ$ERIE#C3eYR`pPT)$SGi(I*Qm6CX1$Hx1U1$`I(+xyYdV6%9;zy+%fxAi}nsVJWb
zyz_pa>&LIlw<qfVVShfa(ck`Gn#cSd+)TUNAFS5xcy0N0cAf3AA7=82GGP;Zv(Id0
zVc)TT|M|?DXO3xl{og6~{?v|5U%&r2?#*LVkfnA_$@@wAnL@vk3t4kjt2chyEZ5^}
z<KgJius$JfDrdtpN8W6mgtM$d=VB!lulgxo6w<U|nGmzU=H%t1)CYBQZh9R(>Qc~u
zIZ}=@&h4`1{0(!bbMd*&jBQ&#_tMWhVv9QOR(zSHtvO5o>ruUzlXN^!PWQRRC*`T1
z%W-S!gIH6KsugW44N~ViXJucI$(u84p3PO0$m?@FCN41XI{)K-<V??kxlu8(Y{3OC
z*)EN-bJ_1`v`Xo*O0Rp^vSJ}ihFrx<x8MtjRU9JcJP$EH>ks~MV`?UUS%ZsRWUKCt
zW)7v6X4NDQ%|-7GXDGW(a(ocC<A3dKWg+>Gt=vslo0>9}c63N@p36DA@lpKVU>6M*
zwkF97hAln^rT2!$@7*Y|mow_i+bRA_BYfA+&=#AT@n!ar6dwJkGmb2Gc-LP4dg9nm
z&B-}4&Ioi0hr2F3tr8d>Did+^{sh(I$7GL`>Gob+a`j@y^~k^S?>YMYIL<rFS?Cbs
z#U#bOtU@%~WoC<uWVVU})1JpRTCFy>l%?-2wmFu}WRM{JDcSB{i#+EO+f8ma?gnz4
zI_?!R!)3vPPlZP~gJK;mo_sJ!ag2#|sqK0)<wC7Q;>(#gwX@vz70yq8aixcK>MdWl
ztetn0z9>xDk>b5?$(t>)w|YG<*G!)L^vO3~|B(MH9{X`52#4_(U+pxW%U<%3vvln&
z5reog-cw>hau+I=99!`C@1EpSA6uliEeuOj4bNK|UZxtcNzroW^_VLiuD(f)jPsk`
z#x>hUM6X+_UZtcGw<KiNY4t_D;d5@f%sR>K)$;VgwRpKBFE%`yc}L`H=9$iglfKES
zP1+G=YxyjI_ex-ITkq9{4l>F@E9bk|$mHEwc!sA<I5b)0E4!%d%?mX#iw{f{uYCMp
zw1t6bftA|{38e&!qdk_tj@Vpd@w?%|a^*<Ni?n<_*Mfh`3Vtnf&}z-&QepbgC84>z
zSm|H|>#db75A0hPh0MC0nliUcj^%RTxlOh%*K*sgOI(xL7~TATMPKgSzNL<A2X#1k
z)7re55_OM0&xxGHl60$d(MjL1E|C*+f*2fFVrB}LrDb2~+<1X=)(R=p#ObD~ifbw#
zUvkN)<8qT$W0i_i^N&eA73kOYud7{&CFlj$i-Si!HY|-!OIHd@+WvR4q7X~B*P^$!
zO6pG0YSX;C=D9@WB{fARshvvA@QkgnOL2bY>V9mZ=F|)GS<@0WUGh1qH0esxp`IIE
zO9fJ-rXF~xK8;EFhiO}It*7x*mb9ja4n9qXO&=T2dwJyI!}N8T!hy-hG>;efTr}rm
z5RGj=-aF^y!KrnNrWVAu`LZ;&GI`r>YID5t%;wtFh4<7paA^6Iyzt?8;`jf`ks_h)
zQ@lsK_bhYqIH<OPY37=vm+q#1YJJtxrQ}lNGT|Eg{N7a>bJMkxr1*9UH15fox#HTb
zlTY%Fs(7#Ro3`Z%-^D)PEDl4?xc512556uG(o46$=eZ*AT)-t65j$}c&A8$QpDzUt
zWl!Z>PR&}C#}*zoCw8ICsoty(slclyE&nqXspPHRmGpdP;qu6UC9c0Dou&xqMJ~90
zR_WW+Nr`!{xlS&c8h0%4EBEsdpCcb`YjfUlW4V&GV&Uu+?PVowWe#s{h)&FMxpQAJ
zFkd2Jc30Z!-uBFA^XDo>{`c+Aa#QX9a%8dIDw|tV7d<<Ycdy0wic9QYFP{%C{(D+%
zq*`qL?>kat$FYp-!1-D13?2@5cTArZn!a2@etFLF37fLr^d?;1rF})}{*_5nO1X}$
zY+7C^^|Ip2Q@2Ua57>C_yBRyJZ*AtEYgudWd2jQp`IjJIbW7VaK~QPxwWof3u5ZL#
ze9ImrJ03YZ-S1og-x1@ZJge5;TBx()Ge=h;N7qSChsSA)w|zPx-X$jeu4SWyLTo61
zP|xc>CE)><E}dNRS8g_|ib;5;hU^YXikg+zwM(P9@b>F<di|?1R43)_o|k16xFjxb
zt((<cj-}}ra+lPuIW=kCg#G&-J-LzOk>nO2*kWL{et)KBCX4ccs$&&9&*UVmaIiU1
zZ=^1;;o<kTrP6F7!T(P@TzV$0QORF8Rc0cmTV!sySZv^lnK@beLT9~Lxk*RMa$C#-
zJ*^EsT8{g2P961W&&zmT)bhdR#L_S=$3JTvdDa$%w%(kR6}wK`>BYJ<&z9?DZ%h9@
zJ<PFY^|U9;4qo%zbt!SpU7t&LZ#~PpdvR|5+_}$u-X(BoKS)zMyS3*$-;<^{8~DB(
z@c-Yy&u%E7@6l*>Z_6{D1wIGQ$+R~YDP~_<Z~ip>S>9@^qCA00kL@ChSiif+tXg+W
zXZAaz`N^hlKQx^8x^j)}d3K@fdiSm+3pkYza=E2c`N>>1yld8*s+90FOp9Shi+9C(
z`R|46IgFSDisYFYl-M_Ut9(*qW)S$VP{h!pKkvl-*>TG5bJWzD-^>a)7(G?z$+l&V
z9OwV>9m{_7Cg$lHMqw7M&qpNL7BK3opJ&qjzfVo~w2|W$UICj{w}uDHW}JL!*7i{2
zh(7!0G>*^i-}Kse`LFbD)c>5vy-XoN^_cF-2l+SWellabmAW~9Wy;!9>bI?Cx!q9b
zES`S1ZDCy=<6Rrkkf(2%lseBdtra-_S%5!<K}Ielbm2mSLbJWIgx%lGJn^AD?UqRJ
zxo2i>*((<N|LS~R<MM(nKDi<K^`dugPU-dene-phbG`hougJwETEa=^^OdGo@3`D9
z|GZPN<M=)omzx_7IalZ=Dy^TQr>{3nLOQ=KC)G{%zxVqaw(DiW^=B;M)cml}!Q(_w
z&Uz;P^UB<t7}UNhX&We+zbSDw{T-Kl{OrmL3E3Ar=CoLG{kwL{anG{KBRN{)M<jmg
zJMp+`^{=ix(xPEp;P_yZ?s=pCZAQyDg08NRG`t--;gsstZ>75`C(La$agWjMw=rHR
zmszzgWy+~lM^4<xzI*QGS*?XD?|Lm0TmMch&G}?4Q}(^DEB*K$zjxvNb|q%9)rJtY
zhDWM$rjBu@xr&oYABr9SR-s_xe&wOs5?7Cjdhz}4;`t^~|F*sUwJnEvo5bgL9~vf1
z3{mLcV>0oCbaGeN{;XS5H(itQewJJ>f6!GeAXz_u#Y4$&A02*hRs6rc={#40?)!YD
z-D!OBstg(pS>C0lRXQiKKAF9n|F|ehywo)3^e2tuTdn!G#wZ>=#hM#q|4Ke|mA3c>
z3%#tf+hQ8Dwy?A(xC9xnnsdyK-Lc}G>Lh`(w6!Ab<{gRU3D;};bJyOh@{-@5!Y8=s
ziNLZA(^_J61sA57zqeT3neP2gFzk$&%fTx<I`bE`@#SaR{_mU@GW|1ym)Nt<D@!NZ
zcTYRcQ@-ryl641;<xXySe^8^Ky3i$mr`z;ARkrP}<tF8+7uwxrdHC3FYze7a_hq7x
zM?&0Y-|ibe_T|O?nVByZeL3`T=9wG&(@vCqPdr_*-J#~a^6Zli@)qs}CxT2^YySI}
zM*i)IHthYARcb!#jd+}UR^ECBmID#1jaq(uclfi$cZzAVNret~g(HuNqlraO%htbF
zk`MpleC@lzBcJWEpI__e6V`J*_iS7JMl{}W%1+kFmvrB?XXW~>zLK&@|F`EqF|Vw$
z)ZGSY^9we+f2-@uy2rNEwr#t7<ObX2WskR-iHFHenQr~`hFP9=U!MII@n*~E=_Y&H
zE2k?zntuIvf=+~|;%71AmUh+h_5y|KLzRhKJ@FR>bfn%V9s8J}vFFR8SlQbF2h)s$
z)^^G+J1NC<TA9DQQfJ3%@l6-ZKCEc}b!&#%vQHbH8dW{={hFcYF1qzk$(iycB}Ni2
z%U2Y$F#iAK!PIWv@ZG&Wd)M-QcE2g?4fd=#5|dh-XJ6@Cu(N;5rR<8ems35wThcjy
zU;5oX^KJK}JA4`MRh%~b=&G+>$oWt!w|ME)wNH<3xcoviWU71dEtl`J%*t0<nRrf~
z`1p=#Z17~qjWa90{`gvZb@GBkMkO5*i{8F_nzwm(!R_7CPwUMv-}lJcZtwHmt+f_c
zqo0?sRnJn^jIW8%SowCgd*)=Z>TjaI_h~Jgd-+Gw6BB=->CcyN{!d}QSYBQ{$@I?u
zn&dCvEzH=K?>1k&dXJF#S@m#1bH4v4THV#T8Ps2_oL6ME{I|_!nIk*wID!_)EO_=l
zYW14GH~t+v&GJ8Q|MKVizW;w+6C1luftMp{LB~4vziaJtFT3ykeR}$a_f^hbDhAHB
z#{0YLpIRkN54!k~+krzsBjNIaCRTQ-kQo6CGn-jBt$KbGh|KH~WsOs54P<a{<WT2Q
z*_^82*`^-2EM#ZU12Ikg*hy0uR2Um(F$r-j$ef(&Ki}S8ZQ`Pot|^P8m*mV`6rwp@
z({CZy1EtajAq#AMxlU9~%?@85zu#-ZqNRze&6wWa+@!uh?)p~s+gv@T%O1q+F8z4w
ztao^R{J#44zvjN)Dt0~UZ4|E$&osl;jeU`u6idZaH5+rJjN&Y_6rT6YwQhQ5Bef;1
zZHZ;`9}BgI8P}NNSF0R0FncqJNte4q^M6>g2&ctl6^=Tg4h9Ae7AAu{o97NskFzSo
z%gwd={Pb)gU(UJUgwJz1r*zC!NIf*=)AO_bf(z;%wB2U1<mj-U`cI5c?REH7t*P99
z|NVJ6nQM2;6k(5Dua<QyEM?~Iao9PB>x<^07Cx3&%cf2e@MxMdsUvf8m`?W5X=N-L
zIfh4fC|%sr!uENU>;4Frm#$MnVpeWgo%8eM5z)v~x~eBw7&f_|Hc&R&d{bxES@j7a
z(Hg2M=AK5Ux|U5@u~2Jc%ZtU|C2n~cPdZ@J*}WuADY)xFj+gL3&dQwN1q><*U1v0>
zdA(Y>^qq?4DhAd~U0UlmeKOY8KX~fGS!YS%*XBlNigf<3-Oko@<*a1ztcyVlG+a#<
zN(Zl+xKP$Kw$n(lUg_(mMg!qY1=Z?LI#({({XTg$(pfuW%8e{(jq9p`w^X!^iiESb
z8?kbnyYY-imV?7A{;wA2$1}?MwqIW4aH`rbFY;ggW82%w(e_IZonQ6;$K`hpZ}@!7
zBAEiC$`^129y%A>IN=1J<&!D9%N0u5SA{P~S=xQz!kieT6{06{JlE(3)Ia*7<)`R2
z(XefH>Y5EQtcP}}ciHbfWf)w^VtMtvMbX!V&1<$8DTYPf%|ET}yeahLr0BGuMH+%q
zou|8^m5i57pXa+^*}|JV30Ct=gOm2mf0@E{e%ZW5T+x5`-N_GFvN130zs^#LT-A+b
zS1#q~c}8;aY}k7y&pUU|H8GyIn}jb#D74%9c3ut>5Z1_-VqayQ-6*D<n!;G)vNBaJ
z)3(!OlC!sr&Xp|H44xLX_7^E_i(<Z>ILkSo_l(g|t*q_I{k|)AJQt`~XU)j;U}fp!
zp!s<YZzJ?Cym*r1e!*c51IL5NPfJ|=E57+%3S;<wXLgp|&lR)sy9>W_Z?G@$KlSp!
z>-(Y_Om8+9^Ql&phif)A%ygab`|a*z<K?$SB|I8=tfw$VtY;Mve6!W#&#t;DGIA^G
zFIzkl-h5Oe#Prkc`*M>71PYfHFlR3CYG5&H+;GUa`LvHq)ukoVGdr7ox;UmzDpAof
zYYkA8+j5OFcG{uUbC0yFDdhC&Nm`b^G&92~`mX1x_5<_w@VEIId(Hc#GIhn*&aC<*
zp9Nop1sD{6vapIUFfr&bFfcGQFgY+XaQtUr<q)}%wP1E*E}K@#(hUn&w=kdN&<J*0
z<l34lm38Mt;S`2`DMvmL%?Vc|f|%Z14Z5-Js+!@OjNFu?ehRA_%~{u+KDlxE`T0`E
z8D(D;-*s5bH1U<qsVOTi&wIqyHAzch)uEM-LV86uyn1{s^s!BotZLrXUF*`qr`paA
zS<O%yTY5~?D)d3fF7NkTD|Ux7+?ycnX%@Tc;);h$+sijeFV;-AW_+7<b8~3X>&x?U
zShhUstV~YpotfVz_F?Ad^h>;!&!3!>R8Q}eiJTc$F*!7oajXCQHjND-l0h*|@y|{;
zDo=mJYndvuZbJE?Af~%I|4+Im9D5n(Irp2^1QpGL4(6Qy8*~%6XEt#~-%K+13J#w6
z;rhi}JG1Oohpu_?eE)uX&G5}kM^z^TEMD_yqDPafs!-US%A?`sp_Qcx&PhLyUJZ)n
zY@Ht%TiEfi-HX$8N~rj)tK!A}3tN@C`14GzcGn)%NbXsEpyH~7il4^=haZ_YM70-p
zFkbT7DiY$ay?8^DP>aAMrrRw(+r(~ng@k23ohmgo@tznHr}#yO%M-5$ls?XNc{cr8
z<~89e=bew(N;0`r(&j`gVtMNDsL_1F<Rx2z_)}+Xy)=oz;AV?(UdOezr;B>NEfI0=
zH|t6YnKJK$U{q?MYG1_6RZm|=&UIVr*Rt3xRitypBa4U-r>$8J8<;!)SEMF&n5K$p
z&*}-B%;3V6z&+dB>(%A5m1j*vHf|Bse(JyY?McDeysXk|{9jBw*g5?W-%}YUY2We*
zZUS2y?>eMUV!u5{%6IjIT~eLma=X@ES+u0u<81qk+D{kL=QDgO`ym`(dQ~FfKyYP9
zK@3OKwFL*cwbulMi*~W9g#QdYlX$d4Dofa7O^1bHo(!|hf=-q6)r%UK)>$N<RGO_(
z)TqyWCbYZZq58%nQkT*eRKHxB(p|CS$&z&E<+W=Hk7Z?T(OK+meYIqp!^DlO;xSE`
zTm4gf3<ItPU5?DVDru-P;Re%$yT$<*!b~RKN||zDHbcyU?nMdC`7M{!CapCINMXwO
zfAIZ6cYEHiZkxp9UO)6PTQOmcm#JOEl4kvP8(Pa7w`i`Jy!g%3sHDz{PZn8%YP$k!
zmUdM(*SyTE2=KFfcmK8W{Uxm0RoU12bic{uF3n(8?4ISefpzz-_w7p7dwjh&rLB3_
zVEr{xdzau7>krrGy}Ug0ntf}G^W$GfqKlavzV9tbm12MAvUmAuw%w{%&kA0MKCG>;
zIr9$7{rlf?Pt33Tm3hOkb;aZf;UXf7na*XTvgs&HReLgVPUVTx^+I#qBuX5-6%C4J
zNk>l2NIS@PPNz}Vdg1|#j@gAD*zKivO=!Q+w0icJQ)1ht{V%XEZF%sMMYXGI4qthP
z(n`-3pPRd-?)(=C+WLv1KZfaHkYM+Ty-Q4&C|tPZp{a1mw$<Z_RhU!fT$TrV3`I}!
z>X@cD{CH-5(3GV}NaWfpr5CH%+%nvq5~nWCa@FSeBoJ0{QzXpoqmtdbr+!?D-bYw8
zPV~o|3s#+^{QAj@Rn}fw_Q?XSj$gh>R4-_+)fSrb$>FffJdw4Pr?YNl*e#aTeUxG{
z=i;M}GnUW1AK`7`ShB=n-}SXoK`X6dBKXwaZd-T1^yBmjC*gS~oFz}4=oX#%V4_2c
z)5h7RpJ)D1^kQ}P@LlC&P{esT!P#ktLUvWb?1)ndon9(klQoN*^@I`+I4K#-2s}FX
z-Jb_bvnCrwEx0hx$g<O^i{<~}eg#g|7ptT_O*l_AZpfVMF?p%eZWYrUPM2(^7fhn-
z-1R!ja+T6oNr@{|*)`5+jAoHMSK>8cuhE19Ra;^N?;c}VsK>?Dv(L+4*JEL;is-Y#
zcIz`Qtu{@SovEQS^>%W=UnQ<JSDqHtZJhI4%zQe-+@s3rf?a_wN~^Q}ZP9mF+a*77
z)$vs#S6Az6&9FG~VdDCCt!r+(tUh$&l5F9Yj$$T7!SY8IGaQuGsfUElSo3L?#YCGf
zHIs#`wk*-yRU+HOzdGdTx)sjy^jlW5`wwU3l#)|lPO4SSdE@y>sXFqs*p#W9xpIN~
zrca*Y_athLgpzyG&Re}pu4#s{+)q97bV|&BE(OIR;Rdnm>dU{leyiQpE1zDc%22jJ
ziF>uj%_l2doU~tNuG9#6=fvzB%cl2zmT|YY<@W4zt~Wgz6?pO+O*Sq3d8S>iZK=Ec
zmDTsEz8v2Bp?1Cf9qopnem(iFUY*`ew`ET~I_8&iLM>yt_XF3dNm;?KFBPmg?xMms
z(Z@@s<C;a(HuZaD%J)|lI6g?ZHE+W-%VjMWiYs?^8VIFKT65u0^MOaxCySma(Erxe
zAz;E~cPm1~eBH-h{vV-|UuOh7oVzS?<?ZYH404?A&lNn}6}3VBaK{?EZ-$WzV;r>n
zB~M)PQNR1>maNS76QR910n*pA7B)UoQ)~A9oGASIUfqtL|IZfmeRg<d_}pASEu3lP
z(YJER;m>^5m#4{`Q+J(x_<~k$<;&YV`FfTwk{nsAmR6|hg+@PE9<uT9i_>ps7+q?+
zpH<fV(6di;dWet4HYKjANqWWaN)&vz?7GT&*=3EZ;%lD^zpkyDwsl>mEm!pOs%!iI
zb*yPL68-*QM!Je*S#`zQH)+h(HxC{Ax@NKL+XVgJ*KLFkY}jfmB6GX7-&CllONiZX
zTIJ@17{m27HyZDK-L-q~=dbSe&wj`CeQ{R!B6?!)*S!;KuP+vxox#ha>D;#9ZmvZu
z-xfE%T@y9<c3iT(Gs#)>P-~}_#UjzEACB&1cj1x_%;t5we=|q(w(`2%+*$vZ1$FG)
zF;#EhdWB;$$+|ZdvifR-ua!xBnd+k0miWC!%U--g?SfUw!tXJ+-Q{<wWz8$uWOUco
z)!pjK)v7PO4`N<0pWJqBzFz6d;@FoS$9G+9jhnt=_S_d4^3^wX=1$*LAdpb1tS$eP
zPkH{U6#ERdHHo+HbX)A+`M!St_Sx)BH!O{__9ZKL>i$bPdCEZjDQ9Z$d#BA_lOL;8
zPUp{jU>B|9W;=^#_5-IZ`!f@-bEQ<@&VKvSQm9Dwm{D~@j>N?h?z?L@+C(hY_~CZr
zVwcW}85(QC?wBq7rxfd4`QoHXh>>w)*fah8Ka(o<ePMl2^UD4Hugm3L#YgP~Htk~n
zb$R{1|3B8RuX&RkFLG7?|BuP*1wIAe*9%CmzQ+9Wm0cW1;>&Z>6J8&@SHA!I|E~`D
zpWV_+%mjX46JWm~{AGdQzYoIfGa49w2v)So?YqEr|D$2b1YgE&dOfMZ?QDe#79w^g
zrUw~aRy|MLHr++~Vio^Y<ANqT>9Rz{sRC)n-eE35OpZY*4r-?vEBzC_<Ti3zeHWQM
zK{CljMRS_=?dx@Sw+q%O3)BS{2kt1T4Q>t0Xbt6P4ck!@exo&d1?x<K)<g|~<Q1)-
z#Ov=I7ksn5=Kb-uY>u{q7j0L>+e;+c$|Bm6JlaDvT5C6#*c7+gK1?;8CT6`*qRBNP
z=UI8zk2LAPhUxz&HZU=Cx)e6_bu|3l5HTmvbE~uJlaJ{x2Zi~rSN{L%VZ2cGf2h=+
zjZ!(2Bn1|_u3wU{<hZlU2G*ko88=$UNQ#-JOcJ>NSoZiN3q5fkro^g88##BW2Y(b|
ztutmjv%DtjYfIz`!8hgt7dYB$h1(`FwBLMDJyC(-&Wqlt3%&PO^gNR2yS0M#{*1n7
z9(_y)`rcUx1YYQqp4clR*w2*M|5l>!r$qm|A1xne^gda^`eX(xv*d&u8GSb^Ca`m+
zB+Mu^4^uvw8u!Ma_|NmakD?8^Y7L)F1-M^yyxZE)Zy_Kp*{QHI?pN3(1x*HxlljsU
zCvil?e>LD*vZ3=;p$g-2y@UU6q)RskaFwerd}6}=)VUzpHugawQ(__0fk3yD2~y#9
zt3qAeb{6oa2QWLDt(_3GW@=RrlbqG|R9?oa6anYoUnG8XC|%wmnEj*f?hK!gC#GF-
zpOC+ET7e{I<b~dno6|})8OkN6mvT<exj8*BVEQ)+&Sp)4rj-JzD+L2T^mnhE-mN*K
z#B)ZY=Zt$EGbT#1PQ2OIBsqi0ar(U%y)!4Uav7G`g!`o@b_nk*f8QFBe=V*1hVayk
ziis~oCUKZ<y~+Jotn=>$ZqHBoM&H$I%3=>I>T*1v!qphLdZX8-jg@ABl~dKD{0|CG
zG&5anuJ&ZnRF}rNQ<e#=WS@HI5u06xY~lan#66!)yeBm>O_aP}DEY`yZcafDlVgnT
zLe`)Qy+3~Rf9ja=VTa)No%8?fnE#z)!OxX5ziZ55v05-EbH=?H3+87oxW~czcjb(?
z91BIHSOsRy7tvw}Sg=qmYhfe<tNgEpa#0IKq!uaMTF@q0yXRP}XOR4xX|otK;w2>W
zB_i`TW;SfrtdPFgAQd=U(W2ABDt^lYJ=aC%^-9k66IJ9mRBeAQo@&;#<4RRjv#saK
z!mEZ$`O{pqZZzqf5DZ(X#(Ge|bcUe*4KB%LW=sxDnYRS8eo2=UsV)eMwN30fcrx91
zJL|_6a(7n<l&uu3bDqE_wW9ZCUw7n+wyYH`RVzBHR`mWqx#Ew-ir%gT(>E?;a$G63
zYN6nUg|k*I47jjz_O6v5GgdB`$(p#LB&i{pHK`+OTF1Jb9X%EKa+(Y*8mqUg5;T{Z
z<kzLd+EHO?=(&Beb@-ypBbSz(OU^iTN%DB7^x?%CIUD92YjXBf&Wzj@u)@82iv_Dp
z!_wX+y`8Di8=qFZiqz5A*tk_(P~)KLF^?>kz$LRL%y41sZPZ%uOLBpX)xtj=EZ?iv
zfA3)VeQW*O6YGDRS}$I;a;g+7gEj-Zw!oiN8>BaGu-~}w)QX-+f!YUQe!5=V!YBDe
zIrBfbF8<Osk;!n9ZP#M1gNwhaPO^QtSpBuITD7EJ<s==;*>(#zDMxSq@&6jXr_h|r
zXWUB~oOe#Ie$p0x@{#a@6P3#C0UAPGN33K5yfUS)PFc?$=)XE@`;4ZjE>%`Ty|o);
zY&CqWIm{ncO2$ZTQ4-WCx#iEiFx~mZa^{KCze_B17G#uG-2T>MyWPVL%oA5KC$5~Y
zB_I&6l4JHt0fi0C+>8MqwomC|srOzl5Vc~7*Q(`Jt1=tqkC{&Vf2_R5Jb$-j!|s*&
z0vZCX;+@-P%{E@WdZ(1-_qGVPWb0o`Cr@|OIs3?EM`PrE4Rv!vji)Co&sA;ler>z=
zV^><yT;|3qzO=bFce?NzvPv^FZ96J0?O_x4WS8yfd4E{9>9K2kytK}*qqX72g2v4L
zg8!cDZ53B>SPSrVZ+|v>A5XWcg4D)@8^t|=$$uQJrEbR=z1nDOwOIRggG#j|?~P5r
zxesuCY&he%**fKb!@^D9pYgwOwBBRnc4Vr`j&zONij&!y*6#hS6D+B_yxUahd4R`C
zQ;(B^9-ULJ8V1G)O%*&Q!+Nk=>9}srr?pH0GFef35_e~L-V%NvIeFGbp|uydbf(VW
zdnNq$=lpkC>npigOSD&sRjr)0YCDqvW7p~(OaVJW=WOrxT2<gNt8;ox!_-;b+bh<r
z?D(XXzDaZSeCy4bI-7IW2yC<xslDD&l(X62aF=XO=Zq6VlN8<N2d=wlt?I%(S4L6L
zD=5QRyX%xywf+A@=~KVvybKgrklqx!+S$vb`{u4aZYC#P6c0-m>YZy+nm=)}?Lk%c
z^qwY0SBCN<3|bQcIQIR?Sg@*QWx;BJn%5!<xArkb7Jtm%_`PvsYiPvx+53NZFIM&5
zl^dhV^k%>HEe(!68fSHM^fKd@>#X)z6m8B})%RucEOl?mJ13txZjBY18o0Ai{l?NW
zfo#jZX4n-9h%^cw3fU8_DG+;es*d=U$BxUl-a7nN;K-x}3;x}iDY$F-quT--eNMMU
zZ|~Ne*&n^?gEd!YdwKhg@^0~p)IVvp;TLoN)T>l4-um;5ZR2bg#!KOkxWAblm#aN4
zJNE$Z3hm!4a{`2Q?9Xt`K4>%f|MN(%%d7n!xhy=w)nc<{xAqC{@0Fa}=K8Ikdq;AQ
zpJ366jjYlJi8FGl!knU=jtI_in|JJG#hw<a*Pbm`vwJEw3wEorMyzQ4aBCa)uI+7+
zr{{C6c=x*c%yjK_VG82W`w!GiJgjp@(Brtps>N1Pn>7|4IDSU(<e7%Q*#~Cd*uRwf
zhMD()y4XyXJqP|T<II^9?|y6Za>o-qTVxBXj{E;Qn__g1N#IcEttQ<Ug}1CkgtwiP
zet0M#C2?)8Y~EwDEG@yTsM}U52}*6IZkMztC<+#7&pPl{<lx=uOMjJgUOzfTI^w13
zv9+&OHJexHue|uvX7x6!1GQ%-Y5o<N5^*E_&zX<^uQl}CEjM0n{x5;sIFo<hfwUc-
zm-by~_+~IkomV8kMz!qDjc;d<Nk;N_hFdo^SznD&EmD?p{Cv6lQRlqIjP^NewktL-
zy}&h5tV-m0l|$nm&6<<JR)rcx0y}RApEjI#^|y$Oa7?XE5AQ*jYf22uwzC?j2QN<O
zpL4TSX!hx-3&pnVd0tx5kLSjH4Om>EyV%I;Kx=FR+nl>BV)r$SiW_-vdc?&~J0jU!
zdti?30mg`A{+xjQPABHEop9h4vEjV3u;RqsGh6QSY?EXcx%s$IHD{~C!YRI$TQ@~d
zwNG5eG$AqXa&%mFGt0M_=ZunUOVigVupZOpijC=LWIZO-yL0XTpXD9*b_VKQl6m;z
zn9L^A>e-=9vw3{D4&2J$H{s>vix1u<aQ8en)-`;!)<AM*-V;^Xr#v_K*90W>-4N=3
z>e{3hbvfw)f4knCP@4{+-5#rTXBo~_vk-CnD{|tt@Qs;=LQkpA`XfB&&(z&Nrru3_
zt<X46De>IFlx;Cj3bq{P`t8fVH(|eQ_w~avFB@auO*nDAdhR9-?i(-8-O%^E(H$%Q
z%1^|9o{+Js+jZSn%z<1>9k|vROk`RpRC|f<n?t(pMBxW>7q5s`b+MEAx8c0ElZxp}
zAu$bg_Mn4v4lX%%>vdbw2?yC{0m@SZwh1cTdA7v%mPY(7&HYWD-cz?6k>+8Pjr=cd
zr@v<zQ{Z8d^1~rUYzHN(<12;ty*&JOg77Tvid~%PTW00As55l5&h|V1C7@n%W990t
zp6T!SmW0mV^uBfD&qRe{^Ab}<uD}aQ{k}Ikt~M;>zLY7}`8ky9t;40?+w#R@`IlVa
z`^@UR^a0=Od-v8g2)+1wZ?fT+{)g{E&VM)-^P{TvexA>!diE{4zA=Y3>CAN8eYog>
zgW&F%KB-NC8LS7pyqQi4*sWzrdn>qXkNCbfwi_QhA8hv16x?<$Ug6`fuQ75TX1XlY
z(3j(QVp#q4w9d=U_$O`mWg9C`E3WF9sSpv>z>*a@TWxn-YwD&g3S4U(_-#({B?+)v
ze!N%Pv`OcGCjU1Ft_AP-CqMkI`1^2;S^T#T62_VWdY!K#9{tm1y1Dk;bH%UfvoCDE
zekMX;-h*hlpNL~SDkhu}Gh*MVBI1y`z@?d=HA=@L@W8ajU<I)+85176H?3vcrF3JN
z0w<#k*EGQcK?%;2OnD~>H55OPXk5(l<4xxTAMNO`If6OIvqW_3|0-DNzF0j^ziDgV
ztxeBQ%ZJvpR!go)4^{|SpCmX*x5F%Pf8AmQg^QcIM6IK~WVNon?l0Wrvz6mou$9-9
z8B4$ZxhO2084|&`*Tu2pP_To2RN$gDudkku-y61ci=Wzwu&mW(?zK0yFZ(^nWL&zg
zD6M?@lBmr|{}<NDR{YM-on86+XOg+vwk*aa-T$7<-D5Vp`3$d7y6FT_?H88>-SpQ9
zS36E-_MLX>WNNo-k2rgg=6UJA4%3<B`eZod_c``Qn{6;(KWBmT41L8(fu0J_E110$
z!b*g)C(mIC2w32|OZb6CV^(5n&szuM)8eZ;9$lCgwc^pM#p_t40vm+9ZM3&H2VCht
zBFZSN&=qL5_@aUEYf08log0!FqJ|4jC<*E6G$)Bv{SrKCw4d)X6Q`<g%u%B#7Na#H
z9Q;N-Q^F2yIxerv{%QH#AdkXUjl;XH^lxj4ka?}V{9ee%mCaXjgjZEnT}-jwpQY)`
zVC1oLgTgGXq`oPEtGf06UvRPK+T!Y-9j&pq<pH;6gIDgPj!W)>PMr!f0)k$iYCIr(
zfS2*q3;$&^!lu4dG-A7ZWdRd6&#T3T1)G*n3;UzAc4}Zmp_abZMw1hLjdu!IZ(7G}
zY>tfHVAZR|cu0xcPq=+)*wL)zM-A3qxZotXN>K9kHo2&z<aE)%Edoo$$`&~!iJZN3
z>5Q(%qSsnn8b><T&Wmz8nH0a<q;pa1DWy)O$xYcW(?vD5ELg6&GDdUhv}LD$^@v1o
z+qB&H@}G^ZQ-basJSE?^^X@eT)0ZtA+Y^P0EKS^lmu=EK^haoc?4_)iGp>mxuFZ5<
z$ey(BMFe-(x~XogA_=>+J)b5`{`$%K)_<L!W~(x}__SR8gP#7{%(;x=uMLM|QwEdQ
z$`?02oP6H@FD77tuFs?kOJb)+u_~FdE_k&fjn!it&%X&<7az9cUbI|p_lrf@Q$pP&
zU4;Xm=w{?M`OPUds&YE5efOkkqfGiKoo=V@_Qem(bu@lO?a>p`EWWh7YSxvexyv$S
zSnq{0Fi#S#FS_0$))M|v-P4!N$mdMhzm<E^=63#GE;{+2rSX%+GuB)E-ePvWM(17T
zw41-pj;`P7^yb@>2|a;6Mj|ho?p!;W%gi5D_44(a+4l8s*`r?AcyF9`+ubB+<?5|3
zf|6$PFaMS-5HxocOOsN^^)1VVzu#;Tso&*&x9FL6*o4>r6`~S*8U9^c**N1MPpU@a
z1C|eM;X0>zcW+|<<TkNWVIm*bgp~`EY)*)YSuS-cKNy+Uw3yRSAW%d$=hXg90<Ah3
zNgkZ1ZzpbQV9|JTQt!))qi-fKG<sYKaO*V`v{e#VUw1``OF+O?jCIn?4GIpzn;e@Y
z3Rqe*E!UL!o|5i#ShjRcjCT~%M)wmwNt-5o^kIm7C852I#icCfMDwlan>j~5c^HHS
zMMXc14lCNq`K*M!P3Ttqw-wGNttPznzMJPqGP!VloUmNseWOS9oyYdgTiS&=Zc1I<
z7BHVzCO~1&9u9UP39t5sT!qa_9)J2I=e9j)$z{#FutXw6U|tTBb^HI_vxBk@X*Wuo
zj#wttAo(M?O>e^ImS$~HZIvn013MIjn<Sg}E!e<y?CPWHPU|4Yf+Jm}8=5+no<5{e
z@kGCAAzz8#m6i>YO#I6<)`_vqPLE>}V~vtf)33b9BJ$vqaNq^u%D)zi)s{%CKXuhv
zq@Ytmn`s+Y>sby%>0T$f<xl(jX8O*b9eB$pd)4aQ7fjMcxE6IEn!b1M)Q+@?Umk9a
zxgvVnRn;jh@u*nFQf<Z;r)F7X-frl7Imh|wlYmU)rH<Q-uLTQz&X}mwq5rtqv?rrQ
z;qVbD_k^FP|FeF`D3d;M!^i2+dj`Ha%NLzwKEG{M<h)aBKCKRQ((~X_s9G-j|JL3A
z5et|^tM42Vs~0{X>-F_|(9T1f=V~%*286N&Id<qe>{%gEB4W5LgU#Uet0c7}8~FBr
zNtXT4)ii;HDX`<_H{I@+?s0FDB;<~sHkhTkm|;gm;MKH;)#9?vlW$CP-B!?7R({Jn
zz9{&Z&xh8;+>6s0lpMLwZb<GDIB{gkzDbe~CM+`f_vw1FytK!zJ(<Q#5*cZE-=+VR
zsir;X4&J$Cd)z+J2rY#(r^5V}d9E?Bn6S3uA@}V7u1nJzdBoHXi#!XHX_RmenJp#d
z)Uzzq>Ein-DNoOq+-Q(}*7H{BaivEOH&gg#wF%16H=a5=T{<1dRXk0|gxg1}LV%ll
ziM>N*+Jy=KdwhN#oT_NyctPzJgW?{uOSvf<kH&e29{OAJN_=W*tFDHQ!1Waw#s_UT
zYb;=3{luqY!EZf}p-!Z=CCF{@_bv(RQg&sl@F{K~QLTN~4(bYjZ59ogGJV$KfZMa4
zzKdJuIx*4Y;ga1On7B&BrX_w#yL0x6vM`gw%PkXFvJ7peFWY)S$a=|f_5C*%$(qQ9
z_rB{EUpT8Yc*g3|@Lz19Tw9&Imt-7fjXx@&r_;g{qv5=9qHMCB$3pAdFNN>S67$`x
z5N7%Hb6@9KDb1~2E)DLhL$(WM2+Zg>b@cZ_1D{h1w5CaXR!YfOu=Z%vf}?qyQ>05|
zRQ0cMU7D4sutKZDqH~hxf2Tv`o^gU<1<bOna-1!4Taw+gpK~x)USy7ovd9xM4m`TM
zM@eYH#U{058@g`wF>y^~xF+TMMNg%N(fv|uaGH*9;Z)`GX_xN$yi-`fS+z;YlWE0W
zDc;@<Eq=>-UPm2GQJ?I3WtNYY?@z8Pj-RJ`^ywdHNt@}IRJL}-XJ2C(=MR}OHf4OC
zJ4A)oeP}&-pfPpxUgnqr>pMFGnDj~<7*t<0-DREdJ-E=3DgM=m>hrSNBA@%G=qeRH
zeC#$MWZ4s&$2(IEZU!It%&=2o&WBk#oRhp6a~>S~znR}<nL%TB$MOxo9{T56ESz*u
zy@l7zmE+Cw&I!IA(%D<(PwZXnytA3*|0eEKjpfTvWHKr5|8PL;@8^K~DH4;F8Leyf
z1=>q-UJSbK96w9pblRfm-Dm$~xB2&I+VcH9y5Gg!?O(^F&)VTJ^KZT^|KOyQU}6|(
z<))PXPTc<F!ID7JzdKi^-4x4a<YiRhT((P$dCk_VD^~3el#2hnHNn~}S=%hB`#@^2
z`Qa*s83Jo|OLy>oIgm8Dsf|H({|DBv432eDQxv(mqB8ev;?l0X%)jMThvede9EWCk
zNiNtbv{zC^ZHj_o(*y0}%saSOb5=AvwrntCZ8WMrZMMI_a_we~c<!~24saC-G3YDU
z9pc=azLDK}<<8uXyx!VEtE5?35>}iz+2oTT9(F+KzwV_b>61O%8hAcUvZ(&Fyk#bL
z$Le*fF0q_o5Q}PHVi90{GP5VDgT?$1@0JAasDeeM4;I$FX#F$0dw0N*Yh1l&1a@sq
z=y_1YwMzQHbjei#G91%SD#aQLvT|Gd{_1hyTEfA(B;8Yt!`hllSa<7U>ut%_yQ+7`
zuCZp^u$!IfnBD>FJ<=Teiuw1xKHBntNl1bDeDTJgT&p&(>RoCmb&T1NQDLLlY3=V5
zq;@V^s%+CCeR)sgi>0x<_!~8jYwqAr{A4Jp-EqyE<J_v=3hkv$4ih3f<((xtvV^%*
z=cwH|ZPxf;VqnLyh|TTs4<_jEW)1$ZU07>s>dxk{0@kn}i)X!%`Csv|^DM^})5}dd
z3O&0P2%bH`qJDB;6~~qarHMVp;-?Sn3tcdG=?uHtPNi?btE3Y(XHJ@F{CM8XAkW2I
z8}^>u<<DZkc;n=)9W!RBbgaH1%JygL_2vWNfo7aO+mgMvac}YBQ{hN><FMw*sWl?p
zyR?~0ugsWq!iX(WdXuBHkTplPaQC6ZX0roqHa;-R{A9NK68{dN?P3$BTr-|3FT!D-
zp<r~wL3y)dfe**FGoAJeXY<XTz;#kgwA0GNX5Hn2X1@;%Iws7*0_u!2HiUI7>{oCt
z7GyBbT3dQyh0cPt)*A)=-n2hEwfxc)PsRZ8Dy!+C3KG9x&rx@hQ0v<4)H3I#l<WWN
zNn03Nj#MfhI4K||zHst_HJoQ2cCG!vyWw@0UDLGiBVFQJlR2GtUt~JHYqs^y-LCIz
ztYbwu5_SCfT{up!?$k5r7J1Yl@I{4hb1T<DH+RV?JJ#?utvL28Me6U&zMm)kLxfff
zdoTN^!78~#D%D5fYsmSg4;#<jHk>VC%TUC>vzep9q{A%0(b3WDTnop|YUz|m$NM=1
z`WQ|eo;lx~Yr_L26RDR%;SHR{ub6}r{0{~$f3{NC`1Tx&Q=8*%SaezGEh>?H7QI>1
z)6*elMrq<A)vHZD3C>ar*`o^P#CPuS-J{XGLeMgB=93SetBt#tO_~<<Kr`&c)_7w_
zqX#WqOT7Mb$C!QJaq8e5uf~SU0kbaO^4P|CV*{rlXT2orZRx<RG90sort()GbmbJ$
z;`Z3A)fyzxzb9t(`PHYl25I{m`utB&;Jdu(T>vNdWv@dus*MLO{QWp{+s}@#hFwhs
z;twSRH*E+CoXu5V!F>A2nZ4TES4p|*&z&4~g5}FEy;A`aH)mZ^)HL1JF-i8PMwLXf
z@~7z<xl8Icb)U>yn{oGWhUyW<B}-Zq#EUFfSzOfLR~afW$&bZaWn=dQj>e-#6E_4o
zSV#Zyi~VvgQRLXp-PYgcgjZ}0=TmW8v*h|-V~$<U9eg*!*`@|J-3b47=X&=A>-aZJ
z(@$)e_=3-_(DccPc?%^s{{OWm;+cix7L}#lYtO%Bxbpl2-?hi)XBHcFFEC7!;*<||
z=ha?NtkdyZ!8S2Vwfu{S&|dHIZs{zaWqg-8#g#)lI+odqc2ui0zg=MzuX*CkOic;X
zP3AB9qb*K&9SYT1%YKz>SNErGMafAfGOX@8`e7cs_CIiWqI_F#t;*)LyYhC<S^RT<
z<BQoGkzVX9W{JVUsk6Pz_ImLiIbd=0`aank``@1OsukIly-m}&({^JQw^rZ$sW-k$
zt`0oV>Dt(_#85TfTHv(l<@m|BPqT{fWjnh3<v8=1!+q(V?M&wP3O42Itl7^ryV$tX
z|D)Fo3HR?3#{+7oREuz&{Lr~TSDG!e`Tt?XzK~vvXCl2-J6eN2G)U@BHQI2u|3r6T
zg82RgdLB7*>l`J$*9hNVpl1+ddCOx*&Q#$uFL%X!4iiq~4VmMYX?Y?nXM;g;tERIx
z*J6V*UjOo=yFWhJ{oD5Zp%Ax?2Rhh9yPHL0gdQ|;8c4r9$Q55XrOm?esO*GLt}6-A
z0vDOgqNY69$*ar8X7;+FFOW-QfuVU`Pe1$G)&I`+-^}go(&(Qyb#~k;?ez~iri*f1
z+TFkIVaEyYzAKwKQ<KjeQ|0*C7{g>Lux1q}>q=XOgPju_q#vD_xojnCph3`9SB}F8
zQAJG48*c{%#_X%pI5lO8DZf#4;FetrW4hmP28T;u{%=%!fV=CI*}79^Zv)Dm{ST}S
zetYm{%Z$gfR$R8S>#)u1Kj>nn`Jl<|q><5vWuh*9>n|NwHJ*|_>-O2X(dS|xd(OLU
zc#_kL&ur(_6u#dx-F*8doA&=&5HRJd{ErV(!JMK#rfZsSgos3mPIpNBe&YgX)*V4#
z4$W?XM;3R=HcTk|IBn(bi4z#@E@k!V&WsA^5zhL1L|Zr6>36E>L6_B=S1kV=7R+)z
z<KhzMU0h)~@g;MQ{*cv85k6D)*RMhL*iPdJ%ca*3>fC4+c+e%#vE8_%W$N=Lh9?zv
z99921TAp72<Z`Uy-~G=Mo_|qzQS&rCzK2t<AhGd+<V(w|w-#*YJlFaE!Yn7QWf@$4
zJ{rx^M^h8*BXv64HYF6F>Qim?dv&APfOD!r%H`inBepp*xZQP+buU#pE~3V9X?7x`
zg6*Z(Qj*P?Vf&Ih9xz=qR_)O>+#zynp(;o8zeMM_J^k+z)&EVMRj}V-k-;||*6<$_
zOm^;CC-?Zt0?C-)9ox?u#ZJ7ve9vvJJ-4^!T@HQsc<q~z%BPQ$-dwl)_ww(|?1Nj=
z4;v+L2C6K&Fq8eDmV4b^*2X)N+BR|WZ8MC_nRDQwiKz`^!<*C6YZ?zcbiWpJcI)Mj
z9;*xfa0IM7c%dw${eq+LHt(=!PXax2!`Y;n$~bSfDkL#o-1_D&@4oHFx%wQPPo2?p
z{lB<kUd9&AAQP_(7J|n<tPid^mof32^RG>>WURlp-jDRL4taSl=8Sd8vfaN$-9DFO
zzvGL!`Zm0$;(482M$JFB^^)(uu;f>y<?mIC*p=3wG`p$2ptbGX%uOM${;<95d6s|F
z_smH*Z@(A)Cnk&dEeMxk>p$Tf+$k3oUv10ozLX)>X42h`R<(}eDGM6JI&L0KT;I5R
z3a|Nu12505?wqkUU|aE&XIrnx^IZ}1Uvo5D#6dIjMCz+rpH7u|oO50X|EYT<_C!X!
z)l}M3aDm7vQPaoHr=A*KeX?_H^wwjyC75TM-;KArmR!Ix+s3WKiuYCjflp6Py#Ds*
z@pJ)xO^wM$CpY~!ZZ}eV{KBL)$2l$Gj&m?u-sh9Y5+|+rTq!+SwcH`puEXSO%F~9B
zT|BC@d>2UB?VWUDThvi)U2dNl37xzlQxDzx^k(K;ar=&2&KEf!rY=~ax8=YQ^Or7J
zp)M^7N6lYaeKXqdv`>+_XJOd=XXzVTBU<81eYY3*ZF|FZzNqf{9^;6&Y&Ytbsg|kb
ze%qJcGVl4~i|Gw|`N`J{nl|jNcgx6#FKe(XDPNb~*tfA!Pbbm*+jg}FJhL0tOq*9Y
zeV&;An-kY=99+a{b-aUdeo5mCmXm%R75kT(_`cbBmUHTZh<UlHGTw<ZOZkJ>eG9lG
zGHY7m+;zP(Wq&9*bIfTi;**yCfAXqXu$A|`-o6B7)|;E3-reecTe;)LjI0(8@smOa
z7KprF+|ptHsr7i0gX^!Y%cmTeu|0avU-QFN<yyxp*X^mCoB!?5m+O1iWq%j@_2y-W
z<IJWDSvi&8g&E?%i@1Du>@3<`m2>*s_cQnIT$#t|FZZ_pK*uH9L%$|4^%q3y&B(vZ
zE8ueXQ%SMH{{Yd*TR#^wK1vQ+v#{XVqZ=h{53fD@xbXf5<tKgyPs8$_m$^o4Z|C^^
zENZzyiu1zvXO6}kkN<w*{@&Zk%ECIcbkFOYf8MB(zv1Q2%bPp0uh&fa@ZoD*#g{UX
z%jNGs{cvrmTX20_htUC>zsnYIXB9Us_`JfrgPC3WKjS>9Rr`xSWGtvqH(n*)5&i4M
z(Iy9nMAnH@->fK%oSy5>@apfQ_22ma=g9SFU$8ztp`pK7opbt&%%#tHlW!HVy}Q6z
zE6=!adxxD*Elbot&78g$!Ls5VyFS_p3UUXB957BlWy;23mU5y%>2T{Av1TrQBb~!X
z6D1i1)xFk4I58aSmNewLaATtKi3v*XlXN7F7B%Rl26TPNC`vv(Q7(4hlAn`Q+Gkt2
zJ`*{an&Lmf%<$VDi_d9}J#!o;-He>HG<cG|s~)Rm=z-9QOS~p#Ij_2UeZyLo9@i78
zURS1P7_ReuJ?$~uj-X{8GX*~B#7)dKn07N-Fh!HG@#iNIho;Ag|JUS;C#$F)`1r^-
z<<`xt@b2WbWhrh~KRrLaIlX$H%WdI=^VeQpX3E)_^mbLK^o^yb&dl>YmdLKRvv~T2
zx7FM87-gOK{8w(|KCXAvQf&XMuX<meOVkG)x3bYV@l-o=vr6ExUhNN$n8VX+igzYG
zRP8@BWuBas+Y|kZ{^#TBt^R&LG_hQ=s^mwhknh!yU=>HvpvJsV)k!t=nyLqa<`$?Z
zdT=v|O==Nw`XSNYSNb(kdg+_gYhr4?Pv=!CEDU|*Q8-m1Jf=+d7;Ep<g+C@qbM6zl
z=pey*jLUsZPm`duGZ(`Wk2MZz4HI(;Sj05F^A25{qQq<YbkdRu8#}cXW6N$z{C|`1
z<aE~78<)IvGrB~lIdktkIl+-{>2aaCin6Iw=E^^`6!A29D0#I}W#+?EvJYF?gDR3g
zie_|uF#V;mWa6<S%bQDu$~2q&RBtTkn`9?(u}N&-_7F$8rqWd%(FF@vdFmKl3tOVP
zOY1-k(-GI#EM6y0h{fdwG79C$>};I4NP<^d$vsBqnUG|suluotd9zYf+{447CrwMf
z+I%sUY0KrVl&w<>r6u=%J;o^hV$tl|yAE#r9azX~rZKfv^Q(xA+v%t5lZwt9Ex)nj
zq}lZQA>tR+;wqLVT)*fadrfT~gUl4BIO(T$Q}jG}t)}Z7|Nip&#w~tlrHz`^E>4*|
zhp)^_!tuWh*M_2HdA&9dSID?}Z0=Wd-M#s&($+VM?n;w3y;>!`M&rO5h25`Sb;=Y@
zFo~8q=yj>v<M7tWB1)36kJoqaJh&p*TWRIx8_D{fOSW3~9%@V0$$P2mwyhwATQl0{
z<FYyYwIUM_R=@r!c=FP^FQy&|{Uz-i9<D73lI(B}Ii<VqhwBQ47Wdt9?r#$Lgda2-
zZ~NdU{rX9IRGV{`jOz@m(0kqs`&3?e$7~Jak_`S*wfL%v@7v05nUgcN3Nf+tU;MDW
zJ!H*BrXNddE_5e7S>wMa`LyCz3EoIs&y+1ek*P9UFCP<;7U4*!QGU33Ug^G<)1{9u
zdi}ca$cx|FL3`9H*$V^yGb}p6=*9Q4cEx6o3marEYAEF`h;3|3)d_#1Yx(T<$$bhL
zO8+Z*xXljAKEIVdi9_gk{OQP>j&4p1mI%B^RWea@^WxY%p`%N9rLbp$iObYUF@6kE
zrvf}Zjw*gy)5W}PvgoCG%eHLa@jN3tQlh6uy&-n5N_J$hU|h)2U2}Q78?6s{I5m~1
z^oejPJAG2pY+w;fYAx(m5KNx3Xp&i9MyBx5Nz&V+63zGh^(}Tfc}LV&ZSuL953TEt
zgv^cO5Z!xsVO#xAKDmud^P*jzC{9?tP+57plaWhagn(dxq)#EYv)B5`^GZ(5cS~A1
zd5@~2wu<V0i=<tj98bA*RrXc0rR@G6`OS+Vw^7P{Qk!?oKGu$i8l~=6n~psEnKDi7
zruTH0OI=z<twHlyPEP7EVSC-mQZ2D^(l?e3x{NW)MDN{cie<~RyAb)bam}9@mzO0^
zDJf)3Vww=M`H}mSt~=bRTPJetx>xDYQk!e>;G#!?%C~bJPl5|0EPEWkc*-%Cy;SUV
znO<?~!kp;|0?p=~sV7|8{U#o5G0>0}aQBhCB)qC`3Ew5pP`5TANv>t3W;%C*l5Sk$
z+Pf~D|CP&Wi@?RYKPI@@ow-{w?U-=vvrX#JJFbQv5*Lbj*VU<1xafm{65}?X17#*j
zJ_~%7Z{KL~vd3C;vFDT{DnSv(DxpFLBu-q|CX)C6VQ=x|$06?9-Cw$<OcvPb`%q<*
z|B8m^rg<5z(=!gw+AqSTojPM?BIi|)TOLCF+L}{TZ%fMU{rFUI)|<!1zE0=PrcGJ&
z@=P$Bb@(Me*%z(Ld%F6TIZX`Bo3gBTR+4ALa;>OKWh(287H#pGA{>9~)U-o7oQ6x?
zyo0V~>a!;$D8Fs@@^{-j%V(FT+3SnCJvH2OJ~qd(KVKO;_sNpopPQ%usS=WK*8CK3
z>+$l1L244WwkX=9eDK!fnz7+-$OPsSmLj`OEwGp}b)h*=?8!L37xz!87B4t;FvHWr
zV^Zh2(&7bISFk#7U;miH)y|nSAS?IOp;?PQq^f+`;9<^UJ;f<Q=|6XprbmFzBnQ@I
zM~!t)@@B6Jo_lRYd+>*a7vq#d6h4XQdbuu2N?LMci-Tx|P=Hg+@_?&zVy-SyRE!A`
zV+lJ^^RXhWL~`2RA1VuCFK@d(sfS_LEw}B*CWbXzXS%v&Dg-X7RH|$F<k?o*ow{y`
zNV-?QY}h4(C%1Nr=3ct(rF6Zmd*`EojkRwUN}P6(uYEhIb!|-M{<Ndl(zabvl=f<L
zy0OfwbcUvy>{HF)YYJZ%{cQBx$C+gQ>=YYon&!GW=iV+$Fuya`N^AR8seH?z=?|WH
zsd%2__Uy5qlwiC5N?YT@Pj;1ipRe1vMv-+=$xct50?!7K2MaxHmroXEY+^}RHRpeC
z65q5H%ta}8UiQXw{yHqwwxvAc!X{_ug<t*FznCoY^GM25F|7u3mC4o>KINSIiZ#5;
zRFv9Jo%Ldx_u+@m@tEgJrbi3>nHS~#XOTkl4?i2HfFGW(w{S%jdL`{j<cwXamC&6j
z>7u7Nv&-eE$4klH+Eq_Jyya7I>bVuTzD;m<2(QYztW}=#EQB;=A}hESta`hmVEvW@
zKh-!RZ{FOpV(QB5qlSJ;tNSb`T{9_Ku~ykrSaL}~+gnziCsl7xE1o<n-j#SHk!{Wl
zmnh@e?OPU2Xzflh%0E2IvH7?db7jY!@FKPSFRq_w7MWlzeZ9oTZ^^~Wg(rOUuFbv>
z^iTKP9Zt*t(xM_o*Q_+cPue6_eAy(h>_Urg;z>>0cbONf|7hvUbhc&57WV0~T5k34
zE>5Zyi>?X^^PkfoJXh-K1dFfNw{f40n-rTjL7+xi?x0>+h`-Ts3+Y{JR<xGf{-}~=
zt(K^0rQmo!d&Nfyl|?^&CB%=No_p|wbzsWH^de=m14d8G?{Aq|euU>8+W}>t`GRcx
zVN(tq`7)hV_gw53wtN<L2TOsXnAVyN{$EP^p4~b5#&yBxztQtz;^z0T9J~{_cSSo}
z-;Iu`bJ})ZYqRO7oWS*JN5Hw37B5eamL)Q6QyBRswwzwdvGR&U-`yFjp3J(sri1@$
zOg?Ai5{IcCEY~lFipB-}Z+vaZnthhjqlalykJB_6&nu>E69XbeCJA1*o$fIw^wy8o
z<Q;-(9?~~F8^6uzj2D&iW|=l;OPkA$$a_-{&XK%kt=gmA!6E*pRkx!<e(S+Ti;yQf
zB(vYdyWE(oe6wlZLYIUu{m#Bej)q)&?fKw-#e!x|*Q0_Hmqtp>Q<&yCVb=1uEU%`v
zXd0e9x^bGi-Kn>QEs`6qG~SuFVntMyK<}E1*WO)?xt18H=gE7CA@~CWf5F}(7ZY0z
zryhw&nd-8}$4QY*EN{_N4hD%?Em1R%P0yLh5_9yhijT{I=-!CviV011vH^c(Cq(7C
zC+G+>i*hKelb`d|-P-Y%g;vkOkG}PW|Jzx2F5Y=3`L9bvPV5}ZT7{*Bb2|ShADcSa
z;oD-?v_)rjD4enpe&ZqUSobJrjg0q+<DHRrs&@8y76_ex)AD-BX`NhNkADi6?woqJ
z<J4N6ixVccT;P;-V^F?0)kR26cJj%%mjTy)mxj)Wof#f@*Skb;>l{&!IiXP#k50;I
z{B+b;yCpXG%fduQK4#BpyDcZlo^g}h&^G(!<WD9^2SjE${}fG-yb>?+Krt{(#faxg
zDqpG30%Nc90T<>67u<_3n3m?MawK(@=H6M$=gxcCJB91zk<b?&*Pnzg$VhpRaU#g2
zrRB!F#Vvi7FXtIPTy(j$rT-^i2&;d=+c|z74ni__ugUzEcbmcJ!SLvJVaq4S`J4-n
zgs40<4xR6r!Ks<TCX+co(Ca|Zj&Mi8J6oTgNoiI7^yli|3F1p;o_>_$e#~cHLBs^6
zUCCZOF7-Ccy-##=7YY0<IQLZXd1b+Z<uMW&fsfeAru=yjSJg6UdCv^zTOqGv{9KC~
zYf_GKmmYN86~X<==fnn)-nS1Tzg!8tnk00y^@DDkV9}8eGLns44l$eu43uPZ?j8}A
zO4-QJ>8EslTH{5JfQ5lV+~TD-w>TsR|2R4&=a6WTaL$5<-z4O6MWT{IpEBNbe9Cdc
z`sP7HxeE(ogO0Aae$=6*WW&<Zuq36#2duXyI&NDU%a-yWL&T58?~=uTcBd(E$Cva>
zPP-CS;CS8Q$coTgr~W?nKa%9~&!tIXrLLpehPR9Kx{h3&uyWmwC6zLBSyUQ?n$mV^
zUU<qfBh;vi!IFEIgUU${VN=eh9tpuYZnsOWGHLoabUt_{_h|Y(28A7*0=rn|?EJ!h
z*kskOFWsd!bF?fcSQ&a0>)b6jU0(Eoy=uuUd%h*CY?^LNYSWfwt?bj97!;GJ@PzZ&
zs>C$07YF#fj)<H&({uVHyLYC?mOBbO4;Qpu_3*zq=dMKW-5ai6`&eg5y=ahIIE&A}
zsmUc&Q&GQ{CG(w2X9P!h>Wic)XFU?;dNw2m_XvuuonY?i_)AtOUQ502QXqp@Tz}U8
zgcCe`H-5A~xtY=*^(ZOkx!R>vkDPnyQ(0aZ9>4fzQWD42Ref<Urv@DUraF0F`xoD&
zHIsO9?s6nAyL8BJj{Uq2U9rV_yEr^Abu~=ONn54juJYofoQ9i*L%Q05iDz$pj12s=
zce2w>8DlTk&<)ErRX!|keQ|Te)rb(c?KRw*IS0h1#O~qiDSGD3*0$V#1OL7kOZ{Ye
zG!v%%;Y@H*WzTn5e!VPr*VEG>AKc2Dp8E!P9lqLf%de~4N@7CY@)*-)6V6T8qQmh!
z_Eo6qbIZowuB#8c8CEa1a65Bx&P55|t&86HF38->Cx2Qg<J7(u76Cm`!~WnD!8WGu
zi9NmIo6bI+_Mc@!#yhdR6`Wfew>Hf5SmLSe!MHl%!MqPE_BY*m`c-hs+OXSu8xIKf
zIqwzTFL=yKHq7qdx}r(Xd)Azr;B;d`l+J3t^~HUePwRRPIVGmL<}Z(3byHAzO=#HE
zPkvhSg0*(>typlABaG$sj!QXnThH7&D|Y47g`dw)-eWn<Hl4+1HG@h66HCK~6Kh;$
zz0b)-A7ML_%hTqUc1@$_;HK7fkxMUxuQ*mR_as|~?3H`12hv=$oL1dyQ~sOC6UK71
zLc)7`R<33Y%TXDYJ74=RFN<ckNxwdC1Lx<*--_Ni?pn1)<H_O|t51mJtS*^U`ABEU
zmNQ>o@hBg<;l1>RM?uOGg=If4{P&n0wtCXWCF?GF#T-rX*z*4ID;K64lZ31uW_YrC
zd#rd~kQeE?`0JF{5=y*h=6Gsri0+@p^04IKBSyc=>tdyqyO=oVcsQK#Qn=3Gxir}|
z!Y<8a-=Vcj0t70|R?OYOe&AKX0j5@^+b0#BkAJrNr1vS}(7s7*oqWspv>adPH*4!s
zj#q2di)GwAJ7%foANtmH(Wi0JSy%6AIv*{MhM2xH2;1nv;xOCJeWQrP=@^!*eM0f)
z))wDwnbvdYLeTW>HJ3#+-g+<aS@c%i-cHH>*xl<N`WsSTZ4C^Qyprjzamq!*&w`I_
zf9JBo3f%)Ak6W~_{j)^(<%GVQT$-$}GH!Eyy3F){$^Mv@@|zp<mnU>?jSO1Sn(Wfe
z_mqe4XV&LCn}g=G3dx&zh|CO8yy<b`eY6VW)^l_Ge4<|DY&0x75mEEbgP}9IhczW4
z!|nZ&^V6nIEO{X@d!5lq-%H<w3+Ep3|G|2srnhjWQ`MiVpB^qrG(DrOd7)S6hK%^7
zM?O4vYR}HQ^LR1S`okBqyFBLbgkMpYeD7DWDDbILKw?Z7hg$T~30ZTaPVJbNC3#yT
zn@e+cOUXJV)%7!%Mrpk+(L8=8Xx)NO(faQ^j;s&qbV=88c)Tq~^mLKOjrBqcio$-l
zZdBl(S25eOGW(1N|5lxETWb=vuQcEB3D@4@%=~)#Y{@faK1ZA~|0`GahDS<q&t1_S
zD4t+Cy+KnXsLpfBfs=Xp*A{d8$=TJdwR@F3!D;KAvPo=WJuAz4f+q7Ubvs*aQnd13
zhu^}4HKz(D+|ru8@r;M-o3<+#f-Ksm&uVe_y~$DH%Ng0c5XX%|D>9wEPw6lE<RWZz
zxX*>b*sUXN;f`rXE_E1A^;tZ<V)EJvjtk#jm3cqkC1mNG3GHXql5+1HD(N_GedyMc
zkBcSmFnTdO=-SEHEPbo@QA*UY?3Tq(jd#1|$5%LJJz8z&#d3Obdvpl9$Fmfp_qv^$
z0gM-axarIjdRC<GmNYG$Q|flJ#e=q*_dVfj7Mrm4ww#Z7aVq}zj>l~-kBgQuefV!0
zKfC4ct@j=b^E-@%y`PwE>5y~WttM^ADeo0+sGoXGMXJ%(bIy+GjJoGGoDhE~F{__L
z;dsr4{3W*+XjpKXU*gn1=Pwf>zh1T8<oTTY-)@>kNLjzNJzPA0d+y>7oGPy$Z57(m
zrrS|+M@zvm*xQwRdhQH?`wHr3b$eyE^naiA*y@X~gNf+!903mh(6=n}j9A3mTO=0#
z`d#|vX2a@fnUUgmT|AQ)-p+V8J?Feotg717Cha?J`9%k_HQbzUepEITyK_*wZu^Qj
z+c^K{TMn9RKK9X4wQbU~2Hqlh{S!CJ+Va<gR$t@pFOB;scXh^&^D`~<Vp}!C0-WY9
zt=idJlJZ}F^`!52go;*I{<e2J-S@m}AM;OxeMWzij>KJmzdX5R5_5QT#Ko&t5m{ed
zf^^Pzd3zNbmK8JiN;__wp3!z==Jg{67mi6k-w?yn?bT!L`t)E;d#F_w^Q7Fxmnw^x
zVp`jGtB11}AG@#ZE5}&G?v?O7Pd??ulDAgk@7W|%^4*UX__TIJyLQ^J9|{XN{^V^)
z%c;xi+%F#R)lAyzx8=;a6AK<BRiEbl=&(OxX<-w8=+%i&yO>JK%O|%i=RR;N+|cE7
z>5BI)e@|`QFokcHF8{s!FJg1Q8(RJf*mp!@N4ZCA;;fisEE60R@2$IcKx2;9F(=6f
zk9Z>d!o?i+u3cc?_2%FIz*(AcpHEioHIrD-JgMOIp9A+;UQA(lAzQmt)mC-)CI1a|
zSHoPGx2f`$7X>f*e!ZG!F-zLE+m}RAb}>Ei`r~)?qVMZl)u%#APi}Z%<eDG8u~6%V
z@-DxCc<BH(kDBgj^TQhj7aWpIXIytVaQ)L8>r%A>H%RM0IC0?Gik*#`4fp+z#Lv#J
zG5O!}=kk|ZU1A!ZyPvo0liSJaP$FP3VL^isD{Fz|4p)Z5ja-6mDjbH1j$NY4aZ`3o
zOm^=Vw4C~|rQpedNvRxNA^}|rE2pWjPI+=-(z7%D=7n`nj1s$8RJrQdLN^$8HY{Qh
z=J1%+kSV~(SS0thBS=ZIDU5y6L#NRH4$>`p#as!_6GRVgWIf2GQT^@liuJ4~m#9Xp
zo~o-Sv~A1HUF8Wg_o|vmO{pk&;CQHsTiE8z#vbV@u|nb|IU6==AI~%3esJrDpX8K0
zooulO$9OW##Z=jjbxc_Fs&}pWvU8l)i`{RnNwz<;*Xq%f4f`AQ@2tI$=zMdQW)SDT
zIhMsIYQwwaI)k2-H0g-irSG{_b*&|4nNNU{V`B>o*9Cpnd0Tc~zmVrusn@(!;d-MA
z%Mn-4`TvTx?SJ}e#r{UKji;t{+){Mr;yb~>W6&t{$icixVUNe5t0F-i+Ik$1KP==b
z);?^s`ox3{Eo`S}tYKLvrNg?VYX#HU^+qv0>o<mF{I9vFa6N}P&#=F)=fwsEOQlZ+
z9NErml;wh@PHca~)g+|Yt|!8$u2nmG0(Zx@ITOsAR^_bQw}FYJZ|R0}J$`3QIUlju
zd~E3YtiJMwc88|*Tq%w79XBSwQa)aHyG!TUjbHDTr{7V!u$bxYB<*z_FIKhk>`D9M
zee%AI@{3Jpj<w`1^0M8{y!n{T?NdBw1DJT0p8I2@yycfqrE2hHx6Nye!Wwz<JdUz8
zXIwu~#_}y(wKJt}k=8_3<3+riQ!d}^kVxv-(Y55uB!#!fw!FWYTRcfqX?4t^{-di|
zIJV#UbB^`<F0+fWnO(fcy!{(LGX*ect;h@NSFTYm%D=mh@6dV$A%*`!5=!fLu&(oA
zyyG=R+2!Pl)$^h_vX;&@T5@B7BcJK10@L&7Y+f7+docNei8N=Npz2Srt^Dq-I^93F
z7%6BUJ*dZE+{-Pb^?9|DUdg)!n}lMf{Jc7Q!W5l{OB-X?-8i^lnO4-}1-x61XBBRq
z_Od^0n#YoL$=;_`r>{ybU*K1?YUQ%VOLu;$`Ke8CI%m<ns#@WzP{S$Bx}Uuo4g3Df
zTu@%Cq?NrmOhxPThd_~fZ>6AfMHL?!FKG#^VP;=+Lo7%oBg{Z4JnTfT^@$mRu^0Aa
zH9zs$`|0$SQw+L?`Hr}6{@52Lpm%htkMO6R>&+kCJ~h=^E^23wwU^;y!{9ptC8^=}
z{$IFu`gXwOs$X|kosatcCf@qpX_fu_|COd3D4u=xQCIO=`=@KZ%jSH3IDgOM?Ezlh
zix?uEW-p!iNILU$#aq+$3`~5ArBPfA4b8h7Ti6oWUiKf1aBedeJ*8tQD6=bXlGsT>
zm!c+VR>@n5-DLqi3U4!<k3A_`IM<fNzAkY>k4R9LnFy~P(}c(x53%`+&a#&-e(A*%
zn5unt&N1HenvCZxSR}(;mK$+0_DYHbNzL@kcRef^<kzDjsKTq=?fdwu=<LlBSATT%
z%$;<CZIOte$-N^^VXr)9tP)k5d}O`y$AxpeE?wGIzw%@?*T-q@pSHViInpnZ|LbJs
zqzUG%DnXj8C;taCH90s~&DyB2W%Z1RBPlyNCOA#DP;5SIc5s8g7)vqJjXP}1KG-qZ
z95<e>7%%xsNh(~yuiX2IlXOfX`;!HH8QZ>`d6&U*X3dYIqU){Z)yO=a&bPC%az>%x
zDu$&q^=2>{34EL@xl~;$y|d7)^<(IF0j3!(ox(NJAMdDKZ?Suv@nnvS;>3(io#)d3
z3aZ=+zAp3ELtu-mitoS2k42U>%&X$P5?ABIRqXdzG-@7OXlQ1Tcg7Wm9VaF?Tl_rF
zo~+qWmJ!0RWAY&ukA+s3BiPbD2x|77P*}WW!a=RP3|8+%4W%CfzHYNNPVHC}yjUn=
znTX7bRM(u+BP}w8uC7PL{QmFKU{F!*U6fOG%V9>><P}9rn9nz!5IdzlM>FB9V)TT^
z4i7vgtkiw7N@il%k4NQ>6Af0*VY;zA`eD_!9mkeLm-tS;C1-YW?wQPq6W(MCyi#ef
z<$V$-y>haO>D8%orX+HQInGhltvcz)m!+JOc$CfDLZIEWOC+Ikii_FyR?QtD#>QJ%
z`&?Rftle^mPr+KCH!qOukZ7^?(u!RD+3ntKa_@aUES&l}ifyLenT#DW!B^wgcuN0P
zF?ZQ6ncP3g#cxZMzxp%lRbRtS?p?;_J~!&{t*vW1C!C#-z5Chj1)={e6sqQ}=E?oB
zSZ?Qr){g=u3(BQA82Eljv&0oO3SCum+QRhTahvJ!fID1EodugNHckmzoLeFO_Jn}Y
zyi-#|^zY0K&B;8Ra&6UA6<x;9K7|4oR-6(s(e~*II<oM1NTEXP3kRXitLIi|`5MQZ
z5tTR`8Jm>(@z}YlcavH~Bo0kc^m%*dgw$^938q{Nyu9O-zux#E(7xen&O__Y>lTNW
z{QVecZuC;U?ZvC(dS?U8Qyv@1A7wkSbf@Nmjx*6r5;q0+eazl(o~YsX!%5)(H*clt
zXCBOdeI_t-CtPU0rt?VP+d<!mNiEBAj*2Y_I5TtV!!4g@#!3g>;tHSQ$FTi|s=uj@
zMo8v~lD)#4mhRcH?C=i#J@a;*tgv=E_q6R}x8=8Gw-f%}|Ng&r>mKjf*L?4-JQbJU
zeTw1Or%f_hveOG%b7CemvAoFE-*a{6xy)dll<Yqju16fz7T57;ygTh??+^1NrWrRJ
z%x-nL`!ATru{(1u+arn4#CGPxt{W34Rcv1PXw6ZMi|0z$i+)(<SE}rJS8w8?QyW|M
z{J3WQspphUu$7+2iRCgC;S~nwg!v4f%n7hs?Y`&MUG9~|SwTjYTQ_Xq`qeD+*u{tQ
z6%s#P+rCcb8EfQhmrGNQdx{w?=;S#to3UG;=~M59TJAR?eY#Rc7A_0V9Q@{RAl71Q
zM|9w<uYS_UrcDjBxqESfTc>tLQukd>uFH0(jaTR;Jw76Gx<l?%-u}ZU!=5kO^MCF2
z!|(dHyO$dzhwtu@)U#ncKee|wfvI(V0~4Qp<L4>uAD=m%m+G~7b6>@>Hu4?!lp@`q
zYgeYl+*{~Te#i5AXON!u!-el>e_SoX5_VgeV~=s~=?3`&HB*=JKm7J&pX$UFlglk-
z*Kj>Di<(fjB}rJAqm$2c%@vh>5i5Q!JF%ZxFQzeEbXBmTh0COGzA~39j&HeFw|3|H
zZwX6(T+<BdY2*7;aCXANR_!ZYM*~j^&--yi{l!XqCDs`&f0SJ~qaW;;bXMzRUeZaf
zq?40a*Sz6-x+U$=u9cRXI<?+(^jj?CR$zMYN=xV3>(gvG%rlb{*-k1wc#>bIH_6RF
zy(Le15vTh9Z-;MlEqb|6*FtJ}qm=uFyd>dMN*7NgT;zD<Q1M~u6o;k<hec;Bzc!ES
zj|lTGy;Osq7k^fqu8CUfp?BoL43-i*rf4I5#bpN`ap->(SpV^yGUFL#p8W-JQk}ME
zl9k((R-5Q2vIv~n(ZICO`Np*mxwAIRdo$ttnGU5_?Xl-x>+WEPTk`gKo!(cLgs(P_
z;@X%B0}i=*v?+XWPB@|Jv%{fjLErbX{A`<oI}AM{S&2zzT0deMi?=bk#&jHK@!KWm
zw96(<bag)8>{SuZ+?@kdgP$xe$y)ztTLN>D;p2I|>%a7C3VHr-5<J&2apJT`8V`Du
z{~Xa!OE5jArMRk9aSo63{|#PBUCAdOd7oO=l66h<X_v2$9P>OUCFK=Kdv`2(Z{wtP
z;n?YGsVZ)d*5;jiR<vr51*3rL2RX)cO}5p=HB2u*=-8*_Y|{Iv_UFu|6?z9PPO#oh
z;@<IYi-#ju6K|GIzSAA`4>~~)7V(@o!Y#X*{XN$qrr%CGnHFc?OW(82g<Cy=S-_u(
zuixg#g^vde)XrbvVP{fWnxLTaKtoA^bIAiWRVOAJCfiwsmixRvX`S!2YN~y3LG<DC
z6*g>EYYxvhX{x!!+Ptc0@;af{`yQ*mVrBTJ#u)Ve#k?kkZCN&bO@i!9iFp^bCbx(L
zBq<#1wHN47)_4)KMo+2c{h|{MqB?B{|F`uQyB*Sfth^+pyU68Kng&bkG(Ekd0DBL`
zLyw$FukgOv*Z$zh5)ON%-%3rtj`?TG<!Q%d+CMw2f3N>!nc=@F41O0LPk6+_^W23Y
zE|`1E0(DL$CN?I{9Aopv4%`ur+$;{<Tq(jz3S1`|WTP5IYJ!!fG$>5l>^DV0np^3U
zOSa;nX44`;smmM;S2@myXgpoUbLxPgvbFw8gGY=vpDCSUt2q<+C5un--7D1sr+yE9
z^<!){VxBCX7aHa)IyWhS>xmnKSf=uZM|1C;wBEF6qspU=S}TNhuG7)cyl_RS&@Aaf
z*W$z&C+)jV?8e8Zur%oTooGAqZi`ch@vB3kd`lYtb2Rg$d6xQRPji`4<}^j5>!esz
z-=rB6l-z@EXNk-f^S&;Z^=OlUYUC#F*iYQ?L26%Jm6kM#<TUC>Hi|AvusEI|yiI{?
zf-%#U1)@t9h?zL*MkR`5DNcE{FvhH1VOk(VgOCPy64R=Qns?Kpl-AqayyQF6&D8Bt
z)4U=*^{ks|%{)H(0tbb3xC;X~mGZ7^(Z4h4@|mp1%hJz?m1I8(@;T8f{I;>B>Ai)c
z)<TbkqOTrqX$ZKh;qJuZ6#B-orSEc5-{m{AL|WJuiJm&7s&~lfvAX~ABC$v|1>>bA
z+I`}@p&Bht|67<GmL^$F7xsOvre1tWW71a*l?Q6K9k~}J@ce(3$aAGhHe#XNqeWs%
z5_oPna@|Vw%5@YLa^TvMAX2kXcHR{&rtXNu&9U8~49gg-7N@M9lM;7*igc^;zAs-T
zH+DT)V7%{whvF)ixT~FE+p|-TeaznXSn<GT?<bREZwTgIn`nD;qr$z5>(23LTNY&g
zDPfxQK>Pj21@D%L_cpCjyO_K|i(8*>stdPGlv&*B)J+#S3$0EotEB4)q)*xQu3$@v
zx7cOXxVEKfYj|Rkewa1t+N~5myhX?5ic2Jy%Kwe;zUA%l2%5F(n@V$t>dpz=-61^B
zS=G{o)OlYl=t|02`Y7jY2$S1Eu|*8$vL0?aCgM~5*)y_HWWoc_;}4=2HzfXdOI$2*
zJaNWUg%ix`?2jzF7e_vtBE4~0qG`YT4DXj8CitCI*GfKIc+pkyN-DpnzaFoN)xQt{
z=BxQj5^Na0*vtz3XQyRj_v&Vtd0v2csiabA!FjERn^rzrq~E$;)T`0g^Rnt2M}3Yq
zlh=zx6qaz;bn$pi6@7F_?2V!b?~|oJgxq~j%I#=+xbFDT2kZK;=--xm7xtXl{$Bp0
zvci8spKqQz9L}2I_iDYuBnRKy<qG%0BUz_MKgi*ElgK?mfvcxj*rg-tqypEr1;JC3
zf)#`A3b}A?OUT)_ActR6u}w+SXxVD9BrC^tR@;sDEm^9u_Q=kC-mQ8`r*2JHo1yn*
z!r80;W6lO1ZgN*DIBoUN@<HLcG`{C5O3d=^ZH>v?N0?tYOjBU!E@3vA*k{slTl{0t
z=8nIUeU+ICO*n7wc*i<<$KoH|Iu@LPMT;D)UKw(#=vHiSzRKIUJxMrgq1c<G726J}
zroH^*a!BmZjV{}(<%a4lQKgrkY^aTRxy$SA{(Z?z0YNs~wtYOs?781e?ev5KZ!^z}
zW=iKmBpVk;zHJma#^7V_$n|8Kn4A%(ohO&jf&WVsI2CdmCx%YADBvK>-#DKo?&PPo
zuYH>0<?o+NVf`Ceq&v-5;jv$(zw!?gVRn_*F5Phf`T^gyBhM#qc#|g4DtPXhn#QcD
ziae<^RYfOjT~z+hv?Nh0Ti9}Hl9cb=08PbP2gGs~ilp$Tw`os5=c3SeT{Q3E(x6jf
zv)o<M(oIq|n5KG}M;N+#sXxk^ShLNgM%V9O$n<~54(q&JX4;;lG3APnw^^Owty=!l
zh0_HixED3PPSa5MwSfDO+0@eD_$=-TrK?SDEo>CEW$CcJz^on|E?(Qk;`MxPX!}N!
zM^99otUg}-e`1xdw37Jzz`{sL#*bU2pU(76S(I46D=RkF{cF1Nnc{U;i&p&1lrx>s
zFZYf4rpmq@M)?k6S}%@i9J%DjAQI2CqQqHBe}iw@;YSL#ha6AtTBN0sZfx@EO77~g
z*Wx;v`W<0!-o4oo<E8&>@tgl{hKgGj1wFc;v|h|(>xac1Q$D`o=lTCbRk6q>BPd0B
zWzNZK3knx4^qy9$I(rfSPvxSb$rD9=Cx&RA3s_{%&?von;|Ye>epznbU&MU2Y>iFy
z7TTWG&|LVnZ(H#rjibx$4rMzlZ7)ugv~Y8>u3-7HM#=wp%RH4=PubqoGl=eIx%iQ*
zdx7;Mzc;V*gjO~jPf~C{KXd2r+XtF>BND%#OG;#WyGFh^_~DL8a=Ba@6W#1p?%(P@
zVbt>T&bb|C&s<V`O;nz!IP3lU^>hAg(Q{=%2bv4jdH3WhZt`Y5VD@^uoQcK6n=Zbd
z+c#<2oO=|`rDNsA?sT}_*qmF*>-KkX)`rsml}uCptd^!{EP11l_U3?CSj|yK=jlpK
z9bS&{tNpeu4rFMXq|6ky>}e=NmCVTrK5MFKs`hwA7TpTm>7sn8c=t}V_clv&A6%Hf
z<@qivM^&$brxe5HpYwUP%16%fL!$9SyXZr9?jN*mcS)!=E!b?e;hC#fo`F-eXxC*U
zr|fdwCo)~8nL4kg6`h;qAj93wnd`u(5_G4kt;uI*mFhJir~c+G{U;x1?=_e(`zud(
zQuO2mb<L_HlO3Ww6**IidD4Qp-)&!M^m~H&PPP6+>h5zCB4Qa9IdGLtj@ceM`S-6a
z`A@c_xnGa2K6>=sQBJ4nY3XHFUdNu>s6P6toK(G>ciWBsJkylts-B!TsX)Q}`m-O~
zjfGC#49+sWrYEZ&n{?>r0$l;sT^gSPR~IWyUYqp&nyvZ)-puDFyBAtn<b^j_u38**
z@MeTpLeKFq=S`2qk{<ureW^al^m6$`7OA>KV+UzXCNW*zEenJ{nsb>X@|ZYsO;`}a
zen{NSk^9C%F*`ZSjZ>8`nQT^{qoGlAII;Gr_?*Myd!DK$<u7)4Rvs+8Wx*r&M4qrl
zQM<aNR|~~o9Tb=Oa<q8mi&oo>TumJ{9P01ZRC1Il|5+b(tMgOO<7J#lQWHPDT03z~
z-{ScCXRl{IJEfB(^Elv5K2z4-KYY`Zlh}3S%Fo%=>M?xbue%!K%dEZnKYJ6;!|fKc
zB-n&SX04CV`)6~Z_`-JUDczcB$CaCyS0wcB{_ug%dV=QPerZl$-T4zZn-;W3GQ2q`
zZj$)SbD`Xyg^$%G18wR>UNwrEB>I@H65Db>%<bSJ(R+Dm_D>s|ujU(lkB+@oDc>L%
z!L2h-M50~v-ZtUCj$d5&%lF-vpJM;>@4}#^3mV)0-VjXcR66vlQ(fuZfnVH@#JD)U
zy4GrJUS;~?>JIfyalbX@&bpwuVy{hXvinBc+x9`XL!9+ayX}ojTAH*dnN{#1qf%oN
z11F1!kCK9-Q!A&mRmlvOL(9U}$VKh(SeAIKMOwZrrTC-L$_Xllld@C~gdL7zirT95
zUpTOfWu}*kMxcP9XOoJB@Q*8;CC|^b2=cfkns%*lY~gm+3b<t8+2XtJ&DBdwT4UE%
zt#*m+Ra$V(gHdqNl_c|Jach?>-m+!&ijamgEkRP}L<N>=+*_3T^O@@HfP9e+&07<$
znxus!g}sr;zx!(a{PlJJ{`jtwvHbL;Wv|e;Y>l1s*n-NuX4%DVIwPjXxt-(26RRiF
zZg8rbY<W9jTOOmd-Xpb})`=h5_wIb-XQ8^Vknhw?^PaQ25=+|81xD*!7N|a?cg;8Z
z{Sn)R-+~^qZg_WPLr4|li>dZ1{c*L`jC1tPh39XqJG7DML%F`Ckht)N5@CNU)`<=R
zOd18V{5d(V{J-Fn8`Cn$S=5YWdZuKUNurZ@)R9vjo|aoWU6Xr_g0zzxw**>inO^bI
zo&7__k;(L4#N=M~yBQN0gj|jYB$Vb}^wajt+34A^#Wv)EtNkvwi%~&+Mqai|3KPRR
zzA0Smi`ly8D3`YT#GvU6+*38L&s^at*eaImwXsz!S;f+8#?}o?k+%O{oJ>!vjY?1T
z{Q6@NtE5J4Mz{90n-7n<c10~&p}noZ(XCTV_4=$FCY_f8v&1*DWqHil^*mt3l7H!U
z3!mlgdgXc}^Si*B73aLPvo2dMiC-(hX%OVT^wM>Kda=}y`#F|c3IPl}P7$yDq9<HZ
z^i*gyS<7%W?6rW$jVSK_51eyMO_DrPdYUeE$EscNbBj8#%`+)^g3`0*6xV}kEI|@%
z)5J>qHo4A@+W45;I&$sOMJy`}Z%kI(Heu2<1}5`moZ4YotylMpx+-=ucxDDoTi)`J
z)hJO(I&H@8)dp_dw*6Nxcc)C*JLl%<zX56o(#@{-Y))Tb`z12n>b}fPi71sT8H=w;
z>M~qfsu~h?ng8ZTA@yZ;+n?kx_1p@`-{1L5CZ9id?$+yDzM9=BSUyYluGksL^gFB8
zwVralbLr(TZMQioC9l`M^ZmZ6Zj+ixd~K6?OWeAhuXrsUcC9{MQP2Cd>z>D-pYG)z
zb8-w@eNU*(U~uFRz7jB5z)QfviAPH1|G`O;aX%JjuAM$*p{rIFPoDEMzS}`g%99jz
zyvw`3z6smcVi{;%+;{Pr=aL-5Q}36&&`z~=jyL@f$k}=#q}(Ip;ibTg$n(pNE#9!R
zcJ>z~L06%d%j4A&qYiMYC(WJ^q&~0W+e-D+pYLbRWcb%2qA`i(g9-DVj;UX^?3ps<
zNoje&0%xfeldSigQ_Nd1h2g}8S?w<E!Qu}^o=v!Hs^`O4taRndT7%F-VQ0@OUCvBg
zYqMg`i$7--WgD;ZXty7hxzrocx7y#mV#b7D3GVi}65j0<65EpnSxOW$gFJ;6u-wYw
zZV%qGz)dR1HUFw#lvHDwcASV9|G^Zslnuuvt{mrz{%^8OF6Ba#?ybZb`n`>^+Z=ir
zWF{{ub5Y=2o6*j4<5E}Eg=g-8nJ0Wj9_jQ6F4JjFREhX7Nqiz_LrO=W&s?d+qFXnt
zk=MO^=CsvG&smm7(wSbINw1q`Ja6SCrN0^vGGnfBX}V~uc<<)8By_3GXvaaZrHYJ?
zass)G?&vCR@9t&a9$*$&SkQ7}Chx7H%UgXsC#xv2%r0zNysgcGy-@p!+TENPa|`m8
zD?U_w&Lztve?0Pt{R>O~)w>Qzecchj@L2P3O+o3z7~gf)Q;d6`m?^Z*6Ii0@yQQP~
zj7sb6#10ivN6#%u?MK^eMBKQ2Pi#5n)OGu=Ud$06LCrb~A@Rx;|5cMFG6<<G;H)pu
z;S_p(eQw7QE}fOFy2%HEkGwKeufF1`n_A@QI;TQ#!M_*ouY);ex2a9HILoQ2*cD_u
z@09(YkN#7vbWUonUS-_<Vx?$n5T}G}QoGp>$LVrf%;Fv!X6a5@C^Gq$gImnQHqD#`
zJiH<Y7N6NAcs<3yDx&$B^O6SXv{hWiEpwf?o+eFSt+{RH$;>s}dpkp~U0}<T>EtWV
z>$tkg>6+C}u9GEJuLTo+zM5n5MAY^w=Ng4s?c82Acbz$YxoA#k6uqu=(p}3uYLdq}
ztDUk(TlRIW>v@yH^R2;?#VznS$C5(fq>a-VUI{J>%V-k$`eJd<zZHvm&ZL;F`(OCh
zm7{7`b!d^{jF_4FY_2m;EfF_1x_$Gk&(6&)fBg5U?92)aemP<Fy`HK6cAuyT<}|6l
zv1qlJp_I7U0<*1K7c~h#aFRUb(4<+gKs0p*6DLaod-#L|rW=J)>su;U3r+3Vz2w#F
z+!sR9!qzFwXWsO7g}#~fWXCe|D%C!QI_Ko|q3Vj~JYTOen;m)ek(c$H2YmOJ9A;IX
zU)J_L@XiB=-xKsW*3EfS7-Ro;v0Uvs^%H7B5Bsf->$IDkOmHj}n_k!<lDJS!dGqbN
z9ubcOY<gK31#dLzn{9MAir`dSQz52#MZHzP)}PCLSGW+zUuCEFh5c;Gn|tTIn6SlV
zYW>uk^Z(CdkcekAJ2LhB-i}&Z-dD!2d;F7c9O~BEVv<sHkay=CC&`#VHqJf2#8Mv|
zU^&+Hfai82kMs^vgVP1(UWEZ0wbG_KFH>NZ%wMH9FUvRC{)a%uYqsDEVSXEJqgPi>
zW107e@l)ne<yQsIKeRtg;pH#>y(2&RV*5cQ=Vwz~%{zTEoFp4pxbB>DbV9)vl^efa
zh=)H?oMN$BG`fRRa>)dSnMYF8Eix?vsva$FH-DnJJiE)U$;G49XQ|QjpEJ*=T@whZ
zyJb9QABU!U=Y5~zV^8iXK0W#R_XNMbB<A(ax31oK(8XiCmWNl5S<*rwF?@^TItj5|
zE%HGwiCmMFFNb_H$^ZXn>&}F8W-~rrxiXXU(9x{gkcj-k8(}iOqMO)e-<~5ay2~?4
zvHSPt;?@1uSO2}$iC3TZ?M@%}d(LzJYSg#BziE_pC1>u&5MeDJD_LRrLkr{=-d@N*
zb3toS`{_HWE4C@@TD&!2OMds143V5`veI@^LF?D0sF+!m9LX1%=zjf#&uaf#rxSM;
z{Qr}|!>PG*NB;Uy#uaB)%{F_m>~A7}n8(9unvJhn%nTGgcRu2Cvj2BGDyciHz=274
zm$Rf<!)%wAfAf^h$~b*{cBE+^G;3)Mc6JMWoEZMw@`~NIZS|&GS6!Q7ny$`iv9mev
z&VTk(SI(s0dMFpg$5>IL-f(bIp0eS8*6U?bf`vl-fx3&Q3cgsTyCo<(*rZ|iBVEab
zLh+XbTsMhWBpO&GS_!*1@jF(`3>6Mq$hp-Z{@2y`@2>HGo(lXoZC<*?Yt@q`y&v%o
zmjynv#;!JCi+dPnu{qv(NBpFNd_oF3y^|6h%oUejWJ|fgreVmMJE53!V)6Cutm14X
zvTcb>6H0<EYyCPH+sx{;&5-fzHqBrIMotg$Xk(|vOcA9QBf^Cvo<DC}GbuTvLNH}<
zJJSJ~tAZVqL$&U;Io}BMlNB`I-c;V1EZFNI^g^-orJCOkr_{+7oz0h1dKcH<V+m1y
zAz-X%y(KAsSCHb8hOWB@D<_02Ef9C(59HQh{Liq#y_=t@+ApwaMPWDF*D7}*hw!i6
zcMXdS)SJy;WJWy{VBeA%{ZVp@0#}j)|2}i=X^PPiHv}w?XNTTkb2bkzQwUEj;O1SJ
z&F`kUx-Co7k?VjP+o5IVHD62SGC2h^h(6xl7R_L9C|0(mv2NK%$I=^hGY<K*FLm1G
z)UjNN;m)$W9gFL^BefT_nr{s1tY2*Iyd+P&LyFm4QDb8Lu1Wb^l0ssVQJbISZwqkH
zYb@X|?7mst#K$Pgeo=YrCILl>!f%I~3@uugD;AynA!)tBQ}2ds*db4*2@+cbH1-}4
zeKSE|^#vRK!(6i*xR)9Bh+UMdPBP`2SX{8Q^~}RC$+Z7ryGwd^v4w|6c<=ZUSUy?%
zZn~ysT2A*=8Me>TDbsQ`Ow3`>EX|1!(l0aBG|RC_OHNLgJ8?xSrCcjmSoX6@8_SD{
zmF-eq+hpf7bT}UI4Z4<Fb=-GqX8CnRzrQVhb;l<xFSIgS6!72Bajmg8_tk)D?z$R6
zLJN0t&J7owf83g>Krcb0HkyIq;j`E+pOhm7Q}zn<e!I%OIzjc^k0~?Dyw*OMy>^Pg
zmH_oboUyYExK<a;`Rkx+x|nnI1Id2}xRM&?Dhf{B8(w|>DSsHVo3FBF(o*k}(=|#}
z`yyr}@|sMGcaf97I?HsKPj;oIdYHI<Q_i=H{@zfvZ<4vPN3^mc|F`o`lu?@E)aIeR
zuCT12c>>?Z84s?_Tz+tY<&z2Xi(Gxo{q`=3=JeF&OxIXz9Ho=wD)zA<%q8#FbU`Ig
zHl?GDFNC}uSff<~7pE<9=5Q8k43p&l82=$v(1b(kwE^3g5BAoav2LgMk~Z-1I`VHh
zz~^$4|H}q0b>^1U7bIf?1!Fgg#@uTD8^FD21J{=WOXe{yORAd8u+3OM+<0nXL4s;u
z_{w6>&#ivWH7>$J2ScXC2+Rs??vI!(v+ZIAr(yrYR?8FLsv25l8dt5zz0uZvv^{+K
z^fO1=RUft=+NjmIB6;3bv+fnz&B87&)BO88W!Dr&E?H_mmm%kyinc$Sj%(Qelm=tx
zb*}_wWTbA7TKGwcVX3pw2_ezLWl~FDRIFeMJS3#IaHZfArnXWiuAB=t4BxF;%4Z8s
z>R!IFdx3}0^;gPbg&9j%XjfmFvUP#c#R&et1@>PRxI{1Vhwjo}v%u)OC;w|Jt~UYv
z7k2WUnJMy*p~d6p(k%-(nHF${Kjr^oz_sYZ+!aoeDM4HhX9%1*ZnA=H<MkUG#1o6<
zglZEz9CMw8o-Pm1njsSuAbi^@q`=7OS^J6~lDSzg=CfucHE2yJHJ)Jc#OZ5#*n=?V
zsS-jnrmnVJlFMJ2^222!gT$6atdlt(NUbr*7wL!=EX`Lcl+x{T{dX-h@uI4pp|W7O
zN7A+bvVK!+qoq~Sg2Y50a%-_Wm<RJeH8u8@nj3Oy$yWoe#|cwH9?f?4T>8?BKY0V6
zt}?@-2}Z1|s~<%f`)V4_S!kbKF!$G0?lm8#u-}%Q8ps_Kz;-}2>*8^1g>9TQQKqrR
zoRhm&bY*#Lk5KS5m7X~{r}%{C2KQ3+Z7XtTBrO(-&{~$eeUnP+bGcQE%-b_9ZL$SJ
zzI7aEiz-?k<!a{tH6`*+h<x}nf9umK+?zzCC4~-G+46PF5`VZ>(A8eN(0a@DS*N;0
z=C^khJab6A$dw#mW!_a$^0{+YmXzQ`k@*XII1Sfr*}xUhB@%keD=}*6nhEQ&7}tL~
z#rIcX*;6g9#UK8!yXeXN`oTJep1J=H@Lhb#|9XPR)`kOaUK?J{>S5;I@$R<0X1eVB
zW$IfuNIm`8x;#+o%}s&44JQ3b;hY-{UhQC&wDvV+lUB{{7k8}qJyUp@uxlHu{q?rG
z)KAj?JU3mq7XB|Y!d`51cKZ>dl={s@3wNHLFkz|r^JNQ{JLSo((&jl)@!zoV=e3E8
z*tVLq2(gB!gs)z!lDN+?%^+1VQ~!oQ>_)EF8@Mj46kW{FJ71A)$urRuD=y>5Jwk_<
z#yk|YIJ(_`mZxKcarQy3IWGKb3=Wzuo}=U3qTlT0G%Ma^#<J-)#^v99O1x@Zlf6rR
zZQx7f-YQUg^XjbWs~w9UYp)CZzbkj=+BOx2mgfaR22PtF?p*r2YGco&+g_!G94n8P
znI1OGsnhIe;P9<BtuLB<<afDL_42Yqr)8gRS+&Z`zwxs`>t}%lt1K?=vDmR{&z2x7
zrim;HKLxI8XRX{JKhxb>Dp7Oof?2L6jvGDLE=LL7sycQhOK_*B5Nn}8(1-PFzSu81
zz{l*ZTFtcKeCUR?Px!kW`CnKqduGM;`oe{`-m0v3Pb67!X&yeH(|F<EV*812)K7U!
z9yMjXWgaxaXrq?lG_EJ1|EF1>TCB99$MMv}usc&vO}!*_My%g&lCY8G>8ID{*SqGN
zZj({3=+C|x5s(-WasAAqZxNF#9gRw~yCly^aQ&b7QGBz5%a+}*wK{&vTPGSwoz!)l
zwrKnHN|}fG(Gvw^B$Sr~2$(I@_EQNFS8tLimkJPM%{1I*ykni;Nl#`UN$*p92Q}F|
zJU0})-j)=#)b|!Y`<xSB7`z_clAOzO^1t!ag<Fbd%Vz1QPt|=cnXJvYR$#u)U)Mb)
zw=Rm;ZhRq>dw@&iV`btYwnV4f{e^l{gEY^pN6wZODplkRVz^TrsHT|h!6$yba0lz;
z3-f0xix(@lU-7Cddnr&eqhSrV%<3SSxMacGF1r)MXWZjy2o&=R+7@=fr{RoG>T&Iv
zKM%^8ZHW||HM8`1=Ze;|zZ<Jxo?gv2vEo2f<<#p|J=?8T1pe2($RzYE$6D6$+Mdf+
z`;4tEZpr@q6Q%eoO31)OZ-x5m152tei5eG+3T<BJwQ`E_;R9b67<sI;|5WM~oO<%)
ziPXa<c1+ia?~m5gtJToGc<WTKi)Ecq#U!D46AgR2%x??kdNkhNxbX7ifWzWXr1tFH
zR8nZ_#jr`?b~oR?s`HHMJN9h)_anST{YZV4v@=iL!)?;%(<dyQD!9IBbNJcRW6}+V
z+);OyZ01#pUhD86;l!3~H*KK`sW&yQOEWd1FS?5{UEg?T>+9DKu33ruPC6!WU2Xca
z>-9WELYvh!jg3!4u2ax`q#!QoT48Lra@o-rFZKAcEIXPRme~h6=xtg0YP0SC+goIp
zYAyEEW%e=@HdZiiPIt7QYNWpIqTrlK(UQV<nyprBwmHq0u2#sa{$>;Fq<~Um#yh)p
z&IGfUUY2c7c-%fGN&0xrnKc`I=l;5~BFW;xwpFjpR$s1}@a&1e!@#SnPQ6<cC4bGQ
z!T5^B=ef(X&Z)S03$1b9`@1hxQAp`>(VFNZg70%Q&%LV9JGke<xzk%S1bNo8EKWM@
zD_5PICdO3j@F%!>V&fwT|7{PyJ=zhzE+`;a$8nNMW!#L!%)g62KFpN;&#b@VhThtj
z0v>)o%R-etG)K7DvbM85<uZJ;XHLlX895(1B3i=KJojGl6m4(&ab%X!jN;@c_sTYZ
zJGW@6#s4U$EvNQ)FN|8f&&TMwuXbMOlRc~%f-|RIlWJ7|cH!FdBK6?B#VZ5_i?*^V
zS~vwX`uJX)6zji&gVkWolI=@bisrp5+9mXA#i3=7zDYh;Wn0AHHGSbtvk2p5B2y;L
z_nIM_^{+rovB>7lCc#+((vu9LoR!2+>2k6$-&qzfZYsFTrqRQ1-kX0j4ND%rDSkLD
zJ7T3z?%jlBLCZ3!?8B^gYGuyI$nNM~SbJIi^<AwPhNz&)DO<H>t~zyA>d29*=>mU`
z)c;7i&ogtMaJh;{x!_XC%Ec1t+@X*A&Hqhaer}nm&X(hqvnNbUK487ZK#+H1<D+QB
z?z;0DoGNQl1!5Qkf>i#i+%R%&WMJeJ&`Y@y;4~%duhg~dh>eSmbc<<n9q1@ne6&YV
zx-7-x(jw1(De-Tg&AoUI_sa)z{ovT7<`MrUVxrR4mFkjhv$k>E*tAk3ASj4MBlOmU
zl|q_cDk_2#Tn-&t9lBX*$|hGX$;B>5i$o2!uDqeiw9F%P(Y6FvJ>JJnOQNSg3}~rf
zJ@Rz7c%meu10(ac=o=prkMLNkZreRW@zMb%4^D<X6?&=1#Mb6zrOx1ee4fKupGE6d
z@!deCpyg`13w)i8)<y;991r+*bxnx&t97@eQ-oN<w);%okjh=~xU|yPM<{XA{;mtg
z-?BJr)lNh$4$Gb5@%KY%pYq}9GW!4L)f|7P&H7{VjFs1_emszjTCRO}QAz#Ql1<8{
zS!b3RHeJeI=qonqh?1s|kAV3$mW<a2Tc_>L;$GNu$*Fs(k&8#uGPMI)ijVIJB{a8s
zs7zca>>hH_MfCBb1N!@RJaAesp>LgID8u?-Nq1|-o!3TkyC1L^$en(%q(_nUL!rA~
z%@LlH2L+1WtZ$nm^5BHuI-SK+T6at-TBsxVsOZcYBkqQU%|cEZzAZu?3Z0kUyy8&|
z)0wr|r{&$2jwy5Bc&$AUcq1X#FX+aDoQ2=TCh2mvzPi=5(A{39gQt;0N~=?nH+aRx
z_E$?@ssy-7b<V4bs9Zlk^w@=#sk1+ne7)NFC()Jre_6uTP<6XnrVS2^Om~=^Jtuj2
zF~o@{om$PaFsf_Swi8)qPWM0EO6pm+>T6ba`cjY6j-0n%WpN4ayz$LI`s0$qh5IyS
zcW-FDvGwcSBWql?`z#Pkz8<?@AUou&nd#()Dc5h!@M$>jrmb`Cj6mJgt;b`Ud#8lG
ziCkgEnCG3KWjcNHi<@_s1s$+(oBH6;iiaiLQ`S9m`*>7u@`cBJa$#YKt@Gl%uI9}R
zb-CFm_Wy+<f7}A$*UQCx!xnn)GfMx&n)c6)@wNSiFP9eGNHXQTpvkC`xM~s0&MYs@
zwVjt-x;L6CHc!oVeo~;xXZs|qf%CxJLvM}ryS0xBZ?CwhzqidR-0i>Y^I4Y`S-gMy
z@k1Ni{!1r!ta;;dLi_KmPapO5i_e_UV^#Qkde<JFTkG7UZ9N*h&u<bAvlU%C<3Zc2
zH*fr$elOPE+NfULdtq)+{rYKPSrZ?fUBK@2Eot?Ql$=$owHK|J7xZ=U(s^}mAE(H^
zWl2aEUF<$l&g$)-mFmyyG=!FlFL|vMZ#(gv|4XI!M+H<vKA*~twC%cZxbJ*m-K`W6
z-%a1N4j5)QhB65xG@3Dr@fxON^Cd0VBymG@kwXB(-_7m%CW2G5BUeXdf4tE0<e0$j
zr`|13nWAF;xUmVjb$ZQDPIO(i#G+-w1wF3rh0hE&I|&x~#d}J4z52i+(kj`b=VWr`
z|A7!@p#^)M849d+*VUM|JnR9RqD=F(eN%#`K3KzPaln&De7O*#@0TXqrwcvLg>B^f
zel>?@v4Xs=%fYUN7n{ZGqC@Q?7IH>?VGd1hh+F@{QNCq`zwPJKC;qTh&U5(XEOK&o
zXYCu|6CX;Jh<!QWvGT^niSBwVR=*Nk>;EKA*(BHzA*^z(fot&s9z$10r_3&~h{yc(
z46No%3!k!VQDu!zbInuo6714xG8a{BanY}u*j=Qdyq|HMf&P_=v9qqMQ`bBh+9rKw
zV$2Lnqe?v=-jiwE(vuh->$)hibH*0NJzaVA;>(hXS3Yx{4jZ%`5Z0Oek|!XlP^0Ze
zljx?6N8juG-z3^F&9m#*qw66)I|aHP2v*%%yhOw8qkC1xdHF99M>Vo8o(%laESntB
z`rYFuf0W4DuzM<+9ybnm`J7;{s*UN82|u`bhN_pf-R0zqQLfX~p6S;!Y?`G~*DB5|
z)ZL~K#NDziLuY-)!YJ`1*R>Hr{E?=uTEV3OO?!-3W3&>vf(61IkBA=5J-MMxcZ)RF
z|4r*oJlL=#`{?CVF80+0YASQ`yUu32IlEk2`8l-0wMS5r&A;Ww=fur7WsgQ(?Bk6H
zH11C25MApu-EAh1N>FB>TGqsqbzhQ1lYT6AaS7~xUt5;3D5CiX3&UpDXH!C@Vh_t}
ze!5g%>CSM)^Za=+r|AEO&$OuMB({tF>#*xD*|bPzZL7-~wT5M3TY45PI9m0~u<fI4
zuA;W{30Lc#$NmR+>}LP8Qe{*0!qxvSmvDugy5Tm_{gqcr#-sq9B{vQ!@=2yMKiG7_
z@uiZp^h@2gYVB~2nV!?PuPR-WI-}d@;FHC<|1^qhyG~C`@R*@;v%qkk^9eIsb5$<=
zBOUXz-@oA4<abzOqoUoE$**1<65b!y!c#B0#+LD!X0(ErS#X9vcXrQ=l^Kk4XR*9o
z_3MXQeaM$Rx1?R|6Q<27RWPqH`8cm+!LKVn6wlV(_mq-swBDj2Qm#}|C9kS|>taWj
zQ@>Pc#)Q&iDbssQ>b8kGlo_15Jpb4K5*^l8_nj^&v93}&qP6SbW|2h>3zx;+tWLfl
zk*{)OYSW(77!J<YdyYtL5%`v>=g_0J@5U>iZX2r%zowo;I>F0WJQ~+?OMc~l8?<51
zGAqLrh8qd23Ek>pYecImMMNcw9v_vOWjLkRK_uei3B&epJ;E)?`rnL?KiafBEjf+T
z?aK}!hjZUfeQ;h<rOdfX^Dw*O6<3auJ$vQie<<JCx1znz(0WUpZKTrCx2@h8-}YLy
zUCF58$(;KpIXAn0_wCKPqT1_T&)+oBm)q{GeEHG^Ti2E9a&ZL-+&q@B>Q&Ej`F#mz
z9ZoCxF)s1cD(aT*v{^DeZrTYwXUR0@F3Tw?d5&)X+4fy@O3;|0QYbZxgY(iE$0TJ@
ztz%DTp1$-f=Y%SE`nuK!Z^Dmt`t9PlDXwDOCVIL1#kpNR%N&B1t7}erF}dwXgxS^`
zPQA>pG$Q7%d68CVHF@@lzN&!ri~Upkx8I!i)_Xs<jbY?X*SGu5YcOoPop`L{OjzLN
z(3B*R_tyF@?DlD89ggpNZg$*V%3cuK+xQ^*!%DWPRX;T}zSfJa?RQUz?ACBmI4}K6
z;AQU2jJB4o7tQB~b4JTY>=Bru=w}pjCpTXFmbtLO-`r)c1$!miINlhSmEH*OoxEI8
zO3lwf(=;n;yUU?%H#fX}Va2Q;C%Dnv#k}z9=CCh+O1ov2T+f;Czpwnt>Ng8L*Wcus
zsQdeRP3Ehv-M1fJd*Y<kF@fvWVX?0_>ZPL|{x?ftW%y&yG=-Pz;kKz6=I5IF{Dhdx
zUa*>E8Be>k_-s_G=c$>`v{a5b98hbLoY}aif!o4UV8^D18exsrUlruvADpl%fTJgp
z!^~4S=FyHe55eDDmWfq+cc1Lu^H6_hhV8+VJW4wik6cvDJhWH2X3o)oLq`;Qf(oYd
z7&|T8tk`0(NTWskqG!kY9kx%hI25bqUS)JX`Jl6vVa?Ty8C6kpH#>Gn+@5>yrB%#h
zr~9k9G#S>+ZrC)#LF32-qX$XGwk}I{S?u1O-CC>V;xuP+*oIaYAD8eAt!_C-T>i_n
z>MmIMb47bqvzGUljZr6BLL=;-JBdEET0eW!+P)ntfBkfRy;1Mw&H1sCtn(+xb^I1t
z__1BLgW=f*?le!6U#q%HB0Kh0?k%^R9(SZ?M~6_dij$>Dk0$5Bjb3xEH!J#h_n6;g
z>d52}T+Oj3%TDE!(=|_}n+;P{ZW)GIuaVf?8^+Mpd_ZFHA=fF4)^BbbvtQg1C(RtO
zY17n-ZFWn{a&I5^>FUc{?OynKTcPmwKezY)eJp#E!=Y8hfkA}9T!6v4fYo|KYv&q|
zt|=#qL>R&^oamV2;k9#3b+wjpRQu~)OWyz7?RaVNnWU9lFKw?{A$g(eq*~R<+BMzZ
zyj3DQnZ*BJk=Qk%L376XpNnRPGcKF-`)GX6!l|bfBYv(w(6vF<(#btoZO-ILb8cH2
zK3sNe#v$z*adX36T?)%DY4&WA_7+a=k-cqD`bx!Kkau~Y^Y%09H($sGbsV0jv(mRy
zfOEy>tyW8Z1<S=}u;1Ic!ug4CN25e=p{%RV8CRb(y)GTSIX>169w&Rw^ocZBSDcAf
zXw^;FrT%%|dqIX9k2w0Xrm8*mUg*dZe8t_PNo86Bt4;Co*WOKQcTIB*=HB4NlU6fz
zQ`Gd+FJ?Dy+?mp2U75(E?Zu#aqv_m@4JuxydxB<cn%r|nK%zQ{HAGw3R`PiG<b9ov
z$1m;P`rxJChUyMo30M37DI9r|*Z%N6pTD`k>CFBEf9x4W9NJPmI;Na(TjKHOjYn6?
z;noJ9wv01JW}Gm)acHV1|Ch_kqH}boNvQlhxt48hcik4FJBK=JTvn7WZufLDo58tP
zL&5g_k<$}qHt5b+FRe4VIMXdziA&xl@W4YBU+u}aoZY$`HYmny5dAb~?Hk)&2Le|w
zaY|paLDQz^qG!{`UAD`-=Vr%nG`<vmVWZT-Fhx1*w8@&~vXAF>t!zEH;PmZaomPb)
z-4CpHtL(3BIo_$T{eaDxFpV=NvB5`T&K#|AaW6UYug9aM<BU}-hxO6mlPMiGPlIh;
zJEAjK^)8&ACb{^lA<xVe0WYJ@mPg9X`p+S+Y;5#Aku@rTJ<33|>_Y!#uK5g=XLsdv
zJZ2H+>~dSGqAA=Y7;=>7Aji4elNB>+0yC#HsknATmP*)Up1UTn=)A&_S0z&~N*s6T
z+1gdfE3UJ5+k=xC8@FcsT>h@cF(Z)a;^n5<QT_$d=L>}y;x#<}t!8a$InS8mVZOot
z@0JsNQ#>YA`1Ja?^=x6)n{X{l`SO>*D;%Xu_O<kL99{BZh2)GW%9fi#>epVebke8|
z^-N3Vk56FjcIMG!ICZN<R7Qj$=~MS)>F%1wP}x%xv$dR3=j>Inbd$~HQ2(=2J~wE=
z%bg0j*BCx*co4AbeC0Wz)g0xg<hQtRERW=M%-s0s|7}Cpl*2R6_PpViT7AN3f`)6$
zuhWiq)DOp8zE>E0v?ti@jlaEXhxH7flXIe0uDxY@_SU3bXN)E^^{6a1NaeX)W8bwv
zW!A|MuiBMc_J;gj!M>Sy{-vJVY1zA{RA}9g4D(pooxgKhSH#|+ugiqju0P<Utu@h7
zWywy-)T{C-;&K~rJlwPO8<U-0=GFsQyq6EGwlcKaC1B_!Y-px2rK>P<0>hdsE03*r
z?I=^Y_FCJj?&F2m%JU90?9Ypw+`^#lRw%qWp*3eK>)*}o-~U|u`DMSA==JV39zAa^
zcQFKnSDffdxL4zJZ&$;;%TN7}Eb!^lVb#rOXS#bhQ@bZexpV5QB|QJ%UZ0V<Y8h8V
zKZ7nyu9@hWHLR?iv)8Qc)ZTtIRapDd)=d#Nc4po<%%HYohWNiv2X`~L?P_p4QgQLl
z?{4)Nr<=hWWFiyfuUeko*v-AULuPHa#F-tN8ia)&AG)~WYMaN#E)B6+xrQ$}d4fEq
zwCz}NjWsHRbAJzaL!p(`BJYls7PC!#qa>1}JbHqUFA4s=Cs^@X@an6VPxNfGS$d{*
z#x13F9V*j&RIWuS=W(dkC0S4Ph~I3cB`i8MV%~?BtUOCYyhR^-v)+hWsNyr__O@ty
z-FusK+qU_>2(YoV<p|ocjZ12ZaQA_^TaK*tI@&7!XVs+rTO-!KxEZW06BKNE<ixSY
z|C(Emu0FKcP@=)}NY@XyqpP;w%$=Y)>n?w(u>Zf5fUq?W0vJxc5(~c9RG=GPq<im)
zZ}@Q@spo$mAKq|YH$XCB^+{LXgM3#4jP2s<)~=gg65o`{-(<Rk{p{197E;qrcz!VE
zQSFe<h}+UG>(zOed#bb75}9Lba@>wKy7?(I=Pk2zO5^ZO?qFG*FmZ3fVT<SMZ6=#V
z&Yq_0E%Z8}+hO?()tePXK^qD-K20(Gmvh*x_^grS+$m=*7(5sxr#+O=3%j#p#-@T7
zEv}P21LLgD-dfpv=Jb&lYg*$DXC&>O5qH~zXTmYz9={`*l2N@4AwdT(7%poqO^7zE
z>;Kg_Z+rCPOaEKKuiB(dd6wjx#VVE(=6+FOPsEn>x48fPJ(!l#W9#;cElqs$rkxA5
z(j(7#RR+!}T-TAwV)*1%Xj{Rq-AtX0f`%7;pR7H%dcT&lN*1f9n){WxGZ<?pPu_Vq
zVBhKWUzAii`tm%FZ!35qR`-z2NWFavw`$*0dsCj5tenqllN9AL4~0!JQ(!jl*f`Pc
z&7nRIqZ>@3byuFQoM~zF_}sP-%Si`Pi{rm&=Nk3KXIlH}?elC&c<T|QAFm-6-91Ie
zD(uIl`#U7Frmf|epsdwWF;PTQO)nxaC(PjJj#pc}-pO6HJQd+|;-up70-;;84ax&$
zoVQ(Ez<jAI;W@`YHHkeELjS|=RK7bTkk-L#?R;$p&$@p-XM7W_XPPn`u#lKCrF-T3
z#tDl$+SXY9QD=R?;FZ>)r1wZG?$MgfK8!n9nb|WZT+3jwe{^VH@RP3@EbbiU67S}#
z^l+X(w_@#^|65qkZgJScdSm*jD>tido4h--^Xrk7X(2Y2_SFikDR1`$-(tv4G`YRz
z>|(ZsA+@6SeO}c>?mKb9a^<;*Fw+FbIJHN2oK6ZT36zPRP32HzDpJ~&_c_jwH@P`M
zc4OENS?AN)+YV%NREn*7aj9&H*2$)6ZzoUD?06$OUrp$P;`37G0|l+RijA5YjVJwh
zdSouFt~{9#cR%Tv`$vxGIcZnif<l7UglGLv{kYIJ;*Z!0$vOJBS!RjcbW94eOo_7e
zES@S;X`tG<v#j&ok+mJib%o#U^O|2QT7K+R&tpZc>8YDUM2u`qHMd%s%1E5d+iW&_
zt6BLJmKzObA2Ol^%fd4rnOS^0bfUn1YjfC+W-|pw&Ib)fCoY@KU^;YR{pmfeMmHFq
zEbBfUawKY7rc>8^m!~h2)xOMidp2iZ-+Au668*8+H(h168&BA<KJR(1Yz@zzidS77
z7Zu-^c2CILy@E$_ZR|6St%r6@J|D~Bzw)Yv`;*7jqRK`oI@6z+F`VhjXxG)hcIZds
z;fn8fZ%kkhWQaUl%;kUYSR9A>jLAxQ%z7EjR{M)z)tdJHuk$e1Xl;LT>)|sdp#omL
zj@d>TXHGw9*iz8SE78@x=9_qC5`!+!)2klQGp2>j49Gcs{AXeAD*f|w+QcUZ9IQIf
zE%AKct!rI>d!@H-aJ;`PV&M@Jp_vmIzI`mz+^{3zl%&+xrsdqqj*~vI`Dr%J$$$4-
zHAhV@wd3IRKpyLx1zzrQOcmEF?QSca>6%e-L!nprUyRV>xw;+s4aY5tW;Gb?XkE5H
zW7XVBHuD0m`AmE}j#+CkU)#@Y^q}liO0Awo^X-QXx*CjoXEwCwG;=C2UVH!dKl4A%
za@O4E4b#8>QPpGSyTS0}X@=^$j&8kg#eXi9@Xy+q`%x^ZGjM}zgeOz<|3!tjayqu0
zC|puzB6h39F79gR*9fIQ-_KrG44<l|eBAro0>9lJmlhrhop9mmq)k>02}j)+92u9K
z*rK&!L+EN>PA)Hxpa4!sPA?S)6%WBhF0FF5UOobW4#&IXbhRb~9lLgXnjYt-6wL#D
z+I-P>c4Qe}o+fRSo3nN1p+*Hsz8H_3hZjzrox;2H4#y0KV|>#&nNCf(X~4SN*WBps
zjZKqFyDvA{oK&k>`As=uTGXqRU#|!0Iqs7CeYdvr`ntPYvp@b)<@k{r&-m}x)~-*2
z$qB7qY92EUTsJjn_DbgqOsL%4u+VvDiP(YEZiY)Se!?uZk-E#L&pB1_N@(|jtcxtk
zkN@ixmzF;EWA*0zz<b9$zll>s!d*{+v6wT+W2vs6K~>50tI=`cvo~DT2wK8b_ua$I
zVRkc9RHc2sz%GWFYdEfDnZ{ZDy~;1zrBbZ@;a~snyHhT@Nt)br3=8t~4L!ydedA)A
zFK>~7t90&!#)-|s6;In(Qg#HjvB>;*GFiLK#5BRNe8(fU$Y{}$ZtX>mS7*qwo_%q#
z!_qbIw0rYU*6>=U9~u!8n*LrD>euBx8EyY>%i>lSRn7wuQ?d=e%@B!>iHypemKz!A
z7&FOD$}y?RB`x>sOwZ_<EH95e^Ui(uEl^zd;LpW#_2yPaGI?fvoaj(@F8qSSV_ug4
z>3S`bfQ798w|<h#@8a5!;kel6m5^kIP)UHc)wN50j&Du2sC!<zyZug->cXf1&!u@w
z8kkaAo?et?-FdOAfnm}CS4n-Y39AF$9=aS7QJHdyEy|~Gf>NZV?=e=%m<ErlENd6+
zP-qa&33PKxsR)&1&uwxQZm?R|aaq>pV#FLDuA7&-*xp|(eKIlY>}`Mj@~h6=*1tA6
zou9O|XOma$XRd<xeU~KP^y|-ZF<2UPN>k+0ihf_QnW?^lCXtRaQ*~l<7t~I>GQ;`i
zvI#R*6i!mFEjVQP+<oiDm<f4}S|;}$*S4l#*vz_>O~%n~y=zPYzo4$r%<_FpCdc12
ze08y!)AH6`_bEZT2Q(W0>z!h&<|@>RU+J-(MK#}rdqD$B^cQvQEzv2dmp9A!Y{-@l
z+v;?cE#^inTUW}FyIgn5e)J|jxb>|ixHY$DM`Mt&QmV)!Jtm3AKI<0D__l^=M^fc3
zma8}7b{TIndEHaKC8=91N5%5Pgufe{xNW)CDsgGA-XQcr>)#b&>w^i(dzaYuHLX1&
z5}qchHa$+RzQ%ijVgBq><;zyysxY}NmN9ov^7&%#wY5k7-tg>6eC-|Sy<GUKp61<4
zKflJ#+INmqHPT8me)pD%o=h<*b5?!#R$=u#;prKw?8)S|Bt?)@qalTBXJCuRfi=8W
z)`|!}aPogtwr+vSYL?)L&WJUOlsI<%XN)QhzR=i~)NZ7sr=I$uYe&$LNz$cxy84eq
z-{);<_5Jb4fb}3ZzikkAJ?8?kHx~qJ+!$?k_Be{HZaAG7rpQ+I<#60$f#&_oepoRI
ztZ8S@_DSq`GX3z6W?5m*1@SN31-c#Ej82`7b8MR=d0|GIPjtZ9%vs;}y!*JUvhL*B
z>>y!{t6u`mn?gFiM<_R3tH`&w^|`k|go)vgO_0j$gvahPo(VnBWwD;QDQYQ)5tnDI
zZt#7_{_+x$$-Z0N?N|h*Z%jJC8g%1PhV$o+o7XIMS{Eo9FKL#3cE^BGXv-p=Lg53t
z0Ug^nCM-2NxXGt0%ws3R&lHC#vlE}P1X!Lrn^OG0u~YGVMpCzL4x`@JC|2DgnOj3I
z9G3UH!lKLL+#lGX)YYDu{w5-@zcPy{?n^|gT7wEhnN~`<waf}}+eubBam(7jt!Vup
z5h?#r@VwD6>0r6#3ht&$+w9~oPujd>`7EP3nrZrzB|SAWgIagInaJYk;K(q6K~v#~
z@-xK~4vQY!F$#1A_FX#ZaL;(c7hNr@W~KA0icbZ-DlY}qZ<2Jfa&>8H(6Ukf8Mu-|
zV`9*aLvoJ;`~C)f6$&rpQ43h;$*C65wAq5SAok<z6Ef2$FP3q2F>#F9QRcK@{e!8x
zn!h-BuRj%%T6b*y9TslMEdkvXiX!Dn7mf%mKH(qrrCDs2!2jkqODx)|-3n!y7O-o(
zFh<s2Xx_9YqWwU`#w8cB%odg@%{cg}$u<5;G7sP8`&si^?4Nt6^-kWXdgD-)$)+A1
zQ<diAEQiTPk5{dkkQbGA(&my;&oRwiua!@!aU0#8x0Tc4)Y2@Wl*&t!lY~~ft8(2o
z^VAI5kSOBQecrcHU5HiZgqY>({$dWJ)&ElTv|}6sLtdQPW;UVES9{B|-T;wZXCf~b
zbIqLk!iIfU&brq{NB+!lx4Lf089l>o(rGnw@i$yQ^B5;vziHQ)q+w9M#<_EWXzQC4
zcJYR$GLgkEj@!9j?AGXN3i@@cd+(Hr3-85_%l}ho70Jm~f8vl<A+2%G{lCFC)uxQ6
z-kVhN+$yIp>#57JY2G6@VdM7L{+gW%9s&nuOrFr->9jVmO(?`$>}uTCSheMK6Q?*G
z;Z9$Wb!xtxV}Cr~4I!hdCrVy79Q;mA6FgSJa-mJ{_7;^M%fiBiEXrKZdXGs>^KQ>}
znK4iCnbq0rdgYQjYb|2qzfH2fd2)u_mVl)G;|sEGu65>l-SKFZrl_^ynn%Yru0A?R
z=ytzphm(8rNhPO)Y5}@Z$K<yL^c+_CCTP6viPFTG+oXI?NuJwr(QH1aZ>F2F)Ri}Z
z)yjK&ckpMvdv`FiYIorGlEZ5S7HGO;uuS~!9<pnBS#y=H(z+*=mz>l6@|pq`ns_*!
za{kZtwQ5~#Y<8=2<?}gZcRiVe8o1h)pG;t}m)h{>Z232<nE8@-7l}qM2$@jvg1=(c
zwQC>VxW6+A>_7PAvcc*J8~0@ye!d;)dt`a-D#z|Gi<|kP6?c4oH@}-XP)D<W`}Ca|
zQ)hQCYg*#zqma7J$t6l)y5;sIGXDhySaxjnj(TOYW<gv@gkgNso^%^?Rr#NGpYpA@
zKgn8S5Wj5JExvi~r6D;>I2flZv3A5R3+0;OplNhxp43Hw4XlASK3Pw$Pg|q?X~O+o
zk!taY(MLXQ`hMv2UcSR!b+@_|CY$y!%(FOuX>!f8Jz_=r8Q1&ew@p(pfA=Ug>Aq0>
zi&WEZuiEw9KAL9y_ioviD6K4|+h(Kx@!a+cth~ovo|P0#=SeNku~M9_uD&+<ud4at
zb&d&TU;7!<{pU{TjS>)G{Qvg1a{jVCYK`La8m`~WO4)I%@^(R-S&ov`>n?xoDGWc}
z^s-zuJyxqAFlPsc`O>p>XHM-p<I%R!rR}YI?-37Xp(AY@TiXiHb!JSKjZkQ;JtFI)
z&}(q7RWM?=h}d~fwhYyaiC+#|U3J>GK{)MZjAiS@ejU&CFDH0&i2P=~68q!u`!zzf
zf95->M7m_QO`G7EmcuyRATc%LsH^GuZ7&6H%s6>&Ps?|~BXbgBbI+Vq__&~?N77N@
z;Jh5isED8?3l{qAx%o>l^mZl7|GBD1_CFK}yx4wA^N8aHC+#cDTt+STCJHW?>6;L7
z<iA9uH%}ttR9@E^zPnGdgk163zQWa~Lds*~L-DlpJU5**GaO49j=g5>NxZ>reZc$V
zMg@&qto;w?#jwauREXw(r{J_U>7Yoo<Ab=^JM$Qx1n^0==~tYZ!V++pRW2{^cAl)f
zYV2*ns<v5ICd#rNn{L~0yoyz_plMgC;<vL4#oeZR@8H=R%jV5^;O2sa>2nhcbK3W-
zie6<o^{4gtwF>q_bA<#ATSPie2`KSy+3B)!h9Fb#@y9o(H*mBD6dc^8lF)j!m%-rZ
z-4`ymezYE+Ipw%vig!=nS`WuEos*}3a!Aa)^MAv|I7gO~j*PO7tjDx>xt!d~67wd}
z)*$9^1Md-q3oJ5o|H~+TI=RG2Lsq<?(|1;zyU+bgj?)=R4!W;oyB2!aug>LLWYSMb
zcaaH>)*+9ityCpz+$%h|sygqMoOw8V>D_r}7c%@gdHW}8$UPCKEr;&vy2wX`I$n4@
zcM8jq2aBT=Ca_#_Tcb12H^x<x!?o$Em$Z&+(^keU4yU;^Pp}j)?D@!Nf6L>k>!K2l
zX(#5+O}Ha5am6EtliVJ=THczrr9MiEy|U;~#xmVoi(*c++Wu&=z2Rg#C2jG7<9~V?
zEd<ZI2(&PjT>Z!LFo!RG%7Va<j59u`mR$PDdbx0RZUN60j_dzVUJwjS>%UrfGc`f>
z=h5h^q4AQc4)bM}MlF%Q!F%K&r^A8H@T(6*f(qLgD#+AKS!z=^-S(UiqnbmEo<Il>
zYlBUP<GNm7jkAF-*do`o)EPct@$zGrd*<^Zoh>MVt;mmKRqI33W9^v-Q+=QJcPyF5
z$eDAMN&5<u<`q84j6)I`jba{+ZCQ>YB9SgN8U|kOQGD}6c=%HsB(G$ioY8Rds1M_e
zlrt`t{{C4aLetLty257q?%0_%u0?t)B&v=`WHrW#wR8lk{t{scP}Q8hmhJ0_hy6Xr
zOJlm^J}o>k_u}CTY5z1>7|yD>6^MHHOueVu_CJ@^zU=}F$1|=@pL<ul{)c~XkxX!H
zS;NA%QU0-F@7$zCx^ongz6xw=TsHBH(q54xZujoZm(p_Vd6uy+VYx}ywKdOp6FWsb
z8jAz87-L(OKH~qUaD18CqU^es%_pBl1+;8g8W=Jo%WzX*L}GMC!<p?1kBLYL$xVz-
zNVr?`W#Y8hg|l90?8sES@#dsN$nyCg5`HnBFDrOj9&%QtR_M>k_+>{FTX#J!De=|K
zX{?NTblu}XwPV1op1w^D9o-)$`AvCQ=fN7<aAM{wc237uC#NN5Yuk=n&SFdAf7viY
zLNr9jOvAAz#Zj}xy+~!kmqP*@o!(A3q}J1@meVMf)7V~>AtZA1a8ziA(xOF`E_Okk
z{}t1Y{B`l{<m10ImCdIr{L_lo%!nzMO{W*`Ix<y|H6UoNJI7MfZEyP9dRcrfrA}$z
z=bCx0CwBi%Sv5(~aI@aCe{v>IV!A8nDp%#D@7I1pm9bOICDc}TrOVoXdt4YdcDT2+
zYIB@kH|J&W1lER)E_Ze`g$caW*W`9md$xSz#6%_Dh%Jp`8<-ZdwZ&dZN}F_V;Xh5i
zTdbMy6jy9u4ml^kB&(^|;}xT?7Ne}57t8(WVzCJv@gG8S5-LI$JPdP;XblZvdlq+Q
zvS<6J!1*V6rZZLE-rjh#eCh0Kjc1Fr!qp2LXY33S+2FW=>81tSvCAA@Aw@!t8rteN
zCCgjfE5d^Q>qNc|RART|o4lr{BW(g>y)VNf7go**jRuU9=eeEO?66+f<Z{afdFM~+
z>pf<4-<a-nq+@Yv--$r3jfwrd?;akIn%JZIqRq{*-OKT?Q%*<S3(p6~uXyRcU-ERh
zk(5u=Q7OT>Gc@=a?H?X}^2F^Xr^vY%yDSaNPQ961GWW34@#=)GOJ`zB8FgjfiO8;E
z_cc8`>+H=ao0m~8D=Qq1%Q;=Wly`Me?NmiM_i16AQL|W-^x71?4*0(ajkKH)k<(yu
zHsfx{t=@ujOIh3Iu45D0?h=2~)2G2zS7k-Uy|)YMo{4Dl#HwiBH+pqIj>Tf*vbLbC
zM_f@ZKO&b|wK4qQ_epyBKU*zobL*<Z7t&<rUs*Ei)zf)ezOU3S?Fsbxw=C;SAk*cH
z9g0HIA2q)2$tl!bzUCJz!<_qU?gsf1uggWc=1z1x8@GJ(BE^G=Ld%~lWMH}}pmm_O
zWfK$20gdTJbsMyw-C>;Lb(!;GOZS2k$Buq7+50+m!h(h{1-<H7&QWsL^jf4vxJ{-q
zMtV5jWplG-$!YP+cq^mRrkB%h<@wj@jS2q*r;w9J4t@0AqIz?}+0AR8beQE$v1$8I
zqvKl3a$JR<aW;?dvy?2KMke#N{JbewHm+el{zm!t6Yr9_-lfyu?2?@P%k|tDkq>X8
z7h8Q|yL|J)vB-;gs(y-Z9~-hr&XCj&T=Ty+XXQ*ocenHshLkNU?;LlFVBr)v5?3nq
zbdNwpNz}DZo8~@vDYV;UZO<0(+f6NVPd|E^s+n-~wu$RehhvO_0q)bDEpnTh)ihZx
zBjceP<5I6^$=lPM1CQM2yf2x3WV+RrCtWYRO>&C@a&tembzHkHd%N&io}OH}Pvr4p
z>7{Q9*1mXgi%ng^_2sEmUn9c%FMaKoDY|xbw#XG7yW5^s1?lRnQymkG%4&qA3$--W
z9W;x#Ec_e4&1##+k41$Co;Cy}?76gMch=?9;BEC<ERPEP9^DjQpOByKFX7mbpXq<&
zsm{^h11mSHxCVP1VBxu!vw!K$8`~H)Q_CfE4jo$mC+PG4{Pq>z5*Ow_O^(hh{Wopm
zzkAB@lU=sWD($*<+;6g5M4-r;e;zXmLdv|)n-+baW?1gML1gCM5TAvyzg`?E-4f1L
zyi`hyWm$tkwK`X&MVHg=#6pLqQ#QF<Y;)!g-%=#w?vYW_kZqD|zIN#$H7|~+Wm1w%
znORI1*)!yqs;uK)@8Bx^z{KzO(y$CyDc@JFJ5IIl=JMLn<yN5?rL-<+@<)mORZX)C
zlQj;6Hh<2xI+l1jE$sh!Elu`SEt#R+D`hSpcNVe8sdI^VY?xp6PH>&p$v+!c&&pY|
ztL>Z5<27M7KhN>peb#fqjvl7y?~2Cl>h+43PrPI2<_ZceI_u)~{+#0fsueT0umx6|
zJexM{tfuldZtL<X4%?F5%e)<=?X1dmH@=Np^TqS+@df_JmEH2coSPWnaggoEx17^u
z-uJfUR+r4Z{xgcfQZic9EZC~*m40;9mX!+I=V^RjUT=PESH+ZqFBaSeR~R>V|LAw!
z`-|cJ6YFSsf3sNkwQp?2RbO4=oRE0;s{$hv!@-#!BWq2T%1nw?^Hz0u-E-sbl#+&2
z=j>j;H}@+&RE}``T=X+uiD|<BnP<20FOru3zM_BkLd6MscRLfEb~ODgYPz|_@fMe#
z_|aQWu8C-rc2w`~Xy3hjTSd?A4HJIPzrph7-<pc#5|?e;D?*x&tStD-eg4wXBGdl%
zU;mv0TD*9Fu3RUW-??V({579BpQkEZ_gs7a*N==Wo9o{vl&{QQTDqfU?#YMSRvbT2
z#H)2O{K_JAv*0-M>ApFKSz2Qy)-N$K{OI+ZXG!LEkA>Wuvu~>`KK*U6dsXh^%BAhQ
zLMB%|d3AnX^RBD6_gwSeTkXl-v%2PJS~k;)5Qf>WJ}fYFOnB$MqHwa}d7&!rlTqfE
z7&cX$**g0{*2&XvSiBp~HMvhLN?o{PzPHMkN!O;#O-X&=Tz&DEPW6eFOL4#5bF3dQ
z-s744BRj0K=tA<-(>s#4sBB%)I@fOAleOu4o`1QO^JB~2)P2<!UJggLeXlMIDdByb
z|NOtlsx5z|E`P}O+xMUQw%r3$(>dRNq%tJuFzvj)yCqb$;n4Msx66BuT|QpY-Zn}5
z=d3NCQ(I>4Ox^D-BDDLZ$`{@9{$a^`_lDf?cE9o@@7MfFwW}gkXRinQu9%VYPgyyj
zXUS3LO)LwG{qE|xHZZVEH;OsobJHNPxrLKU%q!r*g@-LH0<0b@W;i&w^-5bN?a?@(
ze7slDwM!*ZDD_mIrf9>H8$qc))0I=NZK=Ghc4oSTv04Dbfuv)aUduQ%qC%duO!8>u
zn!Cx>%T3Q?dEUhnPqofZ3R}wcz^m=`fu2;0Nf$H1Ub}5F+kGU)+NkqtnCi`rl+fvq
zV|Q0Xo;11bmMk44`H?H*@^oeCpwj=(m&vqF;?X>i_TNh^!8>L3PR>bpY-R?kd(B;(
z*!E{lU}sBEdCtv~9Y429_vS@y)eLRpk<^pn@tdjO%JiY#ON2*Xud%{SRc3m3j-K(B
zV;8KfxjXozQcpbB*uHwo2g%Bl@3iJm*N?1CJwIvr#rG_dyY_ngxM6zid`3LOuWwwF
z3jAgTaxFFav{rM<j3lk4yk3XA9M?<TX=+|4cEpQ8Z^I*%wVO?@yjsh*`P2$sfo!HM
zVMc+|0!KxHXK@=a$_XbJh^N*VF5dS{ipL=6R~EZjZ%$^b_OB3`Alp6}rZatEcQlsr
z8nH2TZ*LLHda<xwZP%yW?@hQex9IK-Ilo1JFVFwU9-=3oaPBqU9PNL`*k4=toQ3fu
z28NzBT3(tq=27c*KV1~G(tP!*Q(W3y+CG5+>-p{mIG<uM=w9QIU!{Fmz;e?TWz7>A
zI*Y~3lb#v~Tjoxj8YcETWMl8EPazu*b1C;I2N^qgFW!~&RC7~U#KO>xJ99GI7nem%
ze6(v;(<Z;g$<KZK7}C~j`>8x>s#>xA{u1A{J90!7&RM+N!MxToQ~Or&xk-JY*Y3>j
zvNezWX|~EQ^3Som9CwS%wak`1d7%Bx>d&$4ZOeZ6b*(q9yuAAS(sLe8dW{T0zk*kG
zunKHgxtQ%tz`-8Yj0uZ5be}i{@HK^SI$Ii>gfvVNf9jFSDB$s*fm1Wd#PX<c;3JWx
z`#vlx6fvqe7RccDylc(0o+wSJOoi4x*Kdn_Oby<rv-V8UqW&P8w`ZqZ$q7DVx_;Ts
z><#T}#3N^)5L>%s`P;eEbGt2)U->V7WY(qJ<=S*=g~Ij)U0SZHF_Nxp_WkO*=&CBa
zTE$EK>I+?OL*JGAS2U~*TpY&bJ#~4g>88rhrvwk*ap_HaX7gf;61S<r>8dq*rusSs
zh2^f@o%btY)}7*8)<wQ;n%}0(dtF;Lbwk>$oim)A-tScL^0vOeVAZ@%<v;WMPgXwa
ztr2(aITY&G$YQYRc1*`>y$_FOTkuVCJg~jOz-gk6k7U7*>_uBu{9g2TUOWEZ?f*($
z29_1ZdP%|yd~&XB?){P@qP|J6$IHD-B5|3RyKsYcRHIlcuYq&q=GeDkO%vBPa#l%*
zD6)5p7|i0$Z_zx`$ufn#!G=>o%VXl905yjxogY`K1pX90ZuVh{L#9i9yOzkJh8a^`
z0z1O`tmZI?EO(ly;>gs0aAqeTvxNW4tgiK5u`dKJFPf>^CbEmEC+SAOswhRa^aVl<
zVt<V!PUouhtZg{^#N@KoxdVDpSAtL0t!p>i{IsR=T%`2gE3VcLp1!k@ObVBqx!Gfm
zgpTVEM}O}}pY=tQ-l&|;<m8(k=JBRLo6#%al+t51nFl^OH6_j$CagKMaoraG{P)sI
zUQuEH?WYB{38yT)`rEE#(~nFRM!^X;MdostiRGpUe_t$<d@W+NK@p3YsEz0JmXcXX
znH$t@@Z4rAe558SyJ<$juLWNZtvp`vsjoaU*@L0P$b9{rNo|Z-eMiiiw)fhowkhvE
z72MLaFYelr`JK*5ku!d19Fr9(H2WCfq(618O2z}1c%E3#fajV)8#m6J7u5MIOC@j@
zi?6<oWKY~4ju}@a&x$Vjz+dq1>e&ZVgUjCClD!h4q&BzUWQB>T?^mxI?tY8fvQ@uy
zdRB1Rnl7DO_G;F&l4)8?F3dSwb!E<h5J4kWzBg^$LJK1_7c=PxFY2DRIwsYpBEp4r
z>3Qw9w|&Fc9P0f3|EiGdDHq8UlO#>9r<<n#5@rxkQ4pWem8-`Za<#iCWEZQ|hKw1J
zb4@-M@3zP^Uat1@b&6*6%~L_e-gmP-zpoJ%Y%@q<lHqtOqGKX(*{YLAwLnPxN3+n1
z4I3g&mpt?@T`@6e*^ze1o3lg9w2$z7yq5oePUvDTt(Ds^tm;g7+!HCa_&{gTM1yHB
zWu7tx?|hXtWmlMqPZH1PqlbC|wYw{mJ)hp_TePuHWUcA-ibKI>Q)8Czy-+E(h2y0D
z(>>c`H5TQEeD?PEujZ{*bH;z~f*bM0n{p3mDHZGuQuer0nz`9Gnn7!&RMeWdtt_z`
zj*@0~68ZE4m8JEZ3zr=b>5e$^Uu*sES36YL>_U@rk`2QGT5QxBq=ZjxH!_@ZDl2pU
zqy<Y;MP{w`c%ZdNSJi<_RcUT&_O-hYt3sTQ&+%SwQ51LD@mjyz(#&bcJI`jbUEQ0&
zyxe<c%(Hx}P&+|p-TWIm0_M{h&oMQAo?{%t#5kkjFjGMz!<!r?-up8Ts~Rv$nmw4f
z(JPVLb3&8Q5(8(q?n`a|AD-ZO&v|&y(cbyZx8(hps#Yl2bvgNKWe5pI1x%GV70|2A
zxopCom7<B}8JoH;J~c855>6CRPfK*|*~D>a%MlNQxk0hEMvH^)9@Vsv?%<l{+7Ud{
z>e>X)+RTlYz68CtlunqmHf`^oJ*j!|&*T66cUo1N<Q`w#-nJ@i>7h`=KZ*SNt%KBd
zbndwu#H5$7M5Flf!=3xGyt*3w9mJI5eJ!r-bDmpT8x*L0!fdW$>a1A{>ZOakWUf!%
z;c)YeB-fjjF)sUAy4IDrtu9S6ofL7_LETmO22;cVrQC<9GS8-@mie=;h@9W@a&xd$
zZsvWR=a$nM&!_z;aC;NOWS#xvg_^*Zx$o~x<I8yR(r0<O)6?GPMn3QJmT&P`yl_*-
z%`T&<44X=F-%BppJZah%%YRF~8dm$)N{Iv=ZLL_xz5Ug$d$yc{8WT^P{c$Iv_0O)Y
zeR?N0d&ja}^ZJm@v$)uEm0Qf}BfV0h@7J>!ST5#eoAf^S<^RsLTyv(LKdbmaW%jLQ
zm)-x$T~cf;;m@?{oU~wz$wc4V=PqAhy*g{dy*R;ReeX_(CGxG2UM>5;Y3|+FKP&T}
z{4n_(uw(<5(La%+N>Kq<L<D-}f)5#3++6<QTm9#x#^oKy_Wivfm>u+_Ct<~LDb1UD
zfwKFR&KD^&lx!^)o<B)-_D)d~qZb9KDR;H)HmmsMJxe*HtdyjrvOpn8t7T2@X`7RK
z4Y=p*(|I<JSKXj{&Ij(ZW=UQmPE1uD<`w6*>`FNI<ji@qGv`;mJHP7PiwWn?r93L^
z@n&$D@KWo@Js0mS%a&Nw97=rEqP)a^yVP2%B?=7-nttu;o2T~7f5rMsUS}>H`F~E^
zNO#Ar_iAeI0uMcUd!#9FQnqs0%j56f<?FC^J>q!sPM<-&t2gHv&rOa~Vu!D-%2r;s
zW@e6((TN4`cV!3XG~O_3>Pu78O5mTMm1N<R$-T|NTWx~JF&*7L_A`1feU>a=x!|m2
zNmA4~UXvxK%yoDUxA`CL;=gw)^Cw4=`L*Nb5nPr(7AH!nF)%2IGCGJxeysnazT=eZ
z{HDV?Y!iQN(Z8Lbk(6{I@0gd$x#b;kE1j-s)wR5NyQS&M9rqnu7S%T~e<)yjtoFh6
zMnn386B{0!2-kCFShGg-*W;#nyn>F4+eIeKX48=2HJnrMDzl9%Dj|tk{lKa<Tq3@w
zpWE@AoALhyOYnsARts2uZseY#ci2ba)g*o8K7O{-4lib`;4wLK_D0UCB0qLf1qPu7
zep-{1^f!sBb}Lm1sIJn_n-jQw!6A-7&GuvGF6kU=kK*KdsIS}P5m4f+GKJ&O7J+)N
zUbXEi4Lxl$0uC+OrfvL+{{suNzMEILi&J-jZ*PL@lBBgWm6~~<9ol1YC`3wI?ZmNf
zT1l2Jfj7_Ss%=WmT=!{zR$rwA5A)$OPtO#6|8!c5vBys7jP_R7?>dVPJ(zfR#{y<E
z!Fx72??O1(N)J0#Fw3oNwfJ?oT5XeP_(W0RV$uKX97akEQpU;(!733SeFb<ORTQ{P
z7~V0sM9x!ouoITs`=4L^RMxSaHBQ?a9=3I;1~(nQ;$Ih(^k~He)2W5t0=%-mZRsUx
zQh`hi+duM}H$USj5UgK3^-gB<t~WZ#$7k6j-%@&AcFxdQVa+nR4IU+ZXSX%5RK0ut
z;0*60T^Fx_S59Zn{V23yeem*!q4J`23Z_iwMT&GEoVmw#=w6n=7P(D|H#qGLjYKtz
z?2RYM%$%&$Yb?R)@N0<zR};5p8<VA5=I@q6b3bu*@VNy1So!cqp2KskD^9up%}Thv
z*E-MKsBYr=#4fd1#b4D)bxBQ-t2xs(j}Yw>0-s(?2o`zF_c`E69m`XZ&S<ZXW{;Od
z{Mlf9{*Cs}?mJqB%4!aZ%U%h+{$FHgS@=StOF3#oP(aep867<9PB~S17;HaUZK2Gd
z_Q^X`kRw9SP+nNf^iZOVux~OGqsb$m^HY=~Hd>hki<vqy@R=wtI-vBz-N9VR!OhS_
zyLXM4$}tDGwH8g<_qwL%WY%q(5STKVVY`A<b%+{McCVcCf1dNbf8NckQps#fRoA&-
zY0FV{u8?8B+ii~y551K#<{DNWTU+iFCY9y=*mBu(+vLs<KH-gdFQ@piUcY?)j4@m5
z*TQ-09yz5bzU^fzUu;(xu=&$w(Vv%<v_I81UGBVfMY%yqbq$koq6bHF3eT;j2R}VA
z_Wd;PnBk5rAJfhi+FY(z`m&AfMZ#@N3cvjK)B3wk!*fdUt`$6=7a6i0-mUUVFRizz
zVXgioBi(sLMp<ilpDmI**Il~8$dcXDs*Ou`>!NSVKI>?jS*@O=o1dQ)|LD$xOS;ic
zdMi({ZPe1A8=|sgio(MqI~*q7bt&}zt;`XyQB*iMJ@#UK)i%S(KqF=2e+St7?v@x$
zJiz`pWP6WGu$y8GXNXOaph=*S>$y`VlI!NR2?nKqHtp<Lmz881*u*{Qz|~j%|FW3e
zPIm^gJ^uJ>k-BTT`fCp^r%VsAB*v{xq34S%bDIQb8;5RnT9vm>K)LCVo3`-FLhI89
zcujf=nT=S#PCVbXm38Vl-8NA+<#N0B^Q&e}Ra6p+`u$(HWTmLsaZTr2AuCF_W(1il
z`-rPDG?aR8GUgNM+QOjQ^ZDy7?`=7gl|N)=o=y*7dA?oYvGxiB$?uc?TO>XDq7~`(
zJnpNhs@tnj6|W4vQyCGdL6KXoT~Kn(ZxTr{DUCVnR(s@AgqH4-15Qm>HgOrWUbsG4
z<D<vrUM7tO&$(9)-}<cP`%IX<>8{&Gk=jM}2TJUZi)5TE@sTuEz86yZ_)3QQXElKv
zYH{XjUDJ%duVdftz}>`TA=2xduVlb_Rjpy8w%ijfo^$zPp*eDX+KZCZ*G~@$T_nYM
zp4U25dh#-{U0jh%CeKz<a+r};k;io=>T6!*q4u>?L#7*BRlHx;^<On<(ii4zQ<jIq
z=l<!i9o+n4(K?n3#kLdItC;n?VBvkCq^c-tWLKVCY|fyh8l+?x)Vus^{!zB3*2_`m
zO{MoMe0H=c>54ckT5yqNfy<<@H^-bgvse2YGfAA(&*{`yAj4WHGIK}9<qC6&j#n3U
zWwp!@_?BkjZoIF<^;JN|qXJiN)kS+cx_p+KJ@7L!Is7R?{h~vdN6_MfMqwU7{Wg0J
z^S@+OJIA{3kgwg2AgkMkbJcW`0>bLv7X5g!{YTeJh16+1^^4nn92Q~xVNe{FzPdD{
zdz*^P4CSV8vrL_4i|$a_WvP0=(mAH^QbGo|*+GZgZ2!}ds!?x#2P}EOCF)f2Kl=pV
z&Fu61X44H_Ln<cy&D(Wh+nHsX)KXFxRed(y7TEVp%qsoR(I_Q{4VlkkdOTE^O4mM%
z=qu>tUv&17@wpwv=l=M-IIx*zdf#~#qoVVh6#{43y0+-tTEr9cpu2Nj*xR+tX5#Go
znR^d@QNQ-1Z*8R7vmGi8QK}KsOD;|<jTBt9#))IEX~Vp2%BB-E(lpxqF1;~fie28c
z_KW0#9u0}o!%Gz<*O(_g+F_tpef<Lm$A_%P36_g^7kYnOk+F6=cj<=jF4<4Qc9aLV
zEUS)ODE3BCxhqXCYN2eHq_*X?F9tVsvLieGyKdZFWcB^D?DI*?77jn&6{UVW!*cvl
zYO0f0;r~VYsoSC=yf*(cXKZRPVzppo`Oy%o<a@NGFTT1l_|Wo~J4!=awjHgU?aDN#
zUX0;_^ZYFe9*dML=6#b0R$8-a+jNJv+9T6alMekkHOq8s61U%}e^EOsvvaQ}v=qNO
z)aY`q+k&-8;Feg|Z~L{!R&Z^ckZ^Tc{f@b&i;miD%L@uKe-z9j7*zPy`bEP#`RV6g
zEUQqcE>(WR^Ww;4<r7W^e0}Y*uPWx>D)ujox;|r;`wHKUrj3nKO`3m9I;?bCb~TF?
z918rdV!*^`5u~yxNkDY%>T5gavHdh=-l@i;<h1tDv1@s&Z;KZ2ZQ>4PYQ39u++S~9
zV94$b-k%sA$6Kpr6#Q4)kl!-%*o8%G+j?eYtEW08@}3F{__@TSna7A<h<}YwpTwbQ
zUeR45kxiAWG#G?UKW>^IqG~0b_Vk^nQ=qr&ZE63D24_uX1z$`Ix}E+#Yis!JlEjat
zs*~qPTKvni-o_cLa&L!W;qLQ4?`&d;VDeJQTaqvES7e)pg;?)y$NQ4Uyo3Xfo^Q#2
z^p_)4(r`+k<<p!<3$>8HYfh_d*ezXsc9EWeq;#!|v}y95<{2vSdCSzES!q05^l*2<
zO&>YSTRg9#U$pQoD-M<K&0FGIWuqzHZJKm`**ezPs`Ff{bR~-Ha)0%F-h57ePr7o<
z^~|=zflb?rZ>L&3x|3?%cyX?8?*C{th81%bGwoQdd@(~%ebFz!2_M*do06uStrb{u
zIcv_b#G)%6vX#L}$L+Tys#~5aOlo9Gj-T&c#m~R^gvX*i+){hRb$6NfeA*!PVukD`
zncp(IzZ!7NHRFz|@s~5?(cgcA#Xe{Ac8eo<`%<;d)4qy3{g`g?=evmSZoh!&t=tiZ
zZ|}&s9IJgx_F_8=<Aet<l@+*_#BrM_GD+-xdEd6FIm%U|_)_lcIg8abDo*#C{`KHY
z@vvZ9o20XLQ<cPQ>G?BzuCA?iY5Y`GxXj!(v@hdnM{r+ra??D4Lj`kp%xGO_?JTvw
z@@x2eotI{ZBC?f&%yP~P2`TQJ%j}g<od02S(f?Zqt>f>Uock`j>F~^dv2KcuB`F_2
z=`&s1d-18oy-M2-&;1?O=dP%Y3Y;>_*g84o1Dk(w;FKl1eupTvom%zwN}orFFt=>$
z#8S<=8S!zu59w@Z3S;^`U$S4_U-FvYyR><MzkgTqbYJRO#<8d9?UR(+-Zg*BwLL0X
zLPCP?t&EU68+GWr9?#j{ncd;DUKGUnO^P&kVN)>knW^B~B*f||q{1_yL9u~_fystN
zsX@t=QB;`YLWhIH@qRhRk|Q003@5u(8VmMx2&#B>>l((fSbkdKJ6k7z-kFt?pL;Se
zG<YN)?TQVY<hJ<5pPgG)US1wBpUHPoad(4nn7`VDBGKe)|5r$;bIdSsb-1}XVQ!wl
zfyTAbdSPpe1P#Qs)F+kaKie89zW&@k-{_@#J+nRphgoqayX5>>l5ntxb%J8#j9!K#
zxdvQzo>`k2qUW35)|%tc)U;vw@nzE%1U9s6Ss#3V&VfWOc0JV{Qxt>U%4go&?^~i7
z=vKkWsAe_A!fuaCk??Wr83#1xTAkFpZn#UsImSWqy&lUHiN4w1?6-wAG-hnDYhe1$
zBkMHh|AxBk$-SSmuSOPUp1l_Mb-H|=8Rv?r9-azflDbS+4t0sinWiqN6ME&M5N!VN
z`vuRKM3%&`XpN~Z?JBiTudsMcS$bI{x@1RdmwwezB{$=wE8+>wwMSY1OP2b!T`X7s
z>7nT6bnD8gw0XRl5tCFDm(5`4+PE+xO=4@Hg!Z(%DUs8BIWr|C*K9d-Dy3=S#Yo$&
zCKW-}DIN<Q9L+kv&Wf;Fy6nK*L|=#5xiWXkqzfC0)HBNOGge+Ml<-X#F3*&FwSqyV
zKv9!DnxkcrBbUm80B24X#Xx5vRsog((TD6UD>;6&O!8&%lbG!1am6HP&8i@aptS+M
zFS0@$R1LVA)kUwKQrm5HRm*de(rmGnd*V*4VNgkGnVKyjdgQ8Fc;wZEK@yr9r$m%2
za$Q`PeN)3>XUrDY>AqHzOs>aOYu=PdnSOHijlBu3%a$aBc%@C==U?<pGDBji<*om2
z>swug=3ddVX=IUpb2B~fz)!Up$=H|}nZ?Hxcg<lqtk@pQ-Z`m8_~ebdA#IWlNpqs}
zwiYgUUd6TH+r8>Jx0Zjd`S$J2=kurbNhSK|xkh}cW0*JPm4w9F!wR}wn}o72IESy*
zzUr0iE7ah*+*E$MS?!La9ZVS$TUde&-iB^rs^}IBz31gPq07%#aYAIp^d0MBUTaBS
zzgP0n;%50|-?GK~9VdnCc;t{>7PP-l_nSqsk5%21$4eaEMLbpEJf<2kIaN(%#@a2;
z25xUTE_kJFYFw?JyXEUly}agYav6^kL|@(QdwI(?z>z(3SHL-e+N>a#<+DlxuQPmO
zce%*Xk@EHbg-e@nZQX1kEC1la@=qmNp5jt#w_ey`{MF|6*KLRV4oOTqzq<7plfXew
zfvT;n+tla24NZQ)*CKZJTb6h9Gou4LKk&MH@9bv!snk&Mv(@QWmKV!GA%jyUp6bGD
z#98Mqnh~g@nvf_Y;px{t@#2YAv!_3K3wB&?eUR|nTJiM^9(S|3PdFM<EUtK-a7b%%
z`YMp3P%W9aLcy5tf)CS&gEOnk!sHn<^lRSzp1FF0LaRoGBe!DLp)0$}g!S@N?89G6
zip=F`Sh(b*<8030_P%RVlJ72w$d!q|dThtSUlU!plP?@fc@-|Os=<Rnu_-J#$Yk1{
z!V@~WPAYdpj<7Op)3)#bf93YOIYBDNLMKh?JN$sPUhE)O(i5-$mx2<RzF9J6@M)!4
z3C&a~R=cpIQ{`<&vxr&3?pt$R0-HT01q>IRcAd!Lv1f5eE=SW!6@`}Id9ys$Hz-UD
zI(fuCB>QUXu4{t&&sJr{-kD`Kw?ml4C6Q<Lg(kHln&%8%8hd*^uSq|(aI?_kX7H3?
z4qRZu6;x3m{5g7B@Rdk~S}E=tkxGtV9!6`ka-T%iZ|=OE7VNR9<@AKMg|m14auX_5
z7185$Nf4QT$Z|!+CQbRfFMQ>ay=J$VB!s;3anHTjC1JVT@=&9nkLx!XXT^3aGm-vN
zEzCyKr?RJQS$LC0sKEHgh36}*9;f~HbUAh9AcF$eEQMAT1rx5E-e94{6Pir#9zVZE
zkaxE4$D}U;4J^7Yid>!@3__C{<V%{S&ObYOvGw9bg3qR2bSal{R@!=Cs!GB~72Q>j
z6mBnD)HsQyL1&Zn?rTc}54&87sb8qk=hnXC)T%QEf^S~z|JAC(qomn8@p1Tb3vs6n
z1)5Wnl{%-eI3K#Qc3qIe<=BNK#=8^wR4Qg)no{#b_(GS<`I)Of>0V0<+PgI7gu~^S
zERStgVH+F_HFO&vc_<mOEsmHj`dWaYR8=r0?BnzKa+Prvk5|MjomH;x_ckuz+Hcz*
zu7cr`O!KQxdQ4p^GDRih^(nCrRuToh+zn5xuJ2O&|4Pr(@LC+-5&=(X;UM3<4+jH2
z@+94OaPV}<kHT3C!_|7f%nZs&WSVd}^;pHv!#$E+Hu}!}XKoyx=OY^0>9Mrm<N2)+
zO}B*xpVl1J(hKHZtf0oV*n_Ees)?d`p8h_MXUp6;Ox!Z2eNT@*`%K!L<-~^2nbB$0
zy%U5a*YasLx;%=K=v{U}ao5ofy`D9n{wZ)zJEpgE;=4mC6B)c!{!LrBcw%QyTB!I#
zZm)>u0IsEvZ8TMT3Qig=ukq5d-yLPL($je3nh;IB-^$Wo-Db%Do6E#!v?ry>Yv!rs
zu7+o~LQ@^5XC>8g$^5L`8rv$kM)cw#wLG3RlQ$^)3SMC8Tom*8zv97(CqLC1h3FM6
zoS7ry;KRo7`$p;htglmDA}2N_&KA<!bYgYv(<1KJE2h(TRao0|E_*SFW$v^SHk!@6
zX<C!LGGCmIIaxgSbV75~#S`}f*4VhtiR|vo3Q;?@<K$kx9S7<sG5EH07_L5`=aw;X
zilL8m`Fgoc7anPC32dI4UG!w@k%JOj+K&j=OC(BJ%seK#&+9;c_Tx=kc(qO#Ih*7&
zEDPM`f9Wmr@18u<^x~+NC4q8{t)BIKVhdX1E-ft2n_Q!3<+*0d6f+Hud(Dw0yVr(T
zH7(UUJljFq(DLlc9}Ql13)?zB2_;N?Ciz?W(bgTFU8+KF58g6xC=`{g4~kju^?&Kx
z8=u!*I%Sp3*{8%cJ0bdBZ}g;I{uA?``~Gpu4tk@)zFYOYbCZ)&fm!xP(}{kXE4C~Z
zVB8q`^P$y8{d-CGX5>V>Hr$#vWj1@?bo=*`O&;pIF516LD9WC6NO+yWZvE*8!&h}K
zO?go6oUli+<3;SpHSt{QzkbfrG;2HMkawZa@DSgMnZ_q14pw{%)jgiQF8Ia+@%Y~h
zm_EF%|NpkE{k8FVGbL@Who6FOycgTBqt)i!_M9-mo%c=z>_2dAy2?j(kC=@`O;5}o
zIeD~C_2{~qmZg^0dpO?Av1-Y8Z|g_0`_zNpY!h)zcl4R$_-j&zcc6vVRm;_)QhgWs
z4lfcs8xr^5E>TM}G38}T&6C5m&!!4rQL$U&X1e-Yn7Fv2h<n|H=?oEqwPpwTk`*|U
z7jT}QE+&><e&AU}dw86ySnKiQI^s%|oD*9!n$4mU3VyeWCfvxeT4)wnXn)H|_vhAl
zw(AWG*c13o8!d&>pKfAh5sRE4nwPJpzxIRI`t-c^MEy@@`rhW%%@P@&=27Mc+fzQ&
zpEOT4O04LeA+OCCnk>M%_<`f%16&b>x}TZdy<cP(E%i0u;l~sd9<P#mXM0)bWhoB@
zhBxZDx<cI_!|P`+2v^?Dni9~R91)q);q~>q=zDjw2GRN>9Evqt(|A41d5R;LJ}RZV
zr9UgOs0&WD2on3x79u+7h=b*IY2Knz8PV#RXOdFK5-ZC(CT^CPdO7fsgp=frJafg;
z>A_7ZQxaP*hIwzac_bdU{g4~eqOOv1+o$Z!Cz@CU8$>-A3c`L=1iz?tQ;z#4WOshC
z&@2YOSPj2eMbUs8{%eA=Bxa_3c$#vjxvXfq(8^7kQ5lsV#oM$v{ls^Q2v|t$^$_xz
zT>qoEN4d#9oulfXYSsVGMIug&%x>cKo2z`LCRA(m{d_3tKPhipx?@YDe%lU_ozMGZ
z()!E`t<tY>EjiFJvBR3nkZtjTsX5G@W!I-}G7{vPSkH98B+a;M^$Ouz?ut_E)9)@<
zS$nWfoxT3^_UZdqgztPY{r^|>$tPY+GL$TumMW38xLZz$^>kQj{beQjAhVk*l=2Q6
zJZ!40RPJq9T->~Hrnw_)aaqKIVDURIWILy+c&sd$bJe2JBFJi^Rf~sb>qRcRCe^u#
zR{d>OQJRczTWhR2#OA*+y%*{m<iPc&KynR(nM`=$!XKdl5_3KXPvlgnjQA04Qju~$
zblU9{iNc$)s)_wqr?d(lnDMdQHG-v0B(nRJrb=t3@Y|DBg^Wd)j?IV@oZ0+SWLl(0
zT4L_wN%HN@dfi87+B;7PNaFT$%JY`Y;A$%IIVrjE=9I^2lEPvcLRB3z6{Do5b+{jz
z8hg_?Rdm`z53aH+3j<wJtv<`&_FVM8nNdh$Is?Z70d@B7BRk?BxUlWm!MSrrH?P)=
zs?-^s#lo$fi{I^>=5oupTBRYVp{VNfjJ}&9sf|igI2+Om&4UA$URa^ToKoyu)SFl~
zU;k&{G#9pe8XZo{)7P!^oWop_KC5rlRaLW%89v9o+>=U~CRk~Ba0z{gQ7o)GS!8l|
zbIOksi;laO2UT=QeDqyfD(SY6bNitQ&97Vpmp2!;#PR(UHgU|667m&%)So|HYkg46
zx;8~2c9A7ct8$kLsdt(^N@+`&v3Riq+j^<h8>&1d3KPZEX8ig%&$Vmy?o}$6nH1Mg
zYzq^dS)V+=No36r)wFyK{p^WLnYXY?OfBv`A^Klhqx#dOSq<A3`x!b4YgwvH>^x}H
zx1nX~=WARtTqaN3tr-{^lNPPkmg>~VsxzCIRpT|i_wYpNK&iLK!>cM6TXrpGnB9H<
zmT}69bqeZB9JF*<R5vJuFX`I3n)USRtm4I*=1PCQic0(zNngmaC_TXRka(t8Z{aeP
zJh4sjvqa5S2@5%JZ4>a`l)kK8L(QDA$x3kf)3l(#%Dx*>UTKT0A|I)QnlCHWQrqJx
zw06eaQ%+fS!l}ZI40T^ut(t6=cxi*$!R)&~W6y>w&Uv+B%_5<1w^sTWHt+gmJI`u@
z+RoOen)Pd~xJ*xNvyfVCRk+$LakWn7YVpV!+h48Q`%A>fSn|K1Hbc(qc}2Uow^iCR
zMVr-5(G6Y3*nPXf+tIvt=bE2}fyIXHcdnVUJ!{~dA~ugx)YE}&twBH6uXg`PNA_Qi
zOMWkCOcZ`=(IJuLHE+t)wF~BGX-)q&k>&eguDJ_$r$p}Fw|e4>rRzgg7T08U|9iE*
z{ih0#cYtoE?zz)@%r9=(|Js<}+N@VoKls2#?xSY=gvI+@dZWA)*;y<mhKPkRCS2!`
zu$vz5WUb=yv$0WQUEMOr3C=;>Qfd)O43ZwJZFgHONsx@PTyB1mZ|eoFN-YNW-y&0I
z)&0HFlBg|wU0G4kGmJ|yd*_KQ&zG0~<5+d`<%DC~{7*cxP3;!eyeRDFy6XScmR5tV
zN!2SSt*NsA#lEBU@wVdV!!sndTg0qx*nQZ_W{2YQZB@T_D~tQFm73iss*lrJy8fVO
zage#4rNqs)Dv5)+GjDg8pA7O@y4Hd_(LYdfdsm+CB8exftkyDAUFM!+oxGx#ac<nt
zt)CwGaR@9{Hd?$nV~=Ik;<}%E_WoL$-(}CU+oJXJwsp5e?ao*%ylKI{TKJOp4gu|r
z7r8wq9~Av}-8eAIgG*zUqIO=e!vej`_C=hc0zZ~!O72?0Z27cenN-B)1tA?~j+?&<
z?VmMqzxTn-n`cVqKG^Rb#GO1#&Hd2vNnEy{Pc^?kb>Q~#isc$YCyVPI9u~a4`OM!b
zCc(^N|F5!SKPr{KAY=E}ac+9HZtRUi`fKJHoLSs{^Q4I9lG4>?t#?GqRv&(I=CJ?7
zbLNIS-f~W}yLG}o=6wC`X`*J6r>3eM?rMu)$l9}7Uggl_KQ|?WRNKvpc1AV^&bmFb
z^o7VTD@pcZ;pN(nmaBJ3o^_0UGsQ2^OYn;Rd}pqG&%K^z9ZSD6QS)q-h~l1-oa2n$
zi%)RxbvS(7ZqkYVm=jY<Pxx*;q223caWlM^+2PJ?bDrd-d4~Hg|JcEp<<WiRO4vrt
zutfgV-KVZqm4yGTatdGDQ6U+<Ncd0m>5pZnw*~MkU*(HbtdzeqM>MEVK1TZj^E^=_
zMG2jiXEN*JcGsp{U-bV+$AP+9;YsZF*~^6@JSsDoyR|tt<XW!|IDKQosl!#$N8YWR
zr}TY;^`3L@ctk(!IXZLhhWk+)Ui+MH;$E^)_SWth8``{e!zRWDCc3MtGxVH3^6O@R
z?M(Tb+r-YU%C!nQ%J_ZJ(v7U*8Nz%*1+@{?ic7D`=-zp1waEVN9f9p4>n#><iCEmU
zhjWPoSF_A{=Gzi>yuED?k8^2D-uSet$!Fo#S*nX0?#<tF^O5YVyK7H;_<O=<>T!mQ
zz0##8|5geYJZtD@J(W3eqo1H>E#qRH*S${BmVb+8t-opfN!xPD<z)#{Dw{JqLfj9e
z`3mj16J)?B%4uVAu|`Pt-tuQ(vk$)U`~T8vz3jVCdA~!;XB=ppdr;zH*V-AYG*7Nd
z$W*iFyK8lC^WqeV#@9E!8*kdho_Nu7?p@E#H*@7Z=gEJ5bHZWX`JX&DAKg`~Rbiah
zF35b>AnKgm#N7t}KA-fzBXW~XqU6L(-oHyLBrYB}Sr^uLRF0!>%k#AhpS1hW)L)!^
z@%f7VUxOBQvRTemywjh^{r>Hpmb^pjU({{XxNJ4+-jW5o*G%Z_t-EXfXTPo7^H+QB
zp8PKQK(?vTO!8aG%lYRl7TTqA?z+syU9_*}*p3zELJqsX1&J+FPwOZVlD9s}uD15_
z57ESFuYBKjMPGd8*Ldf))SFqNj*T4>?uWQ$UfdOuskVL1)c*@sPu;G=kdd@JrRSRY
z#%mM8yGk+@rb?~9F-73nRjD}dvUegkmc6)cbgs+o_Pg^_-mZMK(s<vYm>t_J>)u<h
zJNl{U@LL|S-)|0k)*b#cuVzNA&=2KX%IW8x?K$ta?)>+h^L+0=*voz7UM;~KQ8~$S
zQCW(+w~mKI;~Kut@t-b<?X2A?IyFE!deKeu;|r|&_le(eVBW02y0GHh;bU^g4B{QD
zS4eI(cp)oQ|J-X4XTptQoBHlp|G9!M8RvXCaqmT#=JWn{3m!?l4CK!UjF;G+^=kS(
zV~f9wAMd?8b>hW#v8mpXAHoGsq)Oa3dw6+I-F)+pOO5_2UUCyX<EuC&bKjlaJ^#1x
zJ>xxb(*NT9y!Sga@4KcicAu}f)m-{b=%nY}4@)Fxo$_CA^{XgKV4cOx$NLu=H|>_$
z+HEmC*sR&9I7^l1#^smMYK&h@E_QLR`Ni?ZHAQ{>wamFH3tOLPX1A?=9JDEFo2Htd
zx*Efp=OLDFqJn;APF(P-FmZ{%5j&fEZgoQ5@9fh2KYmlX`L#wU;NRo5Rqo4Ia5DQU
z#ZFsu;geM9{Q#x>+g!0b0}|72{t(HM+ji)1xURm0Pvxn=o1O)A&mY`K;P&dP^83%A
z=-?n7#Kg;`a`n{$hG?d&t(&%HK6L34)s5T3;lj|sA~QMTVn?8gYcB_j%EYFiQ(~IQ
zW)qc;YW;tBCd?$0D_}*zlXI<B)sI{}4Aq=wt`XSuP;<f=jb@+nyn`Bn3tUz*sq`r+
zH;7ydjAkuK<@)ye+M2k7dS6$CIz+A!UG3MxHT_X^bD{e!->L%cnZ7|R>wJ_H*PqZ7
zQ|Fko$??E6%^p^*wo|Da9Zxpv^0#$`J2a_qF7cQ!>1x>S)5^VECRMYNUYRvVYr5>3
z%)8QTy=Q7@M6`n~;}XxuIqqtm2UtXW#pY*RTH?rXgttg+N6lv!6($p<NIn+FVwDrD
zOdtDfmK^*zcd|<dQ&jgd-PtGZ`2F}*YW?%e)BXQ4H;Asj^M`dq($d~5|BOAhwkB=n
zaAaYP+OVZ{&Mz&e#{UX?bk=a_9=YJitG*~;fg#I}3rhtYSuU;=^w$w|6?K<9>>?EX
z<-*a<m{VQ`j7vHaEOgaw6)Np7`=Wh9TGQ%{0V9{Tk24=jM39mC+gVATjXMf1EjlHW
z<G0|Q&;$i8O{e6eV)JWmtz59=j7ou$iMMjF?iDVTrDCSKO&wD>%=U0Fa<0-oyL;k<
z6NXFmZ%2DAoAs^Q_e{H3tH$#7f>lQkn5c*PUfOdkC3ycrHK`Ad{=89}l{l>DoKo1u
zSCp^&{njro&COg!T5GrH#avz)HTzW7!eH;qJpQq^CK;NHiv%*d57fv8F{tu+72Vw5
z_a>C#<f6SoH<Mm3;)&j=_sD1O)c*><eD`-oX$L426(0Jsz>L?bJa$e<@%CVrVq1e1
zVNu<>ikB*kJ<f&QN=)7{VZ!RXgjWI*?=71J>@*u2H@mDeel{=LZ+D}q)%nv?&2BQS
zp7CS$`>j!LesA|J7gzFC{LONS)4rs4*Y_=dPaV~`+sfeLb%~8XsK6k|P-^O;1-!pD
zXf9s6cEid=o^vK%Sz22=D`REj-wFDSTRyzL(VQdVz}#H*CyHU6zLwBN&Pg_#l#Z%J
z-CnSD`-DliPOiJV?N*n`<0C3|!VD%{Yc|R?ix}QD;k|N|<4uUhp~*%8ii@kR?OX&}
z_p2EF#-(bZ&gV-V-cutM6^b|uC$$E;N>AOaWcfebe1+oQu>91omz7-id}Gu-TeE1+
zk*6U=oRJ$#b+_|znQ8^E-Sq29%+4xP=e12@uXQ%gi=TJz@ylnSx9>bH>c4Sx(TR&u
z`_%%HxK#O^Z!7E<%FfBlyuUu^)LZ@TX*)h2`Iob_>5ZdQ+^oi39*>xo=oLSGYP)r%
z=d6;$`^%R2DSXX1TCnAbOjpS9CZ-(<^LY-bx7E5^satYpux;%;u$MjXi{r!nZr|45
z`=VC*@YHE;ULn2vJx5X)uJrFdceIW1-*JZ>k58U6GT-7~spTv-YksQ6788-c-b@9h
zhuuLcuA;tz5x%o0R+}j@>TY<VBKSh^hMtm&0&ivxw`TOMOXd>)kLq~V`A*`Vx+=vy
zUQwO#!p~dBSd4Zoc)${->mB^6Mo3E`ilzMMZjPXuAWfI3Q>CR=%9n%$G&B=(xvLBL
zeZAD>4NHy6RwUm{jd9<3EzrfUYf}evsN{kFvP&kEJc>Qrb-ZnU;*l!8s_v^HU#GtE
zJrTP2N2<YYrG^_U?=$$dyoILDJQ@=l#8n=n_Dk&1QI(rZ>rS@_xvujR{4DdtJ7$X<
z&&jLHyky@6>CBFN&sHS0gkz)Q`-7g@CniY!yplY{IOnLb>5lVloGBJ}Hu@blVw?3%
zfXhymJMzsNV<YibeM-D<raL599G1Knyoi79LoLHB<IbWG`*xG-+PvS@m^l9zDyi-D
z(DlBos3cUVxc$yN?=+W*D~?WCCL1N1v1U>0xgSp>>p$M|KdROs%gL5wC;Q6fQpb`m
z2~P&Ut=)GYsV&V4)z*vI@NCMqmv)Z@(kBVEbg6#5p>as`NeXx7l7|92!zNnv%}`AZ
zkLkDEDHhZx>an%MGeBs@)SMhm7RxK@{?AYP8L3R=bA00{wJGB8BA<=ob5FQv9=c(>
z>UH3Jktb(Qh5Ai=^m2`u<kie&+Z7Wz-&aI!IpuPrN$gpxYuLJ6@6VGm_XNeJtUkS`
zqx#PypVtpW{j+s=el;DDc(m@K!l%W`d8S8ZvQ+phEC}>htNt7rFS@B+)RK3fj??;^
z8yM~XM`rrwW+n1>uQSM2c=DPf`iq?0wdE-qF^SAiPh0g|P~XIKF(c{g>jMX;l<zm)
zvm)iSl=rJoEDrs%gX02biBJ92=PW)cpk=D{!el?*F29sV-4QHD1zbP6NatPbGU`!!
zJ#m7j^{yvrDl6>dHbxkv$YiNpv}oPDbgfBOrt;)vA(C3I$7)ZCp3GG|Ca7m=-miRw
zQ=aq3{&Ptz6Kx8oM+XK^2sqj~r8<XIa*Mz&=cDNxntEfCi>Jq@aL!Pln(Q&-<8Fb_
zX~l<{8RJ^3zo(=`_J<zWbLwJ?sn3!(Kg0Ov3HYonjeZ+hVUTh|*L$Ak<GCFZk3I{Y
zW+J$G#@ASuT!(xA1Ge<+p5PnK(jR?TzT|7WNSoCZrCpL8F~Pm(CiHoytxLHO?VmNF
zMzmnwe@)5!AEr{Lbw&2=lPhp#@p105dSbrxo$u3(2=2PjBVxPtmbSfW%#rw1?wn?K
zVyB){EX(tC2OTDVc8s!0oMu!QZsJ=0tjkTS;#X_3*0+@^RW0G`_ExNj*!;b?^UIax
zmv&svD!%Qcp&+0Ya8)Vt?lq^1o)0fHa;(aqL`CE;*{GkW8@{4@Zq}t=YOhzdu2nu6
zsr|bnY_7+)4Pv$(TlJfY{zj$UILc>srjs}FxL3&(b8%xn!|sS*N#Rd?-NMo*dfdLk
zcKU@^p;u>pYSg+9MpbDGMRWc~T2He+-})k7?u@t4TX6=1T_$HYO%zd6Xw{rhTPvNx
zGs&cQsaB$F^_fQ}mupY556o{A*?91qQE^a0)cU9I?wd}oIH{dBE!||3R{x8(AE))F
z`itBWKKL^)X8G&m@6?{!JpXwnv2RORXN>{(x2awiOI$ek|K9eDSrlSab5uR-ql?@!
zrS_#}O6sq)3L=!XLM$%3inYJZa*<s%;g6EUdhwI@v-a@xmEKt*SNyfM`QVo$EJjbB
zJN!{<Jh+d=Zo9b2MAOI5SnStun@{sB<nn#)zsEyqe(`qw;LqWCJKvo+6L{ov%el&z
z7M|rr?=A(nYOXxhb*XE^G%dAx3pX%N+3-5^zy0l9+)H;%+&77VVaMdtAJ(=`Q1Gx8
z+O*gtW9Al>nPQol^EFS3r6%n-ePA({%c=t&{QF)_a(rl45jAtl%ylJ&k~Nc7M{q9v
z6s$XIXK%=E(~`{AwAmeLv$q|~YHthRxZb!ho!d;(ibL_p+C8^9)!t11ewtg-WOJ$H
zWG3cak%uSGx7^FJ#K3@OyFQQk%&J98R%r4@9%gUsTDrnqw0nvELXO2;`<57SNNUgC
zywY&lA&xDMdW-=)Yr3Q-dg!gWwees^-&sw*j7@fXRyZ`D*thr6zOHCapGVy{KUjD)
z&OEIoaE9sVnNR(96_5UzFh^?7`j9m{Lv1)TeY&Ox9}UyleCYIlOP`M`SWR5_c^j~>
zY*|$7aJ5O`>B_aqQd3_|l-o2xE>3Em&t<iF3MLZ^_&5CG`QRY6Wg<hm;5yz9D|ZC!
z-tu{)W!2{NGu%6Gcgh&Le@@U7eIpdO`ACzJVqfL)r4!uyEBmWj`WaHVR%P^Gi4tAC
zv-kZZ$&e?O$C{nmG)_#h_E40O%ue2BTI`X$n}a>dKB~p0d%>a31leDk4)I2+bX_>K
z{m@*!D-#Nf=P|Yz6bc_Ux+CyB^6=H}y?n)cr~3$s3ko{VwiSBVy+ODqPm*)P#_jWE
zI&D){)k|}lSIPgJxlHg(_u5Ge_KJOq%@+F^_br*>z<5INZ8yVt%j2t;9P#w|&-s@5
z^j;y6jVh<T=D2KL<D$93^h(8)U(QnDclPX9VmYzGa{H3gr)ICY7PxuZZr7s+k19#2
zExhTP!srk(QGk(UnZsvQ79F>vF}s%L9BaAEeY#^>puw~=5`21^y^EtdrWkOfYwkX@
zK&?5zT0rFBiXg8gJB79=o+_Lo(cR(Fyu!=Bny=II_-&WtcaHczd&PJ6i|@QC%NZ_C
zZ#y%m*+ezC$u(@vnl7Df)xUbbCi^w(n5HCe4rk%o(X;f!@6$#pnn_>I6(;*kWig*J
zM_{^0=QJJOFE+FIE1mOiofMrt`}~K!_ly^(9CWs|v9rI^=`Cy_ctmcursLbrZu6iS
zXJ_u~E%?8~?awN96)|ODE$%0?tXFz93371S^7C%;V(DP$nRtQIiD920!(@%l2BC`r
zM;EaCxS-Ood{u{VtASrnm+Q_uelaO)4&9zBxs}IUc&fSJ7TG^X@BF$LbJCdarE03l
zCF|E0W2>x%T*Q2wwC26ySf6bo=B)KLc-jW;vnLC9@^j?3RqxY38Yq0`tl~%R_m+!q
z{_*D7+VSp2i{0<VduvYg9O0NIt-71%@}32Uy6>#FmEF*J$gls(&aMrTGXtjo{%p2b
z)ANJP$xFe@7G~RWOB>#--YdXpICDwJXOlCTO03B<&42IgsJuPVNNV4&DM8H!cGATH
zPmR44ZZl~uJhQ~$LTllFK9SxA{;$G{7tOhO=6{ypGyOU_$hKqEoC890J3dPAD{bEG
zt1I5R;qsCVW@o+qB10r4k2-bDy4Jnx+Kldt#-|<U%-;WI3y1UE4HlPmCi6JQOgdf{
z!@VXTd~(W}r;hVv|9TfL;c&Tp^6Z7lmmIY2e>&5;!|h^BXI{)DPvr=MyYtp6c`ZM}
z@odwnF9ICwTThuhUB>l^hrjaj%QYNII=z|Ahm?+-*c0RTWAWv=2@94a98&&rBO)g(
z({_1;m3WiHN*R*{i-kF-GwB}W@pru%`L0G_dWzC=&+VLxC+ePZZhx$0^+(h5hwZ{u
zK|K!+Yc@oMoaNTH?bs11{clMW-vj^ud;WKvy(#lDqwnzzg-0*?_*q4sWJF0{oYlNT
zndhuv(Vq)XA~;Ofc0??lAyUn$RXru`cW~^UVD`?ibmKKrrlt#CoVK276j^OBTlKtB
zP*7~puGm#wt2qu#j|z`jD0kfE#)%vU_Kjy$4xAA<y7-#R;tdm-(pQ@2yjgwbg8$pZ
zRgN(ZRk7=?PdU5)t9ti~l}%N_l3VZIt2MI?m9?9CboZUk!?xP{=gMF9_I#0e#YOb|
ztk{t5j*#o04Q@nFyj2~-zQlG>;?(^Qr0(!|2R5$@P`>(`=hUB)387avc3rrYv#^Qv
z%%qz!eSJF?TYu!!?v0qY;O@<k%ly2z*L8*Nb`2Eo>GkdUzrTHt&CI!{cT5qs@7<Z1
zD=euKcf8B!>78p6SVLIX?LYifzfe~@yK{P+&zWYApm__r>z3R(9eVwelh3{h$4~n%
z%9wb!rMuhk2KSP`LX9^$em(7wd-hQNTcXlq=?-K5x5|$`=*siT_@4UUurl=2i{^b!
zxBXh*#=cr{!+Q~D>mmJP)e6?Kk&F-I*$vO>-xKIPd!yCh@!bX69z5mb^y`}Tku&#m
z!rPdY&)0U8T60JQ-uv(*x!m`F>ZFOS8O}>2pTsZZel8@jeys$<3JI30eO)*Cc#=}u
z7se}aF!V7rc`iF9ExCKSZ`az|`Hjia>-KWUt>)nqFp8Ypv3{<^Udh|Hul}FpwkEbU
zc4O4MzP8hoCvJ$|#@ggKv1wrk|LNb+PmW&f?K#I|c1LDck7b*&bJo)Q)vAuKFP{8*
zaY|G8*{!>@&T;oKuC`f{^z@g^=H%E%mx6wHo=sJX&HX02>*(FihQ9L&ToVM-N|&Wq
zdp&*?V{9b6%_nkGF1JnEHnB^}dku5$HHF2wZ%uC5c5=nSy&9XkX6#wSn`|qwFf!q!
zXe#gSn?6h%OfSCW=ui48+`MqX*%MFNA8Y&jJ#P@^2#m9nJ-u<4!PERRbNaXOh=isJ
z?|nMw+0#0vg@UCM_`NRFq;|#_Ce*q;u$yWrVajo@z^1$4ad*Y5FWwK#!uDqfUrE-B
zi8%G2yCF=y=R%;elFx#Rs!tNHi~C(Kd8+y^`PkphJSzn*>3)#AqkU`tpYYAjS1;+L
z<|v)ryG;6rNmSK5j_9^jfv(*Te+YFlNVai3)^&S)CD3DnZO75RoGFvvh)L~fU6Ipx
zA}7W3v3P3MM1xnX_c(tWZFPE-b@|gBE4}k}VHx)KA`6ZEvyOE*9h<zOvCHDh_WU@G
zS#q3OdVLHZ)GsvLvMWjI%8<Td)3M#q;-pB!_pkmT(;m%U*U_TbzkeCW`m?Kd-py@I
zeBQi*Z^5@vo>_fodV;)mZ%ud>eaA|B$C?NG&suk}<V>6}#f#JD#nYqze!Kn;el6t3
zRT%R6oG({(Y{!~E{})O6Hf+?kKJnq$Ne1_zzW40&=a)}oSb3q>zV!p=Vx4cv$;r`?
z_l_>C^NX3lyx@rAURT!_wq?1^-~6t*U9mCSvR%UP_0&b3&+fh5%W-Slf!>5CufzD}
zOnawecwquZQM(`qtAO=J=~D+Igl??mkn{G+{J3$JTR{1`0AZQTvw^9c6Zkgv_VMn`
zleZ7pCi;HeRy(T@jxT=ub7b#L<cPU=?V9=a552pbvOVWsDx7fnoA*yM-l;-!j~#X0
zl6HNIndm>e9XV_tPVSi&dy{+7LT8cFXP#erYtGHkq_q23guLF1pzn@(&1dK4etOLv
zzpQ-Q?gO*VB}+S<zVRmHx<KgtdDZ9sU!G?C>~zP2vx^UQT`&vQx!%RFS5*G|`U8)z
z`W064ezUKWx|w&yH0S)Kr3?J!dvwijd93%n;E>r+&C|7E_O^%F&$$EG<Ku;|Dx}A!
zM&3<rWXLO8&hdT%-}{WdM+~pt8(cicslmTDGyGoO@0)8cSQP(hIK=nApdd7%p8ra{
z_>a2z7nP@!-h23+-9B9CpIO^$QAO1+oh!O(L;?=1TXOF)N4-<<F2AHKr`?Yxz3gmA
z3Q+$!cR~S|zU;jlZzlEU%!~NBYTcUZs*4>(hh}ttRoOH%wqAQK`<eGoN{!Ckd~x(<
z(>_ljKhbN&A7_0LxY~Th{j5RN!Q68Dj&lY9PYePkC;WffoKY^%UbKz1p~yb-i)`n%
zOzFiU9GlO-o;baz^<mLmvyP(UMceg@cJznZ-~2K6MwQ*e_}9B9?wF9C#CLh7`j3eM
zHK!Z$d$xSObl`LMjK|?zYfTdGO*JW*XOP`BKd`+&@9V?s^-ck0+wZD46nKX(KH(w1
z>g&{8_p4V^ewtf5)~sKCOs~cy?(5r}ZPT~EnwtD7U3qiXX1^6-Su&4)z7aV&+qY|4
z)b_{eg^Btne{b_Ny)*Uyy{R={7Ok}5V4SvOomA#>o(2YkPk!Ibw^+VAGhc1hq)A5t
zGIte*`fVs%G9~nVgyD|=WiPcaaLt<WXu&&9sh954pSth=WXR6J$Rx}m^50>@0tOW}
zVGR)hMo*T;pluu~SGg3A^+=g>?U`|LF~bCf>`al!3r~(WsRS40y!@2v7;VUUqNMVp
zn$sNFL@|}4pVJ+~m|D0}HW!K<3v{zo65Zjce6%Of&`;N+?a}2m949UAdOXv-up#L%
z(}4~LhnpL->e(h56+W7xTlk#o$)>Wkv5mo8@Akg_uAi`xbCQR^gK4h=`oCFR<9_pF
z<NaACudn9ZiFoj|SDQ6Q?}p!j^u~pX-KkrzEcd@8IXf%m?5xKUjq6wy*82Y5mV47P
zKUwCml#bilSmV5|+!%+PbsZDA7tFZxQcSaz)vrHDpyQZ|L?Z)3(}CR$o0;}6*X>wP
z^Z&;OhI8}WZ`;`(Vqkir9-!oSszviB>zCZf&;x8tH$)$}$@m;R8W&u+F~PC&;1<>@
zUC)957iPth36WyEROH-5ltdOf@P^(HYUk(_KN!jBl5p3>f8(XDfVDr2W;x7lO%YBp
z>sqRup4@mvqF3KjEz;ZOnnqX_lM1J#hpwRMm24*4%qx@h&t4NMIPllSZOW>(OH&=A
z|AY&)$xnK@G@j98!4Z-Cld1x)9C=R^QuwSaLhJu7+^IgFIrI7)k(4`Ej%CbGy|$R4
zU2MwCMK-2g7Z>ZhuDZCmCoiRYsgz*liisWnzNWaDG?}Q*@{T+i;<)ap+wE10x7=FA
z;9ojZYwfPY|C$$8={H)&tl1HCG$v%Rf?L;&{G^bGnH;7m6Bz2hot#{A_TKdSbydA5
z1?S2L&d_x#+jfM}Ep@7B?&kEgCu=s#G@S^ne67?n*Ws3+O8!nsRqHrMURCQEiP001
zytNIQo~?|UZCWXGNat(C#Dl6oOF#J4vvN+JsMg38F;%?%>yx9)K7~zWkh^^<dQxcH
z%jpGs%ced(Szxz&;*6l2mNmPd?OijC$;(H<b$0$c(diAmY_*%?{(N*bxzx{Fc3*6^
zg4l;6OghspE}5ZQwe_l}z4qk^(^P*6dCo4~C3d`AGStjp>3>Re-qt#s*MVEkRjKA3
zEL@Vhjw6=)^@-Uz3tJOgeOFwc!yu$mnD2dhU-q)LJDe=}-cMz(=sB{uN*}rO<Kg$s
zQ4gx(1a{oLc9d0As?dAU&WxpX6CScIHC5WJ>ss*s4fE6)fh=$NLhilg>zsa|li78G
z%jFc6pO0K;I?gGGm|3?saKbrj{*a{+yG6c!GC9i<JlSoL?8YenS!+!Fcek-FTO+km
zNAwgcqkG_)i<yd>FY0;EW_ZN)z<Xbm=MK@kB6VWw3o_3}zv1R!Q2fclD#F0Tpu@nx
zz|g?tz{tSypMjM_BqD3UK_fQyZC)A$3mMGU-kIX0v7yDGi|Ir{%88AO9fO!WCVsRi
z5I#IH#baw``s&2BWotb;BVDr<v>RVbtdrU?dCB?tvd0-kElw@Ju+U?&mamqdgJ0w0
zE}_&4lf>ZWMJyqrGfG!Ag!;}`nraoW=HQ%^=~Y*w%wDc4&1dboRTS*7x@_yKo2$ab
zm)x4j#`s&6Gh$()8K=0IoTTxG1BZA;f92+se|XZ;VQuw>V^iqt6LamkMfOH2XzI<m
zQuOddrm6Gw{;QMjSj;Lrel^8TIVbt%V-2(B>3N)yMoSM&+ZuH>>8Mw5Y`>|s+w-4S
z6Rs{UEzgtPVVfP^7dgA?zqrA^KL<X#9zCt9|KQcCd2c7q`f4e^>do&T)0fFwZ&>mF
z>wnQu(a77J1(zGyyjmAHtU0ihRr{?*inq3IYe;zDl7=hdg_5Zn4rz8%r`_*RnYKej
zTzTy=)*MOB3r8I`Y_t$?GqP$Ea9hVQVX>q2s})l^Cd8HmN;=NkAtE_@%R=!uriVwn
zBo_ND4=!~J`x)Gr6S;D!dwA-}xsHoVT(4<_zg<2xB=DyCm039(Up||jbE|yXbj!2h
z_iM}fw)!<k_dF8t(YYBsYo_zUmXL|ejTsXbWf^wO=-^S?vM`p-Ei2gleCnx-#e7pQ
z-pY+udwQ&gxz{RcB~S0IwuS2tzId4@r}$q>K>9ytijebmt_iQ*HA8Pq&^6CwlK1@Z
z_Q<IjR<-W2^=JA%$y7XdJw0{H8{h0)r)Q;GySClo3dsq--1u~D^({NooE0xV9T!;G
zvhMW6naAr&7v`H^Em71q$uwfq-pVmyPOR{$qg@`ePn-%YXq(8AIh9-eYQZkqsX~hx
zgsYFRiZaFWJa%1oFzi?X)09S~l}=nKTqpT?Rid0McU^MR)Nel{z`0s+$;2~18%6F&
zWKVQ4kkoNyyFB?^Rmr>Bh^Xhm)9q`Yn?y%)e*Lo5-8m;@YeJB#$mML-7t$d?ox*DF
zv2ov3JzLby>UgFpFMS!kW`l8C@mG#7L2>)m#b!9MN&mmXrQLH+Wk#}k;4cl$9zCC#
z_Z8JlZ*127@0;noXsXk(<!;w93f)#Jrp`WN{ww9f**#O0F1c6Do+f|dXrsuNmeX4o
zaYmjm+HzUz@tu8ZuBp^sUw2hD_4K`p*fPyMRl&^bZEksJ-qvZTQs&kVT>T?_Z`kKk
zKSgyH_c&g261lcnpy2h+*<U}sSbkqnQel0@g>I&U{vZ9<U9>prux7U#oA~_gZ<PE0
zKR%Jh^+>pDLZZt2-TaYdq5GN@GgbEO^47hvko7sk`JBS6$!v!Y%dI@RTr)}7b;dEZ
zLmzs4Z54a?*=IjG$g}CJw^y+6wH78P(NzU4*Sq4jPEi#NOj#Nmd^qv^{~gnW7j~rD
zMR+axC2@!^YQti&yUFc$KWtheD9PnAYnEtQLy&b_PFrxyohh%>)K~W3adBCc*!sxI
zS#x57pTgYeq?0yDT8wLQQnzIuelAeD`L4p5n)i=5Uh^28xc0L<=v3h1eO=ph^E11v
z>~o%IteR$*@-@fhq<7qh2KBvcRco_X9$w;K_kH)uD_V;GI>Y^6ZB=KykR<CjN5S^R
zMgFK;o8-TKaKGzzQJ?eXVKJ-ZR?ibca}+X9D0g{BBu)8gU>Vif@W!~~cT9-=S;;4L
zl8<j>Z;+Ij(E2Wf=VhR5b&KF_0nYTA#S!<XF-;1pdXmm`qnkm=WXsOLlas5uZ)vb>
z`G4N{<>&eTa?anlWTIxcQQhaBV5`KFOQQ06CO+Jy8qW1E4tJh%l@a-y$l<PWNW^XF
zV#&QL>J7Aly2VtN2&87Xh~7FN%cZdBV$({;uw4P4u6$X#!bkS<F|`Sg0=cyp?pYEr
zH%LtVSg+U@hpf0P$J13`7RmkXSQWDT7C)oQ*Olj9sV>M#<d6D!bng3{iwAg?NgO`<
zEWguxd74Pj^s^r8jJSjsxQA`ixMH|YK}j>k{n3xf=3bZ8*FDU37xI`CxU+G?`j3kp
zdM+!q8LX<QSK(<dX<bq&r#^A(g%+`8sq+FV-)=6tc!b|oIx4W^+m>a%x>@1YQC}SR
zw=6rNp5bn|`}+TDW!nxbudPp89(^s#iD94bT8Fyq3HOD*PuU>4G$&{$SIu6dX>V2>
z*PY$v9s6k0uAOV716D=dU3Avia$83G0t=V6zY1^qV@sA*y+6!(!bjMC-9&kJgR=cs
zuS`#RYU0x`7BrzXAw|)$sKdqOv+sr0H**>s*W~zY;+(hjN+T<Gyz>X8?HX@BANnnl
z*yd?r!)LF%@P?F}_vTF(gdT_R%+e_Air<y=V9}4Gntg^n@ij?vxQ-k#IBHm3QuA2q
z{Eg!dW*>V~d7jMN^kbR1EJtChjey+kAIp`><Ysz!JQZ5@)P8kKs_<kl{$<aet_<Nd
z4rN~y%Q)qxLh8Cz4y&TfSA2W?KVtSv@o;UC$ORj^jN`6cdSrLxxs7#LL+;kqOW(}U
zNL#f|G}VGv=K5@bgIcZ!B{Y2`T{h1+DrB{3qAX{U$HQ|@Mh42W1q3%lZR|Yj^>o_`
zwXVWU?wGLY*LJM6(+%&KeanX-{N$nyrp_gEUpIb;QkW^wyw-Jfb|Ulk>qqRqZJ2oX
zwU0sd%?r=E*3OlEn=)Db)`75Z8{2Qc&d$Dl`+*q4`subYh1$EdkH1~Js{HltzJ#AU
zeS6;NZk7&>{k2$r*Osh(>`mUyZ+_pGUC8cyWsX4g<z&%j#`ucphdTe)7_Z~;eBUwa
zWX6)5MQ#$sf(f?Q4fBjzX8-Z#h-E$@aB<C<4Qu}kF`P9@nphkqY~P@`l-XbXd@Dm1
z&%_C*B{pqJlG^>_#H`*z)<Byt3xww$oo!piYQDRK(PZvRzy4ootW00Ix@NzeS+VPi
z>Flp-vlqS&*uU%YLBDS+r`NtoJ6?6w^mWNL`?+rqq*~p1ZufPkG0Rc=@{NVrUU#x%
zE9d7oN?Z5cO0D{`NIt=B&y2;p(%#xlJu^YvLEpmZgHY_%m5Uj87G!<&?o@9Lbi7rZ
zqGW90W#~4uj{D0C;o_NXTjqUbm-d+Z)N10jJ#*@09Jq3d^iT9GJaJJ)>ZiZc^cNEh
zrfuX@Y<+NSh4vC2b`^!>``%GZ)=zsDHQ6YC?AyM+R!F@1)}eoE|8H!b%jxSA{U+*!
z)781_>b_+w3cXC<_3NR1y{^l{zu%7k|NC(N|38=G|NjWD|G-}FXWqbYqMrRlJ=cx;
zx62#8hBv%!FSQNJOEa(A7Z&_PJ@42x`9&LauO8-l{2=DF1HZpX?DeN<*QUm%ZOdm{
zn3n!5-{T>FhFL+@!PuLnI_1LtAHR631^Sp7N=O7rWQG>HoDj_Yp!xo)pyYIcS(ELh
zl?A>#$1@qod}l8D-7H|)B*4&S6yf2<sK&5VL2#LqpqQe7%7UU82V<EHQU}`Q-?B&^
z{*tgqNk}M7yy7BXN<n>|dTEVBBeP*?;Bmp{=>pHgJD!<!L{2DftEihep`-8rjgH<K
zoP8F#kqVt%Gdh?UI;VDY&brar|D$uBMAv)`hDe95c^+NMBDz*}bWPO|Sbd|btwS^^
zA$9T=_3BNjanC~Xwp9cl71Y-d*q;$PLA-~@vB%l4lJj8fK8?^KW!Hb3LXTBwIBXP9
z-fl3@Ttp;Y;N{b>Hnr+fRn4i%;`55CUmUL9XKo+$h>f3JxhYVk?qk94h2fkNwU3Hq
z#j&{l46py}-u3=?zvcFRhRk}Vg8D#%32c!QI4UP_&78nvDZu-4LS(}PLCXnCJti^<
zvWj+26px&^B4c7C1FOu+E(Omn<(Ctses-y^oG52GN!WAJx{iqkon4QgIm_KBIH#Jw
z^?ysORcHRrmV9Q#X3oI0^kYrN8KF$+Q?kEJ_L@0`>0|z_r&D}?PI2;VW-g!7d!1kG
ziv9yp6R}GX6=so~?Nu))`P}HR%iX|gw8)TYV+*rle2%4n9Q*X*Bdmpz(<Lw4%Uo2p
zJtQcUD6rF9=vhS@+mZy2g~kpaGnpnxnWXbiE*4)Sz{>h^!lPrIGcr1#m33NP?_ibW
zoG`Ov?#Y?6C1=jb>|%X6bAsh84$hhEoU?dN&SGCVi)ZF6KF!(Uk~1e-PGIF^-M(}7
zW=)1&nlpE33hd9Ez{FS|IlWYKS)zPJw?}Kp*9F}{Pt)43cTY_h=vS}2s3Cn~hQMu2
z>8mHF-02j#`+uf4(*s*!we-iPkzR%ADhD;sE43^WpI=vOHRWU2S4+uU*ZJoSCF@nH
z&Zrkn46OFdP<=8(;7*J0yUT4q9A~f#7R@&l;&8T`S?;yW(4fe$T4tlv1^3A44U$*c
zT@QH9QCcOk{zR8r)FS?!6Sy>4**jSR7tGd^TC6X{Y8172pXXwxfW;PFES9$x+x!Bp
zgl3jpY`ba^-^&IKF*Vbh3KvvkEj{yZrshA<Y<AG>@$@VRjaq8pA#~qUDQcBiyp@vK
z3;yFwZX%lmmRyjS#xhN@aiMEOp}}Nrt)@QaM#HjM0^T#0ms+(l6)a%d&=Q(q_}f@W
zdvdGYC&>rSnjKticK?M0Sp(Tzg4(o>n`<>WxG}YyW%pR&++KZAU+4;74#Uj-tdj*J
zyCz7^=8T*zF1biWkWuF0DjvqwoQkUj8(7v!F$O%Cy<^vEn^~(p7#R1@TD@=8>b+cR
zc5tn+<XUs;)MB>6Re=JX@#ZC`Ug|WrcYmEAt}ZdRJALUsiOMO*^Y6?`yB9SjZCTvS
zpTe=Ldcu}gFfA6S_~@;oq<yea;<|=a=S8lJs}`A0EAMdFX-!<#A|}*T<R;!|S-51u
z0q0hOO-d^QRohLAI-W}H5%ky?7~cAD+00utwh|KGG6Xg-3!OP7Ad@K|6FKY1%+)HH
zn^sp%RFvMt^l%gN#Z89Rn|TyB>;KoD<EglYd1k+8N|e_tohUO$zgr<eQCkAN=J|52
zJ0QHw`%5>&^erj}x5zGB8n$xF&8_}XmCIsWvm_6(Npp)xXir=HUHDFgrplz=O@RU#
zn#y;UESEQr_c~}*rzLRMP)5;R;81vxin+kL1zJoOik2LzPAO<(o47qoaND*+Qpeo6
zVxF46J-{_dVf8wRMKd@IHC_vv&0c)`)SA<?*6i0}5I7*Pt8&#;jkTLOD$a(k^=1v-
zRgvHReAjLb2BU~j2hPw#H<qeink%z#cmA#2uWxbRE%wki^nbW?o~NKnPoY)hlHPpP
z$a^Qax~r<zsqZatb(<Qr_vCZoiAUofM=ZGcW$*0&$^y@RgxeL`M{i)<_r03$^n#Yh
z0`GqaPYf05xG8x4_;SgM3z-8IPcd$CHrymzINRd1z|NbCr)R4kidywuJm=S?Ml(<0
zPi$%K*;~G;?T(tXl=Zn_eAL`i9lJ$zw#s~5D%T^gSaUGUso(}{4EIEtqGjTuRaVER
z)F?E$?%N%gTTm$Sxv1Q0-|Wp&K^`7#x0W}k8c8@h7;TnZ_<6-OZGpv1D>)o@Fc-G9
zyxzM>FtX5M{}Dx@p6IlOOET-SRxv-^?|E=Glg><ro=xkdb}ih!X~)f(Cp6dIQ?A%p
zvUb;xU3)Cj4riq4MXYnaxb*p|<8?kl&u`_2%<Or5Ywfu!$Gsm`-2A^)|GA{bJ8kK5
zv8t>kZZe98o0qwEeAKw`TF~&1V1Lp!yT%nA$x`zRg=an#f2<L~k!>1%fUz-aACL6{
z)kFKjJu=#6i!45nDR#YPi9n|8?EQZZaYZ+<G<a-^_Yq?7nK=FSYUYKT&;FVyIJ2I!
zC#iU|f9ffH)f?O{(g$rKx7_ENC&M!@c((jQpF<)xB4Rqy$~h{p_o$}3?cNz;5;<FD
zpMgQ<EBVCVhwLx9W>{}ibDE}HR9I{!lJ9BnVYuDr#PafN@nud&G&gT&x~S|D!<MeG
z@~O(v_UxLpM+qNpYk#}oU3TJVqs0Wv_A~a1v*yq2@SZ-$y50J{pn2ir)L6CS)Bl@?
zuG0>E92t7O^Z1RE0(-P8PX(U~+@0nid2X*tvq2*NzJ|1yHUjp0<D6ofGs;5q-NXzw
zHvc^%_+h8!lfz*Oo36h6;@ZS!ae9iydh_{pX)SDw#@pKU9={gO*PR}GfU!*b^v^)|
zUFCkdi!|ykUNsDld+)>*c1GaqZ;9v+l2^nP589jwNna!zSjsZ9K5D{}&4&;CI3(i~
zU7@aVQ1H${yTn7=Pp^I8xpZFcK?Rv+@0C-0_b!$1Ik+-c;PgfPe;+O`IWQw2``nVZ
z2Yq(mUVBzxPwnm1cW*zt(-bz<`^^KL6ANY1cL{vEvn*UjMQqmTny~dVj1Fhco*`>&
z|76Btxx(dl{;TwFYE<H$d(CN*^5P<+hzNIyL_xnb*GqP}>~B7r_Dow}(Rxv3X>Lx*
zq6fD5b9S{%@4gvzysW1ux9oV>>f`pc6^mlmUF|$!SD0q)dnaa>2-i9Bw-XN9?z~d@
zko(&Q{<i^K?+k9s=rQzgt}kA8W!v5d4S&wwnYGTtvEZP@<2Ii=J?iJrTilP7)vo=f
zxOd`_w=2V<K1EocyOSa!{D9f6^V)_Ux9i`mTlbi=zK}lMY57#`&^~_-KQBjZ!}cA)
z7n?;G{f`U!A3S4x?Trqfjbnyg{+!$`dVGd~dsUTfc0Z3%4dDu9*nQhsQ)9^ku5yms
znX^1*Up(ij_(HV2IcVY3kN;VO&pi@vvK9MPb27=gH)+bX)NVai#S52Grq@TQ&YCAu
z{A$4@xr;T{0>TH6)cxEmVrUfFwDS4lz1`lA6m%W$9^Q3hRmBGBx$AcCE}mzXJ5MjF
zYVE(GwZ?x#jyrMRkE~RibI@*Oe%LMkeF}C@FYI{mtD@-=pYlDfr2$+*Qv~L)Pigq{
zW@q+Y*@c&nojsu!==#5MkHbFIscwZ+|GkUj6sah!HWa$<koZ1p;f9-6uC3P5dQ+|Z
zD_ArBtUzhmDRHYCoPqmSPpph)aQs-ZOt?fx*yn}w*;|j+Jd(S(bV=>)Ew6=l&Ak<x
zb?B_B)2~UTuhJ817S6W4cxAt>fKTYsuMTTB|9{ILzW3vS8TnZu{Qp0A-_m3F|KM{E
zr;x1Omjhb_cD{Wr_y3kx{cW-NAGLOHo@a~`Ns`KVb?08i(c~ox*Q1}bsVOD6+!Ixg
z5EgE&n04Wt%01<W7haSI^khkU$FPY_tbWF^Lw8eOa89Ob;0(=+Tsm)7R&4ZNqus;$
zuW8qhH!^iU_P0EGQ1|h`k4KI(<&Es#$pxARy7EVTSbfa0^!HxAzX$ks+!FTCKCX~!
zeowV0c>-thg~@+6aGlS3^X~!wx&W@N3eDTJ1tve|{&zt_Lrz$?@wcl!gLiIP-F<Jq
zeX99!>sss|%RWpye=efDP&}GD%)Zd|W1MD(kcIU+*ZOULo&Vpn*!2Bm>+Koy8!s+A
zWcA?glj6DiUS@pno+^EFwS%0~4Y`9333gvkBq}Tl_`|vJ%Dw<@hQ!rV*GBDQk#U(3
z@X)D^SJtjYLU7Uks=p?Vork(&_g0D6URx}x+O?wPt6T4>C92&LzH6-dP6@S6W2pa|
z<>;uiIeBlW$CeAi7cRzXm+_SU$_`(3*S`8+)YdHZ1uJ(4EzR4y`s%7M>uPpuFBa)(
z3+*?iDu1iCWJ?TVWRBjbcvLGdGKeXn*XZ|#(0h9;|H?f%qN*sa7krA#uH%NHXiKz!
zRNRXS#gx`Q*<#Jd8+s4#V~W`+$gr!W;S9T{iAc}REB8Ap#Pggub}MQcE%_gHb9Jjg
zmTp+qNx46NC**`jm&HiB*WH<>TYaXMb$$DW`28F9N(9As#4ap*z_jN7v3uIT9N%5K
zcx_R8S&x5}Y4*2Q;hW|6>UbQPuzabemx}J9V=vm1rc`BJWMT5n>g<>0D6%Tp#pF8k
z-1Mp(&F9-5@cru3xy|xn=}z9i7XvgIZxy~f)Vd~lt>U|5!5v3MPAm5-e$2{pog8-T
z_Nr;m1d>$sjs7JyFO)whWwdY?->Z)cl%qX2O;vttwZgc=Dws!+?~{^_VcWVI4#T6X
za&nlpgHKPIEFzq>_(X?6w)e^AE}2iBtq+niM1o=_pD?uayt&Auk#AM2Y?#Z*vqvN(
zH>v(td~(O&UEgE2RjL;^FA+}IS9NJ|=cRWijJ~*5>4kR84Ru?XDq^_kf|CQ|qK2ep
zts1Y?g*7%cte>Z|K%imTT%T3`2PQ0%Xk8Te<}^!0{fmoCohsIwj(fJxNDW$5-<I@L
zJ}p=AsA%;X*3~wChd-GGKNR44cRSx)G<?y670%s@3_o1ST2K{r(?yTb?dQhh3h!Nm
z8Z1s8n5?L|@Qt~boTS$$pH@Mm%lBJZjC~A`9SAyOcuc~$XH)CeCy$Pw<%@c|pkGuf
zVxiFqiMI-RtPzn62Ig)n^elg_)qc=gx8|*xC+mrg<_q|IIre!8-imCn<|{hfYjnoo
zac}rN3qQ+eM-ET;FB`=Yn|{U6CDF5c%C$lcH<8RoeQToFmb8miopa+*2{t~dq#C4c
z%DLEM(}k9$p|28IIXPS|zR;WA<mAnxvMD29R%1a#YtD~flB`WVuOj`s8BGNpE@*7p
zU^PkOM&Ww-=v#?L<v2Aby|b+Lcs^mxa-a6Un$H9d^}34t_q_P_R;bBPV?k}a$M=Qu
zUFTfiW_@Cfzq?`9h9}0&J{F5=MZ4^dPUN3ex3D(y=IZ19(LW=MW+-P>8ZaJ7uRPsj
zd828A5o?Bp>xFgdy$dhh5J^a5++=U`%Qji<=)RQ26Br-4nk{|EB5C$eol&LB+|)@R
zqKi%Hg2ED>SOaFZ(+sB8TMqr_TQecVHEdyP<XJ9Gi<|w|BZO?<Oj#W?W7Scuh=W{T
zR9Zx5U0_-jFtt-IutjvXbZ5Pbk!5|45?6aPYt*f+UCNuhR17apbq~9`s_u)5a_(Xd
zT?3`AWYuiZHzyw6Y*Xb@c2jJ+uH<^r>k)tTjwX?B3Dcx=3#YK%YIAv$5VY`Ex!RSB
zDK@nw3k&|u(9}$tm>(^1LuTXD{jW0LE12DWJcUKfvyJ7Mf>g1jQ=D2ur{<E$@l72+
zG@4H~FX&gCRv4KnU}O|#^~6{$_{6ClEfS$_m(sScc{1B<;pUcvR@r2RyAf(C4bdwM
zP4WsHGh!~b1x;BvGqzxv@!bF}vn`Ko&HhUs-5s&9&rrutc-JDoyrv@4-W^<ut54j%
zog~!i)3eZa!r>Ep-3+x1HzZrxN_mPn7EScn@ua<eHebOXK_M0nhuKX^{527btQ;)a
zvH}f<Hr+Ygr`)$t`L)2wl{)2XqBp*>n<~`5)kARVx+zo2_9?{L`kCv$Gvs$smI}<v
zddMIDB`-OdTk3towRIaOoL<iK_Cj$})FnPc^}uZftU+_Pw5o1p);g1TSYy-k#UI~#
z``H)CeX~$D_-nXC<V*31)2G}Gbt65EVho;&rJda86Bp#8=dAtbUb)&*SvBLWU#`b|
zIxY}&MRnEpQ{fE1wZv?Buf=oQTngOU<nH9?60~lE{QpZ)h22YLIWb%~lAw8q!O8y6
zD~&fVuKEcWaRsI*wi^l_)>xJ-IOjpGsL>UEA-9S1d>j;ne*N)SvtUA}SM3t9i8pc=
zRJoq$V>oK}ws4}mMpw(;t4=DjEadJN39ac$&{L18>10^e?OJ`~a32$=NIp|2|K+KD
zks^|^GPajQzU|YB6?_@8ysF%0{;ci^-+Mb8H*Jc4ce70-cLA^Di3=O;JQwQm$#8U5
ziHP57c<K@B>k%f%Hqk9;lj5{BXOcyj3>gzXCC$p=>F~apDCDo^x-#=boWg8@3u|`x
z&WiPw44G-fI4A9J)~&^7emq_L-)zJ5My6%UE?ioe+4PK;-H>_G#Q!njH#{}8&n3w{
zWQtn(@`{o$hlRk@Vqw<ib&K7T4s>Mw-YCcMLC4Naur-&xed08c)>9%Y`z4MU%a%WF
zkqt8oU<tp??W>?5{OC#S)Wrb-AA6L}q&s`}D|~o<_0pqLJ<+<2D%X|Ej;6;|^mu)2
zDl*#?uvY#<A-~g_!|N6<WOY_$6n824F3Wdxd8yOVmjz3drpCWHvBL8&&yQI<Ur(Oe
z$gTRLbVK7^M!k{++{e@y#nm3n*lWY8z2l}t-&v!pO`2ya&vki!byAz?8?|FCcVf2F
zG6gk0pT^pE)sk_igTgy`cJS~r8^oX8^k5yIf#QaVXHSSk%x;#M+GfIXC&K6Yng2#t
zLZ`BqAC8-KXib#-6Z!YC%A5Lb>nXc^+1z&8Nl5z~m-qD+vCCiCt$Qj~>#T7Rd5|Jn
zvX@CaaD!Lw$zPp{?MqqBw|6KR=&xJ&xNMS9T9TT4Vv=pGHCNc4$p!9_7Se(0HXS!i
za=2D=7DU}}YF!|*=g#Z(lW$FMVA{Pqd4^~2`uAP?f61o2Nu9wItSYMCxFKn}dzbLm
zui?|VUZ*Atg-NuX++`4$RdHm^&rAIwD<5sLPBC5U@yRixN5J%^#G%K<y-`WVKd-;|
z<mUSMsYJv^z4r#(2U1s@4v$`FB3!YMC9U9a`n2Fn-={>mMd?jAzg$>${(@f7#{!4%
zMjaB?aQV-WGuQr_V6)Rihlyd_6F0oodeK-|z3yAYnfLCq{=b>}G9vxbos}hDa=R5i
z-Z4-s+&$ew`_oIttB<XpOn<-prfJqEi&&+Hm9<VW83F>OEB-B8tof!V>-DoOheUU!
zsY!OLopG3%@VFt#dRx4;8KXucYt8|_gafR5*0fhR9GcOjRN1w8W9u&A?&FJ9GKFWj
zb%`}aas-{|dF{Ay|8Jp<NgcjYt0NZmY*tcjTfn1d;ULg0e=Wg0FL>#MncM<*Hoj?G
zvSQcd04~EIjaK%>y+@QL9$m${h+D4TV^vT=YmkH1sZ$Ghnr2ylQ1CG1m>V>4?h2U;
z4?8p_%n21}eSDe8w_@%8hle#%6{j1SOenMLI-RH<C87AcU~_V0tFBXb;>o7_(sui1
zieBF#a8Zc$>FKrBw>x|e@>xx?@30nrtHJ-vXkzh1m-Yj^A1-nIFA`E;*~=(=bp7XD
zo0WOZc51V^bR7N7z43Na>g@w{lcz^D^fCkq?c5-)Q8`KJ;Q@|p;pZ=1g}#`zUa)Ro
z;h3DlaqPuzZ=sDIyLhH+PMTO~m>JWdq}&^|X=D3I`O;=d+1=YOPBK4so5f6FqI@>{
zS?PHanbYeEjHYgCkJe!2f59Z|V9QuA>q>Li#!8pP5xR^H4LvjVrbL<;GR)Z;xwbk<
z<LaV)RR=5?8M^vr*gF57Qy0NL>2k~ewHKR06{oyh;UTxE@8jdcJ5ToAOy1vYb7Gt2
z);AYK-ZxDAd8zNCgml>A{o$wWBdYmkX{>sd(f9PkR<jFRGnASqGn&O)J0uyOO5!%-
z$T=|OWz*7}Q_FJ>aK2$`n{kR`%Bj;`UW^9YnhmzCb8#^8*uAfzYk3!+hY@$&90T{L
z4zAVRHJ;vF(WhVB+?MXc5iE6#C;OOgLr-zGJmU?96CS<a3gv$&`a}pxc6uBwOBUX9
zdv(hX?N^$P9TE!F-P@ZT)(Czyzqez0%fcg8G5Yldt=y-ac!Q+OJ(M@z=$v=LqJIa+
zGb5|@5_T6R>Arl?(QTkyCeZ4mpliaiS*Kw_m%!BjyDu)Rj5@9rB>nR@m!nbFeaZQL
znSwtT?|S!wuX)EV_D}i@!W#lrn(Y{^*LyGhC1}kZ<r*4d^C`qN?8~-=TO2iCaKFo3
zP{cArY0c4=hYq`pPw!9gk)EZ}tl6C;!X045y?e_Bt%KTmIjZ8>tdhYTrVl5{zm#BP
zSZ!-`nyIRzw)(ur#jfH#{$4D0^$qU)AD9G88<*)Y&wry>Ftg9DWy{M2tPiX#cdcaA
zm$uhFY{b)eB5#pp)B`Es6HIr7n@cO^7;}qkW!&<p=Hx5p!(t2hYFq3}zx1hWo!h1{
zJz}^0oFn#8mpvCtb;Nk@$jDl@K!^FO%C<yruhljjoHpB*yL9|t_2ks57Ke2?9A>^8
zmaQFm#T_hPOgSe^zMC;^cZtN`lG9nM4bl~7c0^2S^N<MoFv*lL;7UZt4@2QyJR5iM
zY!|QT;k1z7-?Dngp$#%V9C;}nvMn9IAG-Z&@jmX-p`hus;f1iD!J0o0x3d*-pKUqh
zut+5Mbw~6H1xAbJr6<=k&p7E}GG*=y>109O9i2?m95~k(b|xDxy2;encR<Q&4ZEAk
z)~TP*{Hof&p};Ti?@FH-j=x>g)pz^dQaJjK#d@Rj(O(NUmdT!PbvQ55dA(WKf5Qq@
zJ;~mMQ<#iAgtirMtE`-PY>k0wAur#Xp2m~@_Pvq{U%Dkl<PR3~NbWf;64U!XGiGV`
z+n#W(JyMS&F2A_J&U&@=;>MFa-Ku{k&b`Td>EL7y+bJ9`*(*%Us!EuRKX^Wyz%TNa
zy{n0-J4k7^0K+r^hBJ!0KF;ocd}iNOrCXC*1bvftmE2j`E+94EN$~UUuICEev-bo|
zK6QyLi!c1g{+8Kl`fs-~sa)QqyUkda+bWi0Yi;n_m;>AHo?2NFEaJ?=Q4?&L8*Onm
zdaccEF&~cITOIX&oIaajHcxVy?JPNqM4JwdNykb%S{9mh?wA*J<6Qd;ReM*}!#A4t
z+sxNqDRIDs!|-6pzB5ABu0CSiH`@gbHa7=s)QGq~W8U71tH0;$saef^++)EmjpM5)
zs~5cN_+MSX$}{WKvW-rcHMw6c;FuiIdjIp9<egJ~ewf?k;n%9b&6n!;(#Jh|>ZN?q
z?mXX)ix2hH3MZ-5m@|If8su}m)Yqr1HvF^B`R}FityAN<j`V2GXfSy+FHM<)amCRY
zo<e)8I)3vUtFyeZ!Q)0&cE_n+9;q+4+XVtHf9>wdJmwP{lXpa0oV)jef<;))g`>3r
zKXVP*Hkh|A;C<>CxhCM|OsVOgce5VJK6Sx>b$(69Jx9S$tG8@(yrt>v*?M5*)Hi(v
zFCY4Oiz~8)J@|2G)*8=6kL;H?X>DM2Xew4S3_W1zdw_k)?HzL+wp@LrbMJQAn&_>6
zw{?CEHYhvrRfTzDPuG8zXo)RXR++V(saHHG$9cuflVeTRhQ^7z6+HuP++f<Bb0xoe
zrp`?R@fHrf&Vwf|NcgCl@7Q&4)1>)V{)Sw7&~lbxgG=q{n;kR%o|LzdeAs$mx^Kl*
zyNSGwKZK>MI={WpoPB$u&J~?=x9<IjvXr;G_gX?K`fJ=PUG7&`y6S%)`S0Y!Xnj2}
z<a&ro+UJt+AEw<Exg4o^_mjgqDtWoN?IusxWaJAt&+DKv@8YwI8(H^M2+1ZJ6o|O_
zu9FL1JC8{(=HHTJuT2iEmTn<J=3IJ)GZ-gNZ+zakGUEIN-def6>vtWyc;Z;jmmBu7
z$7U`JaOOSQc;T6b=fh7DOTI~Os#NU#zw1Jmy6?o>7au-cC^hwuhxAc_aBEqCgZ2x9
zl2)EL`8WH~#n+y3lQ%99F<WBem2f)z;DqRvF*XM0xOcd6>_`nZH=S=>dwJJYuQraX
zxqOco{EK$Kr($w8+9<9=uPl1+TZ!Ly&C+KtKAyrmCo*R3wX7;rYvC!9Ue}(;7si~}
zo1)=4!^~sSj#a1ggrDpV(N4Q^rL|)9PB8`_-=2dll3Z4?o24gR?YLm&7utSfa_z!c
zbI+^04w$o^;yt>F`<_Ir?VoGTG3(<!v>1P|+LzkL9J}}MVjSN`L%%nv0W4|jFZ*rC
zUY(S7_iEm=k15YgdAmzy-@oa1KH)Kw<_FK2N!F?pWB>2oAnYrau2iGa_0N?}`fZV~
z6N7i2%9kV0{v9c`6W%cAdD{#}j&-LwpY661o!?&Dai(F-#E<WORtnc!^6uGW7!?$e
zZ{vN-V}r{{hkccSqBr@zBp+Yayla*5qlwqvpDF88J+`&w^GSw3L6Y8Pi$V@C>qN8r
zTwb1bS%06-V!14f+E=CzPuU-PxA!;q>t$~_pBF56cAI%lgUlW7XBn-B*99E8cTAOK
z@7}0GxgkPxGB*TFnR%;YyBzl#OSya}(PL)=6c~Lw3T|)UcJKKS$m8Gpd{>S3@3U8S
z9m{UFFyAZpX}>_IT+ZZEIubX0Ij$erxNFC}hSCQ&0u~(GVzcO_+&zv>|2rMFF7tgl
zZ&PaLf^!}RyJGF~-tauL;y<5sOgp{o8Tb7U>1iF;^|&40f97cq4;8W2Owe-Z6lzRp
zjt!c0@RjxkIUQrh=kD$b>--cBBuE^Td+$^Bj@L5bccy%&E_cxL4T&W^yPiqp&Y6)L
z{JiE|&pwXZ2CkCr7Ww_2?`4%0I%a%~{Ns8llhYtZ@Y9aDk2bEKDYGgi-zf6-t!VE|
zhV++{H9t(33!Y(jd(9iKEuojc>7Jk6_I5>B$FgZ37H+$3`LuYETlrq!@}mc$^S*yD
z@O!oRUgpwgk96arTUTV}uK&(`|5PRW9p*a>ng>=dUv=G!J-u%Cs<qz|+5=V1WQ9Cb
zng4saS1|rqC0Bf>yZ-cAZw^Do$&pSx^7kbIg@l8%J~eCn6qE7cW$)IEpM1$;(nL<T
zhi2I-2bZ=l@HzS{o>i(avi8lr44Fp<7b(umY5#GEv-7k1X{-2ie(&y_yZmiyh?DEr
zub-WcpSpJK^R;(n_UErVKL52Ye8$@OFF)UGi2bxbsG{qCHmh-9wMFKPj(Xvf7hd_<
z3wx+0GDyEQlf2>{`g8y1Ut6+xG%IFkt`G`XrNC-?rd032Y{?hBXTKa+wj}@A{S6Td
zc&_%YE!cgQJM)+GBfgU&f3wS?A74@zbC6`-ou%)!L}nYup4(UbzZbA`FT0mHU+#Oy
zj@$X?lXN~N{k;48|Nk$)Zu`$~k?1|$8lm@NRhp3__pdaW>K{+pc|xlyST=oce!<c*
z<6p~yYTx`q)tfyEH@V-jzpL$A-7%HJFGJF(FvMN#Lh2Xm;{i`R_hzfNYgb%9c=>D(
zV{rODqtG>S>C#3o0+=|AX0uF4G7e}s5XQu^Wz*ITp}JwoZa<?n7A$gQ+^2IZE1g@(
zC2XxnZ;taSo|#5-P829^KC0?HEp^A4HJS&~X3l2vdU57N@x$|TCEJB`v%Y9tTHw*h
zrF-z`%8PTnm+MXS`uZwJ&($))bk)~a;j0q+wa$86R+8)vvW!f*{Z;qQ%B3DNWmbMz
zcQJ@*Tfxkx(r(xNE{x25+|vyL|7!{x`Mtbdt|7`ek#$GW<_%M%_j0X$mXlh!FvQ5l
zljA{DSVP#}w(w{fi>w#fi<WRq%8Xbq=G^u+s#odp#jEb-Yi~_aidz;kbF${n2^FVW
znWyueU+n)rV2iUjW2{%s`*nY-zrE#NEG(;icV+1nv6HLy^Z(W!`*36H`S(Wg4z(X{
z?7V(Ie}8rT&Ap-_RX=2>&THTn?FemhUTS_VfOpe2g+N{*jfKtEr<Mte%dTR7nByF|
zH6ThwspXne9`916Ztqg1hOVl;D{cjDUTY#9%d}<TVuqYAKa%s72&hE_US50rRzOGO
zk;eGew^J`BwUxe5oFqB*;js=E&!tx*t>0P9?*A{=YH?-8#?&KSu8B-P#iTbMeKyI7
zulC3U-8B;wS$%Z(E=l*%?VT#Tz>0O&lmMT(VIh%oL`_#Lur<{Rm5h|SC9-s~Rq4xM
zU(0q)|Msw-nkyD|d97H!+OulPq6G(8ABL>i<|^P=HgSsd%oLxEtP_L8rmlANt6LR1
z*~9c&gxln;=@-+zBgHb8xHxWo;*{LC>z1SGq{e0L{G1p3q-XmC+%0C)%18)!+GQ=;
zcE5MJV3T{_anY;~yMBc%Iq_5^PjX&jSY&G0&FvFv{r?`H@5(Vx+~D9J&K%i;9?PtC
zSFG#vTIkC8@Ac(?PjRltysW0~Sh2hBnEBMMBHpi$oHP#qc-8rTOY4j~ef+%b7u2Lb
zREdc1Q##ss?seLeCt}h%Cao7-bi=kV1WKv02Ktn7+;s}rJWVGk`q`GnPA&^qehrFF
zcj3}Z^V{0#kZ|0m_!^VNkpNvT!?zhL108EZS1C2A&5@p}=pHP&?o0PlY5w9}naiH$
z?YjJM1#jNEX<3$b=1&5}CAIcQPcjX;Gts&#D@HZ=t3golRo6>vm~xapp7$4WRY`EU
zr=ykB<Y{>Hw%fyBHv-p1OxgDCR#NWGK8;>8k<2AqkMDf8cCRa!<dQ2%{<?2Z*j!3V
zEv*W9>XR9haq#KX*_Zc!h^$xLKQlM;>Ao|s`tJBvv7ZtxV?4J1a{cG*<8u%F|8tc;
z;V*YHpF!A}=BdtqRbmqjA3QNRBJx3}K!<gW1dHJj;dLQ-YVIHJ1lpB3b|zkkjV-v^
z({<_K>4+9LP3?!LW|S*Oyj*IrLX+pxnMYH1e9<wODjfCgL6G$^pHA=NMxhLW#ip{o
znl8$b9kb4{tnS&6D6;uxlw+LownZV@QORB&fp$g9w<xSHT*W0K@pjoN!+T43$}_~$
zGZr12ednM^!{IYGjhx!{sGc^;%kY$Z^I)cStf%_3M<S2(Ts0kpiY9b=%P9#*E<H1G
zqgUzW1g3+VW{Le>uY1<`WK7gT@qH3(1_!LVCdaieiP3c9^eF7n*t=s&*aDZ7xXey%
zQ*ovL9$NlpNm=){T~lj#5tz??Z1JgYeTUanJ#~q5oG8w^-|0B7Ay49!NVSYF+YYpb
zuqj5JRBm@WqR%*GQS2A%2;<+M&$jj&*Z)6tf0an2=V6nMq=iWl$1I*0E9B{Pq-9>@
z3_fypaf`uxZ^?zHcT}!lz_~2LF?#ysLqgM}&IK?0VD-Xj$~LKGhG(brEObcj36Z+u
z%j#VseO_R{+Omnq3Ol_sg|&}`99^_0O`)VP_@+hD#BEz0xvsm2o={>vTQ~7z$efdl
zwG~>I&ivz|y>3Tq+`c6CyMH^mE1h}0Z!OsEG52WG6YpzZIVvp<b8ZT5E#?d0J1OP1
z?bJNyhZ;)79incL|2b3EG^+gIX|~yN&Ooo&QT5%lMVoK9@;TMbJiw>BsD(2kBr{k-
z?A4wrv(J9e$ep3Nc=IF9P&rMbDA$S8wt5uJE_=|XxHqRc{X~FGx6cKKsk_rsqb^R`
z!`fea{fLidMbVr+Ph0w`6z5BBn>G2AkGod2a&PdjZC7R%Ea=ZI(D2<RIlthYfBnbD
z_lxItZ}`RdA?A+YRp#kIA&UciY+gxS=igl|^^HSd&y40Um5D){)-!BL)kunr*|KO;
z&x{Sa{}*5E4q0aOXWuK=oS*txMc2gD-nM$~Fw!`0$8u`Zt}bn%tBGzZ34x4vx*}{{
zgsw)pwh6slx<px&L3>U{q`>4u+5cTKA2~VIOvwt}SQ*#s8F=d&*X(JJl<i9IEO-`i
z_VEs<_>w7HCcUesMSt<LHjPX!dcR@&OdHXYIwxmq+ily&oBphC;-}fE|8`u}EuUHt
z%ljluTl<-z|K{Z$t8Ry6SW514)3bgXdoi+~#ryb&)QG2^cN3VHwc<9en&Le3!z-EQ
zsY>ZfW<|2`7RYbcn{~y!(!yx2=lVl7lTuDA?bbZXwv8d`^VM#XLpfqj5#GKJbz1sb
z^d6hdUo6G_joa!%VOQoZhOJ@JruheToeRD{y?A`ntv;q7g^f*b_qXx-I6Ql^&br<s
zO)_kS2FDZ?YsHQe{R@BYeU&T9@i1ES=Q_9jVuAl})>og{y!>Y>S9fKUlk_fuwpWq1
z{B;E}OB50rBSM&-om+A2#vI3u92b?WjS8=CU~-+ju0&DCEH*01&1}=jD~d+b4vA&m
z>5_AnJhAulTHSALscz?A>eL1E=P%}xNfVrzl{H28eus{Wu~F#qn|oIMKDTJ{?Pt?g
zJdcf8y5qRT+-&i{;>+KoMWm)*7V0uhe3_)PQYd`Mn#ng8S8d)UU8%$W&~$1=+uV6m
z#SZr{L@nat=vnae+tH(ZPq#&$%JMSlD2a61wKOF0o<>w)f1k_w&1*Nz3~Al0e0x&j
zRk^AxFPp<5r*<@|Zr#U|8@)(8TxNUz%!geDb3f1XvHf~|-HF`7|D5q{pLssVHQAgG
zchH~9VPsIl#P#i^`?Ceh=5y<@tkK(j;<4Wk1=X7o3nkpIMuqK2m@qAL(#}Ae>Gutb
zoE+vph_PoixbL~HN7(ku1(6vo&Sm0i?NJLK1Z=YlT<j6S#Ps7Ai-7dZ6F)X+ORkgh
zSZncU3+DkJCAW_OeHKYS+9RS&a|$~zI++EX=u}>tadhp6`F%e)rxY?~u!c(JIPSZ{
z)GqR%@CH-xQWlnbfkzj*+*1hkyW?njw?#JLWH^iXeTKf!HG&><=Uo1I=iJ`76kTcV
zZ3&BI_}w!D5`WC=Tp+(|&CDnX?#r6<H+6PhC~fpQar2bUtR_xbClj~1fj4aa{*OD6
zaFkC_EXbpQsX|^$kySgktvyg|-CXfKIf>ONinkR#KW}kb`u0-Z7N&Jix>Ft6rS2Ws
zCv$y`=sc-ce3DP*X8n;6f8)J!>8!_hy(M$+9cmD@yx4wV1NVV5az}2=d#2K{dP5xl
zzE)+Xxz%@OawoY~uVv->d6|>*zCaXb^qvltv`4r1c768coNU0kr_?pEr8P0O^=f5&
zi{`=WmGM7#ZwT#cbx&mVPGOOlbTTMmn*J`%&@CZ{&R+WW^pelt`(k^JxSPm{*U0#q
zw8ULWblxy2IMmIHA+$d;&iBj3fRz(jr?#Js^`F@}lPk_osnAcu%(u#QlHk79cMs2m
z2XJoD{6A~a9IjVAN3HDUa&#)4JQ9AjlPmm%&(j=6*Nf3pJ#M@e@P45&YyV%x%v_J+
zKSAv(cQz$X6UjZZ#$wKwKl8IZ8y~Obe>l~tc*ZQ(FWj6H{Czk0FVQ&j;vswKo?F&(
z0?d*en<QJ6%g!&EA@2W5rS6GJ%nsqm#2&?2;!13@{rOT~Zw<CNs=T5^puzF(a+6z!
zceL(b!Mc6xg5C+yKU(KM`QfuJW9}=Jvv(M7UDrJlc`y2BW-DLa6bZxIJ{n?i9h?WY
z$_q70tnHc6^|j;o&82#i_&UF^8Pu^z<uOS9=#~95YgL1?eB-T&w#y=IPWwFy;c|Pz
zHBbBqL+b2_k-I`J*<Jge99DDc_=N@4J6l|R-2A+j#>`}Vvoq3q-h!>3KJhYQv#$n3
zn0UVBVB77e7<yolYlVES(G#uOw)740I!j$`b_vWpc-N`tpi{~c-G#1wFN7rj-50y*
z&1rD@i7d~b8AqEXcn<KWR^ELQSm9pbar?@`<>wh&SuE$K9gEyt=wGRKw7P8B=DX@|
z9;Un%WVH)bs$shkwNPF8k$TXZaQnEUpJH3PeuT%Dr0$dynZm(viGerfgI5A?Yra%#
zv)x%;rK3L=cFoU`(APVks_5Fu__#|^M$qw=VOsYblOt~@a6}|7;|@zZ+VCLt-JPs0
z>M?h2glarpzbCD#>_vUj`3o|g>c0Q^R`)#XTp${4n%QEb#PIig`oGKDYu)S9SfbCI
zUal~0u5QFG*D0<ZS6Y*fMhdm&dr8Diz5DJ(i<8d1^c=;^g!rcuj_K`sSagWXE$@ul
z8xQw4eWhNE-way=iu@a8E(g`|uQ=#k7IXDz3BSeM;EGx5)qe9fFNrf=neIPfam_E4
zz_KjMwpTkf9yK~-t&~bVZ^^^GFLG*_YgdWKv^nXeg$EC-Xw04z;3km~Z>n+e%c=_+
zzFI-7>pye}rDO!oS$Xk}bU4S1pHEpbzp+@@J!!A)j63AzbyxMk&l5RIe7zcZMXyfk
zJTd?Di%Z?xm?V815^Xa39)+a56p>f$4%u-?cESGz#ZwNJ)hPHfEDWodzVV{U<&Dp3
z?zPlzYE#~q{z>A^yS;p`INCD#SSJe|-5k{V-t)w*P|ia_nbW4k3u@iaE$f*6jdk}!
zK5eF13{x`06;?B-Fl^FtDA?t7OXB(Zo8H<NUu2xSG~;0ho9LW)Hz~<4ZMUu*cAoQa
zZoyG@$(iC^r;kdUoWDcRzgE=$g4j<Me}A9QvbgY#cl|d$JYQdP-G=u?Xx!vJso6&b
zG;*FTSK692C9XAXm#}SC>+>DD?qAhIUY%lnnyy*G$QIbSU+z9<;q2{eyL&z?IO%ix
zw$<6aH+yCrYu(xVawAv6M!h4C(_DlCB6PhpVxA;LS9(o<;-=sCzsn%3d-58Q!!unv
zwye<G+N(R^(Api@QyZn86?CRdo8|sdFk{6<qu80Ri!N)kTxFj$?Xp;3Ma|3=WvzQx
zT?&!u>ffh2!7AnEBc4eM-cH=t_k~L_+DtKM3F|j8*QgV#rWmb%w!%ka*V;{ui&p2Y
z6nJ!IgT#%s8Vg-nrfE*p30kn`@wxm!pLM6wTo|^z3D}^Ped&b2+FKnev%W>CaN9+>
z`AiHn;6H3BvtXJI`<rgbDNT|erue>NS;u&4DFfF^Svk%nM~h$ZuKDJD|IOrkG5!rR
zCSNOxJ6w^~R-?rz`S6q4rG*MgtNy(RnvgZ8^k73&e(Kq^tDY?n-6r1l!nNnagHNXa
zXVp(rT3_}w<ci<*zspwmyh~WQ^l8k|_PF&&B-Ce_T-+wcntvy?Yw5{%3DeHW6*x`t
zJfG&0{#Rb}#+m+&7jvH`r7duq+^8Ts*Y(lWce3}7$UJx9UODG=RVX7bU!bVcn_AZA
zJ!^0FNlsS_I%w1}VO#svQvHeRq65G9d|EZH%%>>6wToXiR-G;WQtZ>Yl{zQPj=f1a
za{Gf?cFBDwl?K-XMnM{l|HN*k8^~?aNOPOf#!wMeSun9$a}wk2rRNrD+*#VH%b>g9
znuxg19Fvb?4|}5|MWzS)JPBUnzji9ql@+trZau$gMw8!H?tdn(<x4hHYWeVPk6U$V
zMU=+T&7qH)c>e3~mAfhjyMAruu{G1LKO`RYvp^&JHec7>32Jkzu3T>tW9?m`_GF5Z
z);;MxGcUEe=^Z)4Cz|Z~yz7Mblru3(p0RvKx9*y&Dty$TXWe$CJJ~y4=$W=m_;K|3
zQx=wMVrE?K7Hr~=EaZG&M3}Ej3D~;ss>M{Bt0prpZkcwpprKDt;8t?r{?wzl9%w3Z
zUNkigvv?XU(792dGb|wEr{T+QRr#t;`bW;BsLVG?^gWnx@O_BOyU0`PU0DtUoG5Z}
zlu3F!`J>acz=EUtzSmqg%Xay-GASFU>uwg|@ReKdUwHSEZ0W<w9Ff!)Ecv#%zVF4R
ziCkSerMTkV>PK_Zm(@&f`SJgC`Kb%){}n?DJSBHb)V%TG!>@GzW!tRcS$XBZE(l7h
zs`_g8`tp{R*60Us1=h`de#1lk*YpRU7C3ER@L6z5^S1n!i|Z3-ls>*RyE$uqV8Xgf
zb2+X}S)seI^ZEkL-<=W%S~+gKG4-BXI78!u(+i>7Q<@yTrt0R*2#!^h-`ymu!@ja{
zR&hsyqH*_`i4P(p`&0RX&iNko*X_Q$qfzq0_61W8O7A!%EmK-EQMc-p*%OELA^Z7u
z>$*g*Qj&Ddk>p@|G3V6LYYU!jaP=z@c$K1aeI3t>E0T#zcNCq!{avy(N}4tLywTeu
z8m7G(&PRHd{J9ic-;<?s(X-KOrUpOL?H@}N{}<1$dLrh-pvm*i*TkK3N)wwF-)5O;
z%Ru(5o_R-?95;zxw!cBWHlkCq;$f)Ss;C*W-0y0yDwN9Ixp3uQmUVj{GbnQMy)Wn#
z&V04=fr@mZ-C;9_cC|T{`oZ$7MJiv`{`;AKdD}wekDDsjG&?QSQQGdJeDuk(HBXLx
z3e-53Sompu>BWlFPNx-|XM$Wk8#`Si%0&W?dWa{#<YLMDB0qP=b`}|SrzL^fU(aoB
z*`}~_gOa0ghf}LGOP?f9g<0b<=`~VmCZ_{T)t=~ljc!Svs5d*L+pATblRw@_HG7lN
z;%)Wq&t}hjubBNl@+Ox!&orhbKf0r>{&6|<sH^C%Umy75f8V}_Td~ncVz(dh%-HRk
z@?=82@A+$PUrMIsZE>8`eXK|A+pOz5tL|7XWfEv{VHe<=%r4LU{ne7Jl;wUg!85Yg
zh9(>ju5>Sw{btI<#q=-Y%_HmV`v#MaoDDzF9e>3*gn!FP?vk2ktPY>Qaq`Ls-q2c?
zWi5V#CsDIO$F6GXoTt0k8yS3lU5swIsNT4@GS%JF>u$tZt+Vs>qoz5Pd=Z;&Rq}34
z;Fn`+GtL$#1^%gJ5%*wFiRE8Wm=(5jmDa&6(pwtcIh8UrDkN)_v-dehU(}08Ww#Li
zz2UIQf*BX5idNr~u_*a%c_`b+_V`h2ZMzjZcQ^U&K9Vt^P5Sh#OpDmk{}wYFE7VtY
zoPN#n+*~EPhG+5gts;D^m)8GYF(*|>X4<(5tG@XDzWYZqgg(r2Srho<-sX2Ni{Gue
zafIjGs$ScaBmZ|EQmcsf6PaM{%F!5d;#<wq){4&y%(?HMnI;`6b5B7bwINE+@0-8R
z)QO>OYkmfBw(ntZDX=)NXQTfwYkR%T>Tl2IEU<H%Team+R>;abVVZ}2UEBOz%Yoy8
z(t^njjO>=OAv-=KXtwbyx=pb#RMON7Rs)?T+~n59G^s&Ez$LMPMc!G?$1<p45@)a-
zS3&Tzgw7c$Y+M;j3)EN7u)pb%;@WjqBWSTgUgD0<o(8ieQ4@s@l$<`-yp3<$rmJ25
zdz#u!>W!WxZC0CZlq}km8nJAOc6gc0v_j`)J~OS?@$D;on)V=$vGnJ(o422?@!6_+
zW>IX!_p|YOc_O=RYG0oseW?EbDc03yO3$~<S6?N{TlnSRr5T6I7S6c*!0{Tlr9q42
zw6eFIUg~@@9;#cMuFlP67e8m9=yZjvyS{Jrgs<-(FV9r-w<zEepDrWn;j!q!lT)|U
zC7CK-e0|_HBiCTD`+Y8>D#nx(TpjZ9ijqc4ZTPS2Z!HU84DwWTUR{>;aOK=EvzZMd
z<{zwIm<!20TGV{~gw*LZLAMU9=MmDIa)vjUW!46kogaR6^PG&T?$>2wy}@n2d6&e-
zqee+4I-89I{%bEh)@*0uYj`p3l)v5I4xLLLomLGRo*O6Cyj^f}#$;ax4f~=^r<9w$
z-z|JJN7%u;L#SwlqQ0co3q`||Ho*)g8%-un?zXy>q1@{9$xmrg2DcKU&GRK4%a$Gb
zbVlo;$Hc6FunA2kmapJinlZWcM8%XT%lW!W0_H_M^y(DbxM<}TqdUgl#&YFG*_*@^
zg>^i3$LYK=+;c`~!R5gC)-%qRgG?Xu&D!<jf|2<d4xNp={>|#Nl(J=gyVET5^(2qy
zbLMhHb*x#^;gz<r^sQHGn)O-7OPhGj)wZtceYe1DUy0W))6V2~i#SC(!W$WO{N}WI
z$YkNc)cWUAr#o-vrjyHVi2WCCNceY0_`ve^Kf((h_DkhS1SsESp1>27v+?-?Cga>z
zVNQ?2?4x1-bdEZUpSHL-)oh2tjFg!Sa+~81mu%@&6RV!<6WW|vlzL>c>f(*AjdjN^
zW-#s&Fm!uf^eW+2e!ESEXYNEE?R_i7XGLXqZCKW!BO^M?UvW+S;`ipumzGW2HDkvO
zg+p^z*>Y^xD14lh_pD;^tp(=&KHDE~T=zPWf9uQ=4WG^nJ~B%GZXEsQ+hfu-XHT=q
zq2PuZex;M=ZW+u!aij3*yo!b1e{H8PYMtBk;$d@#(W}MoGbfp>T6tnc*TSf%gL}2-
z<{0kOjcAu!A+jT5(_zDqohL4c-dD6fA?kno{{l_J#=O6;`FL;E`l#)alm4oINQbAQ
z>biiipU1-&+_O)7X5Kn0q3zyxW3R%O@h491d>rRt8u$AB{%y<HoP<|$2v}zS?01^B
zdCp-^)}lKzukXny*1!EkJ24}vKl6%^G{;pI)0hWL>NkoeT>Y7^*kkRnNTlh|_bDIj
z#aFx9a!r})!|A|enaLb`qgh`3;S%18z?lwVUu^#H2)f#OPd(WaX1qbc&A>!z;(<vL
ziy4euOfOmm{qmUM+LmCb7Q^FvdY+1E=Z0f>4Z_^cQ=E<&zjEu&bn0WbHnuQuTvEXO
zarR`Hpb3X8T?*|Jg*RJF6tv7$=j5EQfJNO&=rkw8#6-jY(_S3oaA}*Mr?g~pm%q;o
zgEkLV+qPf)kLA8j2ng%awDO#?R7B`--&RLXrbAt|D#eE*;!cQ_YBuYB`=r(Qr9f@F
zl9-TTvZDOlr3Rk@7bjhKB&p-Jx+XehnP@As?|mQP=u0iP3O=!KO^IF7`)g;k`OWLT
zN~$d}UjwJ@yrC}IF0!2G)siXf8$UZ#)h;RK7d-Fy?&O53zISP-6FFKGzS!}_zVTnR
zm_wy~@=4)k21=sY3q#%nD)enVB)Vk60kMD-EjHGP?&pm~_uDN|Hh!>_CrzWdW6AQx
zDl4xzW_aJ)&bDdGtT+0up)OHAala<4dwHc!_~uH66UTyAwz6hh3QDv8mp77Mrf8ZK
z@_W|Ve<9O_E~FY~8w;@)IB&SV=Y8!>5%0TIUefD>J};ZP>!i!8jI*sGE#ADE&Q4s1
zW*xp-*q8k2{_ZIg7$%EcQsLfs=b6T)^Zj=W=J2eHJyho8>1*VpxT4$Rh)C1Cg(6Qi
z3^kt~P5VAicLBS0gyw~?C`DGijH8BfLRV*1S?=A{&_83HW?FWimx_MVX_ME%*PPO3
zhkP~My5YdexBf-Dr)ep@u6BBKK*YkJ@rhr<POgfR5l1CumikPOlJ^WN`MX^&*VI*u
zBX(ir?rH~*Nx^%b9r0j#wlsYGoz)Y6T<MRBNs;=&k<h#|)9aASjvMd4@jLK@DlU-|
z`meA;AfsgFtcD<uZ^}z2x^^*1cl+jYU0I$e?tO4+LByI6#pXBXr-l3#me{z9u_wWW
zOGv}e-J~f_G(RQY@~7I=JKI?AJn&~Qc{VM3pX4=<S2OC0j+}{peQJV@Sb<W!k&)ws
zJrnG>W-8S$kry{w6t4L^dm?w{lBXZGsDB9gsdzS9-cN2(#I2a3$=%7Zi`9%y1-fWy
zi)p9MFjj5VEqK56(3BH7J2MhraFy2yZCn(k{3*SI<Cl8Y)<shL!=wFU+l-SP*A>mw
zW~?~q&BQsup)4dO^WB^7i9Kt=8B_x1a$kR6H}%wQNq1R2huN3cY@2!G2*-*iQvM8k
z)n=+!d|Elb>ObEp{TCZ0B^6R8^EO>HjoH!vQet*|s&tt>4=;Be>m9D@3N!6>8Lb`n
z4=XlKJa5P%9Q$+0cClY98)k~Nb@WZz^=j42X)EP5wRYPkFXarKZW8#{aKnA(N4gym
zM}###c0P9TaMoHdfk}Odhs4fF6LMBro8Bw#y1*gbdgQ%&!I~diw;bJm>xYH8eB<iA
zgH7`f*Zxt+>AfV+?S4Ibk*1}eT|&d*M@7dRBbB%DHF%2My!Sz8NiipbPJb1*#@&kV
zigu2%hXTx<<}Jy)`7bjzHGfmu3XjyY`F=~+7@4;6@^o)I{*x<HfY-k2e{T2o)MvTP
z!gsEh&3%*GzTv_&)d>ws;{TN=mA$*GzxJh1^e>g!UQEkH*R9`w`GWOtqdiVOb6)*h
zy73s-$&dM`UzHx^Zh!2jWaYA>d*+8t7mP1$^sDk^Vu;~Zn7?xx$1{-zb)L>nT}iLV
zV2wPt@+7-me{D6MEmYEGEO_Iv_C}M4o*SDVhy9QDl1XhwB1*rcZr@-#T^_vPzg*Ck
z&2wccemuFkO{#grvBTTG{A*wP{-8krjr+2llb>)%Hma=nuk!Z$IZq{nE<*)7E<phY
zmM42S{j^=4a|s%j9`(pC$WcsN)AQt556h#EE1pJeJMc)M@5-(PKee8k+O$4CajG{b
z`GgnW$uArye{lR3Ii$3V;Y;4S=T>jyKB=<)zw3R@rb#Jp#no@8Ke8+;Q*%4xm2TmY
z9$2v`KqN~^X{lzKy2BbiZ8JCRlr!aP&fcnO*Dc7@F5tAC^mLcQQOzQb#yJ;6vo!4w
zt&6<zcGj2nH5|NGqYfL3<(fpDR(SDlhuVR6YP!}2x^Y^%e4#yxQJx&z6}Vg$`7L4K
zpXi`@EJ0+E16SK3Q8!0!vxQ<FjDlMoRP7!uiaBGpF8|6Wb@fvTW=euAS9L#~YW0ab
zt1v6c>Qs&in@6PGNBaxSJO)j1PV4_lto)+jWHV_&*_OkN8Hc_~sTeb<xXx)w_@S2g
zq5G<e)615L-&&kYbl9|BHQp*w3a-;n-u9@~D9!dxqW1qA3BNoP%}y;B$yxY4s?lLf
z(ybj$qB#rY+7|IfHKxjKc=PYjom;uv7qR6`I<;bsOVtvukf3A1E4nO`od105J#vcw
z=o6KrY@8oBCJ8DlsU&2GL@bnzSolw5lE^j(;Rr|JH~Bn#fda1+xy>3y<yvKK9~AqN
zulOcKb&128MMrWECFK<`t0p%kh+L>sns_7cz2ctL+e-vZUU8fV>FX0|TIb`(Y$(v^
zB%pL5Mcv-%nbU?cmX$2b3a8(6K4(f`VmkRgCOhKYb0xh;JKm&eyRJQ^c1(Qwxsnor
zn_slGC6a9OF1%bN_-dMyVuJdwwBs5~E;9M;lG_u4PCU>oI4;)a@AH3=lGd!I89NS$
z-FX=E<FP(BixwM8`Hn>!_PltL<Lq(nxsif?^R@>&u1w&)Yw-TZxyB#sE@xyYF`ZYC
zeDr?F0Wk(cPLGAMED0i6i6T)8<vbciBN7Gq7s(xC(7LA2wXH$4#_`p=W7cZv$~!n&
zUO!iTd`!*p0@L9p%U<RKEFYe#Jo4U_z_jh~H6e#DM-G3HTV!SSRN>Q8)gNz-q&LX_
z?)`J2*k<Y@@x8A<)h*I&a(dZjq;sqIK*t1A>o%T-^EWykDSuwe%Vk_XX`^V<*&A$6
zl5Qp3Vsg=b^)Ttif@GsKy;~En)V=w=r(bW^+SI7FKT96SGk<cv*#BS)^FtQ@cnK%#
z|9tCJMHaE{4|WYb^n6K!;%k55xX%h1#ade$4yY(_#ZBh9buh>+k(;kYRKTUl?Xv=(
zj#^w0Lxho)YLnLu4F>4~#_jyO=cvR+Jg)ik%)89U(rQAN*#_qQS9<LXl!O+s%cg72
zymU&*iIXj<=$8KbcY4ZdXI1B%R<#oG|9kd+-TQrc7tCb315}cxh%I{6AmDJLK~t=a
zV>6R>)(W0yK4CIR3KohSnq3@coiuX}im~KqM=aFZqV!!vEx6!h@TyeqdL=H&-Ux+$
zmP-MbX07J6EWZ4$FU+x6@tc0Q=jZ1yT3q@=9&Sli+{(cDEdSq|Mv;F7+>;V`UNNdH
zSrBNjTHya=2kvbNJYJh7zH(*Q^GLOeB~<^-$75Z!Ek|s7FV~z<xwGY{=Q0kqizj<e
zbhrNEo+sz=C2HaqkwssYEjYsWNaa)O1Gbm(+<6Mh$B&%xKlks-f@y1Q8#jtFO*P~A
zYHOaTKE*J&?u+$%<>Y@$G`$pm9Qjz;)5#L`P%KNc@(rV&URkOdtNu5ogGsON_N>%d
za_aCMt^40D+WZxKV3K5QqGWBO^x(x31@Fm`6{Sj5Wq;icibXthj9hp`B-mcm;oO(6
z@_{AR7rd49-l`rc_FDNYaGHryKoDc<8?z3V)oWNp-oEfSbWUxv=F^HI=GJy~%|aip
z<N4Fx<U}4nbEV~z*@B|~d4B6zx>Xny7_35~?GI_oc|?cnTwWpO>RmKRN?WbZPqArH
zqHMY1wNLxylJc&tDOvI^Nv_H8<by@GR=Sok=?Ha~-&v%kdRXiYbF$EyuQT4ut#vW7
z=Fmxcs$ekr-L(z8n!J}EO=$bE?vmdI6UO!7p1~@&UwDK%aZP%lSNn)>>48g25(EQ9
zDjt7$|L&;vk_6^$<>$^Q>H9Cyy>YmC3X|Hdu-~UjqnUg^T~QLgZ0@bHNLbL|^Mc8_
zcK)JEd#t}Mb#0!e_+ZL^u5-^>c09ON-1@-SqRmh{;-QFrQ{e>>HpNt7i*8f?Mc;2Z
za9Jb-En6#MvBWg&dq_{FOAe>5z~birI~{IM^0Z6YwqVWr*4%A>{^>sW6PoV6MdkYX
z2^ks1-P`?m-oIyi|G|nWoHIN;wnXv7;rBsbDlB`I`@SnGG4P8z{ayC0quIjR>(l*B
z*M7ZO{Po6T-90z>zCG&RxItvilf<JR)c19o?tA%hR!H6{k8I%pw#6p0vnQWeR&3Si
zWAD@V_{bI?iwUtcK^n2^o;DqRTIa;FwORAKX1q{Cr)1M<r{ba;Uy8gAO;9<|Bdnzq
z&|c(RpfP9dN4Y}*rA@DHIlNk@G}Y<rB8%7k`^*zv9+w?Y=UMsGNIv(H$>#E>2Gdtg
zo7l45Z<*4TPXX3x9+iqoD&3a+z2Qo2)6Ar%E6$2!_-_?*Zdn5Ns>9X6YR-2Sh`w>q
zj(I4yL$asvB_GRyiW8M{w%(khc-6M}jnKxGv;BIHtEutqeOjBH+}C?}_aVMXX?b?7
zoA#gcW^~d2B9|nl9;x`}t0KeeI4;f~X&&}!Zb@DolTLh{{IXa?&7!Sjk!*x!bLf(l
z4IWNDEUR8II~6dkKIVVqOn&-fmc1-N3c`FUDh}M=j{e)i5c%=LGEpTio{6Hb9Jwos
z&D<YG=02PmZeqc<zTUCPCT!6^zNVF|4J+s0_;=;K!khk8@=nWD9l1>&^8cS;X6nFo
z=Aw*0ThGZ`VkfMW($bWA((@H2S{l4FJ+STAp>LjZr<lEZR^z+L_kZ1D;av^hr#lrE
zEfxNBL*db?^T)1O^cZQp^-_|F&}eto{LJ;7rCaO!)OiYv=J_=}@cL9@Ia^8OG@Ix0
zM4@FGw}PfHya{6AQ7SsV$iUBImI|+W0(0KYa&zOt*6Wu9#7*2wDh<_EET1a!XQ61#
zt^k3f%Oe|QcU=^Z+AQ}dC}R3gg~^pBGrud$nz>@-Z{?Dx<y#YYOde_ov?gXRlzp>M
z?oZ>Mv$y3_-pmf27SETo_V~@ybKKS1*o?ed_I^xCRxVv+_@-y6cFYGUbG`MJ7y9P$
z9`7sl+1)Ww$*ljT*qlZ;fiIgr$9;31<aoJYwuGt>!<w~DiT8~UPSyK$OXbI(V21w@
zlclY<FrF;`v*FtjyDJYk@)~O2P1@?*AX%vTW4r1eC+iREMgJ@m)0pIz_`*zZso0Ii
z$m0rJJuwV5jj}!OGE5Kt@why3;&p|q{;OW|uX_Lf-!=Wy#lKcAc35uexcX?qYSBq%
zHy^gftq}ipQ2f_H`8#{gKD;CUEAia(IcHBga0?v^ds40VKw9a}?OjupEDd(<=RcIo
z6Obd{q?)eN`*26^>MQe(w$5GADXjFyLTST!4)v3_QrG^oP&#!b_U6$EAH^nI*~0RD
z>L%4B3tlOkVwtbsxYl>KOqRdyR$VYfZrc>2)l#<M-#Y@dy%eh+PuNym`MyN@qr^uQ
zvx8z{g;7fzD*t!R@IB~kJ#XutY(*|N#<s3KD;NH_Wtw=KwLwO-Xmz3RtmehLPX6s$
zXeqEcn)^x4?%lr?4@U1fzx(XT+4r8_kyl$3DZc35-$s!w4$|U__PML;yqZ1fO~c2A
zjgM6h?RtGH+VjPJ^#H|tH@9=lQM&Vej=|kI?=m6<(tIX4$K~=asFYNCq^k6*UoojB
zNJLFXA^WBRTjJNpK>{J)-P{*TuKeeu#8vj?`<huUJOBJE*>`hc#lMOwZk0yfjxWKS
z=Uo()(&x|MvDqXon5xwkr*zwq+aObY$L}5ew_kWV3K!ivbu{jD^}o}zqgHR7wOVx2
z$@4O8uN==_jX8Jo&e`X6XMaCb_+Rx<;g{lj)`a(>P49opDLuP$HrtUaud}7<c{79R
zlpPHp#hJX$eUIY#TN61+Rb8QYznM}@(CV@^rl+oXZSwTc`^PQ2WbKJ31#*wnU#?hh
zTFX2?c%$jur*CEyy4*FBYiD8Yp5&bLUu(CCW=;Ik?~U4?TlvdMJ)W1G{iJ!r==hBu
zO|x=!ub&J3ergphH)9pl>L_vaOk7#-A89J2^u0j9-jqvXn%T#>k@1(mA3Ahtk^`69
zL2<K2kx7DYR?ardZWKM15G7h9-1p}+qf@ttuIC~b!3KlGLmEMRZh|{pPtWjO;<j<q
ziB)2|qrUzr{qw>w#l2O9^@?_<(NeD##q8Xh|C6*A&YhCyu}$bf*3v^WIBs)j9QIU{
zoU5@^!1aWy!=)+SQ`LNS9nHMFIArytIb9pNuda<@WqJEc<Z1ZSWYKA%mQz9wZRWT-
zP4GdO!^-(_bBi7sr8+UNR!`P&kYGq;+Mp_|(phEvG{B5gQd#C##PbU$ITa_D+zDHH
z>lA10AywxtAMJUZ(KDwhHHxgR+G>`0F=Vx@cA4|>4RWS#-`p6ZZfrcIap2nw6}96T
zzwWNOd*PtgRE3ZUM?Q#!MJ?))OfB%9!oja9`Gl`-cSXw`Ub&jJtmemVZhcYsCUkwc
zef{Uxm-pMpH`p`&{qg(#Q+cPqj5C7jnYb7hH2z}s2x|KE_3{7HYSCMagq=hug-mWW
z;w*URBgysPSd2{5im(jL+@^pK&XpPsUBVn27`=ig98v1lT&HoM$86h<Nxg<jM;b%?
zU8i0Nib-G<o20^ev@<M<H<V30fA7;J3{k05FHF%c-4x~&Q>V!=J#C^&+KgB&l|?gi
zX3o4BlPS~|lo9MP!GXz3qaeNH$kH$3^SM~n7~+Li%#iS0vmj)JQ~RNn0-8$_mR;6h
z$}tMjp1pNzq|dfHCQ{nb3JQ)QYp*D?3Po)wi?O}_bFpXDzdz4rt1<m4o5ZlS=hH0P
z9cR6|bf>e;x|HDQVUQ!l>lzr~=%FCZD#Ro+C7?9QW64PkvEEBwZqkh`|1Z_ZK6MXT
zt7f<=VW)=XiU~VjsF}>SP4QT$>14@S(5`KBHak%|Z^}hC$-FH^-r9lJO4uc3bW*)F
z9j>Z2r1o6uxX|PNaO#93UUqAOdlNOcT#0t6>?u5<bXf9AfMcVLNSMO@rD2-6JGL4~
z$~~Xjz{2IRL43jam0OnkRdrpPe1dOr%gj_~^(A6ms-CB22B_V-)aP{aZpsXnH8+%8
znF2FJ8UvTy2$Plm8sl<IZCh-Lw{6sfpl*hfoC`Q5*F<PBZs4>G7rJ*u>4`x8+N`3x
zrJFY%k8=O_!|49QPGvv&$Gk$J{W~=ocf@;6oxEjTO-vx4qPBFvilF@}9w!quWpWDt
zSG_XN-RP)Uw&T(<5yx*|v=3{o?D*Zs<=Jzof6wGEj|1&D@fijgsF_$F-!8rX$;_}u
zpDR;U0(Wkl>ZoaRZI-X<*EI2X-8-G%i)LDe&*Yk#S?83wPjgdLj8DPyxusenED4-1
zbH2~7*9=H!3exDE;KF&pIB)h;U#noLMT!OuF2UTFD@v4G7&aub&Y2Q*;(*DeoK<3L
zZ!j?^O=t<sWaJD=XlLjsp5%LDkyX%z6TZh1rtIu(65(N-Beq4EHJNikp1Kc5aloND
z|2I0B-de@J-KFUW>r0myWp7U5(oO58+8D<E+u*LdM8RQu?kb*O3n9*B4s5k2*YOx@
zHnVtL;4=E}vw(Nq4p(UlgI1oaeogMHPra`T-#F<;M|njFSMaQkNlKyXGfOTCDfDkz
z=B#8iA$HT5XcoztCyuD<CEt=0e>Y__3&X4Sgw8&Zwnvk#jM##h6jU_@i@aPX9Wh?W
zc{W4nlG)`gNy4G8L$574z!v23V6xDqrRv)i6jeifxh^#JCGsTuFcuW4o(p8!xa$~K
z{e$#bUlXS{DwOz^O-!+Wy=%@BgYc=RHB;=LTzS^$wMd~*>xkXg3kTL~J>k-O^rSSu
zuvz=T2{-E%6S?yOlqKK1Z2MlxlPa<$_G8I(fyH+W`^}Rsc8fG!Sg}E)f@@)hQ_>b&
zMi$OVGS(8ugfcHoap3>2dByqEM1ez`&hzqXv{QwCiYVDlsXCg>r5zk_pk;#5TL0Qr
zXL!`jCaC+WEvRGSIABzMP5r<deFM%0zHY9mN4dPdPFPhb;v~o6z~I#tlo#xqI%S!r
zU+$^qBUPLm%RDD+ezzf_bdrO>f?)H1iY<Pnj~@n3T6Jcpnj?3&SI3Skspl5EOmqx>
z^KxfEP{h=bi}FztT$yp@?2JAQ+}k&Z?~O@h*UL%ZPI++X<&7I`oD&xIew=c^qrWMj
zMM70o$a}54_a*nIh9#<!Z@aP;yC!Hn4O6{jye8lI*$JJ!+f?!`XBOy}u2hPhq<^M1
z;;Nd+)pbjZrfM$Cd;3g6(qra|T(19}XLb4|HTHLu8yt1vxL2muV9mJN|5Yf*ym!a$
zJ)aXk!}9^3^qY-BOo6Ih&2P^qT(ykYaA3LE>lfOgS5E51ugGfpwq!Yr$AN`&b{v{#
zd8^@!SK@i=ElJGAvo<)Kj+nIOk;f6MnGEu!>Kkvk2{h@%opdgoGU?%2K5Z?lOfLtm
z%WJ=sO}NJGt$rYU`rNvwUNMj4{PU$QKU>;T`_h@?_l{2~XDe9yDulEGOP6h4cFW(O
z>7}=9{-x^%;ZFlvKXEQ!7Ik3uo*otckP~6723$N*t9X5~538~4IB3Gu;MekV;e2N|
z9nCE>W13P0j%|t*D*SmYYEo2q5_9*2{GZJt!c{r{rB5Gt$TRhkN7;drmgkxJ&5eSk
zf6n-H6fW6)J8Vt2M&QMskeuT(FF$JSJ9;YU)sAG1ztaN6g}IeGlT3E}*rjq~t%`WS
zWKWCG+=dntRYm1s_k`j!!y|2*x0W6{b^gwi&Si>614WJE=gu)#eeA}0@6>Of*B-$e
zAKG|Xo?aP0O@Tx8=^K7sH^**P-VQm=BzE0j2L%~B4w*fj^4;s(;{|mp>v)3{6r32m
z+IT%y@)W2z`UOlEXz+_j$UnHDz_V%5nxe)dJLcvdiF)#?!#DA%&MsxS%H@Wk%10I*
z)1K5j<ut3-haZVIg*C;4(p0BRdw5)Rqoz;lsaCDllgTdUx%$@sFS)$b{AO89(dBiP
zX6lJ-%lxDCR`LIDGM3*K%5~IFA$-wlVZ+Ud7lc!am+m<=qic(99LLQmjipWDQ*K<{
zTfsjqPE}!b>8eR`ZXyS)GCpQZ{i1C5<$=Gwn<87_liT)hHgwqEP~`eslOj6HA-qBG
z^tuUK7HzcI$(i;xw9RP3F5bm64NmyP>z{}++oV_|=XpZmqW!uQuAm1idcNFn?>$+z
zBS=ALt!ta;+~}9r_Wb>Eb-w?!+xENlmQSrydHLwchBYya^&h#Ns+LX2*AC{iR(f!^
z+UxGR0O?GJZ1!@&OMKpqw)vG$H~p=CA@kw;1g)oL`Z4lGu2VLPvKX1E9Z?Wx{I9TD
zfotoAyw{iHqzt9mHu0@CX#5ezKgXP_H(Y+Z5m)kJ&Yx=Xe>d<YU8wkyCVFFw)3iY0
z^_LTb&GdCE6y+BQ7|p2NE-s=XE~+9XsxK71Zd<cfyZh>?MH^C;j~tFNFpH6Dwec`a
zR#{jRdolaQ^5jFRwq|Cc=35Oi52<=g6WIIJHfN(j#-ZS(i01uUCFhzluD&XIA+RQ<
zP=l{sgG<CD;hIFRQjSndi&wB<>SbYL6NmCPq1jHR?OV#V7uG)%&xlIk{&!U4{w4lJ
z8@MKl3B9{0xFgYNd!gW(3l*;l1(P>${SBy`bcm1Z2LG1?U7e4*xy6E#8A5Ugx#VAD
zgf9HAAe+da%plC<AhjTk`-hQ|wxRIO8N$mx3MU^7t+`=f#2$TrdR(+X&Hn94s@tty
z0}ZW(TKOMWpJ^7l%3gc>MzK?%?Zq}BwPur&LQU1hB3Cmy{xO@Bc6hz<@R&BSPL;7Q
zC@rh&V9uNExw(g9mj!Z_ey#aCJx=wbPqnE~&k0|zEeg|}{aS;$?-_9|I>5!cP4I7l
z;QwP>ldj7Pc=B(VP^sQHajOECEQdMM1L@ZZLVFDgwl>L&YqCxJp|HS$?M*-d$I1!+
z7!$q<D$f60pe5L>&?IECLcr27KxQ*%uwc<-rsNG`hPl&~war<h7qkg3R&v`I<)@i#
zrIf8dO(;<Czp~CG5ywe3@s)j?j;t{crnaXfJ3bfnD(|}#9Hjcu_Qm3qM5mOUUz1;6
zR<V4lnrvYgxUDyLqqu5fNJC-B-_6Py+l8KA7iB8Z$X+g#dpva7B#9*oGk0`!cPe(=
zTP{!*+MznJJmlldhvD_<TTKI8`2TGce3PKiT{dy;j|qzdxK}6WPEW6#x?N$WJNIgZ
z*-9CeUmUu&-IV*PFj@RZqpF4w*NNCk%Vzm)5p6rn6*NIuTqrUmkv*VQ;H#6+x(UKc
zPMYqOJ^g`Gq8$v~4V5ibrt%3-jlP*xqmr$rKJ{vIUszzY>P1#@W!r<_rk{U4?f7xY
zTbqj&r>dovi`0Ge+~4|t{!)>?cNwBr*d+Ulgaa1~d{lS%IbGmLVBRlxnP1oAo*XL|
zGKl}5Zu(0wpTnq8^hKw*BHQZ+g4Yv;xPIgp9B$aFp!xN{#5EWAO}!>=+B9)n(c+oc
z`S)(%N`4@>E3#4j<?Pi4mG)eEuUiT>?GW;+suX%Kz2H-~Oj6Yv6Up+00vd-nQ;mx}
zI2|T`unw9qk27#e?)IYX#_A@M#BGuSqdHr5nA#X^YtcP6fA2LBi)ljE#|&yt%@1#~
z)p1qx6ST8*4NiQaYG_(g<QUu}m6~m|wCl^v*y|y_r4rGNW!EaxG!-36JgnUnH75y4
zi8ykxgv?Yu=(u)*>8TavKQ(oDFaGE1+a}L{Q*O?7DW>gQD=u{{v*7wGu;lR1uCEUK
ztC`aGzM4IAJC|T&*RGd>8&?+m+c5EVk$mSfLHEFj&UVYt64$;~p)!UkQbAR_SGZ1(
zET5C4xkR8Sb@4i(mUWXA=WUoa&uc;3?v!bVu4maCnpdsD5E2=v`zfn_X0`tcVXd#z
zKkm$a<{`>HOY+tY(NN}&dl5n_S^7DoHePZSZFY0y6PJ=aC>5mKV;U^fu3_|0Y*Q7h
zlb{3Fz6ZWs?2}m38Ilwx_iu0ba$(Y^z{PSqxTGx?UlWsO{U*q9P>xGn{^!RvT+`)$
zBo<6q=KDXjiJe{X>ju`vA0m`5DTd6}*cQM&yXpVdUmD&w53%k4D4pxZ7IjL=UARbh
z`+BaxWiL%x6~AZs&lD(SUcUKQ5vSqyT9fTs8v}P7wQfq;VD&KQ=8ckMA+Z4S6_#Z?
zwr!tY5h-qUb^6at(I1D)e{Xj74P7u{cG*&q^k^ydi7qoaUTK&+mQP=~>hzVFDuT1h
zL}%4)5=x)SvgCt{ydl?I_JWxQ<o|t`@TFn*q@sMw>dIX+J9lNf9NJx>_)vGdw!*8H
zgc;hMvKv{G6J{C)&P@~zi@L(K?gE$o;=M~x?Y$YqEomtAHGz9Sqj;5vpvIxBPZjG`
z53bM37TNK++WOPJ+!;LrtlKxY^-6qhG23L+ViNTE#GcrP%N2tE_de(l<vK0;NL|9}
ziEsf&%sY+)Tn{CfR_(jRD#dhxZ;7F)?39I~QWI>IH*p#2a5k!c{+_<Z`_NG##z{(k
ziX4@)j+^IuOx*Bu@xF)o{7mv($1VRFuH|5qfAvJ+#g)8$s`6hP*`f@zVrJ{c{nl7B
z!8@T^<MtuJ_l>@nj#zXp7QD7?r+800>r#PDN{SqcoPnOpv>Me9v`i7$vh8)GP}I+?
zb*+KlxB?F^7TUU1q{?c8?J|R1;qxON*?1O;S8A=WT(%;)QMB5|u8(_qO{L~UEu+a&
zI;NYC1qrun2Wq4&6Y8>@*=>}ev#6c-al83Kv268SoPxqKC9Yg|(sOlKy(0fhsJrM~
zkk*r%IGf9T(oyZsdqHa_ecSxfNN?h|>DRfcI-i9rI&!@<OrJ7S@W4c~%*6>FD!!AZ
zO*nE!YL%h${1~$rx5S=vFB4D+T+4Ir_pEu(G|y$OnWwWT;LGmn$C=q%k12az+-~i@
zfzik4T=I#o?gQ7=gN~c_#R|$kQSW)1(SLdAg3H^51m8GtHf~&SMMM0~#-)i;T1*qP
zH~QE<_6ZhHPPb9_=bZ5Pn)=t!_zi)U|58u)xnFvav}L;Rk+x|9r(Yj_Gi}Rble5z{
z`x@k4c8%ta+#vO#@aQEM-^;Ij{~hdLFY^7u*jTtst}jWl@v~`M&y-mkSSKxD&Ah#D
z_0#@h*Z<3M!;h`HIxk1cFfh{mjD(GtXy9>XmGFtx?nd*!`Uv?MrMQ-ta1}<m1$k&*
zEz#SmR(Vpb^Usbd&-2mGW?HjoOgU};zql-|J&k{|MswRnr;VKY#li~~Ub&d9J^iH+
z*PdM}Yi?Fvtu~&-p!Q8^L9Xc0O{~E#@?P4mhh_9a8`u6^Iq9<_f7Ax9CoFo0bdLN^
z6}o?$%lJuqLEE}TQTs$Mp1pT7;Lk0eKV5f@PT$ArTCcn4-1@S(Q@+vvHH{95$L!vE
zzP?tZZr6fCSBsUWiafZ{J9kN6rihhQ;|vMm{*1!@+=EA@bQYeTC?w`3B-^vAauGw&
z21XXHsXuwA+L|1ia{K?n+ZUbMizD=V9~3z8Z||C5wDzuA#F4*&y8jFo`FM-i{IU4c
zXyLxM-PmMR8rR+_f}$68vo<IzP7=79;<HRni}l<SnH7rbHO@^5)@7K)P_w>Gs*HQP
z$#S8BSt5mhlwZu4rrmhFu1BaLOT4=3iT%gqoQua@-D0k9zTUF?df>L;^Xw;^0@eLP
zPqK#y6=clqDJ-AW<KUdKOk-iFgo(>UTdte~p{40FWm34eh!`~o8gnLuH6OGx{VWvn
zaaDDy)BNeTTzLIV|8~u)@L0R{))~*UzY}G2bSozA6MgYWP{h(je4}Kalb}e5ppKX3
ztOaLZo>KnryUa~-{qf#czq_oH6aGKm{`t<NuaC~W*zWmbdrwl0_s02C4sPJLo9^oO
z<l6C<u}mjCSL_p!(K-C-26u@_kD<xS?e{_rGgKGtlj4?I+H;UG<(~V}h0mVNWcj6+
zt`qraWt;HaIJsXwK?1&~qbEzO6JWAP_;^tMpIMif<KhYJ`DT$X+!Y&4?!Ie(dP(=}
zB^w+0)dgCdb$%Kf-!q)w;`(<nx7@bOg>qXR9k&Xu%8<Jn6Z>lEx$32Q=l*mZ)3~JN
zcJWnd^VIUwJ)+xeZ2l!xdtcX_WFz-=yM@~UwW+-~%y)cUDeB2sdLh}aRI((ma=LzM
zX@XE`__;mzuc=oD8hdW&b4i>VGVxx@I*q-xS@B!{OU&mMcRsRqdCtux6Uvuoi<WGP
ztZ}Q;-L#NdDx<4X=x?BQce?;%y7qhxhMkpyXSmWQnRm&bkP`4bvqvvt%N*AC?6vZN
z+}HL<)>#NnHsZRg`BpW_Z2r7}`#e(vRw_TR`FS+F=B}k?ia<ro+E)*bHOHnHmj1~u
zcocPJ`rSHz(S35Y3_K<6M?6n|`||ee%2Row=i@GLb}w+9tSOi!b=puzWR<YZ7N4K(
z>PLbO-7~i@-|FmjK=$GjZVkr-!N`f5n<g>|Ea7;R|Mgy@(XGa%7F@3x6duZ7Uga;+
zD}4Knz+Z-C!j7>DAL{hZ{`#<ocW<Mx%=h^J8bVULnU)3e3Ph{?_Y@46u%MxlX^Y3D
zMVY6UwX+`YSrxi|DnroXEglae3cMW|g?RE5Ig6M*rf3Eic~%A;5aW|e=1S<?q;|H+
zJoKH<P1Ysnn;clDFnWrlicg!{P_*mi=AIxwg?zWSQzm82T(NW>m&Q@ilq*dM!DYG;
zQ=dm}$qdyBn7Lp<lM2_8ErHSDs&{uQ=&wpqUGV<s{yI-iiK#`C8xD7JPm(nK@#uls
zk`o*oST}8I2;0Uc?voV0w&fh_-CbXKCYfY#w!UtB6Tad83d7e~Yo(MA$gehB{pV`(
z($28grCY+*UJB{ozM^ieE9Z*q{#UfBmM*;=yLOZ5tygEbf4#o0Exh{m8rCg*{r@+W
z7FT_J@ip4FUv&3JrW?JlUPnF6tA7)5F@H~H&$2)h$DAp?Y|5#VPjWGPb{ncVzFe%k
zq@#dKQ={jDlkc$v0f(>;tJAkfd93tZsyW5N<B(=fRHl{K(nFf+Q`D4Ctq{xJa&gt%
zl3SiSQ`MX^4dhv8=xE92`)D?*Y|~&}Kes8QlXt>`79%&tWmC?1X;$9qS$^`?ihRQ$
zy<I-*%U*anaB)uY@H~6LAt-pMIcq@ixu#1#+I}{U-3ncXZ$2L1b64%v()m&IRxiI0
z`s<`WPg#{@sz_B<Wc$UsYm&|L?B!*rOnts+v+mkYc7HD)3k=q}YB$AeuU{ae+Nb9s
zA+Hy<%Kndf@p9>cpKg(^*8}UELoU{3ROT(eR<rS}d_3RV)rL3zSo17Bl#ngD^stKH
z2?a*JQ(=m2CYdYDbYJ`yxbZRWYpaOnmWG0^eitE6hO+Ws9&0B@tO{?K@u@aYRqt16
z;Mx^OgtAvHZh4Y&Y<|`vLyea`M&1mKir-#6+{985p|h!H%fv0)oxD67SyX0BU%%y>
z*U6M#f3pJhqpL(dJ?*}skeX)Xv2)GTD6UgE-skW6O0K;;X@b*%rd5yE$uroi<V^8#
zeR<UO{)<Rgk*I4kXNg5>{+ckitL%09-uZ{;@!4mdPfET1x6Id%f$^fU-t)j|zbDMw
z;jDTsd)-W1g99i3DLV`Q4`N`yW6$oWq;V)G_)J*0#ukkeioOwy{V@)2S1wiV)6+R|
zWcnl5JDZhqWo9m|JRR_3P3zIH+OJ2?am`hi^?V%BaMN*8cTZTry3?nc_NC1I-c<3-
z$ZJB@vTUEZ#nTVz&cDwqsq#FqE6(Jl{kI*-^KN(<H>tceoM!rNnb-l_K)034r^jBm
zl-?DczIN))=*iM0d!8it-u&nPe)a2XP5a!-tS2bEHk-BL=;||{x7-R@BLAl0-25X8
zX1`nG#IsGRRpQ+BSzJ|Ip$ZS@=Qf0L|J{(Rlx)${AoN*KcZz^wAM;5EgXwy0CD}?5
z(pTnNRcOCcSsknBI9Wu^VM23`f@|;$hyT5C>I~B#aJy+mJrcY+$*QweRU<*rOJuFa
z(XN$Cv_fp=iW;|x$u&Odww6?WeQlD8@gh|Nt&6daFM{SDTfpkrtSOLY5TtrCC(UQ;
zjhl|QrnsGZ(~`V+Ww4OSgB>4wPRqah*v+tP+MM;0?Dhv9`Kg|D)|rsvu6y%j-`z)t
zcU^5MGJk&Q(DZeOWusi)IiHp6FJusz^RTDMcJI%#c~!yO{)<vfmxo-@%2wRzp4;Sb
z@svO!OUl~mOT3*<?Gb2xak$TOnTkzU)m61cAKWLjq;Um5XnOSK(UB>R94gA*ojq+#
zC$6?mjGxE3I3VMRu+rad^{tOP6@#yEsRgby@hKAec;Ua1WJ6YtQBFeZ)GW1SlOGmW
z&i;IJ7R!a17J>ch`I|03yHgT)+t7rw`cl)Sh~WM2=J+?wShR$pV8^N-Pl9GyAC!;s
z`l>$TnLop^Uz7jWFkYU0?~0k_*Lk)%f&9^1l*@AusB6D)V$&?~2<=Kb&U}3Gw$Kn2
z2d0SQKe9af+4;C?g;^Gq21(8LVp=WU{)pGW{l=R1oP&yYvpk}Wzxc#N1ua!bbryN0
z!musDpHsL)+<#KUgvozO=O|nh_fA`MWO~6vWu2Xz7L0DHEk9q+VO8w%*NGOHvqM;Q
z?t!kOGTSDv3R$YnFa1{ZXouevmIY<JuAWR2uK3T}vQaO7(%LAWvWfr0&iGW=1PV;~
zmZUIqp-TF;1lM;bI&ZyPpvJXh$}<(mDH>CizSqsX(zmHjQYoyZQ{?8^+;?wgNPft3
zbiDOab&-LnW}wdn`NG3j^mcAG`E|r-SH{Do?PbCJd)~H-9(tmm#PnU`uT${EEJgmh
zh=YgDZCUB-b%SZQ$n~?T&phTOS?7He`DUfz6@J)yp<tktHbcd?CAwLO+>=xebk>+{
znr|rV(6!mbW^2OAqes6kkE#gU+$1qY<hXZ+wpf(y7L)!(7o0s^GSg?B*J!Mc%5e9#
zwFvhqGu6HC-RZkM`RUwaU+S4&y!5Z1wR_>a3Qy}}=`-&tFj(I>#JA4l^Q_xv(kyQO
zPvTDgu!22z=bWz|y;*-3q%tU0uHL-SshQDaWyOU{2aCQQ7Ee;u=14c4^Zm*yCDS*2
zt3#?*@n<P9Nlu+NE1+c>*YYJDg4ZT`Crp|=^F>CSyjgDSJx}4%qQrz*x;Mi3%%(It
zp3#WX^vJBXeR%oS(jGBm<r69jt#6gSEBPGK^{6Q~vObmLJv)%6N#w~4Yd5yzZ&#c-
z`(sBdgH6d{hspC47Y9V!ce9@|yQESY%5|kIX8Vp?Z}Vn!amPhJxGlD=z;)lxTFGz@
zg$EgICugzdwTDe`<nCshzv#~4)v;HzCcf8>iqQ4E%Ug8m!u%%|l|irWgx=b|L`FX<
zw#t$#N9#txe_p>%^TxGG$Nkbv^iLUhcNopo>f7yV$iCR7R7GULW)4sJ$Ze;u2ra4O
zD^WKxdvW?e%hIMz9LEFMg#(+Ty316~A3wAE-q)U(Cv$6J{SwQA<AN70*YIc%SzlCC
zk))NX+uX8DUv#P;>x|%(pI4SU9F3gW`Kx^Y_D5&((l*cH_OECTS9{PcZsEbUZ?eVY
zMT-qvpUhWI)%Ms{yYEutr&U`2e{QY(^mfst%thG?6|SxGWU3WCu;TnAmmBY0;(s;8
z?su}C+fgWS<o7AdfE@{uembm)Q$jt%y}YB=uejW;vuH-i7V&p;pO~g>zkbs%Mf<kC
z?e1S$SGdzks;8w-d>FYe_rIarCyki3`HT!!;oQg1#Lh2U%~$``KurFr?4qjeN%xWt
zm8boBTVG#o6TYMM=<eo5V+U)225SWiW}Ob!ngdfj9GLHHH=EEjXJ>z0)%w^d%_m*<
zleKxj{g}1B&?JmuVSG|+ZxA=z<Oxj`8>V}6&tu-osocy}EEMM4&68}u@ucO{2M22n
zcW?G+x_)T+y-ceqA128}TdB_JQ#iS9^9(b+n>=-scuu|8=sRPs#X&Wt!}G45T-|H5
zByfSzlm#ni?lNLHq^bLRo*uJSH-oW0<DUJ(4CV!`mUqm}jrW+Zb}~t}vDD#UY}q5-
zxjF1Y!`ABcA&Qp4nkKt8bMJCC3DC54`v16dqtxoC9X8)N7kVrfjg?w`cgA$Z+nn-6
z2G@f*UqvtH2<DvrV?$^%=du?&r8up&2<*GNSz^J1HF3AMhOO>SeB9a1xOU48*B8;<
z6Dp@|@H8}kWjF7FT?)@U>#Uj9MY}Ajdc=P3)B3Wlh1*=$WN}W;<eQGYDiZ5|d^pgv
zS}(q1yO7Shzl*n5ALgk)yuGga__5~A&Ds`C#uil}oSZ(K94?)#J_lGz99Vk}SX^c@
z`ruIbg{RkhaX?a^;1j8^g8A(+tWgynWf8p_llY%6Qt|pFP~gLv6QwpmLaCxr@L1Hu
zU>@n?UUT^_@0D}WdzvlylzZ;<oQXRoFx@efe2^))<i9uf14}I%t7CK4a5rSJ${wD(
zz=_jf_n`>}M$1&JZ6vKlYs6Jota#-(iQ@*#p2bG-7p(P*o%bsnACTU2IM{g89%tL0
z<Hs&LzHHcRs=RP^(WZ@--L5_^n%TTwTU@GUO#ObsrEB&UZZ7Q!Og<hty)wIcx4gER
z7h$_kt3&CN+CitIb3GO*ap|a3O*j&@rgkQuY~s;rIg8HBS#<hUQ}aew&gwM{lHCcv
zXU6JuFeGqqW}d0c*c)UrdrN@Zk~^JminqNxt$X-|uNKRsjN2`5tTfC{$r^E(D_n8B
zdPw`K7XvE;!}VKCVJEi7U)q1?kzVy>`?5FtYq=d7iucs>cyOi|v;J4{XZhmXkTREb
zi(uVu){cl19!tFEeNf@&nHs=#(Bq+0RHes1lY>nW{S!_*^!BLzI%2`|rM>JzQ_h`3
zDjNi*R2YbdbW8UfoHE05-i)(TB2LO3)Y)poEick_;p4$A5dp%=R_T)s)0jEcI8UXR
z?2=-h*QnGg{#$T@#Cqx5Jad*fJvrEON3r$h&0|kQ%*{NxpNlwMHwscI)K!_Nu_uU$
zb%xjMFV>bR-ZpE3s!MEiz8*IA?mXUoU@nLM@!LWAp`3<iFKu3O_+`W%6X#V-pS1oq
z?*3~z-+7};6z9UIg{_NbZt;4|=f%8I>eqpnD-`QDurhM;PCT)C`5kSgq`d*&izfcB
zymIhU$Eh05;NrP+R&*+z+^hbJ(=S){k+R+L69KCtx@K|gTXREjf`i@DY&#t#GhHv=
zDG|OZhgVMsWV&%{!|h9FlX-gfP1x2~VwJXM@tr1~&(16InDyl&FXU|Eo=|x9TFBa`
zCf1x4a^;+y&Rk0-cs3|~@~imbT>a-7v&xPp4u1pQ6LmhF(F=~V`dn|45H8=!Q=!{Y
z5_`QNC7f|XI7iI+T@wC}CY{q*-TjQ?M%$isf7i_FJR>Q{bE9XAMgQ$Rn+tAC3hw0J
z$gUoE?d-)l6H6?&Xq=oBBRlQGvdf$rCuXhTTRU-PiK}?#@|ZcxvrT7;7#~yh;S_s%
za=KvT;-ZUF{@++T=k26N6M5!zZeQaqC(qmaBqvOnRZ`VA>bljY?H%09Tf0pfqxNiC
zVP1L3bYi!Wgz&M~LB^(wi(==V*0?O~xjF0xlaaxKojvlFYdgI59I?K7#7An%spK2K
z99=r5^6jX(VVlZ+&C7NQhwF)yGtvvs9IoO#{PxEAnl&8U5`uU4D$JZ1?L6b$EZ0eA
z9jEk$t~c0s^{3K{;D&@bve&K5)?bh)QI_1Y!|wg%$O#rb8J5TPF6hs^GWV2(*0QZ~
z=S#b9PhF8|YL-<yPuh}~bq0gFVz|SvTbm8;tqbigDP`7sz@X=F?~=fg4vkrH4`%(D
z<MHr7+&V)|hObhLrFtG$_^$r<Jh9p0#LqA6rxiO`efRojJI=lkF_}l2d#bp<tL!9K
z&eOU#w|sDva*SzH+_z%EL<L(RHtuV4Lpdc&B{do&=WP%Y`>k$MB&4xr<4J>=vu5%{
zZ(v{YCQxCH@b#YVr7cnFrYh>OUV1!1-Dl-kJ%oiAI5!@3pL*kBnQQ0zy*<Uc9lD{$
zF9o7GuOv3kcAoX1LGuB_I^W9&gE#AHsP5|#VZ3nJPE<Q#WwiOz1@){8cF#S3@QsCw
z&5_BM7G}44t!~wLIgv^6kv0#jtoY999<j+>u{Ztau8Dpeua&f@>1X^U72kOQ`_@@R
zzMOFN#p0{WQXT*H>WVMzzQd(ARpHnp!7$PPqCCmDkJP^D6)GF;Pu-t6l|j$r`YDBb
zUwBR!-gKXEA)G0MvwG`w*{k7|teus6!%dnyjXt=WXg8Y~q%3l<FwbBXn!#YWHC*am
zP{_TY`ak=P6gn7EZ)~{r$Wbibc+rx;4~qp;eZ=O7Ph8W@pZ$1?#kxsbE*!UA{_wWt
z%SVrAUYWS!L+2`m=XaB@R#^$(S;)<1*Iia8B*uAartPlTtipE$8`f|~$;t8vxp9h=
zX;=kzvL0U1z3SFF=Mc^S2EGZ2#zHoyv>GM)u1}c}Ykol8Oq*Hp5yN5+2ca2FYjT@4
z6&epeZsI%<e_p_a_k}>%!_GaS+<!7(x|gi%&5oGX<$vh^l4W5Vv^O|&yT+cmX3V+3
zY?4Zl&e30zzNMyfH|Y3$<(Zr|@v6)7mnT-AYu%SAYt?)5?7g2e(;uJJ-niIo#o{mt
z*2yo;<Z@XRMRuq7y||kELhR>pBZcPPYm!D4_re}9#HOW2mpwEpVBYbp$#{3Gkpklk
z4W-C?O}rmof7;V(CXlr6S;NT!R%V^%NolFlk(;A4Si?QEKKpVu-|0Lz?Z%8}Z>}od
zXNq2`>+x`%_Y3)?o$O~jIehP25}7M5)5Wd4Z>~?;lqt*kUAuEdBvnEJHA4c!UI<;C
zn;skc<dVTVjWrME&FzW0@PPN9P?JM;{j)8XPPlDW=Dm@kp<3ZH&tYEC|GmwpJvIwz
zy!;qu9J$O{?*_xB3}&N(IH3$yYlY^+YK>p-F)7?@lH`2Z!2Pmn&kKcP?>9|(Gdm$;
z+c9@j?b`-6q7j|)u07|Q3XW|1Jtd$q(|yh(-ctoXbM#zqxA;o$)(^An`lKiymAtjU
z*8S^FPg~O)(?ZUhOp;)^Xd|}x{Hv`?9lE^##>YwD-6GK4{_|sqF7I=lOEapUPcis(
zd8?i3bh9UJ0<LXB$2lf{Qtolr<FNZ=xZ^;sO7uffxvjqIoT_|N4f`F;H@q<_xM%jD
zX?xseqXYTj8qAy-%vLf<k$w{+#o|uHz27^J;kl{o^R3K1b#eAC2fm8kGj4uk$uCs%
zpEtF*Hu3mxbGDl~y6*#AH?oV&Nja}jSSaQBy)}Nu>|Eu>E5>Zs#4_)AmM1HkFY&!_
zZvK?W@bI|`6AdRGy?JTDp_)&TZ!e@9%w{m;Te5QPyVblID|mLV_`7RStk>0Py3Ffe
z7}<V)_LRwz{pGKFO-i%gPuf;yqTI|GQ2a}8f3E*#qYoM30*j1xa71on&Agv&x|Q|W
z-3Fr<?iR|cHs=Uxh3;T#5^bOB-Qr>rxzhO8B)$ICDnA^~?6TN)G(_gT*`%Anix%B0
znKfaJwB@Vy7q&jQy|?b;^d;vy^TN(<xuFz)Mrmq<A7jV0YfC~)-E>oRVsF*B9eOZR
zTH4-Psb}AhS9|{-n5nhrll<(GzK2Eb`<o-*W&i&6SyQ3eY)A9gwr|g-Ixn8#Y%Y<s
zRwFY!gZW`Dt3~-6vk6bTAADxy`Eu$3!{292LJG`g=C^y@zG%vyH@H5-`i5lv(Y~2=
zE?<7E2)(fWhg5%5$DEGPG@WhTOsu(`rs7|>cDF^T70pWX<DaX%%dt1m^put?&*K0=
z>yyFZ+!0HbU(326YaA=~dy>diNiCa0(+>*GQ{2DC&aKudVM0J`Pr^Hy(~HA0w(t9`
zJ=I(Jw0oz$moTRWYi;aiT~#Gh^_RK|T8h((D+-%St`?g&aJqc|wfg@zhw|p28x6V|
zjYbu(jVITcui)9B?mtVTIq3fk=X>HO>aQf4&wgma8*HL{+Dt>0S!sjsnfZ0w`J*IS
zO%+z&?U>!PVitQ6zu^Sc1_6Zvg@a8@)si9?^c4@aaEgn)xxRY-{<>%`FM$~@2~E8$
zQ%j~Oq&7|AWm@jzIV0#nOQX0fmx<;CZ^xNTDVOGGK6rYr(K1o$Ny-GDL-XC(%N)6`
zs5>rXI>qsXCA1-=ap@@z-K`=kg4-EbYIv;$9GX~^wl8YEsik{+E7SFDilGIQ8+=_@
zCrz~W%9CcyW9^H|UY)QuZ0!<_4=2oWnHYmke26&{*ucce_3NI}i&uV<e8P8jWEnTM
ztd5Q`o3L?ntMkn0b*FqX9(TFqgy#iq`(LQs*saZ&aw<p9IgPnfH}%#H%Y8e)7hNxB
zyH#j9V}UH=Ro1xe?=JKncz$;$cZFWvg724>M(0YhZjReu#r*nWlKKKU?Z7RsehNEy
z3+pn9{_QpDQd@l?IM9AkPf&tA`#JRnjnzBUXSN7)xd_%3X)e1FCOuWfHAQo+ibIF1
zq3gpI@rzR~$NF<R2#2^hnua>s&y08+W-?7fk%>v+pt7TEZlZ)X+t;tx9BLISQ=D=*
z4lHT$`Sx^4KC`E$q^66g=7p)w+k(BV^CjEjgd{m#mN{~1EV$_4wMH{wme7WsuF~Ib
zY;sLv+K?GAk7a4+@>HQU9CLN^ayB|@&+$FLnxXpNC-6{q@PrFqlB>5Sv5O0@*%l>v
zb>f6sx$ABudY3HNsrV{knOpnZ(!Dotta4m<&9cq*r{ENM-Ng<&5_A{!1p4c*?oi~?
z{&7+Hn3nL*;PVoTFDU7_{#~P#W4m>O@-ZuJw`gytpBJM7wzD{zT-bikP9(5(-;ZE#
zkyNQA4;`esOdLCuw`EQCREtd!W>>w`lGL41dNnj8Wa15xgKXTPSAAj({si}go;)!v
zER6F)Cqq<X%*JCZkv1P4<oZp+m>4QvX!$rxeGzxmTD(<EyGXU<x`xY1SCLb%es1MF
z%_JIjT8#5e=B7x_3s<^y%fFrsOP;DJu)^V4rP^!;;evt}1u_5cD86)!{!`YTo4TVg
zMmjFs{oCb>m20!gZdRsmx_#I4I+tXWkGfvnn;yf2My4Jmuj(}!m%eY{bBYMm{lgh>
zzvkbDN#^(0tl6}{{=lL+0WFhScVB(VXFGe1nDjyG2{TzHS}to364=7JH+t=nqaqWZ
z{wV7-+Gi1PWaDkQ&2M&dPq-3fTjv@e#A)+QTsugtvq*c5z*e!z*>_ou<LUw}1-|$u
z=?hE_T2-^u`I2}3gzxnm3sp7Ox~y0mAGfci>Vlfq!UmxNtpg`@OEj<DNcsKPv+B==
z?o(#VZ%)y+Sg_(?gKoo;O@0i{Tg?`1@L~`YVW{AmG`I0eZj<PfxvWc_xpW`=Umh*=
z^7y)@sSDzIoOr&xaNm0EqI}qeWS&JUT3xOs>M}ma6*0MURU%A-A?0hbz4Iv<=bS4O
ziUgl@ocPqP|3Zk>XyeIPzQoBw9+xIHu36`1_V8rjm#rdgXEhX=7I*dZDGA(q_^yY!
zaF$eUxWIi?pWdP?Tzxui%8d~@PAQ%YZ!AtY?wFu7p;(bav*6hD&p+MtI&7vgumwwe
z;c)0OlbXdM<iJ+FK~SUj<`RZ8iyCT0Q!Ep^{nAukNClgzE6!Z$ch_s?gy&OytvC2^
z+>Z0~{`5d7k44hIyhw81*)5BBg97I7&~WPCcU8&Z4TFrXhlK*8ig8qOg8Qbhg`E4=
zDmnaL{NlKw>q3FXKkafiRTjysZV~9ZvAAgBoB*L^0YcN~uykt``iQY^(l?&9rr$PD
zusg|Q>HHlV=WN=YEOC>G#V*L7!BvTMN?*$+9ZnTb!O)<o*LJ9!3<+U)Sj;hP*^-NU
zI2Oom`!UJkR-%Mai=N`I#Xj?GjCM7i@MnzP!Kyjwn%XHJ!(|z>4zY!cYH`{HG&!Z7
zbLh~zu-J<&WZMQ;?JW*mB~RUpgajKTYPNQzO0Kp|{-MYjyn-`q!iCxUqE1*X`>D+l
z>Y>Oz>sru`R<WXA?J<v)9$8wuU%I5~GO26keUC?x=c`mx;=hY9XjE=u)%(%Cz~SKX
zq(&z9rZpOxe9ix7w0zpJ@${}P9Z!j7tBG1O&q=w=eK1)_^2H<RIgb7lb&8i<G}{<-
zlULVcW(=3Q_GAX<y<AqDD{m*oyo}`BnyY>A?^4|*pJ$xa?X#Pik!Js^%=cT*qIOZo
zg;jqJ_JwYEWX~P>&`Iz4ymZBj{ROUhQ5RowU-2w5^i()07Vtg%n&gqvx+jbLPfVC>
z^kfP5+Q~hK>#9RoT3FIuZ{2ifEwa&_XjaV`n(?*a()61~6^mGJuK2;VXHVk{{@Nok
z@zx!S7&P}x(fz6I@n=<sRr`yp_44-~uuKm!zo)iD#I@<;<!M3(#gh$=o0@U-#<FU?
z+>yjR+d+JX*cLwBfX!_?W_i^7&#;KC@>!zz{0#r52~z8nrBY%ZNOkBRc`C1#R1}-+
z^f73KgnrYWaHr@qg1#w18jWk*)LBk)YWy$V(9~0ykXgtjrD-DC`jxLgHbdw5t_M>Z
z4youUE5|)*a7y@ZbxXhN$?cY{OFU*M8tLnw7E+!W#r@}FUdH4%OH$PgHa<MGiRaM4
zd{*Pna}L@ZzuL>=pPr=L5d6z%``$aMKDVZsHGc`Q-p%K~p<P?xxkoxrw8Oj2uQqb2
zO9vMnUDM&TVsENP_jUPO)8eXRg_2Zv2*>k9cD{0HIlA`X<#}a(%B?SED_P(7>our)
zyzA-<|D5ey9ClJ16L`)Tscc-Ne{Oe%G!x(d$vUYo^&?MZw5&e<VQq=X;mXWb4^J+&
zDMnHnA1`0_zMt~!=hG7HkE?q`6m~rp%WdF3I@3Y&-G#^7*UV~Anc=kP)`z0-yD}>;
za*9fC3E<8>9=e<RV&2w^mvnXWSFe4tDm3Qc$)^jNg<s^QKDu&0n0uREznWR?{kC7}
zkF=XQn!oMhOq^NdU-I4CsPfonhih^Mt0bh(2Y7Nxr9N?U;ZC{WxwAUp)!gqZ9?0s*
zFgCDkxsXva`;6|ZiQKUg`2uD)R2NRa_Axp~ar$+oI_;%-y^dzab95Wn%#5;_#oyd+
zbDo;`?1yPWk%dBIMf>q-Y=&j)Q{x40J(g=)qH<KsiaW09e}(y;hf@El*_>*cCu~<Z
z$j%kR`e<IJZ?~v^vTMm!12)YJMXt8#OfNjxgSR<MRFSqQe3h{1{LG$?eMQqI{Q6&d
z|F;^;xBB^Ve<P<<3bXR-aCR_B%B@maU~<0Ex8KUJ-|1k_Ny&(2Q<+1R{!RfB?mu0h
zrrcOlc`~{rLcQ_Eyc<_q66F4GX#MkmbCv^Z#Rjiy6>W@L9VM5<l!&-5FK}2~<16WL
zN+hIr=?0m%Gd;RL&gj1B(XBbvU?Zcj=tV~!7N?$WA<6sZxkm*{XF8fJWH4D6ZFyIO
ztHJgM(_Ei!PtW^Pa}G)5I8|pdc62Ue>G702<muQG>Ga0)LdgP$n!EpZoV?t!=H{=h
zH}^kqnfukH`@xCZC;OxwBYHiq&9R&&r^+BS;r`#HKG6~j+<04moN)bV(XV9ax!iH`
zMH>Yrv6g>-IK_7a-Mn$k*QYUO$3d0}ju{h@3JMrr>}3;Qb1!{Q+uDb-0_MiBHXfI~
zlEgWwUHU`N8{X&w(P-UQ*WNswbp4K#wgsPB(237y*|%P4`qInv^(n7JR=h}o{P&3q
zP1P99W_fm$O8<E2xnrihSnFNCCGj>p?_2KTeX}RpQNY!D(}_I?=lxN99QI<i!;9Jb
z6d6BmIJnqz$~1*JlXPU7Z6%sZr&yPTxUTIunR~v)WzIC8n4@cZ5)%}5SH%CXneS@l
z@;_;Y=cOYd^BNKyk4%w{Vw3xIWx*OYgItCezP#dpLKd<Fd)+uDHY2J?rOW1Aa_^ft
zB{>H-$;w_yjlAqAd+F;E2N%VOzOuVw5=C!0iRt<OoHA{~*6AWW3nkC_hhE|rtxXB5
zW0|ujq=VBxu2D5Ua-w5TvgO-u&kIKqezGL3oGY<0_C9Ot<ux2{-pcM}eRQwT>A}a?
zZiQJkS=WAw9$CyO()gC=%#?mNT@lF#JdOq@CoGxfmV2cn;*nyW!Zy{jUV)dyVlJ$@
z<0xIhmf^$uHGo&Lz+J2DEcdoWIT2j1cY3{JWaZOq&CY7Op292XaqzUR{QD*TUwq@m
z%M`yQ%9{oy|2Lbq(5x@gO=0<qf><@9<+G*;@7OCGwdAJff)gdPP6Y00sXZC^=jpk!
zTTgt&Ts|-`GSwWdP-<j(!IQDHr%GtjWRC=A+w)V7w6^FjU=mVjHR+GGc&fyjX3Zh|
zan3^aPqV!mm(JF4?Rpr(ac^2$f%^)9u7x)owcm8-JP7Yzps-?!6X%~}Q5siEpTzLJ
zo7U^=!SYjKO-+EcO`<c$Ly>hFS6Nj9q7=pSo_^ZnlUbQqy>+_B3yt`~rN1wncgwls
zTdCT)^JLMY8x!hMci#zd>T!wR*kXC?;kzq|J_jT<9&*&KoqKS@V%J>tl){znH|3*W
zsP2_KD#s_4CfJfTk*T_A`TvuDk4Ej_X%n1u#4stwM$LI5lj^V5r@4*{DjG&fX>J-0
zOlu;Z-))jUBa;!qCE0WK`W8pc1-+F5kM;k~a?W|S@n)Q($ZY2q3v|_3Ds}`FCTcC)
z6lCDaI9spP=vuS>24~}Khi0AYQR`_GYf1bTnq2ruOVaI<SyGF0hEheBUzEdg=Z@$p
zm+m;^9C+Pu!u5*e!mX!vUp&!eakiZ4b=ZXG;cqw?GN!S}ESQ#X<M7^er$5Q|N=qEX
zjvUr}kd)BLpf)LiWm)2%Gm1e^T;GX`gg$eZSTWn=nA;TB{l8Z4=X=HS=FGXROc^_5
z$`*Jn|9hqV(F}(#H`@2{s@^(u(CCuG4#)onv8<9e!nOM5v-B_pg~^IUCEDsNovzcL
zbnk^)%fgU%8j?k+p;fYOyB<#3$1i%Y#rvVQUv7fOED!T(H?2BTGsIkOOkMisM$4{^
zs&n2>zMsn~UzMr6N_&P>i}YG$@heXct(ot5<H@3v*;_ZYCbLb7<8j??#;;r_a%PT~
zwg}t&Pkin{ZVW083>pk~POY84FL&mQR)!FvnKN2hzJx64V%KAvx?$@w{jF!0&g)G{
zY0HmvKPY!Rf6BcqpMz{32iP<Yu*B$He)BfXGTzus`KxG<N!1CHx^ru1y(@f``$|oH
z_nLM0`fi;0bbfEu+BaEqew40tW=NAcb$x$J$L3GsDNon^FIRdL$2RL4Q)>m2=bTgL
z8bU*21Xw>+_#bo)VLr9|idIC_ljYxh7AQyd*B^WB_H50Wdz`5}{*gTX>Qh<oygL6w
z?bw2U8h=CF_OazGeIfQp>FVK4M@x3Z+<dYAsMeLxDNLLNjK?p@bc7!JBV#b@h{(de
znEx}*`8|6f;rI0St(+gXaynZT!p`NiwJdPIF@^no=lMmfZ{KA(_4G~rr}Ji-XMF6$
z3)578Mm`reWR2Q4v-OqB@24%2shLq<#IK04Y*l0VD8wT0S^DA3Y1g%q;v23BMqimQ
zCHITWvo)S+i3+>{OHST7mwQL=?EFV>?>);c+NKv(F)8QGqJl^PpO_12ZzBHd><qfS
z<B~Lg%bG;@gU{MlZG4pI`_99I;aSqV`8Tf3ulbmg@T_50tEABK)5{(PeR$RI$zOV#
zVzP{e+N2F9c24d+bNb?;;H160Th>Y2muAjTGW2=V@t{{dQslgY#GQwdoXN{rH96)S
zR7^PR`Z3Jq<kw9{cNs0yHlK9#XKUh)Y3zrm>|gZ$l8WQ+uvGVX4_QBV7Ekj$t9Ew&
zAH}DY*N)6Pb@12wqjs-erFMy?Dugfc?2JwIs7o<g&pGMX8>wX*&Mf=v+N5F-zp-lX
zX`|ztC;fYQZD-eRmx;{ti}!5Uz&$(qB9oXSk64mv&wh)R_xqmC743QNm0jL+@ryyv
z`wJY7w$uL4Tl(^ZM4X2Hjj#hN6+&k@8_kG`;ZIql%PQ4-=Y(-lU7>>hr@%^~5Q99+
zgC8H=yS2tO<Hi=oCr-z>gWj>c+4!t#l6>$IHNK)(|1tv)yUZ#GQoe88X!JEWd&~BX
z#;p--^Cq+;dmMYc<ND9Ne%&Qst*&v@?saMXXAotwC0#K8!OAP)PZmF1$+N4@WMW9f
zVZoL!w~A`qt~BmF6D=bV^ZMLN-}6V#-kLp2M6=~c*e=eQcU~8*3%R7>b}8e5&3evR
zZ`w@MkEnYIaJFff@<*p7uPMmMStw%Q`RC+`gZh7@j&O=+Iyr3X5Y2G&nt1yb2fMLx
zQ<P!0{?WbLu7uTmnB+I{ztpC@3tKkqv}EDYKX&6~%j}0NoqJd$3Z49R$bPGP89Du{
z`tj*OXXO8ty8I8#)+sJoY^iGfi05p=`LBCc%{@2kYrsc0AEQTwlhmgwtv{JmR+F(p
zEJw42w=P$Q*U!N}eVsh>jzH(qYT+XFgGZFT*_yaBdNp>eHmL}ymfo@VY==m;oKbFs
z{@ar0efD1^m#j36dh&RN!ldSvmk$|U`Fe;a=8ycTA6uO_G#+#<>^|P3Eq~+lE2I3G
zQxYZ0UgV@q_~e=TtHN@1!G!K7-}iFPig7u-?E9q6r;hGw?1??f&~0X3y*V{tTj;el
z6TW4wzN7u)UCyL^n{R*XbbqDe6I!rumcjqL#S2reZ=U%)d;7!BXT83~t-XBPW0HH5
zj3bZvEMDD&nJo!_Pv1N%+0`JpMQGM#zB7FK$7Dm_I{$8o(%$93;J~t=FM&yWRnYm>
zMvop%6^kob?P0v=^n|8uAFg>yn<-20Nca7u)|=v+ek|dR$&D4a@>96jUlhv6=LN8r
z{s|LO?(b9fKBjVCJLo@$lHHHLFSpHdo)!M>*Nm+n3g$<6UpU<{C!%`c{KSp7OW$3N
z*%zNum~P4Wu|+0Ch{ZEX;X!ow>zL!Laq1763_ZVCoSk4267#=QTW3i^@p}i6GS2ha
zvlstN+A}Tb_qUmELL?_N9h45x3aVr>w#gFynftu#Nb{2ao0eAvEv}jrR>QIUsQanS
zNA%O%-fVN+qIlx+YMcFG^A#*+uK#@Cvhl=oI?T+x0dID0w9~#BS#k4cEZ=38Mcya6
z6iWWu=BkVH?3J|;n8LHndB<0eTRPR*OKmqUS!k}D>7BU6(NOtg_4~EYvTCor&wb;!
zC&)bgsPPYWHQSj}|K3bbQte%D#m(~Sg!WI_MXyv=+x`8M6u@Z3Ra^R9SVQ^o+b<sd
zJpBJXm$<a`N9SBwBFj0COYMN-v;T8nD<$>EUN?2_`r<I{yj|aw-ANw%7qd>vTd+5Y
z<@(;wDfPy)rfi=1QrGG==MjIK*PpkU&+|NTx~yr^&o{rz+>h0{F|p=e-}!%z@#aZo
z@gL4t+NAF^TK(buezRlswT9DMTaJ8KG(9$I(c{FG^B3!f-A=5RF?F>rIuu#{fBGir
zh-^(6#aW9|WiC}uTe`hx39FE_PF!q?(2tZ~E4&TaB-xTW_8jRpJi58KnYo3Dk(r%K
zr9vRlsg+w&%|zotB2z2ByxEot!Q^AzqNZhEUR+ReZBjD!3dq>>z`aRGNP|Pc@uUc&
z@D!do4j-O5&SYA`F=Nxx=YCDrI~DFYZCG)t*^|{k)FDmtQe)~42i?yhE03<`+?^5H
z(V#GiCFsT$p@eA((qWRfd2+6PQ@FL6>DDI28(krX!m`<)$({&b$FQ&IHs3jqwvfo(
z0sjTJ8{S@b?O<62tBPPiu|itEil7(EPszp4nip|axpnMRa&K8BJS*-+)Z)w=oTm0F
zYlTveUtx~+dy;!$>p_(Ro>6&wCm6rJbHd#`rmiT(OO;DR#b92s+POD&#ayy4{yw_c
z_-HoQ4TE{izZzpMO&34ERAc#?h~<1wwuBxCTORoO`}y@7RyQ(<BphgA*wDA<r$~%|
z;^s41%~LoyPkB47V9R=aaHoRFX)#8to;B0Lq9!JpvH9xwHVE-I`I=3QuogTfVL$1G
zYfH~-pJOtma|~Ul3k5XU%#wKZQbFnH<d@6m*9a}JZRxt@e^C9jcNdQq=dM$${+FFu
z<?veNM#{Pa7oz@mi*P)Q>NvxB^Fz170g<T7dOL)yJA9iST)Cip<V3<zofQf#tDG5c
z?eso!ieuMWw{C-KJ=Lo+u^bm~2U+o&Tq|qNSmH9(Yu(PDc{VSCo7%eHDNL4ld&MnC
zl1uc8h-lK)&<Mv>onb!4!6quvrK*`tni0ERsJ+=0{prZ(bJ|+AdfPJcyVV~~-lX_b
zZ&Db8pM*2Vs=o_jUhK<QovP8uymUjs>HVj21Dwp<Bs+KvHl8}IQ^7jhn<2<TMZ+lC
zWr|a9sOi%;3+e-H`rPjo<#IeI7M-{LexqOR4VMndvJ(og)I<*~;S`zS(8Rm!!;I!t
zI~H`U;uR9$Iyqw#^ZONZJ7oTAIW7rM>tC5`xa#Kut(&bv-mM>$r{5`NcWK(LvSdTJ
zehZKE3B4+PAMRHNbS7o5?=v_vS;Xi<l+NqpS(0DYoQ%4e6w+=UTKcd*LVKf&H+xz}
z?tQ7O=yx6Jwp%wnlHgsZ7<oU=_h6m2pWb6lwHXRej_lM}$a73KU#95T${>al3)v<)
z-FVCMzwgF5Lm{r`U#G>cw_W~V^|un%#;}vmw69AS9Qey1@W6pZ<wfSpE!$19&sM3&
z%#F2?b*WmwXDw3bay{hA+@GDZ>*bU?7oOi})u?q%;&<Qj2MNC??$u}h^WkXu<5fmJ
zFPc7n_I`1Q#pqPPgPl_jKJNTCUD#>U=l-cW|6gfnF8te(7St(O`!Co|*O0}Yn{$?1
zg8KwL)x?J2(|21QDKVJwwCVdMyIy{gq~2Cg5GUI+nRVl~1E!h23|~8X7MxR3+H~=K
zLaFwNlCZ^6Gj`-EPvW}4r^!0Y=VyV*rOTrH56$*^u^K9FXcFbM|2$!n;qt&4o9?X>
z5?XUuab2}y^&K(gbcd-HeLOc-$Cu3$UNdt+j^~fyfT~GHmwXaf68YDAuG<P%-QN*M
z`aC&ho!`7zvg}MC1BVBLz>)`g%BM~^?n^$bcEX^mEHWcjOr~|uwIs1Hg|ih*C#G|U
zZnkL&?BNdPNcB@{GLy_=X<zqCMc}#E4(C_TrWSeo9Zh5UzduT@uR7|_y%R^m**-lz
z&Jtb7_=_chiE+w;8O$#k8v82uB@404+>jh2z3jj&O^-x%uN@~%TbpF(Rk*8}{_LFC
zv1z~gJYnT0oC};vrp;ecA*lGnQ^B#p-J!^|z?LuOYJ|=aH6AVAmH=z+7_%4CjaMGc
z<TljcT%9Jk+ryV(V#os4`%R*@wrT;hL~e8ImrJrPe5M(*Lss(H_dP*+I(b_@UcVoa
zWiY*Ib)NaAg=tyNF3lNkW=xB3c<Gx>KAbwyw)=I`FNO3ojS^EGPvo2kI}pyLr{ci$
zS$n1A&d+lbD_u06pG>#gYPD;b$;&w(4<<Xk`f1$y=A#npBrm)4as`h6$Aa85RtnD*
zC|cH>bHce&)}!OtGzNbyS0?GhF-&S3T6O=3uImhalcetI{W>w9N&0|Njn{_LaZY?Q
zUMQz=PRVK#nsa{hF_By&&P_gFbT`k4=1i+Q6JYltRBT(w1BOQi;j)pksoYx@d3q*R
z_;%;`Pr3hce&;ny)yHjD8a~hIw~19>@Jw!liDyT{Zkgop)&;DUx;`E=`HUpqX{9u&
zq)5%yaB<`>bMdcPba~!4s}+u&D`ZnP^!)wccEm66U0cB2LzQx$HUzH=Y7gA<o<Fq8
z>y)NS5XZNN%UhCzgI)@>iwQ5<uTy6DI`oDb_YS2;O{<PB6cgQi%C;j&(CJaX%J%;a
zO5SrG8ko<0siMfp*ip#95Rlv{b5uolveTJUI|NM6J-5mA5vys7zSI9@qTRJc%jFlU
z%(u(qKFKbaA{1~#{Fs*Ox=K!s-C|nn-z5nt{#wOaDIo3K^HJ?MU)<9)A5Vi&xBle~
zuUF2TAO5VzWru>$xf{n+CY*jZv1fYT3#K1Uo-Y>J1=q4TYNapKY~Y(Q<H%F)W!z`N
z*}R(bwR)?A*tfOs=*`t((EB*Chtc72!OUG(m@Neus)UReJc!^7I*_(j$VsK(P>;vL
z82-RRw*-~B7u2U{ytC#!5wmHX>Adr)Ty<BZVmZ&48tmQe;d*n@mi-atEu1%vwAVeG
zxNhsx`5OOo+}-x8DgB(r6}m!(Gvw@L;oq@0J0`7q>3uckNcUFWq?I!tJ;+gNOpxq5
zEcfiuW3C+nOAdAHOf87_czD>h|KgK2gSL;24GxbP7#1EAKW}l2or&dM#h%Azb*~*v
z7>=L2%lGuw#?EJ-u1~Zzt681R`egf&u8FK>`kD`=yqzBYo#^t-Z;}5@ZD(OGo4r|^
zM7J3%NsTH=4|Ob!`F*$W(5&?5ir-|G`qzI>Hg7vRVd8;JsTpe*#%q^--nf3oa@Mzt
zr|z^nFE{mwti_K|yXg5JLZ@j>W!-(4|9fU&d(^Qd*M6=#Tsm>TQo{%Cn}!pc7M-1u
zz-E1~(A?=t#k_~lHkV3qKKNfScao||{gO3r&TurHVGaBobzAqQANTgpht%^fx9X>5
z1(XN(3smoo(~aKNF>emrLhfx&tx}>>c0RxDtl`A6WN8qSPr@g`DIIE4nZIe|TChH6
zI{h<dnUhC?@Fy8g?;o#>KYnks-ed8lcE*GA-s^r_?THt2zq6i`qv@!3Z2hn3Iggnb
zcK!?9`~T*}cmD;F{&OrkmX&i{DCKy=+oAxz;w2A$+He&|^li6W)H}yDvah@E+2dzo
zNzq3BSO0i1^(pPykaz1`lL*`6s6UHNi@b{Cb2`&@?$?SH3K2>h>Yn~s;TOK<UCuR~
zU$@?zeR1q%-wV@<6AGKur;Dk-IdS}g(d!i*|1~f6dCk0+r!awYD~HR<T}rbg+8xy1
zzMHUNQ(E@64a-%dls>S%zY@lKmFYq5tat3XOge0jc$qsD9tAuRW7_^Kn=9$9`ZWL9
z0!d7Fa=9*OXb2sdW5Iv(66ehYty-Iw%o96wLxO3oi+<-1&CESIx7{x2zR_yF_r&Gg
zlZAGVZogRUW55-EWEs!Aj6=_q4|oK)>8Y|kY`g!b;ILg<WYej_bN$^5vh94%Xn$JF
z<CQ3Q^YDg<9yO5!iv*nyAM_t{2{0I}yJB(9M1bMRqX|;aFGxKMP-GHNwpn+qL4mQ)
z>DmJg#!9Da36C_+E&l1UID8LhZ=JT_-Y(luFKz4IDlE$RuJd0>X!9Yfs-)J4KvlUV
zd#35=pLi<#p6iT}XM9ehh{uz&5uR)-JCxU~`xR$+&g{|tH@v^^q;)Mi5vX#=ctd)!
zRWI|l1zrrjDR<OsxH7h$QGe^8J!>AP+VqoKWO`=?bYGmbdaE99?Vb&n9X3@l=pQ??
z;UC-kr6mEXk1n`$1S!Vpa-F!ycsy0(%0+cGfAuX|o=hj|6OIVH<&5Uyn^%-}a-C|$
zG%klV$<MVkz6xpTw(&U$d3}4d$R|p_#f4?zvnQDxkM{^QJIQ4=d!5Mq!ZKOm@ZP9}
zvbP%ItDNptoigLfW}b0^o&U^%8#(8gju|d}ue3a0`L%9>47bw0{sU%f!bSe4*Rrka
z7HbhVPWroI(Zes#`5*F2X7)&OUy$UUP{uYvA?LD#z@oJr0jgU99p@BQ9w=0m`Jj16
zRXyv_-D@9@hdh$GyTViNL!`ll=%jaj6LOesCn(I}QC!p`d`GJ(go!PgQ(vuPkA{cd
zlO?RCC!M1fiXFPFyu{)99QNN~D!FQxl)5h%<t{aETAIAaFU5nchDo>P%JVl9G%xma
zsq%LmS#g17Re&0Q(xn^iEAPBqZI#RXhwp<(!-uv>&Y5d8`aelM6_9opaTi)7<r$(m
zq3G(pTpO`|^(_Z76gnmzS)4jS*i+44jV(~U%vVl0P_sa-q-t5ZR8x%A2f5>uW(KKn
z?)$$a;KCEVTThzq7=2C3&phP)ecfvXlaA(HYn3-GYyP!TxBr}@mKWm+p{aQ%!h!<b
zGhVuc6`Ff2I%JZ@Si?2FX%+LCev`}V75$90<u=xY@a%}19RDZl5!=Zt4aH`bdVDtx
zq&5dSGF>q5T_MSMz=(%aW_yD0k;j_uTs}sjjQo%8R~;>iaM#+RCAa^TLfzWFzTVKk
z8j2f)m6xqK_2;7Ul}$FSecwzp40`kypL0@_oW!fUh^<*v$s_Z1O|!Dbq1iK-*=LF>
zEq$SUM#Ds%;f+txg?D)ub~!~)JjoOFY4Zj3cMAeyJhivRu&ei<tWM}=E?W7%Mo5D-
z-<kXUsRK?*6aIg4=`d9m`|2R^h)@4`hQ}ng#@vrPI$hfjCD$kk82VdHW!iq{@D7(_
zpGz*<^En-vP{OQz&0B1hBi|`qpZDv8l5|f@&V1MO_0ShR=Af_FTFM;+wPuMJa2f}&
zZ8qSatSGS5vSah_jwSn-_y@nrZc`IZelW%NJFn^u6Kgex!|Oau3I+cfbXQKwzTlE(
zTHX3H?u<^V4o8VzcIp|)uj|(u@a5!B{{QEEp!6ZL!eGS{+a6bJR#cj*c&E&c>w~wQ
z=#6a-+yWoUSQZ4o-InM$wOB1cdCMuw*-UZ^h3Ecw5&h<3=yqdkX>&!9=?vw1z7Ar0
zbPn0hbk}pbR&Hj#Ku`1Qh5wU9tFG$a`fS~wuNv3<`x=jO&9y^M+>+0<g_yW0#C@N(
zr;qEyyXrH1H7AyDZSvf5AhdQ$*TrQUw#ck?%r4^IY%nWqHFs3D)&zq>GahHxw+;_N
zH8PKD_-}C$JQQc&#CJN(+*xGZq1K?c1;?Lxb1gcca!es1p7%yll1y)s`|PIcL0<**
zA3cqks`YBIN3F@T8H?(Z-&u7()3Etgbo<CRo$>`)>wTYaHAX-5`t5Y<4wJCxA@750
z^$&%VOl)^KBchzYirqKG@`{OX_lDPe?MzI5D|Tqi=+LSaS$nY~yiM}agPLP8H}z8<
zJbuX2&fhE~nf^x7Jnym2p~uT6R^MM%ZTO!l?`}YizEaTi2<1Csx%x~Nye}1uOdSLA
z@?U>bi%XOJUlwA_RC31IHsEUF{|+npayujbFl%R~Ee+4Nb%%*7Ip0W~<d?5|P9^R|
zQ^hge^-e7%oZQO|k}ALH{^7HXeC=jBO|;DMD@WmWo^@NdR$YHnGhNlWyLOuR%o%SV
z{Fst7`!o*|%eEa0m^XAJr%bmp+QRd=^lHZOjxE;{#6*|*gt%l^a%>dTnY6=Yi~01Y
z7nGBk-38WvNIG`cB=p*n1wp~33DOg`2Ruude57vkCFMF_n{`XQ*eohGsp{nV%|Frg
zML<F~w7>EAPkp%~lgz@uo;mWBXGhkFBaOSWQ~3{jpZ)LqBIKI$ozuE%i<>1~f_lW1
zxBDMhad^U7j_Dc-DOL@ckrqd?W>iT2a+&0vb$7k#+HdWixstY99vSV@SSXltH%v-g
zsWY}RS8;33)`?55mnr@^AGcJH@yH`(zbfhCC;z!F7?~W@Fev4oRTNy!<ipdQ7oHa}
zZL@NZD`WGS-YKjr7jIS$3)K{vZaw!%hS@@S?de6Ut+fAe&JkDX|EsiMgOtwer3=4h
zl^l2DW&CBx-&U4+J<MqN#MLh{{t8c0z4?ZvcZFN`=BDkA_w!~-h_oFobvV_2iR(n=
z-Iv_gzZU%QUGV>$d&EqQ|K~NNm6n7iO`ko(g}I>ez@bfx9k|PUgpd7C5cWtEc@S~x
zhFQ(SSG?Q?Oj*6^JleH)-^L$zV7x1?!7;5^<Glh)_}nQ^>T7))SG3&t;q0h&E%Ya6
z&buJD1-x_33zH9L>zjPcJY=P;_v7oa)w*U8E@lUfxwge<_3ZwcbJ*qc;iw&#8ax$k
zEZY(%wVl1H&C2km<VHb-gxaI`?Sfa2?JDcK`$kD=_C2SPrT3?8Npz96^%IV0l;yZ{
z;O0V6H^=?DhZgT=s@a@mHu*uG@SzPoKQ+Hj%dfTaC~1;yzxCwOjuJ0-h7}36-<XSf
zUr0Z%uoKGEmFiq)S2ibqeZ{{Sw$Y6|DvRdNeV-xe6!B+sRB?Eb0{dkt_je7`YkmH|
ze*1y5QaEWcYqj<q^DOPv%-6Q6FaI9fJZY+H)ag(em*|%kQd1cO^SK4)h$=OUp4xL@
z@`CA#)-LL^U8*CebM90OKC<@BhrC|5YgfK&W|VwPlb+4BZHe8p;Ip2yo*x!i*b?(#
zrN)-ZMGe&#H#&KIR%W_)HhJlkrj4gFm+iik`Cci>zv<_WxF4JJc5O+`ozwsEm2=8A
z(GAkmq`3dcF`0x~_<INMI6d1jX!BpWbBvl+8q5j|&)z$1Dl6-%$oZN-!A<(y(+kIh
z4oL@EZBpKSFg@xjhXBKis~xhUUEAkuVJ_&JEy&0vcJSuGx_8pAS4A+m%vRWNBS*!d
zcDqH>lF${Q|D6ghKaN=9)Su?(KF>#SUR2072fe(|IcJwD{#&WY_oE>@)X8qzq*pAq
zo{6ihuN|}1Tw8f3QAIoDlU^+Qmp>dL>koe2QyKV<<q4N;N1;Y9_tp#CsoaIPOgDx~
zo^IXrVbkAJ|G3U<K9uXL%&@=wR&nsvFkP-|sUp{!#QZj#S(<QA-09iwKOLEtMd|E!
z^^Y&V%iCAo6cd*_^Yet+T}RY8CMp!Yt-C$rP;H3Klapz;cFf9^s1S14ve9?<B5Um*
zEt59A)iuh<T*3S8&`+gbnZ5#9x~rw*qd4@_t|u|API~rb_uDJnv%|Q5TWO!UZU1kI
zmQXtPwTba-#ImdJUTL_S-Tpr|CwR%tnQ{!?zYfMeec`xH@@M|EYU7vNYrn)xJ)WR2
z=||In=tkLN44i#(HHo_pUf%X~@(eRhrn|hSgJs=nqt_lfez9(a);(jU?eE3c{CHeq
z!T6?>z3JPHRZ|x=Y5#a&pQKxIUhh*x=C3BBKR=wa{v12+Q+;5A*vCGeCYuw{KcXey
zUQF4<GQlRs`QM6w*Kc3&O)=$F@G1~Ik?3@glS@F$V}XLh;Wkbt6$XQiO0En{23$8z
zY+QW2U%|Oc#&eQ#Q?JJ&m5B{0k55igjX&gbVB$0PS!UT>2RMV$+~+U|1Y8O6X3*|i
z8?jYN^kDM2IkKsCS9c1o@?=y`&DzT)+z|esQ+_MQg-x#AQ#La#QJJ_Y^|9Etg5@3)
ztz8{rCzdYH+4(xm;r2EemV$>C63<Q?%w-o>3$9q4bZ~<8q=yzedl*hnvyWO=vvV^8
z=WM5Kmb)gF%)TTW-Dsn`Ks395{X{MW(V)`f7u}L4clplheb~E1ZTqIXY37<&x7}Kr
zcfIS|0VCA~-u-^IpWnY$F^qq9sPXp{&ihL}0v0KFGJR&eq|PDKu*jvyfr&xTpk8?b
z%jegZT>|X2!`8_MvMg|y%59k#D)BQV!fRH}#Rtt%Dh~r|^+YokG)YRHxY!uO$jH#D
z7}+7~5m&1*Eo+<PQ7%cLTkJv|DhDkl^(dExENWS}_2|X_VzXapxJC)NC_41liJlC1
z7rqj4iLHfGV1|3N$uZaD>$YJ+>I#WJqlHqI+z`x_Z`7PLV@WUPq#5yAKQ9IzZ4FV=
z&L}FmQz#>P^;SWyhl87>UP_vixayRICWgi$l??V4u7($jWE6J^E>bO6GB-7LM#9V_
zmcW7wEL@r&COE$f{W8y0GIHs&l`HyERyFeNQ*BU5v$W_E^G*CQ-<u(Csa7aU-iy~O
z)qPtWgqHLeol;BASmY`q7*@J^*6MXqSM#?`^O`7<?6C8wM9NRmR*}Glqk_?OXGO2a
z$o`F37h7#vwj|m08*6@V(6Q4u+`e-;wDnE$<hr!<li#%)DRPZz>Hl+ARy|R5cb|Ck
zR<5k<*JXig_nJioN&iffkltMxc_(w*mB~@s>n0d6PUXxGaZA%)9m6KB;&bvS+r!@F
z4MKuz!v(8FDsOBK|FTv~X&vLgUpY<ZZ+R?o|Mz#w!!1gAp$Zo?G7V3;E#APauD#i7
zl1{y*_=*Kvyys^vJb!rJbOxb6ytc^~rTQ{n-?7b&ShPcS>(#|#319aHWw`uWv5vv>
z(NdMkpPkx%i23{76jKj2yE$>1ruNCDS%r(jem+}1sch?<4JX}h&5^#T)bP^vrqLW0
zm5A>z+*B>Ns>QpcZb_E(2_A8D6j6Edd$~Yw)kIg}!n{8pz5JHAW~|*9%&omO!}rvG
ziNFAVlTX$krG)%Xrq7=k@PK{lmf(BY{Fg$`)rI7)WB&B&YUM{U&ue0K%S?Af7wlcS
ze`isxiECV0;`F#j?Gh&zyfOdtX!_Jg)3+ZH^of_VOqgP`NVcZYO-1N1Z<mDHB$3>L
zX6eWo*&)Hpt{iDoUYzJVCq|i}xARHkk5>}=B0g>mQ%Sd6t8w%s&tyrxJvwQ@j}DaY
zbC&z8;*cvO829lJ^L+k`>I^=`f?>a$6%}5*vd=4?ec7pKk`5P#)^|<$TDL{6i&=Ap
zqb9VtZ#m|sn~}t$+xcj{j*4AuWJiw7!oZF=l}RgS3R;J*36!*VQtnMy;ljB?-o46Y
zNw3F){Jj4!#rE4xQu)xJ-tc}^U;>lzjAN1pMY9**>CjO3(&K!PuwPU0#UU4=(4>6j
z<uiGX8<|O4zI~RcmE+g-#$?XJIcs?GcKD=uF=)KY$YK(CFlD*T(+O$Maw;yUTvdpl
zX_poCSxv-|Npr(dzO$04B5NETtF?rjowVju*mIre^=?k)U$?$jbBmVWaPGTw@05;4
z>4#!}?|o9<dgQhK&)3hRjeaWcU7_8`>@t%{)qE05(~}0}!dTCeF9)U{yfE=h^$m}h
z&bdZwFDMHge&C^VO)Biiu~t>Z!+BC>GavmpR-*j*smL-H;}1RQHtsHq&S@Ol{FULX
z^S6WNzj7TBp6RoBR)xsp|JGBxLgE&#uen$?EBGbnd6v8Ey|W^OrQHQNU!4@Tj=#J3
z<~q-?jdt0&&pR7h_HCWmW0XH%`sMN+OxqV17A~38!mP0Bm8*om5ZfYY!Kcpk$~VNT
zR5+6l$aq*pO*%7ms)~Y&=WIE%UC%fc$S!MssoAdF+FJ8!=8@)S$?o2_PThQB=OMwo
z#!b+6!X~H4*%@7?i=)o2FM4kuR;gvpIOWRQgG)01Gv>*$ZV*ai-J<j5^zuDrEa7$Q
zSiD?s-H&IIm~@D<A+0(yoJH~TlCz>4bVF9WE?REwxqm^9(Tlfdn0G2Pep=U=B(Hcb
z(B_$M@3OXL#Vx$N-m|2Z-Vk>^ta&yw;6Ime-$TuoqmJh^r*0N}wjqgY;i>Zy4~^Yy
zmlfB#w7&YWWS8F#Uk$fqr}*0vO6D*8TGsXBqi5|F4*7oy2eO|;DPNzqy>(i|MrD<E
zi!a5oY!iGrvEQ!mV!_f1*;P*k*FN)Xic(dO+h)=+(Wc8txt(RQ+{=!W{W<R=maT5w
zcxP%sg5I3C;BB4<eGH2DzUq0hbT2)mworE#>&ZhAoHCOvdd?XX>iBdCFBd+f*%H5L
z%^b^Y-Ic#KZaK!S?U`|Ur|{$zN>K~$NgUwxTdd(Nyu{;>sHm;d!mGUV3XSELpWl@+
zX@T(y?MUx`Cye&j?v#p*z7f9dM8lf?9PfA$QHO1Roc8~Z{r_U;Ha&5N<+~^PICbim
zYd$Dqv<_@Dx_#p$AEzsKa^OkTPn$k$&F^KB*%cCGc6(KcPHxO%k;&UWm}})SKl2No
zbMnR^Iq8Q_3rdcdO$k^#H}>aaG1pHaZns&KvoDskTVA=+lzk&oX_2Sc$@C8ltVf-{
zS1RX3vqrBkTGt(Z@}%h0miO!PqUOf9G~Mc|7E|;7`J_jN`GMK?*L^;gy7L>KxE&F)
zulvZleuhuzUV-DX6-y<jOwiQvY3ZBmGU<>QukmDa^|0q5FMoy<PwVCOe|*Yv>ARBU
zcXG5vIy2tfRN-1O<6iG>Ki@}pmsib_Xg$JuVS&ZFYjZU(ul+fzDtey9|ERuK!K<Ub
z)!p&P=f1W@?(qHSe-o!k&r{ajud#22`)gM}w-1vOHz`Mp<Rl$muO}SP<o=~dWuMBi
zhKZJ;vlNBz9r?Oc?+NdN=g+6!JL^!O9;DD*B(Zu!SlO0uswur3+549<HoJVB!WgWR
zd1GUgfyC`InXP?1l`j^&J<=~B81U<mgy^em4||^L=IynrtOc8rH4{oaC7MmYf77fv
z&bvITed;rJ@#w&l>6YCSUfHJbufIHRp<cRY#*zP=J1Y)L3f8J7c_>X!d44b@uce4<
zMxesOxZ{WQZ#q?6yz1errg!qHSW;u7!F9okWE-DDDyeC4os1=C#Jpq<rZFv0Ww;Rc
z-(z|%$LBcj^z?4i=C7&M-KuJfA4JrPsqK0qxXf5)>w|`r@6vs4(ifQ3naa!exrY67
z6TPUevuJ9q_Ya-LV#2SMx43;4S$VAdl4$nq4_RBp+Ttv-D=&++T=$SO%-*WjsPZtZ
z`-=Vr(d;kIdjC{}7MF|9a*r)Ms@E)>s~A|dkX@L&RJQJilZdIOxGUT9h04nW4JW8d
zXBnjjZqI$@mhj_{pq`2MjLShw78rgoGkSYKf7_*+7wvWW6It9@8-B0}%{suvbB*i1
zJJ+V8Ar<L<-mYe^FXu0PCYt+L)FHHW{x*dt<+cYbLSzq`u~szrZfi&};L^I@Gx=K2
z3sbJk84~}UID~RmWFJqi5aCeCS*ZN9v|aY2?9pZIS(jp^FIv2akf<@Woa$T>crfMo
zLGP_o)V@V{mw%HkJQV2vU7CBb*S?Q|8%)%>mPKwg$iMz2`Ixid>;|u=*SMybx1@yk
zEZ!<A(Zo=-JnPqIw>XKm_#YD!Z-mJ+)+jc{#s8>MnHa}Er9xoIq>k^CZfw+jP$8Bb
zs61nOyY!67ygOq11Dno%t8Dlo(ZHd<LOrh8Rnq>W`LXiq@2yUsL#qS7Pbq)YwBxwc
zGqxbv#7=#aJk3Wz+Z_8(y0dvaaM_Vs`+l(?SGnLa4Z9siJ&&1V^aGP(LzMI{y5ze_
z-@k6-X=t%hqwdSc(Elv%LNbr0Z4XU68?M_nMY~he|8HY5M^kbcv&p1Nf4zx4+8cXP
z7*pdtdS_eqp4DKOzc?m8LP(vn=4M4j>+;@3o?-gWJswVqF@B!<CnzAkEj%f$&&x<c
z<bjH;qVa<75g#J7^4m%$tjPUUF~>k7$n-|ZM^{7djb5u;3=dpvmTk0J8^Fz@F>z&(
zpo0M8)y<slA8n%9%Wq|NKbaZgyQ02mbB4su*5g+vDoD=XKXI;7r)=+XeeHA!^~z4=
zj}G58CBM9!@3B)^k+b-b;KY^;@yZ)wE5ZYAEp2zd)Mt28<+?_$*ChSa1!fzsPmyYs
zW-4&{_|#EZaqhp4sgo-Vr*?GyXMQCpb6NGQqk(pk^ZX8xwHNqw7Ad&7+O4ce+`1#A
zsZ(Bl(ZXm4t~Cd^Ow%N!w}%8hUVPJe#?$RBr6R7$i-n&&pXQyVI89O|LXCCfq?tC0
ziW0ai1U-bb*k^@2>si>iY(eC*U2gMpyebyH6r3Em@W+yv{_lc?yDHiW1BGVj28xBN
zUJo<*F4?wO@LH<p(~KOYjkBc%r(Az3R+kDoW_ka^$;OTHlf<W{FPd{xP3`B261j&f
ze;i%;O;p;lBClcED*nsOGmi?cQqFi9X=!_SZlC~bbi;y;vsPbL64HG*@6y-a1gkdL
z&iN`g9W*lad$uoPIyGNo=Nfs(`MYLKimMR&pBvb&%;eB|*}v?dO0}bLo0Nm+r%LWE
z;R}w7H$+TH7Rylzlu%Y@bPpHW^1W%5SELGSP}onYSjnJ?N1e7TnL7Q2w3g}AIV<v*
z68PDJ7au;Tl*cT3X}W8fd1(Df(HzMb;oYmB6i5A;m}I`pwNb@YtISrutoutydC-xh
z<*MaoN;8jJ6rGZ&&sjK=+d?JzmY~oI<?ItSf*Q**xrD+mZ<cwudEw0E5tD=FO0I3{
zn7{8?udLyUjW@-wwW?H>dUj<=$e)N!JI1Q)v3#+spu5SORmpRdjXF&q1<vLY)>Jb5
zE8(=HQK{sUAbTYDri?9fE>AC-t#!3X$Ya99E5CX-iOv6?Br)lK<oth`m8&!-n?7o5
zJG$m*SGru|j*w;RE||uh;c7p-Yf;XRiA)JGM~;?WZ=B5MHG8Ius{5j?s)`F$4676$
zPL|S=Y5vyn*{fM5N@y4RZnZB9_itY$bC~V0pv;toQf#XY%(M!AKXZRBp0#qulHEbk
zGG*>tXUtGIogeZ`Fz<PpYGN{zLXn1{o9!y?y|cm|9$K37aOv!yJsDiRk5}tEN$1?;
zs4;lbX0dy-wRc$N>muci`(!^_p1ZZ9>vdT2tj*3v^V2?h1U0nD?5LC}EKZnKY}#b8
zuzl+*4~b@p4#AebW$e1v6B%Z9^>1~lmYG^&m?5#?Rfp;-spXE+Gd0rxb6wtcq|)lT
zCzou*b_1Ihb-{w_?wAjr*+-iW&++6s)2w#EYu7R7z>{97j8XIZtB>?qD_zjqet1`7
ze~A{ukpt{HM_)C?Ui><Hws7C_-IGOYBG@IQ-ZCs|Q{2`d7WwsPuE6Bn^wS0^NAo(X
z^BAQQVvVGCX)}BkNMCzUCU8RHxk=IYKDXR-HhZ};#PL;P$?1^4?xF@wd*vMWW*6_B
z;IsGt@ov3IDfc=X^nWj%D=9N)P0&0UnQ-P)Uw>>?_LRJ`V-e@!iW8IOr&(=rmfoVm
zxMj(kQ=4s0ZRy$Rw&#e=>g7KyXZ1_(m^`~JcU4g{m!AHf_G4VdwUZAxx6FQiVzzCh
z!~e%!85x!G`P(?-xz(nX&Q+Xa@Uuep{-0eZPgW}~Ika?h<hBjIhI`IUbv}G7RP6kY
znMZ}nVw}%gdMx>VhRs$w@8#;#UT;o&HSOq{zN}&ODYcyyYLypO%-Oe+>4-bG@VYen
zt6i&A8*>y5x1KcHb#7U)3S%H&#!Bl%xtx#dlFsC2#LW5OB{ONOrJ|~k`W(Z9j!rDb
zPOQ-)az|wB51qKOTJX>D<H^^xe=GJdtWdkWdT+8z{gvt~vTn+EqhoxIEuE#)JNL{<
zU8}4AUX`#sUOB&WUsw0?8xMD6MlIvHdn$$Z^p>1!Dbx3#+I4NQ<>p%}iX?9M3l`Wn
z1+Pd9%bA`sS^BTa|DMOi?vf?-dt>Dsy{1%&elK+KNGNh&9<ee}{Wr(lMOufHT&w$Q
z`!@xyzk4jcv8}Z5*jXRxwT#ElNl(mD6W=#sJ7?l@Z<d|6R=su1*ktN*%F&ub*fCmX
zLFVmkGD4m_7Ze!RHe9aI7QeGcXF<ZGHHDjY?9#opcd!1B&E+d5t&88bmf0ottI^px
z%y%DWw1@LVc%~ICFRtl78kjCEt>ooY{wO}&c9P-WMNDQg<#!SkTP{mA?p>x-tJXM8
zDR8=jS?&Mn9_xba@_z1-xhUirYp(aLK>y<EBc|7{obs4IXZPF+nIqFhmM@grm-z5`
zS&-zuhc9iFwtRdjC3ABp=nDJO|5u;6y=l&)RN2#8WrURK_8-nYvW4d{bJeNe)^}Fb
z95I^bzuG5zk~%|Hs#vpeRaeQ1GjGm>oE8k_*1u+}`Y`qGbrHt)wdK;knmr8jB6bG)
zSUyViZr;YI7S8Jw=QLOQj?}8h7g9~-vwo(D^xAwDEZF#|{OpWdYu1Vg`8^9%U$p+W
z<1L=}jj?KHFP>c-d|r3T`K_`-8FRImeAgT~HS?0kg$aDKdY`}OshdC9aQ~NTt$%Cw
zdlhAW|0yO|bY{O}09($K7=c_to=c1KUdb3vk@l6mzQXLjlb-n#wWF1?*}J8t^~NWy
zsm{AIKi-M$={JX0*SNkmcwO(je9%a~XHV?}nV$d5`)aRoKRDC3@!A>Nn@ggNlteDx
zyBn~V^~#Bp;*)DQtYx0q$$WmgaC3`P_UzZ{F-!Dw^V}U1^v^C*R(zzNcRDh6*(0}W
zGwYTm?R}IUdpg7Rn(@8)v-9S^DBFGT&TXyjXCl+W`<@6kpAv6wF20f5%QLmF&+*{Y
zT_5EO9a?Hd|0xzp@a(%e>FnZNug{!3xNz6gGoI(<9ZMhENOP5)OIufS?aDKS@C?Ui
zt82LZSU7F3M}A>nX_cS-MRDtwkWSgBf8$qkO%145YrfYh@_7C7uJo#DAI$Hao-upP
z%c~!bdr9uz`}qI4ue+o!UXi-B;?Mzq?UgI~j;`_x<6FA*p5NLf8>aj}k$mjfO>Yez
z>HJ64noIRx`~Gk{m}~I0>x`fj%ew9fIkktUUb!3nKtD-XN!<N#hajs*jPCDE7d@67
zzv#_zTIOVXjZ$IFsgGY6!ldN4toqlws9oPsm1Wi5##PdO-dRek6!!hTu+vh@?n9jH
zAEx($Z0{#c*`3Xk@ke*({L}B(OtU+ZEPwuL)q;x8H|PIVuGoC$e%0*Y-yef&BqF45
z91pg!zBn)X!?`oe+hxA=X9=zM<rm&rC%d`&X<+#mt9qNPnOe~qt82P%F*vT8dLV3T
z#MZ3T)d`1Mxw1JMBmx&Tv~SbkD%i0h%rT6~!pTHpW3qdP3hRtHGlG&<cjzeY`f?-a
z|5NWE<G@Kuoxu*iW{b;~t&uF8d|{?myOWKU!pjT3DJ&B*e{FfGvBGbm(;BT%hN~<5
zwx@lK3R%;jn|yYjtG1!e%~jD$JT$cxJEv?BTJE!_I{XpKuHaKnPacIkJeV2tN>1*#
z_{SqGo+=y@K7@5mTlg*QP|wVrOFX8|NZsdi(@n{1=Gq%FdCpxthnDN#E?M(^L+aYH
zwQXvG3@k}Z3_Fjjc`lIN{((hcBF76}7Dg9^y{t0r9CtQOIT39ZslGTWA-CmDbm4N_
zjdy)+iaWaL=<kmG^-<?Gcg1<xI`8Rg<Er=UkNodF$sl^8)m!`JR#v==E_)aoEMiLM
zdZpeW#iFIe@n1EhbM;y^uT$zBCVL7FcUZ7|bQZK#nRtl(aFq9=C1TFnu4|N>COU6g
zr_8OVDb(Sl!>hK4dy$-CcLvjLmJ8gvii)=sln$#rYH(GbI;W%ik&4%XQ^#Iha@wSO
zD(S?L8EPh-TV_eU;W(r2&y}lm-az=Wakov-iPcZf>3D5_`fL`rp6;<5Eh3F9Rx1=8
zMNhpryNE~kyppS7S4N<_q*91-7uPP0y%R6+Og2`xW#%>c@4}^}9(L?b<`pK-$PI_N
zQ+o_l#kXd4tz4b7ch%NeS*5SSBtol7*N7?y_4p+U?>==*T0VK>2_@DAtB><O_~dPH
zbec}rt;F@Bt2Z7~=vou4`G1a9ez32p@yp9oUz`?E40Ey8y&8MfY3kNDWlkNcF2`5v
zYTj8e+r#{sl9!&+9kxCnZr=@;*WG)!FL{Sq&0*Q@;Io&U&+qZluhV$^^;&bro<8NS
z^(GVJe@#)HY-{DwaQ$DihCPdDOQmBu|LhG1Ipt%{EEfpA6?s)Xvh}0JhL*68uB)R?
zpL(%k%CWDpYeS@*8SEBsJGVAh`BKuF<J;JbR2BQrbRShw%-ZCcwR1s|=iwC+YjSf!
zX9?}>5LgkkPELJuVfQSdNh{K3UAV)ae#E0%@?c*?m89|XKQbp&`yG=ydY=CA&0eRm
z>5fzPs(*j@7do-tJ)zWff8L1;GkIKPmA(GU2GuT}9Aj^z(iO*76Lah6B#Whu|DL@G
z(hp?46}~7a>Xqx#J+HJXS5JAKRcSQqI#<<W&-9Pi51-J;eksMukhf93I)q6{@qJPE
z3EfXzYjh4B+cfQ5!sRG4-CWg0>+0TUtkPZd$Kl$W<moQX+zi&D8>byS6Z$K-*FDNm
zf9ldRC9ErZ{U<aqJS^Z>5<D?kY|94LRS%vhuz%#MI1s$?=Ypq&rJ3%pG}Nqmjx1|k
zSFGq^GtE#aD#z|wm*>AD!Sd67s@rR4d)B)tPh{>*P2>4=X>#7QY4c1AT~@u4T)*P`
zlJK>z&cU~jNIcl2${3TWbRy{FcO|A{dwmj_5~IWZC<HC}zj3FoQnK}Jmm6#oGgqqg
zm26R-BH_GLPwjzW*U3pK4~(Yhsh%s{s(dvqZ~KvbvCHZ+v>v%MF*!7qxGCnGm>j+B
z%bq|}KV=@ySm&&Yc~h3GYw2$4cv;}SSH!F<aAl0=F%?(mn*w<;&LY=N`YqcX;Fu8-
zH#6Wv*QO&{r%uf{p3D{2vhB=6Szq0ZO8Jk=D%R>csAfN1eJOF%v&9`RRBM<XaBn^(
zzIKYlu{pWc&rO1be62jyRqwqDGxcOVAS~yWBxD`)&cjWJvFoVTs-FtdEQ=hfeOt`7
zWF9_b7N+0c+MfKpafY5er(uVjm+GpPR5!&Se#U|+i*-Yu&bHB#7QgiWQ<CbF`O3DY
zM!!1N6e}l6K44Pc+$m$I+fy_9+`Jb@gkFZs&{djsabDQ!$BJiHc)s?^?h)5s=%tb+
zF-t0TU9s}<gtM#e)W}~?`($doO3*OX`MKB*hqaro2pXhk-FJC5WfF_R!S0GL6QACD
zB(}cfq_Wv$&AA1$#AcUe<ukixE><v&=;As(WxdOrM8i``wQ*ObO8GAJwk!AOKk-a?
zv$glKH$NrMy?eKgIfQc+i~5>Jic43xOV2Xezv;|#-nCm#YIl7-cgXMNvQ*tSBHpHr
ztl|%zyf8@eIVrM4A?EYfR*!~@-oJ0myrwRCa!pDgXI}Rr-8%-}svnm0J$v?ZV?U>$
z-TyU}DW?Ul`2T&`x99wvc@A@%CRE;R7Zsg&;&GZpyx-N$!4FT#zOu{O)n68}xI5OT
zUU!z4_$?)~$C@uSJb$eG9@EzTR5ZWEy4yQ!!jn`JTkg2miyqE-d*!h0-w(o5t&+aX
zTN-|)TWWrv`&`|?t<i!vCq~w72`yOnaYtt5y2J-JRLs`}>m;-CM6w>>%wGD{k10Ys
zx#6+<ny)AF`VLNx*>q%iVCc8W_J)`1vXUn4eO9`ZRfgMM{>ZHM8{(V#bz`F#dL)b=
zUpP}TX}ycg)QHKNOEyiu<&kK3b+>J|clpD)CswSoiJI5>LPKUAXV;9ASC(GPnz=pc
znaj1H+i_RJ=9V1#FBkIS(KP8;UftVU-o?HsQqlbIR4=(ZEb?>3aqq`BJg?YtPf-<M
z%@1sA=HXd0%WCuMDJ?JUYv;z5o%nF@O$pzguNFF$Su39RcDe3XQr%E5-~HxN;Ozq&
zLax+ZTywu8Li*<YePx>s%P#dUopRk!b0*VyU8(6y-l&M0dGF>qp~R<r%Q2lZa9^#O
z(%dGABZq3@HZ+^rER)>&V7ct7%d0x3Bvr09OgJO8;$?blUg=tmgk(Y1dm0QIX89^b
zKPa?X>~eO;N)@JmT@F*134fH?+xlq2#L_cM3wG){u{zG*;`*i}?&jS3CHARSbJC=a
zsLiPiy0v!Rv)Sic4l}AxJ2!DH$I|}_=F*c+=v@eZZyHswL+#unqb+OwmW9kwFWa<R
ztZK(4xl32~iM+}ErOWW)OQK6{a)(A^&EZzPyVqCjn>c@JRoBFI{gEuMr@J57F4*89
zV$gB0-FtcTmTmVBt~0JoEDyMG%klEPlq@3!p<9_NBlVw!vN<wb-@M<@X76U!KW4e^
zyKmiX+q<J*Z{p?l+XqyR?mg<Z{F2PK&j+;`7gSfimw#Vn>1a|NJ)Qe)zD8Tc&&$EP
zBmHb_>>|~bCRJLL9@|m7d4i#b^QTvl5x!>ozgH-gs%-MIIN*1{d5QUHO}?kAyqBsh
z`D3Uf-jJ{U@7#>jtj+)0jWjt9F!VId6jnKNayHAA|4UfT2(iQ*VCCWVQri3?d2_M#
zbo*+d&cI!{$s!?Dohwdnj=W^M{<G}k(+<;<Cd}Dk&gd~U-g3*#4f7%cxvnN{S(RY2
z&_V9YrY)}wyX2NQSQT+?pUlK?KtU{1?DeFke50)oXKnm!IDcaDI{7cF4osFW_u}03
zLHGUTRUHe~=}qC&XF04da@auSu;Cvk&4Bflr<S{I+{VXZ9&TjdzR)c5vDpM6^P@?J
zgPb;NZn7=-y{(UHd!Lc!<XQa|(sm+Ow%R+(l&_Zcm}RaWxyU8zpo-AO(2XwqA6E)=
zE7ea@x~gTp%xRXJwDlFHqgRa99;ukMy`rbx`OxcP*Uraw-M71?tpBh3V9ecXtu41%
zIaJtEtn<*L1+zOG_AFW;eSEjp+L@g{jr$%s*rb)LX3214>|oeF$t`P+Te?o)woO9Y
z7fJ6}Eu`SlkjHcU!Hs2nM!R@*xJvdM*q+Fl7Ja;U;*3K@?v{@oawf02I&l~Op98CQ
z%L=*ltj(U<xIp%)weU@)S&@ahZNHCf)LxZ+(~-xsyM0b))~;T*CEPoL-L|}#_-fY1
zd?CG08x0S7$bX*Pp|OVhTX3hQimaiGoBHgI#5arN=XCV=aMl%SKUmPsa;X1m<X)4@
zhl4h)cer!7Dre!1-DamfIIB0bF4(hT-kR3GUbaVy_TF|jn548+P>1XLg`Q)-_Oea-
zZ@`n|a`u+N6X~^8!eaAoZC~VIJ+o!s#u+Bd9mL)yyBhowG}yI2;)~t-m#ca%D}P<A
zY*8ew@Y;HZ(aBHVoliZCc1H5OIN`Q~`S>JZ?%iuna?F^VR^!&$aqM}=$<rckoe#v%
z_i)_x;Ja7B6Skmh_pAfuDW|Wd2s)Q+<C}7-b%Rl)m5S|Yk&S!AI;EC5KIYt*!1dJl
zgk+3Ki_J2rDWdm#bekr7T$^ID-Gj-OiLYl(_nJ&Go0%OSHd?FM%zw)xZGOr1S&Fy*
z5(|SSpARP84>Ni;HFQmV$;TMr)jxCd{4Xben)v6uJe>Xe@WfR!98LIZ49z!O*){V7
z|Fn%87A??BF#gZIM@e+^mpwugJA7I$JbS%;%bX*E(WisIxU})OiYQty^ISEvLTja<
zb<mBbS6w}KSFBRrVwg8;V}uCzx=Yq#nUhoud*17)D;V==>F^{>>^S1meQ$-(^9Q|!
zE!N>{cHCtUN^`mN;((9X#ok8+eY+IS3fdmuVz8NQw!wv0yNa`qyUba!c27t3CdDHW
zI>Ec#S6n`R^TzS$q~JR|0lKlAvv+a|Z`|G#a!yi5>S?4$>zsX0*POq>bx7*v7NyvW
zTLc{5S%&O6;pcm)d$+}G&4Q3!)$6ur+Hc%&bf(OncROcAI`7$&v-Nxr$84UfGkwlS
z2==xt>NO8MmD_wi?C<&|I>E-p|GhdKyuQnHM;*P|5fK`_bZ=+H`S4y_ChyQI-2JJ+
z2KuG^ACwd>JW#)-xNA~}ikM4})nBXqnHM^LXl|Jiu$kkWeQ(E#LawhiT{@0p$xoIY
zpJBM8VV!UHQ8Cf^d;i#N(U|mY<Hac)J16z1>1l8)Rm&@BpE(-Rss48Q&WDQ&*9fUJ
zcz){fOgnSpQG!pv+HQ@nfjcL16u30Jx}dwbnu9llv-Hcgt9K4qem-FBY<_x{`^;Ta
zgwCjPbPMn*3uK-Oekf=Yb5ic%l!K3^IJSzMJNhE*+7b`$N4lI0?mLukJ&*{>zalnu
zGWX60^Om+6aUbe^Bcd0w+c0*?G&8H^hyFz5bpD_HDaJQYl>6m|t-cBV(^p(w`}c;?
z*TZ&GIKJJm{b<3xELrnt;?2n!<`u!5JeRdSPigXZU+~FMS)n;=dEwOLO&7{DO>9E9
z9G2h`x@!NH+n1rh=j75Dh6M@?8GWbD-gz(~X0fcZYRh$X&2BR-^#jK0p~gOPs<Ja$
zPEP6$(qp-#*y@(GMZwO>=YEH$3OAq1=iVRIJ)Lir6gnFAJd{}*;%mWbV(8o1YpUVE
zeZg$=0o_<0n;H?5t(%>nxKAhuHt&tz!ok(}O1jCVhi7N-Wvg>@Tc<9OjM<VZ_=GcL
zJA=bUhPciby}zFFKG_k!D3oDk==CU<b+W#(S+ABc>h1rS{6BshOX$l9O|O4+w`>w$
z5@Y;5#w$KFbn1!F2-j}zXIFWzJ>a_*8awwv_^+#rw)9x~tYNa=uzJs_$-I(}-^Tqt
z=(~(r;O~wNnv(<*oYu2_>|Dq&aXHt#oT-sEQ#}{`3S(>ZmRDS?G9&&w^9;3Pv1)CP
z?7DdNSuB=tb=?uvxslQGz=fmLp_BGs2;UUBPSLybCv#9|!YvKAOFw5v-0|3S*OYIn
zKqyPY$;P>zPb*^Y`F1uubh27EasP@X%wkVATP&Q=barB|s?Fr+*%_CGmk6`$4O1+W
z+PWfg&eoKaLn+&S?{J)EcjE2+<)OQ647Y0DU?}p^D^c=%_T=cE$0>=P54P<1@0nKF
z{Wa_S=G>>zxm(kjS8mlkm0GRQSLnX9xpRSa+?v-iQ#&~s7Rh?}1?a54uupdjXK%ok
zgxwo9J2pnIT9$cYp;ecNaNzPuQUPBk9_EhgXs|!(!gq>M;`GvpXA!Hf^ER;rFlq6{
zs(Vh7l(pR4x!AK;Pi$tyded&3zNF_1e45rCn_=@@!#26GO<J!cqj;H6)|DlXrrwir
zy4DqU{8Izxv!e?qKMjg~Y$cp^Z<+U8cBX?5H;cSY4X&TF?6#%oD@|R+zLzOojuLye
zc%`P!7PIm4yRvD9&~b*0SDXx-5m)xzICs$E>7+NOGZ#5AAG;D=CNyhKXYNY*<q<PK
zZs>`f`oAOg=vA&|SNZ+CxW!)26M4PCESpa+n{!#{Q6U4%7X2SuiXD}=e{6a&!B4pK
zaDOe6anHQO^~ulDzr}H`oGQS3QSy$9T-l@<`{b-9>MoFEUd16ceX^UHZ}-+$QO$g}
zHP<~?y!p~9Yq8m?>$0qmYeHXbe$=A+mDfg0;!Ildz1E1cEibk@`AoG)|H*y!a6{16
zN3jjPeOit-zEYcyHD%@R4Vs|&($;6YXXfqXvsSKAFL!B(pH*0Ipm-vl@saPnOo@|n
zEqC5|=W)q?J@|RDr_j@R*{{x+EIaTw_RxL>^^j|~9=5&LT$IImgJm+~Lf$mN=zp&^
z-+Q&ijQg)17sGr`#_9i!SM)er`)n%=?oaYPzDS|hRZd}=#nm$}j!*gV+O6w|xASRv
zzbzixcb2_!;fcL3P_D(jbD!MHDAi**zd}2&hd%F2c%I|K#Hz71=(XP#k4ZA(3>W%3
zE^NCMSF-qfU&NJbk2fb|H0H&&ruF<@>v?w`XQ%0lrfJ9CG3Hfynm-FYBF^?wm;J(%
z-nb`kFBQ*xJyRoP5zoqMlbfYScy4ao#dV+G-2LAA<O!Dr;-4J3FmvDK4Z$%-)hyn$
zop=*wZFFMRbC*YYT5g;bZ^LHuZEv_TNu;MJD(idadwHLdBPaHq*zl_Pfd&(M0spi7
z4c|ilvwi*d@5^7lubZD`+2|A`ri!rD|JPU}6a3HfUFf0g^AGMlTW0R%vPk&h)OlY8
zj=1Hp`}!S{6|<k3spT_KC$RI~!pS=#ll101ms+3n@mNyW<&AR7m1CalVB+&R{2@H=
z{qq?;mws-3Tkm*jfnK?U{8fv#7jms{%htuKyS=!r*Z0%=%dX0Ej*F(V&oB8Fdi-(Q
znkQwl(<O>G?<jpGY4Xyz%<g{X`(tuGZBy&|k0v<Y^2*=3P%lee#Pj<yhY!;#o-KK`
zvq*n)Y>%IRw*T?>f#H=K6<#xne_eKF8?*SSr_cOtwe#+E7tCo-++*sfdZ{$M?}_ox
z+g@+J%KW*uK<nr%zHPER*9-33TQ0ki9{#vh)$Imv!;1_5-(-q+n@Qd6T^$&rmutoK
zc%#s#bMvw)@AGn2JQuU?HcxxerdL#6U$m_~N$t7Zs%J_En(kSY$Gy7wV&eq&sTvij
z);yu&YfePviT<0i%9-CHJ#+Jg<t_PB=PD%CXvcf*tFf_nU6fRjrO!3zTBg{=z^l`K
zd2inB+PZF{$j(jfJ<n$?o}>1lfBMPx?2Y>qT>fnH+Gy}Ii2uWM|H@$XufgK~_|l@6
z=IpicStucO`QY-e=N^dsR+zb~H}PCYm{Qe|1^2)5Z@+LU$ndfht0?zGS)H83nI;_-
zIriTx#CfBwD!(n@YfzYxBz@ib!u!+N|NqExZ>kmbQFE-{#Mq(9#G)ecKjVW!Q#0FS
z3716$3l6tC@Tk>91S&dri17N!%qV<(phebEPe;?iX%Y*klEIWsu3ZdaYdttus4kf2
zG@HMiE8}SK@^kZTn%QKtE<8WHh;1rgiq@Bx2g<B(-!zKY`pRLAckQ-SSHG@4yTQA1
zQ<K!?HBRf;Zhu&!S#rhEXm-n_hnC`R{fwen&g@-1eSJ)GH9Oz2>>nGPGz~=EZgM&t
zIMURiZtuIOqHBsK6RXw}r30TGE=&<DmdpKR^6Cn|@Dh;^Mcik@HnH9~+<m#^ZR5^N
z={tLWFL;0NQ0aRfzdv6>8BgZl+Ex2m;bK|RT&@XHS92m)c(hJ))iSoe-I#h)LdE}|
z$^SczF9omHR!v`j=xf3C=X<S>z4_btll4t~#eVyGrY+B}`O6!M2)c1FPGDf@w>1-J
z7LR2~Y-Zv7pdi30x?~{(Baes+lb4UeQI_B-9oN{b3l$D@v3Gu($q-TyI3ry@NbRA6
zV9b{Xj#^JOZnko%9#M`Hl=&j)WcesV(cNEgXRsIR&Mnt-tXQ|s^}Y6Y$y0}Oj;^u(
zX_YglMf%-*A{?0Lv|M1iP3F?jurS5c({o~Zx7?lUJ~uSDEaBgjyYp;v+n3L>oyerI
zz+@}?)oS*=*H=`t*=lJnW%gZl%dypV$x5g9xgRejt*caFT(<O|NCS&k!na8bvs;(V
zT<Krv6|rXPt^ZM4*{rou({t8(v<K&2OZ<AfU{7hJ%qISy&(t>_*SxZ3<9RXenEb>0
zINubVx_d1~H|Xl+++fSz)M-(bv#bLuvbG7`+m#|~vcj>!&+UQp7bzF71g2kG*VpxN
z8Jv<RKB!u7;Qt}>4F{R#v#m_v{T_Y2q2=t-h!yT^njvo9v#cKa$#E@cVNsdIp}m&b
zcY|<mK<-5)aosXE*9_*uPr-?^j6wt^h&g8joU?gfA~;FqtIn0l{E8eVQ|wh6CV8hE
zxH{FRWf5!sj?n8{CSOVPYE506#$*~bJ8<txnVTnjWuJxHRxX`9w?a`$KlMY9tnl7c
z-6+i(CaIF``}r$h6~r|E{ja*<LG}5n)dBN&ExF+rWDv3?xg&U4!>XA(pG0LE?9!<0
zX(;|@`NC^Ci}1d@uTjyt$y}^!3omZ-Tf2Gf>9{v<Ha`9uU0j$H>R-HD`s$RDKtt83
z(-L1-uDg@Ly=s{c(~|0U&d(&>6dJz-vMDU(IGsM>>$FE}FD$CzzjCjJ=|T3(xab_F
zRU(S*ud-H6Vr85eDPP*+&?+9rQrRl}?EGa9HzO^f&NZQJo`OuGlNQY=T-a#PeA7A0
zMe?LegS*3{i_H=yl3d?9oUD~O{oB4xUX&T2Xvfv*rLuu@`B@d6)PtQLTPAHajXdk$
z`s?o1EQ@tApC)8qxScqQFUV%a0Z+xW)n1c*Uxci16kO`xQ7OH%@4Nc`o3E=U?GCwj
z=FkS6u>PR=L3gsnj%ziytTO1E?7-mlLtu)5X36A02cfX3Gta&`>SnWXan^?!nyoA>
z42nNlSVb6^7<3pI7#JFu92glm{xh)d5OUZc>d2VO$EPx5!ULxkc4e!Q;I5>ZMnc|G
zA1F2SFz89BcF9;yn!*qy$JwW7;q^#uVoJcQob=TRYs=QUbxu6t#jv*7mXl4@bIXdr
zrq_IGy1S-iTwWZo*sJ$e$yLWdq3e;N26@@nn^&!qyQ9UonxW9;tW@r*cM4HkGd%ky
zuL#TaUZwn8N_JP4?CQYecU%WNFUOr**}=NO@Ks@kOLH&3RE*DrO)jSem6>HNuPnLJ
zHoY~^?ar#u*+Gg+iv&)lZgufvPvrKH-6#@rfk*L~*n_t6nPyw~m;H%oQ<m(JILYy?
zX2Q;sflFB2YfoNO5jUEBV#6la%~RKgZH?IY|J0FJD}$QsFAJC6DapM5Ihr-XH>)(y
zd&{NsTqooW*9C43OB9t4dhU8PEXjAF=={)ss?Q!Y2dp#=jb%zmT^J$Wb!2|1OqRw)
zi!~8TPim~W6T<EIR^pn3m%iYU7?(?5pGdTrT+4Xe>o)OHSD!{M<7LN#8_j~UPDrNC
z?p?RzV%MYtXTw4ykIT9*_B`Kv<<eBuqb;%8U$2-%XuoY)epJieRb?@Q@uL-L!OAnY
zKb!40clwn%?4_;O!dY}J#G)QeaGh3H^U7U4szl8-qqX7M%+R>s2WLjKKkZ%U>92k-
zrQ0@cUx#uUdtp?FKik2VD;X9}S|#DXtScqbIpm$1U`8M7|Eq18%NB}E%wc-4OD$*9
z(reO=j<1g0cHH6m#7*+}u7?*3oaESroVK~Sa%tuyNQTzkOm**?{-E(VpZ4TmB>^)t
zn6~oi)+vf!30m{ws>ZB0Os3OAbuVh)cWipvy?${Hm%-^Bd6$x|>EyjC7is?fX|+MD
z+sEe%S}K!{#?M;((M2iY(%cMhTg#1~WHyV=?p)=uR4?b#N#(aIj2!LCt{8d77MR=?
z2|CHMCdK0Zn+Z-f;wqCS<*9T|$UQN2!X-C9ugqj0b>8hki*@(ij=4O0gJZDml36FZ
zBvkUuM2-fD%1pe$>h$#fO(vDA(-yMJYBbzVS-eMMF3Z`aq25uws`GbOz5G8@eG$vd
zP%+2aYq{GZ8jZ7dw6nzBeim5sa_yDsSv?2+923hTS1k)Fe04Ep<)X~&l}q`qa%p!v
zUR#-QsP*3r7lr+GA{j@g{mztDTUeLoxMO-)+qwY8cw@)CMSjUns_MU^H?T;C%S_oJ
zykyQ7gD|($trsmT=6FUY94{%q-+H}E?fuV^bvx`&&d95LbviumP}jBPD<10JR4;8V
z@?Nr_F6<z0{H^bc_!)ke>u+f}+E8^n{+|r%#uW}dPJFKd4qdt;x3opWW$^{38Cp;0
z>TNxuqW|-Or^h0h$vs((8ZE)ILar@-xZ{IDEJKh>j+@Zz!Y7e(p1qDa$pV*7UPxCF
z?r`|u#AdR{a;1lnwx9c{hFO}u0^a(aNsDFNRNr$R@NVJczkY(%Oe%Cj9)nJZ#BmSx
z-Cv*fr1hNE`k+4Fx$|Rh0OR8uPgoq74i@@lJUONH?9)`!UxhMXlO9LPux&KuTH#p#
zia-3>@e_-8tWJ0rBvJkA&^{hMRiE%J?pvo_n;2}XQO0K!doC>8$Ukpc$O+@iFE2cC
z4mUlb5TCp>)60?V;nRNM{oPy>>pUjilUeO~R{A9G^cPERi9T(fQh4LEl8~uDR%Ueg
zn`u8xCYpWSnlD;k7{9ma(e~Px-KxKtmRl?}o0P}b5&!(pY%B2<>YVpZ&lP;R?2v-R
zBsQZp8UoU+?Rwk(bJb4j@VRkhSv-T3QdB~)%`5{>cJ^fMup>v3ZwPYUSP-n&HPxt9
z!nY?q!lUioR-P&Rix!?aQ84%B6q61s;RemBXDZ<{6Fc@8a(w0pYPwm`(#BWhcbV_g
zmT4}!_G%AXrtMK~h-zS0k5&%cwd&>$gMIGWX-96$T{a6{5fr-W*O}WP-;XV{V=BsT
zTys7Ay2J^am4PxzzS`l}tK62n`PdWrRz2pnRruPft*c{swIjaoI<CENYoO5Y>_={f
zJB~<xo0rx3r1FiYP`j(k#%|?PTIW`XuJzfrX^Jkts`lxw{Ef|G^Ow5ozL<EndBzDj
zb*Y8lg04wkj9^O<Q001&<9@s2zeK;K<IWWxJ>s+GOv(zg7CK<G@RF3`#$%>J2Qwur
zRxgX!JFTI>$SEl#G)1EHQ+8<5I*%Hcxlg&SR~!HI`{8AxloPimcInhv$%`*7{I)OH
zZqb8)nuoi)e%#u=@5kQtbx&X4`?v3V{<|;w_4m8unWT3d_%k=ZPJGS%z#HH7_*r7C
zc=yC{JUs68eadRP$FKc){jVMV_ousqoh$0XwI<oG3*7Br7*4AC(JULQ?cuxQ@x}*f
z>6Q<tWVn_8QIp&1w8HLJM(;fBGiRoRw6kwKJ2Ap)Li>_sC!J$1U5Lus;wLweCt_dV
z>UB@H2uv!@()q+8=6i+DbK%Feu`j#qF8{e^dRX}X+6bwm8ppKKEtxl+D-?6}b~*T*
z>*dY!HGEqZu*+t+q@K7C>=H6Zc<&1v>xzp)$tDSmf1h)^{<>6uOlTQ9w}p#)%f`8V
z>i^z7nY#U%Z=mPO`757tT{ZTbc6Qp?HSvEw-MnzgY0K9YuC`T;Tk8alUf(C?VE-$y
zTSDBlnD?oQhvJgkD;l_zHcZK#wJ3c`fUk5^|HQydzsIriH-%D-4nI9qvvW_gjrPv-
z%${3JcAeznR>%qz=Ih>(dZq1}lW^1vgLc6+_qjgaKKS+P_IdI?AAT{_JXkC-ao)<E
zAGlQKs9w3lZ)&Q@)n*^_h;RFjW5(w`HkSXn$jkrpu<O3UN%b}b%K!a;EEZi?G}-)5
zY{W&2`hPM7Kiqr_gx$W>ecxOBC$C}?tL^ojHw<ryte>H9a9W2w-y{AtoCz;k8h>5p
zXUY-3{#bwNB*SH94-L)k?6@g(F!$6FrZTzNJ?^}mo4(l1a*3;ZqM#eB^rlPb?t>F1
zCA;?i-q2zpcyEE4N~oW&@WPZ--S;7?5B_9M%Xzvqd3}*fDPPYVSw0Ed@OuZ7C3Z-d
zoUdfhzxR2r_))(fHdPn76VzAO?|l`l&*r<}-q&UKZC<(h7hayc@7vP+zi(6ft8Z+d
z_x-?f+YjOL)wjj#)kRP3oj)<Z=GpI`T4|ki^Z(0a9kdBJpDJlF|D;sBLs|T}Qr2Jp
z*LeH%NB2*=E+}bssrhdAlHgjEckQP(R3GF!y>yzv^hq7NpUX|W{!VPl;zyb*9$P6L
zwo)!u-k2gXWnu2-BQ`reH5jMl2}$@I4z}rAWOLjpCjYz6%0ed*(Xz`;5+BuUZ$4Bu
zNfdstku%Dnp!ZmYvQi;uc%hK7lF5o96OCqbk3uVps+kIGGZnaOBNT0aG&^&&*k!c1
zRJ6G6Xz{wy;`yU_`S%tNi&h_vRv(QP_ZL;xKbqHtE5>#-8@*^1Rc8=%Z%ba$oMus&
zTp?<8B3)O!nM1stXIp|-YQPI8evM;ETE~)<H<r{{l-6sM1}b*=u#5U`m!A?8!gf(r
zZ&F;>|HkA+ldVNl%cfX}>V&XmeK*o?4t8}k7IKt6?9!MoUhb*Ju+30_%}`)DN3O1_
zj;)J;SfSv_MnTiiV3ma6qUCC#N<wkl;xrUPTv{#74cP((B(xQ)S5NO*v%Kfjiq<ve
zg=^j0E?E>_Ucq|BqnE|7@Y;^vAJ=;Cc(g?+Fhot@eBjYH)1dEmM&FB${O2!v-*oi7
z-O=}{qVLOz-VZx^AHC@NcB41tM%A?+)z>uyFTCiv%Q1mXvx-BrdfN7^EpBy-mu9~Z
zuUBHs5m1_V$ZVp@!xEu~*?X^*+zbvmUZ&a3sCa&9v~8i&-qMCm#!}iQWx<I)(;B_>
zE;-vea_uOVI`jWiF1JYD&L<itnsgLcyUrWOUffi_%~kipLZ5<i0p%NlI!OXw!jyCa
zo2@i7MV~kCzuvqqz0dAO|C1e4BQ+VEHKxT{PJ1H3nWV{(Q8_Jh=d>(MhR6le9&1cb
zx12t40c&C9^iofOD$VINoqgp$r$3jN5p`jDOXZA*5i>e9`|4-1c4_ui{hZO|*`HY{
zF!^Qk<Bpl3D>z*}3S({**@XwB9OC=3p`9sVmiWwBf$SZA<sHH5=0W1K*Pbl#n_lAk
zod1!c*V+f%2U#T-H%zjc)-cs#PX9q=H#VJl5{<$d3?U4$i#TG>CyN{l5pnt)eWFRq
zt5{&QV~|XNY|MwbyDBP}{$B|GeaYwSLe=mVQ}<A*B?7EhSM+|c?0w47cT1xGo8|nO
z6Ij2jnE!YNYt_s7e}B$rx;6ii#SF%(1qG1{{z*>fo;APGb2<ap^m0i~?yQ-dQi9xG
z3x!TCkebCBxL|(1XCLRS1u9+(`Bu$Wk?N27(a&L-{`ZL8^PsvN)1o9gOWwB|?KKdx
z)Dp5@rS|!G_AxfK>6a=@nj{?)twf9)R)^?Kn`qU_K1ZZEy8EG_=;S5AiyIa#R9@O(
zeOgR<&W|$J#u!1xWr7<!1s}#1EV42EY-434J<rhgN|CL>r-}Yw^4t!_hOd&5Rb6sX
zMDC~>Yt76BPb8)bHcnGYTv4YvvuEdwp8va6^xax9L90(*iZ#k%<+P}kGqP67?OG|%
z#X9HL%2`$`7tCS}P+(YeYvnSlRqr^b&%3p1i4^O)S1adwt&)pcIlF55lvyoY-xrm7
zlvGyCGRs=rA|ZJA*DNn}(W72MM`x`$ajV@mu~aTuNn=vI%A(*^u3WJuOONbatA5bw
z(ATvFDqRa_#IR4Cb9jo&<6ld4COfZvVZ8K`G;^aMf1xp3qm4+R&L$5wQ&GX~H`ZUj
zxOAVXAB&=^Z{<7<!JZ{f61I)~ot1r@Q8O5BEo_rom@hd)#!Eo7Ya{!{RYDtAb!4rQ
zmS%mYxoMf!CN=9#s@a<qtT(AtZ_;|bNtb(*Ame6rZdSGb*_$n@HwOxETHjvqdPn6<
z_Bvk8s70y_d#){BHM2x4bE4qFi7(3+=Wp5)@O#U_t}VW|lY}GrKX2!fE3BUuR2Ik>
zt$f+l=Ap!#OQlvq+ABWgtP2%M+HSSxFxQ$xY?D?jjoGzU=ArU~NN)yi#VjsS716on
z)}oq?Q{ux!PEQfIx0zM-v8?JL%ZDBU6Hm{W<h_&S<HAW%J7>+_S+H}}l3A;ko?4|4
zy=uj)c}sS0WEb2eB)DsXmC!0}ArA!xvu>6^huzzvckig)y~TR>2Cmf$X3aFMh+ABl
z?Y?}L(!n)>2e;hmGTOXz*2{KN-9wJj!NNYpLae7H%Qsr*Gnw5A?F<o$IrRUd(Mid5
z>=);-B<?f(qGFM1eE6!(MM=S{KUIzx3FSJtoZh_Nc#Bkk6Vv}!TBbp+Hl}`drAse4
zb0u$({WigrOPW<MVP}h$V5{Y3<%1h#xJ5;Kc1`a-xJi4Hpu;X+p4IYa4w#)jASk#y
zfPrzhG~>2zhHc%4G|n8-+p}B1;E<rgCbeJvQ-jmzUX9$IR=2ZEz~eT*_wqWoY>`vh
zM^0LAJ-yo?to(rBLc?(Ll-Z7w(yB%`PDT4_i3EG;vIa`6T9kZA$Y_q?F{>bL1=dDa
zH-Qtb%MMT8FO%4q`D^l#sd*9;1x`*8`uQ;8CbR7I*Fu$1O#wHTKe)J8Y}bt0w>Pev
zy=(D*?cLdn4lym*y;J(+1e=q4tq;wZ#k%%;#`CAE7ndIC5S%StIZ<TinosT`EfV$f
za+DX?bew&)=fn4sC~u{=4gzkjTBZ}N<(t>)6)iP(+7|He^y`gBmK;)+WnK3~%IDRu
zvS+8~Bv^HwopZ*R#b|{^+1exJ*J1?g$_3q;_p>})FMT8VRcl43o5=YshBKJeYy}zl
zs|5IEM0tG#-s>=Yv^j4qeg1>ad3M9|0SgXk^&I-!!=le~!Gqy2)7wM8Z4TL1PiNKS
zYIOA6`s}bZS5BSA5fAkaKkZZdmmLX@Ui`lNVu<u9*EJUvdL>rsT=L4k#QyN&vFgQ#
zzb)>|&}>_%)}1J+{(t3ZjZ`mp$L(uawr^4On#!1)*(4cbV<etB+04_sV)Zdy=6SM>
z-c~;qp0N3-2nv;c*SRuHvRK^uyyJ5BiL!c=WEO8wHF~z=;l~q`tFOJu**W?22^PWK
zLNSL{ckkZgeR47n>y$a0r+1y)%3ko_g<Ew}`$zXv>vV1$^IBtCbm`!%i=Db>4&C9`
zV@vK+j9dN0@IZ|1*3jr%tlN6tZaX-&)9qpB%V?3)IWe6d_bGoAxEgcTNx1t~jllL5
z$Ic$!-^X;|mpNNp+Povd=T>Z;_m)AAH)f@jw~(B-(0iRjGCqe-@UAvqbH}3l&?(>D
zYOm*8pYC;-9crVqc$eE^Y0grST9L5-)}qceHx$-xWm$MheCFcM&y)U6SRB-Tqhs&d
z`@SL{ZSPB3<{UK>QM;r%)%V<-k4kYeT1&q;El9Rnd^u|E=1KOoH`zEuxjnYuzUE@~
zX{pdWiNh?%&wibJ%INr|(yNCULuLl94dF2j_$XxeX_=}A+r5d8>TU^p2&`<gImi*a
zdFq|ryS1-RJ+rcFCF{&J1xjves!se;oTnNy)|}XV$u;na{8{~ZYqscx-P_)I#4Kx~
zM_k8o?Rzq@Yv%Ply)esgW=PIeYul|~8<Y3kXW4Nx;N#vU4V|)~X9PE%vA8-%X5*Q~
zch8)>xi4Vh_BXqXk4_Sh*qE|2P)qx$wCAR!Cnid5daJVOKkwOnSGD%cNLCI!Clx8R
zcP*#snUzY~cjZrQHrf4BefLVOTU9f~CEg@w8%sTX?>_O6+QdFzfse5h*Y~_~(P7Y9
zJ8{duQ%}!cT4|TF?*ach$)~z+Z|vUp`c9$JzYX`Ic4vRx@Y=0!i=X(B2yLNfxv#tC
zZVCE$@1R+Zewrn_pj!Ci9HBt3^%f!uhc#DjzQ6RywoRdWXOD3$oxJob*TW-|10Qq-
z&$}_#{*zkPt)(Xd%X4Q5%yT@EINkRhTYglo>oO)e)k9p#7alE~wUV!X<&;-5s&qAr
zu03Xby4tOLO|$RQ*0mphUcT3s@x=U5ov+-J;AsB3zfb16t_l18ByFDkzOs(l|1a#=
zb*-XZ@LF)<l*LoZ^&+3yeA^o>wNKg6_^jr0mWlgZ&!1V6yG+S&U+~?ovUcN7QMvn`
zaBYfU`|(;Trl8wv$qW8XsjEcKv98uNEY#fUBVieM_@&2Pp1l)P9!1*5yxQa^q&ipP
z3$OYLnM<$#mK=+F6PW(SEpYY|S;0MGIZt};x!u0B?*QN01pa*+Ccf4^vNMf;PaXHt
z1c7}Ie)Q=JUiSVO{O^-!=9^>#;rL0O%8^Tw&1DYEoTO5`#5zB9UHlGiyDKegF7tLs
z#MO%!9lqkDD`;ZOD*R!NX6Zvo4$+dHT<6$|h}Z+1eN$q5MO5Uzsn-cSZ2mCq$z%0t
zf3?&;dH&xsYjw^kne!J{?7I{sck2DxHM;Zv`Pa>k*!;KC_Fp&eX`T9xh{s7QT)8;!
zB>i*v`RzSF!^2ggn$c^swtiMW5VkvPEz1_+1FNPnbcyP6c~k@@AMKI0&3f}aFvVk{
zs`j_l&+|nz!$j2Wwp{P(X=qGw3srI5sOr~cTpV>LbJHT{E|*q0*I6OTE5nxF%2C?<
zR7spMWM#~rS)m6)yJEI;%?P@<+I5xG;jSsCQqv?Eb8bfE6kkk?=`HNsmHBkpnR`M^
zza|MV%wU;*FqC(DXq3_O^WC#q?(F$Fk!R&erZ+bi-F%m@HZb`W=Zok1DK9QAa&KOD
z-M{oz-CjHH53V0gkH&TWPmi8c`&-HLg8IZy+c^HHcwN<qy}l~ETXt>7mtPx&l~<V^
zeYCrK@}70GstT_y`g>Ko)nM_fxFxk$i!1JhY8uUYowzM@Rj76Jx0=|rjQ(u}H>FFz
z`|IrJt+4Jrz}Ua5Z|VuPTW>Zo*v{(g?N*s$uwq);CJ(2%dWQ;>x!YHHIL~spbHRC+
z64yk{uv-~Vb-OGo4UTQ%nf+0TZ`qB8MN%gpEIBNpujQ=2*GH?lTd9A^o8toENoNdu
z=ZG+Pb!#-QTG5m>u|qV><t49j-@Ya1yr=9_{=BG9=hcfddQ(%3S57$L<0lema#5*E
z=S_=J*RvxIPTW-wzVw+g266G48igjO|5vhzyqGd|%A1YtBH?>@iUaEPluyj7H(2}r
zQvIx~^kDyFCD)5){;sAgr@V?hsmiDpwPLC-+b_cvi+etCgbM}NPS6Z`aZ72{yuMFe
zU3w-`s~2r%nb5T;ZK}~FkDU%B6Bl`|y?a$u(OYw2&%U;&42B0z%vk9bBEI?Mq7!o6
zo0K%wH?I_}&MNFsJnJF3c*(^m<twJ~zjnU2JCoF%Yu>#~xXYeVz^+2Mt=c*_;J?~4
zag*zR64kj>7A**#Y-lCjDq2+~$lBzSrMb>#N<?Iz+=YG7sUlHP+`V&kloDH?MVT}v
zh+I84d;PwLKYTsx;{P!2n)~^WW>iD<t8UYa=Qmh?_<y{8^|P&uRx8F?Yzef=;n>+!
z!?VFzP}q96(k5M_Rd44<Z4BKwDI!sMdDtzDr)$&BiZ)Cx(EN4#SQpdxj_&lIZe8!U
z){7XO$g=O6wsyy<A1m|9uExE5U8)&1UBoPJ(xtn{X1v`kvTDJqO$)T6lhRMG)t<_G
zVM^A=rD9jLwIo)y?2|4&8^ZmC_gr__RM~S;pRTxUcQsF2>f`oWM1f1!L~@I~@1he+
z=7qXD9(r=c;2aZYn4GACUGwojJuBUFCw>X6_u_o-_3OZrD+&EqmU#HKCW>$PZ_4X3
zdFGC3%kI^*iv}&4u6xgUVmxPqh<RiWv+RkkIXzC!6K+me*R@G|TdvFh=16az;3t=r
zyFH#po)9`6zGc#y4LaJPnF~9jmu+?Ip5mmEC(s~0XOnn-tc!t_6%U8xigjHH1;t{G
z)01UP=Ks9H6t8*HH)~19q|YHAjkaxQ*50(kwJU+G{M*knBFksUzWtH!u-=g~cNxd5
zs~PR4M--Ja*FBcHs^QMixyb#;5eJQq-cDJ;2B&jEMr|4eIijy6T-6o{aJpYq60%*b
zeE!pu&2<};{NDudUcMI0eeud-GcE0qU;`oPKqX)MB^~lPGme?8eb`q2|6{CY!b*dv
zOUG0;Gu|uv^@6{u$5Bb-B2Vp`kCtX<Rh*bSX1AOuTgb_oP`I5-ko|s}Qk1Tnlgt0a
zE`|@+*p40W=n!?h<F>=#oWisTbEG)a8>HHu7rLCDdB;WRxkk|T`7Ua?u2U9xEC^9$
zztrj_k=6Pt;yRb_0v;80#e1uA5_gB?Y~;-=I4s){G$H)xvWlQ5vbL`r`JY}%l4a0&
zkX^A|%VaHA?V^Q#PV=3+LoXb%T%8nJ66UC`k&-C-_XLZG(Z$_e6HNRU3Giy0Ju~iE
z@~kRg!O`>y1@1jpGes^Xw|j~G+&g8*V$nA*ggk?i`Ri|F2hTU^vcD6!FFK=Z^?jGs
zo*$k|-w@PLoqeOzMMfxK=7Nk|9mz#Dsw)%}>n^O%_&RlsNBN<Az0<9tm12pmi^R72
zye>4{bK?J2okfSuEY|7wi;9ODv_z`98W{adx^7qbX_3mQ?nU2%&Pn_;k+J%uoe+NJ
zk!%r1VEBxMJZT24(lLeY;W-yYPYC)}stIil>~P{KId$Z0-m=#IX&bYjR4Ql2Y?i&O
zvb=P4f@IMN=l+>m;S7-%T!MZ(^6Qr9b0slI7X1)u4HUSZ+#<DJE?Ak1WzHFS@z3tM
zMvV<klf=1xZdx)wY~k|b4vsuIf5K}zoJ?jJ_SVi3iSYZfc!FT*E2EQ*sZpj)8*W`O
zO8TE%knQTTqQ3K)-E^h2K&dB3&F%(!ra4dSHlMVl|A_8-i$>{tU&;jYr?qIycwBSm
z2%mP~Z?9HMsLq3||3_}Jo9Y#YE|iHin!Zk2-_@xnG5lEz)6}$yO6ma%&-kx<B2Xsq
zMRkXy=1hqw0c)2lJiW%vt~Z1(?%BX(EE&oE<pT?=O2hUYhRk{&yS&tTt(KR*ohrG|
z;M*1JtD4+Q9$Lqa1qAzQWQdk`9s056`18OWJr#={6pLhO+4{!lm|C7~%08vdHl24?
zXl>Uk-$f4Gfi8?I-dtc32@~Y}7WO`nd8fwAq)U@FAL>2&zo6aWkeXnb&yG4qpEK{K
zFa=GJGpS$4SLSrGW&HvPl?4KltZU~;3LZ0fmOaP2@XoHY2V-wmxJdo$SU=B5^irhC
ziBq>a!cH)W2^<nI)Z3KEuB#OJzuo7RuGJMq{=b_Fy8cXX%j{j?D%0$^S81YG;Xxi@
z)5Cp+38nEtTNdeVSm7GJCoxE%_VeuMDck*~EaXnrXyuE%$k$RiB~>q$RYc*yQp0ae
za$LJQPn`NJYv|&{y=aH<GTuMStEQN98(DwRnxov{nZ9u6#1@BTzYp%!NSNBYYU7i0
z`R4+At~L8dO3YMy)opaSrEOo;tK;Vs7If;c1er2lt6X+dV^;KqBX;%eQAKYq@2V15
zBdfDpOMU-h2R6sRX~LH|^lX{-ScJ_>v-jq_WKwT(ZuY0GOuNsd#w2p*-tDyw7Pwf$
zmdahe{Oe4OS?MOGkCG&3KgreodD1LVWvbla|F=~dS5M?VUU{zT%RXPfe<z|Ps#S8s
zTGo3Dc6PkzeA;W%J%9f#Tgd}|1R9ev)6aNnE?}~%(%p5Cuc++s?ASu?=sQZ=x7Svl
zc`H+Cl*+@|rzO1hHOt;vT}>7<8yf5$PkOpy>(Wbdf!=bi%2OUUGW{>!`r_e?wU=ki
z;oM((VMgc;4x>t~C99V79yI&(N_g37*1*@BR|N6;J}}npSk;{{KXtMA``evS8K&I@
z+l*%#Fg2N}dMaP>Sh-DW)#OXt6sE|sMs<j%v^Gj~1Z5~Bb4^p5)8uPmJl%s&{br}K
z5vTT&h2_n*OAEH^*-Sl<C6<`ErpRfkr01r?MbnmO2%VU;`G1#%pm?P-<ArH5S%TYt
zOxu*uU7#t%w_>8Or8xT{qYI0yDqqRW(OCO^#dNhtJDw%3J+zr!=92E@V&>ouV@3mJ
z;Q;$x4pOZS{GB^^?`5og(b&|K#2Xc`#Otv}l|-v&5wBe*!~C7=uT|_`BE<YevbEGh
zYnw0w#|2aWCVhb~!eJdv^ImFRUBDXnc#_jbky)I(Lx1#r5^5;DpjQ>KzVyWWx*1!x
zDszVI5DqlZxO0Pt$C)p9=eD9F9HxbPT0ZvjJr;jabi9nmQ0VlYbKDZazgjOeOHEYX
z8z#_^dunz)OOGjw661@8UX4b*2Thtcm=<pkXZ<B8ae6{;#on8@IbL4w_`l<j5l`jp
zqT+qoS>5r9t-23vA9<@Q3aj@T*zGekDtOe8_H(wb!W?l%Ma7vO1y({vGh|Y39ZG!Y
znX<@^ZP9|`6$^AP@7pJ7Zm^oCb;0UBF3DfbO4B#%Dpqd3T_m&T2W!^_{>__Kq;qS=
zY3T<h?DB0knS5xgUACOa9Mf>&RdK&}`kXLjY-oB}s2SxjZ!#BWNOj+bNO7m%r+9og
ze;+>Jcxv)zuXT2ctGZq+^*P|s{b2>qW!Ya7w{3mJ!T(rl`>AcGW2Sx%kUe`sNac%B
zk>vIj2FnVICtlL&6zUM;o50kbFd_eN&&dFR*(W&XH`typRN(Gf#;-Y1H%I#UXNCJM
z=Km|3CmlJpT_H-W)>~1*Nb$%jJJpZdxBl?Wxw&BS<iiUaT;#MEjb3!WjoEQfp;=W}
z>Aa9^*X)fdvpB_W_qbl+Tcybrv|x_Wj@f%OScP9O`7GhE7TCF8n|~eW<Owrod0aa7
z!>NgR_u{|>!i$AlXFjxAugzEXz+}<{M^;U%HxqY#`Xu~*g=Ftd<-fCbckbw$8)=<=
zSwDI;FS8J<Zoz4{nFs$JFvxx^U(48eC1Qr%rah;%`yMFuzW*_OaiUb!AL|RvQWuqN
z_aq3)o}Rr|V|x4IE>+u}{FNO|UpUlV&*r~u+V_It*d&JCnq1$1ERgq9<9F<uwqSz6
zUC#CoC%#<~xHFsk|AaN>Mqg~}wbc(xo;WDjFjdF1KVeh%j&rAkIG<Vz`E_tt1aaL;
zo^&g_XHkwq+QogR7HD7lY!_HyK1pEjSAm`J1+CjHrghAk(lE=O#gns2fHmTGpWqyo
z?Ppp7C(N84#A&zK!LF%^ZO@7-3)g<32F9aH&PT9v$F{dT5Vmie8ya%(<Q@*znKO%G
z0uxQ$WuABypE>y82S2kX$Cr~Tx!k;pXMD~YbSU%aOkg>CLFH^cbGT%aj-=A{t^;dq
zVlA%dFl)}}yymlDS61tZ3nzLHoYnkjd+*Gw1E16qD+NqiWC{*#H~BMB_?KUQhx1Dp
zp018=OO+t&H=PYnn3T174op(`|EXq@B(uwk1O2+T9sP|Lcbwi6t-;Fwfk}A7kxA98
zcatV=y=8DsL6W1RW%|nY4-aS5%<K?Kb(`DG6U;r|@ufz*j6S#Q%$T#2eJu0>lh?hJ
zXw~^}toudlHP0!3EmwT(2n+wkd(CU+l){^9dDm6OsLFk{sovsVBNBAkIA}()UH69I
z-h!a+jN4rqx03}qcl!45y<oDt)1ceY-N_-%o;Z2_l)Vxw1*9)ey7(%n?DB-E!dLdq
zoG97pDOnh7#Iszfc$$pioM|jK)@05xTj432xNir?j*7=J$8K=;ty!?pgj3J@ioxqe
zJCjcBsGOXxJTJO{dFRseD|m#OR>(iJ;I)Ws{oh-7>}S>0kD8MWPhR-lJ@>rCF`tT_
zzn57IJ=X15vc!6priiWH#Dly(LJGYJkpiohG8{Vdp_V`AsL+m&KBpwle(}BW)%E(~
zjQhO~;p(Cu+f9X9uf&E&Y8?C|Q?0a}*HZexU8lQq7S7!hzAdrqZt}UOOFG1icqE+8
zt>);i)?T<JDd@-y&gRQb(wh`dzEa4&I88Q*`{&zp2NHD5c+Pjpp4Ztur<=prh+~D|
z&7Q;3t$+U*v?%r02}JU*@YlCEk(Vg=BtfE;bMd1H_P89*WR?TZIsyVuh$qXqz06n|
zm15d2)!b)kwQ{x3{JE+pCeQGCcz4YzKC53=Mvrd2W03gFFysH(+k50yqgQC!?pd%#
z`RT$<4MA#8o3wlHbX$0I-SE?!aH)24NAkkxw%x(@awp`LCGU`Z()FULY0o9;!iT!N
zQXLtSFD^N8gPCit50{`|LgQ+W`A?P|&5fC~<blQA*?U)Z>1uSRP2@Vg_Q4}((I>Aj
zXZ-X#nYnMVh4jC<%bjewuDo16=Y`;k4xXzg!_FOW`Y$oLx}a6hfH}>K+4up|o)^2{
z^#(HElF*yT8@0gt<w+)YZH=ygqdpv6JsFb8Tjy_O(oE@9l}~wI`ivu6H2$p15%D|M
zP9|`xwVW&Dy<V#qKHWI6r1m~j-=&P344MV|SZ2*%vPXO3iOwCCcld-JTw^}>Ke6{*
zzk?IYHI7FyYbM)tsc=t_vYgWt8$Z$F&Ylwsm6yh!VCpH{eR=;;_3jruPDdh6S(ta7
z>S>Q?mOM6V!VU=;xto)GEyW)02`jqTy<w(@#iA=oNr4fqE{yK$Y9v;cUete<X_n+J
zc>2~lQ^U$Pf$NNxu6t%vz2mro>R!dO+?%I9Hj?O3ay+~0fwPU-3D<Ajt6nH><4tMg
z6VTYpvG-qx+q(NYsXl%+FC+vL#3#S#=$U<Rt-y2bU~bRL0cx$A45q7OzwCcu=(j>}
z)r-9{iPMhTrj}T3+A=A%&3U=<M6ulmB3|tFSnx$g|FBc<vvB>_oLY@jm-<XO7jdy)
zK`u4z*?&EYW^?UU>&^2Y?&FBb*~<UhAeHt0|Hkci8=pUK6IP1x%9Z8FJCvR;>wEc6
zx`Ja@v1#~^zj?xbQ@6jpyhNkh?io|r+79t2S59_>mtOSUQ}wcR_G&@PJBQiU%=+-+
z*_n)IF%O;`$$qw{E8MMzm+$@8t=uil4|lD*d|*Y#L5-eG9}d^JCU(u(A^NRDtS)ii
zR&iECja7wutKYT9XH3*$%@gB%)XAy5E$N+T`r;!BeTLiiyinp3GVVXbEaAT9_<80%
zW@0<d&1|=vl`i-4wLW*pX4`9(`O$W2xoYa~9qK+YOw6uY+~N2vyLU#B&T@|3x|299
z7VhgkBzF0vq3Y*C>EQo{vY#cSucR4uEx4lgO{4QnoB67Oh0}68k5o>RdFaf$`qa0z
zsarpAp8Uy~W^_gB@l(-lcIA#Cj<HvCSG+Yees-tqx#7kBdwr*V^L|+5wwSro=f$^@
zmu61qpWI12`7Zxz`el*u%*^{b=f8wK7cHIIq^r=8w^(C`)OOxX&O6)*&$CZmJ*SYx
z^v#U@^5O&Uzv(ppwd->F_TKF4b@kbo3pR0?*Gy=?G)>{*w5*w&wkpi;RX%L7Qo0~G
zw<}qp;^(&Zj73!mQw-|onp$#vzs93~Q)=bCmv&jYkKew$rnOP#{;PFt*=FndHy(5U
z)SSFUkJ;z}cjm%R-3xAiZqwQMS1f;F^#7gr9&7a%hi|V~((%ddTep$M3y*!7zZANh
z+9w?7%}C#LmXpcren^h#@sz2@uU8~ZG%D!vJ#fd$wJhEL$4Z4Y9c%9>XgyJRz46pO
z#p64_TKvE4ur>TeGtaBtn=}_LPmcR`!LageF-KCGK-h~NCs)r}Bl0eLb)vpwy4dxq
z=VI|6S-+l}vi;_s^!%%N-+CuDgmH-rCiZ4jFI{k_yZ-A=yKlTRriTAqs<gzZ>$ePR
z;Dk9p=CCW(6fItG^t&k6r!&XzhaEq^>|mXMipa0-fP_9doipd9l%LN&vtE!nom+ZS
zYVU^!Er-vk<$q8TeEHyvdd;RApO)}gSNBTz)l9tMAG50O|HTg~9XF%iHLNp9-mo<C
zv21#>oce1|+t(IuxA%5Tn;LgXyES|7z9O~z9G&Gqv}(Vtn(;%;G`}G0^|9Wf9koFv
zUQ=JGzx|lMd|z5h?}q5_>L>IRm|Ih3d%lZ#(Y@f${GMG8|6gO{Fk8YhNl`UHVPzAO
zI9I~u4G#~ruVD$86ESfSL$`!1m&Vm&3mEhy+=HHMII41TidOihB^r|+iA~gJ%@N$t
zxa{m4k^H_VL7P=CEMU4bMe$Z!>dbkmJ5-itOh|T{5a8}K_s5hgN5YnDE0SR?5uWMG
zew<6exa-WcFs7WXNztoAb;A~$FuiTRsCRF7mFBIl>I?Gs)_wiMtLO7$!^8haI%=;b
zzuq=AY^^-^Bu#CF%+#)_vyG14_+s>8$$a-Zz00kOGP_j$xOEjpzP`S(DgE-g+~3<y
z3B{BYPrP(0NtZD{S5oC_*5$CBC*5_`O{f1X4vScxJvHvGMd_RC%E{aNLK~uXNnBkf
zQXi20Y(a3tKc3FQwDaF3ZZo&OzxQ;?te9m7mhZnbRa#`K&;o{aD_9-`FihLHAltK0
zp&@%k;Xy|6lszU=dD@LC4dKeYMr{#sr5-_rQ#K}fTW@Q)?x*{9!zNDczh6F0?`gbx
z<9x5htsAEkOn+VGPu}<A@C<wBTgprH-KBnZ+io-Yxny$APNoFyyeUD)riM@2sKBXb
zD{*Q1|8I#`7+qs0%uEbeGjWO9tgKC%^0NwqN|ptboVppro<FT}*-WM<p3mkBlsPhq
z3rny^T2Hu|<gcw<H8r4fqMVE5lF3R-1eQ#mxJc7+w$rVZB|Mf#xU?5XEfHKP;aU<P
z$=Uh!@^qEl4f8xUY&+69L-DN2GjHR+TV9LVB&y{2$NV@Fv~Hv5)|`MvhF51sxd*D`
zdPz@OdT9lRrMPrNvaD!WMApg|V%CMdhL7Tzez*!IwQycrm-JCcEJasipQw0w)YOBX
zsTp6hZ|+}u_hwLwO5zHsfcc_#-W<p{X7%ClR@azGUc%E8K2B%dz0I&imRo^UJolif
zQFPiuc3EkS)o)@>{a<*lHDKSoV}Aly+^7}#Ty}rr_o%|PKdxV@dL5`@;^uO=CHS!3
z-SYT4-)_%VhJbFBk|6h$s#0!Ol2TLyU1E<J9`!8sd~wM$-1=IFA{XmRu_ZTx{0nqf
zg`U>P{_=u*g7d4nD)-%;es0%Y$MVi?s$A;M2SMJgOJ?UOmL6bTknHqt(y`@{dJQwD
zPw%vRR#vw0mPfyRoY@?0-JXjtD`xSUJ=M8sxy*aL@)^rF*{9~cnYr^-XU>ah&pRj1
zF8Oy;eL<Svy4!0eX!1uUAMI7uTr$N=zIN%k=>Im~uGMc5SR&jx>!NdLZi;|*l%lP$
zzv<TRpL30Nrmv|Fw$TvUxKTFy+qM6xTM7*_^&)5NV6Zb<+gs6L*88ZdBScB8tHD+0
z8gqEmHK&OaXJ(6hm=&HZ<$Und)bnYaj1q2_HcOswX={mSKFjGic~@U%PpjS&iN_zV
z8~i)Dq0MZHFi-r7!yLH>112P%<`>@DIe8{)<CH|cuPmFU_Ujl1&ynEXKktNomrEeS
zpGlGBTNcZHIO3-LJ-PGZtHna!?t0pPmb@kQ!&vCQLvNW%X6v=Sqy=kvpL%t?zSNY|
zdYtp1d$#B9du<nvDozVn?|LvWZjswWv26nDc-x#~B&Adu^a6W)5?7sl=eB5>$09YS
z`QDu$!#-OCa%D|fSabQ^q*7<YbwcKwK3{&7a(Vgxb5l1>Te9cr#~(MOKj^F%)yz8j
zrODGaani%Cqc&=y#?7lY`4k5E*F0Eu>e0&P3VxfVo0hF)VUpakBqhvaam34trKc9Z
z)$)n|73z0OdCf1*hKxUj_b(k=xn|wP2`cBcm?hUNTUA?>;e3tB>+H6p2Mn{)xMy}Q
zi#+5yshTmE>#vbUMuPIiFs7xgnw=N&jBZ(Q*mf66g?1hYI<$X+l#a8uj0A7;(o0>8
zsY^9y&Il-adTHVogLd)FEBV}w8a2NJr5su4BQjC*iYDLI__I8xgjZ&qubs3=B;tsq
zcxaN0k=WX~PLc7vcbD&5RykQIDC2qEJh81Ci!M5_d3k9X6kVHG@Sk_m3SoyeG0qy?
z6W6t!^S-ShyL8fKhgD()i#--8ICF8vu-UErJ?E*8mGr}{d^swLQ}Q@mS48~PD_-Q1
zGf&TY&((ryD(Y9C%yEdmc=jq^+4}0IZ*FWk6#i^oiT2E=*^SRVr=BRDmbWPPg~RNf
z4?k|3GUM!u7ezOBtn^(VGNEvZvSPDjn%v|SXKV6OcQ}Nkd0p>{ov=vMvnlpXnbPhC
zn+}J*Pqy!5T^io4?A&lpa(V1T6}4@jPX00Yq)@i($lQdrinhwTgwFlZZ}{ii88xp-
z?=wqK=)y%E3c*TZ8*IF+1Z=vxG!6wAc_g$5Kb7|@wy>U7QF3_YZuO4LQx_%ja?JlL
zUwh%Mx$2SV<{4(DcNKeNcMBQ$EDR0PS;nwy>Ro}R5Am}CH;Ql<$?3Yabw4@im%4)M
zSpKo}43@J2YuzTzU(c1x70>G_q3OhWZA+m_-5>w!Z%!!ubn4)#UJ}sV&8)=NyRbpW
zbA@A&iDP_LDpT}=CoNH@0=Zrun^I$?t7AIt`I0FI*78YtP8CR*Wk0>he7VYG4cXjN
zH%}BN>{^_$G&?!s59|49wY#1!SKR*mP~4>B=WI%yEH=i-$cN6{ew1UKOOv){nWM_w
zgvlP4W`*=D*{V8q(z9<p#l9zFG8^?%<*l9yXHVsL+a1^BxhBtP|E_f^8S<N7StriQ
zT=3+(sZZno7lCSzj%P8c9{k&KB5dBDvuxA8>+t&YcS+o0oAA<HV^^eCr}?S~9{X-3
z7AAp)D<NG6c{4QT3KsHk&P@`o{<3-f%IB`d!qV;gr!2j<&B8N5OS<>=`$dx7AERZx
z3#YxQx~{6CxpZZ2QN(5&Rqf@Ag7e){9ydIH%5wCTPsg&9+Z(=T$Z$kSyp_H-v1F!7
zvC2Xj>!3qcQJH5Xq;9KdZ+YAH*SIlkexZ}w#V$2Y^$puJTIXJNo~XC;nPX9_>ij^S
zm;QP@4_qF8?mTU9oOLIo!%~AVj*Tp8?#<^m)t%Q#viu!=Zlz}UY_}ap?%sHEr_Jn9
zhSOTF7a<$&INs}>IpwbUe_ei?L&g8z@BV7Cz(w}X(hbLIPu{xTZ_0dkQu_C&%jSPA
zUBCPB&G==zm0#+;n|d#MQpxw3H;nJNH|<{;w^)3g!AH(dy{i6~_xp#xI?P$SsCV-0
zGv7_!W;0BEuD*3zjb-9pv6bmgIZyccj8tW<6x=v$gjl$ysKzB4RPcU!C+t~ZF=3&3
z|MaKsJ(G4U{wLm`dB-tI^k}4}pIc))L)Eb<0v_6Tm@eyfbSCog)P@|c<vYAIqC0f~
zTTn<_W6Z$?b2c?h@lN3Vzb8t2ZrfSQ)-w-%s#7AX_b%Z5dZ(J@te)W6sh%^XPO>#J
zxK0&0IyHt(a?V|k63*Cz$3t$2dhU^ToAE!IBeZMI$Dl=14zWpGX$bJVd-ZO$k66sd
zz=bc}ZW;O=oERtCbaGzdK~bsscWxfM=zBrZiZ9JHYDsLE%&)nR9)@}O-1YLgyR`8V
z+qFdRH+%}K_*SfyRlId+0q?Ap6OZignBcX9{Z*u!_n9N55ASYzdy9YCxl?cFH88Ng
z{u@<!P*C$ni#hA*Di-c7Tiw^Mxft}a-TSGeuS<{B+>4VY9$~4t>eqSXa7pCFy=_}A
zOunwr`rqbs;E#*;TU*?I1piWu*u{J0qKUF^i)-c7j=qyOleaX9962(3Z~LQ|V@*7+
zfdPK|B|BD}yesRF;GyEDdFPPSHpSo^PPZG3uSyvA?Q}V8`d_T8Q-*zCX!pS@KD{pX
zNilMhB9Ar(A5%PYtl)rSnZm-E0-P7w+H}M@qmC8bb>k{hdj8;{$B{XOQ<!>Nr*7SQ
z?DNdt^$9n_BBxHio8YB!u$Hw|^GbAM&vEZtv6po_I6TF~9|%fb2u&-Qe^Vra@nz^%
znLuq5wr&=&lqG?ZD_pg2-0+y-s63I8UEslW2@lO97apI8y{mh)`AV$xofOHqL`fTt
zuw726XGG;ySpy5ip2YY)*r@t2H0-`%+@s!W*&%YCH<otf#$_7DJv%t5`=xSw<E1AP
z!=9I(lP<fayHjB1g$ovvQ(tKcuXcS_b}_C-Bi<undV0xpwImmHhdXay{+CPVYIxhJ
zcDgc&q43a}87y9NTrPObuGW<QEZXhUanjR7UhmN{_hm=??k<{UDxddb(e|m0j!%Vu
z#I*dhIO?*1>&FRKZw{6U)d253^CNj8`wm97aJHZL!6QE9(3B0W-h43(oh{l^lCL@>
zG|Mco(DJjI^?36HN2^_j%(gA$U$j{FjY7{#w%d2p<=?5-Uvp!BChe}PlJ1i7N;65x
z=uy98N_r1l&5Q;4vwR<(Z1>pH`P<{7dxc8dvjyCG_s(tguWMnM`uB(k3$NEJwfY>5
z$Uv4RkrRBg8oSs$lsny8X58HTaB}3l$7%N_d{($IOYw%#GM3aFCuy69T`P{(sXU$X
z;Q!>Zld=^%mP@%^6LSi4vppewVEM+HHtjam-~i678*?L~7TzwrbSc3@?Wo(#dx^(S
zcB*u#1k70}pCu>%Nimyek)qQr#V%Dhr$vpc<R*kY32Hr}a%$F_(r3c4kGx9MKAgC`
zu2=1~W~k`Y)=z(BYIaUH_jT0udv$MTOW4GR+I-#1Pbt(IN_$3Vd0kU-eKV!z#iQJk
z+v%3x^{%ILXRvR}aW$+9uHJClWN-AGqqk37aPbxJi%e<Lwn*@}B3qO6kTFeUU+ZIS
zk+4RU2LWFeWM8=}+wzk0r$XSB%Uqdfj_1zUdv<AVk4(a|rRi%~j_*CTQ%ogy&g-KR
zDlG~sJ+3J(dH;iy>e_A?s<<?@^}p1#(T!X8Lo-#&*CFghGh6G%IT>y5j)#7nllv<z
zJMQt6SZS|C8g&mA)?N~P$a3Oe$zzp?H*e&0rtW&8bycfm+Y!O7jDe?mKYd}7oX{R-
z)*7~~{jQ;(<P1j@*9Un54@8+{&ZMOtcM@ur5IeUhl7&lW>&$7dr@Veq(ZdrsXGc}G
z!!^E|J$!bjW^fDoygj2Ced^)z+F2hXgH(c~l+DCXGp@O~EU8pCOETlxz8<&tooT^6
z%Yv33$#`~eQJdP`NoqN6SxZix7um7Q^MKly(pR}+d46|}_#Rml@MKwdt&i?CrNp_f
z+V`<s3CYVnu;QP?oW1uZ*T4NQ_j8T>i%YKUG7nuh%-<<_yGeG+&j!{dG0Xq{Jj2$~
z!@{{*JI1qBrNio-LWJD3vvu+Yr?Q=2UH5qCKkrn)$3-V+?Q5I8EAxb*40r5;&2x^K
znWdg;jBje2pZ3LT^HJehLhrw5^=hf+%brs^G;RGMxwW!EvweK@{Cb}Fc{EN<bm_dL
z<f_rYpy4Q`>U}YD(H0(s3Ynumb-u^@V*LX+V*gI>GrGF>-;3F`T`Zmv2@@NgLmvhv
za7L896it(RRIq9V?-FgJwb{AaKUTTrtn18PdgT2N&-1!#rDfO0{nLy)c1?a$(BUPQ
zxF?IvoiwR+#^sx8XPM6D9^hNE#?*CM$&q&p{?EIkclGu&IaifJ?<X(bcnanOb!_xl
zwR9J|zt4&4r!73s=IE?)w@!?gn7&HHMbGVxTKyF@w>OWnF9b{f*}&;}m+^2`fK0N{
zJl;@+iwcs`VTuwae!-C&)-h~3T*1SxzQ$d83scQEoil1Dj!x3{V%2YcH~qu8rAZ(C
z=5%a${Pnp{=_Us$dB;zSvlO1>P59*NxPG%%(x#K_GSV6!g7Q^XUEFqcjc|DW!7TA9
zd)YtD%;zhM*>7~eZb_ij*2`=)SO4sq>=oo<^g86?oEO<LbN}yK(e%cjljR{pvdi=U
z4$*iIwy@ZrwV!_lb~)eZ4&2tvQhZ=q#Aa{Z6KAGPYx7;ZSn>av7YictHu9a_HPu6;
zRWoLxW=_cCpG%YeE&OmH_0qMas{Pqd7`MC%(B1Ja^~6+duHT+XrMgQrx6F!CNsBYd
z5?OUGwsTeLGEvpmX1j~G)4sf$Thiw7@m|=+`%$m%7a5xQMSoRX=Xauw+l%qrQxiXK
zh9_JNlCP)y`LXQjs|4u^U+wQM^}NbHVMkufi~nVK+<6Jh%QTjfDbb~NT$LNvO-O9{
zdy`4h=OKsSi+~-D+jcE!Gt9N!c41RqocTO~U=h|xevjaWrDu1&P;p;$xH0*c(VL^1
z`YroD$jvfHeC4_C?&GGoj=F;28%&RTR3C|}Kl&-P_T#({&qTB`Kb_a_&kQK}ucGg+
zyY=UBv4EQ$eHj`3EB<V2S^4RU(Q32A#QPVv9+A>B?N;M>q_e>{s!edyjo8M|TRDQ3
zgdDr=6Ob^qQDlKJ=LHQ5-;SQ7&C`8W&iFC2VT#23NQJPlg@M5wcXf+Oiw`u+Xo}XI
z7oBy<;V@_1u7d0(p0mF#50F`Q{gXjGQ_lH=$Bjyl+~V7IdS=4f&SwiIJyTmIDsp|T
z*A=71)82~g{-PWA(ND)$e)f_np;ITXWuH1TOY-xVmgM{XzfE*kO{`8{BF(O!%XooB
zG(G3aE3b<4C0{Og=<Ha>HDO(lOUpza&76Z`Dt;G(KFyL?uy3JXdW_<^d(r16J&$1L
z;MAHWx$A#Z_ow1$-;mvlzV7JiR5L$wnJukt$?HcymGkBlMcI4Txkpa1Ts%oJ<7H1k
z)sxS>s~_n;-sE;ZuG~;RZ|2*X;cqgkQ&Q($uQcV|790K8=!b~T=dAf!{{MeI-n5O$
zqd`cZ>fff83Hww2^SEwF)AkKI(`?g`vh=gd(#ZVh5&<5{1>&koXK%AjI25(o_($B+
z29d&0x4He|+gQV&wmx~2b!BdMQTg4U6(MO)r<uu`r<FJGEbG{~!TEe=^S)zQs#~-f
zU7y(8Ti<Np^f>(^hn!~PTArOh*Pdug^$O>@>t3-~Q+<P*TYAdrkLEl0&#e2ZE48?A
z1=HEN&OanxZ)yE+nK<+R$K9J-Say`ucy0^O=?>i9vYBB*P_v%tM3z+%-eD6g!gG$k
zOgh4})P0xX3@6VSCw4gX_%|x=Z1l-dX1E)@m^riN(3_a<#pxF+&N^iua}x=EQ*1s@
zkwN!_bb*fh()B#6zUQzc#hahmCR%2jWc2b+OOpJDTA2;$w<;pHDAZ*r%wB&)_+k0W
zIcx^rE7tXV^Lx6Uo69$2dvIpBWPp!@(eknd_w}#2=NY-L`>S@aW8XHf45h}3Bi+9e
z7HY0MxFgB0X!BBO7p8@JijiCe|3&ArUbw%lNp{Pj=`$kjgu60yJXY4Tow~ov@j%W&
zSB}-<QmqSWMD<v23M}X@YngiGm(uV5I-J*+&Rb&eJap%V<8C~wT`n%x&pTn`{U=N&
z@1g9eTNaFKws0Ta5&X@kqwV#?I^pNvx80ug#&o0CjXN1m8*|>=YTF>UPQv2Tdo_vG
zzeS&$v~D+RTW+>zf1A*Pk6UKidHz_G-V_m39QgfXN`T4?haEi2mLFN$)V+Mdq20NT
z1s`4&xY#_D$&T7v>N5AC<hS>!UuyrokDFS+=&+?wwV|7#nf-KuX5h1_tBu(1?hp)E
z)^nsYFJ|Y;h*yfPz4H8RI)w#G4o+h7Tc(m3@X%|rqTrz^nwL~h&C-g0W^y7??fgvJ
zbg@60mzOszQmmc!Ai(KiP^*8UpXH8Jwa{rS8>5a!1^j<;p*dw=o9PWz715b7dzZ=H
zGF4IE%6@m1s<qjRTU&y6_{=m&<&NL4Xzn4X5c<Jim7Sk^#)26iSyY8ZEaG$=J6Tw#
zu*Ucm6>MfWRr`69?G3L34$VuQyh?&s@g6l&J<+3*So|%gDWzpw<?h+#9zh1~oE&)t
z7H94+InkqZ`Bd26sJBtB`Pb7<wS|eXCgd=OHRK5`4V%t*-)qmo7Jk+O8ILyRigte4
zS@ky>Q)l*kH|2WKWcbj<SoGrIS2~Q|uNJlJ{Ng=f4J-ef1joO8HU;LtQmiypxT>p9
zJKZz*lX>{AUCYUX9K3H9HBamN<++9D^{y^g(T9sx^Y~qO{K#?FiT{tBR_JjRonGt5
z=frE;+`z+Ws?b^!cyb%d3zqGBzI6qy*A$=Kx#6JL>>fi-o7tV-&HH#X4H@~gC;R9u
zy|rq4zm}BNfzLY*_?lnhNlMkYRJYmK|Nfp$m(PWHC@$SG!OU_+0OJ9#bAERxDJKhC
z@n$U*d**f1sVj_4YXwiV#jTPPmu@{sGVMHdphPJ!_Tt&8A)8O1PzZnbLL=<60MmtD
zJ*!zB*JFy8PQ5-mDkxKBSqHD!(&ZholeUW*yBMgX^EX{{ow39>U|CkUjM&Xw)-xNl
z)3`TvS-Z4KnID)?V7+3s$?j8zYEe_)e!E%0UQ&K*M}iiI&X#3QwYF$7yyJeLx#Yh`
z@rFY@d@H?<%B}86)RW!p({$FvT7!{M&rIX-8MO&aCVSp(IdS!B)Z&aKMki*Rb<(rw
zn5=CSA>7cr<=E2i3m&%^n>>Fs#oEK1ae}j7!!w>Ly<K73W+d3$c~jin(>_Pf)aJX*
zsS~!nUORs6o9un?<dKaAQv{YxJ-RK-S5LzvP;o2wmG?*3cx%?W2wg5Sh?<>yX!flw
ztFBhB5V|k+;7EJaJg>WluRcm~t(lrK=csvg@E-R{LH<ic%IR90JohJ0IqA-~f@klk
zLwuLI4QC$uQ*`F)$&dv;tUN(Zjb~Gq*RJ2UthW1X>iN%+442+a$xK_{X0P$1xr^N@
z;Gu`5h}v4S|AkX{f^Plbz4&6g!r5sa>RVYeHE$gdSrcTj^u*NHUXfQ9b*OFH>RVCT
zdOv^JIbZFC|9$jM97vw5TJ}HbxA|m&N)1;H%?Avu6%OYf`f462@^YH7NoDPfqY6=+
zo;x-vO)k9J6Iu2#^v|DX7Gj+nia46?gx>kIR%~u+LZ9LcZh2#gYX?=l=k4@K)hgR~
z_Ju$)(~X4;O&6SJ+OX*>Xm{kvHA#8=%?X+7bknoKXtu_#p0JbA5Bv+#_D<+Y3^2Al
z_hniz_wzsRGIBknGz})Xh0kzQ6g}CIeJ|B)+Q-$=Z$Hid|L)n;X#y%uY)Rb}F_({B
zN>H)o?i9P@VX|iD)Duy%yZ$elWbnjeeT%LxM-6wgnB<N_u>tN`RZrBd%N<>gPHZ;z
zoa9);B79}X)8OTv=d~SQiq7%OV_vj)Qo@n|XT4TU-L$IMH)-N$R1WCPm?`;e%913<
zt#97x`g8}Evus$Aafl_z^7A8;*Ct)}9k>G4D6lS=rY`(kRx9vSvPh9!>4YdA-=MV$
z%|i3qj<j<u4OdvB-Vx?xzoMtNUc5!hbqB+84!I+|xf`6<c_p^}O-ac;{7RkSFUQ51
zA>58<Ufp&vJf+nANw>dxcbMGPs}6Z9U1y4#%<&A2IzE5N>kzg?P01r$mAT3UT1<LV
z^L=KlUzj7XxFlp@-<L}UCrx}3le=VR{MTdK-Fq@6Su(gGnEz8qbBT^g#LUY+Zr_ew
zvzVgJ?CWv*lIVpnsUuzUk3@>68?Bqqnp<@?a{0v>hdK3Xr>fdj`o80<oUp5RsY=d5
z{}!oRZvXe{x_>d|RXlycXv)VHd*RCD-^bJiZDw&4G(<}$9!)#Br*nncB$=*s+t(Lo
z&w8sqH8;SaS4e2>nr}<aJIAPhRux(JpnF^9#+6Zd+QMJv2&w%DQq+p~bUM>}&7$C9
z;PFLQ49>i1D{X)2W7ue6E$?N{vB9{bw6kKjDWiw3Y++<Wo!eRWd7%vFetUjdbbW6A
zBH?AXg4K)ftZbOY%c>q%c&*KeOZ4xn&E?NCC%ea7`7a-SF#Q0lr@XVTk_dyD>SjjQ
z3o%YhB02B7G>Ba5OiK(puu#PHFxRt_+q$+2KUx*W5VDXfs_u0Z|2r>7<-Y0BMOJH7
zUcE9;_{qD2Z@LrX3yYW-jTr&TpCw)^sTm7ptP;5DbmmNo8iUudyaNtlzB9HQ<h{-)
z;OZhHG<C0zx!yF+b524shKpaHzAVkGd1Qy5kyE=!e2_uxk5$nVZCY7Y6dewZdUnBz
zS)ea9!&%egfg8uZHl7xd*AJtTUKq`I#JfyF#`T!i_C0MHlj4p=FHLRA<b9hqvE;Kd
zyT*$s;b1>)O}~vxJtOos+6e3Hb?O!4H98%@S6*!Y^Gby7U+JU&S+AB(DZIO$w^+^C
zwe+N&jfVhlQ;vd^^v!j*xet`EFSu5*)UBY(qhe~wp1h>bX_x;5FG#Qz{%mzhAvb`}
zs{i}7vT2T-D{Q^Jn6ysp7CLe({Fzt%pEuh5G0Q_EUUrlpE$$BHd~)s!pT?REbKDqb
zJPi52#M4V6M0j)Tm&M<n^12$Yx_tM}w}Um&1sgTaEehxkQd_9^)hp6^LC^eKmrhl6
zM8~bu-u|KFnYEm%?MtEQQY$$V9Ljo_^d=iBN9&w)4s>8x*IlA^_*|>PXRiH+nU6RB
zP)RMVP1||-m(sj_s(085?=86UYwCiVhvf2G`HFA(h+Z~OJzuQSk+gfWoQtEt|5;2w
z@80!hG`8N6WM;42d{@zTRl-@p(|T_Y?0lA-9<z62=8sC{wtS|CT*`V|rk0%CW6Tn@
z;p~!6agqx%Zyqz+yrGpj?BRw<Ek#q-MW}8+m1XPm>gLIH{H{ABFDb3N-}L)l2H%SP
z)9$`+=b3y?V)hND0IT%0Ni98^_Z1H7L>y{6q?WV##I2YacHLdm|85C6KRf4wTGUZq
zF|#$>1EXi${I>1--gh2}mfIZs-}CTI?Aq~=Z?2MP#J%G7H+x>4V>0CZX|YHouTh!n
zkZ4|`^@EnMcM7XLj*6HZQdU|da_@otnP%Tt%l8@RJb9#{@=9Ho&m+od3G?1Xhf7vR
z@o8wzV`}=Jqp~&1Pc7)&<W-A~*mdbV({yy%z<Rl<CCjfl=UlPd%4b>pItgc<Rk3)r
zoVhOKuxe7S+LT<iCViC&dR)ElU-Ya`zLICc_h`eKGp0$86|xR<uhj5MTDGP`ySr?e
z`2miza~8|8Jb1&9%Czmkg*Z<8PkA2ye5}{#E3a^WXVkh;rYVc>oXr}w1yZN&KIl5i
z2-u%`@ZpDZaE!9UpI2W?nAnp&FHd1<6w+LDXI-DqnlGE)H2L{I)^Zn$%_*AEqTrJJ
zZHq^D&`MGB^fewDo+bH5EEvt}_<p3g`$VPxsF?U8&50|O=_Zo`=QpKWpB}S4Jeqo9
z!M96?{L-FPD6y*jW10Qm<Dlf~i6SQg()T?ISe0INLt$@M+uUm_d6zVBM<p_79^!kW
zAL{bd>c}Zki$x}Wk8RD4D)OD<`@k=+eyCdSQLn+Fzj}*46f9+uPg1C6eQx){S0#|y
zy)ZH?BTi%E7Avj!0ZzhSJr75%TN3h0eZl*XjDRg3hDx(Fb-t+TB&=gE&5B9b*tf%>
ztU-@+@`jVsw4D3SEA+ir6Li-0NuHqOYHGzAxJyf|;?e3|N@*Rd*BsKdIM#V)!?~qR
z%S4ZHtjk$=cA0~c)6=3j&e{k4wg(=Eo>Nvn(VWF~_M@D^LKy*p$TgcyT8qrO)wEn5
zsq`uS<edLP=$xL*`b&$PpB>73CglF#t!0y+_U5?c52qGzbG4^79P--bs_fZ+BxXa=
zlvYX4bMq98>mCVac&(gTc6y2qyP~1G*lyixPLqpEdYP9TxZbw1tRr(q*ZQAdj;%;*
z^8IwU?$O%vb?@K3N>SUi(R^Lu9~M>J6-+WG`S%`3s_t?Nsym*{aG}O1HB^D=;HiZ7
zO-ICHmde{Jr3EZYIriwU-67F?57;IKM!q_&YtH@Y$@0e*x{DRwb?sPG-*Z&uj|b~h
zW7e0#%C4UbKXSQwytYgBH1ZefPIy1{mHY7sCa(rRwf;V??p(+Ei$7$xs7Y@*nZ{(d
z%17CY?Q_Oc?$<p_|E-yGLhE|NGlgS3e-e~<Z(k7mf8#^>0%yZz4G-oW%aU53b;U#Z
z)tbz#4{ontxcD8ppJrn7^Rv2DztB?ETs1x6-jY1FVns2f2@hDzquSIRmQVY(D0N>d
z_o^=LHCvi?F_qYT)bLxgHRQ#Ux)AphA`Xq(Q}-}^&+4C=#NzvNGEYVBj5bY9^ToSf
zh5VQyz?J(+T+zfVmv#G|?o7E(m*pG%4)F5wvj!Ll%djTQ-IE>^buih6HBjqHZI<iy
zq^0xfSc^=qD%@Qgt5#^FeM&KOU9?l4;$v3ZwXP94P7@zY?tf%#7c3C|$alV10+$1O
ziCRLqva-=6gBK0w?B0K7ja;V_q2#OQrB{Dt+mWOxF{`$x=(+t*(sSsU6m@u!KA(x>
z>^1(T$Fz5Z`fq$ybZLch0&^<QAv2Z-+`>-o2OgP;d;IS5p5qeY^8B>sa(1n=dRA(x
z1`M3;N=w#7DCK$`;4xU#ub1Lz)$;V)@y{PV=_dMbHtC(jb7-?pipSeeN%5gBR(z8c
zmMBd%$rI5O{+yl?tafk3l+;zu`b95I@--MMG|FPkW=y)Es?K0IeTm`bGiim>mUW40
z3KbhUPl_yGGg)Few~YJo31ZrluIWZ>jCS~Z`V!OkG{LOElgIn?7@HIrnG_gj@OSj(
z_+NhXgMlHtUsrdA=eZiydkI>5mhc3nwV9L|tIP@1t@`{s>B8$%+LjG&f&a}F1G+x%
z-IdO5rqFk+SWRTrZjU5azZ1SH=a%WXM7VP6y`E%qe6#I$PG<ksx|FXr8C!qd;0rG-
zWine3u6WpJUe}BTStpwG8GGK8Bo-x3n_6tOLc7MK%-gi+o|35dm1hbUpS5gI-TF0a
zyI2f=Q}q5&1_p*o;e?!ecIyyjYqO-)dkU(uAE!Q2+VenE?a{S-w=X>9uQYETv{?A~
z4U2=pgx2~OJ=`J3k{Ku0_I|0kd8ur6>*72XC#$!0{Zr~R9%X(LPyJKgSF+^Gk||zV
z&Zg7X%t&j`Zr544$H}kt<#oNq(QhW}28mhN3trrC=!sd;7MBytC5}7GcRKJhc_z7t
z{SW*kvbRU%lTst|g4T`Se(14frx)FvpwqTEAUpm?MXK?Azm9qDbQk>NGyj@aU0nQU
zkLR5<CrhEP-BS+x9op`<=&;zNWo9og-<q>EH1bi3-uAG^XKk9o%HmTNYYEvoYn_wP
zI@opemeuy+l@+m$<;)eLMqc478~BoAzORfj=iHXy*Sn@aGCaAhXZoV9(!@=r%FcDV
z5uR(B+@GY|3+^=f+L2}!am?pa*XyrcUz4)8i?+|)?zlI!@_1R!L_hOYJyW6|^37er
zllFMh{dcdjHj1ptsMfEXHd|D)@$s}wwz+q>*Uf9@xxsH4(JOMwA>)vG%_^PqyAqsF
zmPlS^ogth$`~SZ+_HVkAq`#@VY={yI43%g3V&m)NaIKcTIOVtaf`Db`m@2rv^u7Fw
zeO}$@%T7<)@y&aRKBrQ{yyrXT?F{33(=u<V1NV{z^Oibv?o3(Pw*F^bZkoS%_{U(j
z$L8@nrm!mBQZETxRDAtwO@K(I`D=>{Q#+P0o3fjfrxfj4v@L^ohlN#K@%vdRE4SCJ
zYu#?f5hHOk?T32Yt=8RTO3TjHE_t7MW!lw4AI!ET?AxKK>v1@Dr*B?rn%W;0>9=M}
zBebiU9?$MLobSo__lTh7&Lbu>4x5~4;5l)wdPT9qnhNVMW!qIb)n{HP9=OGzFZ^u-
zYwmHiM@C}Bj#>R1b~xlr?f!pt$FC5BhSkb_)ukurygD(Z>qUe)r;0*#1Y^Ll1M^fB
zxRxkzZc*S`lwka2XMowmwO&1<SqoPk=H1ADsQ=@R3G$ImQbtcBc5cc4?pkKxoU`V3
z+->&O2@2aB%FHUWtt~67l^M1j2r^4HK4IcF?^$Nc&rFUFb66$j3QqP}J7wyo={%hh
zm2R0akHWf->73c4#9PYjs@z($^nl&9o;Oadyl)g|9m}t}^{)I*R?Y1VqKCH2Epumh
z#TanNeU^Ln?9bw!&SiPqTDkvDUp_yjuOu|pP}y*eHm9+&cyoSR47-{P+q}sMJZ~C#
zOdgs`MRVV16ur^d?cvP*%8@&wQPd=H?*By&!aF!zBN?rhB>bGv@7L^E7ID3U<Iu~D
z>H>WaWy4utqg>nzY??JDy2-R(H4CsWHpp6gbE;3p)VRzn-RIYG+zOX8w8mCdWLl{(
zIJ{(VU<#3#aU#X5Tf^jv*xcZ2CfS@NzYd*QYqDax%c(Sf-F<I6I-R8(oA1;;w>WpZ
zs%P`uWtWn-Ia}PhEw(F4#`5=+>|*X)3*-zouUdGxa>K;Uho=X<^9)tIv{+_U{lDjz
zOsDUc=~;hUBGob>+;w`%)Er0tKL;eYv<2F>b*_uHW!|{;fS8#g(*%X1tPQeRi6Zlo
zd7NafH>Mqa-6%3afosbG-h5k2F6J%Hi%K~q`#sZ-{tvNUI*ar8%%x?5k&)kzPRiwT
zOq1Rrx9DuSd27S>ZPUZAP7K%8J2t)d?8*>#CO_lpd7XEy+3txdPk(0XWu{oODSB6V
z@}Ca(`)jQ`FE_1qOWZ9zwdc<6nw<=mCmMKPHOlX7;O(;NdcAP{$>rAX_U>t2X1S`J
z*J+OEtpj3LHfo$oyClh3zh8306q5xUXH$Bl{rt?r@7MYW-06!h_Dq=Us&#mt?VWa^
z1mPUU>#rC7UX;KsHBY2!pO~5A(f6go#~x(VL^jGwEEL>jwCb@!PE5hteNAlQ-rnp6
z4AV3=)v5cfX(;)o9-?1YsAHcQsjD~hkwxRgrPqo)q?E!Zd3tz={&%lv{M2L=b7N`x
zk5jx(+bx;BStZq&e)b11_ddW<cIrs$4gWp!{5t9%NxnZ)zH9x;cBai=!&aT{UbXgD
z;KX9)Z3}psW{GZV|FV0R^yYTU0JQ}?vKNoPtC3@UD|WkjQN*c9O8L{6x`lo0yq6eQ
z#x7{k@?w-WdbMMvkFCm_HP6<+$ZlQHFKEqWwYw(T(Z9~_$W{f;eVIHH9JoCiwRUjm
zgvBublHAA{dGqg%e*T{~eRm&U7+Wtfo&Bc$)v{B1m+Gd>nwH0&%#{>W!Kl>Oz{0>H
zB9b7m(1|f<M~<LE;z9;hVO6ex1ViOxy<RFMcdocG%<NGS@7rQwq^ild=GKO!|H=$%
zPBXc~^<-2T)Sc!^dQSZKGPpsvb?vRKSw{<+&j)RLv-P19Th?kbyXc#nLfgDKSNP<%
z>LkAoTRp!l##Z#gsa|dKrCuBbw;$$;Z;$pqvp0H?&rH3l(-S_<Hq~a=Yq89^X~6oC
zMN?>r!j~NZjzX+n8qxbABo$rH2x@BI*<+yeL`B%p?plu3=A&2UYgUW!1}3s}EXk2g
z-P-zP*?fLsk4T-Hi<?-4)cpMnVw#$s_KN#^=p=kNZ!|}z$kF2ILZ`PJJN54<JlJ)(
zmG5->{G!+c(>}evu0Ll}Y|^huUtXS{?=NJ3$Vln({r&SN)?K~2JV28v#QZ@cD_6n;
zCpNVk{}mdA{4(ydc&P|Dv`Mi!Brxz=ZWG9q6l9<1z%^6tqT30<=|?;>0y6})g_K-_
zw3lpf>hiJ4brO=anUwU{XYrqw3jrCjsoa{;2h^u%JG+Lu<%UI;PIV3Y>B1JxHFt`T
zc%<XY)EU*R$EMFn$~$;aB(uoiAPd)ym&=Os_)I3SFsW{km@CHFGO@Gd*Uk%btN6Tx
z7H~;wCAx?mRC5qIn)z{pNRWls%DK~o+E*@>_1wt1JbJ=MP08FTMysTKZ>4o>Z=d4C
zHkoOMmtup#Ij_~_syk=N)vLVJn&h<S5?9WKEn&xBum9RxGPnEB+K@F(%Bm5unyEP(
zj}|c|tYBb}m{svV!0EPOm{5keZ`*|V9a>Xdm|j}_QmQrC`D2~a$0*PFzLF1*2zxsG
zw73+oRVFe-I!NQHTLkYeS0~A=ofE@K@_$CLaVGLTn|>r^g5t)wXht)`#2ATb&x*TE
z)nt}V^1PN&_{wcUi`eWHb68Fb{t!93Q)JaP0avT-%Zjt?oTtnz5ZD-fCsWh?;3gK2
z<9oK8Q4_vCkxAwMv>Re72|8LUdG%A;onn<k!XmSlRtc4Fiz=Dn!r!=je%0-*vh$oC
zYaU^|%TZgi{Z6s4nz>UcxA($s;eS=J>bu-D8rIDbk<PtyA?WymCVsbd^BV<~_9-}h
zyEdir$*k+?`sy(r3lD5{Oqucjz{@X&Y_mGJ45Jn~$z-)$*vfh5-VG<klRE^0JT7h#
z_#kvrqvrjr#~z}Z*2mff`geb{IQ4$^1g0wi*R}_7e_N}gIcb{ENA|RYv@ZX;8G&n8
zJ`*X}ck-Vrv+DHBWu7HbbCyn3Q*_Op<hHPNiF=cUVzF#i!$$1|bH%>yVp+7>Vsetg
zQmKyp>~<6CgDyBRnLN<cNbODJ@eZ9R^5~#i?F5Ys&S-6)6|384^g29ZjPZQ%>`?29
z4f&5=GS#^qnxFPoz0KxLVy%$KnyEc59U@VkFSLXkE(xY|vc3!xR$rtsdndoENYwOQ
z+b&F+B$6NYVb%pNv+0u!o~>55S8NV_^--VkzeK9menr=?SBpf~PDo6yzj!+SP{@qK
zDsHv=6JuRjoTL~REi;-Ld0otHA?vc-gBl)-A1qONFyqBlXYN1&2J47}T)I7n#DW+^
zc6t>nT{*(0yuL8Ry+KV%@M?dMXLi(?8DfoYr+lXM_Avc7S;*?!lPI$LNQ&--6I{Fx
zp0qIih%)Tw+}1PWl+tOJG{ftUx;D*8wY!y>Sas{OY-_`E>%3(C>MO<bzgav{yq*|+
zXWz>WA9|i!r7so`5dGxWeMC6#&%@i_7o^VZ`NQS-;mnDrA}s<Oo=1wfULN89`_d_>
zAd&0vO@TIx61@{U)~j?WC1|KN$=vo#5H|Uw_+EtRnAF`9|KpN#<~g+;SzOkc;$dbu
z>08d_m7k{ev%KVvoPEQb@yVA=i_$|PqANQlJpFUBPDr8E-l((X&)?)pfge=0k6pM@
z7I8^FO2O6E;AMYc#MZfDrh%@B%-4Rez0{b}`(VkrF8hCh1>8{*hrhTzyi)&@N%89h
z58FQv`Qtqn#tR#X_V%1KZ7~s6RPT8f^+s~_zP(;y8Jqi)9#1pp+B$h-&)chfg)>%*
z|0>wLd6vKUtIs!2I9=Pcw@cl@zk9;Toq;N^ydQ1d^7U5H)pY^=CwaQ^v~LSW3N~>4
ze38aFU2tYgh*C07ret$aciErU8m>3zF_>*+wC(FDJ>=-OQ)RJUsj1fP_5Y_F&?r*Y
z&}>^ap;xilw#KoGBjZv0e<@enV+%OL_Dom(Gkd+R-pRh$TS>x9fj2MwdTlpD^lYH7
z_6Hq)A8$<;$Do~Op6XrSz!kRP@pO@Os~MhkO^DNAIV|qxVPAETe@~wy_nfmGH+Zty
znI9gve(kAgeV93RD`&!N?n1+@hbC_;eDovFa^17!t251ucIj^FID6v$qnoTX(~h`)
zzL@Cdq|Ko5-+jyPkTtxl2>~q<U#u>9z0LG}y`V1YNZgEWMRlDJt1B!07#uY`XNy~y
zED(AXptWmm%8tj;LYJN|mYvJ#{jX=jga{TjvrQ8}shHSKWYAQL`pW5EJ$>(+-h+W3
zt*`uF`s%onHLLn%omCG4a(1@*K3Nv1%;mex{ULYQl_X&yu>;laf!s@S56NwCTcTED
z$n|{_)4D}@$0eLhu5nBgZ8|bfLG}0Xupg5c*R9EDT@~s(zmCJ<*1=t`W2dG}Z8f<p
zD)i8-_00yKg4+kR)~!%C-#B^x=ZMcXmCX;6HKQLLdp4QtxyuQU1$!o(ue)~l<GJT8
zKHO;u2iICTv&?mg`LgCi4nwC`&0C@5$$|?{M($~hn$EOLAtRCTdaO|6$z%H~E*Yd9
zac#OcJ>ZpA=wgZ1{ZrQ{EGgN^Ic@EmsjsrmED;iD4ajHA{5W@m?Fog}MH?2$DmzYj
z&Gg5M@kTPwk{_ztX8#kp<F+MVKY2Ap_r}GppM8vzJ3{(J=CGAT9W$D?Eo6h(xpm%g
zMzck~tk|&nuOh>a$D!{E*IVBeWGc_P&i<}qe%YPh54r2NIB`j`uQ6uVW(d~&H{<k|
zZB7eTPRslxF~?vV-_|^yd7^1?dB%G-_FY{(S6W%+^P=y6`i|bLl=>E}v`o~WySH?1
zTCG_-W4qupp03|LDkm1Od@Xvf>XargC3se<N~8Urj^L#&ktT7mC$?so3s@^JcU|2+
zu_vcRSxPtmiRN1k={6DDg&HcC)-YI25>QeOEq?Vn@P?{nk)CppgXrYQbl2e5hF!h~
zV!4IcE}a%ADq(&S+P8>v%Qn_G3IF-iw`Vh5@V-Aagd@;#+msOQHkLpEN3n{qtk#^|
zrJ-lSxG%INiajeATUf5-A*7Vi#I>-z%3c1NT1Auto7*GP7FIJ4LElzUGxHy&ZK@0!
zVG-K`QdAx$zipFyvn+41YSf2k5sQvG-wBM2OcacD$o~~m8~VcOSd-A=r+KG~3eGWE
zwluoj{uHI*ETq0!-QZBH;Kag)&8{jJ+vaZ;W-^G0T4I^nVw>R-aq=qH;stRLPxxmq
zNLZV|FSIR-|D%3#gPX9K=kylguNzpuUf}Xj4y`U#p726KsVVmHBjfcdtQLWuOaWzE
zQoK3Ly8Z}rMor-4T~?OU;Mw(}OjfM?pTPg5y5r^LO9g7on}iNFZC}x3!;(~S)YEW9
z?)fERvB5nTrZlUoMO;*2$eo~i;fdq?O#%)oayFj?G?&WV7O6VZoc!d8tjRXnTN-`V
zYVsLP$+v!(GdyZ3e=PqlEK;~B|Len)FOC*mL6%&XI2TUs-*+fA;+R6HSimmzwAJ5T
zxm6{z#2DTzid779Xg0EzNYyypXkE>!Q){MevawCakt<|lgIa09xy?C69U_W>ZvEQ?
zCYp(P9570G&>@-;GHZj!+Jfxa0<4Q4@V{vg*t#KZcUwco!|Z@|>GRW6mXx)vznm#P
zgGFq`l(em3ncK51ib}US@JodW?apWwo7g1BrlYsxe@S$LipDdQbp`C%AAMFnVmmQO
zH1`5qc>r6ppjnIoW6T7tOIKQ*pPAXE`MwBjdC4eWbu;hlHK%&X84a1O!7BvXBo)GE
zMs%HYV)iU3(vV-Cs=v@VinCmH?bQN}WTjf65W^JL&!3e(GQ~)n7um8D6-ESoTPC0y
z?q<}~o>e09f0I$b@uE+n9dAE$OkP$}{X;lvL5#zrj#&p{o=z`5FD$5b$*MO=a?KHw
zAO%joWo1Hc9T_3~A(`bBspWe*y2B#Nw``g#shE4mQt{Et^1lK}4dz_Z7t*;WS1b)M
zZ+2u-cVRp4$aHa`TJS=(wota8nr2+4z2}dz&K6j}e9G5&$$z;GpXBnI7g(if&ps||
zQf4-9d+(Ervi}PGz8%RoIFxTW#bVzgfrUp|gUq5lkIDaNv;4Cp<zIyC&h)6rZ3_RN
zq;5SP8PB36)@0SXMBv1CJJlqq^Wjs@nakW%VobjrC_goQ_SL`})0GQ7<TXr%JREIL
zKU4NlY}WAP;xLeYX3D1bl6%{dx~&J=Jqwh#D%5{Aw#`*#*yxfpc~k6?!V*!{hSe&p
zvlYT0uN2VmP<&#k7-^|pb82plSNP>;{LLDyMiEX)k!&3o*{Tc~Ki*^$5}3cO(K+`3
zqtF7eoS!nuGqnO{$Y~~OSDaMg-m1c_Dp+N?%GIOAQ)1P+P17fyY%#d>->*ZHp>3wz
z#|)v69nL)~SF<eAS<%{ZKs+NMWhVQqK*wJb>MzYKo)&%TRO~AwwzVJH_C(E!JW)7-
zv;DkiQK_VG=?md?o+4*mxym0+)!fRuL_lz%*^<7C@edw({++7#pS7X?q16nD1X)4j
zpaWdx?()|%!^IA+7i#S0-q@tOV)=v0dH+{U-SVls;W(d=gU{(3rc+PJ#|CRH6^P9>
z5REoq<YJzln!vv80b6HcMBW48_bOI%%6%7<_uXLZd%w}_?#BgXp7M_^Quq@6Ow#;b
z&gh%Ps%-RYVf3SgJAc^*E;C=VT|x3ys@|&9@G}4ZGomImTYkQ1k!rTMs5z~rMbRbm
z|CDy8LPt>#p_kG(Rs?x^DSAvyf61Pq-?Xh|s`3?Qp&Ku3T#i`lm@XA+oaB+Hv?o;l
zbC;f^V1vDuhVx;Em7K~Ax7N3D>ujx9F8Q$OQW!t$?e!5q%l-v$w>&p(OXZf5ZoVTn
zZGDqOW|3@aAe&GC+bSVht_y5_O1g1I3U*4#Q5~{Nsut*f)7pA_dZp*8+Q{86mhE=4
znDJUH|FxQSK-ePLQwu*w)Y@_DoRXek8zi($v}O;J0^dRwVFSr^M`sGF>YRNgA;!6-
z?wa$(U27FsXa0D!r}5ZYd&k6}qcLVo3GZ%g{kTf<tD;`*Y^9bbf*VcPZdh)JIysjy
zdg@cn11dBAcXKaH)c%pUQ=_{geO|qK_g)A7)e2l&E_4eEZD<fr`nrKJ>2`CLWAM`K
zo}~#|nMrO5hkJxKuuANn|7zz-rGuO)4GNd5^PJNb1ScHkS2WV|K76M|;={(u+0%U&
zn;$lIJ7WLFpuYHsQOb0W)6NDaGkyy$QvaR$uf&2qh^=k2fS2}`-HTE=o~G_QK4a10
zfR{zO%QmlxwvN8#?YgZvUGAu}*RFP@_O)uPwtp`5WH5^OyDxogu3GVXyPL9L!72v<
z_Y)#Y8vIK*C$2tG;ia(cM_K)@9gSJ#dKKY{*UCymPxh~w`fmf*>IG$+gPO`;@t2n-
zI~TH_KQgUjVMU68MBah_tkMe8O)~PiT@R%iOk4NBO}|EVx1fS^#cuXnnv1KZzcAaq
zde`oWM_29C-tC#uvR<p;&WrpL)vH(STz&P-(JfB1{GQ3LZPiW939O42n!hvf9NRu2
zr&*Vx*0LNrxa8<8hlg2uS7SUDdgZOs_^frJ*iG=clE<lAQe7=ODt~p~wak6yd4d1U
z39gMB7<E_3m(6>o*}cZ#Ak&5oTMqCkvMpQlAu&URtAoQrO0B!molWJM@<mgLi$~@C
zABu@3_SBa)2QnT0z;bxMw%~&qvagt{&VSSrey}OixXQHljPo>EmP42CMEH5nm~n27
zoaU(``@00Ttx7$<d(nhsi;IV}e}q}EfBl~l6dmCAy8jwa4foPa#|^qxQVu0%wNFiz
zB@1io-7>1w^y{w$N_yIuJMH*3Ehva9!{H#?8nZg43ASgxu8YcU{Vj81?lRVG3_HI*
zY*IN<K3#`F(2akS#i>cYC-cKk?mfV_WkJ^00RBY^Y{GAv{I_!T>$1IZxG8k!fcl9Y
zWu`Z0vWT4vQD|IfprFFAJZHsL-MU#`(<U&xG(DC(aC^FU)8RRjWcv<A-&(la=aKBG
zv#Yjg@7Z_yifWncCN05$XIJ9RMxKe5|6F)Z$Z6*FK-pKpYdoJg&A++#{_kr~ZO)3%
zy!Tbft!T~Kw||72X1HaSh)kPgd?0AQ*vI2ijr+Y07)2%g@4h2BugA8V+xFDz4Sei5
z^Gdh!{WrR`wP0S5CwIR#!<G$vYc}w4={-!a;+p;?eaiyQ)d`8uZgTGtIk-sSA-@e*
zlEBSFLh}^ecDAr@`g+mSe4RjtctkSuQg)`p`fIBW{}C|VtD<z+-8<>9Wv}e{z{3{3
z3r_AebKL8sKil_#WAgRJBbuw7s+srbiTEGoUi7xYUq)-ok>3k7v;-rkrDWc_^5@GH
zy{v%U&n<7%nrFBjYgI}=A8yrOXnkbi_7Xo4<*&6L)(RAy*8h2kO<7bZ^w#mYE`fsX
zibZY;?J@R&6WkcAyQkQm>bJSsr``Qmfs4t3f6+buQdfS_e>ddc@qd}{zi#mcF44K!
zi}&*NpFO#*u5_^i*CjPBn~4YK>K;`2SEjR&QzYi#0-Kv#H^{|z3MB7*bAOw_8H+rX
z->p}4%r_{>?Op8@WWc)LaQEx$nnqXd7_GizQ!RHwH)ZwnyjzT|pCq=(a=rL_Oq+v!
z%aKd(&c0rK-Ro^agq6e9`}gmxS@3diOo;rW+rm$>1y!%Q9(PrFtI+(>Cad+&;?LeU
zPF~;MFlEiPtLMFh8I2B=^Ve+Pm7T{O{YoV6<YIw%X})a5J^|r7`~tf19FC6!e7SeM
z%VIKk^uzS?)(x*CJ-O%gPUd#xPyX;(rE!DG!3+BqaC*o@Y)R5OcA|%8v4xuA(M8)1
zv(Nf}*>vsYb&HB4_?4{II@PYc9NsUGxZdvU-`=vDn{GZ4sCQ=dI)6sw)vBOB^Uw9P
zzRUYwv^n2hHZ^c=waBi1Utjrp`z_3z%EcOYg9A*29xgtoZavYkc8-vaWamEjD4`?9
zHo4u(l7+p+I!k{#*`8Fc`}Zj*T44W4DV=(s8!lHvR4s1yN6(#UJ5_QYm(V$(#k?0+
z8@$rByY;5w_1}ics|zM4Z{WJ5##SJjc=_05&Uvpb6}i4%C==CvlOuj|i@~41-BVQ^
zPcCj)pp-K4VVc$%i)rTXa#Jrh-wEU5d?b5flU$wT?i+lKQT=ig{=en?wd-Tzf=`T3
z_wK%8_~lL`r$8*re;yrW2Ze);oT7RuH#R6PZ|76>TcXj>?PA1h%(o|E;*~>9^6XAB
znhzdN;S^O@^EhPGC}Ons#)>l*#Sf0PSQOrxazZ%WX(Fe)?~$8n3Ytb9uD85yoXT9S
z%c-If7#XT~wb4^3D|UrLL-+<J6_rkhX;bXCF#Y=a`TF{6D*|6f^a?3YpWCqT>w{~r
z+`XqXRIPEjn6%rEXK&P^u$`fI!VQzEwy^w2U0QZmqU!I9wNhKNrluU&a^tIzL)8NN
zt`(BA;!Z?8ytQp#L~h2?3%<_g>)7sXd%Ws<{{4gX+c#ZZb=G|evw7i`8#UdBo-;4c
zanwxcYM8>cJb1aC<EbgHFE5Su=lm~cQ~5^Yto9uRg<oG=-?E<)zwl3qeQJ!J-SejT
zJFj+Gb*)fl6ct*b&>|qSLYUD@u)(p_L#SbLvx1Mup-D=61RVIRuW0OQ6k6fo!ph{K
z&?w~3Qn=t&#F7=Roe6I?hz3O{E}gotEyz!!^~4R9<+pAGtP%~Ik+sTVh07X`rj{9N
zPMvIBkmKQ^dR%Jtrj{G4j?3>gS-D&;$|P~AJmbGr%f$>&Uh$q%cTJGhJIrKdtK1Hj
z1;rvp&u5)BwvOD%y}_RGa?hEzSDq&V_}2yruZ>z_;KgvCWruRmgLeW67ST1zewTf^
zjW?Msw^Pzo4%&VrgIjOe4vo;)L9vEjuS2FLb^niCJ2^?|l|++&`0FCsd__iXpAbbY
zA+E57q}kFh6c|NPPTWjfJaw94V)#{!<(zAGtx)J#km{kbj)kM#W%{HE4Ex+!4@_Ft
zRuOdKZp(wz6U!EeT-tcNqBtl^L76pUsqFF-4^DZmbPDk})w1VSmICXPfEy3jOF3C6
zwx3na*0_4>RqEP>FFv-*8HPFLidw#wbe<BW7R=-0>z0^4b!v5FtEltMiEHK?D=$8-
zcy-y=O4DsD7aFGc6u&;3;g`Fy*&^B|b)#6r+@x0CuvN=uc}(7PFOb2=LeOlXXl82B
z^_3zQvUb1Q7rox<L(r={#SdmTO5c4~+r+;+p8Zok_y1qN^%wq6(foDmaJPoe9_97D
zu?7o+1l}y+>5yuF@$snet3}G`tSX$JT69V@G%iT~k6cv6yg`d$Lx7E8XSj9D^o1p<
zIV-lEic)d)kp6h7cEO7i7cQ3Hk5oymFmvPVTKOnTu}@gl-F`>H)LHx&PpKS>=<wF_
ze70K7H%u~Ps^Pn;$c^UZ%i5=wW+bYnZrxP!-;|Ft)}!vH?5g{Rb+=zpTILsf@=APt
z<tkBc<x7PbDF?o;T`ziSv#V|mqeGC6y86Y3PVWO!CqBQE70$D;OK64)qh>*(@VZq~
z8S+Y3w>D_lTq`_Z5q87r*9Z5)bEnVqhz3nyHtlY+-on);wP<IeY`3E1Pmlj@3PC$E
zwz_l*-EhgR+cZJ_ly_9nl<>VJ<@)PXm+<O(c?Gdt;8S+hO_0gq2veEFvRRX-H#;O^
z3R~IKl>z!pvtPJI`F-AdI3-_p$x2r18`C#WS-32{Y0tesDo*>I&6cEph+e`VwQ_>H
z)u!5wFON$++2XJ=GB<1Xm&x;=g>)NVT`d0fgKHGWOON<1$raL$r~2J)%vtyIg=ML9
zzevz0$Bh9?RX5){BDUhQZ1hpC8BbU8>8Tsc{aldV`s$=#?T2G>M;|`7JGH5!E@g(m
zf1lIsB3WYUr!MShsW@G5rfNddnU}t@b|)OB1}}dq7Q)6IdBTxr=5wbtC!E_CbSQdd
zEoSifKV^cjrh`^rSD@3i(tw&@tKGI6-?J@m7nf)BWH=hFdQrq_4(q+*q@>IscXlPl
zl_4jacP<fKtKqb?KPrd2BZ}9WW2)qD(<gPzGc^;|9J<sJ^rS9Y>9WR_g2mFR8^riT
z4lv7WoaDQCSjDU>!DZ(+<%qP4?m9LLIW@)R@%0_eIi-`ynY~q8l#gWwXXAODKn|ts
zzq$)qcgaNNa!e|CX0p85`--ssCzn<R!Ke~hafiRNrWWNMwBVkYw2PtTdS{Zi>OyG_
zv3#e9b#J{H{{EUCd~1VCr{tNQ-#WL~Y}?StmzTh;aHm1!*UJfAy{b$>D~>983VXI)
zxuPhY&8_@YEBybqFUo#zCO-+B7!~sUV3<<A;!%w`S`6WK>=lu7mPTwh=+r#p#ih`b
zC$o8yTb$c;t6OuKN|$Sgsa3jLU6u4GJF)v<n$zZ{^5E4BzowlP*=HIp{8MT(SJC~6
zcgkmrUgF={a#TV7&{gM;cP<1r6h+Tp>bKx|*R(exLA!4<UU0Q}*x6fje8O`Nft?3#
zANi0G5~DbAp;1E0#NZskDf_B8mZ;U7J95N$>GME;?(&r@bNEw(TOXKs-*A57Qerbf
zetZ8ibGf34p_h^s=N}7LGox@4Z>q*mR>?XCwNDONfy$3dC03od+oU39bSUmu&@Qzr
zdfdw=gsy!tdG)2xhcoV0&Am49zi8_o=Gisw?bpP(x|?p?5^H+u6`slZwB!1(Ip<G(
z6*_pTFC@Tcjfi&B4aK_G2YR1QOj5Vndcv=C(_4*?`+JlY^>R5>3B61>Y>^bD`on9>
zVn5|@DKYm=9t%Y`xEjq2a8|n^uu3=kVQF6FU!xgeM@%_8pKNHq<8fc>Nn>xywh6`E
z%TBp$y1CJpcf#|am<NjKJbUl7e0_0d_fs8ruE%}vb{zj3$iHr)>hoJn+voqj^LoZ_
z>jNwgZ%kV`Vak&0S~vH2R6qE=D)j60S@y~oS|3z?D3N+nAUUg}>4fDZnKC~mv938*
zvQGl+|4&V1Hk3VXe$L=(uPYbhylbyDghVD^?r{5Gyz*2bqeRk4InBlYN=~>r>x!xR
zCU6T+=+dtgPv0NZSruepImLf*c5MF1iBYR`{dcx(?yr5BtP<Voy?D;Yi(6SulM8bE
zn46z+T;pElx@NlmSugFkOrOsLd2AQaNz3Bl(tfLBXFkVVgJBhe@QX4ziw1|5^k*)|
z=LjW=E!cQ&;g%wfjrY7NLJiglt(TEf)8z{CU(y=k@!UI`d*}7{n=B^GESV_cws4Ka
zQifM+a=vU?WEXM7$AeAe)t%`v7PVcHN!wPsC~&R$(4L=p`gJtB?)8S7H#B~~@ZA2P
z(|EN5&yQ;dZy)Y|_kFKK>(^<qQe7L4-8i!Dejs!3)>q~w3b*b4d$rHJeMoP4eb}T4
z<{Za!RX47g==nBpJHrk8cfZST{=Hu-y>IK{{I&)C-+n0`{keO;;~b{d9IS$i+FmrB
zcsS=-fdgZqL+LG<wq!m|#^r38%UcyJTNBot?cUe&Vn^!&p+=z<duLDN*uA2)LGbXU
zMXg%%+D~;Z{2}=GlHjeKb88h1pC&2<O<=mb(Uw8d^wXif_Yt<e&YIm58FVgLEkDh3
zVuSF-2SQO2tO2W>q8<nzP>?*?u+6w~vI4jB+@L8=6IXsu<aoo$<EtPtO~`p!u+`QF
zlbl!Twpj>oc)j7_O%~ZmzQ?C#8GbOD|5|I8u(oP~^>*VUDh0bYdUV7k8M`kL-S&UC
z^;>~Cx3kuXxNr!T$W1cd^K%nd-a@rwyBD6V-Xj>f$oIygl8K94XNw<Q+!4B_L*}L6
z6mHw43VW9obd_Z^-}=46<ziQ%;p{1kB;rzL#QT`v>25L%=#5dHQrf}&e&eQNAK1$T
zn9CL%I<bIz>k7MmLrI&5$FBccYn`=Z)+DonDJs&zW}Q1s-w896&DiLEN_TCv)!JtF
z%8>4=242P)!X8{aWeKa=es$zVZ+Y;uqab@-%LEo(gYDP6HP;%6$1SpNF){a;vu+g=
ze{@twn$Urk1^j&_`Ufq={s~w}O`4FfQLJ%;{-hlplXE!o7tToy<Y1e)oVVqq-<bnG
zB^*LN5?B8J<Px$GT<Os?qd>6DVCSj`arW$ktrZ7vNy@dqI;=fu#UsJJ-Aa95HCuI0
zoSYrPv2e45=7pBpHES0Zth9IJ-SJ?O{VMkNFWr47@Ei>2t_$cn*415n#*2Y-R@4cm
z3lHq%4$HYe($r37y*R<@{EB0`EB84tZqmLiY?H#uxoO^p%SH`%HtB9y6rO0k$NGqq
z&k>g=syAM*TDNQJq@vYnTw<O!4$m3)xEHJMVwuqHEz)7INaxkG9bNmJjm7H^3N3D!
ze&MvZgr!%Q&)NGet~HVziv{-fyqK<E#ig6j*?Ll^ZHMKx-CTu@LMnH5+)0)VIyLFe
zCb@kt+V^<$1P1IfyrF#VzeCUI2==lgJ-S;CoqW+VH`2G$pto*<|F1K;@hjL3C0bKH
zoBot+J@vwGS3u8c37)%NleY@=WH@Wpt&zGPxs*?u^_Mf>+KuyGXn6P<oM`-DBs|HS
zPu5&_M?jvphwcvh%GaB|1T|^CSlS!HX>-}*d&PRjC1P!#4|HbuDhgR}sHn4TnR9mG
z#i}BK^Oie3o6ontI4R7+AW(I%^TXM;ii5WTycj*iIjT2E^c-C~)3I|^&tng+=m(BI
zt60N2_!Ree3Pw8VKH#2xO1|FU%$rDO#s^MYmq_k^(6e&^YhcmJMh-KVNt=wfY~0B*
z!OTQu)+0m449PjO+~2?H(P`nH(<yBI|L=^2FWAd0ByS|J2cF>mZYdlVVfpyvrVW={
z{8d>4t{Ta6dhlj1%rDmKoU*Rv4wLSUZ5&?qx(h@FOOF^Iwr&WP$$scE|7QQQ7aT?p
zntBfC|Jmg!mD-W~)`izGjFDMPTFS=fk4>w@WuXwIX*ZgDuiChEn{aL3bB5XW#3T;w
zqnDlph)2Cy<onvB;U-JKZJj2;=|)+8tt<SR3r#yeDyV93>l#eia=`Rh2lqjX9=n;6
zvp>$Lv6<{N%V@R2aqC_4qohOXTirY7x_7;`-Ll~ZdlIwn3guakOv~1A$94DU&OB!C
z$@6uM-8`?J&;TnZr{2fX^ZcIf(rpNsHTR66LMzw*ud?~t9-c3Zerucmc^uHfaxqu?
zR^X56J2xz8kul%6&_m$UPSLsiA-($bpHE~?>UgFl5I6T?P9sN25eH+VL!x4sY?LR<
zOqoJ2ht?I#6=XZQqkU_&E(zY*!TQ;gbBD{4n4spYF3~9lrI|V{hbHPj7L>lTL8t1H
zykpnm+F3O<UHMCWuif_I>|lBk-BGnca_gEqi4QN&S+#X<iPeSDLneQO3~Ia8F8k?q
z3LhwlEZgC3(#U;q#!NHTrEdgTHJy9wj%>=#mOOD~Q{amYfxKPzORUdNS+_-d<-Dsb
z-nW>Z%Cc5}Xs>rT^RV*B8Jqrn%(rjfx|ra~F?WWq($QT3Az_*SMUS>}h-dK%X3yQT
z_|3J4K00DG(=~X*HFnM2tLn$R*yK;I$#2thCxbW)ldcyOUeA!+THCa-W2c{HMte$e
zho`Lf{G0o9CZ6YzGm81W*J6(U?>!Om3EbVT-9f(+sz0z+Z)mMjNPN0c=dnTGMejs2
z1*@ww>?X-B-6_BvD4{Bq#yT;!J8p*C7aQT+nBL1f&OiQq{=@BUW!h1?#!<gsFP)cr
zM9<h<Xzzahxsgjng@dXdF5P|p+!vjXn^b=k1bU_P_0<I4Yt(-+LviAs18v;uK@DOo
zJ3Fd^wA(-5%Xlilx$|*HK{S8o<II<y6QT{SeH2(!GqFjEL$}vJYUaVaf}PU;dE^*Y
z@hBdgc*|klvCw(F7V&F>owPff!d7U`nRI3A3ZwPbOHBU6T&z88BJzm!))ik?t*N&|
ze6;!)X6dY&6L~t9+a&UCwFqne?knGGc+bD!Icjl6<+S15OTzwd&1^%S`MxmQ5%BEM
z+!M|++ch*+a);iVw(-#I)b79n>zJu}%XEEz-nw=1V`#c=M^A@^ELZ#wpRo1?7uyd%
z-cxlgQFD6n%|%@SM~}R^cf5$>+f=Q6r6&D1IKnjimUeKi_^DK1*%cVEUE?6v3!UIS
z!p9n4Ea>!`(G_Q#Y%;yZmfL!g%0nlWnAuDEmOkL#_n_<QuI{f1yvHJ@vn57LB>FhH
ziZ08X^*@fo>s)5*k6yN0O(`j6ccs#z40xtgUOwm%;m~wA%lLF1i~Tz1q!rg#B6kI}
zr0mxayT$)EVBO)wcGtUg8Lb)_^QvaA6JKqdammAApR~cVj_w6ae+4{?{0`(=MF%-9
zUw3o$cCVAi7JIgJcwUT@k!L#S^6)X=OLd_)JFYNlPV2~aUKh`2=yklTW2wx%HjB{Z
zvpA1zooJS^=%24-bi>o$1>q;A#w>UfGgC-qnRo6R$L<TNOVzdShHaY|9%k9YrSNFM
z(`7qb|1C|q`)+#O%U<gP8b%kISnZ~4UvRn9Vak~YcQSXz$js~xd~o-LUX(!S{?0w`
z-0!rkdU*Z9+#7!`GR6OI36KB7w@c`GW#2`GqHAe_0tdUb=QLc~`|qAh=GoW1?|<ZR
zc4a76%h~QP75}9=Y32%po`BBMm)F#jf)+i9p3JfFWRj+#66>=qTun12n3i$eZkE^+
zyw>-_-j)Uavu;UjWhjgZcH@8cASfwQr2eIV+(RM1#M4rHk3~MzEn=4V__k$3;e5tE
z;oIQ~ZSwZHyw11wO;p=9@8^=bmhM-<ajF0AKE2u^cs5JD|I_lg`PFCE-j}v}^IOF?
zLvDS7kl@9g83&`&m6E%EnC}Zt{!%CQ-Zok$ieq!DC&!xuZ3;P^3yLelIA>l-UY#Mg
z=1<oI?<Ofm0ae`;$7j(2GgtJ#uzB)7>dUc72i^NFNeiyIyY7ig+x0oCK5&`O_#xeG
z$Z^H$O>l77+iQM@?V56@7C&83#C3aFO5MR+$wV&e2#&k+IG*@)Snud<mEenQ*=OIY
z>G)K@aF4q^({7z+?ugJ&Y1@|EUn_OX)P1W1k3EymY{oO^Bjn2@T0cBfh@TkvW~w&N
z-HYFn#D0}=l$H51saQ-E;5gPNm%a6@`qcY}Z(Xa+n_d~e#q2~_quSg>7F~L)c>Nx4
zS?-Z?yG2O-V%OQEz59%Q+&<j3A;Yxo;!?f)C7K-!>s5-T?)xz<EmJ`3;f!}bmT!0|
z!d5wF-j^`_pI2r0bRXD03{o-PqueU{>6>KcjO7s(|IbKnd(-p5`IGlryQZbBCtkRp
z`C7TqohMLe>7|C__6JqtYHk_&bJS+@3oPvKIii++h(Gg_$5+4N&IK18PydpBw5<8X
z#iXZIrN27lMP)bhA7m_^SHH-z|MO>IMoocLTrB6Kr&@SPG`?~1D{;F%SGQT~KI7CQ
zcXK2@zG-k@yhcZ2=kcBVL7Jv--vynM4`mBhviy3pE%nL46=&|(+}Stf-_#8e>%a1K
ze$D9o>iM&GO262%bB2z#PXqW%XF2hFxGk)d)_VSfvc?ff<;5MkFZXty=W4T97Gsz%
z&Gfw}?7h(enWyd@h4~Ivb^m%6=O63NX8SVj*~ED#vda%jmHe2hbo2kkvpQO97};%o
zHZS+O9M5U}qS=?jFo{)@Q~brnQz}<>#Ju{{l__z@d+ik=Q;|&029F62Ub^-Bcy4`8
zJs)Np{r1+DrW<Wjj&z9K<v5{nVA|ZCZ6<YJIE0ejCP|sk^I3WE$;mz~(_L9Vi=KIh
zm&M#H+L}6<q5f}G)}NI@8fT~ft@&nkOJ6f+>E9_vXCv0f?~VGB5u4gRb=tMnU$R2i
zUe{i#-}^6Pp6RJA_h(EIy5n~<$YklAol=$=Pl}4X;uuT+rd^pmX|mg)GS-rdGb|oI
zn$jew9p<?sMDj4xm!hheNQS4A`;DgwIdm1Dx}fFD{bGsW)~7R9@vr772y$d%^<?q-
z|9~?(dG_71>ditIyc|CC9PZ^-_v<>l;-XY#p3>psDGh-sE7bE-=T7<dSSouf(|xb5
zx6?Tj_WRCvIP~i))200(3J#78zc`uhoR6>myCUjN<kmxT8`id%$7pYp`985pLsw(6
zx2ElnK=*PEhlidbWhM(5_={FtY!+K4GKpdP5|&4;0=tTiH%9jao{!Pvdpfzt{i^Ze
zUayl9QyBhUxO_T%&4o=~6Sk~ro*Me~iwA!w(}5F9^4%Og7u%o9oZ{lem2*%$zWC+U
zr<46psa{W?KJn)CxK7ouHioRao6BbJ>y?a{oqVh^f@%9*HC9Q%x7Va4rT+e09$WwI
z7~29SiL4B_l>a*F2j+L4Yl$t@;?#1;Uu3vz`Q*18?h1`U5tWmsdfiZCT<Liu?A6Lo
zQ>S05WI8CddQJH<sZ|VKGqPUIQ|a8Xpd+72`i-YyuEv`U<~I|$C8I^AT=YM0^?Qo{
zrAI$Pw%Vm$@s0>9yt+2*&4x=;LvKyeJ`j<<kUL~o=1T4>(_JiIzj0jaBIxMmwMXSB
zTVdvwgWmt@UasF?@$l$TF}C<)u5;t<uFZ;#GYVUFM15oC_3UFwK9?)sa~8~F*>pDi
ze$DU3U(;tFUE1==^WNDdo7^;{J-D=cmzmruxz4)kc7gHsjVB8q=015@AzgfCUe$-W
zCG%>(K0LnV(vM7)`+09A&$c#M{LigO*eGe65q@3r_qVStW=ky=Z7iQq%dHu`At|{x
z=+N8qx0x<9r|7<k2<4vO#C7u2-N#xtmxSnZt_z&(a?m>bk+Yy)vTamE=IZF68GBU4
zqs2m>A1eF1wK6t^BW$CWsCk&?jpg=lBvLl0vK1be6!a}|tGG9_A?Du3MK8STmYtdH
z)+SUHR5<<85+m&cD_x#?PO@5cvGDZ7LmClVQ?B$~5Ehu*vQqLU+w0toWi_|oXnMvi
zt$Y^0vSFplnN%UqbsMG#Zq(X)-n8Y@d6E4-3Wc`&+f(iz6wA|amaa3My#A&?gW<+b
z-CGM<xBX<&@}JnXZQCbZ?!2ekiHUQ(jw$JsrT*vEO*6diZ8Kxi)*nsk3#+`{9G-5}
z^l6BXH@cblp-58t)lxmK$(Iwjm}h%U?Q5U?IB_P2@iM;+nkJJgJ=Iy3+RTc1qq1V7
zgz$Wq*+HrYOAac<xy5{we!e#){X+GlMJt-;dP#BwGAB)7IG`-2dE%W{AamHyH$v8n
zFBW8reiHCBDzTqB<AigS&jb0uB>OE-Tbxv!o-pN4R0_8Ua!tK)h;7>kQ|_Pq6RP)c
zDn)N(^?kyd@1Qm5{)aYmRn~K!E2^(+r3We3SA0|rUNC1GOT*56Cr*0zOFrUZd@Om@
z^u0<{j;gOrm`L(Bf#hkKGx9T*?|qW8#60H9+;>``XPrOh1^(B+c-KentMz@C^W`(T
z?q%Ex<o4q{J6U_q<Fyed3>iJn&D|_G_2(nYeJ2X}!<Pg#@OG|rn0~`W_48?!e<#=i
zQg$xlO)X${XX;whf8e}m>H{ZEjgv;^wpYZWFEHKPrYg2^PfP3@rBfaYOV$aBv9@$M
zXL^O4a+Hnic6+&UvD*{@=cyb$Y7D{${QM_fp7ZKx+g#zE8TUk%d;MK$lqEDbaoVnB
z%e34KdT;uQgePCU(9-4=%re`-M94jCYrC1y1g7$RPv&<2GPYoxb<=yzV(FJQSt1(B
ztgdbkWnXosyB2;~K6lS8L(UcNryW#^b~8BX$-s4hH`hVBUGAjkPs3{)`@j4@&357e
z+i|W1ELmC1>^nY0@cB5@`**GBI-$VI?8KSe+AW?VwtevxIkv*K=9PE;D6&?}bD8)3
z7Sq#jPA)gzT=K1dDHAL9)pAydPf5*FZ6oC}p+g$wiPf!hjsITd3j6bNO-IwSvR>iS
zxn<?$wOU-CTVC#YIAL<&Dv^e-Hd|Y7?P*{3e$M-tr03x`7UrvQ1<d#DUu@sBEoH$8
zuN>`r({-$;H1d3S%)}n`F@Tlr!$H=x1+q?uS%pMq@ce$%FiR?`O>=_6LWQ&ksb(9l
zE?OX_#;(>XcI#o#w0SCp>q^&h`f`{_@Ag)<Xgj;}*Te3ro`YdOW_60>S+&l~Qt3Y$
zdUD2phHb1ezb;L;_L1P%y&U6}+9G`NYDbZKyNmAnb;7YbHl<GZ5pyx8%_#ibiKDBk
zq*gZ_^Zl(kb<)mk@hx_lvkrNCo>6lO@11keDqZBP=LyLL_jFG5vQ)h<+W%$M{&nt5
zDuvDRe<LCs54AXTwxmc}?e_3UOW~GKY-G9h;RNHlmC6z`H8__Av~5aFnIyA8N8Iee
zO+jC?Ej)V6w>(1HI74_EO&A4RtL{2zKG=Ej6x;2<KZZ+hKia0TcdfT>qo>;Szn3Bo
zn_f<J5PxmjXua}5*{a#Q_T<R#%3QT_rqtUnN7Ns^ycuL7`D|g@p|e%(<`ok@`7YI)
zdar2SWs7-A%|D_v{-=ng@jv|8>o4oL^+U}wFGk@Hd=0!&0WE1A%Di)!*p;dx&rLaK
zo#G(PIWOk*62Am)frH&EzlXL3ZCL+UKJC7JZcx{`ut^KeBQLX5a7u4d^x$4xQ08_?
z^zoDv-(DDdo@Qw_T03ptxfjYc9|LB^c55tHexm7ZxYvXjkp)}}5}F)m@G%=d-KlW-
zl&he~4C&@vzn;pK3HN$71O$h!Sr^?^sn0tn)adSYP6xB!J1>i!nr&!P|6O%sPh^3J
ztL$9{sq|^Lj$|+$Sz%_Pu3Du$!R>$DyP12qHZ-W-OT3s?byCY<lG)BHsqZgjCd{t6
zn(g)C4XXj8kidjVEOQ!~@(YX1|NAVM#90>NZlUn%zhxG)P{3s+;oXeSm|suS+SDp`
zV#BTLbthd-tWNf3-JZO7&y}egaxZ<Y{k-PZtz_;`p(k0YR!_`IpY6TlNJN2s_;fuM
z^N3jwKRqvB{O<6Y`QZ;eSzdS+SR8jrIkK;F@*YvwJ(<EEH4*~v+~&9RFHGqw4&bk@
z^s@fb$>ea`n`OajmeX->!`Jo5xaW8{FFc!?$ih91@ttb-;u10T9Z^YvK?{CxyM2gr
z|1w8*g5#-)2U-gmU7jq^Ju^r9f<hbbgjqWdihYS;yxA{3fwAr4`3)^+k7nMH$Z^^{
zA%OkpjYy88K?l11T_#-qc_S#~X!wlNfg28MU2y)NJyjq_Gd3sFh2`gAjT@)eZn<z`
z#gPcfo1Pjs<11wS&)!eBiCpn8z~1%djl^RDYbB4S+?>|S_rF$JrSDwIhR#nqZL<TX
zx)nU?oZJ3@(?KAE<-~<92NlP;bN%*gTvVZW{LBW%*j}b$ky~Y~TnjT<Tu+=S+TgTE
z)I<Bi><wFvb2v#QIL5!)$=K=><k}dP{x{)shJ?7rwe1rfGHxD{eKGIt-%e={hvQov
z&GkYuwlvBX#7N(9lDu(imX7zyLdMxU9IdA@b*8k~%<A>B5X#}WeM8}r#s=O?iQStM
zytlsb=(FJL<6sCo<DuJeNbAH^4TC2U8=BNEIQ>7ExOuA+$Gn#RzP`8fp2RCQ@fM1P
z&ar9R5vo*lR^Iho%dJ$&TW4A9t_Hr%=sz#g{?6e+*~@b*yOO*rCW~EM;&&^hA~3-u
z^~nB8mgOZ}jRJR$Wt=~_G5DKH@0Az9s~fs%a?VdEJYRDn(0b+hWQi#ZCtZV{-r2B4
zN+fEAY{5a65Dp<fMwcaXm`cOkw@7hLa7>RCi{%oK4C#>;nc-n_;(2MRw1?yEo+(=w
zc>8o74bGU*v*lr*Mk_<cA&CpkJr|lKz8nu)(=4&ZS$_j_+Ahz?6(U>qOvw(FmArO1
zSFv}?*89GKNp3ug@)d3#kVy)Nb(=Avlkex8e3N4#mOR@+mBe2xE4F>4yvpT&=*na2
zpXTPPFfgb*pF6?v(N@n*Z`+E~j_YZ0P2F?#@!dtjvG?ZBWoc??*PSqNgT&dY9nl8A
z=EkpW|D|#u`Nd*}4gQQb53&?AST0(eJ~zgMakh&m!@Ivx&5ZHQYdhm6ov>D$;aSqs
z!6M|f>PRA|`@^5fy<0AR|H*8-=8%@p3#|?&ZI?9r3r#LoM>Tp5Iqmbc*GWm)Hc@R&
zQ^bOdpS5y73+Fp#9Go1`_xopSLSo<32N{8yH*-(j&##o;(RlVhW2P6!%BCzn+f){z
zB`rmdI3r`jnfR6l#9cN?y?*+M%KMbd=8|68b)NgrE?Ky+rBQ%`{aL4+)veltlUY{8
zX#a8gUomgF*9Y-wTU?xD*)}XZyU#`VcxzPZRG9|1<dcC5m%d5uc;jDwjpctrR9ECe
zj|cvXwFDNO%iNT4MdC*DZXa*^4NV#j%QkC1(b8#U=xFTlTOm>LDsa!ju#^|UR}O1s
zIPJJ_NFzsULg%5rmmNRfvRyqY5&WZh!J5lCJzlfC4o2LZr2BS?!oSlSH>FJyTy)T4
zW`JX)$E>*wE0%0o7rRBwHSe1A49n{i4zbNvxHuyx^j4+PuAQAlvtB!Vk)M99HOg0A
z($(F~rF}u=<wYDGI$bS`8q>8UdqoNwwH?xHJL8yg7*5Z)H%sS!TZ(deWKdEe<BOI#
zpCfL0PdL#iCZxzZ>Ho6CTXJhzuJ1U1td7&oLd+n`F>=O2`!&rP3pA`_R%rZjaS3a0
zyVk5>(CJ^2VSB~d<<g<RmKPd4oh^s7tTk4|KIkY})iU+PX}4z=ukmKPedyMktMZqz
zO|P-<aNlk33DVoy*3RzG_GM^v(|Gwtg+V|-u0f{X;^WD^nqD{S0-2Yw_(cf^$hDb9
zDp)Gssubi%_%}E7*zvlvEbB7PXY#V!Z4AEeHiyZF(ff@5@{OHNh9RCST)Mt`UE3BE
z-n3k7Pte6kZe5)xRy~Z?Uk+(?G)~G<yVsy$-H@Scb5vtO)`S+VS7jLmBF`dSQgqHn
zY-mW^U2tFgm$U8x!P>K}@>zHPUrIXT_f+Iy%iWJkJ}VS^4q2>?jgXYgxVc5h?Tgmb
zl#reUCOHWw=FVcdxb2*4Vc+Y5b+_-<p4`dQAmVUO>#z2F+mlg|rxxrzvO8&>!OKNe
zl9QG<JZEp}y?EnZW5N?3mNzUjlzlHK+iJb?=9oQOWyP_5o=orEHa>K5adBj0O6k4e
z-0_XsIc$YXm}{F~t8>;NjXe)p(=+ZjX!Q3mp62s&+oE`J-SpTP`ED`RK8BeO(q!`!
z&lTi)UJ9B$r=yd{lc!&dwJ>$|tc;5;wX;={ay`BDHq@+jV_<mGkvos2KVX~VI~le;
zwZgY&sTVrD6t6CvZpGfnz;9{zD$tHa=C0U(--}b8A3XQXqfvWDla257lA|j{Ox`_y
zcg&%&g;~C>Bjk?U;Uh5&H+))L+FUlU^|aZ4i3s}A{HiODD|y+SfM?mU%QSqJJX_>r
zzu~admyWP07eW-~%DlRfcPEcyq0zUw^Bn`+MH;&Do{I1LlbdyD{)b)5Hnl}vFPs~%
zvWTDE?Ph3X|FqkQCHV%Do6}#k&O9oW7;;tM;@tc>tKa5atT@=lyy?Bmf-|KIvl#ZQ
zej<6Qz~IywkMpN?2x@-0^8cKGq{rFG0li12GTQc?O_bg6w}jX4z^#~?6FvgnogLFM
z1g|jJze>Er=s8F3Yoz@5iyW&L>FM-3sx4W;(9y(U?cK_>_J8=279R7B7x<pMOS(H<
z;w@XwlS^rx8#q4ZT}ab&emo)X+{}o#Lg`;-m*ig8dMTpLAf)2Z^Ug<)OLdBuFteGL
z|NJAqr)CE*Z`~ux<(u<zaz&<O{5>~`RP8O|_x2shw0ru4XWjZ|*Q{GlSuJPT_uy2W
zr=#Q_u4ASm+(sW3ONIEDglzIq{LQdM?thH%@~?g}E_)gjUu<d)2*`+&Uoz>y6^-ZX
zydGrzINRzy_hI(j_@wgrn+><!IpJFM;_czP+#mQ>U)txnQAjE7$7wy;NBYsPkBA6w
zEqmhA!)cWI-bZ4i|Fk7LL-n&aM<$xIZJ#xJ^_*;>o`b)BwQ<YGMIKG6NSpq@*h6lw
zU(4S*m$=5|T0u){moC4^amsP(S?h0^b*toBZW`YT_-<Kr;L}D=JEaeKPmC2Cc{eV7
z>U-yqT8`txmWA=j_Y)JgbRSa8-pjcq=5&oUU$VZ7+`%Gmt^*f8xn6qqX-1B$fUw~P
zo*j!@&UqIwigKLJ+5g2`!NIL0cT$+A?|n}0jXnEWOQ#qu(B&&F;^;aR^0AG3s~3-Z
z(B+ng7qgk<j<fQ$I|rO+^V)j&u6Ggh2fK*}6jwiJvOMMTRmOSkg93>VZ5GATJDpZ9
zonyK_NP)R~k=>L`Q5VltmXq5~rk`u&6l|Sp?H$CEl9Zg6Y&~06s%3h;W%7Rs=Q+Y5
z#~1xC$!UynIrmM3b!Cc5>f!mRzmIMEdo8Q1Ps!-f&d@HQX5SJ~SL4arzBBZ9y((*7
z9@y|lvZP?M8UL*eg{d(Ug};iFH!7SEYw&4ay!=HjqwPGori%)UI~n5>T(nnoetfv}
z^-ovnjEE%3H-=X0>@_RB{uGxpUsAryq-`VcFyq%nO|#l~?_LgTotM1YCQmdn81pk5
zMKT;kJRDazNLyWQxwuW<@9GPW_XSfJ9~D{Z9$B)j@>k`0X>XDFE<T5!hzWRz|2($W
z_i{)6^+5Xv=B0*rwuw&L>mzwwX_4g9z-@=r?sOhxG?-KLwI`c}+w=E<_reL9NfRzV
zNEKMPLUiZqpZ@<_b+1mG+VuM4ydAqavus7)Z*ojLbZzSOSBVC(4t@fC7SWqecFfrP
z$5EqYW;DNchw++O-)FJE&Sic5=7hz8)h;Q`_q-o?O*DM_`=-&2KUcEk*2n9-znOb(
z%e`gK6h!|!>VH2L|4oZ$cF$d*?aHcrnzPT`%`R3F(N|~Sc)f6c$s*;hC3nu09uxf=
zHLtjZ<sZ{xX>+N&!DmIT7;h?xN;-SKU_q+3h0w~Gx7w#3<kp+d@Nr(o-XqcRv(yA9
zZO}D%Je^}1;{^4OM;`X<aa+yO{jci#-rn0Mdi7@nZ9IK7S<QO0@$Sv)jOVgj{!Q!W
z^{zg6`d{chuiuR(UcbY7m$3-!kN+PS`u8%|=kz}sLdm&TR=Z5hvdNjSdy>|2Ut`T(
z?@KHSFEX|*|0Hn!c9yV3`9*W{^P<Y_CcPj1{|QX}aFX@gG0VL_W_<H$jectNd(W{2
znqgfEZ!f#d(Q4RlVIF<`*3tI6QA{nXd86;#5IT_RBxxROwxzIn!fDw?VT&z71%k<L
zjBKW5Uw(X4c4u7YP$r@ov{*L0Ea&E}O-JYL@BW(-_r~<|vWB|7zg~X1DZIk3{$I^S
zm8>h*0~UEr=KGrZe&yP#t+%!+9qkgC=^MSxYhuyI*Ws#6OFSgBzb$G}$-G`B6CKpR
zvZI)_C)CLCR7-cT!KbIYr>{HCE5ROrr)lD8cRiW^dTMqx28Jn&b8LdxJlT>Zdtwce
z_PvN~c-Fnhnf;qYr1C4hIVQ#XUhEP+c4L}1lShJd#}waXmEZqm6s|jJ7VgO@vGn7Q
zLdF^0n=fWe<<)$^l(ltLoa~l{ZTog!o~o_b$igb5vg2TddxOgt6_qWKJ5xW%>Phd|
zqIfv5d+t+_?GtV;_&Lq!l!EjA?mR>P#nu9URyZ)`DX!ezt#HO;QTGMo4|`Z*D+3lq
zM=2_EO?k%EwM^)*q2N*Bzbn&^iY-rCE*2Ic**eAd*o72s%`F{H400zwd30@SV(~ul
zcANIg6CBLDwb!p_(eP<FzU;;!{lnj^dQKgdc&%Z0CE)ejjT%Sa{QtE1sn=nj%_iQ#
zlg^q|IzLdlUEuA&WwcCqL2q}&&I}92uEQ_<Yz$8cU+Ccwy|S}=j`fs_j;2vt%#Ai(
zGWPdee0a^YDu&$F;2<H7OS(NG0SwAJSrkH*KL(sVq4!oK;Lw_IQ4ghS8z;>u4dG;p
z^<=O<F{6Q}(_@B{vH82lO)N%YC5uJP)kF%N?kzg{LM6*qC^Kr(mfV6RJt4oYO|{NR
z?2HM|$#f0xsS<K<6`mf{rCHJ0bV_dJimX;q&*L||x@GGBobq_f62~3Vd+*eVD?3Eq
zX*}E#)w}vZw@`xcwLJ{GZ}=XZcc;SO*tR=LMtUxG9Zz;<EPktW+JO1(>0?pn{%52b
zd%o`2Jky*xbL({zp|eJ_ttWlHY+x)mYu!toO|Q;(_uX#k3A}N`_d-wCW-je+i%sjM
z7Tb3JxN<<~pWdDg2c@moeQ>djZe$ShP<R{oS!CtG_A|Q=FAU4usSz;Y%WebPeyKmR
z^}hUAW^lOs+q>I`3Pj%*6dsj-Qm`~l^pqA?m`{qK>zUXX!($TZDMydhmG(Gw%O7Mi
zTHK%?B)hI@=d*PWgHqFP=<fOOYqj#7Qua)*Nr!npKRwXk?sMj;lJ?^YJ%`VdD$hF4
zb!>Stl_kONw%JU*^ESe&PXl&*yCS^V?yJw13vUg7#Z}vWOlz%i^_~1>bC}lIE)g@G
ztmR^i7P<eIu6mSVYoWZl>vVLI>46x{hoP)G3y%10HQTzDw@JX~(2*3;c}sjVqB??w
z*ydl??kRX}^Ej%L<76s#;mRj1+J&r}XP7QveAvz$yx}f`szcW@9aXKb6S($mSTwaZ
z)6qTgXJGV|Nz-n|KG9vFby`sHo7MdpSxG|8ix_Sz9q4D6py%_6<6Y79xepJf8cK3T
zp0H}#)Yrh)Ti}`OzGS+r`PntXros$44n@+|Y6tyfuPk=&Yg2!;LDBMrXH>C~5!Y%S
z^?>KA{Iv6)3U25<n{z)f;*Flus%@+fCQW-I6?4c~j@5!+?~YSI^N(c;JqjK93Rn0w
zxt|Lj+gs$^XPl^`n)JVW>XbvgsT=q|d2C?i^GRS9x^S9B$Ma0Eq;Xetf~)R2BQB!{
z9wpXALT<|<&ngLWD*N&H99*Q)koWGQ^}a-*I6bDN2RYVO@O<Q+dg`XN-k(%&74Eov
z8Rhdv8w1m_FR5Gn>=b-;RI|&$p&>Uzb?&2r_x`y*=FQ&}zW;d&ORcA{!*8`$8Z%sZ
z7mJkgxjt9YJij5SZu+9WWgoWL2Z>B<+Yt0}g@MsVj(sA_xC%7gHgp{E>N~!S<I55j
zA!VieKT4FZ3$5aE4^rq`QxxOrx{z~blv9D>YTl(10+M2j{fef!PS$ai5O&=<EmNpR
z&q?Ibp`cRNZ8KJN$unw4-VF(E{9n3!ws3TJj9}Qb%(Nw5RWd%uSq{w5-+IQoD#bS;
z;Be=RyPH*F3Vcu6CCxl~N^r_OmRqsEZ~AvLeDXhav8!-ao8)tqT>mh^+yMQcxy4(Z
zxHjKPVeoG~d2QXPuJqubnB&VB&fHn6yfkHU9_w}UgDV4+G<u#sZj<(&x?`b8uE2y(
ztE7CA_=H1`t$C7_{94fWQfcYUD|ux*e}}za7n-o-Qd;V5my?^M1o!5wohwpwL|pTs
zY59vc-pT)*mL2=>tXzUoF4TRojrKv;svl3DC)o=<=s$3B{(L6~AwFk~pOIez<sPn>
z$#P&{-yhCu3Bi@~4Vv{`6L%~U{>Sm>gPDDQl*|8i)wlD@&b;;wIU2f6s<F^L?Tzn*
z7a>e6nkRSY9ZJ4b)e_!znuF7|;?Od=ovO!6T|-U>a0%(K*%>X-$w|!?YuouXA!6UA
z!zVs%oup&9S4vWUc9cx=<(kP|au<u-ncR-3O-!9peeraS;jyDT9W*zzeoRe%6ltVa
zv$45Rw%1*0_xZ~O>{A?=8(0>&#c`yo<S%=v9A>a+j@NY0MPA7(N=z1c_<1(w-YYFh
zu+o&{lgWw{i@H!5P&x0|L7}3JeRHH`HpLh<^e9#eE@OI>^GqULH22`LPcge*?t0tx
z>&D90&&?;>zOwBr?r!1ivD$dWdjGBBdHWQFi|rrBzp&hG-unL?hX4x$Ll1)@qld@$
z<)11hHvf95%yo8h`&aQt9PZ0i?tVT|pwMb=vr*}i>c@<QI;Jb5ZX^f)zJ1`xJFR&)
zJcSZ1T;H4fsyc*ZhAOXL)xB!jhAh5iF0Fo{uTIS0btue4vQlP7|CE!J9|QN=oLO${
zJ2QFRvq=tr_GJ04pT4_U>zZPx-kEvXu^Cy9`zD-Pd)~O4<<O;U$*|1woLkz-`&l;q
zmOHkuz%aSxPQ|gBd%-COi*rtG`g^1QMxfsm*7A!gYwd$lW^F#5809)qIm)lQC3rdO
z*QmC=QHu;`H}<`GQvXgY<|t>l?NQ%9+Y45D8!7bj@GvejU{$zl$*#6*W`Dr{(&vXN
z%}O`F&whV)Rfe46=V^7TnNFXMUG!v$rUj?hKdZY__FdY&@5J<td8f8r*tO2Tkoku4
zUjBKF_DOjvQ=*etmQMcnB{u)vg!Mla_7=Eby1uXGW&T3Zv-_mfUD;k%n`nREE%o1G
zYwOmOo5~I5RvuwX|Gf%|S<W@VNqUOZx+Tj#>^M6~OL9fmeBKQ*XS|w1m@b_1TD0uQ
z;p=P~jLmv~#RC2wd-HJD>Gv6Lj$T^Nlzu$tn*WyvOn-A;b<OB|IVbPj8y%+)Im!Qg
z&TmucDOz8#DfdXzdgFu(ui1LpzMf<?Z93rNe6cBw`RfU#4NUi^wQZjAc=L;MKLt)@
ztzgZLO60!se?7am5@W+t$8)SZr>y3ka(ueV`JE=5pVw$VuJgSi_JQ-^s-?Tq&Kwiq
zJ!%m0?oq-j&Qd$4x$j=@%M<W9*1~$0#n;Q(xQd^*o_~*;$K8ylRm+&)nY9L8VPE_s
z<+qHNvV+#|OX|vt9tSuu3soiu{o(y%)4b$Mdi{(Gt8bkDysu!FmS$PPyAZu~FZw#{
z{_*RdSe(6uE3|@7MWA5onbTdXy1B1ri?{}QX0Ym|Y%JY&`t&~LuxU$POk&w%bHQW-
z$L>=Tj%|7U{F?t?wY0NWcv!UgrRV1D^Dr<tlbO6lW#fjm=6-2LTl@@7n2e8QWJIa4
zGje52YPigl{QE$r+|9nM3;zq>S)Jxk?6sIA5a5^1?PMrYoh)UU$;ABrlbPo(zMy`G
z^NivKUoL%+ET3r2HfgKTvoAYNnHLE1<oA6Q3i5olx?;sT-&ZFq?40*?oWH=<bHV6w
zTAkY&AqGaf)=6?_ep)F7igi{0(GW6zb<u%anagmYTI%9&6Mt(ITD~~`n<YUjq$l{q
z10M6BehZ_JzH=rzLA;tplF4fnnE7umxL~r)`7Z;9#LYKK;>S4W8lG9SY0@RF<V!u`
zS2rbX<4}3&!>}Vzxq$t<kTUC2gYRx_!VJNlKMdGRlU@sSr*^2zFgstI;iqK0tpC>W
znN!*(d}Enla%jPyiB{i^i<%vLzT&l^@&BfqD|)mZ2kF`~X**Az-?EM|^fPDh<~3KG
z^wbvc{%@U?_t|q<%G@_EUv;$eJx{vbwfy_H4?Z!MO!xa9Q}eLg-VkiHNKV-BvdqQW
zbIT9iQLi@P36Kl4UVqv0gX<ByM?(Dt77vzdvM$j2#;)~rz0UJRB4=M|trfBpTqfZg
z%#gc|`=PMe`^z`#3K=i1(k&N!{^v5o-Vc)7A15|l)Hyq8-I1b%Qy=)HuXL^0_=3-*
z(~Y&^(VFWIxb-WW4cU|p*S$?%)n{(_$bjEC^^_r7=Vkr+&x=ajj2x6+`-Lc{T{07B
zxVUO_GuOpUch*_lI&kS3--JW#_Y8`+xID75T577iMEm7`CHsd-3f`YRH&3#=`Q9RX
zjc4gZ#Xw_|#T{Q3aoAjH@;<B<ek0B1gosVbE9+yAl}(e*-MR3s?6cnI(_Tp;`#YW+
z=B>S*72v9Q=6lp8w(5R^YC&bDkHIgN+$|8f_bnxB)_IPT&#hl7EnM=9_32wRMbk8n
zrFBMEZ9cxUxjeaH(mRXWOH3*sdr4keJ14`_y<chJib=<U#1bb@a`M$qFBJ=K58Z3T
zobBUucro+qLz5o8f7%op5ueA@wKZaX9`mAZUq<CuHHTS@7wJb9RQQ+}Ogrvh?q+!L
zq6K@l;d2g^hBP%voe*UeBh$4LRHiPTAfWu|_~jWVJ&%_%Ulxk{?h*I=|J5&QCzmRC
z#vg6VJ#%I9JntFH6xu@iR(UmWPjELW;Zzi2@Va%y%v8<qztXomohmnvsI(dz{ov;R
z8qi<of1tzHQaHi<+wp)qzBLA+HD1&FU*yTHT`#1)Ch!CAp=W0g$%RJ$&AsfQQ}5Lk
zeCt7&l-?YpcMm7=eSLL#;uJyCd_(WY#tj;p+YI=Wb~iVOv4z}yWV};byf?#k^*8P7
z>%;k%9oyvXl>6#W(8R56qPZ8p?d%DC8uE^haSh9<1uV-NLJHm~xqZ-kyNShrLjKq6
zo(=DED-XNa9^SC)Y6_pO@`*s@tS^0br{b6&XWUy?$E0av+wI16)W54FcDg~P+tut-
z|J!m*COD+_SO;xY4xO60xJ>+Yski%dU$3h#8ZLStJ~sPYh|ke2J{P~lSN1BglyT4F
zN#qY>DDrgZ2(P-bgw6W^%PxgHO9ifpFSOfUu1o2!T7SedML)k*J^H1m@&-Qt|F5eB
zHyeb1N_%-I`K3|{w^n}7v4aNUfvT%&yB-S)f2?MDS5r3WC!b)jwU&*c=$|T{fOE$F
zyH;knhx^8_OL|qroLHV=6E1l>|E6E@?vz(I?r1TqzIzk*=4js3(^Cp{j(uAe*XMJ&
zETVU+l325={vj5Fq*ms|ECxz>$uoFN-gO8SInNeVN#a+wx?&I#!1`CmkSW-Wx0sFl
zx_;y3B-{Uo-WVnMU3;ee=6j*FQqkgMx2p9VC)q8oh&J?3nK>urMOt^xnMdbWXtgfB
zQF_)Q`CD<uo+*u=w&q;e694_`!?NuW$uG_~d{5vHbNKe7^{j$#+z+KVi3yC6%5gim
zeHJi3JHayL`vl3#iM|VX6eK6zJ|H?%R672=*-<WYU-hK_?+wmnx+tqyFuwKM-{;G;
zwK7g9I$Ept@RFYm&i9^#FMP-HKxJ#)Hvc-2(!eGBflJlYcQ2n7J}uBu$;Q@4^FPb`
zZ_9ZdeBT=AZ(Eb2Bk;&mX>o=yo0<33M$7Y==VydXpZ3DT(5^P{%#te(>N~PkG?{px
z&F<`T%Is>nKQp5BWyJ0O)3~h`^wc^}U{#o(9Fg~X=KRNH{mfAgKl+8$96Z!cDsQ<s
zX=m~)JIn75Y!~W+3q7~E9ygZ_H|ack!syIdvukV&N(|p$32GcSEi!(asT1xY?qy*l
zJl(&Wli9)Z>ZFiE&7T&vxJH_L9uD&=oHbu5NU8Ly!s%sY>0Bkn*H@gsvSNOBx%q-}
zSG5X3Gm|ej`dOE-JmF~j`qR|Qp;GwfME08#m48k+C^acoVbW=<ww(tOc(*ZIJeb~d
zb@C;*zl+SPwa@cDeqA;9$=oL<1<#o-OndCtf865u_6q?!7gev9J+D`La)E5fH6^x{
z(^r3HUoLF3QrNmZB+W25Ra>ie!H+`|{(lM*;A;5z%qCEq@m!Ob`T6jPeoV_6xc4<J
zzbLUaxLjQEhIZ|)zAHh?_XaVR?`GiUnCE$;<>k$J&pRd<FKSh8nZO#E_x0xd*|S%%
zR`tujj|d1jm6F<&V8x-Q#BgTqofYX9w<SmjORYblxqesRev?H8VITJY^WjsMx_B(S
zXPfzk%Euc{uI-#Hx$WnPw?Wq~8ARW|$2alY64`afws<@=+*DR_cYWFEDKB?CXg>R6
z>*w3%MYoTomG@=8K2+XyeOgq4|Bscs6&5VtwT4w?!p16J21|C~pA)PTDg}2Q-5oXQ
z^sWArxB7X-DtW$5w@A3I){tw!xhye?p~2?F<SloE98P@xzv<JYzV2hI^lc)m*T3!9
zdxCGjM%uaS<{pnX%uMf8RZDrjq~Bo;Z-V5d6~gs`u5~|EF8@1HQ(<|?E+<e$t9r@#
zG;OJPmDc<3-YL25)|z!8Blx^^#P63PKXldyDa<$f`t0(f9Y1H@)w16xbTIWP_x@#}
z`^;KABSPc+Y_6H@2)q$(`e=u7+WgtS=ex@8%H=(kv_e!+BZWC(cfAZlpUsJrYgJO_
zcIGlGn<(UqS|sKiPx@5m{P2+UU#>-4%}?=f-xxQ6C*WCX(2S_`D}C464vM|9{JeX~
zcSHHk0uT1qrSlJ8-o|ikeme6Kg*%g{D##ZdvVVVuC!$oc$g|NZH0@Q{_y3}O#x=Rc
zGLfGoR&{<i^_a8MT2^I;)db1N)zic0PdCrwjazg20P76#HK!XYFIE|9G%1<=@u{sj
zp)dE?_|fg9f_egHc1Qbg{WuhI?Mq(l`s(%D_qiDD+sXX>U$%dK!G?#?&$0_nN2}H5
ziwJ(L4*aWo`H0PC@38xAJ5`DzXXs8865aaXo?65AbYZ)tr<a(m*c*B1_ngo-#a??K
z2j}W^7Vt#ezuBVh@aaXRp55G5U*(;iwO1H-&0k!->QJ9(s>jl6Zq_|_PQ=>!{qfXo
zjrl9!!|-F?`|RebBU5#xwpF}2b!t}My072PZ7V<j&1yr7gllDTZ`jWH7Ex-~dHXG*
z4=MjQ&Sdi3{NSvm(`kRBVCB9qF~=B|@uw~8jCr-&a=EI6j(SkZ<6=*5<3okJc{;7i
zT!Z@_#~wDWJ^Xs^Rz<nQ?-4s!SedxoJ>nh1V0U+d{LYD-^Nudw&+|uO(%SvMo{Amm
z+`qHTH&o}}iF%vI-r{%9pS#xh#PE$1*NuR6|C#pZ`z!zQ@oZDK=>B{=L}+7(Q04C}
zDtyuklT-v|xG=2@V-n&@*cy<yI&94?jX5_iC?0NEqOvWKwQ!MJ+Y+rH8O;V?msTO+
zIv>wXs*+PUyeBCYZ<xAv27hs$*M&)om|BHcJOYGF7B<ay@)Ub}vgFaFWlR&4dZS9T
zPA(4UjpEG;?F{_C-cz(m^ug4O$W0l-O;Za(7;kSYcq}(pA+hN$r_86Kmj_f=-=C+-
z!lD=0w&CHS4t{yP!!vv&^(O4hxoME<l&Z>Qz|Av3!GVE+li7#m@Afb83Mzs9D!dC!
z88~OV<p~-@FBIY2ewJ&(qzjueC&{p|y!YN?_V)hLc&-OaVjS*2JrOMwZV}7S#KIt?
zazIxht)xqQfA{i^4WcW%jfMNXPNZG&Hsjs9-O^+JpR-YGPfXC$P5kZ8TH<RUR}}nl
z%7n(qNK*w@HoMjbE_sZqQ|kY23|$y4vWs;=8{;EWR`wX1ur>xIt^~GZwQr`6x|m7=
zof0^EUp#VAoiRb6NA1cD*Z+NLZzA0K)uudjl2qqAIfGNND3D29=0R$Bth?$02Zmrj
z%ZV8aHCvW4L@V7`>hxQ2!Gsx{6Mx=qu2-B8E2MiiEIQBGGhl)clb@q>ep!%;LMrFi
z)H}uJT)#heNx5?EPKU0d*Q59bv+J(G4NO`G9HyvkY%S+h6bW+T*=jW3V+PlTl`E9K
zP6_gO*gHkbTlnDdxbl@wSMRuSUNo8SvP~nz+nWEX(<z7Dwo^j`gmN6+<|XN_c+<e;
zWO^;?!atKMftRy1#AC{DepQ@&r|-LlrvcZp3kjW5PfH~BERF8!nCO{ylO;%FLd%}H
zoB}Nw(a%;t-K4wKY|>2OnVcIotz}GH@LwcKWr?R#^bt0Zzyhr&x62QH)jYt)lk5|H
zlqdN5^WyV`heC>5s?O+#EJ$4*D-@J7@%2dq<?wk7LPw*FG$lJ*KG!^Z`m0F$$3yqF
zjJSn2-z*8#TylVIkF3TTrwqRqwWiySM}@Ddxbm7_3d*+?x+Ws1J*7=7DZ(sTILG5U
zn_Sb5B^@u1$tEW~+P5Nry?na{!#%w+h9&o{ul;!3Z^u@dG$D1zj1+eHM_Uu?Lz8x<
zPMb2Hr_7)E_??g|3@dI1pLUw6x;&zc>qfLw@Q0bdL^iXUN0%CSTM2gS^euVuhHK&+
zYtOEG>n>~$5}wi^lCs5l)ykQ2{(&C%DsNqKm3n$d@W020nb$9PUUT+0{-(ZjmZ<N{
zV=lr!%s2zQ%PnUvFWsd4_sP}zTvyh$fnM6CVSm58e-?7-kMjzXh?$A;fgOv)mUQqn
z%qa9`diZJDjs?69<qsJ|9ga-rW9U~hle&<=60CQ$#2~NmF_Ywuqm8GYw8hOn$TuxW
zt>NziH&d?0hb{>_=C@oi&^A#y$vacY`N@|Jn(H!W{j1pAswBm3ey4Co!?X<bl$mbH
zhjvCRXk(sv%i~yzuV7@!mK(x1dyjPMUhF$1bZv5A@2Rv|(gBs{oHyUuxm;z*BoB{0
z$Ju^G<SRLB?p&@m-E{v9KYyo?X&dyeZ{>X0pDk*n5+LVp#&gy+SKxn2^kj!4ddKFx
zaLH2;TGVt!Y4_b1lPj5*-nIImq#&s0VXM@m8N70bs^L0MkpStbX^~6k2^1B_Ek41w
z^VsPZ3K`Q|Q!Z>`nKt*$7M7CT6PtvV956CpsHf<3WO5<P1a*7O7On`EWd98}rwAVF
zP}(K!X(1k}tuyUR66el@4{Sn=_bS+Xl`n>rdzt!&YO<)jkreeZDiWG{W3tNC=H=cA
zLE1rA1jE02oLDE9)E&R^rP(9Sl&E>n#8+x$*DYKe#w2~iVTaBF-J47kZ_AY)<oR}F
zNdTw#PMZ@!M;bQpD7Wyis};}8{Fq|=?UL%#rh)}_-?sRiW=hMfyRm-ZKkpThjtu_=
zqMY57-YrYKyr(zV+;H|K-y@v?j$$Ie7j1M5OgEd;;8Su;Q+tN$1FP#BL!_?1nm%)8
zyP|%lXpO`p+m;Q7`j%MCKmVy>a@d_OT6Qnx|E_B@VUg|K5aFDox7h9CU)v2iTsQby
z|4cU9J*8mTl#rl~OQnliO12r??-A_z@a=5DL^Jnt=cRo<54InA@uDEs<Ibd*n>$Wf
z6-|ptdYx1(X)yoOqz&<h=NLpMsjXGrHa&S$U(sKut)^R7cJQaB1#p>8yy+0Axbkq`
zycqEn`*_>~Y%Av$mptD+{e@pyN~}m_dAfJgfn)QzJv*bFmrj~xvt*xAqDRHK9|B>j
z{lW8<IK3bIPoC84ain_Ftq|!s=KFeX9I1HtSXA@LRiQ4qwVDwdSGfMYF3RY3I_ml)
z@6^p(bp?7=VmLPnda-Uu+MlywsR7%=Nhw~z6~5laoDWu}+ax>JJpHzOb)sU%ktqz#
zn%hsLR6T6&@CsSBVMD89+^UKhYYu-jiJZRH(;=tA+3l9dMNQUD-_W&Pu^$t7k`MlT
zSFo+X^+u<AS;CV?E{}YKEVWyNwGN+iI3b`_t99hO#Lnjx3~P^7E^=3V-T2r$>)J8<
z+0QSE>^$#$L*$Ov|H2)7pQ7r8Hf1W9ugE%*lyd8WnkwJQwYjs_rbrr{oZ&NR1&i^n
ziSN~#wo7p?bE`XiQ8Qe7$^V0rc~@IAmrj~<gmJ4U=k}0CLh_q8?fZD=lK#xwYiDg(
zyez9+Z>MXiQS@xykT(*=D!R^U`FV%c%#y!aT)ER^vi{VDtT5w%6P;6zXBlTtS`u#a
zU98~)SJ4DUL95CYQF1lGq3+J#TSQsglgrJR1X;>2FL)^2uvQ_jyJc2?xTT|%%gmz0
z%RH@@<@pLba;1av7S2#n^Q{aHiP}Ex_>UO<)E<tRTUvz<amP(uIz8jvRDpTn@4OE4
zR&5Di9CJyb@x;!aZ3i~w&YB|1!{Fd>;NnDwl^QM#oGi}gF4fLR?flfXD*jo?o;2av
zYO~fEov_*-|77iiS&NNli)`GSw{;%Fs{Q|u%~cV6ap!02gr6}g7vIiImiate(3<N&
z`m@u?+6KmAdF-~Dy{&(4oHBdmW>;`z(h=`T_nuu6t^KCoe<(14$z4f~m2YaZ_9V4x
zm1xzcOSegP&6zrF@AtjdaW5V7*Uj3ppfj+iN%ZwX|3hv6DlRijpB`;~b_=7zMa6xY
zcb+aw%hcH1diCnFySG@@&z_we_ti`0-h(@LEBF5|bA8#zGgYdCWsCWTH7-l$&zPj|
z`GWiYPi=_<@Aofvllr^YccWts$Mb~ej|J8?6#kkfmibY>$6f4Fkr<ywk&=6*;1S)L
zgCe?ZGWy>f<yeXpT-Xecs$`fm?m8%J_e}Drij-DJMD$^){|QM7$CjjPYxsDvs@-Hu
zU3{>~Mp2DvvNLC4>a8^Ah;1QrCueLauzeH2J<FlSFeTOSu+-XvY8xjSY-J3+Afglh
zJoQp}*`@6|nb%_Fn|<@9i57_)-@M+GbVT;o*Yp#KZk6HHe2N*&Y!%JdHJ>C_a80dn
zRA#lgm|1f}c;UB*MQK^eW}c?gRrHT$IepSFDk`crj%qp>wKOEV`mpknBiZ}QI`tM=
z)Lc|47p&g$BsuSgte1!2NpYXOtSKkkL(Dw75-wRX7)Wp3z~$<}y(XZ0O+kr6NNU>A
zXq7~@$gtJ}87@<{v9dbW<+kUAp0G+T%!@f-b^J%!eMRdpVQr6HjQ(FvG5Y$=m1|>u
z^Z{*+MB^Es{CpkjXF6$boYZzkIANVrL0wzHhDYMhrpo<q7cfbb(@*67wm?AQyTEL=
zz{`^Y6^;ew7FG(+5EolwZg{xJ@LG`m*TyM{l7)*{ZWT6Ix2xKyOa0!cy4WMeWQU#8
zjV9d?=e}?$=H?O)_7Ia5mj2=%ze-!)2r!=hDs8=GV!%h|11(al!8*q;rn)V$cVcYi
zTFB~sBrhvnO6G7VPq5+jir(U~^o2J>t~-`;ESUUaMcW)Ff8CGyVLQw7t_l==sckwg
z@NcT-mxbZ&?d|*v+g%bXn^u?!IY>@6VV&^NZ1zzJ=TDXTm&}_Pl~o<QYB!tz@2$vQ
zlUil-D0^1PbiJjKsvoN6yy%?xIobcGwBCyJC5DsEJj=;ER3h^;c%1~};sU!Z8~C<*
zaKE|06||uFP-)lno2KX6Vtfis{ViP1g}VeQFl_bAOE&NgbQN54dG_^E8TXg1+>d-}
zKMC!<?0d{2;HpGjz~Nqt#PAB2cEgPc+{?ATe4n#$l9sHaAg8#0_)gp3=F@Bq?Nk{X
zN{^&(O(=XZSy<X3QXr&A>8kjvDHBQ;H_mO>>tzg9Rtd^zcbRce*lJmhD5sR;f(Z6c
z&c{n@j4jmcw=CFoCC5v2VW9a!uhtNkQfH$hQmQLsyk0b0xt2I^sCl=t6iyX*bAW5<
zNxuKd4W;@mGOH}4mVXpqm#AQ)$q=%Y(^r)-%ze&g!zob_Qvw)W1Fp1+T~0eR({-6)
zTfo94S0<)&TrgtpaI0#cYU|jSeRb*=jl#tOLP8reLbS9dOcSU+8W63(7#yIXlF)IW
zO{vR5W&N^_D&r~*F^NeUoy)^JSBF@vR?nGj!I%?Vxrx1Ng5*q<pW+)lXEHBj<xpk~
zawy&NLwf0p=!w6Y<#%=Iy5v4f?%sMpa7_UB;s%9%GrBcSc4fQz+)!-wHEvBY$lH9-
z=fsay<%ebqf1EA$IP^if)U7AJb&mNb43*9vEPE-=dX!l%M^x_jRe>|bHunx|{WBN*
ze!YOpe4dbc|Nrly{VxN#f8LP&eS=Fzz(eh5hXjk>V<Ww9I~(V+D;aIIQpyPal{WF+
zE`94#%i!jj<}I_9Wvo!_ObI-0G|PaEdod^P<Mjrb405wVrfHRIWiJg_&bsH2pn8|_
zk`1g8=5o73HX5z!w#{(%cFhzCbP1X26ILNJnQ^tfl&rItpjV{fwJTFvPR)r7PPxS>
z&1@kUd`iUhb(`^F<A>tYaz(WIujT7EPg}czRXBhx_k)shl6m97D3e3xlV6xO1kVVX
zUDbTK%DX7C?BR?P*EYqR&N-@58ohusN`Q0mh3X`2*5nPGdmJa5XwD3{*<EmY7H1<%
z*DBV)13rNbEICQ*3s0B)pZsdp>YcNWx-8y0(I6_pW$RAqvn-<1zxi-A8eabqdtrr+
z4`*H6*VUIx@|9ft-aM6g>26c8WnPop=C3IQb2iPrc(aa))jzM*_V#80fsOSaX2^YY
z5q{n&@YSH<<z)G@?#hco*FAd_)F&vZ`elOp5jBfe{ke^@)4inT3rgv)XktrBS@0`b
zt76iX)iY0vmb!~gT3oe3QhM^jUEV@m+bp!WOiuGHR^W_Uu<fDuh7Zz{S$?OOZINHw
zIO|9B265>{52T$vx3w6QEY$Q|;q+8W_;-)luhhq8R(zKY?OyFTRg|WC&^L95YmZR*
zGZU#>h5`XrLQy}Z)3;2$`0?OE?#xFw<^G>sJeNtJCNayd{djnc0i&@~XYI`8tDa=f
zw1}>JsIpCTMf0t#4s&MO8t!0Nz`FQAHPZpUEgQJF1G&|uSUq~Y7k5>!pINdfYUPve
zjjJMKqHGQ@I~|=pS+>(#s9l?(>(q*g?V+!x<-K_(^S(_dXZz}c>uOI`{Mv$cUiu~R
zvbFAz@|t@;;^l0F9#s|y9v3-zs!dnX|J@7!3$F_F9Cup=_ODqWb5n~!V!QTguHx>(
z`A1f-Q&&;V37Tl1zCYi(E5TVxedXR+m$yB-y?yzM7D2C2Ze}_4Rf{GXuz7x16ls_e
zJ^572D$|6RmCS2SPgymKQMqN^#}WpE4c99c$EvAY{oMF}`C}J{uGP7R3{ONHtUI`5
z*~is9i_&jz<P6j*IPbjXzsQl#SISQ*SHx7=h=1J7(OBV_ESzG%sHl_^A?-c2G^gLQ
z`o*8~Ia7_Uu(2c`*!OHRAJc&&i*Bx5tiWm^b!5Bv<l~{4mwvC1@mjU+&Z(l?!5NF1
z*LAE|myvrdtUNF1Sn>gbQ;h~)*~ff7x~DI<3VzVzs~K0IcD%%APF%)n)48if9$NQ*
zk~w&4O4zTqk10zZt<d^6^@_&ED<4w@|32;Olc{?WX?wY9H&X(?{$Xv2aJC>*-OrQu
zOqp!->(Iq(w`VyX+t2dbdmZDVLw~kq++FY~+d1aV>64+(Vz-4vx)x7z5d6QjXHl{M
zYyH~=r$e3p-(K+V&}nt&Gr78x<K|2%vppiZ>dZ--o9h;al!j|sbxHT!N);(woG-NL
z_>INN7hRZ{Tup2>!bFarGHSc!BN#J5HZW3WTf!Qv%~Q9$74O)iAm6j}(~OMV4NL#H
zU-_^`AUR8Ti2~!vZ-*IjH;C-L<1{DaCL;^)WP{Bbv%F%uqb}@A{&O@)=K9e)H!^td
zN$frP+xo`o@_X^#M+~9`bT+Sewe1*Z;4$ICOPdYZV#{`{J(a3+_~zohtsA2xRTkcQ
zaQ&7_<MGIiQ=U(m!}V~r<Q#!J88#0;&T+XbwB^uTGwZ&Oj%z+mIU#iM#9wy#8ZlvZ
z!;>--cmL<y*#Gtdn+*5*JzM1Wn`}R}=G1v!(W74{`lQdif4a;2_$g_tqjeW8rC+ai
z>sjPdw`i6EtNWV$GgscQy!Yg<_o+YJPgnoAac`~niJGV1yPw{4EM*a73~X@OtIeQ(
z*U%^N0k1P#blb75nrCmC3OZ}nUd_4ec~kwynN8bY-TL_L==8m^74El>OPzT7=X|VW
z#@bC+HX7aew8nj((6e*8lRLB?_sYr2`(~zmsovLa5YpNmbLY<CC)?-o<u28^DE{`M
zlI$hpRrjL#&c6M7PuJNkB~>PV+UyN_&o-_T*lGD}cERhAsVrsMtnu?iQ>OJ?dEUm+
z7usO-`ck3H>}aEB<_~_g-TXiI*W7pRa#!q@3d-$z5cqJ-BiTP+>)5Xe-*cYVU?cxu
zeHq`vtIH(!^Ht@_)h_6`+m#fw-neB};-wq?x+(v>7yiCw!2J8AWUSg9ZUZA1HKm{X
z#AP3M^q$g`EN!smN<3Y?$NVn4jq^uu!Bv8}lWRYO{(BNT=ghtAm5e+sQ*1w}>{pwi
z8`^bCD0!aSxt8PRTxnLFJ*7$vdVhTi=ZR*Rxt@FIb9Yhnn%;wxcOSf|$a+QcMdZP^
zxg0OH`o4(QI&|`JSk4u>_lgCd+68sjgx4J9>XCctz;fwrN%+fiUrqi=dnmB}&DvJK
z_wM6^r;^`4USN0hrEl)hDXH?dJ3Is!bvD2HzjofXQlDpEbf2mHP1O&)FRT6RMcRXH
ze4@vge9KlZUfQUW-u{F2-h=G+1DgsTv@{)m*{U&PFXzLED~qEaKGDf&FgtPXm$>2^
z8#z6{H?F&1D4%=~%)hh9&hntPZ?Xa_*UMV#6koR*8@WZ1e5D)*j22(uT6&V}aQfbo
zx0(B<HW$jS%6tEQc4y7mNX?x2vs}|<gRbR@)n5N=z2Cv^(}_7}oGNa5Hhhm;cT@NC
zM)lfDX9IVB=s$3xujjM9+`P5+Gv|u($E2Ou7|NjNpfFumhK);R2kS#8JthmqnjHa)
z8rp=#-L}jSRBT}J6jJG!;IeRS=Ngt7CNmA6diLtZD#iS~^gl_P(L}IsSLP(%(`Gu+
zH$@y)O$}gF@w}Fmx>_`-)o89B%ZHFG@4is$NkReM8mq%5EvsXdzLv?<lESJaFxyNg
zYFnt_4%QFfa+y}k2&y;;1vv?625tLs*T}8m!TKgvZ4(=b#79S%PU>v)h&YgZ(vLI0
z%7^XqwEav=Km65leI(Kn?4utNF>z5;H}6W15Q%7p%qvW@(o}*!Zam$$cj=Z)b;D)e
zv+W~yz2OYD3<%2&n`NRW@ZwNd%<StT8$PQ_2JPOt?a{fmDaDL$J*MSfeV4x4H|;c+
z0e6aG6r;l(?hfsby8rq@SeZJqlv$n#Cmdk>IGdk?xv?WbIIJnTU3A_58#kFaSH8R4
zv9FV9PXA%1|JK}EEV_>ZocU~31Q?iJJ7>5Gy1Ts4<+q>0ZU0-O#otlJk!6Fse0|8p
z<2!A<oYwFAb1S7=#ay!4vvH2dfs<;&&S%VK<{iG|b(7_m&_bP^UyOWC?>Ta2^FAd3
z5AK^=95R~B)RlU-ZQ(I`p`iCJq}S9o`d5dCrkN$b>V~%29IL1COy#)PDi9iC`DTf5
zhsc^Tr;z271z5c_7KsIMt&ED;nd=|8f0Gy2;|-6zR`{|Uns7ew;H83vVavNr-KJ;F
z%?Rs!uqb=oj*6{a>ot~0T+=_mBBdNMeG2R5NbPv8sT&R`6t6KmdL(9r(R7u`W;fOT
zCr@_jR#B<Q;!<TPDmJ_I<ch|5+gNSiwaZS-_^zoW+>x}z{d=#T@+TIhl7&y=O1CU~
zwd(S@rFT6%C9D@lWiqd>>NB#i{T8wJ>q^O!vkLe1IP(~4{d!asX!Z4$qB3VsVJ(01
z^oPzDY-8<tvePD*FOqyFY3kyg!V>xIhmJy(%&f(W*jWr37mGgW@Zelh5!GEV(M9K^
z`f9gpukzV1@5*jj(`WRir+{hoDV^=-zTIp+(Dvr^iD@wz`bGP_!WGJvt}5PYwev>w
zqU#!29Lg(ytm9Qa7{BQK;=XzJ!j~@4$r5?+SRiT2rzYbrn~R+pv-+ky-}JcCs!>U9
z)$-3>QCeT`{^v-GRpZl@$(izMOV^^(+xh2DJX_JQboKRJQ~qVFKmY4jHn*whWwFS#
z$w#l{KA*k4Bl1o(e@^#Z7h#8{Tk}`#-nJ}bcP_KGbWynOC&4P;$nvKlVO3X8>7RZX
z+Wh;~=jp##wH|fx-rHj_ZJxD6xWFZY&NY43tABHrE2+sf&N-L#Dsrl8j+y||#;5i+
zzt6Lnr>y2>&9Ijd;Sl<;l{KixF;(~VjMEl}q-UI6Q*N+XfzADh(fo(Y(|<NjT<i8t
zo<a7kyYpqi1|FA9_bhm`9wtq`WuzpnYIxDi`<K|{B^FCVeFM7!vxLM}&p4_bR#+A9
za!=TEg}}uOf2QNb*Y4iAviJY)PS=_eClA*<toGMD^0j0bJ3TxWifm-bQJr~O=9yl)
z<w?($Vs6Q!OVqwwc*JIDI2d}pn&QN^ps}#(%f$1Xx;s5PUu2c<V^V$Jv&|yWb8SUd
zm+vQ~h_#&t>#MdU-b;A%!!*in<+P;3PqmbHSoP}U=bkD)sa0<|^{$V1<(Ea%88&>M
zsd7riPyb+?cb@Zc)ngaSF3p}U`(BmRh~>B%^W~1eD<1DZ)@ZDI`R9Glx=oW`akMmC
z<?>(X<u1bh&&4+7w1;ZE(&q0WLe2kJ9-fVGag3R&lWsDnqgJCriECF%2KQNhXU8k6
zR_NsFc?&<f=%V4IZ0GBF&M8cPrK(`=%>TXFp{t|B+z+}=-BuE+UAxkomErQtRUt}0
zR9<=<T_$;)WAfDNk8Xum%)ESfis#HA=gH|VUF%wHgmsFFuHXA);@ckaq~O{V4)ssR
zCQOT&s$0Q1FXO__2@S4U6Z`~2o^pwb&6;s!&f%AyUPht276lv+e$XhZ{&iRQ?YHa7
z_cQ0be>0ikq3gjZ$7d~d+j4cr`h;K~wvY~S_06Ya&#m4R_-Li>(v#tpso(S;&5GX1
za>4etPa=1b(b~HrNfSN?Uat9Q)p1wnke#Ym%PiL^2mZt*opVfNpL_VHuG2nEtACsy
z=5j>Ntq?0)wcy2h8}7jK{sL3O|LVBdS!_zM(^Wb=r**}G|5A4n)DyoM38`xAW;uJl
zu_R?d+o6DkMU0P{CWsx8Tg9=)ulvyD4)&wkS!Ox0JhwG8ikGe|=*|vV@~A#t>H+KH
zCrgDoS2~1gC3ZiUGF|(qx?i=`#50eI3?=haZ-!0Z6PJ`0dgFFCo5~@zizhm_EEbyo
zL#RpjK;bpjz`z*yFRYUn-dMNhkm`~G9Vd+vz8v-Q2SpB;q)vVw=A*gf!jj&lXKo(&
zcdERe_1=yjNjm2;j-A&1sQGmo*X$Q}CvDpL=EKattG$gk>zzreG%iw4+H2(L%ivn&
zm1OKBT^_hK=f&Zy+~+bc9Q{*gtEh11UyR!D=t4_f0;kIUIiDxb<2lpNn)LqxgZ#4n
zyRJD|i7B(rno?#goGYI0w^H#OpUJ&ob`xc`)Iir_iwhB#ckTUMc(HDRfw-X5+eLfi
zF4T745Z7J7ndjHJC1it;BXfGok>spYpQW*v*~6<3US5_NpuQ);wlw10+MY8Wo<3}X
z%`>gJ5+~o8HPe=HP4)TAi667Xl#=b4cSVXG<ansIDb)6AK;h#@X+bBlCU4=|se1Tl
z;U?YX1q%IBRo%rHmdmTobKmr+#B^u2_9C0qe@iAjRykBOb>kTYU15HMS&FR<g;gGI
zE9cxkxMXR{^<U4OR==&Bd~#W`PWMf%v}w!Lo>g9G<onXU|J3PYpS{1C*@>q4D&8|K
zfBW|TEstkc*0`B+&9qwG^FzYFUeHSF;hmBorgQhhmaDVKXgwA1nEcUWyPA0ay8x$b
zF+r&-wuU7W-+hTxSaw&^e)r^r@=}-A``)ie;c~rD`dawS*H;`B-yUk@O<b|WkfXxr
z{<PvWz5J7InKzVXuHECN!=E_KX!hA{EFYT-nUc8H?>o4@ymp84skdhY{yx7KW)R>k
zZ=F&Zr{yZzQ@8lRyo|0k-<%+ijGH^uKW+_UDBZ;?%Hi~4@1@qza>X~(ZYvZhZoaF^
zlfLAjw0KI$-TZjll_EYoss~Q$NGdLO?g}zF_fazRsmJEJsqwCIHzpO&Jon(fjQhvE
zADCyjbcw7f*rBldzYPP+;>)L(^3K-b@G&+C-W-0>ID4a|)MU=Zo8+v5)r^>U?<7vI
z67JrwXq+C%yCQ1U9SM$08C{V(#rGZPDSKh1P%N@>_rZ-`eJn9c92ahpo5S(;MW36a
z!zIlw_h_!K3ap$F6JxFSEWaW6de*|Q#r%R?ilQfV)m)@HqHH2>>m7Z}p&l|Tp4&O`
z_9o+?Nz6$SEGB$ke{9+4+&Pb-YheY?tcV%j%Gw(&^a3w#$cgUQqO6&dY>@42%dy)m
zYqch)at~w8&P9(6TqgCbs_fY((6QG;xKly=Wu)>=q3-X343d}4w`X!3pS0bnS-x(w
zZe*uq<UzZ)hV$M&UOZi3`6ucBeP5i-SWWCploYlbO{rgK;XAX#?`P-I*V_Xx%3XiC
zV?yw;z*qYp{j^W;nY(+}>XSbWZ)+M&_vri@Jf)4%(2Zqfs^(^<t~LL6?VV?Qd_ngC
z4izhYW9=EmM|y7XFdFDwl$gH!)@EJ<-Q`_tgQjfycGAJ(A+PeM?z~+)CRK3B^;kHb
zJY2?dsBWdi>l>Z6F(U7NY*~=d{J!C6l|#o1r>>oWGY>yjpVR5|)k=WdQgPZrm1yRP
zM}uvS@17XHd!ka*oRi6u6j=^^y5TWLq{F>=it{JWhqDgHt>NVCv}ky_u_M{xf~Q31
zoUR+$E_V`bZ|v^8zIel`DPF5X)@<F$bC!Ff>zn_!%*i?jB{r;N(MqotnzzVhrz3}$
zwPRWGs;A7|70t?x32S?HOy<6$Vsof_!UU<6ip@(t*tPQPUvDiE$msBlLDt#OL3OjE
zXXTRb&K*B~cV=YGt=h9V^@z`I4$h=Pw<*Q;rIR`@zZ7}t!1eIeuBAT=3x!1GY`mIV
zELc;Td3GCYsu5+k*~{tUUX)_Icos)~%JJKs#!R!<9M_nq!NIv9pvf%DaL*T)YX?u)
zvIqxV=}>YK{(GtA_y*Q_bGTYecGOpLwc1E@Nba96bjWA1%;v@FDr;D!a#&k6JP)nr
ze|>xFiW&ZlE6&TF)Yks8TY$^VaL#!{nWZO~E&8K`OrQ95U-*A+`HF*5EJpcRDz7AF
z$o>(3zlEdBi*u38g20bwqf<_7oOz0Uv4Ox+Ro=~Jv%0)o9&Ni@eY`VbDd**$Q`(F7
zec))}SUW9X-?A#h=>Z3uJGL*seLV1pnB^gdjFYPtEEH@JSZL$au~M?xHcR`d$By^f
z5+9gVnEsqpZ{4xO>C{C@i-aj0iECI-iHIJ!ZJ+AGuvcK(qk}zx39YKa3jV@pcSUjd
zCkJ${xSZfJm+jNZOFeFvMAQ$a%(ay9=ilpjG26u<#fq<6XTJ6Eh1%{5t3%erT-@NG
zc0Qn^zksXUSiJYd@y;2GJ^vnGsj+CsNA92%tyMcNum7=Zs>2xuoh45#`o0?c@9U`P
zR7_FtkmT?2;R?BX^2>>Hx(!>uB~4V{v+my%{S(Y`j92xOgPreN1w6{Ike1o+eCx!z
z#Ru2y2%9^F(^2%wOaVQ{NzR$NSC+EKUUL+=pL`&%X6K5My=7Cq%D<lFG3MdEz2`<E
zPj#qE4e!cR!51IYaGo&`QT@rY$-z7|%bfR2N6E$GD<_<O{@8u{N<(k2t@9G6s@y%Z
z)kSF52G)Zv7Bf9g-IMa)#2DgyQp7xW!3mZ#Q5EM>bb`}F<n*F1cU50oA93NtmdNED
z!G0S%FN(NbRynizmT-Hr?<Cf!&0W52w*$I2oHdI!UUpZL-6b%9wR`dJ18*1^40Aho
zN8Z@&`QK`Jht)%oEgROVy(sCbHoRfdD5d+udxP|q@UPdV{h0ouV25WmYgtj}QWYn&
zh8gB_B`&3$=W-PI&tjIM>zumO^PK_TH{&J6vw~+Y<@B$<bE4$%*Ds;wQ#g8g_(IO!
z`6|L;wfE-Sm>Bld+iqOzT2$68jNy2j(0Y9e$1)KW-lb=+RG$)(JtFwkOYp7lCYLMa
zPgGAn>d}?3oBEMUD%NoNjUzoB+ht-o>X=<a)>!iJ%(<As+dgB}Dkp(oe*`8nY9G2J
zq;05r^W^1Io>LWP%dGhkak6vEj0K#or-Ql`WPMi#CvIu|rg7QW_wwZ832Qs<bbh#V
zt~S7$^?_!0@ZuG#KkxV-WM}J}r4!P$@Y<plnes^nogq`qtBqgE1aXEKyM-FK+q$1+
zbYJ=RFkg#mDpP}$>}D^M2=m1078f^9-Rf||Dlt%1Xz3)5BxU9>kI+t!fSi`-7USg~
zTqM}@?skfJbx*k(yR<LcSMucT^{2S`8M7XQ<eKEa4SUmYhvC>GsWX0ao}9FvJLAEP
zhph)<{x7|p9qWIN^{%Yja<;U`m*;M=(79C<>UCpk$np=iWxcVtTmmbuKA!z*@6Fd|
z?>?QD>nx{naqpZ%&J}m|sx|RnNDvm??NXl2*}8VIzl@2v%zTM^lf`W`=KSnX;az)P
zs5fzAukHka^LsBZEx5XJg&Qm1O*hZ~e2Z)kbUlvW^y$p;GY`+_-ibZO75nX}KG);v
zCnCEaM24Cr%uh`{_BF!s=f)c)%a8uyx*rpD_Rv}OoLhIkgl{ZJT)J0v`3jvX&29T9
zF#Vm<_4ekKZBMjpf8Pq+X(f@TRX(jV(Drs^Tu|$c&X8j#=gZ#RvVqTX&9Ns+hu+TK
zvgm?G@0`9lwinp0U9;Si^yNjW+T|Mw@1AbBk+ju<|MP{I=nHp(*Iu5*8~a8eHgjrp
zW1z32YWTa7yUSR*t<792&t7+Fd6_#o^HX(a;pDx!iM)cdH)vS#oXbclGJM!NA!WPF
z+VXR|Ha>N~;&3##&+607Rc1=uTXyW8zi_5b_p*I&x^G?l|0?4bCzD!dS<&rQ<pb<#
zC%y#VKiRmW<)O)fA4jjTbpM;$8FTI7oSF+-=Q!TYiJVzsGEe4N^0w6(A*ual9#>QD
z{5ctLYU32MRIl@|V=Dtgc<ybe*s_;TZgcK4i<B<j!gs<-u^gHf(<>IPjf_h7n>26n
zz6p}c(w82J6593f%LGn)rTf!3cGpdN+!L@a?nvm~siBM;*e&jc70%`86noS4EGc(y
zWSr~Mc(HDochQV9W;9)Uq5JIWxwBzXcV5?u&7Cij*}`$Xgz3tavlph+aoYJsS+6r#
z^(A@roXk0W+qNvU<$C9JB4N#alhZeoCj~OxyZ<*++t6+iUskt8+~vAXbFKeJS58`N
z?HFM@Q>{OFDuXA(%07nQwyVlk%X+DZNrkB;U+i&rI=ekM^@7bQIqCa?;)T~l4z%`6
znO}NlEkj~P?#D#FV{=WP?fNJ3zxS<leyZE-o$6EP<di&nqw#)u!~5)Q&y?05Fy}5Z
zOb+yV>(JjcYuU+=tv`8oAG~#O#V+o$b2m4+hXo&fIp?BOnA)+%S#J#e8Z|zfJmg^%
z?%FXq)GWULg%iJfvBxD_i3O_HU4EYSJRS2$ASceRyUB&4SA@gyc`BRgm)CZO*cQ$2
zV*ApmP-2;z*AwU8>z2GDE?{1c^Dee`?t=br6mm8N#a@f;v{s){lKYZBHhewL#gO%n
z?ehMAbdlMh;bphKU>1K_m*xxx#_4gJu1@x^<y1T?+Ic}`ex#t!+YSzeH7gI!dwa4&
zOPqJ|<DL}7?V|N6e|@KZnBsOIu0K0|Q(pZUcGnoO@X$MjJ7mqiH@jR|?(k)2+Zng(
zH{CVQ#=ZM|{^u9gxI4S$RJ<j#OLNy0xTMc@-dwmxhi}==`v1p#Z}q(X$<2A=&B__l
zsz2VppULZATk0nwS{bL-e0Kexs{a3R9ZQsojjfcD7SB@6RMJ%1KIMI%>D8Xe<&~44
zuiyH%Pk9|{b-}x74_<U$`)O%gva9U7`+eb^*Mu*=x&1=##q8ztpXeo<nB|EdfA^=7
zkBj$^<u&2ucT{Y`{^x(0#wp6X_zPRg(o`Ov;?i5EPc3rt_FD5*OI`WhMBUhW&gKOS
zEgjYVdq1&kf5Y3AHQiD_th3-p$A<?iKcvS8b$_T><d)d7!N=#w=DjQYF8tux`H)+e
zyYr;J(BzaF-Q+n73?f&a$UawpduLr(XI#v}tsNnvRVU7LB*g`^`E^R&uWYWlTGgL?
z!(ZLC>Y3}$@N$zQs_LII&92myckOOJK5atT83E4Zi3}Rz=g!~r&o+F!Qutpv^NHtQ
zzfbezuAJsx$@1zg=h{Qpg<lnMewfd}c(uv&k9QY0-|X#`Pv>>XUh}BlVX@%<rN`Si
zem>9b_4sw^Jcr};7i>%&RZMKxHvf0Mkg&FF+bh+W83hjxL<y+6wP-Amb?I5F$8{qk
z&|!9?oOM*rVXjA>9m>%glbA0qb)CVKQn$u)mD;&EmhKOqL|#@u;>*<VNyICpA+W<C
z|5eVbFPc|orRZ?ATChEu=9{u5Pf)?NNkr3ANM)kJfj17C^1GLnS~(uLy~5YL=E;RA
z-0a(86pCgr6|QgCSARX~2~z~a?OB{GI@@kUFf{ac3vxJoh*;}-#`H9wkJ16-j1Etw
z0JgadCl4I&6?730c)0N~$BOJl%VK_)Fr5za+a!6lOT@cNBD-%&_8*H+Pfi&d$y@AM
z=Ge@<n0rI$o@wrHgtkBT(R{F^{QYTNvF$rT{`~j(?A)Fw>u-^j)E&E(YpMN)zYYsN
zZQS*~zDBz)%Hv`v6T^h1k$$lomrfI!@A*J7X2#LyR}%k4Ry-_BsZ@0kNn7|-%4KaL
zhiHrZ*>8d^DHBy0HFG97v1zY8VKKp0`Buf^)=Qz=A3ND{sRWcV@g9$LI#l>%j-R%x
zX^;o=%p;e|)m&YLi`?I~EDm=6xFvO(*Io9&>R+s>7ehTJz6_aZv9$7;#Ftc4(TK#_
zZI9Y!{WMJs5&rRXdVR^;)akJ$Ymc(dJFeMrBSFAb?CK)<FtwQs4hGk*w6C@82+mk}
zvNbA~X+c<0@T8z)Gg8^q9FJ*4|1FE1WHB}U;cBgiE)&;meD;4yir@K{S`2GFzH!OA
zSow+PXm62<e&eF~RqCGmnYakKiu<)!Zihr%3%TlkBg^`k!|kZu@xHA|voG(kIO~2)
z>_yX7*N9}vjEv@4Q?Du{WqmxUz9=xXBXOa>>M^k#3%zGAwrg}vlM&KVQe><(*ru3l
zz4gk8*Dhk5A<7fvZQU<S6iECt(e}`_DIfb3;w?5Z@TBsj=sI|be>%$T7!t67NArk)
zBY$X2V6pE_Ep1N$Ps<4vJN8|9b1|Uv=9NnphDUS#Z?2m@f9A8TKQDGC#6=qHJht~_
zNQ98fLZx)e+6=L#xq>p&tA)9=4&2UHKQ({fpNCi1FB8*!wQ^Dayd5`^s;B?wa5=yn
z`D<c_@6;1gD@;#*(<ovS%8L}r&)cL`rBt}L@}TthH<ptQe%i6v<yv5f&Zb*|JfT}$
z`nO*^FEDY^Pyg6k5gPMe6s@){+EM&YWEX>*pxEwob;myqCpYij^C|J<&xu08JRLXY
zneM$iHNN`o>7NTL{?7}3bf3TE(GP)%^KV5Y^X@n)cHm@$&Z$TFa{G@<oRV46Z*(~)
z=g6LK_p^_bm1?h8b0kACF~}=JU06%EUEP23efJ$#Oc+DDv*c%T)h%1*xaz@@n1IO1
zvtC_RyKDL0QOfIn*QrlaMVpqk>gae2i4;r|^>g6Lapv2{q}s4IfI%t$ww$Oy>WsIw
zEggRY|DRnkPic|LqbH91T=$nqeBROMvC5@i^J0d@S3%F0Yc+Rj3PeuZ&U)r8kCu?;
zdCAzP^KOLp^q%+`x-RUx@!XCzO3POBZK_zdGjgh+`njr8=iM$@WuEkFUR<%D$fiQZ
zwQ6-%-yDUz_mWq5sClSA^%4)ivdQMW$hv}KoD9h-0()O_CiutRyuP1nnnX>b_oREx
z3T+!Ij%-x%sa%`ET<}Ys;g=@6vdZds-+*JMr@Uk-kv)9jsLm&&nYB6|r$i<>x0$F|
zN}W!3C_dF=bxCEyAHxh!Z+}0DjETz?jntoB;Z&LV%wv`6(Wan{)0DQSP2}=h>pShV
z*T*^KPX1c#Q|1RY@cv(|E*{3`mRBQrBJsmCi32xS7d9C7FjyFF-}m&a;lz!+E|YGo
zJU)lRwEN|i&K);qO{`jG@}gTQ<%gT=XGfQi+Q&(5g-Sd1l^h+SFC1Fs;W#n=(5B_$
zLJth5T3X4z^eDY`QrR@y$m6<hsO7d@oWT)=e&s;|?)$i>ha7SBoL|N3z&)v{;Yg8Y
z<67>(_D5PF=T6BtEfRH~_i{x{P>a)3Ms+u7$+YOjhl1xmD0R`(KGFZKU7-7qeuEUZ
zR^_^#?)kU4yj-;T-d=hvRH*iRPVmzhtM`jltAa`u!e586+h$L_SocyheB*PCV+Rhc
z?`fO1Qc=cB^z({iI$A~z+)WHq0{$Nkih5yCvawsR{KqLL*6W(puakUdyx5`g&UYvC
z&D)E(Qg#SbPRXA#@rrwC(cLM0E;}!*<;fFD2-chs*q>RlD!A^cWt8OE42dRXzn+!u
z%dbp~Sn7ODR{7Dr)Qx<q*J9oz)i2(8&Gv%o^inSy{VOL}4y$k3dpqm8-i}VgC8sOj
z2x;@I-Ktk@`gCeU)Y6?wFCXqHnwrDO@VepEEw0!_IjP2z<KuNjk|$q0k$gr)?Rd+S
zO_v;lj~AKb%bOmW_-V`GUS>7Viln1m(_XY}GAr!&U-eZfhq3iwT-Nid4KbnrBYtXz
z-iw{0{zh$WhLT_W%bvD5hx|KO&dh(w+&xdH=&;9sDUpTpFBg7&(<7{8R-&N%>`<Y-
zaP-tgU-Bn&m8A16o8`=XYf0xmjVFB5CO0Hrcd^*%dq}8Y!4ut|=GLl)D|DA=FXA)Z
zsC3ZJJ-~NkmQ`!y#J}&|6U@&!HEnhcU**N7Xz_?`v1oIQqI1Ne1A4kE*5*V69nerr
zI<@oduH|R>Oj&$=!l&8fSXP^!+gfOJCa2%$w#CiYJ(kmw|NhiSHJY8~!nN8sXH{Ua
zmzG~;%HO%FzrW9(?8>KPT>U^W_1auJ#)oX{o91)XP1_uIYZJpImJfMzr`3KbeAs?M
zsBxo<;4-m?i;4y3GTdNOXk&c3HuzPKu#J5ySIe#9j*U{H8h+FNpB1a}&61kx^88VX
z;nZ~+XXkqdzBD?vINqi(qCjeK?eQ#Sr)zgE?&XYs^rzy)Ca+x=C100hpZc!+AvFK8
zy;IggKlc3|p<B&Y{(SZBN_7K6+M72^ROO=ISI@QEx=Cs4uY|`jMio&XyFN(>&*Tx;
zVZ4<5T14#B)eydxwW|NZ9^X-ZTY6Gsx=g0wmYrUkrtQ5kbK3en&P_`;pN-zz-=gYy
z$@i^R|D8)phCw=$E1f+GHpzLLpUerp_9b<7;!N+kZ~gbrF1po!qxek9g+pye%M7}n
zw>+B{v)O(sgV;T(ICVDBkQx?K)))KJtINGZ;<A`7&M2GyTi0GNbxL%QW4Or0JO6F>
z>o8Bhq*C<hPc^g5!rk|4cldc-o}G5xH%#^2iRyX(=XdL`%$03eDaEK3dh_DB8wY3Y
zshk`#bKcsSm*4143Kp2>7m(26I`1xH$LAYU(r;Wl^0!aJvwxE)Te#``|8o~4p9%dK
zbMAM+*|@^ng$=V~4k#4;3H^70r_a=V;*b7<HMb|lTxL2Hy|!SoyG~@v-P=Vry>EB2
zE5~{SzUiA4(K=N_Ci7#QXqQ-mg^c6`r*MZmUNhM~243-c=<*|0rfB0#uA`EE9dlg&
z1PIECB+j|SxAS<S#KfPj(_7cvTrr`|PiB7MhldL!``GtIe#%++FR*8VExT0Vxl2c9
z)k~bcTIgc>Kau6y!xjd&cJ~PP>vPX-)Ckkn@MhAw%HK7w;pg=7FK)LNK49H8X?wto
z4&O^V6NGk^uzu=jJso=GnM&iA4cwn=`yX6o**9Z>R@1C4H?OGK^z`Ti_r9H9bRi+`
z-~;Y&r{Y8Jm3)zS9?7ygaPo5v&lg*}JaR6Zt&MYUVV~OaXtFJ<Wo|%-PR!Lcoq-~E
zI(iQ7ca;ncP@JtI^E+U|jNFG}7Z%O5Je9QMeEO3ZdAau4j3IM;8TV~a4Eb`CLurch
zKc!PoTMSNx6-|72(B%40U$!MGQhN82d<?_?`gSd3<#Au?mdxPC<a4KO>v_d_vHm;7
z1-{KUw@UIgNs>`)ck1wC_-_<jmeS6+i!ZoSxk|Ekb?i08NY5z88!9X3T#4!SKdGd4
zmGhHF!>^h5->W$A-u2wavpC#Mjw_TeBKP278_(Vr>E5MJx1D5-GEz2hY&D3yy&)vI
zgkjpw*xP*X9=J+y1}M(GvN6!CGT6e+Tf=Im-;zfa0XGk7w%fdlJCxbm8si#Pam{Yo
zDYc9PS-SB*&$M;!lwo3UkbKp9RFOeIMextw`KJ|C+IO})o>X*_<=Q85Jvc@Fn+ezK
z%q7`1)Bn`A<Z8Mz{SfxDbt_qz<}2_#_(Mz{PYa8L2h$dnf*t%yZLz-Ru6O7xn%Eh)
ztyH;`Dbh7$?u#=^ueGMXWLbKp@lnnHnTsuxf-MiabUOsSt6kc9#QlSZ>b-?CKAZ66
zo$_aBXyy2HGW6`ic{e?rY@a+lsJ?8%*<Be&e{X3rwpv_$MZBl>_L+ctuU7b&1}XDd
zT|OI=ESb9avZ~SxQP~Sq=9zKwRxM%i-qFJJ)qS6?nxDiRQLAMcXJ5Q^_3gEL7P5r%
z;nU>4tfLxtgzx=LZhs*&@!jR#8S}F==JPhr@zZ(YQp@tiG^Bua;#|*=hHI=pWR@>+
zo%M6${O>Pg9&s`h%v<R3NX8?oMYMHEvO(Lj-eV0*y8WI`ba<e`xKv#J=aZrfD_^`_
zIrrI<#8$oyJlrkM1YZeo7L^DW9l5>6R>2~YGs!Ad_5a7HxW7_MKk<Dw=oeWN8U6OT
zhGvHYZ-#cEuYr_Cgwl$zEhn0iSpIH#>b@f)c+Jy)dl$645;Z-RTExM9_RfR9cb>Ss
z2)dT&68}wH^Ic}o2B*m?0k0M3B-U~FNC{mrxU)?owa4z6t<uq}J~vojDmR$)9M<JH
zGWCMP1F^i^V{-)jGQ-#eSZ<}O32J1%D!wY_(zCj`CdPGFE5n344h+}pIIc|o#wS-a
zfANfE4+T;RA1yji*TVEdE&tIXb-^{i?>;(`IqOEwLd8RUM>NEa-SRMdm09GXbh$Ps
zAZe~kP7gy-?8yM1y0v}+tB$C<xl~QNTQo%}(MmvT75mD4i{^hkepfA5`+Sl1E`~WG
z9{is?&M%elNJ&VId#SW{!h;oA@_h}fRAk%YSy&hpf3mQOFfcLbFfcGMG%z_ZGI0E7
zVC9gp*l^}R81r2LnHOCP4>#tDO?sfTz(>+Zlr<^l#Kz>~fpXT9xHJz;Ylz~G%#1pk
zm*CB~CV{WVbAsB**%rlIGS??QKi6ni$|dXd<=L)&6-TZREr*vD%l*W=UTb72Twb8i
z|H-E6>MDkLj+{xXU0;^pT<;uSdc~<I<>Caxn><H4CM4gT!u^zMinQ=z-<B%pQ+t1Z
zOH1f){JJSA`Es(R(VQI~dOik49?i_wu1jJJ7CW3_(Jxza!|;(?lY;w8U)u|Bqy3o7
zI2&{qh#m^l2;`D6?wS_Ww?CO{dStPh&-^tNeO`WIrI(+xtS(xkHCN;E>$t=FY*z*U
zj(i$t+5LaJoSaqa8|R7A;^#hZXZZdu|CjKKDh26f$Bw_ge}4adccbWnjjaqZbKGvl
z<jZ<oN@Fs)60f<SbXj0_%7mg@b;%-bUAo_1lnMFF>N&`#t<UIwr}JOovse%9WiOPv
zGa9E()>yMpT*UwCUzNvwhLcnmM{qw>ZwPbR`XauQyHeswx6i&(ai`v=stzszMMq9e
zW-@h~5gl<&%|TPd&FrQFyVeeQH$kr*0?nC4OC~t*gt<(dDYw(it5)fu$pVLJJr!y9
z=q-*ne3CnyPIm_KZQ;@m|Jx?#xY{7}s>W)M@R!At?}fznJ$?8^!*iOFde$_rwMi=O
z!M^EnljqpJT2(h^+5aUXi`N`Ue;PSK$dxOx{#l87edi{vHx&%CBH8A$6gslboG)l{
z)Zx6+iHVLJTVLMXTG)2Q%SDWH!^MsFlH889{^pEK_t<?+*}<A+>!!|*4+p0vh2Jlo
z@@f@x?yUFwyiMERZ!nn0D%2Ohu>AdAf74Rs5H90A0f%gx!u8#s+1<&Ed|#T9U00E+
zDbk+q#W^|tUX|yCM43p9%L%I74{jxGkLoN6?BopH?9f~%HgkrHqPMiW496E^#{{m_
zW4>&W-fTN8CSG!@H4nB?4BXqta(n*nSuxzclGkR<xo%|2cF$s>+pQ@p&nF(2DrMU6
zw(RyFCLhr|`OQ;a-YrR<_5F4Z_x~^3?&$4%b!T~@uxZSOXSs6>9cO8rTxY9SEfXEs
z!JKDVU%?fU(7-Y&Ol|Y5@18Ta@=m=sv6*)s-^A$x%Q`Q<=7?HR^kCPuOAnpiWgYa}
z^XKD>-^Dh*U#$;5>T#O;aj}U1<4-Fu&iM8zQ#|JC4Z-g#lE2#U@z{Jgw%D}b`(lOl
zHlKVA_2+)vc<h#b*17Wdy6^s><*z1W_}U1Ri*dKud)oigwmfbkDQKN>urF$Zva{UE
zI~7G~5|wY4ds#5{)T-T(n$Egf)MRD1Z<3J^$J-pA>8k_k?>!OB*7aa8KHV8zcXeXi
z%abWGFQt|S1)E9vy$YK0>Gshp%MZ&PI@@j&Gr#h`R+!<b75?$h56*b7X#0_+8A+Wj
zyE#P)4{7)&=BQbIx@E<o!BBRgS?()?vh6R!V^u#E%l$QYTx2uJxah5W<MeK2KZX#U
z6h^ON*_Tc3i&Psj&U!i(GV#vdGevi|&XSC?zs_XdIkD=@)0Jj#J_W^^Eo9yF^g~WE
zcl^P9;&vUjt&<-*$NibrWD}UeQTtTbSu0$VD{#f8sq0HpZ>Mmcj$9=)_1NxI?bGHf
zV<+*~p3YV^D3(0)NtA2)rZZ2qrk*?#do5_@)-_%_@tO>w+s@9qcyQVMB9oQXjG0|l
zii?hAY~p*!GeK_40(ae4f!#*mn0o(Kq}xBTJYRWev-sZ+?)C+0|INcztnrRalC<Y^
zT6b*6v5B2dGBOWmOm;fP;?qBoC+CgFl!cWl8B7N?a&)Y8y274ivPqsyjF7PWwLrjM
zZ%^md0(SxaUPYlb0<KHbw1QRsZE$;faHZeuPe)#KItkm$vIewEg)EZ#y4<7oQQ%aq
zE6YA@S!rAP%5Ui`vt?1GL1C>~f!kMwtWy(R<@9tFLz~sLO<9LJgOYvsFS3}Tz;`Zr
z>!PD#;f@CkCVgC&YLhH={zSO(uhR7yA=7sqW1jTui)&v8Ys^NbFYZjGM_T&tE@p6<
z?s3bd#q0W`=ef&{Se*-c?Y&^FM~lH^lQhdq_9+^RJfawPHs44SiQ;%_=<9j==6|X6
z0h`ZG$os3t7XEA7_EkC}1+gnRABcV5k>s$hX!7YhPp!hIp6z{Cyqr7sRaE${8?||+
z-MjC;suJIMmsP)b_ilj~VVjk=Z|T}M<NH2OP1fbVq=h;;W}GY8Ry_CXne}zAInV!n
z_j&(snKcY7J_(IqnpD=_^-qwH6#wXwX~8^K!I@uavg5^+X3m(Kt){Py^ggWFSk#i!
zHSO4mzK|6YE_n&2vMSA<v#>=ZP~xy~=9JK7pGwvwzFpN<eCF{&HSv`*6Y|vRwAvJ;
zBN^PT9h$g$CZ~#c<(fqqMw5?6r6>mPJbCo!r^%ano~lV(o(c^6G;MXw6Xjyb6;a2E
zrtChGX2SgQ|JjVTO%4sG)^9#{Gr!DlmGH){YjYO0g_Yb*V!WAn-DsKMhR;`C$Dhr3
zabRAc!snM9wp~J*eJ-1pPJFDODY5F-k*Zx9C0Fh=o_cWnqD8jC>8Or|4?8A^uFor7
z;VkPY)%2(}VhPJhElJlC4os8NwuSz>=_0L{JA0{%;-0ftoeDc=$E2G7-f`}l%a-N3
zZ?m+2-?}7rVCz=hw>}$OPu~BwZQF6)yxi*Dx3Alo@3?sOUFq)VyC3|_cRaqESGYg=
z-q&N__in!X#?rrfcTVLt(|+%#n~S!`b1V^DdTnD@5!Z!%Un<n=Uha&qyj46;Y`cZy
zz03zp`cDqXFjm%Ixth69r{ti{|Fiv$|DSIX+B_%uXs2ZQqbDXo9>@CTCzralHU%Dk
z#xhY~=xo58Yn#3<oxq#7v;TjAlVG>e1kp7XWz&vx-&Hi;dE)7@UE8<Y<rkzMe4X(9
z+jZrItsAoyxuO)TZ(W`DZS(fpw|VKi?>zP^-*Nlz+lu+W?|z;4eb@Ke_a)}L_ikU;
zd9d78xV64$!rq`$9*j}zSaQ_k1m9lY_igH@OqmVOSOZ@E=Q{A2>Ej=}_>h+uqN^09
zzw5BqZ#(3_d!IpRNzOUvL&rm^T^zTGbI8v>+G;X&Ta?JNFb}=g%TgZhy1qb>nIqnQ
zg6|fm11yTIB5TgR&7b`H*R}Ke9-jT1yUG0KZ>A2G|C2T={CnR%|IdqlmtWV{|NFN2
z|KE50_J5ws|JOM2|KFGM^=cRP|Nobs`2W@O`oHZdEC<DM4@*D1T(IUr!PljJUqASX
zJ&zaK5kFr%L6)(W*|6$YW25UrJ+TfYA0g9%AZhVs4m>KwRU0$oHYHASu`EB>#AhPl
zs;r>!B(Upgwr^7N%%@`O8U-~~L^MrH)Dl&Xx0SiQD083D;-SIdcA_Qyd5b4UYtW4r
z*BPv?Ct7?mS|cM`0~1=~I9TIuw7N>LCaq|V)nJHZXp7}wJwB~1Wky?MLtD;{wp@$$
zJdO5JiS`1G_Nt8b9FDfE9qoxbS|6$lxQc}sZIwCg7|yEVHs$~FjH;>rjinjg-!kf0
zGno!HN_k|?3#$@UaaDd;)OJPhOn4XbK~9EA*;d!xnHIXQSY}xhBC<|RMCn-Mbj2uD
z2dTZ+vd=cvJT8jzF%mkUk@IF!m7X#yQ(?~U@|;)4dr~Uu&Pnt>{N6M1LtDy?)~gl0
z*Jt$J@@Nyh(0h-gFHnHd$D;50kKP9!eJ@w^7I*aBI?+-r!J1<saGitIM}qO=jlQ2R
zdW$3aZfXc5ZOd7>MTUR5U&5D$zoGs=miV{36@D!7|NcZ!?s%d6b-~#)8kbcF%iQp=
zFt#>$F81$;hst4bRpUgJjjHn{n%6WW%`Ox-J8s!}v3R?=$VLvy34&46{y#1fR1A`v
zC?$PRWJyEG+3>*4hEt?339c>3IoZ(~?jhi{Q@~$xYHmksVMS}0Cc}w{mU@l;m`;|M
zom1l^rv)lZOR=1m7C9}Wa#~jBG^PX7;&x80_vp`^&U!M{rf<7OlUikOTIJO1(`!yn
zpJ5?C??k6eU}oD*!RC{d9Xn^VS_*VZ&fuOnLn@+ZQgFn@>GGQ=O_c2HGFYi_iph#;
zLiKtLfkh|P7as0XxmdkdJhC&Tn|q<amT6X-jtTD3U@(d-vAC)7K~!i{VvqVkF18;%
zVK;i8ddxW=F~|F7&nt_*ryhMwj(ry_1uk;V4G>`TaG0Ch*?(>4+^dmuuUF2!yK?Tg
z{~as>ALc&gWPQRp@7m32PjAliU|<bkV0@@4@aTE%xil|s^Z4gm{DjRLg*YaD4Qc#r
zA|SnUBFn~!;wJ^!v_v>}RdR1!z$H~B;3X&baQ5Sm%`=Oe4!nrknwZF5sCw&?VEe)*
zgN>33uLN~}n#642ocA(wk9pSqwul*yCB7RMavz**Y1qw@8YI0)+Brfx=))8%#U)=(
zw0IZ@xMwXnIb%uk&Lw_Q({d%322?Ex=~@zYO3+hkDR<*Mro{eeuGZA>6yK>Gg{B#0
zPc!OfF6-}DHsggL@2Z&vr&Kzf7X2!86fx^kkXSx-V^&JgtQ9L2cC&Y{l<c0&s=Kyr
zk&K|aMw;%G|3Oh(b_z&56<ahzNqJ$^Vh1*k#GV75Q@twZutoH-Je<R=xN6ZX;ka3I
zKUOYT+13A4lVSC&Ri8K+nF8i<7_5GIa@D2G{+m1U_8yhyvT*o)(obNeKX<185v~RN
zorQvxjlW$cI)7Xyv{G0}y{Ib7@>LSQdWJE#qxf^70JUjGf^5Q0JJ&wwYF@Kwk-^Tc
z1)M9Fxe0C0T(K(5X!0ugT!Z!IQi5!%Q@$1|dZq`>p3r60s3ZMo_F91@<{wvj&)lFg
zacStPRY6e_OTGHPX|&~;rx-u0*n5zRO~mcflx6iVH`Zy&N!?h`^m2yc#2Ni7XSCm3
zrj@-(XZNPApPTe%3u|c08~^X_RFR&kF1hZyqj2_;l0OsIYAs#9(Oqy+<rYQ7S?7%u
ze5?g(omR+foR!H`BY06@X{uGn5d{uLwbF~a5(ingcIs`_klQ5KVt#S<>qRQJrm<ZA
zA-2eC33K4w*jKB#CoWaIxbpUo`tOxKKPA_^W@+R+IpOD)_`jx|f2Vdbe~ZtU(s|+4
zg7aPr#9r>Wti`~-ZNe1miL6{Zr%&D~cx$Jk;lk#t3pg5u>}M%$FH}9sCM+Pf=)bVI
zf@f33FY&eELHaw?49b^VGz-XGTU2yqOUk7ZS52Md=ym41#E(i#m>C+^E?Ru)llry=
z(q2VnPEzeMhEtg)wq3ow^hWin^oqU0*{vS`C6*~}vXS5}Wm_`6A+>VSkBx7#-3)(M
zN}Lo_mfmT_xY;6bvr6KA^XRqf!*?+?uHBoo?M2fR$z!|z%ob*N9kJDX*18wPEQ!^1
zud>cWuV4+5T6tJ-<1>M+-&R<+XP-@4ul!8r0hbU*Ab+zFo0sR_h!=wAPR`-F-FEgy
z>D6P>`%X2yc+&8rb%KQXj-Rdmr+ytiDdjJ;a)<cM9ipAWXKx8!k=pORN8q=K{JszS
zSOPO&o3z|Gr8lu`#=Zyp`c^M@J0mZ>($VRNywlFaz{BEeHVCTDXwqUg4w$ZbFln*b
zExj3mT;}PPMlZ9@8?j0s&{=;t@ZUibr;oezw@ax92NnC6Zu_6m!?BU`xTk>DnnN;*
zX;BKvWh*-hmu_s?HeKQNzM7v$*D#%GS-IKh^d_c-+qA8f{=YW5Sv5n``e@LZBP<g)
z_eSn_`grR7>`l%V5r-zuJX(FAQ#8xGT*0F{%PV`uHuHl%?P_a05Ad`I@Gz;}byASi
zVb~FNR+{n9qk~~`joYSQ<I`w76q%5F_1fx%Pb!#9;w^O62)vlUyE2}^s&h*B4!P$G
z1ZFJ|&Z?SMy<o!chW^#(6&E(LWpT5n@HfBQr<QnhT>zK!3;ulx7rsrn@b!Qq|KE%c
z+WY_XT$K8|KwLva*lPdpJB3A85{eursmWUEZLfOWD1LB~lT-FVWsT<eO}kh9j}UpU
z)!oXq^5v~rmMerGZ<@T>`dF~C`M=W@&#W_~yO+(obFy}4xZjy03bhwq8c(+_Y+Pzy
z)o+=(MCSD3KgT$}WeCP{Ej_@+vQ}EQNHwF^QO@b=0UJ3nuXW5k&T|BHB(Ck+E`27t
zKr!I7PUEWsD=LNAKW%B4ds%McLF?0k9V}`w6S^IP&sw+7_EC~v+~A;dtV%W2X!cXX
z*ClJ_$R3ewU&Fr4UvA@(>B^@R9dEJEl76$qPkEColLPlU6SuC;TMKpff1a4JFnz|q
z1a32{i*FkQwj~@=e#rOw3;$L}uC!eK8(nvDeXE!RjxIl=^2Eu=`jjA(>LQVrfPypP
zF-*os{yR#P{yn{Qqh$Za%eKmLw#OOI+&=#5l*-pt+RGkJp6+SkV5!JyBf_=mgoh*l
zyfas)2wvUnv)|<QreEBrRk=@FI`Xfckebfkp0I-RuL0Mm3Js&zot=^VZ#HzkNwAUo
zbV)Yz4$}j^_)q)-x%^8WaD8)lr1Fk`zb_Z(Gr=6+)6u;TjO{k>ICE|3buLa@0p}T(
zC$se)cNwYOyrZ}=VzS_kg+H%1dI?sa6|7!+X7NFfa>bg>Vv8jw-k8m}jY(h|^TQjH
ztn?o?Iw&h1*dcr;OImoVK*HHwJ36yBv2Q!d|MvnXujMy^qf?`=nk!zA>D?i@_j#S3
z{9gy|i46?=dFOb|6W06}nEzTJbJ7jJiRKY|3mjHIP~5tJGthub&WOun0c+G>{<jYv
z#p)zXf5yK(LAvxF|NaYnsSo(JUx?cJ;kLTquAe%mx14)kzi#LDyw^<2o-4kc@$Z4&
zk{d^VZS>%HbV;UpkK^B>Rm<O~FIre&)AYobO(*K|-r2KGt5@q^Vza7PFEwegU$jW`
zr^Sm8@QK}B5SF_s@XwxY4boc~?lRR)%z1lf>z=FgG}vbQz4OX-dsC3JS|LGt9&4gR
z^w$9XH4mn~6D(zuubQTJ)!OfI&XMQa4sh}&a))2yJ9swS<(|rvyC#)h%cuWzlL_8!
zv^=7+*L|s5^@7T`zdk;Zanx(`6}hbxRrg<3NW^im{-hxGRJnDI(lHAj7x4=3HC(WG
z?TkR3=idsrw<WNzTks`s!x!&-?g0Ob`tO!W&hY<rals1R%?f*WupHq^R^j3h;LMC$
zBNombRBr!CB6^|{+e87z-eBL^@Aj?!&>r<5Jc@yJ@qwev58o|}V|!~LzcP-iM33QH
z0M~A%M{}NENc+;bMfZV+%!8o4mCJg+I&C~sQ(tBIbFE6CG4D1<gBijFBA=x$&SZ?9
zwIor}X`{NiuN3p9<F@6$)+n8DVcfDZG-%!GOXk0t%R|&xatK9V;L6`CBerip(?jm9
z8{Tbu@Fg#R%{Ts4*@DM*{EsCq+!9P2_|i{J<X-!{?!o*2-XDrY-AoOWbMjlWHgb7P
zVD0vMecvqe%>q^r1*O`lF9Hj~-b`Tia1fsM&dcM%`!so0hK;A@Gc+-Ii0W7rEMTbq
z#<HU*;)2+ru(d3IHh*7#&#8rzwSeVjpo;rMW&3$5D+HdNnxJW`^hRAl?PQ-x^j7W2
z3BHa-vYx$4#-|h<gV_1YQl<uH1}*iSYu6ez>5Q0O$WD<xvnG2T*JRoy^hYZp``U`s
zrDa<@0vl#Gtz+3z`6J9BYGNwO6b0=Dmvi&d{%YMdnX=+?*tS=SYfQ^U*4FLiI9BuK
z#&m{wzPC3wrXHUv9iD$}mS*s=o|We7z4qUHvntG`X^mU2kgD6{X+Dwvm2UP4WnO)F
zSV{4;4#&-SpGQouE=C>KvfO>-!1c_XvYOkcm7aBXbP#gA{5q7WYHhcY!|B&I7R~zp
zTAj;r$&uBYHcR*2i{Bgg{x|2<kkxV1G`8H}>^QQYgXzy%xo)*tIYy74X|I@jl1*#r
z+)k+_-HOq-GWm9Fsxn@;=Kj1#0t*GWr8X)b_fR=u*!t?@QTh0)%^GekeZO9>TvFwg
zr9QDlN5h?QhSMoc1??-E>I%;%c{c=2QArlo`?Si?t7+Sdjy1X#3nrafcc@87(Qj7M
zY5g5<j8^jmoyg%@vDr(=OGWFh%Zl|)FTB=RUT1mrcKd{yl}D$AnQT7HYksQWFpv2z
zjm7`DjYUkEx0**4K0Xm;a*~Pb-zEp9E)BPvc_Lx|v?r=mRYa$X26~;{AQ!l$!-?BF
zT9nJycTq<YkH5R1b8ui%lIvxkFSGe=qZT+Yb$Txrc)Y*q&gn<{0~a(f-DtTc$k4gu
z+wCpkj4wXAi@bd$+#JoPduZkMX?(X{ELJ_8dg4Hs*UA-QVNna+y{g~4b}U;Km2`4m
zXP;7$K@*d6kY-RtPOIBernntC3jIgISLrU((pa;9SI8@0MiVy;o&Loo+RF_l&)J;9
zctIsY-!k0J>s-gwqMw>gyt7t_%@O#tWgb8GsyAZw#@@acoO}(M70kUfnKSh!`R(fM
zYO?N|>z|#d{+~<b$c4!CFRxan%HD93Y@b?^Evee=lVNPl6+d|wKa)nwC;x`RjGq}Y
z+j}RkpJ!+yyY%&kV>dFg>tAsCbC><Oyk7hZM-+Qh*sny^PL2n^H-u{(3!EbwW!&i`
zay0wok+<7L<6js^{dK*&;iQ&~M^;(X!&~!z=3LX?_%QC<+9mwnB`X)#u?0=s)E^_W
zZ<AQ>_T`)6gT8M!2%nR;cnYt@G20W%B-A?G_jx@sd?sJo5|OaPO>f1Nl?Gx8`g|Ll
zG*+uy|Jt<RE9a^1sugB4guH*PUK8hK^dL$|TI8DGf-Rl~hRH_{vON<I3V7okT_NQ(
z<CAc$=aI^71<nh&wmP=lc=TU<-G)}p8%(nnOSY*<J0{v5THIN;;DY=ciKDqus)r3V
zR@>D?yW1{VwoT`SyU4L9ckbuLE>#(foKXVef=3Hnp0&7(@F)d}C`{l~+_gbr>xpw3
zaWhyMdmL5HH#Obp36@{|_Lzm|!9Iz=BIEFc6{4TASha-~3mz^xAr}yH!R_6PLnSkx
z80t5kIv2BO(!Gr=6GU~Mv}S!gR(ZBtgfrWjd8b30r(aG|&<!TdtTwSZo$D3celAIH
zX|kPMxT#*ydj8)%hxRXv$Vpzm<WktKMW_CKb@5X!_CCRNNNj7*8NC#(#<&*G2?@WR
zOv!0F#gLvdgD29xN2ckFZj_<axeck!Y5xzXInGJr7FX@he!HWy?!xEUVtt$C+*}?#
zdNEO3cg-TPFAjc7d9ItOFyvVre(5LD#384_@gRgrAl=@fz2Q&I3p<vDu7a!$rZ)3F
zcQk6J7jkS5nA_`=`!GxK=9*&{`Zq8K2R58{f2G(K*1DxR#yRtCgv-qPnZAnqn^!&(
zG54G0<~c#C$W#B)*F`L<>t`STtobSD;GTuI4vDU72+>}2++@kbg*-727{sj<LRa~1
zWscpT6|zu^;k{tXl<%Je9Sc+%=Gv_BSo<>2v*F0%4}Vuo$>LOXJo9|X>MLJo_+E^d
zw!vxIPv1_3+FqfbTuL|k7U*a#;kX>Q<@(7L5*s!aa-I1f_$+h-&rY_9-wIe4acsRQ
zsMV!VmU;A3+YCXwun88kHiWdzP@OQXsW0O0%d5-Ub_n*XrHH1k7Fm15psT9qOUE6h
z1siV!^80cq2&+2fiWoha?Ijzomby7a&102W;DO^Y8w6eR#Y-$+_qp_Mo}26CBq}5L
z_u`%a$0v$cqdnSxWKP^#ky3NAK+P|pNbB3&Czn3`4*B<L`~LklGwWFm8g+eED6=i|
zX8M?VZN-J72Qx)aPADjkUvtJtdy9=)+rdX7eMfsX$o?z{j@c%@Zqf`M!`H7g!=Ih(
zJL5C?TZEpY+eWr~ohFlfi{~-yN?_7gU@>;x`}mqz_yl29m;6Jaa~MqiH!?6@cx13+
z(Ug+^!KNB(l+^Yz&Ye`cEO^z-Q=xN}qqFT?x-a~Fb>i$IPRXgReluVA+I-F8u?{no
ze)&wt&gEJZcQDK4nFm~!A6AS_HZe)zeR{Lp;_G8iITf1)X1iveW-bc6`><_^TmH(0
zg{$foTU}l8D0P+U0WEQdzb^y&f9b_)aIAXIWE^aCElyyvaKD(gDqHvqj|a^gqS;@E
zdKEPa*KEDMe(|K~t5Vh|MBg|N{$cvQCXV0wisw!SOLl}cUwGlMIH8$QmU~I_i;T4P
zr8DBSRrI%9dg6T6dDZ#6LZcA#Q=;Vst+HH)m9$@Fs<Lj>5Sg=Qt*V=%@V!e-YF<}w
z|G)Y5bEj00hLAYR<%3s`7yNE?Qi%v$uXcXbDd~^X_PK8A^4c2O6sE?(_1m$3;{DD2
zit<Umetqj_+bwO<YTWtZ&hKMgN6aTZQ+BTX+#T=r@A`iG`YeODP16_7&$)UerquAg
z*ERXZA~#P<j^_Jb607w0DzDTzsbjq;e1BQb&G61t$8DcP!tb5d3vVp*D~Y_Z`^i%E
zYQvm4Yu??EvYk}E#30#|$@WN|-YI5QLy`1F(<N-%ZCv-<+_9|bh3wak<H27zR@A&W
z=(<MHO;~^<;L3+GnZSg3hFdmN9Eg1S{w`0u=q!(E3QM<rnjpM<nMted!e-};dtaBG
z=s0(B-j@i;n~ML>e|PyR&SB2HGDv6ZN=Nb1(5e4lKIz}Zy`ar*)zbCJue|p&v(D6V
zTzB4%F=FfQptp5DE^n`0y#1D9$@lwL@6LMEady_xwT)$8W__{y61i7J>fh^W7H$83
zHCt$<ChTARd%p3+jr(^@-26fA&*w?ov-nm@woY@l6ffckcIa?5WOUMG-R7acG=g<&
z;NlY>)fKZe6^;1xtk&5q?ElU%SLXzip~Y;C2mO~`uf1(G+oyuJ>2YtIf~6yuzJt*E
zL(M$`J2g3Sx)g0x<a#<9cL>;TY<2UJ@VK<|_o^lHZtE}D?38J2weU5^QtoAXIvn~e
zhZhSww*>6!xXHq()yY_7#o5&LfBmJcO^t_Hte5?+64ukO{&sM~zt7#)d&HxYtd}}<
z@Ehwa5!5~qHCc6|eabA!FTxG3N8|!-I?VmZcWOhsvms+sg`G>IIM2$B%P!3E6<prB
zY0A+R;$;%dfeF&>r<l5`d9qHj>?-8F$+5#Qf?Yj&&b20+>AN=l*x|auk$-Mhm;LM`
zGcIT<t>Nsin2`5sul28$?#{9v-m(uDZ&{qE^-qcUnX+k)$kw38$6jkYf1JIqPr<(B
zgk7Nux9t_@!ZVz9HIm<6Yeib?d>5W%az-NlFxP>i`I8J>_IJ!ru3ow++KiFoMC$A&
z&KvVfj>zixESng(%>0i^zqfX!NaxItixxR>{MR}t7iQ3(9MZ+9BCK!G+_a&&><8;L
zW6nt{bORUcI2g$)*~Awn$gI6uBe23@VgjF?*75_K>n}MitTFWDzc{zL(sHISgKN?f
z^({M%Z|>Ai@mjQc^`d4k!zWHle;k`W^VEtJr@Bo%HaA!;+Tg))c$=AL7yIW`E4`0w
zTEM0IuxoAe($JvO-&W4pd_;TV4z67V+}k#EOkBvlyMUV^i`j=;O-R_;ev0+Z9G9w`
z6PBD4w<NG0(A>ip*mcT6<bd_8un)@$we<NkCwej(uW;nMv0-X1qucux?K5ZXz0qvD
zw($(#MB5{;k9^je_#o0`UeUx`fA-$*Hi<S^l^L??k@Ktz9K8SkUOM(-^D3X7eUGQu
z1+2Dvb@<5Z;&W>X{67A+lU3ed7-IVB&irRfPER_Z^@e+Y;TOLpkNy0(BsxtTmI-&x
zF!X<2=%Lxjy<D&%{<QYtIj1!rG_2p@zy82{-5E@OJk6R8$p1ab+46;(=_%)q0_SQ@
zhj8T%{h5+VlBdHoR;>8Y-c!J8QpxPZxpsaJSJZ^XlUFa&^4e)5v}B%+$&3@ipR}Ch
zbvWx7J6jiYp4`xCx6txO(-Py8CR$elmwmC+`LlGX<*w(y4=qu?Wbo$DnjK!NmiQlT
z@tDssKX<j&eudLvkKK)KOxk2{p}T~utc8o!XTFum>DXk6SgYM@eCF4>xcvTY_J3m)
z7xOKa(>;ca2CkDA1%y9n(Q^<9?vT4Nv;X$x-E%I?QVO1U?1Zr=58v?(+vjl1`+Q1V
z%=ehP?>vuOlPe!*dI;MsU+A_ZaMpzb-2UGCa_$^kEwwd(_o~kxtygDUo~azyv%LCF
zq_gnMB~{r=D;WIW1#`dt(S2mj@#PK2WgWZQrmT@Pl+=s3Tz%7h#}s+py<B>0XROsZ
zF{wg6t^0g(R&zwH3diD!W}b#_B0PDeUCk4OwJt8vo;CT8z@g<8^J2AFmjnrjmR_H3
z9Oz`#$-8I!SF2?w8w?k>EdH@8a9;E&xm+&=)m@?8r&N3|Ejw|lMVH&y_e87hwdB~z
zCW&0kK@$HNFJ0w0ZEB|D?H^;Mz13MWP&NEh&}!-K9aFejTqo`6;oco{bF$*(eF0X^
zZ@Q;V49!2=X}(v|YUbrKSB?aoiQ+3c%zgx%aS=Ql(&cw&=B?~2GiG18J>`l=@0R&D
z&w73KUB>C>FMD;hQ0N+?bL(9FWQ+GbJ#r-E^1crt$HR3y*Yx;3+v1n8)W2<NSlFwM
z*FLwGRh%tLIUoMHdx=A&+QP24Q$ja+o3Z|hJ{;n))Ir!-vM1-XT%Z<r(1#sKstd$~
zmlpc2UUGrO^i-$N+JN2%dnZ4ht|57<HuK`786B!RJzWJV8a9UWX3wa98u&lRl)+nM
z$!stEEq9fso?;Z`+`)V2|NXb~mv?Sl@q1f$uY~2D8%qyNf3%n@?dqj<+5xK<FD)rC
z_|&=k)9*E#ojYDX)oS*z)|}{}@3icDw##W9&fqIb`yH+vlTdeAD=02H@1FL&1yxZQ
zI$C#S?7TK_d92O4NZWL+)K!LOQO}$A1-*3&4ffmZkkGOq`c?C}XEh01wqA`8?POz%
zTOAy|B4Yjy?+Y4{=jTi~&UWaA#g&MTMt{E>OnZaOp4)n-Fn6#0;?5v@;rq+uOO-Fo
z>riR?Xpw59y(8n&vK1#9HpVnKb)2!ie&?m^JqeEF5=$}D&dW=8S~#7OK7CR1ioO<0
zM1!xq?vqE6ckjA<i8Wet``*__FV3Fh-e&(_&)~#_h284f9j04P9OcsfqSMLe7x(+O
zsaV|c&|CAjo7@bZ<^SU8{EfogxkkIp6y_Uda$B%$*!`mWYvlX`Z|=SQ8++1MnDId8
z?3aT2SrZ!{M6Q_F`9Sd4lo#zcm&7k$+jGZ6>+A#H$H}vNUSD|{ye~rS0pqu**WRJw
zPh(%!biHD6f4kX^O)t9W%(<<5{npLB9VL6%=-3g1yskYOYTR;Zab;Z@a_2Q1O>Ija
zTh2+@xuJVvq~W*b8S;LIlZv}C{_f)GKK<_vN3+R^uZ7Q>8Xl%Dejt^ZvV4cC*4c=6
zo~u_Bdw3t)?(k*x38M|>I_s{t#8<Y)$4?NDid9+SGvnIYL;q#ZKDxoWK~Hbj9n(vx
zV#!HPmx^sv)7>6VWQ+Tub81e2u$-a}t5Juh>{5eN3G2KWeFpOn)pCV0y<Bl&M!(_X
z<jG88@0gfMot<<xP707reEW=lovK*!5#uAB=AX2>H%xsz^S-C-st`ZfyI0PJS!mr`
zb8M+*bA;TvJd+1`CZTVS_-zag&7AhE%8WZx>>1a*2gymVv)yjAn)*!q_vmFv7iXbi
zjc3f_WT}H{Zw|`k?0CW1JtK1ct{Yr|s}6;3GZtfD%$;vj6R~`T{5-dgl;*^y7jNE{
zYztl$W@IKIzbv`(N!VjkGmD3vOfruq33ds5e6q;r?)Rc4GG(iTqrK{nCjIaB+qY2o
zhK@+y$2HFkpJlAL@cQ-MxYnZyn-9!5Ew`=u;Eb35_HQ!Kk<@(QJ42y4;0_y;>SBY*
z%^dlQqbJSLZe4nDr;*3|uVMMSJ%V>MIoJJ(m|Lg1f6=`Di;wp&koz-}NpnWS{JO5S
z64pVlq9@(lx<gd$U`p`1M{zs1M4Bl!``o(`HiPTM#gNsmc7BUj`7$T0`gCQ@qSWxE
z(FQ(OQ@^D?@=FgjPTMAXdn51hovP2`_KByOb#LR{pTuOpLf}%$x`e&kvUl`-+`$%|
zR(5;wW^LvFtMl|aw`LbC{V_>>{<U9|KF#Sa(rPc#Ob_1uqH~$%zR97hKX6W-buZ^#
z^`tp{GyebFBh(!!|6gW<+561Z)^}38y3G=`>&mz`w%lQ@iZf`ucYPX1kVm5CltW$L
zb~f%!H#vBB@4FX@wl^MsD}1b$snq|)^Z(uIII{}Fu9m=jE%*0xvW3#B?}pD>@@3tj
zFB7*dukrfz@uJ^zmald1H4@|gJu!&B`+36B(we~2<{$H$Zj|nfy=EM@+jZYN(}UgT
zvvT8Yy%$z|SjNG_so7~Xfr(FnS;%9b-{%kSuG|tcHw`r{R#|@fr{GrOd#gSg-~Lya
zy{Wfrn_GFP?~z^8UO!6v?%pq<P~TN}U+eAFIIl<V*jDM+sZZAK_pko5Rb9LyKKT7=
zi;Bg{-N|NFp~|n=md^hlCZ6N8W<o;h8~;gP`!2|rxIH&tnDdkI@|x(}^$m&5JSQfJ
zb82$jv6%1Oy<M(V@%ik1#ap^0nXT_GQfW<Le%BLLa#*+HNN!Qmt>bR{vqh$VagG17
z!2Q=N$zQR3(adkll5L+yw{yws<{Y{g_vP98Fm|iZ^If6;FK*!9utT|9YFX8{hgD14
zCDXnJg`diw5WAH3N9O;qrlu9uYZK<Lzo0se_nXODfu*;(S;aYzEGk*ZaLsb&gyq-N
z%KjaTTB=nPfBXHFA19a9EKc8cVMoEK|4Vs%dN(Zn{&w-Z_soA!eE+IpS*Y48kumK}
z$ZfmWbDgV~eBZhCSy4HcRDJnJp6|MBng3Vq>YBG<weImx>fgU;=70S)>veO={FlLX
zYmVQW^n!)4;@^w#NiXAtHrN07F0)2)CYRRCdCJqT{gS@<>)ftCQ{>Cq(i=-YFAJ!Q
z6W{If<?p5+^P3ia{&|05;DxJP8~-!q1UHt=3}oPlW?~Z(NmzYlnr148o=(Ifg(DrT
z;z1%8Hh4R?iLp;Be0=Kh+@L(Qdznww7B{r>t8U79`APN645Rc*OKvVzJ2%HFGwjXI
zi>Z=&Of5y6R$CGSd+m(V_UsH^7SQcvshh)Ws^zoBpWUzWR%pYN#t3x{M(YV{8rJJ9
zc9T81ajEt+)h&|*9h`L78B1U1UA6Q&ps}xt{a>r6m&5A+Q9`$UZXTGv^6VaG)nd(y
zAC;fAPExNb(KcFlFQ`msl4kHK=ckQ+zc<CLEjG=$CKxNcWNp-<T*jT{kKfr=r@iZl
z`NSp~)i$~CN!3*D2UlVWKCd{pbAqE^?F;|4Ia|AU^Db6uPGPndRPncaR>;_{J8jdg
zWq%KZz1o?RxYJSh+Dd`f{Q4bR_c6$<FufoqU6A@9lIe<hBLl}Fk!udAjxU7RLrxrv
z^{$zy6z`bBae&M5wWOM$rUKs)uSAoz<q?T8UQ;e4tM60{l6<r=McI9BX~-#UdCw<-
zF^;+=-kQveHv?kQ1h!0a@v%~ocJa$%c{)YlVaAjK<6O6=Q9_9~E=~Jy)cO2!wB=KF
z;SzsK^=Z>tYHwaD<QB~ob$RH^xWGG>cZGsu#k-SI^H@ADPHpB9OIg7n{AcDg?*)P)
zPZl#-oC@Tuu@!mdFll0w@^lMXH6`!0H-h?;^tJ^lgsLT8i=0@-`eM~&-oRrM`(;#A
zH7A$PGz%(Zo%$@JdflFfT$5#gJWq2u#+R~Y6N_Q>?R?F^m$AM<Vpnv0P6csa%)90#
zJ#l-H?#oLv(_bB94ZIP0oVDn!+4ptD+54t0i+B;%sZ$<!cguS>-kItS6aOFcp0H=F
zP)dP|>pIV4%VzV1q-d@!D2jLauIf^_z>3r3!hANV6AC9fyJRl6czFa|+bDKu=jGY|
z_X(XSaM3=d{-{c>?Zl+cX3h<qdj)fYCWK~gdz9qJ(VBMJJ@Dqk$1avpS0?r@yZgwi
zFv4;B1o!O;k*9Q}cTOq^$lbBXbW=cSmD#P#)puV8{r$%hc)phNTG7nx>34$yLpU9t
zACA#Eb!lGg#k=D3>a;i%qg2y1C!S(5snd^P-Wu5E74za$z$`v~k@Dr5Zcl!Ccj|`8
zYbGg&xn=S*zj4#%p7`OWQ(DcHqf1wv<TEH-(CO~v%*|<fsVB8ZS5%zsvfYo3-O1_O
zy>I7n3+G?lwCCGr;jiK$aWA;8widm9wdc?69J3|cy^Hs|%nqJgz;Ez-m#9j9?uY5&
zlMlVx8+uIaKc}e3v!n4Bucx{En<mC$x=gi0NWmpQt&!tyueN`u)TdKh6g!{vsEM|4
zNJm^*KBI@pCFh5vqs0-$qZ}<f!4k7{pD@c`;7H9``|8ZspyZ|287vhk3&fYmg$b?M
zc}R?x(bH{$xZbpcu1dF%xK9Vt&5IlLn{p<d`g<pA``Nj=%FkAFhpqa)Znnin-%r&_
z0ePwG=e{`E@%7~?mP1AhH!SE;m1OR6>Dn@3M}%vi{mj#`QzA4!tZgsO6c&8J#-t_0
zxg_NMYz9r6L;DsTakbOsiHq?{)Q(=bOm$Mq`DmSBwp%UkDitrA-dKH}7<4AJQu|=Q
z<{nd*@0Q)(H>9p_isD@NUrn}9VfC7m+S@0|Cuj0ozf|hl)8wi5A(TZa`OQ=PUP<9k
z4n7P|I;RIWNuDj2)Jb5;o^O1=bl$Ti>Ap9#JEC4~ZryW-J-Mc+)8g<GbM}qJDONKD
zm{~VCus5zR>_5%9{DImuvxcsGLtzIt_CsxfGRM*waswTiEQNzAI6|T&q`Xfku(oj?
z;l1*ZiRbI5t&*&oS`k^Zf{dzWPF#?!BCR_~B3otgqJ)m%*fpv?$y&inbDpT}ljaew
zzBWb6F|F%_5_3aR^VAxrOw;DaeS12DM2z-?C}lY+`0m);>Nq!;t8G)(qFXPn%-z!D
z_rdYZL8D2>|HZtWwNzTgAyLYFOWS`BLzP($+;1;%32m96d`0ups&(O9#sPi|>sjQ|
zbQVqO^H|NYCHq87K(J->+18C41J7zsjgY>7<cW9GlS`@cr*%)onXQ`4nk=O1BvY*_
zJmbKTlhZr2A`L~C2=i`S_UnuD%`ZLab%H;WWTZD~u8LURIZ0C>T;%9RA4$GBEG{C8
zI9w(?xzy6j8kyku;-<sgEeHGcE_2`Kk($UoFC*W8ajui&iAi5O#0@_^oZw)g)VfIF
z?4k(a2G`d~QcGSgE6>}JKY3D-kfLj<UIbTHz4qn!jVu)%t73B{4NARdJU9`}yDrIl
zcZ%5@k%^u9f|G8yY!aWtmv&g{-@Veq--SL$&QSaBS?d|hUX&#Ixn*g}8<B=9Cl+W%
zZ0(Jz<qT0gK1pfjAuSKijkE21j?T<$71_4IU01<MAbdg+Q?!6wQb?1*u6^64NCk?C
zZ|Iq&n;zK~8|LZNH$8&svQWNGrr3<cLn?=lZ3<qgQ#>VZlK0En)27_tR=YeqGfPjn
zb<4h(x4#3r*IzXgt$Udq|JS1MMXq1!tlMke?(b@Q(Hv-%_~v%5(B;x;ni^fqkN@00
z&XKf$wNodBQz_EhN6nv6qv0yAj-g+GxUkLCJ_WT`2~n(F0q@p5Z@SZaLOOX9r>b|#
z-jsy7)s~;)8GG0cZxh*|`!_g&XYEeapjVmu4U#T13-7!gZTSCH$wUr^xgjkY(HB1K
zbX@aTuwD9s^p(@L_Pdjxtxa40sd;Pc-L)|{ZAukFbo0U{S0_(fu%_Fk`^qHIm-3I+
z+^&1I?x_Tq(!Ec+mWaNscrCW#&irFsJ6Pt)oSM-d<ax{V{n<4=W~Y7|Pq8UF!t?MT
zhq7zsq*JG6tDak&<7iZEHn&{ssNhdEqnXMjTYt(NoWy5REO(dn<}sGU8zoh_v5&q4
z6}-35H7-rwrxa_hxBc~svUSCrMVrF5s~k7|`Jz?#(6yOQa^8x_oH*a)`|&*Irw58t
zoN9kmp9*@%@kBeh;MPy8&(ddC7|ZNz*7QqiJ9lo%?R)p)cYl<<wcGIGVo%3s^Zzea
z@|$&H``wax9~zf8e_MNEJx`iK|KoTepMUc<D;!*Q^4Y}`D@>YLb(!7s98dhav)SCH
z#$%^<+@U3_rLW!El^LAjct`F~mCF&)r2!%hi<{C;@EtGOaA}h3jIi{!6LYfGFS^>(
z^Mp&@{gR2}ONO@+`sp1uucIpC_WL|r^W5V6rH=J;{_9Mi=X34BHt$*YIP(lIn`%ml
zA2?Prr8C<!D%*FKw~bWi9+o3B<kdskBbG!r8FVywgs*7jDXxuW;%qoMrFE6#^aT$m
zKmR+Ye6G0TkI8GlMkTU{xIF3UuIOMtc`W`$Y)uR6R?XIK!D+5<rr#C`-aqA_Y(@Ot
z4-;H@T|8X=w<w9Wez<Y*u*AKSdyk!a7;`pOaZAFqJ)Ut-0_9}xPPs48`oHI)w+LVM
z!CUnWVS5gWRLnTYs5UvRF?xX_|7*vVo2&t^eH&X;_#zj$+8kuz+%}6vh2bWzjIR%u
zv&*!pd${@}<_rEk6R!~OvgP2>SQ(c)?$2}CPG7v5a^U)p)cMjA4yN4+P`k>Z9Ld7|
z#v^4a6YtOKE^Ay=1ShMV>~vq_Vi!40)U5y3gZX-OaXt@~GIV3V>72h{JI|YAvdfRt
zpF-~WOlf40>e*wV(CvBG+ajUjgP3^0!H67zW{Zxf2XWpSe9WGVB{yd%Xf?j$b+L+D
zl=EXo=SN@OQ!E{SZuxxpf9CaBmWvYvl1g0f>pl{h%5?P#$EA?#Pi}<wN%*`zFr8n>
z#jW(<bq!C)9jxv*94p@RYI>dOwYWI<;6xvb8$LNH%jWnB&h?e>bM)MHPjahV2Cp!e
zQ~&PDDIJ=@=Xs;Tj<o#xI6a`?z=8*Ny(^L(TVy%r&9Y{Ev`Y5^(?nTCqb~0cXXkJ*
zxCBM@>U4g!WjVc*Gnn_LkE@c4$0C<4O|~ZYSNb&G=sCiB@hb0C4%ZMBw`;D-rEaee
z@M)YA)LPaueJj7~lL@+#J^Xe-2@@xUMckbHpylMlnch6heoF?<%5~jp+p4wEbN<)k
z|IfPozj`FaZ=UZyUgN$vqibR&X_JdR{y)AY(UTp>y6Z?Z`=yo^mZco?c(NBgUbS`B
zyMt-o8Voj$N2D&ru;d=(jbtrn73#WkYpo}rZ^JoTHTjEwdM1{p)=punz1UIx!y{8C
zQBsEGRZm+r!;(1`UG5)vgBYGo`g&S=2Gg~gJ2E>PHEn$R1g51uYx(uoMdnm=^{p7u
zpiVPBSIM`xeSY*y>h&rAi;+4MeePzj%(f#9*HXlm$$jK$4SgsZYB?cNBQ=aQ)n|f2
zt5XY$3gb;xMi-R^29^VvX7gPPe4X-+%nn(4YN}JcQ)k)1yTJjA7`8Om><}&E;9q~>
zT>6Kj?i-d~pWyaNWL}7aq^7Npme-Oz32v7(w^Wt?^WJ9!O_b?RDrNj7d2Nnv+Ki`<
z64MeJwz24^o!(Lr-MHwq!Kx)bPDi>nJ~Yh@tNb#laoe@*nQWUZ9+>4$&hTQ%I5+#%
zn}|6+XRHNVIK&ps-!&^|l`^OtUAx@n#C4X^>s@`*eKh1HkHkv4u>`bs=td+}d2;n|
zv`pymlAO_3xWmb*X;n;<zj5S}LxP!)CeDdnd1cb1n}HjqO!d7z^-J9Hh9@FSM^kH0
zP7j$evqDi~W_Mu5V$XM+2O1~K?&I*D_h9xO&L%roaoJGeT(!yzZmC6+_g;vQS|}K*
z+Sjtr#Y*aFHq)yGn`X7`l5%8mY~MDEA(VyN$;I)Ks=5%%hrR#3Lay9eB=z|0jw5P2
zR;acKh<h+jF}#|*h3l2W%oshEc8PoLhQ9GvIcCZFPb|AR`5fy}A&D)Cn%arnfgwRY
z72S!?j!ZjsMb|ge&*G$h<IDMbrS}CTUgbS1(HY@i5qvP=@&TV@v%8uTT@wy3IDKtn
zi_}WB9jDS3IJx9pYH7QsR(e#$@x>LEi;LIVu3hPIq^pVJA`9DfK{gkk32iHmFXd}-
z*uf-zAhc>?<Bc1Q0b7`OD_LBf+Ilop+V-xMm>P7vwmZT_{#2y2PVTZzI~m<}Eb;ju
z(|Bn4RJZ<_p8nO#BwS}KsddtLbF}6E4$g)ZVtsqw$WBsWm>08mqUNT8iS~W}r8B>H
z$-BKMT6m_tYc5mF3I!o0MYaZ}HR~95ymMe-a5<6prtG!157*oqvB^76EnLNJzx3qZ
zS1uotazCA9`M}qrx~s!wVq?u3uO`l;ae*vHyJj8<n?I#&PQyx}9H04@zFqxvr9;=K
zW#fex?kY1vDq;k8wN@<XPdt{s>5R*Xb!*ZUPaFK>cd3|lY7+Od6?3+W#{OlI@@7d`
zYJ2zQnUxBRJy(CucDW;AdG6T`E{B~?296mTtj;ZMMHxZ`FC26(G>D|%I<mO@AxnqG
z{q{etw$I)>g|2cbV2Y7-y~`Exg>lB}LmPN5>Q24)_U6eGPHrhb%iQiqxd@h>?Jr%o
zPgD0n>HmdOGbW}fX{<l@q@Qo~{S9vWU0oh8S>jgX+4b!ab8lw+6gjihhXEJ00s@Zi
zY?Za(TIC|6x^vq)rUlt=PwDMCwN9p7L)M+;W-5<t<J8a(ToG4!|Ht`-z6y5X(Vxx|
zB$m%4y``B;`Ta8wM)nDev4J_M1rt>^OG>PleDF&|xK||R6;JP}iHAh{4Hw4$-FV+y
zq<edz{@ozXljmIO814q-$TdD#uX8$N>$;aS-pO~HUQWD{<L$Ee*{vL($Y6C<gZZn}
zIM~+iJmkU>Y|yUez{c*#<gwXg|2k9u&2QKFx@}P4OUpeMGhyWvk9Bv?I54{T_B7Qj
zV9`90;kbiI(qiW3tN#m<>`rc7xcuHd<KRCA&P}}xKE9Vt5_<(sexKGTu>Q$eTUWL7
z>Anof51DQ~<z4;wnffNaBe`-@|F7^72@bo+`T3TG`{t@IyWWZ%o0KN$bMXFzxhxZ6
z+k{%)6luKmI^uoY#Ao$lp({q)Lzkp|SSMJ|Wq<PhIhjJojtNse8k1kR%B{>2E4=U9
z*}Cok-~T+e!Wrv-cCz$dco9^;X|w0-4Kb6V3l2uq9h0_D-EKQ=+k{Qi4&B!}E%di^
zjsIf<?^ewRJkx!?g*iKa^-|L|D(edVqhI!==(6ZWfAP<!xw0-Eb79$C$aKWSjQv1T
z_6F1C+Lz28HyBFX+{dD*f93W<gF3ta3;s$(?pK@d>EGwyC{pwAx?ok3#;wmAU7xPA
z;a%->*5kv~;I8|hTDDdQWJ>OF>Sa*rFZKP=!X3CKvi_p)=C01)(-daidlh>+Wv8P_
z!u+$fU74GtUW<7JMi-luJ3KUfdEBX@@Tbn(-BZ|h{o&;F+hFDPLFW1q<M_|X><*Rf
z2EL+~ogyL?cKdyrc`UVBnp1!6ikfXt1h#tiXaw5}Mjbkr_q_GSj?6>7_t(#1VZX6L
zY2#B5gKH|g`zu&H*{^hDbKH9q(Uv&1VZ#(gP2qP@w_lwKTB%k2#(mN8f-BGev)mTR
zIb&N@5PjTR;Np~Lr~36DzcG9IwKO=YBFF27-e;eX{|1h0wlnPb_-0edr0a$z`T;Kw
zbqZ&8a2;w{-|81`H79JBOV=OQMboYytku<2ana7W&+oQ$Vn@rWj``^o@iX(!xyjBJ
z{C1&y*NplFb03~OamF)Il83uam37(&uPh7B+#LVX|7)i%;&sjro2K^N??>N;yf0x7
z6_>iddK6?bZI#c$`=9&E^D-5=Q|5S<>noM_RK-qM7H08n!Kp9aD}J>x9dJ-#ShHV`
zp<o5mo{|nFnNyBNE=N{ZM6^0`y49UOvY}XD+V#T+w>mZ&q#s&$YR-8N=@~558bUE#
zzh2Mj^YoZhwQ6}^)22yF?zhjL;+}JfUrg-joZU%*e!pxr^-s-@ivPc|=&<#o`9@36
z<o}sx+I7%qafIRHJl2MkEhW2)bQ3uar5K(&<&w@<95J=&zDp%*vVj@@kwoW~N!JSn
zF1$N>JdZ)camsy<$BU+3e&(}#+mRFcuj<(LlsBHz*id{?rR7G$+gB$#9gNc}3?|8b
zaCJR;^k&fVup1MUOkJuORcbg}B|qezlDwvtIAhJF4_-3;JiGaSas1g6ueSff)AX}i
zJ2WDU+-{^#br8IHY|~FZ5&P|ezJ~J|Vp7a{7v2t8UacoSU*T-d5i{9iPDg6*Ru^T?
zFxhET`PPkPoqNlw^Sc-hSUo)cYmIh8n5W7iz71FF=X=k&`tg)V__3B1EML2|ezOVu
zUy~r<dth0Ti07#uo=g!o&dC}&U-(+yv^-DPD4FB)zP<VPzCCqkBaM=`)oy*FJN3ld
z)&GLR_sAU(&UQ=oI5wxoXR78-6?X&m{S2$m_5J@e^Vyo8ygOOFH%!dlFe!YG&xJgn
z%WBUj*49pVTNfU4t?%>W`#$w<O6?5+u@4p~zxCzc8c<c}``Uo@^`Dl|jNZTt9uH%m
zxZbjK;LuQYI3OLIx`X3EP}0H%AwF3>pBo<%U7G|rr@aW;=zXk@#XgQj^Hb7^2}-_`
zWIPQXdQDOfJvPO1a@xrmnvp^}#exg{^sEKnJuynnxH!vwqFSg_F}I_UneZVSt<ucU
z)n3tBPdcakpWHClgk4CbbwalgCnL)O)fHiBOj8;34#i42I7v6H)qAuzW4o_p8&7Fn
zt(Vh*%iT+FO-KqZ{up|EiAP`1ty9x%r>-^gRY*OaEor{a<CpA<SDR;t+3r`;d077N
z*frU>?QA}?7HIaaW<BucM_}^1c5&glr8gfc6*e97RI>kN{WJ8L(oE(0x`{u(%!u50
z|I4ZkJg?vJ?5}=$$noJ1P3dJm3a422Ht0$x9h!gBWO?!frI*@oR~fy&y5>-*QI9YK
z=Pi9TW^R#{_N=TgvK*%#+LXVt(Mz*=%0iX`ZRahMJQ~*Ws4mguSX$+k#cgD9(VHRc
zlh6UZCMoWX%K}f_c(UkE+W)G+HM`TQveeCaikUSO8<iffp5LUsR<ZTK<c#HN_0d9}
zSAVVc(|m1kDoEwwsRdlBQ`el)SSB>R*mG(B4wXXfxlXtHe9bIpG&OTRn{&XARqa49
zBj>jxlULZwPYJv*W7&?6`~JxBlw5F?<SlVL%-6%YY5A^OnF<fJWDNFh?Ys10?GBa+
ztJN56S_2p^Y}0P!Ipm_$vuF9dtW1RyEFBXA9Tg|{s6=xn9i3K{6?oE3`Q#68rR3gL
zUb9{=?c|$xi@L8ZxxL`-iCg#b&W7#GnXdHVtl{2Yx~9tYO4E7tF1Su_{(i~ZpNn%!
zfWkc%kp<K5sXRGr9=BQKRugyq_7m%EzkB?@^~mPIi_`BGITfmS2Ift!+R*d2<q^}h
z7M`9D8Z6me2FZ;MZ9i5SX+23|QoZ!)!r>|IDcMH^Ra+u`m(Eghb(PFt7OEx|dByws
z%p4VuO@UvyX1PynDqUruveQ}URL7wy9zNZ9mxUHR*SK|JsY7AaNwrzKzHL(5sOnOD
z_HDy2&bM2eh1N4$KD_tfoXzJao8wr&ZTbF=MP<YG{5}809GDodXm@E$6m(y8Vdhq^
zJB4hoi-U_i_VT{f?&TLw&?yPrw<4s;<*$I|Mc1m0qL%~90~v&t=)GCAJg#(ZipS)x
ziQ%&pmfqEx8ydcH(`k>H3pDjOlINDTMzmhJ^*E}0@6yNN|3&+4W?u?fS+u|F^gq=n
z+eC|Z%-SWcP&04CzJBimLBgx7-yL!8C@nE(s9@53)-HaJ<()6XeZG(H3+n%>Zm?(N
zD5z(8A>+uF;qdV9>4g82{6n8yp3<^X#onT7MyO1v=&G0k-{gl8l^#YD*132j%XbE-
zOR;R?KldcLLwRBA4uw{hb*;LlB|R@D1x$T;;8f|9&5`e;wrt*e#l_WXSC5a>GNB*|
zL8Is$lAb3N80C^OEoJU_ef4NkW$<K;zBoyyWX%iHc@x$~rz;BtsHn>b{n7L*T0DE+
zk1OVupU&o&Oj#_l&Ea?ztAR74%p7@EagQb*KNT(knFV~`Ejmnd3YjwgKk+%S!ozWG
zfC^hx#DaCxoo2r9vq_oZbYDP2Xh%nv+L9hm2DTLqW(sblE0R+s7l~Up`J68mk4##d
zbjZ`w%d1$+@R4g4qmJCeRuQ2Ul9I=qxKa*f>8veq*6cB!SZr84>!gdI(d`Rdxptci
zAK!keCoH5L(RVrKvElPwkCx2)H8VB-*D3LRQy31Ednz*hyL{i)TBU)*rIBZGN5g|H
z3s|e%5_wKtVRE!{7n-uLr_O~@f|*lnL%k!Py?0<p9hc~_oJgaftt=144Oh$;;9_W1
z-Rr^a{>!&<LXL*|<qPc>gf4G66QVkgiy`KN#^DOp#)u79F8cRg(KlkqS<$9-rt^PA
zxydA(j~YI6ISsY7wscGMhD|Vhq_#z~Axy=W;b0c4oaaNXQ$1f@mu_g)&U?tU+h&Q!
zd6V9Vt(n|Aw>9yktmx6zc+3@E!l)7Z`_h6<LMMOkVDZ(}6qLGn!tNoDs$c1)%PcQi
zx1acTwy&~V`M%EV2HhW7s~2Bz;+w1xpy`sx9Vl=>?N~$T_YI3g*IaeVj?CuPR54Lw
z{xkid$*Q1PELxo{MPB*R%Wp@Au2P#)6v0)#$?1`hiqlEiHM*x044k<ROt?I?Q`3HG
zOVETQ?jMh&*)Or)OI@|r;JMTD>TMhJzHd1aKj*<Ap{boBw_{WGX|2o;nlwvUHLPv*
zHj^~%hNJ&)RXqq*SjM<P*P~JQ+E&MPUo%?I%AAR+R)}DDdq7(|!ezBVZfDmO#@mH+
zOVtiMX)(S1v$MP`N#*^F;MkYB*R3A5ZV$PX9rwN_z328!=O&fL2{zy6sO22w(#>fS
zeRH8n%^-%mtY@KYa)*<)n_*j3Mx*T7jwTVajjasV7s>i=%=4A0y8Zn`hutT~>#lMc
z8k}+NYEny6wsBk$`YzUy&EmMwN5jl%QrwesQ7?~p8cbgxYiM2^##!RU-#kJ0NcM4$
zgLx}17&QhRxv)#<27}zguqnbnv$@!IZ&_|#J1;=`>2#eao^F>S>lnF@9TkOJ!&*(1
z89k0=BuQqOO`ImK`M>92xP)43_|>$9!CM`3_bayQ-n!VOn<9MhuSi4l##4*RGOmg-
z2%YOVedO6cnX;apFNLQ6nB{iT^qY-PMe@9F)AFp>Rrl3BiRQbzDA~@;>Yg~Gu49~b
zgNlnI_cnzhTZ5ebK#`kiQ5{XTCKqpMC?s-ou4mWm&|#f)A*>+iz}kY%S8p#fI5V-C
zH>bdGU*K;WpY&6%Tsuz*dCiJT<xZ`bzWht-{z|WgI^*q@XA9##|IB6Fr*tw_Z|WzJ
zP;J%l&F`MBOL}xg^k_ft<ve!-Q+K&*6B~_|{y6=L;}PFlrOigU4_K}yt_=;_&}cMm
zVaDQy%d%U`m(O`-`@B25FS-1l_W!AGx9r}Nq?14KL$;dLweJe4MGl%<7Rd&2<R(`2
zp1HK?klfaeCYD!$i?0dQaR;4v)@;t;&m6~+dhO_A=gBuVaP3-t`5j|k=(Rr{6Y4KJ
z@>v(p7W*ADX-}j|`?7#%yCzkOwb`i6l=9kp$LZtD3=d9W<t5Lb?Kp0w75}s7$wnbR
zx8ffwJYIS^ehq525WaV{YSlKGEgohc53kb`U2|d`YkG8}%DUC1o=GcvHl4oE6+flv
z^|ykr5uA^@)&~S8g{R#9f16G5#9F^IZ#SmA$^XPwpq9<3|GMAi=<zRae^l|f{CRip
zkDrR(1~mm|?uvC{d|s<_+=SQZM7&z*w#0!eYsv2aOfOiN)t&dfQOe-k9?~D^=fAX}
zn|p<WT2jA##d<9Z4V5|C-_(>^+de$EaGPFsq}e8uNzaMVrM*O>LwMPPhBs&L<@D6Q
z)2#o;K98;AS;f19lbC|uoqisb@}jTlchaNjCd<X7`c<+Pin1*ied9kffOnci6Hm-R
zF_|Ww35$6yoD%hD;JNca?9K!AIV+v^EE4f)_B!IKy&=8UugCgGW~k1yIX75sUp<e#
zvGU!!XYc0mzc15z|7PuR6$56~mp|?#6&!O|?YI0<#X7e?FJdd+JpI&q?A<el(swq0
zmI_Iph~M<aZPp`k|Lh8>%;0B_9nBQ$o;_~NI2kzW&|a}4PeKkk{I}C+?mHxv$|Pvo
z$?<*7*>60_TFbxh61*62>YK_DCmYTwDT`9$yi|k^xUDcyW6+$k<}~MO@BBBe^&eLJ
zN@!78$FHCC<W~xleg)g_f}=0;+E(v7)%oGo!7N4oxyRP!EEEWBR-M;*qoymSX_1iN
zqnI6Q3x%|AuqgQXEbHZ(;IFXy?u}eA^8<YqIsO%I?p`Srkv=Q?|6FiOznafP6|D_i
z*+Gv!ICI&ZzT0t0^(n`%B@DLzjw+u_nCf%b<io2qA6D@)v?^q6V1K(<f;W{-bGg<b
zLG7K7mh@={WW3MGNNPQ%WLs3E#<buQ+jG|JOKR4vF7G&5qZ187&hkuWnt7uk?0=n~
zTL|v~o=uv60@l50Fj>QwWpg+(&bg*fRjJ8IIWbk4lex@oQtXwN&&skIK6uu@dTDH(
za{YxSr`_cF9}d00@v`5++w`Z$y{btneg1Ju271~PM4oO`b1a_!FVp<%OQlyUxxc3r
zNO9RdV&7Bqs$@>5Nz6lonZ=fKKO1;Eul@COThiJ1uzrP0YvQv4Bz*gul(be^>1nUK
zWO!=Rq>2yrXM)v!^|1FR*{1iOKXPdapYH{yXN6wJmfEP>XLp6LsLQ##FwLH@i-|#9
z&`Xcqgmd2qB^IOKzb0PY(;xnc`$fYujavuYW@trZDK5UXK(qOk$E+o}GHJ>upPXUa
z!0=j)@xN;K&mYHhQiS@GF7;RNY*~@VGwA_~Hvj1_AKt&~y;rg6{XahD+D-1EPRARB
zG`m6`O_*~32e0Ssq!q8U3^$$R`}E4-yw=)tulVL|7M(TOa%+htYlCqfW2{;3qBWOR
zaH$m?*rX=cX4iFUF~{qjPE4GaMKpJ{H7E5;9i7B;lf_7h#fYt?c-d=C{YTErCaBmw
zVQPM_rIH~0!Kkpr#m{ttS|0nMN$WhrmkV(|whrO3;c<?2`*hSU^O@PicR3fuc771{
zoE&KGXuefp2A>y~$-;+=9-Lz9lg&vKS+d~NjfG;1*2wMh*3|p(!)1{OYs0*<HS<^$
zxc+(GpB5VV^<-q}=g9x%m$$6b5n_6E;#h(R8{grKM!BYyVv{y{+b4TZ6}FmN9CLcA
z%AE&F*QP353+2zxKDX^+Mo!9?9l~}ewmjxY5;kaw|1N6(V)6D(lhe}PJD!?+Hce@J
zSd)~!Q(Uum;I2jI|LG{Ih}rQTv3<EYiL=-4`Q{60i~QBwn9H&>(=t5Tr|x<p<nkrh
znd!w|mq%Bt+)9(Csl9o0HRRDRKixNByf&_n(%ly7&wZ3GWTd|>fk~~&M}#R)=JlPR
zkbW(1)rDnuH8vbSaX_xdk!5nj$3Kb-+#dd^2<6^Y_JF}p_z&aZhE(A{x?FL>n%Aaz
z+?}St!=I=B$m4mKC+n(*LEWO;LQnp0Iw}6@p!`3x0><)!w~Zo89JJ;ZD{cSceLd7$
z{}A6T(W0$hMR&KlUg=u#>`I}K(^oUSqB5nNEl!EX-)(PBF4>qQ&8y<Xd_?`!gUKu<
zPhHYoUbF@$Eh@e?NlonQ!IJjyFGeq3WNX&Bm9k%Y`Nc^;<IANYzUSKvxIcCpzwI>n
z?3r)9SzXtFDLi=og>4^A9J%MMliMU3vsfW#st(hV2Y;5V5i4>%ed2&f?qr2&Q%#nd
z)|CcFzAT?VskH99=YkEUr?yQM-6I<DYN7bQZSsE>HgNuE`1`QoH)A9J4RMb|<|CmD
zw?dmVBUCM#)hr%2-3-;)abR!Bc{hg&iS^0}dnYUWcUtRHtdzj<%zAB+je6D>6_=8W
zSJYn}-jKIQ^;M3Vfti}msdL8ylfUz{253p|T<Y}V{bZlOl2zLCZw0Dqos~6JS@@5Q
z)icqt@R+>h)ASa#l{XGJaWXl7WUgOewt3ODp9vxN{U)4@QrvP#KjDF|YT72XlLrpo
za?G#%8s>g5T$1lWoiF#52;nI{#Ssf@c(+G97xQ@hX~I)8Pu_QVe=8n*n<n}ztl<B)
z2KI^q&KZ-~zvr_akmpz=ueV69=&YWJgg57*DXvT=0j_HA6#fUyRQPbpv;UM&;13NG
zKb5qAlp@LLGn-V30>16xh;b`_l67e3i{iGm%fAaLFV$I6y1XIo|AGk$k2dOtY@3z;
z-7U}4J%l%CqF<(zNcY4;Zd-0l7n_p7(r}=`lOf~FiERv9e@qu$#4!I=X4uK?5C3uI
z+)NPh^5%{>eW)cTJa^fQte^<P3%W`P!WX{R?96uE)g{Z45S|<w{`K3537H}$jWxV^
zQKnH*uczPNX1>5gG5Ys(F$RknEIS%%8yo&AF6G`K&Ua&}u*9<8j>}?aDO4${<=t8q
zx=7B<p~5+($@_S7;FBd!a(s_Z=FHwXZ8no#-pur=RS9fqtyf=LNvj8%ywYATIKz``
zbs$G_kVf#U%iaIZv}<3pEx(ccN#vr+6T8QrY6<}=7nssamc*&FJo~UHO;lvg|Lml+
ziGHr@`dzooRQq%|W6O-PK3_GP)!ESp#SMBgUL}eIbl+FlId|_gkptJVZzgi@Dyvqt
z<cjDP^-$cOcfIq$%&l|YY)P0Z!saK;^Zvx7s~^%tw|&pO-!(Pr>-Bz9$LP<NC(E|V
z{VAK=Xg#?xdNS*DHJ8XKCwcfinL>*~R2!tG-CZy>|Dn=3u4$hGX6mE~S9qLln;BU4
z)2e-fV-xd|GcHTRJlcf6X?+M`cU)FvrvCJSmF}u;m1V0_Qd_$ASC)SMtyR)Kdu>yh
z+e6o7y}zzauUS3K?eKN+8;zpJ!cO+-%;@R~Kb~+<x97yIh4Un2a(;6j*yA&6V$&AA
zD9`Vnx&NO>MtW{u?CZ_F#&n5ul}K2kXjhL|)$JwH5le-C922!TE?m)AS<}eMFs1&_
z<mNvbHa|5SBu}_1ecG1LG=H{&(GPEVb6(kSFHZ%XF9k_$zncBCPk#@!@!c^^dgb=0
zq(Ymwr>keG?Asl1B_=*IGUddl=mk%`7co7NV_W+%=lr#5X;-G!y#6)y6*A`E!qQTc
z>hhnOoqzwRYS{xJ&4j$;OC0`uyQNjKv^=%xQniZ;-|z0jc_Km#oGK2&X}U}k8btPd
zKRGSu;KT<tij}Tej3>F67rHl!%=6=l&|ZCO*Q|af&MRdPlXknYE|^#t9sPCM_U6nT
z`F|!&ynDlP=ZVQSQ~uAbUGB`Xv-4c%vfC&14^6q<Fq>t;RP#?UUe11JPPwWy*lf5W
zTKDL+kkjsT%as#foSDef7OHjTb`IObX^U81`|Pq_eOb6vJ^i}?vr!q-gY8H5-eX<I
zDb-atSFLDL&B=57-!3lKz56SJeYMV;v=2$O!Uwj<t#fueC}yI_$R}NUa{33S*`gE9
z9Q=?Ztm0ra#q+7}`cGfdKQF1=wW1)hRp?6eoukTomM~g0Mpi8iowYRm*V0bAWwn2n
ziCZjZb$D56qc$T}W!}A)`K|gzPv%@{(=u<UIFqdY=t#+;S^uVVN+lniw)27b@&#We
zteCh%cxC0yGp9uYJsveq_?3{=WS5b->VKx@TK;c4yv$YfPKKNbZt+-j{y@})T?$FN
z8TNj^lWZ6pP?OK~YU0Htr?e+R^<RWjL;1D{eM#jFTK|^&TF4Qexn1QCYUjj1e^K`~
z&D5Pqpy8Q-PwvSDCo8;euG-<2xwliLVT;8FmgFlRI*inW?IzYwc~v)iNAvF+Q*B>)
zI^B9c<!3h2$@p(BDTjEU9C3eig!dMMt=(h36Al$`4x}_au`PRKa$&CY)xBCDPTcj+
z|8gzo>^{%0BG=v@+@$l}PN#q8V!!yz3zPQvuPS9sYGPpJQsEE~c(CA5GpCqU4adcW
zExx%d8w4HL79Z;7*Dt!m@iF<xL=|s25lb%z?dZ2rvL*jD7x+$U7V?-lQ?s>a3QLfg
z%ES%D3xqiBC#!fYI*@U3iO+N;S&70G44hsfeV?A5kGEeNv%Tu=_fQA>jY$V5$#zfE
zIk{2guVzX0gtZJi_+HADbeHGF?k&IX)bjfLG&Q4ja<>$X8@r{$%c7<VK4|N1Fv|0i
zRe89ohee3X(!MNta`3T>?3S{3J{3%MzQVN3W9BAS)7KL>^XCOv{Qj2Nv1a+4Z@d!^
zJ3e6AR^VXU%3L`4jCavQp1V5_9cN!Gza;L~!l%u1Matfvi(UBA?*?mwd4HYFug~{q
z?^e&;86x>JTFs+%M!~|SKYY?gSI!4GJ4|#u<-BIkaseNXX;Xq+So@~@50qw|lyur@
z&5jgqO>0RN*Fa7cg@f%1O017Ul-oL#I<%xj7BOh;+Myu%^~us{3}2_HOpoC{S^C)7
zQtQfN4=cTj0Nt~^lHwE0`T}RmUiNgjp}X$Jqf@f$Qc7glW2Rg@rJI_=a7XvTU8VTO
zWiyw}a+3Tgc2O)SgniEJ-B-8Wntk-9rh1->(e^n`lXP8V3eLNJS?-+86_6obt#@ig
z^)lO(Tgl60cLlgj<`U3ccHP!$?yMQpKHjNS`6+(U=gieBr<z44x@daHwr)vi4zpBw
z?YL&kDK~2!&58i&e65Khp<EoMYnTowvFdDeI>>R=@u-@%LnyoF?g@?pxk27hF5jB|
zFX?@M@oAWT%|nifG5a)*oYG3ZqV;Z3UhksY&P#m^TBWz`V7yt{DZ6EcY-{LsH_bz~
zYKfIg*Uo$<mZ>$R;1H9~hi7Ufaw-?HL}!~w=N#5t@-Zjo%JK<$!7t1<&f^p15<FFB
z7m~3^K(k8Vj6mRZ&NFW+w+o!@h?{=*^qreFLd$!VcFbk?RJiHj>}8WyUD&|z&0Q$a
zVd<{tjrx{HR(lCV?oBw!KJk%|<ZK&{u89Iuja(V78Lbsqndf}uTYZsrik4?Ii{+Q<
zyRT%$wcoze&C^}HVUg$B>Qkxx-jVkwzR&gBX2>d@yTg!?JvTyAb$^V^_ZVqy%d)i9
zi_favcA9kc+y7Z}>-U|kdbza!$S<cQ&wCf-9|=78>6Gr}eala6`MuBB(}~^xPcpyN
zk(VFYWwSI_#N6OCf9sWVv8!cjR7-8KTx!3<=RN8>vajv8;+}fBRKH}O(@Xh~^1D~`
zLPWQgW+#5zQKq@Y{9o1G_W`zU_xRWSov7OIH{rO%wLts&3(cZmKDgV48FCdis;d4M
zPKeJ4S<JxIAj?!SVX~csuwG6=@iEQj$sP-4)PFoJ+Wk^IIU-{+r*6JtMv{bh$Yz5+
zO_n4s?YVCmwhA;Vq_?lte3aK0AtP(XZdLnBZsIP6JtwC;GqeikxKOoB^WjM!8w=rv
zMcYnjel1v);ito?dCGxH@4t_t=0)kw`ZZTg+IX^+xK2e$&fcum{iHzURIz8{Edws5
zP>xswMb@k+7Qy9~E))H&MVq!AaGd-{!;^odJEQC+{)2Zr#1<_$+V`d1zG_0N+?EK}
zUB5OSIJ7p&eNlv~ZPh}~)n_xN${d+pU6;}y^~9x>J26r6mIK>)&rYwfqDc%P7E87E
zZZKRO<SW6tq4T1kRwrMjZ|JrK$J(E)^!1%H_l?iCSqoWDskGl(<h&tygY~M<)vZp?
z&-h$w^Y(2MPMvXx<<H_1+t2#W=a-xyFRn6al8n{@{>-Z$DXA9kG+n~qR(MEno6zvx
zFnbYaL6q~g>u$SN`cLec(zJDn1NT3}RsZE*OE~v`*yIqn$(d)O2!rNL$8PD3$)c|l
zj%pkW@aEWbsA2I6R@-Y=*R_^RlGADRwzu)@{-e7>j?4I&^R%B%H9xuI`6p`WuiPat
zTcS(YJjM4=fN`hY&Ni(Je{LL$3O4N2J>uuv(-GBsL#>@-xrNGNUe9MzN1rVT?&5#R
z6#iF5($7IKxY=s*#R=&qZ+OhCpG$JC*s(*vNbZ~Bso>`|Cl0Xw7U5o8n0!v`0NX?x
zsS}E#fpZvklr31Zvt_}Ow+lRZ*l(>q8OG2o(lBw`f^9q37W+pfv@(mgsvP>cMQ#yG
z_tmV0oin~Ai=Ihs)!=jKj<~c~Zta{r9e$&(zzvJ#7ymC<8vE+uozRX&@{URF<@KvW
zp50h%ab<U_lSI1Difi8#{9-vcq>BZ@CoB?SmAZYXAVOhdps=aN;o$5k3G*MN6t8{P
z824tyO7;AAWy!fmWj}thN|d@McsO!PlII!81SP}sl|rpjO)48$&u<K?e)qHEd)!e4
zmbFXd3<58x^$F?DN+=0ox?$L~#A%8gw*%KcTXBZtr4!;FtQC86W938-t+s=y<+6IR
z%@K9G`k3Sdb}vdW^4qG!5Ev54;E~F{{mASqt$)IVlNNgGhE067|Ho5B2^-FoicFz3
zO&9iYb-rIU#hvXUhg9MP9TPsyH3AC)jw;yZoXr*!3gb|&N{&-b`X5smtno1Aa@i@%
z6NPhDY;ufrH<4JV{_%{Yv0lzb+b1vkQl)0QYV8xYy(xLGQe}4Hk136-g904V1wXxR
zS5e_)>GF^jo|64tAw+S(U*%2WkA##bE_6|AOx`NQD`*iV?HDA{>Jj_ua?qtkC-3&E
zsL!5#r^;oz=(IEM%Az%=g{|6Mux7zgtE8@&vfs_q?u2^yrl-W+zScU0_sT@({d?7?
zPdT)=rjFyWjGZduor8Vs8(p=7T)Q4W49H1Rb~W)YUSY3$yv*72Q1*!r&O2r;S$iv`
zpe-`bMK8i)y^(Y2rpdWdt~1p(oH1OHeE!&**wujw{)tmqECMIKja+eb!>omuW&dw_
zbVX;?v6Y-=E-F^SKQAuPQdkrgTG#Y7x4wku^;N;jqr1Cib}=pABGD@G;$W<{bpg+!
z7p#HT7mE5FD~Wx=nq|A@%}L(A*7qrLu0b<(8cNp`>6ERXKkN5FU9WG4#b1{4hxLTC
z{ruGJ;a56I*UnzmKCQ+#DmAWNlFhf3Q{fWV#HQEHs;#;OrHPXnl05HiVSS+Eu5e`M
zjF$|mn<^SVD;nI7dADK5rpN#9JTqI|m+GS&8NtoJkzMoF>fIi-Z+(6^W~dxKqV;P6
z|I2A=mMe2IWDjj!IP;n#=hCa!C(J$Gt#RU`#KLXL;#VW(x5o+Z|C>@K?ZJO@Ynq-Z
z*JD%1u(1F98&f>Ds4!?UmfyYV;M*2!xYcp%1!b?Yf@vno#*ai3!o;H!z2|QUxpjn9
zbW!DT6A$f=`XNDuOb;rzEEUkqC|uc8nE9wMUfB2F(ZY>^)#<9z{bsDH-_^`piq<YK
z5;QE@(Cor8Epf+VY3Gu#-G>v+(}fNj*IKbh%7)2CosikRxp>|oWB=<mxydC7i$jw*
zN~Ht^1CML1TjZzbn9Msh)Z&?)+A@K-<1!zM+yAQQcAM$4B?e_Via(mFyH$Z}v4JxC
z#@N*df<m`(JxWS_xykG1Auf&*fiDVrd?_k-n#7|G#p43S5(T>|ABs&?Pp>U>R59c#
zN%9fhB(yDoed~w+mD?1${{?ezy}-5O2G^Pc-8l!i_MPaqR;^kh;L#XTB_ZgX^jUqC
zA!qc3o{4NS>z)TX8`{pk5PD&<%f%hiw=^<0CWd=#5jdn+b7XVtspl?<OEjK-)aY88
zbyQex>ohrw!*Wj<v*#SjdLbwuvP0qXMi0rWX2m9=SDNeIO{|Oloa5pYuxXR~iLeP#
z!h+wf`@h?m>$imU{!)Q0U-TZQMqfV~YxUGC`XHa()8K@{g5-mZb0a31J!+J@k=7ZZ
zw_r*Rds36OWAyywLi3eEYZMesILyQkaqZwx*=L~q?SaF}q~IeKiu{WV<enGGiu>O9
z(Y$UWtCs?6NTi5=hp+NOsRtSVtKD7{?PMzsOG#8&81bvZJ))yVAhC6>(=_$NIc-K-
zJtlQ;irZ5HZ0}5VUv$)3pH1=pW1*rK0!s`g3JHn|o9e!LEN}lXu*_&ijRr&D0#?Ne
z0S{I&-!kP_VFg}k%5yj>Wq*h+$Pk-zvhzUdG^PSQ=_Rb&zIGox!M5Z=_rVt_{~p^j
z9q9b$P#JkpblXnRZJcp$8m1ic=vg6{#F3oZx{TAGqqqBIRrfR<TM@%ehMf7ErcP30
z6(|n#YUx!83|BOkxqZ^e*QE6fqw(X7*;g5B?WVWJEX}G<l`|6;G~KSA%u;gF%`Ggw
zT}C)Mc)2O>%L%Fzr~mcrX?-TArx-PR!hch@MHaudYi!)^yg{%!o?Y-|vevrkx-&$?
zr7H>wKSoRJYSijnsFLVC$<vcVNM+d$G3AH+TLoB`Z|!!7<ac`MVBj^yc#^1W#Uz<i
zqQ!>Fx({b9J}|}Z)nZu_GpAEiEPwdsH7!}^J~hlUELtH!^`lk%OMxST^U6aq^CFin
z?NrGASzX;Ed!Sji{gZ^NaMH8w>Ol`IJ=wJxRju-NwB@}hX1iLK_N#7ciW!&EjNpsy
zDiiHg)fhKyWWDyNS3|9)Dq;rH0@gPRL_L_qdUvf@DAAR+P&YkLd4)#0W9mxxw3RiJ
zS1!-6|GI#6^@Q26j;rS16z!D~yY*;A<i*+UFBeOe{}0v_bYA?uu<e@jqz$Z-9!OoX
z5bgF92~se$xZ*rppx5roYX22Q8$V|(EtVE)llo}XxAtOY$0zkjE!kzlvM~qd9r`R$
zuN0yE*)>i?a9Nv;Q83#Ev04Ge+R00o7B0!<W~_VSEHdA<@ND4vau+r(PQltGYWzio
z$I?VnrwErV5?0)lz#+uCc7wU(s!4)^DoU@E1*{h--B_oxkaIOdQ%vEc`57TDNt1R+
zPTF&0@tO&R>p~USoi@3?n!?nudZS>q{%a{mjb62K0ZR=tRb%O!H>cJrC01VEyo_1q
z-u1TK!5eozvpwG;dtTjDOIh<|d&G;M0&g$-vvt|Ve4by{wso7+f1#We0lwRFHhBq&
zCI&QUMzbkSZ~nQCKhaLOQE7#ta9(w;pktVvE7$krMO%JOvRfpOdpvK(Esf<b1sY~2
zCMj@5ZWM4-6?c5P-7`#RtLBuo7gE;EESzl3wj^NZRPP=c@ioaCcFtPBx<iZe(u*qR
zXMsz!Stm`X5;|OUmLqZTZS|$rDcp=}RSkD<E6ltuWpZlz+N%=<n;-Sf`lO+e-1;VB
zos^m6I=5`uZ$9s5?0FrvrRLX+qt~MNO6$DMqea3ea0NxhS_^RpD=a>sy(A#2D_P~~
zY9YtO8NE(9Y-#fMuFp^rWPS5YcWTJWuieUD0=Vrp#mv9P2x+UR1}^eCr19_d|FUf+
zE0u5Q{#(6Ra>o7;V~cf_QD?rb@^~39w_PuJ0q4$HOFXrD!#HiNZxmfzw0YXHrAG{1
zd?Namd@MO1EfnLWa`^Cc-`(Ms!9vD9DyE8xS*w;Y9*qj9w$+)obu&}+v9h*Ot|=GV
zlukF5T7DDo{<F>Qq5MX{6(<fIvAov)T$wR^<A%C}_A@F{r=D;yOA0;{+VJLDS&@0V
ze&+VfHI5$3?4395(0C=VWdi5c1NI!t)0nOt%Xv6+){PWR>z!tdOFV8mI_%Q>sbw~2
zH>)JiuKC)#`rlOT*u~lPW@oR=ZU@E0_rHDl1@+?&Z{Cv}p%){2>zB-=#C1-|elI=b
zjFoFm9{;atpW4sexQ?qad#UF1Cx>;|6ctL9Swk+b`?Is;yVN%Biwff5(HG11J$mjU
zT_CSuq$41?_0?v9MhU?m*=Yu)%5BR#nkQGBQMI^rwc(BqgTNEr#SU?0h1+%B2z<E^
zZ9aXY!i@vUvxS;vmQ0=<YZrBJwgH#W=S|XoHZ4E6VCBh8zkS$p#1fZ8`^Hx8t~B<0
z&=Gn1u+-W|eymH+Nj;nQPH|bz%C@YR8ZU$w^6!*Ndtn<cebB32Va?I4?-J{hx=r~R
zg`)#XZnou0nbsXw(uoz<Q4JHCrJ&dSyZroCzLwP~fo5W_)1qZu_*WOio)Zmv)YP$e
zLact&Md!)+_seFrt^U8AKe3T(ql4bfnL9Wu9~|x8XL<hTui5FbJu|IO9=zo&dFI6S
zQ=8^%Gt67f*<KTzxA(-F!wU+dH;2F4a{iYbuWfiQlg+a#Bb)EWUqXy8tE~HYB4XB~
zJuyE7%!6yU{Mqw!d9sGunerp1tDY{K=d^qk<2C^s=Y|r+zbj4GG6a6xss2sfbd7Gx
z>cW})g0UhqVn06fN`A&~JoPqHMsP)W)S?ZZIoE<jtUZ+z51g30{fwbXj$?U6&Lk@*
zudiknYZ%0?95wy9O<DEifhT1OMpqgbcX8ggHT&^qO4k}zDVgdWzs|D<T3F7AH4AO_
zkYc^rDRm)v?fuC`Oa{|}*ZLjUyysow|HBiHhUaOX;+`b@pJlq%85!|L?K_h%t#5UA
z{UhWU<SyVER<Yu;)!+8uhYzcNJp2%RR9;V1(7|3%@Cs|Wf|y0;Hr4!dNAn&o%{y-@
zW_;CQd1~kVO?M_eIVKp`Rb3eHSvzPci({s_*tw;#Q+Mduv4%{SNVUFneD}!|>!+*2
z+)f;+*lFG>m~FczckhY$t659-o)DK=?RRXIkJy|g+^spE<b=#_6qFvC|IONdP1`cH
zY{A0ktbT_p)+Msn)n0#oU%AnvL&ZAHL+A##;A~mJo6=E7Gu*f*y0d(8H;GiyUwG@(
zoBPi%pA(pve^ZA`W{X#=dIPtK#p)-~+gcZ!cO7qx`!BR6V4^bPV}qx8%eU&qJS<}`
z>Nt9pzdhr?9pA;XPIL!76L?zN{CMs8$vwMr+E$luIvCY(@V5_F>*e76=N9tLadvhQ
zt+P$kIc~Fc;(gJ-Z#6d?Pudv1cGEP5d;PX$r<>*_S-o5@wQ&0N&w_&Q4$nU;vni>L
zug^5)^#t=o9o2;kUh-Ws_7z(5IJ$6;mr3iv%xftIsnNZsbm!bm%UCEn^Ty0A$~&z2
z1jEmMEN>Jo{`hg*O!0N^CKcCgj92v1+s9Tg{S*5}uX#_zbkFTbQk)f6IQyS--s?%R
zuP5zv=&D*3crsp(Gx6U}uI7WUHw8x*&W^XO4AOhHkx`~zE3@Qj<W|A|H8DF~{Imn+
z31!r#x*m*PJ7Lk<rz@UM2y%_Qv2^F@S3i%l{`|Q|Z(`I+-ZpM$^|#NXWDaV7E=?|e
z=`!Q2%-w4nJ&k5ZJ<EN6=C0YQjRk?9ye7q++$*qo?wzx1KQ_rv**3LX@if=R>I2JF
z*{n{bY$;G+tcbd6`u5)et|Wz3div*j=KoO7T%4r)=JTJuvS02=tPxmIEwID>`p2_*
z5>ijs+OwH(U&!wJ8uG(={pQ^d&%JV~PJA@)TmMDb50b`y+LCt-9|&bGcqA?SVP2d6
z^CaH9_tJX>wAt>h6O9t-yVY=$>z|!~>qCJyt+PsXD@+`xd)Le;@l08&AwHKwtch7*
z&9DEQFP^Wzf3TT}MdHiy3l9%8a!G6TytwelrHRilYRZYiMfUN$j$J%E4W4*b?|XA|
zRr>0%wPCqQ=cYt%o;o`$%1!dkjSbUQ1b+Q<G1)YWXK&qVk!!2UFFf#P{8DvMT6sa(
z(H;>&E8QDY6Zgk`jo6sH^uh{;pv70XBvc(P&+v^l$?g0drfa=y@2;Y!x37jD3>2EA
z;c@8Vhr}aX>Kr!?+$el}Bu2=8o5s#f&U({WC)_-<(&v%hjGZmhWURIvz074P`|inx
zt%t7jd2`O#yQ}ogjVYz?{bUX-%xB~hQVH;gX<+GYUOGkbXmyeHsZ?HXaR*teTd&1t
zN9SbE5J_*%k=dlU_5a+dx<{7uSf}5YD@<ezDpUKWzGY=V>(@KQM|RD(b^FWmVd>Jc
zti+=S!Xh$Tr?5)ru&PBzbzTq)S6dU{q<f5GgL3PXA`czk3_%B%s6AaB(*nah3q_Wh
zZOTws$knyct}SiL`2&l(yt=3QR`_`92AKqQPi20)`lE-iy4Hz#QGMP{1{(d757~Eb
zTky$1dv(YQ<5r8yZ%%7ldg=NY-Szf0IxFIp>wEZ`w?pIMYpW+sG2145evz4(u!8^L
zPb~R1egC$21oVAMadve0EBxNCeUoud&!H&=9CqIyh%{XKt}*$_2909{7Z=9GnXK5~
z%~Ens{ICJjO7+PuS)r;^b0RxW{twIAy7F9f`liET)-gA?UXQPR8XA#MxiytXch(XQ
z<*K$6_l1kwq?+fh-STSrJT?AHr#EyQxV2h)FAvxD`TPGczH}?gNKBs^8WGt(wd~}i
zQ_;o8w~EIEiQZD`WSJMzw82|-hLQI-j<7QekFv~Q+N5dlctv9E$v=~XPacRqyXdN(
z*%=c?yVr-z81?LQw*{Sw$lYcndnqEhyP#Ol{O*%ER}?j6O3$U)<lFJ2KR<ALiOqW%
zCFetBI!))qf+l!ZW*#y5`NcrWd`-sTlL;kxS=|AxCmmQMzgRhLP!+lD#C3Yzh3lK&
zP7uiDdSBqmu;uQ)R1w`5jyW1pw_;nxsw!U14*4(TE7I_)UG5jxN#$H0n^y<;r*AA=
zua=!5%-B(F#He97f6^wCna|c1w_g;gDmi~MIx@AL*ZQu(h2Z6zzJ8l>gK5{>A37$6
zZj!D&7e0J>v-k9SmJL(A?004_mHlFo#U8}?&-a|A*W&qCf_!v^&L}nX2WxQYa^#;6
z4G+vM{WH-|$}?<QOw&r<up7s>PKmke@H;%Yo3*U&PxaFmx&I!1`@g~LijQ)uw5XHB
zrU^|S+Lhu@hOgXpRP?y3fv5141lG+;4I2!Wiha3}5UMmm=8%R1CucT4r|u>RE{+D~
z&W8<C71S7-b#;w1gyy~oP;M}kZS$V{U{Ye9vh*#Mk0qNGCI1UlKHg-&xV&hZ5noZq
zVv|-CM=qW${__HdH@}H!Hwt^gmz3k)_O`*W(rvZtrLN#zMhD%wq9l~1bc6WUCVD$^
z{gP^a#<6hYT3x3C<JmW>u5R1GdWnyzAbR~fg$75tpotwNCltPyb4#3iZqQ#HQ&&;s
zxXd-6aEi^Ho;`~!7Am-ERV23V$|zh=vhb1vpQ4|(w)zI$JsWHdOBNs6^<;ukL1TI2
zs#N{k%+~Es0u*O?@=m|!B02MDdIp!_l@b{to7)pT%dID$Gr4(4&aJS`-_`ic6tkm(
z-mzR&#>PHh_j%6LebKHWq<PZg1pDf?B`K;blRDSla-Tf&gn!+^N6$CB`~Us?AI&GZ
zOr<9r{;M~bwPkWAZ)ne(w`_KoiAwy!TO1J%t@?$vrxl(uc?bVm=veikH$Pc1bC=U8
z#~jnt#(PfZ*=M)!eXR4Qbhn~niUpGoV}obuo|{Wk-6jY3eVJh9#43E+A}Qov22ZME
z3-7C0R+2L;kNa}%l;hcZOirh0iOxkIo{}6k*>zs&BDX)zjhx5n_LiepWZQ#ft`pQ)
z@2gxczS_O0cu@d%XVnt}mPfZ_q*IQwJS=odww7-F_hR){*|7R`6QmU-D^L8^Oqtjo
z;-MtV<k=nY&EQvYM;gB;n{G#c^#90d)l1Agn+)FB1shJ#^q%Pz{D@Vv>gp0T&B-nQ
zHGMp%2|DxW-o0U<{nO0QRwV1xgje3*B|?pk+vYvJq1@=Zg{h8jqindP^NKs13A1kd
z@|11$wmMvrZq%-H<)X->xn(Q{jFT2!F+cxOK-J^H-j2fcmsGZDHpp%+IwI(|U**D_
z$c-#&+YY7w`Skq!uc!V8=jg8Z&+{wciPdp259ydJ_7e+RFQ^}`oO-8?T|%oh<--ys
zRqxKln;dzsIqN@8THwaJLzFpblGVXNjyWzCjk+w?4Z}9QOetGE;n<81%Zy9rRu@QZ
zx*Nc=V(CkX>ZHkrj5d;`3fb$5r$sGGw))s}Yw<lJ<u$IY4Lhy&ESyuJ$?jm?TWY){
zVA_W@k0$7{{AcJov9*xl)f@)v+dGd%vptT_KDyOrQf}_{3(2>l5<MJ^u7$V6lqgE(
zM!NHfY!ciTDgCZ$X;+HWR{l&b+gAq<?rr@wW$jaipgrw7{FU_FB=qiN+_#bZdCA9|
zML17i@!Rs)JkC8GntDsiFYh|HR;~Ta(L{sWB7Q~hrYrw%?KAxvVzcgF-u2d58@Ci6
z6Wp{+-74m+a5TR%tL57Eb&q`b>VJyPDgM5C>K`e--Ji2H{~QayC-mTu-Rt&*-?|kS
zSxcE6S#P~>em8p}7vm0XhI{jxtyLB-&2D8|!u*JF*NM<6fri_>WN&Zm5LsGTnBd0!
zqOj`X>H>lGicIsr$8^q$TmI*|5%V(Pc&n(@ljH4{MrW6=&7M=e>_+N?g&9uQSWox#
zc!$q3a*uM(`R+1R*=72w#0kn9tai3{2bF)WJgIUoM|+7)rPg_g+dcCv&P&dz-e%bQ
zaZLlmhv~;spK~mm*JruITj3qoga)P^0TJ7t)^40LcT2kb>l-JFQYQ92;drRKdbVdY
z%VwEKn}?>eZyr{O<2o0XP^Wb#^V*_MQ?^7LUvcx}%MBqCjiO5)u;|}#IN!N_&ib64
zXHA2%&ztse?RxpJv{qJaiTu@-9$~y0J-Ru`@tNEutM0y?!Kv}?-6#LETRhAvU;a5f
z!SuD;YhkC`QA=4o63=~<_4~ei|NTEjPxhT{{y(|KwW6-F`@)}3bt}J#ENakXIDMbb
z?#0`?9rMDbH1gy{SWJy^oSu7f!@-jZ-QBvuQX6}N4SNkH?$cPXw&1{JZJCaa1Dqy1
z#T#e&zHZO5?)g!~)1AR{<FiBG3&SHbnD}Pw)Xvnq%{}vc#=*Xemf|XUhML<{7V}JZ
zRQfH<w=LStYvx>onY-?u;8R%A_w0e})x*MCr_{_IcbIYSKI~xD-mqx<WNWRS-H#8<
z(fGp8`k*09Vvl>V&dH*_<F|VaOZty;A3hpwWB!0iXhs9?i>(Hho1;9ITHdj_Ijbvs
zA;;ejDwmq)crW6)vU$&{!am-Qa!+sevKTD9HhZg*^5m%Mg*yLFZe4Il_4j9)2eW(5
zNA&u9*}LsB?=<D+n-lELX>i9BZ_Yfm%C(RqjALGi3(wueqaV8L^p#{eS6X%7XsL7L
z`E1Si{HAOC6cL>q`@RX<ry_Q0T38(NwmAG?>e;MgM=};SJ0Cmg&~K!`IOpMh$E@a{
z1D!@1t-?<@q66H%D>Q3*uDmc=>iACa3Ebl6H7x@Q`O5b=d|SclE6||fVfH-HQlYp{
zpk=PW7gfn8d*?C9PL^EOEopVlf%oJLyHgRXmL76kmb_}|;z<vg4{2}o=sR%Y%>&s}
zmd3f-^ZvaSIwNWRUHjx;1-DZPyE!$QjaYU#N-c?&(D0BtY`jOOHF-&N2J_7S5qpe8
z8ccL-=I1aAO<*t*SY<Y&$)=E5_>6G)j@GV~p6vq1k1*_r*5D3%aX92Pm(L>(=QT$z
zZJc#?!g}6}oo6d;`xb1yziH{+A3Wy@4sxvCbbjN`#6^>g4R;w#Y|vf7`DC%4!%yKC
z);y=A=LUUQcW<K0-3`l_HoCSSaGm_-!1rK|?~e{dbf2jZ-v9mg^pCHXnQ@%dp4e&k
zu*pWVS@VI=7Y*O;7v{NdwmY9v<x4m`;{lV5Hgj`UtJtAV-Um#-1bu}JRv2ZnYAP@n
zS8fd4Y49k;^ihqJue8hK7{9$m{GGxR(i?qNX{k3dI_m5^w<DxuhIP-3)t09mR_V`F
zl-{Lj`(JUBz2*tE?EX91e7y%9&j&8rD9rHV&2+ou6H9M89K9p?caN%>h12^ReJXDz
zzR%k1&gr(^=cKR_vxkz6QQ)b!N7fj1T!>a^zI|kA%%lrO84HbWG~IeLV`|TZsXd*X
zH+<)XDC%i6b7n9fyEW<d8_)j@D!XoQIGY@);XHCmd$J_&WHzM>Ck+B3tuKf-9cfSO
zxBqy~-SWao1~r33qdC>{c4@g;-JJYh%J^-Q+P<1)udMA>Pj*S~Hal`)*WDAl*gg6G
zbMX~&Pd-s|>F<OlAqfpOTb>4KWy`5L;XjVeOi;V9rFC)3Y||WHp$8Xc8#umDXy&|O
ze?!OX*ao2&E6o4B+-~~+4ui0W1^d&pY@hpEvsSDYJTST0;^L`UhhK!KeC6NBJ;Qa0
zhsEPwg{yuYnTnt0oQ{`PyP?SXrR3_IMH2I0?LGaX#Zb}X^oAKyN3LpE_dE`=`st&l
z+Qi?;I4j_+|EuDo+bj;vf4S<F1nW9!*7b+CO^msAl0#juVU4J(^K26ykD}&9I?Z*P
zn{{tA=qfPt9X)H5!DE}NXni7JiAaOcga)IGX5lCH<{BZQN?t}8%{5;|h2NN(-)K0N
zu%Yho*>g@jUUPZEAMl*Z>0hUE#O6xa)0TObvnK`pJ$}jaYDeP?Ro(LsviT3&1SntK
zn{d0oOQkhz&!u;-PHSGBoT=q{xZ>us|Cc9E_T~Arr!VT@woj{lOy}xL&F=A?sQ-J{
znLypBz8hx`a~z+o(JFM}I!~!@>yOLVLc&E=85tkknscim_ensr@a@HW&I&0o3mJs#
z-DuD&X}vusTqd|<^IP+EPa7T;u{Iuc^E4JaKa=;xfv$fuEPPqz`MC`bYaA%y6c7;<
z<jeLut}9wO<-pn+sh%BuzoVj-`tm6Lc4${{JQC59AJU=upy}=<!@C}l39PG5o#2Ve
zb-AQrrX;*<3)ieGyw?_c_sCai?QY>^(71N5RByR;-_4Ew%LF&4oju_e=&Ga2{C|(y
z+HAg04BCvT&Atx;Z7PK{jzlcGI{n{f<{8o9e6qe<{vW+=s~K`^QGm%!2G&wm5nbk6
zcbK+5Y~Vc5DSFA%+~9hLXhZQ`2E9G4S$BBcdQbkg?O7HRc-;8G-wQg2KA#QSaIq^^
z?ET92y*CA_B3J`YOcbe{mL|z6eRuWbWPWMZ(<#QD>OVIIJ>~5$RkkqUvADuB`|7>R
zrF^G<#4<WC-IZvbq0lNhxzF+F-1nO#KQbJet~%va)6K&fakp}(L~USSf2nV|@-5x$
zex1opEsaSvCOy0vA@$0QnjdbJq@D~rU|1B}V|~I?bkp^zPegD0=_%REY4qSwU!-E#
z-p(u=lRbMo6E|+3{$)9Px6Xn+p7$P{nWMmZ@l}800ghz>9Hp5;YyV5K3UV5j2{dz0
z>xi@H&Nbu`J)Cm5<M@#m38x;Ul)t(1O=$UIUcTcti4u!?W;#30v^uSLm3PM-*-MFc
zqgSO$<*fVo#P(XP(T5J+N3E&SCO!9L(-;-flGvKmyVl+P5VYpj`M)<#oc(aZSD`tx
zFYI}1-=cz!z#owXYqTfvOjcoV<x-x^S#k1Q)$FZruQfip{yD2z(AMP3+yIZ%v&ki`
zLN}U<bz5D!IzKZlnK4EB-_Pev%YxlkuW{?LSy^+#TEP9Rk_qF93G3dpaxUWtj9~pM
z!RpB;P{z>3pV-<Z#u|0QIqX7<*hcBV7Xmy@507kM|Kr1RCa|gP<DI)53q{KKMgGf(
zc$J*~8<FT`tv9zSv~R=1ds`C~3OJ;<b==>w=*@<8M;@#?FvaY>P`}c)r}sn}j+AsP
zGF1L8q&P$9*b#x<|Ff8@Gnf~-h6(kVwp2#!lnsoJOx~K=)0}<w+LZY=ii}%$BeqPO
zf2wMB^<?`7qdm2+<+pw_{o)|@I+mkqLP~dDN1#Bfnq;P@*u?o6-19vIIm>cdJN(K6
zyM<o1mkKbH9q6c8c;~&PAMfOj*PmY<mf-2TaVO-S`#l9k;Yq4(8&$;**-ouY6u)-!
zUh+)S#F=7OEN@PI+8JV{b=&st--UOc^0H(^PCe#vDueq}Cg0WFea6|WsoOftvRPFf
zPEK*;KlA_YwbaR{&Q?T!yE}Os-wo>pkN4zyxdfTGIGb!Q+~V{r?6{H5R5PAi3ol(>
z%^oPg?7Jaj+l(i@dE68B<@EXq)+exU-7!tgQ*htTY0FpeIscnrurSc!k6F=W*Y1j!
zv-|iDe^@)~9>3yIo{6IC<Zk7zy7%(WwcbT#94s3;B(6`7QsiIzX!4`tqkC=EXL(IA
zEH*orAd=R1RF`Y<=cuGPnXT%6J#Vf?w9J~6AGbGEn{~UUMU(ca#-NMSRm7rqFtaqz
zuJ@Ard0U=o-Ics~_T2_6{z)sF)<+q)Z?L`CwAy6nuZ-?R8(vJ*>z*(0fjh7Jf>=W+
z$A{7YF2jiS07+)CgVR_17hvY|<MDfEuuO36stsp640}E=7P`KzLwr(C3un;mSB4W(
zug-raBUvV^=P5kzneE-+SXF0PnJb}37$k4}n2~U|zu;f0`KeVpzW1_E@7lQNU3wPZ
z>uXjIclN0XIu&#m8LgZu*sl36DDX?g<@#@)uFbQ5rkOZ6@o(pJJ|wNI-(pjsZgKd-
z?!xIN7W2JMP4JxGv*ee>qre9XL>)_xiM3`gWL>}^*c8yc_h7f;OZEjnW_x>eO~@<k
z%g~6)pB&zl8S+#&TutJNuEf@ww=sNkbzjU84T){)=h?ZXLwtQA<BK0p63jHu-h4IT
z)t3kBW-slTb+>n}YtJ{q%a0WU@@6%t|L>gXHE~K%0juVNSH(=(Ld#YeRa|a*q+>5u
zwe{KSU&Wt$^d<aHb~(#!7FyXF%22?&bJ_ll<%SWhD<zoMzVC{9VY7My*U3^Yo)ka!
zT^B<%e}{1ke_gGr7~OMhgP%ao(>Be*{{rXTMEsSaW4%g3{T}orz54xSj^o|tO+A*p
zhnIi7ryO=~h2)EUwhAndj4r43z38(u?tXhCE&04h!RxGLCk=NdSKllNH2CxQZp4Ms
z=e(E9c^2wVo>#J`Ijmy_i*CSKj<N~gn-9F0BKK0QlFR)Xf7U}4JCmKE-7$~jt2;J*
zIH}>nc&90aOZ`l3W}idqy)4_yu0c+go0L=|O(TD4|CfBVU|FF;*qK|I^785Pm;QVB
zF`+fY=IHxX!XFp(rB(eo!ZEMOzHISL2Fuev%eE#p2q+XJG&OQ@Xay{Ku;73S8@rW^
zM&JS#y>hjrlFO@nCC!CmdAB?bY|!mV-8MzA!0?H~1c}(B9?gwQd>T!%@2$BxX{kdH
z)2owfR2xF=>;F}4T+ABcu(oa=%a7OBSFdPT`H$sGv)0xXS7!WO;q^u|c+I&@%<Ju1
zXN5UL)#ons^Enid;Ml__tLCBMxM)gaklK+sMg`732hv!J1Q;s16E?@N6j=2bZc3Ru
zMLnoaCtx$fxfqS|nQLYWPd`4#$@w2oWSaO{P1Y%^bW**w!=@&0`@d?=?{B&Dnaq5@
zc%ItMFh7^2PiV&O(A9I8qkUI;Pv8E8MQF+WdD;!ltEXzQ-_vqV;%euW-f@G`{^O##
zW>S}T&rJIFV{Jdv-qT0&=T}8qE1$i`blD--<Kh&bzgLPy!vbBu1~*6jWEKe*PCYum
zRs0l7i0!^B$^P0YE=QsruW2euBr<(b4Ni_uxRI33T&U)dv7Xn(t=EWe%A`KaT{n{Z
zr*kqiFnI82G6*mWO0r6xx#<?;bKLVt8$%G6%F?O+o1C7ygx|_oJk6@sQ&!W=>x!4P
z@Y`*!acQj@2jZAC5--da_7r&T%zRU`p-klB&Q@8bNh%jARC8SJIpj^5_@Yq!|I~x5
z?0rQK-85w;gm=lFJIS)5fA*skZ|$fXV#3V{g{7j+IR{HaTMIn5yvpU;B=tHneQ)Zj
zz?G|>PP5&1<#4v{+beFzG?H^RwTeZ17*1!(6%v^o<8@}`<<32lY7B`DzSe6QdN~g~
zN}C<W_9W|(SII1gIWIcaF{G?poj3h?+cU`?gCaqxEas!!`l%*<S3FCjCS2;8tjkxE
zxWakr)7u({_@>AdytdO3(B5Xd=T4rJqRxa_LT^v5?B>c@@X|H+ggV0_rphbF#AMdk
zI7+VFanN;9Ud~3wnbJR2u8jP3SA25$%dbAN=i8^Ibc>|ueVy#ET&p+m(%NZTSFQfz
zs~d2Y`G0Oi|Fx|>hrF)O?pb(LA}GVLOe9F9v5dvj<A7P2#97I=C0sremUl^&TTj@2
zNA8(!Y0dMyGw+*oUD4QWqI53#p`3ACSnL_Oli|`kWo?!o3Fy=O=n%;@VfK{Wyk9C`
zEDq1y>9e5SGO%b@>#tPXtp?_TlG%qUm(TmN*Ejav?SJi?PNw)6Fmg&?og*zS9kJ`K
zh~`|K3nHo=TZ+oJCrW9!T|M1Wo2tjV_M@2pTGz19Kb`Vnk+}=^Z_!`2ZvN`1Z$CD@
zetp2thihYWM$k5qAP<37mOq<yBN=m|=S~Y<zU}I}GNH<t7zL)~8A~}_g(PFT{~S{~
zT+rRrkRAAl<&a)k=>Ik&g*#U=*p$}QepKU(7Vvw_7rb-Eo&dGuoL4nOp3U~1kXah=
z$}>-`-BvGVd8_TEb1V++6CY1hbWoYmAbrbWrT1ON&aw&~$)syDt+zBW3*UI?7kBAs
z=K+^=Cmt!kZ5vvb^FGN+zS7w?<(pO6m82w7mf30(mbPx5tDBp1Ir;X5YsYPcCiLe$
z4(@kr5#;hJ=$$Ia({1)+#_=Ub+^io>SJI9YW8q<8bXs&-t?GvG%q4yd$2l)g^=cQ?
zjqRMWZ1xF0XUhpqo=bRVeolCPPDEHh)nGdFO>XB~U({NCmK(`#mYy%7&}tTPUQ|WE
zfiuR?b-HgMbE=1bQlGQtk(}c*|EC;~R8?POd$MM7hRw!#7T1@u_N){Pdo|~(i_cTL
z@<YY7A0JO$|3ZDuZ_)1hUp5L`7O;BM%T37ryC(SUCl((Y)eXrxMbqL~gbs3W6&jY`
zPKZoOaXK@hQ#dwAJjE!>sWPxI)|4w;#gSR%+>$58KfA*Beo$@5o9Auhsld#w8#%N2
zXn~TYZXb{Fvdee2FekGsyKb418az`X)G&2Lzl)yGQO*dLC8-6hQ+8}PGX18bO|$~D
z<SmCb`_!wvcceSY+pg{1G$%A8HE|Wgt77i?8y?9EIWBgXwCvcXEfY1@bS%Ce!MH7X
zrZ0Du%WQeZ%o`3$ViWSBbX=D_j$o9yG{JP{|KOn1B-vdrSr$nwT;ZdWp?Ln#3Pa^7
zoIZ(FR}9_Gc`$iMYu>tG)!CU>eCpuJ9QSZ(-HDyW%g)TnUE#tNa_HJ}F|KP`#g8?6
z9vDS>Y?aM_vh7wHr;0!8!PA?#%7tgeEMYlx`I+}a<$3F@He0U~crBa581~u9cuvY(
zPalJ+*Ip++*_)Qi*?5CNJ7lG6?VD!xm<Q_i(G1-pYb1g<e%iVJzhcA!HkKC-qDlcw
zRuX<ZodqqT8Hw!g@|3vFs&0P6baSu8^@U=y!nUdeC1nRbHxjeuNw>3*oY^xiX3|da
zUhB)3L{CSUZMDpO+@Q_SK5=Ejf@JaH4enQ1KSe082RbnPpLXZTZOsJ=F)Pk{#T0d$
zZCNPdlY4YN%Y!b4yAMxaX5L&eH!5vYm!X=n=IdoQUOk&}<v5=~vA^b?tR1=@3Bun?
zXGzxwaa)|an4P?0@l6ix6Ni+l`deLdVw+wpb93IcaT1?eiTbA-hH=|;f^Im(2VIdk
zyGTO0^QGVenV(8}If_g4{E8+8-xj#_W`^kEKTDoEc`gn3daY!QgY3(&m&vEkMk&SZ
zx=<LC6BEsKU>b|p3q8rz4jHB_{(D0{l_*L-Wl6BUee2k^t-O-z))O77U#N0M9FBgc
z$IY<jo9AZnhpQ(Wd|I9n_idBgrWm$$)0U+L&p5E}=k*7=H=G%EPrAQ8>frxA#`~Ki
zemQ@zlRk64-F(M_oqx6+>0Ft((;?DdB0enWsnA5#>>H1E<p^#OUU?$xWVTYr8lO}B
zKa`>-DaNQRnb`Xzv{-WXYs2qMFZb+Ioml<Rjcabo*5g&XH)yVXcl;=0;w9IrIZDP)
zEp01<+Z0y|x7o}*$(d2az;!5Pg0@Ls|Fj0Cb-@!Ye&<xxa~WS)=I&;;IbFeFHRtTd
zyW@6qKM~uOu-f+$@8&+0E*|xTB1_d28ab!T&Fh?hAXU)6j7cuHFIoGx?9vIxUDV4i
z@~{<d7JQ}l(oB0(*#0d`lY>eZ)!)cdH?yp_IBjwLc*;?wr-i+?Z6}pvbCtJE?Z{JQ
z@{a#st{1YvSZUo-J=NPRJH7Tv8oAA?+f{0M*Jqy5B8{aJ7|t58+&=xuYWrs1n<^Rd
zf3{5auon+<5RzTeFyDOtB+CF>pR<3Ru21Eu+fe;|Z~YNvDNSe4!g?W#Q(_-(vwZNs
zsoBCf>pr7cz)!;jMoSh4#sCM)1uSabCncvm54~3GI=w~Gr9JmXqRQ1DsR>(dax%Si
z=_r}O=H=3{HDGJNh0ZlT=O3PR>5)2pGU4<_ZZA)^2|Z$c94r@(vh+?omd<+S{X)4?
zgLof_hr28|kMXnw+;Gxd!Fe;WWpZO|YQv1ujD<IK#kiAP9DNS@9+~g+r7!qy?C!I-
zUH05`-4l6lW6S?Jid}OfMJs&{u*uAOWE=PB=|j1=YjV@(Zl1Y-GpJ>e?6f4yTZ(SC
z7W>Y+dYA3x#Cw8bLXJ02b~5%Q{&DRpkYe4)(QPBNoP$l7weJn*?Kh5(To@kaG8}xn
zGm1&ZQ$6t9T21ysj`{EJN<I5COS3a*{nT^1pY9bF2<D%dv~eQKrjM>n5i-YJr>p2C
zI2FusvRE7!5aYvhdES|mEDbH^Uiet76!)AWZvIWyw<C1#SNVM#d6pZ-$j!R%B%yF*
zqw>+(+l-dcshRF3eREt4x~;D%A2+yuTy{!psQ59>nR9u9xhw_TI_9(o&a(AuxvS!m
zutDzej)WC^@2J1x-EqQ&GqUaf>AlB1kH)!Qb4hz}LeuK>q$x@LH`qRLx*WMM>rju9
zE@Q+@juw$DMdfu5ZroH+?7bM{;ZW7H(BaJ8J6GpQt#Yru!xC*8D8nTuKH=c06USwh
z<~@Fx7~=Fa<kiAbjYpxEQj{+}d1sNd)bqGw#oe&JIM1+!CfhokdJ|q<?fu*l;mXyb
zx-Kp{Yf)=~cdQr7s$4eFfQi`?lK8z^GC#WW-}BDgIVF<epju(j8<ue2Cn8^2rzc-)
znR27!Im`71PM3|IJ@ImEg|VGqwl)dJE&JLS<r^0fxn!xm(R3e?K>v5{!8cvL_e?*T
z<MRJ+{Jva={kHR%UZpWPDd&F+_KjIEujI^srx{N<!yX>DO<=3L!?x^M(MzGIS1lZB
zDXDYh6m1hy7bN%`@r*SQ$xOSJ8z9o1(H;4N_svTe{<JHrGDRMqVBz1iT;o%_N0N(3
z#IsDtW!--kZBT8`|0CslQ7EI#Z$XYDLxo}_U&mC%jC+xi6&9E8-<<hisZ`~k)?l}#
z@=+_~UL}gpI-1-$FQxaX@->%^f~Dbkr#k8uZk{o(aT~|&oQSPQp5`7%XSF=$eNjVK
z@w(L7TV1QJ#oA2g5`9=}$Nv04lHRUq*_D1zzg$>%v_<>ttSM)tr2mL*-0Cz#=D4@q
zB*VTthSSteWZqzZq>>pmhb7~txmSzZmFNe$sg`j{R;!Z#Gd)YQ{&e}s(Um?jOKfhP
z%bb=PS){pHXZfV5%OWdQo{(L_^(%AU5`|(Vk@<@>`hR)MedWpdsg<QPiu<Q8Z{F$H
zV=InN@IJHh`Sv9-DRVBI&UEwso8Yte&Pu;&A{xuO;y5OUb$-6;k^NER+ldr`nHipw
z95s(DJfo3%`o_cHu&0l|Cd%u{Z48oit;l>CyL9nGzs*Tf3+sFen-&-ZYPYFmbHv?<
z(mLj_XGK@RBbS0>r#>D_t~ebe=68B-*0O)8J_kbhnU?vTNRM>k%e^zDVCK;Yj}{d@
zV9Sr>OLfZ3-I?WUaD0QNdw}a*mmH>Puaf7<9ebzp`p}`?q&1pMK|01kIsX}BbL^R(
z7dtJVU-f2d-ix_iIa?I7PTcb>lUm~QXVP>g6^E=FC%6Q;-C7hDDKKZMF4@QVKjy@i
zlmOSV>_`T`zMXS=i$Zqhq_FQ?-uvz8vOTLOdN>}~kfmza^QMQd!Rg_y6sf~bN=s(m
z6`MA9W1TMdyOS|BOFLTP!j`SL=B7JI^U1AAs;REe+jC#%3a};L$bBGlRc%*Hx29k3
zuC+c3Iiz!r=ol^2chmG|U9s=ov4^LQ<Xdu1W|}_r#H=aXg7l&?3$7epaP5e}-$`4R
z<`_*|WA`aIzb^V$ZJa~Zn`cRibC0k*Uz%5zvyy4)f)iUgPk&os*EE;?(#q#6ACykI
zf5Pz9|6`|G-kVKuYV(*QljUf3;iH$Evy(LUtmlbGQj>bpw;$=amzLtgxpt-AxxOb0
zJf13~uTgg^T)*;|_NsXgxHx@MbrhE~tW3#xrv6u{@CQo+*X+=Lnzy1Z-JTP-MP<=H
zvpy%wHCr~ly&o2}b>ExUv(k#6<sX0cKDQ(GNZj2+Vi`;P4xXC$+TqKaWhRrN_r)we
zAhA2lt#6hq`_BiQ_t|cjT;jest;nHM@b0bW7SBF-#^zQECFh-c?vqz1ckIcH6MUZX
zhH=j(hHhZ=H+XEbX0zCPC!-C_*-M$)_3CA`8TGHUX0H42D&x7V%bXjTuk<3FKe2N6
zrTK<<f0bIn(fi(Y)yDsxBFpZt%8op9<+oJfp%r|O{M3XWJ<`9|60n8wwvYOeO}9GA
zGHz+DzIpDW;^H_VW515rhsTXrrhj@T>dpAz%38MIta}~VeaGGl&Rc(qt!K@_AfC4p
z@AqEX&)4%fPVIR^U$*A`R6owgO6w;3y;I|Uv!>Fsn4x>0-`$r{CmavGonAKYmD9#Z
zD?<@yBj)y%CUTFJ{k6sZ>z#R}B3is)e%_3QUd9aVd0Sm?-{x4=Dtd%<nyzhLp<eIj
z8)-78ljlBU_5WbwWnJ93ttf7bi~e1|7Rmcd{Zh1Q)}?!FRnhzv<GbLs{r(w`e;oa_
zO<!q#(2F&FX`8xpE!qnw<}jIWc8TWr-|=swBJ+o}+a@q5o0MvNVxF&hhRNti)^Smx
z=aX!ovIxuz{g|~_<&^hyM)wIk(RtHcezk7%+2nk4>5hMAT@K9KndP(W(kI=itqI#U
z`rMIQc<7d=Qr?t|b4kyX+&&mQ+atvDvM92nZ}O-12SuF@5^Men6}nG;Dq8F%-dy6s
zZJeBV++Et`o9IT5{8P%Ow~8BY7I&5`)H-Ixzrf4ci*rh2Jad`kj05q)WeL*X_Ofn%
z?UxlD<i0&=g6;P^(z9KqW(T;L=AK^YmU-tv;B^)?!Oqe(8zQ{hbW}_By?S)8D){*q
z){?O228Y*W9R4P?(R6j0;SJlT4_BI07C3Tl+4A7vmj4OU8zujkN-~s*PjL{R(NH-3
zK*DSXc9n8Yjt1@ugZh|7@q3;wHS4>7q$9<BEoZXJyQP~RdHJk)+cs%h(Ic^kmp$iB
z{B<v9?)No?f_|1O^Lb2Xh2%Z|(3L!mspID-_wJap91lF>U*|X%n%C<Id$%1sW*Q^@
zo?UE(y#JiyFhk#GCu3SVT5tQ^%v%uXSfs2yA#Strl9Gl(@v=_&Z=LTs>X*1Uu8BG4
z^~yPpW19zC`F*xj{al|DcN~_lj%0Y4{PV@5YhgLbECoMYF8^$syhotHX$y<$`j-AJ
zNlRvlOgVO5Y{TUE&+gm5P5sQ?I9+M_X){)Bxkq=O<pz0W?kHJwg6IE(vpxmP+vDxG
zCrtiP%<WJ#`#@2-1E)hdyTRn1E0()=IV>u1W2ljG{pK3$_h{ka9q}Jb>NCvClFcPM
zlzKNm-=*ibbpD1nrA6t6?3u|d!P$Jl`={+q`r7cKWwOoOysz`L4{GIY*d<V`-C}%h
znd)hg2?r(0O(KQ2y!)Wnwmgr^%(AKbY3H)MlT)`$x@wvwWo@dy`*@Jz-8a(l_FVak
ze{9)a*?F3!B$;K~drRr^_ufa@)=y&-cX5w@<7=?Eqx&=4_9IV|69RlcY)R?YS}LCF
z#`$&0ZG|9b7D*G%`Zpr0`PltckFT1%%kPehwhDJdn#}a~CsR9rM=KXZZugkRUHSix
zUJbMG<^nYvugko7(|?LFEbv~wvp|_^k;<tAb*_ZRI}7jcEV{h2`25czW%Ck#wc^(u
z@v|Kta(wrm+F1BGi~WM)2Bmw~OJ$RiK5cu$Z_BpaIWB9lcU9zzx4X-CUy@o;<K=Kb
zr(I}O`LQ=_Pj?j6JWro?K(spg<fqnO+KYvB)(hq5963;vX6x{bjbqAjn@`Ig-kcTi
z*mQR5VeaCSw#FO2|J8f=i|KjhrQ5#GgKo8c*84g0^QQU5hwNFVH(6{cKG+dl{d9K4
z<+u|nd#41kJ;)VWz?^sY8Y=_Cgnut4|9d%kA47)2y)UjYTH8vub4<0~)%xVdcP^8M
zPb_vkvT!)x{Xb1*>+BU-ep}x=?tc4f&H4k=IKTNcDnz|@eRo?v$b^|$tJ%LqLEa$p
z)=TzNrWY>z=DF`aQ*v-7clk+ivrc*QdG_XK7Tce`zw?*=c83J(;`2Wpc((QYD1G>$
zhP|LcEZ5-!Lkklpmk5i-3I>M;22LTS1c?KQN7{vyxi(Z>d?dygy~IP%z_EcvRl;-H
z77JG8Q<J28xB1+h#O-4wp1!Lj@|lufw`u;RFOfm%-nvVoHVQeMT5@rcN3W4>)TXnp
zW@3@9r7O2&1a${4Pdbw&tTQ{zI<qhLdXcvfr?_ZCsdvyxHa%&UoEb(Ra^v<^|NM4#
zwnKh&cata6Hj7OviAS^lvuc#xneiauAd92l2Af9a>1SkjOnDKsh$rJ>s!xkrZdJ+4
zi%d<EbRL>(_}!ef^U5~shwFq`J%x05m^PZbbT9}pDasw#arg<tH0F&R2X-EP!DN=H
zcyf|Ta(B>{@2s3+zdgHU{!DvPxld*Jf*0P$-n@=;n0|<9lE+P>Z8dd&YkJFMTPNC1
z{;)Y?O<0!V%Bi|6#%m0xOXPGq9+)!2VZk<z+x<(pZ{G5E=CS>AL517+PJj!y<{Ss6
zI~E6Wx^H<sSkxo2yW?W3gy-y!Cc;(*4>!y1zA({U*o&j+guHc!zk9cjK!PWuSH;>b
z-A^WeJZfZhnaB9F$bx`I7QU#Ag&hsq|2q~MX&MDHFz9rB&}Et6kgd#U6Lng7Lr4_o
zk?sx6=as$}%;HqH`#!r<eSyQ2rk=2ULJlj<B7SB$xOoRnxwuhA*5FdNQYe=e<C;%X
zwA^l1gfy(Yp^&=D>9Z8K=6aSD%cn7!`g%86eO6j=m{&E}!Cdv4Q0Dq~UY|^lB*pFu
zFgWr+D2ySsRQ6iL+Zj`HHl$1M5i{aeKD^cOpkK@bqca(&&uqRJ<b2lT>V?#`ORj&J
z+#9RC_DKRa-?9U_O>x2xczqsrWWTsy->$4@`Qo3iaH6SU&<>S0!&&ckS(x5ZvgxTi
zyIGHsMU-J_-_J)!R43Fwa=F_3S!kxp<ljd+4ytUC`2X}kchu1t&m8r$ny1WM6l8iS
zEJyL>0T$JY54wvEN>wdb$ZAlPncUAh@nuYLu4eeM$bA~G(nXaw-d<;aE2xRrV7Aic
zGf#96^T=H8o9KSw*1n5x<?}^^Jj&<qIdNBQ<`qq^ow{0&jU}v=r8kRg|8hyeKs{>C
znPbOYf)m*UcVv8eeqhxeU(4845lmb4KFxS+ZP>N)(X7yUnrflZ%-!mfefb3JrW$rR
zaBYl{nh+ee>r`yR##yIAf*6(Ga)tERZO!-|9C>l?_fHYJA%PLTO3IvZmMqz?W^L6!
ze{;{Wy```FgIK<A+GPFvjd97@(D}97@06Zha^sO`L(eUf*^jnymU{nBh;L>3!JEKh
z?RQ{O^PGB9ci%ISlRXWN8=TR7qr{`!7q5~i%b1bKy{OrHv%#0S78hHS6?Ep#3n~-5
zdGWov?n9S1K0EfF+<Q{V^9EC3TAQF>E`NKTrRb-o>(*;_JV_V7=`wxRwgdYgv~g=N
zo)Ov@X1ww(yMfZGX(F@TTvsJ(8^*nk>fCg7hH#^Aj<(V5sYxkp(pOLPEKQ95bt+Uv
z;cJIS;h&2WbUsN{Ec8`R+wyp7T*`XU+K=9>FVDLCIci}nYt$H?oW!T!ZkXoj*eo-9
zhPU62=^nGx)!&>tT@_(&JNbaRhV<kEY8t8w0!+_qFPU_6<%h*yK@m$snU&@(-ty?b
zX3*Qbsb`ht>Z1Y=Kl40qleJM)On;H=QIACC=o8`JIwF1CUY&}(-+awPw{7AF-?Z)>
zS{FMHsp=JNG>m;1bKfF-`+VzT-P~VS_wFrF7MfV1$;j~K`T?e6bE>!m=5x8+a8-CF
z-ro3hz8L2VyPrw2+_O?t<W^oTZ;O=M)eyk&MDby}L)gWOUo(@ZzTrL<5a#l0!a~i2
zor)1PKhNyh@W@`TvrE5P%!Os5Q|52aS-xx1Ce6B~QsZ|;v{mPkV@~C2=czwurT+17
zJ9zX2uUQdu-@R*BEVuP~T5DaJJn4(n&pq$+eXY*kXZpTKyzp6nl$3|UUw4n=oiBZQ
zvpEl|xbNin|EkqjHSrenU9Af?vLVHi8_&miUJB?tnpwT+voP1mrjy?)K5b}KU309-
zb;`tflT_S3-gWKybYs$)O`BpfS<eMzPL7?{60BN#`tzPkU1dksY`(I2Ufv%kfrLoo
z^%7a$Ykh7-Wre<0ez;4V^^Ch;>B`gFml>+NJz2i7GBb^P`@_C{KaNX3Wnkwxq^ZF4
zIa7ww<ipysvx)YcT#j$c%>8F4aK^C;v`%`UW>a_fG{^DJC&joL7FQ-dJY~0J)gG7S
zVO?3Nxj(fx|6dZqpgGs8zD6S9q|gba6;mU9w(jtl{UqB?zR>&Hrih30i+Z~^OjYtP
zoV{?{9FIdTs<Q)gTg;X{+W6ljQ|0W0FrDM(Oq#14lP>ibce<|&d+C*QMX0sy+fqj_
z6`u}~B3GNNBTCn&s+7!?c{Z&kRwcP*-QTmpLZ4O%buo%5AAZ8&wZYg^pmOHMFQIE{
z#l2S;EEjM{&+}|}QY`B#D{*jc?$bp%5t=8IJhawYU0uG+>GZ^~r6E=8jA!}(n(`=e
ziQL}PQ?lASb-Ev@Ix8!RZMOOnmF@QI(AA{et#wOv4wk4dzx76G+1?#X^S+7Ac@k3k
z+ovgX?;PQRu&Cum@3-E$%X6vzvr2HW5JOAccPXobY)u>HUq0_r7v5C!a2><6weuUd
zrQAHv#ywTN@`CRa@ohp6imH-Ni}N3IStRlQ=Hxi5S@PF+n%k;rZa?}?IH<crO!>pT
z{#3(jT#V9IOJ`(to@ZgXJe%#b3eUd@Z8J_joL2B7+C^4+XF|v$BNfq=6>9?TtoS8q
z?)UXkY0cS*dXtvy)$~+1teSZ2*9=t$rE|v}{a#OKonv`=ug%Guy+6xm{z=*Hm$x8L
zZ3VNWMru!lI#<*R;av4s(-QB!oK%yjpK$tToY$i0vsH@5o+k>#s-7?3WOJozB}dBC
zlR-<TM4R7RlA)Qu{mGfOAC}(fw-qw^OZjL2d1|n7Z>XJC4qNT>qS+SyoA>>E`LuSO
z*_1sQhgqZi9k}-F+M3^@eEy}-^6L})`8Ibl&z<xj#%fx*(f=vFlis{_^m$%0vFxza
zwVc%HeS0JX)8^H9O?rPv(?(`N)3HOf6MxwKwUsy=!{ip(z4~`d^^xSm8d0t_lU$x^
zdwksGEy{EGaD(bP@okSTFWYg{$NX}$Q|J!~PX1X&i!9vzb2lFKb~aaw4E6BX<T3lI
zyuLWcfgZ7oiywIXJFY9JQmC2!^OPo2+=I`qCnxAIIG=wS)5IWt;-H0pwNVGpp~YXH
z9BEeLT%_i?IEU#aLyJEPYrvd7on`A<&*+>zapnvQYoMC<CbOmORm%c5I88N4y`b}S
z<DZ^QO0O$sxlj9)tW)u9i&5(4B?^qYmRGMh|7)76%DRp&wPeGh?iKI)xUK#xyua~$
z`;Bzd1CLB&d_Q<FUAvSS^X_R}RqDsOo(YRiU$61CtjpzT%)N1};AG9&lQVo0<ut6A
z1QxiRS7Ynqo2bCJ$bo;UgYXsyVWk6K7OnmK%~{|#(;1ydx4aI8P1Cb)Ipt^fTsQCh
zw+Gy>j-1iex}Xx%CX&ObnRaYb-T4buFaF0c9WXfb<6iIEoD)m;G>I%zVEDy!Y28ys
zXQmZZr=s<k9&THrx2YxW)$<&$Gd&?^a;+9TTH&?o(F=*mf%%v6z3%v%-SdC)<Hb9J
z$N$t0cn9be*1akG!9Vd*O6Q9Ld%g`BDSR@YmAE__+qw>Y&q@@|@_K&iqw|rEx*xol
z&b(H#`mf|SiBC;QP1(Rjex=`Jh9G9a)oDu(Y{|Ny|8Gq@AI~|JrHYKcL0+r0<5mW)
zNNT^3r(eSMc8$~2YkEO70g(dEv+f+asBl=UdR2`{GxI*p9oN*k?k>Cf!A<Qoht8&g
zxTqFx!S{22sLqq(w_sSIKHu9?CBJD+_oqh}loD2)G-%0`JEHOR(C0t=4Ei0i5ev1}
zU($+fJY~k8{K7!X@yM-BIhkS`*Y8?uqs6vJp-UxA|CQTDo0JaGn)51W`s}k#>SlY~
zk6BtC_E9Nkp~$v|??qhV?3<19KJGb^{=1LM?UMVGoF%y(f_*(lj@GFBEqmf|qhIae
z)BFhKe`ziK@6ug0{x6XF*uUn$1nHmtT6ZVy|Mxb_M~N${QMRUEEF#hD>7{3S@6Bwr
zekm;J{MIb!-L&n{V{>mN)g&gnwqkvc^s_gPT1aLq%;uZ?kEOMODK5zUtlP>dpMt`y
zPKRz2o*H$5^?lBL2kmoJ%Y{YHZuB{-UvwcjX{BtPTG+bB?7LH=C7h!tT<9)Yy?x09
z`|z`B>n`YSDwz9W`L(j5_=2-OOSmo1OmLig*l}j>O)Yg^P5q{2&Ac-gKf98`o89N^
ztz`A@Wus01^BF6OB+{%NT-0w+V2Vmz(s0}<b)5~{sV@d87Rm|Q_U*TAo-E8+6qlec
zbTDb(6<?3bXO#kurR03EbNcA)^xr!z?ZWf6^J)=C)gl`81$(%e5_F3;E<d8)Bi481
zN?Y%x<+%oa?mZLpIto25Y{_FuaJS0)`)gVLqsOZqCQJFVNLMo}e)IR&W^%n-WWJDp
z9h;zeVxGRr7yAswim-!9ZVxq+MO?D_z8rY8An3dbpW*r<ll3f5<sFyQl{KlJW48R=
z5@08)AiP?2!{-b4m_*pmsz#i;VE1}c-Z8ZoK}#8WwY$>adL(shdlI*2CFgD~uI8z8
zm?j<jw0Osx<8hy!&z;46z3!5FtI;Qh0|{4BKJ{GM)$vTbx^LZ{Pl;V;eebl)H~SJ;
zevTo1i;Y64a6@|t!(#vE20^~BZfU6r<{oWY^FLYmh0>%euXT^{D#!7+7+tV6OS>y^
zk~#XaoA6T4=tnl8mz~+pcz)E;Ts1{a<$(CqR}T;Pn<#zmJu}5f*VwZz<=7d0&mxw8
zPm~KJvQ;_RCw$VE_AmCAR5FzmyY(WO+a!@EXB+?aAi2LH0m2hh+LCm#6wO!<eBV-Z
z$fN1`v`dafmz^07FVH*Gbj)Glk+nh5Op_E=zW61WEV0q@D34uK`^JFdmg3zzY*t$S
zbYXI1tcX&2y~*#~wQo~oj-JoRw8?L~Cva)w5kLK7Nm^B$2Xhp|ALlZiNVxik&+4?Z
z!+ZCgJX6#~4=KuQzb<C7>(mFHm7<9!Cfuyzbz%*O>gUPa_@6s^{pm!t6}#KcF7r@g
zD!9fUE}ninYwZG~#Hz{K6T<@TKXUbMI(4B?<V1q-F^4@h7cBd9)7%ShE<0?sL0NFC
zS(}Q&*DIHt(=IB_2~RfoC{&nVZKWRgU|L$n*ThIuMRC!}ERTCx0$cf5ww`ccx^}|n
z@|V0R8-2AW`#S2`-AHxY)#Vo+_CF@9{$ywx<5vA83(6NK@VGH94ankV`N5xlUH{*e
zB_;11Cpd6TP~iFy5T(qtn?uyi_=j_Z@v}FNn{=GA>?M@n1h&L78M<+taSGkHsjTG2
zChm|!A95Bk+LkWjI%>-`EvMSQx^RVPnMZj=du7$}*DezSUzpkSGqHUAucVb_pp~O|
zaat17jt75#Iej>>fagR5%NB!3?vi@GDIaB+G}O(bj!P7;y=pLT(sQLLGTGOQ%EX%Q
z9_lqeUn0D~N+nTb!zZoNuhZ1qo6RFmt$N*{wLYyxua7g5;ja>t&Y`%C%vI78ne|V#
zG)+m==HygT;Mz3Zb7#5!tum3RkM7Mdv8i1t#w6~b#^B{tmVYUK^|kY>Efz5Qy!%(B
zBQ5G)|2@CrxA>1a&RoY7xHx8r9!soTQ)G6!TO{vFeR^K?tWPDqIlfvm6a@}6*<NRu
z{;hZEPE%!`nW6>LcduFh!7Dy~(RvG~N<~Y5lNWwBKCRbkQ%$l+jdDqL3e9ot)VT4#
zOfh#t$LC9m{WBccSQK}DPMeUlc~<-82KF%b?$_HERWeLhIoBkzNaETCiAc^6_0nc@
zsdAsXO+F_KinPU@WJPB62lX1RKUv_{<o0V<+Ly(5y(W}T%P2b^E3N6G&%~J(#`Vc!
zlUrj8r@`}GQ)6Z}$DaM_?9*Se`Krs$P61yfzMbBmr|_<yG`()lEcap#l@o_Q++kK{
zKK;@sEy1_6?1lQ9eba5GGp#XfDKA(a8pbs>Ze^QDPX=ogqgiu-g~GH*F~`t%slV69
z7Dj%T>eM%R*swMz=o;guC1EnM6*IqA1?-&UxXD!i)30gkB{yZ8uWQMCwJm{X!L+QO
zfr75f{dE3^R)+pqe<XR1CsW%&%g$ddr~Rt8=w7t)U-8{0wf*zYH#vtoVy902DztEJ
zNyk*B&Qgw$f;6SsKRP1=nZ%mZ5AX1qARX17tFdxx(FBH|`%b?9ba%~s<EyuY;jzW0
z|4R1rZd)yBkns>~SbCr&lkM}m$lcX?`u7U;r_9Q<ZRywE6_(}7ZNSld<n^f)%zk=J
zEvvUpJiYj<1IJuf>C<vQb<g$cYjvKQ#E|$ha!sx07L&?qxz+Ywf(!X|w7t64?me{g
zUDXM>==IKX(k3x1XZB&Z(HNM)c+}JJg_|PhrE1gJCl30|ntA)fgqNq%|BDzrxo~Xi
zwE5qb7B5lYvS>Q-_VGT=|83>R4{WNwzA5rhxxRFZ=ud(DlBYwQxCHXIdM^6*>(L~w
zW9}baly~&Bcud`R$MWo&&D-UBr+DnNsZ<YrJI(rc<ow;jNw4ozq%8|As0easozKcp
z@X<Le!jb9Rgq;@)ip%df-sYO+zN=VL_8Cud)47UbR<Ef1?Ynt)es)@4tjv+t?P1xY
zQ(|g+x$+jDu9J}7?^KT@ku45FPmUx=WtMC`T{Jgm*KD_<2C0>fDhsAnrQN>m%NH`^
z*=@Tc-8Z%`xY2+2#$?%>-G6nsZQ{>f(syXmJD|JBWslHPE>Gj5)9=K13V+jjv&kkb
z^y<euHhaSl=O1U6eYAVc-x-$Y-EQu^Tl`=6l5oOwZH{#Jh5T<V?#9kqc4fA{gv89n
z(TtmK&Ntc7@nM7AtK{|D4(yw_=j7xXZmA0oW#dlPmMc%GS#*q{IpD=6X@OJI_w3nM
zZGDHydYQsPhusGZTdFv1HV5BR{SxDMrBr*z-6>X2%ib6*S?8y?*Ib!Dw==Ny{<Scs
zdmGj*kUn{7$DXB^HfG;B!S-R7$wZzd3YpAy0W<&2jJl=RcDKB@wt-*b#x%LU)a#|3
zarOqWjlHuL2r)eTesq%cBIPNuH@DZ_e7LW?vvz+h(<x7-_sid=NB3`DxL5y*Q2n~O
z`&2G^oP6uNtw<y1;N4xaQ<PV?tHjtJk^OWy>fEmB88>Y*YW@q{Gkh|sQ^|47&d=YO
zF5PuGAvsT3>Bj^Y`E~}m2bnRmBBdTS*F1c`n^AFIx$V4@^J`9O*L@c@e9x^~S^a(G
z*`9!o|27M%tnS9Wuv>C^>z$`N@4UPsDHIvE`Tk6uHO6PxY<rco``O>hQ2V1Dsiv%q
zd#7f+e67S9dSY9MpvnSQ){rd58GDkrCrnzl!~5d&w9D6R6Rl(w1UMKJ7A!c>#KtMc
z(lkNwP_u_hPSVwNqPxSGSkxR&TueCDr{G+6$D=6maI>Olod~B;!>Q>U4M%($RMLDH
z_yYM-Is;OV&#^WaQ_=jK>@&f=^4F7_6VjSg{8c?>?s7dn%{P2=p48^8+Ts7##&7Hr
z-PyYGgdxX8u@LPBUH5Hyf?0x=;)ypm2|0Kfo(^YVpBTd={^Zhj1Mfo$mZdH|TNbde
z2yMGzbY`+>aFC*IxLe201!<=g4U>=DVM}?wZ|f2c)?5qD>}%|ka}=+3g}%L&JZ;~-
z+=%qV`^>@&0(}WT3hpV(i^ZLtXnE+WzWpkn1_dV}PR}JN(>Jy5o@N-feZozxd4GNw
zKHWLbUw@x&@RpQq2ku8ru$$TU`Dz|l<Tkq(@0fNRlWOSpHN25{LigCDMwzeb4b3fI
zkGD4RDIJkW{3u!y?;f?mNYZJ3p-7UWL}Q4wlSHCPQoMH2k+yWsrY}j-iwlprX>Yp`
zob=y4nn9I8V3%aukpLatt+yunShi0PNr?%$m_A$M<fSYP#^o`=vMXDJ9GO@-XH7L{
zED%l&@;Vtfo%QOHJ8n#omCuR;j!HU;1ZO5rjY#Me4U4FZ6Ah0nSgNYw(YVfZMRl8+
z)M5`o5fkx7CLe*IIB%(A3jQG-M-DC(ZTfb7(b+;3M-l0+8IkVO=C(z48*HnX<*}aY
zKvv=^v5=@PnWI_`x{4ovEtt80?e=t$*;|}uGgwc0G-1OTGwD^E_b7S>i(GeI$Xb5q
z<np%_9}7><+x~j4y2Hw{uch~QK3>-SZr95u7d@x<i?a2COD%uhS}N^RC|=KGAeP+1
z&*iZ2fY>LGg8z=wo3^lu3whnh-_PBwa<NaN&Bf4Le0@Yx@3KdyCU%`zR8@F_Y0r;!
zktW+5PxUTU%=WgGHeZqEak_{_WHyiH7C}FIvFnoyT-d^=I67E7ztkVcHG!dgaYUGs
zCg+u$Y593}%fC3Oay||4wCy(e;*}uJyUlM`=h;d9+M=pcyxp}`o?VnW`t!=1pup1Q
zVHM?5x6hB&j1pMDxcKVIMXkHSD;8v^>aBRxA|Lf5(&p;w5)sDf(HGb*h?!M9oAUE?
z@*3s$Ww+AjzwFXVS6FsV{H5&P4DFMwGy5*IZ{tkxo-*h8yh#PuVpX-ZnO^Z}ZMfLx
z^5?3YVQFmHwWQm3#qQ^2&-?%7#nEuv+Ln8*aXUYmzW83JGj*5Z52<VFcERf9QxbQ}
zOgvw=NA_*1L}Ty?e$8fash&B_sj6%2n`9PVXklx%=eG3GUoPx$T4cpc)=4@g8=6I5
zxv5z96dq}uqItHL<-$y}4UgO`x$a7|ZJE3vwNpQonRAlQ51l7FC-{aJclvg<tzD6;
zX?apwnyXq!$Y3LjYU-w2vu`~)bp1k#TH&OwkZlV%Up)1+t?E*`zpTUd&!7CdFca<?
z0ntfc1hrzLuUyc0R5ah9FS{oph4X=hlKsg>m!R3+6P_D{TTU?WaV*oh(fCE<crQ!8
z@>_?wnJKO(!h9z*cbNFFMy5;)iOiX>;s5R#>t7YyJTyIRap>feLY<SIH)1-KJvQ)_
zExEAgd8@na{aY^ELn7ye#`(v!eoWe6_I#h;u@l|(Cfv)ljxRV{=@-E!6=87N^vwQ`
zsR@5J%VaoxpDEj#=dR3ku`}Y)fr(q!h?+13W~OLP%IVXa{Y`*VvS_Bt6V<mZ%!((t
zCWde6v0zcR)=s>2&P-Uv<*l#Q9~FU(VFGtudb7C|l>)DwJlv<!s^FP?yZDXfat)8C
zQI*Y$T=q6C?<zVPP*{2T5zq0!{Tum<<i2LOaT{?NP5H9;#b#CS8%~-<QpNiJlEqkA
z3@*njJqez<d1azf!Gt|KK9&gR_;jbSDqqmK6t;<b;{UWJFD0*e!AlQC&9+&1YmG>o
zk@aoEr`n+>LnjxlULBj8TJ<g1d)^A6xD?kDk*AlRvG{a)M!{X*$jjz(TlY=XPvt!s
zb$jZyyLVjc(j*f+N>+Kd90}<?_2gvY$MPF}C)nJT8r6cmo;YS~T$i-rGLLlMEy4Uv
z2aV)&Ty;yXxK0<lvOv>OSI|~Sr8MfShm%f{cipt5n~qM<4D}LrQTVj@$~hljgC$Y3
z<_U>f`L<doDKcyB*>z`<S@_N~y?LG&GlG8fEUJkPnNxbzDOPXR#-7Y4%Ql~2-&Ya)
zz4p1NLzVL(v5#}NuhZZ?S-n1E&ylht4wJIpG*~$u=X&@hf${Ro`qTdxNm&2tKH&3H
zm6N$eAn2xFlZ>})T9W8lr$v2-0=~{zH^YmI(`M=_1wKO$iA^@EQeLlCQ$1SZxzR{$
z{f;HG6<EVlZR8}6_w6bB%I}rPsk{AHX|cn#3H`1%-uv%`a|tuNNoY0qVSia3&#uIC
z;8(c&G6T&ylYJ+oNM<Y&-1;af{-1<)-quY&$^;efsT`i+9Nk^zBc%A2;htkw+mbua
zKA*FwNzA#PsX8e~Xsff#$#%}H^G@8U9`)}!-P)$G?7G^Se<60=5@p|)p3%E5b<EjN
zWtkD=5+`na`D@k+_uRY1XQ!`y-#mSl;HJ2@ldkGMkUqS!REI&P-9I+%p2_c&v#<Vd
zU9x@8Q&;VLw^tJHwweZ=h!c^Oce~NR8+k-9;6#YQCq)ml(<RfNo_^u-XxD^^Y%1QS
zff-9aBr46ec2q5k(~dsFb!*4%h_`Lqci%qS6}Vx?z3<!1HD8@|tZANgbe;A3u4><)
zCgY{+Wlyd?bUt@l`t9wLZ=W{K{~~?e*6+>+iSQXllFW+S6B!Mqi!7Yh1?$KPUOgmV
zJ@=K?=@6l<GMjZO17Ga=d*Z3m6Su{+pU*B7+IIO?WqWU7pPNOH?!+0H2f5i&^WSQk
zE>Tu_87%6skoif5b*$>HEt198%(Hc#mfgL3DtCL)Zr{(dqj!gG6i!hxzZ>IMzw5F}
z>--OV+wb*V`G5D?oW|W1yz_#}*IHd~e{^E^r$s5gS6Ch9P70Z58hWFzV1W)p^$Z3P
z=Y+XdbGR*Bg)G+xabL`1d)v|@ntr|R)1p;<>Nytq+w)o9Ejx6_L@w=z$F9XG0>&zg
zi@*4No%QWq`R!%jk8#<3Xp}!VVP(d>6U)tahadZW<@4{$EAM|PU2b2%cWv&qx7HPX
zk(0vTXZ|^UyzTX`l=%zH-tiykDle0refFfnWYvVDS7J^kUn!cf@TT$9tq)$t?q9s=
zv&z{Y?Sfwggc?;?Jqv}b77MJN+7Prg{f~&2%E15w6;V}3)8mC1?bl>{#btOz6#^Am
zqb9^mDwB@?mMR+9q%osOBSZRsc9^trQknh_C8G!<6OXdAaDjizJ!Wnz+h~-%E3JI5
zS?sH&<uViFu77V%+gMPuuw4DB_?~C6pH1ynu*ElBFMkuP_#{OC_a^b`iQ-d&1h%%z
z)El-je$jj*p5$~<qdr)NU#v>wxX{w{#AU@HeJ+w0zqchlt4ed^dg4}hmXV=1$g|cl
zY3FyFrw5Y`9_QMcAid~n$F>I@y*oPM)jFfw+9!%yp9?YEAylKdSn{`+wclmQw_zcv
z#~fF-OO|WIpD1imx>md9VV6Q<dG{kbFSU*nf|U$n$|B$GPpSy@l*YFyvPv1$e`go^
zd0lYojjDMQn?)w;%rGrB>BzcsEo6;@-v33W4QDyp3`(LF7^<AUl(A0Hv}&63Q9;vn
zTcZ<{OHCd&URPxGR%wbWZ(185_%?uBJ){3~NB<(#{_heVn%6@brzoj!O55})lJ#WM
z_D_x{7EO@&sI+FmglU0gmNS|cE-bTIsHLuK{Z6&l-J*8Yj24~37PIH=xs2T&8BR*-
z46z(-T}MSuhY56<vl^asuQ(#?{H(6Iv~RDfKs`%fv_meZQI(Ql`?BLwlcsjAZ`X<r
z$lJ3~=g|#uhh~v$3uViKllPf-T1=YSf1@&5vNQ7J)Oq2#3KuI=UQX@y@K~_5zI#*W
zvFp=pI+}kk%UtDDEcbGvOv<F2$E7V^)_yV7Yc{U?|NeXJrtR`ucSyes*JNw%e*Vn<
zut@#KbO|pNxwi@2|2lfISGJ$JVI=xY>(U7k=`BXd%N$M^HB9YP+M!&!VyBSbcY#M5
zs&>i#UeinLKKJ?yD)5)8Pn_2D#iD=D&e?x%%w`Ihx%8yqo}~Vxk^NtHXg6+|Gwo)G
zf^eEjkhSWirYn(BKgB01CN`USNEYpEUcfF@k(u}AQ1ZvEVF%0Xf?vp$h*qpJ3tGKV
zUrIro$v`}&Lm=*i+xm%hGd@qMFLwVo)&4)DR+dC`{b7}Dk9+2cJLZQab{7@7Jni&b
z+@rr;d(DO^b0%l>7KSdlB76OC>HB9>V<i_&;Fz{&)6`bQX}J{({_hi-rmPh;t8m)X
z8M!V$`;VsUs3>*KWz5eMuSz|Z)OxV1uA^LS=k#0Ct>0~Cof0fylP>XfllV(<_g_Wf
z7dCb87IEM{<m7hT&D(`5_gj7clcKClG10<Kk!DAU6fw5Q-g7sX-n_vTeWAhbvVYKY
zr}-j6Oh)P<it^`NoAzWjO}^1zAJz1|L%?cz{p2f^tbvPmZ(6a>s6jSx1*_nMgPd~{
zpBWmo`|=*q^7v`ZC)sQMVx{QL#o5QDXFTk3s+`0cE+IZOZs{t;hnh*(&E>bU*~V)a
z{%4NZ+GWsIIHMu4O=qI!-$RA0jloTdm30?2yA;>#e-agQDLQ=e>b4g~;>K0~HwQVc
zPD;$o)Visq?5UZTbvz{Hz?7OsmRo@x_ZNDFb1u@=TBNdSDnHxGt7eiKrvyKr68bo6
z(Z^W}CVH&6)io_~RcdbMlv!+DOC!2gN?9|Bud4mHD)d`H>6PxO%rjJ0PS<?7;Zw3b
zuk}XVm8)gCWl9%wW~KH2KP7&FvwfxT!m5r<%G!P(jHKRQ;#!ia6gbIb|Ivg8KLRd1
zj){4`eBaI4KW=RPA~EM6=gRg|>s_Pg^!!@qRlUB;YmVlu6+K;ZoVDltdOnSDc00?>
zrlT)rF31WIvs^WO;YJ<r){Sjp&p(H5Ez_%Bx~;lAtx8mKy<pqfL-Y9`=^vQ2agIyC
zO9`Q4Pp$t)Mov7>S<G`v=(bzE%|^qsM}$)xRL>iFEiLJgkDB)8*VHMe*C__B|IQ^e
zvwQvKW|`Tmcj`pV@!Pz};8)wtFp2Y5)>{7RTs%WNUb;I;TWC{79*?%r*6&kv0;~Tl
zPWe00puMzh+N8pr8cPpVYwpfi9$&H1>xjyhM!^v2l}DeiJ$uua_lU@>n=@~$&N2+<
zpJJK!u0nm2S=lzm6~8rBG`?E?+hWV!L&E=76i-g<e}8(X>Z`PEk5}j$@9(wR&$@Bv
z$5$(u9;7Xe+!FeF&&6&f<>r-I?Me@?iHlTDG-Rr}f3kdqR&D5ta96LKC+6`PjylTA
zCuvE}Z@gKNFCujMTA|Fx`2PpFC0?6X9^<O)Z+BN+IJquFAXsyLf`@t_gY9?5X>)fp
zFY@Y5_+5EJE3x)qmBFT!dc`|;U)s}u<4DxcBMC2$1pV1E_w;@R!<n)d_lEOm9o~5~
z&SyQp+WJ3Pi&A$kP_I6=Y?E}(PG#;THGji4$p6&3o}OIgxkvQ*hI+}}3l7SD+bMla
z+kOVKCX>T<larq9TP=^D7T;>A`R>Mv@OH^FoJUT*SU=svC^NHOG`QW;BRqcDGCq-$
z0kc<3?wn)yd-loBEmq}Qmgj8gmfF8?&x-%M_kXjtz7xHlBk;8T<^BJ)r`_p3{dM-~
z9d}M|(h>6DPCu3-;WM>U;-TZ^E5~BAt?sSd@PD~h^@pdfJf}mSGJC%`>6%t(Clnwa
z^})SD(BNyNMCTcy8NtDIiE9o^M?9bIzI>_lMYEz(NA=G;RnBa7iQW*RR;3k}o$Hi3
z^U@Q^&gfZ5KaZ$&E#g{xBy7)qFQ1gyJL0^54OM3?P+?lU)LMJ_ZSmP$X{)2B*z;W2
zq<!!i+miA*mZGO?wLBH%rma@^80_Vyn5Yra#JOk9!Mb)HUCs$hbRGu0yU`cM7|ZJ0
zp!#}GqS>;ljw5T0&xY<WjJ>rlV$Fh6Cxwr3p8oQCUj)zoFTeNyh!%)`yPyB8$k$s-
zBmZ3Z_F?B-?qg2e2iD)Y*3P|b>YM}IEsmF4nx~zRy4H1_GhOfh!Dp+2Z(LXX*{pU}
zSmMsv3q|sFrMgT(VhS=sMb?Kj?}!H*1e<c6KN@AJU$dq{M<{BOu=Djb>l&*)1I}yT
zJg7P?FVs+O{>7=Oo*sH#$EHPCEnc;$`0m<8k%kjai6`k?ym0%r)9fq9?+Bj%efx;6
z&{WHXhm?+;(#@N;OR|mS(v*$Inxqe^%$y{!`uOgLQlFG%f(}Y0TBOWNzU-s6=gg+d
zdb~nYHro^zZ$AF@#6-0_N7Yvzp1Esk*4~Rhs=qrLHmaQYW_^0oowSd$Py4^#|99{0
zT#JT3yl1}edXPBhnqkvUzH_HD{-&Ljyd#(^Ca~npqLasVUY=ViduiI0Z8y2gomV9P
z-*n8Pa%EWh??B%bcHTQUOFv(hdam%b=-xS_;Frf&GZ`ne?$LOZTXI<OR<)tU&5MFJ
z3ZI--_FA5s6dEYHo+bI`wOtqg94_4<uYGM!N3@}M-`&^JI??B(GJ1PN|E7r)Pdhtl
zm+Z%j7jH{mxhvj$)^l_BB{3PD2t{eHU4nQ2N_iD54gB@^Owt~{-4P0jQ8j|<HK&Da
z(q)gRn)mE`aa7fd@9AYjyO#<#PF#0->80}W_}74U5;=Zn_r}Z3N%T9KwsGeNE|1@@
z`~O7G{9AjPsUdY8-|KCQU#*gRsKj_p|MP97*$V0}mYyi`{po&v+uG|2df^GbGw=2!
z-}h`iD!2RYbA|sY5Bh4)O#1OsXSc@*g~qq+BH=ufZgy-vcX*eYwV|xumME8?!%aFu
zCug@YCCCa+V0g#Z$I0-%w~fK%<2%6w{}1cl|Fkpzljr}9@56r+8CT~IR~%zFA1Ii#
z3+_BCv@lPC|GcFjztlpxtGx4eO5MAortO$zvo~|!#oEBy>a$PRcgSpA9WGt<oZ&;j
zl)!2>p%cf_3RTu$_O^N<BiQlOM5K4$=e~2F?e2ee&^ND6EdI&<AmOgyW}Wr_&hAt!
z{IV-}{hTFt4&7-J^}iE6@8hhD1L3F7M4w*E>$5}r+qB{zyBv9vK74J-Z*&utd7d-d
z_0Tcb*O3d3J(2ky`L@Pz@}Z{S|2O*V-yE+K<&OUMMaC-m{SNoHZHglGPh@`D3H`nI
zZo|zV6ZAj0CcmG|Z}K_L<TD>b5JT29%a1N%R`cX9&g-A))O{z*_toadJ}L7~YG+S5
z+^3V}_pR;YvuP?zd?sGnuv;VVs+^Xk=&c=s$Dbao39R-KsGs}j`30pl3l1u@oRjL-
z`_nVY@@)SL*)>Z-GlEXn^zQ6?&38`7ul`qdti+~&is`m<9)9`jZvQW{_8M#8KUu}s
z91H)g=ldFW_rtn=9mB^T5A+#&FP*@k=<vXyF^5CW#v^NiQwy)Ok<W{Q2P@jy%{gV9
zKP)=XEyCXSkz-PVTaS?MJeQM(OAiDJE%FeYq4M9+d4>?%A(b7h3x%43mvQ{q`DuY)
zV+>zjMn_3P0G|u1#o5Ry3j!B2&*odx<<Jn;7{w^_$7=F|u(cj6XIv&Y=nBnNKRR#g
z%5S+Xo3qbfvh^0ozc(|VeVHo9l7$bNh4k{2m^V&c8^si5nxh`_Ddp5u-Qd1E7pA2#
z&Stk0`ysLA@iFyCUK?M7uc^o9D{|H3PMYG*D9wM#Ez|my*Io7M&1`ai5`;K|O4NR6
zEm+GiXHq5C6d{wPelJ5=BgFG0v&t57rktAcVtVyc746bVl2_BdRD64Tm-}wc>U#&O
z=BZ9|n-RI_#rd^AyXABbT)UN!y2XsosAB<(YO{t}lZ?m$#|QrlCciLm_YhK0<~2T3
zAj0D<)L_PKDwQ#rKh&$Ur}5n6BumZ%$$kp6r)B$X;%sr&a%aqY;=r(!<%QsGO^XTV
zG~a~fbgX(UBc!jof0A{F_LC>4xE1&BlRmLVZS@*&g{?_K4hEZd2|du(U^UvTxKZW8
zqHP@fT{*VjzAP3tV*DViVPvCvN%-6~nM?jgmd=~c*!^bN#Ja`hx3Iw$+u*3&t!+|s
zMGm!iM1HXK4HU_<_LD6-6>QEbbakQc5|2f`j48jkI>h7GY*bv76zip=w}hF?bLE|4
zFRS>}T#Xf<B5n&Qez+Hvce(l4oSYk>rI%KUUdURRzwaEAEJt)T+pV&Fx&JvgZ8NI2
z`TcD{(S7Og1x1&AJMW%5q`G3R^ZPyg7p^^e!yn(yWOHF-ox1Pa`##~53T(JMm#i`G
zvXV+(EbJXpwyE7-DwTW2l8no{8yh~Xd;ES+cf*eNF_$E4SSJNMuxu<?E2{Xi&gRpx
zg;h2$J%U&l&E%f7^ifOqsxr0g$Bj%9FV1Lecbc`~;hr_#d!F7CI(_rg0>6ex)2VL3
z8tYP8Jk^iB=rvE;czhbq?LQo|z6UTyYxd@D;{1DlKl}X_-QYW1b=LWNYo#BnKA!RW
zk~hOLnLP!E_|0uTPOh%fI;lGK&h6A#Wq$onpXRU{7q{Qw$u+6eWRJ~1zh-Lxd3W8#
zC2!CF53w(}(8#uqC%@&xwh)WkZYG{8e9kH!uAO|_&U|lfdB4|HGpi|3L4hGbj)R54
zA>rr2x$SYkSk`Q>E0`K7=Jjl0O-#}A_bZvFZPnc-m!aR;<2GMm<B`i=YY*-h(N9^%
zx+$LF>&&a4s+Ra>&7b<=`keJISKI#<V%R2^yp82nr^33^!FvTZsk54WDACxdkj-N{
zVT-1h-%9I_B~8wnPOZm+mK+et&S2iN{oy)Wy)8b6XDGL7MDT>3lsX^YVK(<f;rc=j
z!=Ten<}RHHc?zB@PKr4k-u8=Qi5<_Bp6v~r^*Rr9+fClMNW@o2uVP|%G}8tpc29A~
z4W>_;O0%YH`Sy9{|N5;T{qsY31;rl;7#+Bf$T9ih9kZi{+9F;g90@-6F=^6%1Ib>W
zH3t<P>z+(;O>$MTpEc*KK#{0|oXaF$&IkWbZSgcZ%AXk(I4hj3X_|5sCugnT8J{2J
z6LinAWY4*x;>a^|dZO!t{E}jxWGM~zxU5xva~wm`Eq)w`>R2{o{>}$k3QA53SM=Ez
zxA1BQb$Dk4ZEl@>+E-t5aqFi9pA?}PCv`S2Zjoj^+J4LCgnwPBUkRIg*3vyb(c2u@
z{y78&KeytvyLwfiHY`x%@1CB8&y3yLuPl;HpTom>e}^2is0Rc4FL9?37HhV%f=-*1
zALW~=2;E)LQBfn(@PQ#y$YI|9<*K?;Pd9FAQ(N!AKaJxd*Kw8&hi7OsgiX3Qv9Bym
zWJw3V;<brSR0B6Tf3;MZ>?SJnZ(G`lq~1veCp4qt_9>kT%QKo2dCFVs@R6m_b!%G+
za(t5&IOZIm!)KpnsyzRc#6jDWo@=tdUcTwK>0p9YO5h<Ij|L+Pxz;FQMUh00w^B@(
z%Idr{#5M)2^x4F!V&=!xEWOmC_4199S1v1@Sp8PCyqNGRXWNFib0#K79b{|0!an~m
z%ZqeDp<~sPR+oIOdb>xD*{ND?gJ@pJivun07rP?V%o=PtTLrk4I(tHl*k?zy)@+vU
zlHpfZte&*2Nr9oL$bYkT_me=jPs;CqUYYoR_bjgpaqnf3`%-uPnw@&VX{B1lKUan`
zT(cr?vETQ-w7vGDZKry6rFW)gieXB~+thETPCFcRpRuV`I`fg6WZI@<4*Xud35m`V
zceCC&WLbGKILc)IHixia#t6PGI+jVBG_R?=*;DZ1)*@E3JJ$@O=J?o0zMgPp9n03>
zKkD|{#>rd%ZJJpbdpBJEmwG$bl7n1}dYn|&MD-S}I>}_1_x_#6%+D;FEaJT4g6?u}
z4cxaSRBfiG()Mbh<!?khyqj+NPv0VRhHt57XzwepzEw~EJC-ihOpKo%Xq5i<?34Ux
zhP?RKPp>u3`Kq$~cY@^otQVWatk$%%X**u9a^&T^ysql#e{Bwb<>kuTm-F%@J~NSJ
z+9SH`MwyV{RWZ8~!RK8+HU_Aki&U|m_xA1^562S!+4ZH5zL{K&Sf+QzA>@kw<xTC!
z7g(3=-tqKktywg~W2F^olX4|%7JghU$m=VuYI^Ooj-}G()oR`g=J>YU@mn<eQm>zy
z*i*jTMDC-4Oa4W)3Y`4)D?^C0!S}SKrQ5Xk?b@I3b?vF1YxZ6(+#$&;;L?|=pE8y{
zjE$P=(p2_*<BsxdNA2#T_g9}>ZR9)kZS7N^Wye%QikFD;eDulHHcq(07`?VwRxNh_
zuQMLbTGmXbLULSu4W`Vf2$}A=wN<j(W|6@1g6OHHHz)i^kC8n3T(AAt?f*_TQWA?*
zZkp~(@0tJCHQoE!v9uheMa$N%bm9JaaPdo%{N}|^3jXJ9Xm7o-qV~DsL$4C$QW^79
z7p*rFqAc&p3-<_47H(J`cJgq{s~=v!oj$#qrSd?ia!()2qA9u|u5)L~a)=t87yTM5
zqd0fJdBLRQ=Mm4!4Z^(qSZwOWG*#O*ym)$bzc{bFd0nPNE538#Th;43Zpo=~tjkWn
z!V$(;8QtG>>wLCK>&if#n`TS+HgMm}-S=46*6ow>-=LFhNmKvse*RcfXl8iQ`xW_G
zTh<14)J0_XU+R%njCiklE%5Z4wX40KWzIhTyev|(TCXr5Q-#C4N})|)#`cK<6K?Ik
z^YFiQPs75K9tZpb&ELP?ttC9~(FO;{S2Kcxq|!5|MsoLko+Nlen&bUPxrLwPPBDv#
zB=@dW<hklBmELH)+DR(>)~4w~vz)DF^$7E)CyFUwn$u~_IlG|i;p*079O6#J8$1QM
z?lx;rez0vy#i6MQ3U|Dvy=U)i_2PUS&B@rQ@$~l2eW!O9MX&Qxa$fM+{M1Ip=es1r
z6lE;EBv(9`*OcAAT-*NYt67E#tQ(_vz8_XfE#|O0Vx^bW@%6D)@kPTK!q&{p`>Z4T
z-4E_%o+N$Ma5rONk8<#CogC{M7N&=dI`1E_HgJ)6Jj*U%&&<NlD?|<Fap!Q)d@!Tj
zNa=a!=BE$4_f-6sTk-KgGe<+j=|1U7n>P!Xlq@aI&+1XGly8@?HTB&4z@zVYke1cz
z9*!NZlRFqTW(a=?P+s-9Rl|leH)ZXg;ssL@7Ptv_@CY5BT;am?Yp<@02!E8a;^)=(
zoH!kor}({I{NNFvl}W#`41f1UrpE>Al&iKbR^GqIvuVD-#%ANLSZV2)-{zKAMC8{v
zzmQ-KQfOAP*7&j7-Z_WA=gRETLfzlR+lnVzwk%lf9&%ipWs}e&@v4yB-dCJ_uN(=w
zJ@;Ycfo2cA;Ef$Se(&_<*tKWziCHyV8J&EOE$1!&DZyX8{J;a_Z#Nd^&1yI;eQ-sH
z@f#toqe)z+ojB(UDqUJ_F<Ik3e_J*OpGtpQf>`zDSqmLcTSu$Tl3+EHXbtn=y<nj5
z#F^vz#bX;LZrQTR`$2-GM+&djk=4xs(*G^g%0;9t51M7}>b=)_qE^JKduG$Z7N0pg
z6wPd`mQ^lg2zG2w>HVO6^q9-?BOJV6wU5|`h-rDPU2o0d^k$b#w(pM$ac1s)`7(+g
zAsT+d9n1??c$aX@J?*k1xbtwq<cCbo#lNSpf00%7?%CTcR=#`2A|Kc1T74&Cj`ndj
zX+E5BpxdH-!91hPNy{f4HIVf9xMce`jf2L!mqZ_4cIrWsT42M)#VZ>V-1I!B>7G>4
zIC%c|PWcPk{xUsoVW*b(TJSb&@P<XWxL<ajz5B`(Ph<BTK><GkL^eoj?_G3UrR4Zz
zX5a0?+j(U<bT`c4|LQKFx+^Su>uiCU-(6(ZsaTeH&6EgqR6OK<>V{7z2M2@VPZm}Y
z1||j_1_lO(1||na29Ey>tQ;aQvKBZt=kjrt7;gx5Ze=>bp|fy<$dPUd<E%3`-aV3F
zT*G!LN`1n#hEV1}sgRu;mwNZ8h;~U!1UsCaZc+T|j79JQ4aUcAZ?9P7B?LCxvi=d7
zQL?gO8S`W=m6s(i0vVk|ukEQcT;i~P)`^mk8-9zfFO<63aOFin!s^X&_oYH^Z&TG?
zU#>V$wbd+nbqf>Y|EI4%tZ@iR*-@}_M?^w$Yk$0wSj4I`2X{5FMx3cJG=APJq~E4?
zINTw#b+WJg8LNWHPa30HZb?~GuML<KvwT`i`lct2({$>cQ_h;Ky0rXw?Kj`LKg%!A
z51O7IGX2tyqR_KTv&(<hBwU@ore5g3yvCLqw+HHh>w+UEt9pjXM_(0<IDbuJ>yk}v
zCN+hBzWrxl>kw8?Ir8jABU9_PIjPfRk90b+a=s91W;4}D^y9m_ctXINh3)ek4#uts
z&SMuX4J}k!mNBLDc&9N}30tZvpO=>9@}QuLqUM+Tm;%g&JuT&0X8USqeHH3t>RZIB
zy;a6Ux_;Hl9f7e~nkMp5GebEi#zZ^)oEbNL=Cw9grYVkG;<H_rDM`+ma#1Z+*y+nt
zM_x6P^mKj4%8A+1bFW>9SUq=1pwH?LW_}&2S}dyG|75?&mg|=;zvI@Tx7EvkwT0`W
z@^hB2=1w{7d1{VhzfpMBN{LjiN1jf9mt1OBTKWIVY=$ihwmhlX{L_gme$9tYQNJoF
zkvG#8h-Hhrh%L>0;E=_2fpv56foT_$Q(e*$T?DuS#M7A1R=#yek#d=sb|dP9c>Kd}
zS6BG0y;3HiJ?nth`?B!4JA%vA(?ZwpJEI!9e&c_?EG;+N-tAK?*cVp`9rEcsb+1t6
z=In__8uL6Z9~I7T(U_6kbBSvK@7!(g+@iFWq)uJ4LUdA7UE=R`ivpf4)i`5uS2KOP
zBJZ-bX?a?0ftz(*k1=j8nUu85@BY4&&yHEYj&E{U(>v$lrj!W}XElaTTHd<!b?Ej}
z3m0F|&<LE*m9wU6QQVRbH}^Z$mfe~+$tvcyMd#kaqb1!cA?G&j{D1Y7hPM1S(X2HL
z=e|AcaNoize%kHZ8aD@F-Q1mAwi*tOk&B-OXy&TeUf9}ctGh5eX#eYV8)o%1J@IB_
zR#kBmIm|J)vPfHXrG~d<>*H?!*eI3aHD_HuzG91IRbIJR<js}PpyFAwOIW1EmpG^i
zt-SrUGj40uS2o_xy#>EFDl=Gbxe#3UbL*AKc42EWGA6XGnc#LWAnW2*;}|L4*Vo((
z`H#e(e1GZwKc8l!Jk2G%OC1xW&5o<hmsBZYN}4?TRDz)W&K5<@M@^|em=5&3I3Irc
z)6}Od28ZTq9<hs#_FO(EecLA9HLaGNv$%g7Z$A>Lqr9}aLyc{d^O`ROBC`bi1^y>W
zC_i-ZQqE=$Tv=pc7PclyCEy9Kb(Uj))rH0K;cMFMUtQ$a-Q}<FFT#8BFSUlMKe{T*
zANoJLk$Tu{*AfT5DRZ9h63}gwSi!@V6e-Iaq|g|Up&)cH!R?UhiMdl27`A^|+-27!
zrdAus;M+7^qDx#kuryumyyv|_6*rl-T$N6flG#p2m?wH_rA)GST*~0H^HdOD%hXje
zsooJcr@uS$$zrkymu~L#-Cj(s<zbDSz7YrclIHcxy<D6mUVZRrok1|e-;Oki#g-G?
z!-{K?XDF$>Ozz&q@*+hd(KXX)kr87<a2hAmlc-Nmgi5xhWQrGZBv~*zw(|I@e@~g^
z@FMcR+Tw-WAy=C^Uidr~VD0SdwB9(&aov-3x+#zHqy%sCDsMg9)yA&0V9mqjb1pjv
zm2O$Ih+~n5rB*{1*P38e+n0Z`%vbG6JwB;0_>{YIl^o;3RR!vatochaTy=h#bf?}~
z6rgyizfnZAhn@4WYt}=<2~#hc>=EL6$@1dZs(<W#f|=8vzlu~_Wa!6mvCB6{Z1KT$
zyLuEJD22x=wM;av(witYu_Kjj*IZwNAccLGPPnICUb3xZ$>LDOrH%=mr^Eb;rn6sc
z(e%}t(3j-GEzr2eLBTt+)uPOD`dvRE`%{O{MTseAI=|7jJ$g!%@5wstz<%EzPrJ{G
zo8`{*EXzs#J!$cOyQ>P@L_PA1Hyw@booIH;DZ`E7hU1<+OPZhAaGYX09+O+iy7Q5Y
z%&BKr#OyaF+A}dGNPj!jDl|`beqBy5>*6Vnj10c3J5Oup7@5z$!4SOE?&ZQm^CojD
zaV^-lR%dETaF*GhcjfKvN7p-UQAqeN6`_!HV7;1=;mN;B5sLqGQyZCU1dkX+C<x2x
zHgVf{{pmV%n0498hW^E?UNyYv3|hl#T{X*fewE-XuUdJxJL1WA;+}MWxpDHIk48h^
zmPaNfyy`PboI+RsYj^05J|idn_&|0bqf%wf8M*HqGqY5An{%#qo_&-R*F1s2q@8n9
zUtqN$w^WPB+KML<rV|hKY>@n~%U5=?<Gu9+pA^Q!CG$4Vd3`3`M*sP_lD2}mjBB6I
zO>x@Pe(&>aTh*>W3#**hk?nrHFQd|y>Cf6b<82=62cNrKOT^8NomjTeGvsZ{#)8as
z-nwrVN*Ql1)O#Irr6~FC)E|y4JKvhDbBkHKCG5+Y-9k+79a?-39SykP>vn8f)2&IO
z&0HtKmt-_&|44BxHxxQjIWMQhS2MwZ`@rpcVIC@MM(;Fuj@;4ZEZ^j4n%B4>n&Fw=
zb_S(|t=9e07uu(0dN%%8<g=EkKx5f)k@X%0^Dm`$vvn2*it|mBl0UMI@BX<J?zzeJ
z7m`JnNF>Le*}?h8XNpLi;JaPDXIL$YN~0$If4#|R8Bf^yD9K3XgLfRcoMMY2RbqJ`
z9h$!1M$kaodq+*8X@B&_PL2mN4>x@)-Zr(bq_H7d;qAQdbFc25SRVi6ZiM}Iw@vc|
z*LkbIn%Hq@$8otAr?MP3&XRCmtQf1|6>M0$@#>cx!E|5$+jFl-&a7A|T6D$dJCka1
zW_MeZ;{K;og7vTWu6q!7?asqfR@b`rpMIF`_WDWMcY_AQ9>!YV^XzkNTz|&&upH$M
zn4@#L{>jnK86Q_oypfV?mh5oo;k<c*4$cQ(ZvS8QPP8dcV}>>N?(ciw?adR&kCs*6
z+3)mey8VQ^_S5e1oOAK`&1=zcIq7I<j-hzGjX=Bn*NI}+bN}}(wEWjO-Tr38lWIS%
zaPH-|4vW=I@;F)XSM-rc<&&dYm6LXUNJ$T@RQOviu-x3T(@^05cZV~die9w{7Eco3
zOO(tuQ9qOx@MXJ9&lb)SQ=<&E684RpN#z28?1KBv1s=5J9e<uD;8+^9Tv%O$K~SKE
zDX~O~v00pvRbPW4fPs-|Vzc#%W(y9%L~+*0gmj)|5!u_hRy+7i39~zW#iuwuI{t`j
zd3d?=!g4OgtZ)v&h>rR<6MXlZiYtX@8!$U$PchPOj@hiH^RL;y=Xy-lq3j*sEaxsS
zn&6yM(jh$YqJ@e|_`ekAdBLVqp=qt#Woj12WxKF>2&C&J)~Ni*Rr(*%tf9f+slc%5
zdef%z&L8Ch0TWoABv@I>%91BoE&7n}S5g~tBq{catJ6|frUeC3iP0N6%H2%6H_j;E
zyrOQ$kAmvu-8&_+Hf1F9Jn!}pb=%8e!FN%QIg!65T#C!oZ$-I4-6pr089HJM>+L>E
zFlwX<Y;;djWnk72NW5CawlSr1qRF(+x?+W@*<S@33k4oFc`$uQ4Sj5>yuI;oVo5N2
z>CX)3bK<PW)vLFwS3hrK5nz~LRWX67U;>M!fZdD<wjL7#1SXz(-v4g~>qLW=bal2x
z3(ESgm91^H+winy)-u6&n_DZ~eYdR;-fEG&{f3ap_U=PBChhw%Nq1sqw208_|IX5D
z(i{^{*q_&k-TrKHykPsIB}Q!vvztGQeGoBj`k1X!(tdv{tIP$9V{Qu1C*_<zmYlC5
z`|4;_&1T<r!%lX`>gewslPsE<6(=w_HQS%4-oCwa-iuCGkJ_a(l5RXta=OZ&_=Ml&
z=OhzOt$NGeqmTH@ABt`clztmvkzvA>;vkcjmKMH9edc2IkkWcH7ln(9By_gRGVjot
zt*I*&62?`SD!NG5J6QbXA_4!+Re2)0Ar@Rl9QkjLbtpZmv42q=SkSD;k<Yep+TKnE
zTaJ9Q7uDjP`8jDVvX3VopW<`&h)+%C<O?jVuB=&&FNLmF3Pfj2I#x8d>7|g*QLcCF
zLTmo76l8wt7dA;aZJ~HYV%v)?^QK&n5v_2@dpIS_$SEeFz3hiXS%m;&guC<soo+{k
z)F8?KC5nu!3P%c!Y;Q_22X-Vf8G9<QS~5nKFYAasKKu5?&N(Yszf5CIQt0|Ot;OG@
z;OmCk)fORw;tF>v1+{N>Z<?{l@L3m+c+Zlw);$%A4J*qne|7KP5ncDwXn&~m8a3x<
zB~rULRou-Cy|zM)y+UAqrn~O-UTyV0``|v^hXR~}dbbbt#dI}D8_v^IGR-t|nBc-C
z_@Gn#X2<(wO%p2wBGdE!t?0DiSS}Jd(PhR&X-h6y<4pOVQH9UvR9Q~iuF-m{bMY39
z$-8bW(*Njc^509|W|iWVEd@b4rq5ZnqWt@Gvt5ESyH;3nP1g0CypKWtmWqIH68Bez
zWtS$3^D#P9G5HxgPjP0eEL_+YKhd&G$^Y)v)cN8$Nk<uj4NSToI@DB{Xl+_8G*Ljt
zWl7@-)vn_U)|dynx=3_A>U^ojAYwJGw`1W4wi?k_oskX+@={%W%VsG5D!aA3D{jhk
zE3ei$U4oijE4N-<XEAY+wUxk|2VI_7p<5Slt|?HJZIoDiKzzllDSCqQH+ro;USy`G
z&hS@Ua(1MD|H71eGm7SMrV2-g%?tL>e`vW<li^SpTZ)06*%1%6+ibEALzWmceSTJQ
zldDwuN40om#_^RCrEX62{2$>{duq;%Yje2UC(XN-85Fd_W>vKID<P-dg4+%>?$NMg
zdceKrn&6scTlO`Cc-<Cvy)Et@-O3rxFaNQXX<^TqjwQN{F)@=is5MK)H%^JZwcvcl
zye)2_^8^JCuuT=%IA8O)gt(w?@Q3-&o*F+tA<meU5<Nj6M_@w^w;<bYVWpBCj8~ez
zv?X$WTlmMUYjH*`^KzeC<%`xzte<YR#Z*ht;MGbGRRv*o)!kQBWL^j?pDg@-mF(<U
zlk`sRv{qwy_lVnGEM{q-lxnht?&S62n>O57;m>|)NtLx_2-i~IQxajXEcp%!%w9ZI
zU}7r&L_Hhv{%hI-->2+|u@uR5XkEk5RrO!WY{Ij$^{nN$%_nWSx`O$mPvDQ0Z%?gT
zRJKBoalf(TT(dOa+Uxs_+*)hP3(~Ie?Q59ySDPVF+HLy<zNHWNQnzGny2vO0`M}!%
z>7T2WnD4CIP@R1J$)vvxSr1q4Wn%Ix^oW_ZJVvEP@XW86>=~wU8y%KST+MMweD^AW
zsvX<+ORYKNrBE!bVECt(iN!%~k8n+*@0$x;e`dKxx2&6TYSFvfJ7;AHJXS9ZVrre2
zR`9r3Z0%|R!&|jcb9z`@_uEd}Vtb8ij)Hu`m%0qGgBgFex|eR<_gm<Vz=o|2+NXAK
zwlapEjdGdFu5((k_x(&I?&|FZIeXYX_6lnsZe7!$uef*b|J%ij3ilf7$gUHc?^L*H
z$uH5HOXg1N65RK|F6YiMubNuFFZo&D7e*OyX&via^T5Y5b^n$X)7@C3-zRtRCvyK&
z*b=;Y*Zg2E&)HkpeGdG4z#sgZ|DA)+msck*1%}*z8Nw7Orl9U+rFdw%_}K(A9e-!P
zGpQ13ZZl6`bokmb|M0i76AHJVb~^FUBCKw;2XnVc{~r6$q*xCxuAB>JcAs7ld$Uo1
zF?YdLyTCU)#qTU`U$t*>_2M79PQ9PZdFptl=&gmT9r)KMR4*<#>Q_^{;YY?HCf2Be
z^}i2y9h#w-x%48(9j^ZuY6WGEM7~*M@p0F@?n_a>MQ2}7(R;h%oCZTpkm|4h56!N&
z$NoNiyw{|tHl*m5Mc*IED-UiI9jz8@*&}G@t9U<1;N_16IgWa1rUD`#Hwp<#ec@23
z5t7={EV$0`B)cKo%0C|W!+o!03OJ>6*SNZUGuZ0BMliyowP)4}UB;Dcjjh7vTXu)?
zM>VhnGFYtrS{CKPAK1W>?2!D`fx9ZJWsO1I8i#y|+`{I$tZxEB{uyv_Ea2RFfq&lv
z{twqxW^T*;_ke4O!|8tudUo-+EqcrK{yN{=>1i*84{TRe)Uds+D|!8Mx9_bd0!z8}
z)&|-$3${-$7t5#!yVR2{XR|s?D3>jei+9T5W0!i9S*7>g+WvKCyJpc<vC}(RJk0ko
zIcfhlmD%>8RqL2tT+ia#RHtnXJ;Bl!j;>khue1K0)-FxYql+F?Zz%|=$qf0spk~&B
z5OMd3Nd|d4uk*i|;Qj6}mtd#pJ7qQx1FrZdeEEWxbYC5KD{$#w!6rpr1_8abZw*4e
zW}P`5$Y1B{mZ)a*C?$>YwLH_7ZIwDHDYrx!rlvf)q0=IB_>RU2eZh@aC!LdCcw(t$
zK;Duy8K-B+KD7IMkk7&Ug75FGOo{B<64=)r5GXeg2!6-5E`j}{b@DT9S*O+Iu~$?s
z@7$@i?Zz4fu1V4Ss}Ee4*c%<a<$BZu*Tn_Z%huS%zNlX8@ZjH!Q-2M(0u#~$4{-ep
z;GW6A`nuu%2loT#{#ObFXLCQfz?D9?(CaC8s^G!w+!bDPT0g86_%eIzjD50S71pv%
z+)~ecas~U@%#XL<x}^Mcp7-TTWfqg;_n-2Yd6ypiH6?xPIi;pGr<U%vxOgTh>^#>)
zsl~PP8iMb!J>=dx;a)%<J5QqG(gOB%22Ys<4{na&e{<kMm!#Rs#S#2RPySzX)Qk1j
z#gGC4NA4Yu7ydc5C1N6Lr^fW?%lxYo@)mDMT4Ny8qs1j4z^dxSEvU&AYa;MPfvZ!)
zZk7P+tbmXt2G-Wg&#&>-Z29297r6EL%!lV(KYp8_XL#+Y=HIS4v!8B~=r%d^%xj%^
zgYc5PMAOF06)WBeS>27%%@2)ux`gfji4Df_S4t!{>WVkG?-fqjy646xiAbqprU!d$
z)b{3Hw2W)hb!OgV_C!0n*V*=3a-6n^+E0(yx+Q*fmJ(0cV)EI<g4n|L2{3JNnCX_a
zC}HB96*Zi3@$G4+7<xnUYipk-PkJ*!bN-5mRy~0?3s{{rHs^9?ar80N#_s#;z#Z7Y
zFgs$-q`&Xg7e4#9flpw;hgE+<UN5Lw9q=R3U5G8#)mp@Cc7^L4HHQBe_?9H_FXeSR
zz~eT_@4zO*BiUygU#+w2)U*?w^E5U8pyk2+)yL<)&=Q#a=l6R516wz6KFAb!Jz46O
zVdcKLeinvzR=#U{$0?M_B5`Jh+P&7-w+*MZzFG49f5>aL!oQ47cf}8KmIYmw%W;f*
znDv$C&8Cm7+Z61M_H8-&PJs2|*ZIL}X&)>T_`hX*^I+KMHRV9);w>B}Hm#D|UG?SH
zRbhvGMcuN$EF2ql<W12CTjb(;k}2qlE906i3MUtNE?D8S=GD*S@}SeJ>V$6gI$Cn2
ziKryK__<l*fc)OcfhM^+5$oeF1O|FtOKw#;rB$zDc&&-aHRNhwz?PbxPZLB>Xatzp
z@?@<S(~K5ayKk%NfoXfYvvaquy83R>{<3WzQ$;>p%DuY1l~>+vPDJ6O1^JxZu{sCB
zwWHT;>sb}LBX)J*T9!Ye23kfBE{aD^-Bz`0Ll)z_!2N2TY24Ghga5N!XzR89BsOVV
zw)C^PGtBe#QVXW<^xedpb9mjFNfq65ettQ;<k*BgenBOP7uGUwFUpN_SbI#RmUD;d
zgFbERb$236i$DH7y8D6sWS0AS#gFU$R_R`4Y4ph0xMKdQ3`y4>pG_SqOG7=PR9CaH
zdj&9T&RUr+veND4dqp1IDArDn8;q~^cWiQS;_gstbeJ61^h$MukHCqRJ^z1kuh9xy
zz_C`mY>w1n-5!=7f}FG3E;YHUG<{KZ)(~)La^B*h<hsA=)+ufM^M4v&sdm4*7_`5x
zX5)$Z(ti(6-1Uj8d!mZO-Iy~DC+}FD7vJ^9(x1V`^ViXNJN}g%75g4lxOl3Om*tiJ
z7Dh{MFzN4zQf#)mazo+l1`pxIBEdHztj}x?I`N=|`)<Z%UPC@N?ND>RCgyj#w<|XH
zc8BH*hcfu7=&tSc_GVsd9p-Uzii)4UQ10$SWlQsf8O=7j>aVtuP8Ct)W?s*0eMR^H
z?@JG-byK3>{ZYLTG%efgyrO=V&tV-85wmU`Bh3s^-&0bp^DfMp^>X1tryi%}%UA@I
zH*EiPk)=z+pysu*M#f6_WtsXysz>MZC98K|U^sJ1b<zz%W!Ey+h~tkgaCB!LKK_M;
zN59PD(t26z)mD`o4$0Y-Cmg<ZXSK<{2QF`IcFo$mT5J#KTHr;u6<T?g?o?=g8I<qb
zw>>Q4<eUHJyFCuh5>0k=F58{po}7E*YJuwS8#lSCuNvGJSbWIf)}?=I-#qcS`%UTY
zA?q_cyXRcKZSC@QTUK<+&Z*L-$(^A*@2?z--YjPwtN1j)*hg@|lv9t54{C*f`f^;}
z=HQy=M`v$sR^SS2TBItx*6z)6u|Nd@hlR4N0<C)%yl@QUF>vS9?DE(WkomMt_1UFW
z0*-1k+-|RxVr^o)th0WaWcHhOF<+JrmXx)V-v}O$v&+&vJKMl}@)C(grOm6pJU#Nl
zjWKBp$G?jOZaycb>+RZq@_<D<&$I0>CUwt$vdP?>XBBUUmNV07hc=-TUEI7m3G6Hi
zjz=CWV_|AI$I`<lUcKev|ALg{v)5T1GzH^Yf||L*INdWO1XmqvJ(T3i6S;X#_k={@
zty(h0^VAxGW0$mVeR$xO%hxT@jC$=G8+)_uJm$u`x<yV5oTD2iHcQj2y)CTLU;Lhv
z<nf#DQ%s%3Qx+LbUG-(LW<Y_b{f&iN`%)c6vRp1|$vj%7<Deky9hmHX<iQjX$2B@q
z+{p*UrYTOD;E|TR#Oth-u%WJmkW%#(CJz<^wplxt+JB8;J<Rc0bwSHSm2Z=;h_7|B
zKQ`fs;Ubqa^S>BxStPnvWa49&%RR>ClVsc(Zm*inpjR<5_SL27;&+ycoLFG7(;-V_
zd1CXXf16y6<P?}3aq+qFAz32BYZXuW|HEhbCI`>hCh9i3J#lL2wKl0R(c{w^X7eW%
z{akbD!Lm&E7GC3>#=eK=q}jJw_NPZBE{)#O)Rb0mSo__E?cFAwQ(8B2SSv8F&az=q
z|0~4Nyl)d%*@Wd{fej9C4@LBw`K@tCh;l!0ZK@SZhLW*Ept66-^ZsWu*VR}jzL>6e
zY1#L+PCQR`%eqEv<ZJ7ESDQC!@*kGA{R?zj)19^W&#rO0JZJ9tDftiO=dEHoV9!~g
z*CZ6OkgN27%f!S+UK2ldISuXjPeo1Q7Z;gxA8+L}ITblm@J9SD=JVTm)&(<lgqZF&
z4u4j1JUEpj)KW0`$cdWIx@)-3@w0B&R<Q4AW0`Tz|CDQy7Ke=1&zyMkR*A#rxwY4d
zMM7PA^lv5h+wNS%{n79Y+vy0Ys1GlC1s+d4;o>M*8{48`yHPxaC169d?$J1>U(>yh
zJ(`lS(MdS6p-JKz;}ludC*5BKj%rM+mgu&4QWy3x=;IWx@Lvmqg3by>>8@}ySv-*`
z(<?iwYVJ1QCtJQxORe?Ruh!K3DVN42op74L{$`hMK-c>3j(5sJ4o6l6O<wn5&ojPU
zW500SX)3wd(z&}d8N!z49^4!G`c7Ht_U#et3~G0GJhGW1@#Wfw11oF0Z3LZXer&m6
z9JN`{#VK8MvDgavzh9a{cO@`gscN6%7#TdVBwb2li$<?()GD<%|2-BIf6SI$H~WQ^
z(yOUm4r|JJPr2+ZXgL{Za8uX(kn*_~+wc0zT{Tdfb<LT@On1gpex-jOSm!L4dfvqJ
zL7aJu>%7>9U13`mp6_s)63*(&#9q{}Mrg?c{oMgfx@J)fnRl0k9;uq>IsfX$zPqX7
zLb(zi)3<Uv%A_T}b87qVmmZ<1(;F))eE2Bq>no=!r}6H*a6?!*&Ei1d8<D0Rj9i}!
z%$&D`ZdfZ6=_&YI>0!?lF7e0rxAWXE@V_F`aG3e})XZMdd%jOIt*?sRh`FM(oqLAZ
zx?VwpwM^@_e7>O>-WU^h;j!byo+rCCPRUnYXj(36s8z-q?A3B?rNN3jH}}t<W&FQM
zoUP%@L2m7n-6pY1%end1KXvVCKK)}7%Z&X;c7*EXU5ip@v&>z~KhJpanbg*08Mh42
zJUk+H_Q={@2U4ZQ%_Qf{n5o@%ZLeKfTc`2q<jJpePZ#oUW!WGjsQhvE1<sh`+>>Q*
zJPFS@Z>1(7`kA}@&*y2JUy60#C|)^H!0D=$B^;=?sMD~+$<gnmkAB5+P9{?|<DxhI
zy06aiMOm<%T>NH2eU?t?EF)7R_OHJ+RX0yx5p^`Cc9m8b`_`+4jK2aBo;F1&ukiLL
zpQSR*p68kV+Ao(>s`g22>8(9<S<Gm~jPQ=Y#JG>EBtq5Og8!#(E&3(1>hOcKFj1GQ
ziTTC971aKxTS$NCO?vF6ed|KpjoJFQMco%yooPGvv!mH=U61Iwb9(QLJ6(Epx9Bn$
z?^(#x`s--v#4d?{DN0=BcXk)@D>eAIh&!tUcWE4*Z+vJ%<V1Jg9ZA=&NaTE9GsBU&
z|C5#e`XZ^Xk1W4i_59N6Iq&UoG~qy3v_tn~hdyhiCl44jKNzphn08*l^!yHygpF$s
zCo*4DV7(R~lPSc%@bR9*I~LyGVafJ+XZH+))jLd&6m8&ItUcd*>denP3qJSlHsV?G
zSaFholS-81#R{cm&X$+EyNgfroQv4GaF*tuoyJ!_HJ$uu9Bm-{XXDI2zbEwtbhQ8I
z7h2iAD_Qo%#9hBCyWb|Q`rkZjWBugGp9ADiF5tFwp4q)a_}2mBoeBKRS$oz6E!Zfu
z+HuZ;Ew{FCRxNNf?s0Z8nRLU%!+6c|i;H9zEy!PWq^!&2z(t$fO$yr{G3~j?Uv07T
z>SPh2D8*3jY1bC>rAAv`Q)rFU=6U#f$B|!q=Wz4I9kz^7=?s-rN{&8w=mGBqi(U<p
zX-Ou$iCemYym>Nqb4>bu@alzG|BQLwI<I}dS>~g)Mx*oe-!JuyrDgq{ctR!}J1pUT
zctg8(r0U06v%-wlpHOs&j9ULfVb0MVJE}sof6Q7Ff93ea2~2tgyFO^l^0nyQpRp+2
z$514Dr%233CS|u)7v7s6j{00toV3F-#&hcbj?2B=%A5j|-Jb{S>;B=PsN=-r<+kME
z9@SHPsUq79PpqFZ`Ow7;?cZLtoMP;+p2fFcdg=bnlDTUn4_fp5+t_8>$!)u5mBU8X
zwZDwl1u2}EY~#DwhHKTH?H6ohyDe_a@bcTeXU(1CtLF5v|Jf7FxQ|DLljkFMP&03+
z$-)x>X3AUkrLA5rkbOFQjqA}HijgM!j%rN3cbh5ZPLup4*VR9~qYSrfS6CCMIj1J%
z^m&bGhcXlywR_GaIbIR)es_WM(`UEv4Khz2EYd%;E$7P2{5KZ!7xuIyyDy1U`0C^v
zU8Mgcz~M)g!!NDglTK%jeDFG~!5F#bEQbkC<d$W+GZ>6k@cic$ndN(6mQUcx72b|h
zYj!>@@pz^q-LYn?_#anUk%n!O%RdzGos#73(_wz`!Echu#<e=L_XlWQykWjHYtE_@
zlgw*o{FS!9!eKFg(&2;ILe>|Bf7@8>xZ!;C@~WVMW_y<fMiC~CJ$p_BYr9^tcvf-t
z*5ujFJ^?ohE+j@@I8-z{cb0y9&%q}%T#RNk`PX#s%V6JPW9XIXp=Q*h{mJ;nmagEP
zduv-n=bW~TFzMkGV2GP-DkJ1C)ZG)N(;HY~ce#1dt_@wLLR<>Yy}vc*nM|=d|7QCU
z4fkIwSQ&KAp6HZW9ysNz7GK`UAeIsb&O1BUn+_Q8%AA?KhV$%2<Fks~IlccEuAH@6
z(KOwfM>N>)diCieE7YEE=?a?BJL|OVjTIiAf@?ZnFZ0{nyKN@_TM>p}kL9g0yX;sE
zOlo<9PV_9_tn>S{=iv<@pD(WU)zvt3c(V1$$^LsejvU_X5NYFFXtQ36nL{hUE&Isn
z*9|N7xO>iaaGB!OX|jl;HAMN=)wO5NMn7WWG?=wrs=rFYRnWCpsCH|j<F!czis3EZ
ze;C$C&$YaFXIY2Lp(8JB<*KG0SYi<Om{)2mSE1^@M~}U7d`+)JD69_H_WzWK!rZ<G
z1+J+pmtE$tIju8cUI~w1ZSUtD-3ui3%XDTLdCX!mxe+^aN9CI!E|<&KOfFxG>A5*Y
zI$Kxf|Ggy<I|bENEu6OhLC>Q%d{0VtX1onO%%KyrbL*UD>ATiSelpu$9Wi*lr$g|!
z-HtcYGB?RwU$D^QVbgn=lglGd_bEFbRnS>HQ|!8e{}d&2JxlrHf?Uh)&iCIc{AY{5
z&6ZiKOhcDES;Y{0b)(T1N0R^-Z|$E>0q)J~bGt5ZuiDc4IMn0Fk)GrWt#3_?YcH(U
z;n{Wd><-t}KGhedtqtuyVjFu)kwd6q+b@rniP}2_7lzol2E`}_|G9Z1dg0NCzNZrc
z<sV&+bH09Q$JTH`yVxtPMX?v-4+ow*9dqFU&#^DFjv7q6a#S?ca9UNWOhHZ1=Ly{!
zz80l17N4~av?ohFnqcw2b!r4xOT@{b8+&=<@)R#m488YsZl6(t$hLsqRfb`YWE5ZN
zdWbqXGBeMX)8$RO8$A24f&JPGn=*CuUIgs(?aMQYXTKcB_~3-V?-Pe3uAj&Vdue^5
za?!H(i~Oex#B3F>uD-P1;pJw=bGKN3t$OssWF70B?$irBPeZSqxU(yiXLs%G{jPRP
zlW#_*#x2!uG`hi%*ret$Gx!nf$^DLj5zHsA8lPO~Ep<TQ;)&X@XVw>Q9`4<D$CcyT
z!lOGZW!3~v`Vk``8@|1A&5;?7F%x~gZ!S7+!(+kAt2RyK`cskAhf|8PS#@tL|De^n
zSX2LilkdG+^{X}Ouj<^6ntJ<s#R27ivH#n)`m&0izkS8G@=LVm#v6B+JdTjzop<~8
z-3;yX8HYYz;1+taIyUf*NSgYh<cE`$Tf=|xxcWUQvyp0^GCN@BojJ}oV{_Mti(com
zz0cmggZne@^NRj`npS&$dOVTvY`No}wkO`Yl>4^<Z(Q*Db-DqZzZN>X+}^d--Rp5E
z^V$nRAHD9Z*y3Kj<(JlD{j<s2uRb+jcHGph*PCU|i6Dpgx(5&U@}?gPHQm?CdG>)v
z_d(C@gX~vAIbC*X&S(%+=8P_2KfvXG#Uf^>(v9TAJ026CAFf^h)PZkFVzW?A%yyTa
zpdA-)U)Y*+_v-Ig54#y+xnyIX?NyRLyD;!a_b=Z^lZ0>n*PX<);^rltT!+oS_Qc3N
zzOVcE{?Z+f?>?BQ_HxoOZ=M`hyUWsI%BC+@=uFxnnh~6Oto3>6z9|u1ygRFwh8aDw
z{kzPYEomLg@(;b&Kdj9tjMDjhv{%w+HK+EY-k2-=((y;G@IGnC*wu7By^&*&pj4bQ
z-wK}RY^$Ee{#&?do3!E`)lHj{O--M!{+(fRG%Ppi_K^h-I<F)wT;}c3`{1aBXyv}%
z6P-8Rm%bF-#<f50dF0wG^Rq8cDc)$!d$^(T+QP*T_v+j~``A~8_ln2w%iB48!Yp5G
zw|KQ=n&{WNXO}<Qvn=w(i_JQoC$nbO*>#D$@yzADuiLx5jkl|tFZW{Ze;<YarMF)1
z3+0*U))Oc6Zt7c$&1E5bWa1n3q<7Z7Japh?tmCxj%lHoO;dr%g*7*l7H{VH-%`@22
zc`lB-Z}GnzUp-Nd>c|^Al#2H~+x}vql2%^DsjO$2%zJO}*xQ*UDjkv*i_>4bgF7j(
z=kd+I`>g(#@b6V*WY$;Eaq0`?e4g@c%bG0;oadsq9D4RpPut+>4$knj2gkg)-oNL5
z|KX`yJm*)n_PGvwo~GsLC9kg2@BghIdL@!OFaO94&lh~#mh7B*c<Y?P%*0?5$Ineu
zbZ<0$cFki~D?A&kbMJ}idDYJI+w^z83{Y|J`H_2jSvGIVl1Y~ee2&a`xxHaq(BsuF
z?w&p|DdPVfy`J~MpS4ddWqn`xJ=CsPtJ!Qv^AEF^*2lAL^k1$v<FU5qd1JTl)1~D{
zZ?q`YCrB;vxaq)itZDg2yKR~qdN~7_7!}@on&(S~^yK|IAs1t)ZT`BnOXS->*JaYo
zX-Ug`6PnlBneV=sKZ(I)omXC7Ci8QLt1P}-j0*aK6imLy1{ObJVD^74b>A!J?h%{*
zo?UidQuaT~)hkR+FH6=h3$E;q>6)@DGx0#-%R^>mKlfg(6Kj~cMW#UAXU4MwbL<or
zC9`_XlzMCuAGP=>*QPLO8M`ft;Tq;Ij&S4_3W**y%H1jWE@qY51Xbxnc0JyK%NFKr
z6!?7hkwUXmX74wLivO<)TCE+X=<H?9_59pe#C)fs-#S4mVXo?ThTyC)^-YNhPi>QJ
zpXX1tzkEm}sz{#K@jKs9Mb?~8c`xTY>EwOIx#C6Fyf2>Xd6&MkxITd|TFUcq-j(@x
z6E42Ezw4EIP{C(s!$aG`7JTzs?((={=F!J9L=VZGDizah%{Jt7lJ@zo(VS5BAR#V7
zczXaZd)n);-`wHRJM;6>rail;P}%$M{gDSxh3X6b$6fTeEp~Uul>YT)DtX^{t9fso
za6i3s>T&yh-7`)~oA;dL-?q^7bfKix+3$RQ3Pm4}ZMZ2=n;71+#ZxD5FGt+w@+%y<
z-t7CLHog;m`~A>{PvZP}t+H-YwLAY`l|8oe*ZRYiFVbvmPiJ{neLBMPq$b~S!T%$R
zO1iSnZe)Gwv7j-lS(fQ(zy5+3(|0^7XbtyRuu-URn)v^Z_CCsfMYFFRUw*N7dfAio
zCl}7L9SgHmdU>bm+VYY~3{_HBo@}47VJl;YCKDT1hU*7~LoKN+69fX<d^nqU+4XF0
zs3{)nkuuMcG4^86>|-|P%t-fk&}~UsG9~kA+Je<)F>zKR4_+nAoW0b-LMOPXyFriH
zddfo1F9|C<*u8h`I(#8fvn$4;Ao9hrtB3hwl*$Au12UX8#2wEQnf-Bv+ZM^QlR~{O
zymi>Y^kmZ9?m)e$T^#*dwbm0ZI~dhz3W?0QSP=97h-mgP)!h|_iA@cX=5Aa&g^!(N
zGT~QQx+7%v3bySz0y~~QU}KD4<}<_k>lTK2(v}=H9#)EP8rKHDK6WE&u?S<33Qvep
zBA2+<CiWo38+-Ih`a0M*3H}sJKCy8P-z1+Ku?(I;Y`pa{r9!{O^w#BZWjHVMSvfh3
zb&h`V@+*h-NtA}2krdy=puf@mSZJOE+kEZ3Jt4=!N?wY`rV4m<2sB8CdfaSLdg^+i
z=yd9p#5P4P16fD$t{+olEv80HkJI6MHnrkW;UU-D3v<gBXE9x=c;v8L_jt6E7l%TM
z6W0rm<ZcdE*9)${BcEJt_5Boay;NB0N}r>cC*zZjtf@g)9sldkEDQ<f|Kt)ZnHl#~
zfZg>j%atU~dru}axR^$=h1v@q5p(CiXByz*^|AEA0{x^X>2uq^t$Y!1L9r{ttx@QT
z_)?}2t(QU5l$OZHw;tOjn4YY-EMj`o#1qTImxPI2oiuyru0>AFiETICws@7jc2u0|
zbWCh--h{Y*_KVY`#9z*36N%c%y8E%i%8y@k99RAoi!C@Ps@^Chw4v?Ja!s`-i6uc!
z6?e-xHyWKPd%A6UtoF^+BVJ`|CVZJ2F7BHy=M*{Vl6FnY(n{{+k1IVUO3vm}3DXR}
zIZ@4z`|I;#P6_XhCM=!lS7~@mGJDU*78O00M-ggCE?NuSZOxR9*L!tbOZo3%B>g7h
z)TbXsZZ1VFGTY6wU4o}EZRrx&p>^`vyAzt-KPGx=C|)@>ZONV|$NW4v*(B>kL^Y;F
z#;$!aIVwglib;~UQg+I<S!P@rF;V3U=SO%7-4b(KktI>yA+6SOy}QhL`jR`<?^JV+
z3pCYiU!leO<@0Km(xA39hr&&(7-lH0o4NMF#Dkpd$1Y7v-om``+T!+|Gu<xaDs}ql
zI=Tp%FPV5;_BiVsiQKsI6zP*TsuwkczXZOMp4xt)RFhZ5rgoL@#eD6*8`}k&CtjQx
z_F1Yk$K;`iQGKmbhhKi}n!D9sU$kp6^CnJm?7mX|Rik^)-xa2et^4lB?K~@(;M6LW
z`1t>#W$ngamvK*+<*;n)#7xc$9zhO5kyT{^tU@l0{IfjVJT9*P*SW&#r;vlKK$GgF
z2@m~vBd@w}h=|Y2SnKE&d3(v3G|i0*3^dxDLT;Y=w&<;DmcbmajxdH)bBRf;L5fWB
z{aVYMJ|ubt+kFvhb1Aat&=rjIu{m*tNlkKN$dkf`g)y#L;+FFto(nZ@k`2h4VtMrg
zi-*I*lYxH5thdhaX{=<N#Ne8;wlN`F!7b~q)b$f{_Rr0zQz{hS$x)=YX+yX5<%QSZ
z&2wSf($iGoWyRZkiA8eD4o>IV$oY%pw#-oGTjL_|dkL4O#;L$%{Zmp6=d|XDi}mTW
zxiojj24o!;`!DReroOK@G5OC7@5Vhk=DW^xXbC$xv|M<jAY?F!q2sa9+_&v&%!1QB
zq&_*m6Fkx+v02T1RmjrbEss==7<)+jO`f~)VT!Nk!wYw(1#es{^G3n&O{WU)wMk5>
z9o+91a@JjXBDQl&Z`o3lkT#2#x_jrcF1Yw}sblz0m*7*WoqQL4LiXJ_cUI=;!k*w4
z4^LcLrOI^7_}d?)H)h*gulZ<~Z2B3KKkKL^<B^NnSB=yh!<M&L3mqxFGl|zpRpmmK
zpGT~)(-X&ahuc2zsCchcxaZkmWYOxO<Lvs7<I0@?gJ6+0iDfT1orI2TcoBJG3LE##
z@SiWHK0R@?MnS6Mr&Ff%#<2g#rZW7MSX0vOBKI^unS0??*J3TZgo_^T2D7uz*oivy
zt?gTCF?)W79^3V$ZKWy-H#y9@U!83-nV=HWpuqRqL{xd&3Z?YW6HU89CI3Hoo*wh^
zP}3VtDbs@~zG<IBqApCc-6~e7Jcs2`c(L?KkyF>SjbcLbn^y~ZzC5X)d3J)H?5mLW
z7g@&BZwKhg9^+i|_h@MMWEt_}O*>jLHtw3cP(vbl``T$rElYGvqY}BjgHxowm2Sx9
z3z)}zW8<Hq0!70PfA&ph8=q`nvGdXU<m189-y{@s^SqIiR1t8!BXneDUV&G*G{?q9
z9|`sRgfnwAO4!>(Po#WI>502PZKJ^Ce6Rl*jq=C695&zCV4>JO#mmX`P(+;M7Va#^
zk4K}fvaVa!zkq!%_y0dqiV0aMT-UmkD_w&knl^-U26Gf0lem3S)m3bh%_^b5{lD**
zTTZeTDu@c%5nv|pwcTK4tG~;G&l?v>JXtw$PVmhlrj5pyERnp1UFXl;kqF<jP$cWK
zm;HZ9Mfv<Ct~}4)P7wBbH02;`FRQP~;-l$1d+I`_XU+7H+^{?Fm*A4ncfU*=9p1*C
zY$#l!A)4zpY0d=8bymu5QAU%6jasknIqLkWm#;tZigB~Sg;RoC{L&jv304_?uu%?b
zsP~yR@u&og#?l?GhrBlKSsk)OlUws>+tNn{T^6aK+Y|mrbTXt(t=gu(>YJhKx`ob%
zcX(w~zbl)#;Q8(sA4O*wZ2sJJra(n@=ONR3y|Ud!?zIyF_7?5?UM$(mR1wj$V&fkD
ztvB6ntQL&8Z^*eTvLd#kcVd_?dy|03nZRwo-@oW-6uFn<dOpXoOJ85(7N56(<gG1g
z%dfJ`*L}Fi@5Pw~SM;LQE7uiE`Zx-FoAEjD{cL?ls{_k#8+onW{C2HC>8g$Q=lLp!
ztWV7I*LXDb;M<FaZRZ~EY7sjUQ)M{)o>JNR4uLnfR)s3bdwWzpcQxqSK0~0%qVQ8)
z+zhwp7OM-D{M5JPwR9|RZ2Gfx8vmiTz$bl;e@k<mzI97a(cfTj!tR69f1^3V)*NmJ
zKHdy%h!Ij)V{uz7v+7$i!<R{R<u?@b8Mg1=5jFKD_x2+qM`rIA)qVNnWd8T2e%{R?
zHy<53<69;0%Bk09*|dZ1>KA4TYBxN1BEHnGejT6dBoVa)8Cl1fC8na6CT5?S$mt=o
zjLRU1iDOb7qlN;5v(o~%1(6ffzE(||UmGo5HFu&GyPxtc8E@T`2@e}tP8*7+s$TQ!
zxNz>O<UCcMPbndXBF@e>XjPpnXOqc$o#$v+i6ftFzp&-ps|Tf{U(Q=>GWTfDo$G%m
zpE%)aU^-2HtI)EmtS+h3OG{@>Drh-zQudMUwG#!6zAL5N&ItZE7*%z2ir}}q+jd;!
zX#Ib6UFFfoe?{ClqDv=s9}_&{xusEPLSsYkyv98)laCzVI&<FnIqh2}-uCT~vq_vP
z`Qmb{h&b1yDE`XW`DbR!+>%?hP&n%0!|gZjMm=o%{_=MBfwn1um#?iA`drwPrlPb?
zf#slJyaxwc!k?f+lI%wmTh0n_GHw(%5>0e^GWY1l_+PDy7&;dHwv;@U*z)h|ncKW9
zX@d6^CQW*B)%7Ing8G9;PUSX=TeY0u=q|0o!Mu$<K*!(zM4QS~2ex&*)|K25EMD9T
zIoxtP;%a7Gah%I(cGqv|8Plc*dA4`2_wdJFNKB3mJaRzj)Pyr`8@j}E?}#mYyua~Q
z&lOn#u7?vY2>!nM|8YyrB8CTxkJc{ot$n~Q)%&N=rS9wX|1ZPeeUNwbP&~Y`k?HL$
zk&MLUj%N>B^hkTm|Kxg!TWFElqg(0+=Q*$Gj9$QH|1&P<YV@9m_c)S7`g&PcEx6(N
z^Rm~^haDVBF+AsjY}<Q(dSC33iS~$p>o75Pi+{q0I46d-gg5=)d>d>wg{X>g$}Wum
z>Dcl6=oHOE@p}U#d~}YP6uP|USQ@UN@-bDog5!WjkK?1csZLv(7=-Sz<TTEDI{CW9
zWB(}(?8|tqCJMhdU3^r>%|EDPR>)b0lQ&Q7INADl=CL~$+|tC%=P4vB-7-;A^FG$U
z_UXd?2iaoIcmy^j$Ig)5y>QaI2mjswxhU~Ii)-tUdf0kPX&cv@4adHnO)gY8QXs<a
zdvjvT4IiO3@=h8JA~uoPEh0-CJ5F%8Iea=HoOiQjhqox-5zj46A`U?@Da!paNzHp*
zLO7B;?;LmLaclguEV(T0t?I?3E`=5c^%#%kEi&RxTNXSwnDe71LDy3GQtbKG8S0ty
zJhXRm-E3W&-o=u%AuVXzGq)9fMvaS@BIanw9SL!AXVQ3j|E&1tn=aKn3rfD+e%SC-
zd!M4mid5aSlZ6~@*Ahb(J(&<wk>dC_*+Xe!{e%}WcOvKZruq2<)E#*kk@EOzr-zx`
ztlYm7A{Dx8{wCKbDS4*39AJDJFzZyhU6{v;|FN>W?((Kx+20zOnGr9Wdb;7&-5fV|
zJ3bcy1(#`D4h$L%g*n_C8Mu#2xIKI+AGK^HLk@eyJdXD~Pb12NwemV`9or>8c-A{a
z_?~$l?sFq}r(DAg_Jm!Bv>8rp=3UizqvhpV&d{|>A{IWmc2;^xS*+K$d++bg_F1<4
z`%S;<qI2DWZgp&J`m$$=by_|PuB=Gni5KJjDk^fKGHu18Gbs}q>!sX7R$X^^kgVaB
z+LXkS_h^}_og~vli5$lkw-5JQlg^YMc~mFnVm-x0_h0`zfe_A*E}I-rJDoV@>(gtp
zm;LY*2bQ`eb8fISi}-vFaS1)>nH3w<U~#5(<1}9j)*~1GrzY2__c<-IX1rc-Qo}h@
z;g$@m{4)-PM~M=J>GwEfmOh)}TldVhl_fys<@}aY6O1%jPu`t$;9OyY=C(bz1iM_O
z-&1p9c<orC`8)OgnwNr>hZK8Kjubc@uU(tj;`T72%3UdUa^eJ+^_9WJVn?;4<Xd;J
zSF89)Uucmon3oW6`$ezEg+$S#K3<X?(mpBDelId#zZBlJ>~_)FjM$DP8XGShob^Pu
z$m77hxV{&yA6EM4W~N1OvaP6Gdi&M#@OQ$!Do@q;G;GhQ<$Ze<A+y5vn(_=YjVP0c
zt_v3blXTNob8(0Yo*l=tdD2mhjEtydZu)&5+mB7&>UR4{*z(2+j$3a2KN8y7Hk0wL
zp4E|yx0X(JIU4-e#c$0*29DTEcKs1E9CH&hlJ918Jh>p0x5`a|JCY~fXVRp&E1Du_
zvz%t6-7!qcx}^}*^Gtf*;%|+aA$dpavW`sSb7|T&|6WwST~_yfPwls1^XhZiXC=9m
zuJGRUPUXGCZKmESCk@%oi?Pg#nR-6vs?&{w%t5z(j<~#>Di)Tow2|ZS!YEf)n-kI|
zM`yozndERrDvZPE((47KS+%Qj6_-6&98~a8(Pe?I3;TylCwgD`{hO+gbpHFc*>#Jw
zR8EPmlvsMOGHv6TDNG&?Oa*IXcg{bNr5~xnz>sj{YgTvYvZWpimb=Nd{B%0;sLJ*6
zssEyp+YWBF(@s6g>b;{$b_x^chEpuPCuaVP3G~e~_{AexGQn?0nD>qMe~KQ?d%H3=
zXSTnI^6eb~+<}66S6zFf9<%%oyOij1^M_~bsW7&tj1rOQ(n2B!{`IDoo%4Fgp>WA3
zcS)!p<Hznh{o#wWJZ8K+prWJ`IwiJ+>-2)O%p@N5oC7|!MImJ?Crnr~eck-uq56?3
zjlL_@A4Tzrvpo6n?aU*)xWaR5biXT#)VfsMU=m*boMl2|jmxpn9ZllG#y%_@0b35~
z?l`nPb)Nn+`LbyTLw7n&3sGqeJ)#?OL+-z?(XuSXq__N<((V=$PtM7zh}w`d)0K0A
zsJp?Y3B3j!)Bmqs7RI0cO2wvFYDI^?M-0d9dtO|puVzh}&vc~V*UQ{*c`m%&E_~ss
z=k7eX?Rh!)%h892UfJ?TYwHwFv&q(8!;*PWKTPLjP0O)oTbWo&Z-;H^^4pPUXV?=l
z<Iv?Pjxxee_dBM$E??Doi`CI{QhKjnEYE$3hefrM*7odly>WIb-=oc|U%0*t$cZ?s
ztMNR`NyXu-m#I_EOU?8nA#9A_UggfPn`u6K(hn=esToFd%G!QKDT;eAh{q&0Y<WFP
zqD^)UyU66|=;+V&XXd?eoGPuectIxrbt#>M|1TU0TRy9HE}h6~ZT}%d^Y+O<DLFZe
z%dVA_Tyc}UGb{Q2@@~&<6Po_3uLyfpuX{vU=(5O}E7R*%hBvO<U=@5iOI2k*Z);ka
zlgP)56P9|oOcR>U!n~HnWu9>G!QfdE$^IWC=gFM0X5@<3Z}nH%et(B+^7WSMb^fMq
z$7MfH>%6eVxj^XMCLw{Z?S7BjAMSE{mf|;M)3amy3bs3JYmqr^(=M@4_mPac-+`m6
zqFpx~n7J~sV@^g9o2uw~-g60oTKb<aiS}-ZZ&YjeJu%0Ln?WnvblNMmwXQcK<z}>O
z(qWzXv{b^SY)QU;^h4ckjM9%x-`c7>X`Fa{QK0Lmz~_ytznhpZ-Ew00-z5tZ{Xe`4
z$Tu}tyV`es!$p>y29^o(KV>f)&b?oCbi;4i{}a;Af4<+-m-=3C|K5)iC$}m+;uliq
zOWg7{vrb$y=E$9CWnbJ%?|VIUG+6P;%k$sTCr&bX(}d3|b2sGd)^w98blSw|7Th87
zp+&k$W0}hLm|4ZLc9nY{Wgq*pw7YTk;XUr+w=ZX3XnW0byddTD&$A&y4)4E2-O<>s
zX84yYYH!cap3IFkr*Cd6Pq-X9f$dnJ#L5!I_lepQd2gMXFiB{=O+JH!#fCghEoBY2
zE597p$ZN4Pd~3`6Z@QYl+eKIY-Ue&E*7jSG^IJ~+<@$V4J;AnXvDXHTg-a4ITdaxc
zh}v*b>T5;6c0>RE58T2gJeMEukXBs7*uKTyGKWR$&aa)P%>HMuPIBMecEqFbr{nEk
z4o?c)UhHQ0V7*_yWwPnYNj5D0dy?*Ked9J~{iBKN-#Z+4m$-T<$5UL|?YBrx+>?MC
zS6KWaZB~lg_|>dPdS+Oo_U!YFvu4V(f~M>dSh{DP&3Q9<*<x$|h2bqLXPipwG|x8J
z#bWa7oOnj#{bzRT{BoWpvk16cd~S7MxnG{tWsw5UyL0?*RK1UX_~~X++R{9+PfBaH
z&v;z3`e)#q@0s_4{^<BQwMe>euxgNA-tmM_-R6C+Uusys50i`4YRi?bJ9a$u==`^`
zd&(BeGeO5&F6%rgWoekGD!Jvo<DBFBWX{i<#dUVKqjtj4?%-pG8`>(1CUsi;f3@&O
zMxp8MFAc2S=UG$MU+gRX-QR3IL7jo8zxtP}m6M0VG&cvK2?y2I_=$Kp3W?ONU*i(H
z=N99}PZL)M9dzPM-~GEfKX$dWU&iVUQ?uI+7ddXbVSP!M<(txx{q8ao58K(Q3BG8$
zap;_c`;Wc%`@18`gs1Tw5i;f#UvMXIrpqM}7XA1wx;6^h``j}Z%1CcG<a(W{P214n
zN|t)U0)}QbZ<&xC6B3=<`4x?Hlp9VT?BY@nn-kHtn4wSJe$q^jHYIVsXbYd42B}Qm
znnH0}6BoEJ`p#k!G4NRw;3UMk<i-T0#U~S37zGz`bSOPtA*Q={N><|0F4>tH$u~A;
zUGMs@d9H&kI`8JA4NB8Bna;`GQAk*GaHINuGoR{$oZGG9kDsajo^aWti*4tWm#0rY
zaA{7_dvtWg1Ow-zOjfcwTQ__>bh64YJ<Q|On=>qocG5f>3>Y&RmrC}@={5!*oy9b1
zik|mIo|$G!tatQ+FTT6eta4K-M|nYLOxUcin-X`tR48s=+$F72y5X8*G+Xy{q3qRD
zb%QRlPS7k}9jg7}uC?5odw+K`M6b!Z`pI<f)b*e4d%yLIc(gIOTkv=E3T_7O*ZJEI
z*=U_Hxbtrj$5PFpTMmt!Dl-zCSXDe67<o)Z9GIGgeta-#c1c;WWv*X{L)YsQ3Qw&x
z7bQI1?RzZ2m78(Z|KEAt@3%-KPce*InL6dwwigo?3LTU>ZT@6N&L+j(B3u{j`<Jks
zJE^@<B;mkKl`f6N|JUr%*<-dW_{6D~9JBkD@;XZAx?I^;6s*W<QJBiDS!0#q#PRyk
zp(P($j552YhB;;HI<co_3r{+be922~YBq<K+T!dfJABO)mhvo&*|NO1E6TB_=TX!}
z{tIDmGF?}^yOp`o>T{RyiD_kb1QxU~E*9CKcv$R_=px4558m6yx!>a0e(R9(!X=aa
zvND@yB&_m`nk`~*NnJHT*wCLb<r1f7dy=q2SJ>o^1P|YcmFYaqZZo|as<vEkIT04o
zsnA)`t(+X_8*uoVLcRDTC!Otl{~synYs}xnq1t}zQ}DuVBFz#HTY2`aPCCZ7YK@|H
z<<?IZSw0)BF>N;4w|e0z4)4f@KSy+KDi|I;^J#&R{G%i$RXcA5jw?+S91P}9rv!KU
zCRY2NpRHN*?Itfvm?sz4!Kzs+R~}8R;?~pIYI3Qq?=<(G$8E1?E%X#G(zAHxCfS$9
zXtvXD;p=s`le+Gj<t~+rKEZHK&wHbCUr%7pN_US(fAl8DeUAy$QS3N%VCs$LnpT-`
zT}}3Di<v(@YO>#~wB~~ox9S6f!#t~H?;gHv|3_?FbJ?WIg;B3uox00B5|7V+d_&Ti
zt1-*7>zkr`W_j$5Y{jJ~TBGI%b8e2wzxT6k$N!X$CB5_V@1^ehm)v2Xz4VW7!%L4<
zId5l5&&spX+m;Z_X#VhwPJ`^u37VaAu7zjR%~>&3vp!GO%XUxDnk94A=B}-(RFvK=
zVKk?;WVxLGEQfnL4z%r_+QbqQaBW3aAit5Ow>O*fROuJN79q9kdDlJY(|(iK9;CzP
z-S)D>!{LyEo<bXQlUL{FIHAdzZ$Ff*{ygEI#}$G9tP@L}zg*bHw@6pztxDx*Z#m5@
z#a51-MPg@GvP<9E+NoKk&J>hzSj|pE#Ar{8ZuARJzdcC;zj{45rFYzNVtY32_?n{%
zB_ao+P8n@%e^IDcw$c0hn(y=96(&_Z%<MVLkUZfer&{$S%l|W#qtzIux~dsX?(A_?
z3f?qB)Ie$N#6!P|%w{Q^;B;YET$&-GFvb5;OXu6)TvA^{P6-$my3e_ChLN?wo8?_l
z4i{tp?LF-?jMk)^t7I~3?^wL5|HmW?VZ(*G4o}T*a~u%8BGUP6nafU=oflerugJ6<
z*xAS#(J^N}_rw=AB1%_MI$U1c2wHnfo)^?)@jKbvq0M{A<=WfN=3f^~pOE0vWt6MP
z>7%$Js48hakLm&w&W;0CS3MM0`!d!RE$-I6*c5oiV<mUj#~I)Gm@RvgGK=_TawIB=
z+Zz8>tvNKaZ~Y0DO&=zl_t(!9)?sC6m?P)1e37B*_ASM3yeps0<7*Vo`Y$1+w}s<_
z!rmMXMrEc@RwK{pf-m_Vo>Ds*c=M3h_LYjJJF~c}11|F#O8Z4x8ch7N$H#saw~wR+
z#~fBk?Z}5eg@4_ta9(+^W&N*U2kmkvH^EYF^;MhG(>QhuT&->Gap`gD>{&GB#Iu#%
zzl6N~H!VGU=|ak8OV8NWkR~DSt|cP58LiSW58K}DSi~b7>=PwY7PwWwXIt9hh^Pfy
zRBpxTSnRBd@RT`xyS?U3IKSq~xW3&+o?6otr^o0es%`e{@F-V&Jh|_+g)wVDU(l2)
zPKNAd2Ttub`%-DS@~QI=v8y{Kyqd;6^Ge9eN%tmfJ7Ig_3cqAb;Ji;hYDFPNJ_$T3
z|AX2mJa9B$GvT>IyW&Mtk<Dy^62WgO<1RnA+<w)1jo*)(QyUszD{M|#uC0E><jcfF
z?yZ^5k|C#Mt!G>i)!ydXy;_sOI^eQ@sbxo=`>H_Iuix}%I$wBPB=h85Esy?f=FFdI
zT>D;~`m#$!`LKwJK*QI@OA1zRUpliq6INdN%;oH4hLSbW4v&mZ?)q@luXI&#EN8<h
z-qaO}^&SWI9@F(Jjx`AL;4^k+yz)V%rRe15G^5F~R)S{VjJ(9{MQ*BZeZ0Wilk35c
ztA`IWeT={3G-<zB^bv07ZHn!B*8?4TMO@8QG?%))xa<_|EHUldmX)PnyApLDw_Q1<
zu%*mj^TPExd;e#sUFZ2+sKn?T$Zp$toRvpF!E)Ks*NHxgS5$90&FS8F+(q2zbnVO{
z^GrK;CzUgPO-ARK1o9Za_%JL{+3qw?b!|bQMQ>rz(vCel4t$kL&+#jLbt{KkaD%Li
z<;%0vnU=9X(BrIFs`>B$i$%eMBDEPMQbiT*y&S?q>-F3&HeM8)rnF9O;vxCj6Ar1p
zxY{MR^7V!{ZgTeieNWpmW^8;UG=1^W(&f5eGK7s5T`w$~5^ueQ>6*^IqmJ(sUe7&W
z@tk*}e)&9^H|-B3j`K{;oiMR@0ZZZfiQSD=Un@+d_tcmy5q<c}$*%3vd(G=7@@5_g
zSUIOb+t_8f*MugYC3hZHe(L`}PjKf3$1M(A#^MDHS8rSQ1%0&&@>p=>p`xy5hDMaV
zwr~5R(nr}z!j%H18(r@Ix*+i;kbBXFrE>oyj*6^IlGEgQY4J~@dCx6H$L1SJB7**l
zn%Se3BC@nM1+Mt(IEm@iq9=7%*gBowEf!c`@$-ax$U-kR;}`vPKNiXV`_Qic#&MG9
z?&H(8{kyVm?mIDSyUYZ$w9}CS*S^@sZPC><JP}@-{g!Rz{}WRt${qaNG<}V{%FgVy
zWl35=GyjAvQ!*(2zu7SJ_KfV45{qs>Y7)+Sw*HWBx`GnJx}zC&rxXjcOrNRDzu>|3
z#G7G3m*Tc#L32(i%z5>`s54he>Au^+T^^01|F<bH7bdAbId<TbW=YD0hpU#BsQ5;!
zy?a;k;mMW*@+|6_F-zq7nmriU-E&m9y3$2h*v~X95#@TN{U!6T43DvlFE`6F_o^ka
z`<U$>EdIRb0{g)YKYu+`Ja($W<0xNN>+@g7uJI@xlRLzF`-8%&%#<&z6V~-j7Fd?Z
z;;AUMa^ed0o$mtD>duyWG^HJTUg&kqKf~#iL-M-{uihu@y*Juj7~EFQa7k!zVpeWa
zF}pBr6_?hU)xxX^YeW{wb{)U`MwKz@xW<Q*^KR(=*mmffNg~gI4n>2t?FE6{EbbhI
zEwXL`aVhQ`wS}6Sn7-Duh?q3S{aOBP#z&0`{qG*WiZc#r{%^?_yPc|aqfzXIVsq8P
z&qpr&JP^pkpTr}7NM7ua_@74YTL;8$1Tfs1sF;&`?7>CZmxW)W7$=tWI!GMqyT!C^
z9b3}8JjEL2ex^^a-dxzeV}YUCBJP-F{DLeuX1uw@aKT8%Gj$#Bt_{9_SDd}+rnTE)
zrLWwhD6KW==XjM0wBDWK?Ol`l<;H8i$O{Ue)-rfrcp=6#V~&7Yil0u|677Fkr87LZ
z6nHdry&2{;#r<iNo%hH#;kjD{)8Qo-tvwtIiqf^7w2IU$kxQE>`>#_Z&n5JRS{$F(
zMkOY<Eeb!@O%VHk?Ae3GvP|qedkuNc7Al_j7%P>e_jZ%s)nx7y53HKbDg5v9JNAcH
z=}F+VCm(*9sTqIK_{%g=@z0BwJ6v9gHF3!a{Sy;@rKVmbw|J?-ChxeeiRV~!&%IMF
z_u9B+O`V@-?>j%^B?%!jywZ-nVHS5{F-)$pQavluB&x#j<i@4>El#l}3w4waNIh=l
zjc^QI)X;T9v3Z`8$FW6XOPa(Z9l0EIOh4vnFL_{bOu;QeQM0dE)a>E+Ee&oajBa)7
zZM@%pt2xhg^|SffLY}aPKeiQ$PwoGC=;F^A{tUksihnS+=S|YHJSl$1nft@)hBpVq
z*e;7&99B#+TFuwi$EIq?{;^*y>37=eKU`_oTe|;#aZ<G843BCyuxgyR#q+<<GGVJM
zqyMKDOuE9aJoD{;HiI)f6K?Y*bLXi|@$-x-P%@L+sF-l_$*JB_uf^(iPxfqbyEmg#
zQRnEHBg-sY0*#F()?9Mi7p16aqNug}%i}+Z+7*V)EDT>$j(pBq^5qt*L}`mg+Y;Gh
z4SPOuKD}mQvoDxold<L&1<nJXzfV|j<;4`g*mXKTx45qf)Xi43+<QTM-v?dAMU7gQ
zUp8bmUf7@*vru1j(b0x=qJBCrPB^<<J#Tre$51#lP0IL{Qqcpk&kF9V1bmMP2)p=9
z=`pBC=4#VQzqamR-51|i=iDDFnMW5dv)B;)IqCgA<<C5e6j&I{-H%v%C3C-N6xo%;
zy~RlL-LmYqbdmoj6n-_dN9HC5Zfnx*Z};uYP<*pggO5pb3e)==!8|Ste{2x^`D&r`
z?}yTNP4!q6^lpDu`0`or_h!#O!Fu9O_Gbm-ew)ULCW)9i@X9U<yS4GGU6Rp_iz-QL
zc+M#uJMrZ8w-^0K);s;bsCz(P=~u@MktENYGrzCBubj5lLE)446aOVOOH<}#r>F;|
z@+;kvVoF>2_LfCZ+JzKuF~d7gT<%^8VM@+ql3e8OpB|jMEq_*uV{*v#MTgBcDr-(k
z;NF&`ZnE(4vJ>)?3auv=MDQ;1lIED{_E59!s?8}O-FPJzg{ICW7j?ELh?y!@EK>?F
z?g+RW^z%@G%tYn*_on)9LgW8`HZ%TU7Pt7aXjD)BorE@>4sM<@raK)6?wn9Q!nswZ
zoICA}Sj(amwYCiNuH?r}Arl`5JC`jL(KP<0t>-p9eB$BPcGq7Cxzre?ExW{G@Zg=X
zh{N)<m28<)Jy%?I-WH%ZOQg8eRIg*3=gH-Y-}tl|zIsep(7hwsWAXdUw~0Lal(~c)
zgl{acv2o$5NqP2x<>I0T>-S8*P;jw*-_r=+Lh;G(f6hAp^U`-aRtNi!!Etw|D+-h<
zTD@0z@HPLeVf_DTijTHe-Ed^8;N#0an<jLmHzb?sj^S382aR_c+`JQy-Fcw1u8U{g
zaZ91%zfag2k2g8J&baaFkfZDuZHpg@GA7RdALkp-ZV6{7Eb-oc(_rh)tYx<}ic^D1
zoTV3~u@{&bAJefbGEvAc_1k>+ipI3Iim-4$v;9mqlR~oBXRop;P&B)cBNn2z@X;!<
z4(*UYu6>JR_IPlsC}=J@z!kiF!3m3K8L!i~PMVk`3m=-8|8lyXXi~jz_+le}F0T&7
zcMaTT4ctj<dDI^3cr9Fg%7N+B0a1^J3G!P-h4{7#xff@0E2Z#lU!Y#Pcjct@d@0h0
zij>$-R9xk=G;a%(DDgd}80WRqDs89Uqmyb6mIW-Sd$MkYmAS~?<-)b*OEzfh%RRMj
zF^%A4yJxpV&ER$2p6g8iS*Kmo$Y4CCxZ&u$2}#_q#2x;>FXeeE5hvR;OZm_&-bZzB
zrzt+!p?FQy-uQ4le^bEr1Yx$z`m7Bsavnh|GmrB41#}$dJCVTjqasmVVS+QOp{$bb
zp+n|U#@EZ%C>%T#eDb~ixkLVTl}lXG{?+v;uE<VhEl`@vou<3#)}MT3>rd75dP?$=
z_V_JM-_@-y;We*m*}FQ)_i;vnUtRTAby-dmb8k9wFvPI;wy&z|lNCyb!Yp+vw<HAa
zyg9p3O6gfqU1!zo`!_tVm5NT0nmzM~%u%O?)tx+dGLAK!<Gpo&FDH%Lk7w&Ug{=a4
z%0lZ}cqV7;)%z2sw>U{cU!bj$hiTg^y@iI4o9wr?$|e=b6!o!g4FA6;U7%RCXxZW|
zIX}xIojCK<PR0CGY%nuc$(zQ!aa!sZg|6*EB26BX%(=pk=Un~aF==PCJX7=yU-5;O
zdb@rue`?;R%#=SnUuJ)$;++1tLYI}dx5+t4dmqa<%2yU}D&goY2Gt1ehY2|b>P?YM
zhg%q4lrvi^sj>Yz=cFpLbFFAn%kPPLA2_@OD*Em$N)mgfufX<S;!sPfTdQlA;{KL4
zY3A@Z{;5hQ7Zx6MVJ^`B#GID6?dG~Z;V`EQWm`Opt{<Fa<~@bkjF)MZoq(3!j48X>
zW=f<j^L5eJ=sEQ1_q($6E$+7!-(9j-jyc4?sPRUkNLZtcbcbBV!j)l)^{*2}|9|`J
z_3XOanjPT+d2I?U+&Kr&{)h<rxyj(xn&-C`7@lgeOq!H&_ST}RY`qrWr4jFrT{v_+
zMC-Ugv61Ydjm#5M`9DmY_UFXDcgbEe)7-W#bNalmG|E`ZTSMaa-+#XiX71Vh<xz5!
zV?_B(o(o|eITPRQ;VG@+(8x`X+ka@*(wlX+x63`dB=azgAtN!!ZFcO+!-rokln!sy
zxUn#<&_(3KW-*sW(P{c|;f_2n4$3?(*VDhd^|x8v;un3dFZxVLouOdfcs}w4&z0#e
z&r-r>9lxbyIDhG!i&e?{7a2O5wcNhdXR_l}9k)Tz+Y^RfF56p#ZAugsoo=x9ggT2?
zd!?l~Z@KwDewtF@jcJ=s8#@?fRGJ)geY|2`S?-_PtCW)VFPw44@%qx<DY-eN8;yL^
zuEl7+Io4~)9QfOD<;z6H7mZqZ={(mM<6bw)ow#^-^TybkjdB?eFR~_xE>Jq`>Uds%
zj_4EB!_Q?{o-S?Fx})$cTv3W6?px3cu|?1G9tLtw{<Wd&MbEeDKS^r7cE5jgC~0JI
zy*px9kbPqLg`L;+Pj2R1r!Lv58a#W;5?#BrGaql6e*aZ|+~!T&_x%?Rg}N5Mld{n>
zYxJ2=v165ch3KK2Ys;$SS*_0Tys1&DG?NzJ+WPL2tdiFIuM$q(H^se=O%Qc?c=)&D
z!{3UQzY`xQ#xmSE_<;AH#ecSW@9h7?%EU5kE4=hmR?Eu&e(jM5*^M%_jj~4+E~Xxn
zVZE~a>!K^NkF1o+c1St%H~LPRy=jhi64$g8p(xqL&YDKWomUI@DlWTisJ*P3;ib|K
z{!^ji0={0)^@S%(Xr}!-?P9xgV!*T8%1XB;F7^>fW)=%fcWY)`D?9D*Q|Asd=HMvX
z9cp)GdREwqzo|Ryy5g{v2jju|dGi+u>A&BTzjUYkwu!PQ3T0gu#+v-Gc#*jBrDDaO
zhkx6bysHj8TwA~)I8W?aqo`TU5k{xrwRKEfVkxF>!mF$P@mv#fT|LjSlTABH$K%5y
z2BYAdotqYaR65b5%sPdoa`_@{#<jON6#gGQsiYpnc2eX{&_r&(-ry-7f*VqWr)x&n
zh0Qs%bU|QqFpG}v4U?>cOB9!f?d>vM&Y;CKDQN4DPzQ0w)Dvm8-C@Z$wnQe!l|)~D
zaI4Ae<-Q|cn-lJ>irJwND8TUHs!9&;q^zymKe(E8iN5#EnJD_WYwa!04KV`7l80C=
zwF4%JeTn4Eu#{%8JvF@{PAdEA>Z5u}YfD$(5L{!LH8rn;DI+t=b^SFh#)GbFt8(|u
zdms2F;^i`R2NCV2H%B%I8+bA7j=J#r>*{^0rX6g%6?suPe?$2GHc8!W`Sn-DSM#p*
ziYrMLs842G7j<>*j~NWJ6Ru7P@|(yuHGo-c2Wud+Q|SMTeoQ_JtbSr{5f|B*G9o6l
zNqJQWhseZC5%H^id*ymv=)t8IBUq<SUF4c7`a~&m&J@#R*QFc`;XZSA+~{<440{sR
z%f!;T#MS!UiloVHl4etwJXHi3rgF9{Ip7)|wZwr*$m60{=o$%=XO4bm-xT6*gnpUN
zu;xb6#ENYbgbvP>Qp`wqOb#d!)V}oc+ij~YYdU3%|6bYLrSbKG@hOdeUxK^U1bN*B
zmi9QgoG+i+cZq+=q;H3BX)U(6{?hSz>=wECm2oNlx<OkWa?KH4nsUKQ>~5mAr})++
z?)5sVZo(-g54UvshU9L%9PFiP$QpFwUy4YFx9E$I4yFL_wK12ow*J=&53A(9=4u&d
z`fk-TtCykS(krV&eGE>eicJ(_-EmPwm_=*`182_8rLKR&wz|&HS-X42+{v5;1s=I#
zQ)CafbnPri-P;u&DBaR2Ce-BlpflZ7DB;e`<E(#gZ*&vPF}mQoca0L8?cNK<$24BK
zZog3Q>g#5|)BL?kg0@L(F7uzhuvWRD{Ce-Mb52hURRo<^U%kwI<u&Jpmy1OYE*H%1
z+8Z8d8$BV(eI47}C4x7%ov8YDVC}Z18L@1pe}p7WY%RjK$!RMd6id;G2x(>tvsMgf
zN`CQOL?%mRVT)%W$0Enx=vKuiqP;tm_M}ghz7*~-?O;;q+!90a=S)*3DZkJc6Zx+a
z;~eJtey8Hwt;*uUe^O^ThGtDk4V}#qBk?|bYUkA%2gIN3J{-v!ec_;4V9v!ks_Ub6
zcg@LN_{wG9MW$66uG!OP+aK=r^k3A@r_PYOXouO0g<IAHWfi=+@_38AUvRFe<Lkt`
zTimWDFA==<=fD5fk3XEaQhs=Z9Vz2xJkb@Nn5B?(t6&C)+>*q~#@iYS#^R}xE?g~&
z3VvEkbW(5fYLq#jdT?l}-u0ENaq%-xNAyfte$KNmI(;U0KhGsg>2nhjY*K{7a-4(}
zqEZq~UUJ{#5whJTut8NU@l3r~n4r3qX5#8ai@b9hCA)Q8O@us>LMQ#$*p%VJFz4kW
z3EfY7H2%9daBp?+cU^V3uZ<^NVdAX5ilWV2j)BK*w%qVrmA7qHP~h6EcZMDRbso9(
zny5?^b>zRU*CcoKMJBhLW`}Op#ns+%lT^26xT@bi&=s)B(P&A^(b8?7XN7d7`=m5T
z`)d3+bwX6dcap&b32O<F1v!aAOb?#2GhSDi<KxIBtdRI<!6L=ovVAU03+{+%TtAa0
zC+upa<I{g6RqTx9lh#*<1YZ?Yh)K$Iy2l3GwP@b5<l@TU*5^t`146beRjKhcJnDS<
z!x525CZcK`ypylGdMtD-+tg~X_41+8bzuTg4lYWo4f_B5`k_&OAvy4qGQV?>tK6TL
z{tIMOT@SFSHTX|r`rjmw!>WC2nM2&wL*ic(GF@kGJJDp)BJw@Owam=0ZTf+;O8uKT
zwt6({^jR!jZZ4Vl_+F=laM)$f<2ivZbeIlbE(~e4<eAR@cG47?%+0Y<ot?f%R<7^*
z!>ud7wbOf2x`39Lk73qYujq@HmIZEEC@Z|#Lwd;sh6xK>li42L@ZYngl~=$uKAHEV
z|0DrM%?(YqVxCK{EpgjAllN0#`;}W+0+o8Fy^2;nPHYu9CUELmj=IG`&!>xP3io&H
zQ(qi4JIXPkSbXn<MTv&ho@b6ZbeUZ(5{|x+>u^$omDSL-+xKQ_$QxIugRd?r9o-}5
zH|^%sWpUhv7oMzq`pIOm^wHPuZ2vW#gDeD<XTI5>qo;J&EP98NbZ?&9OPv#@Ul$3g
z6_$mDze*Bc_+m*_mdpNHrC>W#qXm05Bnq!lVg1a<Bf?;_-cxka1d$yJ7+7LgZjgF;
zJ<0h|)Njko`qqb+70-V;!I0J}8Wo}A6WlG>Emc(Ay3$oc+i2C1p3d@JbM^e!mEKs$
z#j0U1ndtF{ZQ*i}>n)$Rc6DxBBv|0M^~4*N*QM)!9@)Xr(f&g;DUvBfPkpm1(~0aJ
zUx~}twtL-N`@cpg?8>RtWi2bkB9FXw-4h|W_t?&;pc9kYc2%UCs412!zD{6SRFKGM
z@JXlHt$!h_Zj#ICEfXZdE-o;yKH=EETuk*_`+pVVMT!4fEtYlXb)*_|MF(~KQcXU!
z@{WfiUtQSJX=dxZ3KrfF@r+mzd71T9g7&4D6)eThVX5IJ+(CDbymVPGH+iDDzEJ2!
zol2e57cJ(oN0c)1`R1}seE%`&bjr4v&hv@79t=k+td~z{+@E&p+qzAQ&exlrQTw;&
zgxH=ZRw=QZ>YM=#jIU!QG%dJ}Xdm;kcQag3H~V(X*N&M>Zf5dZY<y&J)ACG%+f#$U
z*OS#O?>=x{;CdmE?Wo!bEiaE1tWOqmtaJ33dM9Z166ebkT~FTd`F3FAqff^&il4nY
z^gj9w&%uh+)<eN&9y1Qixt5)7ec<3?CFz=@drmX7KF>cFk^bTT5@GAfOe^M0VAylh
zh^^X0YT*i@isYS-XT9%WRBGBXIrVmOWSw}*=KN&C+kfxcH}9!^%a?gh`eupaT&DbO
zW*a^Inw>(rGa9#N22E3`dSh`f?ns;q=bEh{U(Ref^K@_ILd*VBp~}jEtR7Ew?}pBa
z6}S`_9hcD<yEQ&;#x2hTu3x(MdXyi>p9)!8(X(^8+osLI-4E>aD+@Wb>nlo&FV61q
zRBYZOs?6$}QgU=AUqT1>BfBVv<>re|@h%fNC9~RerLD{}t2?`&eYHzXOuK9E{dsS{
zX<5t3W!%Tl<)l|nk9hUDsam)}jN$pKg<ee}%uEj-O%*D#X%+D*jb$ng*>~)}^SzRs
zIZIQoO%db0QT#GgG^{N*FIhb%z~Ak<gj7dZL1<cLVb~N#^{-nvITNJfN>t-Q+_oAs
z22W6p{G=NDMKUeKjme=|j9ujOWYd7pQmX}}FN90~6|if%q`^AT_{|5S8-iMmON=5v
z*Y=b}PH4$!KPD!5kT0qrus1|hUaYRW&Ge3<fr%@h@D}ky##}|h=GRgM_?eW}s4+}3
zs7P)Ic38;y{kvn(1`$PfAGV84Cq5-}WC&cCB69ADh58rKM)#_S!lGM8EoU5OHJucD
z{8;ScExvh%%AS|v9z9ilwZ%%Xql!r&@82O+rUU8WLNW6z)OK!<XJd3UnOLG2Dts{g
zzh0Pnm-TYNpagEkWis_jc1#L(^NeKgACz6QK;YCz%~mDB-Xg6P8G2hMu--^kZx4wI
zZIm<%ZT!rnQ0*AqD=6~Eob6wGvr3BKB0~|gmbleQu`v#8I__d2fntGajZsH?v`dQK
zsVd)X6^*{CB66&REiwObn4#nKMvlwQ>C2SN%GKsw;qy$FToMrR*Te1n=JeXDC0-(K
zfeo$A%L0yH<X&vRHuKR$A7i#R%^FQiazbB?`yZ(@JN6_`DvK~IpMS#R>jKu_-zL5L
zn7Q7OzxIjLde#ETjgxMr3+k{&vpVu|boN#yW{Ddr_+|>sQec?1fi<8rxYB&8#X{B!
z3EfEw|3z618BYYoT)3*(u*110ICfKuQfpx94~<Y=Az{Z)v6@MKoiq9ux_6p~R4q5}
z@0zIkStVd?dia_hUFnXFJi=<Z423-l;~!mQ3um90&NQ(=jWuwBk@!T7!Y67>jT(wq
zHBT-KxU_xZ`oNyNAZgyjVB4Lhh3s_^hg+YP3(73xv=Odbe=%USLapm#^+k`=?;hq`
z^gv^((WFUkg7*cZKcw5NaYzuqQcxW(9H6QAJzQ{>LVGoHy?BX){zsh)k&1^yW1<^E
zzDndbO?C?WCU4m^rJB>vI;H<iNTtw>R8~osjE^&tz62(*l_YHoll`i4$f$bj1+AHh
z5^5FR9#<rUfBu&g-<XiD!ceqKq%fq6Q&o1sB;%QrjMXHK^_8`r21`dx*7P?vUhpU?
zA~55z<D!g@nW|ljK0f5<yyzB{G3!!d`QidO(Oa53Hrnp9H03{N7qX(YW>UF$WAqMH
zm-pseE=St}I7LDu1(#iMbLkW))nF})(5<*s*s7?gY^2oeG0p9o=)+bq%PZvt2U#bs
zD8ADo=C;XdLHqPauDO}3re|m6zK9g9IhMO*1<Uiuyu3%nQj9hYm3j5ErZ;mYcUu~9
zYAt7C$hb5^W8V(0`JeK4F-qHV+CP6NE2%i8{gu_?hE_#E&EtypUkzlBtuX0WG;@lR
zK+mVy>578vTbJxe@2$%EA6&^e)gwzV&tf%`QS@?4`&B{QCP$<F6<HU2OmI<ae6AGR
ze?2xaaNdknPA9{}{KE8}?^qUoQ2+DJ`QBa=eym&`*i!XJbB36pnyrQYl0-N0z^>9N
zz1Gr+`igUGB4uAc$eg-Lj(4F{&(#*0rICIbt-&3O@>D#(J>cdH=ie5<pQ6C(TP0ZV
zBy;IkBg>`IyI#r)zsgjUoIL3X{}V?!L(U~jCIoW^7J67sl~SC$?w8beVWGgs+;1+J
z&)&co^g%VgQ|FXwk<>y(-If>|SIes+6#^Z;)2}bJ{k{2=o3h>%k#lN(*&CG&H!9D)
z5&H4#vKd@%2|*J3%Eg|Pi7KwJj$HlUe^0R5ZfBcE2W{3er2BlEVZYFXRZ);-;<o-E
z{!4+jGRM>lZ%1@Kn<f1;v22n(Qv(0`jk4;cL4qw3jow^S+qQ`Yy5GAPu#Z#rmayEm
zhbH?H<&J#p4xPA7x>T%kV&3*utel2i8!ffg2$)Vkm9+S_K=g#EbsQGWOD&kE_oe*o
zd%-!c%dxUkLdEn`?7p@*&p>4-7T0T1im%(l9Jb5f%4$?h460wbUF5O#ldR?i5BKam
z+@G>!0Z&%5IHM8AqHyntQibjUVn<Ec%M0c%%V1itF+4G|`w6$<)Ag*38#vWB%)cV^
z@iyBhZH6BY+2j+s%O>pqd0W29ID^%ki&@aLv+2KWua$sc>o&8m8|*XJ&p4W?cyRku
z;lk`o8!!DVD5{8F+~g&0ow26SBSnC<<JTtPOBLc1vn8E2U!1aZf0*DsC(%p0<P%)u
zbeZ%Gj^yOb^o>cK{&I?yd5g_ViRCMf>u+mbm+>VqC$ajrRrRw&^Y5Lq-e=C0qg=gQ
zQT2@0Ubc<u_tmGwbaWNI=H@=hrI!}eB)ud4w1eD4&yAaevSYMaCpuUt?qXHcx+y5V
zPttu+)6B)uCbtu%6;B1GUeZ|Xl#usn=ej`qh=Yj?qPAWabX1RvcV*~VZCRtZu)izt
z<crMRtQF4Zx(+LToUol^g7=ykQWwJ_ywk#6Cu}?_v8<*y^x=P@T%mP-;p@sS>8sBW
zyZBjnR=_@n5CQX(V$-%|7){;xH%s8H_l9i~3g)To*A3+Q{8~s+!tCOn{lBEmHZKy&
zjS)yn=C)$o5b>CQ%|n^&39@?3T8&RKA6G}!<p}P3b6TuhZq31kt4q8tJ(pVOy>HJP
zv40P=N{#xY8K$jG6Euxpv&U?j!^F*5Gla7bZl2C;`E<$2xqD(~2ru`MULodqv3#f1
zC($G2E0tfoSiA1<sdY7CX5S0`2ioL5*s8#~P3K`+SWe*;<LxVd&RS(TdzGPKV$bE|
z8VPk*o0%oYvSK`Qa<o%|1vDo*><GHT<e<$|aD_{Sp?qq$u;cdjB8{-%oso?Hw+Go?
zHa>f_gQvCi_qGsU(={7d<Cd@-oTB?uX!knfdBPs4X@!SpWI1!&u1%kqGw096NJU1?
zzgJe?ajCBr{IPUx{Il!KFV!llL?<{3XEO)sIIfev9A1A=jAvI9OO#;3lRc~(-E@B?
z_LOWq)U)qZwBY})Gsn-Eoqlu1=ZMgE?Xy?T>{o5Pk~8P5%vEiv4G}Zk&OFmzv{TmM
z;px3vP68j!G~Z#IwNX&|a=WY`!%bGhX5McxY!4MRV=nA5Q)0?3DhgEM@6~%^B$|Cu
zeAn#5kCptS_WC`(lKOP-Ma759Y;&epDlsn15Qq+1zHpCx*y<yzi`v957VO#{7I-Fp
zZ9{l+_x~<d(TsGV?Rsqw>ZEtroE5U0do1&eyr7<$XRnY~?!#}10^Yf2l_rX<V^Ayg
z&iFA$U18nTptrL39@=ei<W9_)oyb$`6meDjDgU<%T!98|;+NYq?`A)%P&;mAk@BD~
zuXb5`*9~VsWs{mGxtd|3GYzeMZF&Wp6np2b=UEr|<%z?_bPb97nGbg>HX0t96`{MT
zwdnXh#^_rodG?>P31TYVzvl0){Jw{R^UnU-DF1eu)22<VD<9r|87QiJOX7e}d9Rph
zAaj{@=AC6C1sfvHEBw;<d`t7ynV={G)oZugqYDmYGRAf<7POSknPy!1a;E6yJK2Wi
z&P#$-Zm0Bn{XWr^`#&xF)w26r=Dk=eC|Ro)&*ar5cxhSA{To@QJY)83E~(C8DC9TR
z;@xtoZ?k^5PC%?)*AfMLoi<lay<-j4bF<5i*~?xL>bs&g@!_3)Z#5S_OkOPb;N081
z90A{0!NRzAp&PGorf^4ZP(PHp^ZBAcU*^Xbk8{;-dGt#zXlBtP{iRXuHFdEPQv)3m
zMV&<Kmg(87a}uAl`%dl2)kfEu-&(ovPRU;TCf(4#v(P|l$EoCLV*C8|te-4uSuV&O
z>Hl7BqDRYKp=0S!s`loqss2uCGMWFzME<_8q`|p6XM9-B$}@ghxAtAG%|pFh^^^l{
zc~5S)Upw&V^_Q21cb0_S>8J}xIQ_rb{PPP&`+dyyo1UF!Q}pS&s_n*kU*Y0fiT2>T
zuau^#GvyeTIM2Qzab{WUY9&AWJEvsMMxNml)tEZL@6qd=z!`VE#7(|l+4m`m?_*w7
zy=aM7*LG%!5F;bAi2OLAnq(Kx>wU-eC|%i}puM9>h;53{{QNIkCERkM5Bw`Uif$V_
z-QeEXc=`9eN4E~j1`0oBynnUzcInRbtMZgy3Ox&ETrqFX)?HE$V{;?*rkVeW*V=N6
z_g>V;c^+5#g-_nOz4FN6%OC4xpSJp*`cNts_C!TM#4zlf+Qnok{>c2v^WL<G{(L6&
zZN8#vBFoJhRtuR4`#0M@+&M|RtWWIgeVY<K0VbCJTpNrxJaA~?6qVazdSR+`Bd@a7
zk_bbkBVj=iKZP&wFI?Zam#s<U#_G8JRZOq0u5$05>m9Z_V&iO2Zw1?FYQMj}zItBX
ze?F7PyCoVy3y*in2r9|0*z&@Iahr+jf*B46S~R7XMLd}`;q-y^(jr9<13s>1&`Z}W
zdSx}?%mE{j$4YMwg*k|~<XEmeYJDM3a@*2flNM>r*l@^?Sy+3^?0|yk#$HA44XXl9
zEeo5hyKUi#%1zH3*C_0-wT?77dWJJN-zy+;VG{#WqI!pH^#Z2VoFW_(4%{w(-?Hwu
znxw_j#Dh)cEO&$!#4IR!E;5mSipK@3&o3_vy3M~4R{6E<zUa21;Qwls;qNX@=e}X|
z>iK~w9|OM~I=SkacbH!0+9J&cx!NgzR=OXU&-LYn!&w0@O-D9$>DC=xQ*I=6=?dK9
zTHB%};-R}$YRQU2T>H5M54U&;H<&O!nswS)EIMTMM)Bw)*@hdk?rd;nT%w?C+IU0Y
zz;QWtrb}xt)`<ui$+sU7Txb?y7`PzpmY{;M+-*f=PF4kx72EHo92W9s+%(0Z(~vdb
z&>~~aS4KXKTWWHd!lz9sKDT05)(J856Au!&_@Xu{pR>%pnbb42V3E81tT?L>hG|7i
zS?bK(A_p{=&2jQj@ir~a3bs<u^->5^VGve)u;hT3PGf+q$JG$lgwm^xrzV9q{NHpb
zsoz%5<c*2Z{=ATnE(wu;OckXvU0xV4PN?-^I4BjpO)0tDb;YK8G2RNDx)Y{~ExgR~
z;@j<9R;wNZRpC{6i=sRxI#>i9PA%}|yi@R0q4|z)hh^i7uN!)dq7H74YrONIC*GDd
zgE_!bTFJR9?8Zm^1C!ZgE-de4ty;f`HS|^2%54_QG#+=&V5(GDX5)5Ze$=7`r<63+
zu7oyltG^EL<I`4pyvC@2&&;*DdfSTiQa^<n){5pIdA)i^hg{W;E|nWgdJoT=bIBvW
z{fUmTyrcA`SISGTPSZZO#Lw4DgLT)XMW)(Cn^&n`7Rg|;-yO}7(4uWMeV3Jd&<U|B
z$weHA|HCb2ENE%*deCHSzU@K6y~>j_8hWf5u20wgprqv5&G`4@(dBcSqPmnOTUTWn
zEso-`eZuzoVv~jL<vYGj)AVa9{X#_dUtAO0`0mvjzCym7qk8+aQuPk02rrmCyPZwS
zpz)bdPFmx<w^OgbsLh++legBtWAhtcIS!KmQ7KkMBQwd2KPQcpcZ<EcBH9uAR`=2+
zQ&zrzv*Vur+NO8kikU5bfms2gQtFHbhF9O#*K_v7<vnmZ$7aU3b_!#_b`56sZ3*pM
zMF+(iw3=B|A|}>veJa+F)hv>wxT&|6r{rU@qLUk=Ot+6hnXuNm2#*ry7Z)?jLNeyL
zc8Tqmd1l~vY;)rO=oYcHsuLH9-g@LCyz-f?*i>%CpqfYPW~ppj5s<0yHskQETUYcL
zd%P^No5WpRrV4Nz4CrN^$e^jVUByOI&+n=xGv}Wt7Dh5lME`7x*Y-HjYG3p~dz%LH
z{8u|f!$l^sd{(;8_QSN5|JX_%=?90TBd<&d%V-ptDb(R^ame%HL_I~Bk588OC3&4_
z_*BgMh{Gdd;U@h}nF=YBn!*G=tqR|ptH7EwHR4p$gs8?0KbICBbp~Pa1rL|%Z+o>+
zfVIh`yXNi8F3WokOO{W_E?Z@^Hc|QTlB3Ek4aO$1FXMg)CE2k}RLokkg0Da4;XbiP
z9!<|Z1(;S|?%L#Vgm>)**86<_pLdygH1HH&x-8~lz^XTay(u-}!Q*!uT6rflHcIb!
zpt5tq{zetPG?@%%CiM+FVsARRD|-ugnwrh*nQ7v<{Bn<%=B2)?rLLUP9kEVHV&|hp
z3l%m$mV8%bG4tueDSo@8J&bI(v|iEqs2jkX5Ps;{UX@bbuArx0I~UCo$P9b3>V7xZ
zO(7*#lXFF$pI>wRIS}Ew^DV#4<nEZtTMlxJCR21$I=!{IGTOreMAi%6I26sQ?zs2X
zB3|K$B@a$HF&WiN@bBsIa=f*nl`|wrX`=8F>37!_wS}Az|ED3_?d2}iy~1e9ghWkm
z=@g@SkJMDLM%gzRGlJQk3WaTW+O@94bo2jX0-Lpq6S;f89XiwU;oI_KtGG9JJQsh#
zyT0n&ldjDxI?q1emmhU0lq2ZvSNn?z<*R%?T?=sxp0&m;x8{X#_?1&LXO=gGu8@;j
z@$b{q<z`dZo7=RT(>XF6xo4eiURvPHD5=$W^lP~nqwDFj9tSQk{aVc{wuM<-Lu-Su
zXba24T))=Y>JH&87epuip04`XN-}yvA~*A*$(awG=STffx7#8yzw>&xK=lSCP6L?}
zx6*DIaUXnBW-h|v-+1Ejd?C#foWCA+_x><esBhgVdNELY)))KMJxASxvV(%BYo)AA
zTRZ37tW8;)R!k5IJLVf(6z#O@ij_efr%~#%!WnuMH-!WLPht@;eG$DZpvgt=8egBV
zg1B0O9OD#*NtJ&;%=@M(#<{C;f8%T?=|>U|nmCK*RZn@5`u4!~U9wB^3Um^6Lw<Um
zEtwW*;mBco%%CfDL6ykz$aaYv2Uq5{@=cj@C1yp0_^~aepF=jcPR$WESsmT@V!COz
zVuALj!&_Fkx*mIz>DqB~Mn>x9?cTy?Hf_2&^#;R(+FQ$fgqBRWTF`lU4kx2{$^}bT
z@kJ}#y=*3$<R0{&@$^{SvPr>AW(s+Kzj+@>U7*l>(&1$EgO`gtzdUIBJLmYCf@j*f
zJDNLOVy3kfFW@<qdHBflr))p-Zdgt1_AAkO`LJ<GdLBpOIWe<El2cYZvHS0lc~Zk_
zc2?j9e}%0VTI&Be@-t>MiRxao>=YK-HEHRGDQniWie%)3HE!WKZWp`0|9#VDKEbOw
zu3G~)hlivnU+&SqBgVkQ*3{DI!D#YeN?OQn)>*p6;%B1md_&r&lof7G&Azv}HM~-V
zb;<K6#W<ei-pyN&7^`Vq58m)*srQ#1O>a(K?2L|)7CXJ>#I1!QHNF>{nZ>VLeB7=m
z`h5C)sZ9@0ChgkD#5gC-%(u!q@JUR+Dzm0&k^HyLC1R_eozyt8fvf*ZlIXg$pAs&H
zU4bX|$R)ox;eVsB`j^0d^+hRlPA+#Z8M-VqO25?WbS_1!>h+Hmq3b8;E8cpX7{2%T
zgv9^r0?YR(2{ZERS@XS%eIhh-f!gYfU2C72+-CCq72I+=nc;iIA=cEhH;i%>^}7^W
zONAP~&YUGTd3$c)qBf1q<(1NHtHqCNS$r#4VtqjUj5hD48%&j+bL^|QD|fd3NK)Zb
z-YeK8^M}(??1{WMk3@?FN5_mF$u%5RJEe^-v~+uRgnroDxS;*x!;a7sEy15TX5F@I
zT)`grfuB)=b&*7?;e-Crh}IdmJMVwqxrTeK+7auYD-9)YHAxC}tlc4yv{-ljkB#3B
z@6!1)L3Ot9dgi9}-Oefnv%(TsR}?Cp-*8yVXwv_QvoscKt(-J#=VT+XLLvRbDusuv
zIZ~8aBb1FB6^;Hc=Gaiwdf>HnW5L{Y0Sp_PjhUQQH)}E5D0c5yxkD&H!|j)F$dlO}
zA2wK6?ltuh2$I-1-$9^pL940^Yt#eIK!N7TH*6np+f@lLtKaE-ytuv2!mjKCN0>mX
z#+QC!p+j0J6E7T`66mzFZ{v)xhNX8;2$?j>uF2qz+r@QOfP3SL3Gq%Gw>iyt0=bu8
z=1DW!wqU2?*Uwz`G5b#z%!|9&{jzi2k&J1vo=j!cM+>FQ8Sh9cTohBa<S|oVy%?~3
z?@djH4CD1D1uZLOERq*AD>^nd^3`X$_MP6qCA_m`gUUn^&&DkV&4vpvB(gt<cAK5R
z>T97Gb%7=7fMsKZQ<Z?gjQ=9MGq&t}9L>t~agH2|*SZ43h0#J+75SDpFV<hPva3V#
zE7ziVJJ<3UN-IVgZS~kcd5YKuX^zcVi<e0m6f#b_d|;M3Q}6OwyHXg99oDScW;1Qu
z5BYN&SSR0Ny1wGb%UL}!lbHhUFzl=5-RQ)z&*tdFkfYwo+E<)8!d*H}9h`VHW#Orl
zOTuo<>C>1S6>(^a0Bcyr)^KjO?-Cq_0<AG~_G+y0ycp3sO@Q^z=_yeU_|M-w)qkPo
z<!7P1O?GR3xbc77Iyb9(y5;dzF5PBl=9V~d%-!g<HOGwSVavXNbL$hjXT4-u8LXG~
zaRSE$rkc%i3?UplCR-m;IG|=C$eePfC1L)V|6UB8cSMZ@R(`(1!&<PS_qKfZ3chNg
zj{27t4W1p=GY<Z#;cHoOkmcj~9xik13mj1`{h=3FrffJG_+n?>1(qvMI_BNsn8tFd
zD{;!S3ezjA0&dt$+wU@2^MIYKw6j>Ck%`L09TnU{p=Nppf~w9M)pr6cUwU;-2xOFK
zUh+unyaKDn<-?A5=Dj_%iDjV@|4zS|-0F-8y%CqJZ*usY@L=lVvXPRWdOJXvQGwAy
z*gtwlv-VNxKYwgaRQO1Qa)f8_U)VIG@5bIaA`H<vP7@a#5)SH|A>mv0f^+$T%bu0K
z0mXY4O6#sWArkR<ifyau*+llhzy{YP9mau8xA&}!RdkR0&%G?+lxV5UM8UJimnQD7
z30Wr8*(3aPW|fNkS%dv)lda6BG=02z&OlUt%`ByoTT&%O9$g9YMh55i+?iTBbF%J@
zOEsPP4v)Kj`Uo=45}w2(KB3z2n2d-{3xD*IR@OxMw7r(oZcOR&yyo{wax&N49>Y^D
zA6PXbW-i^bkDt}~t#r6yioKDDz3eJ0K|^WYmm%v6CN7W?d*XAI<>Ri#4c9jq3$FX=
z%sOY6ilayKpTLsW7Zn~(%YC@KYDZw<&S_GMJNRNRMFe$xU7~j?;gU(DOc;+!hm*G6
zQ4Qu*2ak8@9QwH7*qix3uJFXB3bgFhF}lFy|C(djht?kkUCci@t#i4S|JS&zdNHiK
zBeGkOGr;)r0@e`54eaq7+I1E9eBM}Z&x|tOc~(aI`Z^Bx+g==@g5B#i7Q}EaGybE?
z#(8e8q2Ssrl8;K<eMCF<HTWrU$+Z^9J#*y9D7um3DK2yJ+&Y0E{>0|c;Es1OS63?X
zynERddGs{z-CoWUd_Pna>UQeaD|Y-!TJbeOfw{Vyi)lk=^ZCCmj_Vi#&bIQLTibDB
zA;-C^^P@YCPG+9_eaAJ?LxR&c&HE8xS+;^TETJ{@c8lQ!CWAQ<?Q@s%|8?rFSi9+{
ztWc~_qR=u~Yi^OdGySH9SIH`MS#Vp3a9Fe)j#lP7)3E4`;%S?Z-RCqs<YgmX-`o`T
zKViO->wy$OVY4oQnWs!udoQuvWC&+ju||RUT*Za}5fyGF8Ma3aK@~dghZdap9Q4s=
zHE-+YtCu-!mx^38G-SPYIm_ho`Jcg8vbxS1sn%~0nP3sMYfWp}io>ls^L1~u+%dIU
zJljzCnOkJg?3Nw-Tv?CbDG`m^JuzNr*~7P8cV69UIkDfKN3Qg?<fA9D^E|t&Otcn7
zscqSMe8=7^4-d#mJe~NOB`DOlqcvvHfxlOlEDBvOaw4W%{9>uW=~hXek_|SqxGOy;
zGCXQ%dcEG&=%kp;Nmb3T?uO*=QGH<<{ogruojh=D+N-W9kCJ;MJEtbJswT1CJ{qfe
zp-Ce`R@302>whs(_jw!d{|IXd2)Y+)RWfDnlGTb^EW*9+&h=W_@kDWwe2}w7UfhWd
z*I&qTtb5F>s?_sZ<#cn4Q0t$0k#XC1KGxfvE3`+JBju(24j%Pyg3C(^Q`SvLDUWp7
z;o0B(gX;w698L>~wHww5|J?ldtK1P~lXD6;3p{Un8->04!Bu_Z;JmK_7qp(w7mW%j
z;#$0web<`Zj%ux+7!#LfuN5jwF$!eLJM!Fnj;iX?Cz__hE?<vna=6cS;%Iuub$5^8
zn%2Z`9Q#+D?XJ~%n)dsOxaw1579$zM{YPG`-M*BYEm0!m>r8dM*<BZArAW$G$j$dH
zI1%_`))fiX3mTU`&7BnOeO2%OM)`AZ6@E9auxZk$^Ino3w;_F<a^o!Vj#~^J8=9s5
z+|h}7)%xLl(~s%XmY&JYj1}$TzcKsnEm>yY8%)k_Ph9g-@{J!FWNnUq%A2)unr@=_
z)?<RIKQ44t%$$*U#n>mXJo36pO<?f5nJf*?8(w?8{*<~gr`t-#qlj7htb|aP+$A?d
z<$D!3bQfOQ)E3cEDy@0-Zp(y*pdZ2kZZ^>~Hn?grA60C&G*;ko;yz-v!Mj-dSdn=6
zleG4Zw?}+Be)x*%X0&(zyxheQ<u7^5|71d0TqobWyHofc>Rw>cRCstzX|kkui1!yE
zW6#-3d4&ybx|umX%$1qyJa^q&G3_^xRR#Yq%@&)ucKr&`D93Ky2_cVVmpzDe|Lz@E
z_f&F^!!q&ONuoyul*;V4mD+DfHGAfpv~B{!>#RqO5-&B@brhcLF}u-nS%S5dS6$O%
z_O`l>rw;VqouH6vHe-=ncSlhlo7Y5M?vJ&<g!{7jdb20=2lKan*ztQ#vbcPQz$Vt3
zgTif<zILjTqK`VRv!>i!>wHU=`L^v7=RPJ)g-;v#vOW58&TQk@I&C|r`LiyK#V6Kr
zNdCGIKJoRfTTlG1X>NGRwMT4YSI7Rn_rwf^8#EM{HGfLU&gD5c?a^VrNo!~AdXrgv
zwjgt=bpJgK;eE}YI$|fA+<K>z$?LeBC-2d+?w~U@6IFEoPZd9$&F6DD=+y<jbqpeA
zZXeC_k|)=2e7d1r@vO-#YpIRv($c)=55rbX6?<42mKU`tec9B!n={_2w)f~sE`B-h
z>$1T6wQd|szq%D(Ntq$|e#bJ7PiZeoeB8gdg?~F_bWdv&pXBX?o%u5>?{z(U-)lFo
z?-~!wg(X|>oSINJ;e*P7H;tJmPINrqHa(Yz;f6<f$@bnZ*Vi`Bd08biCIz3lVER!Y
zE1B(Qj6p89_phtf@f@EXtpDYubMzz^$D{_%jPq_jOhz{fr%t#jm+ql)rR6rO^9Hjb
zmG(<#{O)VIzt*`Y>iO(*?8>6AT;E<zx}`jMn$)pPvlqs!mM*sPeERz7(+&Thtde;e
zH>s@n$TaKnjh`oa^6s}eZ&X!o6y)V-U*(kU@amk9b11`d|0_m{!WH5L=hksJtgI>y
zv|amt=A^yC&n>3&zq$V~GLT*RMdPu3hLXb0QyG?*|GIr);uN-`_1Bo5hJD>qp7-2O
zVBcpYuYac_-gmu<O>B6V$M*5q8{xny%Vj0l%!C#u_FU_cR<nP=a+OJ2aCgDafO)%8
zZcmV5nl)2V>sjt4MtjeYx8FV~<}}`53IFEjX4$)F@)JGB?*^O<t7r6z#EIXDm~OND
zYWegxB5_lN!waXZ`68BBxb8){M7z+QBJHN<Om-_(Tk>66)2r|1AK4S4T$cD`W8IUJ
zWk+KE>rd`j8~QTqQ~z<z`fI1_%{F_^YuFxV)%H$gRg93F({XNTBl&4BwngNyDTa2-
zX*W#RuySfc6YFgWmy8Vw+O2EEoOC?2l#aAB-SF9yA&~57CdC>NbE9pldzYl9-W-op
z3mdAovh0c3q1wozDj=-k5n#}GuE~>2>xZU8qYx`omef>%jmsI9arZJUwK#g#DOzKF
z(%G(0elF2vT(@?ye#>?=Uw3PYR&W=OxL&lG&rAc?CMM3H7fNfROBSAOs@>}JhP!nA
z!ToKl8i8JcD;SPN$ughg`nu}T32SEeH9eM#d>59fb1>*8oXR-euwCL#!h~*yt4smn
zDY+94J87|I9{;a*;$(@75nJu|e{-*vM^A|j`>Yqi_VnGFz#A<-dUs1o?y35V$y~jt
z;K;bx#xCdTsjMSW(M@J&Yo<KA`0euj<$0>x9>}gr<-MgYeKFMK?`PIO-{rl+ZeHw?
z&D{~`CK)qfBdd6Dj>1tEA)f^ftrb^8r&phRd1axaT-A;cM<IhyrJ8A$rHi_p*K*wO
z7t=Cq6=6Da<5sqL9IImAlC2VwSsQd&<}e)R+&O2W2<J`Z{L{Yd3=D#b7iWm6uVtC&
zBqW=ns+~I_Xc9wsTIk)-grFy>{a+UxaTOABcs4zW%g0Q~B|l7Td8${Y*%Xo73F(sc
z5q~o$gf<tcHfWdaNSY?i`+un`8)yDe)rYZ?hay)7Hr<?(8B*r9RkM3;otC7_WI@ru
zm9~>!1t$yq{c_zYSBgbD%xr1Lq}8n(XNAt5-x?;+zf?1Gan_cL$7bc&b16s{Ei+X=
zkat#(<?W`3pVOXX9aYVaaXDLh@MNLD$#1zSWm}tKH#}!@TNZY=w`fuY)0R&`x{7n#
z^Qt}+S~*HGF)h3A^hfN=RN18}I~1Z*d=4IDkq+=kY*Tr3RIXF`o5{y2kz||LwdR2-
zA|3LZG9IOVo5Z8#HD_H%QEG|rf+H-ZVk%ND`wyA1PB314OlrcZecOw-F^Op2o+xO@
zIdLniEbo@oM_s!EF0<M4N-(XG+QRCn_}|oJqSG@mm1m(zRh<)NiWlCU>yrF=+C`58
zJ6oR23hBSKF?GYetVqdPK9Bj1pH-{EqOuDQGA=3Dd&g)_rJ!l#ydtBz&^h%B*C?GT
zc^qZrr_|LI`b1fs{pzuT+fkKkicZOF4%;91RI1N>)v?XzH|%s3%${>X?QN)ydrNg5
zn`rEpEKhCeuM;l)?3*OAct-4|Pf@u_4xevHio`kH+1O<6<FR?>z2~Q``HaLFq(c@+
zb4<I)Cds|)C}Rs>{0{?PsYjVJ&n(sd@-g-H>@$B?UwXJdD`mqs{fOCzudj(KQ<{DA
zO}gson+K;XlUdBEp2@Eh`96$=h0#XSD^TmiyiH*f`Ro2`wlo~<b2lhj`oz~?JENIj
zv*pR`ur16=hAdG+O6{9=DYhxClX`ISUWy3cmZZQ8&E~~S+gW$+;n8=!*ez6CAtClr
zK(+g|+=?)_i8ihm6@p`A65A$e>ODIj+#A@l#^w9|P(!z13z0yn(i4kJ_bm^#%;8`E
zB3<Cpf;_8Lfw3DmJ)L$#(OY|4bH^5oiCeNX&zg(gk=^EFZ0(f9UG8(*i)q^ep}&cr
zw1jsGIYlK3HZ^Rr{^Zq}7Lexk)U_{BG)w--E2WLSj!R2KjXPwkr&@|M>KH|yNn!9W
zQ-Ar!D8%1Uod0E$1TzzNT7+jpj?jwTt9-l7ZM-MtNSQc3j0tY~e|VwCM=qn2*;c)?
zBWKQ9Wwvl#>A~YUaZTsS_?#y7yBx{BypdzZg)i$Au1;uJwkSnUmsx#INucff#>L{S
zS(aTnJ}ENU6BD)sDIHZQG2}R#^29*Z>X5*dUQ?#tj7OU$#6LCE-kQQIUtDaQq4s*>
z9!9^-t4?-4-Jm+_;+tg*UEj{+r@2-LdNN5qndv#n@w|dc$Za>FAloSum=>{dKKtZ1
zbN@*L)>R6|fgQhYo=!~SGYJ2y8NU0}RZX#FsS7w?hHh(cUo*!f;@GFfO2RK4)_-za
zE^sj+&7;w&HO4oV|E7@b&snPN6Ef#U8E3hit6U_O^`zBU=S|P6ZJUn#C|)@$`oC6)
z$r8Dvu^SggOx0P=aq5cpzaYPasX6O51*xB#!qx6ya`A;lvfJqluh74{r)C)zOWfKe
zSdu^IaYvepV)UZy{+q8JZ%hgon6J4yviVMIfxyd{rJCLz=VLr1_iWXf*O|S`IVjKN
zgr~B|x)aOec(Ve;gW}foKCcq+ToqRF<L0@<qSeuB=ZC)LzV4G3nD=|n`!vsM)1&0d
z-OhXoJv+hGSXI+I-KAMUDSFF^S$ewKCU+BD)~GRP9qd+m&*C{liFsMilnIAr4;IXP
zkSXOK$GXSFtHEgJ^uT{vQ)ldqeX%#=&b@1!vKlrNW%b&#9#|cax4f`NXGKuyV<l;q
zpv5mU-2XQ|O*!y~%lu~1S+Rn}v2WFmPl%Yg&TOxz;6qmatZAJ>zHCg%C6O-H7nE5!
zgEbDV`>LrXv~kA&UzN|SA6uTU;(M0yeYL+vSk`>8O%qtm3>d!_E<DP;a^kCs7I%fG
zfd{i~1&jWkh*{$J!^7n4k`HFJqM8#9=0qP7TDkM^(igP>dz@TN!wzgq3@d5RmP>2R
zKf<>Ds94IzAP<MTeykD}XEkq3KI%2URr1)eOxA9TCsWq@zP=Hk{In>%F(<)FdzTU4
z?Cq!7iZTmBk3}1%8LghXCa|Yx5~r`i-F7}1ajOHZLQ@#;efoAb)=WyO-)^UC(7n5_
zb!=sCGcKRCX}8$4?Kb~EWIWuW;9oUWBxP;7=){GUVOm*UEvu3`&pdJAGVKldoAd7F
zp6fT}R9l2Ew$i&GHRZUxx$fqfdqd{1I21O=r7jdUYu%>P(P63IEZLLNls{GB;A)n2
z>jQpW+#+Thc`igDzvi3tlFxbv=dr0zI#QWwSY3JW;<64Aj-ss1vAT!37pfms+;m8&
zE#=CrHyx`^*Cj1&jAhuykS|@f)+b@R$F0w*JM|wnJLg=MnD*D?X{I3aRI^Ff!cwkm
zTqksXhU{zePtVLY?Ye%pv24Hfy;;-CC4CYE&WkT$u?uS0>wEo4|Eyy#MGAc%q$pi}
z8l+v$?mvxtE#Er*WZCIfo`)p#PJaLX-)4!*+1o4yJC2>~l_^zReCCHic;BS+C!abp
z$o+i3dj9jfw@?2p6#f$MK%`hqviO;pRmq`ibBpyfH2&Je@8^7^%W!eJ!<YB(4}9=q
z_{Dc({=e9cdd`Mmro|o|%QSB7TzKS?=$upq?n|686H=ti7~Q6`p1vF~LyXs@v?2D>
zjMH(2K~pybyOgj`=eRbNLu~ch>mCP=-%^=&<6+d*53`OpUVE0}AARRe-iG#vveH{~
z<Lh1L@yXrhE#;9|7?zRC!C!Xq$<NlgM>_*#=FRtwe)REZwydJC)5NfXDZ2^}h$hKB
z&s_MdQdICNOU4sUch%T^d)afhoIAheXq4mK(1PiCCe!~jKRdQWc7e~{<5zAZvbZ$H
z&TRP_*|v3uKwzlMrI7Pnn-cmz9y_M!wn&pz@{ViR%sbZ#Pq-BNop23FQBaZ(n$Msr
zcQ5o>z?w@xG<$x<#9f#>|CgpmlgN?t2k-ymY|u)Z>v&>5!-aV5Q}H@YapGI#-|Mhw
z?u(8-&?$1@Sp8G!22P3d7Q(O2EWD@~t1#6uN|L8V)%T)<f|tj1y?u)^uXM4@X}Rv>
zazmHphUB!{zUPm2N`=i4zGZR8T<a|7*28yB-gKOE{O;ZxJ$t)?at?@HVaZQ?#OHO~
z<`D0xFXxqCDcf5qu2>T`$6%@;Q~1+D2GN*#yQd!bmn$dkC+l@&QCG-?|4k~VpK|gE
z-E|FAab#j~U}9)Yy5f;Kg>T=)sUBzDOhRuil?+$T^-hryk~!w~Hc_hBHN2&Cx}$<4
zONlIZ==|#{XPr9~Tu$875M+_OqLBPXw$`yF()U2{r^`D7p4tUn5iv^KS$p5<Ld>pK
zrC(nkozwKtD_Z#XY`n{{`xB-(u>YB(Qxz8@c0^}gQncBF3jwW?91U@NC!0zdoaLf+
z@y=p85|gOLz##K1k&PjV$$>%Tpr>!+=_4JP3;MF2@LXQm@ya(UnwQrlWJy?!n*XBE
z8@4V}UHY#tZ1JD^jJ<J&i^7qQshu4QS+eFWaBGopym2UL=2^8Z3$k6?vLuh%N?rIb
zRjPFAsRBP!n`q~pOo_{dc8MEAXG@AM7il}SLo`KV@5zlveV#_fiTSBKZ;aC}o4V}k
zm7g1C|2n}Mq~XZa^2EdJS+$jCja8?k&K+L{WnEsyx@9VoS*@=`TwYt8D2%)%W#nFP
z<R0_Vh5JQgq&F!lIxclI@wZA^n)$=c&7t3YPRj(Fmaj8-gRjl_W-6QGks4;ZqUPpI
zl{i<`TQ@rG<~X%=eU`lWl;^J2#QPduPtSe5qxnlldzI4Pi}F#k8vdS5(mJ<P=aA~f
zCy$sv-if<*<zM1F4Y^wjJRK5+mU(EYCF`a6Us+zlqma49CuJ6&mrKW+o%8&b#hDxp
z@#Q=l?KtN@vlh26r)Sy2c^_M%+NM6fQG4NN;{~TR0Y!JF-t;^kbV!&d@~E{KOa7rA
z`>3`hS6G64R!qtXHMlA^N$^RxRs7Sy10n$p&Q;IF4oSDv%IS1vxx}b^jClNWgKKn%
zWXl=XBoS8sE$K`wj?<2%GcX*SeB;$*h6AiuBWkO}*<4*}zqyFy9E?80<GN%?%#5~I
zKYFIVjh=Z*rB#i+*~i<+Ycj*1SB1HY+Y*<qn%gpKY2=5_#S2eK2DizJ?tSs#?vW!Q
zp(kwbFW%N6IBj;A-@>^ueD}_3v`mPP{*WYD(%AGR=KS8J702TH+vdG6^{sF^GU3wF
z7rO0-zs2~@JXqL~dZBgE|AaV>Gm$H+<1&+8Wp#SFZ&SO?@M!tV1%3H{)%y>yW_d(5
z9X#&2Qm)8jafyw1X`|Ncb1y%r+;MVgnAv%^bHy$7(z8`_@4EKPni=3}Q6*th$>Q`x
ztzx2+|C<Y?a;$uZ6!@4}?>S1>Xr!l!y8ZE)czR}6uwu$CwRxpo7mF@U5>?~bUE1JM
zm6h(~va&6%e}cN;#?}W5(qkG@0&?CY=`E|_(%YaVvtIAWdc8Z-vRrGm)*ZT-z9yql
ztmal)!Ij9p$2V{|oSrphe<a(xjGJ$F=%k%{QgSA7#Z+$2L#x9YZnSPYeSTV$ql8u-
zldt|w&*MuomhO8!UpBI5;tSocvU`(4|F?=P|M2Zpi^MDqs}sHVRy6(5K404%Q+3bv
z)cwDE<6r#LxVueh@|OABJ_3`!BylpRv2Wn_%UDx=>D5oObzQSE)RI;^cIZ6(_Nx6(
zy6`^Uq=<EAubeoyCgseeypU&4V)unkoWsEqrR`!O)0Noyg=ejKQ|FhG#U34x%X(Tk
zl|(&WXtHR(oiQV#?JARET*RLn31N4<V%DhGDyfI%Ntf1n6;AR>JvymXhrxN*+wXPi
zm2FzAFCR^u;2^b%|KW}-zYWW$UVF8c>)kp#P9}+W?J9=9_q}gk<nq|+T{XuMrvS;v
zN2a+?bK?wVnec{3Jo=`S%8chyEN?p3z3>xOn<%{VMdp9DWqJ#gc)xI*y!Peg*AtP_
zd;IU5$-ecgHQFG4=`QW=Wj)_~rYh!L+hY~l>cZxJH6~)(qkpFFmzTvo?@XU?O!W|3
zrq-@hZ9Yw}3#xxJqmGmvJn|`tead}xp=X-?OMdQCUn{ik@0Rz6_vQ<yvo=0TPd&lW
z+oHMpn$MD{g_k)Sa%B?IA3X`>Fj(gMIMip7Uv4AYuA8@(K4WoiQ7>(u5%A%qTgyk^
zm>e&5>52dj%e7YLB1MAMCjTv{vvYN@ROJ4yEb>YyGgCLSHD0bf{@S_CFOPm;32-Q1
zmCEaEw6gD2sK%O=Qk=~n3|6ZNda!M*7Ave_TD?xMFva0xiOm)UpOXJB?A!)fPLm#f
zQq$?%tGW8ATiYuR@r^Rmb5vd|No6hP4PyL$A}>Lb$>V^S#46+MADN|#ukKw}RoGsY
zC|{Gqf98*2fWqYPt3jH)lO^(&YxF(+#FituAW7x8YX81?ouI7~>*VgvI-sNWGED#Y
zeyf+Face$Q$!l>N{;JZc)f5Tlc>P$crF^Gp*S^x|V3!!@79ID(*yv-EZya6j=@`GE
zQ^dnDWf7l;W`h3r{8<6A66r^cTlH9EH4<DHC65H$d6>y+>|*l1q~gI<zj9vd#IxD^
zZeCJzXG=D#c3qySniZkBDl_xB)$hsN$9;4P)gv}V-`OU=r#0Q-y6(hZ#YHJ8|0|1B
z#0C6TUt{rC*|Nrp(^E+AvE91$=Srp1FD1_l`MiMb)Va=<2?b*FUtZ;v`M=5J`-U%9
z0wgqVIC*$z#B7aSUF$ME=W=TyXReZJxz5ERsZG7UJrhlr-@K;%?%9H!oN~_=q<n01
z>($%3|J8~Ezhb_+z5f!m>7V4w%NL&NF>RRG@#*(hCH8=ZLr;o-FDRO7wCuQ?+Mj!0
zJ8zgOJ}wOnydO5zLVx0$-Wx}HPF&1jF)O&*)6mCuTIqOs_>GPGM13sFSR}uhKJ2?=
zwZOMFN@jU^2EUkdN{eayjOy<8__td)V~trpXR<8s?mF6iLzSN+`(m+~bbsPh?$tg5
zZ3Vm7X8m7Ypp+whgz3l!+vU=%DHXZb3x41H$@~9`!(x9ap>yuPbp4lX@>sN9(E8+B
zv!iQ2t&-I{!dB|>$jF2(vc%EHq-68fE}xB0iUKCR{W@KouPl4JSaPdmnqHFmHifft
z<=;7y;{%0!I(2<qZY^B*Y0me=r3p5LtLLfl_e>6Z+3=$C3HQ^ZK_}aeai8uCJfqPn
zP@%c#{;#hwO%D0mj}}EgTJ%@eeWlSGCyh1V%1l>id0eQhe!#L!^Y9%_bG74j7Rg;d
z9KXJc-M_v#TIu+f-^GU|m~7O%Ld<)89RHsBA@OPXB~7zc^N-20a^@^q*?8|>iOlEy
z^-*iqMrAF!73a!-tHk*KcBegSlUb`LZ%ZvGP84im-|zN!U0L`>%YA-sPcvW5aZZ_g
zTKAh*h0BYwuSu?MT~D9xY;{-j^Z0aR&9b6tLWZvL8yChu(&nyxm-MJ>_OT_wESL7X
z7j2($MDe!&m5!c}y#3j%6Kqy*KJ?_pwn8D71If3pt_iL>s#-I(At-Fip>>Xn>Z}d-
zGW~uhRsB(^+PTcJ(LDdRVuf$$r>5)kze=qPtDf1c{qihd?|!!@O%3@G{LgksFWcGt
z<G7tx?E1YioAa6es0evDiexYvY}?VfKi?tij>TTKlSYr0X}iW>J|TAIM?(6?Q%)>}
ztUqTi+H&bf#m5b8W^b2dmZZHEo_&7W-v41P?ap^5XYMJV8t-~JsqFCo^{E0b*=Z89
zayIVUZJchnaC&f<?dJ&5CRMRzHWhB4&UU4%WzMQBTH^TI@?8F!<9cdrjO&)PcRW75
zRhH?%l{3#5%Wv$+&wQl!J=Q#LiGATE<`#xVW>yBti~<EF-z9Dvi%#^3x@a;m2yCek
ze0;D+T0iW`k3fehx{~Ic6OsZ?I%o=qam`VBpn9TTnx$aJj@Hv2dNMnB-e?9hsQ1{`
zM~Qd^WCZrP>k8#s1vD`jt&NIRVZEZH-4nI3>}%J=9)@-DtNC=iH%vRA%O%7U@*wO%
z%XX$G$Bt?w>K)`{y61LPy8I9eUvR~=2SLw1JTy}KziUe5ouw@d$0n+~zf({!I>@KC
zXP1WZgW`<yX$HlHIWw9QIE$WYaXgSJYhc;LsIsE=_BMw8-2LCg8W@y6bRS~<@a^r@
z$8L`pCQK+gv1;1k#%KnC_Z2d`-?OM2_T4NBI(1ToWxxMJzGD4jGqPAUa_9fGt!(Vz
z+@=0wW<tat=EGcX^r!A$*f6PKO4K=_7g|5~?s%-;d`#$tBcqs#hNIa7;fCJLCL#;w
z_H6S=(o+?gp}oG}<4pFt10gCs%=4@o*7OJ^&AHfe!9w_jTX)eJp3RLFDjLUS-b-t2
zIZ|;dvq@7;An=Gvnzup+Ll$$>B9;|vI!--m5a8puw@E}}XU`h$|H6zd2ma_T<YwKZ
zXJMMUDQ_|dyVOxH^S5t&yC-d#;itNvWr70d{%toFENzU7@>iR6L->U4>|4bpfsAfW
zulyJ+f1Tg1qBLO<?<ylH?{i+RM_*|9xIbLc8>l9uvG%3clI#Uv{<VZE*k6wJH8|ws
zeP)A^!tbeu!d^3SHtYH9$T2)=_WQ;0pH)1(TQYVVFmH+t6xK0djq+A7yFKp+Z_Hgc
z+37Lny~cSKj9VJ|M58u&F<F%8PWrBGc3d>mzwz6h;w2vwxJ1;Zv&8i;(_LopW6?ad
zS}p5V{a@!beVAKrXm`HeY~nCg$i#brAG_4*RiXM?t9d!>)lN7yy4+jQ+Z*ZmKeAyG
z>y00)S7preVyJd{7m#Y?ATv>a$GxCH1FZ@z9yiIU^ESrJT;ST%J$Km+2Tx5E0mH^M
z^LZMwwU>&BpKjvHTCMwVf&#~mEf-bJc|E$~wqVh#y2-b#T27tmTV!Hrboq=itJ8_a
z`!+s`Ufi=yM5C+2XwoC^{YO>JeSRoK>{xOtTUF!2WVcmTCmRI+cB#46PU^q7!kvAk
z#_yh}xfkz+sCzJJh8@!inUL{sMcMt7dtA;F!qk}cBblYwY&a}sy|3k%ocF&QLT#cM
zn@{Ulhc%qmYpz@Hx_Qr<?#5lm^?zRWIXsOyck9z{rZ?l59lY}MFZbSCc6ZD7f~}34
zt}Ff79rZtN!HMfvq<tkjot%Bt8eVL1;@)=gzPp%*-e1K8?w^k?i=8N#H06v}vHn9p
zAH5lZdJ0*ID?j^kd^TRWsHBl=?QH)*HP^OQ))gAR4o2i}yPz=P&AOPVhg^KlM@47E
zK538LD0wdA8dvp}Q%a&bXL_coDPQ2|jAVK_Ph{`&y=k#2_VX1P#ZM@huG;t{D<`mF
za*D9(H&0IQcy1=g6W4Z}n7Y>Y`|pnb3@45;%;PPLx^P6oY$L0$2V;N2qQ`TkDwaPn
zNauN@a3XSBLyIGmYe9Tv3d8?tdW|{HTUj(uR<dkZ={eWMvxIZ$!d*&M-!-zh`)-{)
zYIH~>%<OHc<irNCH1+@X(OfKPi8mucy~?KEl;!B$rf`&P$Avbd32nxQl%_HCY+chE
z!>=4H;=#Eto$b#LuNql}Gb|bjcAN{=M{ix&`@5*vySRwOBJjcyjc+Qfx*mxti#raf
zo<BM9$Hv}9iJN_V(wjO1-w4M2OP+lDS*XnVBl@vFq*r)uF<@)@psh1UhF?R!X~jz3
z9F+-0UA{>_C;FUHpQd}nizj`?k>wMvC`CtHZA@9Va8t`ENA5|Qr<Fn!XH~9p`f!m;
z^H7kHQO47bwGNMJmOMC8XxrOuee>jkRbt21Of+Ql+vR(YCn$7dX{N%lN@<4oX%qKc
z^fgaxdi?*yE%V>8DwC40sxvI!z^VJ+#fh6&@~D=dC?}&wQ+>wo=<wgm!#4#Pt<Si7
zJHb+D=dF@Uo_<bEQ#FiEzIG7GmsptWI%z}eB%O2qOg9g3f1fnZ`QULilb4+C!InaE
zW!y{GiCoaOwb<2n>Rj0#XMvR|)pKqid>$gUab}))CU^IbZDNPtbf&H{_suBsk8<;s
zGoG0_gRj7G_R7RwCY1?x*K4?Uzc0?Q-FaJ7bB8GxPoamdh$ch(smVE!XZ;jwoiFVF
zHFeUiM(H~Zo9<fGsnmTB(YG~j^5pdL(v6<6+`%Y)QbT2*g2-1-pF5l9^%bQg%7^xI
zNwQ6uUvn(!y`nRh;Wl~CtDm;Taej2TwZS>QsOP`aR^E@zO&-%`ehtwIc`0z7;}V-@
zfr6XeArCRu4M+8g8dfG8i3*vYa*ToDVRpC{$Ml}bGJju9?fa9gUnj3V*}UuY-iXd{
zhVK>*9T7)jqb9U4$Y^f&()_Y-8&5-utE+p<GRu8A(+;HSKf18uhbrfN)(>J<-V@G$
zV++#vQEe4x)pTEJmUy>SUw60A!hqI_qtDJPjqrLUe{a@NvDHefes;;eX-UQ5vIbGx
zmPyZk-E_J(A^6Lx^Olo#6fNDueDg^8;(~~`M?PBoj9snDeJZ|LY?H32=lU6+rk`K0
zuw_%UL}$5-)=Dlf11*ltC%j$KHC9Q!X<lA>YKM%NX8T;Tpe6r5^NTNadgs!<RQ=MH
z8D5(g-JTX?Ro*@20&n1*7ZZ>E)=u0QP@j<_D1DjbMEn-1$sIB*O4AL^-yEJa{nWOl
zzgBK<<bVIZ?zOmQQ$WjtK(iID8@&Ag-t1Kno4&^VScnB<hSQmy%B~{E1Xv$DIa?T4
z7=7oQm;L1SJu!V-^AowPZ<==Q@!^bEGkJ?b;?_Co^JZBW%onUtY&f2-FoCJy@{Fbo
z#+B!6Hwxd3Fk{QSRkWnt_jYct=H8&D4TrWzu5{JhmJqu=dAmhnu<7cT>z0}S{`D^7
zR+GvSgRZDG%uYMk9+gkuTO5D;<N9DX^>yc24;=fy^<-`Bo7pS=TzB@Wd+gES@PFQC
z9{=<8?l(2JZn?BHAhOZdTA4lBpqW4BA$xSo;eh(R?u=(#7yB<)my{D~y~H<V!ozp3
z-#+U&tx@}}agM}&Bh8OzZXD>EG<A*6-!RX-YdfSTOuo6-!(sFNO-p`Q?4PbIyWeh(
z;Qqqo_2PCPL_F8%DzP>klF{g2Zu6u(H+7AK+<!UF$^R4HJzMBr`Z-CxOsAXSx5rH#
zcZC-|lS3Zuf8eRRYVB7K&I@ktfnl7lQqJ41IcILNVJ}Nk@C}DwL2r~M^_QJWJ(8xT
zypE~eZQ1t_H-?r>rB`_&XS~>+sxwM?F%)Iri)fs?q=9GAf#|FcH+Wv1Z9Dtth3EBG
zuYY$4Fq-{e|7D2+*FDZJ4>GxD<O<JN%e6T`VNbI^!;+f@kF39?F0y!}+HzcVUGJ17
z?`+fZmBrM)u{`7b`J`Ac?e?d|cLJU>_3K~mV-C@I#1cCp)nHMo*&~OjgFih!C>y0r
ztVmm7<-+x3sh=6s<9+@GJcq6cH9dKjw))w*)u&#q(n;Fb5D@di^~n)6MVY?G9Ixw@
z`0Bs;@B8&!NVwtLw-;Q2dHr%53-#7bda(YK8o$rBuJtVjYYhCwFKWFI(q>qDY;M=O
z{#h3{ozc`W(mpT77MQ|%@Yx6LoI`;HNxZHn=B(1wFjJbgXp#ND4cmJ1w;fn#F85Z3
zyJPvKkJ?rGy3YcGc>E<q|MPQhJ7762K%y}BR7>)_ZyB%F1+1z$`+Jt!`%~-xJURbd
z=6MX$($9yIxs*GUG&Q(SKH&Pgl$-O=LaU-zXHIiFUuvD@%b41`@Q^YW+lT!|NuGUr
zOv(X=ZUiVK^|w4=zO(PpVWVTrpHs`tShQC$336W4i8@n$<A98AlZL%p(jtCklY`#h
z+@Ak>wZ<W7VFm9#hr*mC2|N)CMQ<c3Nxh%=<h=NK{y!U<eHjwGjgI%<TlwX}r>A`d
zF*jUvPEK0=jN|hRosBF4^GqCXa&2I+O;OnQf!i_3fcdjB*JrgI%mG)H{$*)<$#%@#
z#bc_;Dc;ZyX5E~STTK0GPc?3DY}liu;rc(oF=^QkhQRGnAKt83EPwUMZXQD`A0@?b
z;rul%bKY#6mn6)2Hec9$owpD}z`PHiS4`-S;ZW|pY+(7xN#L@QO%eBWVI8?sSA7nB
zIT4_y<n!&C0sr1Znq5k*yCzS$raU?C(h@Tv>(?KYUNQb?(NMNYyVsSG;wGSMv($w3
zqeYOKW7I;~oQ1M#DZ(e7ZP>!4cQq^j$0<*i*XDfBpPMZbF?5&Ty-EA3l6dFDvpI)F
zZ!dY4)OY>KyN$>C*E2kK5MQrTnU{M>CHKuUd+R)xaKYzO7JL8GFHUmj;{Tv;d`Wr#
z1*@ZH&8GxgzCL3xQ&`1b%SPzU<bRJ0XX_cUHZXk(P}}+cwDe~KrU?z!PQ}VrpS3&u
zWhIxChZG!<a!Co;{NvaGvsz`Je&s@sg93pU8h!<vy<V^V?9#kfpFT*eH~+*p{}`Ln
z!T_#^D_?B7U~RC;-#92p?ddl*Q)Q1!Dwa*>ye|0~d}_7ow_5(7DdLgp#*fO=w`|?u
z++ugCb<<=UPG8+HwiLZ4TiegJ`b$KEj$M2spz&{UN`hOOQvk!S;z=LYPwMh@T`RQV
ziHL>D0!5`IvL_sVKNER$Lcge~^pO~cx%nf_=>?73`1Nf0K5H@MnQXBT%YLDvpgQ%U
z{_KhUI^G5wliW8e$2@d)PgRm)Px?IXqskVA%?U^K-CllaSfsZ2TyXvWLnRgmOY};W
zKTK8DDOFzLpxt)rwa>G%54~RgY}QLcwuiC4O$gOGR^qGgsq*MZ$_thx<!$mG4H>NS
zmDW#aGHbMO4AXMiW+~5>{Cefh9nJ^XCQQ~^^_9!*>xzl~+cpRN^I^&lyJMtTy6c6q
zt7~HTgya$r9XHbuan6kBhj%?bYbPwz(pe(<E?6O9t9_tIuGFXcRVjZK@Q7Yd^X7T2
z%e`E5b@IjsD;1*-rXNaitp2EQZ;j8YO*dwlPWr;1dr8di;*y8=ybJcd5)nM$s;m&L
zI!SW-)c&N>nCfc=oT4gu{-OQCD&ApV9yovJ`|K{d`GAx`SO%+6wQ|WlW~TeeapC{Z
z#y{Nbb^W9Ank)RZ@0E(Oot0EHc{v15Hstw#(|UX^fB)h+DcfVXG*91H!D-3wZ{P24
zKh0wGhL~T@jJ#hhH!=r{T-DaoE^2I2`K|1s?_OB=E#VK3fu>Fg@AN}SLOj+}1I<FU
zQpBvYI^VqFz5lDA<St8^PD=TlX(Dl7m7288d#=`%Zoi%*dOgJCbeZ^RzK|D3Tz{A4
zoqlv>ea}<{Ce21er^ct7Zd~bF@Z{)+AJb06l}K(?C|ExsT6?*{^taRIO)r$?YB6r&
zDe_L*XR6~8!5)4i<#~%T*K{5;-j@;6lovI<%rOi6;ZhRdIrUX?P=%As>R^Q>>-aYE
zDGT^EM*lyW(9fS37*M<Mi`Ug9DM~3_&Hf5m2NOLNBMYZ@zvisp^vrzmhtE=86{mMt
zcxSbg78PEf@orUk*<#^<dn>iY+SHaY1p2HDazEN@ebwr&vJG=u|K3H@ijM#E2`*vX
zX>xpBTGeu$GhbM5C$11s5xU;ulXbQ3m)eaEgD1Dl8U#!l&g4wkk@BdGS@DjE#fobV
zf2W_}^KWE-l(V7BOq_eF@{WAnh<>gD3wOoo-aA?<uh_dOwmM#NGhY-_yv4jsWQL$=
zjPV=p|H_BwRR*p-cEBuERW(E3j;nq1Mzwp%(kF|npSoLYV$)vdl$e`s=l^{|<#DHV
zO_}Ze%1S#$|19l({(ooYt|xwt)BD?mQyMGh8yd04S}@ux1?u)LnA}n6ncl)*rOf=(
zLZzotQp)|{^~E_MsaJ)xb+VK-zRqS>;!}R=mYS5VXw~)o@{e!DWz)`2c&$5kcEmlN
z$h0R%W*Qy}^~~A!=sWAh%cb^eOh$sgPvqD0dfoWuefrNtdFGc>!y^^8=ug^oKIhD_
zsc+X$wLIjy*h`;#!t}<O%QslH&6JRHbXv%%wEb8^D|1x}<F4tfPSY)&*zYbW;I^{9
zc5AJ;L8&3ry5q(UVr&6_n3_|SJl6!O$>fx%T@Ln4H`o`h7MfkXU1!3i1vfb?f1H{r
z+sGs+&hpFX(bBrpiH|G0UrLsr|9{NfavE26@%+d}k)BCg{C96*S4#C;y6*qXt^XtA
z48xXK{&uq9<dTdkvGLYT`nF^jx6SvSYWquDn2se*yOOAXWMk}vloy*rb~a6)A~tdU
z`LtC|_S}IwDh+JQFa5d1EmS&r#mx@3={_qwC+=Sp`l90S9&zoJCwXF1e(pJ+woz+Q
z<6*Y`MP3Xhs?mlwq8+C#|7G3QYt(jpQLXk<mePVF$EKcZ___7st)wXHN=cbD{9D(`
z-kEOhlr-mN<<%XRB+PsJ-&=o4T_lzk&!Vv1P(0ncGmcj`j&XKt%LI#Khs`f;;Xk|G
ze9yXNSvJ-=x4ydsntA(|U41rBS!25AL7(nFnf?FoOsroY^0-*u-nhIjW6j2<L;I7q
z$zMOEJS%2Z%(}NKI$~@#dRAZKI=*gMo4IQHQNgN%Do<x`ZQ#8y`IztKIj+L@r%883
zZ(oq8qwRG}>{Vi@ib77st-8A^t0qb*-!N@_n|dMp>pYF@NnbN<K13_64p{woy7tOL
zmv}E;%JuLI5KT3Bl6vl^v9;{<C(92nO*-Yc<4W>Zr;?wmQw^5sI!)i1?ElA`nZ;?A
z{-HKsn|4+uPuo!5_@FxsyVdSAr|BzcHvK3m`l|G-^I}WpJw7pwGaHTb({D9$T$w#5
z)abWht3m#E6{eh*IZZZOLXVzkJ#~kd<!-B4sPo<$$IY)hYPN3v-&|z&X`$+_tExVy
z4HsRttlm+(C)Iut593ds_1_P4MX<G7FEU^L)g_8)+XCLc+xKqlZD8AIdwcVQ^N|_u
zAAXw5QJ%N^XIk4X#hWJ43n#KvOqG_H{NHV@(nRwok$X25?fvd$-p04G{cIGctmE!)
zPu|a7HT~kIB#Sn09{ct>`|i4X3UO*rf}M6;+Bx;xql<-C7Vj%}VZC*F=~dCKYxhQ}
zO|oP^{OIawtIlOI+iu<{bgihe&J5E#G>cj7dg<&SM<?m+mY%`w(vnY$Q$J+i?$^$F
zp{x|CcE<3H%I{ze-Hv(EzQ#(EvJ-C~S3R$nvbp{+r>oKy^TreXGFE47FY~B;j{dLs
zIsSvPmAvk}8z-+W$q3!`#^J7(z!NPEn^RloZOhGEpx;#K9dzmVfw>%KCRZ(1-T1v~
zbMDrhmv57;<J4W}*(;TuaCqXo-?>KU>vWmS(@rlJ#?>~O37W>F<((EQIi_@}$x0()
z)%>cxyo;tj{k_+B-hr;Q3jA~baPly0-75Ki_a)uCOEyZ_Ph1gthEMs=Kc!!}P9`i0
zw>Ts;3KXQnm{}EkR!m4d)XFApRdOP2k!$-By(pQf6COMCGdX0*XeL}fVyMR2<GG_q
zNwQ5-IITr;lj_OY7R6R7%9oz1aq>*|n6OE8AzN>dke|SYkOLyi#b<Nfn6Tl610SCz
z&z+`F2J!#B@$7A1XKh{`!8k{!=wZTEZFLqV1`8hrhIbB&%1@@`s3tsf?yeL3CCgE`
z<o<z%)PJQN8y+1!#>39P#<Yk#cJ2(xQ?ffE)$gbXPV!J(#kw>uY)RC{pxD(@*M_Z)
z-@a_lWih|o+YE!3`Tod%c*a8UpB$&<6YpL_{ixC%GCMv$7n|)o-!InTl18)x_k_CQ
zu1TG)+kGPK#9rTc^+K}ke%?pTszpmG#C^_}r~ExC%>8Mt{)NJ{eN0<}<l}d%9=R>%
zq0r#s#G&y*AjaXw)`Vtmz9|n{nL=g+wD9mvabfc0P?#bTm9g=tTGdO{2VHNsTzb^a
zv}N(dTBZxv`0{rjOiTFRYx8N^<bI<#m&FXuY9ULU4tc6vEO=?UbK+zbRrgr`1D=`(
z3d5|XO!7T$xkbo-v8pO#MuqBD(P|z>)+-L)z8|j4J}>HeH9R)A@Lq6yYw5k>_@$wD
zQ<-nF-t)0rcP-kjO>3F*lpa47NzZK&*H*ZCw5Yj?L<zJ$(pa%^+pO$0XV_+ETRk&d
z>9>q8MXOKvu6WiY^Mj{e<oiguq`Irzt$MZYMd9*^>#ep<xl&q^o0#TTV3)e4f~U|>
zE>isM%@;-cCQYAE&6hYkprbKu8>jRJR?mR7ViRw#a$bAp_zbr-Ay4F;-&<W_i|;T>
z>9)1|WWlz*;o8%i4x9>%Ni2aADjxlpSiEA#qHd;Zmy&i&e*0sw^BTV=1xJ{!2qzzt
z&emCcTxNUn?V~nEAs<~H3GKWv@y(<5Ck{HFR|&hDJmy;JYhHSCg0IDCo4He~bhGca
z2d(kEw9#$%%V(}3zE@0Uab`Z98p#(Jc=ak%LBra4mz++!M&$UOjlIEdwdA2gpuv(>
zhMBKgKQ5LCRF8<aNM$j1b=#J+-fg1W1mT#eT&<d!%X%+)Wi9NJy1il(=Y&<x%HA=X
z&L5bWwezva_3(t9`*zx_+!@|CWzsp$iGG$>W*lXC^U`-x)AUjy1_P<ITV4y_lodF6
zi_P8j+ei1c3UBEmhixZLh`*R|RHtDVho=4fm&SVhoBy8_`Rw7=A=SnaC2}agX+s?Q
z#uFd9`KI50{H0RZv$k<fo?Yd>U4~0OI()uvWPkk2eU|?v99K_$$~y5nXpdj1b}Zw1
zPpiis(@Iiz&8T2W4|x?Czb!#&i|A_0Y;CT93C|{&J>|0qnR)r$8<UCapS}#adgx5r
zuT<G>vff-(Coj5w+it(>>ocXY)qHojif2rG(k3!<(F2KX4sE_ATBb{)<_H<K&phsX
zq|@igZN{Z5w0>VP2(g?v)o+W15HnXo0}o@TM6TIUqj{ND`ww?+(5MjKwMWnRoo82d
z*CQPpLv_X2<{&BAOQyNKEv;8BGIc#VJfUPEe;TuwymQKyO^r+5c)0%0y6*5!Y+~b{
zlsL03(`{FzcUN`j$M9B1@T$7omb~??eQ!J=TgYfrf8-LmEj#5Gr_2<Xcrt&AY08r4
zNl)bs?incVGg<6*?1jt}9+xIzMo*tjDrc^obji53u=`?<nP84;uO{<HWxq3@L{|nn
zX06_QOY(iiG~>G7{)C-f+nz8j+_6KrK*clK$*p*ipJz|l>q$JS>MJGceibVnd?UX^
z?59igHMY>pf%jI5c+CyXJW|^lC0;e5eR6qci@Nf`3}bUE*F_>rGp1H4Z`x34y`rX|
zT;pkmX{D=gZ^`-G?}l=IBDZXgY4k7p6m`!h#CWE!+r=bnrb)lkJWkDzI#TC9NB)0a
zgy;T{yYE+WE==uR(sd=oF0k&$>eM4{l5-}m;GV&gDekv$`K3*DPgHbvb};U(5KeiQ
zW4$<Hn&%P)U*olJ*KtPPc8I#Awk$)~BBD|%^u#f_)o)j3uU_1@z=KI+zwBwnPa+dL
zUTCtm$m&hppeTMPW0qu4j+s&W5?*Kj%qv>6wrxoho#g7)J+CJAV~W=-4nKM4Nsl5}
zs#4bSMbABQrsd1GhSj&5HpF!>2&jr><p(uwlFPniYv!G%DWf^1rD>*yK#qo#hVLO2
z<=`g2^~=OQo(~VK6#l=5C1z&IQh!+{Be|!?Og9{vbJ%L7R==g`Dx>D4u*jL4PCA&@
zZ&-7`e)a#V&_8xxly+TNs>#o}zEwtLjljAL$!e#Eo=?BcSXrv=tG1H$B1eKx>D|?C
zUw$PnO}M<oyz*Ajie)mZIyXmVUWwwIAMIzdafUG;=f$MIuG2TG9*rpfc){bC$ue)&
zi5;^)yh#<FzCBQIrP$foy$+H(6BXaK@7bifK4anK_$h0{Ca$?Fv3OUHT;ar>RUcl*
z|L&?f@bd__sm0B;+bh1_zGAmLL*>_om`Me9rg}N-e6uj@9bb0(gF<iRcE67M3~CLH
zoOv3~3!kg`xx94zq~f}SL0(fXM*q-(nWFkT3ts4DdVIQfV);{VNs;d5f=iR~R{mJ}
zbSCc^>AzxD$sSi{=(KX}`QPR6<e*TSpZ}N2><1FX3iobvcI_#N?p7#PwS6u5GR$bo
z-ntiu`)#k?nqe)-zi5V4qwk3X_M{ud*LIwb_<hyoh7d#Vn;phwktWVc(T}G6Gh}q&
zJUioy?vy%jN0UXG5eGd?3X)kTy0|#*5M%lG{$fhkrY|~Nnzs%JI*CTB|K>L4Eq769
zKhd$eXg1Hz43_Akvt`rIB|0CP7o2VxA7d+aGN>fbTqwCqO;i4s@%D*E6W-?>FKet5
zmfakBY@<xMWxcBBqKlhLUNm&*`Tw7AWwGDamC<`&MNGH4D_6RH_L^Ht@%5_57D;^j
zlN+>3Eb_g}1-l2oBu~6$W6QduZvHJs>;K7^uY!L1*+;oNr<tqk&5v}C68f=OC|JF;
zZl|hu&a&+A+lmf~AJ0iH+M;QrtMzCbN4Ng|snX5t{V)5(Jdb!zFT1SDk(#{4b;(Q?
zq3165q!?2cC*EJ%(y-#<bWJ_yh6w^?WgN%3Cpo!(bDHt}SI)J#Fq`!|x{j{hw9`DM
z#c#&<l{H1Hs+BtV_ZDCHt8Mq#hS4ftG3}oX>uteQo0C@MPG^GM-UY9FDZa4wtLlc#
zLs#5Wvhqc4t=_+yL0EL#frG9K9w%6c`V_heO$u~a`Bcy~#D(W#L)O=@jHf~;w^T@M
ztB5sXb6uQq=Tk*ak^8r6j_RyVGmlqFC7SM=<gT)?=|AViJPSd_h9b!g*CplI>bsT+
z3Kp0ZI|?-^RdY^Y*wdb@Jkj>~R?hE?MKV+MH?tKnSd=Xdu`N2TEN^ZY>+03osC@Ce
zW!pmm?dvM7&szh9iZ2JrUSVN)V6K*46mwFQ!ONkJ^Ly^OqL}1^O$Q8%P1=ML6{}wu
z8aSyjd`b_q`ce7sh<I#!joqUxPt)=rrDc2*>xC!Q|6VREwK=YDMMv1y>@Us<6FIyc
zA4Q$tZWuYkZXHw3M$tsx#D)fTRojWRqGDDlPc-;Ca`+>xguiv=n_8za)m49uxU|vn
zvx#8P0u%LVlEpjfPF*w!c~JSxtUavIug)mhG)=7JqU49P>i_=^2?VG|I<bkT&0wf=
z_BEQ3Q70^ASs}?Cq`6g`v1wEC)^tI~7Xo|So42VKMo%bT*;MmqdcXVEPWP<>9vNL%
z3{?ac>RC?dx>0Ur{JidFdYJikvyI8QObY(*rkgmViz;{cmAtTSEYnRDZ7y4;*l6K@
zAxwYyw)kHW8NNTtTudd+5A}yebZtx$*xJ;^-XYn(U0YAQd!boYc$3n}M%9!fVf<4j
zm%Fe<Kj>Cs($Fo{3ERl(I<ds7aN<=(r#H*RViI(fCr%Vs?B`(=f4xFD=B3{?iF(DG
z?UIKSy45tEztI1v!La9e-}M<)6$@G2Eo8kl()2uq43~y}G;S?DUikm&=IK{1$4_}U
zU0ZT;i(ugXBZ2M3&UJyt>L)df#M`D$lsq=6;QzAjIUkz?uIIb370j2Mbt9}U;Gn>H
zrH;Lat*TALZZJ)~kl1X#uzfevZ08@{|DRY)h=^x8&=Ik<jMK3*Q+&?V?Q#EGvzLU-
z`PnS||AxTvWcf|5UB*9M4K(MP&WyV-QE=^vR@0ddVlTQ<L(&&J7uPOiyuvz>nURG-
zLHq?L1ILDW3>)UXojK1zLflKT=im>upv0!&iFr|yvK9+{&uQppH<t9c_3l+?c>AsI
zri9YX8+j|P_sum_l)Mz7%4Q&{ly509-A_q_?UZDi66-gKnW<anSS_^a+9?t6ZPNeX
zFj>za38h3m&d;r7&$PL3*4=3qS2(6&&hGmru;=f}SufS5D!r0X+*y)cDVTpFbLY3B
zsTG1#B|7_G7zeN}_3xOz`IyQNm*{zAMZJp}CKpY%TsT>6VnwV{mhcL}3!7ONFKE16
zlAC^VUg<%_o@?`j9hWmaXzDdn@_szar`Xj0a9xC>>63-(D`(2ZT#%f8QNN<A*LsC8
zW3pgbqRD4VLGDY*sh_7My%b2?IhChSr?y?V+)_owD|PXe!t;r%Y=n%Z#asO?R_(f2
zQ*wQgSaOlZam&LjiH99)onJ*K&5$#>wVFS|B-2^x)<eJ9uVfe>v^l7iXfCxrdO};%
zL#Srg>i>41t(q^Enj3a5$eMK7ap~2IOOnh5{H44%9*J997Nz*z`|v^=lTT@f9VcJ8
zHMhCk!E~38MA{TN3D!$0Epu*7HD9<qc!P<IrQf3C0>5sGM9j>)Qz&=-lz`DRQ~hPV
z8>b5ht&)_QwUV`JBbV34cO@A!Zt1>{*eJ6*S&7@ffRn-PGTT3i_3T?#1<a6By>4mh
z6c}kWH`-;csDzv3(V{C4>&}JNN-8qmFqB%L*qpXvv2a4&qk}pN%AJn7Ezr^SDVr)~
z>==LVmCOUFrTaA3O<N&2^~ctSTr!%O>%zUuqda6kGOgeKYN?{4u~eJz>1z!^L5nj(
zi_eK!xqcLqTT!|@y!ZbM#?rZhEL;ug)ma<F1f70sI;AIy>6A?ke<}KGo3=K~baTOJ
zF)L@j`cW+&*!J$(wBBu1;;*N&ziOXWt;6)FP{lD#wtZD{_pXo5t>+xm$|YOZWo=uV
z71}*>QTIyo83&ifWGpjRT<rX+j%Bx~^)E$^1^K;YYc5Y*!)eZX=GS7|)wMTW_Ad1-
z3A|l*+fn~>P{*BJvs=Tq^}dif8o94f-TTYbIdbXtD#k)(?$OI%hd!Jt`OI~Fq4E^*
z6++^Ps`;;FQnpUeJ}5h<VZAhmL0*>Pnb#!@-MYUxi%qZP9@`-s`%(0)M|;OrxmQOr
zbXdgSUdt1m#L)DquV-7yTa{gZxWxaL9IU>1!l}rxcmC}@Jsyp>OXkgNWN-^s**aU&
zGG?iY<079QHF~=bUr>|@xVG!Z!`;uFqKyL!`xg1Aer(w8v|S~!{p$0uLsBl5<<*Kz
zESo&rPJL}_4qL%)vi4?nm{3B?{G{w(GqyfBy)~+OU3B(3=Fs@>Tl?MKc<c@mc(!Pp
z)fNA<D(SHiR?$J*&y)#Qo|e;WN_R9{`QgR1MYHF{ONy)2%<Fm8q|CIJ$y#WRdIVqN
zj_`x-YFlc<4b4tFvUX$%c5}&2S-tX5%fg8=;#ZEuvOn9${cP8^GpnQ;`|p*WuBp^&
zyLtFX<Dw&u((5Nldc>UG9(Alu=M;PAvQR^*iBpgMU$uPZ@<}XD!|Pr@)H(2Z^U-X#
zEu1!~k+Qxg#n~IzKDPEgd*qmS;=Wg>&&5g~-==N3_>__RqHV2$J~ex^ZJbZ!t`<Ds
zHhIshxhr;7NG~`c$8>0VVTQ?`cD{>w&9k?(S|42yo*dt}!FRKmi0n@0k3CW!H<*8v
zoIksFRk)dMPVZHXU3X5LlKQh#y?ViiExl>Q8x>A&)GO7gYg~1H;^Fc+mw$Jh_TxBW
z8|!wubE(;E*LuOUzk4=MsjR&geyq5)TT11u-dkhqs=B2ww;Wh{MqQIJ`h(u9nsd2x
zLPJiKsW5KKKO?Yn=GsXT+a??iS+=#_HzV!Ra~XybE!NEL3lGC|)3^UW$z%I>cC>4w
zG4r2dxrpYTJtZ$*>c2d>exCKN9@|SCr`nt+o;<hlq_6eG?ZT%Hd`!~c<~L!lBv-*s
zS*vM%R@3S=re&@aoW{M9;qX+6S8jDat9HtqkK)}d9lZO0#F0+PGsfMiOC`FNzluD6
za9h`Q_aoKI{P%FiG|aSDSK8=&w#Uoz<gcTrjaYB#9J`RQLdZAIlCeHE$L+qaWtA)2
zt}UL=Y6W+Gyc*RzCr5S;)3vNAf|kBb0_kRPt3oB_|8nInJY%={{LgB^7^m(vlcQsW
z)a0u!@MuLg&Ny&2@uYL($;-P;ul+t5JbCI3-L)G%FWdh()aQFht9F`&nz+;@$#yo4
z|6FhVlx#NrOWLH(_efE9V}YiBvV#ZLLzlm;R~&1NG%MvT{#~|{JFho8pm);JGwPdN
z?;h_o^bCqo?cMa`!bF+roF(~6B55Zc9Qb^6NoO5jx8HK-gQ3B<wT@m@Y>dnJyEpI6
z)>!XryTr<0^$H*OdF<Vt*)Ig+S*F#PUU@7Os%O}If4TCWxJ}D`Nnbdw)|M=F;nS=0
zTW+YH-Z<HMW?SN;?q2pfPG6zvr!UDm?d<GLmOWD~`{%;<%Dpo;iYS?=tFV|Z&C|WK
zJFo4j=we%eZmw!0k6ZOg)rp$btIs@~d8^rCA`72~$8JUCZwrnsUV56{Eb8snwVr2A
zpPSi|Xu3_?YxDoDw<33O#ab2K*)&5qGU&y(S5r9IQyx86JYTy-`1cJXVTLJRbFW(a
zUE`DCJuNVItysdjpxI5S-!q<loF|-Odp!4E+}hojBc}zk81BEAw_h?*sQmGLi$a;Y
zjXvFVri^!TY8QUUXcT+BQK!O8%A@naXU|7d{N4pTYI0;yn#{X*DsPqVVbQtY4?KEl
zT3aM6S1Z?kY~gLETLI#XoIc{3yq8wrxs=8^UGcTPaqcvwT0u>x)XRcTKE08dFy*n&
z|Hn3OzWiTkzj;pTS@zX?e!mG&y7T?tk?ZHa#_bpNzWw$vQ}OAK`9cSdhVE-tuX{aZ
zF++r&`vN-wr75po6gsWl`89Zp)%7Dk&poN$SJxk!yYAZCKi`XT-ly9MrS4Ai)4%SN
zSO4&8`JY~A&pw&9d#?Mg?w>s0xqPqe<Y!YJf85j0R}wVwog-tCVt-YmcI5p%lCuR{
zmOhu+e(valy7^Q7$~0PCTwdoM|1qh=UU0hhXD5zBi&wXQT$WL=^7TIco8pC!4lw?4
zTfA(^0nJaU&sSAhIn_9D9Th*qC%dcqNvlZ19;3Ir51lnz^0oZQPciwY_xyg{nB;c<
zVm_w;!z8_T;)3_irkveYJZnjPlkiy$hJ&sL92%K<xn?9y=yGgiWMokM$-*kaz{H@#
zz`(%Jz~sQl!114fb&8<Fh6N1atb$T86$TFvg{3M@(+u3eBgrT(nRVyH#Rn@4S&hqL
zB2LGx4dZt2OjJ*}tHrp6WlxRggk@(YS`-V(EDvTlKUb=rOCUTT#BrhRWUZqs*R8m`
zjOD1LtW{`MNOS0NDG_t7*G}`IrW^5C3$2V;YF3*i8JfLNw|cFIph#tzrt}P^cgIpr
zc<K4<=l=dq)uKZ2?g2LL>rbo2Hy#X1wJ4maGgCo)j*Lm#J&(4_M~#-8nj*w7b27sL
z(O_X2j}5+A*SHh8Gm1ZcN^RTfzxc|HsSj2hV|n^bY{SY2)8;Ibe5ZCJjA3`+tfWm|
z-ZP$knW4W?Kj!OYwX=b%JNW+Z_qw1FxhdInk@)JY6%31AZktbCFSv@~x69sV<^KC?
zE5CQMY9us7m8Qxyyq7&H?N{CF9P2piz?EsbYi}qu=oe1hG>2jF1fwpGl#;Kr9qV!}
z9&~Cfn;|S2U2ySH%C^X`fS&D!B{FVpoJ$t_EuOhZ&}*@->O?owNr@^Qnqu1pvrb)A
zcCeOSb%am*wLsbAu5`|&Ps2GrxrhYDiLoqeN=wX4Yx?~(J;70FUwYi^eT5p&HQ0?<
zt~q6DMJ#o+`TA0Lc75I!_37zKW^N^KCB9tCY!_V>;&Z@IWMxFohDG6y2Q;?@O8ZPY
z5xBhA(`_b$;i(l$8FM?sCd8bas~YXN?&yCvwrr28SJz!<&f9ijiOi(HJH3{S%a$;R
z>Z!|WUgX`yRd%{*yN>_rHAx~Bte(-AL#~}N(e-_ud3Eh#nYGvF1^(kSShv$}?(>kU
z2isQP+bt`);!1_-*Cnxui`RsDFJilT`IN>4u_Y^KGUtbE*!^J9u|?fFlcL;u*?bdc
z9$7Z2F#7P4up_M=>(6-vOyD(qa?E4nxoJ`<JO|y_IC&a(O_`+7xz$tqo51auML+ed
zUnE5{JvDBMwlw9NYbR*CCD8f(mD!;ou3Jk&_uj31UT^<ie9PR1e{bH-_i%r^^>U!E
z_NkEM-H~!tC4SkVvmX~uExi=8e)fa<wA{bN=ZpT?yeQy5_5VuweF3|ha~RCcwXPkg
zEo{^DnWj{>GOkzqYUZPZy-$qRb5E7g%!=vY+UPhxdfQE>#nL9fQd1^;ip*$;z4ppO
zSvth%Bvb3Xiw-56$F)}+)(cB?JiF1^ODyEnN#Tm@yxIq!IF0;G4_(OLBKIZd{LD*V
z3zYuk?m7G1r1qZqMVHf7TR05tbl+S#<Y)V3!={s-ERh-wlUGfClz3XMP$PVun%jH(
zY0}w&-8#mSZ8y#-v<V;R)jQ;%$Fn4PnH2Adf=l~*Pq39Le!MZW%Dipemp*0VVxj#3
zLESZh-i{6*FMs^^M14U|rbVpqQH>p+)Ff|B@Ke%?zI7*L`OcpKl{P#|eE;(+ocRO`
zJ3Q7T3LT%eO-V?%BRM%vWZ{p)tu@|Vc7OOI=gl}Q`RhVfqf??^$c_n)Yd6F(HFH{L
zIP#lq+9>vAg{y5<V1M10#o}dW7TdotZ2R(2yY5>8v!<E)f(A*JVo%3A34fYSm_&J6
zE}0>l#3MP!^J{A`f2Ys$gpkF|%QQmgFz{xSEVc44Vrdd!iS^qy!Ro`;C5H01TGsa@
z?`-0EUEY}JD&jWLrb0E<;=mcHA61Jv1w|)2&zcctG*Po5MNio3S>-chVHV%lM_<mm
z*1XW*da!MkWbn)fE@>94cP@KgyME?FlM`A>Dn3=cNAkj^NZZV3-;n0}yyj)JyzQ@z
z|Ib(bSlsuI<%NCWrT%qq-0HS|aQ*wwR9#J0M@_>ptby;_iT_JA8VoKeIBOhZ?aRv1
zlll50#w^VGX2+2!8zecL0)7hgrKl(|ZH&-nuUz!zDZje$X7*OLaw**tPn3<Gqz2S}
z7UUD1>X)S9A2e~*m1TXpE8ly*3KmvrOEnW-^=6Njqe)=sS~Fp%h`C(htQSHz1zm0T
zT&oqrJL#&{tj!A9rW|2UcZKdwO<J22`#R=f*P?YgQ)l@|_g(1`J7W-*DSIh0S^BYr
zd&8-VeHCYt?O6<NHid*Sl+4kQl{D!J>j_}U@AOtI4si^7vs6^%?TN=Smj$OgUh)x`
z*z-Ijb5-mA8|#d=>CQbKAnJAEPFC`I<H@cjlV+cJ*~PmzbBXral?QfBaH?CoW2WA5
zg~K^=!d5r-<{56+?s{8i>h$34x{95zeAad-3NLcVSK}y(V_Z|dH>Ebe`u(rC@0-N;
zJ!{pjIiD@E;*$B^&%7IIuHL@)W7qZrpMCYK?=QZ`d|iA$Z`*sPZ{8Dr&ANIpt}s2C
z+xpB?y%pN4E#!aRaO|(0wL_kX_l14<8-K<h&6}pR8anEECrez9P(Gi?>d$$ptNv(9
z<KCU|?QSU-m8`iIGpmG7H1hCnV(31(WYUd7jcYONRsjkJb*DN`TP%?DHbZ5xrn7X&
zw2e`TDvhRl<@|0>SYa=8?EmuiC~c1`USXf&MEAt9zAyj2hEw-TBtzY|l?xbsUpPwd
zQfOD}@Ruxl;U@m;Vqag$LSw}YFXmg9q7IlW`F|(FS6k}h1hp^A?cY96V~aY!V%h7a
z@Z4+r*By#n?Vyt+`uo^1uW#2XWjZHIY|8GlSb3d=@ta$fqea8c$~uG0)ZH0XYQlF_
zgU?*!P|B3rkTtu^tmx#o*qFmB&88d*pZ#XZ;u34|#wl*$%RcsKGJiQBD!3%{UC!FX
zuFsBhDhfhBdHZNDdXT;~ODT<M#^%Pyv#<TVXS&bMH*4AX*#%yQ%`Y=`xGdYMQ0V3E
zbWw=C<d~-3i|+Ks$71h)9MNC*v5)(~W0n8UEl&By74;_1c`B1%c_uyXqu1@4$1PzN
zEpmMEzA`lrXMU(W`1sVaq%Nl@<CV)hE*`h5TXf^)-yh3m*<U@14d2%HWk#X@kCmmp
zDRVc)yBE6ac~1~O&QkaHmw1uwoGDL3KIp78-4qeA#h~M=#!};5f$p<;X75d3vr{oK
z)w6n%mwxnx4{IZRXGXfMIup@z=l^Q+!W(;nn>RRf2O5;FaMyixPXE@W#nU#t;8seU
zq<`eg3Uv=L54l#46h-0H^?zU8wXVJ~x#pYt{=M%q>kDrm-}imv`M<A~=dnKg9ruH!
z@b5|gchxtH?Z2Iy{`d28|LXSL<rlrSCbB2XW{0hrc<t|*H~(cnOWcgnPuYC%TE~XV
z>qU}`kNjmya+j}+;*Z|4IB=aT=TjNu8UNIe74<4gNIk5t2^3i5s`K-Ey;P&r#!Fmt
z1@pEo4U}Q5TxKk^VzJdi5vgT?UUMI694)mz_)to5ljW2pX-OYbb~4B2F0@;#z$&o4
z$iSnJ=|H}?d$X}d^UUyOLGxzo8I^V?n(aLVW~#Ta8n)PYv^Yi7L<+EaooMl>Xfckc
z39KmcvuO3!X!WaT4O!6=aG@pUN2~hv)@X~isE9V76D>&*ZES`KcU{%jf6MyrlGO9G
zgrPM_$1FhWT9T<*WXyJfY7c?<?Sd)d$vlP~sp1OLzb2_ZiTubMmHR)?{<LdKp;BkG
zTKTjN;SzNR0cTd<Z2||1WEQLt3=!*MQB374PHjD&tTwq~>Wwa|OZEmwSw#*C^n_b-
zR*1=O;Y?Pj<O!5{n@}%(h-(IWAx~l9u^Y`t(tE;RR9WumS-ZU|P@s3BK<@>K-fI=T
zH)izSvS_}Q(R=4b^Ti*%_iyw*+R=NjqVMjEzUv&VOTJhARpYvH)cY=bUD20_;t=;8
zX?3g@v;XYq-?gn&DzRgqn%L|19GS+X<ORtGn4<~`yFDjruUes9y)b6?CJWK25-T=Z
zvl!;CeId>w(WG{n&G2LHY=*{vhZ+6V80JlET(rbWLQ$*LD8@1<&*bH#?CIj~{%;fb
za;?*Q(WHiQsh?>ATNPLzMu-RPs96;*;GWqsb2@9lhi2cOQyg~G*mbakO0xQAPL0%L
zh~7EXxPv7|k~JZ7YO?0E#FbOiZca_}oW^uuT3qI|_(*{S%NB2rqR^*p?(7E2%j|rw
z`v)yg3f!KpuhzkBIAc#(S(E0BmJ@X?H%pDeG(yq^+GYyNG6nJrDisIHENRf+YSQrQ
z`ApV|iPJ5FxQ!j^)EShv1_}oYh%_6f2n2CErb;w+g+H6D*cc?WFo<vB?Byy9TRYMi
zC)v7gZIF7{{cU@4)C3kg)0WE?Ep8_STqWnS39<?vXufLMd|i{_s$}oYn{!zw&dv1{
z5d6<Djm2@^<C9Y#M$UWTDG(qqFMxqD;pe=MkpiDiPW`~i`c;$RLuJ(&$?Do;p^FsQ
zjGhQ>{^s^o!M#GIuHp&*&Sm|7PYAv!7yHmIENmsjq^ZC(bK$p3k$GW4JxmfGmkH~g
zRIuet<vhYAv{Yt+rw)^c!~&u6jh}LLnI>=fT5jdo$l5q*%NACvRVjN^rPe5F$yTWG
zPZn7`#a>~Tp!vxd&6y(Sk0xuc$PoN6$BJ>PD@V&)%jUP1(;l8&8aHcREZ4FNlb6a~
zZ2faAzdW_1Kq5(Tng8Ak{$(@T%U>k1zic;4TRv%};AGF`-3u4CN=#^2B`nT0LB^3^
zc2n}_#buK@G?e};rJPx+RH!~{v8DK$C7sfWvls1HEM=H#-??&4N-Dod%(6?ViDE`^
zY|`sC%B(XKa1;_$Rp@37?A{Tace6~5;kxkAKt=YfinSY9!#pz{NVG^jocm~}z=M<X
zj_(q9!nrJF)x2}N<_QD{{9ZZl>@I--0Y?9yQ+zn8r?4%!y<I1Gl3j0V|IdnS=Ew=%
znG;T^<?zoE-fgy!wKCZ&aKe{Wb>FIHB#W<j|6MWnSwCNgy~ZZR2EmnVo28qlOsou)
zm|$c%{c_Z8Hi`Rc(X2~S-!gT{90;7U(aPORi+@L6ci<8$N49l}vKC5<<P5zhsaM7-
ziXSlI5_#mYbphwaj@miLr*QvIYzf-1CM2sRW!IY1S!xlI)6#deJTIMol*NEyaq=Ut
z_HvEojHv=i?vY{3MXF!Tc$+k%&T3_v(TX((#MFX}Wfv*#Jvfs`aM3ys$N8RfDhyW|
zeN~cMEv(6@v+Rc80<Kk-8x?tfD6?r;vrUoQ{3BH?Si@|o$g!Jz)@E#;2`w|+TdsS~
zWpY}Uk|mJ5bN)}swO=~t-ty|btF?gXR4uF3ItEvFo?Qzbo9_xVPYz!mS-M0irF|V!
z(u`1P1<_SoH6C$)nr@fZD43S5FZ#2r>EVQhk~-dw8+@KMsCq7zN-ejv5)%2*$+d-X
zS%r*Kl>MS9d6Ny7XeoKt|JGbrD1M;Z*Zluhuc^u2y&1+W&H}RymYS@d;&F1HpXQp^
zo%3v8&nsFf_%?Id;Tf!MuiHGbqiQOX{H&L6)L!w6Yip9>ipc5?p2n?Wzh+csZG9<q
z@aM1PEItP<4Ojerm3n%j#+nIM_Rj>Ju5JBgyk|?b+R}iPdYd|#4s;40p0#o1%IRIR
zjP?l5|7Gx2P34&AM5#t2mW_LlnC?5UQ{;Qurfr7=_cxcZ7_vRw(PCyj_t>ww_c?cl
zubQ&%`r7w9Y9D;u`L&|oKYZP@<LyS@vj1`91X{1Ry|}Ku^Vq+L8H}D~`p>u<Z;I{v
zkp0fw{f+r<*@wG+>=LMYvySo8f)CfX{@x^$AIN?G^lq+>$xZ*aOuV#h5!+==k;p)m
z)e^NE#p^DzZDeiCpRFWjoqAz)ilnrccBi$_?vq{@*_PGFwC~aCJj1q0rSe>~<uXH^
z=ntDz47oy13B=r(;{KxL=k9|4%a(mD@%Q3(_nDo<bY^*f<gsqe^=xluoVz2uvHM^}
z)xk?X0z0I0tganr6YNmEBb*q1K=2KJ?CJ?M7bCm5^yDoK@~;cO?NO3QEuYaPtn;SR
z=4R*TGaE~}9hD}os@|>1eKB>dK^oJ?RqL|^w!WyiIjt*w<L(oVI@>m^W?OhfFuf({
z#M<S@b~+yQzW2O#>$am4TKk_~Kl<VNuI!u%YHtq+t+~X~c>Kzl<1I4|XgK8@|9|pQ
zXuHrhgBcdFW&0ciOg~>{H9Rm~<FbIw!Jf#RH5b;~<#H=WiMKgh{=I%h^5T`%Mpu6v
z7ngf^V)YuCStij!9gAfRt(IP1VzrBPzK-(c-;%1)lFQbJD0FuoylAg<NPEf*d*?gC
zMIW_o!~z3PDb8%0UNB?Ji^X=wdd@JeNjk-Ito}#4fX<crRX57HZi?NRkyE2|C1&f}
z2es9^ZvI;my!ayD<VfST*>-CLrku{Xd@<&VS?|J@s$1<9WoktWw5=uX?cTOJ=(<&p
z>XFy>j9Uzh4|NJPuRO5#kY0}RH=XUqF?Y74Y+rxKLGp0!N<~lRjjNXL5}05pxc%Yb
zr(#E#A6=0CZz$uwXTll*+3zzVOJ5wV@yRLWxqS7{nUtC<+HZGd^+a;SoMrMnzW18j
zk1Th=KbIyy<^Ol!3R8eFD`UNFZvDs0d`lN723+D}kK?k?y{LC=!L+Hjn0FrkyXe-N
zy$`3%efa$Qy442td4=_2NAFtvoX~wRXN|%njgK38ABwAGF1BU7R&YBexL~8zDNl<_
zk}D5+O17%DrduqY5~DhM@{CQBtuCr<xs%-C9`Z(D=G(oO+<b3d{&V)#cDJ)ix;I#p
zyfxe(%@OGd?c8WRBj6I(|A6|X6P|s$aMR~5*VhmH-x!`3Jd_E@d)itQX?Xex+eD33
zmlY)pb?%)#rM>HR=Itl#kFV(eH#)q0rSMmu!?Rc232{peOEWa%NtIY~`288dr5jeQ
zaZKwn7vI{YdQU}cSCw(gLe4D#k@|P_ZQpWRJ-xi{9XE5&!8<w%6*>xACU8dVe0}z3
z)#?EL#R*gW@0BL#>}u#(aNw}_!!<{e0{Crf`8$*NMcy4-pep|D!sR87#ti>dmOkM7
zv^e;m6W`~<_sibt{WoD-azU)+&ID7=a$`${J(mt1EEHtVxV?O7eMoN1I>EeC+Ezk8
z-Y3>7+urI-v+dHV+<USxNHW+#E9nu(6REm|)@5f`?0uwp_~MdpF_wD|Ke^~?p>vR{
z_Q-+rGRe&^xVIJvZJQ7<b?w#(yvHWr6?`*+)#LvJ)<q7g|90E`YdLMem~&v?)0`0D
zKn1QZ1~Yc>Omp^+^jpc9bbZ3IzxVBA#s4Y1<@mwP`7y>SbqCkZXZmV4*V=xHyMI7`
z{VlP19s4%U>^hYEyI5*7@4oNN;*%#WT60y9eez3=;BphA?M6C=z4PuYy&-(!?)PK0
zGSbdLn`cQIHVNrG?NSY1wdUh?ox?^7Oon{()0Co4G%^~rRlF82QnX6tE_=W=U;fvE
zeD3@AF25^ZxN6;{>7nb{-t1aw|5PTBdu9W}%mD7d>HTXy%v*fm{?`RZ&fl05bwKD%
z!_?IVkq7!%I0RUa>N9+KFy*Ly<o*Q@-YJyMNIk%%_f^08t^PH>MgJd0=Gocv{5rb2
z_R^a357pm3e8zvjw9!5GbO&3Y@y~q{=X5niPo8j;Ia&DLBa`LohE3uO7ne<8U`(+n
zbUfkO;MB?@Yu6$XsB}b^Nh?gH<D#NCAItKRm<|D@<-Lm1el9a7B}h)?KPjMcwe7)*
zX+oK9GGBuh9qXJelb3nbt091KE5|0Ot*7QU)CtXY`;vY>TswO08;%W^pS`-ea9h^T
zrXL{(f~I^?xz-%@dEt%Dl$lO%j;<54jauZEndIs(BBcA~j#J8X){`1rswSogx^~B{
z4UC$=5uu(CGDYL-th7aQRK3>LZC&7!=z49QZqT}4e`_<=i)luGoi-^?cIuXfvTy(2
z{B?dkVQtjCz1L^et`^$yzVpqo39DSEB)krr9l@F4JUL|bY#~p<`noAU0@`l|YJLA_
zuqWRrddCd?%s?g4dvA>M=07s9b8MA3T~PTmNO7hnkE&L_=gy!AtqvAOo+%vmGdM0~
z?!I?SfzNWwEyWWWOPahnbr{_?ALR?bQ}C3p>5Rg}1?F*uCq%7tHnJ|-`7EhbHtbqt
z>XbkiLFV;$LtZ^M>AbUQsoY|PRqUFE1v0Ior>kUCn>4m`sBA8*l4Nc2m@=nZtnA1t
zRbdvBhdEQvFSA+e&d4I{e8NZNbGzKx2tn1Zz+aK<O^bfLY*nZ}k#SUh(-EcBbHi>i
zzMfxpB18SZR#Vi>b8_~wtJM9ksGTxa+E%o<WbKr)BNrU~IHS4@T@)8>5H(s?d|@Hm
zO%0AkA32h)s)ouYZ50iZy22US;c+r_QFx?dsK!hd4c7%md_tOgTOEWNB0KrAJe7Oq
z)N1V0HJkC?hH-+a#-bE)uhK=oYn;FCWK-Q8(D3?%#M507=}uc?XVuK=P;Xzi_`Al!
zj#W>0a;v`$Vc4*0)hRx67LCKbExR=qa~liUMYuBF;@}PPH97srSY?p~57*f(GcGG`
zpQiM4;n5o_(=D2AM7jza8eLSH9C)KJL)Ory&xQNc1Oe6EffE`Sx<Ui)ZSqjDyKMP*
z-<@08nZ9<bk8f>v`Kj~YBkac{K`vdf>Y};xk58y_u3pD&t#tcD-CC3DdBP0acwcXH
zb=!YPs{37MS+tkV-qOdrHTS>eyAiaH^VV<0D-kzUG~RJIWeb~mD5l1hE;io2Yo?me
ztE=x=F2sB;u&>W>_<Vu+yKU1Q;RL%5zbX&qtMUF@mGs^vd)gPAf1I>0k+pzTX=YV@
zMmYoL6DG%Y0h^bqv5xak6y5ve>Sx-qBKoM2FYC5+XEEC+cYB-p&72$B*<UT;ZWAtZ
zT@d!><#D5`cUGVD7=J{$8y>aD{_k3)6<ay=mwRQTSi-h0#f{F~a>X;%SSED7`l)OX
z(BLy6Ya`b|jt04ZD};m&TwA=%V#5FAz!NTg#|*g6?Qs<eTjXkctY8K=+j9A&CZE}S
z7Ht-ti^RW5v|dwUY_v38Dz|TgySCRwrGQj_f9XwA^`{BBY+_yMG-vW`2BoXYJ|CY2
z9eEi4V9APUaW<|&Ou{qlr&t_&Gj*eDy2??p2*V|dwyX_3(|k_)DTj9FhF38KB^_&(
zKJBR9*u}eMf~RzVYn!i1i^`hA$$PI93r?7DPWhbIiETn5Le-8f4tqAn`o<}pn7-$A
zEMK9V%R7nHHCz3Z1SZ|7{`OUX`|8`^tt+_RZ+gULtihA0xvNz($8&<7z-r#L45x!%
zT(qm_=ro%7BG9lql3OO#F+J)6i>}0X<5Lcw-TMA_G6nM}H-rjImJ7_DEFt$;{Q8bG
z(VHqfx^l){NqK<`TYmJY9f&+PDYn;la%0mZ1tZCGrm3>>E>6t6I}BAcJWY;hu?B7u
z-(JV7sgdY>TI%LSDG$xfIW68PZkL%>EoWGxnXuB;`d6y9o8c<n#S+aTHx8_C|Ij7k
zv2$_xI|;$_OIIy3xw>MN+jjw%RnJTK9Ao_hgHA3yrSAANl4G60B<pUCCzE#XP6<(D
zvPlxmSSGkL%a8Ms@afdA_QJQ)`B(3l6V~veT3c{Fr_3ksXpdBruM<wnM;jj1dDSM@
zck@<oSBvCKMgilAJ5Fi{6iDpc!9Uw%v6M(!+KzK?1blK{w*OZX5a`@lpp$ZK^MZse
z3R`72GIKG$^-)upz?ms@u0bwa(b!&_VgG|nBk9?<#lwXchiym>*gADlV9z9VhleMt
zrv8$CY&Tu{#Jr;gOh1$&{;;?hX%#v<<h+)=oyH#?vVG0kAE}aTCJIJV8hKP7FjZIz
z#Px9`FkNo+h+erkbj`Iur8w7<zKyNHuMJ)(^!adZeX)4WBD2$R*MnlW>TGKKtHlyI
z>5hJ_g~INhK5t#aolDD=mJ83iqY`G>cvAhTYJ=|qbEO}%?mhZ-)Y4+7>W)=2V-JPA
znP6mdMse$pqx`HN_^bacl3n}psH@?XexDFs^A9~u8ckb#7##vUzlpROasDsz%j0^P
zBzL+cR^)Kh@eZfs_NSRnEn;BwebaW_ZeLvB0)|$-Vm;%3i(QlX4xhVoMae6&AStjw
zrR$rk%5kxy#g?|Jn+{qoJuh)L<F3Qe=Z^L+&bh*w+HO{|ZqhA0KN?Fa<^H~C-Eid6
zcAbN|tO~8`HcWB!oRnO0OIwmZTV09amQ2}=Xt_H}pQte2JL#YDR?sHyh|=V^*qUF)
z@~hmQOk!6);lOf*jVbk3#bP;?YSrM$PYY*EO~`%8w)%$aEtzfwCdn?zT?tKbLN2bl
zr|wO%@HlGk9XKbl?iA<!o65U-l#+{H9kt(?sn&EMsog1$>$J&2Rz?@cY}bi8nU)f(
z{x`65t`}c>d}sSl0iT_n`r<m1G)x}9*Kl#-i8cy0%-q-|Qno?EX_0x+{X+*Q&R(gx
z=dDQLf19waQ-XIUiQM2{y2(YsuH3}D|6TjK>@{zzzS}+0m2*E~eUFc0!3+MAC6{&w
zG{u?OD&6>{nyavMU2x+&%epCx!|i|WoONu~%{ziA3f_qmS~x;)UW!`cI^$>v!~Z33
zIO68|i`V4di@6lNG<loPovA{nOZ&nsIaBKu9|^D8v!k~l>waI(Ve7=(heWp*Z2WQW
z0oyu}xPlx;zQ_-r-OKJw)i=5Mee#D*eJc(IZ+xrL@%$*m$E0t`hP{o4jz%&%R=#?+
zwdLdS%Vy0tXa4v<TU7RGamrSQ^hc*OJY|+IKQYO^^dZx_0_%65KQqsrG;txzk_{my
z+Fnw^)4inZqi2*XiB!Mpt$bI})4u*!?cP>S6+NY<Pn=J<gnrD-sSTX-_^j{8`9go&
zyRSZLl<l?PTx4WDRY5Bvxck=%lO6{x1|Qj7)f^#{JMKzOGh)yWOcIFvY4vT9ILE|}
zAS+WY!71X(tQ%Hpmw#$MCm^(ABYW;=hL0a*U4<>1EDkmZPS%KMU-m+XrD4IGhjS7S
zO}@N<{j`#K!(*dAj@?EJg*0{u$=ou^OOouoJd68PyWxi!iv;^?Z%>l{X~B1BN@286
z`JolkTzZErcTPLeE5);A=>>uRGo-nn2`!#^dqTWK@5>^a<HgOdFRinEqi3YhxRBZ9
zqfmoSamUifOBcU3+jCm;A46BihF0gmZILf|-+z$tGBFLm>2P0i!tX4O{}EdG7xgn%
za`q{#>Ny}bU&<ghxZgv_by9!<Yj;1B3LkrRXN+Liiyu}eBw2qS?o3l&5|pjUcbW5$
z@}fm2_`)WrPrW_ysWI#BVhPd35?4NK@~hNsp48LZsdi0!cGZXNdaD<SzUr>p(EYes
zc+V&9%_o|A4oa{!SzKE+amB_dWpg;TXw3HCxah^_%?8YK-Pd$YykzJ0Smh(Py;pXJ
zxA6(b5c>`Vm#GmRpS?MEPwwEI(6m5+%S>tO|JNKgl`}PYK2QJgX=+IKk!c#u7ZbIj
zs`*)KIIcWgxn=V-_7>URCX>k9rqgRUrU`I$#HeyS=}THUH+i@9j2p7wRvDx#_Z?~6
z=D3peam0>E-5qxrl+0f+@qb|2s<q2VqdA^ylGW+`hOZT`ISDIFQPPzZI)8d~`wF28
z3)&gFSQv#C1TuIlPMjFpF>y@|*ZzZRcKwm|S?I2mIPcC0=?(_&cnx6-nZ*`sEVp;*
z@fF&H-`JV-a_9S5vU)EVPA)lfa*of3pj~_~n9hb+Efa2VDd|`we0J&={>9EJ*DS3r
zt8|<UaoO{^zio%lH|O>38QOt|7yp?tb?xO{7PC0cK2(nSdE)=it^>&jWh2jMZh6d^
zUO88H1{0I^x$VlnO*3})J2({1blA1nb!x>HBL~sgmma^E<_1maDHZHb65&^TymxnH
zN16(Uz5>VAQ-^OzSjtIuACTZV7uCuv#1I(4y1QbMT-O>62H|_FyBA(OSen4v=REVN
zboXTsJ7H~e2j)4`9BgY<OdKEcPB>*ZiOD1G<vI^$`;UudTV~1%*`8>132dqH=*;kF
z<MC{oz+_jc)Se_hKX7`<%d`JK@N(YSFL%{`|7y-jYtGI8!n>$Qb}`rV_sJ9du9z;9
z3W`tOm_3>E%_c8yoq6oSOT<|Zp7!Jn$z<kQ$^O*il;Mr}JfaQZFIZN}O#I(jxLQPo
zulljk<m9!65tFyZO#U@dD(XP++AXK0jSkDbT(tB<@G(ovX^-bS&fVv`a-U<h<)JCg
zM_hcYOIU4mR?mC9?)Qu{za{h+FzcW5ak-Eac+bP-af#`_t|Rl9)^jp&aV1)m&Xmy$
zHGOo$DsIZu*uc(kL9v#HN75=?^A$tySzPMMIQhL{>$5MR%-$1rKJHL2o$$t|)0gR7
zXT-d;%TuQtosPc~)U`u=riJ*m2jcG~0yslC!xK2SRxvZ4I4ogp@>L|fDnl}xMf9}7
ztSNz;Dp(~ld1e^t9CkW<VSh@vU3AD)hVE9cEl(Z<OnxNV*s`Ti^F(yh2@%zffTz5j
zb36XGws>gt-VhJ%e3rtS+~aUOaC+<<POfCLGZ!y<p9%c-$R*UoGgs7LOLk;y!cF#^
zn;&#K!xx`aT&gu~#-&b0G4Dy7?|(U`8BXYqoiL$9?!fBx9XDFc8qC@a`mUXF?_p93
zf55WRq%q1%quSw0*j?`_66QUhW^Z_UdYOpxqZ!*&eAdaoYCj;dczfzPBdaq{4_J!L
zymIV`<@|1+(@W0ysp?<Mv0CPAf3Va_bm^HX31?ZNeUIK<AGkNt)yC@6uA6N(ou_>J
zul(_O<Z<Ur?ljE_fnigXqIXQaJx6)!g3u`+mfUj?4U-fLIv~q<rz8HaO!VeylOHg=
zIwJN=r6V|oV^YtO|5ZEhvhI$3#(C?Ktf}6OxqDyl%4p@|-W~R9_SV4W&ZH}A7kcni
zZ8^Sj*AjCHjxY|j4Xp87ABgyI-l*r@+`uYfz1whAfQf6eiQC27p4=BdBu8x76z}VO
zLd13dv=2U_p&q@g57JE6>lU!Sz09}e1ZU6&-;$g56M2tutm*v6?D?Ld|9s-snI?00
zoKE=I!!cv&O@6j(odFzEL;7d_<-7m0&-=4ylY!g>4>Mg=4*owIe!VgfWSOwy?V>la
z-R?_vwFqh22wuN;;o-C&N_Bg$ThttWV|ZqEv)-J;mf!v^-Lp63#FLPP!bkqy<vZ=`
zkSe%JZ=TB0sdx4LL@&hL>F_vf%;)=mibA6%0~f>2jX$mLPGC7UO~8-Cjw^1a#C2KO
z2^#v$DnY@e9Xn4aWGil1zPUqlMuW3o%F8AE69ru7N!>f=dbA+3*?{d7TiMm`M&3Us
zUpu9^BqBMAZ6(Lc2Z{Z)e0u_Ry4)3zQl26IXwA<r7Ijyr>|OYjqqXD9+RnzK``UgT
ze``IpF(ksP_(ozHXGiJ9jtMtX=H2*{`mARv$KOqUhGCqGpN4)*;^dog;#(;P*F(_&
zYw^mDRx=vzalQ-uSTOadWxB<#2a|Y$R9L60*Pav)bqF<i*lsby;Zx*nmrh2>BR-Sv
z`LI0*ne=ehhfZnXmF)(6Pq=n&y)m({_(kQB345=c{=ahaEbH!_GP{kp7wK)kCGfUT
z`DN6Z*QXBedFy<O(`vtKW=QO=1H3V9I}Xh9xHIFO<{8mMy=UuB)N;)6d)(S_XI|Q!
z<Fy@UdCp$%vDP=+wLtH&ZeK^7Rv_mCmJn_sw~MPn8#}HnU3@P=^gheE#LQMBjU~Dl
zII=X)UP_U><?H%P#UMv<LzhR&J=F~^de##s9DTUd?^)rsX%6~wb2}{Do|pW5lH=N$
z_mwMb*PQi7pZYG8u0E3bWpBWtcgK1Ex+i2d|C}4&y7zwbp7@lsv{P4COfNgZ!1ne+
z7^mg~rY9TvR05-(?VWz%-^F_j4rP5}_U|6N5ZzFAFX5lll3Teq84j=e|6BOv!VOx?
zhH{TTK3{!q#mq-qQ%+o~*}2icHJ!uJQYmlWHU?!g9%c7jJ$1I1o>zC;+&S#6%KG8b
zmA$;{^=^bLe0Jx2*|Yg!CuWxA8=GbEPJ2I9<n1rNso&c=&OX&-J~++D!F0#L$Ek;v
zSFg+BxwCk2$J93r6B14?-l{dN^+%RlpEcWSmyac`k<}?X4|h226LYnDa@RGbLt#T-
z!@1ssX&!wkyVXMPoOv>PRo}uVGCejQ{FS|YA6*Nyn;LXaM*5=RE4`VMUiLQLd}{jY
zsoP_g1h-GEs?Su-pQ-(i|GjOA#@?LH1<#V(lry$H(V6zlq3)U9@}iq5KDT`Oek3<P
z*(US<lgJ0}n&}@m9O?VxCx6u5yF}n$LD21aDYJSyUiG90Of!w%Z2Ie;X42mW4BV%<
zdM+*aqT14UCd%fPC7-xNe`3Vcuyo@LGovm+DdljnGdu3SINY+X|KE$<vYBtHd=~cl
zEE2Zb_w~)<vLc;(nO9TZw@%4C-*UEXLg~E)FOQxr?Y!`QiJAR1mC(6*$tRcaC$Ogk
zyZW{<e4FPMdTr-{%Q0VXZ_NC5uk@Ef?$L=>&1vGPub1@9y88Nxo)c5{%Q^kLKaQ~c
zp0+8>+RQ9$t60PKvJRKHCs#XPcUpYxU(n>?_T?VO<1Pi}nLDhqrzT9<daYcri?ez|
zl&k0~Q|}FqwpS!1{(tlByL!X!hwZ2LlkfNFU!1COLTK%$uh-(ma?}2oewwl%Ld-Q(
zBmC#o>l)J(407W6^449v;#d5=X{yhZVvFbd&kIhwUKgbNv^e-yZuzs~Cp8X<wq+d)
zo`>B0?l|SnrBpHZ^DFp{bk17PC3r^JDT{gc*~1gdeC9Ww`8dxf#Qp1{*CC6Y^OrnZ
z+V%9)7qz}0YLA1a{tgrgowmLD+pVuz>2v!eJUM6hwg;^DQQ=#0=<zK_iI9!E8#|O&
zxpiip&0<w=^{$HDZnl&={7Aei=Yss*e9I%Z)qSsE|J$zc@xJf1pB{z1TaR94Te_I{
z=8Rt(UQbv#hvCgJ_f=e%KTf;E^x<;t|M$#kTcj=~9gCkJ{&RxKtx4Z|AKmYpZ2t4%
z|Hn`N%Ve<4)d}wwj(<Pvg1%7zlVq;)!WCR=<5z#@TFN$&^JqKAr}Lq^PM@4}NH*<h
z@6z)ci|?Cs_VDkRR@QmrbB9EIr$k5hR+G65970SM92yY;3XY6RxF#9C_@H>CLrA_)
zD4<Zut()oA&8+a(iPD-(JEjO+IQZo7Bz12q8_S8OPtIVAOl#5jpy1UU!&1`daar}q
zTw{5rrz=0N@LZ@IEwr{XNG-S{AlU28%8-@LGbE~|N-9fNGR#VtDE8&orUelTGnZaF
z(q*Q6uuXW|mdvLYrrL%tIiXWxHE}Y-fjK7_*6o}SGV^~s?@f+`GdrF=-k0k&@n%wG
z^FiL|4ev_23%%VgurBAgF=@h=DZDGaxo2GXReJkuOj^I`jDiKZTCBH!^uCZxI(&+Y
zdy`g0VDgbiM`gw8WhS2a=)0G7!o+*ulMnW9ot8gy(v9>RA5@Q~{rPv{Pjt@<^-D4K
ze{KK#`1Jhp`uzWPbsa(5CS1C`YDz5YR7Kg+OND`VgBH9Ek;s1D`D{W#+t$Vz3}vi8
zE)+4P%y{TL+xH-ArhqL+V29w;h)2z;ZD|*p7vw6kX6kb(9A)tmVM$77eOb2HQNHh5
zvg10NkS>NjjUg9WQ>SbQc4p!znVckIHgkS_vX6SK>)IPZiXKW6Mece0->%vc8u;;U
zSej%eSHiuN$vYpOnw@WVM09HDsqeF0DsJsmpZ~e{=kv<0wVqt!(JbzFop_v%bk5^b
z-8Q-M{>z=y9fSGOFSPhhdNik4;O5cjg{v4XFEo4gatN|7y1Pt-U0tjtGIN99!>IBt
zQcrHVC~+>hl_OKr8to|aW=6bZuhR}!$t5v6WHe99USCtl_x0%ojmt(M0*!v5TdsF(
zjnT+0e>e4e+>%#EIn^h!PJP6d|MH&Vlj)Z`t@B+TO`0wjDsgd3{`~5DX0ofdef_pR
zU$B#N;tnP&)29o4|NWY<@slQ(qNZ?^g<yME?Tg1QdVN|BJ@U_=Z#=%lmqEGL?A8v$
zzW+iC**2Zv`keM;R*c%W+?_(4OI~jMdu`=Xh8tTyWoD(=bV?uObL3p=qPpa!*PNv@
z+=bawk4Oe`rifVIE>A6a5GHQVdvx-R<cU{=!ymgond+0~%OX10^5|Ky$lQsgQ*Wn!
zlnA+P{%o%B%<9*%v;E>{*KU94H&s%+v{zbX#^PGRXW@OU+~+(G2^o5*Buo-dS?eTh
z751>GXX`;P>Ddu%v(l>1ixs-)ZsIV`TAS1+={TeDnbi9Yo=dd*=Z5O#xEz+6@^go&
zujNAL92s}rNej%y0+&xs>%RDzBW%mj3pW)bFLY0vcF`><OKZ)CD?*XH%Xh8Y(Eon+
zx+N!FC7rx@12zAD5`S&{|L^br|Nox~SN9HG)gUnQcCDJ)j`LfcBKN-ZJDoILUDUUw
z>Ah!QX2O*PQ|7Kw5KfRhp7ltJk>jP_@x_vd&NNS*>#1FG>GRF65<-X1JErigxp8Ux
z22;jgYq_%ulH?s{T$^%Gmf!EQa^eOar3prBvVCI|I;8WytS>$3ukFUDFf~El!s@SE
zcLa;Vo4QoFHpvY=K2cnW8OaX%Y^OA>UmkImP&q!2*DdyBmhel7$qRQ)NaUKXcxT3@
zM*=q)U%NAN-poj8U466heVBw`r$>&-tT@Y*2vwtbwH!YrjV3gv&0Tfv`I7H8y*-B_
z*%+6)3Oo_NpmW6Q_Un7j+m5jQ=PbBX%QWFu`JE5apCfbQwihHhKN8Vz-gA;~PRMoL
zD8&O!CpzoCzB;>CZIXemn{dN7uUnB#pIo)pvR8h+xa9GnNcm*V2?cyFW0;F?EPPjz
z?yA=uIX`B}LEZ&x+?WDp+Nw)B{Zwi55P#e|VUmm81{vP?{@r30zfHV;Y;dxE`{SCL
zT7cnqPhNv>$%~{f9c9$&bMdou?38@tF3@B3F+QwV$+*{J!lV-`dG-l$-+DIX?52jZ
zsxdDa<_cI`j(2p`NL5NQ6K0>bRp&y{%Z-oRWu4~DJ9eZi$A#NdYx|UT$s<}1L%0|>
za~aNk;HeRGHaH=vYF`PfnsR}PP!und`w7qg-TK>ocs^TAnI*85OM98-D}{7+sa;Bw
z$`p3r%@(^U?Qw7OhT^rabNs#vJetUz$jN%{c|gX59@7o2A}1wy@6Hs|;Y<+TP;$ck
zghK1CoRgxAMf3IUmvqM*>Q;3xxgr~&)sQ5|p|UO__vfh(r~k6P<XeANt)N*u=HaA`
z>B5@pEMA9&ZmRff7jL^_i_Wp7(cc7{o{0pyxmm~_J*Ve3;fQE&l(Oe_AI6r7O$y6w
zQdLU#UESg5_0HF3``N&VlY99q&$u3Q;EKL-IJ$7kiaAquP7mYgoE(@lM{v4J@b;_u
zzV|a1HBB{{FgeY{_pA(m%VMkfdjHOM@4xu+u!YT?f3|@M|4&Rk(K)};t;fbxX|4uy
z!U?Y9tfh&Lse9vHo>kmAW>}cU`miO->*<8ZM*YyH<f+|JR}~|8BPZ??exuBrwAiER
zO>@iwCx!MuhF?x~_m*Fo5bqE^G04r^nR{!;)*ZhW{F*gM@BXh7(_CI(yD8Vf{V{pc
zggXKIwXUgj?ebZvr)#tC^TZE_CS2OAx7zZNm8O(U-!gtP)lI8*2THBa3-OmO=Q(#g
ziDi!U#;kJ>E<f6&$F*xoUB{+h#*=z$-CF9__P=@IDHVCS<CNCt2W!tr%YWm$m9~53
z%}LY3H2tobm7IKgf9Cz*?te}#B0>w|9XuJ{YKV37h&NsLx_pIo!I~pB_A&o2UzgH(
zILYAS+PGlvh(%GptGSBugD=kZZM-F_%XM{mzO>T5R;&ExxTP28Jrv(}>f6QnkEC_p
zv&jVcIZb%J@3?WEWk{IC<D-YB@EfFy9*FYSz1)7_;~WWl(@5QQAJvwYa<4jgrg*pV
zs<SJ0I<9f#eHT`&b><7-DL=(EDykcdw38zpvwS$FeU_46Tls9Gu-Ei25g+&Li#bwx
zDpI3eZ;|Iar<5~CI0ed@wyCMgoDeGz;>w+>q+&g#k*B@PS>(3z=2?HI)kSEE-8rr%
zw?=o@yE6WDToaXMwN)4%J@)m>#_8TKc0d2&R`;anj6u_nzL%lRMp<Xexep!u=oQ4k
zCBb|8{{!Wl(OxNOUXQH(rnt$PPWxD&yly?iv4ql20gwG#H?-F;Rb@<J-4daq(JDJf
zLnQo3oo5#F_we|tcdCWIFQ;(czTu_w{@~HI8pouzIy=~$^OE<QT{daoSN+;2O;cxd
zCtTckS0Ve(p6IB!>`fLuhUcdGU5RjL%G~e#;KoM_DN*G~k6Dr)-dr$Q_@@!qtGqQ5
zLN;&DtuweZ`DMGKfY`aS9V{X(LNybz6s{dqiL2Stcuj{VqrfS3FZ*TDmd|s7xL*o>
zNto_p!mS>9L~R4(g@&-!3qq1R7_T(k-6e9HyYBA3j0iV|n0-9qCSM=SGr3ix;<JSJ
zOaW&rL#6y1pOcbn|3^GtB2g50_2Y)vvn4l5Gy9%C;8>k2{CdN&xX$Z6BG=Z2#yy?L
zA$F*J$B!AYF(S?^7o8nEokZBA_Xzw~iC%8&qP^v?TMf774mal|o$r^<e!uqoEE!+Z
zhn-#)Q;VJ^%<{P5nQ(<)HX{7O-1rx_KSj=Oij>;R*i*BESHSOh5O2(G-l?x^CPsHe
z<R_f_S$MDiV0>srjOE`Mj!Q%%r#^5@m07%pPh3s#|4o_Gy<C%D`X&54cUJX*xZVPx
z7jw?V&SN>_^(^-0RNv4I36mt<dX|Rp%t#f}@@>&bbZIm3+-BpkX)9B@;>l?hT_R`P
z3KCkgZ!%8YBUbt2!TX=C`jS%`QvQqFl8unu)4kY2_~F;Z_NNvHa=3P^5UhGLHy|fU
zG$^V?gv+@nVB?n~MwUl()KqG}967S#cKj9Jg$*75Bo)5o&V16?zs~0%^Q~D^DlbUb
zwJ=3QiXC$23_Vfh`%rhH{|3!Kkz=gto}CL6qP0@5rOmzcy7Kh)Er-}ePqQp@2}!&>
z(<N!f8qPk=_&rk`J&ty2GKh+PyflmBq|&c@cM|T0-R#tCi%_zPSBje${31kSS!#Yo
zWM}9(ZL!3MtX{vCUey&l^2;^e@yz`{o==W@o^_cvYnO}53C$y~4NiI3Ogz5jn1fit
zij=m~Peb4HBzt%@o)dLRm2xrSa$v3!Fb)cG()%A*`M~p#g+QUpku7u7zR0v+ofx^<
zwVlz<!`UKWa>Fq$sUz-NnnZFKcI|Ds#T)1QM0tiw%bXI&SwF5Bl^&e^#$WS>(=46k
zyBrn;c|Nj#w_x#&;{xBfMe^qR<f&XO<%qbtFsLW+v4Ns<-;rjIWS3dD+^5ZQ%~Ll?
zd+sM0CmD5N#Z;arqL0P>Bqbh6)r%c@`g3{eE0<@!?D+?#I>wx6eK6zVmW7N*m$BcI
zmF>E$vBk;u$wOxmRmnY%`L;FwNm;6Qw@pAdFf6G3@{5}jQl>Ozss?mKI~Qa;{*n4C
zM9DSi(O$))s#0^J)LQ4d2p@D^-gZfO!lw2+Ki!nI<Xu0VubT8f>ZgF`B9>!03wXt^
zMED$x@72A0bg5`Uos5pz?1WiSi}v&dow=&SC^nPpaS1~|lhFgib(%&}N1pD@T(|K_
zxN2tUqKLg;j<)%ncRI3=-K+nzzy<bMk_8^e&9Ak@x_GosNzM#AQFmssPg_)&UQ*Gn
zqn??)ozobt#kxC_S`9w3Y%P3seeFqIn^bpB&QGO}?#@j5akX>Rw{8YkfzE&h7hF9Y
zojMj6Bx=|#&QWEXYa(776Z1i4<%*}B?FWKS{9#jMTwuHOjCj>T=A<KOS<mXD=CNoT
zcsf;F%x>xqg%?JgtpcV;&K*gMDZJ$ubTUhcox`am{c5xB7N=`7#cT^Rq&fb(I(f`!
zDCwP`<H8wx_U&Fzc`*qg+rS$S6x?Pq-N}6F!OIf0af<et?1iE&nlEn3UyFCEm>B*n
z<E!ADV~?IPhUV04$@=+MEq%_ZBPGvQ-Ev=C!>+~EnwZw-SIF8|*taq=G2xd=eBR5I
zh8#20uCdn!DDU-hxiQCi!AsAeXH#;Y8D=dL(pWs(EVqAy%Bj0kJ5R9h<nr3>bM4Cy
z*TM$Aw3%s3XXs>4xS5mHu_(*Eb;+@!X-%_du;1c5A-g4c*`!9<1D?(dYkpsu7IsT=
zKI?MFmMlA|HFIaW|KnIRYs=e@D;`g9d${=O%HRd7;w|!)SspFb3#(hQrh$v~WXOfN
zA_9_z|7A?dmgMJM{ycTHLgkE_wi6;P*Z&`tI#VL>Wa5)LuE$&aw9~X4nD4Di6LS&T
zlgTuPhvjXsTTR#c(#x|pv`lmon=EszWLl(4g_xXONV|t)^Vw_OW)p9kM%-1pp(}f0
z_J?E3%_JsvTv5^!FJe2nWLksZP8a*13&SjWf42#ST=Wh4>egSy*2^@V&+Yjmuk?S@
z*7^D9i3V|M$*H~UWtlol^|+r(^jFpA6R*Z!Yd>@7k*=86`CIO@)ZV`BQ=F5x;c2SN
zdAWOsTw^w!QeJM@voJ%UEr#ow5~s$^qfRdp=Y{#l=Qv%xrC4fng4dR%?2OkfG4I#_
z_vMosGgaO0mFe>v{!jlqVeX2si3XgP_kIbw!+9d~z}@|-?>+_PoM=)N`|SCXN8`r6
zBS}UrEB7p`Wo*wVc;fNqMAd>+vCR8o?aLoky?Ht<yke@jb*g~ZIgj2Qd3K&ITce)M
zex$ec-MV*7D;QK7m^2u-ZCetolA$BT8L7MC#nTzlUtPa;%-{L#*vz%ryZ2>l@8S41
z>0s!&X~Lb-Lapl?VwT<$n-w<A!P?6AquKqNHD`N+-g*2`*zu7!c+-vJQd7eZrMpc~
zeY!@q#^T*{iKQ2&K2MfP5aV7gW4$@@(8iy8r=F|3<L6><{Op?P#}-}xrShY9Ue=E{
zGoPt5Yw2E=Dq>%3{MN}u$l?h5dzJ-P{<G&hidEL!Y`*uoR(1uK{;K#8?-sijZP9iu
zO_!Tlch$WQ)QNJ6@6gzCchbD4^Ok8%o_jXz`iIAV0zT@lK3e6Jx7Q(E;2Yasg)_#2
zQ6BXd1M8lLuM+r>mA58KlCz05Omb?hq4BRTizCwH7v7YRZshR1C7j97U$-p9&1H4!
zyF4b2hCpqHAn_xC)-DVj4h%2uC>`$GF51b<EG@3kazLfm)G5Y(UEJqQjP5p9js~=U
zd8fVWPR0>0UQZ3r-}8FkXCA$N#?-){#bsLR1D!=bKX8c7bX;6;TlBh1zmu@o!OfM=
z7F}T7X7+f|W1bHcRWS)doxN9&Fn>NDcTBM3%bEcD|3{dD(%v1)tl3g*ZG9voTFv;=
zskA>jzTG-0nOB!y@VS0z0(X4!W=-~ao8r<1Ga4n1)ICfLu$et?rRVLyjZ40)?>`l5
zQ&gfqGehQfvdF>HANO$QeO&u8?1|X*mhTTu;)3OCn$}pB8m>7sD>!08#j`Jx{3)Fh
z3w>6d?`XXxY*!rn`{I+iOXB}Le=GDUP3cqc>+QidTkC(lYp`NH%dt+&Y^lIiMZ5S<
zeGiL2?{S~pb4jhc)ICMk*@KaN!a?tYEgyfLW!!jV+s)L-bNPD88?r53uFl~Wx_U-v
zq4&;(A-qPDC-QwW-ab3vZpgZK`gH;^{AuZbg^G8*<?o8P^@Qnz!Q=m5O0_0P{gnN;
zV$IFDQcE*r{yyYqDKBn}zLt2-%hRnTeR^@)y3fv2%7V>Hg;Wj%_V0W*X{V#jS7y0+
zQ$2NS&um_%cskeGy-mX+Wl1>azv8Q$TS_(rIVQ79o|(7aENs3>aFS2?QlD3|B%bGJ
zPR_f_@BTSg=JRC_VXGRArTzkz`sq;>+VfcUmL&CeOw#_mV?unw_Q%ibQXlIa@zt-8
zf4gz|^3BeFHf}Lnz@$(V>|8qQ^{#q*m-Ws^W~IM&xp2nioA6JSW0PE>Ho9J$%zWLx
zdF!q`7QTl)t<t^`tJceV9oaRdDNKJy^UJO6pR(?U-SEp$ozkE$l9{@&Fhp|-LsZuP
zHGkLfCe{lk{O0f9HFr{KUQ*UIaq~N0QZ<YJMr>M<6KTk?`uNh7rF;(y51v#=zQm@{
z&}8P;VHTu*U`BtXfJ`Y9i$n1Ds`c;ndS&J{E%`QA#9O_|O<mR9*X6~vPi0pnS$Nmm
z<bQTE)q3?^qh&$7LiOgi*O#h){Wo*5f%ziK8FP9zNzY)sz$~@oh}azMDoKTn#c5ac
zcQO`+eLt2fQGULn!<|K1eCMj^zZ6Zmsv4h_Ri@|m2uN`s=rgH|bb57HF+7REt+GA%
zPW@iV)Nfw5I%==$1!+Id&^}Pdyl=g;%Q|yYh94j6SavLw(>Uw1rt3Zj_e2?&0<N!D
zj;p?|+-3K_rKF(cn&l}=C*R%188=nm%z84f<Cv20)3{ib^obYNI-fotdhCr?t%ZLH
zL%}8&?aQ2VB;K4a@13>h;7-Ab2A4DXcSr9QoXW>Fr}jk!-y)M^e64}st|p#7_lVU;
zXzjLs=goK5f1fL~C)#71MDv`rX-A^lB5Nju#;1I{v`*9IjGnc0OGbXp(c&7%f*Jvd
zBY(w>GWC;ZFP3rOxS`OH;MmO0z3pMsgoO-7ES_7nPIxsOX%jc(T5w{Sf_w84x70g9
z1xwr*bDV@^Lc0^Rr?Gna&UkccYC|jA=_#5sF8fNFo1WaEd3D*+)$QwYUWz2F4)t4>
zx?>CDughvz=BMQ;{ka+Z-(jY??al(#jAy=@jHbmxdu|G6aWdu^8-?65RgT%k^8C@A
zoyOV6cGf>WcJ|hE#l#l#(pOK6jus>w?-%S-iJ7EwOpjIn@m}tk#p(N*JZ_2{IQcN+
zVwP+A*=mC=j)!7)Xv|#T(!|8cly_fkg+eD2;~rKO0f8k64Lc4^WcFZSNNiFGUY3)1
z=0>sR(ijg8g~T5Z4_{0@GwD-Y!Jmrr&js6jDmPAD+x4Dv+r*%YeyiW6M(sSAFS!5T
z=3k{u8$=jb3l?kZeGopdW|o&?vcoQ(mQ_4#-XA=dpV?*f!9IL}(1D(LRt_6@jEl4u
zv+$Vi<~iDCm9WC~OG3}bJ>sc0Cv;a>{J+t}EvvaYkl|F7&DrChPbl!3Zrx?QiAR|`
z>5`|SY;w^FMa#$>Gu9~|-fmQ3d9X?0NfWmM<0B5G&kDx9lXJOdie&{e@F@hJQ24CG
z+^cauC`nay3fHaA3SBxzD~%VOx!t1KtfS?+Ypqz-gcVEXY~njOOKNMXgV*^@t=;W)
zS<OAaY_9SLZC250RWm<iG-1POohcu5S=c8};NJZ}D(BRp0^6Vy$8t)0!p}<;>xOHH
zE`E2dt;uH1rbEj#Rzx<<JKNQh@Z9Fwt){BGYZMPGS+Y7hQ9R1n`*w@92jk;YwkLKp
zUt*oS;3;3yIt87D(Q6DBF-`T}D3o_fZ+F&7g~h#W{Qp0_Skfm@)^o1=%Io5uwkT_j
zC1nv+nu?lcH~M*kN|d^^f|e9@Fwf_+d2Ju$*0+IY<&9OH3gLCHHXLYrC-A^kIQ)&$
z5n+GsKn78Mt;NlAeyW~WcezhbBeN&tOJrm$_o9t2wo7a}ow>aHuccnp)H1_Yt9BpT
zcvF|{*Tzj7nX}FAr2f|Gj5@cfM`&Ryzm)&-)}?c%Dez1eSs=XhjFZNed|%~`lO2NT
z8XpcQxphp|-t2Q-LBIKRk<aPtq7FXIO($%M+n53zxL({)xxFk!KTCT<`>BqMMb|Z*
zG`=0+$?H^{aN<S_kAmbamPM1V2`>=qj1j6^DZeyBd4Frv#>2aUbpDsFTDd`mYlp)U
zmCyj4ZP!k5UNdSEYFJ?&b8ywQ>|Qak+P#<9!;EtTqjZ%m1e=PlCjLCs!IOM_{->MG
zD;fkdr!K2!6-l`35G-}c=85O(PKiu5;n~H$0Yyg(d1jnCD#Od=5GQkG&NCGS)|WS4
zE4V0j`E`iT-{P~D<=#oYIg?V<ndEY6w|euZ<U|NYUs){LIO&9#n!$o)G5HB0mRye8
ziWW?f?MPQUB)LD%L|gAoj-tWh0CB$~n?)rjD3};MxV>YBp!v(L{3vhkA41J@%?lOX
zt#3JazDVE3Y<1dL*)djP^-%%84S8*?*+NrnoOkp;2y6(R;P7ONi@+(>M4z`x|2qoL
zv`jd^a<zizgafmdZD@RP!cedCrE;ai$z8`xQ_40U_Q?3rGVyCq6VL0Niydz{blqr8
z5kBlJ;yLL$SGYn)2-mBwGdt85E(pld^A_$|)3xnrcTQmz`@soDo0}93%vff9XqHM_
z!|4&5>RrqJFzwH-2c8F(^fL!*sI#jXFtV<gJok^Xid)>vnRoY|H~v(eo-P(7clpL!
zg}YTJm}@<yHaoa83eQp5y>ChYKbw1;Xv_-7BPOhZe$Um)vpPz-KIwnrnP8(JoqEur
zbD~UKO5*E}cazUNo>Ud#qm+B2iNzsJ>By8u=AfP=-=9Yt^>UteUUBIV|IT(|r;YIJ
zYUWqx|F8RI+9GjT!RPHP)7jg$FXhy&E2_Mhn)m4FY?-sZcS2&LrtHu(%TshWns`!?
ze`N#9im7Y&Mrv=l6ER^^X<AOI^vN49(%o0MYR~&2y!hXylm1_2`)cO|bWN`NYWGp#
zhz`@&<VSVlH=|v1+=DW*S!E*yj7!foO?so?bVWth_nn48QBhBd&Mf`O5hi-dxgETP
z<w0!$$I54O_)153PVoJ*K}pnFPI8KY?o89KE6qN|8S`fN@4hj4hs4I`7E`xO|HkU8
zcj+!q(Z{A4ubh^-T3e-CEj{3OdqVfxY}5Qo?dN)Cf8Wo0G4YI}#%KFeUtH(2ER3#B
zzA_^(wR^E-<b#YI3jbL-*O-<_9!zky|9yeQKB)Lh7oY3Yc7guR_n}gzC%6SG;@dJR
zJPs!NG6|Z^Gu*I9OxyI{X%9t4medeyopqTqypN1C)7GwbiV`gTIQwyKpt<3L->Q<^
zjCB?JLnjM!tZ&Gitg5QsaI&XlQ}Kc|Ck_;xoYZr1VvOa=>l4p}8k}!3bc{Q>n`z%#
zkCM%^71~YDcF#yH6wEHJ7M~oZmhZQDO34|=?qr#ZCtbGmN#?a3<Pp9q)S|TXi$P7C
zxOi}xOZl8c=j^DT%J=djn5w+zmfrj>>3?Z6&&;3QWs-}|<dp@V-FNEQ3V+%CqTQ4A
zX3rCiJehTlGbds0)^%qau6Dg<`Trt{DMo90-n~)*Z@G4b*H;aDyll-`Zk6vo{_SFg
zn3GiMt0VI_&Usj&v`vjk-RQ=Lq!+t|?i^&TpKy?oV_}Kgi_=F9%Mu=JHgSBkBJgmH
zNxzEkjJ=$XqN2oHw*AQS>}0*bce~MzsXC^k!m0T~qvrbddB0Nq1mc#|D9M;eukQGi
zG*fozlojbm=dOyGrk1t#sB8MRAfHtWr%fxk$GhgiZ*l2F<7=DNah@zXCKNsO!5j{Y
z%IAkV8Jv|ATHF8pdCuB-NX%>@<DHOo*Lgl}-CX7(sdxQjWbsz3ZTDV3H90SSukZIK
z)4h?QRiA=2WsL9mKKiz=^#8@e+UVVm9M98coRR&nnE!u6@|i=e+J-`tUUWTiJ+?RD
z`3E-dv|58!=|?Mfs$|Y%yZG9jQB}i5{MWtBn^$E2e<E=AtEuXehAnyOz5@489d~pp
zbgizau3EO^xR~DWkB7W3b}gLX^fgWK@T=R-lSTgvW!zfz?DD3zdC%+8j(>S_J?YfW
z#*$OZuDyElwbC%cQeScJ)D(dYN}YNQKQrpNm)vj?Iwm0a-hH8{$~2CrMo#*gI&KVa
zO3h+j51kBX`|{VoY4gKBf6x2#6`8iQ`c89NeqdK_GslO821=aU6c{s|C#rDWO;ApV
zQfJ!jWRvjN_sxplAZ1gBWhMsxHyOAUtu82BQ-1hP&+b=;h}W5E|NrS8^;r0oYmuB5
zlZ}|3$ty=i4+V9mWuOaqD!SQY56R^;vfXVK)ljOo;5*)x@3BwwvIukUE~XPlULEb=
zncTy(cFLP)*H)Qtnebtci+P)_`7}Nr<pr#)2|PCrDw!nmxFzzKES#$3ARNWW$Lhd+
zt5MX1QT5mYxsD5B5eu&iao!L}Voq4BvSp1C+ah<flVJ@{BE<@~)v+AAqkm!9lYd<*
z`bm28cQtwDEn;eA;WRw7QKI`TSJPRa<bpYCy>z+{2r-;j;*2%xxWmE3(38x)CYfu;
zh8qSi557s|E^|Loa%7Hy+T^+vE4L1-DD?oZlZKyM+#+6HaN)X?v9kS+^Oyfzi$q(_
zEtYewNl+~>(MVWyNHJ<cl2VgsQ$g~b#iC`|ni_6O0?AL$D9kyKaqPp%St)AzhgM!)
zk{x#M$ea}4Jy%vPP*Et^mV5n7w*uQ4g?itK41uR)R(Zc!Ew_z<b4!EhnvG%_O|n^u
zswR%pwj_u=x*%q9&~i(H@SjFer)Hjg271e!|Gitee!^43swMl(crL$K|L=j`8=DW4
zIhq^oUaGuTQTKTKBu3e+K$n5{&}}ZKI}(eUcN`Gi=D?-nsUVxfy+NhLX$4pR87<Z~
zCwk7TIn@0urDLm6)8Pxri8bx_(u4w8{2n||R;wrs);c5?z}hC}#D4w7bdE)GGaf1C
z9F+Zk>p*$Q22q_p(Hjm*ZycSf-Z-sTt(e2QdPTP4tb)gToaZF)nN8|{w#wk7o;LUG
zQ;MRWjF+!5E^+2$IS^pxXyLI??$ZXb8;wd^94cN3{JE9L{VIU_M1$zA^<p-s_>ET=
zaX7WS%XoE!NA0PW(A!mWUUaNc+4%a|38sU2oM+eEJn&8_?~s+l#r!kLQxsk(oLMCN
zZ$<PypG^v@Ssp)nyM~3Kaq;FPL(TpoThW!JY75MZ^elF`_;M5-`0&~8>*niuy+6|y
zN2)DQGAno>aCVUxud<TA5}!z-Ur_n4wdE`cA~yXuZzKq72q{J!l5%YpUJ|(G!6H`H
z;_oLGl;$K>m?<9mFOb(B#pqIcd0JGT%YxM_9|`I&d1)22xcA3J`Rkm&PHA7~Dk{o4
zY}{HhaYu)lr=yam4|hb8NR2wzk_H*)AWv71KgSMCYRTg^Nx0GE?0w4BdrHIKoKI2z
z(iOQjn*3Oyc;eH%Nxu8jGI|&|Z^>mFx=u84Vz%0r8$0Q6bDZEjk4BBVuVNfLINh2&
zR`Q&ZSb37?tC;6!cbiwt7Z?BewAT1s(L1Sj<rhmtuNlQhIjWg0<GItQx$W@21;^z?
zl)4KFobntE?2n2Cq&m%6{v~Z0Z%UTF%;m5<Dux$NNH0HXw0whhHs8d5CESupiU%(!
zZWS@#H^D=F`J_vmCoxPk5#Orte}U+YJmnV#Q$;HR817AY7^K6=G{t^mC__dv_w}rp
z3tZ+qj3zI7d@3$z{ehmnD$0z)AC%fItG+%Gm%8MN+@jSMg-?1~cJ6xm@6Rz4kHgm2
z`%DDNM6Rx#|0m<jx|DO5j#}_}i3w*}3MKSU=rLd9W8tKs_adkGoU27mv$v}5XO`!N
zPRWj(XH)pqE|t74i&$Kh(aCdbvDE6o7J(&cy1X?p1unN)^z!_rw?6Nw*`{@5mE7(}
za+;s5?j%K>TPd%4;Ac+no_E6LXS~g&J~e5beAyCmO3Ep+eM?_NqVO??1CtV-Z#gij
z>}=L&V++1XW(I5))qx2fj`pvT_1B&etYwM+pWX8M)sc^zyp11-{ID_f;Z)Xmc-)uW
zBjMwt^B*4VcT#@8;?e>k&bTSe$}^NX%hc^|dL`XgWn}tdE_O0jN-0)DeFvK^%lT$r
zj^djkFB2VjSnN+K-7pF;Iq8_w81gl6aSKz|k`;vul@hisZvEHFlgDzRsjWB1>DdXl
z*sYhieqKIlGVRn3eXdRZ42uptnszn&aZ{Az<rslQ=3IxWcQ1Ld<w_MNQ!VGB7mE_I
zw~E$YJ~heXviULB-|LJfpWw1~E7ACLxa-XU(SPO*j6YIiR^|L!>&_v#Pogj`|4^=1
z)9E)adYJ@YH>_7IVyUxu`1;bj^#SYdX+`WZc;&&vt-MG0|Nlk4iTOdB-sovxnDnmZ
zWlG|X`P&#6!UfMidaBT{;K4V)WaY3J{ifTuCM&i$bJ+-eyz%&6nAu0K0!PgzX?c^f
zpSez!0p(fKq@tCqSk%6iy?01isyk_IcG>gYC10+6x$ORNv)$v%wc4F&m%o|sT*7*G
zTK0O@zXD8JuXiYH(>HjQ{QFk;??+~g&&{SyaSxfruE}s?!vE_jc`H@_UFWTE$v+(8
zb}Xb%BLDvu%}H<d-)u_SFSOm{1@GAxOS~=w`{-RSGEVxNk-WBvAs}~0QZuhnOP^?x
zhVI3~ZI`wr=5Z`Jb67di;7ddjb4y<DkEa#g%WP~`E)!F7FIsYfM>Bcz6aW9`y(5;M
zSJ=KFhntBnvax^Zg8sz{yoN=y-5W)tf`v{jkXuk>a&<=a)teF9r=9BZi+(us#hUQi
z`{DC?`Kmwe?B8h4wa9_n<ly|P0lb|Ter1H&*Eux^RyDkRpun?aVY-{M#G#;PUptx4
z#y;hUdz^2n@3fEUaM;D>!oNOVn~n-ehUIY_w&tFe_?GYgj24mP8wz1kMq<uuvu?EA
zJ!-+=V70^Gl--KroEhhinRnD)w%q0KDBPPmd55tbmt<jN(l3=XStlpiCK>KSNm-3K
z+bUL0`Mf*Bch89(d-lyzxOz(Au)&JfwW4npianbt+7Q5HwdqfkBe&Hi*U(^wSB}%Z
z6o~w<apbzUbMD_+efg|6ZcXpsx?sMm2Jd0ZIi5VquMdj9Iw&vqd*k1O@{fLf<N2vD
z%bh{&7T=u(b}1?I+CJyUe=@YFs{J!f)%(%if=R|);*<4fUa^YMmwK%7{iR=nhp4yZ
zog2YiOB~kR&k}7H-EehAk&G8pnfEl=+*+HQsWMwtOE#1hz7%NG&^8U`p4rpByga$t
z`Gu3vypy*S4wk*VHeKY;LB(G)lb!`sT)R*a@rc)Wl4!yP)9V-H_VFur>Ghu%`+aG8
z?c1BX|IV)MeP{ag*8HOhyrQ$k3?|8Wtr3^oEnl@zUhTHH!kvS^A1eG>C}!d~(X=M*
z5z`{xLx;JO7RGjQ|DWNq&Fo>{tfS>-nkAcaRD9+Ls77q@<uz5>m>c%{-><D#_El^R
z-IZ^pXuRaeTPd>z0b%_vg3CmgpO9x~V0Q5Nr&n`X>p9oeldpZU4V4(z2`cy8D${wt
zf>(`&H?1tqQqRmy$$?2VuUN{T=gbTp`@3@#E_bgvxHWsbCBLU4=aX4Hag%qnR^)}P
z61`%eY#5`wg@Mz}kw04a&!=1BZi*^5ChYtDOW{q#?+-gJ884d8Tg@#Xt=e%xN%zu5
zMe8fd)>jOf*b+A=h7@y0=}(M3|23{{A@`!IoQGK1&xtK-6<7VZ?@?!d^rSu}-rOMR
zDFzpPkDl9l{;{W>gm1F=6kp}0{}~qBZ*FcrR_d)ft;AJ)p$+pgrdGccBVWlT<&f=)
z8ndP;iv-VfV&z=q<=nCTZ>7pf1+E&sd9NjFmn!g@CeF4Fn)5eNWDA3`VUbAPx!64^
zDf~)IS(<ZN+eJTpNHi+qHZVFYXsf(fVNUetdp~XWD=(;hDS6=X^%pn(&KHbc5IuRL
z_@0A*6A$_Sd-8kXA>KuY!s8fX<esJ4J&TNc#@Y1LT`^fGa<j9Io7gP{v8a3|v4ZDY
zjox$!PX9b{bI;^WN|BGcjXP`&9BviPSbO8@BBq~T4II`g&NyPdx6W^8ae_YAevb*l
z4L@UM-6`hxs&k(<v8mZ2Dsn}u)#fD$uGiD2{Lkt5WOqo?_tnZ+^{Hn*D(DzF-QQ(f
zQJ|Vu%=ht*<I^hFi-qe>PP`v@VcVAguB?T8o{o1nhuyfeS8h$e>ZxsFW)IhP1_=L%
zWH5QC$vt;Iw>8(K17E-XG~dd9`ROcP)~JoPdpEk@ed7D?nen1$(RRvyu@_&(EoAqd
z_}<|f>w+|nrk~U01MeK3c((bvsnxd5Vvk>0lP0tJ{9dFlcK*iNrG_lpvpKwLGJfyB
zb>x?tho#5QsS~$qFkUr(p8DB(@e1+PTWt;}EZY3)vNJEEQ^SEq7S>IgH<nFpDB}|4
zQV~w=W(fOs%w^)Cb7H%rz8=eXq>&)(9`-dWQ|V~ml7{{_N-it^AN4!6qH}AMr$>;1
z`auoBB@=FnI8Hvw5w+#Ro{EqJ7L~vXz8(b!Z#CUnTRNnSv#x4|Nqbkly?TDWeSG5Y
ziK5<N3HptHuW@BqZ<x<eZ}s}r+V1UoyLp$qc_RA2JpI9*wYPQ@DIW;a)SGkS$z744
zB*&BWtOYk+A`gf(N~>{C*pa&V>G`Rzy`DME>~^?X5jkD$#I3EbjhJ`yl>D8wtsy8i
zq*&@~bb|VArZ#T<AJGrq2Qf_&ufO|8|MQ&7-jjtKx=v4-&B>&~A*8Tegw<lQia<e~
z!jC;%MOET6;u3y0M$5cPF%__{Yx>3gPQGG-!+*|-`3xyNpDxE6D{44Mv*~&KXVcuq
z6VlDV*E@+pNHFJ7znb^NCxN<adIHaDcuerP+_C105tn=Nge0NPwH_C_>eg<V#1g{v
zfaSi!M^1%S2l1)mtU^o%tWO-=M4w*NSiIrWldd;cg8TKq{wRJrH8It5x<jl`2!Bb^
zy-=x{OTRvvtX;=zHbI+dhvorYraKqAq*J#9w#{Yw+BQRXZNY|_vzWd-lgOOFIVCbq
zXy>$9Gptl!Wh}7_WRr3{!mE|JY*yZ>m5nzlcSV&j#V(g##$WsDM*b>aLsurQis!Nn
zOedsTL^$t!S;g@FZ-i9;(w`~w{aTy6a#9)7b}7Uei<+!qSiI$8-+Vqxm$pi#p1{Dq
z#W$4xyRv9^wJcQIw&k;ur2N&xUe;=v2fcdN_9!)q-N;SM7W({B<)FjQT_P%>A&IS`
z;W33r0`<Q(JoS&{)W73@xMZj5fv6(iKVe66UVc$^loU@{c2uss#_(uLmrj6fbWY;&
z#L|h5OyVj}G#NO9a}KbwN9Q<5OA2!|%#e=pad1mmGU1_&<BW^#(^DpuwrSLQO*Wg5
z;p^->%Vp8ZYctCN%f+NFFYaxNUcOXR_Eqk*LP4{Q)88F4bG-V{HAbEJq^|kR(+@*s
zZzer1y_+ShaWS_1;mPGJmDbC|TNpaIO)RRIJU+zrPkq|0q4|w>$M(gw{Z3rXA;MeU
zI82kS$Y>FL7V@84(>wPEf545ai(Q3GuerBPoV{arr-);pQQJFD55p(Ef_Jj?*Dsx@
zs5&)5Z~fCxiLrCSXQ=Y4*v4HguiCpr;jyV(t*}sudvkO_gT~LY0>{nL?YU*kS!8wy
z)kz9Xx!9HDV$UME#M9q5enS1r`8$0B7n)Wl`K~P5w2eWlppp6K*$qLh)7izISP7^&
z=O5U{%DP9ZVAHOtt64T2JjUq~rce^&R}(75nRCc-zLdvit|A>n&kG8IXFOl96dmRi
zTT;i<xUh24hSsu>Z0D2-O<avG>seGBncPes@$cO1^5q71E7L@Whb&E#qZ$3;nGBV*
zF9eBg*mG33KytC4LYnyQ|2h2%MU4_ieofrF!@xj&p8$jOmIR5pf7Xa-I{R?OTx`>v
z;NZLV#sLu_qejgY`2jDBT_t~oO+VeTBy+;8Gbh!KZkarn$0(R#!4;08S+{G9vSxMl
z-el5{7N2)1BwpG>vhH#x^D~#c%7;gm3hj`-dqOw2LRhv$AyU$#sEPY=<J9ss85W;i
z{1jf_6nxyX-AA66rB#sYnuE-j`Tt|y>EC|R@7L72z`1sv!W`+lrF=pQ3tzT~2<^C<
zzG%Yw*>_lMKlSum-}V$Z#cIH%#ToqV(F;R^!mQqz${9;G9p}=W^01nD=h68p8%`WK
zu%1Vn;nt6`Wy@wyb@a=daER@d`IciE>HjCCWna^<7ha<IAt*5PGf&Pl3C~L_(w15-
z)bz=prTN?<U}|c1YN|}J?EM4F3N{?NT6xHL$LAx<ZNDpVhz48iN<JDe)mY^k_tBCG
zEBRDPEYB>IO_b1OobzthwWL*hH=MX~d}XPE*M##?J5OBO|D&tRLee$r)QZUyU3C^z
zwx0XOV>SI!$x68`5y!P&Y}`2EWZ*{DkM6o6zif|i97z{`@YtcsjCH|OMgOcKR!Npf
z^{%{wylXoy#2<;A<!6^0G=Jh5PkGjr0aa(S7b?w9<0>+pZ4|J5hSvk_Ol9vE7kVxB
zYvwU8PL6qg;nR+yYH2^tNpm$d4urO5uQuAhSo@e@O5wZzE6?nmaf2(NjH`IoqdDOx
zU!Go}r#-hcu~mej!M|z8=C1loE#{L2OMOH6H{`^;k@`AIxPhxgDe@wV_K)AwUzMsD
zHWYgNdG?&kl64c$x(5v`JjLvekuNsS-x%qXlX&phAGWO%7hO-6cxr$4gP4`}v@2b%
z#Rro<hD5AebGA6YM`rb-)a8|p57Me<MXGu@Fixlrt$q_Z;pdX)D_IjirE@*jSZnoa
z)q2lq6~#*Tenn=qf4#Jv;npt;gX^tDdmc?>y7lW|=By1}Pfwm~t9i(8^3V0$h0of@
ze`rlVB9tP#c!9g)i$Lz37n()u>?CZDZ0KsaxO08!%EkI_9|I5m(M(HGIQ;)?R3f|4
zgERV*_Uhz&Ewr=V@@V3sPpX|0%>)Y5!nmRZ($icf@1IkVn?A=;BsW@e)?Cjg*S3px
zTsbt&?6%mFqxCCa<**cJ8c76LPEYbxzo0fjZ^bc-^NGH9N`kfo?ftZ4Rg$#I>f%&i
z$4^#Iww+<=@d){(;Ah)hT;$hdE}6Z_!2Zshj=isq@|$iPpT$43)5ULZXga5hVt`Y6
z(hu8-x(CFMPtxX?#4e;Fl&E>Id#`cyyG02zI5gU2Uss$Ady-zZ#z{E(r;xUt5<^r*
zlbDEbE5oPL8GBw_m3z09bq<HpE3+Lr0l6nONZ*>UsnOzy{`Q9X+WXwQ+|@W9EEn-z
zI_dwD{gWT69o2H3)mh8?aUPSv@#uI}$A<2e4~+I)(Z4I=tN3G9!RwG?3xh-rZ)c_m
z3aPA}%M`Y6f7Pm~j2*{J&TvjWdW>srR%Micz5l*hT+fWnCH0fm^-SjdKBfOf<TFuq
z5B+UP%j%Xak^3vrDj^WPDe8iAQnTO5uxCO2p*ANyr6buF9h~&MZ%c-(ko#W^7sX>;
zj6afVL_72&K3+Vyp)<Yx<(dzli$1gQ=7#K3nWkP`^iDIU$<#ku_u9|rUkc+kUElt$
z=epR_%h%5tS|<5F%}To!ocY~kM~Swd<TUF`kIxk4+)JBXT<DYc_D^o}ou_$~O5f$A
zzxXIzvB+donbYvUfMt{UlcT<Kn6-SS3v*p$^G|YLoFFgS)Hq*|A?}!_%ntLY1ALRi
zCFR`tx3jw!gc!YHa9^XqC3BHKYNDvF5`*$``9%)gFCG<rap1O$;AdkimVU0X*r3qj
zO1hFqrRBqjpb753)k-YbC4OyFV|G+$dMIA^Na3j)*F!adb!{4&*Ia`au&5_z#y)SY
zf7rU`iU!XVjh;)vDFzOELz9!$+v~O`uW=Cw{Th6+P<G~1tqqfezcb~3TAKFAk!|gS
z&`C<EA=5gNE`|QMX08)r|Mh}^+hn=k>nYz%JCfU6rwF@di*>|b6W(f2x%+y2fl5{G
z6O}gCMy3PlUoP;kTi*1>;6K}I24Uks?p4pbRE_wyYp^{&!1eloXtp8SY6Z9Qk6lR{
ziYksaHBPhqdO&>6H}Sb{0+&_@*FEG~$`EpfJ$7<QR;Lkb(1+eImD*JwO9B>W1s%=~
z*j&qGkj=70*!Zx0-z2j=OUzTlokP;<nv2X?F4moSCiUdGOy=g?gBQ)dF7LG}nK0KV
zkN=}_!(-XAPb?;EPiy+o9<zmYv9Ko3iizK!$zQ+H^4n0DLEZ1<C53Ng0%nsMQVimY
zIb`2VGO#*W`6YmV-HPVb4&1HBlvXXTVoFM|n$fIvyvU(bPi4F4B8Lozn+A&tBD4!5
zX1S{}DR4z58svOY>2OtJ|J=NH`BV>&|57L2TQ3OOu^1|b-Vk^=wPAvz*`pE-wP*H=
z8ij)zq^4eJ-7!^#<)dd}aNi5|;7kiYR>m@&&gB0uO4qiuTZ+|xF%OPe;K(>Zn|p~9
zpHgZ$hwR}^9qT`qKWBA5tzIH|qh)qNg=&bvw2V%_uc`HlDfv%*m@bC(UKfs^Vvshe
z%70On*GwV3aG}7Hm2VW-UKjL)?c~zcVAy?=zx~+kU7p=7PrDADoNaug>4i~q<ie)E
z0g5UUBT^j=;|^7qUW>mi8IiJpRWMZi`1h$z3xzLDYF%)+H~MO6tjMfcA4^hy)Z9Oz
zRl(Rl*D?Fejp@u2`&L{_G@oj=?IP<O1@G^{0@DxuuRCuTl&voF-rT%$q4XKXh2O#_
zM4c!&Y$C@gVDT_f-%FwVQiy<Tm2gTy{VkRD8J8{V!^O8>RxmG7W-<)pXJnYqG2`43
zWut}Ze;07ps)>7bs_xemPcjg<{#E4kQl~bI-_29i!>V}|yQ+DFlFUQ?);86{D{Tyx
zs~S`$FIm7EwN34hS<NIf#l%wyXW3Psxa2+CGVQ0QK+uP2oJyua!EJA53YVXpzjxVm
z?kBxgAKMI{EigVcN&k!VrYT+xQO>T5>w8Yl=ranpRAijCxh5#B!01SMj=RppoxK$o
z*&@@pzZrBc`aE-DQ)fx5_`5IxJwvu+!Gz?7>c1PB*`MjC1pVjUrpfTdfK71m?8Oe;
zZw$CzCm5_gz{e)s(==^$V_4788wTRvCy8#(IDJYp?}@7MCAMu2y`_oV+Z3{{zEa;7
zz-<~R{7$H14a2&JUCz;^^K@IBw{mIzd)%78llA1sUKLe_$qJ?m6$PF@%U(3mbIL~J
z@0`-+mo;DS^fFs&zDsezi7@G}6772eV=g?4*_XJ0$#KF7v)uS{^Rubz4wb|dbNHOv
zR(~pF7T<DV*(g^NCZ8XTzF($^-~8e$8!4$UGmGny&S$fH^FsmU6>>|o+I@~GR;h~n
zHAQUEY7s8fvtBWI@dbnF?%ZZ8n|?kuaI`8s8!Z0M#g=tryr_FuJ!8bx%m2Be6cme%
z1bDvMel_kicjB5U6_<8fPx}(rRsmgA$AsCRHD_P2eAbmUDPa1egUV~g`YQ#O2Oe%K
zk1pk5ud9ANJu$t#eWvD-Hm|Cd+)oxOEgj7^eJooyiRF>H+SiFI+dTw=RFnHYI%(WW
z4*e#@cS*;5kskB&mFZJwbuFKD&Sch46M?q}JHw{ve|*?z#V9INzJ_ICRpT+GE0!4s
z%bVBzu*v^8>-^;{B^#?%g45#<ZDv+XKO<eN93il60VnstwdO_24^OczVOW<Vu#Wk&
zDEnfAL`AXl!g04|wC>r=8KGG4JW9N*dV98(pW~ve2>10t3G3$@u8-Kho@s+|%wx$F
zivM#WKYK=AHZe7><NY$dAld7{RGC$R3tokbXFuP-;#eo3z3_Ya#=2cb(I3qJtkV8a
zrvKT{Rk5*vvt!fJg@;NL<=Gt_qkl-LomR*xu>KRKXrZ)trjzTMV_7p7v4$-YHafW_
z@{uZ+S&PW?gIp@Mdy}V4+bn-GiaYEP|F;J=;WA6Yylt0n)>$*5wcTmq)>GU4mRe7m
z5|ma^yKkAP#@BU=N|JwC<m7t<Cv88XcWlRm+dK4^O6ec*me}1pNqR?-X_{T5{E`Jb
zOKqHIygX!iQE$$!)RnIrEoO@wD+_)rGIHC(x<l!tm!N29Qcup8S&=nvr?S#HrfGH`
zb0{(7ifWk3S^QrtIdET=<0;F7YxLRIRW9#cc9hS-HR*cwUZ#(yQ)e&;ezcsR$obEh
zb5f$qmx;BrHX8q}&R%iQWLk7^+T^xF(~hTz@3ITcwN2IB@G$zgVeYGR^Xg49%qPuD
z=ZJH^l(Dw)%`BaG?MTDDlJd_!`Rs~~+C94-u=ogcIDZcF{kkw!@Q3rv!^$ZNDO?XE
z-fp>&)FiC0D9*4nAw;^k-ZUWi^jyoti`AF;-yN+iQrUWW@!UQqah^ii)06hKGpW8e
z+RmpTAU9Fv6StRxlkbA0?X}h?OMmv>I(RwedeD@T<0r3qb*(;EFlVE&n3-d$S@|oe
zkTAzpS8|g!vp%RkQL>_A>Z<?ATvK$qVkVuvbmEtq!l#tj4cV?aZt>~5X%-VtdWW1`
zlGu<JdTq6(Z`1|8Q>Uv}@W{r0QuT~hyxu7;UA#DI%|(gktwkkszbs%4dwIQdqoK;<
zLS{$tb0_7W7;Jx{U=`q$x96}m%fr(P6fc|j?l6xP*u|_e_lxK3jr$jF%&u=ddq*c{
z1)~|$#al_A=L<>avhI@F)@ZgXHo3lYL*eHIsR9d+PmG!GwD9oSa~D;7dv~Ti4fGX>
zlKOUM(?#9GFD%zB5Y}m{IjpzK`Pw1}uK<^yi==*Dy0CqBg%gudv5M{CNTb<*7rk1e
z`E>%TmSLIdMRT*F{0Sdbo}LQ5-g~+DWY(k$|8@SK-CyigYWwyqBg=+>Vu#aisY{!@
z>u%-nxJa)oS)LmqRlO&7f%mM9tMyY?*Yq{Y%h<J_-E{T+Y~h)|oYo##sjF3z6f0$2
zvsvha@w11*vzDCPze!o5FnpSLMCO;GCDT`%XPjR3LS^%>+3jBq+%+^S_G+H1I^s|j
zemnRyE929^Z25QA>-cWSU;n6d=DKC{c2*<Z{X&Yzm=d?U7hgNMM{@PGW2=3qPZRW7
zVxqaNb-L+Msm;2Y3$k;!A7tGlo4cVT=J6VtHHk4SCuL5nzTnvH_n}w(lx><<z`Cgu
z4@Ilx>#P?3+-7OArgaAAh3h64VwSm7tlGtp<D&g?q1DF=>!ki~J^b>O;AXx{$q{wS
zL|u1s%@Lofbt1iU+3HTQi><P@O}d&lnIo*NxZLg-GKCd}?!M98+Bm^PnA>v_^C8Qx
z+6?n;FE3cg$~kGi<EMEM+a)>PEpP5V&3k6jk!1@Go6WeoraU)&LrKT!!$Dg9bKlLp
zv|?s=`Q)e>lO*38Uwc=2@G6VrF6$@TCSU4Y%N=gPnD)8l!y0Y*xg87BODj11EtY1+
z3M#amloJ24VDf5%?!%l*N<PkZ)C}8lYmOh2*v6`ajahrjpA{}!Q>~X@7#&%;MCa42
z3*0Mi#S49Y+-sWNFil*O(Q3xs7ZXaJrSq+yu#hu)!x!KAl8X;~@m<V%`c77~!GBJN
zqP!`KgKn$W-D?-GT35Tn@ygYh+fxIzRyj)ZSKqEalygcj=0xm!p;ZU3JbWQyd8mJ~
z+@@&xD2=-^o-gMt5-^Jq__wF`{Utrq+j_AkuloF40~_K!Z3OmfR_XYw+P&akCEnDW
zHhInUO0RiUqQ;+2#eHA8kt<1{sd8hr5obnh;_1}|;xhVNyH^XpNm%x6nfyKLw?A#y
zJu~vmT&HqfPE^itJCmaRe*g9H716f}|9ty#j7LwB`ImT!&99_Q#{<@Cs=Rw2vE<cE
z4}k}<5(@XqXFirF7uUQY-?(POqr^k`(HmT*b!=|TxYxJHUvPr*><OQ1mQML;D$Hiw
zoxmut#pVCSRok-?90S+<+9-8n{ry#;U;kWej#xdPp@&H??a7Rd;qDU@UHeoxKPexN
znkACO6WM=FE9?yi*9vntkt2cThJ8}b@00ciCcXIix;&^sf0B*yB%uRccV_gY%qo&;
z70hz?mE4?k$E5AK#)dFfhpS2sO%1xUPH{~r3Ole&w~#eQ;-u>VmMt>dTkb>$HQe3b
zU|jTxJ4ox8k*x6aEvHv{Cm!h)FOG^?Rpfq1WoFVHo|~W3FQ}^aWQpD?nccHy*`}FM
zyG)K<mkgY|YFDSv{Ry*g=BORslBFFjnf=mt!jUf@FC4q*wdVT91<R_w{_2!c6?~(`
zXc=xBW7>69V}>y2ugezyOOow7Wz3b>cg{;L{;q3mcHCorYVE8jw~73oh6@>*GiK#Y
zSr`*^(jiK3RlBIKj#Q@dr;x)QhgPbr$y)Ws<0gypO1FzoR&CmLDXYgfgZZ_2=dGKb
z-J1#yXK<-*o6*!Qxtzh%;G4~6?&IghPJeWNV)R>EMbWY9%!vsslUErpTcYIS$=LP8
z!*Y_ygr=UbTQAlqUvqdB)NuGfkCw{HLq}eC_gxTKsb%Jwbk>J4D)acXvbgHpZk0P;
zPK<q1I8ORMRaqdU^7zjg4@T7=Z!-LQ*hCdNIkv25wf|?Z$h~LPH9=>Q?zlsI%IvHw
zmd=ZlF`BB;(-HN4c~98h_n}{{l0+B(|8eVcPY~mY&*|4=SWjeK{~_RYL4NuT$MXv8
z`nvu-Z%vZid6MJjPQBK5BXF(O{Y}!XD_0+D%69JCBvih3<Gx)JWV<_*UM-%!<&X#0
z<_le?-K32doz~}bI%TeWXWN4&uAUtaChk>IuYPl)kHziNgC3z`31!8lM^;Sf(RkNm
z(8HS0b#iILml?{OLJTi;CW_x}nsVdD;t9%J(R`hU&LxNyFX1-UD)ceSVP3cRQqIjq
zPx_nd#C6NGSYIxUs?f3ufB8#M>}rgn$jY#VtNF_87;mj?ooZ;(7-jpat!SEBeN&S0
z^1#<8wf0U?`sCyIJK~x}4Db3T?ZtUprBa1A?34bQ`9CaMwKaw@XUar3@zrfT?hWlv
zIQ)((|1W&dQzj#nc>1A=kJfIB^)=5l6(=W5zO*kz*X+s_6;8h^3e7PMW-2yo(pI@-
zEf%$iUU_`ll+9|+madLWyB0qXmA#_-?LlpMP=vqMD)~mCoT+=h_84iIta4s0vNC4p
z_v7NR2hX37i+{86XwB#Bm8z%1Zteg3_nY}erM+Kk1h>tPy0+0%k<EQ#=e(fUBMO|a
zj&I%;JSD8MCG4EHsQWcT#R*q<V)7L)MwxwCxPAjugP*g4+q&t3OAIvhgJw9yJPWnx
zd(f(E6O`!wZmPw^FRqDo6K_1uQxQ6^_{Om5Vbc_=z_y;7tw{pilm0X4Fi-LmDfE_&
zR1)-@<f7uGr=lu+WpP#UoO@f(8Ec>J$bNK0wfE{9r!6NvX0s$%^}XRy*><Dl!~}KM
z%Np+7YYb+2y!vP!z1Ka8^N7FvcZt<2TU)Zet!)3Bv(PRqXRYX(4AxTt$2_iXNvhqm
zSf#*AxHX{kh{Eaz{MWx7=3Bp_yP4^Of7!0ddRb2|T7K7jP@oXuruS}%cFp72<q;RT
zJ9~nr{>W?>TXI3X;%Y`KXS1@QF8hq@w#VF$>Ui>Id+e0r``qTTB<4T|%dS+RSH?mL
zt<$zF+10m)uRTSob5@eM`Th%hDl1Ojo%3qSru)493<pnlEba+jQDv8<av)nIMd`l+
ztCiE*zNx=%HUGG=+AGyx<vzE(^TFeL7bmdPZo4t}+O6YuKlb`F<Q1^GzfI`wZ@VI@
z__RZt<pI~dvm5&stm3lvI4RJ3#YrS3nR9iPBY#xRgsP@Bnrm(~pN`rgqINT|*L!x7
z+oDU4^Os#Zv1P^5)r)@|7MrEf$H0?%XzzvBam9s+Zr_fE_T@~CpWm!FL1+4szBdJJ
zeZmfAwNuXa1?Y4-%lHU#CL1a@v2~ph@jT&hX!#`1#ECUuz0WuWSuMGyr<}83ZirX&
z(g$lMig0=~#B)kZX})n^IGuG#jmuW+5>=r|y)i18iN>DZL6=0g3pD8ZWCisc_{JwW
zbFC~(!0rFn+!9p_7mJnca^ChrBx9jW%**3{8udyXCy1WTJ$1k6r|G?SK@8u1#Ijf@
zu^us8He<&|@Ay*z*A9hD-rTZO=Fl2W)jpTYff9_js{F5ctgPWI?f%MRag}BH??Y{K
ztu+_N-#E}!_UYWzz?+jrUTFF|T{KqC7m0HdO4u`T3v2!XtvqJ$Pqy6U`k{|b8?-Ht
zw(ANiseN}iL(`;7%A7-Ci300OkLWwM`VN>WZ`j)0V?8OUZ$}yT>>F3vBb~H@UT#_-
zv+>^XIqZRfkBVY<erdg2ro|xteU(b9j4R70lk?)a30y|Evbck1h-Y`D#7F<gie);s
zIAc@0d$wwDT0@4*XTSeE>-<uZj#~t8?0FNalDW*-{YBK)_;2U*H`?SRhAehHF+p(d
z)`w>#-s`AcozN&cQ8d&o$FXa|PS%5u&#F%~@t)kwCZuw)HniBMcgkALNfVbOv?_bQ
zp5?Xb+SYY*3wsTdZbzrHT;DFrke$vwDfaoTZ3hpQ#$1~0@LS+|=!D2^{(mL3gKlR<
zoqeZV_3NSV(o-#>TW*votrcZ`y-U^YkcwLRPVTfx8k^U?u9Ha8`+es8*T~%}joVF5
z%TBVdf94f=sW<Zo=PaYfyoqaW@>XBDVrP|V$f@c*WueF%L1)op{(|9$B2@%FIw#sx
zT$63K-FNN6yGEN;0#E1tEK~RF`DDHLzx3S~9<h5KNwNRls?PeYBG-{k)8PDqkA+vO
z7R)-;XA^l!QghvEtv^~ZOra|+j;`E#O>pvz3yUr-_Nuaf)2{9s`e(ZOoTiA2yh8e~
zy&2-4GL-LBaQ-N?hFj1(V$NOOBW26Kc9t!z=AC_G;@(%1=S5DJ_j!FQ=wWG4K3e(y
z?n93kj`IUmq=gpO96fT*^K*~!%zGxw-50egmRH!V*!MUi@@<yeG0m+jx3*?w<|Rk3
zp0RCJf2Bv$+o~JQ&rG*V$jCN}z1?1W?ADhR><rGog&ve!uUPV?)qlp~b<dwodXe9G
zXo1>?)4IoJ>u3Cx;Fk|yzxnsmwY=OX=jfilc5-UU?;x-L)lXG7v|gLoWE%A*w&vw;
zucL?Wdw&x5tji3mXDDtdDW0U_e(NHero#qbX^{h`)lXCyd<$x5y*;^g@fsUVmFgb`
zlNV3!d8NjEWrfJ+6=K4glbrY_aB<4)-gsey;rR`%0xS-<Z?8#OHFePhzQ9PUs2}qs
zdJe8Qtf|3l)UF{K`cOf(#7u*kvp#{fpj*6WLWAz>UE-YD=MQuny<WBFh4>Q>>uTYi
zx5eh#M>^yeEz?-pV{W}jDXZtO!5*ba<`WiZE1ujvN#L+Wl%o5u9*<8wmmVy-*U8KE
zagi~nGT+Pvdk;)o`l9!E7gx{%6SY$&-+s;bdZ^>k%;gLL4X!*5EEz6U|GS$0E9r(i
zOWx?%dNFuy>}{9W*DjMD9J&73#p|}I*Kbpa!yMNRaX4<-;>UB;<W|$)ifxku9PfNq
ze#xM4Z*l7arhTOkwk?qAd*s}|yizaKd1|JjY08mVRTrAHn=K?QWwuOmIvKh7YNYwO
zN1G2FI3}-XZ=B4Z_shA@VM|d`zwS?=@6N(syBEBPI6lc>`CP#*413g>TP#?wsB1CK
zvn_Nub<4f(&T-d3hX$W1HY;~@DYuGr@0?z286ACeCNrndAG<?Zdv|=WX#a39Vxbx9
zu8t+Um6y$SocY6J<zmiN)$?R^PImdMm>1-@Alh@90)PC1RSq-NHE%TNF5q6zY<*c_
z_ksT>?8GM?JO5yzfhSLCwZsl#hSJweDYL96Y}nnG$kmm&+vtq($`BcM6}d?}giKX9
zr!Ta>@nX@HhQsS)+)KICPbPQS#~huf**ZVbV4{bMUFGRjDJ~Nek0`k~oKilbl;h3l
z(O@)T<HRi2xWgQGeoego`^XLB-Vh$CI}@3HKh&Hwp)Y#Rw%9w{LO;0XO84E{yzO!E
ztmj>`LX);dp0VvrYE65yJpR|xg+IHi6ggj?o>+XjTdwiM3B?`@>(#qv8Ef^peaJRo
ze>^>}Nw%*;^7~=^U%$+`7}ce{CVzjd|19PBVJG2B5B4)2=3tm}`e3!l#E<T*8j=eb
zPgEvN=?*^d!Gzo4|4Jp6)$Tf*SM6W1QQ!-=O!bLb-5bC4=uVt*KBoJ;%xpvN&1-L4
z^8L<Pdr)&p)X$FY9V=&_(q#F2Vur%fz=gY(U38rEr*pCPrd6Fr&cajLXE$+n?AstL
zWz@ZJ!A(0O4GDvjJS~dNORmf?l9-_v!((#gQ05_?n$_Amb50$4Va-@^agu@Dc7Zi(
zOnS|aEYg3p=w^b+(OW!sH@IBu*y6=JwJ+kd(;JS{yH20#67N<%v_MnRNWsy3gRXw6
z?bY4q;~twPF<*W#W36$kVX(}$9*;ABQi5+Mt-W4+xu<|x??D6Si_<r~&V>F6O0+&|
zn9F=uN%WqxUfyiS{MUOE?)V%KTsrSRm(<jdInRE`X?8HM%x=1TrsL`!^OK%)d{sN&
zezv=LMevn2=l0dw^S^jbkYEn`p>~tuT&Yp7xn^g*^SNg-r~3kWi)!2_{E*afb@{aB
zcxUi==Yv7EE7du6FHb(o;hcC{SK~||Z?91X^N${#zASB_2Thz0n)*LU+`HJ|E73P$
z&te7c1;K`*VS>BnO%|OObkL)fr%8*)OS0KHq>1-#zz3;LzOPyvR>{=g<gT||rm!%u
z&q7Ggqv!fd8H2#9X-YS~6y4Y()Twf5cjrk#+0@gOP6x_29+6kvazkK?{wuE^%r}iC
zS&a^aPPoxyZ*zKzX79rfS2MnH>qV@cVBmTp!ga#`g3D$WCnhDVJ(8m9sKd-RgDGq7
zTI-BvQQwVbCxWLJGw;b@wK_X-lZRchiQDl%K4r37S6@|*3{fqgyD8wZgysq1%Ah9A
z4m)4o&6Uag8Y1=w1VUxIBKtM<lV9!Iwe_qf7pLY9Sre%p;aWS4lUSRYw@eolj{dv7
z@D<0Dr;?_L&2wk^tt(KTC^4^H`apl62<wcdyxz3}M@)rd7XLreksa!KM1k2nqjghr
z&&6IIA%*7Kij2HZo6HuN7@Y{VUUApx#+^-z!!lTTCvR#syutMB<pOyV&I=DLFP@1i
zs|}yO>B0nutr{zAgf3n1H|@Q4q~i$V4RzVo?Y&Ot3wECgknsD@;niU|>El-JWf?to
zF838rnx{A2(CE5vI43^$Nw=8}XW^1_F}yc7+=-ufRA9Z&;y%S@UPf`(*|T*Wb*@^>
zz5L0`XobzqDa%=IG|a8sYjW1V??!N3@>-)AP5Yj3SkDc!UUAZh;Z~+;tf(@xNNX(L
z2JSm2gN-C^UEa!_<?Ca<fP3NGhAYw!c{P|#N^PY!Z&6UY8!gfL(@^T!Tb-R#=cL*k
zDZj~drG@9Msrd)t^$pH<RK+;kwr)=hif`oYz1TVT3ZwF)HT$1lxqqa0|Dgj(lQLB1
zaq>B4D}Q>!(JS2P6nSmpOW~xa9Cnu6tf%kXTx$7k!4{(%*DlN6H8O||Z@7E&|JKXA
zJ1*;b@QCHLzF+jfYz7mbfnn~`u-vnE7aV<}snEDTM_1@X@SIeB2AQ~?IomeGu6UZQ
zAgZ%#b*|a`WPaZbM%%9Js$P39PPcPym2bmYy9o#O>3-ump0M5cwrAr^#Zv}e!e=>_
z@NE8dwWqJ~)X%GdQl~EN(&l+Q(PW>~o`0=8#~w{6Gr6hwS7~L0yiC&ropYC0#M*|)
zaGKA!cBggjW`)h13XON3Hp%T{ty5!;evqy=<9Ur@hTe`=*J%xv_Zm1IE*@EM&1gac
zzn`+vga%#(=9Yh3CDfQ_pB21kDY-h;bK*D0RSTc?E2?a0<(b0g<Z&_lMX}%;=H8@l
zJ#U^)*#3X*GOagmM^7ZZp1SX~=v?=x{be!d4sN|_?Za!PwtR_R%!SU$Q5R-S(d{#5
zk}6~4Rd3^AozUR$EGG1fgI6AF^RjCeii}39J&iUr2VK}?v>-*guvO?oLimdZLJ=uG
z&zf?2Cwo6*n0zSKI`_q8T}Jsl);$W1lccg;#Zqp6TCw0}li>r=<x-r_d3RN~@-wSW
z{T0Ew*r;ogoYmvdq;4I~cPrT)4@S65Y?!&^-hzjU;y1<RG_?;f^kh!EY0Mq}Zug;6
zmfC(Z6h(R`8E_uHyUe^>HDa5$(xDeD?jn2k9@+H9r2CYRP668_%P05OsYI^7<8^Ga
zWXUCopc^wTMe<(!9B=l2#ib|DUiM_X2%V#AuJQVo?rzbk4_)UmWF6gj<iHcN9n4qm
zY8X{Kv0f3Yw}ZLvUX$~>^jT{*I=wqBB7IA2-!&tTCs!74>EXNFXK=Ld+okp=M=WkK
z>1G%yJY0LOn7?BO=Ld<_hco&*EOuq&^64J9@g%I_d2^Da<HA?D(eu~M+WTUiqNmr&
zj@>(y^EEfwW!d!UX2^f_c7MtvY_iMwfb5+c2Cs~N1@>PQnzxd*T42jbsS`HW)DvP2
z(o7xf&k1u@pRT*4mfYLb_C{%IU$#$K>&;V_zuaRm-OhUBt+%Av!C5-J0cEXbH<$wV
zeH2q`ytLu<#9vtxFFBld^*G;f{;`Iw-hYcnT_Rf_m2!$N;S8%#c*xNeYS0yGG2zi>
z&TfeXpA9q?7-uf%zJA_;t4fzsXja_*1Kna-Uq5XVUw3M{+MbE~G>qqONi#aa@tO7h
z!4HBOk{26{d74(PmU?8JtCp9m_bypqDMg%t;Y3IJaSj#+$xD@a6E}RzejjhelklH6
za`Ms8$#*BP%<H--VCz+U(?T%cp?Qt2#)0`RF9P1|GTYJWbg#*}<F!q0YtXa1*1F8|
zE1#I{2o@EN$}4TI?Ays|emBhF#1-d^kdR`R?FCO7uJvs*aXh=^JIAW83t9s;L>mwA
zy;|6p{JU@FroGD~nEiHjEYOf@)|lmZYGTxpTT{iNAOCN7AknyQ%h7#RC$DSzynTDu
zw`Jz$Ip0d<rk_o}Z@%Am@l->1(|K3l&AY1nTqx+kBcm{#{WtXaGUZpu9l0cvzkYtq
zqy-N?|CDw4`D@SJPZbwm%RV!;$$heITeH)%u!U<2`2wDZo(((nO=E+fM~wFg`@N?7
zL*@kQ<+O(!4ZFZ_`}cx40oF9j&d`QwkA$1oM9f+@QzRy-WBQ9urn#CkFU;?2pVx9?
z-r09KPb6yVSF~<R5UFOF8FK%^p^1g6K8KF3l!<%y^!$U(^Yude1VSf9EW6&v^=)m0
zu1#95n37HRqK9c^KSViIE->|p&id0Dsvz4rxpvMG!^M{}m<tkhT3+ur`5#n!R8z5j
zozk}%OxJI1jaPmcox%LwFW)xyL-W~K({RI%`VW(hqQd@M4z_!GWJ_d2fvUp+*(RnO
zPMH+~3!PfG3z>;97%rOI929Z2+hW5#Eng<Ludm&`g;&?@b?Nom>32ZO@XeoAk*%xl
zc`|<ab5Yv0DB)zC#qowW%L_9O_qg;c`OeySZ-$D{B(4dY98XPa3>Jy&jVOI^ts{n|
zrd6vf=hFH#(MgW04Rmg9%e$YI!X2u|z~~{OCOE--{S>pcw<ct%Ht?Qr%<&ME^R_U&
zvqeglzmFsGp~i}}9<CxW>!&_wTdQJWyUO7}#$H};?g}Zx&L$P<H&X+ReywD>$^4n`
zzsuUdgKXhtYJ56}m-ZeAI#TbwB{_S+r~9YGwOTl89+d33w%$EPFX6OC{6CX#Sv&K3
z8lt`&VUbl{CAQn}b;QXpm2o1gKYpz4l`6ix+@JBs@hhL_26*slgf+TwmL|B|=lwEQ
ze{sXI+W|rzh6hE&luQ|8q&1E-GRU3!8rH#EXmlk~xr#N+IhJF>v~I1sr%LLnTW$z+
zWvvwrRM+A<kQB)@CE`(^3R7gtgk)7mw}crxk9r09Ww9(}2nzBPo3hxaO*PP`tnHfG
zoJ{qGgsmMbTUa;~+NUrD-I*jc>xS=xI|1c8f+o0>xP=P_Rmr8QyE=cIxom-ZXvt|Y
z1|x}!9$Wrj*nG;t=;CGnI{l50G^+kxIozeTI)qJC_p8pT2U8Y*QOa_0<K>#V^4kWX
z)-30jZla-)iK5q|w#Xg5)_d)wt6Rh&%`Kwhu`@kbqwTmHl2@HJ>Yl#Y;eo1Sg5Rb|
z5nEebXNF00Sz69?>gGLq!{Oec(3ugp{f<RiGp&&p%vi7W;aTY?E`vtx+!M=MB0lZ=
zd4**y%c=>|S6t0ExSY{!x#N)778&cDD8y=5l5EK{{YZvqV7L9=jFNj=_an8qbpK{V
zP4Z37V0ut*@-yVT)aoxz{MzO}#RENL%(E9bEpxllC$=V2MLLgZMx^38nXDxb9lYeP
zhK9#0>N;GJ<8%2Me&|1!bW$gC?%6e)Ot-ZvZQ7;PvZjA-f#9`{*y)RMG`UKPwsko>
ziLz?P^6k<MYVq8;eMY_TRo!V*XL=t~n=r8{GJ$E!#?Hj}N466Z)_OSRpYtp*JUTPe
zt6)xaZkEWh`3u~iq&suTuL(Tr5qkYsz~xp+4bA8clQNwDb|^VC1ZO$Tw|~>~X=A{f
zf?ND6m=35k1Tenqdl_i+yk@TbzMPFapGti+4Gq1`uj0Bk>8;g^@Q|GFx`@o$+H2h=
zb$9DF#ni~|{nLDJt9W=fGh0MbN3FGl_H8-0m^qz2ZW74{7BdvoHOxG7_l@GR6MMbm
zxhkh-vMH@!xHQd4_{wyaQsskf0sjrmg)~@SwLM}B-@%xWpUKL3B4NYNgA;VNWgK?>
z!#J5mvtRRdWZY7>w8>Fl+$)#<;`3oyGQmakM2K5U*V{6)DJp79J5PM>(~mkbi_h#y
zv&<5KwrQeU+XCJ?%YO0-^<!DQq~xsI&aM;JP8{eIXI<`Uyh^b5!H#KLR(80?9$0*{
zD#XlQDO;i9%VGwrg*whzPdwb4x>&XqNHQ8|wh6wJ-oC~}K$r7Cp_JyFomRm!k2Yp%
zOz~!5Q|**}B(s!9<pvYG+5twR33}5wvzh0gGH6^eQ%{qz+FLL9s(1UAChL>Z9*r*o
z`Tv=4%w%km`(n08AViRL*0L5Z&2u7$?%rv0`X9Z3E2!Y>-q?w}dj6MB?WikNKH+ez
z_Hs;m(4+|r8Bdq$ZC7-*W(Yo@A-GI*jYs#6wn+yHd0Ry8DfMjsqNe60v_bJ%6W1=A
z%Uxwx7RA1K<dxBOX#J9yD^2Gvb?i}jvhL!9#ObFq53_9xU=#CeWGNMDkUsTbdgI?^
zEHMg<;uZmH<`WW`gcTaOg3G5|b!Az>E4YeBxlE(_hf|2NO@L_UN`X1D9O>taxxdW1
zHQ}Y{ij^)lj{+uS@kd`#UbMzzj)VC|#s!sM%mg)~#90frB&a^^=9;*=&F09K6^g!k
zS=^5UF0ByQuhy+|ib<#cQ2E>q1xXVMw3{b{nM~NxsyZQH;{QV$YSWjUP&~RI)@ViA
zQ6<&Gt}Na&A~hb^%{(9GvBr4nWew~5lNgNFlqj(7boEyK;58?&fx9G&S-aX=;NCRF
z9a2Ik#LNQP+D}Z=;8eW(@|IIbQ{a<UPtM@kClp!})t1gYdud(iv_;l;zc$HkO<>h_
zJJ{vBAZK<|N0Z2$Ni7;G2WDLt?he@#ZnrMx=xk@ju)bMS?;qD#($|<8pYZrbR9V}5
zy$hF4#Qsu}P}e*c#qdO4<>#~uOm3U%zvpo5RJ!bMz3r*dy{Z*Yc8T~bubE)N70A$?
zA3TeHTF_eUX&*CeW<C0Ar>>OypvkW?%bQPNI`i6$%slpMZ4W-pSh`N(pveDI8)U9K
zem~&6bB6zNA1+;!^PS;aXMWoC=F08NNFCoRt9U1-KVvgp(zR~|dxY{Tf1@Yq!qFEF
znQ5k^8*e<ecdepSmW@>MqP0`I&mMYRSA67pNaox1FJ!#hS#QRRE>(3aYVon)Jh|BY
z_{5DYX~_yN)x=aTyqjp~@?wfs=h`6m%t;eiX1kf3lxUW7D_X-bIbm9Gx9*wnQ%Q`U
z^rjzcG~3e29KC|0J#XV2&76ej;2&Mm>n^of7btGud+o%Q%U@1ssejka?Qqgv^3tcb
z!2C>t9QT)38=rl=)+F-O^p3;}nbxn{%<sKp^<{DTti=_jw|J)~SC(2p{H6<DT;UJ(
z19`Oni#2^WuM-Tka4<bo5bOS4FDte{&2q6h*LGH~Ur#nfne{32Z{DT&GsV<?{pElL
zyDMTsZ+Rpew&k}zv1&8VJ0a$G<MpIn2gT05>y2*oQJ*v2$vIEsDBH3NZIYWVwXSKl
z*c`QxHAtmYAxQPqmG-32wMC_?JH4jt+|PBftS8hndBSJ4B?)Dk9t$$wpDJ$owCE_;
zKGn4;fld2oiV6Phb6vOXlC}2fqykg3JG$L`HbHwz*SLF&eQ>|I`0&X|DjdeL*8KNU
zoQ-5I=&AZCt~h(f(QG1v$<aA4S9m&X;y$$8>?Y&Q)E&;+W(JMI&!#N1oZ2D1?%C=G
z+g2`B*Xek<<Z#Ke|G(^%PS|>sZE4W>DSBeJcjEbq)7D&HE=6*ENXr+x#Vz$ZeE%Zh
zSK?)QGt^Xl9^Mby{XSrsQ02NM9!84x{{w{P8?$JzFbat?*eq?0)acZnRIALBuAKSv
z;Gg*?Cbl|89zU5GXMKYy{ekak#weF5$L42F%vy4TIWCxO>tqHMhVHeV=LA{VCFY(J
z4cETW87Xso^36H<DlKzVjuxjLKVrgt*x`85i3dezW|s4`UYmLIi$LH^&W2fvqO-Ri
zcHS$g*>iW!RmocmWxLkOW-#39eJVLW_d=e{!Y<p^ES}!mYp=RxCOZGQ>98Tx`N7=i
z7k6W(^0$AuD)wfoq+*mahs09HS^pPv3PtZZAu})V)|`$=o=cGzIBsq@-|>SZQMFa1
zW7fKfw@gYqCvmt*pNVpr;JGVu@|3TVnJf;yzKc}!&K92K?A_w@E;otmnrDtC%UlET
z1rr|4i-^`(^vK~v(Ci6*Zy0<uw_NF)${6~lZ~EL*#xKw9-01XKmG8^S>0dUAe@(nE
zbSs+eu5`!?=@$oOtWVAr|26w^MoV3)tArW1iPf#JA6}1lDwdf@#e5MwlIxg|FpoiK
zfs=s3lqrw%Sr<H8JO4y1xAYg^lZ`iaPM9-8AwhuYT*8jv=!*NNU0v9A2|Lu?a}ZKt
z+vk~mrY%RKfkk3&{+Ui)4W9!aPP%MsE&C$1E%x|-eb%cEa{{C#+;2~4TwjrJ@v7`C
z(cWVkE#alJXRnkNoOD<JR;19fySrCR75jSg%)z^&zZT!wIj2k3Rb0q5%JgyIh1k{z
z*R~0*cWrNNQ1HDUd+Tb#tw4rFYZ)XW{@h)(GWb`lRA5bVeJQWHL0q(nP@3z!DHERN
zue{}6b0p<YynX7uDGS70rXF_+nQS!gSj3eXU78LIDhE_PEsfVpuHk&99QH6)>j2A|
zd5#$^^D~~z@tA&K4_~BDyUxl-MS@%hwm9lcYAl$*s2$tE;&UKT?@7X>Wu~F`)uxDR
zE|{*iBHZBA?XNq<zX|%X>M3U@CgdJ+HF~6W<YvT=h4=ICT^I3L{6AVy{io=iFfOj|
zS2|=lXB;Y3=Ghrva?U?ir86OAMnVMljtun^vKR70Pg`*<C>0Qi;W*Y&8>{=kS9-_f
z+DH*6p(oXROPF6h6+OFTX3j;HM1egj+*2K05ARq|<ofJTM=Oue{X-3WN0v-*pXTUs
zEu=8^MC*(tEILb^uDRtdk<IdW;FA`dwWF){oQl{j{=ka!9d4{4FWB;29x*>-i<;;*
z?`>rFtTw+>Goo+!_wup&z3RTbH1=-e!aoHZ?FuqKTRkRhn)st6dGShL*<%myA5467
z_G0*pi}I|GCT*A>Jt4JCS0rsm)U-bCe>wLKZ)s9KCccJYng6bnIXy3x`B-w59{<nt
zV-TO<mpn^pt13_Ahm$Wj7_^^he!O{b!p<vcMULrxDj7}BoTOH@L`}}i>9a4oZ=d&Y
zR@4I)(<3#3SqV|Emb{8mNt$fWrW)z7VE<GNOVQ^MAyP8y6c=~8h6x>gAbKOPBx0#y
z<}8JSvn?WA5@PIqIoE~FTWQ*2`|^g}(`4s|ypshMI2=4`f0bj#1Lb=md|kV|_r<KJ
z-E^Z>Pi6TtwV)q!ie1@*{s<*owKQ^Ra4ihb)>%=&A?+E)am)9`7Of|(RSVfQS^Uy2
z1{{$PUUuxv+!tN1R_}TCx+m#XkEbGARu<bKF87rh0bOk3MOuAqvjY28iMz72zd5x~
zRI1-6^}qEs<uV(YHEvI%OQh~7xC*sARtVFy-o`53!<4pzBXez3Z_TwwEN(L<c&>Fl
zI%ArSS?{q8jP2@q+KxW%>wA-4Y!K1;w{oY_ZQV|_MRKn$IjxG033+i>EzsfZ{!h2}
zJxE(R?M=Y0*51ZdwI{EJYCJtGw_Ix7LP^%F1$+^!h3*B|%qiV)!bYj}_gij(1M{5@
z1Rwp&s;RNaN#WRCfeTw#L`8SGuHjv0x#@95mh=XxHrrJ)X`y}J69nVZF6QiralOdh
z_RcNtS4#er3nhZ`jkgZzmp*YXO7nGm;ggl{YT3G$eG#?GV)uwWn>EXGiIOYF#+6;m
z+V(zd6J6)lla+Vq-qQbZ58ko<(%zf(a6WJE9;y6~L2Jbi&HbD<aluaw$x~N1Oz984
zCUUm(X!yE|#|!63>q`ANwlH+ssz-MN`yM3zWSbFjgSX<I&iY@yQE@`2q}n$A(o%9`
zn<>D$&NJhn(cGs;JYQ%%3+zdnzsoUADN=HQqPAdGMqAn<rw<GFb>5xomZ8_{6!76v
z#@bxJw|P+_ZWCMgx%LPAT0fI*eMH8pnyA$gAC3m*2?r_k)=j-AGG$`eu2$0~SF=q~
zxAP7b>&e`nbkr;^>NeBHM%ANhiXL9J6}tCA>;9j9`^=W#RWkQP7P%>C-%(@Tab^9c
zsIUzs1+&hCM``Bo?Raolgva}@Yr({W|93jA*sqwiJ4<0}*a8;4H-3CiU$^PxDdk+T
z%jdAy`gWq<U2x6iZ-$@#J-KstMTCg&iZ<QIo`;1^7bP-TuBu!MkI@O)@G5@gWf@kv
z)vNMdPQ3kPB__DLTVdDx-(q^28*&`B=7@ZL>$s)+pOE~zg_|<=gj{6PlDZevs`55N
zOLx(wcU<yEzv{eN^yy<n!rM(X9}Vj_WdAJWS^9x(`UlQt0R{2Yunp<vqIbm<q<Lqb
zkX*|7z2`(E%e+H^5}LDvG`1i8)4D=A_l~gq(WzlK)&-s1HX&7b{bZ$C%hzVItULS8
z@6nRHDQzbHpG}r)Z(XG)b9gnU`$F-sL<jC-lgX{uyg2^P-Y4@(ZcW^Wb=`^m*IubF
zXwB&9D)DeuFuuJaYGG&3H@#)gT)mEG%kPhsyUg9{(t7vr_4=hJ`?)k%PCBaeS2Xa-
zMVW>ET`sed)~;H+z~hI@OLKRZyFu3?uYHQJ@RUvna+Wza&4DfPM3M6ZPRS>;*^5JX
zo=PR^ecm6nWwwcKl(K8&Y6E%o;*jcBF4GMh&AVN=KSv7pwgn`IGrZhXS^P{uGjtzo
zeedL3i*hZ+H?B5w3tF-HmfWX5o{W+a>d!y6FebK5S@ZbbyR74xavRshPZxNne7ASw
zuL8!u{F6V#$0SXAo*K}-w$wOZ_W)P1SlH`76ATt#GyG!qp!ChhD?wZSv+}0eulrJ9
z{KcU8>!)W=Hoe(+D60LIWbdU4Vbe*9K_RV2lwvwAc!h=(P2`xjdf!E9uURX(3foP~
z)>zH@eK4M>=Z)rJ(?Ef?Q%z^J%Ei)r%9yIyqIP_FA^ZJjXF>FBRpb3;?D4ue+qeJO
z@YU+>wzyd@^Ub~EWR~6Fb2Z-`dU58fz$Olh<aUeu*B5z8buB24p0aI9Q&H0MLKhc?
zCog=TILtfQ#hDairjn?xt$ps%qSRjLbtiVNS9--Cb71kdjC;@4mNES)jz0fwnc3QF
zehC*mI@qidn&z{`O>T4Fxh-*9Dc5|1*!SO_{8~Mg+354PLdVN1Hm@*Ikp8qsY`x#E
zE&rohOMFkMA7(2Nl}f$w$fWFr$x){CZ%I26>W!`|uv$!f|3j%%ZrS`mei!+t4|_w>
zziim9axp(OBS`I{1%uk7`!8Nq^m5O7Q66zSZQ0@tG1<Eov+am(-jM6{wy?NzqPRub
zoT|dj^JC+WPkFsGTRtt&t)+Ia!8(W3kc7>P3w2_?cqJCcJl?3ds_fJ8jjPnVEkbt8
z@3|%a_`zhiKdeD1+cwXZiu|>zGn7+1{3FAwwU$hkc1}W?JGGnGjb?1VEYWP0-(HY;
z=K6fu9|ij5n-^5C*u5)%w-wv#E4o_?58Rzw>f&3v`otBV@A*;hV?^ZoTKeNO<JQ}!
zZoZNml5=O#@?ZaZ^_PDSyOc6Lz48I?v!y?C_my7rt2xnav)N+j&fv<pW8RmpbTd^f
zU|-hj!|8q|{~*hSyS=HOS$B(G{&72$L9=c4_HT>gdE5PqSKW`e@bI8`^}_BS*Gp2v
zvyL*{HDBdfo7!xB>}`dW8H>&T`y0IaEYl(wKK>}rDx7g{>Dq0_+?TUWQ{S}PpoHUM
z>B;q?cQ*g(y=VR5)boI>_z#C>OkV%<qmp^uF%8-C88f?ovfsavA@k4s_ha^@FM9s!
zv0dlRcvhWP=-^xE$XVjZIi>Mg@xN!)`@($~GOnHe^J0r%cIyU4t0JMJs?mF-XReYh
zdvoJf!Ngk~3qGGXyu8i2e@_VivHuSWWYWF__xM*$H+dAZttoiL-dO?_SKF*iPn+2v
zDyyITtM2{3JJYL{OaJ6_-uWrro>O`i^X^L`A$DgD^Z&VXCGB2F!QGH&(;GQ+_A$(G
zc&6>bd0;=o2@&=e-L-G4jzk?*W<Daq$h3pwhLJ#`Q!BTyoJ_<7#e<Eh3VbTfA08bt
z;xXoW5mET~P#>qgo6C)(PaIUZmQKi=8pNQ<xF$_$ie>}vnOQvXx6Wi1J~-R5rsa@~
zWr2uKSZ-ci&gD~Kd#Bs7-cV9i>|{~3+G#Y$Aoa8uCzD8tsr808hqj~(t0nB%y88CI
zY~f3;8sQ9alQTc6J)C{*{Q;JS_kCxpr*BAT@%%5QZgQq#VO&f0+eLG0B6TCr%o6nz
zFkt*5!X<1`pc%V}C+lje&yupan^+Frh}HB9cr<hMfySVcynq+gitX)(1g)MhZB&Zx
zPt!JQpZUkq=>`8%ZU=>gr(acqJOV@i$<|bKFFv6n8(Ly{cDhmgr8QSKPh(iMRwyC0
zBYe}%4Xh_SME}blU|^M*;4t}<b^ycS6JD!zPFVyyFmZ@}$YJ6%Q^@Ul?De2&k<l&<
z<|9UD5`0~R&I%l05ea;=sO3gLQD6ffi-YG;33j7}Eh2?`CT`_lej$r%2g`%p)k?mz
zl9(G0oC;X<@<_3V(kq2m8H>A}4sk5hK4|S_DV3hWp?X4K(*H$M!{!BLtZcmG;$hpG
z;<Z^ctSC!aW7&(GX0@QYpvmXL@)i{@u>0(zvE1ovOyAnC4M~fqM&0VnS6m}v;>o%D
z)T}Q?tP|FJS>V+g<L%L^<z2AQOH!&d<ms`Z0ES@Q>Z#gcmcqu0Q3t)fPxwcNp3sdx
z&-FFroJn?p@h%M(-blVFPZu%kg;jquW!ZVFBZ_f_ZC^L1#AGMOJ4%;)847lO>WyJ+
zxZPx-zVdRX@MMdZFIF@?$qiC}E>t9R{}qSVy7jud7JFJYEaeq=u*AlwKfvXw@B;sq
zQ&aeZ8lq2Xh0gdOb#?8pX~7R1diFb)Zr=6I`1OHF>~@ll6XHW1u5qf)`Jdp#W9d-d
z#b?PQ=wj|<(s=Yrgv-U7qTUk<7s#aA++taj>teV>c4cYfOG$T?jKjQ^3cM!MRV8#9
z7$#Y0c;pKU?Y6jgz{|tDbSm@poy|KG4&Tc%HCnAQG2v6t+tS4%4Pl&SotKx*y7$NP
zL2KTv&wi%6PjzaTF{eu`U%>XMPegCGlu1#?(x@AWp0TZ8*90zoki^yJ>?T<<rBODc
z)Ju0?_U3~k(jGp4%(7$4Ht{4!>auC`eY=|MD7|*e<(TF&)9k?T7pJmLYaD$x)ARo+
zGuA&|N{tMg*c*PoV}4QRao!-c!+cA%GlxazIn_Kx4eqRRg&(C&;>#xVT`teB_C57K
zTJiZhXAh=81=eTN!W~)|UL`Om9<Kb-GOzU_OJnM$*25OZb&sX=gwGV3=I3@?mqSvZ
zMNvfYRL9qu=Y1YH$;>#EX>r9#pf|&1ZQ#9+5h{!IKJ=tJ|Gt!yWU@u%u7?>%p%fph
z!B+l_D_XVoNS&E+!jv_`znbC4JrTwU4O&bJMjSJpejd8Et7S^Q$3w46j@t^%GAB-q
zyH+*((3T||ekQ(DDlAcUUsXJ@Yrzq3Vb4yn2*KqX-zFA|_;@_pp)|#u^;BZ}El%lw
zEISWfbn%;by7Ot!p%pVuTvTOR!M?<5^3(}`^^8j;Wp6ZmcJ%stuG{M1)DuxxH{IL2
zNpF7g(iJ8r{-4^);-)z7&efMe?%t;@R<s;`beGdH*yggq>hud)zMp&!Im>aKsJy1c
ze5Y^nlJy=ATwDQ8OdHzwuJaU}bNBN+h7BAH3LFd^|D{h@DhstXcl3w2b6vN&8MLm6
z<GIBf;}iSdnJy@Q=wJV1b+{C#mHrF`N2cf%t<19KhZrxWG;-e*eq>?ZHT?^x9oNcM
zks|ipHxC|3-aa#+)}(BP&bLhs>{i`gvb!!#{5h%I$7Z2W_zNBJwhR}2pJ(TbW+;mt
zb#prxbHcdOY__zUaDV9Lmsh9lV3^_hqBD?nx#&u(t3~rnn&o{vtkw%NYC3p&dR;i>
zuxIO0-5GD!1V>D|eP)OM-v1j}y|`ZX%PC)S+jUB`e#(-?_n5dYbWB(%TbHGwEU$iI
zu1&xq9#d|n(od65ot>!7GABm&O`zKQ8LRD9vRp{enyM9WkIPtAR>0diazWy;ETc72
zX)Pyatw}Cxa6PFaxJBucbF0l~`?J*?u3m>*bp!NgUq2nk)a2m$uGW9fU1lfCo1Nn8
z_FY->T84vZ!wLb0lveWu7KPU$My>1TsMfN7?9D&FRABqguXUUg=Lar29NySepvo!5
zs-ApE;OI6HHNHKZhg8Dv&)9H$gOJIa2|He1e6g%$i*(Do143&iv(=x`Szt2r?Ve9F
zIXxd}``l%^V|HcX+|Czm8j6B^`4+Pmo^AV|u`Tx2mNllP*@}hEvJ04kH?wVC;dZer
z<j*|OlQ)wE<UHo;bgmYgI^pcb2Wq<$P43&Z-#O1H8PXRTBo`o*6yEys>$;FAKlcwB
z6K&q*Dh6$M5~NUh({UC{0sEAbvt)NKpYedR(Z})0Y{@bWhnJa}?YD%dDNj<bn3_46
z=cZFw*zyG*=f2z0!FEsQb<j)~eW?_k%~P7pHMCn=x*VtXam3b`uVLNz>ckP1vxdFx
zdme9OYuUB*TKk5jPT57PgYL?_GoL8!SNJPp*UoM4oK~%V9p|mh6lL++sm3^LhmQl-
zebMWYEx~#<W=Ga0JnQ;%?ezISzUuL@jehaUO``RYM}2?%mzAiTu;_Z5ubyInAVY2W
z=LgLFy3Tt%3!3bACol)k(EgETqVKp$X2!&;Eo*%*o#}a*ICrmsOWun}zR)L!r8pP3
zI_`Pv(3hnv8McR8?^{5RZ?mb|CLM;LwXgct{@UCBYPRC5H>K<6&lZ|`Us}yEE`2h)
zlEB7IdEL=G*EfhO>3!cDZqGD#wUqiMH!bhzeAS1WtM8whYU7r`y*8tL4(l~Jlb_{_
znv*+bdpdfwYkDcRe13mbGh#)<&y$)jwz(^GKc1G;=GVRNZq#1!BNLo2&%Lzi$@lng
ze5a?Z$eEGtc<q6EY}QS^$u7Zh9bKybp1!x>kACv?UH_3?XITWwyBsan{kNE|?OOJ4
z)?Ch?rcQ450`ETV`l7A=_tS<Sp`Tx}%}#bQ+q9MA-hF`-?w8)-|Gz9f{~=}e_p)TZ
zaHR=TRZ>3fYZKmH^diK<MN_DVVczE_YBSz7<}5s*;aU*4tt^8h`+sTi#_#&Jz28s7
za48;7desws<Dt=u92Y%}Ku(uAO7hbx+^p6-oIYjQmMJQ7N36~)Ta(thDeavTAEW!b
zE{}7^4Nfd&S@M+YNJ@pS;}-28r_}BY{@ha;id>Hr=B;(8%R4=Vee$$sdzC|dJ|9lG
z8JzymWfhPV$Hl}p(T{1@V)ZJf3CsH5NW4~C=A`)Mi1j&M#XCA09RZDP0d_3UHVU<B
z>itie#Nf-o<h@RUS?}7R-!gA6RxGbE%6;E(G-S)6OJ3cVUin4HtmU!i)vaj19K{`(
zb!vvsTGNyhr`TAwKTC`E(EYxL``UpIpY-^<N_2Lf=~Q&`|B}!o;QL5GdC`}GwF2%h
zviY32yOQVYTxcmszIo=5`7GV~3nv!*TP$opaY2w%?l#|B9v2OlEa+!B{c~EsgBllS
zev?Rwk~%}Wu!z&$M~lT6j)`sXU;QUHWZR)XKXe`Mywy3u_CU$C%x1BUMv8)gPg>3?
z=S3g?mM!~NaZ2i|{v$S~%q@nBbs7J^cpT|sJ2ofvn9aFVi)Y6~9(1kXI=*fNBkvl)
z^4whlN=%}L&ldd878L)elalw_@7z}wU#=+*=6TKvyiGzcJ4<eG-I~RxmhtYF&LPDu
zD(e;~v(C<YF-@=jkCxIW59S;Jf2mU&SlA|UWvhs~7b<jaa4R}{Aw|sixQfJS)$>em
z71E2$UT(V8$iwzn;pU?)8i#-X%DWuE{YPu<<r4)TG!l+YW8D_U{BhPLomT~q_ARN~
zl(O9;_tS*4Rcq3Bbe;aZL!Hrzf9?);Mj^quN0jCFD>MC^DEgVVA#0+!(nrOes%KU_
z|GrAO_FB7@Oc2A0TwP`r?JWv^EUWdEm45D9tjV`xA49sxHx<8C{)#z`+!kpy9f5{6
zYDy}H8Gq|aBsNKQb2^)}I2-+6=Mtsd=C-*bOeHfd#XUgjXimn_R{`xB?^KeS|E+3c
zwpze2{e_J12HD~q*}E4P&gqt0>#TT%^M21{3w>cZULW1>hEI1eG44}S;!CpbP^r-n
zw3zaQ`Qmb`AMLtH0xyFMSf3v)>RZI(nUwF<WW&vM=GX!sm1Buvf$C>^)Lsh*cwJ&Y
z?_<zkp!OlfPGLFc`8-bBMLxF_xm*wF_o!^W^g&ldWk%i0XH_fJBikLz(#ub+j}S^?
zbm^_=TWRLcYbv0xlp*{`Wu3zt_4zv<Ts>fLEv@&-vK61tc<xLo*(sznw{7ySly~(U
zg54U%a=Vqja1^?S=Nn8_X57c+rDP~%{y#COZ|$=eo|f-|7;dn$oIGLeI??v)vRhUY
z6o0tazj2-WPmNu8kwJHeWs;!xKW#hK2R4_^8VVoM+V<@3qnA$_E=}**DBdn8Is4<q
zKEdpe72Gi|I!d@)3Rbu$EnZ&qB135%vxMkTfkn)FE=r5)t7KhXmeO;~?%lC8U&hs1
z$zLu$Z4-8k=<}F7S<v;X-1Ci=fuXvNrmE*AF}=R>h3AsWlTDJ{Z{*|OE0!pGE)#aR
zwd}>aQ){)1d_QXSJ$j|Fgva*tF^gj=lf6D|2yzO1_E<Sy$dH@cw&_FE{&&hxS$?HV
z5}k7)F=x`<69@FSWE(TGa7%_Tm`&mJZ2mjzsk8GHQ<wiC9TH6+yPj!EI&J4@+Wtx4
z<_1mejn6;r5c<6C#f}4;++I%%4cw|buUPJ>V0Tn#RN0i-N(x*?VO%PX+>0JqCrs6j
z36*pI*8PY}Pko{)gWt^q%C%fcdy3Y-bO}(hP|R=0*L~V#edDZ;{3^S@-8@p9cPkDV
zzI?6q>aim~^ZQGtf2VD8GI}Fvy(XP&s?<w=_ZSt^@9#5`R7^L2d+5^?Ws=8`znQ`Q
z+%bU<oi<<1SHB7=|7QQ8Da5%{ZfSzB$H7JaRK<@oFia4(S(hBD?4<i&EJ^!<N=08;
zlq&P{lS0Q&#C+?lJ+RqflC$m0qlMj5>!v(r@xO2~qKTL1i^}3sJIVjYe4c!cl2=pv
zvts|B&HMkHw*Q#(>qX0keaUL-EBkL96k9T(_*TN~RU4-Hy>@E!y!7hbMYEC*77c&>
z9)EDti4tmZX+Pz%Fi3q-r_v|Rgv8v0_H4$3nhXsJ)!W6p&uPxh`=WieGru!fAU`Dk
z<;CjXVY{5xe`#8?OX8co!!$Y7M$sD!>sT9P1(HPe6*;(wrS4lMmU%90e~0ArmqNRl
zyi7J)S{N3$^NT1ad8dadD(w*7`6%1NF<soL#PQOEDN6eVSZrr7iLLIl%b(cvXv-2N
zzt-m^In%;Y|BJ*$ay%?b%i=E$+-#QJm~}Mh(P=X!ndI)r1;OSsPvk7@ki8g|`}hC#
zhAZLf54KxKZ_`#P5G<bN8<to!>48<v6}j}QI#ZRy&t_l9>bub7;yEuwsC?_o9lVYA
zPb<DUrhH4HPjS<Hy&`T6qaU|+Xe%WM&r`Dh_?eBdW#N{-w-QOdZB2`W+$`e`Y09k@
zJ@J)&Gp9u2Dvd<9irCdA(>_(a6g73*^07the~R#w|M_h!4J=W)pSCpd@66OLKa(I@
zCiq`{-;N!}XWU4X72`Z+l*Alfdo{=+vofE#+|Wa@-nB?!%L2F5nYp@ZI&zz4%-$F_
zCFk3;HQG_ztX7!l%QIQtdtRrsqfvMJ-hIl9N(U5vWvEW|cVN2NpwQ)$%hY63{w5+Z
zLwJ#h+J6Oxh}FiI_}jdjl@?rcc5C@(;}ZB;KPYg?il828#dH7XTwlO`Xnv&bu_>uJ
z<;B6h7P*bZxsz{B_f=Xv{dlLbn{Z8mO0=VJ#hC;5#H^05cUYBiLt`S(k^@WU{#<%k
z^=8bD9@USHtSr6X*uC0xmM>-LXZP|7DCpaAB;RBE!UBT{bMl+7K4yDqd`_Z?U6MQD
zUGc=Ot=b}zMS4!JvdR-z96Hf<-8uF26%l<!8Go(0?<&(fu7rHivTaR{`7*mkc<v_g
zxj(L3U7BuDY_dK#Wc}=juRo+XR1ViqRAA(=o+imubK&JvudN}aCi&-^bpHQZbXL3m
ztv2Tri<!3{b+?~kI{jaL)<P4T6+drYdRnJt;n1*N_@~i|aOoEkkAEx()iASAog$zz
z-Cyjw<u3*Gvs~_X*W@?Voqk~VB%uBFKc%gK{nbZzhy{HORowNa-Mo^)*E#ac#vFC4
z<5ipI7=508R)k}b1J^MJ;RDq?M%Q$%h(9g5nZK|*DY;Z}k%GZ)Da}O-cHOzA^F+L5
ziGq;zZ>3Z1<|_ismpH_*9Lf=yULq#+a?9JE?`cX0UCrC_BGz6}S;G>jwW!ZoaFfho
zHqK+s{FxeD+?=fGicfkHQ>-UWZ`MqkSax;|?}R0B!HWXhSMi<QRq4d!vTD=z8&(O?
zA=Q%>WN&JpTJDo*?B;Pj(>;FYzS94Zi-S#{HciRjk|2Cw_Y6}<2D67LmQ|Jy&1`y#
z8@A2inK$RuL<fPJzuhIJ6;Bu`DM`<~zg>WFM;LE!$Tp2D3qoQpvBw;Gz4ub;_X*0X
zPo8^8ezHEytUaOsjghs@3l`fVCk?%9l?RpeY78zbzO^-K?415PZhwoL*}A_oTA5~?
zTB$P2eBuxDlXEWkvoTI!*t69^=*ARLH_(xydK)cH*Yhl`-no0PHRq&;4b0a0&#eno
z4|lRYIk@S@*Nnb`?MD?EYUaJUqj=6u|L+|gz8{_k+fFW8+jsGV*TWw+mFKt5an3bb
zw4y#>rC_&z+U>-IPlbE<CVn#J)bE<{G-cK2oJm6emx^A>6I<9+yz-D}gLVAPq^lLL
zXM~wt+j41k&7Z2npQ=yaS?8NK?R!;?xRS1<-SsDj&Rmw>+r7E=vZuVdnc{>4CqwVn
zeZF4T?jpR6;Y@bq75{y^)_GZ+mPrlKb4wQ4d|3I}`5FQCi}?bJ#6**VB$7S(t~W3)
zNxj%Q^Zkbd^`dGT5w~|Uady3r<lVf@ancda;)I?L>`xb~TQ!MF*eY#eT~HelRQtnw
z(QIb3S|+!{#~jY>(w=o(rTU8ViRq0Fd0QQXUriUg=2)1KDEuhythQa(`a2FYgg1Js
zeTekxcAxfk>Yv1)b&8V`cvkJPIL5I5w7a%Nfw@@G&0Ag1e#so1`rrB1(pZsxQH>c_
zRrbZ+WtX*Ca8>1ogKA{a*MohpBs0&SVOkv<ZWk1jUe8^%Os8|h(ODY)+P^mJ*s^@8
zuV3|U@n(mzz3s(c?kFvGfB#T+o~~RS)8<!q86NCbW~_fYSu`kCxZr0>hhTww>CZV4
zDc)(qGN1nZyqz4I&&(jxlV#<_==Fk?clOM4X{MJNUNwJ|R(<6tqrk!;;*hoAU^6?n
zn8*%+g@@Y(1UE@VOyFJF#lpWW@kE+}bl6t5HXh9hJxqE`7CsLR5}TN&3o&U-kW_H=
zW#BA$b!MjVqVw||nz>}XwqykA&RWv5YOCJ?Ek<9~mMcoBJOV6k^=c<hi9WpczaxS-
z@2^(F8-E7&vvNGMuc=?#E;u>orqQzpVvH#(xOQl}z7t^#;`t|Pk@$WJZ}7W&qB|^}
zI!=lG&X;p1;=%ph&8#tX9&QTL&&?Hd5;KVUa`1wX*;c0=zY<noV@mM0u_#>R5SE*B
zxNWas;`<vs(KnanZs1+nx3G2JwjG6!v!f5B=6g+O*>!df0|UziO$Lv0Mn{$fd$j~O
zB3bU8&gTs%Q;2EI5nP~aZEqd@gXz(8`|kUSmCYG<)k7i}I5Z|i-B+9<*TBssa#VtU
z6LX+QbVgHLz0=w66UuqzEG9%sWpOO1XD*cp>`>uSNlJMcyJC^ER9tIIsi%uTD@)Lm
z6|MipqH``K_X~I`HaZ5KS$dNtNW?54%`MkJfSt)A@|IZ4oufymE=iS?3|K0AgwrLy
zsx3f>=>+Sg0{(?BC(PQ+dUb+hKG)GS$J`#J0Qd4orWdle-#o^pz3s#{bxF<-TZGFs
z@>Z^B6fm@!vbc+lRnz5Ju9Abunhlq`T7o=Rbc#)TbS2K^x#bbDCWnPu(^m4W{I#&!
zZfBRkV#Y4>32iB&T8=`Nd4W&X7xykaDiJ-wF?{23DeZt*wyRH9J48*8oZ)cUNb`oH
ztCdOAbf>^)6Sns_Zke&E$}NN~<vCZ#F^AA;ChHsoGcI~r*YF)(=D7CA&&zu^_=Zea
zz{`7GK=YXB*Qo#LKb7wAm8cw?d7$2-<?H<w4r^}2F4|_d<!aZj$`_(*1Vm>P9{pVz
z^C^ni%xlUTJ4S{Uk=Y)Kopae$pC%uXdG<=|#LBc8o0fb&n*MQ#_W6jr7cCz}JQTC}
z^`fLd+2ZYlrH-Q72PQaA%X|_aEMj&rAwGLrXmYUAPu<;2Ehm@yXSr)VTg{Z=*rmH>
zi`etMzl65uyXF1A5;$iy?{=wqtT(p>9#*aliM`*W-6bYeyYbGCCk_kQUx`f$v1n7<
z_b+1Z(JA7tCKvfjV=Py*U!Bpy68MGd&Bj1s?_+95j`>Y@oaJ+Njbn)Plo-#z48;qt
zZYmww`Y~GNneg^#t+KKm*TU2!|8JUPyKL>W8_FuIl3(ss>t|o&4Uu+PU+aG{|B?Na
z=)waJJwv3|N~FiQpFbL=?`)|k@UdfYm&byLMV|u7wmI$kw1mT%&G16cszY6d3Jb%S
zCh<FMGIpGjxT1lbF}CvE;e{+_8V2^CCORZdWBIb@xhumZg-Vra3w|zPcb)XRvs3Qq
z{^lodGOi@b_Emk|BG)G1k#{?GbIqj1jT26KG&oMWzV4`~y5~v5x0@!V%n-NQV#}f$
z(sM#>fwIOrmNV*=-ombz%B9;|1ZxjyDofsH(YU#W&1y|VjOg=@UDH>xsVwerV0d}L
zVs7J7oqL{4p*5mwRbI3N*2##v2zgq%ZR<&2{$GegNNIyr+}=FBV#8@~GTd!%a5_Yk
z3n~fmD%rGOd0K8iQ7*2lVtTA@T9nTmpM*80Gj!4lE5p{z+o5)NW&!6?V+}=TUG`0S
z?70g?w@%>w(_=JIkLy6Sy&#{<AJ=1FIS(kYF!nkJXbRW~`uKld<!UYECdjG7!8!4k
z#GQ(14p$;K=-%hNr(rgArA5)3l+8)Y?Ic+>OOpyTZtm=Ic>7rP|Jq{jtcwp$ip)CC
zRrNH(Y}3TPS>K#*S?XpnEx7AYrtUI-MbLEqmoGN8B`sItTjEg@`DntSG|7#eM?LeU
zIeb>0Xv|%zJKd~TQ(m@d@pX&I`A&fxfh#QCzyD2B&QhB5znG<9N1j@q>Dmd&q9?Zm
zGrBl%Em<*p|0h+}wy>|OyI5X7F-cK)uhKB7Hu0`P_mRbk7h5N9UOm^%JxY9H-evWZ
zC$o35ExJDGP3Za+Z_cRIiq6=_aIAt!<4pV%5!J&A&$@O#>Du16edd{*XNtW$b-t}=
z32qKxR1%tTJ5T0Q@S`baW?5WwvOR7~-&tX(bVAU}{XzPiCkOfy?~2T~WNq*h78FR`
zA1lgw)p?#o7rXX?*U#0AT+iFv8gmu=l($nhGTc(L)bUuW;F^dX^M3~j=pN2h=L%eL
zf#cEUU23bY9u5gv@T<mr52ryCU%pDK<GG{lV!VlNhSq!k@fCWP@p?5>{{I=jCx<cn
zF!QA?A-66)E@br*DD&}M&JnaRj$h-&o+zPX8)g~sT~&XUecAY|$D`}Nmc(a%37^B%
zIw57764%YG8v?f(__t>qTx}FL(|WUz=j63oC5?Bc_BFHy1&DF4E7&_XRxEhtu5WxX
z91qjp|Hztfs^!_@<qKy`sa<~5&?lSw$Zw77Qx43CRLl3Y+Ip{tbB(t~_{_7D*Bb0z
z<2oyfZyEQEELYn#Gu>`(@SK&Il9gBbctTU*(-miZJaQEu*?q74Jaez)@{n&P0>y2r
zhxeGyed$$hH21FXajoJ=rY@5%AI9TO#vwOZjgMu`Um~Tq^;OrJSBqn}id@+JW9Ipb
zr<OSXR}|&kAJ}}$)%;bUh5dPTX1%W1>3grQv&fFxeduCb!mfma8?G#MY|Q*7v-xdD
ziDy^-x21Dh5ABK06Wp4@qBTchl6hnA4|TbH(VZVvUvzwVHhZ<}vn30)S|y&}&|ugg
z*vq+zHQ@j62lh8hRtqVlB$(z-J~k`%h~>?TyXrnY64N(u7fdqUuVhp5*l1<jMjbPC
zOaGkU4PsN4bO>$!+WX`YpVt?)!>p%&ZNFWe$=a<v>4i~We~p@4VfKaV`kAZ5?(X!p
za=l|Na%<8eW9h`^bFU}*T+et=`1Z;dw(CI`Sc=kJw59l)joe;`?QVDewL5EzkHobJ
zE2}2P9Gp>6>wUMq;s5r%H&0r3rK!X}(tNv5k!PE$@vJw6;uRXL?t5ZeCRHpj`FGaj
zTj;f{kUd$kx*M-ADh^cASyp!R&tn(PAKxZ;p1sso;hUV|@oKr{`-d$z?jCy>{wqOP
z<AuZ$G2xfn4?3<FohRA#U4Hsi744T+%OwAw5W0Wl$kvx;I`Nen`xoDOyjo0hhWzaP
zN=v4xDj!lg`?#iW=1+|&_NQ#LZr)V9xBr&dEr<0>76-n3|IPSMv(k#o|K}X>&;QnG
zeMe?lU+^Wb?X16(omHK}Zf{NgEf}*QsX>2ItnD-V=PKE%&6!+H4s0Qf?~aHq2-cRV
zC`)xxp3<W1RU9bh%C=acaIt`?nnmsZDaN(Z<pL(f0>RIOrKT2|9QIW<F=aMvc4=&u
zoND^k&}&AzK>T7AzAzE9RI}&=GZR+HdDqjJ4l?GYxlXuf*0ww?YEojxR@TNObIFUv
zE0$Mf6;-Vg7nExgNOZBkZX%GOqT#J7a8g90(^w$tg7AUz_6iGyjHB8I9~Wp<q%2ub
zGh3izvv3E`!RE>t9S7X4zit)3lvKOrO2W*ks+OUyX5TCCHMw@D3(UF^l=LZ*U(q~N
zT;zgDz2|k;b*`DKO_RQd+r2!@C3~>M&M;b5O(3w{Cqz7>$Iwu<$Vu!**2PkzPmKy}
z;yoV(mEspVvZxsC&k*1}6w7R=%%Rx3XQ|=;XmR=2>#;_k8cm(zZCFgzf0R7(V0&!Q
zXD-nkqQ;=!A#P|9&YoDPbEA3L7jJ&U@OK^+6F)Y+x9FE+7qnZU?jVs{eX)b-p-PXa
znY)LGh(h9RHpvAi(z;GG^aOVE7e;k`a&<adyqdkl>wDX-3W2TV0xBC>%T{!#dAf55
zH0@FH*uTkhNkG=(0>QNclOshZFLe{BZ0<-7&aqtKYF$w)@N=^5Neln99GU6K8$?(I
z3tBg}<n|l-q+B+C`MmSKd!qVwantRpw=dT%5tXjF=K4uJ>A~kHu}p!b(*=K;i`-Kw
zofyU)a-GlXxr3gvCdZDfGR+?L9gW*Bm8o8uo)qr<f6^3%C`0F!WeO36CQ3iD?S=ZD
zc=U;Vobjk*n)FM77cYd%CQb7XpZW4ej)^hbEKWhgA2ZoL&a{$D6<FA>6gsQ#yTC%t
zDMk_@Ev7aN3mLC4wRE*sa&)HIEzG^Otuwo=c>5ROm<L6Ujebl`p;D41-?!&w%n+E|
ztR|~y@!{FT(B=70Ok{VfhqYEts+~FM2)j;GM$;MxuAf2ko>b0z##wJTBb3{49*g0;
z-ea>~S<Ii6*>UHl#kKPJOF}YrG>e-YB@Zv@Jb2PwWqIY4t=_>WCEiaL{q{Xieq|TW
z!E`V7&_}}EE{df$TcU!VPg~3(^VWgw(hTkMH)1uC<7}6Cbbb;0&$`LVfGsX-W88Zc
zL-9}Y;f8WXyA;f3Exvf7cdwXJZ>7wOPU)yrnL~{s*)!C|W~tW+YdV#-)qeFXQ<^p9
zWa{V6aOER`HWt>uTLqerFSWmE9raE4(s65LQvpp0$&-%BDGIDxDiw}yuJCS2<Fpj<
zzcO3qdCOz71^XjKL{^w)e$kKj@be25iT@IHC1i!eLA7^o3sax?Y*}8cbkJJ0v%N@>
z)w{AFqLGF7xQUSXl0zS*c6?u18a$2du@v`2{ku~_6K4qax=jvk?fCg}mUZQn|6;4-
zqWY^0ZM+@}Ut<dP2`UasSv`$o%JoB$ev*-Pg`t+>C1+kOh)%LNH8by{nBael9X|fz
z_WzcrGcinGp6Jo2>3RL?^y-@dl3&)cUtIperO|RxgY_*f<)B#BONuFvdiQ@@yq?ki
zGt;UspQL_Q2^3UDt^F!p;gliz%V)z2!70UUlPkloc;)=9sF>WmltZw;X34DODn(43
z65A{Vyi;0k98)RFlH%qRT&!r@ZK%pJZK|eW-L=gUT8>gjGzGGR)zU(H)mfch9-HH~
zaKpmnC9gg&XcTKd=fXAlmc?|gfVhqBF%wrR-e{k>q-I{1Pf&DqkhIXY<5DMG{MNZ!
zE>4}c#$39ExmlpGBSm}lI?GmT)irlsDsOP9+_)<u<yu7gGMm84N$wR3+$E-Ft*(!8
zT=hR?XV<=iD+H=!YPbcjag_!fU3fdhNyK=ufz~25=EdI~)y{Vq<~)=&RtnH7%DmvA
zHZNOQdBw6w(^WrDt$#FS{T;3y{EX9AN(;tz>u>$Ia=W5$>Eo@NE0*l2p0p%W#AZg}
z*B`T_Q#DhoSM$H#u=n)_@#$U+m6dzkCKNUo&1g@Mo4r7QQPp2XO|N}9lh>NRrBk<X
zwa#ZRPh@OKy|E^5$>x(U*R1Z^r@C#0xS-vt=RSoRlQueX2~C*G@I2o^FlE{kZLNr`
z7oJs4)(0lsvMB1%@7TO`+v<I1xfZ5yB=v;NN?0|&N4f6K=}ix<^DZr}xmhIcow?fU
zLhancrVq*z7l_aKKfS}Ysw(0~QPfh?l5a`f7fy+;`OtszV9B5MwRc1KZ=aG1Oo+Xl
zx>??M9m@=P9#u!T92fZ)z5L>R@`1aSpI-lqt3CGM+9t1~+r?Kp9G_Zy%x-nm?$@U+
zf~;mmuhK}i+Bo}XzhGma=89RB6?^8*TrEFCcf*m0D~C2ZpD<Ot9T75f@0(4m23uP)
z(+-AMis=0o*C;%(EYawJ=#ix=)8c=3O}?4;SxmCQE5GTrRCxE%B^N^3zt8;_t;6tV
zW$5bS;OtY@?E)7zp5ohha;=TPIvs&EXHLmjwr}1zhsVZpZKc2_jcu*f^AEi`rsg?e
znpM^Y7QupQ$;~Yi#nBV?T<I)vIr0Al&)$+RyShFd(B-NBZfwW>v42y?!k^Qn7i94}
zaK}C?^oyUE_$#<q<C2o4u~x^+Y}-$ZAH4})QN3$T_t8y`LThIq{b!TOoOm=+@j`@8
z{VMPLom-{kBt`b_QB6KmohNhAYQ~-vuRSNTH>TIj;&<el`&jwv$7Q#QE4SS=E%p+*
z=H2<nbWR4h$h}84oibDIeDmY`cu?lM+Y<3(X)m|i%szeg<${t$c@BpASDoSdzFH&w
z=%TsI9gI5dE7lY%B%V^aD78^XB;5KcTi{hY!>bX!SO2f+=Bz0&aSq|w*yo^lM&!&~
zm2e#!Cf!)oT!)F<H$Mrg+I7fv<E4#L4*5!o^E>YQUvfhE(>19*54TUeBkC75QSz7c
zty|~6D6x9HQCd?r<Fn|l0PntlJ?p35HCH)!IBd=?Ze;=P&(oL=9A3TfW=yQe65b2W
z7tafD-C)a2x}hV&KUdVaVi{A|#jHD%ol-P6_O|czEwH$q@6(wp>wT%@R%D~hiOW7y
zRkNpDRc;Nm=$!Xs6Vt~DhFtm$g=fm|DA!Lqm85dRQ9a-3Y2KMtYVJ$BS+ctxT@HQy
zbZ+fKtN+pSqExqpSl?$#xT?S@7#(~6*V<FRG&O#|QJYv)&+9vr`CQ*w_RG6n_N3|N
zao0^c=B^vfcxjJS%jMupnQqt9&x(3E8(fQVZD33K{P}d--1>%C?*HEsxEXe++di4G
zMEmG}b-zbj)`rX2&D6>4WPV(G`0(%8`&N0}R&uxO<eqGp)4fSYz@G1d&7H>rjgJ{+
zCF@R?-2Z&GcJpncYY)P$4vMTTJRW^aM7Ccdwe5V8=3h4(oipM(XZGyyymKX6V9(8n
zIpUY!IL#@2J^RoquXA(L91FMbU9>v5n01P-sEE|=R%N@?mx&ddtKM3=Z4A%<l74@q
z<AKj>^R1u0STXnE%jlO%D-Q^~F-tx%E6Zljw0+A@{oWg!XxgjD%Kc^AuE-Nw+Idq~
zXiPXU`8bzIh^VGggwBs+Q&(1(R5367y3FUpv+(V)GO}!?!iIjHd8Z3zJaSuj%eLpq
ze?vV{vvW7?^iI!`y~SMk<fffS*xI*l``$)Ad>d=K@SEE^?wSj2zd}RK?N0r5TJr3%
zUmO>O|28cac-An{`9`h4%)$v-cCXL<c^_VJXYsrD57Pwn)bF01E6LB<#l+-TqiJ)<
zSogxSomo$xH|uS+<GpfG>t3gobgoBcfNJ%*z6-I2kDVUAnNwS!@m^TD|8mmPyD2&X
zQ!}4_lYAw4PRGYt_ph7mp{f=&_d~l{9-7%(242{s?9u07J#BB@yN8R_98T5nH6G?n
zw(qZdWVY|IePeXEt;nN)kG=Mta*un<vG8$T<b~*<Tfwp4_!ZyHc(>l+;r(+hZroi0
zVo?t^ygeOSRq&s;?`izGr$=|kAF{r9AUiy2SHH?ND*->74JG1RBqg+4kH-}+@I7*z
zol!OXlAgY=)Z&Q?F4;(wzY8+F-SfmtrF_rzsYk42dwsblZav#l?XdbK*X$Rseq3J9
z+s<n9MT~8F)2&Z2iTBn1M_KEtI~pFFEBMFG@>)}WLG|q9Y_679u`BHbZ42j01T(C;
zz3mzMx!M=oUrk7Tom`~3GFVOVMfP;#o5v09Hhf%fyGQEP{q^teOWom;{x~ywOGLEO
zuC=$kp0G{a75D!QgQ7!%Z6hnMoXd=WhmO%Kl2RoR+>0DKS(U9s8VnPUw#wRWJALBf
z;sar<-hDQm8_ph_!f~^wMKbvRGoM*ZW}Gs~AC@)DVJ-i*MskaqWUDo6o}hwj7mH@}
zBo%=JAq&L%g}MA@G=(0RvR1l9>`Bpu)eX8bEN!MIzAm`AIpbuNi&jJPt>%)8N*W#F
z`Nvh57W(xn1vtgeZ01#$)L|_Y()FF4b(5)*d!}!STE3EL*BOnVB`g(ex)Y{41kE-M
zb~_tsoX)7v+9P*$(^TCFOxfH!=2|D7?&r*xzVbKp$A|UXSR~y2CjEH*)X{eLyNoTB
zcUrG>i*4zP{eAJ--B!1WpTd|LSUjg3Thd+P9lY-C&K0uS3j4}`ecjI{<{^>T(iCv|
zf4!Ci!|85?P9}aA9j?P7F{>Px$%O<QIQn0%q+sO@FJXnHQLh&IDm*bcrJ?*-M@Udv
zAlYc+g8Kqe6V)cZ>DJuYY*J;oVv5(J-0eKVQof2W5=0WFC^XLT+|_q%N2aifsFIeV
zvhb^yC!6Q7oLVqvf|A#&KMO7eE%>mfM@w<_hD%-R7VP<SYR%6BLJi#JSF5=dx4X?M
z-fhPEsG(~cgFwgr=6|QW6*)i6K6$>G>&*#&6(J9WCRg5=Ef+mxQ>O&62r(!%dY`;)
z!uZMJ>lPa$o6{GJ!?I@jE|Tedle;W5TZAEOirK9|#YpR{iz>0)tCk**o;iEv@n<cm
zlRPvd*Ph$=Mj)z8ZTYpur-B*JG<WK4T{7d)t+nTFg)R8<|HhMT)~}A~o;b7MOO77n
zAFfy0T+Y8;%+yqbW^G?FwIhmiJFl~r#`SmKTr-X~W+<&N>tk5AVqP=5*a^Ary4Vx(
z{PMGA9GO*c=j!5VpRWE`AkUv=bg!v}<!408sz09Ua|EPPJ?FXT6{XIsoYdFD##*hl
zN?-KW%>{<63dXlrrxdNuXg&4FOi%B^+iwff(vyBE*nM0T&S1y*)mzbxeX-~MzExre
z_HX#y-4NI)P+-EeWpfw@$LghWvvq=Q-DtTn$*;vSKl-cKkJX`H%`!}qS|<mcTF$uq
zWx)~Fx7vG_OD&zYyKl!N2hMe73HjW`*LNA~oOe|V|9JH9wSI$t_hkQSYyR(0S|utY
z&9!mazD(b_79Fk9g75GD>+Ag9S|@X7@r1e3l^%^PE^+yj1&v;*FP@Zs&#Q4>xzO@w
z#Z0H3YH{zHv}WlfrMai)9hRQ5%8_04&8jum)@$y2vuDvdzs);t96Eb$Yx8F1qSRSc
zDw*fMosMJKFg-w(r8KfwWUoNe)_(%08tV#YGYB41Z+@WK6m`HS)u&Y}dc~?TF2>IH
z5r?#euCat|KRv@orA(%6j!Tk~3a?L7`NkQ&DV4!aa{^t`C&=D98EMzp88k~t;FHY;
z1zYXwVN*72*0flbDQ75k#%24?Wfxgv<5u%82snPAB9u3W%cMi(n8PZwo@KII89EPr
zxVHZPOb(tz=GW34K`jF9r7X&&b0#_prERp?6sVyrB*7yZG)bT<r@~1w^kd(0kA8{H
zl-@ZreU)Y!y=Kj<mY*Ug5briuk2NDI=-?vv3+yJ}Dqm-m-HLbIBQ$NJ0<ZtWTYT(~
z4vFpD!Kzt2>$LXA6jz2<ssVo|CoEtIGTq~l8aJIwdcl{=8qF@J6zUD5??{|*ZQ0S=
zlrPQ6Fze7RmwRi&X2i&F?KR#ldT5F2qc`VxSU)Xx7gJ4HQW2Dva%qk9yr6o!BkC5y
zpLCA5a_KLX=KRTV+2!oP^|ssh+<*Q`#kFRe`+P3TgnQ3S0~u>JTFrLodL6S}@rLac
z>xwJU-C-Jw9}E0v$=8g$_$EhWNArry%JR*gN|DQK9xgbeb~9S#`=Vl_shkZ)h9~Du
z$P=?`PVsVLkm6|&dEgnumaCF2<Wcj;`LyLzew}a4sr;(q%XjUWG%NCT{C_PAhnXg;
zW^l@+-MO;s+Ga=L;Ka|Dm)>!6+4Sk;I&MZ0h8aRL&(Am_#wV=e<Pssoua%ytypuOE
zkx8h8O()Fcp>IO;7j20}Ui}V5n%>r*g8QEpTb{kN>hib9y?rJfT3r`IoZkJ?WU0{#
zS@-o))h5yXnhGI`+^Uzh3oKfDPU+ktrWvpK?TaTFvn^NP^Iq)pVTD24(Q`YMBNw|r
z<Cy$1?E5C2Pm7afR~t;-_A&qep2C$rtw+1$4qjPqnyK!4Df+y}tB}cNwUO0dr``LZ
zr=lLYD^ld2^Ys=6uZ9TjCWa{vYMN6rRnBf&8qsGYrnf=0Yw=wz{@W_2W5ram_NrvF
zoO~6(DQbhlUv52y&dLc}J9*>kEY-xDBSUhwMJQeuQHc9u<zjS4!=>@<+5$ffVVT7n
zb}NgVHTRB~eCF7ZJsg(4{;E5L&mC&nYsi_KVJ$6wlta*PQc+<5Q-EdDrgY|aGY>V}
zc8RoGnH;aW+%L|5X?j4BJJalf*ftBUOUCC}@=NmGxdgc{R&)+?cl6`bwn!70vFh^)
z<HmcALZ0!zZEmJc>RCUDugP}SPZh7gNB>vWJp8ojtV-&X->iBCtQ-tnU0dyE3OO)*
zIy{9_WXBZ$(4|eRNmEyBI_dk3n>)i+J8|cdOA3Lz)fk%pK77a%-IS%Qee>C`Ya22x
zb{wuNte1|Mu=m##QFYI$6VC1wI2x4VZtdog`1*~~jK96gYdumXb}C<fKF8ejv6!A(
zX|G665VvOA$J622NtQX59~YEADy?<$c763h?A{5c7tFVP-Yz*HdAm@iPv)nEu+8_2
zk1pn}D-}Ex^CqmPRqj2<fwsf5o#aw3IEwK~U+K(yD)YzctE}g@c{)Zjm6ulEP(E5U
zLr9LnQQ($bVgkdqn1t};R=s!KrE5!PmMl1V^5B<jZ>Rre(@$?*(fW#MM?vi5y`}L^
zPqTREh%@iMVZVN|>Dg%WG{;q9Nd|dykDh1Q?YPRfExS6+O!Yt44*p9Kw+^z*t(reI
z;j!q=y%$}UIayXZS{28soxL-!bN2Gn;z9XK*SK?~d1nbuPnJ8Laqe@XUxrfw_s?Ge
zfuH1D<uzljw}mYTWDw2gyJt`*a6{eYo6$^LUzb<qw(YymZwnOcS+Mw1*cXkD3eNAH
zBz)(48`)J^ug(9weYtJF$$bTvjeY5DR=RAHtd1=?rgodTbt|Lz->ck>TF>KldNtqq
zRrmc#-@F}6>PC+}3ysRG?+9#q^ymM}S<CE17=#Y+DVzN}Y`?Z~cjoN>yC-ef&39>=
zc%<lKFWG(FoE%?xXZ|qk^6a?#c$>(sbrS>3AHFa--fSXcWvlVJZ_Z<z4~Zgnjy<kF
zEf?M}s*C17dz0(7kZpnHUX@#Gg;^%u_i|WxW2f&4@#}#r?`*cUb2@Y**y+w}i}wp0
z`aX9Y%W8l8c;davl`|stt*=l~%$zr~a!ZqiiUG?c0~?`rr}}<>nwRz1?(#zIkJ6m~
zeyij}%1b}dYM!ArBV%b;lJHVbgOfY0RzF_5`nM_1;#nWPJDiW$^IIQq5n1-<H8=lZ
z1KwoAcq^9`8q2z*r2ie3GUMspec4R*xAFd$yAN?64Yt`n@we^7gNCgf2U`W^G)e3W
zRQNCUOUl``de5GTMh7famUHl*&UW0PwCMj!KGhWmZ#f;4uGw$>rEksWgXuDyx+^z6
zuG;(j)!N&qE!58(`_*8pwS|9r&~~$cy{$W3f4^{AxMANR%bBk?Dtg`4nZY63tkJ7m
zA<+CmMQ78pnHszKKl2|kus8b2wK`(*PsSZT7%eKc*vs7!`?=YmV57@lWy2W<>}$0-
zQiQuCijOe<nf!0Tf=vf!?r7M2aYpZfVAq4fd@|8fr?3cd=`5f2+4i@^o&^c5x>tCF
zLN@W=a{b|a;?PCQ>gq24z}0SF_|91$J73Lj^|4Pg$RlG*f2%NG=8<hri`Lz|Y|*{i
z*1F1RjYyxh)Z8bZ{`Y;Hq}XR^BBOnLj#r0n$V|PK^+^|1w#|_%X_=&7<ISPHw(5!H
zryq+tyBr^KFtHlgFPzD=GRf-lWYci29cD9CmuquGRVk&#bO?S{<!Lr#k~uKH+lT4T
z@}>soZI6X{S4$;S9cd4-sOW6a=;+oxu#ImfC)b_bJ(W{~WH#+@K0Eo0;k6}-XN^uj
zmXw>l<n){rua*EV)`i>Vs~qFp)X$R8rFVJG;wLiADN~z%`z}l2TIF!|(P7>jGi_I%
z*{qTyk+nqm-JDZvc+Tg__-ChxwoKg7YN7M?6UT-c*Q3^3O1<{ES{=V*a&TgXQL}@$
zLelZqlec^?TJOotW%PjMO|d|=&VL2gMGC@~?LQ<-H!?K&gqSiWZ~c0)Ydr^d(;u0r
zB<^3<CuV9G%hnil`)s>%>BNEu`~O#82;_2_<#Eyd$)bZ6XDzJxIuBUi%XAIuIj7k(
z@lDc1nYB}c1*gsU<25O<`OB)4n{vEWANxvtH1#-j^tCnD&8|5I9C&VYdHib<)LZN<
zZDM@HQ&r1o$G6J|d%kEa@0q@LRnNqdxpxfDx2`yKue(z;(#eAL{KOrn%-8De6SkCF
z+WCFT(J9SG9YXbIMCd=>xMcnY*O>u9Pj;W5o#WkeYpX<*vtm*p<ID@D%4QZZeGDIt
zhjB|qS#t!so(g+)L1xdYi&+M(8?2T&a<38cJ#*#%5k?uq6&&jlx~{UPT6^Bvu>IhP
z?hxy;L^0k)fp<72^5mSo&9W&!%Kt$0(c`NQEO=m@BFYtSslHcoj`CFZvl^0(8t(UV
zE-m<QcGit1DUn4#mAM~XvGHq_JeR#yN~QaNOK7Z*QnBJ0*|l@8YW8OT-ez;x?^8%e
z*WtyDxlW(foN9jCXLZ*?xG5}as$#;+j_DbkO1ZobZ}`8^7Q0qy_wUHcW(CRe7NPQD
zyZlrezXyE9O9jMwyHq$BLS~EgZ&-Hv$cDZHC;m&#X3?-;zUz8*l09Fn4`;T4?N6Da
zsgo}rzA9we!797@m%)Kq71INi%@|!)Gu~nf$_%cP6^ytzAy@i8XNRET?I*X^2dw-W
z(|2-G*qN-AO*1@OPMklfth=`|s3&$_(bJ}L!b+!aPt)JI>H4EZd%A8_%si+kJZqhl
zt&*qevPQp%BDWe*$y(WUw+$x7JKm_jD&5L@-fC`4{_CjIywmJMZ|_?aGM%TZn^nne
zwf{EJ!!v&P=c(R2pE1eh(1gIs*f+}EMMpJ`-`L3(*yZTB$Y04%frCMuXKUY!GkbIT
zIRA!lhjIt1hVEK2pRaU-(N@!fNR1dTpO>{qgy)`#GrG8T^BLbgH(O3zSsJRXljZW~
z_Psye7tO2rf>v;Z9q`+`apQ#Uv%iiyDa{P~&2Xuu;ck}dl{vT03f1h{!78*<=f9(@
z(RG%0<~LuNf5`OFkL20MdMfLZl_={RmWu1QxcYB(nHH~{eCV&vx?h+2B4Uo!p37S$
zYThc?^!MC5mx)camm9Y7tC&bk7CPTI!C~f~I|VANW(#gyb2V}CHDSINQR)+W;qB@R
z9KA0C`){W{P!n3jB`I>sQmM^ZUe#lj!htCdYPGk2d^FwlazOV%AJco=8oNHSHZX8(
zsK0Zw)u8+5O8Fy4=36^zgddHY_%V7`Z<K?g+lr~Ta)Q>nL~bnqX}MxT({u^Z-cthk
zQG#oAPo}oIbyNzrmx^9*ah=F;<s7ffwSr!iS{0?VsV<im-qH=<9s1yF!GptFCVvR%
zIT+BuFhM#YQ}gkXzqcdVgr1c2++4(IsLU7Fx1mzBqk(}z@h1zb2m=#?4g&)NLj#io
zBLl~O2G$)y4jUFQn6n9M#b^X9m>rhqHC1!+1f_$WDLO`Xc1)PU5R}5z_i)Dqo|R!=
zb0l+)XI<7PTO)K##Uj|@>@1aHqnPalcbw+2G;)2A+_K=pVynqov9knL9$dWo2yd)w
z!5xj(Fzz^2kJ7Bbl^#pAM2=2Tj%-Rd7s}nWVYT0Omi3QB0{Zo4x0at4+H*VW-LWv%
z-&!m_0S0FeHrjG2?1^BAKguR~I_B20g4Cwmq;0!+B8m^4mD{fJ#ATAOc<VCGNn!#<
zDUv5x{MGlwY`EgqC6#;GY{zY1$@|j&?N5T#7B}3L^W4~^`ylI6Kdbr6DSorI9QW<I
zy>iRdbz-Xx7iDn%*Y)WBv+I*N_c5RSb+W5N*VG&`-*N4E=x^uQ&+o5_*Q#$y%G^3J
zB=4>|L#T~n#R3LSqaTw41*DcF1abu}^>h0Fa_Pf1nMp_FBQs}iy%@T9&8Dk9|JN>J
z)qbmz(yiOI(a_5yM<YGi$5e0mVqe*BFCO<h=w0JdUz;l^<hWL}<cV9#iMH7c{(ajf
z<goCXOmGT&_4JhU?!YZ7Q-p7drNqQbO-Xc8yPDMR#ae3KCGloT`HeZrT2H4m`#*fC
z;BWWw$@6&)ec{g+xL&<}N9(c6(W^~=0-uJ3^vP*lUBd18G<ZH!%Zr2tSHmsWf|~fE
zCW|dLNL$rk{`Bee_<Gsv6018B|8EmWm|A)*Vy(@*Q>#+yIJd<mvz@)dma;Y@B%st`
z-M1*oQ@6g|$@0ADChf!$SRLRv{pT~cnIU|~Z%sa2^h8|K=+*KGB`Y`+VoTzgu630!
z&TyJg@nY`tc?^#>KAKl!KG#D;`=1;~K{50B<?@|O$EI%BDgEhshGW4?_V}hpx*i`F
zi8Ez<Z0asM^YM5~+nt9qy_QbBR{zq_rYn9{&%rL;wHqQMdOd`eNUWKEP%ScHv)0zl
zCmf1Z1U-*+iQRVPp88NFn^iJ<jgypX?^~yEvyfEBFeQ<TE&<nNCe4sl3x9Tqr?KSH
zEbXnI)O>6%bs9&{eO0+Uplw~VMsxY?yXp68;{T_X2dF)Y4GphhlKMWkQp=>grSZ|*
z=UV;md4DuUDprMtOz(bd@N~Mj4PWFme!HCyyc(wlG|V^hi}z<;E2`DJ=Fq8g9&4^J
zOU(Y}@ICnr&&R0|iN=y;S&nBIe@Cv@{O|KThr96D?6fw{fKwCQ^jUJ9LZ+$7Oq}x|
zZFcUG#<f>2`=6dQ;ac3Qk|!Gy^rufKGOM2=e$sN{p*6dha*CFeF%>+Q)%-N;M)em{
zhwmH(soy&L>pmPl(7ZAyaW;39pT$?hmcUjA0bvF<!w&iX850t#I;U_LORRLA8qieb
zC~>H$XxWc5M>O|G@z;bocAH=H&)pq4(<kSPpzJ~aN)4VxC;lf)Ub4|}ZMH|S$K(lO
zOMm35ct<Rf^4t{G)#326%S*?vaMF#%|0N2Q6>hp{UkK|yTH?+Sl_4GU?}*!BOOD|B
zThWW2b!@j(dev8UMp^E?#>z96UNRGymMn3TIAMG1?Su*5DoWxnEnKaF^85EFt9+fI
z;TW{Ec=qWf%%K_!cp?QRENBT)eeZVAX0KmiQSsu}cP!-Gb&V#yi&W80Sftr>WvQs=
zqYJTFLbhMmtV-ujY)!Xm);`P9pMErkM|)DH@Ub%~Gf%zgHo2ax73Q<qpoMdK)(3UR
z$yc65Jh;-mmB+}hb<4t<;9HCS_66HanvgBBL_u=bffZ9X9`%{=>CyB<7CZlM5^b8G
z9nE>Ck!jY!zRp8|?6v_;;TtbBTZt?b2+eq|RCUOG;)i6huQOW2Oq_(xrYw>V)o{{R
z?^-)0WRXT{hKul|fPPmWO~E4qEapWQgsz4xH8U0TV@y+O;4MfsiJ7@K<6vLZhP7h4
zMjW?9n)T*5JX_hCx%{IEi_1LD!T^_$OP9EkOpF2^D&2g%h@oeh$@PRl#l*WETP2cp
zZ-w+Kh2Hkyj55BiTlBCyRWSOmRg%z@6Fv-YGgvh*tq9&d^~8own>BuGgOrZ7oh#~<
zbxpB!k32o~ik7;lYnr(=mq4uRX0^X>JdIUv1!nXGPCfc2IW&6b<s`kWGnFr9dRs@I
z{2w<>XX)7~JKa`CU*h7}KIhQdJbUg1r?+0)G5z=KaG59O`y6?KzZ9k>E$_bZYTct<
zWx@|CABm_}U1*Zqy24$O^HO(9&%zZIg*>aK%$%N}$)bMHdtUWa&OL`FESauS8NJ1(
zS$Ad8=lyoK?oEn2EWp(56YcM<9ieHoW~Rf`n2*6{CfXEDt<-yJ^6N9lf3e51VJVLS
zWd6QzcKvlhX!VsNs-+p};d7G2cJEj+D`MlGBprQ;{Wp%D&D_{0`RkGJ=ZK>Y`wY9R
ze<v$SM;<piR=6kZj*OK2%q@JT8j6-(ffEc_TxNbU>?%C<RH$<2^5+X52HXtfui`jx
zCc~`5o@K)`v&}#MpUqagaqiYb?y!o<3O9BJ7S-%nz517=$D`;)IYl!kI~t#jSnj!n
z;hmRE^MkK7zTPJs7>os4e0C{Lo%zLCpy-@-(viyvOt%)Fc54fo7|7u9l`m2C^sBJ@
zH@3NpZAv?;z!i0V*R>t{OxKpjW;4FIb^XA<t?P>KX2(g-zHwY`+uFjj*{RvnZk}k{
zwmE;UW>k3etsVQmZ7}}&CcSv~ZPtTE(|$(YDZIaNqt)L{yGpBK-#GO(-1(cQdw~01
z)s*Rb*j~CjemVQTR^R15%ggqCroq#jZXA-^`XRslTCwoGvrofiHcox_E6FEO&aGl+
z)3PI`d1s?_pQ)VM<8wUvq28qX?^oUY|9alvIjpiD87q8VSg&5J|7h(54I6dl#rNHo
zIJ-PM!lM?*z(2V?FIn1rrpv4YxtC_^a7k(JefhDe>eBLiC4ni6UWNNtU0d7tbzSz}
zSHbc}uN{9^x~~53n>6#?5-0k~wk@}PQ!u^is`1v64d?H^3prnXrCV<Oy;+>PE`A%Q
zvFAV1@17a=eBt+*zZ0Le{#Y#kZ^iM)kN-StkY4g2{Tato?n8XC-||)E1>a08Sz*)J
zzw%4HT!?1W!h~xYJF2{1y^8I36S$$j%X3}kmPw1Q+N}P%FRNwYv}~^i6(NUC9u<Rg
zg0(7UbG3aG6LXr&UR_#rK&$QA1Hop7P48bPiCf=bx!Cgm8>dm-$KSX9Fgfi1#jUjd
z!}9-sXI(hd^K^dQ&)fF@eysoh=evH}pWpNU{co>lSiw3op`KNup53E?ZASw~Mg3p(
zdVY<%AL5N77L8&z>i<r!yJcSYaC_q!wYrOC*>hd9=d#MFDMfZJiM_9y|6po9BYOc;
zdgVVghVS2$8IB8XHQ>q#sAZjK{cEZ4%1?0{lWZmh+8iqOIj~vUg~KOra{M<@AJ$`n
z7mQlj4l=Sjs&Y+hUBf7JS6NVw(d-nHN$Q16tA!>z673B?2^uE}d{}IK!7;H)Q0nJn
z-#^U8_bze;PN+*ZD3#D?^u1nRKclokqpqo<bmoS#u<d0`1s(FoI=cV==;-~?F;Rgb
zQh{Nzhrq-ItWzTzSZ8$3@o1Rvqht1p&h8bRTozrC3s`4bbj{7^TD_ucSwzFiA6={+
zZu3?&%%9P{qoX_9y`keoN9T+3C+QvgBYKX^C{GZtcp2vK#VM_KdL<WQC8K!d%H;yH
zw+k8?#(9Nu&lV8&o}_FplxP^z`y@i_k#JaqM%arT8K%Xp?=l2>(gd$h7EV|!$h@#c
zDlxOSP&LL#Fx8=kjd22#15?Za#<k`~b3?MM3T1OA2(54vxGN}JX_5Sfy<SYR;rsIX
z@5dXsZZz!rF>%!l)`<#}cHihysGPK82J0M;Niz>j+F&thp2j32%SpPClZ`Sb8~k56
z$trWQuIJ>b9G#gV0vbCf?Yhw*bRyendBi)x{4Zj0raJ`9C8o+YRq9Ffswhr1IwA1y
z03XwVsa%2li>HZ*T{jD;kVux4{9r2CaWLNGqtCZgAEoa;heT|Zrx&&aPVZ%u_*^le
z`4DT!q9QgnznYWMGK+NMl}tUaWE?g!3ktRQa76yIiT_n)S)1n>N0Yf_!v*X;J7>)l
znC>Cq*y+Y9IBSXJtPBy>%qg>$ubj2AbJpsevsOsXj#8MtC9-pK<?O%-tQ%L(-h6WQ
z&YiOl+?;)2CTqZg*+(+x9GyA)<jvVE6WduP&N=pT_CCwG2RmoYnb8sHp7QsL+!v?(
z)MFJ<mCdaG3wx)8&r>Pvl~J7MwQ!!yCGNWx>G>VXc_#!kH@EoT&~iQ@5E|O&BUJ4X
zWb0>?5gTgDa4J+<Q7YwXpU>qguME3&j~1vbG<};<b0oc0B(bLBrgGKN1#_<m$_ol@
zbC}xHIBV((PNkh4_BSVSF6>%hIceLCMU@(p45c_LS9BRyEuQgWj>)XWl^%<2rC85#
zE}5pWgh^qE!>o39t0f*%tN{y_xN0r2t6E}zYl+LPIa4c^m~gQMPFNb<wKQ(lY?E1&
zIV(C8IOLKGA~tVn`f3&zqRC*;F_lMg-kZp&MjFfiPn((-CFUO~P~DXuwleLjfzT}v
z0j-b5=F6wCUX1%LDcSg6F#f<|tyMwsR>`41Jf~fCk>oM#pDCqM7by_DA*0Td&E-=o
zTbSAl7L@~n)51Lj8$L?;x!SyEUj2TGy;tDO3b*##jlQ!U%uxyK+@?8e&B>W(H0SJ!
zTypf+>|<R^Tz9Pr(PB7zYsrPIHNL;*oQqn^X1Mm+thE<TExGBnmZgy;kYU{;sdZ1d
z)<vya`nYPH&#5KHJ7>+?QRW>M{L3lCB{XgChaS#{X-{Sf{9Tqd)qLLnu6h4ot!GJ`
z$I88def9<`#SJ{KD>$TemOPkh^q9M8sl<ejt87FPyI(2)KH2tXrErg#s*-p0yBQfx
zW?^qS!hDl-R=v{NqLg`Taew&a%<hi@|NeWhF=e)@9$-E8y^r^lqR_PP;uH3%Q?lMD
zNbNf{)AO{zo?BZu53W8Nxs*+CNuc-Epspo?4ojW5SOo=GBX4hwJiR3T^w#H6+n5*_
zuW~V_a<k@eGkQGOR`7dUJ~wOD=_RGB*Cno+#HYcP{wwdUY5v2fQwpWh|FSJF<|^>#
zY!1D-+_P{;`LE@Fe(jjZy|bihd8&yxyZQY8EG@4@b$Fid^!rkfk~z)qiVtu2uA-H4
zmJhYr7HYlwq%AA$5;kMgqFp-iD;6}rR6Y>cc4Xm<-bnkWm;APcS8wrhxfJNu(l~SN
zhSi)KXU_H%IA^^!pqoYd;T8>-y*HxQ-RfouP*`_Ydfom1r}w_}T6ZaPZh-oF?ZaGd
zrujYP%}+TC*nV%|+AVV3VmZs~_43+AOw!^(SGjyccloIl+MBGjH?j$CZISop+Gai@
z-nBY5L159u-Se^*Y+kNd*lN0NlWO?pgM5LibH4SnUfk^4TqF8X`TWD0FDyca&b0@O
zg#<UO@rf4DUn3rnEnxg-jy}&eqdRk$8usSR-fG{&!qLDIX??`OX1nv7ZJs{c?|5&I
zs#;>r*{B+-^np>>^=ebe!SyY>k3QvCE}OJ}QuI#a6~YNNYI_;{bGatwbsXYcxMJ?_
z=C7vewGWS(9TZn-7hZZ@`^(N1_8!ve%f+ux?o;V*ePOY?t};9(f=%fEMk#L*6GQc4
z9!Hs!2JO7de$~r1in<BCaAI4e*uJJvv@}BE&WkPQxeuS-wUyIxuW#30ZpXbc6Zbxr
zI(>WhUO|P^uYS$m?zv2D$J|%1d%oYw=l!kxd$-`%>Uoo1r+wz$aKdK8sWWG|r1yWU
z-hZ%X|Fs_*en)MIzbR8ws8l63-{z6slvfLXb~|;InJm9};7!Fj){mjDWWug~Udb}c
zZ`bmbGLD=1y7~>WxlCVc#Y`4B;$CB`eb9+pQnYMY%4PxMO^0rEgeWKO<t{vYb@t(N
zk*AZbw`EChm(blV$$MnG&eE#rWg&02e^QGxiMW)k(_1-fYHE&v>5Lr#KQB$6p5JkL
zdC32j`FjKSnXYp)9na72T3+|+vIFns$-5Q%ym$7m&YxHwSG~vDsPVwDEjFvTuNG_i
zEV_MVUy5(~&0R~Reb)%A3_dzPWE1P$RVo=j1YZ_cvzxUo@3Kv+aJ^*`Z^dY^q+rFx
zz&&48+}=-ZHs#r?;<(P0Yn?;Zy3IPLTYT4TubI74qtW}=>DS);-?h$Od$ZxH&ACa@
z$0nK2o4!^g)90pDV8P{@{fyg<JY82X1*8}I`pY<8uiCTGQ*iejU4|Dq0vpfvZPq<u
zm8jyzbkZ}>^uYJcA<nu-AN5;mGgw!*vfSL{y|5rr(@a}N@~r_^ob*<ks;zO=OJcc~
zB);D2Kj-dE-X($mZ5nG<Psx(nQ8Oz)d)eid(}F#x1$uVhGt9l@XnRcNQ-%57{eJ`Q
zZxg9}_;lIRsh3L&?_2D>WcBb~_|F|I^90!E2{V7cs%5xBdeMrHrqg?G#Z9QPskwOd
z%hZR{s}6kcjE_Gnuur$OIrH|my@K<#7-SMm4jq)byV>_m#~tZM(u#QkVh?v8*(_6j
z&H3o#ginW$1r^;od+_91fm3D+uWyaqccOBRzs%enD;uIN)Zcm;Y!VW3?#G#evZw!c
zt$((1efYVjVe2YiN8X$?_fpEad6Tx^<UF|S-vy<(NP*JAXA{lOcz(P&(e~yQpEF*J
zXQy7TEWW6;H9+#7#4gE2=>qH~>tg#9|7V}u&}-Yh?%WF>RrPh^f_pZlo-h&jnXxG-
zllP&Xq>SLoRf0^F3uUa-)Ypix__Ce7E8+2ViQTHnb~78?CGTnMnDS-UvTY(eTW2qS
zb#r}+&XqTr`!n7>JEV6xP_o&wv2up_a@OnjC#c8G^nLa;HKgGo|Gxz8ZHfH*CRl%+
z@b+KB+k1X*BV+DW8D9Pr`|jn>%PH#~Jj;7y8DY(~@qU4i(C#I1k$<nOIxUwH`64ND
z+A8nD`KMogyDEN;S%qn>z~SAD8!xWBdQhkS%%g`U8oRq)K6tR1g-aez+W0N?q|rke
z?&L(y#MO>^ZhY(ZEnd*2b1m4xHKh0N{ff8^8h1b4d~>Gizv6T5>Zt{F&;AyCF4Vh!
zFz%Ug$K{Zj&3y9&3i9~>J>YA4`pi0ii-&9P`r6QJHUsB;>!p|43h%!EHuLL>>koZ)
z?E3yynRUJh>nXv*)zzYFZfjb*o?zMRwc=p(V^s;ML&|f$IXYe7SbI|b)}0o^Cw8q_
zDu44|*51!i`|(Bg!JFE5^=c2Ey<5hpSJ40bN6UTw8*@IJ+3swczx-?N8xx5KPem)2
zTu}Nw`)y7T|GEVGyBB4@ec*2Vs`n_gHp}TV>%ARGa`zPvPtC1+{g&bJ%cWOKd9HT;
z`n`Am1W~_fC2Jo_teY<@eS*b))AGVcY%BE^R=FrBww8X9&h8D%t(Cm2zk0*}EaR?S
zM(<s?<ov%~FkIlgPxc*?ew=5e$J}@M&-PiZe_d&HCR*<1={3(c+<(d!ea7WIKZE0X
zIqRs^EIm=GSyR-)zGkFeJmh!4h4J63$RiQ?Np4-@ws~u=_sh95eq-696lu6rmr<v7
z>+BOj2acAljrzIi`}ukMCohTk^u%(b{LzJ)!ECl!Ay3_wdQ6;D<8^Um$kKq|YqySY
zrJe5F$}(ZrjjSm_L2FJm+5Vm?>N-W`_OhC2(={=>O5ffy^-ANGoD{?IV=l)IJyFJ%
z-n@k!KQ_c1i8)cAsjN`+WL3@EZ3{ap7o{=I2%Kj1(yeHw5R+H2f$zpI@q5<xF0!>K
zJfn6!<$u)8OCB4=&IE4hTK4B|{)!KAUu(X8PQ4?SH|J!xxV+!@?=HPN8<$_Pn7bn8
z^+9W`t$DEy`>T=`xpbDq)lSh_l(n{tY3ag(hyPmIm*npc+Z*@z{pP54v4y`Rm+t*n
zkhN5H&JOPj;*52f%SFmHHXWZj?M-H$Wtr5g_tuN|2zzP1b@~(FBJ5q#tMc)Lv@^qc
zk0i}?BA!Xk4N}*4O<c59?N(reT;Z2Mcg0E>BacZ{R|?%#4&T}+8WvFH?Z#*^v&c>R
zuaM4$DFU+}>FaG$X`XDLySm$0+f(Q)uhG**JVyIGB8As)I>B^W<Cfa4&)QwD{&e;D
z-b?CS8g(WqUF)UICXN4=@mZW#EL@Hocg?%MXx2#$uk&lNT~<1M>agklb&`ck!%ebv
zYG_8{a<Q^kg&tQn3fvV?42(!@w_ywube`PD(&IFFp^gx9=f(yrPn*R)n<l7)`>f*F
z()DVi>eaHG%;!Nq>+eokxvOXIv@5rJvRZpLvUGg**!614B;C^IN#B>0J`B=|t=u?w
z#T^~ZUFB{^gT8ogG?MY|+|((e9o~@g-s;9w6Hn#jty4S%7?Q4?&^R1fHC1YRj9u@d
zpa#POo1Pz?ToL{H(!yWL=K`arZ9dKy{$|6`zG+F@j*8b8baCyfD|r@a!DzLI&&Id;
z_D|<e9gElR_*nJwxZST+(yW~tTz7Z=&-Bne#l&f7aaDu+o6*Nz*SBVLwycPA>ty0&
zymnE4Z|S<c)+yqSI-XiAfnBSXm3@o$y8JePB}&h?n0wEPL+V_6wDt>%+nfk9$yzNo
z`PG__QFhbU*esQfJh`WMsg3z1Pc>t9^Eef4cTT^$muJoI@3Rm%Q@YD)UxkvnS~jmy
z`Pmj-53^?>*-8)23zaAD&QM-@F7V5vGt*Ao)VzA>ZJq0si?^dTGFQKy`+VQDCmx?f
z0zdZ4Wmb7CmhQW}SaGhV@^7728jBa(SBZu8MqG4JWO}^#>7r%fS6En#9OVO#q>HS+
z6`;0Afq~I+sa&!{lh^`<2B(e6+yNO6Tmv?V3VEvj*ZZg88l&VQTmPn|P9rIvYw2W>
zms<^5uSIc%E@>Be|0j)SvB1pe9+QbLzP>2Wz8N_88q?D9lN}vzy{=Ennyz&9X18gH
zp-<uzA=f<$Y4P2KJ?|tY&*+rNiF4Z+kYe(f?ZT9WJiOf!t}4q^Z|Y9;f9%!yvBh1;
z=tqj>uA9+cf@YZfQ($*K-Ffr0&JqJ-kxRVmDh_MEQ(%~}GbMe_Dpp&i&Q)92G=(e@
zXx6AH-m<Wz-R}F(lt8=4{=XBJJI`O@?qp=y7JRW?^ySGGv80)uhF6+J*L6KnvH#?t
zlF}&mFTwp*mgkx%mH^=zYYt7(A7=eGg}C<B%(qW6SzxE-JFDpb?GA0PjYe}U){EBb
zJZx&Z=D1X)uTbvLl!r#8a|5<rp1UtXj(z3hb*fC6N=rV+dc9dWMU^pdzMaz~p-96c
z@`@MLG#1_!3KyTDZyU%E{NRAtzAFJ*s$2AO-}rQ$DzLtIBh_eEcC*ZP6Zc}teBG1T
zyh#(nEjQ#e`8OuDu8J_^n%tQmc<1mu=@c~v;~fjRQ(rWt+<DMuy=RST>mokg1K-YY
zdiAc16K1}B<oT43oGGD?cTZJ2GJ!MD;7R0Hf!6C=1UPkd47gM$cU&=X<o5h~sVS(L
zqk7YY16@5%A-@w_ukb8dw8!nBLUL!*s!bDG7r8Wwv0k~l+3V4g`V~iNg=bFq@1hno
zIqb!fmTeplx9<wlZe5m9sPRVO?y_XlpT1Y8%@&AU%-6Ie@Xv?&DUwRIsoB%@&u!~?
zziRG*4@*U^Czg1Z-n_nVSBi*^@00jrtURgTrs_8q9@|+I7?JLgJgdmnEos`t-1OvR
z8;h`~ex*7m^E4(M-@9yyh1ubgx=b%OE#x}Dt9N6C=9!3%LVvD^wC6Z$?-3FyO9|rP
z;9PKxL+Pa5B=zaFGXA>P5(UCKoTf#dI1({q+kTA<#>ulbe5iP3*v0W9bkdg$1C~Fx
z_@i7H8E1a*4Vco&pb=p)d9}v;f@?-w<JK^7rGJ&Y+%R+7-Yg|9uEs;xx;9;T=XF`_
z!JeCblm5%B&Hv>*%U@|JLyp_pyIY!MIdAF5*-2aEEz~&vZ{`Q(;5L)rBFoSG^X{8`
zxyINmUBh`>!cnnlTc+NBI61CtTVQNfi_itpDIVKg7YU^<nR))rHZ8HavaSNP;(9J7
zsvDc{PBF*|yg%<{+0NhMX*y|Q(_VG@FDb~Cv9#q1DiPn2weYUmi6?u`o|~en^=*po
zj_YjJ9m|%h++3lZx6r`&U&-Pn72F#gicV|vJgpa7$l9hIlPD9`A{#X!N~CD(1RV|s
z?lm_yZm5$K;{Ba08>y{d%Cy6w$|0KPDYNGWB_qw18=8*u%vqpuD1G8Ssb~i7JsUL?
zmOAWhI;|eGE_zE=<D~x+zGhe&_P%uSIi!=j(V&gDU6I#UWUhc~yZX!7+cWPjFP!=^
ze80aHcl@<UQ8T{e`8(w3@*6)tFDDzg?d7G!-5IN6cZEc0hNbuxw$7gPFiESsUvNfh
zll13`z$4vk$=s4BE#gmG@FWR+_C6?*_38SA*1n?MkGnrv=mmbeSlM24SmADfhq2F+
z3AqKWx)qz|%xbWDC~GvWpoz0J>Q}qm;tm%v6GJh>DZ0XIUbyQT9eiYXq(V0Hw$x39
z2g;tNy>@}KZyo(x?9H)e@eHOpv+G?H9d`ApYpC?GFg?+2NC@0u*T|osBITi=vEhfi
zTG-#foXV%OVrG8Zx+8MR@593EEB{BT?o}6@eqJ)F{MYfh3weq^{a?Pq%>4h^Z$DU1
zhqYb)6SiMObhf^@$At{W1#S95H=LSYXj*Q((x<O<{<GcxJ<bY0E8CwLopSj3QgYIV
zlS#rWHO`*D5-`uYYEkGW?gJ0@zgfg>%sIn*cXz#|@)-`zoi`+Gl;=JX)O@eRd6vO6
zbTP+_Nt)Hc6NR!^_up6{u*XF3&WfmxeXIv3<O^;SRb2XGf_Z;LYg|<82Tenb6D?H>
z*aKgvZd=$__hI9KPZK|9Y~(n+uv>zgM~kb+!bT;cmG`ykl2;uPHmg^CXrFjt*2EX<
zIW8?JO<-l6#6M{Pd+Fpgoe`}*9OjZ;3x%32OiZTyFZngwHmLK%L}wREvp#Q;#~*e$
z7EapHIonQ=d9TI(y%TtfBwg+toZJ40DMv^$zGHTfWcQR{rH7qIBv-E0*r_}7wa(sx
z755HJ(_YjuPnpv{hNnJruHH+_z#}}|FIV(znj5xdZivjm&Vm`nN}X?4JN$jTuxce&
z@ZtScjN7cay7wlwi&b_#tmw2lv@ghF$tL4QJ%h&RW=VF&edm=H7G0iYCuGAhVf&)Y
z4L>?MPCFZRY4AyR?>P|7p0(SspV3fGX>(nNqeevg=ADKIn!8E`rpis+GnrFCM4GEg
zqIr*ZfBL1Cg6vr#QiuQF?Dx)e_WCIO)?wx%Nn^vw9>&#6jDt7-@Afb<Jv>3>wc5*M
zN%IWW2|3L{5}wPNC0=v!TzSE;GlKUL$Eqt6j!u~~Bl)6mPT>CY3pp;F++2OiQ*PxP
zwUzcNN!^|?D^_L5E{)i-YKPL52aEO<XzUK)(Gxhj%D`dg1C5o3w;qUaNKn?4Go1I(
z(sI(Txk275n;llJZ<xF-!SwGffxsJk+l@QJE~w4kz#40;{FrOE^$xk|5nLt3I--oC
z(pNb0PtRHr#mdv&;<HHfQ^zbFiNjqD{69a;_g~naV$C0Q(NTATR=0r77mm)V*9uW5
z%y%7_AE+TeFJQyWs593hPAn_fk-czU?_^)=FC9+~Xxg%v1#zr2`{B%EWAF5Y`+uY0
zIVT>8r3we8MD*&;VDJ-|!lR-({nv7N#(BT8Egd%d%{Y8?*A9C|iCvv9c0N+lyZ6ZN
z+JT+74({A5aOCkCE^bfP+fqxWB%Hs%;c2WS+~wJOsljSZiE`J2qq`d>$4}`zDK#aL
zNq&)o@45~<;}=Z)0uAXb)~y%ys}0l-YWBxTaS9o4OkcEg`$Mz;60Cx^C%&1nG4R0>
z{~(+172DFKrW^G%?cdG8-PB|(!cZ@BqH@uWnI{%rS6N;x)i>Eu{>#M;9e-40nU}7r
zlz-w7^rgtI@6Y0wlkFn8Pfk!d`Fgiq*F$a{O-8*FmvmcCF8Flm7t`hGYxE=EG#Qv)
zHqs5g>=69lsC32*l}1etd&Y*&y^Wn!4Ei}MC(3-B({#vK>7#m`k@(NYf(pL`r>yW=
zer3`dlT*RB*R~1q-10m%=du0nh@)#t^uKF0n+2R-Q`2+ygofp;13Nn=OIe=Y9iX`{
zquD)$nWv|7n!}DTi)AV$zFa?B<BY8z2U+D*TU`Gl;=5zx-%W>f6fU0J(~+*YO{scy
zU_r-T<->=2`ehsCzEv#aieBC4z_IG0*#0eDy$jsTwg_D;(z#VK-|CLVoduivA3FzJ
z_BH(J?D*&G2@bzUCVG!dTq<?_T;JN=UNmhM&(&L=ixwR|(y;g3?gu=+F;}uX=gvMX
z_Q7V|wFf<qMS6eq=rtEM|2Jbe`BRIzBQyBXCI8k}=dXR3oPX>3#K5VCe%tJ_5UBp3
z$@qbZ-{ErC$@StdjYL;<)HN(vxv=AKci<m|{;O}=cPby7UpUdHp&>b{edDR7FDo`g
z{`3xwYz<9d^%PpVQgmY8+k>BP_O;6d{aN6l^fszzi)8N=k1w}8=5*Mpmj+D`IjK2!
z{p-m*Mh#ZZlcTQ+TwX9sZ*fGgi73Nf*K_BjE<c*WkTWTGy@Gw$2A+V!*S}2ewmYnT
z(UNn4%*s`_{q#MrEPvzL7r5oj(K`Vu=R0pOoa1#}_GiC`YNy^zuPR4}{z*J~2?xGU
zK5~8S$ten3e`(L_5A8C1&~0aO$12jBtLf^0onV=7H#vVk($o(KFutL2w#YXr(6HWN
zw*H<S8?p=o15`67>l_rmHpN4AnJDK>E75aZT+c+83MVd?*c7G3bNyyWeBb7?uO)Ys
z@UGmU*{r5?<l&s_zn5OGz8deHD*Si}$M3uM|CjP=Zs;vdbTN9d&dP{eVUlt<^A(Sp
z?pI5=-y9LUAhGMR?D@JxUaeO#yli(rhO9XDqx=3AlO@ib-+RuV4Y>XD){RRAR<TFI
zZ)~_*yi)lL$AyDmC361G6m*n)`azTbLsPoTJ+m7gzbg)(FWR1DJwHyuZ3E-%z22fz
z9j3}e&5`=mCntNq>u{8g+|sf|9z82Oo<szxsy?=V<*~(g_kRstPp#BlI<=?v$DW*#
z5^UP*DZVb+VCn7sccZ&Jg01g8?tBm}zVH0>JJI&fZmXY)w)0EbHP!X4a0740<&Tr*
zM7_8!c~iZn$o@z2ekD!~CtIPP#*<4tZ=5<B9(ZJut^@CWkJL`Bb=4bJXct}Sj-26p
z^uGO<D?3(BW_zZtcuA0R&76N{CQr$5xD{Y}?`c<~_OhrAQoCQYKkVgUt!;kNyRWU<
zbXS9%uh-#^5u%=+hXof(e~DdqA!@n5;Ps+?hl^|DCv?xA@XNV^_1vGi*Q=(cGaie7
zsFG6Ucc<pr&h;7RCU0nMJk7P5`}8UWFRwi=6`FH*KWNdJ>E);L^k6{5eCI3wt9FV9
zZ0Qz}JMz_M^6TB+lM=j)H1*P7UcNQ+;f)nt(}W`VZq1fyQ+%VL82*AqzrfA-#Co;E
zQ#e&;O=~+WttvI|VU%5$M_0_|g?hIa&D*3_8x$FQJ51u_q?CtOw|4hV?b*Nd)h^Y`
ztCu}q^DlYU?&r5AX3b#9ZvXkJ^8o|XwB;YWVto!8pNZT*_s&Uq$(I*2Go0tH^_6()
zvUG*#-aAt~y!O_axSzhh??%)4uifXr3eC2T$a$!7cTwtz2P|xJgg1BG;owo)C2@Mv
zHsL9Ycnu%keZ6o&REON$KKXD4tqeWZ@Ex7|Lr&!69nM<ztaR5!Cf<qN#_6|wHun|F
z-HA!R`^Wjeq2!VKJ=dLE(izXKdu)<d!<O*x>e?rV(y#8F)>doskw@a0S0|@{>m7z%
z+r}f;T_?SKGO^H5seGkwqiX)&*XQm(N?5MrFL2EDLkaJ>4HJaVt!QQxpE##yz1_(U
z$2Wh^NL<g*+?R3g?u}l<1KVz_6ZxFgS(cDKyFiddemT=p3k6pxy}DUHGu@SZjh`6s
z@v29g8z=YAv5RcJur>CvaZ?^+#7VvfOrrCn*R0Fh6WS$yKFjvq?X_}Gtk|CHx%yG&
z`9}j)PDi~I6WPnv%L>KAdzJ4e8?ru`@*->Tj3(X*Lem@`+&X#r`le4!T}SL?CpFw!
zxI=?=<&oEcW%;c_JVzr^|F2{z?lQ=!Nle{0ja%zMgT^oWD`&5;@bHAF9@)~je}Ra%
zow)d9v4(_Y9SM03HjL{tA9BfSE|a?`;uUG}ZXwsjd!dEPN-SdTe_#3b+u7W|r_SCz
z*7NJn+a9@hRlU!j{kyMbpI009HS{?5{uM9FexAQ_Q$p3tvUuTxYa3h^`n=d3F?prU
z``z<iB>X=jS(u@FFrhn)t2NVqRm=HSP2MPxugfn_vY&Iu+PG_Dfb733-&Pra$&7qC
zS)$+!->k`JqdY1-=JRF!7tAtYj^1FJrREy_(%YIt_oLQcZymu{wdap(mp|TpHA{7V
z*6K1(@$FfQGm502m)DwCELC{Yc6?g%^>zQ%^o!Q2b(&0N@S7l1tzmETH09{sVyTxW
zErcSDPwfuVV7}#$x|HX<$ihj6`z@EHd^VlW@a@~3!{4|sg<iShTJ-4Jsk@dhPq}r+
zm~ox^A?qMsV)M}7e&U<=Msia+G;MeBdEUP$yY#L1`nSc~<WkO^z1t#upY2`wy7X$_
zys+&;QRmM6ja#&F`nwl#mEKdCESrsY+%PWEoci)gk?rxdyA&3En8!VRcj?X#uXU~R
z&+QfGIXCx3Uh3($4-&8cUUIp#v~%6k#pf@dxNd4>vU$J6s@D+{{ZH4GO)T57&_Mg*
z<HwQnO84|Ayy8p0^t}7~E|<4U^$LHvJUMlCk_FpqdHo8>|H=P1_I;dt|L^ZFf1Cb)
zO!)q>)x6H~-D6YHf9t+oYFYmH+56-!|01>J9~F;Bug?4T-2ao}^~Xjx8Z=jU^)A?D
zd~msK>1(ayPx{L3okEo+sj!-)7jNS8-x4p|8Fa(cysLxjkNI?^Q>+|%Oe|9*Gd?IB
zYGJu664DW%Fx^Oyb&KJRt}6~<60%$g$rGj??UnT0D5-it#hr0B-?oE0QWa+QZsXgw
z%9LwqLmgY`uhWs|@1I}b(Ed$Y>&uG3_^&y-Ya=ILIl5xWtpX+GhBIQTqd28>UToFA
zzCLLuUyj8F@5l|B*Ei*AGvqq$$a{WiZT5G)dwV#x`$T9ixOTtsYgBHMGnev-|1F(@
zO)NLP)*m`*@_Eusr3a}_XP9)PxBR-W;6Tel)}Cu?A|JlE$jrL0rNY28g>k7CS4GZ+
zvv1d>mQ8=)<;1|Lx1YPH!E!=b&Nel}M=}~lPt{g$yCs<uzFKzX)PGZc-Tqg+d+)n_
zZ+<e_*ZtA2-gaemdFtt@v%@~#KBE`f<|7%jS5-{CcEjOF2EHq^9Sa0C95{<)mpo`-
zn<{Xtnai+aL5%1*)5K`8c~fJ&B#yN$Y?oP9^1zYB$RaT{+Qv=Rcb@8Y3GL3tCl`B|
z8d9dZNtUKO4()P%CQy5GZdpL~!j;eD)!o;%&6>2}t9y!<w1mpjtaY+emN?23s3ke^
zY<wE(@_(VNo3z`K10I)()qlRY=+HLtQcU!0nPrn0mZX$l@G-k(!WQJNnR-6x?TsK|
z?X@CYS3}AkT~`i^w)7HPk~Z_`!X;UXCM%cng`P@{nj5!DVA(u55v?V;rj=S*oD*N&
zU+U#G<<%rj)0G#*f+V86leRL1scCCWHCenqt<1!$x9@7%8;5?)Y{C9*icfNiv~yox
zm=F+kTFd?FrKPh{gSVJX*v1t2aN6c|5-oFfay`6uEHCxTH@%4cKc6ct=bE~5nRBLG
zh??YD1J;O5Z?|mpl-{;S{C>^1o9*`(En)hR(#*`L@t{dfEMif!h}sXa)hhec48=v<
zdlXu@a(gbElu_PcbnHK8C6E3d<>Wt~nwYazY&yjmwddNo{M?v_=WRFNIePBB){4(9
zn${T#msyo+KX&qpsW67Hb9xERshE0G=z6qzRp4B?i#rzIurVtO=rLP1H^5aq^IJ-N
z%!%6TYZgd`sO$(b^Hg=Oiq?GX`1shatmxDYvxR0X5e*6SX5;L>FsD@3?UF;J@|8J>
zC4EK98H{<Y0(6(;OcY(bG}<&QD!wvm<#E1)a|+MsT#x?oX6x-b>vuc37r5--W)arC
zKU^^SXX5@@mR=uE$j93(3bMR!^W~g8_gnWfYT33<UzUAY>lE=e_4~Jm<6N`0vsYTT
zX3e^Kc4DOH%-eOjI}}sqhyPBUBgXzeW{1a*H^0`EuAj?tbWy`rM=luwPjQwh2Y3wy
z6h(s+Sa=s5n8~@oV(-th&7Zz6;44UMmhV2@>hwZPz%xT{@y?mf0XwD%yj<7H+4&$^
zX@&yB*Jbu8GaPcG79N(pr5vmDP+Z`(h_K?Vl^vz0`g0~^%s0|BwEwbFxJd4?pyo9;
zx2xVB+I!g7?e0udnDgVokDetedNa}#3bMKHi!|Bk6)+fY2vq)Yqe-Sza$SLdA@|o;
zDf(MfpA@nsPJX&>tI$(MBWs4&F`KdiyL1cM>MJb+UdMe_=yd5w5!#tD<J1LK_pVbZ
z*@~j->DyNZMP^M|xFlNUe#Zp$m6LQ|Klad0ZQJz!_r@G9VTD%hKTA(LZcXCu{jH?w
z6}as1k(L={tGq(jHTf{AOr9B)`Aob~^UQPIZr#c?K|gLR()IhB+np)H@k>U@V)qQ~
z*yCF2uDe_WH&19x`&6}*TXwm7yUz0RuRHh>N^Z32<}8@2{kr!=M&=@qq7RM-A1+`O
z3<`MYl4AZ(rSah_W&0m3T}cyJd{y<;b~l)1Ecl<2@W#sOboZ$~7th71=S-GtX5;f+
zV)y%nce2-(%@rc)#=V-So?LRAEZEz^xadNCnA_ocqmsCo8BUT>j`Npli8eUx>6+`a
z{7kx?mqOUatp1+GqRXc-b*0=6>^Zk=@z05cp0}5D{|b6JfBKI97kh)NvZF16ujM6e
zZi?hOJNcQ1xPI*>W8uJQ8xEu`er}<*;kVH1gxe{T9JEp_t9tsn1S$nwjf70r2l1+;
zUf*;zVbkUXvpS@`8m0GLVozi$Xx4t0aIRw0wyo=Gb<f**1-0`S&P?f0Gfr%h)V1m=
zId<~2xV7@x7ooX&UURQ~KgRBMXDZtqj)!MNb}gNCvB$We!&-b33&Z@D9P=%^6+8a2
zxwMr(^RM3ZSlmx$_p}*b-eq0A`JBx|P{wG(#uZMXCnQzfHr-ngR`-puuel|-?O$1`
z!hhYyun(8xcWp3mRw_y0bkwk$bJgdNz%}pY#Mckzr){(Nqv$btQPhOn&z^7lZ}8Bi
zvw3#3eihr%IG)gg-3uNymK;*hTDP&-M)>L`pQWoiXDK}`F<Ex2gKz(x)vr^pbgfmH
z{Om-|ncL~_l<w+o5j=UNH@S#S=9G0&V`2N=N7J9~)IYBJ%wXDQZl|zKlUz6ph0mS3
z951KL`fJ~-Evu?I8m%f;-Tks%ar=*VD|TGCzk1O(KHitSwOe0zwOI&FSupFgrsvI*
zp10SXTU_<!&4Wzw04>eUGh2JQVtXb?_7~|IZ`<hPH&@_}(kj(J!>KVVk3NZ1{=O#0
z>7C;e2EQ+@!HQQ?x$`bW3jQo)wan$v-Cd!a+03H8y6WiM|1Xl%>u)}uVEKeQWBcmZ
zI~f8NRmc7(oG6<Wz?ZjMVS(-kho)eGD?-axck>lUK4^Z&zpiAnCG)Drh%lq+E5D>^
z<z{*Y-g>#rFd&6xd-=Y(Z9!+xeOmhU+;;b5v8_u_PHFle<rf|!EESv<I`f#Mp~>ln
zX#HnpPwJbySMdftUbk+Jvg4ZbL90EK!!~Gk=lm7Be@kQgPR;POVOrV~-Gb*ldg04(
ze~$O|uFap#SGg6~Jhz+>bV$H_MtJCI&k`%v*G0!4v}{NS)7kf8iolt>ueB!no>*|g
z;CYY?kA3Qn)}4n!rnSv#T|do~_0H)awyQ$b@3@z3X84zLGic8t(X@^!ZNlq+9KEi(
zWPVg^{93)rBf(KF3*Agt|6e&X_JWLM^vSi;m+>00YCa4w*}LtX!?B;Qn|-EyWsc0X
z+WmUn>UYQYOrOe|eetxIY__P1_eQD2yq#~uVm@u*I$X={`npSqC*n4n7Dvs6qP$Cg
z)GYP}zE!z<_HEm@?c4(1i7mNOb)uU>7K_{cKUk~v@6+b_hn}tf@blq$v9G7+D|7cO
z%y}F2JYd&}BeMP{{*@g2Vd*kS#xuM&uy?6UALjw4vh$8p4$TPZO?+`7YlrCC7kADz
z`v0prHo-<Rv*!NR8&O^w2h%uQ?rh)^?+Ko~CN9>%MLJ}z^Pf;F&vQ0sFW+$R>*HaW
zCCf5PSImvaXZGHijy2~*Y>q8>5+u#RQp53o7Q+oj#|a0U6wlq8dFam0LqcNm+a9>P
zESa{`mBY1`bD8bEO|iG06nZYn^vY?uH)&_s@`L9NE$oxH)VlcYP02}fWVgAVTIw=0
zV_wb{$DcbciLpLPYnXS};F8%(sjNBk;v{BE8}$9Dm|4>y_ICzr)6&@m20WPu65dUb
zjpMm!%j(8AvFXFf+o2*YvNmkHZXOqJi$2}sCi%pF&%$fXA&VLpiiK;$JLfFa&GWeO
zWwu9U)M3V{23oytOMMRYh8)=G^Otk+YnjEKCnv?82{X7SeYSunO67vjUOtD1JlAGk
zH@Ycik$38;uV`NH<F19bralont?0sRc7&-+)aH@guHHG1dHxI99Gi2yf_tj!%?rKv
zM79XB{W^Z1=gHk2ZI?Jx-u_7DTBeZcAp0XF!TC|^w3njRjjob8jB=^hU8h7Q_Rj3T
zvS7m(uN!X{oaPh@lTdV8a`%?(8R1P&9L~68?2T6IO}sO6(YDOU@EeRZJDOxW#3D0p
zhI=IcF_a68iU>T{YBuSp%bl5OyIL17<%>v($eyWeVCk<J;&y&*>o1d~oVG0M65CQI
zJcx3+9(b0Ed)4L53D;luUdy?{-_#@CP#Gb)=W^keY46@fY&m;5Y->|BuZwle5tafc
z=@ZgAQt{3=*mj3r@SDq{Vj5rXD(`sYk+z%K{Ik*FHMdNb-Tysven{U1hN%BD^tZ(~
zubjXCq)%p7+7I2w=Pz7fx_TkhCQ`h`?dsS0441A=z3?pmZlcw$`*+qnuJ?^n_u&d%
z=gMq#nR#1d(8;qeas!`-F1S>gFnR4%Nlqmvk=WQA&+Qg1ObwIo`%GW2di+YEkQ!rL
z?6l|oXM!JFw2L-A*|$K_^VI?`8MdO7c8?0b=NebN@}|2)Ec{b4bALv3%DzVFFLU!+
zo<Cfu7{`&S*{UcRB=^=L^@(X5&mpfVtWvFd;?q(y!fsu47j605b2NO8U&NOa9laW|
zbzB{jRxlK$t~%>?>gmGATbwkfsIJSL;ZVS)#O2WUDylDQdEYeE6o(|MhsPpv!Z+>Y
zcK`FAt@dPZsLgG)V{SfMnyPpv?C)5lKT*l?%@P-u;|XnIol_bmXS6m2oC(m(keTGB
z`qGE}>y`68XIy`Z7#b=6Fb%jgXNu7-*EwFwXKrN1v4ogj(!6qjn`6__l?O8xHqGhO
zot$@%l}RnI;Ny9wYZ0f9D)&AOV_GG{{7lS^LHJWH!><ijQyqND(mW?K%08&Me{myg
z8DEmqmlrH8mpS{mV|)8KtXigj@YmBi=Mv*TS0v0`NOHZEd$MZxhABs8OSDv95?gd-
zu@jFMW2~#ltw<rU`B{!nO9juZdU12f%bPmuj;WlKJuLgwXqWtgZS!{&aqW4MwKC_<
zu|&7dIH^3V7cXD@=kJ)>y1PnUXXlG_fhUt@EWVS&*efTx`rNFxXReaDZpD7y{aKTh
zp1JPaBKLY@^zI6-&MVWq71(FnWX3)7wsmri=Q<D{rOCPDuHy;s6t~qU<&I3A!I3S*
zF=^T<9ig*HUP6<!qPKs$Id|IYU!fWj6VnXamKi?F@md?uV!Jr)V!Yv@`<Yf3cnXhl
z1|5sp@FH*CVrLO`rc<p2X|D>Ro?V%xe0`p>k#6kNje^mYTFMV6I0T%uv%B8*$+!2=
z#GgJ5>uw6U?|HR))&h%*UU54XTq&BC&Lb(`E82hJGXK>C`KKInRI;DWIl^z$?p1N@
zYDFKP=Rxl&QIRcTjk7e<*Yd0J{%?_dGkYHIiY(3*j~wKix)weS@{%x%TDC+nWr3Ht
z?+o>`cbgX+-MwzbzYDJ&b692Wy<2*BUD%0d(z&ktt}To{5_EkRYZ%L$+t;|GOTAMg
zj~(ig`zqBE!Pml)!?yltTbyL{8^s>ym1|-{<6SD;Bx`cHJ6V#Sa!d9&Y5%#W$a{({
zL4{}6o83Cg3hsozIlwbRZ`QU;IU>`TRB|WnnY+GBm&KKF*4Yoz%>2pHyEHEZ#rS1L
zsAT5lZ1FR@7m~9_!|L5SOTV?zF>y9!O+7awBvlfeW9D5=%PN$b;ke{cjLYRqx_65w
z&bxTm;M2a)Rfpax_;ug;w}Ih7z)Q^|tAF(}ocRB0&C;oHcd{h+xj6=$PnJ0S^~b9O
z&qC2=$?LsO-Z8G`n{+Vg%8}Y_P1|3rbFlmv`tj!9HzKNcFW!55%thwJYMqA<C2y^G
z6C3_g>C4l(&+pxAJ1BOk>DkV*Wv|!xJfAf;PxjOQy%U!vu5jI!m$zxvw*w14u4{Ez
za8!J1ljIeN*HN)@QKARxia6FON?h$ta}c*sd?e!Y-jV0yyEU7H7&PN{_`i{Cu`Ct;
z>v1c;;>Oj#6JwqD9(GASamkGTw3M$>OS0mab>x&Cnu^<=tYdqq|GDs}klvA$eXcsw
z4zcZvZYo$=XD0tnbi<y=E0<O+lxll+&MH}Q_sc^wpQW|Ez9_bqW&Z#BYc$TX2|bWH
z^kNm`0^8@G7rD)--D2=_Vz$$P4cyCF>}KCOmw3`)#@pLoo!j1QDBk)i?S!t=icO9w
z@3juU$~w|3WwrF~S5dLNxad0%<azzQ6y6o@QJ;H)(>0-GR;A&ITk0QnG;L>i>bLOa
zGcCX8TTaivd%V2x{>g2pWkqhd3m=h|h;y0Z<iA_#PHd3omy|t{vs_$Wt4!{C=4X^s
z`lVHD&FV9H9lWceYPnrXl5-cW==8e!PEY*WC*$QcfsQ&4*q>)lE-08<v+2M-(Fv<2
zW!vP(#=QS3)gr)bCir<)7(Yv6)y&FM?vV`1yBH-_Tdd^HUGtD_^K8~ricf`<4gH*b
z{|D{vS;H93TJzwzpOl%S#{Bx0GM%<F$KPa076jB^y<Yd>Nv=+mxPZjDMQZK|X{^<y
zomVos-<IvocxT-A^6drF8{0BCZcD_lsCN}My`40tczN1Vzg0G62Gi%<ay#>(cVCv1
z!<UFK6M@Z}7*g~rceje{(ONO-VvK3=L9e?Fo<+wzS!ydIUH@bqd-fpWrT6+JC&jh-
zV<TFx<@WvG(xlyUuC^^KMQfq7$(4Zl?{zQm-`T5>wUbFCE>Y5<bkEJ>K}FN+w|tEL
z5IW!W^_9+R<+-2UvTQm3DrfSN9S+=|F7>_ktksTq|H=8owY<>N|1>}S-^wMvSM5(?
z>0TKVj?L!oG5<vq-NOaWzYVmMOuy3gjqgbO@*^xiO10TPI;uE0c{niK$vu3@Rrhp%
z()D8-I6u5hm^1UROXb^+?vyu<8-7OZ*m+vArRr^<{8oKwp3m{yn8ZCCb+oQ#3f>P6
z-qAGKaeC7=!^cVCg7fv#G`HWJdBWnE-?49NjpmpCoyK%O#PNn++AOZttYvdHeKz>?
zGL6^3WS{ZrJ!@7ne_ww7K9m27BsCY?@(qQLj}~m=KYm4#OY}KoQkGZE+w13kt#~UF
ze?#c5Q`_pk?NgKAyXsd*Yc$@i)0(>BSlhM*4_mn8<720vRd7#u^Wf8`m>HbKOTKLQ
z=22y*=jM_(=O&Z##75nWiamk<e=z#H?4G|~`@;EZm4!Jsn7({falUgf`AzID7wy*%
zm#j56)QSAi{LC!h_{Gk!3_fEc8Rzu=;)@e%x+9dA^q1d>=y<XCg10uO(L0Iq82wdY
z%sT5`uTRNuyD1f07G)CM@%^30fikVeGs#<+5=xvj7$eJmMpfwFo=~=1;U%l(w;z)X
zQ<ksS=gyA1<(XK*SGu8tZ)L2r$}HVow`)r@7r#H|CnQmy`&azkvEtdYW87ERuY7qq
z=J92Xnw92%6gJnks&W)&$eh2$x8bsDN%x!+Q}3Hfec^r_w{(}mFQ?gZA5WjFTy=Mr
ze|)-S+ysTG#{VAvkD26Y9k*ja<m89z{P^7?{$Dh;JmO}@{b|j5JN6#}yopMZvYLC^
z6SJeUH17xKvjk55;qI^VB=L3ABF!DEywvt?+};$nvf)qqyET_)-PfKgkS?}AQ0l+s
zcH`(f@0At@Nf=7zZ(M!wy7t%4({As!+Lq829lTcK^0br7KTT2oIXS=Wv@**%ewH)*
zEYnuoe_OZTxy3&5yz6ZTt|FU{yRN(u3qJB}d#BR@O9u`G<%R^uMm9k%g{1)pG}`#Y
zrFuLz@Hz&im`%(~U7euIxHf9y#6=f-S5L}2C7_eM$>+3?7WcHu&JEs9j7Gv`GcyfT
z56&`Wo$^3w%8GLfEWBUnBp<ut%V@Q8i_gzvSI^C5TVJGDA^KnA`r26TsJC2Z4@0LK
zSKl*PZT9A7lWdON-!6gY*EXa)R%)3Y(h$}hZ6WdHwYi@9Au~(GusH^W2M<?X=E|63
zX5_TLQ+T@8<Q>6|XJ>w%_G!-al1#=$oQuRRSZ+<>3syT2X4JSqgfTcze2Z>FSq$fr
zsUkdj4vqz?T$4B~{616^sR}J~Q9se@{a~Nj_8D#+mr70<MJ=uq37DubyXgt*hD&|{
zwLkZ)QOncPn6L2nC--d~pE-qz)_pSBH#<ab-I7?bbjCIB7aK2KOEOqx6j0=RaGKww
z3!+oE6g}pf?6sO_*;gr_0GF0ELWKuki)`W4IPK80((2MCWnS~9BpnUw59|Ij_xA{M
zOLZ)eb$2m1p~xNWv#Ci)dWGTfoJ(0JS$57y;?`7~k-I~)&6wkKOI5eFWz#+v4UXBz
zOgtK2Y@5;KK3i|iim4yZ2_IMzcIUzy2O$BY00VhJqy7_$>Rs6j9R#B^l>guQ*r71N
z<xvBVO{Ug~70X*DWgp$##`Vg1myGtSrAxHFcCMYwv5G6$I6}*L%Y;Q$o3Ffolr=$L
zD3|Nw`RL?a?(i7<w~M%CJeA*UG<B^J@{H*<?U;Oa&*IZ3FE5xnMd3`Cbg#m-1){#U
zV++^%nQ`{aZt&9Le)VZP%Vrh7SjI^K4LiyXmP}vpDCu@W*MS`ik_;HLdIW7RDcEc<
z`LOT5ceY|Ucb31B<I&&<+ZNHRei0|`q7!OHZh`!^Modp%EROQHx!_oo=~08F{<pOj
zFPoPpbxMCxQ`f1~h<q;oWz)>wtX7Iv%6=(!afijrgVA$#9X@<X&o#<5)Ag+8!6mb&
zX}{VL)?#&O#p<-Etd)WP&h6UawoTAW?f)z86SK^h{>(92;i0iqwEtOH-i&EJI!jyo
z#Lng^uD4sUbzjQrO<qr1lrsvRO|jmtz$qjUc(0VXV$Tkf#VTg&xxJ@;P+mDf=t1_>
z6C3mvnZ-N}cClx5ko|f|tx03O!%687<>*(6CzL1Ln$Xm|;>3)TD}*LAt!fDh&|3I?
zfgJDpAL5-wH-F47``^DZGGr(Bs_rQ}yiYC>pZjP6^IJ}>ms=*=d^z=N(~7iPfA%rD
zX{&v|Tcq`G(q}Ef+N~!hY5tGg?!_6=6xXb@QTCb3A)zx9mzC;Vy5if&QIgQ4^;mhY
ztG1$DR9Loz#(tK6y-hAbyoZH03FvXWvp5_W<K(wvo`I^0kG6;d(*j{>J%%)^YY*dY
zcnB}t@iE0J<-^6<|E6xRIIC;BB;l}zM%etH916;hXGYW-s4@JPaB&FAVm`erS#)OT
zD<`wbH_MNFU$8G=g=*BxMXTN{*cm<Jp{B+{AITL-ZS0c0Y34hdHMyQ~g*$lkue)(U
z_l|<%DU}m-w>Dn=YmqvA(}SlIT(10Ikh$yF-q?o|;*)0ksW!5z=zTcp{@-}P7Ol;P
z_o$qnc*;slz52)mR=Y_#yd^w2A_^9j7nganF^W6-NiZ<J*ksZ>Timhaq<48|cM+3{
zuj_@+%cfk&v|%@z7a7E~()ZT}5yzNVP049T95?py1czL4zcq39fh#L34TQNk<(6nU
zGQF)9bzJPJSF*4%XFX3zR;!i1iqzzih0LZ0FMehD&=$nunxQc{c<!-s#dDkuArn+2
z%@j{*cl_gX-?1?!#%88+ubYYE3`?H08C!LNa@?l2N~t>8bzI6RG;v(g5j3?+__D&z
z3ks=14Zf9{%bHXa&5T?E_ovNJc=y9&O8gTRqyI-2&+XWfv{E=SD|5y#m!^m}r4r1K
zLJYt8y5=qs&AhZ&I;Bkb?8!BLf=Qm9w-VgUCoEK{&T-bvDfHiwrIw+V_*(LGLAUL#
zHxp;9zAPPeiPd~YWABPx$qJ_G`x{)3WwD){by;Dz>1tau{obomsfM?gERPM#p7m+g
z9SzA_VO%W<b%D`6Cjyi&&k{4LNzR-wg=OttRn4hOOrEM1u8w?lMvajxpn;{1d)eGa
zMK1lb#Th;c?PPITUL-py*wG+;f>j}(Kl9Hy!52<$yv$N)d)=p_{CCR4NpVL?D?8^p
zrKR(0-s4k=zQGXVTonG|*+xIn!o#z}LeIYxdOt7Furp1^>3>_$4aVX^UFK-XDB(w~
zpHy^idMTV{n9iIS*qoodH1+jz?Oz{V>wnDlj`d1Qlv~z&ShX(4UQ}}Wsf?w1y3gbT
z0&1qq+!RnWyyf7S5wmSy$29}CH-=6|KbEH~Z8d1#BbgzzNQz;-ny!PPScb#ml<dHO
zQ!;l%OP2{;ek#&uqO0S<RO2Zyd!4}<HqQe&35(UF{+;Yy7HrsCvtvcmp3ZZ(0%thB
zn=|i<B8#273Tx=w<hUS%t%@tx9u;9!Y1pxBC2v(ooTSHb!O)Lytb4_lto~9l*Lc&_
z&_dmS==@EBd@n_-8_#q~DzrR&w`W^ez+}<pOXgb}ck(XIW9CZMo%-nf|4mi4=7*NJ
zg<9W|zj>s7&%LR+{7vQx>XUA+R_VzRxp4cKY2PyQr(u4)LYArle!kO_Ula#B&HK%@
zNpI2-;U!fkyuMGlbXLvx(NQOh-BmTgUAYDN_561xoBy3|Gh5|e+{W#;LVtd*RaZRh
zB_-n)+In8J;jP2wtiNfUQqfnpY_l+{pOigA#aCUH{p$N9v$>M2S7(_0jOh_eahN*&
z&kb4INx7b@@{XN|vJQ=VxvhscWyZ9CxVW^1yC)dmdBSx0%x)FFsR^5tKjb`I^e}nF
zff;Y-`hGIi<7mE7-F3q1o6nlkWd~0<+)|j|ox_wU7hK}8<U&UDkMK)Ne@HsooOS19
zJo#UATSsxEqKl24<B42BqvngpZkU|9Gi&N1AHhtmdy(god_PV+tMgcKzv=O5s=*q}
zqJQ^2)_h)T_}t`>pEhqtl&tiV9G&}DI;_v@nmb1}3!8Lgh2=}?$*32d`fHS_GJV3F
zq-cf{s=ppBIq;S>v(K61;e?h3_nGcbuNBX35iUP0t@5F2`)kf$;+n?}a7C{OXLyr)
zR87s`fvwRMQC9z}`~5G)+jQ#J8U0kbDG*n4XiZQ4lt}M(=Q|#2Y(&03XD(EJxcugw
z^i;MBnlJhp_yx2z-e^>{YNR-6^&D4Km{4Ap;$EcVv5vbzB*lPlaeT{K53`J$Y3sJO
z@u~%I?$tTu{C`Q$SF48^RY^_LdYW_cJah8$7O}*LC=`Wh+~QK=DDr<H#B!TSfpdYf
zZ(h%n7OqRT)_MjdxgR>KwysafOZohb<PM=@9TPfM@f@;~S;T!a;li>mk!=e^1=jvP
z=f`kqou~o3(=CVgszs_DT}M5Zt_bV(;5olANGJT-W4@10kBod&k2%$vocWZtOknb&
zgnb%<i=UWUBrvHdOj8itk)`?7=P)z>iO+m5{yt%Pq}3xdacN`5>ZYK~8)8YE&IX2J
zN>(+izZDsHM1A~Tb^gU8r|CitZM)tY2&__2QQ7#;TdU8VnUg8V>ZQRM=hC43*+PrB
zJ@e1kJyF`SzU<i=*%?_(|J_y<zd5wP!{yy9o$E$=k1IU19~=|by!fGPMOMS|dON-l
zIf05RhCi#?%UcdHtzK+(NBeA*7gwI9#iZt2XO?VFJMy_eMPYkFiiYR$Aisut>t8;5
zmeJ*-FpYJ|HRTf?oHq;nmWrt@wa9AebG{|yFnv#sPt1vBCFj*D`Yn?Vb4*>Qu$}pm
z1?wY@WbPV{Uu)joy>|H3vz&7~)+O!86gl8mcH*2G(-I!HrU@=AIvWJF<I?_Zb5iup
z|CF&#bLulqdH<}CXWBaFtE4{EEMn3ManlG{matA;(D;0`L?M$IgOL2i?R76(8}ecW
zoC+JfnR1*}D}A>Z2x>iElv}X6PvFwl|3?ZvrWtGtQWBY)l%J8SXy?3bLF#su22t-z
zJ}sTmYVN*yf{rsZ)!wasz{a(^V2yFH%K?w%Uk`Hc&e&+;Td26gOY*F@(w6Su&(4T>
zJT&LynL97@bAx)-js?tZP8D^>|K2FnIK?p6$Y1;CB7VXA%mP+Dv6Bqq{nDyO>(pE|
zZv|Q{{rn*1kd^X=p0wgu77OH#y2c!MI_(~-<sH^DPgu`PT2OdvlSK#j^e+=*Z?Ia$
zP4dWVG1&LauJ){g@QLGIeO4ykEdE~BO36wFpO*9-ea1D#B&*<*&=!*r=h=&nY@e`L
zdF|WQEsM{{884rp&0z7x%<t0PvNyM5yxZS6y;4$E_<#A3?}H`hM3$X%3hoeU;tE#h
z_HO1rdpJ}rjfeZt8>S_qDhD`u3%d_#b+?r02yWEhGwE~GvY@zULVq{>U+|3Ydh=rw
zzfWA80+U4qH;3q?rnQ~=BzVw7=<u5DEDeR{SldJ%96J+o>)MpUYm0sVPqNcpIZ0=t
zr&x>d(hwPSBR}s6UlJC_X03l=AQC=>+j5!04-H|PV{2!1wC}(7YAs)m`=-~ft6nQ;
zUEq3rNJQ8r*f(pHRFla8J%6#Jd(U`OPKl`LH;H^&%(OhSEXwh-MH`QWVpw|V5~iFB
zFLdlRj!K(%cW8Kr?D|-Htx&ewShmslrv*Eoa;e<z&DL*K&5jh?{;%ea>vGmpRp3%l
zu)Ji<sgw|Xb7}O}1jQYpTw4^lloYtOG^jJ@>K@LqII-D?|NIQDpooBThKJlDI9}LC
zW_ix(pPUwYS1sAdA;8*daf16|$L&iU<FmOFR>d|gP_EVFs95Q9O6PXZyTr|JeRd=W
zZ(}*@Cp7s}FZVhQRkyIn<$7BK#MBlusj)VtKFx8N_bP10s`EBm)3jG~&ge?MkszG%
zMLA0G$GHvfVwNU&wO-0v#^Z5NRN|`Wl@Go-jAmQCFN*X!pVAC&bJ6VIoSBxIZOSTL
z&FZ|zNJlVGYsVMs6^nJw8~I#ay1mHXmV1$nu1MRlf(v?0(UTpxmpE`uO5px~#+2)r
z12>BU_oc0(5evn(B=AHn6x-6T*Q@PYuI%?;Fz1<x;@7WgUy9mKO!RMdHB8Dk=zp3Q
z^}aE0k@^o8`(H`liVR9CGjG+bSBMI?KDR{FtC443v*m>+Vy@3}%ug;|<YZjaAvagl
z;7<3;w&_;e7*6^)o9x$i+~dvdtK4C+Ozh1;<!xStstLkN!V9Ap%022*<eFZ6d!fFH
zPw3l&VoN?&bDCFkCf!?NILBjQklNEPKhz#;C?8XLs$R3)LRvtx=JMYdChN7It!pyM
z&t0=lWR~CTqI>s}!X}kiAr-EOgL20lgl{Qxt;piJ(J1<7o9vbb*-c);m!>MzG|Fx(
zSNJb<;B-vKavo=IAtn1>%_411*B<ePq$k<mmgc0*u(A-4_<YEoz3jxC)9yK<2_3oi
zvKnlz+l?~}OsqN_=USX}YY{eUc&_B06n~<tQ-xuQgL0m8kxkgyq)aoF196w0n5L~L
zTGr6BG=W)QYHx4A*&i&wR&XfZb4i@!dA{j{>sKG!KRg$5oVeK^`5Lr|m@)3D3gHsC
zF5Y!QjA=>d5{<&##4^q{ku%#x-5Pl;o01=OiRx7<9}4H~+>&1x8h>zNP?3`7VKJTK
z)4W@g1TRN09VpSgIZ5G8nXUMb+(Onivx8zw9>j2#^E}HH$!e69X%g+R<hqnBVj;oF
zGF|l5!{7fS6fXq5@Oc_kte#o)(xlDwl>Q6<y1h4N`EN9tpgV`XDd3c){vDr|wDpFe
z)2GG1D0NNYW_ufOScl<2>&{(gEjXTA&SRDryb<+Ra>2z3(;CW>WnOascQG~IYBIa9
z`s}-jk&fKQHk3W<t!7^;_Uhma!&|)f#BFXZs0l9p{ps=Ih1Wvt4~Z?(&K5F=uwA$G
z^2Mc$7WetyJ=ifh^74;{gTb*WTDP|3#7WIK{jGAozsG!Dr)a@LD+Jknwk?o5))4Kv
zP|WRM&CLeU8wc`tmJ2IASZUhGvs-hKX;Z=W1+ll5Zl9&R>=tW=yT9<(NP}IvkA8jn
zoaMVVI*(;~VA$-X>UuBI{)=33%zxLha!u#;#Y!7rvh^Qkmk*o1;YL!bq2h$sO$8sk
z8~9!(CF?e>wmn%g?d%qx(^q1fl4E`xQCnSLE7x{Duc%L4RXz8b!U`@^{$omI)BLw^
zsWkc93c3DXQg|Utz3{K|zX{(mcXLg0mQvBUX}^QZ_RP!jMZ(_Ojizo7ni?r-@Q&3?
z#&BB2V%^TeI|_bIzY`YFXrOoO%eOn0idtrhOAd&+IdYpYuG+ewz_?B1)YTbV891Lv
z=Bu~yoGHy`p3ae&v43%I;jil35i=ALB(@YR^?Ukjt-V36oA8`M)oo#!l7gFgzojYl
zEG@}bK7H$^Q*z+O&rV!3vbeLv6|(}l{?E`Zl$v4Fs(tjK@$SwIWs41dZDHQyFnb}Z
zYS$d|sSCcZTz;J=clqiq56bioO;+yBbbOTQebDOJOEH$4d^wFa|FU<#NHM-Sk+J`)
z;{;P{?q}t?0TpkbMl#QuZ(5@(8?jmV5&QO@GxsN125~4l)ttWdC+35o&x1eao4%Sl
zI@x|R3q2ea=cQru^XLBm(hFXvR(SiXraw8B^=6WQYz|+tTh0kP{gaD??ry8O;d%YA
zNaK&$v3_+40VSc%ouZRH9BeoIl<LZ4Syp0@<XLoBT$<rnfXas3FXlOWvDFlIy^dWa
z$(5_r|L{p_>D}x5xlQxM6n7a+U}fq&*5F?uqM|Zu_5T^KG8azc`Ly_ibJLestNjI2
zrf!YR5jOO^IL%9;FgHHK_xRjfOOkgVm0ff!`taQ^y0x5o*Y93C|J0k!$a)DwTZh5@
z+amW>3wxXS)wVx4!1ZM946%1RR`C=lKjdC?O|s_8DK+b+w?c(2!oSx)-0H0Kb?c#r
zy=q6b^UHEX<2RmpV5w(zP;{F@bn@J3ot*^*JD=w$-Mjg;W%FVswnGxVg_VW(^1rQ-
z@Jyb~(VCPs&$RINn-^<VU;2>AsUt4#b!zQ#RkQORmg&o;hsw>pDwnH#uKsb$=F(ex
zLK${_jZsYyIrQuGPSNU(x2|!rZxUAWUgbOg{vMxfbDPe9&6}s5d%o}8|MzzHIhC09
z?8`s@_5EA1r?*?@v)cA@)-Ll@U&+yQ^6dfzsrrws4F@h|DVT>x-1?QJE%>>(FKS*-
zNAHtu%X-#XJDY4^jTSRGm6eybWX3lIv5otyRoVqij?X!(etTNRPn)&UVy&EK5>6am
z*wrWe<ml=?k&JJ_zRc<RNj5T(SzmPuw>xkzsa5c4&tTn=v-W<@;%eqILb_{zvwyE<
z`eGgZI`;la`~1Ys(evl$Z(ksL;=%6aZ=W8tJ|moR_VS%~AMK*Qiv9l_oBz|Ei^0iD
zL^p_o%VSe2*L2;K9a9u9FPkzujEUt&QR&5L4bkhg?j6*4kbHEaie}Q2L#d}crm93v
zs`+m@iTg}fblSEpox*9pvy{uZCU6F;P3=l;m{#eTmJ&3>n^op4r|GIIEA=foUoe@j
zyt*uADwl%lf)MEqq9?gFREgxQ+?=Rg6u3yip^2eT_y!9@Fw;t-0@fp<0u0NTRIS<f
z`xsryb3JM%tm3J#RPAV2?utn%#=@UN&ZeF2d&3jF`BLE8SD9-}H-+ui4LUI)>&FI`
znMSEz7Anq%6sO<Ou*~^rn|O*vQ;5Yw;a<2S6KglGez?=5pN~|kZf0}}s4{8Bui21&
z^N7@MiLf<qI6I7%DOrcj%4%E|Cs<YU(yNPGAx>}~%imwUzwhVE*9fXK-fdtNnjzbC
z!*$|ahglg5|DTjt>~Z2$)tf6p{S0ez5~t~w^?XTmRXq3QVTaPA6n0lT&J9YT(jTT=
z7ISC}zo^C?x`okOKkUhEfAfD&F8kRpGMVfX7Fx0-#8b`ka<E_IHx(a0yQ?StT_QtO
zCo_d)3Qvzdw<XLqaof|#nE|(sczT^u6*v-;^Qa}<HE7DlPNoh;HHKvN&?OBb#y7uD
zDlt|~5MSu|BU93KGUtL99#ccNWTf^6nO$0xGr{SUhH`F`@bbkfS5IY2?K|e>Dis>#
z?aT8rOoY?O?;ztEVV?yIYqxwld83HaV}o|#ZZRvNc>!Oux^1@_tY~F8w&=K*_L?h;
zPHKGJ5YDZgyd{`lQ=;$E|26fUp+~fLbhW<D^}8ZlwNB*ay4~}4F>x+A>JZhl&~w?E
z36FAg#kn4E`~2nju-<;FN8?O4soIh(ciGS_8yL9f|C-dPvNK3B!0_gipo!^uI-%Wp
z8!avc81?2no$M|<^~y=L>Z66H^`38035`-?6+3I(_*CVX@P(tB=cyd^6jl#55Q~`C
z_)E2+#`E^cjmZ-xZd}1IqtorH$0FCMAwl8ptkagMG^Sn7{;@6D+gf$2Q(N5$&k1ub
zw%UD*m{(xUTh20T%cjm`A(w;C>%Atc#P5Fl#oA3I;pOV1Rm`zJ)~E?`1e{ir;5zX3
z?pu}$frevqrE@m0n*FGJvB13JO|ziiiT}El;@6rJ^}l|3?f-`9hjI1WE#W1*-+s+`
z__bpncfcZsyagOKeusA_PW7Kt<Ta_sHZt?0w~lN4r=wT;-_(4Oi~qCn^p(RhA!pp5
z?fG;v%BpnV8Rg2CPou(mEk$qW-#&7brQoEf!^O&NFU@?T4~Uu{eWA}`q3KsRgIQy-
zuA4#It$Tm8f>upBqms2asd?ERHHK$_?~JD~i$>~rv~fuJ1g=yPS?_mazVJUThuv1;
zyU#6cO}V<b;AP@bjV%SsZ4a_U9P4b#(qcJsY^QqM)g)Kz6{d#+uPBv9De%_u?673m
z$Q`R+a@~O~Xu^pz+EI6u&TbAfpTslwWxS~1WA7s``gQ;JEEHvrp5dXS=D4cnh1gPw
zOUL$ocd#k*Nxr)}?n{@`l!**RUIB{tnfXr|t}#eei+Y%#Jn_ktjV(_DU;Os+xfdkP
z@Zm!``$MKhEK4O8xIWMDU3^YW%lYKb+_g%pr_8!~Xq(8-i7HK^ip-fl%sa1aQO>>p
zQbF^W16P>GK?e>Y50)(f!fy&*#6>&sTSpmMF!n5zJ9kLh^G%Zc{JWl9=PwDoHaXaJ
zSM#FN7pcR>HydZFdA+b<y&3&NWQhWsD`VWU$pWo69J{Y3cJwIkjD5G@lmDDQ-SdrK
zwwJa|7jD_-ve2Ti^?i(En`rI%M#(uR#mW|GO-L+R9J65JQL#(PO022>)ny8}13a#|
z&sgHQG+pcPPaaQ?8D1}Ib536Jh|}^g>|G+_-+15q7)yx3+b?Ol%17DWP2a!sMZSvT
zgacwXz6mkyb1^*B^k@OgSKa*rt6F_0CM?$ASUEjl!lZ<&6CxyAHF+knG@o8A8KtPf
zaD9f>^q#0B;pmH8^;5Ehmu^&)WLu;g^kJQn_mc^2IS2bb%4E-4bHQn=i3wZc)8lH4
zMoKMjxXv{-6gxfNnY3ro1eQuA#x2ZE;$obeS}Su8wLMuWmc?QvR3~Jz&DbzGB*#0t
z{$(lmQiBMOw4(M$wJ(&)4xBg~BN)@J_-4C6b==oQO2YGBeC+Eu(pj?b`SItgbS7N=
zUz_mjLd%J@5tr1=ls&_f+JrQZEY&L#d|JDWVgCLJLYbEiY8`(NCgZ)=$Yf?>O8rtM
zvB<vZ`{N6JQ)hLDZ{bv6T02v*aogs3nUO8BzFUrNF*>r6Lq9NHaDvO$tVdGOTbQ~y
zU7{`>*!tjviAEf^RBZRtmkJ!64@<HmTw607`FRpM$~H`veOs5a`_8_UZFkqodRBcD
zis$g*PR+>sAQN^+gZuo#bp_5-SLQqw+PR1)dt+-y^P#=ZlX%?DI%gg;=}LRecrr;m
zRWP7#^RrnCww$o|`NAi^YH3#jtM)VFC*{SHjP);uERVbNa>gz9^cfSBRDTx+|MPH?
zwSRarJy4Y4KjYHPb6ELK>$@5&E)om9=5l_M&i73Z*t8d2J6*%IHsN4TjBb*kP!<<w
z-KL{zY$E5<_L)U{Dlos^rLmS*{i1E{RVUMnYC0!_zQyotO4^eY;TO2!Tjcpg?2}j%
z-0Gh7op5gJ%4yZfTCsplBY)0f=jexRW+Ky-^Jets-D-%LX>*iUSVPnOmO}c3R?DTL
zmc>4QYSpZjjXV<1WyF+U3Z0{2wCo|@_d~kLn~k<SJg?&Et#b8{Sa+kV#p#PF)vOE+
zc5l;FynD|DO-_hsePy6ta@$E;<L0Tef38op?A+Nf(aAeR{)g(;fDJ}(Cz`#z;Up|n
zaZMwKQS7wjM4l*vB-76S`#QZhKC?J%*^#uMS>2H{%R$IOwl$<ZS+;v){AT6d_9yx#
z#}(W@!SH(dZC0<{Tgzvw6wT+FRJ*ZL)^*LPln=h1o(y|_-CedMwSbjf<%7s!|HS9!
zZ(n@9az}XY<P@gAaxcAfx&rI0zpGx-T{*Qy<-^NOtdoK~Ey^Y9es#HPuBhkJ@~N=0
zo4c>>i^atG{>4SV8D}r8^C}1y=ql5*&b@T$x{2ZXpu9)NVv57%W{WJI_x@^c>#e}d
zsi)L7vHcCa;#0S_pr0XQEo*eoA^(&J-}S88TAEv)1$QL%&S$$~>2X|MYhvBn{kQyt
z?#KoGzkAl~-$C7!`))$t6e0{9V`4V^XN-Bkw#AUKHJnX|A%A-So8_fE;RCEviR`9<
zY$YkGb1o@~9Ca^MYbrBK=e=(8%S=?O*+}g~lYVkxM6%~oRnH~v)!YjiW=>&XFenOM
zT4dZ<lfKA&?Z+Y==VI3g(=}V-gC1rTPHmYclAzCM=IQ9Uj?wh&QH`|+*a91TSu13M
z+cng;Sst9EZO|z3RZu>GBlOru^Mi|6C6%mI9IdZEE4y{9EKsnW-=&=~R50ZVf9(tY
zZ1wgXpUZY$<-fC$aq`E2jhEP#Ut(LE!2bCfS1N-xQ-Gw5Lu%>;scje7s;&g8d`vsP
zSm@9}B?j{}E5XWTJCwv-3%Ea~nKuckC|B{O{}+8(tfDGm<{T>c=CkU%VzIUm)%`o{
zc3XsmZW23sqp7z&q$sh?&p6U%hx)0uOwH+;i&LC93>}YG7%!ccWm#C8|I}-Jn!(ab
zwM#y<p8wHvX`=H<v!at`_0tYIAO9HjC&lfLdV|HmdgG7nArI?=KDPf0;Q#-_=$4s5
z|0RP%%-V^c1h*^LFl}IrexN0=aDt2hV^|q?%7qDu8%4^V8EO_bT~cPy3ha|-Zjw?i
z(3ObYx<xqlgYA>aMl0MsR<-w*T=NLNq>|%km*H65^-S!@!XEa8GWHhn{7cLyIcgS!
zrmPhyo>y4>EVP9+P4I0({i=&>K@5_r*ZSC&^i3AE_&M>vL&SHXYz_6dhqyAtMem4p
z6q^fWO}8$W2uR=AUh=)2<tl%n#q`!2{J%DFT{@=!XGYoP$%;!7TKPUt{16tC8^9K3
zHbZ5hAjb*TsTQoQ=7qiK%Gy^u9hpS5E-KB~DKy1BO@5_&v_j>%trHh;s@OiPj47Bk
zXS(oxVdLOJ72k&{V$OCB7diF5swQ0u-o{xSy=b=oK~8>k4}H^&luyA&W>)V!Zur5O
zE!aTfTeA8gcMW~Unw%tw&&le)FN%mfsYw*A4HVS8P#(6hNMCx1<LwCV{G+--hik7c
zVs$v&mtrA&>RL~Gq9mILt7UogR@U|(GZc#>`Tv*nC$?Ds<23l^@SoqJt^VJQ{;wDK
zzbG*L*uwSt%d|V+a_@*)^L!LyPo5ZUAei?-dao&?bO0Aur-;H<*4`C@yp5C2rFP4|
zoH*5-^>*<j`NI>>Pc_^yBSW$@mZe?TL~^oDXv&t3bl*fn&h1m;4?DU~R7p@&{(8~r
z)6z_3MUAkXhCja6th_SCZN}2z$xB(TE#1Cj%A!eL>kqn#CYnvS#1#^$XCY{j>LFzP
zW2*5&lk6YnzXcg*Eo%SGDO`E0ozc0xrX!{C*7BMc?X_G6tx+r1C~&QQu%Oz5do_cB
zw}@k{1=l5|j%>-4>0SN*1Om-AHt@bIPcg7@Ewt&16g>Vo+U*c)=Al)6>B=c-{|odV
zs;`<MpvNV8d$ZCK2~pM1Ap1|1H<nhqrWtk}Q3*c8xlJ;7BfIEYPSpb%)wvI=?5o72
z8)QnHGlO1<?-lR;w{dN4aMHZAum_Q0m#@bjT%ss$A|%n`nDWpu!BFf0NA|LyhD%<e
z)yY+iv+8e@xv@oYf0@v~*_mH7X~F6ZT->+%>t6QrcBK^V<Xf|WYYT&cpcdDb1h+2?
zEB_wgTg<RBb;|TocGZ6h5h5%?T$wX8b_ssu63TyKAmzAdePOM{t(8k>+AjPtacfo}
zm*8T>pTb;$m8E93yLRQv6vx(z*m9eD1b=e>($#AmDOyoBSzj^O?4+aGQ|&jQHI0i_
zzwi7nT;M##?Wa&gRB^~fuVvp`mVWO|4XjH|u4lTy_akiTr|l9UT%m_9IccX0f7+Q-
zev7+GqdiZPf&2FILUUc==;`fMishQQ|2Q{t+~VJIVZ-YUTz3vFn9%Jn^*j4AOMB;5
zIj<?x1iRMDTTe~M)>3?=sKO+ASlM%q16$@H%{9AMv0YCycHC?t7^`K-IcKF~@PhI=
znhbkZmoJpGmHez^v{8BPbFr0{Ta;7_oV=PXm!_ZlJ;{=F&UtCEU7f=2ize;)Y3Da-
z@6-^_6P5|bJG+lq+9z(B^X+kUQ)AKYM?xpqr&K11mkCLBDy^HEw12@fjc>1)J~|mT
zw`iV-qUNie+g4uu&lSDEzM{K#$~O5KtE2x%PXFuS#`S9c-x(WMFVJ7{uwN=^!PHm%
zW-1IU)?5;+H!=xqSag9~a<$6@Z7zvR8<{M)E=|kbvVe1o!@|32<&r@MmYIiJ%dFp^
zW~l#=iz||i=i?^rs)D?2o3#@+7h16jhOUl3v01J%ZL{~5EU|9g;@AonksI4%Z+H|g
zQj1d&Jmz!4)a9w9|LG;p$y;qD<ilk|{ghh_z6(VkUK&>=>B}K%_Fd?<Mr)Y!GBGou
zwC3>2#cdj9Lb<cV-`|{CdsrrT*VKY2d6r9Z`8(PLY*zd|V72*jJNqidc5m)Q23)+^
zJNFu#nikF7{c3@r7Q_4)t$!Q-^CcH7SWu(LerkbWR__1pE=dRY7*28hv+7`EP5Ewa
z?d;CB{L^9XpGNB+?lSwd%6Jp2)D&TzM0u`*nhPyhwH`KZ<66D=g(08FEE6r|n}>yq
zo^`o=>bhqt996W&Zd3Y&>@7PbdXCJPtp9M0?`)N9#q+Ifq7S6=_8gBZF6{FXkhyNv
zd%Y#IW|1cc<Jrfn_J6gu7g^F9^w3n!#ru1cRG3DRcjEysYdwyOD+*35_#LFxe=6!<
z*2cvjR!q6PK)>gtk&K^gG=HV%A^kc2OdI%krFPb892Qu;!h9Ro|DsbpT*6;JT>SZJ
z<1|BO>x*vJIcz$1Xz;1V9DQt%S5O=3%XYWo|JBID0*1e|?k?UO#2_1Vpljko^~=n{
z4DLr2CDugjsq7aIcU$NlxOz*v(cY9E5zE%e51y*HoodM|oj31?m{5cLH&w>KJAKWY
zoF|qYpW@=ZC|h`uqqk>gOZ{D=sL!UoC$?qZ=3BVPT;FkNewVuPbK``Wp#hFER~|+%
zUTyDOswv5Rc+DODt|;!}lYB`I{Qoy`{kL?s|9fitUcPOA4<>KmQ}Nj-Kj*f}9=10D
zf_>475<Hf9E0%95y(0aXdvQU>aUY?>!7%}4sYj=#hL;J3O}lvgIOq8}j_2<LDzM4D
zerq_npz^z-`rH|tz3=27Y;3YUy;$t4(EUA1%U|AC$_%>0{9i=->Xx(FlB%9Uu>!_(
z7kKQ}JoZ{Rqg*WH*Pk^WS!x%xva&YG<OZ443hj23T$;Bz^!G#I_|pB?JN75K3O>4V
zL5pvmK#JJPYy0OYX&D%biIhljxusuURik*WZK+cKL@vMoo|bB74^G^D`m4fWojr#(
zp1G{O=alxHcC%x;_Px2M{^ox9QJ=bGeyXPEpA`PgTY~!^3-WBf%T+AM(HwJZs-T3U
zpq@8ZSngGqIgU|hrDiu7PMmf%<?lWHSI^H2@5!$9V40pjPeV1_P&ALnZrw^dws$W+
z-w_py^Vqe0jr_|wHpR`HiH0`+jxBqlV>M@r`;Vo8iZ?x~mel@{3|PEAp!5H-B?YCP
z%o;`3LfvQ9rbo0jTsompC&P1$Ys#JtUliD0Pv{SlyAsH^a!bO_DZ;1Le4W3gCpSBq
zf8FAhO+Tj1k~&k`@wUuE)czP>)x*?4x74x+sgL!NwgqrMJGU+TN70jyQuCXvzHQ-5
z-f-`e-uoNnSHJiv8$VPJDoA?~TWNPt==z@{*Y+GSJt$<lu-r~?mi{Dxs^hDtd7e{W
z`r>6zdVus^gO3HLqB7KjXGbhj*~2OPb&l|YgXfRWP&>mZenD_?uxjt4MH10V85j;E
zSj`dUiH+4hr|}|d+d?DBq~I{6w)sw$q3R~0v!}55E}38TlHVgv(DH9KU-trr;@dn~
z!px!n1B!W78JneY4m>*+s;Tr$C%I5l!GtR%-|@-8&+%8dZf)l*th<`WZ{*E*mPvtA
zK2O+dfu8u*Icpf3{<+<^ns`lkzx&*o4;(JCIzJRC_<Jtmy;1z12NT~-@~M20A~@4T
zLzq)tX6vHOI@cu%9;>f-;<YsBReQ^n-=#}@ESIf1_o`*<%>+jIrF$=0Z!sxrEDd<7
zzPEhql&j9kACem_bl)_jh{w2Hh<mF)ad};If9Uxi`8%ywAAPqi#aSq5rO+m>3AZ<h
zH6{O?^7lu{f;;p5FUr4vnD6~X&o}h%8Bf90qM?G59Ol2i`@GJGQ_pepl<yBK&4rKd
zu9Q7l`G4;J1U3OCi~og!0T&)Pwy>TQc@pC0Ftd%{Qc)&jL-)~6CJs5B%PUVG@nd3j
z(s^O9@MI8&vslc`3#q4?3}VkMiM*_;+04AXV8)@;Zbw!#UNw&ZgT^Knp(Qdoshcx{
zm+8&tnh{l?CEXmbIY{*=*K-ZVgv?1wTQ?*~Gp5|kxs>JIa64-4EHTjuE-TJ1;!^Xt
zImzVYG+pUsfA&-xbc$^Fo1tZQ;>Lo<Rcy-QB|ef1lh5?WhKUKxEKWSlY0n)HbMwBM
z?&64*X;*iNt&Vz=bbOuqY+2i~EqTx5md_5$zqccK^DUQEKh%tlpH;HgOPKt*HTs8_
z(1+rc2bTuD@p`x^^K`-orggXHDE(hhbMWKU^UZ1VKRowu`e=IYe#M7JPVGPCrA%(#
ztN-2Ol5}Dz19vpXaz=iufCg6a5{~8nxSRr({4#m<(*Cf}gKp6+e1}#ktg$okaA9Q?
znCQ|Hz;p4)tb!uLwKK(D1vnXbtUNj|YDO08@<mf7EKpEwa`ND0Y+^a3x4T0mfr-)V
zhbAZIwiyhkbn9$Rc&YFz1$A*Ol5l3IKVYKabM%nU37^?s%DHFiF1Q$PzF=Zq>|^zC
ziLkSYXR<1H*o%cn{q2iCZ{F@ud-`pFv#{%z3xRHuhcEgb(>NE%=J|6{PoS8W-zD8W
zmsu1&mk4~d(w?c^Veo32R{o?<r)DP@w9K&K*ziA6P^rV9#lpZNe16R${e@hAxeFGx
zbzIfo&&0j!#FCC_q7Rg2MOm$w%#rQ&GKz7-R8KXY-CH@AH}0_D(kN1u2yb9`q4jFT
z;d#5gSFaGd#1+@_qSt(Npqj`jomE<2S7~nN+vKXJDYW7A?dD0BIJ`7ZOmJdil@Z&d
zw(HfT2f;1k+9%|XS$u1sTjzGVBIW;|MAfY~>Vhj}tE!T3KAl$I{WEoHWpw2Bho5ZE
zS<K^RTD_xbUa`29=aS8you77j^BFAWx5~F|KIOfBL4xPHa|$u9xyu$Ct+MO1pT27A
zEZdCdpZo9s6O)>Da6@d+?}cV+SEj#B*(<gxso7LzvFMsvhqB!MhsAK!yxAQ$MZj%4
z>x@9fIk!%3To83DQ}si0u2)jh^hqHH*7>S5Oq#68I%~nBSyQGcaOo<2J}!Qts&n)F
zxLZ2Ck8fD=-IT977NP1s@#v9)E%$_z_HMY5eKBQ<bEIco;QfZ+xQoxld*jT5WO74v
zHth>Jk^Y$H)R$L&tOp`&mal&ua!G%-+I^dkODatNL~v{MZ{R9id(}4f?6Fx{O@~;w
z?fo}pt-YWYtLH5?-K<7mhAT?bY9pK+nhUg}lMPR_Y;(2ve<oLBhBrs7S-Z&Z-Zfg$
zD^GGPe<ZqoLwmgM!IM$9K8Z0NSt`n^?eSD#nMk{)ib1dfGkb2wgheuL+*==<`JZAF
z#$T@>*g554M0#JuvMP-<3#J#Ywh@JULR)<NvVYx8Gf|s$`o<EuC;``;v@*9DO9Rug
zmPnp%N|t`~fhWU8*qfE#X#4k=+*)=?|DqENGZy*w*4$sosc0DC@gdgNj?Gb8ng6Fp
zb4IU&Nu~6Dv5-Y?Q-kW;AFDV$Ni%JBUCFz4#>GbwEQ<t}UUFU;bFnLIr<cghz-G>g
zf=*dix~g3CiozPN_pd1tP!)AyJZN%4zqV6*A48fZ`{y~n)0%dv#yu<Br0gpcd)=j~
z;EnleiKAK`oB0gaG>N{JIBLb>(C=n@PA&1&Wvio4mX+V@RQtdpC(5e8_UvhJ;O{HT
z7cBT+zV{WYNFkeK%u)l<Y40kZE6iWir?N#rD=@I#^8L>%#R|WaqZYb{c}wvmToXPN
z!`Tq7bk##{`vq;iC11kJn0+^OsR}T99Z^g*j;`gg5Z?GUz3JAbPM<W+lRb028I~A(
z1sJ%C?4Pi3fy$E3y}w!y9@x65&raJX_|Y`6imgiS%(E2(J-1!_CLrn+sPJf})3Ls5
z7npRuE@;(Qbb~)|jZ#;_$H{6D2g7_XIQiWB;=cIBLDvOr(~MSkxO^@sHa0)?rdQW`
zjdXI^>?2I->8k}_v>f;}wUNzl0oTPN)zcQSe(FuvJY(2b;!{0$Y0C#2FM}lsf|E>H
zJ%buKBo}1op5FHVt?v=VUA}jG>ax7uFKsf2TkEZH)5+`TMkZ}<RgG>R%gGT<&s=o7
z1T`-xEsEZzoM5=bOMBag!@CO#R&X$`_ugZ5rs;<fmmo)swez+<!yP>H|0c9cY`WQR
z*x}T@NAR%5FGqgEHH$k_LwYp6bg5lWG^j{Ca@lIrgFR3GdNMT4GWf!=bap7ib8{vG
z1KVSn8eu()1D6OC?n_%_95$g@CsEVm*O!V33q@NNKKy(|%xZOKpr~#}&~5qON|K&2
ztZ9>dtwR6pNO&TCS>KVVEAX~~reJDfN+a7H4{w`Ef|Eo(C@9SAXHlIo!9VKtwsM}V
zxJj2b95}Z+Ub#mrc=n`2HNO8v6%$TQ$@=9wZHMR;-Rf&u@mUooKAme>Q8Lj<<jqyJ
zl0K!*V~bhiR=*72F>kY6VecZ1Tc>@?I2uInTzM@n`6qMXmCc56Pkb^n1Y2Kj-I&TG
z(ax}^QaV4&No1x#>*|YAoW2a{qJbPAB>6-6uO7K+F}?J<dibBqm$=R<%>N~l(Qkg-
zqjZ}7;-IyEBBUa2g!u;lw!AiD@6<Id4YN<)KHS-yF=^7RETiS^yPr7xoxFX&#yk7V
zuBlEjmD68Ndf~c1b+%?c=i%70>*wu^e4cfw_$aB`>}55VopK^nMO#q#k!J^kW=*1+
z@z=#-g|1&D9KzzuLJCDvMGh=3`Me_PO0wvG#r}mk9*>u+PV4kb`SfTGAJh5Xr7P94
zn(n?iX2DhWFHL0D1t+yF3p%HAD+ot+tk&2h&^a|-QFyh)lq<`ppBIStfBjDFxP(B+
z-ov(D)sDq|#T<|4N@T~rap`XnDcQs7IfZ?(B1`55H>;o@bG9srntXn1@u%;MysIZ}
zz3M;3TWDgI>xomga}#`&GA2w|!lE8mw`cp|XrBcv3L5)mv;r5Yh?t%}eb$H7Hh6KC
zh5xyo(*j<0l?r~c*!kn;r`C_DLLP>aEi3jM-m)Qk>F%3htNbf2Z(ZPi>io}{tmo&c
z?WmtCd|O0p_R>Ixi5=S7E`=pgJ<hLo{ak(J<;IdgnX;FQSNv~ZRj)fcA^6&n-`BpV
zvAI7Nugw(U5|wv4w!L$q`6<(ba?K)J400DcN`1B9xG81S!nsL4S!G@10XHqqr^0L8
z9_e!a*<G?&SN-?y?H{M@aopUTpjX$qZgT;rfQX}@%LM+1=21$m)u-3JJter)fV;Dp
zGkKMwwxZt0Rou<q9V(p-&rb*+3$VPpLjQl)!bhvywV0G`KAJ4_FsN4IJr*E*ddA$V
znUk-6WY^xaLwn01tuL)rH<)xU^zwVpTbeA%Z@hee)ULt^)<A(~ABTB?9V|7IHh4(1
zI&xZVaavvaVA+~%5y57`jSB4Z12qo4Skvmn@uzso<H+{snj&QnmKiGSa`+#_#n~|R
z`XVXC&r(e{8iFq#d1YXvvUomI3djCn^ELtL2`i?}C~}xpwV-pyf|di*X6{&EkZlw7
z$Ean4Yjg^~h|2nf606P%sw|(tb-`favss<yhu5^MP%e+`Fmo2L7G`+MWU=-Z%cH6T
zEuQODB&?@*C|&AU9hkkZIbok^qf>yk`?n80#}m|Vew<MnvPi{CN$<?W>lM8x1sQZN
zG?lJkHM`X05g=zOvx`TV)#Cz7l(W^e!d*T;CVM7ZnFQ&)+uVCts{Lw*vbC|y!kJn<
zH(R7XA9CL#_nrIDtpbm-6}@v_?w>z{Nol9Rs*CdjWuzx$ZOYqxEH85QA%X2PKS<qE
zGPnN!Xu4n3c9jd#PpmkC7i|{_T)N0*>5D^MQkl%pKCPV<(NUG662vg0?83y>0EVE0
zdz!V?TA91n#aOg3pO!e{nDA*1$Ay-t6T)Rq98VJu%{F!qVqRmukViMlefO6cy(gXa
z-f`OZg|+&Fcl3((15>&KwS0oC&IA@UhZQh~7tC1wTR6<YdEw<2jT738E6%Lmt+ed%
zvPFxntXT|SI%;?7Y+S2tX(c@W!%B_kC)T@1vUa{vx9;hC!!bW>W#_L0qJA}o^H0d{
z4c1hR_M9bPKI;r;hlbx=E58R%%<>K!O{?gfyUKNH#hz&eM|Yn%>Mo?!>>=Lrz-Cd$
z_MVGf%l33P6=|*h|H5;j1NZ)mCZaJ~H&(aCPVr>?Fh}=?_79%p`c_JRa)ebjIhC(r
zukPTv(a^gxrBzllfN5@5n8b|j5)yXY5<w0;##M_$8(0DhW(IaJ1!lNtthjK@N+<I7
zg{j*6rylH`S?F=nYRXBk{T9l6&sWS7v_5k6!eQZDIjK@f=gQv8llbBnc22W7Ag|oB
z&eOr@ffRR%gYw$*yB~N?j5yk<A+5f(v-`x!zYY`lHZGa9LgtN(mc~t%Ee_nOvgbe8
zsLpyZ``ZtJr&X&W|EyeU6MW3E`|TrDqaP<{YUsBLHc0PrlYO#hrop6L9SoWt-U}t>
z&1{f8<ze6?d%US*ZsD07dygE_`hVt7_h;_|B4?^5a36Xi5TwO=IK;=?qoZMG^INW~
zEgP7lGnh3cn74ge?wsgsq-CWm(SBcm**kfj<4ifXFTK&d8|tLg<7CeUdN_B!Stj&2
zG_>^$$LXumS6Q59A9*)})Avhfy6Pq$la4!2Ty8{pvdrO}{iUBPX1mhWP3spPJv2dV
zy25qU?xUx6A2qb@TC(X})l0D|$%WMu#YKJGtgSeNlNFCf8fM(tEa7#5lXJ_>Xz`pY
zO$R*fPOjK_=k}uK$?ok}&zFZN#TUphsa{yQ=f=sIb9OA?V)YGZ76`4^<*;s0^cQWg
z7HEj>;M!87e00;kiKjR4@$${EoWhs7xXr|V;(w98iBEd3JqRtF<Z)v|FGsD1aPQ3G
zQzp{{8TLxCwrh8U-;`qBvNl>UXj>u2p98u-cm(FK*mP~^e5f*$eP+<M%JXM9mo8t#
zS#RvMsAlP+996Nz=1m^!1Z~bQ{k6Se<*9dvxOg*{wQwvu#i@2+E$8A(UP>DmN`46r
zo8g@mbw@4gc=;qJ_uf0Yf4r+X&K#Jccw~#-Yo{|ubt3C|BPZ3WM1OF);Ubcvd-uFc
z>{Z{p_MRe3Z9AM8r*H)yv58^59orkv)x}wHQhfLP)hEwRmC^KG)Axhp=z}>+4xHx9
z2wcgcaxTclZ?;1xYty-BPc$P$!k?Geyxn@Nn6qJ_k@(UB3oiV(j{c;wckPXdVu!A8
zu{g-8c=@RocWtKsCfSH)3m!}6<5^6?9e3}Sx7?_jaien04cVX58K)j+spVpARow1z
z^StZL3$?+s8y@T~XkDYg9O69XL|5dIRW}!Hy*cx4!n9r;sil$AFGo%~v*pEN*4v@I
zJGi#pj%~fKu&>U*Nw<T+?h(_9ohP5k&dk2LL{Yc1_s7ErP51sg1|5oYJ#2k0LC|l_
z*@Jw)PVRSDw_?loI>XCbMT5C6nH)T*yYFC<!RmEY&GF`I#hW}C_HX56Js`AagN)8Z
z2Ia!!^M%Rwsmw;1A-YrUbZ>4wwKit$qnN{Yt}I*d;DRd$XWfH|rcYV~p0Jm32L9LW
za1)Dc-q6gh+FI)r<{dWE>P$+^(G<UZ8<KRVwEWPvSJb$5)*yJ>UD2?4u74&gZ|Q7(
zu%~C1Xtu8(Q_j6iT@L>xsn29L<=S3XUVGq`3MZS_BRSsAuqpRm-;K-Nd%bbdx#ogr
zvlICFowPVRHeRY;w^?-WCGGpkN4Pe4cUw&|w!S@4V9mrj)yRgue2ul+&&+iE;LBCf
zmwb3d+MhS)D|!|(=mzx6^?JuOgSqTUKW~Od+k?ra-IH}Yn#^u`ls%Z}d$XzF#qNd|
z%@#K&w`OdYh>R|1HS1t9nsC$fS4QaPgbDj1C!OnEta?%89dF^QWny)SqStz>7PSAL
zm8hB$q5Hp1cj}g@D_T`1UplL{J1T$f-nFimIyYQe@ayE#3S%BmuM0m8C~o6==43lt
zLX}mLbybLZhDj*zMWN{dLSMWlebKs;DD_%FGkeoN1#45TC6g}id3kRBhSsPR%=YuL
z-NF)_+-5kLB{dhYS{JbL-e9?Qb-DL9b%&Q*ngpH}d%bz0`Sg}+s;kPIDXW#P?^K(f
zvHTFn<nGgg(qhX}vey3PRD7^B?Bcv@va0uTZok}}p^zN8iRt2$o`6e}6V4SKjWc|_
z@@;AOtD_4FXFBo)F}(lEeUJCpCz-r$B5se$R(+f@;rkrD{Tnw+1up;gW!>(A=RbP<
zTN&eD`*GYi(|Y^m?H-TUasN|h?6?@Q`bW%)gbUk0M0`1x!Csfr!q(d|@5Q$p>zw6M
z@(Wp`FEr_TJQR?-D4-T-F2rDT$3VLEVP4Wp@p~pd^D<|$t<l|dzU6=|Cwp(z1viDx
zRBpj8Ib9x4CQU`YOKNIKd(X{q)d~{yciqML(P!0+Q=BHv2TyI*T~XzG{<V=;uGnSC
zLM@TNOMEkriUqEci}(7yEkx?kD}}~M=4O*NzH@ELeSP+1_k5e_rN<5~4QS%bXm!?m
z5^%PtaT(|Le-V1+Pkz1o<bLhbUa?4H_6*bdZo~eR@E7|-|E1jLcv^f(+u+RMW9JU%
zt-9-~vTw@f$r^jOx3`@BKf$<-u_wz`^#6Yjv252(zZ37R`gyD1?xq~i*_{fklMj>>
za9`?LQKDCUw8XbF#q-%4+wCPsComk$RWMq0`lDY(ZTPnb&&t1YnHg-%5aaea$XWX}
zh~?w;wMndpBAy?-n|JoIjkW1FXRFPUCtv(%TiC2t%$SkUYS&@r-!WzF3qAji^!wj-
z=Y5VYXtjQjJ$*t(t6E0OiWh+sn(i)~x!Cs2lzS$oPt<Hh)TjOvW?TMJ@Oyt$2a~NR
zzmrw}zQ=()P5iD&o)4Lx)rfaJe3RNe<KBX8oxI{(bZgfw3lZF^>X@_6Tz^?onww+n
z@pWgnwyru7-Dvz+dt>ktrLK=wGtT~wUZ-|vt%y*Qvi_O>A1Ai^SF}8M?Ie~Gl=?ZW
zfI}!CW#Nfz6Z6o1VLKZc^-B8>JkhCD6LJm+ZCw$5!*T0Ni^|ihzc1SPhUbD+$JyCT
zzC6F>Rv&S(=ZHP&8)vE6wEXG+0?pPAjs??l*Ra0&ZgTxqjo<CC>*>qhPW1db^TWMY
za$M(woA!R3_Gxcna?Nzn+0iWy{$iTub!{>dleoF%3wJfl=*jQTa_{){?^}9&apTOg
z)^+)5?_(yimWS^sD=?2VQfNGIqv`DN3DFs?yc<?e-FBCCdS;$a#2?oiO*1|mlHh$1
zAEENcq;Soo>=fO77cynje*fpl+rCk{&hg{y<CP0nILzL(Hf+1liUZGN{-5tMzS<=f
zFL<DEoiT5)(4YQI3Ky)SIBd;2R!!l~66so|Ymr!0sqm{m!7F**n*4?WWrv2<jcn{<
zHY+al>9#US811<na6q=5S>4Mf<M`Agz2z|zZ!EeHCe4_av-4rlF_qK2YmMSeECLUl
znaQqhCKG&K$-nh))i<FB{tf!gj`e(hR&IHIZ~<R4U#=Fz%0nwcXKJOC26Q@V3NP!h
z*}=Hlna^f>kLZD;YoeNCZmp`d4p`^vm%={Da7Ot9FUB&~H>Nu_t#8m{_MW=2yCOk-
zKa-W9U(fga_#^eQ)+(tRKPEOWwP`N<vGeoY-CUDACOB=I>Uxbma(d0(sD|w3t>N7N
z6O6mAXteIFm%nnSYs#}pN3t*a+}jYgdatLKbAMfUQ_~(%_OLx)1FXuzO*MCI+I%Tg
zT8}Beuw?09<*IKlFLsv)rzM}{jZWg;@pamnwMNoA);#B87oL+oq0BHb{bDKG-HWFr
zqC}3)Z4nNNPz-ysB5k5bw2z`gd)`*p>oJN^H-aOWI#e!2mlRIC?xn}H>_UR-rHbpR
z>~}K~`&88?xG;I;T=dFWvq90p)77da#lKN><r4=ltIU8Yp3l@2C7syHQ+40wWJYVx
z{-SZ9n91Y7WT8~90=4kuWiKbNoZsk~HYdfiV3~(mWQUNXN=y2L3YjzR$5>KwwoH^>
z%i%JAV)g%N57{cSITOO9mlhbXa&F(LI7vIzCosshtiX^l(dJ?2RNa(}jgw2lB15G@
zLnfMvgsw?))sFQ0YBHIL>5bI{mf#$QX$+a3=F>J#RNDQ>A$mipfcW`I5z2iA(+qDp
zv}vY@t5>p$%}kgwby;Nk{E6BQJD*NU%gcD>_B}HBo!pcLSH=_C4ou&@Hk@pUnp$={
zhwrDgK=$FxWwUqKbVjS>EY7%kOk=C!^#JX&yjNCl5ud!WA(CD1iF&i#?&-0P!4{|H
z>6T4+xI(&lRXCeubW2E^ws!FXp$+_n8y!!j{3&NYUli*;Z^fBM+gGwIjye$_9jGu}
z<QUgVEr%=q)@Mc5T>5`=@^qF@*Q_sut5;bhl_>dDDBVzK3vG+>kUvu9e}n6R*;W6;
zwi9AYlkX&Fg>Q3G$=&udR=Td~cG%O&^~#>7rnG$zQK`E$ankkNiQAL9ZMl4Xysqc+
z9a(lXrLfhoR4AuV$hWM5N6=k;z25{Et@AQ*=PC;eSIVyaQ(pa!H8+qgTXt8{ea|_k
zVl+8Z3K)zlIiyTJvE(N{xgW=*dc0=k)nq-!{X67JO0PPox`qb+`{Hc(Q}Ekm@1Ny`
z?<c#i<+@-O?0(c<V858=$E(-HHd(ycbYxR&PP*{ym@=1Xd7Q_T8aa6)<4gpCJlLAb
zEF<oSD9=1EQQ$PEv2Ux_#+~OS|35L|+7T|qrm^<;*C%xhzh@u#cTDQ;1}EbKFV8w;
z{BRJH@?<c~k@D42XLx3jo@aXN7Tc{23Rkwe+dj-Zs(VL4esYb9s!9j9;|YbvS!-LE
zJ7zJkryV=0n<S|FtW!kz?_t&}e~<I3Y97|T6|mCi2wQ2Pv;f!R$@0vaOO&{kx^<Tv
z=vo>0BI{}Nd`GsI{Z%v8*L+#PYTFW_Tz|!hOVHr5h?&U*{!U&!l_0UJK`lzH6Lj+y
z9qD#`^6W%#lG@biClz|XZ0qb?_`$>SiQnWeNs1bQFTDB}$LN+_;$!?{RGetUF^f%T
zmzAVS30Gj#(Fv74Mv|AqqgO_7IC~h`=x${Bza`k4^O{ks(3u?F0-mdqtgag2Qx-U{
z^_aJK()aC}A**>GC#rkRFf}%u)WEI!!29g253<vGSk#m!22{jsYz_RuRA=khr8qgs
zu6e?G@eUI%YnB6jp=+8%zi@Ermb|cQzwt_onKSb{*U!NAD~DV5ed5}Xw@v4NXG;XL
z;<b%@Nz-O4y`CKXz^QA&R8Or*ty3@EJiX1uQ)1CMjg^b11g2dpHK@ql(A@u$bC1j-
z-FYkr6e49$JQHGQVDV^dEa$ut!M**^=0_J4mUJx46Io;-?bX@JzgqFgD)WsQaXO6A
zx3&At4ji-L>Saz@abEItfacFX&&4!;CQ0`#lba{ll~JDXKTPu71~I)YjXWt4ofetS
zY&%b=Z@#j7v5e@Kjg194{SlgPefMl`)qTS<fminBuQSV6IbT~e(evd|jj)B$E7v*+
zOS$^x>uNQG-cikJkU3Gb^s`oXPST{LL-Or&zAg;Nx}<nx>BZ<npCb3I$z7Reywv$b
z_o>f*9DY^Gt@{HkCL9dyT+{wqdeWa|qQ^sD)@|LBAM@Kx@uX*G(YB5#gZ9f?%|kaA
zM=&a`|Dq)nv?SH<O2~(n@~KkwSx-{ZQaF}-6>LgW<f`Bmiqlj}5Smcp<)B$M)8WpB
z<+n^vsc8IB|CW)HJTu}+mDMy($t4YQI2^ag@0}8AZ)GJKWssx(Z$YcZ|FA@lvJOY#
z#WOPO-4xkexFSV@-S~?3w#O}wU<uFZIApX`lVMubiyWiSj+;D}bOaZsMl$jw_&;IN
z^%I$(>AqDp{>`@McDFmGFm>Ev`R<hA5ZxBbmD$5uc*Nc{UqCzF)gjA9OZVqLZf)kn
zb0&5KJKy6t(Dv(6&+>NW*;~(TRb2IMVYW)lBBPs!ME3`1mIxVWTwT<j*8g^<Hp^2D
z$B(WdY7djTtB#A`=Vj@v+@l<9VJ4g&-IB~c(JU-z_nYVmPRydE9`>vDaFy{Km0v8u
zYWrs+S0KmsJ6|MPHR_5y!a5c$uAee1Od_&Bv}d>c_iHUJd)}M~%V^Y*X7X+~nE5}p
z`%C_A(@#s(4_hy0l`?H?h<i1?Qt3+0Rn~_oj7<y{b^<CK3{Mo;o%**OSZ}ssL%wR^
zdPlECQ}-;Gquy=Jp~vsOR5Ngby1{v&>|2T5@q9~U56^rzqid#`iDodPhyRC*C%3Q~
zrT8j(zxP~!PUKoptxLtW2UkQKpN8LhVm#kSy5p2UsbJ$hmg>Geax07GtTy{mxTMJb
ziNueyeR-Rmgk|5pkf@68*Lt0z6|=5=Pb*Vu;F^Wo-&mZoI`ZN6$}cVp)l^RhJvk?=
z!0pN~YsQj`Q$5v8e&;VUFg-ELHNWbA?^E5S_ccXBk6hlNVPh43c4F*{MQg=HMK8<=
zShbx;>B_$U234!d4)c7dzBJ?UYV99lzqA-Uj?4ONpP;#Q=Dx^6kH3!h#n&zT*1I6n
z_*KC5w5w+_Tl-gRyRQ4<xRcj?C+CRn&EAqHQ&&_kI=<}eo8POqJ$jyd)NyIrgxg(B
z(`3W#H#q12JiM#Z$<~0!`bn(r6n{o9Uq-crUozJp%g(&#${U*U;Q!0{Q9Gp!IS;Tf
zbRPW3B0gckvA-cVC0u56y4^a$q0xBVv1973FLCT;3QRR~92;a9TIMj+hzfn_(93i^
z@WWeo>x4r$!cYI~J!{BUcQPhsXZSf?&nQ<u%`1nVYOoaF=w7c8d`Va0@<%VB9EU4~
zeCul#N?zlPzH;Qx-4>Zd+5fj6N|Ya1==^5J*B!F5Ztc1`2PFhpx6hT<-Edf94b%O<
zcQszP=qgN~8X97MAo~8p*%DveY<n6tzBJiyIJIkSGw-$Y4kyk#@~~=8Ir%KYC5z>>
z_smD?bCou3P<XlZ=&OxKJ4>#|d^xWb+7VGYpZTYp9@DXS38^EzQcSVkjtQsPTxX^9
z2&HJWeUNks={<E+^GJB()Ng?tUv>sWrAnQaRV?i2-tp$x!3z@#HiYM|5v_hGQPeYE
zq~p=i1G7ZDj{myKvBi-8c))Du9rCtnEdE!Nk3ErhTEOK$<F-VKlJ*?w-7lk={aO{d
zu7_ku+P;Xke=vLU*J-Rq)Awg8`QLDs_q_g}z3S|NP;t(A3IBUIJi<8EYIYyqo6w^X
z>bgoLtl^Gh!JOl&3<6$^(mp)eNACQ+)Bi}~<Vo8LkIu-1{ZW}(aQxg#(fb7_9XVv*
z*|@)`=@XX<zS=n9Vyl8?0b`cO@k^QxFH6t3IUM(CY1dtF*TL|pOjl~;GFiD>iBV?#
zx>KB1rNlj(=<lvLo#BG>T}wH}u7lgYIw`DccB!4c`zXJy0(*#qNXdjLtra(meO-6l
zZ1zyNG3SV9R*h?G#?luvSs8ZDjCwPrOJ>f|pK@Vm<}G+C*Z1Y<yUyuHTw`v}P<w9}
zn&RM=6(+DKHKw&9^yJD3t#{J=CtS_@cRK3o1jmk90d30v&D-WX&UwtfHpTysm))*D
zp<0(r$LEd(N6K6-#+^}iO*m?o)#7TPUh*c{&X1+i;H;z1*&RRSZRgxhSa;~kAKBu$
zA;p1DzE4>0S|M5E#2%XPuq2VS)Tcqj=OAa>(bC2fl?Roy7A?BvGiTnyw4zlh-cc+k
z7q;y?bE(Bnu{Ka6Qa7<~Ys}Y|0TYi1M5+2^GDHM!3b*|*;lP8wJt-WQBYnD?6falK
zbIeF-mAuUI!`rbYHLW)6?2d!F6-V0V`Z<bnc|GAt3uL)AcWLCYhp|@@d++p#tq}Om
zp;q}RK)s89##A?Tp)}9GEb9!W1P43_S<yd3>sit-XQxP|S@TxfZ(;gx<#bt0OZ|nb
zd#aYptHzGG%UCy`Smg0Q^5%n^AMUp5Yq1z~O<G_ipsytmcHpc**}-_7Rk04ob|>6P
z(plVc<(W?uhgsUJ92fbL9c-&wZyn4@F0i@q<cIfVpPt!98o?TKS$f1?RkVmF3W{$G
zth(VStq@po!B3KhjXg$ip~VqfqmB(_?6N|y|N7qE828*gqGc1Gi1j+HV>+!tDnVP7
zE^U1zW_^W8yXIuqEVufy4t2>EcOzHNvb5GCNuG6LTXizF)XfYil;O8};ZQKAU!_rO
z!eYLK6Y6jJFPA&=cR`HKUDp7u7D2wLrjr;g|IKk!c`ByD*!(c`@<*?$trxC%3ja4s
zJ#)o#sjJ2TmWb~8FFd6d&al7fsQF}a<QkS!9M`;du$;2#zxD9ODntKCYA?$lob2s=
zI_c3uxjd)9GhT_mlxvyZMkl$>bUV7m>E0H*DXPmfFI(k0{XFD)?Qlb`=luzdig|~!
za~nlMqC_HcdaPQm`aa{dVwpQ{(Tk%Zn`Ru1-t$P^w)?fFjBeD_c!xQ<$2gcKHf9EO
z-@7%5d99213Ku5HV-LC}CH`3(xK{0x;jy$C;=u;-OXmvZm%aL0I4kGFbk{d*2^S_z
zs_9%Px;*!VV$m*bLrsR$3wdUK$XM~DJ#l7x;>;UozT9Nl(kSjL=&y6)?L4tEHR+Nk
zkIS#kFS&E*zeY}@i_^SnDg7%EF2y&OU;W{-RcexA;_A*>8rmC}4(A+_NLeeA(fG}+
zQKaPkOh+|;5#b}(Iyu_3yJ|e#`&PShbRRkzm-)s2R?GASX764snsi)}g`e+8xU1uY
z7uk7HEDXm*g{%@|CSKz9)p+<|QmfD!=bntIDmmISGA?lXSYK$pxbsEJ%*5UlgMS_k
zcW1F!8nSS9bm>koI63R`{0WZcFMHlO_^&f)ExV=f`D2EY>aq((kDeSnH6x7or$fkP
zrjN6ht+aJ{+weE$J`cNOO76G3w=*q2YP@?sdlKvH1#H(>vF+US=w;?oExruxc}klL
z(mYNr(wgd)c`d(kq1q*xtp8ls5_mtcE-`zR921bzaC+;W{C`zbMLbsg{nqVR^J2l@
zXS^#z`8KvKe0aR*Pr%b0#v?`<*EQxctvTZJV&a7lCw2ey=$_%Ls^E~B=+HIMGww#C
z#N6W>KYjcg`@Bkm<3HzyEoyI*?AGVX2I*XH)-T}PZp7o1w^q!hQLkG22j7t{uY)sK
zHurox(&Oa*uVVEMF7F#_9=PXSy0c<+%oW|7X`dvodoUeY?dYHpadFB1583Tf27bRj
zPB?N&%iH~|tPYp!hVy@eto?Z;e{?+eoy&A#v-6eBbzW;`Uf>L0aq`8U0_j<TzFQiX
zN_2V@O!H=FtDW;kVyDEPvun=jtvtIccJjR&|5rU+<5ZI>-)L|tNZY-A_T{E8m*>57
z^~${}<oP6Jty<1nkri&<^CgWUm(G@%Jj)|TJnzlyL!5@6kIa#o9A0r$)$sG2dq*Qw
z&fiiyoyPY<_Iz%}E#9EHDmPyw-ScDH9#Fh+p31R8r^RNkdSXsz>E-q9S!SfSZpmHl
zwF`wzqdH8)I0K8adw7ojU2*HcjYD#`H!F(<&-@S?_j>9jRfaN2E^*0;^)0T_4Ik4~
zqZglHms=QCHY=>@h(USP^W!frzE=~hirA)mMV4LeeeqtSFNZ#D%{#g6n1FW7d$}zA
z?^Wx)cbkL-7#cqRlKuLN0OuXHAIn`s#An9}rMDS!&0F1BQutpdZtI^<EvEe2x9l*i
z$`w6XJ9Qt=Liyj??L`{<Yc?}j_mym)vx8^u--;FYO+GBR#I}FU)<a2t)=yfQ3f>xT
zT;W)9wkll5cw%Su_0Cqq$(MGThnwEm+cSZwhU066YX2;k9@Bn@2YS9IHaPSA*i_j5
zUg}4r6HCM<Kld9=NusCDaGcD3s(yJ@W?Qm_p!eGbMME#`LcP^yvkKnoPqr|cT4CgR
zJl46;uv<U;u8Ezsi|F_Cfc5#A2}0ukL*^v9Ha_YIdL6U-mE3KGtW3E8&7~V39P?E_
zVmP^fYsCD*Ts20?8O28zZZ(~EYk}oSjhNivsV<6_*)+E_@}FfXzQSa^?D7BZ$EGu1
zgwB3HHBPgI>4G=IiAtsmmUCx6<haQ#XegMfUFLsc@uo>dTRtAic>2!nRKW?0<wB8V
zjyIVWO>xQ)jaI*0R==!G*CO{z+UsrmOsB8@(zd`XnB((`*Ix|W-LubE<h=iKrzEbo
zeO@Em3gO9bcNSeb)Ow=aP;{Z3c~!6Yr09-44<7vcq#&ShApB@a?urZ3Z$I(V`dux`
zS6B11&8lVDw6=p?8m&xi6~;O`2Um)G=JGjiZfv_|$NR3M1-*Okc<)+mU1#skq;X(o
zOThXoC)dyLyUqG~65p<scNEV|Dw^Np(#-tlq{WVfXM8k$PW3H1D)OS=d-`tIh=mvC
zZV^h(|1Z4S%y2cE{dUe9c821sepJWj+?W=5B<qi9u*b!pUw)nY<?-wM?2U3;3YN|7
zd~)a1pLDaWp@pY=1GY*oPFZy8#3Hd{?^3sBO1%q`%!_C@4USp$H*Tl%lO=w6@z0F@
z-uLLsmt$FVi)qU~%ed>_-RE}BnQp!J$xlZEqq!;$f|>5`jlLyYn>D?DS0-~LCEdMF
zr)u%#y;TZaiw;~&vpp?R&=Br%fMtet$q`QP&qgylt^P&0WG}8_pQO*Ri?d|IkH_V#
zx%1aMEqU)KF^7{aqv6;du0D@j72y>p7fkSOH~k-(s8ZQ<&11uFg}qD;oPV|RyyEke
zKGYPNKTueB;7(nMN8kVK7W3>WUe*U>BUJzGnk=yU@%Gy8Kix`uZM_>8ein2o=dq1*
zDG_Rs5p6l(Y|-Lv@p{VLB~utRulRgo>0a7x`#Jcy<gp*cODeh^iykX$y-~jKfxJu4
z;_r?-qGlcUmJE^zW@uqyWM<V+tPn_a;tUe=^Y{^PK%_%JmG6$zgww9w(mF*uPOvU?
zV-&Y6WIUPq$i;}|^pZI@iWGdNbM6*cWBFkj!%X(*Yi~}j^jSW~n4QUk!J)}xk@rk9
z)mvR6+Kf3Y6N)mo_BMnug>6#WTC_lyaVy8IO|uSUMr~!exNR=O<-Ud;%<p$7W|S-3
zvu1g{Y_IBrzSYs|nE3U4ZcNn;FZ&<0Q}c40kECA8vYZ(k9Gjfmna!2o*f=(xzQFEo
z?{nzO7oRQ0!OA&%#lHD8GN&I`i`e~uMN`N!F2qWx;z-9n{xBBKOJ+wNbB3?W**Q7&
z%M<VUuT!{e%@`ORSR{O%mWf_j%M_8C8$9p9&#)KlL9+H6Y*v34IIn&t>;J-^E3-~c
z$TF^5YUaV|ufurDFktnXu-P6=ss++3wy3ZKEik#nF-hPM7kkkMg=Urjhb|*7o$ezW
zSPEJdL_+7Bc8PvdAb3pt{f5+I;%_6)h?+-bE}mj^LZO+v-)x4Hd6U~>gTq_0qJ<QB
zS*tD`ZsO=FI^1qi^U1VzS+UkBy`^7@7U&!m`JZr5>$Hf%>E?5H3Kq6?37<3T<C_%B
z!1wCTAwO$=CqIVC21#5?TPzr}16pQ1ex<nKm(7ZRmLD=5=N$wmiKzPR+Hgf}LCK{l
z3;Z@XpU_}fa%0k!DbK23*aoeUTB^lpW_dzu)skCTOV=*S+9<l|%&nw@m#*nd)!-GL
z(9p9#;!d~TJ^`+t$b$mAk4`(t6TN2R@lz9cJovOu`!t>td3Dv}oPyNZ+ZVh(e_a^g
z=*pqXyS#Y%!Gg@_@CNqhq28UGQzm3|2RciwSoBaMXx){Q6SSQdPdhNHXF+Rak>|yk
zcD5SJJC|<OvsBsTt<cHxCpJraRmUVPd&YOl{R@72#N{hY+p6?`rp+psv{l!B-jE1j
z*my!|#ay!`8PmO{i~ZSp=3-kK_sbTWB?U<m*-H%F#Kkus?UjjrxUnZ_)7_*KoE?88
z4)M!NoxRsNA*9Nya;eXXt(O+5+&Z<RQ>!}HXyzP_wK@;mG%|KXzqVLwa&gWH$F?OG
zcoLdaxGWMDxd?HpSjztwF7lAi?Yhvxe(B22Ho>DFfiqpUcb3fg*J5HdpJj)qh97%Y
zY=hqiGp)w^FZRlYe)15!sTCUO%AvjX+v2OM=5Q*lo)%UiDqVi#)Rczq%%%0yqO5Nf
zC>!YsJWJx@<2&!ekbhX+qvqTL)$ex-&W2j&RLqw5xcmKS`o5F5-|WA>a?{Iq{{!x8
z9+)>-L24yWf!21J1i|AC4ptt|t;~0Mo;|NB=FQEtE!fehG)MTMrKnRFoAJsG>~_q7
z{4Q22<o|5pX%U!tGw9c6`I)|{4r?U@&rkWT%{fOmj??l|)SagatOX42Rz=SJbs{P@
zwZ7)&DRQj^D;m_#9&!G)JAQ9WklA-u@65%OXLEQqZJBaJUsXmb=0cLPMe~Bzo-9k=
z8EoSA2Q}Y%woAvad{H9U_hO}g#U_pdhffn-WAlnLPu_WMV=i!Z=@RwTo98Uha~1Zv
z$}(xjqs})PQw82mIBA{IaMQ<(^Wc^{I$`^T4`_59cYJtumDW+=(;Ybr_ZVD!-<A0J
zN$!@^|1)_@v}-mVS(JW))v;iy%k(c?&vZIh9@cz!e#VdN3H!R9tgm}3=y!N!$6L8W
z>JEBGXC36c>#VY3ntyH9Y{Q4oj_%@|=$*&oEHmfk$$+#?0!~Y&ywB;k2|mX4eZi^b
zwVR&T8Kh4Q3s`Dt;e7Aotw86tB^RAes+_v`pj;s3+@xOFB>8=pzMX1YmD(<zI^FsB
zwLqh<8y3muc0IT!ajD}L+rlDe-`P(h_8r;TGHdeV6HmXS9LduAG)ur(Bw%0DMq9@x
z6YQp*42TIbbX5~t={{9<g<0B^z|u`KmfY$JSs`@3Q6zuvq215V@@?8YXKNErLeHvM
zSGMi@$~d{s*z<nG%`EAS|E*GIc&(CmmtUnCRGJ;X_|j^IKcUC#pHAtIc6zXAN0EVZ
z<Z`R0U!Ij7^IXNE(3mVP>0iO8BrnwWK_N16W{q9M{FVtla~HUtKN+^e`pcRL8h;ZH
z*RXt<FDa*CGy8#r!9!718MTsiyGxWrGmDq*RBXxH8p`1$zh%GnwXX^<eeyh}SMR@8
zq3O+5>1*Ks>(u_Ex%p>QgQa&y2W^pkYf^bT&)NG|%q<zeK90zkA`2(=D_2dF>tj-a
zwN9>ZmhG~X_;}exW$HACSguJNCq0$J#WlFZmaY`45DmFKOJ>E2F5dm!8x*^zPmNlb
zvuf3!RqGCD-TS;Wdr|zBv+dJ`!Xvtl{hzFsR}l5~)NRqj$GTE$J6Ptq@2mOQU3k#*
zgb2$D3lGygtLfGfMJFa09A*pM_{inS(H|yPx@~<<I;oy$WL;orq<A3tfg*=iblWE1
zO~Q5w^PaQW-+roVzobHE^#<#!pU$`aOxJn)VO8UTCbcK4T-WgQ_KJvhT@+j$a$d;y
zgud&mDJ(B8WoU&qT@<p-(7U*LN!2;M$eSE$YR|G09MZg=28bV+bxPC2-Zj>fNpr;?
zHJ8M7*0S8(FGUuaxmw>kSR|xp^j!RSnzmxLsH4lwO%Hb(U$>C%2%lU&|H7QMBO3w@
zgYQW_iniZfdC&KBRHV-7*j*AzhRd|K*_UMGYd^UC@c%wVW8O`(_i{D(6|LgwzZg4F
z@u%R1*?c;CK38k7dqpz*ja6_Gsm`xA7rSql8h%`5x2WNBy)6q%Z&t?~J2<&>rmdVw
zf4$%9DgPRd9|~tZ&l4T?{K^&6rLR^`*m<t>+R~1Rw|{uQIB_^4W)tK089t6BVy35N
zaVx51JY5?tC4Tp4!mHeZl-RC{_?-)`bzJYUzP|3pj2k7(&1;!v&TQCbA)ul+LBYc%
z>ixCFD_<_XvC%Iv-p}nr$DE4YOM=>-cr833uwsFq`r95Q14osWCu)Q(CUG7tQhQmM
zmSOLzz3SH7mbI_^>XUw~($(^=@Kb)P`*PDwsi`HGR?UshijO;Xt?keMwbS3`%&a!q
zen2W}@%GoboGcAaCVqbx&b^^~-9O>g-OKmhs9qL2e&b&hlLq6&f}L?0eJLTm!Cbq4
z?|8NOKX;I0Xu=|vc~NuqZkAl!{@Xi7Nv+RQp^cw0S2O+T&GzN&e}r!?Iib{dXz7Bo
z>%t$?!x(;vTArS{KyJV64$%!SrnJw~tT|>MGgIi(jR_T>-#uTO)On%BZ<<p69wCGC
z8tc8JHsraydl}T<nb6O)yPbK}x>pK^SS~ej?^%{(cKCIimZC-P2D3-s7i8{#;m0~R
z|Nbt|z_<@5lLCIRttnyYDM?yuZN|o^q`;-~`u#SRDSX|lqFgsOoJ}ckOYci}m*L(1
z>3@2}lmu?A$6Y>aK7V`9zTRNfgQaF(&Br#K^*Ge~@4$wy$JRzK>0@5j_bubRRo;8c
zGYfcp7N~kXP<$ew_#)tYiM~fc=lX9S6waMhs_41<?ZuCMd4XInN<MT?_@}Enf#<;_
z<&zg$lD90FQq;Ri<MbJcH*;;eZKU|@EH0kj;_q-Gx#f)WmJ9i6Wygj33m(5xRE$vb
z?b7~#BZX^1v#?o$sninA%+=>@@?UbKi@!g1^c~N$0=^}uPMA1p?JB!?A@7{`t`*`l
z^_XV&@n{P08h^O@#GP;RM?P<b?{Xg$VzkUYsr1_%^7kmvy|FG(%I&@YE1S1~lFG_S
z*9`B=J=&jDprG;J<C+iyqlyFfmh<nAyk|e|9Qq@(ysUT1xtwMX$MC4O!%>MLGWy%=
zR86KdROfXV&FSUR%>Ojy;wOjv7^D6=j<sB^CzWR>nHIcI$q&2~;(sUL{e^&ho_PIl
ztJZzH@*%3=ebkPNRyB)Qb~&iCH}QEp@L4zTWiMhe3(#l`;@i!@=i8up|AVG)!?z_1
z{(WiX7goC4<iA&K`Q<x;N;@vePS5`-vh3=;cS#a~Zv);Yt@|jYc}(u=lH^GVJaKLn
zQEtM~ZL5t6C$j{##5mdRdowr0?YGZFjeSX+2b0R5tdV=RX2BAtonA{Fr)6><5Iic<
zb?eG1m7o2}zLRS2EX|cTyO{4kZ=Hv~uZO>h@P(r*Rxk3<lUmrvwDKL(+PrCR0;G3d
zG&|iOUHnNcd($hQMcF*d9fcGa=Dm=%RW5Y;py0CT%>~||tgik?tNr`@fBtAr^Jrx|
ztI61)z}WUQbc29dQSbb9T`5($X9S!a?j<WuKCb95Y--MB{ILDlwubCwXRTjuw)0-9
zJ3-C<wi3@vezp7zsq~<4|5m82KgyaH#8({jtM8IJ=OI4x3vA7k_<SE|+~43ddlPq2
z(Kjyzk7az{b{w@ARbUWGa=G|KX^oK5zvmXhNpJQ9-HW=CAk}f7gY(opeJi2WKd%JM
z5nAaczAAZ%!|A>a`t2f$CwLUjZr)^O_{i*k_Gv%Y&5s`2a0ILFJH(#oaXO+<fp5uz
zZI2#TT#PMS?YXMRXzz(zAz7VTmvfAYy}$G?p0aTlSCYzMkC$^cE{?kVa)sbG0|Av=
zLekp<zZVDx6c?M9UsmyH%HFrkQDi}KlY@o32?N7}KyQ(QLQd9yQ|nEbEn^q$J>=)V
zjlt4pt82Q_+ed{H#9joR`P4W~>D`ZP)~|WV+a_Oo@KS#A73*eRVdnndLhJP=eyAvN
zv#VoMQgl+<emZ!m0++e+%>r#dQ+^fZ3tWZSm+q-YwN5qj>HnJ1!^1q~)s6+smo})L
zaZo?S5L4`5dZcrOmBYk01qvNSn(qZP<_bAK7S*br<ox>o)@mCO69;9Nz<v#ttMQut
z)>E7~mo)VJc|OxgU;~?BLl1vRPtN^IMuO42D^?lnt}Fh<=6CRo@_Z+bv}LR23b|~V
zw#aUpf$pp1`Ab6nhG{=um9&+o<;E7L%BHW%>P&mZ4sm<0iA-PPQ(TyS`O5S=tr^#{
zB3RB!C!X|+UmqP8Z1((vqC$yPhOt^+uz9hO^T`F~o8OrWi-z4+;*j||iQz%EX8);k
zou^iuX*zQ);%$33$D@KR>oi}e%+5MxdA2VvxHL)en_X^z^$zCK3ff#vSL8pJDY0yt
z{K;tb8|NFx4pnSfB)05<%2Z$Ft0LT!r;5)kSa`)`_qmh^vv4NHDPKSR4^>;4$8E92
zKg&PH|2pqo0lw19W|x^_XBIi>i)hUbim5en-r3-6-N1Bi0#m~IuP!1LvR@TcwpN~*
zrn$_a$KY(eh&tEo=|BFdgd{7^?%)YraJVYeiEXp_sWYNg{ioRMRrQuS+&Gc5>6MVD
zf19VLn3wfNVe1ZFyA^Z)32nCUwhs`SXY$fEFKufSqc^{j%{B$!OYRLn)fU^eaLZ1}
zKl4S>{`}&IbK-MzbDygAs(tdlbWHN)l{I~3jFYBGeci(RV~WBYGqVL|(#%&|SVg1P
z_<8JG_D>|#LfVv}#hjh>L7?XpO{NBGHl_B&<o2D+9KQm+3j){gQJdAFbazIv!khmp
z%8ynDJoOHF9(e!K#CHK-JH2~D6O-R>yPS6NO<3@vZhNt&JlF7Y-LseYE4C%Ml)1HS
za|pbCs95{Gk>rhZ>q(IpSEcKRf4U=lq-7oV>(q>mS3B30GEFq|PwjW+4q>>lAuc;a
z?LsJnhKNS|^;qu%6BU0>lnjZRdx`h8X!*(qJm&>U{svn(E4Q&bRdh~QP$_l%XR0f9
zq~d*G#w0h_n_)~ri;Y)!CUmXWo~tywtWrmkEq~_wD^i9@?UG7NJ#6Jo$wCTT+Zb#X
zuBzG1zvYxpitn*aeByO~um4^)Q-SOJnORJt;n(aBEtb0^k#^D{t#pOrG6#v)!n9g3
zpY}qD$Nz86Ro&tA;>wb$)!X~Zm^3Fa?Q;LNOH}DfsJnZGG-GL&pp-f9vS|}CE<XCy
z>U}f*yyn#JGurr*+6{jxYDDQ8%~G(b3VOI)ek()Acdi_V4IPSy*4OpFnI&9ma>(B!
zHz=cU#%;j}k!xN1H9{<oMdUv0X5B8@dOZ2_rJmB>FyZp4QOCq8SbIM#`!eI5^OMQk
zO>D7tLUCLMtG;cv{VXv<PKAe2a?QkRMq7^h_az7To2#jCWHny$uMT2}yUt?85cl*K
z@9U^i<16av>zUt)mZ(2ab^WqPzsf~Wz+U#XVqB$?00U!zrQ$i?hI_e*yDv3dTl7PK
z!$X2&W7-xggV{QT|4ZDBe{Y`qCb42l2~&@5-4q@^l@;N?XB%_Rw<xqdu;!r^_pF^a
zXC<rIaDP4&_9*S&VlgpeZ)?7}AD92@$y^(d=hbyPSEQoPQ+4Sq!})>{?It_sC%&Km
z^-EH^ahCs&HS3Qi7%64TTQN$QGW0pMG0$$RzM*PyYfcB#lFpZVi*F>eFmy!w#Iz^}
z2Rmtg$xxYL!5S*s^-}PyTGFQCTX~YH2d*Bmesx>b?3$(o&!L%*mfKFVJCLs+yj|-;
z#bJq0N-WE#2QA_@{k~7NTxpHbg*y@H$Deurn{jkzMg)gtrv1$5vr*MXvw1IDm6m;4
zyV!Z%Lb2Fn^L3gcai<?R7;fSD^naG~$~-2G2VejFn!-GL>P72Qf)h4yKALl)Z14T$
zIX9;r6g*VGa4ceL5DWK5#WRI!9s$!;mDL)47T?&jKryLgS+Q=<t1aS-QfEI3+3{w|
z1)dbw6IwT#AHFkEy?4F7-byiK+JtZ0WzSgJT=+R}YRa_3>X!=5a`r^^%@*(}xY+Qv
zgh_0{W$PQS?o?@9+r^jZRL!5X*4JVo|4beWljv)@opMq_r++K#nx<PMG4(pr#`LDW
z=@KiJ>-c<|tq}LrbkDKqPtu)}YgJU-^Wv-aiMh&2_jD!t$}68fSYNBgvu-X&{=5~h
z&gVW|bx(Qa<l{OA-bP+kk8!avxuwde`F+*x;5AAAUv`G*t1UM?#(PtZ^#JegjnZcy
zsCh(um#I|Jid-ijxo+{Nl7}DGsVblRI5j#?qe4;XdMBrD+1*9buflcgRJYZ<n3H~~
z^7XL_<xS$xm8Kb2<vvzEHc581`R2q6_xN7V&J)?<y&-0E^siTcLL*jh{X2bY_mzXz
zo!$4p>%CsA@VLA}t50*q7h4vV`bKHTbC#T48#88d^evn1D^t)kHFVJj%S92!Oy(Bx
ziR-IvDvz07nEYbG?m&y`h9l-%^#ZNatTX>=>|$oLxTV38kmG!7lHIO%_P5@Dou_H@
z>lFW@>l<Yi8x)dv|55s<zIQ_G(#e1O3=Rd`A9}NA)4pdHyPkghwEBMp>*5~=f*(!l
zYf5I={{GaNFNudQ1lB85t*vl7*j(j)+gL{N$_}wTm3=N}ihfq;c0F74E8^>)2qw)X
z>upP-v)`BP|5_6JV4~;kq->{2vLO%JjWqQuPpp<-Z)thb_NVjLsZIgse_xz^^N84u
zWM)Re-5<XG@Tsc(5dCw*?fapLlK*sHDgH9#<PbYk>wj6zeR?U+@}KewEIEP-3(g#9
zia8+=b0UC2yeU=6X-dQeMW-$%5uutB21>_TWo)@TCMk5X1g(v#O<6HXQJYa;bz97j
zBDI4v3^+F(?AWa4JIk7LTjEu&=cn4NZ>PkZP)oIr<_+E!xah%(tE-t-30;{Ldh*);
zR-rvYBC8v8LRm8|FPnRyjHy*q=26kh3$2Nri`B%QsaA+<AKuIKYnrCvj-JFrZKq2#
zo!gkj`GVD4rU`{Ho@C){QsGcgV0g|l*IAfl@1!jZmpS@osc|fLn6-a{Ark}B0<+xP
z8;iG3<2aI^e@{p_=hqHTCfAcp%OXPk3Ya|@&spj{*)Z`}aXY`I=l9c{N#E~0NMt^s
z<H)7G_~GT%-&Y4cJw5lm``-8!Ar7bJFa_wfa9!P}8?>BL;hIzEUNb3Iwy0DYM{cf!
zIV@Zr6Bwcd!d4t$5%)XL*cMW7(3LaBWy*|>Hqj*!oTY+STsg~^R)nxK9pFe7SKE@|
zBxyb?u<bv0>Whp+NzJW49wtj|DpiaUW?R6JK4<Oolq^55EwdQ{f_4gIgvv?COm$Fc
zaPSOrS=cHPBr`QCYkg{EY7XbN&IME116|x^Ci^NLWht?8zaSQTWao0%+pe517D%dI
znc(IvxMYHppkmk54sJuQz{R|sA{w5a84rW2-U^yb_BFmL9$dYcOW~D2%S5A)<vJUU
zruc2t44LATJHe@~F5v2=zU6DTD7G?9Ni<0aOP;#*YQu@7kf~8;)|EbF%FX-{v7n0c
z*TvWyY+({f%2H}el4Wu>9!!}!QRG0{0#*SRhR2K-RI(Qw&3@Xr{-kxP^V+-G4*OUX
zKg?#>?|X86CTHAN{r}GF`cn*YJ#!4XG#5X(&ZQ|Bv2^`b5!F3Ghf3<Ew73WhERiYR
zX(=vSa{2AaiS3sHU*C1Q&naN+Y;69d=&W{Wc=6fWwVf`ND;|9ptTJciO}MyIuvYY<
z9pj0%8W!6ec^`N479s7woB`bH8TN6DW%7Dy1U4tjcZ#*8%Xcb9Pq1W5l5|oNOL8f_
zs@u?A8@NTu(csqBfOzwN5*ED{>vk-fppduY;Vlonmllrit9lK?I9KglI*oDKv+$|Y
z%tM||3Dmg~cXgq^UdHoHmK7n*!Wxdx#8Tbb0_SrdeVJaNYwU8&G3Cj;4A<+OEZ-N^
z8D(X-@GI&EdQ3^)b#&btF45rswemNQ2)mjF_QwajTlh;W$W4^>wU;C3fwom&W_nFw
zh|JfWD#FR5t<Zc@Xm`$LE}aScZ{-|xZA_Len90DfSzuzxk_HYNuWcKqO5~Nkj*f2N
znRhC6=2?z}*g28coo2DLlwalMWSxE8>5J=z+X81wlu~-L*OYFo?VX_D5y4>{;Bk0$
z%FA?~i%xH!U729bsgQD#^HE?<xoqc=mx}Bk1Ev&&9llYs(wlQagV;8;Ew8_-WL(dF
z6d1B-$`6kvH6|N7>sUUBe5+g~w&cd4EIF2G%8ng{1tr~W4~$fH+??oAz56!PcQ*&|
z*rjfjj4FL>FBcz`(JL%46nZSPT;*hDV1VNPnv}ddB?_jcdD&|sSe>&BWzO%_G}S%A
z@k+zUYxRPa$!>QJ9Y3bTpsB&$q2bLp|KAq@#%E~?J8I53WJS8%|LRxHX*gxkv_~r<
zbXkKY1YOMV5Ms4In-aW8c$V*$^u29Ymd`RO<TTrvv1)b=e^2<%P_K+syO^!c;fp?p
zMt;~Xma$b;kKs!)llD__=A@Ly+pn%M+p^3o^qrt3ZycWS>F7G<joxveex0az@O9a>
z3D4pgR$StY?O5NzbTB^1>u%nrC5CNV3Km7YToE6&_^^)GCuLdDmtIO2PPT3dDdiUA
z7Vw-V_bTAI;<b}c#imHesr?d3b5P|FHvcqnzR=ay8IBGAMYdg#U)t)P89t>Wc;j@F
zPM0$^mWAgp9?{s>8gS8WZNOt`)dOZ$`*ib4jL-CI1x**zb(z=jcs6H>$l_h+G~AlB
zV-y2flw9=ua(iQf(swMH=P@tDi!&oUNw|BN##EzXKNkPz%f1NtEL)Y_Z@VU?aRaN;
zzm>^e9NlxJoPrhEUUfX&c{eauO`!A5xvi(Qmn|{)G$H=qyM|ADggloxw8gp>cLy@+
znG{~K&35}5dX4#lN$k<Cxeb$RR(Y-Sy2XCYEGLmke$k1z0u{lDJ3K1#YqR4&&G0`|
z@OqEV>+}>SZ_x`JA<g17*YlbTRL{vS-KQDyf?Kj_X{(7?@%oi#dH?<wW%U$D;hBBG
zNkps7vS+F1W)V?Y>!^c#VLlayf=;XnwUY9>pW@TX6Syk$#TKuo7M^JlAJ@1_EK*m=
zJCdq>x;Job(3u5I>T#SC4r$DqHE*R;h3Z}A^pN~TlNR@0+#b2vReOg)qadSxba}Nl
zqv56PJGs^|hl|Pd?n-ELwo{wf&6A_-`uVZZYPCg%S6Udh7%Xe-aa1te@^JQ<J(JG$
zHj1p7aQ?8FA)D@mwF|S2*0`A5d0^TbakeN+HNwMtVx-aW@)n^f-RgIvy!8FcrC-i<
zn{iwqtMFI(iI_sqqC3%XuAl9KABTR~b#ubXX{V|de!7ysc#-WA+wU5Wa$J9&Gmrg$
ze^Q{<^|c=Z|D166{A_b{_sp;ze@@1o4*3u>@8g+WcS_fvXL-@W`66h6qV>gQN6kgj
ztRJTdoyu*>_<dOJt$4n`g$*p(Nj!qmM>R|WFA2Fv+}Rr0c_3ug&U;%X+<&Cl;VI@L
zBoHU^V1wh2KntIVVrD&xbyCc8)%dJ!&Wb2k{qi`rve@YLhe@fEuG~C5r)Bd;6$K{G
z2~3*5ntkQJ|Ii5FjXI-gWT^c6$+PIU&yJ`ao3N%gn2GJo?br=Di-n$iebM&jb&SwS
z#sfx08+>jl%w2Am-mS{O$$YU%O~;3O?ZFh0sQXMp|8_gA`n~;O2LHyXZ@TXvTIIH>
zPVwt}v8D$xeg9o9mT2kfW~O^b|6;u~_aBF#`rlhV$}E8=Kkxo9<N5Kx&)2ReuomSx
zs7}s)^nBK|{YmRn{t7)4JG!%OXQ=eMVEHeB*X`CRu$FDl&PkSE)+jJVA(i3Rn;&It
z`*!rtxuO{UZOi<*MQvunnzi~H?sb29^Oe;}<Un6WSB4VTY=@^+K9>_WMW1<d%`2eY
zcvW7ZcYOH<=I(H|N6t?;>J%mY{}<c5D~uI0OgHPB`6pNBr2bsBE^ED6msUNSH0MlA
zhF-hVof6mBE50u-+FE}XN}ZCj)`898x^io=lJh0TkOdYOlFX9SBKlftWmdRyZm0=N
zNaWqfHTSv?+qK4rjv3k|YXA4C2Ohp&yH`CUba~l=6xIA~tg27dg93sUhN`UyiF#~Q
zu5H#*$1c6Xw4!;F_gYu)Sz<m48#A>X&DWUv2tO3;3(vaQVzT*|(7{&UHEP)_+Ok$&
z)10qlrCZpxt|@fG#Ow`0+Lxb}a$n>VR%WzT)q7%8Jxx&R;x)-7f-K@!5_UJ{wtdsz
z|1thnnm&(uo_u2fk7yDXdlF}9$AqWVFP~ZdVqyu}z!`X;p7$VMkOJrS&9w?)UgC@O
z=Qww8Z;yY$X0TDQ>)}S}pJ{o&!lbuoBy6cD-H{>W9vH-QJ*xbYVU@U?*;4_*FG=#@
z&c<dT&O4&&o(IG^wQma$-ufV`?qmIbHPaNm=gFpx>DQ(RFMY20Vsr750Pgryx2+en
zKQ3zA?VcuG5n&XOG1;Z`{fodiYzm(~hE}o(ueo4wzr5z-kN)lzUAjw}vX*xo5uc#P
z(Y$qH1oz5_$B9Ck+e5sUvTi80-gH&q*u_XCW9v;SM(mEktEMK2FjnjhafuG7Fb(q#
zs1THI)F@yUoMV>7tK81mA$tFER^*G;$WYhd&-E@BW3nnlw;EMBJ@kDO%+KQJy0*bB
zcaya5B1s`er_k$`SxiA;57i<sNf~YlkTUCEaItJ2OY@A&mPQ>6+*?^^YY2)i<D9ue
zLGWv}MoMkL%BFQY92uJpf-X7zS5svymfmV1{hyoDX-1IH(?GfC1!;<$(uWNTmrqpR
zuc2`*QBe0<o%UoGSB`q8(g=&mAx{hY7=8$cx5zEBH2L{)W>iQ_cxiSLOB>e-`Lrb|
zYZLM{Dy-fb%{HA9Q6j|oDJ@a>FxQuDoY4h=+blbcCCU?bmYvX?&UJM{_tOb2k`qNO
zCo(&-&2^dU{xL}JY1Dz8A!|bnG}Me93#p$HoOxVAb=tA0?T?duJVK7YXqmXFr>faO
zuSqjCsBcr*{Nx`#-+%hVMYJt&5=mS!IcY{#{Ejw_!nUVZvO~6L7kO|AJ;*)KIQ`x-
z!vLkuFHUoKzgs?QvsznNz|p`WD8M?=WuZt^7dLB7R8ijlipu;oiCsK`h8<<ivo^3E
zn8}hG-ckHA@J~^Gil&nehg|i_e1>9up6}D$c1?(F(i9S4ed?%YGG&JDPS*M|fxi|;
z4-<`;40@P08riVUH!(?`UMbos%9y%nzQ1c|Q0p?MU{lkTtp`I*=iCso+%BMD*cbOS
zR8w*G;ZDJcEP3JP0&4_VS#~auU}Q*gVtYSr#ph+LN~!Tlnl<XI(^yx=-+jJ(+O0VW
zOmpx2OzR4o8+~A<$W4LTXK6f)bG#p>t!hmWX`IUvXjT5bcD;(d@@DV;Z7tiRmOXcm
z?hTu?XYxEf$qJpDg7c3FcvUU<F?0U$s5Xg>zF~=#H=iW0XPUBoTlRmSg)v$O+13fg
zZflrQFhgkV1%;`LyCz$$RB6$F%9e02e1hyNeM>VomxFvgvnIS0O!)jNVqKz;^@(Y$
z;px{NtvGJEjy-e2k7v`jKh$rW*{u;i<Bn#F#KD!kj7ybP%H_M6$#0&)vq?ZhtY;_J
z{5R}{(i2^#Pn)T9Ti~<0;3)}Jl|}{)bGEe;O#gkKxp-y0!FS;~8NJ5d#Xb{-9~|-h
zVQIDN(X2lch5mLJmd>nwyfEbO#mcn@x?8fWrcH_WJstmYmD9P%t*R3@%<5{k@Mt}v
z!jS)L1G`(fsY*`R$FyrF)4G>UbiSgV^IP5Pgm<9Qyxm%YOP2<J*y+8$B}%0-BxptS
ze}~Q5xm}z89$ynvv4(|f!9@23^POy+1GBX^YMgSl-BUK@g^NJZ4Xz-CIR+=zNlPt!
z^(|h|RcO|4r@L%HJU<2hH8GrR5i(AZe`CN_xXj?r_ojbKcdD<N9u%Fya&w`9htyBC
zl_%Ugb!K(VD^c5PxJ$Bh$<`aHo?4T+UN3o_Sbou4kdJ$#JL9%RU0#-Hs|~(7OJ$2j
zbS<6xOwqbk_PzMl<3CmTm#174+L69XXpTj;f3q;dA`u(+1rIXjJ9h7Uk?r+Qv?=hY
z=LW+Sf2-$wzOgtkY2}%f5i=MkCSFS8me!o@uq{q{Rd!i|&+QFvQ|1aCT*dSuNUG49
zy>YLQ<01W}&i}1K)Fg9UHY&Ruh_ZhET3nGcxjsB|>bIyvPXtsZyQl~Y9kvpByK9Pv
z^AuZc!7Ga=Z+^DVopE`h%G%CvuGg}(%{K~fOW^m++_~)nAG@Q_CW)H0hF+nab7mPI
zJ9Iqg5zCGg7q+UX&V2R_PN$X!a8KXRC^h@?QP<sz`ZN+)vz?N^PY6G~#Je>^Yg&}^
zmkrmF_Nx97RC%+th-oj&#toreqAHUMcP>^xTCrVzca;1@*@>J&eS(KyHyRr*+2?nw
z$msi~zt<}*85LH^?9aMYANAstkMU*>4MBJ2EpL>!u*yi!T*SHLz^RNc>AQPs!!)I5
ztcp-5DbiF|6I!|Ty`_(c%(nlR{><a+PKZB!;(E>5%Q0t@W}moWF0=SU+Sg?nA{!4&
zds<&QJulefgp)+ied`kjG8(5Cw<}y^jY%-y={iGOy)tG;)@HZ1=nDBGrlu|0hO1&?
zUmM%)5Zfs^aqZ&mYq?ixYYNT@`W?W&$G}-@;q=edhkh<rdJ^z`R%glquB;mUavLZ0
zu7!?K7qdS_T>XCR=<#D~ZY=u!X!@3n4PsNfTRbHNWz=uoIV=5oqmzfrojqqCY&@}T
zr9iz~I}f9``1if;jY}<Ddkk`VCYqgDyzor)g-v?TPvxyz<~3*8wln8#I8z=bFSotE
zNw@n%#N2f=AFaxjoo=~H|7_;8<34hcx6kZ+c<}$yQwJ~o37R7{X}(waOB=QwEZY=s
zdM2#CQG55`YOAyIe{R%VyDsb_(A{}xRr8563!}C;d%L_)S)tmJ*tOkj#=Hf01Y#Xn
zPnE1+a%}CgviVn9L_K7ruP&K9ce+Sjp`GtSp-q=!bWg91U#cC=God+6)$s16M_#HM
zZy%HDy=bwj%yo6pFQMJ1_X?bjtucME{^;E44Vi&eH5-~{Zm{uPydz~*F#FOQ)^qQ?
z-r7@}5FUCZ?)SZLRfhMxS3W$ucTeX21mE(ndj&qrp3|4A%l>9qc_>42w~T7$Nz2k(
z-aBOXT|T)jWYdS4vUes*?cW#?7_#g#$L5HVQ(=a)8nZWVR@)gJ@#w$sX44&OQnntr
zR(SW3PRW_F-ZK%ut-MWdXuK5nc>BnzPjDxTeU7bw<Z<<#A@kH#Z)~$VY>>0`xQCI-
z;#;$7)(aVMP34Y><cRradHeqzA+O0onHAX*idD1I<$G5PecfQ1#ut`lbNLC|@qVkz
zyp4MfXXm|Nu}E~*ZVu%R<uAFyU#D$)y|b6gS&_wSLq_1r>cEzJs!cmDr#MY~ayI?*
z+pDT~?v+e?9I^4l_qF$L=&btRdq2YPzQw;q8y}uXnN`ZPfKftPHNvXrw&D#h5se*I
zrHvI*ud{D=aZ7*eoSc+9Ye(Pfv|W?^ShbS3$VXk?RI~T-F+Tn2t0J_x7xCRWqaJzl
zztqE*8cz<&&ACx`G3ds->u+SFyXL-~_V)TcxtsUx-W~b4WAOv^AG758uif1KYe{JC
ziAA}Gc><%>*NN=j${Hd3EHLmwYxL`|#;36yn-?s<AW>+SGC?SaN0>kNXu%58w-cTP
z{S>@9_s$*h^sJ&gQ&o<CSKXuW)JfPrV`E);@NNC5&Lw5F1xAvKo-QmE$vk^$pOAjN
z=z}-+AI+1#Xk$4&ZkykyPt*P#nd>KL^Z(P8Oqo49nza+_!UWa*<}H0+eo~up)AfUb
za|HCxA2c?Sk(T*r_-?CDO3??M=kI0Lwchx#Nh$7{{J~Rg*77d`8#bgZU*{>HWyi&n
zHe20x&d;Ljr{=BvFW|HIiYFUCv+3U&lW-QpFT93Q6F)rp_x5O?VfTEw-t#|RfBc~#
za!~Z}BkvnGYo0HCzE|Ls;`YY$r8eh&&XSi=TgV!5LbJ*H!VhVaxl9Y9)-8)!ClaTp
ze(~sS4*w~sc{}$oePsIZQTe~xL#t!Qy92j;EIZY?aQQ;R&a1y~UQ0N<e_bq7p8TrI
zKeA@bH9Z(qqy6;pjt7OG>jd+^#J)T)V>)5)OUcB459ROA(YkLbZ+fTZ{ye_?>GRI^
z+!6TXd;jzLe=qj`za(>x!O*Lr!H|hvPNq0OQL>4{GgnX{F+r%6OEK)ojKIeSc-bvE
zepnbRan)n?o@TN$@ad^ujmRXH0)zi)O&p#}GBU5eQqX3!GTij(XVLNiMw`e_JgY8b
z1Tp4xrTIur4GD;5&)&MpRf{QbU92h71OeAY;hVIQnLHZ8a&NJ4WVPK@N<A%RH2cH^
zp$75y4vYQGzg^|{vEFeX`^QfyGXfv1*qbWLedCM7rY9#R8HcUwn7K4<At$TH#G-&l
zuIqvooSr28THt+cn&~9H*Fg`bG%raN_MM<|@T_m{E#92P7t*gZT6W8u#7)x-Ie4f!
zyTr=imqSoWQP}&NUA^b3wrX9=a&KGou9<^P>14-|Jq%y8uJW@?GPJ8<G}QLGut>GD
zw)L~&y89>gu-r=KZ=7+(nVW;9;e@U^=aPzoZr}eZd%Ah|H2v~j{ID;3odN&K%N+~(
zRF!#@54nBJS=_{T=*^KvA>|cEno~kLkFCpDqv3k}L{y=|IW{R)C0(|zB=bvFvve0V
zX58-ZWYn<W@MO%B)?9KsXjezKwt3T9B|oiQ8A~*SSFPCkNy|y8k>$ayMN`YN9<J24
zY!6zs$l~P}?ekXkhl-WwPJ6xbJb(2pufuDO-P*W_WzQ#nlM6c5-D|gAa8fYW%$elf
zy~|_fs>@+LpYj&0`okl7#3huA>%x2~VTFx>%T{w74>q2?_1dPoEu{-~ut)?gx)3qx
z(`H?t;-WK?!%kmSTC_{#)V8j!yBgciZ+Y?dtZcBWo^?;h?XVpS{}(ZJiAHG8I=6E7
zErULu#TP^u%xm;2-OStdEP9Ir!x`p)g<mghULVIOCeonv>reOk1V)iR;tw0yY<?*G
zzo0FVz^3lk;CNTB&cL2)^=^ZJ)oiCe#7=Wc(_H!dgzq$+$aLR8cX4Sc4aJ%9Vhdc_
zs-{>To6*=)>e|f3^ebJYzHO%498n{VW3wVZy}5ot?60X8qr$8Oi&n8Vl&NZ3PYUvh
z4b<wqIem_8nYYpLBPZUhXX{#UNu}+|BHuM7LT}DF+8+CTMerq0m-_N&Ht#G~&RF$o
z)tW`yL^(A-iFSSLQkuJEE#v(|CEUJmXKBTR2HW{5hW-DPRlIIibk4P@)1S>OKJ{NO
zSyuT>T>8eZCu@#$nEbU%?%by7lq_Vbb~S9a&PGq+-&=b&&HJ@cMPTVP#Sc4JMQ-P8
zeQQ%SVQ=lqsSjL^udpgeblEOe?Nz&dZdl#d?TJ0lPH%tk<!AJNg^r6R6ZZaMGbwQR
zJ9!1?mYrO?x{tdYw_PE!L0jc?v@1u&rgJY>YOeCW+hg|0Qu3i|hR~JBK0S$pyhglD
zF&71$I<`&FRq#^k2=ctTZkhE)%a>Uvy*VdsOHkE5IDz|I5a*2}Vm7*|EbXfWCoqV$
z&U3xGx}N=FlgO1+?ztw<inTI(?#@#>YU^w0U%pEE+dQLnJsisdxFkixA{uw_YpE1v
zdtUyewY>Ge;e}L@H78_3jULHbaV_XmS{)kdrplm}rmgwp@#O3cqEjXxGz`;H=2~~k
zWA&|{1*PjA-%&dz(ApidrowItH}}ObwPf8O)f2xLY?0Z@8g_%Jahl?TFHBpv+dp+t
z_j{;x>Kv;c_f|#?%a!gATZ-*||9!0>8Xe&yaYgR0pt0|VNPz?ImMV0cW>h9k>{f9%
z^xl)oVbgF%+N}J^x^*qDd=;&iYMng96=qPNY|!*{rQh?N!L0@95^pXl7ONbZ!FkqL
zqwC_uS3l43ywjPt^tHlSzM$zFGaj~xN(#AeC=(EE5{h7Y;c+u9Vz+eOrA0A66TQxq
z1onj;Da@MGaDeU6{=gUiReAz6Mdws4y0m!Dlg`*2g$E)lT%8zC6|CK%5_nF-a(nBQ
z`l5y1%lGY=XgkeycD2o`HNSos7WM@PxJ;Crz!1DBLMu&`>8Qf1Ew4OpYO;jKzs*uI
z7n-@kQ_<YA^vQ+|ZL@CvF{tQwT>a6&iFMJQP(|KHR~;Rc8}Hqi=zLd*fobP<#RF;^
zR^AdmaPrjDiDn_sr!o5EXlG6IwiA0^xlqyHjc<zmSDxZ}zDZ4NkJ@%dNtspG?Yj46
zhh^mDZHre(R<2!<uv(xz^0T$iQcbrrHz!Xv6fjsNblq*$s$G|6awz}ukv+$>>wIO@
z)YV!}9GC7{a%IW3D5b=l6e(FQsQFf4(f?hO6nW2IJE1A~KF3yN^7RR>_k0(n1Ul+G
ziq&pq5PA01c*54B_kK9-vn=&krtr#Jg!l5=yXz*+eZ8n})w6A^Cp^1!d7^@!F1g@l
zH04Tgo$ggRQ=cg}o@vdQw}(rP`LV<CKPy)XnRL{BR*GGCgQILqvr138`Lzv2p%d3l
zJngt`(uL_3nGso~FNzONu~BEWj9etMI{3h2DbeWljca+rtwNc+-`8q2Y|mj$6)o~r
zjN{oNIO*BWhEK95ABj!)U}YG-W2K-&o`rJux}4n`Rz9`3EqG3P<26US9&=&mtI1m1
zdb<6dyC~*=-7$6N^04@C=Oq=zcOE~x;Pj?hf^8ox5)R$^&(R;cW6ANFBbOBxu5k0+
z6u4#I)wPSK^*p&3HhUr)kBJr62d{~Tf|#w`mU;gSn>E|&ionHG;es62_57C}I*H#l
zTpydY_J7CisV~f@u3a!Oa8g=~*y+o+%np@>ZT5Ye(r9|DXUWI30(L<TMuy!^FYitX
znY=A#;;|DMr9!dAJMOJhh+N+o$=7z{Yf4C?&Z-5@Ia*5}-<ZP^P`&0upWOR7BGD!3
z=TkXv&f0mipeG|(L#k}L?WEG@iOp*Y_{9z9{jZ&HFU)Yl;i-PzyFUf3xg?x6F|OoH
zyh%W^@X?IR$2{uRtX{oS;AyKwVNkiP)B8y!XH`tkOxmqF`ODM>U$Z{z%liNCmPx5J
zsc|vw_;tmjS#<T6vt8xJqCN-jEw}UD>fg@j*TniyA}N{WiPS2~z<cZ5b-!62sbh^z
z{I^=BcS=Xetvb)s>1%dLAK$Cf$+*f#?#a~40^N6|P5)d9K0E0`imiRJ0_*N?Z=SI5
z6)ju&YswVOzX$f@96!}wn5w5*C(xH)*_V=-JI7M)ciG-Z)x0{-+xquD241vOzP9Vx
zZEO4BhW^+cM|8JbTx4zhY!P3^$)nFNwLRPHHeDmb*#6x%aU(8+_*qwVA|E=nnt$n^
zvczk#Tee$xQI^il4~0%Sx4n{#?$6RtSr~Y_D$-f_%0q*T>i$>Ma)JzFPIGG+t=+?E
zHecZXz7vzJ_w3ef<xqd<y|U{EZ}(x2JHk`X{xqMq>45gD?bB8XXF6L>TOj#X+wp++
z%tOrP>xHzZ?$~%rLa0NcyEuA($>!DHvbKNyZ6_fmEfFNO{zg~W>W<Z`ZSDkithqg7
zcY$#DFW$K~jQ>9}erUi|thlo)frC4-qv`bwqwclWq8&Cx>}}+5{uH_PTK3+)&s=w2
zZ&<|Te6z^;cG2Dm%?BJ9EsqFUZ~Q!aN~C&k!ScEy%hm)F?yP-$tG4ZU?d*^vsw(KD
z{N$j@!!4^mZj_y(EyE)gyHatdlm7Gv4LcT@FTE^k^iwF_YyPi*PT@mSChpp`=5j~z
z<R%S|MLTbZ1Z|kWa^?S=s6QeWCp!fuuZg|l8pFc1(CdKH&;6poT#rBWZJmAi3fG#c
z9S&?yWC}iQe*B?x_v-@!HCpR`tT$h@Dob0>b#b5V6hpBohN550(nAhTm1sU8psCo^
zA*0;=_SK3_FIS{U%AK?jJH26hwYTfGlN|~o4E1-q)>celXXO1asmtkjf-~?$iO=!f
z1t&6RsqGFp(c*Jr*M<`gmeb!X*xPCCY`fd`R?=RJD;<3{X35TGhB>>ff2s99>Dm<B
zY504Wrt;oRFWlZf)_S^m)vPtkr(K*m>&r}+$%1ozWF>w&F67`f+GNt!<i7sKtoOf6
zK5MU^u)wM{V0M*Am$u{j#)$sFGaXeE|1+!)5%%$1)O=!pwKRwDDvkyp;aL{TW-X9u
zd?IhkwQ^_3V(*WdJ3|&XEIhrlr$c22$FUD0>MGs(EywCMJFv`hI8tMq@rCbLj9coO
z9kP}uc}#ReH}2TWXtMBt(iG2~(vFV#PZpfta#F0Mr%2?uq_un9DvpVhxXWF(`~Puy
zuyDK6=2O#V_?~_suWWN_<B44x4sg`#EH1rc*?WOkbHmaj8tU^G?8uGkG@G;c`5KE!
zA%`0;SQxS_zA3~|Eg|~$h+pf49cDe7?-y8@ZRVO_EYGW=GI_Ib>yO#Z5(gQrMY)Rf
zDwDQcQ0chRvmnLNWsbMYo8q1&Hm92}=zaXXL^fphhfV*FeDhxK5yIhj;#ALr25mv%
z<-!dCECN3y{ocP;jSuQjis9U{X1(`m-#VTXTb|5!UC1HAdqU)HfYL4#k-e)~ZZtW0
z@&s{g%nIpDOF57==fH_C7hM!3cNHH#RCFLsx@)hsu0ta0{@I?vj2ihmdO<$~HP>!D
zsLWkda_l%m_rym>RDX(YeWX&k$MIaCm9p?A4h5$*)+_Fx33_y=SNx<+uMYR5J?G4x
z8bmvqeORMBwdRc7Ux!%jCHBhdFDe#%f3ur2%FxN#Zt83GMV?z~xV&DjwiD@D?)2rn
zlqiGhiVlStN4$USoGxHBal)*=3a>4S^SUOTt=@9>qY&4+D=Y8wC@FFNzYzKN_`VR4
zUWRj>C$8Qv3GB@fDT|r0WkN^TfjKNs`q)Z#-!DFJEaqD3n~Q0p8`Gb1$;5Ut#GFyu
z>yTFMkbBlTOLgO&ApgI&oQ41R<jvXQUd4HN&!roUzS?{FTR9pSYDGLvPPg6?_4VT2
zw#e)7W0y-4%>A?$-@9VCPRMWFjv3;D9B&^wy^1+^>-V{?iW{%a__f<|ygPI5?UIO|
z2{(GT?ma%?hQ;HE-lGw1uFl<SH+Oy5`%b`_`%758^VCVo*5}S}OgXIOs(Nhou1Lo(
zGEbY93#Wt});_gq#`SbT-Zr5I|Ile=cdvi`r6>2K$$QU=Ij3AWO!dRI-fF$M({*O&
z3g!Q<D}p(g5|5Z(z1o>DyQ%i#)Phhq!E2XS1SW3@UbNwwJnItc>TA19H&53%qbPc5
z@>ZW5(;$^r>$5dl$2MH~x29uy(e;AhJL2AVw&zaU{b_1bkG5|p(=CO0_h-7-soqky
zp6Yt@s;d+~@877N31Yh?SffuJz5nCJ)QH$uXF4X#jp(htdvj`RPloM{ujjg27j$m+
zd)?y9!z~+n%G~}H=L{2v?^}4Be#`7_IPb!ET%hX8Yiq%BDJj?0A#EJD>{PsZH*}ZB
zo^8}Q@qst~L+u1kgW#zTqym0#`NX_+<-!Y^D>c4J1<S1I;PB<-x!UoaDQxDA__a2|
z>z{fHp1mOWReaWk|ED+{`n^7e{^XbtpVfVIG56Njm${Z4NL%`VJMAQASeV122PtbF
zOsnnTJC|^Lj&r8$rQ5%inI0WD<8o=@V=glW_t}#Z^Rg$p1YR#t@jc_Px}}NtPw12h
zMH57LHCa8HI43kQ#`cyn-#DXs*@bi7meM=D2GLB0(X9eAE=?7<H%X`5@bHb+h-<Qu
z`gwPIC6cFUBwxFF_g3qT8&=5^Ts%#lT(gLC6aOdj$b0vU-j!-|J)NiW{M-_m<vo#W
z@fC@_hsS>iE|pX~{5Igo$6L{@*Bb+xjz&*XzIkwB!=CqAyB5x#<on0&X2Pz@;LiF|
zq3=n@6_ecmB?SIcnD9d>mbdickAVM^Yi@SkVK~QB*U3_6zjf*Ql`{=E*%EJ5@0Axl
znYb_7Qsh}e+qD$2uAQsb-1uZ;DQEcAC#W}Jstt=xvqJ0ilWFe1%+?(^e@xX^y18f7
z)!um3ZjO^GAA_%M|9V8D?3m`c_%4aHZ*L~)U+XYDmVQh0vB|x3(_^tl+tO?Ort!YE
zy}NmtjkBuKTBrFM%d=zFOx=BR1Gn{&QeG!H7tdIpaKV&$E930<gp}mybv|>S@uEMl
zWqw_xoTfp%O4zh7p&CJ~TYdDdewwYg?bf8;Loa2DI4fsg*!^qizubiXp`C7i^S30p
zD;d4|%pt+TmerJ)u!4EQiZt%Hc^CH>@Ck}}?w0*8nd%#sdgUV5yQ~dj+5Bp+d9HD+
z(d$l$-d4Wx%$c5J`)j*$+7i3;7CXgt=0Cevs>;1o@|KHd)7{KHHD~YqjJ@i1#qpa}
z{L-Vhj&&w&m^%OX1Tl`EQ$Iv*uK&h$BlmIdg3C2!u~QG^)ZL3PW6M3u=V>D1FKm|B
zFSFpKr|A7C|9ujAcJ~B-rl?PMezK=wqSsgd1xt6f+=%0HR8svJv%cec!_8jC&_}Cc
zANiQg?!5IxN3&DEkTb0_o+<R*(xvf#uipBb`qVpFhQagcd^3I~PNffN$Kzu~BJN%I
zS1{vvSk}^S(^t3&Pgs-{Z)UdkUI%B%^qs!XcJO-sJC^^4@Ab`F|D&zn%}H)~rtJH|
z@#3);6OJB^?a4iNzWC@vF88xqrO&4;Fu!3sAsZ=NekA75xetQ%w@>s*FZbBn@a~O%
z+{g2Ig>`Z#FNJ-SJbzc}dTj6MoTXW57TGrv+%jyO*IwTB_=wTIb5FRQ=H9d4*0bO)
z$Lr>uvo2=v?UFn=x#w-HOY4uAghv5e9$c7j$DwMF#57f^HK?QGP>Sz7A+3aX?aF{f
z4bwK<OxYDwT$x?$&XzUpL*n`i@e9|9M<0v&8kb}JJ(K6C__oM?jt8H1WC*Vo^Dz;Y
z6jJ*bta#>6*|odepOWR$4ovOQc{Pty{a{gV$=-KdyA_lhIS*;+ahI?e1ty!T@B4o?
z>gLOUdd?8_=;J>&b#usH-}+{HLE-Tov9<TUR?EE=k1v$|?#DC5A|-xPmTfj;6z8Es
z!hX}!rY*=mpR#<qqSwC|sW%Qc?w0X-URLPYA*sE@^3Jn4fh@X9o<4hGSS&BJ-Z$;5
z9M@5~CtOzbkFNg7Ow>w!cduNmSO3esfMYN7jFnuO6ie0}y%i|3(xkBa^{$SU4##p;
z{gdClPdaw(uiMSri=uOnR!m^|(DOY%?d-+ZM-K4zRAj&5nto=t?7`Q&*IknCvkp4`
zeUWV5)0?uTZqsJlO)WboYB{|$Sy=VLhQ2o^EKKyibv})d`Tyu@XjO7~uUW{igy}uT
zM|L#X9p?P<U(H_h(<H@vizKehG~JUG{^Zcr+!wo_Y21IY>3z>r?OBEIxc()ivfV8I
znkm26CP{6ji;?)}_pgM$*>XO;CL}oDC2(_j?ZfYlORp~Y_Gx0Dn7m%e)DOkh?T@~7
z?EjuOn?=Eib0uqBv@zrLsuL!!FR=f;`2WwP|F1Tw{k<&yx8Lz~*8vB+AivK3ip+8|
zjadpe=bb;)x?o!Q!F3vPMlUS)?h^UO`{mqS*(dvMh25VRwD@{}T}Qp>pZV|aJQZZ%
z4On#K*P><1KPJ9^{#Kl8W%%Pw58k|cUwZwy$F<_km;GL-otgJ#&#~*%B0K(-s0*dc
zRrPXFcB$xEcX+Xh7JG*w6Pu9Ee+`EP3_)zHIf4p`oeUigVoDJiA0HiJU|#BBaHHU%
zdmD?h-V=if-JZ=F{#*~7E~I)l$#@9ONxG<dZnDA79ZyUIk`A&kF$jogi7dL%=*g=m
z>b2!%K$8b+$eO4r4}w=kP2>=WDpUwx7<16;tyaRD$fo38wGM{BQ`6ekL~T@xIG>=~
zQ}y|k?uqaN_xCfu=Mr<c@!)Q2=Wo7k6+0dz?{45^Th_Bt*71y3uHch1l@FdbuI1Ra
zDl}?K{ISy4d?{ylJjiI~;+~WgTyW*}Z83AfCwnh!XLxX+SXe)A&c|iCL2PCr_bxs-
z^7tsTN8{-W%Y1t*XL{Ga-IQ3y7_w`BkLN#||G&OG7gnj{D=%3cHsdYlg7^#j9RK`!
z{aM~a{(tRX!7uk4{=8*hkoZDzMO@y(O0$RzPO&ACy0aohW;2NCwJ6r~B;Gg{!?fk&
ztlafyrwMd1&#idmwBzZsBu5TO6Skf`2Q4mWq&%<?=uJ5=W3i*vE7d2C5{gfxB@YXV
zxJKCrx`;Bk2#U-NVrJZU#mSTLz&w^9AAx(>(gG^WX7DZz3Gm?Uy&mIpTkyLeyXC<n
z5=@*PCvOxp*M7Y<*I!8FV7Q>jmh`DFdsQ_YTliQss+kU44X9anu{|<wiqsOtW--xe
zn#-R=X1rYf{ASa`6lTt{TTWYdEW5ty=vJfKS!^2%7Z+|R`|mZcV6W%#*^~B96ue#}
zZ7C(_R(sEC0*jZAA?xhEOIt3?DPAQWv_-5j@}9$E(IvT(5{#ue(;nPA6~|JrSN&EE
zlhL;M^<ug=t|z^n_x;`O*J2v$8@^a^%uJcfd!<kJYe<Hc+w6bZ4DHh=?RwZDu9_3Q
znt5u4V7EkQ&+RTdJ{1*B7p@13j*FeFnAoHI{)dv|;u$@H6Flv#CkRD!c29H@)KoaS
zi76)g?Tn{Ox3tdSSsL=z^%B>C&Mj`-2EyBHHl_%uyUz?=8hGT@RuPx3Als#3)r&%(
zZuhP|JH<69`RA>Lo*L7iWGp_WzHr&JrCkv#D~|daWc3`q{XV;0cUyUKtLuNU9d}Bl
zYYZM8E8S6$Qz;uaA$y}%pMm4*zM$(zw03!Ptvz`5kLB{&e@@AIUY(n}X6k}+Q^uGx
zNz09sz3%^-TX}4!$-LRCUh`E6ae5fK9bXe7As^48=kk6p+p5(y4acAFuGxEO<8P@?
zmuzP|xWAKkjrg(+6WuEBahvk&jGOuTp5-sidYMg*4xc<GIJ^<x`O$T*@=4E02lwuo
zDAGI0eafN$@u`<}T{zb?%U1d%O}b-xHuHAjt;!U4ndLfJno+Z7IqFqR<mURIB<m9r
z|Lw{_78M6>CIx2A4)z%ruN)SirWEnrC}pCA*z?x;hp)sYt#E#Hp}@%OPS1WlqgCB3
zX^B%h|Ho8Gq)!Mt+m-q%`{KGXVOKUq&yq#AoBHCW&3(1yd34pq2WNIIkzp}w;kJH!
z?sC@hd0A68s<B+0CbiLYLf+eTeUDznZU~q$GwjY$ImT$Yeap;kt<~0BH<g5c;(IyY
zo~vOZcV_?}@8-RGV_1wAZ+z0WkA2^%*cQ<>hDym=%fx%O^1Di0zHw4dX}|iu_sJZA
zl6`TOOZqzf6!&zU7Mr9j)T5Jb8K=Xoc>0>0{#-$2N8tm8I~h#oE(z54ntXTWITg8d
zz9Z^^g_FY9JzBCLV(skeRUKEjuAa^RTQpnYiI?l4plxasduK-`@yASZ;R@<GWzwpg
z6wkBl{GrfiN)64@`~QmsZ$HKnQo!tc!S3TVajAFN@e-Lak9m%3F?zhSSl|=*tl;v_
zrYBhnvS074y>vxf`=6GlYOi<MtgGTrJsdfMGW-SNink?bDR2gFIOkwCL3S?R5tZCK
z@gB)RK2uLjY&>YARdu6z!mSK9yEDvBKF_(X_e60;RfkDjoyZiA6<(ro3ph3Izj51e
zJk`xgTy?6tvYH{|41Zhuq@`|G-B<6p<`KU4SA<i*ig4{`McikEJe`|TC-yzb?C$dl
za}g4_+*mWg&|$^Rxl*T>uf5qb<9TMsj43P6%$RbqajWvJGfbiy+<Ghl3L*@P0#46Z
z;-R3qEOkv#)-us6b(fVA-f+E2I~6ZdQ0A#vmGdMf>S6ccgrdCTk!Rv3XwR)r%TGIV
z_PK59jp-bcDe*Oe^9~0kYi%v->MTxGeeo{KdvaW1iG!(M@v3HHJv$eMO}c$~{j(UZ
zJzb-}B|ELkL3;OW36?%aMh3;7EUY37Obj{<3=9knOb(0;9RC?uIb<v@h#qWW7uJgL
z*uZkwh)3DW$HL3OwJjxzb47S|f_52`i))POge4A>g@af5So$s8(WuYbB_dJ0^z3Z&
zP)`v}zvT>t#?F^`tb|_#%v#*!S97x{=|VTttSA``Q7y+60ZWBq+(VVFPkJ5EDRp?=
z;)b%Vw{EUdpK#W}$U-tp#3EcVrfrSTHl?jb`A+)_Rd<OV$og<Ftddb(=jtj^Nxw$^
zB1wx#25~(Zo#bt2GCw`u&2&blWk-a=?8YVR^On87nCGRpnw4jrMc4cIn-TeWf(OcZ
z@{P7dbFVPX67ibc5V=sqVf7P>NvrociT}9sBXkC{>!&k%A=c$H*{7EM4}T&1=kzCg
zW$qC3346~?f6sb^-9KQ@uiro3cW>94vgp8(7S<p4OPOMic*V_nk~+=p>kTD_g35^p
z<(=D}rax?zxOYTcb7knVfUMX6wMR~qr>>C9V0K&<pviSL^txBx29JO&rh}=;l1BH$
zV^Sn@k3?y|?U|(D!6Y;NVXf{}mjGW~)tUFo^j&Apo8%t0Q>Dyi<s#W!r^3#u8uuE1
z-gSNMD;ZklJ8xyESA6NzWnMu?C2y3hS+lq`H1lTC`R>An8PTz#T+=6YGk%&Wp0qsk
z$@HW~xg%aVT$<k&CG|aW4Uk@Hqjs}(YV5O@%jYp2pY64BshIQ<?Z#&dGuH@RdEsL1
z^*`jI8vDnsp4w@gPZSE1*DSi@G==ZT{8DyF_uE>=irEuAU#PlVDW7N?t+3_GTA|6C
zOy-8jZ+Ud{`u*+Vei^AQJAJ+5W1jv}UFW1GX~CsEtLLECI;NkSrZcR)@JuYehA$<0
z!(%p`0+$9;H?A(hP`Ae&%&i&$snHc;hT;WlwKN>rIHzo6kltPR@wjQ<w*?MMj#_=#
zWtx?8-qA73Xzn%x{Vi!uMM2l6?hLj(o!e-m`ZYMvuG{&U>*F7Hg2K$*!#>Y6dmJ;_
zL)Vd4c&3tSXzBF(VYwl$QQ{d_&U*x=i-!29`Cd4~zVPlc->8Kw8$4ILh|itC#dW2R
z;hV*a{FEg-{<lhGH+Zv%YpZcbygbnO^XRdK%(h+=4o+LTWD&#Ud@rT6c|{Cb;&UCt
zESx*HY3*vA$N9z5wd<qlYp>*vU9%mczjVLWC^y^QRd{vnq}`?4=Zie6c)mzI_gaUw
zs9gDe``LdgPPfng{Q0R}zr`tT-uvQ>!JVg5AFsYpRPCB^F|}KF?VT*?cm^Ssf?r3s
z$Jc1jS@COvlU!BCVg^~abFN3GeeD16xark(FBS{sKe|uq8#(>HBwSVACa^>{xggTz
z$&_VFWqUIvSp2V4h)r4&;VUA~B0OV_n2=%Th8AU!$?Tic1qF5)WGqhT3TRO~{PxBN
zb-U_?e6~BB1-RC>-=6LD`Tq=em#2m;?H+#=oo{_nn>Hg?!6(Ec^bDIq)P}=dTNb#B
zWgnba^`%L9wS!=kRLS<=2UG)?Qs#Jxw65RB`y{mb_xTtGwq0U^N6d0L_+(ixwKY98
zEP8wU&a;Z(w#&VKrd*bxPBzcleyh6Ft(_H6#B*FAYVDE~rX5owznIC~3hj+NBHX%Z
z&Gya3C%q!>Je(z<yQp*3)=5(xdsMXww@qL@_o>37`jqZ^G28G<&!#%f;xuS~c`C_^
zziRuQlOI;J3y1CSwk$oQWc|Xizb@qStiLPT?O$E&ugfu>efv(D+WAYf*iNQ7b68K&
zy?gvz<5Ydc)QY)JPX$P}Gk!``i4N`jar}R)=9~BPwL5)R1~hxvDQyy6esBuo$C-;8
zd{3C4tVxLr`{q#LxWyvivx1v@pou_{u4ZyU;?-l1F7ovqGFS2Gl8D+7WUw`aLBIE<
zHtPp3!!sco`mFxmu7a0^-hEka$GghA^;Ga8rcf1k-B$q<yDtB`Q=(v=x@rd7rQl_4
zDN8j)uel~gth10@8zG?+<*FyKc|**{uBavK;yZ7g)i3+J_U{YUnEP7cjb9!s+@3Ps
z;moB4RRRW~%Ck8ZzDP1++;?IETW<PV4&xcle>WC=kMT$s+BCD-XPUzFlV_fpEZ00?
zQj)-3{KV;thxC4p4Z4g?$u4EZw{E;_wTQZKq;$^_C&m9a!-Th=)y*r{R+;rJY};C%
zbs8<+F>h0xcxDPXsJTqM^H}fu&cnU$w3kZXeJb{S*A-E{lI_v4&#JEPyqmkUbg%Zk
zH$mll?##_AKjj@W>zCP%y=!%~W_GSqax9kJav(o)L!@usFY`I~t}JUfd~zBS_Yrxf
z9jlwzH$B*$cSQbumCVl?M~llB*cw@a*R++YFdTD8<4&oV+i8<>{=}opF8v}G?f$>?
z7Z7%Fm9p%0eD%oCZlPE5k5>hZ(n3Z&`aSuAp(hXicywUrmS)jO7n&HHq=F15lq?On
z*5bQd=c!ik&r@M+dcL`@;(U*MI~~XOc{-~>n(FMEr&Gi>&pOZZ?8c=n|IcRk8P9pH
z^W5O}m9q!-8W$v=PSyRr^K>oSm-(CbT+%)K<pNXVr-kvi*3P`}to7ETuE@<EnOiSC
z?#`RLdYSp#1a|i$jqD#y9A#$sre9O_uih1q^XJ=g`;AV@&1Vf`{~U7LvnI{G^3Q9p
zvM<|hXQ}DGI4qK>5N+d_$7j*H<w7ri(}7>2AC4~$x}V|Mc*BbCNm%_#R_!eUth0Wl
z?J$~{>#?Nfj<fRCJ12Zg)TZ>r+*JC0cjkqb;;jbni!R?j^SbZI-mi1tS5Mw~|25wa
zufM+U8*l%9z$VVT|6^}{qsX88JkQGy{qMcg#Jy*ysJq3{$FX@&tM=?X<@>!j(<|=T
zq?$b+{)?8M(Rp+xV_oyCYdw|q`z0Ub)de5mXDeOdxwg2eLwLRIbPkJ~AN8uX9ukw4
zJi6Dcao>YCFKVRws+cN2`&DwKaPcp0a@e(?wC2(J;D#eso2{SvpPsUJ{{JWUqpuiW
zI8wEQXNe+Xs;)Dei1**VMVs7pUllzunJ#&@-rRU5`}!DF>l%e7bDM|PbtgPAt5fLv
zzJp=Yg*!SfPCKU^|MS#%V&%E;y3eibJ6~w_|2&`l@5`M0Jx?{~vtHdT_j&Sno4uO$
zzZ)*cecQ3$_O0T*JJsu@&*q2i45%sIz2j=9XvW<n@!O|-{j#GJ<3&95*^K_~=!jBU
zlGb7W%TXq*z=dI(gZKY?&lcV4xbr@Lli&UX#%Evt@BH{~;v?nugyRC~#ytu%Jw5)t
zUl|#Y`7XHpVI<RIL-T{2r`;@aK9p>kBL3Sco>|dfPRP2ZNxMdnHEKbAWI?`UN8|Uj
z#$V4G@02wv{%BNs(a4n0q@lqu{d%sBg@CSxfUZQdx<#{5M6*dnv*8R@O9@u1iYBWc
z%{CG(YuQ_zHCmiFSY0w&98TnVb2RyBG_7^7U}}ukE3y8r>ihLT{7n|W|E39XKN6T5
z6Qmy|+))us>IhGq(Z-(Emb0VHE}Z|Ii9mUANwpCd=MQP|gKS$T3S4EYue;%vyMa^j
zn1GU-QgL`alS0zJEgc!*WgP#v7&19Ftl4D9w$X5*u(VDP*CY)AzKKaG4T&;lf*Qst
z6^D&JT*}q55LmFiYh^{{x)n_`#W`(%bZK*RuiH^IQGj*RjPAe%-Fr1y_nc_jYteHg
zqUYF+?t>LQ`!aeCRy3Wc=s7r}Tm44QDURO5C%UIy@7`h261*cdIZ>w2I5T5wSlpGG
zbI)tUg-Un|Yb_4;nI#r`yQx&O)}=0yTePTd$`!W_+mx5FO4KF0GpHzhpOkF=t?qrg
zl=LQnUDw=dS^}6DC69=C%-qU4>vR3|gPg5Xa_WC5lpoap|EOV^aIQt5qK~;iRkQY*
z4XmC9%?2-;*c-c;1Y0MY3mEMbFyfqS@&Dx{rU@)7I9TsaWA*qj*|D<8S(4SRa*F-S
z$u5~w0tKeny`18Ea!P>Z)S$?z{+Uw)E2ny_oXV1z7WpA>cWUeVOSZp7i~k*MjoDGl
zP*!X|UEt=EX%h?E?5?+&D^CBu(f8|ulCLX__zvk`*)0D2u>Q{q74DrA`<=O-8h1=l
zEMvM*8uMUA-^m#he;OoRXZ^HAdU9i_;NgxQi3n~{$<uAgOo3cW7`tRV%jdVs^P37z
zGtsL0?yz<Nr=UX5UA5kuCnlZaXx{sxNASSxeK))Ja?aT=$-1|)Nw9%sbL5;OoXw{q
z=P(t_Ic6y!$iT{EIQOdO+<=6+=TA0W<75?RU}1VN_rZV7c@KZieb~w1!NB?`bBch&
zTm#PLHRUljPXohF@SO<f&I;|DsTN+awYF$$ZL~yqnQ9haNG;RETF%4;c8i2w?GWJ2
z3TJuPH@8flYgPiISUYoL*7CqQiGw*mW+WD>O!5<7)Se>TI#a04P=cd1poTG0@Mn_P
z;(B4hM&X%}|F%e)J+frx$ki;As`;oTDwL?>zDW3^75nFgoD2H5%-FOw1-q_SnME|a
z?wsVvIWJU;Av|ko#IB`Lx0c4-n!@~WDGTE~CdXybQOjbgmL+NldUUkNWdsXO5ef5{
z_BCKytVcq{t5%k6ZDo>zWjB{+t*B*USkbUcKxmafOV<kbMZQP>8_J}L)crFM_hv7x
z`O*I7b9*(TeRD=B^TpEMn=2QrDxGIFle2MVrv}5En;jX`1&*W%2pn8ow9!N5aB@*N
zTZ%xG^iIXK6U;a4<lMV5#b}4XIZuIeH)kLHHRnXs+)I(m1ShZ@=bU>&a_xn#rS~i6
z+^<|4*0uIV*Sz~D*Lply`{L!CV;<FsW^C$T^(LofKB?#{Ug|r0nLuiVz^fk%IN7qk
zyz0|*T;DcR=&p;*+6NAqjplPqv=nY8w<lT^FA;3qwBg4KF;S-Mi)Ir#Ce;bdUissZ
zh1oat-pv{{8acXNmT$gjPBAo;F)&>4NkQgP`>Tze=8l^C(uCJ?imug^R{W_@{J(IL
zch<5UKPPi-oXw-i%5-swwWWZQwB%teLHBBb+*Qp=68by7>Lq8o{<z6)T)v!f*|e&z
zVxD6Ozg7wSkXj+w)h1ln*1Bu`<4wNR8i{W=^oun*v=}ea&)&v#nXAW9Tj85k*Uy!l
zie;-Gtz<bk<44M>`qN3PUL`H_6vz{pC0`{mp-B9<xM0_8Mb_>aRmaPZ6jnUiwdVW7
zH5s>dUd>!~Pm@7FVb>yVRt|?<Oc&Qad^y+7bIu8mb$)5<o^{onSY9*Z*}DId;l->A
z&W6`s7hiwi_f}CU!5>;#sju7GPc6{CxV2#CR;`a)%^epAut~`Va#u1-Xh|EfFp4v4
zPEt5Lq2>R>tcgYK44Q%~iWYHSjF_>hWSXkr^o0Vy#rJ=?tU2{!z08HA@2@o{ewL7u
zG-#jgAyVXiNJyJkLvT*i=FS_+?1|hhi#MCi6nMXLnPaxFr?<e3$|l8(#^X=76iIDO
zO<Z0nbtqEWuQ_UZ*R4GzJKM^?E>~S6bo}-q1DokJQd#OcC50ak2|w)P*mmfv1NVc=
zQk9?eI!;Q98!ozdY+;eK!AvRpa&5syua3-LRT^O~Sto6|yYT3WRjXFMnpssSFrh<W
z(adBi!{kXBtN9)Z6g`&;TjY>pu=%*kfhpD-ei(7>jht+|dh(8yTbLit6L_<0S4QRc
zwDd18wZBD*#U62;xhym5KWF$ej_}210v~^!VCPz&*STPF&!Kr`;cRRRxy7<(zL|DU
zg<)R+|GotNeH(;WW*vE@xX>(|KV_o!ingd%8#ipcB(ZUp(2|A*XP<~Qg&N9T?jonP
zz9$`->?AScQ-s26yD!;Ee<$x(I&HcB*P;p5dK(h;{%J^PU7VE{vvS@`g(uDuYYe97
z-kj7WKCfD4vW-WY&WwYKD_BFW9lT(opBlL})AEpV&fz1kx2DZrVd*1~%DAo3YuhRr
zgPtgJS;0C!CdFFe&5FM)S(a>iv3tGTwC(&#{teoK;<+7h<^Izru9_QZ$g*?->%mQ|
zr5@`XE-?kH*8Drux=m4BQ8D4@tle72xBgEsdSBVSDLp;tTklap*{!p~UQ31VeeJvJ
zc9_fF`FyJq4$MA(j(e|I(1Hu)idC1F&-*5Lt=qiBdd2ddgyn0letW?GZ^KoSJ6Bf7
zY~vJ@;4|CvTH-3F)r!9Y?F_5MR!$R;k&>x<q_w<xuTbKVAA;K%&$uuB;66v|?00j4
z{_GoVvWtH&w&%}S{5yElw|4uR-NL;h5{BR0&K&7q=FIhI)7i%%g4-r}rZ{LyCC<P5
zV``{aP|`}7pxyCb3zmP=IizyuyhhKpUop4MV-D-<T*$kj^vUBe$1{icvl2pz$C-K)
zcFP`C<yqlpyRGEpopaR-IF^MUX3XBVVH=Mnm#pW$m(~}U{x{E%FI!b@EfAr8w2FHb
z)5jw(dsn^jtzUZUh8%AI2eaUEMnU6ahHPJsE)+f{U@b6rhQa2n8LOC8c28QkWC3eR
z=X|cXzWl2v{@(OWzqz|Cb-~5$CofJHn60zkd|}_o>T9_-?r<D)O}cUChfdbt1nw!`
zS8qR8SlVO0?tsL@!;k8=ZZp}p`kzDKpHo*PFWph$Te$ezdZ#Y^N9xBd3hmRkDD!nc
zG3k!ts}@)#DqX+TooC1CiHjsw3*MMKSMi+MCM6+5k<3lMev7sA%K!J4G?}EgLFvG9
zmID{fZtY^K+aNZ3Yr)p=*;}=Bwpl-Kx7~NG&T8ScDXm)zT-PM{ewZZwr{}rR|2Nkr
z)I6^K$e*k$P<5T}{{-`;0bENGxKlS8W$#`cKi52@@9@3cLpD7N<rb<xc9FQ%d!Sro
z2IHqq>+V@Rece8*>u5~PfiK1Pq<bUTc<#5{p4sgiu=3+ki){u{iCPEWlxCc{q-3lY
zDJQdilKEE)nKuq%|7Kp99;5$2{ZwW0sU(h9H@^2>v=z+SE0ASd`|ki>Ra^Jo6|7Mr
ztO5pHqPq?+ymMQ5pP*7+_*MhHWC#A{R|>nWPNmg`XWxFg?SRCx82-t8g8Kxn%{?c*
zCV}6^@1%e4`DEFTaku$J*aFKI3qCYgaQ)}jc(rc(!;kw5H^j<VFz(!O{jyeF<C#fI
zFHSprvGc#JQh)c%Zd*x}?)}1(lR7@%Hxpd4spr5>oiF)1f<EPTO%WDvE^sY!&^M@g
zaigz<uj$Sy-3#UOUbOSw4S4^shVR0JJzS2GtOZ)p4>EimEET^nv^VChe}CvblOz9P
z0oKJ2z6o6GR(ki+MelC46W8_x{>k@VOz3+QJpXOA_=Zf6ujv)fb@u$YxcB+JeL^j#
z1aE9pke+y9=Nc>JS7JOXo=ul}HBENn^v-*&wMi2fm&#4L_;#(ptd&PImY1=8S+zD%
zAiz#QC;APG>Ki7-@=P0n%o_2D)rzTcIyno(4o_To`5fPE@psaJwF|ZxRVH(nJ>a^S
z&;HohZduN$E8jD@UR|mF|8BvAJ3k)Cx82>s_i88CubU_KF5u)?z`4Q4diM-gy<p!r
z2XYrbShq!hlY@cv+5Xza4J-@`m(JT?_wUujtmo(FGqmy6e*LVvV5)8Wx4NyfH4fZA
zYV<ZIF(-WWJpEQ))-T30gPwX#(~aHNvQtRad5W>L*c$V+DJP@k-fr!BIjN^%p)2bK
z&QlXaotN<3Hp$Tvd=)a;qcLpitqGwaOEsf)W208_u3VDZTol<k?ewj-=vzfI4<&L%
zPoBeaBRArhj?dm|)-Sr&U2iV6EtzI^_hpfA*yNNG+p;;fgsu%-lXfd*=Y~Yx=(RaJ
zHz^<JTYYYu1$W1Zfa1MpculjvNZ(d*yt?kc@1@mS`@;4{c`VA1=_p=!FwkSm2J1-e
z;3*o5-PT@Oa@4}Qe({X!mv(qHyw|z$JF{!iO8M;SZyr0F<`ylmjb4(Ineu*K&F>(G
z>;IE;qrcVt`zy^S(zD@gU1sWvuoFTJqAZe81*!*?nm1^31`A!V=MK+eUC-_7)Xl|s
zA&J9f^PO2v4Lki>y1R8lQdaN?e68|Yx_n)hh8tr}$jT}EPboCFOsU8z)X<Z;U7)&2
zNH{@VCDds`kMD^O9GlMGl3aY~j9bzPImR5xv%HM&3Xb*~-6?p=8-76Mt!?z1N?+SI
zSqt06$|4feMXNHpc#UUDylP!ol@ZzQ7nLXNENtj<QR)9A4Worm{Z!mmrtkS!^>X>8
zIJZUxq0{?LPnlMy@wR=gPS0aqVGXyHOXr6!xWHr+ShdofTVsp$%0*!{QE%<*>%30L
z$KTRCD%<w&;-$UST1gs<%%TtJ?4O($rD<=w*!R@3%*aD$Znf4V<_d=`$zj#_*6}D`
zqb(!l#DS?d6jE2Lm?y<`Pug?lMz^(6SG3Gzjw+pAJIy7hL%6{?#xr3mi=2h3nB=E$
zhV9!Xbg!FYqO(a+KUw+GQq_;XFFiJkDQ9mCG)h}x+@`U)Xxo>7$8WT(_mnuF5!z*t
z>|^e>`LOZ%ORIw?8ya0y>RPb5Ly22s(+Q^o55wZBZl*e?F5PtM#>SQZ#az1Wc3D1Q
zS)t0M(c^94SoQ0p)ud1tNmt>(4ATzL`)(iEEmc?~R^-)(btb*5im<$8^dT%zkNW_t
z$1X)P&xIR(KmBp6+4uQ@^0ui#_msaqS({zG_T|(#=ggOTbBd~-&RP{4nR<0aiN9B8
z&dW7MK@*g^@3O_mPndG!upYO2wXf6mM@w!UI33d_o44XHe|P>07ry@Q7b*m$(^j}V
z4WDG^D&ns*uSc={Ot$Km3&Ea7E7SJuST7`HwcquA^rIaMRimXOjqdqm9pCx?#XI9g
zU00q6{pmiN&@tnb!2zQw2|kHguF_E-CrrMeBpk_bi+SRNl`3nM0?Kz?^LfxB8`YEa
zpC>z+y-dQ1+t(q0o4=<;V!~(j;M641VyV_On>O@)^ibz|ydq>NkCH5hfx3{umNTA@
zrfm=X(wx6sDt2j>ASXvhl1-9B6W3+|PXC9#+>0Va%H*1Kz6TuXKJn>M_`S`mnfESN
zV#{Q;&YS9H)fvFBfy-;#2bQ%{zGU_p?iAbR6J(ycFil}aV8{9`PxzyIda`*KPr6^(
zK7ZkhqZ8{jk1DHjp7fliIB7~xikdFxV(&gyZ6>P;o@pr(Y<pkp|BbruE#zi1dCiKc
zN3;dMaFvNqXE^4zJ*V?S(xTH-8&s!jc5+EixiaDC0hX!C)jTa~&ATtnY$<hM^qp}g
zz>aasBeBH)zqrq4n5hNrxTvr-!QJ-Qw!XJFPAaeoI?TDywk^=)l)vhk7Ku$0G#0n2
zhkngiIOo+tzNj@yL3=Y6dYn7S_cG(5%B%(_5vA61jS3}U4`pu4y?$u=vEx;snkI*>
zn<-n>5#KQRw;Hw!cFa}fWl>?e5af00bAQ3cV@u*!9u2zoxPL;$lm3+<o^pws)}$|0
zDPC7NtuA2K(vu<?+$O7718j;;F;8t2{+M-LQ~KwX{S&<7{4WZI<%gVId#GczU#W`v
zp{SFqUaj-^dXT$AO3l0Jt*cAEZ(Pulbfp~^LMs(+#VH3Tg{^<1=OnJ&9Czuv(7v#q
z>H3iYf9_2d(%O68^o^$Qf3yGU48}7U8rEer>wasWSXRgrX367seZhxSGF%O^Nk0-<
zW*T>GJet)sGr@Jv1mRY-WyRJ>3)q)(T(W%P@K9yasw-A|*j7GrdFZ$9<#CA`=dDdo
z3KTbJWh^miWYGE+63Qvdx?)W;i$;di_BSVPubjs!_-(@DXj{+6QWGEZ7X*22=Pk^4
zthpK!z^b~UZs#@MnQc<N96^erikuDmPslOVKQqqDU~RgsFwKK)o5(HY!(wI|mKN_k
zq&y>*Gr{D}Ue`(9>t881M@T9qZ4hcblGLlxb?}zn#z}`JPT#U*=OU>o%XCy@R>Z`0
zSTyX}w#mgs#9HpqyG>qOH%+P#=w2Gr_rKggfbphNu0@XS@#B-&4Oi+MtvRWd6E4A+
zFmuKGgMOzjUG8E$=HnB}s_?!pkw0neP7~|mPJ5FsmOvSE$4i2(sV@%B3cUK$I-r3`
zyGi-^BZ)2Y5h+vEbTkg_<y>GXF*m;accQEYPnOwjQRBOR8ZT&cw1}NaymI5+ldV2?
zPAdG+Nx#cy>+1bGGu$(D$?DiHzsUKQ{Yzf+`_G;kJpbMG;tfAbH#r$ehSfZEV3*>S
zoH92&ZTE?rF$IF+OBPH|R`%0UEL-*M09WqDqT4s`-YeU2(Dq&a^xL<ei*4UIca`#i
zFY3;_R3>&lva)G1^}lClR}}qa@ATJ^`H2GORx=!UZu`G!HjC|E;R6YdF`<EvLXth3
zI3?d4zOiEGM*qhvJG5iwoMfH9lUq%oL3-KF8^&5H?U#$f?58w0_!vlaMZL&a`cI`n
zjw@<s<c^r7VT_Fqt7~MFBKr8ZZs04Jc6s@i6SpP4O$#VmHYZ=eA!tdS(aWboEus$u
zqe^Bh=5)M%q+`*871s}HzG$uZJLj{i$1VHNl^^#Rwu{#%`@ghspS$YK-aD7#>zUtZ
z@8hoUt=He69W(I(OV*}?dRmO)RtH%53KTzf*YA0(T5osCc#cMPW?_fv`$zJjQ`h+I
zJst5@>Duy1a<`YTYrhJOZ}V%|`gMK!Up38_l^%LOG|iHjwM*8EuAcCpwQxnvjKjq>
zdqVlnPu);5V^XrEVN}qX4ZpOw4;bA&-7bGr|IMpSJIfYXolIa4+B&n>`U3Nku$W+m
zTXUYt+8mx4`TBd*;lj6te^@en-%J)=<dD01%VYjPhFw?L7PfA;T<$Sb*d<YP9p?($
zOMTBeq!-WT-B~$*CWF8t#oZo}t$`ituAi6B4lvDR<hymDW7lERw40Ov{?PnzTVEt(
za$@(&Eaz=yiQ70XZ+lz0jla^&y~&KPm@{MX!R3z*UI{iY`h8GwkE4>wAtjeX%3sW`
zL?06B?qwF7^mMVga<ca3hSi!ToiA4&+IdLs4X2&l7RS?z1*Ih|q#M;v|DIU(dIRsn
zuKzP6n3J=3P6WuhtlE*v$(Og0^_~Q0^^Vq{7Y%I{&EW=<F9s+~+OT-~29cIk>*^e=
zR8BBeTxvQcF<T~TszyZnnZhY^FDO1=#2$H2cMoU(jaRHy8~7M6w1h-1x;4w{X$MoF
zhv*cGBTr@sUyHDA`mtuRwEpv|#U7EnGZU?MUDmX{xOIMzSo}rnE`cQsn>(12`LAtc
zWeGN5yS*gH!$!Q(=4kOAy&Y~RUNfD#?AG?<*i6Ya2Y(&rkF*iLxiRB6_bR5nV$p^<
z(i`_#7*>|7(tEH?Pr<gPL`(0)@wzqc8QHcQxje32KJJmVSLF7R<{sOY7Q?>Ff?S53
zOC9F4zcC7X#PEN6!k#Wcqb9={wX-)&nXn<^7Q;lHiSo(ZyB)aIgE{vu@YvVE%Q$nN
zN~C~cv*)xAMoFjkwHZwJT`=+JBdM?l8d)<}&fL}7{d0$&z+`idoq4ww&t26X`hj=m
zs&<_Q1}o+{eVb;#`7mqggae(EOsW!EOA9nNl$d<HF?)ga!hRvvs|v0M6Q%A6oStT(
zen!F}`Zj;|OzX=Nd1JlB7J02&vTEDL2ZG{_8+2zJl=N{rxXAWsN=Ko~{3SOwc>QuL
zn0-dU#p#rfkE)N)85eU!pY7*LoRXUky_mRtr;_jGGuyR}Y?tfi)%#+ur?QX#;=B(x
zS5Ms`u~NdKEZV5;gP^=;r^f0T|Gyb>cP^HGv!knQg~g8|%?F>X@^-d{X}IeP95;&C
z5WQ2rCy4W{Q-gzb=W-DSjRgyzX`Ooz!S!)h&kyej>|G0V6l54bO!;`C*U55<_@XKI
zFF8+Un0o2glw~?=tzWJDy~}i`^ue@=ow=GE_P^FYWZ00B;&#B=fF){!uhfDAO3g<$
zUpRT%t?<n;UfcbAZ<+Xgxwdxr3aSPQCG3$%>CQ1WJgaT15VKdj^kP{_V6BLR-vRf!
zHSYB-fj63Mo97&pH{By|yEOC4TnR@Rqv)Oo#&*iLCp0q7c&<6G{U+0-IhQ7;T#^Ys
zIjzEOIfudt7v~@*u0sWD)=lu}Na;!p>gf|~_%Hd`^WMac8LN1vZ<zK~!lKQ9QE$b}
zIUi3hxiNdYB){eB{mv7GPYJO4N{I9etzX&Q7npElMI+zy1;U&@yFE^N%~~;=vD9>7
zlZnU?zqrR&rk|KI^MUu_kKRefX15A?Qkg9dJ#jo@<8=P@v7D@q2;2RqiqD*yY&7}D
zo}*8$p1*u1{_L3q)@zBbhx7`}l0{FP=P_TG)R`R3eWgZK=ShEP(Ant)XJ5^nCouD@
z_`wO&8F*TvIUl|hOgN-@e{yfx#(rDZb7fo?EMB)iH)-xSZm!go?E2vU$A>fGjjqSd
zwbq=wG^0eWdGQr3niV?7<-w`*zjp2Lf9Ps)%hirCz*l%xV$bgX!Y#acKlS$p2c*OV
zFr1#W|MsMG6`R}LDom@6=^0!+F<Ub4i5u_SnNfc)@SAeZ4BgQdv4=1AaFGbZ+^aXY
z+}OzfQLto>%@z*BMW!<r#a=AUG0IIgROmT=k=ZtHvt!Fm_q^zC>8-ccoV|D>*ibb1
z#Cl&3h1e4tLk)eRHkkGX@O~6m6>jKG;n=QfB=>vz-6Pste?&5FxHRR*i3uhXC$70P
ziD$-So0CdA&n^f!yx@UEP*CvX9iCd&Ys_?Ko@F@Y^IGPH1pkz%HF_4z#sOZ|3C6cM
zw$*$Ue%j#@yRv<Gk5m}LoIeH~r|&99XUFdDjM!Nm+x2(r1=*`NJb1aPH;d_*{jW0Q
zkzm>JF!Yc?jmq(yGsk`IJuTgF_^iRL;Q8)d_u^75lD5Vr9=)gh<jh$e=9IT_siAw<
zBuy}Fb;{_~zR|NiW9cDnUAvn*uI>7D=*Aa;#@DOU&UUq3m>8sSr|pHW$&;{$-kq~W
z&K*<Wt}a+?+I4x;>3-)uJP|)x%U|4MFk)S~p?%kB<sV0Q?JajszP<MTtMfL37X;V%
zZ+|Sj>&g1uixH_I4;hOuq}do8DsEOVjpW%IF#Ya@>AjHxtP^L2cFykQl=tJDHT5PR
z>x}8F?rkp`X6U*ff4y<*4LKp%TT)MtFPfTk**i+ycCSR}-V3kY)!iiOp19X+Ili{_
zqQcXQH<)k#R|rksnCs!aYMD{O#w$Mi9<09hw;@UJYIOU}ruLak``<pcpU0r-BB!2q
zmYeb1mJLfSdS`JSv`Z6OHpOw0VQ8o7#eLfx#HAA3pZ^HSk`!-S5dL#Q@bnX2dwT9J
zpLy5fmW*#`>eLN4_L=ma`*HeywPRRpsMMdUFGCN1kxYv6-O-YqG}+-^T;7?MhYv$;
zZFqTgb;`5Qy|+T8_r$%HeVl&oUJ|Qza;w?<t!byfY^$6&$H?P;HS_bWx6CK_aPNG|
z{~~anW^4SDgC50kC#P#1mj5_$7H`x;$p^e2ZDXB+kDQVy&26>lRqj~Pp64boQQ%;8
zX{Ym_HA^biiR_8kdwF-`hxPwcQZAV8Y-LmBWpUadzw`ok>dl;#NWQ&^ivPHk{T@A)
zeI&mvQs85vg4o0<Yzgw-$9t~GEe`Eod^B@*Sd_4H_o}mIdDeSZ#&Xa9m(+4F%HUd*
zG^@vT>1bcC#~W+2T1#GS_<GUIZ)0`x?XRgek*7RbJ304!n!ZbDV#zh9wwX;1awi@B
z-R@k--PxJxqISJwjotJBfe!Dk3Q4sSa}Tp9vU*<go^oqyw$lciGqFM<ZWmKi16a3(
zxE)OG<X~<~4W63%U}>tz(R*=WKF36!oz;7mRQG!7hn)Qdaj{$XcqXO2y4ro&?%Dnc
zY04#6Gu^gl^O+~yjobJs?!u9`IkykZ5P0%G{8U(ho{*iMZ^1Tm!MXA4lRB*S-q-WZ
zd-kuJi#50K>811ExQl&P7C)R77|}fCw0F-b#k`Z}{jM%vwnsVe?_#@~(<Uy+U`)=}
zFEZ|_dzkU(BIB-SCw9AWzdgnynmOa@g~+AvrWiar7x(f8%ZG=%53ScrPu%C^lvdEp
z_*l#B6}x*B=lM*vGiH~#H*p-dQGfPfkzAIv?=@qy#~W;u1^-`M9{1R+%y3VDkw)kx
zCB?vP64}#dTyog<ar3c{TOJhLFKl)`XHeY9C4E@-s43_ExOb+q=ak+XnJp0f*eRvl
zKh6H?`fDGq#Cg9t7WT9w;@~BXZPL|_p<8njbaQ2o+*EM8_dkL+x7_a042ie@&7Q4#
z_42Xj+f_l&EVmaI-3-0qcPK+HORppC&DpdB-jb|sZxdv{JepfLRp9x842i6}{aa@>
zY0kY=xj35XcwUyaPPA7in{nqq-vS0dojEO^KIYD`wN+Wlo#>Ji-tPAB|Dzk>@efUJ
z&Ogxo{!sS?#zk%!%FV}CC+4!ejE}!L<LcAtx(l9aepAr<EMxt0ir(@3H;L00+?<(u
zFJ*q1>Ga-jQ(y8=EM$NFZT7cs{3kQ#iIp$8+RcC6Z1J}byy4|bj@>Fq_E>RGC1Z6~
z`fHZjHy4*J%QB1eU<^2MMtV{&%M7>w)92^Ls~=sx+qARar<1clLS}#A7QWa2A9;Fi
z|HiMO{FZm6=d_H&yOtH!8mq`$G;+Chd1--ChNg8}T8?%($Gxu{N!bVQdz*=<-b+||
zFJW8d)6&oXBHqqP?=*YA;e?Hx;>QgO&AY767r)>Yw0(ZiuK$_+^)pWQ?_FE>?8fQb
zGcoR^*;ifE)4!_vwhKSkozSHDBl(dS1OLqUXBUHfuXi^xhkvd-BP0<%J>&59d+F0Q
zn9XPDic(-*+RbrSs?$E}y=dNhF|!R5x0d>H>B#xq6yE#(&$`4dClV8<|2fw3vTJ3{
zG><(C1PZ1aB%KNW@Mz`Dq<1&39=R%e@68im`DtvqHrKw%_7||<FU!AQQ1JcHt@*V}
z?zvyuT>GEz`QLlt<x;7y1l{)vxj%k1zkKDja^FKLwZ{u`@`Ks}xjQ;L+sl8*EH{_T
zf6aR{$Y{CU&UpgIJSH4rSh}Dig*kG9dSUR=9lPuf6-~bIDd_9n13BWU#%B~){*UtW
zxi(MbeEPlF>>P|t>|8pBT@yH)xWt8Y4qs3_+##eMv?k-D(y<;1Su2|psiz!`qMiCg
z9NZ2Z>|>Idm$LJds{0K6K&dq+kE%_d9kc7jnam66esgVdh4$WD>a%>2SE-(A)s+lM
zztnktYdycNxMCEzH7|9Q@KeVrOxt>7J<1-6Y-Ya9m9jeUp_pdoW51c3Tu+K<rr&K7
z)!wkaA&lu3N5Rvyg{%KJcCv0$@~KcbBBahf>BsH~XK(i`VVQJOrJ(0|>tcyKwX3V<
z=^kUso~U&B+%(%8+hWd^T|QMdJ80+W2@hi*sY&w1)N!lZ&GBx3yx+UzrGi4D6JzWm
z?Vi4x2k&PxGb*RNh<Q-XD7!0$=Y6>1&*v<cy;J7TvvT}#hjo|y3A;ys-_F!NSD!tB
zIYRDfi;%jUsK<gvPNj|nKlY-+gb+>9ipwqhY@vy*j*&MO_zO=9JQb$UwIfvBSyeSO
z$b4=`h_8a$(?{((eHO_*3hP{y`=Se--0FE4H3ZmG3I$c1^ZtHaTqiw6^+F_*kHH0&
zU>D1!Q-bE1K4__zJ#w;6?4Y|~+JA|uY6sFpHU=$YSh)1$g-G?ShPN`*IIr|Mt=*!;
zD!F8fBUe7##~<qA4NSX29JTMN1p8dk?24T(S^HAnx!>&6izQQ?zFk-%pxg3fX=_zW
zkmvk)Uot(C=3Wt6wDMZZitxH^Qdj(~5B|~$3<|s<vUX>l>(#aHs!y{+_rJ4Rvo=<8
zwQzru=}8q=x40{dorF)Qwr;8PZ3vG#7i4&08?#{9jI5?it$?H!&M>YWb9_T)#`j15
z(sM4Hq&Y+P?G>W{-8DHISo@DEe$OvDZCfs1a{1zS)#C1X*TuI?GwqRcV)Nd9p+e#*
z`+L_(Su&Yv(yy1_JO1gY`+Qd(Zt;)?&Bc2@HT)N2JMr+e-szYHArf&Fo7-fb-C;dz
zbxk7qtje#RBwuZ{(k(sq&v$IO<lBGZd0)&scd^Lg!U?()<fFf+OpWzlxP;-ZkjaBe
zmYv;!yCfKoopQ=Jc9TmpXa3Ha2|1H9ziU)1-ucWaYu1$SY4duW+U~O4Oj7q)mBlW*
zxTpO2j;hCABEil^qEQz^dcE$+9-A3^$MVHOYnM3j&iW^To=OY-g1nMee%bZ9sBfLD
zuPtY4^_rdMyR_f$dgT==nYJgxEi5`yRWvO0Zi!9!QE~s(sTpfSQx>1&GI+c7Qk-?G
zkU!UhX*ZJfr)+cTo?35~&ai3qI)|Xe^1B>T{mdrr$-Ug2pYVU}8|zPpZhM}ZaFXrQ
zV#PqUs;HfJyuB7o4a;~{|9T@ogXE{3|7zYfa<6>gGDFztPsEz$_p8pkaqKW+pBUh1
zY@Em`UgGrfhEKY5RU)^!$imqT9o|a?7^d$inIM$gsm}C3u{-3-q9umCZu&)1Ld7A*
z9~ocjgfm}?*|%L_LuRkS>7)}=RxEYZ(rw8Rti11f$H2a;qlG7V&WU^Xf~3rPj^#54
zUY5ABRh{ujVL{H~$qYf0_p2UUA3UR&dDp2A`;2xRG?@KZ{=~0}6Rt^FO6^A^lz-ni
zx_!d)66KvsRChL<3~2SzSz<KxjoPx6f&Z44vpwYWZ%FcTmTXa5nzHn8)^(Bpi5EAi
zolp+jkhAi1oaF0iM}LLLH{P7K+2di=)mL&x+m{O-tl2oR(XuBcLw&;6(7;Jto{JqG
zs5JcIJ(9b&!gnr<G3z~_V1_F?3!JL9*sXiP=5o;SxfhS}d8ciO9_EofVVj&+zIqef
zykzkWJK3WfTPx0ZOB5<puxSalsBKc}<L)lq#=Kx^<)&0iVF9<$YexRdKD0#ddco~D
z%XmYGo|;-MXJy)V=gCDYh3#cL1Sd-@mOg2+O!<=H>}BSy>bp{x3w@Mx+cd*AtCT<c
zZ}&XYr&|P$^R$Q-CM~|RL$kpkb@S|;=2Z+_<^n6-T4olUy6nVvy-PQsP$5}GRa5YU
zrIXbEWuj#UJtnM0?BQ2FJ8xd-srhDukW=65iJpJ86PU9#`x{bJTF)+dDdc_QXw$0F
zrAZN{UyT<8bDap#oT-z-xAfX}gEdnFey;gcGIP(xN0Ta-na&Fh`h9v+;v0qVp3`4h
z8pIc+>Cg0<@JNMK$garK#USBHy5k`w%l&CBtMWUO_ox_k9l!E8t|yaK{6ooNr=aQU
zm>4EFUn@V?8t}QHb;_KAGp80C?Rd0SB|%$3wD>?zgFw<w-AEQqNy8?sk|gni3MbVr
z#c}W2zMt!7r<>_XBjIgl9Hsw0Ig|AB5JPCkK@DZi7DaFChwlm$=6LY)e@#|q+>xQ+
z8F^B$-$F_7`j#XAeNHPn&bzT>{nk`PUnVWLZ8vN+Jsdbq`-Fm2*G#xy<y~C2gd==w
z=h@adnsqU%9&^s^@ZIoeqQcf2Op&)PbnU)(L3I0$g=Lp^#i_AP-&1~|$T_NzyGD*>
zUG4(jjtkx~2bBuWPDuHjrhNL=Nm~`2sWv+ESm&f4)+?L5{Ikjech%+peAEK!cg_(1
zC_V9?K~(mgo0Ed_G<7_~k5<M!6wX|FX?9X!)%2;4M5b^A`80}5UU6r2Zm#j6D^9A<
z44M*`C0*KmzEn%}oJ91y>^G*%54e=qRsRY|<P%ml_S?>*>M&hfOmyzif(I%EYk5l4
z9=G*bJlUPIHr;Q9^vvdK)9t_f{a^OnBqQLQsfcygJN50RMKV+7yjjF}ZNe#r-URp4
zw--sYntN<j@i{POOOc3Up?AENz;v!BZSLA4?m1bHS}$v4rS6uRG;fK@I?I)hBs{%W
zZgr2A7XRAmaks<u&Yi0{mwe+2FJ;Y)`e<f!v|k~o`St|$BB6-MoNIU*--&rloN<`5
z`jokjYcAWT)cqXMw)-v5$Q}q{Gn~zDv{b^_Wrss-k?LNx7h6rQJuH2<CVAEQj;qEu
zex6C6@?_?l)yAH$`l7OBH_JBF+j^*~YKvWRdOGd2^V+5Uv&*ks+pFPqPII$}QPw+2
zVf`JlrmJ#y@12)+q{c*^V~u&vIghznO2=BSKmE_ybVqq|8&A{9<5?2UZH0a}4ea^N
z=4U+a`nv1jyajHZOwVskH1e0;eSPXKhD+5NpP$(oDqS*jzU(RJYHqb(?50)0V(~xU
zKWzV*SFT@Uw>`e=Mam*)sgqyXd;a#Eo1K`LX}yxq=e^8v^L5-i3tC&P3tByRi&@+)
zI&+VF_<5Nli&-gq1t(L8gnr7ErvA4*cO`lY=P<o!4J>rrUJ<ygAl5wcp2iUwXCF4r
z35UXS-6r$4=uSA~F=wXkiTN`cJFe_J^vaQ~u=Rni!;yBLSVxX%j=Gy$SZ9~rIGDSI
z@uvpQ+=<f8qEq(YO*rv0Fly#~{;oSq_S~y0Veyc8_)=k^klKGm;jUP<pOML~3s=lJ
z{^!e4`<D`*3U40Hc=*F{f_`X}Ugpsyru^Y5tzk<f@4XGuzj?$Zz&CP<>(&5n$2($Q
zRqnB99N6l?AeY6GsM6+aGjG4Gn`l(a6G4|}jML06_GiBkbTpVCQs6lCYVhX3@PZJ@
z9X$#qIekp5w_l!R*(BM*G%;LJ@XDtXM`~VNu49pR{p0`9GGu#dfS6wwAKR3sr)R!p
zp3t3lch6Ia6yAt^ogrsB<m-E;v_72RvL{}B(|jkD$LASuFIAEG>#E$R!p+FZ@;}mL
z<<YSJS0j2g85p%1%^%GOWWA<%wtuz(OXx}8u%o`gsiLP7LvLrWNEQTV|M<_Voc3U!
zz_F<hr)qN?6U#UneQ<%pjz-Bn3T0E2Hod*7-N^kV$6focgv+T&9kyv-JSVuuO}G`a
zB;l^4u#)0@-G@Sbcjwp6y*c&Cd`_vKOP9FhC?*`dd*|%jtceTnolKE3I=}4Vl4Y{q
zE-BoyLi1woh&>8yk-as8$-{xE;&I)K{>P5<oagvP^jwrYJI5j5s$y8`(t>$TG4q~t
zGQ8kyC^l^qDL9aJAhcrUyaJuO4t?Fy0!yk{&Xg3mmT9zKG(BGU$HP_3rKE>jOYhca
z&lx*ziYKsLiS6>)ncA~6GeyiPWS8pvT330eik7deo_;aD0Xgy>De)qGt=~OGzD3TT
zR`Y+^p2lUj&Ps~^bDcKlS=E<TUPadS(oXMA4`+=zIbMs?BOkl|h+lbihJ!}~!-r-0
zO^?m1k}Wn#$WL6R5*s*O=8?-BPU+NRGkE$~+orPRHBPz7#AVg;=WSfdi<V;r&lk!}
zTGBB`Ky8v%(`}te(n_ITDl_BVY%aatslKFSNlnv}np4l+uchDQj9AckNxidQ)XqC?
zi+ubl6}O;UlADB^<X-s96Pusm+WfCaa+Bf|5n-c~(^~I|wcT3Q#>H~{Zlau6s*uKk
zK&d!ajipIEZiy^zYxXKJH=PSJyFTZJr?kgGwht%0Cmi_XBE6w^!T%Wdn;K{AKZaM=
z^0`YKDPMBLNAkbgj-$tq{!!Ikl-_NZDPNj?#$(z1*p+H4T_2P#G+6m$FYk;wzAd|L
zI}F9-UFO{5-8#Qv+p}!F<m^S6*<v!e=awCk<#0|(HcQelzZH>xX`w?!vN_Z8f~LhW
zG79l~jwVm&S2k=3+qwAk&c%1%_%1O^aN&`B{`72!;1RV0Qp($$G;<DGWQyr<9+}Cz
z#Q384Q<Z>EYa_P`1}Q9Du#NMfOVhdfq$jnu_rFNSyWV;FQI|#Yq56ij)9y9rH^?na
zJLl>ymwlsm$;%~kz1uYFH>OPYSp6Z=tx6@scUdCu#V5^nD?$#O*p`<aaUjvil2bPB
zw9AW!d5;tWy&O8ko(8^q=sM+pcb6NB%M$J<Q-aGj-n^fC)hxCpjH}JIB`#qLW4GtA
zUlGUqb8c8qz5HYY_oTS%PZVyKuM{l}%#obHFs+bfi^j7bfB89XDkv>nbX;%Blq{al
zx-XM%CGa_2S-53(O3^_!k!e$-gz{EOOpKYo=}peJt+8J;9_&64;P;`YWp0{wqe8pw
zrCEM0vkr;)oYlOdC794NE01qM4x^UEx0Os9M;4T=T_AS#dxP&)pDsr;E`@jOENNP-
zK`qh~PtM3`oZE7hSMuaynRe#@c9$<~w+>u=a_3-_(50xjE=h?=VHboG7$z-Ji?29z
zx-w1j#MVd`x6G$3*&DnBYo%U46<b}av_OyP|E=SW7sP_J8-z5b`}u8Kr1&aHTgpx2
z^{RDYe{L;oas8gNcJhUHvcJ^czFL^i<s$SVuS2aNuxrH!Erz0NC%rFduTjmPv(CjS
z#O;tC%SK<-YfE)qw;X$Jnq)2D);sBPgA<#AY_wwF@v^_I$8Nbf=0sYB&i={M?yPcb
zA%l?1jC)riPp$FFV=I(hxg^Lh;icN%h5vUcGKi~ZzPeE+V6a6m>fDwKZysK7{^qK^
z^*(Q|+bkoO-#e#x>#jA_b1}Y@c|}dIzu~Mo+o}St1J|Y*I9KR$g?B#XDwvX#Wwq(f
zVm+2SE7%<0@P_?+{a}H{>0B-6D@-Mxj?x^LHV25mwagPZ{Qs40W4H7k$9T>IvKg)%
z%X~#vw<H!EOwu_oy!!T%l*D!0+Sys3eYTvx?$N6Lz;!}Gt2Zn&lv;22EK6_GKEoRm
zWz#pfPJ1@(iNJ;zX-nL%ESupL(0@Ti>fKxgX5Q^}C-NFN=6`YX+rYZ(+p}x?R`1qg
z=<q7~FzLg4Ee8gUd^P!0h8X3!Kg0I68J%@ymkc?!Z(d+9UrW&{BPBCo$DF0Mce;yN
zkE!`E?7ARRHuY1S>YUs+8l{E-Hl2d(Pp-bQoULu>cXUUSq|fzNoZ-I@$?dXnpDL@r
zW5to;DQhMlT6EhoEpgVH=o!Zy8jPpRTE70-(-V)LYPGSP5xaIKE695C)tEE?-$b1F
zEM{}jaY^gBXT|5d0_~=^axflam@5A@^F+&r<NHb%UJ^_G&9TKPeZv9W_iW_`?Dwa<
zDO2W5*L^$bgTsY9p+A`&O^@BR9Xjg1Ofm3`4E6e;`242XDu)dvrz2fv-1!jR%akS}
zIr+$1CmB9ThK(GP90i2k9^T6qogevnVmP}{8xNOD(B-)Njs**syuVy}<WEqv<PL@C
zhgUU@-Q?f4d~NUPtx*@M)~VM-b~y*6JyS~T?p40FX6w^khSIwWYZtMIE;#r6*K_Yf
zvKRQKJr7%57<7OC!zHh^GI}`5UiEb-1e!$u$m5iM8W?Akd^s<fJ6|BdC^g-vbKa|u
zPWS#={eSP3w)AgQ9Pj=WlP_-44Hvhyyqy^0rtfderS{RgLr`)<S>l#SUagA~XE3>J
zX=3VqRPZIPzirXRzzHT--=~XjkH396<=y%Q$H+v-n-LAKA_P1c*bO%Od_Ons8AJM#
zeZ5!q$<Fu{%XdKX@r<66PSdv=&69U-h&-tJ*2Vivd(XYeGd6G<ySQK9&d{^P<GFcV
zyT;ZPA1tztRK%THt$E~hX4zt^sa8g8)>`>X;zHeWnr1KH<mYBl{`JAHc)`6>3J&)I
zM1FtC{o<y*W}ah@l4M81>$x*!O}@%*ak*({s(A5%lapa!%Cweu3X`k+B)%_o3-b~$
zHcT?wvi4W2;H-~3{Qmn#i~O9px=?!c#>Bm*^Zv5RM*H8G&~v`&<L%7vk1svBYkPjF
z<frC+C)RTc&0pX>>+h^Q57y{_Ps#a4YTOQL><zQ0f0!M<O|3g)>mKf3X}+mE6V|^p
z@L9Y#^6hcn?-Ay`2h)yh3U`vz^kL9E<K}xHbnD#{Ij6VHe0s0=jBx(coB0#M<Mi4$
z*8WPKwC2E>BXM=^cPfjW_qv~WbbYaebhlmJbAim&+pD%7dZW76c-`^M>&|~(w_9=Z
zm(3HNtl7X`y`fq&{cWkXlBi1x=d3OF(-I0U&nn+C-`vGIw>rN1a)DD#{uT$71GV+l
zLMjY%+N%Xr9CoNT?Cjp{vD$k6wbt(<|MM5OJeO!J@^1Hgcx1~oBbSUTU($Nm?##>8
zeq&PEyI}Ljw$9C=U6aEPZEuvlW8}Q<I%DloQ*S?~hDYVb%ck7Qu4Da?IBB^Kms-`m
zr(wY?$DO%S@2S^%`A0pz#<h9t>D^x=jTKis+5O?|qN#`dr#`QFS+~xarzqbh<CW02
z*VA@(JUOPw95Hu)<($Komv`@+z54eY_r2y`JKwh!CGl7!FE~F>{K5~feD3g;!mVtY
zSD3N{T&xfDvh*zd`Akl-Vg1X(8_WFnZ1g=|s&jmQ3j1CEp5&)L_-7@v6uKS}e`m9D
zd!hKhB`WTzdfU|;Z$63_Dls-+aj7^m!u+ZKzNhgMvp@bXb*s_5`DIajjeL2__AS<5
zoWehNM}F{L81d5j%WuaIvo~$8E;djvX!u`h@_t$M&gXSZQ>q!>?0f&(ZYO{G6Yomb
zjzHHL$@}A6q%yBEhlSb%=}%h9dAGpx!H$xIDUCu$w#o1HdRsE{NP8xqm*yHjx7D+^
zZTa~2g~0U*zrMbx`gp8+quYm9byu|9pWJ+E);)3ap6E3Z(b|>{1v8Zrx*VF=wo8aC
znDFpGYY2-=jK)N6*G_T6v@1I%CLiw=RSpxe6jV8GBy`e6XY-`fr;O&Dno=cM{LE{H
zk>}Bz$(yg7n~|3?jp<jgnz){qiVKf|LPOJ1Z&sCtSq=?M3_){~co&5JZ=BQ-ZLsT;
zMp%P1V`>nWf@0y@lUqejPWiOLP}ki|=)Kz+?S%I+`<EP>Ae68^Y<Krs56>G-)0n3V
z#VXn9J>22l-Z8=1ODEV#>G@_M!(^M4L61*P(mK7PXmM2O-7TwcPs!DFD7|n|!j^jj
z&#nCY`?Kn^d1?zQ-rW%|zb|)g<B~wV-nr^`jx2mIjhA_?_noG{yBJ>I*)R0v9AnN8
z-6wnV8LQ=Tj()o$z9?U=Ch6GEnfkZI51f4Q?}4i2M2!O-Y-`Tj|KT&)Al&<8*NJWH
znN0zQI5g)7s5efU<kg@ew4m$oT00+w4u)-9PFjMVJ`bEZm(4g}-QHCE*zA`|QtPpu
zIsbb!%=Wq|ri%2Qy3Jw8SaqkVH_T|&(y3-kUYy@}E{`Q|o#bDsv%S`K&2NpFPda#?
zzS^L5dX0wdqDjY|x+wdpo;h>p(%IV&ba>C2>06!L(!zBrc!4eVD~@A36J$IBKC+$t
ztl-FNbR}Rz0?(#B6BSfXIvD-6I;+HDygUDrx8bG`lNr0tWU2<9R=66n#mjZ-a<eL}
z=Bwe}noCcMt!(02ymnDn*6P(|wuP(XRVOH&=r~p~S%*=|<>ii!pjVsLb$H!ey>6ez
zR}X`OHLqXaih8wCRp~H`gU`k(9v4cKFB=@<@$u2E`er=++@wOK6K>jzmxdi;(Nw!7
z=B4YivnlHAy)4fEdN&?4i@!Rw^Tm!EE!`~&VdpP)EW6RS;GNd7bGv)b-mb9icl=jz
z!}sOqDJsu-TGlMlW8w+8-@~<M#er#mwF?e*<oOvKU@boP(~Vo3N9x5w!w@INRf}F3
za4`DBSZN8@+X}uCo}5s~)c)d5#&ILVF5Z)CtxpLmZ&dv?QCTHYQB{Al?JX7E9M*#9
z6Jaf-;iq~|ZAeztj23z@ZLg>G>2pR;bh`_T547-nyuHJ~+pYTgQVmlb_OP|86O*b~
zKV0yc$L-5#_h0wRfzPjVzx4R6|6;Uw)#J;h_NH7RN;fs<-7Rw|w7+8ABDJY@p`E-K
zr_u%9H!_L+O1}4$o@zge31WHuf9pxr8B5mNU7hFhm9u(o1yk2mql~j0FJpusZ2j~3
zthqgdM6&g_4c!OIj2CZzl{F!1Z_Vb}`n~VgPn#0O!f>E_$08A(EN5-E$5OvmE>WE(
z>@Z!%*y8E~?)aq-8UkV*+q%vyGfOCH-D_byi{sKU-agG%*_ekjpU-*HTr17B&nJ;>
z=Za@Brx*3}nlBcVt~zJFkn=8|_h<QP$Kx88IPa<Hv>o|6#od<g<+<`bpIzP>6f%2n
zNvhhQEU&w*J$a6iT;rF+dJ{e@SNfxPLBdgKI@8pR6;d;2Hay^Pvs62g&a>(9g_ddV
zHm8=j#;xqSJcDhj;8YFQW|zQDp#?3gW&KTRPyas=H{r;Mo(;K^TsA#nJ}f#>&emX(
zc%abTZ5Ozr8Q%E{aRy7iQxHEiw}pqvS(W$T6d|c~N5oQoN<WEl+^O!mOm)Q+@5dPu
z>!(e5B(_!1LHf;Bq2q<UCB-)rueDr0W4W=@w>&#oa@9_?ONqky`w~@;#4KOl^!H@>
zqm{7>OO~&2>1+>d*>%FhNL90<&SjF+O#aVLULLJ!o9Jzv$+;qH(yD-!T?@1)b~?ri
zY1{cJFmkQna=##cakY<#wpW87_uUqi?}<U}f9}X{{vdRzs;p(wwJg`|2A8_RVwUro
z^mMZ~e!9dtX#$U~Y4`qJEV?t_dF#Y3O3XPUBz{Zfv&sz3gI@n57E7L9lcwmN`pRpU
zf#f+E(GKOUuR{0k+|z3Mri0)6ypZF!sXe*rwxW|GPF$!8o48Va?JdQKCHFLi7SEWN
zs`0x2^dBMPo;Ny<=C2~Bd|Tmu(657A*g#4_|C7kF4v(yTUV*_&1Y}bFv9DIyDk$x>
zYoh$gT#u~D(UT6Tt_ix~q_;LF;7OI2H%rk$(Im;YUMB)hr*I`qS-H`5`l$qNmS<wq
zm@eFjymw&7&12dsr)*9?ZTo%hWXY+7`Nom2uC(lGI(wph&r9FpQw-;>ZPl|$QA&5)
zmwm28t!vK>#|}%$AUPwORlQS}>e#*a+*5H**ip@!^K05A)j#*1RNeag-gis+|GjVg
z7KZVacKy(p^+5Qemga}kDQ?qPmRiLw(GzWqp7QO|{%((KJ=LX8{<;+g<yQ)M25op5
zSydFJ;%YQgPj+p@kxK`HKY35fI}jcjaOO(K5sQ|klh34S1zlOmb-?(7=vlsrcch;-
zq?teGT2!)dj#|IY^^>O~`O3tP$-mriH8CMD>CUNUm-jhI%<)rC2p>Lk#6*9hwbW@P
zarRGhI%_AUe0v(=@vO^@De+iOZL3<-Jq!2Qo+{}zdr!m{rnzVEPP^E6w<+=X_K8kx
z544n5m2^!KSbX1Nl5$G*2SeLS`~Bina%Ci)Z|c3&lb_g_qkQfOn_Kr1spJhUW=D0o
zeqZg1{{NstRf8vub86=Dliw!o$bZ4aQChL{nb&6JE1{c~?}*%$vHbL|PqH(noHts&
zd*T_N#fN3YuSwe!l;xznD^<6;SesW?wKe!n>*}2+zr6e9wQASrl;jL8nSh3QSqm0>
zz3}>TdNQN=7EP~%9ZiCDb^+CAr!2?|$hoj-+E$&{x(dB1UMr?1R$T57S(|aZ<KWUu
zx*iAPja|YdO?GWxBDK!&Yt~Q4aHY`WXZ9EdAJyoSPo7kMSJ$NQ){p#kCqK_WzH$Xy
z5$CEC3mo-6wA=IDW|=Yn-<OV>o5`Z9ts>XjZTwPmbxls_>Z2}g*WTT_Y+lqlGv?yH
z4W4^@lsnY6NM|pbxc`&S|91_!U+!{dN3`F|(hQk+E7`Lp^Vf^>=8OA+)Q_rVF~(dw
zEG5aBrracPFku=`6JxjwLq_-8B`lu3hsAXiEtW0j)|*=9^zKY(hVzq6DTgE%76p5&
z$L|lhUh>Y=Xxq%gIX?r7XV-2PE-;yK_bXpL_v^2IQHRb=Ec&jtXNz$DLWd^5uCIBD
z{Y-IA2AR)q{NJNIt!v7uj_-dvkE~SE)co($clLg5G5>?rr{?|6{A93L(ngu{`PGXm
z0`|Q}+|<rQb+cJM@SWsmJn5+Zvex=dsmc%3Hf!)GpGXjPdGje_wd@H8t`iQ|R&Z@g
z@q3olcfyRz$U(rOM~h`K*NLv4TT|Yg{J-Jt`3j9wHGXxg-X_?w|Ite}>EpHNJNKod
z!)jeJ(-MUU6$j334KKeXzdW(v<)dYZ2hN<{v1ZMw9P4=j42uHJF6&z}%Oh!*9{1uE
zXU(3Sy~E{jg3sQB^^d~Qo@svPy99b}tWdhb5vj6>H?b|4$(P~Ix;~*^yA7=Gzg#fm
z^5&dyEKKRqB?bis9~I7J30zGE9JPxJYLb5jWqq{MRkB&DY@(-J=JeCRDeq02h}L_h
zn6qMBrzh&A8%<cd(!r^frTv7(Ba6Bd`bB!*GV;Hz$-cFyn>D$R<?>OsPW4q!QpD!E
z*Tp^i-ctBnsmaPBfoVlDlS%@Uhy#;IgXG8cno?O(|09pY$rb##v+T!~z&A!pKl_gV
zyre60o_F7lmfv1(^WWsoS7Tc5(JULBX1_sm{}V%XwpDV$M?-_os24p-6nSufA&`NA
zLE*G}gg`R)1SL+HQy-K%Lpzo_YOQ;uv?%7^c}BqvLW0ZmB|bzM9m;cC$ZPj5T4j-_
z)&!Ol4Jv_5lVh4Ie4LGDE$>~Sa-yO+F5?MnWWnvacg1!7YMzh3&&n)_ITUHa!SM8r
zRn-UfdI8NIHbu)6%}^J{X_qw3U6hP`Id?P&TR!CI-;g%N;MJqF<ay^_AJck$=u`5U
z$6Uv~k~Xa4-ne4zo{eu(+LSKktqy35eKE-`qb==)-o+Pr`Y$)U`+q`Z1A}pf#la0G
zhr+fgO>w!DF{wy!e(Q%vE2GY&37I}Rl=bApha?fJkIJ?aMPgEvr?skwevE0FD6u?8
zIY?b>?@JTS)^lqzpZmNw?ao%5m(0~u$TVf5WZ`6~(#w*n3XD~s%_lB9$fBUBvLH)o
zQI^qL?KmdwyU(;XU;MFc<DD(*y7dkTZau8n#wv5#(|T$DFNsBa4_uac>1qF=ljopu
z(4&<70j&EUoDaQa{J?7|?~lUw{|qye7KzjeC`zB=73YhR>6yx+z@WTI*+bFn3%`(N
z;F3R1heb9hNAN0JeR_P$v4N?BNq2&<p~*`v{g-?zoIM}vo_g@#;$x#*iL>XHPycPU
z#Dq+m8MeW6yC-Y$iYeRrU0+|CSoEs#4U5LXx0Y?2Tzf9D_aD>zn6Iha;L&NqrKH5f
zqQJB$fyboLbD;_6j1Z<R4%~|xw7P<|mKseo`=Yh>6UX#*9E;a+UUfdBv&?mqA6I+|
z*ZxCX>z&swd*C}egZsKr*BcQ#rYYTS$6`LL+2`<x=R8NH)gs}DW$PcjoBPNk{a4Y2
zcN_HCzw%0^@P7^Xcg)>+P8wHL5_eO8$7JCvKZNTpF*!JT)nDs=sI=b5`jG>FNVd}?
zBTud+6)p@0CCWLCB3F{tOT4!yDZk9o36DMa!Ru*b+|p$(nuiV@4R?R36j3O)I6KmL
zv!1W<tp7{0r$2jrF6Y8?horq(#*))NO3wZmUB6kfwm7D-IOdA-rU}K)T_w(o6u7o5
zaG&JBZPqC2mdHJ+K_tR)>4^ow{lVVL`$Q}*<{o%!Ip6rrlFQah61ZAem2X{I%TuPj
zBlzx-K>rUeDh=)Tk~VDeX*2!WX`_2c_WfD=8y?1AvTb7@g$SIw|6!t<hho^9ry73(
z6_|yXJQKNA9Wxiaq|qd#R<tPHojICGK;@jW!&6Qbp7(mcjw_k#ToGV<EE8bNd)Q!F
zi$ZRrf!#W}TM9?s92ARa<Wc!@Yr;{nR|^en`l5Fn6+7UXBksTS$>fxT4YOZ7KHtP=
zvh;~tA$Mrm6^#?k?Eh;`1Sf9NZVEB9+^VhDWYK=vqBGdboG~}DQ8p`4WJU>(+d)s&
z?XfSn7H(~j)tfH5ZGoJ?VgnxYLg^l7y)E7wC%x<u;5`0CTb9*#{V~qtmy*wX@;w~n
zw^l%x)5+Grt;?=$j#=5he^)$HidUxyNA1tBcqi0%@5I{=oAiHf3N$~&XRBne^1CX}
zrT2^2j`B<j4t3#L_sptkxmA<Uokvs5PEWU5V32%<?;cP8mAI+OH7V*wm#-T68yrZ=
zaX;u*vij4WrJ`;N#TX<et1af;Hcj*rTkhZMkM|sHU|k@0XsOt@v|DKxbf2G9dFL#V
zZy-8(wZ!>mGxY@KJyRtsH!nQ0RsMgq@c)`(3+1)uhDw*UOtp;9wpvdYdGxh@OM~dG
zg@y9p3t1avRkn+nJS@Du)$6Zg@ki0v#0uw4Te32yS<by-xo~s$435O>%8AO}T`O;_
zJ+`iN`{wTK=iSG(%eB6h-^)#X=VS9ybKjKXHXl1<FSu5DU8}m+|99K+zW8g}i`W<p
zw(@-1Y|*?$`;QpMUUA>`$`R|1vGd=o7Tw9-eLmu1gi_2^$Cpk{hKpopq;T^j=cr#P
zu79FDktNqZ$*AR&oI!U(_E)CWVPQ8F+*e)WJEL#0;>e0wnSXwDPIck%o9Ary(`@2j
zvx3^4S(A!_^_et(m0GAOdGm)e)EOvEi=5f|S?Pb53Dc4VVjhXHvK2YZE9AC4s6VXA
zWmV3@6Cs=@F-QJ-gK)T~ssh)dNUy!kqFzT!)*H>6d3p7e4DK^uT(>b#y!@p!`pK&U
zlgfTZ`mt8oW_<6Q$Kn6*v8BQzyI+rOJP+{{>sy@gND*GbAZp4dy_N6prGV#)s+cF+
zFHz9;NZ~q_Ap9pnbMB;=&|NX%x6CIll8nE#L8OeM^@zSzroPSETH{Q8y+UO-w^uSg
zVb+1lS&hX@A54C&xN_&WL;sdXUr<s>dOU^I*;8S{why5~_B$t67#hWFQ8xN<!{E4`
zOR3@nL%-dk<_Alqnzt-<57)YV%em-itEtkZ43)N}4WbMhT>t-t3ro!qHE9%GRU-0-
zv1s|s^&fA!-I==n_0@Hf)y_*4EH5t;v1k-ou;T}ZSr>oV6O-FrmT8+$PSV|6@n+up
zGL<r!uPwVzUCyoV^S6Cfd4exRQ)>Oaj$W^8axXv1Wh$laU*31S!M^8NxYV&Ua}^Kn
zbu-$fm(DcW$u2%a_#)F=C&$U(78@zWFO|-$f9!6gt@`LWli_=%TMI?EC`?h=xRP;&
z=(cZ-5eu`<HW{~^61#O^d7h)n(?ff+Pl*PcIQD9(m)+yM{&SIQCc5`7>gp2WU=1%2
z+;o8PW`9d?Nw1~Syq&>{G2TB-wagFoPl;60v9X-I$m8Y$xmQYD|DuW+Z^!<3Kj<~}
z%8{i9j$DnLJH6!SLKChf2Vz%;2p^C%H2dQHIMQdQm({j)YxoT3_iifPxNc3`p0eYc
zUY}c4(`lr0cja7xty|l}-!_G>b|`Q-rcrW3qw@ZyO4%pDKR=zG`@KTNbc07Dw^Nm-
z#X$wu1KLx9T$LWRs_H7uIdovvq4#Z9tRy{+w1q!aJ>JY<ro?hMCq`FkM;ODF1&yg!
z8%n*yq<37%ez>iI^J$)ASXAcLN5V}XroD){Jdwd&<lu3M=p$y%H=-k}T26<y-o3Ts
zzKzyHsdb^Y;c>RtG;6NuCmk|;v~i_~;h$NtcTG1<P~e*MU}K{#Q{CR!$q5}3P4;X}
z;CW_R|NrBy!q*4IUmcWxb5NeaEMcz1ov!?ZjTR{_Pj4@G;9TOs(VUVfAGUerj-(b=
z>#f2)sp{olTDZQo^f<CtJeeDu_kQBj-Z$4O5>5!lo_dk8yj|_a<i2w21HTq)pV?#4
zxmlB|FMfXIRqI90Do(CK4=yO5yev_o{H2}OO~jG^Sf-|r6Z_xYSDmNiC_eq!D$JSQ
zbkEvo*Mg@FIjU;sSj-PcdI*J{TcjfIaqdL&$=V4A8U-vvmso_hc$z-2SZv8<av;n!
z=32kytGP30O26rnoio#PQSjQj*Q?`{CYUHk#4tq6@jUwPX!4(<Gq=RfyyCSwggc|A
zXl`u#>xJUa>Tdl$DE|M~LHR$7@4wD_|F?0LulVuc?<FrIlO|{+*L)CQtz?=}u;$ns
z&fk``pDmMfwx3nAuYI|tx!dsEiNy;KZ}yk1x_9A)|M?X?@4C*r-4B>)9%$aQUgBzC
z;(v3SSr;9CO%ONUDfp~r(g8zN;arC}A=5){IVxsfY$kWv3%5(vOUSOs^j4aZ!lO4O
zG-{#7ucu$qSavS>I@hX#*+q7(>x{K0uL@iiU6)+JZN=xAD06J-g0Mu}b+1oeOTT+f
zUi$S+OUt$Yl-|WF$<2P-+IoyJNqA-8nux1O;j8DZulgrp^yc~nd6#xU&8VJ?^<tVq
zb54}J+0l7M%t+CFnuw;t(gt0v?H+~yUlw%<X?iVNlCslC-Ft?9{JJlkmzQ}>HCvt|
zsKt<Zaf#3Jf<msZE3fDVE#`Vq6zVX2jq~oRzg%W(ZfuO*SY^6eNc+~52+P#c+uzpS
z*^$CBL96+@UYuX*i76L<i|5Y_YT|tUHoEQfWlk>@6@d>H4NVOFlPA2*s$9b|HHYQO
z*ISZro}boy-6tnC>y5_x(%|WNste-QmN8{)&)(|Eq-m7mRp9BfXnVo=j=$HQIypHM
z@kN`N%-@uFibYdsh5Ez=E{;sBo;#}Z@7ySw#1XjdiI_tjkG4@D%a;84m!7}T2#VO4
zZ0hB`H|(u3_lNqs0gqzM61)Ok5*B!|?`v16<kS1{KcShsO0|_CDrQBe#NwWfiDJnL
zD$%xkZJzjdsyekSa@n_}BgtNGpGIJh&bk``+PmL=o9NDzqtp;Rr{Lq`ew$A_S}UzZ
zO<7C5yb>;}x!Y!jObPI_be$S3CYf3i>NjEO6tmoeF@7Eozf6saNnJW4GHY4qvZV0B
zjiO=8W`31akDakK)T?~elQ4DLPb(+Q*Zlc1eSW==mcznMy{;AEO_P=hB~IdVd%0v<
zSl3F1DU&{llyPw!xLeQ3;uyx@T+1c7N-K9+kWk8oNkTbs2e~rWCaq1&_S&$?@l;>z
z$t-W}6c5L?nF)=Stn3$jH#9N{TdZzixu~YFHu+jm_uH+`p0oc8t=8T6h+i{GVIiyb
z{|$>=YnN=96fd%N%j5vvr`v)j=+EAw^t%366Klfiq)?BC;SA-fthG!cYg|Rd`xbJx
z=Ln=saO8>&;cr(9lDX(&7`MdGRjPhYp{rTZlZo!BZ&ZR#X}m9)9H_tZjIgh#Smk6t
z`?oAtPP+?TEjq8M%NiQw&nX*v(WTgNYH)bq)M;Lo^LB=<3z}Z47Lm57_1fid^|rLj
zY11Z(-e~x^ReWg*dzhDW`O>{*(lvL#E*25a`5xglsdV=$A<dPq^dlLUU7493Twi)C
zqG|HATLHVb8qE&h;rSqC;(6cp*r(SuCpwgSrsy4f_PEk-9>Y(^Z{JHFRlfco+8ngy
zQNRA`%ph;KSsQ{rYOo(Qa}!CvA{?#Dq_Sk5ZjeI71}|p!gRdIdgzOVPFI>EQPxEQ%
z8Sh%7O!g=qS#v>neFx80zKflzmu~R)O1ItT?t1CHrOy84?i+P~^iQ4N_e<ka1i!ZV
zv@<W=9jcDBPT)8h?v~j(InJo_rAMQcY2hU1@<%F*WmLUlth`t^hAepzvBBHD^PHDW
z$C}j|-Yq&0oo4u)n6}K(w^dt)E2`wl%q?4v>MUglS$k)akW86td+n>Hg>qjtWbIak
zP1jOd)}_(lee|rg*2$Q4ugX@1ayl@j{Nl<<aMor#pcNn4GnIjF`6O*F2D7=1zS3JF
zR>%E+=#|^Bd%9PSp-5utYpaX9{I;z~J$|l~+o8(mkay0;&Zb;P?&uHxUcXcq{P$=S
z+2?TL&5e(GjCa&I10`Jl)r6mnI-^&=dG>U%p6QaBEk#p4DstC~EHlv5;i>+i>7mPW
zi7UEbjgq4B#PEvt(`_4^gnul4s#EWIy6@_x=}Nad7(B~_7$>Qusz!D`$-6dr#uXPX
z&reg&Ry3W>Sm&alGyBr{Sxn~^g{IDVwB^**b6>VB=gpfNmaKT~&daU#M^op0`V-7&
znkxS2!%_Y8+vy2^m}cx|5HaiuJh1#nn`Bv;s%a~83{T)^*K;l+J$$7ZSsJfIV;Yxf
zr!@Gw)lHvtdxFw`Mz+mz-^?y6yzum3;=Xi9O-PI}>EJ23!%0VTZyC0^_+8YlYL;93
zK$U^-YLe-j2NP7bHVBkG@DrYN<<Zxo=W^d&GMqS9WmWr}GT;+EVz>3dyzgQ{%EGOo
zZYjQ=_D71YZJu>?&A)_pu@@Va#66$5mTZlWeSd4wo^zq=pK4#M`?h_*%u^0`Rq06+
z=a_UZvFV(}9dapAK6{F%L#9>uqDySL+*_vhOi+#I5<Y!z<{^<~p1Rpq>R~k!QER7Y
zX4!Fd3ox_Zme*UgbyH}_Vlh)w<yYT2_B?IzR@=OOokECpph&~YU58e&cr>~v=ByJJ
z{dH$!$;YtGUMJn$Cazz|(k-&?|D+q*+)3O{dI9B%n&)?UOtcQaq|LQOc&@$MM%Ie8
z-KzbKiz3z;b48o@m_<pr2A*bJUG}wHWR^#SNPs4Tip&lt*5nVG!eO?$DN_{$tUa6#
zyxN#pm7}vqxi@O}r-yY{l!Ui1NE-O=*(kDD;;5M9%!Jbmd>9Yy*l;>$nct$$$E=T1
zRLqY_nk8NnnxWa6ZZ92mp)co&inVHHR^luVzuQhPymtD8*m+;~=~i3!akF-Ux981$
zF>Ljdd~+)ljc+q8o-wcO=)R!f2A`8gcQxx(P4+g4NP0Y+b$7!eQQ1CaO{D`NK|dZR
zyxqti{#ki8ui_DfF9xgLzB(urc0s-V$g9x*wznAMLtdVpz?5-DMD&{8G?^O*R@fL+
z`n;XTy0zWrdUe*fZCs05mgLtykdFUx!d;p1hm-2wh}B+pHX;fAORqQcm4!<!Y7L#z
zQTJ~1{H`Bfc9K_{)}J{x|DV~R{gZgVox8*5xjyQ(!%@Gaw8v>WljWo>l><MhngoAO
zUHDm5;MDHTPLc~bU&lJ}?eT2Q>oUp+eSJvs+=W)#qRD*Ti61tWDEW6p?O0f4vBLSL
z@OEMON2z7qh4+MxZ4Yu-;bl{I|Iss*@SCBYbB<41f9^om)2ugK#Yc@c=_h^fWomdV
z5&dxI-HA>I7fXB*lgaE3ba52Uy{e=;W0h;z4QJuC3C9&`{~zR<GvScf-X^8ltZwpN
zax*4H$!~29x@5qazi6u9YdOQnn4Aq?D&NMx+&rgOHm5!#NmN34;$7`a{8c?2wX0{8
zIcF$#eaoCNqyDIwy4Od+pq3Qj4)xAQvipA=x8|I6^N5;v$CG54^3BKX6pJkE`Av%Y
zCZ7yRyk)svX!-VV->)IUxwgHfPdi^<&~CTnv1e7jP(SPE{LY6=tJd+w%9VaxB&h5A
zk+-~NV}i;23098--pTB~X_D8YEPGyUg<p`-73IW@H=S%xOkkPD^Yz@v+SKne>!cH>
z9ydPYn(_3?n(bV&7OR*v48FNJ3Y1+qv|2eTa7$#0lFlTdqhWUY#l)Jg{m)LASa!&)
z;kX(D$BzcBiDhwxVVUa6EYHdfoWj34aQ|}W*E0%PZNR1UBL1HPf6@l7qN99C2l&cE
z?Ms#_)uk%mVT&j~#%j7FSHV$e-y!ABP{m0G#tt7H+Dt3EQ&rBHI9^h8oc2{F>5$Cq
zqaunS#wi=s{w!0=cZynRUg&ICoqRDWfzd@MT|+QDTEtvL%-uz5d7I?(=$nNxC%(2F
z2siKBXujlfz#8$IMF(Tjr^FgR7d_ZiEGSU-^KkLL0E6J=tj?+(>yzqUFRus+m5aP?
zIXR;=+f3wdYNEQCkHEuhg=Io{=?x+s?Q7TtSAPhb{eoYmLtgntn9L>quNU}#T>sy+
z$e~<lq370$@{HpG-yBL+4s*%`axongKeVD^QoE07n~7eb@T}#PYKlUW7FS+hVtn7F
z_u-Nhm*(E5Eha&QqF#r4S57rutkD;K%=Gz_s^bq+U#{r$yrSH6P+a<XU+OaPG`F<!
zg%YBlG{c|EPqh&1UZTz4T6FSz*=cpLB@b$@m8QQ>DppKYp1U~4?2~--1?`8^rCXj@
zUN!6#Updjw#C6(4*;yBZHMUwMmD=-9v{RVY@a}QvPNT-@7qio+HVFL?)!muRr7Gwi
z&i&89zIR&FyI_G$j@|Eq`ByLC+!DaQSb?i9+;9<tec9EVub*>Nlek$0Bg{RT?_3i!
z)o@_k@?ZJc!RFYVLhLPh{jI7~LR+F8BE4ofempi!-X(w5*J<C?oPHk@`*kh<|F`^w
zUzH0VYUEE;F9<B0cu7=jdqMM|0&A1#o=>8&8e*nXg!*Q>WNXayI>P#cL+n8L%wx*w
zF<-RRuFt%<)Lv_0ptGaeYDa-9-zAgXixXRlIURIGo7}HWat{`nUF@N|<)+Jpr|#<Q
z-pz*_v^nQ&ofMX+n6u~e>_gvXXCD<%Uy&nu%u_N#(B&l8>H}Ph9IW4~a=kGKUu3}b
zWkJ%4it=v(%DWW>CZ^AmYxFs`qvE_q<dow*)6^|p7x~^@;u}9P@_T0QtsN=Bjj1c!
z`?ejL(JZ`RYg6h<*Z&1JOZoyQ_stEE=$Kf|-&M`mC1J0WK2v$3ruqcF(sohP^h806
zokp{g9l16pb8k5~Yt1EDO+h)X4-@Y$GM^V_X}vU4uS;_FLtS&OdX1xT>=KP}h8c&h
zP2#$-*x5=%Ok;AA;1Z*o++Pypd3S{A&RpX9%5KpHwwkM9IvUFFH*;z1sOK!4JljOc
z<EC9vYq-IUoP8Ip{v~j~DVVyrVQRisgv!CG&WD=gPAX6OFty~AGIvVE^)0@a6IH!6
zR%Fi<IhC}+bz@|r(}Gy#sJ~_+Q5GVmDTN%rTH_aLtS|N7#v08#eTLxjDoOTLvyDWi
zG{yv7amo26F_&FT)x2-+PQk5*{<j-$(Jq`K)S$sydbs~2tK{xkLSK)r-rqHI+aq^H
zuJV1JU5kE*9=OJ>Y*8L$Cg`!VbNdT{jTuvRr%cg2*0{w(U{OHQgP(KD!<8OI^6$;?
zImRLMa%Y9*#E4Io1_p}rFT4=q%#^j-mi%T~WlE5%bC}5Wj+EcK=HGkKtIe`f|H`VO
zw$%45sa1=F5|q{BFQ&d(p_y>a|C7YR%+F^0-ckp-Bm*Z{NI%`kts1BvuFax3Yx+h#
z)9>1A13b1e6-~=nm9Q}FgevRq<~6Z`lT!j44@!1fNsBqpY>J&Ipn0)Td6IzA3+oG^
zcG0T@&Ryfby3=;cf{eWZA-7I$iM_hi$9h@B>Hk|yE?I3l$j9%@miKDelbgBLJ2)9r
zmPb#B=$N|v*+s5B8T^ZiWcySZ&K0(ZwX_JC<ma8-t|;Q9)*{@sDnF*tl&focHK$6N
z;ert7g|Ba<HdMB4c)D_ucOi$>&XmA(ar2q8xo5t4GD}&Vfju&I3zt~rkyU3|wHI}}
z#V-uhpDv_dKFjbr8<SvALCWqQMXM!$ix%zJ9e*&>IZ!gFq2$QU&4(j5A8Qj}a#%aJ
zBJ9}9&J7;?Umf^2ZuC}up>(Qx$`+??l}&tiC8bYtX&&?Bp1EMkK9i8OE?d=ptbNUG
zr0Jx@py9Lo1^*+fsm2lWCWI#!MXBhwFV`2e?7PmjKY{<h!T$qG3RX|C==@x195*rY
ze#`vtC)fXx-0(a_%5clV*Gm>`n{<#x+;qcs(FxY#zR5e!d{^IgyjmdZQ2E5!Tw7Ul
z4sgvaY)>dTC~F;f!d&bKmstB|y~n?-)vVks80FSH%ABiqWNw?e7USa8kK`412D|PF
zW_c{PTX-p7#1dz#47D9xUZRc7Y4Yi&Y^yIc#Jc9lc<^u8z!#~rMY-bG>kFYCp4>t=
zwk+OIFTW|<{N&m<Q^JHBLl`rbDJxZc@D7ubI8mLt%*bikr^of%#gsxcx24Wrm$z^^
zV~Ekl6n>^0?t*Bxn+q-H9+u_&qk4X$umsDg*hh{Z%2qs3VQ9YXmzvlYdg8xY_+vHc
zKQn3;`NeJAQ7c*4@u`4SdgYweg?ij;7JOP={a9<Jc5Hm&uKe;{DS>W+u5PQNxT6nD
z++!oQugXJcLdVgr%@1~aG8ya@5Zt?A$KHE)yp~sXa~zy|*}Gd_qv=6Yec^Qb+f`fm
zJ@}P7wrsq}|8!FKy<JIv-kkr<b3sjM-j{BDX~&2skzD6q^t=w@T6^Kb^QC5ojLfeW
z>E60%X|-(C&sUYsVJZJ+^?GgVWw@PEu~JCf=WNJ{6|sjmFf5M}i8++aI5Xj!v{3X$
ziQA%oEY8F!GUk0eB;>klp>^NNg}YuK(f+Y%!oFtx2iG_4uf80+(EQui>V3aOKVH}6
zI&2*r^MB7CF(t2+ifc9;JEh(9ax<5YCPT9C-bD@8iytU8q)$monEQPr->K-imOrz-
zt=JYhY~H1oq|DJ3Y%C~qD5r7l)_|W=T6J#(M&5{7eM7Bj+5ZbxQ_pm>&NVW9AwKC6
z+uDMgc^A0K9&nW<aL>tQk6s|F^GkN-W7)eGC5kS~Oj_*dd!yyUwe8ANPMI<8P%)Wa
zUfKFU!oPIUbhhWWYb0mdZBnoPsopK!)}|T#eTw$6IokVjG~X=Ix|(=d<-6!cuE75u
ztA9SK(On+6cC}E}73~cdwbMh=-zMzN-@++XeJL@};4+)o`WyUj0{E4R<kuAJz3iFj
zc9Z`O=e29U&oy70`+M&H;C!BAzhk@K7;wEV2wOd|{NA0gALUzA4)I-yT(W_q>m=ts
z#cy#zG8Y<n8s1x<`2A*|;=X;tkNLL-aQAFZf1avrR&?_q)8lms+(vqjH*UGPG34>V
zB=#bgi{&b_uPtH;brJi;Rpq{T{+*6gQ5%a*I(-|CAN;3wXtlanlCbm6ax+HJg~4KV
zXYMYP_nu&&GeQ1W&B^0WRrUmI`6T%|>7L!4!;1?S*Dqe=vF5U~;G~(2i*GwU4_IT6
zV<)0$@$gQi{OS*iniGSSgXV4&yr$Q5{paDezNgpjNuImuqxD~dYnRLRDU}^ty@4y|
zjoo*{y#+Os4wmf=sgVp{vqxi-x8=|JG_U{5nsrVvXoO{2*=2Su+se0nlZv-ypMLHE
zuIX=I&$!DqYwgVie{ased-E9It+{L8oILktzAxLQqFWc&SzhE5sl4s@K}_~k&?)!8
z6&<%CXHP%KQ9XTQx|%4D!#qJ1+fUQGUpl}27d2<s=`E>kuiuI;)Y9DT)xLA)`@LN;
zUqfZL78NOR$3BXECiyfV#f0l&u0Z@kK}+BF2UcHxXEgcinrm}HlGx5Q`mhRetjqcF
ze6GlQ{=2K^D%}rL(y-bf^y&IZ>wB3GE6b1aKU{qL=icCF{A(JP{=M^Xg~g+f()y{}
z)3xt$Iq-XH@;x?-`(iBj*xK%~^|?25_ulm5zvX}a%eo6}vd{lNE>F2w(rmThwy4Zm
z5xzec<zz)Y4~zLsj#B&kLC2*a>{%7ZZPUe!W_g0&f2e%_`DH_)zJ#%CnqZf>udZon
zv;L-6mnEwt6TUpPi~X>*C_Q=0gv1iJSGHG+&RzLaB$-vFpb<ZrX@cFp;s+5?)|Kr|
z>m%l>hSh&F{PmY%t7;f`?Q^BP<92CVl{6JA_N48Zd!xa8_vTwYVYBV6w?F&6&Fu9-
z|JVI*xeIPv|6qEvZ=Tr!y*IOa-<<b<bFu!7um9iEd4I3IznQn-jr0cDn-~8sUVHN5
zIu_vrx6W;0e0F}?^>f=d+-)g(`A_{we$L1I*3YMyt}7o6DyVp5;@2$N$|=Ca^1n#%
z!pd%k<~dwtfj2xGoLZPzr%dt)R5;QuWSaNn#>K@4f>NFOL?VlxxKH6?Ss}6_h|8-p
z*KX4!kIkX(Gnq20?r1)E<{y;9($}Opaf*MZlc#KMdiLVAi<u^iEY;e&s$pfAshWU>
zBG>9FnZFmMO;^R+N6pHZE4ra;-u}?lZX2asy&LSy*4((76#6=EZ(aHCpR3((d`LLR
zbcikPj$uwjV<U6+ls$!?j-Q^csjU|CZNvV93mv8PVs3?7@2iR4nB}_a>#M6<qfTzS
zy6RhQ__m_=*K*UhzrVM?^*7tQt{opAo@kbTf46H#uAY#P$8x*7Ri&?jwsG9JxA%oe
z=Yyl%67m1%*-Y1bado-Be#3+6@8Q-$YrN*Ks{JGMc5%!6_{6#?r)GFCFlq_~EMVf)
zX`XC!LZf+#ky1;5-4vk-O}$|)PM*$;7GDfj@|g>D9p(*G>e4c*5<0PU8B3P?qOz1t
z0ZeOEf}E76O$m75Hf2+ir;?QFgN>^dE2k)Qa=3jwD&IImWWoG6x5%XiRbh$Gb)s&(
ze7dG8YvJ-~o1M6WjV_!^+9ZDPOv}lWeI~CvJk3^j1uL1UOXo~6eDl%OzdC2-asQhs
zlUMeJahQfM%)H~!XA%7-V@u#;1>3w!uG&k#1bG`DExGJ-I^e-&Sy2zCkRW%{Qg^BC
z0ig}NJ7y<!hcCaOaP5C&_R`R>n3Ing_UM*9eRuuO4uP_Wq|RP`JEKRZJ@$9299p5_
z$9KzPVH?+`ZBgx;mKNO3_52jJq<6=njan~-Jd$=V%DWZjnr9ldYO%1Y)x^VQ&x9I!
zR#>c3y<ZlnRmA-^ZqbJmh6<NCw#412oV1~pTR7L?gx%*859+kH%}{QSoY*9KE#|ew
zwtYFmlID|gPoK4X8naXR^3>Yw$kZos(=5|w#jcY~n_ax_3*VIP-Aku%t`nBp{d!VG
zs6t8oTMnDX8EX1AU(N~ZXMMZ&IEnk~hP?isZ){25k~VyA=CoUX(r%k_E6Wj%Ma-67
z(MpUvJ3bubD1N1(y{2nYf^g#%UcsFI2P_|zJBPgyYY=goduv62rjUeC{<>9%qPuuQ
zu1+laz!dh<L)!W}$4TD{Th{m-oRhNJ^T>sg;F8ntZgrGiv+c6|P`3SuH*?Um$tC4(
z{lQZ%d#he8sd&(7E_eC9_|~Jnx=x`E9;`wfr6IDD8{#^3fBf4T8oKj*X%}bS)4Z@x
zC*DsDkKgGrpLxwy{cRE7TT-{#Z?slyzV&0#()HX;i+VYf{^_zP^6Z?p_1|^Dc9ZV1
z(2XZte3;hd=JMrxc}#TYNjIFe;dH*t%agtyDN2kzk2GFMxk`I9>ND=~Q1WZ+I>2B)
zS2{P{ifQJ#RV#!n*NePS776A_Qz?>^i#y{x@n_<HLCZ-q%~S<d|E@UN+H!f$0~f>8
zqn77Rm2BP={eUM;e|OT-T_@(~9$M3_y?K$?>IxhEZB0DR$;zc`%;!#Dpq&|F=r21z
zOm20ku5HBOS=GO0tVz)guww|i=EiYy;=dG^9H~mNMz$pf7*#bKPg-@g+zK;%y|YbA
z{>lkf`xA-||JKHS{KX+1@$ul1g>K3{n=iPveGybFbny^VY8DYuU|sk2Mb^aVPKUfN
zUW^(KJ%lc}=srt1qOSPa=y=K9IlngYZ7?}mda32I+p3p>Dw|%;l22S_X8owUDrJ4e
zV;5!1vypx!vnGoO{RsK$_4QnEB!A|;kN4Ijq-s|`Iu*7{=={1r|4(Wsob|7JZ+vaT
zqxLn`8+lcG_ndH)P+FVukUyIB%C#;QPP48#OSG9DEOoEDB5IMB-7Nc(bN8yGODhUC
zO);FvvCmCQU|-I{#R;nuFQ_z_s`?yg`(!!YQ(Qu<>I<{cVgXj&7lnaK79ElQcjJhd
zkDKN^TlaM1%aMgERvlfP_BAbP?yIovw~WKLI5N2?ufDM8x>m1>?^?g)?20>9raQcR
zv%-V><vA;_FR5D_U*Wr?mZ@@mYU-0o+^bHw-7340dp1Pt_RfUeHCh5w!&EmpPZW2b
zRO)DSqpf{yhU?>on-dEIpHAB<a@BX|g%I&DS1!Q`P4QhtJ{Mfpcz-d-O2{%4|6g{3
zRcFhA$<uVqj`W}5Q#}+r+5W@=R&$Q?<ukqR6=_bjd9i5o&X*?lN@|m1pZ;uPUeGk}
z%bm!*$}?Y2yR-43?a9}BKOEHB`#{+L$BD=HVsa1Gyzaj5@S-@}PvS~W%ISP(eWm?Q
zm)z_56(2MR{}60n!<$>V)Qfrbq6mevf-C#~cHI55Ofl$?c8hAPpf;O2gV-jgLngA<
zOP3h1auhFF?0(6~U+eSX1zZj#TWqszUnqs_y5OX@I^mMU87Ka*4km*w0<GER#Ya+<
zc4|2K6iuGE;)&XQ*CR_OD2bLEZ`Ml_>`rQ#Ic;yoG40DaJu%m`#1F<?ogHXsm7DU}
zVC!e?_X_{tT9+>|mis!xef8Y4IrV?Fo5fGAb(yAnAe}ov#kfeIeZ|>{`PGVg6Pa$-
zHEvR_6}#v*$y6!ul)CGTt-S8HE^^JxQ<=8l>(bsL5uYH=%X5PROATUm8O~+8TwRme
zI(biI_l_g3B5NGH1LwD>AGokh&P6Xm*JWeZ!41wl<%!27M6UT{h#wU%X*#)O>MED4
z@Zj4%SDYlL%s!WI?RBSh(L2ktvG4rWrd<7yvfbw8%T+axK8Nk@`Y!VCr2o4O{O6jl
zy?1zcJ-ksVB#KF5ha1PIfGkl9Wr>w57$tpniyWS{Sm^r+F*}|4)AN=ziz>Ia-<*&r
zc-C8K?TsU;?h`-WnxM$=zllfp#v}#S3Eod`<R`XI`Cue|h-GK{>J_eet{3??38da|
zeS2x${J$TT__Y~CD>xU#J==UYWusY{hO_jk2W?>~8+oELnzi#Ta(z+hUmfu^EBvhA
zx6SFHDbg9mvnLxg)}D@h=k?eldbXSCp8GqqqN>l!|9)Y5AbQtQ5Bb}FzWBV@{C8{p
zxuPWwY<2}llsc9%9WLq2y0pDN#>z$Hm`&&>(a#6tf5e$L_T2Y4cY504No_)ncUC7h
zZ9l*BpIe|rGs7C+<y9RAb$sraMwvKHj$4z&6K_>Eoh9J+ZNE&Nuve}T3r;3HI<@YW
z*Vm(JI|TAPwl?ypOqg95b#ULQ#{E2H3jYtYoG6IZ^I0HrFxh@r<8kZnyPGQKzk2uO
z4S$^M(~v$P!}V3MTjPFk9r*TW>d!3s+Ti)WtZe>Ti6x%YXDIIfw@W_JQ|xEqf%X*+
zjNA)YiY>M$+ISR+^Z)6n*=g~!U_-azRKa9}bI~#`j$D3L6PNy2P;*lu$iQd@lcDJ;
zZI0%>eG!JQFSck@PpX@-bK{2|ElsVkh*l-;?$~J7T`Su40%siL(CXMIoTA0`fN9;T
zij66cobooYuZ!e+n!zkIgUQIkIi`s3r`7%qvl#d^L^u_i?`kk}YA_pBFS6buY4mBA
zp0l<6o>h-#&NRxf-c+Tm_>phxs}7NxS<6;#-g(J-*Z(a3U7J=PVb<YN*<)|Lq~Bx7
z{|l>+Dyz7hmJhV)@T-~4yn4polRLU+>huO^o)zZsy1eDkMw3gAJFXds_i_luOHB{w
zT6@KLYdDvv!bFZ~2c+ee*q(@37AM_W8gM9Z!K57qT2TcvqE@u)Zj?PD(R(Rym%@={
zDqK8illar3Chb-@zO-qw>QC078w?w)`iy2US;x#|^VAcXz?8qaIc$dLn$>geGfL@Y
ztSji=Uo3sXbz#Tz$n6zgTZ1Mtyq&QAZKHX|mZ{a7w!i0`U%zX6LgIn%3L1Si^M7!t
za=n?K_;P*oW2W{jvCc&+4y+PuvywTqx{IM>&eYec6PgUB9at*LBNO_6kIh_(ou?0P
zIQ5EyZ)fM?gl6A>qdb)-yCyQ}eqc3kl#qUHsrrPcgOPWY4bQm)5=&b+b@v?V^*Cl9
z$*cLoN^=+UPp#(l#m%c18yU?IH42cgFyV_lY4x+%d6$g!9-ZCxS59v{?0&a^Su;U>
zQRlpaGPBIPjdyvwY|nBzHbwhe#gR2DS9{Jm;>}~T|JJG1Cr<pevYAq`r_X|;sbluB
z!%M;p_j)bfz@H?(@Ph(ZWbZr?$0;6S7f<UjZeWjAVU2mxA^-3o*F+AE*_!uUww}M(
zRJCKTyi{+>Y@y{klh^&>cznd{XT`aq9HH#nYPp-u>^=%RJY3`S($86Q$|q&k=bKoC
z{;zP3&S2j0q+!D(QFhJNb(h!e*wP&I;JEwCxgMGOJsa%{cl-al<NyDRp2;QNZO*2z
zf3ACxX&$1ue&=iZ4?p)uFdkskkzW=$|80Og^P2VSU(6?6S?$oo(Zr!Lk>?`!m-S4m
zxLBK)Oq<fr9DQ-Mf#ZyhIr5y7TwPWjbLyHNB*<Siebo#{mqOm_Hx$Gkt&!sNTf*gc
z^b^<hDlZGm?!}k98XqVsyjplR!%x0xllo+iW=+2<FSzwud9VElx+&0cTSD^j0oC{1
zebE}trxg2w9yD;?IL`T@S7*{?BaOqW6X&nHy~*g*Y1=!8^>5A8KfKA<SS~tZ^Cqj1
zg^ovRX9v#6SZ($Hkg?sO)pkr45nEgwe{EiL;L6X44SYHldzD-R{~S5+YI<>^=803D
z{T)YrnRI-zo9C{u;W{Ypx7tnmkI9`U2iFMr%#aXQ`Z_1E$@axc!Ne{m&43veJ9$hl
z9$S9KEH5dn|ASh_XZGS4p*MGgqhCzkAR_YqPMFid<M&qj7l>|m6J)p_5$qi~@5^H*
zC(Xm=DRN~;L{@7zhk58Vo$>!8;!!Y}GsMw@<*a?1OJ`K4M@*o8?JV=Im@5sN&NRRB
zV7T4S&$@;)M9h!Db4rXwcdqB3%xSzfT~m&n>Fw~mdRTB}hn(hxd8=-mJa>6V-wBIz
zuXpfu?iBvHdZ7&KO&iO8fuL*uKS%UG;F9TLnsRov_TEeCM=xb^Ptp5(>H3392C|a3
zH?*$5%)6qLm2pCp-JDyNq7738JEl!szuRTz)WT-tH=;Wi8vT^sZ~bPG{n^u79(!-Q
zdwBC_$;m5u`<y}!nXG=fVO~Sv71pQr{RJV%&se|fklL4Q<@v>W&!wBbraPR3Oj_4+
zowwOx#NoT+A&1i6vnDCN;h~x#O`Y=sG<PRX_qJZh)1`Bv+10mci^xj1+Dz7Xo^$=5
z&aQrOb&<fb>B+3cHWHl)vSus&?zy^W33c4Qx-9GM<fs+B?``hoZVmsphxy)54(oz@
zeII(ubDA6WtSjE?|MaW>-z)wMXYakc8~1xltNVY!cx#QgkGl7wI4)OZFzX)Zx+Enk
zkz~$1b@^UbsmP!9?5Yt>cSXNj$$z(SU|1Kx$6@2I^B^R&Ls)9h;g<ou4H}&mN2l@}
z6rMYc{aHwR^Mj5Gi|)%exj#vVh%!u8<ys_hF!a`*$)_#OO^Q7tx^;a9k5W{amaUQI
z)S&7YlNNss(qGEEOjYvUil`YgZ&?d8L?4Li<aqSlG)al?k%C-OV?~^yZ_-pv`z<^n
zHag1M+TJ!>jAlFDwy}xU-<D*zWs$7h%s#_~^M730?-XKsEV|)h^sdDM)=fwDPQG$-
z&K0Mi(C&*@j>&GZz2?%-;d=f_M?j2?>*g76HKvHYQJFT0GyMOaF8)K0`4V}%Cx&__
z_u5_T@K25nyS*Y_WLe@Jj$c!^%5<8BGd+!IF;$%ymfL+Tb86V_&9-VM&)L5e%3GVd
z=66`(*{6l9hMfy;#m&1F@3zV#lUY-t@zv4mRd3I~VG944oAzaIShUCen-A6mKbn!X
zEv-+mL;s~v_M7&nYuXhy$`!g@X!#QEnliOrC8CmBzH;`|>er&KU;Fl*y5PG<zva<p
z&($|LkICmgeBNDplJnS^6BFgSn(uZ_<|=MXx%E+#e_z+!4)J{pBM(Na3V!Nb9J6rZ
z<2A=36Z>a$&3kKXo;X=&YwO+}iT#behHX)kD{slMJ(E{^q^uS`W8O=(eO&7Q?{z5X
zaT%t`8`fBDYh5*8&HC;HD?5?<%SG=`mJE(g=ym!SXdD)G_LFdd67Q2WuMTctug{2=
zS`__yMf9N~a_=n89C>n^HB~C2an|nc)yHM-oS5R`X`;gV?1jxgj@A!ncE!b{?Nkvp
z&WN<SJN0V^%f8Ojr%a~4ogI9OWA{ZHGYQ2hy0P;`IA$IYkJ@^aHy|;t%cAecn&xX~
zRXiA8zI=23+Z&b}P1#$+a$Mt_9p}6|vg<zQ^<Pi?<ECY@1<&bU@V3j4Pwz&<za#rT
z9`XNhw%08)Dt#Gm)HL4UM>8HDOKZK!ooC0LbL805mPr>HHmMhS8Fe(BVATJyb^qHq
z^HVqNE&n&~ek<^vEj0bxEK$ZO7d|@5Gv>W-yV~!1X#K<^2X-7ga5Q_GeYQ%wfV5E0
zgH;C}^cP%Qz2l_2b)@K@j@br^y$a254Q>iPxj5@+WbcIDbBEsK8U{@aZJ5Es!0XSz
z_x;V)$6duAFJAv3aqEENQnyztuJTU*);mk?(HylT3GpPQXC1TGWj4+zytOU2GvQJ1
zhr<3HnYZQ@N|du&g-o&E`)CSdvS!yU9i^<Pa>3yludW#+3-iy+nD^?^hF9ljypuS_
zQ-481LGg$g^J_D=8zLb-;e|&m+I&1p`nT8JXb%l-4ZGv*yDGp%r~N}np5^o)gS~Sl
zrg+NP_&0T4NKMIGn6mhP!AkqUh5Hz{`igCB(%ZNvlJkOCuFv@|KF>7-)kPzMO%)S8
z+G@_<-FCk3L+Z+_9U;?S-{O;r-qCCnu>X7Z{<L+aA9eGx;(DJPOPjSIH2crj7tcN!
zCUWcX@La7FUUlG@S{Flv^z-1KIYn-wr!3PO9rD90^S^P)mmMoBw~KJGInWSwpjYO5
zo%Hv^+^R08^tn|m`Wh0Z{dqp+jzhPS4exi~LdoL$do&LuX5{^h>_2T#r`SAWZg*ej
zhg<hvOo`y+3$IxBqM$XsbLN?&l8g8vX8&kvi)Fphu<SH%<A+^JW_)W{e@^54IYY0q
zb4THf2Zd7@ih9;@O<O->y{Z4Swp|+k%WoOpn|`(8)e_&{s0&4hBedFAzFOeUvx*}s
zhgq*nvFBFu-d8K4P0cpG6Ua6X`@A{s^GA=@`&(DrOg^*r;bZHxuV+kb0;X`B5Z2ku
zKb_sX>0q2{*n%xrC-@#%6*%?MR}Zl+Dc3JT@5K8Z?@F2MJIc9L>R^d?{~XR+GO|Cv
z)cXEmx|>rU$=P-#dsRef|AO3;?v<+oDihXqg`~#`L`$yz-4Qzd&$R0uI=32LTi?&-
zo$_d3+O_j(Qmc<WEB1Q9*ZW-U*R@VNuEb^f@^b?dzD?7(Q1&?|=;ON9Ip6lG=07P9
z_)-_}(LnyWyZna+??lOjdhHc#bKW;LMQq>ro_kN7;Qw6DO7pHC4<fqXI(Q^q<l*}u
zeoJ~o_><XBITmp|4C~>VnczEX#YG3knxM<Bn^sjcOi)NrIM~F-ucWiW!GVc0h3}qF
zfqR2&7mK=^j6~rg-6mN>wJ#b%N(Xxsee*;TiWbP~rKlBThzdP)nN%Kg^On$M(bZu~
zYPS76x$4@5u&u9du6n+Dp8YbP-#?qROtr4o8Lv+|yUKJ)=!$<?u7)=X*05|!JH5zv
z!GeWTbaQWW-4H5Ycc;1N{js@94*K`^*8TqU)X3G~;lVae?gyJ%y&oN8me1?exoMbs
zj9u3}?M!Ep;~CD~94}NDw!ApWEbO7E9LyknZ3*8tMTy-Ha$2@V&;BpQFjFD#WQxbt
zx<A1V?oDr5w%nTJvXFt(OGKkkFSfJc1(TRij9tKQ$3|w>s{4Kyb~$pY6fKsYxiz`)
z>sqf9Ir;~BkNjNwV!FoRL@()|im{4U=4qGhaB34-v!(b-;NB~jz3ut!BtlyRoHQEj
znZ6_`I*J$VSlAv_TJg|9JS}2Eht#Z&N1>8Yt^xMmnHOV<JvAH__nJPcSlnmfrhb%#
z<*&pN2KzjNqbzPpl}jdhh*hRcbbS||5)<@liFjO0!px=9qVr}(CPkQi6G%>0JLWm9
zw~=jG9(&=XqiOkE49n*fUfOvxyX=x?RA%Jc$N<gosjdMGXE-Mu5nI1>dWO@Q{{ma2
zidZ&&x}_1``$<50vB9lVOZt_(WE{5&Y<Feqn$&fbCHjYX<SH%25W%8c7W05=!L>)U
ziq>%j9AinD;wUY?@#ex{ZS%VakFxE(wmj6AO|XJnGy2M>^TED>rn!!~Tmk$}xA=~%
zb-bb0qPru!w>n^_Vx{#u`+pgN2a8V670r*#K5Km~O4!5jXms93Z*j-cr=}Ogs#t9V
z4pwuv6(qD9u8~OW=3RC>i6wDc^h$B{mYL}y#Wn(I3v$_RIVgFXtzwwXBQkq2tEJRS
z$DVE1X8TMt+odMG<iZ{4oC%v26)%t2yKNJL{Z-~wj&CO<7M*0#JR9rir`&QkG^jvf
zwfnmNx0=3QWfwJ{D4lxYdA8)bErzU|3~eqCuIGtdXOomExR{+IofF99ShnR*W6au|
z1Fo)RHx_AcI{D0Ofo_gRV7L9g8DU8tebJ?ho=DVJ6h8SkK~SvR#H{3TaaL{L;wUq(
z=Z_tJ3SCKwv*)Wxc^zzGdGz)A_pYi_f2+1xi&w4gOud@G6d#!yF>B?_)T765?*1sN
zyk9c8x~uKE!#UwCH@{p>w!bj%TD#kY`OLNFC;V7$7%S(P(lUR=g~@CGYve33kP=$7
z_TV3lziW=x2k0&e^0O=x@?i`Y{n@dWCrG>H%{m^9OlL`v74jjxOL+3K56{2l(4;$I
zq1VR3!wD6ptVUb@ALJFb;CGBMoVaglZozg9=lfin)})F)X<#ipsyk`(%6|_H+9xJ*
zX)h@dsAM$M74m3e5oyq`JDAE`aFAR0FYAUzK^BAhz=*_EH#QvgIg=Py)p4{$Vp4A<
zPh&KrvXe*JEESJL3+L|Mb1IjQ9iQ>6x9gXrW1I5)$6FpLUU=Ep!?TEC<=K?}D+|w+
zo^U?#p~Zz!IwRik)?qn+)zua(x4Q4VOXfEJa7r}qf+x#Mx2C&O5Ahyjx;<e>!%6w6
zjXbVJ&Dvj%nN5qi6cHV<#N8so^yHc@rEMopN?dDI6n&Z&b7R(>&Y%+A1SJ)hXw78|
zO6?O)UtJViyXRcrs+o(kFP(7zzf4%joK;}AYU5<j4bsaeAKN+o?}E!-#V^lH-?3!r
z>6(>uKc)1{XkwQ#idfcNvg_WQUy;k}<yflj3Tyn>lOEuhd$c`{$Md&D=Y&7JXBB2E
zF`r^H>RRwHM?vzce4)oghNZeDwn7seZum6`Os-kWvn-@j$J2myhhjUku)+kNyA!r^
z8Web_6x?Qs*y$;>Wf6~1Lp1a43tb6?3wU<y;$)A`m}vdvqLG2gS>G&=Hi?BAvD1n#
z8gF*pS=_pMHS<fa;A0=wHE~>wa}42*oU*{Bb|>pP@0dqcuepvk-7W2E3zUfGXEtni
z_32J}HRHpBj2jcOQj=4xI8I7s&kAJZ3F55$-?2C)ZMLaniEm#PtG3*v{Smw_eUl=2
z^V}|WmDyafsG4#};IzBlT82v|8+AK`)D%p%C>~$7GUb8#<fw^8drHcKHJl$LEIJvO
zSH#Tx=~?Zbldb-f7;=tjvHEWRV7iEHD$g1gJ$5-I&rO-%^CF$4*RdIeu4OnUCb`AE
z{g9Qw@uzK$TNkkFzL>ah*@h-r*2erWy(1PqqQ!-?om2!Z&5vyD;Ira*Hm9lch4W?3
zgxU8@F1=EdQ7PQT=efoBxz5@pyZQCg%0J2&S|84Mbf~J6`z*8YcZ<gJPcoM&Ot`6W
zF6oSfRZ?3ur((u;76T0i4h9BJflm$%XD0LuvFKTpXq^6kL3pK~;;KbOIlVqB_AG0;
z^2k)d^-YS1!KE7l7eaqYK78kPLVJdcsM^vUuF~H=Z{=O_WYX>;R?fO7T(Mgl&)H`+
z%T|@9bN2AAOx@wERr*O=IBU`r$%P9!xhuNeb{A-?t>om*z1XJlq{!&rFA?hlTe|`i
zvP4!3u<E|L$RF6#Bo|}W8=kc=VB-?gYuQs9%)TylQ86>T{{Po>k?OzV+Dz6PG~+Jk
zz3_aaE$uRYwcEyHo0f|zb2Bh9IXF25Y@YpKLsRc2PR<P03vcK8x@~a~W^l<byJfwv
zOR>f0gwvkGvJ<#mvP`=+2;KG-SlOz6pG9*K%cbWNHyu5~r?1HD`}6Vtu6-NVDhYEu
z^;W&q#rfl{s@K*2pwDR;_opfCD&Mgr=(xAS0nP&jPgUPLPTM^-bm=LvwG$k)ubJ)?
ztX$#Zx><io7~}oGjzp2~Te~zO5?@vI>=OISu`9y)kWbp7J2Ot~3D~(Hh=0wmi?ht7
zWX(kHDA%}J9J;%wz+$)Gxk9_lIf0)FglF%)afac%QOu_qw~lWqOO}>a)Z2IOi-NKG
zuK$<k*S@+p|I3l@a?aKtSQ^Wily>Lnhpk$~+<D)1qBECE?iGavn_8z8?_dnNvB=@&
z$4yKf+!Js3G)~Q(QM~a%4*Lz>UEz#-R(fSurO(NlbiOApT6z_ig3E02i@~<<?^d0k
z)crs7$;E9J0Zv`VxQ<_Kx+3B9RqGg6*59ZOxds19T5c%5S{9MYBPGTBW&hEu8y^*1
zj@il;%o8u5aOt$p%jMIXAO8~Dc`tR{$NER>7XNtN8n`jBQPk|t^($|LCr{iYJC83=
zdP`vRMArjqTNoFo++OV9_NO2$Uqd?eX#eSo=qIhy&Tcry%GYw?OvITC5yv|hc=z@s
z{$<JHY`DndV!fqNXn~uD#@U?Se#Z=V$t_Gz&nhffbKAk?_%VhPs%I~4u#ri=;WSq;
zTsmgX_Si`bF4t%Eh}YKKY3R69<Qw#m=fV9Qhs1oiE`2?u^+ZP3Wxm83r^w7h5^I{9
zH#nW|4VAjYWZl8|-+GPrr$TvcllxXC!Jka;JASxc&HJGH2eY<|BZ~@4Z;RgxgB!&+
zPPj+PuFbsr^3Mr}73V|(W=wj++Pd=G>m%n5+~{k6*;?@PT*u$}vkasY9oxKHJU@r@
z&$@9}>sl<wy$G$mo(voR<cSy4-rNo2h!o;bcxo`y@k`R=NChXKQyMW`k^+x^TF6QA
z99_O7!TsrjqdlzREPi=2d<1P<JZo4QzD!Z;o!~XavBXxY)*#@6;;qGjQK5o;VLnlw
zF)r3x-kz~6Hs_9bJvbuP)A%FO!~TMkT1S(H%+Zx=nSvW;t>|^KkBI;5*t+7Pv%O7x
z@R}wy121W_$@U)cj(iKkCWQXi&S1PUXa1VXCy^?RYoDIp=W%eY<weg&t{y*b9b$<V
zO%$9Na%N-flg>m|$%%nZFC@Zb7O?Db^H*>=W0K(5kTA7%f#Z==dPS1bEU6sZ{H?b%
ziKralt83@6^Yu_z#I%5Gk%G_yjTV*@u33zJ^Lu%NPo6r~8WSOP?tSeY2akvK5-g`!
zJr<?h^5O6}@%B<^BiH5LsrEY(S4_R7@gv!OgQD%4Lt0A`9j1kzU(1}J^h~RV(Kf)<
z?%N@aJIU3$moG`qb$pX(yt5<xMw1%HqcFZBrh#{_gvu&9u`4^>n6Tm5HQtjdYMgCH
z({9WaR{zCqJy+KEQQC(qmlLjDzUiwl+d}%kUy<9rrzh<?dmQdQ3f-fS^mkE;=ds70
zUKytI0>Zok*;<X%T6Dh%EOby_b%G@{%atkQtjivOcQ5C;TnO;+ajE<{&sB!yz3LI}
zor@hWEU&z}yyj9%?WGHwpYU%J@jZW)S$ff|)-}%O1D~v5eAaxWNlhe8dfCGB8=KS`
z6df&Gbt9I1FFpOe@~Yzl<&_4lwhF1m3*z@Ik&oIU?-|wk=O}Be>BIv9K5?>V;$F(f
zJN6v+@nQ27Gdib|)Hr2}Y|EC+_J_5z^}n5QOu3S#c;w8V6i%hZJ2G1Xw_e<7y139L
zwb<@j#e%EF9*x>loU->$b6hh!#H*31C2sYZRL339ot9?)?`Jv}ys0TQ$-}lms+oh~
zT;K}3u0vWDcURahvtM&qI*eO;&EcAjOlNsp^gUct{!Di?S+&UIRsGlI=06K}=SJ>n
z<ZBXLv`kN8`8N+`vn2DEtk?IdO+1mNR&?WS$)bBt`trooZ7SO|vwGINYMUu?<nCKG
z$%y{Dviv%IJV(zYZ8+vBe&&(mkK3UN4J-?M(!RcM{IbaBO23QeYc@XLb(5O7%z}z|
zy#DuaMbC&&>FHA1+Qjp;jb~@0jo5|SP0vGLq_14i!g%w7@lWPej%l_H4<&jO6Z@D#
zFHAps^>9NN>sE`od(5=m%vx6`9(C8siGF!naH5Lv&KJ2>7Y~XY@rnxk|IfBZj8VN`
zOlLad!3Vz4C&XMt9FAq3VA*!mi{)36_LrpXSJ_U*T`N{&k>27|FoEf;*zrOWuKzXc
zP9DeiF|G}jn-TRRM`>!i$d}_a=h|63C%CNP%3CTXbj#QG%F?SDuCvv+@^W3PZZ7U}
z;XBV68*+uox+VAGlf^AQtm$rQmkKj3iN-qZkulo2w&Ta7)*q9e=o}SuY0L;ZzTyqX
zRmOLjpE|PCW@by}<+KIgxTw~?G0UsyOw!W1NjvWzE4sLm_geCdMMvLgIo+6>eBsf0
zzSU_n0v*222vcy7O3V$rFtPaB#q0$<hIw7*LezV0@0F>!^L475`ZX}Ay2wq8^cCd$
zzxKwd+7B}i<Q#anD0|7n^qP0iKMD2ia(Z8_rN8;gaorWyS9fxMd#f5UNB2jee!5so
zSmv#;nv9RJt=ku@XSmU<pVOr9N$~QLtG`x;?!B<SdGFQVd*i#;z3AGPaqz{}zc(`u
z{ka;i>ely6qbu=fN14thrkjjXEJ7yBgY3Nih-A9tKIkb`*jyCG{^=N-5f_{CW47{w
zycNd+7b$GBda5>~C*<FRY}?0<1?=317fjnEcF8Qru^_xymfJfed>(I$#zEdY5>vfp
zPtQ7(9=1hWs9MmkX5}`QqgzETFImUon3I+_$76As#}c6<^;h3MD(KKZ^7hv)=O7u@
z^ulR|eFxQc%>Ew{;(op?YuTZ~UKIzWpu5YRZkVlo_OK{6=aOH$Z)fhiz8IFG_YB$$
zcI*CM%sQTP=IN!oJug=!-qd1{>VFYblp&DKl6!1p-}CKmZD(Fx$l9^4hgZ3@uD~(J
zD<I+Z@+TKF^a{A!U$0woq$Z$eih;m_z$5Pp965R1-o9)r`N|toIoWMPa@B`8L-YKw
z69TpOTC`mpwYD2Q&G0r!@77o0(B9y*V(CYTj7BjRMssgQDbX1j=Z>tI_*vkh)V>o_
zJ(MO|nF`LicwaYi&A-%~!y+3thQ714-<)~pi9>+&gf^Y6mIeYE+B*vi*rblU(o1m6
zo-$Q(%kP|4rrB*q_uBGKu>80Cb!6JtH5T2Uf5#o)o6D;3m4kDM*L+vb8!t{dzu<CZ
z=(wpeD{sN&29|}9(_&7fu66vQ;FPhpD#0~t<x=eo2F)vLcNDl9NxIxoV-Y(%$Eb2+
zTg}FfABQBwdxN8l&*dBv%V>)BaOAx_%d*{AkT*!$o&8F}mLGp|uTE7nb4|UL8N6a!
zT6D4Sqfdr6W(v2@eez`6<(VxPZ{FN$Tiur5b~66{=R0rRBy;q{m(HD?&^f!|SW<E0
zYzdY*4XgExUT^Z<mg(_)smm3$3q2f8agHpDH#@Pp>xI0tOLi_e$T(AgJNKY(&%GND
zBza%9$L;CWJo-(AG3tg+H$!ggE<gQz`QraVpXC~P<b#^79<5pNV{S)~i)o+>mv)Ym
znKz?(wXwzS(*m`|;mO}kT-){?T>tB=e)Prld#{A8eV8Bf@_ps(?J+VwDQ7hpPJd=d
zf5Z4+KGmVb&YNvQ(Cy<w7yqoua}rs}5&a=8@WaGs{inn#&bEjxy0YEzLQc*Tt*n{4
zrzafRGgsb8<7`B7YesU#l>!I9*OPRYmfim%d1xZ9cX7Dn3!^!LQg?gKrs#y$`mN`y
ztl(rkCzin|@&3A4@Wwv%SB^TR)ekE!vF`k3XDVr}`X}J4SV(DDwrRHI3`^_EJ(=v8
z+c(OGR?LXAbk>M?ke4O%n&aWUkII{#Iqdl=7yRgr!pZ;gt9SZw7_H{p6x+W@QGQxa
zA)Ca*;;ltF={`nD=4G=(K4mO9&K<U`*7yp?>`!x*s@~*9%O*yi)nlJ3p5y$YwSDu<
z*(W30x3jEX#~HG6O5PI@-jzNcD(_b@n)iHuuG@1^Y>IxBPoIhVX5Em_rjwac<4Q~1
z8%6F|SwxpN#%^CX`+Rvw@`l|XEEc<4p7s{15wV*7#N*zrP3t$Cs|Fs4V@*)u>D~4u
zVNZ(f@A*DLR;Fz%b>d#JKT@)foXTNn_g}?RAo28Um301>@1Y?Di#$r$SG%=)S<Fy7
za=vS2xr|Wm!mIA{kI5@WFx{BuE6iSdgeBx4?}Hc9SY}Q*F6mc&tLcB_FZWlEOnlD^
z9J_83JuRR?guOwfFX-`)eq+;M<8#6KtrO=LK8oBLol`nxb682s*OG?Pskd!-(vO$>
zRJx0ws`#+BIOFBCue!W_%btHYadE!s^F0ru>?Kw(d9K`L8l$}~CSS>m@Ak|!vYiK?
zY-kl!W>--E(zULwcAdiQHM`~}nVT)p{+6+Y^Ij?|15e5J@=cFjPf6#V%HHDkR>)-W
z_dM-6Q?KOg_&=qeiRFNkz`v-Pb`}Z4U`^4*nJ$)#csDxS`n-)R^zFCMJ*(YrFs;q8
zTkUe>{+)J~iHvrV9V{CT*&mo=-1^=`TD~~`N^$l<3773_AOBicU%dYNF3Zy#6&2P0
zw|DiwPqsYP!md!e{ZE^D)4|%x{w2Scn!lP+-f~gC$7}z=9pziOKiGVaXFK|2&hwpj
zX6xBAXJ)N&bMkhZc+BmT-b?>y{48Qec4xbCDo1B)bm&iF+V{EULwCEoI9J&WpB+pt
zlUSHUR3tJkBphsHnxY|aAfS<n(@RBT#)VVooWuE9Z)|$}s;{A6(X#5vj}MC*detMh
zo#~jQ>^V`F^~4g(&kN5^+vau8>g6fl)n-dih&(-%c)Cqfc!|N>A10hL+rQP^+#I?-
zY<Jj_69sp-JY6lK8>BKp)%#;kB&Vl}O~|2-Z_iHV^;gTh8I~8<#4pa}(eOS0o(iA*
zqpPpO9S^f`{b$n;o3lgV$x)`sIfk#dygYn{Up-^qrjP?H7v@`Qi|$Q*DZ;oiTw8QP
z$X7AGRsENqop`GF?#}i~?gvU=)$&#zs^w<T+url>@X^xZM~9Yne{Mgwu=vr5mEG^2
zGBxBVZk4U6`1ojlaY}KKh&JPM`E8b-tFE36TYGQcR_`Ahod2>gtGxIh-^lPG+q-E$
zkB)#NBZrNr66c+4&*gJ6CTSjOzmsut`P;n6mrHn9KWsS6pSfwVXmnKJ;vTaT8jsCy
zF1z8-#HliCqY_KzoQvx>dE89l@!fVIsas=-fDxzW4XsZe%GZN7?K|Wbwc@~`Urn3V
zPg}*3@71^NN%kH+ev=IfEc;#l2RLZ57zB4|F1exLYjQQ^@+OP;*~Q(K@~eXNuYP%w
zbYAnvpUeLC+aDwaJY8b7<+O{auLuL9RPL9P?F}9d-R)6<{ZrT@liu5U`v>LkeSanS
zsbdi9jw#B){+CLwx@IahPyHch)!*~uO=jl4tr<Z&iY+y+FBTqbx!S)dX@1s<8+)ge
zX4^NZXrx-s`!s2qYSzlToy)SnZen#=&>*yGiqVcLKC_E+jHW4d%4MHesP?$2dtvf?
zrPm&eZ?YyWC^xm9e1F=Psi*bo%T053);~|;h}-^v*CKJ+o1`C}$5h<JHXW1`He=j<
z&7?ct<o8pZEfcJb!x=6I8Jn9f4XFz}f3<7%e_`_(m4|!N-p*LObI09|&u=XrtO<Un
zXk=kIWwKElo7Br$Q_tPp{@kKV&w9nd?o(G+t+fknFgqojAj0VJ@QaGsE<e-P!4F?^
z<n}$A+_W)OUTDHn)}uZX@>rEaw<vvIvGzk2r=iHj<vcrCWpafAOOlU@{Y)utl1QJj
z@!0;n6wdv7WTajcvAjN;?Y%$al+o#Zhm~h6KJ`_p*7^!3)8wyAhhyHX|97Uwm+SNu
zn_%DD+5VHi-N<^*;cwkq{&V3p=Gb2UC%>26^nbdgdfu`pP49xL7+2Z9u?SaWWon$v
zFd;!W^7J|H%Vq|9rLMH^I5cb7+St~sd*?;3WYzA=vfMgV_eRox0i{ym*&0d=Tco{z
zHO*(5#@EPiIG0Ivg6+kptt#6(61F!9O)+Ze=JB|BKy1b%k^i*<4gVgWaFKGGaYSW;
zYT?22oS$0_(hPMHa)W0aEW2v(@d0a6o1^SutMbWmpYj7I2|hZiq&(rhYHE3^<z%6Y
zS0pDCvYDtj&&+f>ETz5cnCrfoy<(}?w%o0mn0X<)letxF%Y4oU89^)W6<+(X<=%&7
z+;LO<N@i`I!{YKVhyRLl<(^{`13PlVm;(5h-w~d_w&T6Z)*16f#5NtURr6BfoZ#h=
zSE$%qkfJJmC_wFC@{XV_3wd;un!C*m7$r_}6ir-nu~%%$bPe7M?eliuelEB2|MYWz
zH%!wlOwrr=WS8OJOA3XbW~7ybpSgQ-rQb=Jdx<^IH#snJ&1%*XDQr6|<nmEywh3cX
z+Nor_k0-TiUqxSCVdJ{3z2Q{N)7v8M5mQaW3c7gXjYV%9VAlS4u}gI8hU0(zjO737
zPE<SFHTw|DvIXTAkG8T_d(Ev_w03?7ckZ=ADXjLEdO}x@#bo|`@Ly??PSd0d3&M@(
zz7mlL5Hae|`<=K@+1=|xi`<)pA6^fO6qYTWeo83Lh&RlM|J8=`o@bI=S+oL#H&|O#
z@~refbLNDL*wM+Bp(fhVXT_FpIBIRE-s8JEGH+U&@$5pE-cz1^7dp(YEsRv}b$4FO
zy!S=u-Ty9YGXs9F;@6&f*1hd=PIJxG8%HAjHY;wLzwL(8F^%-_?KeZF%V-Gt+OQ~1
zoU?+rL;G;oHUlrdBPR+)JyW%EuXr7@Fw9$cd0|HSO7FvRY|}e~=Dv!sa=LDnX|uNV
zZBbnK?Q4IZR&UvT`Axxfwd7xVW`}~$hIU5V^vUq=ydZLAeoyd<tfq-;J-Ge9t`pz8
z`s>na6VF2uw-WuPakS2zc1fhVWs=>Vb?vpwotCe@wNv(o<SC1dtK5}7F1Md_FQrvu
zjZ9q1>|L5W5(|WF1GSwjBZW@1HG3;^bKDd?ByjG_R>jh2f9|LoZ(GhSuUweuyEEiM
z?7b9&+dISC5`P<n%TD=kV6umkH?(I)sq3>O$<q?Ew=7__$ZYIPO*}Po&JM|mp@k^{
zwOQF;wj8~canUTcWY*<<JD$vWnU?PQxT|#EWVa2gu2#t`TsJLJ$?esfw!W*DJWL15
zqjFBYvW}HzEx6=t_U(vQ?#ruTx@^`He$V)MsBR{Mc*qeYp^#9f<y%fH2)`M5a;wpy
z^){TID^o5vD~K(R*>$OX<H<yu@Z7Cxd%x(%yD!^!Vr6nrhS%&XOVpM<=_{9BaCwf!
zgz0JDJS#7%h@74C`^mh6-!FCDdS*89z@#{D#+(JaL=qxD&0YU>s`AVze#_{h9LGy6
zA!{tmRvim}`>{mT&Q_WA?~j}QMl5Fx{{LUSA;!oyT6W(wrQ($yE;sJxrP#X0ayW~7
z<X7Ez`^8*oXP$%;JD0(Zmg)sR*Ov3fdL_0*-a4Y!s@byj^oMf8GqI*~&Kzm!+Q`tL
z@8@8(Eb>ax%$tUyPn_l{o$j!?nx=JmwQ+P&_}o3A*-?^#E}w6D3Z!1kdZc5S$Gf@W
z)s@`6JN)k#89v$k*F*cutF7}3)wcb8HRaZ%W83mqg{bOGRaHJ$b8TXD)zjLy9GQRE
z{u;!rPpucP&Gecay?^`G++#YMZ}%CxTv>ikVpXHJbt)VGdv(cm##66<cjTSEbFoNr
z;X2LOS1*_Dy4{%^w<%kHU(Tc5&%E>FPip@xo!1rmJp1JTN@M?hn{;KXPv!a7L>tD(
z)b1|Q<y%yzzqYFSbZZ;aa@{2#Kd;+7J#2TnzWE_bro$`mmK+J+TyjB{Ir;HnJ;vyO
zx;azgck7nK9G83kvA;Q{c3LT$>7vNqZ=Q7`Gqzf-j;!SqU$NQbjpxiY38DYL+?fCL
zTk}qf!Xv$ZSOw-D4&Z&m-_w8X&`<Y$v((Soux?nfRLo<c0-N4{t24z0JcoU{i}$=+
zA{4+>zAFC+Ux!XkWE9tfcS^6gvRm^`PZwffn8sgG_T0zi?3G(C@4^hOhP{24_vq>`
zy`9stXK|co+p4@vEkJMEN<FT_wL1CMx3p^;&drTFWP7f8f>Y}}mX#q}{yz)q+o1X{
zzvrCWZLjxrHS0u+0`8jd=;!EPp4Zfvv50$qo<7T~r9o#(A~fEZ<bJdGurG!Gn^?!u
z9F3z@>l5A_RrE-2e;1H=>*(@-sXyKroG;7bb6?{rag0AzfB&tucWaLRoW}1ljsKTL
zU-GPuUn}0PT-5c%Y+}W<%nwu2{%p_^DHM3tvsmPG#$P|C)9akm)1H|v@G)0tZTP|Z
zrKG^-mwwK%XExiO%~tDI<ubVR<k6c422975w_3Ho30T3{9cYu+KkdvJ$@v$>cuzCU
zG;~imK3(NON7w4NF1h7iZ*^xmaeAHIUU9rEZ0(MSr1w4Qm3_y}8P4y_%KfnAgW@(F
z{k;Eg|L*Yl_~lVz)AG=L-7hLMZ|!;+{AGpqf)5<$+Zc{&OXQwV)OFYIm^gja#Vi@k
zZxQFc_N`Ma%&MQ3YN!;%SiM12V4_m<xvw$?%)VZN!Wm3ampUyTXzWN}p7dT@Y2r_v
zWTE1mE6ey(R`H+kVl&yKZ?@sYiDxMv?r~H(6<T(=ty%V1rNBV+<HSs%^MA`GD4)pT
zUXd#md$iW$VfH`%kW0+x@)j-hdSuP^NJaE%Lx7RZqC(~ELV6orq(oC4ju;$rb62(f
z@YTy}fe*)29VaG>1g5mdmR=3q5iEwzotv&bVx9a_`nJoJs%)lXg;#2p`-Qw^J?Q`b
zhaTJX=kL?jvMu~SiDw2+=9`UL4}-WGlI;XCUqmG7Tz$4Drua)i>gQW0ZEta&oguLI
z#A5Ni$GBJ<=I`R(v+YR2s)-I?mU2(Ntb6V(<L6$U>mQazcsD-zU^!Qq*>RHYM&WOA
zL3(e_++UD)l!GtvPl3w%VjIB>`+JM@PYWpfC#A$}QcVo9+qr;c*8-MP59~e*+kX$X
z|GJrnE9GY$i_WxuA?tO?GgheUUsU&XTeV8dY=V<skl*TK`6A(Y>--8u<5fkUYyH`E
zraNt-{)Z)E=K6Z6!V~|!w`R~hXR=hR@#4!C!Ao%=jH*VeCpYZx6J+Ww)H`@sclVKr
zUP3P3A=aAe45tL9CZ!mx?y}?mKiQM}jRCjLlrsnHP9I?T5X|6nz|Nb&+uOl=`4yHF
zqn~xp{a&3ix}ajT`=igc2BV2D%cpcon{TctNtT`b{C!a~TW7MIpuc=%lh(})f8Wn4
zKAW#cF>z`v{XFSh@V2yU2i6?1QnQ2F`qKjR3WOQ9E6=;<uUNQLR3h`?8aF1kr<P}y
zw3}_<o$hRPHuz!Nmi$X^zh0Z9;N$Uz;js6&wS2BUyoKl83<H#;kE;eJ+x?zw-~83?
zcbolA2gYRzGxs_DoDzCPt<Yh%|8WbZEi<~hR(<V~n_||(<J2$IqcSB^$W`mGsi-(B
z!;L<U))LDj?zdjNv|glbcW$eW{2>>MP}}DJCgJmgWlt{9WIL)`{UxkeB$m0Xe~B^w
z#o)_%PxPZVP2ni^6c+Y!erb2M!G325!;TVf?+1!L3z%Q|Gnjqz@;{VpZI*0#h-J$Z
zuaA>;8(RFb+_{f6#A~S<on5l&Rm$TDCs{9URGoElhaaCwI`3+({%31WuUjD$@Sf?@
zC2o$xuXnfc-u(0;n4vqg!7p^qS@G}sdsV*5?Ps3&Xr<__gJQQ7wXdWFtMi8b<o!0|
zQOmXiExS_q4hqKyZqnO)Sxoos!wgM<-?{qlx4Q9$DcxqWdpwz|E;NGuyNSQHCEs>#
zx5<?^-f2sf|J>)~sPVCICqt2ao2rwau<>=rm8VkgOi^?AztL%H*J`^d&o-P%|1i<{
zyvN#*@0#^(e?BdWY3#k(wYHX}jN`1E)~%P;S55|9IsE)e8h4k-B&7=u?^>FTirm(2
zw9O1rep?!*7{oX+FhT!<@^azE2Wk!?D?HeB6DNzP32yd0&EUFd%Yi!yUay6d^}~{N
z%aRShDOx!sYlo$Hep7tpQPp;}Y6*MN^err_vJ7^Lmx(NhNe*|A|0vk%!mwq5$C(r^
zH_s1M&u6^ht5Ex~L+i$lM<!-3zWCb-uZ~mpk8aBN9uo8Vqra(%KHFEn53kB1l`^B7
z_N09N<dkYwzS(<=k<BS)&dqFgVy_i{oSdJsMRL09MXd?${^@si80+f(Z?_8kq+I+&
zFL%oRV+CJ7C0AM=R<%?pNZ!Q!qPU<@wEp9pg3UgK*@sQeIVrL%vS)DUWoF>GBEfhi
zqIY{{Bx^`y{Sw_-Uj@w<YP)gzS3K{GS!UwU?704`Y0Tl@^L%>jKKAAv^8b8s?;;`1
zOeZn_ML|i$RiBfzt(I!L<(JRbwY$PRanCpL&+NK4CwCoQta-<1YgkbKAJNJ{QN53A
zORg0QR+pM3Y%-X=g<I}hit0A6i`VTQZs#_+#-bJGeSiCm$KR9PukAXt<AA|5SDwkt
z8V{yL7hPatnf>3HWxFH;Q$mBQK;`MJMhz~HSwemaMd-XaW~6bwF*9t2R4_x-S^32O
z6O5DJ&YaiT9NZ_jvqwu((#FTQC0~gvW?Ol!Ap`%d-RDjQ-)IQ5?B~!Ao#Vk8EM=N=
z%sKn~W3z|OQ)+Jnv2}0fdF8(PM(eFZDGuJD+m$<iet7w1Y1r2=rL~{ccIZjoo|K;*
zrT5@-`1$SK-!~UzuiQ1K(zT~4;FLteg&X#N4$q#vYxYBlzS}nfWm1DGKB!2Y54>XW
zQ?A@W`nu1YQsEQ*JPO<2>V-S9TBb%ko7SBDKFUxtyl?fKoSrNB^N%vGjd;}aMQQh$
zmiaoC3m5Xf(W!j-U}b+rcx~3h^5^Q@t=HXSJ*P4;hw^{8cS|^Yu6W)|!`D;FyS`7G
zn!QC&IVnZw|Do!Y3Ei8O4Ud~oDVyoF_llkUwWReT$>wG$&eKw&ch4|3vwZVi@xykW
zH<LYohZ|TXF#a<#Vo~Yc8_CjHy>jQzb_sz?QtuTQ98!5BejZaWGEFX<@_zNt9lKUX
zSA5cwy?(LjuU_T7iIpEpCbmxV-@Rm__vStFOs!jw`0VO-`J3f(Tse!CGdpw9-bZit
zNk(Q124%~>3NcGmJfou5bGb<FP5gbae1FMlk}|W!Jd{c}OCwelm4&9Pvrub{w6Fh}
z^1gW6^ux8HsizNBn79Wwi|{n4DX(O9Sb2R%@9vu`cP<VTYGBWdVO0OAr5br$o{906
z1sBVLzUDb;s^yXUXB9pC|77=a)08z+fA2WIYRlGcft}IUr8C`b%<}nhO7^DnZg;WY
zA7duIofH{S`|bCONoy5cf2#Pj@Oa(a(lA56JS1OoPUZaXjb{Yh_WLdMnQ8m^tfJ7h
z&H0jB%71$nODFlxPWGQYqgm41Tdels^^22#syNwP`e`}k)x`R2XM@J*9n35%`+v{2
z{ku}!V75WeDn<n+xkW)tTT<&)=Jr1Exqb7klEgeyVVgbGl{LyHxd!^bcD8I!KC=4K
z%<m?73hT|EAI`eQT|Y}V)M(dy9mz!#Cj{#5;yv)%ao^$WgJSzw9kf~IaK~r2URe~}
zctUBVc-8jc{-9lljvxLOWOUyCb-t#8TH=49+H$D~e&<VTkIj56yLj1x6@pDir_Wl+
zZ1*6uraI-1N=3n>rL)w$Vyi741u`&jZejSOkl&|mUUd3V;r=Rz@2*kSa!(rGtPEq*
zRM?U_Z&P#rv(+A#LxZNPJ*!$bSLm6uQSzf(d+N?TD-95zpS$PY<MQes8-#D`w%uB%
zuxHapoz1d)UVY-7>h$>h)GG!Dr`I@hdM}mRdh>eH;}u(7x79wM{yH-Hbj*X4uY$EZ
z7cj55IV)ImHtUTY*XtTOe=90Hy*Op|@#30y@s+C{#^_1;zrSCnz){70OyTPB1wVBb
z2i-cfD?Kd!vdQk|?%mU0?%pfi{I+Dz(c6_P?#z#7(z?CA_tt;?H5GgHR34m8pP^<~
zxxQLhAv1WPXz6;-FLRG1PgV@wyDXN$=e@c|;|3KT#$b&o{o7MNZi^56#iSl{C*0$K
zg|&&n9Og;(hhN%8{7(B=W_x-`yXRBmgBP|b_E>Rr{64z3viISP-oiQ7QBA94=lwap
z`1nzKn*;yV&c!oS#q=7k^P3`iV)pqwD?Xid);hl;PX3=^(jcUu#&N?a^^gc7E5F~v
zf&d0*#uy8Z28)eKE_%X-X<JTgTztG=!MSTqhl7KsiqMiRS*Dwoo}O-y+&9Z3Z5e|Z
zQv_#*a>EpcX)?8SIj+x=nHb$g3v8kSotTz0P4P>S+PWxYNyN&iJ-bSqF8yC0svak+
zJt@aygY4O^wN;@Dq8ZsNeG)v{ygBt!Pwmi5Zd)HOTzQ&7EZ~sKbfFIZ@);RPh91K5
z+}sR2JJ&btX@8qFP5b)la8^c#W<P<35+;V_!Hd}x8cMIt*kC>F%?i_RZ<Z=~pL6R?
zS-_yXZ)?v@BezpWbeVeO?>n{qba@=K%|+fqc4z5*eNm-5cCFsu-^XmL;P<yHySPZ~
zJh#LCeM-N-%bbpxryp-u-R-%J;qS@)_Q(GUf4m|ez)+{6;BF&&BiP-d^-7!H#;q5*
z{GU(UGzoMwe4w@PRwpke&j=MqZ)LZoS2}`XRb4w{jxKZX5{g-R#6$Jal50^GZK)Ig
zM|1l;cpM{Gc=2Fj>Xd*d6W!D*pVVwLjl7@gTBxYX9-R|#q038SLd&#etcg(!32F>1
z3=Av|YBQ6W8W{wHcV3${+c)Zo+HA#xlF#RsZM&(Kbu3q@VSYp2M$Up`y)z~)=*Zi2
zbkV8SFVh_JR8y8ZvP5=Xa6VxgBIq_JPfJK$rjhHFn{e-~Np7MuvnH-kwr!o$WLc>a
zvPRfZ;_@1?hfx#P?mWbzExq{0s^Dhpr>hpa3kQ{o2AfV(4RBYzrsl9&MJ@H}TxCtw
z3*np@s~fi46)lUDtavCI5nuh4b;iUt#cMNmw@*DVV^8-<Yr(xOceV4yMXx?TG`S_U
zES#PD?9zt+>37@0qB2*{O-sx2*r-;R9USU%HCKG{6T#3+$4YMJ%GdVfPEPSkxGQl$
zP<>8Gki&GB(`QZFCzf3D6`U)=e)WxXk#OdzV)t2WXXlnP_o~O0zwW$#&;3JT%Pa3!
z7oXg_@=bE<7mp1|u@~$1<mFCm*yhLau+j0Ym7w2|J{LC4oHZeV(m9(${N1<x5Lwi%
zR;O~XM<H<b+P=EtkmLtev%Wrl=ung(KFNKZ_Kitj9j{G%qI^<)(o>&v?oCr-A}lqh
z#i(6;dN_Ge*e-_U(JU=9nC`4yenNEj6m^dc!Ec04Y)kCkJuknF<HdZ@Tq8GE-ToOG
zi+a?Szua69%JR~ImD51;{}(pJl+0xUwjzQ>w}VZJeEgHWpBFEYS^fI^i{<x){MS6t
zO86-%ds{J}ruj&rx7XeKkKBKYpS!kI@nYAuS<k2K)Jch{c<B`PYm)fk6|V~?7&<2I
zQaD&?c+60AP7j-EyCrYjF{9}lWp=V1V?6icMZ$fr8yOPIW{I8S(|)!2#JdEK!@o}+
zQ?Z!i@wVi!U{CIuBnOxHw^~h7j6Bafju<Xlv`o2GMWWxL+w0!hNzR*R$3E}c5Xf=y
z0JB_&#@gVyuM+l0HCg^}d7I^56%@E?n~3IyM&~%@lXW}NCjOnUHvDhmQCa3MlbIqt
z`ty@ny-%>6i1zGo5>{+&VVR)Vy=8Hp(EozD?!U}Ey_f`Q69T(KxFqzrG}bY92Mg_%
zTfXE(MN5&gxAOC4Qzmd~B<3&tJ!vKrLrdf`&eB||`Ln7lb{h&B`ft9e>K>;t>Eqs%
z@?$KXM#&P(jXfWhU+}r-qNK^_8fn}WoFn2c=jy<u)}qKivuUZ=B;)%FUd@@;<om;C
z>8g;W#vx1z%dK^V8oP21sy~11aec;2|GIa<Dr+}*`fu7cN%v@9!K>Hzw_kdsyL!jd
zgo2-f7RQ%56icpbj13T<c<U--pP3s&cOj35_N9xRC0F9a7<;(Yg4o0Z0upAuY*2q*
zbm+WBPOILTmq+%cJNIYyn90qY%O9O|M)KW<<(5kx{^zlGI&<TuUJ?h##8YgS<~*FR
z(8^DSL!@1O?vnt)^edXn&%d!0TA2B9&73Tcii|^>e-|8C8`Q1sF{{jUSCa};)Q!dK
zlNL?3%zNwj=c1@Li}nJ}(i2J<!kingxv_0$)yQL-F`MB@dUu#hQOdMAX~DCSrnUNI
zPwx1<<dKNF*RkG99?RZvcm;9^pSjx3^F%85`MNFJ_o^<4UKBk;*VA3?ieiJQhd|`9
zIZ~Sx9=WxDTP$n*ROBj$izD}}hIbNfhHV-r3RF&qJo8R_E;Zq4rw`|ko9Ed!aq=b{
z@4OxoB4TbJyWZ;kUZ$H2HPzqNv?wg8dNQZuo5~~G<+|co`j%W-pZ*80miYI1zr1CE
zlK2_HIa}Uu5N(v-6t&Q|PoreghKI|dRRq?x7Cg{<qSeeKub#xt<z}eb5#A-J%)nK)
zg;`_j91l)q{htd=dW9Juawt0mpVvrWyua(^p^{azmMiTDJ^fZQz`ba;F2|?-H3{yz
z5r%A558r9HKU{dVW43hf$AHtGd)GBYK3=|7C|x*3L8M4&=Zc3pjREmrwazU$aPM)#
z^K*t5`{wn!ZTZ*6uN?SEWd9-eRfeZR*Ja5lOTT=%obl>J)#4KA2C<b7n9hFe+rZSN
zP`=0`W@BPlYtKo={6k!GJXVNJs{NA6xIOZ$o2uJ%6B$Oq<Ba}nE4O$pIy1@U(5?Rx
ztBke@W~#n$nI3)lN>OR5QEdK8m1U+2^L8*?C@P=)>#4zNKfn7^g^ThvoD=<2cE3pO
zon<1J%D7>1*qel-8nbHODjS#XKFrckx=?b@e97k;$sYNluVWtOmIWWa$h_RiV5_ms
z@1JwU6t~oyFZ$qAzGUv2B+)RoI}zpWE(dw!Bm)Yb3irHrSs=EmL-*13t|*U(F6VA9
zIko-L9$Cx&WlMY)bcnwYH_kDMd;dy<^`q;0o=F#%PX8%BIZP&a#^Mjgj$&sg1Rg%r
zCi-!u@75izRkt=i+O=f4anG;PQVs@9j#jbN3tAa64swOLFqS*NUFhDH5i@D6ZPG(u
z<Iu@B=1h_Nug)2DBkSJO%b_n`<m@RI%a7C+GFc?6tDKdqz3^ePU{NmD+@eta*te_9
z8;^OezhR`)Tx0ZdqpX-y=Iqzecf&;ZMOm-Pu5{Hl3z$9iFzXh(MGfutl3vFw&lRrx
zs^A{*Q%I)0qMB#sfekxDW+}9A`Ye0Ac8CAtkZY{sD-XI@^S$35GMOdZckfBX9f^;Y
zt+KCwb>Vuf`=Yrmzvo!1zhU5;p__MQdw+cC-nkx&|F>x9nw+Y*-jgIc^}EcopZk?Q
zT{XT~9M5IwH|>!CyP+|=dg-gn(i{^Q0tJd#uL=GYNmL4K<ar^lKha-ok;2rgTsEII
zl$na3G9|SeCbdlz*EVzfuezXE{%ewvnB(NYpvl3>Ka&I*ip8x?FtQw$be~@0lqTh~
zJXrFF?bWba#*mhG*M)X9O3gM%6=p0;3FrR2yzIgBvIoag@1(U}X$ukk9Q`9L?Pr<G
zm5r>CS7icSWDC<=Yr+c})P<_tD(-BQjaZ`jp2bas(Jg3FJBPVx5VJx4({|B^E)pyJ
zrCwyDDZ3XwkuNi`7u%TGaa7*oV#kKWtk;PVxtF<CZd24g$Qbm&)1@i9_ppU)N3y_1
z%dXFs8(eZWZIN1XxF$!aTXdmXSewYE4rkTCQrDx_cehC&U7C9ONsgxgYoG#apaA1d
zbt8`x-4Df$c6WH+G*vfzD8<0Q@;{ovz4QoQih}2kBMo<0rBj3$Lo)<_rpbJ}ZgsW9
zCg(=E^1{S|i(dDq#NRR$;tKCq`k2lw;s4=F!7mFR&ILk|!s;>^jlvv43*08?9@Lg=
zDhzkl7ThMUY%VC3#2szGm~w%w?xX&jrs9uGLAn+}ima1VN}DGg*8goJFR3d3XOg7A
z#b&l<121(cKlc{@<t;oN3?2-zixq5U8MFlkl;(bSzAYvkay@msaBkp(@CjyIU$>OT
zyqFRoIpsxjYD$ohjj_zP7P+8=`kZMlu}j->GK}l5$2Gc#$tjxHCQmEcQIL5v{`Nz`
z3Jv2jH`CIq?Xn+@gjRHPT8QXP%;bI){$HTXY`(Yxm!$gSi0NAs_+=k*t$onB^SR)k
zM?&jVojz@}Tq9Dmeu`>zP|mUC(Wyb+kGAFbJj~tvvDV~?THtXhhn1z*j!C`Up7KbI
z!7@|$fFaw(tCMFLbiZnDJ;pj^`{P!ZEmj$3T+s_Sk7mmFN~YZu?UfVkdH=1>Uo!u{
zh2UXkn@`7$b(=*FEsSeA6qmou=EBE%;e$5z%JEE&0)>h3kFE&tKNM&X7nEhJ^u1Id
z#~7Y(F0J{{Y?qRb_>SqE--LKNbz_CA_?I=x{t(=DfK|9bP$ETNG(=z7RB&QuhL2K`
zm4*tdGox*jli^7xU-!vol8YGAT=qNWEeR-lWybYCFk!Zb+M@lOA<G%tPF%~m$TsK1
z(X#j%G0`21ud#*f7nKbyO*`z&*0rcDsMYm`X1Zyy@l%zG8$#_*zX;BqR3N@k@QI>a
ze9O{oQ`2;pa0$bOKNJ~-zRj;~)e#WOoUqN5CCH31a~a=D`G!=vKY=3rmm=0C@P9J4
z4;K}#T$thLq&okx*L|m&cqgeP#Vby4x9ZYTi`ZUfS`{0vHdX9!j@vhH^IJXrt8#T2
z)i-CxKAo1gAS(9g##ZN_v(ygGx@cy%Lee>-NNwMe($kV5XDwG>@C<QeWmp&7+7u*m
zqH#`CtH#BH8nx;*@2(2|XD^r_)c^3yG{ctu;9dO}qO_DIGW_QdU}Tsn9PmSsy;A*l
zl$l~+N8YQoW`zo^q5-8(1cg^Fh<U)K^Hsk#u~<whShA|bafN}Cgqq)VsWaPG>D^j?
zgNea+=9JXw!mk_FE=i~lomO^wMyu_q7G<j`o;PZEPcJ@Q5h7r{)?355#%M{&ahXHG
zZGJ2=sh&38QJWG<Ws5C0DK<{i;x>^D6z^!S$Shr&GgYqXMOb6m^rTf)|4#bMFgLOi
zG*fnFbL^H|Q&3g$Oa5&^XtuLnDdUPQQ&o4ES{?mv5I3o2V`lf_ms0zVbRWLjx@oat
z<TWAL&9j6j){5`0OV}OzBs@0XLh$9Z*|Qr`AC+&M6}A0h=62rS@glP~{x`ej>~KLx
ze)sC$YoYsuU0r{r-TNuStH}E3X3<>}+w;OTg&hsQp03ybEtC_qR?s3M?X!Puv;QWe
zU92a4eot98QDa8`jt=If3c{XThL1%3E-q}fYH@wyG_`BzuGx*B%rtgt>Lj_BT{Y#p
zvtu{^>)@?{xf>qNu9mLLiZ<kIU=f@UYVdiZOP5^6H<|1-7n|lK7Z~TMdWS`bNT->$
zYrKvRKBXBEWfHk7%uCrc_Lf|B$n-?3W%?JF3A1G~eDY@%+QP@#`Tt|3{cD9{t}5+7
ze!)ohTm`EY4^_LhtnPoydF1AOfn(!Viz#Q0TYoHfmJgo1OHydp66qyB>((6<QLUN(
zKTgeO<|^mg(|6mpupGbXzeXUoID3j<j`S6d(mz~6mrnGoec1i>TCeMi*iDYz3Pnf!
zt{6qnEdS-UNWM{H|3xFeEgE-rYE*BpzkfUb*el<Oljc3XGVf(_f2-QDzY)hiO$iYR
zFS5V5*8Q7*tHgrN6$)mJ!gIba>sz{vgKPI{#!{Cn$B+F!erC%CY2ySY0oG^(uB{5G
z&&%>m*3{LX-mv@U36HN^l`q!TpPubkD!q<#j>KaZ{m(J4HDoHEZwzB~4Gk5%>RGPR
z)pjMZXaC2!jnj`lI9VTGxzF#_rX4v2Uo|U&7KQDbBl!Au*dd<%(=Uect~9S>n$c0Q
z?77s84F~6YdFcGNaNJ@USom7e!!z3b?FFuDrgj;#o$uSs^6NaQr*lGa;YoYfZsQ|+
zPH70u_F3gLbNiOXYC#FJqZo?p3eTsV@JahUCnn=u_UzIFB0Xn*9=h0l#6U>)PNwfq
zH4R=~->aNLg0>e|#4MRrsu7YS+s!oX&5ym=nzkvT@yr?Vp_7c?daCBV6nrZp^vQF6
zp^(yc*96WtYvnGQS}E>w<=Dk`a)N-+u5z2mkB7OGxK9LEZi_O|IO~1IcaqdK(b(vn
zQzs<|-(Pcu!)^QbVCl`u=jP?~d~-KmY}4lCb;{}05w|I`7nQqBd@gsly1X)7@ZRl<
zmk#D{iz#?w-Cp)W^Zcs<sYcuX2lp(MZruFowp^-<wARMX!lcYJXLn(Z%-v3Azq)rd
zdU{w2R;{cS4!+Fg9<7t3;K3nd|9P=*)JLb1-6=L}&RHy(Rj^4d^t)5pr~YIGkH<F-
zi6pBZQ#n@`y()K;we{2E<=Te=RX1IEsy?ZqXV=@q&ug~tQLf$nwnu;0wjC+z%)Lj>
zO%>ApoOXP1{xue15m}+bm+z*$y?bS)i;I_yCZkQ|^=p?dUfs0j=)bl#91?!&f49AG
zJ{BS+q#Y}GnOo_5NF?_q!PHxVr(#-iWRBZ%2e8-)8tXD_Q@9e6dzi<2?fI6Q!ERi^
zcOQ8B3h_3x_=>Q4OgQKgyY1tel={;mvj2?!>nFP|$_dR-yXIoGL`CXi@l2P8wWnMU
zZWOV-*LcE2cH-WXj54a1_NHd|N8Tyly2f|=9N9Bk_n)x{?XRiuyO)0BqRh<dj22(E
zUXug=Y9AOj?w|R6^OWsojjqCCpFCtYo_%w+n_;fH0MB{xbAmq=J)VmPoUz_^=Jc~u
zU-z7yawv4=xlO$Z6QwqK9zA#2J?HZ53+;z495SBsGfX)7$0L2l$17xBK74z30^dEE
zz`0jcUy4n9;kirc#Kavh*0SE)sgw}9M!|O8`>E5OTVA>@e4pdmrB{b_q7H1nb8GW9
zBW>omUDce&zyDq)T%lm^8>x`-&{v6-`FF>f2`mi0Z&GI8K2xXuKbrTHNpYIt=113G
zZ@l{NY2(~$nYNXuQ^U_R-V5qUf4uvi*Q++=;_ExtG<eP1l(}<XUhg~J%rh-)N=c{g
zzgT>Rab8E2=<+{-9!swkW=9E5@%`{s)5GDQaKUA+lzGc1n{Kas^l<&dkX4n3gCBaY
zmc2E7)%M5JS=P(Wkef7TV{&RO-${j@RIMKCt0#K)rX0K|o@Tpd`_=Yybv0_oYSlg-
z&siP(=$jntmAzAsW%~Bm&AC4Nh3UH&?)sJb^Ip39U!3Omaw6Z?hqf>8+X_xwe)qNG
zoc)Hnmy={pG~P=+(YEZL;JY7U^PK%Y<X(E4_i&ZvUe}2a7-fWX4!-{D*~sy8e)rrP
z|Cc-v=KJ>k?4R#qh7%9}dt=2_Ua_5%b(Zj*n$i}3XM@1!A7!TQm@9qxw#(IASDn7J
zS@B;Uul;%Ov-;fgKWE$jT5vzZLeA#K>$f5Mem&0k^)B((TKh`r+#S(31s^>v-x2@h
zxZ||!^MX37-{nN!FY2%rm}p#^Ha}>diD~s2VMVLfy=OKaQvCDURA$nL)zuvv4?R7{
zbGH6!f|8?R=^aJx`!}N(E3SQ@e(Z_b#8b2Ejx1iHd*Z=VHHA+;wMV&5IewC<zbzK{
z@aO$!42lYk3{9-OaxOC_C@{40N~<|cQs{Ky4ARQFxG8nATc5Of??;YF4<r~>j`t{O
zGAv~<(vF@~WBKa8s`nJ5>|;4IFF*B~su?>?heJ5Szsbg3)THZ*dQg+@Ofg?g2j!W*
zEL%AyI20%!)Qw(}cT*v>i$OE`nAgln1x^RIupC}@*C2H*!%mjV+nzRDU(LW*ZK?9a
zX~N`&19O<Hbt)tbosP1y^NGxGIHVT6R!m+c)WG=Q*%|zscFEjQ$T-KLD{K=x;q1Z7
zOKywQtleNFd2ws*ZMO{)4onK1W$H6$J^5Df=xDF-`C5Sv2Tso=E*FEg<s8!uIT>=l
z<gQiOtu2eC-kb}nsd)W}J1xC)pN;0%7oM`d_qQnI{^WQme>`5G+DkKPQi|(|)q6Rt
zx^<YjMG~4fElK*I-^{jWjqa}h0>Tpv*raD)(Ay?a6JXEH|2V1pfYXr+h7VXa996vF
zA>HTN6qBR6^&(5bMg_^-kc_Rr^^{fJ%&cB-oGaqH+DoyG#c9%l<C9V@Ex0pDnBi1Q
zUrdI%7VC~p3)E%1H<>F4yjXkk>5h`3Ge)vaKBrrk^lUnNf$bEpkum?NOP;1TRbHHY
zS~Yvc`7W(nPLumgm{wkx(8RRr&_q5XPL0W7RxefNs%`BI^42~YqCRzw6_?%AH^C=@
z1LyzRq7me7DjGOp#jaBa{;zt~<;b|yIJ=AKxKoa9ENeq{*M=8d+UxhEsGd0Zr!SUg
z6N^BH1D8?M%GRkF+^aqwZCT~o<C<d~&2ziuf1V4+u}cBU9w(Zz9CO`MuAe>cWplIj
zyP3xQPz~R%Ig3q|b&oyh+x=$KV?XUuR*g+7mEJGrw`~^h>tD94?-ZZia;1ABNq*{O
zTA@vjPj~ESlwQ<z==T|>4T_t&4t-78a>UC!kS~6luhUGy!y9z<qRxF-uy<WHms69|
zpPdW;Z^>9Zx9yfEZwzCSi^jG;bK-U>ibYrYX11R=(UWzfNoLa4Rfj|vI-4d`Oh2Xb
zw$wOR<8{vD?G`6pI&zvNoSGJz9QO0bQMteLx}x!OndpVO_v@<PZ4tjT$-h&{L#at)
zNyDBs6YNg)iM$TzideD2MEw2kbAn&{)Mk`P#a^22cm8D2%>N<c4y9{?+iSnxcsV)K
z?)K~A8#Z?jPOh-?7yc!&XOhx4t|HDw6CC0hSyDe}Z59c+8OF>#t2>;(dhTky1kppQ
ze<ytpdb8o60gwH{gECg}4PW?$`xbmt)mm+Iq+!ndC@JYj-!`{=l0WzI>Z-M2+IzN|
z7%}L@vAZNNiE1<)`>W1*(a<}5&FA&s16pmxO8gJH2=6HsJ=47Wk<apbizf;8EoxC0
zS-9L}heJhdx=z}}rsAll0vvZwg$b&Nh4X1BHs48gGQK<^yufS$qvkRvJLk*Ho{f59
zt32u-cFjs%a>67mB)5Sx@@)SMwk2URycs;K!_(HasB8W8_4HsmkS7qd{Qvg5ybLKK
zbDVEXn{eoCX59A5<SF-lEVX&|(%WZO^c1B_>c?gIizHrhywTZy;<}+x{mhf|R}?u3
z-?C**+<N-PQI)opU7>G+Hm{!Y+Ucf;>ps58W;a7tow&TRy|!!WvRf(Ykw3kb9eA#?
zMCxAcx=Esa6(_XMO>Syh(yrb9CM!JuA$Q!5pb3ebfm*d61BAPU)sN*iifm)x-LmGW
zy_%Nq>?d7OOzSHaUp>R2_{Vu-rpHCQ%x#zF{`6#cEb(3DgsXt{5(QRmpBer&&x+@-
zd7+;9VW+>)q4ww%o6d4=Es|?%Tb97lnxPxJQpl`i)glp7MJK_-jw!VThVlpbPW=fs
zl6v+3dDvE=ZobYPdJ1MoIxG)IUa&eMq;f4yDM-TEdd3Tv#ga#|;(cx<)k*nf3yQkC
zh4Oyzy4+P1)s}7hRa0};<n2uo2L1~rgB9La2<%nN^G-ZG`OXfmWo<4k3Q4zvo!0zv
zXViJ_5zxfJEZ)B7AIB@@P0>&KH?Re)c_})VYvKJJQOhNlr99JU%Z*j5zrL|&s#b+y
z=+ngH=@BQxu1%S8ROfY3xYoIM@5;7D>J(N?l8cFCsoa03Tg*g3RQJOrrHo45Q>)Y$
zPTjRsFZzO9M9GRX8EVIzte!M&tT{cSS!9_@$IVIilmupWec_n>U0Z9@zbtz(-*VS)
z$!E83d2KIUx!UOe+JxuUlT{nnb(LG}EIe1A!Fp-hgUzD0P4BCpO5c|g{JM)pQ$3yG
zmQ_RCcXx)p8k(m(PxmMtTg)Dq;ViT#p>x@W1^jviZrWXk*!8a{a<1cTn;1H`>CGiG
zE#>aD3#L1N&RWrK@b;}&cGjcQ0Y@Ug-q2+7JZ#cFhb?PaVlTtTZdaYOwY*C*n6+6X
z7K<#*UjBaSfufeh$Bg4bEi-?uay%pDZgtn!DtK2`%wxCE);F1Pk)qOyFK-EYa^5)O
zyiQ5`hC+&|ey>U77T!6ZXWxH#fq!me?o&sBZ#R#=v9$U0vM)ul^ZF5|NmF`c_+3s0
zPq8iDR@PVhWihW%=Fw$EfnR_A7e6)2IcTn6cgpsl6YKesd|tOc4G;d|9Y5=&;=^eY
zhK)9HB}T?ewByoNY+D(amem`(a^~dvfUlYrvAxUgx+HQ37N~EQ;W)S{Bwd7I?)wSV
zDo-*$OP#-U^nCTfom*<$c;_>TzL;bB>P1V-w#_?O79LBhyjUupdT4RN_uV&p^I8mU
zSV(R2J+!WP>I<K#PHJ1bOY(Lo$co9Ny<X(x6n5wK%vn7T`2zDlomdcDyiL*c2GjMm
zwNiK6?)->axp1>{&7Q4#x0r6M$YgC;^J@D6%O$JBboKKOtUC5|($_U#tF}&h9wn%G
zW9vjirm71Q#TGBD|5!KeWZHzxEoo9mmz}k~_hZ}th1NHg=%t-oI8$%Ji^En&W-Wgm
zZSV13=cMHFg1v8|?RPI^oqJPOGb(wqU-^yGchh&Q{<N;TF=o;>C5E1gTS_IJZzETQ
z-=3~0JoEPCt`){CzWW+Zd_4Tr>1PwS#A7Zu#kFZ3&jgM9Us_DJ*uAXk#m3OsznR{O
z=DQxHEl^wAv?RmB(5Z~+aQ6(WpK22pIqE-VP~AB#!*Ji)6+XHR_B9X1<J~gc_DzUi
zQkc2ySmrIg-Uim)_RX_)zk9yTpy~NMEiQM7*7t1&t*`H1c%)+!7gG5!LgjP5Yh#t8
zd(F(By@#ETY}w+hmu>NPa_iLVm(-52e^HZ}xG39lRY6tngu}8QKKwU~l{R{?x?TU)
z%2Nyi8(*%{{48zIX{5s`t^2umQ`98Yjh5=h91lAzPR`_F6gFD)dQp^v)Z8pbF_T@3
zt@bUDa#*>0{_8^rSDaqAEYL14yJPig<MT7@4hdPdW|&M+w4SzM&I|1=YB5Wks%7n7
z&$XB4U%JU;`|d+Kl1=XJ<oDF<=$g<G(P^I~+1+I^!B~gm=;C=n$%nTJi0#fWoh7-B
zpSkC8fSB{7KEoYqOHWG4tehB?Y?o4~#mKQUXVJuq${p(}_l7B`RcxGed&lfM9Gjo5
zP@mH!k?~qE<KmH14~{(K-uHMzPX?#!Q{kf*Gk9MJ_hv^=|GrCKevV#gx3%b{LvNM;
zn=@%^R8HOyqhcOWvX#Yn{{%tq|2y{j&7Rt@daAbdj#-oTw_VtOzi7t=4W-uV9V(Z$
zY&yZyHQTDi!t%6*<z?Yj`8p1_J9;PXJ~+wRl2Ksa-V4&zk(Mn7CYPyj7{3%|E1Khc
zs^io_sqWW{BoZC#Y7WjzT2$O~uvx$(TH9f@<E|Ayom)RxIOT9iUYz#-q2Y?%yrDXW
z6!*+k_0ichd5y}Q6a6(#n^&)iSYrOr)1kO|4a=W3wzIp!I{ALwwDye9-M3oc`l^n^
zRdUS%&f7Q0-8nVyO{C4OUo$gwEci4g9$R6<Z^$FCaIe|LJ<Sz8EHe~2K5;#GyhHYj
z<3$GB!Zo~CjsGvtU)kywqM&lYwr1n<X${*%-W=I_=SaR4hpUhJb?@aaIeT}Na40(;
zFFxt594#0aGf|QIc(IaG#YQQQs)=VAdMDiG5$2jYX^&lV#R`E{OW!T_srYQiF1`QE
zm3`H>4H8ti*Gn8@FzlGjWGWQxHf@qyv$D;f=H)k(JGveCCnryL5IL~skC(Phhx|qN
zj~%?HD;$~&4(guJY7yw0zFJF_)0l0M`>sWOr!FdqmK+rQ<IZtrophq(9u*FWGpGMB
zY+GX4yI{5B0`4=nk9Zx)=v}GYd*qMfqYWO4CI0=1hbGAA{50`79CPxwjngg8Er0Jg
zxnDkid*YJcF(O}XcdY)=zc=8&;lAVv_MCezD44$SJe(3`+N?4Eg2lSWLcX7!r4I`>
za5ywXNc)~RILk%Fc8!Z`>y1;3mHXxwP5h#AzIjFKWfecKB~z<^**;G8e)#*yZXb>v
zOT1e?T-+|xH&^M>qFG`kBGcE@xIX64`XlMn<I;7;;lObhp4W?AE1H*n?=X+u;-kCO
zwPnL4mP6*OlSO}Zh;}x(O|k55eB3of@^D4wv0D*m>o@oQcj!2AvB5WjS3>86`l5|$
zZ!K;F%(^JNK(jQ6v4VH`YbC}C2MOjAVGR>1zf8WaH05gK=GvBN3tu09amO!b50CO^
zr3Mip31`8euYn1Zy<30Sx4c+ZcP6C%Pw*n9{~_%a`_4ND{x%G0x4H88fbm*ko!&22
zTRwaAs)TKk_6*&%@4~Mc#}~P(9GUAqt1FOUg0F)9N|y^WMH;TIS~$CAZJNkdMnjvo
zg4cIWxM+DK;L#+_Lk#oV7EcyymKU;CnPbJ}E3!{a(IMOMh~&p(xy*-}e{9T-yn5|6
zOI4N`_pXcEQoQ}<oPPL1Dn-y^Mb(X_7nfT!g7)-mY%MXCE;%3ObFquZ_Q;h`gV-}i
zBZ785wqxIY=7V$O$J=LmFP<?B?I_yY`=&y1v%}@n8T*Z01j=kJSU)X1IwSa^O5beN
z3Hmj<oHv+kRXlEUILNjva^U5?b-{O8<+R_1Qq>+;i!vQvKKXy^#g~vP3?6&W_;rci
zej9Vc<&Ur1Qt#*$?AuCj2c<@Dbv=H64eKu5+yAA76b+@sTO2xg)!+Z|lX6}q&U(Ch
zub15EtKa|Jh)s2BesT3zr~jc<=YJm2(K~Wh)^di+<MSa|Iy!&mrfyX|Re17Q0}n@|
zJ%4h5vB+r-SB}HUhZ7u4Exsr&^a<y(<(s*6GGFTCidnWRJ8b{2I3lRZ`}3h0)0-Q~
znvoNDdt0`ik~fWR=D76YZQ#X>h>DN5A8TCNXnRT7G)Cek&n=$bg{Q2}2^bvf5uYN`
z8-6QDbL$HI+zta*h3ewSxr)uR=5nk&5@~9BY)a;l1#|7K&dwALJ}aX9VBy>UcbhG4
z{aYa;W9+|R(ZMzo4vuraYm_^fw+PPlitf-g@{^5ryLwLPZo;<Sje&nJZgcH<W!tgk
z@7W!5Z;MR3EgqNH>=FH7N<?RFXkcq-i_H0!2?+&B7cW=un&8@Tq?F@F>79dLLMMoX
zsmfgwbX}wVE@o0mm_hEfTMV&HnVd^hXRPsHNV>!nSiO+%FxRFR*O=@$0$0yFyjkwZ
z@0*DeH5aVBYZ=2Muu}J!G{^3Pv(=ito)z%1Wggk+q3>m4rn{g^K2l2g_p&83#rpOh
zTw`<ZY(#Ij^Og5U!;j3kvGnS(mK`a7I&S!UOYt%DV6#ivyXD6ElqUyS*M#ag{R=#Q
zT4L&b&Hs^S9i|EuEj`7upX2cw!>REnd^%#!^}p%Zt2={(A>wYjO7h&$m6<Zz9(M%K
zR*Gts;C@oL!0A>0lG;N>k|*|_efH*7%JRKEHA`+?*%7s4%Gu4X9g<<U-CU#HQXeWB
zJ`|sKTB7XUQHR7b&WP@%J6_+3miXf*oj7a3?T3#TLY8knEA#Qj(OGv+wq82^_v+t;
zcUsCGxmJhW4ixWP=k##plJ2@x2WQpuQl0e=ckbVPbMFkMAD_F{7|dhq4G0eIHdWc$
z?67slya3B}hb_uhF0c%@t>bvs!E-G0a$Cn<-(m$R+ta=U`fM(}nUW3<PA_|#`)c{W
z=#a7usm`aVVI6*EyQ<~>pWrjQ{iZf9XZ6{(4_W)poRh1$mE*Hu{;!pP78)y;Ef<N3
z*SULiXBfu}Q-g~evKwC}Cms{>>yG@Z`gpSE$`q62)!*Xf7d6fC6SLoWxw*hMMYvBR
zL8fzt_T9x-PDYBlzgq3eWDsloMC+32trtDYB@@d$LR?~Jt!53{?UF7j#w%LJ@$zrr
z8dZ*4^WG|M%&l&S+ij8_=F0PS$+-u0S08RW7}=`yq4&Z5MZQYUCWe@tRB6jQ%dopA
zCMltGO^R>)u}7QVrd>O~u<Pk7^TvebJsY^Ef7~#|?69Bl4wJqMbGCHHE7}Mr+i=y|
z9J26gT@jG+h^Z}MxvFJ?o@WEY_RBA3{FiI}kiBNcGqy>~7xFz}>sRzD={x?ipVPei
zMZ&fh9WtET3!)~@`E_CIikE3Q+p<zFzRI!okY0IXuSx2QONJ}c6rcD#E!=piuzGjN
zYM-e`KfZaKTJh2K@q?MY8S;tWdYS)cGd0SnCR{c=5}T5lwlMU)Uv|;$h&jjp9SUbI
z__*wi-N{e7*FWh!|D^x@Q>Vldo$p=O6`Cc3-yUtqZO?dnb=~ua7v2iKe^q=kec#jD
z!u&Ch-%a~qs&8HW!ao0n1OHx$|HU2qVi@l_C44b{_ipae1uCCc%w54DyU?IWZ(DGj
z(lkk{ZWh1y*K#wPZivqN!+j&QJ1uH!+MWxgGnJbn^h^c+cPK}SHFh-Ix0si#H<LO4
zt5*G4wnfJc*G;c5TFt@nt%IW=>3Q#Q0cn<oj#SIC)cd^~9n>Y<QyWd*HH3!l-4hpZ
z&#iG?qAbsmjb=fY9>guawY4xl@a3e>%Om2Q<}BUmc0X><te-cSu0MMHb@oCf^~ah4
zOec17ryNn79$6IrIo>*ClAgF!3&+`Vt@SMhd@ol=DPCS_G5vO1>_he|x!wKIo$4KC
z<s6;o-!}dDaqHOIiR*JGm{&fIC|>t3VR5BHj!#N>>4VGMFE-oRE3VsBo|JKB#k-eb
zkCJt*-!T7r8|FFniH26)Ze591#*@~K0ZhD++;^BNOnvXV{Rz7k`#9C;f5#h3Q~r(n
ztwP@1H|jZJ5&q3lsO%JPo55t0nYO9wWp|x>V$#3bo_pM+aIjg!uX^Y2*^g&$%bJ|>
zygJptY2w)%(g$B9Z!Ej0@$nPO>n-;Zw;wt8{OrM*<?l)>U7y|i7->>9dF|7wPycil
zt%zf*J-g@6pB48n1?^YvERx;TWmLkaI-_B;LFStBt@CR?9Y{E<_N%DLL2jpfA(zhT
zlQ%?6#2I?Uc^|$1(PHrLp?l90|GMUkf0NtAmDU#9vF0s)nO^iV-8Me$bor%KW&ZE&
zAGK{e?`ZCMdhH9B>T73sx3oX>_x+gRc2br1)Are_%UE=8t(o!5&wqmd{r_`WI2f5&
ziv<23NId1x!m+*JhSGtBhZ}{|gVtnBRBGy$)Ky!-;gHbO&m^SaF+t&wYo~^)ir|G!
zsa_KdRr9`B7(PDU%Q<O+r0Rj?4q;4DoGVN>-gQ~bG*zwFvqbI6ihzkhOR7RvIV_4|
z&#Tp%sCl{3sk`s&DN*fPTher=iJov{(A=8Q==9aH>xzb1+1E$4zl#-OI~$+!-3hs9
z@bGYS!Y-DX4NrYKg0^r>nB>wWG%aXJ&csatT}L|jg(r(OTu_?cxRQ0<JdIsn&dimy
zKD2JGSSQQj*(Z8(Td&-6c_7kyd!28L(#OX^uaBnaUD(-Bl`#MMqNu;xm%^?_3+8^m
z^4oQNz|rs=?W;BaKYl4toh0b7$~NZ5`@5&*f82Xsd%%#LyTSj){t$<)=4}p(x2QKd
zEZ}vy?ZB0{iM55lZ`uL|4ljkHt>R&3jy~%nO<W!Ma}_RRiq1Xe+PP3LBrrQ>$uZAt
z;Zqt6Ld;iPCNoH;nnd^Yo-A41%c+nk<Rzr2G1<lGn?``AX_U&PN#;jco-}#7w9H^|
z+WM%wO03oN!gTLYk!Jx3UaoRMDN!!>qEfw{h|WnDHC-N9I!iNbn#jhJ*IWx1ZoTSL
z_;YGdLfx&U@lhq3uja*8{C)BwwwtR(b!mI4r<A0GTE<KU1{H=D&sni$GhJu*o)TK+
z<#SLneR1crhf57LQ=(kg^ZjSh4(qvjEMbkYWn`fDe!UYn)sk(tE)O{s$a1%UN#k{n
z>v_BGI}Vq_j^t#P#f67hUoZW=mLV+CRHQt0ns#{Pi%63dwR^*!r2Bk0q<wME$8E3W
zs)e^czrXL-x9js8{(sWmkYIl>UEvUWu#1B0heXW)5#dIwkIO}a=eQh|V4ry4n1uR5
zh72yZqYGUOy;zhw*=H^@a#dU2qTOvE&aOIzTQB00r)k*~mx&&~Zb%3yT=Y>6G38I$
z;ult=Hfvh(xhb)(Cw5tvhE$iOuA80b_xx$Z)-6}P@*jIH(R2Ok=VErX!%yX9yvWDz
z*AsH~*JduAd`;_W^AanTNUwQyPs*1oa{hm}{Xu{0otX{-cdfRbQMosFM^?9W-Nbd7
z$A7Kfz??eo(uDTntGWf()Y{hWPVC~Fx`lb7oNUA+t*dJt9%MbgyYb;k)4VIw9?v$b
z`^gn@|J}!n?(^?xGDh6K{fE)wl*E#X8^-H{I$2m4<~6;_ObeRuX4VtY)(q_@+!s#;
zoSex0ASvX<{Qr-BG6x<^neySa#}*&!aFLd?5}Etz-JHM4*PagvvSeftsB`S@y6DCv
z%(1mwW3{*R%qu0!IK5gdg|1AVqsFAcn|2|y<ded@79}VBMAlqG!^FmSuCttMeR5f6
zb1ZovcG&IgT7#fWnXB6-tYm9sGYUC$>H%B%!gB(JZo8QO-**YRqIlP^P-*t;6ATM?
z?C(@LP#k%1ZcNAL1J790qcQ^dwmGq{eJ+xoeP@;MjssiO*5+)m2s%93<;K+`RWo}H
zJ_SA8!}e`=<BN~+)pwHj|6(<7e(7B)nt5KV%vj)*jHiOZL%}~fM)oW|YpNtK3#{E{
zuGC(T;WFDqUgP+3qqYwIcH_dx-$9wejem4)Idv8|D1A8GD{@jL;8Nsrw<19eS%c}G
zLJk-F9vLs2dNL}4CtP@C+SWUXAAHj@m0izhEL*zd^vqnZXWaKw<t>GlRcQWscI(Qs
zv)fvQXO}JR$a7QP)cK*;q(wPKD6G{&c~+5{>aO(fQCIc~J>=^2`g7Cg|DuRF>eJj7
zIP3|HiFi0WYR<Vi$tjP-f|+Dr2C)0A>yCYWie+AQ+S}e_XKt>CGt~2}uH@!M->c#>
zoB!iW-kcogIM=vS2Av9SCOlmZLUk6CUq()<n)ho)`ZSG+ZzEG&7F+h7+Hqi$z}oXQ
zuet-XCsY`98nhXjuIxOgbU;DF)xX|iRp9?e>B_%+pD4@jOm%Il+;Z%OepSE9)r=OW
zt|$uuEuk$BTs}lO3an!F6}41z|1P!LKGndlaM!bvEnhD#c+$DC{8W_Z=NY~d`!@86
zuUzQK&pO?i@o3<45r?~8vnF-evLt&%bUJSQbnX!E#?Y?rtM;*~i>tb}ZHryxyZY0U
z|AC#W)@@JulUq_FeJ8DRV$>;3y)6yK7aueg?l@VS;herm_eN#5=3}Frup^s&>>`b|
z>Ne_lCMHLoQcK&D%B&*KZp5}=&Gf`KygdrMKdpaqH8>c@K5_B*efp&#XNcvirBh0t
zT=r)@5UhB6Ub~eh1MB9rRnLOwIRD5z^2v0^p%=x0_Tet4XTB(z8Z>E%7n{=Qja5^R
zv{=55VhvIb%2o6CW%ipjA@jDRu-NYK-5n;jFSeTMhRxol*nj0zk<INLA#>c=(m&-0
z-x5n|JvpJcr>g12N`@`pw%%>#wNqDDuC=Q?<mtoFTkRUK^FbN+l8mD>)5V{eE^l2_
zdH343LrX<W{}*Q~Y<XOma8~Hj=95o#A6}K+^K4sO<DKp0lVi_&$68Fb`*m&!>#p}H
z*Jtj(vg%sZv^O`tC20$=t-fU9Hn%BoyFgZyB#WY#>#Od0vtC_YasD)OqNIv+ap<N~
zEtewRUi?#c_3C`bm1-WJmqt`>$#$P5;kn>qlgK%q>C1CIuB(Zi8`oOBd;g;u1`SKU
zC0cXcT%r1HV{d6;*sf#Y)d!DhuUYW8PDFO<cFV{h;}G+V<VR1Gt2&I_XZyB4-4vxI
ze{SnKz20Tw>8T;NCSS?1+u@_Jm{sR$Y02Kyb47~(3Z65CE1X<;typ*7$M&d{okyNp
zgq`U)_NqerT=qe|YoG2cz039B|M{R!?p^6kH@EQI>}T2BQTzJzo-glC&o*6Oxra63
z{V9j1!JoJ{xCq_QWC_VX;CVpBwZ}@pkK5&ya?;F$yiG1Lo2*RZ)_<5RJ?V>xey8JM
zx!vKFrl*tYBb0wm?(%bS64H|M<oXb};+SsH+ea=U4?_>n>RtTvhW^tTPsM%C=cUh^
zp``rt#FIHc_goeF*<IpxNyuV%Rap1l(As6EW0&Q6ZJJXW;8r?6;o7GA*XJ-?QWMIq
z^|zg^u2uB<fZw^BH~q>Z^tK%|3TDVv=sB&MbdTZY>ZSExlIvbv6`scR{)fx4pNjV0
zu3OC6lD9ov`Bmt^d*09^ub3xVse8I?YETY6B=$e-?5cx$6PNDm=hjeDXik&$j0{hy
zvv{>t@3>9<kC~!3><?|#a}D%0IKfcb(p129W-d4T(j(JNq(k=1n;RG%+9RDQBYef=
z^0rjzO(nNYET?(Y@?KpapnWE0($b?(rUqY_czM#z2^m`>P6$Z5eh8YLanCP@?a><H
zmanJQtl)TJ!E$isi5Caw<-R@gD208KL&EwmcaJJgm(Wd&J9^~kN>`SPo>E6|mR>#Q
z8qxZ0;<0(056`Cx*G_cY=yRlM;T7|r>`N9-SNM6-#&e=$Wc<+$lk{)S_juuPHb#`e
zK=yLs89#?u*Bt(KmPbvT4eCpKvRqtxJVJdoM5P2ANluujCpcmK|ATDnXUKjnoY8*r
z{!`0EE-!qPCo*c@I20x`Pv8)nkkGNKITI=-Bn3OL-c;}wU*vk`s_QPxTQL^%9@+YF
zeNy<oU^1^GM}|hD$lqK4V!Nhf&SzP1iu=;#sf>4Co)vpp+MjXel*pe2?*gJ`e0`WV
z;m~`@*8C|+Z&vmN?TBRc41RkxZtmH0J98e+{1cZs!~KKd#e^fR4G)fgV&wTeH?it!
z>){Ow8J%r%Yp*mjEN%^O@qWPHu$1>(=o$Z?3tbr=Daal9(L4W2V8o4|3c-;ozYcVK
z-!p6F&z?}<K$UH>dLQR^zKoyJAi81Y)bj$-Uj!c9jy$5=tD^bCGh%}0wxvhzeZ3d{
z-=~%3Q_7kLeQbHrvoqYagB0fLidi3Ovu5KzvofHOVXo_(gvmb_PYzr<sg-SYLz_@!
z8<T{u)2WA7E+jDdG&D8DO%GUnU~4~r*pc5WR4pz^#7tz8mgx@FNECm0uF&!NF;m53
zp?8X6yiWu^e0^d@qEKpCWypliDcV&JTs>5GE}imJhwIRb_Qp4h&$3Pm=3-D^b9~Pd
zHWwR?wv1EUZY{?dCNn%Z@Ao(8?}k{zDp7$q52rNpT=Z;JXp48NiZhPl%Vi0;6X?RA
zIoIPyn%5T|ozB!LPoh1JT$*f=#CXWX|3GKpnsY&?j%r3N-=Zm;bT7icYyr#L8No}A
zd=NceacPn6|7S`Ie<=GtJS*hLVIU;9y23*<^w@ERb6G|g4ddKu-l>_~I%d)*moblD
zvVub>u+j9Gs_;jiq=rXxT91}0W!$Qec^$z1wsiTd(snn^2TV$7(<k`aOk=B>G(E-Z
z+DF~{Ua4xeHcywdJQMR_tSLLzn&Z}!)%BCNg=x;RJF+5Izg&^aJ7aZJ%th^-h>uEV
z$P>j)ZjTMFW)-NUZ<Eq!J(0EG3BRLK@|?MUr%uhj<>P&$zl-toZP!2%UQebOH~kyV
z23ZJ9&roM86MrAu8t<DB5|hA^<m#9Af~`tAsLuDshLs;w-E>wRy;ka<arOvTkc&&t
z!Rlkn7u|bs+Qg$Zrp@zzPEYNjdxlbesUm_xPt&5L=GbkWZ=&aSz%fHO^y2%1#e!k!
zafgn*%n&^GcB$|#t$4lD%wFP&O080|Y2}&=u55^z8+RmnLFDv1f<BV7(#2i}|2opl
zv!r?x*YnK&be5~jPEMQZ+ah#lMUID-foMe5G`8F?lj@(|_I<L@;YeE7)5LFy{m#c)
z6!I8WtbDrV+>JYm6VA_QX@9!n!VHP<yV{d_B#KVO-~O4c-o%oZ!zi7TJaa=+;18#g
zkJBEwwzha_tYunuBq*k=ETQ}vU%Aq?87@cX{mrbr(KbIqY??~cq65+{IgMg_RxG-v
z7N&7zWlfK+Y_fLetG$AL3>pmvK_^_fTe23{$<?d5ff}ibKUr8s7?>Dz7#J8B8kigy
z894qkuyV*)Z16qUn8%?N<8fgsLpZ1Cq{N6-XI$DOjJagC7W5ozU6OTk^VPhB-BF8=
zY*>7>Zz@9+Q_LZe7n7bHpKT$&?)Q%11b;^RX1+f&H?O$3*sh)LiAI3Tg;2YNTvJvB
zys}vKc*(lxrzX!W7BKDO647p0*|3hMQ%+SoWckHei??ayEZv}XX|`$P#-!l#{4<NH
z#BcHb2{#DZ&BMzTqSIh_e?|v8TUq30-j!i%Z-pE=@iIBRErzX4Ey9gq`bAD&qcoij
zC7D;5ytd0lWG%>H+{(IR=h}%qua7Z#YKq9!D8>h+<>ztv#T-yGD&f48>h3l9(sbL*
zRQVOL)~P3*7P9?M-P(0Ee8uy_Tn_Qya;lf7Sx$2G7WfzRJ;3r*Jipn>hDg`5Pvj#W
zecT@0=owggy@^L}hK%1YUKOuuX+xK5P9dot3iVuE0YT0DW*Uz=RHmH>(4M|j<A7Mq
z3DpCx`o}D8JH^gWTg+g3?$~UH#kx->^fUjBnBqJ$@k_MiCe!ew4%b+<23@8rLQ5Ie
zF8OlFtG+fpso8p^X~Oh`M$NPte{(aJxh2<4of;fjxMQA0Kq`lNd(NuuQ)j(<`(^69
zls?Z1^SL-#)BJL$R)(Z7T~vKKv+btK1n+*$BjVCgLTs6dGvcaNI!tl;CLY^T`0Ay@
z+`g=v4qnehWK-8}xTKXfZQ}orGZ{9YJEgI9ht`o>j@^o10-d%>e!1<QT97!=L0FPy
z@ru()QZF6!vPxb#T&UzwojM_M`&@^+ds}3+UuUpS+3{4cL~e(}%<c1bJ)K+qF64VC
z$ND`YP7?a{pY=?<r$6Faa&E^eo-g_PZYx<#XiF>0P@4SF&}5+l|Ml65+C0`#5{D}G
zUU^t#!RhW+te}4C$-?c^au&rZ2G3OLO4)Jajg$W43}wfZp0GDLhVLzu3#^URQe601
zCAZ|L@`N~Lq>A+3KBr`K(y*l@)>^<TDJIj<Kcc=ec4q6Oqt`;J*9mP4j-Gh-YH;A&
zmGiE}<i)<YUAt|2XvpiehbP`Bl)DpG_d53f)$ez67wJ~qOsL%Zb-t@;*N+B?%9-yM
z+htmYpX`Y9y_`_9HLIv|%~g@u!sgT`L1{}9!rIoRf3uK!weab*%M-U9op&+s%Cjx2
z8J2|j&6#vUR5Z7E@r1<d&vi4EE^FOW+|O6}z^Gnf*Bj9j_affbiavbSW2I_acPIP(
zy}CDrDf2rkpLcIObmQ{vg2PwthXz+Y3%eS>KQ&bM!dBIr!SCI3dj3@YV&GAD7vSQs
zb4QQj_LCExoP?!V^}I`cPIpXbX%SJfWy#pMQufT5DXW%0ePX0MbCN@rBbV+JZCx+P
z$y|@8Ej!ext5dOC_}->X%Os1Qw&gxLrSZUIx%45;cGD#fga3!@=n@s$=<VwBC~Vpc
z7pYqpI{p46&iQj}J4fV3+psrW_s?xQEWPh!yO~S(EtaEO5AW2?ZnRo_@6nMXW?~aX
zkACK0(40K|kwv%S#7iM@UxJSNu&R|WkqTX;G5PrWD^DWlt-jNl@k#5_)<sHlSBg3Z
zq^m_rwi-GdR`F}wdVp1JW#q&p=T#g|UB`QJ6y$6~kGGib3`=dQEMI(MSJF!P$W80x
zw%e%gE1S7JZk@YGA=5Mc+DN;&SH8-UT!O#OmHQ-%1v{@6NHb}!G!G2<JnQhEr+0fY
z?bz?SPv=VP(n)sm{<$_?WVJ(j<rPoEh)L~YPXpu1ZzwLk_?N%v?L&#!mf-(83%Neb
zSAF%IA^3`s#8w|=PLZ?jS=UZ3PMH+$$UMnGh)M8n%7(I~j<<RxM2%P2dMc`1aq`)A
zNYH%I<RzXqnO5BvTqS8om!=9zKS=wveN%_BlklxXKUM{<Y^NS&Th^@VBCWo;LZwbI
zimyadqk@-(ZCvrk@s;mvFW3DqOIDbfuJZkQIcUDtk>yc)R|S+uC@yo@>Le-TEp++W
zA?uwJsvHGpFT5reESZ-o>J_52qw9oF)P=+A>(;v3vS?3;6Ide8wX)sm%kc?;k5~-5
zM1158Rd>y{$u3@MHSw9mOv$EGt&=Psy7a5f6lS|+SS#@)(9LbJR38U>#R@0U6LXGD
zSpJ_g-SS{@j`t=5wq<KX=H_udGimg^^H6I0juXAlG&*+Qc^36~=egK*3~QY424*bU
zd986>+2Y?ZPmR9sKKFH9#rD;AKk}9Dyth@a{JMAC8?)^?Z)^4ZpZ~u5v*@y~d*y}Z
zqF=W&7k%EpKR3N{n)>;v7Z%HXdBJYWvf%+M{}Fkn27?w>rH35)EwWNO)twJ;Uf`Sj
zVk?V^tK;?yYvk^#E##OQsjhfJpnb`<a1j}f*t}!`?tdqEmnQH`N)ex(JTXyedg_Ga
zUZbuRod#ZqcIJ6cKl4O2fBR~`H2w19AMVWi7Qz#DY|}K?jZgK8BN^h>-I})k%u}Pu
zGnYqfE1GuQ=9$s<oBvOz^lhG{z9r2hdgeZxx0>_b`#iU~9C^NGTEP6@Heot>5o@2%
zSy)x``_fc1sr41HNuR@#9vryjykT!Zj-NpFl(+&Z<!+^msp`U7i~*~zOxw&E@8mu?
zi0jgom3((s30~H;GwbrIUUE7u{%Dt3g4>bP6PL6LD=W<vOflcSIMR7}t4>?PB+X?e
zr(SA&+4x(v%d6z&+H&(x25;Cz_r20RU3zrw{KDvPk?U7kcAcO5uJm)r-S=XyY8Q0#
ztIkW``?jon&*QauHQTH2{5rRN|4W+>wcK;={ZBIwcxW5jWIgBpyLszME3H13c;B&l
zw5+1+s<zJ;x19R>w(ANna{5%ZX~%H>FWb;0dMLAg%b&{@jXyiQ7Txf5VGEI-cxOX-
z+@h(SHyTb9-DEW9m^Jq<kMhEqPR{FnC#1wzo9<hcv7RNUsaxZ%l9=JP&NiNfW>=b3
zlzW8QPA92qb^kmaBGxhEAWxd_n!9y5;lCZ;JTGg!QoE}}p5xe7huD4o*Wb@#*z@4G
z-`$r|`#$9H+dTZg@5h1kNAJs??|veEzT8mb-lwVi_dGK`|MN(B+?mef)sME$N#7y7
zBr7E@Z`SEOF<RD^oy={@3Hr7X?DEqt&!4u_p`>2qUUtj(?w}lBt5f!EF4t-&MVi`G
z_PN)ot=hc2YnnsL+|{i;VGFqg-#S<v5O~-B?3dc&*#DWjNdh|ro=<B~2~2MYe;dHv
zBD+=~bC+&^-Y@li|GsbA|L^KtPQw%@;k@tlf4^5Szo`GG&cM>qz%_%FTZ3Vu0t3&F
z3Z5Sgf-4$CRy6dNHwxWoke<OA70@Ur!72ZuL4F0R(hOF$h$gv?CMAics0U4Q63zNQ
zn*NzL8%8vnTQq!nR&Q;Q-=7}Ow6NLfMDyQmdDpMT{{QNyeq4xSTal)_fT&yI?smaF
z)2o>uR?8bE8XgyTXRcb%Xni_e=8SpUs!c|743fkd>sS^#xLsrwxS|}wpqphOJ%4fH
z4JK(O!DKc^@udyP9-EY1otc;%n3lQLa33mN-YR(TGT*iZnacl<a78JwwxyT1hj(_A
zS2U-0&hV(1q~2_FqseRr>jH`9MHbDv5nTo|n&x$MMLKjX*Ju*p=$d$;YmrCy<{6wk
zKbp35bZ?L7-r>=;|3&xOj)ub?4N9s(5;r=HS5%zNsEj_AQstUCtu3?9LZGcwd`?+!
z%8Kgi8P&{%x>t87REo5%P0HTZ=072|rdmXDVqo^9gISS9eKjIlONF%7JkpwZG4fns
z?SfFF<%v-@wwp*UEOifL%XOH*6{xfRpz($<kqc$ETn`<6m#bXeCa^YudzJ#L($+?C
z4FR4N0w**W6gzwPPY5Vm3aD!eoRDCZmt;MDVv^F!Nrp4K^&%&mR8BVgKXY>Afk~Dt
zS#4fUw&t9o$I0q=a<X;h6#L96_MTImUrrRTnBpoq)%4}$vlaPgr?z}5Nr>86#PhI-
zyQwv$L!_fs<DrNCvFXKAM5iUb6iTy5x?m*Wm0rvAuubraRN=`0qeDprE8FzTMEu%y
z*A<qYy%y2qG?7Jd;_|ODJsd%AA1F1Zm+w5$tu(b;HM2=hk~2_Y(u|ceXKHrM-8pmK
z$(c-zGXp2gocXe0xhBK%oimvNXRWXlSogDG`O8_$e$HB(IcuXN<BpxPS4px42(ZrJ
zWL;}HYl9}ke#tpYU(P--bIy{;?!_}I_b$yn9h|vEJnNPQ!(WC(gYeei9Rk-gd)to+
z%=-Uru9V}v8<Gxt!rPs`X>6XBHCb8mLua;6xv3+YME_L73B{6gTa~n%gtJ!Eu39R%
zc<FShhgON1Iv;1&acM=L36zl$o0uRW^EN$KE{J<#hwyQYNm8d8WL7QI@0{$hbMmT}
zlUHdj+B$Ql=Ym<sb}l+RbC#6i98E8QLz)7{rxv+4vII6Pwwkrrrfc#3pNsc(PSKpf
zI;}l!v4i2~5bie@T9S4OL`lq>eQm1y$0cHJOYhFKGdi9r5hlFwScFnhjoRm^EK3=d
zk~S4LfhtK|hM$7PH|8@b)~+tixv?mlOE9^zqP^5hwabyM(L$glLrTR!aLrcLLmwAN
zI<lF#3wXT}ocrHWK>nn_YEOZcm5a2zR!+G!`=saWEs-l%Niy!2WEGsivUKHariHTw
z6&U7atvXq`YVohd%dG@F66Odrun0D=Y_3|pZPx02oUHRt%~@+v8L=$)=|v$nwTPCb
znPIc$avz+_{jejfLM_2^S;!3GSczzN!@fl+ma{|WODI~dc~DdBB5;b`vW+Es;pWJe
zkM`|HJl0*x-ruJEcZS{d?W}j)rRqF$%mby`JdDK~+j%SJAG&D&RY<o(GFt9olzI^N
z><P0JXLbv$Sg6%G$yjTV;mSpaH3c?SZS;7!NdDkPu8oVWxL6f#FIMSZ9PGKoJ!8px
z7k}?xE&Sg$NiA$O;@)gwxH<5D)zV8^4*yiFmKdbCcTL;Q=5X~?jY64ZnwPPy&~zq&
zcBX?{Bp+^(P!wm`HC^YEpi_`k*$aVy>jIn)+v_ztm>9R2DX&PGAbqDxWc^kh)^5Qr
z&N7yZd_Aic^{?7EIZI?+=El98vuFNXxrS@i-jk~W3RcaFTD5W3YL0{*9o?%pSMAvM
za^>lhljL5MEpkZBUm{)~I+tl-#Lne&Pj$@=t7<i$*eem(d+}6eswTs7Yo&``0*_A?
zU+;`~W5AXyZrYXFZc!Nd_N8V*m*$JAY4sl!nNlUvwUz!n$^N4`@yhf5i%MHv*c~1{
zpKtOk!b@mR#|%NC%IK{dG_E`OFIMQ3ytP64Ki5X?z}ZsW!gAgMt=bIAyZ5QCp5x;=
zRpWHwOBTH?2e`ae?R@!pr`hQuv2U4CJ9fDS?utAiz#!9Vqh9Qsy;ks*hP}7qn;DyB
zpB3jxF5p_YrKnTTi+j1}^JSW9fg6)|3$zNZQ<hHXE;$^wSKmpz+0^u{fuM}__Gj4}
zCtJ-fx8Bj{z1m{Vs(^yk=5O}Rd^P8Q)lPNQ>Z4W>U7`H*%o5Mf66_G|V>q)*^5K!^
zH>(4zT05Q=`R`dHpfgWtQRcpYn%`aX_gPEs>lS|M;XX5I-Ln<DKS}iW3$44+q_s9_
z&-T#T#FLC)v;>o?v}2`@{pqsqNSYzZ=)sh|V$LPUt5p;4{(rsVVe}%=-zQ#W&yqa7
zc-70rHos<1d$Tz5!^#lLsk02&+$9oD9NzRpdhIXo%>f!qquwlW{kZe;ttI|Z{Of!U
zEZBY2`niygBOA}>*0hx#yR%Qr8VBeo*RpRF5UUX>=n@GQkop~!+?}E#p{OH#a5+<e
z$=6jmxmHRVHYcurD$Rd=NYzt7?bnW(R;yN@+Ws`ATTgQ5RP`gDz1M6`UlX$8$QK=k
zXEP&ORh6F2JbI?<)RvmwgqKD87DW8J&=Ihxm*H*Zjm(H0)6X+c++}gPTHx%FEpvpw
z$9R7GDKS}e{=VOq&5XzIOKnjps;MrN+#6<fXyO@(!n1CM?KgR*OD7x@7hJFQKWReU
z8RhLCb@m*-*!;%&T%jPxZqE72rYcMJDX*FmB)P~lvgsw;DP7eBorO!bPDlv0Ik4uA
z!ty%`e`1XOP1qF`!L3(#gvp>x!tiux&Vl{vOVe`I_H5t3uUyC`R=ibEY2oM72YR+C
zwzcI;O$azG;8lL6+E>8AU7#WMpw*=10mmg?RW6q-n;^E(<f_cg+AJyI#tCw2NrxUT
zWxB|>MuBtJ&(&|%R9;-^m+vw+$SGpxZT=nY5o`8VN8C7{xmK{x;<j7ju6rkUtyz2P
zY}CaS(euJJPAARX<#mhe>xZM)llb>N;NLf4?zad0C;r|MFuPD*tHw|}KmOPHCtjBd
z*A_>9>g)K=)L*N3Y5Hd^zob1&9oP<<37Gu#m!5Ox$oJ`dsS{?mTo%&OOzjc;Yn<Z$
z*im$+NSTLIbb^h$lz-sfB{yF7ns3~nHS_A;_BDcRcVh4I3+SExGh29_OrN%nX{ztT
z{}1>s2J(Mf5OMZz>9^{MZx`IWJl9VCkOHrb(NyL`pI=^-;k{QFYb0cI!(3OOb+3Tm
z?@Rwhl9w7f3Qja>-YJmqJ6YUyh3me4`6FKUa)jPa*|lr$nvT>6z3G`keVax0J{9Ag
z_kYQ)n>YC*@{Y`WmU;K$&gY#^x5n`ARDU-A*t5L8xtI1n;+mLwPPS&91J}|Ed}|AY
z{uOZ7)Ct8Of4;WhLMYEs-v34yc`ogeH@vvFA@R!;L4&-zO@b*((>fM^yK5}h_U@<P
z5kapTX_EaRXXZWZ=RSBzYwm=H>eAEJI-YFPSUka~^rB4m?LAl3uH63-nBdEmGFR~b
zY5^g)hwX7!SI1oSp0jjc-c_--tJ)j=xS!X{1ai;#sUWPk`M}=`v3IZX-{tRMOq{&V
zKHw4my9WXRMi>4yNUr_B&-8)++ktl)kCsNPn9CECnEiJuW6Wt8+b!Q+B;B?ASQ1ZU
zYTgs5c`Tb6<#+pfZS5Jq*$49_Z^#QyKjV}0-6h#b%p}@6P}FgPYS=cjd4dmGZZb7I
zs;OHtYgzT}&bdi(ch0}%R^Iq&*6Q8_%cnE{-+rN_bTKC8;@1a)0s*tOYg8^axN4at
zmT>Nw2hZ~r)>ngyj(#=Rf6am`Z~<p`5`RC7s=@P)U%^8E4!o<@6VQ3jwe{j1U4Nl9
z4$t}KK04edEMBWUd6M9Qx)VtmhWpRlofhPNPwV=9tK;j6?oBkw(Xw4%*k#KeDZ;Tz
z>h>D#1I2Y3b7cPN3OMDqUuo2sAN}=a9Cv&Sx0cO?_&wK-_q|sRdcVJR=_LKrH#NET
zSaCgc@{jJ^wQnV-kle#J8@NiZ2Yp?T8TnzwUkC0zU4^l>Y#s+hHuXpzko$8)SKybj
z%D({Vw;%Y`6CX+CeUs%6TKnPX-Kb+9JRWa1dwDwj?{=pTPXE=FtnG{?7hYO<Z;w>s
z^7oeFU5-yC_;y^6IdtA%AUWW*LZh|P#Xq4;%U0z+{oC;KTLL>n;^{tJ#&vHvEG#Ze
z(~bXEb5&R%aM6*vy)0i2-#9Pt*1@FX_QxXd$&or)cP|skjY)1j8s^i!gs-1=rfxNh
zOX79!<ozB0^sZfQa`E2VyVWJNbBXKZX+D8Q8z)H|UCdDbT0ue3;9ZiguYjBH7S~nN
zJm$&QPEyi15VyLFMS7RefoW@XtFu<d*~~Q3KQ+0w^4rnV;Y$-3s~FpLdOqdvkBay2
zW0jFKm_2!l=5`?mHwWp?8NY-2JZ2WVoG<jBHcf}&mCJ<y&#g&UpOtDGMgM0hnABmt
z)rC)a=bL|fqmw>#$mH}tSrYl<Q)k{Oo#&tCY)(JU<l(tTudwvV8zpnsYc{`rEqkZ5
zFpR&(DD6!9wqx5iJw4_pZN2W@f%f><3)gqL{9V7EeHp{c#uLl;mwoP3S=Djrg?%T(
zmv%YE>P79-LbD>%r|3*sl|EHtLc-JiQ~q3J>C#Ah<vvAYL4>OlldB*@*Tf_#SB;es
zAKT|jWLj>Jz54GZx6+wgjF;vIMXhA_V7#+2&+Ty4B%OoomuCm6%wMuc=-}=pb2_}c
zf>uA*KA|R+EqrB*^h&EQznZd4CZ1Dyt71NLaqy<FxWk*xp4M6WHQ23W$l+pS`I>9N
z*ztPv|8tW~s@DZ}KAUoa_vD=#y+uK8%8PlPdc5%7=_Sfqy7TqA&FZ?H$D&#fcIq6;
zu&=Js3{_}(&~&;-h^650RP(r<&Eer<tCp^tpOM%;wKPmJRV468#HyPDp;jA|r!Bjx
zv4}x?Qs=y=Ev*^Lqi#s1uJmD%czUxgqSK|pW77$jm4;eVyo95)u1u86*ndd#IR9~l
zEF}r$g-iEG%{nE(W%w#lwM*y4&P(!52DdrZ=dI+*_Sqr#<*0l4i>9mciNZDpA&2&f
z%=DY~;na&onYB`zx6Jgpz2=F(`lPo@f;LQk@vP&NaOd++ckc9wE(~wxGFuoL{5B)(
zUCz>}W_>!XJ?~4jXa2W*)X~k6J+aStT~@&SV7;wIi@GH*O+K|-@5Q{}-+oF@j|nFT
zn>`S5IBx3x>HPH82gV*j=S~$Y<~C+wztdy>CcDado>He%qGWQ5Q@75M6DcBPNkRsX
z=6gMwBO4WQQ|y4jaStu7Wp0T(=2i7|#Ry-L{;zd>!6Jdw+xuV4y!%mS(T#KR{Elzb
zA86!k+GY^u<I*o$HsMvr(ZEF)mMrafwd>pl?U^Z0=kJ=4b2Rzw^{iWB9k*7pe%Za?
zi`08PSHI5X6FmLiAJG1ry!5RhclCPDm6iK^7j64J{lpm`;ca(ZmhjB^A;PHMB)uxo
zm%+;WzKh#Ylc0=cW<s7*7BQGk;!O(pZ&b^)@CM&j8Bf<EIYN!x>0W_sj9d-DiPE<h
zibx!s>-i*a_sj&_OOGT&e0!AsHeA(zV>qE%<Vk_5Nwc)q!Xri#7V&7a%+a1Fwq{;Q
zhREs&cI}W9gR(O$u00FIwN;+%i|TRa3AgDwG$~2=*a{7ybeCxoS=WzgxGZx0QG7;%
z<$x%Q%OWxTPL6N|0n0#z*L_+GT}AdR($`qCOxdzaaDR<Z<0;p3Dt(W|UH@I=3%YSw
zp6{cIMr~5}(!E~oZxg!3YlHg3HY}dHM&bUH6-!ny@6Dg~j@zdpq-|nc%<AcT?s)2(
zKjQASXwg`Z!?9+{&s9NFSBXa7IOh^|@z9waXJ-li5A-Wix^&=9i*KXxnq{T;u4!if
zI~%mX($mEC5Zj6)&rAb1`!Y8lKI?Q<xIvw>@0weL#_fVc<~=8+c>N_bv)tOaneX1r
zI<@%R*4XE54V?a!Qywq2O?qbY{<pVn&BxVYSB~>d5=s!sf5;U+C8?k}c#*b7mtFSN
zB-e|Z!&)LP>a}`4)pR*1E0hqvE-=H9AyUG%Z&L%ym9^~p{}PS{be2ySS=%ADTDYt4
z*J5{FkFP3kI8KFI3GHP&+9GPTSNg5WqE3F^e3itB9sMV+cSZeJ5psLu@u-}OM_=Al
z4cVr0y2zkwlSU_pQ{CCwd^dM(S^Ch^?BU84x(3YhQel^ydVA0OU#<{gWOLWka@)Z*
z^G&2D3Zw-DH0-*?QGa;(<dhA5O^=ugWlSdtX{{1fWr<G9JGxSQ*9Y?y2KAJ`%U1+$
zSrnqAIbVf+^Q7o4hk2Hr2wlfA!Nqt+;?CJWr@9=u9=7l-i{ZttlL7p0iiQr1D^5kT
zMzU_;I#$BUwfMz!hN4xh)upOJ(|$j43rkd5y-Fo9?&Wa}1%?e1o_<$j<o5AkV`KE~
zNfj}QYWM3>V&F(oO5p5bVUp4mj7mw`&$zi`O4LlRPZRys{z-QG|Cp?*)2X7em&LRG
z&!Ov6cJ@bUGb=TB{5+YWBs}xX67N3muiD`|4rXezY+7bwe5=6df|-AI(&R(`HHGqD
zJ==IU=95d&Y^l7W&BEX3tYOp@Raf0EoavmME<H<Z>&lng{C?euy5_?zaaW`ysJ8I%
z<fF@ESdB!|bCix|?fVo^v}uvG=#uD-ZiV~j-*7F66<~P4#eUnI!H;48jCtRbjFKXc
z>2n91aA`=|#I<n3cE>F@VnyafE!NREC%?!b@Ib~!5#giWJ^w21wd~$vvq<TU+N&+4
zBKtmx>s?LeaoQ+u%QuJBcg-QSp2v58ww!g<%)B^JNN}3RqP22a?#H~9SE~NmWGkVu
zcKPZC)(JmyHeUO5u&t^6=-T9~wy6hXjyGkPZJsGR!;UMe;Os{~<8)Q06{{MB7p~bl
zQ%Kl$(*I~*m*AO-&eL8Vc^5d5dv>+h)O=r2kFXS>^=~b7G>^PVe$M4kkf-s`$;R@u
z$a7a7kBEpsBSmHXP0D*bR%X@d+r-RvNtD%E$gb_?Xu0wVua>}!$KL0bo>;Ic+4a?%
z_PzxQF+H!u6S$LF)ymqZ=yWh0D1YhN`X<%8XUdGF*N;`jDF=&9-N+Ta=gZQb)R;4U
zH!EhA>~vj|bMf)@e-bBkQW6)fzc$;sR3W?0{>s`@7g+riYtpCwGB#Trvz+Tuo5&GG
z=2=_Ip5`tKQ_+rDutI9rt#fOPmEPsJUeWq5?U$fCPkp9&QA6>HnoqF|HP4-{KbUK0
zKL3d2g-X7=?{Z|Vng1VKxa7zES;C>4J9?592^vlf<lud)dNZl>!^We!X989l?K#L>
z{$Rz*1NX#*W@;SSA>1J5V%Sy2C|dgMmB6X%jZ5#hzF04NiK+X=!OR6*9bywdyz|LO
zRFddjQn={EHs;8!p)1xeQJ$7-$+>6Mw$F(UHPQ3;H#rDq^8e8=KfH_Y&LuU=%bR#*
zEco7Z-!fjoROIktfv`(AhcFAzC1%IWQ<KjWIr6{iKJ#k9g;^cvZt0w;(qWt=64fbk
za>tBw3@cf7O>GL^b5r`D)yuW_PtDJlY(5<*sU^8APjJySqehiihP$|@WgGV#T_HH<
zMwib7gOADwHcpsV7tt>2DcZHe{?D%s|9e)nb155iywF?eA!e8`XZZrII~PqeBPWCk
zv<ke`-?u?7Pk6Ue$OM)Y!)ZIjrUe)%B<=Pu>hSPUORL(QcFJ(lDr*()O+J#F_Hp;|
zDQvppU?QU8+ElFgM1eVX*K)o+%g&tWE6j2&3fg1Ks4jA+cXp#f>+YlMSwiZUX1+{l
zy*8ou!el9D&ZEjU$7+Q*^&Uv2K4Ea&x&4deifgxebu$!$PW0IbGn52vnir_?Mx+1#
zN=MDxZk$TY|5nZplQ7X<X`=a)J$#0waL6LF1XeF$z6fCkfsJB8(yi-?Cv*$Q9dF>2
zRb16EV@`*JeM<3?6_=SVcQzSwZS2qxtIj;2p*iz^ZnyjoPafTm+c^Gce-Bs_C85KW
z#M~QgzFko|)yH$wLz{EmJW6voX4o8>d58DT23Kd!6%!o}Ryr@Za$s^$$1#Z|CpmtJ
zFcwW-JVkio3&R*ehJ)6f5?&@xe{APaYJT;D&rX`NH(FCJsa<VPN8DjE#)hfkFIf0L
zFbQXz(yLH1T&#5R#rpZL-B(4(>1lPYF`o8*x5*bF&P}Ui3nuPbk}*AyW5&AM%4>Je
zkJ@GS^5lt)&Ms~;>$)?JobXxizGP!SkXnF{l5BLx&d-MO)w6b=w%)|(>VHJNbH~Xo
zCcgdMo}CJZge-h7PdVA4vFE7qj;p(ub3N+Y?0HmJgwx#kpkIXJe~v}!2Tr(lF4(j6
zCC~Diy+%L!d=uQNcOA@)>bvr}cY)5@IW2xQ(Fap|Ca?Un=jjLIc5mL>8PboGrx%Mb
zbOo((_;r}|38zlu@uxy_o-)k|d(bZ$J^#Cnv&x1}mPq!HWUj!7u6C7qRS{h~=5Ry_
z_$<35*7au>d*FeEPxfhk^x*u`|1xRy#4QfDW*p7ZI>o?rapH}0GpBf#O%9kP<M(Qh
zkpK_h^fx`%Ppl~vIkoQxZyoa#zDwH$pKO=u-J|P#s^f>Ff^LW6(@pG23$La4=1)@W
zeBgHCRiEiWum73MvQ|@Vv)I?m@WlR_)R)1r$>g-w9QTbZ)*S|%^E591zP$Cz3CsV1
z0bG_=OB@!pzfM|dnR>=)^W}esmfF;A?9DiGNJ7oyNxx^zGEb33PFve|>~;x1aYP|{
zp--ZslJTK^1-=hwgiLHWdBJIm|B{139DO&oxL$wF%jCs(GWb$;(Wax?p|^Tm@9)sP
zYSZ~xhBM~LQJrRmh}gc_lA<qnO!0m=|C+N%%ASL|61s_dx7Xilk`-NXV#nUQE2d02
z(>Ez)MbSyGODFoQKN^+z^iDd$_eF;D!o+J|D+BC%&%0!s$Tyoz47_0YaZ=p`f$lki
zamC)fJ=d?yn9uch^)~^Xqd_`cyM#-E_tf)VT>bOHRilH8bZ!W|^_}H%>5Jh-!JHfW
z986|U>G7!L``<aWhgqxj)S<Hpcg=-*j@kVTwVJr6(;>|1<wbYTQzw`LHHA8F_Uviu
zww?Il+?T?hBp=}q4Z><IU0Ew9P29oC_<~`pN$c{DYp*!;IIHYj@|C-%=1jka^JZtm
zoQJ_%7jI0?+P%6~wB7Pbnp23}YoFcCi!R8_zQ}Ux{tKo9*<OO7Zu~K4J62q|rgKzy
zD$kaq=O(i9w`^6vT4eMp#%T9Z&+wz?IxigEClPbSIOZkGCPic2>!-JSh&oPuz{1DT
zx9;Sc39_B_NyqBDIygIe9{S8r?e&e+;cdI*_T)_1>kVEqlQw;d@ZaVnWp$UkJfqb)
zSV?|%fYTc@mnqR@Tfz@ib^M<pv3<go3w3)Pj&`q=ak;^NH#UpwuCQ)ksBC9@>lzlX
z`P#cZ_f6$_y>_yiR<ndCXPtAH<XLlxo^z`#-7a&*&q_Rch2xT9tuDWa!VJ?r%15p}
zy}@FX(LHg;rB@Y`CRcFWNKk)0siXfzyfcrtxbuUl1<nawGv;P+NC{s)EfeB(C+chr
z$F9$3<f|9)`kic)_SIW(`|i}93v;goPU*QGBC>1mNj-;4O_!vkMEE4~LZ!~`k>FcV
zc}nupo;`0w8y@TUJ=ExD-n2(LE#^#8jMOXP3>Bj@UQ_Ce9!cL}c+Ju?({$mzoMu5=
zk(V7aw*+nqd@x^q>VikcclSk1{{OR&E9vfktyZq}%sl;p+^I?KPIv4oRb7^;atb9+
zxW_u7=i>EcnNr+5%a5PZJ-J$BE6c@$+E3g%qYuRPJrhklq2yH1l=#deG+^bE{jP}}
zb8lP;iM;paiQuLi{5~e<e<^&L%W-;ciZtJyt!FQua5yNU#gQe#cP2x`y^e3+>iYo`
z@7V~sfA}Cc^99fJX!kou{k1kGiwmo*jO|?6duY|#D92K3naxMI{q!~*-R)+3>7vu^
zi#E4A4HCE7c5eTAph=rqh%a4A@Am%EJHpRW!v3Y?i_L7Zj=141fAv>1Yt-{A3Vl9P
z{SKL)3pWjl7UgSvb~M8D(!4nx={kJ(GLF58IU2m}PF>LE|Do4nKK)(J=@9njZ_I&-
z_jKG2#jNAepVtv5msxZ%?6!g3wWkYHA`fhExn*@Q+4kBB?z7h)UfsdvWzOq!Z*E%j
zffG;M)6RC+o?7_sWk6xdcHXQKJCVdzjqF`dUW#(oEfaqGwmTrq)_2*f!>OBjS?)W2
z%yMLW9We2cz@NBIKaq@-rw=mjn=NpQXj6zdHr4xQ_^nO4H_u2NVT!oR^z!bEg^Cw<
z+CS<SyFJG>B;kAKjG25_GUh$-y5iG!D#7<o!XDwZ{P&g%<fTiUN_Q)Z6BbDicFCF(
z@^Es+*^B$$TuFH-yyeXqF}~ior<cFnxg@ey>fS?b<%|XMt}<(-PUO_R6qWV=o>+r=
zUq{!wZjG|%713c2Eh5h}OfjpxcAb^y(owflU9*j6u0B>V{Yu4oT}Jc!t~o{CGIg$x
z`L=G?42}!SS?DU3#m91Rv0ucEXMPWVWZo`wxN7$_^zV}&Y@)~4-e|hER@C%r<Kei;
zuU^jF*!xA6^R>+D?{a;b7JO{|9GV}pTpPI;e{uBWJhbnrHIvucmebiat8ebp=vPx^
zJ|i*zXjc1vwkvDqJ~%Vwnatv6*C#3Q@@-mC;&t8oErZgb1z#;>zd!Nn&D!4k{>Qi5
z{vt<x%Tg80iw^J>T}&(bQTATY&wIb~7UgZe0gjF)hx2>49li89J@!k{S-Df9P73Ty
zS6Sra|5sb_nFzm|*zlplpy2U5NdX&$b2Ip!FH<xV$UOPtY1{3YhLPuOm0ArCOgzAJ
zZ`*<c_G)o=yj&gaJXy8ge=56IEXJoJmim*8r?@YDsbAI}*X%P@=Va#YUA{~>OX`Nz
z%Z$}KZ}e67&8{qQ)%+mrx|vP<otggGlr@hgt|;EIAT=)LD|i0Q`2D>ti<9`p{V$u^
z2~Ah)QRX#lezU<gRr>$#(6;BUW)HLSKb6_}`fR`A=9a$Y>bGsSPMdgR^VdsV@{2xy
zEv0kAn}t`@YY*|7ES^=iK5OEJQ1(qnAFs*fyejFn{9zpz-?B8f>ytActP}41CiUJX
zFYT?3xuvk}ZoX3)%m2SfJNSCV%)Q$d?oeFtQG+))aE<PSrjK&3g4Um%@ThF2%%S4r
z(Z1`2zpeAVdnmtb+m~ZSr;H9<?9<KDQxGwky-C|YKlMrZtUJ$V?%bq$$TgJxV&(6m
zS6|-#{H8ERKU?!gVAT5j%=sPPmd;k#`eF7>kty4=FIFv@m~rV?*Cl~;qm=r|_m3^R
zsghpF_Ptfr^}y?Avi`5~%FZgPG*4D%)O&EHH2i}{?5Fa7(bsxHbdG;h<9|D2f7$-&
zSNxa$S{Z(_O3v|3YVMo8M%TVQ|1vSOA?r^=RLr?mFHDyUGoQ~1O25|f|G>pxA}z0M
zKgjsMxA4ExD{a3}`sgF3bG|>nB>r6U@XG&<oo8RXNO;C^_v^hcUhb8b{^xv_`!|pM
z`?l#NNBkpZ$uHbc_J_IugTNb?ThZ|b^UF`n+HUk>$Lu5D<*#pAQ^)sT?n7AF0XFls
zX7ia=+IP-69%w!>Rq^Uq#kqXXS4`RMf2n0(?Q?d}Uc^aCm%SYJHZo;x&EA}P=ztKD
ziq@Kpk4i^7W4sC;ZYWGX(jk3%ie~7h9tI)Jol{C4pYmPV!{l&o%Z-PNoMu`W#_h@6
ztbSpxQ~R$qGXj^pF0nQiv$YCwnBEoWY_~S+s?zkXIqRbCbcH5_=|yI9X|xJ2b=VTe
zTC(fLHr?wxicU{^=;HQ5j4}GyGg*sokJ=g=AA1QJxH>%izo&NoBGC(O>t-AEpXYPg
zz0^>XajjAIvptats{<FXZcx%~_`KrG8k_lQN8KJQTR7AFZmzKOH^;_pY}c|}-A={s
zx0tFHdSa5Q=QOv2lS*DLyIS0svcqQERO`2Q-rQpKTYD?^;rGUe+b2(tiqkk{6mnuy
z{jM929?pDUSoZ(+kAGJ`K3~7?soj>1G0_WE7PvB;6TjYgC~}KqqLZmbK#jlcvn~b(
zE{BD!7c9df3shOc1tqth{W`1U7OVO#zsZ6wZ0Sr}f=@flb~F*@)R`EVENSki!obe_
zbw_ZB&djeut#)l8ms$-npSiW3$}GF&9{TsIxL*jLX&mR>OwD;smbOdphx~V~Joz;I
zUvB!+fDFFvkNa)nuBJ`@eKK>IQ}oOyS3}e|m#CLnA2gj1>GW4pBf2<9?OGF)?zW7C
zdR;Av8Uem93mAk0PC3@(R)svMw{R7^9>KmcBCwk)HSASz&H{^ortaQPw@R1fTs&3E
z9O>T1a`|aQVyjQzuEh=_7efR+*X;>8wtOqwY1YX;Tp_Za<$F`6ZEBn5E>kJSW<5RV
zhFE&2WC7RJ+*IE%(S=Ukz9(13rEV&}-p*wzmcLW%W%^s!?{Q1sIy5t#S{?Z#Hq@;t
zy!EJ_cyP`_1HUO@PYRZE1y;zmF^ld@X}#z*<zlSX)g!EuOaFAOU%5P1<<=?-(J7Hm
z|CyeuDXZ%WN-FoptO!wdH=cSxzm&<P?eRP#%a>eUF)Xjzip~joEu1C%a&_#q+Y_^=
z&L|4oyEgQm-t)Oma+A+o-BrqU;eJe_>>ROqO>r(6(Jk*HFV2sTPYqgW8+CI9L!+op
zNK5;|?i+Q=;Y%K+%;)<fo}RO%<CVkGsb&I?)wmbD%3@w>b0L?h!)<o5#-@!YA2YpC
z3&>Ap2tTGPawuEplI5b-n3Rl6?@PLC_xLT`e9Tl@#%nvvoL!rf?;c!#=VsAX)wvIz
zR!^MeKRbNSbfKveuij68ClY@sf!F2Oz1z}0YF{$9nTOnY!teU6A=5PXj%$1B`tU?9
zXPpZopEp}ysruh+uYKX@jo3S0(wCY=X7?UVo)dVn-J?XV%CWC8>|~E^!V&>V!_%q#
zn$Fs992N$#@O!!lH%Rs^oxm4jU}WobQZ{Czp^-;_J<~%5$yrCW_ZKV>$$xP_c+ob~
zw;w-LS{|JA>74Si<(f~k3NPOBo3(vU$DM93_D2r!3w}85Y~igs_xf_wnN9n)pFEoT
zaC1lGk1G?qY`g+%CQotV6W%Ww>cSNK!0F(TrLyss8@v>YZnk|hpK>~MDeugO$-8wv
zX<gd#BJ}DdA5Feuf#gDiwqvUTw9Ab5s&GEg`=fNSb;;p1(=%99jXeF{Uud$Pw7{js
zc#W5v-p=)*i<ZY7DSA5N++&gdI|bHz-QF0$`R&@?t}nR?uRqRCyj41*X(eaez0H?I
z*KQTtwA0(SJ}J$WgVooGm1o+$&8J;IC8-`|2v+3R;yH3*DNEyqi(2fNXVyy@8}_gk
zOtZNY8s?$6!|I%75x<SE_N$H3oMmaub1KdhFKc(?^~*P^YC74!<AJcQX<Lg%VduGk
zORaL2uNTOxFmCF2GUwPI)tT0YkG5`ja`98c6!qJhn%8$IPh6Q}I3ZqUzQck{?>RY3
z?UQz1H~RTG$ocA1y-3NPm<G*|1EF4A(<V7AWofvQkfmQQBz0<qovZ&hP4|L|Eq6k-
zKS>>5nPbXzC%H4K(dew@2J^KOm`>yzJUMg6|08?~%7^omirXDHLpmj@UPQ4@a57$U
zR8_OYuaRfE+^UGBOXF{?oHL>Q#%hoEIo3B*_0M)p*9z_LdcggtY*(;%d8fa^(_rb;
zsLr`FOnbR`Z#n(BaAP~$ReufPfJ52p0g)@BRLZ#+S9mTeJv1eQ#o9DU(97=0frr!5
z&1S9UPzZPJigF1k*_L&bMeniElnDz&Sk^wd6t{iziJ0DIDcftDzB5&W)Rt@51wQgQ
zx#;ln2^+jKoHAY8Ec6Z)v33SeJbC%S=_&IxHgcO+#_~VhGQ%g-ze{xGBBv)AH{S34
zq?vs3W&DcDi6<O`mv#51#jZ%7y5IL;&=Zqa4t-G}OZ2{K|KB|Kahj-!tZ3|Wh3gx$
z9zS6^G53zD`9{fBM?EKP&74M&HLFv?dP<LPVt8*?f7dgGr~DyXw))XeNl&kDS#~Oa
z;iTo<=gxI1yV@_?lz*~l(}r(Te%zevF{3+G@<i0FNp|PUBxkJjUF#6heM}&nMQQ!i
z1}24~r;RZM&OaPaCclVc>#R7d#3_2QY+sfC<Fs}WcFEWtzDd)0>Ne=;&79{hw`z@J
zeF(#IS$U)B8Zr|CR6HV2Mb>;|jn|J)y&Bfy#Vxq?`Oh~quO?X=__P}Np7?wE+H*ti
zrnXgXM<Q0;a5XLK^o&Vw3q7*3m~n#ivM8m!Jdu&@f3&)yYD8VR6^gkW{|EN|w|LuX
z6Mo6iW!fIS4jHdwMZO+A-m&T@Wz;uTE&2K~ICM_T>(l34?K6r_seIp?$R@r`|IpLz
z>w~+p|CfX)czsHmDZg&XjD451Yiw0DyVw?TY`j+YL?(TyLteV-6btL6SLVyf8W%QJ
z_y#8kp7s>_C&HGeG@mE1qm+kPLH6fT9x2s(PP2oQg1%4aeEig;!^5HNi;DW?d#xvT
zpIqYnQ!G~D8E@C}iO*lnWtgi`^m^Cyf2W;Uxg!)Dj&{l<YF*x9oNw=^#@%vVt=wRj
z>$B|H_aYuWe#qA>yi!SQqr)G&36rWPhb9YFbS=r9vpt&SSh%9=3ps6BnQS(*lwO&&
z|37_`PuY5KTS8}*`t(ijwwSn8T`F>xeiJaWJ?+huIfu$KX7i+8Sbk-0@V8IL1pd1w
zJX^YH_m_D|Z|B9eRupPzFW)^=^pnq)qnTU^;s-t_N`LCVZTZAlX(toQgqEdhj@vr)
z>(s<gs`xN<Zx`0<d@OGLc(U!RJ9^ni4=oG!)cCC7B^|iO%jQbk^bHeQkKK%%B;uLO
zaCSM*$(e%k4V>|Za%E4tv@&nIa6H6Svhe1?7x%8m-27z`DR%bm%u2x-nhB`|hax0I
zI~-d!&0P?*bipQDR+l+*9oDpOKHAD7$+I9-Ze<UfoUPwBSH3P6ImMM;p$50od4wcw
z_?EnF)6F>;$MOFR-&-!to%4Qibid&_?{gwLl_AVC;Nh#MGV5;~yu{h?-a&*<Gwj3E
z-W9y^hZ-M9#&zyVIr{$Qt?DgP9_h}J+Vs#hWXkhGg|aQZVJSY{H#)O9S#o$6u?Qrc
zn7e?<!-0vzK}l~>kmf<X$P-6>9<@E0`s+)F%bRIlEa9IxkIHbdZ1+7EyypI&P?xhW
zgRLLP{PA==6B_hSaPD!I$Sn(5C3C#~?&X{GrDt=;g$1g;E|zm<hg@>zywW-$UUZ*t
zMGyN`&y#fru3uG{yGJB&^-b;{AG|00X$k4Ld;E{<>@8OkavX2%<qwvWSY$C{)&|+5
zhKrf{ByX=sJl4x9t$O=Gq{|oH|1P$DmqlcbCAgfBuIhSYal<gr{nuL;#hHsVCpkIZ
zSm5v=Vx@!BpIipb8%_+G?Cw7P$%c1hLOW!i99wR{&7Rt$yX3g8Mfapk7O`oLZc&^U
zIZo;>a8XfZ-#zn6kb#)*Gxlo^XE(Q=|I!n3=<B_JFN;~{_4^fG`~Kk!uTaaItKDxJ
zl7sHvR8(6Oq^k5yR7rVSj1WhIveBZPx4aG)eNOERc)m5Ry`;mbh2zANi<52&`x#tV
zz2&go(nY8Luthz&_C-T>3PaSTnuo;}ryT>h4g2PwZ|P3H*do>{aOq9Ymz)V_mK-mi
znj&>gF(vCb^QTh}OC_s)JX3gCo1WfETFGf_`Ttr#%f*>%UAr6RXS$^|R<J&Nd&FX^
z+Lyh{vfbwAu(ok8lGqV4sf2gQlOr7baYrL!TQ_a!=#Fp^@^kEUZ9gBt{(oXil;Lwn
zkJGGMoxIO=dw)9RD0AF3#3$j}k)yFEVi>0#R^;Ag$o6Pz`kXszRb^tbQEIg(T$5{k
z?LS?b^CmiAQCeukj1w}wrWalJDtK9%Dk<|FQQ}*nl(h7lq=v2DnZ^&QQBg;tu5!7)
z@X(rhKeEV`sj@$y<x$`{nT{ML=^d&uuiUIpEIl9Ktu`~`(g!XdlVi&Sdf)tbWH60^
zJ@bx(#qo_N_$L1muS^v4y(9PdZpv=nBUgWjUvu@@XQ<%E^Z#YtuL(097Tj!by|pko
z@#hJHNVTaS9#6V@DdGdCdJp@L#;};01#1f%MAj_J=32xQvs7rwN~SfLLR&ItImOS~
zCFt_RHMfXWcCTC*Ph?wFYFA84m&OEn$t3{+^Lk5X7u@37d6P-phe7;_jCYE_&$YLU
z!{#Z6CTg~%L>>v%w3&3iW`cCfD(%D|agC#Ip9Wlw6q{^#Y|^oNli#wxy*U3x<igrV
zM_6Vhu6=hrR)=qWMQh%xmkE(aSW@mcD7|pCSj3dF)K$gtZ$awL4z8%a<s}s!rzJgo
zTRKj3JZfF})OCZZ<dP)YJ25d#$LHOd*gfgqzuGPiQ?=N=EpMkZN*csU>iiG6{qy*=
z46%?8rxK=S<xhR(_CuTD*{TD5-22@YR4?U@c;l+zdyp+<X4#SFbq&u#miWfZQM=2S
zr2a|KuO{=*RHs90(i%3soZYwPL{|KqeVWRLrX5~%N6YYn>c!bkKNS3aBy>B<1;x2!
zGAL?JQw?g0PHS1Qc0v=2&8p?=u3kQyQ4yE8p-W8U&^*t?=bE01MJHw(=B1|z%A{>f
z|9B<6TF>3p%Io8rHD?NQWurX(GWqL1Je(mIe`=*i>=ahXH96lnkKAE9aEDFfE?1J>
zvMkppif0RyW}k_7lDrU}v?7eRg{k$?!Jo_2n>hM*E-jwem%l)?#G#!dl~uEM`ph}k
z`+xk;*cZk6semP^%u({e>In`ZuX#I*XE^Oycd{n%Cdbm{t0#p_3}W5)N^|ZMt%+Hz
zGM>}C=XT$WjGMK9(;`W;+C%e3%JFIYay1uq&1gFRDYDhXF>Qf^%cf^<{Zt&8SPnMo
zE&5iZuiSK_@{d;Wg`TaNOS)S4e%6U@m%IKgK<?Ru&{GNvUrac}mg6X0qqyc#x5%$E
zKPoT1Qthl;B;YwAFJ;m7=i8(OPpTAdxf0THdYj2K5yv%BQu<kT@8?+XyZ4^GGiOpt
zg8Q+xagr%A$HR`?y_Czapy1ou56aIL&kE@1GT=OF<IeEZ!|jSk?}j<wP6$7edRLe6
zB8KO>W6SE_j1xp0{;L$`6!2RvsQQ}0WV~_ekB}=OIkgX_N-Sa(W=|_k4bohg$Hf2U
zz^io^4osPva(<Bp+iHW`brB!@6i>ZfmK-{z;O^AOg)@$_HGTYY=0vE`qtJIBT?^ia
zHmzqU&_DThZEoG7bw5@Hq#RSseW77?Yddec>kX$g5ufcc?S4<@@$`Ot-s`$2P~}Y8
z8D7RTZ=vq@Hs0Db0_?R<Jo@IVe|}Q=Dfd%Q@=wQSwKKU7ojR*`G(GLu(e??sYbPnL
zxS05R5xdpVo<$vN8J28tT4G?V-EjArWdF`f>kk;HuGMI$@DSR_@iVGri>Gwwmi$u{
ztE0E^yfir?IJs#<2%GDIqbnD-hzb6WikRZSHq~iaE34-gW3`v>FP0cKoKfXuR7>h}
zbxpl=sc{m+9!H;qcgc@VuROKt&AHA=U54v(PTiDBiK~c?KNA+8a!xHKlb<p4@T9=K
z-xlgjFR9Eq8qc6)Vi_Ddg<p1k?%h(gZ@)Hv<9e}?>xl36mtkq`$yO5GrW*UMtU06A
zvRziZi$^QZXVVLrBO*3!T{|vcw_+~~+nSpxs3z*Wc|(hI$`bCz3ES=n#7}k6v_9GW
zYmQi&k7UoMfFrMj{%Xkniq@&u+h6dh>fj{iX>$GxZvAa2l=`~${jG)Z_odja``Kpm
zc%`@{otJN{2<)2a%JKTB$Q*-jW}l7tTa4qk-CgxRC+>k?@28tQ2VHpmvm=FO{k$8v
zpiH}xVPg)*(e+OvcI@MrpL>ukWy0G#^V<tp&dHcey5YB=<{=yFgUu5d9XeY}1<y`6
zF`H8}<S?tRct1<XkC><m@sJxY+<41U?!A_f{~Fjl>(IHG_xB`T)+qi_@>0t0M(B^z
z^G|FEII`t{(P#PSRwfD2O6JA)?ZoE4iU<`k4gav+)xa-p!og=-<j>7)6<PAS(8sO7
zcanCFN<wKs^Y<R{sPxu3pVZDi*k$PZa#Cmf$9}P@(n&iHNk?>4Bpq*4?@y2yD?8mf
zt>*3fs>s^y{2x9E*dF}kee;KSF=u0jZi=zw!Vlg5lvEpfxMXbq-w!@uUc=pT_tT9N
z5u1<v=qh>Q@$cek@h;aB8@>h9Bxz=?-X>;zeo~j?l_pDW7iXK6&bU=SbnkANa<sH}
ze(Bd#HC5kjX+4Z#ZkivaK4dvBnk5pp+o^opy+d2Wr&V$H?O(;OC|l>^nmk3(S-aom
zEzg{kR*tfxzgC<)bmF;>%QcVF?%Thvdv<oF<X>KO4T)L4nwNKRfBrUWen`&B*^=_B
zI%V`F)dN1HYVUN~A*jnb;h;yfa{g>5vAm}dN0N>l>)rW2nZ^BVtV{kMwx-GQo_7xJ
zzSSfadOTLgSNcm&l(hH1RZ|a}{+PE!pmj;#;hBxOr6MuqVlQ@<?#s;O{~JAh?SFBm
z;6TnlC#q`O_Z;P3yEx5!=aq;T?bF)(XRYzCNcf<&a^}N(uF)R9_OH3gc<KJuV#d_l
zy8=?SP5R$!ntfX<z32bac-JFJ1)Y!fXR}W$Ze7HgUi>jwJEw^yrR~*^J4ZUd&8rAV
z{u8QOx2p7gXX*Vi&l!!c&aDneG2HWV_1#8K=lNEZcV4IF`m8LP^(Sync)-cr0>+Ka
zuFpbV#%rIE?wvKwP*i5&#x$u@zh;=TZ`Zn8d9iBsF4qm;_itIe-{skw$1VaIt6h)m
zZB*MC`e#epwFfLGXSim}4mgoh?tMJh#%*a?^s}$Fo%2?2nSQC*Zc_Qp#<1N@%f3ym
zc*gfJ<w~jk|H6H9tk>qfO}Z&QVVkM2EB}TM+6|#<p0~9U7C);w!u~x&s9c0c_?D4Z
zvwv}a+Ilwy;bs2E1o!xQ)~t<ZnZ(V&pwe*H=tX$=k<tRGt#K{K)7=*QFxQAX+WPHf
z-n}(n*PkeQDV$$&jAg>1r8jNne?PTR<D^?f_<=~-xfe956}Vqd`qBE<ZpztzmF90N
zR|M6b5!yd**5wcmhbb?DmicfRvG6KA32N(ZXk$`v6LDype574SG4J7~RPH0bvaCK6
zcQ7Wq^s35oDcqcp;x<J_Ax`9g<C8OeM#^C=D}<Le7zx&Y6N%b%C19~orPrGlq30Kt
zx-WE_daAQ~N{6EXTZe?g|HcM}nC2SahR{_E<|&JeR5lAFYxm@^Y{-({$Q!%3RB_kT
z6I~%kd^P@i$*6Aky&uSA_Ods^+hJ{3?ve?bPoJero=WF*;_3RhT>89|`Es4pKa*Z|
z_^P&TtJsyoc_miqkY1*hnwuVzt2GaA^tLPAdv{Oi(2!9oWIV#I=W}ktPp9W9CVmTd
zCLUr@6<#)FWrmeORF48{Lrq3$N#$o}>n8T|QeP{-Kb<VKmcwTIk$tl-ntgc}d?1*y
zwpxo>|7y)CpN61Y3%oklRCzHluq)PlaQJ5;d|>TaIpq$XW)-7zW?McS%Gm#1rD^pN
zLB=C5HXIHx5>yI0@#3^xv{d&Au^_eoHx4;4CS-T)yJfOC=lJ}p)f3DYI4qv9S(Vvk
zlatWG5+T(a9X)4HO=`0C^KSIgIB{C<po+&~BR-|%wOe;QXlm9qn~}t%snQV4$!YM)
zt3gF1AgNnK{ncb8#z)*d9uw3gvbxL;R%h&O&noIvo*b~MQ_bVNv##CkSI$``|Nls@
zQu^Q3q;Z9_TFUfPu%XgYMb<thZ=H2pCV4r$=WxkfyX#eukCt8kr0n$_5s4Qy5?NPx
z>uvFnTq+u?9A#!S#q4u|$-@<QKOHyfx}tOY`z>w0(1qLHp4^dj^6aG3sxzbWET2m#
zDW28Mc$Cn@D>7q2Tv(*2LVT^_rr?Hke|#O7WW)bY_PEe-!h_#q@`=aOF6Ap`iP&~%
z+L^|5RynU;bXLzu)bnq^r-~JeM80@wtdsMxH0``oF=;=~NfpLLf~u1$(-y|qbUmBx
zpY!qPMURt`Zb4Bi+ub^&6jPaHPnTR*xSsSZX>DP%_6v*ECzq5RZaq=(Yk6Mw+O3aN
zyoIC}7i@kaXqmNIj6>0Ix62EKrX6MFYfW$0eD>vh7Sa@ICF<eCbdYnqaVHDM^56zb
z<xl1XtV<p=g}0P5P3b-$6=Y&Foy%3{lG(zj^{PE?|6`Oq3wN0WuP88*J`w26*>$-i
z?Y-C{KXn)1l!b0bYu`-L>)rZML4e<85vyWW<jD>_?ySwHmi$lY>k)8H5qo*jX-}R0
z;htMZb_f3Ql2S4=`?ytBDzjvbsOC0{j@bEmk9YMp8x);YTyb-+uG+H~ervZcD>%Gl
zVa%J5ty~KIRwgp6%i3Z~vd=L;-afJ7tT)%UP1E<OC>S*SeH|~^;=%TK3#$dEUjh?@
zBIm+MEZ;KIi)}x~d0rEqs9BQ2^(OgjfKmD3_P^@_g=X_+SADTo{N<*x|D}Ik-J#Wj
zsr*_gT_^T2C<t=6E^G5-;A+}XvB1pcnc>l{xibaSOiwL5Y8}*lH>0f0_*8Gc#$@KG
zixP(=IxN&|3~xQ!bjo0h?yW^~0yF2|I$^R!f_IWg^qiT|=hFGjD{_okbpAh>vZQ6>
z=C!;^&wgD}y|1>+Xr-&~+&O)-?_AlS{z{v%@{qU-i#ltlmjYLLmxEB8q<4gyBX?2Y
zLbYuRIyqbtc?uH)Sz3&Cuyh0|rJM=jGZsHQf#ZSi!@y-8H8&2;>v*Da`iuu(tkC+5
zus1==JcFawl}x#*`Lx4D@v5b_>5)w<vb5EHdoeUASiW{>2@N_eKa;~~zZu5~OHS|L
zx?AD_M%(iBoHyRv-tlzjB#vy60#7OFup@2FR@wgBRvf(;bCFB$%2Bb?Sx5C_KH6;A
zvS_i`s|-(h<;DNE91=a%EOe{q(*)*IZ81?nx0%=<373B4Z+hD_z2r{F+$zK3_+MID
z?Ek+XUA0amlxg;h4Ba0>t>IaXs{(H{i76?thGiVo-P)3+Yi8J1cI2oy`{M*pmgzG@
zW|sEWM)CS(NbGd=<XPtwsCIN)=YhpL{N_HN$sfGrlJwOh&$ER!d*qd0ZaO78Nj)$}
zhGEvKGv-?}Gi6?+%=))dt9Qf8fH|?ZE?I063i{L1SzFs(UG#0yBH6W(u~)gRZxrma
zW73Fl-F&$A&e4mhf@|frK3J<8;mCHgYKn-N@T2~`S$DQ1Zc-@rJ|TXxM9sg-Y)bXy
z6M;Wn{W}%%3-87Xl&fCapfO>gT=E4Mmx_~oVH=WpiY~T9ur9dgYoNyTT((vBhGIii
zg*sQ;V&VJ$X8&)|owG6F4a*cKcGijKTzymuoz57$3r#$?(Njz2riXu&khACAtG0b>
zI8Lg4N%Z{YZswiovApZa?jo*M^}WVxtA8zOS$CvMJ9q`x{dK3;SH&)m3(64o-BF++
z&tJL8`CbFB^Q2>YA~N*$ZR(EZ;}TTYudHTbn7`qMSDR=nzZ+A7M#DMvhQ(VKiACKA
zj*AHFzGpT?v_|AWMCsJy$F?l0S^MGWg)g_cQm=Y2tlQA~d){+p?UNxIj}K)sZCt85
z@80J5S(`qxSST=Zztc|m$tAJ>&(gcWH#M1}KOC)`xZ&R6D@h^&vdvYI2fH+uRPlV3
zU=`Z3p)>GG^SLz%s~`DvcxnFkajXdl+!VMbIZ@hOc=nr>ob4a8&d$5>?&v2&{@gGd
zhihL_?N=DimQEL&a&g)dN5&PW*aHvUeXrM-zwFEHIZs5?*`r%+q;I&y#)#x^ms0ID
z>^i+Z__us*@@^N^pSjjgEYcZW+_oLKWF*6wXA*Sah(gB8jGmmt{zDTI-PS5F&)l8S
zJ;7-Aq>T?J?YvdyX<%9Ilj?N2<g?f*nfosbt=@I+=x?7`=*+T*`TaW4Ou2WHG7ieE
zQ(;|sb<Uc&ZMVd=SGB5Ln_a><XP4+22F`P_oZc(<U(yOJ(^mg?B&_p7z#+Ai`|%Dz
z!dZSiN=sLFE)o%Y_~8JHYDhr(M%L`z`Fd~umq}!P6m%>-cXwyR(X-P|^)N4J(0nk<
zMedC0I>G;I_I<Jr?b%+m|ErONgzwJCtq(Xff4qn`y?jEqT+KD!Yt`G1-LBjDwjb@C
zR(eUV>U8w;AibiJ%`)uI&YNb>QrN4u_Q9JttK;6TuQa4f!WKHjc0CP@5%IY@W9>An
zvr8@<TD4tjUSLMz%lD!VB5OQOt0erNdP833>f!}nf~S^>&Up~*dce+jjrQa~Baww3
zO=n9xCa{}2wmp2cuXvtZ)RDL8uj>UWkG(&a))D9IRq#lu<k5yhYM~~Jl_a#47#g&v
zXkKnO-|~me|JPFWNnBG^7EIXV<Kou*sex5L?0ou{v&w!3O8<pc`EP2^s5<m$$pYRg
zy&boF9vPgMTX071%>oTG7NwYjd|wXnMy<+@N}PD-sRNHo+JProLWjJcu+2YpWYasg
zxNQv9YqB3O=oES}JuA}N-LcX}>KS(`pXw0-zi03FTYPZMa@@P@{jYngc^fsko&>CV
zVZbyYVS1S-U%4l@NhAO91d)FRFK;-W{o>un@I>j;Vmp?j?G{OGGhPb*bW_oD{~n{1
zv}jSF-62H{9_3G6!iq`0b{lPGF)fQYle%xEaMn_80Rh!xno2P~AsNpPobhDnRbpg1
zua)&q<AQ){%k$1HP6<aQob+n?bB6iUtflHr&rZpyEp$s<!qCF2_WrCw)7JkviNY)N
zKYOg&-gaz}UzYnH4+kcJC<)&A5^KNJby=A?TzbcP*}|jwnP-9r#}=<9rv0sQua~Is
z^nZBM^5K*d-}d+NUkmt_Ch*PgUv)-1BIxbvwng$PhvYMI*G4ekeCL&PuYcwg2bDi3
z%@&*#?pz^sf3a|>21EVX_bhAGLzV}fNe&5WQUAm2;I+C)uYgM=p&-nUd2_ajz;b1g
zWEJ}fz4s2gPSJ7YW4>MH7J100PoR@$l^@631EQPQtNN7dPNX`UEa4PB<Rhi)`+;rB
zte3kPcq#>+Pkr%Z>N}RbM}*v72~0CNxIKwae6RMiV`p`XKkA;{u>0Hv?Y(&gOgy}T
z{Hp&OIQd^27%&#||Ng+s9JrTl<Mvkz#SIS0vmB6D5LT5CQj!T~_;pbH*1>STLZx3$
zuU!tWQBzc2mY{s{?6H4JCbmI0dzxmMDZLFznicX{Seo@_U(-#wM@k(}f+9MMOPtI?
z(lry(ozm1Te(R-V6lzGN=~X}4Z*<mg2a|G_76aQdl{N2dSACXP*c9QGrF4M(e83|&
zlZFLdE0!O-XzBG}`7&qO;`L1Vms(hO{2ml??@{OaGRcm$ecGC&)fFoxmp1%bHTiqX
zc{$}K`GdUj)lLO>iVK!Fv`U0H9`<j|&~)0`P<tlFUfAgDi$o*wAfvO)M!~)Q8jJ1)
zd}#Z$s!hy&>i<K@HYplGIet@wRR3oLix@8YAEhA}du(D~u$y(ByP<2ah2BPv>{OeH
zKUyA5>1pzKt-`72*YWAChRpe}uA_F}I;^G!6@5CPHT&2+ozt5%R`0r$>nIR%d6$6v
z#B}}3JX6EYir+|7us(FH%=2a6x~nJj4R19%6zlO6uF-Sn;4bV_yIde%{I;2;cp97W
zpBEGO{|X4OGaFv@IeR69`_pRmKL^E?nu25}2l=-e$_wvU#`C%+S@_m@p>PrIa^C+=
zOe#hbmF0Dm-IfRF^}KO2H8X5_yWp(GCf5yiIp)PqX5ZeG*BPr9iI{cstNto6`Jl=;
zA(YGJbJd(P5iK8?J3og1U!vo+cx&XOj~}a;YLt{}1lZ0W^Ez|jtJ^CN?}x0H?zq@a
zT~(;FR@SjY`N72x3zFoQE)Yw5^7GkMxhzMnj%6=BdiQKwmBDsc{L-5KOKTFp6*};5
zb1-cBz>)4Kx+PKZnj?FW)A}r)EeZmiYG==d>^BHiGMS?Id!hW62C-i!l9w(@(G}Km
zFjF$QBBwD`Uw~P6%i#k-kAhAxgyi~!Z08eS>fnFuok`7!c9)em>O$uJ;8OJy%52{1
z?mIQ}FE7jAQ+n-Rn<udEKJ}<T>2UaywfFPfTy|dbzvZPBwqZuqC)ZmKYWA5b-D>0!
z2)WCl&dm6%;GgTOLqV?=t=s33J3XS)@Bg2peuhp3x7Nr-JlylEL$0b*RB7SVCmz$i
z)?WFQVen{?+?E9qKR<oYDc50L5MVl;Pkq`duFw3}6GX3di@jJF__{I3@Ty_xc9k%3
z)l;txv?eQRPj*f(FYAAl?Cn%$xwUB0HO6Eo^^`{mE!SQLIL+LaVB#$rq|#ktwp`UJ
zFv#K3O|9cfo1Ru4@(e$@(xQ=FxodNEZ*it-(mbOJ4ysN}OzR7eh5Oih9B5!wbrDDs
zVLoN2uf=qD;i510Z{AvaI_gVLt95x1d*8J1S9wvnr(L9DL8WK%@dI5GLNvmB3j%c`
z`to?^=kvNZ%4IAxjLFp%wJ?m*^id9LkQH+hFG>3UadXn%bb~3b$^C4prdLwDnNr;k
z%ey3Av$(P)DlFmWp^zN2%)RU(;l?TxctRFChsK(yzR|at(34cKNG&9A;}@=){?qRa
zmS)6GWSs6ET(+X-ui9=Yi@3{Kk;_cA?tQQQ{#f;FGOObeyOz^uRg{c#Jp>-Ve(}YC
zQ()>ek*_~*d7rBAEQov<$k-?PBF7}b^UR@B8HZNM@);Sn_wiP1a@{*Ewcg)=*;s-5
zQt?p-k$)cyvKN+!el}EAGLqk%Y$~ekwKKW;QLFNzDW+TH*|twfoRJ!CnHm{6O>xn*
zcBkOKiRE*9)wP&j$8~U5C>b0$thG(SWb*Y8d!|`pImOo&&-`DyO?`sytQ&c1Zq6GH
zEPc0S=FwdWc2*0I1T`H{ahkL7XWN0BDvhBGI}cdBY0+W((jC|&_utKP;US5w$2IPS
zE3Hyj@^OjCFm^S$eDBgVw>kZ*BM!<(JXEOm)?4$jA+w<GMxuyR_}N>l#WFGl7M@?c
zfN$RSK+l_}r_b|`IK!><<ipQr7v%kSE%E=6Wa@ZYFGR8RkYTu(Qpn6@`4OqH=1M6N
z5l0mEXGtl?%rN@Bu&mu`%Ikw=#p0?lW><eC+y6Tfyfr!G-x8txY1{#q18yu=6H1%u
zarxPV8;Vk?o!ZL+@2s<RaO+xpsB7(vcMG=U1PMePGE6dBo3)sw(5gWF@V1=)vqdzX
zuT^=l$TRSfXMS7fEeYkUg|asm>|f(mU|cS8C1lNkt|rduq6~Qk3OB?Qrt|afp7!IA
z*sX=dJGvV`Zk({vznIf>>E?!McHx1+xqE%Ki<e0mg^I7#+-&H$v()+LvRtd_!Dd<#
zhfhibJO52w(HULp{B4KxHKXcAxsoV73E}O9(PgFKd$(<0AOA$L;!y9uM>kS3u1~ri
z{P$4tgmM#KCYue)u1B4%l%A=@B+b6Ic-C^IWn#<RcRcX$P<emM@>WD(=5(j(=B)uj
zWm)ILnX3aC8w~Y$PQ9O$R;Sn`xzUv2_5=M7IuEi~R&8P64B_1IZpl@#9}SAq*W|9B
z(u#`wZzi$IMyql1vgxOa<lL(jVq=^ltpwPW&k6|#ms_nYyzxFRT%)hb(0EIKm3N7I
zqG;H{$$t|?A9WjL&)B)xX~*A%vSKoxZckbl?3l5_M(dAx+QD11e7?+bX}X(a;$`vf
z`n6d}lge@gE{c9kvJZJ|x1qv(ZTagRr?WakV;@KQ&-<$Jph<nP%Lc~@8}248pZnBr
zOT)Dnmsh$eG8yFWyOlV-W%Wz8a&8t$w-x-;jV}6~nQ$(!X{yqKnxI};5A(hTnJJpa
zrGYcAEPb?manrU|r>TiEU8)!b>Tk(cHOhu1o`_r`9_Gk%t+6E1QDlNmbL~?333Cn=
z?^gGldF{>NqIuiw{{LEZ+D*u0w~WU>rO@MfGnEf%O}nb$bzI}Y?^(`9UEf!|vtxCi
zeMF;Sa@TV4wf@Q6Vn_V%GslN$oLR`y+j>ZMpH8&=oP#rYq*$w@a#L0dJovTX)C}Fc
zmmZFGH^pXHBys<_c+^}<{r$z)8a1i9cB!F1mp4~!|9X0wwcPP)=@s^N%aea<xY@QD
z))h>?o!nm|a&4jft41*w#pcYq6|WOTT@rVy&N2#5<hkZJEkEXoYLc7IlQZ5APqi|I
zTwbp;k9EPc&VOAI9lyKWZXd2X`?vj?tm@`@E_EX2pRHUE#Ci6GiJ1o0dZx^n7NYf}
z_k62T@4sEWRY|?wnQN-D4Hv|4{oj?8;L>sDgPcz!AM^Vq`Iilpewm1OcX_yT1bND?
z44rw^vd%d)Rx3ib+_G+muKg2rc~!5-vYQJp#Fq28C_cDyMt;WbE036NRC`aB-z?u^
zyDV^SsrqGc3F|%e#}pMd**JY(rKn`$suZf;)Zx43R)2W@s?2p8;v)9{jb4@Fe%W^a
z;zyzvJ@?m3DtCPDs1sc~dC`;$<;>fa;?J2kPY}KNsj}lslgP51oX(@Jw;ZDLuibdQ
zLUh}KjTRb@TzCKD+^-`&!^5=pa`@UOj7|*)8d<odY&r}U9%}w4#`PtfKS{TWPn&Pe
z(xqYc&Hr3_7ik=bx1aderC0K3mZ<K8)h1V^|DVjeqS4K1&b38-!c+z$){{9(ss)!F
z7Fe_92znKqa?tc<&%3FV+8wAWG|8iLS8Gp@W(-TgM6Cp_;Pob_xfV<kSQFN0BUon2
zZNBbK;~chqx)I@u7mNzMd7oMbG%_44=2V-wVP!9andt3rGAAY%CeCH;w0d$gYeACH
z%-51RTU8mIR9jgmY@4y<K_+9M5<{-V265lntf|slVt*^-HSK2o_WSOI>j&;L)%ND>
z@M>7`WGTCo$bog+4?NjY_kFeMg0Q`zi#Fto=r!E)e39*Pk!$7bWnUIC9am62p}O8-
z!(=wEEG^H|wZ2S?{H%5z&E!&<cp%Y7Q#G_n+QaZ*4X0-6{|h40JSGbR6M2_hi9O4E
z#nn+(?MZm-(i)EmHFcaJTnTlhrb;zTB}xr7d?$BY=#;ifx!KF55)c|bQ}N4Vr<tst
z0uz}u)Fw=HGn%IC_@2|HD|O~ZM=tHf3F#Ab7qA+HXR%dQE@lY~Xq%AEG*h!7ecr`o
zX^zq>Z%**oAlRZdE8R}yqMP!?!(7%ITtxyszJ75MwB52n`GC(B%M^9@S!<fqa=)mA
zpI>0~QbpL~aH~7F*Xo4I$IdKQFQr5=2^_r=;p^HeDn5J5rBnJ`A3dbPnR35|tl(Q%
zD$vQoxa@VWiPn-tCl#jGVkuK)P2M=QG+Kr?cKEJb<21?k*o<Tw%e4O&wlY<EPe?UZ
zJt915LTOll_1c`E1q>JDo}`9~&Q)`C<kI-4lgKjBEmTOb=Mi_#YfF>mg+<5AE-m=`
z<q+SB1Dp(ZJgy!6Is1^(OA|KR9f_5)UWa>)+!m)DeR*4A(U(md94Gf}@;ZHy)8Vw$
zCS9r9j$0=71UOgaSuJ0|w8i;?1XGWD*9B(VmkJHDa~g#%F#Y-DuN~y!Fm-RUq{fAC
zVTX*6jiO;k7#;TriIlB4s>PD*c;eL5vL4ZcH_A2?$uC)-Ak-li;rP`{p>^l{i!Ny^
z0;fe?*pb6~<&?(leOxM%F<f8X-6>QRELtHb={0fgnhlTqeWsk6sqXTZ=M<lALtgs-
zr&@2mOpYmMRlV--lk)9wm-oV7o4X`tNqrHrUA$YfvUqF4EDcS2#ZnQ?uZmCQG>x}D
z>bCv=<8%L)1*#$I8En)pCe#PjitcgF&-@+n<C%1c_=fD=FE0E_E8Lyf7(F3L@6Yw7
z-`gGPj<?!BY}3wOvTcT@$C5*{U3oZ`oygR$=5q0PqO_p1@u{1_?uj#ee>rjoI=o42
zImA~M5opM`W^t;NW&f*5ljlTj=#kTsX<e!iYS6f$S#(`BhxL($6}*z|VzU@ToemV+
zF|Ijorr9iUN`cMrhRkv?2WOFhv`=zD6CAsiWVl-AB=YNAV+vcF;ViyXVeOR;wYXc&
zbGAshOk4BcvL!pFTcC|6XxomO7rOg|;vKCRb%Y8V`_h#!ZnU_vW{2#OTr;ku4MGQ$
zLs|1Zm=aVROPti)f;tO2weJ0&)UqS+z$=Bd4f7Qrh6?NnP37*55`OVSW#e;|oRY#t
zC#EJmTVAy!rDU}Pt4jultkkt;IU~=>dYVb|LL*iNnW*qNX07Zz5bG}doKw{Hh`^4<
zE1JISf{~6&uEN=u-29b1lU}@e$Y0QPS?sMss5YDVQH`w&yS}V`q{f=Wy;H`GbLX!V
z#|xL-8h7YNoV|4Q<{dTheJ9%8`XZwz+f<%dF(s*fdL~cxiLEm}X+D`WVPU*~)I!;S
z!igepi%w**O|m^K(emHmN7J>vY@gQExhg2ze(q%zy>`&_YqP&@H2aaP#7U(al4Rvy
zw)@0g@_y5^sN}JP^Wg}wNz%KFl5cqiI~UxL7L}T+AYd5lEn0N-&YC0@k0mqDPl%Y)
z8na>b5zkaD=STfp*nHOh<CqbB`_?s`Ylp>(b5q3M-)d{s+aj9xAbT2%fPV$o0a2z8
z-Hd_CjWuE{I$}z*T`wG3Z87WVe5ME8soLE)P56owD$cBN`1Mn%p>3PXLeV9*Pn9HU
z1zi~Xg&i0;F5fyB<ICl-+QY=K^R}DVGOzBLoUZ+BcicXG3vqLjnIu&a#h&$Wn{<fh
z>!3@TFG~E>Zah5aGR1gTRY#NPrvD;GUv!=H=G~O+tL?m~GU3_E$Yq_WmMSNFnj`uH
zGcG8^nRc7jexA@WH)wx@)Li|BV_groHi<r$DQIM#GdGMS>|lfF0bMDX^Bm?s_||ni
zmz8?M#@HgFsCZVcpYPC(2fc5@IBupFm4;5YTJ&ClL-mA{;3b_4a!)pJDozlwFct_i
za8v)${i5@4y1*Hc7DL~&e2+X9M(w;15xz2Mo&CJZr9X^5Tb*7ww=#k0>U^h`RG!l_
za`v1Fb~$^&Sa-LKwu8~csVCghg<0YfGnkK9JoQ_)PjS)Q^0su<O{M1xN>|%5wYl!9
z5?J5zZP7ww=NtRE!Y+z!Y`*6(Yu1!?(@fqcKK|cjc*S-{&klttSJ$}c^Cs`QklJq;
zwDFe3m8pIuj&{p*-*g^MH+NA}JTk|}qmWxj#f?pK68l1*DFTn86=XdbtSXl$Y>$X}
zB4?oI?Y2@xZ{@No6=S8XJ*s8NoBHyME*qaQ2{B51w##oXi`A9@Prm!!_kG{;=zSMm
zZL-^|yJy46v{>Iok^Y<-9X}5F92PDR<@}Q$ksIjUt5VS{Zad+Cn9D<^odt(k6WGly
zY83jGeQ*$poyekBpcDIY%TJ+}31a(qS;|f-+-g_5>UhL5&jX*9FwK5@AXCs&{R#J{
zlezbFJMQVnm~7N=JmK)5Sw1q3>F0_pt2ySEIaI3utP=MO{XcoaRhNfR=Btl;224Bn
z`Nx)`tM5dXEHLJp%N6YFx=brV@{vq=@>Ub`#goPUOu4q>^Ut+)d?jMp8;?4_o_j&k
z&!Xk8-KXXoi+H<aIxg}SuRl_@bY6UOvZItrr2pzdg^Bzt)<wxCUu3Jx>-XN25wZK$
zfgJwh2X~c5)*Xp9`xB-RAai2JIZ4T5g_}fKCe3E7Y^k0l^X;Z(bYFVP^vP?T)|9-~
z`1JnVzMmhqPPPB_^!+~8OYa$kB_6PqsoRLok&MppJS3R5VX<W0txkTi!agyN$CLX6
zqHK>oZo4AzL#}Galgmq<Fd3M}oI4iK%5z0=`}6;aTtc2!VN$kc;`?0#gH8S?*oV7X
z1PVl#n0vPi{3@zooygL;Jn_dO7U>PF@r$FRwoCa4vMNpvZQmr=k|g~2nzU|cypT$L
z(MA4uAC0Syb5%cgt-V^lErDIAt$=HJfrL6kt-AO&hL&AHb?2BYnF}M%A2RRVDj@A4
zcGsA7CR?lMrOfo;O76t^IR!eu-E|h3Rk9~4*D;#23byI)P_%9e@{Y(h7gG5>Su(Fe
zz(p{J!%?J*v3l22*Ts`mf|!e!9%N10!WvkSEbJj*QYLYR-RAYOyxB&+sZPFo9;cZa
zN<V+hzu3TjgHr)dSOMpEWuEXD*B=Ge9)+xhiG6Hc0_I)Ib_j^8)v;gfVkzWa7x16W
z(@Z<FB6*8g#cVgR*p0ekiAB$oi}qWH%w|<R9Fepxt$K4>j(xabPob{T_2j)B0)K-&
zSDU;3Zf+<IRav`%@x<3apY)1{D-xL&G=?fNMi-cW{bVY;%wdTjYgB?-ag*^2!~S9?
z<H)J;$5P5#75g)n*Ze!c7IlHmDx-~6(!X^{iKwB%S0=k19s)h3bxeh=ldj90J;Yk`
z(f`|X#Ti@8)TRj}M@)FB9?dSuP%*h>!e-^?8aZqSW0t9_G&p;%3%58LCX)L>*dd5J
zLB%}gpopgfTkvFtm<18WZUR9ISRaV%<X`gD7fg72t?BF3JgbM|JY|WEQ!~WU#g<A;
zN-O+7)!t&t3wMEP<7__33IQg2j-?Z$U$z%s%;LP5Q_B%#&gSWPD8@N0+mfw5EpWQY
z#D<5HB<-fizTLoh)~#DCu`hZ;CSy@wN~XY<Nh}{;#Jzl|rg2$dilw2Gis9Z5LfnVF
zH6BjkGjw5QbSc=TDHa&NrBPO?yo+h$ERRe9*B!GsFV1RWuW$+%SSvZ(`9%G@MYi9a
zg18QdZcP<<emzszXyP07p45+Gl_?YBJtoRCDX&bcU)wezE7LSlQetzOU0;jE)@3U0
zMlpPkRBVn*>@>@%y5X@^Kx)?{DXtHbKYtX746KMx5=dg2k-#d*{@C{(i<NO{EcfG>
z&Lpcf+e1I6{hzf|vh#L`)Z4Vu9Y4CZTXwCvnXsy(fNSAY7QxwE3kx>fEMJo`o8=*!
zWRyhbmBKk{^_N!YlppM_tc-dSF1BE+SxZY~-Nb3TUkaE_YkzLp!0y;!Q&oImnt;ng
z-Ow)!dqt$qADS(-j6piB#k+mR%7rY^n-!Rif?71xf))sGQ0%|<QF_)-naWG*0?De@
zTe-e`NJ%PCo!e=&wPgN34F(61nOqmkS`BA)N60MRz^M09z^<b3@pCSvtht<q6^cjo
z#U@rne-Q1-6yPnYo0&Fe%5o7&r-_qG1*Tk`_~x<p+-q66%#%1L&UqGW{#av@pSr-N
zWh)kLn!Yo%;m#vNP4$>Ft^Z}WZJ5^<GQ&mFE3hDYk+RPrkyObivf3`zbCeh+1aaSQ
zvtHF~y;^()-$AaFl~d1a39NHS+@`Qr^jX5X4+)olEkB**v9o;z_rpc02P2Y$bxyBT
zWDQ+e7C1rfWhBSZbxcnqxhFDg?TYAJ7`-!9uTFW1U75h_!u6aN7u!Y&+FufLk@8<~
z$R*`M&MC3dgGQ$9jus47Ar}%uHYSH0OH?%vwK6mmexztDQIdM<W5-GMsivC+)=XHg
zAhmW)XY(Z~|9Y*Kbr0B#ydyT95Vwfnl2IrUjhemgWb@We0g30cSUzsP<vxF@SL5E!
zNj{F#{vWC9UF8-dysq%!x+zgv^S_$SS^9sg#wLN5;JNB)=1WhR8)};8uaK<a6!Cvj
zm9oIXuF7I2V+ot!>dm*-PIl%}`kZ2vD0@4U|LCibnVSStql9dqRwpbMo}47tdt&X6
zQuTimCU<@8y84PO)nKP_HTzNrwrB^Yj$N#&3Y&uuFfRR&kjJo#ed5{`8VhZtyH-Rm
z;WR8<{=0xbN^1LYbJbtAFMk!P%~}_{d+MiyyS1k!MRbUN{5<2ol-+rb4SC$+OAfGI
z3~>!t5pZBOnH%D^BD92$u}`72@ll3U&1e1Ws|^aw8tNMbGooTUqrGbnEjzN!%ISzx
zdQi%b4!N}J{hwAZh!5TLGc>;<OlF><sE23b?9+@f2LBIqU1SpqVyhH2n|GQqm|^F-
z3kRHzFYop?zrI>P?Q8wlB?1DP3_Z6FKIz!~`5@zRDV-Wew#vdoue`D(BI|X1+7=yK
zx$fK6c<Z9iQL8q%3b#8Rn%p@3XLGi#kH9G@k#*^kPnBG!FxG8rNV%h>HuZA$ZP%Rp
zW@}!q7MdB%{n#|`i<<e**#b9MI$x#<7cu!oepG(+oGZnEah|ll$^+%Z$<5mx*rWp%
zr36fkJ-F^p*VMkS89uM3P0Dg#=yWKhK-~JzVy?i&jwRyOuMZttR4J_~(84(V<@3q0
zEDo-P*$;$z|I8}B?6J-0r2xz6s<4(RZx09`ko5A9(3*6uJy@ARAml&y=O4?y3K-A$
z<dqpzrY7CF<KR(ym6-{lF8lZFKfEx#z_h&fTl&8S*O}Zct{%Ij1J;%)Y!*1q%6o||
z^}>PF1a7rIn^Fx}RTb9q8E!E#JotF~Cb5s31=6{NJ*Th*D$Lv#!QwcFU9d9saf_mi
zPM42P?7~*R)$2}2OU2)wws2CEjnbUr;_Tmxg<q!#bUCX1SaVWTb$bJ=$izD`Jk^Vr
zOj%I>TTrq$KD((#`S;ACg**OK=dTp8YWP%h)3EmW5w6fJjkykqqSIA)=OpGHV4ZoJ
zZIcG0vd#$yi$l{+vr0ek<WCcD(Pa>RKFjTI0oU_|9&dLsU%VpxEP-#YK*(j&zkmK`
zsd5FD->EY>Sa-DO#3AoU5!pubZ^~>w7RDZ1PdV95JXn0%Li}vDUWn2uKVyNn3nVI%
zgMM!kSpBir-9n=E*L6-$0YL>;zF>jkB8^Rq%gc6kNGt9yl$v?R<z}H!dA94B+O#tl
zg)=zanrp&0bB43kHcxpP&L(n=E#nMpS;NG<1<iQ^$M>vgWR8@2Y;%r7@Z6^qv5961
z2PEhF9c)Q8RD4&XEx6|Dq~znSh2pZC+>iVAEWUbziBY*Ou&pGc&3aN4OW>rXT2hNe
z#8>`W|7@o4(m<{EALSN1h#Yox<(OVO_fAtk=e^X8r{9;Uf3@AzGuuV+V$IK&HGkA<
z+1zR`ZU67OtF`v}@>-h}2|6+7ei<{$IP7duWy}|R$X7NyHGs|BmrYV_w>sO3fZi@c
z--m*cVv9Z|o|`G~-{nN?3{$B?+A2<0Pkvp=HPOBIPNvDDJIa}dukIEQ)eyBW^1B}-
z#^fPz(@0=O%oC1<$#&-AaV}4eu<n&=z3w%E;q%G8sfiau`vls2CA)=YO^!^ja&q}m
zx&LQx*v#IWCDUEY-Tmh8EjP{<;7jM0=G)RRH8Hh$iuk%l<~I)*dF*%QirIP3E)J}_
zZ56SMWAlZY2Yiu@HMVY6ayNKRMku=7dBc3@oTeh*qKGACD<k#_d|3SQfK%2S28n4G
zBme$UG>j8G>vTB5`hQhgM*Frf)hua|T*G#UIF&Qro-FRy+Lkzk84BN8ax~rinabSw
z6;tN_4CDWOZTm&_1V+0AW^<L9Cu-}&{rQ&_xYfR0CcRc>1EcGWuCfbkQ>_IyR9}(f
z>s}J^E;RF<h@ODk+jr|M-pSqT+L%#z)$9F@sI^zJ*l!%a`e~JIbnzWcIh|=~k4|^J
zTHqFSsrFH?%(Z{&j4rc^nW9C6<ckEaChhRc5#isEWh;{NVQ%k|$r_@K4SP#2B=t=c
z3sV1@5m3I@OMo$W`F&}ZH&cB-AN46OJbJTqYI!T$ty-(MF4oVBli3}0w_MRHX~;Xb
z_3iRjzQXo@%l+p!TUI;>w*Oku{y)&!DB<kwJ@)fTS~$E!J*4)}6$r?#i_=}H`fGRZ
z8P3qzb_E+>l`1Y~bG*CGs%XR4%IlqvrPKGAABsu2^xa{PfbqdSThCR!v=s>!xftd2
zbar~jp7i@a_WB;nyL5bA{?jk_&*jzbVvAr?PrUZ8eAVs(lZy}6zvjwSU|e^%OYGP?
z_x^XO9~T(${yzWbY_sU)Z+-8tNOfP?Ra<?-_01N)^KuC{(z&;8$loMoe?W7J^7pC7
zPy7+tBXBTV;6s$o!D?|6rjs9nW_((_YMb4?Q>TT`xITF;BFR*tYqDIjQCaex?UN~Q
zByv7HVOgk`aOBhi7V&VtAXQgW?wk5G$|gs71pfW`|5M1~!aV)zZ9-}}p{qk@w+kvd
zm0Vu6h@nT)H1Elci;vyI<t^JpZhrJz$(j0Y%9D)t#cTUiY<Gz~IJH!E_O=493olPi
zJ2&64dE2JY^9vi=*F<ehI(#ZKIPA@wYerF_i}e|q%zq`T8hIXH&m@!(qNLE-qA6{V
z7kE18@wA?TyJb%nZCrPEr_}BpcYkjTIp)V?!O5XBY087%`uERL@7yR9(kznQoHKd5
z(XlY5U0ru<yc*Jb=eZ`QOy0PKAt;5fe9P4=?}n3d$BTHn1DhB)Ig8$3tKGJkMVIC8
zRcX6B(*qf|-nbg%wQ6ed^EEPZXMIeJ{oj~uJbdIt?PFa=?SsiI@Bd%=eQN40)2j46
zK}*WbHCcO(zq9-E=eN|==ll8Z?fdZ|X6MnV)-_524lKDJPg=dxX<FC&;?@MeU2R#G
zehpOyXSB>1Z!u;Y-CZJ(z%@<fF!$o47AA$tS{6}jco=`((r`V__V~lG-jpXN+zuK2
z;&zicI;pAWjM?QECfiTE;5d0{!k%?1DyFY9OsAblc)F>l<IM`$>p~pfO`BK(4k)W0
zIOD;|I?1Eh*W`9gu&<%vFD*u^<EuWOQ`vK8!LCk~PsYkKnC|lJe%EC3Wob`}k;ro1
zj5*yZbCkAbUh-%ZdKI*IRd#Cd@)@7D1czlru3HuN>L8bPy_rwxRbInGT#G{tUoBny
zzb;{`>*U@;XI7Z47tsiA5WW2?qp7b$x6?(4wZOzf>UGhzn|<Fd1u`huYVmCCS@EUA
zy@!qa%`InRu7qtj4Wx>FHi&p6xK8WgDOWC!TqI<2$t;L>@j3;SRMT$;le@!v^KSE+
z-{lb7ai_>QC?UpBGVaG63-ejQOqvR(Iy4*%^K=7xIlo3I|BkrDzj%VN)X&9}{r7$B
z)3x9@x$Jk0lm3BOSyCGgFtUdPq(_&;ZP;q%F;U>q^R9Ut#JE{^7#xm>KCA6Kae>~+
zdA&mWdX-C3cjw+L|D0sLSZrl!_v>}1`vf;;?iBrf>*#^0ymwoerzv?|{$Lx#u<QN0
zaAlp1N2@*mGfug)Vyc?D@+Omm%=yQsm1Uh=A*Q$8!-31#m`OM`_UNQyozk~QHlMRO
z&SY_B^+~JC+p^q~a!XyJ&3Bj1D%x~-+Jw|s*Vo?koN<VkQS0c-<@a@cU#M44SaxdR
z(gjIfA_3V8kN<qU^jy{g-t634q0Z~md#0{7ULVC^!nkg&Z?d#^Si`5&$4}h&d@^l1
zv-oo*>5bRd>$dhBefC~(@&35;+MVlv<{N#i;?sV1bz<NIMw7rO1x=ala`Uz{@oYFb
z>yQ+iA&=XdoUNAOuQmnC{}J(0`Erqi(>Q3}dxlji_YxD|+leh;{K(g2zKc~-&7o~B
z=ZbW;TM_TgzD{6wP1oepJoUeD+Ombtp8gH8f){sBD%dBK$`W-UYSF)G6-Q5=SfSVa
z@j=SDO`S^<T%3Yd-M)8<W5=W;+TLlKgYGkZSK+tK6M3pNH{wdEdyh(j&va$(XRkV>
zEgSRoL_-=_gxZgu+>@icZFX$qm&?{SZ#I>yWXxP7mFyWE(&k|w!nMNTC|{IKf5@^^
z9+$VMukPF6HC1+|=o+t=CT|LtOpPwQrDd6<^xpSaxI|$029rhmCNWH&E;W<s&!a^;
z&V_6Be>y+A8E{2cD{o16PoVUFsmt~g54bfmU0C+aVv)P5j8L~*kaCdMrzJfXxRqbI
zoXwiCT#&uWrK!>7Ns7t>hC(Z~R7uT9Uy=VVMw35IwpaSX=~i>8t?Pn%`sYT;tqmrP
zv!A%8KFmDIwk?61%aA>D{iLW>Q#vO0b1evy;e6>a@rU5bnNw7Ql3%8n39?q#9r1bI
zyeN6ii5VMAg_UD>oc6VSbh)Z%g<fon@HM+fs+t+v(MfmCOx6mPEMa?>zw}^G_>7h*
zMe`=|u09z(X`M=F&xC`{z0xw8d%m~}Ol3(Z*7Tm4m*pvFC#0@AJ2o~}@$h$-Qz^C^
zwK)wQ%Y|=o^qeHp5TE^sP45k-vT$mT>^1|oRjE%xy(Z_R`bSQRJm$7k((q-<dZBfZ
zWmk1}i*&9#RH7H_XxLCGcrDBRk;Z~ot-@EMo^5&YT=4&Gv&rr@ht$lTKiVccl_hEM
z)&=HoC6pL<1@>6%I+u9k<((2awZdEm-qOXl6wHq3u8=z!wNr$_{{Iev1y2&^9?vV2
z{d}W^Cy(QeNJEQ)yIh7t*@1}0I}FNt8@BUiPULrg7trRW+QlLqu&z?_F_%%sLS@Cf
zdCi|Maj_m>)>J9NJ!64{<SvEDGta~%F)V-iVAIEp$mY&m_dQ!$l@wpDY!h1hkVRpk
zShtI3-W8!DlOXQR>q=**J7s&Sd{`>WzGg?t!~<e?6f~|%Zb_REROC@|h?Q^4p><!L
zI!L~0n6&H55~VjsJvmPVuxXSp=hfvl?(s5U6|$%-NS(nn?Onp*gOC1us4cm`bR}zR
zWcaaXo8)$`(bS%ueu}ATlkev@+Lnh_y?fB5yfcSq)*U4u#vh-QCvD@7c@UH|G32tc
zf!5)ir7JG;?>}8<C)AMdEvsTHCT_HDYP$QWpIPDQM?3<Lp6Q<2HPLrT@ybpAW;4o`
z?>3mldZg0x0NbqurwLP2)-f4ru&W(-%6v0%FVh3riqj6BMmdchSD!ga3!P@QR+)Ay
z<-vh>w-PQ!-|6Glwch4<`$D|TNu3Qr6$)-XrxIjZO`K<6{eD^OPyVDdm#l9m(vvrZ
zS(P?sa-WR);V<2sdF+ab{#CB88=ij8lzE!z!GF`^{)|tL`X>u|2mZ3S>la+K?%&y-
z|HgL8$+tMJT`_4{BT;y#xK8K9@f4Tc$G&>xZt;#{^<Z4D@cN`|*S*O{@0o58dnqff
zc7WCEQ2(i)rL61^g1+ibSty#L=BT}<F2!8LSmkJo)s>aCU;p1&#3Ec_b>h|mMt8%o
zn*wRB3j-^^AM-KW{P%;qea*r0>hE{o$<KWAz}{N?aJI^72mS88{gb@o@3vi%o>;LY
zIr!&~muAYd%6_<~eEBJPm(^osp!8Im*jIk)Y03xou63{SPPBe_aW=d7<y|s8s^)2>
z1wpq~G1?m^zu5Ho=;95pH0FKDKC{K?xY+cwY+jD?2kyOD7gon1ek!0%V%wQZrkc$x
zt9aJ%M(muCTkv0pU0{~j>&Z8Hb%Q<Awkf=I%boelPrs4<b^w!dvCAp-Gu^c>7Ar(c
zOj>c0+vfG|*AJ~9t!NK2bf{Xu`YgFeO>6ee8+H>PZ0Oyw<?W)0M;fPPuG*pfXwRe@
zy6KlXZvE(bYTT`KdrGeY>nbZ{^H;0Z38|}}T-DUgd$mJL+j3rSLf`D$9rlN0ZvPNX
zj5hh;Ir;VsgW{JQDr;ucZR`wFXxy|}gHJ$aWpej^4c2b0t;=W2u3Nq9ZeypJwDsEC
zM*4erw(Za^QZ`noJe0S4uDF(n`4x^GvpAz~G{p#Oo;&5-+abD-*(T!S;WaPVR~f8~
zi)as(Xmu80D4QY5{)WpXcxuRhBa<j0PW#)3qpZ|!C-exL96WKD<BYY)4?{IZfun*S
z<$anb?roST^@f9=g(v7R$F<!~L6cpxt+eJ|?l8GHiM^yd@-fHx9b1oh&QVWfeO!Im
zS8(EbheOggd0A)dnbaYDSE2cX*RCHN_D8rjtg1Hq{<!m#cC)9D07LcuBZm}=f;&xE
zjD<EB6-_=S&TKy8gPZo_{VB@pKi=ZlIh*^t1hcT=N;?zPM<=?fUbNRr^RYi!!=BV0
zv5=infc5#w34tGYswzC{A9L6oKHN~uvr5n|MrKp*7FSjUv6C4VyFA47ez-7lEeKt6
zAoh&XafXIi%hfT;bEY+mPYvE9c7(^&$g=o<kkAoHnWA4DT06IH5R$6S?C8HWMQ%&)
z+}oBn8RjcxGnW^*sh*j2IjBKTN#MKnCZib)`?<DrYOqRL39NSJ>fW`lNSiUMP(N!k
z$Bc?Y`loqj6l|)i>QCIvYIdQi>bCW&jHPu22cllEhY7TXHb@<pWL@@>wM?QpP=dKt
zq#>^1#J<<<M?(&r6zM#uy~m#0<m{e_XJ#1e_{s5JlH2|ERKFfx4{O1>T1TZDcksNn
zaTb&bVi4Lc$#dV!wzpz$R0<En3>oc6&*Kj23wh?03%TWFvKqYNcruY`8P6om8&l#N
zc5qE<5ZTzDRlTp^Nawr9TR0o6?>0{S)VyoWW7+N<eDD7|Y!mp^@oCC&_fF3LMcvT~
zt-2>x#R;&wm>f8o!hTkQIc8z|!-x|TSG6A~kqNu8y{TrNa||mFXTbfQBNkR7Dpz{@
zS;Ufpe0yiuT-&np$`y`~GZ$Dz6;uQo`2Pm<I5hM+FzH`pTDofz$IlMKBFCebrNxY8
zavfz}1}tjJXkC)RbJ<BeZs81Fg&hjUb1w-APhTl%sk7aT!Fl>dW`^d82b7qF6q-*N
z>{`g(?_jv=6W1h*IXvbq+S4<dcUCT)_=^3nNOS3n)_qfijte%oFKIt6!(6?Z)x{?`
zY=vl*0Q13^*3gI}YZBNE51jYtQC%*%#bK|DgY};K96_BYMYmfx%KW!HyVJn4=d{6%
ztRB9Y#U}%1XD8Zj+`0B3!@+(BhAzd$+L!i5ZI+(6g75MUXVpN3z6F!cXz(U2+4>~d
z&pq~-X2p5A<Q^qse({MxOEQ=h7Ot=IIq&?l|KtU})VCdzXL?UP$>VyPV_)={cA3_?
z1O6Q~C#pX1th~U&{)N4);zU#g`}u+cbpmIOopJWP!BaE4yX(#N?k^o@6SgP@yHzhW
zP!(9l@r0vOc{caoj&B#cdlu~A6IGc0q@!81;Vz3pRPDyT6|2r>bWM)d>ne@t_n33p
zgZJL;4vF6!R~Kyb%?#)*;B5$Wo2DTi>)FJ+q1~)t%c-AoMuL-hCvfg?3frvMx$}Px
z|KuAt*M02NU18kC;uFZRG&F)eO2Fro4D(`(BY{7HCnmA_YDC#iJy9jW+;*lTY*o8y
z1gpcHGkrUJA}U2+35Fc25OvUX>h;)q@`IehXO6_iD_>7?c%;fV)ap)s(sAe37MYu3
zl83M4zHIWp%aQLnSNBoCv>U!E6Pm1J0)oGCEKNFCEY*6rVO4Jf=kb-SPh*s#4NR+N
zHGSp~Z#@**s1-47LuB11R-*|`7AfopMHs3a0{*iE_*N`%jtQUW6wb>TzAm<vk=5eT
z-NPqdNvYU$pULDF>U5Yaa`dReQRkWRl9@;E9~D&4xY%P5$hhRj*%dv@cFayt?w#;@
z$4nDt(f^kt`WCD@_BG=92Jz<(eO0d~=tiuL6<wFBHCZ=e;*Yl-n|^rg$l+YDLi^(<
zj#<9DZXTSdt2>)%-JK&B82Bw!k2l_}ieL>CU~c^q{O1e%kvq475?Rfs>acF{;Cs1w
z&kf`1(}%4$d(>}Wx3Hb-AT`%D)#S32)2V9lq(+mVDOb8rb4$HkbUFFxq!kCVHk}P)
z32m|6_+oDN^9LOXDpx16Y8QX%u=+AdX&W!+gSqFoO0L<${K?wCp=<8!M2#a^%olSz
z|0Q3mkiL72V~R#b1Y^U4iHq2iWL(MwJbHil-{+mwwZ`Mv19#ga&W#eRnh~q3p9&O2
z>4qs!te+d;Kj(_eG&R2ejrZ=K749!Mxt@W6)57nP$Iku}_dZ<ZS(w7A%DFA#sQA1;
zadDd?jth7_kC6GR%q)E7NVJbr1mC5(i%*5OC7w2v{X3U4oMCgoohFs1Znqxt?QSwV
zGOJ<D>d4Mj%jcwWMqgmKpkOjF%OUeG$K<S8=MF_N9eE~bXVUd$jnK8n4k8>6B|4fE
zI_f{Pn>A=!?v#pKso*9pU)uO|Q$Tdjjc~4~0`b3&tk0Y_sX#>G=EJkEIU3TQTsyor
zqw#3dHQjkiJnr{8T7Bd1>ud-)I{U#N+nkA^OKyl42PH?}V6obm`jB^0b&+x1uGN8G
z+_nXDHt*8^)}^?4@-Cwd3pMw+Z#MM*{QpbG&Mld`0k<Y)T-T6$m~VEV&gH-(4~O^+
znU@K2VZw0?3WA=$rFKj>wbf$E=g5ikoYejVJ()j`TY6^Pda;I=M>(eRT-+SR%XzX#
zOgB}vPAlT^jh+b^+w~0I_Vym$!FS`%{T>PJu(DLuH_A$PI8xWhw0MV9nk^B(#3}vC
zEL>uP(FB#EW}}@GwrPFd#`_~Bb!Cvwti#O9yj5T4%=6fAW>$~aoM*Z7j#wT$EX3#H
zrSrTg<87f^$D)QgPds~lHyqZTz;s89A?1uqSzO4rL-D;F3{ztlNy@oCKGqRn^DZOk
zWJXk8A4AecPuJ?3iA|!Nynmn1WxBfWfuNsg=UtJB>;7BZ-?T9F<2utQY2KJ@)6?(n
z$=dWht(p7%AIIt|k9r<3yg#jPlrbfm!~NSSzpq`HLIye368n!DJx-1D{C$D%|Lp$2
z3rwwFGHt|m6x`IQ6UpL!Zr$i{;Gu=w@`QOu@3a_BVAAbq5)>D#{wMIWH}%$O?&(K1
zEdAE;`QQfEX#p}D5BPn(eI)aRhTn}>cST#zD!0ih_X&6QtrglXcE4lq)&%WOeLKYa
zd{f_7D9oJ0E6AC`drE5Z+Ek%6sSd|@*Ck&1aazS>Gpm_4<E_kA-6Mzo-0~{WKFnnI
z;s0Dtna0oNK4u1`a}ILn?wP$b{3W}I>s#Ar#)WBb9mStFYsf8rAoT42!d&YsO?Rd-
zl%_rBm>^uTQ08=DqWtxXdpIT+P0Zr6^wls@t9Q-g^Yq*&dvGC_-ohkK#djQD=eB<8
zd(_pp^~{RbYGJ8jMfsPuPPPbSSf{A-`F4JH)n+%%3FY3#Mmu`f-adDxAxB4Xp(T&d
znQt7QTRzV@%hS6@(Dq;2#9al<R|*!pWhJGs3S4iwE%#CAeh0^&$5na<kL?KM<4n1@
z#6e2`C_~Z?%@r%f`Nj1Dg#3i1W&dXiJ^dIyFOKEG+1Cx({Su4UHQbr_S8Jc<d!_at
z!D;W;X0e9m-7j0wb5d}V%9c+&Oqo@mavK#l8Cm{R`aENYN|^Fwm5t2yA9{?k9{L&m
z|6p{lusMh!S+uL{8_z1SF74_!bLUs+RKISwe!bhCrzAWpcUgK>Q^VV;1eu<AE>@}L
zTi2RypUV|&)D*t{wr9ihO#&0e*;E=8=YD*cJ;(dEF8^6qQQ3LPAzpsmr?qE4(tGzr
zPB3WBqW0N>(=N<!`dlt|Z)Fd|orZT}n>`s=IyS7$<0;L$S{n7}>p88Ss^2>AXI-+C
zICK1{ZKK`f#O>mXm(2cQ^!l*fnFHFbyJW4@A19j@FmVc`u=dZnxo}QqtKkKvCG6=!
zaiWa}K0H!*#nK}4Ou;RPp>U0Oq0skV>)!vW`uA&w>D;ioe?H{_eI`>ja(Id!JaVms
z=S;M@>$Uj}|NHMXvo?L$Zrd~CYlpeAIalhw%D`Z;o$Y%ieCPEXUQn^<pv0`zp&D!|
zqHU3eJ8K#Q6ap47F$zu52q-d8bZ+!yQ}a|%U_8noqNz3Gk=p{@#xk~Pnvai7lMR=C
z&-2G(L-O%Py?8zs&S2FuQw+0rEs4B5)q5t>y9p(!?MrkQsJ6?MUR@SCJ1AvI#!B@A
z@}cn|e<RmcU7c>bZriJ?S=YNnw1ZO2avil@yBxMzuxeFmK3(9jtN8ID*^cRX_x6>_
zI>|&7EMRG3R^=+|{Ju8nSmW(ocOG5JJ*z346_>V$QC*ymY3GA4Zz{Gl7|oBJB{pN%
z7Y8E)%ZY4~(rN098`7EodrWBIS$%U~J#+a@uLA{)OIS7?^n6tz{rJdS{`a~IWKU0B
zx<p}~#BPJ~W=@qFe*Sx@j;xGlX8RZXv+?WXG@G(h)xj?Lm;T$^^(QB9IX};|_S^TM
z2By0YPwB1`sYqzzkUFs-C~wL|Hp$ci$75nLJ{ucZW^#(S>V{l>?BAh$?U0}Dnhi!=
z+OvBWo)imkRdf&uviUNBVG3u0a&PX<8IP0gc3pX#VaxS1xxn2hY(}ED+Cl~fFFuu-
zDNMc}gQn``9+RBZ7h3r=Gd@pp*^GQe1&5jXj0MXW7#I`}7BYF72jnj?QD4rm#w0v_
z4%3!{UPUK(r^HC-OMOpYP|^2NV(!uZJJVvcuW$x*JG}L|&Q)gNwcvo`Y^^Jr&K$C3
z6RQ`+En#r=J;b`yd(|;jS7}{NhSzmV&b|s-tNnJ#{b-xBT2mZs1XD!<Jy=60Nvj!#
zdP%00Zad00?d;QQ2}~LWGni~A8Hq`zE!=vvJNvH0I!FGauIm`OWjpo!?p|FQu&VOt
z))_9Z9&Ndi#Cq{}P|VGrtdm__uc<6$VEWH0(;m66<Ywx;PBn+ayxa^?IScNZ-Od%-
z-(u*~8`cwiT(4A7Kwa_hmZ+Sawp-E{X`dIn>%GqO)ukdP74w)gwQgL&x@&U|zE4~g
zbULoeak{(i=|6q~^L^!X&&K;&nf6Y)V*l65)a%oK*@|x6$o$-d>+H8Ka<vErWhk~u
zdl)iausdw>c8jT#n7NM?Tjb@W0_JJv6RMf_n4k98HTT9OH)T!+zu;eHt<&rGr!Mq1
zY>p9r(z8FcXi*#YlQ^NEBP&07w{0rCGq3cVTb1@)_2X?NOLnLHnwUSyE<<~5iOR|X
zTk~m_D+N^x-cI^<Zdz>F&uyV>Yy6IzEm+T_v;U9(L9wl+U!s&-*GR9O;>adBH<jZp
zOEITd#qBE5KSCFt`mcTxYko_W@j$>le&LVWo8Ov0?PSuwpeDuf<jsL8`hE<Ny99hV
z)Ecx4845inNvMZCmeLF1@DaLvBVSEjfSa!+Wrt@2)BEQKr}O-8FJ)4^;=oZl=b^!I
zi@l8^9Lr7i^B-d~6v_zk%-S?@{+$2@bCwG^B3Bl$ruI5+6NuJ45w?&!HN%-HKqAq=
zSof2}+KlGvM>)TyX((AP%MWXLcz=b(zJrpQMzxkV9L$1MTSPxJcQf~98k{)FpVVi#
zOY!iVA4eoEN_r_&ZdzRUZN+iHXM%wSCza2}HEquEY@e_sGNdR*c$fRtbE-)%B|qL+
zI^*D%j$GEp`N?}E_FehVWoWch)Tb@gD>3wamG8-W--Ldwd9S)GYEw_rqDSnmUraaF
z%n6yaJ7=c%LEg@ypNFCo;wolr7s-s%*9<a>KeS#qBQb&Lbsvj{@BstIH>&?-7AWxC
zy)kQhL)&b<pEFK4wQbZ3>6v4}*dXF&sCK?^qeZ@@-x;Z&zL%a(6xmjN^4tl52J>yp
z)#k-0iNA3cp7mja1J_?g=YFMaW;+s{+>Z#bPdTyhV}-kdu*4a+#z_0VyvzklZYJf(
zDFyKted=zBewk6T^3vkJ?+Xqbym;TrLZBxy&%~1T#r+8zma%v=@_uURoF8*(_Qj7U
z{Wm0^3An#$`Gvn5EdzK(3S826xz;Y7GW!JI>ZWw_Mk_VzkPs$zfg^&8IaD=P=uGpR
zAUf&I&P&w~gjcFRG?$n@Ykge8+v|yIT<soYd|Z0WWSL`KrT0Fgwk0-mdspVEsEU6*
zA8iu)|Eg)@<g?p-UM6yXx$!t6dG`5&#O2JM>V75_mUnNIY@Cwq7da`}seIoOoq|a-
zy%S82xV_C@ziMqK%d!9=`Ck{roCKp=oKElM^>J|av3cfg&3R@~C;ytDCz<zO9iP#p
z5w>-nO=g_)uR9UzM7)|SFXf+D5qN3Ix8}*lyw&B>YV(f;D%efUXEFOIv#M)@<x$J5
z`F`TJR|I|aTebUgz~e8=>>s{7;5Id@pk-G4t*wX4j%FSWJG=evO@a4^&g!0E@mhFl
zg30TtQzoi*ayljLe^g|>N0n#qj@x(kKYhQeuTb;Dqpd!*e@&0?JpHKmx7mKVyJzcK
zYPs22g?t-tmY=vG!*S$4i<sZ3m7kZ*dmACmDqwuMpi$Uj>m~-17pW6vIaBxiT%YxD
zr)uS{V=YXQteihB4%Kok7yVa~RuD5sV{55N@tLC<^N(unu-Pm!V`JyL#Mnpa-`S4u
zGHUbm+WlL|I-hg>1iml%H-yeIg<bQOY4V)9L!tFl*JtzS0+~#g?iUZvlq)zibj<!^
zeaQOYt+Q1@O9I*dWS#F6Q0HkoS-|0$eeA-;v>K(>fc;F*BPKeT_XQdinAGgglq|lq
zPcmojVHT}@hSM)rEiUui+-om4|CjC?!G_w!k$KY^qZ&*bXG}Lby@~A<Thq40#W^06
z&&^$T?8@SkywPW5-Wkr^5V%;Oxyvi#|8k)=*04!ZNweqfeqng=>+WW26J1{8*i7SZ
z_vO#{mAOC8nR6v{*TbbHTMy2VSRktQ<krH>|3xi3zD&Dl=r1RH#?)_Th_x|i@Tctx
zjuryfT+;(C6l$Hz%Uyo;?6>EMi;|0DCad4_WNld@khzR&?WOGQKQRjPH|9+>$xLg{
z+;z)N=a}c@^$M%G6ZF`2_uc*{kZQCn=ID`a^O&<UQr^9dzjd*=XX8JiBR9(xW-pUD
zB*8XE!`)|jr@QZ-ojl8Cm7Fns@ni;@v+pt%n@sL)pUn#__TOmR&9ZI(IWfOiKcxM&
zqR#8ro!(*>HTlZr$}XFyc{lxeU*1~(>c0GYrQ|ISe6{}fKJyaU@+U2Q^HbfYyYG2i
z+P-kH1DluR)dw!M+`&`+xnJ_RVfaJm$;+VdgH~2^(++o5<u2z-aa?Tww<>i1S;5t%
zKWk0R();aC?!3MG%tN`JxGCalH~HWF{^))DXZQBK@r=*Tcg~%$X=h8Vr`v;#F^Q^P
zD>BcdwsIbmng7`IT#m~9i7PrEK4D|;^!u&qRU&YEmhZXQGVCv22EW|e#L&liG;q#U
zQI<EAbMiVL@D@!xwNcKXQ@kQK_T5vtxSb*I1s1N9cvzm{C6p&J&$n^8X2ViepQ#Fd
zQ+eja$ON1|$nvq*E$eJd`qt}HA1Wj;PF>e~EJ*bD>^XjO1-zcW2z=f#;s2|iRwdD?
zGYn+UiipZ`GW_H{@@7YW&)G>Q9Vc#%?EWDT_9x?paN-?@r`$^KuKqb0UvTGiw8f%{
z9e4XqdW-&boA7e(P6oC+nR9!0#_K3P(C~B6oVF;{=b%|`FndE==ETNj9TWfc9E@HN
zQ&7Oc+IRA}%S^E{vE+m^?@MobxA2^tFnN{&L)OMa8_x#&R?Mq54ZUPJuZi<Wz(l$A
zXZf5q%=L1e{yMkwgT<Z2E9aZ@EqL{pm$z<$!aAO~y%QJhY!|F**ZJDF?8svwtph@S
z46cQaoGoGETdupcv`FrmDyeaIS?sO<9&P7bJ9ns@aR`|<Z4D2r@5v`I3ZEacNaS^9
zevJ4p<l*qZvi+2d=$hIk1v^+?cs!OlHS6odwgp$BSR*C1PXu4)ZMT(~t-a+?h0TI;
zMVI#)GZl{ol=#jPe;U7N#|_WB@+W`Bp0u2mz_ZZ3_3G<~Gvc@0(C}KSId75HwnZ~$
zIK01lMC}Owj)aBzcOE&(2}t>J>Q7>0%T0J{!M0CRK6;nS%mYgq7!JzVCQC56uiF`X
zP}MzO;8MYy#8|(@B2(sC?2AkE=&=$~u<T<qpX3&BOZCXjm_H`B3`G-D!?-JAmI+-;
zE?WA?R!Y2ePiSU`O9mH%!?Xs6DF-sS8nV_s%iNUaTXHa4$<MJb%IR2d^n`;`Uo6wt
zn>VB4q^Ygs8IAk@liQ}ct$XO5b-yW6e(n*rMIy2bj@**FIm;_f-rG!e`o%!q7Y|DW
zj<aznSWP;<BC>6dD$Czqw(XvaDlg6Z#k=VL+Id?QPxi}-^=_4ozj=duYJ14M8MZ-7
zLpLe;q^OGTe3muMkyGSFw4&PLovkTzT+b%9J?M24KgRZ`^vnlOzVwrK9w$CYKRVAi
z^m1)wfU!|u{I<3itTO@w7r)$j<WOQe`$FD|0>`OG5@uZS?^TORDtmEv;$lm!XA4qg
z9*4|*b+Dgjr+chNi;dJvwYlx7=aOx$GBty)h`mUb{E(dSazaMevdoq<S##3FSzdUr
zNMm2KjG^OYW|>14mrK~JhUwD|{+}haf?H<A>}ScX4gNly2bP|F)GKo{#pk5jgj-@I
zo$O1ab-t;wbqO07&C6nOk6OoeocD;~tC(0dArn7kj)y+yeM4V=h~iXSm0U4D|3lk`
zo58v#o_O<2EZ^kc)0mt*>jql^qg&=;-c8T@*M(HuNd?@y_iwI$pb+2D(C!BuClg|#
zC+u8)^`LzB(;JMh)S14erw1I+%F}S*U6C!fqOq-I>%LG)4Q`e~{sndCom`GM7c60V
z`f{!iTXaQtZDBX#!imWblF}<Cuo|)L(R{_4x!_Z+q~D5@E(#4I1uBJko^vdO*1Szs
z>`dWYsj=?HA(0>FlFXL2{ypEr#GYuhY@?9m|EC``4!jDQ`0FT--;va)Wofq_Z+@Vh
zQTHs`(ZeHUW!9pX4ZB!oEqdnAk$IwICFhg|&Mhy`R=qsCE%SWaky$|<c4Z9g1q`89
znpJ-;+wX0;)OB$l+r23qo!fa@=KoX6U)MAFX6CV);6(vX?z^;zJV^BQiI{srIN?x>
z{LY!hGC9&t>Kh}kN^Qz*zB#dgYib!E%NN1pVte|JTvT#gGDpaVA!A4P&6jaEt|l<P
zQ#g6mSL4$Bop+Wq)+N>IWp535%CcZlXy38vT_>~qG`C8<crKT*ZQs<w!q;LE@3t$s
zxJ0-xsH|mRVPM#^Zq-TE*&1(Gz2x8785JdQJW?jWdEe0op$q@N_v_i4s;C|z@JN-*
z(v#(s#fmR0-+DLPd!D;$=D}$U2d>qbPJJoszb2<K`Pqu6$6ja`&JFHcd{j#>c$)sv
zy2t!Voxk2`ICeZ1Ry?M5S3d2O@`<2#lk?I})-9Xbq*lH1&9rT*J{#0#P0O4$YvtK>
zn&<QiW*<vyJh#%$t029o;h)ml*WX?oVe4U-@>(KhR<_6!w&<C@GnYnMv56|g`<CRM
zT%(%9m+qVKL`_Af<Coi<El$fm%~A@zUtADp*SMm?i%;n&PkzSpjjgQj*1Y<4DT{yI
zyzi%8-Po8_^J&Q&#-ux{tB(ere)1?Xv`S5T)3fPW?@oynJQvb<5x4q9+mUY$7yjRj
zXj?wV>(U`THg1;7M!JcwvVGsIzO%=bL%JpU%%W_DRo)?q1zWUz9w=qb_;A`|=3YL1
zW$wqbWQ1cbElp<lxWCdbRw(96uGS&GP?ej994EaOew@K0y88FE`5M37jy`*Fly8CT
z`@7oWV&ygM<tbjvQqq=PNUmS@X!6<2ZA>Yd3`=HZy*$0EWyYZv!~T`ir8aYx9ANsa
z!#uq)Q))VkaBlh~<x8{Ly$y6vKiQb?(IECD)1=XOy5t2156AhFW?dC~C9PVtXxsd}
z9VefC+;D#BTRE#^b%x5X1!g~Q>67A`EBm!`-jiFAlj44`-Y)Y~mYg7>_{!CJ%aq@h
zK`L_DkCvtX-<){ExnFjR>$+95Zfw$PV5%wEoRqg^pNoY0sV!mIFBWb5R*)UD@ZHuG
z?o2F<GW;#<i%v$bnf118v%?KNKjUM5yYDbPVo9x8v7Kkm)P+Y+rMN8+n&6;vA^*f{
zi{5J?=d*o3C>{U7#^k~fJtOx^&?eRJRX6!sI)&W7?Q7M|2|e~sL$7;^_UVgdD~ulR
zFx9cwnJc(b=k%tP4eZgIzlte~w>b!8d`dRyK9X>{jiFxhG}n=t``*mz%RIx?z_eL&
zj!}U)OEF8Z<3+Kzrr*!Knw0$d(np7uZ!+n#qq@s9nAs&i7_wbEVk!LU{#m!rk65^l
zE3KUQlxy?1z1OtaUP)(NH27ce#^CIVn=<bTJp8VHlfCn~>fE-T6jsGck5!UaUJJ7}
z;qrC;@*}6=<*aW%q@;w^4R4CBc33L1%R?>yssE|e!Zf4TZC4hjR;X=Zd+g9)SP<}J
z%PO{WN!r`ev^gcjrTrV0w`BWlUgvNmux4VJxz#bh^y7ZY4*vG58vBp!Ki0-HwKyl>
zz_b%(*6Lhryc?5md~vC1Pw6OKlldk`B&XQB-FsJLtlphM5l+!irKE_ws<T`B_HFub
zK<oIu9~z6zf~%4zzN$!!J*IcmNa&i8(6djrK?N3p9+?gt4o;QTGka?5H`XX`&bp<;
zWNmEnILE-fIHxVlG=JOVq-Rwp3^P7o`Cm4<ZPsA}pB_h-S+AoF9z-)dx3W)pbXF!K
zVoCbSg892QTR+~G!5DhOO+8TesQklX+kUt83m-1(S>nwnTC{(b?f<C9Hsy*36JPE;
zcI%|VmXATT^VT~wCavWP30QUdu~Jx_UUvNKNAcyyLZ_9aR10`bXL~WLP-=6K_u@z1
z>_1-Zd$;~{q2GQNzxkXkzfBsim-t%jcys8;rq5e8y}!2f=!xD3&))hq=JX#+6N!`h
z$;PUvAA0*|bWL@reb%!aKFu3Ws^-tbkGF*<-{%nYIwRA*&}YW!5G}tOhYDFN@2``H
zUGD3^*e<kj{fvOgjt7)Xk1qauZPP!$156Gt_54@5w7UF1)we8NGQ(h5Th^?w1|zeX
z>z2(<+R0qJlX*JB4DW(;f!w00NkWU<FP!)|^ZSqAhguA$?RH==&HGUC^VW_TQ#kwf
z%*pCDX5s4*`D1u+&zz4NeO81<<}7%6O>5KBm;}#W#e2t&bv$l4b>gy&lzDEiMeq*m
zWRpvrwtO!o4NG4a7F@}UbClXY#ay)h`>&5je;Xco(q<C6OzC3L!FslK&;3WI_Z6PG
zRpk|*UGF^MrPJwUE!lkv$9%4DX;gQdDWG=tcFoz@o-+(8d?kKAeQopfy|f&Adx{Ig
zR%@f3qD8(6Jnae9Em^wW%f97QOn$gTYRNm9&fu)h>2kSyUE?nmulZ2-zx#c~nqMyt
zSH9Udd(*tooP)*gLf=de-(Nf_|MHZ!unZwHgYDCk_OrTOdt=mI!(6_TIat&5_2RVk
zf7~P&&RiY#*Q4%fc7YsorOJMnUx%lhIecX4*OxDjTc-SY$#MCoXx7>`<`#xVW==5^
ziwQjqElj6naw-HK9cbfMmRjO5F+qrPHp>>N7ZVj$n~Adah-7S1T0L1cx>1B<Q>ye-
zMeB7fC(;!BCb6EJkag5+IfI!U>kXlR*U7uhf<!z7H4>LkYUEnLHNk3wmJ2J(>bSd>
z7qky<VA`%_P%ZYBiB<TTUx9Zb6T6=54vrU07h4+kRI*O{@#}j210z*7aW$VEK97#X
z{Ff-cVU&1^MN?>shQ}m@h6aY2%nSk?3|kx(*n~MWVs|~rxV+j{S)#*XYr~C=<yB@G
zzZKr?-}V2S*xlJ?E=ExctVADN`QgwP&Ej+Of%T)$&t+Hw+W961AAG~MIbS{^T;WH@
zy^Z`T_5!6%ES|@<IC0gc9sKce{dIeb%ZuHE_I-VSKeJmz(@VO0&FdSpUMXDR@>0<L
zvMNbHn}yY)TX(Zc1h?~ciw<o)ty2#k9d0`#ps-Q!Y{m;c5&l<;TKIkmop6<4)i}r^
zp1I{>i=NMdz~e?mCI{WcBA+Z`QCfLpLNn`)RTtJUh!hwZ|6a1k&#_6QWUZ18>w{he
zmK}47mTU1}`Y)uUdom`Mr*VdG0*@}s4VAM^i!Ayqjb=noR<QZZ@>$!K^H+d{UG;7b
z#g1#YUnopx>Fuy%{1(g^^pmASM5D2xD&)dnheIpX`?E5SYc$%$uF^d9Ci{Z<`gKO7
zhe9t_UDeqp7I10pjvW(LH}8J8s$c&_mFOFT!y?W(8~L9;QrL82#;P})&NOv=y4hGE
z9ITMVZ56DL!|Rk3sM*NCYsS#fpc|7J-~8hAgLT2-+s<<&g)8(uQq53oUUDI!yVc@W
z3b#G~tfU2;Y)lago+}vL*!P)b?eu&1B*YGwUiWfVYScIpP;#kf_uU`b{}sB<C(rhp
zrK!>5th;Ke?iy2*gRd7<{f}&zzH-*jL#oD41i}S6SvGbH)rvfDY5TM5^m<8FrKkmQ
ztQ{9K#FOt#bWv!uDSV}HT4K`)zGV(Zht=l)nPq&|LM}QWJKXKPG0TJw1y0RH7n~;P
zniOunb#+s##M@mzLq2S|`s-Tt!D7!zhx|1;!ZqUSuZkpaH+>iBPd>gsOJiZDN6(6T
zRl4_UPC1*Nl~9?<BXoCxTeg(OrAzivvB5oo|2r4VH{g4<-tVh)=89=Ozd{<y;)D#I
z&EV9zu<8-(BrndeUj1yRi1^TKrYO@rQ-5_$ky~;7buD*}TViJ@PmSJ*1#*!mH8!8w
z#L^@X{pQ>qacNTpp$YK{6=(Nt``6)q`~UxUI+>598Nz=FIkLTwX4s&fvPj95Wn0L8
zHmeCaDc*iAcaF%gUCQ<Un>5EkEkpjLNQ>3AmE2DjT%P-JhC9Pe#f1(c-{v;2x!@&K
z(Dj-n)yn-Yd#S}AZzfey>mU^t)iu@~@41q?r~Ks&Wow)%w6SU7tWsa+=*^)W4k2vj
zU(R%OZ9M%nqGj*ms~VXGw*q^;4k#~v+3N04)YyM&!!hBV4jz%aMcZ@C7&RJmwcm6v
zsNHh)cwXje@0~wSYq*38^#9?l|2tVoq57}~pO}hoSMlVBuS%A$muR<~bbE%Htdd}7
z1H1B9&ZqPFJ{f#baV)If<F`1-T;TtOzV-`0=PWH^|F6}&MLD(7=3a1Esp#{)USYX2
z=f!kwTWuisxrbF#s$>WInnNnjs=Fp~C!0;sIW_0K$qYe<E!$dVU#QW}FKbS?TX5Jz
zcj~j*tQXJT^ia0g!<eV=K}li8smyg2vMrXZJrAb;uvBYSS*VmUfr)K?)4^Ka6z=l}
zh1I#6)LRsN6qtlNN*zwfhQ3oNZe7LU;JtXFq!HH=dF4)C^E9go6PlLHknZrw`Z(|I
z!X9pcNjF>OMI4{4*v%|5;dFZ!%h|-Eg$k$dBn2d1S+rDkYNnsJRBz3ZkX2J3t_;@>
z;<RW8-k!kGpD64xMUQLQzKxq#%Q&aFm?+NwF~u!OWvBeqLnpTv{NJRb^z+sN#~G8C
z87`Tmz|+bRx6t^|PPSD_<(?gyvTmv&Qb$U27cBHLm=+<m?u)8pi_SZ*O_!#Hh^a1G
zJtaTqI@|w>ZRPsCchz)U7Dmne7Ga>%@r~uG&q7Y_nl8cnYlT*=`5d~;WcJ51zwAEm
z`?+&<)R*c9;TM(f6z=@2@QR5yi|fbQLQC_VA9JoVe-}4&JS3XG?P0H7@Mm$|4bG*S
zyxPYnMXXTyb#jvUT))yQE*J9-Ua)@j@m%GJ%@f4bJeZ<a98G?meE8>=WBac^)OU1S
zBy{qeyQ+(cilj(@Z_+Crmq?#8VxcY*Q@OV4d+Dx<w_@1JS8m~IspNCjXM?e29>;&Z
zoSgw-Ur${9*k;Pe$n)S<hD@^Y&*DAi>p05!C$;?F*Djj(Ymz)~kzeuN9bTNBllN%+
z2=6EeTK||;Ra8Pru0HUHU(KnECJymk%^us765^gZ3TX&=FVxd;{^%RPwpw#;%qr0Z
zK_3DOc6M^kn6!0XibwvgABz(HRz_IORufl9-gipF(UvLWn%Tmmp|@EWO+5;hEGX$Q
z-w^du|8bP`Yzg+|k3DW(%3s?*Lr+&V|IrQgHyN#~3Qu;0c`)s;R-3&mVe-<h58Wyf
zF_ZUPTjIFS&1KQq&Ih%zJ2nK>u`<3^oFlm3X{F4nBgtkxsxHgc9-dp_vWZiMb6c^K
zK+vA$8XNy78+3YIIJ$>pYNkxsv%tTXuE?`y9<e<o=>Od5`0B71eKDmUyMOY^>MJr#
z{WGaWY|Dn$9fc~Kli!sq-<G-_xFJbo$&Di#T<^^Gvz}Y4!@PQ%k(`BV*pqpCJ@Z}v
zuv~1FcG;?@l6cBm^3*}8Q%S{UDZXioB6}z1Em2$fbLQDQf)fnNN;_gZz3ZBUZ@q01
z={g#F$y7C}v7t+({M=_py*suJ$EWceTa;_6SiJA_6EV}7j16I5J2DUShF;lP_dNf|
z%5QqI)91aFd9dS#GK21vsR|<cujgF(xvpPXdPjIt_5J){`L5V!MhnWU|A)+6_d-n0
zO3|ES^2{@Hnz#=<Nm+gR|9WxPoT+i@4LrA)HpeIB+I5B}3(nzGWc}p4a7~}+g}!ff
zDgT6wJQ!wWUg?f&`l4L9`GmXLzN-`8^-imq{`bkr`^iib4BB?~9twW5fVCoWea4$v
zv2&`EcdR`fvh7~O*2mYUPul+})N#`HwuL(XxjC;-No^JA;eLC3^866%Qyn+tZQHqj
z)gIxQ$awzFmnBO?Qy%?4u|~OPooLocu`GUvcW+)DST3?6V^-Gc*L=CAJG!<@<SKmg
zow()cm98^$mD-}O`7yW&e9>yU=CJn4t(89C)S28Ixjwv8i16on;P;c~#V)0!OCsy+
zc4V{eXJYXAU^U~6Z^7I3Rvumo?Vr1P@4x!*YQwQQ&%*1bTmSxumF+?NUl-}Nha5Lw
z^;Wwescg^Mr7VXuoKhK76<)ts>rtR(WTe8lFU{#hA9MZ$KJmPmMF~6%uQUoWxF>Aj
zQ$BIRL~oA8y3_~*r8`|aSn|F1969#ogTBOybvp0=a2>J`IJSNltG`|#!xep-5Wc5Y
zol`!%FA`&!nw9@Q<Ix!}eNo1q17+)!kN9kS;`}^rQBdEz7cW#_vR(KZr6|t2K(6BX
z%dAF`ln#lROOJlZs4dIh@l5k~$l<pip80H0X^2?d*mK7C(Q@+~W;4HJv%1%F=LAOm
zQ_sJ~`DxdC;~T6TOA<sDE%?RLB%4&gJz;^VLI?ZdBPIv_8!Dwa?+Vx;!?}L<F;1l?
zoC@DJn(g8=;S*BiFOuf>zsYuJokcEB#CpdWIdOBCniu(aT|2$k?H$|8(^F0?o+shL
z<+n`vz}qKFe5ZymsXL{qy*O;|@kur6py2Zl{NhXH{(oTSPhvUFuqUEX>{?<_R-<eL
zBYz|NJhqK$o&r0a3N-sSF?%Yl?RagLlCzVcK`>S47~j&|MGccUo_{*lpd@|l9T(FA
zi%(VtMZ#~HbuVwaap(L=t|mhcry7qGy@#8MxS0N5Yqm`I^eo^bm+!07Eq<cfk2VG5
zGx$IM%9C{Z&YP(j`HaF#9diN%uj>hv7HGycALrXBrRZjQMD-e*)5n7UK65U3nw)%j
z!t3&CmPKn+8r9j8G}=G?lsI9xW~tHHz`0wvwLBj&-}Y13s$w$bli9wbTCJD8`GPF2
zGh3P}adjA4nw(Hj(O=-EIO$wJ2m3KAZlxQav=Zj|v?v_fYgOoF)4$n8m^0<YrX2=f
z{{$IazC3lEfbH%ATh~=z98N3jd#lj1LiC58=J8`%%M0y8HJMBjc|Lr2nAAUkElc>$
z6Se=w6Wwy({yNtGZ>{}7R_%>J%6eP0w?FvA^62Rahd7%l&ay&E|C^L%9@c#KN>`aF
z;hIa4PNBj+VXHHnl#VpJT`uA@<r4YYqodj&@*^m~=a6ztpq*f`XtKLO;-cG3TIxmr
z4eec;L~a~VewAooy=c!eZjR>9n&EDy7fu?p7+sq;F=oO6O{tdMPuydV9NBeloxtC`
ziEoaXDKYpxa}$2@?A^A8M{XM$c6giT2>dSi?A5Jfk#Tr-Mz8O4O-11%#mSR%Q%)3c
zoHKv6-e2IA!h&K2j;uJ9LTB?AZq<jM$Sk%yVw|`{g;6EVcEN_PH;P3g7m5Ys=lna*
z9lA-4?~GmK!Q?l_fl&wfC9ga#)6v#Frfz1cDSky^Q3AJ_<0CglKb>_-2l9Pu7hft{
zbSW=r_o`1?n~lEQIF`Gmp<vtAbuXSJUSe@kHB~yaX@iLH{2%WWm$wM(ui{Y<EmRS0
zJEAW9-6w|g|IrZBCQF}<oBRv|*De0vCBpEZsm$Y%*mqM#_C;b#L=yXzQXece^EaLl
zw0QmxmgqG_xn_xKHxhYvJl*l^qvl*A_VVI+Gp^c|B<D^mwiG_+ZF*VY==#*+&&g+9
zV|pg1wNL&zX|iSa)k)JX#6DiN@W|Kn<DrI;`5SHpGJGj6Ou1lb*k9Pwyii2_MnZ`K
zkJ@ClSHiklHsYx){>)RWwlD;W=J;<_GB|phCxAhDQNqjMz#P*%TqnhfrY5(PdNHlY
z=3FFZrl?-})#^!D{$Ecg)hkiY-L*Gg)m}LHQ_bUfD*Zv6OfjNN?%z2wUpucC$;#1L
z>Gqea_;Qf&)?mHk>B92=(>7c)Wn9w0^JQvI4u_hQcc4V~i-vPSb$y%8Og*#h?N=Kn
zFX3(F_ngmNYe`~p;yCRheVH%m!xz(er_Gt!y?>bewlHQiX%y+uNIrQ$aAsko>s$G&
znn_>2ykDnn%xUzF;p=+=&-%VM0jI@Hp0q1YR$$suv_NnDI=4oi2jO+HOVwE#CcHDR
z^DQmBpzZp5yZMRGX&g;?A6k?RPWbA4op?aS`erxh*RaH#<PgqB1*-~{<h}T^=B(;J
z@0{{Or7O>>MDuIbn56y+^GeDO<xZ0Dlt`1lBIkIaHS$K4yRg>Epku41=BY9`?+oR>
z^wp~8oAx2oLe~%lF>xik5`QHoSK0sJmV8Z0bDf&`eptQ8=eoA3$WmneBhw7!MQ)x<
zQ|~-}8KiH2?sG|uy2tV<`%guPimH}$oU>j3_(f8<#x2HjL2Z6^rH5fZDmLn0t7txv
za>GopAtLb9RhA@;#ShHoS5#e8m1cdJxU=7Ac2T&wo6o7~7X6<;TnlHqG5vwaa%a{>
zzw9bIMVm5APklA$Qh1VTC3(I5@K;}_DbBq&rcSMhOW8R!v$BEJ!A!jUQ<d&#p&JTf
z7K`GxtPjfT_I?o@AJO#var5oZw$5Db0Y##Qs?)XzBpfb__daCb(U<#nn+C&<{CWef
z4$)sBrVb%D=BzwwtYuQLerxWO;OO50&Xp1M>HlZmIx$t`-Ss+lrum)MW}mGTn#<F;
zc*f+0uU0G82zT!kuKt!4Yq{4(+(ubx|05lxPtE==?TfB%R@0N}N$meG^!f9yv&u%t
zZR<WxTe6*HZHmd7Ns^Tt9xs{n@m`+Sm9&z#Ayt=Oef;<6*c$GMA`{IPZFJ-mT>Hnj
zQd2Uf=ljk{p96MmY7%L0&rTP%+A@Fn%$d3~RynH`&->;4d!{<y^l61AG3$R#J-lo6
ze5LN=ml}*;SiN>msH$98%^V`_uON|OyX8Q3;EpEqiol4k>mE=3nz7xM{pQqf+)4~b
z=HCu6yS8{kx^dga^FCK1=L$1K-%eYwW9sFsnT3TW9qkdm9THX%|ND7w724a~R^SS2
ztv@@v{jlAy4@X6AwZ6ztzVoTjvpFJKG2}zC*M6na{MjmOISId~K36;XV$v3cBWvm+
z!xP1M7Hlg~5U=c43^SLPoOgiJc;af~%Ejj8O|f<5AEK5u+&Q&uX{tqc)E;J!%LQdd
z9O-#neAnM;*3JK{8(i7EOjyTs_C9~hY2t^t&t6;iaaLXOZqEMP^+%@VF=Scl+_p5|
za@~U|aYe+U53AAzcirE1c-NZj+$9aXIV<B$XRHb?;(mLJ>*rLJWi$6PEb$V)eMJA%
zM&|H3i4^VoI}2{F?<k%6^^fu+o1IEr)3*5UQ8M7r;+VCrrYzRb&@KMOKJ)G7O8<ZD
z_}Sa0A=N9md1});gW{SJt<^6+JuZ8@a6;Aca-XW~>EBI4cI91pC;7NN>O<j9<t7=>
zVGb)N{&m~2P4yDr?z#<vwpHdy_giDyCdZtboupT3$=hD=gVoPs(Jz~0?c7Q6({fHa
zSr*#n?(_e7@$IS0M>g4+|7rWSI7~lw|LJS1xSvMvzUA!vcje2QtKDs{*vhVUF4>T<
zQ&yK_?a`~X&Ce~;_S)HTJe`sJ+00N+e%A)|8Mf(<T4HsVGVGpJUcGAKyBmM=bC#FP
z?$&56mARFjJa_ru4+R#^<reYtq^4|~;L`eMk<0DLo)0~<YgFfNS@l;r<-*+~J8y+N
zEK?Uz|Eu)>#r0=f*J{qyy~}@pzY^yQsqM?JY+GLQe?kfO)rb^7_cICZ>uqP1i67e0
z?hz;x7*IT+>a)Z|^>03Z^Um5En{II3?Ozq`P`ry}1&d^1Oj?iNW%0?kw?`Z-npge!
z0{`7(wm&cGRtL#WS}iU<ZR(CGhPA7VYnyBDT{^vO%9dMp5t3FM+I}bc=S>&$H+isr
z83SkY+UZNTS+4k}utLrzY(?vl$*=D&HQ6>LTA(3!Tf($oru$p(&8**dc;eigc99n^
zoO@)CswN0I-1*Cv`sT>x%3B*Z-BOjFx1B-fy`=g)o5k1b)Rp+^Y$dr47ag1YchQ@h
zANi9$#J}I~A^p)aDOl9Ku;c$db1##kgDkHyYYW4qGx{Yy+nW~)OIayQ+~$31ySddK
zF8R{jM+TkJxz<ghuf7{(GoH@(RP&G8`E^I+$2qBy5w!<7*PXfJ_0~;Bc&2e`^ONl#
zR|w|Y2Gvbw->ZHrCnsv2|F*wY7PD{Nn*98D{pITB=ZrP0-``$+QGV4``!$_rAHUCD
zpwszTvs__9vqF>atNn*gJQY%4;gIpzu;5@bhp?7N$AJVU70yW>f(C90tDD3lId|+R
zSa7UQLA=h!qDa}TWeLlMM;a4Cbf>ZAPMjsNY1!FUK{d6^ip}bd44e&Iyj~?4LYzWN
zb0!+43D5Lh6145h%ZaN_tzwGoJ!Lha_u&7v%qIm}vWkO!H!~evWFt|kpt(NReQWN@
z!gUNQnQop?T@kkS>VET!Tmf7W52m%Wa~4VN*zw_jCbKYiM$CjF_fyj(v-M^$B(`WY
zv))oVA^7ZQ8OQA@HYX;fdR?xMtW?#w_TWw9+LYy2ZmbI3dy)0_)=5kU3Y`>~C#v|b
zn6u!L%3OA)PQ3=J^DQ<L6&O~@YD{=5DDT!&5Yok>8*SDr>H6$#_UxPwlcLUsK0JDh
z>Bv1Zz6jlw%b0$cPN)}^43UltR;`#G%_G&4(46<MU7${6(bp*sLT#!u+WK-^7o@N?
zu4rXY<|??sqQ<3g(05X*n<zW;K?x!88Jq`Brf~(P&Hc}CT~x(UMEM$<aQ*6w;R10|
zy<Ze2HGFjw&~z435%&Ef8WNM}{x#%IK#Y*--9S%W(_>;OJ%_laXfKsq+RVM|iCRmj
z-ju-bh^4NfZS0)e7KT={&bSa=GVkQna4A8p19e55858Cent25XDf}}PoI2GqYC_5k
zH#VV^HrbMhB?_vE6LeQZdI@N+x_WZz#Ox>$p}O2B+XJ=V=89zNeq%5X&^;8W6~J_&
z@yg8&OpYxwMT@k=bsTRievoD0{d(fD!!_I2Yc@YB6qhS`>bYP_@RBR9mN;(Z2+{IT
zR`m>6vyJ!2I)?ZAqVLyyDs@R*-1X1$ffK*q*R;J{L0^~oe@~LQ_WvN8y3XndalM{^
zc&_ykN>djIO?k=2xp2jVW6X&;0^)OKt(vh|kjF(Xm#a2}`@qvxQ4=QiZ8{R|wDwI}
zz-p!&m%NUKx?Py#AI)_kbvmblYHN_!#GNN+n#PB4pW&MN+T@CMDr@lm97*=CE9IX4
zz8X?hYH?wnX>09^*!+{MFPb>HW+ix~J=fe6nj*eMbb{tDsi0-mk54>uG7S>Ve83X8
zJ%A~2O~vNbW@jrW?~ytATJO5F>dULSzb|SAtP$UO@l?*<ziJMr)wV7=>9B}zi>!xz
zczew2)1tHQyt-@4GOsM&?aA8hFV|j|lYTr&Z0m>a&DSPAUO7){z2l~}u0ie(=dBg^
zKhOVO%U0j(o7dVNxScNhA(mZi&iq!6+n#|xMf-Xa?<)2NDcn@ZnlvNfMeMH#Un9LI
zg{gXBr#e%6PsT50nKV;c_n8}~reFHBqXLX3?uD9M0jHxr+q{wWtX_Rm?z6}u$L^S3
z<>yiz=PItPSSZ@6!Ku(MGHC(p(u{|iftgW`r(CV1`trj26lEPtB214xWUDF=ihO34
zvEUmkkM5EHk!zD?Ot6zpG0X0L6w9@A=N5y)ya=9vMjtbCp|p!?ikoNr-LqoC70D&M
zUloFUd%g&szo-+oQR<Q7qZ6VU4+~5dE5>S{Dbiuo;V$QU!pZ#ig@e<hb7hwsIb3#p
zmAtpI-S+eU$dj&%Qk-R{@tH5p6#Ce8+*2t|HCAO}%<d;A%z~tr$K9!x*st(K|E*|R
z!g{CsPDk9mc@2AVwlZ?p?Gc^Tcg0^pSxRu}5`km0-{>p;{rxCulBED=jJlPocFZd;
z7bU@a8zx(n+zcsrw(4#~&}_Rq;+{)B9h}iIA*}gL2WR^vCxfj8-1;qCQ{MbAnf79p
zvT2Nm-O13&`ASPdcQU5eW=%RZr7^`<e&+cIk#9%UmmYBr$my(kHOXtQ!ij9&zul2$
zTaM2Apu1S{ljr|m-viUMjucF93X)RS^vV%asZ@LE!D0SWeB+a4Ve1dh)V*?Pu0zrD
zvKvV$kM>QS_iao2Y?hgt|J|Yt9q)WNmAUlMREIyAhh?LR6%ScvcsNu}YBAB*XR((4
zy5`bBw}X?aGg6;U{2*f6>gh3^T~I!^;N|2#J1-__uQOsoi&=SHKCEA;xNI5Aru|~7
zPi9P4Oz`;UzGI@MS>ps#4xu`i2??_@xIbSEbr2R=syOv))0z+I9#LL<3a_%ZtWk7k
z^5YO;u}NvQkMat9K2v&m?AKX4)~+g?A{9LA?}c`cv&$TJz6#<iJ0BSEdri~27wVC+
zEQ=L8l|+_HYrMS4xvL}a{vwA}0!B>7Ht>F3_g5p>r*X=4ErBgJG;Gwb7A$(L7H!SB
zYF7HGvrpU2KWfJ6UaFWdRq<$almq{czW-Jy6<=PgST}2lU3$g372BG_rykL@SGRF4
zjx<$S+UWbZ>Qd09b3qANe{=RtyRl5D{{4|(LF*bfMB7b~>`7!lEzqOBWvXCq(v0t$
z=ck_CEzqZUO~IgPVyZ{RrP)0q29A;^rm`>{?BsOes?ZM3FEP7%^$e$)|K(MoDP>b1
z&+SrP6dRfAeB1xwJLBzlzHX`(w+Z!qb-F1tEo08xDce>YaMdb1XgrT!bwSO>{Bv9n
zSKM=Q@)Zc|IuZJ7MY7|ii2Z-Nv|fgu;P!lI8I<@nb(>P&lPfY8)*t)YnHd$(Sefnj
zC{=Wa`5WKr#=}=~+5)yHc4h_Lwh@R7zG27^y4gYW(7OLm@`_B8Ss#55`7?cs-u3Er
zEb-oljxYRkU~9+gV(r+MYBLv?UN@Nd=;o4h-2x{$Hr(dqygBiI(SegUmMB^;n%0-e
z^t6$|lZ)k=<mzne3sWU_A4y-U;QN|oazDqZKJ$mUtGC{{+N9iarQukI{ri`bA8^gq
z3aMG@@UKUp%lf=N|DwFv=UObcscJn8oz)S!M8J1`$9DtQzR4Bq@2;G7T+-y3qyG+-
zmnv5@EDr81y|a#e*2%s*%fd<vwd!k)g0E;y4SAb;v-xak8khOpv!d@h*R9I>64m@i
z%i!aOt87xsn>I<@ylQQxqwDb|!FjjBmDNkj0?i&b`l#62E>xQuX2QDn=l_ZgZha!#
zvnLhJj#>F6tm?~p0f9{219QCgS<RYZCOH4jo+lsod`v$ZV)V23$&@*}A9jS?Sn<Bj
z`lMX3n?+mj+|T!qy?oXu$vMr}zG{NNZsok-)loWqu3KAuyE2jmB6OlYuHL=p`y|F*
zhnDvjW$Y_2%6EUhSIkoMSwf|B_wQ+ecQ4O=#r^!?_59X?s?D`;umAbI+|l-_kI`(C
zr{<qBS@zCcZ1*}M`tG3*^}j*_gZrmly|3_H+~HsQy1G__Ki3we3oU<E9m{CE@p%k)
zVElS@qwiq>{zVFcpE4yDY4A=CoFtgZsu;0Xg<<vwP4<I%&qE_BjvG3Bi+DCI!gx}R
zT8F^@xf@e>zxX&A>uyT6_h*wj(k`60T&cOy<iiz_lm#ZKMQVYCAzNIdvw~WDjs4R%
zwRp^^jAbiub{FMyZ(u1;4B3&k@4A<ii>=Bv;h>A^5~{Jfj$YA*)!dHN4c7t>nP<p;
z)a+!>3*S-1y0NC$NYHGDl0<|+L%D!zbNe&Hkb6ZYjTI(A7o6HWI&@t+x;#3(Cw8>%
z;ABiS78h(aa|}IXYCeUdi1%7t(V?maH}_?mRL+Wa{V0oBm!7ooOW5VhNg3@8EE`=#
z1J!cO3y(f;<TTWj{g}+;(8Q#WB4(Vz<j~%bX5p=-%X-o2+(v=Q3;}^B#&aL`Fg-A4
zQqW3M(fWU7B8zId(M64l%No6N7s_xJNQej4$uL$iX2?9fl)#zjnQhcM{Y5mJV8e5c
z?6vBNA2KQ(OR7F48ZYx`D4v)uRGPk6A|u?Rn{N|q+LT7qhi&`a1?C*`DXI{Dezjb?
zL-~+|5a-F1YuCKHgjuV1v@<bIikLEyeImn$XZlPU41d|HUX}|?$;ejt==(dpp4qTu
z=8Af~$jMfjGS4Imwm$3Qy)GyxCe#)*A-^q$x5AO*a@ga?Vayv{SIrP8I@-0`B5b9x
zV3)IR!&JkHb^*oa2)W>(o-hH97G=Fpxm=BTmkI?=3F>ktT4+0Ua8=f@Pqez|GU<+l
z?2U}+S3hbz5%m49!D?-+VIo{8SaD3yT+`^CN57$@@%M-{bH*vzArfni%FG)BOPR&Q
zrz-e;QUAHsY|WQaH_zr*WmCClDwUc`lz2>3O71S+A<$)JZhy$T#-sJh41qhF6c!&~
zd+}VjbLQk9>~lB^C-2u3_|xvXBsueKxbURPD#@#ki9cdiU-3GAOiiORX|0BNe6py>
zAvuA^2IYadRl?J!s=9?XDedVHIFQk_=A^vdiIo1fAf~_`HbKT?J9=aptt&dGb2Rpz
zVzhqg;&DcaF?xX@dr;rpitP6u*)HV*drJG}WyteIN%TlAoN_Z@-c*6FtU-cPtAjt5
zW>mDMB;<NG&d<$Iy1k6$|I^Zm3fs$XPAk~Fvia*z*FQT2?leU*D(T%4S3k0IvPI?`
zJ1+s-TN;1UMNi$-ILlf8ea4b&rCpIL!mi%5%{5&btSL~Ynr@p|R<qDfdQ#j4^~FUy
zCr;X6c<j3H%ag1|lZu!`EKY``Gzy!{O?2~K=5Se}!aJ?E<%(q1&!yTTlVU<VE_U>`
zF1C7g&4J^ians5a){E1eCJS>WyKf7aW2aadklq@0Vg~!M1;5#Xo_?y|6{h+zw3S0>
zRrD#jpF8H3CC^Q`p;z9h!F^)_%THI?8CD8Pf{TP$H@QkEPF)?efMu^0V-Cma!m^@g
z+kD(2jg&IAu0~8Z%aTZI)bhDHNj7ovfA)=&>}I*$s9JlAYssBoi&-7l-mRL$6u2a0
z;vCE3b+&>wA9tj3eDj|dQq9aM^z)JY(i!<l;a!RCOW97Uvu;$}q~RSlOKip??d>;p
zJ5DrrYjiw&SQFA{xT`{-`=jRT%#@nzg2yD9?_>ywXA2y?sO+(@cXg}Pfgst7AA9e1
ztWcb};>1M(MlXTWmo{Gf;-u@?FS={N;+Gl%uOvdQ#XGEYUY}Lq;JvCit$JI#nb=e_
zUzY~OEG6YvWhNUpvNy_bhsbgXu!Kl6cmyuv3^z<M$&*!_Z_c=_Fl@2nN8>9Maxy1o
zZ)RHffJ^A_^=)R1+f84tY%pA+rCAzoK1JWOE>vmO|38lFuV;3#x-LEKHuYxZj&GOj
zqC-M543%E8Zaws9Q|Yd$W*hA@UT#&q+O7QA-ndD!*HNe0Yk7H<fNb~1mRTznJ(|{h
zYWj-9k~$s&owo!}e4OX-F=f)hB^-g9FVC7hO=HQo6)WG**sOH8pk#LT4((R{?Ouyt
zq@T{7d%JYXPb-CR7cTGU32BNo3QF7RH%2`dop{W#JtZaLUiIeGEdm<ak*!<i3l&e=
zd}-2?TWj~-UVBYyo#U#>H@TL4f4%n6tF<4yCr|Qfa4y<YsI(>YQzT!+`V}*#9BYi-
zwMD(|puw_Pi&+B&S&g{bmRqQ>?`{p=owd`darrJL0S1Q!49pYf$r?NSpEWyH-J`*v
zBPikbyxThjE?p7`pOnHRs3RD#OO4t3`b7b))hjpxd#(ryHW%&=e#(6yTlkRmEPktn
zLR>Ss7iyPSuZ&m{^S`RJy0ChWxcSCc5?8r}CK;|)-Xy}3xajt4?U@(ng>0M^%W3iC
zw?>2Gwl&SO^W51MvkCp*El~NU`u*=?ZY!5^mgfgr?KmTr%Mr13-Hrn}g%T1xj@pa%
zh6%E8E?}|UXg|%{A#;}|XW+DcRtK$!iLFu<ZL=8iDrFZ1$(&x*{#mkFGv(A;>#a<U
z%X_qTa$G#z`sA?DjMG<`iY=sqIl?!aiOZN?T)b=d+F!qSzn?DP?IRk@v&Hs$v?ovX
z|JIEQr~lmiC#a2~x$xOz&HNe(zk?f|DIPy2$nZ)s{a9B~-JEk_6IC;AbFF$PT^=oQ
z?bh+a*^{r#+W$fN{GZk5f9yX0*h=6_&Pt`tXDlA~vBcy|_YC;ab>Pa(9s1X%@HNZt
z+r0F=^wf$Zu_Xsi__s9*EIe)hao1I)ipWI*9ba=ZG}{HaB_3~+FwHrA$76n5lz?9D
zF2_IG#au^Pew9mkuV6eS{7rI_v8~RNkD7Tx0c#tMyqa?*f<q!GrvJj6G^ZD0>^vEB
zS9&$HwaKlJTR5@9cG8+-fxB5|TrxYj^%c)a^F+4^(Zb=J`R8+{Fvd>!!jt_)<G^b#
z$N5>pNre|e&v^ZB)#+;d-5?TsqVvts>~gzTlP{i2JT}c~wrt}xCP!AIO{_mBpDZ}R
z`q*3fj*X;()AElur%V2vx9-ltH8GQxWxE|&B(vhpO>V|pmy=HCN$$Q>BC)?jbVcA9
zF*B9ZGIO{TR~D_ledtZyF~ioQznv4Nd6vk|^nE2C>v3`As+kGtSL1)QCT?HV`thjL
zR}qe%We<G>1QV{cGKzQ}(B$8FD0=mkXSs_DPm1hkt;(M(@cGR7zKh3yyxD*C*V?dy
z_Y@2Lz110Z#tQ%4d&TtXl8ayOaIKjVbm{`9jp8Pj<cd!@XTM!ocio`ZX2wRJ%ND#s
z=Puo3ay)dj#ZWZ)5R+nx!X*0{YkmI5a{79U$#(Jj%AeJlnEKGU#v*LtWsBHNTdb7w
zgShv7xc@v?>v(PS(^K1bO7D&O>{`cN<egn8ZK`-b$B;)l{o+K4$Fb*PU3T_edt7I8
z$xBf*_rT+byeYo6<|`Dp?8sJW=$Y%6WxHG}DeTwWdTl?pK!t~w8ZA%U$P{Bp$bBlG
z8j<KJuq)#x%M=NfJb^hrPv`6So|!d$mF)^Wx#bpfC#pq1G<<hT=Jxces?*(GD@6KU
z9AB=t;$r=$=SL1l?zc_co_qWDx3x3Q<oKRFQ|gvEvqn)qZAQ-W6xSO$uQx>AP<Sd*
z?RwJx+H%vst`V;%C?~JU;*kgmSi>~oji~0e__zuGFK&6dh4;dtpXa~6**(RhlhOA6
zow&Cja^D^^EWJMEcF<$TcdM4X{B?xsUO=7i>v!|kM|E_`Ca&VuOKxHe=lM6~w(0YU
zN*z)?`xpvTc~}m6PxCvs$l-F~;bVd+v#NcBEzQrIlQihr*vGS3=bdoE>PLEVhYjyO
zlG}T#Dx;@ix|QI<iXa2ij-PM(9zN~2+{603z<0WB%DYgud9I(W%uAE+R(@($=8}BV
zHbKZiR&!n&!vY4Co8^zCg#_nI3Z9?$%I2e6=A@SwPw%~YPwTygE0@g9qE4Oo))Tc0
z|NXe5vF#4Os$WOUL5__(&U4#6*c85C(_LA`bG?Tis^<tOc|;UgDE|-k6KFkG?)(3J
zZRErA+ixv-`=Wo^t!DmRlcj%j^KZL;aE0-mTMb^iUW%#Dt@J!<)3(ih^?T3H<#}f^
zL-fo~Mo;Q0u=m>|eeKgUBV#*9v2)x)DaE$m?+M*M8MAl$w<hCo`xmn&Ok5qw{fTws
zCf3R~y_4RwJf0Qq->TNBSYcT3`Sj1F@6LbY`TH&V{!P8VbGRgCW^LYocb&t^$|~=(
z>+7dnd$Whv_I<&=fGs<o{FAbEyV_X7C;24vQ>3<*?(JQ*l0Uf&zf3>Z%OL1*;6NiM
zo1fy0fQL?P+=6^}E(JVt=@J$c@yQ5iU}+L$NKmN=Ty(5cUc4)$<I)m_$$}EQbaoj1
zSM-|29qV=TNSd%_*fy3cSx+}Vy3qAi;+m4yfft9Ca?Muq_1d6$F-&M7*9xnUjF8Sq
z?M;uQrX{Qn6geGs=ZL}E16z~C=ZP|GPrSV%(|qF2+2Z;$byGyV6gvvKryLBL<P*XW
z#NfHTmzCwgj0xRGPWGwVi3ograG_%=YsI&m$S)a}0)=Jdtgo%GWnAOEzfIQqO-S3@
zDO?+LzpZ++FyZD7Ra29Sh#jft`#dAeb`-UqnW?@+yFp~;%d0EjKi!f#`I~O=HlA}n
z0w2;iTb6oQ^vqHaXmjLPmVRdT0*59qsjPiARwf7k&X+8D)vx$9EXs5550S~48qGW^
z9Gq=gQW>g~{s+vOpvcbd^wOy{V6ubpE1^|O#iDy|y<Kt4!(^kz$}pEzn&ROqTt{0n
zq)(Wwf5N<Rk#wWiiPb`qOsfsoH;FLlC`nH8UbJnR$_m{-7oswDw9c54H)ZdaSFhKp
zcXnkM=r1+k*}LS*VMDE7Owt+$7$)DEq@g9kbaubFZT07~M(>XVFH+f}(9si?b+T9C
zdX%!sLdW!!j$;c&T&Jvbkq^DHLR{4Lis3?)*1+ReQml;@nsNNv;?ZQq>ZhdXKUFVO
zbN=KL7Qst*pWF4)V&|vb@06T67RRtUaBPmNoSM{?aL~pdbbjgqTZMB^wDz_M9Vt*z
zP^r;TRNzxFUc<?9V;VE#^gU0fGN1XBa$*UKNtc5%r-zckk|Iv0?>8Hj78Kl@`9}3b
zOr`RY6?a=LSQHpFPZ;Q}4&(cFQv7(`p4<f<y44j5ZPI@=U5i+6c&%AevumP;ruwgn
zMSa1dp$jLlH)&fPU!fAY%dA3aX>Xv%Oh4~^haRu;{=84d=Z5Aaj?1>jnpPhk)tEmP
zRqDDmhhwuT=b`g{ZExNzTGYV6p!k!8RfK_wL5G2XfuVuPfsujZKLaa=$c?N8s~cHa
z<z!B5Sa`TCS7Msrfu{=(1c@0(si-!H%nW2Y$?;@Gp~B2ErdKzUZsheW>v<EAIz@9r
z+R51##joz1+`RmJ+oOgk7R?DO7y@~k|B0;l^5Vd<$82t*7NHDR+F#d%#qRPtu&Q*8
zOK#%v4bvE+-+DX>slJ?cXcqI;O{FicEjl|Xzw})0iD!xNbJpDZw^eg;$m+0Muc<e0
zPYCurF1a&A#Gx(tNT)EXUyKIB<5Wgr!z>+%iwiy4C$le`=6yNi_{756JfWF0A1~Xk
z&3aABblGc%DzVMyWIQ4od}hbia-Vrsm2iF4g}tw8?g^w^S?BA0PRb)&GiJ`InN|Pg
z{o`zc-YcBvzVXUn=a<lHtS0Fe=gez#jm~mutk|Tya^I32W#XPk?}pxM<WPGNtlc&-
z=$KkePGOX@(#2)BT4%*ZTx?maxnhF!BHk&IsY;VLq$+zpy0|%hGr7X2J*y=3LHRMk
z=MQ@=o_WND-HZ$k>3<Yiay9f#;M3#@?(5XAIvmnmsqFvo>WbFvty_}2Z7YAOOlmZ~
zb>c#y@<ovu@zS*?1vHbyjvSj|Vih9btFkpHrm2y&Ekbj-quGRcJRffcyH0v)@~A>^
z>%oV<OZTRwb-A)eTvFqa)jE(Hxh?6mMv{POdT&W!Y4~I}x03>ar&`6QtoYQse4d-{
zO63b$>&*VUcuzibcG6X=&0JUJGHkcox@02H!e1)>tNE71ZRC?&IxleXnx-7*=Rc3F
zsa_uxx~kAujs3BP$;{Six<zIdnX~5<S;hNBPO^Tp%d&Lm{2kFPBI`ZX(=-IcLo7@(
zoL5Fpc@h2cU&yQ^@!p7IZ8KY6$hyn@eIi(|cc|mlve|Z=S)RUb$KM_E`?X?Y=yRhI
zxe2a;t|1eT*|iv*65H6bvE69X;~?#?GeW)3etWVlBz{uW%?r_@zwcbzRi7HN#g$>c
zW`J~?;<XRS)lR*^sZ5_$W=RWwi+Oa(`KBs^G|y)i&5XXAZMP%eUOChy@tigHSyjR{
zk<b&K=iVxNs1(l;4sx{ppDlQQ<*&8hA27+z-gJUra^1y{D`wv2wI-XGrZFT<SS0Vb
z?TWL2`(l|>g4$V}9zR}imEBZmW;!$b>$Qm+_x;+d_G)c`Gu!*!?;a}_F0u4B=o1v)
zz2%cu&H6VJWkN!2=ZSAR9r)BXu0-#C%lH0{$4hEHDWuL{HL1b+%AQjXCa+uPyuHno
z&0Dk4TdtkwwCA0KLh~o<E@iRW*6;i%@2gPQ^XW<5-1fw%WP=vvlPp~7%32|AcMK-D
zI36+Kx9m-q3z|_Bx?FvE;At_YZPL1XmS`+0>fJm|$%1udlEMz<g+F#(6JD&*9%<p^
zwUI@9ZN}6!ZMSzn@U2o6D4VfV?#l~y+y4=UU0N^OU8gh23Z8ATT6pzEopy|3^0z|^
zw1P6?#0+oj+4Vu7`Kz{y$WO_)Jf8&Ezoi6-9`b1w5ZRWpwv}hs<(Uj|OE?W3)>&S#
zjNGv7(JUda{!k;6X%=sGOb)a;UCAfN9<-~-=<Ux{R;M;`C@u`$;b6t0Sr&7(vTy0h
zLoX+4b?rX2iEsJg9vA&~y%N9KCyXZCHC!UU{*Al-3q!83TW7p<e?<juF`K8ys*@0?
z<M}_wNaB&fTZ7aOljpN+@~JC4pg8A;(wjpkSXUMDWN)0jY;VG20Ta=sriF}moj1Ig
zuyO0coXJ+wQCHL&d~_~OIw;bic5X4JvyiKZ(VXQrtPB48*KzT+sb1!A@RYm|VrkUQ
z{4>ExeAoFlv2zih-z;~LoHZ{<?239_jfM?#=E)~VmMc!rPW94Z@@}6J!ti%RyZsJT
zfhh~tx+Y|wjr(?eWyP%!^-vX8%}1{1GWlA%w=EXvRNHinJ6F+3`jvQkTKdFh7M;f5
zFO%Mw7%RoSI;$*`8R!&su`evgiMw8V>7%?Q0j*Xg^V6QHJ4;UrJGJxcB+E=~d-crY
zyR2N+Jz1IjNXjXExx%^pQ<M8OHi?T!TzZ?t$JobkX4C4nFNdsW6+d-Ic-dd4Q&=9l
zOUR(KNW-G@v-N!0_&X67m+sr~K6lB|2buSeEHip}$=fws=~Ca}El2)G1T*wyKJzc>
zmOA<2n)8h<t6aA&S<!apsovrj@hrD3G>y5Ah#7S{dD>ZU?U|ImhX16C=a$z3@A!`F
zKKAHc<sR;PZ|c79VNzT#v1K-AU`O~KjVY^R9{vufzjS>)&&%hm6N6&AltfyY_Hsww
zP>eikbVQ!tQ`3{ZDWTD1hq_j;XKd6%?&*3Pl~2}+yMI#JHM`0}*;3Wnf9uIz23#(s
zf^RhUt6Z7T%yLyxPj9ia#EY2PsSDyP<oG?+O7?~H>MBPpjm+b^bag$~g5y6$J0#9<
zg*BQ)GWfqcIeE5E9Lv=2ih+LmRfR#Ry341Z3jX)WVTI1qUsjRx5@L&-F4u$~6g2w3
zn0I21zg%`u+liYS?OwXtmpKccU8s`&^ry~+D$k?*(^GusOuB6Naml0h7oOYCU*YI{
zzjfjLDvupJ4-dI`JwDr45vF|R??$&Xo#$?<q^Q15IV!QJz)od*@$U6$?^jDQ#ckMk
z<XXTs4;2=LyRqSw*LPnt+Hl@FC-HL>!|r!qH%{(-lg#b1>%1Jtmf2_DWJFirWM<qZ
z)^v7)PP6FsdxqI+*G>zE9jjcjzGTB(!9NM5yQ3N2rhQ0xK6mouW!889tvKG=KVxTD
z!E1ZTe>z3V=`WhfRhF;x{t;5X=+PA(t^hT$10rTE6IRHnD6w+}Ub0v$ATa%ziSn+G
z#-0XerSw|^cl=L|&}Fclpyb|Q6F#rhIYqWOR(bcy!_Uf8HYo_q{cOW+{oe9Iqn^N`
z_ck|N#2L<a%Y8}vZgbmbx!D!TOC`Mp)e83EyRL1X(7Ciaa;??Psg2%A@8`Zbcdz`J
z=dJ5;@2kEnmfvTwN%wM_S))wvX0=V83pG6^a4jp-b$vZa==<YsrlF^-8B$!60=_@K
z)8%ipeX8Mglcb_Xg9+-Br_Il<J;dpG_LyF>$Dxf&t>0$E?^Ri<?mtcTQE6L6?5?%v
zPi#r#XT5bh?qz_6)t1cx)s0{FDenAUdvW8r2@kg4`*AS;+a=?lLV-Kl)tJkcHc1QR
z&*}Ml*IcvOe@lN5|J9zQW{=PR_nP9tbu8w_A~yNEF@Dbl#rR!HC%<&sCw0h`XQ`Ug
ztiX?%LB%28l?yc21thCHcU3tqF!i`_^~0>ip9J1smgjjGDyYDE(=2smdFtwR;mzg(
zLKdaGg;oB~1@4HkURYLqVOkZRV$;d#)w`$FsB7f%GqUPG7p^K5xpkQ9*|rEP$MlM4
z0#+-^m@YP}GPdw<w6^tVG5-|t(~M0a(73zRJAYx!L?eNO&B8H{Tud2VZ4`xP9~CH{
zF5KZ(@w?DR=3;Qao7_aT%=hUIS6N!0K4jCMCb;}#iz=hvpJqoU1&0YqSv(V^*Ep~(
z+Ab0)R&C>6wfK3{bv1^14F(Sbfko~eO#e4_X#NoB-qFG1*x6+vFl~myPV>&`7HKb4
zxn?bBxO7n9dU_et!P-(b?^P>=S4X6;v&dgDBVkj8LPmRn*^c(D54az-Sj|0L&XCsn
z(%j;WdXmyNfm<c@EK@S278RV)5D+#m<W6^!2^YG2LqH*j&80M1NnD6sSkq^#)AoZ!
zx?837#XF=Dn`{M}C0^t#p5FL&MMoe5s{lhM(}qp~1qQ(lolJ%Om)TgSRP-NG7x)>`
z-@c+$M@43f1HWQwx_W4>`NT3uj)|%V%MMyhR0`#<+9qB3F`YZJy)4}^SH*8-Xe-;L
z)?*tJi<F|hf(yl5d;(J^c?7o?EUZ|f;P5di)AGNNhUi9CnFEtW1br=b`r18gYg&}4
zz0-h4F>Aks(1axEPmJz-jIm25bj;h)w^L0ZXhKIwWzM4M9hbLFt!nN&5Z=F5yi4JT
z?rH~ayC(udo@JX)2&;>Bi)z|UIm)NwTK;>Y<?Uec?~|QMiq!rm7aTqj{K-flIAMlr
z;S9coGxBfr2u-&VG*av=s!U-CWfcs$G(*E&P2l{NnaqI=)?)rVm)ZC_1kN^=@Gk6o
z6(Ohdqlx9Gz#j{N)tU?pp0gQe_6tm??mJN`ry}4r!}-mH2>Xhz8|>ZdMe-dYYI7^|
z<qp*r?G!oxbAtJ!vg%{HJ{c}r4};V)C$&#<JoeP|yRiC~|C{ACJA-Ws8)_VdgFPm%
zZ4mUT6uvm6hq<sV{e*x_Lyx+efV4p&TT3Q;ldoms6qdrkD)x4Tg<|h5^_Ud1W(w-h
zJuIka>Ygy8>VDYNon?(NCp!YCuwGsk8+9T1T>G@$PGU<X{Q1ik@jskk-MHw;O2KnC
zYrkF-*eEgA+$r*_1Gh_{Y9MPy!Y9G`huexREfm!hBUlPQix-}V5On7%?7AuNanoc;
z!$Nn}%5o34={Fq$QyU^~aZT}*ysaTHFS0>lx$@b<0I`jnf(wmV1G!Zi7ur1Rd-I}W
z-}KpEEBYBYo3}{TY?jRSQsdg6Df4XE1h<YL6R-TXSBu(ib%|*Toc{mPt6Z&x=|jW@
zmX&8bSBgfiFkhJd&w+nmu$WdLzrjjjwM>PlKh5HlTg5_zm%Az_G+VVDoU|e;X1%Gf
z$w&L#0*Q)875(iY=Wa|EImkL!d`j{bVHZ`6XelfAn~Dy*yuCU@Ei44KeXtgiNLzDV
zAnHQD?ZN7E*Qz2nq?Dz2d?=Jz6Toff!oS%i^24#N{F8I+KhDw8%71@s(Us|os+w0i
z-tzi;K=;`(`F#O(??1)QS}w@U)bo9pu(OnA?>BY9<$|2rg(A=A^9UyM1t#Y*D5+19
z%-G^9cF4`j(R^;Rkk5_s>%Ro#Ud-BaEqg<G)vd3NImLFX9dz#&8k-xhJpOaq#{U^B
z?;WhQ*xj5bxnArce_^|D!SNNcD+O~oSGukcoRGEtHP_sUtMb<!44t)9X~nC##WR-P
znX)SBN_k$=Jo7*k5$)Fbn=?`tOzEFi`G?83Z<^HipLHUFnZX+v-B~pzmMgpL6rL8i
zblI#m`*tkIi}q7fbWmelc)nGrQL$;>kCeFZh4+sY-(g+!aF@&$gLRsME+1YAcimcK
zz%~8LEMb1J>2tlUW_zvUyB5mvX_4WnEoU_sPZmwoVO{&-8khXRZk0rl6IFq%-O3!t
zlU!Tp3r^Sn`N{Rb?w$ayr92liE<}VRg$S5@6Nt4E;5j0>uAs8nspy(E!#qy6s>jo3
zA7ER$bJ?5!0Sgn{J&!Po9b6e<ZanAi@?D?RB1^W->AAIec6R!Uw9TJ(Cv-^d|COD;
zszYq;1n(c)(o3H6|9il-v_O?<f_Gti!V0#0mN~mE828(EOuRZ%{;@`%{lY!5(F*(%
zRhJzVPnfZlRZ4B$Q~^^-^+%PhrbgRLXRZ$XT=CsV|M$wCTP&fE3%4^lE2vD=C~t4I
z{5Vhk<C^oPYPM$Uo@z0u&RQpMW?g&LPW#y_RG&`B51+HXWB-+QRhfgeeY5iaJ>c_w
z!aXaQ|6jtcV+$91zft_TEx$We^Xr3r9wp(N2h(?SD|i|&DBH8Os(6Fi#SPYlJ)c$z
zwy&Ha<+JofW|5e-P)xx575@d@?pG+Jt`gUtBsBdpmyxr`r&Eh2U7Zlfv)|iVZiV(1
zzv!S1OD24A;LdaRUcF$KP)!$y8&}es3I8_uePcMrHb+v6F(YQ`fq%2j_+$3_tl7Dr
zXL0bKL@m|wd0ct|*(TNAj(2YQ?z`dAY-VR=sJ!$bSF_``6c0I-wr$_O&#>|IJGn<B
z_ri=nMuJNe#bp?$s_2}JS3b^MXt$7gs^F`jnq|WCqlG@L+L_9J+IM!|g4z2&pX%Cs
zLFd&(*4xWiBM)$GX<<p4aOCfroj;$6zRc$T=D=07CGXpTWg>Tvu1mPcq<C!UgwxC)
z`D0y$zULf0aOa5TU*Y$&YQ5&HGmGNhZg{@?|5hRCZFMm(#AgX@h>)Hvu;#eboef@E
z4DMP2uD(|~I8Si4@0lQJ?Wf9cS)<qDp+l+z+pQUyH?4)9tVw3*7M5SV%-4cz@rQ`L
z0+IXQtSFo&t;@9KufWBX)w^E(>MANLd$VBe%nvC!!QNXHxCAC}9*PXws<1+1YJOIx
zSpIYVHwVnx_NLDiU|o6V+SdTCw--*gNUhAhEBVgh!1fLDTsizp3l1;|Y%z6?n(4DY
z{Y+OuIDc}2CG*6{Zw>P-4Hfk~3rtxL$=}>+8LM}ubt`9~xUie}+sx=)UD2<bwly@>
z-=DKhmGQRJ#XVMv3ua$G=znC=X)d9&QQMqq#JZ0L>7Lys{@*fX>jHVl-xsW!R!)C&
zq{oZ<<BHg6*0q^EESzr>?mv%<x)A3Y!vA#vXXFCTA9M2;H?RaMaJ8T1`?y0p>CY<X
z83#n?TK=10EVuT-(gJ~R398wZx40$+X|oDVZ_e73t#_(4<{xv<K9<$dGn6g226O!0
z&T;7M<CYmsHcN%Og~OwI1ODz|OBPyLD0Sv-E0@qo#oIxj0=ZY+K2XN6mpSlBc+-Kq
zvMU+qrWdZ=uUU9$xrMif%#El9mZ*SRvrh5-)DYG@%GEoUGg*P_x)kfI16;qvI0HBE
z<%e_6oG?4^0N>QDTmchUCtiK9LiuQd&ZYk!ZcUcGHA`8_Z1d63m>1e-pM3v6`@y}v
z`zx0PZ7|_7F?P4o>RWhAIA2iEHrPaIqRIN;rA^hVjMkm`6db)Nd)u7!o~|CnmG1;1
zN@m!3`s$oBoOgKp+e_#ACe4>JWD9t?;OL8>jD^ycDn~y{mG68Ql6Qfv>;aeSyy>!$
zWrd#nUmrNLuwAyhn0G0C%aS{_dFt|annVH(RR1pEEY*AB?7_X&oGbH8_N)Tqmb*6s
z3ylBYz0h&CdJzNbCq1q-yQ5mA5>kZ^Ci-1m@_<WA_1Ig1OSA7jXUM(x>eZXivju0)
zUa@4xCEk@;DhKm547q%ig_R70Hws#(g$StryUZKJ#_K)faP}3C+AHo=OFmv&Qgf!~
z#Oc)!|MOhEq`P+-yHxX|nUkE@jQbm2GaE+!QgXd=q~u!ygUm*r5T>*TZ0kO-F+KQl
zl25P1?5KXVNY=DCXY*?_s&{T!GL6%6vqtG+kB!CUwp@27<|j3<2ngilG(Y<<%X()M
z-@{$jvuCk-7_j9Xzg~AX;qL`m0R^tjC2Q9tM1JFw>5JvhwC9?|T+W(fnUr&L@>A}J
z7;cAd)djry|5o4qAXaW}eDCSOeVmt8Y<$9f^`v6vVxd|Kx#k!)#U{a}1_pCK3T)nd
z+o^M@Afs6@$04<x@_Qzewna@|>UU?capre($!LY`Rn3Pj86}UtlK&qSpduK2{{ON6
z8-=geN-J&^$};Bi{C{V{SAi`G`Rr3VH^wHgr9NOw3wX@2E@(?jO(4UAq<b}-C(plS
zlUeQH(mVASmndtbz<q{?>*cf=1C4rxOjjj{gnj+f+0^91W6j9)%QB=x#miMA(&g*s
z8|&lMG-my}cv)#;+|e#3cg2fN=kIGYZ*_5%e3~CC(q<FBvE;@&QO9WxtQI*}SIuRp
z{?DQz`r`b&^~=|GaV@9_eI2$gc5fZ)>knEdR%>scwDyL-f_AHG^v$N-E0ZMsQnpM=
zI=OPu-JPYZQxb2s2`B7o+52j3#}BO}qvpS2Y|=NgwS%Vo-1KC(>m~**re!&QW2T%A
zJ3BkF?%$qP@65G%X1mwjc(zJ=TEqYEML)V{@^D_>yZ=<Kc4g1iDyC_6xyQF%yB7Dg
z=I3tJ2YjbrG9^4OyShx|v|j4A@L0V~;?`jzC)1rKbY8ph=yh0cl576X1EI6F^axE}
znIxi-^dk0;CYSH2<ZO{Lm!#I^fmtt??~F2WY*<(qb}?gaT*RxT^I0wnC(Px$zS~2=
zFk#hmM-G$34U2ewOR_e3WPMEEso`>xi7QNMkERgEqKI{t^?x=lHLA{8nQ=mim0^-B
zTm6<+(WQM==cWs_E?QyHt+h#L?Yb>@PMun$m(r<YaM|dTwtlzOX6^cIQj@0Wt`TbR
zSe?^&+3d2%#>?kUvpD$pAMFZybAIiYOWf=4JhKY<&v5h3r&9rDT1iWLPemy<o8I3c
z_4bm_`dd?&&zf+*4GK)YEv#exP^3X)gGb<2jV3wM5bd2>Tz(B_SPs1Qoz}5th1j+;
zOqpk=brrXsc=664dG*_zokvB(PZgZy$+#c1dEWFbCtj`KJlf&H&A6q(iCe@_h@n%*
z<Y3pdo<E=6g*6sMoL^+>D#h95GcEh&A`QPQ23#spu9N2KO!%?#^vsGysS3SZd7bkF
zZ_09X6>Uy%Ul66#c_Mnjl0`k?9vfF3T__Y@bVBZN$COE`ll4W*)~tQTYn*1Zvt{P#
zz{o@44Rf@cN`;M<IEaR?-liQIa`?@j%FP#LnBVMPqP?icXt}%V+yAS5?Z5gkc<a07
zn(XchD+#&fHSw&OUWGQ-+b*N8?_RD;T3c!ybV7meRo5hG)*}XUnKy|T`y_UnExBv3
znD^G7kEyGK=3lDxmU+9h^<Gp)R|*efMq;|?B$LiL3xq$4W~yt1%`CXze8bZsqKZw<
z`ANcSNyBAd6*_j$?^YB@_1Pj&Q$BZc(4lXKy4@<vr*ONvFf~<Aobw?z@WdH|)MHyu
z1ynGuX%9+MIj(g2+3d@E?s@C3WIFvgHj-<Wsx#weZi72J8s>BFHdAo54*nbUM$1On
z^~#1o>z!N-tf3R%GHg)#{kg9!#aVK9VXG+1%J{!tfowZ@F674ToW)?)syS`SvH!vI
zE+ueWe<o_Jn`Qj3n@5~MS=w`NR_FvHkyUG)x*`fy^X#~#&zm)BDVd%OXETZ@`;k6F
zPsN*I>4T{!W?b5|{z|h`q{a;Uu)xKk5hvxpRB-w&d9l7pbh+GX!3$AFfs3U&yT!h&
zaPRfGWZS+&o9n9sr_iG|oBtCueXJfXa=u}>sF?GBo$N&yTZcu*n+)D`F`bZ}6>@CK
z>NP$gJ~cO4*M}_get0yXTk32^#O9NW8gjLB9xn(p)9?&T{WMu8@L@uc@z%30y=|g5
zr8KoK&GgOH3fb7>bJb+Utg|yVXzW{aHuz0Q)7F)}!8@3i>7;I*x+Augr(&b~xhY1L
z4UJ1IxBbsxPdp)}ZZT6KGOwUn`%OakkwDjod(x+^6b!7oBTmP#FJh1WRJ>05Vv~I?
z<H`3bj|6TXEtdJ&>7?S;=-QjY?Ya8)^3H%QyX!iXY^{?!?SC;IV_kW)^V>y#wPOpG
zEb3U|@?dka`+22H(^^U<O?FWaSCG88SfsTk{3mO8PGbAn2ggiQ6_^c8JIsElF4JE8
zrc2$h`IJ-aGozW0j<9iFZM|5sh+C9#0VAtI=sFM4hBbdqo2ee1>b}$~$a7@~yJc$d
zWrk3rnN9N7f-Y-D<UKG>d+TeYy7JJ5s}sT|F-n|&_iAe7+tl@WT(^SO9OC;i<%-5P
zhDpck7{wW9EbMUG!~8#YMe2=-&pvk_y*zFA0t2=ZP6wVv8vU<Mg=#xaQsG`89j0&6
zXi&B!Md_f@nw>>K{e}mQ=p2r6-N6$zWow60oQIH#lx3CZ1U22I=GO1hYK~6b#P~IP
zvDCB64_5hX@7*-*&dJm2FE`2V%sM*t<>grs>#Yx+FL^fY5$}7Ajcliv_$HdhZ4OP?
zrWAfF<7mrm9j=6)otpa;d}o$@J$q+C=DxB3VQ(Yuux6LC?lOyO%yA#m9^AUXC9yNd
zTjZvWef%OD!!?h3ZfuTK^D|L<lG-S@cY*t%7WUnJGAp*ecbcwtA|g<1?(6Me79H2R
z#&~fC@1!NS7q1I*&=R}nwJcS;_i}K||781hHd9s2o-!F)D>P`%FA6!LqULWJ(dP3i
zb47RNeE;o5lK&_D)L$jy>u>%w#O(K!fSOxeq2`<CzFn~?H|lT1l2_Ncb{()hkzn|g
z!~W;W?_3AgK7886x?d^$|1a_8kG7BE1UtpqR=sbyr`zSkP&0L37)!fa%3;3q?ke#c
zpB`7yoq4<`!X;zkqJ6UkJ#~L1PimOzEwbIU+xdke*Hn(visp?>&$yKI?3+$2D}8i-
zcHye7?~55aUsoJ2IGnQiu4|C?J?l@WPaHb(Ztk!1iOy1b`)%`zg3T|42_Fb{fBEF7
z!?BAmR{NyhI4v5`xpm5lE!%H9rD_+>erBaM<t)qnzbXGOPGDU9^xawGnL*u;eTqdw
zw-|hV^^v>UB8)*m%z^V1!;CPI@7@_gCk`A@@R+zoLumB`|HsE3Sl`Zcs+{QE-KI2=
zb?(OI&2?WMYguI7?B;l|O?j!xnyVsq9(|qK`nj1YNm;YXrz@NiP5i!?(OByLx|`;q
z%Zv^dc1i#Jm)czI#3TNp-9F^h`KFCk#tW*lv|YQ641HRwf?X{_r^)PD6~Ay*;+yZX
zp1wQId$MXpetbLILDqP!DC5J2FKcHxEALUtlduR^s&|Yq4Q8laJ2kdr6KBv5o!VyC
zB`H;TD$`~><mnZ4C~@guR+W0ltEnbVWXpuk$}~m0?96ZH`70G3O#1)SUahp|L!%DE
z67vm{!yOmqmuS_T^qKeb{=9VyKCTtt$;{ReIKS=_zl7=6x>nPb2ftVKTjsxNvNxH%
zsq6F3Pw$sc>DA9-_L(;Kcg;@Q&0<TA-#omx;u3Sbr%GqBf&LF^a|fko56u>MbKiI=
zHdBLd;)#vV9hb<Sp42}{r9awMCvx($1#3D^Z!=9)TJ>U~ig3fPU45@l^qzRkv_@G~
zRD}1<C9XHkySiR5lqDOBt>L&AVdrPK`q&5Ubq~5<oL-;6b?Ddz;Wx%hFJ>6qFW$X^
zYk@_vwAE~G7NyyoQVI$x=9{b5>t5)&x}t;Ez~sa3DesuL%>`B&I?oR|ICc37MXzkt
z|K6{(PEQuy9U;>CU{By`j;<dZZ&#X4{m6QtVvTvjwABtq>l$>=oK&B7iL3Kd?-MDP
zg}=8wjaa82Z1VMWm&_Rp_2_*vC7a4mPk#RT$g|&@igxoz>8#$_eK6Q(yTGh<rNvI`
zn|Z>$W>s-7UH4!K_bqe18|EhyR2!;oUTd(<iQ2brv*?yez1`eKr*C%ozSho|w2Vo(
z;hJOjw1<-(e`Ggbu*=Axm2<VhzsvLbtb}K9?#d2U$=jkf{ouT_43noMEn2F)@AJm>
z!fRZ%pB9n!aeDWn;p4932erARYaGS*a7j#2@i}AledAOgP3K@9*Tuh#wn+6#X-)P2
zqq)^s+frF!PxjIOt1T>+Ph53y#)7vm%$uBT);?f4=xpwIdx2B7qu!G#x*wdjXW3{d
zZ|}8UEi`HUUWwKSLlKLfo_-gV-7B`7TReO10q4cs-1QaHj%7)GetGQd3A10ys+KoR
zJSKa&7)phdc&*Yg_kJUv&gI&{V`sh5>7}&d(FGo=HCu|AJ@}+}M1}0vojAJa#dhD9
zJRA2Y>=4@TQ)&9)7MC#39FO1KMkkD>DKP#JGVv;z?R)v;q76&>g1zE1=XzdpP`oVm
zXf^xl8OvUCau($*I-AnJ#M>cPaK83OA&(>_r9}6&1=H3g%)isIiMMFiY7x=89-g2#
z$1ARIt(ko=<ctO51KuTpJ%O7R{NEZpD@4V|WcI1ZJ6vYbQWnWh8w}i5EV91eaA>tw
zkA2|gP1&bkIq5rHUhw&{Q_B~p^BFcS%Bt><Pm634Uew_=O+&{m@ceg?2J<^ze!o@S
z4=O%h!4ap};qYpkONh$G0<H}ndQ&%<{oCx2m!p!`bY#yb?S(-nCe7Ksy!*nz$3|;4
zm~Ty4x~1jpY6Z2!Z+gqEJ%T?T7i$SvU2U>A!94hoonz$15FS50?dG+dQ@8Kwikf2`
zv+3f)Df@IS^mi=ro;8K%W&!KI9=%OU$8HpO8F~2lWwZV<?u)viJoA9hxi3B?pX6pU
z$}b9;xlDwi%;s#@3GWi?*>@T|g^r#r-F$H6@8kcUJmI=|r%UL73(wIDix^IbTy<|a
z;$+s#v*z%o)e0wrzHn_Y)YkvZ`#Q>RlgyMCM?9tpEO@i(gnQ_fO;e1umT*5jbM))4
zU?JD-OD-Mf<MQ9}C*;%PK!G9~eo>Rv0?yvuJRvt%JxJ)?_sV4c5$<!+tVS84RtLGl
z940>e%vIoWps4uR(+l1LdpB*q6(Dqr%WscJhsjm%mzSSdpY(e=KQmOV)#R!&k0QgW
zT>@NwHP+@XQ!fi{4G^_5zkJ~0<rC(<J1>?jy|~lF&SUCA8L5MnyhnPqn-^X5`dR50
z8Fl{a8IxmIBo|(g47;&HSMY4M@RE+G$&U(D+@n_N&kcXA!L{6w=f8?hS558NqOC@6
z-*B1l^}3V7dGpGJzO%b(RRY)Q+??|Evi;pDd8NC9jFt-IT3_@yv}N^fkJX!9uXb(v
zvyJmri(Rmqw5@sEuX7)MU72>Li*p9k(o{Ql;Z>ru=B?nJ6`sTzwzJFYa#wg@hw9u=
z%Q;u&L>!wIY-klazUghRKuYJHs42FVQoD@Ixoa=)$lYhTwj+V1+sZcVVRhI)%kXt9
zPG!v=8x*fR4hiqR+Fh&b+MVjU%G>Y0j8YMA;Q9xi^QK&Tr)>Y}4ewr;OPzZovQ#g7
zC)@4V*>z^}R5ex86^T0x69Nuv?Yp@nYQK%6cT2GJodsI2ne1fsC%f8BX}!EFwe!Ev
zl<4&W9%~um?(g-r_KmzNYBELEzNKZ~*#<|kl(-EuA~#r^IB2`i-CFI{<^Zc=qm`FV
z?7Yl9ulLH5LMf}}o+zeO^RjmH%re*5tT~NmVSwuS4|mL$o(;(qwcDQBxo(mwTht|=
z&z?I%b_6aq&}EQ_KQNv3Xp|z$s>L7g>}ico4GIgGlkjBn1FmBUkDhYPz7-vGRw`J?
zd<xIe2f^Dni`-ul>1y%1@6u*1x2tC*#7^#OxET2AqJs8y%^xDN*Ak~Kxc?!kcc;jl
z6}4LzE$C%!itT(GbN}qYHlIiKo0n|=$>lw%_p<1<upfS1slMJ%F0Y+oB<kbX>l)W0
zlkE8U#_AKt|DQ;RTkz`c-6;>$7O$IVn=G4}^zs$&bl)q7f2{O*#AFuCqhaFqF4x+{
z)#6KOPhesEr4v(h5AdFKh&mY%wXAe@TBz^EhFjYYayLcXUGZ09b-}_5uT}R*w7LiG
zc>H3jh32gdcM{B>T}b&FBXViq>YQD|O$WJi?S!Z9F<+*q^Em#@*;!o+E?(%kBP$ZV
z;p@TPyY~gM9_}|_{$RT(E$;HphG#MVCQHR#KfTub<Ew|4Z!l>xq^T|ljNiwVAob|P
zgp+r#JV<Ba`QICJ^27bJ3bAUUj&m~E^(Hj^s6J@odrfQdg{Vs_=P*Sd_+nK(r~B#Y
z`_-GhYD1ph5s7=8`}Fpm9<^)1|F=s<y7YO3<=vZB8nW4CN7+2<(r7;q+s!MTq<W_%
z?o_^Cq}@D!u7Z%LpSH2K8=HTjUYBGlr{;nCYafOzH<_Jw_lc%jtLoiZfd``Icb<xR
z(BwZiuI2Noy)qX3v9CD(aq*Yhd0l&|aKtO{VQP~~uhiG9oV>UVwHI0rZx`N|(YH@B
z*eUS+glU;!PTafpt#b%0d@Ee!dp+l#f*fCHz8;tA42FVrnbQpP*K0f~EQ{^hd%c@A
zW8S304{mSKh!Zu-V2{@5Yg;U$snBRN=kbXLz0G0^T`kk26`NfpgG~S4?fS=ckN4))
zXAiE`y|}KnbVZ-9{=e93ZMnCebuC?X=bl04|4^S0?p1Nq*`~~%%Ht?F<(bX3z9rmp
zXJSLLPJcHvVLZV5>u=)P3rp8T-YWRk8xksF6e4BxfIB2mbPmhim90{%Mc?}{KV8=`
zd;M0?JqAx>6`MIXoV`_+`#;L=gtU^-TE4=pOg(OO#XVV~as>{&o7R2{Tzlg!%m36_
zW%oEmUkio3(Ef7m&V*HWwnkVVdsVVJvA-^o<9=F#Uhjn&O#9L}_lGsH#&20SL5FD#
z@7g=Pf|tVYfB3N5<M0U$srfVCtWMZ5Uo&Bq$ct<5n6CcwK9I=C^e0`WB>Lb)UM3Z0
zJp-8=H!O@AjC^&Tt#im-|G@oT@8#9XOO3AGe|EMgL?&5%^WFcNr>{@gvYq$S0!h7)
z)LnYtj=q|k7qWcgg10wB(!X)phKB?>#oI92@HWLKoD0zpdAjJsGs&gzg~M;Xny2^f
z9oPG9@4e355?!z2H~ah>(FanWW+>cj(A^?``bs_{JNI(t9_t&+3aX~LX*ZuJ;?*k_
z%l^V=wQ~KXZLhd@eO<|SZ~4UsH|)OH>oW7ZrAIr3uiugQ?bL#Po9%Jssk?N{w>-(n
zah-Cg+0XDdSL+t--4Wi_Y+8G-aqYWj&iW(!?Z2g&x^K?D_5L%<Ci&3hpo;Wg&2QGP
zxVn9YP+PXW=7XkRiBlAHS?_G<wy)K4zq~`RM1HqM?pDb&#}9wrwy^iKgjDPIH~;zl
z-x~7U-7E`Z_RqbuFIO^Uv)$68>)dv1uqpBK-_pIM>|E=WDP=ZK`ASaJrUdTH6AowM
zJn-^OkM@R!FQH<$rQHu-w9OBnc<Vud$odPO&ViSLu5~F~bb7mBvRS~wpdGBM8;Um6
z*oYQ2Td&wvAoaDkyNX-4Ijyi)J^btbif${$9>u9y%UcebiCr`h7wvLCr#*cspKRrX
z@(=4Q_U|;w?wgtzbm-}JeJ<rsT+3&!J9xD$yDh*kNMCkZQ-N9Gv>&fszw-)<94`Lf
zz4V^Qk*{6l_iNT|SgpGD>dAHOe);WY6PgytTnfu*zBn!7KtbUyB?lpk>YKLJ-ybqv
zzmWg0=<oj^37<B9=eqkw@mCi3#AH~ly=}<0>a1Ep&%TQ5Z3Ti}$@|(LubI5br?bL$
zZ}pq|FLphDD<ZY|)!*dqHQ%0m>$7{z^g@s8S8RdQ)9hdK7&HSIM5fp9S=a|jm0Ix`
z1WM-r`m*ITGY2Eni5{U7X_Fm{W=C=6Xj~9msT(7=ttfPvk7o2bmV&~^r_LTVD%Y{n
ziEQ$gWGvU?I#AKQ<Z!p#U5+;|PVvp`F0<RF^L0|%nKn)DZ7NYyUIZ-mnm)@U>QF||
zvY^%=QHju3msdt@O%t6Usw&RsvOFht*HMkg#+1uk3oc#IJ-)+qWt`{@*L4lMQ@@`p
zt(boKQfI9+)3?**S}Xq_2n#IIQjREyIl**3FJgB{7sDCVn_K~xDx03I<!Lm_DGt%}
zHe%X6Nnq71?wLW$_%Gj^bwgwJq!fv3!V#<IHiYGF%h6DMkgRE>!MejUPv&Rgtfbx|
z&54tqdp4f6T(Pg?Q)ThnrG7~|nT9W2PPXs9miK<%>)(6dNS7Hz&ClT6TRHz#_?r)J
zB0s)mjZBPkoHj4$<;vqC2@R!}CKMji{JNk)R4SCqO|r=FBAaNDnn1?P#uY6rUOo<C
z4Z&AKgJKmvZdC}}K5>$SHt$xWWarMiD;D?iZZzW3Wc6*Gz>urrwuE8!l_U|DXrGI2
zZqmFV$GmfEzE9TP?h<-^stspJcj158t&fzvQf~-O3Dn!PBfNgg!4)C1U0!ZgiYt&3
zRgcRLN>_~cUm7AFEg!Zb-8p8-(x_bau&LpTcwf4!$2Z({&G2O1)xLs3!tC3OsB}rI
znQ752ujX}nEjc0LJi+RTs&klaDAS_Ugsztkn@p!j&-cE&O?1(2zAIh2driO0&XiVZ
zlVOjUA$G55yMg<q5`)UnV9sSTzGr!*Z3wx$MYr_o)smDgWwYiyxtAU^=jYByjolvv
zuQD#>J$pUR)8N+bS$n=EimLDVVON^B$;H=PGI&B^!72y#aIcI#L6KS47gZH~+~6RR
z&b9C`Z=lA3W75x0iMP!%yK|wvu}veiQ%K;?#<KsbqC%&v@w%E3(Eo0u)2Xve95)3f
z@d_?6KD%A_NQ!of$3r1c@l2P=Ma%w%2)k*ooiRmv%f6oYb$Xlks09UBhEBWovG+|x
z<mJ7$gtkX-FBgkuJoH23dTCuwVNHXtOL^pOwPm;NF?mE~#---Wy5!;)bSx~xoAuR9
zhaUe66Ii%x&RxklC>j~%IkkGG{ywIy*P?c8yjU~iYR%g<qU-EFZ+nu@v~|--mXr%D
zLY${fuS?}zoN9R2rDWb;%fJ~*9Fh<9wM^GHoLGH!)2GwIsyA<PDr;OVVX*qYv1*d9
zeL!Nzqu2#4s;&1weicoNIMo&u`7Gd(5|=@xr<V4D7N#JdKBbfYKW55aEn*5?7qXh=
zk#^&s4h>^&zH=-=6E1&TqvB%3+s9&L63yeyuubFE#a)j#S1on%)-S#p`sd2L&5HWk
zu?(I;htd{qv~&$@7W65L+p+xl*$r`hi+JbF`n3G%R^58-;Q9QW+cl(|*2FwjR$APp
zVSbRq&zeuOM_MC=ZP5uer+}5+g;J9zf7z&O*YlgFQ;L26;U%jSmrM-uU@<V5zg0~&
zN%3&5)^w|-XEy4TiZ~>(?GO)|u+Z)6f(7}vF1GoWbi{q~>eAjfb;6n{ji&92cRY`|
z?x~x4kT>~|z?B1=53|no&htGewO8zl4dc^*N`sOUdqYap*M3>9?fh{HLx76pf04C<
z^=^_kq-?jkIV%a~ACvYu*pXy@C}B;Ko92aMhn}BSj`MJC(!6=nk8RNf564udz?r7f
z=KF<3g%aNd21ovSzK8jlhZcvU&;-5q<0`Bd6t^B&z_y-kt8Ce7GaXKztLuW6O*DBf
z#^czj{Z3%t4#UX{oJD5a#G0LITGX4vqwCGT*K+FHl^+%g_xgB#$(Xw0&v`|u&@NHW
zhY|;CwWisbtPI?jv8c`GfH9ZHO>f_&Q_NJB7MbaKNwO*_y7R3mvuj@LI&+c)+r3kZ
z3)ZySvRvp>-PtFiRs6z%$8!B{g;gzAcJ<xiD4+VF<h7MXir?D4&1*h{nylc7ec^n5
z*{s-0|BqD(`A&2>TXp8G=Q4vT&4A`pLUxaBP9zsy^*V9Nq02w<SX~>d;3<XlDL*wG
z7^Vbrd)*DbwjwLF!90s2@VcZjQ`R(%7bP4lKO-hA+<Cb)WrD*?u4zK%H}rDM3Yq7f
z3Yl_!(I%cqr8_<;&$+^<B$eqjuHwGBO~HzF`;I*<X}p0m?`(RqAvRt5W&CNO>y!F&
zre2)NRh4r>qUWob;ij9RiT~QS>tyv9NM@Z8Vfx}aZBxin-M>CUYhDI1syHxCs`j@&
zYAg`3HIQM_9R<mtYIjDSiUYw1uQK$vM6UcWQDp6p#~W@HDswVCn^%;?8y3ne?2{;z
z_Vnfn7qfKPyM+&|{yRTp;`I3=>pkt^B$<u69#$*wRMoBC`Sndosek32z@`hU7R>P~
zzMMKE^RM<QM(sHcTq}=7n=P5G+L-NoY0(CCt8I#mNvqY<c`U-DRwTW&?unn6?K3xS
zde=;$HOUUlQd4^aH(tNvCA96Qv*X95yep@6W%?YR&?lt2KYC?MZ_zRbZS#T)vjcpk
z%eHmiW>~H$p3^K+mdt+rR++NiMNKYsC*g*$v)h7_&rNd?QoCJcFj=lfH&@?C=t_ck
z&bn#pvwk_LE8f3gTewJg+QvEh_MMy0be)dp-#ur6<KFa{2YK$X9ZVPLtqZAnedgpc
zHKzS1E%oYrJeg`m1sFKD=oin3{vYPaq_MEuW@%YhOIE{G&xAWV`3*C5v-<k7_#Mou
z4yE7h$TCoom{72AJI}>}5c5L<^A79~QML0sFnN=kpLx(x*OyI_9OveIj5#Wr_G0dO
zW4-HoF8;k7RocN80xPus3!J)k`p1`dms*c)$W|(s`@D%YC1*3+G~u>WX=a&+cl_MA
z^OVrul_4IBy?lP|R56t*u%4a!<s6f5ld-G9`Lf3X4%-%qPV5u%Jtv%Wt!#t(HV(-d
z7DmFN@lwhUQUt?ToeCx{dD0SO#T9ZmEjjUafOf!DqlIBDVY=Cp{>9ICJ6HVma@yT~
zu6km|hKoWjudjW0FJ=F3zPXp7Mrzk5S1%vQ|2eN>4_dZye-rkoVO3~k;drq0M&25S
zIIpUU!b|sgdel74>MFQy+M@3Ce%g)|pJf*<;R|M4#@n>?)4K|5i9`E1`LxXn-ap$i
z<EZd+zC+G3;^8xny;0%pS6h7eW4}x4ku3G#Bc3-*BlMT<?9GXYe17$WyPxY{yS|ee
zviAS}bQmPad|dWP(d*OS`}Qa8Rk%)A`j`IXUG{<hc*@O?mYa-59miO&CHOj?Gxc*`
zafL;THF@oozgv#qlswVJa5qRq*79e^Mc&W}0`mfOZasCK>tG_qx_9!`ueYyv-p$Ls
zTXf@)YswK`yIFY~r@7DJ^!4Dq{Wt9Snbua>iz&ITTwA5qIR0<@Gxf+EnGmP5CxwlU
zI-Na!*H%uT_s)Mt$;CVhV!zyD9)ucliUod%e3lS5Z{_9k-YX(L4KFWFXGoBKxi(J5
z?1;=Y#h0uNOezkYHzw^``f!y1t8Y%XbC0+4pXrz09(^Agmd<-W&2k1SCquAjlJ*n%
zgFmA`M;<A@5n1x&$f|}T2R&unOJanMx|Xdx9=e7jwp8&+?LzHG0e{yj1v^A8m$-K;
zLS(s$$KjrE!>O+Eg2I6s58~(YOKw@nQpILe#um6B?xm&6oTE3NZ;hP0l52X5>&0B(
z-kOl}4PsYhJ>@+gIq&GZn(H_v=8U^eyR5;H85>%<_q5sS9=)QGxIM)`WX6BS9Kmpn
zr;igW9)4mG|GdCA<*2AJr%K9HnF$A5=Q7;WmhvwU<!JoTU#Rl%^b23fV-r0RJ)~F3
z$!wb^TeigE$wRrOHr-$R&(@sESrfmev2UjCQKeVM5>*x{uM7Uj>0oOxwPkD5WFFVm
zF)sUMf?^8h#NKItY?xSmht>9_8>dHy`<rmhI|si{44ucLn7d$!OVQ1+Clh)+m#%){
zUA^GRmK_rlKJbapIjZ*JR+zx?lnal<mYhiDa1&=-z$tgzT5IC{CI0O<@7zt9dGr9s
z&kL%1bIzPGyz+C(L9spJ+b;Uonz9({slBL8sWm+3;dhkHtF@@$apDrcS0z{Gbtark
z5py#5e?db*O5Rgok;$z5khvlrE^BHN%POX}PHgvBa%$DyuvnRecPr2S5S(Ja?UC}&
zSYE-@;0ec_F15)&x=^4bSadC*OjCL$=LLzr+3vAEnNtq+3Oo}tNcJ=lOO9F~cj`)H
z&N7>s$J{dfDoWygKKSR%ICeUAnYYz>mbD8ky1W`Bl@_m<BRoy%*u{XlNv9osU1pmE
ziSw|k8J_m+nc;fF?}P<QxU70}Qfu8OS>3*+MWJpwmM5#{%sD@`|69V8kS9rTZ$m3N
zk2Eu|Rjz%)^s6_JWAXZ{cdAv>oD*(uy_zVb!eH`_b-u6K>OvL;FINt;*xZ=Kmw9io
zJrxfy4CUT-Xws7Z9ydAy*6^}=YX}L=nppcTct%g`!jcK)XI%b=1Uc*sv$z^%)AVw}
zO*M%}viX<T)<=pwip_|b)hc+=TX>qk@F9<~(%#dVtrBH3wokqMG_hrBO!}+T<w<M~
z3#?q2UOe2E>Gq~|b|6c@+(+HX5=x6CTN@7E-Zk@Tz*a|VmeVR>bDy(DobeIqS;`{W
z;{EnOM5yJFY=a<;S;>-jT93YL-4(OQ-9p5>hUNIwLw-k&R$N%-n5){o;(o}ZWw&lD
zt32j=JMwt=iPjdS1t&FGTjzN-D6mA%x}vrwHOfk?RqyDOkMdnzH>YVnnO^tW@alx4
zIcHpWM0Q%W>@srQJ<UI%SM{&S|HVNU5?`Oqcwu<C%fKb?Nq9e(1IMal&$w4l?{G*n
zocy53$u-R-L#-{dX|b+_r@Mo?Xc415tN)pZTRB@BrAws3Zn}71XqmQ0%ObWjtn}s7
zu30>r!j|zY^2p$}(enxoTdFlTGh`RLFt7Bk62H<H?Y+Gsj%F*IRqlyhi>>8a%kxzA
zOylaJ&V>OJPyau1@o7e!cE_2j#N|N@4GUi-GEHbbU^iECZ%^OC8S0n1TUNN`{kT`X
zCcVU}MS0q}YO^CjTbnOVU@Y2lv1mz)KPT%!PWOhyMGSK~%Q7AnI$T%qyVJguMR_gT
zk;L}HnWDuJ+$$>M-{x{&GSWEC`XVNxD<S8<({m;T-MqGk6{};Uj&A<8SiqO-`;O!t
z?~ZmbEpFJ8eav#fiiEWc0$RZ=2i@5mnATj63kj)mIJN&<WbU0~eFv5}e|f#sOv1t~
zso)BexDSJPhRfH9k<YfWOt0hoI+L?Q_bFc;JL^`)4KXcuqh`77nZYn8YWJP26OUG{
zs$`Ft>2%Uh@XWM#MUC0_1hd!e5UBo;{F^c5-HyzQpIlRpxj&lg<#}@He=hOIovVv%
zm)MEACalYTI`f6(ua|8qecN<fuUWA$$gE=Gh-KH(_;&5Iw4%qZhZ801e7w`Hgq;!8
z+;DK&wI#g=E*Dy`yS!V~cJpS|)*~}G+K-sJH*94r37D@Q^gqDs&5`!lIp0LNo)*pD
zbWb?oUb?Md_Kbg8V)2D%?>(FSi1X~UMa^-Eah?Lv|JV<DxHZ^J+crrx<-k<2WgU}_
zibg8jJoVw+-&b#fBF;{aJ)!TZzLQB!UO!hq=%(}4InOhn_vEp@Q)_r9ckFH9gP<2P
z!x(NIP}9%T30S!D)r4~%a*S6G+p%aaQTwXLzN7X1536H&g|WZ3G0m88)-vhH_1G5s
znaccaZ%_Q=oE~vP<kT(KkFAV+Z6aMSfAzgA*w|^Xy`}EM`|zgLERB|zX&D#l9{%dc
z{`lkh<A=OUdHRzy*2=p-U$bCEVSvi+Y3}|B?_IYwo;|y|Oi0Umdds!44GcB^4MI8I
zxJ_{L>N$CJigfSKa9<y-;1zFQeDIX}*p)u<;4&fByIzdqJ8l$5oO6@m<~KfWKeJ8f
z`>b1z;w=p>bt&8xFFb9WxICN_!(Zh$pOV!+by}=d`|bim@rrdiGxMq}Tb{3amQui`
z6Z&GZU{CbS1e5&Yl!(vUo!pGwMb|#L+I-?2<3FXDD_3{vm3HVYVV2vpx9G{+N9!CS
zzQ#SsxU*z~^B?UAMzWHIa~FNgbl?|~6V(iKXi@$2z<okv=8;YQD;^)UI_AUEk}xNw
zec2J|h||yJM6B&qy8P(4-uI0=j;;K8X2!J_GydLNmK^eIp{JYSw@t}Q3ulLIyS0JU
zsmLYD<o{~tmFwcwj)*fYzj%-Pm}6V{LgUn(PcpkpLL!#TzZGLD$QowOVsqN~qtMZ3
zXRfg^25)-f=Jmvld-1vB0#of*_c7c!tir~=B-cTEiSpA`bEnqaaC_@sJUR5gQCx#n
zX28Uk8`9>~UOFyzJ*~XP_>GjX(~UXlKkqzASS@LPE~;+boi|&U4}N42UvlWpvKg$`
zyC#(Hw&6)JDXnJMq+tHMvq{9*^{dDNxt=`UmG4+3Tf!Gh9c7daU|72$L?(xmZDqgE
z$#qY7K5tsEcxuyR2hWo13v>A<CahukIdM|hCLe?O$4XBJoKa$}`+O;{Zu>>GCjzq=
z`)0dDn6yOXH2m3e<%r4ui;*HH79LF9>*M%viJE`<oK&CB^8&bT8(ns0n_Tt7yo>AN
zIt7l3kLn3i)~bEy-7UpwC13P%>*pMI3-f1Y$JgzMXvxaD-j$i4wmY(@Abe-B+KiwD
zYi~=Hc%FG%)O{tuxb?hU>xD`)@!*e(ws`jMZx=iLIIGVjxZpc`r0YwbGfxtZUG++s
zbMbuSk5iq&$1Nix8zhwf&$aSe%APv=LDU?p2A0_^QEY9G-h}n+3i+}$%*G>*X;YWq
zhc{^_5<DB<-g_6izdfv+H}TcaYb+N&>fJ~a<aB%^>GUD+Ji{c`hf5aB2w0*nVsw9_
zQTQ%H_2+i$e)QB&x@#!-#i1v#>&uom|Ap9H5?;BUo1J+ydC>}IHI};*4xU`<9@Wtm
z<PjomZ+!YjQdEg!gGqSr?nk#Tcz=|bY`sjZQj~wo#bcJsKeK5cRr>r+*}J49rSDpc
z_t`6Mjdv?^1-9=}WpP&6{y9umQP8wrOzB7tzi0BVr~8gGy)VlWe~|auu+Ya*acjTO
z8o7<p(JU2DnvQQX>XO?RG$SpN_s*hATl-u)HhlkWaV^dNuMXn^-|E%TVdv9ScqSO%
z40@xQ!7UhD^f_#T1}pdF;O>X7<=Tv7cWjXM+Q`oE{?T7{bEDlxu2=V*==u|PAd7W>
z5})%GkB8|y`X_x|@^AK5^>UTC2^Sm8W-Zb74GoFi@xPQsd7?&V_ThT#34669?x#0g
z`|-TL&Zg8i)vHx{n(GU}Cryia<dbH5Fdj4e)w5;&nJ-JiqkI}C^Y1zE-g9kye>6u|
z?Y6s*4|d$-|1rJI(zCtt+3td8p@N;K*^X`h9IAeR{j;7;^2D+&TvJY*in@8MFUshB
zpw=&C>1M$a_qT1jDpPwsZm4A(a5&?my@6?^Q<CnVK<x><4kdyPZVrx&xn7bs5fegY
zhB4i`nf0`7N`n!XuGF500_9_U5~flj5d|t96V>d~o=6C(I8IlMY1`wmj7PF5hAU5V
z$HE696NI?b3|F`?U0mR>HDjW{QH9`Tj>;~Q&RR}Fy0f==bSkv|=g~Bq-SE#uLy<F*
zbM~!}BOU<^>PB-6@`Mz^mov<@TK;2>k>jbD$#%uFwyZAjifxW&;J?=-tnNQShKGBa
zj^y)Or?^<B>1ghF?6HH@I`L1ALCL`u1?TC<R#X*bUth02T}yelh`1&fi;9MXf@1?i
z*>A1Dl^-^FZIgbVq?2c!da!-NLhc0BouZnr+1zJ_fX<-^yc~T}^=jJXV#aMe*RovO
zULP)F%JS3+-Z)P;Xl>L@t<4j?i(9u{D1I@$-D{#p!&5FNfyHP2r2HBt=}DiozwQ;V
zVlg*wQntc`9V@z-8D)$$*EE$ddzvp45z^w+^l)4+7IsV^VYT6;Nl85#i~iqener(|
zWPxZ{*2N?4VG$dT8i@F6X>;m%Xr@{}dX}WU>7dYSPJPW~6Bar5EHUUYeClJAbLvIZ
z>Pg*N3sr*8nlAi3X~{J!<4nsrCRZ*jI)6&y?A9)!tQ8{W78`lZ87r6%Y}ehSwR!<3
z%Z!9B-fazA7VvW`CGXs8z*X4o%VDIy>*bX%Q_gM`3+m`;f5W(PTA)hCMY%AyN)B5i
zrL6W7QLi$ad$@M5Ix5%cw{@wUna9ViyTabATq>JdDtk5Zy58eOPE+l6W!TR;q43aw
zRXsQ<;2{@NxkmFZtranhP1+nQ7H!K~DyO?7;c2XH#>Bq4fo;8yjvKy`Y@Qpn=t7b~
z^o{?SOXo@)wld$|v}lDFljfyIeY&={tGc;&iX7rno-1+o^}}sEm$xYNvK}beFi*mC
zg@FKXQ;zR}7_o++JoVL+4{zg(;Z48!;8DtA2kxfTn|ht3drg})*myH1`w1_|Z0)NP
zv%2+6xWxRv$c+@Hhf^-Bb>U<ab?D?&d2n*W6yBtgQ@p!+*6=ppW7Eu=ye#?ff_8_m
zA5C}1Pv(BSUH;W6y+8k~^KCV6WE`K@ej{>D&!?!5OBYmSZS-sq`qjN>S75=`R1q^1
z^>6N}tOcp>nwIr$W8pNMol|<!B>Tg`o3>m&>v<1(Np5#^KeEDwu|POHW+kh^>Z5Io
zBSP=JtdUf;?r{6>`OmtHMIcMKlC>jr@^ZD{Nk?CKc-~U*V_w~DxcAiYq8wF$y*fKt
zitFkhPwt$~Q+uPhZU4V?ec}E8mpVEbvhB>w6kqA#Uce-7o!ZeTs(6C&aTr_68Fl|K
zZw3aT19povybL^q&U`rXZSuD{eT_B79h`~D=0ajDtxYG-O^ym$#j}+uoXaqxbc?|1
z&vTZ`T~$0>QuSkf%vH|FbrK$RJr(ogdyk85{cv$T)AI#AOO8Zstrqq$o8+36%b}w9
zsC(C;P?3#a^{ieN^?q=gp=Y|RFxIE?dRLAqSA4*Fr9cJdr`z0ZeQn(PQ#2$rS=t2O
zS*mIUN**rJ`lMpLQ#GJ2_`lE3PfeQpY;w3}-w0Y<W3=of!!iTy&ED+-UslWu@S2>{
z6~?oD>iz{U&nwSQoRAW<cymP4lFU_=xBeAm&boQ#nNJIsgTTHQx;NIWcH#Qb>v*Vb
zqKd*Q1#Oeg6$_ZPK6k9L($P8b*?{SAV91encFq2TsDo|ael&SFJn7pI=E@az<8TRQ
zQHO5N3C?ha7ah!9GaPe1nw;6WjE6_7S#m?h!jQg8>N~YNm`-$@x&G(ff>TF4ZaCb~
zHCmg&J@57*jhuy@4n=t)K^;tMmP}{~%sI4}=|YR&JfYTo8a%O%f({3mmbI>I=oijv
z<UYoBSW|C8qT$4ssXI4KnyR^Gy+I0tPuPE<k}E}DFU6d^xa9YeHu0&dQ>TWWO`V-m
zeswmdt_oMmg9{Iy9pK3{T;sPmLGzY?Mn0nhW6+ESc4aB2$vj$&Crvz=c3t60cU#`d
z_VH8+1GlfZn|nx;x0qX&V^<jGgV6gc`{SO!<deU7NG?yKd7I89w&)uRMN~9eWn%&-
zo3)s!X$!7MF)Q+({2_%|Etuo_g{c?X`&hSal?~sKXRT+#b*yKXY+Bcr3w`njwX~R|
z538|=F*=H|aNRlVp~j`e7AV28f$OEKea*$gjVhahlvnd8>xaqA@l%>q9hn)ZmT=Nx
zqPcpywa+axwNnQiLQ@qt-497kyUK7+XTe?8;D0H?|F=vmS;Q^EJd0sb+N8x-FDa~I
zRcP8OCA(_kBOgwW1Bc75OYuuRnK5al_J<Fz%zAu2&%82`r{c>r-_|!7H8&peX}U4)
z;GLMn-Ew$&an!-7WhvrHLY}L8YoBmgUrli~DL%_oTr`uvaDhJCT<w4h%k&;fu$fEu
zZE%vvnxK>HvQ2r?+MO**thyG{6Sb1O6u2j~X2%JKyjjuCpprhRjw9h5(}#B3KZ<v{
zu6$XRsdWFbo@MA#rK@YbCNPETc-k*ZI-(x?>XPzL=|q#KJnA!de#(AUBJ`2-SUvY4
z$t@axGnc2jtVvpXrN(Wp0CUg<-I&P-S_4y}?{KA^-jdgU_f1<0<NqgBzT9tK&)Z}^
z^K$9(h_8}{nhICjIynyTuHC`GsaLE!lPBFiQbl``=F$m8H@_w(K1}H15}K;JL1}_d
zU`U_vhKI!)ORhbBmAgCsx0I21zRr=ufsR`gR2bJJiLx$awdHDB9k=JEJlo38DUW=f
z*ZnvwU%chqk$Z>4+}U~?>Snt5S#+vuiabr8c0}{!v+Wv(t~}K{*=gh<^!Q48?kpn~
zP3KgRC-W^jtrR)^IGQtOX3aVK=~DKhsN7`%BCHjsPfe=aI_<63io1ygGu6t@ZT@sk
z>h%Eum0K+S<z;`*HZ6FbXL+w_U1@EU&Z3u3F77laDB3LY?h30bU-u!`+Ksll2eSV2
ze9Usy?2h}w?`E;EtLso#*b#%rSyMFcYo6Huz-H&X^E>6rzqMMqw{7;_-J&OIuT}6R
z>4m0n!Npq7p2eb%R70l)`F1XJI=RBjx>@7I30w1%+M)V0t{s2ZWhs2W?8On;%SkFp
ztL6L`d{dedb$Z&mD6@SX7Y_Z7eIB$S`%qdo&ubUyO-W3po^M<DT&jwBQ)3_WeP`#5
z<PTHG=l<GJ&0=CGD&^{+cSU=L+MFFWJzSk<D(lbgx~ixAzFk`7^RxO{=XLILHQitG
z_&(D*xvE6bBM1I)Eqcha=z+r`aTR8!Xy+onzH{rkALcZQu47I<)HiY0>!e3ahT9tU
za%DX>%QO6MbWA~y@9~4~IzP3bH;1;q(zZ%y`x5bPe^~$5vVLx%4j-X68A+aIEdf^+
zw3$p0Fs<s#YUtOQbZ%kP<MKLo^AnHSS(5YQl(>U=^1RN>n&+n&utq^3(`d!HyfevG
zQ{K+p^KLc6;e9SksuM08R#~iLc2NFEzN?4g?pJ#M8d&}%xCp;GxN!-yt;fQ6pCbxi
z_*8EA`UV_Xd`xG8$-&BmrDvD;8%X#wNUfL8QWSpm@WDME6+1y?n~Mrgf+B_&RixH`
zoOYu9$)WTIPU=q{u~)Y(XYe&@%39i{<-IM%a9P5|ckHoESq9Ub>xwvQ{rC;`v3%2D
zULBJ^RV~>hg6~_Mfb)M9pDw9I_Drdd8`O2mUPqpKWT)cnT+rM2C)>HsSMglo(USh%
zQOzy8oWd?8du(xV6>-{mpyU6W!2iz-;^uhle4x6pgQ=zG(bAGdZ7omY-5TS!FmV1z
ztz>fhW6&jgOhM87V6lLp$`ozouln-)&#25$;-2HZOD)JMqOpo6>48dsh<t#7#)NZ!
z8s)Y$$VL^Zsy}9WdPXHph-qW?!#@vQEiSCT^u##lnD3u97T*m^`Q9e;9hoS@8K|~U
zPwC~}8J)=yZoi+cHd>{UHY0Fq*@jCyioV7yP8V5Y8liV^LEyAW0*7V^D$fWA$U0kI
za`<%3ahEBZe!g4w{L?e_>uX*aB+dT+rA^^P^Q#BU_Dd9+ceL2*O>&v#6noF_OjwJn
zl9478--LTco+=L9Z=4ypp2Z#E7jRd+|0l63;i&uxU0b(AZVyE_vBKXcmUyRlxi2~V
zA&c?jqctMY$s&sv3!5swx^_wEZ2<Gtm7XQ~{!uwoCp{~aWB>YWx!x`p&QDAiLMB{1
z!74E|`9i_UDKoS#J2**J8&6kKNzc%~@?+&e5v8?0ryJ&|>1wQSe&p5D;BKg%eEO2F
z`n_k@j|s~9KC<(F(GsC3TN&*8%d1n%@WCy{xH--{%6$LCtx8lnxFpUiAwiXEQ-kQV
zR@qOvW=~Z;p0q|pEHrqnyS?DFmepo458;10Cf5H2jPBoP@USyZ@beN5^SYPvM3Y58
z+3V7-Uy7>5A}SK+6ok(0ic(gMKg_3Y^zd1ZTI&`@Rb|hsD(>Qb!ksOOCHebxlFn;A
zJ+E|L-NQ}Dh?A8+S5M2?=NXg!Hz)7qLXWR3n<BILE0eO7K#T9^=Q@tf7W;!trm^Yu
zwIwmF`F3O-cR|zh5aodC$(-92*1aqA)p?`y<e190Ebf(^dD||u@jbHYGQJ`5sM0Fc
zdD-KTkk*2APH}%2cbX{fK9%XD`P`bJ?;Oj62^}gXA5(c2IK&-u@Mw7Q?nH`a)I;O)
z;xm{08G5ueCp<VK!o(?Hxa5>-0?(rz2|BOt=zI*hWEOi^?Z4xr2RTLOR$P5x@aU{q
zo`{QMbZBYR$s<wzf-$0pgzGQMo#<r#-4b#yNml!+u7&IG8OAG?F<2II)ykap{vf(C
z=DfP!`!5GnY^sI*c28#CyGq8=*+{BM%_5Q8>Qdx~S0QsU8xQq$PxCjpbg|`}q4hld
zwg3*9oqqWrlDQ`>h<&BFWXpq)R8fx!-fA})t1a3+?(r<PTIDLVNwz6fSfrsj;q~u^
zB<&k#|Ho;a{os55#)HZ{5j&yx?n|ethK9x*o$4OG=|S42%#~$K8?VYg^AWCE=)Qhi
z)N(Q9En?xn5+5GX4_$lh9E$_f>1{cZ=frbdy?QU}KlmE!t^MiAqet^D>HVKqwDiZ-
zB)7ESO^>>n$_->x^+g`9ymiI;X^Lc_r%t7xL5MPYyz(@mGhb%}7`@0kZMEi}7Yo1R
zBc9!pca^P8R(|i<H*NNnM;Vr1kJT&@;}Yi2b=01;+}ZS?+`h-QcQ`hb2`iZkn8v4=
z)rRu?G_@07R&mBhG3nHjdC#qvF@MxE5J@j8*?n35h9i$p&^d((a(z>Z!WcJ%HRiY`
zir(Yt|7RR!IPu}9HxXH@PNZGby-{d=r72kF<s-FT?-!z$`}qQ;dadkNu9Wj#E%dnV
z^_D{S?T@y2NfoBq7cl+yc|32)x`S6XD^7XI*}uN(o)PB<t)p^Evf5=OzTfrqzbbmD
z+y1|m$UPxJ`_=+Ek1(!jPv0I{?`qQcSK&~?1=f-dRc)0AdOHGRZB~@GIL&a<NYHBX
zSfL)e^gWZN0F&$ng*p=tasS-)KEma~F{XvGNhzW-tHs?ItFsv&EL$xf#`y0~;sdu$
zp2iy<YNRW;JyeUD$bD~n&|E>sD1omY&mPTqIa$h+C7)ePkU928lf=@thSCcSPoEgq
zOn6rI>|jV!(bQ$n&S<Tkly>0Crb(<*t1_EicAc-7?4o$$kQS$Ret^%04{1l1I9st4
zv#Os<lX+2-nZbRj*EP*tYt`Y!%a{XMo_Wi!)p~YmX^gVq%YabMyqM0L(TU7a`@fnx
z3C>(^nl=CE%>Ud>=5GUdHZ6|&?I`rSQ7*z!q##w)MUh#xNaRnV=%1siu8AT+rZX2u
zs*381zRxLK)!A6$tNY?+($5_}MT@*A7{+lbNt|a<Z~AzO?S_Ki`I?dm373y>Tz^vk
zirtUXsb)u4lHHA@r_-$M4mC-B)R9{hxR%r4d+{Q}X^UK=d^sK^^R~Udci56c>FysU
z*At7DzF52PklM_a=H?fMoNa!m975LdXoiM-$~r9?JvGF$G0Ld#%gm$G)CxD5PK#Js
zI;lI^MI?UGto~hMr^LGdv-V^zl+SP!S>~L5d`BD?UrzI{2Sx(BuUO2}HMNW`l->O(
z&-c*8FWZjV=x;MyXCTD;^}j%CJ?D0dPZn&>o~dGM7j8ewF8p)1O8ym}WFs5rg`$fx
z6)ZUtIS=Vguexcp>~$}fq!-u2%aa!~$oN`@TsxxIFJ^V?hWizzaMtwpf62mj+o$)3
z^L^JpttEU~;G@y}88dXNySV$rCRF-vnI>xD7-f2KwWyMCvy0H->*rn^tc;m7XY+L5
zEnm;c_@6&^K~+F9xy;3xW7<-gvpRd+4XZXzZpkv8-#THUV8Tx0h2_=t8EkKgo-K?|
zKJnm@lTsVct%jwRl}3lu?lv&lM|CV-zh=EtgptGX{qK~O1h~^vU8HMoNMC2^{pIAM
z`Qbs4wDJrQGd-DF<${$<XRY2iV|J%H^Z(kf!u;p>w%+L65@yWaH%m3Zy*g3WEm1b3
zQ8wsL<fAiD_r-Q6Rah;mQaHlAd$G!=4`M4-6em1!^2p%XygEJIl5fR~PW?+?YE3e^
zH@!7wDEqo@j*s6-cJ4J_)B4JeeBzkurl_%u<8(~=6_u&;muBwD@hy1lJ^$#;H<OO6
z6Zo`={a8%SS&Op9nirm0h8yFQ1rwVutWM{ferQ$etrhv3)bfijE)1(?J+OVg*{ri+
zM;|ip`tB~g+2P{p(mC!6?`x#jSQpkT-zx7jNi=L>rug3-(Xlh614Q3`aCJF)<;C>n
zEDZg=k1om`Px#k(_B7MAeP3QTJYKKrd%^3O_1+*yhhLKaeGg4MC}bA#&U?mR(+ED>
zjs92H=^UBme;_c^-nS#kUdoJX&+fLhydpBn5`VVF{<3|V^`b~>W|bJ%(MO5h>Irx9
ztFBJFx^Quh;l=wq%~xKxRG0K;dK<v7=a+DEwaB{Ft3&6k?v3`<`onlXRkTZrd1ZTY
zcw_ZWsoYSb!zY(K5L&l#eps#k-_-Rdt+iTZ5BCKN-4XlqQ2ftA-b<<5%dVZ2m6?3&
zv|fqQ<t>*bEEP^)eKIA^S)q&BU$ffRV&MTH>$4|obhLNxHNAewL5X2r!Lo3c<#Y75
z$d=Z>T6Ra@wED&)t4mG=mt^PPh<UZQ&5%7LON#%aN8!1MMmes$$EQno{eQ{xpy9~5
zR}143p56~jRr3E67iN1U_^ayN-F0Gq!oPg0@9RA9{CF|;z0hl)+F-+t787Jkm&}tt
zZZ+lUIi5pu{|<&N{&3aDo^w;nlPbNZQ;e-^PpQm`eP#Goo3$xPVAmo|aXlHI!!eH&
z>rU9rIeGiLu>Jg{$-VNjj;U`h-W8kXs(7oBcTqxfZuS1y#oG*%?wM=EncYzldUt5^
z>tnjH3@;w;5~y4KR$#-8#0P2{@9$%NWIj(!X7%dNYRns}M6W5TRvTTByb#H=Po5#E
zXIb4p773e<3kmyMcm(xwI1CrLv~vmmQhISd+&21aRAtU}H-{5uYgwk)WOzHAYF>NG
zBk=!%BMVqmxmXUYvAn#XNyS>#LvTatX_o~Y<`N=SBAS7WGCW0+uf9CEyqw+Ab4k~Q
zRR{f;)|9<EG$B}g4f|;smh1~_8rDY(7o|pUt-7<b<oPv`2`-tj6WMQW5IOKFA)!tE
zzZ6gAh9?gKnaou0xTqzjG}p0CP;%y4wDt_=w<@z4DeGG_nzgS97gT9DGV+HTuiM%+
zjiLTc#KzCt^F*}kzl8O!+M0KDRqL+GeyOYHziUJta+i!h`YU07-Eq^aMiJ-ZZNnDH
zEnD|{!q)wvNiS4+<o|ss|5AK7c+H6qzrTOKD8B4lI{*H)W!s{@>PO!2oH;QlqJFaH
z6-g&fj|~s}=i6%hKOn-+*?J{}slipOg(Gq4)VA<S4TqGOl2dLr79C{`YcFi<2oMs?
zxp*p}L)An%b}HwC@Dy{Uk_Wxk*F<mj<itFacX~VHNf48v<Q5fYp_XGQ&R!l91g83*
zb5V_payxlpN|4Ke2uJqL%&3wt7beBpZ{48WQ1Ir<C4TK$Yl3cTtiIyht+D3I<mdD5
zTs+w|_jad~xBXs@U&|}MTuFLS_~na{+WbX56K}chS$I-lX35XP%S$(NJ~+KRMej*m
ziOAGfk}H>`bp^P6`;o+-B@>%|BT;GVqc(<}To1U`YyF%xNw;i*QX`Y4spLgJb5D<j
zalDeE6I(779nc9BacZ5~%I+%mU$QN=P)IT?V`W8vdRFX@DWS64zJ~1#E!`TnCwHSq
z{!~VVl5pq9TPhCeJZckKIt5v81Z1*Z*S@a7E_UoxhxS63Q(~M-KJO<pNiUh}xaG#m
z+gf)v2Ay}hkgFauXYrcFvGcZmX_K)%K6B3;hU*v2PUSX#OP;5}wsF(!S-Z|IkG222
zc9m2J%f-d6nK!mx7g+V&cgd<1U%zacc*aP3u0$jI#jFXse?3!<dzQ(RbZh6n$-dsi
z>GM$MD2Hc;z#YGnx(?fJ@9HggmXz9(=Oi{ydKyFO6vHH!x7S!zLp(hkPI=V39xLdm
zefoA`-)*52_a{a-tGF`pZ94Iw*<SZ&3d`&-i~eh7Pv|;$ZdTHoW~uTFId=8ZDc?K=
ztdlmFostTc5BJwy+@r{#yOni)^gD(0GkmiqER%~{!Q&|}xI`#Yzu{|JR?w`=Jc;rX
zzbAD`YOUFJ<8Yu;g!~h>bw##oZn_-Ww#m`g@TkegCiagmm;JpIx3caMZB;B1esG)T
z5aSY)RE=ukm<MtddU|hobp<sKM{7j1?JHdB+-`lM>xrx79><kxTp0qI*W9O1x{^`E
z;i%{m!ZG8}R$qpm#GWS-MoSLGs7I`waKrJ_^O;Asu26L7IWc)#!K@XjKEjHx+cg$6
zOCPf+6!~(YFaOuUzQ=1k8Sbt;uC+^I#Y>$Q(TyvUPE6{U)$;#Dx{`DA`Q}TD=Ni7?
zQG8|TA-(Y0s+)7%innt3JY-{0I5eY0ar=zZH+YIX!expyy5=alsBPQ9CY!3MXw$J}
zjp)?nEvqN#Y`Wd1G)Xa6^@tYJX`Tlx9}bosJ*rci-67D<v_Utc*NF3Epss59;_kZ}
z6n8zD=#s-RQ9-fAW5vT+47V;GR^oc#`MdeVK93tq!i`*JaefS9*%HA0;^H!!LK)UM
z1=|fBbdRT(2>C56Fx$McH#2nQWbT=_Bn2y+QbPDX9@jY1;1;sgSb4#rb8FZ%1t+~(
z6~6aXl#?8ry`PfH#BCN&rzss0{^$EZqfdlk%d51D>4vU(PAkJ)6_va=)mZ*F^Dazq
zwpcN(Y$I#S+8Xh2s}lk<w7yLI+2r9X93-q+^Tk+0=S)D1#dK!gXGVP18rmJ-FU<5)
zadb}!uE|K8`|`%ht}h#Smy0FsX)QR_=cOv8&e^nF=RjxZt%>ve6^mq(#imSP_0@2?
zV|CeDBxKz-jiWAtP21d!IMkdDp7L`%a<Sz=FL%|)l<?O{3wa`sE(y{+tx^8+Kue8)
z;$+tj=jp$k+ba5abQ3)u%85uWFk)$)`ZLG5L2}VC&tvwptT~L#1CJc>^_dyE$@7ue
z_sN<$3yNJ`ZvXXCZ@nwgQniOsGCRxA^nKzRg&UL8SIsgLd=<&kf4EogQpCh-M|_s3
zu>7x`u;E5}K-+=PiI0xXmW_GHwj`x<+mA(@{w2S{n`1<j%6WC1rZ}+%)M;o*p9;{_
z^PW1*s6{K8@gZB_OFh1MUp8<kCo(<SJzbP7Xp4YlVw%ih@8%mH9C9UGJJTvJgiUDN
zTb&sY(DG>Va-W9WS-H<lCNFbaw`*eG>Y#NlUZL(%-cS0<mqggtD7G4IP;xtVMX&tm
z?rZUiDLI;vDoU&c?2IBtdxF}Ermp-Gw!*+tF)+<z`q~_Uos%9e6PTF0Ut)_sNAR0T
z)4xc&RsZzdXLw^nwWqk7XQm|otH^Voo2ENzy;kJRXykqUTaZuHc>07Ns?$phbKAmC
z+%!4$#^Z#F>;EP(@mW5*h19#5d;O*uFwXVT;FK2d5>iWeHA_ibG3Y~Q!BSz_s#_A*
z@_6`dZYocVHs*DjrQ~71ZA0f&7H(A@y=^WWS?xh0+4`rfO7AkPh;O;`Rl&D0jBzFF
z6qc%GjS_2TYbf6k*gS((rzdtp6Z7|=WPVn4&K;-s@;RmbZFD=EKGkEd{1T@tW-N!<
zA|tX+8gxxkx#A!aeL-x;h6Mr-r}5Y{F7=tDz*^frA$V`Z@ilkL3eTsci+DAz-lg~{
zb?Ggpz50s9aivKDTMdKHe${%=#&-MdiDPqa%hzAnd2XwxMfr<6LJQSG7}OY#|G1+6
zecwf{%#%v;6Fc+{Jqh%@_5ViKnUW}PyLK*Lxtv$L5vq(){IQz^9xXK#IU@gH@mJ}$
zhoyA75=C9l{pgVnDr-*Ae-}6PcW>>({;&*2L%I7?kI%hhdO&-o;_g$Q;wIe@=vTL!
z*imw{X8OyQMvtGW1wCNe|J7_^hmVOfcZt}YEoWO<FV!4a(qEi%cB*J|zy$87o3kU<
ze7Zm5SLHXRb(ObLHu-2Cm|4^%$P>>Z`L$!gv}ktDlL7C0msFJ=doXdML;&AD@s4b#
zOIMDU_AXs4>%pm;ye6TqpepdHrR7&g+hY|!KJ9BY5-B<r)@Stlp_SS9=$U`xo*8XC
zeCJ<h&CGvca+Zq9LaB@qsU@aWE?=B8#Qql_EAwP$udOdOzdNZxD6m?<s8K2@qUWk(
z+Cs0Az|zYqtP0O87+FPkvqzQ}XRv>C7rIz-yD9eIWP!lrJ{DWEE|(Pf1ZDX%32U1%
zzT2vAS|PAjjCG2U{{}Yo2j<41uKX6vvW-ke6O7o_9_I7!Fjik&>mMw*)X6{ZNzB%?
zvM&?eSr>->OW+O)VCzyBENE@pwp{UfC>!6SW@C#=p;FzM&vp9Q1)UPxgBx;v7!`R%
zyo6Xf#3xnpEfHJ$-QoSA^beEr3N|LRPYqW&qR%D3@Ubk--Z62Wqk)iPV&jtxj>5`+
zVnTuH1uYl3W-V(oS;2bARPDrNwZtc`K_T3;8vbj(4&Z;-uDR%fOO=q3e@NZbCnkB*
zHNP3KMJKdz9;|=Oo)*rQ>gr^v=x(@9A$)s6_`HL8#({x4<|2CLo|8=?JT`j%nJQp@
zJwm=+l)<boZ&{TfM@EN*ugbS#F9v1x9~KO$O}ra3tUpUm`0RafNtEApt^oB?PD8_=
zml{1;6;sq$qY9+{v>4VHg&uO&Y<JS!W5}klS@yL9_v;T<+=gsvCTu4H`HMnQzB2Wc
zJTF`Fh<kB@Nco4@B11DjGyBv7u_uo)ratIl-x$`BE?_lNz~-kUe{%)T)VQCMRD+MJ
z&J+!_YxF!{9_aSbekOaLV3KFkQGo^QiXtsjOE*@rZ2K?zbXrHVM&uih$ajm{RJYh$
zEe&B|cGi7T{o{qW=8gQ0WwA3aCbDiam0QU2&LVjI6onuG)}BM5EDxp44!eeYcD)?P
zze~fqVo`TMsb86qbjlZL^JR+rJ@l6}$*(bBO%C=r7h%>O$oBkd0LMYbm<wz#Oc^IC
zhUXeErX8$nn%-k>S@Gt3YCK!`pZ0o=hdsZabEP_T+PA0ea&Pzc^lUJlDi>O{Tg8j#
zS`h0EMOF<#sTXNRs=@O&M$R==o=|4)9^NmIn6Z0__kBx|8H|}r0=U0Ti0!<{rJml^
z)f_%CyxDc9u%Mx8-~!gQqMDnGWdmo}uKv)H9W-&TK&#wi>DLVZ>(&Iga(~olOETqI
z7+NZBCp@urzL4ql?J+YSW$!z{B^;1nW?-+fz#--V8}HKgT|sQp2CT~?vy`RGq!kt&
zt>_Y5=y57d=gNs@^{_bt7pF*Vb&yt_TGi3Z^Cb7)x7>+es{&t&+%u{!Dof~=SX?ww
zVbYJ;e>Vv`T8YnGVR!qh_=)9-CQrN<b|@Q`gl%~ws2@1NYirGKV<}}z>Aj8`DI%fQ
z8D&jWq*)!gyPCB2X0%rSl03lC!)@7Syvbwx1vV7}*3<<aDF;|n3b@rzv6iF;rc8)4
zjWjD;(C)52d(+C<g{1<F9dULVf-XB1JP#xVWUe|UX+HCMZ>FPYftp@%;s40x9qAsk
zJfGh%e_oV1W2t_hpv0O7;>JP}woDQ_2f2RzP*!zp5?WYNc}zg)dj4Mzr5P8wepJfJ
zZDe(uDUkjusl$1`ksxbNaBLBKSZ<Qdn#EReu4-4D%U@RstjZ|A{8;kVV%wTo?(;AC
z?@!>Dc3{)lnUWT~D7H;|+XFV$D7KUd4$CuFq$!49392Y%T>QN}pwnF3vn=lGW&N8~
z0wxRXdCP?hcJzLly7+;H$k)*LpqZ<_IEgAv6^ri_h&vgd#5gT3(SrA)o~VTn??=a#
zA)buMIeWKvdOuHOE3Ay$l~_MfGDo0*H;_AZmO|A_$x};wYBB}tGGn}*1;0+1F+=FT
ze52?3DK8gX__{T3$?`8B1(LVBXPTxcIVv3TtiKS!#i<gWDo_>ufNif5qi^7b(rWIM
z2a6W%WR(_JJ<H4Nm-&vTUsKo)*1azm*l<$2{DQ+RE~$e*HyUqkP+Q$(blmeoWnTiP
z;MVYl(?9i|vhAF;)gr36pVyJ=k)@#5_BCHUGQMy`%|EL9>}69**4l-Js_YYmC$?8h
zoZkJ1DNsYa{c4Ho;)e3%>+4omEPp>m;Kc5jbFOM7&s%v51;wpIlGSZrJZ`e6mUw3z
z@O-L+a@S-n!<9?|W@(0j$(LvCOWMbj9T*d^K~HPnRFgQVO+s?F+R6*<J1z@P_7)co
zZRecmaO>Is{Uwg+&Znv#&FIrTJ#`DasP5Axoe^^@F6V7^bxBe4<S6c(ks-vqN%_|@
z1AU2QjfTr!WiE~Vs^auQv7Du+mMeQ!(za~})YckIVp(MR(lgcb+st(V9?}=s{+Jh~
zPMEc}fL-a@ti76y>vjf|3FuXN@A%4obWX_ZfM;BP6We!j=W`rVm~Ad-ykuvrjBcUk
zk}Xl5@iv0&iw-nNcs)#O(EPreW0LrObBXU^65EcfF)j-GWf3L1EVj00bL=Va3%}M}
z{$+6WmNW15V+UUEX0jB&G$lMF$*SV{I>A}lXO;?l<`NV)x4Rkb!)rB-qfy&#t4B|D
zj@+qPrbYYLIWT?SdMeTRKhx`s9lQ@G`|sKjx0yB4$x1-bS3En~K#}o+xnlU6Da&UG
zd}Y;@ZFS&s$xX2lEIG!hHQP~tcl!HJXXdpU%;h+3{8fcBp+In22Cw7sSu0}mJDbj)
znzlE+=E$_zZ4aEcvAg()9ybyb%s!$ds%0w07;wx)$uLMr`Q*}P(i?>q?wE6vO|NTq
zP*^~202^;~>YT_ObF+H7e_ts1I7^6I@wIyZOZtS;jo}gL$2b=EE_>Bm6j>~vyZS{%
zx?qQZ@EgVJmTOFemfc^uOC{sDWLs0$GszhrFEy_bdK<vK_9`Ebxwzc#Oan#nU$4D(
zl>6RH*}LtUz=tVTzfEE`HrdVN5o8MeFJZH?f+O`}=|ycOYu43g_U+pg7PB`%==TNz
z!_&vvHVU2H5%<`XEs)zg`nXN9^MX|tr<mf)X9Ox_3tVQq!Je3VyvOrYk8a)L_TH-w
z8&7TOyb)xjy?Ld%!mS+f1Kfp6CR~$wb6h_}TVK|r@|lF9j89FVP@SN}s?+DTyixIF
zNG=JidpFTgy>Z*C%kFli`5z|l2$;BXp5dtwpNQzP9h{D*I&KRD>j)m3+v9B}zCD54
zIwv4LSi;OF<;KydO^aCPUu@P2uWX+?hd;^T=t)+C^r}_5;<H)u{2haqeV@}aC*<d$
zxOu`&MvpQMNC~c*EugHVk|-r)^>(%M!!;{gOIDix?<$%r!Sj4evEbHHLn}5x#-Ibs
z{B5P5DqolRc+)^usPy!O*;)HzZAE6ro-RAUD!t(vBd@63If1=**B=qn))KTasmcGF
zYjaHb`l}Y(OON^I6g^H6U}gGXnfK=Lo*MCm7xO$Lb!&N*RHa4dr^kn0Ts-^s-Smed
z3#`4gr1h;%cg*eH^|YpA{v7l3Dbeaa`suATnc0$0-YgZKD5+Yw<&0H@^_5QRXmRdC
zT>l-ep5HC#?sRxB*Qsb_k6hi0vVJ-9I5#$FU*Ijg(3!nml23H5<i$W9!S=oj$!mrF
z3ErK?sIYg}mTdvtmxB3!>=9?#crc(!_4Zi-|C#3ZYjul%c+UFrKTWkGBB`@7`rVU~
zhf^bWHcD->nEgO-ThZ11<=#aLGu9X=GbtOmuq)1NG>zIUpOPgsk<sA2<$NjaJv*y)
z_HtJ;8K@c>I_1yN@p52WXYlB4@b*c5k2dC1G>hAZOl#vYd?Z!GUY0uftMutKUb9E6
z*Nb(HcmI3T@G#`MqiKlny1z%X@<g;&D%;vGQ;R>e+a)&b?6fpa!!vbjE13)e-8L@f
z2zkQ)XikT2hqiT|#M7$FK6*1hI?lDYIr*)b=o_CmXV0D%G5Ejdc~PN3(Z=^~Zd}>(
z1Qy9I+Y)kXSEI6C2=^SpjEFk|@4s!?&vP#K@61ybZ8GnkAIW`JANOdU;_S{R|Jhz=
z&kmBWXUGkh7a4HoZ{6AlTnqPHW@wGQvGwHwBNJ}R^J(cb6Qkt|ugtT!D)2KXcyU!Y
z-yv2$p?I~s0*bw7N}0`1uDbLjcC~JGpJtLk+J<7?r0)v~Ge5GGbT&$;8>;NF^H`pJ
zdB>LXMncP$&RsfL@lERejE0TMEQfj)$OxVPw>!jWDNo{;uAD2vg3Xev8MqEG#>*DX
zJUjXAJIOuzr=RS5r}vU;Y2~9?8$Ax2Jud0(S3YCx)f{u;sNYMkz10UNF!77tnmmc=
z!`j$OJ`OGRA=fvG$_t7w6%pV1M7erXz5kh-ACil=>pi#@AKNJQIwCTRaeA-Bv`yij
zUYr6<EdTjbA}%~|Y~keHrgcI<QJQfH+oYKl6Q(wVEm7IlS+Q}hF5^}wt_`Y&jgFj5
zDsxOE8Wu3M8Vb)_q8V^Vq-i_LoryCiq@QoJZ{)g=xk2ONT*LP}iXI<ZFt;(Feb=td
zt=-Oy>sZ9DzT#5xu;$U{Qn}g|q8+yN(l(}--VM==UmtOOFu#&uzq?ZO+vo80VIrEI
zEF9NX2PHI372@EYV6<UlgD$hOyO_t$4*_#5w7Hj@aCLaz9IMwa_jcwMrx|6!DxGHn
z7eqF#<31JUVVD@nxP{e0gyZ73mmLz7lT$f@mK+Fsd&|S~Lq)=*DXa_md-5U@vQ)3L
zE>W8kmGJE?$7byt6aIfO@mLV$x%j-*guRYm%W`&VGVeU|`{KGM+%q2CQVhEnAvi&_
zj6-DQw5lpW2glY|LMIN+c-17})bzn9<4~i)mDS?=UR`ur@Y2uQsZnt0jD!WBSiCka
z5EZog=+a~%pm6lekIjJ$lAn!y8)R7@9WwEA5mxZ<tNGHytynFxmiuxGOW&Htu9{B<
zv)O_gRNwP>F>Lf*7LuXU!*<9^&0ykX52cxIFLiy)+;(>GcJ?W+m=?5+;efXd_oM`-
z<$aSpHCk>=UZr5q%{*n5(Htd@6-(b-cyFv{@!*k$1OKWm3gLN*$<4YKgf_3;&a!F3
zx+XW*jF4b%=}^7K2;CD`xss!{hWuyzvFVM%`=U?|!y^Z@Uu=vP=K5xGRzbO=J1PII
z;j&*WMMnM)=JlRDnYP_TgL4|!r*)P_5>CMl8;|jnZ|^=fiPy(0Lh_t~!SO9GHk;&o
zhnrd4Oz6J)I4ZN%@>R{&Y{QmKsvZg2g>w(YHqMaBz3^>?&<U}5b9P<P>xh1sz-@jd
zdR2?j6Yb`1Gl!z3y_zX=ZmgT1qd1ed+N0LMnTx+k`^s*%Su+ndC5dsQH&xi3NN-J>
zc6EB)jgBOR4~vv%C^T?;IWVe<7ziF+sH6QYf-zx^)9V(K#|N@or#+qMe)(0lnBvCc
zZQISxt9V79YE=0(Z9x#<tCR)1Sa(IgZ)i{AjrwoM@3h09?V6~^*~7WgOV4&2P0~;f
zdp2#e%Bfjwd(@jECpj?Q<O+SUV)2FQ5cRp!ex11}Df-oO@7ftki+lQ_SM~?H%1;ec
z=bN?n@FU&@Otu<J8}=`qsw*I`arN|Me>weq5u85jU-3N(Wvb_$^=3ih!nGTZy<Qs>
zUAgyxn#Z5UOLFpmShif4u6+HJtKRzi4PEw%9S^RY)vLa)+S=QHYohVa+O^9Ru3Gby
zaR|*lTEcE)&>2^;C3q>L*8;mWyL(P}Tw>Fk;HVv6$Si4bfop}&0+I3!tgoUTidfI+
zWwCfQL;IA#OpnGjJmT8U;ZH8N=@uLio#gbeX4{6wgMm8QpEe%-FLo?}pL3^AC(FeN
zLW(VW=Dk=GCTVp-xWD}9bki3h|30|u+NI23v+}LFB*}9{<-webdo;4MO*|_1r7f0v
zc*5wW@rk}^uKOB0(%jR89csTw%<B4*lFxqf_JjlOsxEn&oS%1}c_5>?<G{w%ip-JI
zXZ4s*=q@-CD>?D<oR0G4&hyfWzkZw(r{FflsK85M-(@aInNT)PC!L_8oMvv-3A@dF
z8klnnn)QUgIvd?-<d(L$?H6OwXyo%SXL|uFr`iQeXWo{XDHob9@Gz}j=rB>?#u~}Y
z1uJ|bd6&rj+i+ZBpYtpoQ3FN;CuPSMEB35j=xcmw(zJf3N`J-<w*IQdbK5WcH(9oG
zCa2|^1sx4%zDRc&v?vu+@v3lpd6u@WG+3g*tj*wZvD$Qr=xl+6WF7O#hD{o3H<cDJ
zZ&zl#J)vOg!RV(oLT*Cl43k+l`zRW{4`}0@@K9{&4F>T$35Gonr*Lb%_189Qn0&5f
z4a<q1E`~#&Op`vj*p=M~k@Q<8zJ7uechN+pwM}Z9XB=3%+K4gA@zg4<T|bl;_(-4d
zTva}4Sz@M(mxpK91&+l{Dg5iTyLjT*jatP7RpxE}GtcK+<i2K+%?eJhTx`8Bu5mP*
zmhZZ2_D?pW6Q4i&ZdP;kt?(;yYknlGJzJ*BSx>VeaKb@f_DY#qXM}dP_dZEHBCKg=
zsNV2@vPMGje$A)4vmRYB+tSD*bMx*^713#H4)QHm$#9icvDozdP*9NmnOJ?#2`dx1
z4)97pU~p~fKDEBqJkea0S$u)gE;}P0P3=&Q#XURc_#NYl5?keM|K!m02?dw8ZBk0t
zz4dbG5sd@uSs16K@Vq&9+LcZHy7QZfzkVcCS1p_O+jWh@gR+OsI;V}>kH!d@6imCr
z<6{2B%0v2;%Y=s#d`G{W4&H8&H2ZJu5&fjB<8G`S6H4-qTdQ6bJ8)>lgf}j_Mn5i|
z=P5K%);sfMg{hBP@m+&6eP>sQdUNJ$swLbDi<!A9JTFZ1(52u>Dw|o9jeD15%{=sO
zSK|p4O`bZQPjb;){yT9m`Ow-tt3tB=%VP00r;2s29qbBSFirH!g9!{Sj@(*t8nZ8a
zYS(|2I8(koL``MVtzHk~2f|a+UGsFpWO_?y*511~LC`LU?eoltOB3Gbwdy>%YjH<7
zfJ@;TQwIZs(g#;A?k7jMZa$f*V6^?HsEd-89=GNr?Fms^ns~~iZcVP5I=$qO3*&a?
zdqF1@7=@-NPT#rvD9>4G&ACqD%^P@T8g#yFi#AH-irEk=Z+wHzdJ50f4cYx=!c(Vo
zuH3%Yxp2LMM+Ud<1V_DJQ{SI{6Rll;p;?Uiys@s!L9VvT$vo}L+CA<(;V<h@yfFQu
z{=xP~T6+pMdN=)CQCbx=S6kJM^M9G^37P6oerH5fA_et&l8S3*%KBQUd03peU&4Ib
zGpWd8-N)x$@iIxbUB1L9-MgRn&vI5*+?MEV2d2-k@R~5`nT6uY##>^ig3J$oh)~p=
z$Z#y_^%bEa#+iM)B2IWLJFNDeW%)Xl!&7e`2=3YGG*x-^`I=J>Oh!`{SV&&}em3=y
zXmn2^cWK*%=wD}^R|#aCUsG_|;`c!=-wTW6U0xs8I`ZMe+1HgK$r8)$W%hN|U2zg_
z-1h9;g;!Mzw#0?RJULq;P%&ZB<iK_5e?;SSH!rH#_VEB4<C`=V?FlleV$U0OI#TDV
zY|B6RcB5F|rIy^jNJf`wCJYlCrt|R2+?x}2BjeY9v8VCfYJZO0G~2oVjFsm@uJwP)
zm)yG5knNCuTFR+)hV|V8EEBrVc%0lF@@V3xQ#0~jPrhu?b!c*D&@P^N5na&%HhHg%
z10z_=R!of(XbqHD9w@--yMv=gn%VRrcNK?8d-RTd5v+j%3{?}l4I^4bL|6-c?bCd;
zs5Y@@;RP$1+4Cd?JC6Nul=-t|)j?sl-zHrPIP4Z3jA3qSOmNz!#G&#)tn-$9*o7q?
zT!w9{4hdAx-?4LkQ-X@%?X9AhySuD*b-WN`vFwvN%n`K1aEWEdij}$?Ub7TtGiYWo
zSI=lzZn#tavM6K6EX{z;%LTOD6&wG}=#SJ|r?T68MZjFgi{{+_4{3c^sPKKokre@w
znk!hXPYYBDF!)L^mz^*cT_bu%_+aP<ZeAVR_SuV^A9J7Ita7s0)$oOjs+6dvly>us
znUZHDXMbdG_`RlBsQ-45f-#fR&(A`>!X19yf-}53lsHYRA5Gj`D0I<5Bcys#`00J8
zFCE)`&@RH8LFL1)YAz)Yrml?v!ksTBH5BOzIv?+ln3Iysy#18$!3gz57H*!^oGC#E
zS}He1J?b>eU`_kA^0dZM)|^$#9Zu|#n6lcaqjO`ck7BFt4vw5x?R5gJqA{!zF{}Y!
z*uyFghNiHKZt<v(XbrS*og8zpZb2)5%hY`zjwIjauHo9bPl9>U?m1Jq1!w+0;QofO
zZ>F~VSq9#73@WO7=JaPtsrxWk6^U#2NbBWvERUYtdSJ=cg(`y8lUf$3^mKLqIMvbj
z%Q>c#^SuJkrpZi38`^axCaRmPoMgTDSHQ}qfc0LP9t#}$n_0RJo%TMkbD{164vxSH
zHnS~UgqxWd_eQNaAUS)7;ez(*F^&b1?7f@y4JBCjHFTd%W)G0zikYJn=*V&Nu;hYF
zlg<vNO^f-yNp~&pl-iZGe5Ih>-UvZS$qn;N_Hpg*xxZoOCr#<*h}9W?4&74WxKkr`
zDoJNAL&Mb-D;Di>nwh|HH(>391l>&wY+{Yg%#=GUq`NQg;dybRDf$MJ>>FvVNcGzp
z2QL2?;QFR;{)x-X7D+jy3oXqD^aI}Pagc5e^w=?R%leC_Z9-(a3xycC-?X1vELwHL
zRq9E%o|V+og`G!!m|P3tnEu*VztHkjfkC>bKUcDo`V>A+hK)T5D-KNLy>Y{7lY#T<
zE8V{q&R-V1ah}Fu{S=OzOX8gxmYLR;lXv&2%wPzb(eTEnV?|)Y>rX;nTTXwO&85)G
z=_SLZG|6M#tO=)?_txz=;O{fxV)B_Q!VN!q_V_(f+T$T`&c`Hf%LUC_d)$LNj|%Kw
zX6e<eFj4oR?Yuo(OugKiJ?s>ZoXT)<lHcf-({iElz}eg@(sQM`qO*C5u5cT&%r`fb
zdoSGJ`(fQ`mp<SB7VZ^S#P1xMoW{(@I9tTKQgbcC!e$Oxr$WWgQR|xoy4-C#4k?GW
zx^T3Gc<fT)aKAa%>;ls&ZPA{~2X=h$n7n3cl*8V0Uv`{sP>TP$YTgTr!dXguGCj<@
z<}439b+KYO`&Di$r)gZS(gv;5ctknoB|FV?Zn!FNTvz5wo7CDn8p1^_9Jd%IFFV{N
zpvvK|$nnv+V@-_klN`yLhdRPK`i==$yUYr)JhIEY#Y`u#Gt+q4@{Cr|piQ!sQ(C4-
zOr0rc6kr@udg7BwsETgS=SiE(Ud-I`inUCD^SC$196{|1UxLnWk(|7g`{ra@9WU(-
zlR9^;*etEmJL!{XP9tBAp;*m-UZ({N`<f4Mt_yVY{(aK2^q3XbY#!@dhq^kqFiu)x
zYnppz@x=%G((dRN%w6$(Nk@KWYv6;96PkUmT>8Rx+<da~${vRv-GrsCfo3U9b2$^%
zMX#7~CqZ?&LbGm!;;Nv|J0Hz!X3Ypw<+#c?ulWMMrpA?XYp$J6lv9&Z-}xe<`&P^G
z6Mk#EV{9b_-w7Xju-Rn6hBbR;>}3DjE6aG;=A@gAA@4k64xU^m#*RZ8FT|de?0TM~
z@lBKCcV>uStyh_@Zj%JJ=u}Rl1H4TRhb4uDJyZ8z>e<rbu(OG=>xCkJd*+Na2Nv>5
z=q+ekd2WWD%Nes*MXT0us+k+#nXc5KEZec}f5VAKi8pnM7O8eE(sZ1vw^nKTSJvtU
zR;#?X^#te|-DuH05HHW;B>P45|4~~8z8hB~CeF#dyho#d?<2u?f`?Y>a_kVC9kiGu
zYbA%hk!j0;vzs-UKORgtoVDL_^MzHl(~UmbJ#U$GCqTG6O7}{UN^K%vt*K4pPEJRb
z`|7tFqP;{8WHA>PFO@uH{YyZP@4@6`rKMlqM1M2le;d2<uBKA46))4@9Z9-(3QjGW
z{<V9H0_&;W_YE&J=}utMbqILwuvqqO=X;;vBODDmnGfQBuC@`3S`eC$CF}M6EH{h6
z-A0YOjRxoVB|{b|_f}2RSkW_kjnA$&DUI7lIEsJvY}wGh?!SrFS<RJcr^Pn$x}Ed7
zmm<5M*-%!>PfzPL|K7thqB#`bS{e0qXf14t=(-xc*Wgjhj??oRxPC5(_Vu~XmT9Y3
zbbo4U{DKed^%*AB4e`+*=9hKMxwrTJpQ%hK_u@4>_;y7tc=sW^RU&!+LERZ14GbFA
z$2Af%F3u7OG%!mu&|A<Wb1`Q32FuGG@;uG6cfM>od@5l@u3h7g8HZk6vt;YKHg*1W
zU71G-4^Ig67_}wdnZ&?Z!1XPnYw3ZNQkz&cHaE*?F{ho9GS6sUzmQ+3LeHFGO+f0?
zKMY*ET>Mqebwy9`IJ8yw$lYbT0=Q2|vIgB~(S7$w|K9yQ9tux?rnf|NO8>9E_KCyJ
zq^svZQ1af0g!eP$FRV>p;K0DJ;&JnZYwx&)yFbMm22TGZt*h(OeM2|pRw#$S-d7^J
zJ@-^Eb~y?Bxgf5`An{(~Y|2``EqvY^OghA`9T)r+H|d5^c$Qjf(ZV*p#$8wW`MoxI
zY}DHS`#7&v$2(D_A1^qsS-v?C8tRkM8m%XK+D2|s<V}vs30BN$`&SrlS+L^b;ds5a
z%#^?J+te1no$&H~hna2Tv-;Y@M-^`GaJnYw_*j!eEZ45r_qC&_<A!&YLPhI1PJin8
zwZk{fand5+sG~{gyA951c!)1L$8A!0{QB0cwX)m-#vLizO4&0rOPqLE51d_kVyTqS
zvOE7Trev0It!gm$6TQE7M#P>kLf>DRPn4YZDR-*IGSP3oqQ6$L$0yv0I`HVqqc^%1
zQY>pHZD~`O>w2y3EoY`(+(sGR*=3!3w0cha%H_8uFJZlGFUvd4fr-U~Nq57FiEf5v
zQ%>D_%h@2LA$*Qwxyuvbt=;!>!&{BEJ&^54S=>`o+qZ4g$8(EPmUZ&o?K)huB<*u#
zS5ENC&)d$;U$CIA`?Swpb2axzHQqCVLQ`9}wk}%mN$67lM9tQxH<+q=mGqk86U5Rs
zzle3c%k<r)V}tTU`;gBoDxaB}bC{eGy5v@1apYsbq$iU(rk$9Pdq8vF!n-_$Z3_#{
zqAc0^QWt7yN@lhEpYcHB=fpr}u6MsD@W<yBtSr&!oPV{c!zW4DY|53^4e!_w32)w4
z=sE9-q$?+DK*@oN8FkEeSX8+FS}b&_oXfL`>wwWj+s3{ut(WY9ccKzvPE9^xIDu(8
zTLH6LFH3(W!#T&a`CDeKh<W|<^M0m7?B8Fe@jkz|#_ieSFZ*V0Z8msyRf4&wWd2=)
z#AXSr>s~1>A3ungbLfi{ij}{*XVKkX;gu@P^E~5q{Z&VHf0gUYGG!+-oR<h{O_+Po
zFXzMBbKZ&l?@}wmOTSK0RE$XL5}q{INMn7s<tp*WXY57`-cMkv&CGg!<$m<aLfQE2
z499z!c9jb>E?;!Gob4HWN2n+Izh=K}c-n4{gooTZY}39y^o#neY07)~&X$eAyz|_a
z`rhL9=U|x76xiXJH+@N>^_(V$Ic61C3^H}Qt*pOGSbAR*ZYnIu7Hg<m|73l9!K(m<
z^BS@p{!iy6d~J!D@-c`*>v)m*g`(Wmr7K<?6hFC<(}UM`&&Llk90wvEC)>MSaI5s5
zJE{Jg^X;gxO-q%p&Ee7PD6om#&dUA#{NJ<LG4DLv4n2D?|Hh82TeaoI`Rf)Kmm6wY
z#csaaaqDi2hsGL(6`NPEo9`|^)*&bVy}U7V+mZh@n^=$R@8t}ytJoW$zhcIW?WH2_
zoF{hwp7F^_>kMycMqu&e=2_<F>!yW&dpo`J$p3in8RjuFk4?*7#`Af5<?97ydq3Ek
zeJZxEPj1;UW0@=$_qq=iY5VTSN&6K%&y3Y+5Kt(1;GoCCucXqE;C-l#Pu{6V<70w0
z;~JI?f)-v3D;uTxw+RMJ?C}W7(Nf!T)v%L+RXF}o&5NJ~O}x2zyjvcpEzoW(<M?LN
zVf@TFdV9~VO_xvkNt>tMD$v~AdRl}}$jn2)!tp>@OXOBFpMn6LgERF{3dv?%d*eJU
zlYLjM_V))-TZ><EotVX->!iuET!!V$hj%dtnAo{ZG6E9B5A|^}KeQHTbWmXuP`RP9
zNi>}?*J_fJrGd(+3$l@&Zk<sIuev4Jw-w1=O3Y}IjX(as&b#eO4x`G=E8AK(gw9ss
zT%us|yYxZ0D)UsG_7{_u-d=MtI%&D((%uhkck8)c{QK6&FsD4{rsMllydQ&7PfU2Q
zDe)AG=2up)g1qfZnWpDR|C=6DcIIvS^ai7&Tke!{zGM$<=2sIDcjOgPnbyJ-Q6W%T
z;CdplmDyA6LaPp|gIG}%N5dnr4&D}-qVn9WSJ{}(oKSK$=W3AXX4h47=;tV07~y2^
z<TA-2r66$$L+X!XoQ}PgPnPuOt4?5%T(r0Jc5=(sB57%P&XU_2vJ*F6%<|ebEy!Jw
zk0ZFjaO2EyCgCZDR~(-F6qy*v^fogt$aCtW5b2aD6Q77k`!t01F#h)vcwQIFdF0t_
zcBzW6_Iy9~MeWf=v&5FoJ+(!gq0aP(KvA3EmqiTvT+@=BRvc^5T2-&QE!lCwBP($M
zCKbiTX-r!uU35P5s4c)fg71h|{sAfPfLXCd-a&cm4{o`=MUpW*P?!DcB|qsP8FhwM
z{odD_zDAx0moPYTwk)rB`myqgf7zOFFX?xzn(_&4uI8Eb&za81*QLIFo*#8VamoQ7
zaX*%Y9ZwFKzO3`~cq4H5{K?Xfhl^v^%x5~bam&#j-Mod8hjiCT39jKh%aIwbB%C4<
zI_H~5&;%xrlY)*Z1x}MDxvPmy+{NTz8IbIJn8jg7=CfO03eF0dEOTR;=(hGM$N!Zo
ztls%nuS^d9^Zl^d!Q$u2DbnABtU^N@V#7a9i|t>!^16f-+xKI|=R>c%RC%Y!#^q@8
zG#ug4JrNqO$^GTgiFLQUuIyVUw?t*4;nSXSmnm<n%unzv{if~8;`qBVYjI_mkmHg{
zy`UKzb-O2et`gol+m$I%I!gOJ(`~LDtETNPcghpKv&=hEa{-I^>zzgKt38Sw<ZFYT
zY`?Rgsd7$uBZJt9?MI9aqcT6_Ik|kiE39-Q;Gt}g{<^l=-`>60q33u)E$Z>NdjT=x
z*?$U|96c`l=;Ghmp<%da;zOp(Hph6E6z1=Ybgr2uu#vG>#phL^q{Ts%*_KN(CKq!s
z{W9Ne;P}5Kv)}PrjMrl0`zlYA;&*wtUYerzF~?gq+JXDSx^!o+rLIh-6AvtkOPaQB
z=bQGqo1z;#GIn-cDR8LyeNXWAl_Sk2tGp)9J|yO0s{PmJpk+|R%|xLJ0oGMH3c)jO
zPHDI{LGEgff<TSZrEBaBEea1U!e_<K3i^If_)=RIpQ7TMwCRE=ThA#S={mVFWBabM
z2kS)d-HI)dn&PrZ<Xw$<&FR@y;r7M8E&9fp^P~198Sv_I=zhN(-Tdg#$@T@Tc~aGj
zp01eOAf$S(dp5)40|C<nrt_vWa7f<lOLLuZRK>IR(3gqltIQ^tee%dTaYsmKi9=WJ
zMx*dWl207BEOELj*v<QYO~W&XZK2&^X9TSOd2-z8d11R#(@-r&IsJ}9>bZ<V#d)SO
z%Nt(2oZ#M+KZmp8@hLyi(|W7kE5rs~6<2hhG4-NQhIG&K<i5*o7i;-S&T-9k4Vf%`
zeZmH_%?C9&yvv<*=BlYQZ&|!?%S_+fob0pS-te)QyqINs?j%Q_IVm5mF<ri&dcN&P
ziITq{%bvMeec?{LT23-z0db6`O43y~TD>%bn=G%+S-nU8ZpS4<);F%98kf#=nIzjb
zE;yCn@%qXwfr?{|A@3BOiV`kNIT9wa?qk?-7e&W5wxwI#mUwZdzKGVkq#8CMWKq<n
zW!VMk+y!YG;pV+lN@`w295b6Tm$Ui*3G36>&F2J7Th{NX761N{1}n=0ZH-S#4}-#v
zE|+`a8k%!8PI-|sW5LA*e337_A5~3Xq#4RI`$f;@cPDfbzPd#y$#9s;K00*YZA-t>
zN-4#}a4FYYNlQ0mx@(&Ss(POLvV!xJW}sc`;j=<&4zqf$_=uF5f95kynC9dWbHQ7H
z)6{)aLDc<eA2e5<ImNm<^cF*s+ic|{0#h8~ITc(AG$%YOO-*fk$gsR8^VkZX%-+*E
z3zF~saBI@g<=Xl(XSt8Y#2lgNOYW;Ur&_8WIXTZ;XO8Rhc8fPj$JZDs-YE5WsB+Cr
z{al2j(LXJb_bUxH@$lz_S7tV_E^5{75p!SGq+9e~BSijyPM-qLH1C|uOo5F%za9<y
zdetROS!$Be^vn}X=N!yrmYshgsJOS2S6($~QddQ%`F7z3ug26^PSuG{cbadM8*&SC
zC2p_tzQe!3bamWnDXYDEjZdp8HpQ+=6G~oT(@`8@q_@#x`Nb*M^s?%&%d@^P&=;2Y
z$|7-wQLlu-OI3rJ(_!v}NsS+^gcJ;X*en)H%5DhV7@F;qx6P6BQ0O`vsb?EFHyz{-
zFIDiGe8}~((2aAGk~W<B>!Iz|xXkE2Tie4-*3SM2zn^vIb3Zvh?+Uzg)BNg<i@k5x
zO{<jg67bY#kT|reNW_4#dipM{7>$Edxm71Dv~2avQxK6fQ}{nkRx#b6*Z7Rs&PA>p
z_H_mCT_d)!JnZ&vABm}+5AGBsE;jCnT4LU);^z_*9Mi#2WO`QgtgBPAM^`|(u>9=8
z`L~4(-rp16Y~L$#Hnu13`zdojMy>A=N4OVt%@o?&_oevouHTvKCPdwrb@<xvWgs9d
zu#~5jOWbMKFaOtPtGL=1DSheUnHF0$QGv7VZDXbG#CY+Qj|3u@<w_O{aQi<q+VwSe
z_vJ_7=l^txNKcgF=oGzkrE>CZ!@4&S@0d4N%-VdU_WbR>x!+E_U0HtqSzc%5##M)_
z)n_nA&)76!7sH3<8IG~4FB-10USMpBsQt>W$$ZV%VTI*ab60Jxwe`a9rU(D$J@#ga
z?FtEXdvnW4=N_NR?U*^)^x2)2cS^Q#SGv#K`s^ypbdBRye>`&n`SX~%MdXuhgj5BV
zI8S5>`uTRt5>=;?b6Iw3Kb+?~>)i}d5puoky7bSNEt?V;=SW@TaCyziaE-Ht@lo&9
zy~o#B9?>}&ag+C$j_yns7fHsuQj7<$F&<>e{mKzHb>c?B+tFJuMmH*)vtdb{Ak37}
zxMic;raMPBSMqAcOqN`FAvfZ(aqIDdlG~3;!|MtdS7<Uw&bWJ_H#)^JY$=aa`$Fl<
zh7T7$4AA<?S2rhO(_e|Q6SKWfM0>L^L_d%YH#xs@t6U3bi{us2$scY{Ix_FB;E~s&
zym>qC{db7C`+9F2qgaBtM1tg!dE7JPS~L%cbnrU&Fh1JTA>z^~@<+~j0%L>d{2be3
z>)!S-rONOga#1-Hv4rD(k?5t&4|82M21suRaGG#OcH^OwbNgIMWL!=}x=D1N=4>e8
zblL8}&4293NsS(%57(o=+(~|t<iAnE@r}}+nbAx&tp;+6B65m}OC}X9IX~a`2vd#J
z%F?4NSs(N(B$&p{y)N*8;ot2^QyBLdoWJ4PFO<+$FUYc{SF}!IQ9KWuWP#|Bxv`F%
zF0AWf&&Wh%$KG*v2zVOsNJ_IamXl%AL)p|L6Q@eVFQ33Ew^Fk3CY$*|f&A2P>$$?`
zH4lhgV6Sk!^=xhQf5$3cm8YRqD<gCT&-C;1`!>k&hq|tu`rzeR(Mt)kjI$2N>|1EI
zNzP~0Lx&407bh&_exUgFt>XSt238Y()}C2=|8i$EiaD}0TvBmVkZb9ZX^R(mv`Xet
z?*`@Eha3*Fg3~gW9Q8QhwnehYmPNiZmAmVRtIPfSmhu|k0&^0)bN5Pml*ZdCxah7q
z;`}K7z!UcRT-LLKvWYL^7u`%1F1lqT7aF8{_utj)mv^$5Op_94Ss*RP!ekaGwQk8Z
zQLhOd_iC=V*S$C&m$J~|M0C)<$M#CI9T*xeX#||Cxx41<EX!$})=F*4N-e(<6&U2&
zUR>mHS#pv6(o`3TPWEXDnSO5nSu8GxPkO4B^(5YQ!b_FYxdyy^O^e<==rT|{&{c5o
z{lzqwXEGw9iY|63LVO<OWzPjY&-<8gdCXJqSCuX=Qhf1Ly17-+@lA_^MXQ;d%j>nX
z81AJc-&w+tl58fIAgkBOIU_Fl&F#h_2Ct^L=dEfPd`HrQI2r!*JN6!IS$1NTN~2;}
zAdA|fNn57bDz)hIp0wJ;k$cwl?#cKFl}5HZZG~novsou?KC*&aP2NL8<*TPi;2al^
z2?sT19q~H3^!QV^^*>S@UbIK>p3rv_KKE1U>B`yR0sVcE_b&HJg@>>*EQ%6m$Y|Ik
z#2BPzo)ue@@a#tJGw;Tph&PirDYCRIy8M5;qZot20oAA#t2U^u>^Z1*Kuy_>CCMjJ
zFERCcA(!eqwftjqZ?DvJRA^+M`83R~Ma^x>bO#nqr{@k!MA92hpS5h6?RBIgEv36M
zUElAPYu&3DiDaw0)2lrB4Xc8@Lo)aMJ&|O<DS7Vsq#R|Vr4Q?Gr8o&B)L)w?dnqpI
zLo{c>0dc;kdoLalJ2a_TYjupvy@&;At^cn7S9ui7<sg#MD)ysQj77=X@AY*<r4BEa
zwSO`%%vcfgVX?K~lTW<Uovp%MYE-9baKE{8EN4r`ONT3&6Oty*>3lt>g`q~?HN>Sf
zXZp`MJgQsc6NJ)SB7}Y~e3e|7cu4DRLQ=oC#bYKNhW{pV5?5`HOjBD>KW*VP!LDSh
zWhX57)8}dG)T+%^W4R~0Ds!8TrDF86gc)a&BpB9AVm>9svde4UIoGl#Q7J*zf@6FW
z7F?Y$C4oKfRbd93{9L8pS&J?|>w11dL2DIXf~>mxkqZSc{63yJS~+i(RpV`y!c~($
zCC%nidv?)f#b54^HKI3oSZaiNla9Q4W9lOPO8uh8vR5Y`I$ls|l$*?WYoV-M24{d{
zlbXhhukD8vUW*8|7AUcBty5pom-U2;g-J&`nD3Z`#nROoj%~|4w&lGRD|s!bDdDt0
z!zm!ic}boVkMGW;4v{ablPeZ+3$?4wiut6QrJ>}auGVrw_rA-U760>R9EoOoshxL>
z?L?BQ=YfSejpvnggfx|>c&*=U+p^&5o#vcNJ%ww3`}&9X%nI;IzIyPXv%@oIk7eun
z^xwz|PtJ&))T7hn*gx}?e<w@hsiYg5Zsj{$taEY5yL92rLM9akk4af^8tboUwtFRf
zSm?!i?6K0VXRA$FUVhEI$maV*NB!xP1uk0zH+@|ncTi#RqL>*w+_t##I!=(x>N&*G
z-Bu8?>RrnTo7P(kdnVg{UHasns_3nw^&WEZA5`NQWCPpg9slJa`9N&)js}Jaifgys
z`<eB4`mvPPAD=r;dB1<@JjsmAkOj9qmENUiKGa_Lvbm`srS_x!yH<vZQ1Q?g*`5yn
z+vFDIKX@5(Bxqu+>WVXkoo*7Z&xy81-Z6XTwSLaVPlb=tp71HW7cA2C&Tin#pYt>=
z=JY!$p%05LcnFB6|Is$Dy)33P@3uuyY)jOEKdNy$Cl2j%sS!N!MQ+V66DixXE<G;E
z(^lypKPLP;N=?$Bf8UkkEDp!LBROU<HMT}ZG+V8Rh`I6KujP!;X6HBgmkLg07R6q-
zRCpBOzD<oSyffQ}XM*0v#W}B@<*2qyUNZIdK9&Nel&NO~eQIW2t}EbZ^pfQjp23mt
z#L|B4+yyJy$7P3R*LJ)*GOIs<Y1y3>^H*P;RM~k|YjV_Q1A&VIhK?t+<m^~{^D-}%
zG^8+PT)YtXU-CubtF0<cV((Ub6j;GHQ_RifAKTs2PA*33EKk0z=bisWptmc(N6d2#
zhvyHu^#Y69vKr5a7`rrR`%GDJMnLQ0taTgairPIhT)VAx{f5<6_Px8!TBbjoXnJwt
zE;rMYJ735x7U<u1%yWy=$3tuuf|@=imn<)}2&JfBuzcw3vC`K3W#sa$4KF4bJKYTv
zdAC8TWm=rFuro{C;UeF2b9LI^tSn?ad5zsR@<i*^^$s${LQ72UC`d;w_!2wA#OclZ
zf6Kmb?sL`8?y+3dasQ0ewpoTg@9(WT(KWv>+gU||+4=~}_gQyXOIeS*T%5p?x+kzr
zg`sejpH+Kdk?C5_&1@cu-v5_gbMn4eys7bdWWV9lOT8&;CfkZeMDXyWPhPS&^C*kH
zn~?pH^^5ca->5Sseep|*UHD=4Low~|48}H7Uv2Mwo4TU>saxL4W0yBIZaY+(En1rA
z`~O~g^F*1~FOx5v3KjCS7n1({uf%80x~{B)g^hg?i;groPjOcA&iiq7OGt=w!j<(C
zBp+-%clF1!PhT^%Z!~}GI-<J!Xnnzw9Xzrg*Djq5xuj{oJ~?m=Q-qFovBSe%n_{lS
zrQ~s@X3fp-(D^g%<)3%w3;WeF?cQYsxiO^haYeCpT+o>vVwN=hi-W*t1M!rU`*Smy
z93M{cX^mU$5RrY8w<Yb&rHPhpJpuvq>mL69zxVFihR-exy2hRjVo%ue?6(FTKa#iK
z^>SV3<}2EJ({x=~%I6&L{k~#Lak78^Lw0^?ziO}T;#`Je&p-M_n6Lfyit(LkpvD4+
z7fU!lsI28%mptv((W2IrjxWMH%kQx+eq#6Yj?K@>-uF%^IqWK%_<~7d+q#Wy`%Y}p
zN?Nt0D_gK_2ivy*X;bsvKUpv5F3XcCR#IOsa_OpD%Z4oh`3KpGEssd0vV_G~2R-ML
z>^Oh$j_W}qO+V4p9`P+FxD@(stII02m=>+v#r-kbptJfzMVj6DgxN2(WsfB4uTDMh
z`u|(ey{WH~#VxDN-`zd<^3)=$DWBg9Dmv(E^RfsVm^4@Z&!6>CW--^21!esKjSfG5
zG4HR;6SICL)jP{+oxwjtxrdgYA9|gVZ<$ef#eKt}2aQiNU98<dOf)Za>X;E6G`Fhp
zOLui$qR*+d)A-K%@vYdU@ZeQoTC$JHeXa*W(+>!~Z}CeiX=KxGp1UNhVvF;Xs=VvG
z2WBsOs>UGX!XV`0DB`iSYI;qzxYfNid!O!$YOT^J`f^Hf-$v&Px2BviT=!&4HuLNg
z+Xe4<D#>-*Sb8>2yz(bP>3Ordcx&^yMNLoLGjk*DigLHId3K0ov1tC#oELF>^BO~U
zz2E2d>okPr9Gzmj?B^xL<M%8aAAEJO__#lES-*wa%wz9=O!SzbZr;Z6{D14-i?_Vn
z7Q1}=m*$r#Re#3q)H)esM;-e^yK~Ow-O=!{&)(pCW^(J)-8T#mzf`q)koNrOk=cbm
z{vY+TZ<)z|<k;r0+b3P4IM-gd6v{4E!Je+6d83KrqNPuX40qHyuFM@*nOYbcnLW2=
znF=gy2wNiH=kcS!iK$UYPHoAG1chTg!Xg$T2CdBvLQK1-NM<a1bU=?OEKKHPkOO<8
zn%pTL4`D?=MzvEjG;S_sna&qu;iIAD(8$uu%*+upODHR3Ww5H7Yrw}>S5}0w7R;Ts
zHQTk5lffhL#JXt>Th>Hu^<oOkj@envS|!`Tz;M)%`8$`)X4eN|n$eRyBz0UBBo8$F
z7v>VPa8*cR>|rlj=q<j9L2t^_Su!VX7R4Ab+j48@iWOQn2lB_c?Xc2vUXd%YE$oDw
zf^*v%fo++mqZ!`q$cgwVrne#2I+{Ovs;82}$&ju<!>lV7iCnB)G97JVKR%^hnYK>*
zl}qOapI=5QJGR_6NNp0~^ir{zK10RvvI56!JC=fSg-wfo`71Cni!?YW$W7v4WQ}d+
zUcI}4p}~P;XVZi|OsXsjoHrYs`($<1rf?tT@fFd~=9=%Fv0m5gWwue{q6rRNyc*0o
z!soVp(r|0sQ{BPTxWYqY^8^#?i(4<s{VG&Y<u3_fP!aM-;ttBPd~tY+-JP{cH(5?h
zTA+W_<v-s^1MMv3rGauzpS^F3MXjB?W!mJ)3t40yU0nE*<wCInr`n7J=5`0Cj&7r<
zN1C>mU4G?VoWAS8%Zu&Dr1=%Sd@D}wndkLLNrQ3Q?1m<L>rH$s|DCc4)!OjJX=>=k
zC6BUn7(aM9OuiPj@M&0B#?2$^LZemBhl?<JIB==|6>8}5Ws}nB{KdoUvvEa<i-)^f
zY%|ZMO#RJgj3)Q^uGn&EpKJe?DJmC|R)yJ>7%zMjC*5Iwn`Kh@z4iuc4&NJB&VJw8
zoMY{`r71%4t%4EPs^injgtC_Nsv0cHU}RNt(+FwY;oYhBq2!n8(up==9(#>;Jx=KU
z{%E#8<DW2@6ON3WJpaYKGz6<G*X-hI5j~jL{+6p@wO0+tn)&H&8m_#JZKoV&>$_;A
zt(+t$z)_-|)IQ~T$}@@AjT%orZ$3Gt*N5S!im*dus(3Wd#*JHVEl}Fl>~i9SboACR
z2EQ%8Cg!bFS`gIm^wgJ2aVr?6XMMO4G%5V7Ve{`Ib8}X|mD7SOgbu85lr<GuHC0L0
zW0|}4e6BsN<%g!Ic=(=<P+7#Hk;P=cx_H%<HL2INUL-b7&}dqnzWkNL>K97fjT$+J
zr*4n(2)u2|+|2eNd`DYsA&;)a?MLe`igiDVJh^D@wT)dYr*#(YS<V!H`*=-C&Sq6k
z)|<O8xP%pPe{{U2)ycsSyjH0)dh!43TMsQ{3XVIzu-D;+3d8*s+~y3cPZV|}%LHv$
zsau=ur_H7+?)9`qMby$a-6vUSazVQB4a=E_7by$;$#|=%EOg-Q4E5E*oM$ssvL1%c
zYg1BjaTJcaz;bhoLMw;DL;KUl7e8^duq<p`+NRgT-J&7FsK3#fAt0ppR^77QqQ_J_
zH9eEWWUnk>$zXKjJQ;Z7n~Czq(zn@@6Qz@vFzjpNX~}JDOm?y{YPFb^U;6ZN*WD}b
z)4zp2iQYJ=W6F;-o1?Za=CNKK|0l34yOWY3l5og(n$ixvMvWYWBCVrhY7g4J$h<06
zjy(BXAZhZ914pLJ;JWMM!zALj@}0(%$q}I<7yeKAKO?Jrr^&^LhK7dcdpx*aTw>Z)
zl;NWK>XXYpD<uJsi__BFii8*6U`YyjdPqp~bLd`gkIsZ0z6=?Py_}ubQl11$99h6^
z5OldKr9@@BaH6)`lVaY<Po_&0OwDfQ$PBYsxM_bCZ;I(hC53zzF6jm}mwGPA$-6|N
zqzxsP6tK$hZQ{B#i$yNSkIR_juvW*a3zZvqQUoN+os^t@TD-A5fAjnqCI`i}p&exg
zz8>eJGNvmiacHf+HS_F(MGIDc{WL@3RjSaj4V<!Sehk$YABs&VFqxKW*lKvBdsT!+
z$h6QSU4b88*7GQ{PE~dADsEg=?>FU$-mi9N{>U@k->!s7{r}pL6WcAAu&iN^@YRXT
zc{;*(U!7F(yO0}G^JS*PH#UiA1EHDyCW1-g7F;(KzD^WeBkru7)E!ok$kdrM#V0PP
zsmAFPkF95HK-VFw$ZKy(nGU2TMeg1`Pcf0lvgOV6h>3?~wtdhx`g1iRwpD44p5S)f
zka=d)pGD7O3!N=`Q!{4!hSn6_)2s7+UnMR1I>GGJR>LP3wVjMUE88lq_Pj4EuJBU5
z@%LBmg-jEA9r=&X@m4D-UlWkv)t7A0R<%+=NW>$G=MtCSE|!#eZxuFmY-*k4rRcBW
zd!Tle(&mj!g00hk_PeOrD6K!i<8-&=mEr3}hwclRJ4Nw$H?tUU$@;DRzfHBT!C{lv
zyr+>Gp)%e9dHz#fc#r>BX|&j<yGKfr^Nh#b%p(_*j3!I&KK8Uv{$od9*OlXUuNLn)
zHT#lS#KdW3x<Pv*?56&gsFwT}+G2i)Q)3Tb)5*Wz-fj}kXs)_pKIu}NiW0}lb6L^R
z>DJ1<I!B~jMIsoF>aT4!-;%(ty~uz~^UjZRsakp=EfJe@ug!EkQ+=;O{qYjrgnWY+
zbFxb}E;V~~O7P0`i?7QU9=fi~s#g2J^~CmwT)mQ+l8SvHFYGU`UK+JOJMML=_{?it
zvze_XUFzPPuic`2ze&Sg_j6Wj*1ob$lf#y1`btLyyUv*qp_{Sn+O(3Eg;j5vex7lN
z{$KNPv0#Yrt{3MjPdPJLT+P%EayhtS*JIs+)skDp+)dk@mCd{7UVG2@_xGxvedeti
zR}}Qkw0G-iWmyR}f7rpbB*$=)T+GQC3$1Ng=9P)XS3mh=q_y#o*!N3Mr$-(=ac#xP
zjUS?7rls><cx3y;>+z!EMgF2Xvrhles+MCF30D+|5||?DDbboD;A_3}LC#^1@-1f@
zt5;>8GdLvg7BElvfsW7j0D~US#S@Ne&iIugt~JN;Ug_1Y*pQRLT95y9*q%?9RDZfz
z>63Dp^eaPwt3N(3D9r42`TXu_<F=TMJEnbQV!z~Y&*sAi_K(q<iep8yq{Ck+_SwAG
zyPU!GJ}2z>9K-+Iv*lt=9G1B4y56k&VeO-Ko+#m<8%Mg<&xkp>e93Gt^^b<)z9uJ?
z1MlDbXJCHOSE<RJb4~Cpq249lA9w1^S)1eR^HH#AouMCFE~ldP(+$<`AEt`*Gv9pY
z@T20@5v$I;n1lQqJrW~#JqRxhwey(H?7wWs2Knnuo|S9(U!J(8@cT@7)Bm&OC;uNY
z>$t0uxSy?8w@l-n%lZFX)_#gR|5@Vfth}^{B}-nN(ahP@S9K!$s)z5|dE7aZ5@!5S
zn!Bny_mx)8Ce?R6{(44zJLQya99nL~ks9?N&m?MtUKi`NCy!2lcz2C~|AR;00+HkK
zkND$HcsHhPFw01e@5}WQa59znf5<`ZJ!AE&&ly~w1e7l2=zovW?>@uDILnJmD#u|(
z!TmG5J16Pg@8Mn{bm(B6fA+Q9o&zgaus8_Ew5(+6OHJedmX_Adr^5Ko_xl%v+bLdQ
zS}hKf+yvD9`8n53&ugF1cD`o@zhXj){vnMw8@hMd>AI;E3TrUMcAd|8q{Z@7EBD0X
zM@~w84>*MvH|6a*@A9Ti^z|~O2*b=Xr~fdyYkTCKKHxJk$B^T>>ZFQ-IF-{+vksmT
zc;r=J$Y`7r*0D<Ao|2^WgqKBLVGRykWgfyw-hA$gUoTD2+i}u#N}gVgs?^2;o>DbN
zfrR{T$5wA*H~5#NV3xS&m(vM{4-t0FY?}W!$}u~8&NynYVfEcbO3~Z+-$X3kJ!{Fl
znwEeI=YH=<U^_g~D{7&{V}GU8_P8A9C%^ixvnVxk<ePAH@Ma||);RCwN&47yVFC*i
z*JgoNHOps5UHZC#i=jb*$*fT~s!>+ufqjprRDXu5rO%F<O<cPtW-e37DLJwFK*08I
zybr>jNiWvpx94x=S~q=Ox|h;{o+^%C0!bp@4tc6f)YoE?4-C*-_B6XfC$eSb!zIUl
zvOJK#%2}hr?Ej2IY1gtot-{hdIeKw|it!oWcQv_9FjN)D{t&{axQE4O#}Xfg&#^ZS
z{S{esDlg#Xv5jjYURSO<a_ZKj3-bcYetp=%=52h~P-oBov&R1q%cuu%En;}Rq`hiO
z>gyYQdmafgM?TR$%_7IXsJ*U7xA?RC;v|vSfIgFeD@I3*wk%bsZBaaVI(k9Ue4%Bb
zDZJiGm>T3>Ezk;Dm~c3D(ISri3k^ldH^ub-S3J&)TDB}|S@)3-t3EJ$O*?JOsB%YU
zt%JdvvYZP_&Py2kmtB-P%lCJ~<5Q=WZaDFROC@vOI)3HcZx)m9Z|QhY)h4)lQ{=8L
zzTzjYQ&^uy^f^sreR?86#9)f{WUh4yF77s0s$Vp*DH}R5Ivbty=(C$>q}JE#5vb*%
z#`x7pk!7_K$H@)zK3iT^b!NMKO`y}Dop}?RI{R7`rSGQ|W_&U}%Eh?kzeC)tm8Z&#
zJvXlXd}89KjQ->UftNnLshsoFU>ifv0gvc3?ni7+KHfT7b5dLmPKwfMn(uHn;#nc{
zem8?XT|EtxJ#2C>iv*SJaPWWJ<R2^i=D|9pj22##F7|u@3zbt7fAA@nUKG3RuPm!|
zO+Z8O+3FzeEX59GmFR<Fuk;kxwS1N1KcukUmQio^ijw<HOIN0BGCR3OcH4sKVN=yV
z27eFX;4*YkZw->+f1<TPQ*Y5nu8oh@SQR@d1^ftU`@yCxJm3H7tWZx6MrY#|MIrU5
z=Zj1yd-?30<g4xEvwEvs{1awPCZY1+*Vk6$Zpl}ua8Y1Y=$mmS>B4ixlqsQ4Otn`0
z&-G$kEVllPyYyuDqbd_$tT$v73;Mp*se0mvSFO6&m=>&Ix%TIo$6^!bWiIU=3kz?F
zB<WvcWX}$3W0jqKH2G%XqeDdsx3{n=^C_A!?pF)ZzAu;@Il<`1#}m(#(&{h&KBXe+
zf1ySv*~36@UeD&V@LnyO3Bk`7#eY^$`1<t}i&nvx<MC|)%1fEddE6BF7Bx<}5PZ35
zf0QGWNXxBFO`rKnv_%3mB@Hy@2h9)h-+N4fQ!VFq-@5WuTA}9y4(0GOGH&4#7jzaf
zN&ldv*2o{Zev<1_r<DDhR<K?7d7kwDjL6DOrKuaI`g{)5)|pnQEEu`(m_Z%SoTQfd
z|1S0&NK4cHucTM$^81sQg{7O@*Go>Xl1=smU%SBX854e*#XU;))WovK7GjG$7*C1W
zZBJeLar>47lGWwqHKr@hY)qbebdHWe)vrfx2`SabmWDn)lyrPc(Z<P7#mp5?Z7E+Y
zZgB6EU+^N=qefQ@oVAUd->zcS+O>N3npPgZOD#$bo^Kf?svrBXIjIOY1ukBcargE7
zAJ0$EX=!wrv}j)ouSIB`{Z`omA=e$#j{PgnQ8m`6o|>7^-}pLs=c}!5OO;l$ojw+D
z>DvLXH6mXgw7u^s%eXA&Hz(z@qFG7wYID_7LH}MX^4Tb^b<Ck!^ZT<Wg00;_c2_+6
z=jj_C3RiG3chOFY)jnJ+_<zyw7p3|qokP}ccs1qdb++JZ-?n#6{1(XneuCeUQq~2D
z&zFi-gjXEjan57%t2KNncMY|cm6t^rPnY3ddE-ZtD6`=+5zpwSLc-lfZI?SPieCAo
z*2#Q?`82;#re9|H2J>(361t*_i@tHN9y7n?U}>{8ewzc2>J5kR6Sr&FKb46J{x{}*
zepd6dK=}I)dh49LW_e9*yn6B1XMWci+5K0wGFe{laVzA%?z3t8k*HqQ<xMF+J^S)2
z%Oy^?@s?|EpX6LvX?Anf^uLd0C~x#pEisvv?>|9Z<n+7h`!5u4JQMM}e&B?~_X`^@
zoa*1(cWusj9iMO0<DWiqz7nauiKSfokyqJ&@j0(&>KC4pvESxhs1j*0VPbOlOksl=
z;qTgR^II^q^QNs{FuyR$!r(PmvhT}9)zezjEZJ&ac}AZ)Zhh!z)Fg?z?}x0pXFhpX
zuJm7hV!cSh{xixiA1QNJC{A(N$0M;Y>tU$vjb**BTdXWKLyPPurZwH&+_e6M{g>kV
zd>^yq@?E5kES1_=XW+}cMBOPcb>VIPcKzn4<jV!g5v!Sh83}Ml{Y+|mnzV{B)MYZ4
z-2P`~iye9*#2!f`bNK}P+A=ft>8JF`I;vg5T0JSXeypY2nK*fv&G$_Hcy6cr#)@wC
z;|Aq|3u=4oi<QeI&FXU$XTSX^dG<${Px1WUmd^DX_h$TGJ5lr6s$eGHuN8X#Dzy&j
z_zH<@vE6#(*D_yuaka5Zf8fS!wck;HB-Z_yaplqSg@Q&C+Vh2PERZV_&z^VLa=940
ze$i8|^95nd%OV{U1=Mb?S+i`hO3v#;Hq-rbmYSHDJ}L9_sBD|cZ)oAvbZo|%YOd2~
zJCbHjNQ>k$Nia~jGVjusi3<;Hd7iZ9<)KRF-Gvi7|K|LNP%AW>JoA8SS&{7a$d4b_
zTxK&&Ye{py(0-Rw@koW%msuX~-jtlwQjExFTDshtm&u~bP-%U|hV2#xzBbdnqmHis
zp?p-O@KL0KSpK}+=;<dGt$yWm*lV)4u0dqT?Tm>ohQT}bHI|j>e9PYV-_;^x#|CEh
zEi<jXBBSN)o$Y45H_~Wk`8Xxn<Ls4LJ9bTIHM})_&HEjCua0NG=-Uz6_v8V0rtsBC
z+cs}ZKkLgArf0W_*FmO%|Bzh9uLZXfc@7D$ZeYzxW#jpu>{@ej&kGUNwXYPzrtY>s
z-%(`r<JEG<OqYj7B2nedfzb;aJa#?3GgDWBBPn#6!UscVX8w(zqNBt)%{~hr;Fi%@
zziZ~~sVA=ANYHqE+-72`;Nzca2iLNy=2krEds9`jGGAdcf84^R&RXG{JNs`Qziq5M
zS83L2YkyXsT|WX>S~(XqKUwp{ptE}3!awK!DRV{ihw4=S;8NzQu$d&(xnLbjwxW@8
z{r?MdE!*6e@48nTv*!C&U&pzkLd!EGJsUk1`l!yaIbiF#XUd#whS_=779HPI6Fob+
z&VB98BH;^9&a9lvzS6kBBPRCW(}eJMjUrt<N*<H4A2uj&l0BhN(JI7iyLsi27_W#7
zhitcy>eSHHPL9$2N5UlS8Nwgx{hg50yld}`cX__&ia#9h4GX^g)5ziPpA9($hphAh
zV$1%S>TQ{NV0D!CuBq1)*R61CR+^Tdv+k$c6Sp^uoNlb_@6c5<@DQJ~mx=R>%dLw(
z;!pSb{wlrq{QH)Z$qg0EE7!Qp`^|RWaq^wbZpSB=8_emp{`}g=#b$Ec;g5m0mTZoA
zpZD>I{(Vo0Lt761pZjWK?P|sDBizB=-aG|UY&>_q`zh6zy=Are*N$WCL8lA1^`FjL
zf7C{GM$p$24Z)IW1}~rPke<D9>f5t3o8M<_SgQEyoUh$6W6h`GX=}<ZtudLge3yq~
z+-mNA(@5>R2_~V2PmiA2XYZV_cFx1ozMsU8ewDlbKDur5u`hwC|D$C839I*rNnMLn
zb>2AXhYX*eyq-2IlY$w?4JB6xM@9}|EtwqxitVk!0(pTa5|y0vc#XB5v;-y}>Xc<|
z5xF@*n2AwcdYK3VOJa)(J8Q;7r?#t3vw5Q{uXGqE`Uf#JX#JTfxFUejqLfecK-%-b
z)+Je*xf$J4TINP<R?C?w>cI4WJ!hO(r%~}~F{8y16SKCOE!1s!dvBj>M0R5AF3$H{
z7fy#Ch*?wnG3#yS<PFyjb1zp)n^ii+$*5Ow(#<Uq$(LLECZ-*mDY@|BX%1`UB{4g<
zyqr5*t=~~2)qB$F<;KT)d}k*f*5oWqzk0SH<qb<<l2-=r7P(8ex`fr-cf=$XKev^P
zp2oLj>bYgPED`ys`pLV)JcE=1XD}#E_{4OK`Gm#3hgAydf<5k&6g>ZO1h!@0+@fl#
zX|^V!JR-J=^MYS)rDn;4B<?hmseb%fU*;6uQuK&U?@OF9A;UFw$wDWYs2vjmBiCM=
z(5aNf8tJUfwc%DuxS+~`Y%bp=t+~v;OO*dRGf6&4cC$?r37o+F@K|)02iJ#Cr`MV{
z1DJ$d5-+;XeqnOGWuB~fqVvVnBVM|mve#5YI8vYA37YHrG%$3t;)|)_938G<*-<`j
z$D$l+f2QWL?rOan&-Ha?Xo67CiwD8YKT9O0&AzL0cX3Y7rKCyisXHF}CCjA?Yx*~~
zg?3CybD0$;ywpwIQ9trV+H8-uBeV0xQ%zDmg0)z*b3HRu4n`Jk4L;o%t2RBec;>Su
zH>Qau-BilR`J<lDo%<rCKZfh-Wv|&gXKlYw<#8hXe(g`sg{@+L?^Z5v@VXo7-^i<{
zk<ltAx%}#GDbXG8_Hb6UsD`bbq1musdEUx^V6)1qTmJ(Wv~@+U=uqle<dmG!mUM(8
zFy>|F*4~XLd(%7<<(<sUj?8!2`pDwx#Hp?-45v5?1%*VX-d!cAzO76|s4C)NjocJ@
zO_ie(S^Gj0HXhgAdSe-rQp<{qXLO&uFABUlWm$;S&7!r{Oa~(8v#j^L8I`rz<hEBs
zwbB>y1(Jfd#P|N@+7r;cY~fplJzFKyB64!8cTPCi-&%1cm{VV7mb?6>O*hlLm9Jd6
zFV(v$@FbUfS(u}&`UHm6%SBr(`zC51*>uv?h-K0X&1=VJ<*Y27H@PhL%au7n0&4RD
zx+j?4p5kY!m>R2l_vGvg<w1A8&k}hw&HL2clTI`8ZgC1(l;;-QXWjlkbDyz>?e3|9
zYxAAXT&k70zcllN;tCZrm3<7;%JhCG3b$Kj{+h(eF!NZ()UBM?Elywg-1}tGzQsMW
zgC$*rlmr@WRLqwI&+VACWFm{gffC7`f=|r?*Bs(dGqibmr)81y)|NF-EyIlG+_)BG
zw*SY;98IN=1tJpWhLWokznq%H)4j~R)wVU`LGr^6yD0|^IaxI<W#;@WSur(mY8TJi
zmor!zHdsc=K2uO_mdUO7yfgCa8pWU~l1`gKr5>2Hgsz=<KknUD+qagds@pa_*e^6m
zwjuRsWU}+U{ZR+4cZROkE328eOD|)={kJZyyffuuHU#ZGA$dHu?BuC0lO}I@o7mbc
z_n%WKiRrqfT+k-7MH_D%Q8{+fn>VmWDe9t{#x$4ynSNX@Qoj9Xn5;SzT)TpJV#5;L
zCbjjrbZ?FE@Q6`kn5DZ!_2=nXlf#_m=d|i5Ds~_0QS+F(MuIIc>xu~PRwGe!H+PM$
z&Mv>C7u#A-thL<7wWQo<vboZ&iL<$0Pm=uE>ej!8-({YQ{6i_Ht^=M~hP{*0?Uq~a
zI5Xi;p$kh!rrLC6O^=53Rlg=`cSal)IeN*&;`W_YEDkfvgMtICcFvxb5iw&~1HaMM
zWFF~xQ$^xS-IZHyc$B_t`1M<?H+d1_A*XFD{`iO9()v4{3WA&G@TPI9PjV4jap}py
z1sv~Iys|#~|FF=ekjph0LEH;vH9M})Z_0UFqTu}Of0QOGqgP<VN*51C!NZJELWSxF
z&v3Db&k{|2mf^bA;&xox4DsEXS#2s@!k1s{y1pPFcR80%dh0Ki8!l$No~^pQ37T8|
z)+(%7&n(><peMQN^oI%a#au-0Ye-*EGEylG(d7_ON|@DdXHv92>!xVxOF`{TqK<;G
zJ<<MGBxmacSsEUl@+f1)jQMx^M2mc8#jGxqnDZ!PV$GE6mW<5;_vdagJN{K!a-xao
z0VQw6e_666HBKAcL)$i6vE(0^V&;FzVQbQdlEAck=^<X7Y;J2dYV{^4H*DBc<R#oW
z@mUs&3e&G}MXld=a{2xjAGp3MY&yro6CPnr6FwOjgsW$spFHWn{+p+kHF7L`?{v;c
z>dVTdnWr~$%0AZ;HTY;8`Bo*rA<KKKgx<H-^w6B1y_pNkpHAz@ys19($xlzY$<ya8
zS-CvecJukB!u5L{1=l;VcwLmd&8fC_q7#e2f`_LZ@`{%)og%WbV`^Ja*4n9xx|z~*
z1Pr+koCy><kkk2G`5&)~aCC!l{j)psttNNdr!{`=ePOdncMEGJ<D$pK&ZUoco|Ew0
zwj*<q_~+Fkrqkw@pY;k*m-O#l<9~AN#uZ_+vJ`KXS}5!9n(xT7YDrFLWWDWd4`D&0
zC5%}rN^A8#oXTM}zMm4AnCy1KOQPr6|0yCz&So83m8qogaE(WW%}uFdp}3-{G5Rd&
zr~NptXI<(tF`Z;q*(}g=+|ol<ZCd5+J0}acOCvlE8#F)E4Qo@YexdxkVg<9aQP6GO
zRad17jf|zX)-n`KjaO#dA8+zOFWfu--@7HsnO{8)*)B>9oBQh~Yxs$c=~EiC8jUV+
zTP|T(roE!{W{F;N+!>#rH}3SvY`^d|tK+z@+<Lop+!H;2neB=5x$e1tVn<J~M5?^n
z#-0t^kNddx9J_FR(ZiUht3KLx2$mnYrkQc>^qkj`4g7l&`xo?ZZ?gNbHsNSzf|0<j
zLr;_{g_OP}ME@|5nrGqE=lkZ1#Y)?>GV43%V<b=i{J-;7+OJEE0dgOYUXW?#(K>rV
z@bGtu?q?SlMN0j76Zvv(#I?hjpRT0-&YN{5$B{d3cE{5lVFHZS6JH+kGd(Dpp%Qd<
z*ZYKX22Ycn=S$1>i>$qU_myZyjLY+<%cT81N<>W_7un59bX}@GshIB%ccPAKU-taY
zAmdG|d6pksCdQDHlr*vPf!Z|ok9#77nO^@_bGf#MamT^MaaL1i$Y)z8CU{MAN!#^0
z%kXEkMbfQ%0_#i!IHMF;wS&Y(7)3XISoo>Du~pQ_yyASZV-0%)N0Q_0CehBtna7q3
zN?dA~$Yim3drCxMzMYdmm};#7qws$-ft+aqzg_DVi)+P*X-Bk({%3x!^7eVtlu23~
zg=GdO<l2{&O&1o?5YpWlUbJ*$HS?mPD~F4Ywkm98XtDDs(scKGVkTl{E^?sMU~_w>
zIJ4-yGKq|XcI}2*iwy*Sm}PxoO*{A9b%DCfrSw#}g+`CWo0uk)vWE$BOiPeGD9HJ+
zlyjqy#bJk(w)SVoU8`0Igf2@=3YXhCO^frm>!;<}k_WTDmCJ>w32J{*eaIGkrnQ0n
zhHp(!=fa={ld?vOX#%qqL>B(Yl>3zIbR;k8VyyNg15u00Err>9t@$F?q;@iion~oT
z{Y`LIv5(4&PL74Oo390ISmFG1V#i!#dCtb3qu*5WO3NF|TA~DtJUgoQbp-w2Ro<)1
z;(Xau;q9UZJrU3JBjFr}6rUQ$eJbl)mf_s1-Z<r&gm0^2fT{iBDOP{ovLX`&r#Pnb
z1UA_fO9=~xZ8c$y5vcPk$#`{LK<Gnz@^=+3!?eW4oQ55)UtH_YeN#CVCfPefFy^Bu
zt07ydkb?Y`=BpNyjK5VsD@*c7vWRICu|Fab;-ML-;qu-sZlcp936Cx%XEviljXNh5
zJZ1EW5uPG{qH_H~S1*I6T@$;fe6mbS)n2A<zgXBZHM~H`Z0d4G$G?GUR*t>gA7d`f
zD7>M;5a-z<XE=@b<5Z5OX^A_9Ju0NQDsrdqNH$HG9QAS1@`n<~Co9bf%jXu%Vt2Ip
z&pJ_CK3Ix}h2edoCD(_N^-r>b874$A_J}KH%RX$|w?#tzpsvrPnJHm%LfeGCiTliX
zsS-TXzW92==^3JX+Um@Hbg)gdX53W1EHOFZrh4wlX?NJBIn9`D=-w$_8lk_n!RVQj
z<&l(^p<N%I_)klk&i#3kkK3HN%H63Cm18EW#0fIgdsfb8?g^jnUDBx%`m#)jecsQq
zSr^!irYI&|SRokyb85`Za)*f7P3aLEGc(wgin$(ZUfg2N!PIyte2z1V{Zhx8r4Q#=
zGv?R6R9ax>73<Lyw9QqgQGnrqXzR|2euwI~3WYcY7@~^>?q08bd|e=Khwy}z2CXv%
zxjGZeJHq}~uUO=@so-^^@u%`dkrz9XzZrWa%XT00xEtBKVWlXWM};p(g*IoTr;wT1
zNyQk8{JG8xxETZblGP6?HnB4%Uim31w1DNhx%6QTrR<7+ogIt6q)jdLNQ_coTyn74
zVa2reh0CHC=TDjLo@b?T@MQ1}*Cp?!3HskCU-u%}@8XgdN0q#}3}uhi8#!i)SJrVb
zu`oUmJ<Qyr${Z&2a7F}UyWh{HA>}Kdt2Ya<FN|nh*zsy*X++P1Osh}s^A1eOH`1Ed
zE;vhYnabsKC5i8PClh71WM-^#UcDny^6k!PdrmpoOKF^*oLM|8uxO=ZWtD`J+GN*_
zapIpue5x8{+$@z}H2z=sNsV&>i#=17PDqsob5-=JwFy?$%T-v#mdjo^vFt-;@Wz8v
z3wKH6o?ORuMg86jkDJO7$Gg_J9<>mdF{kF2=8Tj%&sn_|l=!=tTgxpp{9>%Nq)o0V
zb0I^)%x~KTrk_-)+Ogv6#2IVaL>)i%Z!dJ%k<v6*m{IlA%2kyURJ47%D$*9*obX~I
z!?)=|d^;x?M73+|*5$pX@@D$x-C>JU76|8V%y_tK@n@+P?-Mo?!=%(sigisEGudL{
zCRI4|qR7_G%U55Y<7;NUK(#3<L7xAM)RUbKUk%rGcQ!2-FO^>{n7&%bMP*sU$E_*q
zN&+d%Qa`C4JGjizI{7N|awAngDfT7*)nCdghAodYG`lODzx%15>#n|ktJb{RqSXJi
zm`&RL{?Z-W7A}bQ5NNcRF?ZG?&FK2JpSm}!wr#(tymG5FU!!#9?^>~gtm}?4$}+C%
z)|eM_MX1NZXVK1;b9M`)ipw9mGBbK%fn4I$<u4u9X*G!nPAaO(yXxeTvU88;i#<v+
zMYn$6vsY^G1?L#;C)2jg(h!^E>m9hGN889tYJ;7b*0WZLle5-{eOj|rRWj0BVBJL1
z7Kz-6!E?_~cTiYuu(Vx}W#`iJ=-zNu#?nJFFJ}t8@tpSRRg7ff_Jo`3cCMPIvwWTE
z^8;T4f|WX3oW0jSedctuX#KLn%wsp#PjrlPNtH5ODfs{S(iCTPdu}a3w$!Oxe>8C{
zXgBU*5PZFoM}$GQP)=$U%i)U)xfKP1g%*h~RF*YlEr>8$DJ`vNn7VU!UDuCx;pk@0
z@<qFs&76IE=bmV(l`~iBNw}R63}?Dgv`cgE+0%PcWA+wi&G-DY`nqVJ+D;9TMS<R@
z=W~f{o%D24@A5eol}+KhXSkaPDx8?SE4umWFHs3@SJv>VoYbiQ-qWLlmzg}2EI)Xl
zUuMgN3I!Q;rCZw$E?ek#XX*U*-SbN&4|z<Q?pC_5L2D<IwWqJC*Qa8etxY>)zRbNF
zw3b7VMS4|02jfcSLWgJp6`2LKhu4TsUFq$+QXuE{&eJ=N9$4M6=k}`qD>UalRy=FE
zx@_a?GZ%f%dNs)lIZC|{ntsf1b>h!?={|dvCaufLSuK1?uWm`_e$6HOcG@(!#FjXl
zxT&>H>295>l;ZK~xS^Tk&(u9fl6&SRul0OTa&y;-)YV69o25_IoLt>rVzSW2TCk{f
z$^rg6%XoQW<>o3~X>yt>=;yV=YWc)1vNg*$er#A8v%r0+B9oQ>TA7_px0R}AE|^qk
zGdn=wvbpfjYXZEpcS|WAJrp9xI6=Vutcv*Q{(I9!B?=up-dIX*Jkq@}L9bDc{i)Dv
z_RZQ^Cproj`36mV_WVrxvG(+|O}s4Ec-c0`dK*oAc0lN?Q_h^=W+7RvFUNd@j~935
z{l9bgxO1Doeyp_5sg&SJ{Z^K$4W2fs-%HcKiq1YRWt6mX{R}}C%R2tAM*DW0`QkS3
zu(Gi2X&GzQ>zYcpjN7I)JhCoWy#42*&M8jCx0F50VhU7i`p(?Cq`f1+UrST@u`yd}
zKaWb{(}g=#vgE|9ZQ?>DmspqH|8Bvz@Q6!{kg}k6h}66@IvXVgXYmTMY|B2&>F$&K
zclYg{u-x=(8B<x(3iawXs$2eYdNSwOHp|`=%j@r|W?zV|-l=+jzK>$p#DjY~lWab2
z(V7(ZYPC{x?1mT3UF`4teO~SFI9OwKGE8Ri4Y|f^R$tUQb8hnV3Y_%OE}VOlQ^07#
zNA2D#*4%Sc{tKPCxYTyijm^P7YA*#J<XSJg;8{jy=jjTgm)Dm>1U@@d!+bY?v6=8?
z$&HeNS7O%)hM&pab6Z7squj1LHywLK(@ewo5);m>wR-XJh$i>q*g31_W*_xG^fbax
zE_{tj@;XtA)pcJtvQ9GM3U)O8`ha^+xzwvS_YA-3UA+2en|N}SYz@zk<XLG8lyojn
zw!Rtd5m<J}i;Zo+{vH9Tsz#2tC4Dkl>dh;68BUnwcvaReS$^K-{l>>;&Qn<?`~0kn
zbk)O`){UzCO~OCdT;e)nQE}7!g~f{+NrUy<A2ZGEPyONQXFgYb^%<S0{kIOS=+tzO
z=eoeOz{r=g?(E(-%L~s$v6THclG%MBX0z+O-LrZ-PR>|#ZI4h`<lDK{%YHvhTvMR>
zy6ZrO&FpKl&pn^>?nRx{oDQe`S(pCUF4xX9c=h^vn0MC%=Y4kS0smRL+W$S^@w*^*
z__f(?Ien|@YqJ=H7o5=QP*A>A!tup&TiCYwbs2Fbi%(rtP@gCII`e?)zGaH~(-isz
zvN|uO-kkczPx|Z6bCyqS3|pUH|9UJ>Bwwzk*j(=QKg~668y|~=rU!3eb=1#wIe*#C
z$x>UHb@#tR*<a=Ot0ZpmHXBE8=C9rq+P_I`m7q<{!(|t1Psx3ZUv@RSc60dOrYnA%
zQ|~6cSjb?My-B!p^FF(?(Ry#M&HWbeD1Bp(dvtt|_>N!qcD}OC+I*{V{hn=J8oR5)
z-pN?1xg|Mr9hckqJmZS0=}gf#)(fPY#2i#aaup)~KYXy*<inY@FKjI5dfGm`{<ga>
zckWH`(go+E<bMb3a%9``<pHmt00V>KPZm}Y1||j_1_lO(1||na29Ey>tQ;aQvKGv4
zOl8xGIk7?UP^-!bgDDyfQx13YxpQ97xS({rRbG74O7#Vyx~*@Hw1^zYdtlYW6@GAz
z=jLT+XIm7%lCkJpetx?AQO*eU-~&4rNwcpLG8K7#d>+%nCK(Np$6k%$y=G^2O<t&`
z%N4D;WXFX^N4irIXNewIwJf41fA^`-6U!23Hx^27+oioBG@3Ez7FXWZ=^t**j<LA#
z#Arf(GGiaBn#hYE3l5!}z?v{`=Cp_RyJSWGJ?)z86BV@WOIFm^A_d)ck5VPkJF`O1
z-{39ezIr0$;qm)SRr+hJK5tIA&J>sY=goy3PAiKSUskaT{^9fJ(9%iEc4moOS|oqn
z@PBzmep+nX^QEif&+n^^4_fQB^^r<~_uq3us(ErZmMSJ6V`AJ9;P|d{YkWG>RHj)B
z9B$h?{oZmoTyogD!T6ZQnk~~VSG+5I@vz<OtI8sOch{$veJ=!Vz2F^``69_#vyVf`
zZ%5*kdw%9j*Dh3^xhi_YK}=TdLgBNGM<-0MdS<%Bp;S{-s$-3STB`PMPKWtUOz);V
z_329#yO}(F<#D!2i8EENMe$xz5v*Nd8q%$H?`F&FHm0i)lZ%-nw+jSSUE6ZkL1v@U
zoPe@-H!s8+v2uxOFJUx)Tve~760@Y|S>*C1rQeP{bdZ@gBP7Cn>!;bSxiekU!u{-4
zWkvGNjY`vA;`4u7s|&-e;|fj;%T`WU%Xr9T0mHO~y8<LtRb1XUsH)tMb5DNrC80%7
z@5-8@JDOXwy41K_=7ii}-1TZZ*Tu?3fwv2#g}Y?4G(@*WrOv)w{(!IKR#BB^OUkN?
zxm?2Sl`pd|7cvN6b6aD~S+SIXNy;YR5M$oX3)8t+UtgF$dFw~{L&a-dx!R>%tpYZG
zva<-t;tG|xBk?Hdijs%x+Cz7O?{TTb%>QWgBqK=b?93~>tdIYB8D^@Vec8S0(x2y{
zy5>g(b{kvHRId8Qru+5E`-4^T>$h{cgf;VcZxcxVSsR_|yld0;Nvju_UCC){jpG(R
zD{lHq+DF;Qx7=CNKh-#F#<%|}=d>=QrfxYiO~}t8_;y@Mxo`LG>))0pul)5m-v52c
zs)+hro*B=)&BBG2FD$KFy#MDk6$8!G;JX*<`buA3NJ+8Wv%odEY@fisx<gwyPx7u@
zawX&Wt#y-+$JVl}e<drlf2x{fq;+!GDd|_nFRIFq3v8YCJ1yqRiSqwT(%YFhS6|(p
z_cmbNrzBg!2Bt}0wcTdyy+7d(x2m<y1c}*yi~S0HH3BwoJQ;mwkx|#;vu72Jjb)sV
zY_OkokhhWRj#sGFx6Oea%8h$&&UZReC>m%}kmqyq!C96=4iCP(Y--!QU~!d@h{VsH
z*vYfvc2D~}Y1)<+Ii>~2J^e26Gk!@@uody(|NmF9L9WS1;o>j3TL;?rw@!Gc<i93T
z<oBZ)CpDLJRb)9&S<$j<-G+Q6cHxGk4Jy_z9eRw?J%!&nEjQddmCYo^@TSHVg|6eC
z>=qM~=BagUb8kGLnRn*$0)}4|!tV~MOyEuIl9O)oSn#p8DS}hQeU0*Vcem%wc`Zv4
zQa15LZCLE|b%8s7bBAz{R!heco0AS6pJpC@(yBc5^E{nER{0+W8DWvTMQ_L*n$R74
zCL;2*koY!*1qqs&y(WPIn<N(buuV+O3GVh5y1A%RneF*3u9n%R)#d_hMy-uJF&@h7
zmJ?jHQs&gDJhU(s^n2Y>toXiXdH9hH=bWcqR($A^9{$Di|73|<E*dk|a9U?HIy0`=
zB)4_NagF8-ul67QiYl2I&hn=OLf0L4x5|1{dF_$@CBc(%xstLmJzw_Te#q)$@zLA!
zmcml4)<E^tJsb@SPR=vFb|iFV*HQ<Uw#m9nR*4n21lw-(3<-R(*`HC=Lrtt>;!>@z
z^WrXYDoOwJTQSLaRo$Dj0kfx1p5PJGHmzrYbh5KY&kh~4FALaRRgYY2`;xrY#b&9W
z)GHSiUt>L9rAtvfmZ$R{z6_0BbKW86s{Goy{tB@(oYt!t2giNP;u25pKXRdCqG6kF
zXy(TV?Fw}%%?SYvp>qWz^o*`HT7)tzw_WQLl6`gNoUr+8O8G<oTHI>=-($9I?bPQ^
zE`gzC=aw|wQ}hZnatUbSYH9!bV_}Twq8Rx}MQg1WuI4W`^4+;)oyCo}r$n}T+fK52
z;yxo<GttYjZF*8>BJabN!>2T?j}|B{d$&>3XO^Vt+!g%FDH_eL4$GEAO<A=q>4Zyz
zZ}%-mdr{>VYt}Z2PGV(rd88C5;nBisl*kwop(uLpLyw+MLXXLeL(D%5e6v;yY?yFk
z)wP?-b6!QbGe-35&RvtJ@muj!+mWpug0btO6N{1?zeF(Dr*)KgRz6lRR&n~aK_ix}
zXmN1v_Vp*E<zKyNK7C{2+dW24Ri`ntL{3?zYaEyC5z`d9G-Yaol8S)CtdEQL&wZ!z
z-&E}A91)w_0(Y*Ztn7%<%A5Wxq4CR2MJ2JqPVq^tJ~J$iw22+)+~0cJ-n!*HZ{e4P
zAAcG$#$<@jD3i2~C_F5CZL#=#y+tMIOm`J;v+NRJT~t+du<J^!xSHiEZx2iNowBX#
zi|6e2sJird=B-qbuKCfICLdOkn0AQE@mZ`WJA=yoSr@iw|NhCd#z@^}Ut6A{T`7m=
zmIH|b$q`vic_$WeuFaTbBBAqKtb98wLu)BN3l|fodE3Lc6F9l8gjB>nm|9;m3|rGA
zHrHgus@j-->3%8Y73(@D<|jRA^>){pGgUn7#DUAfGr3MlT<=ai(Qvv=!>oQ*Acv^H
z>kVtC-m&ZX->73KpC&%r<aTzKXLshMPjBXCsBEa&UFMt<IBy?M=Q;UvE(;_dWwi0!
zJ<MD9-B0V!^V0u6U%n|TxRlG(S#I8PZ=&1UYpR-E-oi^Ke^JPAQoYjuL~Dt><2U&d
z^^yX`&FT+bZFa|tZ@Lz4edOI&rs;mSH?KT&<oLTcttTZM^?Vu&8fJV*J#qTF^R%?Q
z^3FqAGZ}(5v#PvFIe4$FbB`#~doFcHhWXoTUO4JMY)z=0_My#5EJU7F_r=V4i&rSO
z8C*H(zbbSJm#)Qx(nkgI@9%xLH2e{B>tA&!YoPMgua$lZ^FL}_pY-^;Y^CqJ?v33a
zy&NXI{gm?F=%%)rO2n67kN>ld>o=q<txPl(zvXjFBxmE6MH7|6c2!N0Yizx;U(sX7
z|L3fe{$Jr%_%C*>@_Rdrz=3+Ezy^VWD!#ym@9aqv12WbvXMJ;}^x)CVqf4Dt1hY<=
zdht2dSUt2axf&y$n03}&c*&Lkm91fd>arC=K}v=>+mETZ9pb7#D)UZVV)HhIwT}d}
zL~~RHRo5istZ<fRpPVEADek>#MEmjzU2**rDmFfCf>(|Q^KIm8c_=XTxWK3D0wrk;
zObV>ijtfi+PfXM>jac3gcD?e>v#Jg1)w7>?sv9QnaCfnN)@bV3^g6L!^@XC>G#4dh
zu@r;qudONuJLJx^+te$j{$PxcRj6hC|GBV9)Th~0plfPvbdlQ(58=wfx~ys5JQD?~
z#bqZv5sP2O$(xb*RXlO|3Av1lQUQTB)*pGVr!{QUVBla#WGbkiD3B<m(U|1WxF)fY
zxwVlqw9NHN^=|d1qV37n3w!d+3*Ihl&vh-Y{-M~uF*fO<%Dl(1cW$^Ct`N3Y63^Wz
z9_SROAKJogToL_D;8jILQi`?=LuBW2IkCxxU$6FhKhwRkT|0SWJktiwZ6adZ4sazm
zusr^j%Nv%MaIv*SEKz1#>#K=XtEaQBU7iuiP`&!0>=yR+9pA%BB6^py^j=hzy>uf$
z{%6nI#FY65BZV)iq&?JMxI(V=Mom{lYWG%wOaH?oBm{%|R3lymcg9>uz4yGf{G+M}
zvwe(1oy2yR4;PEMH*$X6UMc%%0#9J~Dh-CM8Hs@ctdSoY+RPfoAC-!E^hiaNWgJf~
z-{Ez6#x$mhjogAwE7e@GZ-(VBomTX78ei%3R~yTC8Oz^3oX&Jn=*h+kGr`G=0xf4d
z<$mXIHLFlj5%vDwq%t`<=UIko@Mn3C6A~fQS`v;G`TS5ke`Th3lTVHT*Otef=RR3_
zIZf~^n7~_D@-(gVt5~VxF#)D9U+IT!|G(uoMe=jCPCC6J`O*&I6B@B@W^>F6o6el5
zR@f=LYD!JpSIte^ninPJo?R*2`J;y0k!{oD$$c+6xIPx1|9{PtJF!z++)!LmtVJ?S
zC|vl<Q^ngZ!XF%k-hS{3)MRKXOXQVcj96acl%5g)vwMGpL-vBq<VbGTrCI5f)3eg2
zn>WtkdDvt2us~?l^dsRZ+zVq0ONI6c%vpM@=aNPf&sJe!rG=uagk`H@M2Z|NC#tm{
z6c6{vnLAOu{Y3L6!M>kAnjU>E51AqSmZK=bsG_yqAVjSH>qf>;9fc>B%rGw%Zu~0v
z!-<V8a<-CLCfCP=C<8WavFf`I^S2z}THwOJx1n07sfVjv@KE`(oX+V7!fR@ea&38F
zZ+dDO_rrpU70Jpk<!qPfue_N3TD|%D%a*NACW|=EGvC?wlvDV(N2lZen=5>O)Xv)}
zu*^W1GtvHm(Q<{0Tz;Ab(t@lNZcef8jjS!1hcwFAzRp&TT3+e3yvj2x=jwty_1L^f
z;Tq4?7jBgO3+4XzfPbr((YFa&E>jk9T`XS4GNbxN*{R4iW{2b+e+;}Y(fa$7N#_Z!
zsfV+jg(QwAu03L=;;_QCUR~gYhUtuzGe7QV`5YnqsdSz747o+0<ZiJkiG38?bC@gY
zK&G0q;!bmyUFHR+Zmw$FCAYWT`PivdvOBy?ZiI2C^ZyT!dpBKbkMV@~Z|hfWYGVHq
zu39R5>DwBi)g8BQPUc=Xnd!q^amIN!!ZtB|jOe{pXs}V<%PE-Cu>V)7_Kd7z%ZqFm
z{|kw4O%UW)4OFre7SIw<j}lhP5_a4@N6V`0q~Z)`MK-VMGRJKTSWgQGe-h7qzTvCn
zgrM63vu|yVyuFo=F-zjQGm{F#1x@jkNi*gst-ZT-Tjg_?*ub?-hH5Q=ai$*^b0=<p
zow0(ksfcMoM07#LoS%xR2Uw?73198%x2h0Zy0IevW1Uyp7UsldtG*>q(w=@SvPL~K
zdDiY_CGOmd47lE;Wbj31h%$+-HrP-c>B6Zc@a2HUzt1i6i<3VVax*{V7TdTr;Kj1w
z;^oDx!eU={IA1c{y-itavepLejW!Y9X@RLc!oe!11yp|TnV7L@%8l)ZRTT8ab;<?B
z-8*$WEd{3iH#JlU@|3v9#=q1{>=0k!(Z<}HG4@Zm|4vvGoy~nnOL0M1*|`}73Kgr1
z16QAvEMN4Y>*<$n@k3lH9}^ZSG|d+)<GHwN<3;X86Id5th*`a$>WhQR{|kIe0^~}b
zatkk7$&|=l_>lj<gV4JR3-?^+n%^R5AKkn1SM{$;>%>#Dr(B)cwz9(UN4A$xU-BpM
zr9sw97WArK)vtTFb}DD;jLJ=Jx8}?*6!&+r)}L7{esOViu_cpwukEo7TnDW)rFNdZ
zzWz=lOP~Rlu#uwqF^8xJ_G`6wiK(%zzTmt?AaQX*&zlPgksG)QRk>EVd#x0#TPd=2
zTY{L<oA?vI70t^w#Dt2i{r|;aiNJchjKxXUww;|mcf+T7&ys~N-S&QATHEt#d-SBb
zDT?uaNh_yCu~jS=*0XZhRkm5gX~G+ZZ3cn+nG@MfCN5(CeyC90MZ`*6iEE3xhl~9c
zXVFD6uLW2G8bqA0O#7?A#Ua4C%7s5kAhE!R>+Kh*#SE-b1qX8&SwBoWtJ1=9_7b;a
z2%Be0``ZAvcMAl*gz_KKI_<Q(XYZT!Ig*Pm1#$~E>Fa+liYv0u4d9ym@H}VG6b4-e
zX5I@Q8zm1+5%@bt;A7yF@=a^zESeV`!sfQ(WPI~>1Hl~+b{o7oyuB`aor0#I@y+13
zmIC~bv=%kkZnde*Ph@9*D52ZCjJe&}>c2^V-ex|RZF8pHo)&K%Fk5KV>Itl?j|*){
zs9bbl^@c{4NQD`!D_Qf;XkNO*|0ZFViXqp$D0Qx@35y%9JY}r>Bz<V|@^egqhdg|R
zkMB%Q;5o*3@o3iBt$Dk`E`1dgsm^`HDRo=Ah-;F7*hGO!Ua6Yi>vJo`xh`3;@ZPAO
zd!hC3_1Bi-kCo+vIF@uqbt-mGo>03<?z(_|=3b|j!uvm6GM%Wi?2J~w(PlMUp_J8&
zbPU<v7-YTC5%7~NWPiw(rXaalSD@TM@moQ%gDKmducH4wCyFg&U2Pkiw|4hVp5wFC
z<a}p4FYuli_@QBr7r)XnA=Z`!dpa^%r(9aHh3nWEuD|~Uj!Rz*5S3xIID2=Z!19Y{
zGGjdhL|)!y{<wN+0{6cOi}H4^U+Zu`<u$jTrQA0Mqm!+>T(nJd)z5J$t`T&XShwJ4
zPq>|PMBLM*@qFhv1^u=Q#XS_vo7w*Hf#P(5UkfFdUO0c-n~V8!tjwY8qt}95ECfV#
z*Kq`EE393Y=p?*A=ayNjLr$Y&L*lFzo_i$@)+O?4u<g0FHJe){SRn60E!RbPcE$A%
zi}HSmm6lvtn!jf=S5~#oMiJ8_#h6OJ7usxWnp{&qP2(>+v#Z_9nO8_Y@=Ed>{yS&I
zL_&{CFAO+ySS04AP}h;Xdv9*B#B#m;!2cw1`o9I6qyNn7pL6=UXY!~2XD&OoEadY#
zI&twMkzPYhIfVl)adQlIG@qXPLNxNK`HtCqH)9^WoHfxvB=*&GxuQ20Z>)7l<v3h;
zu~K;Eo|EE?oqsZJGAZ7CZYj>wEBLJ3e)*lWJQuTzpLV}7;99?CmFzF>rw_Td=m_L1
z2yE45xY#H7_Ck!hV%e8~g8~M}PnJ#-y(GIwSNOX({~a@r_OOPOH~df5UHLCoruRAX
zs{-2%n>z(-(m7l6#MhL?$+0Hb1!%8{`J!<A?*Xo3HV3%_xhFO-ED9)NvG<s_%X$6Q
zMi#|_cRCEe)_O@-NjQdWbC901?$n_P-gm@aeLN!TA{sBu8YsY&t143XD$Gby@BJt7
z|BV;87RIgMTNk=XjQuj_<b$6%g<d}3ddU*Z_HT`XeB6v)M){CUkKXLK7%BGUmDZBI
zK}))(O0;vO_O97j^l;;)Ng{T2w;}|e9JBS^k|TLJ?nR~T<2M3{%V(e7a^RT!G*)hX
z#lV1uC<CrlOE`1-13paSWOpsd+IMwl4F4L1W6`n)0vYyH@vxq94F9{KYNkVS#-3fy
z8^4zdwIv%UZgI#roperQ)1AUf!GF!WRptwyT(hjL&x<STg+$^5`*$k223*SnPsTiW
zU=Sz#WajTpdg48WFBiW1y;N_OW~g_=i_TY@S12!!IBDO>Q~akmv9ndKE?uN9b%SJP
zVp^Hwtyd2XFZ^%1<$2NhTn7K$aJIC7+iDw*@_u~3BkDWno@!qHXI3rFAKJcuUBH^u
zb6I2&C&z>DAMfQcC?=e-Vr1oB*UZowYTw56igSUuYr@`8k3}0khksbu&?Wwx^F&5~
z!}_YE2`LMo_s?}P{364Zu>7gY?*6|oGBqt$O?NY7+O+0(#D#FjGS+O{I~rH+Us%X$
z{OgtYr3dF0C{6BKqZRZzY;7d3*`1T2OG27`TyF_qSd}0>amkTMf(+v882ngw7AOUW
z>&NaZ<(+nsYX^_!0oLbS8aiRw;_G_2t-h_^I=SKG)Sc6WG$lShGg^9MRqEp~hG@n$
zH<qMcZ3~$lmikSrO;*}Wym`z2-0ZtSxAopO1zS9M#yk5<+RdZA*`m5(y4s96ZlX4|
zHLr`C+fVjv3O%h6$hbHn_Ab|ro)aH?PVdmX@oIX>+JAXm0-HC+Ez_9!iuLVwuI1Af
z&064caowt}x+V_>UzQ0cw8H*Lx2~znYII@<{1d6_^lVF#>!Q#mrPT`m8FW1Co9CT&
zV$gM4sb$Y-wD9R_fxSi?nrjbe>m9w3kdm(>oh<FO`x4Kt4GPb0oXFj_eTSEh@$Ma5
zdhY6T_vl*Ok`ZPw_B-|9kg2Za!81F;Jbni08g+>*IB|HMNCH>$nV2GNqou*YXLV<Z
z91u0RsPfRyc5X<w_OqHRt5z!fu30*T&!y<-f4*p!MP7@gUteTt^r?yA>@s|@GF8^l
z<e+Cy;1=n}DpkKcv!-e+x)Gofbxm^d0-uP)<wvG%aq?JLC-$aqNmJP>RUr<SgN7~|
zn-bc!rt@5KnjHFqvs-(m(WO#tDOoPfm2D!cv!Zh5Z%Mznj3r{rIjf7(>vru}^7z08
z8`cL(cOO|SJZn0!qFZOf<7cbPJvY8+G2e0G)!s7;^j4c>7@hMH?AdV9##m$5l><3%
zO?SOWneM6ndQq=%`E%XLtIbc}aqzumCztDQa#BLiquC=ca^s76DwBI&6e+j%7#^CN
zCz{K<DR@cv1!G4B^9#ZcriHx;xVd_jQBstX$`*@h(>N9jDF2@<8h&x-ya{fNOkHB3
zTUR)`^R7CY<(nnxx+skMRk}!IY}88Y5BGLo(VXB?lsV=14ymgtH?3N7-5FWf=eRfL
zWU^n_zQbbj6Ls$&r{9}tf0KS7qiNRR)hB=2W{=MLKQbGNwlfL)Y&`L&B4zEBB}%?`
zxw8J6o#dS~x!aOgDQfkdo0r9lPH-zdJaaSI+fU_HSMX}NqxCU%QR%C{I9M)Rmlb9h
zsL;_O*7icvkSj1Oarv~iC96ItZF(f<T-n5OK`%IRYWZ#T)h`$CdLPAiB~!aBfmdwx
zL($dq{<}4at~Xg)AMM!9u+LO)W5zMF_NE04d|M6gp77HyN%AQ$ahKkiq55CHq_KNS
z(&2-}t;d_c2l0q+QW0woJl4u8Im07LWA@2}Wx}&0ebR4TP~Ezs%{29f%kh8dGFFLx
z9mS{4ia33n-4}U5VX?GQ&aW@Zj2m>+m8AkthlK61<~;9y=HRW3+cud@yt6o+i9@Qb
zAWfz|;E2+i6CU<1f~>7EN7Z;E+g<i0i&|wZl3RSh({YvB>aU$kvXTT?b%UDB*KfJ#
z+L(O%^BXZwud-IRC9|H~*mY<sbJ5a@<&BF@)`+_XO}muRz&L}$Fk(YjDvx)RmSoW(
z5%DVzkCpb_p8U8#a<*oHqwkdBAPrX~o3(3B8~id-&FNF$`=#TF|8=dti+oE}w&|>Q
zWBR}1s#8U&!fHlOk<CI3x-V8tUUFe^R;ls{^ZhU9R)y$CPo2VRBJb&0&Q&n^`kKy&
zeOu*@&Jk=o(KBav&kt+PqIZU}4}95IObIyd6Wk=CobhbgO_lPi-7N=V6b;@kJ=B@H
zCh4SN&r`c!j>W#~d_SCuiB%JxDaO~cG$wQ9ft08T(n`;~IGlUZy)Jfje0*BUl%N!_
zJ?Y5&U7h;d9-b9r$=cl%ut8x3x8!aXt0QItpZyM<2^8p!ny_^%tKMRX%OZ^-vjRH;
zCzWfTkUX!@V9w#o@Mq7(yGssrZSYK0KHSwL_Ni)H`-NoD&Lx4mVv@>PGuKQ$x#xLU
zTa(z4wHs4aTXp^?B@16vj0)Y<5xX?!)u9>FjndjgndeS3irN)+!Od@V@m>Aa9ctHY
zuO^G0_SOBn?3TsblYV7Erw>#(2P97mP|;Ub6)(QR6|KSV{N}a0e!`MF2YL0BFS#t1
zJaM&a;e;m}eqDA_D|sdw*29#&CLt_rhV`~JZ{E&2HglRRySPYsU3Q}-*Y$TvJE!Ui
zTv;fnnx$zPK0!$5N>a>^*vT5&hq&Ictqe>O&kvdTK;YA{P5VrH+}gWBjyW7#E-n|*
zmU!ya+L|(LLmi$&Ho`MwrwJY0E15NMTZ7B3)|{iOdwCr?D?+t`E_OLyO6qOrlwNzv
zZOxsRdTuw8xzFxgD3YDCuEg%YQp5pQ*X$oRJM8&7JWWLXbb?ND^{v}2s~B3Sd%R@P
z6OJC$Zq9}U>HF4rxb1wrHsY&+k(`o);r9DRn~e--?C!01UGbc&E6m%pSU1OuC;8f}
zFIzY3Zppdv=)8n?MX~nhqFF~b`RJ5c>PBU@mM<!jiNCU#r^oOxU-{8{4<joZk}XS<
zdKag%rU^B^^Hqr9PHGR^k<?{qb3O5e4#U~NZD$0R25sOjy6^se^*N=RTQ%yto<CMm
zGYqi`-p{pYJ-c{mQLKYz>V~rq#f8==D2bl-PCSt-DfMJo^rICEnQhlQN?YIl%=L<m
z=TS(R{{kPckhPkhv*%22Wjj8hY}x%ayITK;Z8@yKmhM;nP(!x*#$ukwlG!;{Q~W*d
zWu88~xxN3^^-Vh`9PDUV`7Pz@T%U$tvn<{iTIR6H>=Y<ceWH@zQ(|KklHI@O#&OQv
zY2RX_MZVvewlOw$_t%i<dCFbdWlft;#>II(JakNqlV!@G$-hsOoikQ`SajMg;mE7d
z_g*?8->saF-Ilv~K=kCJtXAnLN2xh^YAt_d%{HmNRcVhiaQ|WJx9R8W_>}%4Jr~~1
z3z`3CEGY{xeztFe2Y*dc=g}>9<mYMb>|dPNGWWNaME0xPmO9Jb$IDXo)@1E6*xst?
z^I%J_8DChp@TZn$5wkOer<30=3EdI%;`m*`*{h#tPGH&h|NEmAa*L0smX~esXwy~M
zsygZ0mh9sPVhvRnNH|RP`@GY-b3yAt)kZ_ENG;#}jT3uizE<YW*>ynj1FPBoyl>Ky
zbN3s0@65dWf6M;LWc#Vx*8cs}&v^OpL)m37GoLAWDbID?xGXTD`Tpasxt?29nwL9h
z_OW^?y!4o4v_aqabyMaoeXHVq`#)^uzB6Z6GcT77Upv#m#~1e1-ZsBE!}x~6!O6v2
z)GyjuzMhjg+wt3l1uWL9P8{Yrv!HkCB`)crg<_u!@|B%<rMDM4&yZeabNzt^x7MTu
zLP}M}Lf5mVn<r{<PByO;=Df^3lb3<-$ZPwT3e8I|3qJ@_J6h4Fo4`AfVaor&6SjU<
z+SRAj58f1x4(QfwvVHtv->b*YlbwwpdssGe9MNc!c{6*Cl9jm7ralX9dvO{2y_dPp
z9PVk)UMR9<x2mMg+uL1d7jYjvu~Uzu@AqQ+`O>bloHv*U>MqDOYx3+lf1}H^xp$|5
zvz-oKP{ppaPZOUDtX~=6@U+QclXdr6i`{8fhg2e6)&;EiXuRUagWcjH`%|BEIlAby
zRJl5CIhOTg{p{??rnhx>YHQ@w>^r-_+;`1RChqOO4s)md-ewfMu<VFqr(mb-ls$aO
zs)}>m)o&g=cj34+lSHTvU(t~xMwW(7r=8Xr*j`rNa>__iv#YnoxWh-d_pxN_|H*yN
z7xewNTgZ1)n>UO>jJ0Ok<BkK~zYX<1btkjf@4tB<+hns_$gv|6++JwAPYpR%Z(|na
zvW8)Ecl@WNcMW=!b{;ZZX`ir3%fn`;`D$ZjCj-AHHp+^t8XGr%`mFiAc)!c!zMwrv
zZk;fns9<|5QEhF4iR#aN873X~viN4_a6eLXm}szL&E&p<gEOOqS0A3-U79Jrd%_{9
zCF^r)TuXHhnk_t*A9HHINaubPpN(rAJ6;`23_kvFqs#u22b@G!y{$gBbkC^=1>5Fx
z?PO(Ete<ICc}B+Sr_Q++n^}|Q)QA|C%--J6b7r;AJ~bmP-6vc(bUK>9^sZqskGgWg
z(|K~x9FL@w!(|D4|5d!bk4iWn3!E#g<UDhUYi5Y?VI96_A%1(WaJ`UGs?uhBUE;Ta
zr>C*#P>YYpZO0AMvUhScb1$00D|6_yX380++iO%bS%WoLjc2%sSDA&GG@oSX`FX}t
z=i`oRGK+lXG$~DFGM&kwdBbbbZzJ2p?jpf??2A247I8C*aAzmYf3#XD`H3#$iOD51
z6%OoK|Er28UfSmK6^$7p_O}=K-B1YJ{KvE-=;Xd7G7@Lc-Ix#<*StGUd&TZ0XHM?f
zdCbP^>YM5OtGES(JL?&p^FBMCH1e(8vq$8Wgwe%4ET`?}i!?Nu@ZCzeXeK@PNrvRs
zg1zq~gx1Y)z1!0JWXIvP8LXNbs{aoxSmV%gRD|{T&Bn`*4Fa#-***V~gZ$Y`wu+lC
z3M;L?<yrPqbwSI9Q#L1O-f^Doz078^c>3h&JXLeudu6N~m!91*$!Nl<cQNPJa$R5u
z4c64SpcxQ6|8duyLf5O#LYq>IKS*Cz->9Sa%J61E_XHgd>p2Zd2M<WGo{<vSwr9z~
z<|t3TE!?59D;B&7J!%n_pL6c|%F8XM*4r;RaFoaEWro|$8>X4PmMPhe2PQbFIUStw
zMcOuYTlpLRGaD|HUf~XSb3|EX#)*i)lNToQTy__X*63QeK)pn`Qdm)YZP1M$OPNe<
z`2-|49}y3^8+=V@^%kGt!l^p;EXJHmF2CTp*6DlkzvW)x=^^{trrxk#87y+zYodnL
zS`$qzUjKw!rmJ`P9IyzpN{*PdrPtqk(n$fui9##SGx5ysJ@q%|rXZ8rC1u+icN+Xn
zIRZkNLRxy)Kk0Jay~R2^tc-C*m~F(`8!cfK9Y-#5>AyK&5Yr>$DIQwNqiLch#IS<P
zdtu*Hznec!+snwsK0Umz<c5VQ$H9=3#>%(3tVQy!x*7hy=;Jb3ZH{HrMi*t)87gNl
zUAFN(v$?mr)o^){R^3!Sov+7xwoVr)akJfdBDJXdX~PzqJ$uzH=NTV0VU9hjuGlNG
zh54<GaI^2S^1C;Sa_+9UDr_{FH`Zw__a)D{BAq`jTfeowIqS;y|EI2=IX6@2Xo2R@
zf-5?T%_~JVGrWy7{TdP|dy6xG>E+W?rH^Oc;8=Q?d0xaFt`%?N_H!;`59NzuogOw}
zyOG7@Qbp#_*6po(?y)tUnW-c-!Fg|C=*3yyCpW9^*|FMk=hk@P(C9svZ{_-2+dIcG
zfjK<>Zd9)5@tX}!tlo3wLqmd`v@*`jD0$2s^ZKC26Eo$OaMiVwDqYRhi^Oa0u4>t{
zzo{UOQ6luEtH<prGB@`MzrQQ|ac<vKMz7wSR(+M`pQ?v?b^DG;xNQ)T3*W&y@x`6z
zQ)IeBAMM<IF8s>HwH-;bo&?^nynAKW(p#}>`=2stNyN#&wTtuJEPCej?HN3Yo8AAP
z*d6JudnI|-q$jL%pNMc?e0|feR(PSWu<DGnFGHL9uXbIGyl~^i<U<{HAA);dZBBf+
z;?|cJ!a_>3<R3TLB*we<96!1C&fx>PMhCWP3LV_j92=B$TYY7i!;-*_D-2G|d30*-
zTw$9_w@r?g<s7=zk$Ut7cf-!J=cNwqG>%d8HMhDaas2GW`oAI7!VIbb4Yx!h+~y?8
z?%wjg^jVWf-@QA+W*wpZGnsarSbJ!L@EMP|$tBw67W=pSdN@<4XD7pRvm0p}6CT}v
z8d!R7d*HkT>$?|&HYFu^CF<^b9-`BC`>f~HH@8{#O_q*1_TcFZJq6>v#+SUz7FKl5
zxxlsd&26qav5^1sWv?!3j+;0mj#VLT!r6%r1MW@`?llz6RNtt>`E;XM#FN*p-4_{o
z7CB0Oczxi+V!K^)k_*$Wt1oh_&D^)=?}1|?4|XL#?F@Y`c{Tn|Z*+diK8CumC1GKQ
z*Z4om@Zq|YzM#rU_}uMXA*m82v8N`QojGk5-DmU7`qGNKsVxT|JYa}=e8S7LFpkrJ
zgVW&IrPsYF|59EC#C}tdG|cRNvq2)pm}7Bg_q>I?=UaGxPU^MR2)*Z!b2o73&eEKh
zA-7YPK6$B<HKjVIYi~}JPpau<%ab1;UoM<`&2sGzZoZFG?!=o${$yTens(|fxAFN|
zo>elwVP7^apJ#UH#T%yXOIMoC{<lth*dfN*J1_2y==9lfd*7~oebaD9`x-t?3*q9n
zw6`gjlZ@52Em&X_5V~aPwHto#-pzZ?I4$R9!_oP{JPW#GxULD;%@mI~c2MbQ!qiDG
zZ=85~Nc3Ud#O?8I!W%U%+}Gnso)=eh_>uj)jGGf4>s`JhpKYu7k<VZ2G*kN8N6s=6
z4@h3Ly?p0D&ozrVc8Oa)ZhI-TtNBR67PB4ADq@`hQO{kUMlPIY`TOjv6;~aMA2;cB
z*uOuMAG9`3Dt@xV+oX&wj?=EZQY}mstAE?o7_xD~@gHjzUbMCJ*=pmomG{}xQ(bab
zRVEePn8EPyp6b=oqqENmOJ3vR@=7sG4Q$?gE6SDUzvhFc#>TuO9DVa%3T2$-d3#K@
zY~~Y=-EIuQj?%eyOrP(^iXO=pO+NMRzQ(PWUqU`wUp>cIce0Hu@JPV*g;r}Xua4U_
z@3AC5--8!n6IngfO+&6Mlc?KhaB+`8>$3-?Dc67cZQfMYdp4}+lJwP^Q<%1^rKvV#
zZ-3|X+I8}(B>sn4Gp9V{5)$XWxFT(mAn(Nq+#LMhqGJz7|K2n$&&9l;b(>Y8<=QtU
z)zKz9%O)T63o@H|r|s^oPwrZ_E=v827W{i|;C*T%-;Bc3=g-bKzuCm=P=r$-Q{Sya
zkJV?Ea$d-o?AX6GnCDyU^s5)Rq`tjN%VXAj!0_->;(}e9GrdCi)XV=j6n$t~USTJ)
zZSJ?guE15F#cm2e;|e=p{ME!jRI5lK|7l=+xsuD*6`~JM884b}_fR5RC}Y9vrh@Dp
zVGPM)4dVOO>~{`UKWV5k?ODg<t7h4K&o>-R;J;vYVftr}SjLDquXgV{*(W3OH8xv3
z<K4da#|d^C7ll1|4D|P}saBi3a$TI!1y3{1Q`+~gh{qWCEDfoQEc$asRymrxi-o7n
z&isyB^(En}-T!i$pK#UAZ(zBR;VLZN+&V|^dcY&Om#Jn;%N#b@1zvu%(qaF{sOiD+
z8zaP9G<!N;&GfH!ah}d+pJTVE<f4Mz+k+-gb3!-&-Q_cpyXOyYn7~t0o?~D7^jG~~
z;kBqRU+eCD)iuvvn!FA@WBgUzr?2H-!VTSQQ7xfvyPCB(^4G4Yob~;E^OnMsOiyl9
zNG2RNc|767T8_)_3U&o?%)B@8hD&a3UA&382G5s*|F(&bChdNcxjpB-oa)B^DJ-Vd
zh3p)ROzd0_lp1^-o4CY{a$bB;bZKOhmohn_spxFZY|5u|wNPPYGm{M$Pv?guNya6o
zHpHA<$|J59ti%_hP|!4KHjk>xgas+wj^?>6JXcRlT;UYX6vgXva>9`VLcXG#7G{3U
z@@!LFyKPfwexkId^oc1B8XLHsw{UA)nXI0$<WMJ1@T{p?7czsFt+jY`^woF$2M3zi
zx!qQqrTYBu32!uGc{Sr<@_rc({wdj(509KlSAX<&QRD&FriG$;e>JKS)K9IFpQd&B
zm#NQ8;UydsjH;P@IhjhdLl!x7=G~DF6AnM|N^#|5-kEwiqMJ&bUof4`6UsRKaGGZA
z|5<@Kq3alSGrzsCg<0->;Qj~Q`Z^o`t&f`Zp7n@&g#4qLrq7Q*i)&o9^?9RvWQLP5
zXUYNw4lRodJ{&?R3w(r5hD)|Gahct(V%xZ6VK&nSlc?b6t&Ty`mvY&aJd~#%^pw<d
z?FjYrdLVYk!RD`-a3gQ-@+r=@Gfg8x`FDyg@enIJ8Xz5<v*Btz_f*EInaUH>r^lpr
zn2Xn`PEfg*;@PXh(Endqw)AOigi*+$TFsTN%N%51Tu%%6K6mMaey)?5>CSn2OP)uS
z#D%6yR+TA+O3qvB`k;VYb(cV&jD<$P^q#8a@{JCO5-&sF$CinzcYfU|5W})m?ZOJK
z-Y~5$vACy#IcrO;So1c2{1)!Vq%om|X&dK+#j8&19u+9|;9N3?!NtsF%|`Cf_BjlT
zHMXyDIKyf!5U_T~Beyzk$)l~(>sn?8c=_>$Uf32Uxpji}^@}N9&W|>}yuMp-ul4)c
zLbs&F6W<^0Rt#e}bu@eZD#oh`O;@5)C2Bbp*GTQ3AJ;K6>0{)PfECP3y)Nv~tX#6O
zOS@KMHp31<l}!!@G#AWfQCBw*Sofb(W5R(P2JI>L93F4mIw4w%bIO`f8}>gJCT`WR
zcGN6m`JECRGvny1^#!%o0!{^?GI29g3sNpzdG}E?aG}8eKc+Vl&r8OMmzr&9Jj2fQ
zEn+EK{n|{&&Q|NG%f)W%UT_t!EctkpcdC|I!rEPIpVl-yd^NjAarVnZ7qit~I;lQS
z?_AJe?hO)s678eia6xF>Lh(vp$+<y0uS+cviDkJdv}XGur!Q^J8;sB0k!AUrVRkjs
z@yh&!j92Rf_WpSL@As1LZbz&g|L>im`yh+|#Dd$b=4J1n@EYF#b2vw`|6*53*?+~y
zEWWi@I+Ulrxb<@bllZ4!T&jty8ML`o_OmeVsd>Eaf6wE06CQgeuL}4t_Goh(Pk{of
zijTIHxx%7EA5FPHFOM!YfgLlBYWg&K$6Gy7<c(a)9J@gBW60s<E2n5GU6-Dn#b7!?
zujhhRMdiasueKQc-%y|s+r^oDfZ4!>H-(w)SG;51i5a>sTK4uOH#qkPE4Jlu<;SlI
z|L9O`9&9*Karq&+qcMw*Udw4rX*wvnZpjA?9k&)uGv{fHl12x7pYTL+uaw!aXz8wI
zjfK2kjvbM=0@^ML9*t((DCRWdkdVRI92=&}TQlumsQ;g{GW<8QoXexDC$#d`PdXVr
zXSdoTIXAw1#iA@BpDUBhjN(`wLo{w2ShW3+=R(!aH!q|2O<S?|e@hRWp17iO_Z*i)
z%XmI1FO?Ad6}8xIn&FdL?uql6-iSBc>?u_6=T4|@X)!cyTU%7n({Vm0k-v4zVN0PK
zUJWuH9{mQ(r!SeEa`xOLNBh+7_?&~vs*0aHf-YTlYL*faneL(4$k3B;u3OzP<=D1A
zt-5NRGd(hFj_%px`7vqRltn-GmAD4^ZVahk{9J7730D1SISN~wPCQ)3X_mpqR&;$y
z+E<pMpr!=Br57)CJBV~lFpBvwp~v_X^V<hve6v_vrmXb;P}DAZ#NvtHiOusGR5&G`
z+&wwfHCAz9T+#Y%t!roUMhe(-s|a%Fo}0YdMX~zbl2yM?u)0bMoy^G66qs-K|6}L#
z->KO<Hnh$;@m~Mrk^Avx->0wT=`?V$df>FAcxqJ6=GiQYN6jsTmN`ycXs!C^rO?q;
z=Ra&+JzdzRL)+>2y6Ug&ZT(+$<{kL8{9@fTy#-eTgMDsajlI=pm(lfV$+iW~ojit0
z+_#P_y0^LYE*DGg?oW!2kC|Nb7%sMW{8=Tee#17p>3Nm7r*GuqmJpFcljb!&X7OWr
zFnj9HV>6gO=>6IE`!353e=e1^=^NKg(mA1dS~0JNqk65gi0TBzr6x1e543HHG~<2R
zm@B$Apj-Hgw_9+<l&AA*<z^c&2un=T>pAu5llaUBi>`0Gpe)g+aYucdL`vJnkVgqp
zZ80a~jwt+J+UT9ha$B))X^TizjkZCuQ}_Xm%WBVsI?ncOnp>$gb=GP|ov;cwu0vU_
zOwk^lS$kI{*evZkC>N*1V_Gt0?ZMzCo=T~@XlXOiq%8*Pj;43WZobW#zWj{PwGE+J
zv#$mQJ+u9g<MCB(+MR9FBo>u<_eT9qd(P$jWS8`Z-nB7oajh(g3XUEQLP7yAemAAA
zKX~bN=p%1o&*;7emX&wJZZHI&<9uMXc0xnUn@=ZK?(m+sW+}(MCyOS&ZCD<iw_={O
zg6ve5#TQmg*)vJ)b!)cYx?|V2o(oZ*Dz0GU-hY<wpjP#Sv)bz(>E!yc-rlUd+q?7a
zr@396p56R8uMV?q7B$-TztiD>Y+U;t(S|rC(JN9ss)ue|Kc1-Us=Y5MVuH_u6EQC~
zw!1|LuIPB4#pT%Hc*jZ1(rPJ-ALj+7%)c%(rW_P4*J#bMzP$MAw_jS5uBsjruDg}{
z{@jC*SxWm@7}oj*Oby)4y3VOWx~P?5ddG@wA}3NiUmZ|cwu@nI(!7-(7h7YdoNrP-
zIzQj_OuL8`L&v%gg*T*iGUkQ%u_#Wftt<^PYFT16ds@Es5{0Wx#l8%A8l}N`g*SU|
zync8~%;CkeNWG>7@7^#OHf!c3s<Kb+y`HeVT7<Fr{mZs(2etE+(xo;n`=FD5BVBIA
z#-D4<c;x3PScmdB>8<-Q`~03;FYiU{`oDbf(^pd&r>kCB+!c4ZE&uR&wUaU)3~Wpd
zKPBf*FxxycXqo7NIdAuBy*-vDCC`*{&`Cw*cF2iWug*@~70?^7?evVUxS-%x315XL
zjvmJ*tQT-(b*<T2JpJmlIi{Wdbti8;E;=O4dAqYWMk#MjM~>{I6>NglXLl!k=9Bg>
zZkL(jl=aX>JKSPo_i|Cwf9|3#U*<S-C{+EG5ibdMIpKei_1Ifem&+3`dZo5{amqO6
z%wkHtD>mV>mF$hhOIvPEJosWt<BhG1l1FYu#h&(wb#M3SttvTwO<_{|of)d0a$zQ-
z>wm`1?wxVtL2$^##!Cill0BCkmd?@g^`GU)=ByyybmITCMwW*q3=i-7-uoKjDSGzO
z?1G3NABDVX`r|ivWbfcuFjvlTiJbEl(LFY-k3B90Fi3`Okghgxx8^w^a$}<O8kW~O
ztdbLsSXu^0?`%Dm(Q<adakrLJY-&8wGabtpv|6?bEKZI2e)0YhTNYlismF6CB!##y
z-*f$QNPCWMx1-B3kr^`&I-C~MIkjj`<i8(bnrT-TyRfkzh<O?~@Agga<tHRFFC1}s
zD|znc{d1fR42M`5+Ik;MVG{Q^xOT=vC!MIzbHX=h#4edB?W;2HaY$d>%5IUA2SINZ
z`c7RSFWbUZ6}!ANYA%P+j2n!eF~^yU?yR(J34U^`d9U(^o~YKGARDFs)7LE&cKhMi
z@RED0p@fQ|jQRsXY1tMvvD?aW(ps$Hd`qM5CH8kb4L&0oK6h%T`<-@A0~V*46qdb5
zW7ixJw>r)^Yo5b|g$zsNJU=aT({P>fg<omy?SEU^oK?74U&XzPbY(H)_U&=f{xi>M
z3sb<2iRWCpWhb_?Dc<qW==Hzaw{3$gXXAxCGUp$8$knCZY&!eMDr?R^2DZ?$xf!L`
zvgclPWJ!s5wMbp>bY3CjNyD}iPdsL}Dz%)rwrQoLMpC=h-bYzicvB@s(o)<?46oc<
zbISdJr1KQ6%Pfx_wZwBKI2m%uzT!P{YK}^RNkW2YdwjsG_yAewn9EnLE@23W(q;J1
zs<}cz^23Rj9Y;Pryiu~EyZy_R&<9t73#Wa**tw4(PW`KFCa2rnn3E4Im9pQq^WE~E
z+~WV7^@J<Wk`oUFLoE|iEjV<%y5gtIiPu?p(1By~&G;h=ra$@l>|g*#yUjHB4M)Y;
zWL!%7Q&zG3s!)++kaBi-HkG$!jp&hGmU5CJ6GP>eF~mkmA9=VZ(Btq*k2Kv&+dsG-
z-r30IxpcMTlEXFgYx&&bFC;fM&YWiGm%X8d|B>pw8LbgrGP+?dQBp^KJ!FeAYMZnp
zby2~hMN95zO^R&W8L!3Ju-Ku?MI>D3RE(;jh^w!n%a{9mBd#QPg}Yd_oMk;`aH_>u
zK{=ISPJGV)qt)+xWK8*GIOmv_s+IXTaB`(Tb9H&aG9#?+RG8HD#>`39|Dqax-panU
z?A@7wf3CN`$++#9(Wn%q>63Fw#Nrg&nG-B?cyc{N&+({8PEgW1#ZnQoILSf1YOc`i
z#Dy_FNuMo5#kB6sUeF@4^oUEH%eIHD^8cn6Z1UQ^wau_i+;dZUu|;28r+ZCS^xP@S
zmkPRj+<7v!D`t<WyY_>LQ&Tt|E4-YuLg1ax)0J;$tUPuh#OBi6l9$2%w8B2AS?6iC
zUOKA&YXW~~kI0V}{|xc`HT+xD4w|oQIl#K4_tkN=1xsW0-fukPvS!Jg=(OYO7R>)L
z=VsG~X*-3kv>49oR-0E9^1m%E>P6-mN!Q922D@<Qo4w9|=j-deh)xkP^lRL~*<dtH
zWxlTFw>cuxIjwI!A2?lMlDs1QccW+RlV#KD!sgUmRlb-!{nq24y9p|Ztz}`Vk*-Ns
z3m7>K5}U5PEa+M7*CXxnp@pR-@OzJ2sl@U^v&1mn<;fir7yRSaGJA4<(anT8moAF2
zl;%yf=gP5B)oP7#`RA$iIN@PufS8`+s~eF^-TOL<mn?tTxaxI_mc>0+R@Wn)wk}rJ
zf;VkwtTSr$@nj59kbJvv+V{lxM#tPjjaAo{h2{A99Tm`B!*OKuvm=|iTDD#~b#NNb
zj?(K}?cBFba=aIicd9jm-;^WIhcUP0zmmPEUgHAQ=7Nwiol`S3Rg%}lFIqCm@sGfP
zI~RB(1;1*z9lr8t-raa7fd{9aCS5-#*}sWRZ0o6xmXt-DN0@?Sj;&a^K4$J^$0N^F
zWYhmiyZG=0PdH@SnpV>CXwpqq8_#EVeOqh4wkDSJP6|zJQsfC(dz>fq$esyP?eu2k
zP0M_B;KExD2F@c#=A2{6FAxcvvncDvo8J?hGz*wCdycsTv`4zM#r!?(6SMfjyw!hn
zCpswV7V2<>mrC5%o7i`0-A|_3Ek0S-?*#=UrF74jd~AbD=N8B1Ee}0Uoqs*?ZjYCK
z^@fMFMwtl`-lZ>=Gk6}9U=YxpadhH}v`KfBCNTs&dHes;G2NTTcwgT!DGU>97v|5-
z4qOo_5dNyIC-*{|n7g*K%o-7whVs4V7s$Fib!lYCXcC*Tk>QJhWX(tC8j(kL)-dHX
z6|+2jlJ(l5WOe7NJL`NJ<?pxU9#H+VAwy{0{aKm%yX&&G`ZAjKywH@?Vx9fQ>53AY
zF~4Halmx$`{#R$+KWHa@d*PTiC8<oSa$-2cgtyX7y{W%i(@sr18!$IZYl2rs<1>y1
zeS@Q=e>PN?NhxjB)jJ*NYLs;Jq3nH2ExsxaPB{sm9}3NTUi@qGd2FWVsId4?PGgb9
zaWMr)5toBv9*)n|7;}3XH?P=uVMCM13s>`t$DC(8JbUkr*n#NGCI7ql*&h1s%yF6#
zY3|=B^rE;-KzBVy;*BRKx-YC!{kZl@m!@u}G+VLGb+7lXYmRKZcINt_Vvl3}7bh;9
z#BrdYfT2+KgU+hRf|IVd|3%m)Cdkwsop7@`E=O2<f|K<PCu<fDg;;mp4NP~q9IPiC
zoEUPH^<2`4G{u^j%vg_&k2{ZO?^D`*$V;qvql1W_^NP*!d-}z094gFk(%o=a(*3l?
zmknYCeowbFi)}evXy9hc(R+cR^-n`Mi^RJv_i|V+v^|-2<Vk>@c!}5Yl1BnFZ?gQ@
z805VAxm?GI33u!>O%nKz#4)oLEKPL~{FbkqyGw53MW;aPhrCXjcO3O2)Nia7N&Ua+
z+)YXDe|qlUlpgS8`mr3;yx!)ab<Ux~wRE1wU)xzz_C2o_^8W83_1Vp3id({jmNXf*
zYhoFWx>HQGw;b}8KR)5=;nEw;7jk3+uQciYG5EHPNp^d)?TaE^gPic22~F!J?c-^Y
zblLd!&jKf#%}z_oB#-eMdK#ZiY8Bh!q`jwK?FaMo3(j)d?A95LYI6#8Cp3w;9F$-%
zcdYoF_1>j7Lfc}yxu(d5{eJAyn!XNsjFXtOezz_Xd%amig)vX#d2)oJkWdz@c#r)>
zHMZYtThFvJMnCWLK2dex(vc&_pX6+qSrIo;;d9Hh4N14WUORrx6FB1jc1N&wOW$XG
z-vwC-KmYSgd@N_aQf)%?W}V$vyt$7|mR9|??V!e#gJy>8R(h?~+U=HMsy_@93bu*q
zT@CNqC3DJhb+Gqm31gT3Zt;-IY>$}S)~+!vbY~RGaJ21Vw7z1Zy}>E`eTudV<Gk)-
z-^X7z$n28hc333rBH`{J(qNc-#jqrRJtIIbHhs~HO{=4S73Q8e_3MhM;HK0s9cNnR
ztM2|LkZfGkt@Y%v$a%*|jTP0FT?d4E>N~HrlpW~uTGwpkw)K=tME2~}ZN5i1Tuz#$
z<{Z`X)^yooBrAMWBW6jKmZf$GcTx6FU19EEp6|MkkM204^{wlUw(-$$51odovzk{;
zj<OJ&Ho<dl)QMA9{)-xI`?=dX&6h{Es(rTm*Le&Xjbc1A#d@4<W!$whoJ1nFE^=qo
zUa(QE=TMmNj=uLlCdDsS`?9jyy1nVVWWWc_mi#|V<|ovHD$LJM6{*?fU>;oBR`W$*
zZL%}VCd=mz@#^RH^d2?z5skX1Tlg^b?w@qaKKEU09!&0!OpeUDZ+-D_b*~J|Qq^i+
z<>Q5G#1mO2>@a6P*=c?C?UHjbI~T?uKj7JU{bA4>N7t$i4`zp66TTj9eQUM2ZOz$h
zyW=k$Td=ZrpL@&g?Ry=193#0~BK2At^p8YwyWGFtCh5&s_jtR-ddKul%(`!W-u>-t
z|HH}t22+@Kp2Mq6FS1r|7gK+h_n+}(>m!wdJag}c3(vaRn2xG6Y}wt{<5ZTB`_Szr
zpPAFH6}(3?9OcDxqRxD^a8;M+dRz2u|FhLeFV?+hQ_$aZe1g_jtNGvmhy(|#AGBs?
z%dyvY`NCi9Bim^4M#IPN$Tg>Yd;fy<e1g(dH}APtO?kf5%kc0<oyL@s%Jt?)u1~#Y
zYh%<k`_J{u7u;J|3O+a3A6dlxODdbmp+`_5K}0)jD<5Bo#)5lJjDO|qzJ#Bjzc*@0
zRH4$*Oa)CnKGq4GJ6m~9`o?T4n5mSov_V&I>7F@Csn1wc^<0}mc5GV8plP@LNXg3L
z2i}X^9lt%{{FEHbDDPV4Xf-wKYNMaHm+CLU|0{#ng~`@E(U_Dh$+(nd2CL?zD+kOn
z&Q8-Z-X0<yl(Nj{LZxt~&}t#Gtx8vyY0X|~DJx~hQ)GBGFhP9VX4M5^Cwzr(?V7b`
z!x4uq8SGsmKSN8>+tUQw_q-}<Zs4u_>^0Hg!*kz#-0A#N1X2}e2JZaJb>UJU&-=#h
z6P?3;Oy1UT`z$+K_z#&(mZ!nJ*%wXHuFMKubK_!A#LjU0_<t-5!e3-(t#1_g{nPk9
z|H6iCik1_1&aO|W^Naac{c4GC%&jD0=It|-9Oq?B^6i|uLt(;!=^P;^R<tgw=v}&S
z+m3>z*P^OdZsna?)Y;74`)Gmo+<i$CyZe`1^0M42_NZgAAItv-845DC)~AoR7CgD=
zeu0H!(+TDE-j`MxJ;@4Mx>%$@!>g6a!(pMO@P;5J?ZqCxo+spjH#s$laBZI2sP(02
z;VBnx->p}#1#uYaZw#8)e6#4X_xd{yvx*k*F8Q6wZCCo)(;_UV)2VxcS>zS5pc&3u
zmP^+3yz^${2<<rUliA&LdB&m%PF#&_S*=qWwPtfI@^<X%U-U!g&<Q)~;t&saqffV4
z!i`FNJD2yzacODHPn-4V;G%%4Pb(X%nm9EUhA~gu#=T#sP(f?T+eJ6mXQW(>wm5vr
z^H^(CO5x4*OMacw?7N_~I&5=ENVZ<<y-yQG)J{KG!qdNI)fA1FGLs(J9jf~O``QEj
z^H*L5a=yFlsb_Rbp}A$k4vUX2yy0D6mU*yt1U1h7^EG@|uj+)R-m+sHeLUSg${yVs
zi#R4L>dGZA7cD!Mv3%+@@mW_aH&k>hF}-9srtA5*N7z~9-45n{v8n)3*JsVQz5SQh
z{nGTei+{DURrdFr%lvQpvhL=G$NrZVIOr*KCuverYIk%4<MVm9S1^nJ($qKUy`^+y
z@5<7(8r=P_I=dP}YP3EU^#^^|+_)p_`x?EHH;)$k#jTmDAuIc6&vB{RBNIJj^w)cN
zC^asLTB!EX)z;I#lsm-WDyv6?;%&}-wuTdysTIGvb9#%dpK0)&;su;G>t3n%v2GA5
zW9<4Lcfu>|UuG(|%9f0y^UJoZ*f6!sZ|ifzus;_UZ3?cHTsk#OtTk@oi_TkD9xQ5=
zN}ha-#llpvT`DW%YGcGg&aWDrc3eN@1Nkx>nEhHBHwv9`D+=MN73|<ny}`or_wee?
zA6La3YVr(7`y@1FN{hDl$=U1&O?MoqJzXGeH9c#Ik<_iT1(DiJ(;rKS>zy-QlgRv1
z?1q5w^es2klYak7EU}onM0bvs=MQG@*%HpX!>%mK=G*DPGA&)hNA_sE)x(<w9#1Dv
zyXmIRa9BzDjUq$Y5g)}_AEeIA(2!B#Na1<2;%KtR#@1~ahiY|``Q~k#q4_%Es8#cm
zUa>940`H#i+pLiK?_pbGD8G6yyH0%e36;>L6BwIVY?<GrxM)N={5a!o_i5$%+9TJT
zV_MJJ#~93B$0u@n+s+M+#)4NvRYJD&olH@aUUWf^b-6enqqej5%|o3rmljPcNSJy3
zCzrd?cMHZN8M7Waaj9=PWO=@HUD)kAfiFzbwc9y7JxsQ6y^i_5?+(j@N-hqDsccNU
zRXh@XtOVsE-f=n<UpqUmcH;e<Z4CQcS-cqLNp*;^OtNd5bFxZEK}!7e6uTCMCn3oe
z9Qkg6+~$3U7BO8oq9L`(y<1Pq+-A1dmn91{9`dAED|9NWADrZ{>Vi7=%3IH!QmRbZ
z4!u%pYV_XYa7pZZW`Oc8qqGD6TdpqP&b$z=qQ!N5)+Hf<s?$+=cTR@(Owwsx@OoJ+
z?{(g@t)g?cu*TWlxg&74YJK%zAN8q1FG7|m2pxV>tmRiC9C)r}#!k62W!*R90~jaU
zADqNw(=GMHvC{dthf72Mn}nkpH!1|`=V*nl(KzSy=gEba8HK5LHoDr{R4*udvQ{BJ
zJ7NZ(_mnG5EE5+6Jkh%GAxO(e#r6$jXRhof&s7(lx#NR7s@F`q-1<Pw=>O$gH93j5
zHhw)ed)qCAS#Q)9CU`2fEXulYr$AJZg*BVWB=Ca%%&T`38js{R8@~@&HEWSb-qgLO
zCHmpnCq#3KTy={cc7>hD+}YTp-9AsV`Gw51|LgzW)0ng1b-&Xo%fo7sUS52hGo5x-
zs80W~Atbmm)BfYC2^RM*uM_tQo>BAI#f;^c;_Q?0(>(&YtLJ2PCQCQ}<T>Qt{KTKz
zC#T&_@YAO548@~N6(XPSw2PR3`e@iC)hagW!=&RL3o~4dRZ5aJt(YdH(r32!R$vSF
zH1ALAcg%`#ohG=#VO8g*DIy!&?XS6WnXZ1Sy5*C9qbH*`uiuGP>OCh*A9H!HRr{(H
z9N@X(-I`rOx~`gWE7SOnr*3%mY{Sk8n&FF1KNT@u_|?Q};l-t$JI>#jx6-gnvxmWD
z&1B276AV^bUY#cH)Wz{3O?aC`Yv~1#$g-S;a%(ycNvw*P+4n!ljlptttH_ps{;-TD
z(IpAa&JAw__Fp+xTch(LkV~_{$xV@)D^g^oo=!tt%p}|Aj87dpcFXzH1#LeV>bKL-
zk}dEA`|>btg<YMSPI@o;T|d8uzsBojf9Dy$6`!_FZ3w=rw(d!h&*T-iz3z!k*8C#m
zE2uPQLD|nq)~}a3H(#FcKxnx@n$qPNXI$<q-6QF{ao6|R3X0Ds-jO+Xo`=)(j*ewh
zcB$3+Al13dD<_@hW=&l5BeRunp>}MhdFImz6$)y4p#=qJyHt*>sj*qtDi*U=qyC3j
z=&#SmSxcWK2VLM%O!rEXbYx0#Rs6D;S&~)P?<il$c4^J*`Ifwvugc~Ir2JQWEzn=F
zq0FxJ&fg$`%1~zcrO$SGW=#G6G3M~IGpiR$?1?<>nHAtNRcF$VRry+s(}KQN?l`#B
zCDKX#=WW?ZFTB5gZ+%b{8+mHN;l|2kyeqyde(3vjZ83}3m0vbpv5~FSyTA1vT3P1N
zb-ChvSjM)itKLnrjVm&W+MGC#{i6KYecJ_?HTPKtWh`$gJ~v0Vz{F?Ef?wMY>b|?f
z6khsp8`saw4!MsJlA2}9&tD3ASo<y@@b1%(thPrs@P(!7+Won{UAN?!-iH-=7VQoe
zOjGZ%>=H3ZU{QFOwkXAFNy@SWhoVR7d`C2PWi##LR&hI`=(c1}8<X>@Lxob$wItki
z?D{?ayQN2ZJolLAsqXT8|BcncTb%S5-puA$vr}sA>J<LXv)ENOrF7RE`n>1qp`7O-
z7QJy(K4?riQ_gm%(`ym$r<F!gikxebxv!~nsvM{hQ#D|5<Jj83W74=J<W+4+pO(@Y
z#jKuf8ZRfxFr7K_Mq``LyF+VupTFZ>t+f1wz#kTdtWCa(E{URBdS$~BMSeBPxh0BR
zc^q<Wp}ZV(mfWG!ztorQJNy3B8l`^+<#HND<v6DPSp31HJ$!|$(5oEF39rsAQDBtu
zV*0U2MXTRAY0-0@XDh0dRzGRkE2FNJv*N&{C#>s}FW%CYIL)fp&>`OWs6(T-t)eHZ
z;610T)4HtllFJvB|Cd>G#Ubgm#44i?D(g}vuHCXkYgtRrl!gE~tv5AIH3poM|FJ9C
zoV@$N;GT)5(gR+dlI(^D1&Yr$hKe=a=y;TU!*P4nL&2Z!0!Ntza#zV6*HvU+G{r;H
z?#tS2nbo=$S;BAH=Ra~$uWQ=n5hygn*^JGLaT0@yt5J~E5+|XSNAr^Qh%mVr^hLc%
z;j($RV#PAexFu@KHJ^s@={C9Owy@0R>(LT=bD-&yd(^w94(t7wJejNK%9Zw9qw^8}
z;ts7RS;7_*HR^Ov)J-_J;gi*=jam$D38&u5&7G8$<*2ikY0I^@QkjmzT$dF6Tm?86
zDTF)nuxwOZ@$uBTAf<or&d6O7`+sT8cc({sdK}Xxt<;*eG-z9jw}^x2)(QGr@0!nb
zuUhBDGW(ULNxt~;PgXAP_fAudy|a2vUB{ORAJtiN^yaa@S+=NqN(0ZiCwf`#inBf@
z{1bR_U=i;>gWeJ+W&>4~e67wC7rRd!|8wtS^o9Jk6Z|_aJf3tUL8PmA*^zujH%Fej
z*V{kji(crL`}HvU%lR&gF1eE4wlG(_Eyv`QPEJv{RPN@hzU1`H4c-hT#`8*;RAjwW
zju&cpWiaguERa!h?s~)=+rg|CyjLWBFUOY-1$Ne>i`ZCw^iC|*=u9%+sd=zU=-IhU
z9e-htvjH6MgDhWgeN#AGR(0aA-Y2p76U>4GJ^x?2Fh^t~-})T>xsx@XP2iolNGCN!
z_Vq&fN9VUHOyqxCAp3a3wJi%Q<rk%0@!w#wN>_Z5c-TUL_F~1PDXus9rsxIUKer~^
zBJix6<JpR(x+48bnv+zxm;%qeQ4QMCEfmZxDa^#o`=ze+b3ve@!W7Lju70Ar64E8w
zO%vS&nxiWwa)>s0=btj*Q081}D3!ndTM54z|JH#2SG0Hzdq+K-bmP71lj9lxJUPlg
zYs_PwFvWf1A9i`s47pknIn_rKU+Bu-F9_wEsF3kxV%S4@)&@`0p6g#OYMH&vF1ymj
zC8#Hvr1WUTogKWV+b-)kYG&VSRy;k~W;dHscdy8f^X79zZT>4=6+4%sxQIbz`=|6D
zPAbluqV1QZ6fw`>aCdcLdF-;GRYvLYfu=8yxcxSqbo<cl<k+NMX*65IeNIfe2Im=%
zHY4NSW&Q%fi@tPdEhq}h&7Xg4$#((%D{{>XOOm*?R9zj{pUiL;ZZY<B>vVs-Uhb8n
zNCe~68@h7YjMk1xqANPYV!Ff6=7}yimb^Vj&%ZU|wzBZ1xAl8ITg8Nw3T4PjzEJ$y
z+o;i$tf-lxoiyo-?sEla(WzyIvabx!a>lAp<~BKR%y}h#KWn>&u8IBm1FM?W{8BRQ
zPZEC96c~Rw>uT}Nl0(kf?B>^<xVIK8h*+wvn4HYjb?wtd#lB+2j}ztozdEtw)k1~W
zi6Yky+J;?yry!^&Jkcm?tpba;jYhY1<hH&7?$k&{JHECwCo#<v`pXNt{}gC1PYL_;
z$I;_{!2NTFOX|OsoGdP7Rp>R@D7M1%x!#uK4DmyDZFZsun7R)+mo(Y0<#Jua70X!S
zu)R!6;(1)Q$}BVC2T6YF7VVC2OV}oEfBNfc&iZEmKUcI`{M8?EaHdZ^tgv|Aj_I7j
z3)t6&bLB9Ww3=?O>%OtT?Jb+r|GW~Bu!V~I)c(nB6lW65V)>XI^EyN*P~dmC+kZ0w
zSI5xb27>R+_$E4KZS$!We&PAFK!<xpeZ#cuti-N4>+OPk4I*?hWpnI8FKTl7GW<8u
zzm(A)yE=*e<(7)?+H0n@w)P*&-Z?Exf-RT(SfC@nfv9MZd$vNIg~^|!j*XM_(;g?^
z;OgvrWcf!$W<Q^bh-$}U0ox61O54_kGG@p|G|H(t-MtsYkiqC)=_|0ZLm~Q`F0WvZ
ztE0`a<BA&7--#@e3p;rFP_aOzNr||VfO)w330*nIsgo|~%JaX-_$wiI@^Y#D=@*|o
zxt3nwbvX1QM42&dy48bg(Nh`h1C<K&mM}*K#h(BAI6}2JXH)E1rps{_bH0VLt?n#(
zH#My1lSe^InBTdoohSTHZ7JKxA3Q@Su_Rq5bmhE?6$wjwr^%jCT-5Ma=1?D-(!V6r
z#Tnh3{<9R-|K1=_SR(Q-P$1eY?CJ~K7vfn0g`#S!1-LIwia5B{;ZfR79sRXTv-i8a
zD>CzXFKD}CSx&wGVT&EpMGuL7s#M5xoX@&oKKmoSeJR0Hriq&FT&?$6(;`25(exSa
z0)Ez?7pk6pDwdQ}|0z3i>CqQIL;jq)EOq#^#V_wo+t=x6A1-O<X?onb{E^AVty&hX
zla@BfDxKuZcFcOwC}z<mTgCXFZMy6chA_?TGqn#37V@SZzOmx7`>caEJhu7@Y;2Q@
zcxYWzx~)$sf9<2B(nVrM3&q^TLlv3twRqcFo!k~zZrspkAa!%TsGFt2O~sazT5?({
zQyuK~bE_(S*0}eJ`M>*;k69B7wAvgzKlttMif&kyl^n7E0+(*vMAqfy(>jYZR+qQs
zt<w@P=G0O(sQBT0`C3AMliA4@t#^wrd!6aBa$36HImy^0dZH4S)^<1H2*GS$;k+&K
zOds=aUaEU}UH%xmlF)QnuB2D{l9eWuD7<*sIFX6ZeQMhaA^yp?JwHxze{Pwm_Ue$?
zjV0ZFiatIIn4=U~s*JWoGq70gS-I$=!InL<pZkcW=y=cg+~d^t=t|{TvkmbPZ3nq#
z<rTJvxz9PHxh{(Fo0G1V=vS+xVlAWJ8XRgJlB;+-e&jx0X?K&$PTnTioh?&&kCsiU
zE2Gi7D=(H`d~CSlh2gxmp2@pT{nt3Wy=>QAkLiuvmR^_r8^4tE9C(v-^{}|WPC2#1
zn^_l3oVaN1+HhZoW7>t`oMsF&wD>AcdaKV^X__QdIP2xY#Y@#(BTbt)G}W$L^6{Hv
zk?z#XqLQVzDbsyYcvW|v<N~gk8B-6R5obEiY18g#Z5~klj74kdi3Yy#hYh*2#iWuv
z_ew5Le>3sJjf8tY6ZkUv4hzXkD}`!3p2v71q~&;-%r(V4#@)3(3~vrB{8?#oaM6O$
zK(04dEVi*1)GoKguK2v~RI6_7-X%%1kGxXeQl>aT@Ur3@t&V3V8!rTy`lxV~9bo=*
zS<A;#V~yw>jbAl-$EUybz9(azvN-yC6Jvdn$bY-<+aCn&m|J}-w|l}?UH;WdM@_|b
ztiJCwU#TY?dE>>-Ma3};GB+ox{@vBtbX-$b^GcPX&zU1JjCUW_u9c5ezq0G&lWVm~
zXC|dDzTCUVe!0?mbNwr<Aq-;2wUpjEoOe2qaY4g4efAztw?|vuSIwz<I(6UftWC?A
zEZlxsWfWKMF2BJovn99vuIIrm@y|_vs6?=-dKSF(-t?xvXZKD~jnhy3YS$P3cW4re
zh%TS-^X|$=yF2eGcGd0H-g}AN>E*#_MWxsUGw<=;QFtl3=+ge%TKi_d-1*=JJJVKE
zS^K5>#;Hxti&|YiPMPsE+A?5*&b-2%S$S;+Ne0tC7?ioUi#u)kpK7d?pR+~A?B)rM
z->-r@nbS_}Jn(CZf@4v*>Y^v+3(wZv_?|cM_s+<g+R7`d`;!_q3)3ooyn6ljo_}rI
zo+IgDw<cLWS)l7z5Fw;;Li8c$*~fjF3%&Qoi1n&1O!8~*a7$|8Ucou7IHf9>y~LzW
zXR49*=1Kl(;RzT1But(!DeJ7*U@7Iv$k3@|AL%;7>WEiRQp@2xPW$vHzcKLK!}fV&
z?A+NVeX8+|hXP*Q@B2JcPV?6Pw^QO6w;j1Lv+Q#JpLOXg)<-kxXU#v$*wo$7$iyyX
z(_ygiP%DqL(wj#Dj}92Iv5JWlOq}B6$0KUu^XS6@w=ho6X(lI*C1|&5JM-oIe`)m0
z!OVcYFX{5RFu$qHOI0Q$1#~vB7<H;maZ2lAQjOWBle>yR#hH`cF>BS*3)2tA8SONB
zbKT!zePHs9R@)b%4<lzXZ{_omHd-6qR`9#`W=i^k)vc9+x00kc=|4Eg^o{SI_Jh6#
zP9__^5}SY*j*HnG)jFl7zM7`XWKn)^)4_xnZ9S|$nzdCAL|Rs}SFPi)Ry^Lqsvq^_
zQfu(RAQ9FQts5)79XofimK}N#{pfJxaZckaFYY8Zw1sG^uaUfK>GOvDHiv=bsf3Ew
z3xexZlo?(h*#FAf|9srn1?dYMtc9;73GF+$IX(O0;Zt1J|27EAX@7a+ek1Ca<@T_O
zwSQXw&tTXtA(9x^<+~!HMNrQq(W#QtOrV30Q!S~3ZDPiRq%P64q&UHQpQ6e*Jq(hW
zf-A%xH#Mw1HZy%2r-PvDDa(?HeY~QQ(#{@cM+99qD~5DAp8c7&BztXz`D6x7(=Dk}
z`4*->Y39y#ycxvhX%<kL!L?u+Q&7P|k=f=&Y?A_l9Jd9q1Z{Y9-jVCq&E*VVTSUcl
z{}#Aj(poLy)vd9b!@;}a%a={(B)+s<eIC^)b>;ao_FSv$jth&tvZA?iUuk+X?RYpR
zkz;3-z?!&oyDr3So1iE;FT?BTh17ksZZ59nX?0tZ(Jg5mkRh{Xo0RD3U0YO&xC?`h
zO_nHZ)ZOOHl=(j(pNWHAu<(eY$y3LtS|U#!u3gePP(49YfFU)L@9G6LrYY?5<<C~$
z67ak?u`%EI=~MThYR-wC+Vg&S8hWkb(Dj<sAox3&Uwe^=mV^D9HARNDi+h4rsP#Dg
zQs4b+>X}JmjQt%Kx<u5(vfI_BzfkSqQvUMJ$yCcgIlz*O`%BYW%_GWlm?u75yv0lY
z$y>*SS!-0BWK8F(xG)9UPQ3J7EH&1{V`I}&=h(RI6Fk?fIdg5=W5t$l-t4(I{dC{T
zT)Cw7iSLMZYPt23z*3H%ZzrXsuy9>3U&NAOsQuR@YX!sWj==K{TQ(>&M1H+ubV_UD
zW#zVpIqRE>%8F0AMm%J0)%)L|8#Q6$!_{h^L|03%o95WGmTl*%g-$!wul%Yq`ugsI
z$kmr`B?9+6suf6^uBqr)%XZXa7Q;f<*#cRM3(ZV&jjm36lE9SPE#rPnH}}HiXzmXw
zyIxG^FD((0+3$M$?R)R-48QKZmhsG)kYwPLa>inRi_Ce2^2ecvm6nUS=e|p3J{2=R
zqENJIO_IpJN|A6iiD^|GO=8VLtqcz3R#A!u4f8YSo^iUuWxeB2YU%?g?$ZHndst2?
zyx^36&!c4gVCR|C1HwJ0`J6U4@9=T?AAO-`lEdOl3eVl&CQqx~<Sx(}@isbyM?rY8
znsjRbzamrQ^vP4$?EdC3s>Mppp2pHJZ_59YlWi9blc)8G$r|nAZ8Y6@+p%TSvSSSA
z6zYn3)_*#o7I)Q?p~0j>L`k8+@P@PESBb+KRS)~4ELO=+x)P%wrr7pBrB80+rYVnE
zR5d2G9JQaXb+t)Gts$h;OLt9*$9%VqjFG*b3`+wP+54_J_1st}BK)AK^@IVV(TWKz
ze2nva-hJ3&ed>wdey>9;O|28I|KU+Q>UGU=*?~q~o7G+(hs1uSMF@5)xLYtQKHqY!
zT&7RvdCieIT}~NaX0)m_YMd<btou@I-+aedWl6!}zdS<Q{#mYdd#lo;CL<=cDMN3f
z`05$#`;}KXuT^r`x*(>^twNwj%5yPiB)7ZqGqXF2PySD1%4n9}5ZPuOlj~IWfX7wM
zp^5$GAzmScW}Qs2Bf^5huFQ$br9UzzJo?z<mR5AqWU;Zy8$p(?+u9i=%Z|C`sUC?w
z^)gK~wrDeB*Ah|2WLK}KjpvQOZJBQp(qaF4ra-+NyFuY<_m(K9uGykN_SMN90dv$D
zbQJ_bk`j+g>`S(Ey!gnT!;|}^&sBjbGx?U}-|TXHX0&S!56e_O&XD<0hK*5LElMgM
zVs}}*<}#n6XtL&%+UiNAA=e$x&d|?35tq=k+?FkM+X-)n5TOFe`MNF&7qdA$C$b)0
zlJsPIR1fF(uH6$>l}(%z+;!>4jLrL6R`}N{E?gJfKgnNc9gB71|F7|V*NzEHzR|7t
ze|2A&Z`r*zkAq?>q(xk+8;z#)7PMqL-d*$NwE*i+rJ{&Lcio7O@kSm8tam>?#Kv`r
z&1gy^&vpsj{!K6ETJPB6Ec~FsKVF?-*53)KazQ8fY<mui>~vEsy(-C=+4AVul4)Uw
zo=)6wMAPq!hI-GIsPM8iw<hG?oKa(StI6|F*~2#*Huhxrobaf!T0K=~ldQo;R!xTK
zt0yd2If22YBlX;p2`UOf>;FCI*JM4w7nV?>yV#3E%k=Bh8DEqGT4w324EU<K_QxT)
zZ3Wsl8LF1P{Zcf8L-TM}nirdpkK(jEhO?`F^_&gsFw1y5`DxR)N2_iuGgmov`po~d
zf+H_pd=0BQwn=XDw9?Z#dN17XrATe%VVaobI#KaWihPM>(%nxsnaAV0rd#uVnpa_$
zZ*cQ(x^vaq^UWpUDoIx_ok~!;aPq?IT}GWwx*w_oXC4uER9yLf`Kyib(Lc;>RTL#6
zvYX~kz2W2>{NNCmU{SZ(olU&!W=wGKvTnTGwu9%^9VX6gD=zcBzU?i2@4IfrOQD-)
zYAbR!<@V-XnIL*RCF9W5r0iKmCavN#Z?-6w&fhH>y2{O5ZF*ow@`Rvm8`C;Z#5i&O
z?fQFc(!Jodw-{CZ-?@rvOunHwneEKkMGA~|J6qN(y68CFdBne9`lO9l)~%WI^3Ao=
zOONL==)PR#^gnw^>PcxY=^0F%7bd@Wc<(%KSDT4*gm>YMjTsNPv(lJ8WxSZM<XV+-
zT7^@`>b+gcnO<`ycobJZj`?!4)^W4j^!v}3Oa9d5xMA4n7jba6zzm0UksFoS#}B`o
zq`_monCs(;EuX!=ge)^EDt`Q}Uhojt2Cgr4MyFrK$A~=I^i54)J;zx(W`>XTmXyN>
zCQI_W+Dzb2u29_79_dt7U#;ks@q^`xU{^YeN5-TJoj02e`_<!Aqog_S>rLEOx1vck
zs6ss_=Z9fJ1fRR<G}EW|8MDiFa8)_3Ot&_2mkZeVXZwkkC$&r~Dp@9YbQxveoBI3y
z{(t^cz497M>L>c&UY=3)@qg*srPeJQt?vc0t-WluPDy*s(+u;J#=3<9qAQ|WKPpN-
zYZNicdvlR%>jf@Vj%Z%P@W`;tT}K7xEX=%8qSJFwA~d{o8DrL|bb$i#(AC#$*+1G|
zxi0Wp#dMvU_mAY*Kq1v<$trJBqJuxmRX<5?IWD$$n=DtMzxHCm90R#D5raDiE4Ug(
z*FLZ@XRgfo9GFv>Hn}x-;$@+~&2{^?$?0yi=890{ni$;B7@4v_!lJcW{)N!e<B4?|
zHN1(6M~+)>e^9z|a^mf&wYd&li`Wyk25?s#6|?*-VSU+{X`=G_Lcz-_h1>^2T!Q^V
z9yHrVw7D47T@ZH_cQn1<ob_(Hfa<sZ=E*HZ%z|MFUqjUr>-HTJNNWjGTj(~~T;Qo<
zMS)0;1gmflqomM;rcdS)n<V5fYB0?HP|vWU(s8>-Nf>uRc+hRrGT9{V2RHJU-w;qg
z%6Q;#uisOy+D3^t7I{nydQ07!R9?s$9OQ~C?SErnxwXLZD?|M6hhk-h9+xa899{0S
ztyGdDQZ`dv_~K`&g$K*#7TU6Z)LB|4FvlV2;wBZTlPXo|GP@gHue!Hfb5{-6+>^sp
z^yG!Wt?dGjQw5|LWnU!rfBW3p`CNEIglZ{k@7MIy<N*HIXGMz+#WOdG&NuJbZBfNL
z(V3^vsp&<v&&~Gc7gP4P3p8!i(VtZ1pIIGg`M<AuW1?_VN3wYPx&;+|H-s4268JC4
zO)PH9UYcWMBy!xGamosTgG#=kk6czA%viF*<WY#<dJc(j5uuBllAC_kEY+~vpjeyo
zL#8jo`7m3Z?{<OsquD>U2$ZliKPweD6H$No@Z>Wt#j|&+&1js=@<mwLqv6Q4lH7uZ
zq?-bFG#a-*7T7pbK<b6HRgyK+#177g*_%%`DjMe9*637@XiDFrdbiEucAMzl2mMSB
zxL32JFB6&jcY8h4aRKX_68jgic$Emu785qE&^c`+b>mAy`ANN-kES~mW%)k#WmT+|
z*_d3i-R^^D%V%c&k52@bXfgzC=+-Sy-y>0Ut})E*Sc!1>|H(RvtUWsw4ATpo8718k
zvv;{kDtzqde<8H(fUMEwDKbgz3fuC}U7XsuLm*XwF*v;==%ta=uPK{u<~>}|QSUfS
zKXcl*&mEbW+*=jWb0vkWq=dQB=gOH2u(c<7J?Px6ChxXNd0~fO%}N>OQ_T)51pKPz
zZ74Jio$Re9wxr=<vOsD#tJP$~h4xzPYTA$eQ&?mt-W2HT2%8(eAiBYYTQHjWcu(CI
z35AOboD)TMxQc8~6`58c5Ol#Nmtk?7km9yXqn9sKb0<W$bTn{eaF=}$FZtHzQmMLU
zh1-&X{_hn6e_n|GRVbO98riL?)n97W%^JC+D`NJiikgztvl$b<ZvQW0^H_~Nc>0M$
znb%T$1!tCdEuF6Xb;&VTf#anDE5(*xZ?e1hMMcU`x*}cR)#MqGOZ3iat^KUbIzJ-d
z&<=r)=k|M6*t1v7Ubk`bCRdS<kMx+Q&zIlmW3pu-x0A7VWn+_J`#Ua)@`Tx%x7wxC
z*o6Yvo=>X|<jfRUlyPHWt?94$Nk7Dkt`%!Z3V(AD`Y>hk@#8)BBz<?M&Ff*4uRO_G
zQz<C$(IjH=hTW-MiqdQIPl|aQl<j{hlgDDFCO$K*Wf>1+SjN}DwGWofsdSff6ycn(
zc}C-e4ZrFfQa76%pW$`Ds$4?2jmz})4uJ!gxl|9Zs{OLw|C~#D1M9(=a<@~P)EoZ`
z{L#?hkZ#Cpt88|RY-#keQD=~P(9_}~!F<K%th3ZMt&K(!%}fQ}r<}G|y-d2srR(D|
ze^+2kc*;aW#ra<<CvJIUE|id6@ksmQVZEP<tv6Q*+}b73*O7AFQRt52X1>*v_`URF
z4g?t`ZZQdN-~D`Horc7AhQ*3j0!D`!doG$t9TS_wI5}pk(B~b!=Q?wlH+tk9TWnJ`
zbNcnb#BPBvO9giS4*o18>=de;_j~y|$LU+FlEkwGbhm{Rq|6BZx=~4blZn`p6c0(E
z4V<|ba(iEtyfp9mv3TqAgWY$^mmLh-sivNCG|ZX#Vx@s2+v-!3Z!TPrsVyv5*thF&
zpVcMtZI}Kx21sj%oK$jTTFI_xZR^Um>F}(%p-df89Y&qG?MlMG(zh+@T9x!r;Mehk
zyoTEp8|75XQWX<zI1;11CP<cOZgh{b-DV}Q%WC6A?W~vu%}aN<+%3xb;K+6F(Y)kY
zJ|2_GFYGFObvfqi&HedRCjHR?j?boj=yprFwKS(nVD^KViRE#7j96cvs^*>{5O;^G
zyg+W%?5P@0*zy)gFdbvu#}XjCVSSl^lx`t+$_B=&43%;RF6jeY|0nFN|1;@ySf|6P
zLjvCNZCmC?FGw=oQ?u;h3D3thBF87*YmV7yAt7X7@Qdl_<j8$`6NQ4fs`RaQt_kq}
zewpk0ZPPMQ;o`2zDL4L`^DboA$RgkT$|TR>^xVYYXBiUOo19oqlyYxduf3?#;SpQR
zfw@(R8EYh^nl7@1T|aGg=}e80c(&l7YctZ)+$1i)iMYYDjU#c};#C6ePh`rBif>tl
zoX-~U`Z9q_aeGm;PO0~HFHvR7p01aYYqoDZ9QsN(B|zZbrt`8hU0bXt#Z3yWdUWLa
zucBwtLW?J~hOOwB?X`H8cVDR~qwJ4`ft3eVDhu$Wb>s%H2`iOWZ%kk+$ggx{I@ogf
z*DJ37k7RFLVJmC!ZM!JG_QJ8_mt;>L3f2wYrQmd#S9lNSX)*cKT+WBPw=m@%{=KS(
zS>iam{>|bOa~sbY<ZNsf>@rL;adrD&E6shPdFSDqj@{chj{f`-`eTyNZ<m>wT1OL`
z*Q%zwMFj0P`QmP|d7Bq+x9}{15JSd3waY)3UDn!iVBLds6$a;&19^wH_OY2A+dDB~
zZ;On`L$32F><6~6>Z)X|Qz*T0T2eDbpp|XGuZuxlBEnaD#Fm+ea~u`xE{(8nU*&jQ
zjC1O`4Lh{8VwP`rU1EE&pigFR@LCsvnF7tvq%Lmu6*<|QvvHzlk>Cr#BMsF@_AV4;
zn%MICva2)41=kr%ulkl9Ww%HU=~=$xB=f^bFS3PGHgNhk>#JO>l>U@*EmqakT;h{i
z-?j~XZ?|a~E)idUJu>wI+X)e=KQ-Q)lFzJR^7*jsf5L$jdy&M`WdVt69To+tT5n58
z;OEgUOGt>EnZU7Uk&z=?R6wtqvxbAEz`Lzi-5#BCEOqJlUDzRWb!(PN-t_Ia7N5T(
zbTW0tox^MQHY9F}dA0BETmh?nYi<6SUHqv3CHbgr=!Bb|(Hs|h+PmFPI+wlqD<PB+
zJDXRf@fd4(`QdFbOu~v8TQ8rHQMkk&c1C)GQ261S!WX3vObMD+n{l(Gbna2D+y|0M
zhf3M*Ol^HNTYZt>+XoNpioCckanBJ+`}Je8*~FZhf0ys&oV|K2uyzq^V8N4LiXPiV
z8S;K#;rw{$WzWk8nZ8FC={)IN;yy{Gb)^ZDtl!NhH&(_|ITyFrFaLjpqmhB_<Ev}B
z`kW06K^yK`U5f3$9lOSW>xCoRLqWOQzD3d2JwkgGxSHo~`a8|hEUG{zLuId)-cz=o
zi$dye*<Nj9tu%Zqv*3YhDz}Q1%H9yRHJjGwE?D2$csRF!n~Sg7;Yo19&kH;covrUK
z%sViDMf3Z4Opn+aEw>0btub(9$T#E`l@m)9aGe`5m3<G#LRP^I=Js5t<-(P%x=)<F
zExc2<Jb172`QodFA5EfUA6=Lv;4oqT2Dx*;F3opcwla-%fwuJK|0ZvYABKKo?$O?T
z&FG+8PKfZ77t1yF2najqu68_k^0mFm;?oX7K3XOBP5y4__j<O#@qTp}o9BJ5xuO3h
zR2TSKyt$EDV4rp-M{5b=k_+l9YQ>@s_NPCL&sOBx`oQ3ApyzUC$1M-||JhY8e|0u>
z_Q6S01^hSff3^11WWm)RojQXIOkU}9+2_l9wMZ?UsJy!IaNm06H64Z3`%R8N-G1x@
zXY#Ezdq1w>E0hs;oWwVgp-1}D8z-&{3kBac-upW7O;i-u<1m+V6E~ZEU8`Dp^KxFW
zmZ*;e<Ff>%>b18e7j4suc_3T%fK6r5O~XR=<yWiEZCn?(#(MQ-fyD<LT{r{(K5!H<
z5$I3)FqJvt-BUva=0_@y{Cy^eQYFOH7(dE7p4d2pfuZ5t)V?Ja9n<^XxI|wNIT!A~
zGVa$@9swqn|9mPM3?~n$Fe&F`xlRamHsia^)1xtgcV&ktYsQv{3(D^OOE@M88eDwr
zIYm`kY);1Ny9cH+Em<P7qG1`sY&lVnn}<${X|j6d@l9C}%p`8j63e+kHKDs9tUSv1
z#v?a}nVPCOI~5N*GO$l#ir$*}beYcCrM|QITz5`;8@+>hI+w)erPmMCv0UdAIJ#<H
zLsgR7(^%6DVfIl>5l_!*Z@4!jbnl<9QmP5Et3zk92FZQCE`RWRVP>0{gVYy?%S_VO
zXL%Yd4Cv=%J=Heb#c>kDCe<@fbbmczVLZUfbYI}l>%!$s%QVvaWK_i&FDw*pId@Wd
z`pe7fxn4Bc{yOlEY4;DgtpBkKDnI2+o{%{8-rwqrDQ281#$72ldPRcG?lLhvKRM@w
zx5EP6n9O8{9~LuCD;ue8IK``XDk5u*QJ$do%IyXsZ~GceR63j+9Yh>lgxG~X9-i!H
zQs~O4^?SA90hU#+lLLP(S}q^xw$Q!dkXz<@u_`yoqb6li(N8yjV@Y85XbKa2s<-1`
z6}MZ{iX*R=h;us$*v(?&Ik0Rw%dACPCr5FAUS`bcRlMRU$C7ovM%ym<?Kax6<@32l
z0f83_Y`*K9uxC^e=v+G4AxqVBn}96GMYD*L8usl8QB#(ka_K&?cE&7W1~bM|q07_4
zR0OiTo3=f<vRLy@!D<EbXI@&Gf`(gMUH{u;Z#7lb%o02x-p{l;XG-V`O`VNLZfWay
zYR(pMWvKd-=)Up1)9*DK()=H7RnX3F3TnvRzPIdp<38OLMz{Y7A23^(QLVpFMaW~3
z#>@wz9$jX=)!r7q-O2@A50_mC_i41qnA~i-^TVs1jn72R?tZaKx?_sbn%6BnUs#k{
z&o!R9@NQ3&mYBw)?@La63Hv0Xnjg@eDWti`G2SZkZ1nbwqoS%$M0J<*`Y1&*aN8~_
zO`q53mdQQ&qmb&2s%cMG+m;o5SuPhgWu3adZ&)JhiKuBy)ibA5RffHYI=N@3+72O$
zQiYA&O2%)WTzOJtxnpLvme0!_!VLBsKKph0Gk$q;+w?zEvevG}o{Cq`E6@Gfm;K~V
zOx4VgVrJFzX_~LL%+%PS;&D}X`?RQl1?Slx?OAXkGl+X#XrTJ~FH^QDJl>qceR=CN
zFY(h>Zbw}NofXQC1^6mNU0b<y-a~({-X}RHfBSGxIp^me%eZff&Aod!>i8`l{b8-x
zX~G$2@vv?0>fgzJuT)n0H6Ghvt+2<q?EmBM>t9za`1H`(q3I`^KY!{`ZmBJo#Mb82
z{z!CQs@F8B?Tb6pgw>*7YMPC|R333wE^<B3vS1-+U5ArQS7>|0lu7d!YPgvHjO4J7
zQ2Lc~Q2b<ctHiC#{dGNu${RXwIlOY@4!g5d{$v$D|N1ZUp0I8#WlDJP|D2Ej!}f(|
zl?@h$7rjbdUS<2y_e*22quDk4g^G-f4Ne~7{7S4wJwAb*pF0+tU6MPtu%(B^Qp7jn
zpnczmejDD2K|VPfIa%hOt<=<1%H8(lfbT?K1<S_pr+(oRtamKpNr_P5T$E(B>_(z)
z#6t(|HxIga#8f7$O;Hdtl~k{;c^&n4fvjVdBY*6}Jd=k~C${+o3)FkOZkJQ+)nQGN
zDqP6ocJj7|NLPotDoaME+{QS^52a@sAFQ@GP}#L_+k-RDWO6jZm-Lk#dbF`MaI%7+
z%8806<-)m}SUIC7ELQ&*B5BFTrNZ|xUW@gCU+hFh#_7iD(K$|R+>4g5#A<dvXZhdY
z89t|DVoJtT>3NonMr$tGh6TpF{N$-op0skF@Y+cXE52-5{&Ufs?LyB^-!hw^`(a6y
ziRQVP^B#%LII7z9(z^5KHSu%b4~XQ{8GHEtoh+~~bD^T}&CKeEg%?U^eGF{*z0zUQ
zl8L2SeTCDL9y_=lJka+@#bHaJgP)AXnH6G77c;)nkjzm$*>qzOkMIhWaDi2dLL3b&
zsTo&Y4m=X&-1=~4>Is#apy_hg7OIrAw{)?6DPR3~srrURozv`V1icU6IxKc1+g<n8
z$78>85{07^?&Yu*nU$3q>haB)%`jv7vU>;IXCBk++qX=?V7Z5|o`LNAl0D7yIfB;s
zm7L;>N!UI2e_!Eb>llfCbJxxDzU53=^wKjqNPe2d!(+wsTzj-G%({J1_r{`<B3?hq
zTNl_kJx=g0*F9-`vY_4Q3a69w1E%&phb}zawY){As8xH(hwx7vg<hM#DEdkyU%ce9
zTF!qn`$st^uCSgak=YW>Vp|kgbzKtUSUVXST8<rUYWSvm$2hp<^5#pE3lh05WX7|k
z&WgMEV&$@Fswe(>2nzSUII|;9N77L)bYd@;xZW#~(@E~%gv+O{4QBeeRAKgNg=Hs}
zs;E9#C+lU%xTI$)&!n8gnbV!xwoGbRGvmo2Hm^d@DJ2)uE27><TfFSj?Ma%VA-FVa
zS87`S7v<^Z51(oazU~gt`X9FOYliFGNJqAy6U#RnJ>2Z@&Erw=+$G1I8C90IFs+!B
z<;W}_s#NLrdRd+0<fY7kx=ZIT+At+za_!7ze!n&a%ig_o+$L6{)N#kj&X)y;w(sja
z<m^zh++6NLfnL~|C2B$mQ<fBYH~X#kT{dA#yHhK(v`3e;ngbJaYPn{vzzo?F4)>N6
z9B>Su$e?Wb^ObJ(Oih+``Bk5VPVadWerI3Cn|fxQ`AqF52eX5tSJqiic_#E^=aZYp
zjuo38sI^^DR=pc}=ZVVl=}FT53<8DoSq`|`sycEjapo<yn6$)A&?Q~i{7%3^Ce03U
z-j(gmUp>QwF1BrGnrAxmmBQJl?(2@Z-TSYy?PM7DCPtrmM;1tTcxjsjh|XTU#L}1T
z<<VHjCXQHv67`$e6N1aHPyP_9>5~4d$?CX*u8YXDEj+%nBP(Yek_&rXT$VF8dgAjy
zk7=9Or#WRj%zZJ9-MENVGkgJK?#(kroD*!{zX{fuE>|O%GBe&X_kJMv+$on>F5j2j
zb60}R{%dbat(;})T$%HRyGw;_-d$YT!z!maYs-5zHRc3IsY_fhay9Q+Z(FXQnjp-%
zV7h{s)%&mp<B98X8d>~TCU^>+3Tr*IbX~ztv*|*TiyDktA8z)E+pf)J-JKM2XKRM5
z?it?%-4)UYYs{(*+mkYM3p2lz&8ij>|MS;q=gPRo@c%DOnAg92*e0bXpjYU4%wXD%
zo{b9H-{w1rNEI#?I~Qis<`sO2Lw_NMfFI|CSm_R?FCzBVoC^bX-<D8WHL1a!OFR8B
z!{i(4O&S`@ySwlGUU2udO5=?svrjB|Ag#D;N9+11u0@h-W>;*sh}M~IA#!5Ynv+`A
zHQFkHozv|<cTD77!Xl!w)!0(OWID?YyZZ}xm3z3;Z#C&^bUD2;p5>r6g@JGVi|IU<
zMZRn7@tr;YyCFZHvuV?YKAXd8KO~($ywv?Zga7nyn^_&5W&)i54jNk~Id98gHF{|J
zBjJ#ujMJP8hhMyOe)&@4>g6R4DcuRh{m~LGynht#dn}1Quw+vPv-$rGdTxjM(r0xl
zuGG-_*!nbDD2!tRqcpQ`fH`Y7)9Y?-%^BOzD(&6Kd8ktP&~^`L&8#i;FV}b-7Uogr
z<onInCFFenLqFq9rSf3uJ)C-`Mq0mS+y6VbzkTA?sM-5fCOZGnm}MX(H$j-AVyD)m
z&BtEOl)J4lo$b$37LMbLnc{XayI*Ry)+@;_vzT}>lly9+Xn(|nEdj?DT66M92@21g
zD!gY&lT>$5gbLpZqq>LUnm2kUN;6wV>pZg-vAMkY#bNVp)e4TYPAJ4!cB^#AwCwI+
zk$LfQ-@*;f+dAx?&D1%Wc_`>h#~x!f#v9gGnxq>QCEkDJjoY)#PnhAe#rz+E`+xs0
zkpKLU<?Lf!8%Kskp30^z-EAD)=02T)9h}Qdcsvt(3rrj?AD+V+)WB?`#Ok0lXM)^H
zmjeq|2n#bg-anzT@rb*SiMP{|1>Q!RJaU>nRT%4MHt<ep;9TJC<TOFHXTu9;1D4<o
zZi_pWS$uvd9N6x$^N`4nJuI4<0ZmC+yqa7qBlb)-{%Fp7`sl(7dzucMS@cJG`3mDG
z<vCMs?C3sie`(8pBcrpE8~LXs?zpMR5qojE{1e0Nr|p!wPUWBBJpAO~&7SV}kM_0+
z87+LgpvJ_}^U?858LYjE%*~Z*O>-KWTDEsB@nFBK!<yqz%&GRm^5jdKsdFAo*{1C^
zg+Zl%%ZV-jXRa~P<f=8?J;lSbafWMkrN8H5&i6kzXq~Y+F?sFohfa4MHT`zz|JB(Y
zx|;VJga3~M#*8O6>&y|!kU2YfMXRoZz=~h|;a8-Me$3HhUMTD($isThE$4{KBt3Ht
z*6lv$bOmPYoa6n*vQu`>g>D}I9~KUwdpIXd@mue5G*qYK;_0&!Y{ZQ$g&#_+RJ+x{
zrELG>!}e{SdnO;?KdTZTXU#E3XiAj!Dh*T4pTRwyJ2<p#FC^Oph)Q?-e!QbI<M172
z{%;G8E%V~zywKatqH=Mw)*X{GtaqkN>z>0far)v3Qx%-KANU*>;5jF{lQ~+!FI;0q
z{h7^A?=&>dX<%KloT10h|NqS3Tpu49U0boGfdbY}A8k56_H-BCIdS(&Cu66NIpa!S
zjlR1DP8Xv*LvuZ!I;{QT5cJ_7-xuSpZ#xve>^yl?dcj)-&iteq2N(NSE1&w|y<SVS
z<EKZbnau?;N0G!*uAs(1zfwH~XK&t%MlMp0n|VU!abDpnaxrf>(x}C3`B8k=meUUv
zmfYXqQL|C<#LPgYO(9c1cQ9%$syw@oF)>g&_hRRo5Vg?2Vx5jtMZ&6cntG;ia(%W_
z`r5lZLs-vZKTGVT?+rJma9r5;((AxO;hBjS<~Lb<vT&Oq5xnt)#S$-}&b^(Ei6>TX
zY+qP8y=3DR+f`cYd5$%$knF#)IDYf-ivMe_336Vkl?kbjQR!_w`K-ooN5MR+)?VqF
zey^kZW*eM2P{Kd!?REds>)RLmIo%doJ7-M;udrgS@b#JVnq*d{G<BOu+BXI0PQK*2
zb#Lbe3!9iT=i4i7W-pPS&NP4ag|*s2`=?B3(4AnlvbXzcjl{v!fVrNFZ$>)ZoDwzb
z^tQ$i0pS~1Jwz^g_6Q5>UKdF{KfA$cO2N*)UE%(>LL|3FOlH1&aLbjjIp=D1ZC}g@
z;Jk23@saVy-JOPveP)T$Q>vt&-Pphze0%%S{_PCc{w3NwbT@qqJa$<|Bvpj-2Xmyp
zZs)VV9nX93rIqlrEbaehU}xkoea8*E@Hc*}Djc(K$~gQtJ!kggxU1C_7hR7fuGd!`
zI=g*B$gBjX#ec({c5}Xeqx2&2))WILkI*?gK3`eB_SU=8Yx=g_D(AKR&amV_u#=jO
z=f=I$H_EP^{$$?OP0`%j7JUee-g$JbZ}-|WPtLzD*z@0pvrD(5$(y0*Q}ooIx5Xqo
zlE2>V47eveS!CmaPT!0T7kuxOUro#qy?bRT=XK4%ohE&p3Qkp9-BxWCR^Ap-w|BCd
zTkM&{hl}=@pI>C%SSl-3a_+QI#D~oW?~D6eOSyQx74J_?nx$c}&yz#9LzzwW)Sb%~
z7d2v@v?@P8D%_{aX`Cuha!~69i&N0a&LWcoch-c+u8o|OsrPs9KKWx8-~Q)*q9>Rl
zE9NBsY@^dX5r#d-#P7y`P>HS7UDR+l%Je47gAKupLeDQeFnyEh+9rt^p4C^mw=G&Q
z`}Us_&FHyXdNsG^cFi}o6JK=E-gus2sx4=iaCex)uK5kqtM0~3dbqW0tMG}3F~9bn
z@cQ>ueOly!i?g%jx=v@VoA^47efJ!;Z8QD-_^x+0HC%OzXyJW6=dQxmM@nb!KFhcf
zR<`n9@1A(J`%7hf({zs|PP;c-z_(saCdebq#`4r3j&7?}4|o6ci#~Z>cUfnXs2T%H
z{HG0pAw6f`wq`KbbzA7(_}v?Rw@1d>m;dk45Qby-Z{|eyTnU->?}UG9aQEN);&z>`
zpVv(JpLB>l&R<DwZRpZ1`@D5O{SM_R(>)YD|7KNc;BC*N+K*4J_T&@mXmaM%Okhrw
zmHDx2Pmz#bc=qFdZqAf4{zFs3cW}IxElYEpeM0$KV!POD>9f*K?~=Rz-Oj1opyj7z
zWUyhE+H*T);e`q2)yf>3Te$;eA8+*yU^9HP+c)S)9rs>YDc8u(HjbFdKUas{xHOw1
z*n6hw_XE<$GMjDVj#e0-d8Q;<xiaL()fY3ihPW@g|9p{p=R3z+M>^N@9r9t#Oijy{
zZFxIM`DO37E87dM%k@3k9G7|H+oM3)S+fm1Ro+-?#!PP9xuu!$UW?!>!=9)5OH#Mx
zYVKfY>ay+leQ{;t|I~*nFKuRN#OUkJ3Q=k`(S5!1-&@aTi@%?HQT;CW`K$R`??u;N
zJ$Cra#SFcUf^Y9N)^%2H^<s+GV@Z@_xt<*}bGq&8>}Fowy2Y;dBQ&{|N%Oqh@H1?3
znlI;kEsh_*tc;j$F23~Un}=avH}@aynCTzS%(!rF%8NMnxB2SJJl^NtlBj=uB}}IO
z-P=iS9ag4K9`&8Dn-w|5?vNkrmKKh@->pfrGIoBe{lwW(q{)Ax-m|m#T0`-*-I^PA
z&#_CL$0zZw>14}P`G~U{jQS<sER#^(nsDaGp{DLvm6EMr@(fq1D)4Wg(R=IFWQA5k
zcg}LzT($J<)jjzYuGv@5rTAODJ^a5Xeg|Jc?lKL}ScSiQQ*5@Jx~%=kd;00iS{HY9
zPI&1dbIaialW>QjL3(2c({AskEjOEV<N4U-FU_;g+tzW$L@}rG+|{sY?_$&rFZFxy
zeOi-QM*AIa=hm<bM;ad=Z0$cKmyw#@Z8o8)`sow#^D@%){Q1*X{n=;3+90k}<8Wu&
zvuSBxrbry<<a1JOzi5AH<Fm?)it10_rWS27lF)n6a8HYIhXG?qXScE8iJ1Y1d>=IZ
zG)N7-Ywo38@i5_BL*vXB|6<Lq=S4pI+Fn`_bNy+TD#tsw#H8~DO%Cc3)@Mz!7GLvB
z`z~PnA+*VG;c}I0KdR++eoO22C@*19SDKPuDfeGa#dggq@9+ktzE9Jq*uK0VahzM9
zv7ph&pm{;PO|^c9&<lol8jVZx{ij@Dn)E}bN2lxBi}Ifx!mpSgMR7j1T|N8bg;J)Y
z^ZVTw8_&&2z4|$czpy)z*@bgmM?&10z979u+Z7d^Wesl{FEmZ*k91IbZo&EX(%FxN
z{3Vj%u}b?VGp2mc_WR?|ec63o)QtHdC-wih==qedUp2S?)8$Fu`5rVaumANgaR1C}
z&o+oQ_-1%asyS5R8@OJhkMrNWuz9t=64$nUkBzy)H$Udh+Yh=KzLyO?J)OKGYl=eM
z!ima#1`2Fn3zkS<-6zjw6v?+KP2q>Dz`^`a5B0KD{ri9W<z@Xp|JJiV`_H<vQ~fe~
zS}IefeR(>ogz3K5+3S^a)wMUh`XkiPr>DW3Z`Ax-zUp`}V}pRgf*uAvCa)<E6gIRt
zG{#EF<(vp`U~l1;oh2|~ql=_DlS+o5KvB0xV~U59NJG#85yqe;ITxMME}xmK7yRhV
z%Hrh*rm94Xv3M;@aBjBY{NyvMXhqN*&k8SvsZ&COS23;h+N#wMDm^D^_Od^#CiC8y
zvQ}!Is`iCxH#bKManxu9H2QH`zPuJ%RLm9E!TILXo9Jzt59YSMHnXwWzJ}>wF{5=^
zSGebsew&ACp*jqk8Fn&hIfpEYT<>>@$;bT7?TW9jIM=BcJP_J_<Zy?0VcP#EGph?u
zbqcBr=_t=hmNZJ)p)h?y%F~41LQ66(I!)qPIcv7rfisqh&)eF#wr|e)5v;&Ig{zax
zWL3!Cs%5VfiyXNo3J2d4c<Rh@Nx+dsbM28^<)IDC(@Mjf7q3vgpb?~bRn%oG2g5?o
zf<O<2bc;l`8?B6a9tpv+RT7Ijls_GtnJiiNWja&q#ib5RdS+?|ySRKD7PCnE9CYks
zJDMKimX@IsBe~D8C19dJse*&}?w@XfUi^k3ZJeqpCQlvGJTzm(w@f|g%DL&_x2Rs>
zJrWG*XTDCEEW+!$ahZ4Yj>;fk&Ard(w(CBWjGLvh@Q89?>7<U!0bhG}9E}XAy&5VV
z^S_dHVx#87rwUP)rKQuB2zbtV5XPiAOL(DgscTph-!^4Mm)Q*(6VkjEUYj1<v{39t
z!bZKe;M6Uj*rGaigq3ACRLx|2li65%Osbc=!;?{T<5{)9pxr0AE>3jkQ(01AtF-(v
zOR$eZYY~^q#D<xSED8=gpED^4FxeO`cl0}NyZxwIa*xwppNyuhcgybPE>EtA{-J%K
zviavsxoTcss|gNYI6r8*@F`D^tK}<J66mPU{S?1Ny4~Vq!<r3Cg(X+At&noj&b%Rz
zu96>f^OV-%m~5|Ir)*}N+*YN**im8HHo0fjFIK^xl@}wv%-HbuMyg}1P>ZUg+{}ji
zF8fq(Hcl1&uQ)Skk}ubk=xKU7nyKQcIWLz^w>*?qGLx%EJ78y?`Im2g9BXc0kl--y
z2^L}6GV_Rc!6KK-VqXu=T;}(qiFJyXCS$2VjjO7vd>yZEp3psu(0NltQY0!gU3mC*
zWH^VZt`O6nW!AFGVHVSi#3-f>2OmUk+cH^r6=RmzOqSpWH`cCcUE4L|eB&jVlE_)H
z6~?LW)7)mU?6|jeR!?u|NgZ+7l1AI>v6CZaPx1<^p0M@$v9_4^!8ZSEm|n9xCJHby
zPFS~-_vHMbzjLnYI{f3!O|o`T{K=yhJ%i8J?4bHv6BZk$7kv&A8U}fsD*l2h{pB8y
zEat|1u$yvw!Bdt=JCAeze^q2q%@s3ES@c4oPOGJ3qL<r6feSke)K}d;d7@<7Org6=
zQ&wtnpXLa$S^CARRdiFtnXZn7XaDdR=RCeSBg2GI=B0)i>(8Kq36BI7I9a^87GKJp
zQ!?T0(<h3aLdjm6QkphSed`_n$YI8;9kUoVoQ-+ZBrUsmtNq_dH-+--uKO`e!i=+A
z3d|!X&%WR?sgc#Au+mCl?!PJK{HgDn6^%sv<y056S*$2c&1B|gop5qPj_PE-O;30Z
zUTO<#$qG^wKC##%QR$G%QrG!`4>xqQ3db3;b$n~Qw?1p7ce3PLQ;`Y0hWp)5Z}Rc3
zxq2yR`X*L`yvlAPPlo9`j+Cz3VY)Q$|1ZH4X8K#s{^Deq^Ceh3|J}1?<!dt=c(ore
zaEmL{aXQ5F^F(+FOrJOB$I1lFl@d?Bbx!%NvPS366;C0H?V7(!HfeI&E;|u)X=-az
zPh0)VhpEp#9d_8uqa(gnL(qEH<k^ug+QbffyO}$S&pmn~!>?!2&1D%P{Q@n@%!Q5Y
zyVZ8K8XU>u`6zcyBWYrH#rkE5*2=k8IMmF{3YiLGUddD{o6miquUT%R<Zj!#eyYd8
z{Cb`_b3B|p4nK7%WC^+>w&Q}ulp9MDQ{O)KQ<Bv1RBIEN7UC5qvTNlvLl52lOQ&M=
zPp-Vus}Y=$$^KSGRUjy5ih9rsF%LzdJx5a7Rv9J#zsNV6Y1Z*`I;K;;F3nK5EpSOg
zA!1w9StgNyTd#CqSFP%m*)oaA?SpBGvXG!q;fVk{y*ckI7skdjH5K^IX$YR9^!I{A
z<Lt?QPwLlm2&^k8y=32b!NrE@W^~e?sdEiC1kcqARB#bFad*m>?()o_B_*t7=UX4D
zB&CWhJS@i<AgGqQ!0M&H_Ae`|wnyShhCZEdY$_xCqO_cN4Mn4GO<<7@D%rsl=yXJD
z%R-jxIxe-9sv*YOkqeg|IcV`yFC?upXlh<}@5Yx--lEq`7!x$Jys~t^ubCnw9Fkkj
zBDuJG^%F<&DT+svjiWgyJUg~^Znv@j(X={t<4~Jh>h<NxekK0}7q_lmXS}xgNT<^q
z%LO)`2C5+^o+}p@T|Rj*G)(Hn6AfNzum4G>*7q8%QTzWY?SENUaIW6z0Kb#>*RA;!
zxK^PxNoe(|qje@GJI_4qSkQhm@|j9liC`{YW2MRU2>$A0DuUg!C;2?+c^`H2kgiGT
zYL<Jy#Ol6w$M2fQ)xwaoS@UeGM{%&NLb!M&!#k_z^Z&EOGD=)%;N}S|msv4EV_VBa
z-;3&12bJy%FmkDLRP-HVYrg*b*7DLzePWtovxI`5rYur)-LkgG{l)#zuN4AjFVo{x
zPA6>I!fcZ?>k!MMHkoXL6+1asZCQ0qpqlyWGRcF5SI*tYms`xcQfS4+FVnpKr_AM<
zsm?h+dRupez49@sN8N`#OXes|P(S=KFV<F4@g#Scw))Y>iN5_2w+|m&y5Vc$yr{~^
zZ2`MG?tc$hJ==Io{0^(^e4(dnHLjkHti0hooi%*Rx}1hRM`wrb><V4AXsV94x2>UX
z1Jee@G^2x$W_wjjZgW|+XzPxGIiJ4h&3dpXZD-Bwb>F7eJ`}&_AL*tW+P8Mr+i#T?
z5p8^1D(>EDczdMJZ_jOkql!lJ1XCVVsC69ttTOwYW_qRa`R;kUzVts6a1_zV)T|U@
zjbK>vLB`MLDwh!7_IAm&7q=_d1oBp2S*-eXjk~|e&7CTunVVlu%+FZmD#E_}?%qg`
zNd=dFSO1SzSr&Wy<(|@<D=jV;O<kQ=9=tj){^_S@eXpxF+~F)qd0{M)nj7R<mSr?=
z;ceL`;s1W`4>|OdZ=2}+$n~1Pci)MdljC>ZW~WGlMgOyNYc0>No;_uWOMBYd34RS5
z9OHQx=w4|)|8oxSL!YxMvfFk2GhEL%tvuD`a-&JMZNmeu#eXMlw7J!Hr0v^{Hn)z1
z>{fnV(ax=JW_*3X(YK`E<mAMeEm2Gij4^w71&;JHlmwqSaEtfhd7exD#hl_yOQKeF
z3JV;Px$}|bAqUIagQvZ|^xqThyZf=_{@c^*99=4>%>S`e{E5o#jh6SSbmBHHbz%9z
ztJ!kq=}*zKjpw%b#)Yb|{y$g9^}?Xz?@e*PEr({k;aND*^YG1}mzH6ku^q*GJmduD
z_8ghl@s>lhlyQOy@1BP@sxHp1{UbMJj-bn%+s;?o0t*kQZ411p6ZrXI%V*iduZ~A1
zwo3C&?9gQEun_c`>>J~pGIcF4m-0?wmySDLD)WLm<~%HHv|V}JcVRr+LGj0X<#pc8
z`QOSRC3nT|O{kwi*PO3YjjX1ebM>&Tkb7NwZNX9&eO-kRi~i3hQtVupHx)2^Yf$(e
z5S6oY+GYm!LLSflJ7G6ohV7UlkhX`vaAlkzqmpLZ<T{0eJ3h8(X+;+<W%<@RImqC;
zM$^>0OO9NyJ?{NLD%eJG&JVt|g)<&l#{GW~*x(Sr$ebz}kf6w6JI{kp*<U5PI`)F>
z%z&qvac6s#WFEC_HT2k8IBBNrwDULTFZNg%v_|d-ui*ArF~wIs#WgG)C-2!FWtVbf
z5j%L>;l_-~f8_k0gm757boVY`_z*pfhyAUtx3prndeQAa8c!l71v@YuDAJwI6L<f1
zuj*IMQ^jvv?r-t@&UsEkL@q!i$abkfWv6J6mC&;p5~nf)gv{KJ-Dt1vWDULYaP~u&
z|0{3Ke7dA#j%(aTp>2^u%pZf&OBbK9IQBV1#K3Z5b^rrwje=T};N%UD-*tLit(-7n
zg5&fl{M!q-Q+GW0ZFALh;jB9WC(PD%JMyKpFX3LZ^Va`=HNv;=DxCSEX08|>`6j`c
zfsskUk;TPP#G^?p<B-^uZZ(VaV3YLnXD-Ej$5tJc{j-#1&z#G`k7ilbisoxvD3_4i
zz{w!`;-vPhC-H3$AMI@q<e5^X${KUUmxbei#1<#r15Y{*C^Go98t+>2@9q<>w=COu
zBU%g=>P=Pe;y6;Kacp%2+t)Qy-l#5e)LHcI?mQn&;k~}R`!ao9a(q?UIvr9}ZxkH)
zSgIOSsl=-nC_6E=)l<wtF1%sQ)5bS5|HR&_xOLu;Q>C^`fWdcq(I4MA9MUc~?&@}(
z&^y#_yLEbX!i8U)%Ys}E6uYV=vvJ72V&PU?_J8Kf=UY;yo1RGFI`Dt-l|EBHuUQOp
zS?Z3MKk8ZB`&ez(<7s<la4cFpWyRwqAK50hEXYrh%CQUH{^DA9i}y{=NI!v>6Q`xN
zNs7yB%~&dRzF=+lAB9G(U(4EE8bdcQSN_YG*thidM(&!{mY5mK<(77?J9x5Kqm{{0
zU{*tWg`%cMRKNq%c^w?eiE7IkCMY%(^{H;XDYfjS(WZyXB4(QiooF)A$lr2EtMhob
zfa-;W)u|T8dn6VK2Rih<>F};Suf((}c$J#Vj0``Qr;~5AUH@`>+LA!aMZByle7hxL
z`rfhh_bpxW$&IClsbvF`_7$fCd8`bQPbN;ws*CMdw=mv$ljoi#mpyN!I{P(>guMCh
z>v4UuU#3`IA_r^8y%{o&bK(=)B>hERBt46tqm^8HG*jA0WcISY16&d<MF-<;7PJ>C
zY}(WIRmUY@Z`i~@?`4YZIUoF%?|krU4tL(N)0d3Y9iJ_5&lKk4QsiS&KO?zth3ndX
z4bL4P9T9NKny7S8?&%Q;1;%}Hm+#(SWe5yk{p{|ejO$f?5B*MDUb~5<hKZ;1je28>
zK=6t1yEg4fTCJOpE#cCAZL(-(vXqC@)rGn_jj=L|*gtWIZjCNDam87|b+V7|wJ&U4
zWfxt}+}<sA^=pArZ(nEo-)RxE*4Cz6iDYtMYPfvVLuOL!s&<)~`-0}I<ayKhhIjw6
zJjYl5H&;#le`A$ry1`5NnJvfk6m^PXHtby__KGF)<<f_|&%_tqe<I;>a+PL?$I-|W
z@u%d{j9;}p+1MERMCHMul<J!|zZIQ|lu?M1;E-H%vii@Oor@m(y*c(bEY(k7l}}`M
zF7MjD4GS^?+-#T0PHdI>BXV)#H-S~Nw2C&(t&e;4g;Af6=>U&fV~Lleh>FAVytT47
z73`-yy57Xf?9FH$FmJ^sCy@{DIFdaC;?}P0oAqZ2H`ALZ5(XTeJGzs3@49^IjCFY8
zrxg&dr{eY~>*31y+EoF1Tr3}$G-|hMhTRE|-5Qy6ZO((f6C3+7XU}St<zm@Zc!=dl
zd&sM}=Vq*3|KdWv&A!o+_n&=&+N~XeGPP|Q-|{9hHi%>#(#SX@ac5&$R>A9Y8-vyu
zq$x0l^1P}4!>&D{N$kynw@t2ZqqI&I9P^8s8Fu#wV{gl>1un-u9oODE+2OEWS7~<q
zEctsiZ@w&R5wm!m6eV=})^)Ao^-W2WPA8o*e&(Y2^2kD-)<y}AmZ}LNXY?CST;t?p
zDg2|qJhyv!t{am{mZ|KWRu%Cx4o3XHKDI4*%BvgxYHwb>Spi4Cfl!7cPvmmB_9J&)
z4irnNGYM;&7rrg4`@r*i-L56YlXBbCCLGjAxGt7)$lHVQ!#acVy5$_JrI>EBONPjJ
zxiH>-6y@}t-)ln4nnSwPW-K;0btfH?`M>dQu;zV%4@R@a9&0yM<<Gb$su|(hsI*{t
zPJ`o<kjWuuuLeAwpVTRR;I&lmx=&|?WR45U{4ex-&*GKAq}}6mVA~;$9w(6m@zRIc
z{!Y1Xo*l9Or!4m|+lGCKnSa)@R~et!G@Ku?cx6nrzIjBfr*YDx=R$`#KWnvqo^)Tp
zyl>jGbOW)YQFB`yMUrPsWYTY6Kj$gSznPar)EA$+Y4nRXVV2&hA6yQ<r<F*1I6e@3
z<+>v|_wKZ93;G%E2khWD!npX|?UvJKx^uo5-U&&5^`_~Am9c@s_lzKp>wZ~rDi4g)
z984o}JpSGZ4B^}&d+_|kb=xv72i?757^ckg=!LPXKHvW}Qg@Fs88VqpD0jF}vrm1N
zM&TOXqq__WOMbDRzBNau+107?OOsa0(gl&HUwz`5yT#qXx3AN8`ihHcy9zJr98-Vw
zrRubo+~XrF<T{>JvQ;Nfu`4=}k$yQnJhXMqX|Im8AJpDA+I~55>oQ~SSGDMz&yzT&
zDVj%A_&ol2BI+f}jF&AUEA+*8pYsd27qho$w&(Y<>5`2Gt)W*UOF51()>$m~U=$Bw
zFUo1O;WF4!a!e*R`0s&Pe=I(S*sr}N6xg?UZkl^((2UQw))v1g&At6G#Wqvjf5!z4
zh82ESzWGg<^(IJn>T$ExoRv$XcJ?VRoR<7^YVo!e6KBaUKY8~}AkW2F|4S7$t3Gim
z&z+p9-fH7%vO}*=Kt6bGg<@h$R&6S4Y;lj&yR9eRNj{inVqEk^>!y?Dp4J|Z2H*MJ
z^DmY!ne4$7^E~WCPRzct4uefg>$E1GE>nJOZlCw6ZcE<E-FZwOj<dQqmaSxII4r-m
z)HFCobc){keEz}{b4x2tr!4q#^7EmsEte;J6T7CHyuI!G{n*<orMDCRx@HGmIj;G3
zW`ccMM6hm}zu78Ihu=Dd%cN4KsOLCvmn`oy``~2Yzh^_X@RRA$_unt`_V74Zm%EXn
zrzxxWbBBZc1oMJ_dp6{*{yZ_ia;cYAzSAn+zRK4|TaC0gR~~EenDNz$^S1YgyKDEZ
z`d?9U-0KHlh2VBky>n}=Cd~WMoujbvUfZ5is_Fkkt~AWj(YbHrY5l_ec6;-V2_E?$
zwfNl@neSQRZF;VnL#W5e<4<<WwL`^s_ng~YmcBfWWxs6Htg6@N9>nK-4fAC8a}jrI
zc6AmGdRSrl>0M`q!Vdxc@?e7>Czr7A6F9|s)iZMKReiSKyHj{hO>1r6z0mj3*BHa!
z@n21s#9U%1nv@`H_Q!EK!<4nFPbOTne(vv=b<HVvb?Gj@GYQN64E#$zm?>{~_&K4D
z;rcbUhXq1g)iMsMZD``$+#Iszq{Nx>4MuC*Z9W(J#jnu5&1Ym_F>`Z~=yNTxnuGZU
zhq!+fC&V0k*Ybb$@|O3zwSF-kb6g(mwV~2!`(-g7M-jdb8$&-4#xob{YlC`pzUpmt
zV9-!i6*>KXW0*n5cm2g%ZaK;F)voz!?61IBmghJ{{N^(Ge`zWQZQTz}PMpJdcKh$u
z7asAm<$QMg!@$DCz$DC|F@wRgMMY>)#>7oeA3EzsvxID^xTrMKS6nkprdcq>gE2)f
zPh`Q;BoR&~t3@t5Q+vF-H6o*0G#pM%nWd!v>Ivti<qZpDb$>m%w2^0eSIU$=6|XBw
z;)?^OPWvet<h8nNZA9!_tp&YDH-w*zOLbnAb9B8}^wPa2)Sg6a5xpI|=4cr=(*zGa
zwH;gslpb_fY5!WgV)_Qx!xI06`Tep3xI7sbi*>WaSf(WRG@X8R^vyK&bQaGgIa0}0
zS0sFcel(T6y`dy<$irl^Z;bTg^ka>T4DaqrGfue2A*`Y)r`RQ;$!DF{SoQVi>#mFC
z%eMUpWYFZ5Q1y737kt32YxbP><vUt!yfrOcgXSkpY;O4OWbSi+!Q@5fe@M*U^(^J%
z{;pq>`8v8pR(Ws*<y7YCcE3Bc!m<0Akb=_%ttA1w0=2SUFAB4GQgE2{_G6!2LcS&5
z3ohNt>YjSduSU@Ax{bF+r^W=21qUQbbp*}#*KB#Qt+B=C!~zwOSA2_fw@5tJoZvCZ
zhe6H$ke-Ux@*bnbvWAORE>@9ATCX%SW7q$zy<f|?I(Ka<i4s=+b56y3r3Guit3Afs
zB?Oc;T)UFG*E(GJ=-<P{ni;?tu_a@zll~LW^Ve-mrYw@-7iKWrWWoIC_yLo(9^1}p
z+|9joTSN8Z29<3yE(8RL8T(wC;8FTIv?t0oCrmQ_Ywu;R`b%H+!^?#`t{<`!HMNk*
zWPQ0krC$4*g>q<Ap3)~DW6wixPAoDyYreXq*K7Nl$(m=r9$S;4{4uMD@9yj~EN}Og
zJzd>#R@34@e{;^(JsjsRn29DVT*I?>h4vOc(Uq%R*E@Z^H-U>aQ)jDF($Q(H$2&|P
ziWdr>T+b2i^XSoPt92H}+e@O~Joay$E0QuH?Cx}h<`}*IeoU8?yo!`wnW&hnWW{eT
zUzvN*yE}beM$~rCc?+~AE1P)BaM{d1@uFkp(pgneng5!&I<qzj^+Yu6IuSJ^KtQ`@
z%JR1VcQ;;~r=6vIVW-8HU59<Ov^NAZ#+XQkFlw}gHcV^`@yYjNbV%X}+fo$4P^My_
z;ePiQQ;bR=>x$58^JVyDwom%|iRb791Ld!uVke8{Dcm&3-Da(_)b_2(JT;Z2w^v^}
z_abdV*FDX>-6o4TPTjEGwd(e4`%}Lby^mwcSJpf4T)w#7Y1R5gKkxPayIc8uHd9rF
z_AUN+2JLg=$rHkq9tUh-w41S<JKp{NgU?3~O}Mc6X4Mz}4(DCnJ(Gg{{@?W0jZkdW
zmWjIlk0ZiC;;%-qXyr{N?jn_$hVDuM3&SqxEHa*VMbFyYc?P4fS=Y_hCl#5Fw-_cL
z+bg~?)5`v_f5-u6^`*A$O>C#UY|n_>yPsU|Y;k15A_wN_Laj%0N^V}bsS#{7fn{~R
z+F{XqPD{7%&1DM?;EBom7BuJ5ghK^yZ#dmmxIR<G>s@Dm^i5X{wHXFYY8I2Fc{&B0
z&8PToG+)9z%`xwa&XW>%Zeh#m+w>mo)|mNBUD_qy^}?ooPLmj$Z5CcrntsSbr!j@+
z(CWKtQeO`S_MEm3y1;Pp(B|XQ4ftczcFl^ub98Uz)$V?o6h)5%=@V5HGK7_OoruX=
z_}`Et^ogU`r!Eh<EBofJIohtosBj=>qBnc*lb%&ylrlt&1Shrl&1BPwOgiRq^svDN
z1r7_o%oI0c)u%mPlNuLt`MaFlbbwX&;Fc3AON9?hEcN8#jl4di!c9bI*2%efiBVjO
zg{CZ1IqI2W=ymDRBm3wBz3TP90%G(`R!GIQ<nmZAk8XN#IFW0CoMND;WTn&W341RF
z?W?+TGG^*xlY3{Ut_gY^Y7i_vW!X_z{aTrg%YJNE^Yb#F#n>m4AL2eE)u>;6<z)HX
z?;fH-OQNm3S4Fc}p3V{!QeHBlCqz=z*F#MxR8})M;B4wTgPH8AFE3^VEGZX!ncA;(
zy*aLKo4Bm-f3A$F5}Xqlp53zD;ib;V<;eH5;AO}Pt^+eTvRAp~b53%)SG1wNHLzK$
zWcuzcOtZqKzDd!v)>3-Nk#;A9uQ+SkX0BC6R#GohA{Q+2<6#Y+yI6tMTKo5`J7<@y
zc%XHf<KU80fo11g^?YW{X4`dq;u2M+pbTd>%NqeM8(HhPqNStv35i{16R8z%o!x$G
z(W<bCJRZ!FqKR8pn`dbW?B2`iDPsA{a^4D)&S}c;UDs((^1FMicUgCDn$h>8Uyrdx
zE@htfQs6=G70u_0t7nIW7%o!em)cr>K>4&%bo=I$a$=$Wzt-M5p_<6u|0GSSGo{gX
zlE;JvY?Bfvy)F0Jblgf;|9|XCk<Q&-nhabS=Y4<rM8;MMF^W%2;}(pLDCG_1ZdBc_
zosyl~BXCVLnQx=$dYMm?X3E%w$9^j@4CXt##Kki!lS5a+WomnDvSZyWkMu>doT|^A
z*j5$YP&&9rKsv-)QLowUn8WT<XOEQ%`Yk)EZZ7CGGs(wKLU5`5+8*rymMhVW5v_+(
zq?S49HU&7^=1jMY$tWu1Sk%6ZRnI}hThTJ4UpY7<PdhYWomP?g$*WTqZ}=$TFLhb#
z@5kIdzc=|jvDszmb#h|Kk&VekCX%{eeAsR@np~Wz{^r*Gz>1Z8@1D$FyL9o_vqD}M
zzim=-j(IC&_*><);)|JwZV7%`;9b4-|J-{!<}bdcpZsL4J#+jnSD}pC8$uM9M@?{&
zm5O*NVp;ohpH~Fa*E^df`Bc7b=1$;U`b15BPs6^QI=xG5HtS}vOziM^UdU|674K$z
z?uyy+cb7iBNh@dFDr)#4%l>!bcAsb!jr9?8yJw|;eUQTC5v&&YZKu<$^PI{7J2!A_
zzrZES%ynrJpH)WFjEt_P)(N?mFOvW7-C<%>7!g?Rv((D%^6Xh5yT7D<b$2aV{N>x)
zFgvY)Y=NfxiM%al%dPvo<}FqE;!^Z{llHeh8@{WDwIpt;A1XV_qd6(#|F`!de0xqh
zPD-5I68z-2TC45pC37Y#ocDX~#J{V=DY|=#sPcQy)&Ciug^J8B`FBT?`F?|5+Kh0o
zL>@c#AIn`YaaF&baDcJ@OK_(C+-);>6Zx%*CJEX}EcZ6sbTf8`)1#9&FMHh4?Q~kF
zmmR<8h}u7OE%iVC3U<%ZxQkyf9A1>gdf?gf<U6J_On%rTa!)ecDHA97@^fEP(B~du
zsptAq@zz<2Df29>+B=we!|Q|sZ^+l~G&$8Ck*JgL!v9si>W`xlDPGH_tz>jve1%I{
z>D0O294|K&*5DK8t}-aU?Mi+-LAig9@+75~I_H`$O<3)y)B7mq@uN#?9?#<cz3Isf
zzBj+_oH6~z=f;w~iY>pa>Vs8{nwL$_iUt86p@KR4v?|XWwmhOWdC7nF&+C%7*0rmD
zdHna3(*+%Y<!_EnRcf$3@%;I^N0)5!bh+9#=pC6JbipZV;R!x2m1n&zZ8>2-p1js_
zdl=W|)}?<y;HA!$3H*%fbfOI0m|S!Iy!tgM`_Z$WCtUtFY_$C)POAKS_gm<7y-eHL
zYnp=Y+``^VxGq218rNc_vwrKm!>xB-a-CM%F>R6TzJiTYxW65nAbN0v(x0<0+D`0@
zaCzBrS}!WGyC?ParAO|5S$kC4^?xjK=u%00<F>R-aC2K~*o$Qme^y&B>#?!n&~18i
ziL2Xj*ZVDB&g!NZvL~+>;Pl>jiHZHEp3Fu!wzc~HT5d+CK1S|a;<xM6ktqgKZa8zz
z`F}xa({rU)k8OV#-f3a_GQ;O}P;dOTqul?`2VD7Zk>lyte;-)lk4U#?<nBAKbnio4
zkMm546@5~Eo_@_D3>*!A3{+Aw?@jA()+kaGGzxv>KEZBN)QsNKOZZLZJrO+qQP)7l
z>BI)d4ILBqtp8s3$kJeicMF@<Ip1ey8`W*ruc%A<!nQVZj!%nG(?6k0H6QvD&s=EB
zJE8N4Nx6&fWy;Aiy`$kfGL~4dD$hFl=9iPfymba91_c?-eI;*~J57+<*m3{TBdNrV
zUwwkUl|0ej!KCN#PCECKdi;|29GUFKi_|tdpILEs#XHxhOC02ntE|ylVRIsx>(t>&
zKM$_2hc7H@y6f`)P|GZ(yE_(`Sa#{}TWuQlNYi8CzpPH32!3hKXDlCmzg=SrWm45q
zN-+=LaIwOZyXn2_q|TlNI*dXM%7O16>hP)DexxV%WP!&gttG)$$J!peV-@%=bn;&N
zw|9EhN<wU$?}fa5ZffPoUQKI$cF0=M$LT`mlT(I?DU-J7DRKRH+G61JEQmw%>XQ|Z
z{54q{xUv@6-+00i%iN#C)L@hAI<1H2>w5i_>vK*Rh}!b{R(V}<<9(&(UcAoRjNx#}
zlTZ7%I7FQ}bLJMqgOC6v^CJs`O#XlQG^5N#;O8Wrcaxlat~eShJvk686RR$>*O;w3
zWa<s$Ll-tCWwGC`N!RnpP?h<gZ=>}7$fc!zN8WNCKO3v1RQ>Allxz7XR`hY}GA;G%
zpI3G6`MTh5G5xN!mv!HquRZrMJ7kit;UmXw4i@puv-^xK=Q2<H*UlfWGQEgXCtgJP
z-xM8|sjjJ)FZI0CPPx2c%cmT%Ufnxw>dbDoo(qK*o^Lp#X;|vs?2!C>-{jxdRzBep
z*49e$bZvTCloyhl)R1@FH0#{Duovsj9l0CfdGyeGC6j~Pyc2dYvF5#6)W(&1iAU9G
zTa#I5$^C0zX7uH7vn2NzY%2Pqt+&u{@`Yv>&8q=_T-?@+>c4!HTXiP;#su|WSCm+`
z>S>6$ZNBOneK03xqQ10?yX({jp3GG$E@zhf=e7Oo_ijP<!832p6&$kGYcewYczenP
z8?(<oS$^_{OjU~tte7U5+*LBkG*#Kr{%VolGP_ncji#q=2c4h!s=RldQ1VIf@MV>~
zM%HSDF^xtguLL$v*gRV<$?|K-jEMdK?Ig#HX+GztHj7P3zdME1e@Xh$W$$ZB0$I;D
zg`8FqS{-9F(f#eYD3;eL+?hhWIY!4mYdtXz309k7mt-6ED8x4<wEF!voBW)`Ay$5Z
zEgC1EuPOR3lPaKiiIus)`PiZCh<={9mCJ2D><?jo=h1lo$`|zj5jUB&dLR5<ZhTW>
zF<o3BD1P{>dX?$Igh1A2(~ei|I`_Wb^AHP}+hxn`;c@@}l*eg-?%()U&r7lub7J88
z^3iXNTHo`@bCm6k_2pBq%YRSy{dPF#`)60BgjqaG;u)@;^J~@S`D%LJo#!`~bYM}+
z)<EmOr#99li7ewuGkv7|{Y&NT0INS!P70J;F<(zRe)W>X+5p)Ur%ju6Sj=5TCpjsZ
zT3^}xyC-C-!PGM?x=NehDBt~@^>C_4ij$|<BE`B#*}+qFO3YQ#M8p1Ww&yYl^KLof
zCz{6C5wSHXao3@Xw+@B-ra0D`iR}#1llyeJWNO53FBi6@VtbzDGK4SmvQUk64m`Lu
zJLg;UPH|-k^TO7QdYv7WzCp3d=CzyOv(`(bn2LAsPd2=~F-}l8d+YxrR?C=mcO3Bv
zOY)S>x6;~tJI>t3EFo6sQbx*kujxhAUzk-gbhOg0PSIGTY1pLi;Tpu#{KI2~U9h<0
zlp|-XrahPwTzg=nWBO+opVKw_l3L<_w3Uh}r|f7@C|3y<vtnh?Q4g<+dCze2kmy<S
zI<YN@0pBw7jp{`|DxJDGu`V;vetPBF($f#VYGo*?ooO+dmKO4HWteKW)iJX#2`Y>a
zXQ)qOR%1Bf{33tX4#CLyjjrlmi;^zK<lb=Mxv9HGXrV}`O4c{mCBZHk5-P94RJg>O
zFJ7r}NYROKob9Ug#3+2@tGo#Z1XDUH?7n`D6XI-Mcci4deVXM;&$T~pn-ouZ^}o3(
zqUZB1ew$Y;)3`eflpHLi-)RMy$6RpsuRZ6ZdBn1{l+`uJOu4eM-D5&lhGUJQzP7eo
z{nhCgw+0-!>hf=jihN{yWoK;q`q~TSs-+z~f^EE$BD|lOs`zk+UTgoYRa)#h^FWjn
zqbavrpHL`gMh>sUxjCO)(kiEEZ<-dG-nVjf3*+|oorWhSbgQf8G{2l$S=nP<Tvp(G
zNK4vWrK6ko?ZH-UrZvY++xCYu9Gj*7=AhD*s5RGjwe9}3Ry4AsEi!OwWQUK`%b-vG
z2H}%91?O~|b<ABGwdsxT+tNcUoO7o$)$L?;jJ&#{_rvC0y)k=yS0Bl}&2UzU>B^c1
zs)>&OnZDduYj5!&(N@~-%n#G7#opSNDrNk;U7fggd2%b{iF`9qd99z6Gy5OUFH_l{
zjz&9<vt3?uFm1Zz)<uHrc5qs*JG-mha>>76r`ByW?7u9s^IlnP_Ju{KnwA>+lx#Np
z7PIN*yK|zMQClOlQ#b9_VE16^3tk?5TQ|vkR-Ki~DYK~@H<Ucyn2I%XfBqf0=%b=Z
z;-`SeKbCY!on6DS@k9aNo&&2x4=(LGxOysY*^l@FiGai>X{R6UT-&v_e2LsV$;E7z
zW)pY2O8)fm{uaM!=ImI3;=fil^ZPSwzfX#N+&rbi=vJY-?(C4ag|?3hy8?A$iyq0f
zCY9}uIHCVO_MWm`&VTU}&i!jGRv1}44h_0}tVZ1P%|<1jbEb=IN~h|i?FcxvX^)SG
zvSo!Y_Y+PfnWQt1Uv9l<bLL`=<%v7-w-(gAKP0|}%jMmoCh5r4l@ZPIZ!h&!y8qj<
z(D|05%FZRmQMIzqwBy+4YMfqrvqZ1#(r*pZjn(;0b6Do@Q9c#)Q$AL?`{>z?+G-jm
zx9yu}&0k});3QM7x5`%6q^sw@nezCZ7F-r!6tnKU)td8FGjHE2SnBLzbm#irU2Bi-
zy6*hwdh^|Dy|w|aiwd)+t=7F$?fz$$^f5Qqzix#kg0aq(u1Oo)*ZxeYDy}Rz*td4o
zB`2d(snc|Yy1wf$?p|eiu9-9OobIOo&ySv2JJt4fXr)un%Tp~rzjao=FnTuOs{i?^
zmlmzie>9cPPDNm!o4~$PzWIF4yecZK`jZq4p0>|i{PK`{&G}N5$MutEOpe{@9Cw7Z
zb+PTv>F3K^<z@G*y}JGok4Z@0#3Y+z&AtZaGoBoa-n6WD>B;0jJWEdP{^ROUc;e5=
z?UU}jP<<WA@}O70>2$7?idfUr^UL%yVne!RuTPY{u|Dqg+Iz2J56{$2I;`KcI(0@t
ztg>5u<xYzh27kH9jlbPP|EOeda7mjdlU;RYUJKhs7O!2`l=znHyM5DW`tumIBqcYG
zs*WVD=o~L!jkp%8#D8-mL)Y%x__`yeQ}KqiN>uHG-~a!)_}nczH%aBeqI0YV#2sW}
zFEh>jY@>Zr>8n3Wq4&OOcGLFR)ivU`6u-vE@bBHVkgK_7!=9^Zhtp2p*V-zyeXqg%
z!%cA~6*6X;Okwz}fBW2lmVLGtKTiLno>2RF>ynq7=YOs|^f{U3_k6qGd*2kS_-DGy
z?)6#wC%v(MZ_mGR_t4*saW{gmDJ{6U=CtSpeVK3ikD53o6{0u=F049xpoy7vOXAcC
z2~LeUECxL{CL|s1=JZUxV-b+--Y4PQC8N4&iDy4kKwXYUQNrnN-IFGFQZ_C0=`oLd
zCUd#ads1W2l9oR+16Hu~%-+UvV^iv(OPXHWIi@}eVYu2N?UcrAx%AbP|7*S2+tym9
zDzs0C*)b)gd*Wq>MNC3RL>aax-fby=YUZNEaL|FXoK@ziM54kZ4$d#rbncidG95D&
z{=`}^as9&`(NSD78QPnkpEu2JyjL@^#Q8G6d%H;Pq|y^*6Rpjs^iJGz;Oy1{?t-_w
zRGcp;<yzl)F;P^r+dz0<%(|GTXBy2jtER0CQZ1S$=eci+UPE=t`vZsLRg_Z;eokn3
zDW{ul^eU()cx_kh`?x)Szs0alN!4%QmfZH8^+rJA1Wt(u{v4N<CM1Yw9=z<j;b!r~
z7PUZ&L?^K{jfPgyyo!eoQcWET+oilD7PTua(@03)bXB>boB804D5pgq*={?SvuVWq
z&tjHTnc&FQDRQxNo~8N%H#e>H1s)#Dwk)Yg*)t`;&&%)UoP-y9zlu+DJZUZ;{C?`z
zR<XpX>LE^{oD-fo_@zF*5*c|=^FU<!%k<|F%pcpIPwu<wBI(wpd2K?xgkV*mP;=tU
zfCUu~r!oi>&n%gdsv7t$-D$en4pzzNoMT@3Oiw!k^J}JMY0cj=(?vR0@vX(HOr@o>
zF61j-j1taTTQTLe<K}fPrwW7SWlbs%S(Ng|J*Q`pP_YE7_8O0Bvn3*$c4x~q5^ks|
zwhB3VK3ywvzl}T3IlZfL+1B^Ar{x=}t}PL%W|(#4!R9Y=zuxTJwdzEohf!BaQp)dZ
zzn|=vSa&Pn|Nb?*IHGO0-tl-fdBZ|8nKhicR~D~bvhk3Y%QiKaCmXe=PI;29%+68d
zA;r4-Q+i03`6)l96DDk$mag2HLj9Mf8lU4(Ye-<}VAE}!=`Lz3T0g&%^;+1x!iFss
zrN?GZi0;k2Ibp#>EsG0F9b;6kFP##^H9ghc&~3_j=7qYGVkWqyX}HZiq%M}ZW~0~R
zyUaVEX=N?Cb=|yg!Qa?P4~(D5KDb|Y|5WmtirH5RE4Evf+}?Oh>G_)U>6cmLD#Ey4
ze)L$f#f9rR({-0!Tb|6VRJ*p6X+pzm2A<rh)yb~WRZiQU=c(`7{#`GwDe9i-`!^|<
zyed3?&$3wb>CEC6HJ#@tYwZ31NkVYV`;P1EJTl#e+kPZD{dKx=;c&x!^*OoEwM}H4
z`}s{&=1-q~CuNeaoTrS3T5MgVM@W(0<^P7&k!PlD;xwvI4~(jgy(K0p#IU8;sP5<F
zrA!PB3<|Geq5_4(%wCvS`zmI=Jal)iQ-uDt(6iaRJ_|knJQ6?cav(rY#Pc@GgG1Xy
zS11~@oLkCJpcnPEt!6v-{P%4;q#0%ORXP{X&{av>5!T!9_V%!cNTuu1_9w5+J(5nW
zpA@?3o~S|E1J9Yi=S*4iWJgN}<00XBK0lUR-)YDhEHQnH%Zha^Hn|E6lceNcb{w<&
zrpM)V(P`hD=kDh-)=%KI3~ZZpVrGj9PtB(PQN0y2c35;Ori3l~$*bz$w0~B~#>od3
z%4<wkKE@dK(lLF>^eHzM3Fb1gHZpb0ZurN-!m#6AQ@M*Ur|FK}aqmu+y?MF7V@28g
z%^M9>)I!cIt-Ex{ZKHuoV)L?;WjD_>J~2?az9lWW&3(dshCr1j;W+|_h0k4AQE`7^
zqVASba^&z_V>9u@C--=c?2EgpUL-mDzKdIsaO&IhD!Q-xDl`_&z0lfXz1%q^s6%OL
z+miOU4c<O7`?s9F^!B)4R-l)gbH;8P#ZQZuGCc_jxWxCqw4>^hcZ%2Y?R#D<+){Zx
zIP;U9k>3m<p~@TCDayXT_7=u1F@MXDWbM1@WY`WbC8^0B|AXwq1sg4&rv&iwKb`aC
z<Odf|&&3W;9V!-uPI~y#Mm%%&0wY!N{ZmTyI4Tn2vtC*$?D**87a8oiNl2aXNGA6o
z9nak?Wj+^9YznHC;#er;u!7@6P(c1_&qUtP8<K2nMy}C!%pGsJ&;Rvy?&Q|x9@BPM
zEGqk^wB^+e5%148V+}nSPI;+HEfv1-c!J^9)D+LD8r6bJRancPMy9<~l4{7CyV7po
zZjJZ9CJQk19^J8@**E_6QQt(@qh{MCOi0c2zB@@{(nOB#o<mB?AxE!1Q=Gl?iG!Q;
zg6ScpQxvV=XI}1(Q#bThN$%)un6@}{R={cRgkF)QUWZn&UTay~|7m87;r~s`O>&=V
zA70DJZdE1dw`A!KwMf0&+f2mwCA?k#+x3~r-_;5`D|Q@~@_TdZS(B+?cCJEH07qWJ
zVUMC!mpk6;$oHhq@sm1Uam+&QRrk@)zOvRq3~bW9fiKlgM(KV%u}aZ*f>o32g%|u=
z6|+ygC@>Oe={i#s@z%r1J}Y|uigyc(CLSu9E^_gPZ|lLm)w80^Rvmw}tI|{R+93^(
z$vbA<6R9#j`?>Xt&Sl%*KaDHaom5Q!{P@7$E>p+c<^7(oUE)|ZSE?yaS~-tv(xkoN
zr^HvvhkAcHpO$F4;51``->>uox}L1lI5TH2EE3b)nzz<pu!JY|PSxp!Pm2^vt}dAU
z?2G9ChixaDMNI{wADK?un8ek}naCrpxH#v)J}Ha8b5F-}Tdzpwo_1KIPx8LnHkD76
zyHD7&p51OgIr2dVo7vNS{@s;4uJ<dpb;%g!PPV<4$((DcxaO~tn_Jj>)rFA@W<Q?7
zut-5{!>euOc`OY~Qywf;WZFDE_>xA{J)gn{CR>7A=j9zW-*>`k{bsdpy(D47MHzX*
zDto-+ibRdo7(B%}O+Go#47qJBmln8Eb;X;kiX|dqzqw@gH+M!0{fp9KU_RKi>GT7c
zw=1@^T<PTEJ(*pY?H)5(ptwJDJ!gza@{0KZX|L7ye7ovV<8<`N(l?j*bhsARO1k}Z
z(JN5&x@2x!G<m`Q=o+(g)qVLf$uiBw$IbhCEWCwImajhb*y^v|e5dM^Ps^UIn3ZSw
z;t99xta`WNrTfagjLI+A6?LgUw=!M8GU05MYe;lD-#39;Tlcb$g^xGzcZ!)M8ctk&
zCBE)ivG?yYzwg$c_`aRL|JYGwWpUp6nng@n_iZQJRRq6LTwH3YZhQ9R<U}i@z`rN{
z7)*bfcK69irH?;z8<ixQlBM&q{7aM0|H|2N?9!|}9tS3gb>1=g7ycDqO_=*BM14=!
z%Km0ip$UQ-^-Wn!i^PQvXmmPCawbG%7HcwAG$?%({;^4W;^e%I-zAStZ|n#cnp3JJ
zaU(-5UHYh^pxj68fHL_9<%0ibr7QfsRv9WLv@1B3O+@sWx@gda7?&f_LJL?5k0ktm
zCRtr7`1X*zbBAE=vBo+Rn<xX5oC(Uej5GErw|YtVvu}=6x~TuMt#RQeDYsB5#~^|I
zfz_c)3}u2_RhWy-9@d`yq!2k(An%95Yy&nscjMU$7`u{0pE8QGyF2f5_7-_!yzN_+
z(Ft#HN5-UsCc8J+^S9|aS`>zxmz5oK2y-qge<(iTlh1U+va?GiCVWba`ji+UT%MVn
z^qgJZ=AtCiL9N#I@^v=^YQ?*kgcena7d>E>nkL$`;Gpe2Q{`O|{(b2}-f1#^i$XTC
zrY_nnd%0YG)@9j?f<m_vt<E>8vwbQ4zffG{uye`2$pzfcRrkBNxZUusbcs38+9qml
zV$&#NSyc0=F-U1q^}+9bdSBD`oBNfe$+HQkNNZH|d{#MIUevu=VCI#O3>B~6X3@8{
zXU=*ib4Rdum23Q!_I%A5LMaLDUV@R&J_bBg412%bL4I47TA}ZRM^>N89dn#I<Vq)H
zeo~V!33Q&(;_Z~H7O3D7sIV*}H&I+jL9;BJxsmx{!;5qwYmo^37U6GCrM5j#`gTBT
z_L0~J-&~%k<*^;=yZ^a7K(ZoYMuk#_RBL(2o`V%ihEc9dg?>*FI^H4T*ET`OGX5Qt
zlyOJ?RS)6Rpy(SCC03rDbtP5y9gbg(S_0e?{_`GWee+RXBeU<SD5Hs^(6WhB+D<k$
zht))KrrbVi;Jmb3@N=YabCKv{i;3HM4t;96v0NpoBr`f8)leh#e{#Dihni-mnxmpf
zLE+@6Nr^ARwHOn88)vX^6^J(6oaOU*dgUY0?iC4%8|wwyN_o{3Og+1j)01MaYsj#b
zeLCb&&^~=~L}&WfIe!*QK3v+^5Z--%XL%KS_v!DF-OCC?G8JC4<*$v%`)U$5zc7R|
zvuWDmxSC5%M{mr2`KegvMGxy1&c(sKvz4a5Wz=(A$RZ^v#PvaVyJxK6R0iP*Ro?0b
zJ6Yz4Z&lBpR@xgGD;vsv&3NjcreOKyN(VkJ?Dq^d54C+dOLb<C=1jddWd|3jZ8xSY
zKgczYDRbWEqCm6g<3c*zp5@D(jGdw;y8N2&6FBSLM-#V;1%)OiFCUi5zHGmI(=<9c
z%;L&|yrsU^T)Zb$2=eYS?H6Kj+%UUXT%qJq;Q#B^)-MCP4@FFgP^e$zaDJHt+l`4W
zp<P+a#XAnh2hLKp=#=Ae6<U%YwN@auuH9{dq_Xs_{%z%2@)_o_Cs%l|MST;Quj-*4
zy~SvitNx7dMyyX(+Ah*rk|41E5EsL$S+hUssR{P6q%A0%(#*Lc!mFYx_EWU@!>Vr^
z^?n8lH+8z_1hORy3NSDz{$yblVPIm=VPIfjXkc<+WZ?MEz{(+VBWr<nGqbQ(%!v)5
zGh6ssr!>r%z;n1;LU@wUhRlTHt<sXWHic(DTw(Y|Z<Cki1liR+2Fa)9cm^NXG0UR(
zmB@y?1se0sSpUeFzgoT{km=}Oo>?WDfsHQHxg2JFSQb*qywd1yRq%qawc^#YQg(DH
z-k2MFnQO(>bJHBES;hB>oM=moZ4_f~%Mmq7jA5+5X!hoIxWQpF7GWi^9T5+XHA&y*
z<*}*!lyr*abit{Ut<Mg3Fx$6>aa1KbnlF*$ezRvoaUA0kDfKzk*FN5v6P=)aruXLS
zTdk?0_tLh$GW*cDdP13KL|N$dP}}{>rp^mqRoZGkl|QC$PxZ~|4_I6Ni&t3haQb|1
zrv8+y9kxE-m^WT?Syft@+xq?UdA;3v>u#+5n`y-5J^8PZc&fDI^LtG^W-k<cq$dW=
z@{8pNNNg3>G7He;l??035#>z@^iWvTdc{5KYKXTjlZV+&jU<7NiA^tTcU&&Ile=6w
z#O&LS3%(9qstjRnoRwGn7hP~ysg2|fos!S<R72J0|6lb0|FB;kmt)@aEcDD*wY8Yu
z)bMs`U{ZbK4KD4DzfYG{G@NvijNiq1p=~DD)=Sbp)nzLJq?hhFb|KAJa8_XQf`!`x
zI+;|g1Qsz(b##+llW4LiIY6ysIzvvtR!#Q>&$a}*PiZvwQd^U9a!z6F#-iZD#WVgN
zzvI90lGR<UEsw0GuG^Tl>VB0;tonoy4_(#OVaNV$>00+_@2NMNce-gWS$mji>C>$e
zchzNrHu0&;t_fMC!l281u}Ekp)0NAoG`RZM)wg{(7A?GMbM6f;^`>=Vn$DteSMrxi
zXm(w6WD;C6;h@OWAgSz0ZdQw2%gtu!`!2qc)E^nO!b*6r&A%PW;qv7&Q;PiEQiD&1
z%$)N$Ad^+IsL%4@kB>z<%F<INyUjK23JU%hlhb8gee~{nt>rfq&e+t(q?SkX9oe{B
zH|*HT-h87Lb&s&wZo%`#UDpUmtB4(&aijX!()6S{*EzRc+pb;t<|EwH5s}Q4-VxDO
zb87YVyuzsebB|US8GFVTZC4A?*?d{8Z^_oG(<OHwP0-e>d@{wDMc(g<<lNBy>4#N*
zJezqgL}t#G%(|xzNfTFFS#K36m-l=lHBa<-@Aba5HzaF4G{m#judHuvE4?P!&M$go
zMS^^q%8MiXi*|fHs+YFZb-nRxi6>tjW9_-V-FO^x<-2E_|NI|Mx|8<(yz000%4CnG
z|4%gDsdL(&+;*wn=$!gt|9{_K9e-c{+cW6?&kNx@TSO)#UA7c*+aNBy<fev=_Ci+c
z3`c&hYfW<6uUu_^1@iO$TI_Aa75wE+uz=NBZnZB}o}7<M_82W(X>e>dPiIQU<a-yg
zeOU@UobIJrH#%m;bI*!M{_o>Dy|Lt#YUxbxc&kTyW|%xP;>l#!UgRK}$;nW7jA`1h
zHD8x}Ie9KT_@c{SGjlG+)`?+hA`OzKXY4d<;>$>@TrADH-OXZCpu(&Vr=*TbxY;hc
z*x+#bamk|<!m}#{Cul6`Dik>>6D5)^x+Q|$_C_GT*QLes)7P}yo2@wMm$ym&?~Nt)
zPLcd#7hh*_DYCxJIqo(0!8*T{Z)Pz>Y&P2ym}1Hh?UA``fqr8wi-~^<hk<E{cdKg)
z&(uj5nkE=&-FWhJD)&tnZM|z;+g_wr1ur}w{7n47d(qVz$rIu&9-5z<ydr<jB8kHr
zu0)C$J_!_g_s7$Q^?{$^h2S3hhl{P*7yb9I<vTL(+2-XAEL`X7xt1<qlofLnUv;64
zt92pcQFVuxnoByFMCSdQl5W4=_+l34lts*?D<-V^D$1I-QA<@>`NWo_dH;5#G>INr
z;A6VNG2k216pO8yevw!DTQUL^Ohp^?UmfJyu;*p4!A+6R5rwAfzOJ1r>G-NgM0lHx
zqhPu2k&cBGGXxq6ulkvtx{%&tKC5-YLc{q?-Y@5DGiMR%;_(sbkA5Q5YWU)P@t2#*
z(yT|W@BKA(eU{E*DfZLhfgK+=$ewLsFj8bY%5`Mpo5D9P6H6iv>#6onb<IiRy&Z8n
z>)Ph5p7gY<8IcP!oVHAvyCzd{I_J4{*S1_0a?NHvDR<_-P?+MjuQAmxoacCW1vGu#
zk)mdK!ZEOF(v)Mq*VXhdU7nY=WGB-H_rT{$PW1uLl;=08-CA%eR{HDG^35_Q-m?@b
zo-Zj?STj@FiKR)Zh0`H{O<1CxE2zzE%0d&dO5c4>hiuqQKPp{H3AFxq$x3(4<^`|T
zJhy!tdH%B1j)MYa8^lz69&$!U2(xY5z+klJAz$&2!*YCuZH9Xid1n7Oq+z$QE0`yV
zcX!26Rk@9gS1l5yjz=7ADNXM+T%BywX0oN?&x#3}ryg_({Z!iMRdgxF>!4clhGywA
zE2k`<6U=BKuBJ2V<kZbEm+Pjq7^%+Gj<PddR`&C$LwvXJjaMzsruJL^XRQ*6PMjdI
z+$7f5Y{i;|h3~H1fA`PRpj<gS`vUjuqX9=-zFs=5#mW#DX%wZO^vzp@Ws%?EFz>m{
zsX87nS7tWM+je=TK|%7?12+PSXI+WvU|AI^n-#X)>hP+21#9B(W`*SMxVB^2)^)LG
zUx)s-y1x0H>4y5b4+ErUUt4_5WF_mWH$nQ_HqLEbld?4S*1=V(ktb)qv#QBH`5<Z2
zj#(~xng@CF8h<z|%A~BDdU5UK;^dYUDVtk&&G4L8e$-F(*4NGZ5^stdDMrOJRh+Fj
zttFs!pe1o<ipSj&CZ#6V7E=`ok0a@xOnNWO<Nk+sn5_JndUAs^7t@DHJ+3wrB+DZd
z)c*Tdo(x|na9poUVPZw)8RLnqGq1-yvsnJ~3~R#YdHUa;b@lFas9E=UzOd{I5Bpyi
zMOS~;xp3!(-+#+1vG)>}PrmiaU*GH$FNgiY1No6Fy{1kM-=f4<COKvCnU}t~vz}@%
zKfN>3@4CYyzR7V1UN={s{M0qCOK_#1uCgDmjkB2URyo16oK>DR+Iy}E*lo^Lh*m9G
zc<!{9UO<=H#>Uh|ywmfyTW(es+~hig&FFz=vIW=rb55_~d#ggX$8oRkw|yO~+u(Wp
z&bO`4E8nS~{oUvjrLh0J?T6a^)sOi9|2X7d_hI_{*2n3JTxZwUewiL$^T7ZA&r9M5
zK5sn#TgM^s_l5R-|6f<GW4$VWb3=}Z>$aO)bFW2yx;=+wd*So<p8js-cf-}*IV^bn
z?Kk(5|Npnwzn&(*utMNnS^e{}29_7~`zNY1C2}wMAR+B2%DE|D?1#dCG2y9~SlQGi
zIG@L{3CA-P2q?{{NKh3x7Q&QbASf7KDBLd2JGs(sk*>r>&Rm5A&cK8f>YN7c#fIOT
zca~T0`rd3BUK}XEX{yfZA;IZ>qb2Y`OTdX1e~Z?Th}Ny)tq~Tj;T)~eH(Dcqw8s5t
zNjy>QuhANRqs6(rW!G{6aWTKLsWKM?9gU_s%DSYMWwi4Nl~<h*C}9`=X<lx!v7Ie~
zf5tR{Ns0WW56jiJ#QF(~U--X7q5L41N2ow)puqNvY11t#a<;~*eNj`Ll3tW#*7H5|
z<D>$kiJ|*V3~P^7Y}~9lcaxy!m-0soy*@koNF{PBn+OC6uqw2RU0hzv{IPq#N9}QT
z)+^6z_oVl1ZD-kYyyx_do(mDxrzBX<Wb|D1=nYKhJ*&ZRO@rY~2g{v~-n&0~A9D2F
zs_5IRE^wZ``m|ZJ>&CjhN39OI<{ek{IuTmGNTq>$VxIVp{!c}DYbJDmO_SDl5@>O(
z_*>D){>g0b<wm9h-eQj?a9(VTJs>5+;nLd_C#1<B`@@-Mku#HEq3Quv!RrRkk18^5
z_u&xn4g4&ww}{m<Az}aVmP`!>h40lC|0P)k3tA5>pKMdvYW=gt@#W-G+b3^LpW?D|
z@`Dvq+&Nj@Is2Fd7@uiOWpbQq^>Ru81LLz5Q}3LZ8o6_-2Lo%o=CmlyX>Kp4IJ?&^
z{Fd;ExlF&*v4AtxLNw2OYFUY-*Mcn#VcQ&;4o-j5CRk=J-1u|4QDB3(;f$ho!M0A}
z1tB3^50w^+3(qd{>Y0&d`Y~VZU`SD+;9PZy^&&zOf9UPnSh3`V+?v2ylfs3!i3m>*
z*HCkjUf-hEy+yb{m0`9c*P`tLTNkhj3e=cftUf!v@96YiX2!lJ8FTK<m~&$1oTE4A
zFfmR&byF^*QZQm?-wN~SrzWv$0=VB#_x$pG?vE9oJpUzxyLSq3oe+4QnP)Vyfx|-J
z%}jy!mh(&|&im9Uw#I;sD@cHOnxM&we2$y_-#=w&atib9oXAjKz?`^%^>du^4-LlP
ziaJ5{@1bnUl5xzlCY}$;n0rz|m_v9{Ab(Dvfa*rp73@8e(~67}rvzFGXziM;cWRNx
zf<>X0a{~kzBQh7?$yjXhYijJzIrlginHZKtSuXKVSi;me^`=K_*z<(4ucbcUQc~2X
zTbO!<aCrr8E31r5t@UisQR?W76z+H#!}_AE_Ju-6<g(7r8O(*tY9u-wt}bN?6yg$6
z`>!dSZ6;lKkW1S|a_S59(r-d~lRC4PEnm>7S-DC4ossjSM5nI*D@<7qR{Toql3eM1
zh{b$TmTAUPj~$a^zm^s5DplKlLSX9x&ZHA<I_$z%b_ndV6uxj%;c{oyis`Hu)D&JF
z*ITUMc;{vAJ;{0BIOhG7X!w7<>{mvEf#duSFXz4JTqB;ihLcm@rBsZS(OODSarX7F
zkDJt`xk7hLZd{nSAckS0xa366Qv%bYG<(|wyA$KtjaRN@6ckQeC@@j{r>SOR(Lyt(
zsvidhWG{+ce5{l#Fxf^UdvaRKx~v+mjZ>341s$vwNAFmYG{JA<CY_^<!GT`E58KLu
zZ}FF|+*nb$tl;~`+M6+TmYd9yW;|cE?2MLh#;#3m2RHRT=2rMJgU^YpcAIeM|6~`@
z7Y(zXdvzB{{8f{g=D68tVnJ@O=CX>FD<$>IgN2uT5Z`1eyhTEbr*NHIp!!ZpiLDzr
z_eu&YnAIGaS#zX)DzoA?PQ`8azt^r;U%jPE`zD*`?VojXzRdmpWA4{V-KQ@V*k^2#
zn7I8^)b@`n=RN<q{jH>MllBhF!c_JdTel0&WcF$dZq@a>y2em&LAJ4>P-fS|QxjL7
z^xm0L;4xc3Wk($EE&=w7aq1@)a2X2-e_SV|uGH5kVR~EnFQ>$(V>^-qR;j9&Xm(9A
z>Jp4FPkfgWp8sP>;Y>%F>h=?}rysCtC}D4?=2%|E*>U0b(n9vwx)swKCB44BjM=wf
z@1iIC%KtZdnk*G)Ub(EzaG6=tvR7ilI!z*l&y^<Lmeams-myxvEy^XkNZm)Y`*U>q
zckAwSHtFo5&3;omSr$4?$eR887^@6}@aoK_7g`x>B!vQm8y8thi^yy>UMPBHmB2(Z
zfvpByJ9Z14njye;twtuSI%)yy=2xm8jAJKUsS^+Jntx=q*_{5<Tk8)=tvMatP<30*
z-e%45S8`tj%D)}p$~(CI)y^ZO+B?l2@e6vW?N4x$<xv!GT<d4Ec^RYc%+vZ0ye7(?
zVCC|ve7s9QV5Q=eMu~T<=Au!8`Zo@o4`FjUt&r}x`3u{spSQ#mYy`HiZtLH=MEu)^
zL(<x>A57a}6|7LTRQ&(#jTN3TQq@a29XBy$?6Yt@`TDd%_sw80pLshhgz7EI*EAf-
z59E5Mz#v|nr^B>QJehxA0{_1U{LWXGF(sVjaa^`(jX+GAwfWc7Xe$AM2;ttSnax$=
zt%BXNIJTHJb{S2~->xXlVRKyaAm{wu!VJz~I}aZCcxjgL!~@rUxy3&@8)_}KE0FES
zE%PacT*BWDy<zecep0tIB-mW$yv3OrC$r{#?b-hQOl;7RefvI$eGA}Pm%#sjf`GU(
z!`wB;o%hW96uIN)o$cRrB<#;9tzV)y+hF#3liiUq)_1go+02j4nk|sdbBTGP^j_z!
zucdY^>0Vb@B=B8TP-W$=wc6`H6v<2fzr~ezaL1YroT=ieR#W#(;0dYLERW?XchcSa
zWT(R4?<ZR%8<MN?X6+I9>cDN|D$KBA`~I9%h8Nr$&YXI1oA0w}!q*GuciiA&4!c(P
zkUx>@3b*6w>x=enmpR$48N#SOq2~C)P7ki^w(FZR_8S*2FP#+Yxn=F_2+7J<olUu+
zH?yo`6l{Vo9FY8Y;H}q6lY^Y6&S>_SSo0`qsL6WoQ}m74EvD|s|K<CYV^Pb#RBpeq
zXnR%nEvCRD5?ACmG->bM#I@wvbh|h6&Mj?VIMKiC4wt6lGVZ`@Ob__B1#lhH6<C|Y
z?|Yl?zruxY4B{VWU*wro$R@j0apKy}-ZNI(!lhZ(lK)Rgr4$@v-6asKaF45S3xB8J
zG7HwZ2Q>~akzyD0ssARRUm^JOsl>+BN3YCP|0HHwdy;EW0QZ61a@()LwZBjOy>m)=
z;e$0X7orZ`I$yf)yF0hl_ZByf+L-}jiy!pV%&xyRTd#Wg-mL*L7p&Q48gSW^az7Q2
zV7;*=I`+=8Kd1ITxO(L6seDH1H4mm2+&!J~aFf>UYensPUng`-i+s|TrtWmR(}X4L
z=OJ;ig-`t7Eze<FC^>QMr`{(&b<c9l6717uh}ByllhAd2&b{l03<A0(xRj(93(SsG
z-WssFFg>|r$}zs>Z!d67e6Tfv+bov*@0<%hGDqI*JXtP%#5i%yn+dEQ|0l36x)A+F
zCgz{Qp#TTHi9Fgz469a8c*wbt>q>^)UjbHu1)PoA!pyc89yz`$EaI<S`8@I2HD2HO
z>+W5s%X=>A_~Kn<KAWs~ZJvaF;nPcR6{K|~<a95k=?E+88Z29F$|~3-DJlMKY11p;
zjE#ZodxMW^846j{u8w@w&wTOu^ECeb3EX85xE|WQd+<(-S1#nuvlw&LBMm1WTlbyb
z`e*6s-FvxiIA%}e<XFJT)VDG86xRa_t^#wew1~Fk0?*eUTDEU|$RWV_<Prau3pHyV
z#B5h!keT#4^A^8Yf75UC$Eg?i)&*Qsy|{Prk!ycXyt?xH+6i5`#SKT>6s5dhf0{hY
zM&v)g#w*v)Ws__gl%xbF?lQao*<+nl4WrpYnNyDpWhOgTR56|9T4O$Yt$uf{#MCN{
zep^kS+v@9AboO~#1y8;&b0)v1%q+q>_ROx6Y-^ua<*~oN%9fGOZnpRN-#sT6r{wLt
zdy99{_Tv8+E<P>W8RhuU+40*F)<tjkau&YGa^zn$;l-j4(Tf^h7+&IEqUF(@+7q~d
z(>9d*zWZ}I9%1<!0ls+>=l}9Qy7~IXqnPiR7s?){1^S6^IdQh5agqps*hOApg-5?+
zf^JUep7gV6i>z;)uxH%CeNQD47fgN2wQq~SN^e0`O$Nd8`=yiaFOL_Ntg5)$*Zlvi
z-l^GZYV-KF1-yIszmEM#KHCF*f&33#_ww0q=W|EdmhXQmbb1qCo(7k&nYh)y{-4jU
zuvFx!26|2O^m1kOV=(Lr+gry})!8NF>Ag%RaMK!L1uiF*Fs7`li(S_*XsFw2#Caj4
zKy-@6>#DDxudf%kX7o71u|QOl$)zfA(S}PRsZ$$D1&pq0MZBNSV8qnJwc`2v6%0$?
zvh0YC`m$ng*wdVwzn{Nf#Zaw2U2bjqmh0i0JbLr~+|tj9+>ms-PnCQ6+sIwTudfLm
z$jZIFIQ7&oQ?7`HgoBOj-&L(<gj_W07X6)jW=6<^qjRLB)m~5k_|Ryn759gnEupiQ
zPJX^=RmPPsm(2R!N<IBhr76B?&i`kZ{z>hAGi`Bj%KvS**~|+Vm#I&lcP8y!)^696
zAJlfgTv2j0cggI$t67?Hi<84Nx30c2ogsSdDei*g<Gr(u@^(#mnY3*0Z0mI_J!za_
zlf~LqCb~2>+5bBt(W?7lhSjPCaW7W3&s%@vpl6Sd%f%GYX<Sk43w1;`adrjDh$M)p
zc$``-R(9o<XM@I%Y!4oR-YQRScGE^Bp{4`QoyRovv@W<tEhyNiq<cfKuxIa%UuVTW
zJ^OWhZkX53EDgq658th3=KbTVsoQT=;JtXA%Li}0<06xFw(N4b;Jx@&;!=lAhm6*W
zEVfW=?lIDN+{9y*x%rHl4eJl%3r0Uwn!ByJuWtT-!I}52@C8TNr@;(%@<%_PbJP`W
zFnMG)_l)854C!+kTF<0PuLP*tp1Kke|2a4K3(JQ5l@a@AUA6nYh5O8Up{_%{EQ~d}
zCdT2hrNuWEd)(|=6ByyB?Z{a3@ws?)-o~SR;cH5nZOvm0oEd_xnsw}gH{~kt-upt*
zb&5c^AS;(j+nE<i>kSsAD=ayXoIbVmR!oMo?zMxf=a+8c2oN^>c3H)=jN{+-X;to>
zk7C)~8=1J2gI}>PEnN_BF`?Og<tF(~rJWu<<&&d|l~q_0j?Zshm>|v-ctwc$gpW;5
znP&B$jmO2SXFU43qO8I$;AB+Uy5-Z#q9Uc7${w7$8DX3n?fqYG<^HH_n~$#=#iHu}
zG@ALwdq^I(i8$HJwQ(uOY+<8wI*)TU*xj|`%G;l@c=l$$Pi$NAcd))#@_sM#^vP3>
z9r(ItikXP{TGO`$d*5rv1ZVrMTCMM18gYMtvDamv{!epEx8^JN@mzY6(>mp9Q1UvR
ztNpR7BDSq)u3hPSs`rl4U2c}o+R>*rKaC2%mh9h<$69fCYIGaBkDa+kX6saKPOnw-
zs&cqEm%hrpx!zvaWFhN8!++6%lLNo~KJPpE|5tU*D!otb^S(x}yEmU{qQ-=&U#2>}
zKCWT1G4A&*_Gwo_mc#~d+z%IMa+o5*!4>Jiv^j#!vG3xd$RMuEAt#De7EaapZ|V}=
zZM$mwlzWMKHZv{<MFbq0(p%Em?5^pr8x_bMHo=kWR$$Pppz`HsY+o5PGKMBL{SaR~
zXH`e4q0%Zp(>bbgsX@iY3$waDOP`Ys4DmC$R<O!rPFUr+;?0vnBR!Jk_n$bak@6ul
zu}?|VdS;+yAdBDftmD%b`*Ie&?wS>N>M>jRo`<2$h0U6M&)bSh&&qAfaL+z<wX3Q^
z)sZF8({_u3;f6IzLjR>%U5qA95b0!M)J{I``pdE3GwosRtOf^>V;}bQEL@@-*wCQS
zFlm9-Mja-m2-jm4Ng`D#jtog2U0x+Sy~|t}xGr(avzQ1P1$p^sZgo)An0kr3OYz{8
z2md2aKc01LOW20yOfRP{eJ&x-tqyNhGzHVW>OLRZxqwx-x=}r1L#JB23Tuw-;pK|8
zWukSvlq&KvTD9F4?>JJ*7C7etQ@+Q=Stl->G_#kHO>h#P(Y|SIe%9<247(a!7w3o+
z$nnkzJX0}c0#n}-^{K(0Z{03jxY;V2e<tMh2`1UQDSl@Bf4g#LWE{O?AsFKR@T}PL
zOF@r*d@#RY?UnUuk<LuLc8%E^?u2KUvt1L(FuJt!agbJ!)4DB>Y~M0!r}w2#iTb0Y
z8L_cFGB?mZedlDi$OnA0J&ZOdRJEu)j6VAB)$~PIa*P~idW5^NTsSST_NhaN?4_^&
z*q6n*I5Et;apFJsU-PbS8=0!ta6HxU`Z;&;9}kAehFR`g3^vb^-6W?Zq7Z$0iyC9v
z1)Vo4!BMwf>X_u7+$Oo_ZProed6MTQh+4(GNcv=Q=ebBZcZX3(QDEsEn~R!xHJeZS
zRc1N8yk*jPAW>x5y0lpVb616~PCC7!V%KFAuF{BYRleI|LYtH}O*beF>UDf*v1Zvc
z)yR0sqem?^O>f%rxS+E;M%?E?>*Ltd8ur|3$Bd3S9=>)sWm(7sm%2rh_VgT@9(m&F
zvA&x+Q3r}dB#kd>?P6CjYzY$P_|fSY!Qm_N_~`Vg2l>CZe25g#%+Ou@Mb%(p)8rPD
zBWg(z?#CVk^@r+o@8>IC5*_q^Q%d-v%Tw3pyw;n(Xxc6Y_HEnOKEGabZEJPj&k6C*
zo-3}u%GJPBQusj2S@J{xlir4#XJcM4%-JoFZdR<|Z??4Ow$5`iyXkDk7NvS=F{U#8
zaz_`(XncuzVsdHXrqqiI41LnY+IL@7-D<H`km2pK>aDNB3`6d-##XWSKEJM0(&d%g
zI(y14k*7XtrJ@)63S+LWT=VE%U&T$W=sLF5K@27p+YROgo|&-6`h(b<>{(G)#4-*Z
z^b6u%EU>XbsBBT)g{0Ieyq6ELZ1C#al0RkZh7IkXzMfd@z*C?2(?hK28o!j{g~(vB
z0~yM<`8gb;6~1N{Wba>MyH1aFTWH(%y!*LJ|KC{T`CxUdwo<Bx^MhsUc&nZA3Ma2#
z@gy$Gplw&heAcyYiMh8PAJF{bmaF_}GFRG?WgD;De{^D6K;gMP-h%JLlm2~LKV$l{
zurEoGX-n^Io|t+uXIgKuSX9@Fx;y7{R<6&e+qPx#l_rtJ3+`RHRW9hf>o_xu-BIbd
zi8c+_npRhuc*UiEI^(18VDn4`0d12@r+&C9x;$3dZNz*`FkFs>Y0r`y{|!!evGRFp
z@|Rt(j-1H3|NPNJhrsTp;G%O2Ou7%fs|h}^ROe~=sn5Q<kN&aUdS`R>zeS%D#6H`f
zw7Fn>JM(PYtm4@Z*Hl@Z-^r~yV>3g-{l?y7af{zv&0z0Z@%xa%{}R7+74E!(#lbpE
zRae>`X9x<Po#t2aaAE9@yhyL8OG^!Q<xPmw%&=j<d@6I6g~i>F^2O&_?;31e_I$qH
z@%@KBUzhl5>RLF5Az{_k^(u39gLYOf>tDKZ^|N={D+F8Xt92QF&CIvtGVb8ta7wJX
zTlWu>z+M9`!A_>gMeSFtCcI|sOl+E1Q7s=R!P<C3JaC3e;0lMTh*lki8ErQNxFr`T
zaUU?xoWLlxBTrgUFHkr6<l@rVHa8O-*sd%t>sa{ahwh6=jSl5?%1d^5AGFw>XdG+F
zb7%72#SKmxCcCtF?6gg|UL9T(%sFR*vyzYEs!JNXHZg2{S~NBA<o?qic>|ai{|`{+
z{$|}3;ko$;hiE+GzQqA^j%+x@G;ywFRc~0tF7}K1C!M<rwK%E^_!&Pi2~XJj{e*Mi
zq_vVJTwxDp1U7I_W0<rxm^+GNsp(7=S?>uGPjVP`wD20b`bsco1$PTHZwugBwz)xt
zb*2&T5l2yGzSWjoF;|SLA8m?bnXUJGQ#8Zg2ZgN<B{fWfoy@v7r^eW(*6dAh*_+XF
zEVIS7chj+_uQhJ{?s9tEWRk?)Ql+6MaD2VSl=-vtly*5ExnNPGvbArQ+ol<I9+RbX
zMKo<(R@x}9_<f=8)WJg^y4Nkf(5ri(SMx!G(H|MdWbSS&4~1JIKG9lX6?3h#WP%z@
z-^^I_-}=$v$kiPyjAp-W5Qr3VXfSqgwP`-^!E{QYXm0ba#SyKNzvaptJJPMWeNP-X
zxtKLDqLs@>iX)QMk7vOK1~2X_a^+5*+bR~mwA|sLvg7KlgSC@;KW=D^;&58dtY>5(
zqr7B?%9<UjXDp5!*n7SEkS5RRpF#GUB%I6M9KU|xSYo8ztq)G+b9fV#=j{7<NTF%b
zFX4vwMkk&~Pt#qsu1<t!UbFLVkIhC5yIx#A{J^N|r?J?v6z)~C7A;QZcH5-9d;>>V
z#L_s9lZzXsnZID+H;^+<=r=o{URpGDMn{M7LiSZgM|~ukOE)NdXz-27Q9YKu^fnLI
z#sI~g8>E^x96hnQ`@ix|p$(1`J|=Lj{dx59lRdh@U0bBw>W$nQoaP)dIrAjT)++nh
zmDzjKuN+Hf;md9D)`~fO{leb$8OAD?4wt!{{`$w=LDJ6V$uUbw_kBywI|(nkuCdZ>
z^Kl)OzKR;2rb%5Mj%VI0&s};z^H#>xSy5c9wq2YNT_$^^zGwHfrf|PA7JELSS4)X^
zwT-FED}}`o%T_R2FD_u+pWtH8q_O#xIoll$VIPTyDja1GT+UwEE<Hv4VTo?pEuUpR
zr_&o172fDdGU>I=+MyJBNZD8KdzX{C3D?G)OFI`hoze6@V|iL`Y0%1>D-L|*)|EN`
z=(gL|mMPYntcM@=?loD@SbJg5AFls<eNLN+$i9_0(`uru-otCX$0A`N@5&iL{jU%E
z-ehr@BjlJfb6Jb)hK6b38mwA6tPUaETRe<b?L2!j+ThL{SEm|Y#x1jC?nr(7aWpQ*
zuf}OoqRo;l$-SQ1D@=^{ng@qoE#7?H`h519W4DaABx?FIt_a=6aDM5I&}|<yKGwKL
z{=Kx5<M{pEPCre;JU7nyzU5enPFL@wb+4+zzM5!0xZtmQ#{cEvYnLqMeB2cDWy#8=
z9v4{tcJV5BX!7!P%(~#+EfZnAwXD>%YYul0&%~DueZ5nZQXcZ{InW)(;3=flt*v?S
zKn3#^$$r772L-DHQ*P@o6FDIC%3RJ?anlUB_yt1$?+XPfrD#|?9$M=XBp>9lNx=IX
zS7b?!%Y@p~sj*!et&43<{lEBJ|Kj8Q)%o($h|{x5Z{AnfIcM%wr68wA7nFY49AB{2
z>5f2;LCF5O-CI^@aGy5hJ^m(ak4pDRiB>ZPuYRAS8yjqPJE-_XUn!e0eUm}Nnv(}_
z9uhK?>^|e{_w38ev!9JJIX4?TR<_fcZtQJ*{=yCO-RI0MpUd21yZ^=2{i=NZtFFeH
zc18F4@0i0q+cn1N_RR#=zTKUR9&VW9v^waxXw){9Ynv|k|6}RPetKv9hJEK%=S<<A
zI)mBco`L&<+TM@B7iQmP2q_F$V6@9@Ma+`hc7BnjKcjEG(AdiR)cOBvn{A6P^oh%g
zt!0o~r|DI%8|~Q=5ng%YdcY)wo_UJA>s&8-t$lN2N8qKmQxCnXy>w(N_taje39%R6
z|K<9a%KasG|HXrP-+Xv@LT@e4^*{6X*d_-19NWkTD>Q5rnJ<er82^noZQXfq!^)YH
z<GFOZ*PQAx5x$vc6K^8L7i^u7ymRh`1(Vh<SXFq+)w#udo3L7sq1Qfx?)49RSuAg-
zzij`wV!7xuse~^^`oeB?zvezPbPINMvtH>JS{f=l?O4_ujm(@ooYQzeeto$5<)zQM
z++Vk@i!xpGTUJcn&Hv%%bvyo?zVET<>|TxPHP>>E%Dn#EWA}31|I`aVwe)X2nA4Kf
z{r`IC<04tH#b<kF>h`YbGWGV}YS|S2HX+=8GS~a+{g*mUz3p(|*kkx?>ZS!Pi^a5d
z+;8yu%n{UMD${$`OZjO;-&39*Zk`Q$_HI0rGVyQXk*iM@lwLX*aA%5W(t$alFRYbc
zthj4>D|U0r)8n#p8C#R@6fo<kbx%GUKRHNQ!;I(lhwh1!r_5u$e%&JZ=2j;TUS;!=
zV5L(U@e7?ys_w6E&|H3ZN~W9D@=e^!Erj+g&|0G!y*px|zM;;eJ2u<iZaQCmTT;(B
zd_$s4m@!-0-imp)d#AeHWq$Ng_vS~_Gm8~sZts3NSM+|JT1<4R<<(yAg|<njc^|d@
zOV`kS`eQElv{>o?%FmuoE^@D$;;wfrqo>S7ciWU6p$D4Zj^Cf0^fhPdeABM$wu?A#
zoc5i3;d;S|JEbe{&)CoOFnQ&y@Zjn`V<)lGTedJsrgW-4x#cB%=TqR~h$pSmPVLb>
zR+fhxJ9l{2iLB#!y6W3K-<Mb0%Q75;lWNpbKbhY5otW`Wm%HES!L+?GYvo=);ntKp
z^lDnJ_v}+a*;g(_UOg1s#^YRQX_}jvJZ-1%Zp&sF<xOI}^Q?Ppu8786>@_WZ>KB=+
zJw^HE1is}v9%wCGwL$Y-x@G#-qszD3s(cD~9;CYBrB7&>X>O{`+s`akM~*&eI~Lid
z_c}(^f303pU0d=8)r=crcUI3k^#5+&CFOl@ukO9MCCpA^-HX{xr@QR({;<Sq$Cwl-
z?tAcTQJKu+`w1@@<h1-IUt7xSymVEESNEAKGhLP|HP7z5^-=G==ArOSU2`2{yw0tO
zXp4UJV3qKSx>pZhMR=W)epeghXLf&LZqQ4i_=7W??tBQnruX_x#A%gDd27o)%u>#?
z63Lrq<^46<`=G}Q4Ku6XbHa=@TJsE<4{;Xi?E0|pXR3nHs#m|(JxmBTisRD!p*L%D
zzX8u-m2;QXw+1IPu0KD6J4=l(>zw|!7r`eTyR5TV&34QU)HS}eIl91culZ)<EAyUf
z_S-5POF4XJMXg*<K;H3#UvsY;$Lq6a%+kHHT_@?kv3jvTU%aNxoz%6VUu08ta${or
z3QYCG7AF^2ocheEUYxpj-}-{rCi`FLamZxf^?%OBdr3Ag<=mVGkz0Gyx^Kwdifo?K
zbz)}_GoRTD=g)DAc|)JhV@ruvVAZwIdp619$*Fw1!xrySgKD0AS}iB0q;_-5o2-?Q
zukWuYy?Nua@9D3mrWtR{p3Ss9eLODS+Wx^#vw8gIFCBVw=veQRgBF(6HXpyAu<-Er
zo~qWZ`C!gIv#uP+^(k%%4)z&$4qL2ru#=x2|65pU@y{-^2~2zq4?<N_Z*5NBmwV(^
zSVnqn?sdKHn5kEJ{-0xC9-1SQp>*!mhIi$UDoSteD1Cda*iV}8{|VcY-3E5gW?$p1
zd#5j#m%#3^dj`|317$0B<|VHz+@i*Jc?HMv*sa!aS|7jJYj$l7ypim)UG~B1rLS%`
zX+|9XSN3&dP3c$Jg2L|>4a@H<KD%nQJ!y$}h4zv|65qeR{q{OuKV#mZ4;@Zbi`E~S
zH0f%vW#PisM=yRq-P2ZO`(IBpA~<J0gX#*U7X`_`J1$?yWIjBF>(1&JPCx(Cg;Ckt
zxD}4C=ThbV>@lnCeiLs(j<w4Ems<XLGA};a{5kZt!lt{|^2&|Rn<qcqn0j3Pvrbg;
z-Fctyt92jToBH8RX>$GR)1EIT_wKuvwliJ)<(x@Xjqh#0i)eOgon|)G{QrmR&i+6D
zHD;Y_{`U2D8=spxx99)Q=XTDS&d0YzNOpI?gR<@1e*Fh3LwV1{a4qBf`v2<BrR!54
z1up(3G4JaSUZ(k_tM|`Y!2bGc)7Ovg70D~#Eq$KNbno@q#t(tsi=x9%v>iXR);+Rf
zO7;<k4nHQgBB21ch69bvyESYgE+~jIhKK~M$@r+`)*8*`cTyvvOQdHV%Z|Vs&mNr&
z<8tTI32pP?Y*o=Zr=omdSwoO|Vbz<LpVcokuFc_Gv2;V|YBOoivobeA4qWaQtevJA
ze5zO5TyJ4ePXt@G=Z1BB&sbhh;$~mZv~1nl8%qzn?htz#rgLzE(%l92$*)*vZCAg)
ztM<9pnd<NQ5C3<?Fm0B7^TW{bc+PU8J=z~0#_r`8RXd~c!T8Aqy;!>^hqfgr_eeYO
z^W3a*I^HMh&$q;OM$QGZC2X65cFgW=xX0wddH?R_?OqQLvgfU`c(tgo^?bjUiPeM^
zzIWK}%B;Ed!7O6Zx_2A4bd=^u>apifmFl><p!(XU+e~NQ7#siV{wDfv{`>g)jK<m}
z6LM;EpGZiu7ZomW-I2QNZux)PX^}3!pSn+U*dcpGvdwbl!G%t<J0%u5zFeyQFj-h5
z@IeF94Y!NkOe;br1#CUI{6YwOVS#Y;?w2C50XB@+ua_{{gsXUoZFCCJ)so$Ml0`6f
zs_4`x-X)1{jc+-^Lhs$Y=oRvxIh9qA-DaivGKSD?hr&}C^h_eyS8@t0PyWl9@I0}S
zPer_?@a}eXH}QKjXSkO6SxjgUF_Z{&%*nly;X3=Lie{JNNzaQN%G=BZCRi4_Epne7
zb}P7AuQZ5FJ!kE;sZOi8HYCUEM42=?$t+-d+^crBOweEOlULVdQO)*?6PjgLPIO-~
z;Sk$|-HNUgOU*qU-89#2D4C?IZ0a_x{Px$Gmvsf!rm}l>*M67tci#F*C@^a8l(Ze4
zNiC`YpSTj<ulRYd`{L5+eZO?~Rh>K)=(fqv<f8xNi_-=3a%59h><iL8^J?}Kw{7w1
ztZ%Q&O_TZTF1Si9YSoP+ay!E&3T>Gce$ny2^2HZPvs5&X6mlBGx-d!Jdb%fc{eRJF
z!AZ5cnf|?7w<t4AS+d3O(k{z>lUcjjd@BPgp4skAmei3IiQrs1p?c97RlB)zTP>8#
zmd_VleK-8Tf5$uGu`%DT%vgLP>d1pz%s-`X-FAqK39O#oIjcN=@zjf*nIZ>Y-9NzW
zIbjpS>b#ckQ^hY<Ki)0(mFLYmvAr30HgL6`n_XzjZaUZZJlkpk$>1%IPU-qge_|va
zvGMu5ZTF@s=1Z18iMk|Pw68r{d;1#g&Ds{c|8w0cgyPyhx3XQ`-g5D~tL?+ZLGx@=
zcsH{>yeC!~wn};~r;1iy<LnptcUg9y7rdt)y`+U<Lh=@?)BmSEcL`hXtJ@~*H$|->
zY34h%JxQ_tb!`*<LR}h@6<T#Wu61SHY!VSxI9h$}s$6)DZcgY)FHNt8X-5kj^`4!Y
z<(L-K6!xP}e|PrD(_SA$_kLaF^?j0XtW|;H&d4Kr9YGw|7pX7U{X-zHUDGF^%VXk>
zuP*KK)t7JWI&iq<;fg|wMG?u{%+2392HHPlzO(#@xSQUuBL#vF4`uDMn8Dd{#8fNj
zK-U%tsSQ6>tR1w%Hhf4IWc{UL#Tn#L%oeodu1mUkbVjq+7mv0~%_F=q3eIoWsH!e%
z$kqwYjNV!B+#`Q)Pt0ve&+Q(U=9v|!+jefc{&e3_n*$7n8q;S#5()Qo`+s6Z;WRtV
z02eDG!J8UAp#q2Hy4;>juABNsp_w_<)#1vdnYCHY&5vBB-`H+&gU2ORV&Z(K7v=)2
z5yIg+xWrnvY~E^Y_Cle*r{j`G;DJb^2}-uz3mE#`1i9zE^qz3$*7b8wc7HEesJZ*K
zf5M#$T@O1AvQ;}fc?4%JGkf!H((c=`K4(%-cLs%239h~%#;$anapLAhTU^poPH8mk
z4}3D?^cRf)o_32Z91CVUbn?2b(PU&i5SS{x_pJC$C$_sPpOm<A)Los9cP%h{uI^BD
zDbMU+*SxUfu0a9*+sq_At!;vAuI+LS-BZMFK4po&@t2S=X(#uu6Q=f6MaR!wcf#C!
z=l@pI+G_1+%_LjLlciPuIx}6fB&FCg1G_ntC)^9r+E}^sY*AXqlH)#I6ZRE!n<O~*
z=9oF`*IdyNHtlM+nSA;Fb5GBUs#|i#pV(lre8aj`^Q01(v`d#53Y1#hTf<@V(VdGk
zhq3kbq#(P(%TAqfo_gOryLntSoMbq^x3c}2zh3XTprVlAg0(5Di|1#|sCw%@?JHNv
zzlNlk2|rspcP71b&CA-pM=T>yx%z6tKegpiern4d19jY{L<Sx%Oxhx{@zEL`rTw$K
zUZ|vP?eJw<aeN-njDE?)!y9Wp&GCDiCc>4wDqxu_S9WetTHljvc9qc=ysx#cIV{@K
z67n<V#FPX3{%agkahvi&Ld5&T5lyi*u6*0B9k{l&+dz)T`2qKoNf+Eg<g{NdFiH{K
zpm?TB?`@}9=(a^&Q<l3ei19pg%xprbpXA;Yh1Sp+PtQq*FG=6J!rdl3{ot)K&mHDE
zFTx!<FC2LFbzS4u%lCFI30qyze!6U5@xC1a_vXzoSI(6a^?WTc(;?dBc!l1}3Bj#`
zGlE=Z_Z?1im0QlB`Jmh&>&gTM`Q|x`w@kKZ`s1sxRQc7sEekD~r%e>u7I5w4oE7Df
zoUixA{c(P0w!-n>U%u^I4qctz_;kZcL6Hr2rLOOe)4CxhJEgZeDJR8v_p#iwtNoLh
z&RhtIVyK<Qk-%;(&=R@5yZ+n%6<;%a1hqq-YzWs1b^8}4Fn5lZZI2kklKMv(zcd}0
z?)>r8l0I~S|9ytqhD$G}-VWf>`n@EAO|2sP_a!dQpbetZE7qQU_vH4BlIIIIRCVuA
zSKhAJEuF$vyR|LB=)&@)FBQDCQzc(qc^H|ld1vRU^EJD2_3wT3yJ`FKo6vXWxJg@|
zhE@Lk?l1K{x9j0o!#APYwN_uvoj=zsX`ePLRH!|2!GVwV|KBEWFw0dwuX*HvMBq73
zkBhegqFahnG%f|`UFw>!;rn5^@~5(AdMB>hd9Zxy?<JjG*C!P$U3u<gpXr0tx#kgl
zJ35?Yw$1gu)0*{oW>LMX&86kNL2j`wGfy4uIR0Owbi#`t5_ZcEZOTkJT)~?1`NR_O
zhO~t`YBN?YNj0e6&6s^Pl_AaQTZ)f`Th+D?2KLzs(leMMyLuPTY&pK*1+OrtQj@Sq
z+41+ypQ1QKkLd<XYUJ4?ciZEX^3UbP^>#XX5w7>{e(dQGe;M<H#hFj&!AgsqdHZ&D
zB(Ywv4N{Fd|F!vhs(f(i|Cqh%4ogq0Tp1g%;h@?a*$X+-wg$52Y;D_q<<{L;&mAGH
zvOTWN7fyti9G9KJbhU>wuc0+ufVHLN=)I|2)nDXe?_TBn7rS|<B-geC@tt?LH!a}O
zyZz1a_K&+~R~X!@=nVgP_QoI6*kiFroMq0Hb#(RXh%LX;Ryg7R?5Q!Wk~iaBWp0`+
z^l{Z-kP-FhIl_K;%8954+zf?Xc`iK@6aw6Mg8Xw1z4M*mbff98=fRU-g{$^@pL=rV
zOXaB<Gg!C!Jn)$_$G1T!=0<CSfp2+((3%5BD&BDSUh#Xn!R7Xn`)3{R?G*^@-Fmj{
zWj}M?LuRSzZCB<fSGAN{b_m^V?bhYfJT=dCO_vGpu?*j%8JZpLJaac!K62n-dHS(i
z;Lv@Eqv3kz!v6+_c`uAx!pOSUcNSlk-+$SFo}NAitxo^cmO>Nvi9H8bgx)V$(ZZ<5
zs(SY<+Y}~gqY%xBESj;MvIS?R?_t?_p=0ZmgO_$3VwYl(w7D5IV=B8;%ljq&*`1HN
zM=V)fR`6hH?2$8xEuFRT4;TfDcRt|S<jecdr6khDa8bzPfOEbp=f4%a!|!x^y``*k
zj+<iR#Pt=E4bS>aFK~JtGTHsi?8gBvLK+TC8w8)G#Gkq0=5>HQ^2(X12NW}B27A9@
z$$EN^vG>&fn{t=7FwHnQxhG?abQmYc#YpWNM=nWR@c5GG7odEm<m${D3mpp@<|NA5
zf4w{P!m-u7;o>DH?RN&ahpDW6dg^KEe2q<BT3J(e+!fKeH{(ib_~w@p3k>J#olE-8
z8>#tXrpPI2*EjLspY+5P_!a$eVc=uao*;XvXI6G$mnmDQS=bRVA4YR0m*B4BYEv1d
z_qAx7{O5h!(CvM|uiQmJ(r1CB%cRdQdBZzHo#$Nq)gv0k`y|q=pT|ymv+nKrKN56q
z-PtJH6}hk9|HMW4olh&*@-Do2b)zARn`(5wLwjY8msr=N>IwZGr`TTped;>ro&(E)
ze@9iyS-aD>x|A2NX4<;0Pl(jqa>zgLrko;2z!r6>Yt6a>s;@m5xdM~JC%RmgxWaaG
zp}KFc(zm;lS4K+8OxvU5=BzlSDNZnd@0{d@NBVc4*<XAb`Aqu%#8kC^-bJ!0CsQWo
zd`Z|Q7&}{+FM3CWNM%b6>#ZLwuE%;N*33MwSJW?=nQnGBV=<?L*Rk1Ue<LLe_!z`e
zCwyv~P$cI1Akekr|6M+-_WTHC6TagqTBle`lU)ifu)mDbyz)3RY7)n;?y7>s&?h`O
z^CsRdZPo0RIH#%dCE&=Jg9{#T%I&-IKxE@em3zxJtELoKxiy~Xnlr=GEg)r{)AY-_
zlbt03taH1<Z*pwE;ugB&h?vJTCl!Y!S}qI$tDIO4nz0`6nd(0I=$Q$9!mbMUeQ&f}
zn!;325o^p8vFe@5YOBT10*+{GIC$}j^Oh4Ql=d!GbG`4iqft|W<*4Vyrb!nHIJG%T
z7cNNftlY-glEUU+&>c~9^mrj>-a6k|PA{0RrA3~4q?+`4>$9`pe9np7nbP5R<(|h~
z0WXe>1UchIk6lK>^R8y}@wF^I`Tw=+k|%qNro|bmPWab8<LvRkyDr5h>@0~a>K+bn
zPe?{yP02a5>>iKy0g-zSpDr}dI>K7ly{`ATL+r`0!qv0yt#tghjH@Yn>4&5DOqRs;
zyiU%!(kPP7SlYNsF?Q=W*V!+Z&E2Hn)_VTwjW+3=6xTgx&b-N~$~m|<$Yp8SbJvP=
zw>><0bB=`FIpQPZ5`3e_<=GKW37PB#S&~L8e^(w+JCRj;ZPgYRmrtDG&3eb8Z88r3
z2sjh9;-_g##2)t31rO?@W_X9V&MweSE`6~rZS}r2y8i>uq)mwZz2m7`PGiR@&xfCG
zy52Zd5!yA!pnp-X$X%XS{x2fB+eDt8x_<fM^e&(OuN*vHIj}IkXj;dRkR|jZ*O$q)
zsbwKkSKq`}TodzRCtl+y(8x?&#lrZjO;@6Ia>VhRt6|9;Dq1VgemKA#RkTcXZTqn<
zPqs(7TMGT>HqH2QslDOLrOgwM-hH+#|J2o`Gv54p8|lxqaIUKQvMp=xORU=c%JoWv
z=yr~ns)B<T_}D7aTwXW6+LzPodn3h9k9)O{Kuy~DVjkADn|eeF7&KeNHfv5VU(z;Z
zLUi<m;D1gTM|oQ`*QjyLZB3fN!ST>X>0N*Mg!ZksmX$>+-41f)Y~RrFcI~!#AGEG6
z;`KVyRC4t6t@|P+Q_eK$Z(5-9^qFqr$0wPMf=joosF4k>&Up9d;(zfaQo75+w3fB4
zV-Q*UE3IJmTai+Y#s9BzbQ!3XK4>{`O}*qonAkIpb(v}Hc}uPDF7B?m8ouKoOOt`;
zk2PwQt}MGIzu(e0Wr~7JkLW&u%SCsW&3^P!=V7K~#rs<?Hn<1y@l9SBsitxyBW~By
zyp)imJB=<sh<LD*ac$-q?U$$AEkv(hp5!=#CE}b#XXDhO2|wJc?cA+hGIMRNI+*I7
z){&2@;N3Ef@mdet{oXkzdpWo6IX>maL+L3@>@BISM(_8{Xq*+ua`NnjR@JA9Tu+o&
z8F1|mTCgu^iBGG+%d>77AKfl=&67QG`jxMHj>`@=zm8pt+<18&FB0Drk#CkhB}L;u
z&-r%+7oVoapLo$*xK2pJK{(vSzWCFTU-uhwqvW<`xqWFXQc(+ta%sA#x5jN{>6An7
zBe`q$cx!~Rmd|PbTKFJf$<55vZpS0%^ll$?JHD9VhhNsQ<m3=v$vaMSf6DgyoRlu<
zK0QU~9N*o;-zLOecyN5(hHq_8W7lMU+vk}YA+ECa^1PE16^r?O3}^W%3bclNo~`)M
zzoCQsz1u>eY0Ao4I?^rmp{y?suCMPo;yI<g{6Y##unz;L%w`RNYf@WQ@g0e_e&(OD
zkl}h(kjFu`72ahEs*ws$umAaU<wVbk2u_Zvtycn4XWen+IC=AFK&PYu%dD2v_206b
zf4*{FdBeo}e=*nQ0*-LM%4zBui&NXX%rc9^)8k*1-#BS)o_4_RSfkCIrkws@>x>*9
zKc+{%&X2Y&RlNK7*+<?s|LUZ5D|kLupVjD*^o#duWKubJQA}54zn<5Y$Xwp<w|J&9
z9($hX#jd^GB{hoMS}Dj!+TX84O+3Nd<;7i_T}`?I&zv|JcFsBS!DdESOSf`puZU^T
z;g)YR&MotD=AXUfYJOgR*P)ni6MU0J(wVrPHwn*OWaIW&ps2f}xZ)-6A%;!+e5B7F
zTd@7AvEDwJMRqNcIj7E5ef{@O<IkjdOfKeb2UIxkpO8%C^=A;UdZ|*tpw(3odx0Tz
z{Td@LpM!DSB9RkzXqf(AYIV^_HunQht(e9_0m*iusVNJ1erA<2CK*QmT$5PQ@@?$~
z%lF;&{!@EH){D3(6gG-o_f0)jw!(JR){Q%#<Qd1`4WHloV$s)_ZM{matbNHxUbI)U
ztqiy_VImXzn_{M%O&8Byod0nO!!5R8-F7GT&_KDEKvixrafRZDcDdXmuLGv`1O#bH
z_=lFHdH&zZ;F2)))}c2yzLdqBY&DI4Kjn1W$!-zT^lA3%za4vUS1xW^gU+T*_rUF)
zUxk)9Zadl9;2UN1{5XTrQ(Kn?+a1w-Jp5H6J`*=P+06TrQ5i7def=BWP1d*F>y~g^
z7j0*nX~_EP_d8?XjTI%IBLYvA-`Meg8AolA&w}(fe?6|2$udQ*eWhiTVjv*DG<KOk
z!zBOC1FfyEZ7c1kI@<aEbPi=za7!w@$!fHpy~^ij^ArBIc^|mS%Z`2jAkqJgqg{C4
zNtKh`Hmh5{9eUxneZ}P^Hx@n2VzKQ1y!692wIdtc4WwKiiEr)a`u*`vjL@6qYowoU
zD=XB$mimPyL!!{`v#Pj3IfKNmg~6W<SH1dE)gpR6FXKU+&?~Qe2?Z%Lt3H|3PjjwW
zaP17gv86NTOY7XLt2zrzI?v2qGt*Dm>&rc>yk!ZltxN5tWKVC*?9Du}ZA!=HNj54I
z4~X30V!txYF^463LgnV*Wv9b;6#CiOI3JsrcBHlVfAnS7(r*VIiSLw&?(kZ$G_`c!
zoT(Y@d2@DmMYDA8pZDsm|33!VnlnH88QiZwp1GfSDFgpYF%BE&y)8VJdzzH4G(9t)
zo?&)mhS`QwwlSWiXa1gk(7f_pM`o1HiT8iyq&;Ubc|<WCxF|M-Z~y(4B@QW%?u9E|
z=DJr<{KTMQwTWKV4zC|SXTGs_o9}Yu;`eP5cRe2~2NyjucHnrS(vaZT%<e54vSUKx
z;ignC$0tgzjtuSMywe^wZG0fcm~yKiM>QekWRs%1k<ZNn)tSC2Hbye44I;i_Ea}(o
zTsf8E7%jhag5ZLU3gW%0W<{DCH@M6+ik(n&gkx)W!z!j}LT9?R{!i#!9d=M^jny`V
zo9k2L`&6qwymZ`>df4obSHZhOy(QB3TC=yEac**(Tr}fQV$%aoAvFyT9Yx0E<6JW4
z({vO%TV^nQ(G#gE=xR7UkF#vrrJwH)o|p0WetKi(R)$OKxPt;`gp}sq-kv{Sw`yaa
zQxoU!BBdMU8PfNU%;l56H+_nmo|lNgmK_(&J)81Ym9zRhQs~^BVpi02qfKOXKkHdz
zpZJ2$hrYfRJ#<{t^vS`nZEti{R1dBQSXletx_<!!Z_ugztsRHo>|rsEdZEo0+AN^o
zuJGox)3&x#0hXM95AiN$(cdzm$LItDzYX7x2MOK0%VsQa)l_K+<m617W5~&>^}qSu
zF<H&Y3Y={s7Yt3m**$9Ec_?zLrQ6qm`Rw9zY6=-AH0^e;F=aF}c(L{Nye|`!Rd|KE
zy3a@n`R!;tP@=K0@rKBXg>UZ2oH%Xyyv0N5#WH1$#@p8(d1wSxeJWDbch6kEa8{bo
zf_?pIw=ypnG#iEN^yH5^6~xlMYqit5vehM8YhSGCbq+LKx@GOcNzSD^`SyDkZs0U?
z;4wU^kiF*GiGtiW$B%tklCw~rlWCfQyl(bVwWb3;VGL?Zbqu!ZS_LmMHWW2^VPxdI
z_r=!hDatEu=N#k>-?}P8+gJbLDyQ#CY^MeEuD+VnG<m@@FU9pdLG!xbDLm#e$_r$C
z)w|+|i2Z-184B<0vif%<^X}?g@Kr`^y5DD>)TPt@h}=5w5p?3gjmJVKGnVqsi)s{j
zB4qimBYBFo+|ATB)xMn}eHJVecL=hx7^WNTu`c8mjV&>Blj^*7@i@z*4h7C7Q}e7c
z8I>j^@n|Y_C^4<zU9?M~cBz2}Q~RVBI@UGM@AAZ_8Q)IwY+WGg(8X!<DPyuu?CGNJ
z^Dpc)PVHKL!sn#l$Gou9H~0MUy43xIWrzR6Ro*cRO%9jJ6wZ|1WUwgc)7i-jCRoIG
z6wl#rS-JP3ty1bmck^`5hK{Sw=XYHW|KH)6HeFXEuq}JldK-7qRjwC3+_ycoyWh0`
z{Eub_rjuV9TDPkz8pZrKyeAeObMb=WQiEGB)-LJLHrH#t@M_;JJJX}zR&FZ@`>MQX
z$A`XdUKZt^=IxojIZV;TXRp51-E!m6@&CKSCdj)caxdw)?QNpisxhHa^q5!g1|Ft^
ziJA+O_X(bxz;Z?B(v)r!xtR@HL|kU?kY$Npwot}>&+_TJ=1eF$#L4d-_Cn0ukX^@A
zQFzjX%L>7<=lqhJ`bt<D78&+(+I&}<IM-s@BE!FL(p>|k(~hNBG0t$e;}h(vS!dyT
z-8QFN(niMJDM=&ffxErXjZWLFi+xd74vU%Z^tX**Tu~X>>XN&mUH4TW_q-<?Wq<tS
zlzw-E&%7urMDW#wuXal(%zAmDN=fMd#Lg)zl*Cwl-CUJtF*S*(NM#0bGQCJqXLZnN
zVhLbM&T-dfxw%!NCs8DNg4AS_gIy6^4%}~6xH7vNvRQB~=#78uky@9cY`lymjaBQ?
zM6cl8f=y{$OC+PxGsLBCm>luCH09Ye1EKxrPO#19(M;szZBk(}WGOl!#4u^yqeD#w
z?|gMi)ZHpN^aF%k425nqh%7kTx^HDJx7sYGZWiXX)lS)UKbK5e)KxjLW8pic#wd@4
z22C2XHav`wjTZ9{5t<VABdK)pilgrC7rUx{G>P(lan(K$Ji+kV;RsF5Oy7uCflXf~
zsyyJi8Yb~`;S&p1OEIshab}1759TB=eOo-0<$vDPE5|L=rc?)Yurxniy`pZ3Tylh}
z%;JDcVH=eDwoYKuytT1)cFrO3SshJR+#dE<PE_I&Z+x|3(u0X%1&<=qg3{g|`8;!X
z!ep=5>e)g|E^Jlbb#}%zt_53XiEPdjOWCN>a@zW?r`p@OpR4=#a?ETpUg@OuyeL<o
zpMUXW238X`E8|rvJ@?vfajuxQ)lf*vZt07KTDz8aKU#Fd@tQ_MnM{jl^bJqjlHJG5
zxKeu$t!1yA)g)=X&QIfqp_p_-chIDSFJeD8YPN<1ANsO1Yx&8SGlh0;SX93+W2@`o
zCu)X~w|uXi$X@(Kwa4w`#7Zv%F3uUTS0e(i+Hfrpdvk*+eCMqHp=B?Mc($mxHnJLU
zb$LC^WZEF*c#G@dYRig4VcD-e`bx$7mY!TxHZ|JOY@^Thl3cgB506Z;x*^#7hS&RA
z&b7M&uV-o0KAYdv-P<*rHDqPXcfDk*<^4B{mX%tW6>@4We3IAvq*-I8qR+Zu7mtmN
zeryT;q37nOta~%#lmlDKI>T%G*DRjk!28LTl~cqy*jzzhw%c>X=UDAsW^Rl(7R&DI
z(A0Vxcxm>INAX2@t{&!Hmke(ut89Dmc;40W6IrRMZNggI{<XivKe9dxZL`!$F*rNp
zqTbZ-eKo5X7JXhEz#YKgr~B2(XYKkn2Ja1D6<CE9DFnxFNETU~;3~4^0$=_A8Kr7b
z8%pc*_qI-u`?0pA_x(minTX_xP3s;W+1M1f==!1(f%%IWbJXJ(Ey%V!vR)!Z(LDdn
zjn__Lrmtka@06W=X2+ck6WdF?oEW$5JOAvo!YZ+Zl7beKB~Pm}|DJo&W23z3(}iue
z+k5+~rA~C2Y5Y;xT)V>gP{#Z}Z>}k<eR2P&n%Bn%2Llg&-QlXeX_H`d&7NfY!owY_
z!Z!u&I4GKz(JFmv!W|Y<DMv-AQymq@Sm*O@b$zn-V%7g#&xYzvd*iG1FB%p_G@U7a
zDHCY5a`U&zh7%TvEuL`5=SJcL!wpW{7J&=IOct_ohwF0j$DN2=x`1_ScGJ`~S2pWL
zTDjlZdF220D#vG?>EXKts+Y|?`z5<)*#Z@{MB(OGuaAzl!uLegubIkBnDkinSl-ER
zGh()ThFNL8PN@`iDe5dX+Id}6&~L{i<sa8%Z266D1?L}((9-lcA{Hgx{c+pF#R{x0
z+3Lcx-o&MH@)<kUxC>ak4qIElZ-y>|X^`o)L*g~tet!wz5)8LURZh)d{$9d2qo+jt
znT>1c_i0UwejHp?_x14opA%1AIr_irN;TKAklLqp0h|B-4Q{PxSy=b~ht}RDvoA2M
z*_yy}Xx-X`x9(g4TwEL8@HlaQ&}7Ixdrjw9lG`e|KaEV97v!}BnEyYV!m#wFhW3gY
zomqS;dotE2xE;Q=#DVj_jnhX7rG~a6w~w7Qlk(QNq}%M(@$63K<{ahNM^5Gkmi!WN
z(U@?Ar=fXM8&f-vl66Jay#Rm3H*YWR&`r_EvUNK2P|qpo%rdU69zkmal$F*!YIr@x
z$C>NRid{JmdA!{;qSih2a?{LVOZQ@UK4FpOCZ%V)l3Y$5u=Y4o8<WbsEn({^?yWkU
zGp4vPEqaiB#4l4rIqQT%VB68X6L|OWJ$l))^u`VT9KA(rO(q^vT9dWv*uf<ZLMPS<
z>*^djr6+WN@69bfrp!jUYb(TGF^c_NC||Qs-e7}R{zb9B3&l7WiL*Qqdz|<E2xn0b
zi}9Hh&p*p@%^sRAdNT9cv%?%pv&0r{{NL7c^u;ryJ-N;6+_g%Y{obq+?r0W%rc`(;
z^Q2n4!jv<g&rK|v@IvXsS+j3T8su6Rx+#4Vd#V<Z9-rj?V++T(jw6a5im4e%DlLob
z-Z5|gvAX2MV$oNsohFYI85jGj^ak;{Yj`|J=UVKVmTLGx{qlyK(;cq=p6GMkXn4)=
z!kurE!jtm~K5JDAo3Cnp6k7UfL&hbR+)b>L9U3{D-m+YHtFU4Jt3)Q_2~4>gHH<fC
z@Gp{QVc0QalR`tcg2n~8KaI?i%VIm6=2|SY`lEd7OXjS&GsbIjxmdW>uAIHG=hKZg
zl@%h~W(R!loLDQfeWG;l1aqBk_T-E^zntQ@UUnR5{`bF%)qKl=)GZy|B9FQ`iVaL3
z%f~L~&SttNqT1Y|&vR2vBd^cpk%6<GibL6|gd_Q`N3thraavt^ecz&Ua?R5E700r+
zDD<s+CVaN<ZBFNPudJ+3-rQQsLit5PoUdO>Bnj7EKG+Z}94(yPR%{`=Xl>BMw{DKy
zN{O!{7~jPeF6j|sURf-*_Oe)gpyqE!O-`q^yEZHA`7F-uWX<ZJS(_*xxymr~m4aQ`
z;l37bzcUImQn}R_qvkz%&y<n7$Z(@w^U*y<YHl4z^_uf`xE|dSsHo8Iw`aM6)N$V{
zDt?dFewHb55?$oGL`l|>-|@-OdvD%;2??_LqPp(Mip03}F3($L{FiAG<vJ=T$y9#A
z`RtiD>J8`B-yPEsVV%>G^e5@5f!O0q7P@~g=ueS&RIjmEl;iB7Pp8E$=`Hcf^32cd
zjd<z#aD|%6#eXyW)Sh?W7F(qD_tly^&(<6Wc~cQ&8M<Z3uk7A8I?Ox!<oZ5Y9p0=s
z?^BCjfmZY@MTO&Tn;)xwxuP0TFzeB=^~)ThmK{)aNsl&2y8Xu_&4_u$zNJxG+RlZ}
zt;ddTI^${HaA9`anK^Aves>ZqUMVZ_J@R_%Z`!s}^a=A>H8-t@5PJ!ypnyzG`Bt6}
zo%17FT-(^(cIYscxZeMDHgk#^W0R9`Td{wSo|>bs_u4j5tqTXYDD=NNczso#$bXA>
z(>5iDR2{rFZ<CnCGPys;b>8MTELn9ZrZ|T8(3>-*hc=|CAI^Jw$M1aC>B!VV@lrwY
zr=rZP4#F7+<4zab)NT}4eC^A+K>S(gguFvOR%;xt6if8G_++HN-{v4>#T=ESvSCtZ
zncW*x7VaZ{%18N>^1}`*D(JJxzEwE#QPGS^rm#bz_RPtBnNAm5J|A#DVc>MqOy4=Q
zUwNO~Ri2BBLfTz8mE@1VF?{~<d!CX?UXr%sn}8o1)ij>jY2>+0XAPgib=vOJm7b|a
z5<b&C9=b1CmR;s*I*DP|uWTL{#=a`WX*VAJ_h{6NWaOF_<ob8vwW3VnH;LRfyyhRT
z=}bLm{(nE;b*Ta#vxnavtq|2=dw1yy)5^;loke2brz-3y*8IM8Rhp8Y{B;Gb?+TAX
z6!pFr$u`+MpTca&6Y0h1vrR!i+sR_aQZ=z@^Cgtj{3d+))mg?PdQ;{q%idnUAE#on
z#DglQ>v6hUvN1V5P}wY@<mfa-^`(nka`(0b5yPDwT?)b73$}BM9ukx-PT^5qa^pqj
z&E~5ans3YwuHrZ(%zemIQRj`224~G%%i2UsUAJDh#6ulZJZ7I$a62fL<J-&TQzX1*
zT9%`5-L+}cCM)d7vW(6Y?)-eP%Q%*w$?{^k=={%?_d^q2Zne_gVO4t3+VXng@y%7o
z8?4Q5C|u!6U<_0I?;&Axw74aBheC~@M6`bLjbew4<%aw&?~|B5`Uq^jl9QXZJeTEw
ztIuNZ7?r$vD?SLV&SyLBd>~VK#Zj{ZA2;eu@l!E5S#_QB>l4}UOOyiM3hrzx2)^P{
zT_*o_@xp>bhO)&9%THVjIVRS0Ug3jylzW$Kk7}>PhS!(Qsb10ROW};tZ4=p&(6eEx
z=n{wiM;$yWC$20A$Tn5fVEs^En_K#C`x^n(H&?>np5nVcN#RhQ(Dgsprv5mtel~=u
zeUjDll0xz6&4H0=4?KjuX0#fGo9ca#zhqqUTtnt#mhI-;$Vo~`H`TJYIRtQi?L72a
zZx!z`pBH7$s^xp1>Pv+lpS8W~fB6KxtEnmF=86YWK2IpHP-wSM;BfbQs~?^#bd)Rf
zu!`YM4f)5+X`5acq;`8Wx$_)mx$bdDZq2G`7F=GYiM>*Vy`Ns!Z_m^GzkR>mMA>zD
zt6Pq{my~6HSbt?ngQ%Gz(;;y;HiN7!4kE^tZ{*gzS*5mNm!75KjcG9#Bhx3wX<IaR
zPZIyVFh2NJ;qR{sQBg^o-c1!(YT<5Lyy>&`>qeWUkCsOM5Wo0&S{Z-&wP1^uSu3(5
z?54X)x@RPP^mJf3Z&&|g!-BwL-;DLPwr=E5a@=~$Ol9kaLo2yANrs3%Hy2}1yI81r
z;pG~3r!{Mp+U)x8%z2fzb_eHke?_h+)38T@|NjV{X`42O+g0lMVFfAUlEWva@Ej`t
zV9q3Ppy$uSg_9NbOG#=)GIExjEU$Tf-DcIQJSB#5Z!V7QjeB0@9jx>|ucdxkPhCH%
z@z*rQ?G0~D7G67L!t^m?iuiSn+>cYlA5Be_D%MZhzOX1hQ)+6qqHxB;?WWtC>t}2J
zeyI7oQFh1V#}?(=<<0#Xy)|cEl3y}OoheA2MS7()Q)a%nboWo?A1k%HBWJASoq0HQ
zHivp!i9yif$jBv+{5GG~U-fvFq{)86L$i;%a@}xvAk$REZCrKwrUFmjfv4xXzAf#)
zvf|-1H-=*e<alPP<v2zh=lZGSAk5P{^@PIw*NwctPW*RkV*bDXn7xLQ`vkoZ)(jEr
zUya3_OsW!b&rO>3C764StR=rIL|NLD%e3yFtQhj+cxlZt(MOTt2Ne@GiaE_v$Ve28
zNE8iYjAm_+J)-cMsZ8`pgIv(fWk-ci?4Hw{Dad^J@}es@Z+6_`*r@uRBT0zGL5Sgs
zGLyDjPvrEOtQ|9#c;8E#e<i4V<F2xa7SS%(Ztkk~{Zg%G7$PhCXzlVFYpTC<-rsoT
z1MkGIu3JKaTu$y(ybu|avwjtSQt0zkl^Y3{z1Jz+lK82TuyOl>+K=7QlNqKSYbcob
z;8k9x%BLLx8aw;=4Ko`$4G&d@Y=7s|DpZ(1d&=6$8WU$LH%ZURch)q0c;dh0BKZY>
z7$Y4|hbQucIi7r9D%!x^u=BH8mg3Fnvb#6_UNY@0bMem^=W4z))hhm*axmjbUV_tn
zo5<W1E+)_B6d66%b}74S9O@{W5+tkCCU+#z{QW*tZ>J}&;i8AG=xkItlkxD|N7v-%
zA(01d4xDk`_oCq352b(7{m(Y;vso<5bJOSsgPYQ<Y3rVa^|Q^G&=%34=~CF@782ue
z_LKXRtH+8B3%_nHtgKsldh+R`t;-Lxz9?*79Ofv}GEZTZtkuS?J0cfqxI2n)IPthS
zim#oc*zs?Nbd%U~)BYQci)!-wCOq|*$d}@D65?_a;>*t8Gwps%w)ZraRd4^?vs6w|
zd~f>S%xtyt`M`&t=kT!lt(iGXQdW7>%(U1t#e2V6H2!inK3N&k+o-qKEJ1z!**Ati
zC(1kCy_~E5y&_-rlJhwqk)KNvHY#p(FWYN0$Ln;3>{ok{m`$s8&HZ#^^6u^Wj|$Bt
z?o5;|>YlT`|MqtNM<=$<*){#siS3)2rr(xdvuA&XqsW$61{cSuzoFkc_n5WGGB2If
zEA=A6L-D=K1hplCZ`Vgn2{BTsb<ue!d;V&*--mlGkC>cRJ#u2XzuNW*`)}LoSMhrP
zVq@>5nt#|`!RW-G&dA~|C^CWJ;lWlOX{#v?tEMuTu|#ey^q6qQv5VW1>w?DxWw(BX
z$kZH<R|#G{!s2~G|1X@I))2)Wy)|1^bzy^w&=eUTr31_T7chwkgeX1CIKRNH*-E$T
zVtU|8_N8w`98OJdsEgTNm8y9_-g#rv!D+nQ)1Kbk%6?wY=61K-+A4|POW$6dw!l8h
z_@dq$(=49YroTt738|)uYL^xNy&3D_e@0_ZzU(gF-!ndh)z7iKzB+V$=&9*TPEKIz
z44lN^8_kh*N$J3=t17<Ye4!4rmy2))v*bM5lzM0iFLyL|#@VL|)9>+2)_-zfj`OXT
z?Ce5Y{0u6^8Bd<%`J*SWy1|c$M^0yRfkI`|>($md{MY_ew0&54d%wTI0mr#aU*vO6
z-FMp6*>=A~O;9D!nUib711DagE&n;4mkXLmIp(^kI7rW4k}}JO<)VwQV-$meqZ6Z6
zic%Ms;F0+5-+Lnx+?tPiB=?$`xt!KmopES-?@U3Fn_3bxMIME%z7pI&;j2hke=^er
z<5L=2HXQEKC|+v%*vUTWXK4Djl5djQLK8P7I?tXUnAkE)#=K1}xo6Qt7MH{+B3-KY
zUS1Z9@z}^(Y`*O2@?!R%l^4pm7k*1$^o?`ZgoS*(EpdyqH>zYXsW-XauAe2e#Z_CJ
z>j0OloNk3e>C&W<Ko7o77v)A>!;I7EJlCcOyKU7idD+DiyNY4G#MzylVfC^uGZ~I@
zoxU`!OR8F66H`E>|HgkhnhpB%UA_Hvzi@2$&zGM0vpHwuajDfOrTSBsU7cNgb<;VC
zL{88Ch(lFosuNpHyCwIrbXHH;^j+wQcj8>%Xzr<My-r+n8d=p2%+A-l<~8}-mSqf~
zL3gVr%-NB<CORiNXH#q8v5hKo8R~?xHWjh=#w=LSW41h|h<UHiywW}^uFVV<`d8%2
z-kc5p?D+QT_BSOG&7ta!#asc;)z(R!2wKHs%W<t)Z6l+4ONZVTAx()*MqI9%H!t})
zo8(S(xU}n0Rs6ckJAEf4T*;lO>Kr|9cdqT0HJ8`lt^I$4-*)c>*YkaAa<Zr2TXdl`
zLbtW_=&7Db#V;>tEQ(nsKGmx45SMnJ>dmVW{nl+er`2Xp`@bS~mH+FfTefWo=GNZo
zqN4WZYsBk*+y5LN{Jp%FmnPjh^j)Or;*UT@hEE3j*Q_{ivTM$?&k}cS?w`A9$dXyX
z%Gofl=1H!*+#M4y$F}85I!;}>#=CIaR<~FCe+B+gH~bRhHTjl!a6Ip#+}pw2vbLg4
zA!igrZi<Ak+z`0w;vl@Wp!?Ok!_(pnp5?!>2(p)0CZ6`NHK8i`OhUv)z74MH#8wM9
zJHIe&3-t&vNL_h+_L8Og`!^hkTrA*x_tqu;3k}B%V+tpwzMDGpG{;4w<*UU6rGutj
z<mHK(`_WZ+<_nj^o62VFf~Wk~HLdK`bYu+j@H^3UTz0DAdj%PXxcp@dvK#(S$+1dK
z@mR8BxlyBpSJh|DkXkE2%{N!t`(7H(lNaQ0VBy-NB{Az<lSk;}Edg)c^c^{yjE?N&
z`TEqEN0GVnUxf1Z+=ZUUnlEYS1@o76994<ED5D*Av0o)JQ*`x)Nemf^UFx!J9yX7b
zg!>#i<)5bJRkAa&H)CR}l5C3G;?)y3ah04o^JA0Z%*uqw6PMPMU0Ec$F}rofyL6Kc
zOMDrHABd=l7a6q~Jn!Y*c+dR&$q8?HTB>rEOgvwAe9_0P3q{vBsjPNu@|$kKGF#g-
zR6EJk_t(pm?wsz6?;muURJb{C@s?Z@+&lGPNK;yTug%k`T02Dd<#x+1Zjg1qsn`(O
z<M_Yec9grm*9r5mSt6}!sV6*-CM-=8>Wt)_dEIG3YjD#A)+IWc*L>d}Uf=e^r1=+9
zINRw*E?YLVDtx=prTgY%Nc5FMV)ItLvRR*XHSESg(YzT>3X7Xu+g_wGRW0%}=X9Dg
zMUW-+flP?A*R^A>6I-~?X*O4h9nx@|xLoAPp1uo*ei-jJJ5uTLTRURI%=1jD3Y^gz
zDXZTV3wC;(b?mg^IHaJ`;1*T3c{5YAYnqektO<;aS>9`Y7%XBvZy3zb%qf$dY<_WO
zL0|ETHOCtzl({BfXmbC1LpMHRVdptch3!?zy5TN`OYX|)6?ng5FY>&({@{^r#f^Wx
z8aG|(*V)jNckTaLZ^6B*FaI{#+_5)BH+!vX=#8$-y^A(3Gv#!;`s3ea(QOOX*S`L3
zcl4TO?k3MQ&Q+JCo^hO-^KWi;<JRSyW-3P5aVLIol$oQ!Y2iC*)hmuaS}`X>Pv#%K
zd{Vzut$V>MwTv>(m6JuMwjH<>_E9tbXzyBsBvH*3-(nIs=5n~6+F80~l84nV9_wQV
z`s&s=^L*UYt+7S5t(3)&$5HN+Rf$J`-G=59Ns&iWoPG((vd>g-N_zI7<;VJs4h4#O
zS9GG}c0{Q>JF0sj#ye;sNBUY1!&Sj&wZh)t;WFQmw7P5q>-@hwFYM)yh%zZ`S@>`T
zgXrQPcB0-~1$~z%ToYigVwuSIzwp|OO)rjUtgD#1`rIRn8y{A%+vckjtl*Zu!F*YR
zAwIXmZ<0e>aHH$BZ;?yn&-w_SbQ5w^PUDciv@GkljxX1)lw}%jf=|y><wZw7e4(}M
z>75gA%NB(#bGPlNetY7ov%6;PTc5J3rK$(@;z|-H@wZpLGtusv-X*2~YM1qy&vD*+
zukKk^eN=_v=dthmw(V%GWq+IBbjOjQOyIx)Zo3aH!haO<76j(zO*K0v)qeT(r^%nB
zgu?mRtpm>`tIMo6cRMnT{mQZCjtmDQ#jcDN-tV`gR^GcU)VuuYytFV*w|Qbok-9BM
ze6$}ht}MQ_WlI!;V8<Z|ukdHpPp|1I{pBfq{GTa>Q|+hcnj@z*d}gVXeOV-57|D|n
za{TLtlB0%<%CZ~fgs;pyCZKF7%Uxi;V)`~?L!JC@-a(J&<*#s>I;-s6Z=0`sLZg3F
zo^O<^na{pgz*hX1U5lE>WAXQ0TmJQ{+VKY*-Ju~c@!|6|EsGf378MuvOulpZj5LSl
zy%%#9d<?BBTdVBIbCvV;#OHliKHh5%o!hW5-Y_}nz)z=)zv|KQE?MF`Z>3sRPvDrl
zeiOUmVU4K7>@XFUg{F(_dbpN!s(gFhwEBkFlzF+)zC3E%yCjcJ$|yP&{LSE0XYr(+
zv#<O+z9nGK7yl`q6<ql}9(;SZ2IN;OGc~-Mn*ZOW;cIHc_tO6v)oB^5>ILkE1=}Aq
zuroIDdNk@RED)$*S5B_|UZ^gA&`5f^jb?C|!u6U2$3g`U`T1eWH;pv~%!*cS)0x{;
zeCWHKT$`O?QL{;rV26`$^F)PIXD+wIq{fzjVkc=2Mp2o|A~T%?y4nP1PpMz~#A53K
z%d$X=58(##Mf%alMWfxtz9*Hre~qpDm?sfv^&nNbO)#eLSV-Vk$qVVBOB&KYA4%uB
zkzO8A(f(X<!twUL_9kveKM#p8_laT-g)PDdIGdCz=c>t`d{~*m8o}$R5hGAl?-V|L
zg)Hwy<NhGoSOb$U4?EvaY}$NWNN+_}mM{ZnqR7^R^1g<Np`RnRKKt)}Zo0y(rzU&D
zRMt*lah$FZyn(f5k#AC|K-7Z(_J@v16TNhw)UEk!F<Y_h=lAwyi}Nl|RGk!<|NUCr
zx3C6QkNE#r^1q1|a8|JQmKU&XFL)Z!$7@(%zNydf5?A*LA^zpQ-iA%m$HQ)ZY*dx#
z)k*YA-eUYZM1wU+fR9an_YO^I34dP?yJw$0q&{k9E);A}QC{OD)LvSw87{arfuFHd
z;PMm^@4!Oo3sMe@ZU-AR`6jYNG1UKFq<UJTzC797IW)MoxXsu_C%b*}x4<5^Ba<==
zxuO#UqmG8Qd{3{4n8N+BqMc2wf4WBh^Yrf*3wRDr;dz)YuhGG}RhxIC;l3IF`%R2k
z`z!jhO9iHCFcfUnkZ6=$-JWo#IDGE3tSQsP94`xO+upykUDNY&#G&ui&smZtZ53p3
zluBt5dTy*3#4L1PJc?(MwCiFPVFx?Ag$#w07*21jZhj#Ul)!DkDK?o|jDL!Sos;tE
zC+^#h25>~gw{8n%IoQA*-pDV}xb~&M>YEL3R}`$i*~e|z$HhKdv}1PXcDp(i^}nS}
z_eJ~?Z-%em-an^9{#i>E-$mO+Kiq{bXf9-`kw3<@;-TPo7W?SMMKWm<1C15g6|L<!
zY9$W~x4bCkZJc0pkh4gMan1%t8$lz7K!>$rtWg*EPP`DvP;<_8lfAIT+#tltTh;kw
z<bSa@8yKUWw)ak-#cMdV?uJ11j5(c3!pDEAx4-aBI3jfSi_zq;X)4asu1u8CXP%z8
zMYd$C><0D@dGm<ag7DcI5pAp{!l%rZmMVFE%<4{++Q+Wd^+ce1$@I;O6drX(e*0MC
zvP$u(WYpG2?p_XTy9M2fgw!uP&Sw{7z3STgQ7vw*sD%3_okc(0_dVg?s=zAV*tq7T
z!1hR!O__arJN>e}WWyF#M|BEjPFDYRacY-Pm8w)Ho7>_)9SL051ZrDluO+hC7#F=R
zT#;QoiT`1QxT5I`Wz+5wQ=ULA1-FTZkJq&C^cU~cSh8T^Wfh;b3m6X^6Ixzs@_r|y
zoul3HjjTnR{tNi+5C~LYeWReWP&tjkx$VkS-K$B_A&Y`;UGlmcKKY@Lvk_;n_02My
z%*nf~e4-abR%@)~`?x?yaUoyf+HVurUfH--V&arlncPPL)t$}NIRh)DFUNN^Wiox}
z;9Y2wy=9$cvu!h1R?LJBqsi_M10za~aK;9NU;fxR>xICw85)m-J@$srtzYFI@li-H
zXvrhT#Nf~VvO5JV6I%>9mO5UPl8sOZ6tKJy5qCu?wVzS<{2{MUA<J`Hvv-RHy_}|>
z7-bS|z_!kykuPww>23jw?9DuZ1=}+FOmFvD?Vg?MwPnjpF}KyT=Y^S+Uvkxu(8yA)
zQkLpXKC=GZMU8AvO&!Mn6S}-Mm@73e{1oZ*OZ`@6BCq2%tBOP(Cvo$S9>)vk{ZlTv
zywUafO4nX72Onm!zzLCjSDjv_r%JJ=EG|h|pJF}DSvc^L!{#Z@p&knyP13T`<XN<O
z&s2$}Fi2lET>C+_;BwX4yQ?;2K2*$Akj`qJ5P4+=-$g0q$;<f!*C+j4xHen4M0CO;
zx2E;aW%``5^1c|=s7&}*s#!MEWG|D$rIU+U#g<Gt$|e|8EHWvwx>9Iu!;HJ9mh9<}
zpRTkzQ;;=fgHNav_iZ6>zwa9pl)Y|m;<CIH_-}#oleE;!i&RRBeJ0QL`Odh^<PcYM
zLYEO^qq#PNz>UpZT#dN_Y^>$nxeouCVh${O^+KS)YfIj&WhO~mcKqzS@;hm_k;5me
zIdLxqa-SsDcnOHTinpHJc|x<Q#AU1Ub+_eD&3q^BeYV*neRg(2wcwf}MZury8twAe
zj@TJ*ET5RR+$_m(<FDe$?It0NBC{PN9a$uv9pCrZz#_a|d_#-(Vg@OV+x(%AEX234
zSwGb?*|^3&w2iNEO-{JWi6%Ysmup0-1=(}@@&dT>CZw}H=VCp>7-Mie^+0I!gas@Q
z*(&}h%-gMV_0`%NU4mr`8mCqU>`Gc^-zY6~KwaLVaD}%U4`YA8>s=qc3isSzzHpkD
zg3RKMTkb2vB0gQ*`g?{*(1Qq8wxqU{;&bh4`x^gC#W8Obov9$~?fIeAY^k?0hvq@v
z$NSc7Rld>`<1OUfl$?4#?2ILo);|Z;C*M?-2<vV$*?+7gp~R6b-O4m|1FOu0v(Xm{
z@(dVd9N6*#*i;M{UpTS}F(d{4oaMUv+~tJ_-rqU5Z{=}`hX-d~Y5B9Yzi!g5{g(DR
ztF}m&ADkVdaKG7;`zYHp<-<~KE3ZsEoDewC{1RKj?>%1~%)Dj^`rHt(_vw(dZQ<M4
zCGH$>Wg#o?69F4j)#yK7k!kvchT_eKMQyiOB}k;D?O1JM+ndbh>+&=$HIX~of$6r?
z@!*8xIe!Fs)hFd8aF+*g-BmgF>2Sl>J%tT5^4%?`oyzA)&H68Pgt7mwO63f;s>~L@
zn1n2*MmdYayE7}R>sn6=SM6O<ed>zf63%RSefK5l)k2evG!+Wfw!}ov-f+5p2kQ^_
zwDXIi1C_j<rR62LTN%qLfBc=duf*r#8-dT>;@ce9f;Y6U-g*4#!~E?N(p#=cJ@jEq
z2{^ZIf|g14EwLE~R`(uQd-mMAzX!N$1j;6?YgRep7_)rCtp3%neeavNMJsEZ{ke0l
z*|mZvHu)@9wJKI!QB1txF1V)Qy5hzfp@o8*!tUxCMzR~N%oMz6a_77m`wa8zT+zaY
zFGD>;yQY>(Z7=DP584=@(!09qVCi1b;OK{~Hk^THy4nsiiJ5&3$o=S(ANIeTB`bvK
zLP&t?#f6FKYZJ3CU1Zz#fGs89IFrLUX@}!63CGJeC>U+L!qzjT$w&6F$J#xTCj?Hd
zZ9T^NGV<W{_9S(oOVKK#v!1WwHaxkwJdCLzY~gmdGtPx`!m7U3G&LCsES{6qnzeQQ
z?}<;8i|RE5ea{K3yyU^zEPCTek;JsS)e$P&9Jq38{q}f>tbFQF_L#rOFg9wIPO``1
zy(zluS*<rE=Wlr&ciSWM#l?bOryESpu!^mdi8k#Me&L#{@KBQJ#KyUszZbH<+H=nF
z^ex%2nXEz{f4*jX+mV^5qG6QYDLm_rL58y0b|DeL!!uHvKN$OO%uyDM^*0PO;|*eS
zxA=eX>&^_{9T$BjUi6zdufW^Y=vZ3m!%e1tcbNE!onI0XD3b2>l$+fz{c=fY(T(yd
zwu(xLjFxTf`6>(lZ3%r5Xq54UD@*HK(HYj<1r7Btj;q_%NJ<~sdM?9;F~R?Zz@+lW
zyc;_hr}=M^zVpxL{S`-dwu^Elnu~LPx|dlVOt@HmyxXs;(q5?1^z&q)^hE{&)f!p{
zuL~V|D(sbGTIu)xv}*doXCiN(ZDzjv|9YJ7y8I^d_<6DM!X39*1pD4*unRpnxUAPL
z#(^m#giRoVO>W-Jf-}!=)@|uNcc5k73;XxCc+M!8?$g=(D8uB??F2!;y>G)NM%+0Z
zt8j&Rf>O+eos<4E)G*Z^*e+z?es^=xUEajI{(2W@b<Nd$sCjwglMC+m3>|&__jTKa
zw;b4<bkIUJ?BJ?o16Nil=c5r1tk;IPn>+{#d9#rzpr9iC)W7oiwe2n^f0Tci(sBIr
zy{w1tc^;-F{Ji(>$BLV4YvkVDSGj7cRPpzpQt|B!&+|gDH>KuG59`}wG__5}FzIDz
z_q4~3en!GME2e22HshN6e{q1m+_&Uyap$=r?ukd8+j!EmR``8hx}5UlV(Yyj+?Sp_
zd&l}biRn{O-mL%M-d1nhZ-4sSukKIx>}St8#Q0=JzQ@hyR{8}EI$Lxu{(C=P>}|i`
zui5*b^T)HSc)0F?=BL1`9?SwvEdTj@lpGJXHFApTrQEpi@USM+HjkJSt_`k6Os8y~
zytt6$K7q@!Rpv(P!jpbXK59OmK?|80gHCJ_a#$s*ZN^l@m7}~M{rG%6_B4?l#R;p!
zay>RGx`t#(hdCB+%SvB8mthI>Z4b{20iI4Xm?kxd{wPhjF(LkJSE_Wd!xY1Be|W#B
z7QBm%FK0a?8c?1eyZ>9w&CRYm=IJi+6x|+q=ajFu^*W{78(nV}rJkHByi4lD&Ej;%
zIpSSnUR4j=%ojV_o1ERa^}+?7T+SJ>)eF|%+L3vC)zwwy?`N+m;8H)=HJ71m>#MKV
z6dVp6-ne1ETHu0|Lq{5KuqyaiG%hLUyUps>WAQ)G`c1$1)#-aqoLuBO`K9za8Ou#m
z8m93DKNH^}|Np@6U(LP?84_Q6EqJkQL+2DmX_*euzKkdir^P33xF{^|iRy5ibs`~k
zTT8~KRuN7<kpnL5!8M+Wx_@4}FBJcID|<mz#+Ovf`w=@c-DOL^_&OcQDO5e7*xhq>
zYP8Gl!<$v#3OwLroYQj2vvGk)$0>EmO+gEFcgtj)zL|G&MNhPhmhlo@;X^@Z^>42!
zT40>n9Nb`(zj5iATTz^wM8hXU1+1K%mAF*QGI!;3xB6F}SLEG9zaAIY4SA!%$ayX4
z#hy1Rz8Xr62C^O8emf7W>;C*C`9z3sun&Le)GN`Ri~oBp>QnyDY1Zh)z`>d%u)3uz
zRZBUnY^|o^a+XCG)aO*3^*p?+?bg;CZEl$wt6SU_p3-s>zNN*eIZxPwQAD!Kk73K9
z+z+eQO?si#sIp~6pu$0p$=|mhlh}OSuttBgj&ZFe^F*(ccHX546Vz^WEWDni9O2jK
zaD{if4e!$F?q(gUZ&*BLiC8x;&f;BrV7y&j$dd`Fb1v!`Ez0m_C}H=SzGUl^fCUPg
zd$K14x2kQDeEw04%U>}pQkC;+Kc8G@x89xXj3pCVo~=00dR#X{LsmRbBfPE2NK7R|
z+~`$ydiF`PXv^&jLKKSE#&M`^V@|Mc+_0-l>4IZ*&e3I#nU{+5m0G{-KJj1GncL>H
zIk);9-_}ztM(6q*SUL-j2sat|>OJx)PB<ZaXtLj}A7wu7K@4vv1~^qZe+h8454-Dn
zExu&#5~+=D0YO_>v;vtlr9OQL-pcymX8Cl!YhOb9W?cJX+Zt3=`ZGe(t6GsWXVQmv
zjV-bPhg*tLx!$yFSie&Jy<?KH=4;8rVNQ(e9%((x7fy=TZ9Uc${coo5hW~Yc-r8^c
z>-9T)QGJk2n$e$CK8#<t&Qw+nP*~y_;Vg6AV$uyYMgBd#IXV(cbz9nv*6mElQTXO~
z??YSMftP`W9mR8cHJ2wVYTtY%v31wp6@Co*LL9t_ho^h!ro<>#&WlKEvu;#4lE!>c
z@_)IA(Gf+diHac-9Kxzkx3qEOs>W3GulzH4K9gvN{YmK)V$xR*3*F39IDc?%P}@~&
z_J#blJ11RpTC!E7m809yOOaV~%_3{Bw;L2QIJ<A3c<Ro|vOv3W&D>(1O6%uye0%gY
zne*CAW}mN6^9*$gnzlq_>EYN;#cZhyPvx8(8*gwH#Tm(HiaND%7F?P<XSMM2jLkc3
zZ!U6Qv^#nMi^4YPr<Ve|s}x@+SqMmTJ&84Lk#%lcIl*wj$LJ}GLf-kYPP%K!k?zAI
zy8M)Y??Y3K*{5naK8rcoY%%q4OuKTjMQqWr#T*P#ZVOLtx^#1E(X%D$=A8E><qt``
z*}70`drXJej{mB9zpv)kzrGazJK~%3OiTAP)`>U&i&+VH&uvY8bwo&A`GjDAmP#UT
zh+xN^0~h`BSUh-7`zBTtdq3FnD9J*xwLoi`;nO2u0+g90PkL+<kp8w<+*Ibo;ul5}
z=C-M<&{}owpUTz?&lOxcr=JR0K4X{hy_e_Gvx;YZIPg>Ov98K$rVEP#_T1RRR=GO(
z%44;hYcB*A8N6`1njGL=Yb0P6;T^vBAlJ)N%T9=@w)C{8Jd6ALYl58K6NeNDp+`o?
zm->cnNaX3foM?J{;^C+~g))u9rmgBXTh|mR<`q1f_hy2GLn~i?lh)nDfam->-vp(s
zHI9s2`>a38;&Rs3(~GyKaBunl)x~r>w{kzj3Qd70sVsBWy`8(V?Y5TaNfphA&G(#^
zTvD+3=910RsBYuvwC~-^m(#8Yolp}J2xR%WK1fIUeZ-kZYa=JJR>cWspHn()(3)bh
zc*Dd6&1sKka2_gee5JxDz9;!&>zVC}=M5FDdz3H7eOa8cH^tz!M5yyVmd%Q5_bxQ@
zop$)kD??p@rv^J$Om0y$Q#rz1B*k9$d{QW5uFr;BC(KwcJzBfU_}Dekj*q*v7W}C(
z4Q=h|^u5qwC;GEOF(o5erudLT*&{CPB?$ttE2SeIS*=%EH$%qV<a9*%In60ic6SnG
zkMA&>kifmtvqSongQR)R!-OACr%qebW1&9rzhmIHAIJ5|B@c552Or|Sx!Jjh%V(>s
z@CE*vr=zS7B{SE4SG?F$B;x+mg~xPP`MsQuw!<q0b{tjFK07yV{XX{a(ruL|Z5kK9
zaeBM8;+0CIL!C$QL$Aqcyi2s+D%^J1lYJs}63fMaeUdY$axOBHS+hUyhxo&sjb0k8
zvkg~j9u`qMz-koU#x1RJST-Os_e?>;nHg6^(zi{R&VJosg;nZ_fa+7}ZC`etuM6zx
zx$!rzZNu$5Wu6`JHMfE}6szhVlydAppS1tBS;|Vq#4S#KX9V^ByqOu&qP}Xzv!tb*
zDwzk?esnL=ne^<X#2TK9Kb+?o8?#Qlx^t4<a%TmX$BA1@!vC9OHpj3!h2NiZ$z@tz
z$t}Juj0*P;UYB7s@ZWY}vhK}B?v#vXStE@-HG3^?q)Plcc*>yhUlGHt)CUJxZUuNY
zE@@*~>+sZly~)JpnL$M?_nQ_-=ax<4Um+T`VCV7DUIkW$18I!uS+~ru3QoDbS2>`i
z&pT_XC`YSbc={T3&xpGdxMp5f-)OelmUWT34C73ZSjqK~UwU7EnY3D`>h2#2$xnqH
z0=E_Wgg5Me^5u)hPwktj3TB)&L6Sxuv9H4Q4~kf95WLN;a8^dcReG81Qp3oXH>7Sl
zaT<A~p5L)=;``FhhNTV~lCJUZxhLA~U0ZLoFhJtB@RdWymm3GxCr2s%k^Ap0#I&G=
z_d>0r$CDEW%DXq;TP9m6GJn5Q%$!9!Wr6=K*j9H2q$q8f!Fj`AtH?nqCK1#1lC80t
zn+{&?N?XXu@v>9Rde2RT=IJx{ZH=@yI=SS;!%dor#`7C?EqvT$w4k$vc^$6;v*FE#
zmBxnV3d~|YLRU1H-$_`R7Pp!iNE}gY<ead}^d`qof!%x}7U7qt*uCyhmNAjgTqaT}
zSbbu*YmvZ=2@DJY3|~)6PB<{LNqNWK!+aT;7T*pW=(AY&n5*-JrA<2rmno0w3XZOx
zg5~oRxWY~dv}ULW{GM7A#KGob^Qn-_Xu;N}HzsH<naekOMQcR&7DoLU87jW3gpL=o
z8vW016-wB9*<;7z!(7TrjhX@bG&7c@X)}gjaN$#6Hkz@`D5Kf;NspnYan^5*V>_CI
zZZt^SOkEu;e0)NWAD02oL+-NG+vf{$R41G7<JjJ!aBQKJ*3%!|@q4!K`*AcahNJSZ
zW?O<=^=v_=FVo~6ZU6APYdr_UH{pg^k{yD<f^Sdk-BrD8LdV9R7j(NO?~t0kql`s>
z@y4nt5}i{vaQqh#s6AuR$icwKz`*#yoxy{_k;!ARl(OanJ;NS>)&#xGK!>>*tEVJ$
zO$!#wzBoJTGiO}z#NIo~(*u_u7TWdCNp+>RtN0a;=FH|RJ8Z2t>^0lL`lZp)Xa}>@
zlP2pK+vX?lHU7W&pl-yZc^b?zJsi?+IIR_$|DDn~v0&4YKrc_r$zcMbsjt`_li8yp
zSi>Y(9~Q_4S{!KAaVyLea5;QTIB-kbiMiTSw0587Se{|{<khM6fa5JcI1~gL6bd?Z
z9l8$O=48Gyftf|tw1?|s=ZOgoTpD|%4pr}&A+$AWhE0sLvUjF_r_qW%ucpb~(sp*?
zK9M3R)SxndCR0F;8DB$Bwz1Wm-OSM%jIuoZN#2_;I2=7&;(JlT^<UGG$B(BPU*3Bv
z$u?#V&$%-~ms9qgWAI~%(h<4MRDEOLYisibK^-Y9t4bqSt2{bF1zJM|7@{KB4I^6n
zB$)#p&Xvt}+j?=I-^MK;{$E}^?IFiD%N@#60`DajeW*C!dSHrf&FMyidEYe-|6<Vd
z>)=n_wL|+CzrN9ldKnHz(Eyeo*6X%NnUr)MmTp}asa@f;{P^W{f3jNpM7Z)o1fpm7
zPgo(7w|lqI57v1-J<FL-_59F?o?&LBa8~RNlXV7j?(a1T#X<|UkDk(+EWTAp#c`iX
zq|oWgUai+V6`wd9u3R&31M9~t5>hK$AMaQixPW!RqH`?^+Cwj}ypZV#m0&GPm>Ve3
z`a*<3Kh`NzCYb%@=0vGwz7rUJW}3{i^lpD}f$^f#gPk1uB^*X2-kyR1jSftVGrSue
z7z(}!s6OU+W5{*$;=C74%5P3`6cyP7{Xg9?TZ4;z;o^lKdY|s-iVF<hsI}^*;RJ<C
zlj}D)Xex8^F)USGxyx(=tN8|7t!gX11v)Jn%o11j28b}+eB2b`+Zz=Us&%;M^pCS_
zLYG2zZaTAzcdw@(<BQhN9YOU0VTK)kWdh8vYNn?ssm0tp6&TT4-grrB5&Jt85srsp
zyI;8OzF=1r>XslAo?gOf79jbEOWZuvzfEIN--PRp7q)f(n7JTRs&B`B@z?xbi~Iln
z<}cmCvEre{4F@J&0i&abgGDoyrycg1yW_;o4Xclz-qUK?p(`-?jEza>ts5r{k485H
z-o3n6Q$c4|%IW^5N&UapoQRNF`X=!HkzI!)mwI~1O#1I&X0FVsxng<MiXdH4j;ISP
z8f#g71sc|hbm*-W%U#PFxS%!h&@~<3V``;0T31ZmD$KQ!X)n*4s4fq|w++FbhWx**
zc~?&rS(a&S=gVPtm8q3S@L)_=Oy*&^hn$C>biAzUa1Gs^c9Zk|2KEOl6nuH@V}cv{
zN;_0;FjQwWf4*dMGKbscHp8+l_AzUFPC4zH=DU}9QEzRf{)C_z-M92Fx+?b-wdzW+
zCbF!0CDLG+&|cSZ?$sLhcWb;}Op(1Q%jzWAx_s$E!v*Z3za;8{oUU7i_grfE_G{lc
zW0$9IJ7jlV5Rr6e%<zw%8XQ{4vE#}4b&aPNN=m$Sy<;V0$z$R$>Hm$X+?5_G*~&$e
zCYaB<(38cgIYY9&Ag*l!Q?y3wVP4J|9!qD<)ma)NBs=xyg^tNxQbC@POBuJ#V)pB3
zY7UA@XiZfW$z=`Jf7_mO_3jPRD1*ITOQw1y{*u`2?W7jMu_are`o`34(K8A(-OXh=
zG%a>CKeo>3^7d<-W@HmRX~WJohbH3|fne`|J!j`Fir5*#opj{%Ve68`uMBrutho}l
zBP8td6+^4U8#5SWd^DcMcAqw0!s+4FZMDrk=g3(GgCx-|L%n3CYa&J`6xWL|+<41g
zCea+|p;oZfZ|)&KUkTQ^QLWcP`>u;NgjP(vHg{p%msr-tb{|YqUU9pX{`GVDUng)*
z*uH6{cfpyyJWmeIg8h9z+&&6)eAqDaXUL9zkBt>FT?R|^`UGwmylF97!C&FS;T7D!
z_~P2W>W&s38=JK!`W8&EO%Y&Eb?^wimB1Lh#DS-~m-DA6m#RXuF2`b#i3}I5WfDsT
zjz3yv7$?ED@>W}b2*=LHtJa>Yd(nP%>Z#R#(yn>(xq8aJVvdt;>(%M=DPJe6X?D8O
z^|aezz9%amma=v3{1VI<al1~f!*Ax&*ha3C$sLW~IQL9hzMY9{?Tp)NEW2Vf&lD%w
z#AH6oVcf)<q1J!ozSnF?zfk?-wH-kMb{A@oWCZKnl$hh`!<!^J<A>9X3$8CGELd=l
zM^N+siK*9i-Hl5S(W<?*vm!|9)r_cuxh&ept_E&*_M4h0Bd~V^`<o7p&r%aUt8}mX
zz^6DX_<E4iZSJWdH?_2^Oxtd#tqu}=bo2bSEBp&P8gwIk+|Q;rt39>QvJ6U&f3`<;
zokbeYQ;&HDygECZ6K*pZN*~odxJKvnF1bG!f^J2!cRssk8T)a`sd7(|l_|2WjT;tk
z3p?|!Bhe^0d6D3Gv1RAQ9-e!)a;JqK+rph?->e%i1h+>d>g=&zuC<P@^!0>*>3*Wq
z?U&4Zn9=ca((+hNv1KQsceE`w=1M!L^3>yK*UqIGlOmFME>4s>7{WSl)^Sz;@?(#j
zo@_7cTOsQD=J;hZq5m4pay+~`A9txAbX@#|DWHV;f0GrT$<i#l$4PVF|6Tfi<Lm{x
z63j_H9W34ZoW!0Kg`P{`(n?z6ma=r;Qa`^(r5&#h+evncuRhVyzfJ1)I@x)ZeYyo|
z)uLU3&8ylTh_v30yK`&pqsopiFJ>+4Fw{$sswop-nZVTc*6U$iN6f=iTb9d78^t&Z
zFLcVL-87i(eL8(g@1(h>9jC4FoVL|v<>jlq9PeM~Hfr2#xV`)4q`Nw8H!S-Od|IQ-
zsk@(5cmF06zo1)X0()n4ORW^#xN`Ptl~!K~X5AA7+k{VVKYifJG?AN&tbH0E^5?Ho
z%Ubp221n<+x$jClce0$QmE)-E<2?3%#fw(9q|nE28`JcHqNg>t@uqc}+)(gQ-j?z5
zpGeyyW$m*08#X+e@ld%{LVs=U{P4T$Zza6vym3U}3B#==+sqmc+P~*)$=Ivu5qfq}
z=3lKTd{H{_*Ifk^U;fJKxTe5bzH-AG?^j0l+tckov@!4;-q3X<gNyxc3(xt2J1@)H
zFJ1Z4G`}Sx*uS)c_ra76uF5Z-a%@!HxRCkv#&!PXJe>8R9b){P4!%vg46?h0@3--C
zE|ci64Y_svv#8?l3A?6sSRFsGVM6JlzI(TuCrLc@w9fRLFR&}{jAQQ(R?)Sa4Ci0b
zka%@5Ea<|sm8)fcwlF*_o@`+J{f+a&t&W+R>{nZI|MxR#ZuqG(p{eP_&$;a7r@z>2
zxAVRpC-JsnOZSY#4SZiViv|>bOlRz9@O9uao+SG|^1^nvy!JX9CK0~1<xc{ouP{a?
zmF?p4={;Fksj=R)^km5Ish%7^?`|`-Gz-75?dlw+ldi7q^Q-S_x29L~v{am2E`2mz
zFX)J2@-MMN?G5kEoq2Q{CP;jAeeZI;dUnmzHGkNa8%$Q}DO=DLfAO2{4W`F$r{(>e
zfARY_FU?f}&(&UEQ*7w-fAizdjEwaQpMPk)kfr2wTtL3?uDaO556(N<ME?lPtFY%}
z_hjN{yx-6Mh^tar%K32OyDx2utve#0Z@9r={-J_(nOUcPWXPosq5rKen)jJf?^Vn{
z5UT6{@Ayl@sr9<k_X}PU;+v`7dVV^S?t=O`*DL3SU;gv_=l-gc1_6bFgr-I|ex;Cx
z4-1@HImNAFGy)HZF?#WY-6=45?4-wJ<Hw^}^iYH`ct?nc!?VSwrYXlC>d_2#I5>-`
zk;7q9Fw@GgH3Hi#Z?uU>Gdi+v6Y|JXSiOX4mYz)4gtI{{_Pu;IS_hV2n;9J~7JGBm
z+_h!ww<mfodhp1J(@SQbtHkw34%?Viw}@thFJaQm%i(0O`OwF}Cw%HgQSipAvf-*b
z3LXR<n>zQ%(pv$`cy5F}wQge8zvJ^`!Rm|idF!^BN^Q-&z~SF~FQ>pXt7~?R;{Pw|
z4(9LgGSz;)CVOe#+Nz`3TR(fYiOlqkp5xuA=XmPNE3Z3A`|AqdAMRlL*wzw#y;^AE
zQ*JJu{(!&Unrf^Wjs73^I=uSIy5qZ6!l8eyQszYs6YKP+E)-JN=G-!=iCfenp^=x*
zVxowhYs5pR@V!q$LYOyh>||KnqAD0@%Xy?rcdJCpBHbk(6BsI(TwCU~>YJ)GFmY-K
zIP_Vo72MF}z8mPqnQEfuAX&CU$sw)xug4ADZFe4dO<`K=HrpfYT*Sl4OtwE4PhW8J
zVwbk))z6a(n5WK6v;BJIke}|F4Mtp=vwIev6brf|IakEfL-6RlYOe$7LKT8q2fFj_
zrcUTF{4aDxqqiniQkX$WGjNhixl+g^cTLx<2}@OauedH>e5fVZ!!ze%R~u95)X=a{
z(XFdivu$lm?DUX4;<b9sqpH<`XLgDN2e?W;nGzYlQOlJnr&3+8yh!kgK%b|ILGIS;
z*S4&P<I*sQm0lita%O@@TXwsnCtpgQ$1ARer!K2EW(VwPj`hq>u;+WyCB1e>pqFII
z6vmrkGCqckr9lgJOt57VaF>~Va^t3$;^+XKlO>mAdt^(mzYG<Ry}gND@C5hDsd1Z`
zTHa`^yij)MlVdTb#C?aKye#voP4*v|?=YuGYr?tcKji}A^CxjFbaXS;5b%g?DEZ>O
zNa{+7PsO#MtEa;McZXaFI@08E%HK6H<7<$oY3MblB^tkzT20MF9GI*Q%~f`6*>z)W
zx1Q9k3H@*NOrGxk_Q`jtw5V9;g$L{pyMH)zdb33A&zo|wEoFl5#zq!ljRwnSLO1Rv
zEeP81h<o<hJ(B9$C0T{b#p<|z{amnix_O4j`AbhSy0uPs1iaJo-tyZ)-PlC<!{Sfj
zg6F>{%3fOTvN%Pc>14M`X$vPq;{o4m@+~e*K^rc5t!38z8XUT|LpCIQ$&CJop|Pf|
zb)9SZx<%K!3Tiw2J%4(M*1zb9s;eVUT{@S-bWm^`SHyw)E1&9e#&9!l-#qPu1g9Aj
zqfeUJq**sT49tEOopmsZn9UjV-_x{a^(2QcOV^+N(ao);a)X6yn_*sIUo_8?YtcKl
z3fSCTX(i1aQg&xi!zY8&sfK~K-#4)@R+1}JdgB>+>Qb=vw+ru-d}=&}HC=_hW%Ct9
zH%iu}E!o`2a#lIWIL>FuauefSF0DG1a?(@76#8vC94AR~TuN(c(4D|hRV5PG<R&Qe
zNmV0!t4BocB{o5sow|8ut0eE)l$-AoEL7b0n&WZ970q%_mBLvqlNfe|9J2nfns>tQ
zaLH{3CQXrk#WP=rJuA3kxm!_C&88-Hfl%p+Z8uUUP0qQdA{x58lPACbkV~Y;mIL!N
zznpP8*>iIFiUmiX@h-^>NcQx2W#QA}qgDO?hVcotTMF9mzrJ!3-Z(X^OUS-+%_}Vt
zF3%%Ptdomx<!i6iIBh=Fba&g83+=ZW6B+7vo?x^((pK}Zf&H$jz7}JinedTG9L&xV
z=T*+d<Xs5Y_RusC^E#L?=P)1d&zFI_8$wbHI9Cd=2f1pf723*Ki++fb5|TW4#VK^o
zU8hAmf*4s88Rt)$SySOPaY}(;x)Ep8g%`}L`Yk>yXUysRa^gTIo9FqgcP5iAEz%Us
z4lA9d?0HDyR=Kb3yHovjDW`58NMDqEHI2J`vd2QDpP{We4;Ma<;@HJkWYL_lDSx40
zgRGvgrQU?ZgsHO~w8D1=hRKyJX8-xfWy{jt&0i*!8+ZQK@tL;6{lUYfk$O{4rtue@
zp8jKt(!0pcYP+NIi#S^<qhvxo15OK>IB~3w=jCMbF%_75B}ZhHQ%740s~zJPmO1;c
zPW|1WE&8)FUu);8(7$t-M1Q?p9;R)m{{Kwcg#Ws)lTEFZm?v6@PpZqwP0;b>oN6+0
zmzt(&(b1qu8o%_7IC7Zvo~+<`J==fAMOSO>+CYQd!AIWyGp=BGm*226TJhG|D@(ZS
zKKLBqj#H0gsW6<{QGMcsZ%cvJ3g%x*^%B`ve^#^#U#O1DS*f*Z+0j6TISsQqST7}s
z9!_yOxpqQ`Ytm{zPg589qI+S-jw=d@rcPMDkb~il)|qo@r^I+!?@a%1_H(JhPRA(>
z%ri9Er(Ha~wuq_pvrN+NP~&(hF5B(g#f#*=26^sl*(9jtU*F5K^5&tP9j#Iv{@XX~
zpIdq{SZL;<PhqR)FOmCt&NF&u&g!2xnhgrMeSe<pJ-kw+_+G@0?YjyW%g#03#5Dc5
z(aNiq7tPN<ky#=-J7UT#)>n}er3BBc?EAp7gyj(DBW>f~x56c}Y6HcywnSffd9Btq
zYXbXW6{UMGpL@6JI@E7+Utl2IANRxBq1ocdrb<W2Mpf>~C)8%7>Tg`RtzK6!P5y{8
z^Gzp_9UNCCaFi;_uqD^~e43wHXW?Mfm>W8kD|so4&V=8>U-o*cGV1<dnNWBvs<%Am
z|I`f=r{*v@oSSe`prNsPX|(XxRt;~%QzsV|F$o{|^laCh2>tVw9ZB;F-sXJDnY6kx
zRdc(UVfFL#Pin=k-k(|4awkUd{!fwf3g0B6_j#CH7b|@p*;MX5jqBrDt^-w82iH0t
zTc8#DQ7m%z*SCQZwG&p&d9V3!&HdmqQ|-k9(_bITVlA2YDbtBB<qFfcJf9u8Y0TU%
z`*g2l@V>S_EO{^0`r*@viu*dBnf_0ImMdR#{L8waSzH@$#ckBwD8A-FQ_AkZI~R99
zl+71VxE7+q@7~e=;>b^1wpxW?g&S9U?g=?McN{sWbvRgEX_fgti(^Y1Cg%6&Ep$8m
zQXypL#v84BJ~aQIu<*pHwlbalu{9x+Wt7%`k(n91YQABAq{^--cBfTys%NU)|9Sm>
zosUw!z(Os>yGNcm?J9jIdY!8!lWF2wlb!aHIk!Hp=5kjLHK_f%HsfivKBt$}&-)n;
zUHtn~c?%zQnP0r@xW>crMoh!k=__UB&+ZI$IWlMS(px<)EOkpo8YO3+zR@~;swA_O
zpaW-|*W4+OeeZAPjd{HELBoMoXBFY)9kUx+jxe>gxUGr4Gc%qqD0umnc^6t*80sQg
zU$`(m2)t_HdT{6b)(r<E-p>0|bJSl$=CI-dF9*34+XZh8xww7_pNy10wX)}HW}*<^
z#8W3(c<W~B_C92vrO3)O@7vi2|0TP6mVOX9%OSe#N^5n61m7{q1-vZ`%jQY6DL8o?
zP<s=$wDXAII^j(pxYk-M{3m+)`o=CR!>9v-M?-pB%mZEjO?0(hAR9TMi7D*C#Y*-`
zv55|`iUvCs4Q3`bu3f}+Z_4I9k9>~seO$Vz&gb6g$b0u#&%0VoJ1)ZV^y&2DE-cTc
zid1ameV}_;Y^ma>3fB4zmltmrv>u7O@+P72jepn0z>B;`ItveW<}POFIAv0L&_r#n
zql<&kgvM10i5_bn=ii)n{q2k(4wp3|*BySy1&FjfYE?Yl+ajKpu*4!k=c|$~+YQb|
zkDPsuXsIn}-7$4y;555i^PI{~v9aAa92()zdE~!Dz!_f0z|78j^A^s3soC!%619Aa
z#2MD<|3c?pyW6#<$Jg>F*FVQc*AFgPQ*rmZ#3NO;*55KmZ8mZ;)h$|RBfrY?iRLjO
z*GWl2j!QOgQF^HP^u5c1(+_$T_ySZ`M(Vpp>ufx(cl42m$vkZ-d8cJkLF;5Aoo@F^
z%zQ4%VzO#sz_}9_Uhw>iJv(>nj9|v;@@bEMMS7TVy0DyxSa#6Gnkh;CSn#?QDeqg0
z^BLW=V^mk^K6QyYacb$K<i4&HH?P1j4(TE>86g$!dAH7VKDEE);jnVjF-f+AD*`mH
zUUKSj<T=&uS?jku(53Fl!V9d4({@bz((xoA>BeQnWlC~MZ&g|U|51qPv=9%zB3ERk
zn82iHB^ka^hCOo%6aU;rw=1P(+=L?BJRYk)b-8d(V_HZ>UV3=mZPBdfaW59fCd~W3
z_l%P3482!#okHYX(>UXF4%B7bVR1QDS9CC$=VU#XXT!9{n7-TPIZG76Ziq#2rRj-z
z|8dVQ?UIl^(s(8MvPt^h#$>ak#bKt`?g%Wha_dNlyI=BGEiB?}f!3lnfuNu*O>aIt
zzW;V{?-wPBvSrR5Dan4%0;QH_d`dghm{$9WJ+(``pDp5eW?x<3y#qde2d>PLyb>kZ
z6C^p~z-!-3P7a5u(-_&`_?@4?C~Ne1{+;%@ikIEveCKz%cq*~XnwPnJNzZ?gC6DtW
zXZ_lt99$6lBjaw*Q~CA_OFZ733hlV-QNo|#b=2l2^DRYI{?NE-8Z9g#=R{iOOvp>i
zuu_}za?%!;N$k)3!mcTOm5pmZ)N!_U`TdK_?|*!8^2zdJz4Pjv-0Imh8kkma+H`R8
zw5(BTnab6&_MhL+jnWsdo;eu8B5u*1ef61m>%@5vB^#qoc)Iz@Zk-qG$kF~yI`rTX
zGsQ-+Clfn9J@dY@GLpBY%^@*~FH$?ey~**_IfiQ@EMeQcvijI&@9JY|emAk7DWXCq
zEc4a!x>-T>dMi^{R!pA8FxBhj)G|lTDF>#^a$pb9JXCh@;Hg!dH&z|yVp%KJF}-)9
zThNjJ-ACA$xg4x@X_qxSe&NYXxv!CVLeE<Hp0#|L)>d&@Ns6`GZ|+}CS7o6Eg}3;x
zxIPM2S#ERdd=SHFpEpmMO8j;IEK%S)RhbsIEXN_Hz|lTZy=ULtaJk&-vfJ8C?Gx&j
ziYm@4zZb&rReZz2yGpzf9y$k@=B&zGm&N|&`GIY(Zt<P`n(J^%VD7qauZr(1Ek5Fs
z>v!b9hgB|nW=LMVZ5p*`%PtA0H7%x}*1Ei4vC@l*R6Npotfk}GlNA#V1<8azm0Na`
z^GMe<cm2p^r+0R+l)UVH@nV<XwQix8FPauaTzc^#wwJB$LBHFHlPi23F6zBXyIGgQ
zH_7c-vVrFQS&rf+FaJ+YTDe9@=cCw~6(Tw891T<T-b__`Gu3H5gU0*A>*UuoE&l4Y
zdY>H|J5NvV9Ur%EK4NEksxlboc{CoC<5>RWIJek}ItA^A?{XDfyJqnP7EBdg%9y@A
z?bfO*{{Mfy{BNgsI_>27Bv-3jO|}OzpS4LYih0Y?x9t0(mKSO+J5L2oIOY<>77*gK
zwzq-(K4<jRds7^GwR%`YIJ%$P_ZmEy`E+H_fft1iDJ2cJcCDJirt$64E5`?`9Pey!
zd{W>uCE~qX8`s&nA_f9mclZi0G<<macK5GU#Xkf!9fW-ztj!Udy3~Z<lCR@s;zya6
zOBedCaLAc_>7omZ3u8#aYwtOR)@ytIul=$n=ub=7m$2>Ugj-b^s)d%Y@%erUERg2v
zzx;0FZng6t68O0Cr$2k}Jab;lzG;jf>@ueuTD6~V%_qGzD#kir%ybUfy?2OM`RSY%
zfA)&WO^=m+tBaR7tdeTex_hGU1qTBQ1A|I~_F*HR53{QtxVApzbTc{o@yVwZAuY>X
z7XL|Ganh>eK~#sBgX=#(+3r{Cjl6FuGqHwUx$iQ=|Lnar=a^ij54$W#GJg3dg(=wO
zVqVsjvS(~xHq~@Kl-s?Id3WlHNqQelbXjWO8cNM@E{VALDV^v4>(!SG8u;$;OLIyd
zxL(I}wnj5RcK_MiTZ**RKROw2VPI&;eaIq^68y6>`v0GMtb7-Bi*M*gcd^`Cb<O9?
zTK-=J?g}ieNg0hj%3I`Em^>U-xxAa><+60|7mqV0XXK7Vc)A26c*wlgm;Jx7yvXnP
z&Aiuf9|G3<luM<gOPNOVmS(1%d(k^jNk3E4V&&vrjLQ7)4{)uSDx|M&{pO%ozQ-Md
zBd?ym=4_GVXpqd%PYF1nnJuHevZznSL}!1I_Wn)I*@@p0FE8nc-gQc0{Ws6!VJ#iG
zO$O_EvLd^@gx`5A&x{lL+%SRh$c>1$388+cPHlH<o%74fU+cOLi^7ei$+{w|@>fpf
z->TPlB_LKO!Fka;j|IjKUB;DF?VO(rbFaT+))%|^>^b*ggVlVT|JVF0<Go&>eqXDQ
zzgS$^Jmjyp0KduIwMQM<9i8;IIBnkK!tKB?gX?R?Lo-PiiN!M=7#vRRcqXyuNXd&i
zZs(8dwjU9fyt|t>xOml;1q<gTE3+_p9DF1C{4e7t3s!E)2U|ikH!nWBkx9KLYGb5h
zQ|^P-;z+I~tlK{a-T$~mDBh|+Ra5PUVOy@wRJFRihId}3lP_(UY-hx4p7UAi&0!<`
z2y1<(Hycck1~xz0{4MBU=04MBN4DPHV5H;yJ=WjL?B!RVg6%0+Tf}VmE0sI4!;ZXK
z;FB{g-{|MprFYb_b<Iqfj1{~Np0axWpY=z$1#6d)l2z$XBWqUa8>a;A5`Xcu{!cvp
z(Tv++=7Os`8jCmNDoiTXT^YsUva2{ySTlEb*@Hw2r#sJU=V{fyTgS6Gx54h4<C}tu
z0au$D)^i;dcl>{WU$%t7V|$7EMknhnD?dDSEUD_3b}=g~U*65KJURHBY>U_$v(hE!
zYD|v%pM2}$^U>u^n&j4r9?Rv|CVzBv;FX%Ycu^>C!mSb?_JY<cB|cu8ukjw;zV7m>
zHC(q=UhVNX$adsc-@aI`|65)~?Wp6~^i9;rw)*D8Z3YJm7(@+tJO1ab_03$nv*A{l
z<`*&jPewZHE>$0Sw(IhYuZQNP9I#=#b;;tH4#V~%y-OeOh<#oWJK0L_SVl|EWffLS
ziPb#cQl$S^@*g=ScVzyId@qqQqqXndJa%rBIvcbyF8Oea>B`FZx)sV+%Z=NkOv<zW
z?@L+Q%X?<mp2TfWY^qp86g-Y>Xv(;LX!B~9IlI?ORW690wmopO@k_0gs(C#39i?-d
z)rCrbv%L7WVEwClDXW6_#RoR+Q0qRjX!@SEq&Zx<2L)bV-s-YlJG!b=eIrA}cgGuI
zjz?@OjvqOkzwlJau9f^8f<b>ACzJ=iT$Rv&P-D+8wTu?m&9nKAJ@vS<%jJx@Psft`
zVjhkW{-TQ>``wB!TKcih_;|$Amp3wYvb$C{?Vi)vDXBei-`4ij*7KV$|GM7tT4TrR
zpjX9eeP7($pX%+L{ArTWrvH;SS%-SRJS6u%oV&iQYO6!arcXtmK2`lav`<c>e4~c{
zxo`KhwUj5tT>g6Sc+qn8(&aa7ztwIJFR`lpzBe~7=M76)txU$Qx>Mh8rG1LHq0erd
z%rES(QZPMu?&*E!r&SkQFuG_nGDq^t+(=vC(99H}8Kbcv)Ui3o!*-2E!x`sZ{%F@T
zC)5&GS8csj8snkX$)w7ynp1K!XlYZg&=QV_m4VAnH=0L!Wo8DaUud;t7gF67_$-jo
zlUFL&s^Cx)r?7_41BXWD>3p$XQ(pes+QYd%@id2q_LZfby{Z?abbe3smefnxk@D0a
zl__q%mxzkz19J~121W)EwHOJ82}hXzH?Rmim{G{Y$RHP#XD!g-czU{d-m^ajpP#pL
z-1;|1vte0-F0=8fdz(_7eJ@IxbFA1~c*=oq^G=QzYd>z!@N;T@&EtFdiR(fkd87Gj
z19uipzvLQU(y>9oX|h!G%_2*$PtHp-M12AhZY`L^`EdU36#IR9e=KJ@!!J9-VcE|=
zPnQR6wNlmIdFn;d0i#m|i+PL>y>fCC-XV0rajQ}ax6?MOfY%GT*<T%4py!jYntO9h
zMYhXkHmyeOwOg19R((3f?Z7CJ8j@hZz~ZgYAZ@<6XK!njNY|F%Ex!sED6kq7DJoBt
zxul@<*jUK6`Iy8D)9I5YD0I%+HX+HB(M)mi>DDmr|Hk`HA5!_eq`fKIw?Tm2id%W%
zgE^NMJZiesG^I~K=#*l+SdpLlbB&}E3Sr+w7VtV;%u>DB>M+Z9L64N|3b_swCr?Jr
z2fuw7e^}Z2F4kao>R-Rq!S$`>m!e-=G*<jt+|adVo0Fi<#$B6Cb(#%wvctJt7k@sx
zYllP`^PXiB95_>iBd)ENX;#o_OwSGJF*<6H?8Bfk#imGE`DXE@rK+Yv+t1`#U*@@c
z^F(aMDNWbjMa!?S-8ynwciD{~20im#zUn6G5*pi!nm29sF|+JuFx8!O+r#pi)}nK&
zu8*5{?Cs)<eR1KfobL(i-W)~OCKi>1RaYnZnntQk+@D)yw<Id<|0&HG@p(6w&iS`@
zrXA<9M+a8zeAK0VaDiyfgf*P1EFYblYwi{-jFnG%t-JY%l*hpY?-kw5^D52!_e&O^
zxp-_t=^UR2M&W#~+zVI>E}bkjPnw<Dczdr7C*z_C3S62yt|WOGgc^I8O~0Tsd-J_2
z;~lz3^qV)Y)!oMJwean&7n2t|apy8WvgPi&v%q<)@ChsL?|0rQ9aYS4ozl-C<k?Wh
z6SSgtN`+BJ$QGa9nU|MM(hA-3d5)81$gdZ#&+Dz-A+`G2qT;tx3(hT?w)#hs`VH@;
zsfBy&7!EOItZ_Real7!XTBE~(O@};`|NqHzo}Cljd@VlS_`}6pJpZ2QufG4U{n&#4
z;=1#h=5LQ<;+T=~!<F%&ZVNZF$br~XY3fa!2^P$q3M{-v2RV1Oa65J}WNzBh*|%Vw
z)to|!!#YY0Z4(VNgqlydIs^$`i0mrietE9MNli=fseX+`<Ei(roaJZ#EwI<!IWzR1
zu!3I9S%<WQ3yn=`VjFijFsN!YUUW)XsKMpnY*iRM`%cm#fxWF^?jnn?{YsuU@hC@c
z{BFS;$DYlUezYMb>vzxXXWwNk?O5_09UYj$j8z#k7V3sG&Dj^xJn<^e2DMEGue-5@
zFLV2~%)6sHVHSt^$(o6d3U!-A%Hx&``?}>!7D#O7az44Dm*<q?8i^~#N~#ULoC|{1
zReMOAeEq+F@=^72r@+v_qD>65Jzceq2#c*fQM7vFgHxGa!OuFcDNR{g`q7v9XJpup
z4Xt~8_#<mK&Wq1cS({P0-tgrkpI?(w^m+wWMm_J5-fp6_{}h8~w1)?S5Zj}3+aB=u
zeGOUe_~V$p;m2Tg-GwI~#9dZp<!a@!`Z=9}_tS~iH_LRzr>Ld-ggb8GT#z^Eq>yHo
zqd>FGnwIH`OFd36Jrq7~Qb2$8T^{yJ7yE845;2|QtCxRsRg7Lpg*o@L*iCImb_jIn
zxG2vORy?A}FDdFAy=H==#Sx2)Sub=s7DaV=9cbh`($x|3ddl^r65EZfzH@$xXg@r6
zZHZ2yr|@s(!x{{lDpvpdw&s{7PL^M}WU}_$Qt#D@ht-Z<P&HBwDkwO*F!-WV#9WWW
zA}L3YB;iXZkC~ZG?%O(LkxAvo2|BM7ST^b`d9>w5+m^#(s+_6&LO#vBeKW*nP102+
zmIJp|?3tz&VJH{TlX+(4)2^90m1zN!c3P~M)yh-ykRj*GB;S1|r&pg`dZy^^bOr?v
zwq19)g!+pjVv8o~doFaGl<E<D?X#_Ujdz#cO|d5bEf4q^CMB?1U%MSwwk1hqL1mxx
z*PDJDujMM&p1v;6ylQEC&Y}g(8<z(BXuaI{Vf8}2#tc=>7ebw0AB?~F91{Dq<z&dM
z^@|&w5{vD2C3b|SOx3T^ak1l^*k^m>zjgG4p0gIsm*u2>G#p``Ai=)mh>F=oXKt;=
zwlbRuOJYxSi5(JXFJV>E|F=yi-X^fiYQfYsyE1GueHIs$ipc%q+GblW5)rv|kD%^`
zMPh3b&ON*J=~_+MP4}P$+Pi;zzka||kt^!L^t%5m1>3t!6LKa!anq3$F_Sb~Aa^FE
z(_1srDSt<C{iZo;j90rH4c!$|+(goi#FlsH91&Apm%5y1kCs3Gk?HHSlvz`(wV%ie
zUyZ$ZB5!)7tFYzz?m*>(8+HWfybSD*y#873`=U^ZV-9V5uIi>wY-n*W%j%xX?09JE
z@z6KsyO!^H>QY?&Vf93wezv~1hZTi#G6d@RDneTRe|er|7C2i|<<kPG#*N+uT@PAi
zzBKZSy<vWN^Fm(;OF%DQX8Xd!`KMiv-(MoK!#mECD=2Q3h<NhR&e&TQ7&tyQSf0Ik
zZjDoGqT)nt2T||lqBjd<y*o~*ar|s+J9C9cbF)rB_1D$w%eJnb5V$%h>~Z+&GdVi<
zWu9N&ccx$EP+#9H%bOXQ(lLSo+jZC6R1jSoJ&k3}^<8QfJ}>36^NNa}H@2Mc^NS6h
z!?p7HO%I0)snTCs6;D63ugp85r@3*VTI`AQ94smY?Hn1g-w!t4NaCKUmHcs&)9oa!
z^SXOmUw=Mxs5LU(wd%OlyLgHD%5m3UT)*nXP%*b#w<hl9v+I-A{eRY*V%NJs{Q0qo
zJr%98Q_dv6{#?gbyXEupZMr>4rn(oSSN!32sA8;8eDJJixB4%E)$z;D&zZPA|J+m-
zpExO9k;^AMOnolZ6|=Q48&*7Z`@Ksz-iwKeYgVJ%k&n@~98V(lC{5THQPitpx-vNU
z)79F`pLQiI?b$cka_WAmH<#D%i;3*oEV4zxzd2^QXMxA_due8Ri%zfOdGT$f?u}&Q
zOKmr!oX^QUI{T@00*|NZvM1-5jy1BRHSl#>2)$8c;B9o<@LkWD$?aj`F`dma-5a@l
z)zw=4FEADFa8?TIR0wicH*52aTIv7d-L0tOqGpLqUJt`&y)@Eb*&g#Y^BzmYvi~df
z)~wv|!-;bnXZ3>C>V56I475BBoZk1!TU|)tcE{Q|cTV;mVC7U&Seu}~_J^17qK0|z
z@=y1ib)V&a`&3_m4@bMh`#EQ_m48f#zLRnGQFj@ePn;Rk_oVeX75oVTYrMsB4z}d4
zxv?bAf@AU_AB~t3_m{lq`}pCfTAFgmho35KNi2&3Jra|JS|73;u-?<mvnju8T34n;
z%ZVfXnPN%ZLGQM+ICS1Q|FolnY00ACZCR(%);;@hUgGqd{vTdKVa@_OQ-rrLFdq0I
zAzHw9)Zk`ZzsMyo?k7InJ2-F6>Xl+Pu)DHMRqK`FEiTg;dHY!wtateEl1C{)<U~P-
ze*KDf`E~#M&AXI;q$LSwJ-lG{C@Fzm!Exe78~4D32}L*h6nz#cuJUIWJ|3jOq_V|X
z)cer8kclE{4LmIGjXC@sTAu3V>CXO>6q)8%t?-`f;qp&z$|fBx+|5eH3=W>lT=zBQ
z)%@dEUn{u#U;oZ00UC=G_Fd}Q+qOwz%LB>jz6mzTyN*prJ#?7=@GD~mhk1V<?Oc&+
z*u<;-yxVxoYTJ25ie?Y%SMi!=oxlA~P~M*TSl(f|{z7?OPDZ5zKa_d{Tm*k51zcSA
zSZI3l11I+&3#JmOW!5R2zj;*tPI_}<l1@()-|B70T$?UtzgXYPRv@JwtS+Rmc9FxG
ztVY==uQS^UWsiCpiT^iH-@M7F?9<JC1#=HQjrnro=Bc!^|JcucO6C4>jEU1rW63+6
zW~WyrO?t0XW;Gpsd*#VW3+6X=9(sbTmw4W8n&!VzC~5Phb-ygUh2@iyS8bjww^ZTN
z5%z`2OdgI;>n^9az1CaCK3h!ReZuM4OI#V|u^R7Ns%m)nx)=W~zw>in1!xEqm@pbd
z-*Ge2NKT#BV#1)nb<9ENSc0*ApwOC&b$8hKmKK`qpJ;NFi!c4KvCiV#Hw3>kWQQEv
zq{Fyk-5O8RvS8&U>+Z@q3CeR><mp*7>2n@)IP#B||KnTX^pJ}Te!{6!EN4GoqTv3^
zZ<Sb`K(fdJbB_jBiOt#1`Xw^+oHYMW_I$`OVM&6BRhJk`g4c4BGf#4O_H6Q;f7sD~
z@|;5#)a@Vly$N=l+?SW){9SMHs}|vPY{yHQoL9=dFn{*ZJiJdnRIC0Aua$7FFmLM_
zE+@s%SCexDeo3f4l-eZf;LXtRC3y>{ZT=Fi#jXa|E`QRR`c*BT-+%HACK2`h+`-ZL
zbI(1qy_<gHL;9DX^}2UBJg@L*G&>3RY;vD6iSOVSyWa(-u}%BdxN2<Ka;8EwJaW><
zNep_q9KN?+)dw-lGQXC)9Aa{fH{n6Xj~_j9rA@*n9*1O(Tk8~Bd-X(r-s1ncy|to5
zSn|;?v9F~&Rz6d^>b7l4?Cq<niylZSA1o4US{wJ@l)Jgl`>}^iyY9_XTu#oLZ8ex3
z4yQ-9h5mn{v%b$S(Rh*RpOen_1%1U=%C;(*Pc1dGDM)D8>|5u&a*3%$r|?F-B34Vz
zK#g=ome0l_mlnG`w$yjkFl!SD=inC&ZCk?cy}>nl(*^1K>56YWl_vzfI5N>zz#%%X
z=u^kG-4Bj`)^X14<>lFJVj5hU`|5b2Xp;y-CeI{>thdMGuY1~SZ#I3sBJAz@np4^H
znx?8{IB}aST)=Q9cYDM7y#94Q$7Kw?7T8Uh$aKO~;+665qY4GD7KnMdq&weR6lzlC
zytv4N%j2N+gnnbcjwFVuhaE1yG1?fTpqA$Cp<w)Ga@0ZnJO4%7w-sbNZZBl;5%x$F
zi8%ORUZ9YNBbVCL66buM$-OaQ$29*ld4Ew`|LvLSH!h!VtJc?CKlg&`rHgWYk-m?b
zn7NsgbN1q-t5+0+R<R`W{#drvn=jqstw_NMhSbwfFH9;<Rpqukes%JR@AB+SDn9BP
zKe;GbEh=<hBdXHCZTQ}O-xi<M8+WZ)b8}Ai?u700UZg8ea^Ozz|NKR>|4{qQ5A8mj
z0m1glCJYIOt`u;`P0juJs<d|No_*8e`^6$EP3`VJ4|e_DsIF}67UB5Up!VD~{dG_B
zz8sT$DAxA4tiJ1d{>7xc7iUh(HL002MAqi#|1LAjI(&DB!onrr^Di7~()rJI`hz#q
zwg<J`zL!?;D>Yn+o~y61;qcTYi~SGe{Nh>~llr1GIoxjIN4q5sX9Cz08LmlWPj4*d
z;P{eO@Al9;?yB_Lj5%Mv_VP*8D}Iqa{~=*#i}yALpQi8L`%)5rp7P=9Y>Ra&l$lnU
z{JG!citJ)OImUF^j~^?wZ>siGSbTnCeyU-z>CcYW*Um*9lowtttEQ?jwYcW<a#sFH
zi*~$I3~_IBSgtJR8o2(f;*td|(iM&RUUNSP*MHq{ZxidAh=nRQw7G<mRli$gNk{N>
z|D5|TG=64f>%&R)$J0G$w`cM$y7#W*^qG}6no1*LE2NVv*4C7ZT!~zJ<6^sMQ=7Qu
zw7>s5>Wv=xh%v?eSgAJ2H2>o^U6G4^EY};BeSIl6u|O?D{`rU5$3Hc@Whmx^yba2T
zQ<rk*bp5F@t46)af4T5mpA(xbrduD9Sa)T{tz!+f_Fi1qLVKe9)ef`m4yjuDz0^a9
zL2oI;`fqI$E?0^Kl&=p6|MiD+#qM|BkB+=<nw*$Aa}q~-LBi#BY0hNbo%7!w?+}*K
zEm~}TZOf}wE6p38s!DvXdF!4ugTF>pPeUbWv25xzj~-#=L$&UY6#oQrYb>5IpL^L|
zu6W<|ExWUg5`Kk*AG3el#&60vQ~0Xo5{H;;U)SfqP<?+nn&Ea?{3iQSr_k=V-nPx*
zTYWuuZ1VV0kmdRREB|*ZyT_9=nnlb1s<&Ounj1WuC#ZdE`0m4xyaoK3w)OvNyF0UD
zW6s*1TRw|j7M%E`ZM|jTt|jfKTUN;*D)`Llxj}T1>Z4mag{j#O7ulYgGLwIb*Q&Py
zu2w2%rzomjzT&qeS?X*<!=XPeN{_5eJf%+8%{2Zp<Hfya>q^9z|2|pHZ8?|wrb5!~
z`AO0H-_PDJzinQA|CIl)lVx|FYiT+l{>byO&i5Os3Ks-r>(4JV`Wd-=JNJ_BV%9Cm
z;!0Dlruf=hRu``qee=+2$KurNvT3)(%T$=y%$qLQWgZoNlz3+2V%OuXQcGpC76wVY
zRJ*)%_Hnc48C#oUswbN$dMvS=arnQPMo-Mm5aSF{1I9p$J&R;y<VCd9`?rK!<i`KV
z-Jc>7Fgx+O<uuE}`*Ob)7tLO=+s5w~+gW3$vxN~17yg{Jzvk0eeJ;H+PU>XqL0(^(
zZ!VlK6cx0lu(L1flGf#@J;}c4mf6EjM%7)j^Y3ez*VuMhtzf>CDxjdecd6aWrCScv
zCs%3zPs;UtsI)C%w(Z?3;!Y9kTmCcbEa$yh&U*7|`z=}BX>m*Sx3NFEX8!0}?XCUE
zQobq8^Y2^cvzxoVp14pbs~|d=dG+)Ku}uC>-fMzt78(BaU%YwpB8dR?P3KIVZePEi
zu<NkgiU`wV=^JMY-CnS|WwCZ{-;+<P1V0D#$Itz56#dVMOQ)jbtyKNjTQe<ZoD7`$
z#pQ6z+x5mfR(me{q_4D~we-8`owByaVtHS)^WK~}7nwX)R&vR*;4{*5&r}9)?Y#B)
zvZV62=xHy%JuZyBZa+;Vta*N_|0G6@L#lJkavtAhQ3x}eW2DNJSYV_V%~iMf{7%PP
zR$8*UD}M7?e}5aa>erFG@{5JvUy%OK_QCDdk1LNq7;W*rci%cc!gj**-&c3v+o%3-
z`|+&h+D&l<29M)+??3(Y!0qI>i=#}h1Rl6kSNSo2RrZbi8zveWl~tumOj{Q4-jPa<
z&@8&P&XRNa5zf5Dlbd<|Y$+|+Q|iW}DituNY7e(^iR02c-1lpb-~T@${{43C+)|nM
zv-Xrs&B?m1&l7i*J?r$tvb6!XzdFmkykT?pbM`l0wd<4h_UX+^Qn32zXZIxbn-Not
zi3s}{(?x1Q%7@f<9uPkiDeA+(HtWXntAe~q7i_0RHHXEhG<!R(UM$jdrcbzSLfoay
zD@8K9ikh9={9mkz^(<LayOuZU<=+%h^M#j}-}|@yN`LY6t6LvDOq5kPkt5`h^<Y^O
z6Uzjz6B`yDZWmDYnj=wo*~N@g-0q4;p~%WW5$7%$&r1)*7=r@Ld}e-1ancp=_tW&;
z{Os^#3)V9}CtMx;rwjRidvkJvkB9D(oUPecPo<q*CcMxnN6S=kwb>e>ZKt$MwcY=B
zN(gTgys+%$p|FUXTUMp6zjnARWs-^y!^Aw7-Ar$o7J7?2Iw|mc=W<|}u`r&I;n_3U
zm>r4_J8RjkT^f!Qx-e8os^xXG=sTb95><ckXXnE-MhCw6*Y;Q*yr;rB$;Cs-(JNOs
zYFo}F+cXi$p#9Cv^8cb2e(c;=s~+OVwxr>iTd&%be|vT@JUg}CTs*;W@#Xh|;&oGB
z#HbW^FfHdV;n=&d_~mRBhc&iKRS&<rqZq|9>(g&GMj3-62Su2h(i2?fc(hEfiB35h
zpvxzxF6b`hctRjtEUNTDt4yDIP$<)muM>3FPM9%4m*=LF7t`w87mvCPj=c~O+Uz-@
zMI<O^;^Y5fOs2=X+?V|E@S3oAgVPeHM9u@ps$}<e%<nlZxiw%CPpzn;v?JGp2jMvt
zPv-ghTUExpTv;h16p&%W65?#__%mR3h{TR%o;-q~Z5}}rF7kEpZ8a8kt}_!^7!uNJ
zI&q=YBvD1@4zrf4&I)EBYKyNKvc?3*a0c`<cyCmno~dN|>_TR{Vqrw=B+IEc<ArAa
z3P_&HE0*oluX$s_3L(*Hk~w?sO}m}P{L^gC#QljsXHWF#b4i?f#*XEZ)A_vbF_o^N
z*0NRCth5)-7Vh1ivu$#mww~|9Hj{a|OefS2c>ZO5dLi$1o`s`R@E>vg+5D_CChX%A
zR2AB9dR8lXSI~r}mH%A?Hw9f?boSDe&_lfLEE}C8d#_G8%4a$&LpP*%M^g8lHl0m9
zCYPqQh*@SCvh}l1Y)aK;JI?d@3>&wDMmuL><kE>JcIm`L=+*DM<1RThJZ=WR;@gm!
zVw!I+Ot@!z_RV(7X7hC4aOsL<m&p4|CST`$s!}xD;p>%W&i~46r}(R$T)w<R(cSCf
zVP!+rX~AidTc@7QIjF0d)6}|cM*8ZmZ8w~k3%v_?sJ5Q#mdj?jb2C$yPkVAbW5)Nc
z<h}!3;cLztBwLEP@wm4LPUTVU{8cXJrjvQ)(KfqH-H(oTe39TSoqn(A@jkt`-Y<<*
z<onW$Sc2dEY`weH<+EDKdWQ4wPY3_sbNc+hfE2#7VvZu}Q~xawi|Slquq24vO7WaX
zvBk%2Pnz#f=!h@OS;SCt$-j$5AzEMPdyvRFdzU)RnId^pHW{*<_Am~faZqH(cK*O8
zX%AlCIFn;GRp+2iaKPz~@R(<F^bC)93oN|G?ta2LSJl@|>Sq_1vxM+pOJAiAQJx>W
zrY~z*@-ipU+hg~;ulqZHWGkgC_RM8rnC#oiq9wI^w$NR7$4-@roqSGHMD{+PaP{Vy
zpcIGhu&6|SKgU(|oGW=wd?}iKiQ_>qALpI5Z-Qo?GGXg&?~Yl=GJWP{m1n&hS56e!
zvVilhgSy9>gyust4oz7qaw)GYd(pIIJ6zP9E~N4O-zDj`t7%_b#6+b{+Vh>3O!mB?
z>0LilrvKdpcA2gR?-X}yUw5fm>@{V<>Q=Lyha$6%Pk3=qNNJl)=e`{b8(LIWsM)iK
zt`(TPs>S8N)VtXoclJ0R{i5pSBoi5LBDl;`cVcT(^1?-TtiCPi@ldR{mAU%p!}mqz
z*EF~^J||BU+acPwvsm{-%LHp@6TuEsV>k233QMKLRJc~=Il0e1@qR~}S8?i+DVk4~
z7fw5>D!$hsCH~=gm%i&xGksig6c0c2sZ$c{+jcU?rlf1vs_(Kpc~2>_KVD`!bA``F
zmQ7Yqy)L~laBcfIwN*E6k5cG?69N{Go-UuC=^OjxQLwSsI>mL4zW;-Qi?y#9EHR$@
z@Tz2Xv+_NTZ0(kquCVNjeCvv?s{Q;U_w&n7kDkP*Nl{VpqK7?A1rM#8tnw!D?LDt~
z1#!<~1tsOYJuYo2ooJ%8cgnooZ?t?CTe&d3m~w;pui3nhwny^s-kI>mfko+uW2?xg
z?lrRx{8Tz8$H6ElpkQVBTzMw%B$rRk>RGSZwqLVbp|!R&pv|M`{G^=ViCho8rp>Zk
zqIdA?y1b-G>U*0f=2yO$q^6=XSF7k$i;H7B3#*(<m{?Hz8w0n0T+3$1B#Bt~sW4TW
z#CtP;J@8)DXrfZ*BTwf~m#*8oESr{eG<4Y}P35i(%^(fYpxIutb{2@4C9V<ezVtuB
zlHo_HanRJW&;D4*ILQ_1wmr!TR)`kbUmCi3?N{NB6Sp-t=3SiFc4^VsB^^N>Iy^i7
zi*#<!_!Rv#?@7(wqS!4PR4>f>;%@h9mZw1S%dHYdOMG{>%$ubm^yGk-%7$BW<!;Fx
z;ykb7r}S{CM%OEG(X(~#YfT-ecx{*TObxoJGEY=;O_s|6-He6qM%oi+c^-LYdU^d)
zftB~CyH8nUK7Ym1l&2E+<sv8DlImFV`^tlYSn=?Ms`)z)@LqT3=X%`!=fu%GtGN1a
zUXx}fh_qY#d}?bDQm?xDGTZao#|fLanJo|4rTFCOHDT{N8+}rBvL<^O{)h-%wzXv9
zo}h=B608Q={||gAzwVxT^~s{DV{b#Z7B2|C5w`nO&iYHnitAd<Y;|?s9=~-{@oRB+
z>(;w(I+sK#ZCAZwq}h6nZJm_T3ETL$QFkWnbow><{W8O?37cA1KeSM_@ZqTYR=fWH
zliP}pd`2$+a>I9Vgc>KTUex@<WxG)H?^DMjyK}U^B(-O1%$m9Mv)CaUq5TZUQbO+v
zI$r9zWxD(4IpLX~w}##mn09>UIgR?wbAnwowr{v|RcGz;?ms=ZEBa5amE`F^yXngr
zQMK#O#n<Y2P1$hx^(Tpo|8E=mTGe{jbY6L>ag!-+es5M+jrF8Jk@X@IS4?#)tz0_I
zM|P<}kk|}K*VA7^)56UIwU_Vu-*vs+vmmGD=z}E9*_uLWW!idkg4<tPDmHeN+2-u4
z_?4UG?{+2ROnTjO{b0L~CJPN7$=*9sxHzP1^4<x}o0_a5om;ZlN+(8F|Il%I-Cir1
znd)1xT)AyYi`lmAzPqzJOuN2Jn%1y%SA$jVleQ15fB1Zxeo*zf{VstIFSEKPJX_71
zek)1HCaO&&)bH2ZvsO!tI5z~Ic&#hIBBe7=Eq?9O_<NtG9=@X~!zSYLXn*Cm(|gVZ
z*X`eO-?H}Usw)?zL`-y2GJcr_tUoTtx+4G4BAK~YJROW1DyEi{xw7ePREusd6-f-7
zu`FiJF@X^G(wn9!mP;~K7V3STsJ!Gt*|o#P{{;&3Qj*N8ie0w`@NfH2G1Z)_#$156
z%p=OQqVapE8k^w7W2FLumAsD4eakg#oSJ<sD&3~kPkCM!+}09&Lcr~ZU5k4~g>cK&
z_PA(6v512n2@A`#%uF|2&v+N+bbvMUO`5IG<~Aio`L7>M|C9;+F3kA4ti)-WzyWt*
z2IZRn#S+ZZY+Fs8ZB$qnf0Yi{SkSb|Jl#;VXQ}9AmAKRstt+>?ZZoLeT2QBYJ?&L_
z?kw@vx8lW%J94i*u2fd+5^7ZX`#dqjTu4Vk@bR-s&6X;U_OcV7x>#NatS;}~rEI1t
zD9zl`xLdu+#--qwb7{S*khNm<va%+jf;3HzuD6Rl{>NN&n8Ti)_C3xkDLU#<8ULjg
zCyvU??)5>AxwEfVOk%Ip*slJoJ$H5z|4;Qw&lv(ABbviB7-GM)XbNd~E$k26Q8(FL
z(D{b?oa-WQ1H?kjl2XMpER^GyUGJS*(Y7~3LDbUdbkW3h=5of@tZxQ|Yd$owNGr8T
zX+IS#q^J?WbI~SMS<vE$w~v@m%mv~7PXzWqv6$5CR+JJu*)62Qwa)Z#*B4>`DL;Ii
zuD88t^R;cai*t0G*xsdjv3Aa*qU|SigEG5TRa9JMo2tCne|dSEySZAHgpgFEF1MoU
zbct#IH}|A738xgq?r3imbPPQ4BB{Z`w)|&u6{q@tVb-mI(*L6iLIW6Errl^=H>3Ei
zm~6_98Bv==Cogd|{?Y&Ic>j_geLpk0yiW9cF}7+ipLsp3tHrectH)HA7fStGQY*@Q
zxr+sRI0Hjp%D#S9b?vj$b49^|lB|Go;ouED{|`>{z9w+w`E2P+QVPP1?S($aTm;o_
z2t1muZ2ie-##XU{Wm7NREEY`aj{oU1CB^sgMG=o5v)C3@vSy|}S(zwQICX)h(C5ul
zl30~IIHz&n%$u~r^FmViL8aLS+f;j&O?&WDX6tgH*@;a*#A0`9I=6&Q&$&FG=|fEW
zRDoH{GftH0G`6IE?40n)+-~{h-X-axlb0CiFP#u8S}wA%Xvs@~@Q#^fYW@G+H7Y-H
z2>xEtKk2$Fzv5!kgX$)$7A^6Zncp}g>1x-eABjqiN#)5wZ>LVtaT5ypJnw(-{2Ma`
z&X#8AAIrbys8{$SIjF!a(m7BfRhYv|=&Ey**E8XRM>y*m-IfGN%@mYAS`y)6Ds((W
zxu8lputE9d6r;3lMz)8HoIfc)b2qNzl8L({Bd@sP+7^wpgNpY&g<kAj{58CSD{9u2
zaN+J-b;iew9vw^1&`>^I62tyT@LZz+m!a34jn<{90UI@y4_O+iWhl>c6JGgi#XL5V
zMP=Pxlil|SN$=2-Ruf)c@M?P9%4&;G(|3KJzP~HtP}Q1k8#$u|n=h#Ksb<ZbuUXu^
zs_VZ=;bJ|m#n*1Rnn(#=vs(0Ceb$#B1s&m{lcUV*n^t~4>e8<yc#cEWOUuYt%E;?;
zzz0P^uLE-qI@=u05EMxjD)scPl@O9?2#h|!Soq2L#CN3=mOjE4rJ7!d^Xyu1I#YPf
zEZZf8J{5&N3#YmVF16hhXe2$GL1z*J#{(&)-5XV-H<c*LJ&#cObYiNa;ygA^^BGa=
z-n@)5+g&+JJkQT@t;u)WH62|B(Y_xgMMWOj9(9cP9@(~$W3@6P1DAr0Lb%Z1S4x#X
zW&bfX2~12$*rmuRu#s=q0_BJHzggu{ytgKQEQ>stqb9XZd)HbsDZw8di_N9Bb2@J4
z*|^rDeS7)q&HpD)-)tf+_#$&*jHiavttO3@B_8SBlAOMRKbFW#R+}c8h!n0b*fQsO
z=+=!}SoeAgW(Ir9X)v5E-l}T2f$0EeY_z()%jE0Kf}4XZ+dhUURl8NH+VE`LP}#V<
z>G6VH-tH?l?cTjjYhts=S_YN+O`6WX?TeH*KYF$Jz4N?Z%<JZ#GzejgJZYVAkHhZf
zb1RvjvPxSr^h3JSW-JMKG<&_5;C!Qv!ADC}FD`w~ttzi7w31sn<4J7W(=Ap@W^{<m
zRG+o|y4B+UuLY{AxBv4#V7hSoU9QCpXV$W=Il#B4OVg=2oHO#pkM!qNGX$5++B7-u
z!?V3P9$WrjHaQoe$3014TKD?@5;KG?H_m2e-z6-#AwQ``&{2ewTQ!+wH|Iu4!31}u
z)%&*p7TEty_FwR(`Hk9^HiAMMI3ojP)+pHP&#pW*b+fC7aL^vbI_oJ@S@I<2OcUkG
z>)E|SdSw9DMXS`8PRWO=y95~mXA4+-(`4@2udG<=X{LVXt9=>cwzapNH*cC8vskcA
zXc5yIdrq4JTwW(iZ}%5*uhlP_WulnWnz=-EV^!*(6^TJ!3U4OP&{2O`w4RqSVasY3
zCMlbbM#6?Eg44tfUr6r}ZcNdOEY*rUoNO(5L}VL(qucTzOTmI|a}vcFwryCvaapVJ
zZh^!jepfdL7Nm(C6zr_p`z)&0P`l-V&bt5St%a<k;}iqeF-fjfJLY@+c7fg{4fole
zpS{$aK8r3orP94j^<&pwL5`)X{@BlwX4v-g*rig1&(_wJmg?4p=M@*9eyHMJvqtU1
zCiCo%b6aGD*mVVUvkvgzIlyyQpykef)y4xiUhTgnB{b*fn#7}vRSur$*lE!HqDjaz
z;`JLfRbI8EOgTACS<xT6D=#klS>@!rSo&vR{$`gHfs5^omxQfl&5k}7+VDr#%15Z#
z;`E~w?c$4)WgE5k{tjN~x_j3miH5*UM|vzWzl2mW&X5jPGJCP7fG6`=<~);yUDH+_
z?fatsKI@wIO2IQW)+?_Zlu;3S$)l=Y(#~tWNvT^>>9+R&-Bpd^8L{sUouB>f+*@m-
ze=*y*vi3_A*zmf{P@Z@po>#SP&i0hGC)ky)skYs`8X+phtfX=6;BF1&$vMJLtgm-^
zCI{aVm{6^Iwb-5E!L1FwQtNeZ&tPO(pSvk=Dubhdbf9<8=^frC7ndf!(mx?Cx%tug
zPQ~+|d`=%cc4wO7>6B<mMis%!wUQj-g28_U9+V0tUzieecHRuH+Q*tZJ}<Wq<x#!S
zQ$Nku@dHclqq0t4?^_>vo!_l#3yNAF(<5Ux(R7(@gz3Y>Mw1vA7wGw^-!Mpeuu<jC
z#NK`DxCOuc7SLSCdMawK$&W<_N*DQjg(Uu65HY;T<8wg#?89&BvnKsK@&EcC`&@%w
zrC8H7pOs_`Z@mvY=5@LI(na}PrUy!emd%s3d{Q^fIC%2^G*hPStePhS4=#D|V50rc
zVy)A%f;<1jrq2>DQ?b}*<F=MjqQvn`Z=<$Fj*sOe!GMET_g%R5O=2Fm-#z<P^S(_l
zdYIe!j#GK!BEfrqOl1DpPZlbf79%)eTaR!0?#;|wFKpfF*Zm;f&U*Vs&a8`TTI-%!
zyjrV%DbYZvUT*Kvxw;xlm`{pv9%H*&HA#Eg6-C|oeYaZ0Uj6*|G=0l4vx`q-<s=vz
zs@LAqXP90v>)KY%=*gU3g0XS?H(L9wo@C#D`1KZTi?m(Kwp5?yXn%U}UC5!^4-T&0
zxGc{4|9#(Q&m-^kNFDw3JXgr@9(&Nr*RgxIr-m2q3kXqK`uNhis&x&)QEE&Tw{%z@
z%V@C9ne)co#>oEPr1x%3|J`16zf-HXe3)vuU5ansgu54R-nzgO^C7vmKK#uJrNVnF
zJq6QDdrg<cWcuFv5GmvlthTT5oar;6j|+tkzIZ-+;p4Eg8&)PIOmf_|WuvotqPMM~
zKI5g!w!R{#G=$zqyQ$@i?|6NF6`#ZPw*sei8CU=N>{5I9Sj*x4Zx*OebQPT5$+Q3c
z{f8eyVuVzF^%aE`yQ?T=2Y&3`6RrI%@8q1hvr9g%-zi^o(|N{3mJ=)9`xy$)b!!Y3
zH{$2}+P>Aj!%j5F|NH;4t1tA_)BR^`ex9*z)?dZvX?e=F6DPh|cSq@*AD5o#M-{pA
z3g5G@T-p2jXv!NwL56_r$4!1hORNu9z0}{PH@Qvmiyi;n<wa5zhTq+$O4Jo<-3z@t
z>0#ldi4uYPBzix--t+8|eT|!ipt0Fu^|;z@>3iF5+*_e8CwTb&=2yO58(3FfkT-Wa
ze(vp=b>YeHWG{LNmNxv8jairSifIAEN!hJJ7oKcZelS@qf4R&ur-k1iSh}27OyZvV
zP@C(xT--%Y8$og3i$DF-84|r49GaMT<XmP1JalT~72amWA-L#ZCx_xT!3$lY&SreJ
zdNN-FRaUmCxK^1|1|3+|tRved;*j?L;f!Vz{z+4k4ARfB@K%2lVK}0|z{tU;#_-|u
zg1|)_y;3H>L_C;Oh54ekW?j%~U(LBw?TJ!C_l@;T7Co-3gK{I91sl{Vv=}ljF6MqL
zmun@S$g-U8r`8|t(6z^R=L+}5t_)<*ZrsOnO-d&6Qwrl(v8ZJ_J7d$DnXIknWUNlo
zKEdr?d{1MO(%~sg9w(pt2`YNsw;?sXY^K3>=OgB<Z?4RIvbI5?DB0alz@bT~d2he<
zoJWN@C9TOfn`-CPZisuUuKGm8K&@2!%-yr|-S1zx=U6?DJx@;J|K~pkSNq@kze7sd
z`46XcP@881!{z7|ThHWd=2g5ASDayf-|QySp6358#u}WhT*?9~StkAPTeyaA*SW*5
zT#{C=+04QrxO11xrV9#7cf4My`8X!#m4c+;tCJe<+Ez_G#wDJ$ah*bQ%!~CZ0=JVE
zh2=E%JDfaKY_w2KGHcVRHZ84OMQvBD7slV^x?U{NIlDQ3e{)3QhXstnF&)09oLhRQ
zFp3xHooZe(p}|8_=*Gkq(r;zDRtYlZYUs8e^4jR<B<}m(dP~&I6|(1bqR%hz5^kDw
z!CP?GmdmX+Tc_AEUi&=dN}#{)*DC>{x327M;y86h`GnHmPZpg@Mw@%D7#K}n{G7>T
z!q%21)(M*#p8If^u?s43I@$0$<#@C|3-VHws9vJ&zVHQ?7sJ#42UZs|Gur(TSSWIH
z(YpmDm$?rd-Fa)Ow*CU9$7Sx>yvjVcmoYuwtji<ZB)-t*uT@9f##zlGo1(6DPYvwe
zrG0*rruOFIr2c(2C$6@+yk2A9>gRnvnem2!=GKX^#XlCOo)7-=VDhm^0f$uXc>P?w
zWYUBQ3tvac>D--m-M7QaD9GpLy;;eVle&FRDCj0POW9eydgCA)n%rhH`HDif<L`{5
z386xk%KiU@!?LCZadk$diHCepxM7jlEu1RYQr69(`s3{ao8Nygx-<UUGCAtm5@FAn
z{uSD%V-^Yj_h{^&5W(?j1z%IlhItCl`#er+iEDaP>%KYpne(HlL$~e;1=dT=|ILIm
zULOyV3trN8`u47GSM#q+l{4;HcBZst*A)IOQF)=Zi*Im$@!fR!gq!7!DMBt`2h5GT
z@8vd!$zAchd@y?Y;aq8^I}e|HFxwr#xFUk%;-~A&eYmciOmJ1Y6!bJCq<!VTjNsp)
zu3PJaeb>gXTCr?@Pa<1|M6$=5Lt?WpSM-FxahBW26SDV-s!HyO-Ep@|nh#FRapVrF
z5Gz=rJm<adX?M>^!B{o(>Ggk)_)ngE>i&u*^TWNb6-@sKur4T_G<Tu8zHi`xzAs9P
zcSz(aK6K^~^!&bE^_gm?w@H%oiE{NBNBkAGbj^LgsKlmgE=$&&hf3n_jU`uGu1!uq
z<3D4@(F6arCpzWwEL`BU;rYZnX95)qMNXcQN!h-Hae0ZOFdx$*4P_N0E`~m5j`K0!
zWEPmRyYOf>-|k>@^s-_pU6<)PnIXc{&uv+7stqU8VwQ<#JJyKi7|c9*#7&T6pP-_4
zaVo1P(`0Wc6;-yT2@wh=t-3xBt%Y9s`h81Nl+1h==kOz3(f`x>9VarJgHt?q%AFKA
z{87gxXXcX1t{01PFLh`vdVRZR$JGfZZ37jLi7;3k?G(7M^s=JRjTTRjhpP;{UfRTm
zaM&3>71X_DT>k&g+-X8eZuM=KG8dWnx^*4*atK`#zVS?tQv5Oj^VAEA^6o5DetYaV
zN9mUS%Ip&&Q#Nd0((-@3E2lu0QGc0-YJhtB%{7uf)|SCmOP6GRxw-r%M}^Ga+)wqt
zxf7gfehEfP7*ueCdfOlWv9%@P$ihG_&yc0%?h|~rS_>R{<@Q@*;eDTnkr#gklpN7&
zDA7%K=Hzf<XbGD0sO7SvR;+q|+b0o8M#rU$&M(_!f}Go41hglvOnjvBOQu3|!G;f)
zxC2x+EI#^8!$7d(bnr8#n<8!gokg<heID+SJO6Mi&N1%#opdPRghg9zq|@4zpx;*4
z$}Vmd?3U6v`(s06&#~my2ZdZNpSaL5MaQTj#QRI$O}5DseAJHH&V6a|>*Ac$^{Iz^
zoCHJzg8FZq-u!g;%?QU=9`Edyx3f)7^p<2|Vo?0a!YabR#Gu2#z`)SJ<iN<l@t=W}
zL*zwP!t6#4VZM^Z28k=pmI_T0j9`1{62xR+cISlPLk&iG*0wV@E{54MzK!BsbKL8)
zM%kKcianYWrZt2zU4Q7ZWAozk^BqL{q%A@cF18poYsH=_S-hiBnbmIUDPgV9)l5g;
zUa1Tb31sx#uQiA3((>@dz7zR)q{TE=H|9t8C5ErQdtfrti3yi93Lm(%*0b(ORsONo
z;cy$Pd5ENm;qfDVm;d$HOxVCMwT(Y++MEl8lKZ4Bc}31vCcCuoux5!(s7gL+6l3An
zqx;~2i`im_@-uVGMP~Zi9p`z{T^PJNY}*^YHAl00G@@ST6unt_<9Xo0E37=dtM)$p
zI-B>r+J6!GDLRL?$ri3+zi+%dsFZ2XEP0Q8TmCNix-<XFuflx?et8`YosuXoTRO#O
zP0yl}siikJ1-a!UJelnv#ry1{ThHC)58GtQc0_2VPG}2o<tSBb=wv!77VWCV*Ye1L
zb>)<YZ3@#I7de?v5{gKe(|crYp3KZCF}ekrM&iD_TiNHef0sRRvHXRhW~!6>#L!e{
z|4h;Q(_V9K@tWqfOmte~-CFad{!^ANKdMnGsOez4?Z)M<zRJS1n?B{|LhjD%+c+s=
zUYFpr=ZP)Le$H^}+t?V9(za;ljfI7>*WI+uxfC7+Z7B_T>0n;AOCZ>WYeJ?|?u1QI
z?tP7J$6921mq-`P_WA#6frE>X<*DL8C6&8c`K>ntn#>uyg&Q&jZ$I>S*U5S_Q1R;(
zjSBg%OYYAM?o+w9<+7jpC9UgKoYS4&3GSGd=((|JviF)Rhh{YIJG1iQDiz-Et39fM
zvfuCRu$rZ>S<JPeb^pS)s>j_-tzH`#(o9OSoY$_scB{v1+J?o>%#9K1?Tb0wgSK&9
zWlHzz*{Z^3{YrJ}8#ndmDq&H6tKLlY`_;N~@~I5X7P;te5AR$IFsj@t8*z`<Wa|3R
z%Gauq|0K_fUP#ND@w(E^_O3>%!N$ii+HbEc4VokQGPEG-yjht<YV)(#)7lzVx!ugl
zR9-U2iTkjs=3K8>D^+LqyEkqv6Px;9(7=7!vD;T#QvOJXB(!~=EfaI5cVXwwTY<-8
z16K#^Qz@S@eXI1oIpzN*uRpm=XKBBTWOl66WaoQd*6-<>5*^EYGt^&u_J&2u1xs9B
z2p^yK{av-wA2#_{*F|=|xO|W+H6p<A`^qm-OAj;mZ+sTPc`i`*8_&*}46`#fp3*Yf
zx@OwBmly8@Y<?WFZ}L>9{Jgi%%$M4WJbiim{=Gjowx-2lZ##o_P7mL7-rd7}{r?%M
z-jN4|7{oPZF0<|RjE!>lQ9HpG6*56?%L4bcZU?)n4&<qB%V>?$IlQ}%K}+w5;F&ob
z#>)Fzcn#-rifD%%I^=Y=#b|BA!y9Eg)L&lZ)-YOi?#_R=&(_TauY$QUeY1s_W_NdH
zDJ<3s?PIxM#&wcgWp!7aa|o-pkwUs9Q$k<UqsvQ|GH8_8Dc<sR5nTH>hNF?=iP))0
zCIS~OUd-^`^;qiT3E_`(T>G+h6gTc+Ix(fwPw~CB!q<Bdb~b9qA{QU<j9DZ+muu5v
zxi1HnFub_P@5j|5m%oc!_eRSc{kBQ*bq8f%Ss5)>+I7h}bC#l(?xHDr0gA3Z52dqv
zBKE|osW_`t#c?(m8S*l!C-<-jIYoWk$lsLWt0>v^|5e7$&xS@SIW8&N{&e{Mnsc=O
z3+HmiwUQ!CSu3JH9t`{XVzGRI)>6B4LH34Y2j~6|-0YLE@p;rY&Ho~=T!Ykwcvi5o
zcRBJ_3Fz-V;B7CkDS+|I;)T4j9riB_#ry9bRS}%!@#U7-47=PFQ(LN1-5-7Pc5BV?
zud(D?`sSD<-vgm8*DGi4$7QVyY86pjaIwRvmU+&C8C=RUozC6fl(yUB4%dXG+{?qK
zY57@br8c~dnV{yfaY0|m)QD$Grd!sATy^PtzB2WrmePK%t8Fo%E2D*uDsV?0w~-Ul
z@Q!s=Q9il)#hOD+Ulw?)S-w2k`i)ti<72xEt3pJh%NFG((HvWg8C!#9sVG-3yx{I3
zDDKwvWz)QDS53d<9S4rERoY2T^`B-gFk#ZFEp9wXlAqkXWv+V8{QlF!KYZuxu>UFg
z*@+^aVlhfwhCe>r-@hUeeEeld%a1UH<Dw1u(^ubVGzd33X_{9&SvvN4l=#leTk}d+
zR^M$g_`dV((RbyCU&lUbD__YFq#ZTW>W=QzP{o^jLL&~c+i|5CG;cM?a1uVXutRQ}
zVuP&ChE2=nSe*6jp5&?Mx1Uw~!i2ZWriSunC3{55<@Fz&9y!IqD8y`bky?9KVCKV)
z6G7)DhD=vs^%1VklIH5vR;+w<HhR;J&ig-HmRAO<7TpQap5bz^@Q<Rp$1*L3UemAk
zE_v%2Rz-z2W;pT8yl`Z)apN5g?dhk&*KL}%*=7Q(W98}iXGJsi+oT#KSDs1t+dOl%
z&()7Xq5nl@GAy5Tx$A(6`OT9a?}`{-7e8P8OMlY8-Px)Oo%#3uySky`qr0=H?u0{H
zM=v(M4QPrGJaj`=&3xzV+?4x^9&jdqyPW>{WaIzT9fGaX%oaTLSn2;;c<P*pLrfbY
z;yer{BzB!zDka*W$GXF#^5Z(*f?0Fjw=CdH%3ip8Zg$b+m3tJ@vKddGkK4pB!}hJJ
z`|g|9%C>D=>iaa0`^l|~VhlSL?#(TnzWWwa<F?83Yv1T?-+kkCn1|ZE+BdK2X7BM9
zYYv}x=eB+SqxoOg-m3i?-QW7HX#;2Fi?)BOr?kn%DDueX=vusRY`mdby+&eT&@Hw{
z5=JMt%wM-C&!YEX(3kA14*xX`S4eMq-Fv1;Q`Bv4FaLMRdc~B&o<u(FMFP26=?ib&
z^V`l)rzfyT{m6@@oktv|=m{+2KJYRie%F=R?}V5CR}p$E*CDoHUFoXoy)U>P{<_@1
z_iLH)-Zzh)Rb68jYRwmxJrz)AoWE1`x+k~Q`k!}~#G4C!JZNRLk(2R{o5J1uZ7z2%
z9{BZW&66XJ6Ax%V^J4sYO1)pPXTq!sPD8;im(vqPOItpltQK+9i`jGa-U?Ty4W)4>
z0tEMNsdIi`YU$on9Ia&7CK8zOyJ?PJZkD;_D^A7TJKFXC%%5y4WPhIN`qcWIXWD;%
z22ZcLvany^N&8&o1NBu-Yxb2a$!GcH<Nxo!!=ClEPtujRKhHDYKlk(d_pCJ!<UZdx
zDF4p7{r>9rZSo!UwTY%o+%6uKj_Mp{l6u?J9k=+jOtP+GiY|JbX?B#We38c56v;Lr
zZz~naH5b@AFLIf+7Oh+2Jx@t;MS50rifhZlU_mt@he^^lsl{A@SurLu+Y+j`JW{tQ
zZ(h2w%x+;bry#4Fds(h|iQ^2`fPfO`h!&>C7S2YN1a*eo@D{rjEuI<-0Uj+OGX#Pv
zS~(WB#B{Xgin9hdv_uNDCjZDuGDxjU)4P!9xV6BuZCL=<(s}{c%1p<KV57p^O?uxL
zBL5s}+L36s#J#;KqDlH1*P@MCsR3Err&MwnvdaE<G?=LrHD`L#oIo~J0qJ_jYy;5-
z$%lz@M!LHeN-sXZ_BKIiYmr3MhgQEEf={lOIId6>+#s_4N0;Y;GTRlcyc4TEB)a`L
z+L#`8d)_D$WMJK_!60y;h3P<dyaemS3Du|C>yCX(k^Jm$$XG9WJ)l9fP*60%?qDzH
z@qo6aB43V+{5UT1c!t<M2BDHfaZN>r!b@z6SLpXMHY$E~a!87r;OHtTR`vN&;Wx*i
zS%osv8yn|INR?g|h<Ok+|G1^Ln26jo^$F~vJRPo^zY50$6dQCDo8I6GRA9B4E@uBj
z_=JT(q(#U28v;HZEP)Fq9g%3WX)m2w(B^*BQ6|On=hpJ4|4&Gky$JYuUGUb4$pYVd
ztHt{qEh~7hPj0*}P~Tqg?||dl1$vvC3aSLfC$mmm`^`((vV$qHXpTXW+zf#gZCTwv
zgr|Pe*JY}jz0jQZpbgW3X&i}M8{L!+J7?E@cB%hZb$PLX#KlYwLG1z|x3?M$Zzgmd
zFPpSFLTb+s;h+^g%m-U?pU+^LFp1+}w=yTIzyyJf9Sl2e%s5`r6T-o$JG1%>NAJ-z
zmCHZsq(t1<O{cV}Mzp8a%Y2Nt{5jd-rvg(!pT|$(dq0HN-<-U~bISb{0=p|Cy37*f
zMbx(~P>wt)BJEK%B{Xr{V)czn)Mo^0v9Ay;o}^_vMR4N7#2+&xYb2eg{y*Ayx>A3E
zd*_C0BD_C^8y5@hYV2RSRNgFX;_E2_v%W>_JSwwRfmLp1soVE1m+(o_i8H)!%($Y|
zs=RWNj%8bFN2!}pxr|Ci&J543FNJxwPvP1)xqW&4w}<h~OQ*U%;eT_$F(A`RQ_yC{
z_3W;es7dOoiJObsk~+-NEi!g?Ow6!h-q@kvG)<pFByWS1=tZxZ!%>SCRr!?|R%hBu
zeTwtUa`l@fu=Itae`nXCl{MRbba#3#SgN?dtAnL;=8Rs+?s=Irk3=kezPv8og6;AN
zE~e#<UpB<Emd}>Ss%X8+Z>=ftrM#g0l*-|svx{d59G*G5{?u&ula_xC3YWa_xvA{*
z{{JNb>94vXGt~8?%q~_+uKKj({HH}%CQW_vQ<$q$z(y*HX<?Dx4v`lxr51A8EdRCS
zu%qNo$N91om#khkF>#9CtO?aN7InLR73Zq6ax7drNpmKX;Uo^hHQJnQ2@!2J(^=Cp
zRB~J<XT4}M?-Xc!<y9ReWvMB-f9BfeTWb%jT&wp>a7U?R%wg#rje=7{T@|<%{VD2L
z>J`jwIgQJ3TDqmc^y7l;8>M7cOe=Vtx$FX)N>S&QWt~h88`if@|31m-6pKvYD${SX
zSfZ@jHvMQ(vFs7!X5AC9Y*A+2#TEJIKH5GzwCqP|KtWW+Hc77oos%67uHC(Iwxeac
zr>4M5i_NUh!#Dls-2C<WLWVMN$Ak5vNBjPV^H&)*oRLhLK2cGs(8=?4=AR4+6BCW5
zMg2^Ijj4}?mkNnbDO7*)Q*16*mdpgrz8T(*AEgYoOG#Yh`gPGs`dUSp$$Yy+b5YA?
z!;;l@D^@3@m$>|B)t$NBS-qvDviZm}i@LD#a#!)HtDd#T*UA*GJ=j^m{e5}Q>V@^I
zc3g>yc=Le&t3$=@UF*_~3fBa5Nfrs6vP@QdIPY)7uBR(k&iS2aRJA_kKxK@;wwQ#)
zvo!<~PYZ6>7Fhais{4~^yV_i277B@N*4$#|v+Wpb{lyhO7G*9<>k3{alq|ry(W1@q
zMvq-bja|hu+0*%Nk2^Aj27LVgJYaR?6xm11@2ygNc3Y7B`NEE;eKTM4|1pht*<rdb
zfqzCc*SiP&T}LPLPVM6e-@L<f%HGI@FQXUUSfx0FTfZT+tH{aR%W=!?o06G9iaxs&
zq@VY%5IXpEkw$QJaBD^-r{k){uU5()<edL&CGWxAQk&-4KTMP;bh2A2xF^s+Yt8Oh
zW43xju9JdOw{GH;v&_vr75(2VQ)Y@@Rq5o~sbbHxXaC7w==W!B)2qqTr|Qj=o?YS?
z@hyRWNrKP}?##|<J41Q;o@gKI^*)w%W+7|Nl;oZQ?x*}`C+fXcVt6SLSuH3u)hjxD
zOXBL+)@tdM%3)I5RXQ!R*849J*x4<>e$imj|8Qx8Ndl~ihSOIFu>2MmZ`AjUn#Ysf
z)fkz=UF5hmi2sjhLK#P(8TV0U!2|WTHmx{Q?jc#&@b%P#R{>2|x%syST=iNZX=N>u
z$j|k>{GWh&(^Q$I0eppz4EH_Y|E7>*p1yzUnN!(!ET_Ca!})yYV=aN3i|1&bQv7*J
zFn95~mz+gy2M<m?=xuPVzxAP*^Okk(J?mf1Tvg}Dwp8hGM56UVrir^AZu_uUYUwsk
z;};5fhxim6xn@=`Tq|>QhUsj(#)W;}YwJw|XIyF9GGV8(6&LT_P1f8ZtylSETXt%A
zSH#pv&I#a}@=ffUwe!*!ed@0a*X=xheva7s&t~s5)VCH(>inP4^j<RZkHj9)J0iMV
zS&N@FEmqq7`i<}>uM;0;9a?9)P3O5lo$yJ=LaD{Z0;`VfSrc+*hpP%_5O-PDsVA?w
z+n;i)GwwItEyA%@;LGuSK|Xyo=3eIt+uoEd6I{#cp}@9aPA^;eDTmDET;&;6djqmF
zmhboB+NyBvhQ{W15`y0tuFX<q=#1t6e}OOc0pHix`=?pmSY5M^HRgbrudd|ddgh5T
zam7nEJzP=Scw#@dpy<Z4A2*%lo_uS`gj@U@r_K_*y-N0W;|c@m@@*nJ5*79czSC*i
zGEp;rw$lE+in`lf6-0#sUte;|oqwr$&#v8vrMEdQW;pgeQ7GZ=QP$pixmBCi{I5Kw
zS-tPUUctp1Ci$&kjZ~O?`3nCm2F95I{E2@&XHN;}G7EgR=#r9S&W~8`o#tFJZ~4|9
z;QKGo9#AFoKVkp622<v~GsQ*R-?WdH89GnDeI!PAr|#d$7h_f!T1RZz!1;=4D*vJe
zV>iX+iv(8}3J4You%Eb;UR0!aLfLt(z=H5AuDQD>ZCsVx8O5{EX77>8!!6U_2hE#g
zD6~pN@oY9fUv*|wZHD@7!Ri-%=Fj^k|9z4l)W>q~$)mqVes1r5EE{;#v*&!FP-$;(
zasx|b!OpTM{>5LeB^^*}uUfsx;dH$!n+F5qVuj~#CRDGvz<+$soNY@p9&_BNn|EV>
z3|I5ptK0r>h`c#Dnq%X>N8atTWoP@ZS^m|4?Pbu}_b(32+H_?$Q`4MJw|0HIwcwxB
z;<#59iv;I8zDkK^Q@HeMO{ms2PAS{J+LJHoZ|KrLYP(n_!Q%K?L0$8vH<bdbvJd`s
zs@kwgB-J4LSfW>}5!;so7ki}(_dQVKNM!E{=e`+xG~?_Ojnk7SuDjN8?CMs{Q-!zM
z{vP0qY+$*1_To(rp`GFvn~riFnZe4X#C22p;+h4M)->$=I-xqPZl_KoOBBP09$R(p
zbyIk29{gi?ux7)9Zx6WUOn4Bua_8jf550ehy=oHJH=*!gv*5=$;;}z>3kxbK%YRZ?
zpRIPDQ_<;@*7{H4H>c|V-=*?j{<E(BXWjju%=<rC{Fl=DzgudNkH)U7`ko0)0*|FW
zO<gLsZQi_jYnDh&vyj&4sjR;F+TO4sz%*KQi_cM;rC#d_o@ccC=dmx@>Ye+OZM}j(
zSps*ayn4dj(|l6{>Zitk4>grLy!Ipa9FhM4>9GY8F5(N;To+!xZ1n=pFfZ|$4Ggm`
zoY@+4^phOxYlqX;;*5(HT+Mglemj>_P<CdC?xH}0`~ZjR`wX}Q7H~%CMr>D@ko*6x
z#>eOTYh-%=s;xQqjf4H#y>oYyYz{bld}|Xe#qm^Zaf9jcn+J}D+)@hq!yA}%)Sqq6
z0oK@?f2Mu<bFN<VRJ_5l|F@3WubO-5)kd)ia{r%qc1{fHUVG?tyi`WzVi$d_1A3};
z&R<!R=T2C4(Edr$rlna2oVlVOm_|34ZvHeS#4~~+(Eor#6D#L6p@x74+3+%!9h%9@
zx@Q{Y-MN!_<5bw{sIPxGeKfMBXc*4i;$k^d)k{o+DWdjEdO^beCKJ{pEu2LPPEo5f
zHm-gis9+nW#_lGluqxR{*JF{x#YLV6G#J0k%91>qbww+1OUtgFrj*dVbu$;FL~>0E
zo1(Gk*T$ritK_s*7T@5^h|K2^Vw7XQwb}L5wD_4!Z>}n(S}fBJ`et)W_`<5t-Syx8
z-4%Jzy}@xdlht*dp3NK1%rLQ@EjM?T$-x<m18=8H-x{VpbM1c*Rkg#NJmO7zIHw6R
z?AiY6;-Q@k?sgx3<UeYywbnnp@=W1NrduY*?JjI<(7m%){LR0ZXCjOtJhRN#Th(TL
zy0M<W#PtrZ<1JQiPIYC6JqQ0V3v<8Fw`SyBbjqI5v}uiOYs{im^W(Tavl<=9Nj$OO
z=b45TJ$fuNPVgAV6ds;xX0d7c^txY>>fIijPPnMDOt|1Q#c<oM*De$Mz8qgx_v+;e
zv9d711kw36Rug3nl@h&=>aZ+$>au!4htuRxH%p-|pNPT$VU`6F%{u2<Ubu>I%yG(E
z8tAuV<unh44?XjACcH2@er}yowupwwO4l>SH#;Al@==OVb#Hp5xm33PP3Ot~)35Oe
zFPM9>X5q@YL3cU>0;6SekBc3M@j7vu(LUGDJs@=RN^$+d5&_c*Ig+itKDP^(PJMk#
zL3Mj$&Qq6$nC9D|7FUbznp($mb8lPV7i4`cg3&87tRaO}fpzEa8L4I)|KIEG+1D5$
zxuQ4d#EZO;!nx9ZYWH?LShV|Fiqy6ZW!%}x`=9-L9Hu?<hQ<bkn%Ue2yUolrJ&dBc
zR++EvPfL2MA1$&edEe_A-N(EF%CDRbKe(f@l(&rUTaT>#>xJo}%iSJUsD(ucdWwYI
zxOq`td9i@Aur6DZQCH}TkH@tn+<$gXTVrtByW8Pvl~z}ui=$F^uVa_z6OG7^ol{v?
zXL<eSsd^RYIeD7kHI`F-uTH9*nz{FtyYRLfUZ)MJJu;taH}kb_d+U6CS+4Sq*Be`>
zhAp|bW5f9kzpD=%u>bqUZLWj!4AX@tZnzj_obCwQUVQp$|EH!WYinZ8Y95c-`ND3!
zU(D?TrW!Gf*K{-X9x_w$_<DvnO8o1aXOlUjb27L0Sw=Q%FK#)>`RGAd)0)<>A7@if
zi9a~PAg6Kvipf+S#nw$$g$oWV^L*?JFEBsI!?<VrIZ<Es#iB(<UYXL0fd}6lf68&h
z%Tz7QByr=v&2B4~%10^P6w;3BFp{4BYNHs7%97f*${gAY7#fW&Pv|Q(AF`-2<l=By
z<Y`zr!TuT}qejBX|1Qn9I@J^=aISx&$|YTV!QoB|i{X{SE^9gt&Di9?FMLr+SpQNJ
z_v+`}CsfpenKRiPc~bksHuR`f1W#x(au%-7IF^4z@=iuc(GK-<DYdLW9|-=vILkG6
zVtdyrk%S{V1fzE(yG+uYU*C|d{p`-Fh`*ow8=hDQoIIIget}D1*Quq)V?MSCSG{3*
zof4$;$Y!<GB`xO7W!H6Q7HAvYH<Fb#ZnvzoZJ)%-Iq&JF&5SoCS)$)_^<^znRxnEF
zwg^#dRo?vA=8-@?cgl{lVl2z%Mt#WDR@LSZ)J|$Me{jxe>4sL_x<tw9KTb7L2hU_A
zIdN-rGN~nWUi!}>FgxvAvs}`D52=$Zm-?e-9L@ORAQiMNh<i?Dib&$b(~DMwP2Xve
zn!%cwCbDIvUsHz;w|=7g)re<;RXs^5?9A#yYqo3+b@0r7d%1IQ#^+Vuj&1F&69rZ*
zl91}s<1+2ME#voUq2%!^PL47crEdp$azzQUB;38o+c0DL&X*GH#rn2!KD?JKn{!@7
zG<`|-UY+dqsK?KAVoq$p$;&4{39zhqt+>jES7`n%ftJWr13u1a!2&`ICn`3Cyvm5s
zoczDXEx31;|K|-YQo^<w%AGOSYV<<d-c_#PeYDxVWt)b2<BKQxCJWsoeibdsyQf)i
zxYD)j$0I|*Nf#Q=7N0uiBz8b$WsV+q4Ez6C`<5(Y`qu3KVfPEJ=m^`UkbF@Yne47}
zMxuck3srwy4zx4g7~bNy($j3_BL}%(sy%iZ*4w6@bP;a9y|<}TA@YrPwu163wX|b9
z_+^s~996U<1GOinDm1V6S|D%x?0NaZE1`Lxn2%dZ`J3O@?Ni}oa<?v0bdYuuui6+b
zdyR4RgvCi=1&d5|^RHfa_|mv!-6ku&Sz7JZGd`<27O<Zxx6N_LG3MAObeOHlU`F(|
z!YAiNbQsw9TX<(H91dG<G^_sUbF0lh3%s`++IUXpl-T!~95wF(HuAc@QIfj(A}yqO
zvaqddwp`-z*=~;gQ9hjtNeAL>|Ez3XD)?lMhlhIPjsFX$7Jti34b@)0AWU%D#w&|v
zwXFQ;aCh5^ttkr|^`cwP1Wf8&TQE(*Vd5g6mzEPunb&R3U$@98UrFTbnPt<qZt2bS
zl8U`QfzM&R%=<c4+ZS(a*e#~5k;!M-nQ^X>ZwljT(RwbS-B)I`uXreQzI68Fki~o|
z2UcyA-M!$Iwo%B{zVE5d;Zvg$c^4a*YKtEDF(u=PsX*uU8Im_r*o=HSOU|?AUx@2q
z3*)Z#*k&nT8O)b0(aJO}MIkwYUAOM`la0T^8hLC2+192zb3DkD4QLSNjn!yKd7RT~
zDRO0w@9G?5VU-l;LkcrqeB8fMwWrhgWzrl+jmgr5YTKC)=8E~6uJ~VbdfSc@ralIy
zsqJA$R_(hS6;`y2dybD^p8dX=dS%>9bH2S}@7wcoU2)k`_gAk@U3hXTZ-KkA$NyO|
zYUd|;)=YF2m~r{eN*2qM0wcGzsh(|nZk&A5z+vg(QlGwT?z1iGv!80u6H<v|Zjw33
zb@;)$RYh?j2MZh%?!F9(nrf{q6_N68&y&hNwM9akSdZ>;pBR_qW8^0i5V=60bHR=`
zzJXD!+HC(MciqrYzkJ0ovcW}6fA7hxzY<4hZ(7i4*wWjo@rLzp-kNihU8i-O+#<Iy
z{g2rhZYy<}wMX6F$pl4hRkU}$zwSCO)5f#5R;Kpi7gSB<f1kQJ#rerA<>EXp*_S&1
z-Jjj+G%4K0e1%c8_FK;tzioE949lck!rIU6C|y0P@c$%}AIGo9XfqhUZs}R%ee7Ru
zqRG{4#u*E^RaZnDvzYX*(@$`=*W=Y3$`k!qqAqUok7S*7NyOP^ZIf$Yr{o@mO|Lq6
ze|5I3NMNm+pi^{8{>aVshZLutcAgXRk;l=n*Yq%_fz`VGJ6I10%zs+c{H$w7AambK
zE~~#U81y_2US|}YCe3kQlJDb;bv9lmr+)5Oe3{?T%Eb29+<&jc@4t|d`?L7^OD3mP
zmQKHI(<M2c%vkU8NzDGR-NV(6#e)619@9TK?Yr7M-Arof8x7S^Z@avOR(=}|nj;L1
zu2|hTE&lxFhW~dudp6t_6D(f#E4yz;(<+h20>Tv>5}Q}N5n$C&lvZ;T@{O2T6=CFi
zV`G0r_qGEQXD?_soS-fz-5PkIWs(5v)P=fdHmo_EIIBfb)c=eNyUZLj>)xgp+Zive
zIDdHdg~uFy0-CoQ)&(2OJu>Kfa=`A@WLMAGM>SOXo+p^xlJ4lV5W8MHJtlCy<1X9(
zulAnb*q5Klx1h<5QPFO3^z_V{V>&#hWtu#<cJxOZZk?~({O?CUSGVSk$Nf(-`I2^v
zy!_a2adB-^$Aq}gP477S1Or8aXRZixStERFVy@u<)tNIhiY86;Q2(})EAtc2q1POi
z+(&0_(7w9EwBdB02-A-14aWb?4tq2OOo<m@=u4Txxsg>MM=#%Tb&y8m3#O&r)dvqb
z?2nt$$+u&U_!RLN6^>a?1ZL=PNSRCzEIfFH!O8xTepHF){U3|}8F9?Cc2cdeoGsm<
z65|!;x#ZsiCO!qb#cM2lh1M@N_G-Gp@ZjO8#tm+K&AZkf**WQvNRbThmLq&Y2doxf
z-sRwIa7?hlu)_J)gN~L6qwNK=>y<f9Ea>2?Z0^cvji0cwTjpr!g{Ggkc}_b?Hf`Cp
zs$<cCO84tOCY|g#q_@a;Pv_RKE#9xYkD7b&UW-0tCM1yb<me%fy^%h9uY~Nq?9)3>
z$K-F)mUPE0uQypJx$Nvct@)YjbfZH5hCgTjU-sm^tmx;oxcBzOX+KW5{mN(#{5*Hn
zEIXqaj_XhJ2PGI>lQH4lt$+2xzEYJL79T}8ZIs@;WD)+rWNgqXsJLzKYEI5IopCeF
zgTKtK(C)alVs#IX=$jq>2AykXZ|IZ}6q$QsyWX!wF;erBxef|^nYL8g^o9t-oRE`$
z7wzLrJvmdRchTXK)2e0fp9r{o<luro7km~wl{|F3zrj`LsO2LCmHQd`0-?Rhm16Uv
zPc11qm9~*zbr1hb39IXOPWsI@xy(85!9$g^nY%u|I`phz4!^XVU*#E#IbBCMw3!2!
zzP_TRtj!Un$h_-HpZ7^I_bgUlg>JKE4Mqc2-j8xUh0I|Wc>Wvu_Uc8=Kg2TCEIGJm
zlTXN*GY181C)}ECqr5WaCI1_lsbz*|r-?8)r5?QVB=8M~cmd;12jhLc*1qgVPsz#h
z@vCxp_ImQ_o_+k_iZiSCsh4N{Ij=G*+FeZ9rz>;z-o$+mD)?S*(OMH^>6E?o(Mv7f
z>^W~guP~Noo@2S_7t2vyuAb*k@_iqsn0(ef+j%0nb&=1OwLx<h-Skr9ec~ba=)}j5
z2PXIg2=3t*;!&Ml6TrvocB$nG<3Y_CO$XV}hRi?Xd-L;^GGAWdFQ*<AdtUu;)cNbw
zb2l$ayy0EIGi}M4sY_grH7*ET7IJFlYnCS(UTbU)7FZq(JUD0F9hD7xyci?>{x5%Q
z%;;#K?#pW+BCw|UkXEmE1EWOP4UUb6PwVrpWBlk4o-nQKfnwFA<ENhN>3kB{d#R^2
z(%ts0V#E>NcU_^mGQJC@UcVqSwOE%U|E!ib@2!O;*H0N9cbUrR6M8aPlht^E@2%aw
zsXvdVJh^rC@s+@;zN{~S?-~4>Jggjd^WEtTdh^2Vbfd)QpvBkb*si+Z_b5X~WtLQ#
z1hdGgwe4SYd!xGA{!9s;a+A?l?MegB9}|uYSL+EZ6Q4>=WW5`<LW=L&$qUO`qs~S;
zzHzw4J+<@P(Nm{+ug~haUXptKh79i&@4F&@r|`aHs?+6>40USIjYz8$yTvN{{M5x4
z8kTar)Bmr_iMT3p$^YYp<=Vb1)f-k`*3!r|FwU9BBYnFm;c&Jm&!nxV=XUV;?pWZj
zcr-9j{reU3_`e;q@A_E3UO4F&e*l-|n;BZdzSp^y%G@ZqU-U8T!r7R!i|w8q*z)Z`
z)3*<swOJ-;CWJ3;IXnNdW1(ympJsDnOt_1!mCxH-x03r}82eJ+g!t<AWX?IZnbmD3
zhxh}I)myh-xo0{3!9t6$*q}*HzDt7+`|rG+cz8;tQ1kQ`p$9|5j&7ZI*`hb?EqDA>
z&dk5vEz*nm&s<B}$v<VwNshl4X6P`_dDFyW8nwxU<6`W|Hx9Sfs^0a!nsoI>6vw*j
z{)SOa4M|sy-eLI`bM?-n|JN%`6y|8!KWdVFx-;QnZ}QB?r!1${&$XO6X=f(m+;fsw
zRmIL-Tybin>0Dj|S<Pd8%Bp<)r*Aw{4SJqBr(ZUDSB}ioMOP=S<(C&^aJtiD`F2mC
z<OPY(?H7KpxcT66SjW28r%#5f-dZZV<)>-Kud|P4d<wg)Vo<K=yR_Hop~2h-R}%j3
z@_oHY>rCaX!kyP|xIFnjH{>hRsii#~DqC#bUN4^Z>I$#f?L!X|jOBX$j5bL1@(Z@D
zn|x-@s#k}uJh}0BaWvDxKHrRH`BmEuD>#)Fs{c;8tZ%hLL2hxjsOL%9tuHR^m@e}8
zVr~>i9-rp6c!z!W7prDmT@bJ)C*xe}i_0ea{(A{+5ehZ2dAG{?n*R4k8REWy61Ki}
z*Dn6m^7A>J!1%(e!!@&cM#GZJsmp&qfA)YSY9ViE(mjJSVk;kuY5eWqa7g@xY{%3~
zembTcK5B=f6kM!qwpA7Mcq=Neyf}OA)_rC-L@k-N#O9>-Za(@ZK{j}+#3R$Yo?8RT
zUOu{N_`1dJtWeI`M=DzXp2qtazihJ0-mvV7hRMF<60vwj`6WFsmu<PVnCH|quD*;v
zA=l?L1?XNX?2Uc0BV72MEU)Uk;(s^)HpM-<>9;m}b}7%IUlPng9TPTOex9>+eSp=Y
zf_YCp|6Xe>cqCf$C}GK?6;n@Y>0P-zFE!@Lg^hhLuB^+xTJiQjhumGh8@DdIzU4b+
zv9oXGgR@$ie$V*A`0QVuOh}v-ck+tz(Z_YXFFP&=3WoA}X}_7A*1vuVr}MH?Yd)p-
z92F~ad4F~%-?pA7kG>?tYPLoN%4Hb7YT<bG?!xAboe{B1vRCX)P1HQQCCxKt-roP$
zoX%QiFx^jk&*A5_`Pi*Bj$eh+W`3I5`t8Nz%xA32VrFtrtJ!Ot%yMzo>V!FqpBdeg
z@$HU#AMmm%W8ag<nh!EQ1~jeWY`?Nf=(y>m=(FC>12dl={GhShB~^2R)=Dlbtu)#D
z8*^`p33zTj>zr!&;NP9ci}&4gdp#>DqjF(Ufaj+Oy&fI;4^R6v7rQ*+%YU%0bp3y+
zyKlwUe(HGq*oXJty8_vgP89~Zjw%19Oj{?rP$kM+^!V%4d&f&Qge@{?7C5{{o2R5?
z*O7{)Op1$}1Nb<1JXriTe}(mjecMVNz3^l@{bg!#LKSPl!lzL`V?Vo?o{Cd@;^6l6
zFYmE0FPN@JK8dik<eB#M=5N`gBiUy=^K@-gJ~I_RZkzkw?zWO|j$6^nm2=+Kx^4~E
zKO7^&`{|a|x~Npfgy-Rmt0R~N6sP;$%XpNKlc6^u^=6-4uytAZ)w7KXpB}a8oytve
z{hXWMTI6qErpNxGvj018XzrsQWjET&tCwHdE%w^)_(6ldPbUAbtW1e$>kUtKf1&k%
zu7qgcoykl7$6UV<<LSg-e@a6n^WNQZN9VcI<le`ni)*q?Us3$TQJ(i!!0Bqk!+{-7
zE=H_;deN9?vzSrWr2{h>bZ7Lj+)usN`6Z~J*<Xl{JN(VU@DJ1F3IdLQkMaAp)U;rc
ze4eB8yZa6)@5Ih66!I%ojAonv^<?2)Rr%6Xm57A2yC!-ayp8W}<VL+*G2@>1JNc=c
zng$%IGa9aFT$1p&d)u(8Q+sRGbFD>oXJzF@dp(riP7u|6!0=L~yi%N@aQPQi9aFs-
z`;NvR+OE{`LxXi?k{xU6w9b<|)``E^B>yAA>_r@VC5wG>;m1!6>)$q%Jy5QHzy5i7
z^M)VX|K}#XT)*T>#?4UP|0d4^?al?3Kka=`d1Y$c$G6iucTVqQY<}q?_EUe)y=79p
z|6lAn&f>f(`R4+&m8!WLjwDo0%q=mt$#K2Lzo9n6L-dO6nwok3|JoMZ5h!d8OpZLr
z=P4|%s#BNRUR!m)t+w>;oR>@;dNC{lcbozaOljs47E0;(u;5@5m!h4>fuo9tyII)z
z?o@0z?b5I4*u|6iNyV#GmbKxDNnxtT3>{s$n3tP;B&TaSPc!VC94x74TfUFSQdP;T
zooR`{oP<kf4=!X{z?E_Na_H3+OkPtJoTn_fzLv>jqou^R)rU4`sOL$&o~E#J3(M_o
zo3Hvg><(l4vTfp_MiEJ)+%K<GJ0^3p|7TQXnGm3(V0cO|ZNs!zFON-~ZKM*$SCZ=P
z5MZ=ak6WSnvQqohR4)(F3yx_go4d1xbR2^g=1x{gw>IPaUFvo3P^$2r-bv=Z*XLwT
zk6AaTlJoh=?qg|wu}e!C+1Qoto?v|W?Y)Tn@s>40MsK!WbzK(DG3oRtKQ-^z<^C7;
zG0YWyy88Y9sdcH0hukMrs+OA1tMbvbIM`_Id6g~0W^1~TW-I4`u6oHy7sH~9Wv@JR
zT%5phO@k>X$y<AyPl~suv`R>uYy@vAcbe1JWtVbdT+AnGmN%FMNKW6GxY%j>&O@w{
zbMFR8x;t_{=+bVt{3%)_sdZ)6RLP{yu&Hf7w_Ka{Kl0JCr8C@QTP`|hsBKxs5VECm
znLG2|@1oPProNQU)vZ+(cGr0MT(yqtqq$&hkmuE)fZ%B>uLS*b<d_;4BD+L9j3sgD
z^-QtEoe>!Yy{>Et<%?35GpJ6>NbUM|<&vK+%hpTR*_Lob&P<pwRcKlo^He59za^y`
z7bh-JR1-;Cn{zB$a)xVX#Ds{#!Z)FL1xp{V-#_ch)80i{yLGqU4Z0aWovqP);trAC
z*SS6wi>|87e_xm=6WH7)qOt29TZEgs<H6}ST2j3pUI}6_FG-)Txz>fNMKN;4g+r75
zM8YGv9CKc^t2K5M#4J5pn$*R<b=Os~Xr5(@wpd(z(H0&t^H}SK|9k(vNR?cAXH!n5
zu7Al*S&vt%7EN|eO`Fm8cU5)l-c6joQ%c;^rzci<EuWp5%^P-QzVq`e_H>WDv?Bt0
zT%#+aXWUrpK6%dXnJV%A+Pq!y3on115-%C6;Uy9*X89$*N&V!G%ZI`kqlHuA6gk8F
z<K~JMX5OAu6?81W@#u!w3c;|eA~9QvZl>ohiTRXqPE9*EYr)yH6%$gIeEYfSMPa}(
z+0(P!q$E{Lc88v^IQln`-Oc?+^!}Z~PpS*|tFP7jV3%kelbg->-6ZeaU){MkF7wV`
z>v>drnMO)<N`Blofx-hX4m@cMd9cel{l-@pg+|{AHJ4s4m%q*a@cTc<=2Ndtz5ajS
z)iUFD#=@PKS(a^J%stzyk-Or8qk+lJ7EXpjo-IO>ZW|Y_pP;0dd&7mr_exh^YYKOe
zhI3cUMYe?#8r^1Tv7V3Eu9M@sv{ZRZ&+|=_G{WyJ;<cX~D0!k_=FF{mTKNv03lF%+
zY+*4}(J7e_wr9zaP7y;V?<4b99Aa}_)R3RWcTwtdn3Bb&%v8U-uO?skw?+6D?-5mn
zjq;9X76?=cy7bs8KCTmD6}cgi(h#;q+40pkQB@fZ?MbgXL;ma#-x%U%cz%{ga#6;_
z4HFg>1kd3(8j#8zx-o0Qt(gq!w-Q#LRXNSK*5lITI|Vy?b=2JRobP682+zIb%3iu8
zvNym@+55l4H<w!DN)^50m8XqzJ+ox@L^+)*RfvB#$t*SI)@~i9y??Is&&)k@;cwT@
z$<>CL3KcQjvz>y>IXN0;YffC`-zO&Kw?!#g+32JtXVbzGt9J^$*>hGNVp<fubuur<
zoCj)VE0mgQ)|dnd@b<<<unFIKVf5RECu4!o6s?^Hd2LO6&KoV(Sl@8D!^cPIGe_FK
zr3;^$IYo2+nzilXvw|)4Y}zlUZ@9Mhe`Mj3h{sK{^LC0Y&+J|%zqC_aTZ`c*!-|<M
z2m2Qm8L9SZv@R9C_}EiK!Jsw7OM7ONvaaqY^ByDN<bRHHRyaMKcFfmBckK#STNQJU
z@SKC1v$i;EnhBq}V*1~EVY|qrWH<J}Yb|dB_FaDI{;6tGs@?MST#@s>E)d?kx9Cep
zi52IzY3^IPm!;GmoP94O@{x68L^^k9@G=*(B=_c+u%78>b^aZZm}$GQUuBMguFr$*
z!n3@J<F+chR<BwiutY7C!Tao`Qym%-?Hhv?JBlAfNl1vTJbT%NrE!AI*PV+lX&f|B
z*UsQ^G~TSZHO;#scdf8`QiR&3Ym4stXeMv2j$Of7xP<kLUYYMCUyF?|H#eI;^Vv09
zbyio|)-yKGj6YWkIix+^k-bObuaEKZExw6dTwL)%QzlHhEySW#-F!-ZagylDG8G?Y
z#?Uhks?#;yE_rR;z}{tdjVpYD6Vrbu$)|le*Jrex@>ts=*%=o&X`aE>WfhLat6ns)
zxE@`${kUCRN7dVCy$g0hOTLAMS1-CLuliqN6_b!ukYtUJszvJz#Xx0Y-_N3Me(Sc2
z<UBa6V>z+I^MZ34-$}-=MPg4sUDo095VDw^%HEQ;a>|w0lX`cFEa_Fz@(3_w&8TXc
z^U`y@k6YZE%Td7(U&zk7c+Jve;m==pKc<~L73|zqomy?WyEivSA@@^hNb}`I9;yK$
zhNXX;nzpTY;-cfQ(&s7H9eGV*7p9sR?rFcdJ^o2*m8D&b@PDJZY_kquVfSKzsHni{
zdm=U+Ie%$lNJz5iiZ?GNtf;(m+lJ}%%{y%>hMWI$`2|gu_MVugwK{W}<cns>qZKB{
z{%JNRJw5XHN#%46QP=xZ{fyr0y^-#`GpVyMOlXs@u#C;@xg5@C4zsRQ{o^nv!lF^<
zZsz}z-`Xo;Ii=_R6#Z-PQ{k&_+S^q#-CG!Z4k~Y7E3&6kXsYwt-5k~&VdX-_TmEcw
zQJ54JqxGZ6sK>EcW{!vSJs*`<x`A4U0&mR||21V>;?r>1#Iv$@44izuO0=93?<V?q
ze#ukhUhI3O{NmoXxvPZ4u5|Ww)zsd(*UnRU`LxNMk`H@N_fOyApXTCu{Qcv?pFc~M
z*Sz;s=3hK<!i<`Oih-|OX5AA0Wm7O=d8c2Al=9CbQ`0o8m)p$vU+8(Br}3)ZUzu)}
zZ4(Zka5`VMaY|Oq*3%NxIs@F7^qksN{dCRE8EnS0E<WU1_*Z4+wM%dIv2Xn)(mSn@
z`+zIUucy_=Q@6Tbs&$BA;L3RM&u97$7rU$1cPx1Jx%%si{11$$Coz3s+nPLI<!p0|
zdZ63E(54mgs)qgDHkvEa_}-?@XL^yYt&$)WcW{|EU+;weIn$Rlxcb+)1(-~c<cU4K
z;oND7W3mRGXYMdLy`33-)K}9}YT@4L%Xxj-(r$aEUR-=tmc1|G!w<Q`E3bGSxw3TP
zX*ZXaOEpoPdGZ`~V(wdKF+7QqzQbai%H@}GhVf^()ZQq+iu;ZiZaHp<F;{Ik!1LdA
z-9qucMpvcYQ;rTD8`rY_6g+Y!F|H%Tr(R=rtWM8nPVZzJ|Dv3>4J|?qDyKA_PMC0o
z>y!>#D2HqM9!JeRZ4-9f%eisywMF+E2{y?MZYiyug<ocWI&maRWNzMsSe`oGT^A&T
zUd~8nINuP_;+S)YZN~%07^R81i>ejly=u<?C}H_I<BE&Nd^R;D<DD#fd0dWbO8$K@
z-D%IF`y2hM0x!E-v<53E3N2!p+S$o4i$yZ!(l6hZxDCOFe@qj-B_l2*^ShU|C^YG>
zuFvW%hwPVjyj9_v@MWeGC&T##6V^I(y53Rrvr+I{a8-X>&zT6<){SwQbFOB75ooz_
zyzJ^R4Xgi)S}GpQc(7Qht7TD1Lj4@^-csJ)i&KuAd~kq8I520T?$TZ#--(*@7S%6d
z_c-GpuF1k|8u#zQIsYk)&JPyl&TL`Pa9CS<*-@k6q~nc5SC6fNN4f+P5;D#(hCUR%
zcKX@M<L#CA;&KAaa~mf_aQO8EO*+u0<~w(7#nIFwJRvHaSFhfaX7dd{abDV`bC>7T
zw+A2Y%54oj=XL1AbY005_oER<E0=z}bFxw9=q*p*DxLdRx6HkMfNjr>psdcWufCUW
z{)oMo7=D7)Rj)}tZ(<TVE1O%_$(0q69tZBvyt?GbnfR41$+LHMiA3;qWt@{;q}=V2
zkjfLHt>UHW(y0-6JtXCS8mI2*^CuoU&NwK`5VbUd%{gewl0UOvvAW-!sI;V0q3q@q
zZ&N9+uam;VCUT`+evmmaTP||%l-?O<(n<x=ji)-TyZIpckHQVv>67kyD@(SV^lcD&
zktB2_iphuNQ7r5IGgmKnx=anYvUUQi!7LZiWiDc~T-r+?Zpsa?H}sJ1Iu_V>Oudlt
zYAIXkO}QX1)l7y^&XA`e5|jL2gv^*Q|4yw!1?MBXpji>0gqF`?xhtCTQM4u1@bNvv
zmXDG#jtxmaI@%Xc?eFQ8x-mCQr>nK)o2=B?ZqE}*x87dbThJ)@ZsN2ZGVwkw)8}69
z-NkigEvvM~<C%>eM-;@L+-W`j<f7P;{~l>&ZQ2jwCVyk;a#6@|;*4rko=~#P%jbrD
zoVTxQSdNp*t`pN2>O}G03t?8eHO1qizMY2X#<{m|Uhj5GF%<Rrl@W7$M@#sg$$KNO
zPu;>eIp<)hR>RuXB@8L*juNXFSQ;2|mNRrb5i%8RF$h@J;_7{hRavzsY;L;8nI~<b
zPsF^=MD;X@-;0mS3DC}I5DAd2+rpH6Ky-EDG5LANOp~re`?1&SD&@LJyIJ-J$Ms9z
z;CrH~apGrmJ%_f4<)S+aTCOOtmgg`YNqjUj;#%vy&SjA&ozvPkE^Cpv$bWpP(l<e&
z`xo77@2Zx)<Zl*@^ODe7`uEYJDF+RVc>W!b|10?asBNQ1SB;PT&L)Q3WpO=i!mORj
zg4#J%3ZE=EGSvj@Jv}A&I68AWFg#McbY{wtO_>*6R(J+zTwTx|sCdUM#id|fX8+H{
zIg1tvS8+=03|?^eNC3-J{xq3}DLgd`UN4ekSv;?0Nz4-eAH41b{+TkDypwpASFO6h
z_SSEYw$~QN*elI`PAtJQj>${4z3+(P`4#o(Mw(mgWPdx+;C-idO>}s^L};s}#?eih
z*$a-}JL~F_<K)hf>D6#tOhW!(DN9YonoXAkJCauXSkazwlhfymxBHAmleV6}`O7o*
z(xY#l%S*W$ge+DysWvWF(pvOQTyW#GpPD^01YIM2`1Z6N=}`FpY`LCVj6+9apX(Ey
z9^E}}51xA*wc?$iWzyj)^|_75WOZH!%{zAW?!ooPGDA|d*L}-#{^Sx7b;svJ-a=6y
zd!zObD^pH#33K=9HWglrnztxKAXjgxd+kThz%9~6=Q!&YzF7ZqWyR7E$sd{8N7i_<
zIC?p(^8Av$Kr+aCoy6SORh73mzj{r!QC$1n@HSiFLHoaou5bCoFTU(OdGyDjORkl!
zyK`NXUe04QUiy}UyXZ>Co2L^N__p?JU=wJ(Ui>b+@}b1E8_5;9AGY;eSDbWePafmn
zjk--9F1I(TPRL~c(a^qZ=ZEQhdSwUL`HEfKOB%oMw(6wCdj!mnJ-6s{(fw@3|EriR
z-j$`P3yW(-d911roMj-ET)*adR6@rdsc?@A!RKDGhD~hJ^;zsR=N;RLMB!Xd@swlS
zyn^r6e)Rs*W5(9~)*xq*#e+>pc+RhIn=<c)tj_}39My>`AKw4U6FjYX{V1oTi;I%>
zYp!OoV?}FbY`C%`=d94gMlP|a$&qVoL+9{Kb2%2u{$Qav^H#Nn3HfKNvKE(ZaCQ4w
z$#e0b#o~+0SiVLXZmV4PTl0v<2FI=|&bl>IBmZ2^c*4b&9Q|bC?U_NjZ(^4zt?aqp
zHR;Wi+wU@G-v1z3ziT0*aQBuO4`q+v+sRw3SiiYChqI%=;}+K(r;wyu(@ry7y%;#9
z`&QHuwUYnr&L!?A_T(DK-ud6KqT)k;yw(cd-zV??)4jh^>fx)o4`=l{R_hCx&nziA
z@W|<%qf1_W&jsP7M^1m#=#M&KDVX+5@Uh6E<-ar+bJ?F^Ja~Vl%ejg3*8O9>a$)V0
z$0crOJvw<G&Qsre$>QIMl&ZcncRmIm5wMKA!FZ~}@!=7_>Ps01C4@WY?tXdo?UaOI
z{!bs`#6PZLpZiI2z6GaGj#t^iy9Kd2#;?|y@ZK!C@N$mN*Z$D2?{D4px#RzRoA0W<
zt-BY^WE1b2|LXLkWt$djt@2E8eXQHFVCubY!=Sd*GY@lSihk4BCaM;7LRebo_Np1H
z>?b+~&T6Y|nq4Q?V#$|rrsDslH20G~Zh6&3v1P1TST|34uV9O&Olkn<iU~0v(!IDt
zOcdFL1Ao+Q5Wbfc_pr>hu|V&oNG$6{vBfNw=UK~r7!PR7-y(O^Qa{FB!_|f7&X%G#
z2DO4|iTMXD+A~ds{eQ~VuF<%cX)ZY5rM!Ln`&OC%Y9Ae07#A;k@2(%1eW)j~-*x)d
zYb=^aJf?FnZDeFS=qh2ZW<Kc<OToe7%uCF6p7M{6=(0pfm!(bE(^YqiW6F=0a@^k1
zN)~6YTndPZb+(<7#pmfHx}l&+(_;4AtuimxU;VyJw<tA}J%}?#)b`UNha>q`F4YXr
z=S*7D(JpoJl4jkFBlo^OC_H-9zh(9JRsWM>?eD~DH^fyxxPI}Rqe)wMhU~V~9`Wx_
zH<&7~kki$B{YmBEbA_+0DPi+8WLr3v_DFRF)d#g^#`sM!NKL)L#UYsx9eeWmm$roo
zk;08zBW7)VAIZJ^1dDo4utC)vhxwb!RJI2fINIp9q_SS%(tlmyawN_6`|MR$%33Ns
zIX)Tv%`Nj@a`Q)p)9dzIZ;s@y>FwG!P4`}@$j&>@)Gj=Y>o}}(`=@7K<OHS(ddVuX
z0&R0zV{hC)KIMj(P57A~6HVO|{Ei3ocq9ZY7CmRs=@{$v^yLM|4O?>ee`He&Y-;QD
zJ}xn{t)Q@__s-PbJA0l6aJ)WP=aP3LDE!)yw;Y?Qul!%>b>~&%8}qKs1z#!z3=i%&
zw&}X()ZTf&i?d7f=d3BM&~4AE?3(88v9RX0=JHK@E#<oYeU<1^fBH1jCrk0zb^R(f
z_3A}$bdS$ZzTdjC>8Dg(?(;3m8QcwwWo|t^anV1syH38^pTxF6^4N~4ZvvgSZm<$N
z_j%j(zc#tTMQqI*HW+2_Y-^jlQL65M*i+VDUpBTbI?a+UI-_v@Ooo}Nwx6pby;kpR
zIKO$5EPISm=0uT9)r{@;`fHZ#-JZ8y;PRqdn_oCpKNVbU<+$RzpNU+QcjD7W8oOka
z&-8vdR-X6x%cj4AEZr?UQ+Q7D@n$s1Kj;u`TXLgkS_)gsTI<l>xpwdVPrBij!}Dmi
z-$MTnsT<CTpZxE}VP?j5WG73|hPQR$F5&TOL<+7Yn|YMHZ;3kggZumsw<99OyRURK
zKjpi`66ct7ith{muAEs1!zSrl&iuaT#UDj`{)y#x+J9#L@eI)4XqxORl^a<xRrkXt
zefbuV90%=+OTlqZ9|ufPVm;0Jdd4~>(~$Gu-tWI{_sMd)jQhTaEtm7yYGRZea%LDg
zayfE(sc=|WG)`e?XWC&Sa^R(+Ylo1&-JMOY497W{mhrTl=v4M-j9%uUqbc-w3a{+)
zoFg|TKAzSg8()_5a>C)mvxRnkIb%`$%)i~-vb1nUiRSbUk@~2b8=Wf}j6_ySi8@UE
z|LW4Jc-b;m>(ZRa&3vb)c%63bY}#ymIz!gGEqHZX{_UF7uh$ae+H>3{N|sK$qG85=
z(&SFk>bbg$G;UAmjNW?Y@F|JYJ1nijUS1ap&7byCM{4b*a}vt4<g~?euU<~wTBLcf
z>RIMBuVnig-3KOl^LMSC75H|8_njRo+X`kZsyW=k$u*C=VD<y;uPp4WCM{l5xM%vN
z$V_9=GE8&V3_j<)W##5);U6E~{QPwGr9`h^pFTdnzP~Et-N&`&-nrU~yeeO;JI(Te
zdvp6OjSW-Sq93;CZWrNjSQI8#^g%IvjaD;{5X+A>%TKikG3@DNshBY3u-_4mJ%?qO
z)*ls9oh7(&ea3&8H+szJNgth#v1G72^AvJT>^Z-0mCwoUx+o#XQ)>=onD6+;{Mz%h
zn9l0$+n4ohIwc_gi|6FEQyQ0jPVOq!TA|CL<iVgR7j#IGb&|p=-S$Zn7M!yZJ+;}N
z(@ZnjebI|W-22X!T=G!=!c&wnx%~~Zrk!iKm&vkEB2ztsegrLYV({Nza#r0`@T$Mo
zDh)4FSCO4cs}IbKYBjwQIXR?TWZS3g=56ae6pPmEP5WeerEQOJ!iG&|H@OcBn`bT+
zJ)WlI%*C(%d5s3c;WJ`7D$cJbt<YZkM<~W%<&kYVEXRGmZ@PKx&o$l;dcl*rA5U8y
zzVNx-q3df`t(d*?{wujnehn%j|2-18gW^iRm<uoc)z$rb$ql7PpQjXj-uvyG(EGgs
z?~89FM>#Gl^?vj#d6M~NC8qvG>VKUIVzyfB?Bcc)EpU#Q9lmu9k5C?yVN|=6($e0r
z1I%K|68(HX-I6<R6<*krQFCox*sYbWi+qK=bd=|L-;m5O>eE@AEgb!L<{OE#(*gt5
zX@3@bQaJV4CbLCV&X-bmp7z<ua`KeR<Yv>`GAk4`RT_d99+vg$*wu3HiZPR&eAgqz
z6N1ZY;*SVUZ#s8qbN9v7=L4Ux*yzdmhFt#TWYkfh=@I{Op|9Dzm&#M(ZoN>MRdJ|e
z@thk^rUd<|<Enc(Q_5|l;#$A8E4bHgVv5x1Jo4Y`$Il&_!K<fmDqQ#Z-IBeg?Bqf|
z(GrinECOK+i>Dqv@n_F5^Iz)&xs+c;Fv?xs*c*5L-lA#gvROx8Ni0|0a%##U(cBB7
zQBse%tj~CxO-oeR>bFFj|0Yiq=d5t?BPL=yJXy>q2cFII`MjjVAizsi=G@71O$RUQ
zcxqM}^Kni$X5{lpVCK9TUZU47XzTgFitnZPV*z6>%@}pAD33#r&5YO%eHBptKdD2Q
zd7HE5jin+hm0DNHs48E~ZRB|`xv*A6&G3uQ!lX5iA1x_Kx0rfFy=+TY$PzCzr^#=-
zsuC6^pS0kx+tjm0Nqw<b>OYGG^Dl5HH=mjmDz#*_t^iNYhfV+Qt%`GPyHldBEckHI
zg?mj~_Ro2%Xt#<>alethvzl(!whivOEl+fK64%yzE9lY84dmL8dOT~}3ilYV3#>vD
z5;M*@skUWB^6%cVP;S3SgKpGDbN#)k&FPtHb(ccd@qIn(Y%^JfebVDejU4ZMnH)Ic
zpKMW2mSr=D>TqING(n(bmN8S%g+q(GPH<IjauQA!ST<qHhaK%JHfj~k@|^c4W%@bY
zTwzV&xh*Oo7u>HdQB&kR`GD*A&Xznboku@UNEt4h_iaxL_g?QQm!4Ei7VHaI(y^1X
z*D3gr-KtfgYcC0W;Rtb`8TDREIG{;{<x=4ClE6t<Cn*Ok`*5_c>|@<C=lkpbe|o%c
z@zwP@9?@Y<vySf1@NDGyBds>;QBrr*jZ0#wv%>%9Je<*HvpRh32UqPI2e_iGmWXBj
z%-a3QcwM>^r$vcr^Q1Em4v3e9SQ@WM+_ylvq5BheLd7J%Ip0psICP~+^w}9dZ8t-<
zupULO30t2_xCM4sxlI$@r{Jpl%8{*sjmO^F=g`47Mj9*U1Wvm5@MV=;*%a^AjmqYs
zOp?CsQXNyfwzU_XuTfyVvbaHOo4W8kpO25X?TppdiZ8r<DnR+{POBGng*TbX!ps|U
z^BR0_?a}Z)u=;MV%o@ihQ!+1F{4`#6Hf2if!zU6;!uI`_oh%}x(5m_U`I_IV%S69&
zOiH#f`7h1C_2AScD^}@l4NEZEz%+G%Yud~rP4mrb0uI$o@i0D~qL$?q<a;AW*yW*%
z#hJ4y@mXKg-TZXsbL{c-cz(r2*UfOomRm{&Ogq+yB)plRArdIOb_2V%+rchh8$YH=
z+T9Zki;fg~t&@$K+|eld%JNd7;zGB^SRb`Up-xdhuGNd@`z1VIaU-)!m1#$;dsO)R
zn~5#uB1`={y|x(Gx?L|`y6JK7x=D*ptF$U{CZ7tMX08y-`@-a+@~JOrhisQ#+*09k
za-sRs<jWq@Cn`wqjQ@XT)AT({{4euM%E$8w?|<L3R59nk{myL<bTWRPcL=GyHerHh
zz@~5J(~cE-q~AWs)%X9(VKKI_*Z=HZuQ}hg)ki`_{N#kYXY@E<96A^nbZ^ZehbLdI
zstTQ$o*=5wc-rw|yTnYsuJ4E6Hd?y$p1Z^(v_-+6@rUzWrVb{pw}CEY9F=>W(sYuK
zKTz*vxTwQ0)k$dA8n2)`t-|4dK2^t@KHAjvyf@M2{Ncja(VZ&{gB0Jr%{SJX|NFqi
zMf(LO6iv)dw>1Bi?5rtwBG=faUE|jT-RudSmyE7Asy5sfapG_aD>qQTu92C_sCvOi
zwnRKMwa!6$Mb+<c?r^S^H4c**)Z4_q98>YgT4C~3cao$dtNdlLiJ8t3S08n4XuHa{
zG+>#@*Uvdw%U<ToebaN{XFuY8zVpBSl-uV!fBqD;^nAB(R>y&E<>>ys3_ss*O*r1y
z$fF*%f!XG*Otab+g$7aeNTnAp&->1<WId5Dwz<39z*_IGDKr0EJ<Zb`?g`VMdM{ma
z`bB4km)zD1{b{T0EKg4H(djyNBde$P_fO4ofzp_L3M-6tWG{S)`z7ikZ~x>+<&)?-
zkKzp`*;CwEyMvm)rhK-w><%rQby1Ofo|((mCh<jCnr|k}*gEZN4PRa>OQTp=>{iD&
zZ2R=vl$1;sdb&lQ(Cc-+)9}Me#p}@_&0k(^c8A&+xOFNt3ZL;^+V!?bPN3-DQ^xr(
z6m>56*{z%*kt1@3vo2zx;)DeqUW+<j9bu7Y;{4C1#x;rIb5};vF$eBr3Xx4c2PJ$M
z-W=R7vq<hwBm3Wn@_sAB+0J!oG$|)N3)FaI?eg6E8ncSdNgI`CzQ<lHd%&_Ss7YBP
zZ~na}kNzcp`;z>)g<HwtsQIC#KO6FQhH$kmTD^0R6T{#0=BHe=BKW+e7V-Rf<QTTj
z=*!#rOZxR!cyVx^bT`qok4pTo>GAVdYhKKF_FF<_Z(ruiXREKz<C`|`sMskz-3LoW
z-zYft9WnB9G(NM;NJ%M9;G*}jw8LvS3$}HC-Nos*Z{_SiErkVMitm=pmU(m}DCf?n
zb@K&Yi>WUc|IS?0BB;D1flWSNZ9>kck~g2qdXAg$Zw^Zo6+8d=f7eR6Ukl}{4l4dy
zs8#b&v1+0GpM&x>3uoL|So|h`!Hx%dLW={0vP!sCDJQ6(2}tS*TV9a#NG)ekNx}2Q
z+cLSo7@j%Nd{%DJ*$yYmCU1rZK2Mtb&y_tjx{~xX;;B-|Q$==_%0o)aJ~Tg3QA)J9
z#1nt?-MK?fa?JHV4u$t+{hsyivfQG}1}p|#izB{VRDAY4!sqSElC?Eqy=zZB=Z`%o
zy5XGYo>fMEEz^JH^R_d^op8A|$+<1)(5*GT8x{yLvpOUk(dF8e(QeX|@o5Ru*#N~O
zxjGFRI`^H5loD8?H?X>I=;-obGv_|><Yd;aK!!iw?43!{>4)^5PSSXGN&Zcv{{L)8
z4qiRiNdlVA=}IxLt;O`Vy;!_mjm=iYZF$0KJC#;ZWwys!!YKzf1Rs0cx39(2BKVxy
z<tGjPKXy&%pVyQll;*x<(cY-$YKKqS$DJ6rE#QrCeBo6%xu$Ol%Yi)=>&~SV?LEMh
zW#IHuBH6>BMN}<;dxfj#qL(?xmfcio{{QIFtgJ_ec^-)hK4*ORq~lde(uJ4e@=n}E
zZ|7`D;5J!!TE)=+Ns4Wwpia@mI|nw4mQG}JUt@PL>FA7)*5>@1uQ|-Su~6&<<Cdy}
z`#)S#{KKfj7oxOnql0{tqpXtjUq+dOhBB5HGRpWCo(b%}@zG%an^P7|c5>@Z?^@%`
z-=}8OrTRZ*eR<HMMYArPQPB(J`Q%)CKG9?87X=~heNLezAs43byPitoTYmh)E}>t4
zmU{fpO!(2Q5nvRup<m;M!nY^OwdS4DUvlbl*`)4wX*2y?;uM-#UbQmMxum#m?S@$$
z8p}FF#55Hn60;c29owStdD7C&EDqd8+z}5}i@LpL>c6t#+ZLT4ZxxSBDohHTbC2c0
zl~2Cg9oU-Ju)SY9uVRAU>O+q0A<|EcWPWbC9&}o(<co`jlXX@KXYA_atfZtP9Z46O
zOLln}c58F#7bkmM?dAKV(Ea+mU2)LQ!~b_};I8L4sd%Cjwv6|Q0E1$|ok<P}%Z%(C
zjS9cKe_^#!aQ7#({Z5YmRhlgRE|<0O-OV@gZO)?Ic54dnEYhtyp)2C9$kA>szFe%7
zBVf0&&8vmdhW+gP`kU8Gl{M(z=(cc1+?oA14vM=i6uq(V#J?@JxrT*LoOKQ+9p86(
z?!76Wd74Et9@;Z+kz)&CkeMRtvQYd`Aj2j>okvA?bEYQkH<YP8D4(-XW<Q(Zh7JwB
zQ(HxzoqF+{>;996J9zx2OelHbuEwISav*09OZK*;c7@XAL3XKXElxkYyq?TSQktN2
z;oq7{E5-AZl$4Gv65hq&?k}uVpY(H@zgEGNC(FJi?^jCr?<w{+$#egq@H1s<Q@o<M
zGYw{}ny_R2%Puv8C0$q7EEC;y`W937mj4{tdL_R1w|&yxb@`y1BA44j_H0LPiPyfo
z`TFN{^=~%bbt>SN(Bb-*tHvW{&rzD-^d{i?r;kZzbaOX|ZOdZ+T%@M3)lu_X&5?w=
z99#4fx9g~wmi|nVmY=Hs*YSpylbZxvM$pj!vqlkt$;;)ICf8{9tT1pB@mnr7Ib>}C
zx4v<n(d3;zY&ph8_J0<)Y*UWt?RzdEs4UX!mtdUyAzh$WScii%YBf`4QHo$~UO>j$
ztdLCgsb}t9&|I{m_>+b@<B`mha_U?rjYX#3AAOd@NVeP71}eQeQvWZQK`NX*GV%Do
zVgs8G_NI-ZEbGHf8hLI#y#LQc@yItj)$>JjH_rQS#<=;?sn1Gb4&o)H=}L~aW@YkB
zZKmB~M$;S<cW4z==%{?vi<MA*6Xc{^l2RS8phk)F_X%yYsh$f=<hDBqZBsbyptO6>
znZT&!wn|4kSywA0H7z{z=4_9+V*mE_M=Z(%jy-nq@||=|W5V*AT%WV&_PYzey-*QS
zbwcTh!zHHr6SqIOFa_ymoMXye`{mjMuiWq=bKjJ@q)&<(5|>k+SjVk>IcuHRuAD1Q
z8$`_(ifw4K?)Kp>%2S^7U@q50(K{>s9Jx2!Wb1!g%Xog#rZrPVt2V@)Prvu%60`5l
zjXdTj%ozQY^pE6BEc)WYbtPzqtz}^P27`t*0=)cN<eK!#cWD2A?a0Hoy*<5k)>mek
z>%20R8xHUZ=5bGU=MxT1+@_`T?%}ouF7u=n*Oa-gGM_#3(>0SdBx&n9l{38dVo!=n
zO<wx&K7HXOrTgW;cKt5*ntoAxuQjWD*3LfrT{rHD_m$}yB2Qe5xP{h+Bt<MVWXU&Z
z$qZb&ApW^-q|Y~r=V$k8%<wZkJ&$W*-ItVz(~mZv3jUzdK4me(W;y>E&rLq5s7oq8
zpL0i7sH4(eS^LDZxAv<GWz54E+{Ba<B4p$jC7144`I%XN_eOb_L>?8B5Hm69t%fQ4
z56x0GP5FILTy2VBgR9`qZ>Lq9oSx-OR9(n*a_ea>c2$l=-S-27D$LH5Y*YDvq`5rD
z->!qZWJ9=@&LcIAO%4TLlrC3=F1hAjJw>TfIs0Oh_NuTxP3eu!ddz>n6fAS#$YfRZ
z{WO6$Q0dbnRq;r{htKuqX}Kxg%#&4`@3gb1oM-B}<KmNCKbSb`NxpVa<<@IFqd4I~
z9P5Lo<}>?0P5C6J{K={E__L6mvXAPcFWj~HE-4yX@;>*~F+=_5MLI7gmHCTzq)O<g
zZ{M9>E^4CaBz&dJFll$_bv@-RL0fO`KXbcXQ&dMnWbf9SL6yCmMcXQBLJZX$gH#qi
zPFs{LCA^^F#FtAq)gtl@o%MP|jY2&9l)e4z%O3Z2J<rmfz47Ovi0_LFR=uj@TkKg|
zyzqbZl1SAi^Zb@0q93){CT7VTwmAH_W7!J1f4<x;nq1oyw*1m#y3!?j?c<D^JB`&o
zAEc5+f_k%}9`3qgZ1>+&@?&v}VgBcNh9{N?Cb4dbJscG@Khm)>qV)Oi_MLxPQ!UO+
zjZz9z{B&#g8R;u~elu=im?HM#p>%cP66GyoE{&q=qRu{L*55xdX?<YI(Tz^hO)Do%
zs5`K@m*v6!hs7?j{y{<$3T`zesQgt>d0=<!wKKQV;%^(vVoc7Rif~H}v9}Rcx*~B?
z%_yBMh0pv`AG4H_@NE5kWk1!v`ir}(d|$i3zgwmA(61})RZ*X;!}b(=9bb7O?GN9X
zsI?}AKf}4+d2{{0(7AEf89BCfa$=c0|BCf{YDzp0PG;}pnXr4tXTLiOIF9E>?}$tk
z&3MT6c#_`UTUB@WiWw~w+b~lj<L90A6IVHhizKa)|MNO&Yo&<G!}B|>LpZjupWUth
zdlGZ->Q39|6F+`5ewfm}B3b+El7zAs+g1e_9_G0CnVIMCVT~7yxu5KO@?u4p<zt@n
z65&sEo_ai6!)x6+bE`fN=dBK>JF1T^G8~N*Qf4{$c%@CIs(b$t*<a=#lQMrzy)Gkh
z(=cV?txlddi3utVq94p&9rgdDq-G$;`e0qQ`2VHx_k?ZaeQngmR=(V=)fc7Ix9jfi
zIbm~dNq<&$eC^1s6dv?4lp*8bvHvDjJT8v+p5M7uohW*wLGH+{h&P!XE7eyYj9SAe
zTUhKX+bp%Y?p<eR;RcPremO71%-t`&bYavw=CXXjrTnsFn^TJ$l+=5^pY=L)`8khS
zACtLX@S(&sC9A`krgJu}T;#ATY14H7i+jpEj0<Xhvb6la&@nS)l3~(ip(izEwm&_0
zcS}ermmHaRUftk|SH!pF${jh}Hx|UGwoUhpD0rGIA^O8NOHt;a^qKokL9eUN@7-N?
zG5+oTJ0+Z3<n<zE8#c)$oq6z*@!Vg<V+>96_E&8&*{HOlGt9o8DI#;#)(!I;I)zpJ
z*0>ucyLB^J<h{{gczmeppWCm`-_Ot6AGX$G<E8%|X;)_WE-~S}q2e&fjWKwNhsTD=
zN&#K=sw$dQA)O4G(KFa;xlCV)%nMo0^&n{~cZiY5X_voR4VS~$TAysXx?(H0>sHCF
zvr<=IeC@YG_^91eudbwf3)2sJDO`P(>K2sKRHQp+;_J(Yy3Q-9U5z-fVxO?2zs%KD
zb9EVYBK=f~O*)x0MO=eJCb&3cEtL}02vlJB=ECWc?LTGh#TCa?qHfLAIa#3O+$P7$
z!oG8|a$)b$a=CcFo%u(erT6>G7;{+^F|tOwo;l%lW!1;UVH20Ar-klseRE{PWjR~B
zqhA%2ne2C7n7_Vi>#dDSuBS8%6U{ICq&RHj%-wbLTS(U0PPYGwS?fF%SX-rX6%U9>
zWbI&dm33-;7$R}%`ZTvPUlnojyuc+hBGjiTG4xpP^tjmhsnkiZ&-$fDQh$Kzf$0JM
z2i>GygSKp$oFKK;Y0;zyflE@RFl|s=&|vK=CNjCp@mQCxPgcrlHHTNzW;IVanD#8!
z?wyFMu1etcv^hm~SGl!UOKiU|OCr{c&9gelg}>~8r;CvF;+`OGYsJzl{*`aO1ou00
zMT*~@|4Fb*ra)!xl1W~FwocB_UGpW~+g7u2nRGx6tK}0HPK^l-ft%!1T~ms)Ql`3v
zB$x`WXIk;~+S)_sc0P(x`&T9Ens#$Vc(4D*scCL&f|eW+YY;S3P0{i5{4ba!WqI>$
z_`OLpuDd*S`YGwU_R8kS`gpE}SAoJ4kIiEEnkMqr<$I8H$i8pq%-*|1ZFqFLe9Bj&
z_y7+7C*FrH)owMY7c@OnFmLad%cr#-evSN4b|&=8M7MtyA>H=e+zkcxt9u&Lbsv9J
zpLdXFXU|=&$rD?QT-DQE3eO3@T~iQXtKYNHMa_Rs{K5UIe`o8jk#K#I>G<%O)c%co
z)ugTo$Cy5HiB13U@VZE;tdi@_AVWd+y{l}~)l+XA6;s!0a}CriaxGPgjMb4|%wX|I
zV_J{!#-oonv~3p?$@Nk6ytzAJ?-jQlK34Z8FiZ?Do4HoXw{0myL*%=i+v1ACx*3+<
zcKYA7X5F-F`l<h#M02wOWm05I3)o6`7k-uK4Sau*(Y^od-aV2+x=ey5r#>)SZmIU<
ztF95Vj@lD0v*giQr|y*3JSm^6m@{4cb-$RTYq~zQpK~WtDAHuYvbU43ymZ+Xvf@?v
zWv9}RHJ%w279qiZ{ipuZTwQ;rt1GgLRdl_GUgX|eVR0WW>8uwi@ZFl_Dt+#$^4Z+e
z(T|(LcPzMMSy&lhl#}7Q`_@Cr)2fP$o<GH#ue3;;Zp>5vJu52j!_w(XQ(J<pkIX$+
z=01P^MrXyX60J^p*A72O=m}KyoWgM9p&DaH63^=f4(BSx$yEu3y!SKtzb<eVl&kZx
zSZ3UD@Wc)7vi}>BWY<W@FZ5G8CfBq^zmVtgtZUZ=pFg~4Q8VX+{+o@h7w#<cS)~{_
zGsakI^9OxrrWXR@N6aS11hz0|%;vha;(*!DH61z}XE<WsO%qGGn5$H_I576grB=TS
zs>(LA9x$<b%w)LJA1XBADevBIJNJ2HD!mU9daA~xw;@1NE#&~G-@K-!)1E9<joH|m
zRyJu@#9jVr@rp~{+;G?veY3Ob+2W(sP3BzhTlrJ#7PTHbby!1lQ_|c;E7)yW7IZ!f
zTq4G~@Oa9c<MUfmg8h<C@SQuhL*q;4lbT|sjHTNa&-woLxLEX$&WB5WPF|bJGO7La
zRk51XX>3_jW?hU-jknTl*j;P%KT=L~!G0g(*}gFoX5Kk8$2(eT{`3q3tuD^Q0D~I>
zr$2m}`R-BA(<KKEty-(6l~y&^ElXnRs)=f*i$5IotWjg=dfDD1${6K&&6zt;=7_KO
zYj#_aOB$aIj-;*8;6HOrU|(s+!4jj!MGSWy_kVl9X|tH){E`y$w$PY0eJ-atWQ>(2
z*Ol-ru(dvS(t^q4XTTX{vB;#_dEcijw<_KA?L(_=PGEP?$4fEux&<ZUgjaVin6iQ?
zVQRC$OjEbMjEOQ_r`n|?Jv+Tp)pJf~L@qEgmp<Cdt#>Or;*p!VOlGTaOEPy)LCW^!
zH_FtTs!wTKM)@|#zL>tD=K5}>>)sbmio{v`pLxd6TH5No;Y!{af^#{2+vC=IZr*aC
zx4-$<lRe8f<oh?jIxes}VntwwlhNJP?)Bk{T<e)4YJx-<EZBnt9<MBx-I8$hgOJaj
zduyG9ll}(B{-3#TMw(K8y%1+e5&MM+XEXaNuiUNk3GWV@;uliI+0$)oWI2D@B8~QD
zjZ8h3yPH`Bmb?{xz$j^<zTRYe-nm^@<D-Oj^7%a}YUAW$H`h3{{)ULw#B3i<)7z5D
z8v~fU5`B3D!#x;9UQSqk^?IPO(iK)oO(y<>{F3ZJR{2sh#9mAAyL9c~GL73Iwn&22
z_R@=aach!Ac1Pcg)V}Vf(-q2)Tye!pO4si3PvwBgGtd2Zd2x^{?_WV*(WH3}pQapn
ze)Z$we%{6I^B&J>iwt9!zV}V|t%QSNI~=bG7qZUC)3rDEVVt;LFn5lU%6j)Sj~tP|
zhPjM8Sou6Y=^Wz=b9+~|eZiJV1#z0qTE<06a+$kN9JD=Me4Bg8ialOu7Fk}}$~XP4
z9B;70{Z=*UQ&)u!PC4y8`{y+=r`wO^avNt)JM~KHuyI%E5Az5AHq3nIVfNwc7iSxm
zoB8&eq+A0pB$dBe;l6~a$8hbWmfq6PewRbv{RHfv`fqD~Q}b$Kt7qq&E8?79?PrhA
zdFUdlywq90+wydm`0CTk*Ga^#E!S&^=2&n~YGX6Qn}Rr_HR4+F_dF-Y{pYyI;*-c7
zqd1#k`dq=)YcAdR**7`Y`J_&Idx@^Yw$rzpj1HRS9&CD;^r_VS#gn|L+B2pE7S)uT
zl03V`nL*<T&*TOv)>WR`ewTSp?elP!(w(wK#rpH34{Ue%?c|TFdsEnd>VnnFJ-4=B
zEPVRsK={VK2|mA6jbCqY5t6?8Ty&a}ZS>FcC1S^_z6595d9UbjJvZUxtL2-fr%rdA
zE|YD(;}Ea^#-p-EPG1(EUhjUyU3SfmHBvW4<jPh(EK+)VC~DK*Z(BD_eIa7EB}X<}
zWwLzkf;-oYVif$3<d)8i{#oxTw)238Hh0as2bb=ZZdfDR>@#h${z50qYg=`XmIOaI
zD*8C|zh%tCq_>meB%2sqjcVV1wPOBkE9K~a?;(GClFYX$vTaQo2bH<<9to5s8RtEU
z7q}?kvBmoF;Y@{RRd1Jawur?ung%dTRcGEPKjD)6qC-i6pOQ?)lLHh>5*$n1whKF3
z2p(BpBiz^`oLG{lz{qDBVz)`l#Eng4YBF<+nZZPz7{lP&#(Fz3k*ta4%q-#s2SpQC
z2s%Gg7QG_W`Z1+@qpodIYQc5KcnQ(aYr1M7{4*yynQu&8AE^IbN&5Sz5=lpCVI$L)
zj*!nnj6nyu@}>#Ccvx7LYLqtB?Qvo{_s7sbAJc2TG=;E=E*48txm4w7n7G#<)MknN
z?)1o|L0X#M{)=p2%RZuB^U<+n=Zt!(jf`9hAs4SJTvSx_G+-28s<`^7hisGL`m5qH
z6LS79&dGB2v`bRf-eewIpj5d~S=H5i*HYz+ZS~IIaxYygz3e3Iz-YQimG$|yZiWuM
zNe^4!E)#5-==1cE?iVqUWQM>iOIU&(?Avz;UsEx-$(Z#y(dGF?w;u5#MMh228+|gR
z)@vBTziyD{PAz^gso0V|S>$5z{+5<qFIt?%MUHP5cAJsE{X)QxZ=(L}6O5i{xd!T5
zOv{hXFgx{iLc+$FjD>ZJSVU8Ph$ac<eiNRMp=RnG=%lSGq0iVZ&mvmrp|ow0Ww2U%
zuOr)nLwdE}Rg;XO_DlS4v@p?^4p3aVku_na;tIyv<`eqnY7Bt}Z2LckJ2Y376d7z#
z4=6lYCD1&5{qam*XaAG~TpQSGwp!F2Ef+f}F4CqczwMfMSL7t?#u?7;Y+MD?mMYe^
zideTU5n1-N#%H5qoUz#kQPEATbs`UDMw;u`Pm6xHe6qiyn5Ab~ov_I9qq&bOCI@^h
zx#kq>KXG>9r}_gedPhtIXJ3$DNNTWCm7LnB@!V9<lwJS#k8JP4o{d4;NoAswc53}F
z)@SkTvb)lBKG;tCx%DSkN!9D(`^yERHn9H4m~+Bius?IsY4-W9>ho20wm5zlb^AW=
zQ)SD=g)_DXun8}iWP7RP(wCa~k^g6J7xg+{w!m@X<aH|r3lz&<E*JYYUAb6^QS0mU
z%0}JxgRRz#?KdXreR|mXu&G=`ajMk9_E$o#THo}R9B}&Y)X|xo=6X^}x<S!1Ve!@S
z#et8VbUmeVRtl*s6XZ}XzMAa5Xov8r;0bfK&5alAQaL2JZvkgY0)N?o8STYOxp%4U
zde;5l$z$I&<pXNbJuS(nG#H`}_blLye!}iBT|}O*ODyG}s8&<2>Q^(zO3@n@UUwu!
z>o+fW%qH!ykb&{RB%=^*sg<+${Or5GX@z5;>a1eD7RNa~!4iy{8a}tgdATPaGGbl9
zq`y8;`^#gi4=nPM&y<&MlM}yYxBZZyzn1zphyQFr3EZ~f^Zh5zw|1QW<)-qN%t^a`
ziSCmU-M&lo+R3<LrP>Xjx_79y`7bOr`MF^I$~A9)l%ANhz{YZI;YQ&KDRuW_T>T#<
zViv00+Z1b^sO@LDPHN*6@0-e+r@UVuDtCObNFYhyS3NH*I7+%f*!>{qsS})hqKg9;
zvd(UD2{>wePH^exiR#+dI%h90bX1w{Sv8%<d3wOcs_ab$Yc?>hEntt75|!Z=mHREC
z@LN>w_l$nemcqnZeHH!1r?k8T6@wOtp0LPC+f}>TQ}EvLo``boG{)SNa)+1O>#DkP
zMNh?SsMOwaGma}M?}4hrgvZKWnX2ZBTMt&w`S2{*Z_ym@N&oGGEY!Cq&6RRu3%)qj
zT9Gv>fjj-mT+xemb2v07r}@2J(j2waFv^wxYlENhjiNc%l%GX*eM;ots=%7-u=;?O
zj!SvX@#`%jjyqa@i*!nh_DGBPYphaZ)fQebE#+47IVp#ECJr&nmu?SL@`zTl)O6G+
zvdAuXoL6l5O+vncqdb9eYB-Zv)d~r%Z<>9=>s>E(Y?`Q++?KY5sqti0huhc1jMf^~
zf|=Z_-RhXC4o}Wp>g*cl>Uu6Z^Z3psE7iM}ONiQDR=y*>u_csW+JP%9Yw26fNs<R=
z$VBh6PtSHy-!z|FY~6+KP~)g#rX1I%)}=Fh%(#_i?A|QBR4i_zS3dK9ofA!7=c^8E
zIH|LUTQtj~HU8S9B)1jYHgfhb$L<cX+_zIW{?m%UG|TzLmXjxL^$D_CyJ72x(+VwD
z)UHR{{5F;2Zxl4!D8OTG+~t(`v!$_FO~CB4-0vyd8NL=-AKac`ctq(j_tqrt&98Qx
zD&O&}dk52ojZW=5))laCayuICwd2sP1wSN9Vm7Qy7d&Vm<h{WuCvwI9@QI=y4=#Tj
z5#+FXayrLa<x(+MQ)kDAZFf&aa&lOevHM7btZO=We7)N4N}KKvMS41gQyqVpXg|^a
zUL16&DP$sttg-XqIFazA51Ic{T^g0R4ll0!HzAa3V(97*>0-{3_inojtv(f?>b9c&
ze^rO7jMStfVXIbuZJWPt!@kX7Y^e%6$`ZJ953s5>oZc3|6db^|&0(M1?|tcxo7!KV
zv7WfkIh-wsLE9yA&9_?}*Z*ksypigDVtOKCb63eS^Vf<`&v=A6)=9f6oqIORapvTs
ztYUmJv!@(9;3qD>V`jbn#@YUlW_J`W^68een<}2$ARHNeXinjwzsB1Vq|!bcSsytp
z*tUtwcZvTOlfvFd>0ceTOTA6+b+YEYc=*{A+lt%vt~%<;8(3}KkA9men0J6R<p67H
z0N3^hT&V`E=S~RA9AGUEU^B~F`fn#=jKk##w|7jG7V*wm?e$}0Kp+=`)UkBO^-_)t
zF5jFvfA!j%UH=a)@GeVqm*mqrAHR{cp)=U$<wA|FbxKcm&6s@6F);PkC4twQns+Md
zU-<51cICppun@sD*Oy!knY3e0?YH!|8*RTf$j&U&P+DQwwMb*)cmJrAqBT+?zEbjQ
zcclb9U^^irk~$$L^tz&(Ib%$~zL*WBc?Q!CugR7vII5Dgsc_S+_=VXQ)YltpsS62d
z--|hxzGhj5)wvy)l20GyRC>C3kxR@g_j)()zOQSR>ds`jeAWK<o_U?C4)$yH`(6`@
zx+E|o@q#2{h<5j(crUiPO~Q+9B+btV@T%E~g$q7BS(NS=;AnI6&dUi)jtQ()U^U)*
z>GX59`ER*Q-m;}A)Z`j4rf&Fuf1LxHkir8M1I9z;+-nao3V*olc6WxLoP3mz;GwKZ
zKO@<^+dH;9Eohmw|Ick@v9B}FT3KA2bye-$&ZNK~JImYJo?I)r^Zkk%d^8(c6YoA+
zrR%p-V$E(#wOcxAwF(B)PQ2D+T{f}fVqyr-TtSbc;_c#$vkQcntBoE#R=crLK(ty$
z?qfL9(<O-}CZS*V&TOjAPm5gtqT0Lg%<J2t^WUD{9M-*d1Ecv_R*T-#i(=Wz8_vW`
z*qC*1W9`G!2dg%H?A+&Wkvz$=H7;}J6{A}1EoV6wh|Hd_|8&H;FmaKG)6E<wp40GK
zCslg2V)61rTiforZ3$?+laq1h;=;5EGrZqk{a-IxBNnh);=byfIjLKH9(k*M@%|#4
z*L*$M^WB9HtVye0t$LerN#SD%5A(eRMp|Bz#3gTXElpUw`PFI$KPFrA=jO3&Wd_q?
zF0iF8xN_wlm&x4+hprz9+<3%gt)R)?`E_=aL>{tDu0Ff3&ilO4u_cO||5cZUG%C*8
zJz4eM%b&;1*-RYFA|HK}QNJc+>B(kw@#3q-HS5;zR$2QYZ`#Id(r36F*XjL9w9>r7
z)p#V8yGcr#PxNMtw9d>Y?Ha5(+bX^&E_vc2JE_(^P_fv<>sjE{tfimYHKb||r)PJq
zT)p({^Ene2pLoNiVj#NBj5W96rkW?yNweqC3TIxtQ&zog-*xgoW5nUzyFV&B8tSf?
z$iUdJfBW008MhUqn-%YcPmZ4P(z@HBV3uIh=G@oMb&?n@T->~BmA~Jdmfy~yE3~2B
zM_Z@l`1y73?l69moMYU7uPOG9lElr}L*Lcjo4-^z=3R8i=uLuS(m_L+x+Lz?_nHG1
z70*_+-^sDbNxg(sk73I!!7>Ln%TuS5J=wNh__bi~CAF7qd)~aA;`p$kZ}qPCzr#Ns
zwHCbkiBW}9;3ZpB;0bw8#VXNrTwdz`>TYly$iE}L-kDA8$m8bsj{eVd**Q)WeqZ=>
zmdZTiMLhW@48Hj|&QNr#Gj-Y|y!pu2clBx8PD=Gh7|AL}xN8}ec;1v}6!^>XpF<?z
z!UM;ae_UE$o}ZrQ9Q{@4nvg@F!|LW-mMxi2mnl0nZ*yv5$!uS8&`3DyW?|~51+$G*
zb`;!jS~NwPQSNSsO678e^OGH0`CK`stk9n1!M{x;z^#i#RYA1DM`(-I)n+axl?g@*
zA55Dp-PC0JGW3ARMz-Y>o(OHpJ-D6y@iEgM)4F4KF<D7Ixx$*F$vB6FA#e9|jlhHL
zqWW$gtEMI&o2Z(ubK;f4%)ljX69o_KRFDi}zbSD<B1(C6*fs;PyOR_e8&*$F<*uzY
z=n@P2$8t^i<ZcDWmVM1!?Ej(@9pk_Lx%&BPUy^Q^nDgRgc^ehn-YDdYiaxmX;M;uV
z*1U4%{|kGy=d&Ly&wm`dBCz=-|N80m0d;?W+!ycOqo43wcgd7ZOQ-M(O<1ttR+?~v
zR!dwlFN5Y13l1GFRt4^b+@dO-U7M|9xE1xbWn{ZF+gvW#cEq7aM{mEMk&p8_i>#xY
zdOa?BC{J(`_BwvFLSz+hkel(ujb;&yXOD!bWVJ53dP`B6lXHPd$3bE4E?o^R-YzXI
zPSz7=<y$$d-feI7kTO<eloC3i$oOXF3Z+STIjzd(Oi%P|CO(MPTKTuFD3!_R&4$*c
z)~pvk8!Q!gDy*{L!Di#YWp|9Ph<RzpZW8S=k?Q3Qvq*LB)({OyTBYOPqua?aWx@h&
z9##YIhAkdlUqk<IjMMFC)+{j9IDFG>k9Wh)4@_k%#DZDgxQc(3%DLGb$FjzYcY6a*
z)PxM4EYpQ+Of|Ngce@$Z;LD<L)J4JYkxp<}i1v$}U1ps^3xih9eWRu67P)SF;~AR{
z#b<M(h5Wh$n$IZZY|Q<luwa|@jwMI;Og?Yz$z8-B5YBMqy+X(5;5SK(a&b9(n=S>v
z+TX=m;%Yi2o24lAcu@2vv5Aw4J47{vTf}dvh%h8v)m*Zppj+>$?5o3k{=X)0FA8N@
z>B=}m?AMV!Tv~?}mYp_Gc3mm3MN*M-+kucfO$ArJ88z43S@C9pMRIZsW6Pdxm$&Vd
z6;jD_T`hRB%;k{CZFSLxqAUL=Zxegvb#j5xidD-ODJ)hr5NEn9Zkk(o*>}cco7>8L
zECK<KzpK>!Dt0cFowCt6HCQY8tgZaXvtHRdOY(E9Ra-@GElyrOt*YR?t3`94bg)9%
zk`p^utq`*jHJij?f6t43VM^%6`JPI%n8Uf9UTe45UHPiArd4ds6g`WXQa3&<W;^-k
z-Nevs3SUiC=Dcp2qQHE4SL?BFR+Dsg|6kOcv8AW?OhvYE@oKUE7K;UqmT2dH-^{8R
z^I7i#?>YtcSqmN<P&HOvyIN^lVNdcRX7x*RQWEcSo~rV>A>8b8#$Ls8VM^2vrv%>x
zybM}*Qyji+d_TpfSmW@tUaN?8FI?13SrhsG=f6>8xHqTMaBs&1R&il{#()Ax)|*~J
zY6^-vTuYs9SaW_&bKbI5g;!;c;l&?m$;TGXIA8NA>T%)<mBwkJnmM_Pll%Na_D`O1
zHfHWa<%vB<w!MCG*7DEqGxs$j_N<A>YzlIU`@^zfsk@p`7l%+ov(N)`t=t1{#kWq(
zF-TGu>|4^tA?%+bv{WHrnV)m6!TA?WMH(KxDnU!uOnKt+XjgYai~F2ab5q$mt?&Fy
z^7RUN6wqZn#n44DgxhHei}GX!2A>Vuj>aBqBp!?Ag!zYOFFf?k;((%F){1I&&z{S(
zf&@NZO^s+wd~PB5NvCDWirP(_4l7rLXmNQZ=1-KIo%HK}=Y+qZYPrl`t|gYdTzYN^
zxAnEm1uQ%+Cl`85Y}#6=;lwmy>RO*pjbKL^e%_`j4MwY`$XH+G4K0^w*KJ)gai^;`
zL#m<BJQ4f9flqaF7#CdG=@UCA<z~-<VD9b{ZxlRTP8f=^$UHJ}<2F4V<*4@a3`31Z
zckCn4z(vay4Oe|O<}gfW*?Y@vr-|=Gg^e@MPI<6(y(Ejf^TUbDUSw=DR7zPJefL$*
zo-JlZVoMgd+!Rr;Ub%fso(d=DkLYQ7madB|Ix5?3eT=uNFmP%nGHZK2owvu!YhqJg
zz~U|E1MV2G3gw6?E?=DLn!|WyOV;Y?ObN$#6&>Hbpy#E}A(wOc4VSt9H(wA_+-1v|
z$Nea}`N|{BBf__`S1~O;`0I_Poi>N*;UM2g#UnZ{-EyUOnbxlB3e9>R{M^Rz)853r
z9<}#OIc!9qY(5@SW>l=2a+B$XE@KLtqK($syCSR9gDgavBt#u-mwRZj?me#+JjH45
z2j(eL#8PavOf-3Tk6M0x;cl(LGc&;M&?GkZ3LCbCCuRMf&1ZPJ()y-m2Y;=WOZ1bq
zVq2c_=A2w|Z~4_^?h~dR!m5*(&T$RqvNiS)D^Fb*)@s~zP1U=@<I1!*hkj(LESe_r
zj9q2J!n5vova(a2ZPVPQ)Y8zdDHP|ad?IQ~vT5<Dn=95itup>EuF}DBM_F^n9uKAe
zVs*kMCs@23S2WG}w76SKB;9qA*SRhe2SM*Cx(Q9%d~2SZmlEO>SJPhnud-8g?cTfD
z<?J!{q=HXwHq6@h$+RoA>(xw-#?PU8yH<rdhF#qd)nocP#&%=J(Qc+2_ttK*lHARp
zomH0@weGy;);%SQc=B&<a{OlGe^6!iJiomw=7>I+r6Cs{rFgd_V9nR_7k+5OM6qSa
z{(P}3nrn*auc_CA^KF)XR;VyESz5p)?KY`ns^dMmZxRg&N$kBxT-shZo(~A&m}KVB
zsH~C9C-}AX%E=uwE~~sOIsCCysCrA&N<S0xWQGHx3qJ1H$+c@$)0*zR{1X?Rc&$>o
zb+w83t(+@f|AlV2JX_<M+p^%8!vAV*C&wmTooMbx)m4HwyKF@iR_aa{<jg7N<Q5II
z-g?Y=qhZ)FDG@!-1g#|roz-7-W?!n@!LxS9;RSv#g_d=+tFo1`a;havUX|a%ooevK
zN8`4L11HycuApAFsEZa&FU&4GeayADiale}xTJ8a=&5`B%rR}ocea1+5Mq%$a^vwN
zt8=|e{zM21Foeu!?M^9t7Z4LG5bkfTyN4+`z(UaUiZHicm!w%VldxUdO5u4eik@9h
zG^O3jmc3AE+}U))+WlycoAHBBXBfO|CS14@9>eLgabAQXEA!E1FV(_#F`YQ@{#VRj
z75zQsM|mE#=}tK)nrrcYH{a(s58w9Ks;d?s_BnKaB5zItS89QC)vX1qf9E%<+9}#|
zp7Ec;mMv1diTy5vjl^q{SHV2345G`rdb+IVFjw*}Sh?!(iV3<*o6bKJE#T5N@p9ll
zGD+hRi%+8D?#9(8ZY&jEGP}vZanXa01;00m6wftmUb9$xp7}1xrHhSE25&ya%&A$_
zz3Q^@f(!f3FzrA2YS+5m96}n*))}q551Qs?99ntPYT5+OnG8nJ8O^#67%ERs{rLIN
zmh3Kd8}4o1PU<a(Hcak`Xc7%-khwd7xBd5l!xwZP7}(rf#cmX^Y|cg@w$44x0rQ?H
zc6Q(1xcr8NlStS3$$G9nN1_>b%13XS{J*37V`aA<!|Ia(+mf~QeQi3@Jtw5SUhm>0
z*Z5lD<cqy0G-mucIloD1aRFy<ZuIP&$Md&7;#t6D{`v;fnPv@36;8H?O_N`4XDK$g
z(%H{%yu0^>oppqD_zxYv4|cOOnsrqc@@h0&Yc#9xVb+_$pqJ4+a|T2Ig9gqU`$cAP
zZqZ<#v8g#`=EAyQf%!XFbtRae2JBdP$z(!+#lsE9Prov0b+%|tVAZWy5l~_#!PvlQ
z!}**`Y897JYfn$~<5k-mj%`?^z4Yc$lOo%b6T~*&6cc;1Enjov#e&uMyJw5OQMN0R
zJ~iP)-=1mWJhQK?XkR3}PU?)%@l%eL)*UR3#<OjDoc?c_U^UZPkjdKdq5V0H=Fc9-
zIBy)%t#F!lgW+#6>#~?r-?bY#Gu&bdo1+tsZJO<6q|sRHb?oG-T{evS7f;#Ur)jNG
zvh?Hxk%`T7;siSC9Bi5y^mcApyW<b<-pkWXChD?$*}}|X^Dt_e+pHxsWelEq>dn)1
zS+G%Vs{&_q<&m&T(Wpi}o*17f<DGjdy8V8L7FBh>%jCYr+#|B_NMTiPcaYA}E;F`|
zvo0K1?E8DUeBt_Cl9LK=YPZ~4{Fr;Qb7t?~MJE>AV5plcV!gn=@^t5n8%|jzty3m&
z7TEMZiQ#nTo}%m1`=X?oGr`N~0h17ii2G-L^A%p%G2IVSI;Sj<tNQ=j&S=GCp5_fF
zH?SJrXej&9UT1JXtI}EN6?@f&^K}7zbt_s!B{+UPIa?*u%obx6SF(8aMadIhXKudO
z!M$dZ0n?^cpSZoZ9En{L;3{M=NrKrYkyXQbwkVH~mbKm1pqW3pgccR}E@a?6na~{e
z!Dw0!2cy8|)~wz;yVV<)cy@LzvTxztKXcD487|ftO&T^!vuwO(PO#TgVbsrI*5AX-
zbdzbujRuE}tws#G;R`ygUQW=S+awfmEXbl?XvTiWn|_lQaPnnz*JU&Rir!*l$T>e|
z<;@dLwM@eMZ=Y_~=rcU1`|AyB)r!_1A{<o_tof25VF7$)0?cI$lmGA0Ijn6H@PD$9
ziS?YL2QJLnvq<}rqkQyM;g4Mr&Kxe6=W%8pc^$R)z0y^WHQi=jMhg}6i@3UcY<h03
znEoZnm*MQRZ8yyd1jWT#dbUdO`f!@hmS%2Pa#Cw%gXRM!))gAZ7z`pbS!bN;__VwA
zIg3C+PmtaQ&5|b#W(iAQ?`Ac+;I@8=n~?%@){?E8js4~;g!Av-E)vyQzS>|y!{z==
z*38E9<}Mb-m8XCAobUP|_V>%AKmpb}7ki^Fw1n>PuS#I`UC?^xVvAGkl_=egj_R(<
zJSW(DR;&%sQ?%}A{?m1inUm|!qBn-2=O(zU{k65Dmxo28W2z{F%7l=<+D!s)FHH7W
zKW{fj-v67HbGQ}HH%RA9R9=`cdw#X=!rdGad*?^pU^!H!a(EVx+2f`wKPDSh>nuz#
zKB48c^n-!n6DG%%mxLrPpJdwa!tJ-TM00({dCd&wXpKf)3;wwZil%e5m0AsyM7S!C
z&L}$^%^fKdU*P><RqO4am*XPX!zxY(3N(j)u#UI5;_~#msX(h?1gm{$hoMC4k1y<1
zEAHIA7*l7;!E(o%MQ7Q5YaYq&Cb_MScf92e?XsCyc;-mTy58Q{FyXF9=I&!t<^@jW
z<xX8*dPr)6(X<b@jvO{ve`MCS1-_9xg{GesxOnV_SkD5%Z3lfPDKxkLI>>N)>d%d=
zU7-RFn*OY(J6F2&fBXN0iC?*yS#tkMms2{i)<w4Gy9%4@iw*ib&sQAfY}s*p%N2g3
z3r$QmYGD%0fe8_3cyIg(xpC_2?x+(jp%>2A3AB1jw}xsQ2$X2Py|IU(&p%M2wchof
z(M^^Yo9{KOvFo+!l$X)rZj4=c;tChbT=v?4yxXySa-j+W4H>sCO7`_m+swnaRZuW?
zw%+c@iwmx9F1)zlhKI0;=dlL?ljg|Z@!TjG67^`;MZvtyhZUNYZ#gH;O_Fu#ym-)f
ze!{7_8z$)S`Uw?;pW$55EqO>njq{unmzN@|ZUCQ^ZAX*<tKp6ZbryF_EUyH<NWQc4
z@wPAQWfIN5Wu6>MWjztw>iOB9@qYws;0bSQR@t9E9X3Z)jxf1AZn}9?;z;PrD_ovi
z`{(xFp6a8=H8F1G-W|6#?U0CCSfKvR&}?6a`n}MeslH7tMy{`%U87kQ&VLZwIaA;1
z7E9QJY4NNJg$xB!)-7S0%ImkGRX5<8x)n!m%{7rl`$dg9rL&F)S#>6)m@k?Tp1(}2
zjN2mmM{vu6_6FA!l|`+scm3OWLmJN}R7LQ$&NXSZjc|0k^VDMU|K5b+I;mO{j>?{$
zOIK;k-WhSMVHpQ!<ndQGPCgN3o7#IRxtk~Qsp+)ot1==jq#U>W3}|3}B=|LB-e&1P
zJNc9?XYagdX{V&1XxDLE<JOf8>@i$=>`L>h|2Hjq*)6X9PP$>qq?eBk>YA_j6>guv
zY0}m>?T4*qo>+QsChP9v!`ovzqFy|%a!3h_U^k3te=5LydTsl0)`Td7n040@InK!j
zdfeH!^l>Q1>x0o23nlJu_Y;o3&{FOgo2tgexLe5IW_8OI2EAQZUIk8ww7j|KV65J(
zt<4{l!sB`_NGxBW$$I|4EWSH>Edpj!cH|tg>*2Vd{?s`x_)5nOk-&2btyho8he@>Z
zv2K6ayRTDdmTo~>Mwq4GJ*F?acL`bv{)y%Jz4XMF$xLQH(i_8WR2jSuEVxmXkx;WP
z%eNxM@!jLvb?i|Rtalc+{<B#owmYMF13Tkg->Iv2RIlP<_#eQ~tI8?hcEY7+>V(#5
z?5y2ul8oM@^-Q<rw0Bv(>7tm-MnTnk)1(i|3&c7~-SqfhwtC{0#pge)J{s^yF;b{=
z!o{l!tosdRcAJTwuu}1qdMF>tz?yMP!AJJThSqi7uOgDT=KT0DZAJ9L3;UcKmtL$=
zbv-yCupw*Tkypo-vRccv-Yrb|zvfjSNA|Tv9S56FpXf+g#9*_d?8>p<fvI-$UY>r!
z)~WPtk$~<ECe7=Cq1>rAPb|xEdJ~g*Lew(!XhJIM%$#=%pRSwenBjPl<JbJN5pl71
zx9yxYE%3tpK#jENdD1Cti5EV;TQGyE@QK8XfZ$Eh2As|vQ!M(6?`*lVPAL8Vy(V6<
z3@Pyke|tWgyu4iPqLg*;`eBAXiznX4KJ-R)G;yqmP_dub6B;mqV;1*Ci%A=bPR;RY
zQ{f5@+xwlZbDC+#k7*(5w*zDrn#F&;FiC~iB!2eA1#@Rgf9U;@FI@Y1kJi!n{T)#g
zUe-rW|D&bj^66#8#s%}2-O6_AcP&olU(A`nn7l<N>uR7$UW(WM-yas*z50DNy1f5-
zX)w>XJ55msb}u|MxB0``ZQt(HoY$>?UQ(yddF#@bhUcd}uL&5eY;Wr0I<{%!o@*(>
zeM{!v^who3w4R}x*;3%#<mt!e=q>u7WE#}7A~NUgsk^$yi&cWR-SlA6RA7DG5fLL7
zc=iC#lmDwLFU^>%8!`KGST64_zEH_9zIj}n4*p3~q&RO_y}A;{z<JI{fw}L%O5Gjj
zj9WRwA3D9a{pfr9?ZO%HzyI<F-Czo9@UNOvddy;(VdwFu8(O*QdM+F&o|O_NBU5VS
z-JqaxVq$0$|IZ07cbYo)>AZfV#(2YFlJIBW=EP$wN~gcwxPIZp2{E%b%$%0Zyial4
z)yWRUyBUj52L|5t7E-^ryzsa5O*<YNc7y6$8dERa3clpD{)W}X%>C+1IR7g@?kh0K
zvT@$2U1)dP)#)*x?Zrd-g{Q4`D|l~~JkXykKAE>IL8hZ%@*Tm=_DapZ6F&96pH%VD
z`8#7L!-tug24B~osNnv8vxPSxv2vpt-@~Gl0Y4+c)IU2`P6`NJ_~O7xKlNiCO@<0b
z`LCq*ZP~VsQ{lz+&;|hqFNT9nOi_Y&j=XxXqIsKun9B{Xt|k^C4lRQSgT`Zha`stg
zZcJ1@)|VS|S8&3n1W7(2izyoqr#jEHHa>YL(>2XWGCc3Z6vY#lpR@3-eYb;YWr^nH
zWqz~oWo=EJvcK-1-8bzSQy*Mg_H|3uAFqHl44Yyc)Ar7~7H_+4->a)xq1_^@15?a;
zFK$xp64Q*DGs#5blV&2LCkumsLMpe1W*6&(moIKS5INSWFa0FwLXrE~S)6P~b*f4(
zA7N^kU^B<%(26i_@v5B56Q|}(X37%&zeKlzSJP}a_ou!!g%2Eq4(0#XJE8Td@DPXc
zZ#8A{b$6Ny)z^eAa_~HOkd3)nob#UY_esiAy$)oa?sJT0I^`1(pQO>w$#m`T^G9vm
zXXZ=qxbcvSy=UQ;xS%D99#Yk^YfJ8nMc;UQJ~TZxO|+HOcg6z!e}5kL>wl}s@V4Ey
z<#CkuvD{6rT|qu(GZR^5mxc5&Wn7naY!ozk?EW-!NpinT?uHo*b9h(Ya1~nOF`?T_
zC{5mReyK_8<X{umC2j$Ft*MCtS(bC$=azIPh%DJsl+={o$0qH#c)}%t0v28u8J82D
zSGua%uPQmrb6eN8Y`$XSvs1e4Qdf?#sB(B6tKa_r3`;`Lx1EknEU8lzHQDoS9AWfr
z`gc<^bNhxRt@>hByF$XVIUj^Y7i_$AN~3h*CfB&=AEK=cfjTP#*QGy-Smnr|vf;)$
zFE7KRNz(#PN=#*u{4JOqHqEQm?X;8Rto7H$SQrHMOSQBlN=@JllxE&473i|-tmlM$
z)`?2DJ0A;OVas`~=OW{AWNq|>oP&;GGjdYCutly`mJ3mHiB2gGbeF6&pXFgXOWolx
zo96_<hS!2qrc|f?dEQnXJmEp`VwD`hB0rS^N2aBYJFj<1d+ywnP|bR?X$nJ&kXFMP
z@1u7D&zo15EO$4M-m;*2-Ho6wv9}%(Q{9#~P1Q>Dt(Q=WQT;zH^Nxm=R`!G}eY?^n
z?!v}TmHQhiudH_TQekLdnJReXrka~y+4SXpTpJu6t6raFoO)Wy<iY%jtt%q~&VJ1n
zc+}Dwwj(%dhnm2)bBW((Ig}mLlqqEH)xNlzT`*<2%fkEWu1t*+Z7q*7b?7p>uP<&C
z@{-yaD6)*HpkcCq>K4Z~X{Lr%7vJ!%TDR-0*4a9F$L%*BxrwaxxG~i+jdOzNmFU=`
zVft&XU3#^;EOS$9Tn%sZHh1G+KkmA(ye(1qk*UCCwfnS<KOQ_{V)!R<=ZqMO#g-+1
z{TX675@y`HuF3M`gh6sf>z*sA`n7vIi`X9e?ozNg*vI0-kkyrtIInrmg@gZH{iSX;
zxhibiu%+{ewWbb7zlZ{3$JF+m<;tFa-XtuTyK@om+K3D1OP(!fdz+(aq&!c+y_h}P
zK;MC}sQGkMU|Z0XMs6X6)|Q%&6W`=!I4cPS>UwRF5wX>1;Cyl}@!T>&<0QAl3>ME{
zCX2Q$-D22h;B_FeNni~-=L3CDr4E)Q1%m5DT@u+e<%79i>_4E{>g~5P-jQpDM^nKf
zZrPVDD%MJ#*^*1R=Vowa`=mI;pG<3>klArfj>D>DhjvN%D!+L-8(X_H-t@K#-geV^
zG?9Trfr0D7QoYoZjE<`B%M~&O1gC$WIH%HO22;Q?#~CZ9B&}l6I^cZI`Aq<q`3F9Y
z|9@s4VpCgS{@Z7PP09-Q_D$ja+m9@*F5P*0O_nNawCys3+&?MqlMd_?{&X=hnL9=3
zmYa6QU)6rr&Yon3^!fWAo>vaJbbE7G$f|X@2aUgc5#8`0eL}8gCvWPi=}#P%AFMcg
zdU2Lkxckko7cEwD`~9@a^z!|6urcYGC4Ua<!B;K<Mk+o@hEpUQ&t&*_{{5nCTr=18
z`Aq-TDjf$_M>WTmXPcLby`ABZ{VHUkR_GkT7F$Q16S{pBM^t~cGzL65@3TZj_4tp8
zrM97q*EKO8Q`z{!(0doh;akTqSL?Y>a1VVQ=f3$E%e>?6`6gOv?{@6p!I>ajyXOSU
zBo&V+At99t<Nq8M+=m+bEsTtNReS@iP78S`^-Y|k;GxQTq4nadZ?<z2xrDYI=(3$z
zw1^|BBZut}o4F=mQ>qZdGX<qx?Gy4mHk?^~B*S9@LqW>)(xck`AAe5d+`_1N;m?V%
z-UxvVq2d|7vR4*NNNP7r*jyf{a#-xoHpW)VldY3<Po%IHOH{aB(|pk!c*~4QDALGv
z{U%*e1`!3FMj;pO<ba+HA6B$9O-qm586~!M($fHwr}<Ils+S7;Cg1!k#nNbg>%=y`
z&8>H%^31+(;tf8tWa89E+S}wGo=uGrmb|m&kX+?0;THwF=Dxl1sl?Lj?8_s{a$>Kx
z@AR8FL*!P0bAQNm&8trzhW)ReaWz9uFvQF!DdIMRtI>iS2Wg9JyJne<2YK4uvsGrx
zFnTM0x=@~5^;f&H>CSSwN=f6EQf2A$=CfE@CK?t?r^G03ZavC2X}wR?sg#@A4Vo|4
zD%UBWNL|0>rY@&|*NVS9S>Crq+?rS>=S=DAG<>tX^~bYDjk~;>7U6pLe~9f^vxi?Y
zMm*iWzeT-StHUVU_jPo+%Z!`6QfGTQMWT<KN!|0<N#t+X{Lf+6JhQ65+D$zaV}Ew0
zX7ksn)92rvczoW<L?MTFNh?Gyoc;3AaP|eBdt7JtH;HKH^NS@fzrp+Z^n@eP5>JzN
zm}p4f*<liQa-)oPnAWNL8xouMM6TR-<;wrrWe45jwi^09n)fK_k;^jCw;3}J=G{7P
zclB$}dp6!<Q?dlsc$p?geNFW8irXw5_VZNCjb|+9ejWO@_CoEs)NSq=eiAvS=k!X7
zZ#g#g(6fyD+uakqH(&T@eZMLy#O;1zv;?=2`LQ2`Mp{f?UWv4Y`J2wFe|JqMSZi;j
zuk?}4=ApSaf};i2o4+&eis=$5Tb-1%_peV*)Ma;z=Yc1eMwJ{?ZCzbm>c43z-}?L-
zWl6TDA6E!1d2-R@gx3TPAL|`Fdv40)SMKx-U)Yht^w`qGQ!U|bZ*|uSZ$;^|h3ASd
ze9rQ4l3JLgvzfI?i?JZs;={t7j~A_8J+Dsi*qL|M!VVk$o6CH4QSWH8k6Csr<buLy
zrImY~Y9<suoZTpSg=3#C$MOeqDRG7Doj#@-GEEnMSc-=~4E}ICc-azxGu+C~$7|C*
ztv_}~JyDlA;Dmv~eqRT!XZI$?&NNEPxP0e`M&qF=Gbb=|&9ywrpmdk%(8LQL=PzfN
zJM-@JJ{5=5g0?$4*ODx*t-BzUmNR#kt;D-IoHs&mEs;EuA=tjm(KC}%AhXdWbK?Vd
zj|1(6Q$8#`nYV+pctVhSkDpfN1!tQhJ)-XSRp;_b9T6)L|8SyDDE9J`r99@9lUYK<
zn!a@1Z)AD!_5N0o1sM;*-OhCAL`JiGyZZ8P=ULOE2`@w$B)WfaT#4T!_rKBbm_!Hv
z+gO+RrfxQ}3*A%h&J2*4RdH9n^7M5LPRWXk?hZ#B6j=7Rp1bYn;;~g^PEU9KgXjY;
z=e{z!?=@)OUo+9sL#Fe{(K%ZhUp|>KKVe~Uqs!482hE5`k%ecbZH@WI`XDy3Wt(ZV
z?xJ|HI>}ib51IMqGxNzNe&~?-*`e@l(fPSIe-<8>Ivbww=8?Xba_^Dza-MC6_9X7O
zDii<kUaHSRIlsmSKOQQ#o_ti|nQ3}FJ;Gaj#a#Cci9b1vGY#$s{h2KoColabGU?^)
zeQ&S1Kb0-06uBSS^7H0($y-U~3`^W9<dQk#+$-*FosuNT)fUU^=D4WyS8Q~XPmkoO
z$Nx=}PH+1uF1K^xIu?<?DmQ}8%!{#%cwfm?*?ZSL;zn?S)V7Ll$DAg<yK^-x=O4(8
z-?t$mSYVFB6YsC47ryLW;x>izK*o`dEzUpgw!NGY@KMF#%*;nGR29-TNV+Ac=&$4F
zej4ts@j&12%$B+A%vHW0O&*6hJ;=Cnk!O_$N1x*r%VW&}bKQ(QYa~+?Ymdm;ot#;4
z@Gr;l*35?zQ@lT^df15ZukiG1Ju0^9FPqjzCV_Rcbblo4u-<LsmDH3{aPeE_Fs(&2
zXqj``lVfimJr7-0+_}tc#=Tc2XU;u+)UV=rIoE9>2WPTCNYuxRDJI^rT5kVJU7Sm~
zyqr8+Yp<mT`TWm5<5_5?bdU3yd#A`ZMV9Wr(t8^EinCmJ4{<xOoC%n9MEu~5Bb@<u
zT5Pg==d52782T+H_>X*e)>DUR>XM&AgUzHXyN+Enb&bqYtyz|?ueEq?=ghf^vhFo!
z6D96m_?aG9`7A<8W0vCd>|4oB3{uxGs1+`Zp4{+YVd3KCnQZZgLYCOx;{3JLRWac9
z)~<t<N51*0zi5<t=F+Aqy4>zrTXK>-C+kd?vS-edmW3=j`Rry)W}7Ng-K4&(XES4#
z7EI&nvr}bg33TASF~P+x{s`|{6~}cuI{wU&|92zq^<K54pIu&T^KVX^%X{kige_i0
z_hi{L)61Hsu)TcZZJXq87x3R-js0+9y2}=({6D81RhG`1(&6swxM4|OZPt?Nb<4^P
zqQf^WpFZ>6f<=)bek&{UQU&dnx;;A4ztulLc7>hn<E5uA7+*?n;)$z0tKsqHdC;AS
zCQVPWXFU}2;C$_J*L}f~;4QCi8M=B~&1ws~<*-0Z@{eK~!~E?LEKefRg9VleN<DK}
zvZC<Jl$}c^<sWIi@PyOpz>7B5C#kE>FV}h{dM#<oKL6!hFOM~<R@t;>aYbgeopq>-
zdsV=qXZv(<SkLp;$<Oxk%9&n~_If$N;Mi%^#}ZmE{!Q(QXS$#9B<rKZ{C#UybScVo
z?)Ch%H})$hOV7(T_m+@3VqW_ncF(O!I(pRiKa<MiUd!1THa-3Kp5K~s>gI;2k$s6L
zSM~M8Y4yq_oC<na`)`gum-6XF%TtYdm~QCo_H#HF<PbPvs$)(AQ_ICWR~9pDQSIKN
zdS!1^OzxrB3;h`%dS5-!TBFsv>||C}A>*w=O%E4`4LMFv1lAnZYA=|#Vqb0_b8kqm
zX<BK?G5facpPj3gN3G6$BpQ;o>{M2EqnM)WqqR1#)+#J~xRvYbnWYogFVK1YVp7!F
z)l3^!v*gU&Sm?-E;U*&3BH6-bCU~Uoi=g<MuDeIlT28HT)5&#NDJOSMEAxly5w4XY
z8yu}oCaAWp?TX}6(t5Lg-i@Fonl|^8=UB<SZSxEI_3nj=*#Bd7?@pyX4V>uWdS}M*
zb$*EtPJ|v)THKfBdM#JSQ*G<4H@6=p6|;tNr@oi|so5{}Sn}4JA5FpMKIyQpSuo#&
zVbX(%Zafm*S02urtCdo~l)vuwJp<RumkU!%JlmUCjx0-_c0}EJ*W+znZn>=YpU24O
zsjQxNqm$#JgyaU@y4<*66^GLk#l?9NcCLE9^^UH%Q~`%~bYtDpqNUkxDthOB^`=cZ
zFxe$AJuxVwB203M1GjJs?`;Fek_Lu11^m+;yj2b|*mkcu!IdHs(c<UU^CEoGFG=w&
ztJbp!Jo=LD=H_s{D>E-+LcV4?v+A6$g^%X=#4lVS;@Nm)Uyi`5W2fh^_54@4buv6+
zjdlT(tnA`@Wg8ZpYx!4rh;2{gN2m8o>)u!9scbV*Nj%e0ZF|&)RkP~e>+fa9f82V*
zr16gF#hahsa(I#=L^K^zdEPdCJG#AA!Lf%?(=eAq>{x1M=l>$vM2{(|=~3w(LB9?y
z2y$@{D!8>`>WU7__@)UNPah;WCiLa&xF|K9yQ#O@?MWxg?pq4U`rNupgeEk~N4h+4
zTkoOrX?qeIyT6Eo*gH3|b*~o~xUDEKFH2tH`K~%d=;^G3Mtj{QcQ`mc`M@pZ!Z*8t
zPrOBC?`G~P2W|GZb3QSa^!UKi5*Vk^7pKEH)#IQ|?GoFzbyupo^4XSium|nflGhnm
z_`lOio;gzflViHOX3kr?q&+Xz1y4D7z?G%EEbv9d^WLKu?!3x<<#CYhk7W6Z@N;t)
z<yfwdNqaBlYTR$P%DFGcBj9}xTckl3J7ZYBVB4#pkfq9-SF6m{|FtOJX7^_I7n>PE
zwyxh}oUz7u>5tE+awls~nwWj<@g0}D>ZhOhPCUersHr($I?Y6?zQil-?`zIy*Sv~i
zS(`JK@4c-j?6%M4*o`U{Eyn;u?uW&H7Qa66DD9B0gjS=l^0zHQ>w8Vkn7N-Zv!DIR
z$>i39FXamBcWJGA<dqm$^7ygay0j8U?r83%e+oE5f_ST4!qO+XXUyV#T`U=(XFppk
z+gb0)HTT*L+5cldPIC8Xl;MjgbY_cmp6lq+c<6wzM?urHNqWf(HnDFi-MVE#?CpdD
zbs+}*nK!hOIzs2qE;7mC*f#x++ngV>nJp(&OyBtBC)=v6g$ElpGNrKZxK#2@i__<M
zyv4&A83sj?dRixb#cXA@xLda-sJrdZgR6U$9_v~esmy*KJUNto&NhaWuc6{BZTC0K
ze^S7G`@^X<XW}M_{`2fC%W$_>3}<*Cq*uwppwgDP!LKW4(|kdRYOiD49VZEVKOd|j
z<#9(o^UT(^sPedbQ?B3jpRwoDsfFykQBgC6j$NO`7QZ|1hLDVsoVHkUp`mb#fYN8b
z)o(Hwwmf4ynauvA->EDu{r&$tw^p#Oe!jV~xNzx)M_r$z_itcW^7+ZCYjf*eGX8uB
zDR1CRG1abSu@i3LE@8Bv?I1qoU}mu+dx1m#^Ec`7_w=3yIc~7%KEC{~gLm|)^OL<4
zJYpWCUsyU{B%s}IyLVrkNP*M`F}9sE%<pZKaJxNwpTKb!WhHm%7R|5G&vI4<y$Ddz
zTlRiZhwsPOvIpPKhORy<w|n`Vv<05)d#$aH_w~-Quim_S&F0?qM_ek-IemRRNh~M&
zM@Tu#4t0*4!v;$$zMk6oNMz3iZdHaNR>o!8t$S7{&s1h}JYd13+7M?m*O4b;Lcy^W
z<wrC&zxtE&BK4(q`kY<vJU{=ZWh8F+WTnOSU$ex&h;8+TIu<5>f!HY<4ebjO@3+}*
ze7n4Bp6lL=&qEDTE`KyiUcGo)<tg=Ty*=(Hr?>3zGQKyhMs(Jm)AvslX#U7A{%!Go
z1B2tcMT@^VzqaJR|KWl;>t+60PK(;Cg=UMie`&i5=8LzA)S0<0`7txR>V+;VlWm90
zwXgLjuV!^TykxhLF_P`WLPJ~8n{K|#51GBmeNv?QOTlY;A+MWGx37-)ZZ?C-Qq}8c
zd~rOUyQNS(GG}A^@~yqkKmGe&8(qBh*Zts*H;dFti(UVv>WKez`;p<kb%8PK&iS3~
z?YpxYcZ6rpJe}EIzwh^T`Ed6GKU}x&2s&ezID?Jp%xpHc3IF4Nv<CaQO<LsPB<j}R
zY!z+qIw5}hq~!05EAO^``{f=I#ln(g&9r{jhjp*JG^UwbeqYU3cwyR;2W~4G{XW}H
znARQef@gv24f9}~CES%i(}TXvRsC@?UfOd{rICK#1#7)0Of3w0sXI6X5}X_)o4Lic
zY<6s5acL9gb(>OgLCLi_?bM7HLCd^3^^`1qG%Ot%y_&gLG&m+GG`RH|re51pIa#r(
zl~e517Rf~JnYz&|69f&~7K$v2;!zV6aAi8znzD>zLel0{A!|){2N=ExNOqj<#yjb5
zRoTLpz}$$;oTVGrGME*rPZB!N_4d$?T(M`VN83VXAIN<vH1CGiddJ;G|2?-p)oz#)
zxY^>if|_Z0;_9fcEL*C7i*0IA;hM50rLv5xT~~O@oP95&(n7j6i5|G75zUZyXWPYX
zOHS<m?s9!;?d)~B367^ZgEeM6IkYfAoKaE#+8GVu#Ddpcrwi)OwVr;ZDZIoZV=l|G
z1_kcsBRmWZif6wvWn6o*Qq=C~AEs~0H_m^spIsyLU?0o28&4b-sRgatbBCvkNquwA
zEhX*M61NhxH;Nq!SiHs};)Qc#6?3-H+igW&4IE1nmQD>5i+<tS{K9DBF)7vxo@2YE
zCTA;}u~c{)G&^J;oz|N5^W*VrLK7a?{Z6@+(8W9JN$07~SuA}E)e0{asXsJHI%%d?
zAN~Kt*58Y)iWhPg1U)#z5@BV$bc&CcFT+F@r5CGPEgoqse5v)SVPS{SCJ(DN2GbV-
zj0|rNwY-keXjT?h2z{}5;ymL_rYRvs#thSA9-Xw6-YykzC1|pg@`{CBQ5oE9^OL+7
zM3&xi(U>D*Qo3lB$(O7}O$RQma|u*cSbFrxJ0p(cooP8ABgB3S1?lho;U(soJkf^d
zL}yQL&egS1Jvs)8=BryH+gyzE&2>tZjdhzlqF0*e#79rxa{0;CSF!Bf(VdGfeG)wo
zzcwpc;hfLx#|#hOGp+U8{kqBdgwnfR!5lVMEhO^yd^)FW(5Nu0gx`+A=~@0lhTK*8
ziV8j^kqlG6-u~Z}Iy1)1gNem_VTX=p!qx9<X0tAs;AWcrNc8$_lav=PW?LNO<m6nE
zanxXT(w@$B;_+*w^m>=Mb?a?%YZT1BASS)Zqq)y!lc=Uh#HYBmGhzyuwz;bEnKSw7
z7H>1Tne+YamIGHp&RQ<q{CROd%Y)$auh>jAmbmcFJ@L`X)<|RbdX?HMJrmp>Eoi>t
zqxIwQm#%4^KD&9fVy&ln%TKuSV2epWXxjv(!=c>sC-EMdy+UY#Q}dKxl_g8Wm3Ce4
zY<es9Grsh?htte$85<2&Ew=5}yU@3Eh3J8Gwm#Llk1x%;{Wx`1kG_#u^5UaaIUnD+
zUoi7vx?53Q9Hq7}J8SpbUCq<~pWJ#Y>-3d`5Qhl%9Tym1eNj`MS}ehoYGl~zz-|-J
zB$Rz-al^$0?5Qfz9<?rFt2Jlx=^AQB`kN>n3R>u=yw<pbi`8HOOU6TobJ-WpZD|r&
z!r<@tExCg)QdN}eL`%4$P~gXJ;gcWFWaTuu?V@`tY5I$k8twsCPF?GY5Z-4asNP-4
zGqI;-$re?H)+MinoO2EfiOdpgN!b$p(k^1L`NuQntcS0(A5a#0n)^h%<8nudQn*H^
zOI~2tr^`;8j5S*XTvTKiIC%OAEnm@~84@~0<@GTOL9wohZqx5{1qGBkM{RtiCC0UM
zP0ewE&387GZ=0;*x@Xd<(0fn%xF<46p1S&fx}@gn$m3PsSKc|TJ#LX*ec@KO=>JDL
z2UK3`Pj{RbvM+UtZr^IzC!6m5K67~gqc7)Y{aMv1lycZNm+6gn#INZg2af1lrgB+b
zFnn-mi;!T;p3jPJOeQYa$25uEPTfm#^@C#^UxX%?1~lu6tl(-qpgJ*glGjhh17{eP
z2(~TDQE~`a$l3fkpgF>1fr*z>(~RZDa}Ne8FN|5{Xym5Mm)NT4#B=VF#GxQ1zKv?0
zmpf)oQeX|b(KOv`(ITHb&7-?~)~iKWi%GjBif4TdF}$D@ev_qOn|$uW+@~T9UO7q2
zE-ze~J7dl5Nir>JH%?3}*OK*^f98U^xWN+TjVX8JOoc36{$I^h>d|WEUh-IA;uMDl
zCX16yAxZ0N_XdR|ZV6J|dfCl8tjNCY>XNXC7s5+fS{1pnrZ#1D2`~Gxru~5NiM4T_
zHurQ@w6j=qJZ?UaTry8=#lg>ldXh!^3L?3UCha_#^|WcmqCFb5ud^q9-Ns=5qtJNu
zEzWm2$6iV`@#<JIW$>|zMIK-dI>Iw`wainFol}nnPFLc7;^)FK$?xNY<#FMr9&D0L
zfh!Lz-)QEQ<g2mkMvoMWorAzlC->QhL}vB0GGxp;Yq494w?)X;zf-{3Ao|qoFWc)>
z)^=)#YE4ycOV#4OIN=A6P~)<kMv+TCS$@H{A1?V6QWeha*HhCHl~Mcb|KaEqty?S!
z+$-j7+-LIntnyjT9o@Aj{FF6AA|($cx9S8;u#fXtq%=um!;eCxRNHJPn@Kx&7x^{J
zxFjOgEqqcUey5w&PJhRV0*$j4u`@)yo0L%aG;XB|W2oEChpRHKYe~*;)mgWt)5pE~
ziQ_ewEv8)u)XKPz?r3<{dF+r<;-5v{X%ADguUUWD!gtxu`TGl_hnKlFpE)l6W$Pk2
zoqb~Bb*J_2c_{3-7ju2afhYD)d>YmZFKA$xYd!b1AIm(mnR&f0A_dut8hP%|Tp$t=
ztiSwByPfW0&Pva3^L1YzX=RdCbd^|j;r|Yvy|zEkJ6c;9b!^+P;90Lmb+J;XVb7!6
z6aMeymYie||ATExs?Oza+QJFOl6_TMY?g4eSxoUd|LxdP-Cj+fez%9BVHJyA9;pVo
zvSzGJeS7%PvssRAiDJiJOi~I?KH4?=R?><?{il9aZw~qLJlyi$ji7R)n8>L$JFKTY
z2xysA>dUaz!v3FEkep`YGoen+r%PPCC-8i$*qC#_{`;{r=8<>9woG9O+wjQaLQ{{}
zA)kxi_wl^SGzp#DrX;e+XIWXeWa?e5=xJxO%r-?W=V<EYn797U?&munZvL8=^62&6
zxeqvwbLKv+^H%SgE$X&#w_8T{zE2OIy6I-QPYj5ecClb`9?!z)Lh+(Iml*=Co{d$1
z+@Rw>XHAS2pY)agYqh8Ob7`Gyddc|hoV)ydUcKKh?XGv7t`I8mk^A=il$-5^{6~iZ
zH#EsgU1VW;V*U2!^$dGI`Kj+Mw5)d<crgDubvGhU>3!#%5Umm+hFh2Grm;laIGvIp
zs9)Yye(l*L#lq=74yIoED-k>K;o7r4N2dCzTu|A2bV-8Hiqsyp_5XbRdQ!G_l)Mq}
z5Zn}?)&J?5mzoUQ$p=}A2FsN*)-`-NrSe0|@65?l9czDSZRpbARIkYAiD;3%)g+d{
z!uhA!o2UDXM~70|IX9IAZY#}0X0OYWT)8|nHGA9-#&o`M^LG5hyKK+8#t(0F+Llgp
zIHSg~cKL+Nzj-MfjJh5--d!;IuN-~Aot?L7+k$16x|&>#)~-CYRQ6PYiI*=U2ivC$
zFRgiAT-A9$HEZp>6A$LurCir~u_~_V`HRQ)_mZ^@PAw8ru#Z`>;9M_%<@;M=NqZOR
zF}LgQ{o$<6(B<usn)WUEt_BmM3gh>xE)gY%eIdtg#d-eNlC)T$r83HOiNmRwIqFMt
zTJBBhR$S$D&a3rY7K^Kj0$0aTR~5~7SMp259#8A=da1Ffb4u%@z9V8DjH+e_R8}nD
zW>c~<a#uX{>NwXT=B(cSe@m>yRIP=cGBoxC*)S!!DXsTQTD8RM`Kx9o-3uWT6d1X<
z7(QJ1>Z2mN`oyvYZExj{*iTv;BINz`TvMyie+RCJ0wsYK#||a7`#eq_D_wqZA9Pw7
zs`Z5Jd{ahP;V%jGEfPgbyi~Z7nZ*iKBJW*X&bGE=(n~#qX2Uuak6Y`MD_hvxC+I#)
zidr|}L)J>(B@N7{7O=eY*tKajkK{2m?Ixvd+KT&{xeeI%L>=}r;<(Y!p!Ut+TT#=C
ztn(}vb2rEYYWk}S&3NQ6!->l&U{OfhLY~uZQOoUT1${FJ_?E)NFsXsZ>O+-Hv+TA7
z-=(;I$ehx=8>DqAurp~z`-DZIiyjHfrzi;psVz|u-QJ|}#(n7~CZ&x&&PSMj&tt2)
z@ot)l(I1g_Z+D$v-m*b$&9TGxlHPA}eHoFG({e=YUcaK(|3q%KMapvRihou}-an=G
z$HVGX*U4Q9#rqBgNoWS|x>&MAOXjHH=@>UIlSC1d#Pp6No&!ECOa08by%*f#&5BrP
zn797nq6794uQjXl76&a=S$pX08YV3hrZBzJZG1k;9ZmY7g6c^x8P-pByJg7HZXmwC
z-LREK`Cxwat)%JK^s7#>8cxu<bm6kv3WJy)rpwpZ|5+{O4PsVHoODgVz+YfX(uYjT
zs@!A;RyDPf=S|ED`M7WDW!;_@v?u2f-_uPg8$Ri3Chc94AhJb4XkV|W+e7oM4#LMA
zgev^q(*0dBT^2q#tsbN1p1(-Y)X8;=1J@D-u01{sP8${)6*^x$v}oRcUcKT+GE)~F
z;8D6G(W056tY6&RHZO(E_h~!NqJw>(mqr!3?o*wy=*8QDMTTe7j1+~BtT^#@*|LWx
z-bc;q&}UeiW3u>BSBgpEB^IHjPxFkPMFlIo(>i6RQn2EUv_HR(c&PT01mT7fS9PX4
zixNZ@7@y%d9dfPkhg*ru!62(#CQr`-tDP;A1#+C~)Y<+|RNTe)Cg+KMv$Eo5C-(E3
z{R`9%aT)6xo!Qp+VY!h|<c1Tv5=wUI8!uiEG00e5=@s<*#v(4(#iDHMYj2!$NET98
zPfB|8!gAY#B)Km(P1--a)EqoC?@rp}G4azKzQRc}-bSDOsx5g)i<QBGH9_>uB&C@D
zB|H-v_NBd=Y7(l(cBDn6%h^0s=5ML+$v&P0jsr!Hy)G(E*_4#HJj>Z=b*OKk@(O>3
zPivo7C~<}?F<S5};?5TzTb&f)Jl~~m@3t|mH2G|(_|Dz#v+>oE&1wlr5(4WbIGI+S
znj;aK?#|7rRmz>$FPf1k8<8mbLfFDwk?Y(?neuIJQTjT}-!xj+$t?}jcKKSeC4t8-
z^G)1^?O9B1H+*bzm2!^fO-)&4V9Ii`z%(+QS<JmXs)5Tyi@Ey6<_PCahdFu;Q%<Wb
z+aP-8d9w9lkF3Vp2}^7$e3NxF!kThzz4Fr}$^{+^-nQD}wTUIhQ{8^ew@hXh9ezXp
zw`>cn{!f{7Bu6RF<d)ZqTesGlD+ekcQ{al2{+(%w=O<_PZV$o#0_t0M6aIeCneau;
zF5JC-I{$nzhgp1#6CAiL0=<;igsuwF7f(sdSMzf3X4aV&=-NE3;FaxrKXLyu?KAzp
zXQxEu21d_2y5d>L!#5%(5A(Ut3CgVRiJy6mse5|q)&!9(#qTT)V(+$#JqqR0D`ojA
zTF2@w*Be~Xz0C7d_JS#5{K5?~wb~w=jVtTb(l>q9dZorC_f2DllD98Qcp+1Ge_Qy0
zMLZYy4r(0Bot9D1DynW1)-^-bt~<2V{kgtJkoJc|ZHrsGqf#8q7g~6_?YmN{$^S!>
z@f*uUCLPBRZT|nKGbb83|6f~TQeNU~ATYaZ!Jf6H)*tG`n`EkrEk28Gs92;O@2^v@
zUvYoBXKH?I^Wy~}%Nm6l>J}(@l&V>r%sX3`rEx0RI`pK1*$p*=6r22|P7C-uR8}@u
zA3i<rxpCx=2Q9CbPw=yUTJq<O(~)iATDK0lpZea~sJXI(CF<%Ik89%m53jpy658h#
zFeP7!Gj2+dagXeb6{3D2+#ZUYizYYtSJuDVIL)(d(gp1lu?o+9JRTcAXTP4o#_EvG
zD12ZSpKFGO_sMs{qAa4kizaMM&SCGd*Iabp#A$8AqsZ&Usm-s#mp{+l$R6n>=E>uH
z@z76$SswY-DN7zqR%Cb_$@+hb?5*YV%ooaRUz3`|KiyrNwR!uV36u7Hc)id=QXq7R
zjQA_NY1=wxS1fSw@Hvyc(0EFS)pU1J!Jgn0;e;<0`*Y7`ESEWZX3sIxS9axD*^d^k
zdFv*BOI>ek+3y0+?z4BexNNg-WzJEXQONP;TF;E!Ma$J6Ew=5d{2F2Qi+l4T@$iZl
zNp`Pw_>3zgjI@#@t9hGO?e!O*`9d#g<8zhGf3_Xx=uGuH$(lB?g=fmOX^H1OlTvv)
zT)C=#tl_NK7;<cxS(NaDP_Kh)oNsQ8-`ul4)1||4>y+~w8(Y6LPG!mtH&8Upw&C<q
zJHfQ_`=|0MZ<94jq2G%Xk1d#48l`3OBy8D#OXnq#vwgpYt=JxTa)I2Pk5RwY_!NI^
zZC<{$`BlXBwr!`<L!X3oN3?kO&3>E3IA79gUyxewO`)~_ds77@UTi2@sN^7g>XYsk
zh1+a_4gu#=s!G;pyQF4uc@<mj;Qg^@LD<(miL{cP-z9GJDl_yohj1r!2ygzj{P*g;
z;V-k7C2=>H9I*SSHOb#Pt~81FP~9)@43FOy(S_V`Wq<bVv}K>Nns-n5-u~bJ#JaXU
z6T9ATc=@N^wdLa7ht`C5mK$?&)`?|h9n{;B(0Kn=NYA8Q_iyc;711bo;B?T{#cJIe
z8BaGJo~%?+Rk~j1wOER`x|0^Gg2mQF3Kq9R5Blju6j!SJ&#CBg*ID$T(xj(4w<Yi7
z&XlKO(S0ALuSm_kxH)1<=2kZSSvHGfKK$INI;%zOmO%NT8y&?~!8SVSY13{j6fNRE
zSN|+F+evWZ+oJ0ni%hSuG_PO!aZcykY<FX=WtLy(GgXx=^xdItz93^}QESBHXWC~X
z&M$jtu~#@{vEl1%v)Y6CkCs;_Z|t6_)F<gS?YpB%UVr-08IG$%yfz-R+1-_LO5LS7
zS^V<ydxtu=M6Es_erlr1b>Tf1-#pq{ZTjxs#RmswDP}3YWZR-~X5x{2C9X$*4s+%l
znJ*js`PZ$zpXPt9+M-p}C~LvhP_ilEMOfM$wKKI>Pkwv7WKo~<U4;eZ@|pi*Hcg83
z;Pp|fs@WXCdSZ3Q<|5uLOD)-53$1jny_sNRc1>?ao9vvm8r^ID%<c?zt4Vpk<icFd
zy3_y7wwLbMDmFvrr_Hx*)>(_KpM32Ji#W1{=RAM*6*upWUG7m@Y(E}*a>PzQyW{@p
z;P&d0=iY7)llR`a(VHK&q>ZVv|Iwz`r_=4`-hCWh)_2cM;7RtDhPpgvk@K~i^PUR7
zRi6+U>s?;3#UM=6lk>2U<vY2ZYLon`>%AsD4gOnZ=&NjLaq4N7PE5=frXI6DQzjp9
zomqTKwPE4fyYJOs-QMeC!d1QRx%A8JhwjL)Q8Ss#b0IceeYIwYsaBP`e)XPNcWpPR
zgoPCR{~F$6lbKMfxTt|=k>}Rz6#Yf)+us&<OX;3EIP<OC<C?VW&>5~bUQe0K`rX*1
zS>)4A3*Q&(FItuDb~zove{u2^?X7E;dCj{cvg`47mf*_&7w?)r`CX@7&Ak81W1+LU
zDK5{nYU`5E|JK^KZI<#5wT_v~&1WfD*#7l${hM9$S6Slk<=B7S^ZtDQ_+jS15au=M
zVSiVoaw=@|5OmnE;9xU{u-KDB6BO861r&?AJOUG$R9I9!GOw-@TOGy`$tmC|u*7kq
z3TuN(=!Dc0&3cJWJwGQY_%<os=Gd_^VEMtQtvQ@4N;iZ~VqwtIiaoVu<t3G%NgN)p
zwye6kIzmuwK}SPE(0>LGVU-RB(JT?hAeOwFMsBAiT5`6_OyM?HjP07u^(=Hoxrk)b
zVhy%?r>7KLTUsv4=4UOiHn}V68P@`Dw|94su`TzRmSb8NtjBt?!7k$G21m}>Ia?o}
zxVYftB#zq?ro<MMojSbUT}$-jq$~ILvoy-5EG~K)KY4A=)nLyl3A#Z-2b5lFF>sVB
zu&wmh^$=bCZQ0%3$_M%;Dtue=ly%ARx`Ne*K5*O&57)Y1bL9PPLAAT9WcMHX$grTD
zL5NdkYKxnc<d&mqMZJkLnpjPD+>937TPhw~ZJ9Bltuge<!c1$`FAEq%v_h3SSOm?w
z#DZL=1v&~TJV{ENm)SO<m&qsK!hbgQml_PBqP8s;GMEHrUU2c2>=bb0^iT*Dik^_<
z?JmFb*~C27txZ9>?02sTO*cK+BH*q0aoOyd5n@XwPfxVE-WPB#wKAk!<67qPxfRbi
z1tTs^e7Zy=aKj3*&MB>FD`KmHp3G!O>CsrmP<J!rLOa_<mzYIWM&BMe2x%<LaGV$^
zwvfR>EaX*M_o52%z6Fe&v(i`2%bF^Zyyv9Uvdno~L~@xflx44H-}+E`al2Z{9j{#z
z4^7NsT667|$3;~I`MF|Smj~3cC?1{b_%2fYK>3wt?h~t7rmE}iWcm3$xB5-#>D-;H
zOSgyYX4+AGp(2ivK|vr|G$Mf|Fk?xp7`L8<+y4X(=c<51Q{<*xR}*paNoo_Co?+Oj
z>2&4#5&_em)d^J(TQ3|`mM=*<*7DS~%R$FYOxbCMrOc<poF{jQ^m{G+W$5U2En1+o
z&g)~K_NhfKCdF=#m!z&s*DBev#83UmWG_XpQ);tCeuf0-?mU_~Wt(-o?zbDiT3cJ@
z2N-Uf*4%66B~WPM%JnkdkW;-l)av~nL)Ym{$1W|E4?5L$JjH#>iv`S?ic=XT9<@n!
z(Yv)b@}a<2n^lg};#;GnR|l>Q(cY7F*EEOa<EzRm%ePBPYv%A|2i)IScx0ECl4>~r
z^|Mn&-j;B-uAWl=b>-~4yIyu>Pk8@P@2jl9@5Jt!_na43$NhIaQ=c8X+mmH&t6H|?
zp3Z^?izO<KO9z!0IzFH5VR1;x&BbEb<MJ8%oYeJ0?Kc`sJUHVtr>El4)1`(o^J=E3
zEKgad`NE1fYTE*-MLq8ncCJ1#^_a`V840?odtU}!jbUCkp`k_8P?7ce1CuE_o{TwD
zyccruGl{XdI57BK>WI3tsnu;_XC}wXsQWiex(*il<Q1KkIOS4exyIs@vybHbBoVVo
z2YlYvDimE2WHr`cbXclc#Ci3J#ukmUJ`cM!Z4%NHl*A)F{S+IeT(#$XJz#w{O>@EH
zgc%jiGB-p|n=VMcyE*5&&V)T~+$>7{A8st3Tj!|K$f42^vSzXV-x~M0Q~y6-tEri$
z=OJ|7Mtq_e)2`%o%&co%S(+o->y#O+!&GXx9991~q)pf>%hsj6r|bBM1ukljRFia@
z6P*|;%zHT#x&#j1e8TcUK#sFh!OD8sjR_?-O)d>qhFzDYZ25BF*@j16@-wBplq4of
z2Tf4BqjSo;<dR!L?-3PEPd=9RBBu>Uf)xLCU;BK7ZQaV8r>#sJAyG=7_XaLV$=abM
zJY7#hX^o(JQJ(ijN##kKS`J*A*5&x3z+}a)>yCmm%1=ofaWB%`;*qM*9qjcaO=VT{
zjT31VN<tGpESqw1%0eY05k+aiOMNXRy(-?bllh){T%0Gir1X$d>Pv-7ZBsWiDsSD~
z`ro#|T)fV=eFEpgKGoHa<tr5@O*%5!D~RvNqWrVQl`)Mcl-aMWXa2k1mFL&I{1Yiw
zu?yej+8^c8j&PJ|R5~)HVpZw`jV~9LelZOUm?H1=Ygd5Uv=tj9qi2ele2}<oli|W*
z(qW+dXz`wfTZ*^>)xwr+n)+s`?<|j4scQw3tXAJqX)*rzlFge_&_1XjJHIJC;$G(b
z<_s6EcWZ(xgdQB)#BoY@wt{QE3%B5$X%n{j-Bb(~=#Ee_x}qi(dn4W@$-Utg=VIT3
z>w>JfR?OfOn3<fwee9k4?WzA(c!*erdaf_Jrm^;8c&OysEYH;wN`hMY*8QIIrtg=|
zQ<Kw;ESbl}CjD>Ff0VddiT&O=rRCy+-d?3u5{dyke<VNfb+U+_HzhIZ<c6?Sis^F;
zv%(&3Stek1B3!d{%j7uOQ-%jN+&Fr|^hA8<ourv-*DTimtmh=<W!xGwO>=VZ#;~4p
zsm-fWg?;u0xo>L|&RJC4B$yJM*W+zEZA!6u<f$U3RbPZlZ>4UVxog$jA4j4$O*mQ~
z&7fCs!sTq!gbiB3k2Gawt`Xr*YhCa)luP&aH$MfYZPVY^2*k22-Lc|DgrYcCenE_i
zo9)c+TK!>Ny;8i3|NEv1yRJQ&<+mo)Aa<vc>E8l&cl9;bJo(lyoqTs=$f6M6>|0xz
z_j!~}n)KjDMyu8jD=w#!+@t^h&p3YQ>#2L{OApv?EsE$f<t^x#>KlBjaF^y{-OIVN
zqNhqu*4Msf=DRI7(pyrWr*@%as^hEU3Qb$W85oq4lYcg86uVEqB(SOLY1J{GWd&kN
z^XHxsn<g4HPtP(ucgv?5^NiTm4=uYy3^^_K-r4HgVA;BRs-K8W<&$lit1}jRe+w$Q
zF4NmA;S$T8?#h+7T=(GCWB*cgwdZnQoWS{Ux}LwtN#?av0(r6y7-so+C?4N&a*jv$
zD>3OcZJik%8EU&Ye=dnTF=0-CJ$LZ@v|H@#w{|uAnUo#6)R|)u)Lms_;QdzNaNq{#
zZ$A^f?NZG-l&?Nsw<%daL)U0Vd_=hQ^-2HNZ7B=)+!QbT^>LkzQjK}XaqjEp!pn~w
z5p^sLmN@oyT6pw{3Hhn1e~z!*EHg`?|8=VK^&){pHc3U16M}}dJ$G38QY#FuJa@C~
z*?rAs`IZ!}bTe=6ADq{oWm@V!-@LhHna<wGWBm4K!uNeWWW0NKn{CLfO<pGot(;w+
zD201`S!-1L(mmz-_5U~CrR!}A&ad1m5NH#&E%vQs)JK1ok5|rFHrwseuu7lNRL<~l
z{Z6s7slwS$q-r+D?Dl)^mhGAvyyUWF`E+TuiKRb91*b2{I(jh6dugFUAb-+x{jMbm
zSBe`yF&S(R%{i!8Fi*JR?6DlShz5m+I<uasaT-?CwF~^e6H@E-u!-HUBEKcIaB6z}
zv+#-Ey-c6w-rHu^SMF7&C_Y=j*`TCS$++Io*<pTr99N^U{WbxPLLV2SR*r){U$<p{
zFB0%qa$X=L6|~4N_gd7dr;YPJtFn|9gr^ti8akDy3twC$f4|6|eG*I4A&vXV!ig0E
z``x1zJgSp8lng3F<C<!{4Wm9bN0@9BKjc`nWPyvGA=i=xu@hdHpKMj){TicS#P+*I
z%<ob*OQ9H-pyZ+##qZR$%!G|S($y`B6kSsy+b1fNZRB)cDz#{u#^3G1`Ukm|y6IdA
z3vu}@&S)X>O+iF`g_?Sb!uF|RLd7}Te+V;mbk|j+YW*lx3Jd<9zo<}4BdPv*&AAAH
z?UQ>s+cnjuYajV2)h=#k7a@9QdUIt{Y|dc|{>2uK%*M~$N({ai*)9{9I91?xgz+19
zvn!WG|9$jHIcOAdLgUmV8MT9j=7yzTOjBJ0YyX$U`zH3CJ6I5VDBar8WIbc8h+<Lo
zi9V&7YA2LBKAJ{Y2u4gk#J0}F%rPXX@M;f-W3Kh&t`9Ea|BDrN9q|?ww473?#e7l1
zEK|zgVoK*hR_9Iujz{*fKa?gPOk8u(awoHp%C)w6!aX6IGw(ldKeS2M<Y(}zX%qNX
z^lD}Z1$mmDz1TiCOyrCb!<k9ajX#7<ORe*!<s8*Wx__eS-1X9X8Vn^Xr~kjSUB>z2
zbnO@IkEW#a-4y;}D%26~wOuj3L$%Jv!|aK8@_XmF$J>&ZE4Fkww+IMMahFi^wy5{m
z-sLEiIb(684YOj*f>|@2bzeFbew$umw1hQa<ph-{?(zpyU!5?K{1UZm(u^GvPQgs-
z4<ZDPM~c=?7d>q`si;(<#bcrki{=^zZ@q=F8aEBxzf3Ltrt!vXQl_D#<;_mEhnfo|
z6d#JlEp04$9Xk1ySgdK2s9-^ukE6jpLzAD=RnJzY_gGpxyy!msNoT7;Pu=xt8ZXM_
zW;Ss-I_0a)sCzlb?_!O>#5oE|0cEZe1*9rDnUhb58qRPxKC#1_?P09R;pD?2^L!5~
z|6k}XpdBT!%2U99hQ^`Ex-U108ZAlgn(S|ISYiu{)1PCe&r^aoFBi&5OI@!%C%&nz
z;d-_6#gev}3meq^ervXIR*5R^Of|6R@DrTdEh)r%IaiZKW~ye2r>11(LbI=zB5fl3
zyFS$&XjFESFkA7ne_4}KlG*$X9)b%p(xxp>Ob#$n`xYmY*rXa*JEu)(t+LSZMtg}V
zI@>)3yiyn3u~>O}naJgg=}q4ka2Za&cSENB=gN7nR@Qn7Ebv;j*h^6Bgm3OjnexaY
zS(aS&_M|0^&F?-Z^}4UN-{GWxGB19Y@C->A*D1?)9rLz6uIaixaIMKK-KUEbJ}sIN
zEE+Sx;g;Zp|HqugVvky+Gj?%pT-LwPKP%ArFN>31RiR90;q@zmFS@L6Pf<M`zE0+4
z;?yANqbGCPBi9FLIr<nbH598AuTsxtS++S#^bwn>OqYw?A)$^Rb5o?|6$L8HJ2AP?
zs>^hW`HCG*sZrwM-2RnW0yl4E-)zofb=3QjqAT^o^W+Vq%!9T^!wfQyZqhrgpnZFj
zku}>*mL64$1>wJ%d><~DnWfHmYIE&Q8T;Q0oVx{FwHdx8tUTd4?YQT{Wr+*dXO$O+
zHa~RVy5my-L-qO{U7a$l-F``nMKk6(nj0HT7cbe!7<8fE@8Tkv0JrB$CA}tX3|HI~
z&uY@;Qu%MQ#l53zmAeK03o0bmb8kyHnsAQAS)SYB$;*xP*{&&tb@Nmi0}aC0I%vi`
z=3eNw((j|gEbnPkyUIV@+F8Wb!RaXCcU#2rbx&Zqbm7UR7E2@wlZBIKIZeu3y^-6V
zd0BB%VO@BkVhTgaGR69iK$r7T4iR6wzDNr0y0)El;T|V7M)B=gCmk2mX)a{@xOrjL
zs+n9{G#2d@IJkMyE|EK@_d1mY%WkRuduvAd(Ouf+Gv8#-ynoGWdz9i#Nu$LdR&!N{
zF*`Q0Zmj!YqLudApvbyPr%P%1rCB}?XBA{*n^aAxHxfu+IN9&u{vTzD>B-w)UEJ|v
zo5h)Eo889}ukM_<CScC#z+LjezUM0>{-0JXQd{Fxy<*x<q3KJmuD`k5YWv28J7<ZW
z4c$F~DS*FquH;l>pPBp3c3Irvl3bD0KSR{J_*B=LPVEV^6>k07s9)m$U5atj3!$|a
z`7ELZA9@RV?iPu!5r~Zub@JZgI9tG-drM-CsLC6+=$j@kt-`^pw|ajuw2Bec658KV
zn7rWS2BrlJ^)Zb~T}M*6i}SigPrVXg`<Rhyts=kq;I*c_&tKI4aGz9oZ7j4$L}<=Q
z_K67|0;*3trulj8z3XB5>(|82OUFuu((_hUuiv#x{ib36kKD;Coc<n}D=<rIO~tYd
zHs_tMXL2oMDePLa#Ob(TqRNgKb;BnAU217Fj6&z=DF2r!(>blQH(YVwu{jzy)`%?6
zUf=p-)$WBAuV%a4?8)^_-2Y{9fNMoGo0H(5ZsY&nXPj=#6H)YGJX5!8&#Yf(L|$H$
z7dhzpdXenLNPWXeJ5K-JD6y>7uTl1gqP)aI>qDCrrXHO5Jd1nD>dm2sQy+5l>hbjE
zx9;_M*ko`<ghy(rZ&}RaJ;oC%j<Z-7b+hk2$+J}3xqWh>NRnauI~n79s#@RIIQi&I
zVt8^UiP=-)qYC@N9_xqNvFz%pIqKOtqPKYj6K$?~F>djVy&5@jv&`~SK7|W&7MX5%
zJpcU{mnoN5SA9~PUZEsul-FGt-8S2y;mwq7N0+(F_>{y7WUXFv%;n5~o5emM9hy$I
zDKcvW&QGz44ZPylbg+4~$GeRQ?>A*P-aRS!(Mk7`*i$Q~>CxwuJx`S#I;DDc{nOjK
zL{pm=#olrfyKMR6@)4iP3_;O^&qkaJ7(5kj8!nlgvCJ%Uqge0;vDYWB8!lEk&@E~>
zOCYHvbmH4{-^{MGFYI~8YPIcg{H(WgoKIgZdvwY}Yjxhz^yuHJ2Q05$cRTGDxc{h;
zeS1)zWTWKUm}qXBvM?3hx4p6_I(pK#PWc#e#-K_xc*2RZQU~pn<pNW6pFFZwY?W_+
z>+{s`=nJz8=jJ3lcT9<OT>J8%O3aUS$|i>{uD!YZ&9wV(u10E^ZPl?m;I&)#Z_&}F
zTmNrf&8sp>SoCS@*H57pMt2e?Yppl(F{pd^hgaXiZ^yG`jG2KZ=N^_W`*kIzX1%BO
zlX9V}&ihuD7Ok+o>ZTbME|U0Y+17oPoBG_o+~ulH?znyC%w^fFYBwhKgfc3e@a@o8
zwX;kA)``#M>S8|6F7MJUo~!G??5j9iSGn7twJ-10Cb_n=P6hj94`28B?RfK{=1$dv
zI*t=Xm2)3oX;-?-v+~o^tIXxs1Qnm&P2Z{!xuJdH#+8Sr6bXvzA6D-<7j#@_Piyg=
z@3%rZR|=o{_+l|%CC{n7S9o`+?G(9Ys<YX8(zUeQbzj1>b@v-rTIv+;30lC?<Sn|w
zFC{MUIkybUJH;vg=N|7jPb%rkaR_)J^5oNl=~F~v9rcdw-G9M(apk+t$xAL=X_CEJ
zComzzL-F*Uxuy4t7^nJXU7XXsK>g3Fr*B?WW%_paDVg3$Ds$HCK5=)(i4IL6^J(iu
z>~hrKs<H+%9MqlnMuAE7u#K5}xq!<gt>Uitou544xeC>AUgAi6a;)b`r_-mT$qx(m
zh)fSE*jd-CReSh4Yu-*T#eIc);&&I%3VCk4OTP5o=X-Stg?E*I=rB~j6$rj?qEk`j
zK<!%%Aw9u{H!H+X2FkyC^yvQSd0&O%1w$@wXXm@H?Q5j8qV<|-H)YQE{hfC;&+PE?
zDVL&eDcXtTO?zp!yDD7gui=`rH~wd7iRsm;b6&Vyo}{erTJMt~6<z<ZY16B`m!)d;
z-vVA0Hm<m660x3rhQNl;GYY4Exa<3&Km2&<At7%6&k~D2d@TFCVTEK}P2-JoZz7(a
zc@gxH<6{58+BlYtL7C@7k_xN-&y~CDc;xgwS;jh#Q~UJZ>%E8weem+$`Tu)W9>ldU
zF;4Y-+NAa0aC4l+<@M@wYX9D!bN5Q+_rB{LNy5JplQT;U)mcAnU1VpL#P);b&5yIM
z?SDM4ikrSAu=mruc#Gcied=bpWkSLXhF%98npjyU7)1mm9BSm2v}=)=`0$V+tGVHx
z9V-_x^zxkKQQ7L+#nQ|<Nri_YMS-D>gJY8m!;$|>Pd9RM+*(r^%y4?H5Nl2YhsX;D
zK7nqo8BP<r{g(*}e|WR=Q>xP(%h|7FBo<~l&Qe(^*7C|&=juilWe>s14Wfs_w%y|0
zQX2X$W>@Lkdq-z)Pq@>xRc@2kjjja=%QzI7ww->qDDId7YmE%Y&F7DHM@vNYb?jW=
zaeBsN8(j^@)SJhAmI~O!?lKW=T4TPt&Gvgr&gCEzj)cprdKsd>Rb6J2i?KLd#ni>U
zqV&#s|9yw$-<fq(D9qkHN!{xBxst_>C#BtU)jeifAM%x2vPAC3PepHqC5^k=S6Dv0
zu$1{V_k{Qf&Ch<b^ozaGSGtrY^gv~n(xe9mR;39$I5tVxOjx@1KTp>LMHMNL2Tto)
zW?Wd+uHx|8Bq)k;&Dy9M$&L+gW>l?SDVA7W=*k!}Wul@KueFBRD;}xT<BdnOIoug#
zvNAl5XZ3KLkk8!x!sgEfA%~{(3pRPGs;pn5y|O7LIYU!-BTKNpfUfdl(L*(xk6SeL
z1sg9knV7P9fkpf5%j>L!qxu#+Q(7dr!;bftu=%nz*<BuTy&@+rv<ul@(Np{!bV>ch
z3E`94j9WT_Y&+J(bOyJv&Cyby&hQ{}wL*=Q)|u6-tUesz-QOjo?k)c+pn>DSsa0BY
zwk_f`S-$Dj)a4v9GAmX$I>~GjJDg(}*dP&F8@AqNxvGZ$^FLLmqn^)LsvXewf5~oc
zgO-{epR2n?=5}xT6StYe^z?<>N}QYwAEqiyjbYt<?G4kG{ssEHuap$^!q0X)7Qbf-
z<kmfr;l3g4LeA%eJ@=;USo!2ZQ*V^Cs@j%|4;oy)pO6vey6G{Av*^XeHig$23wi~V
z{2A2Q4{dRn^TF3&ONCkDMaZ(C-WT@GZ12n!7ryN}kib)#*Xi7xP-LaZX4>;hYuXgK
zj#sPmre=4ydahimdmwA$QVUV$uVQAKd3PD#>iT`C$iMTG&+q6xAuQ(ZPO8eTo<;|{
zb%J(k2*)TLn(N)Ia9+h@!M&z;wbq|nCjFX}aZ<nCqe=IKg6j6Ls5iOit)DL0yzb&>
ztGnm2|8o~b>%U|;ZLK)<b&`ovd*1$aW&PZymE}{a#eZs@xp25&L9_7Cugi@=UMC!9
zFPLPoYSqe;S?VkNCM7kk-tu1j#p=GWwc&HpdCfwWMbu7Mz?_~evwEqb(|v_?wkFvJ
znNHqlDK`GQYU$kjua4a**upyJ+E16{rHl#5|3ZxR8VH!RW}n(s!=!&jBQfsM>Z$vf
zG6d?5K8s(=I3fF;u))=gg685*hkrMcmNi~EWX|k)moxRnWbsR>4Rw<^v==O%{X(Ec
zbmQa>=454;2bDbK?cE$_?3g5f)(F^ISWZ1ZBgwXHMnTspV-JCfXTtvso@y|@T2QYt
zV}gNEuV&oFc*l9uMgMaoZgk+W<oqS!vbRIUzc=R1#g1E-x}I)$8ObSgCiae)^K+34
zdJR(o?+Lt=nd<mKt1l(|qhHt($G|kjB?{+-KARtFtoaa8R&t+*UCGh!+0rQ<j1Q-=
zIXMNg1#r#zed4)Y`kea*0+d=pL}wSwoM3&oGilqFr+Vw1PDt~yclany_b+%p`6lam
zg?=R^<HVmcdPL3}1@B1v=<!MQbisNBw!@vv*PYN*xRTY~zxldYW5K!NI79t=Eb+Ys
zLXANO+3QO7L_R(fzAuS4C5-1H*I&6yt7kjtE56+<%Cut}tIsCJoFw4}+nPZB=m}?&
zidTCZc5YTP{aUoRVpUk?jgoCQGIY1-nM9=oe^Qf`FZG#uZsM6dflF*|Pv0BPJ93(R
z?G2MgFF#JUgPF-X;nNqgu`nq9WMLIyU}Df=U|?WqU~*t&;P}tLIz`Z-K*7<Rg>{S9
z4Yh@b+XYU0=ma<M92AOSiBO3+^~kM5!C6g4HQ~w(U(P_G5Y3|C*;P!pt_lULp36`c
z<+d?uyVrqp^H{3i@yrrfabmFtXCL#<kjz8NRE4{wIfJHLS?;+x>+UasRiRB`)7w-%
zC%ielp6P<!R<B8Gb_9mqx0=f}^+Cu6&cjBcnb#6ym!vJ^^2@sNa9aD?9JQ3tY1&76
zm*{Nj@|p0-`BabUWUn1Jmp?tnWYFE_GokS1(G}j!?{aP#OLr`ZvrjpiX8iW@Huc|Y
z_TDbbn7==oJHvUE=xVbyQJVUGvn>ms9ywW1q;o0ZP~#0rE)CTU){gyB6UF|g$5y7K
zUs&j#tJ3-DL*RE!)*jB>?u6Adx8B%#$s}to!-6$y4&7)tz-fL?Yw?<*geGpLsSld@
z)+L3;WQJNSY?X+sP;|+98XDTB5Y#a}w%XG5Qc$jIYf`I9<_pDAgJT@Y?VdX;M0%O7
zTn$cC{Q3NfmW`9k0tT&#hF<CJZS8ScMrJRcOb)QJjMEMd^L#q>Xyz7H$>;*ccqd8A
zD-*PtCR7GEr75*tad}vK{n>2JS{79=9n&%qFSD(i1Dq^6jjq%;yo(5#(yW`B(bplj
zN@I~amqU1KzVFv-;frl<tqAURn;EjOv#cs3reG=S^%aYHq^7P|I?ZLJM$Xh%S<#cN
z{&THd<9)aF>calEk`?julwOIr)c@rQoZ$ZR$AqN7Q<3u+_NacDmBa67eRK1Rz|%K3
zPxU?hChOXyMQ`)h@w#&rF1+d{lfSWdcW%b6-rZ6j`}S(zNjesI(yi!+l>421O9M3T
zI$zFh6>wp;zY^ebUDU-*QnHm(Jw0!pmA>Qp$`rA=oVAkkoxLXo$-DgKd=THnH0cai
zi@+w;L{Dw?6`xM)^hYc_lX3OW!e#pB)@*Kb=-jlaTkCYrv|j7%)N2<*ZdSg!RC}~z
z@mW89?yb$9^Rq;nbe7+Um>X%z`*D-eHZ?O3;m0}KlHVWtX1e2B<V&+@ayyTwy2%$h
zvN}rgZn<#qUh4m;X|tH8Ce3BwJ3ccoWp?DXl&+Y~+qXGa+)@&D?8<yNxxQIbG%;Db
zY~3Rb^`?2rD;`hC`xUXWFRE%~(ZQpuUa#78wd$qn)^kZ~R`>Vqdbj8KyWcPO&X0S!
z;?W5|n>CxZ-^z*%XqWo^*<C#TclbuXt-GK1D*X-r7OK2<%Xe>P$tjr*j7Ryj-&}C=
z*Zndl=G$>E>t%1;XK|j4x^-yj*=_&k26Ddt?;aSlc>$|#`-f$gdp#AHB!$;AX+HNl
zUg%dCwQ>CkAN7Rv(|(S>)Fw5H_7!}1Bz;t7(vsAv8(LzxIlaxA9s6&#EBq8}P&+bV
z(-w~xZFZ+0o_AtV+LNQ9a?X3A!2iWYN0fv<cDP-cb#u=#lT|8$oqjEg1b!+X)zZ3{
z6L5HfbHg#`S%OPzSQZ|&%9v<Zb|ty?%L{j@%Mtx`TQ<u5Yw+}V^YPFs!|eL<J3fr(
zj~%!B6{sR`WRk-1%WQrHo!-8;ggt&7o4dF1h{)~C6AxUR*z_hOdUjWH91Cdj^wl~d
z_H{<5^RWZ-w>7+*vO-7nMTq69&_$o7tnXU-Iz^MqaEBL9=1SFwf|FV?zcxAT=UK*3
zmnj-^sCD|*Jx@PZy<Bx&b@I%;XO@0i@O^5AnDC4nC4&0xGkx>oCM)b<Nq0IlQ#>s7
z#GDP@>0e#M?cIK!>RM#|R4~wZ!Iz89wQp9tYTUl@zrXI+=lKl08TuwoI_(;6kyjY~
zbHD9a?ssou(p<#^m*ma5#erLzehZ23)SI=gCuz!JdES+~rJ4c@1zTd&bk9#{SUjOd
zwY}+_B_Av6_L+wh1iU8SQ8-j0wcO~6t_=t4bk`dW(>B?rsq$Zpur}n}-=5*bZ5J&1
zd4@UHza{HJLe^wO|2TYgZC0r8q6b+KtF6K|bbVcu=$ox}<y6?_X<yf-t7gZX)w;gr
zoanluv)ZxWyTZ4xySg^vZ+7f|D@9)p1!1E<U5T7YH;zr(s*=#E!}(TX((<P|-J2s@
zCCt71jpqgS=Jh0J+}DV*m^5t*i(#(QlNS@Ba-1eE(&-NL?r`7m|L@0bJDz%}J5HXn
zeec-`(M|o;y9`*HN)#1icg!v;JLNdx#aip76I5Jfgq8`d@Qn@lG;2=Ki<4y@=j^Jb
zoJBm0*iz+!MV^W_hhEM0{&uaX?L+E&o_5Z7-7DYsz1q6I?)~oje;$2b$*^sGJv&bV
z6Zeh-jBXzq#sA#@YjoqlhpCE<s(-Hd_(<&U30>b}d?ryc@8axTd(uoUH~A#1ZPv}{
z?5e-kX!m6$|NP%2$@)yU`Xzr)p1=3Oa_Qfe0Zm_&6)od*9h+o?W*E5nZ+7$wkX2Iu
z;?d?geIrL9n~-c_<Px4K2R}{qY@Dl89I4PGd}8`(fi$D!<!6%hHqScm^HhJk;{P-G
zYS*32n1egsEi?K4$#ZXCv;LESmzD;ycYN(V1o$kz_wyu6aID;&(d%Q`@1?~($wDpj
z>y~!ylpQ9%uPlUqpPDd5&P-eL@T$0=*`Yl3u3I&9=42(TE1i*@x>YS=ZL80fs%u-%
zm8uo&&E`IR;o5<HrW?|Gv*KRgx^Z|}>88@TuXPsux^d92Y~%F1*&*`RZmro@w&w83
zSE0$jZ+D#CwqvbqME>!rt7ThXi>j}_S$d&a_RVh{>GGUg7PofaTPC|!n)Se~mOnr4
zGci~k=>L@5E@JaQF|$=)O*=D9^`sJ?>g?nn*Cw20S!Zph%dEa3`9y2(pT|?L>^Na>
zSJYd){E77c=S-(8=gpm*pKhTqp4d}k?V&TZQT0%&$wJOWF_ZS)vEq&7m@3cY*<R3Y
zeNU0uZ4I}R-h=~_rqx80IUc&((k!I(XzQGM**OX8UWYTp^IP9Ip0{(;<iBsW+U497
z^cL7A|LyI=w%xZL%jKnApZl)(eD#*>xvd$8ZQmZa_WS<gyWbC`ANtU7fA^#7dF5&5
zVTV7fE;sbt#x<|!;yi<Mg?*W4e%8N|c*uU<Ykw<`;I+Ou72{*M0Sli7>g~3(e(<ql
z`u3XC-kdv4Qq%q3@t&;KSwH8r0{_2vwe{5#AJ?5(UtT+Bv31Z4=brP`*TUx?XT4ni
z(Qx@y&#pyNeFXeudURZ_F8+T+antcNGRKTA-#m0&M3W)9M~N#k!1_kBY-h~%``5mn
zZxZ{ozh3vn{QrRqSQ#Vg86{Ym3>ug;8rWwvu<mGJUD3cB(GV%nDDa{|prVn@qfxS>
zk?%!=a7Uy3jz*;yjr<Y~0UsJCerQzZsQ)v)iCLolH+%E<cJHljei~)YRu}V9T==uM
zB!52U|Lq$8Qqz{@qC&sT@;TC5zBtG(6jy%3AXI&cYxV?59&w?E=5;s2J!Hg-+Ljar
zs29b}$k3dgp=I8dSkaa;qb*}cTc$-@o<^H0d-2i*_5mKndCeZ}fiW$M+hY#2pSs%4
zxv<^+p!wt}YBs6zHyj0H8U*hjWNrJ;Cbxf~h2J3y7DIUk7NL8tmOg0>xzh!{x(iHv
z(AjR<xLBgAUA#*#qKj!l*F*tUwi{jRUv#b3=-x1cb%RB>&Wo;XFS-I5y7!*w-uj~Z
z5J%6E7hMM<diKxgJ~gA~<c{vs6+Jo?4d*htuT*rk9*+<zOHwPgo3Y)_Ff7^1E!m8%
z&*qpv|3$ZzrYXyx3)p@W@U=)$_}I6!T;wf>pzn#gzuQs@H}wl{)(v`=8mlHdU1h?Y
z1FZ7t6AqWx>{>qIP<!>>@Coda6Q$ZG2v|;3*<LtTp{S{NVp>^s__nme2W5G$$XsHV
z75*Y)qN*;jWl};?#!glt?#;!Dj%>UKWnHfJtNx#8w#Aieni_+bs?fwh^-~K4430=F
zR$yhB*vP(Pir394msU*i^z53W(d#)=Aegg0aOPCujE3mShD|r71~RZpYIG-BPD|CC
z#;!3fsgouB=CrJr(+XxzEBHArXXmuynXKv+O?(e?nwC0S?uh(wEz-Uszoj#OscF=o
zv;t3wf+CfIf7@o*J?opWa|XM5|33zy>7D{JEoV;D>~|03ep4>uX5=)*&EfP_*{LcM
z0>q2rPqZceXiHtuma8#)o#*ULnQa9Uvv+kC-BB#&3oJHakFonHaLRQ~Vv+fYnTC9c
zvK$XR_TMy53sAfCQ-C>8kj0USDIwzqr`mKAfnP6FbOKAI{~NQdW$-dy=&{CN>cSsg
zJ}0~1{haqfa{edF`Cm>>39FnRkU2kGlOf4+YQTZ{>=KOsEf+9&EnwDKz&dLIv(z-s
zSprvY#J^xw)VbFCYfELmOY*~tWCqr%9?z;p&lfIh7wL^miDnd5<67ioR;Bn#plHfW
z{}}=sMO8PXSF>o!vUyH4<C+*7Fp=M~Xjl0}HqQyWr!O|wnrP!S(Rx>rL?oN!%!%H&
zmiT0)olH+_at@w&B|U;mR^$p-gc^g+&q-HSib^dJy>ZmVGpOz&le)!r!-?MnqYb81
z8wqc7;PO~0`dhL-X6F>91Jl?~EDwCJ+^3VpQ*%Y_ujOsCSn9PHI<!{g{6D#(^VW)L
zsg*5KD_IUMpR{Xv|15!?u9dT+R!-qs@Iic)>5WjUkJB4Z+I?A;-&Ps5%Bw}bOJts8
ze}dI&)mH**KMJx=QBhy)-lifDTj8<#W$T*E*;^ym9I;w+Y}T61k+btXitbIcI3T%p
zqM~`iq4sNC=1m$M{DO0(kI3HPn#*}OxUpGQ`e28Q3!D7Ixq(H7Q#*uw9|t^H>2KIL
zFHnFnC}8=*9|D_pOc6Y=-t*>q7RC+yiW~Y2SF&+0=;2~);$r36y_`#Xfq;WRKm&uv
zf(;xB3}Vq6rGC#BU|<!9p8BD@VjEk0a)Fm_*{ZJ-l7EHt{bT6U-JSg8hQCEy@}!-W
zEB}8@b~)A;t=Ru{N8bjm)k;z`zj17tWVt13<4lL2bMu|-?}jgC&GhuoUYvS@HNbnT
zu;tcZ?xM)*t>M~Rt-ZFIcWsS1z1Z{BlE7ILo0_);&f1o?Y+K&#i9uRR*B+?(RWylz
zOEsrq#+F^8x0j1*T$S?JBq>lX;(lqe{mEFzpR>6ImvbhTm93ipedUVzw^mO4HGeXf
zD$8sErrA6BH#Tu*B+hd1TJYFz7HfW^WmN0TU8}ujFg0v(J-u-1&&>iGXR^ERp7L`B
zU*XJM+CrZsb~{z?-v4^{9Lqh2R?j##Ti~l;bV-QZ%rK$m*_lfcJZ4Xrby<4NO3gK=
zX6-$)YwxYoZTUA?6aGu=+p%-@hR)PtjkQfnRPA1>HqH<@`Ldm<z`SY3oWxmb7bWNF
zO_z0=ZlJC%DDko6*3SNwoQ1DeX}+<F^f_X9l3Ot7!urxx%RLV6H2gGQD0%@`^+Ac~
zjgm4OW!4;&Jw0FW!@Sz^gr}@lQ=cbgEb$kuTKLZ-S^spU$!q_YKivL=6=?6u|2*UH
z^NvO8Sz8n@Znl22Sz%+JOU!DkGmH8R7tLXFjwy6>&6*JR>xcm7HUZ78F2A<8{@NOS
zyET$$Til<caXm+Eb`=G1Ee);OmfL+Sr}|jVn`)D%#|q6H`Hw6Wzv3QJ-O>I`qIR~3
z$<HG1%@VIBE?e?5fb-0<=4^=qE<w)!KLn;R3(h&wS>Y)#F=i^C;Ytx5<x0t>s0FOA
zu6ix;=3X;Fetv1xs_0Wcr{%Bpp6PXG;Q{YGJAUul`0DU9&n?x_r>_VWE)FpCT`hQ7
zI%efg)f=Y`)^x796MbggP1fv=Gsj}iJXl?HHiqrIOxs1BqVqia&T+NB6I%Ofw`$D`
zf$OgXj-Je@P+s@&R!pYQEY^c1eK*Xfe(7*)pFH!4#V1C~D8=<2(*$^Vgg!p+jFIFN
zvY9$}w!ri^0{t?ZcCsa?MsHI0S!g(IQ^6B|-PebVejgIF-Tb$t&!W4}v}Uu+TG5R<
z%Ima+3RD=lpY#8Fz^%OZa8%$CcbP5Pe|OiZxXU-56Zrq|h?(p*i@#f<WVVILY)#79
z>TkU*#O6xK@5`>fSAtd-rmmjoQ+q6b_pxm0tNyZArN18w-BojHVPV{??J>M+3J)h4
z-OQ*tEn2N3*R{l5U1f*+Vf$%q9#M&cmk&yHpFYB}k=5hE38h5K-#iB=%hVfqlrgTT
zU-0_mvMB!L-lta2zOi(Uh!fB0^)-7Y=<bd*JpEPYrs7|Lb+2a}{C(<>^loPM-E;ol
zSoC-IbX_6F$<@=IoIX&q_mIq*(|7hBTXQDi_pNO^3u6=5Bs0%m(Al^1<n7&>0vlhh
z-e<1*chlL#pqQgm=X{dh&otrA*|T>}%q_U<sK%Vg#X4C)@3_ZZM@Jz=hG+j@35YUw
z&Rw|0U{jv9irh8_ch;@)ihHgzyj`Hlk)s@wAeeQC=}@o1wnH}8Dy2@{|KD=trNjg0
zIWt&}AMV__Xfdzq^7Q=E(n3EK*MBouzIMZ&nU<F{a`RQR9&ntyq;>b9;iVhhD;Ie>
zUJkT5YTtV_A?J#v?-lL3E5SU+5_ujQ-CKP0`Z2S4$I^O^1+U%aH}_bWZ(9DctLCl7
ze8tDlsxDKyXpp~FFT?i4@<pfp|4F-;&dNV)R(h7LcyH!ZiH3yZ(_g9XT6&6WjRW^A
z(NoJ$ySZeXa_QK$>gdhYe=bgZdvTJi@?Pm%o{Fb9<G9x*^8ag|S(kTu+nLQ8g0oI2
z&530y4f$_vzH;u1&1YFR*`CSvIMcVUE$sG-sdX>oB3^EhoqhSunc4H&=FXGVd9HU)
zQY~e|+QXG158v#IJ6l{)BXUjfHRs|E3C3&EEpuN;N$u-2e&?lf<sz3s;dzgvy+5Dy
zYHM$PdSa6y_vYE+n{CT3u1-B%v`+LZM@nJc+pFhp&W^n0aq#WNJA5e#!W-Z4pRVS9
z>L9RS$~`85hgFx}3IAIq=U3&R!|*w;WsA*oPCGC6zzMT2Jo>lp5d;4f9k~zgu^;~X
zU5VK}vEcSG>vxYs-hSZazZ!h^6~Ev$(fLm*7C(_p`S`&|z?@T^|9gRoNoMZZSo@ul
z8iHz(N%o5uJo|BN)z@b?=Ki;wIqR*;*4Ynk-+TN0T;JTIeD-1dR*li$0{HbrZvMN#
z_f6p8(g3bG7q~ub6ZjUueYEk*y;l8idZ)|oohs$K%(?ID+BGxmr_Fqk$X)%2YsmxE
zt~gI4zGG`_U(V;dT@Zg;D3PtHx$Uv^zGG4Qq}A`#^naIL_~e|gz(w01=Y4-f3YNIv
zlbz%;<-F;-u8O%ge+u|+dma4TVz1w_tqbmz_zLA;dGnv`t(sP{TGyq03H(duZ~F3X
zGwbyh`~Ro@s=f8lc@(4nKq0VyV-!C}k<ZuE?AHO@iv(C_2`pytd%$N`_4e~3nTvcH
zy#23E<S$w9hs{-B-vfT3ykFPi<=0%`J39N}$Nzr64Guj#eEx=<mND1CrRn!oHO@WC
zUjOe+{s+Un4=>;UGgx;ydCwB_xMNv5SFQH_H|M*`5V)%6fFJYe0xJ!NhfZz0vUV*R
z4c#t!!pcTE#TOGLyQRdZDIT40*4b>a$t)3rNjzTtOvgAlG#hx-8KYUY?0Pw2DnmWf
zudBf~@)urMz+bJkMfHIA`BwL6v0AS$FGE%Zux@A)V)dG)5v7!Q(!_1@p+JvK8;W;L
zh}*x_`R2u>;}fPWGc0VpB+SaX)OLo2tkBl<{wdn6-*P5yN<BaSQP|pBe>bzw%nv;^
zRag6q`}tMwE167Yf4QB-J^kY1of8aucPUNRT<u%DhGTYjPG8FZwBnaVvz3qE*`0s<
z*xzW2w^#S|zP{GG;mpCuXLp;p@B3|e==tX5(^vM*tt?~Qy}NvmN9w|g_F$7+IWs3J
z`hDg8tEX=m&Cvdi|745GRkdfS{fpzc5<1s(M=`FNdrV9!bxKv85|>7A&C28AhjTVA
z7PXGqczD_op~(d+msefnIMVS-Ft9<$J><oab8TJ|7uNM9sdK4pNmwpcmGh?cNYx)n
z)+UV%#jdr1CKsDLd?OfF%?nf#bT$@tZ)D;$lv$%HwAW?J@~J@zMW+nUywO}N8eg^P
z_|zaXlL=F!UPWG>FRECyVWEZOtm1`MqPr&V-@i)f@v@*3H&RUfSX4_)ow=VT{lCy9
z;wuv9CSPiD$yGJC#LU%LYq3J3S?W547MrsnyPHhjnuIj*D4B+agxhXi@Gc`L<3j5W
z1%9PpBD=OK@?BW%dA{3X(W>57H>K1mc7k`_YdQp4=PMfBY6xxcE&7zL7#1Puy2@OC
z)zN8Td)9tf#<kb?)Zu04?!Jjxe#z<kLecBF(@%HYU|J=nvEafJ_4=|W<?&$?I+CU=
z4!f9i%*W-VQnzII%XHbbWkDG{D(_ln$hJ-tY0#fj*3z+F_wk<!LwnW&*Rv|$?}{6j
zOAC2Li><wN?Wt{I-Oc5vVy3MU4GPij-Qw)~xpK=jKGj`Hw;MM1TD+Jhr)RZ%X4<)+
z74w%>{Wot}vhUrlaKpW`cD^^5%dw~RjB)?MP107@O5civ#4fkC9QBN7HM&*wuIJI>
z{+6FC7oy)M*S_UH&=z5&8QGroaP{1_2+3qe_V1z>9XYJ$d~h;1C^^b|Hnx)aGtcBx
zJJM_%w@ehPda$U~k8#qJ&kL;{$9ya_%ilB6FU&16Rd(a16D*#sTW@TfKW%B3pu+;4
zc^lo#ZfgF}VRUJ9m>k8jLUpo+$<Df}HJcVZUtzRz`_8XA2hYzixKPb?a&FJ@mJ4Td
zqmN(qyZ*dh%YW6QGs5+4HJRx*9lF^P7+5V1I6GzqF{Owwu)7NTl{7XobzED`VxaX*
z;YiccMG0)qOC9}6FZ@?no;7u)7KcUq4V4^G#fR>%jHmw5QBhHyc}gR$W74EGPxAR6
zhHmOJU7N3)QnfXKecQDo0%a@Dh~M2bq4iEeyV;I~JZUeQgsscjd1_a;8|=EPWunOC
z+o32N_2N{Rk)dt>0q<1ah3>ZZ4)s~=%okmJf=Qx#3s>0|9iga)UK0Zrb*_og?p~eX
zp}S>K=e#McZW^}*KTlBLdgba^V0g+&cSHK5Jx{V7xiZ}~SFs8n-MU!)|A{4!#ENDd
z`O_S)>5}T)HRIIb+(w?Yg0s#ioSb!)r+1pD<g6?|<}FtPp0w<ra$?P{1p;>jE-UTT
zNi%4aG)py`JTtWU*_EEjvRQfVR@eVe=`U|MIw5NA+S&Y`w}YREP3#HrR?vRp#Q9^J
zK&jS~HlsvU=VeC)CZ7o|$O?|J-LpjC*h>y2p<4p22FoqYXB=FR7ICh4UPhbwo<qF7
zGcRi?O>FJFC&<Z>FiGQxLRaXTr@lcOCpu+p>@vR;sJG|?m#gM2mHAhkL;^bwxs`7h
z-7Ufszoj6i@03!*nmdNeerQX4SS7wUYLAA-UjbGonVbBR3-U!hGGw$D6^I!+Y?Zfs
zbTYBy1wY3}mIZCftdc(`sRmgb6U)gu?su<TkbUlowX2LTo-)}KX8ScoQ>chlQ_6G}
zi&Pi0>u+sF?NA3V$;;E*U7s<sb8EJiBsK9a^!)E`w{^DPmt~ta@61_YaC%N~se7m#
z%UgEW?1QZ*vTSxFb=(kXninfrymckZv@B&IA<1P+wl!;uH6#_?ZhIpZprdnGE$@bN
z`xd9F@>e`Uxf@;1PCeM^C(@v~;P)IohmSiu5;<%mHuCM*5#qMGK;~55mcB_%*Ztg@
zl*2b|5Dip0Srw>|DJJ<+_>JYJYVN3ZhDquSz77mr9!@I#GCyq`HZAINewi=Q5EQCg
zvR={O);o=>Z9~J`(-R^EBb*D2BrUc!iT*F>XE+hce?sNx;jIC}!KX5&lzAT2)J?t^
zb(8gR$<ot3T4`SXSqra6ew#6EE#s&DX0dG-6}Fw|WKA=W{Xgl<m2+M**0mNnr`(>n
z!P9uwNkP%FGXWEx7g&Zw{@!%GuX)?Xi5GbJCDUgW2c3JT<F54aqqOyGpRR95f*trc
z16<55b{+9Oum5I+w}0Ixt5$|7#qD;*j{I3|trE-R)@87II9y`u;L`V)_KrR5OH-ym
z=fXuIy-sQeW=pm@DhR9HTqm)Bp)u!ZhMQdJ{iYaORUdO7H!dx2kqeh3*RZU)7PPYc
zNC~fk{6?pWkXQL?4Jr)nj*3iy2HRTne<?rIWSy*Nxmx#DAUAVv^byCDxpq^ptJOQ`
zIq9z|>wWx6V9l}hTIx|8d!nZ-;?PWW|Mo&4{NvvIQ@JM$);4UJ#%GlB{~%k~26l0s
zq`(N1c&RQWz7ho%RXOd>%SSeft*>A;lvs5#=);^vytyYF?QXUF=n7x28(n=Y^7-<L
z<*MhqI5!3{`QNpAt;D$L;g&a*MJ$>sD&{UclJzJ0!a9;V-fnW$y!9Z)e9yt1{^l#H
z>jWjM{~Qv%*YqVa*ll@jsulOKXpVzoY%XC5{9Fxh+Qa@y+th1Ja{GE<9>Wm>k-ERD
z4{az8*yr}pqsQ<_(y`Z8$2*vouHJBR;wd>UnTZP&9Q%*zWIXJABz;ED$n074?Y#DF
zhKDy^)n%L5^2~Gpvn^{wt(JVb8Twr^^rl{guBzs}Rj%4)YmEPHNK9XMXQ@u9{Aq>n
z|L0BH*t66nO8L(Q-ucsxh#xRm`R0j+zWu4gYfc%@{o|wUUu=F_^Mvqb`BhINj?cQc
zyl=*4&20r*PiAvz+Pg87pX)yIbME_|<@N9GObXnx(1A(dY3Qp<@BZel-8I{&S@$KA
z@C<451LjK~tP;N{XkMsoy;6wfg!udg>};>+B`3G~I54EHZeOrVpY6>=4})z{4azfS
ztq9&N7@j?6%Zm2(%yPcctOA^qUOiktZKq1x0WJo{iQFv>9ulmdF0rf=Xtq-7+S$>R
zqsS#F(_HGYeICO^nUjKSF6L`?t=cX$PiEyI)}30jH#(IU3-)iAf6e*8?n~z14sJIz
z=`r5Yy1JOvt9r$Mvj-wZ8q7u={Cz*P_dIOcZ)CAwt9MpGZ`Tb`dzo4Gdk$Yc;mnk^
zBRxsvS+G(1EK%3O1zi<KI#=j)pXTkDyyo~M7m*^JGn;fyJ<@4@!8h^qg7Z?A0#XYj
zUUKkhNQ(&B#c1x2erPGWswX$A^G`?TTnWzdN4gz1*8bHHlYTTcqNJm+Nd3qO`!<2r
zec2o}-CJH2tqa@19DIOJ_(hX>0LPpat=|OqN)~fe1<cSCTv>f`+Ifj){ZmbW4%~J@
z27fkMH%3mlY}o1ZV{Yh*NozG$EnLuAeRBH#U;&dy2lQe%>U@Nv99FJ;#TB?)pWS4p
zxUj7y=Z3n28?SAgKQmEMY;)KDc@e5THsZyVwl{-?!!AhPx-Dt6p?8u^?`;pM<uebL
zIPocd-aEbI<ct<id2YLxvpW(Gopf#4_oTXGZVJczoP95ok1W`;$v~T9&BkRPf=}r<
zd+BrU{}_EFL`W<tro(vmF;n3N-NMa6I~JC0vYf%-Z7n>(W^tEqMzh`L-G&BcS074n
zFe)*5_JqG^;cM8}wu_l@#{|Vy2R}S?(mLqyQjjbBG}Dw{T)P6~9-cHkvD&Pxfr&9e
z;$i33x(;rQ3<HO%d1itQI~y8Szi5Be=wM`Z=wDY$>SL!@lB|Xi?1F;xE?0I-J8nO>
zV(a2fTPk*&NEAy-lyLn0$jL6)rSwQb;O6xICm*hQxMWV9v-9B{yTUS@mp-2Iw82?V
zfpOX!hG}b@U%qL2snhuSP1Bn<46Qr-7+;)w7qUF2aIK;-uj7}QT?|rjPCLFW={Ov;
zso7Y8!%Ku?)e*KsJJ?U1@2b|}a_YG7X-$wwK%h)OUpGfwqS|J`hRrjZcr~O~g>Z0O
z?YNzwm^f3h<Hu~tPe)fTP-n25mJq{HeMvF>pkAwjGUH5PQ5VTouRD7ZOj<X1F%~on
zt(BBoEX?($v*wPEuE#N}TgUcBTTjS3!@;9j=Q;P-n)a%Q)-@eXh61hAn=}~?bMI`n
zu~t0M_vqMyj1##z9_zS;cCPZ6B^bOf#_($P-ZQVK_0QS+f72~VtHZWe3{G0?-L&&3
zUuhS=^;&+LucvH7m(84V(*A3(yz{=t*8B7pG|i52dF~zJ`o+^V*JyUkl}}SRUVq-_
zb(PP1?XsmT#=d)({pwh2w76$V@Iv2NT|piF3A+~V-^e>7gZ~Hf9O+fd1+I2Xm@&<-
zYj4Ay{UxhCq^3-W)@Xg_W4B9(vE`CV{j9FK3oJdmj&rg|sBSUQc(VMMOsnJM1NSek
z>s-WrsANgtX_G0Fj<1toRatrH%#!xF3s<$*Y}+oy#rDLT?T+`%jgoAYGrylavFw4?
zJc+|KrhfI_JRZ#^<|T*uqfYK$<oxPM)5|00nthkQj*0mA$M4;jbFB=gS{bhWf1kq9
zadh_gn)9EZL^yo8VQq8X!K9b<tMdV?BdykIu1huvPnvqP$;G|p=JcyaI&L31-X-Rl
z!*QJJ0)we*kjjPmvO5DiL)5vK>)vQ+wYa70vA?ZA`fGNFS%dQ`4ejiiJKH&~x8Deu
zKbbRmqk5abu^9<s^Do=jFx!eG%7!1{Yg^#fvWs)=WA9x|6OLPV{h748bVa*xjj~Zv
z>-yKnE2O!71C(_?DQ%EoW)-xpS~-!=W@fNbaK{plRSmcGdhV<&*qe9Lte@xPrY2{<
z375|;oi%liGh@K=-6w-BgE?$PV=w1&tS{lP^Ihw}dgVszT?Oka*H*-OOtpI&d{cGu
zrnD@h|BGZ!t$5<;-P^bB;3nUze14^(*+r+Gc1Ycuuy$3(?CldT3IrSX6vVxFGJ8W{
z=%g<@dkU_GD4pJLr6cO9nSFG}bc67Yj0sM{4DnwX^aUE4r8tgkbQ01$yJ@NV|0wVI
z337}arW+Uz?N}h!8{|~(EUWW}Q#w-EN4hKCJN#yf_Rma?HA!>t%-O!+g2(PnX5TJ`
z*R9_5*vC0P((huCUtb4*i|mn?f3!u*5?hy^?pk}}@?FoisU0svIzF)YcWvd6X){V-
z^`A56;T6&IdlQe?%0$i%c$DCj6qfSnyNiW}T-0n8okfL59u!30IdHz?vj6O_Nym%U
z?26<tyr$5v9k9SP`G0eTg6CYO(2%37r2!V}I@w}xCDwFUEk3$@hwJS}%Wp4I+_Xow
zjidASM#a_}mXTZeXEV!wU{pG|GhxTq_UMA<S3(Wu2F&WaSdTu~v+BmY&<OFjLamCG
zL6<$Z@0K<BQ|V#Oxo53**E;6liq@EGmpRN0FBe<JpJDYhYn`^F^77<0y^DBeU-XT&
zQH{M~`YbW*Zg(xmHQ%(Ww$Hw4*xj7^>}Id$quqDIWUe^N-n(0C#}#@d|68ovlzlxB
z`<9>NySqZoJC^V14!dU&>CgY3U-E_bVJ%mnYUtG6kNYEbS?_h8>7lM+Jx#1=`Fa=C
z$shQuH!c4d+rN3$v@niEfjf?`{1Ey-!BivI!kgpbRfE{=rK{PQyEAUS@nQ8dXV1H@
zIcK}h67vS1z-=O1t{Pvi*4<t+`Q@t5?JH;RI&va~_u_>1sB;aaarZrLygK67_Vip^
z?v1x+PA<vgc=0(=Aup@H;EGHghe8_1s<MvuyR&<yc0|wY=-YePf#r37sH;6s<fd<j
zo!_S0Ux`qkcOf||l6ln>k15%%3;pM`PQ7#Bh~c&DuIx2C6_4(>dwf97HOx_Q*Siij
z*ZrI`9&^rMTKbh^n)32`vE?g!J7VWtZHwrf)^T)0Yv;zN7v_3Q@HMktHi0XA!IO70
z0=Ktbznm>FC2?Z=74aQSlJ{6Hi(1>BdT`>*<h^&##r*%<;e6iIQ@h&ur$E|u-@BJf
zpFNJd`^4?tv#`9!ZF$ea-d)p;-G0$F>)5-@Y0sRmrrqIHyW47%5~Q|g8_&zfvDca2
zKd+s&F7TdC^>e+`Mo*?n<uBt&oa4nf;X>BJ*}Gq+w_lK+z1Vn`@adb2?EhHbpY!7J
zqAKA(6P^A{Hd(rQOVB*7dvnfSUuZ71WwK-QM(;=aD;|qKWGQ^4a?VKST<zoVCoNm%
zeN(W|VXpJLA(G}<r}yg3l8*|{JeQt3+0kp%QDE0{_u+>(g<Y)Yo1Z2sOwW=(pV+-L
zVp&|#s$;K|*seW|ex&(cR4e|`y1GZ#Uu7+-?0E6?*|#;1cEl-oKRK^=%;JCBg_~^x
zk9A*0h6%ng_kVmM$@P~-)X{atj0FM5*ye2eSk&I3(DC!J<g3Y(Di$U(dP~gOIsfv`
z_7_oYKi1A*kPXtE)}nH`;{$8B!Q}Q!Ru}EUb_Lt+XnlHWhaq3x-qWsz{IB1d{e2qG
zmlx0WEV->T#VzkjX<pj1XB{&(F@`_u-f+2|FZQc|)Ff5ux99R^?1{aV%FDPSHYw_;
z=DexNn>wWWzTJO&cfphpkGT<Fn`im7ra!-HSH|`|uRi~4+*RgSj-^{USM7OT(`QnW
z)bU?!+5bMy6LKFS?#^r}TiE_XtoGKD+np~&H~NZSbklZuIqim6MY7ZMV}^WvnL_M|
z4P1%;>-JieKX@6&5YZUH*EoaMh$E}H_2Z(xhpnZBO6R_HPJJl1@8mMKpE7-~IySu0
zOTSWa>f7pR5$&Sq{~h~z?N-yZSsb5lR7O-iUN4p{ES0_RedX4=M|0HG*2Qg_qARtd
zFH%?U&C2;5Y=<IupUZ9+NN&9`$>5*M{L;@ReZ{8hvRgM4Gj8~-nUG^Uzw6tI&1MS%
zY~_~6?AbZ5n(MS(xy&ca*&Lm^A9AXsSlO99ws}3RSbO4jSViT>jyrv}BG#p`%Zw*I
zxf46-@~h;N=BwXb(S7#ZEzNH0*Ofp1oWK6HgQNCk-@B;mf3C2XO&2LkD*v10U)K5J
z-Mh3SUOxLCoOs;%_VfRHU*ECtrWgH^T4EEoxFoLU`M$^b_g>F<zoM~Y$=V|S)_-3s
zZq2#iwXS7zX%I(+{j?Rl@nNSnXBy67^yK>TRQ*6t%%6$ZwNtiR&YW{W`{N^%!@|>l
zbnUOWxU2U?O5)2a|NGUmmaugQ#afjJ8U!>rws5hQM7-F*EZ!<8Y2|Wb!=t0^qMBY?
zRv0LsXl2zjW3d!mbhMMnuj@<1q@~Aa@TgDA@SLQ4bQW)R(G87+)8`jBv~%fNrKVo!
zaG0ue_fRW$`%+J9CeaI*)h|u;-)i^v)mLqCzl6hWZx5~H7T=u6THsW*?QQh7B5TeC
z-B%Uw29=(k^62$;{RjJ-n1$tXb`&K3-x$a0lkleEkjRO0b$uP@Wjv{jDFTzSmQI+$
zV8-pP%c5Il>=ecn(j0f9Ys)iVrj+!sh^c3mg>4mhw{GdIs}g1ZUMao_k8b$*X!2jp
zl)ei;-QwS7y-d4$swzDw*JI+#bFF8-e@Hztx!&rZ&Ge6=hSyW{57-`>^-1*H`kfc{
zG5q-+dMVzcIGOWLX~@Ke<B}B$P3+f>x7NJeyIe%v-BaU$h;V2{Vv9&xhhn>6)eXfC
z`L2qE3?V5yf?74)LN0Vg6}C?BGW@lDVvj+Y$iyCtdmU5ybd|0gt#nwXo+9ZoEznKf
zeb&K~6FtmMKAGtAjWty=D6CR$^7N(Q386tghDXKz=W3{^IpryYaLs(pdEt8XUtO0b
zr|_+sQcf#WBPPz}I%qbVVXd#*Y=*TrlKi#jX&yPHw&u#@IZnHIpLa9-yOHd@<lm2^
ze*I}lt6nZ`F0*>MWZjKRUeZe@N!>`T%{|H39AIb>uv&oCLVFF<$*KS+d%Y<a*U8t0
z2y{#Ix?E_`xM>l*QC)L(j@Mba*`Z;_oT49WVGgSf?~OPZa&g11XXO+8enz@32#}Ex
zy%-oh<M(B@m+QE%?tCA#W71CltK3tAKU|7lxGx~nTJS)M?a~<sQr4<mJE*O7>{=MN
z^&NvPWhYxF9O2&Ty3wi9@@DuX8ToI9NecC6mPGhzvPhnCzZ)sJ^}lp(Slj%CiBHqB
z7aw^eFqdQEHuY48iB4Xj_FFHVbK1Suj7|H3B*#kKw<f-G^Vg=Ro1S?*apTdek?rhq
zRk8V<aSiV}b@gf<9lJiiZt~0W&|BqO=ia>i<K)|A&c*X%1Me_g`n%14#>3i}d+F_S
z?_byQt@yD=C~rqaM{oa+9gn+~pJ?s0QEPb|cJJA%3m#iJZ!CTq-DbC|PdYL3$<&vD
zH?L1z^IJ4^^19%YOI}S2jje8U&wa3zA!*m9i8Bte-%^|X=HU(DIqB->XWrKip7!}n
z?W@kI3%<Qc%UCdb1LLa&cXu1Fl3up0QEk<<MsWqs7>6TS4FWsmSO5BWTk5~x-15Zx
zfB!3QwGDWF`(4z6Eq<4DXMXWgQwu!7P$(qSlhYC|&$wWH+#?}Yy9<%WCWWzTYB;j$
zajSD46x~tu=wPeB(iT_4#9e+F(xO67W3)pe18OF%-FRU`TXN!Qk8?s3cN}ol%U#GV
z@Osn4F3l%N#ezwbtkgD5Ff8m;UL6$fCnhPe&c}P}%X4Km%4(MnwDHDniOe|g!c}jT
zTKXbJ-n<_z6BaP8Qp(j4$!AG0d646nRd$6>{lMa_wn@3!t)HW=AGx|<<IS@<ZEMae
zem7g}a%{On&Enag*SOA?ytHD{#62o%58JkdwfdHDcs^7YZQ2oN<mWu8Sw_qGiO9)=
z8AVgI1z*gYs{8+UpqksOpUN+ac20`!>vLEk9W>z_^X|lb=Xe@Dg$y^BsNV4Pw2k~Y
zNkl4m=EG1IeTFU$$!Y2HbF!Ar=H3`Q&*<gZxhpr$u?gl_5MikCN@t1a>=_59T@qIa
zTdQC9r6kn$`A=o`JioR{Tc-W0Nj=CDx1pJJt`Ga-P-UKflVjw*$h+uvdS<1rZWNjN
zBIVWcz{0ObW@=A48?bG{y0X76OWqWP?}?GnU%8+pUhi#qh|Ht)YmS`I<~SU4kfSWP
z#W1IxH(Mg=iICu~KLSy1A)SJgyByj@+&X=0RteAc;L=!UlsZvlvb*Nuxz4iB)6QF~
zr+R!gUNAw;{Kn+utYZp{>pK7UIp${dD@bOZ_}wAw*tmN3y*q3t_h@yjpAs1z`?$s2
z@?!f2j;wp}H{EtbioR=IRP#ndSUoaql19<WJ!?AE%q`_o`J%R5k#X`=zjdj1+ZNHI
zKCgCd+jV2(H;)rFca4&*t#4mE=NJ4y*6GL-$uob>`BZ)6`F7z;*mfx~KjBPY3+8CS
z4PNi{s*_xGJ1+BCC_R~*H~0Kqi$n5tb$P1d3!k{GGYV$Z;a7CuCDYGzWzCPoBMZ3i
zUSl#%a%c>i(D-)i`WpFL+DW|y`^0=QPXBwYlaczNS)4JP<L6E{F*eoM=!}Q{*Gv^f
zj3hKPeHL;Gc3oa4kYgDC@J+bfv&(YtK6L!Iwb{u3(a2CV+$HIFk5<n@DWjmA)UIct
z97)E`w|4)~@nw=Yns$ccSXaTuw9A(zrB?4g^wi45EOlvKzK9bW>kZC{H<!M?pSNb&
zv!_37^BmSXpZbt$KIIT^+O*X^AD?-+{|erj9cJt>Rr|T+;@pjMa$Zl8WjZY48#ZO>
z8lA^_)m{$8E2dhkirJ8ox%Pb0>7uT%Eqb2PnM+iD9c!=&s_K~eQrk56vi~m=<KW*X
z7i8`%w^;UU<>>`+Mlr3kAHO`Ac4EU-D_QnA97^ixMw``x)&!e#O`OH@h*SAVZ@Awv
zF`d0_f9HKpQmcF%J6SqL@a>1hXGWK=ac}#|e9*%F^rG*%|Gzl1Xaq7yP1(nF*W<aK
zWP*>@js{D|g1-@_LJbo?s7qFT@c3MNiR1O{L%jF4?~2;RI*Cn2Q+sb%NAb#FzQ-94
z?P@FhTQ!t-rJU{%3f4Mir03EiQ|dcMtS2~G=hFe5p03zsK4I@aZ8}%5)oj(t(k73&
z=`$9q`K0%LnsPAs)ns*#GpTupr(gXt&)P#;aqSM*&!_$<Y>-_&p}aPwj5||d#pAE1
zmY*}4CE8i4qkLk+x#Z0c-u1{a$)}&6#G9JJuKQwZ`3cRbv$uFPxR&L9H9A{Rz9qAK
z%HlN)o5~y(#pcAn-nH+~UQbW0KNFX6?S9Xoe`du#!`EILB>g6ZD;7AN4f{XY`}qsm
z<zn9qtV%r<8|^-&);v@$-aN5ac>R3MTU-;jX>Z6DHT1q6n)#?<Ms$AMSF;Ou1lQ!3
zzdk*o<oESv&Jy!a!b={M<=<Y&?kW~0aqDfqe1yVhZt;)d8BX<&cP|lNHo;TfMxQJH
ze~X$bS8!U1v-BqW<pR!zUEV!Oeg4Z<2ys99J8{K-mA*ay15A~|O#1sb@}yq5Q?{h-
zjKs|>o|#S#=O=UqihJHvTOv65rH}PVwhV@&k-9F@9*h+|{n{SUGj}%0mIzOO>3eI(
zJoUSc&!!&IUBTsa$LYOGOM1tHEpw;8@r`xOo!2MJuKH0y;MUZ+hSRFn&You*rlI<9
zkH-InHv_MA%n%Z3>d-7asP)jJtAue;Y`560yLU1l@;|$K_o&c4Mqxe9$vKU;B->_9
z^$9h$<e15muwVzHOp<8Ymagk7Su!%%i&!7_e@&E@y?LfW?utf>ZpS^bDUR9!($cfS
zU8XqsmbAy;>DOeLn=rMJV_E;4p7tjXXUyEexO(sL_fwkAMarH3slX^Ip(WP+K2c<|
zLwwEO^9?hl7k3KR7xrmdg{;4EYWtn8H9qlSCHD_lM(m#}e{kXz5rI2D1Z51lln)5r
z+0`PnFU8OIrpIB!m_s~0hyI8h>5MqC)%9p8OQ2`Wu^#2Ww%tc(P8V9b`2X7EGgFdQ
zUhPy%V-xH3WWM%)abv~Rj0uc6F~0Lydn2|wl}~76`{kltk)YYKKzo6w%OAg0I}TN@
z?Ua4N`gW$LtIuq&j#JLtF6(Y#%9_ZybcbW(hoibD60d$-=vj1kPT<{B8|1w6T)g=n
z&Mlqfsl<27^`UUowA7UyPeW!2KDw(^xbRow!hkl#(<`pMyyEwI!NS)XaaUF1vO~|V
z{L7cd(SMQGUvdlMvX61T2Rh%D9QV8wsa+#?vE?4igr?V_OD`Qfv$gOb$FXG26|)-t
z_&u$hx#_Eme?W?4Nr=XyxGjlO<2<5e=PgQblGkawy<Idd(M~>TSyH_s$Igf0TPmXK
zRMVXPE^&Bz{_9=;)j12DZEiHR{=d@vaJEP5{fs8gM+Nbj^X~s>oo|#hBjAJkfdJ)*
zf>vKc8M8?ehaaT<SsG#aDn9V${C}Buj>=9v7TMXe_Hppqp0{_BXYHBe+;A*=hFfvx
z#f5J;CHA&uJ#d?MVh-!SS*)jyvHhDRvO#sJ#gmFHjv8qy;wu*1uejtC@#M8)?DVOP
zJXMb-3e2s#aBmg|L&Bmb;a!(q|1NL1=h-ZfxO&d<Ce@6z8FPj7Seo?~`dTe)@k{J@
zv827=_Ld{dlKZ0X1wPh$IjP<6)}>r2gFwzsvy;otP77YdIi<({Wp0Zp?|JDYw~qzE
zn{tJnRQ$6KEHpoRbKalC@+8;1ntAILTeP=0B^Ugc44J~TbVd8Z1?n?D#%*%+?%C%a
zWyJAjr@MNZ=F>u@m{lpeJm-HoJ9max+AK-6H5@avy_U@|i?7+6*~sht?d#=+&@`U|
zw^yI=dKsX2nBk~e#k2YA0z?0-G-y+s!H{fdq`c_a?Pk?1<3%C8rmZG<uKjO9Op}zC
zl_dvQo>}I^6LeJaXlL?@Z_f(sy#EFEB(~fsT=yzoBi-Nbxzmh`Stop49sG-1MKz~{
z@&vY}C&VoX@RE_7E0V(?-m)MqhpkF4I&_;8ht=g7iASY58fLCc@b$^Q6Szq2k%xNY
z+0~z>UYXgq-EGq4mA%)_PHo(9Z>N<%XT{w+5*o>=uh-2>5&rgHGFA1Q?<QGMKPlal
zvEpUfcMr<y3#xWpyW77+Y_f>lJC?Y~K37u>RD-$XdbgcC*%&!lrgKW3%ddoaGcx8i
zJW~GZAzrHJvU`G~k{t)ThOCE>n6n!Ho1aIbXF0xG$S*#DK{7(#_dtKxu_tM6E(z~$
z#OmBH-10JUosyng&~C}K7o`+GCaPVIl3%l-%l}xwg_dRML2DN-d$zDDK6BRng-L6#
z*U5ihIibxh@b|?Dk!xC__OzPKI(qZad;!LEi(_vEAKtfMn|SOl>*{B^Cw8oA%xTa2
z$-?ZUE|GXXTj7XAt6;_!rqZB<$&8Oaym-8xG269)Q?jLh)2HX$&t}LbNHNa(&!=Y^
zbn5Mm9lPGd@n$r=S@3*Do`BejbElq^3$!*BJ#Vzpczf%0i;z@n+_ctDGsX3;zBCKj
z`oQBrSJ;L-u^VLcm$C{={u7c~l_6Dk$1TDC<R|q$&v|yWlc$`$5^^!CKWpCFi?<pl
z2v2wwrG4e_3JVwQH++&)BzeC1g{1{+PkFG0V^UgGnDR;CQxE&M);yZ>t2;GDwyt-j
zf6Dv|QqMLEY^df`TX(}_)~-c!*S(w@B*heaA$6DZg?ULkZseq$)4wv!XZ|_11$<ed
zA-W6W-Wrza&d71vEuqCBzA7^=*Pu%NK=1v%S2lE~XxzLs<#+0<!v%B9_N}#;lqJ~t
zs;jh5Q1ZW_W!@a2Y@ZV=w2D5l-k+uYEv)m(kyT$#yo$ZC>bO!~WN!Mr#Dw)$H?utQ
zta<OqPwZ{3nJK%&N!sGX%(|sB6ve{&BHtCx^_sR$Wc$f$j0P{?eVqEY{VJ=`4nd7w
zzY3Mka<T7t-Pq;Vb~HQJ<Wtk7)hgZ^`@%d#*R`B%=)02|->~na|GT*dKAq70ILX|r
zphGXTlj%~<RlmcRraj!dX=PxlPWAGYd!wEz_MdKGoAqYltG4$SOn3_<xx_0H%yQLw
z&Uno%Ofjt2P8Z5eHR;?c!}-QlI%{M8XPF3btK#ikF8bxg@2?*F#MpgVsO9ps;=ivv
zG`F|rG+Ys5$?^=<<*NU`)@J{PjC~tyt91?f;&(=D+RJ*TJuvHL*^LMbpRR>7?mUyf
zdr9}$jko`J0}I01?zxrZKACZz?ZKMF*0p&o5tS1kCbf#QYyQe=o8Pz5KZns~?woB0
zv*ozwI@LsK%AMTub@>sgm$F+<%0>kLNO;<QZ=?D4n{_|l9o_Wk`rIc?8k-vL3E3y{
zzOgb2^U<C0W|gPo*FCFvG9RrAeZ0DL-(At5TiUtj=E$88-*#2w`K1r9I6h@<V$oke
zanAA785y7Jq-IOV%DkHvn)Gzd0=LSCiSredj+d=JedShc$H$C6C-ucwUYy8i=snYn
z_ja07vCMwX#uuutS>ozqb5$orx*u619sj>=;SRZ&NiUg$3t0YheYJhI(I8~$+sO@Q
zGMRt7Fott~YMS=Gs4uC@h3z_5-t^DY{!Gc~aLs3LwPybwz_QB2(XH_Q=8Ud1&glh3
z{b>efs}`v(R=sYp|I3jjQQPg4U-5DGEUU}1WV^azoy4)VH_Y3<uRND4v^v@4{oBN^
zTm3i4Fhv!*Ul*0~UGPTlh(t)LRHvxq<bVx-^><c9ri<TAiD}z1i}SqSw)+N4+;?u=
za&2C!$5WRdXJ;;)6Ytw*ohIvTefKP1$2QI)|JR{6QdbA+Z@Q}S)nBDF(8rC_^Pt#{
z9o^+>!HZ*m_<q{^NJVRAN7&-h?PZ%%U)>FO7W*;EPjm7A&w)~JmT%n@tYsOJB)a3$
zSLa(ADwXYPIz8RKYw#O|=VtU@_6a|sWOi~Er`}=Cnc;HbLKcmi{rsnN-B(G9Z+R3w
zZz<=j&Z?B?)C-kW9#5n7(q~p=bo{qmaNYMZ->$s!4^bVi{A-Qp|9RQW`Q87<=jkq1
zMb_J|9{X(TU*%DIpELNx-U<(w86G!pr5!9`y}gW;|7XB$u@YzYi}}Bnbuw-&SobSs
z&Gw$JF|Nh)iqCC#eo~da;n*4}J(hbt;*ReMzhqtbFw4KdX+y8qsfBE50gGJSqFwxM
zpT5&r<x>6DP$?<#$>#LmP9D))i=R~at)D0Cx1f1z`~QlI|K9Doe{{~e|I)9g{n(w?
zDaY#?bpLly{*izqKf1qH_pBETz0DTt?Gf61KXkQh-}`q9YZC>U!e8`!^_$dJ#c{Gd
zj_-4o)Uo4Y!TX-y4_g|;AO6duCGlm<v&t{=k58Vw&bH2yrTApq<rb3}|NWG@6nsuU
zP2K-<)`T^(8pc)I9G_Y}DX(yg-|=(x&d{1C^SoytlAglE*%6s8t+_Q#MRiehl)3i|
zA&~~wq8w=!d(-pb>KxgNkLbv;XP@8ZA>jRd#jU?9{zlK*eEa>k@*if0*BFbhDYkAd
zC~ZySbpKUY@0q;hSy6Bm*Lnln^&5)j-#3~kA{w-P?*+A+L93R2`oLejn2+<SkaFuC
zzW-NOr`xTG-}g*lVu77u(v4rDt(-OISFM>AXuIm)sgjRZ@>p_P3gwlmT}2+fXjPc$
zX`OZ?-+S}Ns$ctD%h)#?zpPBO_PCTf!*w--#@}Wi>Ar>6Zix2zq`941Ytpb`^~;*t
z-M^O?ecZBh`jXc6xqG_jc0L!pcB5QMv-DcSm9BjGp8Y>^zwX}0z31Td{h4f13qv<`
zq=-F@(QqgbR0wc5*u=ss=d)u%qH{YlFV_@~!X(#@Ic(D=t}tADR8NF8!R5qK9`#;T
z|5+-TlX$!)EjhKNMRP%KgAr5I%`26gr<|RwdSc20r$Y)Cn`Brfcxr73@ip_5cbjUp
z^??Rs*h;(D8>jw<?GCf#oi_1MYqq9QN|sp3<5x?x8JC{gQpCJ{iuCM^yZ1`86V~1p
zO8vu>wt7Q?q;bP(j=$NPkGSh~$$Pi(L^eF`n7T6U+Mb!O&K^p%Z1;N+S#qU6R%*H1
z-l)>-YcVGZ9yJAiQ)Ap#*nVYh_2p^zn6@#@-<0b0@saQn33<Dk?)*n*7n;ZatNB&(
z;=sc2`v$G2MHugFFW}Ul=w12w`NieaW9$CrG$}ckEvyYJKloF^`sKX&^854tG5zBf
zaA^J{yx{<&<SBi{U8fQ*9As0hnV@$r>5+%l(xmLxDRUw!HFTRj1Rb;mTzLf58J%P{
z9&WlJ(aN)?VfHFDqbBVRk!YF!O0G@11v$-2jB+#=A5_SH)y=JZebZvS@H+xQE9c$X
zz^!zELyLK9Yl~Iql3uq{Cl&~?1iU$YMqt*eGpC(?Ypl}q_TH>$D&ADAWqDtR<IG8=
z!@=iux2hxyOJ%a?UC{l*VzNUjxu=j@lP~qkMK^xYbqX$>v#;!ax+7uAZtvZ{*Io9t
zy}M<1psQ}E(mOV%soYv_N*sz-w_}8TCv0O`$=#QAcIx72)<;{7VoG;kRZ$GJTB$nC
zPbtjf64$Q_3${Ga>fl+_;?$+FWQx*8P2uII#I)M>@LpFaT;KK8jPce%!-aEhNoX$*
zyA@%yY%5E`q?18n+O5n77Uc$QJi-w6X3H_Q|4g43YP&X1?&jm#w8lW2w`uaZPUUG`
zCJ|@P?nvxSo_k~K6(MKcC;J`FebTv+vbdM~c2@UWv+0G|@3uKtN9Q%YU*>Sm;^Dm2
zH+Md)*_PMzbX&2W#r*9WRb6k7JZ7+2e)@4ksGfxZzvgE)FaN!to@nc<tiS#{YQcv0
zubme7o#}i3X3NEQ{K~)CLIfN;9_V?z+szsC;-H_>o9IJqnkpwAtn~Q#VWlXql;Og|
zu0AoFmwOa`^!JkH$~(LyXLVrghAhDYVnJ<Xr}fMitr0bxzvGRE_j#qu9varH*K<Bx
zT#~iP^O8;RV?JXcsX38-rN7oZdeB*(QPy+3hwHn3;$K^nro;bP)|StXT3>fwXSF`(
zg_GQ>Q+~e}_SFdTke%BU=*lY<f9cf<<#(6gJE?}&1;3nMfAu$e($%mt4b!hK`kyI!
z;6G!w@&`drPp)L4#HA^bTrMY>{J$t_zEn|n;tEcgP&7rxl*9S)t9a)<sR9x=J&eO1
zcG+fq6y31pwXTy=WRl8cb;*7248?0?-<p+8Ha0v`IV13L%q$bt6&%a-n+-emF1d1;
z-;1@<`w7pnStkTEu4($L+4$&qgt5_;H7aHsXZDH*m#zEx;h3f4;&V%j)LIvcF0b{d
z?kl-<eBTR~<7so4&ac=q&E59h#q7P0T5jH1ylLagmv;Xy_Sbw7-p;sFH=lje|LA){
zEGOIk^iHZvKG8l$T=;3FfNO#v!y_T4Q~r7?E^8zbHHD43L{#7TD~l|2&N{TVDaSAW
zVC3CtJYqAORQ0RMy6!zl3*Xavr)9>WFFum*6vV^t9IULp*s1$1XoB^Z7EkLrq5s!N
zx~dx;nI*YdZFbf6qgQO)<`_?TsJr!er_<k;)whj97HMo>;Wjg$%gbQCjp$QZ&$pi~
zXT9qC{YFBo;nw=PWnNcT3UCP>(h;1)o6EtV{z&_@he){STT!dlFUnIiR;yZNI!jM`
zeljETNK`zR*GI1JYuhJeD=M-ms_%d574CP`?4_26HP<0M&71}il?tBCJZvEk4WAjO
z%t)B?f6DZ=vr8U?+0-;gTb**=x#f{c{4P(wPfWI2+dOA;zI<iY&K`C-WLb)mw)V+0
zzb+rsd950&a#TF{*-`t%ly%R4_2#RYa=v=u!zJY5cdl)5_=2x*de6^vx%l9P+xr}+
zOy}%dmv=aCJ>k@w^>}5}l}%}KH&dqBzGu9he*9bHZv`WV9Obo{&x^P1eE4?a+Ss)!
zAKAnwhfH!bc-d(npQL#9#H7N#rFXO<w@l#cJ+b{@Z_M{g)9*ChXcN&c5)9i{q{LW~
z7Zenbl*c$ZI{d~$V{1pB$raDfw5)k5s>`SzziQ?b>laJ(UH0T<ta@_wkVejmlH03l
zgoW-uG(4@eY0{(5&szT5GS0s|XWiq;o&tA{?p(LIn)jTH^Re)KPi}k;oO(HZ>5XvN
zCcZ5TSiWvp5wX;sA!zAr{+E|#{;HPjK9H`Yv+R7Aagxw%k?vzWF(-YqHtk7keX1TJ
z?7&vdIeUqQ>QdP$(;`BDotkOoQfRr=)ix+&!knB$rfpl_&$QXReydJD%Vj5hL(j`1
zi~^~r%^Vl){bwY8A!WMq=_AvfZp2L3b7gf$*3B8t%YMY$e(pbSbLUm~N0~4O`-z>R
zyB{r#UVQZ01Q)dwh1SK2YyCWT`&{}JdGkzFvUHu8dvER6E%7%kBCOKX`&wm7OKn8u
zK2)sge=6xOr+?c0%a-<Qn?0Y@iA0{T|NqF{ZtuJEjZdZt>|u<a*f%A`G2^xQfv07P
zTTk&s>4mSmTeR$?PSv`i#YW0!?y8-d<2_&GmV)@gUpwQ<Gdv|P6<LMfahiGAr%QIm
zla!RlE6$2sTsx;ZeM#$@*)#l~cJLkTHuy8^MyIOVrnD`YADZu;Y<ntl<LjpKHAV+o
zgqJj&Y%)BT;N+U6_#|4ycB%p^*Zt2m(~Nd5d$LorVcnj?y8jED+*@aUZku}K`kc@%
z(N9_(7tBj{y{dFgowKHQ(dNbBz8+CpCnOqod{MX|Rm3eVE;>!EWoc;pOQ$u9GFPr&
zt^MMYN`Z<4S4Hs>)q;~6oE<l~-ic)zWM?)9+866v&1U_-H~*$lj@QO}^>;6*xu*st
z{YWa)yxF}%f6Ao={PQY~+WTM3+N;EK+;h`$pVb%px794!9g*pIdzV?6?euar$MYt$
z{F%HyXs@_x5%J(l+7-Lv+Qd#ax$n7+o3?I$lcGL*>*}ph*;}7EU3aqH_DiKGxB1D}
zkG!8=Cz~5x@Ak3F`MNB}D{I-R^L^VMTZK%1Xw|29HIAcjC;wCjUo9=|Uzdv(|1Qnc
zT*wn3I!SujtGq9_`tCayZJH2O*xl3TeShf=)|gN0<6{nrvOFqrd(6x7VC5Sp<vR-)
zS3j9ipvkn>WwqETpM><)X?Y7v^!C3wy<Z}!jD<bfXhL3tW|`dv#dl8(|64FT`gY`i
zACnqayNFlgBM-;9H%=bBbF%t^`pzesmTEanPUpU8@XY6X8swF0xv%-@ipNi5*40J0
zhO(?+e(23*S)gQMz?I0eh(n`+??rCRBkqZ=$8Ic*lRDHSbZB``(wQZy(P^uBJr4fv
z(K)G?!Y9q-HtkKM#G{s7xzo%P-(2hCKD1WtmczSuPjzOoDo)9aoAJa`EP&x!p69P~
zx^Gw+Z}ye9t$cro<%7z3zbzfYp3A1M%f8@u_=3m8Bqraa48IFC94p!K&YP*74-*ty
ze`1bYUYpQj)q=uDYJ8Ux1U{R*zs8iA+G1$l(mc;@?fRAgMxXPSoz7;|EvtRN(YJ=#
z;{Orm+C9n^O5Qe$(l({6mA>0-ThseFFIT4Y>;#F<8*OWET>0oQ<^0w|N;mUXx6e50
z{-W*W5q8%(uEs8$JC-!bM?Z1pae6JFsu=X@M1%6*1FnpZ4ypHe>Djcaok(DKX~<xk
zeD_Y1%BS@P6IfP1I4`>5lv-7i=eq?AXBnn{dOhvHky1V<Z;5w)0&c#2&lj2;esC}Q
z;{>0FNy}ALC-_%PQsrmWG)o9FI{(OP;iH@h2MydGv#IWsJIemr^NETA=b|Dl_jiG9
z&FKMr!CNLBc5}Tt<AXN8R(;jeQx+GPMc3)9G-xQ&<EmY%q_ax-!6tRjrtoR5S_clb
zwjDa2w`j=*SB3vv&$m4)c+InE={mk;Zw!s+DYZl_6nn)OTXf3p(7Crgr*v0%dOm8}
z{pOug`^u~rtb6&A4Le@y$Gk9Pb$EXz>p|S7^EpMPk(Zg)7pSstHfT^%ODm32X=2XE
z&tzEr#mk>jWP!P`gJiPHT)p?7%<|W@Y&6r$6;5+bGgG|sqsx5aM)ReUByDrK)e<w7
zC~ru3FS+#br6%W1Y=<(B34h;an6*spS`x3V#cS!t_t|fla{{s*uC<w6QtSETymiWX
zhXwq3pEldapH};k_RGjwb@x%Xd#wE1^N(Ax6n1I9QrM#4()mh_{kBAsu+I_KB|47F
zvty<-@Dv?-GbvMV!AV|*i^cz+oa6B>{=ee+G%jbot2&)o8C6Xur!Q-I=&?cP=eyZC
z>!xp5!gcbJyYeD8S*PhG?{r$1`;>(oeevwxo|aM%W}hjCGA$CA%p92192hs5c&jQ{
z)EhJI3Sn##_Bpx5+uJ{Yqg6P~U~a*M>Y@w&4>&DmZJOuBueJ1(-@+me?KSMbUvli+
z#JN8x<i;kG313W?3M^_8;ofPWSDLJxcIH?>lNt|`*NF@2RXV(#O2L!YC@fSm<P?r^
zd2FzH5np0v@O+(Co0VcEPds80l?s)PE96HuI2oQfpnsxs&5@Vx?ay27J|^fa^V3M{
z3NVa_6kO@{G2-W%PPd77eytI?-gCZ$FY>?Op(ZVtsH7&IgvFZ*imfL+{?f9+yf&Cq
z#6j!m)|lg=lIKl1k8L&g^^aXWS!(hki-Rw+X9QaoCR?6omG1rcW!dNG(^Ji-PmF(`
zX|ZY2l{L<3SDvfCU6Hlm8Ry=>>|2}rwUm3x-d+<}DRVl|;as4seCLgC#j^3;+st0-
zWlegu?EtT2l9TJ<-&^uSRzKP?ZMEVSy|`kgOJ)zXuPk+*?aFj+k={kmf~rp?9!=Au
zCS`;d>E&y%>IZ~#HhXlI%$|2Grs(9eiZ7<Cy}a!f&p2UZQ&h~Xmf*ehi~0T#rhOqk
zho|_Q-R5)kn$PJFt?S!zk8gFmv6Ahy!sC*H$48poKA9>0e^zqnO^Mcfk%GlRT$36K
zmM+lzDe`pFBq?UD0BaqckH-_=aQnH0YRSCfS{tDIJze+ir{^};y$rbSdVCJOe#Wd?
zJ^GS>b5ejNPdi`eDY><JJ61g6i(Mruby=IEh-q6wm~FDrgEa{aTBhP$eqLsx@7I`4
z{9aM`>Djd7>D_%7uNY@uS`j_@qxob7#-`xdgCQ|jr&g~FX1Fue>ge_AljX6er^lSV
zUZa}uHH_`^Jub`FSIxuR>gpeV?mNuBG+6RxY0dRgEBF4uS&PjjTpR9Zq=|@GEKoC-
z&u&!Z-n{6U&8^Gd?<6Z+du;nZW6Ql`6O@)sFcEX`$o>&=+TL!)#6Q9<{}osAu$efm
zXp)Qlt`ktE7&Wa><%)ush|^yFKW^sHY1<T?p7`szw+miUuv#v<L_t%BWk%06x6+fQ
z%a`xdn!pmMqibB=Io;%W))lVF-;Ov0sLd|ZH7uW@a4db{mY2&Gc+E{?+F+8ib4$+s
zX}NbT7+DzlB$YUMn)<j8<t`WU`B>3^c1r)(9sR7^3(g<+&oVIMlC0UYT<UOh-m@$7
zS|${*n-_c(@voiWZ?4BFc)gJM>(fOaD!AVT%rxnW<q53zYY=#GEI?BAk!oDuxy7d#
zcsUP62TeZ_^@2A(>(!B0e7pVmvKd=TwuuI5Xe1w6zTxWfNlh9hg)a{&1#7N9ov`!t
z|8=5fKGP!uKUTDEjaY3N8Gdo{_L*AKu2z_S7wYbuQ6LyC8CJFP>*Yxw{f?V5uA3gC
z{J`(7WX<!NeU-rs*S0H&FwCFoFrRzTeA%YCCru}CRxME7xnOY+Ls!Mlii>G?n=LCJ
z$EYn-=wYtET3P=$)avi6g1@KCbIN#(&RPcPTU~e_u&deTbVaa`gt>fKztQSzWw~Zg
znlA}yPO=lb-r;*=#`|SgH_AR|yQ!loAH8bT5mf`hEs7!66en4ktkX@KEA`syWpqxn
zc1y&mX>AJk_?G-wFT!Tv@yAEAfZe%orLyd%$kq_8Mc-JGRoBkk)@5DLb9Baxwlc5a
zUp<#4a!y|J`G0&{?|%uOr42k?Gy2|2)QGnh9NgZ=U%f#%X#=~o&&QknpCuOZS8Zav
zUcjByz@2(&M|<|lLIrWxh4$z2Z-*A#4t+B3QvUy4kL2F6na{RzYArOE3{WheI5mm+
zX@Oe4-uC%3n01wpTAAzJ-=zDt*I!E|>QU>A2TIY6Z?b0fDyC$1CYfucoKnmZUtXZK
ziz|tFn}XhgPQ%<r(Q}JL12|^9J;=9%)$Bp5Xw*UNeT(Ggy(xD1a`M`4?XGZ>Dvpl@
zw*wY;MHOD`$h|x#)4!^n&2oi!g5j+LtBsi!C2+eX^0*~(FH&G)aX6?t=a6U=*Oms6
zEec#)5=4AF7e+S9ny6U({}#z`<m-H4rTMH2jtC$6@=l`Qv87hB&7q}}j;S^r)3o6%
zsySY0bI3AAn8iU@=FZ~iV{vnX^S*Dr%n(-p|Et7LQL86kKiyc_)AuJp%eAVpIa6du
z<IX$Z<Zd)7Mkr<J+BN&<%v+z=)_?ugWIJi4mWl7}W4INKELucWoQ^R^*N5c?@6z5n
z@$rYi9V%PfqRX@-+^0y1T-cndurYU8MER=XX}gwXTX-E5+t(zTqu4s7Nn0*=+MHIN
zoQIdS=U$3Fc<HU9^PcFv<|?ZWDb2pP>FVs?3O~+Wy=c_M9dNwqcxcgPuk|81AFFa|
zYqi#H?|*Eu@L|M;?GBft8)YLDxti`w`G3rTE5ea`$$`VB3&mRIh(;s|XC(?pER@}F
zS1iYQ<5H(P?|1a6?%t$s&87D9*wTi>tPb2;5_no_L~bNFi(S^<qEIW<lf8A~L{X_H
z7i=edzEbeN%6Z#?M;td@b@H~x9|@TDDd>UwWYbCoG0S3I@$H$hf&Oofo-pEeOqwEX
z8Wh-cfHB*8i?n%Xa>kCWPfCM#FPt-#_jBpd^#<(1)02xs-Ui$%+qtVyDU>zA`q9yg
zyZ%m=Ia}r~tK*^Ad}z0J*FCv_7M?ZMmkUqrRh(3@Y39|?!t)cpR?JMDQ{I%~ua;F*
zeBk%g*qL)q^4^K}wK@9MQmZS1Q^|oVYoY8S1+HTY<o-MC65Fs>&gssrNJW=h2M=Z+
zJZ^kP{J?C{V-9yt%GPMd9G@9`T;1l7a{Y1H`G;!neA?<DY;^yi>>S<=b7iv{<s>E>
z)-IH<IVkUSUp_#7`6T5u`8OF!OX^q~4xF}I%>2zt_r11_)hX#&4~vs)*2ydlk6Jqa
z*;O6wq?61^%^$yOO#6If<Aeun$@UVmj?-jU&rjYczrs5)_wclw#tXlF`Msw$e4Cie
zTw}1x+<Ap^NouI4^ZBPvbrF{z?Os)^m6`Hr@{`#CI~HB4u5)8->RDU0)nj93syD;T
zz^!j?DqZz*(C%QWa}^12YGBZlvab5W5tQsUL78>R$B1Vsr;K95vYz}eDoXCum05P>
zjzMB`6Q{MP$BUoAPE8A(Yx!)oOja^*da|epTnI@HS>d<RF85bR*6|fV9-JLaVGNpK
zEUZ_AHXO~_YZ#cdQR--3h)5Wd+tw`8uB&RCOiL=}NL|wRI9R*b!%)VxTSPbNSjMW~
z(?nNCy~(=zx!iK|{<G76C*J#W^K)wTz9k-kMn~6A(QR4Y@Z`qyuOh0#Q|6>;IdTSH
z-nl(Tli9FhwQq{YM5(O?mW9nnW>3E9<n~xZZ{70n<$ZUnzP-J*yZB~P?c<VGY3I6@
z+I`AZU!Ga+3XlJ{_s@?n@zdTr-LIAWD--?dc}kEZm&(EhAug4G2Mx?>4GN8(Pt5)&
z<Oz8xHpVa6vQSX-;>_cT?Xnv`3U(;uzDSIHHgQFu_FIk%=Uinsip5HoUrqM5zIZdq
zUz+Wzav<jq%_m*DYh9kV*(Q0GbnB)nJei=qeaEJFPuag33{#~dBjel+XD$(x)T;Dk
zlZ@#&$ta$dXTUm3*kj|e7PePu3;gqUaVpMX(pVVSC?9x3q_w`)By@q?qnm<~Uo$4Q
zh<&|~%59z8b7<P4tr<d$+G{h6+Lpe_nJ7`jlp%ODTrflA=nAQsS`4d}sV(aio8dG|
z+<HxjTY&A}8&xyy_f;&@UcdX-<@4+JGX2iB-&-N3vnko;$AUbOO_~R0F)g`ykxj}r
z`eDnhSKt0Cs9T90Z*z3k3y;|;rBt4`^X$(yu~~0-ZDXBvf2a55l9($gYWu!7m9L-q
zJ@1OX_?#co_YZ{foe@05#K4iDo~Cz))qQTn%>r@JcpV3I?)03(W084l0uL)xubFsE
zv79C0r0mQ)ftqr1D;gLK57%62^yF2Y7_QjpA>L-&-m=h5iOXSnrG563%}ytBKP6q3
z7Znfe;ZnSEF;eR4quZ^?6N657X{xm>)O9*EEpP(=#EYkN*S>k)rp+~1WTN)cH{0W<
zMc3tA>Ni~aCFt&kiJ8HELLXc#0_-HGiaher)C&o1irYH%vAknk(o>nNNBt>Nb<?Ir
z&T^U9k{qNE7?ISbD)^sOXu<k@>PcO>7jLK?_k0$1{o9qt-Dx|SPFJVz>Av<mecx-j
zJ!uCfE4c~<@qB#{uwvuNStYVd!*1u^d34OC`RLR|rrHO4H+t?ku3I7_y|~AJj?<B)
zp?9}jE@Izb{^|I8xr*<R?*yxVJm<e({q5}bd7l_E?3kCFXE<Hp!0=E#HR__5(7Z{a
z2OKVF@~?RyrlGWDf=haH>m`K?^W5b4$~=`98whfK4;L4`dLm0)v03Dnf*IqMg<?*Z
zCNc;mnORRc<Q}T!ta)dH*^D#2Jd&QCoI3)T3}iR5Xa#akKlMO$az&=k-$xF6mTg#O
z7RcduGbxN|#?_-t7f(9WM{y;X91`dIFStrM+r<BijnH1ZuScC1u2K%!;$Y$-%ce6a
zz=!3PM59X2tNs?2ZpDup4dJ#~VOtior|JajT<{2*bmG@sRqf6-bqPWdv!0}=O)S;%
zEt?d+H9~H)h>L<=5_fb=OK=m<lLf4f?#^3MnwOdxvPI3xp1$jfz-6<8df$Arxjz)m
z_w9?EoO$&~*DtNf`{!}`W-y(xjoC0`rp`6Pwp^dtJg1+TxvOm|aPmo!(tdu*Chc6w
zqR(?Q-kdnYbYUWcjVt2^o=%o+1{2H5JksSgr7l{0nf9b)MT^`*kHTFpT+tC{v&^3y
zVpC|E(6KGiCuPIK)@3s$PCUS(rgl?9<fWPGu8r#dtxnHz@KrO|G4Df*hIzK;=XX0>
zpFL!oU$-#EL!mu*Rfei|IhUuj@^wYW#wW{bA9lI>P2aUdOjKt<)+!O1%_sKww1pkw
zmiw?yN~)fNAwu@0d-!H6o7GLBYZ8yHdY9C-cZ<@Cr3p2eXJ@1vGx4%A#9o|OsAe?r
z`3mls6Oyezr!2m{h2yf-``zdM@Oj&R+|ka(cxCy&H_VZIj0W}}I^U$Izn(9rRgnMg
z4|j9LsbGgnmRtp?TmJ%{EG%eT-TdP-m!o4+l%(4dSG}MPkwZ@ot2lk^P<xmwS#tY`
z-dvSVF2zQtZ>g@PiGo!nlO{_fo}A*~rN9--v!v8JR7>+pl1TmklBx0AHcYd7x@>vs
zm6dkV8TUTjnivxQvO|_Ja#hID@Q`=5PnO4R?Fu!xG;yoawDl8I*CfeuM=<~M&^Jk1
z(<c+b5a;@#p_la(hj7Fpfq5HRwR;ldr(AIK=?q{EoVg%knb6eum<i4MPI#}Fn8f|W
zDZK0Q*3S1F3^P6~3O8DBv6eH$Judvr^u4oQ>ht*+&*Ipec1Uj71{Gz=ld`jC966`C
zv7=Du6wi`L7N%{6rJH(EWw_p+@j9}g&t|Fi*1sMXtUej<I%3xBHu+)uFHbpsLto{W
z-Lv9FCtc{D^Hfpj!;1ObqQZU8_-8gbXFd$e5}B=-deJlU9_vYgRW7PiGCQMu{>zH*
zT%nP2O1JN!v4>Ole&a>qS5EE@bN8}!4_<9`wY5;|%`t|#dmOxkr)=~IJzMqqe2%D!
z%_oljDqV&I=G`v)_3lL!_GUlsy(+hM!b+cspWe9b4q1CC?Sj~jbi2nbt3{(U&iy!d
zJv_I0HqYzZ0l&}r+8<f<+xlT8f1TX#2$5s6>;E5?PGzo${Gm~h%(?o~iET!XwO)!b
z4?mr_;BmxJ@L1);>~{)T?koJm7KF9xdR>f+*>T!nre<9DmBQIkoQnEQ6L%@~3Cxp8
ziF0x-7q&?9R?2<I@6q2VyLN);GO<v}GLJ6B?2}(su5^-EyJn+ilT~H)>`(Wc_>A^7
zi=?cY_uo!yzrnH4{;Dau>0do!B8x<mZroLK{keOcQN&~pvEAX)XCA~(|9c>Vo%5DB
zPg&rp*vb5Isdu0EefQ{MGndiZdEfER_etytOnY_u`Xc^3I`ZFW#<7y?J7xEuNe+5m
z`#nj=WUl1dGrzMF<`@(m_d9krzs>H8@asJ<ZQaflCsoc}ef?8qs@S9%Gu<@|AD?DD
z{YJ2U-@UWb7I%l6eTkZ|XPT12FWc*@!c2pf1ae6y1baoEda>0{T61v*bJ&rZ;ERi%
z8d>!_Y3nhsEEDnlGnw=B5(8)MMHgp@I_j@qaaUqSz{Xz&s|;n6<8-rc`LdNCVU|lj
zaf&NyL&BbI)vOUpaiRaij+=#@n%0oGl=X^O?d9be!Y49Br^v81>K)$5TIVWL$C_6!
zn)kX{?6PxHTWG|~wx(MPBcu;TOij)2d#d>PSyQiT<kxR`|H@QLCr9c!it@8m%YUp}
zuuWvPfGHD$(#4Afyy*?zC!*w=1VqydL|r`i52;^M^$6-{jW_Z5q{=#Lfl~b!jg$uW
z2E}UoiV(pCtg*#n$qmK%f!wnN7$+_eOkvc@KEYQSC=*@K{^`5>7t_{7O|f3)@&4&q
zH4nAgJK}$thDN4mA2!lCxk-|#O(*lY;IT!;Up8n@c@dhEp1^gi<#}r&zgm}wnAbv&
zE`~7v3CGIi+>(^8CDw%}$(Q|4dNN5*Xgll9lE%(*y_gLZO|JU;cO;vp$)5d^d~}9B
zLv!VoEtRKV<Znnc;69x4Vxp1Z#mYc4)xfk~4dc{XFH+vP7~Ov%81qaplF|4^c$*Ys
zmT__PAC`c;X=(2=T4X1hC_GXU6pVIHZt;F>viPc_;6qXO83A#R1^FL}KiDjgqhP&8
zAb8m#n~7!hH?|6JRZ47eRt`GA_2fEN3uDxc?}E`6lzf~8GFKFrOceiWz_n#s@zx8P
z`45Y_KGuCM4{3jxwb9sGU}MLh@X%QsC8s{IKH%IiRU$|JV)ot%oNF4&4qX#GJZ;L<
z#Z&ejo8q`q(Ceh~fx`_csZ&or<3I5@Ou*uQZLDN17XyRz44K-ew#f@vUwqEZ*lE&e
z+H|Qn?^sFGYqjZb%j}IJEXyjV8%M~0Pjg^;lFu6Au;F2|Q-_mKf#Sy#X$ngUxXhL0
zTB119<<_;@*Inh)RuPm^tn<Ih$9phy@dtr^Casx=xMnD6o0%6!uxn;JnzSD>owleZ
zu&{4sO5DxR;`B;g)hn7VhsEl*u=Xc<)U${+t<ZYg+7YTA8WFC&T}fy1m(uA+tn;Rq
zEfSfV`dl!Ib;?0Cw#5fL<qOMX6X#Chn3o%F`RHZneb2dfZic3}m(M#guduzmDBbg4
zfLG1;u7>pajpg$>H}vg1A(O<Zy=#Tc9*d?XSD(v|{#Q&0mp$3ybHatyX`24D<&}cJ
z3{GdL%~7v<P_mF`$-))id)IFFTbb_nt3~1FbCFby>K0AbpI7=69{TI9@_%zfaKCxF
zqF_x@V*lae0qKF(Cnjq!J>Y&bou$pmWZMC*nMvF;j3mo8YP?RDEPcqIuyF3I0=r3x
z+|d)<UYt;jY4kpISi*xr%X6y4DrT`&%~q>EYj$pw>Rrews2RHTa(v>~`V(SON#axI
z@33E)x!j3!YUQgkt{V;EI~z_1<;Hn7ZuvgdLy+OSOJ4V@aH*o`?3<0<ue`gX@-97|
z)+#z(v!hAdW4eh(WX-Rpg|qViE^IE{DY#TBErLr(-?GoEpjxWw{|xUR1u{mfWdj#0
zFfLa3*is%Cy=;lbSusYPcALp=l7b63EtJF<w`y>PF6mAZ`?jHRtAOBIf#ByiB6F^Y
zdy9J<{3-fzDeJ@r_jO8ai#}*%AK>Z@;r?^AgYlU5p3fcpVLGv;Iy`LU1>egQ4I6%X
zEf;jpo>wt%-vvR2*;BR~HEcP+w<m~y^#(pU!v>z{hObp!;$~e!+zImxxppwdZdjtg
zv4~}3#e4(hr1&qc#@htgD)l<9hcj5MJn%%?<eMIwRF8r7=1e2muaA;{3Z=L_ljBe}
zIOnm@jji|QiiI~-QspnExO=CrF*k}96Y^PYR9O_)sKn}`(J#Lw?f(>$;*S4m8&_@p
zoottzX({+o%)P?dO_`M=K#}vHNR9`Wrvq1ZU|nn?fAnXKUM0ab3o-=??UsEE*>Kq=
zmaQee(Y?=*>*sOPEt9NwUaPm1w8>JR_x8%%{@u&{GA$q7oZ_;W^^3ri1J@G%7OZ;{
zz`s{aa`6VP#S`jE4cTU$PFPg1+V5w!w`arj>bVWSmd{=ydFEPJOr=QDv#HI$9JYt%
zJ=r9cz0>4ATf_y$rs-XI#V7YLq}ZGPSUIiL{-1DjRb+nSPebk}(^*198?r68U*)ng
zGD)#^>fc!~;pU8|)%(_bjZ&Sm`mf8r_{)|;7fowFicAt><y2TBw1G3f!|l{!(^|&h
zpauUKg9H?74vEO!7Lm$sbC6gwZQ-1^R-Owkt`p7nWI9lGNptQ)&UvZV=Sn5cyC*3m
z|3-0A^jtY3%`FaP3Xa^Zv*VX!YcEn@o3&eGk%QP81um^Mf~yVoE4?`=dqy!s{m4`c
zFV{bwZ#n1B-=e*9vdtQa3Wbjw)y2G>8@)qwHW^IPi(FG-{7m{yknE8X;e!&#CW=>H
z+33Tlyin-W=IlS4mtU`BpS;k;y2^K(;kxHl{=WquN%()h68TahWc&85Z<kc>VNd^U
zY*%tk&{(R5TTn2!z|N~!aBYI=<VG<DgV<-2ZK4=_VkS!PC%M(A$A){%zbIA^xhgp4
z2Y=$yoCT*&t-bSK@#}=SiZ*47HgJ7usIxJj_4R?|s@ME)6xdcL%w26zyy?xU(pw2z
zy3g$Tbn;^Oq8Am5lEPM8`X07VFrxd`(JEEz$+I?PbIN8u6kMk$JJ~(rvCg@tJrQ%P
z<SsfkziFKQyLtLA<5g2in?JuX|MD?%^)CBA*@2xaTk^xs_;{G^SkCHyP<)4@Bd5;M
z^%3fd<?5T<-FSQL!?V_Gy`cT&Rf0IL5btY6sk?qMz4I>2ytwK1#nskqb6?N>^5L}b
z>(k3#YgpdlpS+1{o+87Rgu}egF3P>#pzOOo-F)LRkBze(*_L0?;VN2d^w~jq)+(-?
zBu<_^I&5}{D_AvVSLkii%XqlSqV0dO$?Rmq?yE;SHfP!_luk6TtxmbRBqddLOXl6)
zt0xQ<l~eC>T=%LrQrxxW)(-n?R+fuf9n;v-BfGc$H`y0zeM0rq)@O$4f1ZdYN%tRB
zRFOLv5S?)H#R(C=iDC8LGag5XO0Qg1c~<ek(a`n3ca>&Y&f(s@RQFbu*3N~tJ7t7+
z{aSi!{hnL<jvc;sv&_JDDo>l_WF>~x4&1+&hptxOTByzN+TrxG&ME7nZyjH|T;XqM
z@Xm(1SyQK;op|^3%`3WRFQuKln{)OFoAOh(l|5cu=PsU=XFOMMbI<Ez5xO1*7rL9@
z?roVjOYWo2{XcH^Z=7yv%j{=!luNN*y>(Z?%is53NB>{F_VjfPCsUrIO2)^6A1@Uw
zYf{fQQg|XdJAg6C&DL1MT_Nb<1-ZsH5!*5co<lNDXB;FC-S?a)vnEt+UBce9LxGXI
zk`=f(<b;mo9D04B^toitfwKWyHt@ZiHJATbXx}99*B>;tCLEe1z4LdLoUY$o?Kks^
zpU=_HJs`R7;coFwqDPMIPnj^=V5@&@h2Apn*O!hSd2-Z1`dC_>&nc6hbqjkE-aZrc
zTKHIGi*0sQ(VE`UcgIS27RkTrtvaLlrR6&B9+9stZkoULO~1MI{}sg@->Y7H7Brr<
zx>v_qr9?dZv&qD{H?PH7F5PqUEsI3d0WRftr>1LPa?IIv$CID;7~k^Q|6eUu;4(SW
zvULKho1D;_1BYLFanCw%_^-q1W5PNA9(e4T`)c~f^9v-V9M@U?^}*Tcx{KH+-4tSA
z_@&vl;hW5reNrc^r!{=N`#5*yyUTB%)ZBY#DEDyTxof`-Uo@&-mKOSOX8LEvy?+bs
zf4l7!G`Ty&_U?}s#gDU<eq27ko4v*)OU~?xppVho(9N8Yi3fPs1)uwusUo$5H{Mck
z!lGxE4L<))%~>J1-HCg;BmdSlkE9egL|x5hisom_HjPO?e87!Ac>~w>pdAWtLT4>F
zWbt;rM9gQE^{j6kbmSS|ES(*n!l=``B=py<d7rr+&Fy&ntL2FWU%8y(hUcCS`1@=7
znMDO%{$JovIjJ-I>633(2d*R?6>N(<lN95#Am(j_)wZkAzOpuKDaU^pd}MrHpm$B=
z*5+b)Bk#6r>AZ%O|F6ye-n+8gsN#(v&y(jr_lQU}`oF12liT>C{_Ud9dDYRcuTP8@
zw>oTcPlq8{;KL#Hwdzd__oD9}T_?EUT<DbDQ`YCo>7IEcdA@jZ(iB-o{=Ex!?G2Ee
zbg-By;dN!=;l3lgtoqBAp61&-;nTIqB?n@buUdO&({27)4r0syzdlt~cHZ*PE-@jc
zxqqs<el4{-^rCV~z)3ErtBMuAGnBn<@=RihmASf6xvi)1Y=i2(+qFOCHtm^ZW&a@Y
z{~I^?e{0{ZU-<4Po4_2F|2!%h4c?B;Oe}ISGd?IdbqK0zO>kNkDjk%<vcYjh;3J2A
z7R#m{iAgCQy=vTjzdAPbGE82Zb9J@qg3#5CTRl_-6dD&asc^Be$$CwA;jqw+mBn|H
z$;*pPzO%hTgc29qGyeUxRpdgqb{ONEBO8PR&cAku`ts+Z^yWP6wSlYMdXujA&zljr
z<;K~yq4_+gt5&;Qe6`)nLAO5P;JPStH{F9#i^6hMIi_Vho@PGYVI_Sre*gLToK1Zm
zKZ~wiTFm+D*RPAy_g`m9IA0fgd!Bx^aqiaDrJo<U2HCM{NXGs7@=}F!lKOO`g41G*
zv$uInH>$Oo*>m$`&#k__yTACxEU}-z>HpF{Q+S_WE_S$I_gCcG$A`raH?*Fv{qvLe
z+4J@PAN>Dq@zdYIZ<j)VViSwZ3WqkHkQLl|JS71NT!JwH4SohTd<%Ivy%bh5R0t_F
z@J^o4Br=C9Np+EEbe7;kaq}q|!YvWA1eZ?_%SoO*JEy6Gciyr|g&s<Rlai*KNx9Rd
zR+hDpUDubTLwo%im&B#>>ej4TE_d?Otks^r>%O#$?s|1GMUOG;;%WJvH$00J0$pA%
zmuq@4@%33HPNp}?R-JE*{p}dvZeAZ1p}FNkSVShPtE=SOwdT=Lg-b0R)PJp9{)FL%
z6F1{F3Fq$TCvI#}@;6sJ9P%!}wlp+&yTI1V9m^Cv!v06?T=6w5x_GWqxK@#A#`Spq
zOD`6-uqjQ8n98AKc0O4)=v!p6@T)^N|F6&uJJP@5Xc&Xw(hTLJ)AhW|4i{?YW(M}U
zWi4GVqIg@Ale0rO=tTb{qj05%Y+Xq_YqlA!)!zKdbG2AmSYYtB1IuEbp4|OuQM*sn
ztB2l?)=8zM${xEUl{z=>SEF{)VI{GKc~LfEsl1z%l-6vH^Y~|ee)F*_EQXy{#%nh>
z72KO(dQ)Vw@3tFh%uz3H7p+cTb}#Q&*~|O;N*7OE{<gq1%Q|P_tye2IKXcHiuAbC!
zZ|SsKW+zxqq`lku;=(e&rjL6b>n+~C`}B#B4{M}X*fM^Ym&6qH^W5$C|M!-+=7;~P
zTAFXq_bqr&)SqvUoepN56qvw$@_q1$Pbc-o4>+7LY!`55Hx}{u!s0Ez!kJr8sAIK&
zZA-_IMwdHGM|n$I0vlw^g${fZb-!fm)>I{)DH0U1NYF|ucS_LtGFFAu1#-$+s*Icm
z_I=+`x$~0Cmc|!%yw(Mt_|q*G7IrbkJ1p&3k=~@kYtq;5SR&)9`;Aqgb)D))k)>-k
zJUZ0nqq*^sVdkpNEQingrZ+xItxr;Ll!_48xZ#nne)9?D_17J&QVY!6gF+SyZCx?j
z!AnhfODjXp!!F@v0<DqO3$L(D*eZUnG2)tX(Dg1STLzT{R&$u9E}u8K<#|h2-kM*1
z|5t=P)!K42E3ESL>~(WAcBorjv<ps|vvry2+9@&bmCtsCi7OwP&|<Qr>$H*h?P+Y2
z`wLi9RTVwNHhTzltP$+fwmj#Zbct)$mc^9^8?Og6B#B73<!Z$&4&T!wysc3~F`hY6
z#ib#G-)3pQ=--Yr68F@n?dmx+Y0=D?4F491Mn$YNSfnvC!HZ4&u*8}a@z)Xhv$ibR
zm*Q%|wS_0**F)}8Gu*V94+>Ar5>(^zdg8m`VQOkqnDng!%QChZMCH%;c4nEQZg%+U
zTe54~Ojr39XR#I@<=%VqY-2;QLhcE1-cA?&c^c}B3>TPsEkuvJ@NnQtX7DfHQS85S
z<0&gglm5y}{~w?K_MzAIrDwkbd%E_wJMGMW6EmED8t***u-#$O<U7q8CGwoQ8BQEp
z7hZ5oQk+l}vSh{S$xKn^i>B#pQaanr@gYjeW(MDprOs7xB?3;1C-p3fefcP`i_2p|
zn2LM1v#5&WmIvMTqDy@LarlI>EbHvpm}C&$#cjJuL_B2I%9^~Sb_V;){F`iE)^Z%?
z3U+OF*j)C0%`>l6%y**>?ljoyu`Dy2?ewE=HOW=Y3NC_OZWWU~mIkP4$h^^ITYbnn
zMR#MFz=0%bZH8l3-}olDawM})ikO{~;L#|;kj|~txXF|?<m^jD$t4P{m-qs=y0|=&
zSirnm^<Hk+wyZ~Mv!q@BYrH#Uci>a#);%X&b<Iw1=U`k{u-5e0;~Z`!p@fiK7hde1
z@WU@gDe-ZM?V>$RS45P`bWZGgb8+_5L){kN7d@<)cuDBBs{&`#3U)n3C+-BV%*i<q
zZ#am!^~Z7a$X0bm96$9ap7oTF$v=h<db<~$RtYpPHdE#}9b_ypSCPeR))L`}rf<fK
zSJD(t+erD=G*4VQu_@EP`<BVZ%A~Bk%bV5}IGe}5IaDCja@3G-xwL}O<Hj316v`|(
z(la%tc5s-M$w&DyHB4x@zUr*i)s@e0|Jx*6m85=Z@eIFPO+U`-{xL47e|`G_!}NK6
zs#EN~S<Z2i6<1)8<Sr5jo$&0=oSjU%|6deJS$U{CeYdopuUESI_GkCMYo4FhFfZfy
z*vYbGgGR@Ok~zDJJza{93*IxVQkaldvcxjTX13Uqt*18!?zj+PbHK}=>y+ZE*IY^>
zj>;}OeLI~_-m^0L&#9vw70EvBRMPZKEpBF8o@|Ze*YIZfbbcKxXWI9;yJ5-dv+G4|
zzD0*$S)MpOwYlo4w)U@C8iH)oTC_tFx5qX{N(4>YIIT7(MRE1xLgkcA%!=i^5AJ+d
zES|C{^~0M&3+WS26n0wuzISECkG)Y1@*hKIPvclSBjfg+M_~_ly<NNYS%%)@us|ij
zda<I_N7M73X7U{PSR!-#$4Afb?~k84B|F~nIDWanyyAbw8dvKYmkF;-KWTb>Tr7Xk
zO-3`sUxVeY(?Y}0E9*TM%2wR+I=SJ}#`WK3O=IWd&D?g$yXD2FgDWIGHZJd*c}+xE
z@#}@A_~~I?W(~HpbkC(-J=5o#8`Qh$e#-9dy`GP^AK!arxu)wrsZ6~pM%Q+x3iUPD
z%;p)MS=;f$=jzHPjk$hX*%z?zW<2J$vM}I3r|`8f<LPhb1s#hji|4G!Xtn8IFk#ZW
z&-2w~U4F<^b<WwM!02(Roh?^IZllJsNz3+jrPR+6-}|C6$3yO&{l3L7&emOV33aIQ
zc|AwhhoOVzyqMFX2KU5CY&Z7!Xno(&^k%0x=kMFoJ#3%tT$g+M$n7@K|8F?+7v3n(
zFW&Sl?SHcV?r*2k=HEHKk$LHx+LhC8cr#i|c`i2V@O$}3>PlQY6pKV}SUG&|SbHn#
zKuh+4n47wK4F?hoJK_ra9w}*cb&GXd>+#ieglKp0i!@A)66s55?ztd4NqLosi)G6N
zy%rCVsU1s88+m*ts<#*{zkI^-1xMHO0H%cxm_BQ9DgIGr4A9q1V4e3<eEo~*Z%zx(
zd&#w~z<|Sey84lsRWtfTjIBkx4Yv!c?>KB&$mJ~A%^~B=A-kJLU#G+P%0`nlhD{cS
zc?<UTXK+38*s5U2mCMPcs$lx^!IX~=cK@s93z=!;B;z6_+5h~4PJ<EuzQiM+Gmhx~
z?{+a*%pWOf+^V$sNT!T$i%ffjabn}<bBxOUmo{JIoNKpRAjHNs^oWJSOj+0LEw0_J
zGd^v(b5eHZp}Du5Ir<KCdK=GUX0d#CL3V;d>!Y93gidiTkTO-|Jox01n9&TT`4xN@
zZZ*tHKGxCEq`lK_`zD3mhwLs_NE|+}V1+a9uBxp~8x+zWyXBggt(m|#$7BD=S!<#z
z)>OvqFY4~7^x0n0-0|)4@yiY7Wskecf)6+Fc$|LJv|D>piwM`l6AS-;w(oedy;gWf
z*g_BGraqNs(Pz<0&4)$V|9G^nc6gF?DCy;aE@i#RQ!E@NHt6o)6rSSxa?!fU8kVz6
zcFuB9w!9?!;(wCkx&X@sS?zZZbe7$0vMcIx?Be>ndZ)Ar$D)u1O$)o<9}X(~QC^|Z
z!Qn6|?p2S9=IT=cy{lf&>$u6aQmR=^(mm<(DaA%B-5?&b3ryw~ZabAdcTVw+&S-wd
zame|Qw?m}5b1|RXEfL)nd`}XmYTRzo{LS~;!R1qe&9RdH1C=(%Cim}I+26Ebj`3!j
z(}^-Bm-n2ho|7rK@}jing)hdJD&_2~Z7(}=OcoNn_L}p0v`U31lTn7q{c6qId*mKI
z<dVMGCw{TRj6;5=1<yknj%bN}JG&h&TsqP0;V0S2WOTsJzEE}Vf}V9ztVW4FbqbRY
z&af`IBk;n8H|$1JP{sc0|B@$Ur6>JV?)Z|^`PD@HaM$6TM?{`}uovv{_;0P-p<@%i
z(&OKr6Dqg1u=v<KpD=azLfwNqbtjxz>-SRko|fkWuFmr+leoVuI%RW`_fNl(#H8HB
zb$ySyrgog1V9h6R^yI|N>pKb#HvJHrny~XhG2h3atx5;QG-oi#%XG|1=2CL$d$C|9
z&u6b$3poE=(qF%#wRvT0X+mq$LZOveg1%B61wK4mZFn{RbR_QT;W*uMeFdwoM)S%>
zZ^IVr=nIBBOq?IdIGfw_8kt<~=WsR)3C>eGAue>;&y&mbmbd#Qv33#1eV2n@O=$AH
zz@R(9=&(%4Vd*P}Y;1aXWZYvyTysMX{AZN;HpAx7LK|<L&Fu*?9gC)Md~`Y0xcU5<
z308+S0=}%c>~l8A*Y!%`(Z>NR!zBERH@Y^x2)Mo2)lK?hqsF}8lUv?K$cf+ajN|c3
z(OIDT!y%i;<QdD_8IvcjywUXCaoeE@+$(3vYZ~%IM7M5T(K+kG{8A42okdzJKj|m0
zUY!=a?R|<_Q7^|k<?!DU!HhGGH-((PTo8U@QfQ^Mb5n!&XBUpjT=QyK^AhEb=ndgz
z!Z*r98;lN!$=sRXut+YCk>O8`M_o5l$D8wiO_aJy`FeFYm{OH)I_+pSzR=2jvdy|9
zQ*p8GFQqAebu%m@r>;>7l@<EvVlvmok)L&Ee*ov4|9d^}?&&d0XccAkvK8{0aN+R7
z2U3b>EdIP+v!qv?RYKp{OJ)9!?HwHqx(@<d1i1S+d4i-=k972`F}O4_GHCKlmHG^x
ztqJYhEWFoF-mosGQ)4fW&B~>c4;#$>sAca87d~akn6c4Vr<q50q0xf|o?ib$q7A#W
zIriG{TG!a-{j|50)fJqw{J<S1U!#!IOV!WVx>USUKYnG?v6eIEWaoHH74?ak^Uq@Q
z=`%jZ6E|CCcQ8kCoZG2kuuI^Q^w|q4b1!vs%zGi#@>XQ#Uf<3Wdv8l?ML+d?p=mjd
z<J^3ubG{n04_yuLEfyA=DS2rDcYVVAtrNNrb#%X(!@avhyHSu;=l{$09Wz!=)3_eQ
zy{zPEhw!H3;yKJl8O=r;bc|N;3*BgHt>QGF!N8Z%dQxe5VWxj)hCfpu2cwur>Fwy>
z!4XOi?K{_;ujM{-dDDq5*G1hex;-uqOuQFZZ1^X*?wHW(!1+dZVoAVk+sLVFFLXxV
z<SE&ay2f02BS%lcJlEoNLN$6aW$Of5IYeY9iB91Vt##y&j+?Q-`R`^o>m1uX#$2m6
z+`4)|W&H%LLma9qQ&*WT=-j#CobuAi;Yq9ZSy-`DIcet_bZBflx8l&YrB>Uf8q8B%
z7N5|{o53nEd;fu>@p?BH^eRpZ&0r8Z;QB3tIedYe^zP;`ft1b_xAl92cUV6$`)?bp
zpOfP1)*)2H%BR35Up%)(R66*xK*i7foC>=2lkbEqT>etwsiSVHi;K;R02lXKm+-VJ
z?n|e7iWqyXO+8w8!Tarz-z#L=DmVh}TnX4SCr->YSa)vQ2IHV!4hcD~c?Cxoc5M;q
z4V7OQ9u;d6V>-3*z{=>Wdec^D_C`#+*>ZaRrq%}x2cI9E*LZ=6Gip-mYu0HT^DQPS
zFg9?j#0u+dT$Pq<6+LU|vgmDVjoll*32#&Cc>ifylj`{ca~s4ru^K5f^5`<#*x5_Y
zVldjk{6eGo<gNIFw*K{RUa)+7QRCWS=5xNV+C16wMD@Y#X=O2;Hft|e8F8DP;S#gh
zax!7|f3XXne?$oQU6@dMV9Jz$DYEN0O%r)bb|?il=pN{pE^<uRuJ_T@jyY#NCkM#A
zxuv(PO>V*0UNb8$`7_o_zho^ra9YY$WwEQua*HVG-Uel#p4eTIRSR10cCN2`A+h$t
z^SIkvV~l#Y>&7Jgnz_kp+5Tm{sadU@3asKn>d_j_nhK3;6b;3k`HeK1t?xC7DK~Oz
zG<*8Z;jv9vx8ip7$DDO5yfq(4J@UBoI&%5`iA|agx*b#1owctV>3O>=@Qh2U&A!Yt
zxA|oDoqYSfqTl1_eb1{d5B*{pPWqgAyr(NUmTTsObIQ$rTVr4F`ChTn&ka1gq99Fj
zrsl)ai8oK5y?OiL<Np`*?myuRO9*e`;CrQct+=g2d7~X?wY;_CBvsDdgxg!|&B714
z#&>MsVVh;<@=-x>HAjKc_UMKex(REH4#dC6VC9?8@JoT&C}U%^0&|wk_9urkekiYE
z`u2kHS=V<P{YfsS(o4_R3Oni=2&xH-UXLo24=PmQ675~X*Lx>HYP;y?9|>%K5?ZAb
z?>$<}^<|N$$<4<%4lI((QapaRKS3|<$kFKsIHZ@|Yzk1GC3|uCGQH_<I;OpNSeV5s
zyN&bS49Ap|<|hWZM@_RkJhsY5>orL*SD)ZA%rjV9F!TI`Lp$A0&ry7=Dbrz|{&?%#
z*GWQB3m>Y^duXskQZb`xnb7HFT>tAY?{(#0=<|V7;(W74>*|e$(uW@#Ehso;mOoL=
z+5GPp>E<UceknU_+~i(MNv+De$|9{O&GqnyJ?{b$q43xLjOS@F_ivf`wzi6|we4wm
z-qYicH#I87M$HQ`G8XgJ{d84!LTuXIvm$A&1$i0gWp4343wdIkRC&|#k52wVS%or_
zg85f(?>w4e*PF2A<Ws}IH<vi(T;M2_^ZVlV^6RrM--^i@pXY7&4X=Oa_h(;E_4oIC
znLh9<DK=d<%lBT$Iz4=c3iJF0UwEc6=;<<d{5KP+I-J<}LG1AtzUBLQH#XPbb+0|@
z-n78nSYI>sn=(VkF(Lgz;ZCM2MmF+Mg)`F%1OMN=)3f&_pWTi*?@!F*dpNOn+9c7P
zQ*=+B-r=y&?4hzf@0)39Q$KjlUQ^WXTt59mQOAm`*qqj--+X6s+<Ins?wnM%Kuy>3
z74tT;6|OhCwM?{n%eCOnR+Uo^PW{z0Tlw4T{D<t_>?ORXtDU|ZEbUZtnVu|cZGG^4
z(AL@Z5?*gKD^B~Bw+aa~v_@&n-q0MT!RTfF;K2EUMNTRH>qVkFrZ}y>tms)R5!=O^
z@Y41!%XPQ4r+m-Er2pOZaQTtwBge1g-n#$xX1~o(iMwiAd1toR_@!t(UAMybug}ZA
zyo+CKZa8*a`?zM>lB#$6d_Vtv_mEZ2jN@3$*|a!c6O*j(ds;b){>zy@n^V3{KuEVB
zzkVOj>fmkfey8*P`t+Yok^j=_oY%_^M=5@KY!+^@U!rK^kuLWJfduyhvW+af_Y{p6
zp3!Y)k`pR<x~xZ<F)!wBVJY`w?G8ECBAFinPmWJg^{jH?oTzeos-C=<%+DZ)!)@DG
zZcIulTF$^~Q}4v0;Glj{*LA6at5@1(30<abJvWuMz6@CzeZr$wE44dpU7TXx%ufby
zZfM3FStOcJ_BL{B`u$C&yPvLQ*u}ro;YjuMC2@?g4NU(!$_?)CX=APrQE2s99maIS
zMul~vh$LgKO7yWm0-p|_nj@br7O?Y4#=Hs98-*NxKGf(~%eeiQPD04)vTadU|Npw-
z=EbBcKUv`)uiy{oM@&7{E_@fu&$kC|x$>Dk;+J#%zE!^`t&2TS_U6*tpPSNG-}`ZU
z`|T}G_m@ri_2u2$8=D?;+5P?T`R(QX>xCZuX_qbkr|!_e7`5$!uRy5Ef|~LN*XP%K
z;;aaA|IB-0e(S$j3B|U5svZhS?6-s#F|55)60Q5r^cb6F&W25mMb?bd4>%i5b9wB}
z`Cx_%6NATwqij~0iBlM6=S*Y?63W?hMQk?DOwoxxb~~4Zy4aZ>WeN2&aE*w}Te);<
zeA>=~(_>dnT^c1b<0$K_m=%%JXBBTesit1I>*UcnMVma2%_+Tm^XS~NO_~ShmvgNW
zSC4&gv~xlGf8Adb7WHvm@LYVbQiWBsFR&{@S}pfgVAfoxFPbhhCT-HxUO&~XD`j@V
zwOblfE!kFQFkP7>y>@%w)CLygv|HLsPbh|Z+uK}qUc+$IuI0_feG}THwYP3)iqf7Q
zC#9RGl9WA9dt1(J-R-xWJQg-cnu=U@e5zC;;3F_IThD)O!3J07*Rxs#{n=N&7V!W4
z=*cCAGL;*b{Qp#iDEY0u(U^Paa@?_ktrzB1C};{ve1GI`mpY3j+_{iv2}_{D)}yNf
zDpNxzI8MsB8XUXv&(!d!bv%Jl#;c!(#S}cOF&1rk_%%HK;NLfanLXb^OJdzW_k6jy
z&#Q4>)^ta)ugP+2C1O&Y|2wyoW^Rg{cI%4sezAzGg$vWYt@}*b1oQW4E@UX?Pn_d&
z$A&Szu(*VIYKz!gkMh{EJE_mNmp&5ATU7m4@!dUF{$tlGYJ2Q&%x@KT^RRrnFYU#|
zcGbd-2cOD^?ofQTxVd9;hvKw~x1B3C-mubMfA}1q?w2b;{%`kwdF*Zf@5#REcmM7@
zxxDUh?>oU>qj$3y&O}=*-6SGpvp-@=pqJ_2DO&ZVsne6%IG25oY_eya_1ItW*~}!#
zz2S3SZ@&L;ekmW<i-pCP@|VxAyR~!0qTjFY+yDRX@5RWtVPbFH=2mY}e}=f<CzuSE
zI!x9vXcDM7v)bkGl1Y1>8E@QlW=&qOQ1piXN!&(S3m80R9EeL?qi&MLS~|^DJKb}2
z)MY2$EmFzza&o&}?p^ZOZf2y%C7R->!1>6L&B<k^LYlpNpi+X?XZ<qOfKIO?;`_6l
zgrn~?xv=j(dnn@Kh9KKFk$swBwlkjcSN~A9`sbni{??A&FW&T7=gTzryeV^fyf!D<
zMX({*gl*Gm6MqlKTU{X>nIfxmR42yW^pZQsB(^bRrH)bd>8PA(o421is#AI~McicP
zbpLa!jLUU+G%CZ`4oxv=HJLQqb3)0?(>`s+Ga8>V%Wqn_+OZ`&CisCKbMxu`ma`@^
zw0xy=npb<NKB$zdlwLZ+)22hi!QJ*^V4skIgG{I~<Nu332h^lKdwtvBUMO;k-|}O!
zABU`m$|Pag1T*$sXHrkhU0AK&-eShsvh%azk4rN3R$m-D9&o!TcFlOZ?&zZHvQtFa
zQ{CTYmFwJjnb#}0weNkwaqDhA?Zta5JnYT18v3eIL=J18yj|mbNaCZ`Qq_=iVOvy0
zmkVTaUud3}HRs44wb1ufmS?-Zt!YtZEb!Nje-!t#&1#wc!jnFaoaJY|5?b$Yb!Bb)
zL1FPPBFk+>PX#tAiG+V%zQW~aW?;p{DT`EE#ZS%I>ZU50nZCl*D`09$<F-E?$q%?D
z`i1%$?GNG24*4t@xpCoaS(n?+IbTI$SBkhz`P9j<_T~)+gRoXjng1`>X>8Lzb|=8$
z{;7v)J5}Y+`@QIVYt)h9=`CFU@^sAa77yon787RM2zDB4o|F}OGV#_WZbu=F=Dn*j
zgeI|+#Q*iWo4dzzZJVoi&(;j~wPz$`O?OS~-IuNGUitmvYrfUt|5mt)+?aLa#IgeQ
zhpi_*R2(_cucM~v^{^{g=;U!{txqW(F-uNM{hYe-$;7mbolnl@O*?t;jBDo9m)zIh
zNtk@q<elmz{oa2<yMyS*(kTnii0E~$yfByLbFARK9<hth4*1R}m~~w9VPVP=hMYwf
zM{md!7O`B|ChmSY_u`ZWA)V;QyIu5-%WaBzswO_8l1amYF>0-+#Fxia7q5v5KVWM4
zf8(Zp?IOKXib1he3D0-k*x0mioATm!yY}oht+U*9xi{I~h-;$JY?a;Dbvxbdi{>V0
zI$D;=+|^o}^n8vQtEbl~%~Lbfm(ES{X9%Cw<liZ^&$<45wTtcj;76@nzHL13TPk!W
zfys1+ZKP;W<Yx{B>$m&;-=A+0l32j7$66$!ZCQnoPA7BurdNWkUXANd83dT^aOAI3
zF;`0Y=<cEGaLj3&C68X|%F@&~r<A@aF1zQ+wBvxYqSQuv7n?=<Shh_L3-0RNx5-&L
zr+U)Nh}o)-e)s$OdB^1JTqK&upj@=Hb#3ZY!+Y6N^rR2!q`g{k^GrwJHNPoKbxfzb
zEUTEZ!M$z!f6?VOEDxhRPf5&pa<85H<lg2b>y@I#Hg``}ofNky<($sRY5((z3}$UP
zeM!xyV4lj_DXJ@P_-%XOdNSnf`iYXfX$pS?PfS^6H2acjn%Rxh{Q+sF4&`Te=e-n6
zmJ(A^*mv8xM%h>Vd&{w&+}TV^;$Cfwtvz-$@|<zy+Bw^D8SOmuI42lCKVQZ5;*FHo
z?#Z|JeN)=~Xv)W+-!}i7CtY~RYO9{*TqHc}!gb4%)2A1$a5?;DcSl*sefh5%9@pRe
z=-;yQtX!v!i-x~um`mawiPeh>CM6%*_0g-r_K@bw#mm#xeD_SKyT>&#qTbfcZ^2UA
zy^|L9&Qf(M*Z-bjIDgiqFClyXJDAq0noGs4nWf<@*~N3be99-*uhaicyjp+uxy$F^
zy~3yMmdw6pzcbI`=z)mpBOzikXIa~~JdCttGg`UwOs7p!gi*_kJ;&9`QoF_L=cyzx
zM@`Y)n#z4gsaZOE;k2y}Pye?);=<FCzqG}*z>%}#yyJ{+p@(7#1qWFsFc!`A_<eMC
z@zvXVduNu;lsr&)pJSI)ehiPvLWMgV(p**IeHE>TO#2yJ8hO|_C1s8VxX9GJT%hzZ
zq4wtj<=mdSt7q!IURQ05xg>Ib%^Oipqb~O)GD~%ioqjthW@D7z&q<<PLYWM%je%Vk
zY*{)*LT<UReNtgeWn?er@juFQ+heJ~v4pw*xeh578r%@s;%KLO!IwwUVJG9ZkC*+Y
zcsTBGd_JL3`oThmDUOl}T$}|5T{$>7C6W}kUA|G;CbHpz%hri|Yaf(^io2&y=Wuf2
ze%qq{sYRnLX;z|mKmhv(jXB1O$L8GiFuEJ*mwU)3_o&931%Vlk{HLyapPjzo=cB0&
z{R>p37Aek}G2?uYsnnc;MkBANrLLlLL!=gH@cO^;U*^lTyfMnmj?4dz7niJnyqAQi
z-I3LmU7iWj(jgaua%QBY2>jZq{QIrE%N&kBA8%>gz5hbTC7acKN9%mS$|H)?8Z!zU
z9R7)ks6KF(QIeEdBs+zX-GhOhr&l^+fvXP#XTcHO2?zhP6)^74xV`;km{=}@_7*{z
zhqq_WRM|gM#eL?H{Xe<9zIAzTdm{1S$Oh4V|2fxvdtFl%TE#LL!=60gb-ELHM@d<B
zlJdjpDz2BDLoS^um{Wi8XmDRlL!*$m(L!OlwxbH{pK=^;9k`RRp(#D&Y-)vD%8C;a
zukNg=VLMlQcG=3MdcJ4V5&{yI9B(l^R-}7f+=@+8g+aS?Zbc<ab7pWeW2^C2-^&lW
zWp~YOKX6*<*=fzKjk*>IoH8d|7P!0aV3eG|AbF%)nae>l=YW{TBesARCxe@blMb+K
zag^*>cH^d#bkFSFUleU;9`TK7Eis9E%;_+D<;*8DTb@2m^NhIeY;f;?URYcqS9<<0
zu6#N53$6~AQ%~9Q9oLdQKL78{2^Ow;Sq`rq{q>ys19O&(WE?p7^wF!Yj~4tCdB>@2
zpe8l5rZZ~Bc`=jF_l{GAneH!MIj5=j*y0?~Pnn{hGeZ|$kZ}o+Z}`N&zDK}C!M|`q
zV|3`!w#0M4LNDaBw0*rQy2jD%cVbuKR!2#-YaF{4#j7>2iap3l?UL>|zf9qbxLlG`
zMvCK<gDgk5*$rl9ElQJ|qgEB7;Be@LbBGFmElZ8x0oOC#S$PNcSKRjZd5~NCa1u*;
zv(?MVTn)BYn*4a?aJh-esIgs+ed)~6>bD_W&Xje6hyU>>Nykgh`1390i;Af|>GOX!
zSE8WR(wKsS=Tst2I<8EZ5LDoCps=;=io?olIkUuqR1+#5c9=|&=u4E`_wd@<glT57
z$Cg}J#CQJohov$`vRry>6KjuTJx#2d!lb;bX-12hc#fB3himqs7rr(L3V8>X?|atG
z_ee}<rc&B7xpT4;eHnWCCc7S3{^Z1D6|o*&4_2v4B~3PVjbB|FQrVt5uOBa6@>nr@
z;suq>Ur%wo?9%b#44!i9vr4Df1tr~8*C%>7>h(Q8;(1WSgVEq$Qq{8O24YhU#YBJd
zD9*e0{1VIjrLt!m9%|RItvYEqe>r0g!ylJVHZ4mPUit5tWl<-8s3~5T@!0A$r*b~H
zus&5?Gm+_k?jGq_$=h?2`g(Gf?9+MRvg6>1Fj2{lmb5N^SEZhQog`<!gDy4CbhjL2
z<7mn9Yy33vfXEa_>4FB=8%-{g9PfBCa0VPXcdmV^!7{IuXKr^KyZl_-zQi;9lF#0{
zOyour$Dx$XX}T^8RJ<<vZCoMg)AiP?DEFq3_D!~!;z<h4s*K_${XTZChc)j^=wUoz
z5Hzzd`(B@Vz%tFidD+T}sb@7(gKza5-+EN-?$HyjsX}rR$LCF83-OM6bu4Si)2t7%
zo?BF-k{&)YyBOUz<wnO+cayfa2I6Zp1&+2{zs>TjS3x65L_qt~t%^ih&Li!f6Bshe
z`hFA~eA~55G3$xzlQsX9pXEp%aqpX@!S2x#v7;$lD#`WALfMc6?UW<wVJ<aJDm6J=
z>8nm#2y$2+UAF#C8{39}WeVB_VRD;1-1RpwYp-F_Uc;<?;jl#)lP<@B*$<cHKU!b-
zPi^+g{C{O_cMRHZm&l$LZ7(s(O5$|&300q1k^M#I==NukEZy4WOegy%9KC-m&Oj)6
zw_a3rmBIT+{qQ}Vd%5Ck@3q$5()^$sA12DxU?ktLYeR%<SHOi03|BS;Oc7x6Xb|)1
zemqN=ed^-a+K0@&ZY@TgX)SKC7ui^E2DBZD(zcQ1)M}hCC*Vfoo7jve&L&qsc{yB|
z*1%%(Om;=u4bCLD35*hlPqUnG|Ns1iyMxxF;Fy*yy*{T;6HaVn-09N7U(0g&8TWxF
z*En}FsU8blc7^#s?b_d+On;nMY<mv<F}kAB!)VRHV0|Lx=P!etfR&yN)2EreT3W|)
zFs(J*<Y@4OgT+r$zPw2ZdXl8|SeNhH18woA{CPb==d=Z-@?s(`#k@Ha!y=KW^HB9Y
zi+}@*5C?;yu(3;zV{*gAn7?x}eR{7HoLw;QzCTCdvKGB`lQvruwr!pb>oz>vqULtT
zK&+tPP_mb?dbH-93ctyfM?U=130LsZ_Hno|QLSdr3fH=i4zpJM+u7K^z*knxMLOXb
z>$l^NUI=GDlgRAkZhs<dU9WFr8M$K8q1lS+|F0j)x4&{&tB2|DDSh|)D_R|m8f%(t
zx(;cq2-NL3Bz2+7qj{~niFSg<>&?$@Z!U88n<4RrE6s1Qc&Xp{buB$*v*H4r4Zd`&
z4*b&*k@8{JDs{tK_ks>)PdlL<zJUK?fN8B?)TJM1))=g-PgDFVy230@IlzN4H%H#U
zTqj`3V$YnjD{AKRK8;F#(NW}}o@C;y{pZk;Gp7GIZ#5U3W1HZ3xyWIH*O93fY&CJL
zJwo};T^ocwT_hbi>@+#tQ(RK|TlTu#W;!)(QrfmFVbP9aJz_nMB^4~27JSh<@_u87
zNp?r0+6AYL8=587FqNj1C~-5mUw6*3nBsPP8^f9=jXhufXDoi}en4eu%Ug*(92VmI
z{SU%FPjK}Abg?+ORivTkZdT)6ukU`hbZ6}|+I8&H{d?Dsa^~rptHwMr3hBBdJn!>d
zrG@|*V;2R6k2-G2CKtr@COlntjOk?C^%Wgg5_nC5*X^J7)VNyjB4eBQ#iuXyrnhTK
z$6KjS6S@$eq#5sZknP6dHm(A$k54xw_ONI){OfeFa!hl*=kk1t)1faq%C|JcKZNbj
z;j3$4vAh}kHjBgiMy%YVSo_=)cNtf^ulhQ9>XGd1Ls}C8m7lX|XEZTfD9c<d>9zS#
z?)mr5YnsJ6zG(Cuvij0g((@%#|M|an`UigM=k3~3c<;S>*zueI9+`Pp{xf`ks#QDB
z&-5hg&yq<TU*|MyYq|R$S@=4@L1=f@2LIgiftpi;8V!P1&RN2HKbGZeP1x2YiJ^<1
zr`NT8HG6i}{9dN%`<A8k@>b7cZY;Sa#P7M<#BlXYh0dFcZ{2Y*SD(OQu*l)YwBCPP
zT`kJo9|SuZ=<<uQY@g7*cTM-!ou?;0>rPW{TOZxx$+6p0fJZZ@QDwG^u8d{=@%-!y
z&Grpnv@SR)T|TUp!8Ge-nSRff{OioRES9zjCZ&zmx_`DsieKxUSY@@jnoG%Ty50BC
z31@;*HvN=5`Pt`tY5VG`uevpb<;tC`XU~1osb>imQmofaK6E|v@x7Hxrn4Awe?DyQ
z@}H$d*dna=tZT&6q(2SSdB*9AvEM4HW+yLn>$c@MaC6J7-pqNscmH1G^rPd+ku9fe
zTznifeZC2Cy1e~l>Sq%9<jag5Q|7*4`Fq>Z#Q)E$Uhcm+w}e!qifnrhx&>R>O6=Sj
z{#)yd^H%ds3@+8SJxyu>JKeeuT76-*?Kr5BAX0kautbFItlXbFtyg>93$zvCyu!#9
zwkvOCB>$8t$5PKK&HY_FxBR)BVCycU(ueEQ_b4sqkpK0|Wbsl(#eH3u8Z;k1ao@Li
zddrgepR29elfQg;p|NK5+|t8ZZ&oc;p6s%sIqlfdU0Z*P=ESYHPFONQWXWxo9sAU#
z-}-#$?v|xu>CD#u)5^{IKR-@mw(sQnFM8TVtn-Zhn--0XTKj;V8dLV#8*Iw|?wq$~
z+fL)%naNu|_cK|4VfMDacjIy2&AvVjzVDk4Ejf8Qgi*1aDYIwKWTOxRSCO1Ep*%<T
zdNcleTpyI<SjOBU!WguTLq#L?)WH_k<s1nw1Qs=P@JR=WTqxiY?~$@hyK^Xz!9!0`
zb&^h{P^#Bd?Z|B+4GUC!rg|=!kag6H!7qp@YH3RGrsWOu)lYJ$hJI0*Jy)k)>hI4j
zDHoUddiNDYZt7)N5*595mHL9%^9%SUYglB4B{6Nu@?PZnQtat8UN035!Gv(7IL6dS
zuL%*S`Wg-hv2#y*@hT;evEx7g_C3XuK88-4s5*U#&yGz`4&*KC`S^VO68+A<VVSG0
zuD`BjIP2Jz*XdK|?XTP7)|q&{``)S>f9<X>Dvxroy`cK{tB}I&9Sm1gC+oYv{F7xr
z_1w~aKfMRLT*3m+9P%~V{N(!-PoXI~`^EP*urQhD-c(ueSIy`>YewOXxUQBzzrVho
zeyD!`|3A0aFKARV$Y5d=nv>AQDlL-G%%+-R;Ka6l!UrWb=SK?4dKN!U9G>8&l&PX;
zA!w-8xkIC?Thuah;!#0&6U`;UzEv09jQpOsxHDEvIpH>4X~H6H#%ZT`PO3~j(#gBp
z?WPyQY89(V>WsG}T1A3py*V{)QkLdPG0k(<|NT8r3QcG@bLv#{YMx_n3Y3?c-_1z&
z>)dswr$=a$*2}Y#9!z=R&&b64Wr>5b^p*22jG-X`Zk&6&cy!ymJ-3#od`_Pl>Nc@&
zTIjEd?}DrBc>T<tPT4(W`>T1zThD~4{(hxWT_I$=^c~BFP^BHWf==pp{?W?s)|F98
z=1%HetMA&pXhKrttZA{|_Gc{B?YPyvD%C@=qvfu{?M^){&c*rWr#fQv>^_~iBjl;Y
z<rRK+FIUH;2@^89l~l79Pc@qs<fXL2>r#-$!iY@P2a~_#W}TnPwajQ~OZ3(4vFmd<
zwby-`Qs5mP``yW#am$ZWU$5;ExuBh*qWZweaLTO@7Y=KB?D&6X+HtO}!3{<hPZsc4
z>t*+(PbrH#vuQ!~znK{e8|AE2qdhJ<Sxn7JWVd+p>ZR&c|8{TDE#B^vGlEn?L$~^D
zhz&A*{kTR_@65Y5Li)C<mwh}t6CTd~ZTss*u7ATnhAXq*?`4$XC}0tlx#P&jJ73@<
zhw`4CizcP%Y&?2GH9YX{Dc6wcv1?7HXRJOKobuBB<dPdp#AZvBeg87KAS<(UN>n(f
zcjucvuTyH>w?2E$R>^wm$GGoLZTI&3p_f%3T9!W*v-7{z)I9OTjfC!lQH6_n%%gTL
zJ#BR2K~szFCavTq&TS3BY16o@<Cjh`T<ueP_|cg)yqb&rJhRsYtBQt%g*N>!y*lNH
zUuV5G=K}8^<_?Tq%f!}l#I5&wCe$0s#_(0Lc|+IMR!$*py=p@a-+zZ4H7h1GM|Ffb
zx<n|n&iSCozcAcWk3~<s@3oRtXQG>v)8UfLcZG*9FLY6wAvBYxh39Psv){2r!dDkQ
zQD37m!|TUV*{N@|OWjYNeEn;xkY=o^_FU-*Yc;iLD<(Ww$%*ESc339Yv-GHq9C!aQ
zyE3u%1D=N47R0r9Ojxg3a6q&y<7oSjh_<69>v@-UOpuNVY~q}9K&;Vn{_IQ3_k7*S
zKdCeGiSk8vPxnPGRZ$u387h<J{{HD7di^Bdbv4sde$$p%T@h5h-r+3cw@GN)@yw+R
zIf?)KqpmHI-4Vhi>0!wB=ZiaMb`<CB106z!(^O_W-Z}fuBV+Rv-fZr=m$yy+E;AQ=
zwOrws<%Bw^APZK{6;8nd>a7MQtsIsu&eB5HdSuuwj|cX-sJ=Y(TxR=X$*&p3=DXA<
zxQVDtQ%T_7C4HKYQ%h1Uwy@02%QIkuOV^qLPt|QVT$p;oFUYDTFy~dAk^NWFZ8pJy
z=|Z-%<c_7rLOu-fxic8p_g!Z*YRG3!^|;5jqo95M*_=R2*OQB*4qbLr)vVOiluul8
zfa_j_$jXD7m&0d7^@m+aN?4O{RKsmyYgXpMNTCIuLR+L7jSIipY&#UneD{VBo0RY@
z;gHoEIFxqp{r{sivhZv_`$AR5jK9lUu3gIBDzu3)yYjV4FZcBX>!1kfI+b}dAJ5tS
zXQ#7@N<!TIpXb!Nwpi6_mUzu*J(XI%NVUN>wKLSkGsC>xYE{O%*r_i&L%ftW#`G<f
zP0=`gRO-ipSkIQ+lwUJ)Pyd=0tC_%+=kdgI*Oud&TO9kI&ABHVn|QhV$HwZ^4L22Y
zFS0%Jb5rfOn0H~@?HTWw-tNBl^xegtX_G$OIwTq`aa8-I`RRXq%d_=9vR6A7PkOlL
zlK8h1>vdl>o}R5Sk!SIT)U7{`UEL%T!N?Kz;lzy<a!cl^b1*EGzFVUy*!V*$zjF1n
zwv{V=HDx%C$k$zX=+R?QWYqD$@X?})PeV#}o`~f;HtleXUGQvwdp)t^CRv_BClb~#
z`8T7>GA(zC$n7YNZ!Gb9f6MB6HZI#7X0kHCEjcPlD#YVfBe%4|VcnR6jhtsro<3B1
zf@PI?<BVO2+`=m|r7aW~gzi}Q+2qFm4s&|$nPg<#y0Mj&_s6P9A`=?w=ka;XkdldZ
z$W4e4+Mu@L>l*&LfYlStHZ0McoA^0qR!_>FTyLfet5{eZ@<dzG0yQ+&@Yq)=DD+&5
zb-(fIe~kBfZbuI8NR0?_ua*r@H$7Z8xBZ6L^B>3celGNx@WGk0XOZ^w)Kl+-%5G`#
zi6l7xxxSYvP379AZ|aefr-G)eU81vWg=ozGB{w2cqA&W)-`KNHS6R}2S!UnOFs|Ga
ziz_`1T7{lW<WAYgEO}*(t*P*)d#`eW&o24BeC|sfKlbz`s+^lYXgjw(d{!)++!fTl
z&a-8o#!idt7CQ^Szq&r{>yFfjh=Zqp?nvjJBoPq%W%7evJN_ia^!_}(zu0i^gdEkZ
zIYku{D&>+s)f_*h>~UAk#yZ&DahB9Hwf4nZKP^*sDPU8&({V7ely|D}T;1Cfx^Cr6
zlKD5wZC>=7>+vE}Wq)=QJ3eZW|6Mt?Vz%CmLj9ADo*$RgFzxoTl#6&@c09$zGUDBP
zuZDfB2Lh+0>|AE*oGEwGfobgo$Ml$Ga*E39V`t}yEVsXE^M6svt6;mry3Z{SYvvmj
za|h)#iptf8FMR#5J7yEps^GcjCDv%M3MPqOioTlluGs&j@zb|9_3ZoBpHO+~z5Hp<
z$@Oc!<W59$cHC+b?U_>B5<2Cs^4`c5+p@PzQPAD9P3qdi_hPK!mv$WJI=!LQH|GX(
z{Ztc1MPE;czrID6a@&h`-wO7cvG3cLo9Cx3>0EX;v~JO{4IRfSobK#fqSC_AbEIp{
z6wSMRoZAmLZ<1MKqrt^&d_m{b0`GmNMPs;je(|^`a#B<y|K1ygiJvrBb)5~Bl6P;(
zp4jt@_4sMlgWhbJ1$*APq;2V*u%?}-BPHn6vK*ENIR<(AIt-o^oZR2%bl}l{Wkr@2
z`xj2nws1X5$V<-XKKMiF;1MrwXSTbCI-YA4FxBTV`6u}-;CWt_qkm`d3%^5lGK(CH
za$Z{Xy!^FH!70aa*}Ee`?oL5(`lOov6umggw$U}9MIe=@JwRKqJ4gFe!ds6=$E37X
zRUEiq=$GE&_iWNppP*5G<ALp*C8`yg8WD~sYuLZddh@-H|3j4OPbLS(7()iTx1y;R
zMfYw{QE_0ra^_TvR=-Zap<ezyxy6${yqT!-vR3Jw#QHb>ZH209d?tD&eu~R-o$<WS
zu1C);(Sm_DENkKZMOmwAI=^|GxLDFJqq_F{j#HN|eViZikx92e_k!=`s_gbz{zg@<
zTs{9=n|fO9e0p^P&dSeydhy@l-+Yg*9>`z+t?OFYi);H>uV3JAJGHdU>&W%6o_R<3
zgjpYKR8iup($Oeu5|v?NnCX!kawzeGKZBun=daXw6Ze~CFW;#+-#*}bXU__iBPzUS
zIer!y*=T4T)H7h#)Dk=CtlRZ6IpejS)w8D}3-scSrE?^+W*Y=18BF@YqV@dExxQtr
zQ(e;6s886koX4g`idWtK#iOJtm!7>!;!Z0{nDODsGKJYcoS((KJFvn)FI(?mU(w}v
zpTbUT$o;V?<(9|ufJeFe7XP@##&ddue)^#wL60ua%g_4}X!qhV?_y5XX8|u6Sh$QQ
zJFH=~Z&7vBdtxmAKV{Cp53km}o3n*aXZtz<ZoxPE`b_x?-UwZE_fa{Xv03H88Nt{^
zYF_M{^)4-%al~UDi%5TqS4Z(XkBzdE+C+{#SySV^CgG##VqvT7i#n71W}RcPo~vUW
zY^b`8;j_%)<rSws1S#3X9tzl`t-401(yZC<)Pbe@CPn1AF{vqtzn*OOQdoShkLLEv
zCsn*<<j)?c$iJlacJ&(LAAD1O2$ej)v&^SQ-S<V{&wU@b{TEpp`WS3YU3(xf`iWa}
zh7%WKi1G5HzquwwJ@^o{%hf0+H-_uAvH-VWq>EtfXO)Hr?LrE1N*x}%n!0pWZ=Q8_
z;haUAdt5YEJaTV0Nm!<&>fya5;Qu0q*hgwgX=?0Q3MW{STRy34o?^PsbTCJc`@K@i
zwY1A#Y{?%u9{za8HjypUM!+_o%g1b$chzzhR@Hbf$Lej*oeyQsJ|-k?-FDDwnT}+S
zckd_0i_Toz3$`&_4cI<mO^0VMA5V(R1i5PNyIL3YBb}H+1r7OwRk?yai(W@}tjjyb
zP_V*ZR{OKj%L%gr@`QIRuyx{fQb{rK?wNDxCI8QkKT&In;)LhL@q}w$Dv~gC{ljri
z%U$TK!2%99uhc0E$_|yqed=7(E7Gu8WZ^6CZ3+xpSH$|ypKMrfRj>cDD?Ru!Z;Sxn
z<Y^x#oofHm78o1KR6XhVsa-Dpiv;af7BWBkUu-vZlilafKHq}lf0zC^=JBIysjQ`F
z^n9MnVTB2suQ<j&3Q|pWYSs->U38i4Y9c$+ua2ipK}qX$denk)o5SAPo19SKn&h%{
zeS}(*{PgF4q=c@gcwBc`q||Y=QX}Yume9(o7piIk3@Qzq3J!G`Y5$+s!CP16Ir)%R
z-wCfJ9SJMeYD@}EU9!S6-0AkTM`ac5NdZk^Rj%PZld>yc%a-e1meEi(OSo6SrFBx<
zPn@~$hKOq0DwgHTMPF@6)p~OH(_x*TU;ICdX>OdbCeKfl^=snqO$W<(bbB`SUU0Vg
zBDB_MYq(Tc@*_c}gUL*%9|v`wd|qZ$WV*FTxFKlnN|m(#=|+zWRDV4VVLnpAeq{Zf
z!!9K%5jK;RoP^5^51BB2G}-)a?tyn@k;SIbOWxR|r(NXpxWpGEaZPPnSmfka>Sam?
zn9dZ3HYLou<}o2yNn}E++P4br!(l276HN4L)%-Gc_$VEiKCSFH->uKK%O^U&DrNpO
zHGZ>j{8wZ9M*r-?U*q*n67MgQm7G#>V4Cd9?@m9z$4Yc;S25MkEnK4%CKtS@-N5XZ
z(Kp_gsoWPn>dEoGJQ3)?cIdaq3k98N3SF*teqpJin^f6OD@J~4%UhxGY-QT|@77A*
zOgWE~quaf{o!6N3^`F~mkCncg^#WD9r&$zRcz!tQQLl6-=i$m-n{DF$uTxc;mV0MW
zk?QoM;*FvwOcx#7UjBp6Z^tCRlLx|%3eETylK;0PC4x`?#Ehq(+OjX_u8|APIV#FC
zzrtqK6@%*@JiRNe`NR!xK2s}vrzR~n^Vtqn3(*(PZv4nP_g7^bZ)&>smIHQrXVyH~
zz-Jy@TKPmH&LwZhQm1_@j~wbO4Du0p%vp3tGhDFnSX$Zq5ARg2ELH7j$yh&qLBr(O
z7s+;hMy>^m9_?Y7e*8d{<mZXT&J}Dw!i?G`><eZnE9_c$(RL?;-TM<~-iX>47sfv{
zpY>7n1JkxO5*|McwmSa4KK1O9=~sJIRjw=&n6ZFu#&MtCM%U0(Wr^makDF_jCjS*|
z*8RU`E!U-_?pF7TBUgS*oF6_-F|;QpZG}sPueRE!*E7B-J!akh>gc~$>)$VU5f^5(
z$hBLv<A(|Z+pT$`olZg43(P!kED&Abq&X?)j$hKE3li$r-)4tzOgYD!Bfs59rGbY-
zbHc77*G*^UNG-|xIL*I$m(Jv%&d^6{^*l2^R@_|>5xoA;3h%21w#QaVmOD;ndERrp
z|DlCpp?8IrslJ4mYK-ARp;bnUA82_@pJftk|KPB#&pVfI9~~Vgo1}^>pL@w~zC_XK
z$m?62uAgR=Cq8ivONnh@aq~Xpc7s#(!5UA&hV<#v+EYbTj13cB@p-LzRMIoKI6>dC
zw%t>6`|R48PVfGIwYzO@e}B6D>+lK8KNBh~g9WF}ewt^<sg+pH6a80w&VGY!i+42K
zyuo<Y!pHcqP7`ahl!c;fQu09)2kjY=W%>ns^b5F-G?%tSoZ#QJUg^4((*ng$uGf#Q
z-(4B^F3>&lQCn%#{~jeZ)`BIyN}3B!#5a|xxK?>Bnx^`pbvlOt<8GBkwk#2g?RqzE
zY`bc(Qoyw5ue9o)u$=2ZwjJELJ?pm9)L*fEEBj2#C-^%ZFjp(GE?=#E$NT$_`u`O@
zLcyC&Z}Wr)uHL)F=#5kOzQYq8ukr?NkyD(>7q4sQ(<gE!HCjQ5-`sM!ep2DHsEAj;
z=7!9437cHxBst}}ezDI__iz7C?tSWCUiDc^O!|b;;{^q0JlodXGO$Wm@x463U)zdz
zhWIvpuWF&0{%4rCEfQI<Xw%O{+awzzqdXga9ynUl`S6eKk~InDyTcCc{5#J-#Bjxe
z?T4;7nc3dEncRCjc_QnDw>z7AUFYkD9Sy!VpIPcstXNuTOWQ7njZCxu2Hae9V_964
zQ`_p6P0`(Lr9U#v13i6KUs+vKZX>0-uA|egNOf|LdW}!!huen^mz`yeyslZ|c+&FR
zdz%>>rzgiGtuU>T4UFEp_}8|sd|kOaHz(E}mMmVX_<hytzt6k&8vO|B(KFv_eEo)W
z$iFvkcY;q(46orcoa_+yM?-1ypM>zq#k))%{{N-4sqbF2@Vtz&6TX$jU8NfYqm)7<
zR1=zxUo*Nf$L2$ev8vr(^|C{kYmaAcJ#={SnXRJJ9(|sE_991p`0vAS?@YaWXN&RT
zbB%i&X9*v?Y?#FVV|z{algg)zv;WGmDmZUtT61$}vFhtz6Rg)waBX2?FPJsk{oBU1
zd8IA!8Eva(E1zqf5_)KF^V`;!`fakau9$yRp1f_(%HID~S2pijd#Z6qXPbq`YG3sy
z)}Btk+$T1b8{9tb!MOip(}C~RqPB6{AJ0?G`_#DM?t#N1(GTY(6`#B9K4<mM>OxnW
z53+mu-Y&ZF^M?~>@a(UL`o+J^P?GcE{eP&u^$2%Fa>cev@zr-U|6fjtka6(Rh<Nf?
z<67;pokwS<eB~EaU!I>fH_!ZjeN7RM;E7O|w-b~~3Ow8%WvUj1ESwV~EUNaduKrz%
zlf`zYH;YXZstzrR&aRq!!s}b;xr2Wm8{heHXw@&%xNYILPwU@%YhG6&r}6%2%zNhP
zYYwgQ@tXH|Vv19{?bFN5$=p$M_o~`Ia9hpNa3PTQyO+}XcaieFtO`qV1PuxnFf_3X
zs?FIHz;IYIM$WB7Vxy7^qo`rt!$}h!JN8QmPxAEGwB+Pu4eu<)tE)n#gVYn{rf4##
zX)~I$Zm>L&_1w|SsBuwF=_cPpb8LdosW@jX37TX#eNs;9YPrBgoSvE<N{*cj|JSpN
zs)Q(*t+~XaK6%2F1fjQ%+nI!>v}!NVxw$dlbDHnY@0s@}7v29?d&}%?!rmNjHn9cQ
zQXcQ}f2lTQ&h`xtP8D*l5B(Lf`T3-3=f|erRVGg^rK+z8vu$M-=i{H7=d~kq30rIQ
zmKiMBK}}PRa4!8bZBgovX}-s2PFmR;v(TAQm?dY1P%YEfYu9C?$}MD-+Z)$Q?mNb@
zY4OkgCp!xs9h$c%nDPB%`6plGQpAt;ux{JsIl18fc?PzQNrCJW4Nv+1msMp5G*47@
z&<#-yIo*`+c_A`QB~VSFZB{17^fu`>jYl0SY!!)}>5LqA(%3g{U6jetb!>XvgwE8*
zj*K@<8~-z{I`~XkJhdQ@X@Uz|B$H&SL-=fl<OA&Yb9PNsnG+Ral$I(KRM46#BIKgj
z$`H@R{%l5k-o`sKz4H_&IGmGA5prg?{W8a;B<hHuc*q^eOSKAxS6IcrFI*8RlCq^F
z*e}8`L!^;&W21<F&!jIxjY~LOg|ziKLp6P-npsWupB=<?rFz~*$B=*zTa6yJL~IS7
z+^U~fF(V?Ykz;YZx~$rZwN0;H-RxYT`b8pN!#3@8UIS;g)CLv7YN7l%*KDc$YR}iN
zdl%jHh$>wjc;Qy|gJ~CBIa4QO=k0tk&-)Tf+}W}q#~`6+j|%j-9^Br0h;xf;k+131
zm=fktcG;5ak?D8;mt6b#V?uv)jkr+Ni<h24)grGmKUS(%UWq%>;%VrmJ%`OBxM_D+
z#brOEo!Od;-7GyqPV?D$Bx-y9=vuTd#$xuv6r1p62ilhw?)coZv+h~qd6(&HzIbk&
zc4f;RA9a?P%jGjow7s}k1g2dJbv~|Kvih$@%7m2Zf!lUu^v`~>Lw3dmN6y^+vfmw5
zeHJ#%WSh9rb+YElr7f*ta!Zzq#3flioxV$Sx!R1Ky@_eLX+m4n_T=sQ88e6XqUXeC
z0<lvj&Z{am5%O4K7h2Ug%hQofTi5ofLXhp>>z9{WxVBDuYs|RfrN?sn)GXgU@-9<D
z*UG1OYj2n*W^ygaTwcs2I%mVY|L0`%+`^MeZl1AOf66|0X4)=3wnaTv6YC?A=4RT5
zt>H51wY}u9c$&c0d*?I$WvupnFe_WO;9}#g=UfT9CY1;<aV5mCJ3e6ao^YUrrAtM|
zqrvfxYJ;yyCr`ZBiAN2hty?d=Zr&%@m#=j~|EbCHqAQ9KB?4xwVu#YKbyGYJA34mZ
zRl#l}qriVe?$}|w+9h@ppC9sBHZd?VFgiykc)pu<>`2g67CUyMP9L{R3tB8VoVHnb
zYS$fJBb1xSB{X5d-U~%Ke`?luDrzZ9T|aS5koTE><m%J0w~jBi`nJN9S6gVpt^hW>
zX{Y<u7@q9#67blua4j?Ul-Ud-Kesh21Tm=n_uydW*Jx0g*~=Zvd5?8|isi~s&w}tx
z0n=P&>OT}|FHE(%{P@c<wNr{qCaX7Io@2J%()y{Uzr@EF50zyz?|przDRjf-b<w3Z
zXWcG)_Qa4gEDoA1o?0P)Zh9JSW}1>!rxjwQ=(;Rr)7kiCPs)7@gBE55_D5|=p1Fa^
z`^UVa=QjSyRWJ>1`ZdMLPD=ND)>|GiyVTGv>nC(nT~X%Tu)uWxd6p}B--6|~Wu;8w
z)JZVC|7iXMHjM*aQ9`c#J||suPqG>X<Zyd>FtB<X32ozf;{A296i2w$oLbp*o~<9J
zdoMLMXp~%&U^u6bd*k8_%@%Kx9DMuvG}S^Dy%I=zYxwHFp5r75r;U9n%BQ<RoUa{Y
zxO+t7p!G?Oj88^focd0}Lid`MEO0w?;HiGb&FL|H%l5o#J*sQ}`sURg<|f;d)auI4
zY70NFJipCMaQ!mQ-rgIV7IS(eSaBtvzwv`(iO~$l3D1I}6>qCC?AUpBQ(^IKCG+1W
z&W3%P_{6G%^IF(R=5K`_&Yvd!d$Z%(8dU~{<C+)l%`;gxDaAA5h49k{e8-nfYs)NL
zeI#1@Cd>Nf1D||nE>3;^X3A;nmq8tVPIF5%En_crvZw|na0caka#?!fT+oB3o)vQ~
zcc)ZwX-@n!x53X<EA(daO!k=$Oby1@r}*&d3w6xf`)$(<hL5J1-&g`7S^sbGTUdC$
z>XCDXy?nCW?i;FxO<M!6J5Am%*UH@>qI)=lRl@V)yp%~?Yu2a;TX^=WwJdi0q){Dl
zW9qs;I*f{}4opk}3=Av`91I%@`xp(DO`P>)N8{_WULREt=_{%)%b2D4?9uE)noB0M
z-a5#~llWLx>QO<5+Qk{untuqsh$?S)b!o`|cKPnASu<x^6i)Kl#x<X1=b0`+t4@(^
zlXQ%_RunVeY)J1jdvM~%PJ`69pJ!g&5wJ-iB-)g9_MsT(ZMNSo<}5ayz2nr@1A0E+
zE=@S|M%gr4Xh-1pBkYFEjQ-wBuDtR$Ej20W3;TO3>~zHOb!A&US%mf&OHNl&|NCUB
z#Nz1xfmNFEfn36lGRw_F`L<78?^C8TS$lb!f6C6vPj}qPIt>q<NH3Wu>HKkBkWg}#
zu%_O{WfAtOM~z-D37m5&#MJuh<l^a-!5pVnrQgsNTD0ZxoV7-J;T@s-8|<><^zOxZ
zu2{bJ|F7-qm6{B?Ip+CS3tAkgI62Mfb<o3(U;Z8v+g30H1|>}@sd1KGo8lgCe3;Xt
zF_vpWx(BntW2x&F$Mx@TJX7A2%*gPdgZ*HoXT=;A1^E^WrVG6~+^@DM)^AuEW#Pv;
zQFt%UG4qR8557EDBz1UR(sApEzKElfqBeG>{#23n<`mkL-TmLeRIOR;!pR#kj*~Yi
zcG)?K%-Jjv#&z?5)0%xkCfA;wE?;t0RQ2QQ?t6xx%YR7zTUUQ=_5I(O5BBX7Tz5+Q
zO`<=q^f!}TIc}FWq#RFu>F9Q8)BLvu>4KiWCM$fh%jV8io!nA*W#6+at-1GC#hxyT
zxV`V#`lEaM?jL%5jM<}<^P%+QIfr-bZwxAE2zHw0mT8eVnRCY~nUfn;pP!00PdPb#
zcIxvC$+!J6DkXlOjHdfCxh!3=PWE!jvxPy2VyE0-DZeYXes}WqcS&wXg92tIl`wv}
z@@-PFW2rjB+YPK4ivCJL@zsv;E$z0NsRcdd_OlXeGRq4RA0{-0CjMg9`}^5mYI>so
z#6Ug~CAJh#k4>dI-^@QvVK8t@{?94EqPIQS{8^F3v}DtE28-*}X43^MB3#%Tn^g`r
zTc!)yi1}=B6jFU^!hTSCZlk`>59x#?8F#i|&u1k;3({|-8}Ln(jpA?)c5^mKaen;K
z;L-F_Ta!AQFLE9SBs`C3g@g*J7&p$FsF3Ym=bs{C&?qWzEE<_2w9TR6H>+gx^Xdlm
z4o$|0?sB*Jg%V|IlC^2+t;-d^Uluo05;qg7ZkuW#Sy5T@#C`I$%11_#>`%-VWMmdS
z&YZhcL{Xh#QJBMHCly%-RvCusm1^Rf!>f0G&)TJ)z4v&SreGQqLyN{Dv9pS?c5XF&
z7qcTa<uChEoV<wjh*->VGs`2XJ*V3&|Iaa2FZ&d8Wtn;nQ<=J=`~el8wnIkoQ$;e;
z;?x3tx+nQgG_szu$cOWz(8(*BDbuXqwD~UHR6nax^C3(6(-J>J<{-`X0<-4U-y+H`
z<^^n53+fgYoJ)?McTxOZvKY6h2{(h;PiIDv4fboNCyIqeF)=mG7c6o-BzOO#le)UU
z*77FZ<;nUS#m4T%Hf6=eo<dgcLUt8GmXa+>laliTd-euK&$`&+yTZb=JjHXF^ttVN
zrlqnvTY@`8omm`Hc|Vs%EDPDA;^G!67kAvbCr~58(dc$-+0$pSDxcc68*?c)TKgUJ
z<t`EqOmVHwaAmsCu9e|7&yZ2tFrBHaLUxkQ|CZwws?43sH!9zmEG3*C-1SYjJJf6u
zTjqqNmD5Z;x}RkB9f_Q3S;fg1#h>EP_^9glWvLb0loP)gZ@Heeb$RuwpIMeO!Yh_4
zI&kz@RR}rkXx>~Yuqji>K|)pKXwSjG=!i)zogd9lu83}VT$17>a8xbkQb*2F*WMGN
z>gUbo&bl1)uzYTcMcdw;&OwKTK719)yP5asrxjOGd+IT(6Cu-1y41gOn|>`NRN?Y8
z?n_+X9)t=vi=B{Eh%-!&P4;h-h-k``;#x6(qf(;>haF>>Dc33n@yJ4%4HL!F1ErTG
zzA%bX$P`j26XKr|s1TOKa>QL;P-Wf5r0W-xybeq<yf*2-p+>SpL$z^+i_J6!3-!eo
zCns-PIh$eA<h7BDomNaXpDAEoQPWZ=eM)J`vFk}OE-8L1N`5y@Ik8b~AEUbZbOE!e
zPVpkL`d<V%m8Tj%2{tJSsXa6`?q%<tlc|Yfj6oNKzMiOi#@4R3SU4w5;ZdZ}{hbQM
zk>#~NjD;F9{+~4deAO;YSa#p_4jxzK>q<%)ALCzd5jS(K6i`&@WAo^9UAV|BO3lGz
zNk!%o56|ThGbemmIp58Esi?*hwpp7bsv{?_(n;@LFP6Rb)U26;9U2e2H~thc|JBVI
zxO(2i+4C=YF8{20z{ui|m)D-3vn!Hw&RSZYVpBM~tX4!YH)ez2trNZfZ&$>=P|Sb2
zykC1`?){y0!HvQ!pEQn_h`jcY_#xh(!6x!;y29?waUU!55+eDx75F`NWBVv#!tho6
zyU>cCFY~<;8yLkKb($6&R4Tj~6mfS_<DbKovO5Y3P6Wz*mE|Zd+P`tq>RXd6%eu{{
zZFFa0Fsmq*&n|XnVaae@z2jB4^{!(3S(ELx6s)?0^mi*dOqj2_kyY)hq)TPV<jYG8
zSc88rVmxHF)PMPmh!alcOM`C+P7S#>HPxdn?zwZ!NpGiXRvDhw-e2;9Ql~{l8yS2o
z%X44uV#w9c6jqYSJ!e6armI9-r|J96>8&$5_$A_3UW{L)Xz+2Vea}kwH07*C&)EJ?
zmhzaU6*;wa<#aFe$ss$IPT!&G7}XP<x$ZjSqLYmLN0ql!%v%3y)w*4a4ZD?g-Rj=<
zOVUDO*Y=e=tvGibaIZVMgH@JAtf9H)Xs7C_bc_9&F{f9WuP})T-k@g0GIyG?_rB9P
z^ObYUjdCBf=icm`>pNQ@a0Bbw$#K$$VjVBm-7T|zHY4qk>fWc*gg>}Tyx2J{EQMb{
zsNrqGdM<~}EX%aNi%#TUxuQ-{IkdS^N;6O>YN6z{#JOfka%~I!4VxCYF)6EWyyCbq
zdZVcEwWf8)HgcR<WEWZNc4V=c=B}2;O*_423%`+^eS7kP=v`idj1HDHI-evIdZI<W
zr(7wYa`Irw|Ex#SYV2xFT3b5HmR@=|C0fJT+dDO};&9kX$y+Z=^ArWPeh7)|-ulW@
zDD?HRZ1K>*g+8}tr|A|h=L{@M+_Jo`wW9WE$40IhGYgelW=aWY*cU2_xpZ$|cX3Aj
zZPuw#$EK}H%yphQ`_{2}sWVj<&J<5Q#`0yyL&5Hi;j^|z9GHIl`1+}v@^-C`=$XAk
zYDP=ViPkwM>^%;89bE0t$igfrpXaoBW030MTQy0Ggidi<UhLR?>Q%Ak$K7Xl#heaZ
zyGU5*hFHkToKrSStd0inbq#bqb0l|$QJ!<6&{m&ys*Zhkw`$z~cIwx6$-mb%bRTLl
zyi`anSkHb_$3JmF2=jsk4=4P;5M#>QHBne#zhGCQ@UFAWM+%jWIjD-AyRVa^ad3ZX
zK$FbOrEeZ3X@?&$nKp|{=DfkHjSdOTMp@^TwGY}w3UNHxxc9|*8@JUK;<NXrH#70}
zFwZR(%4xRVu5DFykm1wfXVn6W8T9=(w|x6#?wqFoYf*{V7S5OtOMYxRr026GK)p2N
zxq!F#va7QWJ09I49(=-ci@d_&W#O;u9(Yd6n{heX`=W}mP#H%$lft$tkL9JFT!|@e
zp1l)pZ{GGvRI)g+@#;nf5r@i<1eM;e8MDGyPFq$vjmvXJ6jO^!)%4J+>7Ek3+zu16
zvMO{KSL`~zrfcW)+s%zVZgXcV8CdTU;5qTXW6$-v*n|1A#d8)pUYjKlb;0pqSI=qn
z-Hz?qTdRYz9!2k5t!89$BezLte|PO|^|klZt&UhPyKa>m-g&Ar`NpzKLg%ti-><s-
zyF$cCaoxk88tZdikA69|ar0h{k8&=ISHeR4R&Cm0`dRf=iyO0+yJ&>~_kji1&P<e9
zaW-wju_v3(%DZJg@HzKHCMknAOG0sxO1gn^%_6NCi}W;-AL}mWymP?j*nuvd8{U_9
zZoO5_*?VFSS4+Xd)eZuTferVg7I6NyWt$bSN%hkOsh-K!KQHL6ZhJOeh^4j0lgr6J
zLcs4;(BX-@wPe&bsjT4G=;Xicp}`f|*sjAKs>_nPLL$u`{kPf7x@NON+Vy2??mn`~
zz3jM0(NfS=Suph34~_VzM~*dL>9jh^;1*seT4}0!Oyq+7tQ(z=4qaU|W93Y?9VurW
zz9(M$u{+Z9kVEmFu2m6wOTSfB*mSRYb*@PFx+TZO(%5G^q!^RDxz;qyTYG?SjR31F
z%WRLwC*JVhD9?SqU%mE`%Z(k^T&LT3ooW|4*y=b}hjnu93zJaGGq2X%4nO7i^~Kd)
zrzYqz$ox5#bTl`q^)W+}=)+lc*{?JoXbFE%&3n1L@0Q4!d4^{m-r{;|uzuP-wiH)g
z*UR(U%I*}YB=T0B<$7%|`F!Ht9EV`NRr1UCuZW0ZaZyo^G|=)qAijIm4)N>%b#5K_
z`sO@O?PSK-^RI31+goj#e|vGtx(n*nn;7PvDDRtnBTne~JeD;Np1sjyc=O<0)Py##
z!UvVIC+-!u=q4@scVw4F$CBSsYI;vEo))}$c;cMIq)VYy!CdEqi$0Y`dgi8f3Aw0l
z<)6%wu=0`H-OGk@N7BRhn#C!_dJARkJpI^g+wsHgOC7>mJ7?4#UlF2pta3?b)Zd7U
zdn;$CUQ3O=)~!<YZFS__D2K&XtCni1Eb?S5a}nEOkv#46@x*hps$-wkM!lQ!Tja~x
z6NQFc!bQScHgH*P<P7t<?zM37=e5^U_`kjGUF_JlnXf&1-R>Jdm8P`xn%8`eZd<o|
ze$CBm?lu1xt%(ty=*^oVq|`b0a+`X2llR>hB0?Vw=hU4p%m{scEB8}G+}qCE^O7|0
zr3md_`$+jB*QZ~%CmE&31-?F(dv;ah*}s#`KEK{DHD$llD+k$MiJXaS3$_LRmv&Z5
zFJiuP?rH2j<BIb-QORyk#Gd<Ys?fX7+;@VrfrW?n9q*YxPwY;F`9II-U(M+IHDb=}
z34h*g2v+!%_bpBC;ATz7L+LH-T1yma^kWPU`TbDc{_vu{>ZPATOWFOGwyb^lP5yB7
zE8Wm-{}tkl7!JA~aA;y;;qsW!;<Gx8)swTr;zOcK2aBecPQ=7VuHBM`RV<Q5D%y-W
zZni4UT}mgL)<&eIq;~&L&}}l>t}<gmLQ@N;u!YaV14~(^8##5BX<l5)6SPEj(zF)}
z3s!P6sx2zp=w*<_zDjbj&>O1_Eg_7lr{;+olw`2;$<+8QSP*=$iG_>l#@5*i4fj|W
ze*OC@y<<HC0}FSz%bbXhN&OrW_H8neDNIdM1g*scmM&zNrV>4IQc=Z%RK^*`v)Xc^
zOix_xk<Gu!VziAxmqpb>aHflskmjz+-_x3IKb~e7eL_NBBelEe+3C63;u~5Qd}+KQ
z=r7n%%O(Ey=H}w>8UZ#3^->N?yV%FsRxt{B7`9k$IP1}*^PIck{7zoau-TD!|M%M;
zeJAbV&~$4@#{S<*N{=_P#d>uf2>X9)<I=fpYIE|K7M{{*bmq5a`RL4VE9GtUS-?1Z
z<)S98tY%HMia<`*6dxr<>F>9+H!%ERQesdLVsL6uXmn`sQ0_j$!*G0>&I<z;m*%1c
z>eG2PnP@N7@jcntv8KnMML>&(v#IHcuaM@V4hGIE0?ZC3NAH~B@YA_na>iG0>6%HN
zr|Z@fooQ66;9nAEGV!vDGe56p*O4U?8qE5qC~yRMlqQFC-qy?2-}Pc!1FxBp*-fYJ
zr+TR&U7Q8RQx|ZuL~IQWi*FTl<62hmaryj3yRTkoHo8^1;A4$1|6YzI8O;l)3%KSi
z>avR}+t{m8ePRh4chcfT+|o)<Z-i-mIkAlM|E(JrSeBd;5Z79LrnF0Y-Kt++4Q1R}
zU2B9+YVm6>ZYw&ymQ(D6Cu9A4rU-pj7G(*81AAmv1f+<&iE(I}NIFl}RhSlWQQzIU
z%=e_pyfxlE8+{}?bJ!QXbPC$`;q{gy7GXCGrVH0eCeHK?masg!#beR7o)T#ei}}^U
zddhzGw(G2(u_WYmJzLbgZr4k$LwO5U?45jT_gl6_Og7Dn?rr7P;u6Y;YF;5cyL-)c
zzG{W<pR!jjNS3VotvKn)I{n4%QHlbNf4)6l6V-h~VX5OrXQ9~(r`WA7bBc~yxN?En
zwiykFd%Rq-7M$`oIbC$r=y?;5UPQ|uSIx9J6W6PhUE%&J_x~@~B435uB0>)jznii%
zbw;eRN3x-$8t)Q?#TT3=Kj6IN!sV&<E;e<7q+Z6>Ll%6UCYMaqf`k~|B$-1rw<MTw
zzKBe=^SkYw#mn(}jkI&aqFiZ?OW!ybm2CIcO%-|h(&A~!<TuuL_gx9j55DCP_H-ie
zo>Q)?trnCem`sSS)ytCkAj)#{u=a#6T83hKd>6!h(n+;Ft$eEJM6$&vM`70%m8(*U
z=^UFRJ9+ffdOveWD!H%wlyOekZ(~gOE7J)}dt7xxPK5hES=6wLK}X-xDB>W8+U5_Y
z7dv|#osMK3?h4$X6n!Ij%DGA3UG(hOJeVAqCj2r|RqcK6CH^`h`M~x)|KIDnaq)V^
z{}N*rn*Vm0dD#|ukDkr$zZbr;tiLj)gk`1Xa!&)5yN*k)o^cac7kXjh0ip9^N|gfN
zuQavFZ9ISbU-OJ)K@P(j>kDeHnHL03Vw&=4qQ_R2?-J7){G3Fx+EpW8aGjW#7&p87
z!E+f2ZSRX3iZur>#@8KDPON(A5wpl=;;iYaM-4nx+Bf#hj#3u7oFk}Guqo|}(V|vg
zjV{qQKbA2ZIN7JGp_QTK+&j(alKRmnncl6L!Zs^s?mqS9q~{wS34uo|{p)5;G;BTe
z{B2gbuj!+gWxs?ybF(h5@GuPt5_+M!(jatpKxvln{8v*}^tG+BsX7uWUmNI=7^405
z$^TOm`Zp>hU1L6Z$R$y6dP<nEGE?xxf+NdhBhPRu8_jR8C{nmlC6K=DYG2;U{I*oX
z=72X}4=X%NVJg*Bifei)uA=oZA@EgF*w(B?z8Q`}YZ6>_Zx{+haV!v9^WdID(6&&M
zZH0zczeFT05f5)Ho0#?`CeHEME5ZE{>?+USWSp0Xa=-te(Bjz66@{}lM=a5G&phXO
zPT?<?sasfB)`N*puIw-~{UozQ!$#$V(nYf<uYK%xdK1sz<~m~IziOM3e&=ki`A40?
z82AcSyY$@uaJh@)s+hfq(Q0=yMMlSePwI_6J&kKRF#Xy_zEa1EcYS%w&+A5hUboRY
z*>UsgO*c(7mH*#liI<*eu<xT~+WPpWAfaOdorVp@Do1Z05f?pX_~B8W-~HIflIL%`
z)#r(>-ow1+!jc7z6L$DBd|WK=q8eq-*q~J}c2s`3nU}1l&)SAi=Lg-3go7SW6S3wq
z>0l}UYMG#zb!6YIt4`;n<OQ-=RLpY=xl(nUPP~{FslW5Wou*y#Geub1QW{o;^v`fn
z(x152-EC9Y;tiecouLe?U+|<K6KFMDp?Na=8%OG^?)aWe$@Q5wS4?itFrKaJ#mT7l
zMz;R)3Wn^57W1|I<bVHgf8jB0Lb;fy($_b0*B5R(B6(7V^=PrFp2sKu*(+EBMGw!v
zcFos;t!U2e5RS~iuA;3>i~hf?k!n&n;OKUFZNX9(38l&0mM$}v>-kIwlC>^NYV#<%
z6*y_blhnf&m%Y6&31!tinW)I=c_!{wlEci53%Y{RyW{>ndb?mtcKW7ZlfA*~KOCE-
zaiha<cTCZ<#5W}~6P-Q1Kb=`U`9<cvKULyooYg^RxRi3r?A>m^KNie%C}Ob+&ug=T
z59L}~!vr20e5rlko#EoG8t85+acDu2!9SC&jSSTyVQUtJdc?Y1ig_5Rq8WRp^XlXl
z=KRFyADS1^C8jz3nozLS@Tg{TfW?GBk6ZQvHy;FadvQr~ZrSg9$LQyVJ;igopSTBB
zoDhq+G~pO$|4Gfwc{^Y3`m(2P=kq<kKbidB`qR)PZ)X7K#>Yu>7q>nVd2SKs|K=B)
zk;NO)md7&UZ*IQjkn#SgX2H7t-0bfsR@&$--`JBnYp(F(NY^BHs~ab8rPZ2lT|f1y
zLaar{wH?MU;$0oMg>RLd46P`dxI;|E?rdh#qc(**T`t>s7-b%@F~vyawLPnSHF@gI
z6(wf5-}iMf>Zg^dS1a1;`@M3@zbd%wm+_?PnXglNOeeE9$b0$-MOmD5Sz^(pyW-{5
z4gJ~YK1VO#-jq^R*!ubykLi;5WBL|<tt);A?z3+(i|Ff-%)8Lk_Ki87>mYa9ilQaP
zbuHyt|C)K;Ft8Rp<V~+=P(0fg$+>amL_VR}3d}7$J)8e8;OO8B+R*xM#%%G)y1QrE
zbPMbhFxr`R(1GiWgQs%G)abQs71H|+m-6%IO|W+SvqEs*Zlik&tv3@6&h=)vGeP31
zg<MBxuR+vmi6@RnZWyI_H*DqZvv*o!ep&v=t6j=1#uF^|tEqGx<m5cSw8muhVq@-3
z+ey29XY#F$min{VV!{n$m#95&8Eihh;OqZk5I<4HIeO2TTT{(fwR7=s-V&O^RWsYC
zdF}jHD(6pdBug&sVGy`2)X8>n?U94CJAar&uV9ay#W_pZTyx^&dxeuToF-`pF4qp+
zx~zM$(oM6cCuQ%gSX8Fatg>P6)db1An{9pX9K3(qbk*T?Pd}>LGS2(N{9n}5W3tzi
z)uo^3?vWDOZp3rDTPQqPC~=FBmZzE4Lv8bqW;Tuum4|l)3ECS;ux3s<_GYEqU&j@n
zcON=pvH#?%m5!gx87GJoinz0SZ)nt>y1l#iSF-!>*HiRsL>Y|~m>+HEIL#N$sFcs6
zn-b{ItkA0)F^4;5T9krizrYE;C5A2^7w~FzYAxcQS}^<QOT+$%gK{=gelhL(&N1b}
z1`p?j`evGwOgfuXI+$c-7XPu*-@9p_%4EB@tJEGa%zGv<RcX(PSt?Sq=Ny}rv%z0-
z_l*sF5<NT`E$)w2@Xd9cwLN?PmSE$bh1M${YJ5F-XvGz$KOYxct=?>X`FPI*YujL(
zgSY=noo13=AG3$Et1mZkVVT9!0I#*X1RJ)LC@~-0Gx5ci1A+68XvpM9?!Lcjg3E8s
z&It|HiT2lb@4aTd;fC|Hmy-HRZnA`J5cqzor+D_H#K$vC1$pCB_8yFpsC(q1>?rau
zc~@o8)>B9H@6TTT?5AMW<OYo$y$=q`x9mPPWx=rnGu>{soSin=x1+=Npr-Z4i*9Y5
zi<=lk86}#hWlL>tmS?TDFJW1*AoKXI-TQwgTX1o6{=3b@d%BNx784^&7ysji^*?3a
zix>*tcAqMBp83p_+ZxkEoh|uR&7K*>$?Id;lVNzYV`HC$;@wly!Zy={zHAJ7eYoq5
z$FrMF-y06>dvb!o^}kWIQO`7&fVZ1?3_Tk-e)KBl^eJt*Fnf<@XNF~e#?Hwv0)lP?
z2u$s``QT)afY(|p$;T=7S~0uVOg6r=BjC{l9jo2edz*RIUEwp{BDLzE&Z>i_mMb21
z628<lp}}{Bz^p9kV+mVMzdp@msd{zN1fNU%yBc-<lv{5zaY}1<ZnHU(HM2h{&_yC@
z;r<fO_luU_^jwyt<96%fk?zc`mlnvL3OL3R;lr({wZW@T$#dSd0QJnE;Mmi4u}@5+
ze)}{D`Ls4}Pc*)AfA==q)NQ4!g>NjJ{Ho|!)#rU(S$;Q8$Yq~#dyuoC@x;8>f}uZ4
zHu5!{J+RRC)|_L!Z+(wU=o8KjZTj)ww@E^8Riu5<72d{RgQqQlQKI%7i>=$7d01}k
z|J%KT<rimH^#c3XN<E)<u&>!OLE!R@MSGf3c=((A(;Xcr9JoADlcRIbl1USr4rT<j
zHS(VC;FzKru9p<>;`FATi<T-A0v1d>*)=I}1GBZV%$&~)LvJc{@E$$z<Z?jg0*(gO
z2nm(IqZf|1igw<xSgx6|`)<bmLs7=}HaKx@m3RDQW%npzxAm+&)w7OY(2@0Baz<<8
ze(61Xy(2fdNEvNt|8Zcp4U5anuPWK8r)O`qky?DZNno~cD#zY~rz5(YU6U0xUu-?_
zXTAAKyQUd?rxxxqcI}V~?90jQ@U57X;U$sltbXLdf7@lg+wY#_*uZ*wt*Y<TBSNv#
z`}BX#TarBKkjL#etFLeboPB$-X;-h?uiIA|HS8bqoC|n*JK*TNQkIwlf5Z85#Q3~8
z@66%IJmNd8TXDge)AM2^dR}dx*C<~Y(hxIszL(<E_sna`YBsPdFKN3x<$v_lS4aHb
zESbe-w35k2GfmfjgP{MeNe#!7Wv(9z6Z)jkxM$^r75-_7=LK$QiW+qWe(hlWsOjx=
z@RIcP+0o%<jFU7bG`->MEvvovx_2(e)7}|(to&|<sHpI%^6c)OboCQ!=;zW|UsP2#
zDqk$^;W$xxN4vFyf9d_#f`NPAdhF-wJ7{HK`Z@l=4?D}pR+6p8lDz+y_^$L?{xDql
z<|)~~r|dTGI3&#Q{!5>!O+tSGf6!un-z0DUrj@pHOx$Oi>=Qh=^^VH+KmFRe2hS}Q
zb70ZxxiQW4O#j&%Oo^AH@~U^GcKHPV3BI-?@sHQC%Tvs4TQ8qD^6*~rvL=b!hKeC>
zrCUE|om99v_m|MN=-(4}ES;h_MWXCUh{fS!d*+6!-?~!zCnQJYDxYog<E7_=u5ui!
z?chs|X-IYQpZmDy#T5ywnQf1EcqV#1Y1z+k?0)H=t1Zd1CMaBEa=QDrWMcK5bJ3QE
zuP;>)W|ri9yJN-y{_k@fwhQk3cVNkp8UB_viv-sk@aEDMO;YrZl94~$|Lo{JHLjkp
zd;cx3I52RgdU1V?%Q+G-`^wXpZI6Y&Ubm9tvwao*Csh7KZ?8ZphgJ}$$gWFIj5!6q
zN?Z0hzWU%c?`3kA=%Hmk2`j!>IcoJSym{!@ufVTYy*51Ay~;9uEvL5uPruuz4)d<1
zGesF(Uq4uQ+C)!DVE@FMoJ&qSnK(Avxcrsa8x`Tgee;^Gh3x5G$<tT#j9JgV-DDbl
zIx37=x$D^SjlM@;N^@*XjhYbg$lzs$CSPXd=UdsEO((4j2|Br*efNq`+pE+4uHLy5
za4%&3-6_ehZk$yYIX8FO;+bNd2W_AE^WHHjyR)dThwqT@?UH1%C4EtI*PJ;TbJq79
z>tx<f#m8GF_dSaF|KQ)`Syw_8_DA1M?M<1+%kN>gky$xNv%$u1o3o1PojX$<rZn^9
zuqx_qy{TfVz4?!JQR~xv+fFFVeERmR`BRB_fvIj&Jvg<h(nQ<R__y8{aDA3I@7V<1
zSx0Vp&OLHJv!z2YDCeBP!nQ4o(%W)Q-rFg7i-|dSch}$e^P&%4C`d_uz2~^fOZu$W
z(GSKY^AhBrS~o68xZ`$n<G~%@Zm|4s;n94X;xNU;TUu2!HGA8#Jxqcw$9@FmBsPnr
zHA{$He)&N7fi-W#E}kW)O%M7^dVP84lqZ|aO>gVHl<gCn{B2!^>bn=ceeb{f1>aQ8
z^i~z^mO6LG<?56jYyX8Mnu*0R>i-WlU*<NwEOV}2%(U*sd#5hm%gWj8n{2YIz|>A;
ziCY$@JCD)4-T>B9Uo-<hpXLydZ*?>L_}!Cf#j&*aukS3^dF)ts{FYn?lTz6K;1vGX
z=d5?X{SbNJp5s2bWEr1b4u(%VydT8|oO5Wac{e-ugma=|=bX+5n_6cV#O>eqX@Y<-
zLs{PkRjH{LJhXrH7+m)-{LfQj`Z>q(bJzyo)P0LGJ_cM=%i>ml<~;GE|J={tRA<du
za%@&y0pAv-X&oJ@b;Tyrc-nZMInVD4;587A|CoKwYHnd6)3t~{no^vrFHVq6KW33$
zu}11>!-MrI*-afyjJ~Pr;io4TpH_8$mvPr&=g0p~cd=x2XiWPpaI3OyrQrv;ciLuq
zUfr7E8RaN?HfBoj)~_7Txo@9{a82e{zEbS^dUleLX9x4-+ttc%{<>YgpmyQH>a{uN
z%g%k!-LPs}?YHNR7cL$4-TCuNYNp%%&#YHlOz$l;i#T+(z|ua8Y5&>VGKG3oADSlg
zwVqnM`E*Q2*8YyCR}WNu^f@Y@sLJu`NXeDAuhy}>_Fc~B;dkI>D*vT~{VVV4Uhg{K
z-Iv{K@kz2<cgeAhvvzHn{q+Fz&9D%@&L68jsVsh?y<MpCD(^elgiMythUym#-t*7Z
zH{HJO@O*a5$(*0P{}r3C%jC3oERk=0lxithn)9%#XYD<=q;)y-Wd6q=^o%&%SH)Fw
zf@xy>(Xuz|+#?d*O6uCh{vPGsvN~1F*0}H2jrVtN<WI{zvFYN6n<Y$p9>`v;c9aU5
zxT7~cR-(^cHE*i(zQA{??OP^IxPMOjJ3osS2TO4EgVSoVM^l6+y*|9vbmkfT!!`ak
zw|_WVM+r}ky}ihwDgR#fvG3DN&982nGB3K@Ds*~6$F1!XZXN%7=RJS&%36*kwfEWo
zy~<O4p>|Jq?X<_{|8!N$GfrMR67W5u^xvViA6`CJ;wd*&Yl*kso_tcTn*Dxe;M)1c
zv!*a}SDd{XT4M6Ur}UiF@{jG_*ECn0i!4lWv)lAQjf2xy`_S=}Jt1FH89Tz5*o1Wc
zdvqyiGqQ7w`FL!&>eR|3ZZ+kEo5RtTT#jo}5)(t#hHbMD+TyCk$k3)0zA5MBr=<)m
z%u5ff$(*b(iNjMxWuj7Q_Y}j~CnhvyYzaLPrZiD6G^%^*<ra^nc1JrtygJYmvDQe$
z;=*c%#w7VRU2a1~?VyaERVl_^k5+V;hvq3ols|}VtNeaT)Zokdcq3<bPBrI<hZ@`4
zLeoN4PnZ%NG~xa}*@()#)4DSxv*u|!82fWFIlb4>UAl!uHE7Wf7OjS5(;8E@_1rY7
z?#?~CBeU+`&!689oSVH|K*jFD4u&U(xQiN39<BI1O|YV0^7N*wZfn-cq={eHyYx=u
zeeJxmiomD;E2lDfPCPhoL+t9Rtq+V<CbKwrF><mo-PB>=^fTg|;L^aXsQ93fO^rdJ
z!BBCcgAg~@g?JwpRfmVI%w8uZH2e~r;li@Ub78AkG!Ls}NWj76SGqo1Ze7HnyUIL1
z(%{gFDG|3@mo5$p+ImoSLfq1hi)G@Xk<%tJ#e@V_z1%Bu&s}??`vr+<KUEK;vZmg=
zlo~7aWNAH%tjV+KLcN}C)0jh7rp1M<Q<>tGy~#x)AgDv_Rs*xvhRH(J%fx2QwZ8fC
z@`BphZJnK+N~<In9Vm30P!R27c(8DaTDoT8iA*a2_G5(?B-vS9pU13VD%cob|J&4J
za(<E5%^=r+S(X<=Ia~if7pfFmyCg6&(r`-by7-$+(->qmTY_eoGAg>Zn>`H|@cNxN
zZE~hPYwB%{%-rp<9*Vrb-v&M4+dV1baGOZjcAurY^+MO!gkFvM7_~HXQ*z|-uqeKj
ztT*Ry2Y$O$q{q4J(t!ram>UVq!eZACEiuVa4g0<8s8<qC^2-ei`)`SSNXr#Ud7iOs
zp_-9@OB8oV`p&aq(x-Q^mVe6EW1pT=XqrAt=A7~BD|gNw$!weBU2*Bn>+tBOUOLXA
zif2#tM=;)I+n;gk%CV{xgSW98zg^g|I7g64G1rlkuSLPnRB*?t#{VnNCfqc%y)7@m
zvE<9chRdD~D>_{Szbg0XY`b)>rt$E9_6lpmvXq?l#(XY29`x$><vnC_%@ce2O>67o
z@PwP(Q_G#J>Qth=y6OTan@y~IeW5sM=db+<%luZmZ(j6HZH`%GPk8+7H7l(h7aa||
zFt2LbGxaR5JGC|o*FRmS@p;F?V;9(X0;OudObNQB$r_w6KXdthP4<_sS(dVt-C}ZC
zP%|@<bEV}gZpSa7Q}kNh_1|oGzDw}$lao(%J(*{!9Nc_pyHHB&O8aG<OsCkCXD(iH
zI>!5>Sn!#v_H8{WdO|YO8Dv%*<S;s_73>(k+n~u}@x%!kvy{1gST>0@`?wkT2|e)o
z5j0`HNSguY?uCgw#S>Joaw-2g6&@4y!b;+hN<!(w|GcZ&ik45DxUAE5<F)J<<%XLz
zZCyKm^7<w$Gj=cZQJrn&clLmWn;sKSjj7~BuUm#ft9;ig#AQZ3KXQ2PR1@FUN0l;;
z4=nUo3byoLkL164j(L975sgCQgXbL%OjBeC^k85-Gueyd%6xX)AXVN=oS%G5*bZ%2
zuAwV6r+Wg^sef}XOz=oP!J*!<W38mOT|=;{yBhoAMQ1-1bv_Ps{BpulX=1A9nh#68
z<0AIoEbQ!9C%vft(el+-T~``yExsE2BxHA!&1Y2?K|bGi%NFuGcBtQVIlExObB#NS
ztMt0PrX0JpQqpImaq!L0a?3jox4z?Sn71_PoGtgH^NlX<YDSe}jsM*q&%U2|us`tB
z3C&O^wfMPyKGB_X9aV%@w{@(`IH(jAu64sc$YbFo7ZU-s`5i{LbCMce3#ae0&{$WX
zc}`7gZGw~Wj!Dzg0(y;PIsNjZbe5zTEWLXxGt_R!p-Mr~#s7ASM_hUokm~ftt2XB3
zfmg+on*yh{HDum6toQ7~nm+-aH&;m<(^)p*_R`G?u{N(ICu*>&^1Sd<EPj;fb!_>y
zl!>#n7WbYB5Zk=>*Yj@6!#B?+2t~B<h^gJuIpOc%bfVwLtxv!6rNa$|nVLU0s=mG1
zchw-AyZXkXsYfOqYP)i55}T>ED#NdUj(28CzC}gRwbFO)Cvo3CRQgm~UBp8xa?=0X
z2D)Jkd#CpMFo&;6GVI%N=&sbxNzo?e(}c2<9?IN*aa`b1?y`la?p)eGYnQ5arRMI{
z*;%g^UAeGx!q>9PCd*AeZc+>C*<x(sd1zh}`~8O>U;01Pxx(}J(4Jzu^J`c-?>Vee
zXJCxZKM;Jw;k<>ajF!fY9usfRb^(Wd{8u{_HX2TJ(AwesNujUSb2n#52j|7Laa+^`
z9ap-AUJ5)DlD(wKFf(M%AxDl&5zAICUg^!+ku3dMWVy{$g<Tc5)(X^Zh)_Lt-T7*>
zQ<0@we9Xh?PA!tlK5cu_7O_fkTKl6=yG4Aujy-2m_Gzchn6S0Oyl|$7`{!%hjW$LF
zACS&rDM-}*A9Zofo{5r@`H!yKwomM_+sye>;*j)Kt|=Pzuh)cTc#FAeTlJaEOr9ib
ztN%(UXV<-heAP3aP52=aRJ1Ym+Nv;B4WW(GG;#t2`y76r+`qBNc!F#mYo@h)w2s+P
z)|a`Sc}dw#0ztj&mYDX&yo`RrSFW&Hr$a~IeZIuhpyGlnPk+fh5(xXVNj2!6R&s99
znJ=o{D^J^;C|fFd_7B&BS%p=?@AwvlWx96E{^Hv^zbAc<)78>8wM+VWKcrpR-Mtv5
zM5WuRKlQWgzBq4zlE=ztC;bCg9+~g>xwp%HbE~}UIn}Zwp)dCB^@)yjomu30V(V!p
zov9V8FP4=RdUh9GDs5f;|LFs-)T5hyUEH*{Psmj9SX{m~_<wBs(_=!L0zEbE?lcKs
zDPYS#r7->MZkMeJzUjLxe@0wAaN$B*c-joUgR7@@R!&V4zrD^%_Ran450{xun9{Ii
z4oA3oy5ZT1Sjjlmdu8_@{48~NZG4dN_ln)pWf%6ozU%MyP;_#Ah?WPR)n&ErWv5o9
zDCfKooYiqO@=&Sjwz_1QhY`L%p6<)rZWHfmTh^Q1rct%gaDIYg*=@n7-`koNsy}&M
zoBbwf=cj_BTP@$2uYdCF-X)z2vXcV6#fv@^So_|RHGVCzDfa_cz@E)YpMB4GtKR*1
zPXDzm^O2<s7#MTnD?a8LFPO0WEVuspIp_XY$B8rAJoqX;gXy2$hHsDV@0~B$axgmO
zP2H!(9*&|aW*a&Woz7`I_47titHeYZmg7%P{n^9sEz;`58Y_K-_1ca=mr9oZTUt)r
zUhsYq<5;lZOsxXX+Zk(PFZ90^+4Y0VU-I^ajYqhG6y6vta5~~`+$q7q*2eo(fX}vP
z%1tlstrNmKC5<lfL@u0~r{nIhvgKB<PfX{PJE0TvuH2k^H*#KY@Qf#O<98mJ|I+;x
z$Jygsjydj`b(w*aGtpfx*8S1~&nknsr;fa{OwKBtjPsU>_I@FEQZf3~*C_U~dtO%#
zy%3d_do=NE?}RmXrPhC(_4=sP>z}vQh|bpHU9?g4Hm7d%`u`bx5k2l(6pvJfNPF|h
z^Im)KNpb$mgxk!!JmY(qOpbcSrgW}m=-IXNdR<QZ_p{f{KDz8_m0u_`E%wgsMGtPL
z=P(F4_BXe3E!z=av_d|y_2Tjw6Kxe^N_xB;U1lg}Cbr8w%>J|JKkpI2Z!RJk2mNmx
z^L!|#tQX9h6;rzKcvwtK^2JM&W-eTP^?`3gBJU+dg`j|SZzCqK-rUI`yXMP<09Eg@
zH{COruwDv{<gpArD6xd+Dc@DyC819Ru0{Hk1@iFw@<ePsH*>9vIInN#8qWp2$Lbp9
z3mux7JwZNh>zU&&f_DQKGJKqSZ=*`_m&h3f3%wZ>7`RT|%?MJjJ%0A>lXd^Uq(lcy
zGu7mLQs{fFM677<q~|}|ZTZ5_1j@;sT<DuK&n7L}Z>!v!jd5%bqmIf>55DB_@PdqW
zP+#uVI}t9|ou~HIH{4aPy8f|HTtIEUSA(oi#B_~AM@%Np4xT&x>%^qEEsN3`#J;np
z>6{Y(XmWASN-<xKcFUJazjpR4_2}RKH#~(a;=9N-!=?r9qDOx4oX|fN>F|-wRBPgi
zHE!7%j=3Ujr(_kvSg!hi^<tL2IqB$?&RTEx1B<QiEjwvCeeaQHR(g?IzN)r$ah?@N
zSq&fUtT}o<v~}WD7w&J%q%G&WA4q#OWoFz7H@>u)&TiM%)!dxEb?KQ@mHmZ@nF@*?
zheSpGJI+Zv6DjES@XVLnVvWzeBNPw6JyUbeHRIphbORP&r%V4{9x+dxG`EApeAbDE
zS@Yi&OuAPR)U?dC$?Qeq2{(D)laDtlUapXOqxv*=)x4J{r5<uF`hP)e(Z?HJ8E3su
zC>HQaE3P_{e^PE8<7_3SbDf7Sy|GQ|y|Q9?D&KQUX{VL^PCn9(3sRZyrTMOT>RS;V
zv&*GwuJr7NMU$?vR{woe*WvbElX17;dGntqYcqrQ+WK35d?fbc(G6e41FGzXauVJ(
z6KyNG4z1N(zTu^pgG76VqUk-Ax_j#0E)82$9GG6P%ya6P_et~Q7ncR|*cNVTTR16A
z<V5I6l~)V7R)|=*P5VE`J70CmqQ;{kua*@{1)Z%8ak`l{qapR(Mi;-()B;8IqAJ(y
zMMAD8y)Ps_h$#wPtvC5f#>_aE(92&Z^7zI&$4%u+oO(s2jo()#)rBoJfhTs#M6UzS
zeSaJ|uF{ZEvMi&mZL?Nl_to2RNAJcj^6}DIeOFQ2`Nn((gBjjQ$9^6-@^i}SRs$FJ
zD39GgGk5P>z5AH<o^z}B?0Wsv!R1LTpL|m4tG9{;k7N$$oqKsu@g-;Gk1vWZL?6Ar
zD0TSSte1vM3nj&a>YfJf%DVVLj@53u!yk^Vxk;}CG|TR`xK^dla9D03^kk#f+_-7d
z@oJiy0_^z#D`y>4`hNEM_u6>j)XX{mA0Dln8~@8Q?Mu(fnQHDUFUbF$>E*a0^KYx?
zW|d1ZB`WjYw9Hz`vg@9ULujs&>w)VpR@S|FdEMj2HMi~-M~O`hxjrtZMMRHAD=gUF
zB6q1`6~o@Q%dTC^{d#ZjlZ<8ea&v9{+(rEEw<Y&IUHJb`1ecWhd(GuVzn(nYxnP;!
z5w6-JA9gC2-BWm~H*d-pg&C4MZe4dhy%-C1E;^LRJF6&WifF|yJfA+}Zr&aquQtu?
z=UQtnE%&-Hf9@}LCWZt@7ss7$9^M+4+!y-Z?wIzWZB~4Yi}XHr`Bg3*ZCZt#%U{gQ
z(B2xPxHjXVf=>Qd?Jl95o=;M3>t6U+>NvOD>)!QZ!+&0-kZrSizb#XFw8FJ@sl$&;
z+>9%qt>T+FE%{B$@mpPww3IY=oo&6~<LI!IZA*f3OU~taHIsGsN$MGKB_BJc_CqP^
z?)@b)><zUmXZB9~TiS0d#G;;K;OOGum6B#Mi{JZAdrK>i%1)MR(^kLFdRXsqy7|dV
z&cg*vXEwCha@@G~p+&$&xsXkBi{q(@tcnM9&kB6Jcg<z_r)N(J<Q_LIe04VQfYJI-
zT`%6rKKJL<;r=$^=%w5VeahYyJ(iKn7fx_m*JgOdQ00)qnW>R&Qf8bkQmbbKF26MO
z>IDTaA&%qUPA=hg&lS>J=)Xtu72lE8nREYVYR{6)Y5EyntYdU;kKWn;A_jMtWOGaF
ze2lu}<-r&j(y42zBUq__aM_A)YHPa09ywGPc`N9=x$2@ddy`hQ>g%I%n%if+J!`b$
zR-#b+8|NEB_mW&Lm*hAG+&W$qEpRAxW{J2V|L4_x*H8M*TsSA`Yp8rdLr+HAI<b9=
z#O}S>b8GsKqca<={1<1qUGRE#W<|V(^y9*c4}W#vx=%d6i;Lsuj?e7=xxPL#K3J{w
zUgOo0+Tru%xc%OhKP)DR@wBGZJluYxIU<ZN|6S4hhepoIC0sYxF1z<h&{)qkebs%j
zOB)%V2PHjtHFJS`IqM_2$YP^Hf4)%lbNO0+d5Kq+M!Q4`aDM)iv?Sw?YR$p|?+JOc
z{~Ko4#cBWA(I&R?3HvWm@d=IW_QwzHd|bCC+h@AJ?Ru7mKQ4<?+JnWCzIGWoOMKBt
z-}>#@w69#6TG3n8lS>tg#D3h8kmI_&-c<j{IgJNQYo085?-abK<Ryn&i1dlbTdz-~
zZmPJhzVB7X)Ab*prvBfwW|Gm7e`Z}yQ+#%52|9MX`?hiRo=b~-?_JNg*kEq{OrOoi
zLY&1q@WaeUvddkL+i^0e7kuz?Ij(%t<*gCRoLw#UtWxtf88|b%=M3lYVOXE`&LFtG
z_eakOiHyb+8<y7x*SeU0Og_8z=+m`D`X61=wR}`kbq}+wEGlGQyS4Ovvi+?id!L+&
z-OrM6X2RNOg-icScCA}x_|8TnF4`?ZQd_2e;{-mgj1}3pmfpSpup~y+|Nqi;r%fJi
zW7>L#)AW+jCyBG`gX$Ah&9{0xY<2p=dwAE8@aaaH?L}J76*|E`)ZHyw`tNDE|LEGZ
zskOhTeRs?A#{9^`3*F8Z9g*@Y*{vTLac+|9^%H+r^tiwCy7E4{ZH-~{lM_an;-~p6
zuWg*Q<V_scw;Y@76Kk@A!Z){HdA&Q&SfPCL&xP|vY@Q_tozM5_Z!Got5~61FaHewo
zgUuZq>YlXwWjy>b=lp!86Z7`9v|RZn_^!CM<)=M=%bd$U?U@;31zFuhOk7^Coo}wU
z;C)=NeyQO4Bd+pQBHNF+?N*rZKlHoq+Djhm6^<=S{wuIca&^fe6VYd%OgZX)W-ACV
zHuExgoL(`#YRe3(Rr9O1yvf{sFjSoB=<JtMr>9F7HN5`5=ZT16%iMKl3)Q*v{JR+%
zb~$;N_I8+lJ#{DFGk^X3O`*+2;nhW*>8U+CUU;~?*WUP5%TqPx@tq~R+hqh#Z@Ia(
z=Y46f@wDFWJB(hNCQ5%xsNOy&MtI7zZyezqObm>5S9ag=;ZVByd&TG9tq+dx|GxW<
zj>leo5A)QEYvU~)OUewFefC~pELv9Ze%}#6rYn1#7ax}0arw|@+ua*{7(6b2GV^Y+
z;lKTDj&)GB&%{eS-vd=m7Km>z(w=#S|JP~d@&*4F%Xwz}wo$WMx$H*XBCS<Y>c>*c
zD?HO3gKkDlEMGTCT2$8O%1)Q*Yqsy=e(8{ZP}q0#-1}*(W?KnYS*?A2_mJVH>-yEN
zGHe`E9^d>uoh9ec(~cIY&adso6`MD&IOO%C)!(A+a&1R;#i37s?_IAAe?9B)zD=Fd
z&yVCSJ#a}z;@aEWS?AtgEPcUH93}R-H~inkT}%6fKgV2n>N`j2?S?n+TJ}}BikxUr
zyK&-3>zB_rj#x5S#MoQU@n&GS<I=si{_J%h%Y2)(hPCk)@&4ya`ggQxJ3G|8dU4-)
zp3Q9j3I9TcuJ0<iHeL5d##;O5^H28g$rC<uTw72ok~gojCBnbT<^TT0XQd5qwtKoU
zuXB5T>z>)KLcK4Xs~*@L%hH&(zx;Lr@6GzSl61q?{_DnE4jdDdTn~sfda7^)%y39_
zZs%8Yo5Eqp*w7`e%QitF)yuJ8*s-nVMxdf6qnfYNlSLDHJf<s6-ZSN=&`F=^fo5Je
zuPjSkTgLRpbWQr?6-O4jPE-r6_@WrR+-adziq_VZ{?X32W4~%Hh+8|4f3Fq~_owBD
z=C4&)RNFCa>LIgS(QT<m-56HyELFY~x=Z}giq1MwCNYH@iYwdNM3v-p96lZ~RFt>B
zBT^Xr<Ye7a6%!4HL?1@Z2@Dw;QkqTeOIRncF&%jH>e?D%g(o)(HXrBQ=6%mj^1tEI
z!&4Qrou6DOsCad8zNLLiMwDXl<Fm?(*e)5UId_~By%u6|_e;q|Q(tTGT&b;Pj1Og_
z>LnOYx;3Xnt;{VAZMe0S_wDWJE9C24{&D?ZeSSaV<*FAtOtD-B2R81J%F@{^rRB}Q
z8N11qfm`&@28S(CD_z&JvsMV`?QxR{RO92W?l53fD_-Ghq;{LdZCX@{;G$^`vjQ6=
zQ+p=58~s`JQSpeI;?ea=u}@a7ILY?Oql;xtw4di9omE20tV?bvdG|NX4qDWA;Lh45
z9bv!Do;JStWs_%6R^rm$pRYEmYAjxqb;_?}*&{E1o9{YW=a(w|+U)PpUbQM<1s`ij
z;7qQi@BeH3+9$MHc74qvlOyvDdJP;Ie_qzN4~r_itS9_6YsHmCEY?LE4}|5Gx`suD
zhF&`wCp&3lv}pF3Wv5s=)?AO57P_?Hgn@U5)mJy=^@?xK8S|N@&UMzeVOUbi%XnRH
z(p$61W*IAP-`;gMfs5~$Xv3Ymr$igLN-xKFEadR;Zip<G71E7+q0$w!r{jt<ujzzE
zv1WU##FPB)m7l#msb?F{Z2c8~EJQcl?bMw2Lw&v8JONE!$4?U++dut|(3#*fXQk+o
zCssXEf|rXJEs;u06ZDX1oKopNUDj*sT|H5w6AEgapDa7OCjEYMzsvvdi&HAgP84}B
zkzLf(Z4jDu!bjh>IqLuG33qP4JUsuIOv&qwmqo7RbRKIG30hHCl(l-@DW<?D0@YRl
zi^`l>4V32HDdGtE^De79bz1DI(xXwub~=j=OJ9<_7A+h-Cn-@{saZtVG&m#UyhuRD
zmc^e#gSN12I2vX)IqP-McFnN(y|>$WlscRB&OUjR)oQUbw&9V^rx;7d9nN~Yn<g5s
z-_t(Bb(^B*^5PY{(|nUPD%xj>%T>JEaQ#m#;~j4wyXZI5DwI0D>RtHr`<%M`->-@-
z8a0!DzqSb4_5RwG+D?-#6Otz=|2q{VARVN%=g3o4jV_KP4)uxRM^CtM9h}*x_G!`l
zzbD*z-f*uCOI<R3Yqm%f*Q{q9(-@NfYxd`a871b-TBbVRF;_Lj`MNvH0_#m@3zDQS
z9_X`6;kW$yQZvMG%8hf!)D`b_c7#M;Sy*+#SygS7pIy!(Zz)lAl}))RlekVV(GZ!a
zBc|$5_E0*`%XK<Sn@f6XPUspb^_|DhcJ-!*#5^f+6;ZT37GC;?BgsIWLASvm`^s#N
z7aEUtHY{a}x{}Pxuz|IKL50zK`$b+J(MKMPciLheZF(|8=FsM~XJ1<i-4It_-@K>U
z@ktz0^+|=+cAeKA2~$_sNNGL_p8xl&e%+hJ3w;<o>^IyIO5Qca^Pkoh?~N@vb-A-T
z7;-%~wQp!LC~V<4Cne#VaqLW^$)sg2Qjd3ZW(56Lxs&riY>|UNt4CtSVsCzh?PX5d
zs)-6(Pc9wFa-Y4@bcwl<4i{&>rSta;Mmrg=B)jcu8Vyz~&Q>)c^EMosx?T55{n0t|
zkG%``{%P?<moX+S=G~(ShHiYCw?sP6>0MqWb=50;r;w)0n-i1Ly;$6*pY}<!GuqPF
zw4`K%<Sd~Q<Jt2rrcGiJ2!Czh>A%qKNEz40<poThUQY~K6dS(qT&!WA>-0-oDf01>
zsx@7<ZC@@{2|QfoylBHr-y2MN4^ChB`6Jx!qb7%Arth*JcW&7+X9})U<P^Bo^}@U3
zU8qy^5qS^0FN(V&Pc33GdBd>x$r+WqIuX5F6<BsxF6R98BK_ijo*p(yWeMM^;6B!m
z3*<zbCTrQMFc`0$e#vOlM3KDA*0)B%Z)`42EM0gmYuVPfKD(};UuvPH!oixfpuyTj
z*=eI+a7vO;Z@0z+t`fB@=SQJm*r$2&ehUh}(-reGN$K|<)%<qR1_^5|;anxZqTD%G
zAM+#!P2ABlOIVZdS;8#2ng1?U?o0OF{Nbp6kQ}e=%En$LMj3|g3~iN3Q&iYxE=hMy
zWQ#jq>AA;i<(19{!AuLf84jw<S)Z=UQnF@AUqo`{Lbi)V3(EKAMt4uz*mE;wnlPt#
zyUZ&AHbrA4)y<x#E?kLYHQP4JcHcAatgVsLBQ9+AE$U!7CAsveD*IgHko5nqA`4h_
zOI^IJeP!cQPModON{Mx!dm>!DCQ08_!FzFo|1Ad5ZH>Q5)y{~14zXy9X-aa{E_|&W
zr|NoL*t6wWp`?t5$nWVUdS)tZoSLhhq>Mkgy$jt}CAl%GYq777<T90;G9kOG=Df13
zwBB$qOdx5qV57~XXs4wruaBinDh<^w*ZOGWIz@fy+F3yi&pN*r7x;#D%wRwBWtCRo
z0+vbK-+kOROgTUIdvdtQYSVSqr5v+go|5117wySi^&&?3)f*1R%W4My?mK%=Gm=(L
zy`%N$V}GvFllBP<H*tp~9}Us)Yq#N4R?IZwozK&tVV=vprd1^<?5v*l;R_0k;RQTL
z|Ihi58?{bvt%bMlp|Tk3pJ6*r$*%KwVsqq)-^UK`Ke<Qm{t+;>=Gw(R_e9D5+}M=&
zmW~UArpB85iruGkORdd2dC}byfiaHn?sR$w7FKOz<v(;qVY_1ylY{MK-}~jJ$9IO+
z-Mt$B;$itKMs5ZH7M%w<xlbkvO-xadZc5+_U_7|h^~Vv3I}<oGB$Wgio=*C_d;N+e
zE3aK{V#RmfCHU~`tg6XBC&pvk^&^5qS3=5rxzIfCqwXmmPtLn>r)uU~Zqp@46*jW=
zy#8!>^swsmR@a<f$M;LWITfoduK#YLdHAxZ)!I`wvg%H2SV~*7CAM--FmjLhe{K<r
z%{({N%N}jNg#OofExEqgy-P$QPM-7ShDXQB|Hb+2ThV?&UAtg)QqR{c`+ZuscW1eB
z$DZPy=um&f%~9%%L_yv^IpNiRUc~#HxwQN9kJmPC(KEg*Y1J!PzA}t!`%NZKAtf^p
z_XCp_|M+v_-nM1SbeOKBBwvwgx}w&^<dUQsb#_G@Z~2+LawpHon%sR!POsQBZ)`cf
z`oX($Hnyow>Y93ICaWX}^DI`?IV#4te%hT=M=o@+6@I8Sal7@Z>sG~ukN3FFZ_4L*
zn)OsoX_pqiLtIbLp59$f{Vo%n?A;C?%ivR4m$Q7shx=<j-05o(d3OG5m(IN-0r!*+
zDV*q>6QHfsb3Ah1`fqG&GJ{_HpKIZ*;`3;0h(C9))`NNPe=@AMw@7KXT6eZddp%QN
zCx@&5j1GT`bC31b?O5P+RwhuOzx%dC{w}pAT^oFFb8&F|RQr>b^oMQj#x&NiC3$<_
zycc=TFIu~7t5lQ1vIkr{+pDKIFih6rC^##Ye29VdfSCE&gNs&`i!9nQr#<RHpG0x5
zWfi-%9|woi+W#j6l=eMkJbTpq+`6PkPv0)ex^Yce$&KUsmBmx<9ktpLczDrb8Rt%x
zwdxF88+cPxT#_{HY92bN9BIf|^!e7)<S?HZUjkL0rKu_8uXA9zrID|2{?+<Dhn%Jb
z%&=<tY-4y_>BSw7^%w4SJdaA@Y4PL!)c<u8i@NgsW$6VhVtc#h^znIJ(Nt-fxTWUe
Zk9j?rQxdkat<%seP}|nS&BVlD4FF+lJ%a!M

literal 0
HcmV?d00001

diff --git a/tensorflow/lite/g3doc/performance/images/iosdebug.png b/tensorflow/lite/g3doc/performance/images/iosdebug.png
new file mode 100644
index 0000000000000000000000000000000000000000..8cebbb84688b8129c149108ee4a47736a23dddff
GIT binary patch
literal 17762
zcmeAS@N?(olHy`uVBq!ia0y~yU@T`~VDRN&V_;y24O3}kVBnF@baoENOipHC@aUX6
zojoG>q*%lAX(}ozTO3%HwA^!f8Jd;B$g+gX%t^>YLs5lA(0bE~15p7R6I{bxbFvvz
zHYVzKC%ot9);P5*V#C5uYMcvJ->-gO{rvZ{d-mz)=2$+zvvaaF!;0etM;_&bFgM5q
z?o%-{nP91C+3H!vBH+LvvVhyiB4(w9f|{C;gXFIN!T0VqTnkut>vZkkJOATdCm)|)
z#ORQuu>9kaJ2EHO84ujuk>qK|&?2|OeD*Z`HlLP2rd&=QflWK-7*Bbl_}oO*T`T8N
za^!?y#(;%I7L#}wt~@C;<lZ59p+AZ>g<a5Jqv;OM&FR_?J}A_;*=VV&i*MRxn(}(f
z`#&jv4yWJR;LoR?JY(uk4Hs1p4bk-CW&h_t{c(TNWL@c5hk49aIS7eqpO1{&zUB^x
zWw7FW;iayw&exi1d`|@5IK9(bf!kYR%l`vYHzk~9U}l(d>M@I}gIPn4)~UWOnJd4S
zaXa6bkofjuMWUT`oTb=P&!S}Zzl&RIG7Ihq6svUYd}5ZmX-!Lm<9unBnr{=%FH>@?
znPlX*Wv#;u3D!v7Pqvw1F-I1h{j9{Oopr)Zse9^ip81w+msAxudP$$LakrWx{!CS1
z(_%JDqj&-NWis8q4~s-LzH~bG#9Q5k@94?L?5x_K4|V-q)N=60Mz6xUgf|QRh832b
zSzvqZ%z3`4uYDD_dvQN%JNE6_;iJpWh_gTXGovCkXGfWux{aG8a~`)6BR319+JuAC
z*oqw01h#!M`Enrs)I5O$j6xF_e;lX_Fg8*0yr|rC-6DYHU(*rxOAiYhyW;FL8h+^?
zj(U1kZi#jkw}|p42k|E|0*sp;=Fd!;l2NhE<pWDri~ZSitsB!0tG)m2c|1h^l3;VE
z;}L@-j*D#9kJlFF+X%(|O)NfSTX25ij(s<UcX7OJac_Cu!guKH%0o#f`6n75w79^S
z%p-Y5@9pn51>P^Nit-<+T~Pl*I{m?~19Ih!#;gzO3i85N{Iv_aS#a!rmgS4?Hx6zm
zl~4Rjc`dITusO6?_M6iBE8n*U9GhwuqkFC3e1GMJeXHg9?<v=9IU<(CzG3qX&NnvK
zdQ2JA%6PJm=q0*8vDsrbPvhFL3ppFRDmE>Uf9U_YknQ=DaNkz}&o8|FP`QEs$!G4*
zPxZOA!);=f&Imj)W`2-&Hhq)por{ZS`0@1>h);e0_te^_yv`e!EBA2UFR47v_(7-n
zR8PJY-<@W@az=;x@4o5Zr1X9-Wsu2yd`gAYOIC0HiT{CD9?o9G-t*vIDz9-7Q{DmT
zAI%8@e2W}yJlKL0=CT&d^*Uf;Abf^-=>i!YR_%jP8w7J0tq<gFkT+>aD`1+#DDEh|
zgYVs;t(Q3C8sa|ieNpCca?0S4z3AK{$bE#b(8)$1UPYqQC23-cis(rPmkyqpooC!n
zP4r69-6Xlvwa%SSS$)E159=c7%8nA{^%MOy+JXXeuLx{)Ile$HgF&kC*y70-c})0q
zoxCp`eUY_=?QZL}#p)NjUzBcPu5J95$Pj!~Vg_ege`SJ>j+FPoONN|g(%OetZIH_0
zzs55EOXwQr*9Yw$xy<P^F09@WclZO#obKljH-F@+(cQ<q{<#0c@Q>{k`v0anC`$;u
z;fP}~OJq`PHs}rTTA`GqETc51?Y*D+L)nQQDFSOd-hH_1t(GQ~E}-4<y0g0Tc<0+A
z$B)<vbuMw=r2KQjB2SqhyOmS6v~B4w=_zTw;`wXAqE$(29xV}CG-;7ikY?D*pqnd;
zLY{V=3Z7cGiS1_78`nOy&nGUQ^ga=N()Y>fCw@itu9K5oKQ25HY^3>FW3^_r#&XT?
z8u=lip}wKEp~qKqg+5=kF6{2|D!;D_U%9UmyE^UEBELnaR&=ktz4G|V?JM&$OlGN`
zoj=X#^m5&0S1<WplDlBH;^I}e)f=x?WEEdob@kQNbu0FUsIQJ*!M?&jP(NsWko|(X
z$i42(YZ6x;y!0UB%(WHQEY=hTFZQ3j?&S)}wKEq-F5DUXdCxOr-AU7yhHVh@U+t3J
zea&<2@sgfAxAcoUzx@2N_RHljez}XbO*M<Q4&EJ8y}fL8+4{a|ecgVG{1*CcJtsJq
zkL}oolns(MdsKV=_DJ{mEAzOs^gQT~FiAS!p{;!Hb>Ee**Pp(g_VV4;z3YB^_OBFQ
zGkxiHpY@LK_t~w>Uu_>(KkFac-^;&jmAATA^*`y;k=|MVM3(u=folra650-mHG8ww
zwz;(}Y)d+9+NRE1-6P@l&uylgc2A#Fsr2RE+TPY<+CBY}_UW$-m}lHPbM;K{nK)tQ
z$p*qRPo_-{n_L&HeVOHQ(&fF&zMs`dlS_M<rhO*rOx4-mv&UziH(77IKiP5HhYbP|
zYc^SAf4q0;l+nqe#?<Dg#@eyk+d?OX&b?}zJ!|Xy(s`x5UrnVK^|t#?J6ko|Ec(i|
zt6>Szxz#uMwia&|zO{CHw!EvvQ%f(CRcTjzt5?-F)Hc@6{d@Rg&&7$0yWP!u>MW1d
z_?()NntZPB=c@;=CcNEn_2Ih3(~@^2Z%m&2I9okheO>U@%ckkGiaS2bJa>B@u2-g4
zGo4p&d(6vqk7LAlY}-*+SX@4N$J@f!W#;AnrSZ?}E<4{=KDPMK<2mXFjV~1UrEh-j
zu5Z4+X8qr|`s&-&>6P!l)R|q&(y4!QZDW6}PpyUB>{_dunfq4mn_Xl5$LEhlooQYF
zpR<1||K0vGpX~-q3fmj53dt7&8k{yFKju9>I_s#FTnlR}+f>%AY}u^pEbDpe1d^mC
z2?mK@l71y+rTnErLQzL4C-R2xHpSVe*QI`M+ueSi%U;y6x1&2k`lH4rvml|HCVaBu
zc6mQ{R0oO$E?Uu+A#k=YPOjGen3kIUv;L1qJ{mq=eZ2gr)v<Fuw$kqXv!%8hzl*I&
z_%rFpZq93*n_8y1d_CgXxwZ5C<fchlldaTxW*TH4I-2q~C9x=(OH<V_HQMX`kq4dH
z>ScaKb3?3N9!pt$vUT!#|BLe{**-lH@^$5$mFz*BD?L|Q2iq4fvNWBu&!p_ut}StT
z-MhPfH$6Weeem_O-ygJp`d_ZSq<wjQ>W>ti^Do4HaIRvpb(&VN_Csh|#Is{(zDHh+
zyuF!i)8tLxpRIVZ<e7|EX6GiC<wd<k+{L$#mOTn8be|V~@6?_+&z6+gA`|_lnLW!s
z-2cYgDEp@VrgqkStXEm(m#O+$2ds$H(Ar~ix9r!Vm;T*OU3Ie?&p$YqF*DPs_}-cJ
z%}i;z^Lo$t&Aw}NJjd_koO3%D7MGt{eCG75@97(z3sXPd{^|L7wil20vuQT}RKHB$
z6|^<T^W|CAd)#JvN5r|rx9JtdZArU$-6-y6Yf<~BRiDm_-s$4rZ}o52uj`FRUv@`N
z-=TYR!;~E%TfQ7K+MjBzWvbn+C9f~PeNP>0vFpvQ?b^%N*~C8$+FB@@IyEQs_^T6F
zd$Z46*NOdHxi;bL*H;H#?S0L2O*iU$oo)W#sl5NBzvY<RiP-n?vf9IC(X*Y+uV0V2
z7Lu!&r+MSjR@1`QH}Bijt-iZ#|Js_{)88(7D{GxpK4(?w`n7j6HxzryMyaoUea|#A
z{Pw#AcYCYfe9yUe`bP9V{x_4~R{yJh_1^rO`<r~r2Zage9lM)f^IhbhdR6)B<=55H
zEWEP4Cwt$vALid~QfV&xEBTB3?b*JwR`<V<i7c8@wB^Z*0~;45*B(E5yxK2iPQ@Lg
ztxw;b<WyfjSE_#1*ODVIi_@LX&9RX#_;$l`e(~Aod(X-5dr-LW;iB6|e|fK)&iD7_
zujdQHwqN^o`rUNt+cvjn-8#Nae($65r$??XSe?FZ<NDY=a;1C!x*h6`Hdnv<X2;Rp
zU*D8{dmCQP_3q34xz+sz>)-4<_IDLK8-Lcn)X(ep%I3+s%dy$+`J(ae;@#tSj_c3!
zm|ycw=jEc(P+QZwsNZK^Mi!TQl;8f`dEPp{BmT?xDgT!KyuNGtHGTenZ~uQ^dXIDO
z-1RE!+2X?YT&&*wckSuh)Ar5S$Ns+P-q-#5zZZN@Jnr7#pKh;OTXoC+t?|pxuk_90
zXYF&V*=s(**g)Z#heek^BafMht!%^f+l&iT59ha=F=z-+O*d{;IQ4l+(}DV=r$2oa
z^B8izvnwR(F--g*oywdeHeuxlz5guz-`Q7e{#+m}lDTX1<V)hB^CCFp+1Tz$Fgau~
zJ?KtJSWy4>z=7#&4Q||KkU1YbkMpULeRH0a^73uxray51X`XSfQ0oO(&~63>28Cpg
zAYTTCDm4a%h86~fUknTk4KElNN(~qoUL`OvSj}Ky5HFasE6|34fq^Z_+ueoXKL{?^
zyL>VO0|RG)M`SSrgP1A^GkON8d|+T;U@!6Xb!C6VBf_Gfb~e}~nSsIlzNd?0NX4zU
zbIWIl9KF_l|J_N;Tt6L2R~8nQ;zKeAWDE<Bc2u3Uj4@W?TC^fY)KTEfq9g|g)}yOu
zba?yCE>AC0e{<yBy7&E&MiqZc_OAHy+PMFWUwd%q-TmwAN>}x%v<SR-BboB3;}+|L
zo`x;EdoNzR_;dXUw<Uk>Jvi~d{(aV+yW3v>;rDD4xZl@$EL1_dY`dwImDT>13kTjb
zD9bxsIvV?2Tu^Gsq6cRjUlrL%tl52ANmh1ltiq~&jk8=Aq}E;jw`<4yLJpO9_a*b3
z55Bo~_t=XUFa9tsDm$5fFI>D_R#tYdeM`LYoS(;e>s~%u$J`*$lw#y*HP<cl>fPMF
zzH{+HoF1!$?q9p@p(S)_{e`@C&kGmzFB=|jIeDQhX?^!@J|_;Pi5#m|C2hWGAocaM
zll$}S3Z4s866%(E1qUf?UiyFE)|~vi%cl#>NS&H&#o6~Z>zKy&-UF}8H}aiK5eg0c
zShe@WraimQUcC6zJK2bnXH%P_P;}M1rZu~^ulg1CpC#pzRLvHr#m)MM<5-`WZFJs0
zxzuB6P{i7>*c+uKw-^*1t|}~U+Vpa<XG#d?1OsW6OHvxM?U{TXyxo#qo3EvnZK*sT
zdv?i+wX4<V1~M4TSo7rBv!u;`F0C{Xt1+5;)R*m?<==`hotTD#sh2i;e>98MTq(A1
zw*CA|wd?_p%HI`CNwGOJYwBuF^-DjBxi#~{n~wgt&8jV#opSEXOp{f0eRd19<@f%z
z+v{)994E_td)KYH(7SVLk9pqc$+c%#tie^byLS5i)iXtUdlik=?yI<&a7_L~vg!L>
zzE?xI@BdF{Jo@{~(i`u8$X}b@ckZ3~TF(@diKRz9gq#{~G8`{4yPzAM_R6VstMBb^
zZ!XVrQMp=Xwq@sz@b&-tE}z@oH6`sx(iG3-roZG%`M%$?os+abSG;<q&K&#pm#=l_
zRDBZMW0)&cBD1A=M|x84o?l-M&ndb7L-vU6j(tt-?{`o3-yh$3{h@-4XYn7`>BYA{
zcz?L*vwx+I@$#O>^Pby1nWA{zXhE@QzJE3I<9qFYA3nMlv%T$Pe(llB=kpvFo<DNv
zXK_XT+}2>Wxj))Fe%H@l60`rG`nsJj<CmCRerER8@PWm;oxha2YOM1wm*^bXEj<6}
zir4Nh9<AH#uGp}JNz>J7+NS%yMm%pa7Swk(W<(Xd+vgVd&oMD}a>C?p+ocW5a*J;K
z`S*Lz*`v0TvlLfW{J1_@`DLVB!p}dFcPg((q|Gm3a_)QM=3A=#Z%>Sl8uLq+DZc{d
zG##>G<J~*2Iq1Pg$rBn}lP1;LUYsH#eSPg>%Y~0Nui`MuxZNG7cI=1X_m3|prODMk
zOZ`=*^<ekR^~Jwql{K_(cE3Jn&9X3c#l0Q#ekF*6E6guj+BzepGeg5eY4HR<myDdg
z$?-96iR=0!R&@36|1)n>ZU2*|&3((xyia&yy!hnH8<w{PbOW!gi*<K13XYlNuQXB7
z-rT|Fj*FDCWgS09@3CK2DXZDnANHutwcX^nNb!bGTxNAqtlBE`yaje52OpkYx@V2V
z;w2lu{I2Y&{l)6_?bW$J_Adz^F1)tXQ2lUY(_!O-yRx)azc#!XGiCbr!+E9?yV^5*
zR17mDvtCOV{`{O79&|yi*?4Bj9G6?CK3Km`d7otedV1a2$BW~h%suzJ;OkBH$(~17
z-ih$^_EO!yq9tZNw?IaccQmJ|b8e{WK@aYE4t7UbEl*5p2{68LHAFb4FR7eeLsRdB
z8xKcsfU~UWSI0Ge`bS+`BhIo3G%Y(8wd!?Kgp8b*ch|-lCFPUO&1YBKq&IWRtf?8z
zI)ypuGY{XAp0O(+HFEDU)!Lqz8@Eps{|{Wz!=d7-v^M3gW$saVqXk(el^NIf3Cmis
zY*7Cd{A5nY#SDe0%}u-Y<WH8K?@75|Ru}g7xpSVxcTxR)A6T;Y)pWmHZDdn&omZhz
z#k1j*w7$(rWq*IB)xC3M>^D9WSe_#uFS07;=ox)6<1hOP=TvyeGx*tj=MuTo(IDD#
zXt#cU>XG8f230ag4rw{B+td15U-roH|8<j=FRSg))^A@Hz%hHjZ1OyL_pir;EB`M(
z?>V#XiQVef_I&w6Er$Oey^x;w=t7y9+r3KW%XaaN_79FFnNQqMlJ?5G|I<_M^8pLD
zKA&gZrm$S8r|N;(o!{}NOpiY)emm27MTm+>clT{Ko8HW`K5}xlf|IYiZ&5sXA^P3Z
zro~pD{+uXW6T5vz_4_@npJ#{{tgo3~9<EaP;hy!1A5~W7_rGc!oZ@07xBPym_@@)G
z*Ms*zNY>j|Yf{!OtSPlDZe!tbXaAymhI?->@%2q(PknnTGyKu=H3yEb`JNuIdE=xI
ztuw3r?N9BF*I`@uQ1`_N-tG5K-mB~M{oi<KdH%jbKRn-0oaQ)VPfX3z-Z^#~*p2Vc
zJn(t)z2h$mg!-3Q=DxTZQ}Rl1&7W_<a`nGdqg13n78tkl?CQwYx9u(EwtM=~@bOCC
z<$|jothlWIu#4OOuk*c(l9D=Y&DI<Z6{hX1yI4||=?2Z*8s&WT!O!<0dT)~#?2S)L
z4LGcD`Prj;&tl~~Cv}>wfAI15`v+~W+l${FogNf_qp|Mp=|E%6MZCTmS=-E~d#Yw~
zxCWSbs4JPYAI^C-#e0)a!CQYD+i#(Ma%Jl8{p)96?tgW*ZPyLHe4U3Lwu;4y%vEK#
zliE)nd%)dZ^lIbZ74d(S{rjpqll6AYyuN(TIbCs?v)+s6J$YVQx@7U$mP=-NRSz_m
z^~iVIh8?}Vd78n8J&JQ`4hhQC{tE4>dcs!pM(FIqs!Ls`{5RiEJN10^j^ZcAJ~Iva
z{@xLqDxDSBsvz4b(5=$AX1DU4D?Ta{-Bw68ZRP!9@l8j~f8L3Wwp||DY<l}&3NG`V
zcgA%2_lJl3?Olqe7QU4|{PM<1{r(=Avj_g(67Dy>DAPHqi$&7>mFAk*{=`#T%qzcL
zjQ;uaFy|iIYQ<TL0>WjVUGdI;y475*I3``&_+I_&6loo9b;XIE$0N372y%=6JHnpt
z`?P)P*OJ%wrap~7|KRJ+)jMBpW0y2fV@bX8`PG&09xH;ft-VZgBTlu1-+S`m^SVbn
z3(p(<Xb4i<Ycg|H8;6sG<@Zg}lE%M0&)HVxr`6d_k1#s>e6#)Hqk-p``+FYDtB-3t
zt|7>M%_#E_i_YA>gf}~$@AxU-Ji|6u>HhjvnXgYrxIZ^nW?2&28Nzm{%f*1N|9t4o
zz=ab}q*Q;Ep3SPfFJ#76FXhEfT9f}~<XRkEa<=*%Q{%yu#mUpxm1va+7fAm2@%Uzq
zK{B5+pSIhiZB_F;E{V;Zwq#9)_w0RMSJuY+?b%@ce119C(w~0kc0XWBtGAK%y&bnv
zg=P0Q+3RcL{k#v}n^XSoq{uT%%b#~XSEm2snYMO4bHaXR{$~gN?n<8i=*x@S%^^z7
z7hfKITDGu~>B-NSB~6YFPLo8Y=|>#0tL1xiMETc4lS8ckcXXVW`>XOhVbYRo+xym@
z(%8iN?!zz3r&|t|<c0HOl)T@mpTFm~W^(<%rd3^BBGKk4%@ZBs_|m%P9y-M2v*}7z
zTxUiyuXLDs@yXuj7dI~H=wnqc{!sUR!-w6eU#umHtiC;5vY1cB{`#>4&c8oC`(zRo
zz2m(~XT_R6PyOEGUfiOq%?gfG9=0q^D7UNnW;;u%aNlg}$*E}^M@5}Vziv^PnlstU
zs5f-=%HFn{5gQv;ZWT4TY9`Fzn(3EWa<5uHt>&Al@bo(Gx35e&iql!xc9p(9_eK9+
z!kHE7FR$&jc9*TyDbn(Ab>(HtWX|`W?PVmTG_`N@zI8L3cZ+WQx^&Ir&$DklDn9>e
zhE8hjJ5Sf7%_W?T3riQ6F6B^BxzzPz^6u1@1MhU6_HgSh3%9nPa7<?J6o0$fCm;M1
zU$nya$mN@giN1k?PH%#$RU|#KGuGPHE?l|dRv**5uTLgUUiI+x`)LIZ)AziQzbdkM
zdA*C;dTCcS$%%o1Q?EVN;EM86IBDe6u;=GOCB>6$+jtVX=bllRW+>@<<gw?aRSb(=
zROB2pdsPILe7&_=xFlK~f~WYh=Zo%s6BzaC`lkoI<^sAePrp1a)21BdoEwl+>A`s;
zZmO!Mo6`b~wf*J33({C79c5T?+eU}CE#BPv-Sw?2+b##6^8Nm&yC}P5cm8hS<yYJ8
zrnm_``2OqVm6gJLlU*uz7sdR1TR3B>Zhm6J!50f-E7iS<-ekPJeert%f8N%K$K~s0
zxcu8EQ~RK9M)KAxW{s;nWeV;{+5}x{eP92h@&B25=cdQ&K6@v9?pTsZU`&+Xa=)W1
zX1Pt5t9^NM<pN&)1lhUw+XW^BEnTyNIZpac&5FkL*SPP+FI$)q@MhYMcM8m(uW>HZ
zzii-fGj89iZFjle)F1Ge;-Yr-z^A3%D<ih`RL-wG)AL_Pb^ijM<#+w80?+sGKi^Pw
zRp?&bbJ<yoJtu5xQe)Y&N_y6^iT4*;zq=t6H^*RN=<4uKpS&N>+`mn(a<`*Xyl1+f
z#U^q6{yqKM)6b<JZYw=A<Cl&3y(($%>!tyBR<iGNVb%AaablL~>X{Y~Ri`bpd2@Lo
zLukhB7vIyYPux=4_DiY7LD=ZGT&2S5x;dwV{rwZZsb8+FxU!$~XpqMw#`e2szJGrH
z@om58nc08s;&mgMRJ?B*<$g_@*yo<PqQCe=VYB6%20513W)nQpU2i9COZGGS@Oe_I
zO31~kJl^m;McG}?KK?P*|M>aS)tYbTRG!uDp0e-v``*Be$lKc^ey<MO_QRBeaZ<{W
z*I$Ef?GzWESIG4Fe)ZgZt*jfnug7_;l-RVrqwe=tg=-u7RwZ!UOmN`f^}T!|_j+s1
zbQhn=FE6i`4^wqJI!kIwNK;$Fq<ifAPZreO?QD7Fk!lhkT=VIh%f6V9lN+1QCtcY1
zH)GX0w%PrAcRjl-&~<rhxcsfBkt-tt8!x#T|0*fD)s%BSc0$mmhrI`P94VjL#9e(_
z=4PPU@rj~k+g%_1)^dz$ow7?aYT4U28K39vKKbZCUf#olx7Qs_GFl|*Q*!&e!Mfka
zZ$1%Ww<vxgX=C+IFm3O{Wh*SEZ}Ll(;59PyZ)mXZR5@Mo*Yd%pNBS=h@|wpSOin&#
z_<L@r$kuH4`L(-~jC0esT|BpR`#taAUv=;PJvY!%Q}#M`<be18lM{m3A8hx&Q+&GA
z#-Pe(`TmdjK79<=?-%j^7J0tu&{OsB8`1NNgyQlUP9`&_bTR#$F=yTzeKzBNO-XCm
znD6pgbpFm?n|RAQfAgc|HoLyvnSZLwVwGCqiJ9rfsqfD6?EaAB<>mdM^p5oW^L;s6
zCd`j-xI2gUbcy(u^b0qSfBMsLyYlqIr5fiQ3oGS>CPZIpef&*7?Td1`phm^S27Uhh
z>sdEnK7Muk{gZdM+n+d^KA-W>5l-j-Co|himVdH8t*LS9;0>z{8KspR6~?)nbS7TR
zQ8qUhpR5?|%`Ru&@o#?p={J_~#fMeZ3l4f_&ouhwxJ)YL&Cb+6$v-k9eAH?xStESe
zY+Uc|c(jST;@#BiKYle|H<%^MWA(p?%iFr>&_{Now@(9W?9O@~NME|Zpy6W5LY4g!
z%#Zmep6kivxOqm?x$VNrJ+{wd%T)XAE&HU~&mT=NxRm`M;nUu8_amKT=Wa_4-qN!s
z)w+BAKLrC1gPi)A&s@qrU1HuGI?X(Xv-OO|&F$B=Ic2@}@ySwitxXnKdiAN&>@zHX
z7;~nrS+Vk^O6b+ADJ?#Ed3-B-y6(Sr^gQB`VqzmFez8Czam&NP2!UBnwYjDjTf7wX
z#hYgxdh;Q{^zDxrFZHO1{fV|8mo#0LU9e?~K=Er8HAR!Aucj@BbS-S^xIPz6Q1x(`
z<(VZEx1OET(L$!}TJiRl*?}s?a@%DWu56h-d&ADZE?VLrFD)s*`M%@v%b;1O&uAuB
zX4cIxkFTAkw0-8Y1Y6#NX)*g4!ny@-rW{Q*yCA~z@WaoITW-zsR9&%drK?9;QIX9O
zW39M;_3UtO<Au+}_B7h~D7o6KefV+5M1do-ny#3&8AQwx5fw`?Jed;2IV;KYkDb0-
zq>^aAz<G{6a_7Yw`*R$ACrC`%ZhPs#aWnm&w=Z01IR5xx&b^;o(nCEj%j6w=`*jP~
z9tE*XwWX3Ds_$mZ6#4qKRK=L7-=2N`ZN^z%CPK#^eMm5r6mU|#a>2o3+JT4PCJ28>
zoS<T~l7*MIH)E3E{Ohi6qB~;sQcPw!_nNpy8QhL}@{}z~JXFZ#u%eW6qE8t6PD{%O
zpQ=ZKbM5)J1|E!@9J)$5>2ts$1|7E@IrYqcPpinyUA$?9nf9C6@6MM<wS8}t?l)!<
zdemOn@jl+g>Q~^BPmTZct#8HL`OcE`_4SVD^TR(sJ!Y}Aj{9|%a!lFZw=s3+PX1Y$
znkN28j)!ey)LILfUn$D6bMJfSiulc55g;mf;?guvPp1^tds!_WK^y)qnBl?w&cemu
zO-GLU>+fB=_Z8g8KQJ@=T=V(;r*4_P&h*s$@u@2#KI+GFv2XDQa?FG~T{c9mmEftr
z8ua4DA4`Fh$5%r`owl%L<tWWNSGVM?%(7K%jGh-ZFM0ds!qLCxN#(y@>|38-KKr6s
z>c-ohZi_?0!~ZX;c5`=sK7FF9hnm7ngGpYSDt|KGD0SJy*m!+`;Twk5drxzx{|XZS
zZaZnMng|11bK{&lF|(X}O-xKu4qKI`&+R++U*1!|U)o}Ma6`o}@q10tdrc>?#LZ6F
zU)1?8_U(+@1%EumzuOAxGQaeAJ$+5r>_(e@*8P_|mSw(pwQlj}+b=K1IXe9ez3sKB
zC&z%P(Bp90)9}fE=KT&3|86S~xShjpKYxnw?RcAG_6Exr_w}7Sck|{Nk&9cbq<{RA
z{u(R(-IlXi^5lI%!KmJjoI7`~fyTOoCNUOSH68tWyX?BN;&%lTR-xVn%gVQ_f`-x9
zJlUcddba)lzi!u#*C54zFKm{10b=l8JR0ln?*2T#L+_oHm6cV!b4lTK(BPc(qVC;&
zeSPQd3)r3msaSsY=YJN(e}=;BrG?iSH%Ts^bW-JIiBxFrVV%8JR@DMR6GBw?t=&E`
zq%L-qH*ECm%B;-`axWZ>WmjaG=)n>iy76X?MDPB0n<l7m*3N3_J13^XIfdoS`jAX7
zu1za*qv~&G?Oyv{inB2?%yQY}2RGKG_2f=53lcAvope%V)v8Bdt0q2;H|JTrB>2ku
zSns&seG<OQ?#8mn&b8-Nn|weaTS8^y(&zKeUf&pbb^9-Mg~`_*EEbiLt&#kul`&g&
z`&Y(L*`SpoVXMD}b6Yno%3Aa5ld+VR*!#TCr6vDDI41h;n;7VL^UFmQqpY%(xtkr8
z)~0@5kp>%NbH1H=;@;w;ph31(cTU{enr&XMl`R#p#lC;FqvuwG|DVsV;5k~h#s2D{
z>&14-$p+t~`}SP56nPrHJIQRaU-Cwah_4r;kIZ)7|MSkmTVMCsmU3-=_SeHA<n;5;
z`;_PTZ7z7Xe9y;!I+IS#U#|O2B`Zw->1=ti#o_<+mDhTvY{{AQW{cXyEZJjL%QiP^
z|C+5)c6v!j=ay@~GVkh5y0y*!fAx*UNB35JEuT|$%9F?HtI0X5&pziY{+g`Y@zqmG
z>3?_YR*|&24}6!?>;88~r$WZbbfV(7AI^Unp*;Dx-I+(crGMo9?XUZsqIUS!Ok=_L
z`onEUyI0P%Z;vnLP?dbRczgWQ=Ucz7ZOu-+^x6NzsoTbxekvSc`ssJP-(L{%-SlVq
z&zes^%T+>e?D(O5Z_Y<a&e+U<>1%%cO1?MqqhaRWQ~&RCD6Dnxn&`1wNK^IpZ221(
z9sUb3Z(@1=E_=xZ)rn_Pw)?K05ae+A3*%KY#mQNU3xDjWzq0s?#^=RZQfaHD*IORm
z#aqhqiK{Tu_sDE#6PI8SQPGA;33qRm9tqNNRegJcdq*_87t6w3Qf6V-6SwWo43BHO
zo1`SeHR+&7-+gAM#T)nW-F$U$k;%trPo{HN|Gv07z1pkj?~0Wc(%06j3;9osj&Le;
z<P&1@UE+Cs@tSoJUNI9j-(QfMewT^EM7Q<Ysx>oH=9IoJON+j}smzs?cOS#4M0cT#
zb;*C%@PGLJqI><4yZUcbGj42X$`n@H;IB2ktdZIC%KD^_heCNQ9v1aUT$uc#Ek961
z)OUu(uO9E4`mAM|2hYruH+s38WA5&f<oDlPXYdI5@@@^e&braUX38{?WW%?M<}0zB
z?JW(tZkn`dw}z(Pp)|ur0Tb5OE7o!+ZMdPSsd+HZbfTXNhwEgIsI~+Rrv(cnwzIC%
zT799%%*)$*!_3~Bf2GT}D9ZK=@TgC4nN?uo8TtF0glpTYRbGoe@-*&Y;wt*PyJwL=
zR@<#rUQH9lY#M92xQjepXLGntej@rL*~r6${k3PJn*IZ2!Q4fwR!!?M2ndL2x}WPF
ze96SD&ESMk??HvIaz0l}y{uKMJ5)`(yu3eJ<sO+e+arr<VxMqM*P~C*gJ(Qfx1P6r
zn_BM4ZFkqNeD9;>R~mZN?5RlBtYgR2bjpkm{$dm95t%yW(Uw^!RFYbfJkPK>I@by;
zKI`l0srbySN8+X|$HdZ;ORlcTzpLOab35|chHq@k^g<%T))#upD~o?K`*f#XL`#{;
zbCQR4WVv7TM)z*(hKo-Y)E?I>$$NkQ&vuW$UhnpO<GLkZsaYOp*Kx7@|75P6*8X>@
zIoI2j^6UH0=bz;kchk<s<Wu3d_hDx)<=Z`Ywp(tgq2dXZM{nvMx!P1+53TufC}LW+
z?2hMsiYHcbESB5BoBzN2#ms**`|aj>cYjQmJ5l=ShOW5r$Ijc8smGF-%x1P6Jv>+W
zn8Aj{6HB~X4EgVPd24uVQFVNutoNXkJ@3%%SvBF4RJNoDC(G|@=6wG?BRYCgR_3uu
zGy0C_9lP+DrKq<3)7!)MgDfBO&JmAqKe^}g!O)3$4-apTPky+|D$Kuvt$Z14;AheM
z>+hWSbp3zt)fI(B^74~dHaRYP)XhILly$lMovGLDXCB{mOhs9~epbD0h4`soKYJwp
zGHrcm;v_0l`b#yY@ZriEJ=R@^y5#MiZd5-m?A0{OZ`1jO%ZpB%ieKK5BF5ia)i|w3
zQdM&4>xavWkALW1`L|GPyWK}FvE9ufCQUczx<{^?+AIA!qw9#pyQb9=bq6)qM19ay
zKHhWxSlennyIx-VvyY~^$FXddoW5MP{7$uP{9ePp@6Xm9$(-gn^KMM#=UARMSL=>@
z1sT`;dZyho|BGCvWXs9ZORwKKvu_>SJIRmv=l(pm<$06w=-+O+C8sw!Tl`*7Bv<{w
zaoY8*30I6NxONv`|GRzS;;`cPTen|Si43@`IVs}CYH!J@(&hr3rh(h;bJ`TW7p?ht
z%-iSR7wI>A!WSdvY`J(k)co$j!rI*_ZzpcLD6hTm_mtz?wO@bxx>jY@OoMNN_vUZU
zDz;p|>uKIE(Ucwkf8Fl1mR<6Q^Uy<$YS}WW@>ugJ$L6OO{+=AX;r3e<p}<9ntp~;9
zPcBWD>wc;KF4#je#aU5shwcSc1>@X_r`BARTE%w0etA)*JoDSXM%v9%Y%|TozwfUQ
z{b#4*sdj14r1tlFTVJo+-M4!EzFFdN)$-MG>232YJHD&;J@1^z|M{B#)|gNI#Wz08
z|6ciVvg)1hPb%(JXDh|>nD$89Po0)AAzfa+b>ZQOd#v92&Z~S9rnP_e!A-ndH7-hQ
z^bk6+%l?q>S(D>NGNs>sf`|Lck6pE?IBjqyN@nqL=6>71I+Nq8JNGfp^k077c`DD5
z#hxF&KRLQG|Kt0>@=ZnGMDES`Y8>>MVTF$OADid0alcBUZ_lfJXd0CJA!=^cEsvD9
z<}R!1m*pLe-XUwB*td4^1-YYcx{C~^U%kA1<Fn7^o!9>PEcwNL>Ef*1*SFq0nX*}b
zQT}tGbGzQKowI-J;%EC>=~tQV!!xJzl5Rw*CrsM*WU;6EA`|m=&3n%i+UC@##8%6l
zi7mhDv(WUUw*LN}rI!VJWWURp|4$a!yxjN6?@&|g8*8_ie0s&LuO!O<pwlpU=l8aS
zSM>fgO${w|5z=Cpt=IGKsq2bfXVsp%*?#dR!>&~+9A;`3AOAeG`S`(hhIF{_qB+Oi
z`kzh`{$-qgF8s=st45WFT4Zy6B$mp~S|pI=S9HiS-0;!H-I^tm6YoB~Q}LbI=IfQk
zCEr(_nfsU5dUt!X^mK!VQ_U^^>{(rrcV{z?-GV9M$NUck-D_X`d-}&)JDyAYK2vPc
zVdH1@NnuKa@Z0Z}9qjV8O~>EuXI)#!`FVOs(=vgajFk1xekorr`(Idp<wU=B)18v{
z$#;q`AI<rZ$hSJXX-TxwnzeZkpLFM^ym>K^=R|mA<)Lho^i7NO6eo7J&e$?}Q(pZ2
zmnBS#({C4@oBdf{WMNQvXtUyaasFPJ!?xG?5^t)n|9o=0c7U;D*0uG?_np}d)4oKs
z@!Xnk9#=W@>D2wTJkQTJ2&CLKl=RRRI%r{I7oO_Y>ZUaL?bZAov7R46iPyAxd$?Bm
zd|SNrp~z(0UC;FdPPl3RT>3U+{$rK7fgh`D8Dz@7EuHi4r;S|gkI1~*ETi`C8p)re
z?JwRCd3xG^`RB=!Qc9V#UaeZO_Rfhm)3Y%}kAIfvX)X%B7`f-x_Y4_3ac!4xQ?reO
z%8uN5@#Rs`?h7`IpZ55<9CPT;ORf3#Fl3)VQ&;KNH&<O<IT{bkC%m|zSn~b?-zg@U
zJKt@m)x<XzMbv$pup*>pcLDotr>#?uOk(PO`+e^-v)l-;;G;2*IwjU^GCi_YWvjQW
z#Nzo1Ep0hnfq}v9zwRe{%(oNp%-CTq?-DuxP;TGjZ$AodocU2>Y4JvkM^2uvTXMdB
z!{Ic=9kE~Z8&2jNdR~~cdF7L*&nEEcCRbt|S<Efo7ZGgxdiCQufBS=9281+dShT8F
z%e7s7shITg$fTw74N@;@tIWzikrL8!VM5D+=o`JxzrQXzGjU1n@uHaDUly*}&=Ap|
z=W)NreZ{t}r>C<gZrhv~n`>Oha_`Yz%WoabnkP0c*7V%9RX^q5bNNqemu*_MtMi;}
z**)9r6*0cKCTEUyvHGlCyI|$EnH3s77OI8cS`Yuo`(f%cJM+QDjf*wEMtVx!4xF;N
zr~lrprfHrRMEuh)Z(=OlKE0XGx@FO%vXZ=Tzvkm-<6=*HSSqvV1b3(iaVWm1eQ;2w
zxIli6n?%hmg=hBG?FBlWEi6JuW=TcF1vdYQe|7uShl^`#FPc4lb1Bf7qw((R=UXoO
zFRTq@eELuLrFF8{>#*>R`R<MT*O)i^OL;i1n=I@ncjU~>&ntP_ZZ1D_K7OjM-S;;!
z0+CFU=e#|c-*;7jsaZvlWr;{*i%XQ?uIOcF1(>F<?m68S>wCWJcAU?uRSUE-Qz8X=
zP4f7H&vHMxyx5&TM(xJx+i?}$Ru4_285$RDu`6R;A!$1MU%-+~&(;9O#R3^!o36gx
ztgFkox_;K#((OG@8YcTcy|8cYikMvj8?Tmi-P$!@Ex=f6l1~1|<=zV2vbmo{N)Fs)
zQ+%Q3;I;I^mQss9p3}~m1qhuuJjIW>DNmL0)$i|WdHQNXftqEjZj?!dX$m+g+_3r)
z{^$AwhK;dpo|_MPcxGD8wGv&NB^6usWoi2ICfV{SDod_yyHjH2do>_9P;I|Ko4)p?
zITkN8batlI{n?qPZM|fz&XjGN!cV2GaX*w|60~P7?_O*1NvnQvPB52GJiKV9&7U6?
zRdRJEzu(5XeO<M5x6FZQDh3j^TeRQ1c=pMzt+oCo^K8+HboI+iCaArh-0vdHaAvmM
z%;(E$PZ}Pd?_s@vs;d3JJ$D(+|N1vvFuQ)xY;vYZ>1Y2{vm94k;yZ32`M<jM{gyei
z{gnjPes2@V`et|W$+gefE3(SG7iMz!{#g|{w?3_<A>_5gDwc}eg3k-t_C@Je)X!{N
z<hU{V(XM8mGtufZ_W2xENUi=k`_&@d8;kE6F}w_E=~DHM*i>Q2Ev~mD=_*@DXyUUc
zANTB6JDc2p@7Nm8^B>=ZpT6<@fnCN*j#<k){?%3st%*4|>s8dA)}<x4b~h9*KK}Xe
zrm1Ht^O>H{tu^GnX7Z>g>Sx_Qn|*&bHdSuE|9my8cU1K!wm)gYQ?sXU*rw>TX63{!
zuM#hGHopj*cf4&?R>;KKPup9hDihyu*X&PG-Mq~I+3Ei}3%@t)HOvw6PE2^uT>U;H
z+E;ed$M5$wH!W{ja@qgiz04%v${*JA+qNdxZ+R5g@#wF?v~NrAzkEKi{)XX^xo<t5
zzwbU7zw2-zj~ln!<i=3%=|X*~jGVr=!*(5=w(e}2;1;V`ldEAHi@sLryp!2gdUm;@
z$nxy<c3Vzw4%)bgDOG**ub6*KR$IRt<sQn`iQ74Cy?yQ6?Dcy)<-@OExFWD*O>lmn
z{U-$vmkzZ_7pxwKWhBjJf30TC!sL6o;Mb=ct55soOxTo|6a8MEpWQQU4UbF(&+jeg
zm+srw7+ZR-==81$?<aL+8rA;#dU(~56%nz`)A~N%@_rof>3jN%Nah)z4n>4rZR0Ry
z+4$?U#j_uePnKqwv>aM%)~vQoCG6fV9)$}n4jdDB+3Nm$;=cFyiTAa2F-Nk4?;j41
z>Nub3Dl9c?*~43|k*3X!tGe<l^6nJp<aAB$Xeo>G^PJ>iVs2P^$z<BPsMB*K-)~s3
zXx6#vsR<5?R=#pyw)d-E_Vx7-=U!hPxVAEL`t!f?cYfB(+?=;tVQI=?>E^=D_gbvt
z-=<k^TI|!Oto<gh`^fiDC27sgmY@OU8;h&osdWqQwmj3ko&R`}(V~@Hu6z9+eY}1C
z*;f9vqH{~ljlTSG_RNVn`8s%+;je|ScRYK;b?mO?oOE8cJ*lxfU!E)8cw=u&?d#Y(
zwYO_ys?N>y*fjO<dAU<|Z@FGp9GfXpU&(z~&t~?dOF}iW-;X`t%*@6nSI8p&@3r7N
z`Orgur_Vnm$b9eVx7TeC&gvd|sjFiDfbFJ3;qq@wguccYe_@J>N|LF$_jFEK1;5AD
zhx!`Fl1w(7f4IExYozt9YhuOjvaYcOCOZtJdTz+3U;MdYa&-MorBjo19>;BMy70)r
zGM4{PaLm5sTI;<s6)zh1800Fge_kb76qH}&?NxYdlkumXG-=DqIvzI8)wkR4Pk&oj
z{=&53(e~#CRZk@3WD4b-_4dyaTs-gTv4_Qm5<Op}EN0mB%Z3JSd#QU$j9)y)a897|
z{Ab^!*Et_^nE&|P68GtROZx5m3ibCNG`)UCZNlT_wVsA*%{>Pns%$sNX5Q3euFk&u
zIA6ofS1Q{N^k1EP`sd+wDMwZm-o7#6ees2i@|>IU8uwP)GgqBYyVAV<(U)fl#m^=_
zE?mgfxb3c#-M&S6F@-l-ZN8t%uK9Z~TjDIApKK9p?`koF{M4R>=6@&u?Piq@yL0~c
z{My4O-Q^Ca+xI-(_IjP7y17GKd)bY(YCLY8ckbA{dG_XQZh0l=0~fZG=9zQu#K?#V
zJ2F~UirG~c+vK-6-T0##Q-1g7j^2dQpC772&+y4V=il9ReEywnYgp^Vv+Lr{p7Gzm
z;Qhw*3%T$A@w~9j-d$uR$h3x!dr>Oy_j_f+A1j)i{r7hy|1M2C9B!ZV@yVvLgXwp^
z+>}02XpnNFz)JL2*>;mZd1kJsPegrl{Pl{TshQ>Qx9ORFDeC&R2GOaN%?ZK#RTekg
zYFjv=x`Z#(bn>EohC6P~RcbkqQTpinjf^a=$J|enOeH;zDVc0ta`G&%!s^Ubt5<iV
zv;4bL!zB0oS-^>i;0d?d1l+nOA6nFOt-#Q+ctc}bg0Wsa!|J0QIa?lWix!A#OK@gh
zb<i#EfcvVYGjHv3J5$CV*Ul=cthc<?!=mM6($bCl1YIRGG_(%o87i4fEhrWan=Zck
zWXCMez|Fz8>t?pKvDwJ+r+q%R+}8b&V%?n2I(pN)_p-$_oqc%kWm#v%oYdlFk4tZ8
zyr@<wd)^m$+jynQ!ewb|_{{T`Pt8#2x%lcxp5a0*%~^{z_sEH7Y56$`?uhkMQS4}$
z?Xkq<k7b<?TW5xaOR`4K$yX}&_S3)KKbQJz&ia!Qo8D<HTv=iL<=LNu#}(9PujFqt
zoR$0J+0!ILNwe#<HcKCU{OJ(5VfS5+tXT!#T<eebItsZcC8}&@o_@Axl9xc=^3D5C
z=A<?^-<cz$qsHuI<+vmDvf0gB{##lydCZcxSQJ|8EJ+GX+y9+!^{hoFHnBJ@kj#~-
zyEK#MMU=yaeSHNwjfdYVnm<sV+#%4r`p2I*9-|{lXP1~V$Mtt-1W$P8A+!8=^Gy!6
z&9butmF~zKd-g5KP}0Lic}Z)_?YB-VBoCbxyi;es-o8*i-rREM-(ROajwbEcQ^&L~
z=SPl}pz^{AYwM4kJ3Y(mf{gE@kCg_JPjXLZnk%Om&s;V&rvB>NIo0Pms>G~Laxho^
z{2AfPHtkGh@9p{j+XVaXv^o1FZP<0<Ja5jr?fbQ0!|{ts6Rd9YZjyMo=`ZgW{mB(2
zeWw=PTC*WvE7ywm@b3e?$0aknrz(BVD7=1B2s*^xH}_v$@YY7x`}GI+{p~yS{QSWb
zqrA-_jmt7Mx}3D8CLKK_pAqqc_bxl%qAkVS|Hn#dUEiGa*KLchd8@#UzpZaBZ&g-1
zoVWI#Y`|qtEiv6inltYeai4a(RerZ!Y5(KzGQa*e9_#6R{A1<Kle~xZ5B*|)nYkoS
zwBPQF$#S0ka~bDu|9W|SltHs$zRr_A>wlh&$3NL?eV6w~4Ed{Np68nv)stZAq@sDF
zuE22#Yq^Y)M48L&HM)ztck^A$m=dAm78fVScDT<=qW^fOTe8JmzB6gYZi@?Nb0{dX
z%=BpsvGU!c>Q?*I&8p;WTe7-s`CNPcl`h<wzQ<l}`0J?Mm41K0qKjf{_|(%LZ*#86
zQghXrc<^b_maYf)n`de8%v%s3+V(cyscf5*O3LHg%o#oQ?7zyd9gSVy8xT5m*DkA&
z(7&wT&#aIC>=e)SL*FUe<f8oDu>B>rf9|dmjci(Fz%wsliPfaP`?MYJw}9pe3a>J9
z9q!wbl(#tCH}*~`gGaiNN$aeRql+|F-3rSVfBDAj3)`L8BgRJZ`uqat|CCRh#;`ti
zL;Oyc|9WLP*S1C5MT>v8WnGxMnS<l2xS62Ft7WFHUrs+--_7DCeWJR=;7;CAh3@#5
zuO>}Mk&HBJxUy~H%c|G!HC3ED<ma^)Jk`%j`*V2DdaM7&K4weii96ogxBIrx+`Aom
z@2a_GiXHj2GqSK;yy=TNx8lW2=|_$MmoB`l_-nYgeR211Ip^1F<F2s(d=P!`iBtLB
zKYs<h)s)5d-V{r`WK~~SQh5LH1RuVFsZEjm!GEHYK7O1YKgV^s?~TXTr``OV5xd6w
zd4AWuZB~)+nS$M^CEFZBLxWac3Ddl`&2?I;j@W6x^i8i`y!caj;b?5Sv97M}n#j#<
z879BFAH6Hu`QpOD!+O^jUA%bl=HC~O)-k74zj=Lat@E^0tCANRPKzpCera+rVM6Ka
zYZuo>Z<nq5kfr9DY~Zr2bgA#n6txGRlMVlWTDojW)Q^w1Sw+g<|F(~@dF|g45b@`8
z{{x-G`F^+4HUtIh*4&RWKU$r1_r)Kz63~prqP45EWUWdL<ZaKNEmff1B49Ds@7kJ3
zXREo#(l+OQuKxRLzC_*U{T`WWXWVA?th%`N`azz(Uz~PLd;0d%pP3OI<!XJ8SG<<{
zJ-c)Fq43S=lMKWQuN<^3mtCk~RP*bhb;qRSGJ)@)C7^3d7kBSIx87B}TvkE*mRL9I
z?6cd>7VW#O>?ve6o45Purpv7Jr+`*29=p2z?zwxacB{hlQ~q%47yWPcwfXqtu1M9I
zl%PT_^RP>k!cOYzC|-Zg<k=A5p`NrRes9x`qy<}aW}i@=+;Qp3B7qOTH@%+OKHDS9
zh;cr<XNJkOut4#i2PaP+PMc;Zky~;1+gomr;ACCXQpYoV{7Y4j&PeTTyA|c*r*GFU
ztSY;F-$u@SKRdI<3$|^Vl%Ue5KEb`v(XZ^OlVV2Rn#H@;2~NK2>RBjg*4>#=EG*gc
zW3Qfo)WTzIDXXP7UpmuMzFe^IjraDvw;Eg(ftL$ctzOk}%$Vg9m&S>&%8CK4ZPPph
zgLlpgu<(lu)ZA&O?!lyJlrh`LX{BV=>awKSedoeJi)3fMo&M>r=p>aylXt%^8P0k$
zW$O0jd#?Cb{^k?;H8-aI^y4$}dm0bcexC8;3&*{`MTV0ue0Wy=Wd8R12k$XTs!Be-
zn(j8C&owjC^FrzY&pV}`pPKx&=q{YU_VK&f>yM^S|B>@*|4+qNPtMI=m-6%ue?`uh
z3ZYHR-FLmL78@0Om>N@et?~AaEj0&y-1;Q_b5(d(fmZh=-iUPn_-+6EC(lC^`S07#
zk-t|xGkjmf@!IeA&b`|C{?nl&<sPSn4n2ALNIU<*mc>;YKef-a`F5qE$+K~q-^ITZ
z4&E{Q!(=zxtnq6U=k?f1wIHoS-izfA-u+x+wXv!4zK&96%xQ`F({H!&KAT(L|6p(B
z>Vlc6L2BmV_iDu5ziJ&&(5(4)QAMn7mXb{N!SE|nqu=kEd~ROO?3C*TAv1nnb2;W;
zRo<N&e&@i0#=T!In9t3<u%3OceM9c#xanp0rmstlyOy7ma`I1qW%9eMb-TVY&XfHt
zvGmI4S6g1Hgm$KQbDg~0+wNw2mU(;Jxm`cs8z<ij5>fS4k;*=6tL1U*kV)%e73JPN
zW|>Oo_Ocmeosn2)|DWYh_BBE8CEGW;b((*A)iQl!iCxpi^L>w(Ed(!!vkyIL`E-tP
zr_TRL>GSFu!|%;IDR_Ktn|A2Q)57;2Z1}t`>CTRH37Z8|JoSAOokK+@|NpBLR9HVx
z!ggA9o`l!kMf#C}cN;EUa`HC08CBz`BBOR^QNat&W{Xc3{#v}4v3bYOu4049PG!@J
zAIJWGbW&Y@;coxbcXRR?m^C;jN%8FetvIjl5#yYVTlajs=Dx(mZ1)tE(<b>br%tcm
zvvY3l<dwO8Z{N9H|2cW#_dBKkD$G<<riq5>#d!vsoNL-?ymEs_ix8LZ*3vC1qHE;z
zm(R7Fp_h6z;K{=O`{r+`y6IK*ebR)K%+33woSdemM9q?3vChBnny7kk*bMiX|1J0|
zUwmFWDJQ+t)O15lr7mcZ*?zmJDJCvjvgxtD9nAdF2QC=17kxgge<}Zhy~XP#%q#vp
z)6H6xXcks;{>m(_S+8b!n7H5EP$(<CN_bA;%atV>S`H#}&a9~AKm4$8gBQn=wMNDs
zlUh_%dic!w+GUTvRD7_lS?TgxXTi_*|0m>|_L^SxEqOH|wl(<hDMfc#-~V#46B-V>
zzdt{vb#<nv%q-h^Di;HG&bswBr`x=h^Qe$W<$t@IJtcMayLP<(!{~8)P41?=pZ*~s
zZI)|n<hS~P)`HzwY3wJwMLWA)d%rE;<Smsa6B$ebLK{_1Hn@3v?=fXM-k{}TRyZ@q
zMW|_-3loE@r=a(djc+q})tV<>+Eip!s^t`Hy6es5>GS4tGxVmoIU1chvRGHec+u6j
zPZsXkqkNRnI_h`X7u!3|6YqBlFom8~d*_z2xF^JyLF;1(kEj7h<FXYwQ$O;na7spO
z&(Q0)v70xqUH)uc_4J_4RVTypEt!0Sqb6@^oE5k^Z`0aWL6dGA+-BFlOXE{+eAZlB
z*}J-nFF#AUq`M||YtvM1tr@n-THo!hq~fptlew^0Iku|A(d+1hy{%vFu4mua_BDjv
z`GgzuNso^0YQBjVySAR0{ZsFAeVq69+TD*fL~^c+I&}U1|1({#_YGgqW?jNizMx^<
zU9K6Q4t+i|>$hpJ+P*V$SXY-x2$cQ$^x~T8{9g&&cdNcFjGfopB`TV+sj-xM|AVXB
zpBH}IYOWMoe|Cwv{lVy4pp{r5Q$Z`S%uJ7M3cXsv)LQcDLFz5-^&2amM(z9erF7l?
zZ)$aM@`n<QH~%=!`e#du>HS*U?)T#B6<42Po1-QiDjl|YpP}0tOBbW$rz;lsEvv4S
z`&!zYBA+fa$;0URWAhoG-zrV}Ytf%+{NK59d7i~uw-8_F%O;?~fbDa0MOSA}_!??g
zy3P6h&ese%VlJVf^@dStm(8NzZ#H$?%-n8zl-u8~>!gNa3iIzb<xJx7HGGzQ%J&%6
z3;ub&uE;&(>|=c5M)7Hhs<SI+^s;@rJ$0eRS(}RMx<?)tR?RQrJI?o_%U1TmjlXtD
ze@;{vRu+HURjkRi!!RnAXXhiUOOMqJem<S9p{Kq4xSj098)Zz>4$3|FlNfIFY31`B
zg+F3=+=TbC@u?kOX7u$hzvj*VE_!<(<W%ht+VkO|yG;3!yD^q;nC8tjn7ZD6PS?pd
z2^T7-KjeK<c<||UgRCbJ?{<8wQsMbx`HpAT)F;7pXE<yA+~s(;`z>2haq)+;7frXb
znR#Rk9x&L|SJ@=T={);Y+@Ij^{-({(3%M41EpNo^d{QxoVF$D86mjo8R?h^b4hw#=
z{`#ota^KM;pGT*44`dlV(0p}F^uNXY=W2P`vkx!3f6D6iqV%L&!5v6Tdk(z0_blD_
z@`^QUj-+jV__eBh=bnaDt4z$zCwr)9XlmYcd=RSYt8rz|&AmwmksqpTXY@w*El&%&
za{c<`GZ__sEPveLyAZNyqJi7y)@K16tjs%iR^HHwyy$gI`HFeVgtKKedeW;V6x{SZ
z^Y1~a*|9Y;1`Fh)mdrXXCwr3d`JF>gm-F2?SLA3Sqa?wVyv6F|S>A+c3oS(sr4&Cd
zi3l*(bUG^-BPSm=*=NJTc?CL+m)|N1d`QeVn{r<$Z;_(Nv5ajKXHV}wn!>;`ahbxx
zXBuVO#aG|#$l3B>%S^$jwq*evzS|ysuk<*2*o1AiVdR`0Z_-RocAU;smFjKEQIgG8
zn|9{G>61s_Zb+T%<9;YYMoveK*=a?KlvG)gbcg&g#eMaHFJHJ%JjMR*-D<yd^F{XW
zN*i+YZOrxi+kRCkP4xKIf4kjv+c#z(HQ}9=mDX=}O82A{eg0JUF@|k&>!C-N-trhc
z`1Xr&!|n1pE0VYF*35o>@7;Q-Fv+w%Ke;wPKcga?c_8xA+{OI2venb47F06ruxZTw
z_v-Y?yT{huxubiZxssLTQ<+WQ;m5}xetiF-e7ipTcK!6@#;4W(SpJ;7|F(>rm+U;*
zZN)pIPCwrucq8q<Zt3J)vBP>fmoMII(tB4eBeeU}lZ4Gtnwpx5a{~o-M?||#N<E|Y
zxW4r6pZp1yAC45i+;Kng3j6GdK4xC0y*^s4*?rqUcJAFiy?50jLW`F2T|3V>xq0r|
zzq-uV{?9t*{wFhxclrd42`XHBBByimN(=DTi@pALoI_zt(P!yt=jQe5y{i^tQ@*0_
zd}e3;^qW=xvNnAP-~655-Lvsx)4X(lcTgW<HUG-zt^)ZdxOT7Ws*mqUQ;A=GH@Ed%
zo?HL4i{Q09pX!$@Tn_ztC1n8v1A}UbYeY#(Vo9o1a#3nxNh*VpfuV)2p^>hUL5QJ|
zm9dePv4yUIg_VK937IV?P&DM`r(~v8;?}U!k*9=#fk6XgLuPWaRdRkoWl?5&MhSy6
zjHTdMP>_?V;F*`Kr=tL}S>K1%k%56h9Hu8FwK%ybv!En1KaasBv$!B9u~J-m>JA15
O1_n=8KbLh*2~7Z=1f?SY

literal 0
HcmV?d00001

diff --git a/tensorflow/lite/g3doc/performance/images/iosmetal.png b/tensorflow/lite/g3doc/performance/images/iosmetal.png
new file mode 100644
index 0000000000000000000000000000000000000000..5e2b8bde8c1dac18ff66920f4f2a3f369f81bb3a
GIT binary patch
literal 36053
zcmeAS@N?(olHy`uVBq!ia0y~yVEoL$z;Km=je&u|xmWTE0|VFEOlRkS%;aPS29M6E
z)7T?|BgKxtuYTDpwrk3T>U_o}Epk=)W==v1QzV3&x_CtkS&lb#acDI0#1`l@@MtM@
zwK{H0oUhQ_WyL3!v*Hc&G>gr@-@e=azsCIg^vR#k*?!+wz3)AP!@s$z0x5@z7*alN
zZk_lqZp)7r7pZrK4U8-uClof^P-0ZJv5`=iHT(a%@7o3KUCecsCI9{X|9|KwnROk!
z3?h96M(#(>IsLFO`gM~j(Trh*&5?E6ZK4g%nsR7!O=J;#H{;or$anLfJ&Bxl=7)=W
zmGU1x2Ai2jg_bcn%!})ucG%DL(S%Hn=+Blmc84y%-nIF1giK4hLe9LYTT5o`7BKVs
z=RN1j{-ZwAj-_l2T57X<y5z&!3MEIi?5F=LW&TG;&V01f^veWCKVdc(rOmIt>`c|+
zon&NhX_C`dIY;upq@OVRku!$TPv%-2mzm~2;mgf47amypJ^X37v~bhRi^V!IQkuQz
zMSr@THT3>bt6S*C<Gf36x@Pap*<!Nh$1V5VxL_%k>cU$4{o}M-t&co1&uKc@zm(hT
zQh!stU-RpEziz$>?<Rh=HD9L@H;cLb^<(X=pOcr^HRb4M);Ol^xH5nCOZ~0gMT?f4
zPg9=$_V?z8mp*%@v$mZ#{J^+RLE`6y2Hg$p*Bba9HOO<Ef41(#yn7AbgARHr>YM7$
zKIWj+;hp^H=Co%f(b>{g441!J{N8o{-prN89+Lm>hJ`XZY>;L8zW?@`ZHxhL+ol}c
z_TH>HZON4hziU@b*A;tq=GKMF@e)f;r8X)BFh`xJHDVL<XzM#5TA?J#-ee%aD|8^~
z09&R4H^;%E4N_~^Sr>4;VW@VHT*1=rU~9qC*Hm_a^;{G81Fjuxd5ymvnA{F>8}Qw0
z_T*?P5MW*8lr_ODg<+=SHUZh9_L_wW8bV$TlNN+@@OLh}dckc4hgRFE1zIZ@e@R+#
z&TZ9mY`zeiAtBX#&tZO|h)<hnqVElXZA~u?nE9F(KXfh7+QW0M_4C8r0?im+^TW9x
zB7bnza8Em+|A14(VPe234cRHIU45V2S{DihYHBEE>8=t8ZCd5Fb%9l2?FwlP&R43h
zgr&NdEy~Q`-lA>AAlsa_c>2QYi?%O1ec|g1vl9JZY*iAregE1PCYTtcNr>|Z3-@~-
z)j2Mb;AOCNhVYEfGooh%o~hOmTis)N$SRR}1LuvBH(GDP-qhu2Z<Ds}kUi-4puoa(
zj^sIB@wViLn+jQX)b2>TBYCIvj>bFXc`etE-G21<LuZ9u9rHex`yKYZ`}^v<=Bu0#
z<l)e3F<@%nFlocw0OuDfF{(X6YMq~4lov@G&YIyD<FBK-P2jw+yukfQ9xAbuj5V8%
zC_kO(<>~Gr?s<5Le+ug+&6~6LOwduwSH3=R_sQ~;`I;hKibp3;@;JFHXR%4Zxj?x<
zyHf$FI;k^^WM-BcJvGWTx}F+$QYeKtMRHSjYH_M}O8?2YskNH!6O5;Po|b%a?NsgQ
z)|%!j=8FDPw@>7se16h>t@|4Fp8sYt`5g>67~Z&e!ukS_hjR{bh`L@~TorutO79h^
zSCy|;D1}}0n7DGrx`-te5rqzgF`kYm<21uETZ499(p2p<^12x?bLE_sdt!gC_>}wf
zyw)r&9?^BD#HNZ(eXS)Q@_c30s-jTt(CE<om33i@SN>d;y5em}c6jxQzA*F9{($&(
zbq;^`{5oCP^*QGG#^<8vtgYV9V_TEBN^s>yug4SI_8)6mHf`DdWe&yjpIM%r9(5{e
zUewE|-CO;#dozQ#-rZVxYxmak8TQLHFG_6jiPBs<-E7_Bg~AJzudTc!d12-i(@VV<
z?=HB#>UTi@3is>Y7jLh<zSw--`nB?l_DAja{6CNJU_;)7v<=r6oPW@~aAM+|hiQk)
z4yU`Va`Ranvsi60`!SJZ{>Q1ut{yY@-Jn)7nP>7}wd+2K!R(h!E*D*1owi}-V#)J4
z`)@9=Ja*^iocy?4yA;Q52@y{sOd@%rwW8Ug^ETvcuSs2+KQ&|P<yGfj&EDEvx~#PK
zYppcvY=_yrW^>K*b9{4eZhN}T{H;b=UD@k5R&U~N&%RmwmjA~6!Ua|>7CB`v3V*zM
z@kFB}r%<N&Qn^>rssgD++LsQ$jD9IT*YI52xz}@c&#~WWu`6Y_OzF?dR}YyU-g@!t
z&6>qO@BB3V?0%=@ZbV*ao>2L$>Pf#H%RAfq*iSFM^?2j*dFsbEU;C_ixMkwGjI=$U
zHhmMSn)~6`gSU_N9@~AJ{psWP!7q#-tbQQ<IQrS|6aLrrudY9tf9L+e`l<Eh3~CA*
z3F!v}f+wF$Gc!2OJeTP@%Qv=(&5N2&HOaBhX5((Qc78D7z=T*uekWe1p9|k6D)dNh
z(|(h<Bk;$j5AjMH1Rn}M74%oiRSI|i<{G$onv1F1-u`c0_vC8rk6F0MTo%@r-fgf=
z@tgA;|Cqu#D)+SNeE&>k`N|UK<gTP=XtvR;Fsbo<lP$NKxRs7hL`C()i;G0N^>oAH
z=j{A(Md+sL8r9>I3_J@xdp+YL64zv1b1^dBsZ=@V?b&;$>kfW=^5$7j;oTia?=08M
z(>$tqKIm>x`O?0nl}n~B+54pO^QnsUmzghVU$#$aNxhQllX@(5TiV0TlQ#K%68$0h
z=ibsN?{FPikr|v^ER$08PJh#p7H8wkcAbzhafNcUU}nv)qsJckK6)tgea@#j{il<s
z%SS8jZ`!}8e(B3oy4v%f{d)Ru3)kMRy{po<o_qCmb^CRLYawe}uc@tn8@YVDUj5yS
zdG&FBs~-L8`MN)E!@Yn#5APn^(L49&te=%Xm1qAkHx)f0!ol5UwmWa#AI87T@7o)7
z%c8&K{tP>zxhEziHYDgvs&@73r}DXVI)4kZqEBsWeA{=M|ILAS58m(CdpSA1c4JLp
z-p3Q`CdB6M&fLvizW?^-l=Dl4eos2J_sHHydna9baH+5M*YC17zk{ZQm94(Bde2s;
z#Yx^r+yBn_d!4^RE@S1DH9D*Q)J!P<;5W5CG<s`d=}E;m3+FidXC-C-s$2B`O2F#4
z=#bxA`{LDC_OE%LTacBJaV6u<Jr?uM*~eyyo5fqdDc-U8=%S^oLU(R`!C|3f_vz2`
zCG+#;d97|0_tdR<y5r!-F5xS}t+@fWPi_17ZA+P_yVLOpCl5@vox5(Xdfmo9OV@cX
zU;SswrIl}&cP{_z??3mXt?@<O7j>0pU*C9~pEU3H^GE0JzUh4-Yaw6q?-A>M?&rTG
zYSZ<0#lG3~r0C_7)L&1(Y4?TS`X&3z|Ie!ruR9r!H*Q<)7jC`IGXAy99Q%r2yN-X|
zY+jQWv*YBR%BNv3)*ifVxcxnkuFRx65ARf#nSXxwy4v!)+WY6W2W&KIG`??mGV$2r
zSNDDDvZq>j`SJed`EK7=Ra4($`{_~n#rN@UpXUe6n__dd?$(Dh-y*NixPN2obMATl
z`vmsPsEPd0{zqPPeNVjX|I44x|NHpk<HhTTt_SCD+ZS3L{r%j(*z+zwjx7Ej`Ci>#
zq%P!l#Q)BJE8iF2dA@w|>HQD(O{p{edF!X{i`&gB*R0fJ^gb~wlQrN(LvRp7MKC*q
zDkH-yQ-%)iwBJu99XH$l<7xbVWV7wp9vQ}rbIb~hq%*fpoPT`cO@WzD1)_JTGhCd<
zux<0?zIv}IN$J+o-B<dysHp}A=TEc~OlLZ9ec{TLC%>&+xqhq1rtb_d>g;bX_xI*|
znaelpmcg6FRrjwxZLGg&&mb$rxnX<tp%)Ac3<}8}LB0$ORcZ_j4J`}|zZe)88eT9k
zlo~KFyh>nTu$sZZAYL$MSD+080|Q%<x4R3&e-K=-cll%n1_sUokH}&M1~F9-X7mhF
z`M|)yz+U3%>&pI!M~K0IlTYbQHUmQggQtsQNX4x;cS~o8hRPrK`2Nh>8+&iZUb&>G
znz*-%ku9OTe0xIDf+WQ<_6~)F1x#H_5)|+9?#}ssZ}0ByY0vi5-&ramAlh>EbLG6+
z_&vtu&*m5xKQGTp6H#n&n55!4X^EP*?G?@mDxQ;6UcR4rFa{(x%R=&us^=t?m7S9@
z1^x(JJnVStv=(RfF}v&BzqDprJ~dYJndcenV14D}ztb)20wbf^*4ve=nEYw}lkeF%
zT3<F>t#8bont15g{QGVT6whqu&bkqCvE=+=dvAVrO-;RJ3l=mqH8FL0o#tpOd$zyg
zu#sE;Oplr71uG`+l2*QQpnQJ)p)>QtmGZ-X<r~PqV)+w3H81u|vw!dNd+KsaeWT9A
zU7!8?sr#NEA9MQb-wN?8e)&)5^me_+eb#LkOFxv}sXLlEgRivdTlX>k&HE1L`#x@2
zY_(YC)ssK2uB~5tW=H$h&F#CnMBv(j>h=t_?_UH`@4ByRNx0c&#gxYFZdNYvrTu4W
zsq8cfYhB@g($o7k?OA%K=zKCuDR)@6SA==S-KQGsc6|!Em7*HBEmimOva?HWfBfsw
zwN3ugmS<~coL`j7$TsH=fANX){+pL2zsz{JBTjbKiTA$0tp7>xKXG`17*{}GWal#J
zj7BT1y^J9X1LFc)wbdGXwx}?fF8#BifK@WLy6~Cg@rT=H3%bp?w8%*-t7b>CkcYqb
zu|;z465Tdb{ENAkcdw`XTaKQ^XO~MhW<0t+uCA(GOrfS*RhFa)%BL<~v}D31Fa3Ef
zDl%cBlW%%%Iql<GX*#c^A}38aaex1*D0|LMPL@|$AKFcvcuOmqtzJq>YADLH9(`)D
zDCwk!|G_m{9JW50iHxH6*la#}E?KU!K#ObNtgJt?+Maz6eEN0rgh_ryd)j}!Dq2?e
zkuxq_{noaVYNq#`4@CPq1{$`V^ow${x_HjHn(@DKU3^F3X04S*(L3MuxJ-)G(!N&t
z^U3n8J9*scWrYvFW^#lo3S3yh?>OhUQ;+k}`wKok-W2sphObCy?Gg9eJO4a!u_+8=
z*s{Sfv$Uhl;D}4KM~Bfuj>hZD7i(X*ut4yF=%mLwJ8~!bIqZ*Lyv5B~gUfTHSj@J=
zyNeBEvKA<&Jb$M6{j#2N+o4|Vir;U`E8lcx&8z>Gc<YO8C%bX=1*MbB9ILdwG&R>A
zNhzK*abn};JVut%t5ZcprY03vdwj6q;1=FEPa|uUmVpR&=~dAmrmbJj_}HvFVJXUI
z77!88aDKhu{5f7;PC8vo9L=jf1$CX)+PFLUgopo8p9!qS%Z%zWW!DznzM!T)^YnWe
zu5T4S4F)D&-CQeHd=wO&>=6|uBo=z`g3}Z+(V&|@W_x&h9#|st?z(WMe~N*9h)944
zYpChQ(w9DMcIHyE54S#Be_VQ7@#DMB8$@*zzWq7cqAM+ty6y9Y+VIUOzbfwi_~CuV
z&-x$RK}qqLvrE(G_k1;2SJhtowC4ZkAV1spQC!pN&mWo^ZLv6C`jwX1=M!t4HF&i(
z%*?Of{mU`4@v4pGC*41*xMyGXvshDnX{LGMIm5!9iWjBp<h~i?UQWKbDTVEPPvnsg
z%iA|4T>8Doypqpv`E09r*^_6BHolDWI<57wb<LfM_sVnXO7!m6TFu&iyvETZRA8Ef
z-u{Pv=WG^oZ&urPygm5YqrR?1=fc}{&AQv=s#_k(?@Rl4UYlcv`RAe-9`kuur(ch+
zn74MS9nadvmNfUzCz#DAKiMezO6lI}MTgivy?W4f@Z%QmtqcV*+ape$Zqm6GaGrB*
zj?c{4W2ds1MI8+?9(=j6<JA(Du5DK~Zc*?#vBdaZ)}u%B^7j4S*riu1{8A|W!_zxl
z&+<7F4Vw*SO4Q^qxi6djGpkI~`!sJ(TE++IB};bh`1Yj2=bwOlO}Jy>-o}O8Nsnb?
zYKv?>_8qof5Ur!Psc6>3$&&7Vo5hwK+r9Jgz4XWpTJx1BpQ?O%_NI-|o4lIY&p$o+
zod1|R`XW<liN!mecYA*Z{b|*fw5wr>)?(f<*G$K&W6it2JKo>o{_)|c`;6@i`Ig^2
z5FP8BHa)oF%RlK(!^)o@raqrxlP_vETXN0DnCjnq=UiEA`dZBA`OWZq%N;jf4hTMN
zm>zd7Q+DwTi@)q<;Vc#Jc7|7!+?eb0zldpx%;~Mqza|}9!e5#F<~hgexd{P@7gDbm
z-CXeB@bhQGJ?42@-!!C6?OIQ-+ud&L|Eg$3cg!ztzW4DL>s#cwg5$#6_ZP2wx^(*d
z<^IOp_s`C9JwKWK;`<qSU+g}{$jKD(x_xE4SNT8q7vG*q)7|!-31zPS{F43s=B((W
z3(p<06I~N|rY-2;1MYK6(%W^OouBJJ<Ag<3Xr<`d-+T{#KA->eL|U2MuMf_PZhd>)
zIz8^uwr;k7-Idy3{|UU1+NY&IOHT3)FFSKc*!4t-CWlyFUW=VS4<4_0a_;nv9WUHB
z?V5k?w7&iMiI!&1{(oRTTDkh?gTtOO)en33{QI?<=hfVkGs=$#MLk!4v3jXl%JWyd
z<(~aimp}II?y)nK*_O|?9F(c|3I8>(MDF{Y0!7u;`_f--W6P8KB;jxSPe69Id$2r@
z+^a`-m%oydP2V#0&5hVsZ&!J}HnCQm-)66}V*T|Gl3L<(W}I2eEVNMX#FkV$tL>f3
z-20~X?F@`5>S15*8E}1puo=s2^Pe*|E1r0HT&?K+OwE#>z#p%cOJ)6O`o$*OefreZ
z<;x4M78;6(U){ViDml8&_O&+eIZMUP*=#Bol<u|I-w|o+W7*ATlfLbh$Luu{pFW?y
zxW!=V)wq(_#oOYRzi;?lS$D<b-QJfrVQ1$Z+_tG`S*=;$o?jYrvL9{w@6~HpIrFB5
zmMR>G4(&a9xv71ARb%`6U6Yzr?{C_+VUDSN&EfRT*Nd*s?f>v<h4_ofxzGNV+b>(D
zcwkPXab`i<=NpsHf4CqUF1%Jk<>|Lak1qdre|^rq@c-ZP$#WkZdcF4ObN=Ai4YSJc
z&p$DdS^xRe@VJMo4%Yto_*-66Q#b9JdHm%)=M&~teyd!!vw(p^{{LJ_!T)+jGvwJ?
z1dc4<{`iO8T@SBZxB1Fe3879m?gZ|*qZgCM-zg^^$|}uSIm__HmO2%6wc<mW%qKh7
zr9a;G$$4&T?S&tO=e~#h65X~Y*0<In`RDo1C$+O30uKnv9{#94-@~uUNGZTMF>-=2
z+kC&UEFs%|+t$Uu%6iU-X1qDMpzujlpo62Mz|M2e%HFw6=hMx+xJ>uvNzankS6ioS
zN!yY+&GkaOx92@KuUA<&x0MM#KJK67?P<z;^=Qa%DTa*OTdnU}ZCw`mx@^n)Dc&d6
zCl<OsF0w4#ZF^+*+a;;nRT`!4)qPd9`~6JZ>YL51o|YBPu?K!HGZDLxpY&|qW&M^#
z7uHUl{cUBK(Ph~$nVHKMg*h9pw#eb)o9)#T>l!%4V<-2)X`2)}SNc_LD-e)Rep)~G
zF3W4@kYA-Dt_y3oO`c=Heb!heWoO7CE0rVBoraP%hU+I;zGm8TqG*y(sYJQv+un{p
zX`-q{a!pMRs>>_2*L*$NBW*k@e}BA-!J5UUKW^+=J0tEd+eKCL+0))u7y0d9T)2X%
zkzw;><=-;yd0|>_cguLb4q;EUJ+W)5^ogwd&2_cAC(fC6c)@12p!g~NnZ6C%-rjnA
zHvXu$a{QC&XH8{FW8{+-#T;?jz0q=xirTToGeR63Lqkq|HW%o~JNMwSzoM#T>t+3Y
z6Q9fNZJzQY;?}l|gIzl(SAKk2zT%qgmv<Lk%dVRR8qJVbYY7OHVxD?(%U>nIof$dH
zS8zF9zWDV-y3=*Tl7Op+zZ;l^h#1sgG!mLLSLa`)T&kh~@6R*wTco6#4}VjvcQsrr
z)l{`#p23$p=zV^M4#$O_Dk-N#2A;Dw_U4FOl-S|2L}a`E?R{NLX`9>U%n{p?bF(S0
z_E+KI-szL2ULVQWWAOgYjFlhvyQUa(NjPNPwzK*6W%8V=GlC&`k>`Z@`5!+wb#33(
z@m*8zm``o&1l`bwbLC%IWpgv%Sgd2QmW$&;dFQ)B3CC0R+wR-x7#I*}&~hVi&%~26
zxw;m;3Cj-%2oN=BQP8}4a_e%ni`$nr-#W$czBc&Wr9}qq_N}q$_Ia_}r<k&CO?j^3
z%H_c*_{s3w+UY-Po_Psf+MIFINP6?pmPJ}Ke6;?xRVmeaS($$Qnvt_UT~lGo+v<Z?
zek2Eovc_I7pR!-h?7$Kwk=A?abB|8$zw3Yf+Do1JrZW={$1OM^eJ60^kJwkfTu#?7
zY<cVMwxHFt<>tYe*Y6&_V7*Sw_Sl(l`Mu}3^ZgF2mw)wX+P|*D=3IYnn_hdTY_HF(
zSawfarR9d;if#X+89Z<17PNnw@!_TTlc&!=9$hYcg4^}7mG#Gf=V|M&>HeCR`%k0I
z&ZehbzHY|xcK-AahO#s4i`BN)?KP~QSi1O8@QK{nXO8FZJ$a_m{?o@7-)HVEP*w{)
znzK#tTexBQxBe%&vFFV%r$1QBzffcMbn%8n3G3vY0upVGNnv`P7r9Dzd2N@wTlm86
zTE5nyB`bE<*KdD2voKmm<VpUr1&i8L@7JF`KKc2@jqgv+DSsz9!};oz>p%EP1>Xlp
z?P?a2Ywax7j*JsnvPkVn59`%4cZC+YXN%`$FR4#{xXViHDeH|5OXvNR{<Gvr<c4UG
zoV2NzRv8?PvlZIos#0g_zcn!O&GX-;UqwP$ev}2u>h(`$Y2TGR_n2~nz(lUwvv^Bu
zHZPhNcJ}U^8#jABjaTGZ{84td2suBk_w~&Ub-J6E^*z71++FbUF7drqPu^Hu=h5{^
zjbx3~-BobvW36DF{yjE(wQ3Rfl2cEg=i8?pni5&_`)7pR?=rJlLN6x@9J;ago@Ps9
z<IJ+yTN`tauJ`#@CRaH-{PRh7;f}L8ZkukOF`wW2@V6-I9Fq?+Z1XNne)UMHrjFM=
z#ANHy)|^{&`ZLzNdDZ4vR2IF+>1U12!{BYta#J7eUcBS|#OWFuIw@~<zYl&;Ui|Lp
z^b4D7)s7x*ZP~ia;EUqlX>S58UCxBDcXFPLIIHI4XMR$%aE8yssI6yZeRFs;y1bln
zgjN|#F&<gA{o%3KdQZRds~6oapT8(#@s2OMW@k*6D_^mBvv^|I+S^XM=N-RjrOmop
z^1!J=^)HXFm!}0s$*O2-a;B#+t~z^o>Gk`5FW&M?{Qg(HB>m}yjn3RW3m<x&NqTTo
z{qwmc+ao{3ig13EGn&0T*=*&GMM)jzb5=BDZJID)M#p8bo6EGV+A5CvCb>ng+y6@G
zo`mf@`Q<&wlS&Og9Lg@dwPwenq)zd3$7ZedD)G_Dn!;>wad&wBvps*z*6*2qZt7p_
z;Jsgi+wRy(hYFThq#4v&r^vr@x~IPT^r@ffDj!bX(YiM~sOjhu0il976AQm&zjiR>
z2@sX(O87LVsK&ARh>E&;c@Z<0?54Ro^TH3S+g~k;T(4mhn6^f(HSO$Oo);2H&(2Rh
z6Lv7qy!cw7VZ|G+z&v5T>Ln(kp^B$c52Xcp^lS~<zNGK^zAuZJbfb5?oRw&M#dV9n
z*lO<SwkM*M=O0Q66I?a(TP%CQ&!^8Pzc30;DV#7xr1|RGhxN?AKL7eHyyNM#$Rp}U
zF6-FxPCj|+W3%4EC0Zx83)?B_na(YHYb6lC8xWEtEL>2nwE4Q|WKMr4rn1RiukZcc
z`DTr-PsM9XwUq*MC!FOikMqfy(*5O$h}vG`KFiB?jw{UA;;Ia9a_YUZ`l@o&baLSN
z{bz*z@4DyI*eKm{EAh(C+xd_yuCm@{``Ld?lTEvd^>+95Oi>9E%BlHtZnMvi3r$z$
zPP>NlDV<VF-)!#MRm1vf;lgaitSspdM=BinAFbto^wA|bNVJK!l%=GjuWQl5bKQ-t
zD>T>zR_+WBpAz``^W%bQ53dea+njt=!HVqc11ZY4-ro7Ra$)O=b!!hlEffCmv%uwe
z!sZzc{05(2cjm0#e);!_cM|uM+-&~Z)SllzJ)7;j6F;kluC}sH$fHL|7fqP`x6BTx
zm34Ghw6fw_*uHY@$46g|BpA>BaJ`bV^Mp~0UudP}Ry+H38T_0dH_i}Y-GAPD#mC=V
zO}lnkiA?6*e!0!yP)K-a;*2*(j=BhcxEd4~$*ZQ_>FH^aGtD{n#BWuB3!1C>gs-sX
zZJ&80gF8*MXTiDX`Qi^xeJF@ZXa6_hfN<)u)Lxe4M2<bYR!*TE+gAT*W)oZ8b#+O?
zizco4ZUMqO=B6!@)#_^a)zFs|R2U`NB5>t5@0x8V%9$AFzN(US4ecnLy)l~ada=3F
zb*GTfs|#LElw+BE)Xk@def7;NJ8V4G>UXGHaf$Yx`cx#<WHs+b#08T(h8NfT%rq;W
zE`H(ri?|~<c<riQO71ziw<769Da-4u03E(NeA0(5e5}geuv=+AyRyuZHG5?0Jq`=c
z_E>ZLs>P&~fZ$Na-tBU$mAjT4snAhlZ8;Df-D{sAFkQfV!$P?;+fHn2bu4wYG%e+|
zSZu3eHt%<v*rH8)Wa1Tkrl?Q*5y9X7_NmJ)b;&bJ4|g2okMB*rQa%4chrjX@U3IO+
zw$AN$oHwhSU+8yd|0IsrHAahh!lO)<mzT-Ddh$p6>H0b67yIvTk+&8*^y8{V-o9du
z<0YA8D<-S`?LLv9CmW{3e^hhPmQ8F|A1N4XPnOvJI6(b%{945Yi)URqyl?8A6i!Fq
zP|a4?)E7>Qn!1yp{E)cGG>uO~5;n56`6kXGE?k3P#UBjnTNi_d$~=Fk#qF*7y3nr~
z%&Tt};$V5=oDAZu<eco^Fd00u=H~}vuoZhwQt`}slwvR)B162unRYA2Nn7SFb9N3`
z@Ul8C&#dwCvIH~PS5v$usmR8q7)<Bm4lQkZKke0;bK)j%p5C?hliqSV$YNu6=MIgh
z0qT*HR9<dKlrqb?vB7z6M?HJv_C-^!t`3_VAJP+`>#{RegI)1i#zThG)X)f7KA%_i
z!dJMi>pfcBdHLj-+}_)HmK%OveE2%PG)>oh1-FAo^8>wlM&3V)jrs~VqS%}qX7h-a
z>NmSMU0<QK{y>0kgzm(&J#uT(ANt!nhA8DtKDOTeA4g!Pi}f0I75*v<kJMD%u-Sj*
zt=0a$QnH)>!ScwGNg|Jb-M%f{=2z*-C~cm%XwNFK)rUI|r&%uey>h+Yv*WJWo<4rZ
zvhM8<+{ZlE-}9IGq~vGpCvLUgFFN}3dC}Fc=7q=4nj5`X^SPq@kIbQCjrYSZwz0;P
z{k*$If0b><&x?Pz));>CotC4Mcjq~mn(wcsO;P_D?}s}FY_N(~U<hSQoo>%CdC_G%
z?+F6B1}zymd*A<^%JejPdf{u+@Q=^bFW-#&qZlt5-%~et>(S5*-m4Sq#Y1H_*i6Zb
zJyYu}ms~vad&BheWl;;J<=g%4cp805>1kN>sV|wAZ)|wW@WeUe%*;=M+wW8;_uKwd
z;pg4iZvDQx=jzW;mRG0vZq%*ny7uwe@A<)@;&vuKr!1-0ym_{c;mY*<v>SW*gC4&*
zZJnH_d{g(wlf4srDxTEcDw%cU;@is^D^@4HzLwjval+Ke&N)+rTl`!jHKo$Bv`-fv
zQ<*q%nn!4+BCBiajXN8kJV{C1{wD7EB`4k~Q$q!=)x`&MuuGk|y=$sf{fs4BcrHzP
zB;CAd!Ky7Ya$T23%)OZMJmuk*NCytB(_NZ6dPh^rCr_T-dpd7v#p^7)im#%5JWD1_
zu3&oGZ#`Fh)9(qhg_t)S|05CAvFTEgj+yY%qe~NtFYBnYx0l{;zB6%S(T(8hYX&^G
z(hmPTK10&p+@o^s%r=E;*>LuQkA4<M?yfX0-JD*0aHjgCXB!!r6c4%F(-!%;{o;?l
zg#{ZL4>d8>F!r`A37I6EoVjA%vbgq^mWURyC3_aln(@Qsjo8##r?_|~Pu>w5)fuJX
z;F|hWuD{dZ5NO;i@bO{U$qrRfM$?X6+r58XThk#W6P+c?(glQ{X6W3{>gw|1;1t%(
zlsw{Ut$F_bffj!ICBX(0*R(9|@XS9wp<P}@Z}-o6pC<}*H~EF8vbH5HT(^vA|FQ}5
zCUzcAy|3HLJC(KP|AXWx*<w5M*b*t;)Y+AZM~Z8{o|SpG=Tk|W-6w^@NL`NKdqb+4
zHI!UZLuEf|X1d&7uwj9~;WrA##?0-mOLuKz+HBmWv0T)dJ2)`1Q(aNAv2oqY1`8EX
z3ec83!{5N`y)Qkqv-kS3`_9+PBpL1$-2Qvyi0^^a&=qH*&rYa*pX%x;n&xyQwdC>2
z<?Wa4?jBl^xc$@DLrND)^mdeS{M~(P$N$EMPeQLOexY`5&o91rdp|14OpA5CJ8yTP
z^H<S5pRP3b%=^Ucw_Nn7*m-7uo6`+;ZzKPHzJBNTgUeHXvnJLvyU(wZ?6)n@R`p(9
zbo{UKht{_(ueB|U9$dAlJASmr=vVGDo!ciqJ#T+_;Mf5>KR28Iruut+SX_JLyZ`k0
z%gXaq4u>6kD<0=J^=hZX!W}E7e9qrBzk2<Cj@R>FFbgSN(lY09+;Ta=!CYH<8*9Yg
zP-oM(zo+p2KK1k8^QCjD?v;w@%s&kon4E1ZHgl22i$*QGnxBzs=c8NCrm<z~nBOsF
z`M&LE&(DP|$K|D+-rm}I=j)N^7~?0BelI1C-#%~eR`b1k$EP!K`)}WS|7~&m47*H|
z{QloUlP~w}cPu!@8eV*Jr+RSshCcCmm2FRNuQEs$n0<5JLjF7MlY-7o?zcU;Yg5%3
zHwpi&>UZ)*%$xU1oZqCoH0RfY&^v0z;`8pz>wMq$XhpXCLjP@#%YW{8*Z13EVKSHe
zz8@T)&8|E&k*-*sed0|KSO2|wX3lqWoq3`xTx#b!PE%%UzwGPPE@$)UOfgG=z&h>p
zL)MEep8az7Tp4p<!gIR=uSx`0tUf9<*Qcy$?w_WhtUY#qJAb(4N!0Kp$JO_JNnZD4
zP4evry7$ja=C?Up`Q6@i*(xTku0@Y@@BZk29z5ya+7$EOi~$q(Pd&3uHsiKU>GAZw
z8z$@ad@b9x%`4^6@AVt^+&HyjyIs<k==BMDamTXX*SSS#TnPyio_c3y`LVvoryTEA
zygoWnrOc}L{Qf_a(%a)sP1zp*aC#~0@2P!HH`_g4tm!TJ>r1xJuP4dBww8T(bi!Oy
zQ}<!-bN5FX|Ll*|8Fhxw-_@_X-!HW^PI2o#1NHgkyvfJv&lvaLIdVvU*@m5`9!$#4
zdw3)-cSq%m;F#ivoDv7wPOZ)|>=Bq>_u-e^XW>gGr@BvP)Lv1r`;j5@J?7f{%=xw6
zk-iE0FX=N(&U&y#N}`QXLgu#P(L~c9zr^ilZE#_FW+wt0Uu3(y*;(!L^K<G)=B>70
zwQ5t^KNGvzd#9~_SJM*izSm)Xj`91wpAElV(a|znyL7>xNmsk4Z>YS>CZ-?fn(At}
z+I-5zHKD)$&6U0#_W2(B{7dN%rtUF$mAX#q#%zE4iutBLRek=K$Y`Gmk*j-EIj{bc
z;v%P|f3{RGH&*Fe%KrLvB>4QtFB_^2wZcBWK9<EczuwZe$k^ijtlc{vd|4d$Md02n
zE~j1lj@Rt@|L%Cmwbl<|;mIGjJb1V~?!m*|>y{*(+`i_N{%zg4ft-?a>|RKu`Bt>9
zUcZ0h@%d6GPw9rLsH-mI;?4JEZQRf&#uXU(@c9<*{D(Uq_9v{q;WBm7*$(meXP+2t
zXWIW}UsdB+W2gVGV2O6@i^MD~?GNpTcFVB0I>f%UpXcxSYx|^we;firTMU$ZPu=JG
z6|p^4`tSLjQ<QI;FR%O^7gKd^qexzK>-qJGn|4`N9&p&THM=izb?VzqXQzngr+wM|
z-sto7(w4(<T-~d`DGN4T?VfgdRd&3e?LzZRN!=YEHg&JOv!}Q8^|i_K<4?OzkM}v<
zV^TG@LT~Z$zQ-NUGE<&!I~ugT*E~-4=<>_YE(&lcH{7^UvFJ`&lJKNLOC{lZA|fUK
z?|lw_e!Zis_wkg=eyX;XGZ+8*%in&tZ(;XekNPF5_ta$9z0OzD^;=%K$??6J*uK~o
zi<eyMI{L|7_`ONpoX@sX>uzi+GyTdqH`V^XLtNxvIW0ymE~)JBJcZS@{J*}wbt{VE
zYd$B{vEG3Hq2KaKA)V+wv$jTiy{Ny&(6xPC>eo$6GdAAba?a=0dYjOk+YTotA6{_z
z*Tgw9&TuZD>uBuHoP9kvF}aR$ufJ9IyWH8A*TkB)*?(th?tkr35~TK$Gi+I%S)b?R
z{Y!T#b|{K0vfXGQD`vCt<Bo0qcY;GgzSurrtTBI6mVDREqZ4cUl+CrLev9o};=8JO
z!n(I{Qex`q@wMjq_WxO!`(Ak+ou+D2^*^@fUy|CcMT<1Nk`=^`EZg>Qh1}fCYkO4B
zMm@Q(&bfS3Q1FWz3w7O`CpZLjui5WCQTx=p=~9i3K3U)3@y+0ID$DQStk5+#Lbg2K
zykLEkzS_Ryr`D=!Xt4aXX0QA@;a){y&zVm_((9|Z5)7j=udU*pYgM)+`hD`w?-vul
z<jnrKyH-nMKW6}!TBs}c>yBfI(aI?an^w&@qvjyMXTHp_VclzqgI00XX%>s^<9*V)
zua&&GA+`DHogG~c9F3dW&DILMQM7;6m-kHedhXRP?Yb}Tep+&1+9QRB1v9w&Hy4G)
zYtQET`)AX&18JN8KAg+QE6RE%eoM~}74h!+dG3)Ojcqp1^kaCtHXXmV(=+g5X|21h
z55F`==uyep4zs7}{;0eaVkV|Fr|_49PU`EKMyYmZqqnIk9^1BD=1-k`SJMTaFUPog
zFYuPghHUOy#C6EA_id=Gv$?~IGygv&-}~{RcunBPC)=_-w;XM|QRtK4&$@AW&4N`L
ztAs2!w`^M6<NNA;wbrZE-VqxrR3&qF7e4*E{llRZ(=Atqdp|q#zw=ka#@l+WH@@0W
ziG8(!%gN=G!*!ECU;oLf_xzjPwQkj+KPQv_f2{Gkw0|nkGj@m5dhQo3gAH<C6x2m+
zYFj(|+Q(xjmhX7~aQn(@G9|ArOnNyvKK<Ew@5pbTdG7tl;nTfcb}ZLik%xKl$3r5b
z^Sx&GMT$(la$#S#|0dyFo!DIxZ@>3?IX$$xwl?zUs;SzM`)VwsxxF$kFP45S_UFKl
z^XVHh?*`qg{p&kvcKYRQzS&V*yq>)c%-@`HmMzWyUxW9#{gYZ>bFb7}za(|{VvD~|
zk~u52SAH>NNZ544aHj5>m@Pe0@k?gMa5SEjb3Oh|_uGW|emip2c#@}gZ%a7dvia{q
zEv~Cy)$C?HJep?IQFb8Q{`W;u_Eo=*FMqY`=>5%S<&XaD=Xw0FUH)0;Lw|!GGx+`(
zWP?WobtkrPCMF~-%bmV0C;FrI<o$E~?>)O(^XVc>lJO?bKK}*T)352o8B9$`vXZ@3
zreN!7@bvi6EiW(i+V&o5bN-rhZ^yi^udi)PJ1h06>s06YtY+&e$6Nf>C(oaMTztNt
zg1&_B?rUc=Z*Q`+uQ#7($sN8a?PNdC&d+@NW^C<Q^(pI3;mtWW_S<{rIaZeHav03a
zopDypY<-mP+rH#Y{WU4iCJ4PgWKhOtuXsS<QMvn)eOc1_Ha|J?d;az8X;Rvjcgvx!
zjukX0rpwhP__I%7;-=e$9jzOSZ}&c3HN$5{>oNV$&o*i5_e<WdJ3LKFziZK(8*NRF
zu8~inIa2rmA49qm-{E<(2RHnde|*dP+|it{jTVo0K3;hI#Z|r8=L)ZHPJHZqr}nTY
z!<LtJ4X^cWr8jGJc>5`-sTnm#mZ}x}z3W`@qVZu&!L{oVIevZ;HJq7+7k9rspH%#K
z%heA@E<|sLJo93fTYs=#+K~&{JN~Zvy65v1+dXV5OSV3JsGhH4r`IU`zP6$Lef<>e
zbN1}68;ei<48OQFSNj#){DZ;D_aB|vtTt)hJh$7=ZbtZMsg=tdpHpGuzt5`c$`X}x
zR%WxNT(Jn*Jax1F-p1b9b@SzC2Ra0KartF0vA&a2I$?@P|IID$9b_suNniPOFl%eT
zqphEhg`3`3%jIfnwr1PhyRVy0?Al+?6*J>%_V!Ebo)>O8=cRb*V8YWK!oNPgw%)9<
z@}uzdIbDad+yZ@%s4Yt{Ub*Gn-T9}_-8$5=eO~G$u0YP4cPfsnR;+Q@eXb<q?yPrr
zcReoK_449ZkGh*{F6?yv_Wo1C#+?N>oA#J~($q;4Zd*9{_G=TlU2lu>cKr*A`<<Y)
zz0RETsMOJ=*YA3~{^GLf+@2X{f1dw*DEa&1Ra!T5tBOk2^G%RVd-mwMMAh%JDk?`N
zU(u1Cez)gU*TU}`Yb&z<NCXKb@Az@$+>y81^5t<(b8fr&I5Fk<r-W@!GJ5FgaUk*L
z-{tdX+-*FqZ_&$ttUvAFl#dd-+-kzC*I7RgnG-g3#t{)2G2yV~vzJV%@KIw8nL6W$
z%OabJpRZ??Pn;vvfA;Od|0|apKYCie<MX$84-cOup`Uf$qzR{8bxcaqS(Wtc#N*$F
z#V74&gX;637J*Hlsv53b519}i(4rRdNI2NT*DvAsXXmCm`J+cwGiG>t`W}6|q0wN)
z+O-E$N*|Wxz27E!B{V!BD6sdcpsQDr(YE_{W-V)Po)Q|e$n@TP)s_SAx|9_4#I`7?
z9zA-c-6ZL8a7TaXdx_Uyu7(5!#kJmirDm$jI@{G%BJq@$m(up1Dk@5c5@Y1}#Ws6c
zyjZ?w5!Z6-HjU{sqt?%#A;GT}dNa9>$JWO;)O6hpiySfIC<~9?>;5@w(&zIBT`c+W
z#l1K3bgzfE7e~{<0G0T%+kNY*9GBew^21Ln{j1@BwFTUJwwfkv5`0_s?#Go2P9fK?
zJ?!gJx)IQSR<!r*ng?sw9(?vGVe<@wLyo6To&MNvVzaUHNL7r6i`=3|YgZn8)N?Fz
zy8xfrgSGr`HJb1Cb&4%owTjL6zU7~Ut-CvVJKYX0*tMu9YQB8?ZRa4TH+>7IF4#YB
z)25=5qdrbaLV4|?Eep2oYMNzb|1N_+%y`Bczd2^y4-*<z&60X^E@RQ%vRD5!<PR-4
z_u6%%Re^QRM4QU&4ZD*zM3$`=<aA!VXv>UGtD-`>o2P6sNxH+=vM~7h+N00DB^Vv5
zc->rKb8&`9<Mrt7tkQ>N5f_{4_V6kuNqvrq4D*_6@8c6=qb7LlS<kUGFRol;{ZO&U
zKvQNCs1}*X^XyWhqusgCNh&YbPd*r9m2*OTt;ZylzoteryeFx61_`UDZ-z3OcY;Qv
zObg*M7y|PBAQhg!-=4q2@a?=UzcXl1s^P?B7Uo8Vg#jF~*Z==xQ+*k)wnTf4W}(2P
ze-<9v4R^Ip^n{=E*^*!;d*^^2$Vd*w4^?|#?CLxG@IcPA>1>|2rY>r@&VTwd@1DAk
z?fHd00udo|@9k{bm%C<Sxs;5`Cvk00p+*OW*z290oj2E;U-7(DKk=wez*T`&yG~12
zZQpN_)W7$=Rw8p<MD_$Z@nx>7O80;Nntn2Vt%@g0v*Vkx-8@ewZQ*CS(0gL>^722D
zT(j0OaEVR+G&_|^@qm2T34P;#@7{g)+;(Q^;U+o0pr9x>i#3W2o6q_MIVqXFmY=_f
zF`jkv{@;CEy3-t&e^);B@44UWL%iBIqH+sT`p?byevw=3!_^@1E1mt_&VSvXZI#gZ
z^UnKP$V`jpYc6k?&416&)c5eql6R#$<@kakW4%j!Y|65qJp6I!!pfDa4oQhNXHUO=
z#p}}kX$-#M7fJ+Y-dd)ip!B}?#oX*gk(*ALdz)~rK9Z;VeZ#T<`J6xf%_f?DVTYF<
zcDlGMUb5sE!#Rl(wdZq+&6?SHpB&pQmwffvBl|z^${*i*a8TWCqw0R9*#Do`OGVgz
z=RW2$$0MRq*E{|5$K~-4uh;FL%NoD$@ZNr#e)ED4ap?iUyJm*R%Q_csl{bj^cUorO
zuQIvD4+m>o56SOcv`{CX(>$i0_kFEd^IKP$FVl=(U96tDui&zy?`#!WJ(gbGJl#uD
z{MiW{7Dm@yj^_w>Eh;!Am?pGH)G3i8d*au_RxftVVNn0e-qQT_$KgVoUxzrXo*&ti
zdpBub&8x_~U0bJYOm>?nm#r?7k?{K3+8<3;nJ+C>)LkB~)_O8~C-=!W6*l7Y%8df~
zj2}8M|E#R6kZw|0mNsGDoQ8$%8ro}>^fU!$A9Xnq?_L?oocZ#kf=P+9vJ8j8|JR;-
zz88z;-Y#EOd%9%J?mdfiBi=9Fw2IC7&%QEQK9dRCOSMb?8F*$&E>S&de(elv=+dJ(
z+FD0n^{82^b1y&3<P;DY=95^cskVFeoH;TvW>c?rKl^w}JAd-Ub+&)2%2O^p-G0$r
zp<|Q#tGm3Vp*eSV^%{t6Dk|#eElSEzJmqzI;|-rFV$++>`fi>hBXf=UdzGcG%#w=R
zY4@rhms*u&oV_Spbl}2@m8%xqlh3nh;}yx%J|KRf!{1L;RZ+~9G2Yj-&e!?#?-ieZ
zd3t!NYu*!<OuKS;iilXi%eBXa=lfK79qKFJ>wW2=$7N+lDOR6VPV=v6-JL!AN7~AB
z(}J2CZO_iRclXQ^dDBx_tX+#1WG2Ymi7NvQthX?%zUkXNZReI8G56D9Yd!rto=wtz
z|Fk`Iw&m_{t@S+n=kxFT#mUpWWKEyV-$NXyp7}pmo&QY0ME{{@d#}3u-gzI_+Z=kK
zsohb$f+bf@uJ)PguBB`hFWR2(`1<tuiz%|l+v1mf`tbU5<=3_10ivv-rW3dSsg9Gj
zZ@qfdu8ptr(bDVl)XjCX+=WD2ZO<j}UJd_nDwyA>>Xg34+%S=78-AOQI?TVeq<=jh
zU;Mt=Klq(^%U@oZ+E=MI-wv#uqBHlD?Dzfy;@gDRtzUBeK=zL3U3GiDKAN3#b5m0y
z|DEQ)OSc!Uoqy&DZ=GEIle$&g*CoBE{l0P00xo04M@G9oe(6v@xnXMI@7C$TsijH*
z8hTn!|IB{({N8K*LsbiY-D^?MQk$Hy{NU`x_m&;q+%K?RL!+x{(F%!!S#~uqCHMWP
zQ!Se<nb`e!*X{RDm&mQ1Y5CFCt=?8VJVd0_PXF5uop<uzG-QpFXV|@Tz9(lqa{|{P
zp~Ir)caAQoi%prnup~lAN$7<0uBN5~S?z0FroKE~Qg=5YLE^(pe=FwdSswl2H^UMW
zf)tlD7EXBoZOVShmm3rXc9(uP%f3GM+)mA$)7r@|@90a|*P8bC*UUaSzfbLY$eGpo
zeaqj^6fdm$|G)6j1L?|NC#?%+9-d))R^sEQ>>k<ga(9J7_Wdieo3$`7JhJ=R4R;y7
zUn|#qGTXN4pg~E|u0Q>^^<{m|FJ?C=dh*!BzwCYZuIcB0_4F^(Nqup7z2VEr{V%WI
zI@qmzf60<9E#~v&I~V`_FY)`NN?h$LyKR50re3XVz4GhOoSJ*4J(B+o&)XCU@2@j#
zEwb&q-FC;RTkLM-r=8O+_GKIY`D%LQN>yOXgUkGJ>6II&2Ikiud9gk2V9cJL>-YZl
zOjs5e7vuJNG26$3?vHmo-}KMt_nYWbSxoF-Zdly$yi@QzcZRn3=YKy=vXnBghK5Gi
z=4;JA7j^h*Iq&1E)7>ta-#+vEea})g0VVLtt;e%=%Rg*gE_dwN-D7v;uLs8NXydn=
zbEHuG-U;dT`;RTSH?ybmy|PzUQ~R%wDNFWVGHETaFyec+w%u@ZX<bCF09W%m=g8iK
zFFC#O*QH-rO)NdUDOK<9a!C&7gdo#vx^a%BzOtW}`KL_`UAV`tW&IL{qf)LVB_c{%
z^N!reaJk`h{95FhIY#q#I&DbHmV6r}dErLaoxJ1w%Xh9llC^b$!1IGIxhBt<vt>Si
zrNElFEpx(Re;2*t+*>&N;i1!d&tA`V+njf|>eJ`!oqzheGiOe8>)ui<dH1i&n~R25
ztZr>8c)5Jf$5$GELRn2%oEAiNc>5ekE1ooKR#WP49cx#|&%dFgG)>C4&Dm!azqCC5
z>EWW#D?fJ&i%K8YTt3&Us>sOw>u>qwee-)hia+Cj^EpGp>ZGn!@ng~FpC5S3Uo<mj
zySil2f{l4k*)~~ySrUIgZ-=hIi#3gB>PnPSP3;(NmAv)a?zht-b3yI#Yc0)EqO7wu
zR<2IHeXF%cKhnF@llN=r){vi49^D&dUrqL(RF__T+D4`?#yN!b3|GrxuC^OHCvmbe
zwj5?^_4x2#`*Y<#9#E;&cIX{r`QuaZk-Iid37)-N=nk79=gBSaZ*Wd$y|pc~TYV`j
zpLfcqX#x{x?8$toleho3WuKhIjD2^bH|3t~@^~HX5V3xlqD1Cb$yw+6^xrxO{LEQ@
zc-#Kh+2IB&wW1>2O|I?fozQr4Rg%$c21koE`S&e$txINkUGaA2!<yYo8ok%cw3yyK
z+HXJAH7wKTb-wU2jfqd#{#yTj^<_feq7@U3f*o$&JHKpJnf7sJPYz*Q_Bm%Gy{CUV
z_gcS6?1@LHYp4iMY<l(aE;%!Y(7-Dmhsssl0|P=tTRmp^rt#mj)II4@kYsXt+3k&i
zOO9Io&-$eDa>3;-Cyto27Fa)+^w?MDw$hC4Mwiac6IfjE&roezu2{`}qf2Mk{bklI
zzIj$pR!)SY^>xJdG||=JYZGpzzb|?$%P)BSVf~l4mu8vWwO;r;+pzkQ#ir`?Rb1C5
zg$FgAyizec<gu`{RF)R!D>=3~RujHidOpe5c40pwX1&ls^768CX_t?M*8KYUJmk77
z({6!@`P>HtWu0$c%Uu(<S<?6a-}UR)?V58{JA6aYQLbtF@s63Px^JsX)9*IPOZ@-I
zW41Q>)b92BJ&Qi)+}f7u+Z}dtkL|16X=`pCcym&|;N;KnO$D`zSD#MZl2b0Se146N
zV!rCseQXSG%+#j^Se_1BtGL2FNVVlaO8=td-(o)V|5zE<$DP_VKmEh8^x}-0+l2FG
za~{62^qT&*^M3XXx{QoY5)<FfikVaYls9E(iJ8bX1%r#<I?8gQ_cffne@;37bC%uD
z7s5%##U7o7$r8Ky=W{3;@H8J;6EkVCXib~fb~)RE4A3$lw@|)2M{GT<CbTg4cC)1W
zhOby19x+$nV(V_N?=hnN)<P#@&04qKX4$pR_=j|n+3qj#2^)77-pZ1&`x2&;#=7~g
z-}c4lo?nYzuXHh0?c0olG1d31;;KG~xvkRbjLhBf{$O`N$kZ!8Qgr<InNGa9(Ehlp
z{@jJzdWvd>yyZDsCpQMVs6H)tmzw_Z%4~nb<RFvG%+4oU&K$`JDy(?a+26}@IqYt~
z;>nMe{5+LS+wEjr^RmA5nOG%HUUBP_;r5tv*CpH5&fNK77yHh?uQu=ayQ-A@ZhA;-
z%a$0wmvaw#``dbWo!aekR82@ex<BB-8}Iy-%g@?({BF}`$hcg~a6P`t%xrPPk|`o{
zuASI>FKu=6k|h;qmNAEin@7v4sHwf5areFF`KQ;S*DZh6YG|=0Evo$e`PB6Mv_rGc
z8wIw$Xpza=^IB-p8nKu_Llutj$9sQrU7QyYvb}f46|<Q7uSfTMd}vYkr%EPmb+@B<
zZ1-tJeeX*Ty&UuRob@qu5dQFXJyWP4&%3pcKD*0leZIGy?e?jTFA;*AFLsCS;CDN9
zME-7qmgj#4TbU(m776)Y7IaG3XxY)<e>^L!JbL+#6$14U0@G*s%sJlMv|!&hF<xys
z|0UJe-mhM`WDV15<yjKbWo3;oUiAAC&~Rvx(jg~RL&L^`2z9e#&Ntp4mN)xQu_&-*
z!LnVGs`mRO6+O!P&HuADB;@J>^M4wx4I2J78|Bp9EId354ZkY(-uNN%DEV%$>{F(A
zUsctop)=Oz1zir>5E;@UxNOZ5mUstcMMXhDzQ>;`Hp~e9^VKxu>J?$(`7N$2nH;9e
zcsYL6iil5Nkf^3=D!Tn}+rsvhD_0#86>dM9<i^du^5d`0-meL_&M&_`W%~5TA1gLQ
zegCt=GnP9rIJ#f`*yhEP5?)A|UfpW3Xi?z}9f^kusZUdXI9tuT;gV#lCvxEZXCWhz
zMT-{)2wb^Z8R%1X$4}bnMab0Y!LL6*?)o-CK)&1a-YhlQWm>NvG=4AD$|~oR<zx)w
zo^#KAsq?l&FC^KbyUOazsx?^BzX`;Y?V8#A^@0RwQN*H6n}l}X?K}O{>Wb&YC5OY7
zinr}q9UP}EHNRb8rlU4X#F>c;;@*qzpS^ieE6a45^n!Dp?C)>B*<|lMNo7r#cHDaL
z-FNpT%QdNZ?oWTw`}ltEVYl0?&J*pI7OMR&+r`MoQ?!{cZx?6*1n2V0p0TmAuCA`X
z-{ehS@?Xof(Sd=fj@!}x_gAhE(MIR1k`-IrZa@3hJlpQ(Sv^l72Mv)$yLL&<Ki|GA
zv)#&aQppX@mXNwQQGbP$Tfa|yKX<Qu(VV7jXG3>wDNWcee`mVj$pk&mOMFfoCccL)
zZkQkS3KC8Y`fi+}#kFeDqFFwFgv%RGC+LAzWcbdh$>;&~g7Ni}LA~5bD%4T&%y}+@
z=dG*X!V6yQs#`Hx$#c>c=|(4ooI^MI%X_uBW`wdXk3E_b^zR3E{bW#QUs;6f;pd+W
z0iicDRXi^}bY`q^Y+XMs>LaV@H+7rnvuv4$T(!5OJUVT6JPcziez{>mrc~T|@#&{q
z@8-D&^40l<O;UMzKHx8tZwtrEr2S2m0pXFM99O?4o_n`e;IWzPo-MMk4oy^H*nhwO
zYSz(*1t%QzJTE1lHgLG^XkPX=_(BQC|7XRA8A3xfRy6gx-g5Mv&hw0Y;==ZiSIrYR
z9Gkb#I<-pbTk?vh{H@KrQ>KM3dVBtWudpCr`^K9&?N*YLR6MIyZ%FPil`V;Nye@jk
zN~JQyDrHuHDC^pf#lH9Do`{~>dwY99!HI-_(v7!-?o}6x6pOF<fBUx9%{?>P+xNSL
z#@ar1w>?x^Zs)l6{*TMI1>4+QBMt59IU1JUuRB!vJnr19rd9jr9-FmE*XH*ZVIEnl
zDT>Z*A8uSeZ<v2k%ITN<+>)kOr?|9M*VbF}GVEISezWVlwI?3`{eD0F=(9z#^E3ZN
zE%TK+{YiHzy9)0l70=%)N^A4i8V0nNa%4@ASZlCg^~`_n5if3@Y7r2~;`wpw+t;nl
zw;yXX|IA+#YcN@npYP$(^5d5`Oplvb+rD$I_uqY5({$tg6_#B1x?-}$=Pi#nMwf~0
zG+;^Mmlk$XD2o*MH0{HcyVniZZfu%(Z5OwAOo`n|x$ANJH?M#DcGa#5=KOtM6{EdZ
zT{z@;O>f4*Q1LnEPn6rKE_t6Vt$45IALqR4zo+k>cIN15zK|Rcy|2|gUiM_i#P<PF
z;w!JTNN3Ib_VT#S?TBsBR=mmiqRaN`WdAw0fBAgpNlTg?8D_CPjQuCHNW|}L+Ok{h
z_jRtmn3VkJM2zsE8@0B7&u@SHPsEZjFWjx+{j#$vt6j~#rq!N*wZ~}Jc8{hF^H)w*
z4-hR1*imTNI+eHNwROYib5@Lj;ip>q?G8?qIK1iT=}dz)bL!qLmASuISA&a-OG9nZ
z;+P#xQfkI5wNHJd{u|osM)vD!O>=x6cPKD6TKK~>qry8CVry^o9DVs{OR@gL8+vDF
z8Wr-ypZ_y+D)0G3?)!%VWs^7DT)biSSq|F+b5_nbOugpcvhmXIAI1_U4<uY-HRtz8
z^79#IU(0dW?pFLSHT=?E-_3V7>9i=EntXaMzx`3cvyV7TuYCCDeYo;n>qMDapZH%-
zjv0P_%~@kyuIPBz>HA{8r++s0r5-(_>GLm3%I-Fok@4M_%6klE`v2$ktJgJew{zIc
zcX#KLMa7;So8%`rFJ6$a`DUbHJ(s-it21xId0YZTe3p4^7dvq5sl8F<E1fLqLn+&B
zn;MtTb#j_AH}!M+h9qyr##>&WPG_HdTcgtKXDhk=tc&YGH#PrXO}e6{D$ln(KjXN3
z^1KIjSJD@$c(yM-Hm!2U#)XeJXw4~VnfZ&M=3lp)ukDZKf*^6>r&~9qZ0=j7)w1=(
zkDmusqMz+vp%-y{kL&9lpD((<oR#<J_WL?khYZGjGk?f!zyH<XXq{+#(CG~ew;LtC
zTHo^Obmjk_%O$_v**#-X(#o^(M{Kv>J@rUe)Vir>(W;y{quJ}1op-rxa%|rx3;+FQ
zeJ`GfyyBW(d_8yjrezx@$?y9&<GbBVuOzqPEtgDfOj0bOPru%H<4uH)+8aJ=v(<A~
zP3o=rax(l!Q)1n(BAxAbzN#>0>lj|EYP(-*EMNQ2_~5T1yNX1X^}T;*&A7sI?Y~8@
zyPULZ=;3+q?u+HTSK45eyyL~D)i0)qr2L$f`(*vB<CD+t>wf5?a!@ku%H!|G=`Y1(
z!nnmYOHRJobv1A4zAWj*^CS;G(`pub*;(`NT=yRHyl;%wW=;}ERJJ@i)y?+l?hpQ%
zdn=5)c5SZF-mddxV`Bg2ZCe_Tu$TuQ-*@h`d`)9t!rP{udwv|!&RNALoND^x!PJR0
zpD#yk`BfBGRb*j2?de1T`P2HB)j#~no*y7TEw{vTQl0)Kqi~LP<-5-o8tB}~F!;zm
z_tPzp)qkgdWt;zRNpPID((5b%zUIOS@^clVpY4t?N@sgoWp#H~snV8By29zcOJ8mJ
zcSMtmNqc!*fvLZ3p*i2-_N5Pvmere2ooqF?^{nja(sJI%+dlUn58pIzwf?TAo%U-|
z0@6#5zvb>)wP=;rzE3s#=6+%hzw?u)Jg$a`_iAg+zn9Zzh#nL?KL2F#+P*`wvc8G0
zHKy5`b<S!yq`7?l+2@bdi_WxOFaBis_U4un>8>QFmRB>4YOU`2@1B!Bf0s0S-J(O+
zKioaD{^k9zHZg{rF_vEn^=wzNpWd~s==Npf4_Dheimkpy^7;8IWxn(bnSOmjkyp9f
zo)0>F<su7bIfbo_@JsBRKACNf&wV@LmA!UttIt`oD&IM<*13Igq|A*e0<Ej31#el(
z(99g1EW*&$nan=7f2G=d^Z2^no`u&JXlSj<+K_TChbeHa<i88fDaR~d%#u8MIr&*o
z$;Fi#T~530)C<4g{VuTn+N0Hb`v2_V>U!iIu|aRX_q&RYxLo6tUDqmV56NtsI^TKr
zmKV;oR$h}zX58F5J2p^3r0spU#EMl-tF$JXvi#X~&mr)@5~eDZ8K?8U)pu}I%{2OJ
z{fVo)^3%lMD{t+Ylsfx$<d$s>!ndmfbp38UD?5<L<de5+OXk68y}T`@%<ZQ=^Ipp^
z-1)dNW$TG0%PzGC)Y)5GKlmOfFR&$7{^h%}W6O#!UNvX1itnF&;In>hhqkKp_M85m
z&K#kE0xri^AI*0w{TeW#t;#;vs{Fu0CW{v{Za4k34-w|cjK~QSn*4~Nqs#lqrumzX
zzRvd2yb<PgJcYmYwYcG*i4U)|tw~WizBw;glUd3?%Gu@EAJGHRfhV%{ZBBmrBeiL>
zam4-_+r689$=^s;VG*mA5s$0Z?tU+I=y$Su;Cg9|LkkwI>U(mg<WaJc=+rlNVnqI`
zzhZI}5D<QTFfp9bI<T;Cg~pG)p)7yCo|$RX*=npS5!GS#HY#E3iRi~Fp0}7UTCmCW
z#VEfCdvVh(v3B8tJz@?AmpDCt#&xgiMu<+ploRDW_H#o1d_R%*&8<~hY;!`YV!5qF
zi?-h04r}>6^A}t1n)&T)d4|(bj%%AMK03WR#H1W>c+u1B4L4#V&T4(#YBx<MqMKor
z_oo{h(u@l^<)*~;$3EUKtH&ScH#NRKs+aM{sn0(jT)iH#v(ET_-9ORDx%z2mUVPv2
zch={RH_mb>mYw;;$9{Ns?uC@CkISqhH~k2&No?f!{Ui0J)SP(|IcCMl?{{XzRG**x
zR;i3lzdn{#e&2Vo>{`8DrKcXX>#GN6pM7-J@Nvbfht?SrWiIdVWfkU1)a_x|w)W3N
z%l%hAzA(=J^y-BA%gZZSon(HjT4BKRXw$92)28B^KI}gKBh#6+>!goY5>Ko2_JEjo
z%q*{`uVKg*k(;hOsamn@oNvw=3CrY1g>I^b9~Na-CjAh;Q+c|VN3Md)Gq7*}L?_Ql
z|JIh3ZFY}(v1?s{3%Brd`DCS2k~?|Vy01I6(|u+~nai$+TV?OQVc$McA^I5u$Fus+
zqSJbmcf7c>*`sq4(}UX^CM-X8KI!(+<w=_)&iwkNn-|(;z<z4x?gdkp$4T}aUb2Aw
zb;bLo$2H@uCqFW^56<yga(|=jf+EI=mX{_zNR(h(*W%-6|2$+~MWuvmcG^Sjh@;Q4
z#BJGmR1{Sw%Gb|3)S=EDvF|~3SJs&Y4M$%pDr(L%pI0~O%8JA-z9GtMOICLqv(G=s
zTW)iz^t!m9ukPmhbI&hMbS|9fc)#>m>w=A``>S2TMOt^=Dz*9g!FI_JD~(g}XS=q>
z<w<-ph@HJWXx==xr_c9)_{4oaXn!v2{v4eLJg$suww(7#`{bVgxoEo2zw1hz-7Du@
ze82BZO0n(y+V9-V>+Rk$$SF0hdq1b}-qV)Gm~BOz=X>XMuG22D{HOA0+1bYv+2xkh
z`ibm6^Lysvb9*P6^b7l5E}wlgPhz6F)sn4m4h6myuKYeVJ@W2e;n~yl4_;lPZ(H@%
z=8F0f70-6%g$pt#-aZ@Rx^SJ`v9s59eo7G*msz~2YW9M9^QrHran!}8C%jPLVf7Ve
zuVtE(d1FK3VZPFzGDS{{x14gHe(&##TXxm%<*r3hR%PA2({@_^F)CMk+i%s*Cu!ES
zs(p3-&hE>7@4Z$m>sT3Id;G!cbqA$BJoL;zJNG;9!;Wd03s&r~`B?I9Ligc3RXx-8
zJJZg-ys*+*H~5ao^l1|(+-$Xew|nxroin$*zQ8y8(b8pdev%b@>fc^E{?>cjQMCJH
z=0zS`83j}A>360{?yxN^$(rx&x4Nw4?S-yMlakJ?V_CQRZJwX?KJ7HugYSA4C0~Dc
zWT&!!XtAsMg`LUSIlJ%3EnlJLBgU=QF1z}y-{)`cMbB%UEgmnqH$$Yg=gzhxyVp*c
z8hUx<_mey}d*x+sl`E*~b**`Jw%|6;nv0q;#dkPMGUs|^W@>g?%I^Gf=U~aJ8xx<M
z4f^(aV>>S|?~S)*KFe(F-vtDQ&UwdTRhrX$I?XLmcT2|2SyPUPSjk-MT(c&v<n5il
zgjiADU7gA7bB?n%`yEL$Obqf}l4ia<bhA!~h{#o^vdzy@mM*`vGCMrSZNe!j`JJ&<
zg%?@VCy6LWKbxfTcb+|q@id7mzeQah?|GOdpQG2h?`=rc_Q0QI8xvmXvn|}vB*4`s
z_;OnQT&~!n&>QQuAI)FRq4?y=U3-N;p?B)uiZ1!Gw{zk3`CD#pdbMY1OxeG!cRu{8
z_*I@+w_@_Gl+cin2W7iA-sV5EHfNHGCwtUV@wR=-g9D`{x$pZioDC8+5)!G4O&8!X
z^=r!(;X7!#mv#D_tY-<V#W&CHlNVXty;*;^n0wUi`?|%)(hKh1b<UhVuVTgIn@eV%
zP5V=~pXb>m71?V&$Byq<wZHSE^!9^oxlExq|G8~F@t@1#B5%+m9=<RIcF8l}`%fOb
z_}B05W2M>Fjlt)oYE}f_cw#BT=jH8fIQ#65yLmr1iF!`r^WL>^!E2_C)4i{(bAIgP
z67}?Lkg3(yN$VY^{$tKDla}jm|MaOyOH1pURhgdWq!JAct}8!$W-t}rFfC0tuYU4u
zo^4T7Y1Gy$7T;BjPja%G-#KMi{Bnb0i-G{hg$oxPEM)%umY$;GY1P29@WO-bNgej@
zEJH*#MEV{%8db16N9K%P*#yX$A^%$Jg^m92eCPz~!h`x&Z$HdAuRNbY9n$XxdE0Z+
zl_Q|8`Xm)k-#Jtj5MS%jE~>~fYuUorzZUv+sd!$>W8rGtz?p0xH)okfLYU~BGSSU1
zAMBsw3hFcqa<zW=S;Mem)f<yZDlb1QWZ1XhP`LNnA5Ev~_FA9KGV|yTd~y4llh>Pr
zyxWatc(=3qUj9(EcSnrgn=<RJqKbWK0h5;e*So5J!KGt@-Z$+T?4?sfTLWU{pS`)q
zWteyQp!e;CdUN&)v6)U4y>jg*f2r3$6X)yy@7_IfyTA85|Dgp3!~G+S|6No2TW24(
zTD1MJBJ)la&vxCz46*EYZ+!K4ysY?V_IY0BkkEw-xXi>d*-c~Xcc!lUnf>$6;_nfA
zcJe5)sBP|jcYMx?lgGav%lw<YwcnwZH*UXQl9NqUt?J>tJxuQ3iz^<@J8cp0b>=1C
z(-PmW>iF4yH;P(%RQAFl9@8rZZ!drJnZJLAftIe-w+Dxc&np^h{%Uyle11^yw$|<U
zrM*LQZ5i4UFDyEJF{a?uM4sI*xn%C1f4uPZug|Y0`(OC}srtobtH1j1wPpAo7wzOL
zou=wJ>6_OE+X8M8)@|k6q#dHxcqMH)bjzuEg4oMx_wBaGU4wT0rL;j_F%9{A)IEQ5
zLa5xXg~7A-PqWV7HOu^%Z~FC}jE4j`Hp*{iZD@_(clyHd_yf`+Gjj5N&8?I>{?#gJ
zy9?LG=WjU`e>~nR;WP7xo%6fej?VahoBF>VnIETgh<DHTKg@T29>~@R^h{ZP?DNxh
z{ld3j`vW6HZ`jWLZCw3`|I_Z}sr5^9l<wdByy?{=-aS9BNqq8J#mU;t(#+^|&|uY7
z*Zl{d7ssquR){&j%#?fOy8IjKP2Tsed;L3SVzu_CivNGVM{eAiaXfw2{%M($k8P^)
zemVcvbQRA_@k&Z-^Fd1hS~u`H@(L{$Wh*hg{GWr{HhQK2lap13^*<4lw-3Hq{99{S
zy!!Ovqvm%Gi~j9*$l$dpK3{rf&PUzw_zUvU8|HLAKKAMJrmqpP8+mNr3-!Id*7oQt
z_sp$(rW{P?Igl9|Vewe|Qxjj+R?i1>`|Xn79kX&fvb^Z^+363yR2W3Zvg}>j^S?mG
z;>?=J&5U=Y&i<Sp>-pX6>;>nqKEF$BjO!wrtQDL6)=n}wnpWWUfAWSu%ib`Bveww=
z8^%^|d%XPp^1#BAohm!d{K)Q_`QajqOsAnv^qx6vhflqH9(U@a?X93qg@TjQ^$(|V
zZ7zALrL#^)Z|A3^Tc+MOb8oX|*FC!1pH}equzvCJQ1!&H2WuGJ3vbU=j@VQu{c>;A
z#<Ght$7(*QUj3?AWc~forS5YJlU~~Fi;qA3Q=I2mZureD8Dg8Kg%|w$*{m7!nOVMX
zwohp6jCbyH&#*q{Ik1-P=c~&bb5?R}&b{omOU}HbQGHA5rQ`COGk&J&_=zu{Tg`dA
zEWdl>_DzxnLc6y2bUwC!*6y0!(RKP!(`mh>*|X2wJs+*u(U*B}Z$Hm+Z@)i!+=na{
zE#4R*Ged;cHL&bTz;(S@hua^QFZ=nY_?K|%@87}3S*N#uc(z)3=FSS~^Y)f4Pc*b2
zwnZ(g{bgutGxx<6ol5svtpe3E9n=kaY;}a0oH;~7Z9PiUJSY9*Wm%9Za7}6RRmMrl
z&;b~3^Gun$*IYcBf37aw)^FFg)nohbgI~5EJ0t4Q;lXwIp}@lgj?GzaqK<Y|PYvDn
zeB_)r#iYhnd*|zGw||r-TRx2r`}A{5<d4Hl0&8YiPt{*`&LMbhL~fXlm@(gSW?`$%
z`+v{+>1`(XY3j#=+{zWXZ>}F%lD6UB@l2afH#D-E**@?4F=4lS1>a%G_76`BpU*8f
ztb3Jns5(PpU9`#cvtK=JKHgZ|vQ6jt_0XRTEPIyj`TyrKPxZT4pTLb*p6NfDcRMfj
z<25g>Php>5T-aTiekf;BkZ`0ywoLh*-z=&3F67<5w=2tV+Y`SzwjVUk^s-xSzunm*
z_h0aM*|~)&fBWy$z0O^u^=aa?M~R1|{Xaj^mJgO+_w=fGe3EONq`<S!8|4jNPcNO4
z8+*pK{NCYbT?`)sx!&9UW_Vv?B%GMx^FN2@@cc^V^?9mlR$Z0xy-)Xk4E8*e(HFL>
zt^J+!xoxM|p84N%zUec&ciror+P|K0vbK``NkU7r(~8f>@^4zY=U+K{#N{RtQIVjC
z2#30VW?5N(8;-4czc1<Eqh5_HmCaqNg0^Kcmf!z6+oXSfrGsAxE9>mD=em9d|H!yx
zQsNLO;-e;<^>}^O<4^LHiAM_Bc&n$BAFp+Eja}lNa`TU5jmb0TdzGmwhvjOg^xMtu
z(wCTdIUsPu#Bz7r^N0EE&uG8zS-jy;xx*!G&rACQl26;LarR)Iyb83!<`wgYpZZew
z*Bi~)7qUFvTX^kdUzWoY^}cObmLT&a;otSyK3z^X-lv|)_TTvCc8b@h?He*&CHd`&
z#k;mQybP{?@~V2?({0>+$K$6SFP$E9?n-92#Fc9znI}!zY(hf?ohDpbBm11!qR%MZ
z()@mj=*BX8r#CW_3y!H2JiTyy=a(gNI<IG)%2iH#w06~#+mf4K82^x(SNY5DO&fo7
zE$ix|a$aXS3~EnpU$Q03f4N_5#?0LGQ(N9!?0jCT(QsDuG};Kp`}zpa{UK9ZzL@by
z8#X<*N-sWU7`yY!t>2ORmUh(5EiTKs7a#6k6J}(Xcf9X>$mE$%dUzO~)flttZd2CZ
zZ_;(fg1t8CsNt8VFJnrdJub=5OFcIywZtpq==#&!i(cK`zA5ir_>?x`%2%(lPjnb3
zYMoAgR(^R$QUZU-<k_Fs?`eD99@`|Tq2Kj%=d{{FclY2MR$Zz6wO*;ImiOzv|K1d-
zd?0+@dGq+Q4_Ev%FFbnlc)%{Z=?lN}JimT5I<qY2cP!sVuiUFq?{@tQx>II6EA82#
z*K&{MN&C4Zrf17IgtBz+o9%Qn|B}t?-I+0^2T!_0|G(F%)#B*mq<zk2m+j?a^&Zd9
z$y)BJOj+EuDrje_Vn1)SO!POwk}R>)S2nJn9549bVf49V&_I;MGig7Y&pD6Qu*vT+
z5~_|dYpzoHcxm$Cn-9v1Q<lrr&9rOOb>+3O`{ETNcGGRTN@n@QX*-=PWHM*Ai$v_W
zm=N5#V3XdZ1AkNZI?k1U^@qtfI3R?h{h-B(;zj?%{x4s&Vo93G;@H&@bu<2%y8n5f
zbx!)qWQTu2HH?mVvt<}$HhQL{qy~gVwcQJ2O?)F!(^qmaFWsc?{l9&Yxf`QAeY$)i
zy1TrOByCM>ac23W7*&?r9m^VGE*rHy;q<io_4~JFFkepp?=e%if1&gCjWvJO{`}mm
zvEwP1O;^+1rwZFkRrbF75wa;NB`EggVGltSmF-JhtArR{TIy!X<u5-ce<Sy|pTEx2
z%GE1BnF+k^@OdKC<~Vmd*KX^sPn)uWHW_vF_p<zcmU~P1+j+&D-lhLP-j(=&TBpue
zZ(-2cCpYABXZ-%0q2;yh)4^!{<RF%VY}~ol+g$|Nx^8-Tc@;fAc=>1QyO4SHFP&pF
zZ|*o-=k-_d!l57E1I(g7i!jy6i>7X!^539$Rj>Vm*H)5qcL_e;^0w{~SM!S-VK$%c
zOx|+-%K~YE`E&S^{L`2Exn`byf9cM$V3o7=-;Z%JE|_6acH!rYtWTAnpI$#9QN`P}
zXwfP*?`LP!1g_3{$ZzYCHm7;r%s?C4G(A`Gxm}Aiw!AJDD!*UL|1nE<->Kgx&aYme
zv9E>o^{1Ur`r2PyHJiL9qxZV{nZ)RymRmftXH6DU^|Y#}yM0!U>+pANo{&(-Q0e7+
zIDeGxZCupg$@;~^!2ZVFH}$qNxEpW2d-lUoy!fZ)T+hu}Ha~9pOk2Bqn(`zYm)oIY
z|2Oshb2m92zN>ew^tMYotEJMG&e-(FN$d5i&dsw=Xltr2G3(C>HeJ|i!c+TRc%Krh
z)XPQYw`Ek0xqiI-=jN2S3Ym8Wn!(S{9iClv<^3@`@$<j?q}=uRdG<D|^!FP3Sxpl>
z9RH-<HQQqCTrcNbTW_&7Pp2r)3J#uhyv=mB<vR=hRYA^$_oTw|Zn>p7=~ccC+qm?Q
zf6wiGx4#wNDtFbdpX_fp|M1=G^SVM?eNXRy{?FaL;Ob%h$rERux%P9lrq0KUT33(G
zV&yKZSk%$kx9EE(b4B%?%RO3gzRwr?scPTz4JzV!Y58{N?}^JZr@l#eJnQKd!QQ)4
zFHah5o9=qx^cMg9Dyu&q-Sgv(=$hY|M%(Xy7F>1JF{nnw{@=EoOMM3)_9&T}_FY>Z
ze`@aZxh@N~^`vTFySS|IPUUsu>q%vojJBKR&VFl@mS^(8x~ucalpFV#-qc>SA)4=u
z>;xwcoo^O;d%lH<-8XIde%x<SoOwWtqid+_$(b*Y9Cy+3zPWR!#xMUXtW63EYUi2{
zuVX4(^ypDQ;D_qf96w&qG)@)bPLDF!B9MD)>jACkr;}9Pc6P9QDzGT(-4MMnUApya
zYtyGF2G+@3b7yN;yiN=*Ou1Gak-Wa}t;5e~-C}2so@0#-b?LE(b`%CzzMR@$k$+|F
zoT_gpZT_Az=(=`pLDl7;PbK`$zhoVY4zvbso3s1fj;UfF!<N;Vw)-_lGflW;GhNqP
z?Zb=Y#T6ANmTGi0DH#2}eWUb%qsuqlGk)iGeuzoC|4$%1KGLPh_P{D%*Da^Q%I62l
zuSxxM-8l39cM&#T8%gbku=9uBt9`yYJ@{w*yUZ7f^6&S4u{fHY9<=9KdY5nKoQj<4
zo~@NvLi3+|T)5p}advdw&pN)Bch4;--h4Ozuvh-&$*<Sh%vGPiub<a@*<_)$M)~h#
zWHQ|^>!}O7zmYuB_q%QO`n`R8x{GGoC-bSt7u#oU3_G(<Z%cmahhta1`bd0J;=1tl
zN#hZ}SqJ}qzvtbawo&I-sp4P#{j=j_H!&u={}b6fwdD#+-M>^W<#{>@Kbzibt}<L4
z5i_UuVC9)ze-GY0rDh&|aOwol{|D7S|G9lWXyZqBDKGP&C*_8(rYhIh_dNXGuNP9L
zx7g5h*SZU@r!*e%o3XEYmj3Q;J6kLH4Z>j(Y=x0;cRZcNy|enp<|*yDa}tau&yjF`
zxL#UQ@B2lin@4}CY9EklJ^qwID*bx?e(fb2zaN(r?l;!zY!r_@{eRcC1&b22XLr5w
z=qmbiHpFYvm;5c8XWI%bDV=;sP}ip<SMf)}GRM-n3&N$RHtr2ktH~GO;`XuLb(23&
z`JhS8jSY$QFK4gnI{I)~?sB~)M|N(sNIsD}u`g@e#cj^jyUMI5J@fSI@;dEt=H;o_
z?|0p&Pt)A-IOydCwdPY@T#JIE3Qwv&zqmBny(rp5Sx<BB5*G1Ud5gWj>s8$0i+b{G
z(p;(jyAPK^#*6qx3tiPOtW4JPTiza)Yc*l&M91^Vzb0m8E--oW=t{8P4a3>7-0UCN
zOMZQ|pSvc-Kl#_%B}>&GclD`SUp(5AdEfBIuIvqLH5Heli=Wpzzqq8-`>5CZM`dlt
zYCWswS*yIPzP>y^cYC~d)STk&j{-utWp{mDRvV^sn`_%ce(qm$EM_cp&QsDp*lMiS
zy14I+*MePBc5Ucnd>(uLs`&iI!%0k&d$(LkxmEh^sHXatV|j)TyL^|H_v>AmD*ljr
z^&E>ILV7YaMt-X$Ctr2T`^}fK^jKro(%R$SZiwicel0k4px9!gKmV=rO!pr3Y17u0
z>(8DvVRG>A`5w#9-96c}H7F#+$*IgeZ^uhDD--XVYu%j#L*~45srvGw<96P?QpT>6
zPN%Q!Oj)Luux;mw-20thdY*lGcd_!j3BMKV@j9FG1q&B4?!Mc1a?A75?9<ymT>WzN
zi*+f(NjY{+OWlB1pMDmebo({0M9sZC|J<JU`%;eWIl3nLpYr@#)9!uD=O0hGCmU&(
z&-SxUTV1x^M9ek$-<(K+tCJ@8y*{+<sr`qqXDUU+pM8FMgfnNYVey@dwi{=rzK)r+
z<nGUAF5~G^SJ)XA3h+3_&0OG~JvA(oORxRVO52b$9nmw<i3?sUxTN&hnu}U(53-E-
zK2!VA`FjF2Gk$*jTrO!<B(t1Hs$;g5o!n>6<*|PyzTY<~*x$XzDqnN@+<IN}J$Cc{
z#m;)aE$8JI{vD5hTK35OwwlTL`+8Eh$jft{b-p`u{%Jma`n1AEZp}5@^HV;DJuxgj
zX5=SPAk=wrq9R{>rCso{vrGSOTz)~(dSVmr>6u2kYEyX?qqe<u{6D!*?NnmoSO4N0
zVjBt$71kOq{`h#$&K2{-zPy^_ocq&kS*`8Vv#wKBQca#O3HDRewtN;K-mc==zHpUQ
z#RKMEulMJoFY_?o)a1H8XVE0N^&g(<vjx|gmF=$86=`z3%Q}5-nj72ay&S)^W?Izg
z?AMF$Pt@4*>eB4)bJpEIG;Yto;l^sbZ?1cL@~??=CAQw))Rmk4vo4ZR{i}7}_iLHL
z1q!F0%?}C;Y`p#USl;&ke|tSAsmQWSjg$B_&(C%3jkO!q8q!vBNpf=SUuVqGR#y8@
zy2ZJ`FaMzG@>2=_+IgJaIanvZ{7`%6&%#S0r<N^S#-_-!apT5zD^1W^iIA7Azwces
zT+jN|-0)_`oN0@-cDH9Uc1$>UHeF)Qjo%MUm&^(aE|w4Z`e1MGvX$9fsgDaRG<0>3
z=9oP$+x_%(n2M*B&dau@b=*hd^a6L^YgBVt5_(TBbaAluPp*)c_6O27Km1ssp|8Ju
z(W0dF!VgtEC-JemHcHLs3|8RVz3knawEM?*2hRzc8>VY|&i15F(V>I9+yBqXD_i+e
z*hzqcB_kt4L55HM)f10NDsPVoaPih@xBYlL&)Jo6_EnddHLTlDA3G?eeSf1pXgSd&
zmCX9TGbj8!&ntc^uM9NicPqdBdCk1T<yD(O3vN7ryRsF7gz+7UaxQK9KSQVN1$Bp)
zOYlCGS!e8|`f|UYLO`xu%-VBqTPyiWi<bOVY2NvzU+^K@tMGo+n2mBGH#Jo}+f@@2
zy8Mn$EM3I&%(sK**(G})$F0X7yelwVWjoFOI|G~NBtAJ-!EX<vEyV&-o<|BRH}Cwy
zFRT$}e(p}OfqSalt0Nwhws0SgcRajIQ)1yS=eZ&;{U_DVy`-oxLAS_r(h_EO4A=L~
z^qy2Q!_wxym#SyGF=!3!v2WjJwmT`7#M)muIgzh%!nzI1jyQVU?*Hz}T((-lI8N30
zV$)3(=G|ZJt2XZ}i4bsl@U`k<scie<#+y3+Xa7sNGOm}O_FTH+FH`FNq|FBwz4~%a
z^h<8?Q5EK0U*uKS&$yJRD8RAcwUwcv;fhsrPJN6|HRKL4ZCSO+cm}u9*$=kg_N{K=
z{gJrVu_|VU*vf0pb6GvDRQ`n?jCrBRwJmSqialJbjc2`?lXk?_nvuh-LUZ*Br_0LW
zEX!{@Cpqb?S|sr4_lIAHj~qR=$SlrbOF%=HmVO~8XT(|S=5B)$t<zo~S(`eJb$6f4
zQCBss-2V07!#R)FYNTGCR^zl{NovaEd6RZ1>-#zyNF2%Ae%j%mygI{O2H*RtGtB$C
zXNPW>aX|i->MCs}R@VK?Cd8**Q2HA1++h-5Z@sU<b*8vud;T%4-}_fn&ELkco>%j%
z)#nSzhhJRYD9OEi&-q^d{XHioo0Uq=l<`l$Ut)eerbJ4*ZswuWm#=;PvY1~Wxu0e7
zAzu69w{zcrx)ZH`(a^f=c%9Qdp|-mfa>myxJBw}^@XY<O<lLT5@4BBXlbz)*zb^KN
zE<aD@B;BiZZ_-*9oILw=yUm%W&t==6>K}2t8-F@{ey{rP6UG*cp4u0eUNPZaetGA2
zlh)>tkPyLZr7Q_X(VhL>htqoJ96BmJ?c?3^uPsv_yi{?${r}3oZu50%(Q_TI>&Y6t
zk75*((w?r%Sk5N<&&BdhY4s&};RP?bX6Y#@#soSv++^8X>}%p3_A{;1Eph6NwNK@)
ziM(p={jaEg)bqdN|5*uk-!Ffc`1fdejP(;kJ((-*zb>7%G5(S#SMx$rt+#$^!}eJ%
zVzw1oKa`I|9Iml_7#k<^&*X0P_hVvQ0g;j2iEr-Alj94B3iFIGFT34;;qaBK>^q)(
zOPo_3<6aVbJg02J#0LvDZRs#TIZH;c_2|+WZ}N(YHWl7n%Oah%@)Mf_e^6MUb5NC-
zAz$mBwUsZ@*v}Ttdbnk#Bk1H94Gpcs&-zr=)wzG?FDkD6nbvmqw}R=peS3_bnN7Oe
z<bO~4WQY5^<-W^3W$xPbvgNB*{%mdG5q=qIsd8io$IsU*KCx|>#naV&dV@vsiIXP}
zWHminyZYyYtKk+#6P(H(aqd{#^lFy^Cnx8OkQJ-f9m!vza?CY2GOX*{ZX5A;#%HUQ
zj_gY}nEq7v>9gJP${8~@&I&!WYk}|d#XDZD^3I$*)#c^0tv++Dn)8l-|9$yj&0qif
z2Bzm1I6uG7_SxX_@|wO$g^%WQJ*&Pu?@K$&#IT*qG;cq)P!sJtV|vl>wo9#mpXH%5
zPvT9xq?vaywUk~cjl6NG{`!8U8Ql)N@pYEHap^G!*Iws6wvK7#NgJzAuKQ+elv-Sv
zw#8a)Uf9mHouwKvWj|AWekPgC<~o(R|M>Je4UIQD`|^z<J<jJit`1vU=6(1>BH}C;
zjpe1Ai@r}#{|GsN<x|gUiGK&qKKc8L_xfFXq1m^4*Y6U_F5y~xY4MFcOQ&q=D$?8W
z)v+zEUNO7=*R0=ib+i6-Rf*g^GBf=C!M)~jEr&N5he}VYIloFsdVTNy_H}QK7<N3|
z_PeulcEjma0l{1PcIWMGzIxQo$8E-pD|vgq9OH1x-v9I$d;GDtMUVNf1q2B_I%O^Y
z;99relga$|lg@4G)v2$O;CnOY!ll!vCx;t9>TK^Q)$*zR&RVyo?DLmq^Us$yME}fk
zpVxk2|LWe~@)Cac>@NKYZ*k`R;arv{S7ra!C`t3qjyc=!RkgdzuYc6~TyEjpK=qFa
zY2{HWl@1)MPI+tStv>RuPtjIabobSk3x}tOi3dgow;Nl{>-i$0qsDqcu+1&i(=dAV
zs*XN?HC<Ik(F?q?TMIMXUL<;Paj*Dt?Qy5Snyo71){T`fR<BHm4%F;>Vc}FIq3Cd;
z{mx(2S$v7S0SA_7eaT#VY46U?{AsUpG)*OLTt4&i?pdE-fs5CKEzsbabu4dcq35sl
zKf_r*CphbF-5S?tw_U$X^wnzbh^=+Tvld?X{^H$^x6eGj$7(o~mWux1eRlp&UsFTp
z6&HSyKb@=BEn2`@<5QC_GP~Z#{c=fEiRCQr-`^K4+cay%lXE>Xh2m>B1x0?nY!2FT
z!(+A9{wMeTlouP$->A8|RO6w*(W7<m>i<dh<t9FRc6!|cqqj2mq|Li5<!#R1cv-xo
z=)(Vs*H5Q=yeg4eoVUNj_p|-;gX-}QZwU82-N9!)KQj(?98696>5GE1k4@cvkI&)Z
zthHNSRT*vPlj|_v&|Cd(PyhUB`42f%Pb~ZTdgrHo;yeB|CC~Wt(0k99Q{7*zA8P3B
z{1#UCuTXRE`pT~Cb9Rkii|rTPir$|5FYrLLZ|c@IMV70#KmR#nn{s?s<(G4-J>HZ^
z>AzDxTR-*aBNgrCl9n?IpO|F(WzCs=`+aR2>*`OIMl0up-<x%ILt^>ESo`VeaaGf+
zjygK$^1Acy>WS93ZMKZBIWaqV-;p<Emjk39ty!J-=+nXf&Y`|JF($Lw&DSj0wM^`=
z)bXv``5*Qyj&{*1@ltu<KmUtje*a7#57xxahh5U|9-XO+)wsW_>)~Pd4cit7ZMWu{
zy+U>4XU^Z2($Tl33QTm6yCf>Y5@wyEy2>Yh+2_}6%!eEN>n};(xa?B3=hQYX=E+=&
zygZfWi@(T!ThH<Ac5XiB)cnX}ul4tON2aQ>rs^H5{VJ%odFEE`w1V5c*B7<CzBy}q
z@8<P8=FZ!>a?R=c&IaM%_AYi<(0D6Kf7i0W`1i*Jm?mF6d-6E%lIY*l7)<i-_Nkt>
zf7%=F{&2n40!4wo5_N7zx9)RuDoXr~7XR6jcYoHCIQPpswxap{$`Z!;dZMnkBX-wt
zhnX4wx?9d!ahkJzuZURq!F7#+p@N5QWE&gUEoQx`=`Hd9-|GeJnqPkzmNX^`J=tPA
z%Uj=O?iCgD=dlHX!TV=<*17V&jZzbMx=ncYgr3uhc{~}rMS2&RuvwHn+ZkJ&wQX&}
z+f8S0oV-~+>;J2dcR$(-&fF=RIa$`ZF3$4qqeCHK)q+(WH;wl%zBl#oWB$9YZ`<M{
zC&?eWf1yFqC#K+RtW0f+bXwgXt;=ck=Vvbe%kSFwX_LV7dj;IbOFYv4WGhR2erVD6
zswO`<E_4-l*yn@olP%sox^40K2A_}gcdO)k(%$pm>CXK4(!#8IuFbzc1~yTW{kDG;
z{%y#a_CIviG66pKs5|$nB#fig1%xD&K5f=nE!Z<}ts={dDwU3Q@TRWqf3=%-o)vY@
z$>I5&R)0P<UCuLX`Qyr;XZNj{VLdUs-Os_?fus4%wb-36w^?^?Tj_FC?ZPHkV;Qvo
zk>*Rjt|8Tbk66YsTnP_VE)iC{y5;w|mIH4(9!-*d_xRI4_cwoT#P>GH-d&gRb<ejm
z!Zo|EoL?HwcWBnTN5|)%D&urJ)%RaqqeFIbrASK$i~r`MUYo;mUtBSizI)+i(UZcG
z{Mj3$izaQF6};h@bXbnoqx#rQ3>?3-W|);4t*xH#c5i$B=Qq;vhCl9bXH1nj`e}as
zy^ZC%D;8(=avGHw^c-8DE8t|pXVTyx@WFIJrd5JkPDgCgs!hu0xe^Tj7tZ3Jaco_V
zYwB@n>DHuWzPG>Cn3gN)?=_fY_V#vh&3@sJF1s^tF5|s_=2z#%)#3r0GL3g_6?JMz
ze6i(yvB=;4*R99b8N68Ic&6@)uvd1?M8ineg^Ln|RZF+#MDL#PQ8jy|-pYmhZh5=9
zT1Inmy{vt-^Gn8cnT%tR>fciHX7=dRAN@A1(n3bFJd|-W)58r-ie>9koE#lgUMv>i
zYhHJnYs#1VU)S&8_t;n|QE_bLAET$tZu_j}?O7ybwl?laxBgz|&F6AYc=fqmFSX&!
z{9?0-zxLOcWy!b0ETqz%)Kg5{wPR1LFu#}f_nFN#ou3l}&+l{ddY!gJ-R*B)y;Fy+
zc}we!!h?cQ9bKEeqPAIgUG}$|HevfDuXyKVVRNBYm&fHdQfopL1lkm|?51v$HPX~t
zxv=*7ycG3Y5xYtZ_tzSP;W(OQV)=?MtSfc&j!wSS)tEllbCR}CVPvSv2TiS?()*vU
z<#*ehc)7~1;(hF=Lxm>(eeR~w(34Z5;sth}_Pw|2=$Su-jH|`YKHe;O|MC68$LlvL
zpAY=Fa?8sZU)wYPojzW9{J%?1?DiJZ+gT^idAeolINS|*QTyqp&CegqG8e7pKHa=8
z<;G&=$nA-``u!DMsg<EA=J}3cMW6K4_ctwkowI(?i&>0G>F*DJn>+iA+@|8FOPf-Q
zcYS?(yw^O|FKdmkB8yu^^#`#RyA<zpemBpU<rLA)lF`knXL;|)#I>O&yiST8WtuTn
zO`UpYLz)y@H?Ymhnd$w%@PF4d%L5nF<IX8h-MZ)7IbolQ>rLxko4mf#oydQ;@i^DF
z7ZpN6Jl{&Bi>qee<Ei~nHm~w$?74q`m|K=jo^x|%BL6+FJ=x!<?M&JvG=G-gl-K6`
z%L76mO`6=%S?F?Yv-{=$XBut(K8l`GdCle0<;xkjZKS(BT}yX*dio#833D(NoIb_H
z$LGm5oz==+SH4up%u`vnzgmO;&VLs1IbTi1W=|Goi+75O`I5K0RPBkuo120>yZ;-m
z+xsNuOqyu7@jPp}s-C#qO&+oZY{JiM52~imcXg|>^4zIgXuYOmfr6gq++O>dldkjk
z`rZ&Ze74-)>vd0rkJ1rCb-DAC*Y_!@8aB?>vzd0VR?Xngyw@GR9_osEQ_n1k2yxBz
zsa83^%DD0Md$~i-{5<#5XZ>GQuw&j?>Fdd#k8j9*9CWYxxpPwV_eJ6DGE9|~6S*cn
zOaGPr_@-3P^D~amCq4S``Oyy1kgHb$;v^reF#dgUd$iX3vp;(hEG}^`UcseZH{<N`
zc=_X;vB5u6-l)5zZ&6Ua)|<#-z$3J|$g?l(gQV8osxJvo|48bb{#<y7(|zJ?rS^-)
zc8PK8(~h)s&;0Q=#wOO^{)&C4nkUBvspid#w60t|{Ho^Q%DbU|6eGeFEdEP~PWMfG
zYsMVQo~Y4c+NhYm+53ND*N?^3O|t|RwrlF0S-#~`_t9r6+Sao(!k?HeO>maox4<vQ
z=Zu{EGNl$rS5HOPrJ@>Y(k~@%9yz|NZvVW6!O_9JSDzlt-27%&!>wCVK4Ic{%h!Hf
zY>=v|`h!V|le4p<Kw|pw=0)r)KeBf8_8wfqk(Is5@aCNzbIlGNWXpQpeB{^>lY4Ao
zj-IZjVeTTDCr{q^Rx~@j)|a)C)iuLO)y}Z>%99okyH?Oq9jBt?=iOjglg2#z^5I*N
zi+4W$d2`RdW5G9EY{izhUwPu<)7vV{_e*p2ibF5E6wQvgF4#A1(IlfY-!vZC`W>>W
zShV=VwXpZo(gmu0H$Pmzpyt(4iG@`{{?gr+TG{TuPM<#gvBD-|ZFOB!_{U?ccR-i?
zm<Wq)ZVEVC@}f#+dVHOo?qy+(TOWk|_n1y^m)|k<k?VbfPiy;EUgtH4v`l=kG{$U$
z{`|ehlQ@mV9vAHD?BAN~`S#3*-}%X3w`{NY{dp(HQ72bd)@v_B^7dqN{I^!>Hjl4k
zG<_Pe&hoj(vs%+WbGtbnd)Xjc=T5US_aBYkF;RS8?JQMk?}93!j&i9bB{p5v`TN9<
z?_c!(_59#acK+ULE54ZBOS)IlU-^HY*8kUQ9>8|i{Y*_do^L7A)OeS5I-h~^d*lDF
zzkQXS%9sCtRf68z{iXK)&t_iami+gmdC8F*|NgOkKYX9_*)H%2DQiEPIZr<7<>SK>
zws~b?<%Fns0sj&aHCNDKz+1P&4+Cy3xZqz|$=a5rW~SD)L9e5)OG#!$_nEe9O(EB>
zfd+pS&7@off4q8iAm#AeEs8CuCoD_te)RWi+$rr<hPPvnxZUJ@Eg<@ITH;gxRco5r
z?_FQH;_lA?3B?0hjhBC~C^-=(VS6~i`{KLvdHk&p&d4>L_M3k2&-)1n_q_ady=1d@
zy<c~p`sbS4k30M9Z04n|U8VJ~pkn>OJ1N^H&6u*~`uo{u%pR46c!?xEn%vQOzgD*U
ztYg%%_vu$}%LKaYlWbKtOPsFC^=`#nXO^xcB|FX5AGRuoA9kso-dY@ce4*ZjWvdF?
zb}h@F?;};YXtq7W{CfQvNrG2?AAR>(l3^kDoU)%^%wO*JX8ZhKWAjnJp6BOnHQy#|
zidtK-H||ZDwc6yD@ACt)Hu20o@#A}bnhhJup14%ypkMrx55`#OSj_c%_3D*|me#HG
z^4(wFFVzs>a5#{!_#%D3+^)q+1uLwW`>$o({_|nw9Is3EzMy4m2}UzLJUkZc+BNIb
zry>(y>nom<_!=ENp3VsS$l|nYZ~2V(Z)(*~X=dDxS?V3l9@hQl;>3TuQw*l_Nr$bT
zx@3vUqeqVnp7<XAANlt7o9A<4TpdG0LMC_CpHwzhd8rR_(;AJgrlUzee@jnN@%+U<
z@oFTY@jBR;*pr+x&q+(_r`mts#ryNe?Rsv|h@jPj76BF}MyG`WRgX^>E@{@sey+^P
z1NWwY76o2nRBbu%`s<?CR($Qun<L}W{&99O#o6neH~;YKXq_#eIEz^5^73^H@}qCp
z)wx^AoQ$8T;>jMOAmDJlG$|=5Ak<R#`TP_^ZqR_(yH7d4-|A1ko&0{c*Nq1GYf>pk
zvQ4^euUJm_r}fi&vdYT_F1F?yZ_6Z4@H09-(Jh*_@AUkI3tn<9NHEmgAE_q5^e3@(
zkCAiV0`tUA*VSaxSWH7jr%VY?{Ig7TVe20eG0`A-4*3bTqInSq`MIYpe4Dy&g4n0v
zm>8dc+m_F?dFGzlq_i#X?xAVA(I-ydciU@!V~5Y;O>0=<6}H?iliG82%3n46Ll;)+
zXf8V^e#hG|Lnr!+v9_7W<Mnz^{)We;{ap6w{@Q1&brO!vng4<H`~RaRt=ZxqZ@rce
z2zqU=@bt&-qRU%q%htYrsIc_tQ4tZ54Qm%PEShB0v~ccA{d*gFE+(r5J>&iJvQ|7l
z=4_Lyc7451>#<Wy_irjn{@8f)-b1g3W#()-Yd_wPdh+?+PPKC{lNC=c7P>Z7*3oW_
zV?^)@evJ-$(@H<9IVTeIWaC^q%-!uy9Qd4<<~=iiyDRrT$<r&B&pT(%ZT6;EIC|Qd
zXOI2&ANtQ8{@`ly{)96}I$OSNy493_%;>=sMxWXH%Z{fn+!X{Gh+EDhaboB5d8ZFt
z_J6!;_WM-dI@O1}g68db9kuMwj=Jq{*Ub4V86HzA9(}s}*>a<L%kbDLNn`Q+d)}5U
zKWqMEEISw3dO!M<_++(BW!XR9hiTv5*7e-Icj2nM_o6a)<4=nx-%s&OR?xn+p~-Xd
zz6Ea-Sc?_E#lB0e^xpZgZS~H7$7b(%do6m$yIt9p1wXpioYiMuQ1UryncrN$%DeOZ
z@Bf;=@8M<5`+xq;=mL!koh!FrV{35R+J{pr`p&I$4!Jl_#*t{7bNk+ulZM4F4U!+M
zU7K|9PyS4^azXR?)j3ZtJD3U|?@zz_%jVVJiPQ2eE*!YFHu@uL>7`4XJ^eb|_^%r0
zpHjK^e`nXWx7L^UrLu{+F5jUr|B{yeyBRK<HecE}OGH3w*S_guxhIZakM~V{`l__%
z>sOv%^0(z?GdXcT{Q7I>lWmDH7uE}_R%TZeyg0n)%QcsIJ3ohHX`RaW;d|b3b6@O-
z>3dG?X?>kxw?LY~yMBhF=K<f}cYL-?n7>Ci`lA_(r_WghTUAC^!HivPa@RUfzd2y6
z->bB+Qk$WP*KS+R&91yni%nTBn$#Yz$VkmZ85<PIk3Gx2e&3;qjUq)t{tNyVF8}OQ
zCSMes#E^k;Vs&bzWLwtKWs9b;@op-fCHT1P&s${~KGO&MH+NRP_`BtF$I^nwMfa*+
z%gWX6=@Oq;-LlPn?vdu1cQYqGVN_h9rN3~;8lm0RZC|#8ya^L_;t!6BaY^-+J@=x*
z=u+^886pn-UnWcu*}QdC*NY!1mx6=>81G&P{OaMzY3t*e$T;<0mCeVKpiAt`_>6vM
zY2645=~A}pn;_@LA>gYr!#TO&8Uw2j-qUx4GWVXI%{R*oG{!%Hb@?LWx2MnV{P1OQ
zNAZjsyB>+xMeLe*#6_Kp%WcD~@V5Ks+2jARv^fTavCpe|$SPEGx}x@uNZ9lr5>_U&
z!sBXZg&NzX966m_aO6l(sMv#7)pC!Pb^G5-=knU7lm2m)l$>Olx^dds<lu)-x19dK
zkYE(;aX!!QvzTt?y+xMdb8LFgo3CH0SaGVaVZ-VpZ??_3uvgT0|Ifb9zsnDceKUGH
z`~Ax+zeS9aOXBMO=pFNk^_=6Zn(^5n=ZJ>ii+Ox5_msrHg?{?i-CulZuCnE_nUg2-
z-#b0y<^D$nJ3IS}AMvIp8m(OO`)T$&iC-rBHq0%kRdwgvGwb;K9W(sh_Me@TDjoUn
zkAch$i5tgHc0AwztW{k8c+{Rw#qak{cQt99Y3Vrq;#jah|MJ>dt%3dbPW4Q^YT+xj
zRdni;Z?@CL=kM%4qVf8^{UI^+NXxS)PWC7-k1P&)eCJNxdt;r5spltpKHc-_T2349
zZtnNLcG`TtwEK!SpHx=1Mb(AWJ5?_w&-{Gkd{5@HYTEYMZMOu{V@rjn^Z%1ym6h~(
z(M_8jiL6&mw@O^IShu&9BVFb$)6$JLj_JRa@K;u!NpaHR@a74>Qy`xH?X}a-YoR>y
zc1>GZr|!5Lc*8}vbFP1e(dIp^9W0+FtXrV4;g+IyVCpZUx$))_5^OcUCZ9VnPwmp<
z&~2d-GeS5rCr;aWrQ~E@*~<;d5@CK8Kb@K?wSOj_6mv*^^?dTlS0ys;Tm9Kx6K~Ez
zJLu<7c7BaZ(eeL<w^N&6R(@e{>#Ldb#)5lR-m_i1=RJD7F?r|DIe9hKwNj;t-zpp9
z3|>e3NXqVbGfDfzmJoyFdyiyKw-|i5dNt_z*Gc>9BsYucDC=2@&OW;3xuE;x$&>xw
z9$%(vziH<Y_nX(I>cj<~T)BKM%R)AtdCz((ewrTZtM1R^)Qb0?cYd9{ucArTs?!`V
ztxl)cAKdZ&+J^_l`5$loD1SLA;n4ffixUDK^A>%oyl#;8=eW)16TUislDPiozdzT!
zMCUzAU;X^=Z|j!7PhM87C(!0RgIVA(FIP(kOAFJ-4t9aZdXbZ(v~RxHmA!LWlF!Mk
zRK*8eCyuRH(Cu+Wg-P&`V1eVJMT}FNca|iqn>X{_%a=3fz1)8P`(68cg&o>LtLGeg
zdG_<lS?gxLo45V`;(O1(|C{&yyG`w?n<)<;?Vf43YEJ#0dd}|r-)w2=)*GLR`4}Je
zKELKsz}jP<OwVsDJ6r!~h3$z`%j4YK1^P`Vi)%gqxO`sn&q>vmAMN#vK7{!8%OtP<
zeX#82gojhU^MCy4Zn!CO?y?G@(<e4$AD7SgJniF$NymSDwAWQz;<Rh8?B$@Hr5|nA
z{5WcqH#hENUu?BUne62V=eC*K@1K1Y+Hah@qP_I|y5*KL9XT2WxIS)nuXuJd-uUab
z_nWJ(dOh>q(Nns=YThoFuC@AWpNCvJeE9!Ii+-CQ&Msl<Yo7;1_1!(X?)A6Y4=K`X
zZl`Ye-r?!Cee?UamZP^~Bk%n>aKvE4{ngExPfmz-mpB?3Pp^_aZL(v{pX@JZOFq3i
z9W8OaT*~^M#gREj|DW0S)t-HmjD5k3ibYeeZ~r@ud6vSpX^Nrs({!XRHYv!h_X|uE
zS$=v^))Nl1e`(U%d~cuK`I!7--KXxd*=e_>DoQ_>ec9vnujYk-tk<mj`S;GuHLsIh
zzG#B)_D`pm+9!T*tI0ZF*CM5nda^w4=-R0HyYHOZc=^TasN?hgvcJ5(MR;w#>SOn5
z6Q7$&#NF$cogJUBJ+<(Q53^{q-R=9D0vw42PwpP(k+tr$yzX=4c?)a#KiPb{U80v;
z7@vm9Sp3Kiv)?$U<a6XL>k=IW8NCDViv{~c9+q0~o}RuWSjIl~?3vl`5@N18Bq(&}
z%{pY9zu#YLuItsB7~@hiQRZKhzWG1wV*KyA{G##E_W8Ci%EB~797=kZ2=SfLUR6>0
z-7QQf)~zyB^zyoyeg+Sc3o0$wzpZKB>V9v@yC<4FZ;N!eeJ(G^dK7wQj@7&c-Ujd6
zD&JIPFn{e7ak??H^6|~dSGH~osC|<gx5C<OkDg<+jc0Ol&*k*<hKaw<_N<iQ;CayY
za`KJ46Rb5~tIyoI?)L3f90eQ~N(=WfT;3EQ7}2mUn&Zs_edEt})S7o*c^wh);DcfI
zx172UkJBrwu4Y;t4)u|ZKX~u?8!>lAr-@3d+WHT?Zt42O)pV4((0Vu51{Mz23->Cz
z=HD&nUpXo8%>FX2d*7m@r1k#&l;8VbEnhxA>A?%XZTq&#F5f;uIREYQ4=1bK#G6;`
zoN1aaSganJpvm`(SHdOJ#K-;e%=59!%x}$KIoa()d-yfM6LXU9T(Q>pWNrEJnps`N
z`|~@i-hK6vvYqj2ar@4Q*{e8ObFRI}o8jxc-8g1%@6`UkJ=gDLreB=<`NFS?o8R4>
zzyI5!v-{$|GyAqJFs}Xg=->&7GPZ68#a;Q*9E_{7`14<JF8Y12?AedqGHde^iy{RN
zt=e||sEE&uLgoH1l7|}0?>+Rlu0K9+@B5Dz{@zb}y4X8IU*_PK&Uk0<tAX<N`%^9(
z>{_*n)$JOKcS-5``Tq;93NER?nefZaW(M;w<E*TPNgLQtZ!-Am-`CKZIKwjh{@k;-
zTJxDc>`Z)UGPQZRZe0EOcWM1CEZcp;EGteWov$u4uMV63{Y>%PH)+$WR1ceI${Y7C
zT*zV{xo_G@<LAM%-XGlJ<Kw-w=#58D33p$D=3}>b$ImyDYeJ&?nRYNMN}UTS{&GX|
zR@7vZW4Ctxn*Di(QKE(K*Q-mut_wT8NzXt;{q!~8*&4fwq_o3rPhMJjI^ypUu4DWH
zf9h`AZJOq=!cTAgo=Js|j~my$$?#hIJpI;&>E#Qb-9NZ>bK0kW=X&>iTW*yWW6jvG
z%|&>w+l0vLXP2&8GogR=`H$0cU;Ruub!bz_s$+dme*C+B^TTY}nu;e3*<tGz8eZ(n
zd|`6c+-S!qt!HOuIH}!q30@`3aP_5AsYT`c1CBx~``$fWsb_z9HQUCw+gIP%`2W1j
zo&9sK>i?g3P&E8O@ydv0H)}5@eDs`Jb-u+cHvjRJPW~Ssnv>t$`NYJp8|hc^qT(38
z_F)s>-R;V=({7)BeWUn?j`6d*XYNQQcU-@AVUCAD_Nxb{R<BFEG(&6B*{>yOZ}zH*
z9DKNUcSYu9y>oM{4z)cu7kClvR(PX5Git^%&n@=_x|o`#tTf0vBXQ@Zi2R%tj-Od1
zjaKyRy>0&C*KT)<KhN~c^6vWHIx{!z(xzQ)OJ084u;rTB!HNC4>!jBjRCPFANM|p~
zpZ6eQYir^h{(nZs=W{_NqhxD<Mc1p(lb`E7c$R<nP-Zg!=JY<-&`TmB;({NtLV9=4
zdtdijL;K&`ylI+T7wew>InBI%Aw#pf|FR2zx%lsV4-T{a8T{<7Npoob-Sgh_1XNw)
zeob)=WZBUq$aU!L{8Of1Lxkn$txW&bvs;7f=G5C!iT2OV{Ii<fFRwCH(s{`e{<!I%
zjkEPadObYl|J>Wdofo<P)W#U=N}aF&IC~^QlI+cMl!Ddw9Qtb?;h_=Lr61!|Xp!)~
z{SU{x2I+-XC-wDI&o4BpITiCS@gQTE%^&5um!B`}&o^{t|LXdgZKh?tT)$5Bq@U(F
zOZ?{tX|a7fzEW$&z8W?E`Nx#~b(SyLFhP15i-z7w_ri=jJ6=@JvG}R*z3#m=!<7Dc
zM`OG5jx3CLA}%WQ>K40GseqQwk;f&UQY}sO=k4Lio~C#3r9ebibK>H4kNd3i6kmm|
zEZo&8)>^nL=8LA@v!kZ}&#u#zcDOoYcY&a?*6JBQ&vx$l@g?Zrj)U%wH(Wh48M_$H
zK3|>{QU5JB^7#8}W{X(6e*UxFF8t}~L;lvln#vZhQ#)V37vJ#f=GQm3x5%|WOTG8@
z{?W%}hROe!?%Dr*Z}aOxZAFtHm)B{HKDOeF6VnbD957%x)Ut*(`u?VXmZM*VMS?P`
zgJ#C(ny$=aaOLoM(~x)SjLmY+*Xwz?rY@07S|;6@8`&^3Q`zcwrOdxN@rvoL-b#(C
zpEy`;Q;ahwz0_IQAfG6?<;d)lpW6$v{do3HzBQ|ONon_=48fm{^Y6Zh66>rG&dqGj
zC={9acFDGFe8CwmN{&Xm95p0d{I^fKrrUJTp*%TvX5_5dvkHEH{`klBz_}&CDz{4y
zDcY~?WSM3DW99X?|4lPIkG#G9Y0mu3xrIIp1^IsMn(w<@=!&VyqMghwd-Ql@SeWD{
z>2kIl?!Et5MKJ%^Q~rlVRwv4qtz5bG=mEzq22J-C3UjaqoVXMa?!HRdTSe&TrP4hI
z75v(p?#*1baHC|ue#`yW3sh&SiEu5_YC2cab?cVZCJPNc&4bU+F`r(tRJCzc<;TiT
zySA;BoUY&Fy}N=>p!+BXTXV*(*46iVbK0IgW4XDmF81n5L9W&dGFjJO&)u@bij(cn
zibL~UH0I?v7ys7m5ql8AT9rO4VvS(11xNdz7j3Qw-n+lg*s8sG^V}oRw?42Payi7<
zvcY5Pwx(I>`TeOEWV$a-@_Nk6(kapsxO>;5%ekIOy@fJ*`ic9w?=tMaKV6t{?@Cd*
zidzddKK*m&%{10o!q0p{B(BA8DBFGV<t+d6cjuiv@Mq(Pw=WA{nd|=1e!k+0^vMLh
zSfhBI8QweBtz5}D{dDW~*MC=qKh5_G7W(&9vpJ&sYUulf>wFPn7P4gl`QFj<*C{e=
zlI(5`*3FxB$?5OvPcmm+H_eWnv_x)crqsz4q3NevSFMWw`s&YF!`^j?Y5$j>+F+dg
zzhj-(H-8DW9fp#Z{J(^FwtNu<?Oa_V_w-{$MsDuXWy_u|y8AZhybzO>)dJ<WON`Ho
z&wAl@@X%U`GkdMPpE~sK$)9%^bez>n_m!)>RD=}e`h~lXo=DsOWBLJ)ct3Zy#7|FI
zOH!_!-+#P*-*?&l#}_3nICO%`+;91TDeYS6ClmB)S*!B)Y`S8dIm<0EQSc|LppNnP
zV_DC&7(+s9Hty9G;BZ}_q4lZeOs&y5t57Be28JD;E{-8Hyr<8bdDLU>>tD`hpa0K4
zd3xKu*OMce&++Sv&yYPE(bi{^wbPxg_~oTd2QspkR6g%_C{$?Psq$5f$%lFIUcsy3
z8vbWxl$&>!ywu>6dC2(f{ij3mld4pYK6U>prhG-;-f=?7%M*pqE-g)-Ib){Erc-W{
zsw`byFES}Sxo{{(5povR(;atyvN3{211ITuKnDFz!48`4n+)0iTQmo<qj$>Vf8rg=
zGS8KrbjlbQ7*tDKBT7;dOH!?pi&7IyQW=a43@vmGjdTqyLJTadOf0MnEOiYmtPBhy
zRqFguH00)|WTsW()}T>&>=Oe6g9gZk%;aRN<otrlqRjM+5(Z}&OTn?AASYG9GcQ?B
zM*(EBz7MM-0|SFNOixN`adJ^+K}lwQ9)n9}aY0UErMUFe9SjT%44$rjF6*2UngF>D
B#-soM

literal 0
HcmV?d00001

diff --git a/tensorflow/lite/g3doc/performance/images/iosrelease.png b/tensorflow/lite/g3doc/performance/images/iosrelease.png
new file mode 100644
index 0000000000000000000000000000000000000000..a160c6700e60726d8d9775c4a1c28b3e34b1e930
GIT binary patch
literal 75374
zcmeAS@N?(olHy`uVBq!ia0y~yVBuh3U|7t-#=yY9&a0!sz`!G)>FgYknVihP;L$mC
zI(tO$NwJ3K(^OPcwm7gXX}Ra}GBhiLk!1;&nUj!*hN23Kp!KE|2ciNtCb)*X=43Ob
zY)sVePI%AHt#N8s#D;~R)HoNczF+;m`uXo?_w3Wp&9QubXXj*Vh84#Pjy%c<VQ!EK
z+^1q@GQm>OvemPSMZkeUWC6F2Ma)VI1vNDx2gzOkgYVsKxE8SP*6G^6cmBt_PCh=p
zh|wWQVfn`;cVtemGak6RBgxZ_p+#<m`Rr->Z9Xl5Ou3vq0-JWuF`n{9@wthryH?Jl
z<j4uZi~$RaEGF?VTzOJx$h|}ILVpx%3cH}cM$;Xho71%)d{C%wv(Zvl7vHqYH0AY{
z_kU9U98SNr!Jkh(dB)V88ZN3H8lvgN%l^-Q`s4nj$-2_B4)d6;au5>JJ|7vkea#&X
z%V5R%!b@FUov$_3_?`&9aeAk>0=Kusmj4H)Zb~@Iz|1h^)MFM`2eXD8ty6tnGFN^t
z<95C=A@S|SibOl>I7_joo<+&*e;2pbWER{JC|2p(`NS-B)0&nB$NADMHQy$jU#8?(
zGs(zr%UXvS60DKDpKLS3Vva00`&o%mJL`m-Quoy3Jo7EtE~zSR^pZYf<8C!Y{F$o2
zrp0WQM)3mj%VfHJ9~OyheCc%ViMP57-_etg*;%zeAL{zKsO8{~jb4Ry32zqs4J#}=
zv%vP+ne%*8U;8R<_u_ukcI?};!$+5$5odq&XGTS8&W<uQbsINH<~(jCMs5~HwFw8O
zu@yP032ggj^5sDMsd)ki7=<P<{y0z<U~Hn~c~QCPx<vrXzosMXmmU^2cE#CgH2l&(
z9QE|7+!F07ZV}~84&qN_1Q<6x%%7PwC8J`S%LkUM7W=d3S~sR0R(t>3^LU8-CBf!S
z$0G(w92eQHAFnOUw-Ji_n^=6vw&4829s6zy@8Wpd;@<MQh40YYm4}i}@=r8AXmNot
znMd-B-rL`A3cO!j73DutyP*Dsbozr|2jt2djaeVm7377l_-hw-v*6hMEXx<&ZyelC
zDxdh5@>*UwU~_1(>^G(LSH5ozI5yQRM)z95`Toic`&P^I-&3yJazreNeZ%G(oNsKd
z^_ViKmGNXB(MxoHVzb9=p2oFf7jiasRcu-y|Iq()A=~pQ;l8f|o?m$Tp>hNNlh53r
zpXzgKhug#|oe_9q%={qlZ2BhGI~Nzv@Z;+%5TE+~@2Ry<d7U>dSMK4yUs8FT@q<qD
zsh)f*zB|o)<%|yX-+j}+N$LGw${>^Z_>>B(m#p6Y6aNFRJe<9Vz30KbR9@pEro02v
zKbjK+_!c?Zc(4U0%w;W@>vh1yK==&v(giX)tl9^oHVEc0S|7;UAaBx;R=_leQQT2_
z2j9CxTQ70OHN<`3`=ZR@<dnf7d(pW^koyQ<p_7e3yoyApOVY#?715IpE*(5GJI}bE
zn&_3HyGe4TYn?lvvigM29@a(Dl^rF@>nHkav;_s`UJ=;pa(sbY27^@NvBi@w@|f`J
zI(c6>`XXx!+uhb{i`6f5zbM_pT-*39ks<h~#0<`~{>lU$9VzdFmkc@0q_q#P+8~v~
ze~o4Sm(VrLuMgTia+%X-Tv)v$?(heeIo;16ZvMzsqq~oJ{c-<?;UC*8^#4tDP?ivQ
z!x6_~mdK>oY|tCvwL&RJSw?A2+j~Fthq4nrQUum^y!&w1TP;l}T|m3zb!T<w@y@qL
zjvuiT>RjT!N%`l5MV>N2b}OfBY1`6W(o@oU#q-yKMXQq5JX#{OXwo93AkDCqK{r<x
zg*@##6+E?W6Wh(EH?DnZpHEyq>3t&lr0<i{PyCAPT_-2Geq4AY*hurU#%j%Kjpdr(
zHS$A3Lw!SSLyxcK3Vpt6UD(~_ReoO=zH(nBc6HjRMShD;t>|8Pd*$(!+gIjin9Nc=
zJAazf>E*i1u3qxHBzM7X#l@>`t2bV)$SS_F>gubj>sIUyQC}Usf_;U5pnlN$Ao~S%
zk$c^n*Cehyc<DjLnQJSqS*$4xUhF@4-OCk{YiBNwT(~p%^PXqMx|60Y4cj2*zuG0c
z`<mz4<0U<LZs`|ye);)j?U&16{Bjp<n`#zq9lSfHdVAUGvh{t_`nvrV`7QL@dQNaI
zAKS4FDH|kj_Ney!?UC;BSLShN>3Pr}VUl#dLtFXY>%J>ruRnb~?d7|xd)NK;>|ZIq
zX8O|YKI<Ld@3UK%zuG>oe%3#>zn6d6DsOeK>VMLwBfYc!i7fM#1J@L;CA1wBYxZWV
zZF6f|*p_tIv`wA2x<|t8pW93~?VdiVQt8XRwY{y!w0rs`?bBZwFweMo=IWW?GjYPq
zlMRGto=lq@Hn}cX`!dVrq|1AkeLt&_CYSayP5VsLnX0qBXOGW3Z?fKaf3oAY4;us`
z)@-uK{&?@wDWj7`jj7E~jkRO7w}nm&oqN?bd)C(ZrSnRAznV%f>TUO(cD8D^S@e}_
zSHlvbbE|LiZ7tp`d~5CYY<X9Sr<PtOtJ1FcR<EjUsBNsB`}gq0o{JL~ce|VS)L9;@
z@i{dkHThiM&sPs#O?bQE>ce%5rzP)5-k3c3akhH2`nuq)mrc`W6?c4=dG7W+T(3;8
zW;(Cl_L!IJ9><99*tVmtu(*8kj<<!c%goFDOXHu{U3R{$d~ETd$8*#V8eb^xOW*w5
zUEh3t&HBG__0_kl(<|S9sWZElrBnar+Q$A`pIQsM*|k<RGxx3BH@n9AkIx^AI@7xT
zKWG0`{=5BWKHCkJ6t*{96_PImG&pTUe$0D%bk<QTxfa$|wyCUJ*|J&HS=RH`2_#8P
z5)2Z*B>hUtO8HBLgrbg8PUH>WZHlu`uS@;jw!8g2m%XTCZ%228^hb?LW<f$XP55NR
z?ec!^s16hhT(qJsL*Q&*oLsH_F)cOyXZ;_Kd^CK#`gr+Kt7GSSY^B}%XG?82eivJl
z@MqGG-JI7rH?>T2`Fg~&b8F}O$xV~ACR?fX%rwY8bTs8}N@7tmm!_&=YP8q=BM&;Y
z)yw>f=7w0kJeIQhWb5Sf{uk#@vVD3Y<m<{gE7^lMS9-3r4z@2|WNA8QpGn!RU0dSx
zx_5W|ZhC$``rzwlzdvaI^uJttN&E8r)E_B2=U<5Z;9SLG>ol!k?T65`h-b&ne2=^s
zd3!V4rpcSWKU?u+$uk+T%+5_N%Zqx8xQlNeEqfGH=sqv}-l;uto-HY}MJD=9GkcbO
zxc`l}QT9#!P3^4vSg*3mFH`li4p<SXp|!{4ZrQIzFa5iny6R>(o_}yIV`ipN@x3$c
zo0-yb=k=cPn|;^lc#hx6Ip=mREG|E@_{`~9-_tib7p8u^{nPXFY%d<~XVYx{seYNh
zD`;zw=gYIK_qfgUj)-%KZ__J^+md$ix>4NC)}r=Lt3I6<z0<|L-|FA4U)LLtzU+>k
zzC-ushABHjwtP8ev_I8a%T&8tOI}}o`<^=1V%M8p+qIXkvx$Eiw6#z)b!txN@mD9V
z_GX{Ct`qyYa&5xfudfcg+WVU4nr_thI@|oeQ+fYMf6Fns6S42(WwnRPqGvmsU%wu4
zEhJYlPxHp5t)_*qZ{D}5TYY!g{<Sr?r@vkFR@OSHe9o%U^=t2DZYcJYjZ$Cz`krZK
z`0aNK?)Fx{`JQv{^o{6!{BI_|t^Qa2>b?0l_c!^L4+;~`J9am}=DWy0^{VpM%de}Y
zS$Ji8PxiiTKg_?~q|#jWSMnG6+p~RVt?qvz6InE+Xv>oq2R1HBu04MAc(q^3oQgX}
zTc5r=$*I16u2lW1uO&xb7N<L%n`0wg@a=}>{Nl6E_nwpA_n>g$!$r4`{_<Woo$v3<
zU(XkYZNK*G^t<WOw{33Ex^;Y;{N6|9Pmf$(usVI+#`Up#<VyGcbvx7>ZLWUz&5onH
zzrHE^_BOnn>)n_8bF2Fc*1y?z?C&aeHvX)Csh`*HmCciNmt(Wt^F`y`#k<Gv9M_-c
zF~8=W&dWump|+-VQNPc;j4UqqD8K!=^SpI@NBo!XQ~oXed41RPYx?~E-v0l-^d9Hl
zx$9Ndv&Dt)xmdmV@7mM1r|p}ukNthoy|4TAe=qo+c-+0eKiyunw(6GsTjQ6XU+J60
z&)Vl!v)6orv4O%f4~s5;MjkU0TiJ%|w;30x9?ow!W6%(unr_^xaO(4trUUg!Pk;I<
z<}u`aXIDtnW0?3sI+Zy`Y{JS9djDDazq7B{{JB6{By-p1$(O`M=S6VHv$5TiU~<S}
zdeEJcu%Q0$fdkXm8r-<eAag!=9_Ldf`{q0;<>lMXO@HA2(>&u|q1Fqopxq1%3<}8}
zLB0$ORcZ_j4J`}|zZe)88eT9klo~KFyh>nTu$sZZAYL$MSD+080|Q%<x4R3&e-K=-
zcll%n1_sUokH}&M1~F9-X7mhF`2gyKlz95OvOnSxVPO&ZlGEbKz|g?p>EaktaqG?B
z@&=i!Ggt82FJEP~{AA^liEft9dnPanvYJ`mOxSS!Sa<o}ZP%r1-v52~?(W~W_jW&g
zS5<pGTe_LkjLpTsQ(~f{!a^OzDVt3!%lx0OTlM+x#mj3Y&uS#_N;RHO?w_&Bs`Om+
z^3_$ZYR^j_dH2{>sMTqqg}2V>Me3K%_7q;-G3Qu8_u>Ez5w2FJg#j9E9JxDK;XII_
z;hz;x?Lm6Zd)z+vui#LlH%H^LmkLJ;x|c)M{a}7Ng^kHKrJk+ZUZ6pDv;BNFCypcP
zHJ;Ogl%A&BM>%mYIw`!E`#W)mp5mrH)u3C?|L&4pmi=$fC(n6H&!09fS*V;l`#b-(
zf(6En7hfM%RbArK9q8EnbMdK<m3t5CE(5tsgsb%t{{%-~mRSvJeVcN9lP*ZGNY98=
zof)ahF2Q;3YK20}f~6ld&hon3U91$ns;s$Qs?d+?T!FOOk|`zS+b-U_<e(&X{PEYI
zpg7+Z8=eTC{`xy<rrPoTH|`8qOOr2r^}m18g~M^j>gHw1?F=i{$hxlBP;p}F)rbFT
z&YK6_+F~NJc<oZg+je_5x?eW&(q5&xTE<Z2`>fpCN0K%&K7O^U@AzZIi5>}&*BB&g
z?Br*ay_@L2EqAhq%7+S@=U3)W5xQ(*$di0xp5x^+1sanU9uIYJKjp@-O!uYFl8DP!
zJi|>+h6cMm`1-Rg#ReP;Am8R#E^SPRp8VipFO%<=@NZl1l|Sj9waI?loBx+`n2ohm
zFKa)2pQ<9@bm2}x%d|G-kZV0oMOm!A9E}baN^Dj~goX6~**kUTLlYl|u*@6ZKZ?!j
zlRqY}bKvQ>3pekby4YO3a?Q%*H&13tv^}14%wo2sMO6Vm`}aDQl*t!+<?NpZhfZh}
zDA{7RvG(%sioJCVv!wWpbH62Zoz`m4d2^@s>Yaio!fRKAhxFSl|DCl%!0E!Jf*#rE
zYe6AFjfob&c5eQj-K{X$Z)5W3sAXrGx?b%n-ghrT=h$R_yUyI(m8^HS<(9-gf3<pj
z+SOH|5!0@|pRp|SPSxwREKH5XQn$C|URx7+c$R7Qj@RpUuUNVA=ic94Zi{RFd^{c!
z9v&PVY@BjTc=<zxq}?}#&P{k5_~?qvl2|r(qrycSC4#(a>i%2y!%}9z9NlIq4waj4
zT4y!fyt?;P-TQsIy-NQ#uUz>5bkVsPAFf*Ryo&wcxnhmv(Hkd?4UHxL9K6sVUaYBl
z(L>_!#L!u9i%Z(yXU|x%!g6zyO3Q+uKN26@;St`Mee?TAv0JnHjce}(adn@*P%3+R
z(~DObPamI^XD~>-b;tkgWCspMn`?(BGVh*YP|BxpV|y{TsHyVDUrh76n&ueu<=p-t
zH+Ab`3G+%dhHCqJhqm#)H>`c<>YSw_p?B_BvE!stTlrIlx1*#qv{oiuTjYJ=S6cy(
zL`;(R$!GlcmaN^Wm_5V#xaIa->D$|O{(7~VldYM-MM*GhZIr9pWXq=;jVrC@oZXRq
zZOy~$`~R6HAMdkGPj0^a(&c(-+1p#K0!^=8y;7`XQ<dUaeEr1CuX|2;bgpbH;7FR)
ze(jpxD&5(K0v2pPdUK-Ra?7G89CiEWKYEmO>-O!HD^@I6yS6vQ=;Vos%I72Nj;gE6
z@J;qxuBodln->TwDHaB3_?^uBdpwuZc$G^0Y;D80YZw!}m2)yDFqibcF+6zeF_Z6{
zthJxsO!PS8)8?Cfe?gUt?N{MtU!!wvHdI^ly7L=UKU_NJcZT|k6)TgE{pmixVhyKn
zvi6+KN~$djc5m)o_PBkfUB0I3)6knwQiB#{N~t~#^_jP|Blq?FrJKq`T92mApLM9_
z&4sv_JbqEr(uM~w4&Ifq-Z8x}IGz7zM6)EjiEg#^GagpwOqB^iL6O1kS355L{q{6P
zGDmuo*YBMgYtvuPeDs-JGSHrJ`DN3T`Rj9UOSCcja!B>dN?oZ|dafE6YRz@HV9w^I
zd*Z&AIart~l^doADZ1EwQ7Dh$n7IG``L9*?>;Ko*{e2y8oOVXy-M-&>P75zsetlWA
zbB0Z&k#*Udgy-kxGB_uxObAN2oe;UVOqw&#`LKb^p6ip+YJ(*%aq4Mx7ao5j#l_TI
zEfg{B=()MpE7z@CwqgZG!}EF7edhNnn)T~{o<8!pP_E{KW8II#@|)`ZR?V6<>&LI_
z`~9k`t>3<V>odb3(Zk6~B>mam@An>mczF0u{r|tubyIv6g3IeilZ&F58`Tnun-!9|
zm%RC^^TIdoglAQY$n$501x7($e+~*<ReqIeQzfuo-$CULtAjy^$K#95A^j_k%k$K3
z3!l-tVDD1K`V9)J|IA;q!1MO94Lf-1H#B^k#>VA+YP00owh$J_HR?WxCo=oYF`M3Z
zEsepgclM!mZ>3kPly25HDEVBI_1Ixsy{~cG9S?<;1uuU*+%sv8u>^z3*Q%y3ch7CL
zye`nZ`IFDYf<GVZc3Iz^VN<Lqw?6;Hr>GjH$2S9{Zuyytf2^2ee!pgNfyJ5G`TI_O
zy&m76&yn<Y>6FUH4z;zm@qb^1x92Rq_WG+)>ZvJ;ra~#b%uPKrzbafXjfhK~u%#qS
z{;Fq$*(K2wZ=F+5j>%2y`>Mni_~h_IgZ=-$uHX6Tly>B%6i!=P+coj~=RG|=eRIM=
zCO%m!7Zss{Cce{jqus3Lx^WzA<CTusUuRqYExq2&(UH-K!)PYY#|oL{mphL?wp{yX
zS*B3;Q3D?4_19<5@AgrfeE<FDp8`$^kz&g-m0A?0dOiGFb<w0ZLZ?k2>A+9*(*>V?
z?yP(K(PE|#+x+wGhYNpwSRJC}sx<My>#vVWtWKsF%~;lX`>h*?qt#qDug=p?4{gg`
zbWww$Zoj<m<q1KPyp|g9G`lD@o_>1hWyvHDl@lpOKg(wNsO58XAAel2_uihIlPQxT
zbf!&zIr+?XNOgMj=Of|N1;Qz%!Px?~hunC!-s=)*`o($q!!M?qEiY@ix;E+masT!{
z<<*;`A=mn(K24qR>~Z@|HIpS<{OA96$S{$&_o@84bLVR=1;PHuMb#HdYcCj<#vXtA
zZ9>O!{}6KtrzKNlY`2K_pZAZv%2Aecs?Ee_?%odN-{nPL9>}Tr%(hlZekvrCcmGS>
zJ&Oge`{MdVE-s&|oUd|Y-^<!G-5-Z6)%DIyn|N7Z@BI9C_fD=|@a?YM&a*p<)Xq=I
zP&wm1`(FC#UE60^rE@kW9GsxG-{xlZqWgun|AqAAY|FiT`SRrXhdor9Zof@?d1>jM
zzu#`3nQ6>^*`(7&>C2Zdf9~Dm-j@6MAiMmBACLQuv#zXAH066Dk)_<2+nniWHQBbl
zZ04~W6CM<>WbKiv-tFMUp1Ra^i^`!HX>9G6ude4my(waEmFaA=+(nx=3u~(C>gv{f
zx#<4o%a?$-IKR}?)PMW#@2NDNJ9qAxS*G37<LhQRxAT3x;LQK=Y<}J36r+=q{q0Up
zRCfPRVG~pT_v@J%hRFdpZ{NCA@#&<xWyJ>tzV_zT>-Wv-*55bd+O;sb`agwza<*OT
zUdL3w-KrD2Ys%qv{>@oewdVM#SATz(`s&Kckg%{%=d9mP@KM|O`JDAk+iJ7i++3Tl
zSAv;SENXrj=<oZXG%YnTQtbQPa(QX<yp*r6uBuG-eD&%TsCxY*C!)SN{k+_~x?h=x
z9vY;krrs$$F3Vu^@5f_Mg*;{I)Q>L~_ZuZ2<5}iA``D?e+BbKVYH!OmE`Jx3x9g?a
zq?0QAwqFEPRaHS@AR;af$}IP)-`h$_OYi*i>GaNbyIxnk-}~L7<b}Yr)Q#ou<Fc}{
zc6>f(t*NVfwq1%-(XVnXq~R%Y_-tmackaxysvo}YH2Z$W^6&ZU&u=}SQ=SmOXxi(s
zqq}KQ&@-KVJ(k`2ix;ez;;|#!BR2W->Z7KmCbIS=r&7vxiauQSxP8TnmDb<?FgF~3
ztvXd^##P())W{{1IBnwIPmpGB5pbzKWFl*QWW~nGg%)$3G%ez7pS)}1<V>r&85#Z+
z8B<afGpx!u)?c4*&SzpSlXklO{f5%NyK`dy>qXnuv_ES*{G#fb$AhoGHjDDLoslRp
z+ibDLSa<1}c5R+-kGWZ9?o5!>o8I1=9jAQhu3^ft&N<(YJlmSTG4<~4zjbEoYqj4S
z8cun!t8d-wHBno;rfP?8DthV_wl=Eu@WTZQ7BEadeboB>p2LUv?azQxi1zwDlkA^Q
z-jsZtukP#W_`uN6qkF&K^Nx(XIU&xwZ_dO6@4s&@e(rbc_U-!j`_{zqGi?_Tv1D2N
zr%5@nW66SZ>jjH4c9_U&mI_H4u1NH}UOG)L_SBJ1;h&Gh_n-Ll^K;~`5=~(>p9kIX
zf0RByJ9}C>wnf0i%&h3c14mE{J(=V^!>Uwk``t3>yzR5q=T$U$s2uvf@4M}@^YhOi
z=C}7rPk-KEn0x!#N%i?lHgBG6|M#-}Q||a5O#1u(7|F`YnwXgd#l)Ne>6oS)ee7|+
zy`R57|HX_c*W;>tfBmZ3_vfkp=TE2gH|O28dbjs`+^SWpD*pX^?zuF`=I4{i%qbl0
zhg}v2-m7}8o0XjnO4b)k=eD)6?fY?5|I^*~b<>lN_kDaeJD<<$ysfRR&4&ZbJ(9+4
z@9Y2nuCS2<Wtpz7t`EQe+Wh@;dC#|7*)rB;J)FX77p`7)Rhsyq|KE##P|~mX`}O+G
z?fLT4Qa9$^-SubVv14v>l}`lM?faEAO+S8~xr2$hIVh*8`OiD^_xpYS>T2urHlKa|
zpTD<>ZSS>%&!VCAX-%;kKa;e<+c(^AC2yZPpBJ-q%A`-L1VgU%CB1Hc&%m?mk+#X&
z`-QhRoy+|i>g9E6wPjU-eyr@;pE4}9$6x1Vt6dXL`|~8ySuRdt>5~_YcYpkG^g6Zr
zcWn2Ysb`<_7kxR9r*?M6|IB)x=bf8B9{yocW7_uE>-@i&A{~m{#-LK9_UqLYqnUNT
zW%<~T9z7aU`}Jx`X(=dpHl>^t`cr4$mN;kf>e9T{pO;fsoQjJQvuM<I&tRF`G;fPb
zxkIP-ge$7%eHUMPdEI)%blK|bTnVn$4?k<BOqufFYt`nd{8iiU=9TZ?w)1Ls>~&2I
zjRVg=2Zn{Y*~q1zo~GL)Ydx*>^|gzaFFO|(8}st=GKW2_uBy7Rx7s}Z|F7#SSFcV!
z(jj<ezWsa$g@)sg4{p-gmUGj|+namt+_@@~JsGmHvu9eD>&2~~-YKkZRQ$~6SdZl4
zL#^DEACHQ^eDfyd^|iG<lE%yC^BnJ&XHR%?Vq#3;QPCEsn*qyHPfy#JeqPQj|DKQ1
z#06{D=3e-jnUnM6KqIryJe!$Yv#%%JHnT2&r!v#0=yvY*kNf{!|DW{v+1WoI+U->Y
zI39eh+WBHpw?)ANhKB_+eAJw)tF1#rLs^&@#q{HRYHMwSm-{W<ym_*uaoQK-B8D?F
z44YZG#hz%Ft%=z=$wg`6{QvXj&2wB|y5VOSq~mOHzvh?gyYqs#w{f;{S#GLVm%d*w
z+wkt3S(bb6>u{&Hmlv&fnsC~}?#kbGF7Fj<Rz7}wR^FiU+M>Epo?W+R-Q4spmuI=~
z`u}%~HqGx5X!0mmKg;Vr!zx>_ZQ@zx0w=~v5^V=pY@GZv<1_n}J}LJN3tr6ORhqhW
zaYmNKnS1@)4e~zuXsvqv>Er*}S&5O7-|F_XR@ltqEtD{NeB$SK1%WvQ`sd!h{A*vJ
zbmYJ`-Df(Hb}EW&vzATtT)tuHl2v6Qp<5S!`grm09>c2D5-E3Y%yll>+v%bdf6Cyw
z=mhQJimRdFA5UqopWwBWqy6xn-|u#3%$lZO{NjJh(!b06=AMczzk4*f-}YEDzugH?
z5tW+ybV7`K{Gy8*^0i+CKUT~+Iaxh8DCkr5_te`0-i}txfy-6qG%?HGe6oIO-3rb_
z|M&_r5_bFDzkc%j$(s^}zvQ-8JNwkw#sAx1ey>vf>C>mpCvuY`#m?RP?Q(fb&7Z&d
z^}nO9t&2VV=jZ1_3z@%rf^6jc|JfPvFi+R64&1d)xy9n4r*Q`;$sB)K@@)U>*|TQ-
zcwYZ6-K|gN<co`o{b$!JOg{Ob#LAF|`S!Mz>(-_H`SG#RdCKv}igW#*Z#U1maR5{X
zuZi5u(dxAKe&>_avs;Coo~&0CKQwXn^_ZvJ3iB^7$p3NX<+@Nv0g$v~=K@}qATPc*
z_xv7TZ1$PE*JA#X4P{p>zP-QYW^?WEjLysLDQjZ3{oE({e0$vwzNewe8tbJP3vNAr
zE+uVmHhoI|-`zi&S$Eu#3p&ZNah^fjWzX<x?&`9fq(@&WZ_fJWyg<iW)3PW*{9u6M
z(WmN@7@ke}I%iQ}Ox#S<OENdkPKo3`UwI}V(eQGbe%kG}?+k1znk<dEv)}x!;PySv
z2kKWopHr`3zAg9mPnMRQ|BCeA*KYUEE7p4w^XIA6?(?EWPM*^*T)o(2A93KxOz(CT
z&xKPaUFwo5%lX&fE5D~P(PHHqNl(v8Q`xn?TTJARZaNbe$+B$b!p(bUY8s34UF51P
zGqn#tbzxiXtl6_GU$5Q%;@vy7$tORw>en>(N}Cs*RGpr1yX@YA55Ld6Kh(-C==9;m
z;{Ip7ETN&H&ue5Yi;gvBn4EfYa`O3=`Ht%w6*L_@{<J;lWj6cR!=k8Ba`;hHMz_-i
z&TWscxbm6$9^RHKB`v*CN4x(1GmD>c=l;Gcuz2y}#e&V7Cofv0bgW0xSwY~zDed)3
zwr-u8dwU!2EE^*M4uhm49N?-XE6bwpkHz|ZzqGdBud}u;f0y#<$;mrKr*-%I|Mz>w
zvPv&WzkA<)*8JQwMg4R^(eGo&+;;8S_2b_6eeN0}kM5S=w@v-mb?TItmzS4>X_iRw
z^K+@!*Tt5Ul`VR0rN8%!klp<L(|WsmK(!^PDtdEsGyDGg=X1B;b@TS-*5CW(Qp5Ys
z%V$?F%gnjEYpQMaw}8mVle^#V^DZqd-T(T-@4xA`_n~c}=d+EaI7|*$bh}nLGy8hj
zdG398S+PYyit}>87ly0KuU4*+b^UsxO5yh+vHq_Y3cfIiUM*#g>#D#1vFPSA?d;7b
z@676xJ@Z_R!LRyH%yDxbcHw1bK|SQRy1Xin3cI{cN!VA4#nm_1TAOYB{#cj6XO0>F
zT?_NuerAyg)@yx_&ooQt+?Dra{lh2O7fYLWZJc~_Um@?rcZDrn-sg1RH%$?l9%@wk
z&b3*BQzAj;$ClGFVb}VkN~F*2J(E~=C$Rba`YBKTg$J9Ld7PH<($rkNCiT=B*Dc#k
zE>ye`c(~52rv2NcyEpw-Y*4Wdd;ed?PdzwIZ^@cfWhaEhzUdZAbRWI3qcGWS%>ff%
zEiJ7b58I?aRM@yEO)UPzCgijs;b7CAi4PXG%Sjn29}JLUFNzSmAtu9n_*k;p!zZ08
zGjlU<T$_^noHzCMJg)7x&%U|2x$@u7=bBntNe7!)MMOlNg#VD`V{dJ3&6w48{k5vV
zi7LfjX>-1Y`G4P}*Zlu||M|4&yoU$b<)7sLdzSB^B6Mb^ar%X+m4^*>e7zR^^QeB^
z!TbN--WTdW9=Rn$aOu*eF-0d;Z9X0mHZd^?h=}mW&DEW?>|u%3=cz(10xeDpYko9e
zet9PUe$D5z&8IGWuW2w$K4y@1M&jSw`~P}Zt<t)aXU@vX+I-Hxu+VV(og!|3`@bfj
z7W>JRNw3%MpEqrqm|5PP6XE+liE3$S{g__=Q~cWc`16xgy>D#E6t4gOz1}}EQc_e@
zbjRI16W_%vR%pBreDMAEZ&g!WEv-}4ye&?O^J>3E-nxDJqDk-Xcf0vdRB5v?G0yQ*
zH@{zF%y0YUg5+~}J4U%o{=MV^-WO6fyF4F1^liF%b<)eJpL%|>nke49rE=qr!jD%n
zvv;SR+<Yq9V1c}E^6Ova7h8_O2hJ9Jy}fv%LEGVj`}(&hc+ZynJni4TQ!#ZhzUj}J
zQW<h0zng8#HB7!2)Nol&e$}s!mfzpku+*&Ilr~GUEI0Z*s2WRrm80e}yF8C;>avaL
zzuk6OMbF&-h0DbEeE<FJZ{qJ?-Td|aKCcxUS~xF1pJycfUw!Yj$){zRobzN(^nwN?
zr<_!0Pu4gk^X!=2<rzY4DSqtI(U(i@ZvC)p|5khE#4pYs3ERd5$BPZWpXb=BA&r1H
zORqZg&_FwU-IHgt^UodcH8(Z=_^4ZdQ_0I9&`6~2_B&1*B2ruazVDtg>Few3pI<EQ
z4~mRDS^NENx!*#DPM4;&(c6W3-DVo4o|@32w`4MB-M($@wI`FV$!)#(+hy*Y9ep{I
zZnP?!9NT66<JchvzwZxzN*c}Nk(HILWOh&yI{5e_s5<hUoMGanA)++VBjL7L-F|sZ
z)rlT06*hj6k!DL)E??fhZJU{m9DizRs)>oohQz~dd&1kNOuBU|>e>1E{zfyGY~3ol
z{kCsrX6C)t{orvrPft(InLcd^1|6x9M;;e89ZWD=qbYRoWyzzm-JZ)Ys|a=e*|^W>
z$*;3hna^xx>^S~-%eHNgzW@GMVUuDcxh?m$TXAvmzTQL$ov1AxU0qyOR#q?Gy<4_m
zLBpa&N|#=mY>en>YHG?+y5qs3$kOQ0ux_2+uHCye_4M308ZW;*@~}Xl(}jbD>BTNT
z9kJ8Z%s00ATF5NhuwlY+x$2&8-^xI({$0CvY3S*llTH=tWLcK^Y&-kzFv#$C*Tb|6
z=0c8UmJ4+ggPRi$f7~3nf@Ri;dC8}9c5N<R`0|IMLiFsvTb{BBf;xZap0a&@EN9&@
zCDJu_gHc<$Royxd&Fb)Jzmp~S)V@iv7M=RDlb6MB`s@sqGv9c#e;r-&JZ?krLJ`X<
zrMpw6@Xx(n7VqEmcJ@8>OIH~kY~*Bv9v>($4q3>x|NV_^sez_RCZ?sy9AYBRZS&er
ztti#<(<`2FT5?kH_IK59Vou3?bLBX=r}DFa(}#1`?*+P#o;&F<;bh8%(%KU}%$7Rb
zZmlf;0&gk!h3|dl$howxQPjRRp0B;RtE=lzSv+%hqQuXurR%TX&Y$PBFkt_-MhAuU
zdz{Sb_N`d4Vt%8-booQiKO1IUSy5M%G0QANX794hzsES)nt$G(>z97-cA|k~(e?68
z7f-zY8WbKr{r)-4d$p63q}nFNPu17cTef4zj{94m7IDnc2~JE@Y;A32UblGtbf^0T
z48E5?y)8brVe)e^i3ttG8?r+otyqaXwyz2oSaNcOa<YZ?CNih@>#`ba700pfZ+F<c
zlff^{@4Soa*AvC3SPNDa7XM{hZd<oc<iH#uQ<0OBXMJWRw|~1N{L*gqw|@>Dw*nut
zT}ieN-}I1A_GVC9O6;F@rRN6{9@ktdsyn>$4*%D!&(}SVv6SX-nm12wUd^YIb)7E7
zo?Pqq{d!fiuW{e;B2TV%r_4z~lYG<)cg8&bvc^O6bEjDQDv=bmsSp0|y?gdy1w-r?
zE6%68-c4Vk^t(3ddGCj}sjL<;`|DfTkN#Ag>63K3%s#mXI@p!o>B!46!Bi;Y(d6ev
z;!pN)oO4`}81#fqy4*pu`LfJ^9VL^uYc{*pMCE-bP}-toeIdQt&;I?K-o^vla+5aR
zU|8P}qxQo^$xuNt@!akQm)H3i-HDiXbYWXr-*&eNm+an_uTe9}I@Q#(O7G<H9oo>r
zaFO|9%S{9{M7WM7%;{}(C}dlrBJEIG<?`;n;-@(9SS-kOe(hDe!!<;@KKBZwaQkS%
z^osM%&ySCTWCv)gAidQHWEzqf8xs;oLnPtGw$gWdHAK4XC30?`n*M*Y?#E8;xsbRG
z&=Be37GAt{ZSs!P01c7(iYf^lvkO#w4qXWK^n#i4sJLn4{@NFdc>JbLj9$slb@VyA
z^W+|_YZ@G_1(rNtYY;vY=D94l%PIQOwvD;F@7pOlWLP;}{TQZU_u0cw;*W{1`{l0>
z#XKSHW2c1yJc7q2d{NqTtg$VlrDcM|-eU}%D)Y=fdV7{CbR2#v_orqn%gTTnHGw&|
zf<54xYPLyAF*7F2l)U8EY?!<%!#jfGz!ufciZ11~Z!<Rft#Vb|b$GD}56tbLu+%eX
zJL90R^hl<~i*2zvug*>QtXPo6XT;b%LH}4pe@eZ#%-M&3UEG)yS!OL$T<YT0DZq4l
zo8YXr2F66*uFuT^Qcf#=?hmjOyr%qezI(s~>m@=w@CfOWVdi0R>M(xBeM0jU^S8x}
zZrk`bnKkrGNnx?DZ3--vvr0Oi$$PwD=iY*@!WK1$c>!^^r203d9#;F4sw@#5fAC)I
z@tJnTQtfuPJiKoto!q3lBi3zyvB-QqCyu_i_g#xE&h|(K8<;#5D@s)KJd&(3S#pjX
zf82KMdX>pt!rm$uW%K^T@oU+3fIJFLB@L5#Se!U+-M)QufBk>W@K#gb<TEo2r%ai`
z6L4%P?=;;FYvS(4H{HHqz{CFDZ28X(noV=BX!^QJG#_~UvFOu}l@<K&_kXmyI{CR_
zp^xrL7yk8sekqp8a&CU}OFGlOuEBg><=l&37hC-J7*-~0K687DqVMl*o8MQMd+YC%
z*zSD3TIxxUY<kqwcelg5toJ{%dhsug+2dVBp4_LmIo9*T5(6attnP1n`)1$0zsrpd
ze3`ucdiA39;j9J$(<VP#n16p~ZSDHT$tf?gbxv(P{r3LuOx@@{ml+nRLVVLNuJ?Z$
zYOsf6X5!=Dt0T96Wl&Z-_jH%G{;_0^4VQDO-kaLp*fo99MdKZrM^%5!xiw)aW9PC3
z>Z<mIA`@Ad^cwGf<@67Mr+VjO4E#*(hY#+n{hg7OCH25lVd;`hn<jM%tMlEOse5pf
zLUP(vpV;Q7p=_Ss>0blqwxl*&x7>caT=}NsR&8muf<^VSrzN*nJl^?yrc8NI;2ZTj
zRa2OM_Zb$RG(9uhc<wjn=mXO-c1>0dzklM(l-6dR=xcip?~*-x<?8id^G#Ci3ol>2
z<`iA6uGc@m>(!|o&uzgP_hNH(w`?zeR8XzCe$A1`WlBqZ!>hGr-_CvEXa00*Xoanw
z&_<4n8BCe8TMh?ud7b{avouP2_Jm!NyH>5b_xb4pBb6k<s;WmGw^gKk11ytN*wohg
zo;@--ldH>HQ&ZQfb&~3aw>-JWr8&PJ;S>`Kjtlg%x#MdxZ^?=+BF@)eyvsN|#me<)
z>k*~9TTHhnc{ygfsxF)$uRVF$>do8>IS<W^tzc5@>UP><Za&##=9I8dL7s}2FW)KU
z-n{Wr<)Md2y`y1j>d`Mzf)gsQ23@*(t?Jt6wLeS*6;xNI%<)h;w9wW%bB=Md;nY@_
z-O)F9R_ouL$nh?MDf-gyNWQZt{Buoz-egzXRrW2G>%gXmv%7iT&9zSn^oN#kT&<6U
zFTKx=@$%FVNjf*j5;S-R8rtfW<zp{?c1BTzi+9zJo7)a-VR`8jwsmgoGD9;J3z<f#
z^~dh%O$Z7Ij%>Oqa@fqbWVvzmCV>>8K)Z^suDh%b=S2Qx(wnJraZmTQZGJ-U|M0a3
zC*Qhp=-xA_-PiXYckeY6J12AY8gFz*nR?~FiN|+*{g*G4f8*@t{Z(Blw~iU<s_E`+
zlFo~2JbrV3-fn>n*^jGjzQ3NT;=92p)6FOL#M}72Q;*%1uKcx8-Eb+h@zjJPUs`K^
z&a<}I@wIkd%^OReesS%oF5lZUICj0vd#@trZ&3QP^iI`fu5bB~lizOd=P|$g%at#u
za=yA;C7X;N6KiVC$9vW{cm3M_=En<zw&3GpdiyHAU!P%He5j#v+gg$Ld%wu!|9O{N
z5_|a2$)(%tW)|g7n%Zx3RQ%l!smZUJZ2m0%pK|4pc7#o}iT}SpjJK?kIWiyWuY9^K
zzgD>a?#c82Ub-*Ikv(g)aj9QuuI1acYgMWih4jok8T<YI(OIGAH{`uFi}7RDo~zc}
zo&A2_bk@(yo__ZDGW)RcneC7MEXtJXoX1!8*7nGYQ%-$v?|XZP%fz04)-UnuMbo?~
zOAK{1KR8`bE!x%<e2;n2_c`w>zU_xZ?!OHagC_YcFSMGwoVR*g?(DWUwo5NfN=r*$
z%-?wFbdb}fEt|Vd8=S>0z8deJY5IMJitou2PV8?Dg_rAb?ED?SJJoyEVl~w`=IMqT
z&TmoeT$WSfD^QR*CwobO-?Cj!#|!*dCviw@h*4j<WZ%Ad&FuV}vaheJTlaFYd;hY1
z`~FQ?eQ4U;uL=d+jn^9j>kdvdaJdxl{CY;G(;*f`Q=gT}OK+AZ$mH33hu=A{PPTnU
z(bcmtwMUb7%~Ic(e?4(oOeOdAn4K-hY+rAF%eq9!u<DSYo8QdSdydL~{`XBS@8>c3
zz!z=l^Xu5+zlyxuRm;k5w*FyjdbnZYF_ULLlFe^Q>&2Y}`b$2%I<>rTTHizM^7}rw
z5B695+`hi>!Hpo7Qz@q=Jv|d^GJmnP-GkPt`4>vwhDJzEj!-!`)%@Ps8CmlEe@^e8
zq&eg8^?$Es=fr#!k+!?z@p;kAHFe)^G8*~JG8MjS(&;X&v-@qBnEpK8GMVrVP5O0z
zy|PZ#toifO|HigN<ICwj3*JB0GwGe?TwVEi=~bWIpWO7KOy-u)|IM3k^V#P9pVj%s
zg-N{Y*VlB#ipmKY9XNFK^T`sqMQLY5)<tb$5z4jsHktj#&esN;kNtY_?oHaEqv6G0
zpK@3H6SseO;3D6<znkq<Z*6E^UjJ3deO8pGYEsG8uLY;&{~JDboBru`{GTaT&+j{N
z#4~wwrCN)^y~`(GrO&f{T%&!?o^Sc|hWig5%)7hAZ(Zv3cYAkmd2uW`ToZLp)~w*S
zPP28IWXVxg#`63Z2NWJlTK##$JbC_}e5H^|VQ|I&@4?GAZ=P&CF86p|^*coouDP4;
zw(&~;0QJ*1rJi1~cJ1FYk{njTV*ks1*_Y{FWRaMa@KQs5de65-^WBpFe~4&WCO3Kg
zdEEym-wI}N%hVJmEr_3;aqHP}`I1jxceeIumVL9{wqN<|<LJXCE<QZGVtpTq9#3JA
zYcD%xC@guhZ<@+wtvfq-3=^!?FUz0Rb_hSY*6uUUtMji*tY%sktMT&kx+n=UG#ymf
zoObrm%aVDA927R{6mTA#s%+Ba=jb1`$a=vMRgYJbq>sNSQB{@obUefK^2Pmr0jYfn
z9ETGh@QMg<{PH~REa6sWEx~g^<=0#Wra5~2cJsyWzFf9oM~BOCMJZ`(jSEo~=~uOc
zF6ITj6w%d<?O6Hue1FOIi<hQ8>5|r8oadr@b?VcenO5@CuX|6w{_MuX(mj74eQwzu
zwed~NpM6Kd15SRK8Lauq$up93YO<QargeSaeg)>-yX2D|E!^8Xx8;bZ(U!76)dS%X
zXZN^%*3i{fnc5K)8+`7d?&}*<{Vv~=JZK-pcVT&pz!rIXCmpfHKR?9Xdy%zdv$D^k
z<;@jtMK^o5Z}|WIf8X2Lx$piQ<X@qst&kKF7*WvBo4-zNVuzrA>V|52`@@e@550-J
za_wqhP^{~A_v9-xF1qZ=G+DkteVU8ciWw$TZ+=b}{IoN&=A^65*V)@Uk3QPlsm3z#
z-M2Se=Vk5_eJ<H~ST^C;6f3j5TSs^CUoYICdg#W>&o6K8pSaHd^mb>b6S@`e7aXnM
zll^J!(dGQI7E?bufSZ~}<>fkEntW%Q6&{x@f3la;#8+9a|8d#w!ksY@(~jQSnk{LX
zC2~EsT$azh?5WjMuZiGQT{~jL%aRtm982?$?y`>zaarh|UL8CsKJQJE%ZY@VmEA6G
zkIXhrU9Q6^slEGe>qR@GZ7$+X5x3N4PW=1JB=^J(0neK=%KnS5S)y5)k!*f6Ztnhw
zC?D@LPY&|l4PoQeTl+or`GqMeKGN%*Pgk3md`tOpvpx3DcW*Drzy}f=?uIX$qIq#^
zj{P>4{lTw;ZMl;&rCBOur82!A8Ex`*$@CWTzP*E|@w>?}uA|G(FUWrz<fWyhHNz;C
zEBpF7)`U}2G($o{=9n$JvrTbYOA6N_KTiu*b}JPpho$maO&j(z$GB~KtDu}d`GI0}
z?fmM$lQ+BjS(n*;jC*fCYniad>IG}IiQQhetmX^LvHq0@x3Np9ZaT)Rz5o0s-Jp*e
z>;FA7<TF*9S^Cjwj$Lmud-Sh)oI*J_XHN+DrlvkcK_H_e_>{oSYMx(frpw*h*!Wzo
zvT0+nSWu{W)3U|v7FR#56IdXb*R^_;rY=`dOr+;u8TF;Dj%Nx?toLp4I40zgV^{Mw
zHLkK$w^%>Utuj;aclo<_bN&6E2yQ4o#?iUu?H)!&`|aO4m-H^nR7_px=9;I*GHc!3
zt2Nr^^o2aXz9^Krxx0O>wAn-6P@_a2S?TY&KSGX+I;e0S-@TMe7SbpC*m=Q(SCOTi
zU;f$B>2ZhNmTk_yuE)j2_2ZcNzJmvw*(;yVEjQq4o*q}#IXiDxC#b3^dg=ul`_RyM
z@N9NIXg=X^t>*Hhk1xy!c(v$Get>Y%hJO`C>+jfa*w<a1CLZwQs^y)j&$iYYzMC0)
z;Mp-2*}`t=Wh|Ssm+oEvMBY^W=h<x>)28%jmQ6BZSL^kgW#((E*8SS%in#Sk{j#8k
z=AW*fDBHd?;%8t>!KGzq))znjBG4K)clkE=8&{`2YUGW*V`F;a{WrIpSEoIC{(AS1
zUw4@LT8+0H*V$*Hd0?}=d&knvk4?VMh%r2OzKb)Gzy1E(-cQH=Y1B<W*Ak>YmlNbd
z0pFW8+f03uEt*eX*il#1u+wzj!+#b35-gWix?MI|<}=f&veNSD)2BW&jaYrHx0<>$
zY@2E-P@?W|tI3?tBkcY5%F_K$e%_t5)_2l24VTH0%m)+rG};~>Nacv)lU=adx<_U9
z9)o<16H|@O$h|P-;kXmWKl7bgpR8Ti@pzBiP{EZMO&=#O3y6HO^ydf1oEP$^Ra6%q
zGRivLyr#ZVboci+j}M#2J#+v6TK;Fnf5Anuu~V*HGtPe)Gv}pBn$}!<eQ5)gm&zgQ
zf^%-G1q3+dhCXDFTyxLt$sWNL1-X~Uj7pd3MtIcbTCRMTSG4o7-?Z&p%1rF~8#A+g
z=GyUY_qkNFh1FOz`Rgflf!5?1^%qOx>YsV?$z`hCx!WtKk$7&>QIRU%M;9yV-fj3@
zq3Kbn6lPOzE984Q<KD)({P*GxH^$zseEzlk!?dY_AF8WYTisyc_F5sCvQM|o?pSQ$
zH;<X7KPsA3eI%mWry8!5vz>P;ss5UViv8WzYQ_vEQL6>uRQO0(xkW*!^TV^*`9a~~
z$3gSY{{H;>`~MgfKRc6nZ%-v?R8CYoY=Vc%kDur3{bFNf@Be*QE@_y=!f*3|;bKO~
z4*kblTqcS6*NZ%S=zI6i?^%4urdvpcKT?^w@#hLl&r_%Nt^BHzTefxCh8)l2{*wpK
zed(-J+O75fGjD8<cHyVDt*#a(j?OB^3hmApmTu13GX1z&`_JRsZ~we|Ps4Hhxeg|F
zTg}<?*S}Bl**IBVNB7w43qQVYGvM<rczcAQwAJt347sGFmEEGkn{Jn%;|L6Y@ucEM
zi;w*FB{y!*dv;Dzah=@z^r!BEx02PWZ*M-<o~_QdBfX5V`u62D*0UVEBKGQg1s{3x
zTYTz=){Aj{ZN|)d<VBwM2WR~W@@hJ$@cG%<<a>K6S(*<Tr!Co@yFusRL%ttx{bm=)
z_yi}UEzo7UJ!^sOth?#a&Aps*^%FFU(+eIP-d^##*EGRGQ%nE&+rre(B0Fqay0|Ws
z^7h+1Xo$ElD69J}v|!Nx`|5g$Dr@+?I%}hvDeeDW)_?wz?Em<~^q?pSg{fW>&!u>z
z8!fq%S+ae;cK&YR<u{M5|M!0XrdK!b8)n_gS>_{gyl1KS<)RF3`~Ua<pPBdftjN^q
z$uF$@`lknYD$ZJ`XwvHtC@8m|T~pPs{M(HE8(OsUWHp>!16SPtd1>3qxdwB06+Yeb
z;X?h!PZyUP*0}!oYiBf*=VghNe!kJvMLSl_)6Uy5&H44cC+Yt`^zZq3>vzbt-UVMD
z`0>eeZ~pUa_Ks)MqBTBqo2|Kf=7{BI%`1!k{QCcoJAePqzD1{s?#QXPJzTP32S>lQ
z^z<iRSv5IWFSa<C&k+$3eO#D&*z=B~xO9AVWBk9{{hZ71#uT5P&hhI`MGa%qL56z?
zg5me7ZXd|j<<mU>F;OwQ;#0+67G_41!zYhqZjLNFDL%uznk%op@4CE##D}!WmsapY
z27ivSdinbDetvd#@xFcYb{0Rsbnjl@<(Db~90ELSoGeTxW@bYD$3aV2SFT(4Y*zNV
zh0B+>A3JvJq1gYPCtniwdp|BHZ`+%jX`=W`oloSZtN4)vtS*WR6elUBdaau)vB2!f
zgH<2vPW@$G-I>XhH(%}%yLUI|PSdzMcMr$!yL6~;nuxEh8oSk{BbUwOWjqsZZrH5q
zs*@c2=4zp2lTrP*{jT$bZ%3^cZ&qWOl(XsR&#iK9;+Bynm%I<Gyf}Bu&lQ$OS17TI
zU#(oJT~ct=!oF^Kn{J{@WAnR~qwl}DX?~K5F}ybadse`ldcWvkAFq=$43j_YpU-)l
zZ9CuFz)rmdH(d9tK6w{mCYG9HsNye}v_MjC+Wi`H?)5tam*4I_{`%QF``>nLr<*fp
zh4jxnW_w?<-Tp!9HPvwAqgx7J23Tep-Tx(D-#l~U$5tl`?Po>z>sPA_y*pU2OH)fT
z@&7g6njLw{U#gruGKIC|&xUGG`+QdZU-_ZCwi0~etg1(M$gpwi?eF6(7MtuB{iDfo
zL#|Do{iX`edXq%Ce+~SKD&}{;FMq*jzAe?T?3(KppIQC(4;H6n)k|`9ZT@@hyiMW0
z+}k3dJx^BO|I4j(SCD79^!DSfMq;U7h4(MptX`%RxWr&a@yDk+yFOW|or&_WnRw(a
zcR)mlhnnEyfB)@z9UM7)IrjaVUf-)}`iIf}zLaHOg-+w`_s1`=wq5SMd|}6X{eME+
z&-bTt>;1g6HbN-I%Hp*9r?vZkaNE@Jv)?n;lr-*Kf3I5IXzn3lj=%_kL)&!HVvBf#
zb)~m2474wm`~G*kA@jP>sg^QwO9~9qKZ^RyR{!|xj>JO2n4*&$3^}(SeVYBwA&bw3
z&-W%fc(g1aX2Xt{b$i4&C-SV@HDg)lvdo`lvz9G<ZM7_O-dBsmju+MzpWdSI^xvD(
zzsK3<?%8C1>h9)W`kNxR*}VNdE3;9LWv3EHn_2-+kf36+=ONGTnW1uN;nf|>a{oPC
zTNZco^5M|O%oe?GDyK~eY3DgMz3qd9RA-NAf#XXNrk71MPg`O>9nImW)DzkBJ>vZ~
zT~+a`KRI@kn92BmxNuDG*mSF%`E?f~LrVjAY}0kD^z0nfyiPY=WMzuC>(1EU)bz9V
zDdR!+<%TaTW8}KLHW#V~c`+?)TElI#K-$lVb={k+y~o>EZ|8Lq;1GRw{^Y!Hmjw$f
z-v5ga2|W4a=H{2PH4h8x95XPndFS+J;i8%wO;dz?%{BB^zX(^4^WxmazWnrL{(O!W
zt`krF3wGF8+@0r`dg5)d#qW9ZkG|%A#M|;|)2=ye$;<py=e-Kf3d)!=Rg6K(HT9_o
z=fwBdpMHM)ZLYzSA5E;4mCmk5Te|$86rVVox?+RD{d=$ZpWNoYs}K^w5y8UMr70xU
zs=Bm7w(6f0Lrk3fyWlhB_shB-Iz0IFqo7*iLnOz-hukcUn~ncyLgH(Vf6;q&_5FwE
z{JA&j_PRA|j%>>{%e{5vP%HP%Et$d$qN1YUR@N5Phf8`-Z|GQ>)7NU;qgnRI%!J2x
z!)DXg`ztr@wRE+a8ykDJow-=))lQ>~2^&?Ur6%b}Jxu)jk%c$0xIymv)D1s&o8R7%
znKdCYR4yoNTa(j)rNN>m8Xu*4v`UQDUpsv6n#f#E?)G`UQ)ZatMlBXowPRj;zHH_B
z7rfC^D}R2{Vo#VHt1vI<-J|mAsvjRR<_NO##od-yowkwj;I8CC9qD(oLJwsBUi7Cd
zoxiiE_p@=>%txOON%FF!cMCLy+)8=AQO;$5y7~6PLk<@%uAS&Ld%2n2iGN|6f@>w`
zgzmQceJa~6C@|VQ!Q%Ed_APJkA8*+Ech)!g@(Ia7^S=E#cuUgHJUH#FWXQaI*XzxS
z%Wh>`zm>_>+5Jt;M>=NL0XxghPZE-~T0b9hoOkQi8XjiIP>h~SN$m0!D>!^F7g)^&
z&54(?t-tQAFd_EJr{8~f+|6U?>h9k8xX-%c+068cuUEr8mtO|03KbC*<^BG2&ZIfq
zfB(l`ZVVPrS@UMq-%?>_mb+4?kN>DBNEXk3(;xKU+DZ-yjswpxaO@~$xjBbPu6@CU
z#l^3;DEQZlJp1}L&})yC@aZ`c>a3He|KQP+e*Ao4?_Ha>uM{(HP5Dt0=JO%=%I)K)
zWCi2y+;u)QCB|N}K~3SwrI|7B9xZ=u{qx+Sq%7yYKi|s5#UGbvvRtZ7ef9N;K;+F5
zUxRH&PWA>j+3ZzzI<YqR*=5chTWTr}t>0ey`O`bOB*~MdlbKjzPFWblbj;sZ^y7{x
zE917*ix;o@hO3mhJ$2^jJ+xr@qeDz4vEhan#0`FaU|SlXRb^;$@<!0fKPR{DnDaNc
z$G*z^u>Ib|Pxq%;PdIRY{kad}(<N)tS3aKd_vyKxr@dwWyu75iITA8q@Pj8QQtV;D
zjHV_gozqHk{lYwKf=&}WR1AD4J197qNV#$}o}aFJUVZ73)vLP;EPOOX7H!z@LAcUs
z{`c2gRAo82H=FCvUYIWWaZdHVOz)^gYC%pD!_|`=FD#vCkgFrjq}MxdO7Ftd)ruaR
zxrUQlG&MiInql^1PLhMpe%*zpS(mOPKPu$Na^|Z!7xTlkL``RxkQl4n+@$__<$L^6
za&`%cxyH$EU-<Rrxi*p6dyQO!%=}^>n6YeY@i=wp{gsZTs?vSxEA-0*ieB!iS2<VR
z{`k+?)TI@&QnH2J=XSVWSQ~ij_0k0ra&6CYW123Mcy)DYXlmJPJ~V%|t450vU(5rI
z!wrk{YwsRAcSY|-+bkx>155Wk6a^=~IkJ}gOedZRakM)9OqM&}d8xc^|Ni@xP8?lb
zU3wYo8|$vG<UCcgJnSfsZ-DHZ%?B?1UVXjrkgmc5tB<<{0-o7#&RWj7f<q#Ng>9QC
zQ`cUrqxTwDf0t{Y_MJy!NA06n{TuFTz7CDi@At-?{Pmc#Zo9eHfu$F(Bp1EjlJKv}
z&vf0RKgl~}ZteWOC_v+1!i83sgt@yK8w!Go&Z;zPib^qVU$8U%!>q10zKbHO<e)px
zjy{*zAN<(5yX(gNiE8sUN(Meyp!LAwd!MLtpW$k|66lPZ&S@pl{yCp^u>DEfeDbt>
z^o7~dpF{lEnO=B$n)s<~4}?xg)cpHkX}T<YS9>(iQOO+&HXJ#(-}yFNFP98l^m^}8
zuO(eB0UCNNEew~ZEb-s?NWOq`l@n7ev-G(i4^G`H-oqE7(FB|SF+9Ul?PhR#lfh&`
zp3TB3>%vwZxVFbdSKZH_VfyW2zhn)Ou7wYMezNDyOf0zS@>kfM=LruhI1WHgN%|pC
z#@v`0wIgMQR<7-#;Hh^tOLS~qs;_vutF&(EE~u_Oth>r-p#^iD_>ZZJ)`L4GP74ES
z)FjK88>g6g2&WgQ^qo3gUEY+ddXcFxW+KbQk~SBcw%d;%mt6JQwAANGL+H89npf3L
zcp3LD<Xi5nU@|dYaq70eD_iY3Ca&eJN}Ji%mSfR=L@sAui^JtDZ@zk2F39nz%WHKI
zezfOg%{|NW*UR_4X9SP(fhvTL?WaGeD=4f!z^!<}Wt&Xfvc`u6kBfePtg;Q6+IC^B
z#K(y9np0~|`&_<u&rc_F-jjPeFFDIEUX(p(*Q%z!;pWbVwqbk!*=YvPyJZ{2kz;)H
z<6-t2`)aL!^T{brH2i#O(}gXHtG8>lEO>cA&Mg0atET;5(MUz#vfTS;);XVLUgyZ+
zsC4Iy_j$Y1_mXoxw4JM?wa=bCYhrGG`PRM0%$pgr+}7-#`TcsX+B`G;jw4@oR%fsH
zWi=n&3uiY}3p|-{>5$(RrMEl`hYFG<SS8vVwPqf9FSMn+e9{#Y8}(Lc4&TL(e;ohw
zcfZ|<J=%GX&t*?nTGA`f6yk5a|IbDH%a5eb$N&2n&RMnTt7)si-qlYPHdyS))_yG|
zqZ4Dn-+!60HtgimGta`!Ckw{ad`gUq-OauIwy)LH1$D=Q-tGMC#w8<pullv`DRteX
zPbY%b+*{H;>)P`ExW{hy7AUyLHy`Ib&#uTaOV47<Q|5cwn<VTrmGoj0zQ0tyzHWZc
z&t&yIGJdw}dZ(!W5}0<9{r$ea6%j{{{PJ3I_4+I&54T&D|L=Y;2=QHbA@70wzpv>_
zmNK{Nd8Fs4>F;Q<&fhgH;<Vztx_7p(j`gWOC{)bdJ>UJ%%9MB7(jH4{m6q53FMbnz
z?BMgT+r@vgB64DOw@umH(^{<N{9dbOU2J@Q=N>^vjusokviP4SzRGusCM=&HUHs_R
zlHT0;XVWh|zoGE>*}K*`<!)tr?o}_im~`g|r$x!sHNoIfK2S2MsftlxV`2=ORdQS`
zMUBfz;jEmDYB7hH&#`T}P74+CHcnoaVZQD5<kEKgLbEhA*Ys%R_xpd#O^>&k{%L`<
zB8z)je&XG0b47es9-OAImM3?uZi#DB1v6{Nl<AKy$m;C-!?tY6=8u=>zFm9ckb>At
zrA#N+RMCYE)0Z(vS3aHlHm35p{KlMX5q<BX&O7g~n#pVY%wV<6;Z^td9oP}lvFeZ|
z3$xvtlBUPKzDI)9T2-G;5o<qS_VU$>BTnjKFONO^n9R|r9^r7{wZkp5xw5hrHjV;j
zm$tpg%3h=>b1*!>$2HQhX^qg#j<(FKyn{y;>{&Evg8l3>hLgQ6T)N|<GjrjB9aAnn
za#}KV+2+mCiV89>7cST`>CuAOeb>@%?OxQ<6sL3BP5#JPC$HHFKPC2kJbr)4hANhe
z$<m%nO7njIk~dtYbfod^>z`kP^#hHQWQtcGm}YkI9pAT7lY|PUkkDy~8IPtIMW@YL
zcP_8QdGYGa!Vf<LMTR!szTn6qWP8`dbbC^!pVn%vkJT~im6JG@OkKEjY3I9)Ne+&N
zPnWtLIV`*9_b+XizmFwC!+j$Y1$eu!Eqh$zmh_{^H6(0GQiecp$z+e2DU%F?E-5@H
zO|rP#!o=vZUfe56>c-wvwK{i>{yWEJvb=xQDy})ZPMoqj@zqmv;x@|%`;1$btX`~K
zuW#{}HDrpYFmL&#JGc67%PS?%7Ek$Uuc^ASglmcEokgJX5L~KuDL!H{6!2(ycrbLQ
z&F<9Yo^o}Qo*Z<)c<gTbGwIgD4;LM{=(jzrD|42`+;fb3EoAkq`rao#KIgn9!bH+q
zH1+N+eot=Mc0Gr*ivL^1D_$KfH?YZ@R5;mte~I+<_>D7|XWvVX?LHfxzpw4N+)u;V
z{l`8?2)uo{u;`A{zbDc0ijzGOZZge3-+cUe`mse%|F~YYkjc}}-~YjYJ=!{>?xklF
zUvKt)gQ?<mUj^Rh-0VNRQn^>by#8sNSfBpWo$hsK+JEmZkgzHi*IW4c0H0S{+{y5D
z(N23F3)uhP{{Q%w>~{-2%q{k1+y4Jp|6Qp2=*ALz^V*+sdNb{3o_qZJ$5ZG1PZFY8
zWV;hGZNJyNm2(q&Y4iEU`O5s;fp_?LgTEKv-g`f#XywuV|KImtoH>nSdB60#e`==B
z1m!AHR0F*qe5~qpOV(KvWt0%tDv*+t7+3e|`kCEtqpxr@J}d|h4fc-p<-EF-HRoRf
z@9}=Q<gil3j~@)>e%$om5LB4N*}Xb{@9b>-x&GG`jPC#bTCHKY>&M6cKLrhSvlT$*
z?)aqk?$4v5BwzL)Pn-ES?_VjrVWpgP|N8sY(oYr7b1FBU*^u_N*yj6gQ;WNPi#UH=
z$}NA$^*-_RarToEXKkKwe_Ct3KJ~S={)SIaw_m)g7vwoRHT?bw-f-P#&EfN(-n?#q
zB++8U9GmG;i@ViUPsy+U>t}PM@?E&SdX!1;>U8l*9Wsv(&i}{$v7#sC)}uw*`H!x>
z)>FMz)DUl1aVfp%eYj+>=##rw&u{v)wA}wr`Sttfx{glvyTiRJ;p9~Ff|ET_a({j@
z%jCU1`1<RDXhW`JXZI=`GX0g5=JW0C-F7uSo5!tAA0u77Ix}ZUv>B!(212Xp`BS)$
zI9!-ta{Bg+<8qESG$&OHS1kKipV7xKYr>0N3bqkOjG7-?wIsuzr5|R0{o~M_PM7}o
zsmB}n#Lmt2%XUjjH;UMH?5cTf>&;;I&j-EtKfKFtzUj}SpDB|JtCGZmypkRo^d-pN
z|Mg7vi_E<v{_>Uc_VR`s)!yMr>y17vZ2#N*-paMfZ+>*1iQX{5Q}k)g!&Ukl|1Lh<
zv;UWNO!dngzM3Cg>(|>K(EWbzcu!#Ko)53%KfE#4H+<;-Ugetb!RVOd=KIRDl}~Y1
zK7Q$T@AjsN^G=Ij*pbifUd>{DHT>dDy_fI2ijU@QH_p$|)VKeqaNhnum$7>9lfBpD
z9tCVaw|(DdW~Y{)NB!$gb+6xd+SXj>2uE1)b@O$}A)S+t%m2`vZI+XKXR0t~dF#sP
z8q@7-ceE_H{7C%&ujo5lH~;uHJ^aGHeB1x;)%_1>h6HNrE}EDZB*|^km^-^=_wJWp
zxTL=4KM%;}=bk=gL2}}<q%#pU+~GPh=e76$WPQKyzv6VAIb~ck??3L-R=aTZ=GhfX
zuNNNgTAEXpt9pL_=DyX_bsrqQs$XdG?8!g-|JgDZlmEO~-v8;@@8BJ^M{PE!a5$<-
zJ!>w%w=nC~(mQ{b9<AAvDVu41bd73!!aeus&R=RDw%!h(@W$f6=JWf{ojIMKbad0q
zoXX#>Vf#MF1gHBbOzxOCMQ-1pt@%c&Z#eq$>$|t^ubkf2m#w+HRd#n@(aTu#N0+QT
zj_sDHy!iA^!A+)Zd%qj`@BPGNobLPR!R_Xl`>($LxHdOD;bztU-+6c4rk#~@*O|0i
zJTKWLb3*w1stL(~;@|84*ZoOjxBq*x-*a|MfdpUk#>K}T6~#=5SL8aH-dAFEcA~Po
zgk{l_1#AVN>Qh6c>+``4`|o;7JaEz#KAWJr(NsQa6XO~Y3#OYEZ|@niy>^%2dltW?
zVS3$fwQ4@|OZzf)tA0wT^_N`Uquy1pL&CDk<o){l=RW@~zfktY;*!L~-c0ePm|Uin
zJ7-!|>Lt5}34e|1(W#ocW#+-~h~wAyf36kFxqob3Z1s(OA7$@V{$}3yZ|-};d>6*N
z>5c{wE+?KpxzMN_(lhteqMesHc9l$v|Ho?f|BI-VlHr9rK65uVoJ@W_VOz?xJ3II6
zOH^g=GvAQ^`}mcf+2=YwIo;Bk9e6pd=;NPAkt$xj-G7A4{(eb~x^>Iq+Jz%qeuf*A
z#>h&Uoqp#3bgHfUhApQhVuO!;>wW)mV&<g_atqg;W791B`9~*W&x~EOyl(ZsU$RmA
ztm*c+V=HE9Wpz(8%s%_;a@y(l()#l|wtalTY_Is>(u;R*7Okn%J1ypOEbng9{tOXs
zv9rq-Eo!)Fylg}9*}_X7CEWYXjdK4@{hRXd?!CoFCMOsEW<C>X&_C;x*O|!65qnE{
zXI4yke4Kx>m(-ft*QPCV4{f=$WxHvjSL40k2c`A+zb<zB@N3!jN}YQye{XQ=%rTnS
zaR0#L!sMTSL`2zc?*D9%>2En*FWSd7a>7yZ_*2#I^BxsAvokDK^Go~I<N2ecz;BLc
z>+jNSZ*4qmf1muZYNlZ#OIWe?!9^E!_NPdOPHp`$J@(YB?`B009u)SOn;7T*d&+D#
zdCAh?@cK)&&m}m6(%x@fd9(UpPg%j9g`F=yuTT2b5}2`Rn)kEwNzTughig9m?#j`4
z`sJagGT+t<v_2|s6kv+IUbr*nlmleKBk9HtgM07%FV)GUiYJ!l9_W)jvSJtK+0YB?
z->#gZnt3~<XV#e=ueR>__laAj?|t&Q(D;Q5(ycih*Q}m+T&`w9$;SvWpV<v!`OEiK
zf7s`z@`HJ;d1lP%iOe$ZPZ<<H6rZvG3HKKH+o$*c`}_a6`|WbW>?2>|oSonQoYj^(
z*JDNcOtB!2mT7NeznsdxJ*jIK=auQZBR?OjE|JSfd&hhIq3NcRzjRKY+I=Z{TB}NQ
zn*WnuZ{~`e-XgHe@$o!gn;##%j-8D>@ne@(RNvd>{_mUwHeKhKAbMrf+Iw<uHtQ`_
zUwY)J=7Ue!>ylFn84f>Px@_CDfCHyDigl*#DD21ya*e&dxve#);~&G!i&gy-ZS%Z4
zMfP4*XkjR6IvAAYenlZ?cGsz>Rqp;SucNarK77~}{cd62Exm7V?j63ZZ+G(D;<+`?
z=VhOW-J|eZ((ClGZJVxVepd?+*;ymTv_yBK=?Bqi#>uZTw&g@R8^>PppY5C3*uHF8
z%%USX@|QMk%)MaJ_+sy}y7`^UgkNVpx)^Wcv24ltT5-FdahpHyzkQ9fxnb!kX~>`k
zXbtJ-BdmeXKBScV*N92p-yBwG`);Cn^3lGeqy6(g+GO6Am?zLJelGcS_5Q~ZRtNq>
z3LiM9^=f9woW6ptkH*i8vu{N_ny7ba+1yZ`-zjs>OS1h+-E^JBw|e`h$@TvQ*YDZd
zcWjwjpPZdXITu%Vci{^@9_Bu|8~*MpCubBc&b+;SuKB&#^AnqQSN>dj{lnDjK|&uA
zZ`T@q^zOVIeqzqd$u~cI`@H3pP+E`tMA_gBnS1*huiyLCaG3r3hF43k?(klvsjr<J
z603XLzGg1(?Xu0)x7p%;7g#I1_qut1pI{_=CVoRl;nt-syLWxI6Tf5^sCiIaC(&hQ
z_x1RgqwkvABY%I(&bd)0u#oe_@1QT2)V1r^UwF|r`=ju3gW_vGYW_0r7oMNfe!Qta
zX!h*zyHAr#?shO5^{<fby}tDOd|{Psx))>P-1_8PO~e-F9FaJ7{rZKHFB%t@s9d~z
zLU{jA-r3(wCUlj4x;^#$g;`$;j&_OqtX#in=~mIpVij}zW=Z!Kuh}{G(5s6TYs13U
z)-^A7@BeV@Rp^#kaj&wwK0iKv?>x)dgKKscXWZL5v3K|Ena0&teR0}uOCBreuZcRl
zP`SOb_N&&c%X-K9Bu}n5xGw)n#vCQr^ppv|*PY+n#&Y-R(!3Pi=<9or{mR|_v*L5q
zy&nn8nVNb#WLTh05w6xppO3dbbTr`DwVv&4sKcWZ#St6b*0^x5+jLXm6x)`m4G#-~
zBSL!gN{;!=KmFxow&uKTGgc|d>&EFHb)6L8X)3Ni>)dPof3wc6ujKe8Gi93C0>-@h
zhJXuBmra^nTPyzLDeT@QqoH$P>C2wkyIZ=y%RD)%F8?UBo%!+^gQ{mi({yLFT+TKq
zepfQD`fX~QCf|pnr}dugI^DPV*G>1zSZ4nXO}F3e>T}xP?GUIKw|{rPdds;ZGdF)&
z8*}u=%3w|14@usgmtQBB)S5=$ubx>R*SDaOYv<E-q9Q-`=<NIx)n{&hNZJ0cpn|5n
zX?ON?{bdb#%X*^Eub2BFC>JlVeESTh{o=9Z0TCfB$1@d{_8yKt&^0^n(L(p|Pw)T#
z@n>;X%sSR{?0IS8teJ0K)#*EBD3mz4y6%v*+VyPNh7AI~DdD?6r}9)!78er}O<C>Z
z(RujoLIV+p%d#;h9cQk`)(Ps}yym`b&Buis3_6xsNACI))n{gNNWbpS`WH?CW=Tw^
zvrl<^l}UTd{<>hgf7-Pfg;PX#RzAJyUisFQ`L5)HThe-p4?Y{@-tsZC`KPnL>Y3=i
z+UwJI{GIl?;?vKof8Kx1Z+Q3c*6jSn`xZ?&e2<}ZTKm0KQ#?3DONCeJe=oWsalB>X
z(;M4*^LDIQvsS&Tg#G`w^VuFzO{>@KpJ_NXHAFZ5;KpT>?I){>u(5iJ`#$fr`yLya
z$>D2XxZYERgWX#%{$S%{{*NF0|H*E*_gJuH)}$r|t@N4TE)^*G{MaJ6yZ<{+c15w$
z%p;R6{pP+m=-Z^KP`vs)Q>N;M%ZWyFZ+>duwS0<lXQ7JB)rvds>i$Ng&6<}XRDYm<
zA$QV6jai3#ufJ88+A-1S;QQ~%sgp`dN|ah14IUiba`64<#AT0`=zgfQJ+j#LLBZo<
z>xtW3O>Q-xmOk**z~HiHRCLsfPSt{+OjD*$e=ZtrJXuNR-NBb11wT}8JZmB=E4#!<
ztT926)iFMQ+CMp${Qnca=G@r&GPmyEi}yc1&HpRftrnVmds2y!->pd=#=a_2#~<?t
zh6psoeg9Qr<W`+7zU_*Jj^4|+{*yYsUbuR<`Qeub#nuyMH6DMfAocR#TmA<oU*7O=
zzj*mhMaK8RSN1|nlS`c}@_W8%9al0mnaEo_VOIA}Z~gR|rTmBA9zLKZ)ccs9#h@hj
zz;l7#<sX07zL<5uN4<0Tnm^wXjBbau&GRtPpPjnwW#@8^AAcowxmgJ=-n&?MrKE_6
z*yGO@8_)7g^bnbTzW?~s17^l0Wn~jQQa`XciL^CwY<kM7{Lo<AGZhh@1>BzD>DCu3
zxTatCFAP-NH9;jQ&}iF%_us$n`Z?P~=GMvcb)Og}>+POw9j3!{`&?YSQnT}9mqUpr
z%iAA4QtF9&-*j-DKT}%Ogx}|0D++c$d%yKfx{}ASsuwT%5-e<4YR=s|C3%ngSoeWV
zI#o({&q##ZrdV?582eUNv!~_%XkR8RA}02*%yPmu*Yb4hk0E-|ix0f!4;1(`cZRU$
z%dGr`3s;JAI|fFohCV&|qB$b$(L@8=PM)KxM{m5;kc}1m^~|=k>W_$c+a@*xW7o?c
zA8#phEvRG;xqeOH)2`nqwSQII;P73(V1>l`(}s8CCqFWFvzq(tKqK>&>(}{DI8NB8
zFlWk)`Hia<wecx#Kd_L0yGqx2?fFZeKdf0W&$89Y@JfsVTN7K>o9h?uh&Fp@Oh{u|
zD0yRLSzq>Pc2j|r*<E|2_ngq&{*$G}QoPbI!PM;ax%x%E`-}ZwiL*GJUT%D5*W39M
z6z-SDTNXX%SDU>oGpGK!YTeH(-92#?rr+<?@W!=IdmB8bK-YdJ|EJ@dH@^R*QdzBh
z`*&>e|JMl(H4I@^C;!=2cc@%e|CY_Rb@B0_NRboIH6_-pKWNu}XU++`IJV`JcIMar
z-Bz7?=I5jDyZtJY{;|(~E&RFAY`$mSJd<1fbw~F9f1<eMLdLt@@8wo;)iU3D_UuQF
z-Q?iA<?8pdm;e5wCt}CT?D+Yh@16g5|2w*?G+s`eVCb8szq9T4{;#}Q%!U6x>c4+}
zwcqMvmCOvy$%{^L-uZhrzwqm(l`qQLO4t9HCU5_P!`&)*M@#Qp{fF<?*Ui1XuYxsi
z@p?h;PrKv)Nb~R6F~9!T(ft=Tzcl~zVSc@*j5wd~;Y~hjy1VDq|5<H+G`DQSr;F?r
zUpm7*GTD};2HF=3*8fYd_nMjW$iM!B`IfS83A5Ck8uBD8YFhu?{QpON-K<!9a8w>!
z$-^?Y@Z{mwzG3e-zVQ6^Zco6P*4meUySR2d{PfW(er|felf440p)naZw$1%&Wy5V}
zeE#rH2ixNGCzJ0)bG1HVIV_i?z~EisVR>B9Kb2R^af9li$d=Sfzn|L|tQWfKzi?9R
z)F1y&-AKsw3p@5Je}0b0g8zmm8;bp}&(FPm<$TN`{$uBsEm$GZAN+dy{0AXMb1m!w
z#Ab45*`NJ0$II^Q!=yu7B%8Oqo`2*+3D@g}pRcx8TFvy|XnR6t-m5wHx5xgkKCTuL
zcYgc+&&+!onV9O%Yrb2%`M-}}>HWD;_qdNM6xbOUYtFj0d&age-8GpvL(1}MX2;up
z)UI(g{d3gT`(;-4qJ$?U`wEYLRf*>NI6L0<)c$`j(tT$CTc2_K+Q<KYAKMG-%bxyg
z|3feIoJ7Cf7l!};{Qo;_*fULfn$Jt~uvO|#(b>|=GAH}Fym-}Fbk;ULxyqMwTB^`0
ziO%C2Du3@h{&WBTuhMGzdmF3sBK;1p<NtZozy75CgQxTVeTv&xcHX?>$gaqrH=5tP
z{aE<_`aUPm3#VINJ$g9L@91-u({1h7XaBkWTkTo8YJ4UO-<I^fwfi<-X<T>jtmyKC
zn{<>WavV$$h_(MK^dtRKrG@;8^t=GOlY5s2X#6sM!^YHp*iq%#0@b+w-1+z7eJ+>S
z#IZY`e}205x-iE>-j*uShYzQ<ZC;o5q4HS4u9Yj+u;{J1`un`TP>H#4THKkb;W{Zd
z_h`!8*wbd){!P$dmyPl3<734~L#IDha^Ccl;s4h=c}f!v<oEt$bDzxi|1W#}*|Xi7
zjx`9TGVc4@`~K6j+4%)u7isSJz0aWYo|nSjX>QwZpT5+spZao1l#chJOsTR(bGOG=
zi#|VL`R8f;U&+a<oHrDxG0t!2=U2IShgE0)GbxX+8>-)G^~vf!J+*qjM`>gB4)vgq
zmW#9%BljLZwB_V;cS}+AdhfZXtrqF}hfMofYmghU_xPiAJK0q2wk}#}VLPp~Y$MbB
zCzU4@i~n~lEH!yiY9lMM`}>W6p6QeS`<!1?^^EO#&U@Wfr;GCw%>`cU@^cL2TzF`6
z@S&o259ezy`NJHo|1W0&5A$>3#skaLZ~2DrnCvW{`?w&x>6K@lOdFfig?kk}{MXm1
z-z{wjzn*&}*0ytk`|^#8W%Iv=JoA;A`+jZOmy~6Fx{p1o%~$pud24mFyiGu5kzn1w
z_5TlWv)cKbPC9mJ^^SjwI7?)*(_j5by%O58DVLZ3`kim?F`1DTGLg!udOa?23dd}Z
zKIgvts73m#WP$XA?qwTuJiaV@^J7W9Zfk?#(WVTps70T?hZR*#;;HP}kvU66h||RP
z@Z*m$2I~$l%e-{shDT(i<U7Yh^7fqDaxdS!>8T-ds{Z_(_t}gFJ;LkTH%9dMs5v`2
zGTyy=SE8-){(JlXn`)b$D}FDPDQG$^aOBdYif3<lxtBX%Utpu~qQmpk%EGF`7Y55*
z3buF)`Ocp`(SeCot}(cM+l!h15Aa+yo(|fm`8T+F->-ZA^F?!Rt4;KX3Ak^~b#(d3
z4bmJBidTKn7U|mT-jH{!^54Vyq{rQIyKY%2T1?bmyMt}M^8(J^e)BaC<@xJ<(r*_|
z3e&a!)4tw#(UIxAFFZ~Q|MWen^{#B)<KT<ZCl_W)wH`e>MU>r8xBJip{@yi<RNns6
zPo1?ab^mI%-}{2%6T1=}nB<uEZ9A}FpNn#8+SFgNXD2=1HoH>wX^HmAP3P;Jn?)y=
ze%t01cQ12c_WXNVZ`2oV+&Izve$C`>-^yA9u3W#qIp^l4``&?Db-E9?^9M&oc^OF9
zq~8?kWLdxepVjH<`pdU%`!?rS*YU?TpU)VtiP#7_1<*g&j{UHSFX#}W<BuP1I<0ql
z|McA3+j^HRQ_I`&knLX0XWvt&PDN}=IaxXN?PLYU>3r9HE=5MIe^|(|>(Qj=cAFyV
zue`hdE@bs*^-UUI^S<Bv@A1L%p})RS#S7=jvAROOuWeNhz8AS;c>BRdUbWrPssCn9
zO#k?1?w<RTKC~K2ZDnne-=AJ&=dm1oOb_VfmR-Ac`KR{m`+hn7>0NuV?vIze^*1IR
z<zi|)06L|#=2eUUXwTzE_CA*AVuRdoM`QLSDDf?y-g!CAaN#~F;~JUM+h%TkqgD4~
z?d^&$_W$blY%%jJ_0`PqTh;B=X}<qhyY;u=@{cAAeQ)ocbFYq`@#CnIN9o6NzoRwe
zO=og0_Ilyr;+ZX#akDYj&VGCC+x|$Ke@9O55PiQOB}H<5{>_62AIJJgMskEr3a|gI
zGMn$%&hOIY!IGc&zOT{zCt%V2J>9vTGgV5uKQhNA&3IG5<G!0#4_nUV+i$wSvP1Sk
zDVI>Dr_$wYhRHWY&o@L*IOUP_GN<jwar4L}J}=gW&9=R+*paSs@=Ro#9M8k*dKICE
zE0@n(v}@O_=)9e+DMlx`_4hc~$UR>+JMUEc(Wha1jc4+JmoNo+O**MEE%o8YiVLN4
zBXo|vy1F`YLjvRFWxl7kdoK-|v214k_3i6Rta_Ja?ugOzxvVnP>tWGO10Lr2=Z{a{
z_eGVdkzrcuM6acOKFtC<V)RUWoxMDjCOU95-hTUTxy;faO`X$+Ht8g7ys=@2TX{*z
zhRV-rbNtjlKRc@|a;nOI>E+BxCsXS5F2DRzIsNUCmU*tyoZ<eLem)AB+iquKc|~#O
zinzzsr)B=UT`g&sw@t9&JgZ;qg~_MFTm+hGR2gDZ=C97sdR;0|z&$DL+s%tD3J-)N
zq*!8ZK3AD_YM#8O=fC1!&vc)pjGO;-IPI1zOq{snmrZpWALDd|RIf)b@BQ7I{QK_B
zZ+gnV?|k!*uHh>#X45(S=)u9}D_5@YRM|{EIl*gb!ER7_k$XGENOF!}y4}xWtGUM>
z7A(k=T9yggVQe9@Bj%>So;!c1D9`nK{#%A`{mjh=x8*LnxPt3o=%0fHjuIR>w-0Ul
zWnbdNVd5LS&W7u#Jc|&wdhoOObGh5~tX^_>+&=c2-(XQ4)Asvy?D=c{&j0hP^Z5+R
zkGErLZa!U8^DVUQQ~kfjkk5BRejWPr>Y;naW$vGoPwO4-Gx;F@|3|q+*7L&GU6Ff!
z{bQS>oG#3><K?6YJx43P?z~^LLak-0U$9+~&Ggt!9p2N|KPa#NKL5|-|G(JZ?RaW*
zi~rq&hw(OEKb!O)mdrU}q`!17uX~^(TUX+dEvnVGRiyhpx0y%ztiSfbl66~4W~kKm
zb5}G!a2)bHx!!Eno1BRutgP`flLJIsrf{Cm*?yzsPTJ3kxyrF-@8u@=tUNI<`pxNe
zhyJA%mtFl*qxa|Sjg8ETEZ1X-dE@f08I(WnHQ!YIJx}L!(&c5o5z~%7I@-N4cXs)`
z%Jx&Iyg;XBE%TjyV@IL#wA8@p=+mmxV-B_PO5fOBu5W8=`(oF*Et!{Z>?+lMeSQ6L
z$VnexE_rjdI_>%WZnugM=dIhfJ(piTGuynsnVrAr%Zos_9*Kjd*JGM<Z-Wl>(}~{J
zlVNgdcK*K3%vllBT&?ClJ0_jKWX+l$ko~p4zcCur{wjHPet!S5Oc|Ss3Fh}In8Bz0
zs7*e}bNbY)b91dFt;=*?mdx5){oSbi-5thUy^?-^O~n^&+Q(ko>^%DDY^~X>hGn{9
zlEOB9YvgKAM)Yc{zp^MjxX^F*aq~!%?;d)x%Z(TMJTOXdS$IskT9+wrhP{*eq$z){
zv#VVU`Sve9YJKr*AIA&26{XvcuHLP?hi^yfu5-b+H|>u%k5o@Qw4|taUCWckt!nPO
zRYV;RzTI5nCdMT+^Udb%KWFq_=iGkYXhz)9ivrs|T5LLg{`vgr!L`TS`{ka#-F{z9
zsy8Si;>4$?rwez+fX<RTt-pWHp+io$Zr=u-zq8}ntZW0{$=BoS=T4g@7NaNbJzX#L
z&5ezJk6D(y5Xj&ARjkvcDL~`M)9LYkk&!nG?z=6YBi{OmzcumtKHbwwlRFm8{#&1-
z<)d@QPiv+`o}eh#(d9i43xeYU-456roSc85gv<N1CRgg+U&qexYPr2W>e$8p>l;4@
za9<4;|78E~czp14?~e!S{|W#9q5t1=cj$rmKVR<`9lgUT64LWfyFAAK`@ye&ZtwrE
zo;&43@A^9~JFg#j$sG_P@L+EEyl0@3R}RMin<Re!u7i?t#GXfC{~r18_j>Z6>fh7<
zfAZ@-KIy&^yQk^@x4roRZ%nIFFLQ`a4~mQI$y+g<{pbe;RSVvOSA|RW?L6^dzi#;h
zvj=bgit)+1iE)|9Z~xhT-MHkqs=2TE1(&eSRXkk(e(Fx%{_S>;#+hT*{d!_kezQ7?
z-`RfRLB{ka@%K($keC)@?t1*G#eOdD;9%j<(9p`q3=X-uy3=FJB>CE#uU!k9HG6jT
zbEiueE;y8zn_tYBa&>h$_+Z36AKgrRm4!MDil6y_PMvZUk3G`DDGWY*Xzw?v&(F?2
z1|`<z%iFi#tJ2;-tLn>(z;!zwadmZdRXplcm#{2S36CjcwJv+};KW4bpJkmcO{Y$I
zt=s+1YTf>SRaseC1v_IzL`64NeoiZUbK_u_sP>L8m%Jl)m+7|i%d1WF`0%J(KPWo-
zG_U!chD3=iTer^rUac|JYh(U>yJi0K&mHL$zPUB~db-iU*($ql#Xo55pS9^2qi@RO
zAGTZ1CS5k1{V`Q4eqQp2S9!DR>(!knny+4y&9thi;9^Ga_3yX#?7RNRU(cvOx@rEN
z`3rBT@5q_cmzv%yAnJHf|N87bKQ<nHySe1pG_Dd~gQY=-U!SOAk1_IlUa<Vh@%O15
zyEe_^oyAsu<`U<wP3Q0JI&Aj+oUz6S4n5h`#`Q~%F~9by+kQLE$Mf4vnY>3IE%f*O
zP`Y~c>YcjZZ+FD##Z|vGmEmK*m@&n+`WwTB?CW|$oh*LK!3n*>=3MUfyT_JzPOf-5
zHN4|!QYEvC+T@38x8GZ|VZ#LLcRQN9y11gEqS}53iE^=uH?TKm&Z?NxBq+M{=c9t4
zrqr2Rwv}GEcef9*@ae}C4Ozpf?EhZdfBfOh+-LbG_0P@w|79{O3K<SbKFPNKVV&2f
z${`}cXtjA-{Y3e*kv`tp*3)z~v~-uRSz<7Ig4gGQyW#te?(tr~*m{;iK!{grB`4F2
znvc1EijHypS|(MnYwr)41)T!sbzg7ClpN(PaeXY|$DcZNDvNDxk3?I4^}f&B@8o5R
z{M7fk?r~I}Z|9kpojprz6lR1j-k3H0=q+Wl%UZ15?pZ8%nihCJxb)Ct`q@o}QsMqr
zZr)1x{7=z({*yD)0)iDj?0)c5RxCI0WV8INAF)cW`)cLZpWd{_@${6ZVPE{>KAx$R
z+ZCgC>eMMvQ9RS9?dRv`!9hWj%G3E;oh;`1eS3f3KQc1X?oPtRMXsQtroyK0@WX^R
zH#UZZg&AdDQjwC9nqg6>#Mj=uYL%AKM2@Rhum1cf%g65O>MCPV(9qV#wr=mYs5^P)
zyUX7n+m_2GXVZ~;d&%WYK6$&I!w(ha`fbj*s07;4@%()H=Vxc1|Gk%)k+ER!UfcI<
zb#HDM-nw;b#flXc?>EiSp2gr|7MUKu<jlL-aZ`Pjeq5{T-okmlKINl3H!suP__y+=
zp4-fBn6$BOKQm#^*7Hf_t5d2M`!=dqzdwHI!zsP)+T2NtW5vAPvwc=yir9PnocbD3
zroM9TYjWE%yS?mevW!=GEWV&TBXI4;^}d@jmbfjrej^}UWk%G}jn5XGo%T5L;eFeT
z;Air6kBh6Tt$BHQuUx&lG5I)O&h2L(9v+UEc65p7WYBqn#_8vtJUH0wGuKM=vdPWe
z<@#H<ZUr4(e&lhXm6cUScJ}FZC5bjiS65ch@vWc(4(09tHSM^cb6ZWU`{`lM7J)Bc
zz9dA7+1lDx{#5)O`}p(EDcd(cDn7nJnq$cpL8k{LJN;IfzF)FNWKJIkY{}j4D|e(?
zmnjCQ{uAkX=wLo)%Cu>GRX!YZ0|M)(JNMgn9G8D{yQ67Q5LXx5duQW0A-xi=S3Q5v
z&9hH^7p!h6ysP=stcg?RZ)#K!&~jO-6U3wL<Zx@snp>G4pVdk8?kHMxuJ>}OncU`|
z&y<+g3vV;KvBY=6x*WA{<@~o?CaHaP%c{w3JUB0G_Wbp>6M1~Evn;D!lruZi*{1g2
z>UW?sVe9|b8O`*ubF+N@%|2pHN#4`j4+{!T>u%pr`@8JhySt}BN4-C6lV(%-Z|p04
z*x-}?Pri2M)0;NrO2=L=w3@s3vrd84+@C2+mj`iKSy@$lK5HJir^0aR)TuYN_2%BL
zd>YMh@K7uF%Ju8dKessXb?IxXw!}H#wXEdp{}^_<G@0M85k8@odV6d3b=HL&YK2$?
zyd9NzmAP1t@2b`=>{yfTW5U{|cOpXFOLW=VoICrk{cf6hV$KS$<XaOqZho6nskCx|
zu43-I8}m0FT*zPJ=sWXfamm}3gHG?iW_6@*TFAB0<WJUO&+L#z+#WO4?S60J{KhJ<
zFD+86E%C``zA4kD86_RznB%AZ_;`Q%>uYP@+}_^rqIB@a#^jAU+?uL++of-BE8KlI
zrtarc5mC{?CnqM>?4KT2r8#x#)HnC{&;R=Rdhtb9@r@Z57Zp$Va>r9eXwR2R-Y$zT
z?s<5AlMW~}em<T4`IwicYTf?%uh;G7oA0liq-7V+v%67Ywqzua)aNH9yFOIevN-eg
znQRg4zgbcHPqCj_=*F8nAMD%@NySI(V%V-2VzCi4SZ`;X;S})TY#W<JiA21b$C87o
z2U>~_p0_i&S#agqy`N2=9KPkhI5bge7hC@t^GLh6=iR|2Hhp(i*_gDc&T3#)HoYdZ
zDZ(IV@j8#k8++9}KmVx|^qFbY`t|ko&54KE($36qRGXam_0`ocU%nK4zgzzJ`|q23
ztIZ!4%!tr&b9QEae5_Yltb1eW>#$`$GY>sEIl1!3!*<KEHxkNjJqNbs-nxBzVvteM
z6OU~<HxFH0?5?S+oBI0N+9^|~KKxq6t|K)kOHgt7h2=q-pmXp`OHCKM^`4sUwtDT_
zrym|3J~PKsSX0%y?9G9vMS?tR8+E$x?k+b@Jtb21_SV$+RNMG3U%qU}zi*ef?`PVv
zUTO9N&(6-ixheH@Ll*1JN$bSlNly~EFn7Wl`>f*MK696CSYoUw8NaOXzfkl1efgU<
zAAL4yBb)GP*5kWXPan&Wc|FY~Sje~IPwm{#Ctm*ce4-`960>1T+~b7yC0Z<Bk2Zyu
zMOMCV-S_Fs-&l^0Wvah#soHu*Y(2f`jQ)(s#TVyJka@Lbrn%GOw|~ucv3+*WGI2fk
zzQ$X#^Jvn~B{jR&1xG}9I65*;ojTQLhC!pUd!In3%cjr<F7A~pRy?>G9-n$=M<GkI
zqtE3f%a*aNzka%jmHWlJcWL+c)$X|a?$6)JCsS^0O7*tWc|J3JUgH0Me|2JZ6hu7!
zYb(dE9k%Ahb4P(rmx`ZHr|*0=E1QF@`L7L!jokSSCszg3MDgrzd{A=eYm|2INtL?o
zZq?=Dp1y~TBX&6M*Z(T~&VHx9Wae%4Q`esK2nMBvyP>U04Vq{1U8G*%_b&5iuNH8x
zHNTzn-co(v6PNsFx9o~MJ8K#LFAN2pmDJj=tC;AOH-BmIRTtUC<)xo4@~ZU;o{avT
z^KM>t!O4iuOy_Kq-`mem`tUT8O<T=-4`0Q{<Zh3kOOwxpX}{?3mb;x%^xCIcTm4l9
z&yTZWldn{o@G(r8KE3#?>Gco4Ua#Mrd09=UlO=BbbWnDQy*|^bR7<}0iy&y*WN2up
zh`9LUqMZd6GA3qbn@V4Y$?&l+4Cv4dUX~!i_V{@J^S$5iDGP8I)cz_-J2T^8@p;?!
z!w(M_@Pw_2XgvJz!1K>fZ8>E4*c%-b{+HX9`*1iKNE|WX>65WMbZ>9<$`vaXY}_cg
z{PM|9Pfvq7=(@VPHlI!?duW2rCp<Mplcm{FZ|4)Cdv(7-TU}fXBvPK7nAqGFQWIrj
zYWnc|@0IJ<AAeiMY>;_L1$2;Zb+vWgcIoxky9JomU!R>{yfDeTqNw|%yspU)`EU!5
zlm!=*E0X6Q-R35Jx~-f)qUN_!^OTuu?|eKEHF*o`#w|y#&h|gHX`%7FuSzG{L_Lhk
z${1fpY(KK_Fu$Im#%FFji=5AalY6hv^nI3j^Jd*{VIz4vX@6(`ph-s6Poon)l!se>
zuwaqyzd6I~Pu}#iH-dWqS1xV7Cg&OLb4kp${(a}HWe-cNDj)ZnM{LiN4e|nQD*5?r
z_RcStyeogd-Cp_s?|0ATmt#sUy8irKa;Swf=k~VVPGNPXo8P~C>+Ng-ot)k-SJe@s
zb@a!_$A&!2(@(cvf9-m`RH&2XyxngdyZQY-YKK2PJ>70AxnNl)vzMfo=Ec&v4hjY;
zO9Mr_xMer~t~kFTYsE<yrumnX&U75Sd!wjpTjs2i((=nU<c`Z~NIkeU-Oq*dco2uR
zbGe0EVx(oAx{3MLgx7jntF=B@$leHR`*uximy+S0MkcLO-i0rk)=g78l(}N1*7D`M
zg`*{oD8;Jyu2``~=|QoxeCoYzMkQrsi?f^7y~~UF=X9#;&lXd2VgK7xLPLZ;?e26t
zq;mPDOV7nEzOJdNOOm&2DSPm#eT9y`>-9GKeLf2<qLGdx6yMStxbVlB34Y5h3m!0h
z{ra`Cb&I9a&X3bIrIc?@{S&OnB2wXE=e#@B+22dZmyv0XRR|l)w3c%_j$Dks<?E&K
ziL0wtz|$#u{X>NZVl$?x%scrl{E4VY)k14V*^S2gjcy37YHCV<R~RTLGi!f-kk_J%
z8gJje&6wpD85y~M>h1^~Hx(g80geT)tL#h^CZD`;^JZu6Z4oiCXH%mun{c)t-f=gt
z#!h})?&Z68eYK`8TChOjez}d@aRZ(?e(C#CW}koV%;BgbcKBhzg93|=qe&GuayqA9
zh4bmA+njhR_2ShlrUe@|82q>N*5qtIyd+4|X`#Th)Q4re&(EHy5Pr`vo#n(yk?x}f
zJ7aEa>rIWU*c<ogqs6Xu&kw7f69rwf@(Q$3rssO~w)4(mbp=uZvO*$NENcTjgI$uo
z-rKjUVN=6_<dpZn=iM`EjZ44w*FOB7!4pB3Ns)(V-S@OpQMeH;wdKZi(IrdWCvDvP
zx9!f&=iVJTm4^$Aey^#NNyxY`*(xx1;T3(4v<0fOWSM79U3f;nBVA$Eq!)WrMf{Hk
z*cR`6z9xUAU+i_xR;Qm+zhszn-G8qx!Sk*7VE@siAAkSVy?_3(V$QpFc`<t8tgNgr
zcFog&IrH;{m#<za&GZrCVf*;I{harY$#<gDm1Ouf7d`dL+xPQX%}@W<N1xAZ*nju*
zAN%IX(QnHwWTuFSUzRoXOgFOl{ipDf$=t{C^#}UpY8l&)JFCeZ{~#V5`C!S;rE}*+
zOll3ke`xypy)#Nz&rg2p{XU`Mj>==T>2)v7*6mL+WH;M;e3#hl_4jIc&)a@VHk0pt
zXsB@Ovh&j&t#|&PwbuA%U7dW*WYOH?pyjswi!L2FmvZ7k!j%(TQL|=OpN~9J{%(Ko
z*Y$SC^Y7nkIQ~{uu0JrwRA{;E;=P+^ryP4Qt@_>LOFxAtXnk)4H6ZHvm$aOV_<x7N
zvOMj?x)iy0lYahP;y*?6O^4^L<h|>z?ES68$@!>d`hkSpbLLJQOpON`rtX!oo1cHd
zBBA}TW982SI!bMc0WmRi?q4(MU9@Ns<J(_heg-_u`~SO~o3!uo({HbL*=5)>_$@CE
zh-?uM5fR}ze`J#Hvx-fJ^jlt(uWdeiywyi<`+<Xep6^tHUl;q|7MgX)Nc`!n{p*uo
z-uqmAyX07k-4%sf&m+ohe)q_{es_QS-RzdrU$0+%{=Gs?+beI6htQ#ulT(CmmbF{?
zKk<9<dH!dE{N?rMoEBbKz_n@J>*jT9s`uVw{NA(r{_ow_-dpaOuycZQwn*2)XH8za
zXL?32nXGowr7gs0wua`^6BE2rvYFDuS+_VQGo1BlE3i0u^!lBH^Xs46^{iX$;$nIK
z_v-)0wtA~={&?88q;2typvSLf?>iL#|7v{E(LEPyK0I_=<T=^L$jx78*4A0=hb@{+
zYhLfJKUiB{+qum4^^NN{?wwzi8Ljs4T%Ozcm{ak8zs@(_cqrleM1%7B*Sl*TYhS;(
zBVE}&dbfxAB9*1>*82+Wzt=vE>U($3?dX!Fpp(!Of~HRPoozljY#Vr+&778n&JT3<
z?sUz6wE52YlU%O%|EeE%%}$ySk?D8;nGegI$e*HJ3m;9k;h(U#{kNSkb07bwH9RLQ
z71->jRlBjh{{19<<^7+{Hr34Mo*(&Kw#jC>vu*KwxmnA2KlBA?*q!Iu&CbZcWTe@>
zyl&sLl1nNfPnJHdFrWPF%7v>^i+@&_hu^bm-c`8tfTod#x95TkmEz^Tiw@0Rwrtg$
z12azctXZpQmL{@3?_QH;sav1i+260XfBy4Nc8jcSpNZR-w56g?Hc5D#J$vP3v845;
zSu3t5rCeN{qSJlvi^)#L!k>T6%&?v<sJu!tFxIvG{P7tVl|6qxVu@XUVX}|n(v%}R
zI(?+<+d{nCK0QiO^PTUK>nplAYxRvwvPZY5`2GEJZEBdtIabYsTYPv{$3N+HTKL0O
zz=<cSc;5x(9W{H-HI^B_uKOc-@WO*nAFW#DMD3Pa7^lDf_cyXu?i~B8nbLC>a$WxT
zvnHneZmE{y!bkjV+=m?$RQaa+nanv*AQ4k`IHd0HW79WxO6FMqPvO+A-`B@~|IYnv
z1q;)kPa3wlT$*ihVcX|K#*gdN=2d)fymT}<(jb>9YR}wn?~K2w7B9_rS^qVB&X%rA
zvokiBhbhg!5b(`Z`1@V`iI?6L_c?vMd3c-7tl6^+OTHQ{3gBU#mda}&^g^npH16B^
zFRyIHY){@}7Jrm{{r7jB2j{f3;^(F>Sbfy&sN{*G+vlG>^(Orlc((T4mYPY`uYEFR
zE%TDp`nO<*0T0XGAI@@r7{w1d@n2Qu_B?W#g?sOm>mmX#yXJq3@%-P#uxUQiywG#T
zD<$SmG}){__xu{oRGZB&buTSrclqzdYI(Tf&F$#|F1w{F|4rR4X`ijR*qJGM-ioQ(
zzyI93o}r-1{O$Sqh5s$Mj;6DA9@%o{Uz<Uz)5q%M+us%#u5*n$I6vmegLOOEmVK=*
zshMZ^C1;XZdr@iF9M$<Qjlb8}@;_((dS<SasL+CdKXowyY)qA#4&{E=db7=DvGnH4
zkC(1k!r&{uAd`ig`{S>>9tR$N3W^AFSh1yTY4@^^zq<FQea@C^vSVMmlr^K_w!DJK
zmJ24m34w}o`=uvv_$miPbR18d@G3L?GW+WfVLpjDg`gvGZTy&3Dry+HmYTm#U0|)D
zqjltQ$%C(~np<LS{+-o)=#FK|dCg6GSGYzTO`pj9b=%Z!ccZU;IWoOXfXVlAyZ+%>
zPK9D~w5qo|w+qeRRW8aEaK0I|-R>4o^~}RAN{#mug8~Ci=>LD3%cN*-USF)}|L;1x
z%A?G8f4&LT{XW4~wQKU7&(BY1ynClK%aH8|<II;T7gO3A?)U2{NWFEmxbwqC?@UNj
zhLN*@fWs`!Th>++4}nf<@V@?|X2wd3OzxUE{*M{_7I%K!@mrEB*!%6lC$<6(PZgn~
zPd`64TA(g;^ez7bqY|Ny8j{fuzW<&u&2x)-*Kzp+e(IjLzudOJVfo5`?xW33sgWh6
z+k#(B7LV!l%e<YF+jnWg6pN~<y3t-i%Uq{kPRep+IL{h)IQnCIXYTDKL7K;UrKjuH
zFNbU{EZnzKfrrJe{p5>RMTcTP<}ACgpgY5&bIJqmT^;V1Z*22japCmAHnF5;2@ZwX
z(hAQP&5Z3lSt<9$R*g+Z>6sHpQ6>AY`Lo}yj^A$-F7<rL-!Jn2UE-O)^OyBGeT;nI
zWG0YuNi!~gqkyYviHdFu!{uou$AVp~Ztc(zNKf?T*dDDa6%-Sp!3?g*M7p>e6Zs7V
zn0ziPP4#>DDV2G}q?aOm`DS?<xyuSGSQbq7T5jOWt;i<4yI#Vne(CP|x(3Iug_^&g
z%Orf*ckbEueYewZPj^~qQNBdC@D<;!rQK&A`mEFA^gHW#D_P*%u>uR7Nu4Zt{j>dr
zG$!%}UzlJt+uKDLd>HApwe`8Hx7<4Hs8_fqxBlCH|D{uwT+aNu-4VQx?dS$W@Nx;y
z9lP60Z-1Nnkv00V$sRlVgG~nty4MD1=mn@qakz&Ss$?^r@aSCG_)JKmRqj@w{L%EI
zoIm!?Ips0Im@UP&al!7r6VJ}x_E>*?@}m_(-(<2<KP}lPbN8EK`(i8ccoX^U3zb{`
z#AvLZ&)Nby)ytze#3(VwcGr)b^VMpT6W`z4yZnsn0*l5eL6T2FdovgQ5CQEbUH4i<
zWU7#sj^*laH?Jyv{j?-NgU#KmF}E@G>ob)p56nz>4j-&On|aOLRpDvEF6}d-i%pvL
zO+Liia=`EVhkw=iL1rJ7)I!9QXBWm*yb83tBmVSP^r@G5)`!;5``xf#%5>7<-P<-U
zyLoWYQSKLZ8z##3YU)L8;h0{gB%&$4twimx<Na+ni`J^YfA4!}#@*Tc(@(YPc^0Kc
z`j^(lOa7WSKe|zz!}p8k=PI?ztu;Tcom?2Tv+$wIi?I7|C*OXu-tp9)@ME0Jy)TT@
z?w2(w-@gBFZ?akPvA%->dyVEit%=XDkl}Odm1;fwP$Dl-uOjF6vfaC*B{{*~mq=>6
z>%idkGF(H%xji?%ms!$y^_o{wI_r%*qO4Z;zf#eWe#y1b|FiL~-Mx|Xa%MQlX}-*{
zI;6e7c3R;|AuT3jk5c)k2ktqaiK>z=Uf!pE>yM%2<s;Lqy4US$^vo=nob>kA_R7Na
z9zo^TlA)nZhc9rHZ@YNyV((?)1)CP}v@5dl77H;c#lGFLt^8p@^~x0*$DbNFcpZv;
zyQREjp|x#D$dt!rRvzk$Taq(d<|@kc1qMX99bi4{t9xTtch@PdJ8gVTjwu&5J^#MM
zA+6)srI2eO$rBbX-Pu}U<m_0esoJW@rhi1sVb&IN(+i^B0va02)@<o0G4ZjOwrs}|
z31?xKUDhW&nin+*oQ#+;?OK^p(DE%?<^&vQPEC99?v>khme{+Aa~hpaasAz5YCieI
zjVWOv0$+A@xE@-P+i4P9(!TfycbBHF-iyC`d~&SQdm7(up7WktP?5!NdGJ#O3-1`c
z>jmB5O#(-s_i-Nr?-y`xKC<l~=oHXvFD!QZB<FY(STLE!_3n+0)Nk&(B(&>tpw8(d
z7mB!MG8mt`y?*bZ<L$RBe!NMpxRjZ4+-;kz#j$-af+c31-E@+bV<NA!#h$;p*Gvzl
z79aa`>Go{PJ2Rg&vdcd|zwfi}nP<8-A3g+cX`7sot+Vr$Ti(uhLU(q(SYF1hXtC*Y
zZQZ{QlUsrlmIn9CuD6We{nxuhKH{{$e3a**lEkG-S9JGG%D?wd_15>C2lM5s#I_%I
z-lg7jINBq$bM~@ld50!)oZeTWzCFHL^SE8yVSTxm`iFDlqx>$<<NY&#{#q9awzz!L
z`akONiKnyoCw^^r=iJ5O+1`J9-?!sB^CD~$RO4<)-i!as<Zriq)|3eu>zwj-J$~K8
zU-n_~eEYLgzyC4p&XWCeegEH<UES**pNcg7^JxFSZf03dnQg86Gj!_z%v~Rtu<cWP
z{rB@Fuk+6M%xWnJu;0|R?U~x5yNRck#~u2<ztlK7uCncZ-3HO^=UsKg)Ytyr`S|ty
zM~_y1{mCKdD=hQtM>FGfs~a`icJeEvo`Y8U2JqO5w=^EurmM4h|Lg=k#TiwM0U9Fn
zSyo?fKA_7I$L#i#)%}owr}NzS#j0GK!Vb!rSvR)be4fME*6zK2SKnOga^rWasv|e1
zSaqJ$DtmwH><;aBPj{(aewlY;TBt><!sITYmlLO4KmXv@)6bQe={@%Uu7ztBs{VYr
z`TPSJ)A{>%J~Wal-*Z~O{;y1Mx?l3O7e-6w>Q8QxeE+C*s`f>#^iSdCao#^pl*!dU
z6Yblk=jeaq%+4QDV!AWDZ_nHM=g&n`llYx7XN_LI^C&!BTYuz^*WCX<?*DfwKY8@e
zPyYR9w}sDtaI`l%a$kj%{vYt!oAYfaFG#&8>sR>qZu!QXZ!CAJxXwl&y}tjis@wEk
zKhFBspL({{I>KP1SmV_Bii#$?X7PVIvw6M2|6jXz=6sX5wx+7<dDy-K|9`x{pYZXF
z>7GA7gk_`>FKBk}sQ#C8=0n}8<7Xc3+AaRjbL*$e|9`Q{%jTqIu2s2n)AjGw?M085
z7FT?0{GV|6VBWt!`u}8BXG+`uJbGFtKiW@C?rB7JQ-0llyR<96D&C3z?@ym!J@e(`
zcjs@h7Zv}w6A8Npu=UXtp52WHHtC#65oy16-yz|q*~TrcmxT{3^_rw9(zV$!rI%Sx
zaYs>$+Xc-Rp57(wQ+BX;#uh!6n7eMt5?e(BU5lHoaU8t$3LV$qE>H+@Iq?4I36_sm
zvLY>9oUD6VihHa7-eMIx{BoJXi=&UHO%V}IER%h3Vqe&jjAvi(?{(6;=6lTR!kwGV
z60!5-!xcPStTHltil3gHZn$X^&*~D%deb~Hz4h}~6s!E^+jqKK-^P8}@?M`~r@~?<
zPI3sf{k+U~$*IYUvc7oi5ndh`9qX9%DB_LmZI10*3sd5`zujmwVwuRY`uN6_e>$hQ
zV^c19y7g)#e{Rp;EPHa}G=r^InIG?N6)5@t|EBf(A6l#4G}fm-`x<vy*h_(PZ~4oF
z!|QIGz7=b|?P_Rwecs0FTlKq=4)%EkPE@)yb={@th6ycKrtjJFFYVd)<J^jk3)bzD
zzVh>3x?%pkGkc%Uvo?NzulT&}@=YyWZ34f`a~^CyHTO#Bb>Vf~mj%Qv9$qQ_k#_g~
z%$*TW&h5YFJMFfM%XG(C)z;n5&bWjMU2%QvJ2TnG%=#}IpMtvj?7o?iEN{d<Sw9gn
zZ~yS>aZ8e6h-lT9x5qc^ySwov>oK`{nbr6H%}TJiCQ;0w^wspuy|Bhl%)hQ(fC3An
z+&l#jwaGJhnJOjg_*$+6g-&>?B0BYI2<xf<QGT`Vt!rJcUbyf>bb*FIm!l3JFWb~z
z5^q-cr7E4+;@HQ$Exg&0Y4xS3H^N&DAI#@E+w@BI$9qBM@21P_eJaJ4@V!*+{PHHo
zs>H)ROytw<FUyMO@ju;Q<iZm2WXYsm5ut%8tg3>E&vsw_QRUvV@i^~c5%Z0S&I@jq
zGOb!AGv&QQ!6D{zoDGhb9@@LK9kP+%7#lr%9ed-Zx7n5N?!UjBdcvSeEG0N}&*$pz
z@4T&~Ei2!iESMSi>C+3-6J7T+l_aNlaxFOF`O4Xl#WD46ljNS?FTMICE!%RY&zvi5
zW_I>EKZ|lm2x~xa8}m{_j)#n{Tnp5uUg>9M>YAh?I@>Q}A@^RDiQg7~`TJUBqK81Q
z+y3t|lTTg|c@p4vY{jZoiYu3V(c)kd+s(cF?x($+c2iwuR2thSL?1Y@^Z6XFyWc`<
zw&t-jANNzRKYshsomZ3I)_gk3wToZQb^bZacJXDWSh7}gxQfd7>6g8(my1^niaz!`
ze}BK>dh6wK_EL+#aBF-PpTKQ-Y!WN?jYw^yhes|}<eohp5V1c)kNbM=^BtGZFVy&c
zoHfaAR={M=Q{LVycvu8oWM9s_`D5=nc^^LM{1s^j)i)o!JJB$?@8AdTFVowOtrwfR
z%YpY`fz0yT?RV!?*|Rs7Ub))3!TOGctbM&=NN8x-`Z!*P{dIr0+{lx%?=~|RSg~pn
z*O?TPZ4wo$S2dd0OMLm-^2k%^p@7=vsqc5+=XcruFHUN^9><FI-0UBkG98cBYIr>^
z>xhqklEd43U#Lsz!Uc==O)puVy$|3}51L%NowMz!Q!2AX57WJ{I7TgnS3Dex8s|or
z3A~uq?8x96I%}<?{EQZJzl^{@L4zonEAQ_678Yu@-qgEQ?v?EC+i@qEyYtQcbBnId
zesN<jfA2}rS*JGb+LN-i*tqytj?vuZ0=GW%=GSj&`MiGTq+PeHHWmFY^!Yz;&5YlQ
zUnffRUVe4u_qvS>rhI(#;C|Plq_V1v>nG2@b5412W%K=mucbT9t_XWaNLUqU)FsQ#
z4+)z*d-|$XJNIa5z3Z{L^hS%faONRB*PQsxO}<Nczr2!txS9W++ROJxEo~F8-Oj(X
z^svgS)vpbAZ;~uJQ}AlL{&K5;`yneXJ=%NUN6mC;@s%rapY!)eoqXOMesO*C+?&SF
zAAAyjv_(QATttLb$B+MQ)YbC$+xDr}8?IWlYKC>W-b9Zl8xGE2wQ9!Mv}679{%L7y
z45HQh9{=+E9uz3-qVB1Z+IprbE~fBmX^(tJ$&t=^GHcUsK7YS?%aMaSgpX}{ditl$
zr%OKXPRbX|Xk3u>o$cS(FPrD2zqqifalyhY^ZMPTLZ{1FEIxmlJ+tKI%Rd)t3eTH+
z&$P=|{p&XW*ay3}E7$8jZtk}~a?ke24nt*wS@r*VJ}*B}s*z=MtSK_zC^>Wz<6$=W
zBn6L6S!a#z&Dz)VW?!;~kGbLRyVq|jrxzc*ss701|CWd0wO{4#)nr@Ebp1QSpv>ae
z|4T9og6p0hHjjUL)!crGmJDmC>h9!ny}f_Z)OudGezDzczqnL%&$so<1AomqGU4H2
z1CLuSu8+U>oZ56V|HFgs%a-3B&Hi~=bh+}8tjd26t!M7sGUfLB+Kyni^1nB?u8G{-
zrar$$sKIlx+Qs$p`?uxoIJSO<nYn<;&KKJzojj0XlDJW&z>4>c-dYu*AcN@KwNXc(
zq|Dj%OQ3AI@ZX7bJ6RedoOM_4`!{RXlBpAw*%FOk))lp6I|^vr-`aYKZE7BC<(fZ5
zjFa{YH~$h*l35twC^Gv_v#E(mU8M-C2+KE?kg&I*E4t-;mWE5Ve@$Kad5zGPtkc1v
zU((q3{{H%Wt)%>;a!bq7ob)@HHzRyjhA&%U{;<q?f(v8w)!xJN90Ec&asPT?Tz7P}
z{@#PS^}G6l&si^g;mp9$z~JfP7_x7|^yPa_GJjv|U^3%{S&ok0?9LRUn};68n%uB_
zI;Xp$^wevcZ|C<bbFfwQSk3#hCr9aU`=@X0n|Hq7cl1Y%+^0={Ubrw`%sBYCSjm%#
zL#b@L^yIS~D_-=b+1RnO&7M73J?_Bi;ALAU9L(JwkZtt%qov@|DT$_goy~uh#2=dS
zT>j9rFG-trf3!HGv8hK%zQ?ga=kb^A&-ZjL|9H)P{)ewkku!SCj?FEz+giOpF}1Lx
zSZ{xeTxP(w&zbuo_VpP$_a}aQW?u2*{r88TD+`}LOE7vNpvt)Yc(bSe!c_^ItO~f&
zSQuG6`%gbUm@;8S@WI!YH^13j{&UGPsnzO}z4h!Q+XHj1)C<kMdGw^Pe!?m3_U&K8
zCm(yj?&0mNY;8SzYxebRziOBJ&Gqp1w!RVCDe?4a>W<r`kLLvYEvdO(b+4wLZT;?I
zi95#ge{D)SG~>kuJ@&xPjn&ul46IL{{!;L3sk!lzgFe;!o7b`jxtPond6o5Tb8fVu
z)oO0lX&#ctuU=DcyuCm2+^78Q!J=LpKAb)pV^PQ?`R$5}RZ#rtYipxd@VR|VO$olR
zQrSxN(5j>JjeZ{GUm0Me`R2i~nG?_ZPR=_%=N_Z;qQ0}cPi?IHlal_!)9uyGt-Dzk
z-}TN5v-)~9HRs-@-i#m>^ZmxMn=T8s%Ik+4$t8YYXWNlxys?epVavpKcdZt?yt=Vz
z>CQDjX9x(eiVCjK)PMNeR{BEZT@lf;V@X$xpZzfQcyBZ1-`yh3`+H~d->aFKpM3Y_
z)zuw!cde2xaZY)A^W?Tz)ebi4x!l~I->n!tjw!#nvu$ea@70FZwIZUT4@_R)*j1;h
zzu#nL-eRp6Q2{RN&byqPC)DF7zjJ0?67N=#_qxZQo<8#2Xk+#BcZY8atxiAv>}<$&
zSI^U@78fu7CQ*9!xGz`yxf#a}`~R1&-YpknQONUgmF=F-A9Lh>9RHj0<Zt)CxpUs!
z3hX(Tq%65+>5@g$zLfnGDLLVHbV3koN}$TE0PmmcihTU!cSdGvbaEWsUA9=_P05MV
zPWoBLn0xPADKWSAsyyz^JX29FrQR#wxbKU%-JcY}nSakE1pKdgwd&N3EveEITO8u*
z7-frs-mmqj<&EB+w^&W$=-g?Z&;Rg6__i-Ucz07O_hfbdv=b8)8GPp1NX|a%mYA3r
zZ65bH-0QY9hX=3aq?0NeBi0-(bPen{nsnv*b!C;qF3OHBfi^k28x^`j-u~@NUv}mh
z|MmMG&wa|fWp1;FEmaA8`kK|avi4gzkCfrVgnRFg*{)%aX^Eb^p>p3x=I5M?4xHP1
zIwEG-1w;MbuiDcV>Q9iku$*J=1tazyTqiCnc#G9JR$X6MlKtHM+9&huj6aV~Z0ho4
z59QUG#1N;fYjcrVQPj0niT}jLYwCTg7Nmb^DSlaerc>gK*%vqE=dpI2b5Hsh%{($Y
z_DF7c*@3)^4`oV}-u_zo!sF4YH%H!<|8LERIr!9I;rB0(w(Y(ZD9oWWQ|Cvm+^Kz$
zXOG9PSG_6G5E?2fQyFWSc_tul@8`T#%wIdIjP2&L@AXT$a6(w~jbVpJ%)y5xi_|91
zSfj@BoZ;Pxx4m~)Ig6I~?D<f{vUg@mklN}Dh4L-$o9`}izrNCZ``xSiJlW<S?~M~O
z6z6%;cgOC=+A0y(gYV8hn<bsMTrHsgT*kpPBLV*QpO5x#zZol^SH-@{O8Kw5tpERu
z!Y^(V3mhyE=|0<d`>Bc~Z_)MnyB?QVO<>va?VJ6?0M|KX(%bLfZ2T3#vifYJU(m!a
zYCTqs+V6h|tId|R<ZSk{o4s#UgP-rB&nd-MW$O>U+Oko9=7QSobuon}Zys?{T;nI6
z*UoFH8EjF(^mekARZ5Z7_G{PT{bwDPcbcS9_5I!1dA8Ln)~(C?DE;_r)yuoP%_m*S
zk&R&pIQjb4Y~ypb^FQ%gp81(0Hz`i(V!+An`u#Iav-0D9UH_lR|Nd#Cr}?Bel{)@+
zN?Ezn_MUALvae<P>X-X9<BxClid#qH-~Z(H4$C|C?AxSw<pO=D*H(W&DL73xj%(UE
znd`R?e&W@R++L}Fod3k~g}Y{`zyHm?dH*lIxd$DBS~u(MZ+*O5V&<oVHIr7InX`S~
zk(ADl$1k^i{AT?Bkohm3OFegYex7FS`MRE~|Ns59Z(@IHn&17+v-sTZR_<e4o;TLt
zIltZdTk?gK4^BLu7XAL&lfc;eEin&Y?X{`7eKx1}w2E?K&0P1-7v}3f`CD&$>Wt`p
zvy)ZNmYNse-T7F+MRwsXTbqLWu`F6z9voGAJ0AMqDfs7^6J^r>+&<=n?b}-$(|&2M
z(aLj3)BXSFcKnaqi~ASf-Z<sAeD$<XD;j2XOpd<NUj45~K6aDO9^<E+|Nb7!5pi^P
z3VTs{Ch|_z&5e6L-8tQ|$n#o_W&8ePId|%Qa{Q}0zqdrb+U)z?f9~IMV~+XOS4~Mi
zzq?o4PupsZZvVZ{>~?kk1nr`}=}x-Wu!dt!@l)NH%6p!^(TtN8YKhhS<1}<utKa$h
zzeQ?B_>~lv{h?0})}MdUt$!`xiROccx77mIc)SoU)XG}DF{R|;K6bff`}IrrcQiS~
z-V}TH`*Q3G*2k644%!Rf`)O)^?~C{?Ui;bZ|7v8bbosU(d(V*nTP~*JUgV#j2g7H6
zJGT18_j=cWg5u-y=iKk@pLqNI?)mS_rYye~ea<Ft@7uV(x38bR-2Fc7+pV(_K5kA6
zH6H8lm}kA-PQ2Jy+R@pWIe3}RLg#kAn4*)a3^u>tYzD20s(!y$eD>LE*Un!5AzD--
z!NJn_=uwiEmevH7m6`mh1`<BJF>k+J*>~DS$W+of@Qm&zX$}V?!NuFm9D)Kn+?%ej
zE$&cRxpL*<M;|84e5V`}eDD|V?2|99G=EHHPc(cLFjZyh)6^?h+BaC)q@?}0$f<ic
zQgspUmDZ#es#9mpVz64=d*;lRW3>~wRw*ssZej8FlJ>y|PVHVwQp?t?nUfK8@U{Q(
z$6EqJqt2+ypIxBvOy%@l`S!DyuC2O$;m27;?F$E06RvvpmNklTvd->Xa+BrNslJ%L
zCFYMl{|<h>TC1#mvF3((DlZGIc{L9&zSwlIM8U7I;qJpj=N!u`yYIGMExeGCdH%$r
z@_Np#H=mq)^JSs9Lw@<B#|5Ubt5<1p2<63Fy-VA)auwGk!R-f2HpJ|n%zT@z<X7{3
zSuN+u2Se_EstOG?jCzw2CF5&lELBqS<{oprPGtd;hDXXy9Sx2hd+P4&Yj%s+d$R9%
z%>|9Dt=9an&Obf&uuOi=8uNLRt)~6Dov}GND{DSq`|Vy0jT@JYJh#s%+_hI??>w<+
z!7pdO+^>qh{j2X!s$~L4#Et?*uf-F8HatA=`m0>oJ`K;BuScKX3EDXMQta%N9o#V%
zNfKtlEr+&VQFGGtmkL#jiFP&;VG?yZs1RDJ_T)+~F9(OxnwjR!tnoFz-yfzIe~fgl
zsDH)XqWm$8m*uPI?#OLPwye&6&le>ZNB&-3zv;*6^%i*!cG=HrkFd|zUwXcCdf`h+
z=8Ei7pLJ$fc5iIXe?G;q`RDa5l{~gDYtC8a@l<VXOehs?Yg70)=bT82<<Fv{S>Hm#
zHdkG>`gZ-~mlyXM&)=_~s+#S!qv+E77xUzvO{}*&F=hL`<QogBKbYK}R-m%;82<_P
z0MSE@QL7H8>Fs<I^y~Zk(|t=fSG@nM`o;3$f;I7H{<Q=j<8u*lP&;#=aQcPI6CbQx
zJ}>dwb8n0PUy>&t<_!rCohZ`VWB1Nv-=9y~WfRYfhR3>3Q#^EdvifH0chAo4&Qse|
ztjTX@+xy)v?!eJx;~6$znb+*xyYuZb?H_s9cKiI#vb#Uer0w$gxd*O)xA!iapS<JO
zC*#6dS}WJBT&O6+=HPeg$~6{_*2fFBT$+|T-{|dRRUYftTIXbMPd({xdnS8weBzM~
z#V;qk@7iB%t2*^9<Z*_5gI%mOlfsKLJQ5egf4`Ow3fw)nLTP?MF0be@9ksg^wp+8)
zKONelAGz;E^shNHce#az?z#E1<W!+y_EWBJQ^EuHrl~GXW%PS{N&0r6A=A|hKjIY(
zza8o9`S-*-ZQoC$rd35}B(EFn%dfsy^F(jjx|^p%x8L#nzD0ol|7XXz%Gb5GtdAQc
zAJzI))&F=`w}0~IuA?ljofS5nk@X)QPE_gq`1ttc-R1Aqk`Fwc9)E6E>FX$|9#fNl
z%(wo0tx0&JcG|PVN|vp8VdmBQ2_C!$eAR@l=JL4)UYPmyh!7`_+vCL4r9sMGZ~o><
z`uompyqDwF=gO(hGv{;r=?kH`2~U5mRr&h6;xDgq{hlNx>FKqHsyB-(&Fnd{h-<N*
z&E(5{^7m4%in(^~m7ezbjPz#1s53_v2_AZ=ve@5lcBinL+0q+AS6I5Aopt-b|KQ9=
z*P0Kn_D62pDbl>8>EhX=^*cIrx$P6vT}_{EzISk;uto7950}u-N*6BtSSrB6XgM*Z
zI#b{y`=V>tuX9&SI{BXS$>pM=$`{)fG|#X9vFF+Gn3s_|j5qPGTxEFRq2M-o*-Y=C
z(>ruueXskTe9E`huEOVn<cas!9&gzDN|#A_o5tf4yDToH#O78ycE<&WibjgQsDHEa
zOBhSZ37K~lr4@7cf7Oj~b#;AnXXoVD-DQjurcZYd3=Dj&zIwv5uNQU}Gw!rg|9JlS
z$D_yPjTgPx6k;>4dD{FdVmpqVcY6GPma;}+*&9t~Lrcro{LhR37{+d3E#Hv!SbOca
zx6ii5Z``TkaK6c^OZ&|Wmx!NNi%w<vryXBc6w=*YtZy;-;a{sui=)k%4st~9+<9d8
zR_FW+6Yu9T#C|U3n3k6}>2k_<!%s5~o~g}Ki(mVz?X2{^!}oXE-@RQKu`ye_OCa^{
ztI9W%LXx`XRm4sxFu8fi;li$%(;D1!T2~t`3%Gyr@s3xgPH&W+v}ezQ+>p-;H@SI!
zkC-SbxqCBn`n*Cj*1oEJGybw^OD|HH^y%Q+$&(rnt-E{jtMtw3XVOkwP<*j^e`2<$
z?#3mD{@j#b?Dl=3xsySR-_Lh!Qc4AXIK>UpZWy$cy+5Y8mGMsI&6aPj2CsUzWZ$2+
z@GtkuJF`Bi2=afKJ>%7J=Gx_}bZ;A_KHSZ+nCqzYHACASb6*v3IQj;O^D*CJ-p=IF
zpmq6AXNc%h%^Rt&t_8g`PB0Mmx0`3XkK?jkl$(uQo*?(pjL(H<44r3w{JMKh<X>IW
z`*}xRoJ>D)K>5ykDXIN|I{UuOslOMw=Eplnk#t7!dBtjCrcQ+)%|sMW?FucvdGp6T
zFP2w=J!V~2zaoErWs+N`wcLIFIv&ZOj;|AbsHSb6S@`(aR*Pq!fAU!`TljJQg=cI{
z&$dLIG?MaE5mK4#sp6?L(??REWRi!<&%cwiEGH@ma1_2;DQ>*SUG4q8FFM})qfYXQ
z&pomr|GMxNvBINA(-*On8U1}+`Ni^^j-S-k_O0CMpYHYWzr3|s+xAn9??JZ7A0?*#
zJ}p^%Ep?8`B6IC$bq9_c+b@dWKab;4<H0Fj8dn-J`~_CrIeh5qY>Rh$R!c_BIM=0m
zy5l&j+x!Zt*ZXg}KI{}XObn4fYq4PAM{Z@$Dz~)jfpMFku`6HiZt`&3#p+q~&p-9g
zR^NnB*M)1Bn$9>Q@i2Vp_Wx>Wn`hSj{S_ECt*G(k_4V>;#jgW5h`KIY@LnKqMu~8n
zhwgV)ub%}+gSx)YXJgVh$>Hg$Ce|%5VRFrW<5dkRUK(9K211Mqw*;IfM!%fy(s8tV
zy5N+=+rN&!xv3!-=#o{wCi(3%+g`W-Dho4M0#>eHw`7f0i@EJ;!yn6TJy9&2aHVAH
z`w4<6J~NL#m#=a8IZcagel25UT(;JOjR#LJXgqx4Tht5t0)`0VjkA}?t6yJsx98=A
zDA@vwGZIpFD}r<LG_xy9C8po?(K2doS$9!l!DI&ynV#zxVv0gpMV1zJFX~vZ>PJzr
zq*;-OnBmEWM$;70ug3&#GBi0^+-hKIIsA3XJ8k#$&x$e}UAMmIaIA1J(ObMifGbQ{
z{ju-W)mf*vbuhi)VcW$oab&`}n>;VfCloLhNhQlO%vj^MspxOb-ZI<M`)+#O>xk3n
z6gWM7osz`G$@4BsHtTIIJ)U-Pv9P0nYR1tOOB60upYSNkxg7cDCc8^{?xNoZj9<*C
z+_Am8?O6U?y}xXiE+(qqTx@P@mEIiHs$o{2A1P^&p%Ah8#Hm*&mRP?_{20Fa=I&zg
z&&&6pT%@}D<)>fCB|i2$u38CoZYX&fRQ3H`@A>JcrUiwD9^F;?TEZ;nhG4z4iOIhH
z#fKaUUR-cmetF^b*McmJu7MopyUuw|^4PrK?)~URcW=)-SX(_|X5QlR=}cm8y%V<F
zFqrMDves;mocL|?w`zHQ>2FpfrYy0PFe}pO7GJxlBA(aw^B;EA;Ltglb0y5PBxYvv
zx%EgMza)A;FiiT$4KJCD)rY;?efYW4YaMQGW8&KVeUbNbnI#^oYK!^W5BDBUQ~SxC
z5_sXl1&a<1&n>45F6$ls+Wfwgd715&5C?br_6o5KhrB$!*j*)=zjlX873bweZf<)1
zPgRirP5Q-8UnAbm{aCrj!nTg-{<<eO)#V&l-9LNoR#lbUjxPfGf}w!{F1EH@+#6QD
z|9p+RKX~Up>C?~8sm-wXBf09{_PI;FuT8K|u=)P~r`w(|*=+mj{%ZN^PJQw9kG?EY
zT6606o^OAIuhd@eKc8Oju+^?#{QqAQtI3Rtl8GmK%3sXOPIagbHs+W1$iF|;(RTAp
zv+u!Y=IOM_8w)Hx>Y4a_Ud^5ozN4X~DK~_5W|&K7Kc6_K<k1g#_3MjLc}~jSnNnta
zwMy1@w^(A}vh5SLy4~+fpTDQSIWsU=<J3VPx2)t-ej%ccQzrCSF-7l7d^N?~;{TV4
zU-ZsAyxjYH#p?da9&eJr9sev@nrCL)_D$t_-m}Tymp>9`e)H#7@wB%qo~`%a<59m{
z&X(zZ&DIw4W#3a`)|*^9qZXzYd*U$v;d!Rr940EW+w@N<=DxhQl$-Cbj#t^6=Df%?
zaeEr}Dr!3)pL=DA^4zP(Z#?#U^m(fP#~**|lizPYF0}f_L3d-j!*^GTC%b=h4p|kP
zYa8&b@n7Z|&p^I!)9u%DJO2{@a4h}v#4lxel2$Ue{ZDAET=+5DU4WG()b)xd2UACy
zdMoSR9~CdtMZ&L7e(7OU|4!#kYx0YUi=XdIk3SfBVZNZU+lI`)TBjZ-K5<^8eR|WG
zJ3EUb_g0x+k1dyF0NqMtYHG?OXR~AN>@^>?pRO<vkg@t?{a~M3mr~kh$;fH5kIMWg
zJnXX2aQco_D>S+HU(g8*f6PBqfn&v{rb`xq;Zm1VWeYZousmEKy6>e8qsY>azrtoM
zS~WxVWlYuAuQ|7NOguZ={L{J9`a#0GCUG)<4r1eb_~9xmZ*RP```ebaAAeq&XV4oj
z{lBxlMa7dx#$v<E_t!4G>`h5a+qi#I3;(@J?$9YG-dJ^PR-fm8-EXePyeh@}qV}h!
z&p*G=`S-b!v-$3`j<>3GTI{X6v{+9i@IX@e#F;Z2k1kk#{03)C;YYP)XQK|S@n<S|
zf33S_Bg^cqPq*#=v^TnQ#@ilQ>n=H0*@+CFH$Hx0oG!P!Ok{HH$Iv?;j>qo#_pT;x
z)-KNUD6{$<Z<31khRN)8D}J+MYt7%YRnyi-I{*K3@yBK30?VM)cNX0|{Nm#AO<#5g
z8CCZ$Di1VU@c77Qz4Y+X39Os%r|smr*W+|f&QNT*=byl$w>P)WzV*T3kEEo<+`eVy
z9{q*FI}+dL-K$tXbItU4pSW4l?Z>^Vc6G(6H?254cb;F>rDm0`h_`x1Mwbj<si>@;
zzEOSt`E9fKBe(r9?z4Sbv2Mp#wO4YL^R~xVvXv;c*0vc|Rn!~Z-nsMPHR+1PQ__3>
z-tvCI<L77ho}cIStZvoB=S{->&zpb78>inZ>XR>3o*rW|ae4gjj`QjDPL7UtZ$GET
z*s(8K;{S4OWOP;U>;qq`yDJ~>ZMFIKBB_UuUs&H}cG=6bwkuiO=G4g5_o-BTJ^TH~
zgRkKMA&q=@+lyary?%N5;{Ff+c8R6FJ{xw)*tqakuJ2d1y?!o^t_BtsX&)T!{m+qH
zwuX;KCgQ@i{#k8qvlY77YF<jNunSLK6~HWfZugu=tE!%zuj+oP^}~qEviygHmAJ*r
zdwO&EpVgjtrLV#>O-FD4E2d9>bpP(jF_Q-E=k>Sw$P&EVk2T@?x>%oCCZ5~@D?UyZ
zG@H(HyJU`hr^^Ou5ut@rtFp5UBzR`{sOgAtH##&_*z_Gta9Dmh(QNhvAHSx;m93|H
zG`im0-7|&#-&Z^HyT8M4?Ce;6FY@r+`kiy{m`Hcun|o$aQ0we<`zCCkS3hIHT6Xi8
zO0%!3(m$Tt%PoC;fpd=4E_?0$KW5xKT~>H|@An5BqMHkzs=3W8&|2slUvNJ-ShDES
z&p9@$^yAO|Y&d^?O40TIcTfLQeZP5w%d=&x^;2%`pYL==;+!V)+&$*Sg3NBR-qTc%
z?f?JP<X_$L!kd}Q6|eH^kKX#rqGy+E!ngX_x5Xb{J5NhxGQGOQ+NC6_E}OSEo+WY`
zXZv9h5s?IgKSrll7a95a<}&<j*gi`kVbhJmTRKaGKAi{&pCZhoURb?nLeGL%K09R0
zq+T(*q%f|(?P+wyAy%X5@Y{na7bbSNu2{8-?PSaCR8@(b13AS>#`06F#rT9c+BRC<
z@jV{ux@_IHSv%Goeq5@o#i^yO#c}k}jX8NWa^a8Ts-}9ZXlyHBX+GI>H)+wjWo!pC
z4!+!=$j})v<CcSM)nw7>hwr6q=y!L1vwnq&X7lC8hu?15Y9Lm(Kc?!Ao-}XiR^3-j
z6NC#_MV)x%CKA$A%BWQMp(5smOY1HN-omR5$KNVQW-YLjIr#Ep(gjbAqNZKDtWsPL
zJS<AQ;Mu}t8gh-rA^pJgXUEgLB%?w?mu*|s<xtDfeA6vz7Qb_{)*%-iuULhS1?CTm
ztQ92}*Zi-Ww5sL$V<jQJ<V{u^W~wk<HFeK8UFWn>C9GX|{`KA~r@Go#sjR#c@8zq|
zqVVd&()34#OHTzpRcTDCRXV=-3d<|QXqg*2R!mn-SFjn&wNI6mzW(9dYX8s2dQy9A
z|C`C}d!BVB>WxY7eZ^ICmTcR_){}N;?VjyIMmz4<-C4){ey7INsZ(7ZbLy0d*R7X+
z$+6Nge0|(mVSgLO14}$7zqr0WKH5C|arl|QhmH#tJ^K37K|sKDqh`UR!w($-r>MPW
z+<x0vVNTA|stYfDci6^#nbeW6?ZL;LK|l8f8b;(=yy04;@TjChP$w)v^p2#gW%;9%
zd#sc7`0H%tRgBgBZKnIp%rn!AInwoX^$WW<7H66oF1|{+-?qNu&+AH;P-gYHrFL&G
z$6SeeYawg2XhK(9-`S3<Pmj-7^Zcs#Jg4Qt8-H6Fr&ZXW5wqUQ{^il(otuB2Nq)R$
zvRh^$he_Skj!7<VX;x~@?>Yh>iMn3=){~uKqGa-KpX|%s6*hi_g&*~!mM<}W@O^o0
zzmL$qFF7^7?tJUaZqChn!+vVxP0!6YXYac#eKsZSxk;_rOg{56Y2C6*y5FXK?G)0>
zH?CA+etGs(W|>;rYT3<SAAEi1d~2b3l!v#yjqrqnDXQk@Pfh$`ShO?x;>;!azF`Yn
z=Q$jD`Az;$%I3%U%gh5WJ^uD1YDz+B<%V50Yrpj#pWk=9{zmwVUbUK+%bu~HICnDp
zSH@<!oc|MV7q#SS>V8?ItZUrA<xJSV$tAj{u1WeVSt>ha<x6YRW79vU8y0^wtCBC~
znjTxQDNW|WkGo19RyLXK_LJSK@?Z9<Z7Qn!5y|@E_I7>q`!&Uv?@LM9w=X`_d|;LT
zrJb>An{<LBBX7RYO;D5Jo9v?&7&^75hsPx_<@P%xC(gTxj^<BN<|H_a%s$LlG^fo&
zX7|}P6|N(T6_Y$S++5@0%&XDFw_5e9v)Y1y=wM%Gi?)kNhhG;eb82-)Olb??X+M2*
z+f(C;UwgOSDLQ?1#!{9^2WM^C(Jj#3(s?pr^Q_JmrxlYJXW#Tt5?i)w*R&QUzv=Al
zXM0wyQV}+M^{PqYOxT-Q?2X6P3q(%iyvF__c0%vP3%hy)14UlFdKJZKv~1zW-!3Aq
zbNrqQ_Og7em?LXlHevq!|J{`W85vt_xlE`2UdemxL&fC+OWj-F^m%!Cdl%1He5m=s
zfwX58Hh!t8tgE=hx@QJx^eAxL$l2E1yh?e7vC3S=?YCz{Z4~j<{ZuV|X4^838`Ipf
z>w-V1y4<;EVso~ra<$eAp0qn|#!U)ITW*~^@!@g7PS5gfJkwGPqGzAlcI%DFQ$_Qr
z6DO=59IM=5H}_u4`TQ)!=C)(`yn!Nx7Bcqx6?l8^3%DEcW&ANoiOkCuG^u0J%eQ)W
z%y8PGuQS`)+KvVK&z9z3VVtzd$3jMNp2vgF%Z2-oce*XU@n+=%^RU^yZEneqLDK|V
zt{t@tiu?3Ta6|tT?mr(MXFWa=r?&mIK;9b*hS|RTH7~keOgxomQ&&-{H7nNP@%IB0
znU-f}WMuq_5pi9(Z(p3TPzD(MIkaVGqr-ah0CA2@Nq@~kmKu69Y~lH|{@UV2N?l4~
zt9jc`{|jFA>!e4O^0_oQPALe;ZGX0O$wCnLcz@wpUY1EJ4?q1V+*ND;s^+1?uY>;$
zXB9d)gTUh#j-P`rfmag!c$!(v%(&(5l6a5=0xKU4OjNjV;ldBw$){A~?hCL5u^nbM
zwCNF(mYXgt*t*NX^x=%A>(37_@Lnr(>ZC+ysHuwd=RMquS}wmmc58;WP3Rsk-&D!p
zr?dlDY%^cZT=M&FM^nL#uMCq?j=U}vTD4kH#Jfph$)~#2S_%opI|^@|J<)0Lbg62{
zwQI=+w@;irdo<DT-wCs9aA<%CnOYf9HYV4J6Q52y_@D1d&$UH%*%^P11q#=E__f*R
zZvvwhSJxT^PoXA;yKeIj{w=@Puy0`xd;aVrU6VM<|2^jHlfJ;&&}DnS@b8l96)XD}
zU4QMVHd*AW>@oTO6}+~eHamK5cyyWn=XLMko59k@8k$Z{J;ypd?x^MRp2b<`%VW%6
z%B^~>^x{iZi0iSZ?@!+UUneVX|6BZ7-DdevPbF=pNg9m_559QkJ?$6gdA9eY|EF8O
z;}u?iIr50p>bqr|b^gwo=XTDUzfypgxA(qMk&$0$sOhFf%akOzmCBY&@4ot_`TDzv
z?JeuCzfXB^gY(Jrirc$1rKF_nrH~f>tYKG>ShIMYUd!Ew$6o&UnRz}mH1+GNs|;UW
zU-$of=Gu=)P?DLJI_V<Ymn3iI#x)u(kBrzh%PxNT+}=V;O3Gcc>Bxe_$A!<oBs^~u
z^{GrZbM~`o=XBov(Bt&sHSKTj>n5N4r5j?Rq4Oijaxr_!A%+v35~-n#RlPR7m)*an
zJ#P!$^XJ#?6=wYX|5Jsg-q(Bjk6li6R@f2V@;hF$M4oKboLu>)vHfJ${Z=LKIh$57
z*!Iml;CZ-u;R&Nj7ky`)KR@HL{^Hj!s`mYR5}T%X``FFtaqi!1?BDJ#Sy=KqDz2uO
z?QQ$qqrA`OAK12)yYku5?LR+Ew!2gI^<_qyFE4Me^U6D)b+6wo(*M2sRiG%V<E6_h
zqwCe?{b%p9UdFyld7nkdzF3R-?r}4Z+WurVVQy?Yww`Nh>gM|o4vH@?esk3P^KxxJ
z_l~Qd{v0khD$dm0e*d??@}76UHfKj}PUH2r`>Db(S<QD*{QkP!y9K@RhA4~cRCq$L
zFRoj!aB=B*g<W47b6+y$pO0~wzAxG0>lDS6>#s_9i|HP`w9sDoQs}<8Z5@+V8NYn7
znL*1#jjOu<$&F<8$r9J5%idp?knE|tF{0-vx7Y^rRnPOw@0>i@9BhAJ?yc012h~p-
zs7<{h?QRr!Azeg$eu<Bkd->w;lk%TbIJj!uJ0YI!En)kxqD?f7c|+>WtVM-0=lIxH
zzP)vM?~9^cUuBx^O}%&R)RhYtEWA9`lBP_bo_O}fnH>+CS8Coo^_=_JhQegSZS}SX
zPu#0K@lnHid;Y>Dd!$xt_a0irG)K%<iHSq$Vn9Rh>+1pX!YA}K4!qxVDIoady}zLq
zmy10=-&nC$WAUC{^S<2qR*~ZdUUG3;k7e`ZIX?W2uNdBK$+={+ZChAp+p+fy6B`e^
zP5OAuByEp%@8o0s4)MQK<?A+eoT*DUSW$0R8k(AOYg4a>lY;*K20qJ|9ZW8-vTy7v
zl{P)7@otNc)${TLS(oI)4o}mKo?%mI<lMI8nD1nD|6^&J10y1CT<g95W9p@6Y)qk{
z0xXQp7jGy9H>{HK_}uIm@9%8ms3LPHW!oobIped;a;sKVg`~=7oV~kz?Gs^F!O+kj
zhF8t6r$@hZ5O85pocjA$hJ%P}K<JdkFaG_~U9f6p{jB@KyrR1!e_b{IB3~=IytH}h
z|L-!|`|kQ3zi7+I=;*nr{Iq$^-k^l%<*OtA=5ogRNws^1Xs%s%WXj#<d9AVmlLPlf
z?wzzMplROT>PO#S1|%t{DxaDE&1UIpJC}*6F*o<_{5OmH!>upAPAgZecy+K^e{HS!
zuhPupXTs;DJpI%E^5SB3=60FG8)K`o?=eeBNv)mCmF6^kpYx8F(>z~1;w`CsS0`8X
zkolQz#E}W_gn$0nnqE=%iic&ci;d>x^edHr{;u;DbYV1=d$jiXyXRNis#Y-v{^GHn
zx&Qy?GB0ue=i83oe>TUHzwm4(bL6*&(l(#2*{Iomkx8~b6#wq0{j_zrnyx-RIQ`k8
zMO&tE=l!w1KF4muTGK@fKkhzodmp>_ymGPnx(}ht_I}j*^*3Hn<j$Rs?Vl%p&wujm
z_q*r2WuHovXjy#T@pW6Z<+k<Hm%raP`)0OzvVkOL>!P0h531hn`4w~KYz_-AFE41u
zluy>G;H2twh7WHxpEpW9CGu|X_qZIhvjQtV&cFPOjVUC^@j%qclmr8bjS*)o=2Vrv
zU#%h&v>_tb{Z6<t2h%|Z(cj;uPIfsmdz$C2^Sw8o%)ZceHYDX0(^ePZPmYzYg3C`g
z2mashb5%v*_~A9N+vlBWl1tf@Yj5D_&pIKP#g^~)`Ol96LTB!}#c;9WQv=h6n{#ih
zUG{$N5AG(1pX+p|m}z`@5!T}OgS9$XK!Ek3y^#&asx!;-uQ!{{UBoxjeX8KA?I)&{
z#&6uXWZsuHt8ZbeS8QaPrW<u`=He<P4h1d4X*E{8yx#W~FIm!ZXC3p-M@^dzBt3Xf
zJm%6+aDV;d&0X=C-+d)RLlw{Mex*Oh{2uqBj-!tib#-N3{T_YTaiQ~merHL#=!gDx
zx%7K`e%5TyG|mpwPdoR<|3+Xbe|b#VufPvUS1(+!hzgWGt}^N3u@!}nD+_Pwta^Rw
z=H^UQecwvy@ZS%8cD>7)R&O%XJk2Nd-Hzl9T}qu1GTqB3WUrS#@1DQ!#36mVw0m8X
zHA0lHT)QUFx3n($smF|6v!p^)55B6L|LN$Vsb8cj?Zo|MPFSzEIelvrmr`d+^lC@v
z$K@?YZV0Oz{hF*l^XIeE9IH+lrpFxqvF~l==ZU)`cBScu-z`$Vd)dk6&w2LBr)$4w
zzLz=NBf9#;YumopdUm~MwZ()*R3Cj*N&9;+uua9YN8Wzky}i}ulFWMh{}f$Y8@+sK
z)72j*WlA64$Z!?i`r7P)ycxsXn=O9JmBo6Kj5|YEvpTz!o`r3b=3sjBE6*}GzJB_r
zUlwe#Dz3)rbE^1`_rK})v$?W+LE+auhEA2Mcot@Ex?VDGk%^=@U)=gd8d<?d&#w3P
zHDP7hSox&5!z|?UsmJ`AR;u+KUEv-Y%44Q39#?8xdu#b(lW0@hfc91YY-aLH1+fG*
zJ`8?-)S2;F?H4&`{gaBPlifqFE)%W#Dqxv(Fl(93SAnb7l=61J<G1ihnj9meba-Kr
zOjQ1weQn|MD;j!3HDy(VIb~kHwRpM3SmMfJVI}r;`I_AdTyJ7__qZ-KYK`r>;B6u~
zHR_9Na(zd8g^kJd)jRtHSemY?)@*ze-PR}7dOy!C@anbFx(!SLk92+)={X$>*j0AU
zaq85?=c1pQI~$#u?mg||1dmT&54~Iy&BYm-_&3~J^Ikn;nWMwRGjj?Ko}FW@)4TuQ
zOS3~&=637(S!8d@*+2ZWbIp%D>E}If4t}k#ou%k)mwIXMe<zb@!N7^G%7^`I`aRS6
z|6WX*ns@(1`R;GVpQdnI{#H^}ogG&Ed+YTLb$`uUl9oR-$vW6*Vq#J!5gO>|Te&p!
z&0V)#4UbKA;_~$-b5G5^+R^0TT52oHF!$)emEticI3HI%UOr`N_Pm7i*KA8pi3+-G
zd|B=-5X3l-)$-i6yQv9<h4-F5H>?XtZ<&<;BkOPUmY=IYl?qSydfVso%f+<!RSO)A
zJka`lPN!>cK+GOq*N#njS7XY&v<zMSl>+(`wgg_ec>dIS@ec)|mbSl2Qcp~9TJvmL
zu=(bjXLb}mj*|KOv!-44@`WG%7O!|%4!%8_Q};^s&fZ-~o1!+{etRRwY>uD0n0Lda
zRR&#2c@ytC2(U=4`u^$H>h*#f#R@;0rav-US@ZMd@0t(CvQwTuHC#|EWg03P5_Vll
z@=QufT9V<b&erBDtw|hSO%Aaf<xy5<%u}Ynb)VRil9rla_JVP$Ln*6h=vIj{Wt)sH
z8@^b2<?BI?6`oBCH6nLbzVO(vf6bf#lg87p)z3b6oSNFSkb}A3w*4`CiPJPu7ti+V
z?-y-bHfhGH!_UhaCoNgEifix9#?y&P0kJ=YQ_dY(BFu5++9I#2MZBdgi#+%I<GFK=
z`IxAs=})$RlyB3g3uJI4m;dze|Eys4YHP`JwrhEMYRmXHu1#V3Yn{Gel@@O)V^UYw
ztPY_S+=<1No}YILhWzaKs#P@US3*nU<MR)HG|pYQTJQ1anU`n&dGmS3`yF%l%{2;~
z)ajh^anh!GE%w+wtWhty1qC|Xr8IPPl{F<k{ESg~WTZEpd+(9NT_&$x%W~t+_w=Vf
zJeB9aU+-tcSvIAE45?2=Pb_LYv&JXq*|gj70v@@#9!G9$-#sJ2qR(Dgc>3K#{kJwR
zEOZH(B6e)kt4nWnBERt}`|qB!d!4oTWR~nS|Hto^-%C09s#hkSqf6^S!ScW>$r{h+
z*D`&#v77sHzV5?ipN|<Oy{Wi2*RK7!+`jo4FIiJ=IPEcho2%l@*32fNu<Gc9o=tTp
zW#anZ`><+mNiRC|aq)*J9*0%yLtS$(Zha~p{_%Xhjqmr3&x`)8RMxn;rRkDG6XVOB
zGrod*RgX__^#1pEc@^>DXFAKP$4e@U3?!3Rtm@lyC$C4!vgOGF#vUHG7k5o(ZxRx9
z4b13pVeDXAq~XaJ=+)~VFj+-N&|#sKlp3Exhs4*a6iY|fSlI;^1h^Jv*au5V<vLBT
zV}AK!!{j#?XUomq?6}rRgq4R)znP6s`F3{dfd!2TXO*;9D^Gs6qy1?7+O+rQV$aO~
zr`A2~&9c+4`!5>)ec+U-B9!>@($X*Q?n=);&+ZT!D*F1`+T<G>5@(!^T2S8me-3ER
zfbah$8=j>qNAle}!>;qFYOmb<>)s0#J#8i)NH8(8c3+Zm+Ugsd%DTUgcb1$z?-KfO
zN~d^Ik=~&d;x|5?HB$4ho!+rVU6TLu!uJyEcD>;FmLGF0I5>TQ`E1XY!!L5;jAGeN
zeGeDpag{83<f3O)Ds%L{&eN;r_m`W^ahV#*)?+#I^LdMTlXy>8WSweXQDqyJt~Ncs
zR51Jbp3~D`mnC27R#DY{9#>^ryXkAkBEI$d&3EqH35jxdbPy7mrZeO47uVZA9`~9r
zUVraE@%oxGE2qbJFWb?7a{WJ%ndhJFNoN1>@JjlRWhQMZDjls(H~z@(eDq5Dq?xS5
zJdPD!O-GBDeLlFR^~9nMtw@WfZ@MMz*RX3CUaQ&LRc70k&>8;W>#F=0mo>TP&PaGU
z=Vi^VKhbC0Iz3F>Vm1`sTN-Y(<-qwbE6;z(&Hg%z)g`dq-nRSO+fM<XK4naQ`g~%&
z&52J>vtQo4czl!P<;wp8k$uwlHgBBoD3Wme*U~*j9`SR^e-&~})VeqCQ~tDkt%FHh
z6AZjmmioNg@ixn;I?TB8RkiP~H#hQip3B^y9%=kMFgW<&?e+WH3SP>;oH^~u)068r
z>|MhaaN?cK1Z_R*=_Si|woY^qnyeFVG+Q^@Md|C;9Ywo79)GQ;emnQ74QCx2+vce~
zYj|YNU9nz}^E_=*n0|7ftb3lD6a(|I-o?+~>=eoW-ItP>Z)9|7Lv--_=D_3g4}Z|V
zwz=q{&?$8u_2tI}-tGMDH?QhtXk67h!;(ckXJ?;YTfeJMG1hpcmY%Zl*|Y0PxhsFp
zc3rWm$9OG&a~of-kpN_(Vxt7#KE2(&c766ax6j=+PXAcH`k8>gTBiM@6g{iwE<8~J
zOuW3k*D}5^+}Tj1*>68*N!EKYF`cMB?dANBkN2sZ4{_Rd`rN%wLQYyCInihTd{uq&
zMW#o=!AR!8-RbjGHZ0{|Emq@oCTDT+`Mw=f8Qg?){uFRM+V`}^GGEZs(Lwafm$#RN
zpU;;6d@7iKQ?@6gw)i>Ku9mc8S1$axuiU}Zv`Xn!(8@Wkn<pP?imTc4#kgzr>ysra
zGwsX8z8y0Ba^m9WJDb-((0OsbLs0p~mO9a=9}Bd6vy4+B)85_Lxh8J!EYQhQ4b$W6
zX3m-=#l|mp=bG;|aQ~#~umB57piAGej_cPJZjKa`{(j42Qj1EbMyK<)Zf>ouzgKTq
zwq05OaPy)QE3_nX>-;P~`>=>ytKB>C-eLcQ8xQKH<VPKwQZ5!)c4^}TkDzezr)M_n
zEia$emo8V^Q}x{-Z|CQ%YiBBFpWhdExX`)E;Pv8ugJ<6!uj_n#b9bF#@csF&=j&<<
zPkFwQTI6H#-^q>tO`l_r!mZ6aeyp1O@`15I?kDawQCs`uESo{ymOoEoy{;`y*|O8B
z;98{mrScCM({y8x-@bg$Mf^_l`T~a9T~9XT=6>{_{B!lyIR;zQ7w(cew5j{``n_Vi
zV~&14zxTlM&(n%dwuHPAb`)8md-s^T@w1Enk9?M}-!S3m=6gxc+Kpe{U2MF~#qzcJ
z`-T5shEEjy)3#lF?XpitCZ_BxIdYhHN@&Z@H*K$9UOsn@Jvi>c;eNY?ufIMQ<dRu<
zZ*tJrRb5SsR-EYSIy~KVy58HHVY}ylvc3Hy<C)E``4@tw+3mG5m+E49v@j~I?nk0_
z?M3#(_fmEKZJScOe$TXdm1TA{PhF>pUtgeMbTH$SK`dLp^wF8=a}Ug$o>ur#H2lK$
zVyXFl)s41_%-8RhiT`TbyXVKDY@LW(b3gk@aTHxn6ArQLJNV|=r)+)a6?YFB>+d<Y
zYwhn7vN0WNBsuDgcHV9K{OQTPh&O%=R{S=N)8pk@57!#3oo>CFyMO-&uCA0riKZXl
zKc9az$9wL3pZ7<c{r7rUu*t3FW10Bx_}pWcncFt*UnC_dxZZxQ$$D-_hWfnnxEX)4
zP9<FL(2)F4!58tIwQYX2<X(SE{^idX@VHf5iQlg<Kkl!y*i6DHebc0gGdm9_{dwq?
zm67pB@8Zkel$4afz}SOtB;}P~t>KX?XL~-UT(NoA5@jv9yggsp=G8ytomc<Iuw>E1
z#N_j74vE&+))Ze*w4eW{EKcmqEME@U^!PH<<lp6oWC{&Fwz$@8snFeQw|aSSnf0V&
z0%D8j>DBh9U+NXD`F7On)M{rDv8S7>e};*iby~3SqrJNb?~hBy0kwK6f(H(?eAZjO
zW7WK$o9`v<<l5KcbWYZ;?HsG$uI=8>n=9ov{o>|qR_#8$epiD{{pkhGX0v-w>+SAQ
zpI6beGI%)?=tx*K-&spoHLm<PDWl5PH0|B(lL^dEKZ{@9P|W}A4Ubi9q1~dm`z3j+
zFY7yIl=v=lja-@Wcf#Ds$Ir!{{#kPDc*|>Tk5iu-C%n92`&Qz@W^Ku77M77OZ&;Tu
zj?Lp+=qLN5=57_!)!#R_E|;=RJ@fa8@S=wd8!M(<I#o5N;qL9mWVY|hce51*jV~!$
z_vVM|KD(VVsW$ZMmi+0rk~KJ(_?CBXF|qGjlu%|N=(?e5^UR8Ol7BwW?^jMMdwTx*
zyoRd_l(p6SW$hiGS!Q3K=%EtmD*TtNe_48BwTo%{!@s*WZrU^@W0U`>dIy0Onzwy@
zS2TW#mHxS8)r?)Msuq3ydc}QFNM7vmd4CzXR)2HQ@o;?^w(s`7{x^(o14E}a#noCY
ztLK+->nwci_UUePydaOOW7gh#5sUcO>PD|<&@el7sbGgr=|YXB36so4)}%eZm&N0>
zd7p=i=GqmGw_=^b68YP{z3#iKVfgPxV9tzfLZPB3&t)F(n|nIyMbLzj8?k&PRkm{d
z;?1)X-rq|~3bZ}Fe@6F$BOa#;S81L0Sj7}%+!@;Q<%RVGy}6$}u0_9?^ITbO^2HyO
zFL=~0{aLo`yj!>I{-gIE$v93asJuMO?)Tq+&*GEHD>DmQ6&`$AwsprdhDqld?$`hL
z_-sZ_nUC4;hu2^AraarKzIpA`WeXR6Tzz2v^=5Ik{g+ZEh_J|(Eobi5e7d$}N%|Mf
z=a;H)STKE6<Z;vfXCm*m_sEChe-7`1&mGR&9vC0(?Iil)*Pj_YLK4P1-|6!kZd@dF
zGUITnu~45&VO7nWf6u(H<?Ai`eL#>$_QKTG{QV6Lao&Y17G7kF`1^K8{wM7OgT&lB
zeJ{?wJTmvY?U|*bp(-f_l{Gu&hOs=DRsHYW%&Ajdl089lbMv`ZPAx1+OjHbyD-{Jz
zH?IJl;-`Cb;j#0NGz<kyoLL>$AGXg?nD=k*k7xS*lgfW}&M^9Ozqq$S-L|*js8`qN
zRUX=B+(H$_R;4|>^faaX)w!;j`I}Z*9Q-h``Gb|}wl^!XcKvZMa10F$%;edzd`Z(~
z&a79}b5$nJo9F45XVo>yH}3ellI-{D`;v{FvVyG?HsqhyimUq?%f-&KbVIG!@9%l`
z8J8|dN*a7}IVW{h*VG{(bW)K}Xlsg1n*xW*WJ9Ho5?)Rz*LLS`Nb~=se!6FslKRtQ
zw}S6;J8ibCS{yMcMN!#!cFFYCHEVcwecAlT^i<8BEMBYK-J4{kHNz)c@CdCAUzeD2
z|J%wb&7T?<bSj+o3~^mxxGQ1xSGg4t0>O+^1VUb&`nIt!*wE7Xck-2!YfmI8o=sfl
zWqTuUZ$(9X$@>Y%mgNdv%$U-$Mu9`<VSz+sV8CO=wEf@RwExb(6d!z$siEcYLv?d~
zcE`XIC(fwuP8L6M>_IDgzNO&admO!mf-0U)pKo3P&1F~!*&kET<PegJw~~uL{a9@_
zbMWIi_oh_-n3pv9sAF8U`{nf2wqLhv>k98&I7#yT+DD%w(-vLSP@h*R_?TT}%T_BB
zle+2^Mkh8ZvoVS?^dFxV^~SqxcAs<!)6Sc{@7Y3IU0zx5<%pC#8^>0(<m)EZ7q=I8
z%wE4+s9~RNchFP$hcPP398qn|y~@R3BsmyPrW|?Tx09!v&9>{$-<K63p^D2lh|Ai{
z|L`q4KVeseQe5|eO_jzo_oi@%-<{#VVAr%Wcj|cL@AR*KR_9<LmA_|Os$O+y=xp17
zl?xog+Nb}CF<*L6V(zYQDk0mZx~?`0x>BF<YR<9HT9!tS+J+r#(l0z%xa8&Qhydw~
zsIwtYRF<k9&AFwrbdzj{mC>eRg&pnX_hx%eH&j=gmiuIFd0N5cRQ2HSVDGfN|NkS>
zwqBg<q;l@m)8d!6mBp9G9G$Lhej)Qn<cu#)!3q}nW$bstIyI`QzP&m*&Ghum7@>s<
z<>s%qzk9yq^ZLg}99K3TnYHu%=Tmb`r-bx6Pf^*nKKAGz*Z7Oe7f+6yU)!mxEZ=^$
zWr5tnxVarl>MJ#FpWgPiis@?VnYlj=mvX8<=XvKmYa!Ql#Z5W4q?2Z7EW5QvBSO~B
z>RFYB^&0)QfD3yk@Qdv`$}WF)nftuvna0mA3VdFC%9H7;>DpI8`34J*9Ll*J=(BEj
z$2VE)4Vk`{nU!-tcqBfn+sBa}COFYS)7U=N<GI~o$$$r}Ul$ZQ=DuRi&2DT1P3c{1
zR$tz+-|maPmzNi(t8he)Us2H}L+P*f`wMt^d#8oeweNbC>fsj5o%TQZmpNNOxPATR
zQt8%JP3KxJd@u5NeRbQeOBSmvxp-N$vNr!c@T&iW|N2#0Vg6#`@pY2x_y03HZ}ZQf
z%G2R$q{Y`c+%K-H%9s8qxHs3ZpIt7Z_i&=Rwyx}46{Q)v6*YP;o(zFU9+o9<oEgBl
z_hQrS=ZBtrD7ti{xbo@N@`^VHe+#lS9%@|dSsb-8Yge30sECpE_eIqQweHRMBa*h7
zx%p`K?KIa3B^GnT!p{Gn(ChtW!77QIynh8}x3ZY&D5iKle`I{lX~mCe>G4(bzgo8$
zWL(j>_w&W)|1oL%r9IV~G7e~H`ON>Nbg+P>{`Wo~33HykA_rf7Ot|2=!}ycs?;g%Q
z)}M{{#Tu~(8W}x%(#dk_Thhn!6o>CMJ#NYh7nc5a*c6_#pyGWU3yVyi^6BSw2ks;=
zvsFup#}pXpZRQKTvOR9SUf1Q!)Te*=f97rEv($Wi{_OKv(tgXc=ZOcF%BY=QA$<Nw
z(v{>bHw&#MNvzTHv->R*SNZm?Snb2j(HlzT5B*HPVo?84s4Ty5(!AUAUfnHpnLM>o
z+3nN0<Hgahf<y{`eR^?MbMDRe`&;kU$u91*p1ODG`cIcG_bYNTWmdZOocB}~(@DF!
zNp!}g-G&P^cxPX3O)xdhmECAPm;J7bsfme+$-mbpHf>w``gnqY^$Gm~v)ZuG(93h%
zj_LO*IBt+hye=-lKlzm2r~RT2t5*w4{WpvVu<~b!Uq9t|bDNts$87JW++A-ZJ6$%s
znp~1MopaTxfBMrmKb;x=qU`wA6$g&B&tLrEm$#?9B@0Vq?(WkZswHcey%k_-EHH4W
zbXDD?^{AxsV`XfIQ`Xmg+m_ABh&lLfL$gluwwsQfmjybLx8Iz=l99oo^ujmdtlOlG
z9K07<_;&Mpie#=>zjT{k$GL=KuYG+t2=_j}k!z+B=pdjV#1$m%F*n`xWuU99*yhHz
zUsg$-R}QaOrL{=oVS!bLyR*v0$6F#Dr>^9`q9~;s;k#?TO0mYQBa6I0-BIRGcwMBF
z-t*c^W72`A9}~Ar1g6Xka29Aj+UB_2*<snXU9%D_+Ky%&e_J-Gqpd?FWKLQ1WWkV7
zQK?t24QKeCnfXmAn>lUW&KEjD6Ai-Jx%)3S%B*|(vi*Jj-Z<C1OO2~maWr*YeRm+s
zY=OzTW850;CzUz{K<f^)%vOsA3h=$=UL5Cs^`7sUpO3Vy%3gc--9P$DKi%N_ZH4;1
z)k1Z%T}r<$+_h!lbeBaF*_s|ogv}Q2_Vl+ATC{2xTj-UCMb*KdRE4ixH&57d%RpMi
zX`#TXj00I_4XXsQG8~Me3?%!UX5ZHD(K*fXn)!>~iDN&kGuF2q``=VybFNR;`o#J3
z#}h8}^xC&9(ztT<YNl{3Xl(*>;Guv|tk;%)S3F^ulF!(6v;V{XZ9-d`x*8-_x~#g_
zr6tM9`&;|toQn%G-uw4nv$S&ies-R$|9m-~ns0R_y({_H?W*l3oxJjDXS?X)RS|FW
zKFPjYcAovtTh_osul?=c)TjvOHLuh(oqPA;lZM&*&LtenH8G!>T)bnWwe8tcxs`sa
zd<>#xXRJLZc4*bAuES3=L(`h?mIlZ<9(noE<kSKGNjFo4CMbWcxO8bl^_(A<j6Rqh
zQx6CgbrjiX9ou_-^@@%iwV4$x3s=Uvb@N_s{3RvGBG`VqS3~1QY}}tktlN*bduClR
z)#|u3@wMoYu#yF4lI%hA`KG&u2ETmYbN1Jkt@{k3-h{lpd2G|w`wup~Si$KgS$@lM
zN3L6lV{Yhlt%OZC3?tvH%w=BHbuUL%`}#EzfuPV(7SHz6kNxH{N92B6X6GcZ(Q=OO
zeb6F@j6daqu7d4{KYr#0tv&dmDxk6=Hzs6qkH_EjE?M^PUPLSvSJ9Zq{`sAwz=5o-
ziKRRH`EHu@dEL6>pujO<N~+UCk1V5A*BkG-1z%yTe)T@WViMOQ_0t@OCLXmjh_ty>
zzE&o8W9go!mhJWB6_T$KqtEmE?5U6W+b*B~@2C37H7h#UwA15iOq<Si%(3~ydLe3V
z)@i5cxo=jj>RQ9aWBZPWZ3R=6<HDbM6aH_xr~J~{HfE*n6U#3$p`jahZr<@}nYD(l
zwz|~x!X0~4mi?Z<^0oSP-d4S@zkk2~{mTcmaN_a)ew9qeZ6+oreR3gzKE0u=0wrlI
zdw)I``+550GULT&RV>|$Iuw{P-Uswv%PeMYjLTo3C-7wT*V{pjzxS~RifoL?wTSs9
zANlLOgrmpWqW{yww)Tp>PLyN&-(9tzU)12XOo`Qo>z^hh^g-5pKuPyysB1kO1)`SB
zzHDLi|7z&l_l68xo@V{}UO54@hyspfW}2A`G?=}-4r(`DxbP!aSv@%<^s0h<$8Keg
z8?t$C!Y+T03y<A2H}Z4Xr2Ewh$G@D*f1L(esN%lj3TqZe+hdEnc5j}AbX`o?(6?)s
zKv}!=^2g8fT?0a=UR@o|d|`dOeQw;+kDCQ<M=^lb`21JvjF|H2ZS=&+C+Ek{0M)+!
zOOno)$5elPI;ZyAO&^=rI_K<u%dFe~R_@vQyWjRke_Hh|=gCZUxzu~&ueUnvkdoSO
z6*BE?T{HJN+v)XtExT0|j%QW=duVO({?Y26MPJv&FPUbVeeJ`O$^M_7O!nWDc6Qdb
zye-GpgO(OK1g50yO#Q>?C?Mb}7<eT|0<^GVZiLCHa^w5UuWP11I<KFTT?m@DSbhS0
z{=kjR+?IJ)448gzGkQGvrj6v4i*Il4pZIW!#_he6b9Z|Q@S0hPi%oi0s=%;lf!W#@
zCptB(x97s=&(B%tx;ZRex2);y?eZY`zK7?}EG|C&P2#K4$%vM@*4qNN?U$N-*LnH5
zxgV``8C;~9a&B&#`D&L&W@Kf_yDM^w{iLfiGyWKL@W_}=>M>tt`1o$-9FsbhUzhZM
zewgf<^3A&0@$cuAtF@9ZPs?=?VygVfT5`hET|^-${_wT6w^wkw6)sP`WVxR?Z1TSi
zQlK;SpPZa5X_Ugj0Gl^o(OYR!CzYG0Fd=nj$`;1GH!e9SXmc&x(Q`28c!pKgq6X0L
z!7hWFZzT5GmDO0u9)D_{Y?A$6n3uQL%z+_5^ObeW>%2XTJG0b9qr*2>J(pwRO+2-K
z-s!*nw%(N?4!%cD>b^R>_y70ul1Gl=iLRZ-wXckAtV`LN{FLTRd1?GA=H2hhp#hs`
z?tZ_sKfi2B@|A9tkcr;w_BWqCCvp0CzRlUo>+75DB`64SU0(V8<DLHp-d3!5bhG|n
zZ~yfA_9f;D+Dy-5D!Iewl?$=y9@yD-OuuP+Ex(@4d!2uO9<27MejNJc_kJaf9XlWM
zKUaQ#|HRwxch8qwpE7(AQt+(sb?*0~oOpfz`?cnqujd_0FcRW&m8?Jb^-ks2oik?N
z<N)1!VwB1yYgO`KlD8h?gPZB|Q%_CNyi@&t?~R;o+w%4uTR*|fTtH>A=NY!fJv{d8
zhwiP^I#D#K>r~b%rqC5#?30h)y0!=ufTui88C9p9b=kgkhOO%TiT5_k%eu+@o_uQ6
z${VLYEscMCLTf>IqSgI8-NpIy*S-1k%6r>2&p8!;G<qg&d;Y%O_vky_DOXm1c@@{P
ztgd&ltfPSDy{*mq`yyP9Ub~ueNA}1yJMP#uvsO+<--uamKkN7r#%Uq9OahnWyw-}F
z^~r4Ax5G0_?%ve-pJ9}LrlPkp+1>C;uzTF`Jf7JFhCA1(@c(~n_e?kPXr^-YhLU$Y
zO4G{j6dvEo@;xspy|N?1=-8`P<C))H2K8BD*_iox+tytsCUwdlJ+_5Pi|v0-nwh_^
zXTd6&;$x;g?{CjDm*3gFH2Yi8Ro(JSwz@|S|F4-4ey?i&w{wPPA~tsIx%YU-yGNIA
z>`5|o3{;ZyPr4>QXP@V!BR@VqetCCy`DDWfpHAzi-`!Ojr4`XT-)FWoM@Xon01J=X
zW5vp6yi0x_T(Ew*;yeW*zC@$wp9^P5pJkSl%N2YZ%`I^$>{I8Wj#XNJX1tpHdh^;6
z1rDXA1N_SlUAd5PZkg_Fj_|OFYxsq>RNOTFEqjOcZeeOl?fcTAHS<{KpI2FZePhY#
z&(20?woIS*>@@G!m=agBtEsmuziq9wiJEZ!T50^ppLWd2$NHCLy$%X{-&^=*hT)7`
zf8PH3o$&kme53T~UeohjXP7cK9t-T#=*o%TJcqq}-SS0?nC7lzE{`o0G<$V3D9Br9
zr9u6-<0~%KzUURmx#1s_*YEJ^`{qR{(pz~QCw<-h<HzEcA@A0>iLiVNzxMd;#d+T>
z^UP{D&naEV!240fkEO(o`{qwqm$bEyR=UeAtp5J)i@Z$_&+d8pWp9=#t_s+bD0{j2
z_Da5Xfh#Jd4-FPba+sLZX%g6!r@8Woh;65XZPP`!KzDoX{c#5lhd%xC;__5QXReL2
zqWSdBq!~Xx)|*{;=J{vYG9Ty1_6y&(Z_v76vC_WeQwHYpjGi-^7|Sy{Zar(P4RKwj
z7vOE8C_g1!q;Qvw#Ml4c-18e`_&1l`tuo@Sn(%Vl-vcbMsuAK2f?q7&*q`rq4o!WP
z#q9F<Bs156wDP3wv-Xug(&u1$x_eu|tpyhrM+;g#6o2FE=pe#!_QMBh6`@s&Yo2@M
z?5z?!m3!P_`rNbq_s`BdZ-1!n;MM!tk?XZ?p8K8ev$R^LS?0UKqEPFbTN1gQ`*wE;
z{;dv(-!!d4sdLRMiRx>rU;3I|J3bs0pKp{KJK>L?tH_qq4*Hx|!a_If-z3$1)YGp_
ztxfse+fdGs@SuJBx%S?4D=QOIYvv1?>fo#TTYJ^Sp0rJyHycWSJ)-QG<!s1e>FAql
zd+&?R8#j%+;a!W2Cr+L_cbDAOP{~IV6->VdvY7hS8o1^CE_-o>W$Ev47X>^YczUbf
zeQmcWFf4RZ$eY{lr3O8hU#n}Mb>FgZnu9@~{oQ^)p%trFD)sTrKkmCm@xZ=Ie^o*{
zznsar!)NSe<mg+<J4q!VG`9KTL)FXK%lB-V5)h+kY|Xy>s>_OX(`GPvDLMJ(3hY+z
z<%#z)3-whvoi&-Se$PHrlYieey3RCQR4=?b^Kj*x{QAW38!pHBW=fQ#a;)moH=mxU
z?4Bq%nMWeRIV>?><<zWW28%QnEmGFsF1^@|Pth`iLAhD5s?_`W<nm>oPJGTc{Ig{9
zi^wZ8^TTwUU0pBSw)}m`^T+4u8_V+zPmAd$99*RO!@9&tzw-t9M!!9M+rp-{9n0^-
z*rLZJ!4t>EE9189`l$v-=7Y0b!@}Iv?iTWQ?~girZ|?3H_J4iFbZ`1jzb?Nx`PadZ
z6-bkx`csYety<Mp`udt-&JBYzx3+op&Ud&i&9TqlVlLlYKX)CeqiI4bxvmI_x}I5^
zV=lmQ<Llj`W&8f=J^fn9W98A6=4bI$rceH+{M?Ea<&KYE?Nm@OnW|{wJ8ky4;3>=G
zpEX}!t#S5}i0OVi(~BYkN3MKie(EZ{s`BYa_M0KX$CIusUZur0douU>UGwbTUJjg9
zX)UwDIkbM>8FpSxr^gOfrExYQ3HRLJ8-AH2v&p6`Bs4c_VL1OpWwAABr-Jzdb9EnW
zd;PEBu!`Bqql!i{TE`cDJ@zWQ;%g`Sjfkn29E~O(n0T*mvY&Ocqpg)<(Su*pHgPOE
z06M=WL@W5lQjWbYlLJ?}#+!U=ujn}LT)cDEYYs<+h}c8or}w44SP}T+ZJJ1EZtU~+
zyZKJ1_XS;1o7rb}e(HVg(&}j!S3j2d_U^Xp^+nz%Pn=5ec8vdaerDhL_XeAGeb4%P
z$YsV&`Mt+q=|0V=`zookQR3A<)tbG9;?l3<-1N2P+&JH?u!QaP?XYvty7w|I=X-zb
z_I#bx=+u^W`Rc~~cfYGoQq`__vhw7VYi@!f6F;W^ZPY$zFFE<$hsf{WzJ9L!adP^K
z-X@vuOSMlj?^K<2y^`~<>9>6S%z{HLGMpDG-$ed-Ig9(o|GP>5zWfL|BgXx47em$T
z{<4>8-`=fWxNO@rVf|mq*XL9VSzkZ*{9^syD^6{z^{iU=-q(HB*B<9~<>kg(w+g?<
zuS@^wuK(%bA=w>1OoGjGSLgjTPOjQw9pmoR^z-ulCrhu}sc1E2d@6b=pTD?0+y4Le
z3#;c?zRWwe?RxS5_;<@ce|%p2YyzK6*?;?+A%bq}Pgwko-TC`}-qG1QkLR1;_3TYi
z6YLij;wgDK#r@{Ky)3J<Pe>IAeZOC7dHR0!RP$$R59F8&=WqyKw*2vUZMosT^5w^#
z^SKCa+kWYq^2=!F$Mt=)J~eILedmj`QD3oN{Hb`k`q|s>&pjEx-n#Q1t7XluO!?BU
zg+iURXZ}^#Ek1Wval7z}uC7HIDJ3ub&?Ze&%B3ArSM+tPdGxi)^Vq8uyvIK<J$zN`
zQ~62r+v__%7Ih+})#V$iuf_>&a_QHLEqIY0uqj!-<aE;4N33snBPFfgE!lQ$THCRF
zK1UIO5Ya&OrFGlS_;GPpef)UoyvDk9>X#>`{JRr*gZuaeW%u*N=l3;jUSO~6){}7g
zSLl@EhuJSLF;hBiRQ&uLX!UYT`Q1{6J%2u(zH;S?LB)p$+wwLYTR&y-EeC}zrLx_!
zv(L6&et9H`GayRTEf9QKN|S^0*SNU)uck`Zl=AjH^SProRo}jLKL5Rn-Zgt{*h>5s
zFI&}g>Q$6V^UR(zK00o;8w)1v3gB_{6j;Ii(9ipLTEnMJe>EmeJhO)944duBt9!Y$
zR%IE^c$<>CEct@>%Gb?;0h3vr12+|YzA!PNKGJew2GjKG{<F?EYn^6s7G#sF<~z>+
zqD4o$V-nx``>wgZs)nKM=if4^dWMFIR&8dUe%NKfu21SG=5%_h3#?e{I`xZ-^eYb6
z$rXPkcg$t)(JspR#Jk|c8-0Dn3j)X9eVRD$nPbkC?3d?w*W8Wq@sUr8KFRW{{G3B*
zsMIT_BO;=vf~o=Hhn{_!^sHEU#V5hnp-)u49^_y?KL7AXX6xYa*kjL)mmB}Q`P}e+
zp>?fw>LL~4_7jg(x1L+QU{&wxKTK=Cv~qA9QRba8ZQ4W;o*y-OLWPpkPq)TBGKih^
zRcW```t)PRHy5OCUmicbW=$IN#Ds=TpSV&kI;4fEnH8UYtG~i%tLfGgO)e|+8kD3?
zp7|oYYVD$3dM&44AAeuC?d#*uyOra1w|j&dHl1`)5mO7ZXWp#bA;I-{%BjxqcgL^m
zPJhd7cj(8p;um_mIVRt1S#=^JX6K9C-rIa%#QbJfFlJw`bMD`q|L{&>b4SS2jdgdc
zs^(O3i_2I~c=c#~@G~7p1vy@!6~QO8q>db2-ur0nR^B_GpU*yN;98mRYV`~*_s?&R
zF5i&;HQnatpVvGRHWM6KC3RD#AHM2SoyisSE<tYZx0<Z&^=Ch?-*^7prN<MTxDNYU
zv~OJ89{leulU&uW;%{>+rIL&sOI=&E-|w26T%`JL_h0{z<?~N4_uoAoF8^otcCqyj
zz7#Ug{Pye0;S>pz9FBD<lV-ir5WicZEZp|>$eM1Yo1b>Pz1OXA#^B?lX8sK&|FYb^
zXB=O6`Sq+xx_hj?*1E;j2rrktJk|33z9ZTC`%mtl_e1z`+nL96#s8hX)@P*^YH!{r
z<F=_xxiYq`_+W3c;qg-yAO5*V?o5?GULLY?<(fqqk3wUrP8sg`b>jcV3U5<>n;#4(
zCyD-fe6jpSXr0x^fC+O`6aSU`jpaF2{^8c*`o~wV-%CvRzFS&(H@Aut<Gq^PSN)f^
zx4A_xxGe4IfBwMR4T>E~$?9{C-Z`p$b6Xv2wY1;kPq*_Hy_@*F^7qtq2lb#E=@QBA
zd<Q~y_kB6zBV(rhQnsq_@mli_f3)*I{CV8JP`H;zy4s9+SIMFiB{Hk8<t5%b{(kcH
zb;3V?%6`}-8*w-Di|n1r)0fpA5SG5CknrG3vEK6f=Q4Nh`aSuzy8P3zYVOK67mrHF
z%BjUwn7zHU^}YdbZ@g5X$pi264qtapaA6>6r)M+YtsFDG-EWNMRXpNc<~Nt?z`EGo
zGfXlki7g0ttbX|!8`IX8^RHQMn6>))>yMRtS)?W%5$5D^OWs`U;<ar}a=yc(TlVw3
zumAB05%rr{p{Vv;si&$@_P$Q?rz_>1C#oiWG?4w(_i$DBxrG16Qf*92*{U|rbg}3;
z_e}M2>Vlej(Srd~6xIEnEDAmz_~&d;T<z!Dtizrmk*Nn}Ot$>r^H}rdhMwiiYC2Y3
zPcBU>`WY&|vFzOLnw@#(YI{9`HWj^F8h%+lx?khfhhIh~&YtyMS}MY#;+S}{$<il&
zORxRC3gcg|cx}F)IzFfB<kpm~-!uNa@}KxK=bUvJgS=mDTfSGahf`C8M$6gz6Sddv
zo0K!zL2J&YhE*yNHtAOKXQDcEx>ihT4L-MT#>c}|4MNFhudmLyp3E_=f;E2quQt26
zn=W*6PCGaA-15`=kLFD2a7jMGD*ZgRO4XVBkI(#HJomSSt&emNDgAMIbH;r=epUVc
zvdB8QpU0bZ;`dFPCtIsC+j#DG{)ta_JhYYmE$PBowkH0RcKes3mCNPZ1y(+KtNB0e
z=Q1IkFmATRE<vZB^xa;kwk6$k^74Jxmz?hVd#vc($J&{u#d4;mqI>@S%I28<Z{oa6
z&$xMixNe`$VrYGHj(O*!O~sz&vx4229348_eiXdgAF)-v#dWy=*U^)&Z%hm*wJhaT
zEa~%ExAR|B*Crc{S?UM>_q?)=UUADn-c^*tbz887*0E#y&H~vB{;ZDF*wnL7(P!q#
zmWPuiU3>&T?J84{>U{LlYF1e-|1s4K)^l#?u-(y<59?-}d${v#%7M4vl1!ynWd6G(
zlu_gUvS4>kgpO9G9e?L5-JJUy_(L@wF6O_dF1AK-9#3G9s71~psa0#2l;ks~$CYut
zHM(@d+q%(x7yr?eVj+`fpYJ}i-1GHW_L0}8JdaH3jnjH?>h+-yb+#wA9?J;7!RfvJ
zU&`{|FR!iEK544GQd2+qcUw>j-_(VC;#%vz-IXv;5_l;m>}BNUcwA?i-kh`7SG>6s
z?tHwg_nt@li(RYM%-EE`{M&5pv%jCtUU|1(HRQ5Gtj~irpVucm{WN>Vx1C28``i0%
zJMG{z-+5cT@_fH~?$Ewq>At2#O{<bhcSu+zSx83i6nXY6@bQ+w8<w+YEY*J5X2Cm=
zW7Vol^)C#L@xAN~Z3;eT(X2Xo{}Mfs>`V1KwoQ5~{cYhNj@s;uEv*0c$o={K^!kO_
zYhEz;`-RKZrfE&w(RKOlktyc0+W%ecJ~fNs1<#L5e{T4=O`3VQ;^$Vm8M=HdE^hOS
z^!B#*DhJ3PO4#zq|8wFM?;lk?&pxJC2i_Mr`mR7p!!X6M|8Ca9C7;)&o!FQja9_YR
zf6Iht-;+%|8rqB3@0nb<JkCpF+L2cUk3R33Sf}w|a$DQ6`kt%z`+onkW$j5@^XrQ7
zGx+@dRF#y1owYk+%~&4H+Wr3=X#O6$w|4i%4HqUII^?u&&nGWXA;JJ!YO>Vg$`8<P
zss)yvJLVKE&Mx_EHml@_5+_4Xn(ZVNMNcKKM-%=&YVvyfPixJ~w<k`2SfF9Fipy>G
z{NQso!i=q@@%c->e-M^1+adpS_B6w%b6uy*Ezwb#bmgk6<>xi0U(Q|nc&5Aj;(N9|
z(eL)mGA+LMZ1ei-1$QRMo%!`bxo79!9odg6zO9QtbLD&b#m({YbBZSIYSR!tw|~M#
z=H1Bx${pWy^!B|<vHHQ|Gy4zI>aewo-OnjpNtvQ=UpakJsitJ^I=(rj*E+w*+RRGz
zzW(vs7W0d&-ydkpILz0x%!!Wvb2;T-mZ>_=ThA5i+BUp-^ecP)QN8ewe~R@#yxg-i
z#%N)@SMDN*+?Qr{vkd$Hsn0w9qp?+I-d8bW`#GNP<>qcwJr%jXlK-#&{;r+}=6m*L
zm`+viQ#rj<SuStE8l^L-<}v>B%gXF#d}5NX`@$K!-o8KJ0ms*QQ&xqCiio(nhK0!N
z)7e#$d4Aiizf<1dJmzlv{Nn$XX`8;hyt?z^)OZECRjJR~<+oiDuQhlRd+brZx8?U!
z*)M0NKR+BEmr_~C+odGjqsD(Z^}~+9IhEHq-&AIbiN_bR)><$B_;QQ*4Pm2#ySe2H
z^KYE)*57liY3W_d_e&ZJ66SASbtxdc|LEnJuh(02_j2ER-g`59=bLA<H4C(DKA-u#
z=H`ysi)7v(%r^P=dWH(;Y(MwVP}YA<Rj;%|m8QyBl@=_{HhC8@H*6R0+mq*}9=#D_
zo@CPJEwr(t{kXD+WDDQVKg<_>Lbeuq2QB73r}6S+*8PcxnsicJ7ka;UEO}k=BPCQI
zD6hR}lc8HDmsgNuU;B)2pSWvw8Xh{C-gjJR^M*9vua1_pcprtfs8xoFvWT*<@L0*t
zY(H^fakACKwf7w#m&rBkogMb!eEYHs*>5V?%JPr@%Z=`nI1(b@u)O$!;eO+raoS#W
z?reRVCciYByvBQyfTOP(2g_oH8q;Da{k_{eKB~|0sa`)-H~ae53rh<Uss%L{J8l+}
z&4~|UWStNcnHTc3Vc};5w!8h8L9GRampnf%U*eY&P?5>AKU@AgI#zCto`3i2k}BcM
z%w^5fIl*fz?kkFNF!^m4U;g_*W7_KWNpkT~V*9p)J-H<9asFIx_ScMzKhFeJWan);
zEbr8O(kt)Y!`sWhx(P>G+P}N`26gANa@Ayp&KYNR>+3x~yL|eGlUug>RAw;7pSyQr
zdH(vxQ{Bab_H5ns@rg2vj7ZC|+CRFdMO_71Ccc?t^W|ce-Q16c-r@NQnx}i214VZ)
zmu0RhQ#D_2YZ>{n<js}dvYnP9Wv?0}dWBcqJHO8M>XoZg48KqIkJz584_V~J@a7($
zjPVZnmv!c5v%C3jX68g~ogumT*@pG2<N`w%7T(`?_@$EWS^uagskr&Vfh?=dXLmlI
zcko>(_s!7!M{h3s9mu*c@vCsE*djxrPL)*|i|#+T_(dnjzi#?Hp7+;#Pn`-{<hfK+
zsmS+cUX~N@l=rtzsLwyMaLKB98MUlmCu;0ATbF$PAH%^_wo|UNuhh7Es{HnM)3=-T
zUtDcZ{`FnKSbcusV>hAIo|Og)TjTO|7Ru}s@ywc~(AB%iL`Kg$R-@~}4lfC=u+7Eq
z6>BGU1VsfNzIRtOMf6)j#{PFH&q`AMY+PJ$qi4k`k21gAhf6d)tYq}qt@g_GAMbYb
z-B6xyxG}@_^!^#AAFaPOdCKFE&}kDxqPf;}sqHFz?Y#V3Wyz}>eEC(@yF|;>w#Tv;
z?cqDr7!YkAy5IWIDvwYD$4dcTQSP#{!(STgw7#V9n&XHn6Q^t`r|izTR*!7oSuDvr
ztik)5BavzEM%Tzm=Q=c`Zv1*H_DVX5!FTny3A@s-9(%s4@(kzXv%aC8mbc5NBozF3
zam94<rc(;CyT2-&PJXc4SK4FSC!Pq~mrqJR8NS{6{l<=D^Z2v#&#j&O?$PZ>f3JjH
zop61l{Bb$s1{Y`D#sU#W#*n41e|kgq^fb>Gbv^j#<K>m!+D0bZoC5i_X$HkRt0bo0
zIk8bBrCLLGy~DDr?qczN?nZOCN)2bq?X6DixVACTU9(8(jCJsTU!`P$@HxgU+yDQy
zvDSXI;%<ER)ysVwkIQu*ms!4O*S3`@OC#rqyM{(r+|}s1^`lN8a%X{{{MqPZ?_{Mb
z-oNGlkn*eGHji)h=C$*-T)VwWw<QZS`}XIUAWP%FzrQc<um7*iR($=!kBu_2axAO5
z?rB^M5O-XD-SAzH+@7Cbgv{3IJ#3Qhd$_lL{e#u4w=?4(*dO}JH1CVaAyI8jz12!<
z4WvU`BUa36^`1WY?n-Ozw0$R6Jz_cR&UfhblpC8CuX_23`5;TEtLsw36LS`sM7J_y
zT{BjkwPD>7nZob6kw;Ij-{<|XVWG&j^GW~jy=C{6vQlPtn|C5|@vf!YBvxp7T6V5-
zGLmx^V3~AMVqyi4s-n^ByXFy_)qhS-T~_G2$uLBI)uJV8w|uL-|GH1oNl8hY{BF~P
zsZ$-N*M;qIo7o?})-k<2a9gtS(=4HO$E2^POnctkwwqcMU?*U*@n%<?7HrAbQMbdr
zUa>dr%<{C<(lhFNtage<>MXH56S9*1_F8fN_ucPgFWuPUt0dJsX@V$=qmm4tvNl(i
z&5<|X?`16gFWt;7rxC$0p}^wLlV?vAG`J*Z?o26v({$s-)k`%;!!ni?d$3Nlx*)$l
zWf5zyeEaeOj)^H8%^N2E^3uM0%<s_j=qX$tH+dWz_5`%_#_J!RdAobgzvBs8x8KQM
zy?K{p$(e$<iqED-HhFqBORevnDZ6pz*Ol*V#&+|iJ%tQz{@oE%$Z)Gv-l?rp(ckWq
z%>RlXd3rlOMafnFvCP~3kK1K&=f1~&M_+GHIL*F!=89EaDJ4Y*L$80nl6?Nd^XNP$
z0iG}Rg-WrXZVOtL$>{sl&X&%b<6O0C{+~TQC6hW*Ha_zAKiQ!0=+iF)%b<&0U){9#
zm6)sTp8i>X|Fq-tGBlgdXD+^#b8!AUx9buI^j9yCcm93$>BsteeTOIa9esK9`KDz(
zEGJXCtM5;}I5S&se}{Nnb-(N8dxuhrif*-D57=9*+P&SZ;Ha+oC41p-c|WZ;i>-O^
zr0}@m^V#V;9xszV;o*Jw_C{N=$$XANAAVk*Rb&;Nqi3^-l~F*IIc~pvx1&3QhrhYO
z|G(2K)^B3kZ9iQ)&qjVXi`#Dg>DrC=9zFS^z3%D9uv@D>-d^&v7<5d*#bq2q8eW$r
zq64M&-n+8+fo-YD>1Bl>pks5THd>y&@y1Oi&-musJ1&O`W%9URTI#*ZKJ327FI2RM
ztNP->&z$-Dzp*XrR)4gWLua=6<eT2x3a<A4WLcGc^32=g9}fQgk@b$fW%0*j+0pAF
zWFtg#{!EFQSyXZId}**$&&8G5{!XdUkz2BqUH`T(J7@WzN{7Sc74wl)+h#U?Rp0pI
zO}7}n9lIoA>6z(iINQu~Wqa_Ii4*5CAGF=``JT_N9!t)RIm-EajhkodS<l=xE6S+&
z_y6NY@)0jZIyV>njQREL#8J)gk55nWmPi$fsqMWm#q{9pyNugImAdB5+gr6M^=y&Z
zbQg^ichCM>``hy6pVt<J0gMdU&-R_yF1{8JWXcyf>Bhb?(a++wuJil+(rUk{ZtJO$
z2@`IrYN;w!>YB4;QI7uYZLc{)rW=+&J3d8i&K%<%{F!Z9G3_TdW#)GzE}D_Gc~L^?
zj~6puFMP9MpM;a+cPG1uA7b(~CbQ)#`O?nKad$q`ucgiza(&Ax@4W|Ggt#~7x*P6%
zr*y0Q&4Fd6PnSF`J*3<^^Y>4sIgzd|OiR!FDN&sJyKjNN^=-wQynOHZ&)QvJ<FL|N
z)m<pbm?-kImAP-msSA%9{@Tg6oqBsj`c>gizCvg5FFqE}z1ZZd&D6S?n{OVG&;KoL
zRh+eH=Dl_5-|xpYoL+U<_*IS-Z}0`?dvn&={5W#iWf7O$cFDc@3g7K7b*{Q)RiY-H
znQ7-0ru(?p^|WV)QtvdgIY0MWmLH1v_wmK!kZV^IC3SvWu8}BFQudqS@ttuo$MKYy
zJo7B0&ZBv3-#O%VWgF=2mb*}Vvvj-C3tfgSuT7iUt@w|nUREy6)NH>0{OAJV4JGm?
zon_k}Y?5Rs`0@G2<#_oetE?LCr5sK)6v)j;_;&h#TJ-|PKp#tkl~u-v{cR>Is?YO%
z*bsQd)MbO^oC@1Kp@~&;wLA3l%C#6S-{rUQi4iTdeq-|?BmGrVVh*2p)!eRg2?tdU
zzU8-Vx%rYQ;R<WGL5Sc5=BoPYy!sc4Z{*mIeq@>ZoA2?q+3Qq<c#_I1KkNv~zb3c%
z{IX&vSN<)pjT(N-SN@r>O-{D2NNT@M*P?*<`1O*UpmipEE}YIQ7_%Rriqw!%iwmzW
z@j1lG>1@%n)$l{K#uW#f0Pa(P3bT&|o(xP?$ar{YwOq{IP|>d))rtIbvcIMjetbIp
z<FD8GpU*tzUp$>9WNvHl`JD}Qv7Z9l`JdmF{9N&%@v4PP*``aEE_y08-E2Dk+-XWs
z<tCXtbFtZ+q8Frfchu_n@ae1j+Ds3Rt8PCow@2PlKtsl2mdWy|tCqcX4RKZ6T&%xK
ze9fcYlm3a@tPCRd?8psc@oc}Gnfmn)|IHmP{HwYSK6}Q0J;poXu#nTT5<R=eIxSuK
zmEV?rkKCWk(d^dyXl}pV|MRKec26u(>R{_xbm+@UeIXwA58L0LZYex`!CP>S96#IK
z7Hj4Co6pZUu-n-F@om*&0afK>zj+=Da`eUO-PuacAI~shVv7C6uTp)Kt?T<ccZUNH
z%{JA%V?V=dmat)Y;XzjQ$40LMTx(x{eYFU*it^n53Gc;TPnc}<?@rCSoexWPJ*yN9
zJ)XF!$Lb?!NZD<+>~iJKgRdSv*?j)t)a&<hFN)jkW{v)?IH%r?_p`Wu;&(Cimpk9C
zyeI2!5-Dksr1HMTX8Te-SHb1_{rBcxn)&x%^o_UI4PWgDz8U>XOW*dd#3Y3Y)54GZ
zm}8Q3Feywox>0klbgi{TVTBEQY2Ky>-*yK7RqU{c$y-w<EcHKH=%c|whY2e_vi#z)
z|J`w{kvXO8-Kk}7?=Q%UX+8P$aN&!6@y(rIzBK>M$e(k>DNBFj^{KNWyw$kdn3r=h
z@hz8jH*RNHeYyE+<%XPaKe^_1Z;i7|Y%^>nB~%#q9`>Dg_^uP5<CTZLr;k);Fslg1
z9d`UF)7SjzC6DfY>yWl<=NJtlWu8Rvz3VWODiQfA?@(n@cj2?gm5I-{GBU~4Tg^-B
zi*@T}Ui|bydbIJ(pz6i^*{pu=4<@V4ZdU75k=0u-9#EL}bW-Z}N1rM@zHiHEKbT^>
zqP`{U_NtG!okd(__>P~~_i%Q0emvb_!h&nlKADB4tU0(gB5Tb8E&Kg%=IhSx5V+H|
z;FN+_#9Fg8dfzW5?zv#01v%7hqp?tD!e#Z71v@LBUTnQFS4}O>wxxVtyqk*5^QPDR
z4;Qc0=44wGQ*~S86!TUSm691}pB<J~=4feVJf*EQF;sB*>F)EdokW6Gf8T0))#Wi)
zS-WhC<mR({Ua&)!9Gsm69P<xlnN8|(&FD}oxRL9z$iul-cWbF*37c_Tepb`m44dP5
zn-x1(*4&vGFyWTNE1T7x{aUX!%-}V-a$V4SiJV(#sGuun`(;NPDWy)i<!8B8`P44p
zyO?n>>(XD9na&qg#9}8sm#bumo-S;e924#|kwvecfA`T2rPD1^O%gh0JXW$yMq965
zWjSo96K0*aeX@&*i^{@BTW-&?=?)EDxNx6b^UWi3GW9o^B|MweeNIV`Gr4rf2MgH~
z52NPQ{rGq7b?6f{*%hn06gnc_@~ug8z8sgu_|za$MKEcifnns)w5>{9hZ1WOue@Gy
zYj>w*e2wezyxNl=cSrnv%JZyF`pNoo|3mqcSFE!OnQr!Fl84D92LqYIS;a|)FFW0x
zLPD=E=(39G4SYQ{rRmo%ww^F)wPqLR4-<Z?OgfNfe?xxJy_TaX8K>P}?{9;z!pyL!
z%IUd!b*fJE56)V)zC%yHC7MWGcv+(R?a|6rUPd!jq<V!qT_ScS@SXg7^o#vdn@!dw
zS+CVHWEvw(e1ZfwM)>>*D-S#NxUlTa4ad4VJGR65pYMXtK#F#|Ysz1GZmmFn!#RdU
zeezs+fA%{vsB)gPkT}C}+_vpPVoWsm(n68qke+v;2Xagc4)z{b=u&c<+-IyO-Lb0a
zXx@`8_B<|uB4@&wvmR8R3JldL6aU$(UBKSdzEX>0QOouDbvJ%`sNG=;4ldB&efT7!
zdF|YTJ-0LRw2nV3NG`9MHTCwVIUzF>wpkfO&o!ynT(PR^2G`TZ$85V_FR*FqxcoSC
zO5=Q$&8Jlq++TfMvNZ9##mA+o4|4BluV1Crq>)s<Yu2JotHCQcrJDIyt?Ig%bhu=}
zKDU-*R|^9JHtgQ>!<9R)pMP@6fwgh0fmTI(n{IURU76N+Eo!1`=)r_-NjcY+ul?G3
zF==_$d9!J2|FKPXb@w$mwd3-+gSp0wxUyF73SpdlwC`k6`>~=gY1)xHw=HX!uxgc(
zhgFPR`Kvkaclt+tbC<Ra*dM$m%(~g#=Rk_lqU*1N3iYBK1z1!(52YAgEPmnfDtq_i
z^Ouw=<j+d@iuKs<m*53~-v4roSbj0*l`#FcecaHXA;&SVRN>#$svnlU+$yV9E$Xr|
zFgc~}^*^&!YQOQVJIxAGy$K3#yM6L6uU#w0!BKwVKGTNXYi3!v`KW#NU;W{G(<&jp
zc7YPNGj4zN<7#w-)E06@=sugKUG@KI)qeiQDc{{PBP}oP=sEmW)%b3^%D=GdGPm0n
zUu|}9etf;>nd5?m3m1OWR*&h7`gym$G=IMW$B9mf>N4@*X!%3$K24ZCyV+CURM&@-
zNk!Oj)v7EHtp^2_1rfa~-@9@iyf3TsZ@%jBdM_ifWvkR0)?7GaESbY+?pJ;L+{;4;
zt(d;@>aq%}#>B}_Gn0S5>dMuSP0N>vgkG6Eacb}V$5uyX2dPGcd)GR%%stq2{`Ie$
z#q+o2nO5y>+~qJq^!4_oOy6ty7ilDIyj8H{js2!C_YIpGuBPQjef+o7FltSErNkfo
zSAXq74&)SHSh=8(`G?{`8PJ0DI{Ot@*t0m>k}YE5-b|ZvQBjaHC^WkNWY+PgC0ll-
zCLiCWTC=xKz4_Su9@AaZ%hT%|w<jJqTvi|0{W<BN*0Z`Vnr(b{C9T)+2WhvmzGl9{
z?i}%kFNW{8|1G5=hk1>U{gmu??-ZYP=$C78<c>V+<8$lyTzBr<7PhOc?bv(CZuy0G
zU;NJ!JsDy9-mkW?e!pdR3sZ-c+uTCV$?Nto%zt`di(lf$FPeA$pOH36c)Th4|FNmU
zzqY*;+k5wZ&B@;Sx(Tn^EEXv!bSZTuB|KVMUhsM9@$Cz9wBq-DHR9j*#jJ3y-c(ml
zk&Cx&?tJ@xe5S#I-0kzl@*|G@<+nR%e*3#3L+0D=9pAp~wpf?%c-fLy)OzRpdDW4>
z-s%}u%O2m%|9_^>@$=Cc8O6+vDxN!R>gK$Y|9)7HqiNdx&+d!<nX4bapUm)M{k7l9
zx9&{)3OZp-M)t&++UKtGYI602|5uos$5@ClMtu4h^6%f9-E(SQ%HAot>A9BYRLDNi
z2~*3?Jkn=vetBcVtp0no-E7Wg%D=@g-Z3-{i$6L0{9o?(h0hI>BINmsqk7Fhyqf&o
z@a<*M66s>K{yWy3+h5w3=H^a$J~i^rWcv>S%#M>nv!{K$_9ZptyR38KuFjk9_nnio
zoXvgCPH_3p<-CPYJZ$IJeRy_W&)wPi@%J9D*izQp<!_eQi5#9k)%*D$J_*&SmnPk1
zS)4HG(VO*rmlp=RztTQ6@l4pGgHormrB3L-X>#q@l=-==#Ek#$E|Emjml3~zy8n57
zbLEq-lBZeI`x4)sd3oml2U)f+k|#2icQFKPOtzi8e2-UTs%)U8#|Fis9VM%Lj_$kr
zMn_L^na7i|7cT8{N?e}ISBt-{w4}mz_qS=upMRV@`C)R;mCdVnuuQ((Zo8Lx`J6hn
zhckLKG}c7e^o!n~vS0>p`(M73of^Bl%Y%Y;$)8ME9PG&WGEV(sTU3|#x8#ix93k(I
zKRBauJwN&VzM?f@)~3I|Uz%{`^3oL@842>Y%cnm)m*XCIJV(p>dw_V@iPsn8RtGLx
zRH7eR8dzk#Ww~>|-JHt4E*;MmAFH07{nub?s?_H8z7g{VDc!wG7C*Eq%e#H<->j+}
zJ$u&HxrI?ZYfsGSyS`|#mB5ii<BiKJ_^aCc7x`7bxxI7hrCDsZ%C#2NPMcp5YpHMd
zNkiK#tl(zo^#?-Cdw%_@-edbVnN9R=(`lu~NeSPo*Bj*@trGH-Txryxv;3K3k<F=d
zZ=-KS$F-iF>X#VkwM_nS;K$X=eSd*+uZc;WXNic|7WKIO{YT?<pD*f6F5u=}^CL_3
z=~qvya$T1`wwrrWUn??8F)J}~Fm)}uwDG(|-rm#x?d$$dY^d1LmwS7gpx28JQj<Ft
zbtr7ho3d8x?!iO)VF97RM|Zi#Z>s#>%A<LP!F5^A{k&JRc^$0F-}W6=*qr=2?AzP*
z$KHGij0kDCo#o=<AmDO{*R*DHn(D$97so`wxaqBi$LBj%Jzx0c{WR~N@6OhIcy_u*
z`nBKsIM*+4uL*9Bb6>Ej=WO(LLEo+?&&roMmsu@hI9bs$_cq_P@Q{#(*Z*%@zqeSq
z>T{v}cfa?>x1~=r?bg}t-9A+<NJKM<{pdZ>&RH2IhK8=6yZ5RdZ0b_t5L=?5BqlKB
z-s+O+O1b4Xt@hrnuB$(@iFKMpZ@l5jT08l|XZM2dRiArvruyFJdEd9rTw$?gcciuJ
z7mYIupWR4onvr?kj6Jfqpy0pbh8-u)dOiJdaq|47EYkx6ww0T=pI*Ij=jB-<#)>N}
z>w5OypCZ`jo$~C-#dW!6Y;%_}XJ-5{4=jC<E%)oOa*nR{@stUjoHM20*G-tMXEnQA
z-S*%b)#?q=k2#99zT1EBerESu<ag47e%BRi)-BBq<m9WJv$y(GN5X?!yXBv}(>|a4
z@>FVz@6DLPr>Q%71kYK%SJK;O)xA!&Gr}uN=fE2C_~#C_LXVHfpW2eSIhwb1L+wo)
zxynbDeLqW2WU*MTl->XLa=heC-QIFF+k<c3Yu+#2|HpRvmk%Ev?B$<y?RV0=`X|?A
zqP90w|NFqyyl&5Q_WDB0avzV1g+H~vZz#OUWMo~M_+x^cE^F}OGxH5Q)7AI)=NJfa
zwr7L~or;&Q<|!;t{ciWeq-%+?z$fFBGr2vlr=Ml!w{1-+5bC@Z5WPMi=uYpBa{E;G
z)dn|ie|Vr=zpUo1wa?EA<NfuoT;0lj(>@vRPyQ&!e188Io7i8x3xoH!JU@Q#>-^0L
z-*k4(sg>Jn{Z!~{n8C+|_8&eji<LO5#?)wVc-_u!_V*I)OAopUe3gGb^|*Y}v+1`D
zq<mWr8G5<jKeCwr&Z&>G+B?6OtEcc8l$_};F=ns)xb~(+aPSIE`?jPlm)1rb{{H@E
z(W+I`=1j7)`S@jWgm&46oZF1Q=2n~5?iLS-j5`_o{a;&?U+jjNLWSGD<;I@dBFX*I
z{*{%^R_Vyv7?U}MoQBn}y7yRobynHiml3n@_=D9IPquHL^yo+4`=<G=jga#&ogW`x
zTKn9t<n@GO$8@DFLIS(jtWG$5EcS`Wx1}n#&n$75f3R0IT3U40y~X*DHnQh=$Q+&`
zls7Guz1-mA(dj!={v0mJN>2XeUH|B7r1k_JzD>rLEM9R>Dw1OU-?w$Og-p{!asKqL
zkFM%O?`^GXmF$RUD37b$<jH8d^uu%Yc~5Rn-8{qOt#XK`zfa)HTmOyZ{(fdjiFE4y
zQCa;c*@yE=@ayhN&;Pux_%zY_#?Bl;{qsL3*>A}`eatdAqW(btAvL+br|17Ty}4dJ
z+Varh`pP-$Piq;xVzns#`SovoLdySbd%hnl-edLm@v(oqcz4XPzh5s~d@nQW`Nsd>
z)9b!p_*?Vu(fpeWN>V@B%N-N6Ir8}XJ?A~If7<-t9bNhTs`|{W79Vw1X|}r0tC^LY
z{miiN6#H3`&4o6f9=u&K^>)RJ{`%DG53PKxrUu+Or!V(I__5pcf7jDqT)6Nf{z!bZ
zdtSQ`TTB7-&UqI8cApHmr>{+Y^YbqUYqQMyl@HfEyl(vIO_0@+n;)--1m~W8*cjiF
z{<+<_qW)=BTfFtW`#my8iv9mJhi01>9%g0!`1|jXD*~6}Yvtl=ZR6(8KQxc|M{%Oz
z%LCs`=NlzIlxmCpui(5}s)BpgxohS^%grQ~?>PT6#=gMdV{>-JkC~ZkVmJ0ayxT6w
zRa5^nY~8&BSD)W^Ox)J3AHM$iiH}-lYi>Ka*3Qwh^_rz?A-eBJh1%;f`Q$sb<$+V~
z8g1Py#3|f;@TYiB>V*aS*37i;&78j`$ve_AR@Kq>YoU(i+?`La#b4N*ZT{re+bbdA
z6JmqT{Ls!%IHp~n{&(5dp8r>wdA{X6f4}}r>gQ?AJnyfyez?M#b8E-kOIvKh*4=Zh
zdVl(a?{|~eyQh7>H}{y}(St1;r$074DPy~%pFe!<;@tkkKaYNS-`H9uX&32kB6U2;
z!zR??iMvIyWcA~lU*66+y+wRp%2oAljr~uxR9tfvi`VC=PO1GRCwe?vL#|TQJmq!8
zzkL0TY2}vtcmB59ra$A{oKWc-l?x^%9rgBU%HJ1vFn*ivq44<rMdrnqZTW+OuJEab
zoQ=-+TXya4y!~<Z)BNLqOF2H;{`<zgtv+>+W$#ozFSs8YzM<&dYM=S$Q(K+SKU}qN
z?*7l8wd>P6=QD(cb{tL8{nvf`@qNk3&@8sNbd9Fvm)F5w8-6Lw-uC|a?Ypxr0;HF0
z$&ZgoYkZY)_=fcH&p#H|FH5ghUu_@hQn_`1tNVPR?%>#1XM<G*Deu*m?c2{1T&4K^
z|G({D-o030C6lu?U-09moS#KKnsayjTlG5Qy~(Dmr$(!syp&W6o0+0&zNy}+e_uP}
z)#-}TQ@?L)vif*J*uUWGf_rc7ZD=vw&98XcXyXd6EI)g>A79tMoROM(_EP`N@-=Lx
z5({>%>e`W$bmD`z#;Hxor(Ye>-R`SkfAh$m^5vV79w)Bb@zd_#hXZqGtlDPy^zO}^
z+k0l63{KzvBL8~Z(KWZj&+b&#%guTI?Djhq`@60O;(gbeH)t^|6j-xXQG%nPVZr{F
zk#ak}$LZWNimUybdndQ*<A-?p$A|X)tg+0OieGd8bYt%6$nB-#weInUmHGcSa<_kX
zUCu3Ne_39Pm$z4b>XS9www(U9A^o=3Gu<CD^(ON^oKq^4xpsM*xbV9j%@3dRDV<Gq
z4h^lE^6lXzZ~aB_{pT!n_}b=JPYPV@7JTU%tCq<UR^><Q7k;)B|9mQYeZk`wn{Rx$
znq(KXrE|spNs^P7D_2_;9h|V4<Huf&ESdTHrzLFpxxHuJHd(HZH+a{?{4kuq{)SV`
z)CqoTcDMZ6`&4+<fkz&R7Y@|9y#BU1?XTIk{QGWMb@^3dPffqYR9CRSKO5DsJGNX;
z+VIJg#hF%<r#2m8`r@T4e}4AaX?uT9EU8jqR1m1rejZz+Y~H)O=iO1+3!Y31H(&m~
z=k$(@D+Rlrq$iheOFzOYZcz24a)(TqOU%*@)%ISuEN*Nm<&M6-ez}p(#rgvZG0T?S
z-#z#B^bH9o*>>?u9h@+aGoO9+W#{GJGIQd0&saWh_K|-R7A(kC+?M8ke$BZME-tHp
z>J0wP>F1N{TkR7C1eso{%-k(n>Y~p1#!q~DOcCc{eT(@r?Y-aT?kYaG;v$dq73tX%
zkJPMYdX)Ic%J{qeACvm(Kjlka?(wNE)^pEx%8A<3;6MFc!i^5^6W=60v>wv<C~)=b
zR~g$Kv!nL&?=P2^KQs62x4&0UOmaz``aS;FpLvD+Os;_)?SCbnOgqNl8p`P++9SrU
zrkf1jpWb%tyR*@ybO-sE_jSjX@oICv>=(Fh`SSjLk$&serlxOe?(Ug;bN6-=4U=`j
zv4PTk45uc!d|hbt$MIn6;dKutO;onHbnRGE>+2}rmr<1&>e*)R4o$ey`*$~Af|=xv
z&8525-=-G(xrZ-is(LQF{j%ZVgUkHY9`hdhF!8!U$pMC6<$4bn-Kv^&>GF@P)~17L
z=~s4mOB$-omz5UZ{QB|NmPHTUeyYr!c<9l&EuN-@D)W5z|NC3;>a|5hg8nLzw9{>_
zZ?1DHt!mtH{rJNCvPb;$@5IZ#yP}iRwRz>-x?0a^T7o9_T$+j}i#EqvUi=hkS3Os1
zKL7G^d-*zp_Q$`4x*}btPM?>1d*ZYw2_M&7*Svpo>Z3(|C8y3mUVirH;)M%8)^{IT
z{@(xbO3(L31z#d|m1-rtel_*%JMJ%L>So2>;rZR3?R*nj9N0=YQs(xY)KO%Zd$8eb
z-tm-;m+KEOe6kjK^!EDn>*;!j;~y{GRAaBbPj9hS&ZCFFj|qRc{_y+N&;PXjC(oRB
z?CoN2;Y+Pq6|a{buSoY~O+MbcaIx!#qNmf+E`8kl)VaaI{e0c?35GwF?Pi-!@46fM
zMYh^>rd+Y};lh10>}Q|-{d&i*e;Y+oclt~2oTa?G^3TNM8s^Ikzg-G^pq-u_=^M;A
zS8ny?tCt>ot#{7%UNT)|63Yn%m6W}Lp3?-K9(d)RK2W!J#-0S(_rJ@MOSYss%zt!D
z?a{)lw6k+O=RSG(&v5!fLscK~Z%ek_`8~TmVqckZ`?t5xyDg0)cKuOs&A0XbT_Vj;
zeO>Rt#^dvp)+Q({ZYx>%XtwSZ6aL=?o6azmY-+i>gn6M^kE#cgkDBD%IU31@Ru{dE
z-f~$7-`cD_EkdkSc2ne@#q$a-oL|-Z+A{2M>c`*fH|8CDEn{6aB_ZVDLU(ne#0WKy
zs-oLd3{nc}lWM&spYUE>YVkb&7t8!YW=9*j<4IFaob|{(^H7sT!)wl|$5%tHhALiB
zITLn!rdTLw%dGQb`>s8c_LaP!aP*bx=bM`Bp6B1Yo?o=>P26Y2e%o&*D}ORenr;zo
zT_#%MW-FwXSTD;Zz}4QVv9|hGZuH(B=Tf_#kG6Zvekwe4=$Np5+EZ=w!ZOE4YtoBZ
zL((rOpVT_IL0*)DZLf@N5zpeou1Su1i3YEP)|YnvFZNobKG7g-MV3-?@01nkCTf?T
zuVGA4kA7Jjrn$+OeR~aq#m@ajCEHSRmMtr5vA8(Zp|`zbHP=Ls4lC8Zy&E4-JGF1k
zWk$}UIUg3(9r-=Q=!EzZ@8v#4pR7J!aIQ}{ye(Gd#^zSZ_4`lkt7?7zuSCA=tk;5t
zALE<6TKD%&Q1bBeR1#ac+)U!3i&ObRt}DX#^)JjQ^$r!*R#c3-C%&pHAXH^>@8;jU
z5B{>Qu1J1#e}%-g-t}T<)H5~&2VCJRFA}<bw~BqA!=Z|(+6wW{cJZ!>)9p&m`Q}gZ
z>kJByZ8ezf;Z+>Gzt(nQL{CemgF?%%zfXmkLLI8&3~P6%8bAK*zd<Fn@SJ6>l=$r9
z-}9gDy~+Kd<i)wCYb*E}ToyU@u`);83A^K`EuHnMb5*bQkrtjT<JDU%Ud~#j)N(7}
zSIZox3zzF3Fx;E7cZPo0QHh$Dxgsg5=4P97uRhoLQDl7g{O-1ewzG}C$=u!ASMM{Y
zXLqvZ^Et(Q*4x)Cw~=zmRCB%%?X4hlY}TxnQ2E8{_K7J5`Sm>0H0o~qDALKaLcMPD
zth>RjUSYe1*h)@5->f?SQ-7!5nvYy!Ywx??-h6Dro3abp4+`FG+Tu7#r9~+5=p+9h
z%dVZVT)Owf`x#!_ZlB!#wEg4ifSu{g@i~@0Y0nw|UsC_kBhAqgFj0kb@yRPnzxD{Y
zT?#sw@!?VbF%DMd8M<csw5KJ0ZfLjP-tlcMYm!ayr<0p{1hkgF-uF{tUQH~=gw#&q
z`W5zv4k(zPwyjzgB<wY*M@98t!0i41%5Pf*dz>xX9>urviq6&7=byg^&0QQnK5LHa
z{e62QKArN>GAz2<vTw<59+|J#1UTAHdV3#Ql^v9^;NkXQ+l6~p^mZjy-4-&sYbI9r
z$IML2a!;=5@oeVhUskfLj@VVhYQgoR{`|kGi_A`jyt#cOU;c-(_VilUqpG!y)40;y
z;>3$&#Kg|b@>&&heBajjn>Vd#Yyba^X_8B`jk<`2Xy^)+eYd4V^c5$h$XZRFx9e{7
zOtVxg^VoV#SJ~Z$`A3(_tXQRfSTOnHqW(e=zNOb)_ns`AcPHMozA8`X=3JfLw?1=?
z=DuEMAzof2RqB2H<D<|1i$BYy#IK#cj`w`^CZoACug}cc-t7Feos(n5o!J3rHtqhB
zd^kPcsPftJFLvcB&v{;)|20*blQr4CGAe<WxA(s{Pmhe@q~7Rnn_e_WK6&<Z^8JOf
znsVZs{Ep3?<WyzB8S7jS6q&AKYRkOcdScbJM+K8-&YbAsul>4U`7&YOB1OeOt3N^S
zbl0bxy8XGb^pe{bHu39y``ZPXyEjDh+_C#7-Ww!yV@t={)TuoZ#jJmKKQr@8_fC8=
z<MoZ-pJtz#WjOWEx7^I>v)eBxI!{Wub8@A<hn-|&j$hJ=2cIMNz15g`m;LhI%bl{>
z2d`&`2Mcd5I?5X!c%$n(gF{vsqj|sd+1bU@jJ}@zo;mxe;`-j6qf3r2*gkRM%*Mls
zc{=%9RMuI(O`D(+bMj=$5^sqwMpk=ge(CXfad__w1_p`Co-U3dbNHm0GVbrRXRZ?2
z{HmdUp)<RB%uhx^?hRE}qt?8fyW`cVjXQjOmvvt;`&p7X``{bb;+q>>+Aq#Js=Gbj
zIWKE*=Zru14(}+qA+;}Rf5-R!r;kqRIPfh^nvhyDEjV?X<=vDe%ae^>Oek>+x@0V%
znU^BjW2P=C`SOO3)t7TT@BW<4=dk_DZ#U19IdsyC3%MG?@AfsO?4J3r+5KYuhZQc1
zH!>_MV0~rCe?88yiAVa*-`Be<e)hiH^XHDSM2ax;xxG!Bxr%>=Pbic9HuvKFr3D%-
zXID&^KfCbM-VBqj`|sWJ4GKP-C`_8fy83M0tseGCDi2q$esp4D(W4V8%Tp)Kd*tXg
z=c9c5LamC|S)9kGs8yy)PPgWq{P6d(+F#`p0v0)L%e&{4c3dx|>djgmq0pLpRV_J(
zGPj$>g+2;=wYP4Kn6mUVeb?!;{O8NHdi+186g){qNzvY&PxhF6-(xQ?HEm_t=D2?c
zyRToCG~c({@8>^TnXju$-rnh4)m&}yD)?p4CC<HH{yxdLca4AL|Hld(?M*F*4SU46
z!-|8WCP%2aD2TYF8zgU!+IRUynNioG2cNH9+?#Bs_E>VeVOGYUaDiSoO>OPP&h2~>
zz5nC>J(_%T^Dp6d^*<z=iiGz6Jsr&R`flH^u3DSVM;7bc-O#?ICnl!N;_E__d9%N0
z{C>9o<mUIfPs06umQ0R0rvJ}mVt3we<?X%ijx}wMPrvk}d_|w5r)PTBx3`O)G-?0*
zYrDPhkw*I2h3j{|pT~YdeExCc`aR;?Bh|!KJX&dg@BF3wraxah-6O4^vx>=m`ta#8
z@6TCNCAy!VPq`$0*)r&<bAE|o&YjXfpPrqLG=8f(Z}!fPYPsLi%2iuIz5K_2Tcs<L
z9(iBUnRoPvWq0L^eEr~A^A9b5zyIKpmBpRXu{M9s+&^i~{`0u~yyTCw);l^BxhZb<
z{ULhzyk5dd_xE$J+~0bq^l-!-wW<E;zL`1CCoZ4wzP;YG|NmQ7z7W^U>ERA{$~)F<
z)_Zm_{{D&C=WUOOyHE0~oHl*A&+<ifJ^LIDe?Pr@U1<G{lf~!l&;FX*e(@w*%*vm4
z%pS^}laE-!s(IzYk9;SNSFf`^{5otgM^DAHNlCI`w+zdk(iV}<l<nMebrh#qlwLLU
zNYl<+K6k_Yc5{J4FFg<COjxmct=rVI6*1?IJPWq*`gw8j74}0BZ}dP1EWh&S)fVq>
zTe<RkV(E?z^GjkJRv2cjTE%hv;L{=@PBy~`8H=}SHDCYFtJ&x=;g*#?+h*SFmoF|z
z=VH27RkKjTuUshX^2@KsBUF4sOHEI>%_-wP7{IanbaP<Pigllpw(i~#-J?7I;=4n6
zh7VS+R~4D);Z>&GDRI@bbjDh~Ic3}&qWQJD3s*Fqp4+n1v0c<@;l6EcE0i94eoDA-
za`}nkKR*f&U6crGpM52T-)iSm^D>J7|I)SMi?24i>I-sdS4^^9;I?Ar=dBYg-+X__
z-BzHgnmmDJ>s8Z)O%Dn!J6iu<scg2m;~63Kg!@Wp@s>qgp_e93n&!D_x9R$-izfMU
z;#Qtc9IdxrX}#LR*rTSsGIQnsPqRL!rz>!ThXwAgv|h>VHBmS1Po4cl50-ibhtODo
zW?e~d?c=;EoZssA&RUs&cF7&-qbu5X<Rm^h8g7&nkpJb=$$PBoG3-l^$v-{vZbwE9
zFK?dl*>~#e%x_!TudGWtZt?J%#>FijyjQ~wUanNX5h;8$%SfT+Kt_4t+qdNxmG$=?
zyt<_Fe&^lU_3qEao~tfZi)=D=uYCA2{YCxZy*xUnayKUHv3O*)>@-%}c>49jcemFa
zO!Jtt{m#y3!OPCX9`@ZHdtTf)?r?y?gxSY?UGzgT*3>_;jQUpj$$0L9Z8i4B%Y0L#
zBQ4W5PxP?x`(Er;a5YuAAd-he(X>K)%NLH!w>Mh^PH3!&-#e$nu(xinw?nA(X|-cs
zxf2u>-*fh}MyhX4x+XGBF7(G${k-JM?D-$A3byV(e!t0AUF!T!v-!W>Bvm+9&U}8j
zbK5l8&>g?eMQ?naX}I5>byCNsH{Jb_dw2HSSrnuvWysMT$7FhHmA+l#(`U9zw!A+#
z<D&CM$*1pA9WL{*NJ-h-{h!Xpq$;;`*)l%Gmr)w_%QZ{{S^X!mt3Fx2&+(`6oFx-D
zRozNcMFkn<wzp>-U{~YTTfX;Wxo4;Aigjxbr)+#!GG~Y0hw9gyEsJ*CkxNRf=VlcV
z@`}n0Z8l2YIC<K%ra=AeA#S}@fA+ap&s(D6V$reP@uJbI(iL7kzkXGTgdTcXmVD)P
zM;miUh@ASm25n8&{?qq9{Jv9Fe?arCL`gtwWS_po@ySzoJ_<PHAA7syvAyTb{F-TQ
zN6)xrMR_ew663oP92x2z!@6Cawg2`yNp{e2S|)X7f8K1VzHiQc<Lmd66}v=}w?!6O
z$lQ2a7NaK~mM!bPx@pBK*2ug~?>r{8?EfW@zt^&N!dl_|QuUT{n;)$9Pcy9kWwNQL
zSX0+y@v>Eu9$(%1qDRcg$av!{9+xJ&SImMcrf#vd4E|A8Wy(|JS36vtdE(5e14$+f
zO}k!sX|2vOjFREZp6fV$#e43?j>p_BC$lDZ&yx^6eu}dtdB@~nPj62r4X%9-9h=YX
zpCq{Sr$$%Tn#+^tyjr&HnT6m&^`np3O#MPbMcKNAw_o;&{rXW}p;#?#uS;yMu2Z_g
zoAv4feUs<RYCK+gXvZRj77q>8RZgxy6JPytlzAs&WE`ZCaetcsvig(Tr-}#MQIizp
z-<bH;s^pzd(WcEizZ|=L;DuYi!wRmcXJ4f+QC;l9eC1mE2Nl(Y(^*YJufEXrn^R`&
z7;k$}V#nL6XXodfdzSBf$K+qQ$TrWU=eyR=FiPe-nR&BEHm*yEui(cE!={NA^QM#p
zO^)?&da!=IvQT)3JJXe)hgFuQO?XxKX#Lu>my7&A+_t`?r$1Yyy>-_vD;3UHd+qM5
zj8wVF8zN`z(`(*vcBV_3Th?Jq4N;e7M-P*^iB5~A?wl^{m8jyUBcsqXan72OFTG-}
zqM;fg;WAG<?zbyzPTi97-G2#-nvU(n0)h3{Pyf4XZE<V2_0O4de@dsmxv{m~YQ1=|
z!jz5^b2n6j&J3Eq=(4n@uebHnT{oX8-F}u;Q-3HiD8GC{TKSv`_o%F_WJ%9-1y$qC
zANO9(vfFCk+_`1Tm&xnAZp_VlbFOWlubhnf^a&*@OP5Waa_iHso9_<f<ZI=Y@5rdh
zE}wAg&Ac_n4?Mjx1Mg>SyQsE(Yj5kd-`26#nlEpf#AX)hm+tWUtfTpI&J~x(JD!_K
zmxI<O@b=CZPQT-zAobrf;q+Oz^xRWX1xeGdB^XSYG{<MIy`PEL;S3{FZRO&M?`eEj
zU(P6aZqa3(qQnwo@-6Djx}`ICgi@o!Ki#@AvqgmYW5$Fs+xDX0rPBj9Cac!Zx8iR;
z>ZBv#lJBo}Id{pPMO^U;KKlcccfNb{*<wQ}YyP<zuDJ$%0W3?M^`E>t?N(6n+w0%I
zV=*lU8_rHWsk=S?__Fh9pRPV@p22HWIJdXZ{H|kv4)dlp>Xxd_X1TW(XlQu|S$U@Z
zd~|-s<`Vt#d!My=uHLT9Jk=gkbC|{Gsq&t0&-f?)<XUI7f6C6__=MLNIjl6@R0Iyp
zyEQNR{f^$8&z%qagq(UezS(SfX5N3*E*<5PFui@B+<u9FS3A6ELT~i9fIDRj_f{$A
z+%9gp<gI@sGvsH+#p50;m&TaxQup>>XaCM^SBgu;%dPx7pRJ3Zl>N>4V`ut_xAlhR
zy^?39%kMq@ZbEzE;oX})zRtJ&-^DrS?rMn^g~}VP9#+OX&bItDt+9B_yYJQ)Ube-2
z3(Jk3-I^_NWwWie{rto3@wo?*HVSdE8tgLKS$n9WFaKwk@9pn`m!})wefMb9!@Cu4
z^MzOTaj7is^<KAAviPakz1jO&=H7HsSz}aj<!OvrRzT9ndpWH7o(kRp+frGBqt5(w
zjVZnv5LRb4-Lg9@`GT~E{>$@s&0{-HUn+?6o@Tiv*;ZdpPg#U#!Hhe13VzR>6FTGX
z<0H90#3!cCoUne@-0E+a?Jgf)nEx#xP}*g=cSS(P?l36?P`7x@))YKJ=-nGH3#TpH
z<o4ce%kk(~dHLmqmnm0ncsAZwam;lU-S%bMlQflutGJvRPR={!Iz7Ix<<zsS)-UV|
zRG5E#TU7Jh>t02^=5e?A{2ML4eofSqFXq0x`OBUUXEs;ZGVgeQ(E7%Yo4advl_?+o
zU7GlhT|Ob}a`*jS*<%ZJPtUN*wu`O)dSvO5mEYTUK6sTaSrnGuQ^av|;?b|$<2`nZ
zDCh4t>-_rK{IcqDekHkQt67aL_gE?}zI(UNFxl<m=Jx_&>u#Lcoqyip{qJd04h2<-
zJl#-!xlvy2^DN$;H}|JH)bctWJsT^m_8?$Y#m}XlHJdV6&0{WT8|G|sd3$khjX}QP
zUia9e8#Z%An*Eo3c6O%wcl&#;>-B!FxwVDy{?FPQ`;(31EIa4TwQdODWT|-}W0=)f
zSM2q-|5=U8!PsQO)nDJ-itoD_=6Ca<^N+H(%eF1EzCY!y`pRf=mmPg84*CTq&UR7I
z32iuc<mO?0r)deVC+*aUiasW4-oL!sn*Vc}PujaTKTC9F{7qgQ7dI`sJNNNP1Gb9H
zN2k}^3pf_~eBSA2xzQhPTNQrZ!9Ahoeo%O{|GkF5p4}f}KFB}387_Y5@gKt;xjW6i
z@pVUje|~mpr+d1}&nHhextIi;b}xQD&*Z!O#kJw-Ns9J~i4Si@=PlpiEd55!van|1
z!ZMAlC#$sc7b&<t+uA7{xwlH#mixBC;t#=z22q7KzP`D=xBJv2vp09PvQM^Vb>AYV
zYqt=zJJG1?)8*Rf;B6oof7X96HWV<4Ug;t5{io*tB*}lZ_mfVZQ}};v^+evbW1H5l
z%T`&bnd-86o6yByiq`7WYk!A-Jh_LTvr4;Dn@jzZx5cDeLASQf;K`mcXHALBPt`5C
zlbsdz%g)cg=M-DY`g{M{V+Rby^dcNABHUG^p1mm4*E=xrPT%I=-;```dg&xBiY=46
zeN8_(+UDFuE&a+|R+Bk<%Xi&bvUKU9Ra#=pxg*)4i_DUb#~xeUZ`VF2^2~eno+D{j
zCQqI0eUpE}&&|HGJC8o%xaqTgpYw~T8SOen5;s=2lpgOF>ih2TgWp5Q^f+IT@Q2`i
z_3YfM_yXdes=oOeBp&plGE9>@e%4v@-~5{@KVRSS_Db)Y6m};Tk2!y*N3WG%^<D7t
z?GyRak3QUMJ+XId$(u{<9s4K6{{9veD89VjvpVIggY&K_la~iMO%wU{{`a(=CcQ8E
zZ@VrpNxt#Z`G!R%<JEgESKjPdl(b7kG(p*+&QZ@xrSE9k@hro|H{Pi1l)dts)ou3N
zeavhZ^wcNIp1v-$bxHBN2aG98EF}$I#7xtTJ>RHosxp7ed_7Mg#(2};e4)uxW}U6o
zIWzH8Rmy_cK+e6prYB5|PKyr+ox11Ixi4>mT26~BFJ8{ncSOm4qx${>31t`77R$Xh
zf0Occ*3~zwKLlM^%fB!GTJY+*+Mt^h`CUUp_r$GFQ@H>JKPLX1DZw#I(ZT42r)+_k
z=ic2Lzxtn_Q2kQljO5DyOZr@bs-zB{DwVMabB#(gb8=7#I6GZCY~tMeGq1{@P59d=
z!OZ?u@#fOx^2pHSufhjeTD1f`c-Koe-86l}@w(LN(q-e5nUnigrA@xQZ^ooOA&XLX
zPncj)IOF$;cVGAXd*;LC{X2NZbl0N`^Jn=;RS7R`b~8*X@O;$KC~#z|g_DDgl;b3o
zM;|#BX0Vv58P`86F{_jQf6z#o#VaQBjI373)}_&v%@I~}r}mu5xpX&d(<UR0SE;I3
z6OX-n;j%k~nOj_E$tMTX!}%{-Bl)w|+`Qqv_PW3Sr+_OPWu^Du@md?V^!tKti)}xR
zr|J9?Iy%=iu1bh4-geoH`LfpEF6-*^Mv8H$9=fH!pQT*vMac@U;8z-|>l9RWMqW7B
zX6T^cS@c-hl$CkPtbbF5G`CK0(9p_qewP;-8W<Y&Q7$9vpu&8s?{hz2y0J^!@Pd;l
zho{NOs{z+jE~)-<RXA;x^6|3u7jBlDZ>@_8&gYq}zuj<ks^{`QHrL+AexH1L5oo1g
zvf^nYzvZ8Ox<Ttm7cTr5fBgJ;2Zk4C)OzN*Fv#D`a0q?$f6E>LEz8!s8Pm5klrQ%^
zUh_9%(pn*H@!BKDPnQ?}wv}D6dY$TKUwyW3Z;#Gu-OSmxn71_g-5)N+S(YX$VpGl1
zjbv|CSiI_<5PxUtfsM90vHP3-_Wo8sW%A~gvS-VlQWsI?+F#$)&jy@-bZghK2d=Ax
zp1*j={N~O|Z=JAITgjW24}MSke(!(BqJ@hm_P>2F_xrPtr)TRg*|%)k8MD3nx1U*b
zaI%xysa>JmFK;g8)mz%N{#vg3|2c*;c!QEd{yuqAR*_?MNaDq<wc5MQ*Cjn)$naN1
za<Xmby}Q<iAJ6VwB0AUidefX`pFcY-;QG4d>u=@Bmp8UN*s^m<^sS;~PsMGo*4EfO
zbw0Q6i(5>Ya8)p0dg<BZhwSesy)z7+GyT-Pv)VJheXBXQ`<dRZ(l-b8=-WT&UYh-K
z>idU}e{yeJ`gFphEn81m-nC#;c@#K<*KMC=)r`ZUS0h4a*?6}os4iy?OV#7H``^^v
z{w3}1B-RkEFsmm^0{*VOH6>kI>X}{Zx7<k<WtZ-Vf7Nrn@MNRX!VQU6!{4kDh$%Wa
zLCi?eZ|>&#O~P)<-Nvnctd&z_Z+evdb~2j$w%78UsXP;C3^Jr-*L)+_4WL7y+m8Kb
zvv+20bkOiMWNTSfSI(=rK{d7Tudey$YtipN9V+%uyx_^PcWU?gJ#BHWvI5NPM#cA(
z<mzjf{gyYc*;4YwO7diqs^-z&HIp5t6;|0qyiH?CFo^uPWQU>kWJ%jHnc!u6k8k$3
zX;bkOnP=K!GQD@zv0p|`p(k}-xIOQCJ@enK`x|e}kyv}O;*FWgT)F9|kG1JP+myVM
zvE@L<_kzbiIIrZ!`yH5b?(6hk$5nTZbx%&;xO9ztEgS!{z5I6n{~0Z>y6`e}Lhp=}
zsg}X<2gP|N&bmIC`<!%xqxhkzQ(6MoAD(=%QCn|!!%X?wQ`?@-SA6ZmBUfq}Hd#(F
zrK@`CrNonLUBA95-b~CpQKOJtzVl$(k;(U}YP&uA=S=b4{)kVeTx8;-vPJruYU}@o
zb#;3icrhhzFnZPX`h)xO&+nx56HQ)q9%s4R9(!oR=G_l2xbFC{CqL36U8*}}gN3m~
z`Cr!^YuR+n)a9;TdZC<9yko*zwU%=p*Voxb{68m?cIM}l*KS`_EHx*3hlnzjyHy|S
z-8*6CXR%L5Zs|LuKRmSY@rw!Xj-?krC^|OL)8%ZL<ztE5JiR@Syq5XQzM1qcIJRs}
zn7Z82I}`4QKD{vSZO#pC3CkRTwB5qfjoa=B{JfA;oFHyiH>>$SXY$meS;t=nO#JHB
zwd<Rb!tN8gZ+dPnIGj~3P#z)tDKm@-+$(?(4!25gl)Fu5Ul_nKThY()wvRwl*ZQ!H
zzgsn@{7GYCy7BRfx5&Lx;g`+78dWxae_+UDXe-wJbjmgrPo)>D7w`FG(KVrP=VVXM
zGG*`1yT5PE$~qJ%+J2d_tLf-F2GcHkk)w&*7f(L${=1^&nHxH4k0uKIE8tmvxX*9e
z$Lj2rYrlhTtM>EL*^tbeDpO`{73nldLF7YV^mL7MV`+g+a;uMa+>iEGnP#D{>2m1p
zotH&IyzCvx2U52;27I`7ReAQB8)tsWs}w899ADM{`O@Y7WPPclsn11)IgS>nFMY6K
zR)A=0!~OP)S;tpgS6&zp9owrfaWJd+!N(mI4-54}Sm$4FKbUklDXWWFkcUOUO4+QN
z({s+ceF`tWFZ5!2W2U`I=<xH;$s4T=UX}J1i*d7ChQ&ErNF7frpEPY+m!rOV*~iC$
zt5&b>JR&0F$Dg;HSCQr6g84JF61U%;Ai%=JI6?RmhiTjPeR82@54X(Z{PB0j?C+gk
zxxS|V5?Cf*^(&ghK3AoqrO8F4@ot*3SYDN`caV0mzA%@`bjCxB_JXW)_^da21lMwM
zurq5owVzFUw%k#Gv%TXOOW17Dz3Vh}SFF-%VPb5_Ign+U_~6I&KoQpY+xy<hZZ?#x
zZ!>e_?fvht=%PkyYHG=g3yg~|RxCSu5nK^p_#yu^Jzb%rvi69x;FoPJj0$ob3pdX&
zu#&x>snBqw>GP4uf{PE_?*EA^7W%&A*ycCBseU{9=2!VoS~B@oZsfA9b$(yI#$Jz}
z{E+>mj<-gle*XTCTHl%$#@Ou<cM6Q&b;R7}_{qm*pJvRxRrmV3U-<s5>DD`IRXBf|
z6qoJV_W$OLxi$N@+C@(dpCpxRy`|#s-V}@4zt2+Zh5RNxykhw`@lysH6K}7){8@(u
z(E6zOK~MH~p|efTJcZb%3pcNRx$j`2eeKV%jNHbXf8w;mqis*u?W$hBP0{b8)c?A_
z^Q-p${4KAtS!)8Xncm)KVJgj%s&!dYo?rX*$FAuA>aYKAZZ_Th!~gvA?bp72udKJ)
zd1R&Vb)6~Br*kClN4JH&`u~KZ@YZGKbQd{D9s!ZP^G*ND?{VfozVoo)76t|e)e_f;
zl9a@fRIB8o)Wnih1|tJQ3tdAaT_b}KLnA8#6DuPFT>}d%1B1m&-;|?h$jwj5OsmAL
z;nI(g(+ms@8Xy}olasBI^9w4AGSf3k7@T1&1;>JdoKywRyktEc1(419KCF%m3=HBh
pJt?Wh$wiq3C7Jno3@(|)1v!b8;?h%hFfcGMc)I$ztaD0e0ss$)1D5~*

literal 0
HcmV?d00001

-- 
GitLab


From 5d7d5a76a339a7ee947aa2e7df812787346e01a5 Mon Sep 17 00:00:00 2001
From: "Wen-Heng (Jack) Chung" <whchung@gmail.com>
Date: Tue, 15 Jan 2019 22:18:50 +0000
Subject: [PATCH 0749/2345] [CUDA] Fix invalid nvcc configuration

---
 third_party/gpus/cuda_configure.bzl | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl
index af8876ba17..f8f3e90708 100644
--- a/third_party/gpus/cuda_configure.bzl
+++ b/third_party/gpus/cuda_configure.bzl
@@ -1409,9 +1409,11 @@ def _create_local_cuda_repository(repository_ctx):
     # bazel's header check failing.
     cuda_defines["%{extra_no_canonical_prefixes_flags}"] = (
         "flag: \"-fno-canonical-system-headers\"")
-    nvcc_path = "cuda/bin/nvcc"
-    if _is_windows(repository_ctx):
-      nvcc_path += ".exe"
+    nvcc_path = str(
+        repository_ctx.path("%s/bin/nvcc%s" % (
+            cuda_config.cuda_toolkit_path,
+            ".exe" if _is_windows(repository_ctx) else "",
+        )))
     _tpl(
         repository_ctx,
         "crosstool:BUILD",
-- 
GitLab


From b4aed7c931041d9d8406ad45fed49e56d7f394e9 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 15 Jan 2019 14:12:28 -0800
Subject: [PATCH 0750/2345] Make XlaCompiledCpuFunction constructor explicit;
 NFC

PiperOrigin-RevId: 229438156
---
 tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h b/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h
index e1da19cf1d..de2e485a47 100644
--- a/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h
+++ b/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h
@@ -112,7 +112,7 @@ class XlaCompiledCpuFunction {
     RESULTS_PROFILES_AND_TEMPS_ONLY,
   };
 
-  XlaCompiledCpuFunction(
+  explicit XlaCompiledCpuFunction(
       const StaticData& static_data,
       AllocMode alloc_mode = AllocMode::ARGS_RESULTS_PROFILES_AND_TEMPS);
   virtual ~XlaCompiledCpuFunction();
-- 
GitLab


From d25d54029fe407932155ad0d527510ebd9ec8e07 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 15 Jan 2019 14:30:57 -0800
Subject: [PATCH 0751/2345] [TF:XLA] Bump open source llvm revision to r351210

PiperOrigin-RevId: 229441372
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 9a699ae243..81de0fb45b 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -502,11 +502,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "066d58d687ea0a97d36135d4aecf6ea31b900cfc74582b2349cadf0c7e4a04fd",
-        strip_prefix = "llvm-06a887ac8a688a23aa09252f44e9bb95797d057f",
+        sha256 = "c6e6de0b5cd524acf276188cb6aae427568070cd1b9cb9c0e3aab8044f3ac278",
+        strip_prefix = "llvm-a42cde684126a3797f3eda9c894c379e7a8c66c1",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/06a887ac8a688a23aa09252f44e9bb95797d057f.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/06a887ac8a688a23aa09252f44e9bb95797d057f.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/a42cde684126a3797f3eda9c894c379e7a8c66c1.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/a42cde684126a3797f3eda9c894c379e7a8c66c1.tar.gz",
         ],
     )
 
-- 
GitLab


From 73b1e8ebbee9246a85acaebcce9b9a8672c22216 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 15 Jan 2019 14:31:41 -0800
Subject: [PATCH 0752/2345] Use default value if the explicit_padding attribute
 is not specified for Conv2D

PiperOrigin-RevId: 229441517
---
 tensorflow/core/framework/common_shape_fns.cc | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/framework/common_shape_fns.cc b/tensorflow/core/framework/common_shape_fns.cc
index 3d3711ffc5..876ac188ac 100644
--- a/tensorflow/core/framework/common_shape_fns.cc
+++ b/tensorflow/core/framework/common_shape_fns.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 #include "tensorflow/core/framework/common_shape_fns.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
+#include "tensorflow/core/lib/core/errors.h"
 
 namespace tensorflow {
 
@@ -498,7 +499,12 @@ Status Conv2DShapeImpl(shape_inference::InferenceContext* c,
 
   std::vector<int64> explicit_paddings;
   if (supports_explicit_padding) {
-    TF_RETURN_IF_ERROR(c->GetAttr("explicit_paddings", &explicit_paddings));
+    Status s = c->GetAttr("explicit_paddings", &explicit_paddings);
+    // Use the default value, which is an empty list, if the attribute is not
+    // found. Otherwise return the error to the caller.
+    if (!s.ok() && !errors::IsNotFound(s)) {
+      return s;
+    }
     TF_RETURN_IF_ERROR(CheckValidPadding(padding, explicit_paddings,
                                          /*num_dims=*/4, data_format));
   } else {
-- 
GitLab


From 750b30d2aa9e41802028a1c6693f73fa6ec6be1b Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Tue, 15 Jan 2019 15:09:35 -0800
Subject: [PATCH 0753/2345] Make layer JSON config saved in checkpoints
 optional for restore

In general if a Python string is in the checkpoint but not used directly in the saving program, assert_consumed will pass even if the attribute is totally absent on restore.

This should fix checkpoint compatibility with saved_model.load() even without proper Keras revived types. There's no reason it should fail if those aren't available for some reason.

PiperOrigin-RevId: 229448783
---
 .../checkpointable_object_graph.proto         |  4 +++
 .../python/training/checkpointable/base.py    | 14 ++++++++--
 .../training/checkpointable/base_test.py      | 28 +++++++++++++++++++
 .../python/training/checkpointable/util.py    | 11 +++++++-
 .../python/training/saving/saveable_object.py |  5 ++++
 5 files changed, 58 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/protobuf/checkpointable_object_graph.proto b/tensorflow/core/protobuf/checkpointable_object_graph.proto
index 651f692f6d..f2956404b5 100644
--- a/tensorflow/core/protobuf/checkpointable_object_graph.proto
+++ b/tensorflow/core/protobuf/checkpointable_object_graph.proto
@@ -30,6 +30,10 @@ message CheckpointableObjectGraph {
       string full_name = 2;
       // The generated name of the Tensor in the checkpoint.
       string checkpoint_key = 3;
+      // Whether checkpoints should be considered as matching even without this
+      // value restored. Used for non-critical values which don't affect the
+      // TensorFlow graph, such as layer configurations.
+      bool optional_restore = 4;
     }
 
     message SlotVariableReference {
diff --git a/tensorflow/python/training/checkpointable/base.py b/tensorflow/python/training/checkpointable/base.py
index 1aaa85258c..f2797bc5ff 100644
--- a/tensorflow/python/training/checkpointable/base.py
+++ b/tensorflow/python/training/checkpointable/base.py
@@ -142,8 +142,10 @@ class PythonStringStateSaveable(PythonStateSaveable):
       state_callback: A function taking no arguments which returns a
         string. This function is run every time a checkpoint is written.
       restore_callback: A function taking a Python string, used to restore
-        state. Optional; defaults to doing nothing.
+        state. Optional; defaults to doing nothing, in which case it is ignored
+        by status assertions such as assert_consumed().
     """
+    self._has_trivial_state_callback = (restore_callback is None)
     def _state_callback_wrapper():
       with ops.init_scope():
         return state_callback()
@@ -156,6 +158,11 @@ class PythonStringStateSaveable(PythonStateSaveable):
     super(PythonStringStateSaveable, self).__init__(
         self._save_string, [spec], name)
 
+  @property
+  def optional_restore(self):
+    """For values with no restore, relaxes assert_consumed()."""
+    return self._has_trivial_state_callback
+
   def feed_dict_additions(self):
     """When running a graph, indicates fresh state to feed."""
     return {self._save_string: self._state_callback()}
@@ -351,8 +358,9 @@ class _CheckpointPosition(object):
           # added or deleted. Stores unused attributes so an exception can be
           # raised if the user decides to check that everything in the
           # checkpoint was loaded.
-          self._checkpoint.unused_attributes.setdefault(
-              self.checkpointable, []).append(serialized_tensor.name)
+          if not serialized_tensor.optional_restore:
+            self._checkpoint.unused_attributes.setdefault(
+                self.checkpointable, []).append(serialized_tensor.name)
           continue
         if callable(saveable_factory):
           saveable = saveable_factory(name=serialized_tensor.checkpoint_key)
diff --git a/tensorflow/python/training/checkpointable/base_test.py b/tensorflow/python/training/checkpointable/base_test.py
index fd935ac559..736cbee488 100644
--- a/tensorflow/python/training/checkpointable/base_test.py
+++ b/tensorflow/python/training/checkpointable/base_test.py
@@ -16,6 +16,9 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import functools
+import os
+
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.platform import test
@@ -57,5 +60,30 @@ class InterfaceTests(test.TestCase):
             name="v", shape=[], overwrite=False,
             getter=variable_scope.get_variable)
 
+  def testAssertConsumedWithUnusedPythonState(self):
+    has_config = base.CheckpointableBase()
+    has_config.get_config = lambda: {}
+    saved = util.Checkpoint(obj=has_config)
+    save_path = saved.save(os.path.join(self.get_temp_dir(), "ckpt"))
+    restored = util.Checkpoint(obj=base.CheckpointableBase())
+    restored.restore(save_path).assert_consumed()
+
+  def testAssertConsumedFailsWithUsedPythonState(self):
+    has_config = base.CheckpointableBase()
+    attributes = {
+        "foo_attr": functools.partial(
+            base.PythonStringStateSaveable,
+            state_callback=lambda: "",
+            restore_callback=lambda x: None)}
+    has_config._gather_saveables_for_checkpoint = lambda: attributes
+    saved = util.Checkpoint(obj=has_config)
+    save_path = saved.save(os.path.join(self.get_temp_dir(), "ckpt"))
+    restored = util.Checkpoint(obj=base.CheckpointableBase())
+    status = restored.restore(save_path)
+    with self.assertRaisesRegexp(AssertionError, "foo_attr"):
+      status.assert_consumed()
+
+
 if __name__ == "__main__":
+  ops.enable_eager_execution()
   test.main()
diff --git a/tensorflow/python/training/checkpointable/util.py b/tensorflow/python/training/checkpointable/util.py
index a45263f5c6..1ff0cd9f25 100644
--- a/tensorflow/python/training/checkpointable/util.py
+++ b/tensorflow/python/training/checkpointable/util.py
@@ -666,7 +666,13 @@ def _add_attributes_to_object_graph(
         if cached_attributes is not None:
           cached_attributes[name] = saveables
 
+      optional_restore = None
       for saveable in saveables:
+        if optional_restore is None:
+          optional_restore = saveable.optional_restore
+        else:
+          optional_restore = optional_restore and saveable.optional_restore
+
         if hasattr(saveable, "full_name"):
           attribute.full_name = saveable.full_name
         if isinstance(saveable, base.PythonStateSaveable):
@@ -688,6 +694,9 @@ def _add_attributes_to_object_graph(
                     % (checkpointable, new_feed_key))
             feed_additions.update(saveable_feed_dict)
         named_saveable_objects.append(saveable)
+      if optional_restore is None:
+        optional_restore = False
+      attribute.optional_restore = optional_restore
 
   return named_saveable_objects, feed_additions
 
@@ -1020,7 +1029,7 @@ class CheckpointLoadStatus(_LoadStatus):
       raise AssertionError(
           ("Unused attributes in these objects (the attributes exist in the "
            "checkpoint but not in the objects): %s") % (
-               self._checkpoint.unused_attributes.items(),))
+               list(self._checkpoint.unused_attributes.items()),))
     return self
 
   def assert_existing_objects_matched(self):
diff --git a/tensorflow/python/training/saving/saveable_object.py b/tensorflow/python/training/saving/saveable_object.py
index 4b19294b65..981d4580fc 100644
--- a/tensorflow/python/training/saving/saveable_object.py
+++ b/tensorflow/python/training/saving/saveable_object.py
@@ -66,6 +66,11 @@ class SaveableObject(object):
     self.name = name
     self._device = None
 
+  @property
+  def optional_restore(self):
+    """A hint to restore assertions that this object is optional."""
+    return False  # Default to required
+
   @property
   def device(self):
     """The device for SaveSpec Tensors."""
-- 
GitLab


From ee64c06de75664e937d3f6e7791dac9a9e6731ac Mon Sep 17 00:00:00 2001
From: Yanan Cao <ycao@google.com>
Date: Tue, 15 Jan 2019 15:15:19 -0800
Subject: [PATCH 0754/2345] Support nested output structure in xla.compile and
 tpu.rewrite

PiperOrigin-RevId: 229449848
---
 tensorflow/contrib/compiler/xla.py       | 227 +++++++++++++++------
 tensorflow/contrib/tpu/python/tpu/tpu.py | 239 ++++++++++++++++-------
 2 files changed, 335 insertions(+), 131 deletions(-)

diff --git a/tensorflow/contrib/compiler/xla.py b/tensorflow/contrib/compiler/xla.py
index 2aa5d77b10..0f1be500f4 100644
--- a/tensorflow/contrib/compiler/xla.py
+++ b/tensorflow/contrib/compiler/xla.py
@@ -85,7 +85,14 @@ def compile(computation, inputs=None):  # pylint: disable=redefined-builtin
       with `tf.convert_to_tensor`.
 
   Returns:
-    A list of output tensors.
+    Same data structure as if computation(*inputs) is called directly with some
+    exceptions for correctness. Exceptions include:
+      1) None output: a NoOp would be returned which control-depends on
+         computation.
+      2) Single value output: A tuple containing the value would be returned.
+      3) Operation-only outputs: a NoOp would be returned which
+         control-depends on computation.
+      TODO(b/121383831): Investigate into removing these special cases.
   """
   # pylint: disable=protected-access
   return _compile_internal(computation, inputs)
@@ -251,13 +258,21 @@ def _compile_internal(computation, inputs=None):
   Args:
     computation: A Python function that builds the computation to compile and
       execute.
-    inputs: A list of input tensors or `None` (equivalent to `[]`). Its order
-      should match ordering of computation arguments.
+    inputs: A list of inputs or `None` (equivalent to an empty list). Each input
+      can be a nested structure containing values that are convertible to
+      tensors. Note that passing an N-dimension list of compatible values will
+      result in a N-dimension list of scalar tensors rather than a single Rank-N
+      tensors. If you need different behavior, convert part of inputs to tensors
+      with `tf.convert_to_tensor`.
+
   Returns:
-    A list of output tensors from computation.
+    Same data structure as if computation(*inputs) is called directly with some
+    exceptions for correctness. Exceptions include: 1) None output 2) Single
+    value output 3) Operation-only outputs
   Raises:
     ValueError: If any element in computation outputs is neither an operations
       or a value that can be converted to tensor.
+    ValueError: If computation outputs is non-flat and contains any Operations.
     TypeError: If `inputs` is not a list or tuple.
   """
   if inputs is None:
@@ -300,66 +315,166 @@ def _compile_internal(computation, inputs=None):
     # Restore variable scope after computation.
     vscope.set_use_resource(saved_use_resource)
 
-    # If the computation returns `None`, make it an empty tuple.
-    if outputs is None:
-      outputs = tuple()
-    # If the computation only returned one value, make it a tuple.
-    if not isinstance(outputs, collections.Sequence):
-      outputs = (outputs,)
-
-    # Append `no_op` here so that return value of this function always contains
-    # at least one op that can trigger XlaLaunch node.
-    outputs += (control_flow_ops.no_op(),)
-    try:
-      outputs = [
-          o if isinstance(o, ops.Operation) else ops.convert_to_tensor(o)
-          for o in outputs
-      ]
-    except Exception as e:
-      raise ValueError(
-          'XLA computation function return values must all either be Operations'
-          ' or convertible to Tensors. Got error: "%s"' % str(e))
-
-    # Separates the returned Operations and Tensors.
-    output_operations = [o for o in outputs if isinstance(o, ops.Operation)]
-    output_tensors = [o for o in outputs if not isinstance(o, ops.Operation)]
-
-    if outputs != output_tensors + output_operations:
-      raise ValueError(
-          'XLA computation function must return zero or more Tensor values '
-          'followed by zero or more Operations.')
-    output_arity = len(output_tensors)
-
-    new_output_tensors = []
-    for t in output_tensors:
-      with ops.device(t.device if t.device else ''):
-        new_output_tensors.append(array_ops.identity(t))
+    outputs_is_flat = is_flat(outputs)
+    if outputs_is_flat:
+      output_tensors, control_deps = _postprocess_flat_outputs(outputs)
+    else:
+      output_tensors, control_deps = _postprocess_non_flat_outputs(outputs)
 
-    output_tensors = new_output_tensors
     context.ExitResult(output_tensors)
   finally:
     context.report_unsupported_operations()
     context.Exit()
 
-  outputs = [
-      xla_ops.xla_cluster_output(output_tensors[i], name='output{}'.format(i))
-      for i in xrange(output_arity)
+  # When XLA computation returns only operations and no tensors, a NoOp
+  # dependent on the operations in outputs is returned. Otherwise final
+  # outputs would be empty and there is no way to trigger returned
+  # operations.
+  if not output_tensors:
+    return control_flow_ops.group(control_deps, name='output_0')
+
+  output_tensors = [
+      xla_ops.xla_cluster_output(o, name='output{}'.format(i))
+      for i, o in enumerate(output_tensors)
   ]
 
-  with ops.control_dependencies(output_operations):
-    if output_arity == 0:
-      # When XLA computation returns only operations and no tensors, a NoOp
-      # dependent on the operations in outputs is returned. Otherwise final
-      # outputs would be empty and there is no way to trigger returned
-      # operations.
-      return control_flow_ops.no_op(name='output_0')
-    else:
-      # Wraps the outputs in identity operators that carries control
-      # dependencies.
-      return [
-          array_ops.identity(outputs[i], name='output_%d' % i)
-          for i in xrange(output_arity)
-      ]
+  with ops.control_dependencies(control_deps):
+    # Wraps the outputs in identity operators that carries control
+    # dependencies.
+    output_tensors = [
+        array_ops.identity(o, name='output_%d' % i)
+        for i, o in enumerate(output_tensors)
+    ]
+
+  # If `computation` returned non-flat output structure, pack output tensors
+  # back into same structure.
+  if not outputs_is_flat:
+    output_tensors = nest.pack_sequence_as(
+        structure=outputs, flat_sequence=output_tensors)
+
+  return output_tensors
+
+
+def is_flat(outputs):
+  """Checks if outputs is a flat structure.
+
+    Following structures and values are considered flat:
+    1) None
+    2) A single object
+    3) A list or tuple of Tensors/Operations
+
+    The only structures that this function understands are sequences and
+    dictionaries.  E.g. this means that if outputs contains a single
+    user-defined Object, it is considered to be flat. Errors are raised later on
+    if that Object cannot be converted to a Tensor.
+
+  Args:
+    outputs: Output from `computation` inside `xla.compile`.
+
+  Returns:
+    A boolean indicates whether outputs is flat.
+  """
+  # If outputs is a list or tuple, check if it has any nested structure. If
+  # there is, then outputs is non-flat.
+  if isinstance(outputs, collections.Sequence):
+    for o in outputs:
+      if isinstance(o, collections.Sequence) or isinstance(o, dict):
+        return False
+
+  # If outputs is a dict, it is non-flat.
+  if isinstance(outputs, dict):
+    return False
+
+  # Getting here means either outputs itself is a single non-structured value
+  # or it is a flat list of single non-structured values.
+  return True
+
+
+def _postprocess_flat_outputs(outputs):
+  """Validates flat outputs and adds back device assignments.
+
+  Args:
+    outputs: Output from `computation` inside `xla.compile`.
+
+  Returns:
+    Tensors and Operations extracted from outputs.
+  """
+  # Following code segment is to preserve legacy behavior. Previously we only
+  # supported flat outputs and thus for consistency it was nice to convert even
+  # single element into a tuple. But now that we support arbitrary output
+  # structure, this is no longer necessary.
+  # TODO(b/121383831): Migrate all legacy use cases and delete this special
+  # case.
+  # If the computation returns `None`, make it an empty tuple.
+  if outputs is None:
+    outputs = tuple()
+  # If the computation only returned one value, make it a tuple.
+  if not isinstance(outputs, collections.Sequence):
+    outputs = (outputs,)
+
+  # Append `no_op` here so that return value of this function always contains
+  # at least one op that can trigger XlaLaunch node.
+  outputs += (control_flow_ops.no_op(),)
+  try:
+    outputs = [
+        o if isinstance(o, ops.Operation) else ops.convert_to_tensor(o)
+        for o in outputs
+    ]
+  except Exception as e:
+    raise ValueError(
+        'XLA computation function return values must all either be Operations'
+        ' or convertible to Tensors. Got error: "%s"' % str(e))
+
+  # Separates the returned Operations and Tensors.
+  output_operations = [o for o in outputs if isinstance(o, ops.Operation)]
+  output_tensors = [o for o in outputs if not isinstance(o, ops.Operation)]
+
+  if outputs != output_tensors + output_operations:
+    raise ValueError(
+        'XLA computation function must return zero or more Tensor values '
+        'followed by zero or more Operations.')
+
+  new_output_tensors = []
+  for t in output_tensors:
+    with ops.device(t.device if t.device else ''):
+      new_output_tensors.append(array_ops.identity(t))
+
+  return new_output_tensors, output_operations
+
+
+def _postprocess_non_flat_outputs(outputs):
+  """Validates non-flat outputs and adds back device assignments.
+
+  Args:
+    outputs: Output from `computation` inside `xla.compile`.
+
+  Returns:
+    Tensors extracted from outputs and an empty list because Operations are not
+    allowed in non-flat outputs..
+  """
+  # Convert all non-Operation outputs to Tensors.
+  new_output_tensors = []
+  for o in nest.flatten(outputs):
+    if isinstance(o, ops.Operation):
+      raise ValueError(
+          'xla.compile does not support Operation as return value in non-flat '
+          'output structure. You can set returned Operations as control '
+          'dependencies of returned Tensors so Operations are triggered when '
+          'Tensors are evaluated. Operation found: "%s"' % o.name)
+
+    try:
+      o = ops.convert_to_tensor(o)
+    except Exception as e:
+      raise ValueError(
+          'XLA computation function return values must all either be '
+          'Operations or convertible to Tensors. Got error: "%s"' % str(e))
+
+    # Makes sure even pass-through inputs/outputs are touched in compile
+    # context by creating an Identity node inside compile context.
+    with ops.device(o.device if o.device else ''):
+      new_output_tensors.append(array_ops.identity(o))
+
+  return new_output_tensors, []
 
 
 @contextlib.contextmanager
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu.py b/tensorflow/contrib/tpu/python/tpu/tpu.py
index a267dd435c..b04baebfe6 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu.py
@@ -19,6 +19,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import collections
 from six.moves import xrange  # pylint: disable=redefined-builtin
 
 from tensorflow.contrib.compiler import xla
@@ -503,7 +504,17 @@ def replicate(computation,
       replicas is equal to the number of cores in the TPU system.
     name: (Deprecated) Does nothing.
   Returns:
-    A list of lists of output tensors, indexed by `[replica_num][output_num]`.
+    A list of outputs, indexed by `[replica_num]` each output can be a nested
+    structure same as what computation() returns with a few exceptions.
+
+    Exceptions include:
+      1) None output: a NoOp would be returned which control-depends on
+         computation.
+      2) Single value output: A tuple containing the value would be returned.
+      3) Operation-only outputs: a NoOp would be returned which
+         control-depends on computation.
+      TODO(b/121383831): Investigate into removing these special cases.
+
   Raises:
     ValueError: If all replicas do not have equal numbers of input tensors.
     ValueError: If the number of inputs per replica does not match
@@ -712,51 +723,12 @@ def split_compile_and_replicate(computation,
       vscope.set_use_resource(saved_use_resource)
       vscope.set_custom_getter(saved_custom_getter)
 
-    # If the computation returns `None`, make it an empty tuple.
-    if outputs is None:
-      outputs = tuple()
-    # If the computation only returned one value, makes it a tuple.
-    if not isinstance(outputs, (list, tuple)):
-      outputs = (outputs,)
-
-    # Append `no_op` here so that fetching any return value of this function
-    # will trigger TPUExecute node.
-    outputs += (control_flow_ops.no_op(),)
-    try:
-      with ops.device(core(0)):
-        outputs = [
-            o if isinstance(o, ops.Operation) else ops.convert_to_tensor(o)
-            for o in outputs
-        ]
-    except Exception as e:
-      raise ValueError(
-          "TPU function return values must all either be Operations or "
-          "convertible to Tensors. Got '%s'" % str(e))
-
-    # Separates the returned Operations and Tensors.
-    output_operations = [o for o in outputs if isinstance(o, ops.Operation)]
-    output_tensors = [o for o in outputs if not isinstance(o, ops.Operation)]
-
-    if outputs != output_tensors + output_operations:
-      raise ValueError(
-          "TPU functions must return zero-or more Tensor values followed by "
-          "zero or more Operations.")
-    output_arity = len(output_tensors)
+    outputs_is_flat = xla.is_flat(outputs)
+    if outputs_is_flat:
+      output_tensors, control_deps = _postprocess_flat_outputs(outputs)
+    else:
+      output_tensors, control_deps = _postprocess_non_flat_outputs(outputs)
 
-    # Wraps outputs in Identity ops. Otherwise a replicated input copied
-    # straight to an output would bypass the replicate(). This would be bad
-    # because the TPUReplicatedInput/TPUReplicatedOutput operator would not
-    # be rewritten away, leading to a runtime error.
-    # TODO(phawkins): extend the rewrite to elide these nodes instead.
-    new_output_tensors = []
-    for t in output_tensors:
-      with ops.device(t.device if t.device else core(0)):
-        o = array_ops.identity(t)
-        # pylint: disable=protected-access
-        o.op._set_attr("_tpu_output_identity", attr_value_pb2.AttrValue(b=True))
-        # pylint: enable=protected-access
-        new_output_tensors.append(o)
-    output_tensors = new_output_tensors
     context.ExitResult(output_tensors)
   finally:
     context.report_unsupported_operations()
@@ -768,11 +740,6 @@ def split_compile_and_replicate(computation,
     attr_value.list.s.extend([compat.as_bytes(x) for x in host_compute_core])
     metadata._set_attr("host_compute_core", attr_value)  # pylint: disable=protected-access
 
-  # Fan-out: Builds a TPUReplicatedOutput node for each output.
-  outputs = [tpu_ops.tpu_replicated_output(output_tensors[i], num_replicas,
-                                           name="output{}".format(i))
-             for i in xrange(output_arity)]
-
   with ops.control_dependencies([metadata]):
     if use_tpu:
       compile_status = tpu_ops.tpu_compilation_result()
@@ -782,28 +749,146 @@ def split_compile_and_replicate(computation,
     else:
       compile_status = control_flow_ops.no_op(name="compilation_status")
 
-  with ops.control_dependencies(output_operations):
-    if output_arity == 0:
-      # Returns a list of NoOps dependent on the replication Op, indexed by
-      # [replica_num].
-      return [
-          compile_status, [
-              control_flow_ops.no_op(name="shard_%d" % i)
-              for i in range(num_replicas)
-          ]
-      ]
-    else:
-      # Wraps the outputs in identity operators so the names of any possible
-      # `fetch` nodes are preserved by the replication rewrite.
-      return [
-          compile_status, [[
-              array_ops.identity(
-                  outputs[out][replica],
-                  name="output_%d_shard_%d" % (out, replica))
-              for out in xrange(output_arity)
-          ]
-                           for replica in xrange(num_replicas)]
+  if not output_tensors:
+    # Returns a list of NoOps dependent on the replication Op, indexed by
+    # [replica_num].
+    return [
+        compile_status,
+        [
+            control_flow_ops.group(control_deps, name="shard_%d" % i)
+            for i in range(num_replicas)
+        ]
+    ]
+
+  # Fan-out: Builds a TPUReplicatedOutput node for each output.
+  replicated_outputs = [[] for i in xrange(num_replicas)]
+  for i, t in enumerate(output_tensors):
+    # Fan-out: Builds a TPUReplicatedOutput node for each output.
+    ys = tpu_ops.tpu_replicated_output(
+        t, num_replicas, name="output{}".format(i))
+
+    # Wraps the outputs in identity operators so the names of any possible
+    # `fetch` nodes are preserved by the replication rewrite.
+    with ops.control_dependencies(control_deps):
+      for replica in xrange(num_replicas):
+        replicated_outputs[replica].append(
+            array_ops.identity(
+                ys[replica], name="output_%d_shard_%d" % (i, replica)))
+
+  if not outputs_is_flat:
+    replicated_outputs = [
+        nest.pack_sequence_as(outputs, replica_outs)
+        for replica_outs in replicated_outputs
+    ]
+
+  return [compile_status, replicated_outputs]
+
+
+def _postprocess_flat_outputs(outputs):
+  """Validates non-flat outputs, add backs device assignments and other attrs.
+
+  Args:
+    outputs: Output from `computation` inside `tpu.rewrite`.
+
+  Returns:
+    Tensors and Operations extracted from outputs.
+  """
+  # Following code segment is to preserve legacy behavior. Previously we only
+  # supported flat outputs and thus for consistency it was nice to convert even
+  # single element into a tuple. But now that we support arbitrary output
+  # structure, this is no longer necessary.
+  # TODO(b/121383831): Migrate all legacy use cases and delete this special
+  # case.
+  # If the computation returns `None`, make it an empty tuple.
+  if outputs is None:
+    outputs = tuple()
+  # If the computation only returned one value, makes it a tuple.
+  if not isinstance(outputs, collections.Sequence):
+    outputs = (outputs,)
+
+  # Append `no_op` here so that fetching any return value of this function
+  # will trigger TPUExecute node.
+  outputs += (control_flow_ops.no_op(),)
+  try:
+    with ops.device(core(0)):
+      outputs = [
+          o if isinstance(o, ops.Operation) else ops.convert_to_tensor(o)
+          for o in outputs
       ]
+  except Exception as e:
+    raise ValueError(
+        "TPU function return values must all either be Operations or "
+        "convertible to Tensors. Got '%s'" % str(e))
+
+  # Separates the returned Operations and Tensors.
+  output_operations = [o for o in outputs if isinstance(o, ops.Operation)]
+  output_tensors = [o for o in outputs if not isinstance(o, ops.Operation)]
+
+  if outputs != output_tensors + output_operations:
+    raise ValueError(
+        "TPU functions must return zero-or more Tensor values followed by "
+        "zero or more Operations.")
+
+  # Wraps outputs in Identity ops. Otherwise a replicated input copied
+  # straight to an output would bypass the replicate(). This would be bad
+  # because the TPUReplicatedInput/TPUReplicatedOutput operator would not
+  # be rewritten away, leading to a runtime error.
+  # TODO(phawkins): extend the rewrite to elide these nodes instead.
+  new_output_tensors = []
+  for t in output_tensors:
+    with ops.device(t.device if t.device else core(0)):
+      o = array_ops.identity(t)
+      # pylint: disable=protected-access
+      o.op._set_attr("_tpu_output_identity", attr_value_pb2.AttrValue(b=True))
+      # pylint: enable=protected-access
+      new_output_tensors.append(o)
+  return new_output_tensors, output_operations
+
+
+def _postprocess_non_flat_outputs(outputs):
+  """Validates non-flat outputs, add backs device assignments and other attrs.
+
+  Args:
+    outputs: Output from `computation` inside `tpu.rewrite`.
+
+  Returns:
+    Tensors extracted from outputs and an empty list because Operations are not
+    allowed in non-flat outputs..
+  """
+
+  # Flatten output items.
+  flat_outputs = nest.flatten(outputs)
+
+  # Convert all non-Operation outputs to Tensors.
+  for i, o in enumerate(flat_outputs):
+    if isinstance(o, ops.Operation):
+      raise ValueError(
+          "tpu.rewrite does not support Operation as return value in non-flat "
+          "output structure. You can set returned Operations as control "
+          "dependencies of returned Tensors so Operations are triggered when "
+          'Tensors are evaluated. Operation found: "%s"' % o.name)
+
+    try:
+      o = ops.convert_to_tensor(o)
+    except Exception as e:
+      raise ValueError(
+          "TPU function return values must all either be Operations or "
+          'convertible to Tensors. Got error: "%s"' % str(e))
+
+    # Wraps outputs in Identity ops. Otherwise a replicated input copied
+    # straight to an output would bypass the replicate(). This would be bad
+    # because the TPUReplicatedInput/TPUReplicatedOutput operator would not
+    # be rewritten away, leading to a runtime error.
+    # TODO(phawkins): extend the rewrite to elide these nodes instead.
+    with ops.device(core(0)):
+      o = array_ops.identity(o)
+      # pylint: disable=protected-access
+      o.op._set_attr("_tpu_output_identity", attr_value_pb2.AttrValue(b=True))
+      # pylint: enable=protected-access
+      flat_outputs[i] = array_ops.identity(o)
+
+  # All flat_outputs are Tensors, and no Operations.
+  return flat_outputs, []
 
 
 def split_compile_and_shard(computation,
@@ -1126,11 +1211,15 @@ def rewrite(computation,
       case the core attached to task 0, TPU device 0 is used.
     name: (Deprecated) Does nothing.
   Returns:
-    A list of output tensors.
+    Same data structure as if computation(*inputs) is called directly with some
+    exceptions for correctness. Exceptions include:
+      1) None output: a NoOp would be returned which control-depends on
+         computation.
+      2) Single value output: A tuple containing the value would be returned.
+      3) Operation-only outputs: a NoOp would be returned which
+         control-depends on computation.
+      TODO(b/121383831): Investigate into removing these special cases.
   """
-  if inputs is not None and not isinstance(inputs, (list, tuple)):
-    raise TypeError("tpu.rewrite() inputs must be a list or tuple")
-
   # TODO(b/36647078) remove disable when pylint bug is fixed.
   # pylint: disable=indexing-exception
   return replicate(
-- 
GitLab


From bea8a4a3a1fb0282b0c59a687d25c6ae160316fc Mon Sep 17 00:00:00 2001
From: Guangda Lai <31743510+aaroey@users.noreply.github.com>
Date: Tue, 15 Jan 2019 15:46:33 -0800
Subject: [PATCH 0755/2345] Fix formatting issues and broken build

---
 tensorflow/contrib/tensorrt/BUILD             |  2 +-
 .../tensorrt/python/trt_convert_test.py       |  8 ++--
 .../tensorrt/resources/trt_lru_cache.h        | 38 ++++++++++-----
 tensorflow/contrib/tensorrt/test/base_test.py | 14 +++---
 .../tensorrt/test/batch_matmul_test.py        |  2 +-
 .../tensorrt/test/biasadd_matmul_test.py      |  2 +-
 .../binary_tensor_weight_broadcast_test.py    |  2 +-
 .../tensorrt/test/concatenation_test.py       |  2 +-
 .../tensorrt/test/const_broadcast_test.py     |  2 +-
 .../contrib/tensorrt/test/conv2d_test.py      | 12 ++---
 .../test/dynamic_input_shapes_test.py         | 31 ++++++------
 .../tensorrt/test/identity_output_test.py     |  4 +-
 .../contrib/tensorrt/test/int32_test.py       |  4 +-
 .../contrib/tensorrt/test/lru_cache_test.py   | 48 ++++++++++++-------
 .../tensorrt/test/memory_alignment_test.py    |  2 +-
 .../multi_connection_neighbor_engine_test.py  |  2 +-
 .../tensorrt/test/neighboring_engine_test.py  |  2 +-
 .../tensorrt/test/quantization_test.py        |  2 +-
 .../contrib/tensorrt/test/rank_two_test.py    |  2 +-
 .../tensorrt/test/reshape_transpose_test.py   |  4 +-
 .../test/tf_trt_integration_test_base.py      | 46 +++++++++---------
 tensorflow/contrib/tensorrt/test/topk_test.py |  6 +--
 .../contrib/tensorrt/test/unary_test.py       |  2 +-
 .../tensorrt/test/vgg_block_nchw_test.py      |  2 +-
 .../contrib/tensorrt/test/vgg_block_test.py   |  2 +-
 25 files changed, 138 insertions(+), 105 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 58c188b314..7ae3caaaa2 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -567,8 +567,8 @@ cc_library(
     hdrs = ["convert/utils.h"],
     copts = tf_copts(),
     deps = [
-        "//tensorflow/core:lib",
         "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
     ],
 )
 
diff --git a/tensorflow/contrib/tensorrt/python/trt_convert_test.py b/tensorflow/contrib/tensorrt/python/trt_convert_test.py
index 60c31580db..14ab4772c4 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert_test.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert_test.py
@@ -239,8 +239,8 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
         # Run with batch size 2, a new engine is created and cached.
         self._TestRun(sess, 2, True)
         # Run with batch size 3, since the number of cached engines has reached
-        # the max, it should fall back to TF function.
-        self._TestRun(sess, 3, False)
+        # the max, it should evict an old engine and create a new one.
+        self._TestRun(sess, 3, True)
 
     # Test the output SavedModel
     with ops.Graph().as_default():
@@ -251,8 +251,8 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
         # Run with batch size 2, a new engine is created and cached.
         self._TestRun(sess, 2, True)
         # Run with batch size 3, since the number of cached engines has reached
-        # the max, it should fall back to TF function.
-        self._TestRun(sess, 3, False)
+        # the max, it should evict an old engine and create a new one.
+        self._TestRun(sess, 3, True)
 
   def testCreateInferenceGraph_StaticOp(self):
     if not trt_convert.is_tensorrt_enabled():
diff --git a/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h b/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
index ba455b9b35..afd1b83e74 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
@@ -1,5 +1,20 @@
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_TRT_LRU_CACHE_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_TRT_LRU_CACHE_H_
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_LRU_CACHE_H_
+#define TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_LRU_CACHE_H_
 
 #include <list>
 #include <unordered_map>
@@ -9,9 +24,9 @@
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/lib/core/errors.h"
 
-#if GOOGLE_CUDA
-#if GOOGLE_TENSORRT
+#if GOOGLE_CUDA && GOOGLE_TENSORRT
 #include "tensorrt/include/NvInfer.h"
+#endif  // GOOGLE_CUDA && GOOGLE_TENSORRT
 
 namespace tensorflow {
 namespace tensorrt {
@@ -40,9 +55,7 @@ class LRUCache {
 
   size_t count(const key_type& key) const { return objects_.count(key); }
 
-  value_type& at(const key_type& key) {
-    return Touch(key);
-  }
+  value_type& at(const key_type& key) { return Touch(key); }
 
   const_iterator begin() const { return objects_.begin(); }
   const_iterator end() const { return objects_.end(); }
@@ -111,6 +124,9 @@ struct VectorTensorShapeHasher {
   }
 };
 
+#if GOOGLE_CUDA
+#if GOOGLE_TENSORRT
+
 struct EngineContext {
   EngineContext() {}  // Creates an empty context.
   EngineContext(
@@ -167,10 +183,10 @@ class TRTEngineCacheResource : public tensorflow::ResourceBase {
       cache_;
 };
 
-}  // namespace tensorrt
-}  // namespace tensorflow
-
 #endif  // GOOGLE_TENSORRT
 #endif  // GOOGLE_CUDA
 
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_TRT_LRU_CACHE_H_
+}  // namespace tensorrt
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_LRU_CACHE_H_
diff --git a/tensorflow/contrib/tensorrt/test/base_test.py b/tensorflow/contrib/tensorrt/test/base_test.py
index 0f6293b166..17e0b6f4d2 100644
--- a/tensorflow/contrib/tensorrt/test/base_test.py
+++ b/tensorflow/contrib/tensorrt/test/base_test.py
@@ -70,7 +70,7 @@ class SimpleSingleEngineTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[(100, 6, 6, 6)]])
+        expected_output_dims=[[[100, 6, 6, 6]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -127,7 +127,7 @@ class SimpleMultiEnginesTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[(100, 12, 12, 6)]])
+        expected_output_dims=[[[100, 12, 12, 6]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -185,7 +185,7 @@ class PartiallyConvertedTestA(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[tuple(input_dims)]])
+        expected_output_dims=[[input_dims]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -255,7 +255,7 @@ class ConstInputTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[tuple(input_dims)]])
+        expected_output_dims=[[input_dims]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -288,7 +288,7 @@ class ConstDataInputSingleEngineTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[tuple(input_dims)]])
+        expected_output_dims=[[input_dims]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -322,7 +322,7 @@ class ConstDataInputMultipleEnginesTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[tuple(input_dims)]])
+        expected_output_dims=[[input_dims]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -371,7 +371,7 @@ class ControlDependencyTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[tuple(input_dims)]])
+        expected_output_dims=[[input_dims]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/batch_matmul_test.py b/tensorflow/contrib/tensorrt/test/batch_matmul_test.py
index 1b015dd369..46e3407d96 100644
--- a/tensorflow/contrib/tensorrt/test/batch_matmul_test.py
+++ b/tensorflow/contrib/tensorrt/test/batch_matmul_test.py
@@ -73,7 +73,7 @@ class BatchMatMulTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name, w1_name, w2_name],
         input_dims=[[input_dims, w1_dims, w2_dims]],
         output_names=[output_name],
-        expected_output_dims=[[(12, 5, 8, 7)]])
+        expected_output_dims=[[[12, 5, 8, 7]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py b/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
index 8a5f8ef5d5..ca23629efa 100644
--- a/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
+++ b/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
@@ -113,7 +113,7 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[(4, 6680)]])
+        expected_output_dims=[[[4, 6680]]])
 
   def GetConversionParams(self, run_params):
     """Return a ConversionParams for test."""
diff --git a/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py b/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
index 0083695a50..846fd009db 100644
--- a/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
+++ b/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
@@ -65,7 +65,7 @@ class BinaryTensorWeightBroadcastTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[(5, 23040)]])
+        expected_output_dims=[[[5, 23040]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/concatenation_test.py b/tensorflow/contrib/tensorrt/test/concatenation_test.py
index e6afe4e3f6..5d8742ae35 100644
--- a/tensorflow/contrib/tensorrt/test/concatenation_test.py
+++ b/tensorflow/contrib/tensorrt/test/concatenation_test.py
@@ -75,7 +75,7 @@ class ConcatenationTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[(2, 126)]])
+        expected_output_dims=[[[2, 126]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/const_broadcast_test.py b/tensorflow/contrib/tensorrt/test/const_broadcast_test.py
index 56a297afb5..9137d0072f 100644
--- a/tensorflow/contrib/tensorrt/test/const_broadcast_test.py
+++ b/tensorflow/contrib/tensorrt/test/const_broadcast_test.py
@@ -60,7 +60,7 @@ class ConstBroadcastTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[(5, 12, 12, 1)]])
+        expected_output_dims=[[[5, 12, 12, 1]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/conv2d_test.py b/tensorflow/contrib/tensorrt/test/conv2d_test.py
index d4cee3cce0..e7993b4620 100644
--- a/tensorflow/contrib/tensorrt/test/conv2d_test.py
+++ b/tensorflow/contrib/tensorrt/test/conv2d_test.py
@@ -103,9 +103,9 @@ class Conv2DNCHWTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=["input"],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=["output"],
-        expected_output_dims=[(13, 5, 7, 11)])
+        expected_output_dims=[[[13, 5, 7, 11]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -128,9 +128,9 @@ class Conv2DNHWCTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=["input"],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=["output"],
-        expected_output_dims=[(13, 7, 11, 5)])
+        expected_output_dims=[[[13, 7, 11, 5]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -178,9 +178,9 @@ class Conv2DStridedNCHWTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[(n, num_filters, h, w)])
+        expected_output_dims=[[[n, num_filters, h, w]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py b/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py
index 1c6d411d11..90addecb14 100644
--- a/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py
+++ b/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py
@@ -18,17 +18,18 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+import numpy as np
+
 # pylint: disable=unused-import
 from tensorflow.contrib.tensorrt.python.ops import trt_engine_op
 # pylint: enable=unused-import
+from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import constant_op
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import nn
 from tensorflow.python.platform import test
-import numpy as np
 
 
 class DynamicInputShapesTest(trt_test.TfTrtIntegrationTestBase):
@@ -47,12 +48,10 @@ class DynamicInputShapesTest(trt_test.TfTrtIntegrationTestBase):
 
     g = ops.Graph()
     with g.as_default():
-      x = array_ops.placeholder(shape=(None, None, None, 1),
-                                dtype=dtypes.float32,
-                                name='input')
-      conv_filter1 = constant_op.constant(np.ones([3, 3, 1, 8]),
-                                          name="weights1",
-                                          dtype=dtypes.float32)
+      x = array_ops.placeholder(
+          shape=(None, None, None, 1), dtype=dtypes.float32, name="input")
+      conv_filter1 = constant_op.constant(
+          np.ones([3, 3, 1, 8]), name="weights1", dtype=dtypes.float32)
       bias1 = constant_op.constant(np.random.randn(8), dtype=dtypes.float32)
       x = nn.conv2d(
           input=x,
@@ -62,9 +61,8 @@ class DynamicInputShapesTest(trt_test.TfTrtIntegrationTestBase):
           name="conv")
       x = nn.bias_add(x, bias1)
       x = nn.relu(x)
-      conv_filter2 = constant_op.constant(np.ones([3, 3, 8, 1]),
-                                          name="weights2",
-                                          dtype=dtypes.float32)
+      conv_filter2 = constant_op.constant(
+          np.ones([3, 3, 8, 1]), name="weights2", dtype=dtypes.float32)
       bias2 = constant_op.constant(np.random.randn(1), dtype=dtypes.float32)
       x = nn.conv2d(
           input=x,
@@ -73,13 +71,13 @@ class DynamicInputShapesTest(trt_test.TfTrtIntegrationTestBase):
           padding="SAME",
           name="conv")
       x = nn.bias_add(x, bias2)
-      x = array_ops.identity(x, name='output')
+      x = array_ops.identity(x, name="output")
 
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
-        input_names=['input'],
+        input_names=["input"],
         input_dims=input_dims,
-        output_names=['output'],
+        output_names=["output"],
         expected_output_dims=expected_output_dims)
 
   def GetConversionParams(self, run_params):
@@ -107,5 +105,6 @@ class DynamicInputShapesTest(trt_test.TfTrtIntegrationTestBase):
     """The relative tolerance to compare floating point results."""
     return 1.e-03 if run_params.precision_mode == "FP32" else 1.e-01
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/tensorrt/test/identity_output_test.py b/tensorflow/contrib/tensorrt/test/identity_output_test.py
index 196ccdbe9d..b568eeda94 100644
--- a/tensorflow/contrib/tensorrt/test/identity_output_test.py
+++ b/tensorflow/contrib/tensorrt/test/identity_output_test.py
@@ -61,9 +61,9 @@ class IdentityTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=['output1', 'output2', 'output3'],
-        expected_output_dims=[(100, 4), (100, 4), (100, 4)])
+        expected_output_dims=[[[100, 4]] * 3])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/int32_test.py b/tensorflow/contrib/tensorrt/test/int32_test.py
index 181229c8a7..8cf5387038 100644
--- a/tensorflow/contrib/tensorrt/test/int32_test.py
+++ b/tensorflow/contrib/tensorrt/test/int32_test.py
@@ -52,9 +52,9 @@ class ExcludeUnsupportedInt32Test(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[(100, 10)])
+        expected_output_dims=[[[100, 10]]])
 
   def GetConversionParams(self, run_params):
     """Return a ConversionParams for test."""
diff --git a/tensorflow/contrib/tensorrt/test/lru_cache_test.py b/tensorflow/contrib/tensorrt/test/lru_cache_test.py
index 74b51932ac..7702413e6c 100644
--- a/tensorflow/contrib/tensorrt/test/lru_cache_test.py
+++ b/tensorflow/contrib/tensorrt/test/lru_cache_test.py
@@ -1,3 +1,19 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test LRUCache by running different input batch sizes on same network."""
+
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -17,29 +33,28 @@ from tensorflow.python.platform import test
 class LRUCacheTest(trt_test.TfTrtIntegrationTestBase):
 
   def GetParams(self):
-    """Test for running network multiple times with different batch size to
-    check that lru_cache works correctly"""
     dtype = dtypes.float32
     input_name = "input"
     input_dims = [[[1, 10, 10, 2]], [[2, 10, 10, 2]], [[4, 10, 10, 2]],
                   [[2, 10, 10, 2]]]
-    expected_output_dims = [[(1, 10, 10, 1)], [(2, 10, 10, 1)],
-                            [(4, 10, 10, 1)], [(2, 10, 10, 1)]]
+    expected_output_dims = [[[1, 10, 10, 1]], [[2, 10, 10, 1]], [[4, 10, 10,
+                                                                  1]],
+                            [[2, 10, 10, 1]]]
     output_name = "output"
     g = ops.Graph()
     with g.as_default():
-      x = array_ops.placeholder(dtype=dtype,
-                                shape=[None, 10, 10, 2],
-                                name=input_name)
-      conv_filter = constant_op.constant(np.random.randn(3, 3, 2, 1),
-                                         dtype=dtypes.float32)
-      x = nn.conv2d(input=x,
-                    filter=conv_filter,
-                    strides=[1, 1, 1, 1],
-                    padding="SAME",
-                    name="conv")
-      bias = constant_op.constant(np.random.randn(1, 10, 10, 1),
-                                  dtype=dtypes.float32)
+      x = array_ops.placeholder(
+          dtype=dtype, shape=[None, 10, 10, 2], name=input_name)
+      conv_filter = constant_op.constant(
+          np.random.randn(3, 3, 2, 1), dtype=dtypes.float32)
+      x = nn.conv2d(
+          input=x,
+          filter=conv_filter,
+          strides=[1, 1, 1, 1],
+          padding="SAME",
+          name="conv")
+      bias = constant_op.constant(
+          np.random.randn(1, 10, 10, 1), dtype=dtypes.float32)
       x = math_ops.add(x, bias)
       x = nn.relu(x)
       x = array_ops.identity(x, name="output")
@@ -58,5 +73,6 @@ class LRUCacheTest(trt_test.TfTrtIntegrationTestBase):
     return (run_params.dynamic_engine and
             not trt_test.IsQuantizationMode(run_params.precision_mode))
 
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/tensorrt/test/memory_alignment_test.py b/tensorflow/contrib/tensorrt/test/memory_alignment_test.py
index 6142e6d8e3..cc64329bbd 100644
--- a/tensorflow/contrib/tensorrt/test/memory_alignment_test.py
+++ b/tensorflow/contrib/tensorrt/test/memory_alignment_test.py
@@ -64,7 +64,7 @@ class MemoryAlignmentTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[(2, 15, 15, 10)]])
+        expected_output_dims=[[[2, 15, 15, 10]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py b/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
index fcf89d3a74..a14bb0396e 100644
--- a/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
+++ b/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
@@ -77,7 +77,7 @@ class MultiConnectionNeighborEngineTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[(2, 4, 5, 4)]])
+        expected_output_dims=[[[2, 4, 5, 4]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py b/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py
index 840344aeba..06a86bbb8d 100644
--- a/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py
+++ b/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py
@@ -61,7 +61,7 @@ class NeighboringEngineTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[(2, 4, 5, 4)]])
+        expected_output_dims=[[[2, 4, 5, 4]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/quantization_test.py b/tensorflow/contrib/tensorrt/test/quantization_test.py
index fb09980552..ce1b25ebf3 100644
--- a/tensorflow/contrib/tensorrt/test/quantization_test.py
+++ b/tensorflow/contrib/tensorrt/test/quantization_test.py
@@ -62,7 +62,7 @@ def _GetParams(add_quantization_nodes, dtype=dtypes.float32):
       input_names=[input_name],
       input_dims=[[input_dims]],
       output_names=[output_name],
-      expected_output_dims=[[(8, 1)]])
+      expected_output_dims=[[[8, 1]]])
 
 
 class QuantizationMissingAllRangesTest(trt_test.TfTrtIntegrationTestBase):
diff --git a/tensorflow/contrib/tensorrt/test/rank_two_test.py b/tensorflow/contrib/tensorrt/test/rank_two_test.py
index 85ce89e151..97159bb008 100644
--- a/tensorflow/contrib/tensorrt/test/rank_two_test.py
+++ b/tensorflow/contrib/tensorrt/test/rank_two_test.py
@@ -65,7 +65,7 @@ class RankTwoTest(trt_test.TfTrtIntegrationTestBase):
         input_names=input_names,
         input_dims=[input_dims],
         output_names=[output_name],
-        expected_output_dims=[[tuple(input_dims[1])]])
+        expected_output_dims=[[input_dims[1]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py b/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py
index 530c4992db..7fb2cbde07 100644
--- a/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py
+++ b/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py
@@ -74,7 +74,7 @@ class ReshapeTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[tuple(input_dims)]])
+        expected_output_dims=[[input_dims]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -131,7 +131,7 @@ class TransposeTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[(24, 100, 2, 24)]])
+        expected_output_dims=[[[24, 100, 2, 24]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
index db6be3828c..f13c615cb1 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
@@ -39,17 +39,19 @@ from tensorflow.python.framework import test_util
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import tf_logging as logging
 
-TfTrtIntegrationTestParams = namedtuple("TfTrtIntegrationTestParams", [
-    "gdef",
-    # A list of names of the input placeholder nodes.
-    "input_names",
-    # A list of list of output shapes of the input placeholder nodes.
-    "input_dims",
-    # A list of names of the output identity nodes.
-    "output_names",
-    # A list of list of expected output shapes of the output identity nodes.
-    "expected_output_dims"
-])
+TfTrtIntegrationTestParams = namedtuple(
+    "TfTrtIntegrationTestParams",
+    [
+        "gdef",
+        # A list of names of the input placeholder nodes.
+        "input_names",
+        # A list of list of output shapes of the input placeholder nodes.
+        "input_dims",
+        # A list of names of the output identity nodes.
+        "output_names",
+        # A list of list of expected output shapes of the output identity nodes.
+        "expected_output_dims"
+    ])
 
 RunParams = namedtuple("RunParams", [
     "use_optimizer", "precision_mode", "dynamic_engine", "test_name",
@@ -167,10 +169,9 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
 
   def GetConversionParams(self, run_params):
     """Return a ConversionParams for test."""
-    batch_list
+    batch_list = []
     for dims_list in self._GetParamsCached().input_dims:
-      if not len(dims_list):
-        continue
+      assert dims_list
       # Each list of shapes should have same batch size.
       input_batches = [dims[0] for dims in dims_list]
       assert max(input_batches) == min(input_batches)
@@ -323,19 +324,20 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
       with self.test_session(
           graph=g, config=config, use_gpu=True, force_gpu=True) as sess:
         # Run for each input(s) shape
-        for ind in range(len(inputs_data)):
+        for shape_index in range(len(inputs_data)):
           val = None
           # Defaults to 2 runs to verify result across multiple runs is same.
           for _ in range(num_runs):
             self._PrepareRun(graph_state)
-            new_val = sess.run(
-                outputs,
-                {inputs[i]: inputs_data[ind][i] for i in range(len(inputs))})
-            output_len = len(params.expected_output_dims[ind])
+            new_val = sess.run(outputs, {
+                inputs[i]: inputs_data[shape_index][i]
+                for i in range(len(inputs))
+            })
+            output_len = len(params.expected_output_dims[shape_index])
             self.assertEqual(output_len, len(new_val))
             for i in range(output_len):
               self.assertEqual(
-                  list(params.expected_output_dims[ind][i]),
+                  list(params.expected_output_dims[shape_index][i]),
                   list(new_val[i].shape))
             if val is not None:
               self.assertAllClose(val, new_val, atol=1.e-06, rtol=1.e-06)
@@ -506,8 +508,8 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
       current_input_data = []
       for i in range(len(params.input_names)):
         dtype = input_dtypes[params.input_names[i]]
-        # Multiply the input by some constant to avoid all zeros input for integer
-        # types.
+        # Multiply the input by some constant to avoid all zeros input for
+        # integer types.
         scale = 10.0 if np.issubdtype(dtype, np.integer) else 1.0
         dims = inp[i]
         # TODO(laigd): add debug options. E.g. we can set the input data to be
diff --git a/tensorflow/contrib/tensorrt/test/topk_test.py b/tensorflow/contrib/tensorrt/test/topk_test.py
index 263ea0cfa1..633a51982b 100644
--- a/tensorflow/contrib/tensorrt/test/topk_test.py
+++ b/tensorflow/contrib/tensorrt/test/topk_test.py
@@ -38,16 +38,16 @@ class TopKTest(trt_test.TfTrtIntegrationTestBase):
     g = ops.Graph()
     with g.as_default():
       x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name)
-      k_tensor = constant_op.constant(k, dtype=tf.int32, name="Const")
+      k_tensor = constant_op.constant(k, dtype=dtypes.int32, name="Const")
       values, indices = nn_ops.top_k(x, k_tensor, name="TopK")
       values = array_ops.identity(values, name="output_values")
       indices = array_ops.identity(indices, name="output_indices")
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=["output_values", "output_indices"],
-        expected_output_dims=[(100, k), (100, k)])
+        expected_output_dims=[[[100, k], [100, k]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/unary_test.py b/tensorflow/contrib/tensorrt/test/unary_test.py
index f630d695cf..497ea2848a 100644
--- a/tensorflow/contrib/tensorrt/test/unary_test.py
+++ b/tensorflow/contrib/tensorrt/test/unary_test.py
@@ -102,7 +102,7 @@ class UnaryTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name, input2_name],
         input_dims=[[input_dims, input2_dims]],
         output_names=[output_name],
-        expected_output_dims=[[(12, 5, 8, 12)]])
+        expected_output_dims=[[[12, 5, 8, 12]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py b/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py
index 1a4746e8d0..b5fed73d2d 100644
--- a/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py
+++ b/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py
@@ -72,7 +72,7 @@ class VGGBlockNCHWTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[(5, 6, 2, 2)]])
+        expected_output_dims=[[[5, 6, 2, 2]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/vgg_block_test.py b/tensorflow/contrib/tensorrt/test/vgg_block_test.py
index 051ced055a..307128f1a8 100644
--- a/tensorflow/contrib/tensorrt/test/vgg_block_test.py
+++ b/tensorflow/contrib/tensorrt/test/vgg_block_test.py
@@ -63,7 +63,7 @@ class VGGBlockTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[(5, 2, 2, 6)]])
+        expected_output_dims=[[[5, 2, 2, 6]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
-- 
GitLab


From f9b9cf52eb36a5d5e8bbf96fc1f2b6f584ce7867 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Tue, 15 Jan 2019 16:08:22 -0800
Subject: [PATCH 0756/2345] Checkpointable->AutoCheckpointable
 CheckpointableBase->Checkpointable

In preparation for adding public symbols.

PiperOrigin-RevId: 229459684
---
 tensorflow/contrib/checkpoint/__init__.py     |   4 +-
 .../contrib/checkpoint/python/containers.py   |   2 +-
 .../checkpoint/python/containers_test.py      |   4 +-
 .../contrib/checkpoint/python/python_state.py |   4 +-
 .../checkpoint/python/split_dependency.py     |   2 +-
 .../python/split_dependency_test.py           |   6 +-
 .../cudnn_rnn/python/ops/cudnn_rnn_ops.py     |   2 +-
 .../distribute/python/checkpointing_test.py   |   2 +-
 .../contrib/eager/python/metrics_impl.py      |   2 +-
 tensorflow/contrib/eager/python/tfe.py        |   2 +-
 .../optimizer_v2/checkpointable_utils_test.py |  10 +-
 tensorflow/python/data/ops/iterator_ops.py    |   4 +-
 tensorflow/python/distribute/values.py        |   8 +-
 .../feature_column/feature_column_v2.py       |   2 +-
 tensorflow/python/keras/engine/base_layer.py  |   2 +-
 tensorflow/python/keras/engine/network.py     |   2 +-
 tensorflow/python/keras/engine/training.py    |   2 +-
 tensorflow/python/keras/layers/recurrent.py   |   2 +-
 .../python/keras/optimizer_v2/optimizer_v2.py |   2 +-
 tensorflow/python/keras/optimizers.py         |   2 +-
 tensorflow/python/layers/base.py              |   2 +-
 tensorflow/python/ops/lookup_ops.py           |   4 +-
 tensorflow/python/ops/rnn_cell_impl.py        |   8 +-
 tensorflow/python/ops/stateful_random_ops.py  |   2 +-
 tensorflow/python/ops/template.py             |   2 +-
 tensorflow/python/ops/variables.py            |   2 +-
 tensorflow/python/saved_model/load.py         |   3 +-
 tensorflow/python/saved_model/load_test.py    |  58 +++++-----
 .../python/saved_model/revived_types_test.py  |   4 +-
 tensorflow/python/saved_model/save.py         |   2 +-
 tensorflow/python/saved_model/save_test.py    |  18 ++--
 .../python/training/checkpointable/base.py    |  12 +--
 .../training/checkpointable/base_test.py      |  16 +--
 .../checkpointable/data_structures.py         |  18 ++--
 .../checkpointable/data_structures_test.py    |  38 +++----
 .../training/checkpointable/tracking.py       |   8 +-
 .../training/checkpointable/tracking_test.py  |  46 ++++----
 .../python/training/checkpointable/util.py    |   4 +-
 .../training/checkpointable/util_test.py      | 100 +++++++++---------
 .../util_with_v1_optimizers_test.py           |  14 +--
 .../training/checkpointable/util_xla_test.py  |   2 +-
 tensorflow/python/training/optimizer.py       |   2 +-
 tensorflow/python/training/saver_test.py      |   6 +-
 .../training/saving/saveable_object_util.py   |   4 +-
 .../api/golden/v1/tensorflow.-variable.pbtxt  |   2 +-
 .../golden/v1/tensorflow.data.-iterator.pbtxt |   2 +-
 .../golden/v1/tensorflow.keras.-model.pbtxt   |   2 +-
 .../v1/tensorflow.keras.-sequential.pbtxt     |   2 +-
 ....experimental.-peephole-l-s-t-m-cell.pbtxt |   2 +-
 .../tensorflow.keras.layers.-activation.pbtxt |   2 +-
 ...eras.layers.-activity-regularization.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-add.pbtxt     |   2 +-
 ...nsorflow.keras.layers.-alpha-dropout.pbtxt |   2 +-
 ...low.keras.layers.-average-pooling1-d.pbtxt |   2 +-
 ...low.keras.layers.-average-pooling2-d.pbtxt |   2 +-
 ...low.keras.layers.-average-pooling3-d.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-average.pbtxt |   2 +-
 ...tensorflow.keras.layers.-avg-pool1-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-avg-pool2-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-avg-pool3-d.pbtxt |   2 +-
 ...ow.keras.layers.-batch-normalization.pbtxt |   2 +-
 ...nsorflow.keras.layers.-bidirectional.pbtxt |   2 +-
 ...tensorflow.keras.layers.-concatenate.pbtxt |   2 +-
 ...orflow.keras.layers.-conv-l-s-t-m2-d.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-conv1-d.pbtxt |   2 +-
 ...flow.keras.layers.-conv2-d-transpose.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-conv2-d.pbtxt |   2 +-
 ...flow.keras.layers.-conv3-d-transpose.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-conv3-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-convolution1-d.pbtxt |   2 +-
 ...ras.layers.-convolution2-d-transpose.pbtxt |   2 +-
 ...sorflow.keras.layers.-convolution2-d.pbtxt |   2 +-
 ...ras.layers.-convolution3-d-transpose.pbtxt |   2 +-
 ...sorflow.keras.layers.-convolution3-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-cropping1-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-cropping2-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-cropping3-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt |   2 +-
 ...rflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-dense.pbtxt   |   2 +-
 ...flow.keras.layers.-depthwise-conv2-d.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-dot.pbtxt     |   2 +-
 .../v1/tensorflow.keras.layers.-dropout.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-e-l-u.pbtxt   |   2 +-
 .../tensorflow.keras.layers.-embedding.pbtxt  |   2 +-
 .../v1/tensorflow.keras.layers.-flatten.pbtxt |   2 +-
 .../tensorflow.keras.layers.-g-r-u-cell.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-g-r-u.pbtxt   |   2 +-
 ...rflow.keras.layers.-gaussian-dropout.pbtxt |   2 +-
 ...sorflow.keras.layers.-gaussian-noise.pbtxt |   2 +-
 ...as.layers.-global-average-pooling1-d.pbtxt |   2 +-
 ...as.layers.-global-average-pooling2-d.pbtxt |   2 +-
 ...as.layers.-global-average-pooling3-d.pbtxt |   2 +-
 ...low.keras.layers.-global-avg-pool1-d.pbtxt |   2 +-
 ...low.keras.layers.-global-avg-pool2-d.pbtxt |   2 +-
 ...low.keras.layers.-global-avg-pool3-d.pbtxt |   2 +-
 ...low.keras.layers.-global-max-pool1-d.pbtxt |   2 +-
 ...low.keras.layers.-global-max-pool2-d.pbtxt |   2 +-
 ...low.keras.layers.-global-max-pool3-d.pbtxt |   2 +-
 ....keras.layers.-global-max-pooling1-d.pbtxt |   2 +-
 ....keras.layers.-global-max-pooling2-d.pbtxt |   2 +-
 ....keras.layers.-global-max-pooling3-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-input-layer.pbtxt |   2 +-
 ...ensorflow.keras.layers.-l-s-t-m-cell.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-l-s-t-m.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-lambda.pbtxt  |   2 +-
 .../v1/tensorflow.keras.layers.-layer.pbtxt   |   2 +-
 ...ensorflow.keras.layers.-leaky-re-l-u.pbtxt |   2 +-
 ...w.keras.layers.-locally-connected1-d.pbtxt |   2 +-
 ...w.keras.layers.-locally-connected2-d.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-masking.pbtxt |   2 +-
 ...tensorflow.keras.layers.-max-pool1-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-max-pool2-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-max-pool3-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-max-pooling1-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-max-pooling2-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-max-pooling3-d.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-maximum.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-minimum.pbtxt |   2 +-
 .../tensorflow.keras.layers.-multiply.pbtxt   |   2 +-
 .../tensorflow.keras.layers.-p-re-l-u.pbtxt   |   2 +-
 .../v1/tensorflow.keras.layers.-permute.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-r-n-n.pbtxt   |   2 +-
 .../v1/tensorflow.keras.layers.-re-l-u.pbtxt  |   2 +-
 ...nsorflow.keras.layers.-repeat-vector.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-reshape.pbtxt |   2 +-
 ...flow.keras.layers.-separable-conv1-d.pbtxt |   2 +-
 ...flow.keras.layers.-separable-conv2-d.pbtxt |   2 +-
 ...ras.layers.-separable-convolution1-d.pbtxt |   2 +-
 ...ras.layers.-separable-convolution2-d.pbtxt |   2 +-
 ...flow.keras.layers.-simple-r-n-n-cell.pbtxt |   2 +-
 ...ensorflow.keras.layers.-simple-r-n-n.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-softmax.pbtxt |   2 +-
 ...low.keras.layers.-spatial-dropout1-d.pbtxt |   2 +-
 ...low.keras.layers.-spatial-dropout2-d.pbtxt |   2 +-
 ...low.keras.layers.-spatial-dropout3-d.pbtxt |   2 +-
 ...ow.keras.layers.-stacked-r-n-n-cells.pbtxt |   2 +-
 .../tensorflow.keras.layers.-subtract.pbtxt   |   2 +-
 ...low.keras.layers.-thresholded-re-l-u.pbtxt |   2 +-
 ...rflow.keras.layers.-time-distributed.pbtxt |   2 +-
 ...sorflow.keras.layers.-up-sampling1-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-up-sampling2-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-up-sampling3-d.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-wrapper.pbtxt |   2 +-
 ...orflow.keras.layers.-zero-padding1-d.pbtxt |   2 +-
 ...orflow.keras.layers.-zero-padding2-d.pbtxt |   2 +-
 ...orflow.keras.layers.-zero-padding3-d.pbtxt |   2 +-
 .../tensorflow.keras.metrics.-accuracy.pbtxt  |   2 +-
 ...rflow.keras.metrics.-binary-accuracy.pbtxt |   2 +-
 ....keras.metrics.-categorical-accuracy.pbtxt |   2 +-
 ...low.keras.metrics.-categorical-hinge.pbtxt |   2 +-
 ...flow.keras.metrics.-cosine-proximity.pbtxt |   2 +-
 ...rflow.keras.metrics.-false-negatives.pbtxt |   2 +-
 ...rflow.keras.metrics.-false-positives.pbtxt |   2 +-
 .../v1/tensorflow.keras.metrics.-hinge.pbtxt  |   2 +-
 ...w.keras.metrics.-mean-absolute-error.pbtxt |   2 +-
 ...rics.-mean-absolute-percentage-error.pbtxt |   2 +-
 ...ow.keras.metrics.-mean-squared-error.pbtxt |   2 +-
 ...rics.-mean-squared-logarithmic-error.pbtxt |   2 +-
 .../v1/tensorflow.keras.metrics.-mean.pbtxt   |   2 +-
 .../tensorflow.keras.metrics.-precision.pbtxt |   2 +-
 .../v1/tensorflow.keras.metrics.-recall.pbtxt |   2 +-
 ....metrics.-sensitivity-at-specificity.pbtxt |   2 +-
 ...metrics.-sparse-categorical-accuracy.pbtxt |   2 +-
 ....metrics.-specificity-at-sensitivity.pbtxt |   2 +-
 ...sorflow.keras.metrics.-squared-hinge.pbtxt |   2 +-
 ...orflow.keras.metrics.-true-negatives.pbtxt |   2 +-
 ...orflow.keras.metrics.-true-positives.pbtxt |   2 +-
 .../v1/tensorflow.keras.models.-model.pbtxt   |   2 +-
 .../tensorflow.keras.models.-sequential.pbtxt |   2 +-
 ...ensorflow.keras.optimizers.-adadelta.pbtxt |   2 +-
 ...tensorflow.keras.optimizers.-adagrad.pbtxt |   2 +-
 .../tensorflow.keras.optimizers.-adam.pbtxt   |   2 +-
 .../tensorflow.keras.optimizers.-adamax.pbtxt |   2 +-
 .../tensorflow.keras.optimizers.-nadam.pbtxt  |   2 +-
 ...nsorflow.keras.optimizers.-optimizer.pbtxt |   2 +-
 ...nsorflow.keras.optimizers.-r-m-sprop.pbtxt |   2 +-
 .../tensorflow.keras.optimizers.-s-g-d.pbtxt  |   2 +-
 ...ensorflow.layers.-average-pooling1-d.pbtxt |   2 +-
 ...ensorflow.layers.-average-pooling2-d.pbtxt |   2 +-
 ...ensorflow.layers.-average-pooling3-d.pbtxt |   2 +-
 ...nsorflow.layers.-batch-normalization.pbtxt |   2 +-
 .../v1/tensorflow.layers.-conv1-d.pbtxt       |   2 +-
 ...tensorflow.layers.-conv2-d-transpose.pbtxt |   2 +-
 .../v1/tensorflow.layers.-conv2-d.pbtxt       |   2 +-
 ...tensorflow.layers.-conv3-d-transpose.pbtxt |   2 +-
 .../v1/tensorflow.layers.-conv3-d.pbtxt       |   2 +-
 .../golden/v1/tensorflow.layers.-dense.pbtxt  |   2 +-
 .../v1/tensorflow.layers.-dropout.pbtxt       |   2 +-
 .../v1/tensorflow.layers.-flatten.pbtxt       |   2 +-
 .../golden/v1/tensorflow.layers.-layer.pbtxt  |   2 +-
 .../tensorflow.layers.-max-pooling1-d.pbtxt   |   2 +-
 .../tensorflow.layers.-max-pooling2-d.pbtxt   |   2 +-
 .../tensorflow.layers.-max-pooling3-d.pbtxt   |   2 +-
 ...tensorflow.layers.-separable-conv1-d.pbtxt |   2 +-
 ...tensorflow.layers.-separable-conv2-d.pbtxt |   2 +-
 ...flow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt |   2 +-
 ...orflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt |   2 +-
 ...nsorflow.nn.rnn_cell.-device-wrapper.pbtxt |   2 +-
 ...sorflow.nn.rnn_cell.-dropout-wrapper.pbtxt |   2 +-
 .../tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt  |   2 +-
 ...tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt |   2 +-
 ...orflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt |   2 +-
 .../tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt  |   2 +-
 ...orflow.nn.rnn_cell.-residual-wrapper.pbtxt |   2 +-
 ...tensorflow.train.-adadelta-optimizer.pbtxt |   2 +-
 ...sorflow.train.-adagrad-d-a-optimizer.pbtxt |   2 +-
 .../tensorflow.train.-adagrad-optimizer.pbtxt |   2 +-
 .../v1/tensorflow.train.-adam-optimizer.pbtxt |   2 +-
 .../v1/tensorflow.train.-checkpoint.pbtxt     |   4 +-
 .../v1/tensorflow.train.-ftrl-optimizer.pbtxt |   2 +-
 ...ow.train.-gradient-descent-optimizer.pbtxt |   2 +-
 ...tensorflow.train.-momentum-optimizer.pbtxt |   2 +-
 .../v1/tensorflow.train.-optimizer.pbtxt      |   2 +-
 ...ow.train.-proximal-adagrad-optimizer.pbtxt |   2 +-
 ...-proximal-gradient-descent-optimizer.pbtxt |   2 +-
 ...nsorflow.train.-r-m-s-prop-optimizer.pbtxt |   2 +-
 ...rflow.train.-sync-replicas-optimizer.pbtxt |   2 +-
 .../api/golden/v2/tensorflow.-variable.pbtxt  |   2 +-
 .../golden/v2/tensorflow.keras.-model.pbtxt   |   2 +-
 .../v2/tensorflow.keras.-sequential.pbtxt     |   2 +-
 ....experimental.-peephole-l-s-t-m-cell.pbtxt |   2 +-
 .../tensorflow.keras.layers.-activation.pbtxt |   2 +-
 ...eras.layers.-activity-regularization.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-add.pbtxt     |   2 +-
 ...nsorflow.keras.layers.-alpha-dropout.pbtxt |   2 +-
 ...low.keras.layers.-average-pooling1-d.pbtxt |   2 +-
 ...low.keras.layers.-average-pooling2-d.pbtxt |   2 +-
 ...low.keras.layers.-average-pooling3-d.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-average.pbtxt |   2 +-
 ...tensorflow.keras.layers.-avg-pool1-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-avg-pool2-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-avg-pool3-d.pbtxt |   2 +-
 ...ow.keras.layers.-batch-normalization.pbtxt |   2 +-
 ...nsorflow.keras.layers.-bidirectional.pbtxt |   2 +-
 ...tensorflow.keras.layers.-concatenate.pbtxt |   2 +-
 ...orflow.keras.layers.-conv-l-s-t-m2-d.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-conv1-d.pbtxt |   2 +-
 ...flow.keras.layers.-conv2-d-transpose.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-conv2-d.pbtxt |   2 +-
 ...flow.keras.layers.-conv3-d-transpose.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-conv3-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-convolution1-d.pbtxt |   2 +-
 ...ras.layers.-convolution2-d-transpose.pbtxt |   2 +-
 ...sorflow.keras.layers.-convolution2-d.pbtxt |   2 +-
 ...ras.layers.-convolution3-d-transpose.pbtxt |   2 +-
 ...sorflow.keras.layers.-convolution3-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-cropping1-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-cropping2-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-cropping3-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-dense-features.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-dense.pbtxt   |   2 +-
 ...flow.keras.layers.-depthwise-conv2-d.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-dot.pbtxt     |   2 +-
 .../v2/tensorflow.keras.layers.-dropout.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-e-l-u.pbtxt   |   2 +-
 .../tensorflow.keras.layers.-embedding.pbtxt  |   2 +-
 .../v2/tensorflow.keras.layers.-flatten.pbtxt |   2 +-
 .../tensorflow.keras.layers.-g-r-u-cell.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-g-r-u.pbtxt   |   2 +-
 ...rflow.keras.layers.-gaussian-dropout.pbtxt |   2 +-
 ...sorflow.keras.layers.-gaussian-noise.pbtxt |   2 +-
 ...as.layers.-global-average-pooling1-d.pbtxt |   2 +-
 ...as.layers.-global-average-pooling2-d.pbtxt |   2 +-
 ...as.layers.-global-average-pooling3-d.pbtxt |   2 +-
 ...low.keras.layers.-global-avg-pool1-d.pbtxt |   2 +-
 ...low.keras.layers.-global-avg-pool2-d.pbtxt |   2 +-
 ...low.keras.layers.-global-avg-pool3-d.pbtxt |   2 +-
 ...low.keras.layers.-global-max-pool1-d.pbtxt |   2 +-
 ...low.keras.layers.-global-max-pool2-d.pbtxt |   2 +-
 ...low.keras.layers.-global-max-pool3-d.pbtxt |   2 +-
 ....keras.layers.-global-max-pooling1-d.pbtxt |   2 +-
 ....keras.layers.-global-max-pooling2-d.pbtxt |   2 +-
 ....keras.layers.-global-max-pooling3-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-input-layer.pbtxt |   2 +-
 ...ensorflow.keras.layers.-l-s-t-m-cell.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-l-s-t-m.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-lambda.pbtxt  |   2 +-
 .../v2/tensorflow.keras.layers.-layer.pbtxt   |   2 +-
 ...ensorflow.keras.layers.-leaky-re-l-u.pbtxt |   2 +-
 ...ensorflow.keras.layers.-linear-model.pbtxt |   2 +-
 ...w.keras.layers.-locally-connected1-d.pbtxt |   2 +-
 ...w.keras.layers.-locally-connected2-d.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-masking.pbtxt |   2 +-
 ...tensorflow.keras.layers.-max-pool1-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-max-pool2-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-max-pool3-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-max-pooling1-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-max-pooling2-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-max-pooling3-d.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-maximum.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-minimum.pbtxt |   2 +-
 .../tensorflow.keras.layers.-multiply.pbtxt   |   2 +-
 .../tensorflow.keras.layers.-p-re-l-u.pbtxt   |   2 +-
 .../v2/tensorflow.keras.layers.-permute.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-r-n-n.pbtxt   |   2 +-
 .../v2/tensorflow.keras.layers.-re-l-u.pbtxt  |   2 +-
 ...nsorflow.keras.layers.-repeat-vector.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-reshape.pbtxt |   2 +-
 ...flow.keras.layers.-separable-conv1-d.pbtxt |   2 +-
 ...flow.keras.layers.-separable-conv2-d.pbtxt |   2 +-
 ...ras.layers.-separable-convolution1-d.pbtxt |   2 +-
 ...ras.layers.-separable-convolution2-d.pbtxt |   2 +-
 ...flow.keras.layers.-simple-r-n-n-cell.pbtxt |   2 +-
 ...ensorflow.keras.layers.-simple-r-n-n.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-softmax.pbtxt |   2 +-
 ...low.keras.layers.-spatial-dropout1-d.pbtxt |   2 +-
 ...low.keras.layers.-spatial-dropout2-d.pbtxt |   2 +-
 ...low.keras.layers.-spatial-dropout3-d.pbtxt |   2 +-
 ...ow.keras.layers.-stacked-r-n-n-cells.pbtxt |   2 +-
 .../tensorflow.keras.layers.-subtract.pbtxt   |   2 +-
 ...low.keras.layers.-thresholded-re-l-u.pbtxt |   2 +-
 ...rflow.keras.layers.-time-distributed.pbtxt |   2 +-
 ...sorflow.keras.layers.-up-sampling1-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-up-sampling2-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-up-sampling3-d.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-wrapper.pbtxt |   2 +-
 ...orflow.keras.layers.-zero-padding1-d.pbtxt |   2 +-
 ...orflow.keras.layers.-zero-padding2-d.pbtxt |   2 +-
 ...orflow.keras.layers.-zero-padding3-d.pbtxt |   2 +-
 .../tensorflow.keras.metrics.-accuracy.pbtxt  |   2 +-
 ...rflow.keras.metrics.-binary-accuracy.pbtxt |   2 +-
 ....keras.metrics.-categorical-accuracy.pbtxt |   2 +-
 ...low.keras.metrics.-categorical-hinge.pbtxt |   2 +-
 ...flow.keras.metrics.-cosine-proximity.pbtxt |   2 +-
 ...rflow.keras.metrics.-false-negatives.pbtxt |   2 +-
 ...rflow.keras.metrics.-false-positives.pbtxt |   2 +-
 .../v2/tensorflow.keras.metrics.-hinge.pbtxt  |   2 +-
 ...w.keras.metrics.-mean-absolute-error.pbtxt |   2 +-
 ...rics.-mean-absolute-percentage-error.pbtxt |   2 +-
 ...ow.keras.metrics.-mean-squared-error.pbtxt |   2 +-
 ...rics.-mean-squared-logarithmic-error.pbtxt |   2 +-
 .../v2/tensorflow.keras.metrics.-mean.pbtxt   |   2 +-
 .../tensorflow.keras.metrics.-precision.pbtxt |   2 +-
 .../v2/tensorflow.keras.metrics.-recall.pbtxt |   2 +-
 ....metrics.-sensitivity-at-specificity.pbtxt |   2 +-
 ...metrics.-sparse-categorical-accuracy.pbtxt |   2 +-
 ....metrics.-specificity-at-sensitivity.pbtxt |   2 +-
 ...sorflow.keras.metrics.-squared-hinge.pbtxt |   2 +-
 ...orflow.keras.metrics.-true-negatives.pbtxt |   2 +-
 ...orflow.keras.metrics.-true-positives.pbtxt |   2 +-
 .../v2/tensorflow.keras.models.-model.pbtxt   |   2 +-
 .../tensorflow.keras.models.-sequential.pbtxt |   2 +-
 ...ensorflow.keras.optimizers.-adadelta.pbtxt |   2 +-
 ...tensorflow.keras.optimizers.-adagrad.pbtxt |   2 +-
 .../tensorflow.keras.optimizers.-adam.pbtxt   |   2 +-
 .../tensorflow.keras.optimizers.-adamax.pbtxt |   2 +-
 .../tensorflow.keras.optimizers.-nadam.pbtxt  |   2 +-
 ...nsorflow.keras.optimizers.-optimizer.pbtxt |   2 +-
 ...nsorflow.keras.optimizers.-r-m-sprop.pbtxt |   2 +-
 .../tensorflow.keras.optimizers.-s-g-d.pbtxt  |   2 +-
 ...nsorflow.nn.rnn_cell.-device-wrapper.pbtxt |   2 +-
 .../tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt  |   2 +-
 ...orflow.nn.rnn_cell.-residual-wrapper.pbtxt |   2 +-
 .../v2/tensorflow.rnn.-dropout-wrapper.pbtxt  |   2 +-
 .../v2/tensorflow.train.-checkpoint.pbtxt     |   4 +-
 ...-proximal-gradient-descent-optimizer.pbtxt |   2 +-
 357 files changed, 535 insertions(+), 536 deletions(-)

diff --git a/tensorflow/contrib/checkpoint/__init__.py b/tensorflow/contrib/checkpoint/__init__.py
index 94b7f4f867..99ed4959fa 100644
--- a/tensorflow/contrib/checkpoint/__init__.py
+++ b/tensorflow/contrib/checkpoint/__init__.py
@@ -51,11 +51,11 @@ from tensorflow.contrib.checkpoint.python.split_dependency import split_dependen
 from tensorflow.contrib.checkpoint.python.visualize import dot_graph_from_checkpoint
 from tensorflow.core.protobuf.checkpointable_object_graph_pb2 import CheckpointableObjectGraph
 from tensorflow.python.training.checkpoint_management import CheckpointManager
-from tensorflow.python.training.checkpointable.base import CheckpointableBase
+from tensorflow.python.training.checkpointable.base import Checkpointable as CheckpointableBase
 from tensorflow.python.training.checkpointable.data_structures import List
 from tensorflow.python.training.checkpointable.data_structures import Mapping
 from tensorflow.python.training.checkpointable.data_structures import NoDependency
-from tensorflow.python.training.checkpointable.tracking import Checkpointable
+from tensorflow.python.training.checkpointable.tracking import AutoCheckpointable as Checkpointable
 from tensorflow.python.training.checkpointable.util import capture_dependencies
 from tensorflow.python.training.checkpointable.util import list_objects
 from tensorflow.python.training.checkpointable.util import object_metadata
diff --git a/tensorflow/contrib/checkpoint/python/containers.py b/tensorflow/contrib/checkpoint/python/containers.py
index 5418e2605b..97936d9e9d 100644
--- a/tensorflow/contrib/checkpoint/python/containers.py
+++ b/tensorflow/contrib/checkpoint/python/containers.py
@@ -63,7 +63,7 @@ class UniqueNameTracker(data_structures.CheckpointableDataStructure):
       ValueError: If `checkpointable` is not a checkpointable object.
     """
 
-    if not isinstance(checkpointable, checkpointable_lib.CheckpointableBase):
+    if not isinstance(checkpointable, checkpointable_lib.Checkpointable):
       raise ValueError(
           ("Expected a checkpointable value, got %s which does not inherit "
            "from CheckpointableBase.") % (checkpointable,))
diff --git a/tensorflow/contrib/checkpoint/python/containers_test.py b/tensorflow/contrib/checkpoint/python/containers_test.py
index ac85c7be80..a2d453ec6e 100644
--- a/tensorflow/contrib/checkpoint/python/containers_test.py
+++ b/tensorflow/contrib/checkpoint/python/containers_test.py
@@ -52,7 +52,7 @@ class UniqueNameTrackerTests(test.TestCase):
     save_root = util.Checkpoint(slots=slots)
     save_path = save_root.save(checkpoint_prefix)
 
-    restore_slots = tracking.Checkpointable()
+    restore_slots = tracking.AutoCheckpointable()
     restore_root = util.Checkpoint(
         slots=restore_slots)
     status = restore_root.restore(save_path)
@@ -68,7 +68,7 @@ class UniqueNameTrackerTests(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
   def testExample(self):
-    class SlotManager(tracking.Checkpointable):
+    class SlotManager(tracking.AutoCheckpointable):
 
       def __init__(self):
         self.slotdeps = containers.UniqueNameTracker()
diff --git a/tensorflow/contrib/checkpoint/python/python_state.py b/tensorflow/contrib/checkpoint/python/python_state.py
index 302d5cfb79..969c90c788 100644
--- a/tensorflow/contrib/checkpoint/python/python_state.py
+++ b/tensorflow/contrib/checkpoint/python/python_state.py
@@ -34,7 +34,7 @@ except ImportError:
 # pylint: enable=g-import-not-at-top
 
 
-class NumpyState(base.CheckpointableBase):
+class NumpyState(base.Checkpointable):
   """A checkpointable object whose NumPy array attributes are saved/restored.
 
   Example usage:
@@ -130,7 +130,7 @@ class NumpyState(base.CheckpointableBase):
 
 
 @six.add_metaclass(abc.ABCMeta)
-class PythonStateWrapper(base.CheckpointableBase):
+class PythonStateWrapper(base.Checkpointable):
   """Wraps a Python object for storage in an object-based checkpoint."""
 
   @abc.abstractmethod
diff --git a/tensorflow/contrib/checkpoint/python/split_dependency.py b/tensorflow/contrib/checkpoint/python/split_dependency.py
index 7e77453f3d..3e9700ad74 100644
--- a/tensorflow/contrib/checkpoint/python/split_dependency.py
+++ b/tensorflow/contrib/checkpoint/python/split_dependency.py
@@ -43,7 +43,7 @@ class _CallbackSaveable(saver_lib.BaseSaverBuilder.SaveableObject):
     return self._restore_callback(tensor)
 
 
-class _SplitDependency(checkpointable.CheckpointableBase):
+class _SplitDependency(checkpointable.Checkpointable):
   """Looks like a regular variable while synchronizing save/restores."""
 
   def __init__(self, save_buffer, restore_buffer, name, dtype, num_components,
diff --git a/tensorflow/contrib/checkpoint/python/split_dependency_test.py b/tensorflow/contrib/checkpoint/python/split_dependency_test.py
index 00a805af25..664a4e76ab 100644
--- a/tensorflow/contrib/checkpoint/python/split_dependency_test.py
+++ b/tensorflow/contrib/checkpoint/python/split_dependency_test.py
@@ -44,7 +44,7 @@ def _combine_variable_closure(variable):
   return _consume_restore_buffer_fn
 
 
-class SaveTensorSlicesAsDeps(base.CheckpointableBase):
+class SaveTensorSlicesAsDeps(base.Checkpointable):
 
   def __init__(self):
     self.combined = resource_variable_ops.ResourceVariable([0., 0., 0., 0.])
@@ -59,14 +59,14 @@ class SaveTensorSlicesAsDeps(base.CheckpointableBase):
       self._track_checkpointable(dep, name=name)
 
 
-class HasRegularDeps(tracking.Checkpointable):
+class HasRegularDeps(tracking.AutoCheckpointable):
 
   def __init__(self):
     self.first_half = resource_variable_ops.ResourceVariable([0., 0.])
     self.second_half = resource_variable_ops.ResourceVariable([0., 0.])
 
 
-class OnlyOneDep(tracking.Checkpointable):
+class OnlyOneDep(tracking.AutoCheckpointable):
 
   def __init__(self):
     self.first_half = resource_variable_ops.ResourceVariable([0., 0.])
diff --git a/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py b/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py
index 1facc83972..f36e8d5022 100644
--- a/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py
+++ b/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py
@@ -837,7 +837,7 @@ class CudnnLSTMSaveable(CudnnOpaqueParamsSaveable):
       checkpointable._track_checkpointable(bias, name="bias")  # pylint: disable=protected-access
     assert len(biases) == len(weights)
     for cell_index, (bias, kernel) in enumerate(zip(biases, weights)):
-      cell = checkpointable_lib.Checkpointable()
+      cell = checkpointable_lib.AutoCheckpointable()
       checkpointable._track_checkpointable(cell, name="cell-%d" % cell_index)  # pylint: disable=protected-access
       cell.bias = bias
       cell.kernel = kernel
diff --git a/tensorflow/contrib/distribute/python/checkpointing_test.py b/tensorflow/contrib/distribute/python/checkpointing_test.py
index f37026bc95..aa5b9f57b8 100644
--- a/tensorflow/contrib/distribute/python/checkpointing_test.py
+++ b/tensorflow/contrib/distribute/python/checkpointing_test.py
@@ -34,7 +34,7 @@ from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.training.checkpointable import util as checkpointable_utils
 
 
-class NonLayerCheckpointable(tracking.Checkpointable):
+class NonLayerCheckpointable(tracking.AutoCheckpointable):
 
   def __init__(self):
     super(NonLayerCheckpointable, self).__init__()
diff --git a/tensorflow/contrib/eager/python/metrics_impl.py b/tensorflow/contrib/eager/python/metrics_impl.py
index 566246de49..c8d9266672 100644
--- a/tensorflow/contrib/eager/python/metrics_impl.py
+++ b/tensorflow/contrib/eager/python/metrics_impl.py
@@ -37,7 +37,7 @@ from tensorflow.python.training.checkpointable import base as checkpointable
 _to_replace = re.compile("[^A-Za-z0-9.]")
 
 
-class Metric(checkpointable.CheckpointableBase):
+class Metric(checkpointable.Checkpointable):
   """A metric holds state for aggregating statistics over an evaluation run.
 
   Example use with eager execution:
diff --git a/tensorflow/contrib/eager/python/tfe.py b/tensorflow/contrib/eager/python/tfe.py
index 31481d7685..b82e1bb71b 100644
--- a/tensorflow/contrib/eager/python/tfe.py
+++ b/tensorflow/contrib/eager/python/tfe.py
@@ -138,7 +138,7 @@ from tensorflow.python.ops.resource_variable_ops import ResourceVariable as Vari
 from tensorflow.python.ops.variable_scope import EagerVariableStore
 from tensorflow.python.ops import script_ops
 from tensorflow.python.ops import template
-from tensorflow.python.training.checkpointable.tracking import Checkpointable
+from tensorflow.python.training.checkpointable.tracking import AutoCheckpointable as Checkpointable
 from tensorflow.python.training.checkpointable.util import CheckpointableSaver
 from tensorflow.python.training.checkpointable.util import Checkpoint
 from tensorflow.python.util.all_util import remove_undocumented
diff --git a/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py b/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py
index 72019b3154..0243927ce4 100644
--- a/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py
+++ b/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py
@@ -48,7 +48,7 @@ from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.training.checkpointable import util
 
 
-class NonLayerCheckpointable(tracking.Checkpointable):
+class NonLayerCheckpointable(tracking.AutoCheckpointable):
 
   def __init__(self):
     super(NonLayerCheckpointable, self).__init__()
@@ -440,7 +440,7 @@ class CheckpointingTests(test.TestCase):
   def testDeferredSlotRestoration(self):
     checkpoint_directory = self.get_temp_dir()
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.var = util.add_variable(
         root, name="var", initializer=0.)
     optimizer = adam.AdamOptimizer(0.1)
@@ -463,7 +463,7 @@ class CheckpointingTests(test.TestCase):
                                    14.))
     slots_path = util.CheckpointableSaver(root).save(
         os.path.join(checkpoint_directory, "with_slots"))
-    new_root = tracking.Checkpointable()
+    new_root = tracking.AutoCheckpointable()
     # Load the slot-containing checkpoint (deferred), then immediately overwrite
     # the non-slot variable (also deferred).
     slot_status = util.CheckpointableSaver(
@@ -508,7 +508,7 @@ class CheckpointingTests(test.TestCase):
       with graph.as_default(), self.session(graph):
         checkpoint_directory = self.get_temp_dir()
         checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-        obj = tracking.Checkpointable()
+        obj = tracking.AutoCheckpointable()
         obj.var = variable_scope.get_variable(name="v", initializer=0.)
         obj.opt = adam.AdamOptimizer(0.1)
         obj.opt.minimize(obj.var.read_value())
@@ -526,7 +526,7 @@ class CheckpointingTests(test.TestCase):
       with graph.as_default(), self.session(graph):
         checkpoint_directory = self.get_temp_dir()
         checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-        obj = tracking.Checkpointable()
+        obj = tracking.AutoCheckpointable()
         obj.var = variable_scope.get_variable(name="v", initializer=0.)
         obj.opt = adam.AdamOptimizer(0.1)
         obj.opt.minimize(obj.var.read_value())
diff --git a/tensorflow/python/data/ops/iterator_ops.py b/tensorflow/python/data/ops/iterator_ops.py
index bfa256f8d7..d6fb73813c 100644
--- a/tensorflow/python/data/ops/iterator_ops.py
+++ b/tensorflow/python/data/ops/iterator_ops.py
@@ -68,7 +68,7 @@ def _device_stack_is_empty():
 
 
 @tf_export(v1=["data.Iterator"])
-class Iterator(checkpointable.CheckpointableBase):
+class Iterator(checkpointable.Checkpointable):
   """Represents the state of iterating through a `Dataset`."""
 
   def __init__(self, iterator_resource, initializer, output_types,
@@ -491,7 +491,7 @@ def _generate_shared_name(prefix):
   return "{}{}".format(prefix, uid)
 
 
-class EagerIterator(checkpointable.CheckpointableBase):
+class EagerIterator(checkpointable.Checkpointable):
   """An iterator producing tf.Tensor objects from a tf.data.Dataset."""
 
   def __init__(self, dataset):
diff --git a/tensorflow/python/distribute/values.py b/tensorflow/python/distribute/values.py
index ec0ea4bc2d..e7db285be5 100644
--- a/tensorflow/python/distribute/values.py
+++ b/tensorflow/python/distribute/values.py
@@ -626,7 +626,7 @@ class _MirroredSaveable(saver.BaseSaverBuilder.ResourceVariableSaveable):
 
 
 class MirroredVariable(DistributedVariable, Mirrored,
-                       checkpointable.CheckpointableBase):
+                       checkpointable.Checkpointable):
   """Holds a map from device to variables whose values are kept in sync."""
 
   def __init__(
@@ -748,7 +748,7 @@ def _enclosing_tpu_context():
 # tpu.replicate() because it assumes that you're in a device context where you
 # can operate on a single version of the variable, but a tpu.replicate()
 # operates on all variables and is replicated during a rewrite pass.
-class TPUMirroredVariable(checkpointable.CheckpointableBase):
+class TPUMirroredVariable(checkpointable.Checkpointable):
   """Holds a map from device to TPU variables whose values are kept in sync."""
 
   def __init__(
@@ -1201,7 +1201,7 @@ def _assert_replica_context(strategy):
 
 
 class ReplicaLocalVariable(DistributedVariable, PerReplica,
-                           checkpointable.CheckpointableBase):
+                           checkpointable.Checkpointable):
   """Holds a map from device to variables whose values are reduced on save."""
 
   def __init__(
@@ -1432,7 +1432,7 @@ def value_container(val):
 
 
 # TODO(josh11b): Descend from Variable.
-class AggregatingVariable(checkpointable.CheckpointableBase):
+class AggregatingVariable(checkpointable.Checkpointable):
   """A wrapper around a variable that aggregates updates across replicas."""
 
   def __init__(self, strategy, v, aggregation):
diff --git a/tensorflow/python/feature_column/feature_column_v2.py b/tensorflow/python/feature_column/feature_column_v2.py
index 73e3001fc0..5976f647af 100644
--- a/tensorflow/python/feature_column/feature_column_v2.py
+++ b/tensorflow/python/feature_column/feature_column_v2.py
@@ -3175,7 +3175,7 @@ def _raise_shared_embedding_column_error():
                    '`DenseFeatures` or `LinearModel` instead.')
 
 
-class SharedEmbeddingColumnCreator(tracking.Checkpointable):
+class SharedEmbeddingColumnCreator(tracking.AutoCheckpointable):
 
   def __init__(self,
                dimension,
diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index 47900e7533..5a3540d12d 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -57,7 +57,7 @@ from tensorflow.tools.docs import doc_controls
 
 
 @keras_export('keras.layers.Layer')
-class Layer(checkpointable.CheckpointableBase):
+class Layer(checkpointable.Checkpointable):
   """Base layer class.
 
   This is the class from which all layers inherit.
diff --git a/tensorflow/python/keras/engine/network.py b/tensorflow/python/keras/engine/network.py
index cca9874e8b..41f5f319bc 100644
--- a/tensorflow/python/keras/engine/network.py
+++ b/tensorflow/python/keras/engine/network.py
@@ -1434,7 +1434,7 @@ class Network(base_layer.Layer):
         session = backend.get_session()
       optimizer = getattr(self, 'optimizer', None)
       if (optimizer
-          and not isinstance(optimizer, checkpointable.CheckpointableBase)):
+          and not isinstance(optimizer, checkpointable.Checkpointable)):
         logging.warning(
             ('This model was compiled with a Keras optimizer (%s) but is being '
              'saved in TensorFlow format with `save_weights`. The model\'s '
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index d690df2986..f5a720b292 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -261,7 +261,7 @@ class Model(Network):
     self.optimizer = optimizer
     # We've disabled automatic dependency tracking for this method, but do want
     # to add a checkpoint dependency on the optimizer if it's checkpointable.
-    if isinstance(self.optimizer, checkpointable.CheckpointableBase):
+    if isinstance(self.optimizer, checkpointable.Checkpointable):
       self._track_checkpointable(
           self.optimizer, name='optimizer', overwrite=True)
     self.loss = loss
diff --git a/tensorflow/python/keras/layers/recurrent.py b/tensorflow/python/keras/layers/recurrent.py
index 797278a5e5..9404543ed9 100644
--- a/tensorflow/python/keras/layers/recurrent.py
+++ b/tensorflow/python/keras/layers/recurrent.py
@@ -468,7 +468,7 @@ class RNN(Layer):
     self.zero_output_for_mask = kwargs.pop('zero_output_for_mask', False)
     super(RNN, self).__init__(**kwargs)
     self.cell = cell
-    if isinstance(cell, checkpointable.CheckpointableBase):
+    if isinstance(cell, checkpointable.Checkpointable):
       self._track_checkpointable(self.cell, name='cell')
     self.return_sequences = return_sequences
     self.return_state = return_state
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
index bc210132b5..6cd6cf0a8d 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
@@ -70,7 +70,7 @@ def _deduplicate_indexed_slices(values, indices):
 
 @six.add_metaclass(abc.ABCMeta)
 @keras_export("keras.optimizers.Optimizer")
-class OptimizerV2(checkpointable.CheckpointableBase):
+class OptimizerV2(checkpointable.Checkpointable):
   """Updated base class for optimizers.
 
   This class defines the API to add Ops to train a model.  You never use this
diff --git a/tensorflow/python/keras/optimizers.py b/tensorflow/python/keras/optimizers.py
index 8c3bf110d4..b704b885cb 100644
--- a/tensorflow/python/keras/optimizers.py
+++ b/tensorflow/python/keras/optimizers.py
@@ -710,7 +710,7 @@ class Nadam(Optimizer):
     return dict(list(base_config.items()) + list(config.items()))
 
 
-class TFOptimizer(Optimizer, checkpointable.CheckpointableBase):
+class TFOptimizer(Optimizer, checkpointable.Checkpointable):
   """Wrapper class for native TensorFlow optimizers.
   """
 
diff --git a/tensorflow/python/layers/base.py b/tensorflow/python/layers/base.py
index 5354d437b4..1b84ec1f69 100644
--- a/tensorflow/python/layers/base.py
+++ b/tensorflow/python/layers/base.py
@@ -554,7 +554,7 @@ class Layer(base_layer.Layer):
 
   def __setattr__(self, value, name):
     # By-pass the automatic dependency tracking performed by the parent Layer.
-    super(checkpointable.CheckpointableBase, self).__setattr__(value, name)
+    super(checkpointable.Checkpointable, self).__setattr__(value, name)
 
 
 def _add_elements_to_collection(elements, collection_list):
diff --git a/tensorflow/python/ops/lookup_ops.py b/tensorflow/python/ops/lookup_ops.py
index 0cc3ea5ec7..6a7057fafd 100644
--- a/tensorflow/python/ops/lookup_ops.py
+++ b/tensorflow/python/ops/lookup_ops.py
@@ -164,7 +164,7 @@ class InitializableLookupTableBase(LookupInterface):
     self._default_value = ops.convert_to_tensor(
         default_value, dtype=self._value_dtype)
     self._default_value.get_shape().merge_with(tensor_shape.scalar())
-    if isinstance(initializer, checkpointable_base.CheckpointableBase):
+    if isinstance(initializer, checkpointable_base.Checkpointable):
       self._initializer = self._track_checkpointable(
           initializer, "_initializer")
     self._resource_handle = self.create_resource()
@@ -315,7 +315,7 @@ class HashTable(InitializableLookupTableBase):
     return exported_keys, exported_values
 
 
-class TableInitializerBase(checkpointable_base.CheckpointableBase):
+class TableInitializerBase(checkpointable_base.Checkpointable):
   """Base class for lookup table initializers."""
 
   def __init__(self, key_dtype, value_dtype):
diff --git a/tensorflow/python/ops/rnn_cell_impl.py b/tensorflow/python/ops/rnn_cell_impl.py
index 2da66cfcd5..40c3771f4e 100644
--- a/tensorflow/python/ops/rnn_cell_impl.py
+++ b/tensorflow/python/ops/rnn_cell_impl.py
@@ -1183,7 +1183,7 @@ class DropoutWrapper(RNNCell):
 
     # Set cell, variational_recurrent, seed before running the code below
     self._cell = cell
-    if isinstance(cell, checkpointable.CheckpointableBase):
+    if isinstance(cell, checkpointable.Checkpointable):
       self._track_checkpointable(self._cell, name="cell")
     self._variational_recurrent = variational_recurrent
     self._seed = seed
@@ -1424,7 +1424,7 @@ class ResidualWrapper(RNNCell):
     """
     super(ResidualWrapper, self).__init__()
     self._cell = cell
-    if isinstance(cell, checkpointable.CheckpointableBase):
+    if isinstance(cell, checkpointable.Checkpointable):
       self._track_checkpointable(self._cell, name="cell")
     self._residual_fn = residual_fn
 
@@ -1482,7 +1482,7 @@ class DeviceWrapper(RNNCell):
     """
     super(DeviceWrapper, self).__init__()
     self._cell = cell
-    if isinstance(cell, checkpointable.CheckpointableBase):
+    if isinstance(cell, checkpointable.Checkpointable):
       self._track_checkpointable(self._cell, name="cell")
     self._device = device
 
@@ -1551,7 +1551,7 @@ class MultiRNNCell(RNNCell):
     for cell_number, cell in enumerate(self._cells):
       # Add Checkpointable dependencies on these cells so their variables get
       # saved with this object when using object-based saving.
-      if isinstance(cell, checkpointable.CheckpointableBase):
+      if isinstance(cell, checkpointable.Checkpointable):
         # TODO(allenl): Track down non-Checkpointable callers.
         self._track_checkpointable(cell, name="cell-%d" % (cell_number,))
     self._state_is_tuple = state_is_tuple
diff --git a/tensorflow/python/ops/stateful_random_ops.py b/tensorflow/python/ops/stateful_random_ops.py
index c7a14f2015..155ad969f6 100644
--- a/tensorflow/python/ops/stateful_random_ops.py
+++ b/tensorflow/python/ops/stateful_random_ops.py
@@ -151,7 +151,7 @@ def _shape_tensor(shape):
 
 
 @tf_export("random.experimental.Generator")
-class Generator(tracking.Checkpointable):
+class Generator(tracking.AutoCheckpointable):
   """Random-number generator.
 
   It uses Variable to manage its internal state.
diff --git a/tensorflow/python/ops/template.py b/tensorflow/python/ops/template.py
index 4eaa16a22c..e02175d6fe 100644
--- a/tensorflow/python/ops/template.py
+++ b/tensorflow/python/ops/template.py
@@ -232,7 +232,7 @@ def _skip_common_stack_elements(stacktrace, base_case):
   return stacktrace[-1:]
 
 
-class Template(checkpointable.CheckpointableBase):
+class Template(checkpointable.Checkpointable):
   """Wrap a function to aid in variable sharing.
 
   Templates are functions that create variables the first time they are called
diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index d765936399..7eb2e14b42 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -204,7 +204,7 @@ class VariableMetaclass(type):
 
 @tf_export("Variable", v1=[])
 class Variable(six.with_metaclass(VariableMetaclass,
-                                  checkpointable.CheckpointableBase)):
+                                  checkpointable.Checkpointable)):
   """See the [Variables Guide](https://tensorflow.org/guide/variables).
 
   A variable maintains state in the graph across calls to `run()`. You add a
diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
index 93a081b62b..134373906d 100644
--- a/tensorflow/python/saved_model/load.py
+++ b/tensorflow/python/saved_model/load.py
@@ -150,7 +150,7 @@ class _Loader(object):
       # individually callable by adding a `__call__` method to the classes of
       # the objects instances that have a `__call__` property.
 
-      class _UserObject(tracking.Checkpointable):
+      class _UserObject(tracking.AutoCheckpointable):
         pass
 
       return _UserObject(), setattr
@@ -200,4 +200,3 @@ def load(export_dir):
         "Currently only SavedModels exported with `tf.saved_model.save` may be "
         "imported. Other SavedModels may eventually be supported via load().")
   return root
-
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index ff6b72053b..9eea170bf0 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -43,17 +43,17 @@ class LoadTest(test.TestCase):
     return load.load(path)
 
   def test_structure_import(self):
-    root = tracking.Checkpointable()
-    root.dep_one = tracking.Checkpointable()
-    root.dep_two = tracking.Checkpointable()
-    root.dep_two.dep = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
+    root.dep_one = tracking.AutoCheckpointable()
+    root.dep_two = tracking.AutoCheckpointable()
+    root.dep_two.dep = tracking.AutoCheckpointable()
     root.dep_three = root.dep_two.dep
     imported = self.cycle(root)
     self.assertIs(imported.dep_three, imported.dep_two.dep)
     self.assertIsNot(imported.dep_one, imported.dep_two)
 
   def test_variables(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.v1 = variables.Variable(1., trainable=True)
     root.v2 = variables.Variable(2., trainable=False)
     imported = self.cycle(root)
@@ -63,7 +63,7 @@ class LoadTest(test.TestCase):
     self.assertFalse(imported.v2.trainable)
 
   def test_capture_variables(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.weights = variables.Variable(2.)
     root.f = def_function.function(
         lambda x: root.weights * x,
@@ -83,7 +83,7 @@ class LoadTest(test.TestCase):
     file1 = self._make_asset("contents 1")
     file2 = self._make_asset("contents 2")
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.asset1 = tracking.TrackableAsset(file1)
     root.asset2 = tracking.TrackableAsset(file2)
 
@@ -102,7 +102,7 @@ class LoadTest(test.TestCase):
       self.assertEquals("contents 2", f.read())
 
   def test_capture_assets(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.vocab = tracking.TrackableAsset(self._make_asset("contents"))
     root.f = def_function.function(
         lambda: root.vocab.asset_path,
@@ -116,7 +116,7 @@ class LoadTest(test.TestCase):
 
   def test_dedup_assets(self):
     vocab = self._make_asset("contents")
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.asset1 = tracking.TrackableAsset(vocab)
     root.asset2 = tracking.TrackableAsset(vocab)
     imported = self.cycle(root)
@@ -128,7 +128,7 @@ class LoadTest(test.TestCase):
     def func(x):
       return 2 * x
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.f = func
 
     # Add two traces.
@@ -146,7 +146,7 @@ class LoadTest(test.TestCase):
     def func(x):
       return 2 * x
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.f = func
 
     imported = self.cycle(root)
@@ -157,7 +157,7 @@ class LoadTest(test.TestCase):
     def func(x):
       return 2 * x
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.f = func
 
     imported = self.cycle(
@@ -173,7 +173,7 @@ class LoadTest(test.TestCase):
         lambda x: f(x) + 1.0,
         input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.g = g
     imported = self.cycle(root)
     imported.g(constant_op.constant([1.0]))
@@ -186,7 +186,7 @@ class LoadTest(test.TestCase):
       else:
         return 7
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.f = def_function.function(func)
 
     self.assertEqual(20, root.f(constant_op.constant(10), True).numpy())
@@ -208,7 +208,7 @@ class LoadTest(test.TestCase):
       else:
         return 7
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.f = def_function.function(func)
 
     x = constant_op.constant(10)
@@ -240,7 +240,7 @@ class LoadTest(test.TestCase):
       named_tuple = named_tuple_type(a=input1 + input2, b=input1 * input2)
       return [named_tuple, input2, {"x": 0.5}]
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.f = def_function.function(func)
 
     result = root.f(constant_op.constant(2), constant_op.constant(3))
@@ -270,7 +270,7 @@ class LoadTest(test.TestCase):
       else:
         return 7
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.f = def_function.function(func)
 
     self.assertEqual(20, root.f(constant_op.constant(10), True).numpy())
@@ -285,7 +285,7 @@ class LoadTest(test.TestCase):
     self.assertEqual(6, imported.f(constant_op.constant(1), defg=7.0).numpy())
 
   def test_member_function(self):
-    class CheckpointableWithMember(tracking.Checkpointable):
+    class CheckpointableWithMember(tracking.AutoCheckpointable):
 
       def __init__(self):
         super(CheckpointableWithMember, self).__init__()
@@ -310,7 +310,7 @@ class LoadTest(test.TestCase):
     self.assertEqual(27, imported.f(constant_op.constant(2)).numpy())
 
   def test_side_effect_listing(self):
-    class M(tracking.Checkpointable):
+    class M(tracking.AutoCheckpointable):
 
       def __init__(self):
         super(M, self).__init__()
@@ -334,7 +334,7 @@ class LoadTest(test.TestCase):
         lambda x: x*weight + bias,
         input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.weight = weight
     root.bias = bias
     root.g = g
@@ -346,16 +346,16 @@ class LoadTest(test.TestCase):
       self.assertAllClose(grad, [3.5, 1.0])
 
   def test_callable(self):
-    class M1(tracking.Checkpointable):
+    class M1(tracking.AutoCheckpointable):
 
       @def_function.function(
           input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
       def __call__(self, x):
         return x
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.m1 = M1()
-    root.m2 = tracking.Checkpointable()
+    root.m2 = tracking.AutoCheckpointable()
     root.m2.__call__ = def_function.function(
         input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])(
             lambda x: x*3.0)
@@ -378,9 +378,9 @@ class LoadTest(test.TestCase):
     func = def_function.function(
         input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])(
             lambda x: x*3.0)
-    root = tracking.Checkpointable()
-    root.__call__ = tracking.Checkpointable()
-    root.__call__.__call__ = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
+    root.__call__ = tracking.AutoCheckpointable()
+    root.__call__.__call__ = tracking.AutoCheckpointable()
     root.__call__.__call__.__call__ = func
 
     imported = self.cycle(root)
@@ -395,7 +395,7 @@ class LoadTest(test.TestCase):
     def func(x):
       return 2 * x
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.f = func
 
     self.assertAllEqual([2], root.f(constant_op.constant([1])).numpy())
@@ -419,7 +419,7 @@ class LoadTest(test.TestCase):
                         imported.f(constant_op.constant([1, 2, 3])).numpy())
 
   def test_dict(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.variables = dict(a=variables.Variable(1.))
     root.variables["b"] = variables.Variable(2.)
     root.variables["c"] = 1
@@ -429,7 +429,7 @@ class LoadTest(test.TestCase):
     self.assertEqual(set(["a", "b"]), set(imported.variables.keys()))
 
   def test_list(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.variables = [variables.Variable(1.)]
     root.variables.append(1)
     root.variables.append(variables.Variable(3.))
diff --git a/tensorflow/python/saved_model/revived_types_test.py b/tensorflow/python/saved_model/revived_types_test.py
index 98e10cfec0..ede5922b80 100644
--- a/tensorflow/python/saved_model/revived_types_test.py
+++ b/tensorflow/python/saved_model/revived_types_test.py
@@ -25,7 +25,7 @@ from tensorflow.python.saved_model import saved_object_graph_pb2
 from tensorflow.python.training.checkpointable import tracking
 
 
-class CustomTestClass(tracking.Checkpointable):
+class CustomTestClass(tracking.AutoCheckpointable):
 
   def __init__(self, version):
     self.version = version
@@ -56,7 +56,7 @@ revived_types.register_revived_type(
 class RegistrationMatchingTest(test.TestCase):
 
   def test_save_typecheck(self):
-    self.assertIs(revived_types.serialize(tracking.Checkpointable()), None)
+    self.assertIs(revived_types.serialize(tracking.AutoCheckpointable()), None)
 
   def test_load_identifier_not_found(self):
     nothing_matches = revived_types.deserialize(
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index 207dc2cb74..d383bbecfc 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -796,7 +796,7 @@ def save(obj, export_dir, signatures=None):
             "tf.enable_eager_execution() must run first when calling it from "
             "TensorFlow 1.x.")
   # pylint: enable=line-too-long
-  if not isinstance(obj, base.CheckpointableBase):
+  if not isinstance(obj, base.Checkpointable):
     raise ValueError(
         "Expected a Checkpointable object for export, got {}.".format(obj))
 
diff --git a/tensorflow/python/saved_model/save_test.py b/tensorflow/python/saved_model/save_test.py
index 533b954190..5f9dbe2c25 100644
--- a/tensorflow/python/saved_model/save_test.py
+++ b/tensorflow/python/saved_model/save_test.py
@@ -87,7 +87,7 @@ def _import_and_infer(
 class SaveTest(test.TestCase):
 
   def test_method_save_signature(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.f = def_function.function(
         lambda x: 2. * x,
         input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
@@ -99,7 +99,7 @@ class SaveTest(test.TestCase):
         _import_and_infer(save_dir, {"x": 1.}))
 
   def test_method_save_concrete(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.f = def_function.function(
         lambda z: {"out": 2. * z})
     root.f(constant_op.constant(1.))
@@ -115,7 +115,7 @@ class SaveTest(test.TestCase):
             save_dir, {"z": 1.}, signature_key="non_default_key"))
 
   def test_non_concrete_error(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.f = def_function.function(lambda x: 2. * x)
     root.f(constant_op.constant(1.))
     save_dir = os.path.join(self.get_temp_dir(), "saved_model")
@@ -124,7 +124,7 @@ class SaveTest(test.TestCase):
       save.save(root, save_dir, root.f)
 
   def test_nested_inputs(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.f = def_function.function(
         lambda x: 2. * x[0],
         input_signature=([tensor_spec.TensorSpec(None, dtypes.float32),
@@ -137,7 +137,7 @@ class SaveTest(test.TestCase):
       root.f.get_concrete_function()
 
   def test_nested_outputs(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.f = def_function.function(lambda x: (2. * x, (3. * x, 4. * x)))
     root.f(constant_op.constant(1.))
     to_save = root.f.get_concrete_function(constant_op.constant(1.))
@@ -158,7 +158,7 @@ class SaveTest(test.TestCase):
       save.save(root, save_dir, to_save)
 
   def test_variable(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.v1 = variables.Variable(3.)
     root.v2 = variables.Variable(2.)
     root.f = def_function.function(
@@ -186,7 +186,7 @@ class SaveTest(test.TestCase):
   def test_trivial_save_exception(self):
     save_dir = os.path.join(self.get_temp_dir(), "saved_model")
     with self.assertRaisesRegexp(ValueError, "signature"):
-      save.save(tracking.Checkpointable(), save_dir)
+      save.save(tracking.AutoCheckpointable(), save_dir)
 
   def test_single_method_default_signature(self):
     model = _ModelWithOptimizer()
@@ -200,7 +200,7 @@ class SaveTest(test.TestCase):
                                     {"x": [[3., 4.]], "y": [2.]}))
 
   def test_single_function_default_signature(self):
-    model = tracking.Checkpointable()
+    model = tracking.AutoCheckpointable()
     model.f = def_function.function(lambda: 3., input_signature=())
     model.f()
     save_dir = os.path.join(self.get_temp_dir(), "saved_model")
@@ -369,7 +369,7 @@ class AssetTests(test.TestCase):
         _import_and_infer(second_dir, {"keys": ["gamma", "beta"]}))
 
   def test_unused_asset(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.f = def_function.function(
         lambda x: 2. * x,
         input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
diff --git a/tensorflow/python/training/checkpointable/base.py b/tensorflow/python/training/checkpointable/base.py
index f2797bc5ff..8257693055 100644
--- a/tensorflow/python/training/checkpointable/base.py
+++ b/tensorflow/python/training/checkpointable/base.py
@@ -460,16 +460,16 @@ def no_automatic_dependency_tracking(method):
       target=method, decorator_func=_method_wrapper)
 
 
-class CheckpointableBase(object):
+class Checkpointable(object):
   """Base class for `Checkpointable` objects without automatic dependencies.
 
   This class has no __setattr__ override for performance reasons. Dependencies
   must be added explicitly. Unless attribute assignment is performance-critical,
-  use `Checkpointable` instead. Use `CheckpointableBase` for `isinstance`
+  use `AutoCheckpointable` instead. Use `Checkpointable` for `isinstance`
   checks.
   """
 
-  # CheckpointableBase does not do automatic dependency tracking, but uses the
+  # Checkpointable does not do automatic dependency tracking, but uses the
   # no_automatic_dependency_tracking decorator so it can avoid adding
   # dependencies if a subclass is Checkpointable / inherits from Model (both of
   # which have __setattr__ overrides).
@@ -623,7 +623,7 @@ class CheckpointableBase(object):
     # assign again. It will add this variable to our dependencies, and if there
     # is a non-trivial restoration queued, it will handle that. This also
     # handles slot variables.
-    if not overwrite or isinstance(new_variable, CheckpointableBase):
+    if not overwrite or isinstance(new_variable, Checkpointable):
       return self._track_checkpointable(new_variable, name=name,
                                         overwrite=overwrite)
     else:
@@ -695,7 +695,7 @@ class CheckpointableBase(object):
       ValueError: If another object is already tracked by this name.
     """
     self._maybe_initialize_checkpointable()
-    if not isinstance(checkpointable, CheckpointableBase):
+    if not isinstance(checkpointable, Checkpointable):
       raise TypeError(
           ("Checkpointable._track_checkpointable() passed type %s, not a "
            "Checkpointable.") % (type(checkpointable),))
@@ -742,7 +742,7 @@ class CheckpointableBase(object):
       name: The name of the dependency within this object (`self`), used to
         match `checkpointable` with values saved in a checkpoint.
       checkpointable: The Checkpointable object to restore (inheriting from
-        `CheckpointableBase`).
+        `Checkpointable`).
     """
     self._maybe_initialize_checkpointable()
     checkpointable._maybe_initialize_checkpointable()  # pylint: disable=protected-access
diff --git a/tensorflow/python/training/checkpointable/base_test.py b/tensorflow/python/training/checkpointable/base_test.py
index 736cbee488..750799f030 100644
--- a/tensorflow/python/training/checkpointable/base_test.py
+++ b/tensorflow/python/training/checkpointable/base_test.py
@@ -29,13 +29,13 @@ from tensorflow.python.training.checkpointable import util
 class InterfaceTests(test.TestCase):
 
   def testOverwrite(self):
-    root = base.CheckpointableBase()
-    leaf = base.CheckpointableBase()
+    root = base.Checkpointable()
+    leaf = base.Checkpointable()
     root._track_checkpointable(leaf, name="leaf")
     (current_name, current_dependency), = root._checkpoint_dependencies
     self.assertIs(leaf, current_dependency)
     self.assertEqual("leaf", current_name)
-    duplicate_name_dep = base.CheckpointableBase()
+    duplicate_name_dep = base.Checkpointable()
     with self.assertRaises(ValueError):
       root._track_checkpointable(duplicate_name_dep, name="leaf")
     root._track_checkpointable(duplicate_name_dep, name="leaf", overwrite=True)
@@ -44,7 +44,7 @@ class InterfaceTests(test.TestCase):
     self.assertEqual("leaf", current_name)
 
   def testAddVariableOverwrite(self):
-    root = base.CheckpointableBase()
+    root = base.Checkpointable()
     a = root._add_variable_with_custom_getter(
         name="v", shape=[], getter=variable_scope.get_variable)
     self.assertEqual([root, a], util.list_objects(root))
@@ -61,15 +61,15 @@ class InterfaceTests(test.TestCase):
             getter=variable_scope.get_variable)
 
   def testAssertConsumedWithUnusedPythonState(self):
-    has_config = base.CheckpointableBase()
+    has_config = base.Checkpointable()
     has_config.get_config = lambda: {}
     saved = util.Checkpoint(obj=has_config)
     save_path = saved.save(os.path.join(self.get_temp_dir(), "ckpt"))
-    restored = util.Checkpoint(obj=base.CheckpointableBase())
+    restored = util.Checkpoint(obj=base.Checkpointable())
     restored.restore(save_path).assert_consumed()
 
   def testAssertConsumedFailsWithUsedPythonState(self):
-    has_config = base.CheckpointableBase()
+    has_config = base.Checkpointable()
     attributes = {
         "foo_attr": functools.partial(
             base.PythonStringStateSaveable,
@@ -78,7 +78,7 @@ class InterfaceTests(test.TestCase):
     has_config._gather_saveables_for_checkpoint = lambda: attributes
     saved = util.Checkpoint(obj=has_config)
     save_path = saved.save(os.path.join(self.get_temp_dir(), "ckpt"))
-    restored = util.Checkpoint(obj=base.CheckpointableBase())
+    restored = util.Checkpoint(obj=base.Checkpointable())
     status = restored.restore(save_path)
     with self.assertRaisesRegexp(AssertionError, "foo_attr"):
       status.assert_consumed()
diff --git a/tensorflow/python/training/checkpointable/data_structures.py b/tensorflow/python/training/checkpointable/data_structures.py
index d1e617ed46..5a5b444a6c 100644
--- a/tensorflow/python/training/checkpointable/data_structures.py
+++ b/tensorflow/python/training/checkpointable/data_structures.py
@@ -58,7 +58,7 @@ def _wrap_or_unwrap(value):
   """Wraps basic data structures, unwraps NoDependency objects."""
   if isinstance(value, NoDependency):
     return value.value
-  if isinstance(value, base.CheckpointableBase):
+  if isinstance(value, base.Checkpointable):
     return value  # Skip conversion for already checkpointable objects.
   elif isinstance(value, dict):
     return _DictWrapper(value)
@@ -99,7 +99,7 @@ def sticky_attribute_assignment(checkpointable, name, value):
   value = _wrap_or_unwrap(value)
   if not add_dependency:
     return value
-  if isinstance(value, base.CheckpointableBase):
+  if isinstance(value, base.Checkpointable):
     checkpointable._track_checkpointable(  # pylint: disable=protected-access
         value, name=name,
         # Allow the user to switch the Checkpointable which is tracked by this
@@ -109,7 +109,7 @@ def sticky_attribute_assignment(checkpointable, name, value):
   return value
 
 
-class CheckpointableDataStructure(base.CheckpointableBase):
+class CheckpointableDataStructure(base.Checkpointable):
   """Base class for data structures which contain checkpointable objects."""
 
   def __init__(self):
@@ -122,11 +122,11 @@ class CheckpointableDataStructure(base.CheckpointableBase):
         checkpointable=self, value=value, name=name)
     if isinstance(value, variables.Variable):
       self._extra_variables.append(value)
-    if not isinstance(value, base.CheckpointableBase):
+    if not isinstance(value, base.Checkpointable):
       raise ValueError(
           ("Only checkpointable objects (such as Layers or Optimizers) may be "
            "stored in a List object. Got %s, which does not inherit from "
-           "CheckpointableBase.") % (value,))
+           "Checkpointable.") % (value,))
     if hasattr(value, "_use_resource_variables"):
       # In subclassed models, legacy layers (tf.layers) must always use
       # resource variables.
@@ -410,7 +410,7 @@ class _ListWrapper(List, collections.MutableSequence,
 
   def __setitem__(self, key, value):
     self._check_external_modification()
-    if isinstance(self._storage[key], base.CheckpointableBase):
+    if isinstance(self._storage[key], base.Checkpointable):
       self._non_append_mutation = True
     self._storage[key] = self._track_value(value, self._name_element(key))
     self._update_snapshot()
@@ -693,14 +693,14 @@ class _DictWrapper(Mapping, collections.MutableMapping):
     else:
       value = _wrap_or_unwrap(value)
       existing_dependency = None
-      if not no_dep and isinstance(value, base.CheckpointableBase):
+      if not no_dep and isinstance(value, base.Checkpointable):
         # Non-string keys are OK as long as we have no reason to add a
         # dependency on the value (either because the value is not
         # checkpointable, or because it was wrapped in a NoDependency object).
         self._non_string_key = True
     current_value = self._storage.setdefault(key, value)
     if current_value is not value:
-      if ((not no_dep and isinstance(value, base.CheckpointableBase))
+      if ((not no_dep and isinstance(value, base.Checkpointable))
           # We don't want to just check that the existing object is
           # checkpointable, since it may have been wrapped in a NoDependency
           # object.
@@ -716,7 +716,7 @@ class _DictWrapper(Mapping, collections.MutableMapping):
   def __delitem__(self, key):
     self._check_external_modification()
     existing_value = self[key]
-    if isinstance(existing_value, base.CheckpointableBase):
+    if isinstance(existing_value, base.Checkpointable):
       # Deleting tracked checkpointable values means restoring is problematic,
       # so we'll throw an exception on save.
       self._non_append_mutation = True
diff --git a/tensorflow/python/training/checkpointable/data_structures_test.py b/tensorflow/python/training/checkpointable/data_structures_test.py
index 79191f3448..53cbd66482 100644
--- a/tensorflow/python/training/checkpointable/data_structures_test.py
+++ b/tensorflow/python/training/checkpointable/data_structures_test.py
@@ -210,8 +210,8 @@ class ListTests(test.TestCase):
   def testListWrapperBasic(self):
     # _ListWrapper, unlike List, compares like the built-in list type (since it
     # is used to automatically replace lists).
-    a = tracking.Checkpointable()
-    b = tracking.Checkpointable()
+    a = tracking.AutoCheckpointable()
+    b = tracking.AutoCheckpointable()
     self.assertEqual([a, a],
                      [a, a])
     self.assertEqual(data_structures._ListWrapper([a, a]),
@@ -343,7 +343,7 @@ class MappingTests(test.TestCase):
 
   def testLayerCollectionWithExternalMutation(self):
     d = {}
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.wrapper = d
     self.assertEqual([], root.wrapper.layers)
     self.assertEqual([], root.wrapper.trainable_weights)
@@ -361,7 +361,7 @@ class MappingTests(test.TestCase):
     self.assertEqual(2, len(has_mappings))
     self.assertNotIn(data_structures.Mapping(), has_mappings)
     # In contrast to Mapping, dict wrappers are not hashable
-    a = tracking.Checkpointable()
+    a = tracking.AutoCheckpointable()
     a.d = {}
     self.assertEqual({}, a.d)
     self.assertFalse({} != a.d)  # pylint: disable=g-explicit-bool-comparison
@@ -370,7 +370,7 @@ class MappingTests(test.TestCase):
       set([a.d])
 
   def testDictWrapperBadKeys(self):
-    a = tracking.Checkpointable()
+    a = tracking.AutoCheckpointable()
     a.d = {}
     a.d[1] = data_structures.List()
     model = training.Model()
@@ -380,7 +380,7 @@ class MappingTests(test.TestCase):
       model.save_weights(save_path)
 
   def testDictWrapperNoDependency(self):
-    a = tracking.Checkpointable()
+    a = tracking.AutoCheckpointable()
     a.d = data_structures.NoDependency({})
     a.d[1] = [3]
     self.assertEqual([a], util.list_objects(a))
@@ -391,7 +391,7 @@ class MappingTests(test.TestCase):
     model.load_weights(save_path)
 
   def testNonStringKeyNotCheckpointableValue(self):
-    a = tracking.Checkpointable()
+    a = tracking.AutoCheckpointable()
     a.d = {}
     a.d["a"] = [3]
     a.d[1] = data_structures.NoDependency([3])
@@ -405,15 +405,15 @@ class MappingTests(test.TestCase):
   def testNonAppendNotCheckpointable(self):
     # Non-append mutations (deleting or overwriting values) are OK when the
     # values aren't tracked.
-    a = tracking.Checkpointable()
+    a = tracking.AutoCheckpointable()
     a.d = {}
     a.d["a"] = [3]
     a.d[1] = 3
     a.d[1] = 2
     self.assertEqual(2, a.d[1])
     del a.d[1]
-    a.d[2] = data_structures.NoDependency(tracking.Checkpointable())
-    second = tracking.Checkpointable()
+    a.d[2] = data_structures.NoDependency(tracking.AutoCheckpointable())
+    second = tracking.AutoCheckpointable()
     a.d[2] = data_structures.NoDependency(second)
     self.assertIs(second, a.d[2])
     self.assertEqual([a, a.d, a.d["a"]], util.list_objects(a))
@@ -475,7 +475,7 @@ class MappingTests(test.TestCase):
     self.assertEqual({1: 3}, new_dict)
 
   def testListShallowCopy(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     orig_list = [[1.]]
     root.a = orig_list
     copied = copy.copy(root.a)
@@ -492,7 +492,7 @@ class MappingTests(test.TestCase):
       util.list_objects(copy.copy(root.a))
 
   def testListDeepCopy(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     orig_list = [[1.]]
     root.a = orig_list
     copied = copy.deepcopy(root.a)
@@ -509,7 +509,7 @@ class MappingTests(test.TestCase):
       util.list_objects(copy.deepcopy(root.a))
 
   def testDictShallowCopy(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     orig_dict = {"a": [1.]}
     root.a = orig_dict
     copied = copy.copy(root.a)
@@ -526,7 +526,7 @@ class MappingTests(test.TestCase):
       util.list_objects(copy.copy(root.a))
 
   def testDictDeepCopy(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     orig_dict = {"a": [1.]}
     root.a = orig_dict
     copied = copy.deepcopy(root.a)
@@ -543,8 +543,8 @@ class MappingTests(test.TestCase):
       util.list_objects(copy.deepcopy(root.a))
 
   def testShallowCopyCheckpointable(self):
-    original = tracking.Checkpointable()
-    original_sub = tracking.Checkpointable()
+    original = tracking.AutoCheckpointable()
+    original_sub = tracking.AutoCheckpointable()
     original.a = [[1.]]
     original.b = {"a": original_sub}
     shallow_copied = copy.copy(original)
@@ -557,15 +557,15 @@ class MappingTests(test.TestCase):
     self.assertIn(shallow_copied.b["a"], shallow_deps)
 
   def testDeepCopyCheckpointable(self):
-    original = tracking.Checkpointable()
-    original_sub = tracking.Checkpointable()
+    original = tracking.AutoCheckpointable()
+    original_sub = tracking.AutoCheckpointable()
     original.a = [[1.]]
     original.b = {"a": original_sub}
     deep_copied = copy.deepcopy(original)
     self.assertIsNot(original, deep_copied)
     self.assertIsNot(original_sub, deep_copied.b["a"])
     self.assertEqual([[1.]], deep_copied.a)
-    self.assertIsInstance(deep_copied.b["a"], tracking.Checkpointable)
+    self.assertIsInstance(deep_copied.b["a"], tracking.AutoCheckpointable)
     deps = util.list_objects(deep_copied)
     self.assertIn(deep_copied.a, deps)
     self.assertIn(deep_copied.b, deps)
diff --git a/tensorflow/python/training/checkpointable/tracking.py b/tensorflow/python/training/checkpointable/tracking.py
index 4e96aee0c5..04fd5547e1 100644
--- a/tensorflow/python/training/checkpointable/tracking.py
+++ b/tensorflow/python/training/checkpointable/tracking.py
@@ -41,7 +41,7 @@ class NotCheckpointable(object):
   pass
 
 
-class Checkpointable(base.CheckpointableBase):
+class AutoCheckpointable(base.Checkpointable):
   """Manages dependencies on other objects.
 
   `Checkpointable` objects may have dependencies: other `Checkpointable` objects
@@ -74,7 +74,7 @@ class Checkpointable(base.CheckpointableBase):
     if getattr(self, "_setattr_tracking", True):
       value = data_structures.sticky_attribute_assignment(
           checkpointable=self, value=value, name=name)
-    super(Checkpointable, self).__setattr__(name, value)
+    super(AutoCheckpointable, self).__setattr__(name, value)
 
   def _no_dependency(self, value):
     """Override to allow CheckpointableBase to disable dependency tracking."""
@@ -124,7 +124,7 @@ def resource_tracker_scope(resource_tracker):
     _RESOURCE_TRACKER_STACK = old
 
 
-class TrackableResource(base.CheckpointableBase):
+class TrackableResource(base.Checkpointable):
   """Base class for all resources that need to be tracked."""
 
   def __init__(self):
@@ -151,7 +151,7 @@ class TrackableResource(base.CheckpointableBase):
     return self._resource_handle
 
 
-class TrackableAsset(base.CheckpointableBase):
+class TrackableAsset(base.Checkpointable):
   """Base class for asset files which need to be tracked."""
 
   def __init__(self, path):
diff --git a/tensorflow/python/training/checkpointable/tracking_test.py b/tensorflow/python/training/checkpointable/tracking_test.py
index 17c5461bc2..eb70919b9c 100644
--- a/tensorflow/python/training/checkpointable/tracking_test.py
+++ b/tensorflow/python/training/checkpointable/tracking_test.py
@@ -35,10 +35,10 @@ from tensorflow.python.util import nest
 class InterfaceTests(test.TestCase):
 
   def testMultipleAssignment(self):
-    root = tracking.Checkpointable()
-    root.leaf = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
+    root.leaf = tracking.AutoCheckpointable()
     root.leaf = root.leaf
-    duplicate_name_dep = tracking.Checkpointable()
+    duplicate_name_dep = tracking.AutoCheckpointable()
     with self.assertRaisesRegexp(ValueError, "already declared"):
       root._track_checkpointable(duplicate_name_dep, name="leaf")
     # No error; we're overriding __setattr__, so we can't really stop people
@@ -50,10 +50,10 @@ class InterfaceTests(test.TestCase):
     self.assertIs(duplicate_name_dep, dep_object)
 
   def testNoDependency(self):
-    root = tracking.Checkpointable()
-    hasdep = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
+    hasdep = tracking.AutoCheckpointable()
     root.hasdep = hasdep
-    nodep = tracking.Checkpointable()
+    nodep = tracking.AutoCheckpointable()
     root.nodep = data_structures.NoDependency(nodep)
     self.assertEqual(1, len(root._checkpoint_dependencies))
     self.assertIs(root._checkpoint_dependencies[0].ref, root.hasdep)
@@ -66,16 +66,16 @@ class InterfaceTests(test.TestCase):
       def __init__(self):
         super(NoDependencyModel, self).__init__()
         self.a = []
-        self.b = tracking.Checkpointable()
+        self.b = tracking.AutoCheckpointable()
 
     nodeps = NoDependencyModel()
     self.assertEqual([nodeps], util.list_objects(nodeps))
 
   def testListBasic(self):
-    a = tracking.Checkpointable()
-    b = tracking.Checkpointable()
+    a = tracking.AutoCheckpointable()
+    b = tracking.AutoCheckpointable()
     a.l = [b]
-    c = tracking.Checkpointable()
+    c = tracking.AutoCheckpointable()
     a.l.append(c)
     a_deps = util.list_objects(a)
     self.assertIn(b, a_deps)
@@ -87,10 +87,10 @@ class InterfaceTests(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
   def testMutationDirtiesList(self):
-    a = tracking.Checkpointable()
-    b = tracking.Checkpointable()
+    a = tracking.AutoCheckpointable()
+    b = tracking.AutoCheckpointable()
     a.l = [b]
-    c = tracking.Checkpointable()
+    c = tracking.AutoCheckpointable()
     a.l.insert(0, c)
     checkpoint = util.Checkpoint(a=a)
     with self.assertRaisesRegexp(ValueError, "A list element was replaced"):
@@ -98,11 +98,11 @@ class InterfaceTests(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
   def testOutOfBandEditDirtiesList(self):
-    a = tracking.Checkpointable()
-    b = tracking.Checkpointable()
+    a = tracking.AutoCheckpointable()
+    b = tracking.AutoCheckpointable()
     held_reference = [b]
     a.l = held_reference
-    c = tracking.Checkpointable()
+    c = tracking.AutoCheckpointable()
     held_reference.append(c)
     checkpoint = util.Checkpoint(a=a)
     with self.assertRaisesRegexp(ValueError, "The wrapped list was modified"):
@@ -110,25 +110,25 @@ class InterfaceTests(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
   def testNestedLists(self):
-    a = tracking.Checkpointable()
+    a = tracking.AutoCheckpointable()
     a.l = []
-    b = tracking.Checkpointable()
+    b = tracking.AutoCheckpointable()
     a.l.append([b])
-    c = tracking.Checkpointable()
+    c = tracking.AutoCheckpointable()
     a.l[0].append(c)
     a_deps = util.list_objects(a)
     self.assertIn(b, a_deps)
     self.assertIn(c, a_deps)
     a.l[0].append(1)
-    d = tracking.Checkpointable()
+    d = tracking.AutoCheckpointable()
     a.l[0].append(d)
     a_deps = util.list_objects(a)
     self.assertIn(d, a_deps)
     self.assertIn(b, a_deps)
     self.assertIn(c, a_deps)
     self.assertNotIn(1, a_deps)
-    e = tracking.Checkpointable()
-    f = tracking.Checkpointable()
+    e = tracking.AutoCheckpointable()
+    f = tracking.AutoCheckpointable()
     a.l1 = [[], [e]]
     a.l1[0].append(f)
     a_deps = util.list_objects(a)
@@ -183,7 +183,7 @@ class InterfaceTests(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
   def testAssertions(self):
-    a = tracking.Checkpointable()
+    a = tracking.AutoCheckpointable()
     a.l = {"k": [numpy.zeros([2, 2])]}
     self.assertAllEqual(nest.flatten({"k": [numpy.zeros([2, 2])]}),
                         nest.flatten(a.l))
diff --git a/tensorflow/python/training/checkpointable/util.py b/tensorflow/python/training/checkpointable/util.py
index 1ff0cd9f25..5f96ce0db8 100644
--- a/tensorflow/python/training/checkpointable/util.py
+++ b/tensorflow/python/training/checkpointable/util.py
@@ -1655,7 +1655,7 @@ def frozen_saver(root_checkpointable):
 
 
 @tf_export("train.Checkpoint")
-class Checkpoint(tracking.Checkpointable):
+class Checkpoint(tracking.AutoCheckpointable):
   """Groups checkpointable objects, saving and restoring them.
 
   `Checkpoint`'s constructor accepts keyword arguments whose values are types
@@ -1757,7 +1757,7 @@ class Checkpoint(tracking.Checkpointable):
     """
     super(Checkpoint, self).__init__()
     for k, v in sorted(kwargs.items(), key=lambda item: item[0]):
-      if not isinstance(v, (base.CheckpointableBase,
+      if not isinstance(v, (base.Checkpointable,
                             def_function.PolymorphicFunction)):
         raise ValueError(
             ("`Checkpoint` was expecting a checkpointable object (an object "
diff --git a/tensorflow/python/training/checkpointable/util_test.py b/tensorflow/python/training/checkpointable/util_test.py
index 78a8282f25..cef1075e93 100644
--- a/tensorflow/python/training/checkpointable/util_test.py
+++ b/tensorflow/python/training/checkpointable/util_test.py
@@ -51,7 +51,7 @@ from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.training.checkpointable import util as checkpointable_utils
 
 
-class NonLayerCheckpointable(tracking.Checkpointable):
+class NonLayerCheckpointable(tracking.AutoCheckpointable):
 
   def __init__(self):
     super(NonLayerCheckpointable, self).__init__()
@@ -139,7 +139,7 @@ class InterfaceTests(test.TestCase):
 
   def testInitNotCalled(self):
 
-    class NoInit(tracking.Checkpointable):
+    class NoInit(tracking.AutoCheckpointable):
 
       def __init__(self):
         pass
@@ -148,7 +148,7 @@ class InterfaceTests(test.TestCase):
     checkpointable_utils.add_variable(NoInit(), "var", shape=[])
 
   def testShapeDtype(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     v1 = checkpointable_utils.add_variable(
         root, name="v1", initializer=3., dtype=dtypes.float64)
     self.assertEqual(dtypes.float64, v1.dtype)
@@ -180,7 +180,7 @@ class InterfaceTests(test.TestCase):
   def testNotCheckpointable(self):
 
     class CallsFunctionalStuff(
-        tracking.NotCheckpointable, tracking.Checkpointable):
+        tracking.NotCheckpointable, tracking.AutoCheckpointable):
       pass
 
     test_dir = self.get_temp_dir()
@@ -190,7 +190,7 @@ class InterfaceTests(test.TestCase):
       checkpoint.save(prefix)
 
     class CallsFunctionalStuffOtherMRO(
-        tracking.Checkpointable, tracking.NotCheckpointable):
+        tracking.AutoCheckpointable, tracking.NotCheckpointable):
       pass
 
     checkpoint_reversed = checkpointable_utils.Checkpoint(
@@ -220,7 +220,7 @@ class _MirroringSaveable(saver_lib.BaseSaverBuilder.SaveableObject):
         self._mirrored_variable.assign(tensor))
 
 
-class _OwnsMirroredVariables(base.CheckpointableBase):
+class _OwnsMirroredVariables(base.Checkpointable):
   """A Checkpointable object which returns a more complex SaveableObject."""
 
   def __init__(self):
@@ -653,7 +653,7 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
   # pylint: enable=cell-var-from-loop
 
   def _get_checkpoint_name(self, name):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     checkpointable_utils.add_variable(
         root, name=name, shape=[1, 2], dtype=dtypes.float64)
     (named_variable,), _, _ = checkpointable_utils._serialize_object_graph(
@@ -674,8 +674,8 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True)
   def testNumberedPath(self):
-    root = tracking.Checkpointable()
-    leaf = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
+    leaf = tracking.AutoCheckpointable()
     root.leaf = leaf
     checkpointable_utils.add_variable(leaf, name="v", shape=[])
     (named_variable,), _, _ = checkpointable_utils._serialize_object_graph(
@@ -684,8 +684,8 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
   def testLocalNameValidation(self):
-    root = tracking.Checkpointable()
-    leaf = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
+    leaf = tracking.AutoCheckpointable()
     # Dots are escaped, which avoids conflicts with reserved names.
     root._track_checkpointable(leaf, name=".ATTRIBUTES")
     checkpointable_utils.add_variable(checkpointable=leaf, name="a", shape=[])
@@ -726,13 +726,13 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
   @test_util.run_in_graph_and_eager_modes
   def testLateDependencyTracking(self):
 
-    class Dependency(tracking.Checkpointable):
+    class Dependency(tracking.AutoCheckpointable):
 
       def build(self):
         self.var = checkpointable_utils.add_variable(
             self, "var", initializer=0.)
 
-    class LateDependencies(tracking.Checkpointable):
+    class LateDependencies(tracking.AutoCheckpointable):
 
       def add_dep(self):
         self.dep = Dependency()
@@ -759,13 +759,13 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
   @test_util.run_in_graph_and_eager_modes
   def testDepAfterVar(self):
 
-    class Dependency(tracking.Checkpointable):
+    class Dependency(tracking.AutoCheckpointable):
 
       def build(self):
         self.var = checkpointable_utils.add_variable(
             self, "var", initializer=0.)
 
-    class DepAfterVar(tracking.Checkpointable):
+    class DepAfterVar(tracking.AutoCheckpointable):
 
       def add_dep(self):
         dep = Dependency()
@@ -792,7 +792,7 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
   def testDeferredSlotRestoration(self):
     checkpoint_directory = self.get_temp_dir()
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.var = checkpointable_utils.add_variable(
         root, name="var", initializer=0.)
     optimizer = adam.Adam(0.1)
@@ -815,7 +815,7 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
         14.))
     slots_path = checkpointable_utils.CheckpointableSaver(root).save(
         os.path.join(checkpoint_directory, "with_slots"))
-    new_root = tracking.Checkpointable()
+    new_root = tracking.AutoCheckpointable()
     # Load the slot-containing checkpoint (deferred), then immediately overwrite
     # the non-slot variable (also deferred).
     slot_status = checkpointable_utils.CheckpointableSaver(
@@ -861,8 +861,8 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
   @test_util.run_in_graph_and_eager_modes
   def testOverlappingRestores(self):
     checkpoint_directory = self.get_temp_dir()
-    save_root = tracking.Checkpointable()
-    save_root.dep = tracking.Checkpointable()
+    save_root = tracking.AutoCheckpointable()
+    save_root.dep = tracking.AutoCheckpointable()
     save_root.dep.var = checkpointable_utils.add_variable(
         save_root.dep, name="var", initializer=0.)
     self.evaluate(state_ops.assign(save_root.dep.var, 12.))
@@ -871,13 +871,13 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
     self.evaluate(state_ops.assign(save_root.dep.var, 13.))
     second_path = saver.save(os.path.join(checkpoint_directory, "second"))
 
-    first_root = tracking.Checkpointable()
-    second_root = tracking.Checkpointable()
+    first_root = tracking.AutoCheckpointable()
+    second_root = tracking.AutoCheckpointable()
     first_status = checkpointable_utils.CheckpointableSaver(
         first_root).restore(first_path)
     second_status = checkpointable_utils.CheckpointableSaver(
         second_root).restore(second_path)
-    load_dep = tracking.Checkpointable()
+    load_dep = tracking.AutoCheckpointable()
     load_dep.var = checkpointable_utils.add_variable(
         load_dep, name="var", shape=[])
     first_root.dep = load_dep
@@ -891,13 +891,13 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
 
     # Try again with the order of the restore() reversed. The last restore
     # determines the final value.
-    first_root = tracking.Checkpointable()
-    second_root = tracking.Checkpointable()
+    first_root = tracking.AutoCheckpointable()
+    second_root = tracking.AutoCheckpointable()
     second_status = checkpointable_utils.CheckpointableSaver(
         second_root).restore(second_path)
     first_status = checkpointable_utils.CheckpointableSaver(
         first_root).restore(first_path)
-    load_dep = tracking.Checkpointable()
+    load_dep = tracking.AutoCheckpointable()
     load_dep.var = checkpointable_utils.add_variable(
         load_dep, name="var", shape=[])
     first_root.dep = load_dep
@@ -913,23 +913,23 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
   def testAmbiguousLoad(self):
     # Not OK to split one checkpoint object into two
     checkpoint_directory = self.get_temp_dir()
-    save_root = tracking.Checkpointable()
-    save_root.dep_one = tracking.Checkpointable()
-    save_root.dep_two = tracking.Checkpointable()
-    dep_three = tracking.Checkpointable()
+    save_root = tracking.AutoCheckpointable()
+    save_root.dep_one = tracking.AutoCheckpointable()
+    save_root.dep_two = tracking.AutoCheckpointable()
+    dep_three = tracking.AutoCheckpointable()
     save_root.dep_one.dep_three = dep_three
     save_root.dep_two.dep_three = dep_three
     checkpointable_utils.add_variable(dep_three, name="var", initializer=0.)
     self.evaluate(checkpointable_utils.gather_initializers(save_root))
     save_path = checkpointable_utils.CheckpointableSaver(save_root).save(
         os.path.join(checkpoint_directory, "ckpt"))
-    load_root = tracking.Checkpointable()
+    load_root = tracking.AutoCheckpointable()
     status = checkpointable_utils.CheckpointableSaver(load_root).restore(
         save_path)
-    load_root.dep_one = tracking.Checkpointable()
-    load_root.dep_two = tracking.Checkpointable()
-    load_root.dep_one.dep_three = tracking.Checkpointable()
-    load_root.dep_two.dep_three = tracking.Checkpointable()
+    load_root.dep_one = tracking.AutoCheckpointable()
+    load_root.dep_two = tracking.AutoCheckpointable()
+    load_root.dep_one.dep_three = tracking.AutoCheckpointable()
+    load_root.dep_two.dep_three = tracking.AutoCheckpointable()
     checkpointable_utils.add_variable(
         load_root.dep_one.dep_three, name="var", initializer=0.)
     with self.assertRaises(AssertionError):
@@ -941,9 +941,9 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
   def testObjectsCombined(self):
     # Currently fine to load two checkpoint objects into one Python object
     checkpoint_directory = self.get_temp_dir()
-    save_root = tracking.Checkpointable()
-    save_root.dep_one = tracking.Checkpointable()
-    save_root.dep_two = tracking.Checkpointable()
+    save_root = tracking.AutoCheckpointable()
+    save_root.dep_one = tracking.AutoCheckpointable()
+    save_root.dep_two = tracking.AutoCheckpointable()
     checkpointable_utils.add_variable(
         save_root.dep_one, name="var1", initializer=32., dtype=dtypes.float64)
     checkpointable_utils.add_variable(
@@ -951,8 +951,8 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
     self.evaluate(checkpointable_utils.gather_initializers(save_root))
     save_path = checkpointable_utils.CheckpointableSaver(save_root).save(
         os.path.join(checkpoint_directory, "ckpt"))
-    load_root = tracking.Checkpointable()
-    load_root.dep_one = tracking.Checkpointable()
+    load_root = tracking.AutoCheckpointable()
+    load_root.dep_one = tracking.AutoCheckpointable()
     load_root.dep_two = load_root.dep_one
     v1 = checkpointable_utils.add_variable(
         load_root.dep_one, name="var1", shape=[], dtype=dtypes.float64)
@@ -968,8 +968,8 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
   def testDependencyLoop(self):
     # Note: this test creates garbage during eager execution because it
     # purposefully creates a reference cycle.
-    first = tracking.Checkpointable()
-    second = tracking.Checkpointable()
+    first = tracking.AutoCheckpointable()
+    second = tracking.AutoCheckpointable()
     first.second = second
     second.first = first
     first.v = checkpointable_utils.add_variable(
@@ -982,10 +982,10 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
         os.path.join(checkpoint_directory, "ckpt"))
 
     # Test deferred loading
-    first_load = tracking.Checkpointable()
+    first_load = tracking.AutoCheckpointable()
     status = checkpointable_utils.CheckpointableSaver(
         first_load).restore(save_path)
-    second_load = tracking.Checkpointable()
+    second_load = tracking.AutoCheckpointable()
     first_load.second = second_load
     second_load.first = first_load
     with self.assertRaises(AssertionError):
@@ -1014,7 +1014,7 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
   def testRestoreOnAssign(self):
     checkpoint_directory = self.get_temp_dir()
     checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-    first = tracking.Checkpointable()
+    first = tracking.AutoCheckpointable()
     first.var1 = variables_lib.Variable(0., name="outside_var")
     first.var2 = variables_lib.Variable(0., name="blah")
     self.evaluate(first.var1.assign(4.))
@@ -1022,7 +1022,7 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
     save_path = checkpointable_utils.CheckpointableSaver(first).save(
         checkpoint_prefix)
 
-    second = tracking.Checkpointable()
+    second = tracking.AutoCheckpointable()
     second.var2 = variables_lib.Variable(0., name="blah")
     status = checkpointable_utils.CheckpointableSaver(
         second).restore(save_path)
@@ -1042,7 +1042,7 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
       with graph.as_default(), self.session(graph):
         checkpoint_directory = self.get_temp_dir()
         checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-        obj = tracking.Checkpointable()
+        obj = tracking.AutoCheckpointable()
         obj.var = variables_lib.Variable(0., name="v")
         obj.opt = adam.Adam(0.1)
         variables = [obj.var]
@@ -1059,7 +1059,7 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
     # No checkpoints are deleted by default
     checkpoint_directory = self.get_temp_dir()
     checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-    obj = tracking.Checkpointable()
+    obj = tracking.AutoCheckpointable()
     obj.var = variable_scope.get_variable(name="v", initializer=0.)
     self.evaluate(checkpointable_utils.gather_initializers(obj))
     saver = checkpointable_utils.Checkpoint(obj=obj)
@@ -1079,7 +1079,7 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
   def testCheckpointStateChangingVarList(self):
     checkpoint_directory = self.get_temp_dir()
     checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-    obj = tracking.Checkpointable()
+    obj = tracking.AutoCheckpointable()
     obj.var = variable_scope.get_variable(name="v", initializer=0.)
     self.evaluate(checkpointable_utils.gather_initializers(obj))
     checkpoint = checkpointable_utils.Checkpoint(obj=obj)
@@ -1132,7 +1132,7 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
       with graph.as_default(), self.session(graph):
         checkpoint_directory = self.get_temp_dir()
         checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-        obj = tracking.Checkpointable()
+        obj = tracking.AutoCheckpointable()
         obj.var = variables_lib.Variable(0., name="v")
         obj.opt = adam.Adam(0.1)
         variables = [obj.var]
@@ -1286,7 +1286,7 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
     load_status.assert_existing_objects_matched().run_restore_ops()
 
 
-class _ManualScope(tracking.Checkpointable):
+class _ManualScope(tracking.AutoCheckpointable):
 
   def __call__(self):
     with variable_scope.variable_scope("ManualScope") as vs:
diff --git a/tensorflow/python/training/checkpointable/util_with_v1_optimizers_test.py b/tensorflow/python/training/checkpointable/util_with_v1_optimizers_test.py
index bb6bb8bd1d..bd80fa60f0 100644
--- a/tensorflow/python/training/checkpointable/util_with_v1_optimizers_test.py
+++ b/tensorflow/python/training/checkpointable/util_with_v1_optimizers_test.py
@@ -47,7 +47,7 @@ from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.training.checkpointable import util as checkpointable_utils
 
 
-class NonLayerCheckpointable(tracking.Checkpointable):
+class NonLayerCheckpointable(tracking.AutoCheckpointable):
 
   def __init__(self):
     super(NonLayerCheckpointable, self).__init__()
@@ -461,7 +461,7 @@ class CheckpointingTests(test.TestCase):
   # pylint: enable=cell-var-from-loop
 
   def _get_checkpoint_name(self, name):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     checkpointable_utils.add_variable(
         root, name=name, shape=[1, 2], dtype=dtypes.float64)
     (named_variable,), _, _ = checkpointable_utils._serialize_object_graph(
@@ -503,7 +503,7 @@ class CheckpointingTests(test.TestCase):
   def testDeferredSlotRestoration(self):
     checkpoint_directory = self.get_temp_dir()
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.var = checkpointable_utils.add_variable(
         root, name="var", initializer=0.)
     optimizer = adam.AdamOptimizer(0.1)
@@ -526,7 +526,7 @@ class CheckpointingTests(test.TestCase):
                                    14.))
     slots_path = checkpointable_utils.CheckpointableSaver(root).save(
         os.path.join(checkpoint_directory, "with_slots"))
-    new_root = tracking.Checkpointable()
+    new_root = tracking.AutoCheckpointable()
     # Load the slot-containing checkpoint (deferred), then immediately overwrite
     # the non-slot variable (also deferred).
     slot_status = checkpointable_utils.CheckpointableSaver(
@@ -572,7 +572,7 @@ class CheckpointingTests(test.TestCase):
       with graph.as_default(), self.session(graph):
         checkpoint_directory = self.get_temp_dir()
         checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-        obj = tracking.Checkpointable()
+        obj = tracking.AutoCheckpointable()
         obj.var = variable_scope.get_variable(name="v", initializer=0.)
         obj.opt = adam.AdamOptimizer(0.1)
         obj.opt.minimize(obj.var.read_value())
@@ -590,7 +590,7 @@ class CheckpointingTests(test.TestCase):
       with graph.as_default(), self.session(graph):
         checkpoint_directory = self.get_temp_dir()
         checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-        obj = tracking.Checkpointable()
+        obj = tracking.AutoCheckpointable()
         obj.var = variable_scope.get_variable(name="v", initializer=0.)
         obj.opt = adam.AdamOptimizer(0.1)
         obj.opt.minimize(obj.var.read_value())
@@ -739,7 +739,7 @@ class CheckpointingTests(test.TestCase):
       self.assertEqual(42., self.evaluate(optimizer.variables()[0]))
 
 
-class _ManualScope(tracking.Checkpointable):
+class _ManualScope(tracking.AutoCheckpointable):
 
   def __call__(self):
     with variable_scope.variable_scope("ManualScope") as vs:
diff --git a/tensorflow/python/training/checkpointable/util_xla_test.py b/tensorflow/python/training/checkpointable/util_xla_test.py
index d80ac003b4..4e96a7514a 100644
--- a/tensorflow/python/training/checkpointable/util_xla_test.py
+++ b/tensorflow/python/training/checkpointable/util_xla_test.py
@@ -29,7 +29,7 @@ from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.training.checkpointable import util as checkpointable_utils
 
 
-class NonLayerCheckpointable(tracking.Checkpointable):
+class NonLayerCheckpointable(tracking.AutoCheckpointable):
 
   def __init__(self):
     super(NonLayerCheckpointable, self).__init__()
diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py
index a591dc9fec..3742ebb807 100644
--- a/tensorflow/python/training/optimizer.py
+++ b/tensorflow/python/training/optimizer.py
@@ -218,7 +218,7 @@ class Optimizer(
     # Optimizers inherit from CheckpointableBase rather than Checkpointable
     # since they do most of their dependency management themselves (slot
     # variables are special-cased, and non-slot variables are keyed to graphs).
-    checkpointable.CheckpointableBase):
+    checkpointable.Checkpointable):
   """Base class for optimizers.
 
   This class defines the API to add Ops to train a model.  You never use this
diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py
index d1b51adaa4..dec23c50e8 100644
--- a/tensorflow/python/training/saver_test.py
+++ b/tensorflow/python/training/saver_test.py
@@ -2775,7 +2775,7 @@ class ScopedGraphTest(test.TestCase):
       self.assertEqual(2.0, self.evaluate(var_dict2["variable2:0"]))
 
 
-class _OwnsAVariableSimple(checkpointable_base.CheckpointableBase):
+class _OwnsAVariableSimple(checkpointable_base.Checkpointable):
   """A Checkpointable object which can be saved using a tf.train.Saver."""
 
   def __init__(self):
@@ -2808,7 +2808,7 @@ class _MirroringSaveable(
         self._mirrored_variable.assign(tensor))
 
 
-class _OwnsMirroredVariables(checkpointable_base.CheckpointableBase):
+class _OwnsMirroredVariables(checkpointable_base.Checkpointable):
   """A Checkpointable object which returns a more complex SaveableObject."""
 
   def __init__(self):
@@ -2831,7 +2831,7 @@ class _OwnsMirroredVariables(checkpointable_base.CheckpointableBase):
     return self.non_dep_variable.name
 
 
-class NonLayerCheckpointable(checkpointable_tracking.Checkpointable):
+class NonLayerCheckpointable(checkpointable_tracking.AutoCheckpointable):
 
   def __init__(self):
     super(NonLayerCheckpointable, self).__init__()
diff --git a/tensorflow/python/training/saving/saveable_object_util.py b/tensorflow/python/training/saving/saveable_object_util.py
index fa88d2c6eb..b8cc66249b 100644
--- a/tensorflow/python/training/saving/saveable_object_util.py
+++ b/tensorflow/python/training/saving/saveable_object_util.py
@@ -165,7 +165,7 @@ def saveable_objects_for_op(op, name):
         yield ResourceVariableSaveable(
             variable, variable._save_slice_info.spec, name)
     # pylint: enable=protected-access
-  elif isinstance(op, checkpointable.CheckpointableBase) and not isinstance(
+  elif isinstance(op, checkpointable.Checkpointable) and not isinstance(
       op, variables.Variable):
     # pylint: disable=protected-access
     for attr, factory in op._gather_saveables_for_checkpoint().items():
@@ -250,7 +250,7 @@ def op_list_to_dict(op_list, convert_variable_to_tensor=True):
         names_to_saveables[name].append(var)
       else:
         names_to_saveables[name] = [var]
-    elif (isinstance(var, checkpointable.CheckpointableBase)
+    elif (isinstance(var, checkpointable.Checkpointable)
           and not isinstance(var, variables.Variable)):
       checkpointable_saveables = [
           (factory() if callable(factory) else factory)
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.-variable.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.-variable.pbtxt
index 048f666305..f042097acc 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.-variable.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.-variable.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.Variable"
 tf_class {
   is_instance: "<class \'tensorflow.python.ops.variables.VariableV1\'>"
   is_instance: "<class \'tensorflow.python.ops.variables.Variable\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "SaveSliceInfo"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.-iterator.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.-iterator.pbtxt
index 4f0147a523..682a2b91b6 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.-iterator.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.-iterator.pbtxt
@@ -1,7 +1,7 @@
 path: "tensorflow.data.Iterator"
 tf_class {
   is_instance: "<class \'tensorflow.python.data.ops.iterator_ops.Iterator\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "initializer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt
index eced2e1cb0..0a81c3a663 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.engine.training.Model\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.network.Network\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt
index 2acb90173f..2a9ceec30e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.engine.training.Model\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.network.Network\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt
index b2ab5006dc..1bfd51cdcc 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.PeepholeLSTMCell\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.LSTMCell\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activation.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activation.pbtxt
index da212382c1..8a0b8eb46f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activation.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activation.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Activation"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Activation\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activity-regularization.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activity-regularization.pbtxt
index c910db027e..abb3c23694 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activity-regularization.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activity-regularization.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.ActivityRegularization"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.ActivityRegularization\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-add.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-add.pbtxt
index 8b7b33e98c..b27db4e7f2 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-add.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-add.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Add\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-alpha-dropout.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-alpha-dropout.pbtxt
index 5e3e41ba20..50998ac9d6 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-alpha-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-alpha-dropout.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.AlphaDropout"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.noise.AlphaDropout\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling1-d.pbtxt
index e160b10153..be17aeafb5 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.AveragePooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling2-d.pbtxt
index b6b71358c8..7f21b444bc 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.AveragePooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling3-d.pbtxt
index 5c5ab1580e..2ac86f152f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.AveragePooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average.pbtxt
index 489de2e4d3..f6b1dd2f7e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Average\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool1-d.pbtxt
index 30fec249b8..3da1f43a92 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.AveragePooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool2-d.pbtxt
index 0e983c9234..a7be5ac818 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.AveragePooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool3-d.pbtxt
index ec50db7127..c5c29bead3 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.AveragePooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-batch-normalization.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-batch-normalization.pbtxt
index cbbb000e25..3af3c2a501 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-batch-normalization.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-batch-normalization.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.normalization.BatchNormalizationV1\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.normalization.BatchNormalizationV2\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-bidirectional.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-bidirectional.pbtxt
index 23153d4284..880d18e1aa 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-bidirectional.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-bidirectional.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.wrappers.Bidirectional\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.wrappers.Wrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-concatenate.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-concatenate.pbtxt
index 766c3f267f..1eb0cf1a18 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-concatenate.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-concatenate.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Concatenate\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt
index 8980982271..d9394e60f5 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional_recurrent.ConvRNN2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.RNN\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activation"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv1-d.pbtxt
index a74b8d2950..a0f6dc8097 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d-transpose.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d-transpose.pbtxt
index b093f8ead9..037b92f861 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d-transpose.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d.pbtxt
index 0ce9f6fd59..6a0d027d47 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d-transpose.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d-transpose.pbtxt
index c1f5bfae0d..66b5bd75fc 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d-transpose.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d.pbtxt
index 4aa872c4f0..e73133ff07 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution1-d.pbtxt
index 6e01f7c70c..7af6b2b3c3 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt
index c002042d77..baff492dfb 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d.pbtxt
index f5e5446d2b..63d30a6185 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt
index d5f36f4bc3..7a29cbbec3 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d.pbtxt
index 346fec6056..87c75c0224 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping1-d.pbtxt
index 0f8fe9f05e..f69104ddfe 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping1-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Cropping1D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Cropping1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping2-d.pbtxt
index 68fb7382a7..aa05471933 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping2-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Cropping2D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Cropping2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping3-d.pbtxt
index deda82f9b3..d61f1ddc1d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping3-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Cropping3D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Cropping3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt
index 2eba3fb954..e2d05f8298 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.cudnn_recurrent._CuDNNRNN\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.RNN\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt
index 6ed13d37f2..f650f48423 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.cudnn_recurrent._CuDNNRNN\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.RNN\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dense.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dense.pbtxt
index 919aed5723..06e8b6b314 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dense.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dense.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Dense"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Dense\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt
index f590ce1ef7..9fdf6f66d1 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dot.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dot.pbtxt
index db4261fadc..cbe1020650 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dot.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dot.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Dot\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dropout.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dropout.pbtxt
index 7369552b3b..0efba09b27 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dropout.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Dropout"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Dropout\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-e-l-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-e-l-u.pbtxt
index f643ef9de2..b34c499eb2 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-e-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-e-l-u.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.ELU"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.advanced_activations.ELU\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-embedding.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-embedding.pbtxt
index ce053ae8c4..51dd853127 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-embedding.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-embedding.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Embedding"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.embeddings.Embedding\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-flatten.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-flatten.pbtxt
index db95043077..dcd18a9ced 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-flatten.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-flatten.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Flatten"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Flatten\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u-cell.pbtxt
index a6edba6b7e..f029907ee8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u-cell.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.GRUCell"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.GRUCell\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u.pbtxt
index f8c0dbb273..278ae06cba 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.GRU\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.RNN\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activation"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-dropout.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-dropout.pbtxt
index ac4bbe7d19..15cbcfe8ed 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-dropout.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.GaussianDropout"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.noise.GaussianDropout\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-noise.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-noise.pbtxt
index 947e3170ae..865b898c4c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-noise.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-noise.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.GaussianNoise"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.noise.GaussianNoise\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt
index 17e202c581..3e17aca17c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalAveragePooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt
index 9772c5df9b..b160687a2a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalAveragePooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt
index cd65075591..70e8d51a5a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalAveragePooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt
index 0423de7a24..809dc8554b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalAveragePooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt
index 4471cba245..3fbce8cb71 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalAveragePooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt
index c0e7fae456..70e4103ea1 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalAveragePooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool1-d.pbtxt
index 6975a6e88d..000bf54c45 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalMaxPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool2-d.pbtxt
index 56bd70db7e..8ffbf07f9b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalMaxPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool3-d.pbtxt
index 656319920e..3803d2b0a8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalMaxPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt
index f815e66911..28668224e0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalMaxPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt
index f61f0e521b..b83ed67723 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalMaxPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt
index c58c8ce63f..e689d69140 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalMaxPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-layer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-layer.pbtxt
index 0efe9a4297..bb6eddae71 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-layer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-layer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.InputLayer"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.engine.input_layer.InputLayer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt
index 5caa02e71a..5fb3f9dd3a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.LSTMCell"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.LSTMCell\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m.pbtxt
index f21c7e5b21..8eb6dd9f4a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.LSTM\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.RNN\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activation"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-lambda.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-lambda.pbtxt
index 381d5660b9..376bec0814 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-lambda.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-lambda.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Lambda"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Lambda\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-layer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-layer.pbtxt
index 36b0a86628..c5f91a6338 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-layer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-layer.pbtxt
@@ -1,7 +1,7 @@
 path: "tensorflow.keras.layers.Layer"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-leaky-re-l-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-leaky-re-l-u.pbtxt
index b41662e63a..bde8887359 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-leaky-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-leaky-re-l-u.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.LeakyReLU"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.advanced_activations.LeakyReLU\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected1-d.pbtxt
index e4abfca913..16945f2c12 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected1-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.LocallyConnected1D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.local.LocallyConnected1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected2-d.pbtxt
index cfcb92e293..f05741ffce 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected2-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.LocallyConnected2D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.local.LocallyConnected2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-masking.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-masking.pbtxt
index e0721353d1..7885db4ed2 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-masking.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-masking.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Masking"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Masking\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool1-d.pbtxt
index 0618fbeead..9380d26cf4 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.MaxPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool2-d.pbtxt
index 4af52ffec8..8eb8218df3 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.MaxPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool3-d.pbtxt
index db9311ee58..0c96f86ed3 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.MaxPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling1-d.pbtxt
index bfb15cb447..0c6b230eb7 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.MaxPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling2-d.pbtxt
index 1db962dbb8..eb7ca52fba 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.MaxPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling3-d.pbtxt
index f80d5267e7..e724e9088f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.MaxPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-maximum.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-maximum.pbtxt
index cd772d4ac7..dafbd09ee2 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-maximum.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-maximum.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Maximum\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-minimum.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-minimum.pbtxt
index 2bb6b3073a..3122fbec1c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-minimum.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-minimum.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Minimum\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-multiply.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-multiply.pbtxt
index e1a1f07355..0527cda1f0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-multiply.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-multiply.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Multiply\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-p-re-l-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-p-re-l-u.pbtxt
index 66c4446572..814e5a5d54 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-p-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-p-re-l-u.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.PReLU"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.advanced_activations.PReLU\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-permute.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-permute.pbtxt
index 0839554f43..aa1731afb8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-permute.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-permute.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Permute"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Permute\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-r-n-n.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-r-n-n.pbtxt
index b10695f6f7..9d7dd85fe0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-r-n-n.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-r-n-n.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.RNN"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.RNN\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-re-l-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-re-l-u.pbtxt
index b96500f710..e9bba298bb 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-re-l-u.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.ReLU"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.advanced_activations.ReLU\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-repeat-vector.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-repeat-vector.pbtxt
index a27d93ec62..3c783eb512 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-repeat-vector.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-repeat-vector.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.RepeatVector"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.RepeatVector\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-reshape.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-reshape.pbtxt
index 6dda24d3d2..b8e0882541 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-reshape.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-reshape.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Reshape"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Reshape\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv1-d.pbtxt
index 8a4ae8aaa7..310f369ed6 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv1-d.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.SeparableConv\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv2-d.pbtxt
index a083c1da2e..df19d781c2 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv2-d.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.SeparableConv\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution1-d.pbtxt
index 5d5b361f82..bf909509bd 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution1-d.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.SeparableConv\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution2-d.pbtxt
index 392c338d73..5d66bc6fb6 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution2-d.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.SeparableConv\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt
index 1143604903..88e9300de9 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.SimpleRNNCell"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.SimpleRNNCell\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n.pbtxt
index 5a15f1a55f..9d81c6d4bc 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.SimpleRNN\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.RNN\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activation"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-softmax.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-softmax.pbtxt
index c470d9c8e8..712eb0c6ec 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-softmax.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-softmax.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Softmax"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.advanced_activations.Softmax\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt
index d17d6495c0..dfc4ca2705 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.SpatialDropout1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.core.Dropout\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt
index 2d538b4734..5e4f727f71 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.SpatialDropout2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.core.Dropout\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt
index b70923601a..9d893cb30a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.SpatialDropout3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.core.Dropout\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt
index f453ddd50e..a2ed954e4c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.StackedRNNCells"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.StackedRNNCells\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-subtract.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-subtract.pbtxt
index 5759169e07..8a0818e78a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-subtract.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-subtract.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Subtract\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt
index bfde1c35f6..b5591b4826 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.ThresholdedReLU"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.advanced_activations.ThresholdedReLU\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-time-distributed.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-time-distributed.pbtxt
index e7f59a9cc5..210e4fd4e6 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-time-distributed.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-time-distributed.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.wrappers.TimeDistributed\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.wrappers.Wrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling1-d.pbtxt
index 0354149d4f..da2213a84f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling1-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.UpSampling1D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.UpSampling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling2-d.pbtxt
index fff0e26bc1..e2c303d506 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling2-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.UpSampling2D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.UpSampling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling3-d.pbtxt
index c49fa5663d..396e774c8a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling3-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.UpSampling3D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.UpSampling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-wrapper.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-wrapper.pbtxt
index c961699053..8b6418d514 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-wrapper.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Wrapper"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.wrappers.Wrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding1-d.pbtxt
index 1911e128eb..e8fda4c71a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding1-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.ZeroPadding1D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.ZeroPadding1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding2-d.pbtxt
index 88be914347..50c52d270b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding2-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.ZeroPadding2D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.ZeroPadding2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding3-d.pbtxt
index 2bbb71ece2..84c6b78a2b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding3-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.ZeroPadding3D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.ZeroPadding3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-accuracy.pbtxt
index e3e8dc00fe..6756beca1d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-accuracy.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-accuracy.pbtxt
index 60b4f992c0..bec0b20aa5 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-accuracy.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
index deab5a7a70..71dc294dc3 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-hinge.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-hinge.pbtxt
index 10477e6662..43024e738a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-hinge.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-cosine-proximity.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-cosine-proximity.pbtxt
index cdb3e1a79a..1e39385f81 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-cosine-proximity.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-cosine-proximity.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-negatives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-negatives.pbtxt
index f4d067243d..5432f7f400 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-negatives.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-negatives.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-positives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-positives.pbtxt
index f898e00eea..75541bf285 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-positives.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-positives.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-hinge.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-hinge.pbtxt
index 393ba964c4..f8d47f3771 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-hinge.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
index a86b26e618..bf7fc7cfc5 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
index 5e8f321a06..59bb767d35 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-error.pbtxt
index a2eb10c5f6..91f4712312 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-error.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-error.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
index 7df45a8caf..205e15d439 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean.pbtxt
index 899502cfad..eec26ffce7 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt
index 3bdf3a4167..9aeaa5627a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Precision\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt
index 1ac2aacca0..748cec0866 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Recall\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
index 5c4f26929f..97aeb680be 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.SensitivitySpecificityBase\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
index 48ac5c2fdc..571c2bf9d3 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
index 702f711722..85f80b062e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.SensitivitySpecificityBase\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-squared-hinge.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-squared-hinge.pbtxt
index 6c3a9748c7..71c047e973 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-squared-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-squared-hinge.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-negatives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-negatives.pbtxt
index 3f1a99b83b..4bc9383f6f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-negatives.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-negatives.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-positives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-positives.pbtxt
index e5a4207a32..2eae4df0ae 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-positives.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-positives.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt
index 5885cd21c1..55bd285612 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.engine.training.Model\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.network.Network\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt
index 935fa32f8c..a82424167e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.engine.training.Model\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.network.Network\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adadelta.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adadelta.pbtxt
index 5426269793..0a56293e80 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adadelta.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adadelta.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.optimizers.Adadelta"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.adadelta.Adadelta\'>"
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adagrad.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adagrad.pbtxt
index c39fe6ba4f..14d0894e56 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adagrad.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adagrad.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.optimizers.Adagrad"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.adagrad.Adagrad\'>"
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adam.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adam.pbtxt
index 05d46d380b..fdb1ea838c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adam.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adam.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.optimizers.Adam"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.adam.Adam\'>"
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adamax.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adamax.pbtxt
index 88fd96c71d..ece63ec168 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adamax.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adamax.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.optimizers.Adamax"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.adamax.Adamax\'>"
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-nadam.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-nadam.pbtxt
index 7b824e7f0b..f952f88b6d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-nadam.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-nadam.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.optimizers.Nadam"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.nadam.Nadam\'>"
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-optimizer.pbtxt
index 58b7f27491..27bae902b0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-optimizer.pbtxt
@@ -1,7 +1,7 @@
 path: "tensorflow.keras.optimizers.Optimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-r-m-sprop.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-r-m-sprop.pbtxt
index 8de796edde..e523443a00 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-r-m-sprop.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-r-m-sprop.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.optimizers.RMSprop"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.rmsprop.RMSprop\'>"
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-s-g-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-s-g-d.pbtxt
index 393eeb3d6c..d2721f8e92 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-s-g-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-s-g-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.optimizers.SGD"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.gradient_descent.SGD\'>"
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling1-d.pbtxt
index 85764cc8dc..6d826a8f8e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling1-d.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling1D\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling2-d.pbtxt
index 259da2ad3e..9505c90aac 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling2-d.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling2D\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling3-d.pbtxt
index ffda9334cf..5b1b8f78dc 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling3-d.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling3D\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-batch-normalization.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-batch-normalization.pbtxt
index 56a3fc3de7..ef4c57b694 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-batch-normalization.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-batch-normalization.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.normalization.BatchNormalizationV2\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv1-d.pbtxt
index d72f24b3d5..b5ee2e7302 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv1-d.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d-transpose.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d-transpose.pbtxt
index 72a7339368..57f6d7c7c0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d-transpose.pbtxt
@@ -6,7 +6,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d.pbtxt
index 38a63df42d..88c616bd17 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d-transpose.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d-transpose.pbtxt
index 29620561f7..b70a907907 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d-transpose.pbtxt
@@ -6,7 +6,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d.pbtxt
index f1a2bcbb72..33e8765ce6 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-dense.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-dense.pbtxt
index d1e2d57570..1ac13b5791 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-dense.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-dense.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Dense\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-dropout.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-dropout.pbtxt
index 92e40f6d96..77faa3c2b9 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-dropout.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Dropout\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-flatten.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-flatten.pbtxt
index 087601a3c1..0b2631491b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-flatten.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-flatten.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Flatten\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-layer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-layer.pbtxt
index b052c6bb0a..0a3414d20c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-layer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-layer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.layers.Layer"
 tf_class {
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling1-d.pbtxt
index 9444a1bc76..ffc5cf1c8b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling1-d.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling1D\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling2-d.pbtxt
index 83dcb5e4e7..ff2cf2ba90 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling2-d.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling2D\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling3-d.pbtxt
index eb26e2220b..09c8a31a7b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling3-d.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling3D\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv1-d.pbtxt
index 38d75e8bd5..549e13a7ac 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv1-d.pbtxt
@@ -6,7 +6,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv2-d.pbtxt
index 90fc61cdfa..169ecdece5 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv2-d.pbtxt
@@ -6,7 +6,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt
index adffc55227..4251206cda 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt
index 95746cc49c..20af24633a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt
index 3547b66d19..3205c6a4dc 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt
index 7582fd52b6..14cf5ce456 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt
index 7ec61661fd..e43547b154 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt
index 9617d07568..99381cd7e1 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt
index b31886f736..1fbde9df17 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt
index c36ecaa4b2..8ba92fcc8d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt
index 42128ebd17..9de73076b1 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.-adadelta-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.-adadelta-optimizer.pbtxt
index 1f1d8b6f9e..65a2b605d5 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.train.-adadelta-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.train.-adadelta-optimizer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.train.AdadeltaOptimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.adadelta.AdadeltaOptimizer\'>"
   is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "GATE_GRAPH"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.-adagrad-d-a-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.-adagrad-d-a-optimizer.pbtxt
index a7c05d4849..179272d8a8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.train.-adagrad-d-a-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.train.-adagrad-d-a-optimizer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.train.AdagradDAOptimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.adagrad_da.AdagradDAOptimizer\'>"
   is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "GATE_GRAPH"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.-adagrad-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.-adagrad-optimizer.pbtxt
index bc8b92389c..15c2ef46c1 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.train.-adagrad-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.train.-adagrad-optimizer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.train.AdagradOptimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.adagrad.AdagradOptimizer\'>"
   is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "GATE_GRAPH"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.-adam-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.-adam-optimizer.pbtxt
index 5d17be9378..9c902e582f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.train.-adam-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.train.-adam-optimizer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.train.AdamOptimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.adam.AdamOptimizer\'>"
   is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "GATE_GRAPH"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.-checkpoint.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.-checkpoint.pbtxt
index 5be37200f3..42dcdac9e7 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.train.-checkpoint.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.train.-checkpoint.pbtxt
@@ -1,8 +1,8 @@
 path: "tensorflow.train.Checkpoint"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.checkpointable.util.Checkpoint\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.tracking.Checkpointable\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.tracking.AutoCheckpointable\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "save_counter"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.-ftrl-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.-ftrl-optimizer.pbtxt
index d265fdeb01..f41d9f12d9 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.train.-ftrl-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.train.-ftrl-optimizer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.train.FtrlOptimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.ftrl.FtrlOptimizer\'>"
   is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "GATE_GRAPH"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.-gradient-descent-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.-gradient-descent-optimizer.pbtxt
index c673e29cd4..7399750385 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.train.-gradient-descent-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.train.-gradient-descent-optimizer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.train.GradientDescentOptimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.gradient_descent.GradientDescentOptimizer\'>"
   is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "GATE_GRAPH"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.-momentum-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.-momentum-optimizer.pbtxt
index 8199f63b9b..9bbaa14a6f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.train.-momentum-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.train.-momentum-optimizer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.train.MomentumOptimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.momentum.MomentumOptimizer\'>"
   is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "GATE_GRAPH"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.-optimizer.pbtxt
index 876bb35e39..448e17a448 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.train.-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.train.-optimizer.pbtxt
@@ -1,7 +1,7 @@
 path: "tensorflow.train.Optimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "GATE_GRAPH"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.-proximal-adagrad-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.-proximal-adagrad-optimizer.pbtxt
index 14349a74ef..eb1782e9ca 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.train.-proximal-adagrad-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.train.-proximal-adagrad-optimizer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.train.ProximalAdagradOptimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.proximal_adagrad.ProximalAdagradOptimizer\'>"
   is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "GATE_GRAPH"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.-proximal-gradient-descent-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.-proximal-gradient-descent-optimizer.pbtxt
index 7d982dc51f..eb9a86183e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.train.-proximal-gradient-descent-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.train.-proximal-gradient-descent-optimizer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.train.ProximalGradientDescentOptimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.proximal_gradient_descent.ProximalGradientDescentOptimizer\'>"
   is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "GATE_GRAPH"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.-r-m-s-prop-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.-r-m-s-prop-optimizer.pbtxt
index 906384a287..2cf4c2e7ea 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.train.-r-m-s-prop-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.train.-r-m-s-prop-optimizer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.train.RMSPropOptimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.rmsprop.RMSPropOptimizer\'>"
   is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "GATE_GRAPH"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.-sync-replicas-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.-sync-replicas-optimizer.pbtxt
index 2c0fda3c72..ecce08220d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.train.-sync-replicas-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.train.-sync-replicas-optimizer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.train.SyncReplicasOptimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.sync_replicas_optimizer.SyncReplicasOptimizer\'>"
   is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "GATE_GRAPH"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-variable.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-variable.pbtxt
index ee16ffa505..2206f0f8d8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.-variable.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.-variable.pbtxt
@@ -1,7 +1,7 @@
 path: "tensorflow.Variable"
 tf_class {
   is_instance: "<class \'tensorflow.python.ops.variables.Variable\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "SaveSliceInfo"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt
index eced2e1cb0..0a81c3a663 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.engine.training.Model\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.network.Network\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt
index 2acb90173f..2a9ceec30e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.engine.training.Model\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.network.Network\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt
index b2ab5006dc..1bfd51cdcc 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.PeepholeLSTMCell\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.LSTMCell\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activation.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activation.pbtxt
index da212382c1..8a0b8eb46f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activation.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activation.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Activation"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Activation\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activity-regularization.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activity-regularization.pbtxt
index c910db027e..abb3c23694 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activity-regularization.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activity-regularization.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.ActivityRegularization"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.ActivityRegularization\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-add.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-add.pbtxt
index 8b7b33e98c..b27db4e7f2 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-add.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-add.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Add\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-alpha-dropout.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-alpha-dropout.pbtxt
index 5e3e41ba20..50998ac9d6 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-alpha-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-alpha-dropout.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.AlphaDropout"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.noise.AlphaDropout\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling1-d.pbtxt
index e160b10153..be17aeafb5 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.AveragePooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling2-d.pbtxt
index b6b71358c8..7f21b444bc 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.AveragePooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling3-d.pbtxt
index 5c5ab1580e..2ac86f152f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.AveragePooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average.pbtxt
index 489de2e4d3..f6b1dd2f7e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Average\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool1-d.pbtxt
index 30fec249b8..3da1f43a92 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.AveragePooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool2-d.pbtxt
index 0e983c9234..a7be5ac818 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.AveragePooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool3-d.pbtxt
index ec50db7127..c5c29bead3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.AveragePooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-batch-normalization.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-batch-normalization.pbtxt
index 36ea9d5851..b13f963a6f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-batch-normalization.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-batch-normalization.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.BatchNormalization"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.normalization.BatchNormalizationV2\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-bidirectional.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-bidirectional.pbtxt
index 23153d4284..880d18e1aa 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-bidirectional.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-bidirectional.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.wrappers.Bidirectional\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.wrappers.Wrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-concatenate.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-concatenate.pbtxt
index 766c3f267f..1eb0cf1a18 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-concatenate.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-concatenate.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Concatenate\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt
index 8980982271..d9394e60f5 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional_recurrent.ConvRNN2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.RNN\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activation"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv1-d.pbtxt
index a74b8d2950..a0f6dc8097 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d-transpose.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d-transpose.pbtxt
index b093f8ead9..037b92f861 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d-transpose.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d.pbtxt
index 0ce9f6fd59..6a0d027d47 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d-transpose.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d-transpose.pbtxt
index c1f5bfae0d..66b5bd75fc 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d-transpose.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d.pbtxt
index 4aa872c4f0..e73133ff07 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution1-d.pbtxt
index 6e01f7c70c..7af6b2b3c3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt
index c002042d77..baff492dfb 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d.pbtxt
index f5e5446d2b..63d30a6185 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt
index d5f36f4bc3..7a29cbbec3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d.pbtxt
index 346fec6056..87c75c0224 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping1-d.pbtxt
index 0f8fe9f05e..f69104ddfe 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping1-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Cropping1D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Cropping1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping2-d.pbtxt
index 68fb7382a7..aa05471933 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping2-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Cropping2D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Cropping2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping3-d.pbtxt
index deda82f9b3..d61f1ddc1d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping3-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Cropping3D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Cropping3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt
index ff00ca1bb2..28c1926bd7 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.DenseFeatures"
 tf_class {
   is_instance: "<class \'tensorflow.python.feature_column.feature_column_v2.DenseFeatures\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense.pbtxt
index 919aed5723..06e8b6b314 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Dense"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Dense\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt
index f590ce1ef7..9fdf6f66d1 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dot.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dot.pbtxt
index db4261fadc..cbe1020650 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dot.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dot.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Dot\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dropout.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dropout.pbtxt
index 7369552b3b..0efba09b27 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dropout.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Dropout"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Dropout\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-e-l-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-e-l-u.pbtxt
index f643ef9de2..b34c499eb2 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-e-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-e-l-u.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.ELU"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.advanced_activations.ELU\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-embedding.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-embedding.pbtxt
index ce053ae8c4..51dd853127 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-embedding.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-embedding.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Embedding"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.embeddings.Embedding\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-flatten.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-flatten.pbtxt
index db95043077..dcd18a9ced 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-flatten.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-flatten.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Flatten"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Flatten\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u-cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u-cell.pbtxt
index a6edba6b7e..f029907ee8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u-cell.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.GRUCell"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.GRUCell\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u.pbtxt
index df2ea3fbe9..ac2d8c9aa3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.GRU\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.RNN\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activation"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-dropout.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-dropout.pbtxt
index ac4bbe7d19..15cbcfe8ed 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-dropout.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.GaussianDropout"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.noise.GaussianDropout\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-noise.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-noise.pbtxt
index 947e3170ae..865b898c4c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-noise.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-noise.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.GaussianNoise"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.noise.GaussianNoise\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt
index 17e202c581..3e17aca17c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalAveragePooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt
index 9772c5df9b..b160687a2a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalAveragePooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt
index cd65075591..70e8d51a5a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalAveragePooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt
index 0423de7a24..809dc8554b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalAveragePooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt
index 4471cba245..3fbce8cb71 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalAveragePooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt
index c0e7fae456..70e4103ea1 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalAveragePooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool1-d.pbtxt
index 6975a6e88d..000bf54c45 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalMaxPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool2-d.pbtxt
index 56bd70db7e..8ffbf07f9b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalMaxPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool3-d.pbtxt
index 656319920e..3803d2b0a8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalMaxPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt
index f815e66911..28668224e0 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalMaxPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt
index f61f0e521b..b83ed67723 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalMaxPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt
index c58c8ce63f..e689d69140 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalMaxPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-layer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-layer.pbtxt
index 0efe9a4297..bb6eddae71 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-layer.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-layer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.InputLayer"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.engine.input_layer.InputLayer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt
index 5caa02e71a..5fb3f9dd3a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.LSTMCell"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.LSTMCell\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m.pbtxt
index 33082a6f06..89dfc2a256 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.LSTM\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.RNN\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activation"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-lambda.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-lambda.pbtxt
index 381d5660b9..376bec0814 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-lambda.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-lambda.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Lambda"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Lambda\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-layer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-layer.pbtxt
index 36b0a86628..c5f91a6338 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-layer.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-layer.pbtxt
@@ -1,7 +1,7 @@
 path: "tensorflow.keras.layers.Layer"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-leaky-re-l-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-leaky-re-l-u.pbtxt
index b41662e63a..bde8887359 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-leaky-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-leaky-re-l-u.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.LeakyReLU"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.advanced_activations.LeakyReLU\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt
index 5766528b31..d40a999bd8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.engine.training.Model\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.network.Network\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected1-d.pbtxt
index e4abfca913..16945f2c12 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected1-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.LocallyConnected1D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.local.LocallyConnected1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected2-d.pbtxt
index cfcb92e293..f05741ffce 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected2-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.LocallyConnected2D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.local.LocallyConnected2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-masking.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-masking.pbtxt
index e0721353d1..7885db4ed2 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-masking.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-masking.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Masking"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Masking\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool1-d.pbtxt
index 0618fbeead..9380d26cf4 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.MaxPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool2-d.pbtxt
index 4af52ffec8..8eb8218df3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.MaxPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool3-d.pbtxt
index db9311ee58..0c96f86ed3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.MaxPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling1-d.pbtxt
index bfb15cb447..0c6b230eb7 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.MaxPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling2-d.pbtxt
index 1db962dbb8..eb7ca52fba 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.MaxPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling3-d.pbtxt
index f80d5267e7..e724e9088f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.MaxPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-maximum.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-maximum.pbtxt
index cd772d4ac7..dafbd09ee2 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-maximum.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-maximum.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Maximum\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-minimum.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-minimum.pbtxt
index 2bb6b3073a..3122fbec1c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-minimum.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-minimum.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Minimum\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-multiply.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-multiply.pbtxt
index e1a1f07355..0527cda1f0 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-multiply.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-multiply.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Multiply\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-p-re-l-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-p-re-l-u.pbtxt
index 66c4446572..814e5a5d54 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-p-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-p-re-l-u.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.PReLU"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.advanced_activations.PReLU\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-permute.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-permute.pbtxt
index 0839554f43..aa1731afb8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-permute.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-permute.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Permute"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Permute\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-r-n-n.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-r-n-n.pbtxt
index b10695f6f7..9d7dd85fe0 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-r-n-n.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-r-n-n.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.RNN"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.RNN\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-re-l-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-re-l-u.pbtxt
index b96500f710..e9bba298bb 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-re-l-u.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.ReLU"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.advanced_activations.ReLU\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-repeat-vector.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-repeat-vector.pbtxt
index a27d93ec62..3c783eb512 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-repeat-vector.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-repeat-vector.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.RepeatVector"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.RepeatVector\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-reshape.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-reshape.pbtxt
index 6dda24d3d2..b8e0882541 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-reshape.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-reshape.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Reshape"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Reshape\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv1-d.pbtxt
index 8a4ae8aaa7..310f369ed6 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv1-d.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.SeparableConv\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv2-d.pbtxt
index a083c1da2e..df19d781c2 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv2-d.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.SeparableConv\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution1-d.pbtxt
index 5d5b361f82..bf909509bd 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution1-d.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.SeparableConv\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution2-d.pbtxt
index 392c338d73..5d66bc6fb6 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution2-d.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.SeparableConv\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt
index 1143604903..88e9300de9 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.SimpleRNNCell"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.SimpleRNNCell\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n.pbtxt
index 5a15f1a55f..9d81c6d4bc 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.SimpleRNN\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.RNN\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activation"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-softmax.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-softmax.pbtxt
index c470d9c8e8..712eb0c6ec 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-softmax.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-softmax.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Softmax"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.advanced_activations.Softmax\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt
index d17d6495c0..dfc4ca2705 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.SpatialDropout1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.core.Dropout\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt
index 2d538b4734..5e4f727f71 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.SpatialDropout2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.core.Dropout\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt
index b70923601a..9d893cb30a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.SpatialDropout3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.core.Dropout\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt
index f453ddd50e..a2ed954e4c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.StackedRNNCells"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.StackedRNNCells\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-subtract.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-subtract.pbtxt
index 5759169e07..8a0818e78a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-subtract.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-subtract.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Subtract\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt
index bfde1c35f6..b5591b4826 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.ThresholdedReLU"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.advanced_activations.ThresholdedReLU\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-time-distributed.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-time-distributed.pbtxt
index e7f59a9cc5..210e4fd4e6 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-time-distributed.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-time-distributed.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.wrappers.TimeDistributed\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.wrappers.Wrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling1-d.pbtxt
index 0354149d4f..da2213a84f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling1-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.UpSampling1D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.UpSampling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling2-d.pbtxt
index fff0e26bc1..e2c303d506 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling2-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.UpSampling2D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.UpSampling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling3-d.pbtxt
index c49fa5663d..396e774c8a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling3-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.UpSampling3D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.UpSampling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-wrapper.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-wrapper.pbtxt
index c961699053..8b6418d514 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-wrapper.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Wrapper"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.wrappers.Wrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding1-d.pbtxt
index 1911e128eb..e8fda4c71a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding1-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.ZeroPadding1D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.ZeroPadding1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding2-d.pbtxt
index 88be914347..50c52d270b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding2-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.ZeroPadding2D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.ZeroPadding2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding3-d.pbtxt
index 2bbb71ece2..84c6b78a2b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding3-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.ZeroPadding3D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.ZeroPadding3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-accuracy.pbtxt
index e3e8dc00fe..6756beca1d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-accuracy.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-accuracy.pbtxt
index 60b4f992c0..bec0b20aa5 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-accuracy.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
index deab5a7a70..71dc294dc3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-hinge.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-hinge.pbtxt
index 10477e6662..43024e738a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-hinge.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-cosine-proximity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-cosine-proximity.pbtxt
index cdb3e1a79a..1e39385f81 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-cosine-proximity.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-cosine-proximity.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-negatives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-negatives.pbtxt
index f4d067243d..5432f7f400 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-negatives.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-negatives.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-positives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-positives.pbtxt
index f898e00eea..75541bf285 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-positives.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-positives.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-hinge.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-hinge.pbtxt
index 393ba964c4..f8d47f3771 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-hinge.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
index a86b26e618..bf7fc7cfc5 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
index 5e8f321a06..59bb767d35 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-error.pbtxt
index a2eb10c5f6..91f4712312 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-error.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-error.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
index 7df45a8caf..205e15d439 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean.pbtxt
index 899502cfad..eec26ffce7 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt
index 3bdf3a4167..9aeaa5627a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Precision\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt
index 1ac2aacca0..748cec0866 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Recall\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
index 5c4f26929f..97aeb680be 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.SensitivitySpecificityBase\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
index 48ac5c2fdc..571c2bf9d3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
index 702f711722..85f80b062e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.SensitivitySpecificityBase\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-squared-hinge.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-squared-hinge.pbtxt
index 6c3a9748c7..71c047e973 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-squared-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-squared-hinge.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-negatives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-negatives.pbtxt
index 3f1a99b83b..4bc9383f6f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-negatives.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-negatives.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-positives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-positives.pbtxt
index e5a4207a32..2eae4df0ae 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-positives.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-positives.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt
index 5885cd21c1..55bd285612 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.engine.training.Model\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.network.Network\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt
index 935fa32f8c..a82424167e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.engine.training.Model\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.network.Network\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adadelta.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adadelta.pbtxt
index 5426269793..0a56293e80 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adadelta.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adadelta.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.optimizers.Adadelta"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.adadelta.Adadelta\'>"
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adagrad.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adagrad.pbtxt
index c39fe6ba4f..14d0894e56 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adagrad.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adagrad.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.optimizers.Adagrad"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.adagrad.Adagrad\'>"
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adam.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adam.pbtxt
index 05d46d380b..fdb1ea838c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adam.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adam.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.optimizers.Adam"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.adam.Adam\'>"
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adamax.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adamax.pbtxt
index 88fd96c71d..ece63ec168 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adamax.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adamax.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.optimizers.Adamax"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.adamax.Adamax\'>"
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-nadam.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-nadam.pbtxt
index 7b824e7f0b..f952f88b6d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-nadam.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-nadam.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.optimizers.Nadam"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.nadam.Nadam\'>"
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-optimizer.pbtxt
index 58b7f27491..27bae902b0 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-optimizer.pbtxt
@@ -1,7 +1,7 @@
 path: "tensorflow.keras.optimizers.Optimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-r-m-sprop.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-r-m-sprop.pbtxt
index 8de796edde..e523443a00 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-r-m-sprop.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-r-m-sprop.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.optimizers.RMSprop"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.rmsprop.RMSprop\'>"
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-s-g-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-s-g-d.pbtxt
index 393eeb3d6c..d2721f8e92 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-s-g-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-s-g-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.optimizers.SGD"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.gradient_descent.SGD\'>"
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt
index 3547b66d19..3205c6a4dc 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt
index c36ecaa4b2..8ba92fcc8d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt
index 42128ebd17..9de73076b1 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.rnn.-dropout-wrapper.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.rnn.-dropout-wrapper.pbtxt
index 7721eed65b..7781337c82 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.rnn.-dropout-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.rnn.-dropout-wrapper.pbtxt
@@ -6,7 +6,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-checkpoint.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-checkpoint.pbtxt
index 5be37200f3..42dcdac9e7 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-checkpoint.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.train.-checkpoint.pbtxt
@@ -1,8 +1,8 @@
 path: "tensorflow.train.Checkpoint"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.checkpointable.util.Checkpoint\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.tracking.Checkpointable\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.tracking.AutoCheckpointable\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "save_counter"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-proximal-gradient-descent-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-proximal-gradient-descent-optimizer.pbtxt
index 7d982dc51f..eb9a86183e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-proximal-gradient-descent-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.train.-proximal-gradient-descent-optimizer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.train.ProximalGradientDescentOptimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.proximal_gradient_descent.ProximalGradientDescentOptimizer\'>"
   is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "GATE_GRAPH"
-- 
GitLab


From 5b2fa842fafcffd7fb1251a1e550d8b6ff7a1467 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Tue, 15 Jan 2019 16:28:44 -0800
Subject: [PATCH 0757/2345] [Grappler] Preserve function execution semantics
 wrt to Dataset ops.

PiperOrigin-RevId: 229463147
---
 tensorflow/core/grappler/grappler_item.cc     | 16 ++---
 tensorflow/core/grappler/grappler_item.h      | 24 +++----
 tensorflow/core/grappler/op_types.cc          |  7 +++
 tensorflow/core/grappler/op_types.h           |  3 +
 .../optimizers/arithmetic_optimizer.cc        |  2 +-
 .../grappler/optimizers/function_optimizer.cc | 26 ++++----
 .../grappler/optimizers/meta_optimizer.cc     | 12 ++--
 .../optimizers/meta_optimizer_test.cc         | 62 +++++++++----------
 tensorflow/core/grappler/utils/functions.cc   |  6 +-
 .../core/grappler/utils/functions_test.cc     |  2 +-
 10 files changed, 89 insertions(+), 71 deletions(-)

diff --git a/tensorflow/core/grappler/grappler_item.cc b/tensorflow/core/grappler/grappler_item.cc
index 2d71ac54cc..1323dd9a64 100644
--- a/tensorflow/core/grappler/grappler_item.cc
+++ b/tensorflow/core/grappler/grappler_item.cc
@@ -43,7 +43,7 @@ GrapplerItem GrapplerItem::WithGraph(GraphDef&& graph_def) const {
   item.save_restore_loc_tensor = save_restore_loc_tensor;
   item.queue_runners = queue_runners;
   item.devices_ = devices_;
-  item.allowed_optimizations_ = allowed_optimizations_;
+  item.optimization_options_ = optimization_options_;
   item.graph.Swap(&graph_def);
   return item;
 }
@@ -115,9 +115,11 @@ std::unordered_set<string> GrapplerItem::NodesToPreserve() const {
     }
   }
 
-  if (!allowed_optimizations_.prune_ops_with_side_effects) {
+  // Tensorflow functions do not prune side effects, or dataset-output ops from
+  // the function body (see PruneFunctionBody in common_runtime/function.cc).
+  if (optimization_options_.is_function_instantiation) {
     for (const NodeDef& node : graph.node()) {
-      if (!IsFreeOfSideEffect(node)) {
+      if (!IsFreeOfSideEffect(node) || IsDataset(node)) {
         result.insert(node.name());
       }
     }
@@ -175,13 +177,13 @@ Status GrapplerItem::InferDevicesFromGraph() {
 
 void GrapplerItem::ClearDevices() { devices_.clear(); }
 
-const GrapplerItem::AllowedOptimizations& GrapplerItem::allowed_optimizations()
+const GrapplerItem::OptimizationOptions& GrapplerItem::optimization_options()
     const {
-  return allowed_optimizations_;
+  return optimization_options_;
 }
 
-GrapplerItem::AllowedOptimizations& GrapplerItem::allowed_optimizations() {
-  return allowed_optimizations_;
+GrapplerItem::OptimizationOptions& GrapplerItem::optimization_options() {
+  return optimization_options_;
 }
 
 std::vector<const NodeDef*> ComputeTransitiveFanin(
diff --git a/tensorflow/core/grappler/grappler_item.h b/tensorflow/core/grappler/grappler_item.h
index 1ae551f5ac..75712e9f92 100644
--- a/tensorflow/core/grappler/grappler_item.h
+++ b/tensorflow/core/grappler/grappler_item.h
@@ -81,17 +81,17 @@ struct GrapplerItem {
   // fetch nodes, keep_ops, init_ops.
   std::unordered_set<string> NodesToPreserve() const;
 
-  // Restrict types of optimizations that are allowed for this GrapplerItem.
-  struct AllowedOptimizations {
+  struct OptimizationOptions {
     // Is it allowed to add nodes to the graph that do not have registered
     // gradient function.
-    bool non_differentiable_rewrites = true;
-
-    // By default we are allowed to prune ops with side-effects from the main
-    // graph if they are not in transitive fanin of the fetch nodes. If we are
-    // optimizing a graph that was instantiated by a function definition, we
-    // must keep all side effects intact.
-    bool prune_ops_with_side_effects = true;
+    bool allow_non_differentiable_rewrites = true;
+
+    // Tensorflow function execution semantics is slightly different from the
+    // main Tensorflow graph, and we need to make sure that we do not change it
+    // by running Grappler optimizer passes. One main difference is that
+    // functions do not prune ops with side-effects and dataset-output ops (see
+    // PruneFunctionBody in common_runtime/function.cc).
+    bool is_function_instantiation = false;
   };
 
   const std::unordered_set<string>& devices() const;
@@ -108,8 +108,8 @@ struct GrapplerItem {
   // Clears a set of available devices.
   void ClearDevices();
 
-  const AllowedOptimizations& allowed_optimizations() const;
-  AllowedOptimizations& allowed_optimizations();
+  const OptimizationOptions& optimization_options() const;
+  OptimizationOptions& optimization_options();
 
  private:
   // TODO(ezhulenev) Make GrapplerItem a class and hide all public data members.
@@ -120,7 +120,7 @@ struct GrapplerItem {
   // Example of a fully defined name: "/job:work/replica:1/task:1/device:CPU:0"
   std::unordered_set<string> devices_;
 
-  AllowedOptimizations allowed_optimizations_;
+  OptimizationOptions optimization_options_;
 };
 
 // Return the transitive fanin of a set of terminal nodes.
diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc
index b201c3a717..e06580cebf 100644
--- a/tensorflow/core/grappler/op_types.cc
+++ b/tensorflow/core/grappler/op_types.cc
@@ -561,6 +561,13 @@ bool MaybeHasRefInput(const NodeDef& node) {
   return false;
 }
 
+bool IsDataset(const NodeDef& node) {
+  const string& op = node.op();
+  // See `GetNodeClassForOp` in core/graph/graph.cc.
+  return op == "IteratorGetNext" || op == "IteratorGetNextSync" ||
+         op == "DatasetToSingleElement" || op == "ReduceDataset";
+}
+
 bool IsFreeOfSideEffect(const NodeDef& node,
                         const OpRegistryInterface* op_registry) {
   // Placeholders must be preserved to keep the graph feedable.
diff --git a/tensorflow/core/grappler/op_types.h b/tensorflow/core/grappler/op_types.h
index cb7781ec6e..092748f4ff 100644
--- a/tensorflow/core/grappler/op_types.h
+++ b/tensorflow/core/grappler/op_types.h
@@ -183,6 +183,9 @@ bool IsCommutative(const NodeDef& node);
 // value.
 bool IsPersistent(const NodeDef& node);
 
+// Returns true if the node belongs to the NC_DATASET class (see graph/graph.h).
+bool IsDataset(const NodeDef& node);
+
 bool IsFreeOfSideEffect(const NodeDef& node,
                         const OpRegistryInterface* op_registry);
 bool IsFreeOfSideEffect(const NodeDef& node);  // use OpRegistry::Global()
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index e85e0473b3..2168dbd623 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -3576,7 +3576,7 @@ Status ArithmeticOptimizer::Optimize(Cluster* /*cluster*/,
 
   // Disable restricted graph rewrites.
   options_.unary_ops_composition &=
-      item.allowed_optimizations().non_differentiable_rewrites;
+      item.optimization_options().allow_non_differentiable_rewrites;
 
   if (options_.dedup_computations) {
     DedupComputations();
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index 1394aa5047..794e87dc38 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -260,7 +260,7 @@ class FunctionOptimizerContext {
       : grappler_item_id_(item.id),
         graph_version_(item.graph.versions().producer()),
         opt_level_(opt_level),
-        allowed_optimizations_(item.allowed_optimizations()),
+        optimization_options_(item.optimization_options()),
         function_library_(OpRegistry::Global(), item.graph.library()),
         available_device_names_(item.devices().begin(), item.devices().end()),
         graph_view_(&item.graph) {
@@ -272,8 +272,8 @@ class FunctionOptimizerContext {
 
   const RewriterConfig::Toggle opt_level() const { return opt_level_; }
 
-  const GrapplerItem::AllowedOptimizations& allowed_optimizations() const {
-    return allowed_optimizations_;
+  const GrapplerItem::OptimizationOptions& optimization_options() const {
+    return optimization_options_;
   }
 
   const FunctionLibraryDefinition& function_library() const {
@@ -409,7 +409,7 @@ class FunctionOptimizerContext {
   const string grappler_item_id_;
   const int graph_version_;
   const RewriterConfig::Toggle opt_level_;
-  const GrapplerItem::AllowedOptimizations allowed_optimizations_;
+  const GrapplerItem::OptimizationOptions optimization_options_;
   FunctionLibraryDefinition function_library_;
 
   // These fields initialized lazily only if needed.
@@ -1620,21 +1620,25 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
   // edges from inlined side-effectful ops.
   std::vector<string> side_effectful_nodes;
 
-  // We have to make sure that all side-effectful nodes inside a function body
-  // will be executed after function inlining.
+  // We have to make sure that all side-effectful and dataset-output nodes
+  // inside a function body will be executed after function inlining.
   for (NodeDef& func_body_node : *placed_graph_def.mutable_node()) {
-    if (!IsFreeOfSideEffect(func_body_node, &ctx->function_library())) {
+    const bool node_must_execute =
+        !IsFreeOfSideEffect(func_body_node, &ctx->function_library()) ||
+        IsDataset(func_body_node);
+
+    if (node_must_execute) {
       int num_fanouts = placed_graph_view.NumFanouts(
           func_body_node, /*include_controlled_nodes=*/true);
 
       // If the node doesn't have any outgoing edges and we do not have any
       // nodes in the `happens_after` set, we can't inline a function and
-      // guarantee that side-effects will be executed. The only exception if we
-      // do function library optimization, and the GrapplerItem was constructed
-      // for the function body, because functions have strict semantics.
+      // guarantee that it will be executed. The only exception if we do
+      // function library optimization, and the GrapplerItem was instantiated
+      // for the function body, because functions do not prune these ops.
 
       if (num_fanouts == 0 && happens_after.empty() &&
-          ctx->allowed_optimizations().prune_ops_with_side_effects) {
+          !ctx->optimization_options().is_function_instantiation) {
         return errors::Internal(
             "Can't inline a function with a side-effectful op with empty "
             "fanouts and empty output control edge set. Function body node: ",
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
index a84bb1d62f..c200b7db07 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
@@ -527,7 +527,8 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
       // can't perform non-differentiable rewrites.
       if (differentiable_functions.find(func_name) !=
           differentiable_functions.end()) {
-        func_item.allowed_optimizations().non_differentiable_rewrites = false;
+        func_item.optimization_options().allow_non_differentiable_rewrites =
+            false;
       }
 
       // Function item is allowed to use all devices from the main graph.
@@ -536,10 +537,11 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
         VLOG(3) << added_devices.error_message();
       }
 
-      // We are not allowed to prune side effects from the graph instantiated
-      // by the function definition, because we must guarantee function
-      // execution semantics wrt side effects (see function_optimizer.cc).
-      func_item.allowed_optimizations().prune_ops_with_side_effects = false;
+      // We are not allowed to prune certain types of ops from the graph
+      // instantiated by the function definition, because we must guarantee
+      // function execution semantics wrt side effects (see
+      // function_optimizer.cc).
+      func_item.optimization_options().is_function_instantiation = true;
 
       // Optimize function body graph.
       GraphDef optimized_func_graph;
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc b/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
index a061f6194a..b1b72075ca 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
@@ -87,12 +87,12 @@ REGISTER_GRAPH_OPTIMIZER(TestOptimizerWithParams);
 // Record various properties of the GrapplerItems passed for optimization.
 class GrapplerItemPropertiesAccumulator : public CustomGraphOptimizer {
  public:
-  static void SetAllowedOptimizations(
-      gtl::FlatMap<string, GrapplerItem::AllowedOptimizations>*
-          allowed_optimizations) {
-    allowed_optimizations_ = allowed_optimizations;
+  static void SetOptimizationOptions(
+      gtl::FlatMap<string, GrapplerItem::OptimizationOptions>*
+          optimization_options) {
+    optimization_options_ = optimization_options;
   }
-  static void ResetAllowedOptimizations() { allowed_optimizations_ = nullptr; }
+  static void ResetOptimizationOptions() { optimization_options_ = nullptr; }
 
   GrapplerItemPropertiesAccumulator() {}
   string name() const override {
@@ -107,8 +107,8 @@ class GrapplerItemPropertiesAccumulator : public CustomGraphOptimizer {
   Status Optimize(Cluster* cluster, const GrapplerItem& item,
                   GraphDef* optimized_graph) override {
     *optimized_graph = item.graph;
-    if (allowed_optimizations_) {
-      allowed_optimizations_->insert({item.id, item.allowed_optimizations()});
+    if (optimization_options_) {
+      optimization_options_->insert({item.id, item.optimization_options()});
     }
     return Status::OK();
   }
@@ -117,12 +117,12 @@ class GrapplerItemPropertiesAccumulator : public CustomGraphOptimizer {
                 const GraphDef& optimized_graph, double result) override {}
 
  private:
-  static gtl::FlatMap<string, GrapplerItem::AllowedOptimizations>*
-      allowed_optimizations_;
+  static gtl::FlatMap<string, GrapplerItem::OptimizationOptions>*
+      optimization_options_;
 };
 
-gtl::FlatMap<string, GrapplerItem::AllowedOptimizations>*
-    GrapplerItemPropertiesAccumulator::allowed_optimizations_;
+gtl::FlatMap<string, GrapplerItem::OptimizationOptions>*
+    GrapplerItemPropertiesAccumulator::optimization_options_;
 
 REGISTER_GRAPH_OPTIMIZER(GrapplerItemPropertiesAccumulator);
 
@@ -602,10 +602,9 @@ TEST_F(MetaOptimizerTest, OptimizeFunctionLibraryWithRestrictions) {
 
   // We will record what type of optimizations meta optimizer allows for each
   // GrapplerItem (main graph and graphs for each function).
-  gtl::FlatMap<string, GrapplerItem::AllowedOptimizations>
-      allowed_optimizations;
-  GrapplerItemPropertiesAccumulator::SetAllowedOptimizations(
-      &allowed_optimizations);
+  gtl::FlatMap<string, GrapplerItem::OptimizationOptions> optimization_options;
+  GrapplerItemPropertiesAccumulator::SetOptimizationOptions(
+      &optimization_options);
 
   // Just record properties of optimized Grappler items.
   ConfigProto config_proto;
@@ -664,22 +663,23 @@ TEST_F(MetaOptimizerTest, OptimizeFunctionLibraryWithRestrictions) {
 
   // Our custom optimizer must be called for the main graph and for the two
   // functions.
-  ASSERT_EQ(allowed_optimizations.size(), 3);
-
-  auto allowed_optimizations_main =
-      gtl::FindOrNull(allowed_optimizations, "main");
-  ASSERT_NE(allowed_optimizations_main, nullptr);
-  EXPECT_TRUE(allowed_optimizations_main->non_differentiable_rewrites);
-
-  auto allowed_optimizations_my_mul_1 =
-      gtl::FindOrNull(allowed_optimizations, "MyMul1");
-  ASSERT_NE(allowed_optimizations_my_mul_1, nullptr);
-  EXPECT_TRUE(allowed_optimizations_my_mul_1->non_differentiable_rewrites);
-
-  auto allowed_optimizations_my_mul_2 =
-      gtl::FindOrNull(allowed_optimizations, "MyMul2");
-  ASSERT_NE(allowed_optimizations_my_mul_2, nullptr);
-  EXPECT_FALSE(allowed_optimizations_my_mul_2->non_differentiable_rewrites);
+  ASSERT_EQ(optimization_options.size(), 3);
+
+  auto optimization_options_main =
+      gtl::FindOrNull(optimization_options, "main");
+  ASSERT_NE(optimization_options_main, nullptr);
+  EXPECT_TRUE(optimization_options_main->allow_non_differentiable_rewrites);
+
+  auto optimization_options_my_mul_1 =
+      gtl::FindOrNull(optimization_options, "MyMul1");
+  ASSERT_NE(optimization_options_my_mul_1, nullptr);
+  EXPECT_TRUE(optimization_options_my_mul_1->allow_non_differentiable_rewrites);
+
+  auto optimization_options_my_mul_2 =
+      gtl::FindOrNull(optimization_options, "MyMul2");
+  ASSERT_NE(optimization_options_my_mul_2, nullptr);
+  EXPECT_FALSE(
+      optimization_options_my_mul_2->allow_non_differentiable_rewrites);
 }
 
 class SleepingOptimizer : public CustomGraphOptimizer {
diff --git a/tensorflow/core/grappler/utils/functions.cc b/tensorflow/core/grappler/utils/functions.cc
index 150728d030..357a0b3b47 100644
--- a/tensorflow/core/grappler/utils/functions.cc
+++ b/tensorflow/core/grappler/utils/functions.cc
@@ -337,9 +337,9 @@ GrapplerFunctionItem::GrapplerFunctionItem(
     }
   }
 
-  // It's unsafe to prune side-effectful ops from the graph instantiated from a
-  // function definition (see inlining in function_optimizer.cc).
-  allowed_optimizations().prune_ops_with_side_effects = false;
+  // Tensorflow functions execution semantics is different from the main graph,
+  // and we need to preserve it when we do graph optimizations.
+  optimization_options().is_function_instantiation = true;
 }
 
 const string& GrapplerFunctionItem::description() const { return description_; }
diff --git a/tensorflow/core/grappler/utils/functions_test.cc b/tensorflow/core/grappler/utils/functions_test.cc
index c49920c79c..7720888828 100644
--- a/tensorflow/core/grappler/utils/functions_test.cc
+++ b/tensorflow/core/grappler/utils/functions_test.cc
@@ -641,7 +641,7 @@ TEST_F(FunctionsTest, FromFunctionDefWithSideEffectfulOps) {
   EXPECT_EQ(3, item.function_body().node_size());
   EXPECT_EQ(1, item.input_size());
   EXPECT_EQ(0, item.output_size());
-  EXPECT_EQ(false, item.allowed_optimizations().prune_ops_with_side_effects);
+  EXPECT_EQ(true, item.optimization_options().is_function_instantiation);
 }
 
 TEST_F(FunctionsTest, MakeFunctionDef) {
-- 
GitLab


From 7e090f6a5412fd5f22b5b75ca7ae7844c3d025e9 Mon Sep 17 00:00:00 2001
From: Nick Hamatake <hamatake@google.com>
Date: Tue, 15 Jan 2019 16:34:32 -0800
Subject: [PATCH 0758/2345] Force ICU to look for its data statically. This is
 required for Windows 10 builds. Patch suggested by github.com/cielavenir.

Fixes #23655

PiperOrigin-RevId: 229464102
---
 tensorflow/core/kernels/unicode_script_op.cc | 3 +++
 third_party/icu/BUILD.bazel                  | 1 +
 2 files changed, 4 insertions(+)

diff --git a/tensorflow/core/kernels/unicode_script_op.cc b/tensorflow/core/kernels/unicode_script_op.cc
index 085e397eba..f01bb52293 100644
--- a/tensorflow/core/kernels/unicode_script_op.cc
+++ b/tensorflow/core/kernels/unicode_script_op.cc
@@ -17,6 +17,9 @@ limitations under the License.
 #include "unicode/uscript.h"  // TF:icu
 #include "tensorflow/core/framework/op_kernel.h"
 
+// ICU codemap data is linked statically.
+#define U_STATIC_IMPLEMENTATION
+
 namespace tensorflow {
 
 class UnicodeScriptOp : public OpKernel {
diff --git a/third_party/icu/BUILD.bazel b/third_party/icu/BUILD.bazel
index 36d6b9006b..399962d249 100644
--- a/third_party/icu/BUILD.bazel
+++ b/third_party/icu/BUILD.bazel
@@ -44,6 +44,7 @@ cc_library(
     ]),
     copts = [
         "-DU_COMMON_IMPLEMENTATION",
+        "-DU_STATIC_IMPLEMENTATION",
         "-DU_HAVE_STD_ATOMICS",
     ] + select({
         ":android": [
-- 
GitLab


From bc246800912b450cf5ee7a485a95a0b77a3d9f53 Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Tue, 15 Jan 2019 17:40:10 -0800
Subject: [PATCH 0759/2345] Reduce flakyness of integration test.

Tested by trying a range of random seeds.

PiperOrigin-RevId: 229473610
---
 tensorflow/python/keras/integration_test.py | 36 ++++++++++-----------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/tensorflow/python/keras/integration_test.py b/tensorflow/python/keras/integration_test.py
index febee44cd4..dc8d1deddb 100644
--- a/tensorflow/python/keras/integration_test.py
+++ b/tensorflow/python/keras/integration_test.py
@@ -35,7 +35,7 @@ class VectorClassificationIntegrationTest(keras_parameterized.TestCase):
   def test_vector_classification(self):
     np.random.seed(1337)
     (x_train, y_train), _ = testing_utils.get_test_data(
-        train_samples=150,
+        train_samples=100,
         test_samples=0,
         input_shape=(10,),
         num_classes=2)
@@ -47,10 +47,10 @@ class VectorClassificationIntegrationTest(keras_parameterized.TestCase):
          keras.layers.Dense(y_train.shape[-1], activation='softmax')],
         input_shape=x_train.shape[1:])
     model.compile(loss='categorical_crossentropy',
-                  optimizer=keras.optimizer_v2.adam.Adam(0.01),
+                  optimizer=keras.optimizer_v2.adam.Adam(0.005),
                   metrics=['accuracy'],
                   run_eagerly=testing_utils.should_run_eagerly())
-    history = model.fit(x_train, y_train, epochs=10, batch_size=16,
+    history = model.fit(x_train, y_train, epochs=10, batch_size=10,
                         validation_data=(x_train, y_train),
                         verbose=2)
     self.assertGreater(history.history['val_acc'][-1], 0.7)
@@ -64,7 +64,7 @@ class VectorClassificationIntegrationTest(keras_parameterized.TestCase):
     # and internal losses can be shared.
     np.random.seed(1337)
     (x_train, y_train), _ = testing_utils.get_test_data(
-        train_samples=150,
+        train_samples=100,
         test_samples=0,
         input_shape=(10,),
         num_classes=2)
@@ -82,13 +82,13 @@ class VectorClassificationIntegrationTest(keras_parameterized.TestCase):
     y = keras.layers.Dense(y_train.shape[-1], activation='softmax')(y)
     model = keras.models.Model(x, y)
     model.compile(loss='categorical_crossentropy',
-                  optimizer=keras.optimizer_v2.adam.Adam(0.01),
+                  optimizer=keras.optimizer_v2.adam.Adam(0.005),
                   metrics=['accuracy'],
                   run_eagerly=testing_utils.should_run_eagerly())
     if not testing_utils.should_run_eagerly():
       self.assertEqual(len(model.losses), 2)
       self.assertEqual(len(model.updates), 2)
-    history = model.fit(x_train, y_train, epochs=10, batch_size=16,
+    history = model.fit(x_train, y_train, epochs=10, batch_size=10,
                         validation_data=(x_train, y_train),
                         verbose=2)
     self.assertGreater(history.history['val_acc'][-1], 0.7)
@@ -106,7 +106,7 @@ class TimeseriesClassificationIntegrationTest(keras_parameterized.TestCase):
   def test_timeseries_classification(self):
     np.random.seed(1337)
     (x_train, y_train), _ = testing_utils.get_test_data(
-        train_samples=250,
+        train_samples=100,
         test_samples=0,
         input_shape=(4, 10),
         num_classes=2)
@@ -119,22 +119,22 @@ class TimeseriesClassificationIntegrationTest(keras_parameterized.TestCase):
     model = testing_utils.get_model_from_layers(
         layers, input_shape=x_train.shape[1:])
     model.compile(loss='categorical_crossentropy',
-                  optimizer=keras.optimizer_v2.adam.Adam(0.01),
+                  optimizer=keras.optimizer_v2.adam.Adam(0.005),
                   metrics=['accuracy'],
                   run_eagerly=testing_utils.should_run_eagerly())
-    history = model.fit(x_train, y_train, epochs=15, batch_size=16,
+    history = model.fit(x_train, y_train, epochs=15, batch_size=10,
                         validation_data=(x_train, y_train),
                         verbose=2)
     self.assertGreater(history.history['val_acc'][-1], 0.7)
     _, val_acc = model.evaluate(x_train, y_train)
     self.assertAlmostEqual(history.history['val_acc'][-1], val_acc)
     predictions = model.predict(x_train)
-    self.assertEqual(predictions.shape, (250, 2))
+    self.assertEqual(predictions.shape, (x_train.shape[0], 2))
 
   def test_timeseries_classification_sequential_tf_rnn(self):
     np.random.seed(1337)
     (x_train, y_train), _ = testing_utils.get_test_data(
-        train_samples=150,
+        train_samples=100,
         test_samples=0,
         input_shape=(4, 10),
         num_classes=2)
@@ -147,17 +147,17 @@ class TimeseriesClassificationIntegrationTest(keras_parameterized.TestCase):
                                                 activation='softmax',
                                                 dtype=dtypes.float32)))
     model.compile(loss='categorical_crossentropy',
-                  optimizer=keras.optimizer_v2.adam.Adam(0.01),
+                  optimizer=keras.optimizer_v2.adam.Adam(0.005),
                   metrics=['accuracy'],
                   run_eagerly=testing_utils.should_run_eagerly())
-    history = model.fit(x_train, y_train, epochs=15, batch_size=16,
+    history = model.fit(x_train, y_train, epochs=15, batch_size=10,
                         validation_data=(x_train, y_train),
                         verbose=2)
     self.assertGreater(history.history['val_acc'][-1], 0.7)
     _, val_acc = model.evaluate(x_train, y_train)
     self.assertAlmostEqual(history.history['val_acc'][-1], val_acc)
     predictions = model.predict(x_train)
-    self.assertEqual(predictions.shape, (150, 2))
+    self.assertEqual(predictions.shape, (x_train.shape[0], 2))
 
 
 @keras_parameterized.run_with_all_model_types
@@ -167,7 +167,7 @@ class ImageClassificationIntegrationTest(keras_parameterized.TestCase):
   def test_image_classification(self):
     np.random.seed(1337)
     (x_train, y_train), _ = testing_utils.get_test_data(
-        train_samples=250,
+        train_samples=100,
         test_samples=0,
         input_shape=(10, 10, 3),
         num_classes=2)
@@ -184,17 +184,17 @@ class ImageClassificationIntegrationTest(keras_parameterized.TestCase):
     model = testing_utils.get_model_from_layers(
         layers, input_shape=x_train.shape[1:])
     model.compile(loss='categorical_crossentropy',
-                  optimizer=keras.optimizer_v2.adam.Adam(0.01),
+                  optimizer=keras.optimizer_v2.adam.Adam(0.005),
                   metrics=['accuracy'],
                   run_eagerly=testing_utils.should_run_eagerly())
-    history = model.fit(x_train, y_train, epochs=10, batch_size=16,
+    history = model.fit(x_train, y_train, epochs=10, batch_size=10,
                         validation_data=(x_train, y_train),
                         verbose=2)
     self.assertGreater(history.history['val_acc'][-1], 0.7)
     _, val_acc = model.evaluate(x_train, y_train)
     self.assertAlmostEqual(history.history['val_acc'][-1], val_acc)
     predictions = model.predict(x_train)
-    self.assertEqual(predictions.shape, (250, 2))
+    self.assertEqual(predictions.shape, (x_train.shape[0], 2))
 
 
 if __name__ == '__main__':
-- 
GitLab


From 13ecb4c3bcdd93c85a4bf21080b240681c763eda Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Tue, 15 Jan 2019 17:47:32 -0800
Subject: [PATCH 0760/2345] Document the GPU accelerator best practices.

PiperOrigin-RevId: 229474455
---
 .../lite/g3doc/performance/best_practices.md  | 32 +++++++++++++++++--
 1 file changed, 29 insertions(+), 3 deletions(-)

diff --git a/tensorflow/lite/g3doc/performance/best_practices.md b/tensorflow/lite/g3doc/performance/best_practices.md
index b76414cebe..090f19e810 100644
--- a/tensorflow/lite/g3doc/performance/best_practices.md
+++ b/tensorflow/lite/g3doc/performance/best_practices.md
@@ -37,8 +37,34 @@ If your application is not careful, there can be redundant copies when feeding t
 Platform specific tools like [Android profiler](https://developer.android.com/studio/profile/android-profiler) and [Instruments](https://help.apple.com/instruments/mac/current/) provide a wealth of profiling information that can be used to debug your app. Sometimes the performance bug may be not in the model but in parts of application code that interact with the model. Make sure to familiarize yourself with platform specific profiling tools and best practices for your platform.
 
 ## Evaluate whether your model benefits from using hardware accelerators available on the device
-Tensorflow Lite is working on adding support for accelerators like GPU and provides acceleration through [Neural Networks API](https://developer.android.com/ndk/guides/neuralnetworks/) on Android.
-You can utilize these hardware accelerator backends to improve the speed and efficiency of your model. To enable Neural Networks API call [UseNNAPI](https://github.com/tensorflow/tensorflow/blob/6305a6d83552ba6a472cd72398b60d9241467f1f/tensorflow/lite/interpreter.h#L334) on the interpreter instance.
+
+TensorFlow Lite has added been new ways to accelerate models with faster
+hardware like GPUs, DSPs, and neural accelerators. Typically, these accelerators
+are exposed through *delegate* submodules that take over parts of the
+interpreter execution. TensorFlow Lite can use delegates by:
+
+*   Using Android's
+    [Neural Networks API](https://developer.android.com/ndk/guides/neuralnetworks/).
+    You can utilize these hardware accelerator backends to improve the speed and
+    efficiency of your model. To enable the Neural Networks API, call
+    [UseNNAPI](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/interpreter.h#L330)
+    on the interpreter instance.
+*   A binary-only GPU delegate has been released for Android and iOS—using
+    OpenGL and Metal, respectively. To try them out, see the
+    [GPU delegate tutorial](gpu.md) and [documentation](gpu_advanced.md).
+*   It is possible to create your own delegate if you have access to
+    non-standard hardware. View the NN API delegate in the source code as an
+    example.
+
+Be aware that some accelerators work better for different types of models. It is
+important to benchmark each delegate to see if it is a good choice for your
+application. For example, if you have a very small model, it may not be worth
+delegating the model to either the NN API or the GPU. Conversely, accelerators
+are a great choice for large models that have high arithmetic intensity.
 
 ## Need more help
-The Tensorflow team is happy to help diagnose and address specific performance issues you may be facing. Please file an issue on [GitHub](https://github.com/tensorflow/tensorflow/issues) with details of the issue.
+
+The Tensorflow team is happy to help diagnose and address specific performance
+issues you may be facing. Please file an issue on
+[GitHub](https://github.com/tensorflow/tensorflow/issues) with details of the
+issue.
-- 
GitLab


From a3fa57c6d2f4678597c0b76bb448c37b66c45376 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Tue, 15 Jan 2019 17:57:55 -0800
Subject: [PATCH 0761/2345] Add new distribution strategy test rule

PiperOrigin-RevId: 229475641
---
 tensorflow/contrib/distribute/python/BUILD    | 10 +++--
 .../core/platform/default/distribute.bzl      | 39 +++++++++++++++++++
 2 files changed, 45 insertions(+), 4 deletions(-)
 create mode 100644 tensorflow/core/platform/default/distribute.bzl

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 1a54cd167e..384fe53923 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -1,6 +1,7 @@
 # Implementation of a prototype TF distributed computation library.
 
 load("//tensorflow/compiler/tests:build_defs.bzl", "tf_xla_py_test")
+load("//tensorflow/core:platform/default/distribute.bzl", "distribute_py_test")
 load("//tensorflow:tensorflow.bzl", "py_test")
 load("//tensorflow:tensorflow.bzl", "cuda_py_test")
 
@@ -621,12 +622,10 @@ py_library(
     ],
 )
 
-cuda_py_test(
+distribute_py_test(
     name = "keras_test",
     srcs = ["keras_test.py"],
-    additional_deps = [
-        ":keras_test_lib",
-    ],
+    main = "keras_test.py",
     shard_count = 16,
     tags = [
         "multi_and_single_gpu",
@@ -635,6 +634,9 @@ cuda_py_test(
         "no_windows_gpu",
         "notsan",
     ],
+    deps = [
+        ":keras_test_lib",
+    ],
 )
 
 # TODO(b/121200287): Remove this in 2.0
diff --git a/tensorflow/core/platform/default/distribute.bzl b/tensorflow/core/platform/default/distribute.bzl
new file mode 100644
index 0000000000..a3afc37f03
--- /dev/null
+++ b/tensorflow/core/platform/default/distribute.bzl
@@ -0,0 +1,39 @@
+"""Build rules for tf.distritbute testing."""
+
+load("//tensorflow:tensorflow.bzl", "cuda_py_test")
+
+def distribute_py_test(
+        name,
+        srcs = [],
+        deps = [],
+        tags = [],
+        data = [],
+        main = None,
+        args = [],
+        shard_count = 1,
+        **kwargs):
+    """Generates py_test targets for CPU and GPU.
+
+    Args:
+        name: test target name to generate suffixed with `test`.
+        srcs: source files for the tests.
+        deps: additional dependencies for the test targets.
+        tags: tags to be assigned to the different test targets.
+        data: data files that need to be associated with the target files.
+        main: optional main script.
+        args: arguments to the tests.
+        shard_count: number of shards to split the tests across.
+        **kwargs: extra keyword arguments to the test.
+    """
+
+    cuda_py_test(
+        name = name,
+        srcs = srcs,
+        data = data,
+        main = main,
+        additional_deps = deps,
+        shard_count = shard_count,
+        tags = tags,
+        args = args,
+        **kwargs
+    )
-- 
GitLab


From 6750ba5da025f71ea989ea7f9d924a5e9b22aedc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 15 Jan 2019 18:06:59 -0800
Subject: [PATCH 0762/2345] Update ops-related pbtxt files.

PiperOrigin-RevId: 229476985
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 128 ++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 |  95 ++++++++++++-
 2 files changed, 217 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 8705492c66..9990b8629c 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -1298,6 +1298,34 @@ op {
     type: DT_FLOAT
   }
 }
+op {
+  name: "AdjustContrastv2"
+  input_arg {
+    name: "images"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "contrast_factor"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+      }
+    }
+  }
+}
 op {
   name: "AdjustHue"
   input_arg {
@@ -1313,6 +1341,34 @@ op {
     type: DT_FLOAT
   }
 }
+op {
+  name: "AdjustHue"
+  input_arg {
+    name: "images"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "delta"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+      }
+    }
+  }
+}
 op {
   name: "AdjustSaturation"
   input_arg {
@@ -1328,6 +1384,34 @@ op {
     type: DT_FLOAT
   }
 }
+op {
+  name: "AdjustSaturation"
+  input_arg {
+    name: "images"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "scale"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+      }
+    }
+  }
+}
 op {
   name: "All"
   input_arg {
@@ -73373,6 +73457,50 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "StatefulStandardNormal"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "shape"
+    type_attr: "shape_dtype"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_BFLOAT16
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "shape_dtype"
+    type: "type"
+    default_value {
+      type: DT_INT64
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "StatelessIf"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 012c03a834..8212282623 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -486,7 +486,7 @@ op {
   name: "AdjustContrastv2"
   input_arg {
     name: "images"
-    type: DT_FLOAT
+    type_attr: "T"
   }
   input_arg {
     name: "contrast_factor"
@@ -494,14 +494,27 @@ op {
   }
   output_arg {
     name: "output"
-    type: DT_FLOAT
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+      }
+    }
   }
 }
 op {
   name: "AdjustHue"
   input_arg {
     name: "images"
-    type: DT_FLOAT
+    type_attr: "T"
   }
   input_arg {
     name: "delta"
@@ -509,14 +522,27 @@ op {
   }
   output_arg {
     name: "output"
-    type: DT_FLOAT
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+      }
+    }
   }
 }
 op {
   name: "AdjustSaturation"
   input_arg {
     name: "images"
-    type: DT_FLOAT
+    type_attr: "T"
   }
   input_arg {
     name: "scale"
@@ -524,7 +550,20 @@ op {
   }
   output_arg {
     name: "output"
-    type: DT_FLOAT
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+      }
+    }
   }
 }
 op {
@@ -34365,6 +34404,50 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "StatefulStandardNormal"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "shape"
+    type_attr: "shape_dtype"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_BFLOAT16
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "shape_dtype"
+    type: "type"
+    default_value {
+      type: DT_INT64
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "StatelessIf"
   input_arg {
-- 
GitLab


From d62f6223fc0a1e4d81dffd92c61ada3e9ec17fb7 Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Tue, 15 Jan 2019 18:23:21 -0800
Subject: [PATCH 0763/2345] Move to new quantization parameters for subgraph
 quantizer.

PiperOrigin-RevId: 229478990
---
 .../lite/tools/optimize/subgraph_quantizer.cc | 37 ++++----
 .../tools/optimize/subgraph_quantizer_test.cc | 76 +++++++---------
 .../optimize/symmetric_per_channel_params.cc  | 86 -------------------
 .../optimize/symmetric_per_channel_params.h   | 58 -------------
 .../symmetric_per_channel_params_test.cc      | 51 -----------
 5 files changed, 47 insertions(+), 261 deletions(-)
 delete mode 100644 tensorflow/lite/tools/optimize/symmetric_per_channel_params.cc
 delete mode 100644 tensorflow/lite/tools/optimize/symmetric_per_channel_params.h
 delete mode 100644 tensorflow/lite/tools/optimize/symmetric_per_channel_params_test.cc

diff --git a/tensorflow/lite/tools/optimize/subgraph_quantizer.cc b/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
index 56f2e6cf76..30320304ec 100644
--- a/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
+++ b/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
@@ -27,7 +27,6 @@ limitations under the License.
 #include "tensorflow/lite/model.h"
 #include "tensorflow/lite/schema/schema_generated.h"
 #include "tensorflow/lite/tools/optimize/quantization_utils.h"
-#include "tensorflow/lite/tools/optimize/symmetric_per_channel_params.h"
 
 namespace tflite {
 namespace optimize {
@@ -37,11 +36,14 @@ namespace {
 const int8_t kMinQuantizedValue = -127;
 const int8_t kMaxQuantizedValue = 127;
 
-TfLiteStatus AddQuantizationParams(const SymmetricPerChannelParams& params,
+TfLiteStatus AddQuantizationParams(const std::vector<float>& scales,
+                                   int quantized_dimension,
                                    const uint8_t* buffer_data,
                                    size_t buffer_size, TensorType output_type,
                                    ModelT* model, TensorT* tensor) {
-  params.AddToTensor(tensor);
+  tensor->quantization = absl::make_unique<QuantizationParametersT>();
+  tensor->quantization->scale.assign(scales.begin(), scales.end());
+  tensor->quantization->quantized_dimension = quantized_dimension;
   model->buffers[tensor->buffer]->data.assign(buffer_data,
                                               buffer_data + buffer_size);
   // Update the tensor type.
@@ -95,10 +97,6 @@ TfLiteStatus SymmetricPerChannelQuantizeTensor(ModelT* model, TensorT* tensor,
   TF_LITE_ENSURE_STATUS(utils::NumElements(*tensor, &num_elements));
   const uint64_t num_elements_per_channel = num_elements / channel_dim_size;
 
-  if (tensor->quantization == nullptr) {
-    tensor->quantization = absl::make_unique<QuantizationParametersT>();
-  }
-
   std::vector<float> min_vals(channel_dim_size);
   std::vector<float> max_vals(channel_dim_size);
   std::vector<bool> has_min_max_value(channel_dim_size, false);
@@ -170,9 +168,8 @@ TfLiteStatus SymmetricPerChannelQuantizeTensor(ModelT* model, TensorT* tensor,
   // Set the buffers and output type.
   uint8_t* uint8_buffer = reinterpret_cast<uint8_t*>(final_buffer.data());
   size_t buffer_size = num_elements * sizeof(int8_t);
-  SymmetricPerChannelParams symmetric_params(scales, channel_dim_index);
-  return AddQuantizationParams(symmetric_params, uint8_buffer, buffer_size,
-                               TensorType_INT8, model, tensor);
+  return AddQuantizationParams(scales, channel_dim_index, uint8_buffer,
+                               buffer_size, TensorType_INT8, model, tensor);
 }
 
 // Symmetrically quantizes the bias for ops like Conv and DepthwiseConv.
@@ -204,10 +201,8 @@ TfLiteStatus SymmetricPerChannelBiasQuantize(const TensorT* input_tensor,
     error_reporter->Report("Input tensor missing quantization information");
     return kTfLiteError;
   }
-  std::unique_ptr<SymmetricPerChannelParams> weight_params;
-  TF_LITE_ENSURE_STATUS(SymmetricPerChannelParams::ReadFromTensor(
-      *weight_tensor, &weight_params));
-  const auto& weight_scales = weight_params->scales();
+  TF_LITE_ENSURE(error_reporter, weight_tensor->quantization);
+  const std::vector<float>& weight_scales = weight_tensor->quantization->scale;
 
   if (weight_scales.size() != channel_dim_size) {
     error_reporter->Report("Mismatch weight scale dimension: %d",
@@ -226,9 +221,6 @@ TfLiteStatus SymmetricPerChannelBiasQuantize(const TensorT* input_tensor,
   uint64_t num_elements;
   TF_LITE_ENSURE_STATUS(utils::NumElements(*tensor, &num_elements));
 
-  if (tensor->quantization == nullptr) {
-    tensor->quantization = absl::make_unique<QuantizationParametersT>();
-  }
   std::vector<int32_t> final_buffer(num_elements);
   const int32_t kScale = std::numeric_limits<int32_t>::max();
 
@@ -244,9 +236,14 @@ TfLiteStatus SymmetricPerChannelBiasQuantize(const TensorT* input_tensor,
   // Set the buffers and output type.
   uint8_t* uint8_buffer = reinterpret_cast<uint8_t*>(final_buffer.data());
   size_t buffer_size = num_elements * sizeof(int32_t);
-  SymmetricPerChannelParams symmetric_params(scales, channel_dim_index);
-  return AddQuantizationParams(symmetric_params, uint8_buffer, buffer_size,
-                               TensorType_INT32, model, tensor);
+  // For Bias we only set the quantized values, the scale and quantized
+  // dimension is implicit.
+  tensor->quantization = nullptr;
+  model->buffers[tensor->buffer]->data.assign(uint8_buffer,
+                                              uint8_buffer + buffer_size);
+
+  tensor->type = TensorType_INT32;
+  return kTfLiteOk;
 }
 }  // namespace
 
diff --git a/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc b/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
index 96ba20d4d9..085b66e90f 100644
--- a/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
+++ b/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
@@ -21,7 +21,6 @@ limitations under the License.
 #include "tensorflow/core/util/command_line_flags.h"
 #include "tensorflow/lite/schema/schema_generated.h"
 #include "tensorflow/lite/tools/optimize/subgraph_quantizer.h"
-#include "tensorflow/lite/tools/optimize/symmetric_per_channel_params.h"
 #include "tensorflow/lite/tools/optimize/test_util.h"
 
 namespace {
@@ -91,31 +90,22 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantizationWithUnitScale) {
 
   ASSERT_TRUE(weights_tensor->quantization);
   const int out_channel_size = weights_tensor->shape[0];
-  std::unique_ptr<SymmetricPerChannelParams> bias_params;
-  std::unique_ptr<SymmetricPerChannelParams> weights_params;
-  ASSERT_EQ(kTfLiteOk, SymmetricPerChannelParams::ReadFromTensor(
-                           *weights_tensor, &weights_params));
-  ASSERT_EQ(kTfLiteOk, SymmetricPerChannelParams::ReadFromTensor(*bias_tensor,
-                                                                 &bias_params));
-
-  ASSERT_EQ(bias_params->scales().size(), out_channel_size);
-  ASSERT_EQ(weights_params->scales().size(), out_channel_size);
+
+  // Bias tensor doesn't contain quantization info.
+  ASSERT_FALSE(bias_tensor->quantization);
+
+  const std::vector<float>& weights_scales =
+      weights_tensor->quantization->scale;
+
+  ASSERT_EQ(weights_scales.size(), out_channel_size);
   ASSERT_EQ(input_tensor->quantization->scale.size(), 1);
   ASSERT_EQ(output_tensor->quantization->scale.size(), 1);
 
-  const float eps = 1e-7;
 
-  // Bias scale should be input * per_channel_weight_scale.
-  for (size_t i = 0; i < out_channel_size; i++) {
-    EXPECT_NEAR(
-        bias_params->scales()[i],
-        input_tensor->quantization->scale[0] * weights_params->scales()[i],
-        eps);
-  }
   for (size_t i = 0; i < out_channel_size; i++) {
-    EXPECT_EQ(weights_params->scales()[i], 1);
-    EXPECT_EQ(bias_params->scales()[i], 1);
+    EXPECT_EQ(weights_scales[i], 1);
   }
+
   EXPECT_EQ(input_tensor->quantization->scale[0], 1);
   EXPECT_EQ(output_tensor->quantization->scale[0], 1);
 
@@ -128,8 +118,11 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantizationWithUnitScale) {
   const float* bias_float_buffer =
       reinterpret_cast<const float*>(original_bias_buffer->data()->data());
 
+  const float eps = 1e-7;
   for (size_t i = 0; i < bias_tensor->shape[0]; i++) {
-    auto dequantized_value = bias_values[i] * bias_params->scales()[i];
+    const float bias_scale =
+        input_tensor->quantization->scale[0] * weights_scales[i];
+    auto dequantized_value = bias_values[i] * bias_scale;
     EXPECT_NEAR(dequantized_value, bias_float_buffer[i], eps);
   }
 
@@ -147,7 +140,7 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantizationWithUnitScale) {
     for (size_t j = 0; j < num_values_in_channel; j++) {
       size_t element_idx = channel_idx * out_channel_size + j;
       auto dequantized_value =
-          weight_values[element_idx] * weights_params->scales()[channel_idx];
+          weight_values[element_idx] * weights_scales[channel_idx];
       EXPECT_NEAR(dequantized_value, weights_float_buffer[element_idx], eps);
     }
   }
@@ -190,27 +183,17 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantization) {
 
   ASSERT_TRUE(weights_tensor->quantization);
   const int out_channel_size = weights_tensor->shape[0];
-  std::unique_ptr<SymmetricPerChannelParams> bias_params;
-  std::unique_ptr<SymmetricPerChannelParams> weights_params;
-  ASSERT_EQ(kTfLiteOk, SymmetricPerChannelParams::ReadFromTensor(
-                           *weights_tensor, &weights_params));
-  ASSERT_EQ(kTfLiteOk, SymmetricPerChannelParams::ReadFromTensor(*bias_tensor,
-                                                                 &bias_params));
-
-  ASSERT_EQ(bias_params->scales().size(), out_channel_size);
-  ASSERT_EQ(weights_params->scales().size(), out_channel_size);
-  ASSERT_EQ(input_tensor->quantization->scale.size(), 1);
-  ASSERT_EQ(output_tensor->quantization->scale.size(), 1);
 
-  const float eps = 1e-7;
+  // Bias tensor doesn't contain quantization info.
+  ASSERT_FALSE(bias_tensor->quantization);
 
-  // Bias scale should be input * per_channel_weight_scale.
-  for (size_t i = 0; i < out_channel_size; i++) {
-    EXPECT_NEAR(
-        bias_params->scales()[i],
-        input_tensor->quantization->scale[0] * weights_params->scales()[i],
-        eps);
-  }
+  ASSERT_TRUE(weights_tensor->quantization);
+  const std::vector<float>& weights_scales =
+      weights_tensor->quantization->scale;
+
+  ASSERT_EQ(weights_scales.size(), out_channel_size);
+  ASSERT_EQ(input_tensor->quantization->scale.size(), 1);
+  ASSERT_EQ(output_tensor->quantization->scale.size(), 1);
 
   const auto bias_buffer = model.buffers[bias_tensor->buffer].get();
   ASSERT_EQ(bias_buffer->data.size(), sizeof(int32_t) * bias_tensor->shape[0]);
@@ -221,10 +204,11 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantization) {
   const float* bias_float_buffer =
       reinterpret_cast<const float*>(original_bias_buffer->data()->data());
 
-  for (size_t i = 0; i < bias_tensor->shape[0]; i++) {
-    auto scale = bias_params->scales()[i];
-    auto dequantized_value = bias_values[i] * scale;
-    EXPECT_NEAR(dequantized_value, bias_float_buffer[i], scale / 2);
+  for (size_t i = 0; i < out_channel_size; i++) {
+    const float bias_scale =
+        input_tensor->quantization->scale[0] * weights_scales[i];
+    auto dequantized_value = bias_values[i] * bias_scale;
+    EXPECT_NEAR(dequantized_value, bias_float_buffer[i], bias_scale / 2);
   }
 
   const auto weights_buffer = model.buffers[weights_tensor->buffer].get();
@@ -240,7 +224,7 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantization) {
   for (size_t channel_idx = 0; channel_idx < out_channel_size; channel_idx++) {
     for (size_t j = 0; j < num_values_in_channel; j++) {
       size_t element_idx = channel_idx * out_channel_size + j;
-      auto scale = weights_params->scales()[channel_idx];
+      auto scale = weights_scales[channel_idx];
       auto dequantized_value = weight_values[element_idx] * scale;
       EXPECT_NEAR(dequantized_value, weights_float_buffer[element_idx],
                   scale / 2);
diff --git a/tensorflow/lite/tools/optimize/symmetric_per_channel_params.cc b/tensorflow/lite/tools/optimize/symmetric_per_channel_params.cc
deleted file mode 100644
index ff7e5657f1..0000000000
--- a/tensorflow/lite/tools/optimize/symmetric_per_channel_params.cc
+++ /dev/null
@@ -1,86 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include "tensorflow/lite/tools/optimize/symmetric_per_channel_params.h"
-
-#include "flatbuffers/flexbuffers.h"
-#include "absl/memory/memory.h"
-#include "tensorflow/lite/schema/schema_generated.h"
-
-namespace tflite {
-namespace optimize {
-namespace internal {
-
-namespace {
-const char* kCustomQuantizationScales = "scales";
-const char* kCustomQuantizationChannelDimIndex = "channel_dim_index";
-
-// Writes scales and dimensions to custom quantization.
-std::vector<uint8_t> SerializeToFlexBuffer(const std::vector<float>& scales,
-                                           int channel_dim_index) {
-  flexbuffers::Builder fbb;
-  fbb.Map([&]() {
-    fbb.Vector(kCustomQuantizationScales, [&]() {
-      for (float scale : scales) {
-        fbb.Float(scale);
-      }
-    });
-    fbb.Int(kCustomQuantizationChannelDimIndex, channel_dim_index);
-  });
-  fbb.Finish();
-  return fbb.GetBuffer();
-}
-
-}  // namespace
-
-/*static*/ TfLiteStatus SymmetricPerChannelParams::ReadFromTensor(
-    const TensorT& tensor, std::unique_ptr<SymmetricPerChannelParams>* params) {
-  const auto* q_params = tensor.quantization.get();
-  if (!q_params) return kTfLiteError;
-  auto custom_quantization = q_params->details.AsCustomQuantization();
-
-  auto buffer = flexbuffers::GetRoot(custom_quantization->custom.data(),
-                                     custom_quantization->custom.size());
-  if (!buffer.IsMap()) {
-    return kTfLiteError;
-  }
-
-  auto custom_quantization_map = buffer.AsMap();
-  auto quant_scales =
-      custom_quantization_map[kCustomQuantizationScales].AsVector();
-  auto channel_dim_index =
-      custom_quantization_map[kCustomQuantizationChannelDimIndex].AsInt32();
-  std::vector<float> scales;
-  scales.reserve(quant_scales.size());
-  for (int i = 0; i < quant_scales.size(); i++) {
-    scales.push_back(quant_scales[i].AsFloat());
-  }
-  *params =
-      absl::make_unique<SymmetricPerChannelParams>(scales, channel_dim_index);
-  return kTfLiteOk;
-}
-
-TfLiteStatus SymmetricPerChannelParams::AddToTensor(TensorT* tensor) const {
-  flatbuffers::FlatBufferBuilder fbb;
-  CustomQuantizationT* custom_quantization = new CustomQuantizationT;
-  custom_quantization->custom =
-      SerializeToFlexBuffer(scales_, channel_dim_index_);
-  tensor->quantization = absl::make_unique<QuantizationParametersT>();
-  tensor->quantization->details.type = QuantizationDetails_CustomQuantization;
-  tensor->quantization->details.value = custom_quantization;
-  return kTfLiteOk;
-}
-}  // namespace internal
-}  // namespace optimize
-}  // namespace tflite
diff --git a/tensorflow/lite/tools/optimize/symmetric_per_channel_params.h b/tensorflow/lite/tools/optimize/symmetric_per_channel_params.h
deleted file mode 100644
index 2ec3ea373a..0000000000
--- a/tensorflow/lite/tools/optimize/symmetric_per_channel_params.h
+++ /dev/null
@@ -1,58 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#ifndef TENSORFLOW_LITE_TOOLS_OPTIMIZE_SYMMETRIC_PER_CHANNEL_PARAMS_H_
-#define TENSORFLOW_LITE_TOOLS_OPTIMIZE_SYMMETRIC_PER_CHANNEL_PARAMS_H_
-
-#include <memory>
-#include <vector>
-
-#include "tensorflow/lite/context.h"
-#include "tensorflow/lite/model.h"
-#include "tensorflow/lite/schema/schema_generated.h"
-
-namespace tflite {
-namespace optimize {
-namespace internal {
-
-// Helper to read/write symmetric per channel parameters to tensors.
-//
-// TODO(shashishekhar): This maybe useful for kernels as well, refactor to
-// kernel utils.
-class SymmetricPerChannelParams {
- public:
-  SymmetricPerChannelParams(const std::vector<float> scales,
-                            int channel_dim_index)
-      : scales_(scales), channel_dim_index_(channel_dim_index) {}
-
-  // Creates a new instance by reading the information from existing tensor.
-  static TfLiteStatus ReadFromTensor(
-      const TensorT& tensor,
-      std::unique_ptr<SymmetricPerChannelParams>* params);
-
-  // Add the symmetric per channel info to the tensor.
-  TfLiteStatus AddToTensor(TensorT* tensor) const;
-
-  const std::vector<float>& scales() const { return scales_; }
-
-  int channel_dim_index() const { return channel_dim_index_; }
-
- private:
-  std::vector<float> scales_;
-  int channel_dim_index_;
-};
-}  // namespace internal
-}  // namespace optimize
-}  // namespace tflite
-#endif  // TENSORFLOW_LITE_TOOLS_OPTIMIZE_SYMMETRIC_PER_CHANNEL_PARAMS_H_
diff --git a/tensorflow/lite/tools/optimize/symmetric_per_channel_params_test.cc b/tensorflow/lite/tools/optimize/symmetric_per_channel_params_test.cc
deleted file mode 100644
index 4d3cbe6f49..0000000000
--- a/tensorflow/lite/tools/optimize/symmetric_per_channel_params_test.cc
+++ /dev/null
@@ -1,51 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include "tensorflow/lite/tools/optimize/symmetric_per_channel_params.h"
-#include <gmock/gmock.h>
-#include <gtest/gtest.h>
-#include "flatbuffers/flexbuffers.h"  // TF:flatbuffers
-#include "tensorflow/lite/schema/schema_generated.h"
-
-namespace tflite {
-namespace optimize {
-namespace internal {
-namespace {
-
-TEST(SymmetricPerChannelParamsTest, TestReadWrite) {
-  TensorT tensor;
-  std::unique_ptr<SymmetricPerChannelParams> read_back_params;
-  const std::vector<float> scales = {3.0, 4.0, 5.0};
-  const int channel_dim_index = 42;
-
-  SymmetricPerChannelParams params(scales, channel_dim_index);
-  params.AddToTensor(&tensor);
-  auto status =
-      SymmetricPerChannelParams::ReadFromTensor(tensor, &read_back_params);
-  EXPECT_EQ(kTfLiteOk, status);
-  ASSERT_TRUE(read_back_params);
-
-  EXPECT_EQ(channel_dim_index, read_back_params->channel_dim_index());
-  ASSERT_EQ(read_back_params->scales(), scales);
-}
-
-}  // namespace
-}  // namespace internal
-}  // namespace optimize
-}  // namespace tflite
-
-int main(int argc, char** argv) {
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}
-- 
GitLab


From dc91b313e86e6bcb8c88a4be2e098296ba6c3174 Mon Sep 17 00:00:00 2001
From: Billy Lamberta <blamb@google.com>
Date: Tue, 15 Jan 2019 18:25:10 -0800
Subject: [PATCH 0764/2345] * Cleaned up prose for clarity. * Refactored
 options example to highlight preferred usage.

PiperOrigin-RevId: 229479188
---
 tensorflow/lite/g3doc/_book.yaml              |   4 +
 .../lite/g3doc/performance/gpu_advanced.md    | 196 ++++++++++--------
 2 files changed, 114 insertions(+), 86 deletions(-)

diff --git a/tensorflow/lite/g3doc/_book.yaml b/tensorflow/lite/g3doc/_book.yaml
index 0c79e79fdd..9c48e1e54d 100644
--- a/tensorflow/lite/g3doc/_book.yaml
+++ b/tensorflow/lite/g3doc/_book.yaml
@@ -59,6 +59,10 @@ upper_tabs:
       - title: Post-training quantization example
         path: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/tutorials/post_training_quant.ipynb
         status: external
+      - title: GPU delegate
+        path: /lite/performance/gpu
+      - title: Advanced GPU
+        path: /lite/performance/gpu_advanced
 
       - title: TF Mobile
         style: accordion
diff --git a/tensorflow/lite/g3doc/performance/gpu_advanced.md b/tensorflow/lite/g3doc/performance/gpu_advanced.md
index d0ae123613..6274948040 100644
--- a/tensorflow/lite/g3doc/performance/gpu_advanced.md
+++ b/tensorflow/lite/g3doc/performance/gpu_advanced.md
@@ -5,24 +5,32 @@ hardware accelerators.  This document describes how to use the GPU backend using
 the TensorFlow Lite delegate APIs on Android (requires OpenGL ES 3.1 or higher)
 and iOS (requires iOS 8 or later).
 
+## Benefits of GPU Acceleration
+
+### Speed
+
 GPUs are designed to have high throughput for massively parallelizable
-workloads.  Thus, they are well-suited for deep neural nets which consists of a
+workloads. Thus, they are well-suited for deep neural nets, which consist of a
 huge number of operators, each working on some input tensor(s) that can be
-easily divided into smaller workloads and carried out in parallel, typically
-resulting in lower latency.  In the best scenario, inference on the GPU may now
-run fast enough and now become suitable for real-time applications if it was not
-before.
-
-GPUs do their computation with 16-bit or 32-bit floating point numbers and do
-not require quantization for optimal performance unlike the CPUs.  If
-quantization of your neural network was not an option due to lower accuracy
-caused by lost precision, such concern can be discarded when running deep neural
-net models on the GPU.
-
-Another benefit that comes with GPU inference is its power efficiency.  GPUs
-carry out the computations in a very efficient and optimized way, so that they
-consume less power and generate less heat than when the same task is run on the
-CPUs.
+easily divided into smaller workloads and carried out in parallel. This
+parallelism typically results in lower latency. In the best scenario, inference
+on the GPU may run fast enough to become suitable for real-time applications
+that were not previously possible.
+
+### Accuracy
+
+GPUs do their computation with 16-bit or 32-bit floating point numbers and
+(unlike the CPUs) do not require quantization for optimal performance. If
+decreased accuracy made quantization untenable for your models, running your
+neural network on a GPU may eliminate this concern.
+
+### Energy Efficiency
+
+Another benefit that comes with GPU inference is its power efficiency. A GPU
+carries out computations in a very efficient and optimized way, consuming less
+power and generating less heat than the same task run on a CPU.
+
+## Supported Ops
 
 TensorFlow Lite on GPU supports the following ops in 16-bit and 32-bit float
 precision:
@@ -51,8 +59,8 @@ precision:
 
 ### Android
 
-Using TensorFlow Lite on GPU is achieved via `TfLiteDelegate`.  In Java, you can
-specify the GpuDelegate through `Interpreter.Options`.
+Run TensorFlow Lite on GPU with `TfLiteDelegate`. In Java, you can specify the
+GpuDelegate through `Interpreter.Options`.
 
 ```java
 // NEW: Prepare GPU delegate.
@@ -73,10 +81,9 @@ delegate.close();
 
 ### iOS
 
-Using TensorFlow Lite on GPU is as simple as getting the GPU delegate via
-`NewGpuDelegate()` and then passing it to
-`Interpreter::ModifyGraphWithDelegate()` instead of calling
-`Interpreter::AllocateTensors()`:
+To use TensorFlow Lite on GPU, get the GPU delegate via `NewGpuDelegate()` and
+then pass it to `Interpreter::ModifyGraphWithDelegate()` (instead of calling
+`Interpreter::AllocateTensors()`).
 
 ```c++
 // Set up interpreter.
@@ -87,7 +94,13 @@ std::unique_ptr<Interpreter> interpreter;
 InterpreterBuilder(*model, op_resolver)(&interpreter);
 
 // NEW: Prepare GPU delegate.
-auto* delegate = NewGpuDelegate(nullptr);  // default config
+
+const GpuDelegateOptions options = {
+  .allow_precision_loss = false,
+  .wait_type = kGpuDelegateOptions::WaitType::Passive,
+};
+
+auto* delegate = NewGpuDelegate(options);
 if (interpreter->ModifyGraphWithDelegate(delegate) != kTfLiteOk) return false;
 
 // Run inference.
@@ -99,17 +112,19 @@ ReadFromOutputTensor(interpreter->typed_output_tensor<float>(0));
 DeleteGpuDelegate(delegate);
 ```
 
-*IMPORTANT:* When calling `Interpreter::ModifyGraphWithDelegate()` or
-`Interpreter::Invoke()`, the caller must have a `EGLContext` in the current
-thread and `Interpreter::Invoke()` must be called from the same `EGLContext`.
-If such `EGLContext` does not exist, the delegate will internally create one,
-but then the developer must ensure that `Interpreter::Invoke()` is always called
-from the same thread `Interpreter::ModifyGraphWithDelegate()` was called.
+Note: When calling `Interpreter::ModifyGraphWithDelegate()` or
+`Interpreter::Invoke()`, the caller must have an `EGLContext` in the current
+thread and `Interpreter::Invoke()` must be called from the same `EGLContext`. If
+an `EGLContext` does not exist, the delegate will internally create one, but
+then the developer must ensure that `Interpreter::Invoke()` is always called
+from the same thread in which `Interpreter::ModifyGraphWithDelegate()` was
+called.
 
-## Advanced: Delegate Options for iOS
+## Advanced Usage
 
-There are a couple of GPU options that can be set and passed on to
-`NewGpuDelegate()`:
+### Delegate Options for iOS
+
+`NewGpuDelegate()` accepts a `struct` of options.
 
 ```c++
 struct GpuDelegateOptions {
@@ -130,45 +145,53 @@ struct GpuDelegateOptions {
 };
 ```
 
-When option is set to `nullptr` as shown in the Basic Usage, it translates to:
+Passing `nullptr` into `NewGpuDelegate()` sets the default options (which are
+explicated in the Basic Usage example above).
 
 ```c++
+
+// THIS:
 const GpuDelegateOptions options = {
   .allow_precision_loss = false,
   .wait_type = kGpuDelegateOptions::WaitType::Passive,
 };
+
+auto* delegate = NewGpuDelegate(options);
+
+// IS THE SAME AS THIS:
+auto* delegate = NewGpuDelegate(nullptr);
+
 ```
 
-While it is convenient to just supply `nullptr`, it is recommended to explicitly
-set the options to avoid any unexpected artifacts in case default values are
-changed.
+While it is convenient to use `nullptr`, we recommend that you explicitly set
+the options, to avoid any unexpected behavior if default values are changed in
+the future.
 
-## Advanced: Input/Output Buffers
+### Input/Output Buffers
 
-To do computation on the GPU, data must be made available to the GPU which often
-translates to performing a memory copy.  It is desirable not to cross the
-CPU/GPU memory boundary if possible, as this can take up a significant amount of
-time.  Usually, such crossing is inevitable, but in some special cases, one or
-the other can be omitted.
+To do computation on the GPU, data must be made available to the GPU. This often
+requires performing a memory copy. It is desirable not to cross the CPU/GPU
+memory boundary if possible, as this can take up a significant amount of time.
+Usually, such crossing is inevitable, but in some special cases, one or the
+other can be omitted.
 
-If the network's input is an image already loaded in the GPU memory, e.g. a GPU
-texture containing the camera feed, it can stay in the GPU memory without ever
-entering the CPU memory.  Similarly, if the network's output is in the form of a
-renderable image, e.g.
-[image style transfer](https://www.cv-foundation.org/openaccess/content_cvpr_2016/papers/Gatys_Image_Style_Transfer_CVPR_2016_paper.pdf),
+If the network's input is an image already loaded in the GPU memory (for
+example, a GPU texture containing the camera feed) it can stay in the GPU memory
+without ever entering the CPU memory. Similarly, if the network's output is in
+the form of a renderable image (for example,
+[image style transfer](https://www.cv-foundation.org/openaccess/content_cvpr_2016/papers/Gatys_Image_Style_Transfer_CVPR_2016_paper.pdf)_)
 it can be directly displayed on the screen.
 
-To let users achieve best performance, TensorFlow Lite makes it possible for
-them to directly read from/write to the TensorFlow hardware buffer and bypass
+To achieve best performance, TensorFlow Lite makes it possible for users to
+directly read from and write to the TensorFlow hardware buffer and bypass
 avoidable memory copies.
 
-### Android
+#### Android
 
-Assuming the image input is in the GPU memory, it must be first converted to a
-OpenGL Shader Storage Buffer Object (SSBO).  One can associate a TfLiteTensor
-with user-prepared SSBO with `Interpreter.bindGlBufferToTensor()`.
-
-*IMPORTANT:* `Interpreter.bindGlBufferToTensor()` must be called before
+Assuming the image input is in the GPU memory, it must first be converted to an
+OpenGL Shader Storage Buffer Object (SSBO). You can associate a TfLiteTensor to
+a user-prepared SSBO with `Interpreter.bindGlBufferToTensor()`. Note that
+`Interpreter.bindGlBufferToTensor()` must be called before
 `Interpreter.modifyGraphWithDelegate()`.
 
 ```java
@@ -176,7 +199,7 @@ with user-prepared SSBO with `Interpreter.bindGlBufferToTensor()`.
 EGLContext eglContext = eglGetCurrentContext();
 if (eglContext.equals(EGL_NO_CONTEXT)) return false;
 
-// Create a SSBO.
+// Create an SSBO.
 int[] id = new int[1];
 glGenBuffers(id.length, id, 0);
 glBindBuffer(GL_SHADER_STORAGE_BUFFER, id[0]);
@@ -198,9 +221,10 @@ float[] outputArray = new float[outputSize];
 interpreter.runInference(null, outputArray);
 ```
 
-Similar approach can be applied to the output tensor.  In that case,
+A similar approach can be applied to the output tensor. In that case,
 `Interpreter.Options.setAllowBufferHandleOutput(true)` should be passed on, to
-disable the default copying the network's output from GPU memory to CPU memory.
+disable the default copying of the network's output from GPU memory to CPU
+memory.
 
 ```java
 // Ensure a valid EGL rendering context.
@@ -230,21 +254,16 @@ interpreter.runInference(input, null);
 renderOutputSsbo(outputSsboId);
 ```
 
-### iOS
-
-Assuming the image input is in the GPU memory, it must be first converted to a
-`MTLBuffer` object for Metal.  One can associate a TfLiteTensor with a
-user-prepared `MTLBuffer` with `BindMetalBufferToTensor()`.
-
-*IMPORTANT:* `BindMetalBufferToTensor()` must be called before
-`Interpreter::ModifyGraphWithDelegate()`.
+#### iOS
 
-*IMPORTANT:* By default, the inference output is copied from GPU memory to CPU
-memory implicitly by the framework.  This behavior can be turned off by calling
-`Interpreter::SetAllowBufferHandleOutput(true)` during initialization.  To copy
-the inference output from GPU memory to CPU memory, explicit
-`Interpreter::EnsureTensorDataIsReadable()` calls are required for each output
-tensor.
+Assuming the image input is in GPU memory, it must first be converted to a
+`MTLBuffer` object for Metal. You can associate a TfLiteTensor to a
+user-prepared `MTLBuffer` with `BindMetalBufferToTensor()`. Note that
+`BindMetalBufferToTensor()` must be called before
+`Interpreter::ModifyGraphWithDelegate()`. Additionally, the inference output is,
+by default, copied from GPU memory to CPU memory. This behavior can be turned
+off by calling `Interpreter::SetAllowBufferHandleOutput(true)` during
+initialization.
 
 ```c++
 // Prepare GPU delegate.
@@ -258,22 +277,27 @@ if (interpreter->ModifyGraphWithDelegate(delegate) != kTfLiteOk) return false;
 if (interpreter->Invoke() != kTfLiteOk) return false;
 ```
 
+Note: Once the default behavior is turned off, copying the inference output from
+GPU memory to CPU memory requires an explicit call to
+`Interpreter::EnsureTensorDataIsReadable()` for each output tensor.
+
 ## Tips and Tricks
 
-* Some operations that are trivial on CPU side may be high cost in GPU land.
-  One class of such operation is various forms of reshape operations (including
-  `BATCH_TO_SPACE`, `SPACE_TO_BATCH`, `SPACE_TO_DEPTH`, etc.).  If those ops
-  are inserted into the network just for the network architect's logical
-  thinking, it is worth removing them for performance.
+*   Some operations that are trivial on the CPU may be high cost on a GPU. One
+    class of such operation includes various forms of reshape operations
+    (including `BATCH_TO_SPACE`, `SPACE_TO_BATCH`, `SPACE_TO_DEPTH`, and similar
+    operation). If these operations are not required (for example, they were
+    inserted to help the network architect reason about the system but do not
+    otherwise affect output), it is worth removing them for performance.
 
-* On GPU, tensor data is sliced into 4-channels.  Thus, a computation on a
-  tensor of shape `[B, H, W, 5]` will perform about the same on a tensor of
-  shape `[B, H, W, 8]`, but significantly worse than `[B, H, W, 4]`.
+*   On a GPU, tensor data is sliced into 4-channels. Thus, a computation on a
+    tensor of shape `[B, H, W, 5]` will perform about the same on a tensor of
+    shape `[B, H, W, 8]`, but significantly worse than `[B, H, W, 4]`.
 
-* In that sense, if the camera hardware supports image frames in RGBA, feeding
-  that 4-channel input is significantly faster as a memory copy (from 3-channel
-  RGB to 4-channel RGBX) can be avoided.
+    *   For example, if the camera hardware supports image frames in RGBA,
+        feeding that 4-channel input is significantly faster, because a memory
+        copy (from 3-channel RGB to 4-channel RGBX) can be avoided.
 
-* For best performance, do not hesitate to re-train your classifier with
-  mobile-optimized network architecture.  That is a significant part of
-  optimization for on-device inference.
+*   For best performance, do not hesitate to re-train your classifier with
+    mobile-optimized network architecture. That is a significant part of
+    optimization for on-device inference.
-- 
GitLab


From 8d89bfeaae9540d60e290254a8a82828656d3e23 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 15 Jan 2019 18:25:17 -0800
Subject: [PATCH 0765/2345] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 229479202

---
 tensorflow/go/op/wrappers.go | 511 +++++++++++++++++++----------------
 1 file changed, 276 insertions(+), 235 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index f8a5b5e7e1..fbabb37c0e 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -6606,6 +6606,27 @@ func DeleteSessionTensor(scope *Scope, handle tf.Output) (o *tf.Operation) {
 	return scope.AddOperation(opspec)
 }
 
+// Store the input tensor in the state of the current session.
+//
+// Arguments:
+//	value: The tensor to be stored.
+//
+// Returns The handle for the tensor stored in the session state, represented
+// as a string.
+func GetSessionHandle(scope *Scope, value tf.Output) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "GetSessionHandle",
+		Input: []tf.Input{
+			value,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Compute the regularized incomplete beta integral \\(I_x(a, b)\\).
 //
 // The regularized incomplete beta integral is defined as:
@@ -11629,220 +11650,6 @@ func OneHot(scope *Scope, indices tf.Output, depth tf.Output, on_value tf.Output
 	return op.Output(0)
 }
 
-// Computes exponential of x element-wise.  \\(y = e^x\\).
-func Exp(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Exp",
-		Input: []tf.Input{
-			x,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// NthElementAttr is an optional argument to NthElement.
-type NthElementAttr func(optionalAttr)
-
-// NthElementReverse sets the optional reverse attribute to value.
-//
-// value: When set to True, find the nth-largest value in the vector and vice
-// versa.
-// If not specified, defaults to false
-func NthElementReverse(value bool) NthElementAttr {
-	return func(m optionalAttr) {
-		m["reverse"] = value
-	}
-}
-
-// Finds values of the `n`-th order statistic for the last dimension.
-//
-// If the input is a vector (rank-1), finds the entries which is the nth-smallest
-// value in the vector and outputs their values as scalar tensor.
-//
-// For matrices (resp. higher rank input), computes the entries which is the
-// nth-smallest value in each row (resp. vector along the last dimension). Thus,
-//
-//     values.shape = input.shape[:-1]
-//
-// Arguments:
-//	input: 1-D or higher with last dimension at least `n+1`.
-//	n: 0-D. Position of sorted vector to select along the last dimension (along
-// each row for matrices). Valid range of n is `[0, input.shape[:-1])`
-//
-// Returns The `n`-th order statistic along each last dimensional slice.
-func NthElement(scope *Scope, input tf.Output, n tf.Output, optional ...NthElementAttr) (values tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "NthElement",
-		Input: []tf.Input{
-			input, n,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes the maximum along segments of a tensor.
-//
-// Read
-// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
-// for an explanation of segments.
-//
-// This operator is similar to the unsorted segment sum operator found
-// [(here)](../../../api_docs/python/math_ops.md#UnsortedSegmentSum).
-// Instead of computing the sum over segments, it computes the maximum such that:
-//
-// \\(output_i = \max_{j...} data[j...]\\) where max is over tuples `j...` such
-// that `segment_ids[j...] == i`.
-//
-// If the maximum is empty for a given segment ID `i`, it outputs the smallest
-// possible value for the specific numeric type,
-// `output[i] = numeric_limits<T>::lowest()`.
-//
-// If the given segment ID `i` is negative, then the corresponding value is
-// dropped, and will not be included in the result.
-//
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/UnsortedSegmentMax.png" alt>
-// </div>
-//
-// Arguments:
-//
-//	segment_ids: A tensor whose shape is a prefix of `data.shape`.
-//
-//
-// Returns Has same shape as data, except for the first `segment_ids.rank`
-// dimensions, which are replaced with a single dimension which has size
-// `num_segments`.
-func UnsortedSegmentMax(scope *Scope, data tf.Output, segment_ids tf.Output, num_segments tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "UnsortedSegmentMax",
-		Input: []tf.Input{
-			data, segment_ids, num_segments,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Transforms a vector of brain.Example protos (as strings) into typed tensors.
-//
-// Arguments:
-//	serialized: A vector containing a batch of binary serialized Example protos.
-//	names: A vector containing the names of the serialized protos.
-// May contain, for example, table key (descriptive) names for the
-// corresponding serialized protos.  These are purely useful for debugging
-// purposes, and the presence of values here has no effect on the output.
-// May also be an empty vector if no names are available.
-// If non-empty, this vector must be the same length as "serialized".
-//	sparse_keys: A list of Nsparse string Tensors (scalars).
-// The keys expected in the Examples' features associated with sparse values.
-//	dense_keys: A list of Ndense string Tensors (scalars).
-// The keys expected in the Examples' features associated with dense values.
-//	dense_defaults: A list of Ndense Tensors (some may be empty).
-// dense_defaults[j] provides default values
-// when the example's feature_map lacks dense_key[j].  If an empty Tensor is
-// provided for dense_defaults[j], then the Feature dense_keys[j] is required.
-// The input type is inferred from dense_defaults[j], even when it's empty.
-// If dense_defaults[j] is not empty, and dense_shapes[j] is fully defined,
-// then the shape of dense_defaults[j] must match that of dense_shapes[j].
-// If dense_shapes[j] has an undefined major dimension (variable strides dense
-// feature), dense_defaults[j] must contain a single element:
-// the padding element.
-//	sparse_types: A list of Nsparse types; the data types of data in each Feature
-// given in sparse_keys.
-// Currently the ParseExample supports DT_FLOAT (FloatList),
-// DT_INT64 (Int64List), and DT_STRING (BytesList).
-//	dense_shapes: A list of Ndense shapes; the shapes of data in each Feature
-// given in dense_keys.
-// The number of elements in the Feature corresponding to dense_key[j]
-// must always equal dense_shapes[j].NumEntries().
-// If dense_shapes[j] == (D0, D1, ..., DN) then the shape of output
-// Tensor dense_values[j] will be (|serialized|, D0, D1, ..., DN):
-// The dense outputs are just the inputs row-stacked by batch.
-// This works for dense_shapes[j] = (-1, D1, ..., DN).  In this case
-// the shape of the output Tensor dense_values[j] will be
-// (|serialized|, M, D1, .., DN), where M is the maximum number of blocks
-// of elements of length D1 * .... * DN, across all minibatch entries
-// in the input.  Any minibatch entry with less than M blocks of elements of
-// length D1 * ... * DN will be padded with the corresponding default_value
-// scalar element along the second dimension.
-func ParseExample(scope *Scope, serialized tf.Output, names tf.Output, sparse_keys []tf.Output, dense_keys []tf.Output, dense_defaults []tf.Output, sparse_types []tf.DataType, dense_shapes []tf.Shape) (sparse_indices []tf.Output, sparse_values []tf.Output, sparse_shapes []tf.Output, dense_values []tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"sparse_types": sparse_types, "dense_shapes": dense_shapes}
-	opspec := tf.OpSpec{
-		Type: "ParseExample",
-		Input: []tf.Input{
-			serialized, names, tf.OutputList(sparse_keys), tf.OutputList(dense_keys), tf.OutputList(dense_defaults),
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if sparse_indices, idx, err = makeOutputList(op, idx, "sparse_indices"); err != nil {
-		scope.UpdateErr("ParseExample", err)
-		return
-	}
-	if sparse_values, idx, err = makeOutputList(op, idx, "sparse_values"); err != nil {
-		scope.UpdateErr("ParseExample", err)
-		return
-	}
-	if sparse_shapes, idx, err = makeOutputList(op, idx, "sparse_shapes"); err != nil {
-		scope.UpdateErr("ParseExample", err)
-		return
-	}
-	if dense_values, idx, err = makeOutputList(op, idx, "dense_values"); err != nil {
-		scope.UpdateErr("ParseExample", err)
-		return
-	}
-	return sparse_indices, sparse_values, sparse_shapes, dense_values
-}
-
-// Compute the pairwise cross product.
-//
-// `a` and `b` must be the same shape; they can either be simple 3-element vectors,
-// or any shape where the innermost dimension is 3. In the latter case, each pair
-// of corresponding 3-element vectors is cross-multiplied independently.
-//
-// Arguments:
-//	a: A tensor containing 3-element vectors.
-//	b: Another tensor, of same type and shape as `a`.
-//
-// Returns Pairwise cross product of the vectors in `a` and `b`.
-func Cross(scope *Scope, a tf.Output, b tf.Output) (product tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Cross",
-		Input: []tf.Input{
-			a, b,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // CudnnRNNAttr is an optional argument to CudnnRNN.
 type CudnnRNNAttr func(optionalAttr)
 
@@ -12696,6 +12503,261 @@ func ResourceApplyFtrl(scope *Scope, var_ tf.Output, accum tf.Output, linear tf.
 	return scope.AddOperation(opspec)
 }
 
+// Computes exponential of x element-wise.  \\(y = e^x\\).
+func Exp(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Exp",
+		Input: []tf.Input{
+			x,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// NthElementAttr is an optional argument to NthElement.
+type NthElementAttr func(optionalAttr)
+
+// NthElementReverse sets the optional reverse attribute to value.
+//
+// value: When set to True, find the nth-largest value in the vector and vice
+// versa.
+// If not specified, defaults to false
+func NthElementReverse(value bool) NthElementAttr {
+	return func(m optionalAttr) {
+		m["reverse"] = value
+	}
+}
+
+// Finds values of the `n`-th order statistic for the last dimension.
+//
+// If the input is a vector (rank-1), finds the entries which is the nth-smallest
+// value in the vector and outputs their values as scalar tensor.
+//
+// For matrices (resp. higher rank input), computes the entries which is the
+// nth-smallest value in each row (resp. vector along the last dimension). Thus,
+//
+//     values.shape = input.shape[:-1]
+//
+// Arguments:
+//	input: 1-D or higher with last dimension at least `n+1`.
+//	n: 0-D. Position of sorted vector to select along the last dimension (along
+// each row for matrices). Valid range of n is `[0, input.shape[:-1])`
+//
+// Returns The `n`-th order statistic along each last dimensional slice.
+func NthElement(scope *Scope, input tf.Output, n tf.Output, optional ...NthElementAttr) (values tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "NthElement",
+		Input: []tf.Input{
+			input, n,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Computes the maximum along segments of a tensor.
+//
+// Read
+// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+// for an explanation of segments.
+//
+// This operator is similar to the unsorted segment sum operator found
+// [(here)](../../../api_docs/python/math_ops.md#UnsortedSegmentSum).
+// Instead of computing the sum over segments, it computes the maximum such that:
+//
+// \\(output_i = \max_{j...} data[j...]\\) where max is over tuples `j...` such
+// that `segment_ids[j...] == i`.
+//
+// If the maximum is empty for a given segment ID `i`, it outputs the smallest
+// possible value for the specific numeric type,
+// `output[i] = numeric_limits<T>::lowest()`.
+//
+// If the given segment ID `i` is negative, then the corresponding value is
+// dropped, and will not be included in the result.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/UnsortedSegmentMax.png" alt>
+// </div>
+//
+// Arguments:
+//
+//	segment_ids: A tensor whose shape is a prefix of `data.shape`.
+//
+//
+// Returns Has same shape as data, except for the first `segment_ids.rank`
+// dimensions, which are replaced with a single dimension which has size
+// `num_segments`.
+func UnsortedSegmentMax(scope *Scope, data tf.Output, segment_ids tf.Output, num_segments tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "UnsortedSegmentMax",
+		Input: []tf.Input{
+			data, segment_ids, num_segments,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Transforms a vector of brain.Example protos (as strings) into typed tensors.
+//
+// Arguments:
+//	serialized: A vector containing a batch of binary serialized Example protos.
+//	names: A vector containing the names of the serialized protos.
+// May contain, for example, table key (descriptive) names for the
+// corresponding serialized protos.  These are purely useful for debugging
+// purposes, and the presence of values here has no effect on the output.
+// May also be an empty vector if no names are available.
+// If non-empty, this vector must be the same length as "serialized".
+//	sparse_keys: A list of Nsparse string Tensors (scalars).
+// The keys expected in the Examples' features associated with sparse values.
+//	dense_keys: A list of Ndense string Tensors (scalars).
+// The keys expected in the Examples' features associated with dense values.
+//	dense_defaults: A list of Ndense Tensors (some may be empty).
+// dense_defaults[j] provides default values
+// when the example's feature_map lacks dense_key[j].  If an empty Tensor is
+// provided for dense_defaults[j], then the Feature dense_keys[j] is required.
+// The input type is inferred from dense_defaults[j], even when it's empty.
+// If dense_defaults[j] is not empty, and dense_shapes[j] is fully defined,
+// then the shape of dense_defaults[j] must match that of dense_shapes[j].
+// If dense_shapes[j] has an undefined major dimension (variable strides dense
+// feature), dense_defaults[j] must contain a single element:
+// the padding element.
+//	sparse_types: A list of Nsparse types; the data types of data in each Feature
+// given in sparse_keys.
+// Currently the ParseExample supports DT_FLOAT (FloatList),
+// DT_INT64 (Int64List), and DT_STRING (BytesList).
+//	dense_shapes: A list of Ndense shapes; the shapes of data in each Feature
+// given in dense_keys.
+// The number of elements in the Feature corresponding to dense_key[j]
+// must always equal dense_shapes[j].NumEntries().
+// If dense_shapes[j] == (D0, D1, ..., DN) then the shape of output
+// Tensor dense_values[j] will be (|serialized|, D0, D1, ..., DN):
+// The dense outputs are just the inputs row-stacked by batch.
+// This works for dense_shapes[j] = (-1, D1, ..., DN).  In this case
+// the shape of the output Tensor dense_values[j] will be
+// (|serialized|, M, D1, .., DN), where M is the maximum number of blocks
+// of elements of length D1 * .... * DN, across all minibatch entries
+// in the input.  Any minibatch entry with less than M blocks of elements of
+// length D1 * ... * DN will be padded with the corresponding default_value
+// scalar element along the second dimension.
+func ParseExample(scope *Scope, serialized tf.Output, names tf.Output, sparse_keys []tf.Output, dense_keys []tf.Output, dense_defaults []tf.Output, sparse_types []tf.DataType, dense_shapes []tf.Shape) (sparse_indices []tf.Output, sparse_values []tf.Output, sparse_shapes []tf.Output, dense_values []tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"sparse_types": sparse_types, "dense_shapes": dense_shapes}
+	opspec := tf.OpSpec{
+		Type: "ParseExample",
+		Input: []tf.Input{
+			serialized, names, tf.OutputList(sparse_keys), tf.OutputList(dense_keys), tf.OutputList(dense_defaults),
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if sparse_indices, idx, err = makeOutputList(op, idx, "sparse_indices"); err != nil {
+		scope.UpdateErr("ParseExample", err)
+		return
+	}
+	if sparse_values, idx, err = makeOutputList(op, idx, "sparse_values"); err != nil {
+		scope.UpdateErr("ParseExample", err)
+		return
+	}
+	if sparse_shapes, idx, err = makeOutputList(op, idx, "sparse_shapes"); err != nil {
+		scope.UpdateErr("ParseExample", err)
+		return
+	}
+	if dense_values, idx, err = makeOutputList(op, idx, "dense_values"); err != nil {
+		scope.UpdateErr("ParseExample", err)
+		return
+	}
+	return sparse_indices, sparse_values, sparse_shapes, dense_values
+}
+
+// Compute the pairwise cross product.
+//
+// `a` and `b` must be the same shape; they can either be simple 3-element vectors,
+// or any shape where the innermost dimension is 3. In the latter case, each pair
+// of corresponding 3-element vectors is cross-multiplied independently.
+//
+// Arguments:
+//	a: A tensor containing 3-element vectors.
+//	b: Another tensor, of same type and shape as `a`.
+//
+// Returns Pairwise cross product of the vectors in `a` and `b`.
+func Cross(scope *Scope, a tf.Output, b tf.Output) (product tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Cross",
+		Input: []tf.Input{
+			a, b,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// StatefulStandardNormalAttr is an optional argument to StatefulStandardNormal.
+type StatefulStandardNormalAttr func(optionalAttr)
+
+// StatefulStandardNormalDtype sets the optional dtype attribute to value.
+//
+// value: The type of the output.
+// If not specified, defaults to DT_FLOAT
+func StatefulStandardNormalDtype(value tf.DataType) StatefulStandardNormalAttr {
+	return func(m optionalAttr) {
+		m["dtype"] = value
+	}
+}
+
+// Outputs random values from a normal distribution.
+//
+// The generated values will have mean 0 and standard deviation 1.
+//
+// Arguments:
+//	resource: The handle of the resource variable that stores the state of the RNG.
+//	shape: The shape of the output tensor.
+//
+// Returns A tensor of the specified shape filled with random normal values.
+func StatefulStandardNormal(scope *Scope, resource tf.Output, shape tf.Output, optional ...StatefulStandardNormalAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "StatefulStandardNormal",
+		Input: []tf.Input{
+			resource, shape,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Locks a mutex resource.  The output is the lock.  So long as the lock tensor
 //
 // is alive, any other request to use `MutexLock` with this mutex will wait.
@@ -17519,27 +17581,6 @@ func DecodeBase64(scope *Scope, input tf.Output) (output tf.Output) {
 	return op.Output(0)
 }
 
-// Store the input tensor in the state of the current session.
-//
-// Arguments:
-//	value: The tensor to be stored.
-//
-// Returns The handle for the tensor stored in the session state, represented
-// as a string.
-func GetSessionHandle(scope *Scope, value tf.Output) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "GetSessionHandle",
-		Input: []tf.Input{
-			value,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // ResourceSparseApplyProximalAdagradAttr is an optional argument to ResourceSparseApplyProximalAdagrad.
 type ResourceSparseApplyProximalAdagradAttr func(optionalAttr)
 
-- 
GitLab


From 492a0e17ae125c17fb5a0d0406e7c255fec835b6 Mon Sep 17 00:00:00 2001
From: tomguluson92 <314913739@qq.com>
Date: Wed, 16 Jan 2019 11:07:16 +0800
Subject: [PATCH 0766/2345] fix spell mistake

---
 tensorflow/python/util/nest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/util/nest.py b/tensorflow/python/util/nest.py
index 70e5ebb3b6..dbae97a373 100644
--- a/tensorflow/python/util/nest.py
+++ b/tensorflow/python/util/nest.py
@@ -149,7 +149,7 @@ def assert_same_structure(nest1, nest2, check_types=True):
 
   Note that namedtuples with identical name and fields are always considered
   to have the same shallow structure (even with `check_types=True`).
-  For intance, this code will print `True`:
+  For instance, this code will print `True`:
 
   ```python
   def nt(a, b):
-- 
GitLab


From 1f2952ec47015a300184e340d6f53aa42eb6eadf Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Tue, 15 Jan 2019 19:23:23 -0800
Subject: [PATCH 0767/2345] Update the MobileNet models used in iOS examples.

PiperOrigin-RevId: 229484657
---
 .../lite/examples/ios/camera/data/labels.txt  | 1001 +++++++++++++++++
 .../lite/examples/ios/download_models.sh      |   43 +-
 .../lite/examples/ios/simple/data/labels.txt  | 1001 +++++++++++++++++
 3 files changed, 2018 insertions(+), 27 deletions(-)
 create mode 100644 tensorflow/lite/examples/ios/camera/data/labels.txt
 create mode 100644 tensorflow/lite/examples/ios/simple/data/labels.txt

diff --git a/tensorflow/lite/examples/ios/camera/data/labels.txt b/tensorflow/lite/examples/ios/camera/data/labels.txt
new file mode 100644
index 0000000000..572eccf900
--- /dev/null
+++ b/tensorflow/lite/examples/ios/camera/data/labels.txt
@@ -0,0 +1,1001 @@
+dummy
+tench
+goldfish
+great white shark
+tiger shark
+hammerhead
+electric ray
+stingray
+cock
+hen
+ostrich
+brambling
+goldfinch
+house finch
+junco
+indigo bunting
+robin
+bulbul
+jay
+magpie
+chickadee
+water ouzel
+kite
+bald eagle
+vulture
+great grey owl
+European fire salamander
+common newt
+eft
+spotted salamander
+axolotl
+bullfrog
+tree frog
+tailed frog
+loggerhead
+leatherback turtle
+mud turtle
+terrapin
+box turtle
+banded gecko
+common iguana
+American chameleon
+whiptail
+agama
+frilled lizard
+alligator lizard
+Gila monster
+green lizard
+African chameleon
+Komodo dragon
+African crocodile
+American alligator
+triceratops
+thunder snake
+ringneck snake
+hognose snake
+green snake
+king snake
+garter snake
+water snake
+vine snake
+night snake
+boa constrictor
+rock python
+Indian cobra
+green mamba
+sea snake
+horned viper
+diamondback
+sidewinder
+trilobite
+harvestman
+scorpion
+black and gold garden spider
+barn spider
+garden spider
+black widow
+tarantula
+wolf spider
+tick
+centipede
+black grouse
+ptarmigan
+ruffed grouse
+prairie chicken
+peacock
+quail
+partridge
+African grey
+macaw
+sulphur-crested cockatoo
+lorikeet
+coucal
+bee eater
+hornbill
+hummingbird
+jacamar
+toucan
+drake
+red-breasted merganser
+goose
+black swan
+tusker
+echidna
+platypus
+wallaby
+koala
+wombat
+jellyfish
+sea anemone
+brain coral
+flatworm
+nematode
+conch
+snail
+slug
+sea slug
+chiton
+chambered nautilus
+Dungeness crab
+rock crab
+fiddler crab
+king crab
+American lobster
+spiny lobster
+crayfish
+hermit crab
+isopod
+white stork
+black stork
+spoonbill
+flamingo
+little blue heron
+American egret
+bittern
+crane
+limpkin
+European gallinule
+American coot
+bustard
+ruddy turnstone
+red-backed sandpiper
+redshank
+dowitcher
+oystercatcher
+pelican
+king penguin
+albatross
+grey whale
+killer whale
+dugong
+sea lion
+Chihuahua
+Japanese spaniel
+Maltese dog
+Pekinese
+Shih-Tzu
+Blenheim spaniel
+papillon
+toy terrier
+Rhodesian ridgeback
+Afghan hound
+basset
+beagle
+bloodhound
+bluetick
+black-and-tan coonhound
+Walker hound
+English foxhound
+redbone
+borzoi
+Irish wolfhound
+Italian greyhound
+whippet
+Ibizan hound
+Norwegian elkhound
+otterhound
+Saluki
+Scottish deerhound
+Weimaraner
+Staffordshire bullterrier
+American Staffordshire terrier
+Bedlington terrier
+Border terrier
+Kerry blue terrier
+Irish terrier
+Norfolk terrier
+Norwich terrier
+Yorkshire terrier
+wire-haired fox terrier
+Lakeland terrier
+Sealyham terrier
+Airedale
+cairn
+Australian terrier
+Dandie Dinmont
+Boston bull
+miniature schnauzer
+giant schnauzer
+standard schnauzer
+Scotch terrier
+Tibetan terrier
+silky terrier
+soft-coated wheaten terrier
+West Highland white terrier
+Lhasa
+flat-coated retriever
+curly-coated retriever
+golden retriever
+Labrador retriever
+Chesapeake Bay retriever
+German short-haired pointer
+vizsla
+English setter
+Irish setter
+Gordon setter
+Brittany spaniel
+clumber
+English springer
+Welsh springer spaniel
+cocker spaniel
+Sussex spaniel
+Irish water spaniel
+kuvasz
+schipperke
+groenendael
+malinois
+briard
+kelpie
+komondor
+Old English sheepdog
+Shetland sheepdog
+collie
+Border collie
+Bouvier des Flandres
+Rottweiler
+German shepherd
+Doberman
+miniature pinscher
+Greater Swiss Mountain dog
+Bernese mountain dog
+Appenzeller
+EntleBucher
+boxer
+bull mastiff
+Tibetan mastiff
+French bulldog
+Great Dane
+Saint Bernard
+Eskimo dog
+malamute
+Siberian husky
+dalmatian
+affenpinscher
+basenji
+pug
+Leonberg
+Newfoundland
+Great Pyrenees
+Samoyed
+Pomeranian
+chow
+keeshond
+Brabancon griffon
+Pembroke
+Cardigan
+toy poodle
+miniature poodle
+standard poodle
+Mexican hairless
+timber wolf
+white wolf
+red wolf
+coyote
+dingo
+dhole
+African hunting dog
+hyena
+red fox
+kit fox
+Arctic fox
+grey fox
+tabby
+tiger cat
+Persian cat
+Siamese cat
+Egyptian cat
+cougar
+lynx
+leopard
+snow leopard
+jaguar
+lion
+tiger
+cheetah
+brown bear
+American black bear
+ice bear
+sloth bear
+mongoose
+meerkat
+tiger beetle
+ladybug
+ground beetle
+long-horned beetle
+leaf beetle
+dung beetle
+rhinoceros beetle
+weevil
+fly
+bee
+ant
+grasshopper
+cricket
+walking stick
+cockroach
+mantis
+cicada
+leafhopper
+lacewing
+dragonfly
+damselfly
+admiral
+ringlet
+monarch
+cabbage butterfly
+sulphur butterfly
+lycaenid
+starfish
+sea urchin
+sea cucumber
+wood rabbit
+hare
+Angora
+hamster
+porcupine
+fox squirrel
+marmot
+beaver
+guinea pig
+sorrel
+zebra
+hog
+wild boar
+warthog
+hippopotamus
+ox
+water buffalo
+bison
+ram
+bighorn
+ibex
+hartebeest
+impala
+gazelle
+Arabian camel
+llama
+weasel
+mink
+polecat
+black-footed ferret
+otter
+skunk
+badger
+armadillo
+three-toed sloth
+orangutan
+gorilla
+chimpanzee
+gibbon
+siamang
+guenon
+patas
+baboon
+macaque
+langur
+colobus
+proboscis monkey
+marmoset
+capuchin
+howler monkey
+titi
+spider monkey
+squirrel monkey
+Madagascar cat
+indri
+Indian elephant
+African elephant
+lesser panda
+giant panda
+barracouta
+eel
+coho
+rock beauty
+anemone fish
+sturgeon
+gar
+lionfish
+puffer
+abacus
+abaya
+academic gown
+accordion
+acoustic guitar
+aircraft carrier
+airliner
+airship
+altar
+ambulance
+amphibian
+analog clock
+apiary
+apron
+ashcan
+assault rifle
+backpack
+bakery
+balance beam
+balloon
+ballpoint
+Band Aid
+banjo
+bannister
+barbell
+barber chair
+barbershop
+barn
+barometer
+barrel
+barrow
+baseball
+basketball
+bassinet
+bassoon
+bathing cap
+bath towel
+bathtub
+beach wagon
+beacon
+beaker
+bearskin
+beer bottle
+beer glass
+bell cote
+bib
+bicycle-built-for-two
+bikini
+binder
+binoculars
+birdhouse
+boathouse
+bobsled
+bolo tie
+bonnet
+bookcase
+bookshop
+bottlecap
+bow
+bow tie
+brass
+brassiere
+breakwater
+breastplate
+broom
+bucket
+buckle
+bulletproof vest
+bullet train
+butcher shop
+cab
+caldron
+candle
+cannon
+canoe
+can opener
+cardigan
+car mirror
+carousel
+carpenter's kit
+carton
+car wheel
+cash machine
+cassette
+cassette player
+castle
+catamaran
+CD player
+cello
+cellular telephone
+chain
+chainlink fence
+chain mail
+chain saw
+chest
+chiffonier
+chime
+china cabinet
+Christmas stocking
+church
+cinema
+cleaver
+cliff dwelling
+cloak
+clog
+cocktail shaker
+coffee mug
+coffeepot
+coil
+combination lock
+computer keyboard
+confectionery
+container ship
+convertible
+corkscrew
+cornet
+cowboy boot
+cowboy hat
+cradle
+crane
+crash helmet
+crate
+crib
+Crock Pot
+croquet ball
+crutch
+cuirass
+dam
+desk
+desktop computer
+dial telephone
+diaper
+digital clock
+digital watch
+dining table
+dishrag
+dishwasher
+disk brake
+dock
+dogsled
+dome
+doormat
+drilling platform
+drum
+drumstick
+dumbbell
+Dutch oven
+electric fan
+electric guitar
+electric locomotive
+entertainment center
+envelope
+espresso maker
+face powder
+feather boa
+file
+fireboat
+fire engine
+fire screen
+flagpole
+flute
+folding chair
+football helmet
+forklift
+fountain
+fountain pen
+four-poster
+freight car
+French horn
+frying pan
+fur coat
+garbage truck
+gasmask
+gas pump
+goblet
+go-kart
+golf ball
+golfcart
+gondola
+gong
+gown
+grand piano
+greenhouse
+grille
+grocery store
+guillotine
+hair slide
+hair spray
+half track
+hammer
+hamper
+hand blower
+hand-held computer
+handkerchief
+hard disc
+harmonica
+harp
+harvester
+hatchet
+holster
+home theater
+honeycomb
+hook
+hoopskirt
+horizontal bar
+horse cart
+hourglass
+iPod
+iron
+jack-o'-lantern
+jean
+jeep
+jersey
+jigsaw puzzle
+jinrikisha
+joystick
+kimono
+knee pad
+knot
+lab coat
+ladle
+lampshade
+laptop
+lawn mower
+lens cap
+letter opener
+library
+lifeboat
+lighter
+limousine
+liner
+lipstick
+Loafer
+lotion
+loudspeaker
+loupe
+lumbermill
+magnetic compass
+mailbag
+mailbox
+maillot
+maillot
+manhole cover
+maraca
+marimba
+mask
+matchstick
+maypole
+maze
+measuring cup
+medicine chest
+megalith
+microphone
+microwave
+military uniform
+milk can
+minibus
+miniskirt
+minivan
+missile
+mitten
+mixing bowl
+mobile home
+Model T
+modem
+monastery
+monitor
+moped
+mortar
+mortarboard
+mosque
+mosquito net
+motor scooter
+mountain bike
+mountain tent
+mouse
+mousetrap
+moving van
+muzzle
+nail
+neck brace
+necklace
+nipple
+notebook
+obelisk
+oboe
+ocarina
+odometer
+oil filter
+organ
+oscilloscope
+overskirt
+oxcart
+oxygen mask
+packet
+paddle
+paddlewheel
+padlock
+paintbrush
+pajama
+palace
+panpipe
+paper towel
+parachute
+parallel bars
+park bench
+parking meter
+passenger car
+patio
+pay-phone
+pedestal
+pencil box
+pencil sharpener
+perfume
+Petri dish
+photocopier
+pick
+pickelhaube
+picket fence
+pickup
+pier
+piggy bank
+pill bottle
+pillow
+ping-pong ball
+pinwheel
+pirate
+pitcher
+plane
+planetarium
+plastic bag
+plate rack
+plow
+plunger
+Polaroid camera
+pole
+police van
+poncho
+pool table
+pop bottle
+pot
+potter's wheel
+power drill
+prayer rug
+printer
+prison
+projectile
+projector
+puck
+punching bag
+purse
+quill
+quilt
+racer
+racket
+radiator
+radio
+radio telescope
+rain barrel
+recreational vehicle
+reel
+reflex camera
+refrigerator
+remote control
+restaurant
+revolver
+rifle
+rocking chair
+rotisserie
+rubber eraser
+rugby ball
+rule
+running shoe
+safe
+safety pin
+saltshaker
+sandal
+sarong
+sax
+scabbard
+scale
+school bus
+schooner
+scoreboard
+screen
+screw
+screwdriver
+seat belt
+sewing machine
+shield
+shoe shop
+shoji
+shopping basket
+shopping cart
+shovel
+shower cap
+shower curtain
+ski
+ski mask
+sleeping bag
+slide rule
+sliding door
+slot
+snorkel
+snowmobile
+snowplow
+soap dispenser
+soccer ball
+sock
+solar dish
+sombrero
+soup bowl
+space bar
+space heater
+space shuttle
+spatula
+speedboat
+spider web
+spindle
+sports car
+spotlight
+stage
+steam locomotive
+steel arch bridge
+steel drum
+stethoscope
+stole
+stone wall
+stopwatch
+stove
+strainer
+streetcar
+stretcher
+studio couch
+stupa
+submarine
+suit
+sundial
+sunglass
+sunglasses
+sunscreen
+suspension bridge
+swab
+sweatshirt
+swimming trunks
+swing
+switch
+syringe
+table lamp
+tank
+tape player
+teapot
+teddy
+television
+tennis ball
+thatch
+theater curtain
+thimble
+thresher
+throne
+tile roof
+toaster
+tobacco shop
+toilet seat
+torch
+totem pole
+tow truck
+toyshop
+tractor
+trailer truck
+tray
+trench coat
+tricycle
+trimaran
+tripod
+triumphal arch
+trolleybus
+trombone
+tub
+turnstile
+typewriter keyboard
+umbrella
+unicycle
+upright
+vacuum
+vase
+vault
+velvet
+vending machine
+vestment
+viaduct
+violin
+volleyball
+waffle iron
+wall clock
+wallet
+wardrobe
+warplane
+washbasin
+washer
+water bottle
+water jug
+water tower
+whiskey jug
+whistle
+wig
+window screen
+window shade
+Windsor tie
+wine bottle
+wing
+wok
+wooden spoon
+wool
+worm fence
+wreck
+yawl
+yurt
+web site
+comic book
+crossword puzzle
+street sign
+traffic light
+book jacket
+menu
+plate
+guacamole
+consomme
+hot pot
+trifle
+ice cream
+ice lolly
+French loaf
+bagel
+pretzel
+cheeseburger
+hotdog
+mashed potato
+head cabbage
+broccoli
+cauliflower
+zucchini
+spaghetti squash
+acorn squash
+butternut squash
+cucumber
+artichoke
+bell pepper
+cardoon
+mushroom
+Granny Smith
+strawberry
+orange
+lemon
+fig
+pineapple
+banana
+jackfruit
+custard apple
+pomegranate
+hay
+carbonara
+chocolate sauce
+dough
+meat loaf
+pizza
+potpie
+burrito
+red wine
+espresso
+cup
+eggnog
+alp
+bubble
+cliff
+coral reef
+geyser
+lakeside
+promontory
+sandbar
+seashore
+valley
+volcano
+ballplayer
+groom
+scuba diver
+rapeseed
+daisy
+yellow lady's slipper
+corn
+acorn
+hip
+buckeye
+coral fungus
+agaric
+gyromitra
+stinkhorn
+earthstar
+hen-of-the-woods
+bolete
+ear
+toilet tissue
diff --git a/tensorflow/lite/examples/ios/download_models.sh b/tensorflow/lite/examples/ios/download_models.sh
index 4828617d95..a450aba042 100755
--- a/tensorflow/lite/examples/ios/download_models.sh
+++ b/tensorflow/lite/examples/ios/download_models.sh
@@ -17,42 +17,31 @@
 set -ex
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-MODELS_URL="https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_1.0_224_ios_lite_float_2017_11_08.zip"
-QUANTIZED_MODELS_URL="https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_224_android_quant_2017_11_08.zip"
+FLOAT_MODEL_URL="http://download.tensorflow.org/models/mobilenet_v1_2018_02_22/mobilenet_v1_1.0_224.tgz"
+QUANTIZED_MODEL_URL="http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_224_quant.tgz"
 DOWNLOADS_DIR=$(mktemp -d)
 
-cd $SCRIPT_DIR
+cd "$SCRIPT_DIR"
 
 download_and_extract() {
-  local usage="Usage: download_and_extract URL DIR"
-  local url="${1:?${usage}}"
-  local dir="${2:?${usage}}"
+  local url="$1"
+  local dir="$2"
   echo "downloading ${url}" >&2
   mkdir -p "${dir}"
   tempdir=$(mktemp -d)
-  tempdir2=$(mktemp -d)
 
-  curl -L ${url} > ${tempdir}/zipped.zip
-  unzip ${tempdir}/zipped.zip -d ${tempdir2}
-
-  # If the zip file contains nested directories, extract the files from the
-  # inner directory.
-  if ls ${tempdir2}/*/* 1> /dev/null 2>&1; then
-    # unzip has no strip components, so unzip to a temp dir, and move the
-    # files we want from the tempdir to destination.
-    cp -R ${tempdir2}/*/* ${dir}/
-  else
-    cp -R ${tempdir2}/* ${dir}/
-  fi
-  rm -rf ${tempdir2} ${tempdir}
+  curl -L ${url} > ${tempdir}/archive.tgz
+  cd ${dir}
+  tar zxvf ${tempdir}/archive.tgz
+  rm -rf ${tempdir}
 }
 
-download_and_extract "${MODELS_URL}" "${DOWNLOADS_DIR}/models"
-download_and_extract "${QUANTIZED_MODELS_URL}" "${DOWNLOADS_DIR}/quantized_models"
-
-file ${DOWNLOADS_DIR}/models
+download_and_extract "${FLOAT_MODEL_URL}" "${DOWNLOADS_DIR}/float_model"
+download_and_extract "${QUANTIZED_MODEL_URL}" "${DOWNLOADS_DIR}/quantized_model"
 
-cp ${DOWNLOADS_DIR}/models/models/* simple/data/
-cp ${DOWNLOADS_DIR}/models/models/* camera/data/
-cp "${DOWNLOADS_DIR}/quantized_models/mobilenet_quant_v1_224.tflite" \
+cd "$SCRIPT_DIR"
+cp "${DOWNLOADS_DIR}/float_model/mobilenet_v1_1.0_224.tflite" "simple/data/mobilenet_v1_1.0_224.tflite"
+cp "${DOWNLOADS_DIR}/float_model/mobilenet_v1_1.0_224.tflite" "camera/data/mobilenet_v1_1.0_224.tflite"
+cp "${DOWNLOADS_DIR}/quantized_model/mobilenet_v1_1.0_224_quant.tflite" \
    'camera/data/mobilenet_quant_v1_224.tflite'
+echo "Done"
diff --git a/tensorflow/lite/examples/ios/simple/data/labels.txt b/tensorflow/lite/examples/ios/simple/data/labels.txt
new file mode 100644
index 0000000000..572eccf900
--- /dev/null
+++ b/tensorflow/lite/examples/ios/simple/data/labels.txt
@@ -0,0 +1,1001 @@
+dummy
+tench
+goldfish
+great white shark
+tiger shark
+hammerhead
+electric ray
+stingray
+cock
+hen
+ostrich
+brambling
+goldfinch
+house finch
+junco
+indigo bunting
+robin
+bulbul
+jay
+magpie
+chickadee
+water ouzel
+kite
+bald eagle
+vulture
+great grey owl
+European fire salamander
+common newt
+eft
+spotted salamander
+axolotl
+bullfrog
+tree frog
+tailed frog
+loggerhead
+leatherback turtle
+mud turtle
+terrapin
+box turtle
+banded gecko
+common iguana
+American chameleon
+whiptail
+agama
+frilled lizard
+alligator lizard
+Gila monster
+green lizard
+African chameleon
+Komodo dragon
+African crocodile
+American alligator
+triceratops
+thunder snake
+ringneck snake
+hognose snake
+green snake
+king snake
+garter snake
+water snake
+vine snake
+night snake
+boa constrictor
+rock python
+Indian cobra
+green mamba
+sea snake
+horned viper
+diamondback
+sidewinder
+trilobite
+harvestman
+scorpion
+black and gold garden spider
+barn spider
+garden spider
+black widow
+tarantula
+wolf spider
+tick
+centipede
+black grouse
+ptarmigan
+ruffed grouse
+prairie chicken
+peacock
+quail
+partridge
+African grey
+macaw
+sulphur-crested cockatoo
+lorikeet
+coucal
+bee eater
+hornbill
+hummingbird
+jacamar
+toucan
+drake
+red-breasted merganser
+goose
+black swan
+tusker
+echidna
+platypus
+wallaby
+koala
+wombat
+jellyfish
+sea anemone
+brain coral
+flatworm
+nematode
+conch
+snail
+slug
+sea slug
+chiton
+chambered nautilus
+Dungeness crab
+rock crab
+fiddler crab
+king crab
+American lobster
+spiny lobster
+crayfish
+hermit crab
+isopod
+white stork
+black stork
+spoonbill
+flamingo
+little blue heron
+American egret
+bittern
+crane
+limpkin
+European gallinule
+American coot
+bustard
+ruddy turnstone
+red-backed sandpiper
+redshank
+dowitcher
+oystercatcher
+pelican
+king penguin
+albatross
+grey whale
+killer whale
+dugong
+sea lion
+Chihuahua
+Japanese spaniel
+Maltese dog
+Pekinese
+Shih-Tzu
+Blenheim spaniel
+papillon
+toy terrier
+Rhodesian ridgeback
+Afghan hound
+basset
+beagle
+bloodhound
+bluetick
+black-and-tan coonhound
+Walker hound
+English foxhound
+redbone
+borzoi
+Irish wolfhound
+Italian greyhound
+whippet
+Ibizan hound
+Norwegian elkhound
+otterhound
+Saluki
+Scottish deerhound
+Weimaraner
+Staffordshire bullterrier
+American Staffordshire terrier
+Bedlington terrier
+Border terrier
+Kerry blue terrier
+Irish terrier
+Norfolk terrier
+Norwich terrier
+Yorkshire terrier
+wire-haired fox terrier
+Lakeland terrier
+Sealyham terrier
+Airedale
+cairn
+Australian terrier
+Dandie Dinmont
+Boston bull
+miniature schnauzer
+giant schnauzer
+standard schnauzer
+Scotch terrier
+Tibetan terrier
+silky terrier
+soft-coated wheaten terrier
+West Highland white terrier
+Lhasa
+flat-coated retriever
+curly-coated retriever
+golden retriever
+Labrador retriever
+Chesapeake Bay retriever
+German short-haired pointer
+vizsla
+English setter
+Irish setter
+Gordon setter
+Brittany spaniel
+clumber
+English springer
+Welsh springer spaniel
+cocker spaniel
+Sussex spaniel
+Irish water spaniel
+kuvasz
+schipperke
+groenendael
+malinois
+briard
+kelpie
+komondor
+Old English sheepdog
+Shetland sheepdog
+collie
+Border collie
+Bouvier des Flandres
+Rottweiler
+German shepherd
+Doberman
+miniature pinscher
+Greater Swiss Mountain dog
+Bernese mountain dog
+Appenzeller
+EntleBucher
+boxer
+bull mastiff
+Tibetan mastiff
+French bulldog
+Great Dane
+Saint Bernard
+Eskimo dog
+malamute
+Siberian husky
+dalmatian
+affenpinscher
+basenji
+pug
+Leonberg
+Newfoundland
+Great Pyrenees
+Samoyed
+Pomeranian
+chow
+keeshond
+Brabancon griffon
+Pembroke
+Cardigan
+toy poodle
+miniature poodle
+standard poodle
+Mexican hairless
+timber wolf
+white wolf
+red wolf
+coyote
+dingo
+dhole
+African hunting dog
+hyena
+red fox
+kit fox
+Arctic fox
+grey fox
+tabby
+tiger cat
+Persian cat
+Siamese cat
+Egyptian cat
+cougar
+lynx
+leopard
+snow leopard
+jaguar
+lion
+tiger
+cheetah
+brown bear
+American black bear
+ice bear
+sloth bear
+mongoose
+meerkat
+tiger beetle
+ladybug
+ground beetle
+long-horned beetle
+leaf beetle
+dung beetle
+rhinoceros beetle
+weevil
+fly
+bee
+ant
+grasshopper
+cricket
+walking stick
+cockroach
+mantis
+cicada
+leafhopper
+lacewing
+dragonfly
+damselfly
+admiral
+ringlet
+monarch
+cabbage butterfly
+sulphur butterfly
+lycaenid
+starfish
+sea urchin
+sea cucumber
+wood rabbit
+hare
+Angora
+hamster
+porcupine
+fox squirrel
+marmot
+beaver
+guinea pig
+sorrel
+zebra
+hog
+wild boar
+warthog
+hippopotamus
+ox
+water buffalo
+bison
+ram
+bighorn
+ibex
+hartebeest
+impala
+gazelle
+Arabian camel
+llama
+weasel
+mink
+polecat
+black-footed ferret
+otter
+skunk
+badger
+armadillo
+three-toed sloth
+orangutan
+gorilla
+chimpanzee
+gibbon
+siamang
+guenon
+patas
+baboon
+macaque
+langur
+colobus
+proboscis monkey
+marmoset
+capuchin
+howler monkey
+titi
+spider monkey
+squirrel monkey
+Madagascar cat
+indri
+Indian elephant
+African elephant
+lesser panda
+giant panda
+barracouta
+eel
+coho
+rock beauty
+anemone fish
+sturgeon
+gar
+lionfish
+puffer
+abacus
+abaya
+academic gown
+accordion
+acoustic guitar
+aircraft carrier
+airliner
+airship
+altar
+ambulance
+amphibian
+analog clock
+apiary
+apron
+ashcan
+assault rifle
+backpack
+bakery
+balance beam
+balloon
+ballpoint
+Band Aid
+banjo
+bannister
+barbell
+barber chair
+barbershop
+barn
+barometer
+barrel
+barrow
+baseball
+basketball
+bassinet
+bassoon
+bathing cap
+bath towel
+bathtub
+beach wagon
+beacon
+beaker
+bearskin
+beer bottle
+beer glass
+bell cote
+bib
+bicycle-built-for-two
+bikini
+binder
+binoculars
+birdhouse
+boathouse
+bobsled
+bolo tie
+bonnet
+bookcase
+bookshop
+bottlecap
+bow
+bow tie
+brass
+brassiere
+breakwater
+breastplate
+broom
+bucket
+buckle
+bulletproof vest
+bullet train
+butcher shop
+cab
+caldron
+candle
+cannon
+canoe
+can opener
+cardigan
+car mirror
+carousel
+carpenter's kit
+carton
+car wheel
+cash machine
+cassette
+cassette player
+castle
+catamaran
+CD player
+cello
+cellular telephone
+chain
+chainlink fence
+chain mail
+chain saw
+chest
+chiffonier
+chime
+china cabinet
+Christmas stocking
+church
+cinema
+cleaver
+cliff dwelling
+cloak
+clog
+cocktail shaker
+coffee mug
+coffeepot
+coil
+combination lock
+computer keyboard
+confectionery
+container ship
+convertible
+corkscrew
+cornet
+cowboy boot
+cowboy hat
+cradle
+crane
+crash helmet
+crate
+crib
+Crock Pot
+croquet ball
+crutch
+cuirass
+dam
+desk
+desktop computer
+dial telephone
+diaper
+digital clock
+digital watch
+dining table
+dishrag
+dishwasher
+disk brake
+dock
+dogsled
+dome
+doormat
+drilling platform
+drum
+drumstick
+dumbbell
+Dutch oven
+electric fan
+electric guitar
+electric locomotive
+entertainment center
+envelope
+espresso maker
+face powder
+feather boa
+file
+fireboat
+fire engine
+fire screen
+flagpole
+flute
+folding chair
+football helmet
+forklift
+fountain
+fountain pen
+four-poster
+freight car
+French horn
+frying pan
+fur coat
+garbage truck
+gasmask
+gas pump
+goblet
+go-kart
+golf ball
+golfcart
+gondola
+gong
+gown
+grand piano
+greenhouse
+grille
+grocery store
+guillotine
+hair slide
+hair spray
+half track
+hammer
+hamper
+hand blower
+hand-held computer
+handkerchief
+hard disc
+harmonica
+harp
+harvester
+hatchet
+holster
+home theater
+honeycomb
+hook
+hoopskirt
+horizontal bar
+horse cart
+hourglass
+iPod
+iron
+jack-o'-lantern
+jean
+jeep
+jersey
+jigsaw puzzle
+jinrikisha
+joystick
+kimono
+knee pad
+knot
+lab coat
+ladle
+lampshade
+laptop
+lawn mower
+lens cap
+letter opener
+library
+lifeboat
+lighter
+limousine
+liner
+lipstick
+Loafer
+lotion
+loudspeaker
+loupe
+lumbermill
+magnetic compass
+mailbag
+mailbox
+maillot
+maillot
+manhole cover
+maraca
+marimba
+mask
+matchstick
+maypole
+maze
+measuring cup
+medicine chest
+megalith
+microphone
+microwave
+military uniform
+milk can
+minibus
+miniskirt
+minivan
+missile
+mitten
+mixing bowl
+mobile home
+Model T
+modem
+monastery
+monitor
+moped
+mortar
+mortarboard
+mosque
+mosquito net
+motor scooter
+mountain bike
+mountain tent
+mouse
+mousetrap
+moving van
+muzzle
+nail
+neck brace
+necklace
+nipple
+notebook
+obelisk
+oboe
+ocarina
+odometer
+oil filter
+organ
+oscilloscope
+overskirt
+oxcart
+oxygen mask
+packet
+paddle
+paddlewheel
+padlock
+paintbrush
+pajama
+palace
+panpipe
+paper towel
+parachute
+parallel bars
+park bench
+parking meter
+passenger car
+patio
+pay-phone
+pedestal
+pencil box
+pencil sharpener
+perfume
+Petri dish
+photocopier
+pick
+pickelhaube
+picket fence
+pickup
+pier
+piggy bank
+pill bottle
+pillow
+ping-pong ball
+pinwheel
+pirate
+pitcher
+plane
+planetarium
+plastic bag
+plate rack
+plow
+plunger
+Polaroid camera
+pole
+police van
+poncho
+pool table
+pop bottle
+pot
+potter's wheel
+power drill
+prayer rug
+printer
+prison
+projectile
+projector
+puck
+punching bag
+purse
+quill
+quilt
+racer
+racket
+radiator
+radio
+radio telescope
+rain barrel
+recreational vehicle
+reel
+reflex camera
+refrigerator
+remote control
+restaurant
+revolver
+rifle
+rocking chair
+rotisserie
+rubber eraser
+rugby ball
+rule
+running shoe
+safe
+safety pin
+saltshaker
+sandal
+sarong
+sax
+scabbard
+scale
+school bus
+schooner
+scoreboard
+screen
+screw
+screwdriver
+seat belt
+sewing machine
+shield
+shoe shop
+shoji
+shopping basket
+shopping cart
+shovel
+shower cap
+shower curtain
+ski
+ski mask
+sleeping bag
+slide rule
+sliding door
+slot
+snorkel
+snowmobile
+snowplow
+soap dispenser
+soccer ball
+sock
+solar dish
+sombrero
+soup bowl
+space bar
+space heater
+space shuttle
+spatula
+speedboat
+spider web
+spindle
+sports car
+spotlight
+stage
+steam locomotive
+steel arch bridge
+steel drum
+stethoscope
+stole
+stone wall
+stopwatch
+stove
+strainer
+streetcar
+stretcher
+studio couch
+stupa
+submarine
+suit
+sundial
+sunglass
+sunglasses
+sunscreen
+suspension bridge
+swab
+sweatshirt
+swimming trunks
+swing
+switch
+syringe
+table lamp
+tank
+tape player
+teapot
+teddy
+television
+tennis ball
+thatch
+theater curtain
+thimble
+thresher
+throne
+tile roof
+toaster
+tobacco shop
+toilet seat
+torch
+totem pole
+tow truck
+toyshop
+tractor
+trailer truck
+tray
+trench coat
+tricycle
+trimaran
+tripod
+triumphal arch
+trolleybus
+trombone
+tub
+turnstile
+typewriter keyboard
+umbrella
+unicycle
+upright
+vacuum
+vase
+vault
+velvet
+vending machine
+vestment
+viaduct
+violin
+volleyball
+waffle iron
+wall clock
+wallet
+wardrobe
+warplane
+washbasin
+washer
+water bottle
+water jug
+water tower
+whiskey jug
+whistle
+wig
+window screen
+window shade
+Windsor tie
+wine bottle
+wing
+wok
+wooden spoon
+wool
+worm fence
+wreck
+yawl
+yurt
+web site
+comic book
+crossword puzzle
+street sign
+traffic light
+book jacket
+menu
+plate
+guacamole
+consomme
+hot pot
+trifle
+ice cream
+ice lolly
+French loaf
+bagel
+pretzel
+cheeseburger
+hotdog
+mashed potato
+head cabbage
+broccoli
+cauliflower
+zucchini
+spaghetti squash
+acorn squash
+butternut squash
+cucumber
+artichoke
+bell pepper
+cardoon
+mushroom
+Granny Smith
+strawberry
+orange
+lemon
+fig
+pineapple
+banana
+jackfruit
+custard apple
+pomegranate
+hay
+carbonara
+chocolate sauce
+dough
+meat loaf
+pizza
+potpie
+burrito
+red wine
+espresso
+cup
+eggnog
+alp
+bubble
+cliff
+coral reef
+geyser
+lakeside
+promontory
+sandbar
+seashore
+valley
+volcano
+ballplayer
+groom
+scuba diver
+rapeseed
+daisy
+yellow lady's slipper
+corn
+acorn
+hip
+buckeye
+coral fungus
+agaric
+gyromitra
+stinkhorn
+earthstar
+hen-of-the-woods
+bolete
+ear
+toilet tissue
-- 
GitLab


From bc4a279ca963a901576a9a7a35b189dfae7f5c85 Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Tue, 15 Jan 2019 19:58:24 -0800
Subject: [PATCH 0768/2345] Unbreak non-CUDA builds, since the testing bot
 tests all targets in the directory "//tensorflow/...".

PiperOrigin-RevId: 229487682
---
 tensorflow/stream_executor/cuda/BUILD | 134 +++++++++++++-------------
 1 file changed, 67 insertions(+), 67 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/BUILD b/tensorflow/stream_executor/cuda/BUILD
index 87c8eae416..fcfda1337e 100644
--- a/tensorflow/stream_executor/cuda/BUILD
+++ b/tensorflow/stream_executor/cuda/BUILD
@@ -42,10 +42,10 @@ cc_library(
 
 cc_library(
     name = "cuda_platform",
-    srcs = ["cuda_platform.cc"],
-    hdrs = ["cuda_platform.h"],
+    srcs = if_cuda_is_configured(["cuda_platform.cc"]),
+    hdrs = if_cuda_is_configured(["cuda_platform.h"]),
     visibility = ["//visibility:public"],
-    deps = [
+    deps = if_cuda_is_configured([
         ":cuda_driver",
         ":cuda_gpu_executor",
         ":cuda_platform_id",
@@ -55,27 +55,27 @@ cc_library(
         "//tensorflow/stream_executor:stream_executor_pimpl_header",
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
-    ] + tf_additional_cuda_platform_deps(),
+    ] + tf_additional_cuda_platform_deps()),
     alwayslink = True,  # Registers itself with the MultiPlatformManager.
 )
 
 cc_library(
     name = "cuda_diagnostics",
-    srcs = ["cuda_diagnostics.cc"],
-    hdrs = ["cuda_diagnostics.h"],
-    deps = [
-        "//tensorflow/stream_executor/lib",
-        "//tensorflow/stream_executor/platform",
+    srcs = if_cuda_is_configured(["cuda_diagnostics.cc"]),
+    hdrs = if_cuda_is_configured(["cuda_diagnostics.h"]),
+    deps = if_cuda_is_configured([
         "@com_google_absl//absl/container:inlined_vector",
         "@com_google_absl//absl/strings",
-    ],
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+    ]),
 )
 
 cc_library(
     name = "cuda_driver",
-    srcs = ["cuda_driver.cc"],
-    hdrs = ["cuda_driver.h"],
-    deps = [
+    srcs = if_cuda_is_configured(["cuda_driver.cc"]),
+    hdrs = if_cuda_is_configured(["cuda_driver.h"]),
+    deps = if_cuda_is_configured([
         ":cuda_diagnostics",
         "@com_google_absl//absl/base",
         "@com_google_absl//absl/container:inlined_vector",
@@ -84,49 +84,49 @@ cc_library(
         "//tensorflow/stream_executor:device_options",
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
-    ] + tf_additional_cuda_driver_deps(),
+    ] + tf_additional_cuda_driver_deps()),
 )
 
 # The activation library is tightly coupled to the executor library.
 # TODO(leary) split up cuda_gpu_executor.cc so that this can stand alone.
 cc_library(
     name = "cuda_activation_header",
-    hdrs = ["cuda_activation.h"],
+    hdrs = if_cuda_is_configured(["cuda_activation.h"]),
     visibility = ["//visibility:public"],
-    deps = ["//tensorflow/stream_executor/platform"],
+    deps = if_cuda_is_configured(["//tensorflow/stream_executor/platform"]),
 )
 
 cc_library(
     name = "cuda_activation",
-    srcs = ["cuda_activation.cc"],
-    hdrs = ["cuda_activation.h"],
-    deps = [
+    srcs = if_cuda_is_configured(["cuda_activation.cc"]),
+    hdrs = if_cuda_is_configured(["cuda_activation.h"]),
+    deps = if_cuda_is_configured([
         ":cuda_driver",
+        "@local_config_cuda//cuda:cuda_headers",
         "//tensorflow/stream_executor",
         "//tensorflow/stream_executor:stream_executor_internal",
         "//tensorflow/stream_executor/platform",
-        "@local_config_cuda//cuda:cuda_headers",
-    ],
+    ]),
 )
 
 cc_library(
     name = "cuda_gpu_executor_header",
-    textual_hdrs = ["cuda_gpu_executor.h"],
+    textual_hdrs = if_cuda_is_configured(["cuda_gpu_executor.h"]),
     visibility = ["//visibility:public"],
-    deps = [
+    deps = if_cuda_is_configured([
         ":cuda_kernel",
         "//tensorflow/stream_executor:event",
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
-    ],
+    ]),
 )
 
 cc_library(
     name = "cublas_plugin",
-    srcs = ["cuda_blas.cc"],
-    hdrs = ["cuda_blas.h"],
+    srcs = if_cuda_is_configured(["cuda_blas.cc"]),
+    hdrs = if_cuda_is_configured(["cuda_blas.h"]),
     visibility = ["//visibility:public"],
-    deps = [
+    deps = if_cuda_is_configured([
         ":cuda_activation",
         ":cuda_gpu_executor",
         ":cuda_helpers",
@@ -146,16 +146,16 @@ cc_library(
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
         "//tensorflow/stream_executor/platform:dso_loader",
-    ] + if_static(["@local_config_cuda//cuda:cublas"]),
+    ] + if_static(["@local_config_cuda//cuda:cublas"])),
     alwayslink = True,
 )
 
 cc_library(
     name = "cufft_plugin",
-    srcs = ["cuda_fft.cc"],
-    hdrs = ["cuda_fft.h"],
+    srcs = if_cuda_is_configured(["cuda_fft.cc"]),
+    hdrs = if_cuda_is_configured(["cuda_fft.h"]),
     visibility = ["//visibility:public"],
-    deps = [
+    deps = if_cuda_is_configured([
         ":cuda_activation_header",
         ":cuda_gpu_executor_header",
         ":cuda_helpers",
@@ -169,21 +169,21 @@ cc_library(
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
         "//tensorflow/stream_executor/platform:dso_loader",
-    ] + if_static(["@local_config_cuda//cuda:cufft"]),
+    ] + if_static(["@local_config_cuda//cuda:cufft"])),
     alwayslink = True,
 )
 
 cc_library(
     name = "cudnn_plugin",
-    srcs = ["cuda_dnn.cc"],
-    hdrs = ["cuda_dnn.h"],
+    srcs = if_cuda_is_configured(["cuda_dnn.cc"]),
+    hdrs = if_cuda_is_configured(["cuda_dnn.h"]),
     copts = [
         # STREAM_EXECUTOR_CUDNN_WRAP would fail on Clang with the default
         # setting of template depth 256
         "-ftemplate-depth-512",
     ],
     visibility = ["//visibility:public"],
-    deps = [
+    deps = if_cuda_is_configured([
         ":cuda_activation",
         ":cuda_diagnostics",
         ":cuda_driver",
@@ -208,15 +208,15 @@ cc_library(
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
         "//tensorflow/stream_executor/platform:dso_loader",
-    ] + tf_additional_cudnn_plugin_deps() + if_static(["@local_config_cuda//cuda:cudnn"]),
+    ] + tf_additional_cudnn_plugin_deps() + if_static(["@local_config_cuda//cuda:cudnn"])),
     alwayslink = True,
 )
 
 cc_library(
     name = "curand_plugin",
-    srcs = ["cuda_rng.cc"],
-    hdrs = ["cuda_rng.h"],
-    deps = [
+    srcs = if_cuda_is_configured(["cuda_rng.cc"]),
+    hdrs = if_cuda_is_configured(["cuda_rng.h"]),
+    deps = if_cuda_is_configured([
         ":cuda_activation",
         ":cuda_gpu_executor",
         ":cuda_helpers",
@@ -229,75 +229,75 @@ cc_library(
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
         "//tensorflow/stream_executor/platform:dso_loader",
-    ] + if_static(["@local_config_cuda//cuda:curand"]),
+    ] + if_static(["@local_config_cuda//cuda:curand"])),
     alwayslink = True,
 )
 
 cc_library(
     name = "cuda_kernel",
-    hdrs = ["cuda_kernel.h"],
-    deps = [
+    hdrs = if_cuda_is_configured(["cuda_kernel.h"]),
+    deps = if_cuda_is_configured([
         ":cuda_driver",
+        "@local_config_cuda//cuda:cuda_headers",
         "//tensorflow/stream_executor:event",
         "//tensorflow/stream_executor:stream_executor_pimpl_header",
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
-        "@local_config_cuda//cuda:cuda_headers",
-    ],
+    ]),
 )
 
 # TODO(leary) we likely need to canonicalize/eliminate this.
 cc_library(
     name = "cuda_helpers",
-    textual_hdrs = ["cuda_helpers.h"],
+    textual_hdrs = if_cuda_is_configured(["cuda_helpers.h"]),
 )
 
 cc_library(
     name = "cuda_event",
-    srcs = ["cuda_event.cc"],
-    hdrs = ["cuda_event.h"],
-    deps = [
+    srcs = if_cuda_is_configured(["cuda_event.cc"]),
+    hdrs = if_cuda_is_configured(["cuda_event.h"]),
+    deps = if_cuda_is_configured([
         ":cuda_driver",
         ":cuda_gpu_executor_header",
         ":cuda_stream",
         "//tensorflow/stream_executor:stream_executor_headers",
         "//tensorflow/stream_executor/lib",
-    ],
+    ]),
 )
 
 cc_library(
     name = "cuda_stream",
-    srcs = ["cuda_stream.cc"],
-    hdrs = ["cuda_stream.h"],
-    deps = [
+    srcs = if_cuda_is_configured(["cuda_stream.cc"]),
+    hdrs = if_cuda_is_configured(["cuda_stream.h"]),
+    deps = if_cuda_is_configured([
         ":cuda_driver",
         ":cuda_gpu_executor_header",
         "//tensorflow/stream_executor:stream_executor_headers",
         "//tensorflow/stream_executor:stream_header",
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
-    ],
+    ]),
 )
 
 cc_library(
     name = "cuda_timer",
-    srcs = ["cuda_timer.cc"],
-    hdrs = ["cuda_timer.h"],
-    deps = [
+    srcs = if_cuda_is_configured(["cuda_timer.cc"]),
+    hdrs = if_cuda_is_configured(["cuda_timer.h"]),
+    deps = if_cuda_is_configured([
         ":cuda_driver",
         ":cuda_gpu_executor_header",
         ":cuda_stream",
         "//tensorflow/stream_executor:stream_executor_headers",
         "//tensorflow/stream_executor/lib",
-    ],
+    ]),
 )
 
 # It implements :cuda_gpu_executor_header
 cc_library(
     name = "cuda_gpu_executor",
-    srcs = ["cuda_gpu_executor.cc"],
-    hdrs = ["cuda_gpu_executor.h"],
-    deps = [
+    srcs = if_cuda_is_configured(["cuda_gpu_executor.cc"]),
+    hdrs = if_cuda_is_configured(["cuda_gpu_executor.h"]),
+    deps = if_cuda_is_configured([
         ":cuda_activation",
         ":cuda_diagnostics",
         ":cuda_driver",
@@ -306,6 +306,7 @@ cc_library(
         ":cuda_platform_id",
         ":cuda_stream",
         ":cuda_timer",
+        "@com_google_absl//absl/strings",
         "//tensorflow/stream_executor:event",
         "//tensorflow/stream_executor:plugin_registry",
         "//tensorflow/stream_executor:stream_executor_internal",
@@ -313,8 +314,7 @@ cc_library(
         "//tensorflow/stream_executor:timer",
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
-        "@com_google_absl//absl/strings",
-    ],
+    ]),
     alwayslink = True,
 )
 
@@ -341,13 +341,13 @@ cc_library(
     name = "all_runtime",
     copts = tf_copts(),
     visibility = ["//visibility:public"],
-    deps = if_cuda_is_configured([
-        ":cudnn_plugin",
-        ":cufft_plugin",
+    deps = [
         ":cublas_plugin",
-        ":curand_plugin",
         ":cuda_driver",
         ":cuda_platform",
-    ]),
+        ":cudnn_plugin",
+        ":cufft_plugin",
+        ":curand_plugin",
+    ],
     alwayslink = 1,
 )
-- 
GitLab


From 742bd415306658a329be3f6581eab74b68c74601 Mon Sep 17 00:00:00 2001
From: Albin Joy <albin.joy@huawei.com>
Date: Tue, 15 Jan 2019 10:09:28 +0530
Subject: [PATCH 0769/2345] Typo error correction context.py

---
 tensorflow/python/eager/context.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py
index faaf742c26..6c5b191cf9 100644
--- a/tensorflow/python/eager/context.py
+++ b/tensorflow/python/eager/context.py
@@ -180,8 +180,8 @@ class _ContextSwitchStack(threading.local):
   def push(self, is_building_function, enter_context_fn):
     """Push metadata about a context switch onto the stack.
 
-    A context switch can take one of two forms: installing a graph as the
-    default graph, or entering the eager context. For each context switch,
+    A context switch can take any one of the two forms: installing a graph as
+    the default graph, or entering the eager context. For each context switch,
     we record whether or not the entered context is building a function.
 
     Args:
-- 
GitLab


From e2cc09ed38a7ab1b3620aab93b77c4b1d7720677 Mon Sep 17 00:00:00 2001
From: "Li, Guizi" <guizi.li@intel.com>
Date: Wed, 16 Jan 2019 13:55:52 +0800
Subject: [PATCH 0770/2345] [Intel MKL] enable depthwise convolution backward
 side code

---
 tensorflow/core/graph/mkl_layout_pass.cc      |  19 ++++
 tensorflow/core/graph/mkl_layout_pass_test.cc |  72 +++++++++++++
 .../core/kernels/mkl_conv_grad_filter_ops.cc  | 100 +++++++++++-------
 .../core/kernels/mkl_conv_grad_input_ops.cc   |  51 +++++----
 tensorflow/core/kernels/mkl_conv_ops.h        |  53 ++++++----
 tensorflow/core/ops/mkl_nn_ops.cc             |  44 ++++++++
 6 files changed, 257 insertions(+), 82 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 0f7a81110c..6f7d49c2ea 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -259,6 +259,9 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     csinfo_.conv3d_grad_input = "Conv3DBackpropInputV2";
     csinfo_.conv3d_grad_filter = "Conv3DBackpropFilterV2";
     csinfo_.depthwise_conv2d = "DepthwiseConv2dNative";
+    csinfo_.depthwise_conv2d_grad_input = "DepthwiseConv2dNativeBackpropInput";
+    csinfo_.depthwise_conv2d_grad_filter =
+        "DepthwiseConv2dNativeBackpropFilter";
     csinfo_.fused_batch_norm = "FusedBatchNorm";
     csinfo_.fused_batch_norm_grad = "FusedBatchNormGrad";
     csinfo_.fused_conv2d = "_FusedConv2D";
@@ -278,6 +281,10 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     csinfo_.mkl_conv2d_with_bias = "_MklConv2DWithBias";
     csinfo_.mkl_conv2d_grad_filter_with_bias =
         "_MklConv2DBackpropFilterWithBias";
+    csinfo_.mkl_depthwise_conv2d_grad_input =
+        "_MklDepthwiseConv2dNativeBackpropInput";
+    csinfo_.mkl_depthwise_conv2d_grad_filter =
+        "_MklDepthwiseConv2dNativeBackpropFilter";
     csinfo_.mkl_fused_conv2d = "_MklFusedConv2D";
     csinfo_.mkl_pad_with_conv2d = "_MklPadWithConv2D";
     csinfo_.pad = "Pad";
@@ -381,6 +388,14 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     rinfo_.push_back({csinfo_.depthwise_conv2d,
                       mkl_op_registry::GetMklOpName(csinfo_.depthwise_conv2d),
                       CopyAttrsConv2DDepthwise, AlwaysRewrite});
+    rinfo_.push_back(
+        {csinfo_.depthwise_conv2d_grad_input,
+         mkl_op_registry::GetMklOpName(csinfo_.depthwise_conv2d_grad_input),
+         CopyAttrsConv2DDepthwise, AlwaysRewrite});
+    rinfo_.push_back(
+        {csinfo_.depthwise_conv2d_grad_filter,
+         mkl_op_registry::GetMklOpName(csinfo_.depthwise_conv2d_grad_filter),
+         CopyAttrsConv2DDepthwise, AlwaysRewrite});
     rinfo_.push_back({csinfo_.fused_batch_norm,
                       mkl_op_registry::GetMklOpName(csinfo_.fused_batch_norm),
                       CopyAttrsFusedBatchNorm, AlwaysRewrite});
@@ -680,6 +695,8 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     string conv3d_grad_input;
     string conv3d_grad_filter;
     string depthwise_conv2d;
+    string depthwise_conv2d_grad_input;
+    string depthwise_conv2d_grad_filter;
     string fused_batch_norm;
     string fused_batch_norm_grad;
     string fused_conv2d;
@@ -699,6 +716,8 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     string mkl_conv2d_grad_filter;
     string mkl_conv2d_grad_filter_with_bias;
     string mkl_conv2d_with_bias;
+    string mkl_depthwise_conv2d_grad_input;
+    string mkl_depthwise_conv2d_grad_filter;
     string mkl_fused_conv2d;
     string mkl_pad_with_conv2d;
     string mul;
diff --git a/tensorflow/core/graph/mkl_layout_pass_test.cc b/tensorflow/core/graph/mkl_layout_pass_test.cc
index 6e73ed1b9f..8e6df142e6 100644
--- a/tensorflow/core/graph/mkl_layout_pass_test.cc
+++ b/tensorflow/core/graph/mkl_layout_pass_test.cc
@@ -1289,6 +1289,55 @@ TEST_F(MklLayoutPassTest, NodeRewrite_Conv2DGradInput_Positive) {
             "D->E:1;DMT/_0->D:3;DMT/_1->D:4;DMT/_2->D:5");
 }
 
+TEST_F(MklLayoutPassTest,
+       NodeRewrite_DepthwiseConv2dNativeGradFilter_Positive) {
+  InitGraph(
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Int32Input'}"
+      "node { name: 'C' op: 'Input'}"
+      "node { name: 'D' op: 'DepthwiseConv2dNativeBackpropFilter'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'data_format'      value { s: 'NCHW' } }"
+      " attr { key: 'use_cudnn_on_gpu' value { b: false } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'padding'          value { s: 'SAME' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
+      " input: ['A', 'B', 'C']}"
+      "node { name: 'E' op: 'Zeta' attr { key: 'T' value { type: DT_FLOAT } }"
+      " input: ['A', 'D'] }");
+  EXPECT_EQ(DoMklLayoutOptimizationPass(),
+            "A(Input);B(Int32Input);C(Input);D(_"
+            "MklDepthwiseConv2dNativeBackpropFilter);"
+            "DMT/_0(Const);DMT/_1(Const);DMT/_2(Const);E(Zeta)|"
+            "A->D;A->E;A:control->DMT/_0:control;A:control->DMT/_1:control;"
+            "A:control->DMT/_2:control;B->D:1;C->D:2;D->E:1;DMT/_0->D:3;"
+            "DMT/_1->D:4;DMT/_2->D:5");
+}
+
+TEST_F(MklLayoutPassTest, NodeRewrite_DepthwiseConv2dNativeGradInput_Positive) {
+  InitGraph(
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Int32Input'}"
+      "node { name: 'C' op: 'Input'}"
+      "node { name: 'D' op: 'DepthwiseConv2dNativeBackpropInput'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'data_format'      value { s: 'NCHW' } }"
+      " attr { key: 'use_cudnn_on_gpu' value { b: false } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'padding'          value { s: 'SAME' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
+      " input: ['B', 'A', 'C']}"
+      "node { name: 'E' op: 'Zeta' attr { key: 'T' value { type: DT_FLOAT } }"
+      " input: ['A', 'D'] }");
+  EXPECT_EQ(DoMklLayoutOptimizationPass(),
+            "A(Input);B(Int32Input);C(Input);D(_"
+            "MklDepthwiseConv2dNativeBackpropInput);"
+            "DMT/_0(Const);DMT/_1(Const);DMT/_2(Const);E(Zeta)|"
+            "A->D:1;A->E;B->D;B:control->DMT/_0:control;"
+            "B:control->DMT/_1:control;B:control->DMT/_2:control;C->D:2;"
+            "D->E:1;DMT/_0->D:3;DMT/_1->D:4;DMT/_2->D:5");
+}
+
 // Check that we never rewrite BiasAddGrad.
 TEST_F(MklLayoutPassTest, NodeRewrite_BiasAddGrad_Positive) {
   InitGraph(
@@ -2322,6 +2371,29 @@ TEST_F(MklLayoutPassTest, NodeRewrite_Conv2DGradFilter_DeviceTest) {
             "A->D;A->E;B->D:1;C->D:2;D->E:1");
 }
 
+TEST_F(MklLayoutPassTest,
+       NodeRewrite_DepthwiseConv2dNativeGradFilter_DeviceTest) {
+  InitGraph(
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Int32Input'}"
+      "node { name: 'C' op: 'Input'}"
+      "node { name: 'D' op: 'DepthwiseConv2dNativeBackpropFilter'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'data_format'      value { s: 'NCHW' } }"
+      " attr { key: 'use_cudnn_on_gpu' value { b: false } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'padding'          value { s: 'SAME' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
+      " input: ['A', 'B', 'C']}"
+      "node { name: 'E' op: 'Zeta' attr { key: 'T' value { type: DT_FLOAT } }"
+      " input: ['A', 'D'] }",
+      kGPUDevice);
+  EXPECT_EQ(DoMklLayoutOptimizationPass(),
+            "A(Input);B(Int32Input);C(Input);D("
+            "DepthwiseConv2dNativeBackpropFilter);E(Zeta)|"
+            "A->D;A->E;B->D:1;C->D:2;D->E:1");
+}
+
 TEST_F(MklLayoutPassTest, NodeRewrite_Relu_DeviceTest) {
   InitGraph(
       "node { name: 'A' op: 'Input'}"
diff --git a/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc b/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc
index bc9d45abe6..8faa711975 100644
--- a/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc
@@ -357,12 +357,12 @@ class MklConvBwdFilterPrimitiveFactory : public MklPrimitiveFactory<T> {
   }
 };
 
-template <typename Device, class T, bool bias_enabled>
+template <typename Device, class T, bool bias_enabled, bool is_depthwise>
 class MklConvCustomBackpropFilterOp
-    : public MklConvBackpropCommonOp<Device, T> {
+    : public MklConvBackpropCommonOp<Device, T, is_depthwise> {
  public:
   explicit MklConvCustomBackpropFilterOp(OpKernelConstruction* context)
-      : MklConvBackpropCommonOp<Device, T>(context) {}
+      : MklConvBackpropCommonOp<Device, T, is_depthwise>(context) {}
 
   ~MklConvCustomBackpropFilterOp() {}
 
@@ -432,7 +432,7 @@ class MklConvCustomBackpropFilterOp
       conv_utl.GetConvFwdSizesInMklOrder(
           src_tf_shape, filter_tf_shape, &fwd_src_dims, &fwd_filter_dims,
           &strides, &dilations, &fwd_dst_dims_tf_order, &fwd_dst_dims,
-          &padding_left, &padding_right, false);
+          &padding_left, &padding_right, false, is_depthwise);
       if (!context->status().ok()) return;
 
       auto tf_fmt = is_conv2d
@@ -485,13 +485,27 @@ class MklConvCustomBackpropFilterOp
       diff_filter_mkl_shape.SetMklTensor(false);
 
       if (is_conv2d) {
-        // Conv2D: output_dims_mkl_order is in OIHW format.
-        TensorShape diff_filter_tf_shape({bwd_output_dims[MklDnnDims::Dim_H],
-                                          bwd_output_dims[MklDnnDims::Dim_W],
-                                          bwd_output_dims[MklDnnDims::Dim_I],
-                                          bwd_output_dims[MklDnnDims::Dim_O]});
-        AllocateOutputSetMklShape(context, 0, &diff_filter_tensor,
-                                  diff_filter_tf_shape, diff_filter_mkl_shape);
+        if (!is_depthwise) {
+          // Conv2D: output_dims_mkl_order is in OIHW format.
+          TensorShape diff_filter_tf_shape(
+              {bwd_output_dims[MklDnnDims::Dim_H],
+               bwd_output_dims[MklDnnDims::Dim_W],
+               bwd_output_dims[MklDnnDims::Dim_I],
+               bwd_output_dims[MklDnnDims::Dim_O]});
+          AllocateOutputSetMklShape(context, 0, &diff_filter_tensor,
+                                    diff_filter_tf_shape,
+                                    diff_filter_mkl_shape);
+        } else {
+          // Depthwise Conv2d: bwd_output_dims is GOIHW format
+          TensorShape diff_filter_tf_shape(
+              {bwd_output_dims[MklDnnFilterGroupDims::MKL_GROUP_FILTER_DIM_H],
+               bwd_output_dims[MklDnnFilterGroupDims::MKL_GROUP_FILTER_DIM_W],
+               bwd_output_dims[MklDnnFilterGroupDims::MKL_GROUP_FILTER_DIM_G],
+               bwd_output_dims[MklDnnFilterGroupDims::MKL_GROUP_FILTER_DIM_O]});
+          AllocateOutputSetMklShape(context, 0, &diff_filter_tensor,
+                                    diff_filter_tf_shape,
+                                    diff_filter_mkl_shape);
+        }
       } else {
         // Conv3D: output_dims_mkl_order is in OIDHW format.
         TensorShape diff_filter_tf_shape(
@@ -568,9 +582,9 @@ class MklConvCustomBackpropFilterOp
       // delete primitive since it is not cached.
       if (do_not_cache) delete conv_bwd_filter;
     } catch (mkldnn::error& e) {
-      string error_msg = "Status: " + std::to_string(e.status) +
-                         ", message: " + string(e.message) + ", in file " +
-                         string(__FILE__) + ":" + std::to_string(__LINE__);
+      string error_msg = "Status: " + std::to_string(e.status) + ", message: " +
+                         string(e.message) + ", in file " + string(__FILE__) +
+                         ":" + std::to_string(__LINE__);
       OP_REQUIRES_OK(
           context,
           errors::Aborted("Operation received an exception:", error_msg));
@@ -628,10 +642,11 @@ class MklConvCustomBackpropFilterOp
   }
 
   // Output layout is Tensorflow's filter layout
-  //   Conv2D: HWIO;  Conv3D: DHWIO
+  //   Conv2D: HWIO;  Conv3D: DHWIO; Depthwise Conv: HWIGO
   memory::format GetOutputFormat(const memory::format data_format) {
-    return (this->strides_.size() == 4) ? memory::format::hwio
-                                        : memory::format::dhwio;
+    return is_depthwise ? memory::format::hwigo : ((this->strides_.size() == 4)
+                                                       ? memory::format::hwio
+                                                       : memory::format::dhwio);
   }
 
   // Allocate output tensor.
@@ -673,27 +688,36 @@ class MklConvCustomBackpropFilterOp
   }
 };
 
-#define REGISTER_MKL_FILTER_KERNELS(T)                                         \
-  REGISTER_KERNEL_BUILDER(Name("_MklConv2DBackpropFilter")                     \
-                              .Device(DEVICE_CPU)                              \
-                              .TypeConstraint<T>("T")                          \
-                              .Label(mkl_op_registry::kMklOpLabel),            \
-                          MklConvCustomBackpropFilterOp<CPUDevice, T, false>); \
-  REGISTER_KERNEL_BUILDER(Name("_MklConv2DBackpropFilterWithBias")             \
-                              .Device(DEVICE_CPU)                              \
-                              .TypeConstraint<T>("T")                          \
-                              .Label(mkl_op_registry::kMklOpLabel),            \
-                          MklConvCustomBackpropFilterOp<CPUDevice, T, true>);  \
-  REGISTER_KERNEL_BUILDER(Name("__MklDummyConv2DBackpropFilterWithBias")       \
-                              .Device(DEVICE_CPU)                              \
-                              .TypeConstraint<T>("T")                          \
-                              .Label(mkl_op_registry::kMklOpLabel),            \
-                          MklDummyOp<CPUDevice, T>);                           \
-  REGISTER_KERNEL_BUILDER(Name("_MklConv3DBackpropFilterV2")                   \
-                              .Device(DEVICE_CPU)                              \
-                              .TypeConstraint<T>("T")                          \
-                              .Label(mkl_op_registry::kMklOpLabel),            \
-                          MklConvCustomBackpropFilterOp<CPUDevice, T, false>);
+#define REGISTER_MKL_FILTER_KERNELS(T)                                   \
+  REGISTER_KERNEL_BUILDER(                                               \
+      Name("_MklConv2DBackpropFilter")                                   \
+          .Device(DEVICE_CPU)                                            \
+          .TypeConstraint<T>("T")                                        \
+          .Label(mkl_op_registry::kMklOpLabel),                          \
+      MklConvCustomBackpropFilterOp<CPUDevice, T, false, false>);        \
+  REGISTER_KERNEL_BUILDER(                                               \
+      Name("_MklConv2DBackpropFilterWithBias")                           \
+          .Device(DEVICE_CPU)                                            \
+          .TypeConstraint<T>("T")                                        \
+          .Label(mkl_op_registry::kMklOpLabel),                          \
+      MklConvCustomBackpropFilterOp<CPUDevice, T, true, false>);         \
+  REGISTER_KERNEL_BUILDER(                                               \
+      Name("_MklDepthwiseConv2dNativeBackpropFilter")                    \
+          .Device(DEVICE_CPU)                                            \
+          .TypeConstraint<T>("T")                                        \
+          .Label(mkl_op_registry::kMklOpLabel),                          \
+      MklConvCustomBackpropFilterOp<CPUDevice, T, false, true>);         \
+  REGISTER_KERNEL_BUILDER(Name("__MklDummyConv2DBackpropFilterWithBias") \
+                              .Device(DEVICE_CPU)                        \
+                              .TypeConstraint<T>("T")                    \
+                              .Label(mkl_op_registry::kMklOpLabel),      \
+                          MklDummyOp<CPUDevice, T>);                     \
+  REGISTER_KERNEL_BUILDER(                                               \
+      Name("_MklConv3DBackpropFilterV2")                                 \
+          .Device(DEVICE_CPU)                                            \
+          .TypeConstraint<T>("T")                                        \
+          .Label(mkl_op_registry::kMklOpLabel),                          \
+      MklConvCustomBackpropFilterOp<CPUDevice, T, false, false>);
 
 TF_CALL_float(REGISTER_MKL_FILTER_KERNELS);
 #undef REGISTER_MKL_FILTER_KERNELS
diff --git a/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc b/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc
index b5be87ec55..1799a44a45 100644
--- a/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc
@@ -295,11 +295,12 @@ class MklConvBwdInputPrimitiveFactory : public MklPrimitiveFactory<T> {
   }
 };
 
-template <typename Device, class T>
-class MklConvCustomBackpropInputOp : public MklConvBackpropCommonOp<Device, T> {
+template <typename Device, class T, bool is_depthwise>
+class MklConvCustomBackpropInputOp
+    : public MklConvBackpropCommonOp<Device, T, is_depthwise> {
  public:
   explicit MklConvCustomBackpropInputOp(OpKernelConstruction* context)
-      : MklConvBackpropCommonOp<Device, T>(context) {}
+      : MklConvBackpropCommonOp<Device, T, is_depthwise>(context) {}
 
   ~MklConvCustomBackpropInputOp() {}
 
@@ -367,7 +368,7 @@ class MklConvCustomBackpropInputOp : public MklConvBackpropCommonOp<Device, T> {
       conv_utl.GetConvFwdSizesInMklOrder(
           src_tf_shape, filter_tf_shape, &fwd_src_dims, &fwd_filter_dims,
           &strides, &dilations, &fwd_output_dims_tf_order, &fwd_output_dims,
-          &padding_left, &padding_right, false);
+          &padding_left, &padding_right, false, is_depthwise);
       if (!context->status().ok()) return;
 
       // Create Convolution forward descriptor since Convolution backward
@@ -383,9 +384,11 @@ class MklConvCustomBackpropInputOp : public MklConvBackpropCommonOp<Device, T> {
       auto fwd_filter_md =
           filter_mkl_shape.IsMklTensor()
               ? filter_mkl_shape.GetMklLayout()
-              : memory::desc(
-                    fwd_filter_dims, MklDnnType<T>(),
-                    is_conv2d ? memory::format::hwio : memory::format::dhwio);
+              : memory::desc(fwd_filter_dims, MklDnnType<T>(),
+                             is_depthwise
+                                 ? memory::hwigo
+                                 : (is_conv2d ? memory::format::hwio
+                                              : memory::format::dhwio));
 
       conv_utl.GetInputSizeInMklOrder(diff_dst_tf_shape, &diff_dst_dims);
       if (!context->status().ok()) return;
@@ -462,9 +465,9 @@ class MklConvCustomBackpropInputOp : public MklConvBackpropCommonOp<Device, T> {
         delete conv_bwd_input;
       }
     } catch (mkldnn::error& e) {
-      string error_msg = "Status: " + std::to_string(e.status) +
-                         ", message: " + string(e.message) + ", in file " +
-                         string(__FILE__) + ":" + std::to_string(__LINE__);
+      string error_msg = "Status: " + std::to_string(e.status) + ", message: " +
+                         string(e.message) + ", in file " + string(__FILE__) +
+                         ":" + std::to_string(__LINE__);
       OP_REQUIRES_OK(
           context,
           errors::Aborted("Operation received an exception:", error_msg));
@@ -554,18 +557,22 @@ class MklConvCustomBackpropInputOp : public MklConvBackpropCommonOp<Device, T> {
   }
 };
 
-#define REGISTER_MKL_CPU_KERNELS(T)                                    \
-  REGISTER_KERNEL_BUILDER(Name("_MklConv2DBackpropInput")              \
-                              .Device(DEVICE_CPU)                      \
-                              .TypeConstraint<T>("T")                  \
-                              .Label(mkl_op_registry::kMklOpLabel),    \
-                          MklConvCustomBackpropInputOp<CPUDevice, T>); \
-  REGISTER_KERNEL_BUILDER(Name("_MklConv3DBackpropInputV2")            \
-                              .Device(DEVICE_CPU)                      \
-                              .TypeConstraint<T>("T")                  \
-                              .Label(mkl_op_registry::kMklOpLabel),    \
-                          MklConvCustomBackpropInputOp<CPUDevice, T>);
-
+#define REGISTER_MKL_CPU_KERNELS(T)                                           \
+  REGISTER_KERNEL_BUILDER(Name("_MklConv2DBackpropInput")                     \
+                              .Device(DEVICE_CPU)                             \
+                              .TypeConstraint<T>("T")                         \
+                              .Label(mkl_op_registry::kMklOpLabel),           \
+                          MklConvCustomBackpropInputOp<CPUDevice, T, false>); \
+  REGISTER_KERNEL_BUILDER(Name("_MklConv3DBackpropInputV2")                   \
+                              .Device(DEVICE_CPU)                             \
+                              .TypeConstraint<T>("T")                         \
+                              .Label(mkl_op_registry::kMklOpLabel),           \
+                          MklConvCustomBackpropInputOp<CPUDevice, T, false>); \
+  REGISTER_KERNEL_BUILDER(Name("_MklDepthwiseConv2dNativeBackpropInput")      \
+                              .Device(DEVICE_CPU)                             \
+                              .TypeConstraint<T>("T")                         \
+                              .Label(mkl_op_registry::kMklOpLabel),           \
+                          MklConvCustomBackpropInputOp<CPUDevice, T, true>);
 TF_CALL_float(REGISTER_MKL_CPU_KERNELS);
 #undef REGISTER_MKL_CPU_KERNELS
 
diff --git a/tensorflow/core/kernels/mkl_conv_ops.h b/tensorflow/core/kernels/mkl_conv_ops.h
index fd279ead9d..c49650c1db 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.h
+++ b/tensorflow/core/kernels/mkl_conv_ops.h
@@ -58,7 +58,7 @@ class MklDnnConvUtil {
  public:
   MklDnnConvUtil(OpKernelContext* context, const std::vector<int32>& strides,
                  Padding pad, TensorFormat fm,
-                 const std::vector<int32>& dilations)
+                 const std::vector<int32>& dilations, bool is_depthwise = false)
       : context_(context),
         strides_(strides),
         dilations_(dilations),
@@ -196,9 +196,8 @@ class MklDnnConvUtil {
                                         filter_shape.DebugString()));
 
     for (int i = 0; i < ((strides_.size() == 4) ? 3 : 5); i++) {
-      OP_REQUIRES(context_,
-                  FastBoundsCheck(filter_shape.dim_size(i),
-                                  std::numeric_limits<int>::max()),
+      OP_REQUIRES(context_, FastBoundsCheck(filter_shape.dim_size(i),
+                                            std::numeric_limits<int>::max()),
                   errors::InvalidArgument("filter too large"));
     }
 
@@ -550,7 +549,7 @@ class MklDnnConvUtil {
 ///  Common class that implements ConvBackpropFilter and Input
 /////////////////////////////////////////////////////////////////////
 
-template <typename Device, class T>
+template <typename Device, class T, bool is_depthwise>
 class MklConvBackpropCommonOp : public OpKernel {
  public:
   ~MklConvBackpropCommonOp() {}
@@ -563,28 +562,38 @@ class MklConvBackpropCommonOp : public OpKernel {
     OP_REQUIRES_OK(context, context->GetAttr("strides", &strides_));
     int stride_n = GetTensorDim(strides_, data_format_, 'N');
     int stride_c = GetTensorDim(strides_, data_format_, 'C');
+    const int64 stride_h = GetTensorDim(strides_, data_format_, 'H');
+    const int64 stride_w = GetTensorDim(strides_, data_format_, 'W');
     OP_REQUIRES(
         context, (stride_n == 1 && stride_c == 1),
         errors::InvalidArgument("Current implementation does not yet support "
                                 "strides in the batch and depth dimensions."));
-    OP_REQUIRES_OK(context, context->GetAttr("dilations", &dilations_));
 
-    if (strides_.size() == 4) {
-      // Check Conv2D dilations
-      OP_REQUIRES(context, dilations_.size() == 4,
-                  errors::InvalidArgument("Sliding window dilations field must "
-                                          "specify 4 dimensions"));
-      int dilation_n = GetTensorDim(dilations_, data_format_, 'N');
-      int dilation_c = GetTensorDim(dilations_, data_format_, 'C');
-      int dilation_h = GetTensorDim(dilations_, data_format_, 'H');
-      int dilation_w = GetTensorDim(dilations_, data_format_, 'W');
-      OP_REQUIRES(context, (dilation_n == 1 && dilation_c == 1),
-                  errors::InvalidArgument(
-                      "Current implementation does not yet support "
-                      "dilations in the batch and depth dimensions."));
-      OP_REQUIRES(
-          context, dilation_h > 0 && dilation_w > 0,
-          errors::InvalidArgument("Dilated rates should be larger than 0."));
+    // Depthwise Convolution doesn't have dilation parameter
+    if (!is_depthwise) {
+      OP_REQUIRES_OK(context, context->GetAttr("dilations", &dilations_));
+      if (strides_.size() == 4) {
+        // Check Conv2D dilations
+        OP_REQUIRES(
+            context, dilations_.size() == 4,
+            errors::InvalidArgument("Sliding window dilations field must "
+                                    "specify 4 dimensions"));
+        int dilation_n = GetTensorDim(dilations_, data_format_, 'N');
+        int dilation_c = GetTensorDim(dilations_, data_format_, 'C');
+        int dilation_h = GetTensorDim(dilations_, data_format_, 'H');
+        int dilation_w = GetTensorDim(dilations_, data_format_, 'W');
+        OP_REQUIRES(context, (dilation_n == 1 && dilation_c == 1),
+                    errors::InvalidArgument(
+                        "Current implementation does not yet support "
+                        "dilations in the batch and depth dimensions."));
+        OP_REQUIRES(
+            context, dilation_h > 0 && dilation_w > 0,
+            errors::InvalidArgument("Dilated rates should be larger than 0."));
+      }
+    } else {
+      // Set dilations as 1 for depthwise conv
+      // for future support to align with Tensorflow
+      dilations_ = {1, 1, 1, 1};
     }
 
     OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
diff --git a/tensorflow/core/ops/mkl_nn_ops.cc b/tensorflow/core/ops/mkl_nn_ops.cc
index 658afd9901..07d95dd988 100644
--- a/tensorflow/core/ops/mkl_nn_ops.cc
+++ b/tensorflow/core/ops/mkl_nn_ops.cc
@@ -634,6 +634,50 @@ REGISTER_OP("_MklQuantizedConv2DWithBiasSignedSumAndReluAndRequantize")
       return Status::OK();
     });
 
+REGISTER_OP("_MklDepthwiseConv2dNativeBackpropInput")
+    .Input("input_sizes: int32")
+    .Input("filter: T")
+    .Input("out_backprop: T")
+    .Input("mkl_input: uint8")
+    .Input("mkl_filter: uint8")
+    .Input("mkl_out_backprop: uint8")
+    .Output("output: T")
+    .Output("mkl_output: uint8")
+    .Attr("T: {half, bfloat16, float, double}")
+    .Attr("strides: list(int)")
+    .Attr(GetPaddingAttrString())
+    .Attr(GetConvnetDataFormatAttrString())
+    .Attr("dilations: list(int) = [1, 1, 1, 1]")
+    .SetShapeFn([](InferenceContext* c) {
+      ShapeHandle s;
+      TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(0, &s));
+      TF_RETURN_IF_ERROR(c->WithRank(s, 4, &s));
+      c->set_output(0, s);
+      return Status::OK();
+    });
+
+REGISTER_OP("_MklDepthwiseConv2dNativeBackpropFilter")
+    .Input("input: T")
+    .Input("filter_sizes: int32")
+    .Input("out_backprop: T")
+    .Input("mkl_input: uint8")
+    .Input("mkl_filter: uint8")
+    .Input("mkl_out_backprop: uint8")
+    .Output("output: T")
+    .Output("mkl_output: uint8")
+    .Attr("T: {half, bfloat16, float, double}")
+    .Attr("strides: list(int)")
+    .Attr(GetPaddingAttrString())
+    .Attr(GetConvnetDataFormatAttrString())
+    .Attr("dilations: list(int) = [1, 1, 1, 1]")
+    .SetShapeFn([](InferenceContext* c) {
+      ShapeHandle s;
+      TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(1, &s));
+      TF_RETURN_IF_ERROR(c->WithRank(s, 4, &s));
+      c->set_output(0, s);
+      return Status::OK();
+    });
+
 }  // namespace tensorflow
 
 #endif  // INTEL_MKL
-- 
GitLab


From 0bc02af7db9e43c37a89278a2e7c99ecdfe4fb00 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Tue, 15 Jan 2019 23:12:00 -0800
Subject: [PATCH 0771/2345] Fix demo app to handle some cases more properly

- Don't crash when threads are changes with a delegate. This is done by now recreating interpreter every time threads is changes.
- Don't call showToast() every updateFrame() when classifier is null. This prevents hanging and allows the proper error message to be displayed.

PiperOrigin-RevId: 229503428
---
 .../Camera2BasicFragment.java                 | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java b/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java
index a7b3440536..814d236872 100644
--- a/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java
+++ b/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java
@@ -190,6 +190,8 @@ public class Camera2BasicFragment extends Fragment
 
   int currentModel = -1;
 
+  int currentNumThreads = -1;
+
   /** An additional thread for running tasks that shouldn't block the UI. */
   private HandlerThread backgroundThread;
 
@@ -323,13 +325,16 @@ public class Camera2BasicFragment extends Fragment
     // Get UI information before delegating to background
     final int modelIndex = modelView.getCheckedItemPosition();
     final int deviceIndex = deviceView.getCheckedItemPosition();
+    final int numThreads = np.getValue();
 
     backgroundHandler.post(() -> {
-      if (modelIndex == currentModel && deviceIndex == currentDevice) {
+      if (modelIndex == currentModel && deviceIndex == currentDevice
+              && numThreads == currentNumThreads) {
         return;
       }
       currentModel = modelIndex;
       currentDevice = deviceIndex;
+      currentNumThreads = numThreads;
 
       // Disable classifier while updating
       if (classifier != null) {
@@ -357,7 +362,11 @@ public class Camera2BasicFragment extends Fragment
         classifier = null;
       }
 
-      // Customzie the interpreter to the type of device we want to use.
+      // Customize the interpreter to the type of device we want to use.
+      if (classifier == null) {
+        return;
+      }
+      classifier.setNumThreads(numThreads);
       if (device.equals(cpu)) {
       } else if (device.equals(gpu)) {
         if (!GpuDelegateHelper.isGpuDelegateAvailable()) {
@@ -437,7 +446,7 @@ public class Camera2BasicFragment extends Fragment
         new NumberPicker.OnValueChangeListener() {
           @Override
           public void onValueChange(NumberPicker picker, int oldVal, int newVal) {
-            backgroundHandler.post(() -> classifier.setNumThreads(newVal));
+            updateActiveModel();
           }
         });
 
@@ -807,7 +816,9 @@ public class Camera2BasicFragment extends Fragment
   /** Classifies a frame from the preview stream. */
   private void classifyFrame() {
     if (classifier == null || getActivity() == null || cameraDevice == null) {
-      showToast("Uninitialized Classifier or invalid context.");
+      // It's important to not call showToast every frame, or else the app will starve and
+      // hang. updateActiveModel() already puts a error message up with showToast.
+      // showToast("Uninitialized Classifier or invalid context.");
       return;
     }
     SpannableStringBuilder textToShow = new SpannableStringBuilder();
-- 
GitLab


From 9fd5c9930dc714c089ed137d57e8ba09e0e1a6af Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 01:03:08 -0800
Subject: [PATCH 0772/2345] compat: Update forward compatibility horizon to
 2019-01-16

PiperOrigin-RevId: 229513368
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 83f31bd533..773b3a41f9 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 15)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 16)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From a331eaf593bb91eaafd8953f1bd7d502eaeee53a Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Wed, 16 Jan 2019 02:40:41 -0800
Subject: [PATCH 0773/2345] Move variable ownership test into
 extended.variable_created_in_scope.

We want to invert the relationship here, instead of tf.Variable being aware of
strategies, we make strategies keep track of variables created while their
scope was active. This change implements that (at least in terms of the public
API for variables and dist strat).

tf.Variable subclasses created by distribution strategy (e.g.
DistributedVariable) still export a public `distribution_strategy` property.

PiperOrigin-RevId: 229525171
---
 .../python/parameter_server_strategy_test.py  |  5 +++-
 .../python/distribute/distribute_lib.py       | 27 +++++++++++++++++++
 tensorflow/python/distribute/values.py        |  9 ++++---
 tensorflow/python/keras/engine/training.py    |  5 ++--
 .../resource_variable_ops_test.py             |  2 +-
 .../python/kernel_tests/variables_test.py     |  2 +-
 .../python/ops/resource_variable_ops.py       |  9 ++-----
 tensorflow/python/ops/variables.py            | 14 ++--------
 .../api/golden/v1/tensorflow.-variable.pbtxt  |  4 ---
 ...orflow.distribute.-strategy-extended.pbtxt |  4 +++
 .../api/golden/v2/tensorflow.-variable.pbtxt  |  4 ---
 ...orflow.distribute.-strategy-extended.pbtxt |  4 +++
 12 files changed, 53 insertions(+), 36 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
index c7760a8431..9e7e201519 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
@@ -873,7 +873,10 @@ class ParameterServerStrategyWithChiefTest(ParameterServerStrategyTestBase,
                              id(get_step), get_step.__class__.__name__)))
       self.assertIs(resource_variable_ops.ResourceVariable, type(created_step))
       self.assertIs(resource_variable_ops.ResourceVariable, type(get_step))
-      self.assertIs(strategy, created_step.distribute_strategy)
+      # All variables have an _distribute_strategy parameter. Only variable
+      # subclasses in distribution strategy expose it publicly.
+      self.assertFalse(hasattr(strategy, 'distribute_strategy'))
+      self.assertIs(strategy, created_step._distribute_strategy)
 
   @combinations.generate(
       combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
diff --git a/tensorflow/python/distribute/distribute_lib.py b/tensorflow/python/distribute/distribute_lib.py
index f112787ba2..32789c821e 100644
--- a/tensorflow/python/distribute/distribute_lib.py
+++ b/tensorflow/python/distribute/distribute_lib.py
@@ -876,6 +876,30 @@ class DistributionStrategyExtended(object):
     # Note: should support "colocate_with" argument.
     raise NotImplementedError("must be implemented in descendants")
 
+  def variable_created_in_scope(self, v):
+    """Tests whether `v` was created while this strategy scope was active.
+
+    Variables created inside the strategy scope are "owned" by it:
+
+    >>> with strategy.scope():
+    ...   v = tf.Variable(1.)
+    >>> strategy.variable_created_in_scope(v)
+    True
+
+    Variables created outside the strategy are not owned by it:
+
+    >>> v = tf.Variable(1.)
+    >>> strategy.variable_created_in_scope(v)
+    False
+
+    Args:
+      v: A `tf.Variable` instance.
+
+    Returns:
+      True if `v` was created inside the scope, False if not.
+    """
+    return v._distribute_strategy == self._container_strategy_weakref()  # pylint: disable=protected-access
+
   def read_var(self, v):
     """Reads the value of a variable.
 
@@ -1519,6 +1543,9 @@ class _DefaultDistributionExtended(DistributionStrategyExtended):
     _require_strategy_scope_extended(self)
     return ops.colocate_with(colocate_with_variable)
 
+  def variable_created_in_scope(self, v):
+    return v._distribute_strategy is None  # pylint: disable=protected-access
+
   def _distribute_dataset(self, dataset_fn):
     return self._call_dataset_fn(dataset_fn)
 
diff --git a/tensorflow/python/distribute/values.py b/tensorflow/python/distribute/values.py
index e7db285be5..19967f2c5e 100644
--- a/tensorflow/python/distribute/values.py
+++ b/tensorflow/python/distribute/values.py
@@ -571,11 +571,12 @@ ops.register_dense_tensor_like_type(DistributedVariable)
 
 
 def _validate_colocate_extended(v, extended):
-  if v.distribute_strategy.extended is not extended:
+  variable_strategy = v._distribute_strategy  # pylint: disable=protected-access
+  if variable_strategy.extended is not extended:
     raise ValueError(
         "`colocate_vars_with` must only be passed a variable created in this "
         "tf.distribute.Strategy.scope(), not %s created in scope: %s" %
-        (v, v.distribute_strategy,))
+        (v, variable_strategy))
 
 
 def validate_colocate_distributed_variable(v, extended):
@@ -595,7 +596,7 @@ def validate_colocate_tpu_variable(v, extended):
 
 
 def validate_colocate(v, extended):
-  if not hasattr(v, "distribute_strategy"):
+  if not hasattr(v, "_distribute_strategy"):
     raise ValueError(
         "`colocate_vars_with` must only be passed a variable created in this "
         "tf.distribute.Strategy.scope(), not: %r" % (v,))
@@ -1174,7 +1175,7 @@ class _ReplicaLocalSaveable(saver.BaseSaverBuilder.SaveableObject):
     # We use a callable so that we don't have to evaluate this expression
     # in the case where we are trying to restore instead of save.
     def tensor():
-      strategy = replica_local_variable.distribute_strategy
+      strategy = replica_local_variable._distribute_strategy  # pylint: disable=protected-access
       return strategy.extended.read_var(replica_local_variable)
 
     spec = saver.BaseSaverBuilder.SaveSpec(
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index f5a720b292..6b0422e71c 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -547,7 +547,8 @@ class Model(Network):
       # Validate all variables were correctly created in distribution scope.
       if self._distribution_strategy and not self._compile_distribution:
         for v in self.variables:
-          if v.distribute_strategy is not self._distribution_strategy:
+          strategy = self._distribution_strategy
+          if not strategy.extended.variable_created_in_scope(v):
             raise ValueError(
                 'Variable (%s) was not created in the distribution strategy '
                 'scope of (%s). It is most likely due to not all layers or '
@@ -556,7 +557,7 @@ class Model(Network):
                 'to the following.\n'
                 'with strategy.scope():\n'
                 '  model=_create_model()\n'
-                '  model.compile(...)'% (v, self._distribution_strategy))
+                '  model.compile(...)'% (v, strategy))
 
   @property
   def metrics(self):
diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index ec93038f1b..d31652eb89 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -1095,7 +1095,7 @@ class _MixedPrecisionVariableTest(test_util.TensorFlowTestCase):
   @test_util.run_in_graph_and_eager_modes()
   def testDistributeStrategy(self):
     v = resource_variable_ops.ResourceVariable(1, dtype=dtypes.int32)
-    self.assertIsNone(v.distribute_strategy)
+    self.assertIsNone(v._distribute_strategy)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/kernel_tests/variables_test.py b/tensorflow/python/kernel_tests/variables_test.py
index 4dc6d0a828..028ef11fc4 100644
--- a/tensorflow/python/kernel_tests/variables_test.py
+++ b/tensorflow/python/kernel_tests/variables_test.py
@@ -46,7 +46,7 @@ class VariablesTestCase(test.TestCase):
   @test_util.run_deprecated_v1
   def testDistributeStrategy(self):
     v = variables.VariableV1(0.0)
-    self.assertIsNone(v.distribute_strategy)
+    self.assertIsNone(v._distribute_strategy)
 
   @test_util.run_v1_only("b/120545219")
   def testInitialization(self):
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 2681deb78c..1a895fee87 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -593,7 +593,7 @@ class ResourceVariable(variables.VariableV1):
         constraint=self._constraint,
         dtype=self._dtype,
         name=self._shared_name + "_copy",
-        distribute_strategy=self.distribute_strategy)
+        distribute_strategy=self._distribute_strategy)
     memo[self._unique_id] = copied_variable
     return copied_variable
 
@@ -622,11 +622,6 @@ class ResourceVariable(variables.VariableV1):
     """The shape of this variable."""
     return self._shape
 
-  @property
-  def distribute_strategy(self):
-    """The `tf.distribute.Strategy` that this variable was created under."""
-    return self._distribute_strategy
-
   def _shape_as_list(self):
     if self.shape.ndims is None:
       return None
@@ -933,7 +928,7 @@ class ResourceVariable(variables.VariableV1):
         name=self._shared_name,
         dtype=self.dtype,
         constraint=self.constraint,
-        distribute_strategy=self.distribute_strategy), ()
+        distribute_strategy=self._distribute_strategy), ()
 
   def scatter_sub(self, sparse_delta, use_locking=False, name=None):
     """Subtracts `IndexedSlices` from this variable.
diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index 7eb2e14b42..3392b1aad9 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -1004,16 +1004,6 @@ class Variable(six.with_metaclass(VariableMetaclass,
     """The `Graph` of this variable."""
     raise NotImplementedError
 
-  @property
-  def distribute_strategy(self):
-    """The `tf.distribute.Strategy` that this variable was created under.
-
-    Returns:
-      A `tf.distribute.Strategy` or `None` if this variable was not created
-      inside the `scope()` of any strategy.
-    """
-    raise NotImplementedError
-
   @property
   def shape(self):
     """The `TensorShape` of this variable.
@@ -2206,7 +2196,7 @@ class RefVariable(VariableV1):
     return self._variable.graph
 
   @property
-  def distribute_strategy(self):
+  def _distribute_strategy(self):
     """The `tf.distribute.Strategy` that this variable was created under."""
     return None   # Ref variables are never created inside a strategy.
 
@@ -2603,7 +2593,7 @@ class PartitionedVariable(object):
     return self.get_shape()
 
   @property
-  def distribute_strategy(self):
+  def _distribute_strategy(self):
     """The `tf.distribute.Strategy` that this variable was created under."""
     # NOTE(yuefengz): Today, no partitioned variables in a distribute strategy.
     return None
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.-variable.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.-variable.pbtxt
index f042097acc..341ace0766 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.-variable.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.-variable.pbtxt
@@ -16,10 +16,6 @@ tf_class {
     name: "device"
     mtype: "<type \'property\'>"
   }
-  member {
-    name: "distribute_strategy"
-    mtype: "<type \'property\'>"
-  }
   member {
     name: "dtype"
     mtype: "<type \'property\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy-extended.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy-extended.pbtxt
index 37b620891f..5c4f090753 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy-extended.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy-extended.pbtxt
@@ -82,4 +82,8 @@ tf_class {
     name: "value_container"
     argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "variable_created_in_scope"
+    argspec: "args=[\'self\', \'v\'], varargs=None, keywords=None, defaults=None"
+  }
 }
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-variable.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-variable.pbtxt
index 2206f0f8d8..a80726d3bb 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.-variable.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.-variable.pbtxt
@@ -15,10 +15,6 @@ tf_class {
     name: "device"
     mtype: "<type \'property\'>"
   }
-  member {
-    name: "distribute_strategy"
-    mtype: "<type \'property\'>"
-  }
   member {
     name: "dtype"
     mtype: "<type \'property\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy-extended.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy-extended.pbtxt
index 37b620891f..5c4f090753 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy-extended.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy-extended.pbtxt
@@ -82,4 +82,8 @@ tf_class {
     name: "value_container"
     argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "variable_created_in_scope"
+    argspec: "args=[\'self\', \'v\'], varargs=None, keywords=None, defaults=None"
+  }
 }
-- 
GitLab


From 2225d4d16dcec29f5c4eeda4485618bc3ac4de99 Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Wed, 16 Jan 2019 06:13:58 -0800
Subject: [PATCH 0774/2345] Return the same list of dataset optimizations every
 time to fix non-determinism.

PiperOrigin-RevId: 229545908
---
 tensorflow/python/data/experimental/ops/optimization_options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/data/experimental/ops/optimization_options.py b/tensorflow/python/data/experimental/ops/optimization_options.py
index 9fe1f9499e..02747d3e36 100644
--- a/tensorflow/python/data/experimental/ops/optimization_options.py
+++ b/tensorflow/python/data/experimental/ops/optimization_options.py
@@ -132,4 +132,4 @@ class OptimizationOptions(options.OptionsBase):
       for optimization in optimizations_to_disable:
         if getattr(self, optimization) is not False:
           result.add(optimization)
-    return list(result)
+    return sorted(list(result))
-- 
GitLab


From 2834c69d95a3034c587f3cee80282a47055fa750 Mon Sep 17 00:00:00 2001
From: Dheeraj Rajaram Reddy <dheeraj98reddy@gmail.com>
Date: Wed, 16 Jan 2019 20:53:35 +0530
Subject: [PATCH 0775/2345] Add take_while to v1 golden API

---
 .../tools/api/golden/v1/tensorflow.data.experimental.pbtxt    | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
index a262c0f799..abc98a74b6 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
@@ -180,6 +180,10 @@ tf_module {
     name: "shuffle_and_repeat"
     argspec: "args=[\'buffer_size\', \'count\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
+  member_method {
+    name: "take_while"
+    argspec: "args=[\'predicate\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "unbatch"
     argspec: "args=[], varargs=None, keywords=None, defaults=None"
-- 
GitLab


From 625516717dc584267eaff196a3c9a62f0e18658f Mon Sep 17 00:00:00 2001
From: Dheeraj Rajaram Reddy <dheeraj98reddy@gmail.com>
Date: Wed, 16 Jan 2019 21:03:53 +0530
Subject: [PATCH 0776/2345] Add absl_py parameterized testing dependency to
 take_while_serialization test

---
 .../python/data/experimental/kernel_tests/serialization/BUILD    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD b/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
index 00b05e4d30..16c8aced6b 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
@@ -681,6 +681,7 @@ py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python/data/experimental/ops:take_while_ops",
         "//tensorflow/python/data/ops:dataset_ops",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
-- 
GitLab


From 1f0e45d698e07834c2e316843403f0742a206d8e Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Wed, 16 Jan 2019 07:42:12 -0800
Subject: [PATCH 0777/2345] [tf.data] Turning map paralellization off by
 default until an performance regression is resolved.

PiperOrigin-RevId: 229555646
---
 .../kernel_tests/optimization/optimize_dataset_test.py           | 1 -
 tensorflow/python/data/experimental/ops/optimization_options.py  | 1 -
 2 files changed, 2 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
index a08016ca89..e02f3b1572 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
@@ -268,7 +268,6 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     options = dataset_ops.Options()
     expected_optimizations = [
         "map_and_batch_fusion",
-        "map_parallelization",
         "noop_elimination",
         "shuffle_and_repeat_fusion",
     ]
diff --git a/tensorflow/python/data/experimental/ops/optimization_options.py b/tensorflow/python/data/experimental/ops/optimization_options.py
index 02747d3e36..be1fb4c7ca 100644
--- a/tensorflow/python/data/experimental/ops/optimization_options.py
+++ b/tensorflow/python/data/experimental/ops/optimization_options.py
@@ -125,7 +125,6 @@ class OptimizationOptions(options.OptionsBase):
       # user explicitly disables them.
       optimizations_to_disable = [
           "map_and_batch_fusion",
-          "map_parallelization",
           "noop_elimination",
           "shuffle_and_repeat_fusion",
       ]
-- 
GitLab


From 060b6e32adbcee5504377f5bd54a2e4044d78273 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 07:51:23 -0800
Subject: [PATCH 0778/2345] Use Ophint to support unidirectional sequence rnn
 (static)

PiperOrigin-RevId: 229556771
---
 .../lite/experimental/examples/lstm/BUILD     |  35 ++++
 .../experimental/examples/lstm/tflite_rnn.py  | 150 ++++++++++++++
 .../lstm/unidirectional_sequence_rnn_test.py  | 195 ++++++++++++++++++
 .../propagate_array_data_types.cc             |   6 +
 .../propagate_fixed_sizes.cc                  |  44 ++++
 tensorflow/lite/toco/import_tensorflow.cc     |  22 ++
 tensorflow/lite/toco/model.h                  |  10 +-
 tensorflow/lite/toco/tflite/operator.cc       |  36 ++++
 tensorflow/lite/toco/tooling_util.cc          |   1 +
 9 files changed, 498 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/lite/experimental/examples/lstm/tflite_rnn.py
 create mode 100644 tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_rnn_test.py

diff --git a/tensorflow/lite/experimental/examples/lstm/BUILD b/tensorflow/lite/experimental/examples/lstm/BUILD
index 0c351ee4ec..f39673c028 100644
--- a/tensorflow/lite/experimental/examples/lstm/BUILD
+++ b/tensorflow/lite/experimental/examples/lstm/BUILD
@@ -17,6 +17,19 @@ py_library(
     ],
 )
 
+py_library(
+    name = "tflite_rnn",
+    srcs = ["tflite_rnn.py"],
+    srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tensorflow:tensorflow_py",
+        "//tensorflow/lite/python:lite",
+        "//tensorflow/python:framework",
+        "@six_archive//:six",
+    ],
+)
+
 py_test(
     name = "unidirectional_sequence_lstm_test",
     size = "large",
@@ -38,3 +51,25 @@ py_test(
         "@six_archive//:six",
     ],
 )
+
+py_test(
+    name = "unidirectional_sequence_rnn_test",
+    size = "large",
+    srcs = ["unidirectional_sequence_rnn_test.py"],
+    srcs_version = "PY2AND3",
+    tags = [
+        "no_oss",
+        "no_pip",
+    ],
+    deps = [
+        ":tflite_rnn",
+        "//tensorflow:tensorflow_py",
+        "//tensorflow/examples/tutorials/mnist:input_data",
+        "//tensorflow/lite/python:lite",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:platform",
+        "//tensorflow/python/tools:optimize_for_inference",
+        "//third_party/py/numpy",
+        "@six_archive//:six",
+    ],
+)
diff --git a/tensorflow/lite/experimental/examples/lstm/tflite_rnn.py b/tensorflow/lite/experimental/examples/lstm/tflite_rnn.py
new file mode 100644
index 0000000000..e4aad18367
--- /dev/null
+++ b/tensorflow/lite/experimental/examples/lstm/tflite_rnn.py
@@ -0,0 +1,150 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""TfLite BasicRnnCell wrapper.
+
+TODO(renjieliu): Find a better home for this one.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+import itertools
+
+from tensorflow.lite.python import lite
+from tensorflow.python.keras import activations
+from tensorflow.python.layers import base as base_layer
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import init_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn_ops
+from tensorflow.python.ops import rnn_cell_impl
+
+
+class TfLiteRNNCell(rnn_cell_impl.LayerRNNCell):
+  """The most basic RNN cell.
+
+  This is used only for TfLite, it provides hints and it also makes the
+  variables in the desired for the tflite ops.
+  """
+
+  def __init__(self,
+               num_units,
+               activation=None,
+               reuse=None,
+               name=None,
+               dtype=None,
+               **kwargs):
+    """Initializes the parameters for an RNN cell.
+
+    Args:
+      num_units: int, The number of units in the RNN cell.
+      activation: Nonlinearity to use.  Default: `tanh`. It could also be string
+        that is within Keras activation function names.
+      reuse: (optional) Python boolean describing whether to reuse variables in
+        an existing scope. Raises an error if not `True` and the existing scope
+        already has the given variables.
+      name: String, the name of the layer. Layers with the same name will share
+        weights, but to avoid mistakes we require reuse=True in such cases.
+      dtype: Default dtype of the layer (default of `None` means use the type of
+        the first input). Required when `build` is called before `call`.
+      **kwargs: Dict, keyword named properties for common layer attributes, like
+        `trainable` etc when constructing the cell from configs of get_config().
+
+    Raises:
+      ValueError: If the existing scope already has the given variables.
+    """
+    super(TfLiteRNNCell, self).__init__(
+        _reuse=reuse, name=name, dtype=dtype, **kwargs)
+
+    # Inputs must be Rank-2.
+    self.input_spec = base_layer.InputSpec(ndim=2)
+
+    self._tflite_wrapper = lite.OpHint("UnidirectionalSequenceRnn")
+    self._num_units = num_units
+    if activation:
+      self._activation = activations.get(activation)
+    else:
+      self._activation = math_ops.tanh
+
+  @property
+  def state_size(self):
+    return self._num_units
+
+  @property
+  def output_size(self):
+    return self._num_units
+
+  def build(self, inputs_shape):
+    """Builds the RNN cell.
+
+    Args:
+      inputs_shape: Rnn input tensor shape.
+
+    Raises:
+      ValueError: If last dimension of the input shape is not known.
+    """
+    if inputs_shape[-1] is None:
+      raise ValueError("Expected inputs.shape[-1] to be known, saw shape: %s" %
+                       (inputs_shape,))
+
+    input_depth = inputs_shape[-1]
+
+    def add_variable_wrapped(name, shape, initializer, index):
+      var = self.add_variable(name, shape=shape, initializer=initializer)
+      return self._tflite_wrapper.add_input(
+          var, name=name, index_override=index)
+
+    self._input_weights = add_variable_wrapped(
+        "input_weights", [self._num_units, input_depth], None, 1)
+    self._recurrent_weights = add_variable_wrapped(
+        "recurrent_weights", [self._num_units, self._num_units], None, 2)
+    self._bias = add_variable_wrapped(
+        "bias",
+        shape=[self._num_units],
+        initializer=init_ops.zeros_initializer(dtype=self.dtype),
+        index=3)
+
+    self.built = True
+
+  def call(self, inputs, state):
+    """Most basic RNN: output = new_state = act(W * input + U * state + B)."""
+    inputs = self._tflite_wrapper.add_input(
+        inputs, tag="input", name="input", aggregate="stack", index_override=0)
+    state = self._tflite_wrapper.add_input(
+        state,
+        tag="hidden_state",
+        name="hidden_state",
+        aggregate="first",
+        index_override=4)
+    weights = array_ops.transpose(
+        array_ops.concat([self._input_weights, self._recurrent_weights], 1))
+    gate_inputs = math_ops.matmul(array_ops.concat([inputs, state], 1), weights)
+    gate_inputs = nn_ops.bias_add(gate_inputs, self._bias)
+    output = self._activation(gate_inputs)
+    output = self._tflite_wrapper.add_output(
+        output,
+        tag="output",
+        name="output",
+        index_override=1,
+        aggregate="stack")
+    return output, output
+
+  def get_config(self):
+    config = {
+        "num_units": self._num_units,
+        "activation": activations.serialize(self._activation),
+        "reuse": self._reuse,
+    }
+    base_config = super(TfLiteRNNCell, self).get_config()
+    return dict(itertools.chain(base_config.items(), config.items()))
diff --git a/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_rnn_test.py b/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_rnn_test.py
new file mode 100644
index 0000000000..6f9e2dd949
--- /dev/null
+++ b/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_rnn_test.py
@@ -0,0 +1,195 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+import tempfile
+import numpy as np
+import tensorflow as tf
+
+from tensorflow import flags
+
+from tensorflow.examples.tutorials.mnist import input_data
+from tensorflow.lite.experimental.examples.lstm.tflite_rnn import TfLiteRNNCell
+from tensorflow.lite.python.op_hint import convert_op_hints_to_stubs
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import test
+from tensorflow.python.tools import optimize_for_inference_lib
+
+FLAGS = flags.FLAGS
+
+# Number of steps to train model.
+TRAIN_STEPS = 1
+
+CONFIG = tf.ConfigProto(device_count={"GPU": 0})
+
+
+class UnidirectionalSequenceRnnTest(test_util.TensorFlowTestCase):
+
+  def __init__(self, *args, **kwargs):
+    super(UnidirectionalSequenceRnnTest, self).__init__(*args, **kwargs)
+    # Define constants
+    # Unrolled through 28 time steps
+    self.time_steps = 28
+    # Rows of 28 pixels
+    self.n_input = 28
+    # Learning rate for Adam optimizer
+    self.learning_rate = 0.001
+    # MNIST is meant to be classified in 10 classes(0-9).
+    self.n_classes = 10
+    # Batch size
+    self.batch_size = 16
+    # Rnn Units.
+    self.num_units = 16
+
+  def setUp(self):
+    super(UnidirectionalSequenceRnnTest, self).setUp()
+    # Import MNIST dataset
+    data_dir = tempfile.mkdtemp(dir=FLAGS.test_tmpdir)
+    self.mnist = input_data.read_data_sets(data_dir, one_hot=True)
+
+  def buildRnnLayer(self):
+    return tf.nn.rnn_cell.MultiRNNCell([
+        TfLiteRNNCell(self.num_units, name="rnn1"),
+        TfLiteRNNCell(self.num_units, name="rnn2")
+    ])
+
+  def buildModel(self, rnn_layer):
+    # Weights and biases for output softmax layer.
+    out_weights = tf.Variable(
+        tf.random_normal([self.num_units, self.n_classes]))
+    out_bias = tf.Variable(tf.random_normal([self.n_classes]))
+
+    # input image placeholder
+    x = tf.placeholder(
+        "float", [None, self.time_steps, self.n_input], name="INPUT_IMAGE")
+
+    # x is shaped [batch_size,time_steps,num_inputs]
+    rnn_input = tf.unstack(x, self.time_steps, 1)
+    outputs, _ = tf.nn.static_rnn(rnn_layer, rnn_input, dtype="float32")
+
+    # Compute logits by multiplying outputs[-1] of shape [batch_size,num_units]
+    # by the softmax layer's out_weight of shape [num_units,n_classes]
+    # plus out_bias
+    prediction = tf.matmul(outputs[-1], out_weights) + out_bias
+    output_class = tf.nn.softmax(prediction, name="OUTPUT_CLASS")
+
+    return x, prediction, output_class
+
+  def trainModel(self, x, prediction, output_class, sess):
+    # input label placeholder
+    y = tf.placeholder("float", [None, self.n_classes])
+    # Loss function
+    loss = tf.reduce_mean(
+        tf.nn.softmax_cross_entropy_with_logits(logits=prediction, labels=y))
+    # Optimization
+    opt = tf.train.AdamOptimizer(
+        learning_rate=self.learning_rate).minimize(loss)
+
+    # Initialize variables
+    sess.run(tf.global_variables_initializer())
+    for _ in range(TRAIN_STEPS):
+      batch_x, batch_y = self.mnist.train.next_batch(
+          batch_size=self.batch_size, shuffle=False)
+
+      batch_x = batch_x.reshape((self.batch_size, self.time_steps,
+                                 self.n_input))
+      sess.run(opt, feed_dict={x: batch_x, y: batch_y})
+
+  def saveAndRestoreModel(self, rnn_layer, sess, saver):
+    """Saves and restores the model to mimic the most common use case.
+
+    Args:
+      rnn_layer: The rnn layer either a single rnn cell or a multi rnn cell.
+      sess: Old session.
+      saver: saver created by tf.train.Saver()
+
+    Returns:
+      A tuple containing:
+
+      - Input tensor of the restored model.
+      - Prediction tensor of the restored model.
+      - Output tensor, which is the softwmax result of the prediction tensor.
+      - new session of the restored model.
+
+    """
+    model_dir = tempfile.mkdtemp(dir=FLAGS.test_tmpdir)
+    saver.save(sess, model_dir)
+
+    # Reset the graph.
+    tf.reset_default_graph()
+    x, prediction, output_class = self.buildModel(rnn_layer)
+
+    new_sess = tf.Session(config=CONFIG)
+    saver = tf.train.Saver()
+    saver.restore(new_sess, model_dir)
+    return x, prediction, output_class, new_sess
+
+  def getInferenceResult(self, x, output_class, sess):
+    b1, _ = self.mnist.train.next_batch(batch_size=1)
+    sample_input = np.reshape(b1, (1, self.time_steps, self.n_input))
+
+    expected_output = sess.run(output_class, feed_dict={x: sample_input})
+    frozen_graph = tf.graph_util.convert_variables_to_constants(
+        sess, sess.graph_def, [output_class.op.name])
+    return sample_input, expected_output, frozen_graph
+
+  def tfliteInvoke(self, graph, test_inputs, outputs):
+    tf.reset_default_graph()
+    # Turn the input into placeholder of shape 1
+    tflite_input = tf.placeholder(
+        "float", [1, self.time_steps, self.n_input], name="INPUT_IMAGE_LITE")
+    tf.import_graph_def(graph, name="", input_map={"INPUT_IMAGE": tflite_input})
+    with tf.Session() as sess:
+      curr = sess.graph_def
+      curr = convert_op_hints_to_stubs(graph_def=curr)
+
+    curr = optimize_for_inference_lib.optimize_for_inference(
+        curr, ["INPUT_IMAGE_LITE"], ["OUTPUT_CLASS"],
+        [tf.float32.as_datatype_enum])
+
+    tflite = tf.lite.toco_convert(
+        curr, [tflite_input], [outputs], allow_custom_ops=False)
+    interpreter = tf.lite.Interpreter(model_content=tflite)
+    interpreter.allocate_tensors()
+
+    input_index = interpreter.get_input_details()[0]["index"]
+    interpreter.set_tensor(input_index, test_inputs)
+    interpreter.invoke()
+    output_index = interpreter.get_output_details()[0]["index"]
+    result = interpreter.get_tensor(output_index)
+    # Reset all variables so it will not pollute other inferences.
+    interpreter.reset_all_variables()
+    return result
+
+  def testStaticRnnMultiRnnCell(self):
+    sess = tf.Session(config=CONFIG)
+
+    x, prediction, output_class = self.buildModel(self.buildRnnLayer())
+    self.trainModel(x, prediction, output_class, sess)
+
+    saver = tf.train.Saver()
+    x, prediction, output_class, new_sess = self.saveAndRestoreModel(
+        self.buildRnnLayer(), sess, saver)
+
+    test_inputs, expected_output, frozen_graph = self.getInferenceResult(
+        x, output_class, new_sess)
+
+    result = self.tfliteInvoke(frozen_graph, test_inputs, output_class)
+    self.assertTrue(np.allclose(expected_output, result, rtol=1e-6, atol=1e-2))
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc b/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
index 6d9aad66b6..7aec6728da 100644
--- a/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
+++ b/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
@@ -252,6 +252,12 @@ void SetDataTypeForAllOutputs(Model* model, Operator* op,
       SetDataTypeForAllOutputs(model, op, data_type);
       break;
     }
+    case OperatorType::kUnidirectionalSequenceRnn: {
+      const ArrayDataType data_type = model->GetArray(op->inputs[0]).data_type;
+      if (data_type != ArrayDataType::kFloat) return ::tensorflow::Status::OK();
+      SetDataTypeForAllOutputs(model, op, data_type);
+      break;
+    }
     case OperatorType::kUnique: {
       CHECK_EQ(op->outputs.size(), 2);
       const UniqueOperator* unique_op = static_cast<UniqueOperator*>(op);
diff --git a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
index 5185afd22e..1b1780a73b 100644
--- a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
+++ b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
@@ -1109,6 +1109,46 @@ void ProcessUnidirectionalSequenceLstmOperator(
   output_shape->ReplaceDims({timestamp, batch_size, output_size});
 }
 
+void ProcessUnidirectionalSequenceRnnOperator(
+    Model* model, UnidirectionalSequenceRnnOperator* op) {
+  auto& output_array = model->GetArray(op->outputs[0]);
+  if (output_array.has_shape()) {
+    // Shape already propagated.
+    return;
+  }
+
+  if (output_array.data_type == ArrayDataType::kNone) {
+    // Yield until the output type has been set by PropagateArrayDataTypes
+    return;
+  }
+
+  // TODO(renjieliu): check the inputs, as well as all kinds of weights.
+  const auto& input_array = model->GetArray(op->inputs[0]);
+  // Yield until input dims have been resolved.
+  if (!input_array.has_shape()) {
+    return;
+  }
+  const auto& input_shape = input_array.shape();
+  const int batch_size = input_shape.dims(1);
+  const int timestamp = input_shape.dims(0);
+
+  const auto& bias_array = model->GetArray(op->inputs[3]);
+  // Yield until input dims have been resolved.
+  if (!bias_array.has_shape()) {
+    return;
+  }
+
+  constexpr int kHiddenStateTensor = 4;
+  // b(115961645): This is a hack to work around.
+  model->GetArray(op->inputs[kHiddenStateTensor]).buffer.reset();
+
+  const auto& bias_shape = bias_array.shape();
+  const int output_size = bias_shape.dims(0);
+
+  Shape* output_shape = output_array.mutable_shape();
+  output_shape->ReplaceDims({timestamp, batch_size, output_size});
+}
+
 void ProcessSpaceToBatchNDOperator(Model* model, SpaceToBatchNDOperator* op) {
   const auto& input_array = model->GetArray(op->inputs[0]);
   // Yield until input dims have been resolved.
@@ -2037,6 +2077,10 @@ void ProcessUniqueOperator(Model* model, UniqueOperator* op) {
       ProcessUnidirectionalSequenceLstmOperator(
           model, static_cast<UnidirectionalSequenceLstmOperator*>(op));
       break;
+    case OperatorType::kUnidirectionalSequenceRnn:
+      ProcessUnidirectionalSequenceRnnOperator(
+          model, static_cast<UnidirectionalSequenceRnnOperator*>(op));
+      break;
     case OperatorType::kLstmCell:
       ProcessLstmCellOperator(model, static_cast<LstmCellOperator*>(op));
       break;
diff --git a/tensorflow/lite/toco/import_tensorflow.cc b/tensorflow/lite/toco/import_tensorflow.cc
index 86e04b2393..dac106b398 100644
--- a/tensorflow/lite/toco/import_tensorflow.cc
+++ b/tensorflow/lite/toco/import_tensorflow.cc
@@ -2317,6 +2317,27 @@ tensorflow::Status ConvertLeakyReluOperator(
   return tensorflow::Status::OK();
 }
 
+tensorflow::Status ConvertUnidirectionalSequenceRnn(
+    const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
+    Model* model) {
+  DCHECK_EQ(node.op(), "UnidirectionalSequenceRnn");
+
+  auto* op = new UnidirectionalSequenceRnnOperator();
+  const auto& indices = GetListAttr(node, "_tflite_input_indices");
+  if (indices.i_size() != node.input().size()) {
+    return tensorflow::errors::InvalidArgument("Input size does not match.");
+  }
+
+  for (const string& input : node.input()) {
+    op->inputs.push_back(input);
+  }
+  // Only use the last one as input.
+  op->outputs.push_back(node.name() + ":1");
+  model->operators.emplace_back(op);
+
+  return tensorflow::Status::OK();
+}
+
 }  // namespace
 
 namespace internal {
@@ -2454,6 +2475,7 @@ ConverterMapType GetTensorFlowNodeConverterMap() {
       {"Unpack", ConvertUnpackOperator},
       {"ZerosLike", ConvertSimpleOperator<TensorFlowZerosLikeOperator, 1, 1>},
       {"UnidirectionalSequenceLstm", ConvertUnidirectionalSequenceLstm},
+      {"UnidirectionalSequenceRnn", ConvertUnidirectionalSequenceRnn},
       {"MirrorPad", ConvertMirrorPadOperator},
       {"Unique", ConvertSimpleOperator<UniqueOperator, 1, 2>},
   });
diff --git a/tensorflow/lite/toco/model.h b/tensorflow/lite/toco/model.h
index bfa86c8059..296ed9fc74 100644
--- a/tensorflow/lite/toco/model.h
+++ b/tensorflow/lite/toco/model.h
@@ -158,7 +158,8 @@ enum class OperatorType : uint8 {
   kLeakyRelu,
   kAbs,
   kMirrorPad,
-  kUnique
+  kUnique,
+  kUnidirectionalSequenceRnn
 };
 
 // Helper to deal with TensorFlow arrays using a different ordering of
@@ -1965,6 +1966,13 @@ struct UniqueOperator : Operator {
   ArrayDataType idx_out_type = ArrayDataType::kInt32;
 };
 
+struct UnidirectionalSequenceRnnOperator : Operator {
+  UnidirectionalSequenceRnnOperator()
+      : Operator(OperatorType::kUnidirectionalSequenceRnn) {}
+  bool time_major;
+  FusedActivationFunctionType fused_activation_function;
+};
+
 // Alloc's are used for transient arrays only. An Alloc specifies which interval
 // of the "transient_data" workspace buffer passed to inference functions, is to
 // be used for the transient array at hand. The 'start' and 'end' values are
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index 8eb4d321ef..d9c051ebf7 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -1509,6 +1509,39 @@ class Unique : public BuiltinOperator<UniqueOperator, ::tflite::UniqueOptions,
   }
 };
 
+class UnidirectionalSequenceRnn
+    : public BuiltinOperator<UnidirectionalSequenceRnnOperator,
+                             ::tflite::SequenceRNNOptions,
+                             ::tflite::BuiltinOptions_SequenceRNNOptions> {
+ public:
+  using BuiltinOperator::BuiltinOperator;
+  flatbuffers::Offset<TfLiteOptions> WriteOptions(
+      const TocoOperator& op,
+      flatbuffers::FlatBufferBuilder* builder) const override {
+    return ::tflite::CreateSequenceRNNOptions(
+        *builder, /*time_major=*/true,
+        /*fused_activation_function=*/
+        ::tflite::ActivationFunctionType_TANH);
+  }
+  void ReadOptions(const TfLiteOptions& options,
+                   TocoOperator* op) const override {
+    // Only support tanh actication, so check that tflite type is tanh.
+    DCHECK(options.fused_activation_function() ==
+           ::tflite::ActivationFunctionType_TANH);
+  }
+
+  int GetVersion(const OperatorSignature& op_signature) const override {
+    return 1;
+  }
+
+  std::vector<bool> GetMutatingInputVariables(
+      const Operator& op) const override {
+    std::vector<bool> mutating_input_variables(op.inputs.size(), false);
+    mutating_input_variables[4] = true;
+    return mutating_input_variables;
+  }
+};
+
 std::unique_ptr<flexbuffers::Builder> WriteFlexOpOptions(
     const string& tensorflow_node_def) {
   auto fbb = absl::make_unique<flexbuffers::Builder>();
@@ -1847,6 +1880,9 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
                                       OperatorType::kMirrorPad));
   ops.push_back(MakeUnique<Unique>(::tflite::BuiltinOperator_UNIQUE,
                                    OperatorType::kUnique));
+  ops.push_back(MakeUnique<UnidirectionalSequenceRnn>(
+      ::tflite::BuiltinOperator_UNIDIRECTIONAL_SEQUENCE_RNN,
+      OperatorType::kUnidirectionalSequenceRnn));
 
   // Custom Operators.
   ops.push_back(
diff --git a/tensorflow/lite/toco/tooling_util.cc b/tensorflow/lite/toco/tooling_util.cc
index 2396de1a3d..9c0e8067d8 100644
--- a/tensorflow/lite/toco/tooling_util.cc
+++ b/tensorflow/lite/toco/tooling_util.cc
@@ -417,6 +417,7 @@ const char* OperatorTypeName(OperatorType type) {
     HANDLE_OPERATORTYPENAME_CASE(SquaredDifference)
     HANDLE_OPERATORTYPENAME_CASE(MirrorPad)
     HANDLE_OPERATORTYPENAME_CASE(Unique)
+    HANDLE_OPERATORTYPENAME_CASE(UnidirectionalSequenceRnn)
     default:
       LOG(FATAL) << "Unhandled op type";
 #undef HANDLE_OPERATORTYPENAME_CASE
-- 
GitLab


From 72c2e3562f7b393d414c57fd87cc32c8a7370b3b Mon Sep 17 00:00:00 2001
From: AG Ramesh <ag.ramesh@intel.com>
Date: Wed, 16 Jan 2019 08:20:12 -0800
Subject: [PATCH 0779/2345] Fix for compilation error when compiling with
 --config=mkl

---
 tensorflow/core/kernels/eigen_spatial_convolutions_test.cc | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/kernels/eigen_spatial_convolutions_test.cc b/tensorflow/core/kernels/eigen_spatial_convolutions_test.cc
index 03002adec4..14c6903a43 100644
--- a/tensorflow/core/kernels/eigen_spatial_convolutions_test.cc
+++ b/tensorflow/core/kernels/eigen_spatial_convolutions_test.cc
@@ -1605,6 +1605,7 @@ static void PackLhsHelper(int iters,
       /*inner_dim_reordered*/ false,                  //
       /*Alignment*/ 0>;
 
+  using Traits = typename Eigen::internal::gebp_traits<float, float>;
 #if defined(TENSORFLOW_USE_MKLDNN_CONTRACTION_KERNEL)
   using PackLhsImpl = Eigen::internal::mkldnn_gemm_pack<float, Eigen::Index,
                                                         SubMapper, ColMajor>;
@@ -1697,9 +1698,9 @@ static void PackLhsHelper(int iters,
     SubMapper sub_mapper =
         input_mappers[filter_idx].getSubMapper(row_offset, col_offset);
 
-    // NOTE: Eigen gemm_pack_lhs accepts contraction depth (k-th dimension) as a
-    // first argument (aka block cols). MKL-DNN pack is generic for lhs and rhs
-    // and accepts block rows and cols in the same order for lhs and rhs.
+// NOTE: Eigen gemm_pack_lhs accepts contraction depth (k-th dimension) as a
+// first argument (aka block cols). MKL-DNN pack is generic for lhs and rhs
+// and accepts block rows and cols in the same order for lhs and rhs.
 #if defined(TENSORFLOW_USE_MKLDNN_CONTRACTION_KERNEL)
     pack_lhs(packed.data() + packed_offset, sub_mapper, rows, cols);
 #else
-- 
GitLab


From d122a13913a1d058cb872b9c443ecbfef8c8bc8d Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Wed, 16 Jan 2019 08:23:59 -0800
Subject: [PATCH 0780/2345] Fix for ResourceVariables and variants when using
 while_v2 + GradientTape.

PiperOrigin-RevId: 229561249
---
 tensorflow/python/kernel_tests/BUILD          |  1 +
 .../python/kernel_tests/while_v2_test.py      | 18 ++++++
 tensorflow/python/ops/while_v2.py             | 59 ++++++++++++++-----
 3 files changed, 64 insertions(+), 14 deletions(-)

diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 7a514875d6..40bf4aedd0 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -3538,6 +3538,7 @@ cuda_py_test(
         "@absl_py//absl/testing:parameterized",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
+        "//tensorflow/python/eager:def_function",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:control_flow_ops",
diff --git a/tensorflow/python/kernel_tests/while_v2_test.py b/tensorflow/python/kernel_tests/while_v2_test.py
index 1f2c6f94c5..061d787760 100644
--- a/tensorflow/python/kernel_tests/while_v2_test.py
+++ b/tensorflow/python/kernel_tests/while_v2_test.py
@@ -23,6 +23,8 @@ from absl.testing import parameterized
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python.eager import backprop
+from tensorflow.python.eager import context
+from tensorflow.python.eager import def_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import meta_graph
@@ -34,6 +36,7 @@ from tensorflow.python.ops import functional_ops
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import list_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import variables
 from tensorflow.python.ops import while_v2
 from tensorflow.python.ops.control_flow_ops import while_loop as while_loop_v1
 from tensorflow.python.ops.while_v2 import while_loop as while_loop_v2
@@ -65,6 +68,21 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
       self.assertEqual(16., eval_result[0])
       self.assertSequenceEqual(sess.run(grad), [32.])
 
+  def testGradientTapeResourceVariable(self):
+    with context.eager_mode():
+      v = variables.Variable(1.)
+
+      @def_function.function
+      def fnWithLoop():  # pylint: disable=invalid-name
+        with backprop.GradientTape() as tape:
+          _, x = while_loop_v2(
+              lambda i, _: i < 2,
+              lambda i, x: (i + 1, x * v),
+              [0, 2.])
+        return tape.gradient(x, v)
+
+      self.assertAllEqual(fnWithLoop(), 4.0)
+
   @test_util.run_deprecated_v1
   def testMultipleLoopVarsBasic(self):
     x = constant_op.constant(5.)
diff --git a/tensorflow/python/ops/while_v2.py b/tensorflow/python/ops/while_v2.py
index 50011e34b1..0e427d3c6a 100644
--- a/tensorflow/python/ops/while_v2.py
+++ b/tensorflow/python/ops/while_v2.py
@@ -38,6 +38,7 @@ from tensorflow.python.ops import control_flow_util
 from tensorflow.python.ops import control_flow_util_v2 as util
 from tensorflow.python.ops import custom_gradient
 from tensorflow.python.ops import gen_functional_ops
+from tensorflow.python.ops import gen_resource_variable_ops
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import list_ops
 from tensorflow.python.ops import math_ops
@@ -263,20 +264,9 @@ def _WhileGrad(op, *grads):  # pylint: disable=invalid-name
   assert not _is_in_xla_context() or maximum_iterations is not None
   maximum_iterations = _validate_and_convert_to_tensor(maximum_iterations)
 
-  # Set the incoming gradient of non-trainable inputs to None. It is possible
-  # that we receive non-None gradients for non-trainable types in nested while
-  # loops because we accumulate outputs of the inner while as variant tensors
-  # which are trainable and hence receive zeros_like tensors in the gradient
-  # pass. The non-trainable tensors then receive the popped zeros tensor from
-  # this zeros variant. The gradient for the loop vars corresponding to these
-  # tensors is None or zeros (this happens only if the loop var is accumulated
-  # as well) in _grad_fn so we reset these.
-  # TODO(b/118712257): Remove the IsTrainable filter once we can handle None
-  # output grads in _grad_fn.
-  grads = [
-      None if not _is_trainable(output) else grad
-      for grad, output in zip(grads, body_graph.outputs)
-  ]
+  grads = [_preprocess_grad(grad, body_out, while_out)
+           for grad, body_out, while_out
+           in zip(grads, body_graph.outputs, while_op.outputs)]
 
   # We compute the gradient for the sub-graph between trainable ys and xs
   # with non-None incoming gradients. We later pad the None's to the list of
@@ -352,6 +342,47 @@ def _WhileGrad(op, *grads):  # pylint: disable=invalid-name
   return none_padded_outputs
 
 
+def _preprocess_grad(grad, body_graph_output, while_op_output):
+  """Returns the initial gradient to be used for a given output tensor.
+
+  Args:
+    grad: the original gradient Tensor passed to the gradient function.
+    body_graph_output: the corresponding Tensor in the body graph.
+    while_op_output: the corresponding Tensor output of the While op.
+
+  Returns:
+    A Tensor or None.
+  """
+  # Set the incoming gradient of non-trainable inputs to None. It is possible
+  # that we receive non-None gradients for non-trainable types in nested while
+  # loops because we accumulate outputs of the inner while as variant tensors
+  # which are trainable and hence receive zeros_like tensors in the gradient
+  # pass. The non-trainable tensors then receive the popped zeros tensor from
+  # this zeros variant. The gradient for the loop vars corresponding to these
+  # tensors is None or zeros (this happens only if the loop var is accumulated
+  # as well) in _grad_fn so we reset these.
+  # TODO(b/118712257): Remove once we can handle None output grads in _grad_fn.
+  if not _is_trainable(body_graph_output):
+    return None
+
+  # GradientTape initializes resource and variant grads as None instead of
+  # zeros. Set to zeros so _GradientsHelper computes the gradients instead of
+  # returning None.
+  if (while_op_output.dtype in (dtypes.resource, dtypes.variant)
+      and grad is None):
+    return _zeros_like(while_op_output)
+
+  return grad
+
+
+def _zeros_like(op_output):
+  """Like array_ops.zeros_like() but also accepts resource var handles."""
+  if op_output.dtype == dtypes.resource:
+    return array_ops.zeros(
+        gen_resource_variable_ops.variable_shape(op_output))
+  return array_ops.zeros_like(op_output)
+
+
 def _is_trainable(tensor):
   """Returns whether the given tensor is trainable."""
   if not gradients_impl.IsTrainable(tensor):
-- 
GitLab


From e74388ec6d1e8aeadd7aaa787544f319175dd2a6 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Wed, 16 Jan 2019 17:15:17 +0000
Subject: [PATCH 0781/2345] Fix softmax_cross_entropy issue

This fix is from 24397/24736 and all credit goes to @felix-schneider

This fix fixes 24397. This fix fixes 24736.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/ops/losses/losses_impl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/losses/losses_impl.py b/tensorflow/python/ops/losses/losses_impl.py
index 7f88ccd879..3393e75335 100644
--- a/tensorflow/python/ops/losses/losses_impl.py
+++ b/tensorflow/python/ops/losses/losses_impl.py
@@ -794,7 +794,7 @@ def softmax_cross_entropy(
 
     if label_smoothing > 0:
       num_classes = math_ops.cast(
-          array_ops.shape(onehot_labels)[1], logits.dtype)
+          array_ops.shape(onehot_labels)[-1], logits.dtype)
       smooth_positives = 1.0 - label_smoothing
       smooth_negatives = label_smoothing / num_classes
       onehot_labels = onehot_labels * smooth_positives + smooth_negatives
-- 
GitLab


From 59e9acf78787ee2e46106672533aaff8032f69b0 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Wed, 16 Jan 2019 09:20:44 -0800
Subject: [PATCH 0782/2345] Bump the googletest dependency version for TF

PiperOrigin-RevId: 229569726
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 7eb1d8a3c6..354148f034 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -408,11 +408,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
 
     tf_http_archive(
         name = "com_google_googletest",
-        sha256 = "61eee610f136c1edc693d979647a4bb2ca253d60e6964724b61af85d32a41251",
-        strip_prefix = "googletest-6729a1361150131bc5d394d5cd2b4cdf0953ee7b",
+        sha256 = "ff7a82736e158c077e76188232eac77913a15dac0b22508c390ab3f88e6d6d86",
+        strip_prefix = "googletest-b6cd405286ed8635ece71c72f118e659f4ade3fb",
         urls = [
-            "https://mirror.bazel.build/github.com/google/googletest/archive/6729a1361150131bc5d394d5cd2b4cdf0953ee7b.zip",
-            "https://github.com/google/googletest/archive/6729a1361150131bc5d394d5cd2b4cdf0953ee7b.zip",
+            "https://mirror.bazel.build/github.com/google/googletest/archive/b6cd405286ed8635ece71c72f118e659f4ade3fb.zip",
+            "https://github.com/google/googletest/archive/b6cd405286ed8635ece71c72f118e659f4ade3fb.zip",
         ],
     )
 
-- 
GitLab


From 899f788738014fe438105fe30100e768b5f7900c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 09:45:13 -0800
Subject: [PATCH 0783/2345] Add a missing explicit dependency between
 :nn_ops_test (which references the AvgPool op) and :pooling_ops.

PiperOrigin-RevId: 229573584
---
 tensorflow/core/kernels/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 374aa88d2d..96536e6945 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -4011,6 +4011,7 @@ tf_cuda_cc_test(
         ":nn",
         ":ops_testutil",
         ":ops_util",
+        ":pooling_ops",
         "//tensorflow/cc:cc_ops",
         "//tensorflow/cc:cc_ops_internal",
         "//tensorflow/core:core_cpu",
-- 
GitLab


From 9728b12bf7a2e4cf4010a9baaee5626fd2bd8140 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 09:53:50 -0800
Subject: [PATCH 0784/2345] Update Google Cloud Bigtable C++ Client to the
 v0.5.0 release.

PiperOrigin-RevId: 229574976
---
 .../kernels/test_kernels/bigtable_test_client.cc      | 11 +++++++++++
 .../kernels/test_kernels/bigtable_test_client.h       |  6 ++++++
 2 files changed, 17 insertions(+)

diff --git a/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.cc b/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.cc
index c04d5578ee..e6fda9e617 100644
--- a/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.cc
+++ b/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.cc
@@ -410,6 +410,17 @@ BigtableTestClient::AsyncCheckAndMutateRow(
   return nullptr;
 }
 
+std::unique_ptr<
+    grpc::ClientAsyncReaderInterface<google::bigtable::v2::ReadRowsResponse>>
+BigtableTestClient::AsyncReadRows(
+    grpc::ClientContext* context,
+    const google::bigtable::v2::ReadRowsRequest& request,
+    grpc::CompletionQueue* cq, void* tag) {
+  LOG(WARNING) << "Call to InMemoryDataClient::" << __func__
+               << "(); this will likely cause a crash!";
+  return nullptr;
+}
+
 std::shared_ptr<grpc::Channel> BigtableTestClient::Channel() {
   LOG(WARNING) << "Call to InMemoryDataClient::Channel(); this will likely "
                   "cause a crash!";
diff --git a/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.h b/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.h
index 8570590457..8e1326f2ce 100644
--- a/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.h
+++ b/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.h
@@ -87,6 +87,12 @@ class BigtableTestClient : public ::google::cloud::bigtable::DataClient {
       const google::bigtable::v2::CheckAndMutateRowRequest& request,
       grpc::CompletionQueue* cq) override;
 
+  std::unique_ptr<
+      grpc::ClientAsyncReaderInterface<google::bigtable::v2::ReadRowsResponse>>
+  AsyncReadRows(grpc::ClientContext* context,
+                const google::bigtable::v2::ReadRowsRequest& request,
+                grpc::CompletionQueue* cq, void* tag) override;
+
   std::shared_ptr<grpc::Channel> Channel() override;
 
  private:
-- 
GitLab


From 3a2741544dc6aaf0abfe47d8fdd35d710dc7ffb0 Mon Sep 17 00:00:00 2001
From: Billy Lamberta <blamb@google.com>
Date: Wed, 16 Jan 2019 10:00:19 -0800
Subject: [PATCH 0785/2345] Publish /lite

PiperOrigin-RevId: 229576111
---
 tensorflow/lite/g3doc/custom_operators.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/lite/g3doc/custom_operators.md b/tensorflow/lite/g3doc/custom_operators.md
index 18202cbfb9..2d80668f37 100644
--- a/tensorflow/lite/g3doc/custom_operators.md
+++ b/tensorflow/lite/g3doc/custom_operators.md
@@ -137,9 +137,9 @@ operations instead of a single operator.
 
 ## Special TF Graph Attributes
 
-When Toco converts a TF graph into TFLite format, it makes some assumption about
-custom operations that might be not correct. In this case, the generated graph
-can be not executable.
+When `tflite_convert` converts a TensorFlow graph into TFLite format, it makes
+some assumption about custom operations that might be not correct. In this case,
+the generated graph may not execute.
 
 It is possible to add aditional information about your custom op output to TF
 graph before it is converted. The following attributes are supported:
-- 
GitLab


From ef5d41d827b4130cc3cedf08454e54ce1340f6eb Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Wed, 16 Jan 2019 10:11:41 -0800
Subject: [PATCH 0786/2345] Refactoring: Get signature directly from concrete
 function.

We can do this now, because signature is stored on the function, and it will clean the list_all_concrete_functions return value.

PiperOrigin-RevId: 229578645
---
 tensorflow/python/eager/def_function.py       | 51 ++++++++-----------
 tensorflow/python/eager/def_function_test.py  |  9 +++-
 .../saved_model/function_serialization.py     |  7 ++-
 tensorflow/python/saved_model/load_test.py    |  4 +-
 tensorflow/python/saved_model/save.py         |  2 +-
 5 files changed, 36 insertions(+), 37 deletions(-)

diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py
index 977f548b61..aef8f9ccda 100644
--- a/tensorflow/python/eager/def_function.py
+++ b/tensorflow/python/eager/def_function.py
@@ -447,50 +447,41 @@ class PolymorphicFunction(object):
     return initialize_variables.get_concrete_function()
 
   def _list_all_concrete_functions_for_serialization(self):
-    """Returns all of the concrete functions.
+    """Returns all concrete functions for serialization.
 
     Returns:
-      A list of tuples in the form (signature, concrete_function), where
-      concrete function is an instance of `Function`.
+      A list of instances of `Function`.
     """
-    input_signature = self._input_signature
-    if input_signature is not None:
+    if self._input_signature is not None:
       self.get_concrete_function()
     concrete_functions = []
-    for signature in self._cached_input_signatures:
-      flattened = nest.flatten(signature)
-      if any(
-          isinstance(arg, func_graph_module.UnknownArgument)
-          for arg in flattened):
-        logging.info("Unsupported signature for serialization: %s.", signature)
-        continue
-      concrete_function = self.get_concrete_function(*signature)
-      concrete_functions.append((signature, concrete_function))
-    return concrete_functions
-
-  @property
-  def _cached_input_signatures(self):
-    """All input signatures used to call this PolymorphicFunction."""
-    seen = list()
-    # We are using a list so that:
-    #  - the returned collection is deterministic, and
-    #  - we can use a custom equality operator (is_same_structure).
-    # This is run only at serialization time on likely very small inputs so we
-    # are not concerned about O(n^2) runtime.
     # pylint: disable=protected-access
-    concrete_functions = []
     if self._stateful_fn:
       concrete_functions.extend(self._stateful_fn._function_cache.values())
     if self._stateless_fn:
       concrete_functions.extend(self._stateless_fn._function_cache.values())
+    # pylint: enable=protected-access
+    deduplicated_concrete_functions = list()
+    seen_signatures = list()
+    # We are using a list so that:
+    #  - the returned collection is deterministic, and
+    #  - we can use a custom equality operator (is_same_structure).
+    # This is run only at serialization time on likely very small inputs so we
+    # are not concerned about O(n^2) runtime.
     for concrete_function in concrete_functions:
       signature, _ = concrete_function.structured_input_signature
+      flattened = nest.flatten(signature)
+      if any(
+          isinstance(arg, func_graph_module.UnknownArgument)
+          for arg in flattened):
+        logging.info("Unsupported signature for serialization: %s.", signature)
+        continue
       equal_to_signature = functools.partial(
           function_lib.is_same_structure, signature, check_values=True)
-      if not any(equal_to_signature(s) for s in seen):
-        yield signature
-        seen.append(signature)
-    # pylint: enable=protected-access
+      if not any(equal_to_signature(s) for s in seen_signatures):
+        deduplicated_concrete_functions.append(concrete_function)
+        seen_signatures.append(signature)
+    return deduplicated_concrete_functions
 
   def get_concrete_function(self, *args, **kwargs):
     """Returns a `Function` object specialized to inputs and execution context.
diff --git a/tensorflow/python/eager/def_function_test.py b/tensorflow/python/eager/def_function_test.py
index 77cc8ee981..256ef511a3 100644
--- a/tensorflow/python/eager/def_function_test.py
+++ b/tensorflow/python/eager/def_function_test.py
@@ -247,7 +247,7 @@ class DefFunctionTest(test.TestCase):
     concrete = compute.get_concrete_function(
         tensor_spec.TensorSpec(None, dtypes.float32))
     self.assertAllClose(4., concrete(constant_op.constant(2.)))
-    input_signature, = compute._cached_input_signatures
+    input_signature, _ = concrete.structured_input_signature
     self.assertEqual(
         tuple(input_signature),
         (tensor_spec.TensorSpec(None, dtypes.float32),))
@@ -260,8 +260,13 @@ class DefFunctionTest(test.TestCase):
 
     f(constant_op.constant([[3., 4.]]), constant_op.constant([2.]))
     f(constant_op.constant([[3, 4, 5]]), constant_op.constant([2]))
+
+    concrete_functions = f._list_all_concrete_functions_for_serialization()
+    signatures_for_serialization = [
+        c.structured_input_signature[0] for c in concrete_functions
+    ]
     self.assertEqual(
-        set(f._cached_input_signatures),
+        set(signatures_for_serialization),
         set(((tensor_spec.TensorSpec([1, 2], dtypes.float32),
               tensor_spec.TensorSpec([1], dtypes.float32)),
              (tensor_spec.TensorSpec([1, 3], dtypes.int32),
diff --git a/tensorflow/python/saved_model/function_serialization.py b/tensorflow/python/saved_model/function_serialization.py
index 8e8e3bc856..6496b2b2a4 100644
--- a/tensorflow/python/saved_model/function_serialization.py
+++ b/tensorflow/python/saved_model/function_serialization.py
@@ -48,7 +48,7 @@ def serialize_polymorphic_function(polymorphic_function, node_ids):
   proto.function_spec.CopyFrom(function_spec_proto)
   all_concrete_functions = \
       polymorphic_function._list_all_concrete_functions_for_serialization()  # pylint: disable=protected-access
-  for signature, concrete_function in all_concrete_functions:
+  for concrete_function in all_concrete_functions:
     bound_inputs = []
     try:
       for capture in concrete_function.captured_inputs:
@@ -60,10 +60,13 @@ def serialize_polymorphic_function(polymorphic_function, node_ids):
           "captures tensor %s which is unsupported or not reachable from root.",
           concrete_function.name, capture)
       continue
+    signature_args, signature_kwargs = \
+        concrete_function.structured_input_signature
+    del signature_kwargs
     function_proto = proto.monomorphic_function.add()
     function_proto.concrete_function = concrete_function.name
     function_proto.canonicalized_input_signature.CopyFrom(
-        coder.encode_structure(signature))
+        coder.encode_structure(signature_args))
     structured_outputs = func_graph_module.convert_structure_to_signature(
         concrete_function.structured_outputs)
     function_proto.output_signature.CopyFrom(
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index 9eea170bf0..c8c8f6e1b3 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -401,8 +401,8 @@ class LoadTest(test.TestCase):
     self.assertAllEqual([2], root.f(constant_op.constant([1])).numpy())
     self.assertAllEqual([2, 4], root.f(constant_op.constant([1, 2])).numpy())
 
-    concrete_functions = root.f._list_all_concrete_functions_for_serialization()
-    self.assertEqual(1, len(concrete_functions))  # pylint: disable=protected-access
+    concrete_functions = root.f._list_all_concrete_functions_for_serialization()  # pylint: disable=protected-access
+    self.assertEqual(1, len(concrete_functions))
 
     imported = self.cycle(root)
 
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index d383bbecfc..ef3f67bbce 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -560,7 +560,7 @@ def _fill_meta_graph_def(meta_graph_def, saveable_view, signature_functions,
 
   with exported_graph.as_default():
     signatures = _generate_signatures(signature_functions, resource_map)
-    for _, concrete_function in concrete_functions:
+    for concrete_function in concrete_functions:
       concrete_function.add_to_graph()
     saver_def = saver.to_proto()
     meta_graph_def.saver_def.CopyFrom(saver_def)
-- 
GitLab


From 7226c27f4d0161954ee36b0fcb32275a2942d5bc Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Wed, 16 Jan 2019 10:13:48 -0800
Subject: [PATCH 0787/2345] Use new _ property for distribute strategy on
 resource variable.

PiperOrigin-RevId: 229579047
---
 .../distribute/python/mirrored_strategy_multigpu_test.py        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index 59d711ae01..6727c7d703 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -267,7 +267,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
     self.assertIs(strategy, var.distribute_strategy)
     for d in var.devices:
       self.assertEqual(d, var.get(d).device)
-      self.assertIs(strategy, var.get(d).distribute_strategy)
+      self.assertIs(strategy, var.get(d)._distribute_strategy)  # pylint: disable=protected-access
 
   def testVariableInFuncGraph(self, distribution):
     def model_fn():
-- 
GitLab


From 352525990469da6694abaa4e70edfdbb287dd984 Mon Sep 17 00:00:00 2001
From: Billy Lamberta <blamb@google.com>
Date: Wed, 16 Jan 2019 10:14:45 -0800
Subject: [PATCH 0788/2345] Add tutorial vid links.

PiperOrigin-RevId: 229579217
---
 tensorflow/lite/g3doc/performance/gpu.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tensorflow/lite/g3doc/performance/gpu.md b/tensorflow/lite/g3doc/performance/gpu.md
index 14ca54a158..c738922612 100644
--- a/tensorflow/lite/g3doc/performance/gpu.md
+++ b/tensorflow/lite/g3doc/performance/gpu.md
@@ -24,6 +24,9 @@ The easiest way to try out the experimental GPU delegate is to follow the below
 
 ### Android (with Android Studio)
 
+For a step-by-step tutorial, watch the
+[Experimental GPU Delegate for Android](https://youtu.be/Xkhgre8r5G0) video.
+
 Note: This requires OpenGL ES 3.1 or higher.
 
 #### Step 1. Clone the TensorFlow source code and open it in Android Studio
@@ -55,6 +58,9 @@ run on the GPU.
 
 ### iOS (with XCode)
 
+For a step-by-step tutorial, watch the
+[Experimental GPU Delegate for iOS](https://youtu.be/a5H4Zwjp49c) video.
+
 Note: This requires XCode v10.1 or later.
 
 #### Step 1. Get the demo source code and make sure it compiles.
-- 
GitLab


From ff12131917fb3f9c789063e3a78e1d8a41e49776 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 10:17:33 -0800
Subject: [PATCH 0789/2345] Automated rollback of commit
 8154ec0b4a893657e48b838c06e1f988aeac4115

PiperOrigin-RevId: 229579722
---
 .../process_function_library_runtime.cc       |  10 +-
 tensorflow/core/grappler/optimizers/BUILD     |   1 -
 .../grappler/optimizers/function_optimizer.cc |  74 +++++------
 .../optimizers/function_optimizer_test.cc     | 122 ------------------
 4 files changed, 38 insertions(+), 169 deletions(-)

diff --git a/tensorflow/core/common_runtime/process_function_library_runtime.cc b/tensorflow/core/common_runtime/process_function_library_runtime.cc
index 76c75ad3d2..b236343a0f 100644
--- a/tensorflow/core/common_runtime/process_function_library_runtime.cc
+++ b/tensorflow/core/common_runtime/process_function_library_runtime.cc
@@ -568,12 +568,15 @@ Status ProcessFunctionLibraryRuntime::InstantiateMultiDevice(
   DumpGraph("Before running POST_PLACEMENT passes", graph.get());
   TF_RETURN_IF_ERROR(OptimizationPassRegistry::Global()->RunGrouping(
       OptimizationPassRegistry::POST_PLACEMENT, optimization_options));
+  DumpGraph("Before running POST_REWRITE_FOR_EXEC passes", graph.get());
+  TF_RETURN_IF_ERROR(OptimizationPassRegistry::Global()->RunGrouping(
+      OptimizationPassRegistry::POST_REWRITE_FOR_EXEC, optimization_options));
+  DumpGraph("After all optimization passes", graph.get());
 
   Device* cpu_device;
   TF_RETURN_IF_ERROR(device_mgr_->LookupDevice("CPU:0", &cpu_device));
 
   if (options.optimize_graph_fn) {
-    DumpGraph("Before running graph optimization fn", graph.get());
     Status status = options.optimize_graph_fn(std::move(ret_node_names),
                                               &data->overlay_lib_, device_set,
                                               cpu_device, &graph);
@@ -584,11 +587,6 @@ Status ProcessFunctionLibraryRuntime::InstantiateMultiDevice(
     DumpGraph("After optimization", graph.get());
   }
 
-  DumpGraph("Before running POST_REWRITE_FOR_EXEC passes", graph.get());
-  TF_RETURN_IF_ERROR(OptimizationPassRegistry::Global()->RunGrouping(
-      OptimizationPassRegistry::POST_REWRITE_FOR_EXEC, optimization_options));
-  DumpGraph("After all optimization passes", graph.get());
-
   std::unordered_map<string, std::unique_ptr<Graph>> subgraphs;
   TF_RETURN_IF_ERROR(
       PartitionFunctionGraph(device_set, std::move(graph), &subgraphs));
diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 70e0f226fc..ee216f80f9 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -181,7 +181,6 @@ tf_cuda_cc_test(
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/utils:grappler_test",
-        "@com_google_absl//absl/algorithm:container",
     ],
 )
 
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index 794e87dc38..bd69405b1a 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -27,8 +27,6 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/device_mgr.h"
 #include "tensorflow/core/common_runtime/device_set.h"
 #include "tensorflow/core/common_runtime/function.h"
-#include "tensorflow/core/common_runtime/lower_if_while.h"
-#include "tensorflow/core/common_runtime/optimization_registry.h"
 #include "tensorflow/core/common_runtime/placer.h"
 #include "tensorflow/core/common_runtime/process_function_library_runtime.h"
 #include "tensorflow/core/framework/attr_value_util.h"
@@ -1367,6 +1365,20 @@ Status IsInlinableIndirectFunctionCall(const FunctionOptimizerContext& ctx,
         SummarizeNodeDef(func_node));
   }
 
+  // TODO(b/120991525, b/120986912): We need to lower `If` and `While` nodes to
+  // `Switch` nodes after function inlining (one more PRE_PLACEMENT pass?), but
+  // because of the reason described above we are not sure that it's safe, for
+  // now just disable inlining functions with functional control flow.
+  const auto is_functional_ctrl_flow_op = [](const NodeDef& node) {
+    return IsIf(node) || IsWhile(node);
+  };
+  if (absl::c_any_of(func.node_def(), is_functional_ctrl_flow_op)) {
+    return errors::FailedPrecondition(
+        "Can't inline function with `If` or `While` nodes in the function "
+        "body: ",
+        SummarizeNodeDef(func_node));
+  }
+
   return Status::OK();
 }
 
@@ -1476,46 +1488,22 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
 
   const string prefix = strings::StrCat(func_node.name(), "/");
 
-  // ------------------------------------------------------------------------ //
-  // Grappler receives the graph after PRE_PLACEMENT, Placer, and POST_PLACEMENT
-  // passes, so each node has a valid device assignment. Also V2 control
-  // flow ops (functional If and While) lowered to V1 control flow (Switch and
-  // Merge nodes). To keep graph valid for execution we must assign device to
-  // every inlined graph node, and also lower the control flow.
-
-  // Control flow lowering and Placer works with a Graph object.
-  std::unique_ptr<Graph> func_body_graph =
-      absl::make_unique<Graph>(ctx->function_library());
-
-  GraphConstructorOptions opts;
-  TF_RETURN_IF_ERROR(
-      ConvertGraphDefToGraph(opts, item.graph, func_body_graph.get()));
-
-  GraphOptimizationPassOptions opt_options;
-  opt_options.graph = &func_body_graph;
-  opt_options.flib_def = ctx->mutable_function_library();
-
-  // TODO(ezhulenev): Should we run full PRE_PLACEMENT pass here? And
-  // POST_PLACEMENT after placer?
-  LowerIfWhilePass pass;
-  TF_RETURN_IF_ERROR(pass.Run(opt_options));
-
   // ------------------------------------------------------------------------ //
   // Before placing the function body nodes we pin input placeholders to the
   // same device as their corresponding input nodes.
 
-  for (Node* func_body_node : func_body_graph->nodes()) {
+  for (NodeDef& func_body_node : *item.graph.mutable_node()) {
     const auto input_placeholder_idx =
-        input_placeholders_idx.find(func_body_node->name());
+        input_placeholders_idx.find(func_body_node.name());
 
     if (input_placeholder_idx != input_placeholders_idx.end()) {
       const int input_idx = input_placeholder_idx->second;
       const GraphView::OutputPort output_port =
           ctx->graph_view().GetRegularFanin({&func_node, input_idx});
 
-      VLOG(3) << "Pin inlined function input node '" << func_body_node->name()
+      VLOG(3) << "Pin inlined function input node '" << func_body_node.name()
               << "' to the '" << output_port.node->device() << "' device.";
-      func_body_node->set_requested_device(output_port.node->device());
+      func_body_node.set_device(output_port.node->device());
     }
   }
 
@@ -1523,6 +1511,8 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
   // After placing nodes corresponding to the function inputs, we need to assign
   // device placements to all other function body nodes.
 
+  GraphDef placed_graph_def;
+
   const DeviceSet* devices = ctx->devices();
 
   if (devices->devices().empty()) {
@@ -1532,8 +1522,9 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
     // of a batch job for graph analysis/optimization.
     VLOG(3) << "Assign function call node device to all function body nodes. "
             << "Device: " << func_node.device();
-    for (Node* func_body_node : func_body_graph->nodes()) {
-      func_body_node->set_requested_device(func_node.device());
+    placed_graph_def = item.mutable_function_body();
+    for (NodeDef& node : *placed_graph_def.mutable_node()) {
+      node.set_device(func_node.device());
     }
   } else {
     // If we are running in an active runtime session, Grappler will get the
@@ -1545,20 +1536,23 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
                    [](string* out, const Device* d) { out->append(d->name()); })
             << "]";
 
+    // Construct a Graph object from the instantiated function body.
+    GraphConstructorOptions opts;
+    Graph graph(ctx->function_library());
+    TF_RETURN_IF_ERROR(
+        ConvertGraphDefToGraph(opts, item.function_body(), &graph));
+
     // Use function caller node device as a default for placer.
     const Device* default_device =
         devices->FindDeviceByName(func_node.device());
 
-    Placer placer(func_body_graph.get(), devices,
-                  nullptr /* No session options */, default_device);
+    Placer placer(&graph, devices, nullptr, /* No session options */
+                  default_device);
     TF_RETURN_IF_ERROR(placer.Run());
-  }
-
-  // TODO(ezhulenev): Should we run POST_PLACEMENT runtime pass here?
 
-  // All following transformations will work with GraphDef.
-  GraphDef placed_graph_def;
-  func_body_graph->ToGraphDef(&placed_graph_def);
+    // Convert Graph back to the GraphDef.
+    graph.ToGraphDef(&placed_graph_def);
+  }
 
   // ------------------------------------------------------------------------ //
   // After all nodes placed we need to prepare them for inlining into the
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
index f4752bd584..827b0658c5 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
@@ -14,8 +14,6 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/grappler/optimizers/function_optimizer.h"
-
-#include "absl/algorithm/container.h"
 #include "tensorflow/cc/ops/functional_ops.h"
 #include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/core/framework/function_testlib.h"
@@ -1184,126 +1182,6 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithMergedDeadTensors) {
   test::ExpectTensorEqual<float>(tensors[0], tensors_expected[0]);
 }
 
-TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithFunctionalControlFlow) {
-  using test::function::NDef;
-  using FDH = FunctionDefHelper;
-
-  FunctionOptimizer optimizer(RewriterConfig::AGGRESSIVE);
-
-  FunctionDef add_func = FunctionDefHelper::Create(
-      "MyAdd", {"x:T", "y:T"}, {"z:T"}, {"T: {float, double}"},
-      {{{"add"}, "Add", {"x", "y"}, {{"T", "$T"}}}},
-      /* Mapping between function returns and function node outputs. */
-      {{"z", "add:z:0"}});
-
-  FunctionDef mul_func = FunctionDefHelper::Create(
-      "MyMul", {"x:T", "y:T"}, {"z:T"}, {"T: {float, double}"},
-      {{{"mul"}, "Mul", {"x", "y"}, {{"T", "$T"}}}},
-      /* Mapping between function returns and function node outputs. */
-      {{"z", "mul:z:0"}});
-
-  // Compute: return cond ? a + b : a * b
-  FunctionDef add_or_mul_func = FunctionDefHelper::Create(
-      "AddOrMul", {"cond:bool", "x:float", "y:float"}, {"z:float"}, {},
-      {
-          {{"if_node"},
-           "If",
-           {"cond", "x", "y"},
-           {
-               {"Tcond", DT_BOOL},
-               {"Tin", DataTypeSlice{DT_FLOAT, DT_FLOAT}},
-               {"Tout", DataTypeSlice{DT_FLOAT}},
-               {"then_branch", FDH::FunctionRef("MyAdd", {{"T", DT_FLOAT}})},
-               {"else_branch", FDH::FunctionRef("MyMul", {{"T", DT_FLOAT}})},
-               {"_lower_using_switch_merge", true},
-           }},
-      },
-      /* Mapping between function returns and function node outputs. */
-      {{"z", "if_node:output:0"}});
-
-  // Build a computation graph for:
-  //   is_add: bool
-  //   a: float
-  //   b: float
-  //   c = AddOrMul(is_add, a, b)  # is_add ? a + b : a * b
-  //   d = Identity(c)
-  //   return d
-
-  // c = MyMul(a, b)
-  GrapplerItem item;
-  item.fetch = {"d"};
-  item.graph = test::function::GDef(
-      {NDef("is_add", "Placeholder", {}, {{"dtype", DT_BOOL}}, kDevice),
-       NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
-       NDef("b", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
-
-       NDef("c", "PartitionedCall", {"is_add", "a", "b"},
-            {{"Tin", DataTypeSlice{DT_BOOL, DT_FLOAT, DT_FLOAT}},
-             {"Tout", DataTypeSlice{DT_FLOAT}},
-             {"f", FDH::FunctionRef("AddOrMul")}},
-            kDevice),
-
-       NDef("d", "Identity", {"c"}, {{"T", DT_FLOAT}}, kDevice)},
-      // Function library.
-      {add_or_mul_func, add_func, mul_func});
-
-  GraphDef optimized_graph;
-  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &optimized_graph));
-
-  const auto count_nodes_with_op = [&](const string& op) {
-    return absl::c_count_if(optimized_graph.node(), [&](const NodeDef& node) {
-      return node.op() == op;
-    });
-  };
-
-  // All `PartitionedCall` nodes in the optimized graph must be inlined, and
-  // `If` node must be lowered to `Switch` and `Merge` nodes.
-  EXPECT_EQ(count_nodes_with_op("PartitionedCallOp"), 0);
-  EXPECT_EQ(count_nodes_with_op("If"), 0);
-  EXPECT_EQ(count_nodes_with_op("Switch"), 3);
-  EXPECT_EQ(count_nodes_with_op("Merge"), 1);
-
-  GrapplerItem optimized = item.WithGraph(std::move(optimized_graph));
-
-  Tensor one = test::AsScalar<float>(1.0);
-  Tensor two = test::AsScalar<float>(2.0);
-  Tensor three = test::AsScalar<float>(3.0);
-
-  const auto feed_args = [&](bool is_add) {
-    std::vector<std::pair<string, Tensor>> feed;
-    feed.emplace_back("a", one);
-    feed.emplace_back("b", two);
-    feed.emplace_back("is_add", test::AsScalar<bool>(is_add));
-    return feed;
-  };
-
-  {  // Check 'is_add == true': a + b
-    item.feed = feed_args(true);
-    optimized.feed = feed_args(true);
-
-    auto tensors_expected = EvaluateFetchNodes(item);
-    ASSERT_EQ(tensors_expected.size(), 1);
-    test::ExpectTensorEqual<float>(tensors_expected[0], three);
-
-    auto tensors = EvaluateFetchNodes(optimized);
-    ASSERT_EQ(tensors.size(), tensors_expected.size());
-    test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
-  }
-
-  {  // Check 'is_add == false': a * b
-    item.feed = feed_args(false);
-    optimized.feed = feed_args(false);
-
-    auto tensors_expected = EvaluateFetchNodes(item);
-    ASSERT_EQ(tensors_expected.size(), 1);
-    test::ExpectTensorEqual<float>(tensors_expected[0], two);
-
-    auto tensors = EvaluateFetchNodes(optimized);
-    ASSERT_EQ(tensors.size(), tensors_expected.size());
-    test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
-  }
-}
-
 TEST_F(FunctionOptimizerTest, SpecializeFunctionXTimesTwo) {
   using test::function::NDef;
 
-- 
GitLab


From 04181795a31d2b332aa0d0fc5daf9d254432c042 Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Wed, 16 Jan 2019 10:24:17 -0800
Subject: [PATCH 0790/2345] Add num_values argument to Tensor::DebugString()

Non-CPU Tensors cause a crash when DebugString() tries to access
their values. num_values can be used not to access the values or
print more than default of 3 during debugging.

PiperOrigin-RevId: 229581029
---
 tensorflow/compiler/jit/xla_launch_util.cc |  2 +-
 tensorflow/core/framework/tensor.cc        |  9 +++++++--
 tensorflow/core/framework/tensor.h         | 12 ++++++++++--
 3 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/jit/xla_launch_util.cc b/tensorflow/compiler/jit/xla_launch_util.cc
index 4e3cb0238f..c64981053f 100644
--- a/tensorflow/compiler/jit/xla_launch_util.cc
+++ b/tensorflow/compiler/jit/xla_launch_util.cc
@@ -377,7 +377,7 @@ Status XlaComputationLaunchContext::PopulateOutputs(
     }
 
     if (VLOG_IS_ON(3)) {
-      VLOG(3) << ctx->mutable_output(i)->DebugString();
+      VLOG(3) << ctx->mutable_output(i)->DeviceSafeDebugString();
     }
   }
 
diff --git a/tensorflow/core/framework/tensor.cc b/tensorflow/core/framework/tensor.cc
index 0c96ec8168..ab492a2189 100644
--- a/tensorflow/core/framework/tensor.cc
+++ b/tensorflow/core/framework/tensor.cc
@@ -1174,10 +1174,15 @@ bool Tensor::SharesBufferWith(const Tensor& b) const {
          buf_->root_buffer() == b.buf_->root_buffer();
 }
 
-string Tensor::DebugString() const {
+string Tensor::DebugString(int num_values) const {
   return strings::StrCat("Tensor<type: ", DataTypeString(dtype()),
                          " shape: ", shape().DebugString(),
-                         " values: ", SummarizeValue(3), ">");
+                         " values: ", SummarizeValue(num_values), ">");
+}
+
+string Tensor::DeviceSafeDebugString() const {
+  return strings::StrCat("Tensor<type: ", DataTypeString(dtype()),
+                         " shape: ", shape().DebugString(), ">");
 }
 
 void Tensor::FillDescription(TensorDescription* description) const {
diff --git a/tensorflow/core/framework/tensor.h b/tensorflow/core/framework/tensor.h
index 009dd0846d..44293c5c19 100644
--- a/tensorflow/core/framework/tensor.h
+++ b/tensorflow/core/framework/tensor.h
@@ -526,7 +526,15 @@ class Tensor {
   string SummarizeValue(int64 max_entries, bool print_v2 = false) const;
 
   /// A human-readable summary of the tensor suitable for debugging.
-  string DebugString() const;
+  // `num_values` is the number of actual data values in the tensor
+  // included in the message. If the tensor might be resident in
+  // GPU/TPU memory use DeviceSafeDebugString instead.
+  string DebugString(int num_values = 3) const;
+
+  // Variant of DebugString() that should be used for possibly non-CPU tensors.
+  // If the tensor is not resident on CPU, we can't read its values as
+  // DebugString() does.
+  string DeviceSafeDebugString() const;
 
   /// Fill in the `TensorDescription` proto with metadata about the
   /// tensor that is useful for monitoring and debugging.
@@ -594,7 +602,7 @@ class Tensor {
       OpKernelContext* ctx, Var* var);  // For access to RefCountIsOne().
   friend Status batch_util::CopyElementToSlice(
       Tensor element, Tensor* parent,
-      int64 index);                // For access to RefCountIsOne().
+      int64 index);  // For access to RefCountIsOne().
   friend Status batch_util::MaybeMoveSliceToElement(
       Tensor* parent, Tensor* element,
       int64 index);  // For access to RefCountIsOne().
-- 
GitLab


From 1b2b712b836f1c7e32526f62c797a9d9a46d9561 Mon Sep 17 00:00:00 2001
From: Shivani Agrawal <shivaniagrawal@google.com>
Date: Wed, 16 Jan 2019 10:41:46 -0800
Subject: [PATCH 0791/2345] Fix in evaluate for SparseTensors.

PiperOrigin-RevId: 229584367
---
 .../bucket_by_sequence_length_test.py         |  9 ++---
 .../experimental/kernel_tests/scan_test.py    |  4 +-
 .../experimental/kernel_tests/unbatch_test.py | 37 +++++--------------
 tensorflow/python/framework/test_util.py      |  5 ++-
 4 files changed, 17 insertions(+), 38 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
index 71ae14b5e1..0bbf0e9a12 100644
--- a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
@@ -77,12 +77,11 @@ def _get_record_shape(sparse):
 class BucketBySequenceLengthTest(test_base.DatasetTestBase,
                                  parameterized.TestCase):
 
-  # TODO(b/117581999): add eager coverage.
   @parameterized.named_parameters(
       ("WithoutPadding", True),
       ("WithPadding", False),
   )
-  def testSkipEagerBucketDropReminder(self, param_no_padding):
+  def testBucketDropReminder(self, param_no_padding):
 
     boundaries = [10, 20, 30]
     batch_sizes = [10, 8, 4, 2]
@@ -202,12 +201,11 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase,
 
     _test_bucket_by_padding(param_no_padding)
 
-  # TODO(b/117581999): add eager coverage.
   @parameterized.named_parameters(
       ("WithoutPadding", True),
       ("WithPadding", False),
   )
-  def testSkipEagerBucket(self, param_no_padding):
+  def testBucket(self, param_no_padding):
 
     boundaries = [10, 20, 30]
     batch_sizes = [10, 8, 4, 2]
@@ -383,12 +381,11 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase,
 
     _test_tuple_elements_by_padding(param_no_padding)
 
-  # TODO(b/117581999): add eager coverage
   @parameterized.named_parameters(
       ("DoDropRemainder", True),
       ("DoNotDropRemainder", False),
   )
-  def testSkipEagerBucketSparse(self, param_drop_remainder):
+  def testBucketSparse(self, param_drop_remainder):  # pylint: disable=g-doc-args
     """Tests bucketing of sparse tensors (case where `no_padding` == True).
 
     Test runs on following dataset:
diff --git a/tensorflow/python/data/experimental/kernel_tests/scan_test.py b/tensorflow/python/data/experimental/kernel_tests/scan_test.py
index f5ac0f5007..38e9b1e128 100644
--- a/tensorflow/python/data/experimental/kernel_tests/scan_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/scan_test.py
@@ -70,9 +70,7 @@ class ScanTest(test_base.DatasetTestBase):
     self.assertEqual(5, self.evaluate(next_element()))
     self.assertEqual(8, self.evaluate(next_element()))
 
-  # TODO(b/117581999): Add coverage for eager.
-  @test_util.run_deprecated_v1
-  def testSkipEagerSparseCount(self):
+  def testSparseCount(self):
 
     def _sparse(i):
       return sparse_tensor.SparseTensorValue(
diff --git a/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py b/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py
index e4034cc43a..613fe0da6b 100644
--- a/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py
@@ -68,9 +68,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     self.assertDatasetProduces(
         data, [(i, compat.as_bytes(str(i)), i) for i in range(10)])
 
-  # TODO(b/117581999): Add eager coverage.
-  @test_util.run_deprecated_v1
-  def testSkipEagerUnbatchDatasetWithSparseTensor(self):
+  def testUnbatchDatasetWithSparseTensor(self):
     st = sparse_tensor.SparseTensorValue(
         indices=[[i, i] for i in range(10)],
         values=list(range(10)),
@@ -79,20 +77,12 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     data = data.apply(batching.unbatch())
     data = data.batch(5)
     data = data.apply(batching.unbatch())
-    iterator = dataset_ops.make_one_shot_iterator(data)
-    next_element = iterator.get_next()
-
-    for i in range(10):
-      st_row = self.evaluate(next_element)
-      self.assertEqual([i], st_row.indices)
-      self.assertEqual([i], st_row.values)
-      self.assertEqual([10], st_row.dense_shape)
-    with self.assertRaises(errors.OutOfRangeError):
-      self.evaluate(next_element)
+    expected_output = [
+        sparse_tensor.SparseTensorValue([[i]], [i], [10]) for i in range(10)
+    ]
+    self.assertDatasetProduces(data, expected_output=expected_output)
 
-  # TODO(b/117581999): Add eager coverage.
-  @test_util.run_deprecated_v1
-  def testSkipEagerUnbatchDatasetWithDenseAndSparseTensor(self):
+  def testUnbatchDatasetWithDenseAndSparseTensor(self):
     st = sparse_tensor.SparseTensorValue(
         indices=[[i, i] for i in range(10)],
         values=list(range(10)),
@@ -101,16 +91,9 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     data = data.apply(batching.unbatch())
     data = data.batch(5)
     data = data.apply(batching.unbatch())
-    next_element = self.getNext(data)
-
-    for i in range(10):
-      dense_elem, st_row = self.evaluate(next_element())
-      self.assertEqual(i, dense_elem)
-      self.assertEqual([i], st_row.indices)
-      self.assertEqual([i], st_row.values)
-      self.assertEqual([10], st_row.dense_shape)
-    with self.assertRaises(errors.OutOfRangeError):
-      self.evaluate(next_element())
+    expected_output = [(i, sparse_tensor.SparseTensorValue([[i]], [i], [10]))
+                       for i in range(10)]
+    self.assertDatasetProduces(data, expected_output=expected_output)
 
   def testUnbatchSingleElementTupleDataset(self):
     data = tuple([(math_ops.range(10),) for _ in range(3)])
@@ -150,7 +133,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.assertRaises(ValueError):
       data.apply(batching.unbatch())
 
-  # TODO(b/117581999): eager mode doesnt capture raised error, debug.
+  # Note: dynamic shape mismatch is graph specific test.
   @test_util.run_deprecated_v1
   def testSkipEagerUnbatchDynamicShapeMismatch(self):
     ph1 = array_ops.placeholder(dtypes.int32, shape=[None])
diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index e2b906e37c..732fd31389 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -1658,8 +1658,9 @@ class TensorFlowTestCase(googletest.TestCase):
     else:
       try:
         if sparse_tensor.is_sparse(tensor):
-          return sparse_tensor.SparseTensorValue(tensor.indices, tensor.values,
-                                                 tensor.dense_shape)
+          return sparse_tensor.SparseTensorValue(tensor.indices.numpy(),
+                                                 tensor.values.numpy(),
+                                                 tensor.dense_shape.numpy())
         return tensor.numpy()
       except AttributeError as e:
         six.raise_from(ValueError("Unsupported type %s." % type(tensor)), e)
-- 
GitLab


From b59aa4b89356333027bb38f7364f18c4df882835 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 10:44:30 -0800
Subject: [PATCH 0792/2345] Update landscape layout with model, device, and
 number of thread selectors.

PiperOrigin-RevId: 229584823
---
 .../layout-land/fragment_camera2_basic.xml    | 128 +++++++++++-------
 1 file changed, 82 insertions(+), 46 deletions(-)

diff --git a/tensorflow/lite/java/demo/app/src/main/res/layout-land/fragment_camera2_basic.xml b/tensorflow/lite/java/demo/app/src/main/res/layout-land/fragment_camera2_basic.xml
index ee71ab808f..323b21dbce 100644
--- a/tensorflow/lite/java/demo/app/src/main/res/layout-land/fragment_camera2_basic.xml
+++ b/tensorflow/lite/java/demo/app/src/main/res/layout-land/fragment_camera2_basic.xml
@@ -16,67 +16,103 @@
 
 <LinearLayout
     xmlns:android="http://schemas.android.com/apk/res/android"
-    android:layout_width="match_parent"
+    xmlns:tools="http://schemas.android.com/tools" android:layout_width="match_parent"
     android:layout_height="match_parent"
     android:background="#bb7700"
     android:orientation="horizontal">
 
-  <com.example.android.tflitecamerademo.AutoFitTextureView
+    <com.example.android.tflitecamerademo.AutoFitTextureView
       android:id="@+id/texture"
       android:layout_width="0dp"
       android:layout_height="match_parent"
       android:layout_weight=".8"/>
 
-  <LinearLayout
+    <LinearLayout
       android:layout_width="0dp"
       android:layout_height="match_parent"
       android:layout_weight=".2"
       android:orientation="vertical">
 
-    <ImageView
-        android:id="@+id/logoview"
-        android:layout_width="wrap_content"
-        android:layout_height="wrap_content"
-        android:scaleType="centerInside"
-        android:src="@drawable/logo"/>
-
-    <RadioGroup
-        android:gravity="center"
-        android:layout_width="match_parent"
-        android:layout_height="match_parent"
-        android:orientation="horizontal">
-        <RadioButton
-            android:id="@+id/radio_cpu"
-            android:background="#0000000f"
-            android:layout_width="wrap_content"
-            android:layout_height="wrap_content"
-            android:text="@string/cpu"
-            android:textColor="@android:color/white" />
-        <RadioButton
-            android:id="@+id/radio_nnapi"
-            android:background="#0000000f"
+        <ImageView
+            android:id="@+id/logoview"
             android:layout_width="wrap_content"
+            android:layout_height="47dp"
+            android:scaleType="centerInside"
+            android:src="@drawable/logo"/>
+
+        <TextView
+            android:id="@+id/text"
+            android:layout_width="match_parent"
+            android:layout_height="160dp"
+            android:paddingTop="20dp"
+            android:textColor="#FFF"
+            android:textSize="20sp"
+            android:textStyle="bold"/>
+        <LinearLayout
+            android:id="@+id/modelLayout"
+            android:layout_width="match_parent"
+            android:layout_height="150dp"
+            android:orientation="vertical">
+
+            <TextView
+                android:id="@+id/textView"
+                android:layout_width="match_parent"
+                android:layout_height="20dp"
+                android:text="@string/modelLabel"
+                android:textAlignment="center"
+                android:textColor="@android:color/white"/>
+
+            <ListView
+                android:id="@+id/model"
+                android:layout_width="match_parent"
+                android:layout_height="wrap_content">
+
+            </ListView>
+        </LinearLayout>
+        <LinearLayout
+            android:id="@+id/deviceLayout"
+            android:layout_width="match_parent"
+            android:layout_height="150dp"
+            android:orientation="vertical">
+
+            <TextView
+                android:id="@+id/textView2"
+                android:layout_width="match_parent"
+                android:layout_height="20dp"
+                android:text="@string/deviceLabel"
+                android:textAlignment="center"
+                android:textColor="@android:color/white"/>
+
+            <ListView
+                android:id="@+id/device"
+                android:layout_width="match_parent"
+                android:layout_height="wrap_content"/>
+
+        </LinearLayout>
+
+        <LinearLayout
+            android:layout_width="match_parent"
             android:layout_height="wrap_content"
-            android:text="@string/nnapi"
-            android:textColor="@android:color/white" />
-        </RadioGroup>
-
-    <NumberPicker
-        android:id="@+id/np"
-        android:layout_width="wrap_content"
-        android:layout_height="47dp"
-        android:layout_gravity="center_horizontal"
-        android:visibility="visible"/>
-
-    <TextView
-        android:id="@+id/text"
-        android:textStyle="bold"
-        android:layout_width="match_parent"
-        android:layout_height="match_parent"
-        android:paddingTop="20dp"
-        android:textColor="#FFF"
-        android:textSize="20sp"/>
-
-  </LinearLayout>
+        >
+
+            <TextView
+                android:layout_width="wrap_content"
+                android:layout_height="wrap_content"
+                android:gravity="center"
+                android:text="Threads"
+                android:textAlignment="center"
+                android:textColor="@android:color/white"/>
+
+            <NumberPicker
+                android:id="@+id/np"
+                android:layout_width="match_parent"
+                android:layout_height="wrap_content"
+                android:layout_marginLeft="10dp"
+                android:theme="@style/AppTheme.Picker"
+                android:visibility="visible"/>
+
+        </LinearLayout>
+
+    </LinearLayout>
 </LinearLayout>
 
-- 
GitLab


From b4ae5ba323c20ad30e3f7559d547a6b76d5f33e6 Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Wed, 16 Jan 2019 10:51:07 -0800
Subject: [PATCH 0793/2345] [tf.data] Add a ChooseFastestDataset, which will
 allow our optimizations to dynamically pick between two equivalent input
 datasets

- For the first N iterations, run two threads simultaneously that each pulls from one of the inputs. The threads also measure time taken. Wait for both threads to complete before returning.
- After N iterations, decide which dataset is faster and just use that.

There are more sophisticated implementations that could be better, e.g. instead of waiting for both threads to complete, return when the faster of them is finished. However, the code here is significantly simpler (I also implemented the other one elsewhere), at the cost of slower iterations for the first N iterations -- the iteration time for the first N iters is max(input_a, input_b). Open to ideas.

PiperOrigin-RevId: 229586112
---
 ...def_ExperimentalChooseFastestDataset.pbtxt |   4 +
 .../core/kernels/data/experimental/BUILD      |  13 +
 .../experimental/choose_fastest_dataset_op.cc | 346 ++++++++++++++++++
 .../core/ops/experimental_dataset_ops.cc      |   7 +
 .../python/data/experimental/benchmarks/BUILD |  14 +
 .../benchmarks/choose_fastest_benchmark.py    | 105 ++++++
 .../kernel_tests/optimization/BUILD           |  20 +
 .../choose_fastest_dataset_test.py            |  85 +++++
 .../kernel_tests/serialization/BUILD          |  18 +
 ...oose_fastest_dataset_serialization_test.py |  45 +++
 .../data/experimental/ops/optimization.py     |  45 +++
 11 files changed, 702 insertions(+)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_ExperimentalChooseFastestDataset.pbtxt
 create mode 100644 tensorflow/core/kernels/data/experimental/choose_fastest_dataset_op.cc
 create mode 100644 tensorflow/python/data/experimental/benchmarks/choose_fastest_benchmark.py
 create mode 100644 tensorflow/python/data/experimental/kernel_tests/optimization/choose_fastest_dataset_test.py
 create mode 100644 tensorflow/python/data/experimental/kernel_tests/serialization/choose_fastest_dataset_serialization_test.py

diff --git a/tensorflow/core/api_def/base_api/api_def_ExperimentalChooseFastestDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalChooseFastestDataset.pbtxt
new file mode 100644
index 0000000000..7aa7a59bb6
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_ExperimentalChooseFastestDataset.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "ExperimentalChooseFastestDataset"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/kernels/data/experimental/BUILD b/tensorflow/core/kernels/data/experimental/BUILD
index 583fce6d89..29756f0cfd 100644
--- a/tensorflow/core/kernels/data/experimental/BUILD
+++ b/tensorflow/core/kernels/data/experimental/BUILD
@@ -143,6 +143,18 @@ tf_kernel_library(
     ],
 )
 
+tf_kernel_library(
+    name = "choose_fastest_dataset_op",
+    srcs = ["choose_fastest_dataset_op.cc"],
+    deps = [
+        "//tensorflow/core:experimental_dataset_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core/kernels/data:dataset",
+    ],
+)
+
 tf_kernel_library(
     name = "non_serializable_dataset_op",
     srcs = ["non_serializable_dataset_op.cc"],
@@ -343,6 +355,7 @@ tf_kernel_library(
     name = "dataset_kernels",
     deps = [
         ":assert_next_dataset_op",
+        ":choose_fastest_dataset_op",
         ":csv_dataset_op",
         ":dense_to_sparse_batch_dataset_op",
         ":directed_interleave_dataset_op",
diff --git a/tensorflow/core/kernels/data/experimental/choose_fastest_dataset_op.cc b/tensorflow/core/kernels/data/experimental/choose_fastest_dataset_op.cc
new file mode 100644
index 0000000000..f66d5d9955
--- /dev/null
+++ b/tensorflow/core/kernels/data/experimental/choose_fastest_dataset_op.cc
@@ -0,0 +1,346 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/framework/common_shape_fns.h"
+#include "tensorflow/core/framework/dataset.h"
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/histogram/histogram.h"
+
+namespace tensorflow {
+namespace data {
+namespace {
+
+static const double kPercentile = 90.0;
+
+class ChooseFastestDatasetOp : public DatasetOpKernel {
+ public:
+  explicit ChooseFastestDatasetOp(OpKernelConstruction* ctx)
+      : DatasetOpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("num_experiments", &num_experiments_));
+  }
+
+  void MakeDataset(OpKernelContext* ctx, DatasetBase** output) override {
+    OpInputList input_list;
+    OP_REQUIRES_OK(ctx, ctx->input_list("input_datasets", &input_list));
+    OP_REQUIRES(
+        ctx, input_list.size() > 1,
+        errors::InvalidArgument(
+            "ChooseFastestDataset must have at least two input datasets."));
+
+    std::vector<DatasetBase*> inputs;
+    inputs.reserve(input_list.size());
+    for (const auto& tensor : input_list) {
+      DatasetBase* input;
+      OP_REQUIRES_OK(ctx, GetDatasetFromVariantTensor(tensor, &input));
+      inputs.push_back(input);
+    }
+
+    const DataTypeVector& output_types = inputs[0]->output_dtypes();
+    for (size_t i = 1, num_inputs = inputs.size(); i < num_inputs; ++i) {
+      OP_REQUIRES(ctx, inputs[i]->output_dtypes() == output_types,
+                  errors::InvalidArgument(
+                      "All inputs to ChooseFastestDataset "
+                      "must have the same output types. Input ",
+                      i, " has output types: ",
+                      DataTypeVectorString(inputs[i]->output_dtypes()),
+                      ", while all prior inputs have types: ",
+                      DataTypeVectorString(output_types), "."));
+    }
+
+    std::vector<PartialTensorShape> output_shapes = inputs[0]->output_shapes();
+    // Merge the output shapes of all the input datasets, returning an
+    // error if any of them are incompatible.
+    for (size_t i = 1, num_inputs = inputs.size(); i < num_inputs; ++i) {
+      OP_REQUIRES(
+          ctx, inputs[i]->output_shapes().size() == output_shapes.size(),
+          errors::InvalidArgument(
+              "All inputs to ChooseFastestDataset must have compatible outputs."
+              " Input ",
+              i, " has ", inputs[i]->output_shapes().size(),
+              " components, while all prior inputs have ", output_shapes.size(),
+              " components."));
+      for (size_t j = 0, num_components = output_shapes.size();
+           j < num_components; ++j) {
+        PartialTensorShape result;
+        OP_REQUIRES(
+            ctx,
+            output_shapes[j]
+                .MergeWith(inputs[i]->output_shapes().at(j), &result)
+                .ok(),
+            errors::InvalidArgument(
+                "All inputs to ChooseFastestDataset must have "
+                "compatible output shapes. Component ",
+                j, " of input ", i, " has shape: ",
+                inputs[i]->output_shapes().at(j), ", while components ", j,
+                " of all prior inputs have shape: ", output_shapes[j], "."));
+        output_shapes[j] = std::move(result);
+      }
+    }
+
+    int64 cardinality = inputs[0]->Cardinality();
+    for (size_t i = 1, num_inputs = inputs.size(); i < num_inputs; ++i) {
+      if (cardinality == kUnknownCardinality) {
+        cardinality = inputs[i]->Cardinality();
+      } else {
+        OP_REQUIRES(
+            ctx,
+            inputs[i]->Cardinality() == cardinality ||
+                inputs[i]->Cardinality() == kUnknownCardinality,
+            errors::InvalidArgument(
+                "All inputs to ChooseFastestDataset must have compatible "
+                "cardinalities. Input ",
+                i, " has cardinality: ", inputs[i]->Cardinality(),
+                ", while all prior inputs have cardinality: ", cardinality,
+                "."));
+      }
+    }
+    *output = new Dataset(ctx, std::move(inputs), std::move(output_shapes),
+                          cardinality, num_experiments_);
+  }
+
+ private:
+  class Dataset : public DatasetBase {
+   public:
+    Dataset(OpKernelContext* ctx, std::vector<DatasetBase*> inputs,
+            std::vector<PartialTensorShape> output_shapes, int64 cardinality,
+            int64 num_experiments)
+        : DatasetBase(DatasetContext(ctx)),
+          inputs_(std::move(inputs)),
+          output_shapes_(std::move(output_shapes)),
+          cardinality_(cardinality),
+          num_experiments_(num_experiments) {
+      for (auto input : inputs_) {
+        input->Ref();
+      }
+    }
+
+    ~Dataset() override {
+      for (auto input : inputs_) {
+        input->Unref();
+      }
+    }
+
+    std::unique_ptr<IteratorBase> MakeIteratorInternal(
+        const string& prefix) const override {
+      return absl::make_unique<ChooseFastestIterator>(
+          ChooseFastestIterator::Params{
+              this, strings::StrCat(prefix, "::ChooseFastest")});
+    }
+
+    const DataTypeVector& output_dtypes() const override {
+      return inputs_[0]->output_dtypes();
+    }
+
+    const std::vector<PartialTensorShape>& output_shapes() const override {
+      return output_shapes_;
+    }
+
+    string DebugString() const override {
+      return "ChooseFastestDatasetOp::Dataset";
+    }
+
+    int64 Cardinality() const override { return cardinality_; }
+
+   protected:
+    Status AsGraphDefInternal(SerializationContext* ctx,
+                              DatasetGraphDefBuilder* b,
+                              Node** output) const override {
+      std::vector<Node*> input_nodes;
+      input_nodes.reserve(inputs_.size());
+      for (const auto& input : inputs_) {
+        Node* input_node;
+        TF_RETURN_IF_ERROR(b->AddInputDataset(ctx, input, &input_node));
+        input_nodes.push_back(input_node);
+      }
+      AttrValue num_experiments_attr;
+      b->BuildAttrValue(num_experiments_, &num_experiments_attr);
+      return b->AddDataset(
+          this, {}, {std::make_pair(0, input_nodes)},
+          {std::make_pair("num_experiments", std::move(num_experiments_attr))},
+          output);
+    }
+
+   private:
+    class ChooseFastestIterator : public DatasetIterator<Dataset> {
+     public:
+      explicit ChooseFastestIterator(const Params& params)
+          : DatasetIterator<Dataset>(params),
+            histograms_(dataset()->inputs_.size()) {}
+
+      Status Initialize(IteratorContext* ctx) override {
+        mutex_lock l(mu_);
+        input_impls_.resize(dataset()->inputs_.size());
+        for (size_t i = 0, num_inputs = dataset()->inputs_.size();
+             i < num_inputs; ++i) {
+          TF_RETURN_IF_ERROR(dataset()->inputs_[i]->MakeIterator(
+              ctx, strings::StrCat(prefix(), "_", i), &input_impls_[i]));
+        }
+        return Status::OK();
+      }
+
+      Status GetNextInternal(IteratorContext* ctx,
+                             std::vector<Tensor>* out_tensors,
+                             bool* end_of_sequence) override {
+        mutex_lock l(mu_);
+
+        // The first num_experiments_ iterations, we fire up a thread for
+        // each input that calls its GetNext function and records the time
+        // taken. We only return when all the threads have completed.
+        if (experiment_counter_ < dataset()->num_experiments_) {
+          experiment_counter_++;
+          std::vector<ThreadInfo> threads = StartThreads(ctx);
+          for (const auto& thread : threads) {
+            thread.result->notification.WaitForNotification();
+          }
+
+          *out_tensors = std::move(threads[0].result->out_tensors);
+          *end_of_sequence = threads[0].result->end_of_sequence;
+
+          if (experiment_counter_ == dataset()->num_experiments_) {
+            SelectFastestInputIndex();
+          }
+          return threads[0].result->status;
+        }
+        return input_impls_[fastest_index_]->GetNext(ctx, out_tensors,
+                                                     end_of_sequence);
+      }
+
+     protected:
+      std::shared_ptr<model::Node> CreateNode(
+          IteratorContext* ctx, model::Node::Args args) const override {
+        return model::MakeKnownRatioNode(std::move(args), /*ratio=*/1);
+      }
+
+      // TODO(rachelim): Save and restore histogram state as well. Currently,
+      // if an iterator is saved and restored, the histograms start recording
+      // from scratch.
+      Status SaveInternal(IteratorStateWriter* writer) override {
+        mutex_lock l(mu_);
+        if (input_impls_.empty()) {
+          TF_RETURN_IF_ERROR(
+              writer->WriteScalar(full_name("input_impls_empty"), ""));
+        } else {
+          for (auto& input_impl : input_impls_) {
+            TF_RETURN_IF_ERROR(SaveInput(writer, input_impl));
+          }
+        }
+        TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("experiment_counter"),
+                                               experiment_counter_));
+        TF_RETURN_IF_ERROR(
+            writer->WriteScalar(full_name("fastest_index"), fastest_index_));
+        return Status::OK();
+      }
+
+      Status RestoreInternal(IteratorContext* ctx,
+                             IteratorStateReader* reader) override {
+        mutex_lock l(mu_);
+        if (reader->Contains(full_name("input_impls_empty"))) {
+          input_impls_.clear();
+        } else {
+          DCHECK_EQ(input_impls_.size(), dataset()->inputs_.size());
+          for (auto& input_impl : input_impls_) {
+            TF_RETURN_IF_ERROR(RestoreInput(ctx, reader, input_impl));
+          }
+        }
+        TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("experiment_counter"),
+                                              &experiment_counter_));
+        TF_RETURN_IF_ERROR(
+            reader->ReadScalar(full_name("fastest_index"), &fastest_index_));
+        return Status::OK();
+      }
+
+     private:
+      struct InvocationResult {
+        Notification notification;
+        Status status;
+        bool end_of_sequence;
+        std::vector<Tensor> out_tensors;
+      };
+
+      struct ThreadInfo {
+        std::unique_ptr<InvocationResult> result;
+        std::unique_ptr<Thread> thread;
+      };
+
+      std::vector<std::unique_ptr<IteratorBase>> input_impls_;
+      // For tracking the time taken for each input's iterations.
+      std::vector<histogram::Histogram> histograms_;
+
+      mutex mu_;
+      int64 experiment_counter_ GUARDED_BY(mu_) = 0;
+      int64 fastest_index_ = -1;
+
+      std::vector<ThreadInfo> StartThreads(IteratorContext* ctx)
+          EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+        std::vector<ThreadInfo> threads(dataset()->inputs_.size());
+        for (size_t i = 0, num_inputs = dataset()->inputs_.size();
+             i < num_inputs; ++i) {
+          threads[i].result = absl::make_unique<InvocationResult>();
+          threads[i].thread.reset(ctx->env()->StartThread(
+              {}, strings::StrCat("tf_data_merge_", i),
+              std::bind(&ChooseFastestIterator::RunnerThread, this, ctx,
+                        threads[i].result.get(), i)));
+        }
+        return threads;
+      }
+
+      void RunnerThread(IteratorContext* ctx, InvocationResult* result, int i) {
+        int64 start = Env::Default()->NowNanos();
+        Status s = input_impls_[i]->GetNext(ctx, &result->out_tensors,
+                                            &result->end_of_sequence);
+        histograms_[i].Add(
+            static_cast<double>(Env::Default()->NowNanos() - start));
+
+        result->status = s;
+        result->notification.Notify();
+      }
+
+      // Select the fastest input to use based on the histograms of timings
+      // of the completed threads. The input with the best 90th percentile
+      // iteration time is selected.
+      void SelectFastestInputIndex() EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+        fastest_index_ = 0;
+
+        double best_percentile = histograms_[0].Percentile(kPercentile);
+        for (size_t i = 1, num_inputs = histograms_.size(); i < num_inputs;
+             ++i) {
+          double percentile = histograms_[i].Percentile(kPercentile);
+          if (percentile <= best_percentile) {
+            best_percentile = percentile;
+            fastest_index_ = i;
+          }
+        }
+      }
+    };  // class Iterator
+
+    const std::vector<DatasetBase*> inputs_;
+    const std::vector<PartialTensorShape> output_shapes_;
+    const int64 cardinality_;
+    const int64 num_experiments_;
+  };  // class Dataset
+
+  int64 num_experiments_;
+};  // class ChooseFastestDatasetOp
+
+// Register the kernel implementation for ChooseFastestDataset.
+REGISTER_KERNEL_BUILDER(
+    Name("ExperimentalChooseFastestDataset").Device(DEVICE_CPU),
+    ChooseFastestDatasetOp);
+
+}  // namespace
+}  // namespace data
+}  // namespace tensorflow
diff --git a/tensorflow/core/ops/experimental_dataset_ops.cc b/tensorflow/core/ops/experimental_dataset_ops.cc
index f904e2536d..d70b1650ff 100644
--- a/tensorflow/core/ops/experimental_dataset_ops.cc
+++ b/tensorflow/core/ops/experimental_dataset_ops.cc
@@ -453,6 +453,13 @@ REGISTER_OP("ExperimentalLMDBDataset")
                       // stateful to inhibit constant folding.
     .SetShapeFn(shape_inference::ScalarShape);
 
+REGISTER_OP("ExperimentalChooseFastestDataset")
+    .Input("input_datasets: N * variant")
+    .Output("handle: variant")
+    .Attr("N: int >= 2")
+    .Attr("num_experiments: int")
+    .SetShapeFn(shape_inference::ScalarShape);
+
 REGISTER_OP("ExperimentalIdentityIndexedDataset")
     .Input("size: uint64")
     .Output("handle: variant")
diff --git a/tensorflow/python/data/experimental/benchmarks/BUILD b/tensorflow/python/data/experimental/benchmarks/BUILD
index 2443f7edad..4f2117ec9b 100644
--- a/tensorflow/python/data/experimental/benchmarks/BUILD
+++ b/tensorflow/python/data/experimental/benchmarks/BUILD
@@ -110,6 +110,20 @@ py_test(
     ],
 )
 
+py_test(
+    name = "choose_fastest_benchmark",
+    srcs = ["choose_fastest_benchmark.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:session",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//third_party/py/numpy",
+    ],
+)
+
 py_test(
     name = "optimize_benchmark",
     srcs = ["optimize_benchmark.py"],
diff --git a/tensorflow/python/data/experimental/benchmarks/choose_fastest_benchmark.py b/tensorflow/python/data/experimental/benchmarks/choose_fastest_benchmark.py
new file mode 100644
index 0000000000..4a5a264c6f
--- /dev/null
+++ b/tensorflow/python/data/experimental/benchmarks/choose_fastest_benchmark.py
@@ -0,0 +1,105 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Benchmarks for static optimizations."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import time
+
+import numpy as np
+
+from tensorflow.python.client import session
+from tensorflow.python.data.experimental.ops import optimization
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.platform import test
+
+
+# TODO(b/119837791): Add eager benchmarks too.
+class ChooseFastestBenchmark(test.Benchmark):
+  """Benchmarks for static optimizations."""
+
+  def benchmarkChooseFastest(self):
+
+    dataset = dataset_ops.Dataset.range(1000**2).repeat()
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
+    map_batch_dataset = dataset.map(lambda x: x + 1).batch(100)
+    batch_map_dataset = dataset.batch(100).map(lambda x: x + 1)
+
+    merge_dataset = optimization._ChooseFastestDataset(  # pylint: disable=protected-access
+        [batch_map_dataset, map_batch_dataset])
+    self._benchmark(map_batch_dataset, "map_batch_dataset")
+    self._benchmark(batch_map_dataset, "batch_map_dataset")
+    self._benchmark(merge_dataset, "merge_dataset")
+
+  def benchmarkChooseFastestFirstNIterations(self):
+
+    dataset = dataset_ops.Dataset.range(1000**2).repeat()
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
+    map_batch_dataset = dataset.map(lambda x: x + 1).batch(100)
+    batch_map_dataset = dataset.batch(100).map(lambda x: x + 1)
+
+    merge_dataset = optimization._ChooseFastestDataset(  # pylint: disable=protected-access
+        [batch_map_dataset, map_batch_dataset])
+
+    self._benchmarkFirstN(map_batch_dataset, "map_batch_dataset")
+    self._benchmarkFirstN(batch_map_dataset, "batch_map_dataset")
+    self._benchmarkFirstN(merge_dataset, "merge_dataset")
+
+  def _benchmarkFirstN(self, dataset, name):
+    n = 10  # The default num_experiments for ChooseFastestDataset
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
+    next_element = iterator.get_next()
+
+    deltas = []
+    for _ in range(100):
+      with session.Session() as sess:
+        start = time.time()
+        for _ in range(n):
+          sess.run(next_element.op)
+        end = time.time()
+        deltas.append(end - start)
+    median_wall_time = np.median(deltas) / n
+    self.report_benchmark(
+        iters=n, wall_time=median_wall_time, name=name + "_first_%d" % n)
+
+  def _benchmark(self, dataset, name):
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
+    next_element = iterator.get_next()
+
+    with session.Session() as sess:
+      # Run 10 steps to warm up the session caches before taking the first
+      # measurement. Additionally, 10 is the default num_experiments for
+      # ChooseFastestDataset.
+      for _ in range(10):
+        sess.run(next_element.op)
+      deltas = []
+      for _ in range(50):
+        start = time.time()
+        for _ in range(50):
+          sess.run(next_element.op)
+        end = time.time()
+        deltas.append(end - start)
+
+      median_wall_time = np.median(deltas) / 100
+      self.report_benchmark(iters=100, wall_time=median_wall_time, name=name)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD b/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
index 93a9aca1f2..703d3350db 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
@@ -242,6 +242,26 @@ py_test(
     ],
 )
 
+py_test(
+    name = "choose_fastest_dataset_test",
+    size = "small",
+    srcs = ["choose_fastest_dataset_test.py"],
+    srcs_version = "PY2AND3",
+    tags = [
+        "no_oss",
+        "no_pip",
+        "no_windows",
+    ],
+    deps = [
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:errors",
+        "//tensorflow/python/data/experimental/ops:optimization",
+        "//tensorflow/python/data/kernel_tests:test_base",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "@absl_py//absl/testing:parameterized",
+    ],
+)
+
 py_test(
     name = "model_dataset_test",
     size = "medium",
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/choose_fastest_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/choose_fastest_dataset_test.py
new file mode 100644
index 0000000000..ec7a85ae11
--- /dev/null
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/choose_fastest_dataset_test.py
@@ -0,0 +1,85 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for `tf.data.experimental._ChooseFastestDataset`."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+
+from tensorflow.python.data.experimental.ops import optimization
+from tensorflow.python.data.kernel_tests import test_base
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.eager import context
+from tensorflow.python.framework import errors
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import test
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class ChooseFastestDatasetTest(test_base.DatasetTestBase,
+                               parameterized.TestCase):
+
+  def testChooseFastestSimple(self):
+    dataset = dataset_ops.Dataset.from_tensor_slices([0, 1, 2, 3, 4])
+    merge = optimization._ChooseFastestDataset([dataset, dataset])
+    self.assertDatasetProduces(
+        merge,
+        expected_output=[0, 1, 2, 3, 4],
+        expected_shapes=dataset.output_shapes)
+
+  def testChooseFastestManyInputs(self):
+    dataset = dataset_ops.Dataset.from_tensor_slices([0, 1, 2, 3, 4])
+    merge = optimization._ChooseFastestDataset([dataset for _ in range(5)])
+    self.assertDatasetProduces(
+        merge,
+        expected_output=[0, 1, 2, 3, 4],
+        expected_shapes=dataset.output_shapes)
+
+  def testChooseFastest(self):
+    dataset = dataset_ops.Dataset.range(600)
+    f = lambda x: 2 * x
+    dataset_a = dataset.batch(50).map(f)
+    dataset_b = dataset.map(f).batch(50)
+    merge = optimization._ChooseFastestDataset([dataset_a, dataset_b])
+    self.assertDatasetProduces(
+        merge,
+        expected_output=[
+            [i * 2 for i in range(j * 50, (j + 1) * 50)] for j in range(12)
+        ],
+        expected_shapes=dataset_a.output_shapes)
+
+  @parameterized.named_parameters(
+      ("Shapes", [0], [[1, 2, 3]], "must have compatible output shapes."),
+      ("Types", [0], [0.0], "must have the same output types."),
+      ("NumComponents", [0], ([0], [1]), "must have the same output types."),
+      ("Cardinality", [1, 2, 3], [1], "must have compatible cardinalities."))
+  def testChooseFastestErrorWithIncompatibleInput(self, slices_a, slices_b,
+                                                  error_msg):
+    dataset_a = dataset_ops.Dataset.from_tensor_slices(slices_a)
+    dataset_b = dataset_ops.Dataset.from_tensor_slices(slices_b)
+
+    # The error is raised at dataset creation time.
+    if context.executing_eagerly():
+      with self.assertRaises(errors.InvalidArgumentError):
+        merge = optimization._ChooseFastestDataset([dataset_a, dataset_b])
+    else:
+      merge = optimization._ChooseFastestDataset([dataset_a, dataset_b])
+      self.assertDatasetProduces(
+          merge, expected_error=(errors.InvalidArgumentError, error_msg))
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD b/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
index 4a2e28f496..3a2ce17228 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
@@ -93,6 +93,24 @@ py_test(
     ],
 )
 
+py_test(
+    name = "choose_fastest_dataset_serialization_test",
+    size = "small",
+    srcs = ["choose_fastest_dataset_serialization_test.py"],
+    srcs_version = "PY2AND3",
+    tags = [
+        "no_oss",
+        "no_pip",
+        "no_windows",
+    ],
+    deps = [
+        ":dataset_serialization_test_base",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python/data/experimental/ops:optimization",
+        "//tensorflow/python/data/ops:dataset_ops",
+    ],
+)
+
 py_test(
     name = "concatenate_dataset_serialization_test",
     size = "small",
diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/choose_fastest_dataset_serialization_test.py b/tensorflow/python/data/experimental/kernel_tests/serialization/choose_fastest_dataset_serialization_test.py
new file mode 100644
index 0000000000..936dc22214
--- /dev/null
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/choose_fastest_dataset_serialization_test.py
@@ -0,0 +1,45 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for the ZipDataset serialization."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.data.experimental.kernel_tests.serialization import dataset_serialization_test_base
+from tensorflow.python.data.experimental.ops import optimization
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.platform import test
+
+
+class ChooseFastestDatasetSerializationTest(
+    dataset_serialization_test_base.DatasetSerializationTestBase):
+
+  def testCore(self):
+    num_outputs = 10
+    batch_size = 2
+
+    def build_ds():
+      dataset = dataset_ops.Dataset.range(num_outputs)
+      map_fn = lambda x: x * 2
+      return optimization._ChooseFastestDataset([  # pylint: disable=protected-access
+          dataset.map(map_fn).batch(batch_size),
+          dataset.batch(batch_size).map(map_fn)
+      ])
+
+    self.run_core_tests(build_ds, None, num_outputs // 2)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/experimental/ops/optimization.py b/tensorflow/python/data/experimental/ops/optimization.py
index 22a36646ea..75769b8998 100644
--- a/tensorflow/python/data/experimental/ops/optimization.py
+++ b/tensorflow/python/data/experimental/ops/optimization.py
@@ -129,3 +129,48 @@ class _NonSerializableDataset(dataset_ops.UnaryUnchangedStructureDataset):
             self._input_dataset._variant_tensor,  # pylint: disable=protected-access
             **dataset_ops.flat_structure(self)))
     super(_NonSerializableDataset, self).__init__(input_dataset, variant_tensor)
+
+
+class _ChooseFastestDataset(dataset_ops.DatasetV2):
+  """A `Dataset` that merges two input datasets."""
+
+  def __init__(self, datasets, num_experiments=10):
+    """Chooses the fastest of some input datasets.
+
+    Given input datasets, produces elements as quickly as the fastest of the
+    inputs. Note that this dataset assumes that input datasets have the same
+    elements in the same order, though this is not enforced besides checking
+    that the input datasets have compatible output types, output shapes, and
+    cardinality at runtime. The resulting dataset produces elements that are
+    identical to the input elements, and in the same order.
+
+    Note that the time to first iteration is longer when this dataset is used
+    due to the overhead of dynamically picking the faster dataset. Namely,
+    for the first num_experiments iterations, this dataset will pull from all
+    of its inputs simultaneously in order to determine which input is the
+    fastest. For all subsequent iterations, that input will be used.
+
+    Args:
+      datasets: A list of `Datasets` that all have the same elements in the same
+        order.
+      num_experiments: The number of experiments to run before deciding which
+        dataset is fastest. In each "experiment" iteration, the dataset will
+        call from all its inputs simultaneously, and update its knowledge of
+        which input is the fastest.
+
+    Returns:
+      A `Dataset` that has the same elements the inputs.
+    """
+    self._datasets = list(datasets)
+    variant_tensor = (
+        gen_experimental_dataset_ops.experimental_choose_fastest_dataset(
+            [dataset._variant_tensor for dataset in self._datasets],  # pylint: disable=protected-access
+            num_experiments=num_experiments))
+    super(_ChooseFastestDataset, self).__init__(variant_tensor)
+
+  def _inputs(self):
+    return self._datasets
+
+  @property
+  def _element_structure(self):
+    return self._datasets[0]._element_structure  # pylint: disable=protected-access
-- 
GitLab


From 191f1d43e4c064c6460a6d4329c8e030f6b94114 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sat, 27 Oct 2018 00:09:20 +0000
Subject: [PATCH 0794/2345] Add test case for using dataset with class_weight

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

Pylint fix

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .../python/keras/engine/training_utils_test.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/tensorflow/python/keras/engine/training_utils_test.py b/tensorflow/python/keras/engine/training_utils_test.py
index ec44252c9b..dfc27cbc3f 100644
--- a/tensorflow/python/keras/engine/training_utils_test.py
+++ b/tensorflow/python/keras/engine/training_utils_test.py
@@ -225,6 +225,24 @@ class StandardizeWeightsTest(keras_parameterized.TestCase):
     expected = sample_weights * np.array([0.5, 1., 0.5, 0.5, 1.5])
     self.assertAllClose(weights, expected)
 
+  @tf_test_util.run_in_graph_and_eager_modes
+  def test_dataset_with_class_weight(self):
+    model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
+    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    loss = 'mse'
+    metrics = ['mae', metrics_module.CategoricalAccuracy()]
+    model.compile(optimizer, loss, metrics=metrics)
+
+    inputs = np.zeros((10, 3), np.float32)
+    targets = np.zeros((10, 4), np.float32)
+    dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
+    dataset = dataset.repeat(100)
+    dataset = dataset.batch(10)
+    class_weight_np = np.array([0.25, 0.25, 0.25, 0.25])
+    class_weight = dict(enumerate(class_weight_np))
+
+    model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1,
+              class_weight=class_weight)
 
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From aba19e4143bd12070dcec103c55173cc3c0368b1 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Wed, 16 Jan 2019 10:53:44 -0800
Subject: [PATCH 0795/2345] Add a (failing) randomized test for FusedBatchNorm
 training

I had to add a --tf_xla_reference_device flag so that I could set the reference
device to GPU:0.  The CPU kernel for FusedBatchNorm does not support NCHW.

PiperOrigin-RevId: 229586592
---
 tensorflow/compiler/tests/randomized_tests.cc | 52 +++++++++++++++++--
 1 file changed, 47 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/tests/randomized_tests.cc b/tensorflow/compiler/tests/randomized_tests.cc
index d23fd12516..735014fc89 100644
--- a/tensorflow/compiler/tests/randomized_tests.cc
+++ b/tensorflow/compiler/tests/randomized_tests.cc
@@ -80,6 +80,7 @@ int64 tf_xla_random_seed = 0;
 int32 tf_xla_test_repetitions = 20;
 int64 tf_xla_max_tensor_size = 10000LL;
 string* tf_xla_test_device_ptr;  // initial value set in main()
+string* tf_xla_reference_device_ptr;  // initial value set in main()
 bool tf_xla_test_use_jit = true;
 
 string LocalDeviceToFullDeviceName(const string& device) {
@@ -321,6 +322,9 @@ class OpTest : public ::testing::Test {
   // for use as reduction indices.
   Tensor RandomReductionIndices(int rank);
 
+  // Returns a random bit.
+  bool RandomBool();
+
   struct WindowedSpatialDims {
     Padding padding;
     std::vector<int64> kernel_dims;
@@ -453,6 +457,11 @@ std::vector<int64> OpTest::RandomDims(int min_rank, int max_rank,
   return dims;
 }
 
+bool OpTest::RandomBool() {
+  std::bernoulli_distribution d(0.5);
+  return d(generator());
+}
+
 Tensor OpTest::RandomTensor(DataType dtype, bool needs_unique_values,
                             absl::Span<const int64> shape) {
   Tensor tensor(dtype, TensorShape(shape));
@@ -829,8 +838,8 @@ OpTest::TestResult OpTest::ExpectTfAndXlaOutputsAreClose(
     VLOG(1) << "Input: " << input_tensors.back().DebugString();
   }
 
-  string cpu_device =
-      LocalDeviceToFullDeviceName(absl::StrCat(DEVICE_CPU, ":0"));
+  string reference_device =
+      LocalDeviceToFullDeviceName(*tf_xla_reference_device_ptr);
   string test_device = LocalDeviceToFullDeviceName(*tf_xla_test_device_ptr);
 
   DeviceNameUtils::ParsedName parsed_name;
@@ -845,9 +854,9 @@ OpTest::TestResult OpTest::ExpectTfAndXlaOutputsAreClose(
   std::vector<string> expected_inputs, test_inputs;
   std::vector<string> expected_fetches, test_fetches;
   Status status = builder.BuildGraph(
-      absl::StrCat("test", num_tests_, "_expected"), cpu_device,
-      /* use_jit= */ false, &graph, /* test_node_def= */ nullptr,
-      &expected_inputs, &expected_fetches);
+      absl::StrCat("test", num_tests_, "_expected"), reference_device,
+      /*use_jit=*/false, &graph, /*test_node_def=*/nullptr, &expected_inputs,
+      &expected_fetches);
   if (!status.ok()) {
     LOG(ERROR) << "Expected graph construction failed: " << status;
     return kFatalError;
@@ -3346,11 +3355,41 @@ TEST_F(OpTest, ZerosLike) {
   });
 }
 
+// Example failing run:
+//   --tf_xla_reference_device=GPU:0
+//   --tf_xla_test_use_jit=true --tf_xla_test_device=GPU:0
+//   --tf_xla_test_repetitions=2
+//   --gunit_filter='OpTest.FusedBatchNormTraining'
+//   --tf_xla_random_seed=2838146746
+TEST_F(OpTest, FusedBatchNormTraining) {
+  bool is_nhwc = RandomBool();
+  std::vector<int64> x_dims = RandomDims(/*min_rank=*/4, /*max_rank=*/4,
+                                         /*min_size=*/5, /*max_size=*/20);
+  std::vector<int64> scale_dims = {x_dims[is_nhwc ? 3 : 1]};
+  std::vector<int64> offset_dims = {x_dims[is_nhwc ? 3 : 1]};
+  std::vector<int64> mean_dims = {0};
+  std::vector<int64> variance_dims = {0};
+  DataType type = DT_FLOAT;
+  Repeatedly([&] {
+    return ExpectTfAndXlaOutputsAreClose(
+        OpTestBuilder("FusedBatchNorm")
+            .RandomInput(type, x_dims)
+            .RandomInput(type, scale_dims)
+            .RandomInput(type, offset_dims)
+            .RandomInput(type, mean_dims)
+            .RandomInput(type, variance_dims)
+            .Attr("T", type)
+            .Attr("data_format", is_nhwc ? "NHWC" : "NCHW")
+            .Attr("epsilon", static_cast<float>(1.001e-05))
+            .Attr("is_training", true));
+  });
+}
 }  // anonymous namespace
 }  // namespace tensorflow
 
 int main(int argc, char** argv) {
   tensorflow::tf_xla_test_device_ptr = new tensorflow::string("GPU:0");
+  tensorflow::tf_xla_reference_device_ptr = new tensorflow::string("CPU:0");
   std::vector<tensorflow::Flag> flag_list = {
       tensorflow::Flag(
           "tf_xla_random_seed", &tensorflow::tf_xla_random_seed,
@@ -3366,6 +3405,9 @@ int main(int argc, char** argv) {
                        "Maximum number of elements for random input tensors."),
       tensorflow::Flag("tf_xla_test_device", tensorflow::tf_xla_test_device_ptr,
                        "Tensorflow device type to use for test"),
+      tensorflow::Flag("tf_xla_reference_device",
+                       tensorflow::tf_xla_reference_device_ptr,
+                       "Tensorflow device type to use for reference"),
       tensorflow::Flag("tf_xla_test_use_jit", &tensorflow::tf_xla_test_use_jit,
                        "Use JIT compilation for the operator under test"),
   };
-- 
GitLab


From 2eaf93a349fe2d4d4a2d89ea723302d2b34ec98e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 10:54:45 -0800
Subject: [PATCH 0796/2345] Replace calls to deprecated googletest macros
 *TEST_CASE() with *TEST_SUITE()

PiperOrigin-RevId: 229586776
---
 .../compiler/tf2xla/literal_util_test.cc      |  2 +-
 .../partial_run_mgr_test.cc                   |  2 +-
 tensorflow/core/framework/bfloat16_test.cc    |  2 +-
 tensorflow/core/framework/model_test.cc       | 17 ++++----
 tensorflow/core/kernels/conv_ops_test.cc      | 42 +++++++++----------
 tensorflow/core/lib/gtl/int_type_test.cc      |  2 +-
 tensorflow/core/nccl/nccl_manager_test.cc     |  2 +-
 .../bidirectional_sequence_lstm_test.cc       |  2 +-
 tensorflow/lite/kernels/conv_test.cc          |  2 +-
 .../lite/kernels/depthwise_conv_test.cc       |  4 +-
 .../lite/kernels/fully_connected_test.cc      |  4 +-
 tensorflow/lite/kernels/reshape_test.cc       |  6 +--
 .../lite/kernels/transpose_conv_test.cc       |  2 +-
 tensorflow/lite/models/speech_test.cc         |  8 ++--
 .../lite/toco/import_tensorflow_test.cc       |  8 ++--
 tensorflow/lite/toco/tooling_util_test.cc     |  4 +-
 16 files changed, 55 insertions(+), 54 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/literal_util_test.cc b/tensorflow/compiler/tf2xla/literal_util_test.cc
index 15f4c38da2..44bccfe647 100644
--- a/tensorflow/compiler/tf2xla/literal_util_test.cc
+++ b/tensorflow/compiler/tf2xla/literal_util_test.cc
@@ -49,7 +49,7 @@ using Types =
                      std::pair<int16, qint16>, std::pair<uint16, quint16>,
                      std::pair<int32, qint32>>;
 
-TYPED_TEST_CASE(LiteralUtilTest, Types);
+TYPED_TEST_SUITE(LiteralUtilTest, Types);
 
 TYPED_TEST(LiteralUtilTest, LiteralToQuantizedHostTensor) {
   using int_type = typename TypeParam::first_type;
diff --git a/tensorflow/core/distributed_runtime/partial_run_mgr_test.cc b/tensorflow/core/distributed_runtime/partial_run_mgr_test.cc
index 5f7c0cb3ca..a2b799c3e4 100644
--- a/tensorflow/core/distributed_runtime/partial_run_mgr_test.cc
+++ b/tensorflow/core/distributed_runtime/partial_run_mgr_test.cc
@@ -139,7 +139,7 @@ TEST_P(StatusPropagationTest, PartialRunDoneFirst) {
 // ExecutorDone and PartialRunDone.
 Status ExecutorError() { return errors::Internal("executor error"); }
 Status PartialRunError() { return errors::Internal("partial run error"); }
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     PartialRunMgr, StatusPropagationTest,
     ::testing::Values(
         StatusTestParam{Status::OK(), Status::OK(), Status::OK()},
diff --git a/tensorflow/core/framework/bfloat16_test.cc b/tensorflow/core/framework/bfloat16_test.cc
index ce97085494..7da1727e47 100644
--- a/tensorflow/core/framework/bfloat16_test.cc
+++ b/tensorflow/core/framework/bfloat16_test.cc
@@ -75,7 +75,7 @@ TEST_P(Bfloat16Test, TruncateTest) {
   EXPECT_EQ(GetParam().expected_rounding, float(rounded));
 }
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     Bfloat16Test_Instantiation, Bfloat16Test,
     ::testing::Values(
         Bfloat16TestParam{
diff --git a/tensorflow/core/framework/model_test.cc b/tensorflow/core/framework/model_test.cc
index 013f1e61c8..1d7f407e18 100644
--- a/tensorflow/core/framework/model_test.cc
+++ b/tensorflow/core/framework/model_test.cc
@@ -83,9 +83,10 @@ TEST_P(AsyncInterleaveManyTest, Model) {
   EXPECT_GE(async_interleave_many->OutputTime(&input_times), 0);
 }
 
-INSTANTIATE_TEST_CASE_P(Test, AsyncInterleaveManyTest,
-                        ::testing::Combine(::testing::Values(1, 2),
-                                           ::testing::Values(0, 50, 100, 200)));
+INSTANTIATE_TEST_SUITE_P(Test, AsyncInterleaveManyTest,
+                         ::testing::Combine(::testing::Values(1, 2),
+                                            ::testing::Values(0, 50, 100,
+                                                              200)));
 
 class AsyncKnownRatioTest
     : public ::testing::TestWithParam<std::tuple<int64, int64, int64>> {};
@@ -156,10 +157,10 @@ TEST_P(AsyncKnownRatioTest, Model) {
   EXPECT_GE(async_known_many->OutputTime(&input_times), 0);
 }
 
-INSTANTIATE_TEST_CASE_P(Test, AsyncKnownRatioTest,
-                        ::testing::Combine(::testing::Values(1, 2, 4, 8),
-                                           ::testing::Values(0, 50, 100, 200),
-                                           ::testing::Values(0, 1, 2, 4)));
+INSTANTIATE_TEST_SUITE_P(Test, AsyncKnownRatioTest,
+                         ::testing::Combine(::testing::Values(1, 2, 4, 8),
+                                            ::testing::Values(0, 50, 100, 200),
+                                            ::testing::Values(0, 1, 2, 4)));
 
 TEST(InterleaveManyTest, Model) {
   std::shared_ptr<Node> interleave_many =
@@ -245,7 +246,7 @@ TEST_P(KnownRatioTest, Model) {
             num_inputs_per_output * (50 + 100) + 64);
 }
 
-INSTANTIATE_TEST_CASE_P(Test, KnownRatioTest, ::testing::Values(0, 1, 2, 4));
+INSTANTIATE_TEST_SUITE_P(Test, KnownRatioTest, ::testing::Values(0, 1, 2, 4));
 
 TEST(SourceTest, Model) {
   std::shared_ptr<Node> source = model::MakeSourceNode({0, "source", nullptr});
diff --git a/tensorflow/core/kernels/conv_ops_test.cc b/tensorflow/core/kernels/conv_ops_test.cc
index fc93915e16..a4cd67804e 100644
--- a/tensorflow/core/kernels/conv_ops_test.cc
+++ b/tensorflow/core/kernels/conv_ops_test.cc
@@ -952,8 +952,8 @@ class FusedConv2DWithBiasOpTest : public FusedConv2DOpTest<T> {};
 template <typename T>
 class FusedConv2DWithBatchNormOpTest : public FusedConv2DOpTest<T> {};
 
-TYPED_TEST_CASE_P(FusedConv2DWithBiasOpTest);
-TYPED_TEST_CASE_P(FusedConv2DWithBatchNormOpTest);
+TYPED_TEST_SUITE_P(FusedConv2DWithBiasOpTest);
+TYPED_TEST_SUITE_P(FusedConv2DWithBatchNormOpTest);
 
 // -------------------------------------------------------------------------- //
 // Conv2D + BiasAdd + {Relu}                                                  //
@@ -1035,29 +1035,29 @@ TYPED_TEST_P(FusedConv2DWithBatchNormOpTest, SpatialConvolutionAndRelu) {
   this->VerifyConv2DWithBatchNormAndRelu(filter_size, filter_count);
 }
 
-REGISTER_TYPED_TEST_CASE_P(FusedConv2DWithBiasOpTest,    //
-                           OneByOneConvolution,          //
-                           ImageSizeConvolution,         //
-                           SpatialConvolution,           //
-                           OneByOneConvolutionAndRelu,   //
-                           ImageSizeConvolutionAndRelu,  //
-                           SpatialConvolutionAndRelu);
-
-REGISTER_TYPED_TEST_CASE_P(FusedConv2DWithBatchNormOpTest,  //
-                           OneByOneConvolution,             //
-                           ImageSizeConvolution,            //
-                           SpatialConvolution,              //
-                           OneByOneConvolutionAndRelu,      //
-                           ImageSizeConvolutionAndRelu,     //
-                           SpatialConvolutionAndRelu);
+REGISTER_TYPED_TEST_SUITE_P(FusedConv2DWithBiasOpTest,    //
+                            OneByOneConvolution,          //
+                            ImageSizeConvolution,         //
+                            SpatialConvolution,           //
+                            OneByOneConvolutionAndRelu,   //
+                            ImageSizeConvolutionAndRelu,  //
+                            SpatialConvolutionAndRelu);
+
+REGISTER_TYPED_TEST_SUITE_P(FusedConv2DWithBatchNormOpTest,  //
+                            OneByOneConvolution,             //
+                            ImageSizeConvolution,            //
+                            SpatialConvolution,              //
+                            OneByOneConvolutionAndRelu,      //
+                            ImageSizeConvolutionAndRelu,     //
+                            SpatialConvolutionAndRelu);
 
 using FusedBiasAddDataTypes = ::testing::Types<float, double>;
-INSTANTIATE_TYPED_TEST_CASE_P(Test, FusedConv2DWithBiasOpTest,
-                              FusedBiasAddDataTypes);
+INSTANTIATE_TYPED_TEST_SUITE_P(Test, FusedConv2DWithBiasOpTest,
+                               FusedBiasAddDataTypes);
 
 using FusedBatchNormDataTypes = ::testing::Types<float>;
-INSTANTIATE_TYPED_TEST_CASE_P(Test, FusedConv2DWithBatchNormOpTest,
-                              FusedBatchNormDataTypes);
+INSTANTIATE_TYPED_TEST_SUITE_P(Test, FusedConv2DWithBatchNormOpTest,
+                               FusedBatchNormDataTypes);
 
 ////////////////////////////////////////////////////////////////////////////////
 // Performance benchmarks for the FusedConv2DWithBiasOp.                      //
diff --git a/tensorflow/core/lib/gtl/int_type_test.cc b/tensorflow/core/lib/gtl/int_type_test.cc
index 61d364017c..89d2d0e8fe 100644
--- a/tensorflow/core/lib/gtl/int_type_test.cc
+++ b/tensorflow/core/lib/gtl/int_type_test.cc
@@ -45,7 +45,7 @@ typedef ::testing::Types<Int8_IT, UInt8_IT, Int16_IT, UInt16_IT, Int32_IT,
                          Int64_IT, UInt64_IT, Long_IT>
     SupportedIntTypes;
 
-TYPED_TEST_CASE(IntTypeTest, SupportedIntTypes);
+TYPED_TEST_SUITE(IntTypeTest, SupportedIntTypes);
 
 TYPED_TEST(IntTypeTest, TestInitialization) {
   constexpr typename TestFixture::T a;
diff --git a/tensorflow/core/nccl/nccl_manager_test.cc b/tensorflow/core/nccl/nccl_manager_test.cc
index 58bb84ac57..e65af13389 100644
--- a/tensorflow/core/nccl/nccl_manager_test.cc
+++ b/tensorflow/core/nccl/nccl_manager_test.cc
@@ -209,7 +209,7 @@ const Scalar NcclManagerTest<Scalar>::max_ =
 
 // Instantiate tests for float and double.
 using TypeList = ::testing::Types<float, double>;
-TYPED_TEST_CASE(NcclManagerTest, TypeList);
+TYPED_TEST_SUITE(NcclManagerTest, TypeList);
 
 // Test basic sum reduction.
 TYPED_TEST(NcclManagerTest, BasicSumReduction) {
diff --git a/tensorflow/lite/kernels/bidirectional_sequence_lstm_test.cc b/tensorflow/lite/kernels/bidirectional_sequence_lstm_test.cc
index 59ea47a2a2..707f06af83 100644
--- a/tensorflow/lite/kernels/bidirectional_sequence_lstm_test.cc
+++ b/tensorflow/lite/kernels/bidirectional_sequence_lstm_test.cc
@@ -400,7 +400,7 @@ class BidirectionalLSTMOpModel : public SingleOpModel {
 // indicating whether to use quantization or not.
 class LSTMOpTest : public ::testing::TestWithParam<bool> {};
 
-INSTANTIATE_TEST_CASE_P(QuantizationOrNot, LSTMOpTest, ::testing::Bool());
+INSTANTIATE_TEST_SUITE_P(QuantizationOrNot, LSTMOpTest, ::testing::Bool());
 
 TEST_P(LSTMOpTest, BlackBoxTestNoCifgNoPeepholeNoProjectionNoClipping) {
   const int n_batch = 1;
diff --git a/tensorflow/lite/kernels/conv_test.cc b/tensorflow/lite/kernels/conv_test.cc
index 478df3354f..d0350b2fa7 100644
--- a/tensorflow/lite/kernels/conv_test.cc
+++ b/tensorflow/lite/kernels/conv_test.cc
@@ -1069,7 +1069,7 @@ TEST_P(ConvolutionOpTest, DISABLED_PointwiseMultifilterHybrid) {
                   0.0474)));
 }
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     ConvolutionOpTest, ConvolutionOpTest,
     ::testing::ValuesIn(SingleOpTest::GetKernelTags(*kKernelMap)));
 
diff --git a/tensorflow/lite/kernels/depthwise_conv_test.cc b/tensorflow/lite/kernels/depthwise_conv_test.cc
index d924e6f700..75aed4cc4a 100644
--- a/tensorflow/lite/kernels/depthwise_conv_test.cc
+++ b/tensorflow/lite/kernels/depthwise_conv_test.cc
@@ -437,11 +437,11 @@ TEST_P(QuantizedDepthwiseConvolutionOpTest, SimpleDilatedTestPaddingSame) {
               ElementsAreArray({4, 7, 3, 6, 10, 4, 2, 3, 1}));
 }
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     DepthwiseConvolutionOpTest, DepthwiseConvolutionOpTest,
     ::testing::ValuesIn(SingleOpTest::GetKernelTags(*kKernelMap)));
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     QuantizedDepthwiseConvolutionOpTest, QuantizedDepthwiseConvolutionOpTest,
     ::testing::ValuesIn(SingleOpTest::GetKernelTags(*kKernelMap)));
 
diff --git a/tensorflow/lite/kernels/fully_connected_test.cc b/tensorflow/lite/kernels/fully_connected_test.cc
index d1d29fc7e6..03f4ea7143 100644
--- a/tensorflow/lite/kernels/fully_connected_test.cc
+++ b/tensorflow/lite/kernels/fully_connected_test.cc
@@ -725,11 +725,11 @@ TEST_P(QuantizedFullyConnectedOpTest,
               ElementsAre(175, 177, 179, 243, 245, 247));
 }
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     FloatFullyConnectedOpTest, FloatFullyConnectedOpTest,
     ::testing::ValuesIn(SingleOpTest::GetKernelTags(*kKernelMap)));
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     QuantizedFullyConnectedOpTest, QuantizedFullyConnectedOpTest,
     ::testing::ValuesIn(SingleOpTest::GetKernelTags(*kKernelMapNoPie)));
 
diff --git a/tensorflow/lite/kernels/reshape_test.cc b/tensorflow/lite/kernels/reshape_test.cc
index f98f3eb9ae..e2210aeaf0 100644
--- a/tensorflow/lite/kernels/reshape_test.cc
+++ b/tensorflow/lite/kernels/reshape_test.cc
@@ -236,9 +236,9 @@ TEST_P(ReshapeOpTest, Strings) {
               ElementsAreArray({"1", "2", "3", "4", "5", "6", "7", "8"}));
 }
 
-INSTANTIATE_TEST_CASE_P(VariedShapeSpec, ReshapeOpTest,
-                        ::testing::Values(kAsReshapeOption, kAsConstantTensor,
-                                          kAsTensor));
+INSTANTIATE_TEST_SUITE_P(VariedShapeSpec, ReshapeOpTest,
+                         ::testing::Values(kAsReshapeOption, kAsConstantTensor,
+                                           kAsTensor));
 }  // namespace
 }  // namespace tflite
 
diff --git a/tensorflow/lite/kernels/transpose_conv_test.cc b/tensorflow/lite/kernels/transpose_conv_test.cc
index 0520d84a30..44d1336b99 100644
--- a/tensorflow/lite/kernels/transpose_conv_test.cc
+++ b/tensorflow/lite/kernels/transpose_conv_test.cc
@@ -252,7 +252,7 @@ TEST_P(TransposeConvOpTest, AccuracyTest) {
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 3, 4, 1}));
 }
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     TransposeConvOpTest, TransposeConvOpTest,
     ::testing::ValuesIn(SingleOpTest::GetKernelTags(*kKernelMap)));
 
diff --git a/tensorflow/lite/models/speech_test.cc b/tensorflow/lite/models/speech_test.cc
index 17b7e8f28e..f3509d1ece 100644
--- a/tensorflow/lite/models/speech_test.cc
+++ b/tensorflow/lite/models/speech_test.cc
@@ -196,10 +196,10 @@ TEST_P(SpeechTest, DISABLED_TtsTest) {
 // 200s just to bring up the Android emulator.)
 static const int kAllInvocations = -1;
 static const int kFirstFewInvocations = 10;
-INSTANTIATE_TEST_CASE_P(LongTests, SpeechTest,
-                        ::testing::Values(kAllInvocations));
-INSTANTIATE_TEST_CASE_P(ShortTests, SpeechTest,
-                        ::testing::Values(kFirstFewInvocations));
+INSTANTIATE_TEST_SUITE_P(LongTests, SpeechTest,
+                         ::testing::Values(kAllInvocations));
+INSTANTIATE_TEST_SUITE_P(ShortTests, SpeechTest,
+                         ::testing::Values(kFirstFewInvocations));
 
 }  // namespace
 }  // namespace tflite
diff --git a/tensorflow/lite/toco/import_tensorflow_test.cc b/tensorflow/lite/toco/import_tensorflow_test.cc
index de7f4cdb7e..8ff3f7733a 100644
--- a/tensorflow/lite/toco/import_tensorflow_test.cc
+++ b/tensorflow/lite/toco/import_tensorflow_test.cc
@@ -257,8 +257,8 @@ std::vector<tensorflow::DataType> TestTypes() {
   return {DT_FLOAT, DT_INT32, DT_INT64, DT_BOOL, DT_QUINT8, DT_COMPLEX64};
 }
 
-INSTANTIATE_TEST_CASE_P(ShapeImportTest, ShapeImportTest,
-                        ::testing::ValuesIn(TestTypes()));
+INSTANTIATE_TEST_SUITE_P(ShapeImportTest, ShapeImportTest,
+                         ::testing::ValuesIn(TestTypes()));
 
 class ContentImportTest : public ::testing::Test {
  public:
@@ -418,8 +418,8 @@ TEST_P(TypeImportTest, BasicTypeInference) {
           model.operators[0].get());
   ASSERT_THAT(op->output_data_types, ::testing::ElementsAre(GetParam().second));
 }
-INSTANTIATE_TEST_CASE_P(BasicTypeInference, TypeImportTest,
-                        ::testing::ValuesIn(UnaryTestTypes()));
+INSTANTIATE_TEST_SUITE_P(BasicTypeInference, TypeImportTest,
+                         ::testing::ValuesIn(UnaryTestTypes()));
 
 TEST(ImportTest, TypeInferenceWithFixedOutputType) {
   // Create an op that has a fixed output type (bool).
diff --git a/tensorflow/lite/toco/tooling_util_test.cc b/tensorflow/lite/toco/tooling_util_test.cc
index faa6fe412e..f063ce71e9 100644
--- a/tensorflow/lite/toco/tooling_util_test.cc
+++ b/tensorflow/lite/toco/tooling_util_test.cc
@@ -96,8 +96,8 @@ TEST_P(ShapeTest, Agrees) {
   }
 }
 
-INSTANTIATE_TEST_CASE_P(AgreeBroadcast, ShapeTest,
-                        ::testing::ValuesIn(CreateShapePairs()));
+INSTANTIATE_TEST_SUITE_P(AgreeBroadcast, ShapeTest,
+                         ::testing::ValuesIn(CreateShapePairs()));
 
 static const char kNegativeValuesMessage[] =
     "Tensor shape should not include negative values";
-- 
GitLab


From 5f9d03ee64231a5ed2b2cb1ae984a339300547c6 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Tue, 15 Jan 2019 22:00:31 +0000
Subject: [PATCH 0797/2345] Skip the metrics as suggested in review feedback

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/keras/engine/training_utils_test.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/keras/engine/training_utils_test.py b/tensorflow/python/keras/engine/training_utils_test.py
index dfc27cbc3f..64102d24ee 100644
--- a/tensorflow/python/keras/engine/training_utils_test.py
+++ b/tensorflow/python/keras/engine/training_utils_test.py
@@ -27,6 +27,7 @@ from tensorflow.python.data.ops import readers
 from tensorflow.python.eager import context
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.keras import keras_parameterized
+from tensorflow.python.keras import testing_utils
 from tensorflow.python.keras.engine import training_utils
 from tensorflow.python.keras.utils import tf_utils
 from tensorflow.python.platform import test
@@ -225,13 +226,9 @@ class StandardizeWeightsTest(keras_parameterized.TestCase):
     expected = sample_weights * np.array([0.5, 1., 0.5, 0.5, 1.5])
     self.assertAllClose(weights, expected)
 
-  @tf_test_util.run_in_graph_and_eager_modes
   def test_dataset_with_class_weight(self):
     model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
-    optimizer = RMSPropOptimizer(learning_rate=0.001)
-    loss = 'mse'
-    metrics = ['mae', metrics_module.CategoricalAccuracy()]
-    model.compile(optimizer, loss, metrics=metrics)
+    model.compile('rmsprop', 'mse')
 
     inputs = np.zeros((10, 3), np.float32)
     targets = np.zeros((10, 4), np.float32)
-- 
GitLab


From 168eca540fc3b2090b15535d6728ed88520fae49 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Wed, 16 Jan 2019 11:14:30 -0800
Subject: [PATCH 0798/2345] Update tests to use the new distribute_py_test
 build rule.

PiperOrigin-RevId: 229590924
---
 tensorflow/contrib/distribute/python/BUILD    | 50 +++++++++++--------
 .../core/platform/default/distribute.bzl      |  2 +
 2 files changed, 32 insertions(+), 20 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 384fe53923..3a4f23e597 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -640,12 +640,11 @@ distribute_py_test(
 )
 
 # TODO(b/121200287): Remove this in 2.0
-cuda_py_test(
+distribute_py_test(
     name = "keras_backward_compat_test",
     srcs = ["keras_backward_compat_test.py"],
-    additional_deps = [
-        ":keras_test_lib",
-    ],
+    full_precision = True,
+    main = "keras_backward_compat_test.py",
     shard_count = 16,
     tags = [
         "multi_and_single_gpu",
@@ -654,6 +653,9 @@ cuda_py_test(
         "no_windows_gpu",
         "notsan",
     ],
+    deps = [
+        ":keras_test_lib",
+    ],
 )
 
 py_library(
@@ -680,13 +682,12 @@ py_library(
     ],
 )
 
-cuda_py_test(
+distribute_py_test(
     name = "keras_dnn_correctness_test",
     size = "medium",
     srcs = ["keras_dnn_correctness_test.py"],
-    additional_deps = [
-        ":keras_correctness_test_lib",
-    ],
+    full_precision = True,
+    main = "keras_dnn_correctness_test.py",
     # Shard count is set to an odd number to distribute tasks across
     # shards more evenly.
     shard_count = 19,
@@ -697,15 +698,17 @@ cuda_py_test(
         "no_windows_gpu",
         "notsan",
     ],
+    deps = [
+        ":keras_correctness_test_lib",
+    ],
 )
 
-cuda_py_test(
+distribute_py_test(
     name = "keras_image_model_correctness_test",
     size = "medium",
     srcs = ["keras_image_model_correctness_test.py"],
-    additional_deps = [
-        ":keras_correctness_test_lib",
-    ],
+    full_precision = True,
+    main = "keras_image_model_correctness_test.py",
     # Shard count is set to an odd number to distribute tasks across
     # shards more evenly.
     shard_count = 31,
@@ -716,15 +719,17 @@ cuda_py_test(
         "no_windows_gpu",
         "notsan",
     ],
+    deps = [
+        ":keras_correctness_test_lib",
+    ],
 )
 
-cuda_py_test(
+distribute_py_test(
     name = "keras_embedding_model_correctness_test",
     size = "medium",
     srcs = ["keras_embedding_model_correctness_test.py"],
-    additional_deps = [
-        ":keras_correctness_test_lib",
-    ],
+    full_precision = True,
+    main = "keras_embedding_model_correctness_test.py",
     # Shard count is set to an odd number to distribute tasks across
     # shards more evenly.
     shard_count = 31,
@@ -735,15 +740,17 @@ cuda_py_test(
         "no_windows_gpu",
         "notsan",
     ],
+    deps = [
+        ":keras_correctness_test_lib",
+    ],
 )
 
-cuda_py_test(
+distribute_py_test(
     name = "keras_lstm_model_correctness_test",
     size = "medium",
     srcs = ["keras_lstm_model_correctness_test.py"],
-    additional_deps = [
-        ":keras_correctness_test_lib",
-    ],
+    full_precision = True,
+    main = "keras_lstm_model_correctness_test.py",
     # Shard count is set to an odd number to distribute tasks across
     # shards more evenly.
     shard_count = 31,
@@ -754,6 +761,9 @@ cuda_py_test(
         "no_windows_gpu",
         "notsan",
     ],
+    deps = [
+        ":keras_correctness_test_lib",
+    ],
 )
 
 py_library(
diff --git a/tensorflow/core/platform/default/distribute.bzl b/tensorflow/core/platform/default/distribute.bzl
index a3afc37f03..ea8fa8708e 100644
--- a/tensorflow/core/platform/default/distribute.bzl
+++ b/tensorflow/core/platform/default/distribute.bzl
@@ -11,6 +11,7 @@ def distribute_py_test(
         main = None,
         args = [],
         shard_count = 1,
+        full_precision = False,
         **kwargs):
     """Generates py_test targets for CPU and GPU.
 
@@ -26,6 +27,7 @@ def distribute_py_test(
         **kwargs: extra keyword arguments to the test.
     """
 
+    _ignore = (full_precision)
     cuda_py_test(
         name = name,
         srcs = srcs,
-- 
GitLab


From 42f6aa2d5aeda0a586fd64c82e7cc71c98e8e047 Mon Sep 17 00:00:00 2001
From: Tong Shen <endlessroad@google.com>
Date: Wed, 16 Jan 2019 11:15:04 -0800
Subject: [PATCH 0799/2345] Add a DFSFrom() algorithm.

PiperOrigin-RevId: 229591031
---
 tensorflow/core/graph/algorithm.cc | 39 +++++++++++++++++++++++-------
 tensorflow/core/graph/algorithm.h  | 12 +++++++++
 2 files changed, 42 insertions(+), 9 deletions(-)

diff --git a/tensorflow/core/graph/algorithm.cc b/tensorflow/core/graph/algorithm.cc
index 25bd3516a1..5ad1c19dc1 100644
--- a/tensorflow/core/graph/algorithm.cc
+++ b/tensorflow/core/graph/algorithm.cc
@@ -22,25 +22,29 @@ limitations under the License.
 #include "tensorflow/core/platform/logging.h"
 
 namespace tensorflow {
-
-void DFS(const Graph& g, const std::function<void(Node*)>& enter,
-         const std::function<void(Node*)>& leave,
-         const NodeComparator& stable_comparator,
-         const EdgeFilter& edge_filter) {
+namespace {
+template <typename T>
+void DFSFromHelper(const Graph& g, gtl::ArraySlice<T> start,
+                   const std::function<void(T)>& enter,
+                   const std::function<void(T)>& leave,
+                   const NodeComparator& stable_comparator,
+                   const EdgeFilter& edge_filter) {
   // Stack of work to do.
   struct Work {
-    Node* node;
+    T node;
     bool leave;  // Are we entering or leaving n?
   };
-  std::vector<Work> stack;
-  stack.push_back(Work{g.source_node(), false});
+  std::vector<Work> stack(start.size());
+  for (int i = 0; i < start.size(); ++i) {
+    stack[i] = Work{start[i], false};
+  }
 
   std::vector<bool> visited(g.num_node_ids(), false);
   while (!stack.empty()) {
     Work w = stack.back();
     stack.pop_back();
 
-    Node* n = w.node;
+    T n = w.node;
     if (w.leave) {
       leave(n);
       continue;
@@ -80,6 +84,23 @@ void DFS(const Graph& g, const std::function<void(Node*)>& enter,
     }
   }
 }
+}  // namespace
+
+void DFS(const Graph& g, const std::function<void(Node*)>& enter,
+         const std::function<void(Node*)>& leave,
+         const NodeComparator& stable_comparator,
+         const EdgeFilter& edge_filter) {
+  DFSFromHelper(g, {g.source_node()}, enter, leave, stable_comparator,
+                edge_filter);
+}
+
+void DFSFrom(const Graph& g, gtl::ArraySlice<const Node*> start,
+             const std::function<void(const Node*)>& enter,
+             const std::function<void(const Node*)>& leave,
+             const NodeComparator& stable_comparator,
+             const EdgeFilter& edge_filter) {
+  DFSFromHelper(g, start, enter, leave, stable_comparator, edge_filter);
+}
 
 void ReverseDFS(const Graph& g, const std::function<void(Node*)>& enter,
                 const std::function<void(Node*)>& leave,
diff --git a/tensorflow/core/graph/algorithm.h b/tensorflow/core/graph/algorithm.h
index 45f8a29a92..3479605df8 100644
--- a/tensorflow/core/graph/algorithm.h
+++ b/tensorflow/core/graph/algorithm.h
@@ -55,6 +55,18 @@ extern void DFS(const Graph& g, const std::function<void(Node*)>& enter,
                 const NodeComparator& stable_comparator = {},
                 const EdgeFilter& edge_filter = {});
 
+// Perform a depth-first-search on g starting at the 'start' nodes.
+// If enter is not empty, calls enter(n) before visiting any children of n.
+// If leave is not empty, calls leave(n) after visiting all children of n.
+// If stable_comparator is set, a stable ordering of visit is achieved by
+// sorting a node's neighbors first before visiting them.
+// If edge_filter is set then ignores edges for which edge_filter returns false.
+extern void DFSFrom(const Graph& g, gtl::ArraySlice<const Node*> start,
+                    const std::function<void(const Node*)>& enter,
+                    const std::function<void(const Node*)>& leave,
+                    const NodeComparator& stable_comparator = {},
+                    const EdgeFilter& edge_filter = {});
+
 // Perform a reverse depth-first-search on g starting at the sink node.
 // If enter is not empty, calls enter(n) before visiting any parents of n.
 // If leave is not empty, calls leave(n) after visiting all parents of n.
-- 
GitLab


From 3e41d77991f3807eb796a62573f2451cedee0c63 Mon Sep 17 00:00:00 2001
From: Katherine Wu <kathywu@google.com>
Date: Wed, 16 Jan 2019 11:23:28 -0800
Subject: [PATCH 0800/2345] Add mean tensor to Keras metrics. (From
 tf.metrics.mean_tensor).

PiperOrigin-RevId: 229592753
---
 tensorflow/python/keras/metrics.py      | 111 ++++++++++++++++++
 tensorflow/python/keras/metrics_test.py | 150 ++++++++++++++++++++++++
 2 files changed, 261 insertions(+)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 245dffe99e..d6010e3cb0 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -29,6 +29,7 @@ from tensorflow.python.eager import context
 from tensorflow.python.eager import function
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras.engine.base_layer import Layer
 from tensorflow.python.keras.losses import binary_crossentropy
@@ -2300,6 +2301,116 @@ class MeanIoU(Metric):
     return dict(list(base_config.items()) + list(config.items()))
 
 
+class MeanTensor(Metric):
+  """Computes the element-wise (weighted) mean of the given tensors.
+
+  `MeanTensor` returns a tensor with the same shape of the input tensors. The
+  mean value is updated by keeping local variables `total` and `count`. The
+  `total` tracks the sum of the weighted values, and `count` stores the sum of
+  the weighted counts.
+
+  Usage:
+
+  ```python
+  m = tf.metrics.MeanTensor()
+  m.update_state([0, 1, 2, 3])
+  m.update_state([4, 5, 6, 7])
+  print('Result: ', m.result().numpy())  # Result: [2, 3, 4, 5]
+  m.update_state([12, 10, 8, 6], sample_weights= [0, 0.2, 0.5, 1])
+  print('Result: ', m.result().numpy())  # Result: [2, 3.636, 4.8, 5.333]
+  ```
+  """
+
+  def __init__(self, name='mean_tensor', dtype=None):
+    """Creates a `MeanTensor` instance.
+
+    Args:
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+    """
+    super(MeanTensor, self).__init__(name=name, dtype=dtype)
+    self._shape = None
+    self._total = None
+    self._count = None
+    self._built = False
+
+  def _build(self, shape):
+    self._shape = tensor_shape.TensorShape(shape)
+    # Create new state variables
+    self._total = self.add_weight(
+        'total', shape=shape, initializer=init_ops.zeros_initializer)
+    self._count = self.add_weight(
+        'count', shape=shape, initializer=init_ops.zeros_initializer)
+    with ops.init_scope():
+      if not context.executing_eagerly():
+        K._initialize_variables(K._get_session())  # pylint: disable=protected-access
+    self._built = True
+
+  @property
+  def total(self):
+    return self._total if self._built else None
+
+  @property
+  def count(self):
+    return self._count if self._built else None
+
+  def update_state(self, values, sample_weight=None):
+    """Accumulates statistics for computing the element-wise mean.
+
+    Args:
+      values: Per-example value.
+      sample_weight: Optional weighting of each example. Defaults to 1.
+
+    Returns:
+      Update op.
+    """
+    values = math_ops.cast(values, self._dtype)
+    if not self._built:
+      self._build(values.shape)
+    elif values.shape != self._shape:
+      raise ValueError('MeanTensor input values must always have the same '
+                       'shape. Expected shape (set during the first call): {}. '
+                       'Got: {}'.format(self._shape, values.get_shape()))
+
+    num_values = array_ops.ones_like(values)
+    if sample_weight is not None:
+      sample_weight = math_ops.cast(sample_weight, self._dtype)
+
+      # Update dimensions of weights to match with values if possible.
+      values, _, sample_weight = squeeze_or_expand_dimensions(
+          values, None, sample_weight)
+      try:
+        # Broadcast weights if possible.
+        sample_weight = weights_broadcast_ops.broadcast_weights(
+            sample_weight, values)
+      except ValueError:
+        # Reduce values to same ndim as weight array
+        ndim = K.ndim(values)
+        weight_ndim = K.ndim(sample_weight)
+        values = math_ops.reduce_mean(
+            values, axis=list(range(weight_ndim, ndim)))
+
+      num_values = math_ops.multiply(num_values, sample_weight)
+      values = math_ops.multiply(values, sample_weight)
+
+    update_total_op = state_ops.assign_add(self._total, values)
+    with ops.control_dependencies([update_total_op]):
+      return state_ops.assign_add(self._count, num_values)
+
+  def result(self):
+    if not self._built:
+      raise ValueError(
+          'MeanTensor does not have any result yet. Please call the MeanTensor '
+          'instance or use `.update_state(value)` before retrieving the result.'
+          )
+    return math_ops.div_no_nan(self.total, self.count)
+
+  def reset_states(self):
+    if self._built:
+      for v in self.variables:
+        K.set_value(v, np.zeros(self._shape.as_list()))
+
+
 def accuracy(y_true, y_pred):
   y_pred.get_shape().assert_is_compatible_with(y_true.get_shape())
   if y_true.dtype != y_pred.dtype:
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index bf2f4d88fa..bd94132a9a 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -23,6 +23,7 @@ import os
 import numpy as np
 
 from tensorflow.python.eager import context
+from tensorflow.python.eager import function as eager_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -30,6 +31,7 @@ from tensorflow.python.framework import test_util
 from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import layers
 from tensorflow.python.keras import metrics
+from tensorflow.python.keras import Model
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import variables
@@ -1050,6 +1052,154 @@ class MeanIoUTest(test.TestCase):
     self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)
 
 
+class MeanTensorTest(keras_parameterized.TestCase):
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_config(self):
+    m = metrics.MeanTensor(name='mean_by_element')
+
+    # check config
+    self.assertEqual(m.name, 'mean_by_element')
+    self.assertTrue(m.stateful)
+    self.assertEqual(m.dtype, dtypes.float32)
+    self.assertEqual(len(m.variables), 0)
+
+    with self.assertRaisesRegexp(ValueError, 'does not have any result yet'):
+      m.result()
+
+    self.evaluate(m([[3], [5], [3]]))
+    self.assertAllEqual(m._shape, [3, 1])
+
+    m2 = metrics.MeanTensor.from_config(m.get_config())
+    self.assertEqual(m2.name, 'mean_by_element')
+    self.assertTrue(m2.stateful)
+    self.assertEqual(m2.dtype, dtypes.float32)
+    self.assertEqual(len(m2.variables), 0)
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_unweighted(self):
+    m = metrics.MeanTensor(dtype=dtypes.float64)
+
+    # check __call__()
+    self.assertAllClose(self.evaluate(m([100, 40])), [100, 40])
+    self.assertAllClose(self.evaluate(m.total), [100, 40])
+    self.assertAllClose(self.evaluate(m.count), [1, 1])
+
+    # check update_state() and result() + state accumulation + tensor input
+    update_op = m.update_state(ops.convert_n_to_tensor([1, 5]))
+    self.evaluate(update_op)
+    self.assertAllClose(self.evaluate(m.result()), [50.5, 22.5])
+    self.assertAllClose(self.evaluate(m.total), [101, 45])
+    self.assertAllClose(self.evaluate(m.count), [2, 2])
+
+    # check reset_states()
+    m.reset_states()
+    self.assertAllClose(self.evaluate(m.total), [0, 0])
+    self.assertAllClose(self.evaluate(m.count), [0, 0])
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_weighted(self):
+    m = metrics.MeanTensor(dtype=dtypes.float64)
+    self.assertEqual(m.dtype, dtypes.float64)
+
+    # check scalar weight
+    result_t = m([100, 30], sample_weight=0.5)
+    self.assertAllClose(self.evaluate(result_t), [100, 30])
+    self.assertAllClose(self.evaluate(m.total), [50, 15])
+    self.assertAllClose(self.evaluate(m.count), [0.5, 0.5])
+
+    # check weights not scalar and weights rank matches values rank
+    result_t = m([1, 5], sample_weight=[1, 0.2])
+    result = self.evaluate(result_t)
+    self.assertAllClose(result, [51 / 1.5, 16 / 0.7], 2)
+    self.assertAllClose(self.evaluate(m.total), [51, 16])
+    self.assertAllClose(self.evaluate(m.count), [1.5, 0.7])
+
+    # check weights broadcast
+    result_t = m([1, 2], sample_weight=0.5)
+    self.assertAllClose(self.evaluate(result_t), [51.5 / 2, 17 / 1.2])
+    self.assertAllClose(self.evaluate(m.total), [51.5, 17])
+    self.assertAllClose(self.evaluate(m.count), [2, 1.2])
+
+    # check weights squeeze
+    result_t = m([1, 5], sample_weight=[[1], [0.2]])
+    self.assertAllClose(self.evaluate(result_t), [52.5 / 3, 18 / 1.4])
+    self.assertAllClose(self.evaluate(m.total), [52.5, 18])
+    self.assertAllClose(self.evaluate(m.count), [3, 1.4])
+
+    # check weights expand
+    m = metrics.MeanTensor((2, 1), dtype=dtypes.float64)
+    self.evaluate(variables.variables_initializer(m.variables))
+    result_t = m([[1], [5]], sample_weight=[1, 0.2])
+    self.assertAllClose(self.evaluate(result_t), [[1], [5]])
+    self.assertAllClose(self.evaluate(m.total), [[1], [1]])
+    self.assertAllClose(self.evaluate(m.count), [[1], [0.2]])
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_invalid_value_shape(self):
+    m = metrics.MeanTensor(dtype=dtypes.float64)
+    m([1])
+    with self.assertRaisesRegexp(
+        ValueError, 'MeanTensor input values must always have the same shape'):
+      m([1, 5])
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_build_in_tf_function(self):
+    """Ensure that variables are created correctly in a tf function."""
+    m = metrics.MeanTensor(dtype=dtypes.float64)
+
+    @eager_function.defun
+    def call_metric(x):
+      return m(x)
+
+    self.assertAllClose(self.evaluate(call_metric([100, 40])), [100, 40])
+    self.assertAllClose(self.evaluate(m.total), [100, 40])
+    self.assertAllClose(self.evaluate(m.count), [1, 1])
+    self.assertAllClose(self.evaluate(call_metric([20, 2])), [60, 21])
+
+  def test_in_keras_model(self):
+    with context.eager_mode():
+      class ModelWithMetric(Model):
+
+        def __init__(self):
+          super(ModelWithMetric, self).__init__()
+          self.dense1 = layers.Dense(
+              3, activation='relu', kernel_initializer='ones')
+          self.dense2 = layers.Dense(
+              1, activation='sigmoid', kernel_initializer='ones')
+          self.mean_tensor = metrics.MeanTensor()
+
+        def call(self, x):
+          x = self.dense1(x)
+          x = self.dense2(x)
+          self.mean_tensor(self.dense1.kernel)
+          return x
+
+      model = ModelWithMetric()
+      model.compile(
+          loss='mae',
+          optimizer='rmsprop',
+          run_eagerly=True)
+
+      x = np.ones((100, 4))
+      y = np.zeros((100, 1))
+      model.evaluate(x, y, batch_size=50)
+      self.assertAllClose(self.evaluate(model.mean_tensor.result()),
+                          np.ones((4, 3)))
+      self.assertAllClose(self.evaluate(model.mean_tensor.total),
+                          np.full((4, 3), 2))
+      self.assertAllClose(self.evaluate(model.mean_tensor.count),
+                          np.full((4, 3), 2))
+
+      model.evaluate(x, y, batch_size=25)
+      self.assertAllClose(self.evaluate(model.mean_tensor.result()),
+                          np.ones((4, 3)))
+      self.assertAllClose(self.evaluate(model.mean_tensor.total),
+                          np.full((4, 3), 4))
+      self.assertAllClose(self.evaluate(model.mean_tensor.count),
+                          np.full((4, 3), 4))
+
+
 def _get_model(compile_metrics):
   model_layers = [
       layers.Dense(3, activation='relu', kernel_initializer='ones'),
-- 
GitLab


From 0f91a840d86210f298c8caf0e8640b4b9925a058 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 11:25:00 -0800
Subject: [PATCH 0801/2345] Support lists of floating points in custom op
 attributes.

PiperOrigin-RevId: 229593054
---
 tensorflow/lite/toco/tflite/operator.cc      | 15 +++++++++++++++
 tensorflow/lite/toco/tflite/operator_test.cc | 14 +++++++++++++-
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index d9c051ebf7..088673fd95 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -1647,6 +1647,13 @@ class TensorFlowUnsupported : public BaseOperator {
             }
             fbb->EndVector(start, /*typed=*/true, /*fixed=*/false);
             has_valid_attr = true;
+          } else if (attr.list().f_size() > 0) {
+            auto start = fbb->StartVector(key);
+            for (const float v : attr.list().f()) {
+              fbb->Add(v);
+            }
+            fbb->EndVector(start, /*typed=*/true, /*fixed=*/false);
+            has_valid_attr = true;
           } else {
             LOG(WARNING)
                 << "Ignoring unsupported type in list attribute with key '"
@@ -1707,6 +1714,14 @@ class TensorFlowUnsupported : public BaseOperator {
           }
           break;
         }
+        case 13: {  // flexbuffers::FBT_VECTOR_FLOAT: {
+          auto* list = (*attr)[key].mutable_list();
+          const auto& vector = value.AsTypedVector();
+          for (size_t i = 0; i < vector.size(); i++) {
+            list->add_f(vector[i].AsFloat());
+          }
+          break;
+        }
         default:
           LOG(WARNING) << "Ignoring unsupported attribute type with key '"
                        << key << "'";
diff --git a/tensorflow/lite/toco/tflite/operator_test.cc b/tensorflow/lite/toco/tflite/operator_test.cc
index 215eda82f6..f77780488a 100644
--- a/tensorflow/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/lite/toco/tflite/operator_test.cc
@@ -569,6 +569,12 @@ TEST_F(OperatorTest, TensorFlowUnsupported) {
   (*attr)["str_attr"].set_s("Hello World");
   (*attr)["int_attr"].set_i(17);
   (*attr)["bool_attr"].set_b(true);
+  {
+    auto* list = (*attr)["list_float_attr"].mutable_list();
+    list->add_f(std::numeric_limits<float>::min());
+    list->add_f(2.0);
+    list->add_f(-std::numeric_limits<float>::max());
+  }
   {
     auto* list = (*attr)["list_int_attr"].mutable_list();
     list->add_i(1);
@@ -588,7 +594,13 @@ TEST_F(OperatorTest, TensorFlowUnsupported) {
   EXPECT_EQ("Hello World", output_attr.at("str_attr").s());
   EXPECT_EQ(17, output_attr.at("int_attr").i());
   EXPECT_EQ(true, output_attr.at("bool_attr").b());
-
+  {
+    const auto& list = output_attr.at("list_float_attr").list();
+    ASSERT_EQ(3, list.f_size());
+    EXPECT_EQ(std::numeric_limits<float>::min(), list.f(0));
+    EXPECT_EQ(2.0, list.f(1));
+    EXPECT_EQ(-std::numeric_limits<float>::max(), list.f(2));
+  }
   {
     const auto& list = output_attr.at("list_int_attr").list();
     ASSERT_EQ(4, list.i_size());
-- 
GitLab


From 6d5147395469d35d695dbd9d2498ad36bcfcca46 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 11:25:34 -0800
Subject: [PATCH 0802/2345] Support previously added 'truncate' attribute on
 floating datatype casts to apply in tf2xla as well.

PiperOrigin-RevId: 229593179
---
 tensorflow/compiler/tests/randomized_tests.cc | 34 +++++++++++++-
 tensorflow/compiler/tf2xla/kernels/cast_op.cc | 47 +++++++++++++++++++
 tensorflow/compiler/xla/primitive_util.cc     | 16 +++++++
 tensorflow/compiler/xla/primitive_util.h      |  4 ++
 4 files changed, 99 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/tests/randomized_tests.cc b/tensorflow/compiler/tests/randomized_tests.cc
index 735014fc89..1521cc760b 100644
--- a/tensorflow/compiler/tests/randomized_tests.cc
+++ b/tensorflow/compiler/tests/randomized_tests.cc
@@ -63,6 +63,7 @@ limitations under the License.
 #include "tensorflow/core/graph/graph.h"
 #include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/kernels/ops_util.h"
+#include "tensorflow/core/lib/bfloat16/bfloat16.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/platform/test.h"
@@ -769,8 +770,22 @@ Status TensorsAreEqualImpl(const Tensor& x, const Tensor& y) {
   for (int i = 0; i < Tx.size(); ++i) {
     if (Tx(i) != Ty(i)) {
       return errors::InvalidArgument(absl::StrCat(
-          i, "-th tensor element isn't equal: ", Tx(i), " vs. ", Ty(i),
-          ". x = ", x.DebugString(), "y = ", y.DebugString()));
+          i, "-th tensor element isn't equal: ", Str(Tx(i)), " vs. ",
+          Str(Ty(i)), ". x = ", x.DebugString(), "y = ", y.DebugString()));
+    }
+  }
+  return Status::OK();
+}
+
+Status TensorsAreEqualImplBfloat16(const Tensor& x, const Tensor& y) {
+  auto Tx = x.flat<bfloat16>();
+  auto Ty = y.flat<bfloat16>();
+  for (int i = 0; i < Tx.size(); ++i) {
+    if (Tx(i) != Ty(i)) {
+      return errors::InvalidArgument(absl::StrCat(
+          i, "-th tensor element isn't equal: ", static_cast<float>(Tx(i)),
+          " vs. ", static_cast<float>(Ty(i)), ". x = ", x.DebugString(),
+          "y = ", y.DebugString()));
     }
   }
   return Status::OK();
@@ -806,6 +821,8 @@ Status TensorsAreClose(const Tensor& a, const Tensor& b, double atol,
       return TensorsAreEqualImpl<int64>(a, b);
     case DT_BOOL:
       return TensorsAreEqualImpl<bool>(a, b);
+    case DT_BFLOAT16:
+      return TensorsAreEqualImplBfloat16(a, b);
     default:
       LOG(FATAL) << "Unexpected type : " << DataTypeString(a.dtype());
   }
@@ -1380,6 +1397,19 @@ TEST_F(OpTest, Cast) {
   });
 }
 
+TEST_F(OpTest, CastBF16) {
+  Repeatedly([this]() {
+    DataType src_type, dst_type;
+    src_type = Choose<DataType>({DT_FLOAT});
+    dst_type = Choose<DataType>({DT_BFLOAT16});
+    return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Cast")
+                                             .RandomInput(src_type)
+                                             .Attr("SrcT", src_type)
+                                             .Attr("DstT", dst_type)
+                                             .Attr("Truncate", true));
+  });
+}
+
 TEST_F(OpTest, Ceil) {
   Repeatedly([this]() {
     return ExpectTfAndXlaOutputsAreClose(
diff --git a/tensorflow/compiler/tf2xla/kernels/cast_op.cc b/tensorflow/compiler/tf2xla/kernels/cast_op.cc
index ce8131eeb4..db58c2e651 100644
--- a/tensorflow/compiler/tf2xla/kernels/cast_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/cast_op.cc
@@ -12,6 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
+#include "tensorflow/compiler/tf2xla/lib/util.h"
 
 #include "tensorflow/compiler/tf2xla/type_util.h"
 #include "tensorflow/compiler/tf2xla/xla_helpers.h"
@@ -19,6 +20,7 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/primitive_util.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
 
 namespace tensorflow {
@@ -31,6 +33,20 @@ class CastOp : public XlaOpKernel {
     OP_REQUIRES_OK(ctx, ctx->GetAttr("DstT", &dst_dtype_));
     OP_REQUIRES_OK(ctx, DataTypeToPrimitiveType(src_dtype_, &src_type_));
     OP_REQUIRES_OK(ctx, DataTypeToPrimitiveType(dst_dtype_, &dst_type_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("Truncate", &use_truncation_));
+  }
+
+  xla::PrimitiveType GetUnsignedIntTypeOfSameWidth(int64 src_bitwidth) {
+    switch (src_bitwidth) {
+      case 16:
+        return xla::U16;
+      case 32:
+        return xla::U32;
+      case 64:
+        return xla::U64;
+      default:
+        return xla::PRIMITIVE_TYPE_INVALID;
+    }
   }
 
   void Compile(XlaOpKernelContext* ctx) override {
@@ -48,6 +64,36 @@ class CastOp : public XlaOpKernel {
       // imaginary part.
       output = xla::ConvertElementType(xla::Real(input), dst_type_);
     } else {
+      if (use_truncation_) {
+        OP_REQUIRES(
+            ctx,
+            xla::primitive_util::IsFloatingPointType(src_type_) &&
+                xla::primitive_util::IsFloatingPointType(dst_type_),
+            errors::Unimplemented("Truncate attribute is only "
+                                  "implemented for floating point datatypes."));
+        int mantissa_difference =
+            xla::primitive_util::SignificandWidth(src_type_) -
+            xla::primitive_util::SignificandWidth(dst_type_);
+        OP_REQUIRES(ctx, mantissa_difference > 0,
+                    errors::Unimplemented(
+                        "Truncate attribute is only implemented in cases where "
+                        "dst datatype "
+                        "has fewer mantissa bits than the src datatype"));
+        int src_bitwidth = xla::primitive_util::BitWidth(src_type_);
+
+        // Bitcast to same-width integer, mask off the LSBs, bitcast back to the
+        // source datatype.
+        int64 mask = ~((1L << mantissa_difference) - 1);
+        xla::PrimitiveType same_width_int =
+            GetUnsignedIntTypeOfSameWidth(src_bitwidth);
+        OP_REQUIRES(ctx, same_width_int != xla::PRIMITIVE_TYPE_INVALID,
+                    errors::Unimplemented("Unexpected type bitwidth"));
+        input = xla::BitcastConvertType(
+            xla::And(
+                xla::BitcastConvertType(input, same_width_int),
+                ::tensorflow::IntegerLiteral(builder, same_width_int, mask)),
+            src_type_);
+      }
       output = xla::ConvertElementType(input, dst_type_);
     }
 
@@ -57,6 +103,7 @@ class CastOp : public XlaOpKernel {
  protected:
   DataType src_dtype_, dst_dtype_;
   xla::PrimitiveType src_type_, dst_type_;
+  bool use_truncation_;
 
   TF_DISALLOW_COPY_AND_ASSIGN(CastOp);
 };
diff --git a/tensorflow/compiler/xla/primitive_util.cc b/tensorflow/compiler/xla/primitive_util.cc
index 04b40efbed..3386d2e097 100644
--- a/tensorflow/compiler/xla/primitive_util.cc
+++ b/tensorflow/compiler/xla/primitive_util.cc
@@ -18,11 +18,27 @@ limitations under the License.
 #include "absl/strings/ascii.h"
 #include "absl/strings/numbers.h"
 #include "tensorflow/compiler/xla/util.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/platform/logging.h"
 
 namespace xla {
 namespace primitive_util {
 
+int SignificandWidth(PrimitiveType type) {
+  switch (type) {
+    case F32:
+      return std::numeric_limits<float>::digits;
+    case F64:
+      return std::numeric_limits<double>::digits;
+    case BF16:
+      return kBFloat16MantissaBits + 1;
+    case F16:
+      return 11;
+    default:
+      LOG(FATAL) << "Not a floating data type " << type;
+  }
+}
+
 bool IsFloatingPointType(PrimitiveType type) {
   return type == F16 || type == F32 || type == F64 || type == BF16;
 }
diff --git a/tensorflow/compiler/xla/primitive_util.h b/tensorflow/compiler/xla/primitive_util.h
index 739a7c409c..d32505335d 100644
--- a/tensorflow/compiler/xla/primitive_util.h
+++ b/tensorflow/compiler/xla/primitive_util.h
@@ -29,6 +29,10 @@ limitations under the License.
 namespace xla {
 namespace primitive_util {
 
+// Returns the count of significand (mantissa) bits for float datatypes.
+// For non-float datatypes, results in a LOG(FATAL).
+int SignificandWidth(PrimitiveType type);
+
 // The number of exponent bits in a BF16 value.
 const int kBFloat16ExponentBits = 8;
 
-- 
GitLab


From d80c2aa144b092937ffd1f019cfe461ea668839f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 11:29:39 -0800
Subject: [PATCH 0803/2345] - Fixed multiple broken links in the
 best_practices.md guide as a result of the recent move out of contrib

PiperOrigin-RevId: 229594101
---
 .../lite/g3doc/performance/best_practices.md  | 34 +++++++++++++++----
 1 file changed, 27 insertions(+), 7 deletions(-)

diff --git a/tensorflow/lite/g3doc/performance/best_practices.md b/tensorflow/lite/g3doc/performance/best_practices.md
index 090f19e810..5f41a70275 100644
--- a/tensorflow/lite/g3doc/performance/best_practices.md
+++ b/tensorflow/lite/g3doc/performance/best_practices.md
@@ -1,6 +1,9 @@
 # Performance best practices
 
-Mobile and embedded devices have limited computational resources and it is important to keep your application resource efficient. We have compiled a list of best practices and strategies you can use to optimize your model and application when using Tensorflow Lite.
+Mobile and embedded devices have limited computational resources and it is
+important to keep your application resource efficient. We have compiled a list
+of best practices and strategies that you can use to optimize your model and
+application when using TensorFlow Lite.
 
 ## Choose the best model for the task
 Depending on the task you will need to make a tradeoff between model complexity and size. If your task requires high accuracy then you may need a large and complex model. Some tasks may work with a less precise model, for these tasks it is better to use a smaller but less precise model. Smaller models not only use less disk space and memory but are generally faster and more energy efficient. For example, graphs below show accuracy and latency tradeoff for some common image classification models.
@@ -10,7 +13,7 @@ Depending on the task you will need to make a tradeoff between model complexity
 
 ![latency vs model size](../images/performance/model_size_vs_latency.png "Latency vs Model size")
 
-One example of models optimized for mobile devices are [MobileNets](https://arxiv.org/abs/1704.04861), which are optimized for mobile vision applications. Tensorflow Lite [models page](../models.md) lists several other models that have been optimized specifically for mobile and embedded devices.
+One example of models optimized for mobile devices are [MobileNets](https://arxiv.org/abs/1704.04861), which are optimized for mobile vision applications. TensorFlow Lite [models page](../models.md) lists several other models that have been optimized specifically for mobile and embedded devices.
 
 You can retrain the listed models on your own dataset by using transfer learning. Check out our transfer learning tutorial for
 [image classification](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets/#0) and
@@ -18,20 +21,37 @@ You can retrain the listed models on your own dataset by using transfer learning
 
 
 ## Profile your model
-Once you have selected a candidate model that is right for your task, it is a good practice to profile and benchmark your model. Tensorflow Lite [benchmarking tool](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark) has a built-in profiler that shows per operator profiling statistics. This can help in understanding performance bottlenecks and which operators dominate the computation time.
+Once you have selected a candidate model that is right for your task, it is a good practice to profile and benchmark your model. TensorFlow Lite [benchmarking tool](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark) has a built-in profiler that shows per operator profiling statistics. This can help in understanding performance bottlenecks and which operators dominate the computation time.
 
 ## Profile and optimize operators in the graph
 If a particular operator appears frequently in the model and based on profiling you find the operator consuming the most amount of time, you can look into optimizing the operator.
- This scenario should be rare as Tensorflow Lite has optimized versions for most ops. However you may be able to write a faster version of a custom op, if you know the constraints in which the operator is executed. Check out our [custom operator documentation](../custom_operators.md).
+ This scenario should be rare as TensorFlow Lite has optimized versions for most ops. However you may be able to write a faster version of a custom op, if you know the constraints in which the operator is executed. Check out our [custom operator documentation](../custom_operators.md).
 
 ## Quantize your model
 If your model uses floating point weights or activations then it may be possible to reduce the size of model up to ~4x by using quantization and other model optimizations. Check out our [model optimization toolkit](model_optimization.md) for details about optimizing your model. 
 
 ## Tweak the number of threads
-Tensorflow Lite supports multi-threaded kernels for many operators. You can increase the number of threads and speed up execution of operators. Increasing the number of threads will however make your model use more resources and power. For some applications latency may be more important than energy efficiency. You can increase the number of threads by setting the number of [interpreter](https://github.com/tensorflow/tensorflow/blob/1084594657a5d139102ac794f84d1427a710e39a/tensorflow/lite/interpreter.h#L337) threads. Multi-threaded execution however comes at the cost of increased performance variability depending on what else is been executed concurrently. This is particularly the case for mobile apps. For example, isolated tests may show 2x speed up vs single-threaded but if another app is executing at the same time may result in worst performance than single-threaded.
+
+TensorFlow Lite supports multi-threaded kernels for many operators. You can
+increase the number of threads and speed up execution of operators. Increasing
+the number of threads will however make your model use more resources and power.
+For some applications latency may be more important than energy efficiency. You
+can increase the number of threads by setting the number of
+[interpreter](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/interpreter.h#L333)
+threads. Multi-threaded execution however comes at the cost of increased
+performance variability depending on what else is been executed concurrently.
+This is particularly the case for mobile apps. For example, isolated tests may
+show 2x speed up vs single-threaded but if another app is executing at the same
+time may result in worst performance than single-threaded.
 
 ## Eliminate redundant copies
-If your application is not careful, there can be redundant copies when feeding the input to the model and reading output from the model. Make sure to eliminate redundant copies. If you are using higher level APIs like Java API, make sure to carefully check the documentation for performance caveats. For example, the Java API is a lot faster if ByteBuffers are used as [inputs](https://github.com/tensorflow/tensorflow/blob/6305a6d83552ba6a472cd72398b60d9241467f1f/tensorflow/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java#L151).
+
+If your application is not careful, there can be redundant copies when feeding
+the input to the model and reading output from the model. Make sure to eliminate
+redundant copies. If you are using higher level APIs like Java API, make sure to
+carefully check the documentation for performance caveats. For example, the Java
+API is a lot faster if ByteBuffers are used as
+[inputs](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java#L175).
 
 ## Profile your application with platform specific tools
 Platform specific tools like [Android profiler](https://developer.android.com/studio/profile/android-profiler) and [Instruments](https://help.apple.com/instruments/mac/current/) provide a wealth of profiling information that can be used to debug your app. Sometimes the performance bug may be not in the model but in parts of application code that interact with the model. Make sure to familiarize yourself with platform specific profiling tools and best practices for your platform.
@@ -64,7 +84,7 @@ are a great choice for large models that have high arithmetic intensity.
 
 ## Need more help
 
-The Tensorflow team is happy to help diagnose and address specific performance
+The TensorFlow team is happy to help diagnose and address specific performance
 issues you may be facing. Please file an issue on
 [GitHub](https://github.com/tensorflow/tensorflow/issues) with details of the
 issue.
-- 
GitLab


From 43d9f971759be4a0a683bfdbaa646025552e8f1e Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Wed, 16 Jan 2019 11:38:46 -0800
Subject: [PATCH 0804/2345] Fix more TFLite tests and binaries

Use the tf_cc_binary where appropriate. Also fix an issue
with schema generation and blacklist lstm zip tests temporarily.

PiperOrigin-RevId: 229596167
---
 tensorflow/lite/build_def.bzl         | 2 +-
 tensorflow/lite/schema/BUILD          | 1 -
 tensorflow/lite/testing/BUILD         | 5 +++--
 tensorflow/lite/tools/benchmark/BUILD | 3 ++-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index 03b217c120..3c48b655f7 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -263,7 +263,7 @@ def generated_test_models():
         "logical_and",
         "logical_or",
         "logical_xor",
-        "lstm",
+        #"lstm", TODO(b/122889684): Resolve toco structured line parsing in oss.
         "max_pool",
         "maximum",
         "mean",
diff --git a/tensorflow/lite/schema/BUILD b/tensorflow/lite/schema/BUILD
index 69d5458c6e..ea516764c9 100644
--- a/tensorflow/lite/schema/BUILD
+++ b/tensorflow/lite/schema/BUILD
@@ -70,7 +70,6 @@ flatbuffer_cc_library(
         "--no-union-value-namespacing",
         "--gen-object-api",
     ],
-    gen_reflections = True,
     out_prefix = "reflection/",
 )
 
diff --git a/tensorflow/lite/testing/BUILD b/tensorflow/lite/testing/BUILD
index 19c950b4f8..ce85a39385 100644
--- a/tensorflow/lite/testing/BUILD
+++ b/tensorflow/lite/testing/BUILD
@@ -12,6 +12,7 @@ load(
 load("//tensorflow/lite:special_rules.bzl", "tflite_portable_test_suite")
 load(
     "//tensorflow:tensorflow.bzl",
+    "tf_cc_binary",
     "tf_cc_test",
     "py_test",
 )
@@ -230,7 +231,7 @@ cc_test(
     ],
 )
 
-cc_binary(
+tf_cc_binary(
     name = "nnapi_example",
     srcs = ["nnapi_example.cc"],
     deps = [
@@ -380,7 +381,7 @@ tf_cc_test(
     ],
 )
 
-cc_binary(
+tf_cc_binary(
     name = "tflite_diff",
     srcs = ["tflite_diff_example_test.cc"],
     deps = [
diff --git a/tensorflow/lite/tools/benchmark/BUILD b/tensorflow/lite/tools/benchmark/BUILD
index bc47406cd9..e68997a00b 100644
--- a/tensorflow/lite/tools/benchmark/BUILD
+++ b/tensorflow/lite/tools/benchmark/BUILD
@@ -4,6 +4,7 @@ package(default_visibility = [
 
 licenses(["notice"])  # Apache 2.0
 
+load("//tensorflow:tensorflow.bzl", "tf_cc_binary")
 load("//tensorflow/lite:special_rules.bzl", "tflite_portable_test_suite")
 load("//tensorflow/lite:build_def.bzl", "tflite_copts")
 load("//tensorflow/lite:build_def.bzl", "tflite_linkopts")
@@ -35,7 +36,7 @@ cc_binary(
     ],
 )
 
-cc_binary(
+tf_cc_binary(
     name = "benchmark_model_plus_flex",
     srcs = [
         "benchmark_plus_flex_main.cc",
-- 
GitLab


From e388dfcde40d44d3c6bf5ee6835b823ede73bb7a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 11:56:47 -0800
Subject: [PATCH 0805/2345] Export BUILD file so it is visible at
 //third_party/tensorflow/core:BUILD

PiperOrigin-RevId: 229599457
---
 tensorflow/core/BUILD | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 7ba9afb5a8..6c9afc51e9 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -70,6 +70,9 @@ package(default_visibility = [
 
 licenses(["notice"])  # Apache 2.0
 
+# Export the BUILD file so automated tooling can check licenses
+exports_files(["BUILD"])
+
 load(
     "//tensorflow:tensorflow.bzl",
     "cc_header_only_library",
-- 
GitLab


From 45a95587b973cd0f82ab240b038b94157d015fc3 Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Wed, 16 Jan 2019 11:57:21 -0800
Subject: [PATCH 0806/2345] Don't rematerialize copy nodes.

Copy insertion adds copies to prevent buffer clobbering. Rematerialize those copies could reintroduce the clobbering. This CL adds copies into the blacklist of rematerialization pass.

PiperOrigin-RevId: 229599559
---
 .../xla/service/hlo_rematerialization.cc      |  9 ++++
 .../xla/service/hlo_rematerialization_test.cc | 46 +++++++++++++++++++
 2 files changed, 55 insertions(+)

diff --git a/tensorflow/compiler/xla/service/hlo_rematerialization.cc b/tensorflow/compiler/xla/service/hlo_rematerialization.cc
index 9ca14ca18a..5c1793933c 100644
--- a/tensorflow/compiler/xla/service/hlo_rematerialization.cc
+++ b/tensorflow/compiler/xla/service/hlo_rematerialization.cc
@@ -57,6 +57,15 @@ using ::tensorflow::strings::HumanReadableNumBytes;
 
 // Returns true if the given instruction is rematerializable.
 bool IsRematerializable(const HloInstruction* instruction) {
+  if (instruction->opcode() == HloOpcode::kCopy) {
+    if (LayoutUtil::Equal(instruction->shape().layout(),
+                          instruction->operand(0)->shape().layout())) {
+      // Don't rematerialize copies added by copy insertion (layout doesn't
+      // change).
+      return false;
+    }
+  }
+
   // Don't rematerialize instructions with side effects or instructions which
   // cannot be cloned safely.
   switch (instruction->opcode()) {
diff --git a/tensorflow/compiler/xla/service/hlo_rematerialization_test.cc b/tensorflow/compiler/xla/service/hlo_rematerialization_test.cc
index c4bb475619..102a360ad8 100644
--- a/tensorflow/compiler/xla/service/hlo_rematerialization_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_rematerialization_test.cc
@@ -499,6 +499,52 @@ TEST_F(HloRematerializationTest, InstructionRematerializedMultipleTimes) {
   EXPECT_THAT(add_4->operand(0), op::Broadcast(param));
 }
 
+TEST_F(HloRematerializationTest, CopyNotRematerialized) {
+  // Test that copies are not rematerialized.
+  auto module = CreateNewVerifiedModule();
+
+  auto builder = HloComputation::Builder(TestName());
+  auto param = builder.AddInstruction(
+      HloInstruction::CreateParameter(0, vec1024_shape_, "param"));
+
+  auto copy = builder.AddInstruction(
+      HloInstruction::CreateUnary(vec1024_shape_, HloOpcode::kCopy, param));
+
+  auto negate_a_1 = builder.AddInstruction(
+      HloInstruction::CreateUnary(vec1024_shape_, HloOpcode::kNegate, copy));
+
+  auto negate_a_2 = builder.AddInstruction(HloInstruction::CreateUnary(
+      vec1024_shape_, HloOpcode::kNegate, negate_a_1));
+
+  auto negate_b_1 = builder.AddInstruction(
+      HloInstruction::CreateUnary(vec1024_shape_, HloOpcode::kNegate, copy));
+
+  auto negate_b_2 = builder.AddInstruction(HloInstruction::CreateUnary(
+      vec1024_shape_, HloOpcode::kNegate, negate_b_1));
+
+  builder.AddInstruction(HloInstruction::CreateTuple({negate_a_2, negate_b_2}));
+
+  HloComputation* entry_computation =
+      module->AddEntryComputation(builder.Build());
+
+  TF_ASSERT_OK_AND_ASSIGN(bool changed,
+                          RunHloRematerialization(
+                              /*memory_limit_bytes=*/1 * 1024, module.get()));
+
+  auto count_copies = [](const HloComputation* computation) {
+    int64 copy_count = 0;
+    for (auto* instruction : computation->instructions()) {
+      if (instruction->opcode() == HloOpcode::kCopy) {
+        copy_count++;
+      }
+    }
+    return copy_count;
+  };
+  EXPECT_TRUE(changed);
+
+  EXPECT_EQ(count_copies(entry_computation), 1);
+}
+
 class IndirectUseTest : public HloRematerializationTest,
                         public ::testing::WithParamInterface<bool> {};
 
-- 
GitLab


From ee5d6e6deaac2b129bd9ac0c5ac3a82bb4bde8b8 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Wed, 16 Jan 2019 11:58:08 -0800
Subject: [PATCH 0807/2345] Tag a bunch of tests oss_serial to address spurious
 nightly GPU failures

Still not entirely sure what changed to make these not work when run together, but the CUDA upgrade seems likely. They pass individually with test_output=streamed.

PiperOrigin-RevId: 229599721
---
 tensorflow/contrib/eager/python/examples/densenet/BUILD     | 2 ++
 tensorflow/contrib/eager/python/examples/l2hmc/BUILD        | 3 +++
 .../contrib/eager/python/examples/linear_regression/BUILD   | 5 ++++-
 tensorflow/contrib/eager/python/examples/resnet50/BUILD     | 2 ++
 tensorflow/contrib/eager/python/examples/revnet/BUILD       | 4 ++++
 tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD | 3 +++
 tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD      | 3 +++
 tensorflow/contrib/eager/python/examples/spinn/BUILD        | 1 +
 tensorflow/contrib/sparsemax/BUILD                          | 6 ++++++
 9 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/eager/python/examples/densenet/BUILD b/tensorflow/contrib/eager/python/examples/densenet/BUILD
index e2154fcc5f..56a682ec55 100644
--- a/tensorflow/contrib/eager/python/examples/densenet/BUILD
+++ b/tensorflow/contrib/eager/python/examples/densenet/BUILD
@@ -27,6 +27,7 @@ cuda_py_test(
     tags = [
         "no_pip",
         "optonly",
+        "oss_serial",
     ],
 )
 
@@ -45,5 +46,6 @@ cuda_py_test(
         "nomsan",
         "notsan",
         "optonly",
+        "oss_serial",
     ],
 )
diff --git a/tensorflow/contrib/eager/python/examples/l2hmc/BUILD b/tensorflow/contrib/eager/python/examples/l2hmc/BUILD
index 7bdf9053de..78548c51c9 100644
--- a/tensorflow/contrib/eager/python/examples/l2hmc/BUILD
+++ b/tensorflow/contrib/eager/python/examples/l2hmc/BUILD
@@ -36,4 +36,7 @@ cuda_py_test(
         "//tensorflow/contrib/eager/python:tfe",
         "//third_party/py/numpy",
     ],
+    tags = [
+        "oss_serial",
+    ],
 )
diff --git a/tensorflow/contrib/eager/python/examples/linear_regression/BUILD b/tensorflow/contrib/eager/python/examples/linear_regression/BUILD
index 74ce9e84f0..6a178ddcec 100644
--- a/tensorflow/contrib/eager/python/examples/linear_regression/BUILD
+++ b/tensorflow/contrib/eager/python/examples/linear_regression/BUILD
@@ -23,7 +23,10 @@ cuda_py_test(
         ":linear_regression",
         "//tensorflow:tensorflow_py",
     ],
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
+    tags = [
+        "no_windows",  # TODO: needs investigation on Windows
+        "oss_serial",
+    ],
 )
 
 cuda_py_test(
diff --git a/tensorflow/contrib/eager/python/examples/resnet50/BUILD b/tensorflow/contrib/eager/python/examples/resnet50/BUILD
index f3135a9668..04ac78a2d3 100644
--- a/tensorflow/contrib/eager/python/examples/resnet50/BUILD
+++ b/tensorflow/contrib/eager/python/examples/resnet50/BUILD
@@ -40,6 +40,7 @@ cuda_py_test(
         "nomsan",  # Fix b/118130911
         "notsan",  # Fix b/118130911
         "optonly",
+        "oss_serial",
     ],
 )
 
@@ -58,5 +59,6 @@ cuda_py_test(
         "nomsan",
         "notsan",
         "optonly",
+        "oss_serial",
     ],
 )
diff --git a/tensorflow/contrib/eager/python/examples/revnet/BUILD b/tensorflow/contrib/eager/python/examples/revnet/BUILD
index 4f0d46b1ba..0d85bf63ad 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/BUILD
+++ b/tensorflow/contrib/eager/python/examples/revnet/BUILD
@@ -73,6 +73,9 @@ cuda_py_test(
         ":ops",
         "//tensorflow:tensorflow_py",
     ],
+    tags = [
+        "oss_serial",
+    ],
 )
 
 cuda_py_test(
@@ -101,6 +104,7 @@ cuda_py_test(
     tags = [
         "no_pip",  # depends on blocks_test, which is not available in pip package
         "optonly",
+        "oss_serial",
     ],
 )
 
diff --git a/tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD b/tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD
index d500b632eb..576f60396e 100644
--- a/tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD
+++ b/tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD
@@ -25,4 +25,7 @@ cuda_py_test(
         "//tensorflow/contrib/eager/python:tfe",
         "//tensorflow:tensorflow_py",
     ],
+    tags = [
+        "oss_serial",
+    ],
 )
diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD b/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD
index 2cc2fcbfeb..f9bf82a7d8 100644
--- a/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD
+++ b/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD
@@ -35,4 +35,7 @@ cuda_py_test(
         "//third_party/py/numpy",
         "//tensorflow:tensorflow_py",
     ],
+    tags = [
+        "oss_serial",
+    ],
 )
diff --git a/tensorflow/contrib/eager/python/examples/spinn/BUILD b/tensorflow/contrib/eager/python/examples/spinn/BUILD
index 5966f1d487..9b0fbaa679 100644
--- a/tensorflow/contrib/eager/python/examples/spinn/BUILD
+++ b/tensorflow/contrib/eager/python/examples/spinn/BUILD
@@ -42,5 +42,6 @@ cuda_py_test(
         "no-internal-py3",  # flaky
         "no_cuda_on_cpu_tap",
         "no_pip",  # because spinn.py is under third_party/.
+        "oss_serial",
     ],
 )
diff --git a/tensorflow/contrib/sparsemax/BUILD b/tensorflow/contrib/sparsemax/BUILD
index d7ba754f70..ed4eca1a60 100644
--- a/tensorflow/contrib/sparsemax/BUILD
+++ b/tensorflow/contrib/sparsemax/BUILD
@@ -49,6 +49,9 @@ cuda_py_tests(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform_test",
     ],
+    tags = [
+        "oss_serial",
+    ],
 )
 
 cuda_py_tests(
@@ -64,4 +67,7 @@ cuda_py_tests(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform_test",
     ],
+    tags = [
+        "oss_serial",
+    ],
 )
-- 
GitLab


From 8a5ddef315a5cd96e4e129bbd2b8ff1fa9f23ef6 Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne <reedwm@google.com>
Date: Wed, 16 Jan 2019 12:00:48 -0800
Subject: [PATCH 0808/2345] Fix nn_fused_batchnorm_test.

Also set test size to medium, becuase it takes significantly less than five
minutes to run.

PiperOrigin-RevId: 229600169
---
 tensorflow/python/BUILD                          |  2 +-
 tensorflow/python/ops/nn_fused_batchnorm_test.py | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index adba0b49fa..e32be0039f 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3516,7 +3516,7 @@ cuda_py_test(
 
 cuda_py_test(
     name = "nn_fused_batchnorm_test",
-    size = "large",
+    size = "medium",
     srcs = ["ops/nn_fused_batchnorm_test.py"],
     additional_deps = [
         ":array_ops",
diff --git a/tensorflow/python/ops/nn_fused_batchnorm_test.py b/tensorflow/python/ops/nn_fused_batchnorm_test.py
index 4e5cff5c16..69e753aa95 100644
--- a/tensorflow/python/ops/nn_fused_batchnorm_test.py
+++ b/tensorflow/python/ops/nn_fused_batchnorm_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import gradients_impl
@@ -436,6 +437,7 @@ class BatchNormalizationTest(test.TestCase):
       self._test_training(
           x_shape, dtype, [6], np.float32, use_gpu=False, data_format='NHWC')
 
+  @test_util.run_deprecated_v1
   def testBatchNormGradShape1(self):
     for is_training in [True, False]:
       x_shape = [1, 1, 6, 1]
@@ -463,6 +465,7 @@ class BatchNormalizationTest(test.TestCase):
             data_format='NHWC',
             is_training=is_training)
 
+  @test_util.run_deprecated_v1
   def testBatchNormGradShape2(self):
     for is_training in [True, False]:
       x_shape = [1, 1, 6, 2]
@@ -483,6 +486,7 @@ class BatchNormalizationTest(test.TestCase):
             data_format='NHWC',
             is_training=is_training)
 
+  @test_util.run_deprecated_v1
   def testBatchNormGradShape3(self):
     for is_training in [True, False]:
       x_shape = [1, 2, 1, 6]
@@ -496,6 +500,7 @@ class BatchNormalizationTest(test.TestCase):
               data_format='NCHW',
               is_training=is_training)
 
+  @test_util.run_deprecated_v1
   def testBatchNormGradShape4(self):
     for is_training in [True, False]:
       x_shape = [5, 7, 11, 4]
@@ -523,6 +528,7 @@ class BatchNormalizationTest(test.TestCase):
             data_format='NHWC',
             is_training=is_training)
 
+  @test_util.run_deprecated_v1
   @test_util.disable_xla('This test never passed for XLA')
   def testBatchNormGradShape5(self):
     for is_training in [True, False]:
@@ -582,6 +588,7 @@ class BatchNormalizationTest(test.TestCase):
           is_training=is_training,
           err_tolerance=err_tolerance)
 
+  @test_util.run_deprecated_v1
   def testBatchNormGradGradConfig1(self):
     config = {
         'shape': [2, 3, 4, 5],
@@ -590,6 +597,7 @@ class BatchNormalizationTest(test.TestCase):
     }
     self._testBatchNormGradGrad(config)
 
+  @test_util.run_deprecated_v1
   def testBatchNormGradGradConfig2(self):
     config = {
         'shape': [2, 3, 2, 2],
@@ -598,6 +606,7 @@ class BatchNormalizationTest(test.TestCase):
     }
     self._testBatchNormGradGrad(config)
 
+  @test_util.run_deprecated_v1
   def testBatchNormGradGradConfig3(self):
     config = {
         'shape': [2, 3, 4, 5],
@@ -606,6 +615,7 @@ class BatchNormalizationTest(test.TestCase):
     }
     self._testBatchNormGradGrad(config)
 
+  @test_util.run_deprecated_v1
   def testBatchNormGradGradConfig4(self):
     config = {
         'shape': [2, 3, 2, 2],
-- 
GitLab


From d6f6485eaee272fa4bd72c9d4625c2520f699e7f Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Wed, 16 Jan 2019 12:08:53 -0800
Subject: [PATCH 0809/2345] [Grappler] Improve MutableGraphView tests by
 checking graph and node connectivity with GraphView, and checking node fanins
 and fanouts explicitly.

PiperOrigin-RevId: 229601739
---
 tensorflow/core/grappler/BUILD                |    1 +
 .../core/grappler/mutable_graph_view_test.cc  | 1215 ++++++++---------
 2 files changed, 569 insertions(+), 647 deletions(-)

diff --git a/tensorflow/core/grappler/BUILD b/tensorflow/core/grappler/BUILD
index 475fa64201..bd95e89878 100644
--- a/tensorflow/core/grappler/BUILD
+++ b/tensorflow/core/grappler/BUILD
@@ -233,5 +233,6 @@ tf_cc_test(
         "//tensorflow/core:testlib",
         "//tensorflow/core/grappler/inputs:trivial_test_graph_input_yielder",
         "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:span",
     ],
 )
diff --git a/tensorflow/core/grappler/mutable_graph_view_test.cc b/tensorflow/core/grappler/mutable_graph_view_test.cc
index 597bf8a7ee..b5bf33033c 100644
--- a/tensorflow/core/grappler/mutable_graph_view_test.cc
+++ b/tensorflow/core/grappler/mutable_graph_view_test.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/mutable_graph_view.h"
 #include "absl/strings/substitute.h"
+#include "absl/types/span.h"
 #include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/core/framework/function_testlib.h"
 #include "tensorflow/core/framework/types.pb.h"
@@ -29,6 +30,115 @@ namespace grappler {
 namespace {
 
 using ::tensorflow::test::function::NDef;
+using FDH = FunctionDefHelper;
+
+void CompareNodeFanins(const MutableGraphView& graph, NodeDef* node,
+                       absl::Span<const string> fanins) {
+  ASSERT_EQ(node->input_size(), fanins.size());
+  for (int i = 0; i < node->input_size(); ++i) {
+    TensorId tensor_id = ParseTensorName(fanins[i]);
+    EXPECT_EQ(ParseTensorName(node->input(i)), tensor_id);
+    int port;
+    if (tensor_id.index() == Graph::kControlSlot) {
+      port = Graph::kControlSlot;
+    } else {
+      port = i;
+    }
+    MutableGraphView::InputPort input_port(node, port);
+    MutableGraphView::OutputPort output_port =
+        graph.GetOutputPort(tensor_id.node(), tensor_id.index());
+    EXPECT_TRUE(graph.GetFanin(input_port).contains(output_port));
+    EXPECT_TRUE(graph.GetFanout(output_port).contains(input_port));
+  }
+}
+
+void CompareNodeFanouts(const MutableGraphView& graph, NodeDef* node,
+                        absl::Span<const string> fanouts) {
+  auto node_fanouts =
+      graph.GetFanouts(*node, /*include_controlled_nodes=*/true);
+  EXPECT_EQ(node_fanouts.size(), fanouts.size());
+  for (const string& fanout : fanouts) {
+    TensorId tensor_id = ParseTensorName(fanout);
+    MutableGraphView::InputPort input_port(graph.GetNode(tensor_id.node()),
+                                           tensor_id.index());
+    EXPECT_TRUE(node_fanouts.contains(input_port));
+  }
+}
+
+void CheckNode(const MutableGraphView& graph, absl::string_view node_name,
+               absl::string_view op, absl::string_view device,
+               absl::Span<const std::pair<string, FDH::AttrValueWrapper>> attrs,
+               absl::Span<const string> fanins,
+               absl::Span<const string> fanouts) {
+  NodeDef* node = graph.GetNode(node_name);
+  ASSERT_NE(node, nullptr);
+  EXPECT_EQ(node->op(), op);
+  EXPECT_EQ(node->device(), device);
+  EXPECT_EQ(node->attr_size(), attrs.size());
+  for (const auto& attr : attrs) {
+    auto it = node->attr().find(attr.first);
+    ASSERT_NE(it, node->attr().end());
+    EXPECT_TRUE(AreAttrValuesEqual(it->second, attr.second.proto));
+  }
+  CompareNodeFanins(graph, node, fanins);
+  CompareNodeFanouts(graph, node, fanouts);
+}
+
+void CheckGraph(const MutableGraphView& mutable_graph) {
+  GraphView immutable_graph(mutable_graph.graph());
+  EXPECT_EQ(mutable_graph.graph()->node_size(),
+            immutable_graph.graph()->node_size());
+  EXPECT_EQ(mutable_graph.graph(), immutable_graph.graph());
+
+  auto check_edges =
+      [](const absl::flat_hash_set<MutableGraphView::Edge>& mutable_edges,
+         const absl::flat_hash_set<GraphView::Edge>& immutable_edges) {
+        EXPECT_EQ(mutable_edges.size(), immutable_edges.size());
+        for (const auto& fanin_edge : mutable_edges) {
+          GraphView::Edge immutable_edge(
+              {fanin_edge.src.node, fanin_edge.src.port_id},
+              {fanin_edge.dst.node, fanin_edge.dst.port_id});
+          EXPECT_TRUE(immutable_edges.contains(immutable_edge));
+        }
+      };
+
+  // Check graph connectivity.
+  for (auto& node : *mutable_graph.graph()->mutable_node()) {
+    EXPECT_EQ(&node, immutable_graph.GetNode(node.name()));
+
+    auto mutable_fanins =
+        mutable_graph.GetFanins(node, /*include_controlling_nodes=*/true);
+    auto immutable_fanins =
+        immutable_graph.GetFanins(node, /*include_controlling_nodes=*/true);
+    EXPECT_EQ(mutable_fanins.size(), immutable_fanins.size());
+    for (const auto& fanin : mutable_fanins) {
+      GraphView::OutputPort immutable_fanin(fanin.node, fanin.port_id);
+      EXPECT_TRUE(immutable_fanins.contains(immutable_fanin));
+    }
+
+    auto mutable_fanouts =
+        mutable_graph.GetFanouts(node, /*include_controlled_nodes=*/true);
+    auto immutable_fanouts =
+        immutable_graph.GetFanouts(node, /*include_controlled_nodes=*/true);
+    EXPECT_EQ(mutable_fanouts.size(), immutable_fanouts.size());
+    for (const auto& fanout : mutable_fanouts) {
+      GraphView::InputPort immutable_fanout(fanout.node, fanout.port_id);
+      EXPECT_TRUE(immutable_fanouts.contains(immutable_fanout));
+    }
+
+    auto mutable_fanin_edges =
+        mutable_graph.GetFaninEdges(node, /*include_controlling_edges=*/true);
+    auto immutable_fanin_edges =
+        immutable_graph.GetFaninEdges(node, /*include_controlling_edges=*/true);
+    check_edges(mutable_fanin_edges, immutable_fanin_edges);
+
+    auto mutable_fanout_edges =
+        mutable_graph.GetFanoutEdges(node, /*include_controlled_edges=*/true);
+    auto immutable_fanout_edges =
+        immutable_graph.GetFanoutEdges(node, /*include_controlled_edges=*/true);
+    check_edges(mutable_fanout_edges, immutable_fanout_edges);
+  }
+}
 
 TEST(MutableGraphViewTest, AddAndUpdateFanouts) {
   // Actual node.op() is not important in this test.
@@ -43,42 +153,23 @@ TEST(MutableGraphViewTest, AddAndUpdateFanouts) {
   MutableGraphView graph(&graph_def);
 
   NodeDef* new_bar = graph.AddNode(NDef("new_bar", "NotImportant", {}, {}));
-  NodeDef* bar = graph.GetNode("bar");
-
-  EXPECT_TRUE(graph.UpdateFanouts(bar->name(), new_bar->name()).ok());
-
-  // Fanout nodes must have their inputs updated.
-  NodeDef* foo_1 = graph.GetNode("foo_1");
-  ASSERT_NE(foo_1, nullptr);
-  ASSERT_EQ(foo_1->input_size(), 3);
-  EXPECT_EQ(foo_1->input(0), "new_bar");
-  EXPECT_EQ(foo_1->input(1), "other");
-  EXPECT_EQ(foo_1->input(2), "new_bar:1");
-
-  NodeDef* foo_2 = graph.GetNode("foo_2");
-  ASSERT_NE(foo_2, nullptr);
-  ASSERT_EQ(foo_2->input_size(), 2);
-  EXPECT_EQ(foo_2->input(0), "other:1");
-  EXPECT_EQ(foo_2->input(1), "new_bar:2");
-
-  NodeDef* foo_3 = graph.GetNode("foo_3");
-  ASSERT_NE(foo_3, nullptr);
-  ASSERT_EQ(foo_3->input_size(), 2);
-  EXPECT_EQ(foo_3->input(0), "other:2");
-  EXPECT_EQ(foo_3->input(1), "^new_bar");
-
-  // And fanouts mapping must be also updated for both nodes.
-  bool include_control_fanouts = true;
-  auto old_node_fanouts = graph.GetFanouts(*bar, include_control_fanouts);
-  auto new_node_fanouts = graph.GetFanouts(*new_bar, include_control_fanouts);
-
-  EXPECT_TRUE(old_node_fanouts.empty());
-
-  EXPECT_EQ(new_node_fanouts.size(), 4);
-  EXPECT_EQ(new_node_fanouts.count(MutableGraphView::InputPort(foo_1, 0)), 1);
-  EXPECT_EQ(new_node_fanouts.count(MutableGraphView::InputPort(foo_1, 2)), 1);
-  EXPECT_EQ(new_node_fanouts.count(MutableGraphView::InputPort(foo_2, 1)), 1);
-  EXPECT_EQ(new_node_fanouts.count(MutableGraphView::InputPort(foo_3, -1)), 1);
+
+  EXPECT_TRUE(graph.UpdateFanouts("bar", new_bar->name()).ok());
+
+  // Fanins and fanouts must be updated.
+  CheckNode(graph, "bar", "NotImportant", "", {}, {}, {});
+  CheckNode(graph, "other", "NotImportant", "", {}, {},
+            {"foo_1:1", "foo_2", "foo_3"});
+  CheckNode(graph, "foo_1", "NotImportant", "", {},
+            {"new_bar", "other", "new_bar:1"}, {});
+  CheckNode(graph, "foo_2", "NotImportant", "", {}, {"other:1", "new_bar:2"},
+            {});
+  CheckNode(graph, "foo_3", "NotImportant", "", {}, {"other:2", "^new_bar"},
+            {});
+  CheckNode(graph, "new_bar", "NotImportant", "", {}, {},
+            {"foo_1:0", "foo_1:2", "foo_2:1", "^foo_3"});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, AddAndUpdateFanoutsKeepControls) {
@@ -92,37 +183,21 @@ TEST(MutableGraphViewTest, AddAndUpdateFanoutsKeepControls) {
   MutableGraphView graph(&graph_def);
 
   NodeDef* new_bar = graph.AddNode(NDef("new_bar", "Identity", {"bar_1:2"}));
-  NodeDef* bar_2 = graph.GetNode("bar_2");
-
-  EXPECT_TRUE(graph.UpdateFanouts(bar_2->name(), new_bar->name()).ok());
-
-  // Fanout nodes must have their inputs updated.
-  NodeDef* foo_1 = graph.GetNode("foo_1");
-  ASSERT_NE(foo_1, nullptr);
-  ASSERT_EQ(foo_1->input_size(), 4);
-  EXPECT_EQ(foo_1->input(0), "new_bar");
-  EXPECT_EQ(foo_1->input(1), "other");
-  EXPECT_EQ(foo_1->input(2), "new_bar:1");
-  EXPECT_EQ(foo_1->input(3), "^new_bar");
-
-  NodeDef* foo_2 = graph.GetNode("foo_2");
-  ASSERT_NE(foo_2, nullptr);
-  ASSERT_EQ(foo_2->input_size(), 3);
-  EXPECT_EQ(foo_2->input(0), "other:1");
-  EXPECT_EQ(foo_2->input(1), "new_bar:2");
-  EXPECT_EQ(foo_2->input(2), "^new_bar");
-
-  // And fanouts mapping must be also updated for both nodes.
-  bool include_control_fanouts = true;
-  auto old_node_fanouts = graph.GetFanouts(*bar_2, include_control_fanouts);
-  auto new_node_fanouts = graph.GetFanouts(*new_bar, include_control_fanouts);
-
-  EXPECT_TRUE(old_node_fanouts.empty());
-  EXPECT_EQ(new_node_fanouts.count(MutableGraphView::InputPort(foo_1, 0)), 1);
-  EXPECT_EQ(new_node_fanouts.count(MutableGraphView::InputPort(foo_1, 2)), 1);
-  EXPECT_EQ(new_node_fanouts.count(MutableGraphView::InputPort(foo_1, -1)), 1);
-  EXPECT_EQ(new_node_fanouts.count(MutableGraphView::InputPort(foo_2, 1)), 1);
-  EXPECT_EQ(new_node_fanouts.count(MutableGraphView::InputPort(foo_2, -1)), 1);
+
+  EXPECT_TRUE(graph.UpdateFanouts("bar_2", new_bar->name()).ok());
+
+  // Fanins and fanouts must be updated.
+  CheckNode(graph, "bar_1", "Switch", "", {}, {}, {"bar_2", "new_bar"});
+  CheckNode(graph, "bar_2", "Identity", "", {}, {"bar_1:1"}, {});
+  CheckNode(graph, "other", "NotImportant", "", {}, {}, {"foo_1:1", "foo_2"});
+  CheckNode(graph, "foo_1", "NotImportant", "", {},
+            {"new_bar", "other", "new_bar:1", "^new_bar"}, {});
+  CheckNode(graph, "foo_2", "NotImportant", "", {},
+            {"other:1", "new_bar:2", "^new_bar"}, {});
+  CheckNode(graph, "new_bar", "Identity", "", {}, {"bar_1:2"},
+            {"foo_1", "foo_1:2", "^foo_1", "foo_2:1", "^foo_2"});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, AddAndUpdateFanoutsWithoutSelfLoops) {
@@ -137,56 +212,16 @@ TEST(MutableGraphViewTest, AddAndUpdateFanoutsWithoutSelfLoops) {
 
   // `new_bar` reads the output of an original `bar` node.
   NodeDef* new_bar = graph.AddNode(NDef("new_bar", "NewBar", {"bar"}, {}));
-  NodeDef* bar = graph.GetNode("bar");
 
   EXPECT_TRUE(graph.UpdateFanouts("bar", new_bar->name()).ok());
 
-  // Foo node must read from `new_bar`.
-  NodeDef* foo_1 = graph.GetNode("foo_1");
-  ASSERT_NE(foo_1, nullptr);
-  ASSERT_EQ(foo_1->input_size(), 1);
-  EXPECT_EQ(foo_1->input(0), "new_bar");
-
-  NodeDef* foo_2 = graph.GetNode("foo_2");
-  ASSERT_NE(foo_2, nullptr);
-  ASSERT_EQ(foo_2->input_size(), 1);
-  EXPECT_EQ(foo_2->input(0), "^new_bar");
-
-  // And the `new_bar` should read from the original `bar`.
-  ASSERT_EQ(new_bar->input_size(), 1);
-  ASSERT_EQ(new_bar->input(0), "bar");
-
-  // And fanouts mapping must be also updated for both nodes.
-  bool include_control_fanouts = true;
-  auto bar_fanouts = graph.GetFanouts(*bar, include_control_fanouts);
-  auto new_bar_fanouts = graph.GetFanouts(*new_bar, include_control_fanouts);
+  // Fanins and fanouts must be updated.
+  CheckNode(graph, "bar", "NotImportant", "", {}, {}, {"new_bar"});
+  CheckNode(graph, "foo_1", "NotImportant", "", {}, {"new_bar"}, {});
+  CheckNode(graph, "foo_2", "NotImportant", "", {}, {"^new_bar"}, {});
+  CheckNode(graph, "new_bar", "NewBar", "", {}, {"bar"}, {"foo_1", "^foo_2"});
 
-  EXPECT_EQ(bar_fanouts.size(), 1);
-  EXPECT_EQ(bar_fanouts.count(MutableGraphView::InputPort(new_bar, 0)), 1);
-
-  EXPECT_EQ(new_bar_fanouts.size(), 2);
-  EXPECT_EQ(new_bar_fanouts.count(MutableGraphView::InputPort(foo_1, 0)), 1);
-  EXPECT_EQ(new_bar_fanouts.count(MutableGraphView::InputPort(foo_2, -1)), 1);
-}
-
-void CompareNodeInputs(const MutableGraphView& graph, const NodeDef* expected,
-                       NodeDef* actual) {
-  ASSERT_EQ(actual->input_size(), expected->input_size());
-  for (int i = 0; i < actual->input_size(); ++i) {
-    EXPECT_EQ(actual->input(i), expected->input(i));
-    TensorId tensor_id = ParseTensorName(expected->input(i));
-    int port;
-    if (tensor_id.index() == Graph::kControlSlot) {
-      port = Graph::kControlSlot;
-    } else {
-      port = i;
-    }
-    MutableGraphView::InputPort input_port(actual, port);
-    MutableGraphView::OutputPort output_port =
-        graph.GetOutputPort(tensor_id.node(), tensor_id.index());
-    EXPECT_EQ(graph.GetFanin(input_port).contains(output_port), true);
-    EXPECT_EQ(graph.GetFanout(output_port).contains(input_port), true);
-  }
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, UpdateFanoutsToSwitchWithControlFromSwitch) {
@@ -213,27 +248,13 @@ TEST(MutableGraphViewTest, UpdateFanoutsToSwitchWithControlFromSwitch) {
 
   EXPECT_EQ(graph.graph()->node_size(), 5);
 
-  NodeDef expected = NDef("", "", {}, {});
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-  CompareNodeInputs(graph, &expected, a);
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"^e"});
+  CheckNode(graph, "b", "Switch", "", {}, {}, {"e:1"});
+  CheckNode(graph, "c", "NotImportant", "", {}, {}, {"e:0"});
+  CheckNode(graph, "d", "NotImportant", "", {}, {}, {"^e"});
+  CheckNode(graph, "e", "NotImportant", "", {}, {"c", "b", "^a", "^d"}, {});
 
-  NodeDef* b = graph.GetNode("b");
-  ASSERT_NE(b, nullptr);
-  CompareNodeInputs(graph, &expected, b);
-
-  NodeDef* c = graph.GetNode("c");
-  ASSERT_NE(c, nullptr);
-  CompareNodeInputs(graph, &expected, c);
-
-  NodeDef* d = graph.GetNode("d");
-  ASSERT_NE(d, nullptr);
-  CompareNodeInputs(graph, &expected, d);
-
-  NodeDef* e = graph.GetNode("e");
-  ASSERT_NE(e, nullptr);
-  expected = NDef("", "", {"c", "b", "^a", "^d"});
-  CompareNodeInputs(graph, &expected, e);
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, UpdateFanoutsToSwitchWithNoControlFromSwitch) {
@@ -249,27 +270,13 @@ TEST(MutableGraphViewTest, UpdateFanoutsToSwitchWithNoControlFromSwitch) {
 
   EXPECT_EQ(graph.graph()->node_size(), 5);
 
-  NodeDef expected = NDef("", "", {}, {});
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-  CompareNodeInputs(graph, &expected, a);
-
-  NodeDef* b = graph.GetNode("b");
-  ASSERT_NE(b, nullptr);
-  CompareNodeInputs(graph, &expected, b);
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"^e"});
+  CheckNode(graph, "b", "Switch", "", {}, {}, {"e:0", "e:1"});
+  CheckNode(graph, "c", "NotImportant", "", {}, {}, {});
+  CheckNode(graph, "d", "NotImportant", "", {}, {}, {"^e"});
+  CheckNode(graph, "e", "NotImportant", "", {}, {"b", "b", "^a", "^d"}, {});
 
-  NodeDef* c = graph.GetNode("c");
-  ASSERT_NE(c, nullptr);
-  CompareNodeInputs(graph, &expected, c);
-
-  NodeDef* d = graph.GetNode("d");
-  ASSERT_NE(d, nullptr);
-  CompareNodeInputs(graph, &expected, d);
-
-  NodeDef* e = graph.GetNode("e");
-  ASSERT_NE(e, nullptr);
-  expected = NDef("", "", {"b", "b", "^a", "^d"});
-  CompareNodeInputs(graph, &expected, e);
+  CheckGraph(graph);
 }
 
 GraphDef SimpleMutateFaninGraph() {
@@ -287,129 +294,152 @@ GraphDef SimpleMutateFaninGraph() {
   return graph_def;
 }
 
-void TestAddRegularFanin(absl::string_view node_name,
+absl::flat_hash_map<string, std::vector<string>> GetNodeInputsFromGraph(
+    const GraphDef& graph, absl::string_view node_to_exclude) {
+  absl::flat_hash_map<string, std::vector<string>> node_inputs;
+  for (const auto& node : graph.node()) {
+    if (node.name() == node_to_exclude) {
+      continue;
+    }
+    node_inputs[node.name()] =
+        std::vector<string>(node.input().begin(), node.input().end());
+  }
+  return node_inputs;
+}
+
+void CheckUnmodifiedNodeFanins(
+    const GraphDef& graph, absl::string_view node_to_exclude,
+    const absl::flat_hash_map<string, std::vector<string>>&
+        unmodified_node_inputs) {
+  for (const auto& node : graph.node()) {
+    if (node.name() == node_to_exclude) {
+      continue;
+    }
+    auto it = unmodified_node_inputs.find(node.name());
+    ASSERT_NE(it, unmodified_node_inputs.end());
+    ASSERT_EQ(it->second.size(), node.input_size());
+    for (int i = 0; i < node.input_size(); ++i) {
+      EXPECT_EQ(node.input(i), it->second[i]);
+    }
+  }
+}
+
+void TestAddRegularFanin(absl::string_view node_name, bool node_exists,
                          const TensorId& fanin_to_add, bool success,
                          const string& error_msg,
-                         const NodeDef* expected_node) {
+                         absl::Span<const string> expected_fanins) {
   GraphDef graph_def = SimpleMutateFaninGraph();
 
   MutableGraphView graph(&graph_def);
 
-  auto node = graph.GetNode(node_name);
-  if (expected_node == nullptr) {
-    EXPECT_EQ(node, nullptr);
-  } else {
+  NodeDef* node = graph.GetNode(node_name);
+  if (node_exists) {
     EXPECT_NE(node, nullptr);
+  } else {
+    EXPECT_EQ(node, nullptr);
   }
 
+  absl::flat_hash_map<string, std::vector<string>> unmodified_node_inputs =
+      GetNodeInputsFromGraph(graph_def, node_name);
+
   Status s = graph.AddRegularFanin(node_name, fanin_to_add);
   EXPECT_EQ(s.ok(), success);
   if (!success) {
     EXPECT_EQ(s.error_message(), error_msg);
   }
-  if (expected_node != nullptr) {
-    CompareNodeInputs(graph, expected_node, node);
+  if (node_exists) {
+    CompareNodeFanins(graph, node, expected_fanins);
   }
+
+  CheckUnmodifiedNodeFanins(graph_def, node_name, unmodified_node_inputs);
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, AddRegularFanin) {
-  NodeDef expected_node;
   string error_msg;
   // Add input to node with 1 input 0 controls.
-  expected_node = NDef("", "", {"a", "b:1"});
-  TestAddRegularFanin("foo_1", {"b", 1}, /*success=*/true, error_msg,
-                      &expected_node);
+  TestAddRegularFanin("foo_1", /*node_exists=*/true, {"b", 1}, /*success=*/true,
+                      error_msg, {"a", "b:1"});
   // Add input to node with multiple inputs and 0 controls.
-  expected_node = NDef("", "", {"b", "a:1", "a:1", "b:2"});
-  TestAddRegularFanin("foo_3", {"b", 2}, /*success=*/true, error_msg,
-                      &expected_node);
+  TestAddRegularFanin("foo_3", /*node_exists=*/true, {"b", 2}, /*success=*/true,
+                      error_msg, {"b", "a:1", "a:1", "b:2"});
   // Add input to node with 1 input multiple controls.
-  expected_node = NDef("", "", {"b", "a", "^c"});
-  TestAddRegularFanin("foo_2", {"a", 0}, /*success=*/true, error_msg,
-                      &expected_node);
+  TestAddRegularFanin("foo_2", /*node_exists=*/true, {"a", 0}, /*success=*/true,
+                      error_msg, {"b", "a", "^c"});
   // Add input to node with multiple inputs and controls.
-  expected_node = NDef("", "", {"a", "b:2", "b:2", "a:1", "^d", "^c"});
-  TestAddRegularFanin("foo_4", {"a", 1}, /*success=*/true, error_msg,
-                      &expected_node);
+  TestAddRegularFanin("foo_4", /*node_exists=*/true, {"a", 1}, /*success=*/true,
+                      error_msg, {"a", "b:2", "b:2", "a:1", "^d", "^c"});
   // Add input to node with 0 inputs 0 controls.
-  expected_node = NDef("", "", {"a:1"});
-  TestAddRegularFanin("foo_5", {"a", 1}, /*success=*/true, error_msg,
-                      &expected_node);
+  TestAddRegularFanin("foo_5", /*node_exists=*/true, {"a", 1}, /*success=*/true,
+                      error_msg, {"a:1"});
   // Add input to node with 0 inputs multiple controls.
-  expected_node = NDef("", "", {"c:1", "^b", "^a"});
-  TestAddRegularFanin("foo_6", {"c", 1}, /*success=*/true, error_msg,
-                      &expected_node);
+  TestAddRegularFanin("foo_6", /*node_exists=*/true, {"c", 1}, /*success=*/true,
+                      error_msg, {"c:1", "^b", "^a"});
 
   // Add control to node with 1 input 0 controls.
-  expected_node = NDef("", "", {"a"});
   error_msg = "Can't add invalid fanin '^b' as regular fanin to node 'foo_1'.";
-  TestAddRegularFanin("foo_1", {"b", Graph::kControlSlot}, /*success=*/false,
-                      error_msg, &expected_node);
+  TestAddRegularFanin("foo_1", /*node_exists=*/true, {"b", Graph::kControlSlot},
+                      /*success=*/false, error_msg, {"a"});
   // Add control to node with multiple inputs and 0 controls.
-  expected_node = NDef("", "", {"b", "a:1", "a:1"});
   error_msg = "Can't add invalid fanin '^c' as regular fanin to node 'foo_3'.";
-  TestAddRegularFanin("foo_3", {"c", Graph::kControlSlot}, /*success=*/false,
-                      error_msg, &expected_node);
+  TestAddRegularFanin("foo_3", /*node_exists=*/true, {"c", Graph::kControlSlot},
+                      /*success=*/false, error_msg, {"b", "a:1", "a:1"});
   // Add control to node with 1 input multiple controls.
-  expected_node = NDef("", "", {"b", "^a", "^c"});
   error_msg = "Can't add invalid fanin '^d' as regular fanin to node 'foo_2'.";
-  TestAddRegularFanin("foo_2", {"d", Graph::kControlSlot}, /*success=*/false,
-                      error_msg, &expected_node);
+  TestAddRegularFanin("foo_2", /*node_exists=*/true, {"d", Graph::kControlSlot},
+                      /*success=*/false, error_msg, {"b", "^a", "^c"});
   // Add control to node with multiple input multiple controls.
-  expected_node = NDef("", "", {"a", "b:2", "b:2", "^c", "^d"});
   error_msg = "Can't add invalid fanin '^a' as regular fanin to node 'foo_4'.";
-  TestAddRegularFanin("foo_4", {"a", Graph::kControlSlot},
-                      /*success=*/false, error_msg, &expected_node);
+  TestAddRegularFanin("foo_4", /*node_exists=*/true, {"a", Graph::kControlSlot},
+                      /*success=*/false, error_msg,
+                      {"a", "b:2", "b:2", "^c", "^d"});
   // Add control to node with 0 inputs 0 controls.
-  expected_node = NDef("", "", {});
   error_msg = "Can't add invalid fanin '^a' as regular fanin to node 'foo_5'.";
-  TestAddRegularFanin("foo_5", {"a", Graph::kControlSlot}, /*success=*/false,
-                      error_msg, &expected_node);
+  TestAddRegularFanin("foo_5", /*node_exists=*/true, {"a", Graph::kControlSlot},
+                      /*success=*/false, error_msg, {});
   // Add control to node with 0 inputs multiple controls.
-  expected_node = NDef("", "", {"^a", "^b"});
   error_msg = "Can't add invalid fanin '^c' as regular fanin to node 'foo_6'.";
-  TestAddRegularFanin("foo_6", {"c", Graph::kControlSlot}, /*success=*/false,
-                      error_msg, &expected_node);
+  TestAddRegularFanin("foo_6", /*node_exists=*/true, {"c", Graph::kControlSlot},
+                      /*success=*/false, error_msg, {"^a", "^b"});
   // Add control to node with control that already exists.
-  expected_node = NDef("", "", {"b", "^a", "^c"});
   error_msg = "Can't add invalid fanin '^a' as regular fanin to node 'foo_2'.";
-  TestAddRegularFanin("foo_2", {"a", Graph::kControlSlot},
-                      /*success=*/false, error_msg, &expected_node);
+  TestAddRegularFanin("foo_2", /*node_exists=*/true, {"a", Graph::kControlSlot},
+                      /*success=*/false, error_msg, {"b", "^a", "^c"});
 
   // Add fanin to node where node is missing.
   error_msg =
       "Can't add fanin 'a:0' as regular fanin to missing node 'foo_missing'.";
-  TestAddRegularFanin("foo_missing", {"a", 0}, /*success=*/false, error_msg,
-                      nullptr);
+  TestAddRegularFanin("foo_missing", /*node_exists=*/false, {"a", 0},
+                      /*success=*/false, error_msg, {});
   // Add fanin to node where fanin is missing.
-  expected_node = NDef("", "", {"a"});
   error_msg =
       "Can't add missing fanin 'bar_missing:0' as regular fanin to node "
       "'foo_1'.";
-  TestAddRegularFanin("foo_1", {"bar_missing", 0}, /*success=*/false, error_msg,
-                      &expected_node);
+  TestAddRegularFanin("foo_1", /*node_exists=*/true, {"bar_missing", 0},
+                      /*success=*/false, error_msg, {"a"});
   // Add fanin to node where node and fanin are missing.
   error_msg =
       "Can't add missing fanin 'bar_missing:0' as regular fanin to missing "
       "node 'foo_missing'.";
-  TestAddRegularFanin("foo_missing", {"bar_missing", 0}, /*success=*/false,
-                      error_msg, /*expected_node=*/nullptr);
+  TestAddRegularFanin("foo_missing", /*node_exists=*/false, {"bar_missing", 0},
+                      /*success=*/false, error_msg, {});
   // Add control fanin to node where node and fanin are missing.
   error_msg =
       "Can't add invalid/missing fanin '^bar_missing' as regular fanin to "
       "missing node 'foo_missing'.";
-  TestAddRegularFanin("foo_missing", {"bar_missing", Graph::kControlSlot},
-                      /*success=*/false, error_msg, /*expected_node=*/nullptr);
+  TestAddRegularFanin("foo_missing", /*node_exists=*/false,
+                      {"bar_missing", Graph::kControlSlot},
+                      /*success=*/false, error_msg, {});
 
   // Add self to create cycle.
   error_msg = "Can't add fanin 'foo_6:2' as regular fanin to self.";
-  expected_node = NDef("", "", {"^a", "^b"});
-  TestAddRegularFanin("foo_6", {"foo_6", 2}, /*success=*/false, error_msg,
-                      &expected_node);
+  TestAddRegularFanin("foo_6", /*node_exists=*/true, {"foo_6", 2},
+                      /*success=*/false, error_msg, {"^a", "^b"});
 }
 
-void CheckFanout(const MutableGraphView& graph, const TensorId& fanin,
-                 absl::string_view node_name) {
+void CheckFanoutRemoved(const MutableGraphView& graph, const TensorId& fanin,
+                        absl::string_view node_name) {
   MutableGraphView::OutputPort output_port =
       graph.GetOutputPort(fanin.node(), fanin.index());
   auto fanouts = graph.GetFanout(output_port);
@@ -418,166 +448,166 @@ void CheckFanout(const MutableGraphView& graph, const TensorId& fanin,
   }
 }
 
-void TestRemoveRegularFanin(absl::string_view node_name,
+void TestRemoveRegularFanin(absl::string_view node_name, bool node_exists,
                             const TensorId& fanin_to_remove, bool success,
                             const string& error_msg,
-                            const NodeDef* expected_node) {
+                            absl::Span<const string> expected_fanins) {
   GraphDef graph_def = SimpleMutateFaninGraph();
 
   MutableGraphView graph(&graph_def);
 
-  auto node = graph.GetNode(node_name);
-  if (expected_node == nullptr) {
-    EXPECT_EQ(nullptr, node);
-  } else {
+  NodeDef* node = graph.GetNode(node_name);
+  if (node_exists) {
     EXPECT_NE(nullptr, node);
+  } else {
+    EXPECT_EQ(nullptr, node);
   }
 
+  absl::flat_hash_map<string, std::vector<string>> unmodified_node_inputs =
+      GetNodeInputsFromGraph(graph_def, node_name);
+
   Status s = graph.RemoveRegularFanin(node_name, fanin_to_remove);
   EXPECT_EQ(s.ok(), success);
   if (!success) {
     EXPECT_EQ(s.error_message(), error_msg);
   }
-  if (expected_node != nullptr) {
-    CompareNodeInputs(graph, expected_node, node);
+  if (node_exists) {
+    CompareNodeFanins(graph, node, expected_fanins);
     if (success) {
-      CheckFanout(graph, fanin_to_remove, node_name);
+      CheckFanoutRemoved(graph, fanin_to_remove, node_name);
     }
   }
+
+  CheckUnmodifiedNodeFanins(graph_def, node_name, unmodified_node_inputs);
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, RemoveRegularFanin) {
-  NodeDef expected_node;
   string error_msg;
   // Remove input from node with 1 input 0 controls.
-  expected_node = NDef("", "", {});
-  TestRemoveRegularFanin("foo_1", {"a", 0}, /*success=*/true, error_msg,
-                         &expected_node);
+  TestRemoveRegularFanin("foo_1", /*node_exists=*/true, {"a", 0},
+                         /*success=*/true, error_msg, {});
   // Remove input from node with multiple inputs and 0 controls.
-  expected_node = NDef("", "", {"b"});
-  TestRemoveRegularFanin("foo_3", {"a", 1}, /*success=*/true, error_msg,
-                         &expected_node);
+  TestRemoveRegularFanin("foo_3", /*node_exists=*/true, {"a", 1},
+                         /*success=*/true, error_msg, {"b"});
   // Remove input from node with 1 input multiple controls.
-  expected_node = NDef("", "", {"^a", "^c"});
-  TestRemoveRegularFanin("foo_2", {"b", 0}, /*success=*/true, error_msg,
-                         &expected_node);
+  TestRemoveRegularFanin("foo_2", /*node_exists=*/true, {"b", 0},
+                         /*success=*/true, error_msg, {"^a", "^c"});
   // Remove input from node with multiple inputs and controls.
-  expected_node = NDef("", "", {"a", "^c", "^d"});
-  TestRemoveRegularFanin("foo_4", {"b", 2}, /*success=*/true, error_msg,
-                         &expected_node);
+  TestRemoveRegularFanin("foo_4", /*node_exists=*/true, {"b", 2},
+                         /*success=*/true, error_msg, {"a", "^c", "^d"});
   // Remove input from node with multiple inputs and controls, and results in
   // shifting of ports.
-  expected_node = NDef("", "", {"b:2", "b:2", "^c", "^d"});
-  TestRemoveRegularFanin("foo_4", {"a", 0}, /*success=*/true, error_msg,
-                         &expected_node);
+  TestRemoveRegularFanin("foo_4", /*node_exists=*/true, {"a", 0},
+                         /*success=*/true, error_msg,
+                         {"b:2", "b:2", "^c", "^d"});
 
   // Remove control from node with 1 input multiple controls.
-  expected_node = NDef("", "", {"b", "^a", "^c"});
   error_msg =
       "Can't remove invalid fanin '^a' as regular fanin from node 'foo_2'.";
-  TestRemoveRegularFanin("foo_2", {"a", Graph::kControlSlot},
-                         /*success=*/false, error_msg, &expected_node);
+  TestRemoveRegularFanin("foo_2", /*node_exists=*/true,
+                         {"a", Graph::kControlSlot},
+                         /*success=*/false, error_msg, {"b", "^a", "^c"});
   // Remove control from node with multiple input multiple controls.
-  expected_node = NDef("", "", {"a", "b:2", "b:2", "^c", "^d"});
   error_msg =
       "Can't remove invalid fanin '^d' as regular fanin from node 'foo_4'.";
-  TestRemoveRegularFanin("foo_4", {"d", Graph::kControlSlot},
-                         /*success=*/false, error_msg, &expected_node);
+  TestRemoveRegularFanin(
+      "foo_4", /*node_exists=*/true, {"d", Graph::kControlSlot},
+      /*success=*/false, error_msg, {"a", "b:2", "b:2", "^c", "^d"});
   // Remove control from node with 0 inputs multiple controls.
-  expected_node = NDef("", "", {"^a", "^b"});
   error_msg =
       "Can't remove invalid fanin '^a' as regular fanin from node 'foo_6'.";
-  TestRemoveRegularFanin("foo_6", {"a", Graph::kControlSlot},
-                         /*success=*/false, error_msg, &expected_node);
+  TestRemoveRegularFanin("foo_6", /*node_exists=*/true,
+                         {"a", Graph::kControlSlot},
+                         /*success=*/false, error_msg, {"^a", "^b"});
 
   // Remove input from node with 0 inputs 0 controls.
-  expected_node = NDef("", "", {});
   error_msg = "";
-  TestRemoveRegularFanin("foo_5", {"a", 1}, /*success=*/true, error_msg,
-                         &expected_node);
+  TestRemoveRegularFanin("foo_5", /*node_exists=*/true, {"a", 1},
+                         /*success=*/true, error_msg, {});
   // Remove input from node with 0 inputs multiple controls.
-  expected_node = NDef("", "", {"^a", "^b"});
-  TestRemoveRegularFanin("foo_6", {"a", 1}, /*success=*/true, error_msg,
-                         &expected_node);
+  TestRemoveRegularFanin("foo_6", /*node_exists=*/true, {"a", 1},
+                         /*success=*/true, error_msg, {"^a", "^b"});
 
   // Remove control from node with 1 input 0 controls.
-  expected_node = NDef("", "", {"a"});
   error_msg =
       "Can't remove invalid fanin '^b' as regular fanin from node 'foo_1'.";
-  TestRemoveRegularFanin("foo_1", {"b", Graph::kControlSlot},
-                         /*success=*/false, error_msg, &expected_node);
+  TestRemoveRegularFanin("foo_1", /*node_exists=*/true,
+                         {"b", Graph::kControlSlot},
+                         /*success=*/false, error_msg, {"a"});
   // Remove control from node with multiple inputs and 0 controls.
-  expected_node = NDef("", "", {"b", "a:1", "a:1"});
   error_msg =
       "Can't remove invalid fanin '^c' as regular fanin from node 'foo_3'.";
-  TestRemoveRegularFanin("foo_3", {"c", Graph::kControlSlot},
-                         /*success=*/false, error_msg, &expected_node);
+  TestRemoveRegularFanin("foo_3", /*node_exists=*/true,
+                         {"c", Graph::kControlSlot},
+                         /*success=*/false, error_msg, {"b", "a:1", "a:1"});
   // Remove control from node with 0 inputs 0 controls.
-  expected_node = NDef("", "", {});
   error_msg =
       "Can't remove invalid fanin '^a' as regular fanin from node 'foo_5'.";
-  TestRemoveRegularFanin("foo_5", {"a", Graph::kControlSlot},
-                         /*success=*/false, error_msg, &expected_node);
+  TestRemoveRegularFanin("foo_5", /*node_exists=*/true,
+                         {"a", Graph::kControlSlot},
+                         /*success=*/false, error_msg, {});
 
   // Remove fanin from node where node is missing.
   error_msg =
       "Can't remove fanin 'a:0' as regular fanin from missing node "
       "'foo_missing'.";
-  TestRemoveRegularFanin("foo_missing", {"a", 0}, /*success=*/false, error_msg,
-                         /*expected_node=*/nullptr);
+  TestRemoveRegularFanin("foo_missing", /*node_exists=*/false, {"a", 0},
+                         /*success=*/false, error_msg, {});
   // Remove fanin from node where fanin is missing.
-  expected_node = NDef("", "", {"a"});
   error_msg =
       "Can't remove missing fanin 'bar_missing:0' as regular fanin from node "
       "'foo_1'.";
-  TestRemoveRegularFanin("foo_1", {"bar_missing", 0}, /*success=*/false,
-                         error_msg, &expected_node);
+  TestRemoveRegularFanin("foo_1", /*node_exists=*/true, {"bar_missing", 0},
+                         /*success=*/false, error_msg, {"a"});
   // Remove fanin from node where node and fanin are missing.
   error_msg =
       "Can't remove missing fanin 'bar_missing:0' as regular fanin from "
       "missing node 'foo_missing'.";
-  TestRemoveRegularFanin("foo_missing", {"bar_missing", 0}, /*success=*/false,
-                         error_msg,
-                         /*expected_node=*/nullptr);
+  TestRemoveRegularFanin("foo_missing", /*node_exists=*/false,
+                         {"bar_missing", 0}, /*success=*/false, error_msg, {});
   // Remove control from node where node and fanin are missing.
   error_msg =
       "Can't remove invalid/missing fanin '^bar_missing' as regular fanin from "
       "missing node 'foo_missing'.";
-  TestRemoveRegularFanin("foo_missing", {"bar_missing", Graph::kControlSlot},
-                         /*success=*/false, error_msg,
-                         /*expected_node=*/nullptr);
+  TestRemoveRegularFanin("foo_missing", /*node_exists=*/false,
+                         {"bar_missing", Graph::kControlSlot},
+                         /*success=*/false, error_msg, {});
 
   // Remove self.
   error_msg = "Can't remove fanin 'foo_6:2' as regular fanin from self.";
-  expected_node = NDef("", "", {"^a", "^b"});
-  TestRemoveRegularFanin("foo_6", {"foo_6", 2}, /*success=*/false, error_msg,
-                         &expected_node);
+  TestRemoveRegularFanin("foo_6", /*node_exists=*/true, {"foo_6", 2},
+                         /*success=*/false, error_msg, {"^a", "^b"});
 }
 
-void TestRemoveAllFanins(absl::string_view node_name,
+void TestRemoveAllFanins(absl::string_view node_name, bool node_exists,
                          bool keep_controlling_nodes, bool success,
                          const string& error_msg,
-                         const NodeDef* expected_node) {
+                         absl::Span<const string> expected_fanins) {
   GraphDef graph_def = SimpleMutateFaninGraph();
 
   MutableGraphView graph(&graph_def);
 
-  auto node = graph.GetNode(node_name);
+  NodeDef* node = graph.GetNode(node_name);
   absl::flat_hash_set<string> fanin_strings;
-  if (expected_node == nullptr) {
-    EXPECT_EQ(node, nullptr);
-  } else {
+  if (node_exists) {
     EXPECT_NE(node, nullptr);
     fanin_strings.insert(node->input().begin(), node->input().end());
+  } else {
+    EXPECT_EQ(node, nullptr);
   }
 
+  absl::flat_hash_map<string, std::vector<string>> unmodified_node_inputs =
+      GetNodeInputsFromGraph(graph_def, node_name);
+
   Status s = graph.RemoveAllFanins(node_name, keep_controlling_nodes);
   EXPECT_EQ(s.ok(), success);
   if (!success) {
     EXPECT_EQ(s.error_message(), error_msg);
   }
-  if (expected_node != nullptr) {
-    CompareNodeInputs(graph, expected_node, node);
+  if (node_exists) {
+    CompareNodeFanins(graph, node, expected_fanins);
     if (success) {
       TensorId tensor_id;
       auto retained_inputs = absl::flat_hash_set<string>(node->input().begin(),
@@ -585,177 +615,192 @@ void TestRemoveAllFanins(absl::string_view node_name,
       for (const string& fanin : fanin_strings) {
         if (!retained_inputs.contains(fanin)) {
           tensor_id = ParseTensorName(fanin);
-          CheckFanout(graph, tensor_id, node_name);
+          CheckFanoutRemoved(graph, tensor_id, node_name);
         }
       }
     }
   }
+
+  CheckUnmodifiedNodeFanins(graph_def, node_name, unmodified_node_inputs);
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, RemoveAllFanins) {
-  NodeDef expected_node;
   string error_msg;
   // Remove all fanins from node with no control dependencies.
-  expected_node = NDef("", "", {});
-  TestRemoveAllFanins("foo_3", /*keep_controlling_nodes=*/false,
-                      /*success=*/true, error_msg, &expected_node);
+  TestRemoveAllFanins("foo_3", /*node_exists=*/true,
+                      /*keep_controlling_nodes=*/false,
+                      /*success=*/true, error_msg, {});
   // Remove all fanins from node with control dependencies.
-  TestRemoveAllFanins("foo_4", /*keep_controlling_nodes=*/false,
-                      /*success=*/true, error_msg, &expected_node);
+  TestRemoveAllFanins("foo_4", /*node_exists=*/true,
+                      /*keep_controlling_nodes=*/false,
+                      /*success=*/true, error_msg, {});
 
   // Remove all fanins from node with no control dependencies and preserve
   // control dependencies.
-  TestRemoveAllFanins("foo_3", /*keep_controlling_nodes=*/true,
-                      /*success=*/true, error_msg, &expected_node);
+  TestRemoveAllFanins("foo_3", /*node_exists=*/true,
+                      /*keep_controlling_nodes=*/true,
+                      /*success=*/true, error_msg, {});
   // Remove all fanins from node with control dependencies and preserve control
   // dependencies.
-  expected_node = NDef("", "", {"^c", "^d"});
-  TestRemoveAllFanins("foo_4", /*keep_controlling_nodes=*/true,
-                      /*success=*/true, error_msg, &expected_node);
+  TestRemoveAllFanins("foo_4", /*node_exists=*/true,
+                      /*keep_controlling_nodes=*/true,
+                      /*success=*/true, error_msg, {"^c", "^d"});
 
   // Remove all fanins from node with no fanins.
-  expected_node = NDef("", "", {});
-  TestRemoveAllFanins("foo_5", /*keep_controlling_nodes=*/false,
-                      /*success=*/true, error_msg, &expected_node);
-  TestRemoveAllFanins("foo_5", /*keep_controlling_nodes=*/true,
-                      /*success=*/true, error_msg, &expected_node);
+  TestRemoveAllFanins("foo_5", /*node_exists=*/true,
+                      /*keep_controlling_nodes=*/false,
+                      /*success=*/true, error_msg, {});
+  TestRemoveAllFanins("foo_5", /*node_exists=*/true,
+                      /*keep_controlling_nodes=*/true,
+                      /*success=*/true, error_msg, {});
 
   // Remove all fanins from node with only control dependencies.
-  TestRemoveAllFanins("foo_6", /*keep_controlling_nodes=*/false,
-                      /*success=*/true, error_msg, &expected_node);
-  expected_node = NDef("", "", {"^a", "^b"});
-  TestRemoveAllFanins("foo_6", /*keep_controlling_nodes=*/true,
-                      /*success=*/true, error_msg, &expected_node);
+  TestRemoveAllFanins("foo_6", /*node_exists=*/true,
+                      /*keep_controlling_nodes=*/false,
+                      /*success=*/true, error_msg, {});
+  TestRemoveAllFanins("foo_6", /*node_exists=*/true,
+                      /*keep_controlling_nodes=*/true,
+                      /*success=*/true, error_msg, {"^a", "^b"});
 
   // Remove all fanins from node where node is missing.
   error_msg = "Can't remove all fanins from missing node 'foo_missing'.";
-  TestRemoveAllFanins("foo_missing", /*keep_controlling_nodes=*/false,
-                      /*success=*/false, error_msg, /*expected_node=*/nullptr);
-  TestRemoveAllFanins("foo_missing", /*keep_controlling_nodes=*/true,
-                      /*success=*/false, error_msg, /*expected_node=*/nullptr);
+  TestRemoveAllFanins("foo_missing", /*node_exists=*/false,
+                      /*keep_controlling_nodes=*/false,
+                      /*success=*/false, error_msg, {});
+  TestRemoveAllFanins("foo_missing", /*node_exists=*/false,
+                      /*keep_controlling_nodes=*/true,
+                      /*success=*/false, error_msg, {});
 }
 
-void TestUpdateFanin(absl::string_view node_name, const TensorId& from_fanin,
-                     const TensorId& to_fanin, bool success,
-                     const string& error_msg, const NodeDef* expected_node) {
+void TestUpdateFanin(absl::string_view node_name, bool node_exists,
+                     const TensorId& from_fanin, const TensorId& to_fanin,
+                     bool success, const string& error_msg,
+                     absl::Span<const string> expected_fanins) {
   GraphDef graph_def = SimpleMutateFaninGraph();
 
   MutableGraphView graph(&graph_def);
 
-  auto node = graph.GetNode(node_name);
-  if (expected_node == nullptr) {
-    EXPECT_EQ(node, nullptr);
-  } else {
+  NodeDef* node = graph.GetNode(node_name);
+  if (node_exists) {
     EXPECT_NE(node, nullptr);
+  } else {
+    EXPECT_EQ(node, nullptr);
   }
 
+  absl::flat_hash_map<string, std::vector<string>> unmodified_node_inputs =
+      GetNodeInputsFromGraph(graph_def, node_name);
+
   Status s = graph.UpdateFanin(node_name, from_fanin, to_fanin);
   EXPECT_EQ(s.ok(), success);
   if (!success) {
     EXPECT_EQ(s.error_message(), error_msg);
   }
-  if (expected_node != nullptr) {
-    CompareNodeInputs(graph, expected_node, node);
+  if (node_exists) {
+    CompareNodeFanins(graph, node, expected_fanins);
     if (success) {
-      CheckFanout(graph, from_fanin, node_name);
+      CheckFanoutRemoved(graph, from_fanin, node_name);
     }
   }
+
+  CheckUnmodifiedNodeFanins(graph_def, node_name, unmodified_node_inputs);
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, UpdateFanin) {
-  NodeDef expected_node;
   string error_msg;
   // Update fanin from non control to non control.
-  expected_node = NDef("", "", {"a", "b:3", "b:3", "^c", "^d"});
-  TestUpdateFanin("foo_4", {"b", 2}, {"b", 3}, /*success=*/true, error_msg,
-                  &expected_node);
+  TestUpdateFanin("foo_4", /*node_exists=*/true, {"b", 2}, {"b", 3},
+                  /*success=*/true, error_msg, {"a", "b:3", "b:3", "^c", "^d"});
   // Update fanin from non control to control.
-  expected_node = NDef("", "", {"a", "^c", "^d", "^b"});
-  TestUpdateFanin("foo_4", {"b", 2}, {"b", Graph::kControlSlot},
-                  /*success=*/true, error_msg, &expected_node);
+  TestUpdateFanin("foo_4", /*node_exists=*/true, {"b", 2},
+                  {"b", Graph::kControlSlot},
+                  /*success=*/true, error_msg, {"a", "^c", "^d", "^b"});
   // Update fanin from control to non control.
-  expected_node = NDef("", "", {"a", "b:2", "b:2", "d:1", "^c"});
-  TestUpdateFanin("foo_4", {"d", Graph::kControlSlot}, {"d", 1},
-                  /*success=*/true, error_msg, &expected_node);
+  TestUpdateFanin(
+      "foo_4", /*node_exists=*/true, {"d", Graph::kControlSlot}, {"d", 1},
+      /*success=*/true, error_msg, {"a", "b:2", "b:2", "d:1", "^c"});
   // Update fanin from control to control.
-  expected_node = NDef("", "", {"a", "b:2", "b:2", "^d"});
-  TestUpdateFanin("foo_4", {"c", Graph::kControlSlot},
+  TestUpdateFanin("foo_4", /*node_exists=*/true, {"c", Graph::kControlSlot},
                   {"b", Graph::kControlSlot}, /*success=*/true, error_msg,
-                  &expected_node);
+                  {"a", "b:2", "b:2", "^d"});
   // Update fanin from control to existing control.
-  expected_node = NDef("", "", {"a", "b:2", "b:2", "^d"});
-  TestUpdateFanin("foo_4", {"c", Graph::kControlSlot},
+  TestUpdateFanin("foo_4", /*node_exists=*/true, {"c", Graph::kControlSlot},
                   {"d", Graph::kControlSlot}, /*success=*/true, error_msg,
-                  &expected_node);
+                  {"a", "b:2", "b:2", "^d"});
 
   // Update fanin of node where from and to fanins are the same.
-  expected_node = NDef("", "", {"a"});
-  TestUpdateFanin("foo_1", {"a", -1}, {"a", -1}, /*success=*/true, error_msg,
-                  &expected_node);
-  TestUpdateFanin("foo_1", {"a", 0}, {"a", 0}, /*success=*/true, error_msg,
-                  &expected_node);
-  TestUpdateFanin("foo_1", {"a", 1}, {"a", 1}, /*success=*/true, error_msg,
-                  &expected_node);
+  TestUpdateFanin("foo_1", /*node_exists=*/true, {"a", -1}, {"a", -1},
+                  /*success=*/true, error_msg, {"a"});
+  TestUpdateFanin("foo_1", /*node_exists=*/true, {"a", 0}, {"a", 0},
+                  /*success=*/true, error_msg, {"a"});
+  TestUpdateFanin("foo_1", /*node_exists=*/true, {"a", 1}, {"a", 1},
+                  /*success=*/true, error_msg, {"a"});
 
   // Update fanin of node where node is missing.
   error_msg =
       "Can't update fanin 'a:0' to fanin 'a:1' in missing node 'foo_missing'.";
-  TestUpdateFanin("foo_missing", {"a", 0}, {"a", 1}, /*success=*/false,
-                  error_msg,
-                  /*expected_node=*/nullptr);
+  TestUpdateFanin("foo_missing", /*node_exists=*/false, {"a", 0}, {"a", 1},
+                  /*success=*/false, error_msg, {});
   // Update fanin of node where from fanin is missing.
   error_msg =
       "Can't update missing fanin 'from_bar_missing:0' to fanin 'a:1' in node "
       "'foo_1'.";
-  TestUpdateFanin("foo_1", {"from_bar_missing", 0}, {"a", 1},
-                  /*success=*/false, error_msg, &expected_node);
+  TestUpdateFanin("foo_1", /*node_exists=*/true, {"from_bar_missing", 0},
+                  {"a", 1},
+                  /*success=*/false, error_msg, {"a"});
   // Update fanin of node where to fanin is missing.
   error_msg =
       "Can't update fanin 'a:0' to missing fanin 'to_bar_missing:1' in node "
       "'foo_1'.";
-  TestUpdateFanin("foo_1", {"a", 0}, {"to_bar_missing", 1}, /*success=*/false,
-                  error_msg, &expected_node);
+  TestUpdateFanin("foo_1", /*node_exists=*/true, {"a", 0},
+                  {"to_bar_missing", 1}, /*success=*/false, error_msg, {"a"});
   // Update fanin of node where from/to fanins and node are missing.
   error_msg =
       "Can't update missing fanin 'from_bar_missing:0' to missing fanin "
       "'to_bar_missing:1' in missing node 'foo_missing'.";
-  TestUpdateFanin("foo_missing", {"from_bar_missing", 0}, {"to_bar_missing", 1},
-                  /*success=*/false, error_msg, /*expected_node=*/nullptr);
+  TestUpdateFanin("foo_missing", /*node_exists=*/false, {"from_bar_missing", 0},
+                  {"to_bar_missing", 1},
+                  /*success=*/false, error_msg, {});
   // Update fanin of node where from fanin is invalid.
   error_msg =
       "Can't update invalid fanin 'a:-2' to fanin 'a:0' in node 'foo_1'.";
-  TestUpdateFanin("foo_1", {"a", -2}, {"a", 0},
-                  /*success=*/false, error_msg, &expected_node);
+  TestUpdateFanin("foo_1", /*node_exists=*/true, {"a", -2}, {"a", 0},
+                  /*success=*/false, error_msg, {"a"});
   // Update fanin of node where to fanin is invalid.
   error_msg =
       "Can't update fanin 'a:0' to invalid fanin 'a:-2' in node 'foo_1'.";
-  TestUpdateFanin("foo_1", {"a", 0}, {"a", -2},
-                  /*success=*/false, error_msg, &expected_node);
+  TestUpdateFanin("foo_1", /*node_exists=*/true, {"a", 0}, {"a", -2},
+                  /*success=*/false, error_msg, {"a"});
   // Update fanin of node where from/to fanins are invalid and missing and node
   // is missing.
   error_msg =
       "Can't update invalid/missing fanin 'from_bar_missing:-2' to "
       "invalid/missing fanin 'to_bar_missing:-3' in missing node "
       "'foo_missing'.";
-  TestUpdateFanin("foo_missing", {"from_bar_missing", -2},
-                  {"to_bar_missing", -3},
-                  /*success=*/false, error_msg, /*expected_node=*/nullptr);
+  TestUpdateFanin("foo_missing", /*node_exists=*/false,
+                  {"from_bar_missing", -2}, {"to_bar_missing", -3},
+                  /*success=*/false, error_msg, {});
 
   // Update to self to create cycle.
-  expected_node = NDef("", "", {"a", "b:2", "b:2", "^c", "^d"});
   error_msg = "Can't update fanin 'b:2' to fanin 'foo_4:3' in self 'foo_4'.";
-  TestUpdateFanin("foo_4", {"b", 2}, {"foo_4", 3}, /*success=*/false, error_msg,
-                  &expected_node);
+  TestUpdateFanin("foo_4", /*node_exists=*/true, {"b", 2}, {"foo_4", 3},
+                  /*success=*/false, error_msg,
+                  {"a", "b:2", "b:2", "^c", "^d"});
   error_msg = "Can't update fanin 'b:2' to fanin '^foo_4' in self 'foo_4'.";
-  TestUpdateFanin("foo_4", {"b", 2}, {"foo_4", Graph::kControlSlot},
-                  /*success=*/false, error_msg, &expected_node);
+  TestUpdateFanin(
+      "foo_4", /*node_exists=*/true, {"b", 2}, {"foo_4", Graph::kControlSlot},
+      /*success=*/false, error_msg, {"a", "b:2", "b:2", "^c", "^d"});
   error_msg = "Can't update fanin '^c' to fanin 'foo_4:4' in self 'foo_4'.";
-  TestUpdateFanin("foo_4", {"c", Graph::kControlSlot}, {"foo_4", 4},
-                  /*success=*/false, error_msg, &expected_node);
+  TestUpdateFanin(
+      "foo_4", /*node_exists=*/true, {"c", Graph::kControlSlot}, {"foo_4", 4},
+      /*success=*/false, error_msg, {"a", "b:2", "b:2", "^c", "^d"});
   error_msg = "Can't update fanin '^c' to fanin '^foo_4' in self 'foo_4'.";
-  TestUpdateFanin("foo_4", {"c", Graph::kControlSlot},
+  TestUpdateFanin("foo_4", /*node_exists=*/true, {"c", Graph::kControlSlot},
                   {"foo_4", Graph::kControlSlot}, /*success=*/false, error_msg,
-                  &expected_node);
+                  {"a", "b:2", "b:2", "^c", "^d"});
 }
 
 void TestUpdateFaninFromFaninToNodeAsSwitchControl(const TensorId& fanin) {
@@ -777,20 +822,12 @@ void TestUpdateFaninFromFaninToNodeAsSwitchControl(const TensorId& fanin) {
 
   EXPECT_EQ(graph.graph()->node_size(), 3);
 
-  NodeDef expected;
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-  expected = NDef("", "", {});
-  CompareNodeInputs(graph, &expected, a);
-
-  NodeDef* b = graph.GetNode("b");
-  ASSERT_NE(b, nullptr);
-  CompareNodeInputs(graph, &expected, b);
+  string fanout = IsControlInput(fanin) ? AsControlDependency("c") : "c";
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {fanout});
+  CheckNode(graph, "b", "Switch", "", {}, {}, {});
+  CheckNode(graph, "c", "NotImportant", "", {}, {tensor_id_str}, {});
 
-  NodeDef* c = graph.GetNode("c");
-  ASSERT_NE(c, nullptr);
-  expected = NDef("", "", {tensor_id_str});
-  CompareNodeInputs(graph, &expected, c);
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, UpdateFaninToNodeAsSwitchControl) {
@@ -816,59 +853,24 @@ TEST(MutableGraphViewTest, DedupControllingFaninsOnGraphInit) {
   MutableGraphView graph(&graph_def);
 
   EXPECT_EQ(graph.graph()->node_size(), 11);
-  NodeDef expected;
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-  expected = NDef("", "", {});
-  CompareNodeInputs(graph, &expected, a);
-
-  NodeDef* b = graph.GetNode("b");
-  ASSERT_NE(b, nullptr);
-  CompareNodeInputs(graph, &expected, b);
-
-  NodeDef* c = graph.GetNode("c");
-  ASSERT_NE(c, nullptr);
-  CompareNodeInputs(graph, &expected, c);
-
-  NodeDef* d = graph.GetNode("d");
-  ASSERT_NE(d, nullptr);
-  expected = NDef("", "", {"c:1"});
-  CompareNodeInputs(graph, &expected, d);
-
-  NodeDef* foo_1 = graph.GetNode("foo_1");
-  ASSERT_NE(foo_1, nullptr);
-  expected = NDef("", "", {"a", "b:1"});
-  CompareNodeInputs(graph, &expected, foo_1);
-
-  NodeDef* foo_2 = graph.GetNode("foo_2");
-  ASSERT_NE(foo_2, nullptr);
-  expected = NDef("", "", {"a", "^b"});
-  CompareNodeInputs(graph, &expected, foo_2);
-
-  NodeDef* foo_3 = graph.GetNode("foo_3");
-  ASSERT_NE(foo_3, nullptr);
-  expected = NDef("", "", {"a", "b:1"});
-  CompareNodeInputs(graph, &expected, foo_3);
-
-  NodeDef* foo_4 = graph.GetNode("foo_4");
-  ASSERT_NE(foo_4, nullptr);
-  expected = NDef("", "", {"a:2", "b:1"});
-  CompareNodeInputs(graph, &expected, foo_4);
-
-  NodeDef* foo_5 = graph.GetNode("foo_5");
-  ASSERT_NE(foo_5, nullptr);
-  expected = NDef("", "", {"a:2", "b:1"});
-  CompareNodeInputs(graph, &expected, foo_5);
-
-  NodeDef* foo_6 = graph.GetNode("foo_6");
-  ASSERT_NE(foo_6, nullptr);
-  expected = NDef("", "", {"d", "^d"});
-  CompareNodeInputs(graph, &expected, foo_6);
-
-  NodeDef* foo_7 = graph.GetNode("foo_7");
-  ASSERT_NE(foo_7, nullptr);
-  expected = NDef("", "", {"a:3", "b:2", "d", "^d"});
-  CompareNodeInputs(graph, &expected, foo_7);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {},
+            {"foo_1", "foo_2", "foo_3", "foo_4", "foo_5", "foo_7"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {},
+            {"foo_1:1", "^foo_2", "foo_3:1", "foo_4:1", "foo_5:1", "foo_7:1"});
+  CheckNode(graph, "c", "Switch", "", {}, {}, {"d"});
+  CheckNode(graph, "d", "Identity", "", {}, {"c:1"},
+            {"foo_6", "^foo_6", "foo_7:2", "^foo_7"});
+  CheckNode(graph, "foo_1", "IdentityN", "", {}, {"a", "b:1"}, {});
+  CheckNode(graph, "foo_2", "IdentityN", "", {}, {"a", "^b"}, {});
+  CheckNode(graph, "foo_3", "IdentityN", "", {}, {"a", "b:1"}, {});
+  CheckNode(graph, "foo_4", "IdentityN", "", {}, {"a:2", "b:1"}, {});
+  CheckNode(graph, "foo_5", "NotImportant", "", {}, {"a:2", "b:1"}, {});
+  CheckNode(graph, "foo_6", "Identity", "", {}, {"d", "^d"}, {});
+  CheckNode(graph, "foo_7", "NotImportant", "", {}, {"a:3", "b:2", "d", "^d"},
+            {});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, DedupControllingFaninsOnAddFanin) {
@@ -881,17 +883,14 @@ TEST(MutableGraphViewTest, DedupControllingFaninsOnAddFanin) {
   MutableGraphView graph(&graph_def);
 
   EXPECT_TRUE(graph.AddRegularFanin("b", {"a", 2}).ok());
-  NodeDef expected;
-  NodeDef* b = graph.GetNode("b");
-  ASSERT_NE(b, nullptr);
-  expected = NDef("", "", {"a:2"});
-  CompareNodeInputs(graph, &expected, b);
+  CheckNode(graph, "b", "NotImportant", "", {}, {"a:2"}, {});
 
   EXPECT_TRUE(graph.AddControllingFanin("c", {"a", Graph::kControlSlot}).ok());
-  NodeDef* c = graph.GetNode("c");
-  ASSERT_NE(c, nullptr);
-  expected = NDef("", "", {"a:1"});
-  CompareNodeInputs(graph, &expected, c);
+  CheckNode(graph, "c", "NotImportant", "", {}, {"a:1"}, {});
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"b:0", "c:0"});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnAddFanin) {
@@ -903,29 +902,18 @@ TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnAddFanin) {
   MutableGraphView graph(&graph_def);
 
   EXPECT_TRUE(graph.AddRegularFanin("c", {"b", 2}).ok());
-  NodeDef expected;
-  NodeDef* c = graph.GetNode("c");
-  ASSERT_NE(c, nullptr);
-  expected = NDef("", "", {"b:2"});
-  CompareNodeInputs(graph, &expected, c);
+  CheckNode(graph, "c", "", "", {}, {"b:2"}, {});
   EXPECT_TRUE(graph.AddControllingFanin("c", {"b", Graph::kControlSlot}).ok());
-  expected = NDef("", "", {"b:2", "^b"});
-  CompareNodeInputs(graph, &expected, c);
+  CheckNode(graph, "c", "", "", {}, {"b:2", "^b"}, {});
   EXPECT_TRUE(graph.AddControllingFanin("c", {"b", Graph::kControlSlot}).ok());
-  expected = NDef("", "", {"b:2", "^b"});
-  CompareNodeInputs(graph, &expected, c);
+  CheckNode(graph, "c", "", "", {}, {"b:2", "^b"}, {});
 
   EXPECT_TRUE(graph.AddControllingFanin("d", {"b", Graph::kControlSlot}).ok());
-  NodeDef* d = graph.GetNode("d");
-  ASSERT_NE(d, nullptr);
-  expected = NDef("", "", {"^b"});
-  CompareNodeInputs(graph, &expected, d);
+  CheckNode(graph, "d", "", "", {}, {"^b"}, {});
   EXPECT_TRUE(graph.AddControllingFanin("d", {"b", Graph::kControlSlot}).ok());
-  expected = NDef("", "", {"^b"});
-  CompareNodeInputs(graph, &expected, d);
-  EXPECT_TRUE(graph.AddRegularFanin("d", {"b", 3}).ok());
-  expected = NDef("", "", {"b:3", "^b"});
-  CompareNodeInputs(graph, &expected, d);
+  CheckNode(graph, "d", "", "", {}, {"^b"}, {});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, DedupControllingFaninsOnUpdateFanin) {
@@ -938,10 +926,12 @@ TEST(MutableGraphViewTest, DedupControllingFaninsOnUpdateFanin) {
   MutableGraphView graph(&graph_def);
 
   EXPECT_TRUE(graph.UpdateFanin("c", {"a", 1}, {"b", 2}).ok());
-  NodeDef* c = graph.GetNode("c");
-  ASSERT_NE(c, nullptr);
-  NodeDef expected = NDef("", "", {"b:2"});
-  CompareNodeInputs(graph, &expected, c);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {});
+  CheckNode(graph, "b", "NotImportant", "", {}, {}, {"c"});
+  CheckNode(graph, "c", "NotImportant", "", {}, {"b:2"}, {});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnUpdateFanin) {
@@ -957,23 +947,16 @@ TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnUpdateFanin) {
                   .UpdateFanin("d", {"b", Graph::kControlSlot},
                                {"c", Graph::kControlSlot})
                   .ok());
-  NodeDef expected;
-  NodeDef* d = graph.GetNode("d");
-  ASSERT_NE(d, nullptr);
-  expected = NDef("", "", {"c", "^c"});
-  CompareNodeInputs(graph, &expected, d);
+  CheckNode(graph, "d", "NotImportant", "", {}, {"c", "^c"}, {});
 
   EXPECT_TRUE(graph.UpdateFanin("e", {"b", 0}, {"c", 3}).ok());
-  NodeDef* e = graph.GetNode("e");
-  ASSERT_NE(e, nullptr);
-  expected = NDef("", "", {"c:3", "^c"});
-  CompareNodeInputs(graph, &expected, e);
+  CheckNode(graph, "e", "NotImportant", "", {}, {"c:3", "^c"}, {});
 
   EXPECT_TRUE(
       graph.UpdateFanin("e", {"c", 3}, {"c", Graph::kControlSlot}).ok());
-  ASSERT_NE(e, nullptr);
-  expected = NDef("", "", {"^c"});
-  CompareNodeInputs(graph, &expected, e);
+  CheckNode(graph, "e", "NotImportant", "", {}, {"^c"}, {});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnAddFanin) {
@@ -986,19 +969,12 @@ TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnAddFanin) {
   MutableGraphView graph(&graph_def);
 
   EXPECT_TRUE(graph.AddRegularFanin("c", {"a", 3}).ok());
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-
-  auto fanouts = graph.GetFanouts(*a, /*include_controlled_nodes=*/true);
-  EXPECT_EQ(fanouts.size(), 2);
 
-  NodeDef* b = graph.GetNode("b");
-  ASSERT_NE(b, nullptr);
-  EXPECT_EQ(fanouts.count(MutableGraphView::InputPort(b, 0)), 1);
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"b", "c"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {"a:1"}, {"^c"});
+  CheckNode(graph, "c", "NotImportant", "", {}, {"a:3", "^b"}, {});
 
-  NodeDef* c = graph.GetNode("c");
-  ASSERT_NE(c, nullptr);
-  EXPECT_EQ(fanouts.count(MutableGraphView::InputPort(c, 0)), 1);
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnRemoveFanin) {
@@ -1011,15 +987,11 @@ TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnRemoveFanin) {
   MutableGraphView graph(&graph_def);
 
   EXPECT_TRUE(graph.RemoveRegularFanin("c", {"a", 2}).ok());
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"b"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {"a:1"}, {});
+  CheckNode(graph, "c", "NotImportant", "", {}, {}, {});
 
-  auto fanouts = graph.GetFanouts(*a, /*include_controlled_nodes=*/true);
-  EXPECT_EQ(fanouts.size(), 1);
-
-  NodeDef* b = graph.GetNode("b");
-  ASSERT_NE(b, nullptr);
-  EXPECT_EQ(fanouts.count(MutableGraphView::InputPort(b, 0)), 1);
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, KeepMaxRegularOutputPortOnRemoveFanin) {
@@ -1032,15 +1004,12 @@ TEST(MutableGraphViewTest, KeepMaxRegularOutputPortOnRemoveFanin) {
   MutableGraphView graph(&graph_def);
 
   EXPECT_TRUE(graph.RemoveRegularFanin("b", {"a", 1}).ok());
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
 
-  auto fanouts = graph.GetFanouts(*a, /*include_controlled_nodes=*/true);
-  EXPECT_EQ(fanouts.size(), 1);
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"c"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {}, {});
+  CheckNode(graph, "c", "NotImportant", "", {}, {"a:2"}, {});
 
-  NodeDef* c = graph.GetNode("c");
-  ASSERT_NE(c, nullptr);
-  EXPECT_EQ(fanouts.count(MutableGraphView::InputPort(c, 0)), 1);
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnUpdateFanin) {
@@ -1053,22 +1022,12 @@ TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnUpdateFanin) {
   MutableGraphView graph(&graph_def);
 
   EXPECT_TRUE(graph.UpdateFanin("c", {"a", 2}, {"b", 3}).ok());
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-
-  auto a_fanouts = graph.GetFanouts(*a, /*include_controlled_nodes=*/true);
-  EXPECT_EQ(a_fanouts.size(), 1);
 
-  NodeDef* b = graph.GetNode("b");
-  ASSERT_NE(b, nullptr);
-  EXPECT_EQ(a_fanouts.count(MutableGraphView::InputPort(b, 0)), 1);
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"b"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {"a:1"}, {"c"});
+  CheckNode(graph, "c", "NotImportant", "", {}, {"b:3"}, {});
 
-  auto b_fanouts = graph.GetFanouts(*b, /*include_controlled_nodes=*/true);
-  EXPECT_EQ(b_fanouts.size(), 1);
-
-  NodeDef* c = graph.GetNode("c");
-  ASSERT_NE(c, nullptr);
-  EXPECT_EQ(b_fanouts.count(MutableGraphView::InputPort(c, 0)), 1);
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, AddControllingFaninMissing) {
@@ -1096,15 +1055,11 @@ TEST(MutableGraphViewTest, AddControllingFaninMissing) {
   EXPECT_EQ(s.error_message(), expected_msg);
 
   ASSERT_EQ(graph.graph()->node_size(), 2);
-  NodeDef expected;
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-  expected = NDef("", "", {});
-  CompareNodeInputs(graph, &expected, a);
-
-  NodeDef* b = graph.GetNode("b");
-  ASSERT_NE(b, nullptr);
-  CompareNodeInputs(graph, &expected, b);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {});
+  CheckNode(graph, "b", "NotImportant", "", {}, {}, {});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, AddControllingFaninExistingControl) {
@@ -1118,16 +1073,11 @@ TEST(MutableGraphViewTest, AddControllingFaninExistingControl) {
   EXPECT_TRUE(graph.AddControllingFanin("a", {"b", Graph::kControlSlot}).ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 2);
-  NodeDef expected;
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-  expected = NDef("", "", {"^b"});
-  CompareNodeInputs(graph, &expected, a);
-
-  NodeDef* b = graph.GetNode("b");
-  ASSERT_NE(b, nullptr);
-  expected = NDef("", "", {});
-  CompareNodeInputs(graph, &expected, b);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {"^b"}, {});
+  CheckNode(graph, "b", "NotImportant", "", {}, {}, {"^a"});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, AddControllingFaninNotSwitch) {
@@ -1141,16 +1091,11 @@ TEST(MutableGraphViewTest, AddControllingFaninNotSwitch) {
   EXPECT_TRUE(graph.AddControllingFanin("a", {"b", 2}).ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 2);
-  NodeDef expected;
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-  expected = NDef("", "", {"^b"});
-  CompareNodeInputs(graph, &expected, a);
-
-  NodeDef* b = graph.GetNode("b");
-  ASSERT_NE(b, nullptr);
-  expected = NDef("", "", {});
-  CompareNodeInputs(graph, &expected, b);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {"^b"}, {});
+  CheckNode(graph, "b", "NotImportant", "", {}, {}, {"^a"});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, AddControllingFaninSwitch) {
@@ -1167,14 +1112,11 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitch) {
   EXPECT_EQ(s.error_message(), expected_msg);
 
   ASSERT_EQ(graph.graph()->node_size(), 2);
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-  NodeDef expected = NDef("", "", {});
-  CompareNodeInputs(graph, &expected, a);
-
-  NodeDef* b = graph.GetNode("b");
-  ASSERT_NE(b, nullptr);
-  CompareNodeInputs(graph, &expected, b);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {});
+  CheckNode(graph, "b", "Switch", "", {}, {}, {});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, AddControllingFaninSwitchWithIdentity) {
@@ -1189,10 +1131,12 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithIdentity) {
   EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-  NodeDef expected = NDef("", "", {"^identity"});
-  CompareNodeInputs(graph, &expected, a);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {"^identity"}, {});
+  CheckNode(graph, "switch", "Switch", "", {}, {}, {"identity"});
+  CheckNode(graph, "identity", "Identity", "", {}, {"switch"}, {"^a"});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, AddControllingFaninSwitchWithNoExistingIdentity) {
@@ -1208,20 +1152,15 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithNoExistingIdentity) {
   EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
-  NodeDef expected;
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-  expected = NDef("", "", {"^ConstantFoldingCtrl/switch_0"});
-  CompareNodeInputs(graph, &expected, a);
-
-  NodeDef* identity = graph.GetNode("ConstantFoldingCtrl/switch_0");
-  ASSERT_NE(identity, nullptr);
-  expected = NDef("", "", {"switch"});
-  CompareNodeInputs(graph, &expected, identity);
-  EXPECT_EQ(identity->op(), "Identity");
-  EXPECT_EQ(identity->device(), kDevice);
-  ASSERT_TRUE(identity->attr().count("T"));
-  EXPECT_EQ(identity->attr().at("T").type(), DT_FLOAT);
+
+  CheckNode(graph, "a", "NotImportant", "", {},
+            {"^ConstantFoldingCtrl/switch_0"}, {});
+  CheckNode(graph, "switch", "Switch", kDevice, {{"T", DT_FLOAT}}, {},
+            {"ConstantFoldingCtrl/switch_0"});
+  CheckNode(graph, "ConstantFoldingCtrl/switch_0", "Identity", kDevice,
+            {{"T", DT_FLOAT}}, {"switch"}, {"^a"});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, AddControllingFaninSwitchWithExistingAddedIdentity) {
@@ -1236,10 +1175,15 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithExistingAddedIdentity) {
   EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-  NodeDef expected = NDef("", "", {"^ConstantFoldingCtrl/switch_0"});
-  CompareNodeInputs(graph, &expected, a);
+
+  CheckNode(graph, "a", "NotImportant", "", {},
+            {"^ConstantFoldingCtrl/switch_0"}, {});
+  CheckNode(graph, "switch", "Switch", "", {}, {},
+            {"ConstantFoldingCtrl/switch_0"});
+  CheckNode(graph, "ConstantFoldingCtrl/switch_0", "Identity", "", {},
+            {"switch"}, {"^a"});
+
+  CheckGraph(graph);
 }
 
 void TestAddControllingFaninSelfLoops(absl::string_view node_name,
@@ -1259,30 +1203,14 @@ void TestAddControllingFaninSelfLoops(absl::string_view node_name,
   EXPECT_EQ(s.error_message(), error_msg);
 
   EXPECT_EQ(graph.graph()->node_size(), 5);
-  NodeDef expected;
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-  expected = NDef("", "", {}, {});
-  CompareNodeInputs(graph, &expected, a);
-
-  NodeDef* b = graph.GetNode("b");
-  ASSERT_NE(b, nullptr);
-  CompareNodeInputs(graph, &expected, b);
-
-  NodeDef* c = graph.GetNode("c");
-  ASSERT_NE(c, nullptr);
-  expected = NDef("", "", {"b:0"});
-  CompareNodeInputs(graph, &expected, c);
-
-  NodeDef* d = graph.GetNode("d");
-  ASSERT_NE(d, nullptr);
-  expected = NDef("", "", {"b:1"});
-  CompareNodeInputs(graph, &expected, d);
-
-  NodeDef* e = graph.GetNode("e");
-  ASSERT_NE(e, nullptr);
-  expected = NDef("", "", {"^a"});
-  CompareNodeInputs(graph, &expected, e);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"^e"});
+  CheckNode(graph, "b", "Switch", "", {{"T", DT_FLOAT}}, {}, {"c", "d"});
+  CheckNode(graph, "c", "Identity", "", {}, {"b"}, {});
+  CheckNode(graph, "d", "Identity", "", {}, {"b:1"}, {});
+  CheckNode(graph, "e", "NotImportant", "", {}, {"^a"}, {});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, AddControllingFaninSelfLoops) {
@@ -1332,23 +1260,13 @@ TEST(MutableGraphViewTest, AddControllingFaninSelfLoopsGeneratedIdentity) {
   EXPECT_EQ(s.error_message(), expected_msg);
 
   EXPECT_EQ(graph.graph()->node_size(), 4);
-  NodeDef expected;
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-  expected = NDef("", "", {}, {});
-  CompareNodeInputs(graph, &expected, a);
-
-  NodeDef* b = graph.GetNode("b");
-  ASSERT_NE(b, nullptr);
-  CompareNodeInputs(graph, &expected, b);
-
-  NodeDef* c = graph.GetNode("c");
-  ASSERT_NE(c, nullptr);
-  CompareNodeInputs(graph, &expected, c);
-
-  NodeDef* d = graph.GetNode("ConstantFoldingCtrl/b_1");
-  ASSERT_NE(d, nullptr);
-  CompareNodeInputs(graph, &expected, d);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {});
+  CheckNode(graph, "b", "Switch", "", {{"T", DT_FLOAT}}, {}, {});
+  CheckNode(graph, "c", "NotImportant", "", {}, {}, {});
+  CheckNode(graph, "ConstantFoldingCtrl/b_1", "Identity", "", {}, {}, {});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, RemoveControllingFaninMissing) {
@@ -1364,10 +1282,13 @@ TEST(MutableGraphViewTest, RemoveControllingFaninMissing) {
   EXPECT_TRUE(graph.RemoveControllingFanin("d", "c").ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 4);
-  NodeDef* d = graph.GetNode("d");
-  ASSERT_NE(d, nullptr);
-  NodeDef expected = NDef("", "", {"^a", "^b"});
-  CompareNodeInputs(graph, &expected, d);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"^d"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {}, {"^d"});
+  CheckNode(graph, "c", "NotImportant", "", {}, {}, {});
+  CheckNode(graph, "d", "NotImportant", "", {}, {"^a", "^b"}, {});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, RemoveControllingFaninExisting) {
@@ -1384,10 +1305,13 @@ TEST(MutableGraphViewTest, RemoveControllingFaninExisting) {
   EXPECT_TRUE(graph.RemoveControllingFanin("d", "a").ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 4);
-  NodeDef* d = graph.GetNode("d");
-  ASSERT_NE(d, nullptr);
-  NodeDef expected = NDef("", "", {"^c", "^b"});
-  CompareNodeInputs(graph, &expected, d);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {});
+  CheckNode(graph, "b", "NotImportant", "", {}, {}, {"^d"});
+  CheckNode(graph, "c", "NotImportant", "", {}, {}, {"^d"});
+  CheckNode(graph, "d", "NotImportant", "", {}, {"^c", "^b"}, {});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, RemoveControllingFaninOnRegularFanin) {
@@ -1403,10 +1327,12 @@ TEST(MutableGraphViewTest, RemoveControllingFaninOnRegularFanin) {
   EXPECT_TRUE(graph.RemoveControllingFanin("c", "b").ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
-  NodeDef* c = graph.GetNode("c");
-  ASSERT_NE(c, nullptr);
-  NodeDef expected = NDef("", "", {"a", "b"});
-  CompareNodeInputs(graph, &expected, c);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"b", "c"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {"a"}, {"c:1"});
+  CheckNode(graph, "c", "NotImportant", "", {}, {"a", "b"}, {});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, RemoveControllingFaninSelfLoop) {
@@ -1424,10 +1350,12 @@ TEST(MutableGraphViewTest, RemoveControllingFaninSelfLoop) {
   EXPECT_EQ(s.error_message(), expected_msg);
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
-  NodeDef* c = graph.GetNode("c");
-  ASSERT_NE(c, nullptr);
-  NodeDef expected = NDef("", "", {"a", "b"});
-  CompareNodeInputs(graph, &expected, c);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"b", "c"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {"a"}, {"c:1"});
+  CheckNode(graph, "c", "NotImportant", "", {}, {"a", "b"}, {});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, DeleteNodes) {
@@ -1444,21 +1372,14 @@ TEST(MutableGraphViewTest, DeleteNodes) {
   EXPECT_NE(graph.GetNode("foo_1"), nullptr);
   graph.DeleteNodes({"foo_1"});
 
+  EXPECT_EQ(graph.graph()->node_size(), 3);
   EXPECT_EQ(graph.GetNode("foo_1"), nullptr);
 
-  NodeDef* bar = graph.GetNode("bar");
-  NodeDef* other = graph.GetNode("other");
-  NodeDef* foo_2 = graph.GetNode("foo_2");
-
-  bool include_control_fanouts = true;
-  auto bar_fanouts = graph.GetFanouts(*bar, include_control_fanouts);
-  auto other_fanouts = graph.GetFanouts(*other, include_control_fanouts);
-
-  EXPECT_EQ(bar_fanouts.size(), 1);
-  EXPECT_EQ(bar_fanouts.count(MutableGraphView::InputPort(foo_2, 1)), 1);
+  CheckNode(graph, "bar", "NotImportant", "", {}, {}, {"foo_2:1"});
+  CheckNode(graph, "other", "NotImportant", "", {}, {}, {"foo_2"});
+  CheckNode(graph, "foo_2", "NotImportant", "", {}, {"other:1", "bar:2"}, {});
 
-  EXPECT_EQ(other_fanouts.size(), 1);
-  EXPECT_EQ(other_fanouts.count(MutableGraphView::InputPort(foo_2, 0)), 1);
+  CheckGraph(graph);
 }
 
 }  // namespace
-- 
GitLab


From b2723af89145e5238df417b9bfd47fef29d00354 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Wed, 16 Jan 2019 13:05:06 -0800
Subject: [PATCH 0810/2345] Don't run ops.device(None) in eager.

PiperOrigin-RevId: 229610975
---
 tensorflow/python/framework/ops.py             | 3 +++
 tensorflow/python/ops/resource_variable_ops.py | 4 +++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 079fad4210..6da6637b0b 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -102,6 +102,9 @@ class _UserDeviceSpec(object):
 
 class NullContextmanager(object):
 
+  def __init__(self, *args, **kwargs):
+    pass
+
   def __enter__(self):
     pass
 
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 1a895fee87..c8f20029ac 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -409,11 +409,13 @@ class ResourceVariable(variables.VariableV1):
         # Use attr_scope and device(None) to simulate the behavior of
         # colocate_with when the variable we want to colocate with doesn't
         # yet exist.
+        device_context_manager = (
+            ops.device if self._in_graph_mode else ops.NullContextmanager)
         attr = attr_value_pb2.AttrValue(
             list=attr_value_pb2.AttrValue.ListValue(
                 s=[compat.as_bytes("loc:@%s" % handle_name)]))
         with ops.get_default_graph()._attr_scope({"_class": attr}):
-          with ops.name_scope("Initializer"), ops.device(None):
+          with ops.name_scope("Initializer"), device_context_manager(None):
             initial_value = ops.convert_to_tensor(
                 initial_value() if init_from_fn else initial_value,
                 name="initial_value", dtype=dtype)
-- 
GitLab


From 71472ddbe17a51ba10003d271700bb0123ceb991 Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Wed, 16 Jan 2019 13:29:51 -0800
Subject: [PATCH 0811/2345] Remove dynamic dynamic dimensions in
 XLABuilder::Build

XLA backend cannot handle dynamic dimension yet, remove all dynamic
dimensions before building xla program until we have support in the
backend.

PiperOrigin-RevId: 229615685
---
 tensorflow/compiler/xla/client/xla_builder.cc | 19 +++++++++++++++++++
 .../compiler/xla/client/xla_builder_test.cc   |  4 +++-
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc
index 4b360c6563..20298d175d 100644
--- a/tensorflow/compiler/xla/client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_builder.cc
@@ -322,6 +322,25 @@ StatusOr<XlaComputation> XlaBuilder::Build(int64 root_id) {
     return AppendStatus(first_error_, backtrace);
   }
 
+  // TODO(b/121223198): XLA backend cannot handle dynamic dimensions yet, remove
+  // all dynamic dimensions before building xla program until we have support in
+  // the backend.
+  std::function<void(ShapeProto*)> remove_dynamic_dimension =
+      [&](ShapeProto* shape) {
+        if (shape->tuple_shapes_size() != 0) {
+          for (int64 i = 0; i < shape->tuple_shapes_size(); ++i) {
+            remove_dynamic_dimension(shape->mutable_tuple_shapes(i));
+          }
+        }
+        for (int64 i = 0; i < shape->dimensions_size(); ++i) {
+          shape->set_is_dynamic_dimension(i, false);
+        }
+      };
+
+  for (auto& instruction : instructions_) {
+    remove_dynamic_dimension(instruction.mutable_shape());
+  }
+
   HloComputationProto entry;
   SetProtoIdAndName(&entry, name_, kNameSeparator, GetNextId());
   TF_ASSIGN_OR_RETURN(ProgramShape program_shape, GetProgramShape(root_id));
diff --git a/tensorflow/compiler/xla/client/xla_builder_test.cc b/tensorflow/compiler/xla/client/xla_builder_test.cc
index abc11b4732..feee8187c7 100644
--- a/tensorflow/compiler/xla/client/xla_builder_test.cc
+++ b/tensorflow/compiler/xla/client/xla_builder_test.cc
@@ -463,7 +463,9 @@ TEST_F(XlaBuilderTest, DynamicParameter) {
                                  ->parameter_instruction(0)
                                  ->shape()
                                  .tuple_shapes(1);
-  EXPECT_TRUE(param_shape.is_dynamic_dimension(0));
+  // TODO(b/121223198): The dynamic dimension should be set once we enable
+  // dynamic dimensions in xla builder.
+  EXPECT_FALSE(param_shape.is_dynamic_dimension(0));
 }
 
 TEST_F(XlaBuilderTest, AfterAllWithNonTokenOperands) {
-- 
GitLab


From c432f294845786ca4d26e4d8091cab4dfa1e796f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 13:40:43 -0800
Subject: [PATCH 0812/2345] Restore 0-arg version of Tensor::DebugString.

PiperOrigin-RevId: 229617823
---
 tensorflow/core/framework/tensor.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/framework/tensor.h b/tensorflow/core/framework/tensor.h
index 44293c5c19..4cfad893d5 100644
--- a/tensorflow/core/framework/tensor.h
+++ b/tensorflow/core/framework/tensor.h
@@ -529,7 +529,8 @@ class Tensor {
   // `num_values` is the number of actual data values in the tensor
   // included in the message. If the tensor might be resident in
   // GPU/TPU memory use DeviceSafeDebugString instead.
-  string DebugString(int num_values = 3) const;
+  string DebugString(int num_values) const;
+  string DebugString() const { return DebugString(3); }
 
   // Variant of DebugString() that should be used for possibly non-CPU tensors.
   // If the tensor is not resident on CPU, we can't read its values as
-- 
GitLab


From 440ee60cda0912d2275a61b714a33564716bfd5a Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Wed, 16 Jan 2019 13:54:48 -0800
Subject: [PATCH 0813/2345] [TF:XLA] Bump open source llvm revision to r351319

PiperOrigin-RevId: 229620326
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 354148f034..b8ef43d27e 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -502,11 +502,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "c6e6de0b5cd524acf276188cb6aae427568070cd1b9cb9c0e3aab8044f3ac278",
-        strip_prefix = "llvm-a42cde684126a3797f3eda9c894c379e7a8c66c1",
+        sha256 = "2027c5043ea3458376fbf4263aaab3b257736b9e4a1e9e9984f15e245177979d",
+        strip_prefix = "llvm-82a2c96ee81a617918d9bcc0fbe9af860e437f5f",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/a42cde684126a3797f3eda9c894c379e7a8c66c1.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/a42cde684126a3797f3eda9c894c379e7a8c66c1.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/82a2c96ee81a617918d9bcc0fbe9af860e437f5f.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/82a2c96ee81a617918d9bcc0fbe9af860e437f5f.tar.gz",
         ],
     )
 
-- 
GitLab


From 955d8f1377c3f8e7a2ec4b548126d20d70e063f0 Mon Sep 17 00:00:00 2001
From: Alan Chiao <alanchiao@google.com>
Date: Wed, 16 Jan 2019 14:13:41 -0800
Subject: [PATCH 0814/2345] Clarify tensor_utils neon memory alignment
 assumptions.

PiperOrigin-RevId: 229624071
---
 .../internal/optimized/neon_tensor_utils.cc   | 34 ++++++++++++-------
 1 file changed, 22 insertions(+), 12 deletions(-)

diff --git a/tensorflow/lite/kernels/internal/optimized/neon_tensor_utils.cc b/tensorflow/lite/kernels/internal/optimized/neon_tensor_utils.cc
index cf1bda2661..3e7b6c22be 100644
--- a/tensorflow/lite/kernels/internal/optimized/neon_tensor_utils.cc
+++ b/tensorflow/lite/kernels/internal/optimized/neon_tensor_utils.cc
@@ -90,20 +90,28 @@ void NeonMatrixBatchVectorMultiplyAccumulate(
     int n_batch, float* __restrict__ result, int result_stride) {
   const int kWeightsPerUint32 = 4;
   const int kWeightsPerNeonLane = 16;
-  // If the number of rows is not divisible by kWeightsPerUint32, we set a
-  // flag and allocate an aligned memory block. The flag is used to use the
-  // aligned memory block later in the kernel loop.
+  // Assuming *matrix is kWeightsPerUint32-byte aligned,
+  // every row of the matrix is also
+  // kWeightsPerUint32-byte aligned as long as cols is
+  // a multiple of kWeightsPerUint32. The assumption
+  // is currently satisfied by TFLite's 16-byte memory
+  // alignment scheme.
+  //
+  // Otherwise, we allocate an aligned memory block and set
+  // a flag to later copy rows from matrix to the block
+  // for aligned multiplication.
   bool unaligned = false;
-  int8* aligned_row = nullptr;
+  int8_t* aligned_row = nullptr;
   void* aligned_row_free = nullptr;
   if ((m_cols & (kWeightsPerUint32 - 1)) != 0) {
     unaligned = true;
-    aligned_row = (int8*)aligned_alloc(kWeightsPerUint32, m_cols,  // NOLINT
-                                       &aligned_row_free);
+    aligned_row = (int8_t*)aligned_alloc(kWeightsPerUint32, m_cols,  // NOLINT
+                                         &aligned_row_free);
   }
   void* aligned_vec_free = nullptr;
-  int8* aligned_vec = (int8*)aligned_alloc(kWeightsPerUint32, m_cols,  // NOLINT
-                                           &aligned_vec_free);
+  int8_t* aligned_vec =
+      (int8_t*)aligned_alloc(kWeightsPerUint32, m_cols,  // NOLINT
+                             &aligned_vec_free);
 
   // If m_cols is not at least kWeightsPerNeonLane, we cannot use the main
   // vectorized loop, and we need to process sequentially. postamble_start shows
@@ -114,13 +122,13 @@ void NeonMatrixBatchVectorMultiplyAccumulate(
   for (batch = 0; batch < n_batch; ++batch) {
     const float batch_scaling_factor = scaling_factors[batch];
     // Copy the vector data to an aligned vector.
-    memcpy(aligned_vec, vectors + batch * m_cols, sizeof(int8) * m_cols);
+    memcpy(aligned_vec, vectors + batch * m_cols, sizeof(int8_t) * m_cols);
     // Compute dot-product for every column.
     for (row = 0; row < m_rows; ++row, result += result_stride) {
       // Get the address of the first element of the row.
-      int8* row_ptr = (int8*)matrix + row * m_cols;  // NOLINT
+      int8_t* row_ptr = (int8_t*)matrix + row * m_cols;  // NOLINT
       if (unaligned) {
-        memcpy(aligned_row, row_ptr, sizeof(int8) * m_cols);
+        memcpy(aligned_row, row_ptr, sizeof(int8_t) * m_cols);
         row_ptr = aligned_row;
       }
 
@@ -135,7 +143,8 @@ void NeonMatrixBatchVectorMultiplyAccumulate(
       col = 0;
       for (; col < postamble_start; col += kWeightsPerNeonLane) {
         // Load 16 8-bit values from the row and vector, each, to operate on.
-        // Here the assumption is that each buffer is 4-byte aligned.
+        // Here the assumption is that each buffer is 4-byte aligned. Otherwise,
+        // performance may suffer significantly.
         TFLITE_CHECK_EQ((uintptr_t)(&row_ptr[col]) & (kWeightsPerUint32 - 1),
                         0);
         const int8x16_t s1_8x16 = vld1q_s8((const int8_t*)(aligned_vec + col));
@@ -164,6 +173,7 @@ void NeonMatrixBatchVectorMultiplyAccumulate(
         if ((m_cols - postamble_start) >= (kWeightsPerNeonLane >> 1)) {
           // Load 8 8-bit values from the row and column each to operate on.
           // Here the assumption is that each buffer is 4-bytes aligned.
+          // Otherwise, performance may suffer significantly.
           TFLITE_CHECK_EQ((uintptr_t)(&row_ptr[col]) & (kWeightsPerUint32 - 1),
                           0);
           const int8x8_t s1_8x8 = vld1_s8((const int8_t*)(aligned_vec + col));
-- 
GitLab


From 8ae3ff414ca860f6d4fdc92b255b483c1d9c302f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 14:20:57 -0800
Subject: [PATCH 0815/2345] Update tf.distribute.Strategy.broadcast() API to
 match accepted RFC:
 https://github.com/tensorflow/community/blob/master/rfcs/20181016-replicator.md

PiperOrigin-RevId: 229625402
---
 tensorflow/python/distribute/distribute_lib.py    | 15 +++++++++------
 tensorflow/python/distribute/values.py            |  4 ++--
 ...tensorflow.distribute.-mirrored-strategy.pbtxt |  2 +-
 .../v1/tensorflow.distribute.-strategy.pbtxt      |  2 +-
 ...tensorflow.distribute.-mirrored-strategy.pbtxt |  2 +-
 .../v2/tensorflow.distribute.-strategy.pbtxt      |  2 +-
 6 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/tensorflow/python/distribute/distribute_lib.py b/tensorflow/python/distribute/distribute_lib.py
index 32789c821e..e7d52c6c98 100644
--- a/tensorflow/python/distribute/distribute_lib.py
+++ b/tensorflow/python/distribute/distribute_lib.py
@@ -500,10 +500,11 @@ class DistributionStrategy(object):
         inputs = input_iterator.get_next()
         return self._extended.call_for_each_replica(fn, args=(inputs,))
 
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def broadcast(self, tensor, destinations=None):
-    """DEPRECATED: use extended.broadcast_to() instead."""
-    return self._extended.broadcast_to(tensor, destinations)
+  # TODO(b/121296772,b/121300973): Add logical_device argument (default of 0).
+  def broadcast(self, tensor):
+    """Broadcasts `tensor` to all replicas, returning a per-replica value."""
+    _require_cross_replica_context_extended(self._extended)
+    return self._extended._broadcast(tensor)  # pylint: disable=protected-access
 
   def reduce(self, reduce_op, value):
     """Reduce `value` across replicas.
@@ -758,8 +759,7 @@ class DistributionStrategyExtended(object):
   * `d.make_dataset_iterator(dataset)` (or the deprecated
     `d.distribute_dataset(dataset).make_one_shot_iterator()`): in cross-replica
     context, produces an iterator with locality T
-  * `d.extended.broadcast_to(t)`: in cross-replica context, produces a value
-    with locality M
+  * `d.broadcast(t)`: in cross-replica context, produces a value with locality M
   * `d.extended.broadcast_to(t, v)`: in cross-replica context, produces a value
     with locality V(`v`)
   * `d.extended.call_for_each_replica(fn, ...)`: in cross-replica context, runs
@@ -1025,6 +1025,9 @@ class DistributionStrategyExtended(object):
     assert not isinstance(destinations, (list, tuple))
     return self._broadcast_to(tensor, destinations)
 
+  def _broadcast(self, tensor):
+    return self._broadcast_to(tensor, None)  # Default implementation
+
   def _broadcast_to(self, tensor, destinations):
     raise NotImplementedError("must be implemented in descendants")
 
diff --git a/tensorflow/python/distribute/values.py b/tensorflow/python/distribute/values.py
index 19967f2c5e..feaeea80cb 100644
--- a/tensorflow/python/distribute/values.py
+++ b/tensorflow/python/distribute/values.py
@@ -605,8 +605,8 @@ def validate_colocate(v, extended):
 
 def _apply_aggregation(strategy, value, aggregation, destinations):
   if aggregation == vs.VariableAggregation.ONLY_FIRST_REPLICA:
-    return strategy.broadcast(strategy.unwrap(value)[0],
-                              destinations=destinations)
+    return strategy.extended.broadcast_to(strategy.unwrap(value)[0],
+                                          destinations=destinations)
   reduce_op = reduce_util.ReduceOp.from_variable_aggregation(aggregation)
   return strategy.extended.reduce_to(reduce_op, value, destinations)
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
index c554df489a..4cb78b08f8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
@@ -17,7 +17,7 @@ tf_class {
   }
   member_method {
     name: "broadcast"
-    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'self\', \'tensor\'], varargs=None, keywords=None, defaults=None"
   }
   member_method {
     name: "colocate_vars_with"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
index cce83be3d0..11c1479b5b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
@@ -16,7 +16,7 @@ tf_class {
   }
   member_method {
     name: "broadcast"
-    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'self\', \'tensor\'], varargs=None, keywords=None, defaults=None"
   }
   member_method {
     name: "colocate_vars_with"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
index c554df489a..4cb78b08f8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
@@ -17,7 +17,7 @@ tf_class {
   }
   member_method {
     name: "broadcast"
-    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'self\', \'tensor\'], varargs=None, keywords=None, defaults=None"
   }
   member_method {
     name: "colocate_vars_with"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
index cce83be3d0..11c1479b5b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
@@ -16,7 +16,7 @@ tf_class {
   }
   member_method {
     name: "broadcast"
-    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'self\', \'tensor\'], varargs=None, keywords=None, defaults=None"
   }
   member_method {
     name: "colocate_vars_with"
-- 
GitLab


From b0b56368f0c9269e4b003e2bc07020043eef53c8 Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Wed, 16 Jan 2019 14:33:55 -0800
Subject: [PATCH 0816/2345] Add optional `validation_freq` argument to `fit`.
 This allows user to configure how many training epochs occur between each
 validation run, or by providing a list users can explicitly specify which
 epochs should be followed by a validation run.

PiperOrigin-RevId: 229627830
---
 tensorflow/python/keras/engine/training.py    | 21 +++++++++++++
 .../python/keras/engine/training_arrays.py    | 14 +++++++--
 .../keras/engine/training_distributed.py      | 23 +++++++++++---
 .../python/keras/engine/training_generator.py | 12 ++++++-
 .../python/keras/engine/training_test.py      | 29 +++++++++++++++++
 .../python/keras/engine/training_utils.py     | 31 +++++++++++++++++++
 .../golden/v1/tensorflow.keras.-model.pbtxt   |  4 +--
 .../v1/tensorflow.keras.-sequential.pbtxt     |  4 +--
 .../v1/tensorflow.keras.models.-model.pbtxt   |  4 +--
 .../tensorflow.keras.models.-sequential.pbtxt |  4 +--
 .../golden/v2/tensorflow.keras.-model.pbtxt   |  4 +--
 .../v2/tensorflow.keras.-sequential.pbtxt     |  4 +--
 ...ensorflow.keras.layers.-linear-model.pbtxt |  4 +--
 .../v2/tensorflow.keras.models.-model.pbtxt   |  4 +--
 .../tensorflow.keras.models.-sequential.pbtxt |  4 +--
 15 files changed, 140 insertions(+), 26 deletions(-)

diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 6b0422e71c..e6ede64573 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -636,6 +636,7 @@ class Model(Network):
           initial_epoch=0,
           steps_per_epoch=None,
           validation_steps=None,
+          validation_freq=1,
           max_queue_size=10,
           workers=1,
           use_multiprocessing=False,
@@ -740,6 +741,13 @@ class Model(Network):
             is a dataset or dataset iterator. Total number of steps (batches of
             samples) to draw before stopping when performing validation
             at the end of every epoch.
+        validation_freq: Only relevant if validation data is provided. Integer
+            or `collections.Container` instance (e.g. list, tuple, etc.). If an
+            integer, specifies how many training epochs to run before a new
+            validation run is performed, e.g. `validation_freq=2` runs
+            validation every 2 epochs. If a Container, specifies the epochs on
+            which to run validation, e.g. `validation_freq=[1, 2, 10]` runs
+            validation at the end of the 1st, 2nd, and 10th epochs.
         max_queue_size: Integer. Used for generator or `keras.utils.Sequence`
             input only. Maximum size for the generator queue.
             If unspecified, `max_queue_size` will default to 10.
@@ -846,6 +854,7 @@ class Model(Network):
           callbacks=callbacks,
           validation_data=validation_data,
           validation_steps=validation_steps,
+          validation_freq=validation_freq,
           class_weight=class_weight,
           max_queue_size=max_queue_size,
           workers=workers,
@@ -868,6 +877,7 @@ class Model(Network):
           callbacks=callbacks,
           validation_data=validation_data,
           validation_steps=validation_steps,
+          validation_freq=validation_freq,
           class_weight=class_weight,
           workers=0,
           shuffle=shuffle,
@@ -930,6 +940,7 @@ class Model(Network):
           callbacks=callbacks,
           validation_data=validation_data,
           validation_steps=validation_steps,
+          validation_freq=validation_freq,
           workers=0,
           shuffle=shuffle,
           initial_epoch=initial_epoch,
@@ -951,6 +962,7 @@ class Model(Network):
           initial_epoch=initial_epoch,
           steps_per_epoch=steps_per_epoch,
           validation_steps=validation_steps,
+          validation_freq=validation_freq,
           steps_name='steps_per_epoch')
 
   def evaluate(self,
@@ -1426,6 +1438,7 @@ class Model(Network):
                     callbacks=None,
                     validation_data=None,
                     validation_steps=None,
+                    validation_freq=1,
                     class_weight=None,
                     max_queue_size=10,
                     workers=1,
@@ -1480,6 +1493,13 @@ class Model(Network):
             to yield from `generator` before stopping.
             Optional for `Sequence`: if unspecified, will use
             the `len(validation_data)` as a number of steps.
+        validation_freq: Only relevant if validation data is provided. Integer
+            or `collections.Container` instance (e.g. list, tuple, etc.). If an
+            integer, specifies how many training epochs to run before a new
+            validation run is performed, e.g. `validation_freq=2` runs
+            validation every 2 epochs. If a Container, specifies the epochs on
+            which to run validation, e.g. `validation_freq=[1, 2, 10]` runs
+            validation at the end of the 1st, 2nd, and 10th epochs.
         class_weight: Dictionary mapping class indices to a weight
             for the class.
         max_queue_size: Integer. Maximum size for the generator queue.
@@ -1535,6 +1555,7 @@ class Model(Network):
         callbacks=callbacks,
         validation_data=validation_data,
         validation_steps=validation_steps,
+        validation_freq=validation_freq,
         class_weight=class_weight,
         max_queue_size=max_queue_size,
         workers=workers,
diff --git a/tensorflow/python/keras/engine/training_arrays.py b/tensorflow/python/keras/engine/training_arrays.py
index a356f8b328..3b6a13839a 100644
--- a/tensorflow/python/keras/engine/training_arrays.py
+++ b/tensorflow/python/keras/engine/training_arrays.py
@@ -56,6 +56,7 @@ def model_iteration(model,
                     initial_epoch=0,
                     steps_per_epoch=None,
                     validation_steps=None,
+                    validation_freq=1,
                     mode=ModeKeys.TRAIN,
                     validation_in_fit=False,
                     steps_name='steps',
@@ -84,6 +85,13 @@ def model_iteration(model,
         the default value of `None`.
       validation_steps: Number of steps to run validation for (only if doing
         validation from data tensors). Ignored with the default value of `None`.
+      validation_freq: Only relevant if validation data is provided. Integer or
+        `collections.Container` instance (e.g. list, tuple, etc.). If an
+        integer, specifies how many training epochs to run before a new
+        validation run is performed, e.g. `validation_freq=2` runs
+        validation every 2 epochs. If a Container, specifies the epochs on
+        which to run validation, e.g. `validation_freq=[1, 2, 10]` runs
+        validation at the end of the 1st, 2nd, and 10th epochs.
       mode: One of ModeKeys.TRAIN/ModeKeys.TEST/ModeKeys.PREDICT.
       validation_in_fit: DEPRECATED: if true, then this method is invoked from
         within training iteration (for validation). In this case, do not copy
@@ -320,8 +328,10 @@ def model_iteration(model,
     if len(results) == 1:
       results = results[0]
 
-    # Run the test loop every epoch during training.
-    if do_validation and not callbacks.model.stop_training:
+    # Run the test loop every `validation_freq` epochs during training.
+    if (do_validation and
+        training_utils.should_run_validation(validation_freq, epoch) and
+        not callbacks.model.stop_training):
       val_results = model_iteration(
           model,
           val_inputs,
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 58a8637a84..8b7e5bb5cc 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -30,6 +30,7 @@ from tensorflow.python.keras import backend as K
 from tensorflow.python.keras import callbacks as cbks
 from tensorflow.python.keras.engine import distributed_training_utils
 from tensorflow.python.keras.engine import training_arrays
+from tensorflow.python.keras.engine import training_utils
 from tensorflow.python.keras.utils.generic_utils import Progbar
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import tf_logging as logging
@@ -51,7 +52,8 @@ def fit_distributed(model,
                     sample_weight=None,
                     initial_epoch=0,
                     steps_per_epoch=None,
-                    validation_steps=None):
+                    validation_steps=None,
+                    validation_freq=1):
   """Fit loop for Distribution Strategies."""
   distributed_training_utils.validate_callbacks(callbacks, model.optimizer)
   distributed_training_utils.validate_inputs(
@@ -111,7 +113,8 @@ def fit_distributed(model,
         val_dataset=val_dataset,
         initial_epoch=initial_epoch,
         steps_per_epoch=steps_per_epoch,
-        validation_steps=validation_steps)
+        validation_steps=validation_steps,
+        validation_freq=1)
   else:
     return training_arrays.fit_loop(
         model,
@@ -124,7 +127,8 @@ def fit_distributed(model,
         shuffle=shuffle,
         initial_epoch=initial_epoch,
         steps_per_epoch=steps_per_epoch,
-        validation_steps=validation_steps)
+        validation_steps=validation_steps,
+        validation_freq=validation_freq)
 
 
 def evaluate_distributed(model,
@@ -206,7 +210,8 @@ def experimental_tpu_fit_loop(model,
                               initial_epoch=0,
                               steps_per_epoch=None,
                               val_dataset=None,
-                              validation_steps=None):
+                              validation_steps=None,
+                              validation_freq=1):
   """Fit loop for training with TPU DistributionStrategy.
 
   Arguments:
@@ -224,6 +229,13 @@ def experimental_tpu_fit_loop(model,
       validation_steps: Number of steps to run validation for
           (only if doing validation from data tensors).
           Ignored with the default value of `None`.
+      validation_freq: Only relevant if validation data is provided. Integer or
+          `collections.Container` instance (e.g. list, tuple, etc.). If an
+          integer, specifies how many training epochs to run before a new
+          validation run is performed, e.g. `validation_freq=2` runs
+          validation every 2 epochs. If a Container, specifies the epochs on
+          which to run validation, e.g. `validation_freq=[1, 2, 10]` runs
+          validation at the end of the 1st, 2nd, and 10th epochs.
 
   Returns:
       Returns `None`.
@@ -363,7 +375,8 @@ def experimental_tpu_fit_loop(model,
       if callbacks.model.stop_training:
         break
 
-    if do_validation:
+    if (do_validation and
+        training_utils.should_run_validation(validation_freq, epoch)):
       logging.info('Running validation at fit epoch: %s', epoch)
 
       if model._compile_distribution:
diff --git a/tensorflow/python/keras/engine/training_generator.py b/tensorflow/python/keras/engine/training_generator.py
index 8a5e24856c..23b5d7e8c5 100644
--- a/tensorflow/python/keras/engine/training_generator.py
+++ b/tensorflow/python/keras/engine/training_generator.py
@@ -46,6 +46,7 @@ def model_iteration(model,
                     callbacks=None,
                     validation_data=None,
                     validation_steps=None,
+                    validation_freq=1,
                     class_weight=None,
                     max_queue_size=10,
                     workers=1,
@@ -74,6 +75,13 @@ def model_iteration(model,
         `keras.utils.data_utils.Sequence` object or Eager Iterator or Dataset.
       validation_steps: Total number of steps (batches of samples) before
         declaring validation finished.
+      validation_freq: Only relevant if validation data is provided. Integer or
+        `collections.Container` instance (e.g. list, tuple, etc.). If an
+        integer, specifies how many training epochs to run before a new
+        validation run is performed, e.g. `validation_freq=2` runs
+        validation every 2 epochs. If a Container, specifies the epochs on
+        which to run validation, e.g. `validation_freq=[1, 2, 10]` runs
+        validation at the end of the 1st, 2nd, and 10th epochs.
       class_weight: Dictionary mapping class indices to a weight for the class.
       max_queue_size: Integer. Maximum size for the generator queue. If
         unspecified, `max_queue_size` will default to 10.
@@ -257,7 +265,9 @@ def model_iteration(model,
       results = results[0]
 
     # Run the test loop every epoch during training.
-    if do_validation and not callbacks.model.stop_training:
+    if (do_validation and
+        training_utils.should_run_validation(validation_freq, epoch) and
+        not callbacks.model.stop_training):
       val_results = model_iteration(
           model,
           validation_data,
diff --git a/tensorflow/python/keras/engine/training_test.py b/tensorflow/python/keras/engine/training_test.py
index fa26385cc4..6be4da70f6 100644
--- a/tensorflow/python/keras/engine/training_test.py
+++ b/tensorflow/python/keras/engine/training_test.py
@@ -22,6 +22,7 @@ import io
 import logging
 import sys
 
+from absl.testing import parameterized
 import numpy as np
 import six
 
@@ -803,6 +804,34 @@ class TrainingTest(keras_parameterized.TestCase):
       test_model_loss = test_model.train_on_batch(train_x, train_y)
       self.assertAlmostEqual(test_model_loss, reference_model_loss, places=4)
 
+  @keras_parameterized.run_with_all_model_types
+  @keras_parameterized.run_all_keras_modes
+  @parameterized.named_parameters(
+      ('default', 1, 4), ('integer_two', 2, 2), ('integer_four', 4, 1),
+      ('simple_list', [1, 3, 4], 3), ('duplicated_list', [4, 2, 2], 2))
+  def test_validation_freq(self, validation_freq, expected_runs):
+    x, y = np.ones((10, 10)), np.ones((10, 1))
+    model = testing_utils.get_small_mlp(2, 1, 10)
+    model.compile('sgd', 'mse')
+
+    class ValCounter(keras.callbacks.Callback):
+
+      def __init__(self):
+        self.val_runs = 0
+
+      def on_test_begin(self, logs=None):
+        self.val_runs += 1
+
+    val_counter = ValCounter()
+    model.fit(
+        x,
+        y,
+        epochs=4,
+        validation_data=(x, y),
+        validation_freq=validation_freq,
+        callbacks=[val_counter])
+    self.assertEqual(val_counter.val_runs, expected_runs)
+
 
 class TestExceptionsAndWarnings(keras_parameterized.TestCase):
 
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index c36b637829..3affb84147 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 import abc
+import collections
 from collections import OrderedDict
 import copy
 import json
@@ -1502,3 +1503,33 @@ def should_run_multi_worker():
   tf_config = json.loads(os.environ.get('TF_CONFIG', '{}'))
   cluster_spec = server_lib.ClusterSpec(tf_config.get('cluster', {}))
   return tf_config and 'master' not in cluster_spec.jobs
+
+
+def should_run_validation(validation_freq, epoch):
+  """Checks if validation should be run this epoch.
+
+  Arguments:
+    validation_freq: Integer or list. If an integer, specifies how many training
+      epochs to run before a new validation run is performed. If a list,
+      specifies the epochs on which to run validation.
+    epoch: Integer, the number of the training epoch just completed.
+
+  Returns:
+    Bool, True if validation should be run.
+
+  Raises:
+    ValueError: if `validation_freq` is an Integer and less than 1, or if
+    it is neither an Integer nor a Sequence.
+  """
+  # `epoch` is 0-indexed internally but 1-indexed in the public API.
+  one_indexed_epoch = epoch + 1
+
+  if isinstance(validation_freq, int):
+    if validation_freq < 1:
+      raise ValueError('`validation_freq` can not be less than 1.')
+    return one_indexed_epoch % validation_freq == 0
+
+  if not isinstance(validation_freq, collections.Container):
+    raise ValueError('`validation_freq` must be an Integer or '
+                     '`collections.Container` (e.g. list, tuple, etc.)')
+  return one_indexed_epoch in validation_freq
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt
index 0a81c3a663..283cc6a735 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt
@@ -175,11 +175,11 @@ tf_class {
   }
   member_method {
     name: "fit"
-    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'10\', \'1\', \'False\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'validation_freq\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'1\', \'10\', \'1\', \'False\'], "
   }
   member_method {
     name: "fit_generator"
-    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
+    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'validation_freq\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'1\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
   }
   member_method {
     name: "from_config"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt
index 2a9ceec30e..95e405aeba 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt
@@ -180,11 +180,11 @@ tf_class {
   }
   member_method {
     name: "fit"
-    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'10\', \'1\', \'False\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'validation_freq\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'1\', \'10\', \'1\', \'False\'], "
   }
   member_method {
     name: "fit_generator"
-    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
+    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'validation_freq\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'1\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
   }
   member_method {
     name: "from_config"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt
index 55bd285612..eb1ab1d9dd 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt
@@ -175,11 +175,11 @@ tf_class {
   }
   member_method {
     name: "fit"
-    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'10\', \'1\', \'False\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'validation_freq\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'1\', \'10\', \'1\', \'False\'], "
   }
   member_method {
     name: "fit_generator"
-    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
+    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'validation_freq\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'1\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
   }
   member_method {
     name: "from_config"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt
index a82424167e..c69cf28174 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt
@@ -180,11 +180,11 @@ tf_class {
   }
   member_method {
     name: "fit"
-    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'10\', \'1\', \'False\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'validation_freq\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'1\', \'10\', \'1\', \'False\'], "
   }
   member_method {
     name: "fit_generator"
-    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
+    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'validation_freq\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'1\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
   }
   member_method {
     name: "from_config"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt
index 0a81c3a663..283cc6a735 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt
@@ -175,11 +175,11 @@ tf_class {
   }
   member_method {
     name: "fit"
-    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'10\', \'1\', \'False\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'validation_freq\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'1\', \'10\', \'1\', \'False\'], "
   }
   member_method {
     name: "fit_generator"
-    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
+    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'validation_freq\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'1\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
   }
   member_method {
     name: "from_config"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt
index 2a9ceec30e..95e405aeba 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt
@@ -180,11 +180,11 @@ tf_class {
   }
   member_method {
     name: "fit"
-    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'10\', \'1\', \'False\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'validation_freq\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'1\', \'10\', \'1\', \'False\'], "
   }
   member_method {
     name: "fit_generator"
-    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
+    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'validation_freq\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'1\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
   }
   member_method {
     name: "from_config"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt
index d40a999bd8..c4726cf824 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt
@@ -180,11 +180,11 @@ tf_class {
   }
   member_method {
     name: "fit"
-    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'10\', \'1\', \'False\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'validation_freq\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'1\', \'10\', \'1\', \'False\'], "
   }
   member_method {
     name: "fit_generator"
-    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
+    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'validation_freq\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'1\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
   }
   member_method {
     name: "from_config"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt
index 55bd285612..eb1ab1d9dd 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt
@@ -175,11 +175,11 @@ tf_class {
   }
   member_method {
     name: "fit"
-    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'10\', \'1\', \'False\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'validation_freq\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'1\', \'10\', \'1\', \'False\'], "
   }
   member_method {
     name: "fit_generator"
-    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
+    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'validation_freq\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'1\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
   }
   member_method {
     name: "from_config"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt
index a82424167e..c69cf28174 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt
@@ -180,11 +180,11 @@ tf_class {
   }
   member_method {
     name: "fit"
-    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'10\', \'1\', \'False\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'validation_freq\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'1\', \'10\', \'1\', \'False\'], "
   }
   member_method {
     name: "fit_generator"
-    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
+    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'validation_freq\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'1\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
   }
   member_method {
     name: "from_config"
-- 
GitLab


From 661103ed87cdf0f5a792100b543006b71d62697a Mon Sep 17 00:00:00 2001
From: Revan Sopher <rsopher@google.com>
Date: Wed, 16 Jan 2019 14:52:57 -0800
Subject: [PATCH 0817/2345] Bump absl_py from 0.2.2 to 0.7.0, add absl_py's new
 dep enum34.

PiperOrigin-RevId: 229631203
---
 tensorflow/opensource_only.files               |  1 +
 tensorflow/workspace.bzl                       | 18 ++++++++++++++----
 .../systemlibs/absl_py.absl.testing.BUILD      |  6 +++++-
 third_party/systemlibs/enum34.BUILD            | 13 +++++++++++++
 4 files changed, 33 insertions(+), 5 deletions(-)
 create mode 100644 third_party/systemlibs/enum34.BUILD

diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index 03b5e5a73c..30dfce3c83 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -150,6 +150,7 @@ tensorflow/third_party/systemlibs/cython.BUILD
 tensorflow/third_party/systemlibs/double_conversion.BUILD
 tensorflow/third_party/systemlibs/zlib.BUILD
 tensorflow/third_party/systemlibs/jsoncpp.BUILD
+tensorflow/third_party/systemlibs/enum34.BUILD
 tensorflow/third_party/systemlibs/re2.BUILD
 tensorflow/third_party/systemlibs/lmdb.BUILD
 tensorflow/third_party/systemlibs/googleapis.BUILD
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index b8ef43d27e..6fcccd4147 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -317,19 +317,29 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
 
     tf_http_archive(
         name = "absl_py",
-        sha256 = "95160f778a62c7a60ddeadc7bf2d83f85a23a27359814aca12cf949e896fa82c",
-        strip_prefix = "abseil-py-pypi-v0.2.2",
+        sha256 = "595726be4bf3f7e6d64a1a255fa03717b693c01b913768abd52649cbb7ddf2bd",
+        strip_prefix = "abseil-py-pypi-v0.7.0",
         system_build_file = clean_dep("//third_party/systemlibs:absl_py.BUILD"),
         system_link_files = {
             "//third_party/systemlibs:absl_py.absl.flags.BUILD": "absl/flags/BUILD",
             "//third_party/systemlibs:absl_py.absl.testing.BUILD": "absl/testing/BUILD",
         },
         urls = [
-            "https://mirror.bazel.build/github.com/abseil/abseil-py/archive/pypi-v0.2.2.tar.gz",
-            "https://github.com/abseil/abseil-py/archive/pypi-v0.2.2.tar.gz",
+            "https://mirror.bazel.build/github.com/abseil/abseil-py/archive/pypi-v0.7.0.tar.gz",
+            "https://github.com/abseil/abseil-py/archive/pypi-v0.7.0.tar.gz",
         ],
     )
 
+    tf_http_archive(
+        name = "enum34_archive",
+        urls = [
+            "https://mirror.bazel.build/pypi.python.org/packages/bf/3e/31d502c25302814a7c2f1d3959d2a3b3f78e509002ba91aea64993936876/enum34-1.1.6.tar.gz",
+            "https://pypi.python.org/packages/bf/3e/31d502c25302814a7c2f1d3959d2a3b3f78e509002ba91aea64993936876/enum34-1.1.6.tar.gz",
+        ],
+        sha256 = "8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1",
+        build_file = clean_dep("//third_party/systemlibs:enum34.BUILD"),
+    )
+
     tf_http_archive(
         name = "org_python_pypi_backports_weakref",
         build_file = clean_dep("//third_party:backports_weakref.BUILD"),
diff --git a/third_party/systemlibs/absl_py.absl.testing.BUILD b/third_party/systemlibs/absl_py.absl.testing.BUILD
index c1b794c1e9..7629509ebb 100644
--- a/third_party/systemlibs/absl_py.absl.testing.BUILD
+++ b/third_party/systemlibs/absl_py.absl.testing.BUILD
@@ -2,6 +2,10 @@ licenses(["notice"])  # Apache 2.0
 
 py_library(
     name = "parameterized",
-    testonly = 1,
     visibility = ["//visibility:public"],
 )
+
+py_library(
+    name = "absltest",
+    visibility = ["//visiblity:public"],
+)
diff --git a/third_party/systemlibs/enum34.BUILD b/third_party/systemlibs/enum34.BUILD
new file mode 100644
index 0000000000..a5ac25592b
--- /dev/null
+++ b/third_party/systemlibs/enum34.BUILD
@@ -0,0 +1,13 @@
+# Description:
+#   enum34 provides a backport of the enum module for Python 2.
+
+licenses(["notice"])  # MIT
+
+exports_files(["LICENSE"])
+
+py_library(
+    name = "enum",
+    srcs = ["enum34-1.1.6/enum/__init__.py"],
+    srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
+)
-- 
GitLab


From 715f5c73cfaef2991d393024f019d7906ae27e8d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 14:53:05 -0800
Subject: [PATCH 0818/2345] Update NASCell to LayerRNNCell.

PiperOrigin-RevId: 229631237
---
 .../rnn/python/kernel_tests/rnn_cell_test.py  | 13 +++
 tensorflow/contrib/rnn/python/ops/rnn_cell.py | 80 +++++++++++--------
 2 files changed, 59 insertions(+), 34 deletions(-)

diff --git a/tensorflow/contrib/rnn/python/kernel_tests/rnn_cell_test.py b/tensorflow/contrib/rnn/python/kernel_tests/rnn_cell_test.py
index aa1d7d2b01..d7ee7fb8fa 100644
--- a/tensorflow/contrib/rnn/python/kernel_tests/rnn_cell_test.py
+++ b/tensorflow/contrib/rnn/python/kernel_tests/rnn_cell_test.py
@@ -29,7 +29,9 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import random_seed
+from tensorflow.python.framework import test_util
 from tensorflow.python.keras import initializers
+from tensorflow.python.keras import layers as keras_layers
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.keras import utils
 from tensorflow.python.ops import array_ops
@@ -763,6 +765,17 @@ class RNNCellTest(test.TestCase):
         self.assertEqual(new_h.shape[1], num_proj)
         self.assertAllClose(np.concatenate(res[1], axis=1), expected_state)
 
+  @test_util.run_in_graph_and_eager_modes
+  def testNASCellKerasRNN(self):
+    """Tests that NASCell works with keras RNN layer."""
+    cell = contrib_rnn_cell.NASCell(10)
+    seq_input = ops.convert_to_tensor(
+        np.random.rand(2, 3, 5), name="seq_input", dtype=dtypes.float32)
+    rnn_layer = keras_layers.RNN(cell=cell)
+    rnn_outputs = rnn_layer(seq_input)
+    self.evaluate([variables.global_variables_initializer()])
+    self.assertEqual(self.evaluate(rnn_outputs).shape, (2, 10))
+
   def testUGRNNCell(self):
     num_units = 2
     batch_size = 3
diff --git a/tensorflow/contrib/rnn/python/ops/rnn_cell.py b/tensorflow/contrib/rnn/python/ops/rnn_cell.py
index a2179cae0c..482e547a16 100644
--- a/tensorflow/contrib/rnn/python/ops/rnn_cell.py
+++ b/tensorflow/contrib/rnn/python/ops/rnn_cell.py
@@ -1462,7 +1462,7 @@ class LayerNormBasicLSTMCell(rnn_cell_impl.RNNCell):
     return new_h, new_state
 
 
-class NASCell(rnn_cell_impl.RNNCell):
+class NASCell(rnn_cell_impl.LayerRNNCell):
   """Neural Architecture Search (NAS) recurrent network cell.
 
   This implements the recurrent cell from the paper:
@@ -1475,23 +1475,28 @@ class NASCell(rnn_cell_impl.RNNCell):
   The class uses an optional projection layer.
   """
 
-  def __init__(self, num_units, num_proj=None, use_biases=False, reuse=None):
+  # NAS cell's architecture base.
+  _NAS_BASE = 8
+
+  def __init__(self, num_units, num_proj=None, use_bias=False, reuse=None,
+               **kwargs):
     """Initialize the parameters for a NAS cell.
 
     Args:
-      num_units: int, The number of units in the NAS cell
+      num_units: int, The number of units in the NAS cell.
       num_proj: (optional) int, The output dimensionality for the projection
         matrices.  If None, no projection is performed.
-      use_biases: (optional) bool, If True then use biases within the cell. This
+      use_bias: (optional) bool, If True then use biases within the cell. This
         is False by default.
       reuse: (optional) Python boolean describing whether to reuse variables
         in an existing scope.  If not `True`, and the existing scope already has
         the given variables, an error is raised.
+      **kwargs: Additional keyword arguments.
     """
-    super(NASCell, self).__init__(_reuse=reuse)
+    super(NASCell, self).__init__(_reuse=reuse, **kwargs)
     self._num_units = num_units
     self._num_proj = num_proj
-    self._use_biases = use_biases
+    self._use_bias = use_bias
     self._reuse = reuse
 
     if num_proj is not None:
@@ -1509,6 +1514,33 @@ class NASCell(rnn_cell_impl.RNNCell):
   def output_size(self):
     return self._output_size
 
+  def build(self, inputs_shape):
+    input_size = tensor_shape.dimension_value(
+        tensor_shape.TensorShape(inputs_shape).with_rank(2)[1])
+    if input_size is None:
+      raise ValueError("Could not infer input size from inputs.get_shape()[-1]")
+
+    num_proj = self._num_units if self._num_proj is None else self._num_proj
+
+    # Variables for the NAS cell. `recurrent_kernel` is all matrices multiplying
+    # the hiddenstate and `kernel` is all matrices multiplying the inputs.
+    self.recurrent_kernel = self.add_variable(
+        "recurrent_kernel", [num_proj, self._NAS_BASE * self._num_units])
+    self.kernel = self.add_variable(
+        "kernel", [input_size, self._NAS_BASE * self._num_units])
+
+    if self._use_bias:
+      self.bias = self.add_variable("bias",
+                                    shape=[self._NAS_BASE * self._num_units],
+                                    initializer=init_ops.zeros_initializer)
+
+    # Projection layer if specified
+    if self._num_proj is not None:
+      self.projection_weights = self.add_variable(
+          "projection_weights", [self._num_units, self._num_proj])
+
+    self.built = True
+
   def call(self, inputs, state):
     """Run one step of NAS Cell.
 
@@ -1535,38 +1567,20 @@ class NASCell(rnn_cell_impl.RNNCell):
     tanh = math_ops.tanh
     relu = nn_ops.relu
 
-    num_proj = self._num_units if self._num_proj is None else self._num_proj
-
     (c_prev, m_prev) = state
 
-    dtype = inputs.dtype
-    input_size = inputs.get_shape().with_rank(2).dims[1]
-    if input_size.value is None:
-      raise ValueError("Could not infer input size from inputs.get_shape()[-1]")
-    # Variables for the NAS cell. W_m is all matrices multiplying the
-    # hiddenstate and W_inputs is all matrices multiplying the inputs.
-    concat_w_m = vs.get_variable("recurrent_kernel",
-                                 [num_proj, 8 * self._num_units], dtype)
-    concat_w_inputs = vs.get_variable(
-        "kernel", [input_size.value, 8 * self._num_units], dtype)
-
-    m_matrix = math_ops.matmul(m_prev, concat_w_m)
-    inputs_matrix = math_ops.matmul(inputs, concat_w_inputs)
-
-    if self._use_biases:
-      b = vs.get_variable(
-          "bias",
-          shape=[8 * self._num_units],
-          initializer=init_ops.zeros_initializer(),
-          dtype=dtype)
-      m_matrix = nn_ops.bias_add(m_matrix, b)
+    m_matrix = math_ops.matmul(m_prev, self.recurrent_kernel)
+    inputs_matrix = math_ops.matmul(inputs, self.kernel)
+
+    if self._use_bias:
+      m_matrix = nn_ops.bias_add(m_matrix, self.bias)
 
     # The NAS cell branches into 8 different splits for both the hiddenstate
     # and the input
     m_matrix_splits = array_ops.split(
-        axis=1, num_or_size_splits=8, value=m_matrix)
+        axis=1, num_or_size_splits=self._NAS_BASE, value=m_matrix)
     inputs_matrix_splits = array_ops.split(
-        axis=1, num_or_size_splits=8, value=inputs_matrix)
+        axis=1, num_or_size_splits=self._NAS_BASE, value=inputs_matrix)
 
     # First layer
     layer1_0 = sigmoid(inputs_matrix_splits[0] + m_matrix_splits[0])
@@ -1598,9 +1612,7 @@ class NASCell(rnn_cell_impl.RNNCell):
 
     # Projection layer if specified
     if self._num_proj is not None:
-      concat_w_proj = vs.get_variable("projection_weights",
-                                      [self._num_units, self._num_proj], dtype)
-      new_m = math_ops.matmul(new_m, concat_w_proj)
+      new_m = math_ops.matmul(new_m, self.projection_weights)
 
     new_state = rnn_cell_impl.LSTMStateTuple(new_c, new_m)
     return new_m, new_state
-- 
GitLab


From f98110b559362b086409438b00f5b73c9d870b60 Mon Sep 17 00:00:00 2001
From: Blake Hechtman <blakehechtman@google.com>
Date: Wed, 16 Jan 2019 14:54:24 -0800
Subject: [PATCH 0819/2345] [TF] Use compare and cast instead of floor for
 dropout.

PiperOrigin-RevId: 229631446
---
 .../layers/python/layers/layers_test.py       |  2 +-
 .../python/util/parse_layer_parameters.py     | 23 ++++++++++---------
 .../python/keras/engine/topology_test.py      |  6 ++---
 .../python/keras/layers/wrappers_test.py      |  2 +-
 tensorflow/python/ops/nn_ops.py               | 14 +++++------
 5 files changed, 23 insertions(+), 24 deletions(-)

diff --git a/tensorflow/contrib/layers/python/layers/layers_test.py b/tensorflow/contrib/layers/python/layers/layers_test.py
index d791418c9d..1c0088186c 100644
--- a/tensorflow/contrib/layers/python/layers/layers_test.py
+++ b/tensorflow/contrib/layers/python/layers/layers_test.py
@@ -1356,7 +1356,7 @@ class DropoutTest(test.TestCase):
     with self.cached_session():
       images = np.random.uniform(size=(5, height, width, 3))
       output = _layers.dropout(images)
-      self.assertEqual(output.op.name, 'Dropout/dropout_1/mul')
+      self.assertEqual(output.op.name, 'Dropout/dropout_1/mul_1')
       output.get_shape().assert_is_compatible_with(
           ops.convert_to_tensor(images).get_shape())
 
diff --git a/tensorflow/contrib/receptive_field/python/util/parse_layer_parameters.py b/tensorflow/contrib/receptive_field/python/util/parse_layer_parameters.py
index 0e3c46f17d..92ae1021bc 100644
--- a/tensorflow/contrib/receptive_field/python/util/parse_layer_parameters.py
+++ b/tensorflow/contrib/receptive_field/python/util/parse_layer_parameters.py
@@ -27,7 +27,8 @@ from tensorflow.python.platform import tf_logging as logging
 _UNCHANGED_RF_LAYER_OPS = [
     "Add", "BiasAdd", "Cast", "Ceil", "ConcatV2", "Const", "Floor",
     "FusedBatchNorm", "Identity", "Log", "Mul", "Pow", "RealDiv", "Relu",
-    "Relu6", "Round", "Rsqrt", "Softplus", "Sub", "VariableV2", "LRN"
+    "Relu6", "Round", "Rsqrt", "Softplus", "Sub", "VariableV2", "LRN",
+    "GreaterEqual"
 ]
 
 # Different ways in which padding modes may be spelled.
@@ -276,11 +277,11 @@ def get_layer_params(node, name_to_node, input_resolution=None, force=False):
     kernel_size_x, kernel_size_y = _conv_kernel_size(node, name_to_node)
     # Compute the padding for this node separately for each direction.
     total_padding_x, padding_x = _padding_size_conv_pool(
-        node, kernel_size_x, stride_x, input_resolution[1]
-        if input_resolution is not None else None)
+        node, kernel_size_x, stride_x,
+        input_resolution[1] if input_resolution is not None else None)
     total_padding_y, padding_y = _padding_size_conv_pool(
-        node, kernel_size_y, stride_y, input_resolution[0]
-        if input_resolution is not None else None)
+        node, kernel_size_y, stride_y,
+        input_resolution[0] if input_resolution is not None else None)
   elif node.op == "Pad":
     # Kernel and stride are simply 1 in this case.
     kernel_size_x = 1
@@ -294,11 +295,11 @@ def get_layer_params(node, name_to_node, input_resolution=None, force=False):
     kernel_size_x, kernel_size_y = _pool_kernel_size(node, name_to_node)
     # Compute the padding for this node separately for each direction.
     total_padding_x, padding_x = _padding_size_conv_pool(
-        node, kernel_size_x, stride_x, input_resolution[1]
-        if input_resolution is not None else None)
+        node, kernel_size_x, stride_x,
+        input_resolution[1] if input_resolution is not None else None)
     total_padding_y, padding_y = _padding_size_conv_pool(
-        node, kernel_size_y, stride_y, input_resolution[0]
-        if input_resolution is not None else None)
+        node, kernel_size_y, stride_y,
+        input_resolution[0] if input_resolution is not None else None)
   elif node.op in _UNCHANGED_RF_LAYER_OPS:
     # These nodes do not modify the RF parameters.
     kernel_size_x = 1
@@ -320,7 +321,7 @@ def get_layer_params(node, name_to_node, input_resolution=None, force=False):
       total_padding_y = None
       padding_y = None
     else:
-      raise ValueError("Unknown layer for operation '%s': %s" % (node.name,
-                                                                 node.op))
+      raise ValueError(
+          "Unknown layer for operation '%s': %s" % (node.name, node.op))
   return (kernel_size_x, kernel_size_y, stride_x, stride_y, padding_x,
           padding_y, total_padding_x, total_padding_y)
diff --git a/tensorflow/python/keras/engine/topology_test.py b/tensorflow/python/keras/engine/topology_test.py
index 9022dc9481..951988d852 100644
--- a/tensorflow/python/keras/engine/topology_test.py
+++ b/tensorflow/python/keras/engine/topology_test.py
@@ -358,17 +358,17 @@ class TopologyConstructionTest(keras_parameterized.TestCase):
     x = keras.layers.Dropout(0.5)(x, training=True)
     model = keras.models.Model(inp, x)
     # Would be `dropout/cond/Merge` by default
-    self.assertTrue(model.output.op.name.endswith('dropout/mul'))
+    self.assertTrue(model.output.op.name.endswith('dropout/mul_1'))
 
     # Test that argument is kept when applying the model
     inp2 = keras.layers.Input(shape=(2,))
     out2 = model(inp2)
-    self.assertTrue(out2.op.name.endswith('dropout/mul'))
+    self.assertTrue(out2.op.name.endswith('dropout/mul_1'))
 
     # Test that argument is kept after loading a model
     config = model.get_config()
     model = keras.models.Model.from_config(config)
-    self.assertTrue(model.output.op.name.endswith('dropout/mul'))
+    self.assertTrue(model.output.op.name.endswith('dropout/mul_1'))
 
   def test_node_construction(self):
     # test basics
diff --git a/tensorflow/python/keras/layers/wrappers_test.py b/tensorflow/python/keras/layers/wrappers_test.py
index 00e5276d6a..313a2193eb 100644
--- a/tensorflow/python/keras/layers/wrappers_test.py
+++ b/tensorflow/python/keras/layers/wrappers_test.py
@@ -159,7 +159,7 @@ class TimeDistributedTest(test.TestCase):
       np.random.seed(1234)
       x = keras.layers.Input(shape=(3, 2))
       y = keras.layers.TimeDistributed(
-          keras.layers.Dropout(.999))(x, training=True)
+          keras.layers.Dropout(.9999))(x, training=True)
       model = keras.models.Model(x, y)
       y = model.predict(np.random.random((10, 3, 2)))
       self.assertAllClose(np.mean(y), 0., atol=1e-1, rtol=1e-1)
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index f71fcef130..25609f9efa 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -3292,15 +3292,13 @@ def dropout_v2(x, rate, noise_shape=None, seed=None, name=None):  # pylint: disa
         return x
 
     noise_shape = _get_noise_shape(x, noise_shape)
-
-    keep_prob = 1 - rate
-    # uniform [keep_prob, 1.0 + keep_prob)
-    random_tensor = keep_prob
-    random_tensor += random_ops.random_uniform(
+    # Sample a uniform distribution on [0.0, 1.0) and select values larger than
+    # rate.
+    random_tensor = random_ops.random_uniform(
         noise_shape, seed=seed, dtype=x.dtype)
-    # 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob)
-    binary_tensor = math_ops.floor(random_tensor)
-    ret = math_ops.divide(x, keep_prob) * binary_tensor
+    keep_prob = 1 - rate
+    ret = (1 / keep_prob) * math_ops.cast(keep_prob >= random_tensor,
+                                          x.dtype) * x
     if not context.executing_eagerly():
       ret.set_shape(x.get_shape())
     return ret
-- 
GitLab


From 9850eea1c83422c90a84c5c520070577becec0a5 Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl@google.com>
Date: Wed, 16 Jan 2019 14:58:58 -0800
Subject: [PATCH 0820/2345] Add monitoring for file IO time and TF graph init
 time.

PiperOrigin-RevId: 229632265
---
 tensorflow/cc/saved_model/loader.cc | 43 +++++++++++++++++++++++------
 1 file changed, 35 insertions(+), 8 deletions(-)

diff --git a/tensorflow/cc/saved_model/loader.cc b/tensorflow/cc/saved_model/loader.cc
index 85d3dd01fa..aaf06a74c6 100644
--- a/tensorflow/cc/saved_model/loader.cc
+++ b/tensorflow/cc/saved_model/loader.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include "tensorflow/cc/saved_model/reader.h"
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/monitoring/counter.h"
+#include "tensorflow/core/lib/monitoring/sampler.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/env.h"
@@ -42,9 +43,28 @@ auto* load_latency = monitoring::Counter<1>::New(
     "/tensorflow/cc/saved_model/load_latency",
     "Latency in microseconds for SavedModels that were successfully loaded.",
     "model_path");
+auto* load_latency_by_stage = monitoring::Sampler<2>::New(
+    {
+        "/tensorflow/cc/saved_model/load_latency_by_stage",  // metric name
+        "Distribution of wall time spent (in microseconds) in each stage "
+        "(restore graph from disk, run init graph op, etc) when loading the "
+        "model",
+        "model_path",
+        "stage",
+    },
+    // Scale of 10, power of 1.5 with bucket count 36 (~20 minutes).
+    monitoring::Buckets::Exponential(10, 1.5, 36));
+
 constexpr char kLoadAttemptFail[] = "fail";
 constexpr char kLoadAttemptSuccess[] = "success";
 
+uint64 GetLatencyMicroseconds(const uint64 start_microseconds) {
+  const uint64 end_microseconds = Env::Default()->NowMicros();
+  // Avoid clock skew.
+  if (end_microseconds < start_microseconds) return 0;
+  return end_microseconds - start_microseconds;
+}
+
 Status LoadMetaGraphIntoSession(const MetaGraphDef& meta_graph_def,
                                 const SessionOptions& session_options,
                                 std::unique_ptr<Session>* session) {
@@ -242,6 +262,7 @@ Status LoadSavedModelInternal(const SessionOptions& session_options,
                               const string& export_dir,
                               const std::unordered_set<string>& tags,
                               SavedModelBundle* const bundle) {
+  const uint64 read_start_microseconds = Env::Default()->NowMicros();
   TF_RETURN_IF_ERROR(ReadMetaGraphDefFromSavedModel(export_dir, tags,
                                                     &bundle->meta_graph_def));
 
@@ -256,12 +277,23 @@ Status LoadSavedModelInternal(const SessionOptions& session_options,
                  bundle->meta_graph_def.saver_def().restore_op_name(),
                  bundle->meta_graph_def.saver_def().filename_tensor_name(),
                  asset_file_defs, bundle->session.get()));
+  // Record walltime spent in restoring graph from disk, but postpone metric
+  // increments until graph init finishes.
+  const uint64 restore_graph_walltime =
+      GetLatencyMicroseconds(read_start_microseconds);
+
+  const uint64 graph_init_start_microseconds = Env::Default()->NowMicros();
   string init_op_name;
   TF_RETURN_IF_ERROR(
       GetInitOp(export_dir, bundle->meta_graph_def, &init_op_name));
   TF_RETURN_IF_ERROR(RunInitOp(run_options, export_dir, bundle->meta_graph_def,
                                asset_file_defs, bundle->session.get(),
                                init_op_name));
+  load_latency_by_stage->GetCell(export_dir, "restore_graph")
+      ->Add(restore_graph_walltime);
+  // Record wall time spent in init op.
+  load_latency_by_stage->GetCell(export_dir, "init_graph")
+      ->Add(GetLatencyMicroseconds(graph_init_start_microseconds));
   return Status::OK();
 }
 
@@ -275,16 +307,10 @@ Status LoadSavedModel(const SessionOptions& session_options,
   const uint64 start_microseconds = Env::Default()->NowMicros();
   const Status status = LoadSavedModelInternal(session_options, run_options,
                                                export_dir, tags, bundle);
-  const uint64 load_latency_microsecs = [&]() -> uint64 {
-    const uint64 end_microseconds = Env::Default()->NowMicros();
-    // Avoid clock skew.
-    if (end_microseconds < start_microseconds) return 0;
-    return end_microseconds - start_microseconds;
-  }();
   auto log_and_count = [&](const string& status_str) {
     LOG(INFO) << "SavedModel load for tags { " << str_util::Join(tags, " ")
               << " }; Status: " << status_str << ". Took "
-              << load_latency_microsecs << " microseconds.";
+              << GetLatencyMicroseconds(start_microseconds) << " microseconds.";
     load_attempt_count->GetCell(export_dir, status_str)->IncrementBy(1);
   };
   if (status.ok()) {
@@ -292,7 +318,8 @@ Status LoadSavedModel(const SessionOptions& session_options,
   } else {
     log_and_count(kLoadAttemptFail);
   }
-  load_latency->GetCell(export_dir)->IncrementBy(load_latency_microsecs);
+  load_latency->GetCell(export_dir)
+      ->IncrementBy(GetLatencyMicroseconds(start_microseconds));
   return status;
 }
 
-- 
GitLab


From 29b51d42ba4950658c566a0ca531b799e019ef7f Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Wed, 16 Jan 2019 15:10:08 -0800
Subject: [PATCH 0821/2345] Update tests to use the new distribute_py_test
 build rule.

PiperOrigin-RevId: 229634446
---
 tensorflow/contrib/distribute/python/BUILD | 66 +++++++---------------
 1 file changed, 21 insertions(+), 45 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 3a4f23e597..6ae3ec7fb0 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -344,10 +344,14 @@ cuda_py_test(
     ],
 )
 
-py_library(
-    name = "minimize_loss_test_lib",
-    testonly = 1,
+distribute_py_test(
+    name = "minimize_loss_test",
     srcs = ["minimize_loss_test.py"],
+    main = "minimize_loss_test.py",
+    tags = [
+        "multi_and_single_gpu",
+        "no_pip",
+    ],
     deps = [
         ":combinations",
         ":mirrored_strategy",
@@ -367,18 +371,6 @@ py_library(
     ],
 )
 
-cuda_py_test(
-    name = "minimize_loss_test",
-    srcs = ["minimize_loss_test.py"],
-    additional_deps = [
-        ":minimize_loss_test_lib",
-    ],
-    tags = [
-        "multi_and_single_gpu",
-        "no_pip",
-    ],
-)
-
 cuda_py_test(
     name = "moving_averages_test",
     srcs = ["moving_averages_test.py"],
@@ -501,10 +493,14 @@ py_library(
     ],
 )
 
-py_library(
-    name = "step_fn_test_lib",
-    testonly = 1,
+distribute_py_test(
+    name = "step_fn_test",
     srcs = ["step_fn_test.py"],
+    main = "step_fn_test.py",
+    tags = [
+        "multi_and_single_gpu",
+        "no_pip",
+    ],
     deps = [
         ":combinations",
         ":single_loss_example",
@@ -517,18 +513,6 @@ py_library(
     ],
 )
 
-cuda_py_test(
-    name = "step_fn_test",
-    srcs = ["step_fn_test.py"],
-    additional_deps = [
-        ":step_fn_test_lib",
-    ],
-    tags = [
-        "multi_and_single_gpu",
-        "no_pip",
-    ],
-)
-
 py_library(
     name = "monitor",
     srcs = ["monitor.py"],
@@ -766,10 +750,14 @@ distribute_py_test(
     ],
 )
 
-py_library(
-    name = "metrics_v1_test_lib",
-    testonly = 1,
+distribute_py_test(
+    name = "metrics_v1_test",
     srcs = ["metrics_v1_test.py"],
+    main = "metrics_v1_test.py",
+    tags = [
+        "multi_and_single_gpu",
+        "no_pip",
+    ],
     deps = [
         ":combinations",
         "//tensorflow/python:math_ops",
@@ -781,18 +769,6 @@ py_library(
     ],
 )
 
-cuda_py_test(
-    name = "metrics_v1_test",
-    srcs = ["metrics_v1_test.py"],
-    additional_deps = [
-        ":metrics_v1_test_lib",
-    ],
-    tags = [
-        "multi_and_single_gpu",
-        "no_pip",
-    ],
-)
-
 cuda_py_test(
     name = "warm_starting_util_test",
     size = "medium",
-- 
GitLab


From e0d156b3bfef7388b31b35793a2add7b0a5fab6d Mon Sep 17 00:00:00 2001
From: Yifei Feng <yifeif@google.com>
Date: Wed, 16 Jan 2019 15:10:22 -0800
Subject: [PATCH 0822/2345] [WIP] Part II of #22562: Make TensorFlow core
 dynamically load cuda driver instead of static linking.

PiperOrigin-RevId: 229634483
---
 .../gpu/gpu_cudamalloc_allocator.cc           |   6 +-
 tensorflow/stream_executor/build_defs.bzl     |   3 +-
 tensorflow/stream_executor/cuda/BUILD         |   7 +-
 .../stream_executor/cuda/cuda_driver.cc       | 213 ++++++++++--------
 .../cuda/cuda_driver_wrapper.h                | 143 ++++++++++++
 .../stream_executor/cuda/cuda_gpu_executor.cc |   5 +-
 6 files changed, 277 insertions(+), 100 deletions(-)
 create mode 100644 tensorflow/stream_executor/cuda/cuda_driver_wrapper.h

diff --git a/tensorflow/core/common_runtime/gpu/gpu_cudamalloc_allocator.cc b/tensorflow/core/common_runtime/gpu/gpu_cudamalloc_allocator.cc
index d85ca8892f..4be1bbb7df 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_cudamalloc_allocator.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_cudamalloc_allocator.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #ifdef GOOGLE_CUDA
 #include "cuda/include/cuda.h"
 #include "tensorflow/stream_executor/cuda/cuda_activation.h"
+#include "tensorflow/stream_executor/cuda/cuda_driver_wrapper.h"
 #endif  // GOOGLE_CUDA
 
 #include "tensorflow/core/common_runtime/gpu/gpu_cudamalloc_allocator.h"
@@ -41,7 +42,7 @@ void* GPUcudaMallocAllocator::AllocateRaw(size_t alignment, size_t num_bytes) {
   // allocate with cudaMalloc
   se::cuda::ScopedActivateExecutorContext scoped_activation{stream_exec_};
   CUdeviceptr rv = 0;
-  CUresult res = cuMemAlloc(&rv, num_bytes);
+  CUresult res = tensorflow::wrap::cuMemAlloc(&rv, num_bytes);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "cuMemAlloc failed to allocate " << num_bytes;
     return nullptr;
@@ -54,7 +55,8 @@ void* GPUcudaMallocAllocator::AllocateRaw(size_t alignment, size_t num_bytes) {
 void GPUcudaMallocAllocator::DeallocateRaw(void* ptr) {
 #ifdef GOOGLE_CUDA
   // free with cudaFree
-  CUresult res = cuMemFree(reinterpret_cast<CUdeviceptr>(ptr));
+  CUresult res =
+      tensorflow::wrap::cuMemFree(reinterpret_cast<CUdeviceptr>(ptr));
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "cuMemFree failed to free " << ptr;
   }
diff --git a/tensorflow/stream_executor/build_defs.bzl b/tensorflow/stream_executor/build_defs.bzl
index a7ddf5a0d7..717c13d113 100644
--- a/tensorflow/stream_executor/build_defs.bzl
+++ b/tensorflow/stream_executor/build_defs.bzl
@@ -4,8 +4,9 @@ def stream_executor_friends():
 def tf_additional_cuda_platform_deps():
   return []
 
+# Use dynamic loading, therefore should be empty.
 def tf_additional_cuda_driver_deps():
-  return ["@local_config_cuda//cuda:cuda_driver"]
+  return []
 
 def tf_additional_cudnn_plugin_deps():
   return []
diff --git a/tensorflow/stream_executor/cuda/BUILD b/tensorflow/stream_executor/cuda/BUILD
index fcfda1337e..77a1851047 100644
--- a/tensorflow/stream_executor/cuda/BUILD
+++ b/tensorflow/stream_executor/cuda/BUILD
@@ -74,7 +74,10 @@ cc_library(
 cc_library(
     name = "cuda_driver",
     srcs = if_cuda_is_configured(["cuda_driver.cc"]),
-    hdrs = if_cuda_is_configured(["cuda_driver.h"]),
+    hdrs = if_cuda_is_configured([
+        "cuda_driver.h",
+        "cuda_driver_wrapper.h",
+    ]),
     deps = if_cuda_is_configured([
         ":cuda_diagnostics",
         "@com_google_absl//absl/base",
@@ -84,6 +87,7 @@ cc_library(
         "//tensorflow/stream_executor:device_options",
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
+        "//tensorflow/stream_executor/platform:dso_loader",
     ] + tf_additional_cuda_driver_deps()),
 )
 
@@ -314,6 +318,7 @@ cc_library(
         "//tensorflow/stream_executor:timer",
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
+        "//tensorflow/stream_executor/platform:dso_loader",
     ]),
     alwayslink = True,
 )
diff --git a/tensorflow/stream_executor/cuda/cuda_driver.cc b/tensorflow/stream_executor/cuda/cuda_driver.cc
index 7aea66e3af..c39e4f59eb 100644
--- a/tensorflow/stream_executor/cuda/cuda_driver.cc
+++ b/tensorflow/stream_executor/cuda/cuda_driver.cc
@@ -25,6 +25,7 @@ limitations under the License.
 #include "absl/container/inlined_vector.h"
 #include "absl/strings/str_cat.h"
 #include "tensorflow/stream_executor/cuda/cuda_diagnostics.h"
+#include "tensorflow/stream_executor/cuda/cuda_driver_wrapper.h"
 #include "tensorflow/stream_executor/lib/env.h"
 #include "tensorflow/stream_executor/lib/error.h"
 #include "tensorflow/stream_executor/lib/human_readable.h"
@@ -108,11 +109,11 @@ class CreatedContexts {
 // Formats CUresult to output prettified values into a log stream.
 string ToString(CUresult result) {
   const char *error_name;
-  if (cuGetErrorName(result, &error_name)) {
+  if (tensorflow::wrap::cuGetErrorName(result, &error_name)) {
     return absl::StrCat("UNKNOWN ERROR (", static_cast<int>(result), ")");
   }
   const char *error_string;
-  if (cuGetErrorString(result, &error_string)) {
+  if (tensorflow::wrap::cuGetErrorString(result, &error_string)) {
     return error_name;
   }
   return absl::StrCat(error_name, ": ", error_string);
@@ -167,7 +168,7 @@ namespace {
 
 // Call cuCtxtSynchronize and crash if it doesn't succeed.
 void SynchronizeOrDie() {
-  auto res = cuCtxSynchronize();
+  auto res = tensorflow::wrap::cuCtxSynchronize();
   if (res != CUDA_SUCCESS) {
     LOG(FATAL) << "Synchronize found "
                << ToString(res) << " :: " << port::CurrentStackTrace();
@@ -203,7 +204,8 @@ ScopedActivateContext::ScopedActivateContext(CudaContext* cuda_context) {
   to_restore_ = (tls->depth == 1 ? nullptr : tls->context);
 
   // Set the context and update thread local.
-  CHECK_EQ(CUDA_SUCCESS, cuCtxSetCurrent(cuda_context->context()));
+  CHECK_EQ(CUDA_SUCCESS,
+           tensorflow::wrap::cuCtxSetCurrent(cuda_context->context()));
   tls->id = cuda_context->id();
   tls->context = cuda_context;
 }
@@ -228,7 +230,8 @@ ScopedActivateContext::~ScopedActivateContext() {
   }
 
   // Set context and update thread local.
-  CHECK_EQ(CUDA_SUCCESS, cuCtxSetCurrent(to_restore_->context()));
+  CHECK_EQ(CUDA_SUCCESS,
+           tensorflow::wrap::cuCtxSetCurrent(to_restore_->context()));
   tls->id = to_restore_->id();
   tls->context = to_restore_;
 }
@@ -290,7 +293,7 @@ static port::Status InternalInit() {
   if (FLAGS_gpuexec_cuda_driver_inject_init_error) {
     LOG(ERROR) << "injecting CUDA init error; initialization will fail";
   } else {
-    res = cuInit(0 /* = flags */);
+    res = tensorflow::wrap::cuInit(0 /* = flags */);
   }
 
   if (res == CUDA_SUCCESS) {
@@ -323,7 +326,7 @@ static port::Status InternalInit() {
 
 /* static */ port::Status CUDADriver::GetDevice(int device_ordinal,
                                                 CUdevice *device) {
-  CUresult res = cuDeviceGet(device, device_ordinal);
+  CUresult res = tensorflow::wrap::cuDeviceGet(device, device_ordinal);
   if (res == CUDA_SUCCESS) {
     return port::Status::OK();
   }
@@ -337,7 +340,8 @@ static port::Status InternalInit() {
                                             string *device_name) {
   static const size_t kCharLimit = 64;
   absl::InlinedVector<char, 4> chars(kCharLimit);
-  CUresult res = cuDeviceGetName(chars.begin(), kCharLimit - 1, device);
+  CUresult res =
+      tensorflow::wrap::cuDeviceGetName(chars.begin(), kCharLimit - 1, device);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to get device name for " << device << ": "
                << ToString(res);
@@ -388,9 +392,9 @@ bool DeviceOptionsToContextFlags(const DeviceOptions &device_options,
 
   unsigned int former_primary_context_flags;
   int former_primary_context_is_active;
-  CHECK_EQ(CUDA_SUCCESS,
-           cuDevicePrimaryCtxGetState(device, &former_primary_context_flags,
-                                      &former_primary_context_is_active));
+  CHECK_EQ(CUDA_SUCCESS, tensorflow::wrap::cuDevicePrimaryCtxGetState(
+                             device, &former_primary_context_flags,
+                             &former_primary_context_is_active));
   if (former_primary_context_flags != flags) {
     if (former_primary_context_is_active) {
       LOG(ERROR)
@@ -398,15 +402,16 @@ bool DeviceOptionsToContextFlags(const DeviceOptions &device_options,
           << former_primary_context_flags << ") than the desired flag set ("
           << flags << ").";
     } else {
-      CHECK_EQ(CUDA_SUCCESS, cuDevicePrimaryCtxSetFlags(device, flags));
+      CHECK_EQ(CUDA_SUCCESS,
+               tensorflow::wrap::cuDevicePrimaryCtxSetFlags(device, flags));
     }
   }
 
   former_context = CUDADriver::CurrentContextOrDie();
-  res = cuDevicePrimaryCtxRetain(&new_context, device);
+  res = tensorflow::wrap::cuDevicePrimaryCtxRetain(&new_context, device);
   if (former_context != nullptr) {
     CUdevice former_device;
-    if (cuCtxGetDevice(&former_device) == CUDA_SUCCESS) {
+    if (tensorflow::wrap::cuCtxGetDevice(&former_device) == CUDA_SUCCESS) {
       if (former_device == device) {
         if (former_context == new_context) {
           VLOG(2) << "The primary context " << former_context << " for device "
@@ -425,7 +430,7 @@ bool DeviceOptionsToContextFlags(const DeviceOptions &device_options,
                  << former_context;
     }
   }
-  CHECK_EQ(CUDA_SUCCESS, cuCtxSetCurrent(former_context));
+  CHECK_EQ(CUDA_SUCCESS, tensorflow::wrap::cuCtxSetCurrent(former_context));
 
   if (res == CUDA_SUCCESS) {
     *context = CreatedContexts::Add(new_context);
@@ -454,12 +459,12 @@ bool DeviceOptionsToContextFlags(const DeviceOptions &device_options,
     return;
   }
   CUcontext former_context = CurrentContext();
-  CUresult res = cuCtxSetCurrent(context->context());
+  CUresult res = tensorflow::wrap::cuCtxSetCurrent(context->context());
   CUdevice device;
-  cuCtxGetDevice(&device);
-  cuCtxSetCurrent(former_context);
+  tensorflow::wrap::cuCtxGetDevice(&device);
+  tensorflow::wrap::cuCtxSetCurrent(former_context);
 
-  res = cuDevicePrimaryCtxRelease(device);
+  res = tensorflow::wrap::cuDevicePrimaryCtxRelease(device);
 
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to release CUDA context; leaking: " << ToString(res);
@@ -471,7 +476,8 @@ bool DeviceOptionsToContextFlags(const DeviceOptions &device_options,
 /* static */ bool CUDADriver::FuncGetAttribute(CUfunction_attribute attribute,
                                                CUfunction func,
                                                int *attribute_value) {
-  CUresult res = cuFuncGetAttribute(attribute_value, attribute, func);
+  CUresult res =
+      tensorflow::wrap::cuFuncGetAttribute(attribute_value, attribute, func);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to query kernel attribute. kernel: " << func
                << ", attribute: " << attribute;
@@ -482,7 +488,7 @@ bool DeviceOptionsToContextFlags(const DeviceOptions &device_options,
 
 /* static */ bool CUDADriver::FuncSetCacheConfig(CUfunction function,
                                                  CUfunc_cache cache_config) {
-  CUresult res = cuFuncSetCacheConfig(function, cache_config);
+  CUresult res = tensorflow::wrap::cuFuncSetCacheConfig(function, cache_config);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to set CUDA kernel cache config. kernel: " << function
                << ", config: " << cache_config << ", result: " << ToString(res);
@@ -496,10 +502,11 @@ bool DeviceOptionsToContextFlags(const DeviceOptions &device_options,
 CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   CUsharedconfig shared_mem_config;
   ScopedActivateContext activation(context);
-  CUresult result = cuCtxGetSharedMemConfig(&shared_mem_config);
+  CUresult result =
+      tensorflow::wrap::cuCtxGetSharedMemConfig(&shared_mem_config);
   if (result != CUDA_SUCCESS) {
     CUdevice device;
-    cuCtxGetDevice(&device);
+    tensorflow::wrap::cuCtxGetDevice(&device);
     LOG(ERROR) << "failed to get CUDA device shared memory config. "
                << "Context device ID: " << device
                << ", result: " << ToString(result);
@@ -513,10 +520,11 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
 /* static */ port::Status CUDADriver::ContextSetSharedMemConfig(
     CudaContext* context, CUsharedconfig shared_mem_config) {
   ScopedActivateContext activation(context);
-  CUresult result = cuCtxSetSharedMemConfig(shared_mem_config);
+  CUresult result =
+      tensorflow::wrap::cuCtxSetSharedMemConfig(shared_mem_config);
   if (result != CUDA_SUCCESS) {
     CUdevice device;
-    cuCtxGetDevice(&device);
+    tensorflow::wrap::cuCtxGetDevice(&device);
     LOG(ERROR) << "failed to set CUDA device shared memory config. "
                << "Context device ID: " << device
                << ", config: " << shared_mem_config
@@ -539,9 +547,9 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
           << " gdy: " << grid_dim_y << " gdz: " << grid_dim_z
           << " bdx: " << block_dim_x << " bdy: " << block_dim_y
           << " bdz: " << block_dim_z;
-  CUresult res = cuLaunchKernel(function, grid_dim_x, grid_dim_y, grid_dim_z,
-                                block_dim_x, block_dim_y, block_dim_z,
-                                shared_mem_bytes, stream, kernel_params, extra);
+  CUresult res = tensorflow::wrap::cuLaunchKernel(
+      function, grid_dim_x, grid_dim_y, grid_dim_z, block_dim_x, block_dim_y,
+      block_dim_z, shared_mem_bytes, stream, kernel_params, extra);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to launch CUDA kernel: " << function
                << "; result: " << ToString(res);
@@ -555,7 +563,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                 const char *cubin_bytes,
                                                 CUmodule *module) {
   ScopedActivateContext activation(context);
-  CUresult result = cuModuleLoadFatBinary(module, cubin_bytes);
+  CUresult result =
+      tensorflow::wrap::cuModuleLoadFatBinary(module, cubin_bytes);
   if (result != CUDA_SUCCESS) {
     return port::Status(port::error::INTERNAL,
                         "failed to load in-memory CUBIN: " + ToString(result));
@@ -598,8 +607,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
       // TODO(leary) Need to see if NVIDIA can expunge the leakiness in their
       // module loading: see http://b/13248943
 
-      res = cuModuleLoadDataEx(module, ptx_data, TF_ARRAYSIZE(options),
-                               options, option_values);
+      res = tensorflow::wrap::cuModuleLoadDataEx(
+          module, ptx_data, TF_ARRAYSIZE(options), options, option_values);
     }
 
     // The PTX JIT mutates the values in the option values array to reflect the
@@ -638,7 +647,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                      CUdeviceptr location,
                                                      uint8 value, size_t size) {
   ScopedActivateContext activation(context);
-  CUresult res = cuMemsetD8(location, value, size);
+  CUresult res = tensorflow::wrap::cuMemsetD8(location, value, size);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to memset memory: " << ToString(res);
     return false;
@@ -651,7 +660,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                       uint32 value,
                                                       size_t uint32_count) {
   ScopedActivateContext activation(context);
-  CUresult res = cuMemsetD32(location, value, uint32_count);
+  CUresult res = tensorflow::wrap::cuMemsetD32(location, value, uint32_count);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to memset memory: " << ToString(res);
     return false;
@@ -665,7 +674,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                       size_t uint32_count,
                                                       CUstream stream) {
   ScopedActivateContext activation(context);
-  CUresult res = cuMemsetD8Async(location, value, uint32_count, stream);
+  CUresult res =
+      tensorflow::wrap::cuMemsetD8Async(location, value, uint32_count, stream);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to enqueue async memset operation: " << ToString(res);
     return false;
@@ -680,7 +690,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                        size_t uint32_count,
                                                        CUstream stream) {
   ScopedActivateContext activation(context);
-  CUresult res = cuMemsetD32Async(location, value, uint32_count, stream);
+  CUresult res =
+      tensorflow::wrap::cuMemsetD32Async(location, value, uint32_count, stream);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to enqueue async memset operation: " << ToString(res);
     return false;
@@ -694,7 +705,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                 StreamCallback callback,
                                                 void *data) {
   // Note: flags param is required to be zero according to CUDA 6.0.
-  CUresult res = cuStreamAddCallback(stream, callback, data, 0 /* = flags */);
+  CUresult res = tensorflow::wrap::cuStreamAddCallback(stream, callback, data,
+                                                       0 /* = flags */);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "unable to add host callback: " << ToString(res);
     return false;
@@ -708,7 +720,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                 CUfunction *function) {
   ScopedActivateContext activated{context};
   CHECK(module != nullptr && kernel_name != nullptr);
-  CUresult res = cuModuleGetFunction(function, module, kernel_name);
+  CUresult res =
+      tensorflow::wrap::cuModuleGetFunction(function, module, kernel_name);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to get PTX kernel \"" << kernel_name
                << "\" from module: " << ToString(res);
@@ -726,7 +739,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   ScopedActivateContext activated{context};
   CHECK(module != nullptr && symbol_name != nullptr &&
         (dptr != nullptr || bytes != nullptr));
-  CUresult res = cuModuleGetGlobal(dptr, bytes, module, symbol_name);
+  CUresult res =
+      tensorflow::wrap::cuModuleGetGlobal(dptr, bytes, module, symbol_name);
   if (res != CUDA_SUCCESS) {
     // symbol may not be found in the current module, but it may reside in
     // another module.
@@ -741,7 +755,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
 /* static */ void CUDADriver::UnloadModule(CudaContext *context,
                                            CUmodule module) {
   ScopedActivateContext activated{context};
-  CUresult res = cuModuleUnload(module);
+  CUresult res = tensorflow::wrap::cuModuleUnload(module);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to unload module " << module
                << "; leaking: " << ToString(res);
@@ -752,7 +766,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
     CudaContext* context) {
   ScopedActivateContext activated{context};
   CUdevice device = -1;
-  CUresult result = cuCtxGetDevice(&device);
+  CUresult result = tensorflow::wrap::cuCtxGetDevice(&device);
   if (result == CUDA_SUCCESS) {
     return device;
   }
@@ -768,7 +782,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   // up synchronization with respect to memsets and any other things that have
   // to occur on the default stream?
   ScopedActivateContext activated{context};
-  CUresult res = cuStreamCreate(out, 0);
+  CUresult res = tensorflow::wrap::cuStreamCreate(out, 0);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "could not allocate CUDA stream for context "
                << context->context() << ": " << ToString(res);
@@ -787,7 +801,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   }
 
   ScopedActivateContext activated{context};
-  CUresult res = cuStreamDestroy(*stream);
+  CUresult res = tensorflow::wrap::cuStreamDestroy(*stream);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to destroy CUDA stream for context "
                << context->context() << ": " << ToString(res);
@@ -802,7 +816,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                               uint64 bytes) {
   ScopedActivateContext activated{context};
   CUdeviceptr result = 0;
-  CUresult res = cuMemAlloc(&result, bytes);
+  CUresult res = tensorflow::wrap::cuMemAlloc(&result, bytes);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to allocate "
                << port::HumanReadableNumBytes::ToString(bytes) << " (" << bytes
@@ -819,7 +833,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                void *location) {
   ScopedActivateContext activation(context);
   CUdeviceptr pointer = absl::bit_cast<CUdeviceptr>(location);
-  CUresult res = cuMemFree(pointer);
+  CUresult res = tensorflow::wrap::cuMemFree(pointer);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to free device memory at " << location
                << "; result: " << ToString(res);
@@ -834,7 +848,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   ScopedActivateContext activation(context);
   CUdeviceptr result = 0;
   // "Portable" memory is visible to all CUDA contexts. Safe for our use model.
-  CUresult res = cuMemAllocManaged(&result, bytes, CU_MEM_ATTACH_GLOBAL);
+  CUresult res =
+      tensorflow::wrap::cuMemAllocManaged(&result, bytes, CU_MEM_ATTACH_GLOBAL);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to alloc " << bytes
                << " bytes unified memory; result: " << ToString(res);
@@ -850,7 +865,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                       void *location) {
   ScopedActivateContext activation(context);
   CUdeviceptr pointer = absl::bit_cast<CUdeviceptr>(location);
-  CUresult res = cuMemFree(pointer);
+  CUresult res = tensorflow::wrap::cuMemFree(pointer);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to free unified memory at " << location
                << "; result: " << ToString(res);
@@ -865,7 +880,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   ScopedActivateContext activation(context);
   void *host_mem = nullptr;
   // "Portable" memory is visible to all CUDA contexts. Safe for our use model.
-  CUresult res = cuMemHostAlloc(&host_mem, bytes, CU_MEMHOSTALLOC_PORTABLE);
+  CUresult res = tensorflow::wrap::cuMemHostAlloc(&host_mem, bytes,
+                                                  CU_MEMHOSTALLOC_PORTABLE);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to alloc " << bytes
                << " bytes on host: " << ToString(res);
@@ -876,7 +892,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
 /* static */ void CUDADriver::HostDeallocate(CudaContext* context,
                                              void *location) {
   ScopedActivateContext activation(context);
-  CUresult res = cuMemFreeHost(location);
+  CUresult res = tensorflow::wrap::cuMemFreeHost(location);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "error deallocating host memory at " << location << ": "
                << ToString(res);
@@ -887,8 +903,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                            uint64 bytes) {
   ScopedActivateContext activation(context);
   // "Portable" memory is visible to all CUDA contexts. Safe for our use model.
-  CUresult res =
-      cuMemHostRegister(location, bytes, CU_MEMHOSTREGISTER_PORTABLE);
+  CUresult res = tensorflow::wrap::cuMemHostRegister(
+      location, bytes, CU_MEMHOSTREGISTER_PORTABLE);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "error registering host memory at " << location << ": "
                << ToString(res);
@@ -900,7 +916,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
 /* static */ bool CUDADriver::HostUnregister(CudaContext* context,
                                              void *location) {
   ScopedActivateContext activation(context);
-  CUresult res = cuMemHostUnregister(location);
+  CUresult res = tensorflow::wrap::cuMemHostUnregister(location);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "error unregistering host memory at " << location << ": "
                << ToString(res);
@@ -917,7 +933,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   }
 
   ScopedActivateContext activated{context};
-  CUresult res = cuEventDestroy(*event);
+  CUresult res = tensorflow::wrap::cuEventDestroy(*event);
   *event = nullptr;
 
   switch (res) {
@@ -941,7 +957,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                   CUevent event,
                                                   CUstream stream) {
   ScopedActivateContext activated{context};
-  CUresult res = cuEventRecord(event, stream);
+  CUresult res = tensorflow::wrap::cuEventRecord(event, stream);
   switch (res) {
     case CUDA_SUCCESS:
       return port::Status::OK();
@@ -962,7 +978,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
 /* static */ port::StatusOr<CUresult> CUDADriver::QueryEvent(
     CudaContext *context, CUevent event) {
   ScopedActivateContext activated{context};
-  CUresult res = cuEventQuery(event);
+  CUresult res = tensorflow::wrap::cuEventQuery(event);
   if (res != CUDA_SUCCESS && res != CUDA_ERROR_NOT_READY) {
     return port::Status(
         port::error::INTERNAL,
@@ -978,12 +994,12 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   ScopedActivateContext activated{context};
   // The stop event must have completed in order for cuEventElapsedTime to
   // work.
-  CUresult res = cuEventSynchronize(stop);
+  CUresult res = tensorflow::wrap::cuEventSynchronize(stop);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to synchronize the stop event: " << ToString(res);
     return false;
   }
-  res = cuEventElapsedTime(elapsed_milliseconds, start, stop);
+  res = tensorflow::wrap::cuEventElapsedTime(elapsed_milliseconds, start, stop);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to get elapsed time between events: "
                << ToString(res);
@@ -997,7 +1013,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                 CUstream stream,
                                                 CUevent event) {
   ScopedActivateContext activation(context);
-  CUresult res = cuStreamWaitEvent(stream, event, 0 /* = flags */);
+  CUresult res =
+      tensorflow::wrap::cuStreamWaitEvent(stream, event, 0 /* = flags */);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "could not wait stream on event: " << ToString(res);
     return false;
@@ -1008,7 +1025,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
 
 /* static */ bool CUDADriver::SynchronizeContext(CudaContext* context) {
   ScopedActivateContext activation(context);
-  CUresult res = cuCtxSynchronize();
+  CUresult res = tensorflow::wrap::cuCtxSynchronize();
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "could not synchronize on CUDA context: " << ToString(res)
                << " :: " << port::CurrentStackTrace();
@@ -1022,7 +1039,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                         CUstream stream) {
   ScopedActivateContext activated{context};
   CHECK(stream != nullptr);
-  CUresult res = cuStreamSynchronize(stream);
+  CUresult res = tensorflow::wrap::cuStreamSynchronize(stream);
   if (res != CUDA_SUCCESS) {
     port::Status status = port::InternalError(
         absl::StrCat("could not synchronize on CUDA stream: ", ToString(res)));
@@ -1038,7 +1055,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                            CUstream stream) {
   ScopedActivateContext activated{context};
   CHECK(stream != nullptr);
-  CUresult res = cuStreamQuery(stream);
+  CUresult res = tensorflow::wrap::cuStreamQuery(stream);
   if (res == CUDA_SUCCESS) {
     return true;
   }
@@ -1054,7 +1071,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                            CUdeviceptr gpu_src,
                                                            uint64 size) {
   ScopedActivateContext activation(context);
-  CUresult res = cuMemcpyDtoH(host_dst, gpu_src, size);
+  CUresult res = tensorflow::wrap::cuMemcpyDtoH(host_dst, gpu_src, size);
   if (res != CUDA_SUCCESS) {
     return port::InternalError(
         port::Printf("failed to synchronous memcpy from device to host: %s; "
@@ -1072,7 +1089,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                            const void *host_src,
                                                            uint64 size) {
   ScopedActivateContext activation(context);
-  CUresult res = cuMemcpyHtoD(gpu_dst, host_src, size);
+  CUresult res = tensorflow::wrap::cuMemcpyHtoD(gpu_dst, host_src, size);
   if (res != CUDA_SUCCESS) {
     return port::InternalError(port::Printf(
         "failed to synchronous memcpy from host to device: %s; GPU dst: %p;"
@@ -1089,7 +1106,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                            CUdeviceptr gpu_src,
                                                            uint64 size) {
   ScopedActivateContext activation(context);
-  CUresult res = cuMemcpyDtoD(gpu_dst, gpu_src, size);
+  CUresult res = tensorflow::wrap::cuMemcpyDtoD(gpu_dst, gpu_src, size);
   if (res != CUDA_SUCCESS) {
     return port::InternalError(port::Printf(
         "failed to synchronous memcpy from host to device: %s; GPU dst: %p; "
@@ -1107,7 +1124,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                     uint64 size,
                                                     CUstream stream) {
   ScopedActivateContext activation(context);
-  CUresult res = cuMemcpyDtoHAsync(host_dst, gpu_src, size, stream);
+  CUresult res =
+      tensorflow::wrap::cuMemcpyDtoHAsync(host_dst, gpu_src, size, stream);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << port::Printf(
         "failed to enqueue async memcpy from device to host: %s; host dst: %p; "
@@ -1128,7 +1146,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                     uint64 size,
                                                     CUstream stream) {
   ScopedActivateContext activation(context);
-  CUresult res = cuMemcpyHtoDAsync(gpu_dst, host_src, size, stream);
+  CUresult res =
+      tensorflow::wrap::cuMemcpyHtoDAsync(gpu_dst, host_src, size, stream);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << port::Printf(
         "failed to enqueue async memcpy from host to device: %s; GPU dst: %p; "
@@ -1148,7 +1167,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                     uint64 size,
                                                     CUstream stream) {
   ScopedActivateContext activation(context);
-  CUresult result = cuMemcpyDtoDAsync(gpu_dst, gpu_src, size, stream);
+  CUresult result =
+      tensorflow::wrap::cuMemcpyDtoDAsync(gpu_dst, gpu_src, size, stream);
   if (result != CUDA_SUCCESS) {
     LOG(ERROR) << port::Printf(
         "failed to enqueue async memcpy from device to device: %s"
@@ -1185,7 +1205,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   }
 
   ScopedActivateContext activated{context};
-  CUresult res = cuEventCreate(result, cuflags);
+  CUresult res = tensorflow::wrap::cuEventCreate(result, cuflags);
 
   if (res == CUDA_SUCCESS) {
     return port::Status::OK();
@@ -1201,7 +1221,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
 
 /* static */ int CUDADriver::GetDeviceCount() {
   int device_count = 0;
-  CUresult res = cuDeviceGetCount(&device_count);
+  CUresult res = tensorflow::wrap::cuDeviceGetCount(&device_count);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "could not retrieve CUDA device count: " << ToString(res);
     return 0;
@@ -1216,8 +1236,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
 /* static */ port::StatusOr<CudaContext*> CUDADriver::GetPointerContext(
     CUdeviceptr pointer) {
   CudaContext* context = nullptr;
-  CUresult result =
-      cuPointerGetAttribute(&context, CU_POINTER_ATTRIBUTE_CONTEXT, pointer);
+  CUresult result = tensorflow::wrap::cuPointerGetAttribute(
+      &context, CU_POINTER_ATTRIBUTE_CONTEXT, pointer);
   if (result == CUDA_SUCCESS) {
     CHECK(context != nullptr) << "success should entail non-null context";
     return context;
@@ -1232,8 +1252,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
 /* static */ port::StatusOr<MemorySpace> CUDADriver::GetPointerMemorySpace(
     CUdeviceptr pointer) {
   unsigned int value;
-  CUresult result =
-      cuPointerGetAttribute(&value, CU_POINTER_ATTRIBUTE_MEMORY_TYPE, pointer);
+  CUresult result = tensorflow::wrap::cuPointerGetAttribute(
+      &value, CU_POINTER_ATTRIBUTE_MEMORY_TYPE, pointer);
   if (result == CUDA_SUCCESS) {
     switch (value) {
       case CU_MEMORYTYPE_DEVICE:
@@ -1256,7 +1276,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
 /* static */ port::Status CUDADriver::GetPointerAddressRange(CUdeviceptr dptr,
                                                              CUdeviceptr *base,
                                                              size_t *size) {
-  CUresult result = cuMemGetAddressRange(base, size, dptr);
+  CUresult result = tensorflow::wrap::cuMemGetAddressRange(base, size, dptr);
   if (result == CUDA_SUCCESS) {
     return port::Status::OK();
   } else if (result == CUDA_ERROR_NOT_FOUND) {
@@ -1291,7 +1311,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   *cc_major = 0;
   *cc_minor = 0;
 
-  CUresult res = cuDeviceGetAttribute(
+  CUresult res = tensorflow::wrap::cuDeviceGetAttribute(
       cc_major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, device);
   if (res != CUDA_SUCCESS) {
     return port::Status(
@@ -1301,7 +1321,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
             ToString(res).c_str(), device));
   }
 
-  res = cuDeviceGetAttribute(
+  res = tensorflow::wrap::cuDeviceGetAttribute(
       cc_minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, device);
   if (res != CUDA_SUCCESS) {
     return port::Status(
@@ -1320,7 +1340,8 @@ template <typename T>
 static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
                                             CUdevice_attribute attribute) {
   int value = -1;
-  CUresult result = cuDeviceGetAttribute(&value, attribute, device);
+  CUresult result =
+      tensorflow::wrap::cuDeviceGetAttribute(&value, attribute, device);
   if (result != CUDA_SUCCESS) {
     return port::Status(
         port::error::NOT_FOUND,
@@ -1375,24 +1396,24 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
 /* static */ bool CUDADriver::GetGridLimits(int *x, int *y, int *z,
                                             CUdevice device) {
   int value;
-  CUresult res =
-      cuDeviceGetAttribute(&value, CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_X, device);
+  CUresult res = tensorflow::wrap::cuDeviceGetAttribute(
+      &value, CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_X, device);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to query max grid dim x: " << ToString(res);
     return false;
   }
   *x = value;
 
-  res =
-      cuDeviceGetAttribute(&value, CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_Y, device);
+  res = tensorflow::wrap::cuDeviceGetAttribute(
+      &value, CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_Y, device);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to query max grid dim y: " << ToString(res);
     return false;
   }
   *y = value;
 
-  res =
-      cuDeviceGetAttribute(&value, CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_Z, device);
+  res = tensorflow::wrap::cuDeviceGetAttribute(
+      &value, CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_Z, device);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to query max grid dim z: " << ToString(res);
     return false;
@@ -1402,7 +1423,7 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
 }
 
 /* static */ bool CUDADriver::GetDriverVersion(int *driver_version) {
-  CUresult res = cuDriverGetVersion(driver_version);
+  CUresult res = tensorflow::wrap::cuDriverGetVersion(driver_version);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to query driver version: " << ToString(res);
     return false;
@@ -1414,7 +1435,8 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
 /* static */ port::StatusOr<int> CUDADriver::GetDeviceAttribute(
     CUdevice_attribute attribute, CUdevice device) {
   int val;
-  CUresult res = cuDeviceGetAttribute(&val, attribute, device);
+  CUresult res =
+      tensorflow::wrap::cuDeviceGetAttribute(&val, attribute, device);
   if (res != CUDA_SUCCESS) {
     return port::Status(
         port::error::INTERNAL,
@@ -1426,8 +1448,8 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
 
 /* static */ bool CUDADriver::IsEccEnabled(CUdevice device, bool *result) {
   int value = -1;
-  CUresult res =
-      cuDeviceGetAttribute(&value, CU_DEVICE_ATTRIBUTE_ECC_ENABLED, device);
+  CUresult res = tensorflow::wrap::cuDeviceGetAttribute(
+      &value, CU_DEVICE_ATTRIBUTE_ECC_ENABLED, device);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to query ECC status: " << ToString(res);
     return false;
@@ -1443,7 +1465,7 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
   ScopedActivateContext activation(context);
   size_t free = 0;
   size_t total = 0;
-  CUresult res = cuMemGetInfo(&free, &total);
+  CUresult res = tensorflow::wrap::cuMemGetInfo(&free, &total);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to query device memory info: " << ToString(res);
     return false;
@@ -1457,7 +1479,7 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
 /* static */ bool CUDADriver::GetDeviceTotalMemory(CUdevice device,
                                                    uint64 *result) {
   size_t value = -1;
-  CUresult res = cuDeviceTotalMem(&value, device);
+  CUresult res = tensorflow::wrap::cuDeviceTotalMem(&value, device);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to query total available memory: " << ToString(res);
     return false;
@@ -1472,7 +1494,8 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
   static const int kBufferSize = 64;
   absl::InlinedVector<char, 4> chars(kBufferSize);
   chars[kBufferSize - 1] = '\0';
-  CUresult res = cuDeviceGetPCIBusId(chars.begin(), kBufferSize - 1, device);
+  CUresult res = tensorflow::wrap::cuDeviceGetPCIBusId(chars.begin(),
+                                                       kBufferSize - 1, device);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to query PCI bus id for device: " << ToString(res);
     return pci_bus_id;
@@ -1500,7 +1523,7 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
                << to_device.status();
     return false;
   }
-  CUresult res = cuDeviceCanAccessPeer(
+  CUresult res = tensorflow::wrap::cuDeviceCanAccessPeer(
       &can_access_peer, from_device.ValueOrDie(), to_device.ValueOrDie());
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to detect peer access capability: " << ToString(res);
@@ -1517,7 +1540,8 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
   }
 
   ScopedActivateContext activated{from};
-  CUresult result = cuCtxEnablePeerAccess(to->context(), 0 /* = flags */);
+  CUresult result =
+      tensorflow::wrap::cuCtxEnablePeerAccess(to->context(), 0 /* = flags */);
   if (result != CUDA_SUCCESS &&
       result != CUDA_ERROR_PEER_ACCESS_ALREADY_ENABLED) {
     return port::Status(
@@ -1535,8 +1559,9 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
   ScopedActivateContext activation(context);
 
   int max_blocks;
-  CUresult result = cuOccupancyMaxActiveBlocksPerMultiprocessor(
-      &max_blocks, kernel, threads_per_block, dynamic_shared_memory_bytes);
+  CUresult result =
+      tensorflow::wrap::cuOccupancyMaxActiveBlocksPerMultiprocessor(
+          &max_blocks, kernel, threads_per_block, dynamic_shared_memory_bytes);
   if (result != CUDA_SUCCESS) {
     return port::Status(
         port::error::INTERNAL,
@@ -1549,7 +1574,7 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
 
 /* static */ CUcontext CUDADriver::CurrentContextOrDie() {
   CUcontext current = nullptr;
-  CUresult result = cuCtxGetCurrent(&current);
+  CUresult result = tensorflow::wrap::cuCtxGetCurrent(&current);
   if (result != CUDA_SUCCESS) {
     LOG(FATAL) << "failed to query current context: " << ToString(result);
   }
diff --git a/tensorflow/stream_executor/cuda/cuda_driver_wrapper.h b/tensorflow/stream_executor/cuda/cuda_driver_wrapper.h
new file mode 100644
index 0000000000..ee99908bd5
--- /dev/null
+++ b/tensorflow/stream_executor/cuda/cuda_driver_wrapper.h
@@ -0,0 +1,143 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// This file wraps cuda driver calls with dso loader so that we don't need to
+// have explicit linking to libcuda. All TF cuda driver usage should route
+// through this wrapper.
+
+#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DRIVER_WRAPPER_H_
+#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DRIVER_WRAPPER_H_
+
+#include "cuda/include/cuda.h"
+#include "tensorflow/stream_executor/lib/env.h"
+#include "tensorflow/stream_executor/platform/dso_loader.h"
+#include "tensorflow/stream_executor/platform/port.h"
+
+namespace tensorflow {
+namespace wrap {
+#ifdef PLATFORM_GOOGLE
+// Use static linked library
+#define STREAM_EXECUTOR_LIBCUDA_WRAP(cudaSymbolName)                       \
+  template <typename... Args>                                              \
+  auto cudaSymbolName(Args... args)->decltype(::cudaSymbolName(args...)) { \
+    return ::cudaSymbolName(args...);                                      \
+  }
+
+// This macro wraps a global identifier, given by cudaSymbolName, in a callable
+// structure that loads the DLL symbol out of the DSO handle in a thread-safe
+// manner on first use. This dynamic loading technique is used to avoid DSO
+// dependencies on vendor libraries which may or may not be available in the
+// deployed binary environment.
+#else
+#define TO_STR_(x) #x
+#define TO_STR(x) TO_STR_(x)
+
+#define STREAM_EXECUTOR_LIBCUDA_WRAP(cudaSymbolName)                        \
+  template <typename... Args>                                               \
+  auto cudaSymbolName(Args... args)->decltype(::cudaSymbolName(args...)) {  \
+    using FuncPtrT = std::add_pointer<decltype(::cudaSymbolName)>::type;    \
+    static FuncPtrT loaded = []() -> FuncPtrT {                             \
+      static const char *kName = TO_STR(cudaSymbolName);                    \
+      void *f;                                                              \
+      auto s = stream_executor::port::Env::Default()->GetSymbolFromLibrary( \
+          stream_executor::internal::CachedDsoLoader::GetLibcudaDsoHandle() \
+              .ValueOrDie(),                                                \
+          kName, &f);                                                       \
+      CHECK(s.ok()) << "could not find " << kName                           \
+                    << " in libcuda DSO; dlerror: " << s.error_message();   \
+      return reinterpret_cast<FuncPtrT>(f);                                 \
+    }();                                                                    \
+    return loaded(args...);                                                 \
+  }
+#endif
+
+// clang-format off
+#define LIBCUDA_ROUTINE_EACH(__macro)                   \
+  __macro(cuCtxEnablePeerAccess)                        \
+  __macro(cuCtxGetCurrent)                              \
+  __macro(cuCtxGetDevice)                               \
+  __macro(cuCtxGetSharedMemConfig)                      \
+  __macro(cuCtxSetCurrent)                              \
+  __macro(cuCtxSetSharedMemConfig)                      \
+  __macro(cuCtxSynchronize)                             \
+  __macro(cuDeviceCanAccessPeer)                        \
+  __macro(cuDeviceGet)                                  \
+  __macro(cuDeviceGetAttribute)                         \
+  __macro(cuDeviceGetCount)                             \
+  __macro(cuDeviceGetName)                              \
+  __macro(cuDeviceGetPCIBusId)                          \
+  __macro(cuDevicePrimaryCtxGetState)                   \
+  __macro(cuDevicePrimaryCtxRelease)                    \
+  __macro(cuDevicePrimaryCtxRetain)                     \
+  __macro(cuDevicePrimaryCtxSetFlags)                   \
+  __macro(cuDeviceTotalMem)                             \
+  __macro(cuDriverGetVersion)                           \
+  __macro(cuEventCreate)                                \
+  __macro(cuEventDestroy)                               \
+  __macro(cuEventElapsedTime)                           \
+  __macro(cuEventQuery)                                 \
+  __macro(cuEventRecord)                                \
+  __macro(cuEventSynchronize)                           \
+  __macro(cuFuncGetAttribute)                           \
+  __macro(cuFuncSetCacheConfig)                         \
+  __macro(cuGetErrorName)                               \
+  __macro(cuGetErrorString)                             \
+  __macro(cuInit)                                       \
+  __macro(cuLaunchKernel)                               \
+  __macro(cuMemAlloc)                                   \
+  __macro(cuMemAllocManaged)                            \
+  __macro(cuMemFree)                                    \
+  __macro(cuMemFreeHost)                                \
+  __macro(cuMemGetAddressRange)                         \
+  __macro(cuMemGetInfo)                                 \
+  __macro(cuMemHostAlloc)                               \
+  __macro(cuMemHostRegister)                            \
+  __macro(cuMemHostUnregister)                          \
+  __macro(cuMemcpyDtoD)                                 \
+  __macro(cuMemcpyDtoDAsync)                            \
+  __macro(cuMemcpyDtoH)                                 \
+  __macro(cuMemcpyDtoHAsync)                            \
+  __macro(cuMemcpyHtoD)                                 \
+  __macro(cuMemcpyHtoDAsync)                            \
+  __macro(cuMemsetD32)                                  \
+  __macro(cuMemsetD32Async)                             \
+  __macro(cuMemsetD8)                                   \
+  __macro(cuMemsetD8Async)                              \
+  __macro(cuModuleGetFunction)                          \
+  __macro(cuModuleGetGlobal)                            \
+  __macro(cuModuleLoadDataEx)                           \
+  __macro(cuModuleLoadFatBinary)                        \
+  __macro(cuModuleUnload)                               \
+  __macro(cuOccupancyMaxActiveBlocksPerMultiprocessor)  \
+  __macro(cuOccupancyMaxPotentialBlockSize)             \
+  __macro(cuPointerGetAttribute)                        \
+  __macro(cuStreamAddCallback)                          \
+  __macro(cuStreamCreate)                               \
+  __macro(cuStreamDestroy)                              \
+  __macro(cuStreamQuery)                                \
+  __macro(cuStreamSynchronize)                          \
+  __macro(cuStreamWaitEvent)
+
+// clang-format on
+
+LIBCUDA_ROUTINE_EACH(STREAM_EXECUTOR_LIBCUDA_WRAP)
+#undef LIBCUDA_ROUTINE_EACH
+#undef STREAM_EXECUTOR_LIBCUDA_WRAP
+#undef TO_STR
+#undef TO_STR_
+}  // namespace wrap
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DRIVER_WRAPPER_H_
diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
index 70004495cd..51dec6b246 100644
--- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
+++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
@@ -28,6 +28,7 @@ limitations under the License.
 #include "absl/strings/string_view.h"
 #include "tensorflow/stream_executor/cuda/cuda_diagnostics.h"
 #include "tensorflow/stream_executor/cuda/cuda_driver.h"
+#include "tensorflow/stream_executor/cuda/cuda_driver_wrapper.h"
 #include "tensorflow/stream_executor/cuda/cuda_event.h"
 #include "tensorflow/stream_executor/cuda/cuda_platform_id.h"
 #include "tensorflow/stream_executor/cuda/cuda_stream.h"
@@ -501,7 +502,7 @@ int CUDAExecutor::CalculateOccupancy(
     CUfunction func) {
   int suggested_blocks = 0;
   int suggested_threads = 0;
-  CUresult err = cuOccupancyMaxPotentialBlockSize(
+  CUresult err = tensorflow::wrap::cuOccupancyMaxPotentialBlockSize(
       &suggested_blocks, &suggested_threads, func, nullptr,
       shared_memory_per_block, 0);
   CHECK_EQ(err, CUDA_SUCCESS);
@@ -518,7 +519,7 @@ int CUDAExecutor::CompareOccupancy(int *initial_blocks,
                                    CUfunction func) {
   int suggested_blocks = 0;
   int suggested_threads = 0;
-  CUresult err = cuOccupancyMaxPotentialBlockSize(
+  CUresult err = tensorflow::wrap::cuOccupancyMaxPotentialBlockSize(
       &suggested_blocks, &suggested_threads, func, nullptr,
       shared_memory_per_block, 0);
   CHECK_EQ(err, CUDA_SUCCESS);
-- 
GitLab


From be1e0458309715cb8d0eea6baf1863a90a4f9ab5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 15:10:59 -0800
Subject: [PATCH 0823/2345] Commented out a description of the delayed-build
 pattern. :)

PiperOrigin-RevId: 229634588
---
 tensorflow/python/keras/engine/sequential.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/engine/sequential.py b/tensorflow/python/keras/engine/sequential.py
index 425c8c3ef6..37eb2840b3 100644
--- a/tensorflow/python/keras/engine/sequential.py
+++ b/tensorflow/python/keras/engine/sequential.py
@@ -87,8 +87,8 @@ class Sequential(Model):
   model.add(Dense(32))
   model.weights  # returns list of length 4
 
-  When using the delayed-build pattern (no input shape specified), you can
-  choose to manually build your model by calling `build(batch_input_shape)`:
+  # When using the delayed-build pattern (no input shape specified), you can
+  # choose to manually build your model by calling `build(batch_input_shape)`:
   model = Sequential()
   model.add(Dense(32))
   model.add(Dense(32))
-- 
GitLab


From a799b815e5b497fd85e36ec36e44df0f3cebf802 Mon Sep 17 00:00:00 2001
From: James Qin <jamesqin@google.com>
Date: Wed, 16 Jan 2019 15:23:15 -0800
Subject: [PATCH 0824/2345] Define .value() on DistributedVariable

PiperOrigin-RevId: 229636699
---
 tensorflow/python/distribute/values.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/python/distribute/values.py b/tensorflow/python/distribute/values.py
index feaeea80cb..585ae1bd6c 100644
--- a/tensorflow/python/distribute/values.py
+++ b/tensorflow/python/distribute/values.py
@@ -562,6 +562,9 @@ class DistributedVariable(DistributedDelegate):
   def read_value(self):
     return self._distribute_strategy.extended.read_var(self)
 
+  def value(self):
+    return self._get_closest().value()
+
   def _should_act_as_resource_variable(self):
     """Pass resource_variable_ops.is_resource_variable check."""
     pass
-- 
GitLab


From 7fa13a9954ddd0c0479f226a8cc71215b7dc260f Mon Sep 17 00:00:00 2001
From: Shivani Agrawal <shivaniagrawal@google.com>
Date: Wed, 16 Jan 2019 15:31:20 -0800
Subject: [PATCH 0825/2345] Adds either a Note or TODO for tests without Eager
 coverage. Adds coverage for some more SparseTensors related tests and
 optimization test.

PiperOrigin-RevId: 229638239
---
 .../optimization/map_vectorization_test.py    | 12 ++--
 .../optimization/model_dataset_test.py        | 14 ++--
 .../optimization/optimize_dataset_test.py     | 71 +++++++++++--------
 .../parse_example_dataset_test.py             | 45 ++++++++----
 .../kernel_tests/restructured_dataset_test.py |  2 +-
 .../python/data/kernel_tests/dataset_test.py  |  3 +
 .../from_sparse_tensor_slices_test.py         |  4 +-
 .../kernel_tests/from_tensor_slices_test.py   |  2 +-
 .../data/kernel_tests/from_tensors_test.py    |  3 +-
 .../python/data/kernel_tests/optional_test.py |  6 +-
 .../data/kernel_tests/padded_batch_test.py    |  2 +
 .../python/data/kernel_tests/reduce_test.py   |  1 +
 12 files changed, 97 insertions(+), 68 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
index 405b038eb2..ec543aeb76 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
@@ -370,19 +370,17 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
                                                      num_parallel_calls)
     self.assertDatasetsEqual(unoptimized, optimized)
 
-  # TODO(b/117581999): Add eager coverage for the following tests.
-  def testSkipEagerOptimizationBadMapFn(self):
+  def testOptimizationBadMapFn(self):
     # Test map functions that give an error
     def map_fn(x):
       # x has leading dimension 5, this will raise an error
       return array_ops.gather(x, 10)
-
-    base_dataset = dataset_ops.Dataset.range(5).repeat(5).batch(
-        5, drop_remainder=True)
-    _, optimized = self._get_test_datasets(base_dataset, map_fn)
-    nxt = dataset_ops.make_one_shot_iterator(optimized).get_next()
     with self.assertRaisesRegexp(errors.InvalidArgumentError,
                                  r"indices = 10 is not in \[0, 5\)"):
+      base_dataset = dataset_ops.Dataset.range(5).repeat(5).batch(
+          5, drop_remainder=True)
+      _, optimized = self._get_test_datasets(base_dataset, map_fn)
+      nxt = dataset_ops.make_one_shot_iterator(optimized).get_next()
       self.evaluate(nxt)
 
   def testOptimizationWithCapturedInputs(self):
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py
index c6b9db0d52..5c1ae7a98a 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py
@@ -23,10 +23,11 @@ from tensorflow.python.data.experimental.ops import optimization
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import errors
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
-# TODO(b/117581999): Add eager coverage for the following tests.
+@test_util.run_all_in_graph_and_eager_modes
 class ModelDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   def testAutotuneOption(self):
@@ -37,14 +38,11 @@ class ModelDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     options.experimental_autotune = True
     options.experimental_optimization.apply_default_optimizations = False
     dataset = dataset.with_options(options)
+    get_next = self.getNext(dataset)
 
-    iterator = dataset_ops.make_one_shot_iterator(dataset)
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      self.assertEqual(0, self.evaluate(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        self.evaluate(get_next)
+    self.assertEqual(0, self.evaluate(get_next()))
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
index e02f3b1572..f44ade049b 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
@@ -29,6 +29,7 @@ from tensorflow.python.data.experimental.ops import scan_ops
 from tensorflow.python.data.experimental.ops import threadpool
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
@@ -112,35 +113,47 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = self.getNext(dataset)
     self.evaluate(get_next())
 
-  # TODO(b/117581999): Add eager coverage for the following tests.
-  def testSkipEagerOptimizationLargeInputFromTensor(self):
-    input_t = array_ops.placeholder(dtypes.int32, (None, None, None))
-    dataset = dataset_ops.Dataset.from_tensors(input_t)
-    options = dataset_ops.Options()
-    options.experimental_optimization.apply_default_optimizations = False
-    dataset = dataset.with_options(options)
-    iterator = dataset_ops.make_initializable_iterator(dataset)
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(init_op, {input_t: np.ones([512, 1024, 1025], np.int32)})
-      self.evaluate(get_next)
-
-  # TODO(b/117581999): Add eager coverage for the following tests.
-  def testSkipEagerOptimizationLargeInputFromTensorSlices(self):
-    input_t = array_ops.placeholder(dtypes.int32, (None, None, None, None))
-    dataset = dataset_ops.Dataset.from_tensor_slices(input_t)
-    options = dataset_ops.Options()
-    options.experimental_optimization.apply_default_optimizations = False
-    dataset = dataset.with_options(options)
-    iterator = dataset_ops.make_initializable_iterator(dataset)
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(init_op, {input_t: np.ones([1, 512, 1024, 1025], np.int32)})
-      self.evaluate(get_next)
+  def testOptimizationLargeInputFromTensor(self):
+    def dataset_fn(input_t):
+      dataset = dataset_ops.Dataset.from_tensors(input_t)
+      options = dataset_ops.Options()
+      options.experimental_optimization.apply_default_optimizations = False
+      return dataset.with_options(options)
+
+    if context.executing_eagerly():
+      input_t = np.ones([512, 1024, 1025], np.int32)
+      get_next = self.getNext(dataset_fn(input_t))
+      self.evaluate(get_next())
+    else:
+      input_t = array_ops.placeholder(dtypes.int32, (None, None, None))
+      iterator = dataset_ops.make_initializable_iterator(dataset_fn(input_t))
+      init_op = iterator.initializer
+      get_next = iterator.get_next()
+
+      with self.cached_session() as sess:
+        sess.run(init_op, {input_t: np.ones([512, 1024, 1025], np.int32)})
+        self.evaluate(get_next)
+
+  def testOptimizationLargeInputFromTensorSlices(self):
+    def dataset_fn(input_t):
+      dataset = dataset_ops.Dataset.from_tensor_slices(input_t)
+      options = dataset_ops.Options()
+      options.experimental_optimization.apply_default_optimizations = False
+      return dataset.with_options(options)
+
+    if context.executing_eagerly():
+      input_t = np.ones([1, 512, 1024, 1025], np.int32)
+      get_next = self.getNext(dataset_fn(input_t))
+      self.evaluate(get_next())
+    else:
+      input_t = array_ops.placeholder(dtypes.int32, (None, None, None, None))
+      iterator = dataset_ops.make_initializable_iterator(dataset_fn(input_t))
+      init_op = iterator.initializer
+      get_next = iterator.get_next()
+
+      with self.cached_session() as sess:
+        sess.run(init_op, {input_t: np.ones([1, 512, 1024, 1025], np.int32)})
+        self.evaluate(get_next)
 
   def testOptimizationNestedDataset(self):
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/parse_example_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/parse_example_dataset_test.py
index 76e0d4d72a..4dbb188f2c 100644
--- a/tensorflow/python/data/experimental/kernel_tests/parse_example_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/parse_example_dataset_test.py
@@ -27,6 +27,7 @@ from tensorflow.core.example import feature_pb2
 from tensorflow.python.data.experimental.ops import parsing_ops as contrib_parsing_ops
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
@@ -671,8 +672,7 @@ class ParseExampleDatasetTest(test_base.DatasetTestBase):
     for batch_size in (1, 10, 20, 100, 256):
       self._testSerializedContainingVarLenDenseLargerBatch(batch_size)
 
-  @test_util.run_deprecated_v1
-  def testSkipEagerSerializedShapeMismatch(self):
+  def testSerializedShapeMismatch(self):
     aname = "a"
     bname = "b"
     cname = "c"
@@ -695,19 +695,34 @@ class ParseExampleDatasetTest(test_base.DatasetTestBase):
     ]
 
     serialized = [m.SerializeToString() for m in original]
-    self._test(
-        ops.convert_to_tensor(serialized), {
-            aname:
-                parsing_ops.FixedLenSequenceFeature((2, 1),
-                                                    dtype=dtypes.float32,
-                                                    allow_missing=True,
-                                                    default_value=[]),
-            bname:
-                parsing_ops.FixedLenSequenceFeature(
-                    (2, 1, 1), dtype=dtypes.string, allow_missing=True),
-        },
-        expected_err=(ValueError,
-                      "Cannot reshape a tensor with 0 elements to shape"))
+    if context.executing_eagerly():
+      self._test(
+          ops.convert_to_tensor(serialized), {
+              aname:
+                  parsing_ops.FixedLenSequenceFeature((2, 1),
+                                                      dtype=dtypes.float32,
+                                                      allow_missing=True,
+                                                      default_value=[]),
+              bname:
+                  parsing_ops.FixedLenSequenceFeature(
+                      (2, 1, 1), dtype=dtypes.string, allow_missing=True),
+          },
+          expected_err=(errors_impl.InvalidArgumentError,
+                        "Input to reshape is a tensor with 0 values"))
+    else:
+      self._test(
+          ops.convert_to_tensor(serialized), {
+              aname:
+                  parsing_ops.FixedLenSequenceFeature((2, 1),
+                                                      dtype=dtypes.float32,
+                                                      allow_missing=True,
+                                                      default_value=[]),
+              bname:
+                  parsing_ops.FixedLenSequenceFeature(
+                      (2, 1, 1), dtype=dtypes.string, allow_missing=True),
+          },
+          expected_err=(ValueError,
+                        "Cannot reshape a tensor with 0 elements to shape"))
 
   @test_util.run_deprecated_v1
   def testSerializedContainingVarLenDense(self):
diff --git a/tensorflow/python/data/experimental/kernel_tests/restructured_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/restructured_dataset_test.py
index 87a91415b0..ddac02b9e2 100644
--- a/tensorflow/python/data/experimental/kernel_tests/restructured_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/restructured_dataset_test.py
@@ -27,7 +27,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
 
-# TODO(b/117581999): Add eager coverage
+# TODO(b/117581999): Add eager specific test.
 class RestructuredDatasetTest(test_base.DatasetTestBase):
 
   @test_util.run_deprecated_v1
diff --git a/tensorflow/python/data/kernel_tests/dataset_test.py b/tensorflow/python/data/kernel_tests/dataset_test.py
index db8a999491..f319b24bee 100644
--- a/tensorflow/python/data/kernel_tests/dataset_test.py
+++ b/tensorflow/python/data/kernel_tests/dataset_test.py
@@ -266,6 +266,7 @@ class DatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
           round_trip_dataset, [self.evaluate(tf_value_fn())],
           requires_initialization=True)
 
+  # NOTE: This test is specific to graph mode and is skipped in eager mode.
   @test_util.run_deprecated_v1
   def testSkipEagerSameGraphErrorOneShot(self):
     dataset = dataset_ops.Dataset.range(10)
@@ -273,6 +274,7 @@ class DatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       with self.assertRaisesRegexp(ValueError, "must be from the same graph"):
         dataset = dataset.batch(2)
 
+  # NOTE: This test is specific to graph mode and is skipped in eager mode.
   @test_util.run_deprecated_v1
   def testSkipEagerSameGraphErrorOneShotSimple(self):
     dataset = dataset_ops.Dataset.range(10)
@@ -283,6 +285,7 @@ class DatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
             str(mock_log.call_args), "Please ensure that all datasets in the "
             "pipeline are created in the same graph as the iterator.")
 
+  # NOTE: This test is specific to graph mode and is skipped in eager mode.
   @test_util.run_deprecated_v1
   def testSkipEagerSameGraphErrorInitializable(self):
     dataset = dataset_ops.Dataset.range(10)
diff --git a/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py b/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py
index ef608ebb67..546c2fb2ed 100644
--- a/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py
+++ b/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py
@@ -29,11 +29,11 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
 
-@test_util.run_all_in_graph_and_eager_modes
+# NOTE: deprecated method in V2, no eager coverage added.
 class FromSparseTensorSlicesTest(test_base.DatasetTestBase):
 
   @test_util.run_deprecated_v1
-  def testSkipEagerFromSparseTensorSlices(self):
+  def testFromSparseTensorSlices(self):
     """Test a dataset based on slices of a `tf.SparseTensor`."""
     st = array_ops.sparse_placeholder(dtypes.float64)
     iterator = dataset_ops.make_initializable_iterator(
diff --git a/tensorflow/python/data/kernel_tests/from_tensor_slices_test.py b/tensorflow/python/data/kernel_tests/from_tensor_slices_test.py
index 9a480e5678..72db638771 100644
--- a/tensorflow/python/data/kernel_tests/from_tensor_slices_test.py
+++ b/tensorflow/python/data/kernel_tests/from_tensor_slices_test.py
@@ -53,7 +53,7 @@ class FromTensorSlicesTest(test_base.DatasetTestBase):
     with self.assertRaises(errors.OutOfRangeError):
       results = self.evaluate(get_next())
 
-  def testSkipEagerFromTensorSlicesSparse(self):
+  def testFromTensorSlicesSparse(self):
     """Test a dataset that represents the slices from a tuple of tensors."""
     components = (sparse_tensor.SparseTensorValue(
         indices=np.array([[0, 0], [1, 0], [2, 0]]),
diff --git a/tensorflow/python/data/kernel_tests/from_tensors_test.py b/tensorflow/python/data/kernel_tests/from_tensors_test.py
index ab3c15263f..82ccdebc7f 100644
--- a/tensorflow/python/data/kernel_tests/from_tensors_test.py
+++ b/tensorflow/python/data/kernel_tests/from_tensors_test.py
@@ -50,7 +50,7 @@ class FromTensorsTest(test_base.DatasetTestBase):
 
     self.assertDatasetProduces(dataset, expected_output=[components])
 
-  def testSkipEagerFromTensorsSparse(self):
+  def testFromTensorsSparse(self):
     """Test a dataset that represents a single tuple of tensors."""
     components = (sparse_tensor.SparseTensorValue(
         indices=np.array([[0]]),
@@ -224,6 +224,7 @@ class FromTensorsTest(test_base.DatasetTestBase):
     self.assertEquals(dtypes.int64, get_next().dtype)
     self.assertEquals([3], get_next().shape)
 
+  # TODO(b/121264236): needs mechanism for multiple device in eager mode.
   def testSkipEagerSplitPipelineFailsWithPlacementError(self):
     with session.Session(
         target="",
diff --git a/tensorflow/python/data/kernel_tests/optional_test.py b/tensorflow/python/data/kernel_tests/optional_test.py
index ba5ee9b661..2269bb8724 100644
--- a/tensorflow/python/data/kernel_tests/optional_test.py
+++ b/tensorflow/python/data/kernel_tests/optional_test.py
@@ -75,7 +75,6 @@ class OptionalTest(test_base.DatasetTestBase, parameterized.TestCase):
       self.assertAllEqual(expected.dense_shape,
                           self.evaluate(actual.dense_shape))
 
-  @test_util.run_deprecated_v1
   def testFromNone(self):
     value_structure = structure.TensorStructure(dtypes.float32, [])
     opt = optional_ops.Optional.none_from_structure(value_structure)
@@ -269,9 +268,7 @@ class OptionalTest(test_base.DatasetTestBase, parameterized.TestCase):
        optional_ops.OptionalStructure(
            structure.TensorStructure(dtypes.float32, []))),
   )
-  @test_util.run_deprecated_v1
-  def testSkipEagerOptionalStructure(self, tf_value_fn,
-                                     expected_value_structure):
+  def testOptionalStructure(self, tf_value_fn, expected_value_structure):
     tf_value = tf_value_fn()
     opt = optional_ops.Optional.from_value(tf_value)
 
@@ -306,6 +303,7 @@ class OptionalTest(test_base.DatasetTestBase, parameterized.TestCase):
       self.assertEqual(
           self.evaluate(tf_value), self.evaluate(round_trip_opt.get_value()))
 
+  # NOTE: This test is specific to graph mode and is skipped in eager mode.
   @parameterized.named_parameters(
       ("Tensor", np.array([1, 2, 3], dtype=np.int32),
        lambda: constant_op.constant([4, 5, 6], dtype=dtypes.int32), True),
diff --git a/tensorflow/python/data/kernel_tests/padded_batch_test.py b/tensorflow/python/data/kernel_tests/padded_batch_test.py
index dcfb2f507b..042af7a6f9 100644
--- a/tensorflow/python/data/kernel_tests/padded_batch_test.py
+++ b/tensorflow/python/data/kernel_tests/padded_batch_test.py
@@ -156,6 +156,7 @@ class PaddedBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     next_element = self.getNext(padded_dataset)
     self.evaluate(next_element())
 
+  # NOTE: This test is specific to graph mode and is skipped in eager mode.
   @test_util.run_deprecated_v1
   def testSkipEagerPaddedBatchDatasetShapeSpecifications(self):
     int_placeholder = array_ops.placeholder(dtypes.int32)
@@ -228,6 +229,7 @@ class PaddedBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       _ = dataset_ops.Dataset.range(10).padded_batch(
           5, padded_shapes=shape_as_tensor)
 
+  # NOTE: This test is specific to graph mode and is skipped in eager mode.
   @test_util.run_deprecated_v1
   def testSkipEagerPaddedBatchShapeError(self):
     with self.assertRaisesRegexp(
diff --git a/tensorflow/python/data/kernel_tests/reduce_test.py b/tensorflow/python/data/kernel_tests/reduce_test.py
index 14bbc0bf72..93acc1565f 100644
--- a/tensorflow/python/data/kernel_tests/reduce_test.py
+++ b/tensorflow/python/data/kernel_tests/reduce_test.py
@@ -68,6 +68,7 @@ class ReduceTest(test_base.DatasetTestBase, parameterized.TestCase):
       self.assertEqual(((i + 1) * i) // 2, s)
       self.assertEqual(i, c)
 
+  # NOTE: This test is specific to graph mode and is skipped in eager mode.
   @test_util.run_deprecated_v1
   def testSkipEagerSquareUsingPlaceholder(self):
     delta = array_ops.placeholder(dtype=dtypes.int64)
-- 
GitLab


From a88c90db14f92509e0da39072dc9bfd772a1c1aa Mon Sep 17 00:00:00 2001
From: Ben Barsdell <bbarsdell@nvidia.com>
Date: Fri, 14 Dec 2018 10:14:54 -0800
Subject: [PATCH 0826/2345] [XLA:GPU] Make sm_35 the min supported LLVM proc

- XLA nvptx backend does not support sm_30, and TensorFlow also only
  supports sm_35 onward.
---
 .../xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc   | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc
index eddaa877f2..153aab97d9 100644
--- a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc
+++ b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc
@@ -110,11 +110,9 @@ static string GetLibdeviceFilename(const string& libdevice_dir_path,
 }
 
 // Gets the GPU name as it's known to LLVM for a given compute capability.  If
-// we see an unrecognized compute capability, we return "sm_30".
+// we see an unrecognized compute capability, we return "sm_35".
 static string GetSmName(std::pair<int, int> compute_capability) {
   static auto* m = new std::map<std::pair<int, int>, int>({
-      {{3, 0}, 30},
-      {{3, 2}, 32},
       {{3, 5}, 35},
       {{3, 7}, 37},
       {{5, 0}, 50},
@@ -127,7 +125,7 @@ static string GetSmName(std::pair<int, int> compute_capability) {
       {{7, 2}, 72},
       {{7, 5}, 75},
   });
-  int sm_version = 30;
+  int sm_version = 35;
   auto it = m->find(compute_capability);
   if (it != m->end()) {
     sm_version = it->second;
-- 
GitLab


From ea84f664dd7d4405240da8797bfd21b7bd860aa1 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Wed, 16 Jan 2019 15:37:43 -0800
Subject: [PATCH 0827/2345] [tf.data] Fixing a bug in MultiDeviceIterator which
 would cause a deadlock if the iterator was re-initialized without trying to
 fetch elements from it.

PiperOrigin-RevId: 229639479
---
 .../core/kernels/data/multi_device_iterator_ops.cc    |  2 +-
 tensorflow/python/data/kernel_tests/BUILD             |  1 +
 .../data/kernel_tests/multi_device_iterator_test.py   | 11 ++++++++---
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/kernels/data/multi_device_iterator_ops.cc b/tensorflow/core/kernels/data/multi_device_iterator_ops.cc
index eb08d2b28a..167276032b 100644
--- a/tensorflow/core/kernels/data/multi_device_iterator_ops.cc
+++ b/tensorflow/core/kernels/data/multi_device_iterator_ops.cc
@@ -151,7 +151,7 @@ class MultiDeviceIterator : public ResourceBase {
     void Reset() LOCKS_EXCLUDED(mu_) {
       {
         mutex_lock l(mu_);
-        if (!background_thread_finished_) {
+        if (background_thread_ && !background_thread_finished_) {
           cancelled_ = true;
           // Wake up the background thread.
           for (int i = 0; i < size_; ++i) {
diff --git a/tensorflow/python/data/kernel_tests/BUILD b/tensorflow/python/data/kernel_tests/BUILD
index 737ba28ceb..8206ce382c 100644
--- a/tensorflow/python/data/kernel_tests/BUILD
+++ b/tensorflow/python/data/kernel_tests/BUILD
@@ -405,6 +405,7 @@ cuda_py_test(
     srcs = ["multi_device_iterator_test.py"],
     additional_deps = [
         ":test_base",
+        "@absl_py//absl/testing:parameterized",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python/data/ops:multi_device_iterator_ops",
diff --git a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
index 2ae6a9893d..0c88d7533f 100644
--- a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
+++ b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from absl.testing import parameterized
+
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.data.experimental.ops import optimization
 from tensorflow.python.data.kernel_tests import test_base
@@ -36,17 +38,20 @@ from tensorflow.python.platform import test
 # / V2 mode, we should remove this annotation and the run_v1_only annotations
 # as well.
 @test_util.run_all_in_graph_and_eager_modes
-class MultiDeviceIteratorTest(test_base.DatasetTestBase):
+class MultiDeviceIteratorTest(test_base.DatasetTestBase,
+                              parameterized.TestCase):
 
   @test_util.run_v1_only
-  def testNoGetNext(self):
+  @parameterized.parameters(0, 1, 42,)
+  def testInitOnly(self, num_inits):
     dataset = dataset_ops.Dataset.range(10)
     multi_device_iterator = multi_device_iterator_ops.MultiDeviceIterator(
         dataset, ["/cpu:1", "/cpu:2"])
 
     config = config_pb2.ConfigProto(device_count={"CPU": 3})
     with self.test_session(config=config):
-      self.evaluate(multi_device_iterator.initializer)
+      for _ in range(num_inits):
+        self.evaluate(multi_device_iterator.initializer)
 
   @test_util.run_v1_only
   def testBasic(self):
-- 
GitLab


From b1075f10d9facfdca80d28b881a434419e34ae2d Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Wed, 16 Jan 2019 15:59:42 -0800
Subject: [PATCH 0828/2345] Populate zero point for symmetric quantization as
 well.

PiperOrigin-RevId: 229643065
---
 tensorflow/lite/tools/optimize/subgraph_quantizer.cc  | 11 +++++++++--
 .../lite/tools/optimize/subgraph_quantizer_test.cc    | 10 ++++++++++
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/tools/optimize/subgraph_quantizer.cc b/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
index 30320304ec..bd6e6e81b2 100644
--- a/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
+++ b/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
@@ -37,12 +37,17 @@ const int8_t kMinQuantizedValue = -127;
 const int8_t kMaxQuantizedValue = 127;
 
 TfLiteStatus AddQuantizationParams(const std::vector<float>& scales,
+                                   const std::vector<int64_t>& zero_point,
                                    int quantized_dimension,
                                    const uint8_t* buffer_data,
                                    size_t buffer_size, TensorType output_type,
                                    ModelT* model, TensorT* tensor) {
   tensor->quantization = absl::make_unique<QuantizationParametersT>();
   tensor->quantization->scale.assign(scales.begin(), scales.end());
+  if (zero_point.size() != scales.size()) {
+    return kTfLiteError;
+  }
+  tensor->quantization->zero_point.assign(zero_point.begin(), zero_point.end());
   tensor->quantization->quantized_dimension = quantized_dimension;
   model->buffers[tensor->buffer]->data.assign(buffer_data,
                                               buffer_data + buffer_size);
@@ -168,8 +173,10 @@ TfLiteStatus SymmetricPerChannelQuantizeTensor(ModelT* model, TensorT* tensor,
   // Set the buffers and output type.
   uint8_t* uint8_buffer = reinterpret_cast<uint8_t*>(final_buffer.data());
   size_t buffer_size = num_elements * sizeof(int8_t);
-  return AddQuantizationParams(scales, channel_dim_index, uint8_buffer,
-                               buffer_size, TensorType_INT8, model, tensor);
+  std::vector<int64_t> zero_point(scales.size(), 0);
+  return AddQuantizationParams(scales, zero_point, channel_dim_index,
+                               uint8_buffer, buffer_size, TensorType_INT8,
+                               model, tensor);
 }
 
 // Symmetrically quantizes the bias for ops like Conv and DepthwiseConv.
diff --git a/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc b/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
index 085b66e90f..1ea49da899 100644
--- a/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
+++ b/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
@@ -97,13 +97,18 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantizationWithUnitScale) {
   const std::vector<float>& weights_scales =
       weights_tensor->quantization->scale;
 
+  const std::vector<int64_t>& weights_zero_points =
+      weights_tensor->quantization->zero_point;
+
   ASSERT_EQ(weights_scales.size(), out_channel_size);
+  ASSERT_EQ(weights_zero_points.size(), out_channel_size);
   ASSERT_EQ(input_tensor->quantization->scale.size(), 1);
   ASSERT_EQ(output_tensor->quantization->scale.size(), 1);
 
 
   for (size_t i = 0; i < out_channel_size; i++) {
     EXPECT_EQ(weights_scales[i], 1);
+    EXPECT_EQ(weights_zero_points[i], 0);
   }
 
   EXPECT_EQ(input_tensor->quantization->scale[0], 1);
@@ -190,8 +195,11 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantization) {
   ASSERT_TRUE(weights_tensor->quantization);
   const std::vector<float>& weights_scales =
       weights_tensor->quantization->scale;
+  const std::vector<int64_t>& weights_zero_points =
+      weights_tensor->quantization->zero_point;
 
   ASSERT_EQ(weights_scales.size(), out_channel_size);
+  ASSERT_EQ(weights_zero_points.size(), out_channel_size);
   ASSERT_EQ(input_tensor->quantization->scale.size(), 1);
   ASSERT_EQ(output_tensor->quantization->scale.size(), 1);
 
@@ -225,9 +233,11 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantization) {
     for (size_t j = 0; j < num_values_in_channel; j++) {
       size_t element_idx = channel_idx * out_channel_size + j;
       auto scale = weights_scales[channel_idx];
+      auto zero_point = weights_zero_points[channel_idx];
       auto dequantized_value = weight_values[element_idx] * scale;
       EXPECT_NEAR(dequantized_value, weights_float_buffer[element_idx],
                   scale / 2);
+      EXPECT_EQ(zero_point, 0);
     }
   }
 }
-- 
GitLab


From 0e0ef6e2aab7bf04baad53608745044ecb839594 Mon Sep 17 00:00:00 2001
From: AG Ramesh <ag.ramesh@intel.com>
Date: Wed, 16 Jan 2019 16:18:39 -0800
Subject: [PATCH 0829/2345] Moving definition of Traits inside ifdef

---
 tensorflow/core/kernels/eigen_spatial_convolutions_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/eigen_spatial_convolutions_test.cc b/tensorflow/core/kernels/eigen_spatial_convolutions_test.cc
index 14c6903a43..920e648972 100644
--- a/tensorflow/core/kernels/eigen_spatial_convolutions_test.cc
+++ b/tensorflow/core/kernels/eigen_spatial_convolutions_test.cc
@@ -1605,11 +1605,11 @@ static void PackLhsHelper(int iters,
       /*inner_dim_reordered*/ false,                  //
       /*Alignment*/ 0>;
 
-  using Traits = typename Eigen::internal::gebp_traits<float, float>;
 #if defined(TENSORFLOW_USE_MKLDNN_CONTRACTION_KERNEL)
   using PackLhsImpl = Eigen::internal::mkldnn_gemm_pack<float, Eigen::Index,
                                                         SubMapper, ColMajor>;
 #else
+  using Traits = typename Eigen::internal::gebp_traits<float, float>;
   using PackLhsImpl =
       Eigen::internal::gemm_pack_lhs<float, Eigen::Index, SubMapper,      //
                                      Traits::mr,                          //
-- 
GitLab


From 1961871d3fc8941b60212ea9f577001e7827d4a1 Mon Sep 17 00:00:00 2001
From: Shanqing Cai <cais@google.com>
Date: Wed, 16 Jan 2019 16:05:18 -0800
Subject: [PATCH 0830/2345] Disable testing of user-defined GPU kernel under
 eager mode

PiperOrigin-RevId: 229644310
---
 tensorflow/tools/ci_build/builds/test_user_ops.sh | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/builds/test_user_ops.sh b/tensorflow/tools/ci_build/builds/test_user_ops.sh
index 25ecee4725..9da9c3b881 100755
--- a/tensorflow/tools/ci_build/builds/test_user_ops.sh
+++ b/tensorflow/tools/ci_build/builds/test_user_ops.sh
@@ -239,8 +239,15 @@ function run_op() {
   fi
 }
 
+printf "\nTesting execution of user-defined op under graph mode:\n\n"
 run_op "$("${PYTHON_BIN_PATH}" -c "import tensorflow as tf; print(tf.Session('').run(tf.load_op_library('./${USER_OP_SO}').${USER_OP}(${OP_INPUT})))")"
-run_op "$("${PYTHON_BIN_PATH}" -c "import tensorflow as tf; tf.enable_eager_execution(); print(tf.load_op_library('./${USER_OP_SO}').${USER_OP}(${OP_INPUT}).numpy())")" " in eager mode"
+
+if [[ "${IS_GPU}" == "0" ]]; then
+  printf "\nTesting execution of user-defined op under eager mode:\n\n"
+  run_op "$("${PYTHON_BIN_PATH}" -c "import tensorflow as tf; tf.enable_eager_execution(); print(tf.load_op_library('./${USER_OP_SO}').${USER_OP}(${OP_INPUT}).numpy())")" " in eager mode"
+else
+  printf "\nSKIPPING the testing of execution of user-defined GPU kernel under eager mode. See b/122972785.\n\n"
+fi
 
 
 popd
-- 
GitLab


From 1d2e3ab3f291bd887a8dc292bb3fd31f9106204e Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Wed, 16 Jan 2019 16:12:51 -0800
Subject: [PATCH 0831/2345] Always call PyObject_ClearWeakRefs

This seems to work in Python 2 and be necessary for correctness in Python 3. weakreflist is not set and yet there are weakrefs which need to be cleared.

Oddly it's only the weak key dictionary that fails in Python 3 before removing the conditional; using weakref.ref on its own seems fine.

PiperOrigin-RevId: 229645639
---
 tensorflow/python/eager/ops_test.py      | 28 ++++++++++++++++++++++++
 tensorflow/python/eager/pywrap_tensor.cc |  4 +---
 2 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/eager/ops_test.py b/tensorflow/python/eager/ops_test.py
index 17a090d526..ab4bdaa601 100644
--- a/tensorflow/python/eager/ops_test.py
+++ b/tensorflow/python/eager/ops_test.py
@@ -18,6 +18,8 @@ from __future__ import division
 from __future__ import print_function
 
 import threading
+import weakref
+
 import numpy as np
 
 from tensorflow.core.protobuf import config_pb2
@@ -397,6 +399,32 @@ class OpsTest(test_util.TensorFlowTestCase):
     t1.start()
     t1.join()
 
+  def testWeakrefEagerTensor(self):
+    x = constant_op.constant([[1.]])
+    x.at1 = constant_op.constant([[2.]])
+    x.at2 = 3.
+    weak_x = weakref.ref(x)
+    weak_xat1 = weakref.ref(x.at1)
+    del x
+    self.assertIs(weak_x(), None)
+    self.assertIs(weak_xat1(), None)
+
+  def testWeakKeyDictionaryTensor(self):
+    weak_key_dict = weakref.WeakKeyDictionary()
+    strong_x = constant_op.constant([[1.]])
+    strong_y = constant_op.constant([[2.]])
+    weak_key_dict[strong_x] = constant_op.constant([[3.]])
+    weak_key_dict[strong_y] = constant_op.constant([[4.]])
+    strong_y.a = constant_op.constant([[5.]])
+    weak_x = weakref.ref(strong_x)
+    del strong_x
+    self.assertIs(weak_x(), None)
+    self.assertEqual([strong_y], list(weak_key_dict))
+    self.assertEqual(1, len(list(weak_key_dict)))
+    self.assertEqual(1, len(weak_key_dict))
+    del strong_y
+    self.assertEqual([], list(weak_key_dict))
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/eager/pywrap_tensor.cc b/tensorflow/python/eager/pywrap_tensor.cc
index 30a93fb0e4..35040c5d56 100644
--- a/tensorflow/python/eager/pywrap_tensor.cc
+++ b/tensorflow/python/eager/pywrap_tensor.cc
@@ -501,9 +501,7 @@ int EagerTensor_init(EagerTensor* self, PyObject* args, PyObject* kwds) {
 void EagerTensor_dealloc(EagerTensor* self) {
   // Clear weak references to self.
   // Needs to happen before any actual destruction.
-  if (self->weakreflist != nullptr) {
-    PyObject_ClearWeakRefs((PyObject*)self);
-  }
+  PyObject_ClearWeakRefs((PyObject*)self);
 
   TF_DeleteStatus(self->status);
   Py_DECREF(self->handle_data);
-- 
GitLab


From 0eeb360d8f04fd93b799a07bedbea6e6e899f55e Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Wed, 16 Jan 2019 16:12:52 -0800
Subject: [PATCH 0832/2345] Remove dead code block from normalization layer

PiperOrigin-RevId: 229645644
---
 tensorflow/python/keras/layers/normalization.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/tensorflow/python/keras/layers/normalization.py b/tensorflow/python/keras/layers/normalization.py
index 9167a747cd..171e1e231e 100644
--- a/tensorflow/python/keras/layers/normalization.py
+++ b/tensorflow/python/keras/layers/normalization.py
@@ -413,15 +413,6 @@ class BatchNormalizationV2(Layer):
   def _assign_moving_average(self, variable, value, momentum):
     with ops.name_scope(None, 'AssignMovingAvg',
                         [variable, value, momentum]) as scope:
-      # TODO(b/120571621): We want to avoid colocating the variables here
-      # since TPUStrategy does not implement replica local variables.
-      # Remove this hack once we support TPULocalVariables.
-      is_tpu_strategy = False
-      if distribution_strategy_context.has_strategy():
-        distribute = distribution_strategy_context.get_strategy()
-        if distribute.__class__.__name__ == 'TPUStrategy':
-          is_tpu_strategy = True
-
       with ops.colocate_with(variable):
         decay = ops.convert_to_tensor(1.0 - momentum, name='decay')
         if decay.dtype != variable.dtype.base_dtype:
-- 
GitLab


From 889615c72ef16e1d3151d7ae52939258fdda796c Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Wed, 16 Jan 2019 16:45:06 -0800
Subject: [PATCH 0833/2345] Updates binary cross entropy logic in Keras when
 input is probabilities. Moves label smoothing logic into the cross entropy
 loss functions.

PiperOrigin-RevId: 229651194
---
 tensorflow/python/keras/backend.py            |  12 +-
 tensorflow/python/keras/losses.py             |  67 ++++++---
 tensorflow/python/keras/losses_test.py        | 127 +++++++++++++-----
 .../golden/v1/tensorflow.keras.losses.pbtxt   |   4 +-
 .../golden/v1/tensorflow.keras.metrics.pbtxt  |   4 +-
 .../golden/v2/tensorflow.keras.losses.pbtxt   |   4 +-
 .../golden/v2/tensorflow.keras.metrics.pbtxt  |   4 +-
 7 files changed, 156 insertions(+), 66 deletions(-)

diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index 2e53ee4ec9..f357f0efc7 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -3866,7 +3866,8 @@ def categorical_crossentropy(target, output, from_logits=False, axis=-1):
       axis = axis % len(output.shape)
       # scale preds so that the class probas of each sample sum to 1
       output = output / math_ops.reduce_sum(output, axis, True)
-      # manual computation of crossentropy
+
+      # Compute cross entropy from probabilities.
       epsilon_ = _to_tensor(epsilon(), output.dtype.base_dtype)
       output = clip_ops.clip_by_value(output, epsilon_, 1. - epsilon_)
       return -math_ops.reduce_sum(target * math_ops.log(output), axis)
@@ -3948,10 +3949,13 @@ def binary_crossentropy(target, output, from_logits=False):
   """
   if not from_logits:
     if context.executing_eagerly() or output.op.type != 'Sigmoid':
-      # transform back to logits
       epsilon_ = _to_tensor(epsilon(), output.dtype.base_dtype)
-      output = clip_ops.clip_by_value(output, epsilon_, 1 - epsilon_)
-      output = math_ops.log(output / (1 - output))
+      output = clip_ops.clip_by_value(output, epsilon_, 1. - epsilon_)
+
+      # Compute cross entropy from probabilities.
+      bce = target * math_ops.log(output + epsilon())
+      bce += (1 - target) * math_ops.log(1 - output + epsilon())
+      return -bce
     else:
       # When sigmoid activation function is used for output operation, we
       # use logits from the sigmoid function directly to compute loss in order
diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index 66780de0f0..9144eb0bf9 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -24,6 +24,7 @@ import abc
 import six
 
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import smart_cond
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras.utils.generic_utils import deserialize_keras_object
 from tensorflow.python.keras.utils.generic_utils import serialize_keras_object
@@ -291,7 +292,7 @@ class BinaryCrossentropy(Loss):
   Args:
     from_logits: Whether `output` is expected to be a logits tensor. By default,
       we consider that `output` encodes a probability distribution.
-    label_smoothing: If greater than `0` then smooth the labels.
+    label_smoothing: Float in [0, 1]. If > `0` then smooth the labels.
     reduction: Type of `tf.losses.Reduction` to apply to loss. Default value is
       `SUM_OVER_BATCH_SIZE`.
     name: Optional name for the op.
@@ -304,7 +305,8 @@ class BinaryCrossentropy(Loss):
                name=None):
     super(BinaryCrossentropy, self).__init__(reduction=reduction, name=name)
     self.from_logits = from_logits
-    self.label_smoothing = label_smoothing
+    self.label_smoothing = ops.convert_to_tensor(
+        label_smoothing, dtype=K.floatx())
 
   def call(self, y_true, y_pred):
     """Invokes the `BinaryCrossentropy` instance.
@@ -318,11 +320,11 @@ class BinaryCrossentropy(Loss):
     """
     y_pred = ops.convert_to_tensor(y_pred)
     y_true = math_ops.cast(y_true, y_pred.dtype)
-
-    if self.label_smoothing > 0:
-      y_true = y_true * (1 - self.label_smoothing) + 0.5 * self.label_smoothing
-
-    return binary_crossentropy(y_true, y_pred, from_logits=self.from_logits)
+    return binary_crossentropy(
+        y_true,
+        y_pred,
+        from_logits=self.from_logits,
+        label_smoothing=self.label_smoothing)
 
 
 @keras_export('keras.losses.CategoricalCrossentropy')
@@ -349,8 +351,9 @@ class CategoricalCrossentropy(Loss):
   Args:
     from_logits: Whether `output` is expected to be a logits tensor. By default,
       we consider that `output` encodes a probability distribution.
-    label_smoothing: If greater than `0` then smooth the labels. This option is
-      currently not supported when `y_pred` is a sparse input (not one-hot).
+    label_smoothing: Float in [0, 1]. If > `0` then smooth the labels. This
+      option is currently not supported when `y_pred` is a sparse input
+      (not one-hot).
     reduction: Type of `tf.losses.Reduction` to apply to loss. Default value is
       `SUM_OVER_BATCH_SIZE`.
     name: Optional name for the op.
@@ -364,7 +367,8 @@ class CategoricalCrossentropy(Loss):
     super(CategoricalCrossentropy, self).__init__(
         reduction=reduction, name=name)
     self.from_logits = from_logits
-    self.label_smoothing = label_smoothing
+    self.label_smoothing = ops.convert_to_tensor(
+        label_smoothing, dtype=K.floatx())
 
   def call(self, y_true, y_pred):
     """Invokes the `CategoricalCrossentropy` instance.
@@ -385,14 +389,11 @@ class CategoricalCrossentropy(Loss):
           y_true, y_pred, from_logits=self.from_logits)
     else:
       y_true = math_ops.cast(y_true, y_pred.dtype)
-      if self.label_smoothing > 0:
-        num_classes = math_ops.cast(array_ops.shape(y_true)[1], y_pred.dtype)
-        smooth_positives = 1.0 - self.label_smoothing
-        smooth_negatives = self.label_smoothing / num_classes
-        y_true = y_true * smooth_positives + smooth_negatives
-
       return categorical_crossentropy(
-          y_true, y_pred, from_logits=self.from_logits)
+          y_true,
+          y_pred,
+          from_logits=self.from_logits,
+          label_smoothing=self.label_smoothing)
 
 
 @keras_export('keras.losses.Hinge')
@@ -782,7 +783,29 @@ def logcosh(y_true, y_pred):
 
 @keras_export('keras.metrics.categorical_crossentropy',
               'keras.losses.categorical_crossentropy')
-def categorical_crossentropy(y_true, y_pred, from_logits=False):
+def categorical_crossentropy(y_true,
+                             y_pred,
+                             from_logits=False,
+                             label_smoothing=0):
+  """Computes the categorical crossentropy loss.
+
+  Args:
+    y_true: tensor of true targets.
+    y_pred: tensor of predicted targets.
+    from_logits: Whether `y_pred` is expected to be a logits tensor. By default,
+      we consider that `y_pred` encodes a probability distribution.
+    label_smoothing: Float in [0, 1]. If > `0` then smooth the labels.
+
+  Returns:
+    Categorical crossentropy loss value.
+  """
+
+  def _smooth_labels():
+    num_classes = math_ops.cast(array_ops.shape(y_true)[1], y_pred.dtype)
+    return y_true * (1.0 - label_smoothing) + (label_smoothing / num_classes)
+
+  y_true = smart_cond.smart_cond(label_smoothing,
+                                 _smooth_labels, lambda: y_true)
   return K.categorical_crossentropy(y_true, y_pred, from_logits=from_logits)
 
 
@@ -795,7 +818,13 @@ def sparse_categorical_crossentropy(y_true, y_pred, from_logits=False):
 
 @keras_export('keras.metrics.binary_crossentropy',
               'keras.losses.binary_crossentropy')
-def binary_crossentropy(y_true, y_pred, from_logits=False):
+def binary_crossentropy(y_true, y_pred, from_logits=False, label_smoothing=0):
+
+  def _smooth_labels():
+    return y_true * (1.0 - label_smoothing) + 0.5 * label_smoothing
+
+  y_true = smart_cond.smart_cond(label_smoothing,
+                                 _smooth_labels, lambda: y_true)
   return K.mean(
       K.binary_crossentropy(y_true, y_pred, from_logits=from_logits), axis=-1)
 
diff --git a/tensorflow/python/keras/losses_test.py b/tensorflow/python/keras/losses_test.py
index 004c30f84d..d276854e29 100644
--- a/tensorflow/python/keras/losses_test.py
+++ b/tensorflow/python/keras/losses_test.py
@@ -27,7 +27,6 @@ from tensorflow.python import keras
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import test_util
-from tensorflow.python.ops import array_ops
 from tensorflow.python.ops.losses import losses_impl
 from tensorflow.python.platform import test
 
@@ -593,74 +592,132 @@ class BinaryCrossentropyTest(test.TestCase):
     self.assertAlmostEqual(self.evaluate(loss), 0.0, 3)
 
   def test_unweighted(self):
+    y_true = np.asarray([1, 0, 1, 0]).reshape([2, 2])
+    y_pred = np.asarray([1, 1, 1, 0], dtype=np.float32).reshape([2, 2])
     bce_obj = keras.losses.BinaryCrossentropy()
-    y_true = constant_op.constant([1, 0, 1, 0, 0, 1], shape=(2, 3))
-    y_pred = constant_op.constant([1, 1, 1, 0, 1, 0],
-                                  shape=(2, 3),
-                                  dtype=dtypes.float32)
     loss = bce_obj(y_true, y_pred)
-    self.assertAlmostEqual(self.evaluate(loss), 8.0004, 3)
+
+    # EPSILON = 1e-7, y = y_true, y` = y_pred, Y_MAX = 0.9999999
+    # y` = clip_ops.clip_by_value(output, EPSILON, 1. - EPSILON)
+    # y` = [Y_MAX, Y_MAX, Y_MAX, EPSILON]
+
+    # Loss = -(y log(y` + EPSILON) + (1 - y) log(1 - y` + EPSILON))
+    #      = [-log(Y_MAX + EPSILON), -log(1 - Y_MAX + EPSILON),
+    #         -log(Y_MAX + EPSILON), -log(1)]
+    #      = [0, 15.33, 0, 0]
+    # Reduced loss = 15.33 / 4
+
+    self.assertAlmostEqual(self.evaluate(loss), 3.833, 3)
 
     # Test with logits.
-    logits = constant_op.constant([10., 10., 10., -10., 10, -10],
-                                  shape=(2, 3),
-                                  dtype=dtypes.float32)
+    y_true = constant_op.constant([[1, 0, 1], [0, 1, 1]])
+    logits = constant_op.constant([[100.0, -100.0, 100.0],
+                                   [100.0, 100.0, -100.0]])
     bce_obj = keras.losses.BinaryCrossentropy(from_logits=True)
     loss = bce_obj(y_true, logits)
-    self.assertAlmostEqual(self.evaluate(loss), 5., 3)
+
+    # Loss = max(x, 0) - x * z + log(1 + exp(-abs(x)))
+    #            (where x = logits and z = y_true)
+    #      = [((100 - 100 * 1 + log(1 + exp(-100))) +
+    #          (0 + 100 * 0 + log(1 + exp(-100))) +
+    #          (100 - 100 * 1 + log(1 + exp(-100))),
+    #         ((100 - 100 * 0 + log(1 + exp(-100))) +
+    #          (100 - 100 * 1 + log(1 + exp(-100))) +
+    #          (0 + 100 * 1 + log(1 + exp(-100))))]
+    #      = [(0 + 0 + 0) / 3, 200 / 3]
+    # Reduced loss = (0 + 66.666) / 2
+
+    self.assertAlmostEqual(self.evaluate(loss), 33.333, 3)
 
   def test_scalar_weighted(self):
     bce_obj = keras.losses.BinaryCrossentropy()
-    y_true = constant_op.constant([1, 0, 1, 0, 0, 1], shape=(2, 3))
-    y_pred = constant_op.constant([1, 1, 1, 0, 1, 0],
-                                  shape=(2, 3),
-                                  dtype=dtypes.float32)
+    y_true = np.asarray([1, 0, 1, 0]).reshape([2, 2])
+    y_pred = np.asarray([1, 1, 1, 0], dtype=np.float32).reshape([2, 2])
     loss = bce_obj(y_true, y_pred, sample_weight=2.3)
-    self.assertAlmostEqual(self.evaluate(loss), 18.4010, 3)
+
+    # EPSILON = 1e-7, y = y_true, y` = y_pred, Y_MAX = 0.9999999
+    # y` = clip_ops.clip_by_value(output, EPSILON, 1. - EPSILON)
+    # y` = [Y_MAX, Y_MAX, Y_MAX, EPSILON]
+
+    # Loss = -(y log(y` + EPSILON) + (1 - y) log(1 - y` + EPSILON))
+    #      = [-log(Y_MAX + EPSILON), -log(1 - Y_MAX + EPSILON),
+    #         -log(Y_MAX + EPSILON), -log(1)]
+    #      = [0, 15.33, 0, 0]
+    # Weighted loss = [0, 15.33 * 2.3, 0, 0]
+    # Reduced loss = 15.33 * 2.3 / 4
+
+    self.assertAlmostEqual(self.evaluate(loss), 8.817, 3)
 
     # Test with logits.
-    y_true = array_ops.ones((32, 1))
-    logits = array_ops.ones((32, 1), dtype=dtypes.float32)
+    y_true = constant_op.constant([[1, 0, 1], [0, 1, 1]])
+    logits = constant_op.constant([[100.0, -100.0, 100.0],
+                                   [100.0, 100.0, -100.0]])
     bce_obj = keras.losses.BinaryCrossentropy(from_logits=True)
     loss = bce_obj(y_true, logits, sample_weight=2.3)
-    self.assertAlmostEqual(self.evaluate(loss), 0.7205, 3)
+
+    # Loss = max(x, 0) - x * z + log(1 + exp(-abs(x)))
+    #            (where x = logits and z = y_true)
+    # Loss = [(0 + 0 + 0) / 3, 200 / 3]
+    # Weighted loss = [0 * 2.3, 66.666 * 2.3]
+    # Reduced loss = (0 + 66.666 * 2.3) / 2
+
+    self.assertAlmostEqual(self.evaluate(loss), 76.667, 3)
 
   def test_sample_weighted(self):
     bce_obj = keras.losses.BinaryCrossentropy()
-    y_true = constant_op.constant([1, 0, 1, 0, 0, 1], shape=(2, 3))
-    y_pred = constant_op.constant([1, 1, 1, 0, 1, 0],
-                                  shape=(2, 3),
-                                  dtype=dtypes.float64)
+    y_true = np.asarray([1, 0, 1, 0]).reshape([2, 2])
+    y_pred = np.asarray([1, 1, 1, 0], dtype=np.float32).reshape([2, 2])
     sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
     loss = bce_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAlmostEqual(self.evaluate(loss), 21.4907, 3)
+
+    # EPSILON = 1e-7, y = y_true, y` = y_pred, Y_MAX = 0.9999999
+    # y` = clip_ops.clip_by_value(output, EPSILON, 1. - EPSILON)
+    # y` = [Y_MAX, Y_MAX, Y_MAX, EPSILON]
+
+    # Loss = -(y log(y` + EPSILON) + (1 - y) log(1 - y` + EPSILON))
+    #      = [-log(Y_MAX + EPSILON), -log(1 - Y_MAX + EPSILON),
+    #         -log(Y_MAX + EPSILON), -log(1)]
+    #      = [0, 15.33, 0, 0]
+    # Reduced loss = 15.33 * 1.2 / 4
+
+    self.assertAlmostEqual(self.evaluate(loss), 4.6, 3)
 
     # Test with logits.
-    y_true = constant_op.constant([[0, 0, 1], [1, 0, 0], [0, 1, 0]])
-    logits = constant_op.constant(
-        [[100.0, -100.0, -100.0], [-100.0, 100.0, -100.0],
-         [-100.0, -100.0, 100.0]],
-        dtype=dtypes.float64)
-    weights = constant_op.constant([3, 2, 8])
+    y_true = constant_op.constant([[1, 0, 1], [0, 1, 1]])
+    logits = constant_op.constant([[100.0, -100.0, 100.0],
+                                   [100.0, 100.0, -100.0]])
+    weights = constant_op.constant([4, 3])
     bce_obj = keras.losses.BinaryCrossentropy(from_logits=True)
     loss = bce_obj(y_true, logits, sample_weight=weights)
-    self.assertAlmostEqual(self.evaluate(loss), 288.8888, 3)
+
+    # Loss = max(x, 0) - x * z + log(1 + exp(-abs(x)))
+    #            (where x = logits and z = y_true)
+    # Loss = [(0 + 0 + 0)/3, 200 / 3]
+    # Weighted loss = [0 * 4, 66.666 * 3]
+    # Reduced loss = (0 + 66.666 * 3) / 2
+
+    self.assertAlmostEqual(self.evaluate(loss), 100, 3)
 
   def test_no_reduction(self):
-    y_true = constant_op.constant(((1, 0, 1), (1, 1, 0), (0, 1, 1)))
-    logits = constant_op.constant(((100.0, -100.0, 100.0),
-                                   (100.0, -100.0, 100.0),
-                                   (100.0, 100.0, -100.0)))
+    y_true = constant_op.constant([[1, 0, 1], [0, 1, 1]])
+    logits = constant_op.constant([[100.0, -100.0, 100.0],
+                                   [100.0, 100.0, -100.0]])
     bce_obj = keras.losses.BinaryCrossentropy(
         from_logits=True, reduction=losses_impl.ReductionV2.NONE)
     loss = bce_obj(y_true, logits)
-    self.assertAllClose((0., 66.6666, 66.6666), self.evaluate(loss), 3)
+
+    # Loss = max(x, 0) - x * z + log(1 + exp(-abs(x)))
+    #            (where x = logits and z = y_true)
+    # Loss = [(0 + 0 + 0)/3, (200)/3]
+
+    self.assertAllClose((0., 66.6666), self.evaluate(loss), 3)
 
   def test_label_smoothing(self):
     logits = constant_op.constant([[100.0, -100.0, -100.0]])
     y_true = constant_op.constant([[1, 0, 1]])
     label_smoothing = 0.1
     # Loss: max(x, 0) - x * z + log(1 + exp(-abs(x)))
+    #            (where x = logits and z = y_true)
     # Label smoothing: z' = z * (1 - L) + 0.5L
     #                  1  = 1 - 0.5L
     #                  0  = 0.5L
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
index b71e89883e..ce942de6d2 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
@@ -62,11 +62,11 @@ tf_module {
   }
   member_method {
     name: "binary_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\', \'label_smoothing\'], varargs=None, keywords=None, defaults=[\'False\', \'0\'], "
   }
   member_method {
     name: "categorical_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\', \'label_smoothing\'], varargs=None, keywords=None, defaults=[\'False\', \'0\'], "
   }
   member_method {
     name: "categorical_hinge"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
index 54f5b21f55..cae251cdfe 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
@@ -110,7 +110,7 @@ tf_module {
   }
   member_method {
     name: "binary_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\', \'label_smoothing\'], varargs=None, keywords=None, defaults=[\'False\', \'0\'], "
   }
   member_method {
     name: "categorical_accuracy"
@@ -118,7 +118,7 @@ tf_module {
   }
   member_method {
     name: "categorical_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\', \'label_smoothing\'], varargs=None, keywords=None, defaults=[\'False\', \'0\'], "
   }
   member_method {
     name: "cosine"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
index 673593ed6c..e8809450fa 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
@@ -66,11 +66,11 @@ tf_module {
   }
   member_method {
     name: "binary_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\', \'label_smoothing\'], varargs=None, keywords=None, defaults=[\'False\', \'0\'], "
   }
   member_method {
     name: "categorical_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\', \'label_smoothing\'], varargs=None, keywords=None, defaults=[\'False\', \'0\'], "
   }
   member_method {
     name: "categorical_hinge"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
index 54f5b21f55..cae251cdfe 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
@@ -110,7 +110,7 @@ tf_module {
   }
   member_method {
     name: "binary_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\', \'label_smoothing\'], varargs=None, keywords=None, defaults=[\'False\', \'0\'], "
   }
   member_method {
     name: "categorical_accuracy"
@@ -118,7 +118,7 @@ tf_module {
   }
   member_method {
     name: "categorical_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\', \'label_smoothing\'], varargs=None, keywords=None, defaults=[\'False\', \'0\'], "
   }
   member_method {
     name: "cosine"
-- 
GitLab


From af9f12801a72ef9e9cbfa3e0dbca3510388c7104 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 17:03:57 -0800
Subject: [PATCH 0834/2345] Fork contrib's timeseries into the canned estimator
 folder and clean it up to only include LSTMAutoRegressor and its
 dependencies. After the fork, the following changes were made:

1. Imports are pointed to the new fork except for symbols that were removed during the next steps of the clean up (see below). tensorflow/.../estimator imports were changed to tensorflow_estimator/...

2. estimators.py had all its symbols removed except for LSTMAutoRegressor which is the one we want to keep.

3. ar_model.py had all its symbols removed except for LSTMPredictionModel and ARModel (which uses LSTMPredictionModel).

4. state_management.py had ChainingStateManager removed and as state_management_test.py only tests ChainingStateManager, that test file was removed.

5. math_utils.py had all its symbols removed except for InputStatisticsFromMiniBatch and replicate_state.

6. examples folder was removed. This will come back in some form (perhaps an updated docstring in estimators.py) before LSTMAutoRegressor is exported.

7. input_pipeline.py was removed. This is going to be replaced with a user supplied tf.data-backed input_fn. For now, tests point to contrib's input_pipeline.

8. Cleaned up the BUILD and __init__.py files. Fixed rules and removed stale tags.

PiperOrigin-RevId: 229654011
---
 tensorflow/contrib/timeseries/examples/BUILD          | 2 ++
 tensorflow/contrib/timeseries/python/timeseries/BUILD | 1 +
 2 files changed, 3 insertions(+)

diff --git a/tensorflow/contrib/timeseries/examples/BUILD b/tensorflow/contrib/timeseries/examples/BUILD
index 57797214d1..e10be88ece 100644
--- a/tensorflow/contrib/timeseries/examples/BUILD
+++ b/tensorflow/contrib/timeseries/examples/BUILD
@@ -105,6 +105,7 @@ py_binary(
     data = ["data/multivariate_periods.csv"],
     srcs_version = "PY2AND3",
     tags = ["no_pip"],
+    visibility = ["//visibility:public"],
     deps = select({
         ":empty_condition": [],
         "//conditions:default": [],
@@ -113,6 +114,7 @@ py_binary(
         "//tensorflow:tensorflow_py",
         "//tensorflow/contrib/timeseries/python/timeseries:estimators",
         "//tensorflow/contrib/timeseries/python/timeseries:model",
+        "//tensorflow/contrib/timeseries/python/timeseries:state_management",
     ],
 )
 
diff --git a/tensorflow/contrib/timeseries/python/timeseries/BUILD b/tensorflow/contrib/timeseries/python/timeseries/BUILD
index a0c3204d41..2a22295197 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/BUILD
+++ b/tensorflow/contrib/timeseries/python/timeseries/BUILD
@@ -281,6 +281,7 @@ py_library(
         "input_pipeline.py",
     ],
     srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
     deps = [
         ":feature_keys",
         ":model_utils",
-- 
GitLab


From 574db93f256879e4182760b2661f2c398f48372e Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Wed, 16 Jan 2019 17:08:25 -0800
Subject: [PATCH 0835/2345] Add SerializableResource and
 GetSerializedResouceOp. This is needed by: 1. the new calibration design. The
 current int8 calibration workflow depends on    a global resouce manager
 singleton TRTResourceManager (in    resources/trt_resource_manager.h). This
 has been:    - violating the resource manager design: resource manager should
 be      per-device    - pollutes the BUILD dependencies, and makes the kernel
 implementation not      able to be used in other language bindings (Issue
 #23853) 2. the custom backend offline mode, where we'll do the conversion
 during    execution and provide an offline tool to get the serizlied engine

PiperOrigin-RevId: 229654702
---
 tensorflow/contrib/BUILD                      |  2 +-
 tensorflow/contrib/tensorrt/BUILD             | 49 +++++++++---
 .../kernels/get_serialized_resource_op.cc     | 73 +++++++++++++++++
 .../get_serialized_resource_op_test.cc        | 80 +++++++++++++++++++
 .../ops/get_serialized_resource_op.cc         | 40 ++++++++++
 .../contrib/tensorrt/python/__init__.py       |  2 +-
 .../ops/{trt_engine_op.py => trt_ops.py}      |  6 +-
 .../tensorrt/python/trt_convert_test.py       |  2 +-
 .../tensorrt/resources/trt_resources.cc       | 61 ++++++++++++++
 .../tensorrt/resources/trt_resources.h        | 40 +++-------
 .../test/dynamic_input_shapes_test.py         |  3 -
 .../tensorrt/test/quantization_mnist_test.py  |  2 +-
 .../test/tf_trt_integration_test_base.py      |  2 +-
 13 files changed, 314 insertions(+), 48 deletions(-)
 create mode 100644 tensorflow/contrib/tensorrt/kernels/get_serialized_resource_op.cc
 create mode 100644 tensorflow/contrib/tensorrt/kernels/get_serialized_resource_op_test.cc
 create mode 100644 tensorflow/contrib/tensorrt/ops/get_serialized_resource_op.cc
 rename tensorflow/contrib/tensorrt/python/ops/{trt_engine_op.py => trt_ops.py} (87%)
 create mode 100644 tensorflow/contrib/tensorrt/resources/trt_resources.cc

diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD
index 307bb6eca3..a4c3d9623a 100644
--- a/tensorflow/contrib/BUILD
+++ b/tensorflow/contrib/BUILD
@@ -196,7 +196,7 @@ cc_library(
             "//tensorflow/contrib/kinesis:dataset_kernels",
         ],
     }) + if_not_windows([
-        "//tensorflow/contrib/tensorrt:trt_engine_op_kernel",
+        "//tensorflow/contrib/tensorrt:trt_op_kernels",
     ]),
 )
 
diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 7ae3caaaa2..80d79164a2 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -54,8 +54,9 @@ tf_cuda_cc_test(
 )
 
 tf_custom_op_library(
-    name = "python/ops/_trt_engine_op.so",
+    name = "python/ops/_trt_ops.so",
     srcs = [
+        "ops/get_serialized_resource_op.cc",
         "ops/trt_engine_op.cc",
     ],
     deps = [
@@ -80,8 +81,9 @@ tf_cuda_library(
 )
 
 cc_library(
-    name = "trt_engine_op_kernel",
+    name = "trt_op_kernels",
     srcs = [
+        "kernels/get_serialized_resource_op.cc",
         "kernels/trt_engine_op.cc",
     ],
     hdrs = [
@@ -109,9 +111,33 @@ cc_library(
     alwayslink = 1,  # buildozer: disable=alwayslink-with-hdrs
 )
 
+tf_cuda_cc_test(
+    name = "get_serialized_resource_op_test",
+    size = "small",
+    srcs = ["kernels/get_serialized_resource_op_test.cc"],
+    tags = [
+        "no_cuda_on_cpu_tap",
+        "no_windows",
+        "nomac",
+    ],
+    deps = [
+        ":get_serialized_resource_op_op_lib",
+        ":trt_op_kernels",
+        ":trt_resources",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+        "//tensorflow/core/kernels:ops_testutil",
+    ],
+)
+
 tf_gen_op_libs(
     op_lib_names = [
         "trt_engine_op",
+        "get_serialized_resource_op",
     ],
 )
 
@@ -128,8 +154,9 @@ tf_cuda_library(
 )
 
 tf_gen_op_wrapper_py(
-    name = "trt_engine_op",
+    name = "trt_ops",
     deps = [
+        ":get_serialized_resource_op_op_lib",
         ":trt_engine_op_op_lib",
         ":trt_logging",
         ":trt_shape_function",
@@ -137,16 +164,17 @@ tf_gen_op_wrapper_py(
 )
 
 tf_custom_op_py_library(
-    name = "trt_engine_op_loader",
-    srcs = ["python/ops/trt_engine_op.py"],
+    name = "trt_ops_loader",
+    srcs = ["python/ops/trt_ops.py"],
     dso = [
-        ":python/ops/_trt_engine_op.so",
+        ":python/ops/_trt_ops.so",
     ] + if_tensorrt([
         "@local_config_tensorrt//:nv_infer",
     ]),
     kernels = [
-        ":trt_engine_op_kernel",
+        ":trt_op_kernels",
         ":trt_engine_op_op_lib",
+        ":get_serialized_resource_op_op_lib",
         ":trt_shape_function",
     ],
     srcs_version = "PY2AND3",
@@ -176,8 +204,8 @@ py_library(
     name = "trt_ops_py",
     srcs_version = "PY2AND3",
     deps = [
-        ":trt_engine_op",
-        ":trt_engine_op_loader",
+        ":trt_ops",
+        ":trt_ops_loader",
     ],
 )
 
@@ -208,7 +236,7 @@ tf_py_wrap_cc(
     deps = [
         ":test_utils",
         ":trt_conversion",
-        ":trt_engine_op_kernel",
+        ":trt_op_kernels",
         "//third_party/python_runtime:headers",
     ],
 )
@@ -218,6 +246,7 @@ tf_cuda_library(
     srcs = [
         "resources/trt_int8_calibrator.cc",
         "resources/trt_resource_manager.cc",
+        "resources/trt_resources.cc",
     ],
     hdrs = [
         "resources/trt_int8_calibrator.h",
diff --git a/tensorflow/contrib/tensorrt/kernels/get_serialized_resource_op.cc b/tensorflow/contrib/tensorrt/kernels/get_serialized_resource_op.cc
new file mode 100644
index 0000000000..f68bc2b485
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/kernels/get_serialized_resource_op.cc
@@ -0,0 +1,73 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CONTRIB_TENSORRT_KERNELS_GET_SERIALIZED_RESOURCE_OP_H_
+#define TENSORFLOW_CONTRIB_TENSORRT_KERNELS_GET_SERIALIZED_RESOURCE_OP_H_
+
+#include <memory>
+#include <vector>
+
+#include "tensorflow/contrib/tensorrt/resources/trt_resources.h"
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/resource_mgr.h"
+#include "tensorflow/core/lib/core/refcount.h"
+
+#if GOOGLE_CUDA
+#if GOOGLE_TENSORRT
+
+namespace tensorflow {
+namespace tensorrt {
+
+class GetSerializedResourceOp : public OpKernel {
+ public:
+  explicit GetSerializedResourceOp(OpKernelConstruction* context)
+      : OpKernel(context) {}
+
+  ~GetSerializedResourceOp() override {}
+
+  void Compute(OpKernelContext* context) override {
+    // TODO(laigd): it will allocate the tensor on the device and copy the
+    // serialized string to that tensor, and later sess.run() will copy it back
+    // to host. We need to optimize this.
+    const string& container = context->input(0).scalar<string>()();
+    const string& resource_name = context->input(1).scalar<string>()();
+
+    // Get the resource.
+    SerializableResourceBase* resource = nullptr;
+    OP_REQUIRES_OK(context, context->resource_manager()->Lookup(
+                                container, resource_name, &resource));
+    ::tensorflow::core::ScopedUnref sc(resource);
+
+    // Serialize the resource as output.
+    string serialized_resource;
+    OP_REQUIRES_OK(context, resource->SerializeToString(&serialized_resource));
+
+    Tensor* output = nullptr;
+    OP_REQUIRES_OK(context,
+                   context->allocate_output(0, TensorShape({}), &output));
+    output->scalar<string>()() = serialized_resource;
+  }
+};
+
+REGISTER_KERNEL_BUILDER(Name("GetSerializedResourceOp").Device(DEVICE_GPU),
+                        GetSerializedResourceOp);
+
+}  // namespace tensorrt
+}  // namespace tensorflow
+
+#endif  // GOOGLE_TENSORRT
+#endif  // GOOGLE_CUDA
+#endif  // TENSORFLOW_CONTRIB_TENSORRT_KERNELS_GET_SERIALIZED_RESOURCE_OP_H_
diff --git a/tensorflow/contrib/tensorrt/kernels/get_serialized_resource_op_test.cc b/tensorflow/contrib/tensorrt/kernels/get_serialized_resource_op_test.cc
new file mode 100644
index 0000000000..a91228e4c4
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/kernels/get_serialized_resource_op_test.cc
@@ -0,0 +1,80 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <dirent.h>
+#include <string.h>
+#include <fstream>
+#include <vector>
+
+#include "tensorflow/contrib/tensorrt/resources/trt_resources.h"
+#include "tensorflow/core/framework/fake_input.h"
+#include "tensorflow/core/framework/node_def_builder.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/kernels/ops_testutil.h"
+#include "tensorflow/core/platform/test.h"
+
+#if GOOGLE_CUDA
+#if GOOGLE_TENSORRT
+
+namespace tensorflow {
+namespace tensorrt {
+
+class GetSerializedResourceOpTest : public OpsTestBase {};
+
+TEST_F(GetSerializedResourceOpTest, Basic) {
+  // Create the GPU device.
+  std::unique_ptr<Device> device(
+      DeviceFactory::NewDevice("GPU", {}, "/job:worker/replica:0/task:0"));
+
+  // Create the resource.
+  class MySerializableResource : public SerializableResourceBase {
+   public:
+    string DebugString() const override { return ""; }
+    Status SerializeToString(string* serialized) override {
+      *serialized = "my_serialized_str";
+      return Status::OK();
+    }
+  };
+  const string container = "mycontainer";
+  const string resource_name = "myresource";
+  SerializableResourceBase* resource = new MySerializableResource();
+  ResourceMgr* rm = device->resource_manager();
+  EXPECT_TRUE(rm->Create(container, resource_name, resource).ok());
+
+  // Create the op.
+  SetDevice(DEVICE_GPU, std::move(device));
+  TF_ASSERT_OK(NodeDefBuilder("op", "GetSerializedResourceOp")
+                   .Input(FakeInput(DT_STRING))
+                   .Input(FakeInput(DT_STRING))
+                   .Finalize(node_def()));
+  TF_ASSERT_OK(InitOp());
+
+  // Execute the op.
+  AddInputFromArray<string>(TensorShape({}), {container});
+  AddInputFromArray<string>(TensorShape({}), {resource_name});
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Verify the result.
+  // TODO(laigd): OpsTestBase::GetOutput() doesn't work.
+  Tensor* output = context_->mutable_output(0);
+  EXPECT_EQ("my_serialized_str", output->scalar<string>()());
+}
+
+}  // namespace tensorrt
+}  // namespace tensorflow
+
+#endif  // GOOGLE_TENSORRT
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/contrib/tensorrt/ops/get_serialized_resource_op.cc b/tensorflow/contrib/tensorrt/ops/get_serialized_resource_op.cc
new file mode 100644
index 0000000000..59da73f5ef
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/ops/get_serialized_resource_op.cc
@@ -0,0 +1,40 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#if GOOGLE_TENSORRT
+
+#include "tensorflow/core/framework/common_shape_fns.h"
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/shape_inference.h"
+#include "tensorflow/core/framework/tensor_shape.h"
+
+namespace tensorflow {
+
+REGISTER_OP("GetSerializedResourceOp")
+    .Input("container: string")
+    .Input("resource_name: string")
+    .Output("serialized_resource: string")
+    .SetShapeFn(shape_inference::ScalarShape)
+    .SetIsStateful()
+    .Doc(R"doc(
+Gets a resource from a container managed by the resource manager and returns
+its serialized representation.
+)doc");
+
+}  // namespace tensorflow
+
+#endif  // GOOGLE_TENSORRT
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/contrib/tensorrt/python/__init__.py b/tensorflow/contrib/tensorrt/python/__init__.py
index 7cdfe2b1a6..e2cf253ca0 100644
--- a/tensorflow/contrib/tensorrt/python/__init__.py
+++ b/tensorflow/contrib/tensorrt/python/__init__.py
@@ -19,7 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 # pylint: disable=unused-import,line-too-long
-from tensorflow.contrib.tensorrt.python.ops import trt_engine_op
+from tensorflow.contrib.tensorrt.python.ops import trt_ops
 from tensorflow.contrib.tensorrt.python.trt_convert import add_test_value
 from tensorflow.contrib.tensorrt.python.trt_convert import calib_graph_to_infer_graph
 from tensorflow.contrib.tensorrt.python.trt_convert import clear_test_values
diff --git a/tensorflow/contrib/tensorrt/python/ops/trt_engine_op.py b/tensorflow/contrib/tensorrt/python/ops/trt_ops.py
similarity index 87%
rename from tensorflow/contrib/tensorrt/python/ops/trt_engine_op.py
rename to tensorflow/contrib/tensorrt/python/ops/trt_ops.py
index 31a313182b..1fee06854f 100644
--- a/tensorflow/contrib/tensorrt/python/ops/trt_engine_op.py
+++ b/tensorflow/contrib/tensorrt/python/ops/trt_ops.py
@@ -22,13 +22,13 @@ import platform
 
 if platform.system() != "Windows":
   # pylint: disable=wildcard-import,unused-import,g-import-not-at-top
-  from tensorflow.contrib.tensorrt.ops.gen_trt_engine_op import *
+  from tensorflow.contrib.tensorrt.ops.gen_trt_ops import *
 
   from tensorflow.contrib.util import loader
   from tensorflow.python.platform import resource_loader
   # pylint: enable=wildcard-import,unused-import,g-import-not-at-top
 
-  _trt_engine_op = loader.load_op_library(
-      resource_loader.get_path_to_datafile("_trt_engine_op.so"))
+  _trt_ops = loader.load_op_library(
+      resource_loader.get_path_to_datafile("_trt_ops.so"))
 else:
   raise RuntimeError("Windows platforms are not supported")
diff --git a/tensorflow/contrib/tensorrt/python/trt_convert_test.py b/tensorflow/contrib/tensorrt/python/trt_convert_test.py
index 14ab4772c4..9562704aeb 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert_test.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert_test.py
@@ -22,7 +22,7 @@ import os
 
 from tensorflow.contrib.tensorrt.python import trt_convert
 # pylint: disable=unused-import
-from tensorflow.contrib.tensorrt.python.ops import trt_engine_op
+from tensorflow.contrib.tensorrt.python.ops import trt_ops
 # pylint: enable=unused-import
 from tensorflow.core.framework import graph_pb2
 from tensorflow.core.protobuf import config_pb2
diff --git a/tensorflow/contrib/tensorrt/resources/trt_resources.cc b/tensorflow/contrib/tensorrt/resources/trt_resources.cc
new file mode 100644
index 0000000000..c19eb34dab
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/resources/trt_resources.cc
@@ -0,0 +1,61 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/contrib/tensorrt/resources/trt_resources.h"
+
+#if GOOGLE_CUDA
+#if GOOGLE_TENSORRT
+
+namespace tensorflow {
+namespace tensorrt {
+
+TRTCalibrationResource::~TRTCalibrationResource() {
+  VLOG(0) << "Destroying Calibration Resource " << std::endl << DebugString();
+  builder_.reset();
+  engine_.reset();
+  // We need to manually destroy the builder and engine before the allocator
+  // is destroyed.
+  allocator_.reset();
+}
+
+string TRTCalibrationResource::DebugString() const {
+  std::stringstream oss;
+  using std::dec;
+  using std::endl;
+  using std::hex;
+  oss << " Calibrator = " << hex << calibrator_.get() << dec << endl
+      << " Builder    = " << hex << builder_.get() << dec << endl
+      << " Engine     = " << hex << engine_.get() << dec << endl
+      << " Logger     = " << hex << &logger_ << dec << endl
+      << " Allocator  = " << hex << allocator_.get() << dec << endl
+      << " Thread     = " << hex << thr_.get() << dec << endl;
+  return oss.str();
+}
+
+Status TRTCalibrationResource::SerializeToString(string* serialized) {
+  calibrator_->waitAndSetDone();
+  thr_->join();
+  *serialized = calibrator_->getCalibrationTableAsString();
+  if (!serialized->size()) {
+    return tensorflow::errors::Unknown("Calibration table is empty.");
+  }
+  return Status::OK();
+}
+
+}  // namespace tensorrt
+}  // namespace tensorflow
+
+#endif  // GOOGLE_TENSORRT
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/contrib/tensorrt/resources/trt_resources.h b/tensorflow/contrib/tensorrt/resources/trt_resources.h
index 11930f3671..d0a87f2c31 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_resources.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_resources.h
@@ -32,37 +32,23 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-
 #include "tensorrt/include/NvInfer.h"
 
 namespace tensorflow {
 namespace tensorrt {
 
-class TRTCalibrationResource : public tensorflow::ResourceBase {
+class SerializableResourceBase : public tensorflow::ResourceBase {
+ public:
+  virtual Status SerializeToString(string* serialized) = 0;
+};
+
+class TRTCalibrationResource : public SerializableResourceBase {
  public:
-  ~TRTCalibrationResource() {
-    LOG(INFO) << "Destroying Calibration Resource " << std::endl
-              << DebugString();
-    builder_.reset();
-    engine_.reset();
-    // We need to manually destroy the builder and engine before the allocator
-    // is destroyed.
-    allocator_.reset();
-  }
-
-  string DebugString() const override {
-    std::stringstream oss;
-    using std::dec;
-    using std::endl;
-    using std::hex;
-    oss << " Calibrator = " << hex << calibrator_.get() << dec << endl
-        << " Builder    = " << hex << builder_.get() << dec << endl
-        << " Engine     = " << hex << engine_.get() << dec << endl
-        << " Logger     = " << hex << &logger_ << dec << endl
-        << " Allocator  = " << hex << allocator_.get() << dec << endl
-        << " Thread     = " << hex << thr_.get() << dec << endl;
-    return oss.str();
-  }
+  ~TRTCalibrationResource() override;
+
+  string DebugString() const override;
+
+  Status SerializeToString(string* serialized) override;
 
   // Lookup table for temporary staging areas of input tensors for calibration.
   std::unordered_map<string, std::pair<void*, size_t>> device_buffers_;
@@ -82,6 +68,6 @@ class TRTCalibrationResource : public tensorflow::ResourceBase {
 }  // namespace tensorrt
 }  // namespace tensorflow
 
-#endif
-#endif
+#endif  // GOOGLE_TENSORRT
+#endif  // GOOGLE_CUDA
 #endif  // TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_RESOURCES_H_
diff --git a/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py b/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py
index 90addecb14..cc28cd6087 100644
--- a/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py
+++ b/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py
@@ -20,9 +20,6 @@ from __future__ import print_function
 
 import numpy as np
 
-# pylint: disable=unused-import
-from tensorflow.contrib.tensorrt.python.ops import trt_engine_op
-# pylint: enable=unused-import
 from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
diff --git a/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py b/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
index 966476dc3e..753b47f4d9 100644
--- a/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
+++ b/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 from tensorflow.contrib.tensorrt.python import trt_convert
 # pylint: disable=unused-import
-from tensorflow.contrib.tensorrt.python.ops import trt_engine_op
+from tensorflow.contrib.tensorrt.python.ops import trt_ops
 # pylint: enable=unused-import
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python import data
diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
index f13c615cb1..4f47f80e73 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
@@ -27,7 +27,7 @@ import six
 
 from tensorflow.contrib.tensorrt.python import trt_convert
 # pylint: disable=unused-import
-from tensorflow.contrib.tensorrt.python.ops import trt_engine_op
+from tensorflow.contrib.tensorrt.python.ops import trt_ops
 # pylint: enable=unused-import
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import rewriter_config_pb2
-- 
GitLab


From 8b0e7dfcf0e5f4d0a15a1c32dd379292f2b344fb Mon Sep 17 00:00:00 2001
From: Katherine Wu <kathywu@google.com>
Date: Wed, 16 Jan 2019 17:16:40 -0800
Subject: [PATCH 0836/2345] Correct add_weight docstring by removing mention of
 outdated functionality.

PiperOrigin-RevId: 229655774
---
 tensorflow/python/keras/engine/base_layer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index 5a3540d12d..7bfc301239 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -256,7 +256,7 @@ class Layer(checkpointable.Checkpointable):
                  synchronization=tf_variables.VariableSynchronization.AUTO,
                  aggregation=tf_variables.VariableAggregation.NONE,
                  **kwargs):
-    """Adds a new variable to the layer, or gets an existing one; returns it.
+    """Adds a new variable to the layer.
 
     Arguments:
       name: variable name.
-- 
GitLab


From 1d669ac6ccaa343add66d652f48b2ae943932a43 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Wed, 16 Jan 2019 17:28:46 -0800
Subject: [PATCH 0837/2345] [XLA] Delete dumped_computation_to_graphviz tool.

Superceded by the interactive_graphviz tool.

PiperOrigin-RevId: 229657392
---
 tensorflow/compiler/xla/tools/BUILD           | 27 ------
 .../tools/dumped_computation_to_graphviz.cc   | 84 -------------------
 2 files changed, 111 deletions(-)
 delete mode 100644 tensorflow/compiler/xla/tools/dumped_computation_to_graphviz.cc

diff --git a/tensorflow/compiler/xla/tools/BUILD b/tensorflow/compiler/xla/tools/BUILD
index a7ef32fd35..4412a6ec69 100644
--- a/tensorflow/compiler/xla/tools/BUILD
+++ b/tensorflow/compiler/xla/tools/BUILD
@@ -29,33 +29,6 @@ tf_cc_binary(
     ],
 )
 
-cc_library(
-    name = "dumped_computation_to_graphviz_library",
-    srcs = ["dumped_computation_to_graphviz.cc"],
-    deps = [
-        "//tensorflow/compiler/xla:debug_options_flags",
-        "//tensorflow/compiler/xla:statusor",
-        "//tensorflow/compiler/xla:types",
-        "//tensorflow/compiler/xla:xla_data_proto",
-        "//tensorflow/compiler/xla/client",
-        "//tensorflow/compiler/xla/client:client_library",
-        "//tensorflow/compiler/xla/client:local_client",
-        "//tensorflow/compiler/xla/client:xla_computation",
-        "//tensorflow/compiler/xla/service",
-        "//tensorflow/compiler/xla/service:hlo_proto",
-        "//tensorflow/core:lib",
-        "@com_google_absl//absl/types:span",
-    ],
-)
-
-tf_cc_binary(
-    name = "dumped_computation_to_graphviz",
-    deps = [
-        ":dumped_computation_to_graphviz_library",
-        "//tensorflow/compiler/xla/service:interpreter_plugin",
-    ],
-)
-
 tf_cc_binary(
     name = "show_signature",
     srcs = ["show_signature.cc"],
diff --git a/tensorflow/compiler/xla/tools/dumped_computation_to_graphviz.cc b/tensorflow/compiler/xla/tools/dumped_computation_to_graphviz.cc
deleted file mode 100644
index b623556468..0000000000
--- a/tensorflow/compiler/xla/tools/dumped_computation_to_graphviz.cc
+++ /dev/null
@@ -1,84 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-// Usage: dumped_computation_to_graphviz some_binary_snapshot_proto*
-//
-// Dumps a graphviz URL for a snapshot computation to the command line.
-//
-// some_binary_snapshot_proto is obtained by serializing the HloSnapshot from
-// ServiceInterface::SnapshotComputation to disk.
-//
-// The GraphViz URL is placed into the log stderr, whereas computation
-// statistics are printed on stdout (implementation note: getting computation
-// statistics is how we trigger compilation to split out a GraphViz URL).
-
-#include <stdio.h>
-#include <memory>
-#include <string>
-
-#include "absl/types/span.h"
-#include "tensorflow/compiler/xla/client/client.h"
-#include "tensorflow/compiler/xla/client/client_library.h"
-#include "tensorflow/compiler/xla/client/local_client.h"
-#include "tensorflow/compiler/xla/client/xla_computation.h"
-#include "tensorflow/compiler/xla/debug_options_flags.h"
-#include "tensorflow/compiler/xla/service/hlo.pb.h"
-#include "tensorflow/compiler/xla/service/service.h"
-#include "tensorflow/compiler/xla/statusor.h"
-#include "tensorflow/compiler/xla/types.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
-#include "tensorflow/core/platform/env.h"
-#include "tensorflow/core/platform/init_main.h"
-#include "tensorflow/core/platform/logging.h"
-
-namespace xla {
-namespace tools {
-
-void RealMain(absl::Span<char* const> args) {
-  Client* client = ClientLibrary::LocalClientOrDie();
-  for (char* arg : args) {
-    HloSnapshot module;
-    TF_CHECK_OK(
-        tensorflow::ReadBinaryProto(tensorflow::Env::Default(), arg, &module));
-    XlaComputation computation =
-        client->LoadSnapshot(module).ConsumeValueOrDie();
-    DebugOptions debug_options = GetDebugOptionsFromFlags();
-    debug_options.set_xla_generate_hlo_graph(".*");
-    ComputationStats stats =
-        client->GetComputationStats(computation, debug_options)
-            .ConsumeValueOrDie();
-    fprintf(stdout, ">>> %s :: %s\n", arg, stats.DebugString().c_str());
-  }
-}
-
-}  // namespace tools
-}  // namespace xla
-
-int main(int argc, char** argv) {
-  std::vector<tensorflow::Flag> flag_list;
-  xla::AppendDebugOptionsFlags(&flag_list);
-  xla::string usage = tensorflow::Flags::Usage(argv[0], flag_list);
-  const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list);
-  if (!parse_result) {
-    LOG(ERROR) << "\n" << usage;
-    return 2;
-  }
-  tensorflow::port::InitMain(argv[0], &argc, &argv);
-
-  absl::Span<char* const> args(argv, argc);
-  args.remove_prefix(1);  // Pop off the binary name, argv[0]
-  xla::tools::RealMain(args);
-  return 0;
-}
-- 
GitLab


From 0f6bde31cf9593395529dce78cb4ec95eecbba3c Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Wed, 16 Jan 2019 18:04:59 -0800
Subject: [PATCH 0838/2345] tf.saved_model: Allow saving/loading concrete
 functions directly

We'll want this for representing signatures, and I don't see any reason to limit it to that.

Some fiddling so that the function gets registered with the eager context. Apparently that was only working when graph building.

PiperOrigin-RevId: 229661863
---
 tensorflow/python/eager/function.py           | 56 +++++------
 .../saved_model/function_deserialization.py   | 18 ++++
 .../saved_model/function_serialization.py     | 65 +++++++++----
 tensorflow/python/saved_model/load.py         | 96 +++++++++++--------
 tensorflow/python/saved_model/load_test.py    | 51 ++++++++++
 tensorflow/python/saved_model/save.py         | 35 ++++---
 .../saved_model/saved_object_graph.proto      |  1 +
 7 files changed, 224 insertions(+), 98 deletions(-)

diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 9b63b6f31c..82dea51c29 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -256,13 +256,16 @@ class _EagerDefinedFunction(object):
     self._graph = graph
     self._stateful_ops = tuple(op for op in operations if op.op_def.is_stateful)
 
-  def add_to_graph(self, g):
+  def add_to_graph(self, g=None):
     # pylint: disable=protected-access
-    if self.name not in g._functions:
-      g._add_function(self)
-    for f in self._graph._functions.values():
-      if f.name not in g._functions:
-        g._add_function(f)
+    if not g and context.executing_eagerly():
+      context.context().add_function_def(self.definition)
+    else:
+      if self.name not in g._functions:
+        g._add_function(self)
+      for f in self._graph._functions.values():
+        if f.name not in g._functions:
+          g._add_function(f)
     # pylint: enable=protected-access
 
   @property
@@ -635,27 +638,26 @@ class ConcreteFunction(object):
     # method's functionality better. Remove register_gradient_functions argument
     # and figure out if these needs to be registered.
 
-    if not context.executing_eagerly() or g:
-      if not g:
-        g = ops.get_default_graph()
-      self._inference_function.add_to_graph(g)  # pylint: disable=protected-access
-
-      # pylint: disable=protected-access
-      if register_gradient_functions:
-        # There are two situations for the actual call of a defun:
-        # 1. If none of the input args are resource variables or watch by any
-        #   tape, and it will run the _inference_function of concrete_func for
-        #   forward pass, the gradient will be generated by standard mechanism.
-        # 2. Otherwise, defun will create two functions, one for forward pass,
-        #   and the backward pass will be created via tape.
-        #   When registering the function, we register both cases.
-        if self._backward_graph_function is None:
-          self._construct_backprop_function()
-        forward_function = self._forward_function
-        backward_function = self._backward_graph_function._inference_function
-        # pylint: enable=protected-access
-        forward_function.add_to_graph(g)
-        backward_function.add_to_graph(g)
+    if not context.executing_eagerly() and not g:
+      g = ops.get_default_graph()
+    self._inference_function.add_to_graph(g)  # pylint: disable=protected-access
+
+    # pylint: disable=protected-access
+    if register_gradient_functions:
+      # There are two situations for the actual call of a defun:
+      # 1. If none of the input args are resource variables or watch by any
+      #   tape, and it will run the _inference_function of concrete_func for
+      #   forward pass, the gradient will be generated by standard mechanism.
+      # 2. Otherwise, defun will create two functions, one for forward pass,
+      #   and the backward pass will be created via tape.
+      #   When registering the function, we register both cases.
+      if self._backward_graph_function is None:
+        self._construct_backprop_function()
+      forward_function = self._forward_function
+      backward_function = self._backward_graph_function._inference_function
+      # pylint: enable=protected-access
+      forward_function.add_to_graph(g)
+      backward_function.add_to_graph(g)
 
   def _construct_backprop_function(self):
     """Constructs the backprop function object for this function."""
diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py
index 4ab85ec8a0..5479bbb355 100644
--- a/tensorflow/python/saved_model/function_deserialization.py
+++ b/tensorflow/python/saved_model/function_deserialization.py
@@ -71,6 +71,24 @@ def _deserialize_function_spec(function_spec_proto, coder):
                                    kwargs_to_include, input_signature)
 
 
+def recreate_concrete_function(saved_concrete_function, concrete_functions):
+  """Recreates a user-facing concrete function."""
+  concrete_function = concrete_functions[saved_concrete_function.name]
+  input_signature = (saved_concrete_function.canonicalized_input_signature
+                     .list_value.values)
+  # pylint: disable=protected-access
+  # Set metadata required for the concrete function to accept keyword and
+  # positional arguments in __call__. Normally this is set in
+  # get_concrete_function.
+  concrete_function._arg_keywords = [
+      spec.tensor_spec_value.name for spec in input_signature]
+  # TODO(allenl): Should we preserve the number of allowed positional arguments?
+  concrete_function._num_positional_args = len(input_signature)
+  # pylint: enable=protected-access
+  concrete_function.add_to_graph()
+  return concrete_function
+
+
 def recreate_function(saved_function, concrete_functions):
   """Creates a `Function` from a `SavedFunction`.
 
diff --git a/tensorflow/python/saved_model/function_serialization.py b/tensorflow/python/saved_model/function_serialization.py
index d973f97d42..c79c61b60c 100644
--- a/tensorflow/python/saved_model/function_serialization.py
+++ b/tensorflow/python/saved_model/function_serialization.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.framework import func_graph as func_graph_module
+from tensorflow.python.framework import tensor_spec
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.saved_model import nested_structure_coder
 from tensorflow.python.saved_model import saved_object_graph_pb2
@@ -38,6 +39,46 @@ def _serialize_function_spec(function_spec, coder):
   return proto
 
 
+def _serialize_concrete_function_with_signature(
+    concrete_function, node_ids, signature_args, coder):
+  """Build a SavedConcreteFunction."""
+  bound_inputs = []
+  try:
+    for capture in concrete_function.captured_inputs:
+      bound_inputs.append(node_ids[capture])
+  except KeyError:
+    # TODO(andresp): Would it better to throw an exception?
+    logging.warning(
+        "Concrete function %s not added to object based saved model as it "
+        "captures tensor %s which is unsupported or not reachable from root.",
+        concrete_function.name, capture)
+    return None
+  concrete_function_proto = saved_object_graph_pb2.SavedConcreteFunction()
+  concrete_function_proto.name = concrete_function.name
+  concrete_function_proto.canonicalized_input_signature.CopyFrom(
+      coder.encode_structure(signature_args))
+  structured_outputs = func_graph_module.convert_structure_to_signature(
+      concrete_function.structured_outputs)
+  concrete_function_proto.output_signature.CopyFrom(
+      coder.encode_structure(structured_outputs))
+  concrete_function_proto.bound_inputs.extend(bound_inputs)
+  return concrete_function_proto
+
+
+def serialize_concrete_function(concrete_function, node_ids):
+  """Build a standalone SavedConcreteFunction."""
+  coder = nested_structure_coder.StructureCoder()
+  signature = []
+  # Construct a flat list of TensorSpecs for this concrete function. On load
+  # we'll parse them and recreate the call metadata.
+  for input_tensor, input_keyword in zip(
+      concrete_function.inputs, concrete_function._arg_keywords):  # pylint: disable=protected-access
+    signature.append(tensor_spec.TensorSpec(
+        input_tensor.shape, input_tensor.dtype, name=input_keyword))
+  return _serialize_concrete_function_with_signature(
+      concrete_function, node_ids, signature, coder)
+
+
 def serialize_function(function, node_ids):
   """Build a SavedFunction proto."""
   coder = nested_structure_coder.StructureCoder()
@@ -48,27 +89,11 @@ def serialize_function(function, node_ids):
   all_concrete_functions = \
       function._list_all_concrete_functions_for_serialization()  # pylint: disable=protected-access
   for concrete_function in all_concrete_functions:
-    bound_inputs = []
-    try:
-      for capture in concrete_function.captured_inputs:
-        bound_inputs.append(node_ids[capture])
-    except KeyError:
-      # TODO(andresp): Would it better to throw an exception?
-      logging.warning(
-          "Concrete function %s not added to object based saved model as it "
-          "captures tensor %s which is unsupported or not reachable from root.",
-          concrete_function.name, capture)
-      continue
     signature_args, signature_kwargs = \
         concrete_function.structured_input_signature
     del signature_kwargs
-    concrete_function_proto = proto.concrete_function.add()
-    concrete_function_proto.name = concrete_function.name
-    concrete_function_proto.canonicalized_input_signature.CopyFrom(
-        coder.encode_structure(signature_args))
-    structured_outputs = func_graph_module.convert_structure_to_signature(
-        concrete_function.structured_outputs)
-    concrete_function_proto.output_signature.CopyFrom(
-        coder.encode_structure(structured_outputs))
-    concrete_function_proto.bound_inputs.extend(bound_inputs)
+    concrete_function_proto = _serialize_concrete_function_with_signature(
+        concrete_function, node_ids, signature_args, coder)
+    if concrete_function_proto is not None:
+      proto.concrete_function.add().CopyFrom(concrete_function_proto)
   return proto
diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
index 98990905f0..2aa4924b29 100644
--- a/tensorflow/python/saved_model/load.py
+++ b/tensorflow/python/saved_model/load.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import functools
 import os
 
 from tensorflow.python.lib.io import file_io
@@ -48,53 +49,62 @@ class _Loader(object):
         function_deserialization.load_function_def_library(
             meta_graph.graph_def.library))
     self._load_all()
-    self._setup_concrete_functions()
+    self._setup_functions()
     self._restore_checkpoint()
 
-  def _setup_concrete_functions(self):
+  def _setup_concrete_function(
+      self, concrete_function, seen_functions, coder):
+    """Setup captured tensors and outputs for a single concrete function."""
+    name = concrete_function.name
+    bound_inputs = [
+        self._get_tensor_from_node(node_id)
+        for node_id in concrete_function.bound_inputs]
+    bound_variables = [
+        self._nodes[node_id]
+        for node_id in concrete_function.bound_inputs
+        if self._proto.nodes[node_id].WhichOneof("kind") == "variable"
+    ]
+    if name in seen_functions:
+      raise RuntimeError(
+          "Concrete function with a duplicate name: %s." % name)
+    else:
+      seen_functions.add(name)
+      # TODO(andresp): This is only injecting the captured inputs into the
+      # concrete function, note that we did not modify the FuncGraph
+      # itself.
+      function = self._concrete_functions[name]
+      function._captured_inputs = bound_inputs  # pylint: disable=protected-access
+      function._func_graph.variables = bound_variables  # pylint: disable=protected-access
+      # By setting the structured_outputs directly, we can rely on this
+      # function_lib.ConcreteFunction object to perform the output repacking
+      # logic. The only limitation of that logic is that it only works
+      # with output that is convertible to Tensors and the conversion
+      # always happens. For example tf.TensorShape([2, 3]) will be
+      # converted to Tensor representing [2, 3].
+      original_outputs = coder.decode_proto(
+          concrete_function.output_signature)
+      # The original_outputs here had Tensors converted to TensorSpecs, so
+      # the restored function's structured_outputs field will not be
+      # exactly the same. Fortunately the repacking logic cares only about
+      # the structure.
+      # TODO(vbardiovsky): Should we just replicate the structures, with
+      # Nones instead of real objects? Decide when we start solving
+      # idempotency.
+      function._func_graph.structured_outputs = original_outputs  # pylint: disable=protected-access
+
+  def _setup_functions(self):
+
     """Setup captures and output structure in restored concrete functions."""
     seen_concrete_functions = set()
     coder = nested_structure_coder.StructureCoder()
     for object_proto in self._proto.nodes:
-      if object_proto.WhichOneof("kind") == "function":
+      if object_proto.WhichOneof("kind") == "concrete_function":
+        self._setup_concrete_function(
+            object_proto.concrete_function, seen_concrete_functions, coder)
+      elif object_proto.WhichOneof("kind") == "function":
         for concrete_function in object_proto.function.concrete_function:
-          name = concrete_function.name
-          bound_inputs = [
-              self._get_tensor_from_node(node_id)
-              for node_id in concrete_function.bound_inputs
-          ]
-          bound_variables = [
-              self._nodes[node_id]
-              for node_id in concrete_function.bound_inputs
-              if self._proto.nodes[node_id].WhichOneof("kind") == "variable"
-          ]
-          if name in seen_concrete_functions:
-            raise RuntimeError(
-                "Concrete function with a duplicate name: %s." % name)
-          else:
-            seen_concrete_functions.add(name)
-            # TODO(andresp): This is only injecting the captured inputs into the
-            # concrete function, note that we did not modify the FuncGraph
-            # itself.
-            revived_function = self._concrete_functions[name]
-            revived_function._captured_inputs = bound_inputs  # pylint: disable=protected-access
-            revived_function._func_graph.variables = bound_variables  # pylint: disable=protected-access
-            # By setting the structured_outputs directly, we can rely on this
-            # function_lib.ConcreteFunction object to perform the output
-            # repacking logic. The only limitation of that logic is that it only
-            # works with output that is convertible to Tensors and the
-            # conversion always happens. For example tf.TensorShape([2, 3])
-            # will be converted to Tensor representing [2, 3].
-            original_outputs = coder.decode_proto(
-                concrete_function.output_signature)
-            # The original_outputs here had Tensors converted to TensorSpecs, so
-            # the restored concrete function's structured_outputs field will not
-            # be exactly the same. Fortunately the repacking logic cares only
-            # about the structure.
-            # TODO(vbardiovsky): Should we just replicate the structures, with
-            # Nones instead of real objects? Decide when we start solving
-            # idempotency.
-            revived_function._func_graph.structured_outputs = original_outputs  # pylint: disable=protected-access
+          self._setup_concrete_function(
+              concrete_function, seen_concrete_functions, coder)
 
   def _get_tensor_from_node(self, node_id):
     obj = self._nodes[node_id]
@@ -137,6 +147,8 @@ class _Loader(object):
         "user_object": lambda: self._recreate_user_object(proto.user_object),
         "asset": lambda: self._recreate_asset(proto.asset),
         "function": lambda: self._recreate_function(proto.function),
+        "concrete_function": functools.partial(
+            self._recreate_concrete_function, proto.concrete_function),
         "variable": lambda: self._recreate_variable(proto.variable),
     }
     kind = proto.WhichOneof("kind")
@@ -168,6 +180,10 @@ class _Loader(object):
     return function_deserialization.recreate_function(
         proto, self._concrete_functions), setattr
 
+  def _recreate_concrete_function(self, proto):
+    return function_deserialization.recreate_concrete_function(
+        proto, self._concrete_functions), setattr
+
   def _recreate_variable(self, proto):
     # TODO(andresp): Can we use the checkpointed value as initializer?
     dummy_value = init_ops.Zeros(dtype=proto.dtype)(shape=proto.shape)
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index c8c8f6e1b3..e3dcbda50a 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -418,6 +418,57 @@ class LoadTest(test.TestCase):
     self.assertAllEqual([2, 4, 6],
                         imported.f(constant_op.constant([1, 2, 3])).numpy())
 
+  def test_concrete_function(self):
+
+    @def_function.function(
+        input_signature=[tensor_spec.TensorSpec([None], dtypes.int32)])
+    def func(x):
+      return 2 * x
+
+    root = tracking.AutoCheckpointable()
+    root.f = func.get_concrete_function()
+
+    self.assertAllEqual([2], root.f(constant_op.constant([1])).numpy())
+    self.assertAllEqual([2, 4], root.f(constant_op.constant([1, 2])).numpy())
+
+    imported = self.cycle(root)
+
+    self.assertAllEqual([2, 4, 6, 8],
+                        imported.f(constant_op.constant([1, 2, 3, 4])).numpy())
+    self.assertAllEqual([2, 4, 6],
+                        imported.f(constant_op.constant([1, 2, 3])).numpy())
+
+  def test_concrete_function_no_signature(self):
+    @def_function.function
+    def func(x):
+      return 2 * x
+
+    root = tracking.AutoCheckpointable()
+    root.f = func.get_concrete_function(constant_op.constant([1]))
+    self.assertAllEqual([4], root.f(constant_op.constant([2])).numpy())
+    imported = self.cycle(root)
+    self.assertAllEqual([6],
+                        imported.f(constant_op.constant([3])).numpy())
+
+  def test_concrete_function_backprop(self):
+    @def_function.function(
+        input_signature=[tensor_spec.TensorSpec([None], dtypes.float32)])
+    def func(x):
+      return x ** 2.
+    root = tracking.AutoCheckpointable()
+    root.f = func.get_concrete_function()
+
+    def _compute_gradient(function):
+      with backprop.GradientTape() as tape:
+        inp = constant_op.constant(1.)
+        tape.watch(inp)
+        output = function(inp)
+      return tape.gradient(output, inp)
+
+    self.assertEqual(2., _compute_gradient(root.f).numpy())
+    imported = self.cycle(root)
+    self.assertEqual(2., _compute_gradient(imported.f).numpy())
+
   def test_dict(self):
     root = tracking.AutoCheckpointable()
     root.variables = dict(a=variables.Variable(1.))
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index c35e09457f..9c3ace7325 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -73,18 +73,24 @@ class _SaveableView(object):
     self.functions = util.ObjectIdentityDictionary()
 
     # Also add `Function`s as nodes.
-    for obj in self.nodes:
+    nodes_without_functions = list(self.nodes)
+    for obj in nodes_without_functions:
       self.functions[obj] = self._list_functions(obj)
       for function in self.functions[obj].values():
         if function not in self.node_ids:
           self.node_ids[function] = len(self.nodes)
           self.nodes.append(function)
-        # Force listing the concrete functions for the side effects:
-        #  - populate the cache for `Function`s that have an
-        #  input_signature and have not been called.
-        #  - force side effects of creation of concrete functions, e.g. create
-        #  variables on first run.
-        function._list_all_concrete_functions_for_serialization()  # pylint: disable=protected-access
+          # Avoids recursing into functions to see if other functions are
+          # assigned to attributes. This is sometimes true for concrete
+          # functions but not helpful.
+          self.functions[function] = {}
+        if isinstance(function, def_function.Function):
+          # Force listing the concrete functions for the side effects:
+          #  - populate the cache for functions that have an input_signature
+          #  and have not been called.
+          #  - force side effects of creation of concrete functions, e.g. create
+          #  variables on first run.
+          function._list_all_concrete_functions_for_serialization()  # pylint: disable=protected-access
 
   @property
   def root(self):
@@ -96,7 +102,7 @@ class _SaveableView(object):
       assert self.node_ids[node] == node_id
       object_proto = proto.nodes.add()
       object_proto.slot_variables.extend(self.slot_variables.get(node, ()))
-      if isinstance(node, def_function.Function):
+      if isinstance(node, (def_function.Function, defun.ConcreteFunction)):
         continue
       for child in node._checkpoint_dependencies:  # pylint: disable=protected-access
         child_proto = object_proto.children.add()
@@ -117,7 +123,8 @@ class _SaveableView(object):
         # We really don't want to throw an exception just because some object's
         # attribute accessor is broken.
         attribute_value = None
-      if isinstance(attribute_value, def_function.Function):
+      if isinstance(attribute_value, (def_function.Function,
+                                      defun.ConcreteFunction)):
         functions[attribute_name] = attribute_value
     return functions
 
@@ -554,8 +561,11 @@ def _fill_meta_graph_def(meta_graph_def, saveable_view, signature_functions,
   concrete_functions = []
   for obj in accessible_objects:
     for function in saveable_view.functions[obj].values():
-      concrete_functions.extend(
-          function._list_all_concrete_functions_for_serialization())  # pylint: disable=protected-access
+      if isinstance(function, defun.ConcreteFunction):
+        concrete_functions.append(function)
+      else:
+        concrete_functions.extend(
+            function._list_all_concrete_functions_for_serialization())  # pylint: disable=protected-access
 
   with exported_graph.as_default():
     signatures = _generate_signatures(signature_functions, resource_map)
@@ -617,6 +627,9 @@ def _write_object_proto(obj, proto, asset_file_def_index, node_ids):
   elif isinstance(obj, def_function.Function):
     proto.function.CopyFrom(
         function_serialization.serialize_function(obj, node_ids))
+  elif isinstance(obj, defun.ConcreteFunction):
+    proto.concrete_function.CopyFrom(
+        function_serialization.serialize_concrete_function(obj, node_ids))
   else:
     registered_type_proto = revived_types.serialize(obj)
     if registered_type_proto is None:
diff --git a/tensorflow/python/saved_model/saved_object_graph.proto b/tensorflow/python/saved_model/saved_object_graph.proto
index 38940f4c1c..6f5a952083 100644
--- a/tensorflow/python/saved_model/saved_object_graph.proto
+++ b/tensorflow/python/saved_model/saved_object_graph.proto
@@ -54,6 +54,7 @@ message SavedObject {
     SavedAsset asset = 5;
     SavedFunction function = 6;
     SavedVariable variable = 7;
+    SavedConcreteFunction concrete_function = 8;
   }
 }
 
-- 
GitLab


From 6dae9a68f0e6e902aaf8811165b8f70d2cc3624a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 18:23:04 -0800
Subject: [PATCH 0839/2345] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 229663723

---
 tensorflow/go/op/wrappers.go | 126 +++++++++++++++++------------------
 1 file changed, 63 insertions(+), 63 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index fbabb37c0e..134185a03b 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -18447,69 +18447,6 @@ func Timestamp(scope *Scope) (ts tf.Output) {
 	return op.Output(0)
 }
 
-// ResourceSparseApplyKerasMomentumAttr is an optional argument to ResourceSparseApplyKerasMomentum.
-type ResourceSparseApplyKerasMomentumAttr func(optionalAttr)
-
-// ResourceSparseApplyKerasMomentumUseLocking sets the optional use_locking attribute to value.
-//
-// value: If `True`, updating of the var and accum tensors will be protected
-// by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceSparseApplyKerasMomentumUseLocking(value bool) ResourceSparseApplyKerasMomentumAttr {
-	return func(m optionalAttr) {
-		m["use_locking"] = value
-	}
-}
-
-// ResourceSparseApplyKerasMomentumUseNesterov sets the optional use_nesterov attribute to value.
-//
-// value: If `True`, the tensor passed to compute grad will be
-// var + momentum * accum, so in the end, the var you get is actually
-// var + momentum * accum.
-// If not specified, defaults to false
-func ResourceSparseApplyKerasMomentumUseNesterov(value bool) ResourceSparseApplyKerasMomentumAttr {
-	return func(m optionalAttr) {
-		m["use_nesterov"] = value
-	}
-}
-
-// Update relevant entries in '*var' and '*accum' according to the momentum scheme.
-//
-// Set use_nesterov = True if you want to use Nesterov momentum.
-//
-// That is for rows we have grad for, we update var and accum as follows:
-//
-// accum = accum * momentum - lr * grad
-// var += accum
-//
-// Arguments:
-//	var_: Should be from a Variable().
-//	accum: Should be from a Variable().
-//	lr: Learning rate. Must be a scalar.
-//	grad: The gradient.
-//	indices: A vector of indices into the first dimension of var and accum.
-//	momentum: Momentum. Must be a scalar.
-//
-// Returns the created operation.
-func ResourceSparseApplyKerasMomentum(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, indices tf.Output, momentum tf.Output, optional ...ResourceSparseApplyKerasMomentumAttr) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "ResourceSparseApplyKerasMomentum",
-		Input: []tf.Input{
-			var_, accum, lr, grad, indices, momentum,
-		},
-		Attrs: attrs,
-	}
-	return scope.AddOperation(opspec)
-}
-
 // VariableShapeAttr is an optional argument to VariableShape.
 type VariableShapeAttr func(optionalAttr)
 
@@ -23474,6 +23411,69 @@ func TakeManySparseFromTensorsMap(scope *Scope, sparse_handles tf.Output, dtype
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
+// ResourceSparseApplyKerasMomentumAttr is an optional argument to ResourceSparseApplyKerasMomentum.
+type ResourceSparseApplyKerasMomentumAttr func(optionalAttr)
+
+// ResourceSparseApplyKerasMomentumUseLocking sets the optional use_locking attribute to value.
+//
+// value: If `True`, updating of the var and accum tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceSparseApplyKerasMomentumUseLocking(value bool) ResourceSparseApplyKerasMomentumAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
+	}
+}
+
+// ResourceSparseApplyKerasMomentumUseNesterov sets the optional use_nesterov attribute to value.
+//
+// value: If `True`, the tensor passed to compute grad will be
+// var + momentum * accum, so in the end, the var you get is actually
+// var + momentum * accum.
+// If not specified, defaults to false
+func ResourceSparseApplyKerasMomentumUseNesterov(value bool) ResourceSparseApplyKerasMomentumAttr {
+	return func(m optionalAttr) {
+		m["use_nesterov"] = value
+	}
+}
+
+// Update relevant entries in '*var' and '*accum' according to the momentum scheme.
+//
+// Set use_nesterov = True if you want to use Nesterov momentum.
+//
+// That is for rows we have grad for, we update var and accum as follows:
+//
+// accum = accum * momentum - lr * grad
+// var += accum
+//
+// Arguments:
+//	var_: Should be from a Variable().
+//	accum: Should be from a Variable().
+//	lr: Learning rate. Must be a scalar.
+//	grad: The gradient.
+//	indices: A vector of indices into the first dimension of var and accum.
+//	momentum: Momentum. Must be a scalar.
+//
+// Returns the created operation.
+func ResourceSparseApplyKerasMomentum(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, indices tf.Output, momentum tf.Output, optional ...ResourceSparseApplyKerasMomentumAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceSparseApplyKerasMomentum",
+		Input: []tf.Input{
+			var_, accum, lr, grad, indices, momentum,
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
 // Assigns a new value to a variable.
 //
 // Any ReadVariableOp with a control dependency on this op is guaranteed to return
-- 
GitLab


From f960af59b7d86ed5ece2c604f0400ef4f3afb4b2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 18:24:53 -0800
Subject: [PATCH 0840/2345] Update ops-related pbtxt files.

PiperOrigin-RevId: 229663932
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 58 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 58 +++++++++++++++++++
 2 files changed, 116 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 9990b8629c..3b7a7b812a 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -22275,6 +22275,28 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ExperimentalChooseFastestDataset"
+  input_arg {
+    name: "input_datasets"
+    type: DT_VARIANT
+    number_attr: "N"
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "N"
+    type: "int"
+    has_minimum: true
+    minimum: 2
+  }
+  attr {
+    name: "num_experiments"
+    type: "int"
+  }
+}
 op {
   name: "ExperimentalDatasetCardinality"
   input_arg {
@@ -23647,6 +23669,42 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ExperimentalTakeWhileDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "other_arguments"
+    type_list_attr: "Targuments"
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "predicate"
+    type: "func"
+  }
+  attr {
+    name: "Targuments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "ExperimentalThreadPoolDataset"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 8212282623..92a7bb9a34 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -10435,6 +10435,28 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ExperimentalChooseFastestDataset"
+  input_arg {
+    name: "input_datasets"
+    type: DT_VARIANT
+    number_attr: "N"
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "N"
+    type: "int"
+    has_minimum: true
+    minimum: 2
+  }
+  attr {
+    name: "num_experiments"
+    type: "int"
+  }
+}
 op {
   name: "ExperimentalDatasetCardinality"
   input_arg {
@@ -11457,6 +11479,42 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ExperimentalTakeWhileDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "other_arguments"
+    type_list_attr: "Targuments"
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "predicate"
+    type: "func"
+  }
+  attr {
+    name: "Targuments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "ExperimentalThreadPoolDataset"
   input_arg {
-- 
GitLab


From 52af2770f94f0f660e4019d95821052c08b18a41 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 20:07:31 -0800
Subject: [PATCH 0841/2345] Removing unnecessary parameters.

PiperOrigin-RevId: 229673560
---
 third_party/gpus/cuda_configure.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl
index f8f3e90708..1988c4ae96 100644
--- a/third_party/gpus/cuda_configure.bzl
+++ b/third_party/gpus/cuda_configure.bzl
@@ -1067,10 +1067,10 @@ def _create_dummy_repository(repository_ctx):
 
   # Create dummy files for the CUDA toolkit since they are still required by
   # tensorflow/core/platform/default/build_config:cuda.
-  repository_ctx.file("cuda/cuda/include/cuda.h", "")
-  repository_ctx.file("cuda/cuda/include/cublas.h", "")
-  repository_ctx.file("cuda/cuda/include/cudnn.h", "")
-  repository_ctx.file("cuda/cuda/extras/CUPTI/include/cupti.h", "")
+  repository_ctx.file("cuda/cuda/include/cuda.h")
+  repository_ctx.file("cuda/cuda/include/cublas.h")
+  repository_ctx.file("cuda/cuda/include/cudnn.h")
+  repository_ctx.file("cuda/cuda/extras/CUPTI/include/cupti.h")
   repository_ctx.file("cuda/cuda/lib/%s" % _lib_name("cuda", cpu_value))
   repository_ctx.file("cuda/cuda/lib/%s" % _lib_name("cudart", cpu_value))
   repository_ctx.file(
-- 
GitLab


From 77e2ba9188f4343214adc30de63a98d4999103f7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 20:45:42 -0800
Subject: [PATCH 0842/2345] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 229676560

---
 tensorflow/go/op/wrappers.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 134185a03b..666ebe33df 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -13421,7 +13421,7 @@ func StaticRegexReplaceReplaceGlobal(value bool) StaticRegexReplaceAttr {
 // Arguments:
 //	input: The text to be processed.
 //	pattern: The regular expression to match the input.
-//	rewrite: The rewrite to be applied to the matched expresion.
+//	rewrite: The rewrite to be applied to the matched expression.
 //
 // Returns The text after applying pattern and rewrite.
 func StaticRegexReplace(scope *Scope, input tf.Output, pattern string, rewrite string, optional ...StaticRegexReplaceAttr) (output tf.Output) {
-- 
GitLab


From 69b9e5358b7e64efd52f742847eeaf5f893762ab Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 22:02:05 -0800
Subject: [PATCH 0843/2345] Remove HostBuffers

HostBuffers add significant complexity and have no users.

PiperOrigin-RevId: 229682740
---
 tensorflow/stream_executor/BUILD         |  7 ----
 tensorflow/stream_executor/dnn.h         | 16 ---------
 tensorflow/stream_executor/host_buffer.h | 46 ------------------------
 tensorflow/stream_executor/stream.cc     | 33 -----------------
 4 files changed, 102 deletions(-)
 delete mode 100644 tensorflow/stream_executor/host_buffer.h

diff --git a/tensorflow/stream_executor/BUILD b/tensorflow/stream_executor/BUILD
index 7cf7e75a55..fca28e016a 100644
--- a/tensorflow/stream_executor/BUILD
+++ b/tensorflow/stream_executor/BUILD
@@ -206,7 +206,6 @@ cc_library(
 cc_library(
     name = "stream",
     srcs = [
-        "host_buffer.h",
         "stream.cc",
     ],
     hdrs = ["stream.h"],
@@ -453,12 +452,6 @@ cc_library(
     ],
 )
 
-cc_library(
-    name = "host_buffer",
-    hdrs = ["host_buffer.h"],
-    deps = [":dnn"],
-)
-
 tf_proto_library(
     name = "dnn_proto",
     srcs = ["dnn.proto"],
diff --git a/tensorflow/stream_executor/dnn.h b/tensorflow/stream_executor/dnn.h
index f5eef29109..f60d1ada24 100644
--- a/tensorflow/stream_executor/dnn.h
+++ b/tensorflow/stream_executor/dnn.h
@@ -2069,22 +2069,6 @@ class DnnSupport {
       QuantizedActivationMode mode,
       DeviceMemory<float>* gpu_unquantized_dst) = 0;
 
-  // Enqueues an asynchronous copy of the contents of buffer_src to
-  // gpu_unquantized_dst.
-  virtual bool DoCopyHostBuffer2Device(
-      Stream* stream, HostBuffer* buffer_src,
-      DeviceMemory<float>* gpu_unquantized_dst) {
-    return false;
-  }
-
-  // Enqueues an asynchronous copy of the contents of gpu_unquantized_src to
-  // buffer_dst.
-  virtual bool DoCopyDevice2HostBuffer(
-      Stream* stream, const DeviceMemory<float>& gpu_unquantized_src,
-      HostBuffer* buffer_dst) {
-    return false;
-  }
-
   // Create an RNN descriptor based on model shapes and configurations.
   // The caller retains the ownership of the descriptor.
   //
diff --git a/tensorflow/stream_executor/host_buffer.h b/tensorflow/stream_executor/host_buffer.h
deleted file mode 100644
index 20299da517..0000000000
--- a/tensorflow/stream_executor/host_buffer.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_STREAM_EXECUTOR_HOST_BUFFER_H_
-#define TENSORFLOW_STREAM_EXECUTOR_HOST_BUFFER_H_
-
-#include "tensorflow/stream_executor/dnn.h"
-
-namespace stream_executor {
-
-// A HostBuffer is a block of memory in host memory containing the data for a
-// dnn::BatchDescriptor using a device-dependent memory layout.
-// Derived classes provide methods to construct a HostBuffer for a specific
-// device, and to copy data in and out of the buffer.
-class HostBuffer {
- public:
-  const dnn::BatchDescriptor& descriptor() const { return descriptor_; }
-
-  // Returns a string describing the HostBuffer.
-  virtual string AsString() const = 0;
-
- protected:
-  // Construct a HostBuffer from the supplied dnn::BatchDescriptor.
-  explicit HostBuffer(const dnn::BatchDescriptor& descriptor)
-      : descriptor_(descriptor) {}
-  virtual ~HostBuffer() {}
-
- private:
-  const dnn::BatchDescriptor descriptor_;
-};
-
-}  // namespace stream_executor
-
-#endif  // TENSORFLOW_STREAM_EXECUTOR_HOST_BUFFER_H_
diff --git a/tensorflow/stream_executor/stream.cc b/tensorflow/stream_executor/stream.cc
index 0dceacb228..e7485ca426 100644
--- a/tensorflow/stream_executor/stream.cc
+++ b/tensorflow/stream_executor/stream.cc
@@ -20,7 +20,6 @@ limitations under the License.
 #include "absl/strings/str_cat.h"
 #include "third_party/eigen3/Eigen/Core"
 #include "tensorflow/stream_executor/blas.h"
-#include "tensorflow/stream_executor/host_buffer.h"
 #include "tensorflow/stream_executor/host_or_device_scalar.h"
 #include "tensorflow/stream_executor/lib/stacktrace.h"
 #include "tensorflow/stream_executor/platform.h"
@@ -95,8 +94,6 @@ string ToVlogString(const void *ptr) {
   return out.str();
 }
 
-string ToVlogString(const HostBuffer &buffer) { return buffer.AsString(); }
-
 template <class T>
 string ToVlogString(const std::complex<T> &c) {
   // StrCat does not convert std::complex to text.
@@ -2103,36 +2100,6 @@ Stream &Stream::ThenMemcpyH2DQuantized(
   return *this;
 }
 
-Stream &Stream::ThenCopyHostBuffer2Device(
-    HostBuffer *buffer_src, DeviceMemory<float> *gpu_unquantized_dst) {
-  VLOG_CALL(PARAM(*buffer_src), PARAM(gpu_unquantized_dst));
-
-  if (ok()) {
-    if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      CheckError(
-          dnn->DoCopyHostBuffer2Device(this, buffer_src, gpu_unquantized_dst));
-    } else {
-      SetErrorAndLogNoDnnSupport();
-    }
-  }
-  return *this;
-}
-
-Stream &Stream::ThenCopyDevice2HostBuffer(
-    const DeviceMemory<float> &gpu_unquantized_src, HostBuffer *buffer_dst) {
-  VLOG_CALL(PARAM(gpu_unquantized_src), PARAM(*buffer_dst));
-
-  if (ok()) {
-    if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      CheckError(
-          dnn->DoCopyDevice2HostBuffer(this, gpu_unquantized_src, buffer_dst));
-    } else {
-      SetErrorAndLogNoDnnSupport();
-    }
-  }
-  return *this;
-}
-
 Stream *Stream::GetOrCreateSubStream() {
   mutex_lock lock(mu_);
 
-- 
GitLab


From 8eb3cbcb423c4d840e88a910c5db47404207b8a4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 23:25:53 -0800
Subject: [PATCH 0844/2345] Fix typo.

PiperOrigin-RevId: 229689427
---
 tensorflow/compiler/xla/g3doc/broadcasting.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/g3doc/broadcasting.md b/tensorflow/compiler/xla/g3doc/broadcasting.md
index 2870869a2c..5c0525c1e9 100644
--- a/tensorflow/compiler/xla/g3doc/broadcasting.md
+++ b/tensorflow/compiler/xla/g3doc/broadcasting.md
@@ -168,7 +168,7 @@ consult the
 
 Broadcasting of a lower-rank array to a higher-rank array **and** broadcasting
 using degenerate dimensions can both be performed in the same binary operation.
-For example, a vector of size 4 and an matrix of size 1x2 can be added together
+For example, a vector of size 4 and a matrix of size 1x2 can be added together
 using broadcast dimensions value of (0):
 
     |1 2 3 4| + [5 6]    // [5 6] is a 1x2 matrix, not a vector.
@@ -176,7 +176,7 @@ using broadcast dimensions value of (0):
 First the vector is broadcast up to rank 2 (matrix) using the broadcast
 dimensions. The single value (0) in the broadcast dimensions indicates that
 dimension zero of the vector matches to dimension zero of the matrix. This
-produces an matrix of size 4xM where the value M is chosen to match the
+produces a matrix of size 4xM where the value M is chosen to match the
 corresponding dimension size in the 1x2 array. Therefore, a 4x2 matrix is
 produced:
 
-- 
GitLab


From ca2d0b652e1bc8e6d637318a1fbcb56bd4e6f0d0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 01:03:09 -0800
Subject: [PATCH 0845/2345] compat: Update forward compatibility horizon to
 2019-01-17

PiperOrigin-RevId: 229698933
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 773b3a41f9..c558722061 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 16)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 17)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From a286bc2819f4b3f3f42dcfe3fb8969979d6eaea3 Mon Sep 17 00:00:00 2001
From: Miguel Morin <32396311+miguelmorin@users.noreply.github.com>
Date: Thu, 17 Jan 2019 11:45:25 +0000
Subject: [PATCH 0846/2345] Add details for running tests from nightly build
 images. Closes #24993.

The current documentation guidelines throw an error after running the
commands in the tensorflow nightly build container. Following this
Magenta issue:

https://github.com/tensorflow/magenta/issues/1232

running the test requires changing directory. Rather than changing the
WORKDIR in the image, which would probably affect many uses and users,
this change adds the direction of changing directory after mentioning
the nightly build image.
---
 CONTRIBUTING.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 4a296f265f..4bd9c77d29 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -158,7 +158,10 @@ There are two ways to run TensorFlow unit tests.
    for the required packages. Alternatively, use the said
    [Docker images](https://hub.docker.com/r/tensorflow/tensorflow/tags/), e.g.,
    `tensorflow/tensorflow:nightly-devel` and `tensorflow/tensorflow:nightly-devel-gpu`
-   for development to avoid installing the packages directly on your system.
+   for development to avoid installing the packages directly on your system (in
+   which case remember to change directory from `/root` to `/tensorflow` once
+   you get into the running container so `bazel` can find the `tensorflow`
+   workspace).
 
    Once you have the packages installed, you can run a specific unit test in
    bazel by doing as follows:
-- 
GitLab


From 0b537e5b7d4eca61b058d2415f8f93b253506a1a Mon Sep 17 00:00:00 2001
From: Grzegorz Pawelczak <grzegorzp@graphcore.ai>
Date: Thu, 17 Jan 2019 12:20:35 +0000
Subject: [PATCH 0847/2345] Don't dump the whole literal into VLOG(1)

---
 tensorflow/compiler/jit/xla_device_context.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/jit/xla_device_context.cc b/tensorflow/compiler/jit/xla_device_context.cc
index 1f3afe8822..28681bb8b0 100644
--- a/tensorflow/compiler/jit/xla_device_context.cc
+++ b/tensorflow/compiler/jit/xla_device_context.cc
@@ -131,7 +131,7 @@ void XlaDeviceContext::CopyCPUTensorToDevice(const Tensor* cpu_tensor,
         xla::ShapeUtil::MakeShape(shape.element_type(),
                                   xla::AsInt64Slice(shape.dimensions())));
 
-    VLOG(1) << "Transfer to device as literal: " << literal.ToString() << " "
+    VLOG(2) << "Transfer to device as literal: " << literal.ToString() << " "
             << xla_tensor->shaped_buffer().ToString();
     if (UseMultipleStreams() &&
         !transfer_manager_->CanShapedBufferBeAccessedNow(
@@ -214,7 +214,7 @@ void XlaDeviceContext::CopyDeviceTensorToCPU(const Tensor* device_tensor,
       device_to_host_stream_.get(), xla_tensor->shaped_buffer(), literal,
       [ref, xla_tensor, done](xla::Status status) {
         done([&]() -> Status {
-          VLOG(1) << "Transfer from device as literal: "
+          VLOG(2) << "Transfer from device as literal: "
                   << xla_tensor->shaped_buffer().ToString();
           return status;
         }());
-- 
GitLab


From 9331096d56002c7fcb8bae411684b8b78fc196c4 Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Thu, 17 Jan 2019 06:29:06 -0800
Subject: [PATCH 0848/2345] Experimental implementation of tf.Module.

PiperOrigin-RevId: 229735738
---
 tensorflow/python/BUILD                       |   1 +
 tensorflow/python/__init__.py                 |   1 +
 tensorflow/python/module/BUILD                |  30 ++
 tensorflow/python/module/module.py            | 402 ++++++++++++++++++
 tensorflow/python/module/module_test.py       | 355 ++++++++++++++++
 .../v1/tensorflow.experimental.-module.pbtxt  |  47 ++
 .../golden/v1/tensorflow.experimental.pbtxt   |   4 +
 .../v2/tensorflow.experimental.-module.pbtxt  |  47 ++
 .../golden/v2/tensorflow.experimental.pbtxt   |   4 +
 9 files changed, 891 insertions(+)
 create mode 100644 tensorflow/python/module/BUILD
 create mode 100644 tensorflow/python/module/module.py
 create mode 100644 tensorflow/python/module/module_test.py
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.experimental.-module.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.experimental.-module.pbtxt

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index e32be0039f..b61c33f319 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -178,6 +178,7 @@ py_library(
         "//tensorflow/python/eager:def_function",
         "//tensorflow/python/eager:profiler",
         "//tensorflow/python/eager:remote",
+        "//tensorflow/python/module",
         "//tensorflow/python/ops/distributions",
         "//tensorflow/python/ops/linalg",
         "//tensorflow/python/ops/losses",
diff --git a/tensorflow/python/__init__.py b/tensorflow/python/__init__.py
index 5172dbb836..398fb375e1 100644
--- a/tensorflow/python/__init__.py
+++ b/tensorflow/python/__init__.py
@@ -82,6 +82,7 @@ from tensorflow.python import distribute
 from tensorflow.python import keras
 from tensorflow.python.feature_column import feature_column_lib as feature_column
 from tensorflow.python.layers import layers
+from tensorflow.python.module import module
 from tensorflow.python.ops import bitwise_ops as bitwise
 from tensorflow.python.ops import image_ops as image
 from tensorflow.python.ops import manip_ops as manip
diff --git a/tensorflow/python/module/BUILD b/tensorflow/python/module/BUILD
new file mode 100644
index 0000000000..64c226e4c9
--- /dev/null
+++ b/tensorflow/python/module/BUILD
@@ -0,0 +1,30 @@
+package(default_visibility = ["//tensorflow:internal"])
+
+load("//tensorflow:tensorflow.bzl", "tf_py_test")
+
+licenses(["notice"])  # Apache 2.0
+
+exports_files(["LICENSE"])
+
+py_library(
+    name = "module",
+    srcs = ["module.py"],
+    deps = [
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:util",
+        "//tensorflow/python:variables",
+        "//tensorflow/python/training/checkpointable:tracking",
+        "@six_archive//:six",
+    ],
+)
+
+tf_py_test(
+    name = "module_test",
+    srcs = ["module_test.py"],
+    additional_deps = [
+        ":module",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python/compat:v2_compat",
+        "//tensorflow/python:variables",
+    ],
+)
diff --git a/tensorflow/python/module/module.py b/tensorflow/python/module/module.py
new file mode 100644
index 0000000000..16db0b0e01
--- /dev/null
+++ b/tensorflow/python/module/module.py
@@ -0,0 +1,402 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Modules encapsulate building stateful components."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import re
+import sys
+
+import six
+
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import variables
+from tensorflow.python.training.checkpointable import tracking
+from tensorflow.python.util import nest
+from tensorflow.python.util import tf_decorator
+from tensorflow.python.util import tf_inspect
+from tensorflow.python.util.tf_export import tf_export
+
+
+class ModuleMetaclass(type):
+  """Metaclass for `tf.Module`."""
+
+  def __new__(mcs, name, bases, clsdict):
+    for key, value in clsdict.items():
+      if key in ("__init__", "name_scope"):
+        continue
+
+      elif tf_inspect.isfunction(value):
+        if getattr(value, "_no_module_name_scope", False):
+          # The function has been annotated to say that no autoscoping should
+          # be applied, so do not patch it.
+          continue
+        clsdict[key] = with_name_scope(value)
+
+      elif isinstance(value, property):
+        clsdict[key] = property(
+            value.fget if not value.fget else with_name_scope(value.fget),
+            value.fset if not value.fset else with_name_scope(value.fset),
+            value.fdel if not value.fdel else with_name_scope(value.fdel),
+            doc=value.__doc__)
+
+    return type.__new__(mcs, name, bases, clsdict)
+
+  def __call__(cls, *args, **kwargs):
+    # Call new such that we have an un-initialized module instance that we can
+    # still reference even if there is an exception during __init__. This is
+    # needed such that we can make sure the name_scope constructed in __init__
+    # is closed even if there is an exception.
+    module = cls.__new__(cls, *args, **kwargs)
+
+    # Now attempt to initialize the object.
+    try:
+      module.__init__(*args, **kwargs)
+    except:
+      # We must explicitly catch so that in Python 2 sys.exc_info() is populated
+      # before entering the finally block.
+      raise
+
+    finally:
+      # The base Module constructor enters the modules name scope before
+      # returning such that other functionality in the ctor happens within the
+      # modules name scope.
+      scope = getattr(module, "_ctor_name_scope", None)
+      exc_info = sys.exc_info()
+      if scope is None:
+        if exc_info[0] is None:
+          raise ValueError(
+              "Constructing a tf.Module without calling the super constructor "
+              "is not supported. Add the following as the first line in your "
+              "__init__ method:\n\n"
+              "super(%s, self).__init__()" % cls.__name__)
+      else:
+        scope.__exit__(*exc_info)
+        del module._ctor_name_scope
+
+    return module
+
+
+def with_name_scope(unbound_method):
+  """Patches the given method so it enters the modules name scope."""
+  def enter_name_scope(self, *args, **kwargs):
+    """Decorator that calls the given function in the module name scope.
+
+    Args:
+      self: Module instance.
+      *args: Positional arguments to `unbound_method`.
+      **kwargs: Keyword arguments to `unbound_method`.
+
+    Returns:
+      `with self.name_scope: return unbound_method(self, *args, **kwargs)`
+    """
+    try:
+      module_name_scope = self.name_scope
+    except AttributeError as exc_value_from:
+      exc_value = AttributeError(
+          "The super constructor must be called before any other methods in "
+          "your constructor. If this is not possible then annotate all the "
+          "methods called with `@no_module_name_scope`.")
+      six.raise_from(exc_value, exc_value_from)
+
+    with module_name_scope:
+      # tf.Module enters the module name scope for all methods. To disable this
+      # for a particular method annotate it with `@no_module_name_scope`.
+      return unbound_method(self, *args, **kwargs)
+
+  return tf_decorator.make_decorator(unbound_method, enter_name_scope)
+
+
+@tf_export("experimental.Module")
+class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
+  """Base neural network module class.
+
+  A module is a named container for `tf.Variable`s, other `tf.Module`s and
+  functions which apply to user input. For example a dense layer in a neural
+  network might be implemented as a `tf.Module`:
+
+  >>> class Dense(tf.Module):
+  ...   def __init__(self, in_features, output_features):
+  ...     super(Linear, self).__init__()
+  ...     self.w = tf.Variable(
+  ...         tf.random_normal([input_features, output_features]), name='w')
+  ...     self.b = tf.Variable(tf.zeros([output_features]), name='b')
+  ...
+  ...   def __call__(self, x):
+  ...     x = tf.convert_to_tensor(x, name='x')
+  ...     y = tf.matmul(x, self.w) + self.b
+  ...     return tf.nn.relu(y)
+
+  You can use the dense layer as you would expect:
+
+  >>> d = Dense(input_features=64, output_features=10)
+  >>> d(tf.ones([100, 64]))
+  <tf.Tensor: ...>
+
+  By subclassing `tf.Module` instead of `object` any variables created inside
+  the module are automatically created within the modules name scope:
+
+  >> d.w.name
+  "dense/w:0"
+
+  In eager mode this is useful for debugging, and when used with `@tf.function`
+  the use of name scopes gives operations (e.g. matmul) useful names as well.
+
+  As well as automatic naming, the Dense module inherits methods for tracking
+  its variables:
+
+  >>> d.variables
+  (<tf.Variable 'dense/b:0' ...>, <tf.Variable 'dense/w:0' ...>)
+  """
+
+  def __init__(self, name=None):
+    if name is None:
+      name = camel_to_snake(type(self).__name__)
+    else:
+      if not valid_identifier(name):
+        raise ValueError(
+            "%r is not a valid module name. Module names must be valid Python "
+            "identifiers (e.g. a valid class name)." % name)
+
+    self._name = name
+    with ops.name_scope(name) as scope_name:
+      self._scope_name = scope_name
+
+    # Enter the name scope so subsequent code in the contructor (e.g. creating
+    # submodules) happens inside the modules name scope. This is exited when
+    # the subclass __init__ returns (this is implemented in ModuleMetaclass).
+    self._ctor_name_scope = self.name_scope
+    self._ctor_name_scope.__enter__()
+
+  @property
+  def name(self):
+    """Returns the name of this module as passed or determined in the ctor.
+
+    NOTE: This is not the same as the `self.name_scope.name` which includes
+    parent module names.
+    """
+    return self._name
+
+  @property
+  def name_scope(self):
+    """Returns a `tf.name_scope` instance for this class."""
+    # TODO(tomhennigan) Memoize once name scopes are re-entrant.
+    return ops.name_scope(self._scope_name)
+
+  @property
+  def variables(self):
+    """Returns a tuple of variables owned by this module and it's submodules.
+
+    Returns:
+      A tuple of variables for the current module (sorted by attribute name)
+      followed by variables from all submodules recursively (depth first).
+    """
+    return tuple(walk(self, recurse_if=_IS_MODULE, predicate=_IS_VARIABLE))
+
+  @property
+  def owned_variables(self):
+    """Returns a tuple of variables that are attributes of the current module.
+
+    See `variables` for a property which returns all variables from the current
+    module and all it's submodules recursively.
+
+    Returns:
+      A tuple of variables which are attributes of the current module. Will
+      yield variables inside nested structures (lists etc) but not in other
+      modules.
+    """
+    return tuple(walk(self, predicate=_IS_VARIABLE))
+
+  @property
+  def trainable_variables(self):
+    """Returns a tuple of variables owned by this module and it's submodules.
+
+    Returns:
+      A tuple of variables for the current module (sorted by attribute name)
+      followed by variables from all submodules recursively (depth first).
+    """
+    return tuple(
+        walk(self, recurse_if=_IS_MODULE, predicate=_IS_TRAINABLE_VARIABLE))
+
+  @property
+  def owned_trainable_variables(self):
+    """Returns a tuple of variables that are attributes of the current module.
+
+    See `variables` for a property which returns all variables from the current
+    module and all it's submodules recursively.
+
+    Returns:
+      A tuple of variables which are attributes of the current module. Will
+      yield variables inside nested structures (lists etc) but not in other
+      modules.
+    """
+    return tuple(walk(self, predicate=_IS_TRAINABLE_VARIABLE))
+
+  @property
+  def owned_submodules(self):
+    """Returns an iterator of immediate child modules.
+
+    Child modules are modules which are found as properties of the current
+    module.
+
+    >>> a = tf.experimental.Module()
+    >>> b = tf.experimental.Module()
+    >>> c = tf.experimental.Module()
+    >>> a.b = b
+    >>> b.c = c
+    >>> assert list(a.owned_submodules) == [b]
+    >>> assert list(b.owned_submodules) == [c]
+    >>> assert list(c.owned_submodules) == []
+
+    Returns:
+      A generator over all child modules.
+    """
+    return walk(self, predicate=_IS_MODULE)
+
+  @property
+  def submodules(self):
+    """Returns an iterator of all sub-modules recursively.
+
+    Submodules are modules which are properties of this module, or found as
+    properties of modules which are properties of this module (and so on).
+
+    >>> a = tf.experimental.Module()
+    >>> b = tf.experimental.Module()
+    >>> c = tf.experimental.Module()
+    >>> a.b = b
+    >>> b.c = c
+    >>> assert list(a.submodules) == [b, c]
+    >>> assert list(b.submodules) == [c]
+    >>> assert list(c.submodules) == []
+
+    Returns:
+      A generator over all submodules.
+    """
+    return walk(self, recurse_if=_IS_MODULE, predicate=_IS_MODULE)
+
+  @classmethod
+  def no_name_scope(cls, method):
+    """Decorator to wrap a method, preventing automatic name scope wrapping.
+
+    By default, any method on a module is considered as a forwards function, and
+    so any variables / modules created by the method will be scoped as belonging
+    to the module. In some cases this is undesirable, for example when
+    implementing .clone() / .transpose(), as in those cases we want the new
+    module to have the scope of wherever the .transpose() call is made. To
+    allow this, decorate any methods with `no_module_name_scope`.
+
+    This logic is tied to ModuleMetaclass.__new__, if anything is
+    changed here corresponding changes will be needed there.
+
+    Args:
+      method: the method to wrap.
+
+    Returns:
+      The method, with a flag indicating no name scope wrapping should occur.
+    """
+    setattr(method, "_no_module_name_scope", True)
+    return method
+
+_IS_VARIABLE = lambda o: isinstance(o, variables.Variable)
+_IS_TRAINABLE_VARIABLE = lambda o: (_IS_VARIABLE(o) and o.trainable)
+_IS_MODULE = lambda o: isinstance(o, Module)
+_CAMEL_TO_SNAKE_R = re.compile(r"((?<=[a-z0-9])[A-Z]|(?!^)[A-Z](?=[a-z]))")
+_VALID_IDENTIFIER = re.compile(r"^[a-zA-Z_]([a-zA-Z0-9_])*$")
+
+
+def valid_identifier(name):
+  return bool(_VALID_IDENTIFIER.match(name))
+
+
+def camel_to_snake(value):
+  return _CAMEL_TO_SNAKE_R.sub(r"_\1", value).lower()
+
+
+def walk(o, recurse_if=None, predicate=None):
+  """Flattened attributes of `o` in sorted order by attribute name.
+
+  >>> class Foo(object):
+  ...   def __init__(self, prefix=''):
+  ...     self.z = prefix + 'c'
+  ...     self.a = [prefix + 'a', prefix + 'b']
+
+  >>> tuple(walk(Foo()))
+  ('a', 'b', 'c')
+
+  If `predicate` is not None, then only values matching predicate are returned:
+
+  >>> tuple(walk(Foo(), predicate=lambda v: v != 'a'))
+  ('b', 'c')
+
+  If `recurse_if` is not None then it should be a callable which tests if the
+  given leaf should be expanded:
+
+  >>> is_string = lambda v: isinstance(v, str)
+  >>> is_foo = lambda l: isinstance(l, Foo)
+  >>> o = Foo(prefix='root_')
+  >>> o.b = Foo(prefix='child_')
+  >>> tuple(walk(o, predicate=is_string))
+  ('root_a', 'root_b', 'root_c')
+  >>> tuple(walk(o, recurse_if=is_foo, predicate=is_string))
+  ('root_a', 'root_b', 'root_c', 'child_a', 'child_b', 'child_c')
+
+  Args:
+    o: An object who's attributes are walked.
+    recurse_if: (Optional) Visited items of this type will be walked to extract
+      more leaves. If `None`, it will not recurse into leaves.
+    predicate: (Optional) If set then only values matching predicate are
+      yielded.
+
+  Returns:
+    Attributes of `o` in name order. If `recurse_if` is not `None` then
+    attributes for which `recurse_if(attribute) == True` will be walked
+    recursively. If `predicate` is not `None` then only attributes for which
+    `predicate(attribute) == True` will be yielded.
+  """
+  if predicate is None:
+    predicate = lambda _: True
+  return _walk_internal(
+      o, recurse_if=recurse_if, predicate=predicate, seen=set())
+
+
+def _walk_internal(o, recurse_if, predicate, seen):
+  """Implementation of `walk`."""
+  if seen is None:
+    seen = set([id(o)])
+
+  o_dict = vars(o)
+  to_walk = []
+
+  for key in sorted(o_dict):
+    values = nest.flatten(o_dict[key])
+    for value in values:
+      value_id = id(value)
+      if value_id in seen:
+        continue
+
+      seen.add(value_id)
+      if predicate(value):
+        yield value
+
+      if recurse_if is not None and recurse_if(value):
+        # Walk direct properties first then recurse.
+        to_walk.append(value)
+
+  for value in to_walk:
+    for subvalue in _walk_internal(value, recurse_if, predicate, seen):
+      # Predicate is already tested for these values.
+      yield subvalue
diff --git a/tensorflow/python/module/module_test.py b/tensorflow/python/module/module_test.py
new file mode 100644
index 0000000000..e7c736696e
--- /dev/null
+++ b/tensorflow/python/module/module_test.py
@@ -0,0 +1,355 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for `tf.Module`."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.compat import v2_compat
+from tensorflow.python.framework import ops
+from tensorflow.python.module import module
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+
+
+class TestModuleNaming(test.TestCase):
+
+  def test_single_name(self):
+    mod = module.Module(name="simple")
+    self.assertEqual(mod.name, "simple")
+    self.assertEqual(mod.name_scope.name, "simple/")
+
+  def test_construct_in_scope(self):
+    with ops.name_scope("foo"):
+      mod = module.Module(name="bar")
+    self.assertEqual(mod.name, "bar")
+    self.assertEqual(mod.name_scope.name, "foo/bar/")
+
+  def test_enters_name_scope_in_call(self):
+    mod = ReturnsNameScopeModule()
+    for _ in range(3):
+      self.assertEqual(mod(), mod.name_scope.name)
+
+  def test_enters_name_scope_in_other_method(self):
+    mod = ReturnsNameScopeModule()
+    for _ in range(3):
+      self.assertEqual(mod.alternative_forward(), mod.name_scope.name)
+
+  def test_subclassed_module(self):
+    mod = SubclassedReturnsNameScopeModule()
+    for _ in range(3):
+      self.assertEqual(mod.alternative_forward(), mod.name_scope.name)
+      self.assertEqual(mod.alternative_alternative_forward(),
+                       mod.name_scope.name)
+
+  def test_submodule_created_late(self):
+    m = TreeModule()
+    self.assertEqual(m.name, "tree_module")
+    self.assertEqual(m.name_scope.name, "tree_module/")
+    leaf1 = m.new_leaf()
+    self.assertEqual(leaf1.name, "tree_module")
+    self.assertEqual(leaf1.name_scope.name, "tree_module/tree_module/")
+
+  def test_does_not_evaluate_property_methods(self):
+    mod = PropertyThrowsWhenCalledModule()
+    with self.assertRaises(AssertionError):
+      mod.raise_assertion_error  # pylint: disable=pointless-statement
+
+  def test_overridden_name_scope(self):
+    mod = ModuleOverridingNameScope()
+    self.assertEqual(mod(), mod.name_scope.name)
+    self.assertEqual(mod.alternative_forward(), mod.name_scope.name)
+
+  def test_patched_callable(self):
+    with ops.name_scope("foo"):
+      mod = module.Module(name="bar")
+    mod.foo = get_name_scope
+    # `foo` is not a method so we do not re-enter the name scope.
+    self.assertEqual(mod.foo(), "")
+
+  def test_invalid_name(self):
+    msg = ".* is not a valid module name"
+    with self.assertRaisesRegexp(ValueError, msg):
+      module.Module(name="$Foo")
+
+  def test_modules_not_numbered_in_eager(self):
+    mod = RecursiveModule(2)
+    self.assertEqual(mod.name_scope.name, "badger/")
+    self.assertEqual(mod.child.name_scope.name, "badger/badger/")
+
+    mod = RecursiveModule(2)
+    self.assertEqual(mod.name_scope.name, "badger/")
+    self.assertEqual(mod.child.name_scope.name, "badger/badger/")
+
+  def test_module_numbering_in_graph(self):
+    with ops.Graph().as_default():
+      mod = RecursiveModule(2)
+      self.assertEqual(mod.name_scope.name, "badger/")
+      self.assertEqual(mod.child.name_scope.name, "badger/badger/")
+
+      mod = RecursiveModule(2)
+      self.assertEqual(mod.name_scope.name, "badger_1/")
+      self.assertEqual(mod.child.name_scope.name, "badger_1/badger/")
+
+  def test_ctor_error_closes_name_scope(self):
+    with self.assertRaises(ErrorModuleError):
+      # If super constructor is called then a name scope is opened then an error
+      # is thrown. The metaclass should handle this and close the namescope
+      # before re-throwing the exception.
+      ErrorModule(call_super=True)
+
+    self.assertEqual("", get_name_scope())
+
+  def test_ctor_error_handles_ctor_not_opening_name_scope(self):
+    with self.assertRaises(ErrorModuleError):
+      # If super ctor is not called then the name scope isn't opened. We need to
+      # ensure that this doesn't trigger an exception (e.g. the metaclass trying
+      # to __exit__ a non-existant name scope).
+      ErrorModule(call_super=False)
+
+    self.assertEqual("", get_name_scope())
+
+  def test_forward_method_closes_name_scope(self):
+    mod = ErrorModule(call_super=True, raise_in_constructor=False)
+    with self.assertRaises(ErrorModuleError):
+      mod()
+
+    self.assertEqual("", get_name_scope())
+
+
+class VariableNamingTest(test.TestCase):
+
+  def test_variable_names(self):
+    mod = RecursiveModule(3)
+    self.assertEqual(mod.w.name, "badger/mushroom:0")
+    self.assertEqual(mod.child.w.name, "badger/badger/mushroom:0")
+    self.assertEqual(mod.child.child.w.name, "badger/badger/badger/mushroom:0")
+
+
+class VariableTrackingTest(test.TestCase):
+
+  def test_variables(self):
+    m = RecursiveModule(3)
+    self.assertEqual(m.variables, (m.w, m.child.w, m.child.child.w))
+    self.assertEqual(m.child.variables, (m.child.w, m.child.child.w))
+    self.assertEqual(m.child.child.variables, (m.child.child.w,))
+
+  def test_owned_variables(self):
+    m = RecursiveModule(3)
+    self.assertEqual(m.owned_variables, (m.w,))
+    self.assertEqual(m.child.owned_variables, (m.child.w,))
+    self.assertEqual(m.child.child.owned_variables, (m.child.child.w,))
+
+  def test_trainable_variables(self):
+    m = RecursiveModule(3)
+    self.assertEqual(m.trainable_variables,
+                     (m.w, m.child.w, m.child.child.w))
+    self.assertEqual(m.child.trainable_variables,
+                     (m.child.w, m.child.child.w))
+    self.assertEqual(m.child.child.trainable_variables, (m.child.child.w,))
+
+  def test_trainable_variables_ignores_non_trainable(self):
+    m = RecursiveModule(3, trainable=False)
+    self.assertEqual(len(m.trainable_variables), 0)
+    self.assertEqual(len(m.child.trainable_variables), 0)
+    self.assertEqual(len(m.child.child.trainable_variables), 0)
+
+  def test_owned_trainable_variables(self):
+    m = RecursiveModule(3)
+    self.assertEqual(m.owned_trainable_variables, (m.w,))
+    self.assertEqual(m.child.owned_trainable_variables, (m.child.w,))
+    self.assertEqual(m.child.child.owned_trainable_variables,
+                     (m.child.child.w,))
+
+  def test_owned_trainable_variables_ignores_non_trainable(self):
+    m = RecursiveModule(3, trainable=False)
+    self.assertEqual(len(m.owned_trainable_variables), 0)
+    self.assertEqual(len(m.child.owned_trainable_variables), 0)
+    self.assertEqual(len(m.child.child.owned_trainable_variables), 0)
+
+
+class ModuleTrackingTest(test.TestCase):
+
+  def test_owned_submodules(self):
+    m = RecursiveModule(3)
+    self.assertEqual(list(m.owned_submodules), [m.child])
+    self.assertEqual(list(m.child.owned_submodules), [m.child.child])
+    self.assertEqual(list(m.child.child.owned_submodules), [])
+
+  def test_submodules(self):
+    m = RecursiveModule(3)
+    self.assertEqual(list(m.submodules), [m.child, m.child.child])
+    self.assertEqual(list(m.child.submodules), [m.child.child])
+    self.assertEqual(list(m.child.child.submodules), [])
+
+  def test_non_ctor_submodule(self):
+    m = TreeModule()
+    leaf1 = m.new_leaf()
+    self.assertEqual(set(m.submodules), {leaf1})
+    leaf2 = m.new_leaf()
+    self.assertEqual(set(m.submodules), {leaf1, leaf2})
+
+
+class CommonErrorsTest(test.TestCase):
+
+  def test_not_calling_super_constructor(self):
+    msg = ("Constructing a tf.Module without calling the super constructor is "
+           "not supported")
+    with self.assertRaisesRegexp(ValueError, msg):
+      DoesNotCallSuperConstructorModule()
+
+  def test_calls_method_before_super(self):
+    msg = "super constructor must be called before any other methods"
+    with self.assertRaisesRegexp(AttributeError, msg):
+      CallsMethodBeforeSuperConstructorModule(allowed_method=False)
+
+  def test_annotated_method_is_allowed(self):
+    self.assertIsNotNone(
+        CallsMethodBeforeSuperConstructorModule(allowed_method=True))
+
+
+def get_name_scope():
+  with ops.name_scope("x") as ns:
+    return ns[:-2]
+
+
+class ErrorModuleError(Exception):
+  pass
+
+
+class ErrorModule(module.Module):
+
+  def __init__(self, call_super, raise_in_constructor=True):
+    if call_super:
+      super(ErrorModule, self).__init__()
+    if raise_in_constructor:
+      raise ErrorModuleError("Deliberate error!")
+
+  def __call__(self):
+    raise ErrorModuleError("Deliberate error!")
+
+
+class RecursiveModule(module.Module):
+
+  def __init__(self, depth, trainable=True):
+    super(RecursiveModule, self).__init__(name="badger")
+    self.child = None
+    if depth > 1:
+      self.child = RecursiveModule(depth - 1, trainable=trainable)
+    self.w = variables.Variable(1.0, trainable=trainable, name="mushroom")
+
+
+class TreeModule(module.Module):
+
+  def __init__(self, name=None):
+    super(TreeModule, self).__init__(name=name)
+    self._leaves = []
+
+  def new_leaf(self, name=None):
+    leaf = TreeModule(name=name)
+    self._leaves.append(leaf)
+    return leaf
+
+
+class ReturnsNameScopeModule(module.Module):
+
+  def alternative_forward(self):
+    return get_name_scope()
+
+  def __call__(self):
+    return get_name_scope()
+
+
+class SubclassedReturnsNameScopeModule(ReturnsNameScopeModule):
+
+  def alternative_alternative_forward(self):
+    return get_name_scope()
+
+
+class PropertyThrowsWhenCalledModule(module.Module):
+
+  @property
+  def raise_assertion_error(self):
+    raise AssertionError
+
+
+class ModuleOverridingNameScope(ReturnsNameScopeModule):
+
+  @property
+  def name_scope(self):
+    return ops.name_scope("yolo/")
+
+
+class DoesNotCallSuperConstructorModule(module.Module):
+
+  def __init__(self):
+    # NOTE: Intentionally does not call super constructor.
+    pass
+
+
+class CallsMethodBeforeSuperConstructorModule(module.Module):
+
+  def __init__(self, allowed_method):
+    if allowed_method:
+      self.no_name_scope()
+    else:
+      self.with_name_scope()
+    super(CallsMethodBeforeSuperConstructorModule, self).__init__()
+
+  @module.Module.no_name_scope
+  def no_name_scope(self):
+    pass
+
+  def with_name_scope(self):
+    pass
+
+
+class WalkTest(test.TestCase):
+
+  def test_walk(self):
+    parent = SimpleModule()
+    child = parent.c
+
+    self.assertEqual(
+        list(module.walk(parent, predicate=IS_MEMBER)),
+        [parent.a[0], parent.a[1], parent.z])
+
+    self.assertEqual(
+        list(module.walk(parent, recurse_if=IS_MODULE, predicate=IS_MEMBER)),
+        [parent.a[0], parent.a[1], parent.z, child.a[0], child.a[1], child.z])
+
+
+class MemberType(object):
+  """A simple type to search for."""
+  pass
+
+
+class SimpleModule(module.Module):
+
+  def __init__(self, create_child=True):
+    super(SimpleModule, self).__init__()
+    self.z = MemberType()
+    self.a = [MemberType(), MemberType()]
+    if create_child:
+      self.c = SimpleModule(create_child=False)
+
+
+IS_MEMBER = lambda v: isinstance(v, MemberType)
+IS_MODULE = lambda v: isinstance(v, module.Module)
+
+if __name__ == "__main__":
+  v2_compat.enable_v2_behavior()
+  test.main()
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.experimental.-module.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.experimental.-module.pbtxt
new file mode 100644
index 0000000000..c364b0217a
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.experimental.-module.pbtxt
@@ -0,0 +1,47 @@
+path: "tensorflow.experimental.Module"
+tf_class {
+  is_instance: "<class \'tensorflow.python.module.module.Module\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.tracking.AutoCheckpointable\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name_scope"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "owned_submodules"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "owned_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "owned_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "submodules"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "no_name_scope"
+    argspec: "args=[\'cls\', \'method\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.experimental.pbtxt
index 0c3f04e468..a7ee6d3e07 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.experimental.pbtxt
@@ -1,5 +1,9 @@
 path: "tensorflow.experimental"
 tf_module {
+  member {
+    name: "Module"
+    mtype: "<class \'tensorflow.python.module.module.ModuleMetaclass\'>"
+  }
   member_method {
     name: "function_executor_type"
     argspec: "args=[\'executor_type\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.experimental.-module.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.experimental.-module.pbtxt
new file mode 100644
index 0000000000..c364b0217a
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.experimental.-module.pbtxt
@@ -0,0 +1,47 @@
+path: "tensorflow.experimental.Module"
+tf_class {
+  is_instance: "<class \'tensorflow.python.module.module.Module\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.tracking.AutoCheckpointable\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name_scope"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "owned_submodules"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "owned_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "owned_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "submodules"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "no_name_scope"
+    argspec: "args=[\'cls\', \'method\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.experimental.pbtxt
index 0c3f04e468..a7ee6d3e07 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.experimental.pbtxt
@@ -1,5 +1,9 @@
 path: "tensorflow.experimental"
 tf_module {
+  member {
+    name: "Module"
+    mtype: "<class \'tensorflow.python.module.module.ModuleMetaclass\'>"
+  }
   member_method {
     name: "function_executor_type"
     argspec: "args=[\'executor_type\'], varargs=None, keywords=None, defaults=None"
-- 
GitLab


From 31c1c097d8ee260ab1411011f5276b326cfb2ace Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 07:36:10 -0800
Subject: [PATCH 0849/2345] Remove 19 uneeded C++ BUILD Dependencies

PiperOrigin-RevId: 229744406
---
 tensorflow/compiler/xla/service/BUILD             | 10 ----------
 tensorflow/compiler/xla/service/cpu/BUILD         |  2 --
 tensorflow/compiler/xla/service/gpu/BUILD         |  4 ----
 tensorflow/compiler/xla/service/interpreter/BUILD |  2 --
 tensorflow/compiler/xla/service/llvm_ir/BUILD     |  1 -
 5 files changed, 19 deletions(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index aef3c37159..e39e17c110 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -1062,7 +1062,6 @@ cc_library(
         "//tensorflow/compiler/xla:types",
         "//tensorflow/compiler/xla:util",
         "//tensorflow/core:lib",
-        "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/strings:str_format",
@@ -1100,7 +1099,6 @@ cc_library(
         ":buffer_value_containers",
         ":heap_simulator",
         ":hlo",
-        ":hlo_memory_scheduler",
         ":hlo_proto",
         ":logical_buffer",
         ":tuple_points_to_analysis",
@@ -1240,7 +1238,6 @@ cc_library(
     deps = [
         ":hlo",
         ":hlo_proto",
-        "//tensorflow/compiler/xla:statusor",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/types:span",
     ],
@@ -1508,7 +1505,6 @@ cc_library(
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/core:lib",
-        "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/types:optional",
         "@com_google_absl//absl/types:span",
     ],
@@ -2069,7 +2065,6 @@ cc_library(
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:status_macros",
-        "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:types",
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
@@ -3030,10 +3025,7 @@ cc_library(
         ":hlo",
         ":hlo_pass",
         ":shape_inference",
-        "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:literal_util",
-        "//tensorflow/compiler/xla:shape_util",
-        "//tensorflow/compiler/xla:types",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/core:lib",
         "@com_google_absl//absl/algorithm:container",
@@ -3405,7 +3397,6 @@ cc_library(
         ":hlo_pass_pipeline",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/core:lib",
-        "@com_google_absl//absl/container:flat_hash_map",
     ],
 )
 
@@ -3593,7 +3584,6 @@ cc_library(
         "//tensorflow/compiler/xla:util",
         "//tensorflow/core:lib",
         "@com_google_absl//absl/algorithm:container",
-        "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/container:inlined_vector",
     ],
 )
diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index a7feab5a0a..d4535b204d 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -790,8 +790,6 @@ cc_library(
         ":target_machine_features",
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla/service:computation_layout",
-        "//tensorflow/compiler/xla/service:hlo",
-        "//tensorflow/compiler/xla/service:hlo_casting_utils",
         "//tensorflow/compiler/xla/service:layout_assignment",
         "//tensorflow/core:lib",
         "@com_google_absl//absl/container:flat_hash_map",
diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD
index 1caf753c45..dc17aa4426 100644
--- a/tensorflow/compiler/xla/service/gpu/BUILD
+++ b/tensorflow/compiler/xla/service/gpu/BUILD
@@ -1013,14 +1013,10 @@ cc_library(
     srcs = ["variadic_op_splitter.cc"],
     hdrs = ["variadic_op_splitter.h"],
     deps = [
-        ":ir_emission_utils",
-        "//tensorflow/compiler/xla:literal_util",
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:util",
-        "//tensorflow/compiler/xla:window_util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/service:hlo",
-        "//tensorflow/compiler/xla/service:hlo_casting_utils",
         "//tensorflow/compiler/xla/service:hlo_pass",
         "//tensorflow/core:lib",
         "@com_google_absl//absl/strings",
diff --git a/tensorflow/compiler/xla/service/interpreter/BUILD b/tensorflow/compiler/xla/service/interpreter/BUILD
index 9b4cafa4b4..545662543c 100644
--- a/tensorflow/compiler/xla/service/interpreter/BUILD
+++ b/tensorflow/compiler/xla/service/interpreter/BUILD
@@ -32,7 +32,6 @@ cc_library(
         "//tensorflow/compiler/xla:status_macros",
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla/service:algebraic_simplifier",
-        "//tensorflow/compiler/xla/service:batchnorm_expander",
         "//tensorflow/compiler/xla/service:compiler",
         "//tensorflow/compiler/xla/service:computation_placer",
         "//tensorflow/compiler/xla/service:dynamic_index_splitter",
@@ -43,7 +42,6 @@ cc_library(
         "//tensorflow/compiler/xla/service:hlo_cost_analysis",
         "//tensorflow/compiler/xla/service:hlo_cse",
         "//tensorflow/compiler/xla/service:hlo_dce",
-        "//tensorflow/compiler/xla/service:hlo_get_dimension_size_rewriter",
         "//tensorflow/compiler/xla/service:hlo_module_config",
         "//tensorflow/compiler/xla/service:hlo_pass",
         "//tensorflow/compiler/xla/service:hlo_pass_pipeline",
diff --git a/tensorflow/compiler/xla/service/llvm_ir/BUILD b/tensorflow/compiler/xla/service/llvm_ir/BUILD
index 54e8fe1947..c5d59fb28e 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/BUILD
+++ b/tensorflow/compiler/xla/service/llvm_ir/BUILD
@@ -39,7 +39,6 @@ cc_library(
         "//tensorflow/compiler/xla/service:logical_buffer",
         "//tensorflow/core:lib",
         "@com_google_absl//absl/container:flat_hash_map",
-        "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/strings",
         "@llvm//:core",
     ],
-- 
GitLab


From cc6f6fb453f1cb864643452ff937f9f4606bda32 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Thu, 17 Jan 2019 07:40:48 -0800
Subject: [PATCH 0850/2345] Dump out an annotated TF graph from
 mark_for_compilation_pass for better visualization

PiperOrigin-RevId: 229745040
---
 .../compiler/jit/mark_for_compilation_pass.cc | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass.cc b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
index 50afec020d..cb01690845 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
@@ -41,6 +41,7 @@ limitations under the License.
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/graph/algorithm.h"
 #include "tensorflow/core/graph/control_flow.h"
+#include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/gtl/cleanup.h"
 #include "tensorflow/core/lib/strings/stringprintf.h"
@@ -1150,6 +1151,27 @@ Status MarkForCompilationPass::RunImpl(
   if (flags->tf_xla_clustering_debug) {
     dump_graph::DumpGraphToFile("mark_for_compilation", **options.graph,
                                 options.flib_def);
+
+    // We also dump out an annoated version of the TF graph where the nodes
+    // names are prefixed with the cluster names.  This can help visualizing the
+    // clustering decisions on TensorBoard.
+    Graph new_graph((*options.graph)->op_registry());
+    CopyGraph(**options.graph, &new_graph);
+
+    for (Node* n : new_graph.nodes()) {
+      if (absl::optional<absl::string_view> cluster_name =
+              GetXlaClusterForNode(*n)) {
+        n->set_name(absl::StrCat(*cluster_name, "/", n->name()));
+      } else {
+        // There is room for improvement here.  In particular, it may help to
+        // split these unclustered nodes into classes where every node in a
+        // specific class has edges to and from the same set of clusters.
+        n->set_name(absl::StrCat("unclustered/", n->name()));
+      }
+    }
+
+    dump_graph::DumpGraphToFile("mark_for_compilation_annotated", new_graph,
+                                options.flib_def);
   }
 
   VLogClusteringSummary(*graph);
-- 
GitLab


From 202525cd4bbab377e9ee67f1e4098e40b0925d9c Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Thu, 17 Jan 2019 07:43:30 -0800
Subject: [PATCH 0851/2345] Blacklist certain failing NNAPI zip tests

PiperOrigin-RevId: 229745363
---
 .../testing/generated_examples_zip_test.cc    | 29 ++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/testing/generated_examples_zip_test.cc b/tensorflow/lite/testing/generated_examples_zip_test.cc
index a9a31ad088..45bd59a67d 100644
--- a/tensorflow/lite/testing/generated_examples_zip_test.cc
+++ b/tensorflow/lite/testing/generated_examples_zip_test.cc
@@ -107,6 +107,28 @@ std::map<string, string> kBrokenTests = {
     {R"(^\/strided_slice_buggy)", "119786029"},
 };
 
+// Additional list of tests that are expected to fail when
+//   --test_arg=--ignore_known_bugs=false
+// and
+//   --test_arg=--use_nnapi=true
+// Note that issues related to lack of NNAPI support for a particular op are
+// handled separately; this list is specifically for broken cases where
+// execution produces broken output.
+// Key is a substring of the test name and value is a bug number.
+std::map<string, string> kBrokenNnapiTests = {
+    // Certain NNAPI kernels silently fail with int32 types.
+    {R"(^\/add.*dtype=tf\.int32)", "122987564"},
+    {R"(^\/concat.*dtype=tf\.int32)", "122987564"},
+    {R"(^\/mul.*dtype=tf\.int32)", "122987564"},
+    {R"(^\/space_to_depth.*dtype=tf\.int32)", "122987564"},
+
+    // Certain NNAPI fully_connected shape permutations fail.
+    {R"(^\/fully_connected_constant_filter=True.*shape1=\[3,3\])", "122987564"},
+    {R"(^\/fully_connected_constant_filter=True.*shape1=\[4,4\])", "122987564"},
+    {R"(^\/fully_connected.*shape1=\[3,3\].*transpose_b=True)", "122987564"},
+    {R"(^\/fully_connected.*shape1=\[4,4\].*shape2=\[4,1\])", "122987564"},
+};
+
 // Allows test data to be unarchived into a temporary directory and makes
 // sure those temporary directories are removed later.
 class ArchiveEnvironment : public ::testing::Environment {
@@ -242,8 +264,13 @@ TEST_P(OpsTest, RunZipTests) {
   tflite::testing::TfLiteDriver test_driver(FLAGS_use_nnapi);
   test_driver.SetModelBaseDir(tflite_dir);
 
+  auto broken_tests = kBrokenTests;
+  if (FLAGS_use_nnapi) {
+    broken_tests.insert(kBrokenNnapiTests.begin(), kBrokenNnapiTests.end());
+  }
+
   string bug_number;
-  for (const auto& p : kBrokenTests) {
+  for (const auto& p : broken_tests) {
     if (RE2::PartialMatch(test_name, p.first)) {
       bug_number = p.second;
     }
-- 
GitLab


From f01d357ca3f08d7872dc1059e1a13f7fb4515ef2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 08:34:02 -0800
Subject: [PATCH 0852/2345] Switch from deprecated
 strategy.distribute_dataset() to strategy.make_input_fn_iterator() in
 contrib/distribute tests.

PiperOrigin-RevId: 229752452
---
 .../distribute/python/metrics_v1_test.py      |  5 ++---
 .../distribute/python/minimize_loss_test.py   | 21 ++++++++-----------
 .../python/mirrored_strategy_multigpu_test.py | 11 +++-------
 .../contrib/distribute/python/monitor.py      |  2 +-
 .../distribute/python/optimizer_v2_test.py    |  9 ++------
 .../contrib/distribute/python/step_fn.py      | 14 ++++++-------
 .../contrib/distribute/python/step_fn_test.py |  3 ++-
 7 files changed, 26 insertions(+), 39 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/metrics_v1_test.py b/tensorflow/contrib/distribute/python/metrics_v1_test.py
index 96006276f6..a663e809dd 100644
--- a/tensorflow/contrib/distribute/python/metrics_v1_test.py
+++ b/tensorflow/contrib/distribute/python/metrics_v1_test.py
@@ -95,8 +95,7 @@ class MetricsV1Test(test.TestCase, parameterized.TestCase):
 
   def _test_metric(self, distribution, dataset_fn, metric_fn, expected_fn):
     with ops.Graph().as_default(), distribution.scope():
-      iterator = distribution.distribute_dataset(
-          dataset_fn).make_initializable_iterator()
+      iterator = distribution.make_input_fn_iterator(lambda _: dataset_fn())
       if isinstance(distribution, tpu_strategy.TPUStrategy):
         def step_fn(ctx, inputs):
           value, update = distribution.extended.call_for_each_replica(
@@ -121,7 +120,7 @@ class MetricsV1Test(test.TestCase, parameterized.TestCase):
         # replace "distribution.num_replicas_in_sync" with "1".
         batches_per_update = distribution.num_replicas_in_sync
 
-      self.evaluate(iterator.initializer)
+      self.evaluate(iterator.initialize())
       self.evaluate(variables.local_variables_initializer())
 
       batches_consumed = 0
diff --git a/tensorflow/contrib/distribute/python/minimize_loss_test.py b/tensorflow/contrib/distribute/python/minimize_loss_test.py
index 37e6b418a0..f06c9b7564 100644
--- a/tensorflow/contrib/distribute/python/minimize_loss_test.py
+++ b/tensorflow/contrib/distribute/python/minimize_loss_test.py
@@ -41,12 +41,9 @@ from tensorflow.python.ops.losses import losses_impl
 
 class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
 
-  def _get_iterator(self, ds):
-    if context.executing_eagerly():
-      iterator = ds.make_one_shot_iterator()
-    else:
-      iterator = ds.make_initializable_iterator()
-      self.evaluate(iterator.initializer)
+  def _get_iterator(self, strategy, input_fn):
+    iterator = strategy.make_input_fn_iterator(lambda _: input_fn())
+    self.evaluate(iterator.initialize())
     return iterator
 
   @combinations.generate(
@@ -70,7 +67,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
             distribution.extended.call_for_each_replica(
                 model_fn, args=(inputs,)))
 
-      iterator = self._get_iterator(distribution.distribute_dataset(dataset_fn))
+      iterator = self._get_iterator(distribution, dataset_fn)
 
       def run_step():
         return distribution.extended.experimental_run_steps_on_iterator(
@@ -102,7 +99,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
       model_fn, dataset_fn, layer = minimize_loss_example(
           optimizer_fn, use_bias=True, use_callable_loss=use_callable_loss)
 
-      iterator = self._get_iterator(distribution.distribute_dataset(dataset_fn))
+      iterator = self._get_iterator(distribution, dataset_fn)
 
       def run_step():
         return distribution.group(
@@ -161,7 +158,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
             distribution.extended.call_for_each_replica(
                 model_fn, args=(inputs,)))
 
-      iterator = self._get_iterator(distribution.distribute_dataset(dataset_fn))
+      iterator = self._get_iterator(distribution, dataset_fn)
 
       def run_step():
         return distribution.extended.experimental_run_steps_on_iterator(
@@ -230,7 +227,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
           fetches += tuple(ops.get_collection(ops.GraphKeys.UPDATE_OPS))
         return control_flow_ops.group(fetches)
 
-      iterator = self._get_iterator(distribution.distribute_dataset(dataset_fn))
+      iterator = self._get_iterator(distribution, dataset_fn)
 
       def run_step():
         return distribution.extended.experimental_run_steps_on_iterator(
@@ -322,7 +319,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
             distribution.extended.call_for_each_replica(
                 model_fn, args=(inputs,)))
 
-      iterator = self._get_iterator(distribution.distribute_dataset(dataset_fn))
+      iterator = self._get_iterator(distribution, dataset_fn)
 
       def run_step():
         return distribution.extended.experimental_run_steps_on_iterator(
@@ -413,7 +410,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
             output=loss)
         return distribution.group(train_op)
 
-      iterator = self._get_iterator(distribution.distribute_dataset(dataset_fn))
+      iterator = self._get_iterator(distribution, dataset_fn)
 
       def run_step():
         initial_loss = lambda: constant_op.constant(1e7)
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index c22e4f8ee6..d6337d106f 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -366,14 +366,9 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
                 (layer2.kernel, layer2.bias),
                 (layer3.kernel, layer3.bias)]
 
-    ds = distribution.distribute_dataset(
-        lambda: dataset_ops.Dataset.from_tensors([[1.]]).repeat(10))
-    if context.executing_eagerly():
-      iterator = ds.make_one_shot_iterator()
-    else:
-      iterator = ds.make_initializable_iterator()
-      self.evaluate([iterator.initializer])
-
+    iterator = distribution.make_input_fn_iterator(
+        lambda _: dataset_ops.Dataset.from_tensors([[1.]]).repeat(10))
+    self.evaluate(iterator.initialize())
     features = iterator.get_next()
 
     with distribution.scope():
diff --git a/tensorflow/contrib/distribute/python/monitor.py b/tensorflow/contrib/distribute/python/monitor.py
index 17b7ab74f6..53e35ea6b7 100644
--- a/tensorflow/contrib/distribute/python/monitor.py
+++ b/tensorflow/contrib/distribute/python/monitor.py
@@ -51,7 +51,7 @@ class Monitor(object):
     else:
       if session is None:
         raise ValueError("Should provide a `session` in Graph mode.")
-      session.run(step_callable._iterator.initializer)  # pylint: disable=protected-access
+      session.run(step_callable.initialize())
       self._run_step = session.make_callable(step_callable())
       session.run(variables.global_variables_initializer())
 
diff --git a/tensorflow/contrib/distribute/python/optimizer_v2_test.py b/tensorflow/contrib/distribute/python/optimizer_v2_test.py
index a832ef61f9..e388061b17 100644
--- a/tensorflow/contrib/distribute/python/optimizer_v2_test.py
+++ b/tensorflow/contrib/distribute/python/optimizer_v2_test.py
@@ -41,12 +41,7 @@ class MinimizeLossOptimizerV2Test(test.TestCase, parameterized.TestCase):
     with distribution.scope():
       model_fn, dataset_fn, layer = minimize_loss_example(
           optimizer_fn, use_bias=True, use_callable_loss=use_callable_loss)
-
-      ds = distribution.distribute_dataset(dataset_fn)
-      if context.executing_eagerly():
-        iterator = ds.make_one_shot_iterator()
-      else:
-        iterator = ds.make_initializable_iterator()
+      iterator = distribution.make_input_fn_iterator(lambda _: dataset_fn())
 
       def run_step():
         return control_flow_ops.group(
@@ -56,7 +51,7 @@ class MinimizeLossOptimizerV2Test(test.TestCase, parameterized.TestCase):
 
       if not context.executing_eagerly():
         with self.cached_session() as sess:
-          sess.run(iterator.initializer)
+          sess.run(iterator.initialize())
           run_step = sess.make_callable(run_step())
         self.evaluate(variables.global_variables_initializer())
 
diff --git a/tensorflow/contrib/distribute/python/step_fn.py b/tensorflow/contrib/distribute/python/step_fn.py
index 24b95f748b..27aad46b97 100644
--- a/tensorflow/contrib/distribute/python/step_fn.py
+++ b/tensorflow/contrib/distribute/python/step_fn.py
@@ -19,7 +19,6 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.eager import backprop
-from tensorflow.python.eager import context
 from tensorflow.python.training import optimizer as optimizer_lib
 
 
@@ -33,6 +32,9 @@ class Step(object):
   def distribution(self):
     return self._distribution
 
+  def initialize(self):
+    return []
+
   def __call__(self):
     """Perform one step of this training algorithm."""
     raise NotImplementedError("must be implemented in descendants")
@@ -50,12 +52,10 @@ class StandardInputStep(Step):
 
   def __init__(self, dataset_fn, distribution):
     super(StandardInputStep, self).__init__(distribution)
-    self._distributed_input = distribution.distribute_dataset(dataset_fn)
-    if context.executing_eagerly():
-      self._iterator = self._distributed_input.make_one_shot_iterator()
-    else:
-      # TODO(priyag): Expose initializer via some initializer property.
-      self._iterator = self._distributed_input.make_initializable_iterator()
+    self._iterator = distribution.make_input_fn_iterator(lambda _: dataset_fn())
+
+  def initialize(self):
+    return self._iterator.initialize()
 
 
 class StandardSingleLossStep(StandardInputStep):
diff --git a/tensorflow/contrib/distribute/python/step_fn_test.py b/tensorflow/contrib/distribute/python/step_fn_test.py
index a77d6d0bec..9f48560b26 100644
--- a/tensorflow/contrib/distribute/python/step_fn_test.py
+++ b/tensorflow/contrib/distribute/python/step_fn_test.py
@@ -46,10 +46,11 @@ class SingleLossStepTest(test.TestCase, parameterized.TestCase):
           optimizer_fn, distribution, use_bias=True, iterations_per_step=2)
 
       if context.executing_eagerly():
+        single_loss_step.initialize()
         run_step = single_loss_step
       else:
         with self.cached_session() as sess:
-          sess.run(single_loss_step._iterator.initializer)
+          sess.run(single_loss_step.initialize())
           run_step = sess.make_callable(single_loss_step())
       self.evaluate(variables.global_variables_initializer())
 
-- 
GitLab


From 890a58c43242becfc29f0bfe6c5100b5f7faea94 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 08:46:54 -0800
Subject: [PATCH 0853/2345] s/AutographParseError/AutoGraphParseError/
 (properly camel-casing AutoGraph.)

PiperOrigin-RevId: 229754536
---
 tensorflow/python/autograph/__init__.py                     | 4 ++--
 tensorflow/python/autograph/converters/control_flow_test.py | 2 +-
 tensorflow/python/autograph/converters/decorators_test.py   | 2 +-
 tensorflow/python/autograph/converters/slices_test.py       | 2 +-
 tensorflow/python/autograph/pyct/transformer.py             | 6 +++---
 5 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/autograph/__init__.py b/tensorflow/python/autograph/__init__.py
index 6faeb01607..8fd6e72da6 100644
--- a/tensorflow/python/autograph/__init__.py
+++ b/tensorflow/python/autograph/__init__.py
@@ -50,7 +50,7 @@ from tensorflow.python.autograph.lang.directives import set_element_type
 from tensorflow.python.autograph.lang.directives import set_loop_options
 from tensorflow.python.autograph.lang.special_functions import stack
 from tensorflow.python.autograph.lang.special_functions import tensor_list
-from tensorflow.python.autograph.pyct.transformer import AutographParseError
+from tensorflow.python.autograph.pyct.transformer import AutoGraphParseError
 from tensorflow.python.util.all_util import remove_undocumented
 
 # TODO(mdan): Revisit this list once we finalize the generated code mechanism.
@@ -77,7 +77,7 @@ _allowed_symbols = [
     'stack',
     'tensor_list',
     # Exceptions
-    'AutographParseError',
+    'AutoGraphParseError',
     # Utilities: to be removed
     'utils',
 ]
diff --git a/tensorflow/python/autograph/converters/control_flow_test.py b/tensorflow/python/autograph/converters/control_flow_test.py
index 034fcbe386..1a38d0db4d 100644
--- a/tensorflow/python/autograph/converters/control_flow_test.py
+++ b/tensorflow/python/autograph/converters/control_flow_test.py
@@ -184,7 +184,7 @@ class ControlFlowTest(converter_testing.TestCase):
       return b
 
     node, ctx = self.prepare(test_fn, {})
-    with self.assertRaises(transformer.AutographParseError):
+    with self.assertRaises(transformer.AutoGraphParseError):
       control_flow.transform(node, ctx)
 
   @test_util.run_deprecated_v1
diff --git a/tensorflow/python/autograph/converters/decorators_test.py b/tensorflow/python/autograph/converters/decorators_test.py
index abd76849d6..bcf502c62b 100644
--- a/tensorflow/python/autograph/converters/decorators_test.py
+++ b/tensorflow/python/autograph/converters/decorators_test.py
@@ -121,7 +121,7 @@ class DecoratorsTest(converter_testing.TestCase):
       return inner_fn(a)
 
     # Expected to fail because simple_decorator could not be imported.
-    with self.assertRaises(transformer.AutographParseError):
+    with self.assertRaises(transformer.AutoGraphParseError):
       test_fn(1)
 
   def test_nested_decorators_imported(self):
diff --git a/tensorflow/python/autograph/converters/slices_test.py b/tensorflow/python/autograph/converters/slices_test.py
index bd049afdfc..d674f266c8 100644
--- a/tensorflow/python/autograph/converters/slices_test.py
+++ b/tensorflow/python/autograph/converters/slices_test.py
@@ -68,7 +68,7 @@ class SliceTest(converter_testing.TestCase):
     def_.directives[directives.set_element_type] = {
         'dtype': parser.parse_expression('tf.float32')
     }
-    with self.assertRaises(transformer.AutographParseError):
+    with self.assertRaises(transformer.AutoGraphParseError):
       slices.transform(node, ctx)
 
 
diff --git a/tensorflow/python/autograph/pyct/transformer.py b/tensorflow/python/autograph/pyct/transformer.py
index b6830534b3..9c776a737a 100644
--- a/tensorflow/python/autograph/pyct/transformer.py
+++ b/tensorflow/python/autograph/pyct/transformer.py
@@ -29,7 +29,7 @@ from tensorflow.python.autograph.pyct import pretty_printer
 from tensorflow.python.autograph.pyct import templates
 
 
-class AutographParseError(SyntaxError):
+class AutoGraphParseError(SyntaxError):
   pass
 
 
@@ -495,8 +495,8 @@ class Base(gast.NodeTransformer):
       # In other words, we need to find how to suppress the "During handling
       # of the above exception, another exception occurred" message.
       six.reraise(
-          AutographParseError,
-          AutographParseError(msg, (original_file_path, original_line_number,
+          AutoGraphParseError,
+          AutoGraphParseError(msg, (original_file_path, original_line_number,
                                     original_col_offset, original_source_line)),
           sys.exc_info()[2])
     finally:
-- 
GitLab


From a782c9df3e9c2cae1b49326f9c9b72eac1dfd794 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 08:53:44 -0800
Subject: [PATCH 0854/2345] Replace --genrule_strategy w/ --strategy=Genrule

This is in preparation for --genrule_strategy becoming a no-op in a future Bazel release.

PiperOrigin-RevId: 229755652
---
 .bazelrc                                   | 2 +-
 tensorflow/tools/ci_build/install/.bazelrc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bazelrc b/.bazelrc
index d5432aeb17..c70c571361 100644
--- a/.bazelrc
+++ b/.bazelrc
@@ -80,7 +80,7 @@ build --define=use_fast_cpp_protos=true
 build --define=allow_oversize_protos=true
 
 build --spawn_strategy=standalone
-build --genrule_strategy=standalone
+build --strategy=Genrule=standalone
 build -c opt
 
 # Other build flags.
diff --git a/tensorflow/tools/ci_build/install/.bazelrc b/tensorflow/tools/ci_build/install/.bazelrc
index 2060babd4a..4662e2e60a 100644
--- a/tensorflow/tools/ci_build/install/.bazelrc
+++ b/tensorflow/tools/ci_build/install/.bazelrc
@@ -5,7 +5,7 @@ startup --batch
 
 # Similarly, we need to workaround sandboxing issues:
 #   https://github.com/bazelbuild/bazel/issues/418
-build  --verbose_failures --spawn_strategy=standalone --genrule_strategy=standalone
+build  --verbose_failures --spawn_strategy=standalone --strategy=Genrule=standalone
 test --spawn_strategy=standalone
 
 # Force bazel output to use colors (good for jenkins) and print useful errors.
-- 
GitLab


From c9bd0744ea3d527503e7cc2e44c7306f0d25e421 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Thu, 17 Jan 2019 09:01:33 -0800
Subject: [PATCH 0855/2345] MOve backend config related functions to a
 dedicated file. Use backend epsilon if epsilon argument is None.

PiperOrigin-RevId: 229756902
---
 tensorflow/python/keras/BUILD                 |  19 +++
 tensorflow/python/keras/backend.py            | 143 ++----------------
 tensorflow/python/keras/backend_config.py     | 126 +++++++++++++++
 .../python/keras/backend_config_test.py       |  55 +++++++
 tensorflow/python/keras/backend_test.py       |  18 ---
 tensorflow/python/keras/optimizer_v2/BUILD    |   1 +
 .../python/keras/optimizer_v2/adadelta.py     |   3 +
 .../keras/optimizer_v2/adadelta_test.py       |   9 ++
 .../python/keras/optimizer_v2/adagrad.py      |   3 +
 .../python/keras/optimizer_v2/adagrad_test.py |  13 ++
 tensorflow/python/keras/optimizer_v2/adam.py  |   3 +
 .../python/keras/optimizer_v2/adam_test.py    |   9 ++
 .../python/keras/optimizer_v2/adamax.py       |   3 +
 .../python/keras/optimizer_v2/adamax_test.py  |   9 ++
 tensorflow/python/keras/optimizer_v2/nadam.py |   3 +
 .../python/keras/optimizer_v2/nadam_test.py   |   9 ++
 .../python/keras/optimizer_v2/rmsprop.py      |   3 +
 .../python/keras/optimizer_v2/rmsprop_test.py |   9 ++
 18 files changed, 287 insertions(+), 151 deletions(-)
 create mode 100644 tensorflow/python/keras/backend_config.py
 create mode 100644 tensorflow/python/keras/backend_config_test.py

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 9e08c766db..c4c403119c 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -82,6 +82,7 @@ py_library(
     srcs = ["backend.py"],
     srcs_version = "PY2AND3",
     deps = [
+        ":backend_config",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:check_ops",
@@ -117,6 +118,12 @@ py_library(
     ],
 )
 
+py_library(
+    name = "backend_config",
+    srcs = ["backend_config.py"],
+    srcs_version = "PY2AND3",
+)
+
 py_library(
     name = "engine",
     srcs = [
@@ -1108,6 +1115,18 @@ tf_py_test(
     ],
 )
 
+tf_py_test(
+    name = "backend_config_test",
+    size = "medium",
+    srcs = ["backend_config_test.py"],
+    additional_deps = [
+        ":keras",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:util",
+    ],
+)
+
 tf_py_test(
     name = "keras_parameterized_test",
     size = "small",
diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index f357f0efc7..9677f12d34 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -40,6 +40,7 @@ from tensorflow.python.framework import func_graph
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import tensor_util
+from tensorflow.python.keras import backend_config
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import clip_ops
 from tensorflow.python.ops import control_flow_ops
@@ -97,15 +98,6 @@ _DUMMY_EAGER_GRAPH = _DummyEagerGraph()
 # Change its value via `manual_variable_initialization(value)`.
 _MANUAL_VAR_INIT = False
 
-# The type of float to use throughout a session.
-_FLOATX = 'float32'
-
-# Epsilon fuzz factor used throughout the codebase.
-_EPSILON = 1e-7
-
-# Default image data format, one of "channels_last", "channels_first".
-_IMAGE_DATA_FORMAT = 'channels_last'
-
 # This list holds the available devices.
 # It is populated when `_get_available_gpus()` is called for the first time.
 # We assume our devices don't change henceforth.
@@ -119,6 +111,14 @@ _GRAPH_VARIABLES = weakref.WeakKeyDictionary()
 # the graph.
 _GRAPH_TF_OPTIMIZERS = weakref.WeakKeyDictionary()
 
+# The below functions are kept accessible from backend for compatibility.
+epsilon = backend_config.epsilon
+floatx = backend_config.floatx
+image_data_format = backend_config.image_data_format
+set_epsilon = backend_config.set_epsilon
+set_floatx = backend_config.set_floatx
+set_image_data_format = backend_config.set_image_data_format
+
 
 @keras_export('keras.backend.backend')
 def backend():
@@ -132,87 +132,6 @@ def backend():
   return 'tensorflow'
 
 
-@keras_export('keras.backend.epsilon')
-def epsilon():
-  """Returns the value of the fuzz factor used in numeric expressions.
-
-  Returns:
-      A float.
-
-  Example:
-  ```python
-      >>> keras.backend.epsilon()
-      1e-07
-  ```
-  """
-  return _EPSILON
-
-
-@keras_export('keras.backend.set_epsilon')
-def set_epsilon(value):
-  """Sets the value of the fuzz factor used in numeric expressions.
-
-  Arguments:
-      value: float. New value of epsilon.
-
-  Example:
-  ```python
-      >>> from keras import backend as K
-      >>> K.epsilon()
-      1e-07
-      >>> K.set_epsilon(1e-05)
-      >>> K.epsilon()
-      1e-05
-  ```
-  """
-  global _EPSILON
-  _EPSILON = value
-
-
-@keras_export('keras.backend.floatx')
-def floatx():
-  """Returns the default float type, as a string.
-
-  E.g. 'float16', 'float32', 'float64'.
-
-  Returns:
-      String, the current default float type.
-
-  Example:
-  ```python
-      >>> keras.backend.floatx()
-      'float32'
-  ```
-  """
-  return _FLOATX
-
-
-@keras_export('keras.backend.set_floatx')
-def set_floatx(value):
-  """Sets the default float type.
-
-  Arguments:
-      value: String; 'float16', 'float32', or 'float64'.
-
-  Example:
-  ```python
-      >>> from keras import backend as K
-      >>> K.floatx()
-      'float32'
-      >>> K.set_floatx('float16')
-      >>> K.floatx()
-      'float16'
-  ```
-
-  Raises:
-      ValueError: In case of invalid value.
-  """
-  global _FLOATX
-  if value not in {'float16', 'float32', 'float64'}:
-    raise ValueError('Unknown floatx type: ' + str(value))
-  _FLOATX = str(value)
-
-
 @keras_export('keras.backend.cast_to_floatx')
 def cast_to_floatx(x):
   """Cast a Numpy array to the default Keras float type.
@@ -238,49 +157,7 @@ def cast_to_floatx(x):
       dtype('float32')
   ```
   """
-  return np.asarray(x, dtype=_FLOATX)
-
-
-@keras_export('keras.backend.image_data_format')
-def image_data_format():
-  """Returns the default image data format convention.
-
-  Returns:
-      A string, either `'channels_first'` or `'channels_last'`
-
-  Example:
-  ```python
-      >>> keras.backend.image_data_format()
-      'channels_first'
-  ```
-  """
-  return _IMAGE_DATA_FORMAT
-
-
-@keras_export('keras.backend.set_image_data_format')
-def set_image_data_format(data_format):
-  """Sets the value of the image data format convention.
-
-  Arguments:
-      data_format: string. `'channels_first'` or `'channels_last'`.
-
-  Example:
-  ```python
-      >>> from keras import backend as K
-      >>> K.image_data_format()
-      'channels_first'
-      >>> K.set_image_data_format('channels_last')
-      >>> K.image_data_format()
-      'channels_last'
-  ```
-
-  Raises:
-      ValueError: In case of invalid `data_format` value.
-  """
-  global _IMAGE_DATA_FORMAT
-  if data_format not in {'channels_last', 'channels_first'}:
-    raise ValueError('Unknown data_format: ' + str(data_format))
-  _IMAGE_DATA_FORMAT = str(data_format)
+  return np.asarray(x, dtype=floatx())
 
 
 # A global dictionary mapping graph objects to an index of counters used
diff --git a/tensorflow/python/keras/backend_config.py b/tensorflow/python/keras/backend_config.py
new file mode 100644
index 0000000000..e7c63ac2c7
--- /dev/null
+++ b/tensorflow/python/keras/backend_config.py
@@ -0,0 +1,126 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Keras backend config API."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.util.tf_export import keras_export
+
+# The type of float to use throughout a session.
+_FLOATX = 'float32'
+
+# Epsilon fuzz factor used throughout the codebase.
+_EPSILON = 1e-7
+
+# Default image data format, one of "channels_last", "channels_first".
+_IMAGE_DATA_FORMAT = 'channels_last'
+
+
+@keras_export('keras.backend.epsilon')
+def epsilon():
+  """Returns the value of the fuzz factor used in numeric expressions.
+
+  Returns:
+      A float.
+
+  Example:
+  ```python
+  keras.backend.epsilon() >>>1e-07
+  ```
+  """
+  return _EPSILON
+
+
+@keras_export('keras.backend.set_epsilon')
+def set_epsilon(value):
+  """Sets the value of the fuzz factor used in numeric expressions.
+
+  Arguments:
+      value: float. New value of epsilon.
+  Example: ```python from keras import backend as K K.epsilon() >>> 1e-07
+    K.set_epsilon(1e-05) K.epsilon() >>> 1e-05 ```
+  """
+  global _EPSILON
+  _EPSILON = value
+
+
+@keras_export('keras.backend.floatx')
+def floatx():
+  """Returns the default float type, as a string.
+
+  E.g. 'float16', 'float32', 'float64'.
+
+  Returns:
+      String, the current default float type.
+
+  Example:
+  ```python
+  keras.backend.floatx() >>> 'float32'
+  ```
+  """
+  return _FLOATX
+
+
+@keras_export('keras.backend.set_floatx')
+def set_floatx(value):
+  """Sets the default float type.
+
+  Arguments:
+      value: String; 'float16', 'float32', or 'float64'.
+  Example: ```python from keras import backend as K K.floatx() >>> 'float32'
+    K.set_floatx('float16') K.floatx() >>> 'float16' ```
+
+  Raises:
+      ValueError: In case of invalid value.
+  """
+  global _FLOATX
+  if value not in {'float16', 'float32', 'float64'}:
+    raise ValueError('Unknown floatx type: ' + str(value))
+  _FLOATX = str(value)
+
+
+@keras_export('keras.backend.image_data_format')
+def image_data_format():
+  """Returns the default image data format convention.
+
+  Returns:
+      A string, either `'channels_first'` or `'channels_last'`
+
+  Example:
+  ```python
+  keras.backend.image_data_format() >>> 'channels_first'
+  ```
+  """
+  return _IMAGE_DATA_FORMAT
+
+
+@keras_export('keras.backend.set_image_data_format')
+def set_image_data_format(data_format):
+  """Sets the value of the image data format convention.
+
+  Arguments:
+      data_format: string. `'channels_first'` or `'channels_last'`.
+  Example: ```python from keras import backend as K K.image_data_format() >>>
+    'channels_first' K.set_image_data_format('channels_last')
+    K.image_data_format() >>> 'channels_last' ```
+
+  Raises:
+      ValueError: In case of invalid `data_format` value.
+  """
+  global _IMAGE_DATA_FORMAT
+  if data_format not in {'channels_last', 'channels_first'}:
+    raise ValueError('Unknown data_format: ' + str(data_format))
+  _IMAGE_DATA_FORMAT = str(data_format)
diff --git a/tensorflow/python/keras/backend_config_test.py b/tensorflow/python/keras/backend_config_test.py
new file mode 100644
index 0000000000..59e003d81e
--- /dev/null
+++ b/tensorflow/python/keras/backend_config_test.py
@@ -0,0 +1,55 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for backend_config."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python import keras
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import test
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class BackendConfigTest(test.TestCase):
+
+  def test_backend(self):
+    self.assertEqual(keras.backend.backend(), 'tensorflow')
+
+  def test_espilon(self):
+    epsilon = 1e-2
+    keras.backend_config.set_epsilon(epsilon)
+    self.assertEqual(keras.backend_config.epsilon(), epsilon)
+    keras.backend_config.set_epsilon(1e-7)
+    self.assertEqual(keras.backend_config.epsilon(), 1e-7)
+
+  def test_floatx(self):
+    floatx = 'float64'
+    keras.backend_config.set_floatx(floatx)
+    self.assertEqual(keras.backend_config.floatx(), floatx)
+    keras.backend_config.set_floatx('float32')
+    self.assertEqual(keras.backend_config.floatx(), 'float32')
+
+  def test_image_data_format(self):
+    image_data_format = 'channels_first'
+    keras.backend_config.set_image_data_format(image_data_format)
+    self.assertEqual(keras.backend_config.image_data_format(),
+                     image_data_format)
+    keras.backend_config.set_image_data_format('channels_last')
+    self.assertEqual(keras.backend_config.image_data_format(), 'channels_last')
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/python/keras/backend_test.py b/tensorflow/python/keras/backend_test.py
index 4b83f0bf66..e3f86ee2c0 100644
--- a/tensorflow/python/keras/backend_test.py
+++ b/tensorflow/python/keras/backend_test.py
@@ -99,24 +99,6 @@ class BackendUtilsTest(test.TestCase):
   def test_backend(self):
     self.assertEqual(keras.backend.backend(), 'tensorflow')
 
-  def test_espilon(self):
-    epsilon = 1e-2
-    keras.backend.set_epsilon(epsilon)
-    self.assertEqual(keras.backend.epsilon(), epsilon)
-    keras.backend.set_epsilon(1e-7)
-
-  def test_floatx(self):
-    floatx = 'float64'
-    keras.backend.set_floatx(floatx)
-    self.assertEqual(keras.backend.floatx(), floatx)
-    keras.backend.set_floatx('float32')
-
-  def test_image_data_format(self):
-    image_data_format = 'channels_first'
-    keras.backend.set_image_data_format(image_data_format)
-    self.assertEqual(keras.backend.image_data_format(), image_data_format)
-    keras.backend.set_image_data_format('channels_last')
-
   def test_get_reset_uids(self):
     self.assertEqual(keras.backend.get_uid('foo'), 1)
     self.assertEqual(keras.backend.get_uid('foo'), 2)
diff --git a/tensorflow/python/keras/optimizer_v2/BUILD b/tensorflow/python/keras/optimizer_v2/BUILD
index da757edc74..45afe2a134 100644
--- a/tensorflow/python/keras/optimizer_v2/BUILD
+++ b/tensorflow/python/keras/optimizer_v2/BUILD
@@ -35,6 +35,7 @@ py_library(
         "//tensorflow/python:variables",
         "//tensorflow/python/distribute:reduce_util",
         "//tensorflow/python/distribute:values",
+        "//tensorflow/python/keras:backend_config",
     ],
 )
 
diff --git a/tensorflow/python/keras/optimizer_v2/adadelta.py b/tensorflow/python/keras/optimizer_v2/adadelta.py
index b6cba263bc..a3d5538ea8 100644
--- a/tensorflow/python/keras/optimizer_v2/adadelta.py
+++ b/tensorflow/python/keras/optimizer_v2/adadelta.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 
+from tensorflow.python.keras import backend_config
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
 from tensorflow.python.training import training_ops
 from tensorflow.python.util.tf_export import keras_export
@@ -90,6 +91,8 @@ class Adadelta(optimizer_v2.OptimizerV2):
     invocations of optimizer functions.
     @end_compatibility
     """
+    if epsilon is None:
+      epsilon = backend_config.epsilon()
     super(Adadelta, self).__init__(name, **kwargs)
     self._set_hyper('learning_rate', kwargs.get('lr', learning_rate))
     self._set_hyper('decay', self._initial_decay)
diff --git a/tensorflow/python/keras/optimizer_v2/adadelta_test.py b/tensorflow/python/keras/optimizer_v2/adadelta_test.py
index 442a1d3189..06ff975212 100644
--- a/tensorflow/python/keras/optimizer_v2/adadelta_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adadelta_test.py
@@ -181,6 +181,15 @@ class AdadeltaOptimizerTest(test.TestCase):
     self.assertAllClose(self.evaluate(opt_2.lr), (1.0))
     self.assertAllClose(self.evaluate(opt_3.lr), (0.1))
 
+  def testConstructAdadeltaWithEpsilonValues(self):
+    opt = adadelta.Adadelta(epsilon=None)
+    config = opt.get_config()
+    self.assertEqual(config["epsilon"], 1e-7)
+
+    opt = adadelta.Adadelta(epsilon=1e-8)
+    config = opt.get_config()
+    self.assertEqual(config["epsilon"], 1e-8)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/keras/optimizer_v2/adagrad.py b/tensorflow/python/keras/optimizer_v2/adagrad.py
index 01c722521f..0840aa6fae 100644
--- a/tensorflow/python/keras/optimizer_v2/adagrad.py
+++ b/tensorflow/python/keras/optimizer_v2/adagrad.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import ops
+from tensorflow.python.keras import backend_config
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import init_ops
@@ -89,6 +90,8 @@ class Adagrad(optimizer_v2.OptimizerV2):
     if initial_accumulator_value < 0.0:
       raise ValueError('initial_accumulator_value must be non-negative: %s' %
                        initial_accumulator_value)
+    if epsilon is None:
+      epsilon = backend_config.epsilon()
     if epsilon < 1e-7:
       raise ValueError('epsilon must be larger than 1e-7: %s' % epsilon)
     super(Adagrad, self).__init__(name, **kwargs)
diff --git a/tensorflow/python/keras/optimizer_v2/adagrad_test.py b/tensorflow/python/keras/optimizer_v2/adagrad_test.py
index 1599a54745..864aefaf70 100644
--- a/tensorflow/python/keras/optimizer_v2/adagrad_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adagrad_test.py
@@ -411,6 +411,19 @@ class AdagradOptimizerTest(test.TestCase):
     self.assertAllClose(self.evaluate(opt_2.lr), (1.0))
     self.assertAllClose(self.evaluate(opt_3.lr), (0.1))
 
+  def testConstructAdagradWithEpsilonValues(self):
+    opt = adagrad.Adagrad(epsilon=None)
+    config = opt.get_config()
+    self.assertEqual(config["epsilon"], 1e-7)
+
+    opt = adagrad.Adagrad(epsilon=1e-6)
+    config = opt.get_config()
+    self.assertEqual(config["epsilon"], 1e-6)
+
+    with self.assertRaisesRegexp(ValueError,
+                                 "epsilon must be larger than 1e-7"):
+      opt = adagrad.Adagrad(epsilon=1e-8)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/keras/optimizer_v2/adam.py b/tensorflow/python/keras/optimizer_v2/adam.py
index 263004c9df..4fa7c73615 100644
--- a/tensorflow/python/keras/optimizer_v2/adam.py
+++ b/tensorflow/python/keras/optimizer_v2/adam.py
@@ -18,6 +18,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.framework import ops
+from tensorflow.python.keras import backend_config
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
@@ -131,6 +132,8 @@ class Adam(optimizer_v2.OptimizerV2):
         compatibility, recommended to use `learning_rate` instead.
     """
 
+    if epsilon is None:
+      epsilon = backend_config.epsilon()
     super(Adam, self).__init__(name, **kwargs)
     self._set_hyper('learning_rate', kwargs.get('lr', learning_rate))
     self._set_hyper('decay', self._initial_decay)
diff --git a/tensorflow/python/keras/optimizer_v2/adam_test.py b/tensorflow/python/keras/optimizer_v2/adam_test.py
index 837d0220e1..7918c09b7e 100644
--- a/tensorflow/python/keras/optimizer_v2/adam_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adam_test.py
@@ -516,6 +516,15 @@ class AdamOptimizerTest(test.TestCase):
     self.assertAllClose(self.evaluate(opt_2.lr), (1.0))
     self.assertAllClose(self.evaluate(opt_3.lr), (0.1))
 
+  def testConstructAdamWithEpsilonValues(self):
+    opt = adam.Adam(epsilon=None)
+    config = opt.get_config()
+    self.assertEqual(config["epsilon"], 1e-7)
+
+    opt = adam.Adam(epsilon=1e-8)
+    config = opt.get_config()
+    self.assertEqual(config["epsilon"], 1e-8)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/keras/optimizer_v2/adamax.py b/tensorflow/python/keras/optimizer_v2/adamax.py
index 9a1b63f6c6..3102e28cff 100644
--- a/tensorflow/python/keras/optimizer_v2/adamax.py
+++ b/tensorflow/python/keras/optimizer_v2/adamax.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.framework import ops
+from tensorflow.python.keras import backend_config
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
@@ -95,6 +96,8 @@ class Adamax(optimizer_v2.OptimizerV2):
         allow time inverse decay of learning rate. `lr` is included for backward
         compatibility, recommended to use `learning_rate` instead.
     """
+    if epsilon is None:
+      epsilon = backend_config.epsilon()
     super(Adamax, self).__init__(name, **kwargs)
     self._set_hyper('learning_rate', kwargs.get('lr', learning_rate))
     self._set_hyper('decay', self._initial_decay)
diff --git a/tensorflow/python/keras/optimizer_v2/adamax_test.py b/tensorflow/python/keras/optimizer_v2/adamax_test.py
index fe658761c7..6934f1590e 100644
--- a/tensorflow/python/keras/optimizer_v2/adamax_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adamax_test.py
@@ -375,6 +375,15 @@ class AdamaxOptimizerTest(test.TestCase):
     self.assertAllClose(self.evaluate(opt_2.lr), (1.0))
     self.assertAllClose(self.evaluate(opt_3.lr), (0.1))
 
+  def testConstructAdamaxWithEpsilonValues(self):
+    opt = adamax.Adamax(epsilon=None)
+    config = opt.get_config()
+    self.assertEqual(config["epsilon"], 1e-7)
+
+    opt = adamax.Adamax(epsilon=1e-8)
+    config = opt.get_config()
+    self.assertEqual(config["epsilon"], 1e-8)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/keras/optimizer_v2/nadam.py b/tensorflow/python/keras/optimizer_v2/nadam.py
index 4c94b1358a..d515f98725 100644
--- a/tensorflow/python/keras/optimizer_v2/nadam.py
+++ b/tensorflow/python/keras/optimizer_v2/nadam.py
@@ -18,6 +18,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.framework import ops
+from tensorflow.python.keras import backend_config
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
@@ -85,6 +86,8 @@ class Nadam(optimizer_v2.OptimizerV2):
 
     # Backwards compatiblity with keras NAdam optimizer.
     kwargs['decay'] = kwargs.pop('schedule_decay', 0.004)
+    if epsilon is None:
+      epsilon = backend_config.epsilon()
     super(Nadam, self).__init__(name, **kwargs)
     self._set_hyper('learning_rate', kwargs.get('lr', learning_rate))
     self._set_hyper('decay', self._initial_decay)
diff --git a/tensorflow/python/keras/optimizer_v2/nadam_test.py b/tensorflow/python/keras/optimizer_v2/nadam_test.py
index b877a80cd8..8dd61956f6 100644
--- a/tensorflow/python/keras/optimizer_v2/nadam_test.py
+++ b/tensorflow/python/keras/optimizer_v2/nadam_test.py
@@ -178,6 +178,15 @@ class NadamOptimizerTest(test.TestCase):
     self.evaluate(variables.global_variables_initializer())
     self.assertAllClose(self.evaluate(opt.decay), (0.2))
 
+  def testConstructNAdamWithEpsilonValues(self):
+    opt = nadam.Nadam(epsilon=None)
+    config = opt.get_config()
+    self.assertEqual(config["epsilon"], 1e-7)
+
+    opt = nadam.Nadam(epsilon=1e-8)
+    config = opt.get_config()
+    self.assertEqual(config["epsilon"], 1e-8)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/keras/optimizer_v2/rmsprop.py b/tensorflow/python/keras/optimizer_v2/rmsprop.py
index d459d9811e..e55e6375a3 100644
--- a/tensorflow/python/keras/optimizer_v2/rmsprop.py
+++ b/tensorflow/python/keras/optimizer_v2/rmsprop.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import ops
+from tensorflow.python.keras import backend_config
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
@@ -102,6 +103,8 @@ class RMSprop(optimizer_v2.OptimizerV2):
         allow time inverse decay of learning rate. `lr` is included for backward
         compatibility, recommended to use `learning_rate` instead.
     """
+    if epsilon is None:
+      epsilon = backend_config.epsilon()
     super(RMSprop, self).__init__(name, **kwargs)
     self._set_hyper("learning_rate", kwargs.get("lr", learning_rate))
     self._set_hyper("decay", self._initial_decay)
diff --git a/tensorflow/python/keras/optimizer_v2/rmsprop_test.py b/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
index d4bc7ce50c..a9ddc2155a 100644
--- a/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
+++ b/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
@@ -471,6 +471,15 @@ class RMSpropOptimizerTest(test.TestCase):
       self.assertEqual(
           self.evaluate(opt.variables()[0]), self.evaluate(opt.iterations))
 
+  def testConstructRMSpropWithEpsilonValues(self):
+    opt = rmsprop.RMSprop(epsilon=None)
+    config = opt.get_config()
+    self.assertEqual(config["epsilon"], 1e-7)
+
+    opt = rmsprop.RMSprop(epsilon=1e-8)
+    config = opt.get_config()
+    self.assertEqual(config["epsilon"], 1e-8)
+
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 53c44a9ac4e69c788ef5b707d21c0f364eedb9a8 Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Thu, 17 Jan 2019 09:06:26 -0800
Subject: [PATCH 0856/2345] Make a real repro for IConstantLayer: we need to
 call getType() before getDimensions(), then if CreateConstantLayer() doesn't
 set the dtype, the test will fail.

Before CreateConstantLayer() was added, graphs which trigger conversion of INT32
weights to IConstantLayer will crash the TRT conversion. This is a bug in TRT
5.0 and I has reported it to NVIDIA. CreateConstantLayer() fixed that by setting
the dtype of the output ITensor of the IConstantLayer, but the added test could
not reproduce the error when we remove the setting code. This change fix the
test, so when removing the dtype setting logic the test will fail.

Note that, after this fix, during conversion it'll report a lot of spammy logs
but won't crash. Once NVIDIA fixed the TRT bug we should remove the dtype
setting logic and the test should still pass.

PiperOrigin-RevId: 229758150
---
 tensorflow/contrib/tensorrt/convert/convert_nodes.cc      | 7 ++++---
 tensorflow/contrib/tensorrt/convert/convert_nodes.h       | 3 +++
 tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc | 4 +++-
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index a1c519d908..8b7279ad03 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -340,10 +340,11 @@ nvinfer1::ITensor* Converter::CreateConstantLayer(
   nvinfer1::IConstantLayer* layer = network()->addConstant(dims, trt_weights);
   if (!layer) return nullptr;
   const nvinfer1::DataType trt_dtype = trt_weights.type;
-  // NOTE(laigd): calling layer->setOutputType() does not work. This may be a
-  // bug and we should report to NVIDIA.
-  // layer->setOutputType(0, trt_dtype);
   nvinfer1::ITensor* trt_tensor = layer->getOutput(0);
+  // TODO(laigd): there is a bug in TensorRT 5.0 library that, if we don't set
+  // the data type below, it will always be kFLOAT regardless what the data type
+  // of the weights is. Once NVIDIA fixes this bug, we should remove the data
+  // type setting logic below and test should still pass.
   trt_tensor->setType(trt_dtype);
   return trt_tensor;
 }
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.h b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
index 25e1c42fdf..a02a52e430 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
@@ -158,7 +158,10 @@ class OutputEdgeValidator {
   bool operator()(const tensorflow::Edge* out_edge) const;
 };
 
+string DebugString(const nvinfer1::DimensionType type);
+string DebugString(const nvinfer1::DataType trt_dtype);
 string DebugString(const nvinfer1::Dims& dims);
+string DebugString(const nvinfer1::Permutation& permutation, int len);
 string DebugString(const nvinfer1::ITensor& tensor);
 int64_t TrtDimsNumElements(const nvinfer1::Dims& dims);
 
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index 36dbbe3f6f..f90cdac359 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -922,8 +922,10 @@ TEST_F(ConverterTest, CreateConstantLayer) {
     nvinfer1::ITensor* tensor =
         converter_->CreateConstantLayer(weights, GetTestDims({3, 10}));
     ASSERT_NE(nullptr, tensor);
+    EXPECT_EQ(TfDataTypeToTrt(dtype), tensor->getType())
+        << "Expected " << DebugString(TfDataTypeToTrt(dtype)) << " vs. actual "
+        << DebugString(tensor->getType());
     ExpectTrtDimsEqualsArray({3, 10}, tensor->getDimensions());
-    EXPECT_EQ(TfDataTypeToTrt(dtype), tensor->getType());
   }
 }
 
-- 
GitLab


From ac7f91ee3cd48c01f9ff253440e81c36a729526a Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Thu, 17 Jan 2019 09:14:03 -0800
Subject: [PATCH 0857/2345] Make serialization work with kwargs.

Additionally:
1) Unify the way how concrete functions and concrete functions from signatures are serialized. This can be achieved by using signature from the concrete_function.structured_input_signature field instead of constructing it manually.
2) Add a test for serializing a concrete function, then restoring it and calling with a positional argument.
PiperOrigin-RevId: 229759347
---
 tensorflow/python/eager/def_function_test.py  | 18 +++++----
 tensorflow/python/eager/function.py           | 13 +++++--
 .../eager/function_argument_naming_test.py    |  8 ++--
 tensorflow/python/framework/func_graph.py     | 39 +++++++++++++++----
 .../saved_model/function_deserialization.py   | 32 ++++++++-------
 .../saved_model/function_serialization.py     | 28 +++----------
 tensorflow/python/saved_model/load_test.py    | 39 +++++++++++++++++++
 7 files changed, 115 insertions(+), 62 deletions(-)

diff --git a/tensorflow/python/eager/def_function_test.py b/tensorflow/python/eager/def_function_test.py
index 256ef511a3..e0a82bb0be 100644
--- a/tensorflow/python/eager/def_function_test.py
+++ b/tensorflow/python/eager/def_function_test.py
@@ -247,10 +247,9 @@ class DefFunctionTest(test.TestCase):
     concrete = compute.get_concrete_function(
         tensor_spec.TensorSpec(None, dtypes.float32))
     self.assertAllClose(4., concrete(constant_op.constant(2.)))
-    input_signature, _ = concrete.structured_input_signature
-    self.assertEqual(
-        tuple(input_signature),
-        (tensor_spec.TensorSpec(None, dtypes.float32),))
+    signature_args, _ = concrete.structured_input_signature
+    self.assertEqual(signature_args,
+                     (tensor_spec.TensorSpec(None, dtypes.float32),))
 
   def test_serialization_signature_cache(self):
 
@@ -261,12 +260,15 @@ class DefFunctionTest(test.TestCase):
     f(constant_op.constant([[3., 4.]]), constant_op.constant([2.]))
     f(constant_op.constant([[3, 4, 5]]), constant_op.constant([2]))
 
+    signatures_args = set()
     concrete_functions = f._list_all_concrete_functions_for_serialization()
-    signatures_for_serialization = [
-        c.structured_input_signature[0] for c in concrete_functions
-    ]
+    for concrete_function in concrete_functions:
+      args, kwargs = concrete_function.structured_input_signature
+      signatures_args.add(args)
+      self.assertEqual(dict(), kwargs)
+
     self.assertEqual(
-        set(signatures_for_serialization),
+        signatures_args,
         set(((tensor_spec.TensorSpec([1, 2], dtypes.float32),
               tensor_spec.TensorSpec([1], dtypes.float32)),
              (tensor_spec.TensorSpec([1, 3], dtypes.int32),
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 82dea51c29..d17c7020f3 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -1306,10 +1306,15 @@ class Function(object):
           arglen = len(args)
         else:
           arglen = len(self._input_signature)
-        arg_names = (
-            self._function_spec.arg_names[:arglen]
-            + [self._function_spec.vararg_name] *
-            (arglen - len(self._function_spec.arg_names)))
+        base_arg_names = self._function_spec.arg_names[:arglen]
+        num_missing_args = arglen - len(self._function_spec.arg_names)
+        missing_arg_names = [self._function_spec.vararg_name] * num_missing_args
+        # Produce a list of missing args of the form ["arg_0", "arg_1", ...],
+        # where arg is based on the self._function_spec.vararg_name.
+        missing_arg_names = [
+            "%s_%d" % (arg, i) for i, arg in enumerate(missing_arg_names)
+        ]
+        arg_names = base_arg_names + missing_arg_names
         graph_function = ConcreteFunction(
             func_graph_module.func_graph_from_py_func(
                 self._name,
diff --git a/tensorflow/python/eager/function_argument_naming_test.py b/tensorflow/python/eager/function_argument_naming_test.py
index 9358c4fd07..08a50a8f51 100644
--- a/tensorflow/python/eager/function_argument_naming_test.py
+++ b/tensorflow/python/eager/function_argument_naming_test.py
@@ -220,10 +220,10 @@ class ArgumentNamingTests(test.TestCase, parameterized.TestCase):
         z=tensor_spec.TensorSpec(shape=(), dtype=dtypes.float32),
         zz=tensor_spec.TensorSpec(shape=(), dtype=dtypes.float32, name='cust'))
     self.assertEqual(
-        ['x', 'y', 'args', 'second_variadic', 'z', 'cust'],
+        ['x', 'y', 'args_1', 'second_variadic', 'z', 'cust'],
         [inp.op.name for inp in variadic_op.inputs])
     self.assertEqual(
-        [b'x', b'y', b'args', b'second_variadic', b'z', b'cust'],
+        [b'x', b'y', b'args_1', b'second_variadic', b'z', b'cust'],
         [inp.op.get_attr('_user_specified_name')
          for inp in variadic_op.inputs])
 
@@ -244,10 +244,10 @@ class ArgumentNamingTests(test.TestCase, parameterized.TestCase):
     variadic_op = variadic_fn.get_concrete_function()
     self.assertIn(b'variadic_fn', variadic_op.name)
     self.assertEqual(
-        ['x', 'y', 'args', 'z'],
+        ['x', 'y', 'args_1', 'z'],
         [inp.op.name for inp in variadic_op.inputs])
     self.assertEqual(
-        [b'x', b'y', b'args', b'z'],
+        [b'x', b'y', b'args_1', b'z'],
         [inp.op.get_attr('_user_specified_name')
          for inp in variadic_op.inputs])
 
diff --git a/tensorflow/python/framework/func_graph.py b/tensorflow/python/framework/func_graph.py
index 5202e9b8f9..36f7aa67c7 100644
--- a/tensorflow/python/framework/func_graph.py
+++ b/tensorflow/python/framework/func_graph.py
@@ -63,11 +63,20 @@ class UnknownArgument(object):
   pass
 
 
-def convert_structure_to_signature(structure):
+# TODO(vbardiovsky): Remove this when nest is updated with new
+# flatten_with_tuple_paths.
+def flatten_with_tuple_paths(structure):
+  return list(zip(nest.yield_flat_paths(structure), nest.flatten(structure)))
+
+
+def convert_structure_to_signature(structure, arg_names=None):
   """Convert a potentially nested structure to a signature.
 
   Args:
-    structure: Structure to convert.
+    structure: Structure to convert, where top level collection is a list or a
+      tuple.
+    arg_names: Optional list of arguments that has equal number of elements as
+      `structure` and is used for naming corresponding TensorSpecs.
 
   Returns:
     Identical structure that has TensorSpec objects instead of Tensors and
@@ -84,8 +93,19 @@ def convert_structure_to_signature(structure):
 
   # We are using the flattened paths to name the TensorSpecs. We need an
   # explicit name for them downstream.
-  flattened_with_paths = nest.flatten_with_joined_string_paths(structure)
-  mapped = [encode_arg(arg, path) for path, arg in flattened_with_paths]
+  flattened = flatten_with_tuple_paths(structure)
+  if arg_names:
+    if len(arg_names) != len(structure):
+      raise ValueError(
+          "Passed in arg_names don't match actual signature (%s)." % arg_names)
+    # Replace all top-level names with their actual arg_names. If a path before
+    # was "(2,'a',1)", it will become "(arg_names[2],'a',1)".
+    flattened = [((arg_names[0],) + path[1:], arg) for path, arg in flattened]
+
+  mapped = [
+      encode_arg(arg, "/".join([str(p) for p in path]))
+      for path, arg in flattened
+  ]
   return nest.pack_sequence_as(structure, mapped)
 
 
@@ -100,9 +120,9 @@ class FuncGraph(ops.Graph):
       inputs coming first.
     outputs: Tensors that will be returned by this function. The tensors are in
       this FuncGraph.
-    structured_input_signature: A possibly-nested python object that was
-      received by this function. Note that this structure might contain Python
-      `None`s.
+    structured_input_signature: A tuple of (args, kwargs), which are both
+      possibly-nested python objects that were received by this function. Note
+      that these structures might contain Python `None`s.
     structured_outputs: A possibly-nested python object which will be returned
       by this function. The Tensors in this structure are the same as those of
       self.outputs. Note that this structure might contain Python `None`s.
@@ -443,8 +463,11 @@ def func_graph_from_py_func(name,
     func_kwargs = _get_defun_inputs_from_kwargs(kwargs)
 
     # Convert all Tensors into TensorSpecs before saving the structured inputs.
+    # If storing pure concrete functions that are not called through polymorphic
+    # functions, we don't have access to FunctionSpec, so we need to call the
+    # TensorSpecs by their `arg_names` for later binding.
     func_graph.structured_input_signature = (
-        convert_structure_to_signature(func_args),
+        convert_structure_to_signature(func_args, arg_names),
         convert_structure_to_signature(func_kwargs))
 
     # Note: `nest.flatten` sorts by keys, as does `_deterministic_dict_values`.
diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py
index 5479bbb355..ec5214824e 100644
--- a/tensorflow/python/saved_model/function_deserialization.py
+++ b/tensorflow/python/saved_model/function_deserialization.py
@@ -73,17 +73,23 @@ def _deserialize_function_spec(function_spec_proto, coder):
 
 def recreate_concrete_function(saved_concrete_function, concrete_functions):
   """Recreates a user-facing concrete function."""
+  coder = nested_structure_coder.StructureCoder()
+
   concrete_function = concrete_functions[saved_concrete_function.name]
-  input_signature = (saved_concrete_function.canonicalized_input_signature
-                     .list_value.values)
+  input_signature = coder.decode_proto(
+      saved_concrete_function.canonicalized_input_signature)
+  input_signature_args, input_signature_kwargs = input_signature
+  if input_signature_kwargs:
+    raise ValueError("Restoring concrete function with non-empty kwargs (%s)." %
+                     input_signature_kwargs)
+
   # pylint: disable=protected-access
   # Set metadata required for the concrete function to accept keyword and
   # positional arguments in __call__. Normally this is set in
   # get_concrete_function.
-  concrete_function._arg_keywords = [
-      spec.tensor_spec_value.name for spec in input_signature]
+  concrete_function._arg_keywords = [spec.name for spec in input_signature_args]
   # TODO(allenl): Should we preserve the number of allowed positional arguments?
-  concrete_function._num_positional_args = len(input_signature)
+  concrete_function._num_positional_args = len(input_signature_args)
   # pylint: enable=protected-access
   concrete_function.add_to_graph()
   return concrete_function
@@ -120,25 +126,21 @@ def recreate_function(saved_function, concrete_functions):
           concrete_function.canonicalized_input_signature)
 
       try:
-        can_args, can_kwargs = function_spec.canonicalize_function_inputs(
+        canonicalized_inputs = function_spec.canonicalize_function_inputs(
             *args, **kwargs)
-        if can_kwargs:
-          # TODO(vbardiovsky): Enable this along with the structured input and
-          # structured output.
-          raise ValueError(
-              "Received keywords arguments that could not be bound: %s" %
-              kwargs)
       except ValueError:
         continue
 
-      if _inputs_compatible(can_args,
+      if _inputs_compatible(canonicalized_inputs,
                             canonicalized_original_inputs):
-        flattened_inputs = nest.flatten(can_args)
+        flattened_inputs = nest.flatten(canonicalized_inputs)
         filtered_inputs = [t for t in flattened_inputs if _is_tensor(t)]
         return function_obj._call_flat(filtered_inputs)  # pylint: disable=protected-access
 
     raise AssertionError(
-        "Could not find matching function to call for arguments: %s" % (args,))
+        "Could not find matching function to call for args %r and kwargs %r" %
+        (args, kwargs))
+
   return restored_function
 
 
diff --git a/tensorflow/python/saved_model/function_serialization.py b/tensorflow/python/saved_model/function_serialization.py
index c79c61b60c..172ab41dc8 100644
--- a/tensorflow/python/saved_model/function_serialization.py
+++ b/tensorflow/python/saved_model/function_serialization.py
@@ -19,7 +19,6 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.framework import func_graph as func_graph_module
-from tensorflow.python.framework import tensor_spec
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.saved_model import nested_structure_coder
 from tensorflow.python.saved_model import saved_object_graph_pb2
@@ -39,8 +38,7 @@ def _serialize_function_spec(function_spec, coder):
   return proto
 
 
-def _serialize_concrete_function_with_signature(
-    concrete_function, node_ids, signature_args, coder):
+def serialize_concrete_function(concrete_function, node_ids):
   """Build a SavedConcreteFunction."""
   bound_inputs = []
   try:
@@ -53,10 +51,11 @@ def _serialize_concrete_function_with_signature(
         "captures tensor %s which is unsupported or not reachable from root.",
         concrete_function.name, capture)
     return None
+  coder = nested_structure_coder.StructureCoder()
   concrete_function_proto = saved_object_graph_pb2.SavedConcreteFunction()
   concrete_function_proto.name = concrete_function.name
   concrete_function_proto.canonicalized_input_signature.CopyFrom(
-      coder.encode_structure(signature_args))
+      coder.encode_structure(concrete_function.structured_input_signature))
   structured_outputs = func_graph_module.convert_structure_to_signature(
       concrete_function.structured_outputs)
   concrete_function_proto.output_signature.CopyFrom(
@@ -65,20 +64,6 @@ def _serialize_concrete_function_with_signature(
   return concrete_function_proto
 
 
-def serialize_concrete_function(concrete_function, node_ids):
-  """Build a standalone SavedConcreteFunction."""
-  coder = nested_structure_coder.StructureCoder()
-  signature = []
-  # Construct a flat list of TensorSpecs for this concrete function. On load
-  # we'll parse them and recreate the call metadata.
-  for input_tensor, input_keyword in zip(
-      concrete_function.inputs, concrete_function._arg_keywords):  # pylint: disable=protected-access
-    signature.append(tensor_spec.TensorSpec(
-        input_tensor.shape, input_tensor.dtype, name=input_keyword))
-  return _serialize_concrete_function_with_signature(
-      concrete_function, node_ids, signature, coder)
-
-
 def serialize_function(function, node_ids):
   """Build a SavedFunction proto."""
   coder = nested_structure_coder.StructureCoder()
@@ -89,11 +74,8 @@ def serialize_function(function, node_ids):
   all_concrete_functions = \
       function._list_all_concrete_functions_for_serialization()  # pylint: disable=protected-access
   for concrete_function in all_concrete_functions:
-    signature_args, signature_kwargs = \
-        concrete_function.structured_input_signature
-    del signature_kwargs
-    concrete_function_proto = _serialize_concrete_function_with_signature(
-        concrete_function, node_ids, signature_args, coder)
+    concrete_function_proto = serialize_concrete_function(
+        concrete_function, node_ids)
     if concrete_function_proto is not None:
       proto.concrete_function.add().CopyFrom(concrete_function_proto)
   return proto
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index e3dcbda50a..280a26a0a5 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -284,6 +284,28 @@ class LoadTest(test.TestCase):
     self.assertEqual(7, imported.f(constant_op.constant(2)).numpy())
     self.assertEqual(6, imported.f(constant_op.constant(1), defg=7.0).numpy())
 
+  def test_additional_kwargs(self):
+    def func(x, training=False, **options):
+      del options
+      if training:
+        return 2 * x
+      else:
+        return 7
+
+    root = tracking.AutoCheckpointable()
+    root.f = def_function.function(func)
+
+    x = constant_op.constant(10)
+    self.assertEqual(7, root.f(x, learning_rate=0.5, epochs=3).numpy())
+
+    imported = self.cycle(root)
+
+    with self.assertRaisesRegexp(AssertionError,
+                                 "Could not find matching function to call.*"):
+      imported.f(x, learning_rate=0.5, epochs=4)
+
+    self.assertEqual(7, imported.f(x, learning_rate=0.5, epochs=3).numpy())
+
   def test_member_function(self):
     class CheckpointableWithMember(tracking.AutoCheckpointable):
 
@@ -438,6 +460,23 @@ class LoadTest(test.TestCase):
     self.assertAllEqual([2, 4, 6],
                         imported.f(constant_op.constant([1, 2, 3])).numpy())
 
+  def test_concrete_function_arg_names(self):
+
+    @def_function.function(
+        input_signature=[tensor_spec.TensorSpec([None], dtypes.int32)])
+    def func(x):
+      return 2 * x
+
+    root = tracking.AutoCheckpointable()
+    root.f = func.get_concrete_function()
+
+    self.assertAllEqual([2], root.f(constant_op.constant([1])).numpy())
+
+    imported = self.cycle(root)
+
+    self.assertAllEqual([2, 4, 6],
+                        imported.f(x=constant_op.constant([1, 2, 3])).numpy())
+
   def test_concrete_function_no_signature(self):
     @def_function.function
     def func(x):
-- 
GitLab


From 452907059d5844831b15d4202574a3b6be5d8b91 Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Thu, 17 Jan 2019 10:22:56 -0800
Subject: [PATCH 0858/2345] Cleanup: remove unused code.

PiperOrigin-RevId: 229772701
---
 tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index f90cdac359..c8739b46c1 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -364,9 +364,6 @@ TEST(TRT_TensorOrWeights_Test, Basic) {
       EXPECT_EQ(false, ptr->is_tensor());
       EXPECT_EQ(true, ptr->is_weights());
       EXPECT_TRUE(TrtShapedWeightsEquals(weights, ptr->weights()));
-
-      nvinfer1::Dims dims;
-      dims.nbDims = 0;
       ExpectTrtDimsEqualsArray({}, ptr->GetTrtDims());
     }
   }
-- 
GitLab


From a8dbd3a680b06f617baef53dfad47a10e642f0ca Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Thu, 17 Jan 2019 10:31:28 -0800
Subject: [PATCH 0859/2345] Do not override `steps_per_epoch` if it is not
 None.

Currently if you pass a tf.keras.preprocessing.image.ImageDataGenerator to fit,
with a `steps_per_epoch`, your manual `steps_per_epoch` is overridden with the real value. Fix it to only override the value if the `steps_per_epoch` was not manually set.

PiperOrigin-RevId: 229774626
---
 tensorflow/python/keras/engine/training_generator.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/engine/training_generator.py b/tensorflow/python/keras/engine/training_generator.py
index 23b5d7e8c5..b8defad8b2 100644
--- a/tensorflow/python/keras/engine/training_generator.py
+++ b/tensorflow/python/keras/engine/training_generator.py
@@ -410,7 +410,9 @@ def convert_to_generator_like(data,
       and may be `None` or `[None]`.
     batch_size: Used when creating a generator out of tuples of NumPy arrays or
       EagerTensors.
-    steps_per_epoch: Steps of the generator to run each epoch.
+    steps_per_epoch: Steps of the generator to run each epoch. If `None` the
+      number of steps will be read from the data (for
+      `keras.utils.data_utils.Sequence` types).
     epochs: Total number of epochs to run.
     shuffle: Whether the data should be shuffled.
 
@@ -431,7 +433,8 @@ def convert_to_generator_like(data,
   if data_utils.is_generator_or_sequence(data) or isinstance(
       data, iterator_ops.EagerIterator):
     if isinstance(data, data_utils.Sequence):
-      steps_per_epoch = len(data)
+      if steps_per_epoch is None:
+        steps_per_epoch = len(data)
     return data, steps_per_epoch
   if isinstance(data, dataset_ops.DatasetV2):
     return dataset_ops.make_one_shot_iterator(data), steps_per_epoch
-- 
GitLab


From 5f98ba7eba6538f6cb389fa7326a9549dcd1299a Mon Sep 17 00:00:00 2001
From: Andr? Susano Pinto <andresp@google.com>
Date: Thu, 17 Jan 2019 10:34:59 -0800
Subject: [PATCH 0860/2345] Refactor registering gradient of ConcreteFunction.

PiperOrigin-RevId: 229775288
---
 tensorflow/python/eager/function.py | 25 ++++++++++---------------
 1 file changed, 10 insertions(+), 15 deletions(-)

diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index d17c7020f3..9c3e20f936 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -506,26 +506,25 @@ class ConcreteFunction(object):
     if context.executing_eagerly() or not self.outputs:
       outputs = self._inference_function.call(ctx, args)
     else:
-      if not self._gradient_name:
-        self._gradient_name = "PartitionedCall-%s" % ops.uid()
-        self._register_gradient(self._gradient_name)
+      self._register_gradient()
       with ops.get_default_graph().gradient_override_map(
           {"PartitionedCall": self._gradient_name,
            "StatefulPartitionedCall": self._gradient_name}):
         outputs = self._inference_function.call(ctx, args)
     return self._build_call_outputs(outputs)
 
-  def _register_gradient(self, name):
-    """Registers the gradient for this `ConcreteFunction` under the given name.
+  def _register_gradient(self):
+    """Registers the gradient for this `ConcreteFunction`.
 
     The gradient rewrites an inference call op to a forward call op, but does
     not modify a pre-existing forward call op. It then computes the gradient
     from the output's gradients and the side outputs of the forward op.
-
-    Args:
-      name: The name to register the gradient as.
     """
-    @ops.RegisterGradient(name)
+    if self._gradient_name:
+      return
+    self._gradient_name = "PartitionedCall-%s" % ops.uid()
+
+    @ops.RegisterGradient(self._gradient_name)
     def _registered_grad_fn(op, *doutputs):  # pylint: disable=unused-variable
       return self._grad_fn(op, *doutputs)
 
@@ -724,9 +723,7 @@ class ConcreteFunction(object):
 
     ctx = context.context()
 
-    if not self._gradient_name:
-      self._gradient_name = "PartitionedCall-%s" % ops.uid()
-      self._register_gradient(self._gradient_name)
+    self._register_gradient()
     with ops.get_default_graph().gradient_override_map(
         {"PartitionedCall": self._gradient_name,
          "StatefulPartitionedCall": self._gradient_name}):
@@ -773,9 +770,7 @@ class ConcreteFunction(object):
     """
     ctx = context.context()
 
-    if not self._gradient_name:
-      self._gradient_name = "PartitionedCall-%s" % ops.uid()
-      self._register_gradient(self._gradient_name)
+    self._register_gradient()
     with ops.get_default_graph().gradient_override_map(
         {"PartitionedCall": self._gradient_name,
          "StatefulPartitionedCall": self._gradient_name}):
-- 
GitLab


From 0f39e8a552018eba3bd2916c0a72467617bebf88 Mon Sep 17 00:00:00 2001
From: Haoyu Zhang <haoyuzhang@google.com>
Date: Thu, 17 Jan 2019 10:41:18 -0800
Subject: [PATCH 0861/2345] Reuse existing dataset iterators in Keras for
 better memory efficiency.

* In Keras model iteration, create only one iterator for training dataset and
  one iterator for validation datasets.
* If training requires dataset to be reset at each epoch, reinitialize the
  training iterator;
* If we run multiple validation passes between training epochs, reinitialize the validation
  iterator.

Compared to recreating iterators, this approach reduces host memory usage and
prevents memory leak.

PiperOrigin-RevId: 229776545
---
 .../engine/distributed_training_utils.py      | 10 ++++-
 .../python/keras/engine/training_arrays.py    | 38 ++++++++++++++++---
 .../python/keras/engine/training_utils.py     | 18 +++++++--
 3 files changed, 54 insertions(+), 12 deletions(-)

diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index 27ad6a5c0b..d13e0b6fa9 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -34,6 +34,7 @@ from tensorflow.python.keras import callbacks
 from tensorflow.python.keras import metrics as metrics_module
 from tensorflow.python.keras import optimizers
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
+from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import tf_logging as logging
@@ -528,10 +529,15 @@ def list_to_tuple(maybe_list):
 def get_iterator(dataset, distribution_strategy):
   with distribution_strategy.scope():
     iterator = distribution_strategy.make_dataset_iterator(dataset)
-    init_op = iterator.initialize()
+  initialize_iterator(iterator, distribution_strategy)
+  return iterator
+
+
+def initialize_iterator(iterator, distribution_strategy):
+  with distribution_strategy.scope():
+    init_op = control_flow_ops.group(iterator.initialize())
     if not context.executing_eagerly():
       K.get_session().run(init_op)
-  return iterator
 
 
 def _get_input_from_iterator(iterator, model):
diff --git a/tensorflow/python/keras/engine/training_arrays.py b/tensorflow/python/keras/engine/training_arrays.py
index 3b6a13839a..d0881c11a5 100644
--- a/tensorflow/python/keras/engine/training_arrays.py
+++ b/tensorflow/python/keras/engine/training_arrays.py
@@ -24,6 +24,7 @@ import functools
 import numpy as np
 
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.eager import context
 from tensorflow.python.framework import errors
 from tensorflow.python.keras import backend as K
@@ -119,18 +120,22 @@ def model_iteration(model,
 
   # In case we were passed a dataset, we extract symbolic tensors from it.
   reset_dataset_after_each_epoch = False
-  original_dataset = None
+  input_iterator = None
   is_dataset = isinstance(inputs,
                           (dataset_ops.DatasetV1, dataset_ops.DatasetV2))
   # TODO(fchollet): consider moving `steps_per_epoch` inference to
   # _standardize_user_data and set reset_dataset_after_each_epoch as an
   # attribute on the dataset instance.
   if is_dataset:
-    original_dataset = inputs
     if steps_per_epoch is None:
       reset_dataset_after_each_epoch = True
       steps_per_epoch = training_utils.infer_steps_for_dataset(
           inputs, steps_per_epoch, epochs=epochs, steps_name=steps_name)
+    input_iterator = _get_iterator(inputs, model._distribution_strategy)
+
+  val_iterator = None
+  if isinstance(val_inputs, (dataset_ops.DatasetV1, dataset_ops.DatasetV2)):
+    val_iterator = _get_iterator(val_inputs, model._distribution_strategy)
 
   if mode == ModeKeys.TRAIN:
     _print_train_info(inputs, val_inputs, steps_per_epoch, verbose)
@@ -150,6 +155,7 @@ def model_iteration(model,
       convert_eager_tensors_to_numpy((inputs, targets, sample_weights))
 
   # Prepare input data.
+  inputs = input_iterator or inputs
   ins = _prepare_feed_values(model, inputs, targets, sample_weights, mode)
   if not is_dataset:
     num_samples_or_steps = _get_num_samples_or_steps(ins, batch_size,
@@ -230,7 +236,7 @@ def model_iteration(model,
           actual_inputs = ins() if callable(ins) else ins
           batch_outs = f(actual_inputs)
         except errors.OutOfRangeError:
-          if original_dataset is None:
+          if not is_dataset:
             # We ran out of batches while the user passed an iterator (legacy).
             logging.warning(
                 'Your dataset iterator ran out of data; '
@@ -332,6 +338,7 @@ def model_iteration(model,
     if (do_validation and
         training_utils.should_run_validation(validation_freq, epoch) and
         not callbacks.model.stop_training):
+      val_inputs = val_iterator or val_inputs
       val_results = model_iteration(
           model,
           val_inputs,
@@ -348,15 +355,18 @@ def model_iteration(model,
         val_results = [val_results]
       epoch_logs = cbks.make_logs(
           model, epoch_logs, val_results, mode, prefix='val_')
+      if val_iterator and epoch < epochs - 1:
+        _reinitialize_iterator(val_iterator, model._distribution_strategy)
 
     if mode == ModeKeys.TRAIN:
       # Epochs only apply to `fit`.
       callbacks.on_epoch_end(epoch, epoch_logs)
     progbar.on_epoch_end(epoch, epoch_logs)
 
-    # Recreate dataset iterator for the next epoch.
+    # Reinitialize dataset iterator for the next epoch.
     if reset_dataset_after_each_epoch and epoch < epochs - 1:
-      ins = _prepare_feed_values(model, original_dataset, None, None, mode)
+      _reinitialize_iterator(input_iterator, model._distribution_strategy)
+      ins = _prepare_feed_values(model, input_iterator, None, None, mode)
 
   callbacks._call_end_hook(mode)
 
@@ -430,7 +440,8 @@ def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
     else:
       return get_distributed_inputs()
 
-  if isinstance(inputs, (dataset_ops.DatasetV1, dataset_ops.DatasetV2)):
+  if isinstance(inputs, (dataset_ops.DatasetV1, dataset_ops.DatasetV2,
+                         iterator_ops.Iterator)):
     inputs, targets, sample_weights = model._standardize_user_data(
         inputs,
         extract_tensors_from_dataset=True)
@@ -445,6 +456,21 @@ def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
   return ins
 
 
+def _get_iterator(inputs, distribution_strategy=None):
+  if distribution_strategy:
+    return distributed_training_utils.get_iterator(
+        inputs, distribution_strategy)
+  return training_utils.get_iterator(inputs)
+
+
+def _reinitialize_iterator(iterator, distribution_strategy=None):
+  if distribution_strategy:
+    distributed_training_utils.initialize_iterator(
+        iterator, distribution_strategy)
+  else:
+    training_utils.initialize_iterator(iterator)
+
+
 def _make_execution_function(model, mode):
   """Makes function to run one step of model execution."""
   if model._distribution_strategy:
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index 3affb84147..7e5bc08e5e 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -1232,6 +1232,19 @@ def is_dataset_or_iterator(data):
                            iterator_ops.Iterator))
 
 
+def get_iterator(dataset):
+  """Create and initialize an iterator from a dataset."""
+  iterator = dataset_ops.make_initializable_iterator(dataset)
+  initialize_iterator(iterator)
+  return iterator
+
+
+def initialize_iterator(iterator):
+  init_op = iterator.initializer
+  if not context.executing_eagerly():
+    K.get_session().run(init_op)
+
+
 def extract_tensors_from_dataset(dataset):
   """Extract a tuple of tensors `inputs, targets, sample_weight` from a dataset.
 
@@ -1241,10 +1254,7 @@ def extract_tensors_from_dataset(dataset):
   Returns:
     Tuple of tensors `x, y, weights`. `y` and `weights` entry may be None.
   """
-  iterator = dataset_ops.make_initializable_iterator(dataset)
-  init_op = iterator.initializer
-  if not context.executing_eagerly():
-    K.get_session().run(init_op)
+  iterator = get_iterator(dataset)
   inputs, targets, sample_weight = unpack_iterator_input(iterator)
   return inputs, targets, sample_weight
 
-- 
GitLab


From 887bcfa5116a11c1ee534b99cc1c118c29ff7e46 Mon Sep 17 00:00:00 2001
From: Cong Liu <congliu@google.com>
Date: Thu, 17 Jan 2019 10:51:03 -0800
Subject: [PATCH 0862/2345] [XLA] Fix a bug in dynamic parameter binding.

PiperOrigin-RevId: 229778432
---
 tensorflow/compiler/xla/service/dynamic_parameter_binding.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/dynamic_parameter_binding.cc b/tensorflow/compiler/xla/service/dynamic_parameter_binding.cc
index 3d0eddb0d8..5549cccfa8 100644
--- a/tensorflow/compiler/xla/service/dynamic_parameter_binding.cc
+++ b/tensorflow/compiler/xla/service/dynamic_parameter_binding.cc
@@ -70,7 +70,7 @@ StatusOr<DynamicParameterBinding> DynamicParameterBinding::CreateFromProto(
     int64 target_param_num = binding.target_param_num();
     ShapeIndex target_param_index(binding.target_param_index().begin(),
                                   binding.target_param_index().end());
-    int64 target_dim_num = binding.target_param_num();
+    int64 target_dim_num = binding.target_param_dim_num();
 
     TF_RETURN_IF_ERROR(
         result.Bind(DynamicParameter{dynamic_param_num, dynamic_param_index},
-- 
GitLab


From 8f275bdcede1de92ef2ec72324627ab5fc449a21 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 10:54:21 -0800
Subject: [PATCH 0863/2345] Fixes bug preventing cond_v2 from working with
 gradient tapes inside functions.

PiperOrigin-RevId: 229779015
---
 .../python/kernel_tests/cond_v2_test.py       | 22 +++++++++++++++++++
 tensorflow/python/ops/cond_v2.py              | 21 ++++++++++--------
 2 files changed, 34 insertions(+), 9 deletions(-)

diff --git a/tensorflow/python/kernel_tests/cond_v2_test.py b/tensorflow/python/kernel_tests/cond_v2_test.py
index 050da5ff6c..356c6f0a16 100644
--- a/tensorflow/python/kernel_tests/cond_v2_test.py
+++ b/tensorflow/python/kernel_tests/cond_v2_test.py
@@ -20,7 +20,9 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.core.protobuf import config_pb2
+from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
+from tensorflow.python.eager import def_function
 from tensorflow.python.eager import function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -628,6 +630,26 @@ class CondV2Test(test.TestCase):
         # d2[x]/dx2 = 0
         self.assertEqual(false_val, [0.0])
 
+  def testGradientTapeOfCondWithResourceVariableInFunction(self):
+    with context.eager_mode():
+      v = variables.Variable(2.)
+
+      @def_function.function
+      def fnWithCond():  # pylint: disable=invalid-name
+        with backprop.GradientTape() as tape:
+          pred = constant_op.constant(True, dtype=dtypes.bool)
+
+          def true_fn():
+            return math_ops.pow(v, 3)
+
+          def false_fn():
+            return v
+
+          cond = cond_v2.cond_v2(pred, true_fn, false_fn, name="cond")
+        return tape.gradient(cond, v)
+
+      self.assertAllEqual(fnWithCond(), 12.0)
+
   def testLowering(self):
     with ops.Graph().as_default() as g:
       with self.session(graph=g) as sess:
diff --git a/tensorflow/python/ops/cond_v2.py b/tensorflow/python/ops/cond_v2.py
index 5dca8e501b..6e8f2df39b 100644
--- a/tensorflow/python/ops/cond_v2.py
+++ b/tensorflow/python/ops/cond_v2.py
@@ -91,11 +91,13 @@ def cond_v2(pred, true_fn, false_fn, name="cond"):
 @ops.RegisterGradient("If")
 def _IfGrad(op, *grads):  # pylint: disable=invalid-name
   """The gradient of an If op produced by cond_v2."""
-  true_graph, false_graph = _get_func_graphs(op)
+  # Get the if operator (this logic handles the case where op is a MockOp)
+  if_op = op.outputs[0].op
+  true_graph, false_graph = _get_func_graphs(if_op)
   # Note: op.graph != ops.get_default_graph() when we are computing the gradient
   # of a nested cond.
-  assert true_graph.outer_graph == op.graph
-  assert false_graph.outer_graph == op.graph
+  assert true_graph.outer_graph == if_op.graph
+  assert false_graph.outer_graph == if_op.graph
 
   # Create grad functions that compute the gradient of the true/false forward
   # graphs. These functions will capture tensors from the forward pass
@@ -140,11 +142,12 @@ def _IfGrad(op, *grads):  # pylint: disable=invalid-name
     true_graph.name += "_rewritten"
     false_graph.name += "_rewritten"
 
-    op._set_func_attr("then_branch", util.create_new_tf_function(true_graph))
-    op._set_func_attr("else_branch", util.create_new_tf_function(false_graph))
-    op._set_type_list_attr("Tout", true_graph.output_types)
-    op._set_shape_list_attr("output_shapes", true_graph.output_shapes)
-    op._add_outputs(
+    if_op._set_func_attr("then_branch", util.create_new_tf_function(true_graph))
+    if_op._set_func_attr("else_branch",
+                         util.create_new_tf_function(false_graph))
+    if_op._set_type_list_attr("Tout", true_graph.output_types)
+    if_op._set_shape_list_attr("output_shapes", true_graph.output_shapes)
+    if_op._add_outputs(
         [t.dtype for t in extra_true_outputs],
         [t.shape for t in extra_true_outputs])
 
@@ -153,7 +156,7 @@ def _IfGrad(op, *grads):  # pylint: disable=invalid-name
   true_grad_inputs = _resolve_grad_inputs(true_graph, true_grad_graph)
   false_grad_inputs = _resolve_grad_inputs(false_graph, false_grad_graph)
 
-  outputs = _build_cond(op.inputs[0], true_grad_graph, false_grad_graph,
+  outputs = _build_cond(if_op.inputs[0], true_grad_graph, false_grad_graph,
                         true_grad_inputs, false_grad_inputs)
 
   # The predicate has no gradient.
-- 
GitLab


From d0cc3477ff5e3d9dd71abb2656a2a2b5fb39d04c Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Thu, 17 Jan 2019 11:03:31 -0800
Subject: [PATCH 0864/2345] TPUEstimator: Add export_to_cpu option to
 enable/disable CPU export.

Not all TPU ops are supported on the CPU (in particular some bfloat16
operations).  These models will fail to export on the CPU.  This CL adds a flag
to allow users to disable CPU export in this case.

PiperOrigin-RevId: 229780875
---
 .../contrib/tpu/python/tpu/tpu_estimator.py   | 39 ++++++++++++-------
 1 file changed, 26 insertions(+), 13 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 79340a9f21..022ac4152d 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -2143,6 +2143,7 @@ class TPUEstimator(estimator_lib.Estimator):
                batch_axis=None,
                eval_on_tpu=True,
                export_to_tpu=True,
+               export_to_cpu=True,
                warm_start_from=None,
                experimental_exported_model_uses_all_cores=False):
     """Constructs an `TPUEstimator` instance.
@@ -2187,9 +2188,11 @@ class TPUEstimator(estimator_lib.Estimator):
       eval_on_tpu: If False, evaluation runs on CPU or GPU. In this case, the
         model_fn must return `EstimatorSpec` when called with `mode` as `EVAL`.
       export_to_tpu: If True, `export_savedmodel()` exports a metagraph for
-        serving on TPU besides the one on CPU. Note that unsupported export
-        modes such as EVAL will be ignored. For those modes, only a CPU model
-        will be exported. Currently, export_to_tpu only supports PREDICT.
+        serving on TPU. Note that unsupported export modes such as EVAL will be
+        ignored. For those modes, only a CPU model will be exported.
+        Currently, export_to_tpu only supports PREDICT.
+      export_to_cpu: If True, `export_savedmodel()` exports a metagraph for
+        serving on CPU.
       warm_start_from: Optional string filepath to a checkpoint or SavedModel to
         warm-start from, or a `tf.estimator.WarmStartSettings` object to fully
         configure warm-starting.  If the string filepath is provided instead of
@@ -2264,6 +2267,7 @@ class TPUEstimator(estimator_lib.Estimator):
         self._config, train_batch_size, eval_batch_size, predict_batch_size,
         use_tpu, eval_on_tpu)
 
+    self._export_to_cpu = export_to_cpu
     self._export_to_tpu = export_to_tpu
     self._experimental_exported_model_uses_all_cores = (
         experimental_exported_model_uses_all_cores)
@@ -2284,14 +2288,18 @@ class TPUEstimator(estimator_lib.Estimator):
                       'when `export_to_tpu` is `True`; Mode {} will be ignored '
                       'for TPU.'.format(mode))
 
-    (super(TPUEstimator, self)._add_meta_graph_for_mode(
-        builder,
-        input_receiver_fn_map,
-        checkpoint_path,
-        save_variables,
-        mode=mode,
-        export_tags=export_tags,
-        check_variables=check_variables))
+    if not self._export_to_cpu and not self._export_to_tpu:
+      raise ValueError('One of export_to_cpu and export_to_tpu must be true.')
+
+    if self._export_to_cpu:
+      (super(TPUEstimator, self)._add_meta_graph_for_mode(
+          builder,
+          input_receiver_fn_map,
+          checkpoint_path,
+          save_variables,
+          mode=mode,
+          export_tags=export_tags,
+          check_variables=check_variables))
 
     if self._export_to_tpu and mode == model_fn_lib.ModeKeys.PREDICT:
       input_receiver_fn_map = {
@@ -2299,15 +2307,20 @@ class TPUEstimator(estimator_lib.Estimator):
       }
       export_tags = [tag_constants.SERVING, tag_constants.TPU]
       mode = _REWRITE_FOR_INFERENCE_MODE
+
       # See b/110052256 for why `check_variables` is `False`.
+      if not self._export_to_cpu:
+        check_variables = save_variables = True
+      else:
+        check_variables = save_variables = False
       (super(TPUEstimator, self)._add_meta_graph_for_mode(
           builder,
           input_receiver_fn_map,
           checkpoint_path,
-          save_variables=False,
+          save_variables=save_variables,
           mode=mode,
           export_tags=export_tags,
-          check_variables=False))
+          check_variables=check_variables))
 
   def _call_model_fn(self, features, labels, mode, config):
     if mode == _REWRITE_FOR_INFERENCE_MODE:
-- 
GitLab


From 568b9e56038d2d2fe8927f9e0b538bf5e49116f6 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Thu, 17 Jan 2019 11:03:53 -0800
Subject: [PATCH 0865/2345] Fix Toco IdentifyL2Normalization bugs

PiperOrigin-RevId: 229780959
---
 tensorflow/lite/build_def.bzl                 |  1 +
 tensorflow/lite/testing/generate_examples.py  | 30 +++++++++++++++++++
 .../identify_l2_normalization.cc              | 16 +++-------
 tensorflow/lite/toco/tooling_util.cc          |  2 +-
 tensorflow/lite/toco/tooling_util.h           |  2 +-
 5 files changed, 37 insertions(+), 14 deletions(-)

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index 3c48b655f7..7e70503386 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -253,6 +253,7 @@ def generated_test_models():
         "greater_equal",
         "sum",
         "l2norm",
+        "l2norm_shared_epsilon",
         "l2_pool",
         "leaky_relu",
         "less",
diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index d2c275f787..12b5a8b210 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -1424,6 +1424,36 @@ def make_conv_tests(zip_path):
   make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
 
 
+# Note: This is a regression test for a bug (b/122651451) that Toco incorrectly
+# erases the reduction indices array while it's shared with other ops.
+def make_l2norm_shared_epsilon_tests(zip_path):
+  """Regression test for a bug (b/122651451)."""
+
+  # Chose a set of parameters
+  test_parameters = [{
+      "input_shape": [[5, 7]],
+      "dim": [1],
+      "epsilon": [1e-8],
+  }]
+
+  def build_graph(parameters):
+    input_tensor = tf.placeholder(
+        dtype=tf.float32, name="input", shape=parameters["input_shape"])
+    epsilon = tf.constant(parameters["epsilon"])
+    out1 = tf.nn.l2_normalize(input_tensor, parameters["dim"], epsilon=epsilon)
+    out2 = tf.nn.l2_normalize(input_tensor, parameters["dim"], epsilon=epsilon)
+    out = out1 + out2
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_values = create_tensor_data(
+        np.float32, parameters["input_shape"], min_value=-4, max_value=10)
+    return [input_values], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_values])))
+
+  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+
+
 # Note: This is a regression test for a bug (b/112436267) that Toco incorrectly
 # fuses weights when multiple Conv2D/FULLY_CONNECTED ops share the same constant
 # weight tensor.
diff --git a/tensorflow/lite/toco/graph_transformations/identify_l2_normalization.cc b/tensorflow/lite/toco/graph_transformations/identify_l2_normalization.cc
index dabd4bd209..3b7c88ac62 100644
--- a/tensorflow/lite/toco/graph_transformations/identify_l2_normalization.cc
+++ b/tensorflow/lite/toco/graph_transformations/identify_l2_normalization.cc
@@ -151,20 +151,12 @@ std::vector<std::unique_ptr<Operator>>::iterator FindOperator(
 
   // Erase the subgraph that is now replaced by L2Normalization
   model->operators.erase(FindOperator(model, square_op));
-  model->EraseArray(sum_op->inputs[0]);
-  if (sum_op->inputs.size() > 1) {
-    model->EraseArray(sum_op->inputs[1]);
-  }
-  model->operators.erase(FindOperator(model, sum_op));
+  DeleteOpAndArraysIfUnused(model, sum_op);
   if (add_op) {
-    model->EraseArray(add_op->inputs[0]);
-    model->EraseArray(add_op->inputs[1]);
-    model->operators.erase(FindOperator(model, add_op));
+    DeleteOpAndArraysIfUnused(model, add_op);
   }
-  model->EraseArray(sqrt_or_rsqrt_op->inputs[0]);
-  model->operators.erase(FindOperator(model, sqrt_or_rsqrt_op));
-  model->EraseArray(div_or_mul_op->inputs[1]);
-  model->operators.erase(FindOperator(model, div_or_mul_op));
+  DeleteOpAndArraysIfUnused(model, sqrt_or_rsqrt_op);
+  DeleteOpAndArraysIfUnused(model, div_or_mul_op);
   *modified = true;
   return ::tensorflow::Status::OK();
 }
diff --git a/tensorflow/lite/toco/tooling_util.cc b/tensorflow/lite/toco/tooling_util.cc
index 9c0e8067d8..2c98ff1d92 100644
--- a/tensorflow/lite/toco/tooling_util.cc
+++ b/tensorflow/lite/toco/tooling_util.cc
@@ -173,7 +173,7 @@ bool DeleteArrayIfUsedOnce(const string& array_name, Model* model) {
   return false;
 }
 
-void DeleteOpAndArraysIfUnused(Model* model, Operator* op) {
+void DeleteOpAndArraysIfUnused(Model* model, const Operator* op) {
   for (const string& array_name : op->inputs) {
     DeleteArrayIfUsedOnce(array_name, model);
   }
diff --git a/tensorflow/lite/toco/tooling_util.h b/tensorflow/lite/toco/tooling_util.h
index 53131824b5..517da784d0 100644
--- a/tensorflow/lite/toco/tooling_util.h
+++ b/tensorflow/lite/toco/tooling_util.h
@@ -72,7 +72,7 @@ bool DeleteArrayIfUsedOnce(const string& array_name, Model* model);
 
 // Deletes the op and any of its input and output arrays if they are unused
 // after the op has been deleted.
-void DeleteOpAndArraysIfUnused(Model* model, Operator* op);
+void DeleteOpAndArraysIfUnused(Model* model, const Operator* op);
 
 std::vector<std::unique_ptr<Operator>>::const_iterator FindOpWithOutput(
     const Model& model, const string& array_name);
-- 
GitLab


From 2db14a0ffcb3cd46211a0da8baae2be2afe02b45 Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Thu, 17 Jan 2019 11:06:51 -0800
Subject: [PATCH 0866/2345] [Grappler] Add validation in
 MutableGraphView.DeleteNodes().

PiperOrigin-RevId: 229781551
---
 tensorflow/core/grappler/BUILD                |   1 +
 .../core/grappler/mutable_graph_view.cc       | 132 +++++++++--
 tensorflow/core/grappler/mutable_graph_view.h |  43 +++-
 .../core/grappler/mutable_graph_view_test.cc  | 218 ++++++++++++++----
 .../core/grappler/optimizers/data/BUILD       |  10 +
 .../grappler/optimizers/data/filter_fusion.cc |   5 +-
 .../optimizers/data/graph_utils_test.cc       |  14 +-
 .../optimizers/data/hoist_random_uniform.cc   |   5 +-
 .../optimizers/data/make_numa_aware.cc        |   5 +-
 .../optimizers/data/map_and_batch_fusion.cc   |   5 +-
 .../optimizers/data/map_and_filter_fusion.cc  |   5 +-
 .../grappler/optimizers/data/map_fusion.cc    |   5 +-
 .../optimizers/data/map_parallelization.cc    |   5 +-
 .../optimizers/data/map_vectorization.cc      |   5 +-
 .../optimizers/data/noop_elimination.cc       |   5 +-
 .../data/shuffle_and_repeat_fusion.cc         |   5 +-
 16 files changed, 379 insertions(+), 89 deletions(-)

diff --git a/tensorflow/core/grappler/BUILD b/tensorflow/core/grappler/BUILD
index bd95e89878..d5266247cf 100644
--- a/tensorflow/core/grappler/BUILD
+++ b/tensorflow/core/grappler/BUILD
@@ -211,6 +211,7 @@ cc_library(
         ":utils",
         "//tensorflow/core:graph",
         "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
         "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/strings",
diff --git a/tensorflow/core/grappler/mutable_graph_view.cc b/tensorflow/core/grappler/mutable_graph_view.cc
index 2ba53a685d..e31eedf166 100644
--- a/tensorflow/core/grappler/mutable_graph_view.cc
+++ b/tensorflow/core/grappler/mutable_graph_view.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include <utility>
 
 #include "absl/strings/str_cat.h"
+#include "absl/strings/str_join.h"
 #include "absl/strings/string_view.h"
 #include "absl/strings/substitute.h"
 #include "tensorflow/core/framework/graph.pb.h"
@@ -29,6 +30,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/utils.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
+#include "tensorflow/core/lib/gtl/map_util.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
@@ -134,7 +136,8 @@ void MutableGraphView::AddAndDedupFanouts(NodeDef* node) {
                               controlling_fanins.contains(input_node_name)));
     if (!gtl::InsertIfNotPresent(&fanins, input_node_name) &&
         can_dedup_control) {
-      node->mutable_input()->SwapElements(pos, last_pos--);
+      node->mutable_input()->SwapElements(pos, last_pos);
+      --last_pos;
     } else {
       OutputPort output(nodes()[input_node_name], tensor_id.index());
 
@@ -506,10 +509,11 @@ bool MutableGraphView::RemoveRegularFaninInternal(NodeDef* node,
       auto fanouts_set = remove_input(fanin_port, i, /*update_max_port=*/false);
       fanouts_set->insert({node, curr_pos});
       // Shift inputs to be retained.
-      mutable_inputs->SwapElements(i, curr_pos++);
+      mutable_inputs->SwapElements(i, curr_pos);
+      ++curr_pos;
     } else {
       // Skip inputs to be retained until first modification.
-      curr_pos++;
+      ++curr_pos;
     }
   }
 
@@ -717,15 +721,103 @@ Status MutableGraphView::UpdateFanin(absl::string_view node_name,
   return Status::OK();
 }
 
-void MutableGraphView::DeleteNodes(const std::set<string>& nodes_to_delete) {
-  // TODO(lyandy): Check if nodes have fanouts before deleting and return
-  // Status.
-  for (const string& node_name_to_delete : nodes_to_delete)
-    RemoveFaninsInternal(nodes().at(node_name_to_delete),
-                         /*keep_controlling_fanins=*/false);
-  for (const string& node_name_to_delete : nodes_to_delete)
+Status MutableGraphView::CheckNodesCanBeDeleted(
+    const absl::flat_hash_set<string>& nodes_to_delete) {
+  std::vector<string> missing_nodes;
+  std::vector<string> nodes_with_fanouts;
+  for (const string& node_name_to_delete : nodes_to_delete) {
+    NodeDef* node = GetNode(node_name_to_delete);
+    if (node == nullptr) {
+      // Can't delete missing node.
+      missing_nodes.push_back(node_name_to_delete);
+      continue;
+    }
+    const int max_port = gtl::FindWithDefault(max_regular_output_port(), node,
+                                              Graph::kControlSlot);
+    for (int i = Graph::kControlSlot; i <= max_port; ++i) {
+      auto it = fanouts().find({node, i});
+      bool has_retained_fanout = false;
+      if (it != fanouts().end()) {
+        for (const auto& fanout : it->second) {
+          // Check if fanouts are of nodes to be deleted, and if so, they can be
+          // ignored, as they will be removed also.
+          if (!nodes_to_delete.contains(fanout.node->name())) {
+            // Removing node will leave graph in an invalid state.
+            has_retained_fanout = true;
+            break;
+          }
+        }
+      }
+      if (has_retained_fanout) {
+        nodes_with_fanouts.push_back(node_name_to_delete);
+        break;
+      }
+    }
+  }
+
+  // Error message can get quite long, so we only show the first 5 node names.
+  auto sort_and_sample = [](std::vector<string>* s) {
+    constexpr int kMaxNodeNames = 5;
+    std::sort(s->begin(), s->end());
+    if (s->size() > kMaxNodeNames) {
+      return absl::StrCat(
+          absl::StrJoin(s->begin(), s->begin() + kMaxNodeNames, ", "), ", ...");
+    }
+    return absl::StrJoin(*s, ", ");
+  };
+
+  if (!missing_nodes.empty()) {
+    VLOG(1) << absl::Substitute("Attempting to delete missing node(s) [$0]",
+                                sort_and_sample(&missing_nodes));
+  }
+  if (!nodes_with_fanouts.empty()) {
+    std::vector<string> input_node_names(nodes_to_delete.begin(),
+                                         nodes_to_delete.end());
+    return errors::Internal(absl::Substitute(
+        "Can't delete node(s) with retained fanout(s) [$0] from node(s) [$1].",
+        sort_and_sample(&nodes_with_fanouts),
+        sort_and_sample(&input_node_names)));
+  }
+
+  return Status::OK();
+}
+
+Status MutableGraphView::DeleteNodes(
+    const absl::flat_hash_set<string>& nodes_to_delete) {
+  TF_RETURN_IF_ERROR(CheckNodesCanBeDeleted(nodes_to_delete));
+
+  // Find nodes in internal state and delete.
+  for (const string& node_name_to_delete : nodes_to_delete) {
+    NodeDef* node = GetNode(node_name_to_delete);
+    if (node != nullptr) {
+      RemoveFaninsInternal(node, /*keep_controlling_fanins=*/false);
+      RemoveFanoutsInternal(node);
+    }
+  }
+  for (const string& node_name_to_delete : nodes_to_delete) {
     nodes().erase(node_name_to_delete);
-  EraseNodesFromGraph(nodes_to_delete, graph());
+  }
+
+  // Find nodes in graph and delete by partitioning into nodes to retain and
+  // nodes to delete based on input set of nodes to delete by name.
+  // TODO(lyandy): Use a node name->idx hashmap if this is a performance
+  // bottleneck.
+  int pos = 0;
+  const int last_idx = graph()->node_size() - 1;
+  int last_pos = last_idx;
+  while (pos <= last_pos) {
+    if (nodes_to_delete.contains(graph()->node(pos).name())) {
+      graph()->mutable_node()->SwapElements(pos, last_pos);
+      --last_pos;
+    } else {
+      ++pos;
+    }
+  }
+  if (last_pos < last_idx) {
+    graph()->mutable_node()->DeleteSubrange(last_pos + 1, last_idx - last_pos);
+  }
+
+  return Status::OK();
 }
 
 void MutableGraphView::RemoveFaninsInternal(NodeDef* deleted_node,
@@ -741,10 +833,22 @@ void MutableGraphView::RemoveFaninsInternal(NodeDef* deleted_node,
     input.node = deleted_node;
     input.port_id = IsTensorIdControlling(tensor_id) ? Graph::kControlSlot : i;
 
-    absl::flat_hash_set<InputPort>* fanouts_set = &fanouts()[fanin];
-    fanouts_set->erase(input);
-    UpdateMaxRegularOutputPortForRemovedFanin(fanin, *fanouts_set);
+    auto it = fanouts().find(fanin);
+    if (it != fanouts().end()) {
+      absl::flat_hash_set<InputPort>* fanouts_set = &it->second;
+      fanouts_set->erase(input);
+      UpdateMaxRegularOutputPortForRemovedFanin(fanin, *fanouts_set);
+    }
+  }
+}
+
+void MutableGraphView::RemoveFanoutsInternal(NodeDef* deleted_node) {
+  const int max_port = gtl::FindWithDefault(max_regular_output_port(),
+                                            deleted_node, Graph::kControlSlot);
+  for (int i = Graph::kControlSlot; i <= max_port; ++i) {
+    fanouts().erase({deleted_node, i});
   }
+  max_regular_output_port().erase(deleted_node);
 }
 
 }  // end namespace grappler
diff --git a/tensorflow/core/grappler/mutable_graph_view.h b/tensorflow/core/grappler/mutable_graph_view.h
index 59740f2c71..c62129bcad 100644
--- a/tensorflow/core/grappler/mutable_graph_view.h
+++ b/tensorflow/core/grappler/mutable_graph_view.h
@@ -101,6 +101,10 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // end up with:
   //
   //   fanin -> Identity{N} -^> target node.
+  //
+  // If the control dependency being added is redundant (control dependency
+  // already exists or control dependency can be deduped from regular fanins),
+  // this will not result in an error and the node will not be modified.
   Status AddControllingFanin(absl::string_view node_name,
                              const TensorId& fanin);
 
@@ -108,29 +112,41 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // do not exist in the graph, nothing will be modified in the graph. If there
   // are multiple inputs that match the fanin, all of them will be removed. To
   // remove controlling fanins, use RemoveControllingFanin.
+  //
+  // If the fanin being removed doesn't exist in the node's inputs, this will
+  // not result in an error and the node will not be modified.
   Status RemoveRegularFanin(absl::string_view node_name, const TensorId& fanin);
 
   // Removes control dependency `fanin_node_name` from the target node named
   // `node_name`. If the node or fanin do not exist in the graph, nothing will
   // be modified in the graph. To remove regular fanins, use RemoveRegualrFanin.
+  //
+  // If the fanin being removed doesn't exist in the node's inputs, this will
+  // not result in an error and the node will not be modified.
   Status RemoveControllingFanin(absl::string_view node_name,
                                 absl::string_view fanin_node_name);
 
   // Removes all fanins from node `node_name`. Control dependencies will be
   // retained if keep_controlling_fanins is true.
+  //
+  // If no fanins are removed, this will not result in an error and the node
+  // will not be modified.
   Status RemoveAllFanins(absl::string_view node_name,
                          bool keep_controlling_fanins);
 
   // Replaces all fanins `from_fanin` with `to_fanin` in node `node_name`. If
   // the fanins or node do not exist, nothing will be modified in the graph.
   // Control dependencies will be deduped.
+  //
+  // If the fanin being updated doesn't exist in the node's inputs, this will
+  // not result in an error and the node will not be modified.
   Status UpdateFanin(absl::string_view node_name, const TensorId& from_fanin,
                      const TensorId& to_fanin);
 
-  // Deletes nodes from the graph.
-  // TODO(lyandy): Check if nodes have fanouts before deleting and return
-  // Status.
-  void DeleteNodes(const std::set<string>& nodes_to_delete);
+  // Deletes nodes from the graph. If a node can't be safely removed,
+  // specifically if a node still has fanouts, an error will be returned. Nodes
+  // that can't be found are ignored.
+  Status DeleteNodes(const absl::flat_hash_set<string>& nodes_to_delete);
 
  private:
   // Adds fanouts for fanins of node to graph, while deduping control
@@ -163,11 +179,6 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // behavior is undefined.
   Status UpdateFanoutsInternal(NodeDef* from_node, NodeDef* to_node);
 
-  // Removes fanins of the deleted node from internal state. Control
-  // dependencies are retained iff keep_controlling_fanins is true.
-  void RemoveFaninsInternal(NodeDef* deleted_node,
-                            bool keep_controlling_fanins);
-
   // Adds fanin to node. If fanin is a control dependency, existing control
   // dependencies will be checked first before adding. Otherwise fanin will be
   // added after existing non control dependency inputs.
@@ -179,6 +190,20 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // Removes controlling fanin `fanin_node` from node if such controlling fanin
   // exists.
   bool RemoveControllingFaninInternal(NodeDef* node, NodeDef* fanin_node);
+
+  // Checks if nodes to be deleted are missing or have any fanouts that will
+  // remain in the graph. If node is removed in either case, the graph will
+  // enter an invalid state.
+  Status CheckNodesCanBeDeleted(
+      const absl::flat_hash_set<string>& nodes_to_delete);
+
+  // Removes fanins of the deleted node from internal state. Control
+  // dependencies are retained iff keep_controlling_fanins is true.
+  void RemoveFaninsInternal(NodeDef* deleted_node,
+                            bool keep_controlling_fanins);
+
+  // Removes fanouts of the deleted node from internal state.
+  void RemoveFanoutsInternal(NodeDef* deleted_node);
 };
 
 }  // end namespace grappler
diff --git a/tensorflow/core/grappler/mutable_graph_view_test.cc b/tensorflow/core/grappler/mutable_graph_view_test.cc
index b5bf33033c..acfaba5ddd 100644
--- a/tensorflow/core/grappler/mutable_graph_view_test.cc
+++ b/tensorflow/core/grappler/mutable_graph_view_test.cc
@@ -23,6 +23,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/inputs/trivial_test_graph_input_yielder.h"
 #include "tensorflow/core/grappler/utils.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/platform/test.h"
 
 namespace tensorflow {
@@ -154,7 +155,7 @@ TEST(MutableGraphViewTest, AddAndUpdateFanouts) {
 
   NodeDef* new_bar = graph.AddNode(NDef("new_bar", "NotImportant", {}, {}));
 
-  EXPECT_TRUE(graph.UpdateFanouts("bar", new_bar->name()).ok());
+  TF_EXPECT_OK(graph.UpdateFanouts("bar", new_bar->name()));
 
   // Fanins and fanouts must be updated.
   CheckNode(graph, "bar", "NotImportant", "", {}, {}, {});
@@ -184,7 +185,7 @@ TEST(MutableGraphViewTest, AddAndUpdateFanoutsKeepControls) {
 
   NodeDef* new_bar = graph.AddNode(NDef("new_bar", "Identity", {"bar_1:2"}));
 
-  EXPECT_TRUE(graph.UpdateFanouts("bar_2", new_bar->name()).ok());
+  TF_EXPECT_OK(graph.UpdateFanouts("bar_2", new_bar->name()));
 
   // Fanins and fanouts must be updated.
   CheckNode(graph, "bar_1", "Switch", "", {}, {}, {"bar_2", "new_bar"});
@@ -213,7 +214,7 @@ TEST(MutableGraphViewTest, AddAndUpdateFanoutsWithoutSelfLoops) {
   // `new_bar` reads the output of an original `bar` node.
   NodeDef* new_bar = graph.AddNode(NDef("new_bar", "NewBar", {"bar"}, {}));
 
-  EXPECT_TRUE(graph.UpdateFanouts("bar", new_bar->name()).ok());
+  TF_EXPECT_OK(graph.UpdateFanouts("bar", new_bar->name()));
 
   // Fanins and fanouts must be updated.
   CheckNode(graph, "bar", "NotImportant", "", {}, {}, {"new_bar"});
@@ -266,7 +267,7 @@ TEST(MutableGraphViewTest, UpdateFanoutsToSwitchWithNoControlFromSwitch) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.UpdateFanouts("c", "b").ok());
+  TF_EXPECT_OK(graph.UpdateFanouts("c", "b"));
 
   EXPECT_EQ(graph.graph()->node_size(), 5);
 
@@ -882,10 +883,10 @@ TEST(MutableGraphViewTest, DedupControllingFaninsOnAddFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddRegularFanin("b", {"a", 2}).ok());
+  TF_EXPECT_OK(graph.AddRegularFanin("b", {"a", 2}));
   CheckNode(graph, "b", "NotImportant", "", {}, {"a:2"}, {});
 
-  EXPECT_TRUE(graph.AddControllingFanin("c", {"a", Graph::kControlSlot}).ok());
+  TF_EXPECT_OK(graph.AddControllingFanin("c", {"a", Graph::kControlSlot}));
   CheckNode(graph, "c", "NotImportant", "", {}, {"a:1"}, {});
 
   CheckNode(graph, "a", "NotImportant", "", {}, {}, {"b:0", "c:0"});
@@ -901,16 +902,16 @@ TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnAddFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddRegularFanin("c", {"b", 2}).ok());
+  TF_EXPECT_OK(graph.AddRegularFanin("c", {"b", 2}));
   CheckNode(graph, "c", "", "", {}, {"b:2"}, {});
-  EXPECT_TRUE(graph.AddControllingFanin("c", {"b", Graph::kControlSlot}).ok());
+  TF_EXPECT_OK(graph.AddControllingFanin("c", {"b", Graph::kControlSlot}));
   CheckNode(graph, "c", "", "", {}, {"b:2", "^b"}, {});
-  EXPECT_TRUE(graph.AddControllingFanin("c", {"b", Graph::kControlSlot}).ok());
+  TF_EXPECT_OK(graph.AddControllingFanin("c", {"b", Graph::kControlSlot}));
   CheckNode(graph, "c", "", "", {}, {"b:2", "^b"}, {});
 
-  EXPECT_TRUE(graph.AddControllingFanin("d", {"b", Graph::kControlSlot}).ok());
+  TF_EXPECT_OK(graph.AddControllingFanin("d", {"b", Graph::kControlSlot}));
   CheckNode(graph, "d", "", "", {}, {"^b"}, {});
-  EXPECT_TRUE(graph.AddControllingFanin("d", {"b", Graph::kControlSlot}).ok());
+  TF_EXPECT_OK(graph.AddControllingFanin("d", {"b", Graph::kControlSlot}));
   CheckNode(graph, "d", "", "", {}, {"^b"}, {});
 
   CheckGraph(graph);
@@ -925,7 +926,7 @@ TEST(MutableGraphViewTest, DedupControllingFaninsOnUpdateFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.UpdateFanin("c", {"a", 1}, {"b", 2}).ok());
+  TF_EXPECT_OK(graph.UpdateFanin("c", {"a", 1}, {"b", 2}));
 
   CheckNode(graph, "a", "NotImportant", "", {}, {}, {});
   CheckNode(graph, "b", "NotImportant", "", {}, {}, {"c"});
@@ -943,17 +944,14 @@ TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnUpdateFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph
-                  .UpdateFanin("d", {"b", Graph::kControlSlot},
-                               {"c", Graph::kControlSlot})
-                  .ok());
+  TF_EXPECT_OK(graph.UpdateFanin("d", {"b", Graph::kControlSlot},
+                                 {"c", Graph::kControlSlot}));
   CheckNode(graph, "d", "NotImportant", "", {}, {"c", "^c"}, {});
 
-  EXPECT_TRUE(graph.UpdateFanin("e", {"b", 0}, {"c", 3}).ok());
+  TF_EXPECT_OK(graph.UpdateFanin("e", {"b", 0}, {"c", 3}));
   CheckNode(graph, "e", "NotImportant", "", {}, {"c:3", "^c"}, {});
 
-  EXPECT_TRUE(
-      graph.UpdateFanin("e", {"c", 3}, {"c", Graph::kControlSlot}).ok());
+  TF_EXPECT_OK(graph.UpdateFanin("e", {"c", 3}, {"c", Graph::kControlSlot}));
   CheckNode(graph, "e", "NotImportant", "", {}, {"^c"}, {});
 
   CheckGraph(graph);
@@ -968,7 +966,7 @@ TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnAddFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddRegularFanin("c", {"a", 3}).ok());
+  TF_EXPECT_OK(graph.AddRegularFanin("c", {"a", 3}));
 
   CheckNode(graph, "a", "NotImportant", "", {}, {}, {"b", "c"});
   CheckNode(graph, "b", "NotImportant", "", {}, {"a:1"}, {"^c"});
@@ -986,7 +984,7 @@ TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnRemoveFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.RemoveRegularFanin("c", {"a", 2}).ok());
+  TF_EXPECT_OK(graph.RemoveRegularFanin("c", {"a", 2}));
   CheckNode(graph, "a", "NotImportant", "", {}, {}, {"b"});
   CheckNode(graph, "b", "NotImportant", "", {}, {"a:1"}, {});
   CheckNode(graph, "c", "NotImportant", "", {}, {}, {});
@@ -1003,7 +1001,7 @@ TEST(MutableGraphViewTest, KeepMaxRegularOutputPortOnRemoveFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.RemoveRegularFanin("b", {"a", 1}).ok());
+  TF_EXPECT_OK(graph.RemoveRegularFanin("b", {"a", 1}));
 
   CheckNode(graph, "a", "NotImportant", "", {}, {}, {"c"});
   CheckNode(graph, "b", "NotImportant", "", {}, {}, {});
@@ -1021,7 +1019,7 @@ TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnUpdateFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.UpdateFanin("c", {"a", 2}, {"b", 3}).ok());
+  TF_EXPECT_OK(graph.UpdateFanin("c", {"a", 2}, {"b", 3}));
 
   CheckNode(graph, "a", "NotImportant", "", {}, {}, {"b"});
   CheckNode(graph, "b", "NotImportant", "", {}, {"a:1"}, {"c"});
@@ -1069,8 +1067,8 @@ TEST(MutableGraphViewTest, AddControllingFaninExistingControl) {
       /*funcs=*/{});
 
   MutableGraphView graph(&graph_def);
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"b", Graph::kControlSlot}).ok());
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"b", Graph::kControlSlot}).ok());
+  TF_EXPECT_OK(graph.AddControllingFanin("a", {"b", Graph::kControlSlot}));
+  TF_EXPECT_OK(graph.AddControllingFanin("a", {"b", Graph::kControlSlot}));
 
   ASSERT_EQ(graph.graph()->node_size(), 2);
 
@@ -1087,8 +1085,8 @@ TEST(MutableGraphViewTest, AddControllingFaninNotSwitch) {
       /*funcs=*/{});
 
   MutableGraphView graph(&graph_def);
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"b", 2}).ok());
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"b", 2}).ok());
+  TF_EXPECT_OK(graph.AddControllingFanin("a", {"b", 2}));
+  TF_EXPECT_OK(graph.AddControllingFanin("a", {"b", 2}));
 
   ASSERT_EQ(graph.graph()->node_size(), 2);
 
@@ -1127,8 +1125,8 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithIdentity) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
+  TF_EXPECT_OK(graph.AddControllingFanin("a", {"switch", 0}));
+  TF_EXPECT_OK(graph.AddControllingFanin("a", {"switch", 0}));
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
 
@@ -1148,8 +1146,8 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithNoExistingIdentity) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
+  TF_EXPECT_OK(graph.AddControllingFanin("a", {"switch", 0}));
+  TF_EXPECT_OK(graph.AddControllingFanin("a", {"switch", 0}));
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
 
@@ -1171,8 +1169,8 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithExistingAddedIdentity) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
+  TF_EXPECT_OK(graph.AddControllingFanin("a", {"switch", 0}));
+  TF_EXPECT_OK(graph.AddControllingFanin("a", {"switch", 0}));
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
 
@@ -1279,7 +1277,7 @@ TEST(MutableGraphViewTest, RemoveControllingFaninMissing) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.RemoveControllingFanin("d", "c").ok());
+  TF_EXPECT_OK(graph.RemoveControllingFanin("d", "c"));
 
   ASSERT_EQ(graph.graph()->node_size(), 4);
 
@@ -1301,8 +1299,8 @@ TEST(MutableGraphViewTest, RemoveControllingFaninExisting) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.RemoveControllingFanin("d", "a").ok());
-  EXPECT_TRUE(graph.RemoveControllingFanin("d", "a").ok());
+  TF_EXPECT_OK(graph.RemoveControllingFanin("d", "a"));
+  TF_EXPECT_OK(graph.RemoveControllingFanin("d", "a"));
 
   ASSERT_EQ(graph.graph()->node_size(), 4);
 
@@ -1323,8 +1321,8 @@ TEST(MutableGraphViewTest, RemoveControllingFaninOnRegularFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.RemoveControllingFanin("c", "a").ok());
-  EXPECT_TRUE(graph.RemoveControllingFanin("c", "b").ok());
+  TF_EXPECT_OK(graph.RemoveControllingFanin("c", "a"));
+  TF_EXPECT_OK(graph.RemoveControllingFanin("c", "b"));
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
 
@@ -1370,7 +1368,7 @@ TEST(MutableGraphViewTest, DeleteNodes) {
   MutableGraphView graph(&graph_def);
 
   EXPECT_NE(graph.GetNode("foo_1"), nullptr);
-  graph.DeleteNodes({"foo_1"});
+  TF_EXPECT_OK(graph.DeleteNodes({"foo_1"}));
 
   EXPECT_EQ(graph.graph()->node_size(), 3);
   EXPECT_EQ(graph.GetNode("foo_1"), nullptr);
@@ -1382,6 +1380,148 @@ TEST(MutableGraphViewTest, DeleteNodes) {
   CheckGraph(graph);
 }
 
+GraphDef SimpleDeleteNodeGraph() {
+  // Actual node.op() is not important in this test.
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {"a:2"}),
+       NDef("c", "NotImportant", {"a:5", "^b"}), NDef("d", "NotImportant", {}),
+       NDef("e", "NotImportant", {"d:2"}),
+       NDef("f", "NotImportant", {"d:3", "^e"})},
+      /*funcs=*/{});
+  return graph_def;
+}
+
+TEST(MutableGraphViewTest, DeleteNodesWithFanoutsBeingDeleted) {
+  GraphDef graph_def = SimpleDeleteNodeGraph();
+
+  MutableGraphView graph(&graph_def);
+  EXPECT_NE(graph.GetNode("a"), nullptr);
+  EXPECT_NE(graph.GetNode("b"), nullptr);
+  EXPECT_NE(graph.GetNode("c"), nullptr);
+  TF_EXPECT_OK(graph.DeleteNodes({"c", "a", "b"}));
+
+  EXPECT_EQ(graph.graph()->node_size(), 3);
+  EXPECT_EQ(graph.GetNode("a"), nullptr);
+  EXPECT_EQ(graph.GetNode("b"), nullptr);
+  EXPECT_EQ(graph.GetNode("c"), nullptr);
+
+  CheckNode(graph, "d", "NotImportant", "", {}, {}, {"e", "f"});
+  CheckNode(graph, "e", "NotImportant", "", {}, {"d:2"}, {"^f"});
+  CheckNode(graph, "f", "NotImportant", "", {}, {"d:3", "^e"}, {});
+
+  CheckGraph(graph);
+}
+
+TEST(MutableGraphViewTest, DeleteMissingNodes) {
+  GraphDef graph_def = SimpleDeleteNodeGraph();
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_EQ(graph.GetNode("g"), nullptr);
+  EXPECT_EQ(graph.GetNode("h"), nullptr);
+  TF_EXPECT_OK(graph.DeleteNodes({"g", "h"}));
+
+  EXPECT_EQ(graph.graph()->node_size(), 6);
+  EXPECT_EQ(graph.GetNode("g"), nullptr);
+  EXPECT_EQ(graph.GetNode("h"), nullptr);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"b", "c"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {"a:2"}, {"^c"});
+  CheckNode(graph, "c", "NotImportant", "", {}, {"a:5", "^b"}, {});
+  CheckNode(graph, "d", "NotImportant", "", {}, {}, {"e", "f"});
+  CheckNode(graph, "e", "NotImportant", "", {}, {"d:2"}, {"^f"});
+  CheckNode(graph, "f", "NotImportant", "", {}, {"d:3", "^e"}, {});
+
+  CheckGraph(graph);
+}
+
+TEST(MutableGraphViewTest, DeleteMissingNodesAndNodesWithFanoutsBeingDeleted) {
+  GraphDef graph_def = SimpleDeleteNodeGraph();
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_NE(graph.GetNode("d"), nullptr);
+  EXPECT_NE(graph.GetNode("e"), nullptr);
+  EXPECT_NE(graph.GetNode("f"), nullptr);
+  TF_EXPECT_OK(graph.DeleteNodes({"d", "e", "f", "g", "h"}));
+
+  EXPECT_EQ(graph.graph()->node_size(), 3);
+  EXPECT_EQ(graph.GetNode("d"), nullptr);
+  EXPECT_EQ(graph.GetNode("e"), nullptr);
+  EXPECT_EQ(graph.GetNode("f"), nullptr);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"b", "c"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {"a:2"}, {"^c"});
+  CheckNode(graph, "c", "NotImportant", "", {}, {"a:5", "^b"}, {});
+
+  CheckGraph(graph);
+}
+
+TEST(MutableGraphViewTest, DeleteNodesWithError) {
+  GraphDef graph_def = SimpleDeleteNodeGraph();
+
+  MutableGraphView graph(&graph_def);
+
+  Status s = graph.DeleteNodes({"b", "a"});
+  EXPECT_FALSE(s.ok());
+  string error_msg =
+      "Can't delete node(s) with retained fanout(s) [a, b] from node(s) [a, "
+      "b].";
+  EXPECT_EQ(s.error_message(), error_msg);
+
+  EXPECT_EQ(graph.graph()->node_size(), 6);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"b", "c"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {"a:2"}, {"^c"});
+  CheckNode(graph, "c", "NotImportant", "", {}, {"a:5", "^b"}, {});
+  CheckNode(graph, "d", "NotImportant", "", {}, {}, {"e", "f"});
+  CheckNode(graph, "e", "NotImportant", "", {}, {"d:2"}, {"^f"});
+  CheckNode(graph, "f", "NotImportant", "", {}, {"d:3", "^e"}, {});
+
+  CheckGraph(graph);
+}
+
+TEST(MutableGraphViewTest, DeleteNodesWithLargeError) {
+  // Actual node.op() is not important in this test.
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {"a:2"}),
+       NDef("c", "NotImportant", {"^b"}), NDef("d", "NotImportant", {"c:6"}),
+       NDef("e", "NotImportant", {"d:2"}),
+       NDef("f", "NotImportant", {"d:3", "^e"}),
+       NDef("g", "NotImportant", {"f"}), NDef("h", "NotImportant", {"a"}),
+       NDef("i", "NotImportant", {"b"}), NDef("j", "NotImportant", {"c"}),
+       NDef("k", "NotImportant", {"d"}), NDef("l", "NotImportant", {"e"}),
+       NDef("m", "NotImportant", {"f"})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  Status s = graph.DeleteNodes({"a", "b", "c", "d", "e", "f"});
+  EXPECT_FALSE(s.ok());
+  string error_msg =
+      "Can't delete node(s) with retained fanout(s) [a, b, c, d, e, ...] from "
+      "node(s) [a, b, c, d, e, ...].";
+  EXPECT_EQ(s.error_message(), error_msg);
+
+  EXPECT_EQ(graph.graph()->node_size(), 13);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"b", "h"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {"a:2"}, {"^c", "i"});
+  CheckNode(graph, "c", "NotImportant", "", {}, {"^b"}, {"d", "j"});
+  CheckNode(graph, "d", "NotImportant", "", {}, {"c:6"}, {"e", "f", "k"});
+  CheckNode(graph, "e", "NotImportant", "", {}, {"d:2"}, {"^f", "l"});
+  CheckNode(graph, "f", "NotImportant", "", {}, {"d:3", "^e"}, {"g", "m"});
+  CheckNode(graph, "g", "NotImportant", "", {}, {"f"}, {});
+  CheckNode(graph, "h", "NotImportant", "", {}, {"a"}, {});
+  CheckNode(graph, "i", "NotImportant", "", {}, {"b"}, {});
+  CheckNode(graph, "j", "NotImportant", "", {}, {"c"}, {});
+  CheckNode(graph, "k", "NotImportant", "", {}, {"d"}, {});
+  CheckNode(graph, "l", "NotImportant", "", {}, {"e"}, {});
+  CheckNode(graph, "m", "NotImportant", "", {}, {"f"}, {});
+
+  CheckGraph(graph);
+}
+
 }  // namespace
 }  // namespace grappler
 }  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/BUILD b/tensorflow/core/grappler/optimizers/data/BUILD
index 413588b15c..fe59dfef15 100644
--- a/tensorflow/core/grappler/optimizers/data/BUILD
+++ b/tensorflow/core/grappler/optimizers/data/BUILD
@@ -38,6 +38,7 @@ cc_library(
         ":graph_utils",
         ":fusion_utils",
         ":optimizer_base",
+        "@com_google_absl//absl/container:flat_hash_set",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:grappler_item",
@@ -193,6 +194,7 @@ cc_library(
         ":function_utils",
         ":graph_utils",
         ":optimizer_base",
+        "@com_google_absl//absl/container:flat_hash_set",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
@@ -261,6 +263,7 @@ cc_library(
     deps = [
         ":graph_utils",
         ":optimizer_base",
+        "@com_google_absl//absl/container:flat_hash_set",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
@@ -324,6 +327,7 @@ cc_library(
     deps = [
         ":graph_utils",
         ":optimizer_base",
+        "@com_google_absl//absl/container:flat_hash_set",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
@@ -358,6 +362,7 @@ cc_library(
         ":graph_utils",
         ":fusion_utils",
         ":optimizer_base",
+        "@com_google_absl//absl/container:flat_hash_set",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
@@ -396,6 +401,7 @@ cc_library(
         ":graph_utils",
         ":fusion_utils",
         ":optimizer_base",
+        "@com_google_absl//absl/container:flat_hash_set",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:grappler_item",
@@ -434,6 +440,7 @@ cc_library(
     deps = [
         ":graph_utils",
         ":optimizer_base",
+        "@com_google_absl//absl/container:flat_hash_set",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
@@ -470,6 +477,7 @@ cc_library(
         ":graph_utils",
         ":optimizer_base",
         ":vectorization_utils",
+        "@com_google_absl//absl/container:flat_hash_set",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
@@ -526,6 +534,7 @@ cc_library(
     deps = [
         ":graph_utils",
         ":optimizer_base",
+        "@com_google_absl//absl/container:flat_hash_set",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
@@ -571,6 +580,7 @@ cc_library(
     deps = [
         ":graph_utils",
         ":optimizer_base",
+        "@com_google_absl//absl/container:flat_hash_set",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
diff --git a/tensorflow/core/grappler/optimizers/data/filter_fusion.cc b/tensorflow/core/grappler/optimizers/data/filter_fusion.cc
index 25dc10806b..7a20b8042b 100644
--- a/tensorflow/core/grappler/optimizers/data/filter_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/filter_fusion.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/data/filter_fusion.h"
 
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/grappler/clusters/cluster.h"
@@ -66,7 +67,7 @@ Status FilterFusion::OptimizeAndCollectStats(Cluster* cluster,
   *output = sorted_old_graph;
 
   MutableGraphView graph(output);
-  std::set<string> nodes_to_delete;
+  absl::flat_hash_set<string> nodes_to_delete;
   FunctionLibraryDefinition function_library(OpRegistry::Global(),
                                              output->library());
 
@@ -125,7 +126,7 @@ Status FilterFusion::OptimizeAndCollectStats(Cluster* cluster,
     stats->num_changes++;
   }
 
-  graph.DeleteNodes(nodes_to_delete);
+  TF_RETURN_IF_ERROR(graph.DeleteNodes(nodes_to_delete));
   return Status::OK();
 }
 
diff --git a/tensorflow/core/grappler/optimizers/data/graph_utils_test.cc b/tensorflow/core/grappler/optimizers/data/graph_utils_test.cc
index 5c0f03dca8..3b6d223fd3 100644
--- a/tensorflow/core/grappler/optimizers/data/graph_utils_test.cc
+++ b/tensorflow/core/grappler/optimizers/data/graph_utils_test.cc
@@ -109,7 +109,7 @@ TEST(GraphUtilsTest, ContainsGraphNodeWithName) {
   AddNode("A", "OpA", {}, {}, &graph);
   EXPECT_TRUE(ContainsGraphNodeWithName("A", *graph.graph()));
 
-  graph.DeleteNodes({"A"});
+  EXPECT_TRUE(graph.DeleteNodes({"A"}).ok());
   EXPECT_TRUE(!ContainsGraphNodeWithName("A", *graph.graph()));
 }
 
@@ -131,7 +131,7 @@ TEST(GraphUtilsTest, ContainsNodeWithOp) {
   AddNode("A", "OpA", {}, {}, &graph);
   EXPECT_TRUE(ContainsNodeWithOp("OpA", *graph.graph()));
 
-  graph.DeleteNodes({"A"});
+  EXPECT_TRUE(graph.DeleteNodes({"A"}).ok());
   EXPECT_TRUE(!ContainsNodeWithOp("OpA", *graph.graph()));
 }
 
@@ -143,7 +143,7 @@ TEST(GraphUtilsTest, FindGraphNodeWithName) {
   AddNode("A", "OpA", {}, {}, &graph);
   EXPECT_NE(FindGraphNodeWithName("A", *graph.graph()), -1);
 
-  graph.DeleteNodes({"A"});
+  EXPECT_TRUE(graph.DeleteNodes({"A"}).ok());
   EXPECT_EQ(FindGraphNodeWithName("A", *graph.graph()), -1);
 }
 
@@ -164,10 +164,10 @@ TEST(GraphUtilsTest, FindGraphNodeWithOp) {
 
   AddNode("A", "OpA", {}, {}, &graph);
   AddNode("B", "OpB", {"A"}, {}, &graph);
-  AddNode("A2", "OpA", {"B"}, {}, &graph);
+  AddNode("A2", "OpA", {"A"}, {}, &graph);
   EXPECT_EQ(FindGraphNodeWithOp("OpA", *graph.graph()), 0);
 
-  graph.DeleteNodes({"B"});
+  EXPECT_TRUE(graph.DeleteNodes({"B"}).ok());
   EXPECT_EQ(FindGraphNodeWithOp("OpB", *graph.graph()), -1);
   EXPECT_EQ(FindGraphNodeWithName("A2", *graph.graph()), 1);
 }
@@ -186,7 +186,7 @@ TEST(GraphUtilsTest, FindAllGraphNodesWithOp) {
   EXPECT_EQ(result_indices.at(0), 0);
   EXPECT_EQ(result_indices.at(1), 2);
 
-  graph.DeleteNodes({"A2"});
+  EXPECT_TRUE(graph.DeleteNodes({"A2"}).ok());
   std::vector<int> result_indices_new =
       FindAllGraphNodesWithOp("OpA", *graph.graph());
   EXPECT_EQ(result_indices_new.size(), 1);
@@ -201,7 +201,7 @@ TEST(GraphUtilsTest, SetUniqueGraphNodeName) {
   NodeDef* node2 = AddNode("", "A", {}, {}, &graph);
   EXPECT_NE(node1->name(), node2->name());
 
-  graph.DeleteNodes({node1->name()});
+  EXPECT_TRUE(graph.DeleteNodes({node1->name()}).ok());
   NodeDef* node3 = AddNode("", "A", {}, {}, &graph);
   EXPECT_NE(node2->name(), node3->name());
 }
diff --git a/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc b/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc
index 237e4b63e3..496277780f 100644
--- a/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc
+++ b/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/data/hoist_random_uniform.h"
 
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/op_def.pb.h"
@@ -227,7 +228,7 @@ Status HoistRandomUniform::OptimizeAndCollectStats(Cluster* cluster,
   *output = item.graph;
 
   MutableGraphView graph(output);
-  std::set<string> nodes_to_delete;
+  absl::flat_hash_set<string> nodes_to_delete;
   FunctionLibraryDefinition function_library(OpRegistry::Global(),
                                              item.graph.library());
 
@@ -277,7 +278,7 @@ Status HoistRandomUniform::OptimizeAndCollectStats(Cluster* cluster,
     stats->num_changes++;
   }
 
-  graph.DeleteNodes(nodes_to_delete);
+  TF_RETURN_IF_ERROR(graph.DeleteNodes(nodes_to_delete));
   return Status::OK();
 }
 
diff --git a/tensorflow/core/grappler/optimizers/data/make_numa_aware.cc b/tensorflow/core/grappler/optimizers/data/make_numa_aware.cc
index d8ea9adfb7..221f4c2525 100644
--- a/tensorflow/core/grappler/optimizers/data/make_numa_aware.cc
+++ b/tensorflow/core/grappler/optimizers/data/make_numa_aware.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/data/make_numa_aware.h"
 
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/grappler/clusters/cluster.h"
 #include "tensorflow/core/grappler/grappler_item.h"
@@ -43,7 +44,7 @@ Status MakeNumaAware::OptimizeAndCollectStats(Cluster* cluster,
                                               OptimizationStats* stats) {
   *output = item.graph;
   MutableGraphView graph(output);
-  std::set<string> nodes_to_delete;
+  absl::flat_hash_set<string> nodes_to_delete;
 
   for (const NodeDef& node : item.graph.node()) {
     if (node.op() != "ExperimentalMapAndBatchDataset") continue;
@@ -53,7 +54,7 @@ Status MakeNumaAware::OptimizeAndCollectStats(Cluster* cluster,
     nodes_to_delete.insert(node.name());
     stats->num_changes++;
   }
-  graph.DeleteNodes(nodes_to_delete);
+  TF_RETURN_IF_ERROR(graph.DeleteNodes(nodes_to_delete));
   return Status::OK();
 }
 
diff --git a/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc b/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc
index 3aeac021e6..5d26d1abe4 100644
--- a/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.h"
 
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/grappler/clusters/cluster.h"
@@ -104,7 +105,7 @@ Status MapAndBatchFusion::OptimizeAndCollectStats(Cluster* cluster,
                                                   OptimizationStats* stats) {
   *output = item.graph;
   MutableGraphView graph(output);
-  std::set<string> nodes_to_delete;
+  absl::flat_hash_set<string> nodes_to_delete;
   for (const NodeDef& node : item.graph.node()) {
     if (node.op() != "BatchDataset" && node.op() != "BatchDatasetV2") {
       continue;
@@ -131,7 +132,7 @@ Status MapAndBatchFusion::OptimizeAndCollectStats(Cluster* cluster,
     stats->num_changes++;
   }
 
-  graph.DeleteNodes(nodes_to_delete);
+  TF_RETURN_IF_ERROR(graph.DeleteNodes(nodes_to_delete));
   return Status::OK();
 }
 
diff --git a/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.cc b/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.cc
index 1ed4eae4ba..e257683b35 100644
--- a/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.h"
 
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/grappler/clusters/cluster.h"
@@ -103,7 +104,7 @@ Status MapAndFilterFusion::OptimizeAndCollectStats(Cluster* cluster,
   *output = sorted_old_graph;
 
   MutableGraphView graph(output);
-  std::set<string> nodes_to_delete;
+  absl::flat_hash_set<string> nodes_to_delete;
   FunctionLibraryDefinition function_library(OpRegistry::Global(),
                                              item.graph.library());
   auto get_map_node = [](const NodeDef& node) -> const NodeDef* {
@@ -168,7 +169,7 @@ Status MapAndFilterFusion::OptimizeAndCollectStats(Cluster* cluster,
     stats->num_changes++;
   }
 
-  graph.DeleteNodes(nodes_to_delete);
+  TF_RETURN_IF_ERROR(graph.DeleteNodes(nodes_to_delete));
   return Status::OK();
 }
 
diff --git a/tensorflow/core/grappler/optimizers/data/map_fusion.cc b/tensorflow/core/grappler/optimizers/data/map_fusion.cc
index 66f8404c05..ce41f7069c 100644
--- a/tensorflow/core/grappler/optimizers/data/map_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_fusion.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/data/map_fusion.h"
 
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/grappler/clusters/cluster.h"
@@ -86,7 +87,7 @@ Status MapFusion::OptimizeAndCollectStats(Cluster* cluster,
   *output = sorted_old_graph;
 
   MutableGraphView graph(output);
-  std::set<string> nodes_to_delete;
+  absl::flat_hash_set<string> nodes_to_delete;
   FunctionLibraryDefinition function_library(OpRegistry::Global(),
                                              item.graph.library());
 
@@ -147,7 +148,7 @@ Status MapFusion::OptimizeAndCollectStats(Cluster* cluster,
     stats->num_changes++;
   }
 
-  graph.DeleteNodes(nodes_to_delete);
+  TF_RETURN_IF_ERROR(graph.DeleteNodes(nodes_to_delete));
   return Status::OK();
 }
 
diff --git a/tensorflow/core/grappler/optimizers/data/map_parallelization.cc b/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
index d8f01d720a..57e541cde6 100644
--- a/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/data/map_parallelization.h"
 
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/grappler/clusters/cluster.h"
 #include "tensorflow/core/grappler/grappler_item.h"
@@ -73,7 +74,7 @@ Status MapParallelization::OptimizeAndCollectStats(Cluster* cluster,
                                                    OptimizationStats* stats) {
   *output = item.graph;
   MutableGraphView graph(output);
-  std::set<string> nodes_to_delete;
+  absl::flat_hash_set<string> nodes_to_delete;
   FunctionLibraryDefinition function_library(OpRegistry::Global(),
                                              item.graph.library());
   auto get_map_node = [](const NodeDef& node) -> const NodeDef* {
@@ -97,7 +98,7 @@ Status MapParallelization::OptimizeAndCollectStats(Cluster* cluster,
     stats->num_changes++;
   }
 
-  graph.DeleteNodes(nodes_to_delete);
+  TF_RETURN_IF_ERROR(graph.DeleteNodes(nodes_to_delete));
   return Status::OK();
 }
 
diff --git a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
index 848642041b..4c9e1f31e7 100644
--- a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/optimizers/data/map_vectorization.h"
 #include "tensorflow/core/grappler/optimizers/data/vectorization_utils.h"
 
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/tensor.pb.h"  // NOLINT
@@ -218,7 +219,7 @@ Status MapVectorization::OptimizeAndCollectStats(Cluster* cluster,
                                                  OptimizationStats* stats) {
   *output = item.graph;
   MutableGraphView graph(output);
-  std::set<string> nodes_to_delete;
+  absl::flat_hash_set<string> nodes_to_delete;
 
   for (const NodeDef& node : item.graph.node()) {
     // Find Map->Batch nodes.
@@ -274,7 +275,7 @@ Status MapVectorization::OptimizeAndCollectStats(Cluster* cluster,
     nodes_to_delete.insert(batch_node.name());
     stats->num_changes++;
   }
-  graph.DeleteNodes(nodes_to_delete);
+  TF_RETURN_IF_ERROR(graph.DeleteNodes(nodes_to_delete));
   return Status::OK();
 }
 
diff --git a/tensorflow/core/grappler/optimizers/data/noop_elimination.cc b/tensorflow/core/grappler/optimizers/data/noop_elimination.cc
index e474fddcc0..851bbbdc1a 100644
--- a/tensorflow/core/grappler/optimizers/data/noop_elimination.cc
+++ b/tensorflow/core/grappler/optimizers/data/noop_elimination.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/data/noop_elimination.h"
 
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/grappler/clusters/cluster.h"
@@ -76,7 +77,7 @@ Status NoOpElimination::OptimizeAndCollectStats(Cluster* cluster,
                                                 OptimizationStats* stats) {
   *output = item.graph;
   MutableGraphView graph(output);
-  std::set<string> nodes_to_delete;
+  absl::flat_hash_set<string> nodes_to_delete;
   for (const NodeDef& node : item.graph.node()) {
     if (!IsNoOp(node, graph)) continue;
 
@@ -87,7 +88,7 @@ Status NoOpElimination::OptimizeAndCollectStats(Cluster* cluster,
     stats->num_changes++;
   }
 
-  graph.DeleteNodes(nodes_to_delete);
+  TF_RETURN_IF_ERROR(graph.DeleteNodes(nodes_to_delete));
   return Status::OK();
 }
 
diff --git a/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.cc b/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.cc
index f30b2230ff..ff64ff1adb 100644
--- a/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.h"
 
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/grappler/clusters/cluster.h"
@@ -39,7 +40,7 @@ Status ShuffleAndRepeatFusion::OptimizeAndCollectStats(
     OptimizationStats* stats) {
   *output = item.graph;
   MutableGraphView graph(output);
-  std::set<string> nodes_to_delete;
+  absl::flat_hash_set<string> nodes_to_delete;
 
   auto make_shuffle_and_repeat_node = [&output](const NodeDef& shuffle_node,
                                                 const NodeDef& repeat_node) {
@@ -95,7 +96,7 @@ Status ShuffleAndRepeatFusion::OptimizeAndCollectStats(
     stats->num_changes++;
   }
 
-  graph.DeleteNodes(nodes_to_delete);
+  TF_RETURN_IF_ERROR(graph.DeleteNodes(nodes_to_delete));
   return Status::OK();
 }
 
-- 
GitLab


From fa016bec45651ecdd789dbb197bca65ea3ab80d4 Mon Sep 17 00:00:00 2001
From: Cong Liu <congliu@google.com>
Date: Thu, 17 Jan 2019 11:09:33 -0800
Subject: [PATCH 0867/2345] [XLA] Actually test the dynamic dimension inference
 for select-and-scatter.

PiperOrigin-RevId: 229782019
---
 .../dynamic_dimension_inference_test.cc       | 25 +++++++++++++++----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/xla/service/dynamic_dimension_inference_test.cc b/tensorflow/compiler/xla/service/dynamic_dimension_inference_test.cc
index 1dd196821c..b42e67b4bb 100644
--- a/tensorflow/compiler/xla/service/dynamic_dimension_inference_test.cc
+++ b/tensorflow/compiler/xla/service/dynamic_dimension_inference_test.cc
@@ -62,6 +62,17 @@ class DynamicDimensionInferenceTest : public HloTestBase {
     return module_->AddEmbeddedComputation(embedded_builder.Build());
   }
 
+  HloComputation* GetGe() {
+    auto embedded_builder = HloComputation::Builder("ge");
+    auto lhs = embedded_builder.AddInstruction(HloInstruction::CreateParameter(
+        0, ShapeUtil::MakeShape(F32, {}), "lhs"));
+    auto rhs = embedded_builder.AddInstruction(HloInstruction::CreateParameter(
+        1, ShapeUtil::MakeShape(F32, {}), "rhs"));
+    embedded_builder.AddInstruction(HloInstruction::CreateBinary(
+        ShapeUtil::MakeShape(PRED, {}), HloOpcode::kGe, lhs, rhs));
+    return module_->AddEmbeddedComputation(embedded_builder.Build());
+  }
+
   std::unique_ptr<HloModule> module_;
   std::unique_ptr<DynamicDimensionInference> inference_;
   const Shape scalar_shape_ = ShapeUtil::MakeShape(S32, {});
@@ -487,7 +498,7 @@ TEST_F(DynamicDimensionInferenceTest, SelectAndScatterTest) {
   // Test the ability to trace select and scatter batch dimensions.
   auto builder = HloComputation::Builder(TestName());
   auto input_shape = ShapeUtil::MakeShape(F32, {2, 4, 4});
-  auto output_shape = ShapeUtil::MakeShape(F32, {2, 2, 2});
+  auto source_shape = ShapeUtil::MakeShape(F32, {2, 2, 2});
 
   Window window;
   // First dimension is unchanged.
@@ -514,22 +525,26 @@ TEST_F(DynamicDimensionInferenceTest, SelectAndScatterTest) {
       /*parameter_number=*/0, input_shape, "A"));
   auto* size_param = builder.AddInstruction(HloInstruction::CreateParameter(
       /*parameter_number=*/1, scalar_shape_, "size_param"));
+  auto* source = builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/2, source_shape, "B"));
 
   auto init = builder.AddInstruction(
       HloInstruction::CreateConstant(LiteralUtil::CreateR0<float>(0.0)));
 
-  auto* reduce_window =
-      builder.AddInstruction(HloInstruction::CreateReduceWindow(
-          output_shape, a_param, init, window, GetAdd()));
+  auto* sns = builder.AddInstruction(HloInstruction::CreateSelectAndScatter(
+      input_shape, a_param, GetGe(), window, source, init, GetAdd()));
 
   module_->AddEntryComputation(builder.Build());
 
   TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
       DynamicParameterBinding::DynamicParameter{1, {}},
       DynamicParameterBinding::DynamicDimension{0, {}, 0}));
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{1, {}},
+      DynamicParameterBinding::DynamicDimension{2, {}, 0}));
 
   TF_ASSERT_OK(RunInference());
-  EXPECT_EQ(inference_->GetDynamicSize(reduce_window, {}, 0), size_param);
+  EXPECT_EQ(inference_->GetDynamicSize(sns, {}, 0), size_param);
 }
 
 }  // namespace
-- 
GitLab


From 7462a8fcd7c526a00fa62036cde53f5d989c5a45 Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Thu, 17 Jan 2019 11:49:03 -0800
Subject: [PATCH 0868/2345] [XLA] Remove the old HloInstruction
 CreateDS/CreateDUS interfaces.

Migrate all remaining use to thes new Span interfaces. Note that
this does not enforce using R0 indices - it's still possible to
pass a span with a single R1 element. That happens in practice
when going through the deprecated version of the XlaBuilder API.

PiperOrigin-RevId: 229790190
---
 .../xla/service/buffer_liveness_test.cc       |  8 +--
 .../cpu/cpu_instruction_fusion_test.cc        | 51 ++++++++-----------
 .../compiler/xla/service/hlo_instruction.cc   | 49 +++++-------------
 .../compiler/xla/service/hlo_instruction.h    | 10 ----
 .../service/tuple_points_to_analysis_test.cc  | 16 +++---
 tensorflow/compiler/xla/tests/fusion_test.cc  |  4 +-
 6 files changed, 47 insertions(+), 91 deletions(-)

diff --git a/tensorflow/compiler/xla/service/buffer_liveness_test.cc b/tensorflow/compiler/xla/service/buffer_liveness_test.cc
index 46218e6499..23b9af0281 100644
--- a/tensorflow/compiler/xla/service/buffer_liveness_test.cc
+++ b/tensorflow/compiler/xla/service/buffer_liveness_test.cc
@@ -638,10 +638,10 @@ class FusedDynamicUpdateSliceLivenessTest : public BufferLivenessTest {
     }
     // Create a DynamicUpdateSlice instruction of tuple element 1 with 'update'.
     auto starts = builder.AddInstruction(
-        HloInstruction::CreateConstant(LiteralUtil::CreateR1<int32>({2})));
+        HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(2)));
     auto dynamic_update_slice =
         builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
-            data_shape, gte1, update, starts));
+            data_shape, gte1, update, {starts}));
     // Create output tuple.
     builder.AddInstruction(
         HloInstruction::CreateTuple({gte0, dynamic_update_slice}));
@@ -794,10 +794,10 @@ class DynamicUpdateSliceLivenessTest : public BufferLivenessTest {
     }
     // Create a DynamicUpdateSlice instruction of tuple element 1 with 'update'.
     auto starts = builder.AddInstruction(
-        HloInstruction::CreateConstant(LiteralUtil::CreateR1<int32>({2})));
+        HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(2)));
     auto dynamic_update_slice =
         builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
-            data_shape, gte1, update, starts));
+            data_shape, gte1, update, {starts}));
     // Create output tuple.
     auto tuple_root = builder.AddInstruction(
         HloInstruction::CreateTuple({gte0, dynamic_update_slice}));
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc b/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc
index 86cc72dd7c..c4bde837e5 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc
@@ -390,10 +390,8 @@ TEST_F(OpcodeFusionTest, DynamicSlice_Negate) {
       HloInstruction::CreateParameter(0, param_shape, "param"));
   HloInstruction* param1 = builder.AddInstruction(
       HloInstruction::CreateParameter(1, slice_shape, "starts"));
-  HloInstruction* dynamic_slice2 =
-      builder.AddInstruction(HloInstruction::CreateDynamicSlice(
-          result_shape, param0,
-          std::initializer_list<HloInstruction*>({param1}), {2}));
+  HloInstruction* dynamic_slice2 = builder.AddInstruction(
+      HloInstruction::CreateDynamicSlice(result_shape, param0, {param1}, {2}));
   builder.AddInstruction(HloInstruction::CreateUnary(
       result_shape, HloOpcode::kNegate, dynamic_slice2));
 
@@ -591,49 +589,40 @@ TEST_F(OpcodeFusionTest, MessOfFusibleNodes) {
 
   Shape full_shape = ShapeUtil::MakeShape(F32, {4, 100, 10, 100, 50});
 
-  auto loop_idx = builder.AddInstruction(HloInstruction::CreateReshape(
-      ShapeUtil::MakeShape(S32, {1}),
-      builder.AddInstruction(HloInstruction::CreateParameter(
-          0, ShapeUtil::MakeShape(S32, {}), "param0"))));
-
+  auto loop_idx = builder.AddInstruction(HloInstruction::CreateParameter(
+      0, ShapeUtil::MakeShape(S32, {}), "param0"));
   auto param1 = builder.AddInstruction(HloInstruction::CreateParameter(
-      1, ShapeUtil::MakeShape(S32, {1}), "param1"));
-  auto concat = builder.AddInstruction(HloInstruction::CreateConcatenate(
-      ShapeUtil::MakeShape(S32, {5}),
-      {loop_idx, param1, param1, param1, param1}, /*dimension=*/0));
+      1, ShapeUtil::MakeShape(S32, {}), "param1"));
 
-  auto idx_choice = builder.AddInstruction(HloInstruction::CreateDynamicSlice(
-      ShapeUtil::MakeShape(S32, {1}),
-      builder.AddInstruction(HloInstruction::CreateParameter(
-          2, ShapeUtil::MakeShape(S32, {4}), "param2")),
-      loop_idx,
-      /*slice_sizes=*/{1}));
-
-  PaddingConfig padding_config;
-  padding_config.add_dimensions()->set_edge_padding_high(4);
-  auto pad = builder.AddInstruction(HloInstruction::CreatePad(
-      ShapeUtil::MakeShape(S32, {5}), idx_choice,
-      builder.AddInstruction(
-          HloInstruction::CreateConstant(LiteralUtil::CreateR0(0))),
-      padding_config));
+  auto idx_choice = builder.AddInstruction(HloInstruction::CreateReshape(
+      ShapeUtil::MakeShape(S32, {}),
+      builder.AddInstruction(HloInstruction::CreateDynamicSlice(
+          ShapeUtil::MakeShape(S32, {1}),
+          builder.AddInstruction(HloInstruction::CreateParameter(
+              2, ShapeUtil::MakeShape(S32, {4}), "param2")),
+          {loop_idx},
+          /*slice_sizes=*/{1}))));
+  auto zero = builder.AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0(0)));
 
   auto slice = builder.AddInstruction(HloInstruction::CreateDynamicSlice(
       ShapeUtil::MakeShape(F32, {1, 100, 10, 100, 50}),
       builder.AddInstruction(HloInstruction::CreateParameter(
           3, ShapeUtil::MakeShape(F32, {100, 100, 10, 100, 50}), "param3")),
-      pad, /*slice_sizes=*/{1, 100, 10, 100, 50}));
+      {idx_choice, zero, zero, zero, zero},
+      /*slice_sizes=*/{1, 100, 10, 100, 50}));
 
   builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
       full_shape,
       builder.AddInstruction(
           HloInstruction::CreateParameter(4, full_shape, "param4")),
-      slice, concat));
+      slice, {loop_idx, param1, param1, param1, param1}));
 
   module->AddEntryComputation(builder.Build());
   RunFusionAndCheckOpcodesWereFused(
       module.get(),
-      {HloOpcode::kConcatenate, HloOpcode::kPad, HloOpcode::kDynamicSlice,
-       HloOpcode::kDynamicSlice, HloOpcode::kDynamicUpdateSlice,
+      {HloOpcode::kDynamicSlice, HloOpcode::kDynamicSlice,
+       HloOpcode::kDynamicUpdateSlice, HloOpcode::kReshape,
        HloOpcode::kParameter, HloOpcode::kParameter, HloOpcode::kParameter,
        HloOpcode::kParameter, HloOpcode::kParameter, HloOpcode::kParameter});
 }
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index 029d170317..3c92554ad4 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -458,21 +458,17 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
           << "DynamicSlice instruction should have at least 1 operands but "
              "sees "
           << proto.operand_ids_size();
-      if (proto.operand_ids_size() == 2 && operands(1)->shape().rank() == 1) {
-        // TODO(b/118437727): Old form, remove this path.
-        instruction =
-            CreateDynamicSlice(shape, operands(0), operands(1), slice_sizes);
-      } else {
-        // New form
+      // TODO(b/118437727): Old form, make the check unconditional.
+      if (proto.operand_ids_size() != 2 || operands(1)->shape().rank() != 1) {
         auto expected_operands = 1 + operands(0)->shape().rank();
         TF_RET_CHECK(proto.operand_ids_size() == expected_operands)
             << "DynamicSlice instruction should have " << expected_operands
             << " operands, but has " << proto.operand_ids_size();
-        const auto& operand_vector = all_operands();
-        instruction = CreateDynamicSlice(
-            shape, operands(0), absl::MakeSpan(operand_vector).subspan(1),
-            slice_sizes);
       }
+      const auto& operand_vector = all_operands();
+      instruction = CreateDynamicSlice(
+          shape, operands(0), absl::MakeSpan(operand_vector).subspan(1),
+          slice_sizes);
       break;
     }
     case HloOpcode::kDynamicUpdateSlice: {
@@ -480,22 +476,19 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
           << "DynamicUpdateSlice instruction should have at least 2 operands "
              "but sees "
           << proto.operand_ids_size();
-      if (proto.operand_ids_size() == 3 && operands(2)->shape().rank() == 1) {
-        // TODO(b/118437727): Old form, remove this path.
-        instruction = CreateDynamicUpdateSlice(shape, operands(0), operands(1),
-                                               operands(2));
-      } else {
-        // New form
+      // TODO(b/118437727): Old form, make the check unconditional.
+      if (proto.operand_ids_size() != 3 || operands(2)->shape().rank() != 1) {
         auto expected_operands = 2 + operands(0)->shape().rank();
         TF_RET_CHECK(proto.operand_ids_size() == expected_operands)
             << "DynamicUpdateSlice instruction should have "
             << expected_operands << " operands, but has "
             << proto.operand_ids_size();
-        const auto& operand_vector = all_operands();
-        instruction =
-            CreateDynamicUpdateSlice(shape, operands(0), operands(1),
-                                     absl::MakeSpan(operand_vector).subspan(2));
       }
+      const auto& operand_vector = all_operands();
+      instruction =
+          CreateDynamicUpdateSlice(shape, operands(0), operands(1),
+                                   absl::MakeSpan(operand_vector).subspan(2));
+
       break;
     }
     case HloOpcode::kGather: {
@@ -947,13 +940,6 @@ HloInstruction::CreateAddDependency(HloInstruction* data_operand,
                                                 limit_indices, strides);
 }
 
-/* static */ std::unique_ptr<HloInstruction> HloInstruction::CreateDynamicSlice(
-    const Shape& shape, HloInstruction* operand, HloInstruction* start_indices,
-    absl::Span<const int64> slice_sizes) {
-  return absl::make_unique<HloDynamicSliceInstruction>(
-      shape, operand, start_indices, slice_sizes);
-}
-
 /* static */ std::unique_ptr<HloInstruction> HloInstruction::CreateDynamicSlice(
     const Shape& shape, HloInstruction* operand,
     absl::Span<HloInstruction* const> start_indices,
@@ -962,15 +948,6 @@ HloInstruction::CreateAddDependency(HloInstruction* data_operand,
       shape, operand, start_indices, slice_sizes);
 }
 
-/* static */ std::unique_ptr<HloInstruction>
-HloInstruction::CreateDynamicUpdateSlice(const Shape& shape,
-                                         HloInstruction* operand,
-                                         HloInstruction* update,
-                                         HloInstruction* start_indices) {
-  return absl::make_unique<HloDynamicUpdateSliceInstruction>(
-      shape, operand, update, start_indices);
-}
-
 /* static */ std::unique_ptr<HloInstruction>
 HloInstruction::CreateDynamicUpdateSlice(
     const Shape& shape, HloInstruction* operand, HloInstruction* update,
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h
index 789ace6a26..2c29b6c243 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.h
+++ b/tensorflow/compiler/xla/service/hlo_instruction.h
@@ -556,11 +556,6 @@ class HloInstruction {
   // Creates a slice instruction, where the first operand is sliced by
   // start indices specified in the second operand, and by size specified in
   // 'slice_sizes'.
-  ABSL_DEPRECATED("Use span-of-indices form instead")
-  static std::unique_ptr<HloInstruction> CreateDynamicSlice(
-      const Shape& shape, HloInstruction* operand,
-      HloInstruction* start_indices, absl::Span<const int64> slice_sizes);
-  // Same as above, but expects a span of scalar start indices.
   static std::unique_ptr<HloInstruction> CreateDynamicSlice(
       const Shape& shape, HloInstruction* operand,
       absl::Span<HloInstruction* const> start_indices,
@@ -568,11 +563,6 @@ class HloInstruction {
 
   // Creates a dynamic update slice instruction, which updates a slice
   // of 'operand' with 'update' and 'start_indices'.
-  ABSL_DEPRECATED("Use span-of-indices form instead")
-  static std::unique_ptr<HloInstruction> CreateDynamicUpdateSlice(
-      const Shape& shape, HloInstruction* operand, HloInstruction* update,
-      HloInstruction* start_indices);
-  // Same as above, but expects a span of scalar start indices.
   static std::unique_ptr<HloInstruction> CreateDynamicUpdateSlice(
       const Shape& shape, HloInstruction* operand, HloInstruction* update,
       absl::Span<HloInstruction* const> start_indices);
diff --git a/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc b/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc
index 7599e1e6ad..fd5759e442 100644
--- a/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc
+++ b/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc
@@ -623,7 +623,7 @@ class FusionPointsToAnalysisTest : public TuplePointsToAnalysisTest {
   void Run(const bool add_additional_gte0_user) {
     Shape input_shape = ShapeUtil::MakeShape(F32, {8});
     Shape update_shape = ShapeUtil::MakeShape(F32, {3});
-    Shape starts_shape = ShapeUtil::MakeShape(S32, {1});
+    Shape starts_shape = ShapeUtil::MakeShape(S32, {});
     Shape tuple_shape =
         ShapeUtil::MakeTupleShape({input_shape, update_shape, starts_shape});
 
@@ -657,7 +657,7 @@ class FusionPointsToAnalysisTest : public TuplePointsToAnalysisTest {
         HloInstruction::CreateGetTupleElement(starts_shape, tuple_param0, 2));
     // Update 'input' with 'update' at dynamic 'starts' indices.
     builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
-        input_shape, input, update, starts));
+        input_shape, input, update, {starts}));
 
     // Build computation and add it to module as entry computation.
     BuildModule(builder.Build());
@@ -882,12 +882,12 @@ TEST_F(DoesNotUseOperandBufferTest, FusedDynamicUpdateSlice) {
 
   // Create a DynamicUpdateSlice instruction of tuple element 1.
   auto starts = builder.AddInstruction(
-      HloInstruction::CreateConstant(LiteralUtil::CreateR1<int32>({2})));
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(2)));
   auto update = builder.AddInstruction(HloInstruction::CreateConstant(
       LiteralUtil::CreateR1<float>({2.f, 2.f, 2.f})));
   auto dynamic_update_slice =
       builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
-          data_shape, gte1, update, starts));
+          data_shape, gte1, update, {starts}));
   builder.AddInstruction(
       HloInstruction::CreateTuple({gte0, dynamic_update_slice}));
 
@@ -976,12 +976,12 @@ TEST_F(CanShareOperandBufferWithUserTest, FusedDynamicUpdateSlice) {
 
   // Create a DynamicUpdateSlice instruction of tuple element 1.
   auto starts = builder.AddInstruction(
-      HloInstruction::CreateConstant(LiteralUtil::CreateR1<int32>({2})));
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(2)));
   auto update = builder.AddInstruction(HloInstruction::CreateConstant(
       LiteralUtil::CreateR1<float>({2.f, 2.f, 2.f})));
   auto dynamic_update_slice =
       builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
-          data_shape, gte1, update, starts));
+          data_shape, gte1, update, {starts}));
   builder.AddInstruction(
       HloInstruction::CreateTuple({gte0, dynamic_update_slice}));
 
@@ -1003,7 +1003,7 @@ TEST_F(CanShareOperandBufferWithUserTest, DynamicUpdateSliceCanShare) {
 
   Shape data_shape = ShapeUtil::MakeShape(F32, {8});
   Shape update_shape = ShapeUtil::MakeShape(F32, {4});
-  Shape starts_shape = ShapeUtil::MakeShape(S32, {1});
+  Shape starts_shape = ShapeUtil::MakeShape(S32, {});
   auto data = builder.AddInstruction(
       HloInstruction::CreateParameter(0, data_shape, "data"));
   auto update = builder.AddInstruction(
@@ -1011,7 +1011,7 @@ TEST_F(CanShareOperandBufferWithUserTest, DynamicUpdateSliceCanShare) {
   auto starts = builder.AddInstruction(
       HloInstruction::CreateParameter(2, starts_shape, "starts"));
   auto dus = builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
-      data_shape, data, update, starts));
+      data_shape, data, update, {starts}));
 
   BuildModuleAndRunAnalysis(builder.Build());
 
diff --git a/tensorflow/compiler/xla/tests/fusion_test.cc b/tensorflow/compiler/xla/tests/fusion_test.cc
index d1fddf9d6b..2178c9b3f3 100644
--- a/tensorflow/compiler/xla/tests/fusion_test.cc
+++ b/tensorflow/compiler/xla/tests/fusion_test.cc
@@ -523,10 +523,10 @@ XLA_TEST_F(FusionTest, DynamicSliceNegate) {
   auto const0 = builder.AddInstruction(HloInstruction::CreateConstant(
       LiteralUtil::CreateR1<int32>({1, 2, 3, 4})));
   auto const1 = builder.AddInstruction(
-      HloInstruction::CreateConstant(LiteralUtil::CreateR1<int32>({1})));
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(1)));
   auto dynamic_slice2 =
       builder.AddInstruction(HloInstruction::CreateDynamicSlice(
-          ShapeUtil::MakeShape(S32, {2}), const0, const1, {2}));
+          ShapeUtil::MakeShape(S32, {2}), const0, {const1}, {2}));
   auto negate3 = builder.AddInstruction(HloInstruction::CreateUnary(
       ShapeUtil::MakeShape(S32, {2}), HloOpcode::kNegate, dynamic_slice2));
   hlo_module->AddEntryComputation(builder.Build())
-- 
GitLab


From 5c08b1f021e0a40fece750b5ebc0cf85b3281265 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 11:52:56 -0800
Subject: [PATCH 0869/2345] Remove alias redirection for RBE.

PiperOrigin-RevId: 229790895
---
 tensorflow/opensource_only.files            |   3 -
 third_party/gpus/crosstool/remote.BUILD.tpl |  10 --
 third_party/gpus/cuda/remote.BUILD.tpl      | 110 --------------------
 third_party/gpus/cuda_configure.bzl         |  17 ++-
 third_party/tensorrt/build_defs.bzl.tpl     |   4 +-
 third_party/tensorrt/remote.BUILD.tpl       |   7 --
 third_party/tensorrt/tensorrt_configure.bzl |  11 +-
 7 files changed, 14 insertions(+), 148 deletions(-)
 delete mode 100644 third_party/gpus/crosstool/remote.BUILD.tpl
 delete mode 100644 third_party/gpus/cuda/remote.BUILD.tpl
 delete mode 100644 third_party/tensorrt/remote.BUILD.tpl

diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index 30dfce3c83..5f1ec1d673 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -82,7 +82,6 @@ tensorflow/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_is_not_
 tensorflow/third_party/gpus/crosstool/CROSSTOOL.tpl
 tensorflow/third_party/gpus/crosstool/CROSSTOOL_hipcc.tpl
 tensorflow/third_party/gpus/crosstool/LICENSE
-tensorflow/third_party/gpus/crosstool/remote.BUILD.tpl
 tensorflow/third_party/gpus/crosstool/windows/msvc_wrapper_for_nvcc.py.tpl
 tensorflow/third_party/gpus/crosstool/BUILD.tpl
 tensorflow/third_party/gpus/crosstool/BUILD
@@ -90,7 +89,6 @@ tensorflow/third_party/gpus/cuda/LICENSE
 tensorflow/third_party/gpus/cuda/BUILD.tpl
 tensorflow/third_party/gpus/cuda/BUILD.windows.tpl
 tensorflow/third_party/gpus/cuda/cuda_config.h.tpl
-tensorflow/third_party/gpus/cuda/remote.BUILD.tpl
 tensorflow/third_party/gpus/cuda/BUILD
 tensorflow/third_party/gpus/cuda/build_defs.bzl.tpl
 tensorflow/third_party/gpus/rocm/rocm_config.h.tpl
@@ -194,7 +192,6 @@ tensorflow/third_party/tensorrt/BUILD
 tensorflow/third_party/tensorrt/build_defs.bzl.tpl
 tensorflow/third_party/tensorrt/BUILD.tpl
 tensorflow/third_party/tensorrt/tensorrt_configure.bzl
-tensorflow/third_party/tensorrt/remote.BUILD.tpl
 tensorflow/third_party/kafka/config.patch
 tensorflow/third_party/kafka/BUILD
 tensorflow/third_party/android/BUILD
diff --git a/third_party/gpus/crosstool/remote.BUILD.tpl b/third_party/gpus/crosstool/remote.BUILD.tpl
deleted file mode 100644
index b2316331db..0000000000
--- a/third_party/gpus/crosstool/remote.BUILD.tpl
+++ /dev/null
@@ -1,10 +0,0 @@
-# Description:
-#   Template for crosstool Build file to use a pre-generated config.
-licenses(["restricted"])
-
-package(default_visibility = ["//visibility:public"])
-
-alias(
-    name = "toolchain",
-    actual = "%{remote_cuda_repo}:toolchain",
-)
diff --git a/third_party/gpus/cuda/remote.BUILD.tpl b/third_party/gpus/cuda/remote.BUILD.tpl
deleted file mode 100644
index 100c7bb7c4..0000000000
--- a/third_party/gpus/cuda/remote.BUILD.tpl
+++ /dev/null
@@ -1,110 +0,0 @@
-# Description:
-#   Template for cuda Build file to use a pre-generated config.
-licenses(["restricted"])  # MPL2, portions GPL v3, LGPL v3, BSD-like
-
-package(default_visibility = ["//visibility:public"])
-
-config_setting(
-    name = "using_nvcc",
-    values = {
-        "define": "using_cuda_nvcc=true",
-    },
-)
-
-config_setting(
-    name = "using_clang",
-    values = {
-        "define": "using_cuda_clang=true",
-    },
-)
-
-# Equivalent to using_clang && -c opt.
-config_setting(
-    name = "using_clang_opt",
-    values = {
-        "define": "using_cuda_clang=true",
-        "compilation_mode": "opt",
-    },
-)
-
-config_setting(
-    name = "darwin",
-    values = {"cpu": "darwin"},
-    visibility = ["//visibility:public"],
-)
-
-config_setting(
-    name = "freebsd",
-    values = {"cpu": "freebsd"},
-    visibility = ["//visibility:public"],
-)
-
-alias(
-    name = "cuda_headers",
-    actual = "%{remote_cuda_repo}/cuda:cuda_headers",
-)
-
-alias(
-    name = "cudart_static",
-    actual = "%{remote_cuda_repo}/cuda:cudart_static",
-)
-
-alias(
-    name = "cuda_driver",
-    actual = "%{remote_cuda_repo}/cuda:cuda_driver",
-)
-
-alias(
-    name = "cudart",
-    actual = "%{remote_cuda_repo}/cuda:cudart",
-)
-
-alias(
-    name = "cublas",
-    actual = "%{remote_cuda_repo}/cuda:cublas",
-)
-
-alias(
-    name = "cusolver",
-    actual = "%{remote_cuda_repo}/cuda:cusolver",
-)
-
-alias(
-    name = "cudnn",
-    actual = "%{remote_cuda_repo}/cuda:cudnn",
-)
-
-alias(
-    name = "cudnn_header",
-    actual = "%{remote_cuda_repo}/cuda:cudnn_header",
-)
-
-alias(
-    name = "cufft",
-    actual = "%{remote_cuda_repo}/cuda:cufft",
-)
-
-alias(
-    name = "curand",
-    actual = "%{remote_cuda_repo}/cuda:curand",
-)
-
-alias(
-    name = "cuda",
-    actual = "%{remote_cuda_repo}/cuda:cuda",
-)
-
-alias(
-    name = "cupti_headers",
-    actual = "%{remote_cuda_repo}/cuda:cupti_headers",
-)
-
-alias(
-    name = "cupti_dsos",
-    actual = "%{remote_cuda_repo}/cuda:cupti_dsos",
-)
-
-alias(
-    name = "libdevice_root",
-    actual = "%{remote_cuda_repo}/cuda:libdevice_root",
-)
diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl
index 1988c4ae96..beacf2f45a 100644
--- a/third_party/gpus/cuda_configure.bzl
+++ b/third_party/gpus/cuda_configure.bzl
@@ -1492,17 +1492,16 @@ def _create_remote_cuda_repository(repository_ctx, remote_config_repo):
               ),
       },
   )
-  _tpl(
-      repository_ctx,
-      "cuda:remote.BUILD",
-      {
-          "%{remote_cuda_repo}": remote_config_repo,
-      },
+  repository_ctx.template(
       "cuda/BUILD",
+      Label(remote_config_repo + "/cuda/BUILD"),
+      {},
+  )
+  repository_ctx.template(
+      "crosstool/BUILD",
+      Label(remote_config_repo + "/crosstool/BUILD"),
+      {},
   )
-  _tpl(repository_ctx, "crosstool:remote.BUILD", {
-      "%{remote_cuda_repo}": remote_config_repo,
-  }, "crosstool/BUILD")
 
 
 def _cuda_autoconf_impl(repository_ctx):
diff --git a/third_party/tensorrt/build_defs.bzl.tpl b/third_party/tensorrt/build_defs.bzl.tpl
index 0dc3a7ba2d..6d00513827 100644
--- a/third_party/tensorrt/build_defs.bzl.tpl
+++ b/third_party/tensorrt/build_defs.bzl.tpl
@@ -2,6 +2,4 @@
 
 def if_tensorrt(if_true, if_false=[]):
   """Tests whether TensorRT was enabled during the configure process."""
-  if %{tensorrt_is_configured}:
-    return if_true
-  return if_false
+  return %{if_tensorrt}
diff --git a/third_party/tensorrt/remote.BUILD.tpl b/third_party/tensorrt/remote.BUILD.tpl
deleted file mode 100644
index 7598e7aa4b..0000000000
--- a/third_party/tensorrt/remote.BUILD.tpl
+++ /dev/null
@@ -1,7 +0,0 @@
-licenses(["notice"])  # Apache 2.0
-
-package(default_visibility = ["//visibility:public"])
-
-alias(name="LICENSE", actual = "%{target}:LICENSE")
-alias(name = "tensorrt_headers", actual = "%{target}:tensorrt_headers")
-alias(name = "nv_infer", actual = "%{target}:nv_infer")
diff --git a/third_party/tensorrt/tensorrt_configure.bzl b/third_party/tensorrt/tensorrt_configure.bzl
index d1e116d9dd..7cf845d5cc 100644
--- a/third_party/tensorrt/tensorrt_configure.bzl
+++ b/third_party/tensorrt/tensorrt_configure.bzl
@@ -143,7 +143,7 @@ def _tpl(repository_ctx, tpl, substitutions):
 
 def _create_dummy_repository(repository_ctx):
   """Create a dummy TensorRT repository."""
-  _tpl(repository_ctx, "build_defs.bzl", {"%{tensorrt_is_configured}": "False"})
+  _tpl(repository_ctx, "build_defs.bzl", {"%{if_tensorrt}": "if_false"})
   substitutions = {
       "%{tensorrt_genrules}": "",
       "%{tensorrt_headers}": "",
@@ -158,11 +158,10 @@ def _tensorrt_configure_impl(repository_ctx):
   """Implementation of the tensorrt_configure repository rule."""
   if _TF_TENSORRT_CONFIG_REPO in repository_ctx.os.environ:
     # Forward to the pre-configured remote repository.
-    repository_ctx.template("BUILD", Label("//third_party/tensorrt:remote.BUILD.tpl"), {
-        "%{target}": repository_ctx.os.environ[_TF_TENSORRT_CONFIG_REPO],
-    })
+    remote_config_repo = repository_ctx.os.environ[_TF_TENSORRT_CONFIG_REPO]
+    repository_ctx.template("BUILD", Label(remote_config_repo + "/BUILD"), {})
     # Set up config file.
-    _tpl(repository_ctx, "build_defs.bzl", {"%{tensorrt_is_configured}": "True"})
+    _tpl(repository_ctx, "build_defs.bzl", {"%{if_tensorrt}": "if_true"})
     return
 
   if _TENSORRT_INSTALL_PATH not in repository_ctx.os.environ:
@@ -210,7 +209,7 @@ def _tensorrt_configure_impl(repository_ctx):
   ))
 
   # Set up config file.
-  _tpl(repository_ctx, "build_defs.bzl", {"%{tensorrt_is_configured}": "True"})
+  _tpl(repository_ctx, "build_defs.bzl", {"%{if_tensorrt}": "if_true"})
 
   # Set up BUILD file.
   substitutions = {
-- 
GitLab


From 2435a1875b574f5d299b1dee431ab5ceccd6132f Mon Sep 17 00:00:00 2001
From: Revan Sopher <rsopher@google.com>
Date: Thu, 17 Jan 2019 11:54:37 -0800
Subject: [PATCH 0870/2345] Switch out unittest for absltest, and point app.py
 to absl's app.py to resolve flag conflict.

PiperOrigin-RevId: 229791193
---
 tensorflow/python/BUILD                  |  6 +-
 tensorflow/python/platform/app.py        | 98 ++----------------------
 tensorflow/python/platform/googletest.py | 12 +--
 tensorflow/tools/pip_package/BUILD       |  5 ++
 4 files changed, 22 insertions(+), 99 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index b61c33f319..e0648085c3 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -213,6 +213,7 @@ py_library(
         ":pywrap_tensorflow",
         ":util",
         "//tensorflow/core:protos_all_py",
+        "@absl_py//absl:app",
         "@absl_py//absl/flags",
         "@six_archive//:six",
     ],
@@ -233,7 +234,10 @@ py_library(
     name = "platform_test",
     srcs = ["platform/googletest.py"],
     srcs_version = "PY2AND3",
-    deps = [":platform_benchmark"],
+    deps = [
+        ":platform_benchmark",
+        "@absl_py//absl/testing:absltest",
+    ],
 )
 
 tf_py_test(
diff --git a/tensorflow/python/platform/app.py b/tensorflow/python/platform/app.py
index 7b917235c0..303b70ff57 100644
--- a/tensorflow/python/platform/app.py
+++ b/tensorflow/python/platform/app.py
@@ -18,109 +18,23 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import errno as _errno
 import sys as _sys
 
+from absl.app import run as _run
+
 from tensorflow.python.platform import flags
 from tensorflow.python.util.tf_export import tf_export
 
 
-def _usage(shorthelp):
-  """Writes __main__'s docstring to stdout with some help text.
-
-  Args:
-    shorthelp: bool, if True, prints only flags from the main module,
-        rather than all flags.
-  """
-  doc = _sys.modules['__main__'].__doc__
-  if not doc:
-    doc = '\nUSAGE: %s [flags]\n' % _sys.argv[0]
-    doc = flags.text_wrap(doc, indent='       ', firstline_indent='')
-  else:
-    # Replace all '%s' with sys.argv[0], and all '%%' with '%'.
-    num_specifiers = doc.count('%') - 2 * doc.count('%%')
-    try:
-      doc %= (_sys.argv[0],) * num_specifiers
-    except (OverflowError, TypeError, ValueError):
-      # Just display the docstring as-is.
-      pass
-  if shorthelp:
-    flag_str = flags.FLAGS.main_module_help()
-  else:
-    flag_str = str(flags.FLAGS)
-  try:
-    _sys.stdout.write(doc)
-    if flag_str:
-      _sys.stdout.write('\nflags:\n')
-      _sys.stdout.write(flag_str)
-    _sys.stdout.write('\n')
-  except IOError as e:
-    # We avoid printing a huge backtrace if we get EPIPE, because
-    # "foo.par --help | less" is a frequent use case.
-    if e.errno != _errno.EPIPE:
-      raise
-
-
-class _HelpFlag(flags.BooleanFlag):
-  """Special boolean flag that displays usage and raises SystemExit."""
-  NAME = 'help'
-  SHORT_NAME = 'h'
-
-  def __init__(self):
-    super(_HelpFlag, self).__init__(
-        self.NAME, False, 'show this help', short_name=self.SHORT_NAME)
-
-  def parse(self, arg):
-    if arg:
-      _usage(shorthelp=True)
-      print()
-      print('Try --helpfull to get a list of all flags.')
-      _sys.exit(1)
-
-
-class _HelpshortFlag(_HelpFlag):
-  """--helpshort is an alias for --help."""
-  NAME = 'helpshort'
-  SHORT_NAME = None
-
-
-class _HelpfullFlag(flags.BooleanFlag):
-  """Display help for flags in main module and all dependent modules."""
-
-  def __init__(self):
-    super(_HelpfullFlag, self).__init__('helpfull', False, 'show full help')
-
-  def parse(self, arg):
-    if arg:
-      _usage(shorthelp=False)
-      _sys.exit(1)
-
-
-_define_help_flags_called = False
-
-
-def _define_help_flags():
-  global _define_help_flags_called
-  if not _define_help_flags_called:
-    flags.DEFINE_flag(_HelpFlag())
-    flags.DEFINE_flag(_HelpfullFlag())
-    flags.DEFINE_flag(_HelpshortFlag())
-    _define_help_flags_called = True
+def _parse_flags_tolerate_undef(argv):
+  """Parse args, returning any unknown flags (ABSL defaults to crashing)."""
+  return flags.FLAGS(_sys.argv if argv is None else argv, known_only=True)
 
 
 @tf_export(v1=['app.run'])
 def run(main=None, argv=None):
   """Runs the program with an optional 'main' function and 'argv' list."""
 
-  # Define help flags.
-  _define_help_flags()
-
-  # Parse known flags.
-  argv = flags.FLAGS(_sys.argv if argv is None else argv, known_only=True)
-
   main = main or _sys.modules['__main__'].main
 
-  # Call the main function, passing through any arguments
-  # to the final program.
-  _sys.exit(main(argv))
-
+  _run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
diff --git a/tensorflow/python/platform/googletest.py b/tensorflow/python/platform/googletest.py
index fe4b0d0d37..802721e34b 100644
--- a/tensorflow/python/platform/googletest.py
+++ b/tensorflow/python/platform/googletest.py
@@ -13,7 +13,7 @@
 # limitations under the License.
 # ==============================================================================
 
-"""Imports unittest as a replacement for testing.pybase.googletest."""
+"""Imports absltest as a replacement for testing.pybase.googletest."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -26,7 +26,7 @@ import tempfile
 
 # go/tf-wildcard-import
 # pylint: disable=wildcard-import
-from unittest import *
+from absl.testing.absltest import *
 # pylint: enable=wildcard-import
 
 from tensorflow.python.framework import errors
@@ -41,7 +41,7 @@ from tensorflow.python.util.tf_export import tf_export
 
 Benchmark = benchmark.TensorFlowBenchmark  # pylint: disable=invalid-name
 
-unittest_main = main
+absltest_main = main
 
 # We keep a global variable in this module to make sure we create the temporary
 # directory only once per test binary invocation.
@@ -51,7 +51,7 @@ _googletest_temp_dir = ''
 # pylint: disable=invalid-name
 # pylint: disable=undefined-variable
 def g_main(argv):
-  """Delegate to unittest.main after redefining testLoader."""
+  """Delegate to absltest.main after redefining testLoader."""
   if 'TEST_SHARD_STATUS_FILE' in os.environ:
     try:
       f = None
@@ -67,7 +67,7 @@ def g_main(argv):
 
   if ('TEST_TOTAL_SHARDS' not in os.environ or
       'TEST_SHARD_INDEX' not in os.environ):
-    return unittest_main(argv=argv)
+    return absltest_main(argv=argv)
 
   total_shards = int(os.environ['TEST_TOTAL_SHARDS'])
   shard_index = int(os.environ['TEST_SHARD_INDEX'])
@@ -87,7 +87,7 @@ def g_main(argv):
   # Override getTestCaseNames
   base_loader.getTestCaseNames = getShardedTestCaseNames
 
-  unittest_main(argv=argv, testLoader=base_loader)
+  absltest_main(argv=argv, testLoader=base_loader)
 
 
 # Redefine main to allow running benchmarks
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index c51b45a49c..3c1a9280a2 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -146,7 +146,11 @@ filegroup(
         "//third_party/eigen3:LICENSE",
         "//third_party/fft2d:LICENSE",
         "//third_party/hadoop:LICENSE.txt",
+        "@absl_py//absl:LICENSE",
+        "@absl_py//absl/logging:LICENSE",
         "@absl_py//absl/flags:LICENSE",
+        "@absl_py//absl/testing:LICENSE",
+        "@absl_py//absl/third_party/unittest3_backport:LICENSE",
         "@arm_neon_2_x86_sse//:LICENSE",
         "@astor_archive//:LICENSE",
         "@boringssl//:LICENSE",
@@ -155,6 +159,7 @@ filegroup(
         "@curl//:COPYING",
         "@double_conversion//:LICENSE",
         "@eigen_archive//:COPYING.MPL2",
+        "@enum34_archive//:LICENSE",
         "@farmhash_archive//:COPYING",
         "@fft2d//:fft/readme.txt",
         "@flatbuffers//:LICENSE.txt",
-- 
GitLab


From f5465080e9d3b2544c5b0fe364070d55b469e516 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Thu, 17 Jan 2019 11:56:22 -0800
Subject: [PATCH 0871/2345] Device Assignment support in TPUStrategy for
 StatefulRNN

PiperOrigin-RevId: 229791509
---
 .../contrib/distribute/python/tpu_strategy.py | 56 +++++++++++++++++--
 .../tpu/python/tpu/device_assignment.py       |  3 +
 2 files changed, 53 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index b9d3924a24..f55e6f0b20 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -25,6 +25,8 @@ import copy
 import functools
 
 from tensorflow.contrib.tpu.python.ops import tpu_ops
+from tensorflow.contrib.tpu.python.tpu import device_assignment as device_assignment_lib
+from tensorflow.contrib.tpu.python.tpu import topology
 from tensorflow.contrib.tpu.python.tpu import tpu
 from tensorflow.contrib.tpu.python.tpu import tpu_system_metadata as tpu_system_metadata_lib
 from tensorflow.contrib.tpu.python.tpu import training_loop
@@ -58,6 +60,8 @@ def initialize_tpu_system(cluster_resolver=None):
   Args:
     cluster_resolver: A tf.contrib.cluster_resolver.TPUClusterResolver,
         which provides information about the TPU cluster.
+  Returns:
+    The tf.contrib.tpu.Topology object for the topology of the TPU cluster.
   """
   if cluster_resolver is None:
     cluster_resolver = resolver_lib.TPUClusterResolver("")
@@ -68,8 +72,9 @@ def initialize_tpu_system(cluster_resolver=None):
 
   with ops.Graph().as_default():
     with session_lib.Session(config=session_config, target=master) as sess:
-      sess.run([tpu.initialize_system()])
+      serialized_topology = sess.run(tpu.initialize_system())
   logging.info("Finished initializing TPU system.")
+  return topology.Topology(serialized=serialized_topology)
 
 
 def get_tpu_system_metadata(tpu_cluster_resolver):
@@ -149,6 +154,7 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
   def __init__(self,
                tpu_cluster_resolver=None,
                steps_per_run=None,
+               device_assignment=None,
                **kwargs):
     """Initializes the TPUStrategy object.
 
@@ -160,10 +166,13 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
           metrics, summaries etc.
           This parameter is only used when Distribution Strategy is used with
           estimator or keras.
+      device_assignment: Optional `tf.contrib.tpu.DeviceAssignment` to specify
+          the placement of replicas on the TPU cluster. Currently only supports
+          the usecase of using a single core within a TPU cluster.
       **kwargs: Additional experimental flags. Will be removed in future.
     """
     super(TPUStrategy, self).__init__(TPUExtended(
-        self, tpu_cluster_resolver, steps_per_run))
+        self, tpu_cluster_resolver, steps_per_run, device_assignment))
 
     self._disable_training_loop_on_host = False
     if len(kwargs) > 1:
@@ -188,7 +197,8 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
   def __init__(self,
                container_strategy,
                tpu_cluster_resolver=None,
-               steps_per_run=None):
+               steps_per_run=None,
+               device_assignment=None):
     super(TPUExtended, self).__init__(container_strategy)
 
     if tpu_cluster_resolver is None:
@@ -201,6 +211,21 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
 
     self._tpu_cluster_resolver = tpu_cluster_resolver
     self._tpu_metadata = get_tpu_system_metadata(self._tpu_cluster_resolver)
+    self._device_assignment = device_assignment
+
+    # Device assignment is currently only supported for 1 core case.
+    if self._device_assignment:
+      assert isinstance(self._device_assignment,
+                        device_assignment_lib.DeviceAssignment)
+      if self._device_assignment.num_replicas != 1:
+        raise ValueError("Device assignment is only supported for a single "
+                         "core single replica case currently.")
+      if self._device_assignment.num_cores_per_replica != 1:
+        raise ValueError("Device assignment is only supported for a single "
+                         "core single replica case currently.")
+      if not all(self._device_assignment.core_assignment[0][0] == [0, 0, 0]):
+        raise ValueError("Device assignment is only supported for a single "
+                         "core single replica case currently.")
 
     # TODO(jhseu): Switch to DeviceAssignment to support pods and model
     # parallelism.
@@ -604,15 +629,34 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
 
   @property
   def num_hosts(self):
-    return self._tpu_metadata.num_hosts
+    if self._device_assignment is None:
+      return self._tpu_metadata.num_hosts
+
+    return len(set([self._device_assignment.host_device(r)
+                    for r in range(self._device_assignment.num_replicas)]))
 
   @property
   def num_replicas_per_host(self):
-    return self._tpu_metadata.num_of_cores_per_host
+    if self._device_assignment is None:
+      return self._tpu_metadata.num_of_cores_per_host
+
+    # TODO(sourabhbajaj): Remove this method we use inputs and remove infeed
+    # as the computation of num_replicas_per_host is not a constant
+    # when using device_assignment. This is a temporary workaround to support
+    # StatefulRNN as everything is 1 in that case.
+    # This method needs to take host_id as input for correct computation.
+    max_models_per_host = (self._tpu_metadata.num_of_cores_per_host //
+                           self._device_assignment.num_cores_per_replica)
+    models_per_host = min(self._device_assignment.num_replicas,
+                          max_models_per_host)
+    return models_per_host * self._device_assignment.num_cores_per_replica
 
   @property
   def _num_replicas_in_sync(self):
-    return self._tpu_metadata.num_cores
+    if self._device_assignment is None:
+      return self._tpu_metadata.num_cores
+    return (self._device_assignment.num_replicas *
+            self._device_assignment.num_cores_per_replica)
 
   @property
   def experimental_between_graph(self):
diff --git a/tensorflow/contrib/tpu/python/tpu/device_assignment.py b/tensorflow/contrib/tpu/python/tpu/device_assignment.py
index 6906501ecf..3313dc749c 100644
--- a/tensorflow/contrib/tpu/python/tpu/device_assignment.py
+++ b/tensorflow/contrib/tpu/python/tpu/device_assignment.py
@@ -25,6 +25,9 @@ from six.moves import xrange  # pylint: disable=redefined-builtin
 from tensorflow.contrib.tpu.python.tpu.topology import Topology
 
 
+SINGLE_CORE_ASSIGNMENT = [[[0, 0, 0]]]
+
+
 def _compute_task_and_cores_to_replicas(core_assignment, topology):
   """Computes a nested dict which maps task and logical core to replicas."""
   task_and_cores_to_replicas = {}
-- 
GitLab


From 5f640305f37a58970f7070ecc074ada1ec14107b Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Thu, 17 Jan 2019 11:58:19 -0800
Subject: [PATCH 0872/2345] Disable nnapi_delegate_test portable tests

PiperOrigin-RevId: 229791853
---
 tensorflow/lite/delegates/nnapi/BUILD | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/delegates/nnapi/BUILD b/tensorflow/lite/delegates/nnapi/BUILD
index dda3c02567..63f6da1fa6 100644
--- a/tensorflow/lite/delegates/nnapi/BUILD
+++ b/tensorflow/lite/delegates/nnapi/BUILD
@@ -24,7 +24,11 @@ tf_cc_test(
     name = "nnapi_delegate_test",
     size = "small",
     srcs = ["nnapi_delegate_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        # TODO(b/122987564): Enable on Android after resolving API 27 failures.
+        "tflite_not_portable_android",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":nnapi_delegate",
         "//tensorflow/lite:framework",
-- 
GitLab


From 1e8a3d9e1fc542bca285682dd0fb9971ac3262d3 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Thu, 17 Jan 2019 12:06:17 -0800
Subject: [PATCH 0873/2345] Adds v2 binary crossentropy metric.

PiperOrigin-RevId: 229793647
---
 tensorflow/python/keras/metrics.py      |  66 +++++++++++++
 tensorflow/python/keras/metrics_test.py | 120 ++++++++++++++++++++++++
 2 files changed, 186 insertions(+)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index d6010e3cb0..d3bd523d29 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -2411,6 +2411,72 @@ class MeanTensor(Metric):
         K.set_value(v, np.zeros(self._shape.as_list()))
 
 
+class BinaryCrossentropy(MeanMetricWrapper):
+  """Computes the binary crossentropy between `y_true` and `y_pred`.
+
+  Usage:
+
+  ```python
+  m = tf.keras.metrics.BinaryCrossentropy()
+  m.update_state([1., 0., 1., 0.], [1., 1., 1., 0.])
+
+  # EPSILON = 1e-7, y = y_true, y` = y_pred, Y_MAX = 0.9999999
+  # y` = clip_ops.clip_by_value(output, EPSILON, 1. - EPSILON)
+  # y` = [Y_MAX, Y_MAX, Y_MAX, EPSILON]
+
+  # Metric = -(y log(y` + EPSILON) + (1 - y) log(1 - y` + EPSILON))
+  #        = [-log(Y_MAX + EPSILON), -log(1 - Y_MAX + EPSILON),
+  #           -log(Y_MAX + EPSILON), -log(1)]
+  #        = [(0 + 15.33) / 2, (0 + 0) / 2]
+  # Reduced metric = 7.665 / 2
+
+  print('Final result: ', m.result().numpy())  # Final result: 3.833
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile(
+      'sgd',
+      loss='mse',
+      metrics=[tf.keras.metrics.BinaryCrossentropy()])
+  ```
+  """
+
+  def __init__(self,
+               name='binary_crossentropy',
+               dtype=None,
+               from_logits=False,
+               label_smoothing=0):
+    """Creates a `BinaryCrossentropy` instance.
+
+    Args:
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+      from_logits: (Optional )Whether output is expected to be a logits tensor.
+        By default, we consider that output encodes a probability distribution.
+      label_smoothing: (Optional) Float in [0, 1]. When > 0, label values are
+        smoothed, meaning the confidence on label values are relaxed.
+        e.g. `label_smoothing=0.2` means that we will use a value of `0.1` for
+        label `0` and `0.9` for label `1`"
+    """
+    label_smoothing = ops.convert_to_tensor(label_smoothing, dtype=K.floatx())
+
+    super(BinaryCrossentropy, self).__init__(
+        binary_crossentropy,
+        name,
+        dtype=dtype,
+        from_logits=from_logits,
+        label_smoothing=label_smoothing)
+
+  @classmethod
+  def from_config(cls, config):
+    if 'fn' in config:
+      config.pop('fn')
+    return super(BinaryCrossentropy, cls).from_config(config)
+
+
 def accuracy(y_true, y_pred):
   y_pred.get_shape().assert_is_compatible_with(y_true.get_shape())
   if y_true.dtype != y_pred.dtype:
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index bd94132a9a..4e755dd301 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -1200,6 +1200,126 @@ class MeanTensorTest(keras_parameterized.TestCase):
                           np.full((4, 3), 4))
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class BinaryCrossentropyTest(test.TestCase):
+
+  def test_config(self):
+    bce_obj = metrics.BinaryCrossentropy(
+        name='bce', dtype=dtypes.int32, label_smoothing=0.2)
+    self.assertEqual(bce_obj.name, 'bce')
+    self.assertEqual(bce_obj._dtype, dtypes.int32)
+
+    old_config = bce_obj.get_config()
+    self.assertAllClose(self.evaluate(old_config['label_smoothing']), 0.2, 1e-3)
+
+    # Check save and restore config
+    bce_obj2 = metrics.BinaryCrossentropy.from_config(old_config)
+    self.assertEqual(bce_obj2.name, 'bce')
+    self.assertEqual(bce_obj2._dtype, dtypes.int32)
+    new_config = bce_obj2.get_config()
+    self.assertAllClose(self.evaluate(new_config['label_smoothing']), 0.2, 1e-3)
+
+  def test_unweighted(self):
+    bce_obj = metrics.BinaryCrossentropy()
+    self.evaluate(variables.variables_initializer(bce_obj.variables))
+    y_true = np.asarray([1, 0, 1, 0]).reshape([2, 2])
+    y_pred = np.asarray([1, 1, 1, 0], dtype=np.float32).reshape([2, 2])
+    result = bce_obj(y_true, y_pred)
+
+    # EPSILON = 1e-7, y = y_true, y` = y_pred, Y_MAX = 0.9999999
+    # y` = clip_ops.clip_by_value(output, EPSILON, 1. - EPSILON)
+    # y` = [Y_MAX, Y_MAX, Y_MAX, EPSILON]
+
+    # Metric = -(y log(y` + EPSILON) + (1 - y) log(1 - y` + EPSILON))
+    #        = [-log(Y_MAX + EPSILON), -log(1 - Y_MAX + EPSILON),
+    #           -log(Y_MAX + EPSILON), -log(1)]
+    #        = [(0 + 15.33) / 2, (0 + 0) / 2]
+    # Reduced metric = 7.665 / 2
+
+    self.assertAllClose(self.evaluate(result), 3.833, atol=1e-3)
+
+  def test_unweighted_with_logits(self):
+    bce_obj = metrics.BinaryCrossentropy(from_logits=True)
+    self.evaluate(variables.variables_initializer(bce_obj.variables))
+
+    y_true = constant_op.constant([[1, 0, 1], [0, 1, 1]])
+    y_pred = constant_op.constant([[100.0, -100.0, 100.0],
+                                   [100.0, 100.0, -100.0]])
+    result = bce_obj(y_true, y_pred)
+
+    # Metric = max(x, 0) - x * z + log(1 + exp(-abs(x)))
+    #              (where x = logits and z = y_true)
+    #        = [((100 - 100 * 1 + log(1 + exp(-100))) +
+    #            (0 + 100 * 0 + log(1 + exp(-100))) +
+    #            (100 - 100 * 1 + log(1 + exp(-100))),
+    #           ((100 - 100 * 0 + log(1 + exp(-100))) +
+    #            (100 - 100 * 1 + log(1 + exp(-100))) +
+    #            (0 + 100 * 1 + log(1 + exp(-100))))]
+    #        = [(0 + 0 + 0) / 3, 200 / 3]
+    # Reduced metric = (0 + 66.666) / 2
+
+    self.assertAllClose(self.evaluate(result), 33.333, atol=1e-3)
+
+  def test_weighted(self):
+    bce_obj = metrics.BinaryCrossentropy()
+    self.evaluate(variables.variables_initializer(bce_obj.variables))
+    y_true = np.asarray([1, 0, 1, 0]).reshape([2, 2])
+    y_pred = np.asarray([1, 1, 1, 0], dtype=np.float32).reshape([2, 2])
+    sample_weight = constant_op.constant([1.5, 2.])
+    result = bce_obj(y_true, y_pred, sample_weight=sample_weight)
+
+    # EPSILON = 1e-7, y = y_true, y` = y_pred, Y_MAX = 0.9999999
+    # y` = clip_ops.clip_by_value(output, EPSILON, 1. - EPSILON)
+    # y` = [Y_MAX, Y_MAX, Y_MAX, EPSILON]
+
+    # Metric = -(y log(y` + EPSILON) + (1 - y) log(1 - y` + EPSILON))
+    #        = [-log(Y_MAX + EPSILON), -log(1 - Y_MAX + EPSILON),
+    #           -log(Y_MAX + EPSILON), -log(1)]
+    #        = [(0 + 15.33) / 2, (0 + 0) / 2]
+    # Weighted metric = [7.665 * 1.5, 0]
+    # Reduced metric = 7.665 * 1.5 / (1.5 + 2)
+
+    self.assertAllClose(self.evaluate(result), 3.285, atol=1e-3)
+
+  def test_weighted_from_logits(self):
+    bce_obj = metrics.BinaryCrossentropy(from_logits=True)
+    self.evaluate(variables.variables_initializer(bce_obj.variables))
+    y_true = constant_op.constant([[1, 0, 1], [0, 1, 1]])
+    y_pred = constant_op.constant([[100.0, -100.0, 100.0],
+                                   [100.0, 100.0, -100.0]])
+    sample_weight = constant_op.constant([2., 2.5])
+    result = bce_obj(y_true, y_pred, sample_weight=sample_weight)
+
+    # Metric = max(x, 0) - x * z + log(1 + exp(-abs(x)))
+    #              (where x = logits and z = y_true)
+    #        = [(0 + 0 + 0) / 3, 200 / 3]
+    # Weighted metric = [0, 66.666 * 2.5]
+    # Reduced metric = 66.666 * 2.5 / (2 + 2.5)
+
+    self.assertAllClose(self.evaluate(result), 37.037, atol=1e-3)
+
+  def test_label_smoothing(self):
+    logits = constant_op.constant(((100., -100., -100.)))
+    y_true = constant_op.constant(((1, 0, 1)))
+    label_smoothing = 0.1
+    # Metric: max(x, 0) - x * z + log(1 + exp(-abs(x)))
+    #             (where x = logits and z = y_true)
+    # Label smoothing: z' = z * (1 - L) + 0.5L
+    # After label smoothing, label 1 becomes 1 - 0.5L
+    #                        label 0 becomes 0.5L
+    # Applying the above two fns to the given input:
+    # (100 - 100 * (1 - 0.5 L)  + 0 +
+    #  0   + 100 * (0.5 L)      + 0 +
+    #  0   + 100 * (1 - 0.5 L)  + 0) * (1/3)
+    #  = (100 + 50L) * 1/3
+    bce_obj = metrics.BinaryCrossentropy(
+        from_logits=True, label_smoothing=label_smoothing)
+    self.evaluate(variables.variables_initializer(bce_obj.variables))
+    result = bce_obj(y_true, logits)
+    expected_value = (100.0 + 50.0 * label_smoothing) / 3.0
+    self.assertAllClose(expected_value, self.evaluate(result), atol=1e-3)
+
+
 def _get_model(compile_metrics):
   model_layers = [
       layers.Dense(3, activation='relu', kernel_initializer='ones'),
-- 
GitLab


From f6b3d83c1112686658c0e6d34c04eef545240056 Mon Sep 17 00:00:00 2001
From: Alex Ilchenko <ilch@google.com>
Date: Thu, 17 Jan 2019 12:15:41 -0800
Subject: [PATCH 0874/2345] Add TPU support for resize using nearest neighbor

PiperOrigin-RevId: 229795283
---
 tensorflow/compiler/tests/image_ops_test.py   | 115 +++++-
 .../tf2xla/kernels/image_resize_ops.cc        | 331 ++++++++++--------
 2 files changed, 304 insertions(+), 142 deletions(-)

diff --git a/tensorflow/compiler/tests/image_ops_test.py b/tensorflow/compiler/tests/image_ops_test.py
index 0e2d840418..12741c4d4a 100644
--- a/tensorflow/compiler/tests/image_ops_test.py
+++ b/tensorflow/compiler/tests/image_ops_test.py
@@ -403,6 +403,117 @@ class AdjustSaturationTest(xla_test.XLATestCase):
           self.assertAllClose(y_fused, y_baseline, rtol=2e-5, atol=1e-5)
 
 
+class ResizeNearestNeighborTest(xla_test.XLATestCase):
+  # TODO(ilch): Wrap each test with `for dtype in self.float_types:`
+  # Some work to understand how that should be done was presented here:
+  # cl/227850213
+
+  def _assertForwardOpMatchesExpected(self,
+                                      image_np,
+                                      target_shape,
+                                      expected=None,
+                                      large_tolerance=False,
+                                      align_corners=True):
+    if expected is None:
+      self.fail("expected must be specified")
+    with self.cached_session() as sess, self.test_scope():
+      image = array_ops.placeholder(image_np.dtype)
+      resized = gen_image_ops.resize_nearest_neighbor(
+          image, target_shape, align_corners=align_corners)
+      out = sess.run(resized, {image: image_np[np.newaxis, :, :, np.newaxis]})
+      if large_tolerance:
+        self.assertAllClose(
+            expected[np.newaxis, :, :, np.newaxis], out, rtol=0.03, atol=0.1)
+      else:
+        self.assertAllClose(expected[np.newaxis, :, :, np.newaxis], out)
+
+  def testAlignCorners2x2To1x1(self):
+    self._assertForwardOpMatchesExpected(
+        np.array([[1, 2], [3, 4]], dtype=np.float32), [1, 1],
+        expected=np.array([[1]], dtype=np.float32))
+
+  def testAlignCorners1x1To2x2(self):
+    self._assertForwardOpMatchesExpected(
+        np.array([[1]], dtype=np.float32), [2, 2],
+        expected=np.array([[1, 1], [1, 1]], dtype=np.float32))
+
+  def testAlignCorners1x1To3x3(self):
+    self._assertForwardOpMatchesExpected(
+        np.array([[1]], dtype=np.float32), [3, 3],
+        expected=np.array([[1, 1, 1], [1, 1, 1], [1, 1, 1]], dtype=np.float32))
+
+  def testAlignCorners2x2To3x3(self):
+    self._assertForwardOpMatchesExpected(
+        np.array([[1, 2], [3, 4]], dtype=np.float32), [3, 3],
+        expected=np.array([[1, 2, 2], [3, 4, 4], [3, 4, 4]], dtype=np.float32))
+
+  def testAlignCorners2x2To4x4(self):
+    self._assertForwardOpMatchesExpected(
+        np.array([[1, 2], [3, 4]], dtype=np.float32), [4, 4],
+        expected=np.array(
+            [[1, 1, 2, 2], [1, 1, 2, 2], [3, 3, 4, 4], [3, 3, 4, 4]],
+            dtype=np.float32))
+
+  def testAlignCorners3x3To2x2(self):
+    self._assertForwardOpMatchesExpected(
+        np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), [2, 2],
+        expected=np.array([[1, 3], [7, 9]], dtype=np.float32))
+
+  def testAlignCorners4x4To3x3(self):
+    self._assertForwardOpMatchesExpected(
+        np.array(
+            [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]],
+            dtype=np.float32), [3, 3],
+        expected=np.array([[1, 3, 4], [9, 11, 12], [13, 15, 16]],
+                          dtype=np.float32))
+
+  def testAlignCorners3x3To4x4(self):
+    self._assertForwardOpMatchesExpected(
+        np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), [4, 4],
+        expected=np.array(
+            [[1, 2, 2, 3], [4, 5, 5, 6], [4, 5, 5, 6], [7, 8, 8, 9]],
+            dtype=np.float32))
+
+  def testAlignCorners3x3To6x6(self):
+    self._assertForwardOpMatchesExpected(
+        np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), [6, 6],
+        expected=np.array(
+            [[1, 1, 2, 2, 3, 3], [1, 1, 2, 2, 3, 3], [4, 4, 5, 5, 6, 6],
+             [4, 4, 5, 5, 6, 6], [7, 7, 8, 8, 9, 9], [7, 7, 8, 8, 9, 9]],
+            dtype=np.float32))
+
+  def testAlignCorners3x3To9x9(self):
+    # The expected matrix might look uneven in terms of how many of each number
+    # there is, but this is an artifact of doing the dilation and convolution
+    # iteratively. The behavior is less esoteric in the 3x3To12x12 case below.
+    self._assertForwardOpMatchesExpected(
+        np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), [9, 9],
+        expected=np.array(
+            [[1, 2, 2, 2, 2, 3, 3, 3, 3], [4, 5, 5, 5, 5, 6, 6, 6, 6],
+             [4, 5, 5, 5, 5, 6, 6, 6, 6], [4, 5, 5, 5, 5, 6, 6, 6, 6],
+             [4, 5, 5, 5, 5, 6, 6, 6, 6], [7, 8, 8, 8, 8, 9, 9, 9, 9],
+             [7, 8, 8, 8, 8, 9, 9, 9, 9], [7, 8, 8, 8, 8, 9, 9, 9, 9],
+             [7, 8, 8, 8, 8, 9, 9, 9, 9]],
+            dtype=np.float32))
+
+  def testAlignCorners3x3To12x12(self):
+    self._assertForwardOpMatchesExpected(
+        np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), [12, 12],
+        expected=np.array([[1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3],
+                           [1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3],
+                           [1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3],
+                           [4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6],
+                           [4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6],
+                           [4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6],
+                           [4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6],
+                           [4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6],
+                           [4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6],
+                           [7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9],
+                           [7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9],
+                           [7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9]],
+                          dtype=np.float32))
+
+
 class ResizeBilinearTest(xla_test.XLATestCase):
 
   def _assertForwardOpMatchesExpected(self,
@@ -444,14 +555,14 @@ class ResizeBilinearTest(xla_test.XLATestCase):
       self.assertAllCloseAccordingToType(expected[np.newaxis, :, :, np.newaxis],
                                          out)
 
-  def testAlignCorners1x2To3x2(self):
+  def testAlignCorners1x2To3x3(self):
     for dtype in self.float_types:
       self._assertForwardOpMatchesExpected(
           np.array([[1, 2]], dtype=dtype), [3, 3],
           expected=np.array([[1, 1.5, 2], [1, 1.5, 2], [1, 1.5, 2]],
                             dtype=np.float32))
 
-  def testAlignCorners1x2To3x2Grad(self):
+  def testAlignCorners1x2To3x3Grad(self):
     for dtype in self.float_types:
       self._assertBackwardOpMatchesExpected(
           np.array([[1, 2], [3, 4], [5, 6]], dtype=np.float32),
diff --git a/tensorflow/compiler/tf2xla/kernels/image_resize_ops.cc b/tensorflow/compiler/tf2xla/kernels/image_resize_ops.cc
index 5a10c52ba8..b96d45316f 100644
--- a/tensorflow/compiler/tf2xla/kernels/image_resize_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/image_resize_ops.cc
@@ -72,10 +72,10 @@ namespace {
 // from in_size to out_size.
 struct ResizeConvolutionDims {
   // Size of the kernel to use.
-  std::vector<int64> kernel_size;
+  std::vector<int64> kernel_size;  // k
 
   // Stride of the convolution to use.
-  std::vector<int64> stride;
+  std::vector<int64> stride;  // S
 };
 ResizeConvolutionDims ComputeResizeConvolutionParameters(
     absl::Span<const int64> in_size, absl::Span<const int64> out_size,
@@ -117,8 +117,10 @@ ResizeConvolutionDims ComputeResizeConvolutionParameters(
 //                        + dims.stride * (out_size - 1)
 int64 CalculateUpperPadding(int64 in_size, int64 out_size, int64 kernel_size,
                             int64 stride) {
-  return (2 * kernel_size - 1) + (out_size - 1) * stride - (kernel_size - 1) -
-         1 - (kernel_size * (in_size - 1));
+  int64 padding = (2 * kernel_size - 1) + (out_size - 1) * stride -
+                  (kernel_size - 1) - 1 - (kernel_size * (in_size - 1));
+
+  return padding;
 }
 
 // Form a 2D convolution kernel like:
@@ -132,7 +134,7 @@ int64 CalculateUpperPadding(int64 in_size, int64 out_size, int64 kernel_size,
 // If the 2D kernel would be very large, the 1D kernel can be applied once in
 // each dimension due to the symmetry of the kernel along all axis to reduce the
 // computational intensity.
-xla::XlaOp Make1DKernel(xla::XlaBuilder* builder, int64 n) {
+xla::XlaOp MakeBilinear1DKernel(xla::XlaBuilder* builder, int64 n) {
   std::vector<float> kernel(n * 2 - 1);
   for (int64 i = 0; i < n; ++i) {
     float v = (i + 1.0f) / n;
@@ -142,43 +144,64 @@ xla::XlaOp Make1DKernel(xla::XlaBuilder* builder, int64 n) {
   return xla::ConstantR1<float>(builder, kernel);
 }
 
+// Unlike the bilinear kernel, which is triangular, the nearest neighbor
+// kernel is a square. For example, a 1D kernel with n=3 would look like
+// [0 1 1 1 0]
+// and n=4 would look like
+// [0 0 1 1 1 1 0].
+// Note that in the second case, the kernel is not symmetric and we default
+// to the right (because an existing non TPU kernel
+// for nearest neighbor resize already chose to default to the right,
+// so we want to be consistent).
+xla::XlaOp MakeNearestNeighbor1DKernel(xla::XlaBuilder* builder, int64 n) {
+  std::vector<float> kernel(n * 2 - 1, 0.0f);
+  std::fill(&kernel[n / 2], &kernel[(3 * n) / 2], 1.0f);
+
+  return xla::ConstantR1<float>(builder, kernel);
+}
+
 // Kernels with more than 16 spatial elements are considered intense and the
-// kernel should applied to each dimension independently.
+// kernel should be applied to each dimension independently.
 const int64 kMax2DKernelSize = 16;
 
-xla::XlaOp MakeBilinearResizeKernel(xla::XlaBuilder* builder,
-                                    absl::Span<const int64> kernel_size,
-                                    int64 channels) {
+xla::XlaOp MakeGeneralResizeKernel(xla::XlaBuilder* builder,
+                                   absl::Span<const int64> kernel_size,
+                                   int64 channels, bool is_kernel_bilinear) {
+  auto make_kernel_func =
+      is_kernel_bilinear ? MakeBilinear1DKernel : MakeNearestNeighbor1DKernel;
+
   auto depthwise_kernel = xla::Broadcast(
       xla::Zero(builder, xla::F32),
       {(2 * kernel_size[0] - 1), (2 * kernel_size[1] - 1), channels, 1});
 
   return xla::Mul(
-      xla::Add(depthwise_kernel, Make1DKernel(builder, kernel_size[1]),
+      xla::Add(depthwise_kernel, make_kernel_func(builder, kernel_size[1]),
                /*broadcast_dimensions=*/{1}),
-      Make1DKernel(builder, kernel_size[0]),
+      make_kernel_func(builder, kernel_size[0]),
       /*broadcast_dimensions=*/{0});
 }
 
-xla::XlaOp MakeBilinearResizeKernelInDim(xla::XlaBuilder* builder,
-                                         absl::Span<const int64> kernel_size,
-                                         int64 channels, int64 dim) {
+xla::XlaOp MakeGeneralResizeKernelInDim(xla::XlaBuilder* builder,
+                                        absl::Span<const int64> kernel_size,
+                                        int64 channels, int64 dim,
+                                        bool is_kernel_bilinear) {
+  auto make_kernel_func =
+      is_kernel_bilinear ? MakeBilinear1DKernel : MakeNearestNeighbor1DKernel;
+
   auto depthwise_kernel =
       xla::Broadcast(xla::Zero(builder, xla::F32),
                      {dim == 0 ? (2 * kernel_size[0] - 1) : 1,
                       dim == 1 ? (2 * kernel_size[1] - 1) : 1, channels, 1});
-  return xla::Add(depthwise_kernel, Make1DKernel(builder, kernel_size[dim]),
+  return xla::Add(depthwise_kernel, make_kernel_func(builder, kernel_size[dim]),
                   /*broadcast_dimensions=*/{dim});
 }
 
-xla::XlaOp ResizeUsingDilationAndConvolution(xla::XlaBuilder* builder,
-                                             const xla::XlaOp& input,
-                                             const int num_spatial_dims,
-                                             std::vector<int64> in_size,
-                                             std::vector<int64> out_size,
-                                             const int64 channels,
-                                             const bool align_corners) {
-  // Picture for a 1x3 to 1x4 resize:
+xla::XlaOp ResizeUsingDilationAndConvolution(
+    xla::XlaBuilder* builder, const xla::XlaOp& input,
+    const int num_spatial_dims, std::vector<int64> in_size,
+    std::vector<int64> out_size, const int64 channels, const bool align_corners,
+    bool is_kernel_bilinear) {
+  // Picture for a 1x3 to 1x4 bilinear resize:
   // stride = 2, kernel size = 3
   // Input:
   // 3 6 9
@@ -264,8 +287,8 @@ xla::XlaOp ResizeUsingDilationAndConvolution(xla::XlaBuilder* builder,
   // Split convolutions into independent dimensions if they would be a very
   // large kernel.
   if (dims.kernel_size[0] * dims.kernel_size[1] < kMax2DKernelSize) {
-    xla::XlaOp kernel =
-        MakeBilinearResizeKernel(builder, dims.kernel_size, channels);
+    xla::XlaOp kernel = MakeGeneralResizeKernel(builder, dims.kernel_size,
+                                                channels, is_kernel_bilinear);
     output =
         xla::ConvGeneralDilated(input_data, kernel, dims.stride,
                                 /*padding=*/
@@ -275,8 +298,8 @@ xla::XlaOp ResizeUsingDilationAndConvolution(xla::XlaBuilder* builder,
                                 /*rhs_dilation=*/{1, 1}, dimension_numbers,
                                 /*feature_group_count=*/channels);
   } else {
-    xla::XlaOp kernel0 =
-        MakeBilinearResizeKernelInDim(builder, dims.kernel_size, channels, 0);
+    xla::XlaOp kernel0 = MakeGeneralResizeKernelInDim(
+        builder, dims.kernel_size, channels, 0, is_kernel_bilinear);
     output = xla::ConvGeneralDilated(
         input_data, kernel0, {dims.stride[0], 1},
         /*padding=*/
@@ -284,8 +307,8 @@ xla::XlaOp ResizeUsingDilationAndConvolution(xla::XlaBuilder* builder,
         /*lhs_dilation=*/{dims.kernel_size[0], 1},
         /*rhs_dilation=*/{1, 1}, dimension_numbers,
         /*feature_group_count=*/channels);
-    xla::XlaOp kernel1 =
-        MakeBilinearResizeKernelInDim(builder, dims.kernel_size, channels, 1);
+    xla::XlaOp kernel1 = MakeGeneralResizeKernelInDim(
+        builder, dims.kernel_size, channels, 1, is_kernel_bilinear);
     output = xla::ConvGeneralDilated(
         output, kernel1, {1, dims.stride[1]},
         /*padding=*/
@@ -306,13 +329,11 @@ xla::XlaOp ResizeUsingDilationAndConvolution(xla::XlaBuilder* builder,
   return output;
 }
 
-xla::XlaOp ResizeUsingDilationAndConvolutionGradOp(xla::XlaBuilder* builder,
-                                                   const xla::XlaOp& grad,
-                                                   const int num_spatial_dims,
-                                                   std::vector<int64> in_size,
-                                                   std::vector<int64> grad_size,
-                                                   const int64 channels,
-                                                   const bool align_corners) {
+xla::XlaOp ResizeUsingDilationAndConvolutionGradOp(
+    xla::XlaBuilder* builder, const xla::XlaOp& grad,
+    const int num_spatial_dims, std::vector<int64> in_size,
+    std::vector<int64> grad_size, const int64 channels,
+    const bool align_corners, bool is_kernel_bilinear) {
   ResizeConvolutionDims dims =
       ComputeResizeConvolutionParameters(in_size, grad_size, align_corners);
 
@@ -332,8 +353,8 @@ xla::XlaOp ResizeUsingDilationAndConvolutionGradOp(xla::XlaBuilder* builder,
   dimension_numbers.set_kernel_output_feature_dimension(num_spatial_dims);
   xla::XlaOp output;
   if (dims.kernel_size[0] * dims.kernel_size[1] < kMax2DKernelSize) {
-    xla::XlaOp kernel =
-        MakeBilinearResizeKernel(builder, dims.kernel_size, channels);
+    xla::XlaOp kernel = MakeGeneralResizeKernel(builder, dims.kernel_size,
+                                                channels, is_kernel_bilinear);
 
     // Broadcast the input kernel where the forward op expanded from a size == 1
     // dimension to a size > 1 dimension. This has the effect of summing the
@@ -355,14 +376,14 @@ xla::XlaOp ResizeUsingDilationAndConvolutionGradOp(xla::XlaBuilder* builder,
         /*rhs_dilation=*/{1, 1}, dimension_numbers,
         /*feature_group_count=*/channels);
   } else {
-    xla::XlaOp kernel0 =
-        MakeBilinearResizeKernelInDim(builder, dims.kernel_size, channels, 0);
-    xla::XlaOp kernel1 =
-        MakeBilinearResizeKernelInDim(builder, dims.kernel_size, channels, 1);
-
-    // Broadcast the input kernel where the forward op expanded from a size == 1
-    // dimension to a size > 1 dimension. This has the effect of summing the
-    // gradient contributions in that dimension.
+    xla::XlaOp kernel0 = MakeGeneralResizeKernelInDim(
+        builder, dims.kernel_size, channels, 0, is_kernel_bilinear);
+    xla::XlaOp kernel1 = MakeGeneralResizeKernelInDim(
+        builder, dims.kernel_size, channels, 1, is_kernel_bilinear);
+
+    // Broadcast the input kernel where the forward op expanded from a
+    // size == 1 dimension to a size > 1 dimension. This has the effect of
+    // summing the gradient contributions in that dimension.
     if (in_size[0] == 1 && grad_size[0] > 1) {
       kernel0 =
           xla::Add(kernel0, xla::ConstantR1<float>(builder, grad_size[0], 0),
@@ -407,109 +428,139 @@ xla::XlaOp ResizeUsingDilationAndConvolutionGradOp(xla::XlaBuilder* builder,
   return output;
 }
 
-class ResizeBilinearOp : public XlaOpKernel {
- public:
-  explicit ResizeBilinearOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {
-    OP_REQUIRES_OK(ctx, ctx->GetAttr("align_corners", &align_corners_));
+void GeneralCompile(XlaOpKernelContext* ctx, bool align_corners_,
+                    bool is_kernel_bilinear) {
+  xla::XlaBuilder* b = ctx->builder();
+
+  TensorShape input_shape = ctx->InputShape(0);
+  OP_REQUIRES(ctx, input_shape.dims() == 4,
+              errors::InvalidArgument("input must be 4-dimensional",
+                                      input_shape.DebugString()));
+  // First dimension always assumed to be batch
+  const int64 batch = input_shape.dim_size(0);
+  std::vector<int64> in_size = {input_shape.dim_size(1),
+                                input_shape.dim_size(2)};
+  // Last/4th dimension always assumed to be num channels
+  const int64 channels = input_shape.dim_size(3);
+  OP_REQUIRES(ctx, in_size[0] > 0 && in_size[1] > 0,
+              errors::InvalidArgument("input size must be positive, got [",
+                                      in_size[0], ",", in_size[1], "]"));
+
+  std::vector<int64> out_size;
+  OP_REQUIRES_OK(ctx, ctx->ConstantInputAsIntVector(1, &out_size));
+  OP_REQUIRES(ctx, out_size.size() == 2,
+              errors::InvalidArgument("output size must be length 2, got ",
+                                      out_size.size()));
+  OP_REQUIRES(ctx, out_size[0] > 0 && out_size[1] > 0,
+              errors::InvalidArgument("output size must be positive, got [",
+                                      out_size[0], ",", out_size[1], "]"));
+
+  const int num_spatial_dims = 2;
+
+  xla::XlaOp input = ctx->Input(0);
+
+  // If in_size[i] > 1 and out_size[i] == 1, slice out the first input in
+  // dimension i.
+  bool slice_input = false;
+  for (int i = 0; i < num_spatial_dims; ++i) {
+    if (in_size[i] > 1 && out_size[i] == 1) {
+      // If in_size[i] > 1 but out_size[i] == 1, then we slice out the first
+      // entry before resizing.
+      slice_input = true;
+      in_size[i] = 1;
+    }
+  }
+  if (slice_input) {
+    input = xla::Slice(input, {0, 0, 0, 0},
+                       {batch, in_size[0], in_size[1], channels}, {1, 1, 1, 1});
   }
 
-  void Compile(XlaOpKernelContext* ctx) override {
-    xla::XlaBuilder* b = ctx->builder();
+  // Output is always type float.
+  input = xla::ConvertElementType(input, xla::F32);
+
+  // Special Case:
+  // Instead of doing a ResizeUsingDilationAndConvolution directly,
+  // while (out_size[0]-1) = c * 2^x * (in_size[0]-1) for x>1 c>1, resize the
+  // image to 2*(in_size[0]-1)+1 x-times and then resize by scale c(int here).
+  // Instead of resizing directly we resize it iteratively.
+  //
+  // Since bilinear resize can be broken down as 2 sequential linear
+  // operations along different dimensions.
+  // Given sufficient numerical stability and a<e<c and b<f<d, bilinear resize
+  // from image of size axb -> cxd is same as resizing axb -> exf -> cxd.
+  // This does not work in the case of align_corners_=false because of special
+  // padding requirements that cause multiple resizes to be very different
+  // from a single resize.
+  //
+  // This makes the convolutions kernels smaller and the operation faster.
+  xla::XlaOp output = input;
+  while (in_size != out_size) {
+    if (in_size[0] != 1 && in_size[1] != 1) {
+      std::vector<float> k = {
+          (static_cast<float>(out_size[0]) - 1) / ((in_size[0] - 1) * 2),
+          (static_cast<float>(out_size[1]) - 1) / ((in_size[1] - 1) * 2)};
+      if ((k[0] == std::floor(k[0])) && (k[1] == std::floor(k[1])) &&
+          k[0] > 1 && k[1] > 1 && align_corners_) {
+        std::vector<int64> next_out_size = {(in_size[0] - 1) * 2 + 1,
+                                            (in_size[1] - 1) * 2 + 1};
+        output = ResizeUsingDilationAndConvolution(
+            b, input, num_spatial_dims, in_size, next_out_size, channels,
+            align_corners_, is_kernel_bilinear);
+        input = output;
+        in_size = next_out_size;
+      } else {
+        output = ResizeUsingDilationAndConvolution(
+            b, input, num_spatial_dims, in_size, out_size, channels,
+            align_corners_, is_kernel_bilinear);
+        in_size = out_size;
+      }
+    } else {
+      output = ResizeUsingDilationAndConvolution(
+          b, input, num_spatial_dims, in_size, out_size, channels,
+          align_corners_, is_kernel_bilinear);
+      in_size = out_size;
+    }
+  }
 
-    TensorShape input_shape = ctx->InputShape(0);
-    OP_REQUIRES(ctx, input_shape.dims() == 4,
-                errors::InvalidArgument("input must be 4-dimensional",
-                                        input_shape.DebugString()));
-    const int64 batch = input_shape.dim_size(0);
-    std::vector<int64> in_size = {input_shape.dim_size(1),
-                                  input_shape.dim_size(2)};
-    const int64 channels = input_shape.dim_size(3);
-    OP_REQUIRES(ctx, in_size[0] > 0 && in_size[1] > 0,
-                errors::InvalidArgument("input size must be positive, got [",
-                                        in_size[0], ",", in_size[1], "]"));
+  ctx->SetOutput(0, output);
+}
 
-    std::vector<int64> out_size;
-    OP_REQUIRES_OK(ctx, ctx->ConstantInputAsIntVector(1, &out_size));
-    OP_REQUIRES(ctx, out_size.size() == 2,
-                errors::InvalidArgument("output size must be length 2, got ",
-                                        out_size.size()));
-    OP_REQUIRES(ctx, out_size[0] > 0 && out_size[1] > 0,
-                errors::InvalidArgument("output size must be positive, got [",
-                                        out_size[0], ",", out_size[1], "]"));
+class ResizeNearestNeighborOp : public XlaOpKernel {
+ public:
+  explicit ResizeNearestNeighborOp(OpKernelConstruction* ctx)
+      : XlaOpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("align_corners", &align_corners_));
+    OP_REQUIRES(
+        ctx, align_corners_ == true,
+        errors::Unimplemented("ResizeNearestNeighbor with align_corners=False "
+                              "is not yet implemented"));
+  }
 
-    const int num_spatial_dims = 2;
+  void Compile(XlaOpKernelContext* ctx) override {
+    GeneralCompile(ctx, align_corners_, is_kernel_bilinear_);
+  }
 
-    xla::XlaOp input = ctx->Input(0);
+ private:
+  bool align_corners_ = true;
+  bool is_kernel_bilinear_ = false;
+};
 
-    // If in_size[i] > 1 and out_size[i] == 1, slice out the first input in
-    // dimension i.
-    bool slice_input = false;
-    for (int i = 0; i < num_spatial_dims; ++i) {
-      if (in_size[i] > 1 && out_size[i] == 1) {
-        // If in_size[i] > 1 but out_size[i] == 1, then we slice out the first
-        // entry before resizing.
-        slice_input = true;
-        in_size[i] = 1;
-      }
-    }
-    if (slice_input) {
-      input =
-          xla::Slice(input, {0, 0, 0, 0},
-                     {batch, in_size[0], in_size[1], channels}, {1, 1, 1, 1});
-    }
+REGISTER_XLA_OP(Name("ResizeNearestNeighbor").CompileTimeConstantInput("size"),
+                ResizeNearestNeighborOp);
 
-    // Output is always type float.
-    input = xla::ConvertElementType(input, xla::F32);
-
-    // Special Case:
-    // Instead of doing a ResizeUsingDilationAndConvolution directly,
-    // while (out_size[0]-1) = c * 2^x * (in_size[0]-1) for x>1 c>1, resize the
-    // image to 2*(in_size[0]-1)+1 x-times and then resize by scale c(int here).
-    // Instead of resizing directly we resize it iteratively.
-    //
-    // Since bilinear resize can be broken down as 2 sequential linear
-    // operations along different dimensions.
-    // Given sufficient numerical stability and a<e<c and b<f<d, bilinear resize
-    // from image of size axb -> cxd is same as resizing axb -> exf -> cxd.
-    // This does not work in the case of align_corners_=false because of special
-    // padding requirements that cause multiple resizes to be very different
-    // from a single resize.
-    //
-    // This makes the convolutions kernels smaller and the operation faster.
-    xla::XlaOp output = input;
-    while (in_size != out_size) {
-      if (in_size[0] != 1 && in_size[1] != 1) {
-        std::vector<float> k = {
-            (static_cast<float>(out_size[0]) - 1) / ((in_size[0] - 1) * 2),
-            (static_cast<float>(out_size[1]) - 1) / ((in_size[1] - 1) * 2)};
-        if ((k[0] == std::floor(k[0])) && (k[1] == std::floor(k[1])) &&
-            k[0] > 1 && k[1] > 1 && align_corners_) {
-          std::vector<int64> next_out_size = {(in_size[0] - 1) * 2 + 1,
-                                              (in_size[1] - 1) * 2 + 1};
-          output = ResizeUsingDilationAndConvolution(b, input, num_spatial_dims,
-                                                     in_size, next_out_size,
-                                                     channels, align_corners_);
-          input = output;
-          in_size = next_out_size;
-        } else {
-          output = ResizeUsingDilationAndConvolution(b, input, num_spatial_dims,
-                                                     in_size, out_size,
-                                                     channels, align_corners_);
-          in_size = out_size;
-        }
-      } else {
-        output = ResizeUsingDilationAndConvolution(b, input, num_spatial_dims,
-                                                   in_size, out_size, channels,
-                                                   align_corners_);
-        in_size = out_size;
-      }
-    }
+class ResizeBilinearOp : public XlaOpKernel {
+ public:
+  explicit ResizeBilinearOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("align_corners", &align_corners_));
+  }
 
-    ctx->SetOutput(0, output);
+  void Compile(XlaOpKernelContext* ctx) override {
+    GeneralCompile(ctx, align_corners_, is_kernel_bilinear_);
   }
 
  private:
-  bool align_corners_;
+  bool align_corners_ = true;
+  bool is_kernel_bilinear_ = true;
 };
 
 REGISTER_XLA_OP(Name("ResizeBilinear").CompileTimeConstantInput("size"),
@@ -581,19 +632,19 @@ class ResizeBilinearGradOp : public XlaOpKernel {
                                                (in_size[1] - 1) * 2 + 1};
           output = ResizeUsingDilationAndConvolutionGradOp(
               b, grad, num_spatial_dims, in_size, next_grad_size, channels,
-              align_corners_);
+              align_corners_, true);
           grad = output;
           in_size = next_grad_size;
         } else {
           output = ResizeUsingDilationAndConvolutionGradOp(
               b, grad, num_spatial_dims, in_size, grad_size, channels,
-              align_corners_);
+              align_corners_, true);
           in_size = grad_size;
         }
       } else {
         output = ResizeUsingDilationAndConvolutionGradOp(
             b, grad, num_spatial_dims, in_size, grad_size, channels,
-            align_corners_);
+            align_corners_, true);
         in_size = grad_size;
       }
     }
-- 
GitLab


From 66ae27d95e18f1abd728dfbc0ce4c876b8a4f219 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Thu, 17 Jan 2019 12:26:18 -0800
Subject: [PATCH 0875/2345] TensorScatter* ops should forward the correct
 input, and set output when forwarding.

PiperOrigin-RevId: 229796887
---
 tensorflow/core/kernels/scatter_nd_op.cc              |  4 +++-
 tensorflow/python/kernel_tests/scatter_nd_ops_test.py | 11 +++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/scatter_nd_op.cc b/tensorflow/core/kernels/scatter_nd_op.cc
index 1b1c59cf34..50e4c66b7e 100644
--- a/tensorflow/core/kernels/scatter_nd_op.cc
+++ b/tensorflow/core/kernels/scatter_nd_op.cc
@@ -197,7 +197,7 @@ class TensorScatterOp : public OpKernel {
     }
 
     std::unique_ptr<Tensor> forwarded_input = c->forward_input(
-        2, 0, input.dtype(), shape, DEVICE_MEMORY, AllocatorAttributes());
+        0, 0, input.dtype(), shape, DEVICE_MEMORY, AllocatorAttributes());
 
     if (forwarded_input == nullptr) {
       // We were not able to forward the input, so we deep copy the tensor and
@@ -215,6 +215,8 @@ class TensorScatterOp : public OpKernel {
       OP_REQUIRES_OK(c, functor::DoScatterNd<Device, T, Index, op>(
                             c, indices, updates, shape, forwarded_input.get(),
                             false /*allocate*/));
+
+      c->set_output(0, *forwarded_input);
     }
   }
 };
diff --git a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
index f30b5315aa..88f7b27b77 100644
--- a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
+++ b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
@@ -24,6 +24,7 @@ import numpy as np
 
 from tensorflow.python.client import session
 from tensorflow.python.eager import context
+from tensorflow.python.eager import def_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -750,6 +751,16 @@ class ScatterNdTensorTest(test.TestCase):
       self.assertLess(err_added_wrt_updates, 2e-4)
       self.assertLess(err_subbed_wrt_updates, 2e-4)
 
+  def testTensorScatterUpdateWithForwarding(self):
+    @def_function.function
+    def _TestFn():
+      indices = constant_op.constant([[4], [3], [1], [7]])
+      updates = constant_op.constant([9, 10, 11, 12], dtype=dtypes.float32)
+      t = array_ops.ones([8], dtype=dtypes.float32)
+
+      return array_ops.tensor_scatter_update(t, indices, updates)
+
+    self.assertAllEqual(_TestFn(), [1, 11, 1, 10, 9, 1, 1, 12])
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From bc7cf8062dc88b55130bc9b786707fca22c75206 Mon Sep 17 00:00:00 2001
From: Alan Chiao <alanchiao@google.com>
Date: Thu, 17 Jan 2019 12:32:26 -0800
Subject: [PATCH 0876/2345] Replace runtime memory alignment checks with debug
 time checks.

average 7.5% performance improvement across different models with hybrid inference.

PiperOrigin-RevId: 229797803
---
 .../lite/kernels/internal/optimized/neon_tensor_utils.cc  | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/lite/kernels/internal/optimized/neon_tensor_utils.cc b/tensorflow/lite/kernels/internal/optimized/neon_tensor_utils.cc
index 3e7b6c22be..730d9b662a 100644
--- a/tensorflow/lite/kernels/internal/optimized/neon_tensor_utils.cc
+++ b/tensorflow/lite/kernels/internal/optimized/neon_tensor_utils.cc
@@ -145,8 +145,8 @@ void NeonMatrixBatchVectorMultiplyAccumulate(
         // Load 16 8-bit values from the row and vector, each, to operate on.
         // Here the assumption is that each buffer is 4-byte aligned. Otherwise,
         // performance may suffer significantly.
-        TFLITE_CHECK_EQ((uintptr_t)(&row_ptr[col]) & (kWeightsPerUint32 - 1),
-                        0);
+        TFLITE_DCHECK_EQ(  // NOLINT
+            (uintptr_t)(&row_ptr[col]) & (kWeightsPerUint32 - 1), 0);
         const int8x16_t s1_8x16 = vld1q_s8((const int8_t*)(aligned_vec + col));
         const int8x16_t s2_8x16 = vld1q_s8((const int8_t*)(row_ptr + col));
         // Multiply the low bits (i.e. the lower 8 8bit numbers in the
@@ -174,8 +174,8 @@ void NeonMatrixBatchVectorMultiplyAccumulate(
           // Load 8 8-bit values from the row and column each to operate on.
           // Here the assumption is that each buffer is 4-bytes aligned.
           // Otherwise, performance may suffer significantly.
-          TFLITE_CHECK_EQ((uintptr_t)(&row_ptr[col]) & (kWeightsPerUint32 - 1),
-                          0);
+          TFLITE_DCHECK_EQ(  // NOLINT
+              (uintptr_t)(&row_ptr[col]) & (kWeightsPerUint32 - 1), 0);
           const int8x8_t s1_8x8 = vld1_s8((const int8_t*)(aligned_vec + col));
           const int8x8_t s2_8x8 = vld1_s8((const int8_t*)(row_ptr + col));
           const int16x8_t prod_16x8 = vmull_s8(s1_8x8, s2_8x8);
-- 
GitLab


From a851fa125b0ddaf91e64a279f568d8ff01f6f76f Mon Sep 17 00:00:00 2001
From: Yanan Cao <ycao@google.com>
Date: Thu, 17 Jan 2019 12:33:39 -0800
Subject: [PATCH 0877/2345] Add a warning for dump_graph when dirname is not
 specified

PiperOrigin-RevId: 229798002
---
 tensorflow/core/util/dump_graph.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/core/util/dump_graph.cc b/tensorflow/core/util/dump_graph.cc
index 523d37ecc2..d275e076f8 100644
--- a/tensorflow/core/util/dump_graph.cc
+++ b/tensorflow/core/util/dump_graph.cc
@@ -84,6 +84,10 @@ string WriteTextProtoToUniqueFile(Env* env, const string& name,
     dir = getenv("TF_DUMP_GRAPH_PREFIX");
   }
   if (!dir) {
+    LOG(WARNING)
+        << "Failed to dump " << name << " because dump location is not "
+        << " specified through either TF_DUMP_GRAPH_PREFIX environment "
+        << "variable or function argument.";
     return "(TF_DUMP_GRAPH_PREFIX not specified)";
   }
   Status status = env->RecursivelyCreateDir(dir);
-- 
GitLab


From 155385b86579489f92c0f5643c8af97364fe5d24 Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne <reedwm@google.com>
Date: Thu, 17 Jan 2019 12:38:07 -0800
Subject: [PATCH 0878/2345] Automated rollback of commit
 7e090f6a5412fd5f22b5b75ca7ae7844c3d025e9

PiperOrigin-RevId: 229798646
---
 tensorflow/core/kernels/unicode_script_op.cc | 3 ---
 third_party/icu/BUILD.bazel                  | 1 -
 2 files changed, 4 deletions(-)

diff --git a/tensorflow/core/kernels/unicode_script_op.cc b/tensorflow/core/kernels/unicode_script_op.cc
index f01bb52293..085e397eba 100644
--- a/tensorflow/core/kernels/unicode_script_op.cc
+++ b/tensorflow/core/kernels/unicode_script_op.cc
@@ -17,9 +17,6 @@ limitations under the License.
 #include "unicode/uscript.h"  // TF:icu
 #include "tensorflow/core/framework/op_kernel.h"
 
-// ICU codemap data is linked statically.
-#define U_STATIC_IMPLEMENTATION
-
 namespace tensorflow {
 
 class UnicodeScriptOp : public OpKernel {
diff --git a/third_party/icu/BUILD.bazel b/third_party/icu/BUILD.bazel
index 399962d249..36d6b9006b 100644
--- a/third_party/icu/BUILD.bazel
+++ b/third_party/icu/BUILD.bazel
@@ -44,7 +44,6 @@ cc_library(
     ]),
     copts = [
         "-DU_COMMON_IMPLEMENTATION",
-        "-DU_STATIC_IMPLEMENTATION",
         "-DU_HAVE_STD_ATOMICS",
     ] + select({
         ":android": [
-- 
GitLab


From 7b690b1687fb506ad9e7faabe5095bb266a0f68f Mon Sep 17 00:00:00 2001
From: Frank Chen <frankchn@google.com>
Date: Thu, 17 Jan 2019 12:42:39 -0800
Subject: [PATCH 0879/2345] Return the task_type if the Cluster Resolver is in
 "grpc" mode

PiperOrigin-RevId: 229799270
---
 .../python/distribute/cluster_resolver/tpu_cluster_resolver.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
index 72b9990701..635948fd19 100644
--- a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
+++ b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
@@ -378,7 +378,8 @@ class TPUClusterResolver(ClusterResolver):
     return self.master()
 
   def get_job_name(self):
-    if self._shouldResolve():
+    if (self._shouldResolve() or
+        self._tpu.startswith(compat.as_bytes('grpc://'))):
       return self.task_type
 
   def cluster_spec(self):
-- 
GitLab


From 094ae1c04d3c021e7e0048c91204a04bb08d3eb0 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Thu, 17 Jan 2019 12:48:01 -0800
Subject: [PATCH 0880/2345] Remove useless comments.

PiperOrigin-RevId: 229800138
---
 tensorflow/core/lib/gif/gif_io.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/lib/gif/gif_io.cc b/tensorflow/core/lib/gif/gif_io.cc
index 3fad8c8b14..24aca854eb 100644
--- a/tensorflow/core/lib/gif/gif_io.cc
+++ b/tensorflow/core/lib/gif/gif_io.cc
@@ -94,8 +94,8 @@ uint8* Decode(const void* srcdata, int datasize,
   }
 
   const int num_frames = gif_file->ImageCount;
-  const int width = max_frame_width;    // gif_file->SWidth;
-  const int height = max_frame_height;  // gif_file->SHeight;
+  const int width = max_frame_width;
+  const int height = max_frame_height;
   const int channel = 3;
 
   uint8* const dstdata = allocate_output(num_frames, width, height, channel);
-- 
GitLab


From 76c2a2512c3fefe065415e338a98c1043acbb131 Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Thu, 17 Jan 2019 12:48:12 -0800
Subject: [PATCH 0881/2345] Use more specific assertions from absl base class.

PiperOrigin-RevId: 229800181
---
 tensorflow/python/module/module_test.py | 66 ++++++++++++-------------
 1 file changed, 33 insertions(+), 33 deletions(-)

diff --git a/tensorflow/python/module/module_test.py b/tensorflow/python/module/module_test.py
index e7c736696e..d5d36a7bbc 100644
--- a/tensorflow/python/module/module_test.py
+++ b/tensorflow/python/module/module_test.py
@@ -143,64 +143,64 @@ class VariableTrackingTest(test.TestCase):
 
   def test_variables(self):
     m = RecursiveModule(3)
-    self.assertEqual(m.variables, (m.w, m.child.w, m.child.child.w))
-    self.assertEqual(m.child.variables, (m.child.w, m.child.child.w))
-    self.assertEqual(m.child.child.variables, (m.child.child.w,))
+    self.assertCountEqual(m.variables, (m.w, m.child.w, m.child.child.w))
+    self.assertCountEqual(m.child.variables, (m.child.w, m.child.child.w))
+    self.assertCountEqual(m.child.child.variables, (m.child.child.w,))
 
   def test_owned_variables(self):
     m = RecursiveModule(3)
-    self.assertEqual(m.owned_variables, (m.w,))
-    self.assertEqual(m.child.owned_variables, (m.child.w,))
-    self.assertEqual(m.child.child.owned_variables, (m.child.child.w,))
+    self.assertCountEqual(m.owned_variables, (m.w,))
+    self.assertCountEqual(m.child.owned_variables, (m.child.w,))
+    self.assertCountEqual(m.child.child.owned_variables, (m.child.child.w,))
 
   def test_trainable_variables(self):
     m = RecursiveModule(3)
-    self.assertEqual(m.trainable_variables,
-                     (m.w, m.child.w, m.child.child.w))
-    self.assertEqual(m.child.trainable_variables,
-                     (m.child.w, m.child.child.w))
-    self.assertEqual(m.child.child.trainable_variables, (m.child.child.w,))
+    self.assertCountEqual(m.trainable_variables,
+                          (m.w, m.child.w, m.child.child.w))
+    self.assertCountEqual(m.child.trainable_variables,
+                          (m.child.w, m.child.child.w))
+    self.assertCountEqual(m.child.child.trainable_variables, (m.child.child.w,))
 
   def test_trainable_variables_ignores_non_trainable(self):
     m = RecursiveModule(3, trainable=False)
-    self.assertEqual(len(m.trainable_variables), 0)
-    self.assertEqual(len(m.child.trainable_variables), 0)
-    self.assertEqual(len(m.child.child.trainable_variables), 0)
+    self.assertEmpty(m.trainable_variables)
+    self.assertEmpty(m.child.trainable_variables)
+    self.assertEmpty(m.child.child.trainable_variables)
 
   def test_owned_trainable_variables(self):
     m = RecursiveModule(3)
-    self.assertEqual(m.owned_trainable_variables, (m.w,))
-    self.assertEqual(m.child.owned_trainable_variables, (m.child.w,))
-    self.assertEqual(m.child.child.owned_trainable_variables,
-                     (m.child.child.w,))
+    self.assertCountEqual(m.owned_trainable_variables, (m.w,))
+    self.assertCountEqual(m.child.owned_trainable_variables, (m.child.w,))
+    self.assertCountEqual(m.child.child.owned_trainable_variables,
+                          (m.child.child.w,))
 
   def test_owned_trainable_variables_ignores_non_trainable(self):
     m = RecursiveModule(3, trainable=False)
-    self.assertEqual(len(m.owned_trainable_variables), 0)
-    self.assertEqual(len(m.child.owned_trainable_variables), 0)
-    self.assertEqual(len(m.child.child.owned_trainable_variables), 0)
+    self.assertEmpty(m.owned_trainable_variables)
+    self.assertEmpty(m.child.owned_trainable_variables)
+    self.assertEmpty(m.child.child.owned_trainable_variables)
 
 
 class ModuleTrackingTest(test.TestCase):
 
   def test_owned_submodules(self):
     m = RecursiveModule(3)
-    self.assertEqual(list(m.owned_submodules), [m.child])
-    self.assertEqual(list(m.child.owned_submodules), [m.child.child])
-    self.assertEqual(list(m.child.child.owned_submodules), [])
+    self.assertCountEqual(m.owned_submodules, [m.child])
+    self.assertCountEqual(m.child.owned_submodules, [m.child.child])
+    self.assertEmpty(list(m.child.child.owned_submodules))
 
   def test_submodules(self):
     m = RecursiveModule(3)
-    self.assertEqual(list(m.submodules), [m.child, m.child.child])
-    self.assertEqual(list(m.child.submodules), [m.child.child])
-    self.assertEqual(list(m.child.child.submodules), [])
+    self.assertCountEqual(m.submodules, [m.child, m.child.child])
+    self.assertCountEqual(m.child.submodules, [m.child.child])
+    self.assertEmpty(list(m.child.child.submodules))
 
   def test_non_ctor_submodule(self):
     m = TreeModule()
     leaf1 = m.new_leaf()
-    self.assertEqual(set(m.submodules), {leaf1})
+    self.assertCountEqual(m.submodules, (leaf1,))
     leaf2 = m.new_leaf()
-    self.assertEqual(set(m.submodules), {leaf1, leaf2})
+    self.assertCountEqual(m.submodules, (leaf1, leaf2))
 
 
 class CommonErrorsTest(test.TestCase):
@@ -323,12 +323,12 @@ class WalkTest(test.TestCase):
     parent = SimpleModule()
     child = parent.c
 
-    self.assertEqual(
-        list(module.walk(parent, predicate=IS_MEMBER)),
+    self.assertCountEqual(
+        module.walk(parent, predicate=IS_MEMBER),
         [parent.a[0], parent.a[1], parent.z])
 
-    self.assertEqual(
-        list(module.walk(parent, recurse_if=IS_MODULE, predicate=IS_MEMBER)),
+    self.assertCountEqual(
+        module.walk(parent, recurse_if=IS_MODULE, predicate=IS_MEMBER),
         [parent.a[0], parent.a[1], parent.z, child.a[0], child.a[1], child.z])
 
 
-- 
GitLab


From aed554b573df4387e47c8f9e7d4ff52963c6e89b Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Thu, 17 Jan 2019 12:48:46 -0800
Subject: [PATCH 0882/2345] Prevent OOMs in too large JPG multi-channel images.

PiperOrigin-RevId: 229800275
---
 tensorflow/core/lib/jpeg/jpeg_mem.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/lib/jpeg/jpeg_mem.cc b/tensorflow/core/lib/jpeg/jpeg_mem.cc
index f7a359eb5b..9e7d1e6410 100644
--- a/tensorflow/core/lib/jpeg/jpeg_mem.cc
+++ b/tensorflow/core/lib/jpeg/jpeg_mem.cc
@@ -157,7 +157,8 @@ uint8* UncompressLow(const void* srcdata, FewerArgsForCompiler* argball) {
   jpeg_calc_output_dimensions(&cinfo);
 
   int64 total_size = static_cast<int64>(cinfo.output_height) *
-                     static_cast<int64>(cinfo.output_width);
+                     static_cast<int64>(cinfo.output_width) *
+                     static_cast<int64>(cinfo.num_components);
   // Some of the internal routines do not gracefully handle ridiculously
   // large images, so fail fast.
   if (cinfo.output_width <= 0 || cinfo.output_height <= 0) {
-- 
GitLab


From 05bef43400c83ca6d27617e27f1661e14f5f05e8 Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Thu, 17 Jan 2019 12:52:46 -0800
Subject: [PATCH 0883/2345] Do not constant fold nodes with DT_VARIANT type
 outputs in XLA. XLA does not support Const nodes of type Variant. It needs to
 see the source ops for the Variant operations to be able to build its own
 representation.

PiperOrigin-RevId: 229801068
---
 .../jit/encapsulate_subgraphs_pass.cc         | 28 +++++++++++-
 .../compiler/tests/tensor_list_ops_test.py    | 29 ++++++++++---
 tensorflow/compiler/tf2xla/xla_compiler.cc    | 28 +++++++++++-
 tensorflow/core/common_runtime/function.cc    | 10 ++++-
 tensorflow/core/common_runtime/function.h     |  3 ++
 .../core/common_runtime/graph_optimizer.cc    | 10 ++++-
 .../core/common_runtime/graph_optimizer.h     | 43 ++++++++++++-------
 7 files changed, 123 insertions(+), 28 deletions(-)

diff --git a/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc b/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc
index 03aba97bbe..a21e083131 100644
--- a/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc
+++ b/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc
@@ -2535,7 +2535,33 @@ Status EncapsulateSubgraphsPass::Run(
             std::vector<int>* input_permutation,
             std::vector<int>* output_permutation, NodeDef* node) {
         // Optimize the subgraph.
-        OptimizeGraph(flr, subgraph);
+        // Do not constant fold nodes that output DT_VARIANT type tensors.
+        // XLA does not support Const nodes of Variant type since it needs
+        // to know the original ops to be able to compile them to the relevant
+        // XLA form.
+        // TODO(srbs): This filter is a little conservative. E.g. a subgraph of
+        // the form:
+        //                          Const
+        //                            |
+        // EmptyTensorList -> TensorListPushBack -> TensorListPopBack -> Op
+        //                                                  |
+        //                                        (Discard popped list)
+        //
+        // Would have been reduced to "Const -> Op" without this filter.
+        // However since we are only allowed to specify the filter at the "Node"
+        // level there is no good way to allow the above behavior. So we
+        // disallow any sort of constant folding on Variant nodes for now.
+        auto cf_consider_fn = [](const Node* n) {
+          for (const auto& output_arg : n->op_def().output_arg()) {
+            if (output_arg.type() == DT_VARIANT) {
+              return false;
+            }
+          }
+          return true;
+        };
+        GraphOptimizer::Options graph_optimizer_options;
+        graph_optimizer_options.cf_consider_fn = cf_consider_fn;
+        OptimizeGraph(flr, subgraph, graph_optimizer_options);
 
         const int num_args = input_permutation->size();
         std::vector<bool> const_args(num_args);
diff --git a/tensorflow/compiler/tests/tensor_list_ops_test.py b/tensorflow/compiler/tests/tensor_list_ops_test.py
index 5c079d595c..3b48aed60c 100644
--- a/tensorflow/compiler/tests/tensor_list_ops_test.py
+++ b/tensorflow/compiler/tests/tensor_list_ops_test.py
@@ -48,24 +48,39 @@ class ListOpsTest(xla_test.XLATestCase):
 
   def testPushPop(self):
     with self.cached_session() as sess, self.test_scope():
-      num = array_ops.placeholder(dtypes.int32)
       l = list_ops.tensor_list_reserve(
-          element_shape=(7, 15), num_elements=num, element_dtype=dtypes.float32)
+          element_shape=(7, 15), num_elements=10, element_dtype=dtypes.float32)
       l = list_ops.tensor_list_push_back(
           l, constant_op.constant(1.0, shape=(7, 15)))
       l = list_ops.tensor_list_push_back(
           l, constant_op.constant(2.0, shape=(7, 15)))
       l, e2 = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
       _, e1 = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
-      self.assertAllEqual(sess.run(e2, {num: 10}), 2.0 * np.ones((7, 15)))
-      self.assertAllEqual(sess.run(e1, {num: 10}), 1.0 * np.ones((7, 15)))
+      self.assertAllEqual(sess.run(e2), 2.0 * np.ones((7, 15)))
+      self.assertAllEqual(sess.run(e1), 1.0 * np.ones((7, 15)))
+
+  def testDoNotConstantFoldVariants(self):
+    with self.cached_session() as sess, self.test_scope():
+      val = array_ops.placeholder(dtype=dtypes.float32)
+      l = list_ops.tensor_list_reserve(
+          element_shape=(7, 15), num_elements=10, element_dtype=dtypes.float32)
+      # Note: Pushing a Placeholder will force the constant folding code
+      # to build a Const node with a DT_VARIANT output. This tests that XLA
+      # passes a cf_consider_fn which prevent folding such nodes.
+      l = list_ops.tensor_list_push_back(
+          l, array_ops.fill(value=val, dims=(7, 15)))
+      l = list_ops.tensor_list_push_back(
+          l, constant_op.constant(2.0, shape=(7, 15)))
+      l, e2 = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
+      _, e1 = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
+      self.assertAllEqual(sess.run(e2, {val: 1.0}), 2.0 * np.ones((7, 15)))
+      self.assertAllEqual(sess.run(e1, {val: 1.0}), 1.0 * np.ones((7, 15)))
 
   def testPushPopSeparateLists(self):
     with self.cached_session() as sess, self.test_scope():
-      num = array_ops.placeholder(dtypes.int32)
       l = list_ops.tensor_list_reserve(
           element_shape=scalar_shape(),
-          num_elements=num,
+          num_elements=20,
           element_dtype=dtypes.float32)
       l = list_ops.tensor_list_push_back(l, constant_op.constant(1.0))
       l2 = list_ops.tensor_list_push_back(l, constant_op.constant(2.0))
@@ -75,7 +90,7 @@ class ListOpsTest(xla_test.XLATestCase):
       l2, e22 = list_ops.tensor_list_pop_back(l2, element_dtype=dtypes.float32)
       l3, e31 = list_ops.tensor_list_pop_back(l3, element_dtype=dtypes.float32)
       l3, e32 = list_ops.tensor_list_pop_back(l3, element_dtype=dtypes.float32)
-      result = sess.run([e11, [e21, e22], [e31, e32]], {num: 20})
+      result = sess.run([e11, [e21, e22], [e31, e32]])
       self.assertEqual(result, [1.0, [2.0, 1.0], [3.0, 1.0]])
 
   def testEmptyTensorList(self):
diff --git a/tensorflow/compiler/tf2xla/xla_compiler.cc b/tensorflow/compiler/tf2xla/xla_compiler.cc
index ee461a3c07..15fd265686 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler.cc
@@ -462,8 +462,34 @@ std::unique_ptr<Graph> XlaCompiler::GetGraph(const FunctionBody* fbody) {
   opts.set_do_function_inlining(true);
   opts.set_do_constant_folding(true);
   GraphOptimizer optimizer(opts);
+  // Do not constant fold nodes that output DT_VARIANT type tensors.
+  // XLA does not support Const nodes of Variant type since it needs
+  // to know the original ops to be able to compile them to the relevant
+  // XLA form.
+  // TODO(srbs): This filter is a little conservative. E.g. a subgraph of
+  // the form:
+  //                          Const
+  //                            |
+  // EmptyTensorList -> TensorListPushBack -> TensorListPopBack -> Op
+  //                                                  |
+  //                                        (Discard popped list)
+  //
+  // Would have been reduced to "Const -> Op" without this filter.
+  // However since we are only allowed to specify the filter at the "Node"
+  // level there is no good way to allow the above behavior. So we
+  // disallow any sort of constant folding on Variant nodes for now.
+  auto cf_consider_fn = [](const Node* n) {
+    for (const auto& output_arg : n->op_def().output_arg()) {
+      if (output_arg.type() == DT_VARIANT) {
+        return false;
+      }
+    }
+    return true;
+  };
+  GraphOptimizer::Options graph_optimizer_options;
+  graph_optimizer_options.cf_consider_fn = cf_consider_fn;
   optimizer.Optimize(flib_runtime_, flib_runtime_->env(),
-                     /*device=*/nullptr, &graph, /*shape_map=*/nullptr);
+                     /*device=*/nullptr, &graph, graph_optimizer_options);
 
   return graph;
 }
diff --git a/tensorflow/core/common_runtime/function.cc b/tensorflow/core/common_runtime/function.cc
index 3df09eac29..82bbb7e689 100644
--- a/tensorflow/core/common_runtime/function.cc
+++ b/tensorflow/core/common_runtime/function.cc
@@ -786,13 +786,19 @@ void DumpGraph(StringPiece label, const Graph* g) {
   }
 }
 
-void OptimizeGraph(FunctionLibraryRuntime* lib, std::unique_ptr<Graph>* g) {
+void OptimizeGraph(FunctionLibraryRuntime* lib, std::unique_ptr<Graph>* g,
+                   const GraphOptimizer::Options& graph_optimizer_options) {
   OptimizerOptions opts;
   opts.set_do_common_subexpression_elimination(true);
   opts.set_do_function_inlining(true);
   opts.set_do_constant_folding(true);
   GraphOptimizer optimizer(opts);
-  optimizer.Optimize(lib, lib->env(), lib->device(), g, /*shape_map=*/nullptr);
+  optimizer.Optimize(lib, lib->env(), lib->device(), g,
+                     graph_optimizer_options);
+}
+
+void OptimizeGraph(FunctionLibraryRuntime* lib, std::unique_ptr<Graph>* g) {
+  OptimizeGraph(lib, g, GraphOptimizer::Options());
 }
 
 namespace {
diff --git a/tensorflow/core/common_runtime/function.h b/tensorflow/core/common_runtime/function.h
index eeca66f5d0..df884f7577 100644
--- a/tensorflow/core/common_runtime/function.h
+++ b/tensorflow/core/common_runtime/function.h
@@ -21,6 +21,7 @@ limitations under the License.
 
 #include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/common_runtime/device_mgr.h"
+#include "tensorflow/core/common_runtime/graph_optimizer.h"
 #include "tensorflow/core/common_runtime/process_function_library_runtime.h"
 #include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/graph/graph.h"
@@ -133,6 +134,8 @@ void DumpGraph(StringPiece label, const Graph* g);
 // OptimizeGraph mutates **g extensively and replaces '*g' with a
 // complete copy. Therefore, the caller should not keep any references
 // to nodes *g.
+void OptimizeGraph(FunctionLibraryRuntime* lib, std::unique_ptr<Graph>* g,
+                   const GraphOptimizer::Options& graph_optimizer_options);
 void OptimizeGraph(FunctionLibraryRuntime* lib, std::unique_ptr<Graph>* g);
 
 // Convert the Graph of a function to a GraphDef.
diff --git a/tensorflow/core/common_runtime/graph_optimizer.cc b/tensorflow/core/common_runtime/graph_optimizer.cc
index 37a979a8f1..7905944fb1 100644
--- a/tensorflow/core/common_runtime/graph_optimizer.cc
+++ b/tensorflow/core/common_runtime/graph_optimizer.cc
@@ -38,8 +38,7 @@ void GraphOptimizer::Optimize(
     std::unique_ptr<Graph>* graph,
     const std::unordered_map<string, std::vector<PartialTensorShape>>*
         shape_map,
-    const std::function<bool(const Node*)>& cse_consider_fn,
-    const std::function<bool(const Node*)>& cf_consider_fn) {
+    const NodePredicate& cse_consider_fn, const NodePredicate& cf_consider_fn) {
   Graph* g = graph->get();
   DumpGraph("Initial", g);
 
@@ -103,4 +102,11 @@ void GraphOptimizer::Optimize(
   DumpGraph("ReCopy", graph->get());
 }
 
+void GraphOptimizer::Optimize(FunctionLibraryRuntime* runtime, Env* env,
+                              Device* device, std::unique_ptr<Graph>* graph,
+                              const Options& options) {
+  Optimize(runtime, env, device, graph, options.shape_map,
+           options.cse_consider_fn, options.cf_consider_fn);
+}
+
 }  // end namespace tensorflow
diff --git a/tensorflow/core/common_runtime/graph_optimizer.h b/tensorflow/core/common_runtime/graph_optimizer.h
index 789cc56942..05150608f0 100644
--- a/tensorflow/core/common_runtime/graph_optimizer.h
+++ b/tensorflow/core/common_runtime/graph_optimizer.h
@@ -26,6 +26,28 @@ namespace tensorflow {
 
 class GraphOptimizer {
  public:
+  using NodePredicate = std::function<bool(const Node*)>;
+
+  struct Options {
+    // If not null it maps from nodes in graph to partially-known
+    // shapes of their outputs, and may be used, e.g., in the constant folding
+    // pass. The use of shape_map implies that the mapping from node name to the
+    // vector of partial shapes of its outputs is stable, i.e., no optimization
+    // pass may replace a node with a different node of the same name that has a
+    // different number of outputs, or outputs with different known shapes.
+    // TODO(b/65453533) introduce a unique way to name nodes in a graph.
+    std::unordered_map<string, std::vector<PartialTensorShape>>* shape_map =
+        nullptr;
+
+    // If not null then only nodes for which cse_consider_fn returns true will
+    // be considered for CSE.
+    NodePredicate cse_consider_fn = nullptr;
+
+    // If not null then only nodes for which cf_consider_fn returns true will be
+    // considered for CF.
+    NodePredicate cf_consider_fn = nullptr;
+  };
+
   GraphOptimizer(const OptimizerOptions& opts);
   ~GraphOptimizer();
 
@@ -34,26 +56,17 @@ class GraphOptimizer {
   // on which the 'graph' will execute. It's passed to the optimizers
   // so that they can respect constraints if any, that should be
   // respected.
-  //
-  // If shape_map is not null it maps from nodes in graph to partially-known
-  // shapes of their outputs, and may be used, e.g., in the constant folding
-  // pass. The use of shape_map implies that the mapping from node name to the
-  // vector of partial shapes of its outputs is stable, i.e., no optimization
-  // pass may replace a node with a different node of the same name that has a
-  // different number of outputs, or outputs with different known shapes.
-  // TODO(b/65453533) introduce a unique way to name nodes in a graph.
-  //
-  // If cse_consider_fn is not null then only nodes for which cse_consider_fn
-  // returns true will be considered for CSE.
-  // If cf_consider_fn is not null then only nodes for which cf_consider_fn
-  // returns true will be considered for CF.
+  void Optimize(FunctionLibraryRuntime* runtime, Env* env, Device* device,
+                std::unique_ptr<Graph>* graph,
+                const Options& graph_optimizer_options);
+  // DEPRECATED: Consider passing a GraphOptimizer::Options object instead.
   void Optimize(
       FunctionLibraryRuntime* runtime, Env* env, Device* device,
       std::unique_ptr<Graph>* graph,
       const std::unordered_map<string, std::vector<PartialTensorShape>>*
           shape_map,
-      const std::function<bool(const Node*)>& cse_consider_fn = nullptr,
-      const std::function<bool(const Node*)>& cf_consider_fn = nullptr);
+      const NodePredicate& cse_consider_fn = nullptr,
+      const NodePredicate& cf_consider_fn = nullptr);
 
   const OptimizerOptions& options() { return opts_; }
 
-- 
GitLab


From c1ccc03a08211dddc409759bfab22ed9af59078c Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Thu, 17 Jan 2019 12:53:51 -0800
Subject: [PATCH 0884/2345] [tf.data] Disabling default optimizations in
 benchmarks.

This CL:
- makes sure all benchmarks disable default optimizations
- removes redundant print statements
- various minor fixes

PiperOrigin-RevId: 229801289
---
 .../python/data/benchmarks/batch_benchmark.py | 13 +--
 .../data/benchmarks/filter_benchmark.py       | 10 +-
 .../from_tensor_slices_benchmark.py           | 26 +++---
 .../data/benchmarks/list_files_benchmark.py   |  8 +-
 .../python/data/benchmarks/map_benchmark.py   | 21 ++---
 .../python/data/benchmarks/range_benchmark.py | 17 ++--
 .../benchmarks/autotune_benchmark.py          | 16 +---
 .../benchmarks/csv_dataset_benchmark.py       |  5 +-
 .../benchmarks/map_and_batch_benchmark.py     | 91 +++++++------------
 .../benchmarks/map_vectorization_benchmark.py | 31 +++----
 .../benchmarks/matching_files_benchmark.py    | 10 +-
 .../benchmarks/optimize_benchmark.py          |  9 +-
 .../parallel_interleave_benchmark.py          |  6 +-
 .../rejection_resample_benchmark.py           |  3 +
 .../benchmarks/unbatch_benchmark.py           | 10 +-
 15 files changed, 117 insertions(+), 159 deletions(-)

diff --git a/tensorflow/python/data/benchmarks/batch_benchmark.py b/tensorflow/python/data/benchmarks/batch_benchmark.py
index e063849f70..0ccf5c57d1 100644
--- a/tensorflow/python/data/benchmarks/batch_benchmark.py
+++ b/tensorflow/python/data/benchmarks/batch_benchmark.py
@@ -42,6 +42,9 @@ class BatchBenchmark(test.Benchmark):
 
     dataset = dataset_ops.Dataset.from_tensors(sparse_placeholder).repeat(
         ).batch(batch_size_placeholder)
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
     iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
 
@@ -72,13 +75,11 @@ class BatchBenchmark(test.Benchmark):
 
         median_wall_time = np.median(deltas) / 100.0
 
-        print("Batch sparse dataset non-zeros per row: %d batch_size: %d "
-              "wall time: %f"
-              % (non_zeros_per_row, batch_size, median_wall_time))
         self.report_benchmark(
-            iters=10000, wall_time=median_wall_time,
-            name="batch_sparse_dataset_nnz_%d_batch_size_%d" % (
-                non_zeros_per_row, batch_size))
+            iters=10000,
+            wall_time=median_wall_time,
+            name="sparse_num_elements_%d_batch_size_%d" %
+            (non_zeros_per_row, batch_size))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/benchmarks/filter_benchmark.py b/tensorflow/python/data/benchmarks/filter_benchmark.py
index a6d86fe221..e0ecf19e11 100644
--- a/tensorflow/python/data/benchmarks/filter_benchmark.py
+++ b/tensorflow/python/data/benchmarks/filter_benchmark.py
@@ -36,6 +36,9 @@ class FilterBenchmark(test.Benchmark):
     with ops.Graph().as_default():
       dataset = (
           dataset_ops.Dataset.from_tensors(True).repeat(None).filter(predicate))
+      options = dataset_ops.Options()
+      options.experimental_optimization.apply_default_optimizations = False
+      dataset = dataset.with_options(options)
       iterator = dataset_ops.make_one_shot_iterator(dataset)
       next_element = iterator.get_next()
 
@@ -51,12 +54,7 @@ class FilterBenchmark(test.Benchmark):
           deltas.append(end - start)
 
         median_wall_time = np.median(deltas) / 100
-        print("Filter dataset using %s. Median wall time: %f" %
-              (name, median_wall_time))
-        self.report_benchmark(
-            iters=100,
-            wall_time=median_wall_time,
-            name=name)
+        self.report_benchmark(iters=100, wall_time=median_wall_time, name=name)
 
   def benchmarkSimpleFunction(self):
     self._benchmark(array_ops.identity, "simple_function")
diff --git a/tensorflow/python/data/benchmarks/from_tensor_slices_benchmark.py b/tensorflow/python/data/benchmarks/from_tensor_slices_benchmark.py
index d7f1a4e7af..4e5559ddba 100644
--- a/tensorflow/python/data/benchmarks/from_tensor_slices_benchmark.py
+++ b/tensorflow/python/data/benchmarks/from_tensor_slices_benchmark.py
@@ -41,6 +41,9 @@ class FromTensorSlicesBenchmark(test.Benchmark):
     dataset = (
         dataset_ops.Dataset.from_tensor_slices(input_data)
         .repeat(num_epochs + 1).batch(batch_size))
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
     iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
 
@@ -59,9 +62,6 @@ class FromTensorSlicesBenchmark(test.Benchmark):
         pass
 
     median_wall_time = np.median(deltas)
-    print("Slice/repeat/batch with sess.run() input size: %d batch size: %d "
-          "Median wall time per element: %f" % (input_size, batch_size,
-                                                median_wall_time))
     self.report_benchmark(
         iters=len(deltas),
         wall_time=median_wall_time,
@@ -77,6 +77,9 @@ class FromTensorSlicesBenchmark(test.Benchmark):
     dataset = (
         dataset_ops.Dataset.from_tensor_slices(input_data)
         .repeat(num_epochs + 1).batch(batch_size))
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
     iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
 
@@ -96,10 +99,6 @@ class FromTensorSlicesBenchmark(test.Benchmark):
         pass
 
     median_wall_time = np.median(deltas)
-    print(
-        "Slice/repeat/batch with callable input size: %d batch size: %d Median"
-        " wall time per element: %f" % (input_size, batch_size,
-                                        median_wall_time))
     self.report_benchmark(
         iters=len(deltas),
         wall_time=median_wall_time,
@@ -116,6 +115,9 @@ class FromTensorSlicesBenchmark(test.Benchmark):
     dataset = (
         dataset_ops.Dataset.from_tensor_slices(input_data.reshape(100, 100))
         .repeat(num_epochs + 1))
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
     iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
 
@@ -135,9 +137,6 @@ class FromTensorSlicesBenchmark(test.Benchmark):
         pass
 
     median_wall_time = np.median(deltas)
-    print("Reshape/slice/repeat with callable input size: %d batch size: %d "
-          "Median wall time per element: %f" % (input_size, batch_size,
-                                                median_wall_time))
     self.report_benchmark(
         iters=len(deltas),
         wall_time=median_wall_time,
@@ -154,6 +153,9 @@ class FromTensorSlicesBenchmark(test.Benchmark):
     dataset = (
         dataset_ops.Dataset.from_tensor_slices(input_data).batch(batch_size)
         .cache().repeat(num_epochs + 1))
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
     iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
 
@@ -173,10 +175,6 @@ class FromTensorSlicesBenchmark(test.Benchmark):
         pass
 
     median_wall_time = np.median(deltas)
-    print(
-        "Slice/batch/cache/repeat with callable input size: %d batch size: %d "
-        "Median wall time per element: %f"
-        % (input_size, batch_size, median_wall_time))
     self.report_benchmark(
         iters=len(deltas),
         wall_time=median_wall_time,
diff --git a/tensorflow/python/data/benchmarks/list_files_benchmark.py b/tensorflow/python/data/benchmarks/list_files_benchmark.py
index 0dc2147112..70f8eeec9e 100644
--- a/tensorflow/python/data/benchmarks/list_files_benchmark.py
+++ b/tensorflow/python/data/benchmarks/list_files_benchmark.py
@@ -58,6 +58,9 @@ class ListFilesBenchmark(test.Benchmark):
     for _ in range(iters):
       with ops.Graph().as_default():
         dataset = dataset_ops.Dataset.list_files(patterns)
+        options = dataset_ops.Options()
+        options.experimental_optimization.apply_default_optimizations = False
+        dataset = dataset.with_options(options)
         next_element = dataset.make_one_shot_iterator().get_next()
         with session.Session() as sess:
           sub_deltas = []
@@ -71,11 +74,6 @@ class ListFilesBenchmark(test.Benchmark):
               break
           deltas.append(sub_deltas)
     median_deltas = np.median(deltas, axis=0)
-    print('Nested directory size (width*depth): %d*%d Median wall time: '
-          '%fs (read first filename), %fs (read second filename), avg %fs'
-          ' (read %d more filenames)' %
-          (width, depth, median_deltas[0], median_deltas[1],
-           np.average(median_deltas[2:]), len(median_deltas) - 2))
     self.report_benchmark(
         iters=iters,
         wall_time=np.sum(median_deltas),
diff --git a/tensorflow/python/data/benchmarks/map_benchmark.py b/tensorflow/python/data/benchmarks/map_benchmark.py
index 65d945cdae..b620eaaed5 100644
--- a/tensorflow/python/data/benchmarks/map_benchmark.py
+++ b/tensorflow/python/data/benchmarks/map_benchmark.py
@@ -38,17 +38,14 @@ class MapBenchmark(test.Benchmark):
         if mode == "general":
           map_fn = lambda x: x + 1
           use_inter_op_parallelism = True
-          print_label = ""
           benchmark_label = ""
         if mode == "single-threaded":
           map_fn = lambda x: x + 1
           use_inter_op_parallelism = False
-          print_label = " (single threaded mode)"
           benchmark_label = "_single_threaded"
         if mode == "short-circuit":
           map_fn = lambda x: x
           use_inter_op_parallelism = True  # should not have any significance
-          print_label = " (short circuit mode)"
           benchmark_label = "_short_circuit"
 
         with ops.Graph().as_default():
@@ -58,6 +55,9 @@ class MapBenchmark(test.Benchmark):
                 dataset,
                 map_fn,
                 use_inter_op_parallelism=use_inter_op_parallelism)
+          options = dataset_ops.Options()
+          options.experimental_optimization.apply_default_optimizations = False
+          dataset = dataset.with_options(options)
           iterator = dataset_ops.make_one_shot_iterator(dataset)
           next_element = iterator.get_next()
 
@@ -73,13 +73,10 @@ class MapBenchmark(test.Benchmark):
               deltas.append(end - start)
 
             median_wall_time = np.median(deltas) / 100
-            print("Map dataset chain length%s: %d Median wall time: %f" %
-                  (print_label, chain_length, median_wall_time))
             self.report_benchmark(
                 iters=1000,
                 wall_time=median_wall_time,
-                name="map_dataset_chain_length_%d%s" % (chain_length,
-                                                        benchmark_label))
+                name="chain_length_%d%s" % (chain_length, benchmark_label))
 
   def benchmarkMapFanOut(self):
     fan_outs = [1, 2, 5, 10, 20, 50, 100]
@@ -88,17 +85,14 @@ class MapBenchmark(test.Benchmark):
         if mode == "general":
           map_fn = lambda *xs: [x + 1 for x in xs]
           use_inter_op_parallelism = True
-          print_label = ""
           benchmark_label = ""
         if mode == "single-threaded":
           map_fn = lambda *xs: [x + 1 for x in xs]
           use_inter_op_parallelism = False
-          print_label = " (single threaded mode)"
           benchmark_label = "_single_threaded"
         if mode == "short-circuit":
           map_fn = lambda *xs: xs
           use_inter_op_parallelism = True  # should not have any significance
-          print_label = " (short circuit mode)"
           benchmark_label = "_short_circuit"
 
         with ops.Graph().as_default():
@@ -108,6 +102,9 @@ class MapBenchmark(test.Benchmark):
               dataset,
               map_fn,
               use_inter_op_parallelism=use_inter_op_parallelism)
+          options = dataset_ops.Options()
+          options.experimental_optimization.apply_default_optimizations = False
+          dataset = dataset.with_options(options)
           iterator = dataset_ops.make_one_shot_iterator(dataset)
           next_element = iterator.get_next()
 
@@ -123,12 +120,10 @@ class MapBenchmark(test.Benchmark):
               deltas.append(end - start)
 
             median_wall_time = np.median(deltas) / 100
-            print("Map dataset fan out%s: %d Median wall time: %f" %
-                  (print_label, fan_out, median_wall_time))
             self.report_benchmark(
                 iters=1000,
                 wall_time=median_wall_time,
-                name="map_dataset_fan_out_%d%s" % (fan_out, benchmark_label))
+                name="fan_out_%d%s" % (fan_out, benchmark_label))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/benchmarks/range_benchmark.py b/tensorflow/python/data/benchmarks/range_benchmark.py
index a5020e2873..375ff339a8 100644
--- a/tensorflow/python/data/benchmarks/range_benchmark.py
+++ b/tensorflow/python/data/benchmarks/range_benchmark.py
@@ -31,14 +31,16 @@ class RangeBenchmark(test.Benchmark):
 
   def _benchmarkRangeHelper(self, modeling_enabled):
     num_elements = 10000000 if modeling_enabled else 50000000
-    options = dataset_ops.Options()
-    options.experimental_autotune = modeling_enabled
 
     # Use `Dataset.skip()` and `Dataset.take()` to perform the iteration in
     # C++, and focus on the minimal overheads (excluding Python invocation
     # costs).
     dataset = dataset_ops.Dataset.range(num_elements).skip(
-        num_elements - 1).take(1).with_options(options)
+        num_elements - 1).take(1)
+    options = dataset_ops.Options()
+    options.experimental_autotune = modeling_enabled
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
     iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
 
@@ -54,11 +56,10 @@ class RangeBenchmark(test.Benchmark):
       end = time.time()
 
       time_per_element = (end - start) / num_elements
-      print("Average time per element (%s modeling): %f nanoseconds" % (
-          "with" if modeling_enabled else "without", time_per_element * 1e9))
-      self.report_benchmark(iters=num_elements, wall_time=time_per_element,
-                            name="benchmark_tf_data_dataset_range%s"
-                            % ("_with_modeling" if modeling_enabled else ""))
+      self.report_benchmark(
+          iters=num_elements,
+          wall_time=time_per_element,
+          name="modeling_%s" % ("on" if modeling_enabled else "off"))
 
   def benchmarkRange(self):
     for modeling_enabled in [False, True]:
diff --git a/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py b/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py
index 391b6711e9..bda7d38792 100644
--- a/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py
@@ -60,9 +60,6 @@ class AutotuneBenchmark(test.Benchmark):
         end = time.time()
         deltas.append(end - start)
 
-    print("%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n" %
-          (np.median(deltas), np.mean(deltas), np.std(deltas), np.min(deltas),
-           np.max(deltas)))
     self.report_benchmark(
         iters=10000,
         wall_time=np.median(deltas),
@@ -87,6 +84,7 @@ class AutotuneBenchmark(test.Benchmark):
             batch_size=batch_size))
     options = dataset_ops.Options()
     options.experimental_autotune = autotune
+    options.experimental_optimization.apply_default_optimizations = False
     dataset = dataset.with_options(options)
     iterator = dataset_ops.make_one_shot_iterator(dataset)
     get_next = iterator.get_next()
@@ -101,10 +99,6 @@ class AutotuneBenchmark(test.Benchmark):
         end = time.time()
         deltas.append(end - start)
 
-    print("%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n" %
-          (np.median(deltas), np.mean(deltas), np.std(deltas), np.min(deltas),
-           np.max(deltas)))
-
     self.report_benchmark(
         iters=1000,
         wall_time=np.median(deltas),
@@ -128,6 +122,7 @@ class AutotuneBenchmark(test.Benchmark):
         num_parallel_calls=optimization.AUTOTUNE)
     options = dataset_ops.Options()
     options.experimental_autotune = autotune
+    options.experimental_optimization.apply_default_optimizations = False
     dataset = dataset.with_options(options)
     iterator = dataset_ops.make_one_shot_iterator(dataset)
     get_next = iterator.get_next()
@@ -142,9 +137,6 @@ class AutotuneBenchmark(test.Benchmark):
         end = time.time()
         deltas.append(end - start)
 
-    print("%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n" %
-          (np.median(deltas), np.mean(deltas), np.std(deltas), np.min(deltas),
-           np.max(deltas)))
     self.report_benchmark(
         iters=10000,
         wall_time=np.median(deltas),
@@ -190,6 +182,7 @@ class AutotuneBenchmark(test.Benchmark):
     dataset = dataset.map(f2, num_parallel_calls=optimization.AUTOTUNE)
     options = dataset_ops.Options()
     options.experimental_autotune = autotune
+    options.experimental_optimization.apply_default_optimizations = False
     dataset = dataset.with_options(options)
     iterator = dataset_ops.make_one_shot_iterator(dataset)
     get_next = iterator.get_next()
@@ -204,9 +197,6 @@ class AutotuneBenchmark(test.Benchmark):
         end = time.time()
         deltas.append(end - start)
 
-    print("%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n" %
-          (np.median(deltas), np.mean(deltas), np.std(deltas), np.min(deltas),
-           np.max(deltas)))
     self.report_benchmark(
         iters=1000,
         wall_time=np.median(deltas),
diff --git a/tensorflow/python/data/experimental/benchmarks/csv_dataset_benchmark.py b/tensorflow/python/data/experimental/benchmarks/csv_dataset_benchmark.py
index 03345ce4e6..2e91e08c79 100644
--- a/tensorflow/python/data/experimental/benchmarks/csv_dataset_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/csv_dataset_benchmark.py
@@ -63,6 +63,9 @@ class CsvDatasetBenchmark(test.Benchmark):
 
   def _runBenchmark(self, dataset, num_cols, prefix):
     dataset = dataset.skip(self._num_per_iter - 1)
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
     deltas = []
     for _ in range(10):
       next_element = dataset_ops.make_one_shot_iterator(dataset).get_next()
@@ -79,8 +82,6 @@ class CsvDatasetBenchmark(test.Benchmark):
       deltas.append(end - start)
     # Median wall time per CSV record read and decoded
     median_wall_time = np.median(deltas) / self._num_per_iter
-    print('%s num_cols: %d Median wall time: %f' % (prefix, num_cols,
-                                                    median_wall_time))
     self.report_benchmark(
         iters=self._num_per_iter,
         wall_time=median_wall_time,
diff --git a/tensorflow/python/data/experimental/benchmarks/map_and_batch_benchmark.py b/tensorflow/python/data/experimental/benchmarks/map_and_batch_benchmark.py
index b17f2bcd12..4b7c173786 100644
--- a/tensorflow/python/data/experimental/benchmarks/map_and_batch_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/map_and_batch_benchmark.py
@@ -26,7 +26,6 @@ import numpy as np
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.client import session
 from tensorflow.python.data.experimental.ops import batching
-from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -41,7 +40,7 @@ _NUMPY_RANDOM_SEED = 42
 class MapAndBatchBenchmark(test.Benchmark):
   """Benchmarks for `tf.data.experimental.map_and_batch()`."""
 
-  def benchmarkMapAndBatchDense(self):
+  def benchmarkMapAndBatch(self):
     """Measures the performance of parallelized batching."""
     shapes = [(), (10,), (10, 10), (10, 10, 10), (224, 224, 3)]
     batch_size_values = [1, 32, 64, 128, 1024]
@@ -55,6 +54,9 @@ class MapAndBatchBenchmark(test.Benchmark):
 
     dataset = dataset.apply(batching.map_and_batch(
         lambda _: dense_value, batch_size_placeholder))
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
     iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
 
@@ -88,13 +90,9 @@ class MapAndBatchBenchmark(test.Benchmark):
         median_wall_time = np.median(deltas) / 100.0
         iters = len(deltas) * 100
 
-        print("Map and batch dense dataset shape: %r batch_size: %d "
-              "wall time: %f (%d iters)"
-              % (shape, batch_size, median_wall_time, iters))
         self.report_benchmark(
             iters=iters, wall_time=median_wall_time,
-            name="benchmark_batch_dense_dataset_nnz_%d_batch_size_%d" % (
-                np.prod(shape), batch_size))
+            name="num_elements_%d_batch_size_%d" % (np.prod(shape), batch_size))
 
   def benchmarkMapAndBatchChainingVersusFusing(self):
     """Compares the performance of chaining and fusing map and batch.
@@ -128,49 +126,25 @@ class MapAndBatchBenchmark(test.Benchmark):
     def benchmark(label, series):
       """Runs benchmark the given series."""
 
-      print("%s:" % label)
-
-      def make_base_dataset(element_size):
+      def make_dataset(element_size, num_calls, batch_size):  # pylint: disable=missing-docstring
         k = 1024 * 1024
         x = constant_op.constant(np.random.rand(element_size, 4 * k))
         y = constant_op.constant(np.random.rand(4 * k, 1))
-        return dataset_ops.Dataset.range(1000000000000).map(lambda _: (x, y))
+        dataset = dataset_ops.Dataset.range(1000000000000).map(lambda _: (x, y))
+        dataset = dataset.map(
+            math_ops.matmul,
+            num_parallel_calls=num_calls).batch(batch_size=batch_size)
+        options = dataset_ops.Options()
+        options.experimental_optimization.apply_default_optimizations = False
+        return dataset.with_options(options)
 
       for num_calls, inter_op, element_size, batch_size in series:
-
         num_iters = 1024 // (
             (element_size * batch_size) // min(num_calls, inter_op))
-        fused_dataset = make_base_dataset(element_size)
-        fused_dataset = fused_dataset.map(
-            math_ops.matmul,
-            num_parallel_calls=num_calls).batch(batch_size=batch_size)
-
-        fused_iterator = dataset_ops.make_one_shot_iterator(fused_dataset)
-        fused_get_next = fused_iterator.get_next()
-
-        fused_deltas = []
-        with session.Session(
-            config=config_pb2.ConfigProto(
-                inter_op_parallelism_threads=inter_op,
-                use_per_session_threads=True)) as sess:
-
-          for _ in range(5):
-            sess.run(fused_get_next.op)
-          for _ in range(num_iters):
-            start = time.time()
-            sess.run(fused_get_next.op)
-            end = time.time()
-            fused_deltas.append(end - start)
-
-        # `map_and_batch_fusion` is optimized by default. To get the chained
-        # dataset, with have to disable it.
-        options = dataset_ops.Options()
-        options.experimental_optimization = OptimizationOptions()
-        options.experimental_optimization.map_and_batch_fusion = False
-        chained_dataset = fused_dataset.with_options(options)
+        # By default the chained map().batch() calls will not be fused.
+        chained_dataset = make_dataset(element_size, num_calls, batch_size)
         chained_iterator = dataset_ops.make_one_shot_iterator(chained_dataset)
         chained_get_next = chained_iterator.get_next()
-
         chained_deltas = []
         with session.Session(
             config=config_pb2.ConfigProto(
@@ -184,27 +158,32 @@ class MapAndBatchBenchmark(test.Benchmark):
             end = time.time()
             chained_deltas.append(end - start)
 
-        print(
-            "batch size: %d, num parallel calls: %d, inter-op parallelism: %d, "
-            "element size: %d, num iters: %d\nchained wall time: %f (median), "
-            "%f (mean), %f (stddev), %f (min), %f (max)\n  fused wall time: "
-            "%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n    "
-            "chained/fused:    %.2fx (median),    %.2fx (mean)" %
-            (batch_size, num_calls, inter_op, element_size, num_iters,
-             np.median(chained_deltas), np.mean(chained_deltas),
-             np.std(chained_deltas), np.min(chained_deltas),
-             np.max(chained_deltas), np.median(fused_deltas),
-             np.mean(fused_deltas), np.std(fused_deltas), np.min(fused_deltas),
-             np.max(fused_deltas),
-             np.median(chained_deltas) / np.median(fused_deltas),
-             np.mean(chained_deltas) / np.mean(fused_deltas)))
-
         self.report_benchmark(
             iters=num_iters,
             wall_time=np.median(chained_deltas),
             name=name("chained", label, num_calls, inter_op, element_size,
                       batch_size))
 
+        # Apply an option to the default dataset that will fuse map().batch().
+        options = dataset_ops.Options()
+        options.experimental_optimization.map_and_batch_fusion = True
+        fused_dataset = chained_dataset.with_options(options)
+        fused_iterator = dataset_ops.make_one_shot_iterator(fused_dataset)
+        fused_get_next = fused_iterator.get_next()
+        fused_deltas = []
+        with session.Session(
+            config=config_pb2.ConfigProto(
+                inter_op_parallelism_threads=inter_op,
+                use_per_session_threads=True)) as sess:
+
+          for _ in range(5):
+            sess.run(fused_get_next.op)
+          for _ in range(num_iters):
+            start = time.time()
+            sess.run(fused_get_next.op)
+            end = time.time()
+            fused_deltas.append(end - start)
+
         self.report_benchmark(
             iters=num_iters,
             wall_time=np.median(fused_deltas),
diff --git a/tensorflow/python/data/experimental/benchmarks/map_vectorization_benchmark.py b/tensorflow/python/data/experimental/benchmarks/map_vectorization_benchmark.py
index a60ba0a857..50e3a5c469 100644
--- a/tensorflow/python/data/experimental/benchmarks/map_vectorization_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/map_vectorization_benchmark.py
@@ -24,7 +24,6 @@ import numpy as np
 from tensorflow.core.example import example_pb2
 from tensorflow.core.example import feature_pb2
 from tensorflow.python.client import session
-from tensorflow.python.data.experimental.ops import optimization_options
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import nest
 from tensorflow.python.framework import constant_op
@@ -115,31 +114,27 @@ class MapVectorizationBenchmark(test.Benchmark):
 
   def _compare(self, input_dataset, map_fn, batch_size, input_size, str_id):
     num_elems = int(np.sum([np.prod(x) for x in input_size]))
-    name_template = "{}__batch_size_{}_input_element_size_{}_{}"
+    name_template = "{}_batch_size_{}_input_element_size_{}_{}"
 
-    base_dataset = input_dataset.map(map_fn).batch(batch_size)
+    unoptimized_dataset = input_dataset.map(map_fn).batch(batch_size)
 
     options = dataset_ops.Options()
-    opt_options = optimization_options.OptimizationOptions()
-    # Disable default map_and_batch_fusion optimization
-    opt_options.map_and_batch_fusion = False
-    options.experimental_optimization = opt_options
-    base_dataset = base_dataset.with_options(options)
+    options.experimental_optimization.apply_default_optimizations = False
+    unoptimized_dataset = unoptimized_dataset.with_options(options)
+    unoptimized_next = dataset_ops.make_one_shot_iterator(
+        unoptimized_dataset).get_next()
 
-    unoptimized_op = dataset_ops.make_one_shot_iterator(base_dataset).get_next()
-
-    optimized_options = dataset_ops.Options()
-    opt_options = optimization_options.OptimizationOptions()
-    opt_options.map_vectorization = True
-    optimized_options.experimental_optimization = opt_options
-    optimized = base_dataset.with_options(optimized_options)
-    optimized_op = dataset_ops.make_one_shot_iterator(optimized).get_next()
+    options = dataset_ops.Options()
+    options.experimental_optimization.map_vectorization = True
+    optimized_dataset = unoptimized_dataset.with_options(options)
+    optimized_next = dataset_ops.make_one_shot_iterator(
+        optimized_dataset).get_next()
 
     unoptimized_time = self._run(
-        unoptimized_op,
+        unoptimized_next,
         name=name_template.format(str_id, batch_size, num_elems, "unoptimized"))
     optimized_time = self._run(
-        optimized_op,
+        optimized_next,
         name=name_template.format(str_id, batch_size, num_elems, "optimized"))
 
     print("Batch size: {}\n"
diff --git a/tensorflow/python/data/experimental/benchmarks/matching_files_benchmark.py b/tensorflow/python/data/experimental/benchmarks/matching_files_benchmark.py
index c53f8dd7c5..cb5bf2946d 100644
--- a/tensorflow/python/data/experimental/benchmarks/matching_files_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/matching_files_benchmark.py
@@ -60,6 +60,9 @@ class MatchingFilesBenchmark(test.Benchmark):
     for _ in range(iters):
       with ops.Graph().as_default():
         dataset = matching_files.MatchingFilesDataset(patterns)
+        options = dataset_ops.Options()
+        options.experimental_optimization.apply_default_optimizations = False
+        dataset = dataset.with_options(options)
         next_element = dataset_ops.make_one_shot_iterator(dataset).get_next()
 
         with session.Session() as sess:
@@ -75,11 +78,6 @@ class MatchingFilesBenchmark(test.Benchmark):
           deltas.append(sub_deltas)
 
     median_deltas = np.median(deltas, axis=0)
-    print('Nested directory size (width*depth): %d*%d Median wall time: '
-          '%fs (read first filename), %fs (read second filename), avg %fs'
-          ' (read %d more filenames)' %
-          (width, depth, median_deltas[0], median_deltas[1],
-           np.average(median_deltas[2:]), len(median_deltas) - 2))
     self.report_benchmark(
         iters=iters,
         wall_time=np.sum(median_deltas),
@@ -92,7 +90,7 @@ class MatchingFilesBenchmark(test.Benchmark):
             (len(median_deltas) - 2):
                 np.average(median_deltas[2:])
         },
-        name='dataset_nested_directory(%d*%d)' %
+        name='nested_directory(%d*%d)' %
         (width, depth))
 
     shutil.rmtree(tmp_dir, ignore_errors=True)
diff --git a/tensorflow/python/data/experimental/benchmarks/optimize_benchmark.py b/tensorflow/python/data/experimental/benchmarks/optimize_benchmark.py
index 1bbee5e7a3..395a529f85 100644
--- a/tensorflow/python/data/experimental/benchmarks/optimize_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/optimize_benchmark.py
@@ -47,6 +47,7 @@ class OptimizationBenchmark(test.Benchmark):
         dataset = dataset.map(lambda x: x)
       if optimize_dataset:
         options = dataset_ops.Options()
+        options.experimental_optimization.apply_default_optimizations = False
         options.experimental_optimization.map_fusion = True
         dataset = dataset.with_options(options)
 
@@ -66,8 +67,6 @@ class OptimizationBenchmark(test.Benchmark):
 
         median_wall_time = np.median(deltas) / 100
         opt_mark = "opt" if optimize_dataset else "noopt"
-        print("Map dataset {} chain length: {} Median wall time: {}".format(
-            opt_mark, chain_length, median_wall_time))
         self.report_benchmark(
             iters=100,
             wall_time=median_wall_time,
@@ -90,6 +89,7 @@ class OptimizationBenchmark(test.Benchmark):
             lambda x: math_ops.greater_equal(x - 5, 0))
       if optimize_dataset:
         options = dataset_ops.Options()
+        options.experimental_optimization.apply_default_optimizations = False
         options.experimental_optimization.map_and_filter_fusion = True
         dataset = dataset.with_options(options)
       iterator = dataset_ops.make_one_shot_iterator(dataset)
@@ -108,8 +108,6 @@ class OptimizationBenchmark(test.Benchmark):
 
         median_wall_time = np.median(deltas) / 100
         opt_mark = "opt" if optimize_dataset else "noopt"
-        print("Map and filter dataset {} chain length: {} Median wall time: {}"
-              .format(opt_mark, chain_length, median_wall_time))
         self.report_benchmark(
             iters=100,
             wall_time=median_wall_time,
@@ -131,6 +129,7 @@ class OptimizationBenchmark(test.Benchmark):
         dataset = dataset.filter(lambda x: math_ops.greater_equal(x - 5, 0))
       if optimize_dataset:
         options = dataset_ops.Options()
+        options.experimental_optimization.apply_default_optimizations = False
         options.experimental_optimization.filter_fusion = True
         dataset = dataset.with_options(options)
 
@@ -150,8 +149,6 @@ class OptimizationBenchmark(test.Benchmark):
 
         median_wall_time = np.median(deltas) / 100
         opt_mark = "opt" if optimize_dataset else "no-opt"
-        print("Filter dataset {} chain length: {} Median wall time: {}".format(
-            opt_mark, chain_length, median_wall_time))
         self.report_benchmark(
             iters=1000,
             wall_time=median_wall_time,
diff --git a/tensorflow/python/data/experimental/benchmarks/parallel_interleave_benchmark.py b/tensorflow/python/data/experimental/benchmarks/parallel_interleave_benchmark.py
index f5108e5ff4..37375af27f 100644
--- a/tensorflow/python/data/experimental/benchmarks/parallel_interleave_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/parallel_interleave_benchmark.py
@@ -56,8 +56,10 @@ class ParallelInterleaveBenchmark(test.Benchmark):
 
   def _benchmark(self, dataset_fn, iters, num_elements):
     with ops.Graph().as_default():
-      iterator = dataset_ops.make_one_shot_iterator(dataset_fn())
-      next_element = iterator.get_next()
+      options = dataset_ops.Options()
+      options.experimental_optimization.apply_default_optimizations = False
+      dataset = dataset_fn().with_options(options)
+      next_element = dataset_ops.make_one_shot_iterator(dataset).get_next()
       with session.Session() as sess:
         deltas = []
         for _ in range(iters):
diff --git a/tensorflow/python/data/experimental/benchmarks/rejection_resample_benchmark.py b/tensorflow/python/data/experimental/benchmarks/rejection_resample_benchmark.py
index a64f7ecb00..9a8ac7ef65 100644
--- a/tensorflow/python/data/experimental/benchmarks/rejection_resample_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/rejection_resample_benchmark.py
@@ -39,6 +39,9 @@ def _time_resampling(data_np, target_dist, init_dist, num_to_sample):  # pylint:
           initial_dist=init_dist,
           seed=142))
 
+  options = dataset_ops.Options()
+  options.experimental_optimization.apply_default_optimizations = False
+  dataset = dataset.with_options(options)
   get_next = dataset_ops.make_one_shot_iterator(dataset).get_next()
 
   with session.Session() as sess:
diff --git a/tensorflow/python/data/experimental/benchmarks/unbatch_benchmark.py b/tensorflow/python/data/experimental/benchmarks/unbatch_benchmark.py
index 6f80df50b8..3f5b9b9130 100644
--- a/tensorflow/python/data/experimental/benchmarks/unbatch_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/unbatch_benchmark.py
@@ -42,6 +42,9 @@ class UnbatchBenchmark(test.Benchmark):
       dataset = dataset.batch(batch_size_placeholder)
       dataset = dataset.apply(batching.unbatch())
       dataset = dataset.skip(elems_per_trial)
+      options = dataset_ops.Options()
+      options.experimental_optimization.apply_default_optimizations = False
+      dataset = dataset.with_options(options)
       iterator = dataset_ops.make_initializable_iterator(dataset)
       next_element = iterator.get_next()
 
@@ -58,8 +61,6 @@ class UnbatchBenchmark(test.Benchmark):
             deltas.append((end - start) / elems_per_trial)
 
           median_wall_time = np.median(deltas)
-          print("Unbatch (native) batch size: %d Median wall time per element:"
-                " %f microseconds" % (batch_size, median_wall_time * 1e6))
           self.report_benchmark(
               iters=10000,
               wall_time=median_wall_time,
@@ -78,6 +79,9 @@ class UnbatchBenchmark(test.Benchmark):
       dataset = dataset.batch(batch_size_placeholder)
       dataset = dataset.flat_map(dataset_ops.Dataset.from_tensor_slices)
       dataset = dataset.skip(elems_per_trial)
+      options = dataset_ops.Options()
+      options.experimental_optimization.apply_default_optimizations = False
+      dataset = dataset.with_options(options)
       iterator = dataset_ops.make_initializable_iterator(dataset)
       next_element = iterator.get_next()
 
@@ -94,8 +98,6 @@ class UnbatchBenchmark(test.Benchmark):
             deltas.append((end - start) / elems_per_trial)
 
           median_wall_time = np.median(deltas)
-          print("Unbatch (unfused) batch size: %d Median wall time per element:"
-                " %f microseconds" % (batch_size, median_wall_time * 1e6))
           self.report_benchmark(
               iters=10000,
               wall_time=median_wall_time,
-- 
GitLab


From f602445c23feb67f2e35256ffdf24e090195fab6 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Thu, 17 Jan 2019 12:59:15 -0800
Subject: [PATCH 0885/2345] [tf.data] Replacing IteratorStateMetadata with a
 simpler alternative.

Currently, serialization of iterator state uses the `IteratorStateMetadata` proto, which AFAICT is only used to convert a map to and from string. This CL replaces the proto-based conversion with a simpler mechanism. This change is motivated by making it possible to move the VariantTensorDataReader and VariantTensorDataWriter into a header file and depending on protos from TensorFlow header files is discouraged (c.f. b/62899350).

PiperOrigin-RevId: 229802166
---
 .../contrib/cmake/tf_core_framework.cmake     |  1 -
 tensorflow/core/BUILD                         |  1 -
 tensorflow/core/framework/iterator.proto      | 18 --------
 tensorflow/core/kernels/data/iterator_ops.cc  | 41 ++++++-------------
 4 files changed, 13 insertions(+), 48 deletions(-)
 delete mode 100644 tensorflow/core/framework/iterator.proto

diff --git a/tensorflow/contrib/cmake/tf_core_framework.cmake b/tensorflow/contrib/cmake/tf_core_framework.cmake
index 9ae5831f47..d8d1cc3aa2 100644
--- a/tensorflow/contrib/cmake/tf_core_framework.cmake
+++ b/tensorflow/contrib/cmake/tf_core_framework.cmake
@@ -147,7 +147,6 @@ set(tf_proto_text_srcs
     "tensorflow/core/framework/function.proto"
     "tensorflow/core/framework/graph.proto"
     "tensorflow/core/framework/graph_transfer_info.proto"
-    "tensorflow/core/framework/iterator.proto"
     "tensorflow/core/framework/kernel_def.proto"
     "tensorflow/core/framework/log_memory.proto"
     "tensorflow/core/framework/node_def.proto"
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 6c9afc51e9..084db7a0fd 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -181,7 +181,6 @@ COMMON_PROTO_SRCS = [
     "framework/function.proto",
     "framework/graph.proto",
     "framework/graph_transfer_info.proto",
-    "framework/iterator.proto",
     "framework/kernel_def.proto",
     "framework/log_memory.proto",
     "framework/node_def.proto",
diff --git a/tensorflow/core/framework/iterator.proto b/tensorflow/core/framework/iterator.proto
deleted file mode 100644
index f015342e13..0000000000
--- a/tensorflow/core/framework/iterator.proto
+++ /dev/null
@@ -1,18 +0,0 @@
-syntax = "proto3";
-
-package tensorflow;
-option cc_enable_arenas = true;
-option java_outer_classname = "IteratorProtos";
-option java_multiple_files = true;
-option java_package = "org.tensorflow.util";
-option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework";
-
-// Protocol buffer representing the metadata for an iterator's state stored
-// as a Variant tensor.
-message IteratorStateMetadata {
-  // A user-specified version string.
-  string version = 1;
-
-  // Keys for tensors in the VariantTensorDataProto.
-  repeated string keys = 2;
-}
diff --git a/tensorflow/core/kernels/data/iterator_ops.cc b/tensorflow/core/kernels/data/iterator_ops.cc
index a8c3f7f77f..da497a5f72 100644
--- a/tensorflow/core/kernels/data/iterator_ops.cc
+++ b/tensorflow/core/kernels/data/iterator_ops.cc
@@ -20,7 +20,6 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/threadpool_device.h"
 #include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/framework/function_handle_cache.h"
-#include "tensorflow/core/framework/iterator.pb.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/resource_op_kernel.h"
 #include "tensorflow/core/framework/stats_aggregator.h"
@@ -277,12 +276,19 @@ class IteratorResource : public ResourceBase {
 
 namespace {
 
+constexpr char kDelimiter[] = "@@";
+
 // Helper class for reading data from a VariantTensorData object.
 class VariantTensorDataReader : public IteratorStateReader {
  public:
   explicit VariantTensorDataReader(const VariantTensorData* data)
       : data_(data) {
-    PreProcess();
+    string metadata;
+    data_->get_metadata(&metadata);
+    auto keys = str_util::Split(metadata, kDelimiter, str_util::SkipEmpty());
+    for (size_t i = 0; i < keys.size(); ++i) {
+      map_[keys[i]] = i;
+    }
   }
 
   // Returns OK iff the initialization was successful, i.e.,
@@ -306,21 +312,6 @@ class VariantTensorDataReader : public IteratorStateReader {
   }
 
  private:
-  void PreProcess() {
-    string metadata;
-    data_->get_metadata(&metadata);
-    IteratorStateMetadata proto;
-    if (!proto.ParseFromString(metadata)) {
-      status_ = errors::Internal("Error parsing IteratorStateMetadata.");
-      return;
-    }
-    size_t num_entries = proto.keys_size();
-    CHECK_EQ(num_entries, data_->tensors_size());
-    for (size_t i = 0; i < num_entries; i++) {
-      map_[proto.keys(i)] = i;
-    }
-  }
-
   template <typename T>
   Status ReadScalarInternal(StringPiece key, T* val) {
     if (map_.find(string(key)) == map_.end()) {
@@ -361,11 +352,10 @@ class VariantTensorDataWriter : public IteratorStateWriter {
     return WriteTensorInternal(key, val);
   }
 
-  // Writes the metadata to `data_`.
   Status Flush() {
     string metadata;
-    if (!metadata_proto_.SerializeToString(&metadata)) {
-      return errors::Internal("Unable to serialize IteratorStateMetadata.");
+    for (size_t i = 0; i < keys_.size(); ++i) {
+      strings::StrAppend(&metadata, kDelimiter, keys_[i]);
     }
     data_->set_metadata(metadata);
     return Status::OK();
@@ -380,19 +370,14 @@ class VariantTensorDataWriter : public IteratorStateWriter {
   }
 
   Status WriteTensorInternal(StringPiece key, const Tensor& val) {
-    // Write key to the metadata proto. This gets written to `data_`
-    // when `Flush()` is called. We do this lazily to avoid multiple
-    // serialization calls.
-    metadata_proto_.add_keys(string(key));
-
-    // Update tensors.
+    DCHECK_EQ(key.find(kDelimiter), string::npos);
+    keys_.push_back(string(key));
     *(data_->add_tensors()) = val;
     return Status::OK();
   }
 
   VariantTensorData* data_;
-  // TODO(srbs): Set the version string.
-  IteratorStateMetadata metadata_proto_;
+  std::vector<string> keys_;
 };
 
 // Wrapper for encoding/decoding the iterator state stored in a Variant tensor.
-- 
GitLab


From 1757ebeaa21b356b954fa8f54e1d0796a8e60dc6 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Thu, 17 Jan 2019 13:01:06 -0800
Subject: [PATCH 0886/2345] Add oss-serial tag to elastic_optimizer_test

This requires allocating many ports, and attaching to them later.
Even with portserver, this is a flaky action when multiple tests execute on the same environment.

PiperOrigin-RevId: 229802480
---
 tensorflow/contrib/opt/BUILD | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/contrib/opt/BUILD b/tensorflow/contrib/opt/BUILD
index 0446e823d9..12320d9e45 100644
--- a/tensorflow/contrib/opt/BUILD
+++ b/tensorflow/contrib/opt/BUILD
@@ -319,6 +319,9 @@ tf_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//third_party/py/numpy",
     ],
+    tags = [
+        "oss_serial",
+    ],
 )
 
 tf_py_test(
-- 
GitLab


From 26cb1cd0e402b179650bba742caa4796f12da963 Mon Sep 17 00:00:00 2001
From: Suharsh Sivakumar <suharshs@google.com>
Date: Thu, 17 Jan 2019 13:10:15 -0800
Subject: [PATCH 0887/2345] Add support for per channel quantization to C API.

Also add future proof quantization params structure.

PiperOrigin-RevId: 229804132
---
 tensorflow/lite/c/c_api_internal.c       | 37 ++++++++++++
 tensorflow/lite/c/c_api_internal.h       | 71 ++++++++++++++++++++++--
 tensorflow/lite/c/c_api_internal_test.cc | 21 +++++++
 3 files changed, 124 insertions(+), 5 deletions(-)

diff --git a/tensorflow/lite/c/c_api_internal.c b/tensorflow/lite/c/c_api_internal.c
index 2923dbad4e..29dba15c63 100644
--- a/tensorflow/lite/c/c_api_internal.c
+++ b/tensorflow/lite/c/c_api_internal.c
@@ -70,6 +70,20 @@ TfLiteIntArray* TfLiteIntArrayCopy(const TfLiteIntArray* src) {
 
 void TfLiteIntArrayFree(TfLiteIntArray* a) { free(a); }
 
+int TfLiteFloatArrayGetSizeInBytes(int size) {
+  static TfLiteFloatArray dummy;
+  return sizeof(dummy) + sizeof(dummy.data[0]) * size;
+}
+
+TfLiteFloatArray* TfLiteFloatArrayCreate(int size) {
+  TfLiteFloatArray* ret =
+      (TfLiteFloatArray*)malloc(TfLiteFloatArrayGetSizeInBytes(size));
+  ret->size = size;
+  return ret;
+}
+
+void TfLiteFloatArrayFree(TfLiteFloatArray* a) { free(a); }
+
 void TfLiteTensorDataFree(TfLiteTensor* t) {
   if (t->allocation_type == kTfLiteDynamic && t->data.raw) {
     free(t->data.raw);
@@ -77,10 +91,30 @@ void TfLiteTensorDataFree(TfLiteTensor* t) {
   t->data.raw = NULL;
 }
 
+void TfLiteQuantizationFree(TfLiteTensor* t) {
+  if (t->quantization.type == kTfLiteAffineQuantization) {
+    TfLiteAffineQuantization* q_params =
+        (TfLiteAffineQuantization*)(t->quantization.params);
+    if (q_params->scale) {
+      TfLiteFloatArrayFree(q_params->scale);
+      q_params->scale = NULL;
+    }
+    if (q_params->zero_point) {
+      TfLiteIntArrayFree(q_params->zero_point);
+      q_params->zero_point = NULL;
+    }
+    free(q_params);
+  }
+  t->quantization.params = NULL;
+  t->quantization.type = kTfLiteNoQuantization;
+}
+
 void TfLiteTensorFree(TfLiteTensor* t) {
   TfLiteTensorDataFree(t);
   if (t->dims) TfLiteIntArrayFree(t->dims);
   t->dims = NULL;
+
+  TfLiteQuantizationFree(t);
 }
 
 void TfLiteTensorReset(TfLiteType type, const char* name, TfLiteIntArray* dims,
@@ -98,6 +132,9 @@ void TfLiteTensorReset(TfLiteType type, const char* name, TfLiteIntArray* dims,
   tensor->allocation_type = allocation_type;
   tensor->allocation = allocation;
   tensor->is_variable = is_variable;
+
+  tensor->quantization.type = kTfLiteNoQuantization;
+  tensor->quantization.params = NULL;
 }
 
 void TfLiteTensorRealloc(size_t num_bytes, TfLiteTensor* tensor) {
diff --git a/tensorflow/lite/c/c_api_internal.h b/tensorflow/lite/c/c_api_internal.h
index 1b1bc6db8f..31f483370c 100644
--- a/tensorflow/lite/c/c_api_internal.h
+++ b/tensorflow/lite/c/c_api_internal.h
@@ -98,8 +98,32 @@ int TfLiteIntArrayEqualsArray(TfLiteIntArray* a, int b_size, int b_data[]);
 // You are expected to free memory with TfLiteIntArrayFree
 TfLiteIntArray* TfLiteIntArrayCopy(const TfLiteIntArray* src);
 
-// Free memory of array `v`.
-void TfLiteIntArrayFree(TfLiteIntArray* v);
+// Free memory of array `a`.
+void TfLiteIntArrayFree(TfLiteIntArray* a);
+
+// Fixed size list of floats. Used for per-channel quantization.
+typedef struct {
+  int size;
+// gcc 6.1+ have a bug where flexible members aren't properly handled
+// https://github.com/google/re2/commit/b94b7cd42e9f02673cd748c1ac1d16db4052514c
+#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ == 6 && \
+    __GNUC_MINOR__ >= 1
+  float data[0];
+#else
+  float data[];
+#endif
+} TfLiteFloatArray;
+
+// Given the size (number of elements) in a TfLiteFloatArray, calculate its size
+// in bytes.
+int TfLiteFloatArrayGetSizeInBytes(int size);
+
+// Create a array of a given `size` (uninitialized entries).
+// This returns a pointer, that you must free using TfLiteFloatArrayFree().
+TfLiteFloatArray* TfLiteFloatArrayCreate(int size);
+
+// Free memory of array `a`.
+void TfLiteFloatArrayFree(TfLiteFloatArray* a);
 
 // Since we must not depend on any libraries, define a minimal subset of
 // error macros while avoiding names that have pre-conceived meanings like
@@ -185,14 +209,48 @@ typedef enum {
 // Return the name of a given type, for error reporting purposes.
 const char* TfLiteTypeGetName(TfLiteType type);
 
+// SupportedQuantizationTypes.
+typedef enum {
+  // No quantization.
+  kTfLiteNoQuantization = 0,
+  // Affine quantization (with support for per-channel quantization).
+  // Corresponds to TfLiteAffineQuantization.
+  kTfLiteAffineQuantization = 1,
+} TfLiteQuantizationType;
+
+// Structure specifying the quantization used by the tensor, if-any.
+typedef struct {
+  // The type of quantization held by params.
+  TfLiteQuantizationType type;
+  // Holds a reference to one of the quantization param structures specified
+  // below.
+  void* params;
+} TfLiteQuantization;
+
+// Legacy. Will be deprecated in favor of TfLiteAffineQuantization.
+// If per-layer quantization is specified this field will still be populated in
+// addition to TfLiteAffineQuantization.
 // Parameters for asymmetric quantization. Quantized values can be converted
 // back to float using:
-//    real_value = scale * (quantized_value - zero_point);
+//     real_value = scale * (quantized_value - zero_point)
 typedef struct {
   float scale;
   int32_t zero_point;
 } TfLiteQuantizationParams;
 
+// Parameters for asymmetric quantization across a dimension (i.e per output
+// channel quantization).
+// quantized_dimension specifies which dimension the scales and zero_points
+// correspond to.
+// For a particular value in quantized_dimension, quantized values can be
+// converted back to float using:
+//     real_value = scale * (quantized_value - zero_point)
+typedef struct {
+  TfLiteFloatArray* scale;
+  TfLiteIntArray* zero_point;
+  int32_t quantized_dimension;
+} TfLiteAffineQuantization;
+
 // A union of pointers that points to memory for a given tensor.
 typedef union {
   int* i32;
@@ -274,12 +332,15 @@ typedef struct {
 
   // True if the tensor is a variable.
   bool is_variable;
+
+  // Quantization information. Replaces params field above.
+  TfLiteQuantization quantization;
 } TfLiteTensor;
 
-// Free data memory of tensor `t`;
+// Free data memory of tensor `t`.
 void TfLiteTensorDataFree(TfLiteTensor* t);
 
-// Free memory of tensor `t`;
+// Free memory of tensor `t`.
 void TfLiteTensorFree(TfLiteTensor* t);
 
 // Set all of a tensor's fields (and free any previously allocated data).
diff --git a/tensorflow/lite/c/c_api_internal_test.cc b/tensorflow/lite/c/c_api_internal_test.cc
index acf0dfc5be..d01cf63a3e 100644
--- a/tensorflow/lite/c/c_api_internal_test.cc
+++ b/tensorflow/lite/c/c_api_internal_test.cc
@@ -65,6 +65,13 @@ TEST(IntArray, TestIntArrayEqual) {
   TfLiteIntArrayFree(d);
 }
 
+TEST(FloatArray, TestFloatArrayCreate) {
+  TfLiteFloatArray* a = TfLiteFloatArrayCreate(0);
+  TfLiteFloatArray* b = TfLiteFloatArrayCreate(3);
+  TfLiteFloatArrayFree(a);
+  TfLiteFloatArrayFree(b);
+}
+
 TEST(Types, TestTypeNames) {
   auto type_name = [](TfLiteType t) {
     return std::string(TfLiteTypeGetName(t));
@@ -81,6 +88,20 @@ TEST(Types, TestTypeNames) {
   EXPECT_EQ(type_name(kTfLiteString), "STRING");
 }
 
+TEST(Quantization, TestQuantizationFree) {
+  TfLiteTensor t;
+  // Set these values, otherwise TfLiteTensorFree has uninitialized values.
+  t.allocation_type = kTfLiteArenaRw;
+  t.dims = nullptr;
+  t.quantization.type = kTfLiteAffineQuantization;
+  auto* params = reinterpret_cast<TfLiteAffineQuantization*>(
+      malloc(sizeof(TfLiteAffineQuantization)));
+  params->scale = TfLiteFloatArrayCreate(3);
+  params->zero_point = TfLiteIntArrayCreate(3);
+  t.quantization.params = reinterpret_cast<void*>(params);
+  TfLiteTensorFree(&t);
+}
+
 }  // namespace tflite
 
 int main(int argc, char** argv) {
-- 
GitLab


From f132194cf910343d4e1c3aedfad45f7b76fc5f61 Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Thu, 17 Jan 2019 13:24:19 -0800
Subject: [PATCH 0888/2345] SessionRef: add a timeout to avoid blocking
 Session::Close() forever.

Some operations/sub-systems do not respect TF cancellation operations.  This
can result in Session:Close() hanging when waiting for these operations to
finish.

This CL adds a default timeout of 60 seconds to ensure we always return control
back to the client eventually.

PiperOrigin-RevId: 229806493
---
 tensorflow/python/BUILD                 |  1 +
 tensorflow/python/client/session_ref.cc | 31 +++++++++++++++++++++----
 2 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index e0648085c3..d17c57b05f 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -4067,6 +4067,7 @@ cc_library(
         "//tensorflow/core:master_proto_cc",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:replay_log_proto_cc",
+        "@com_google_absl//absl/time",
     ],
 )
 
diff --git a/tensorflow/python/client/session_ref.cc b/tensorflow/python/client/session_ref.cc
index 6639cf506e..895624f592 100644
--- a/tensorflow/python/client/session_ref.cc
+++ b/tensorflow/python/client/session_ref.cc
@@ -18,6 +18,9 @@ limitations under the License.
 #include <memory>
 #include <utility>
 
+#include "absl/time/clock.h"
+#include "absl/time/time.h"
+#include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/io/record_writer.h"
 #include "tensorflow/core/lib/strings/stringprintf.h"
@@ -477,6 +480,8 @@ Status SessionRef::ReleaseCallable(CallableHandle handle) {
   LOG_AND_RUN_OPERATION(ReleaseCallable, handle);
 }
 
+static const absl::Duration kMaxCloseWaitTime = absl::Seconds(60);
+
 Status SessionRef::Close(const RunOptions& run_options) {
   TF_RETURN_IF_ERROR(CheckNotClosed());
   mutex_lock l(run_lock_);
@@ -487,8 +492,17 @@ Status SessionRef::Close(const RunOptions& run_options) {
     status = session_->Close(run_options);
   }
   session_.reset();
-  while (run_count_ > 0) {
-    run_finished_.wait(l);
+
+  // Wait no more than kMaxCloseWaitTime for all pending operations to finish.
+  absl::Time start_time = absl::Now();
+  absl::Duration wait_time = start_time + kMaxCloseWaitTime - absl::Now();
+  while (run_count_ > 0 && wait_time > absl::ZeroDuration()) {
+    if (run_finished_.wait_for(l, absl::ToChronoMilliseconds(wait_time)) ==
+        std::cv_status::timeout) {
+      status.Update(errors::DeadlineExceeded("Timeout closing session."));
+      return status;
+    }
+    wait_time = start_time + kMaxCloseWaitTime - absl::Now();
   }
   return status;
 }
@@ -503,8 +517,17 @@ Status SessionRef::Close() {
     status = session_->Close();
   }
   session_.reset();
-  while (run_count_ > 0) {
-    run_finished_.wait(l);
+
+  // Wait no more than kMaxCloseWaitTime for all pending operations to finish.
+  absl::Time start_time = absl::Now();
+  absl::Duration wait_time = start_time + kMaxCloseWaitTime - absl::Now();
+  while (run_count_ > 0 && wait_time > absl::ZeroDuration()) {
+    if (run_finished_.wait_for(l, absl::ToChronoMilliseconds(wait_time)) ==
+        std::cv_status::timeout) {
+      status.Update(errors::DeadlineExceeded("Timeout closing session."));
+      return status;
+    }
+    wait_time = start_time + kMaxCloseWaitTime - absl::Now();
   }
   return status;
 }
-- 
GitLab


From 8879131afb5e0470279a838f4b47bb969a361cf2 Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl@google.com>
Date: Thu, 17 Jan 2019 13:30:56 -0800
Subject: [PATCH 0889/2345] Use Sampler to record warm up latency distribution
 (by status).

PiperOrigin-RevId: 229807679
---
 tensorflow/cc/saved_model/loader.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/cc/saved_model/loader.cc b/tensorflow/cc/saved_model/loader.cc
index aaf06a74c6..10f7abf09e 100644
--- a/tensorflow/cc/saved_model/loader.cc
+++ b/tensorflow/cc/saved_model/loader.cc
@@ -52,8 +52,8 @@ auto* load_latency_by_stage = monitoring::Sampler<2>::New(
         "model_path",
         "stage",
     },
-    // Scale of 10, power of 1.5 with bucket count 36 (~20 minutes).
-    monitoring::Buckets::Exponential(10, 1.5, 36));
+    // Scale of 10, power of 1.8 with bucket count 33 (~20 minutes).
+    monitoring::Buckets::Exponential(10, 1.8, 33));
 
 constexpr char kLoadAttemptFail[] = "fail";
 constexpr char kLoadAttemptSuccess[] = "success";
-- 
GitLab


From 3fa896d2aa9f67f88c825c3a05eff2cf01b5546a Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Thu, 17 Jan 2019 13:33:33 -0800
Subject: [PATCH 0890/2345] Revert the changes to TRTEngineOp's OpDef, so
 existing models won't get broken.

PiperOrigin-RevId: 229808206
---
 .../contrib/tensorrt/convert/convert_graph.cc | 12 +++++-----
 .../contrib/tensorrt/convert/convert_graph.h  |  5 ++--
 .../contrib/tensorrt/convert/convert_nodes.h  |  2 +-
 .../tensorrt/convert/trt_optimization_pass.cc |  6 ++---
 .../contrib/tensorrt/kernels/trt_engine_op.cc | 17 +++++++------
 .../contrib/tensorrt/kernels/trt_engine_op.h  |  2 +-
 .../contrib/tensorrt/ops/trt_engine_op.cc     |  5 +++-
 .../contrib/tensorrt/python/trt_convert.py    | 24 +++++++++----------
 .../tensorrt/python/trt_convert_test.py       |  7 +++---
 .../contrib/tensorrt/test/test_tftrt.py       |  6 ++---
 .../test/tf_trt_integration_test_base.py      |  8 +++----
 11 files changed, 47 insertions(+), 47 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index 87c2d367f8..e2350f69a6 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -239,7 +239,7 @@ tensorflow::Status ConvertGraphDefToTensorRT(
     const std::vector<string>& output_names, size_t max_batch_size,
     size_t max_workspace_size_bytes, tensorflow::GraphDef* new_graph_def,
     int precision_mode, int minimum_segment_size, bool is_dyn_op,
-    int max_cached_engines, std::vector<int> cached_engine_batch_sizes,
+    int max_cached_engines, std::vector<int> cached_engine_batches,
     bool use_calibration) {
   // Create GrapplerItem.
   tensorflow::grappler::GrapplerItem item;
@@ -301,9 +301,9 @@ tensorflow::Status ConvertGraphDefToTensorRT(
   TF_RETURN_IF_ERROR(GetPrecisionModeName(
       precision_mode, parameters["precision_mode"].mutable_s()));
   parameters["maximum_cached_engines"].set_i(max_cached_engines);
-  if (!cached_engine_batch_sizes.empty()) {
-    auto list = parameters["cached_engine_batch_sizes"].mutable_list();
-    for (const int batch : cached_engine_batch_sizes) {
+  if (!cached_engine_batches.empty()) {
+    auto list = parameters["cached_engine_batches"].mutable_list();
+    for (const int batch : cached_engine_batches) {
       list->add_i(batch);
     }
   }
@@ -689,7 +689,7 @@ tensorflow::Status CreateTRTNode(const std::vector<EngineInfo>& infos, int pos,
   }
 
   if (info.engine_type == EngineInfo::EngineType::TRTStatic &&
-      !info.cached_engine_batch_sizes.empty()) {
+      !info.cached_engine_batches.empty()) {
     LOG(WARNING) << "Cached engine batches are ignored for static engines";
   }
   tensorflow::NodeDef trt_node;
@@ -983,7 +983,7 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
                                    ? EngineInfo::EngineType::TRTDynamic
                                    : EngineInfo::EngineType::TRTStatic);
     curr_engine.use_calibration = params.use_calibration;
-    curr_engine.cached_engine_batch_sizes = params.cached_engine_batch_sizes;
+    curr_engine.cached_engine_batches = params.cached_engine_batches;
     curr_engine.maximum_cached_engines = params.max_cached_engines;
     StrAppend(&curr_engine.engine_name, "TRTEngineOp_", t);
     status = RegisterSegmentFunctionToFunctionLibrary(
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.h b/tensorflow/contrib/tensorrt/convert/convert_graph.h
index 5616bbefd9..1f39f56f63 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.h
@@ -82,7 +82,7 @@ struct ConversionParams {
   bool fixed_input_size;   // Assume non-batch ranks of input tensors are fixed
   int max_cached_engines;  // maximum number of cached engines
   bool use_calibration;
-  std::vector<int> cached_engine_batch_sizes;  // list of cached engines
+  std::vector<int> cached_engine_batches;  // list of cached engines
 };
 
 // This method extracts calibration information from the resource managers
@@ -101,8 +101,7 @@ tensorflow::Status ConvertGraphDefToTensorRT(
     size_t max_workspace_size_bytes, tensorflow::GraphDef* new_graph_def,
     int precision_mode = 1, int minimum_segment_size = 3,
     bool is_dyn_op = false, int max_cached_engines = 1,
-    std::vector<int> cached_engine_batch_sizes = {},
-    bool use_calibration = true);
+    std::vector<int> cached_engine_batches = {}, bool use_calibration = true);
 
 // Method to call from optimization pass
 tensorflow::Status ConvertAfterShapes(ConversionParams& params);
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.h b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
index a02a52e430..4ea5775f04 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
@@ -108,7 +108,7 @@ struct EngineInfo {
   EngineType engine_type;
   int64 max_workspace_size_bytes;
   int maximum_cached_engines;
-  std::vector<int> cached_engine_batch_sizes;
+  std::vector<int> cached_engine_batches;
   int precision_mode;
   bool use_calibration;
 };
diff --git a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc
index e7ca126f55..d57f2300f8 100644
--- a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc
+++ b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc
@@ -50,8 +50,8 @@ tensorflow::Status TRTOptimizationPass::Init(
   if (params.count("is_dynamic_op")) {
     is_dynamic_op_ = params.at("is_dynamic_op").b();
   }
-  if (params.count("cached_engine_batch_sizes")) {
-    auto batch_vec = params.at("cached_engine_batch_sizes").list();
+  if (params.count("cached_engine_batches")) {
+    auto batch_vec = params.at("cached_engine_batches").list();
     batches_.reserve(batch_vec.i_size());
     for (const auto i : batch_vec.i()) {
       batches_.push_back(i);
@@ -258,7 +258,7 @@ tensorflow::Status TRTOptimizationPass::Optimize(
   cp.graph_properties = &static_graph_properties;
   cp.cluster = cluster;
   cp.is_dyn_op = is_dynamic_op_;
-  cp.cached_engine_batch_sizes = batches_;
+  cp.cached_engine_batches = batches_;
   cp.max_cached_engines = max_cached_batches_;
   cp.use_calibration = use_calibration_;
   auto status = tensorflow::tensorrt::convert::ConvertAfterShapes(cp);
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
index f089f2c1cc..7548c8ccda 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
@@ -136,13 +136,12 @@ TRTEngineOp::TRTEngineOp(OpKernelConstruction* context)
   native_func_ = tensorflow::kInvalidHandle;
   OP_REQUIRES_OK(context, context->GetAttr("max_cached_engines_count",
                                            &max_cached_engines_));
-  OP_REQUIRES_OK(context, context->GetAttr("cached_engine_batch_sizes",
-                                           &cached_engine_batch_sizes_));
-  std::sort(cached_engine_batch_sizes_.begin(),
-            cached_engine_batch_sizes_.end());
+  OP_REQUIRES_OK(context, context->GetAttr("cached_engine_batches",
+                                           &cached_engine_batches_));
+  std::sort(cached_engine_batches_.begin(), cached_engine_batches_.end());
   if (VLOG_IS_ON(1)) {
     string s("Engine Batches= ");
-    for (auto i : cached_engine_batch_sizes_) {
+    for (auto i : cached_engine_batches_) {
       StrAppend(&s, i, " ");
     }
     VLOG(1) << s;
@@ -247,7 +246,7 @@ bool TRTEngineOp::GetCompatibleCachedEngine(
   // Output shape will always be the same as the input but we will overwrite the
   // batch size.
   *engine_input_shapes = actual_input_shapes;
-  for (const int cached_batch_size : cached_engine_batch_sizes_) {
+  for (const int cached_batch_size : cached_engine_batches_) {
     // Check if compatible: batch <= cached batch.
     //
     // TODO(laigd): here it only compare the first dim a.k.a the batch size,
@@ -495,12 +494,12 @@ EngineContext* TRTEngineOp::GetEngine(
   // exact shape and possibly create a new engine if it is not in cache.
   if (!matched_successfully) {
     engine_input_shapes = input_shapes;
-    if (!cached_engine_batch_sizes_.empty()) {
-      // If user has explicitly defined cached_engine_batch_sizes, we should
+    if (!cached_engine_batches_.empty()) {
+      // If user has explicitly defined cached_engine_batches, we should
       // warn them that their input was non-compatible (batch size too high)
       LOG(WARNING) << "No compatible cached engine was found for batch size: "
                    << batch_size << ". A new engine will be created.";
-      cached_engine_batch_sizes_.push_back(batch_size);
+      cached_engine_batches_.push_back(batch_size);
     }
   }
 
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
index c9b325cfd2..7f0f05aa0a 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
@@ -104,7 +104,7 @@ class TRTEngineOp : public AsyncOpKernel {
   bool calibration_mode_;
 
   // Batches of the cached engines
-  std::vector<int> cached_engine_batch_sizes_;
+  std::vector<int> cached_engine_batches_;
 
   // Maximum number of cached engines
   int max_cached_engines_;
diff --git a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
index 73cc046bc3..b84d2fe0b8 100644
--- a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
@@ -28,6 +28,9 @@ namespace shape_inference {
 extern Status TRTEngineOpShapeInference(InferenceContext* c);
 }
 
+// NOTE: please try NOT to add/modify/remove attributes or inputs/outputs to the
+// list below, this will break backward compatibility!
+//
 // TODO(laigd): consider making this op stateful. The only problem is it uses TF
 // function which has to be stateless, but we can use function library as the
 // key to cache the instantiated functions for different executor subgraphs.
@@ -40,7 +43,7 @@ REGISTER_OP("TRTEngineOp")
     .Attr("OutT: list({int8,float16,float32,int32})")
     .Attr("static_engine: bool = true")
     .Attr("fixed_input_size: bool = true")
-    .Attr("cached_engine_batch_sizes: list(int) >= 0 = []")
+    .Attr("cached_engine_batches: list(int) >= 0 = []")
     .Attr("max_cached_engines_count: int = 1")
     .Attr("workspace_size_bytes: int")
     .Attr("precision_mode: {'FP32', 'FP16', 'INT8'}")
diff --git a/tensorflow/contrib/tensorrt/python/trt_convert.py b/tensorflow/contrib/tensorrt/python/trt_convert.py
index 8de17347f8..49d72232aa 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert.py
@@ -77,7 +77,7 @@ def get_tensorrt_rewriter_config(rewriter_config=None,
                                  minimum_segment_size=3,
                                  is_dynamic_op=False,
                                  maximum_cached_engines=1,
-                                 cached_engine_batch_sizes=None,
+                                 cached_engine_batches=None,
                                  use_calibration=True):
   """Returns a RewriterConfig proto for TRT transformation.
 
@@ -97,7 +97,7 @@ def get_tensorrt_rewriter_config(rewriter_config=None,
       If the number of cached engines is already at max but none of them can
       serve the input, the TRTEngineOp will fall back to run the TF function
       based on which the TRTEngineOp is created.
-    cached_engine_batch_sizes: a list of batch sizes used to create cached
+    cached_engine_batches: a list of batch sizes used to create cached
       engines, only used when is_dynamic_op is True. The length of the list
       should be <= maximum_cached_engines, and the dynamic TRT op will
       use this list to determine the batch sizes of the cached engines, instead
@@ -150,14 +150,14 @@ def get_tensorrt_rewriter_config(rewriter_config=None,
       "max_workspace_size_bytes"].i = max_workspace_size_bytes
   optimizer.parameter_map["precision_mode"].s = _to_bytes(precision_mode)
   optimizer.parameter_map["maximum_cached_engines"].i = maximum_cached_engines
-  if cached_engine_batch_sizes:
-    if not isinstance(cached_engine_batch_sizes, list):
-      raise TypeError("cached_engine_batch_sizes should be a list.")
-    if len(cached_engine_batch_sizes) > maximum_cached_engines:
-      raise ValueError("cached_engine_batch_sizes should not contain more than "
+  if cached_engine_batches:
+    if not isinstance(cached_engine_batches, list):
+      raise TypeError("cached_engine_batches should be a list.")
+    if len(cached_engine_batches) > maximum_cached_engines:
+      raise ValueError("cached_engine_batches should not contain more than "
                        "maximum_cached_engines items.")
-    optimizer.parameter_map["cached_engine_batch_sizes"].list.i.extend(
-        cached_engine_batch_sizes)
+    optimizer.parameter_map["cached_engine_batches"].list.i.extend(
+        cached_engine_batches)
   optimizer.parameter_map["use_calibration"].b = use_calibration
   return rewriter_config_with_trt
 
@@ -170,7 +170,7 @@ def create_inference_graph(input_graph_def,
                            minimum_segment_size=3,
                            is_dynamic_op=False,
                            maximum_cached_engines=1,
-                           cached_engine_batch_sizes=None,
+                           cached_engine_batches=None,
                            use_calibration=True,
                            input_saved_model_dir=None,
                            input_saved_model_tags=None,
@@ -197,7 +197,7 @@ def create_inference_graph(input_graph_def,
       If the number of cached engines is already at max but none of them can
       serve the input, the TRTEngineOp will fall back to run the TF function
       based on which the TRTEngineOp is created.
-    cached_engine_batch_sizes: a list of batch sizes used to create cached
+    cached_engine_batches: a list of batch sizes used to create cached
       engines, only used when is_dynamic_op is True. The length of the list
       should be <= maximum_cached_engines, and the dynamic TRT op will
       use this list to determine the batch sizes of the cached engines, instead
@@ -361,7 +361,7 @@ def create_inference_graph(input_graph_def,
   rewriter_config_with_trt = get_tensorrt_rewriter_config(
       rewriter_config, max_batch_size, max_workspace_size_bytes, precision_mode,
       minimum_segment_size, is_dynamic_op, maximum_cached_engines,
-      cached_engine_batch_sizes, use_calibration)
+      cached_engine_batches, use_calibration)
   session_config_with_trt.graph_options.rewrite_options.CopyFrom(
       rewriter_config_with_trt)
 
diff --git a/tensorflow/contrib/tensorrt/python/trt_convert_test.py b/tensorflow/contrib/tensorrt/python/trt_convert_test.py
index 9562704aeb..3ef18e3e15 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert_test.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert_test.py
@@ -57,7 +57,7 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
         minimum_segment_size=10,
         is_dynamic_op=True,
         maximum_cached_engines=2,
-        cached_engine_batch_sizes=[1, 128])
+        cached_engine_batches=[1, 128])
     self.assertEqual(["constfold", "layout", "constfold"],
                      rewriter_cfg.optimizers)
     self.assertEqual(rewriter_config_pb2.RewriterConfig.ONE,
@@ -71,7 +71,7 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
     for key in [
         "minimum_segment_size", "max_batch_size", "is_dynamic_op",
         "max_workspace_size_bytes", "precision_mode", "maximum_cached_engines",
-        "cached_engine_batch_sizes"
+        "cached_engine_batches"
     ]:
       self.assertTrue(key in trt_optimizer.parameter_map)
     self.assertEqual(10, trt_optimizer.parameter_map["minimum_segment_size"].i)
@@ -84,8 +84,7 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
         trt_optimizer.parameter_map["precision_mode"].s)
     self.assertEqual(2, trt_optimizer.parameter_map["maximum_cached_engines"].i)
     self.assertEqual(
-        [1, 128],
-        trt_optimizer.parameter_map["cached_engine_batch_sizes"].list.i)
+        [1, 128], trt_optimizer.parameter_map["cached_engine_batches"].list.i)
 
   def _GetConfigProto(self):
     """Get ConfigProto for session creation."""
diff --git a/tensorflow/contrib/tensorrt/test/test_tftrt.py b/tensorflow/contrib/tensorrt/test/test_tftrt.py
index d26f260086..090aa8bdb0 100644
--- a/tensorflow/contrib/tensorrt/test/test_tftrt.py
+++ b/tensorflow/contrib/tensorrt/test/test_tftrt.py
@@ -191,7 +191,7 @@ def user(multi_engine,
       minimum_segment_size=2,  # minimum number of nodes in an engine
       is_dynamic_op=False,
       maximum_cached_engines=1,
-      cached_engine_batch_sizes=[])
+      cached_engine_batches=[])
   o1 = run_graph(orig_graph, dummy_input)
   o2 = run_graph(trt_graph, dummy_input)
   o3 = run_graph(trt_graph, dummy_input)
@@ -206,7 +206,7 @@ def user(multi_engine,
       minimum_segment_size=2,  # minimum number of nodes in an engine
       is_dynamic_op=False,
       maximum_cached_engines=1,
-      cached_engine_batch_sizes=[])
+      cached_engine_batches=[])
   int8_calib_gdef = trt.create_inference_graph(
       input_graph_def=orig_graph,
       outputs=["output"],
@@ -216,7 +216,7 @@ def user(multi_engine,
       minimum_segment_size=2,  # minimum number of nodes in an engine
       is_dynamic_op=False,
       maximum_cached_engines=1,
-      cached_engine_batch_sizes=[])
+      cached_engine_batches=[])
   o4 = run_graph(fp16_graph, dummy_input)
   _ = run_calibration(int8_calib_gdef, dummy_input)
   int8_graph = trt.calib_graph_to_infer_graph(int8_calib_gdef)
diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
index 4f47f80e73..db43581404 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
@@ -61,7 +61,7 @@ RunParams = namedtuple("RunParams", [
 ConversionParams = namedtuple("ConversionParams", [
     "max_batch_size", "max_workspace_size_bytes", "precision_mode",
     "minimum_segment_size", "is_dynamic_op", "maximum_cached_engines",
-    "cached_engine_batch_sizes", "rewriter_config", "use_calibration"
+    "cached_engine_batches", "rewriter_config", "use_calibration"
 ])
 
 PRECISION_MODES = ["FP32", "FP16", "INT8"]
@@ -186,7 +186,7 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
         minimum_segment_size=2,
         is_dynamic_op=run_params.dynamic_engine,
         maximum_cached_engines=1,
-        cached_engine_batch_sizes=None,
+        cached_engine_batches=None,
         rewriter_config=None,
         use_calibration=run_params.use_calibration)
 
@@ -266,7 +266,7 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
           conversion_params.minimum_segment_size,
           conversion_params.is_dynamic_op,
           conversion_params.maximum_cached_engines,
-          conversion_params.cached_engine_batch_sizes,
+          conversion_params.cached_engine_batches,
           conversion_params.use_calibration)
 
       graph_options = config_pb2.GraphOptions(rewrite_options=rewriter_cfg)
@@ -369,7 +369,7 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
         minimum_segment_size=conversion_params.minimum_segment_size,
         is_dynamic_op=conversion_params.is_dynamic_op,
         maximum_cached_engines=conversion_params.maximum_cached_engines,
-        cached_engine_batch_sizes=conversion_params.cached_engine_batch_sizes,
+        cached_engine_batches=conversion_params.cached_engine_batches,
         use_calibration=conversion_params.use_calibration,
         session_config=config_for_trt)
 
-- 
GitLab


From 741fb41f3cafc5750b3d1f6ab3ba366b8cf93b34 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Thu, 17 Jan 2019 13:49:30 -0800
Subject: [PATCH 0891/2345] [TF:XLA] Bump open source llvm revision to r351428

PiperOrigin-RevId: 229811121
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 6fcccd4147..ad73b19ff1 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -512,11 +512,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "2027c5043ea3458376fbf4263aaab3b257736b9e4a1e9e9984f15e245177979d",
-        strip_prefix = "llvm-82a2c96ee81a617918d9bcc0fbe9af860e437f5f",
+        sha256 = "986122dee6053db98d4cf8a60efedd6d94e20c557cc97299a0c7d33b9efae12d",
+        strip_prefix = "llvm-329f768b5fc380a4bfa327396f108a8d8f33e77b",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/82a2c96ee81a617918d9bcc0fbe9af860e437f5f.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/82a2c96ee81a617918d9bcc0fbe9af860e437f5f.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/329f768b5fc380a4bfa327396f108a8d8f33e77b.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/329f768b5fc380a4bfa327396f108a8d8f33e77b.tar.gz",
         ],
     )
 
-- 
GitLab


From 252aa8b5661559377e86b4763980f59f214b9eee Mon Sep 17 00:00:00 2001
From: Bairen Yi <byronyi@clustar.ai>
Date: Thu, 17 Jan 2019 22:48:05 +0000
Subject: [PATCH 0892/2345] Address review comments in #24864.

Signed-off-by: Bairen Yi <byronyi@clustar.ai>
---
 tensorflow/contrib/gdr/gdr_worker.cc | 6 +++---
 tensorflow/contrib/gdr/gdr_worker.h  | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/gdr/gdr_worker.cc b/tensorflow/contrib/gdr/gdr_worker.cc
index 29ee4c9375..a2425557da 100644
--- a/tensorflow/contrib/gdr/gdr_worker.cc
+++ b/tensorflow/contrib/gdr/gdr_worker.cc
@@ -44,13 +44,13 @@ GdrWorker::GdrWorker(WorkerEnv* worker_env, const ConfigProto& config,
                      RemoteMemoryManager* remote_memory_manager)
     : GrpcWorker(worker_env, config),
       remote_memory_manager_(remote_memory_manager),
-      recv_tensor_recent_request_ids_(100000) {}
+      recent_request_ids_(100000) {}
 
 void GdrWorker::GrpcRecvTensorAsync(CallOptions* opts,
                                     const RecvTensorRequest* request,
                                     ::grpc::ByteBuffer* response,
                                     StatusCallback done) {
-  Status s = recv_tensor_recent_request_ids_.TrackUnique(
+  Status s = recent_request_ids_.TrackUnique(
       request->request_id(), "RecvTensor (GdrWorker)", *request);
   if (!s.ok()) {
     done(s);
@@ -152,7 +152,7 @@ void GdrWorker::GrpcRecvTensorAsync(CallOptions* opts,
 void GdrWorker::RecvBufAsync(CallOptions* opts, const RecvBufRequest* request,
                              RecvBufResponse* response, StatusCallback done) {
   // This is an RDMA enabled implementation augmenting grpc.
-  Status s = recv_tensor_recent_request_ids_.TrackUnique(
+  Status s = recent_request_ids_.TrackUnique(
       request->request_id(), "RecvBuf (GdrWorker)", *request);
   if (!s.ok()) {
     done(s);
diff --git a/tensorflow/contrib/gdr/gdr_worker.h b/tensorflow/contrib/gdr/gdr_worker.h
index 7af03d16fc..9a85cfd426 100644
--- a/tensorflow/contrib/gdr/gdr_worker.h
+++ b/tensorflow/contrib/gdr/gdr_worker.h
@@ -44,7 +44,7 @@ class GdrWorker : public GrpcWorker {
 
  private:
   RemoteMemoryManager* remote_memory_manager_;  // Not owned
-  RecentRequestIds recv_tensor_recent_request_ids_;
+  RecentRequestIds recent_request_ids_;
 };
 
 }  // namespace tensorflow
-- 
GitLab


From 793d66993c0210fd22eeda7c7048c379521ebd29 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 13:52:51 -0800
Subject: [PATCH 0893/2345] internal changes

PiperOrigin-RevId: 229811646
---
 tensorflow/python/autograph/impl/api.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/autograph/impl/api.py b/tensorflow/python/autograph/impl/api.py
index 6633441e9f..122ea1726b 100644
--- a/tensorflow/python/autograph/impl/api.py
+++ b/tensorflow/python/autograph/impl/api.py
@@ -35,9 +35,9 @@ from tensorflow.python.autograph.operators import py_builtins
 from tensorflow.python.autograph.pyct import compiler
 from tensorflow.python.autograph.pyct import inspect_utils
 from tensorflow.python.autograph.utils import py_func
-from tensorflow.python.util import nest
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.util import nest
 from tensorflow.python.util import tf_decorator
 from tensorflow.python.util import tf_inspect
 from tensorflow.python.util.tf_export import tf_export
@@ -263,7 +263,7 @@ def converted_call(f, owner, options, *args, **kwargs):
     partial_types = (f.__class__,)
 
   else:
-    NotImplementedError('unknown callable type "%s"' % type(f))
+    raise NotImplementedError('unknown callable type "%s"' % type(f))
 
   arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs)
   arg_types = {}
-- 
GitLab


From 8dacf726e788afadf7b8a7ab366f66c10fb2c0c4 Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Thu, 17 Jan 2019 13:58:28 -0800
Subject: [PATCH 0894/2345] TPUEstimator: always force dequeue operations to
 execute.

If a hostcall does not use the result of an outfeed_dequeue operation, the
resulting values remains on the outfeed.  This eventually stalls the training
process.  This CL executes all outfeed_dequeue operations explicitly when
running hostcalls to ensure the outfeed is drained.

PiperOrigin-RevId: 229812586
---
 .../contrib/tpu/python/tpu/tpu_estimator.py   | 28 +++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 022ac4152d..27c8ccf6bd 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -124,6 +124,16 @@ def _is_iterable(obj):
     return False
 
 
+class CatchInvalidHostcallFunctions(control_flow_ops.XLAControlFlowContext):
+
+  def AddOp(self, op):
+    if op.type in [
+        'AudioSummary', 'AudioSummaryV2', 'HistogramSummary', 'ImageSummary',
+        'MergeSummary', 'ScalarSummary', 'TensorSummary', 'TensorSummaryV2'
+    ]:
+      raise ValueError('Use tf.contrib.summary inside of host_calls.')
+
+
 def _create_global_step(graph):
   graph = graph or ops.get_default_graph()
   if training.get_global_step(graph) is not None:
@@ -1802,6 +1812,10 @@ class _OutfeedHostCall(object):
           dequeue_ops[j].append(item)
 
     # Deconstruct dequeue ops.
+    flat_dequeue_ops = []
+    for l in dequeue_ops:
+      flat_dequeue_ops.extend(l)
+
     dequeue_ops_by_name = {}
     pos = 0
     for name in self._names:
@@ -1809,6 +1823,14 @@ class _OutfeedHostCall(object):
                                               len(self._tensors[name])]
       pos += len(self._tensors[name])
 
+    def _call_host_fn(fn, *args, **kw):
+      context = CatchInvalidHostcallFunctions()
+      context.Enter()
+      result = fn(*args, **kw)
+      context.Exit()
+      context.ExitResult(result)
+      return result
+
     # It is assumed evaluation always happens on single host TPU system. So,
     # place all ops on tpu host if possible.
     #
@@ -1842,7 +1864,7 @@ class _OutfeedHostCall(object):
           # The user-provided eval_metrics[1] is a dict.
           dequeue_ops = dict(zip(self._tensor_keys[name], dequeue_ops))
           try:
-            ret[name] = self._host_fns[name](**dequeue_ops)
+            ret[name] = _call_host_fn(self._host_fns[name], **dequeue_ops)
           except TypeError as e:
             logging.warning(
                 'Exception while calling %s: %s. It is likely the tensors '
@@ -1850,8 +1872,10 @@ class _OutfeedHostCall(object):
                 'function\'s arguments', name, e, name)
             raise
         else:
-          ret[name] = self._host_fns[name](*dequeue_ops)
+          ret[name] = _call_host_fn(self._host_fns[name], *dequeue_ops)
 
+    # force all dequeue operations to be run if not consumed by the host calls
+    ret['__force_dequeue'] = control_flow_ops.group(*flat_dequeue_ops)
     return ret
 
 
-- 
GitLab


From 1cb48000f4c8e22bb444920a999942f8c50d17a9 Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Thu, 3 Jan 2019 15:39:40 -0800
Subject: [PATCH 0895/2345] Move VariantTensorDataWriter and
 VariantTensorDataReader to dataset_utils

---
 tensorflow/core/kernels/data/dataset_utils.cc | 94 +++++++++++++++++++
 tensorflow/core/kernels/data/dataset_utils.h  | 49 ++++++++++
 2 files changed, 143 insertions(+)

diff --git a/tensorflow/core/kernels/data/dataset_utils.cc b/tensorflow/core/kernels/data/dataset_utils.cc
index 4d92d314d3..aba991c080 100644
--- a/tensorflow/core/kernels/data/dataset_utils.cc
+++ b/tensorflow/core/kernels/data/dataset_utils.cc
@@ -141,5 +141,99 @@ Status VerifyShapesCompatible(const std::vector<PartialTensorShape>& expected,
   return Status::OK();
 }
 
+Status VariantTensorDataReader::status() const { return status_; }
+
+Status VariantTensorDataReader::ReadScalar(StringPiece key, int64* val) {
+  return ReadScalarInternal(key, val);
+}
+
+Status VariantTensorDataReader::ReadScalar(StringPiece key, string* val) {
+  return ReadScalarInternal(key, val);
+}
+
+Status VariantTensorDataReader::ReadTensor(StringPiece key, Tensor* val) {
+  return ReadTensorInternal(key, val);
+}
+
+bool VariantTensorDataReader::Contains(StringPiece key) {
+  return map_.find(string(key)) != map_.end();
+}
+
+void VariantTensorDataReader::PreProcess() {
+  string metadata;
+  data_->get_metadata(&metadata);
+  IteratorStateMetadata proto;
+  if (!proto.ParseFromString(metadata)) {
+    status_ = errors::Internal("Error parsing IteratorStateMetadata.");
+    return;
+  }
+  size_t num_entries = proto.keys_size();
+  CHECK_EQ(num_entries, data_->tensors_size());
+  for (size_t i = 0; i < num_entries; i++) {
+    map_[proto.keys(i)] = i;
+  }
+}
+
+template <typename T>
+Status VariantTensorDataReader::ReadScalarInternal(StringPiece key, T* val) {
+  if (map_.find(string(key)) == map_.end()) {
+    return errors::NotFound(key);
+  }
+  *val = data_->tensors(map_[string(key)]).scalar<T>()();
+  return Status::OK();
+}
+
+Status VariantTensorDataReader::ReadTensorInternal(StringPiece key,
+                                                   Tensor* val) {
+  if (map_.find(string(key)) == map_.end()) {
+    return errors::NotFound(key);
+  }
+  *val = data_->tensors(map_[string(key)]);
+  return Status::OK();
+}
+
+Status VariantTensorDataWriter::WriteScalar(StringPiece key, const int64 val) {
+  return WriteScalarInternal(key, val);
+}
+
+Status VariantTensorDataWriter::WriteScalar(StringPiece key,
+                                            const string& val) {
+  return WriteScalarInternal(key, val);
+}
+
+Status VariantTensorDataWriter::WriteTensor(StringPiece key,
+                                            const Tensor& val) {
+  return WriteTensorInternal(key, val);
+}
+
+Status VariantTensorDataWriter::Flush() {
+  string metadata;
+  if (!metadata_proto_.SerializeToString(&metadata)) {
+    return errors::Internal("Unable to serialize IteratorStateMetadata.");
+  }
+  data_->set_metadata(metadata);
+  return Status::OK();
+}
+
+template <typename T>
+Status VariantTensorDataWriter::WriteScalarInternal(StringPiece key,
+                                                    const T& val) {
+  Tensor val_t = Tensor(DataTypeToEnum<T>::v(), TensorShape({}));
+  val_t.scalar<T>()() = val;
+  return WriteTensorInternal(key, val_t);
+}
+
+Status VariantTensorDataWriter::WriteTensorInternal(StringPiece key,
+                                                    const Tensor& val) {
+  // Write key to the metadata proto. This gets written to `data_`
+  // when `Flush()` is called. We do this lazily to avoid multiple
+  // serialization calls.
+  metadata_proto_.add_keys(string(key));
+
+  // Update tensors.
+  *(data_->add_tensors()) = val;
+  return Status::OK();
+}
+
 }  // namespace data
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/data/dataset_utils.h b/tensorflow/core/kernels/data/dataset_utils.h
index 23a3d93ed1..b2a19ac869 100644
--- a/tensorflow/core/kernels/data/dataset_utils.h
+++ b/tensorflow/core/kernels/data/dataset_utils.h
@@ -16,6 +16,7 @@ limitations under the License.
 #define TENSORFLOW_CORE_KERNELS_DATA_DATASET_UTILS_H_
 
 #include "tensorflow/core/framework/dataset.h"
+#include "tensorflow/core/framework/iterator.pb.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/kernels/data/captured_function.h"
 
@@ -57,6 +58,54 @@ Status VerifyTypesMatch(const DataTypeVector& expected,
 Status VerifyShapesCompatible(const std::vector<PartialTensorShape>& expected,
                               const std::vector<PartialTensorShape>& received);
 
+// Helper class for reading data from a VariantTensorData object.
+class VariantTensorDataReader : public IteratorStateReader {
+ public:
+  explicit VariantTensorDataReader(const VariantTensorData* data)
+      : data_(data) {
+    PreProcess();
+  }
+
+  // Returns OK iff the initialization was successful, i.e.,
+  // pre-processing did not have errors.
+  Status status() const;
+  Status ReadScalar(StringPiece key, int64* val) override;
+  Status ReadScalar(StringPiece key, string* val) override;
+  Status ReadTensor(StringPiece key, Tensor* val) override;
+  bool Contains(StringPiece key) override;
+
+ private:
+  void PreProcess();
+  template <typename T>
+  Status ReadScalarInternal(StringPiece key, T* val);
+  Status ReadTensorInternal(StringPiece key, Tensor* val);
+  std::map<string, size_t> map_;
+  const VariantTensorData* data_;  // Not owned.
+  Status status_;
+};
+
+// Helper class for writing data to a VariantTensorData object.
+class VariantTensorDataWriter : public IteratorStateWriter {
+ public:
+  // Does not take ownership of data.
+  explicit VariantTensorDataWriter(VariantTensorData* data) : data_(data) {}
+  Status WriteScalar(StringPiece key, const int64 val) override;
+  Status WriteScalar(StringPiece key, const string& val) override;
+  Status WriteTensor(StringPiece key, const Tensor& val) override;
+
+  // Writes the metadata to `data_`.
+  Status Flush();
+
+ private:
+  template <typename T>
+  Status WriteScalarInternal(StringPiece key, const T& val);
+  Status WriteTensorInternal(StringPiece key, const Tensor& val);
+  VariantTensorData* data_;
+
+  // TODO(srbs): Set the version string.
+  IteratorStateMetadata metadata_proto_;
+};
+
 }  // namespace data
 }  // namespace tensorflow
 
-- 
GitLab


From 0762324b237eca697b376e3e5d067d761717230e Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Thu, 3 Jan 2019 22:37:28 -0800
Subject: [PATCH 0896/2345] Add the test cases for VariantTensorDataWriter and
 VariantTensorDataReader

---
 tensorflow/core/kernels/data/BUILD            |  1 +
 .../core/kernels/data/dataset_utils_test.cc   | 44 ++++++++++++++++++-
 2 files changed, 43 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD
index fab1ec8b6e..dcbdb74691 100644
--- a/tensorflow/core/kernels/data/BUILD
+++ b/tensorflow/core/kernels/data/BUILD
@@ -39,6 +39,7 @@ tf_cc_test(
     srcs = ["dataset_utils_test.cc"],
     deps = [
         ":dataset_utils",
+        "//tensorflow/core:framework",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
     ],
diff --git a/tensorflow/core/kernels/data/dataset_utils_test.cc b/tensorflow/core/kernels/data/dataset_utils_test.cc
index 43295b8ebb..de5dc59bb1 100644
--- a/tensorflow/core/kernels/data/dataset_utils_test.cc
+++ b/tensorflow/core/kernels/data/dataset_utils_test.cc
@@ -13,15 +13,16 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include "tensorflow/core/framework/variant.h"
 #include "tensorflow/core/kernels/data/dataset_utils.h"
-
+#include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/platform/test.h"
 
 namespace tensorflow {
 namespace data {
 namespace {
 
-TEST(DatasetUtils, ComputeMoveVector) {
+TEST(DatasetUtilsTest, ComputeMoveVector) {
   struct TestCase {
     std::vector<int> indices;
     std::vector<bool> expected;
@@ -41,6 +42,45 @@ TEST(DatasetUtils, ComputeMoveVector) {
   }
 }
 
+TEST(DatasetUtilsTest, VariantTensorData_Writer_Reader) {
+  VariantTensorData data;
+
+  // Basic test cases.
+  VariantTensorDataWriter writer(&data);
+  TF_ASSERT_OK(writer.WriteScalar("Int64", 24));
+  TF_ASSERT_OK(writer.WriteScalar("", "Empty_Key"));
+  Tensor input_tensor(DT_FLOAT, {1});
+  input_tensor.flat<float>()(0) = 2.0f;
+  TF_ASSERT_OK(writer.WriteTensor("Tensor", input_tensor));
+  TF_ASSERT_OK(writer.Flush());
+
+  VariantTensorDataReader reader(&data);
+  EXPECT_OK(reader.status());
+  int64 val_int64;
+  TF_ASSERT_OK(reader.ReadScalar("Int64", &val_int64));
+  EXPECT_EQ(val_int64, 24);
+  string val_string;
+  TF_ASSERT_OK(reader.ReadScalar("", &val_string));
+  EXPECT_EQ(val_string, "Empty_Key");
+  Tensor val_tensor;
+  TF_ASSERT_OK(reader.ReadTensor("Tensor", &val_tensor));
+  EXPECT_EQ(input_tensor.NumElements(), val_tensor.NumElements());
+  EXPECT_EQ(input_tensor.flat<float>()(0), val_tensor.flat<float>()(0));
+
+  // Test the non-existing key for VariantTensorDataReader.
+  EXPECT_EQ(error::NOT_FOUND,
+            reader.ReadScalar("Non_Existing_Key", &val_int64).code());
+  EXPECT_EQ(error::NOT_FOUND,
+            reader.ReadScalar("Non_Existing_Key", &val_string).code());
+  EXPECT_EQ(error::NOT_FOUND,
+            reader.ReadTensor("Non_Existing_Key", &val_tensor).code());
+
+  // Test the invalid parameter for the constructor of VariantTensorDataReader.
+  data.metadata_ = "Invalid Metadata";
+  VariantTensorDataReader reader2(&data);
+  EXPECT_EQ(error::INTERNAL, reader2.status().code());
+}
+
 }  // namespace
 }  // namespace data
 }  // namespace tensorflow
-- 
GitLab


From 55c6645ec7c65ebc6ce9e036052a915b66c2fc46 Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Fri, 4 Jan 2019 10:00:48 -0800
Subject: [PATCH 0897/2345] Use error::InvalidArgument to replace CHECK_EQ in
 VariantTensorDataReader::PreProcess

---
 tensorflow/core/kernels/data/dataset_utils.cc      |  7 ++++++-
 tensorflow/core/kernels/data/dataset_utils_test.cc | 13 ++++++++++---
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/kernels/data/dataset_utils.cc b/tensorflow/core/kernels/data/dataset_utils.cc
index aba991c080..cb0a183c3b 100644
--- a/tensorflow/core/kernels/data/dataset_utils.cc
+++ b/tensorflow/core/kernels/data/dataset_utils.cc
@@ -168,7 +168,12 @@ void VariantTensorDataReader::PreProcess() {
     return;
   }
   size_t num_entries = proto.keys_size();
-  CHECK_EQ(num_entries, data_->tensors_size());
+  if (num_entries != data_->tensors_size()) {
+    status_ = errors::InvalidArgument("Unmatched number of keys and tensors: ",
+                                      num_entries, " vs. ",
+                                      data_->tensors_size());
+    return;
+  }
   for (size_t i = 0; i < num_entries; i++) {
     map_[proto.keys(i)] = i;
   }
diff --git a/tensorflow/core/kernels/data/dataset_utils_test.cc b/tensorflow/core/kernels/data/dataset_utils_test.cc
index de5dc59bb1..2cc5d357b0 100644
--- a/tensorflow/core/kernels/data/dataset_utils_test.cc
+++ b/tensorflow/core/kernels/data/dataset_utils_test.cc
@@ -75,10 +75,17 @@ TEST(DatasetUtilsTest, VariantTensorData_Writer_Reader) {
   EXPECT_EQ(error::NOT_FOUND,
             reader.ReadTensor("Non_Existing_Key", &val_tensor).code());
 
-  // Test the invalid parameter for the constructor of VariantTensorDataReader.
-  data.metadata_ = "Invalid Metadata";
-  VariantTensorDataReader reader2(&data);
+  // Test the invalid metadata for VariantTensorDataReader.
+  VariantTensorData data_invalid_meta(data);
+  data_invalid_meta.metadata_ = "Invalid Metadata";
+  VariantTensorDataReader reader2(&data_invalid_meta);
   EXPECT_EQ(error::INTERNAL, reader2.status().code());
+
+  // Test the unmatched number of keys and tensors for VariantTensorDataReader.
+  VariantTensorData data_unmatched_entries(data);
+  data_unmatched_entries.tensors_.push_back(Tensor(DT_INT64, {1}));
+  VariantTensorDataReader reader3(&data_unmatched_entries);
+  EXPECT_EQ(error::INVALID_ARGUMENT, reader3.status().code());
 }
 
 }  // namespace
-- 
GitLab


From 7bcbe79420f52f9fbee7d43327fb2edab088bb78 Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Fri, 4 Jan 2019 11:08:56 -0800
Subject: [PATCH 0898/2345] Separate the big test to small tests and address
 the clang-format issues

---
 tensorflow/core/kernels/data/dataset_utils.cc |  6 +--
 .../core/kernels/data/dataset_utils_test.cc   | 43 +++++++++++--------
 2 files changed, 28 insertions(+), 21 deletions(-)

diff --git a/tensorflow/core/kernels/data/dataset_utils.cc b/tensorflow/core/kernels/data/dataset_utils.cc
index cb0a183c3b..8ac74af6ba 100644
--- a/tensorflow/core/kernels/data/dataset_utils.cc
+++ b/tensorflow/core/kernels/data/dataset_utils.cc
@@ -169,9 +169,9 @@ void VariantTensorDataReader::PreProcess() {
   }
   size_t num_entries = proto.keys_size();
   if (num_entries != data_->tensors_size()) {
-    status_ = errors::InvalidArgument("Unmatched number of keys and tensors: ",
-                                      num_entries, " vs. ",
-                                      data_->tensors_size());
+    status_ = errors::InvalidArgument(
+        "Unmatched number of keys and tensors: ", num_entries, " vs. ",
+        data_->tensors_size());
     return;
   }
   for (size_t i = 0; i < num_entries; i++) {
diff --git a/tensorflow/core/kernels/data/dataset_utils_test.cc b/tensorflow/core/kernels/data/dataset_utils_test.cc
index 2cc5d357b0..b503e58c1b 100644
--- a/tensorflow/core/kernels/data/dataset_utils_test.cc
+++ b/tensorflow/core/kernels/data/dataset_utils_test.cc
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/core/framework/variant.h"
 #include "tensorflow/core/kernels/data/dataset_utils.h"
+#include "tensorflow/core/framework/variant.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/platform/test.h"
 
@@ -42,10 +42,8 @@ TEST(DatasetUtilsTest, ComputeMoveVector) {
   }
 }
 
-TEST(DatasetUtilsTest, VariantTensorData_Writer_Reader) {
+TEST(DatasetUtilsTest, VariantTensorDataRoundtrip) {
   VariantTensorData data;
-
-  // Basic test cases.
   VariantTensorDataWriter writer(&data);
   TF_ASSERT_OK(writer.WriteScalar("Int64", 24));
   TF_ASSERT_OK(writer.WriteScalar("", "Empty_Key"));
@@ -66,26 +64,35 @@ TEST(DatasetUtilsTest, VariantTensorData_Writer_Reader) {
   TF_ASSERT_OK(reader.ReadTensor("Tensor", &val_tensor));
   EXPECT_EQ(input_tensor.NumElements(), val_tensor.NumElements());
   EXPECT_EQ(input_tensor.flat<float>()(0), val_tensor.flat<float>()(0));
+}
 
-  // Test the non-existing key for VariantTensorDataReader.
+TEST(DatasetUtilsTest, VariantTensorDataNonExistentKey) {
+  VariantTensorData data;
+  VariantTensorDataReader reader(&data);
+  TF_ASSERT_OK(reader.status());
+  int64 val_int64;
+  string val_string;
+  Tensor val_tensor;
   EXPECT_EQ(error::NOT_FOUND,
-            reader.ReadScalar("Non_Existing_Key", &val_int64).code());
+            reader.ReadScalar("NonExistentKey", &val_int64).code());
   EXPECT_EQ(error::NOT_FOUND,
-            reader.ReadScalar("Non_Existing_Key", &val_string).code());
+            reader.ReadScalar("NonExistentKey", &val_string).code());
   EXPECT_EQ(error::NOT_FOUND,
-            reader.ReadTensor("Non_Existing_Key", &val_tensor).code());
+            reader.ReadTensor("NonExistentKey", &val_tensor).code());
+}
 
-  // Test the invalid metadata for VariantTensorDataReader.
-  VariantTensorData data_invalid_meta(data);
-  data_invalid_meta.metadata_ = "Invalid Metadata";
-  VariantTensorDataReader reader2(&data_invalid_meta);
-  EXPECT_EQ(error::INTERNAL, reader2.status().code());
+TEST(DatasetUtilsTest, VariantTensorDataInvalidMetadata) {
+  VariantTensorData data;
+  data.metadata_ = "Invalid Metadata";
+  VariantTensorDataReader reader(&data);
+  EXPECT_EQ(error::INTERNAL, reader.status().code());
+}
 
-  // Test the unmatched number of keys and tensors for VariantTensorDataReader.
-  VariantTensorData data_unmatched_entries(data);
-  data_unmatched_entries.tensors_.push_back(Tensor(DT_INT64, {1}));
-  VariantTensorDataReader reader3(&data_unmatched_entries);
-  EXPECT_EQ(error::INVALID_ARGUMENT, reader3.status().code());
+TEST(DatasetUtilsTest, VariantTensorDataUnmatchedKeys) {
+  VariantTensorData data;
+  data.tensors_.push_back(Tensor(DT_INT64, {1}));
+  VariantTensorDataReader reader(&data);
+  EXPECT_EQ(error::INVALID_ARGUMENT, reader.status().code());
 }
 
 }  // namespace
-- 
GitLab


From cc1713c0c0cf20972c34c447fe4afc28a7f6ce34 Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Fri, 4 Jan 2019 13:01:38 -0800
Subject: [PATCH 0899/2345] Change the indentation

---
 tensorflow/core/kernels/data/dataset_utils.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/kernels/data/dataset_utils.cc b/tensorflow/core/kernels/data/dataset_utils.cc
index 8ac74af6ba..67a0b1165a 100644
--- a/tensorflow/core/kernels/data/dataset_utils.cc
+++ b/tensorflow/core/kernels/data/dataset_utils.cc
@@ -169,9 +169,9 @@ void VariantTensorDataReader::PreProcess() {
   }
   size_t num_entries = proto.keys_size();
   if (num_entries != data_->tensors_size()) {
-    status_ = errors::InvalidArgument(
-        "Unmatched number of keys and tensors: ", num_entries, " vs. ",
-        data_->tensors_size());
+    status_ =
+        errors::InvalidArgument("Unmatched number of keys and tensors: ",
+                                num_entries, " vs. ", data_->tensors_size());
     return;
   }
   for (size_t i = 0; i < num_entries; i++) {
-- 
GitLab


From abf9518088797e505ead96aedebd4d71b7e03b4d Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Thu, 17 Jan 2019 14:02:43 -0800
Subject: [PATCH 0900/2345] Disable InterpreterTest.java for now and fix
 missing include.

PiperOrigin-RevId: 229813499
---
 tensorflow/lite/java/BUILD                       | 3 +++
 tensorflow/lite/models/smartreply/ops/predict.cc | 1 +
 2 files changed, 4 insertions(+)

diff --git a/tensorflow/lite/java/BUILD b/tensorflow/lite/java/BUILD
index bd9d8f4806..a539a0cf77 100644
--- a/tensorflow/lite/java/BUILD
+++ b/tensorflow/lite/java/BUILD
@@ -145,6 +145,9 @@ java_test(
         "//tensorflow/lite:testdata/multi_add_flex.bin",
     ],
     javacopts = JAVACOPTS,
+    tags = [
+        "no_mac",  # TODO(b/122888913): libtensorflowlite_test_jni broke on mac.
+    ],
     test_class = "org.tensorflow.lite.InterpreterTest",
     visibility = ["//visibility:private"],
     deps = [
diff --git a/tensorflow/lite/models/smartreply/ops/predict.cc b/tensorflow/lite/models/smartreply/ops/predict.cc
index bb2ed4a315..24b7d54897 100644
--- a/tensorflow/lite/models/smartreply/ops/predict.cc
+++ b/tensorflow/lite/models/smartreply/ops/predict.cc
@@ -28,6 +28,7 @@ limitations under the License.
 //
 
 #include <algorithm>
+#include <cstdlib>
 #include <unordered_map>
 #include <vector>
 
-- 
GitLab


From f986acda003966de4efed0e2fe4854a2b6dc776f Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Thu, 17 Jan 2019 14:05:09 -0800
Subject: [PATCH 0901/2345] [Grappler] Exclude Atan2 from IsUnaryElementWise.

PiperOrigin-RevId: 229814062
---
 tensorflow/core/grappler/op_types.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc
index e06580cebf..f7931c615c 100644
--- a/tensorflow/core/grappler/op_types.cc
+++ b/tensorflow/core/grappler/op_types.cc
@@ -713,7 +713,6 @@ bool IsUnaryElementWise(const NodeDef& node) {
           "Asin",
           "Asinh",
           "Atan",
-          "Atan2",
           "Atanh",
           "Ceil",
           "ComplexAbs",
-- 
GitLab


From 703ab9db70f61ab26d7379614149a07ef3e8db37 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Thu, 17 Jan 2019 14:22:45 -0800
Subject: [PATCH 0902/2345] Make tolerance more reasonable on
 _testOptimizersCompat...

PiperOrigin-RevId: 229817236
---
 tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
index 374ba60460..8069703b7a 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
@@ -533,8 +533,10 @@ class OptimizersCompatibilityTest(keras_parameterized.TestCase):
 
       hist_1 = model_v1.fit(x, y, batch_size=5, epochs=1, shuffle=False)
       hist_2 = model_v2.fit(x, y, batch_size=5, epochs=1, shuffle=False)
-      self.assertAllClose(model_v1.get_weights(), model_v2.get_weights())
-      self.assertAllClose(hist_1.history['loss'], hist_2.history['loss'])
+      self.assertAllClose(model_v1.get_weights(), model_v2.get_weights(),
+                          rtol=1e-5, atol=1e-5)
+      self.assertAllClose(hist_1.history['loss'], hist_2.history['loss'],
+                          rtol=1e-5, atol=1e-5)
 
   def testAdadeltaCompatibility(self):
     opt_v1 = optimizers.Adadelta(lr=0.01)
-- 
GitLab


From 779f2f83fa07866ca34b6aff1a45ff5b09bfcfbd Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Thu, 17 Jan 2019 14:22:48 -0800
Subject: [PATCH 0903/2345] Minor fix to error message; NFC

PiperOrigin-RevId: 229817246
---
 tensorflow/core/graph/control_flow.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/graph/control_flow.cc b/tensorflow/core/graph/control_flow.cc
index 8e1e56d29b..66237a3497 100644
--- a/tensorflow/core/graph/control_flow.cc
+++ b/tensorflow/core/graph/control_flow.cc
@@ -59,7 +59,7 @@ Status ValidateControlFlowInfo(const Graph* graph,
           "Invalid loop structure: Mismatched parent frames for \"",
           cf.frame_name, "\": \"", parent->name, "\" vs \"", frame.parent->name,
           "\". The node giving this error: ", FormatNodeForError(*node),
-          "This is an internal bug, please file a bug report with "
+          ". This is an internal bug, please file a bug report with "
           "instructions on how to reproduce the error.");
     }
     if (IsLoopCond(node)) {
-- 
GitLab


From 22bbbb2280f1ce9a04ab67cdb1f63d85da6f33b3 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Thu, 17 Jan 2019 14:28:59 -0800
Subject: [PATCH 0904/2345] Make tolerance larger for cudnn comparison

PiperOrigin-RevId: 229818348
---
 .../contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py b/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py
index 7e1b4062ce..ca92c31236 100644
--- a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py
+++ b/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py
@@ -1023,7 +1023,7 @@ class CudnnRNNTestCompatibleRNNCells(test_util.TensorFlowTestCase):
           outputs_v, output_state_v = sess.run(
               [outputs, output_state],
               feed_dict={cell_inputs: inference_input})
-          self.assertAllClose(cudnn_outputs_v, outputs_v, atol=2e-5, rtol=2e-5)
+          self.assertAllClose(cudnn_outputs_v, outputs_v, atol=1e-4, rtol=2e-4)
           (cudnn_output_h_v,) = cudnn_output_states_v
           self.assertAllClose(cudnn_output_h_v, output_state_v, atol=2e-5,
                               rtol=2e-5)
-- 
GitLab


From 98984347b66d0642d042841e47e571cdd2191104 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Thu, 17 Jan 2019 14:39:06 -0800
Subject: [PATCH 0905/2345] Disable broken tests for 1.13

PiperOrigin-RevId: 229820397
---
 tensorflow/lite/build_def.bzl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index 7e70503386..5e6b13a46a 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -327,6 +327,7 @@ def generated_test_models_failing(conversion_mode):
     if conversion_mode == "toco-flex":
         return [
             "lstm",  # TODO(b/117510976): Restore when lstm flex conversion works.
+            "unroll_batch_matmul",  # TODO(b/123030774): Fails in 1.13 tests.
         ]
 
     return []
-- 
GitLab


From ee0cc9c80dfbc52ce63dea5e9bd9d789bdbb4893 Mon Sep 17 00:00:00 2001
From: Wenhao Jia <jiawenhao@google.com>
Date: Thu, 17 Jan 2019 14:42:11 -0800
Subject: [PATCH 0906/2345] Internal change

PiperOrigin-RevId: 229820977
---
 tensorflow/compiler/jit/xla_device.cc      | 78 +++++++---------------
 tensorflow/compiler/jit/xla_device.h       | 38 ++---------
 tensorflow/core/common_runtime/device.h    | 19 +++++-
 tensorflow/core/common_runtime/executor.cc | 54 ++++++++++++++-
 tensorflow/core/framework/op_kernel.h      | 22 ++++++
 5 files changed, 120 insertions(+), 91 deletions(-)

diff --git a/tensorflow/compiler/jit/xla_device.cc b/tensorflow/compiler/jit/xla_device.cc
index 77cd2f4462..e2397f6fcb 100644
--- a/tensorflow/compiler/jit/xla_device.cc
+++ b/tensorflow/compiler/jit/xla_device.cc
@@ -219,9 +219,6 @@ XlaDevice::XlaDevice(const SessionOptions& session_options,
 XlaDevice::~XlaDevice() {
   VLOG(1) << "Destroying XLA device " << jit_device_name_ << " " << this;
   mutex_lock lock(mu_);
-  while (outstanding_asynchronous_operations_ > 0) {
-    outstanding_asynchronous_operations_cv_.wait(lock);
-  }
   if (device_context_) {
     device_context_->Unref();
   }
@@ -398,12 +395,6 @@ Status XlaDevice::Sync() {
   if (!stream) return Status::OK();
 
   Status status = stream->BlockHostUntilDone();
-  {
-    mutex_lock lock(mu_);
-    while (outstanding_asynchronous_operations_ > 0) {
-      outstanding_asynchronous_operations_cv_.wait(lock);
-    }
-  }
   TF_RETURN_IF_ERROR(status);
   if (!stream->ok()) {
     return errors::Internal("XlaDevice::Sync() failed.");
@@ -412,6 +403,8 @@ Status XlaDevice::Sync() {
   return Status::OK();
 }
 
+// TODO(b/112409994): This is no longer necessary. Consolidate it with the
+// synchronous version.
 void XlaDevice::Sync(const DoneCallback& done) {
   VLOG(1) << "XlaDevice::Sync (asynchronous)";
   std::shared_ptr<se::Stream> stream;
@@ -424,14 +417,20 @@ void XlaDevice::Sync(const DoneCallback& done) {
     return;
   }
 
+  // The call to ThenEnqueueOnBackgroundThread below enqueues a host callback at
+  // the end of the stream, after everything that has already been enqueued
+  // there at this moment. When the host callback is called, everything before
+  // it must have already finished, and the host callback will then place the
+  // task below onto a background thread. (See the implementation of
+  // ThenEnqueueOnBackgroundThread for details.) Therefore, when the done
+  // callback is finally called from that background thread, we know for sure
+  // that everything enqueued onto the stream (i.e., the device) at this very
+  // moment--when ThenEnqueueOnBackgroundThread is called--will have finished.
+  // This achieves a device-wide sync.
   stream->ThenEnqueueOnBackgroundThread(
       [this, stream, done](se::StreamExecutor*) {
         tracing::ScopedActivity activity("XlaDevice::Sync::Callback",
                                          /*is_expensive=*/true);
-        mutex_lock lock(mu_);
-        while (outstanding_asynchronous_operations_ > 0) {
-          outstanding_asynchronous_operations_cv_.wait(lock);
-        }
         done(stream->ok() ? Status::OK()
                           : errors::Internal("XlaDevice::Sync() failed."));
       });
@@ -470,57 +469,26 @@ Status XlaDevice::MakeTensorFromProto(const TensorProto& tensor_proto,
   return status;
 }
 
-void XlaDevice::SetRequiresSyncOnCompletion(bool sync_on_completion) {
+void XlaDevice::SetAllowsSyncOnCompletion(bool sync_on_completion) {
   mutex_lock lock(mu_);
   sync_on_completion_ = sync_on_completion;
 }
 
-bool XlaDevice::RequiresSyncOnCompletion() const {
+bool XlaDevice::AllowsSyncOnCompletion() const {
   mutex_lock lock(mu_);
   return sync_on_completion_;
 }
 
-XlaDevice::AsynchronousOperationHandle::AsynchronousOperationHandle(
-    XlaDevice* device)
-    : device_(device) {
-  mutex_lock lock(device_->mu_);
-  ++device_->outstanding_asynchronous_operations_;
-}
-
-XlaDevice::AsynchronousOperationHandle::~AsynchronousOperationHandle() {
-  if (device_) {
-    mutex_lock lock(device_->mu_);
-    --device_->outstanding_asynchronous_operations_;
-    device_->outstanding_asynchronous_operations_cv_.notify_all();
+Status XlaDevice::CurrentStatus() {
+  std::shared_ptr<se::Stream> stream;
+  {
+    mutex_lock lock(mu_);
+    stream = stream_;
   }
-}
-
-XlaDevice::AsynchronousOperationHandle::AsynchronousOperationHandle(
-    const XlaDevice::AsynchronousOperationHandle& other)
-    : device_(other.device_) {
-  mutex_lock lock(device_->mu_);
-  ++device_->outstanding_asynchronous_operations_;
-}
-
-XlaDevice::AsynchronousOperationHandle::AsynchronousOperationHandle(
-    XlaDevice::AsynchronousOperationHandle&& other)
-    : device_(other.device_) {
-  other.device_ = nullptr;
-}
-
-XlaDevice::AsynchronousOperationHandle& XlaDevice::AsynchronousOperationHandle::
-operator=(const XlaDevice::AsynchronousOperationHandle& other) {
-  device_ = other.device_;
-  mutex_lock lock(device_->mu_);
-  ++device_->outstanding_asynchronous_operations_;
-  return *this;
-}
-
-XlaDevice::AsynchronousOperationHandle& XlaDevice::AsynchronousOperationHandle::
-operator=(XlaDevice::AsynchronousOperationHandle&& other) {
-  device_ = other.device_;
-  other.device_ = nullptr;
-  return *this;
+  if (!stream) {
+    return Status::OK();
+  }
+  return stream->ok() ? Status::OK() : errors::Internal("XlaDevice is not OK.");
 }
 
 XlaDeviceOpRegistrations* RegisterXlaDeviceKernels(const char* device,
diff --git a/tensorflow/compiler/jit/xla_device.h b/tensorflow/compiler/jit/xla_device.h
index 45f18ac9ee..e35a1c7d29 100644
--- a/tensorflow/compiler/jit/xla_device.h
+++ b/tensorflow/compiler/jit/xla_device.h
@@ -167,35 +167,14 @@ class XlaDevice : public LocalDevice {
   Status UseGpuDeviceInfo() LOCKS_EXCLUDED(mu_);
 
   // Instructs this XlaDevice to return 'sync_on_completion' for
-  // RequiresSyncOnCompletion().
-  void SetRequiresSyncOnCompletion(bool sync_on_completion) LOCKS_EXCLUDED(mu_);
+  // AllowsSyncOnCompletion().
+  void SetAllowsSyncOnCompletion(bool sync_on_completion) LOCKS_EXCLUDED(mu_);
 
-  bool RequiresSyncOnCompletion() const override LOCKS_EXCLUDED(mu_);
+  bool AllowsSyncOnCompletion() const override LOCKS_EXCLUDED(mu_);
 
-  // A simple RAII handle. On construction the device's
-  // outstanding_asynchronous_operations_ field is incremented; on destruction
-  // it is decremented.
-  class AsynchronousOperationHandle {
-   public:
-    AsynchronousOperationHandle(XlaDevice* device);
-    ~AsynchronousOperationHandle();
-    AsynchronousOperationHandle(const AsynchronousOperationHandle& other);
-    AsynchronousOperationHandle(AsynchronousOperationHandle&& other);
-    AsynchronousOperationHandle& operator=(
-        const AsynchronousOperationHandle& other);
-    AsynchronousOperationHandle& operator=(AsynchronousOperationHandle&& other);
-
-   private:
-    XlaDevice* device_ = nullptr;
-  };
-
-  AsynchronousOperationHandle CreateAsynchronousOperationHandle() {
-    return AsynchronousOperationHandle(this);
-  }
+  Status CurrentStatus() override LOCKS_EXCLUDED(mu_);
 
  private:
-  friend class AsynchronousOperationHandle;
-
   xla::LocalClient* client() const;
   Allocator* GetAllocatorLocked(AllocatorAttributes attr)
       EXCLUSIVE_LOCKS_REQUIRED(mu_);
@@ -255,14 +234,9 @@ class XlaDevice : public LocalDevice {
   // Thread pool used for running closures
   std::unique_ptr<thread::ThreadPool> thread_pool_;
 
-  // True if the device requires XlaDevice::Sync to be called on completion
+  // True if the device allows XlaDevice::Sync to be called on completion
   // regardless of status.
-  bool sync_on_completion_ GUARDED_BY(mu_) = false;
-
-  // Count of outstanding asynchronous operations which must be zero on Sync()
-  // completion.
-  int64 outstanding_asynchronous_operations_ GUARDED_BY(mu_) = 0;
-  condition_variable outstanding_asynchronous_operations_cv_;
+  bool sync_on_completion_ GUARDED_BY(mu_) = true;
 
   // Set of devices to use. This controls which of the devices on the given
   // platform will have resources allocated. For GPUs this will be
diff --git a/tensorflow/core/common_runtime/device.h b/tensorflow/core/common_runtime/device.h
index 8dfbb21eda..5a0ef28ff2 100644
--- a/tensorflow/core/common_runtime/device.h
+++ b/tensorflow/core/common_runtime/device.h
@@ -44,6 +44,7 @@ limitations under the License.
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/graph/graph.h"
 #include "tensorflow/core/graph/types.h"
+#include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/macros.h"
 #include "tensorflow/core/platform/types.h"
@@ -122,9 +123,21 @@ class Device : public DeviceBase {
   // version.
   virtual void Sync(const DoneCallback& done);
 
-  // Override this to return true for devices that require a Sync() call before
-  // session completion.
-  virtual bool RequiresSyncOnCompletion() const { return false; }
+  // On session completion, the executor may call Device::Sync() depending on
+  // flag settings. Override this to return false for devices that don't allow
+  // such calls. Instead, these devices must use other mechanisms (such as
+  // num_deferred_ops) to ensure the device has finished processing necessary
+  // work at session completion.
+  //
+  // Devices that override this function must also implement CurrentStatus.
+  virtual bool AllowsSyncOnCompletion() const { return true; }
+
+  // This is used in conjunction with AllowsSyncOnCompletion to allow the
+  // executor to get execution result status at session completion.
+  virtual Status CurrentStatus() {
+    return errors::Unimplemented(
+        "CurrentStatus is not supported on this device.");
+  }
 
   // Optionally modify the device's GraphDef before execution.
   //
diff --git a/tensorflow/core/common_runtime/executor.cc b/tensorflow/core/common_runtime/executor.cc
index 07c8c4a5d4..d068bbf1e4 100644
--- a/tensorflow/core/common_runtime/executor.cc
+++ b/tensorflow/core/common_runtime/executor.cc
@@ -1276,6 +1276,11 @@ class ExecutorState {
 
   std::atomic_int_fast32_t num_outstanding_ops_;
 
+  // Available via OpKernelContext to every OpKernel invocation.
+  mutex num_deferred_ops_mu_;
+  condition_variable num_deferred_ops_cv_;
+  int64 num_deferred_ops_ GUARDED_BY(num_deferred_ops_mu_) = 0;
+
   mutex mu_;
   Status status_ GUARDED_BY(mu_);
 
@@ -1631,6 +1636,15 @@ void ExecutorState::Process(TaggedNode tagged_node, int64 scheduled_nsec) {
   params.input_alloc_attrs = &input_alloc_attrs;
   params.runner = &runner_;
   params.stats_collector = stats_collector_;
+  params.inc_num_deferred_ops_function = [this]() {
+    mutex_lock lock(num_deferred_ops_mu_);
+    num_deferred_ops_++;
+  };
+  params.dec_num_deferred_ops_function = [this]() {
+    mutex_lock lock(num_deferred_ops_mu_);
+    num_deferred_ops_--;
+    num_deferred_ops_cv_.notify_all();
+  };
 
   Status s;
   NodeExecStatsInterface* stats = nullptr;
@@ -2416,7 +2430,45 @@ void ExecutorState::Finish() {
   CHECK(done_cb != nullptr);
   Device* device = impl_->params_.device;
 
-  if ((sync_on_finish_ && status.ok()) || device->RequiresSyncOnCompletion()) {
+  // There are several potential race conditions below. To name a few:
+  // 1. Even if the device's status is OK at the precise moment when
+  // num_deferred_ops_ reaches 0, it could go bad before device->CurrentStatus()
+  // is called below, caused by work enqueued onto the same device by other
+  // concurrent ExecutorState objects.
+  // 2. Some implementations of Device::CurrentStatus, such as
+  // XlaDevice::CurrentStatus, may be inherently racy because it releases the
+  // device mutex after a stream pointer is acquired and before the stream is
+  // queried for status.
+  // 3. It's the same for some implementations of Device::Sync, such as
+  // XlaDevice::Sync.
+  //
+  // However, these race conditions are acceptable because a stream (and
+  // therefore an XlaDevice) can only go from OK to not-OK, never the opposite,
+  // which means we will at worst report errors when there isn't any, never the
+  // opposite.
+
+  // If inc_num_deferred_ops_function has ever been called, ExecutorState must
+  // wait for all corresponding dec_num_deferred_ops_function calls to happen
+  // regardless of status. This ensures that dec_num_deferred_ops_function can
+  // safely use ExecutorState's resources.
+  {
+    mutex_lock lock(num_deferred_ops_mu_);
+    while (num_deferred_ops_ > 0) {
+      num_deferred_ops_cv_.wait(lock);
+    }
+  }
+
+  // An early exit for devices don't allow sync on completion. Ops that run on
+  // these devices should have used num_deferred_ops correctly to ensure the
+  // device has finished all relevant work at this point.
+  if (!device->AllowsSyncOnCompletion()) {
+    status.Update(device->CurrentStatus());
+    delete this;
+    runner([=]() { done_cb(status); });
+    return;
+  }
+
+  if (sync_on_finish_ && status.ok()) {
     // Block until the device has finished all queued operations. For
     // devices like GPUs that continue to execute Ops after their Compute
     // methods have completed, this ensures that control is not returned to
diff --git a/tensorflow/core/framework/op_kernel.h b/tensorflow/core/framework/op_kernel.h
index aa07cbd380..06b90964ad 100644
--- a/tensorflow/core/framework/op_kernel.h
+++ b/tensorflow/core/framework/op_kernel.h
@@ -646,6 +646,10 @@ class OpKernelContext {
     static const int kNoReservation = -1;
     // Values in [0,...) represent reservations for the indexed output.
     const int* forward_from_array = nullptr;
+
+    // For tracking actively running deferred ops.
+    std::function<void()> inc_num_deferred_ops_function = []() {};
+    std::function<void()> dec_num_deferred_ops_function = []() {};
   };
 
   // params must outlive the OpKernelContext.
@@ -1172,6 +1176,24 @@ class OpKernelContext {
 
   bool input_is_ref(int index) const;
 
+  // Used by OpKernel implementations to track actively running deferred ops.
+  //
+  // A deferred op is one whose Compute method returns (or whose ComputeAsync
+  // method invokes the callback) when work is scheduled onto a device. At that
+  // point, we don't know when the work will actually complete (or if it has
+  // already completed) on the device. These functions allow the executor to
+  // track the status of deferred ops and act accordingly.
+  //
+  // Deferred OpKernel implementations must use these methods to get two
+  // functions. It then must call these two functions in pairs, before and after
+  // device execution, respectively.
+  TF_MUST_USE_RESULT std::function<void()> inc_num_deferred_ops_function() {
+    return params_->inc_num_deferred_ops_function;
+  }
+  TF_MUST_USE_RESULT std::function<void()> dec_num_deferred_ops_function() {
+    return params_->dec_num_deferred_ops_function;
+  }
+
  private:
   Allocator* get_allocator(AllocatorAttributes attr);
 
-- 
GitLab


From b797012b39f89c45f13336b36c6b73e9ad93d815 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Thu, 17 Jan 2019 14:46:46 -0800
Subject: [PATCH 0907/2345] Add AttentionMechanismV2 that follows Keras API
 style.

1. Adding new base class for AttentionMechanism that use keras.layer as parent.
2. All the new AttentionMechanismV2 will accept tensors from call() instead of
   as __init__ parameters.
3. memory_length has been update to memory_mask in v2 which is more align with
the keras API.
4. _luong_score() and _bahdanau_score() has been updated to not create variable
by itself. All the variable are now passed in as parameters.
5. All the existing unit tests are still passing. I haven't add new unit test
for v2 class yet, will add them once I update the AttentionWrapper to v2.

PiperOrigin-RevId: 229821776
---
 .../seq2seq/python/ops/attention_wrapper.py   | 962 ++++++++++++++++--
 1 file changed, 851 insertions(+), 111 deletions(-)

diff --git a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
index 60ec3efffe..31c62d5849 100644
--- a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
+++ b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
@@ -28,6 +28,7 @@ from tensorflow.contrib.framework.python.framework import tensor_util
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.keras.engine.base_layer import Layer
 from tensorflow.python.layers import base as layers_base
 from tensorflow.python.layers import core as layers_core
 from tensorflow.python.ops import array_ops
@@ -72,77 +73,6 @@ class AttentionMechanism(object):
     raise NotImplementedError
 
 
-def _prepare_memory(memory, memory_sequence_length, check_inner_dims_defined):
-  """Convert to tensor and possibly mask `memory`.
-
-  Args:
-    memory: `Tensor`, shaped `[batch_size, max_time, ...]`.
-    memory_sequence_length: `int32` `Tensor`, shaped `[batch_size]`.
-    check_inner_dims_defined: Python boolean.  If `True`, the `memory`
-      argument's shape is checked to ensure all but the two outermost
-      dimensions are fully defined.
-
-  Returns:
-    A (possibly masked), checked, new `memory`.
-
-  Raises:
-    ValueError: If `check_inner_dims_defined` is `True` and not
-      `memory.shape[2:].is_fully_defined()`.
-  """
-  memory = nest.map_structure(
-      lambda m: ops.convert_to_tensor(m, name="memory"), memory)
-  if memory_sequence_length is not None:
-    memory_sequence_length = ops.convert_to_tensor(
-        memory_sequence_length, name="memory_sequence_length")
-  if check_inner_dims_defined:
-    def _check_dims(m):
-      if not m.get_shape()[2:].is_fully_defined():
-        raise ValueError("Expected memory %s to have fully defined inner dims, "
-                         "but saw shape: %s" % (m.name, m.get_shape()))
-    nest.map_structure(_check_dims, memory)
-  if memory_sequence_length is None:
-    seq_len_mask = None
-  else:
-    seq_len_mask = array_ops.sequence_mask(
-        memory_sequence_length,
-        maxlen=array_ops.shape(nest.flatten(memory)[0])[1],
-        dtype=nest.flatten(memory)[0].dtype)
-    seq_len_batch_size = (
-        tensor_shape.dimension_value(memory_sequence_length.shape[0])
-        or array_ops.shape(memory_sequence_length)[0])
-  def _maybe_mask(m, seq_len_mask):
-    rank = m.get_shape().ndims
-    rank = rank if rank is not None else array_ops.rank(m)
-    extra_ones = array_ops.ones(rank - 2, dtype=dtypes.int32)
-    m_batch_size = tensor_shape.dimension_value(
-        m.shape[0]) or array_ops.shape(m)[0]
-    if memory_sequence_length is not None:
-      message = ("memory_sequence_length and memory tensor batch sizes do not "
-                 "match.")
-      with ops.control_dependencies([
-          check_ops.assert_equal(
-              seq_len_batch_size, m_batch_size, message=message)]):
-        seq_len_mask = array_ops.reshape(
-            seq_len_mask,
-            array_ops.concat((array_ops.shape(seq_len_mask), extra_ones), 0))
-        return m * seq_len_mask
-    else:
-      return m
-  return nest.map_structure(lambda m: _maybe_mask(m, seq_len_mask), memory)
-
-
-def _maybe_mask_score(score, memory_sequence_length, score_mask_value):
-  if memory_sequence_length is None:
-    return score
-  message = ("All values in memory_sequence_length must greater than zero.")
-  with ops.control_dependencies(
-      [check_ops.assert_positive(memory_sequence_length, message=message)]):
-    score_mask = array_ops.sequence_mask(
-        memory_sequence_length, maxlen=array_ops.shape(score)[1])
-    score_mask_values = score_mask_value * array_ops.ones_like(score)
-    return array_ops.where(score_mask, score, score_mask_values)
-
-
 class _BaseAttentionMechanism(AttentionMechanism):
   """A base AttentionMechanism class providing common functionality.
 
@@ -205,12 +135,14 @@ class _BaseAttentionMechanism(AttentionMechanism):
           self._memory_layer.dtype).as_numpy_dtype(-np.inf)
     self._probability_fn = lambda score, prev: (  # pylint:disable=g-long-lambda
         probability_fn(
-            _maybe_mask_score(score, memory_sequence_length, score_mask_value),
+            _maybe_mask_score(score,
+                              memory_sequence_length=memory_sequence_length,
+                              score_mask_value=score_mask_value),
             prev))
     with ops.name_scope(
         name, "BaseAttentionMechanismInit", nest.flatten(memory)):
       self._values = _prepare_memory(
-          memory, memory_sequence_length,
+          memory, memory_sequence_length=memory_sequence_length,
           check_inner_dims_defined=check_inner_dims_defined)
       self._keys = (
           self.memory_layer(self._values) if self.memory_layer  # pylint: disable=not-callable
@@ -286,6 +218,199 @@ class _BaseAttentionMechanism(AttentionMechanism):
     return self.initial_alignments(batch_size, dtype)
 
 
+class _BaseAttentionMechanismV2(AttentionMechanism, Layer):
+  """A base AttentionMechanism class providing common functionality.
+
+  Common functionality includes:
+    1. Storing the query and memory layers.
+    2. Preprocessing and storing the memory.
+
+  Note that this layer only support Keras functional API since it takes multiple
+  input tensors, which is not available in sequential model.
+  """
+
+  def __init__(self,
+               probability_fn,
+               query_layer=None,
+               memory_layer=None,
+               **kwargs):
+    """Construct base AttentionMechanism class.
+
+    Args:
+      probability_fn: A `callable`. Converts the score and previous alignments
+        to probabilities. Its signature should be:
+        `probabilities = probability_fn(score, state)`.
+      query_layer:  (optional): Instance of `tf.keras.Layer`.  The layer's depth
+        must match the depth of `memory_layer`.  If `query_layer` is not
+        provided, the shape of `query` must match that of `memory_layer`.
+      memory_layer: (optional): Instance of `tf.keras.Layer`. The layer's
+        depth must match the depth of `query_layer`.
+        If `memory_layer` is not provided, the shape of `memory` must match
+        that of `query_layer`.
+      **kwargs: Dictionary that contains other common arguments for layer
+        creation.
+    """
+    if (query_layer is not None
+        and not isinstance(query_layer, layers_base.Layer)):
+      raise TypeError(
+          "query_layer is not a Layer: %s" % type(query_layer).__name__)
+    if (memory_layer is not None
+        and not isinstance(memory_layer, layers_base.Layer)):
+      raise TypeError(
+          "memory_layer is not a Layer: %s" % type(memory_layer).__name__)
+    self.query_layer = query_layer
+    self.memory_layer = memory_layer
+    if self.memory_layer is not None and "dtype" not in kwargs:
+      kwargs["dtype"] = self.memory_layer.dtype
+    super(_BaseAttentionMechanismV2, self).__init__(**kwargs)
+    if not callable(probability_fn):
+      raise TypeError("probability_fn must be callable, saw type: %s" %
+                      type(probability_fn).__name__)
+    self.probability_fn = probability_fn
+
+    self.keys = None
+    self.values = None
+    self.batch_size = None
+    self._memory_initialized = False
+    self._check_inner_dims_defined = True
+
+  def build(self, input_shape):
+    if self.query_layer is not None:
+      self.query_layer.build(input_shape)
+    if self.memory_layer is not None:
+      self.memory_layer.build(input_shape)
+    # dtype of the layer is known at this moment, create the score_mask_value if
+    # needed.
+    self.score_mask_value = dtypes.as_dtype(self.dtype).as_numpy_dtype(-np.inf)
+    self.built = True
+
+  def _setup_memory(self, memory, memory_mask=None):
+    """Pre-process the memory before actually query the memory.
+
+    This should only be called once at the first invocation of call().
+
+    Args:
+      memory: The memory to query; usually the output of an RNN encoder. This
+        tensor should be shaped `[batch_size, max_time, ...]`.
+      memory_mask: The boolean tensor with shape `[batch_size, max_time]`. For
+        any value equal to False, the corresponding value in memory should be
+        ignored.
+    """
+    if self._memory_initialized:
+      raise ValueError("The memory for the attention has already been setup.")
+    with ops.name_scope(
+        self.name, "BaseAttentionMechanismInit", nest.flatten(memory)):
+      self.values = _prepare_memory(
+          memory, memory_mask=memory_mask,
+          check_inner_dims_defined=self._check_inner_dims_defined)
+      if self.memory_layer is not None:
+        self.keys = self.memory_layer(self.values)
+      else:
+        self.keys = self.values
+      self.batch_size = (
+          tensor_shape.dimension_value(self.keys.shape[0]) or
+          array_ops.shape(self.keys)[0])
+      self.alignments_size = (tensor_shape.dimension_value(self.keys.shape[1])
+                              or array_ops.shape(self.keys)[1])
+      if memory_mask is not None:
+        self.probability_fn = lambda score, prev: (  # pylint:disable=g-long-lambda
+            self.probability_fn(_maybe_mask_score(
+                score, self.score_mask_value, memory_mask=memory_mask), prev))
+    self._memory_initialized = True
+
+  def call(self, inputs, mask=None, **kwargs):
+    """Base method to calculate the attention score.
+
+    Args:
+      inputs: a list of tensor that contains `query`, `state`, and `memory`.
+        `query` is the tensor of dtype matching `memory` and shape
+        `[batch_size, query_depth]`.
+        `state` is the tensor of dtype matching `memory` and shape
+        `[batch_size, alignments_size]`. (`alignments_size` is memory's
+        `max_time`).
+        `memory` is the memory to query; usually the output of an RNN encoder.
+        This tensor should be shaped `[batch_size, max_time, feature]`.
+      mask: optional bool tensor with shape `[batch, max_time]` for the mask of
+        memory. If it is not None, the corresponding item of the memory should
+        be filtered out during calculation.
+      **kwargs: Dict, other keyword arguments for the call method.
+    """
+    query, state, memory, memory_mask = self._process_inputs(inputs, mask)
+    if not self._memory_initialized:
+      self._setup_memory(memory, memory_mask=memory_mask)
+    return self.calculate_attention(query, state)
+
+  def calculate_attention(self, query, state):
+    raise NotImplementedError(
+        "calculate_attention need to be implemented by subclasses.")
+
+  def get_config(self):
+    config = {}
+    # Since the probability_fn is likely to be a wrapped function, the child
+    # class should preserve the original function and how its wrapped.
+
+    if self.query_layer is not None:
+      config["query_layer"] = {
+          "class_name": self.query_layer.__class__.__name__,
+          "config": self.query_layer.get_config(),
+      }
+    if self.memory_layer is not None:
+      config["memory_layer"] = {
+          "class_name": self.memory_layer.__class__.__name__,
+          "config": self.memory_layer.get_config(),
+      }
+    base_config = super(_BaseAttentionMechanismV2, self).get_config()
+    return dict(list(base_config.items()) + list(config.items()))
+
+  def _process_inputs(self, inputs, mask):
+    if len(inputs) != 3:
+      raise ValueError(
+          "Expect to have 3 inputs for attention, got %d" % len(inputs))
+    query, state, memory = inputs
+    return query, state, memory, mask
+
+  def _process_probability_fn(self, func_name):
+    """Helper method to retrieve the probably function by string input."""
+    valid_probability_fns = {
+        "softmax": nn_ops.softmax,
+        "hardmax": hardmax,
+    }
+    if func_name not in valid_probability_fns.keys():
+      raise ValueError("Invalid probability function: %s, options are %s" %
+                       (func_name, valid_probability_fns.keys()))
+    return valid_probability_fns[func_name]
+
+  @classmethod
+  def deserialize_inner_layer_from_config(cls, config, custom_objects):
+    """Helper method that reconstruct the query and memory from the config.
+
+    In the get_config() method, the query and memory layer configs are
+    serialized into dict for persistence, this method perform the reverse action
+    to reconstruct the layer from the config.
+
+    Args:
+      config: dict, the configs that will be used to reconstruct the object.
+      custom_objects: dict mapping class names (or function names) of custom
+        (non-Keras) objects to class/functions.
+    Returns:
+      config: dict, the config with layer instance created, which is ready to be
+        used as init parameters.
+    """
+    # Reconstruct the query and memory layer for parent class.
+    from tensorflow.python.keras.layers import deserialize as deserialize_layer  # pylint: disable=g-import-not-at-top
+    query_layer_config = config.pop("query_layer", None)
+    if query_layer_config:
+      query_layer = deserialize_layer(query_layer_config,
+                                      custom_objects=custom_objects)
+      config["query_layer"] = query_layer
+    memory_layer_config = config.pop("memory_layer", None)
+    if memory_layer_config:
+      memory_layer = deserialize_layer(memory_layer_config,
+                                       custom_objects=custom_objects)
+      config["memory_layer"] = memory_layer
+    return config
+
+
 def _luong_score(query, keys, scale):
   """Implements Luong-style (multiplicative) scoring function.
 
@@ -304,7 +429,7 @@ def _luong_score(query, keys, scale):
   Args:
     query: Tensor, shape `[batch_size, num_units]` to compare to keys.
     keys: Processed memory, shape `[batch_size, max_time, num_units]`.
-    scale: Whether to apply a scale to the score function.
+    scale: the optional tensor to scale the attention score.
 
   Returns:
     A `[batch_size, max_time]` tensor of unnormalized score values.
@@ -320,7 +445,6 @@ def _luong_score(query, keys, scale):
         "Query (%s) has units: %s.  Keys (%s) have units: %s.  "
         "Perhaps you need to set num_units to the keys' dimension (%s)?"
         % (query, depth, keys, key_units, key_units))
-  dtype = query.dtype
 
   # Reshape from [batch_size, depth] to [batch_size, 1, depth]
   # for matmul.
@@ -338,12 +462,8 @@ def _luong_score(query, keys, scale):
   score = math_ops.matmul(query, keys, transpose_b=True)
   score = array_ops.squeeze(score, [1])
 
-  if scale:
-    # Scalar used in weight scaling
-    g = variable_scope.get_variable(
-        "attention_g", dtype=dtype,
-        initializer=init_ops.ones_initializer, shape=())
-    score = g * score
+  if scale is not None:
+    score = scale * score
   return score
 
 
@@ -354,8 +474,8 @@ class LuongAttention(_BaseAttentionMechanism):
   as described in:
 
   Minh-Thang Luong, Hieu Pham, Christopher D. Manning.
-  "Effective Approaches to Attention-based Neural Machine Translation."
-  EMNLP 2015.  https://arxiv.org/abs/1508.04025
+  [Effective Approaches to Attention-based Neural Machine Translation.
+  EMNLP 2015.](https://arxiv.org/abs/1508.04025)
 
   The second is the scaled form inspired partly by the normalized form of
   Bahdanau attention.
@@ -429,13 +549,121 @@ class LuongAttention(_BaseAttentionMechanism):
         `max_time`).
     """
     with variable_scope.variable_scope(None, "luong_attention", [query]):
-      score = _luong_score(query, self._keys, self._scale)
+      attention_g = None
+      if self._scale:
+        attention_g = variable_scope.get_variable(
+            "attention_g", dtype=query.dtype,
+            initializer=init_ops.ones_initializer, shape=())
+      score = _luong_score(query, self._keys, attention_g)
     alignments = self._probability_fn(score, state)
     next_state = alignments
     return alignments, next_state
 
 
-def _bahdanau_score(processed_query, keys, normalize):
+class LuongAttentionV2(_BaseAttentionMechanismV2):
+  """Implements Luong-style (multiplicative) attention scoring.
+
+  This attention has two forms.  The first is standard Luong attention,
+  as described in:
+
+  Minh-Thang Luong, Hieu Pham, Christopher D. Manning.
+  [Effective Approaches to Attention-based Neural Machine Translation.
+  EMNLP 2015.](https://arxiv.org/abs/1508.04025)
+
+  The second is the scaled form inspired partly by the normalized form of
+  Bahdanau attention.
+
+  To enable the second form, construct the object with parameter
+  `scale=True`.
+  """
+
+  def __init__(self,
+               units,
+               scale=False,
+               probability_fn="softmax",
+               dtype=None,
+               name="LuongAttention",
+               **kwargs):
+    """Construct the AttentionMechanism mechanism.
+
+    Args:
+      units: The depth of the attention mechanism.
+      scale: Python boolean. Whether to scale the energy term.
+      probability_fn: (optional) string, the name of function to convert the
+        attention score to probabilities. The default is `softmax` which is
+        `tf.nn.softmax`. Other options is `hardmax`, which is hardmax() within
+        this module. Any other value will result intovalidation error. Default
+        to use `softmax`.
+      dtype: The data type for the memory layer of the attention mechanism.
+      name: Name to use when creating ops.
+      **kwargs: Dictionary that contains other common arguments for layer
+        creation.
+    """
+    # For LuongAttention, we only transform the memory layer; thus
+    # num_units **must** match expected the query depth.
+    self.probability_fn_name = probability_fn
+    probability_fn = self._process_probability_fn(self.probability_fn_name)
+    wrapped_probability_fn = lambda score, _: probability_fn(score)
+    if dtype is None:
+      dtype = dtypes.float32
+    super(LuongAttentionV2, self).__init__(
+        query_layer=None,
+        memory_layer=layers_core.Dense(
+            units, name="memory_layer", use_bias=False, dtype=dtype),
+        probability_fn=wrapped_probability_fn,
+        name=name,
+        dtype=dtype,
+        **kwargs)
+    self.units = units
+    self.scale = scale
+
+  def build(self, input_shape):
+    super(LuongAttentionV2, self).build(input_shape)
+    if self.scale:
+      self.scale_weight = self.add_weight(
+          "attention_g", initializer=init_ops.ones_initializer, shape=())
+    else:
+      self.scale_weight = None
+    self.built = True
+
+  def calculate_attention(self, query, state):
+    """Score the query based on the keys and values.
+
+    Args:
+      query: Tensor of dtype matching `self.values` and shape
+        `[batch_size, query_depth]`.
+      state: Tensor of dtype matching `self.values` and shape
+        `[batch_size, alignments_size]`
+        (`alignments_size` is memory's `max_time`).
+
+    Returns:
+      alignments: Tensor of dtype matching `self.values` and shape
+        `[batch_size, alignments_size]` (`alignments_size` is memory's
+        `max_time`).
+    """
+    score = _luong_score(query, self.keys, self.scale_weight)
+    alignments = self.probability_fn(score, state)
+    next_state = alignments
+    return alignments, next_state
+
+  def get_config(self):
+    config = {
+        "units": self.units,
+        "scale": self.scale,
+        "probability_fn": self.probability_fn_name,
+    }
+    base_config = super(LuongAttentionV2, self).get_config()
+    return dict(list(base_config.items()) + list(config.items()))
+
+  @classmethod
+  def from_config(cls, config, custom_objects=None):
+    config = _BaseAttentionMechanismV2.deserialize_inner_layer_from_config(
+        config, custom_objects=custom_objects)
+    return cls(**config)
+
+
+def _bahdanau_score(processed_query, keys, attention_v,
+                    attention_g=None, attention_b=None):
   """Implements Bahdanau-style (additive) scoring function.
 
   This attention has two forms.  The first is Bhandanau attention,
@@ -453,41 +681,28 @@ def _bahdanau_score(processed_query, keys, normalize):
    Training of Deep Neural Networks."
   https://arxiv.org/abs/1602.07868
 
-  To enable the second form, set `normalize=True`.
+  To enable the second form, set please pass in attention_g and attention_b.
 
   Args:
     processed_query: Tensor, shape `[batch_size, num_units]` to compare to keys.
     keys: Processed memory, shape `[batch_size, max_time, num_units]`.
-    normalize: Whether to normalize the score function.
+    attention_v: Tensor, shape `[num_units]`.
+    attention_g: Optional scalar tensor for normalization.
+    attention_b: Optional tensor with shape `[num_units]` for normalization.
 
   Returns:
     A `[batch_size, max_time]` tensor of unnormalized score values.
   """
-  dtype = processed_query.dtype
-  # Get the number of hidden units from the trailing dimension of keys
-  num_units = tensor_shape.dimension_value(
-      keys.shape[2]) or array_ops.shape(keys)[2]
   # Reshape from [batch_size, ...] to [batch_size, 1, ...] for broadcasting.
   processed_query = array_ops.expand_dims(processed_query, 1)
-  v = variable_scope.get_variable(
-      "attention_v", [num_units], dtype=dtype)
-  if normalize:
-    # Scalar used in weight normalization
-    g = variable_scope.get_variable(
-        "attention_g", dtype=dtype,
-        initializer=init_ops.constant_initializer(math.sqrt((1. / num_units))),
-        shape=())
-    # Bias added prior to the nonlinearity
-    b = variable_scope.get_variable(
-        "attention_b", [num_units], dtype=dtype,
-        initializer=init_ops.zeros_initializer())
-    # normed_v = g * v / ||v||
-    normed_v = g * v * math_ops.rsqrt(
-        math_ops.reduce_sum(math_ops.square(v)))
+  if attention_g is not None and attention_b is not None:
+    normed_v = attention_g * attention_v * math_ops.rsqrt(
+        math_ops.reduce_sum(math_ops.square(attention_v)))
     return math_ops.reduce_sum(
-        normed_v * math_ops.tanh(keys + processed_query + b), [2])
+        normed_v * math_ops.tanh(keys + processed_query + attention_b), [2])
   else:
-    return math_ops.reduce_sum(v * math_ops.tanh(keys + processed_query), [2])
+    return math_ops.reduce_sum(
+        attention_v * math_ops.tanh(keys + processed_query), [2])
 
 
 class BahdanauAttention(_BaseAttentionMechanism):
@@ -578,12 +793,145 @@ class BahdanauAttention(_BaseAttentionMechanism):
     """
     with variable_scope.variable_scope(None, "bahdanau_attention", [query]):
       processed_query = self.query_layer(query) if self.query_layer else query
-      score = _bahdanau_score(processed_query, self._keys, self._normalize)
+      attention_v = variable_scope.get_variable(
+          "attention_v", [self._num_units], dtype=query.dtype)
+      if not self._normalize:
+        attention_g = None
+        attention_b = None
+      else:
+        attention_g = variable_scope.get_variable(
+            "attention_g", dtype=query.dtype,
+            initializer=init_ops.constant_initializer(
+                math.sqrt((1. / self._num_units))),
+            shape=())
+        attention_b = variable_scope.get_variable(
+            "attention_b", [self._num_units], dtype=query.dtype,
+            initializer=init_ops.zeros_initializer())
+
+      score = _bahdanau_score(processed_query, self._keys, attention_v,
+                              attention_g=attention_g, attention_b=attention_b)
     alignments = self._probability_fn(score, state)
     next_state = alignments
     return alignments, next_state
 
 
+class BahdanauAttentionV2(_BaseAttentionMechanismV2):
+  """Implements Bahdanau-style (additive) attention.
+
+  This attention has two forms.  The first is Bahdanau attention,
+  as described in:
+
+  Dzmitry Bahdanau, Kyunghyun Cho, Yoshua Bengio.
+  "Neural Machine Translation by Jointly Learning to Align and Translate."
+  ICLR 2015. https://arxiv.org/abs/1409.0473
+
+  The second is the normalized form.  This form is inspired by the
+  weight normalization article:
+
+  Tim Salimans, Diederik P. Kingma.
+  "Weight Normalization: A Simple Reparameterization to Accelerate
+   Training of Deep Neural Networks."
+  https://arxiv.org/abs/1602.07868
+
+  To enable the second form, construct the object with parameter
+  `normalize=True`.
+  """
+
+  def __init__(self,
+               units,
+               normalize=False,
+               probability_fn="softmax",
+               dtype=None,
+               name="BahdanauAttention",
+               **kwargs):
+    """Construct the Attention mechanism.
+
+    Args:
+      units: The depth of the query mechanism.
+      normalize: Python boolean.  Whether to normalize the energy term.
+      probability_fn: (optional) string, the name of function to convert the
+        attention score to probabilities. The default is `softmax` which is
+        `tf.nn.softmax`. Other options is `hardmax`, which is hardmax() within
+        this module. Any other value will result into validation error. Default
+        to use `softmax`.
+      dtype: The data type for the query and memory layers of the attention
+        mechanism.
+      name: Name to use when creating ops.
+      **kwargs: Dictionary that contains other common arguments for layer
+        creation.
+    """
+    self.probability_fn_name = probability_fn
+    probability_fn = self._process_probability_fn(self.probability_fn_name)
+    wrapped_probability_fn = lambda score, _: probability_fn(score)
+    if dtype is None:
+      dtype = dtypes.float32
+    super(BahdanauAttentionV2, self).__init__(
+        query_layer=layers_core.Dense(
+            units, name="query_layer", use_bias=False, dtype=dtype),
+        memory_layer=layers_core.Dense(
+            units, name="memory_layer", use_bias=False, dtype=dtype),
+        probability_fn=wrapped_probability_fn,
+        name=name,
+        dtype=dtype,
+        **kwargs)
+    self.units = units
+    self.normalize = normalize
+
+  def build(self, input_shape):
+    super(BahdanauAttentionV2, self).build(input_shape)
+    self.attention_v = self.add_weight(
+        "attention_v", [self.units], dtype=self.dtype)
+    if self.normalize:
+      self.attention_g = self.add_weight(
+          "attention_g", initializer=init_ops.constant_initializer(
+              math.sqrt((1. / self.units))), shape=())
+      self.attention_b = self.add_weight(
+          "attention_b", shape=[self.units],
+          initializer=init_ops.zeros_initializer())
+    else:
+      self.attention_g = None
+      self.attention_b = None
+    self.built = True
+
+  def calculate_attention(self, query, state):
+    """Score the query based on the keys and values.
+
+    Args:
+      query: Tensor of dtype matching `self.values` and shape
+        `[batch_size, query_depth]`.
+      state: Tensor of dtype matching `self.values` and shape
+        `[batch_size, alignments_size]`
+        (`alignments_size` is memory's `max_time`).
+
+    Returns:
+      alignments: Tensor of dtype matching `self.values` and shape
+        `[batch_size, alignments_size]` (`alignments_size` is memory's
+        `max_time`).
+    """
+    processed_query = self.query_layer(query) if self.query_layer else query
+    score = _bahdanau_score(processed_query, self.keys, self.attention_v,
+                            attention_g=self.attention_g,
+                            attention_b=self.attention_b)
+    alignments = self.probability_fn(score, state)
+    next_state = alignments
+    return alignments, next_state
+
+  def get_config(self):
+    config = {
+        "units": self.units,
+        "normalize": self.normalize,
+        "probability_fn": self.probability_fn_name,
+    }
+    base_config = super(BahdanauAttentionV2, self).get_config()
+    return dict(list(base_config.items()) + list(config.items()))
+
+  @classmethod
+  def from_config(cls, config, custom_objects=None):
+    config = _BaseAttentionMechanismV2.deserialize_inner_layer_from_config(
+        config, custom_objects=custom_objects)
+    return cls(**config)
+
+
 def safe_cumprod(x, *args, **kwargs):
   """Computes cumprod of x in logspace using cumsum to avoid underflow.
 
@@ -766,6 +1114,34 @@ class _BaseMonotonicAttentionMechanism(_BaseAttentionMechanism):
         dtype=dtype)
 
 
+class _BaseMonotonicAttentionMechanismV2(_BaseAttentionMechanismV2):
+  """Base attention mechanism for monotonic attention.
+
+  Simply overrides the initial_alignments function to provide a dirac
+  distribution, which is needed in order for the monotonic attention
+  distributions to have the correct behavior.
+  """
+
+  def initial_alignments(self, batch_size, dtype):
+    """Creates the initial alignment values for the monotonic attentions.
+
+    Initializes to dirac distributions, i.e. [1, 0, 0, ...memory length..., 0]
+    for all entries in the batch.
+
+    Args:
+      batch_size: `int32` scalar, the batch_size.
+      dtype: The `dtype`.
+
+    Returns:
+      A `dtype` tensor shaped `[batch_size, alignments_size]`
+      (`alignments_size` is the values' `max_time`).
+    """
+    max_time = self._alignments_size
+    return array_ops.one_hot(
+        array_ops.zeros((batch_size,), dtype=dtypes.int32), max_time,
+        dtype=dtype)
+
+
 class BahdanauMonotonicAttention(_BaseMonotonicAttentionMechanism):
   """Monotonic attention mechanism with Bahadanau-style energy function.
 
@@ -860,7 +1236,22 @@ class BahdanauMonotonicAttention(_BaseMonotonicAttentionMechanism):
     with variable_scope.variable_scope(
         None, "bahdanau_monotonic_attention", [query]):
       processed_query = self.query_layer(query) if self.query_layer else query
-      score = _bahdanau_score(processed_query, self._keys, self._normalize)
+      attention_v = variable_scope.get_variable(
+          "attention_v", [self._num_units], dtype=query.dtype)
+      if not self._normalize:
+        attention_g = None
+        attention_b = None
+      else:
+        attention_g = variable_scope.get_variable(
+            "attention_g", dtype=query.dtype,
+            initializer=init_ops.constant_initializer(
+                math.sqrt((1. / self._num_units))),
+            shape=())
+        attention_b = variable_scope.get_variable(
+            "attention_b", [self._num_units], dtype=query.dtype,
+            initializer=init_ops.zeros_initializer())
+      score = _bahdanau_score(processed_query, self._keys, attention_v,
+                              attention_g=attention_g, attention_b=attention_b)
       score_bias = variable_scope.get_variable(
           "attention_score_bias", dtype=processed_query.dtype,
           initializer=self._score_bias_init)
@@ -870,6 +1261,140 @@ class BahdanauMonotonicAttention(_BaseMonotonicAttentionMechanism):
     return alignments, next_state
 
 
+class BahdanauMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
+  """Monotonic attention mechanism with Bahadanau-style energy function.
+
+  This type of attention enforces a monotonic constraint on the attention
+  distributions; that is once the model attends to a given point in the memory
+  it can't attend to any prior points at subsequence output timesteps.  It
+  achieves this by using the _monotonic_probability_fn instead of softmax to
+  construct its attention distributions.  Since the attention scores are passed
+  through a sigmoid, a learnable scalar bias parameter is applied after the
+  score function and before the sigmoid.  Otherwise, it is equivalent to
+  BahdanauAttention.  This approach is proposed in
+
+  Colin Raffel, Minh-Thang Luong, Peter J. Liu, Ron J. Weiss, Douglas Eck,
+  "Online and Linear-Time Attention by Enforcing Monotonic Alignments."
+  ICML 2017.  https://arxiv.org/abs/1704.00784
+  """
+
+  def __init__(self,
+               units,
+               normalize=False,
+               sigmoid_noise=0.,
+               sigmoid_noise_seed=None,
+               score_bias_init=0.,
+               mode="parallel",
+               dtype=None,
+               name="BahdanauMonotonicAttention",
+               **kwargs):
+    """Construct the Attention mechanism.
+
+    Args:
+      units: The depth of the query mechanism.
+      normalize: Python boolean. Whether to normalize the energy term.
+      sigmoid_noise: Standard deviation of pre-sigmoid noise. See the docstring
+        for `_monotonic_probability_fn` for more information.
+      sigmoid_noise_seed: (optional) Random seed for pre-sigmoid noise.
+      score_bias_init: Initial value for score bias scalar. It's recommended to
+        initialize this to a negative value when the length of the memory is
+        large.
+      mode: How to compute the attention distribution. Must be one of
+        'recursive', 'parallel', or 'hard'. See the docstring for
+        `tf.contrib.seq2seq.monotonic_attention` for more information.
+      dtype: The data type for the query and memory layers of the attention
+        mechanism.
+      name: Name to use when creating ops.
+      **kwargs: Dictionary that contains other common arguments for layer
+        creation.
+    """
+    # Set up the monotonic probability fn with supplied parameters
+    if dtype is None:
+      dtype = dtypes.float32
+    wrapped_probability_fn = functools.partial(
+        _monotonic_probability_fn, sigmoid_noise=sigmoid_noise, mode=mode,
+        seed=sigmoid_noise_seed)
+    super(BahdanauMonotonicAttentionV2, self).__init__(
+        query_layer=layers_core.Dense(
+            units, name="query_layer", use_bias=False, dtype=dtype),
+        memory_layer=layers_core.Dense(
+            units, name="memory_layer", use_bias=False, dtype=dtype),
+        probability_fn=wrapped_probability_fn,
+        name=name,
+        dtype=dtype,
+        **kwargs)
+    self.units = units
+    self.normalize = normalize
+    self.sigmoid_noise = sigmoid_noise
+    self.sigmoid_noise_seed = sigmoid_noise_seed
+    self.score_bias_init = score_bias_init
+    self.mode = mode
+
+  def build(self, input_shape):
+    super(BahdanauMonotonicAttentionV2, self).build(input_shape)
+    self.attention_v = self.add_weight(
+        "attention_v", [self.units], dtype=self.dtype)
+    self.attention_score_bias = self.add_weight(
+        "attention_score_bias", shape=(), dtype=self.dtype,
+        initializer=init_ops.constant_initializer(
+            self.score_bias_init, dtype=self.dtype))
+    if not self.normalize:
+      self.attention_g = None
+      self.attention_b = None
+    else:
+      self.attention_g = self.add_weight(
+          "attention_g", dtype=self.dtype,
+          initializer=init_ops.constant_initializer(
+              math.sqrt((1. / self.units))),
+          shape=())
+      self.attention_b = self.add_weight(
+          "attention_b", [self.units], dtype=self.dtype,
+          initializer=init_ops.zeros_initializer())
+    self.built = True
+
+  def calculate_attention(self, query, state):
+    """Score the query based on the keys and values.
+
+    Args:
+      query: Tensor of dtype matching `self.values` and shape
+        `[batch_size, query_depth]`.
+      state: Tensor of dtype matching `self.values` and shape
+        `[batch_size, alignments_size]`
+        (`alignments_size` is memory's `max_time`).
+
+    Returns:
+      alignments: Tensor of dtype matching `self.values` and shape
+        `[batch_size, alignments_size]` (`alignments_size` is memory's
+        `max_time`).
+    """
+    processed_query = self.query_layer(query) if self.query_layer else query
+    score = _bahdanau_score(processed_query, self.keys, self.attention_v,
+                            attention_g=self.attention_g,
+                            attention_b=self.attention_b)
+    score += self.attention_score_bias
+    alignments = self.probability_fn(score, state)
+    next_state = alignments
+    return alignments, next_state
+
+  def get_config(self):
+    config = {
+        "units": self.units,
+        "normalize": self.normalize,
+        "sigmoid_noise": self.sigmoid_noise,
+        "sigmoid_noise_seed": self.sigmoid_noise_seed,
+        "score_bias_init": self.score_bias_init,
+        "mode": self.mode,
+    }
+    base_config = super(BahdanauMonotonicAttentionV2, self).get_config()
+    return dict(list(base_config.items()) + list(config.items()))
+
+  @classmethod
+  def from_config(cls, config, custom_objects=None):
+    config = _BaseAttentionMechanismV2.deserialize_inner_layer_from_config(
+        config, custom_objects=custom_objects)
+    return cls(**config)
+
+
 class LuongMonotonicAttention(_BaseMonotonicAttentionMechanism):
   """Monotonic attention mechanism with Luong-style energy function.
 
@@ -960,7 +1485,12 @@ class LuongMonotonicAttention(_BaseMonotonicAttentionMechanism):
     """
     with variable_scope.variable_scope(None, "luong_monotonic_attention",
                                        [query]):
-      score = _luong_score(query, self._keys, self._scale)
+      attention_g = None
+      if self._scale:
+        attention_g = variable_scope.get_variable(
+            "attention_g", dtype=query.dtype,
+            initializer=init_ops.ones_initializer, shape=())
+      score = _luong_score(query, self._keys, attention_g)
       score_bias = variable_scope.get_variable(
           "attention_score_bias", dtype=query.dtype,
           initializer=self._score_bias_init)
@@ -970,6 +1500,125 @@ class LuongMonotonicAttention(_BaseMonotonicAttentionMechanism):
     return alignments, next_state
 
 
+class LuongMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
+  """Monotonic attention mechanism with Luong-style energy function.
+
+  This type of attention enforces a monotonic constraint on the attention
+  distributions; that is once the model attends to a given point in the memory
+  it can't attend to any prior points at subsequence output timesteps.  It
+  achieves this by using the _monotonic_probability_fn instead of softmax to
+  construct its attention distributions.  Otherwise, it is equivalent to
+  LuongAttention.  This approach is proposed in
+
+  [Colin Raffel, Minh-Thang Luong, Peter J. Liu, Ron J. Weiss, Douglas Eck,
+  "Online and Linear-Time Attention by Enforcing Monotonic Alignments."
+  ICML 2017.](https://arxiv.org/abs/1704.00784)
+  """
+
+  def __init__(self,
+               units,
+               scale=False,
+               sigmoid_noise=0.,
+               sigmoid_noise_seed=None,
+               score_bias_init=0.,
+               mode="parallel",
+               dtype=None,
+               name="LuongMonotonicAttention",
+               **kwargs):
+    """Construct the Attention mechanism.
+
+    Args:
+      units: The depth of the query mechanism.
+      scale: Python boolean.  Whether to scale the energy term.
+      sigmoid_noise: Standard deviation of pre-sigmoid noise.  See the docstring
+        for `_monotonic_probability_fn` for more information.
+      sigmoid_noise_seed: (optional) Random seed for pre-sigmoid noise.
+      score_bias_init: Initial value for score bias scalar.  It's recommended to
+        initialize this to a negative value when the length of the memory is
+        large.
+      mode: How to compute the attention distribution.  Must be one of
+        'recursive', 'parallel', or 'hard'.  See the docstring for
+        `tf.contrib.seq2seq.monotonic_attention` for more information.
+      dtype: The data type for the query and memory layers of the attention
+        mechanism.
+      name: Name to use when creating ops.
+      **kwargs: Dictionary that contains other common arguments for layer
+        creation.
+    """
+    # Set up the monotonic probability fn with supplied parameters
+    if dtype is None:
+      dtype = dtypes.float32
+    wrapped_probability_fn = functools.partial(
+        _monotonic_probability_fn, sigmoid_noise=sigmoid_noise, mode=mode,
+        seed=sigmoid_noise_seed)
+    super(LuongMonotonicAttentionV2, self).__init__(
+        query_layer=None,
+        memory_layer=layers_core.Dense(
+            units, name="memory_layer", use_bias=False, dtype=dtype),
+        probability_fn=wrapped_probability_fn,
+        name=name,
+        dtype=dtype,
+        **kwargs)
+    self.units = units
+    self.scale = scale
+    self.sigmoid_noise = sigmoid_noise
+    self.sigmoid_noise_seed = sigmoid_noise_seed
+    self.score_bias_init = score_bias_init
+    self.mode = mode
+
+  def build(self, input_shape):
+    super(LuongMonotonicAttentionV2, self).build(input_shape)
+    if self.scale:
+      self.attention_g = self.add_weight(
+          "attention_g", initializer=init_ops.ones_initializer, shape=())
+    else:
+      self.attention_g = None
+    self.attention_score_bias = self.add_weight(
+        "attention_score_bias", shape=(),
+        initializer=init_ops.constant_initializer(
+            self.score_bias_init, dtype=self.dtype))
+    self.built = True
+
+  def calculate_attention(self, query, state):
+    """Score the query based on the keys and values.
+
+    Args:
+      query: Tensor of dtype matching `self.values` and shape
+        `[batch_size, query_depth]`.
+      state: Tensor of dtype matching `self.values` and shape
+        `[batch_size, alignments_size]`
+        (`alignments_size` is memory's `max_time`).
+
+    Returns:
+      alignments: Tensor of dtype matching `self.values` and shape
+        `[batch_size, alignments_size]` (`alignments_size` is memory's
+        `max_time`).
+    """
+    score = _luong_score(query, self.keys, self.attention_g)
+    score += self.attention_score_bias
+    alignments = self.probability_fn(score, state)
+    next_state = alignments
+    return alignments, next_state
+
+  def get_config(self):
+    config = {
+        "units": self.units,
+        "normalize": self.normalize,
+        "sigmoid_noise": self.sigmoid_noise,
+        "sigmoid_noise_seed": self.sigmoid_noise_seed,
+        "score_bias_init": self.score_bias_init,
+        "mode": self.mode,
+    }
+    base_config = super(LuongMonotonicAttentionV2, self).get_config()
+    return dict(list(base_config.items()) + list(config.items()))
+
+  @classmethod
+  def from_config(cls, config, custom_objects=None):
+    config = _BaseAttentionMechanismV2.deserialize_inner_layer_from_config(
+        config, custom_objects=custom_objects)
+    return cls(**config)
+
+
 class AttentionWrapperState(
     collections.namedtuple("AttentionWrapperState",
                            ("cell_state", "attention", "time", "alignments",
@@ -1026,6 +1675,97 @@ class AttentionWrapperState(
         super(AttentionWrapperState, self)._replace(**kwargs))
 
 
+def _prepare_memory(memory, memory_sequence_length=None, memory_mask=None,
+                    check_inner_dims_defined=True):
+  """Convert to tensor and possibly mask `memory`.
+
+  Args:
+    memory: `Tensor`, shaped `[batch_size, max_time, ...]`.
+    memory_sequence_length: `int32` `Tensor`, shaped `[batch_size]`.
+    memory_mask: `boolean` tensor with shape [batch_size, max_time]. The memory
+      should be skipped when the corresponding mask is False.
+    check_inner_dims_defined: Python boolean.  If `True`, the `memory`
+      argument's shape is checked to ensure all but the two outermost
+      dimensions are fully defined.
+
+  Returns:
+    A (possibly masked), checked, new `memory`.
+
+  Raises:
+    ValueError: If `check_inner_dims_defined` is `True` and not
+      `memory.shape[2:].is_fully_defined()`.
+  """
+  memory = nest.map_structure(
+      lambda m: ops.convert_to_tensor(m, name="memory"), memory)
+  if memory_sequence_length is not None and memory_mask is not None:
+    raise ValueError("memory_sequence_length and memory_mask can't be provided "
+                     "at same time.")
+  if memory_sequence_length is not None:
+    memory_sequence_length = ops.convert_to_tensor(
+        memory_sequence_length, name="memory_sequence_length")
+  if check_inner_dims_defined:
+    def _check_dims(m):
+      if not m.get_shape()[2:].is_fully_defined():
+        raise ValueError("Expected memory %s to have fully defined inner dims, "
+                         "but saw shape: %s" % (m.name, m.get_shape()))
+    nest.map_structure(_check_dims, memory)
+  if memory_sequence_length is None and memory_mask is None:
+    seq_len_mask = None
+    seq_len_batch_size = None
+  elif memory_sequence_length is not None:
+    seq_len_mask = array_ops.sequence_mask(
+        memory_sequence_length,
+        maxlen=array_ops.shape(nest.flatten(memory)[0])[1],
+        dtype=nest.flatten(memory)[0].dtype)
+    seq_len_batch_size = (
+        tensor_shape.dimension_value(memory_sequence_length.shape[0])
+        or array_ops.shape(memory_sequence_length)[0])
+  else:
+    # For memory_mask is not None
+    seq_len_mask = memory_mask
+    seq_len_batch_size = (
+        tensor_shape.dimension_value(memory_mask.shape[0])
+        or array_ops.shape(memory_mask)[0])
+  def _maybe_mask(m, seq_len_mask):
+    """Mask the memory based on the memory mask."""
+    rank = m.get_shape().ndims
+    rank = rank if rank is not None else array_ops.rank(m)
+    extra_ones = array_ops.ones(rank - 2, dtype=dtypes.int32)
+    m_batch_size = tensor_shape.dimension_value(
+        m.shape[0]) or array_ops.shape(m)[0]
+    if seq_len_batch_size is not None:
+      message = ("memory_sequence_length and memory tensor batch sizes do not "
+                 "match.")
+      with ops.control_dependencies([
+          check_ops.assert_equal(
+              seq_len_batch_size, m_batch_size, message=message)]):
+        seq_len_mask = array_ops.reshape(
+            seq_len_mask,
+            array_ops.concat((array_ops.shape(seq_len_mask), extra_ones), 0))
+        return m * seq_len_mask
+    else:
+      return m
+  return nest.map_structure(lambda m: _maybe_mask(m, seq_len_mask), memory)
+
+
+def _maybe_mask_score(score, memory_sequence_length=None, memory_mask=None,
+                      score_mask_value=None):
+  """Mask the attention score based on the masks."""
+  if memory_sequence_length is None and memory_mask is None:
+    return score
+  if memory_sequence_length is not None and memory_mask is not None:
+    raise ValueError("memory_sequence_length and memory_mask can't be provided "
+                     "at same time.")
+  if memory_sequence_length is not None:
+    message = "All values in memory_sequence_length must greater than zero."
+    with ops.control_dependencies(
+        [check_ops.assert_positive(memory_sequence_length, message=message)]):
+      memory_mask = array_ops.sequence_mask(
+          memory_sequence_length, maxlen=array_ops.shape(score)[1])
+  score_mask_values = score_mask_value * array_ops.ones_like(score)
+  return array_ops.where(memory_mask, score, score_mask_values)
+
+
 def hardmax(logits, name=None):
   """Returns batched one-hot vectors.
 
-- 
GitLab


From 92122d1818bb8dbbab371506f81b2f7344bf8595 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 14:54:37 -0800
Subject: [PATCH 0908/2345] - Add post_training_quantization to index under how
 it works - Add post_training_quantization to devguide as an optional step -
 Add GPU dev preview to index as an update - Remove testimonials

PiperOrigin-RevId: 229823165
---
 tensorflow/lite/g3doc/_index.yaml             | 122 +++++++-----------
 tensorflow/lite/g3doc/devguide.md             |  45 ++++++-
 .../landing-page/facial_contour_detection.png | Bin 0 -> 294705 bytes
 3 files changed, 94 insertions(+), 73 deletions(-)
 create mode 100644 tensorflow/lite/g3doc/images/landing-page/facial_contour_detection.png

diff --git a/tensorflow/lite/g3doc/_index.yaml b/tensorflow/lite/g3doc/_index.yaml
index 1b3f1d616a..7153b7c6f6 100644
--- a/tensorflow/lite/g3doc/_index.yaml
+++ b/tensorflow/lite/g3doc/_index.yaml
@@ -4,7 +4,7 @@ description: <!--no description-->
 landing_page:
   custom_css_path: /site-assets/css/style.css
   rows:
-  - heading: TensorFlow Lite is for mobile and embedded devices.
+  - heading: TensorFlow Lite is for mobile and embedded devices
     description: >
       <p style="max-width: 75%;">
         TensorFlow Lite is the official solution for running machine learning
@@ -13,9 +13,6 @@ landing_page:
         iOS, and other operating systems.
       </p>
       <style>
-      .tfo-landing-row-heading {
-        padding-top: 0 !important;
-      }
       .tfo-landing-row-heading h2 {
         margin-top: 0 !important;
       }
@@ -71,58 +68,16 @@ landing_page:
           icon_name: lens
           foreground: theme
 
-  - classname: devsite-landing-row-logos tfo-landing-row-heading
-    heading: Companies using TensorFlow Lite
-    items:
-    - custom_image:
-        path: ./images/landing-page/photos_logo.png
-      path: https://www.photos.google.com
-    - custom_image:
-        path: ./images/landing-page/gboard_logo.png
-      path: https://play.google.com/store/apps/details?id=com.google.android.inputmethod.latin&hl=en_US
-    - custom_image:
-        path: ./images/landing-page/gmail_logo.png
-      path: https://www.google.com/gmail/
-    - custom_image:
-        path: ./images/landing-page/assistant_logo.png
-      path: https://assistant.google.com/
-
-  - classname: devsite-landing-row-logos
-    items:
-    - custom_image:
-        path: ./images/landing-page/vsco_logo.png
-      path: https://vsco.co
-    - custom_image:
-        path: ./images/landing-page/shazam_logo.png
-      path: https://www.shazam.com/
-    - custom_image:
-        path: ./images/landing-page/nest_logo.png
-      path: https://nest.com/
-    - custom_image:
-        path: ./images/landing-page/loseit_logo.png
-      path: https://www.loseit.com/
-
-  - classname: devsite-landing-row-no-image-background devsite-landing-row-67
-    background: grey
-    items:
-    - description: >
-        <em>“TensorFlow Lite helped us introduce machine learning and AI into our
-        app in an easy and streamlined way. We could reduce the size of our
-        models while keeping the accuracy high. This helped us create an amazing
-        fishing experience for our users by allowing them to identify any fish
-        species with just a photo.”</em>
-      image_path: ./images/landing-page/fishbrain_logo_big.png
-
   - heading: How it works
     items:
-    - heading: Build
+    - heading: Pick a model
       icon:
         icon_name: build
       description: >
-        Build a new model or retrain an existing one, such as using transfer learning.
+        Pick a new model or retrain an existing one.
       buttons:
-      - label: Read the developer guide
-        path: /lite/devguide
+      - label: Pick
+        path: /lite/devguide#1_choose_a_model
         classname: button button-primary tfo-button-primary
     - heading: Convert
       icon:
@@ -131,18 +86,29 @@ landing_page:
         Convert a TensorFlow model into a compressed flat buffer with the
         TensorFlow Lite Converter.
       buttons:
-      - label: Read the converter guide
-        path: /lite/convert/
+      - label: Convert
+        path: /lite/devguide#2_convert_the_model_format
         classname: button button-primary tfo-button-primary
     - heading: Deploy
+      icon:
+        icon_name: settings_cell
+      description: >
+        Take the compressed <code>.tflite</code> file and load it into a mobile or embedded device.
+      buttons:
+      - label: Deploy
+        path: /lite/devguide#3_use_the_tensorflow_lite_model_for_inference_in_a_mobile_app
+        classname: button button-primary tfo-button-primary
+    - heading: Optimize
       icon:
         icon_name: bolt
       description: >
-        Take the compressed <code>.tflite</code> file and load it into a mobile
-        or embedded device.<br/>
-        See the <a href="#build-your-first-tensorflow-lite-app">tutorials below</a> to build an app.
+        [optional] Quantize by converting 32-bit floats to more efficient 8-bit integers or run on GPU.
+      buttons:
+      - label: Optimize
+        path: /lite/devguide#4_optimize_your_model_optional
+        classname: button button-primary tfo-button-primary
 
-  - heading: Build your first TensorFlow Lite app
+  - heading: Build your first TensorFlow Lite app with Codelabs
     background: grey
     items:
     - classname: tfo-landing-row-item-inset-white
@@ -160,28 +126,40 @@ landing_page:
         We love to hear what you're working on—it may even get highlighted on
         our social media! <a href="https://groups.google.com/a/tensorflow.org/forum/#!forum/discuss" class="external">Tell us</a>.
 
-  - classname: devsite-landing-row-no-image-background devsite-landing-row-67
+  - classname: devsite-landing-row-logos tfo-landing-row-heading
+    heading: TensorFlow Lite users
     items:
-    - description: >
-        <p>
-          <em>“The release of TensorFlow Lite has allowed us to deploy an engaging
-          real-time experience to our users that eliminates the requirement
-          for a data connection. TensorFlow Lite’s ability to compress and
-          optimize the TensorFlow graph for mobile deployment has been
-          transformative in expanding the capabilities of Snap It.</em>
-        </p>
-        <p>
-          <em>Through TensorFlow Lite, our users can now enjoy a state of the
-          art, computer-vision-based food logging experience without worrying
-          about signal strength. We look forward to future collaborations
-          with the TensorFlow Lite team.”</em>
-        </p>
-      image_path: ./images/landing-page/loseit_logo_big.png
+    - custom_image:
+        path: ./images/landing-page/photos_logo.png
+    - custom_image:
+        path: ./images/landing-page/gboard_logo.png
+    - custom_image:
+        path: ./images/landing-page/gmail_logo.png
+    - custom_image:
+        path: ./images/landing-page/assistant_logo.png
+
+  - classname: devsite-landing-row-logos
+    items:
+    - custom_image:
+        path: ./images/landing-page/vsco_logo.png
+    - custom_image:
+        path: ./images/landing-page/shazam_logo.png
+    - custom_image:
+        path: ./images/landing-page/nest_logo.png
+    - custom_image:
+        path: ./images/landing-page/loseit_logo.png
+
 
   - classname: devsite-landing-row-cards
     background: grey
     heading: Updates
     items:
+    - heading: "TensorFlow Lite Now Faster with Mobile GPUs (Developer Preview)"
+      image_path: ./images/landing-page/facial_contour_detection.png
+      path: https://medium.com/tensorflow/tensorflow-lite-now-faster-with-mobile-gpus-developer-preview-e15797e6dee7
+      buttons:
+      - label: Read more
+        path: https://medium.com/tensorflow/tensorflow-lite-now-faster-with-mobile-gpus-developer-preview-e15797e6dee7
     - heading: "AI in motion: react in the real world"
       image_path: ./images/landing-page/ai_in_motion.png
       path: https://cloud.google.com/blog/products/ai-machine-learning/ai-motion-designing-simple-system-see-understand-and-react-real-world-part-ii
diff --git a/tensorflow/lite/g3doc/devguide.md b/tensorflow/lite/g3doc/devguide.md
index fdd02638f9..cbad036407 100644
--- a/tensorflow/lite/g3doc/devguide.md
+++ b/tensorflow/lite/g3doc/devguide.md
@@ -180,7 +180,6 @@ bazel run tensorflow/lite/tools:visualize -- model.tflite model_viz.html
 This generates an interactive HTML page listing subgraphs, operations, and a
 graph visualization.
 
-
 ## 3. Use the TensorFlow Lite model for inference in a mobile app
 
 After completing the prior steps, you should now have a `.tflite` model file.
@@ -221,3 +220,47 @@ devices. To use the converter, refer to the
 Compile Tensorflow Lite for a Raspberry Pi by following the
 [RPi build instructions](rpi.md) This compiles a static library file (`.a`) used
 to build your app. There are plans for Python bindings and a demo app.
+
+## 4. Optimize your model (optional)
+
+There are two options. If you plan to run on CPU, we recommend that you quantize
+your weights and activation tensors. If the hardware is available, another
+option is to run on GPU for massively parallelizable workloads.
+
+### Quantization
+Compress your model size by lowering the precision of the parameters (i.e.
+neural network weights) from their training-time 32-bit floating-point
+representations into much smaller and efficient 8-bit integer ones.
+
+This will execute the heaviest computations fast in lower precision, but the
+most sensitive ones with higher precision, thus typically resulting in little to
+no final accuracy losses for the task, yet a significant speed-up over pure
+floating-point execution.
+
+The post-training quantization technique is integrated into the TensorFlow Lite
+conversion tool. Getting started is easy: after building your TensorFlow model,
+simply enable the ‘post_training_quantize’ flag in the TensorFlow Lite
+conversion tool. Assuming that the saved model is stored in saved_model_dir, the
+quantized tflite flatbuffer can be generated in command line:
+
+```
+converter=tf.contrib.lite.TocoConverter.from_saved_model(saved_model_dir)
+converter.post_training_quantize=True
+tflite_quantized_model=converter.convert()
+open(“quantized_model.tflite”, “wb”).write(tflite_quantized_model)
+```
+
+Read the full documentation [here](performance/post_training_quantization) and see a tutorial [here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/tutorials/post_training_quant.ipynb).
+
+### GPU
+Run on GPU GPUs are designed to have high throughput for massively
+parallelizable workloads. Thus, they are well-suited for deep neural nets, which
+consist of a huge number of operators, each working on some input tensor(s) that
+can be easily divided into smaller workloads and carried out in parallel,
+typically resulting in lower latency.
+
+Another benefit with GPU inference is its power efficiency. GPUs carry out the
+computations in a very efficient and optimized manner, so that they consume less
+power and generate less heat than when the same task is run on CPUs.
+
+Read the tutorial [here](performance/gpu) and full documentation [here](performance/gpu_advanced).
diff --git a/tensorflow/lite/g3doc/images/landing-page/facial_contour_detection.png b/tensorflow/lite/g3doc/images/landing-page/facial_contour_detection.png
new file mode 100644
index 0000000000000000000000000000000000000000..27bb49826ff3246bd5a971e7ecd0926121dbf749
GIT binary patch
literal 294705
zcmeAS@N?(olHy`uVBq!ia0y~yU{+vYU<&16Vqjoc*~Rvgfq`pFrn7T^OJ;FFPGThk
zgGcAoY2_iom14)=SL^S+AS5tBs7iPV2a5p5Tn4U72i-iFSe$w^J8p;u9^AM<$yfHG
zqaf>XrV|kYfjP}djK3Rrv`$$>2#8Mo+pK!-&AXen)#odp*Pi=+_x|^LzpMYpFx*m}
z7oz26%CJe5E3%}+UfWP}rt2GKmMaZBRSl0nJ`LKxVg(07L+AJVkDbL87#Oy{tqfZE
z<Np5t*Dr{0Jgs3+;bI74EvhqPC}LvBeCnOb#h@e3a7IjX2^Yf}A%+>7jx0%Gc%#8!
zaQXO7UWOSfAJ_0R%-D3}1p`BlK!eNUk0~!1uJkh;IQ4M3HbcfPh6fsg9V!f40vJ9#
zO#i^bu;nX*fvR<NBSS+O6T_O||BojBGd~|PRlR@ux<?-)85kNwG$wf;-E;K+^U^=>
zPl#!&XU$+_&~+A*;$_HUIMKO=^`X|ZCFgqPBur#TXfmF%uu%BSbaoa04GrQF>#Vfr
zNg5u^T$Hfxl>5Qnxd}I$GN$eikP|UJXmj&>)TuQV_Mx$g;ZJ8y7o2Ty&~nj_nZi2f
zyEy;P6JBsB^;Aaa`OHlTS9@-L+J8p%(W+0Pk{cJUlB~R6EqqEi)Rv>A(NQZ_X!)s$
zT7N?S1r<)wir3%^eEDgP#MQ`Fop06ql)OsqXYSdsg0n54gGo%KncJd?!$I*(C<6nF
z&IG+_otKP7bnI7#=%f^?*0gKtq`Ry0Hyl}#a_vWcT{*u1gF}hO{jEmn4GXv!Tt37m
z+p}5dFkX1|Yu(K4%op4l&h2}wooe}5ijRSTUnGEC?M0*Dj7HNN&Bi+##S0WT4k-vF
zDRMqzls8h4PT9aQ?U0s6gVGHq>jg634%k_6xId6EIh1L^KmCE!otE7n1gaLu#~l=?
zX$WL-?BI}1RH*7OOlrKLc#cE*qhr$q4iza+r=STXE=?K}m!6PX!lCJQYT{lG-cQCw
zT$cSYj#d+Yd*~ZA>~XbQEaBxmbA@77r=RQ174lMj#}*Y|EGbc()&6Yp><d9#xNLjO
z7CgV0{UUY?du=OUazmh@>=~BL$8SV1h;^zaXU^b>?h{Q?jSvlMH%)$fBX<qE^bxy<
z92RVSt(%WmK6EN9w6M!l%(Ki>UMEv7aJ?b>@ZDoyAGH<S|4_1r@gCp%&gYMWKfe6o
z`h%&4bDvbbP`zNi`2GI>4hjM#9APZwP8y1x6FeiFZg|{qU7@~3z>2fBB~fYO3imC_
zRl>@hHjA1f9CVcB)aEJQQ~D-sKWT+Z@+4!;8B=^FO$eLp<<ahu?z!FLyyxN+-c77G
zC(C$>O%R`KKKc9P*(c3U{CyJGrFOLIk#FbHl%TLcFU`^^-BaABl!s_to$|_YuB&KZ
zc#!u>_m$IE?!Dq~qB~1hx52D^*^=BvvJ19ecz5~U#g`@CR_*hA!Y`O#%zh#M!vBT*
zRf#OVtQgbATU=YYwz6%F&RT!<<SVgPb+0~O`FzFuO8@Hn0rFQAvLmusGCr70%MQty
zbLqu3iEAktR+(;DY1w&KY?f5Ier^>DJRPL%;k{t$`mO%2o5JLolV=z*Nz@#ZI#zeg
z)y;nLfh|czGiRPrdM230UhEO=T|KL8PuaS%)^Cz;<z#)12eq$OzIJm>vCnev=bu#$
zd)7^EnQZ2>YIaHXrH*$gv3=_Twg+sF`@Zg7y6v>M331czh3$3Qd;FJ0ZS>#BU)^8I
zU&mjOpL1r;8`GKgI}1K-Jju@!z#QG2(X7h$n`fOgkF=Czn}iqdX&!wEKB<=yg*`^S
z+ob23N*Sa{{FdnM*)BcbxbEbow4WP~tO*K}&OUMEfXWq@%uUyq98S4%<&?>}lC7D0
zUnU!ON#52uo3lG|_00)8H~!@Ly!6@DXZ$)R#B#cMy61|$6;qxTHjQ)IYaMCb_=uNL
zg=;^p$yzHHd3kN^TJfmx$om@=Z*0ijlv8%|&KB3)*j(cr?VS7@3(NA#ly=9K%?<t#
zY_rT`S%~k;ZOd=|zWMyd^;_SqURbzTtSQSVsj$6TllAG!+Th#E>xxTCMXcYI*SuKr
zROH!~TPttAT)TO#@%^2Le)e(SRlSkg>OJ}N?98&uEtkvucRxQ>F!$lEb6c-|T|ImE
z>@#c6<sP_u$M)#nOM7RZO>y67zDd9Of$bmLu!=d~emr^kY~=@=XFr!u-y1&dy5f5A
z%WIc$FMmJ(-t*j7XFuP2efRO<Po5utUR-|s`jP3+qo4ksl6*|CV3p<hzd3)5{__3a
z`>p)T^q=#;$a8++i4eOW)WO=uBF0w6lF53Mbt=0prx~vw|2kedu0q~Op-Vzt{Ji{k
zrEZ-yee!9i>79iJ-#4Tt=04o=VaA6~AC?Pl6WrACO!7s~2FcrLHnTDfgN-&D3MuWJ
zcw0%{J<?UT=U1v&V%U{6U6JP(&0RFzZToS9q-V!vx_%10mfvmpEpbli9HVunakh1S
zHNJm5oD!T0o%|nNIg;9WebSf7E}p(8k39MD<kgdhHyt;ZpLsX`vDNYfzw0rpckV1+
zf9CRh^?BuK`+n8j`}0`y$CQ>S&YJNR6ZdG{E6_6EYE;@Gf8hSfdesLU+ql_U?j1^c
z_}S$BmPcE}U*3Cp`x3iRjj@*zn=9j@qKmW7TsUJpQ#o>CB>SdgM>rq-$_Xm-`li`A
zO~w1twgVZC@0$Ep&fVg0Yx9NThkdQin+hVg%1k^z)m(E1*UCq`E=BDOyE*&u<lu>)
z3j_Zwdc1Si&PjJ(cjt=dNBptTw|Ku?b%*L7)wSNf)7C}Z+I=Sf=DnvUK3#dbR{KoY
zll3p+B{zL~pml21n$Z30%3@;o{Q5hq#q{jYwx8Oce_v}`vsO1idxB`m`p;3@({1zi
zX4D>Mo6egmy-$>jYrUAI*i@0)n<ws9d}UuQ?^Q2xd&;(z$(wIKyBp0BtrER!+pj#^
zh&7RO!tZR)J+`gw?V7i(x9`87vH!8x$t{y?tu?KuT3351d(A#|=Jd8HwVLNO_D4UA
z4~)+Y*txJMWqQK(wRh*-Ev@GHF7o#0pT#eZS64fKGYg3foBBri&FP!pbL#*6`!@fE
z{ne1y;o_n7whERPiVc1xME6$vo>gpio}GO$D)sGwhnDU6a-24oLek^6Z;xBP?}dg;
zs9f#O;#22ik00vaDwi(5-a4^x(I=J1DT&ONHta}TVYqnXa?O=1KZo2{c`a@Iv&s7{
z{uDjfa`I)D@vU^T`Ag@VwYM(Qdw-{(drRtvnqNQnUHiF~Utc9|$7QqCYab{5IC^?|
zlYVT*_iX#}o4-H3)Sega7n>TNyXRTi+K0Pdf1SL4*T1Swhu1EC*!H`3*=#xU+1AU;
z=k0x3{rb<_1GlHYD|jEX+xl}}dU(wBz3(jV+J0yG9(PCle*I^K=LcULp4$JRpG!X5
zMz1pI!;)9#v+uvEzEgGX3){E2da2J}66N>5zxw{u{rz8_+b_1)uFU(h=UMCL?#s*v
zna$(R)!qE>^((7n!vp@a_FD1L`|UoeKR^H4xzqV?eoFqadwDg>zrQ>1JM(>X`{K6w
zH7EWVSOxtEeE;~M{~Q0e^M&o~<zv6J{#g2D_1Wvs@*m!p>rL(N^{h0KV_-;NXjrzC
zagQy7gFjQk9hL=W<nB*qWMGiI|CaGs`Go&x9`80`VA#QUh0#HU{lHe%2NvuH7O^(m
zV-h&Z*zuI%)uTmK|0F+|h(9mX&r)XCmy+`)mWkma+k@=yH*RD+zH#Gv7Q-7xhtusJ
z|H?2NDB(D;k%!^Ix30(9591%n8!~ZmuszO|VPIh3EbxddW?<kJ24O~qS#u;97+3={
zLn2Bde0{8v^K<nQQ}UBi^(t};7(l?rzM>#8IXksPAt^OIGtXA({qFrr3YjUkO5vuy
z2EGN(sTr9bRYj@6RemAKRoTgwDN6Qs3N{s16}bhusU?XD6}dTi#a0!zN?>!X@`|lM
z!um=IU?nBlwn~m53l$=K0~DO|i&7O#^~`jW4NXiH%q{f{&5TVfEp-%(3=A#w4J`Bx
zjdcyptV|57jEoeZK*>(QrYI%ND#*nRYFAN8nypesNlAf~zJ7Umxn8-kUVc%!zM-Y1
zrM`iYzLAk`QA(O_ab;dfVufyAu`<jAm(=3qqRfJl%=|nBkco*&`K2YcN=i^;Q!6qN
z=H?Yc!znpGS05y!Z>VPg@)X3kxdm_+ps1`!EI?71k_=Z@P+F7&c4tbmerir?Zfagh
zvA&_6A>7b{jQo=P;*5M8M#H61EU3sW@b!h;fz>U!xo|6rOI*uJ@T)EksVqp<4@xc0
zFD*(=buCNHD^XTb0EK{6KxRd1PJ~NlZfag}W`3TnlBtOqvQ#9pl%+W+Tnnt6i&7Iy
zQe6^DQf-xt3=A!G4GeV+%|Z-~tc=a93{7<nEUXL+AWD7nQ!>*ku`0E&GC)<DT9TNO
zSdvIkaZ0jPT7FS(Vu`I%W^Q77s(wLUx{?AsEUldLbMlLP^HWl7m5h<aJ)NBcit^Jk
zb5d=UpuIt`BYl0XJoAc667!N%JzZ>-Dsl_-GBZ=G3=@+RQ<BULb&bs|QglsB4N`Oy
zjg2gIQ<4o043bkV&CHUFVftP2lS@Id57VDQqJICP%=FB>#2m8C0cBrMva#~WEH23}
zs)S~+;M77WD<@L{oKmb3lR=rqR;f6#EHy<5Dws*O{~?B!XQbv~qy^*%hA1n}Pb(=;
zEJ}4uPt7ZV6(Rw!Jfh(23@S7%6*PPklQZ)`8AlW5z~qd?y!6x*TP1xZeLDpkeNbM9
zcupUjU6A|@mP0BRV9JAB+(1ll5e6#G>=Y2?WNKcDtx}Paz1^n$#&HZRdB;3m978JR
zy!jhjtbB5+YL{L)i^MyIg9Yp}W~QV)W#6o_)x>yG-><U%cb3=tZW_)!xn};CqWAo>
z=9lEg9ba>Pj-2(i@~FF^YgfISwrZ2s%&BL5CV2|HbDY5Ook5^ZzU+Wx%Csx-OV?|~
zPZMKfV`6k?@rsD5e!Df2gOL*k;$?rluZDBu%~hJFPJn<9qRN76TA*OPL}!Ak3Ix;~
z^<dFfgn)<IZb}hwvkZj}dU-&=kFJRvVNMWmD7?pIgAf$#OCrIFhuUq1&#pDMw6x?)
zMQ}X)zPAnJ9{Vtf<Sp0szYkSWQTcOF@prAhf}-NX+1*Qdrynf)o@0D9{LOWlJjSo7
z{q^5Bct>A2we^~J_AjkMkZ+_txWb&A4!xK1mj2Zxs2>@{yyA4icF7lv3{^7yjBhTn
zS?yRK@%>}|lE`n{T@GdZ{XdUcNJwbkH?Wr<mUk^Roz8fhr}pTyZ7+l-F6C`lny&9$
z{h`hK<&RJMF5Tu~&|}`B)|Yz!VNh8ncl?I8@rR55O<iicdy8xQ)GbTPd%F9(_q_SD
zUbD6(#MZ~-$1x>v5Z{-}d14#;Pmpu+wKd)`)68PnN~Y+}Xk|RGRBwjv6s_FqHofCc
zt1GttW~`PBiDR>RxPN`c^>y}o^D=^KUcZydI5;;orvB2|)glSjRxwYqI5|1vox!2}
zAv7siHd|FbndxHp3Hy#3o&>QAu}ih)DQ;SAv|wdk$-|e8X55R}c>*RTKa4u%G<}_<
zaNXR7FSETXw%deyXD1maOu3xV`jOFU$*a4;M};pn<!{p7{V>98;@Xy$^}^uT{!#u=
zwtGA4l}l`kEDrEyUDdsj8_^wm)8KwqxlrY$sCYJ>1dmOtcy~wCd$E6E+x_vxKC2f2
z(;Bj;B{P;vPM98_WZ!@4^7MmwEBLQ!-+x*^EknRFyYS%0#Yd-_Je=j2XKZq?VA<o%
zg3{76ecxy)KFn?dC#8Q+H#8U>%Q)M^x$(gDk1A8WqZ!1cvz)v{bEjWpxTeF@@HOd{
z^qY0!!B+V-Hn#KM`?+m-elZ~3OE|dl_p?fyisj$u>gIM#EHPabUAq1Lg*)G0i>!IU
zuY7WCfW}(W?gi{?cvdC6eZ<(4b!NN4s)?Gs3j!Xz`Bt6#=ZCq6sNnH;2hA;~`TA%p
zK78E>cFw)l+U0qvo_$?6OnyzBXj`Hi*2-kK%4y*i18(7;!QWMOPY#@(=ISzQ>8996
zhQ1Q<Pd6`=x3B+QzI*b;cki}8KKLar(19uBjqRWCK(VB7>5ZqDv`&S*FgSEC!F<!|
zpVxAid2nBA)~_s&TCTyn(T!ny#<gRH-}GY(oSY8H8-b%?2V*jir$z13&!61VV^~Dr
z|MJp*t0;OxN#C&hgWmM0xVfeA_vG&}I*7ayE33M{^HlngSigS0>EY=m1`YXltv9UJ
zoI7oX<kaUE1Sc%&5Pkdh{i}#8t8awwTAduI6#hM!LE`FL?-#Yo&o{50=6Ce`jy+b!
z#)aP>YUovz8+Dz3=jWz%`>Y7FK~k37I&q(+hreyw(jE9^UT2!B%AY_;F{1b85?lO>
zeYN`z@WvIbJ|k`_og|miYh`Q2^5I*?zcfo$fjC?K-A%r+H$)$A-Qv8~DEQ%n=*d%O
zcIuw#JABS=_X@*h!MTac<m}i!tlfQUUHIgp*LzlUE)#HlJxRA}Las;WwOm8N>wD{R
zCtb5H<ec_uM*OQ3^{ZVanzjj-U*6Qc+ACIU8hv5Yc7xT1%g!sEc4qMMWw;{odgE*U
z_jjTyQ!-~JwFb?(8@4&9Inn9Rb!KpJ7qd9q@3=3!XkL*HLu$b3&*?|(@5!BIoYOw_
zh~lYh2kxbYACstJU|<!QeWRdH=!^Q^B}*MyMP~CHXJ+{OaPIf2tJ+nfYd^f=Iq+}s
z@*{yK6y)~?1pZLGUKyVm<!P?B+Iy$R5u;;wl==dfb$#iRNbD_KU-PZ(dBctn@h$(N
z53^_XUz1{W6ORzT-={JuMR%>KzJp1(Mc9`I>i?^q4#}HBBHrzId&=%-icXog`}TiL
zlB&FV$d1|Ib!g73N0m3K{jW~#oYOf^b>G*#l0zMd`<RcaU7L4#-VdjxYAURy{q=wU
zc&t3R{onNeOE$%_a4+lNnfU+Z$y(oa>vw;jpS70p_qPJEpJHaF7iNg~&)?1yG0*<6
z+l1Nk&cD4VuHt*~+uiKc!#{avRS0WmKa2lnzs_Zw$ZUoHe);3C9(Jrev1Qk-hiy`7
zDk^^_DuRo-+jom+-xUjZB4+gLL->3f_7C4O)*XMBwe7{Eg%$I%d$XFOyZl<h47S}B
za^v}2kSx7YLiDc3<*xj{4_-Ro*;2eR$2{2T)~l}$*=v7$?YFJT6MOcudAUkPpm6+|
zxccAEeqQL;WJ{{DQx^_R34hzl@myo!l@trPn6qLZgV&t?{FBd3d`5d%J99%qSZc_X
zTecsQ@9p4->X#_vll#_QzVElXKljpx?EEmD@@0<goSgMNpfdmAY##52FLy|O_58@f
za7Jmu<v&^5I^-7X?c1VfHg{{inbz*p`BpcZ886?su<~8=R@1#q49nPdOC{{Odw5Hj
zL?Q2jaG!OVVHV*zpXSfIQ~FBJ>TlbT9F6`dO`GD*ocZ+U=Jx%6=WkT3ak<;!8`Q2G
zsBJ8~#b;62_7@@IuRqP1ws(`7_69A+k`)UNt#S!t@VU$we@ZQx(eTomYJUDN4VkqK
zvpu6ck9&FiSnLd{hwuADBs<BtUoKsfe3^0k9;@oVe^%I5tvI~;c~rL6x4qu0E<6m?
z&Nf=QJNm3+2<Jh8(5^YnD?Gy(b}ZB6NQ*sWVt(%b+vT%PNM?q+)V4il{JG`GyZVlc
zoQtK(T=l+Qn~^G(_*sHM<HVM#k9)GWR8^b2pL+g;nrMH_^yhn%ocbltg{SVesrs<)
z)wR1!<{YVidcG{Et$8byJ^lNGms=OOZsc3np(x^c3glczuyZwTy!aG%>+p=9lJnm^
z<Gav&gI)aOwKd@rGi7CiSRFo36Pox>#Pia+W4m0|AK0~JSzrmn+suuMaVo5_+)llb
z(;3!eURjqnRoAx8D<#z+FLN62bAC1c--oU5?^=COq-pDGQHH5+{pP)x!CF7{JVQz5
zwe@yC8$z^g%a+9I9x^U+kB!%_?$b-%qv>_gODVX=y761^8D_T)p9Kz-oZdJiE|OUx
zc-D*Sr)E`N{$#LfBj1(nUvIp%uaa7<u}krX^k?fy|7{f?@=Jn>ikjkt`!ko_`F8dw
zhp$rfp?$SmryW_nqPnl**w!r9Lp}!AZ8X!;@=T<2UW9AMD_&YtCM`HKOk}&#S*2%=
zTMvEPJLB4_TKA&k{dx{c!8+nek3++6>J_Zb%ssS1r1nGj{$I1tpWDoNx+B5$q1Lqj
zyY6XfeQ?cuwTpwpC9->+=l9JC!RBx3)_p5tb#A|yo>Qu`b>}2jIUl)WX`T!h`LE07
z&6qIZ1A8~9s@nH&LC}M<-yFCXnEgunyYf_-y|wKU6P8=I7VEFGeY5%b_LG;}oHEzu
zSFD;P{clwnKSPZ0vT{=`2IXVV0yx~V_udj%uP19LHu22FKG)lV*Pra@s_ihmb5O2!
zp`2ZwsV+kXrx*iQ`HweWCPXQ)zuKiHVQ-owtsuCp_g{*R+lJy5@18I17O+i;mGkU)
zBr~_*$PJs86Kap=e?E3Wed%R6nVM`1If3J|@-JR+4tOMft7xIGB_Ahe{RVJW%;xUW
zS$;f3m_<PSz=Rde<&REimNg!kowq*z&bBvRjM5*Mt$w#f>U>*QY<8fzt!L~yv9oWE
z#4fTZY3|D1!8cWQrHtp+6WLeOFXwmkmcNSrf4ZK@Vb810mk+PV{QK$WeTA;#DccHG
zYZ~eK>{Y&!>otGY_Uo3nPCs(*mg(kueJVf1P=#@Q_?9O{EVDIPV?Li-dGKAtYxSs&
zoJ%bO-e_?~MaeI*xzy5PUjS-)efZiIduxN{+g!B+Kdvn`v7Nu)c2<7kHofzVXP6hA
zT~S-Q#L=r?&)`0zm&@W`rA`OWzKZywc$PUY<a<G^RCo2hJxM1_F0PSKHEm)IzCJ0a
zJ@e}o-agZ$7ox)1=U1BM^USvY6+HFrzfV8!zp=0Kn*ZyMyxoFTkJdiX-4!tLL+MJP
zhI_|MMXx)|*T{<EkS@$tesGaH*Esf;9K)yIm(zRpnIw7aedN~q<yW(DidV3cOMQhp
zTaeGZ7cu?9pwgt-6_h>W%}dhWN-`XqQJ${5Jt?%}n$Z%yeN&w8=u9}8zMCnuN3nzb
zzR(H}fvzdsJMOCAXnnSJ``<R%y5<PIBgca8mpv?%NqE7MA^RprJ}xHda&<)V9j;Oa
zt9zTWvse{o<XIZ+TYqWSZC#E#bp@wOTvG$0KJ1*Sm#ZHfRqTCrZ`{JX$jcp>sjNMQ
z<{nF~dHG&BeEj`at>y26UQEAc#kx;j<xe8G@w{I3%_X)iR}T6uPd~0d|M>HyjwxH8
zU;j9xW!>pjhqvr9wZ8Kt^zN(#75^i1I#dEJ`{j&%n9Gub<Kovcmzi#7$l1K;c>3no
z{{xjys$XMS{D(OqYn^9=N@&#U+e&xZ((VSlH2W#IJ?HmEO(UV*D|g0n?FnVxyEXUf
zn%-rluZtGu_AWVnDX+TqknVFa!>H1`2|KnOGqITQ!slh)p;zVqpRByb-tcO5QD(Qr
zZYxE_hu{0ac~EV^#wzdZUtNKb7j$?e*DCGq+Bnl}PVmWoku_m8uL4@KS+gg5HN+oY
ztu(jmNNe2n@YlUj%$FNph=iVc6wI@Bk@I@hU#`}VpDPKq^(`~qJM+NRE}!JPzMoV|
zHa}Qv`~T9Lbu;uX?o-*dyFfR!^mTvysdafQ7xq8)c)jb?t~8@bb31C8R$VvTlg{cE
zBP^S<_T%ij!xk<bIv;lLF4yp%`mXwwo{z^5W^fT5uO8uWF=fW(#m+fijV5>47%V48
z#cjPgscEa>jOoY3%u^&DMep#teq{RRN!c4;?OInJe0^tV%^%a1yw{z=r)LVDUG#VE
ztB0i<#0u9`OnZ6eO<$?VmR;KSWw(1tuM7^9bM~3iaqQ<=_dMQ2-`jx_(cHV$)=H}x
zFI%bkY1ay_JEwkSNxpq%C9rjoagW3r!@yHY+2N-*n*DEH`*rJ-t=5un-_^}N)p}%e
z@J<7G;L2_Aj%K(f;rw&viz!T*Pr6O##~#-Z*qvqe=K1yi?;T$Mz8A49Z|bXEs=8cP
zRkVNQ&ye^$Tj;!XBo9Lpui>j`@7u3e#V^TmebZN(JxA*GYYx4o7d@XWN<E&ui$!~y
z)cU_)4_@%v7xMH~T+8grRp#5h+`Yrh_tf3nvX^P1{YBTXU$+`WJ#XtqOPh5@ig<D*
z{J(hl?%n+00I|Fl3s8TxZURRbsPU%rb)s&r!LtG@+r@(BH$s@NKdqd8E#~@Wsbbxk
zcU-1qop@4W#y!PU^s`g+om2l=9?eWPs@_t~Y@XeH_N~zEwHg<<+P;z6mUmTb{}-nV
zO1oKI7NuUFpZ)o{Zjh+$o4d!3RXA+#`1yLr-BJPPGbKB&=AS58Bb{U{{^C|-ypyKe
zP2P>u4Wb@j<V;#RXT!A2+;0pDvs{C&{l5RpET4OpmZ^lA${#sU5B`w6{4}!(D`zIH
z>xh!wU*mVx!79zrn#r|9%)BS;dezmlVUCZYwY@cc&Hr4F6{uD?Z@N&l$jr*uT1IWx
z4lBmM?ORtrYs>nY_wLHchq<Rk9(renm0hmh^6JdGcUFgYwQrTlzEd}6oloiM$okc{
zCnm1?k$2tNRp)Kkw<Ejnttrc$+t>J#TX^#(f#$pBe7af<$I2a#hl-s~)>(0If>YSZ
zsi7KICvFlF5~|||*AV@jE^8lY?AqSG`Q?|J_J99MGepn0=>O(wPU2h3i?i~!a_`9O
z+V%7Pzf~+wr`PS963%!%t?X81`k52lU*C1iUiWt8YQNH>ihB+xFXG#>jcx8L>4xZY
z<xSh?KYvwgZYlDdy-@w)%FBC?zY4FPx%zwk-KlR*70umtJX=pRb=8k+v!}=GKJw^h
z-pZtjnXZ!SXV0k8_MEz^(PjC*zaf*Ge&qSxDBZAXpP}eR*}IFk22GZFtdV+g&1=6m
z)u8bR5l}1Q&&7k64y0|04h+%=)Mi+3|9^$;s$|{xn`Rn&qQ!FBFR@s+znZZx|Htv<
z&Iul(p6*9hn{2l&E%2U_<=MPz+cf6umeGdn+&dgM@7l@vJ}-abyrpkl-9#g=oVWjX
zbbkH+_U(F;x6iLHDe?TBFI#YS-IY=k=kPCG=`Vt|L?1o4?y=l<V@)fru;jcc-^`9)
zmT2XVxvaBFbYJ!9$7?fFk48JLG4GAkFi5z+?{~o#P~N+@@W%HkQ>RY&U=1DwsyDfz
z(y{5qjzxc1yIiu^IgZWvdW>P|M%It>ZtQPM=;bwJo~$AMx9)ZttHZ}0ky$S^-ml*J
zXm{*l(S_a%7sdX)wOy{z_|N2TC9$9H3PnE&+U_LJkm6;ol^r+dsIANFJ%8`ieOLeP
zw{OSM_PT!`_2vG4N&oJ*Z^2;@r|v_+Y`%^*-Kyj7b|3kbT=V_K`v0ZhRoOgUT&}sk
z)7$l<<g3Q5RoUB>bh}JkANg!Qn>)Kp(Y4jkN4?6tZKB)qKM!?W=Ks6Q9BcC@m|@DF
zeJ>4t&V6rbxi1UqV*OA)a5nSo48QrV$3yEY(kn}^8Ly8&XVu*0eMWlmasSK{vOiZw
zyIn2(8ROo*`C5vsu%oBu%W1pTHM^OYx;>JQza@0Tv?`u`NB*DC?OwftvBFi?W9wcU
z2(9Y~xhR)!&z`XV_vGpS-|jD)x=JI-b<@?_#jj6qRSeqyP;Ad`o;xMYM=yjdT7D#W
z{o7!la@lv6#jk7fM9%Rz!SLW|rF7J{cNboiR@#=PzjwI2c28hm&N|*)6YXEs{{M}A
z=k!?efs<9<h2Q%BpYH#&@A`xZ6Fy{b+_FqP<#CMgnM1p#n$EBLcv^h^Jnx;8idYls
ztZc3AG-DZL8bUes4zMnnqWtV&NMz{^(d5qoy*uQzb{<v9ef?mS(6U)!i4J?R6b`mV
zOuuHLv_6n$-OZ~V8?8bN%NCYg-?M5{+w;8GT@P*-w3NQwG2!6On}>EbxIWeM^O<{k
zs%dZWN8_sBm1@TrR!y#dGNJoap<?*8dd=ORzI{$Qdi|V;wp{r<`=1kM$cbt1n;k#b
z{^v#g|G(DD-MOuty1M4y$>!?n-9M+!kV{LNqj|QK#UXn^eaWO_mQe?$+c8Ygyek|Y
z<FjMJ7k-Hg&GU9B<*wvPKAS6gOpoL2MQs-G8B!rvwSx<zcQ$$JZ8)03YuPl{`HSs2
zF*mL!K0<$AT{mhM4|@LLTqE}y^X;bVUt3+D+&Dk9>F4AJtP?jp*eD`Y@TIK#)YW%q
zFMU1M|87-Ixh7lq)78%ot#3KE=QSrk#{s5C+rRAf_3!`<G9Ho_OH#0&nWo45*2LuF
zGHag?x6be}TxVHfD;WIwSb*;Yxy6^8#pYhw=unn^=FypHn<KO9UWs0aia+1}(@0U>
z>Slhb{2ivf6INB+)or-E_xN3=`)j@A7rx%z<uqls%%=0RUmWZbb!Pas+dgpruhZ*l
zO1-~SiMyQGb?wLfJt~LRzG>^t&oMsC?r_}E^|(w=<T8=cl4G3L4EM^;?qR$0#zsN4
zC#!X@(^~KB*`ibPR_ZtQ1x6j8(tCHKoyCu7RX=`2f9Eqgy6mzs!|vVp_f>x|xymIh
zRCgQdv8pEB+z+j1_Uz@kk+x&gGP5%=vVyy#;^vy3KKlBza(EcWu?UYsTUCSf962Z6
zD+eW$Up;7xGMBTgIam5PFznf@D|2<?V(0v@nm+3``}g^Inv>QTtlqG8gT36Ho7b6D
ze{X%l)DRtUym-5oY(u33FVEJzIk$aI7sg)xb?EMeb`SO_>&cxT+t2rHlK=PPGQa)3
zv+LiK*KewgoTT*ibASEB|3CD9t+)H};pWA~$M66Cd3=9GLH_&a^8b^}_b(D(*Ps5}
zzwS!?*Txxg>*i*j|7NcLGPGZ=o}Xc_%Kwk?HNPGn@1MTD_^aL9p9c@WO*(39cyFK1
z{k;#CSNH#3Z+GIz3_b<jUWfM{pCZcb8G;T~J$e;pbS>pU?T$5vB8pl3A5AWG^mrB{
zFpFPh^KtHj4-cg9nYT-9%zm4dcvq`jd0nyE>jyg+#23YTE8Z|VzH3Xv3sEMA4~JS?
zxx4qsF*I?ryw54-Jm1~8PLci4TJP|%{6BXpV`g>#`ge}`x{g?P<cgmGr&Z!rRsMuQ
zYG<(?L+7`Vk%GH-@nqcboE{r{RxJIv`Uit!7VUz};u^9`G#CS!Gq_t*_lH$iO(@=b
zN>+T)nlm5YTs_3Avbm|Z{M!?UMG|k$Gc&l~n_KN{8#0M|qt?X=hKxD7b5}IKyk2x~
zhE&m!0OoT$t!9+IOcRY^Xz(uH{X(qp-xK$vUFlx-HGfjlpUT_SZCd({f0_E1E8kLo
z{_x*tK4V@=(w_(S?!CPC`1tAT@^;ot4mmFm?zLUTxo+;y7cY$7-8s9yOH}=mTk1jc
zeE<E%BK+^(rY8C5^v;O+f0aGy{ofrMF7G`)G3RT1twWL2#g%#!mG-$tS9Q*oNt?>H
z;pdV)>#rSXNOXL1(!}d?Blm<ev)p)83>AMAaPp{Z-s_|*^XV2_`_c<h3)lF_-nskm
zuhouYiwn!9Zf~(@U*C94Ie5|gy~Rpt<_rRB#Cm4Cd8F+Wi0RlP$51vixKJ#H!N>af
zw5yu0;$GhC<KjKgY#x3i=>_LZu2xRY{i@);9Xspn%F@7{x^22_$$aP2pRJk`WLLd8
zW=~yRNzETOuI*Z}ycw;R7QW9p$0DATn){elTtj8`L~&-`9hED34A1Q><vf^m^2oc%
ze|6QgU+-{8^>1Pp*I=t!k@EQ9(b6&oO^u)h@!kxkDorb!qo?axn6Hisa&=ppu~cco
zvBRx7bCk`T+e6j=@7R)Wp~4xmBu)3N^drx|;qnu1P5Ju)lzNYT`S4{?-REypw)M-^
zzdyKl@7vR#&fX3VI@IA*`|IKHjV>O`ZR<ZR{`6D*CV#!py)`~Pov{~{cDrP~<@vbZ
zV~NMsMW*Rqm$^fa?VJ#uXfL%PqyDjl;aaQddK?UsBE?&{or<kjAKB&<=e*$-tK!TD
zOw65;sxc1l{A(CqhOWLdk>_Sb;I3(DR}XX;Zb~$b%Xxot-*;1iJ#R&oDlTff#6@3T
zamx2|)dg*VJqyxLZez7Dcq!wvE#j;gclFILOW&*)@0NW2RZQ4+qS7WIp*n33xiu{<
z_8b#lzA}nfd%JS4h>{g}K>zY*%cTYq$r=-EmuaNBa*8nIwZ?8e`Nlk0+dU@TH8ZC$
z`OF737lmt*(MM!X`@WX{dZe_NcadeM{m#R0yULEWux@f@{PE(2&;fqAy!w)opD&i%
zSO2{KPrE@+wX44D`|;K}?`)TSJ5a$e_vhlp-d4|dv#p7K+j1kt%EdnHOGv42?0&}g
z#}1ZH*4gk<>5ZHH5yq>A-xa^5X{}wm^IXC8sQw*wmH%pvNX$6JwPwv$vm?t-_=)FV
zez<dCb)&D0_R}fHJ})*-XwOS6FH1D<+_HYlwUt{96d$g3SBl`|++QA%+;mN9gSWKC
znvNTwL9AO>zii(Bzy0~=@SIl+cV_(N?%8o|RcZ)B-jfbC>6u|Z=?ow8emKe4$kZQd
zoHp}l{5y$0(d&1YdG?wMok%k&y3Qf*ylO*dIKzdn+b$>GUB0WYv?yTV;h7!DiH8q&
z=$4*+qyPVQz3Mg5;PsEBON%!zdj8j&_l?VyeOGSWv;7zLC8KA5*PB;&U%UP0SKDPh
zaRcwxO|vB~`Ob2i&gg#3=Zp5T>4DjYI;OU7USu8e-0W;pl(xoS-LD@ETayJf_AzR{
zwN-q$8r(+Q@2|7=rM7H3chG-@){bv%yI&tIsj{p8_=)-RqNO@2F<X<HowXwwJKkia
z>b_MBy82JT`R)vUjysEjuS<qEY&3kasaK7CUGqwY6tC@n3w@F=+jq=vd~^4_M@_f=
z&dYl&PEV=REz5fv7+9n6v||#F+vdC%29KQMPp!MKFmhseDeH#3E7eb;e{A~ZP<6rg
z#v;}`m$v6^+$dov)vEm{`tF49<*D{n3m@(Fitm?rb!nx?hNMsbIt+JR|Cq+O@t}a0
zp@HasmVGJ?tC!zuynH3RqC5Gnt%SOuw&KIp-~p5U<`a7NGJ6ZYO%wdmu(s#to3%e5
z%>S2fR_fBT<ywf0$*~Eq1XeeGsY><<*|6o=>P-q)SDoE>#O*>;*G`5PJQDA?!soth
zyv}v)>y86Vugi@ta88Vk)sALhnC0Po{6F{VqWyY{Wan1NGfdi15d60*g<CpIq`Yiq
z*!SL~y9UcVj$FRftiLqI@XbH5#fxNExK_HSH$7bI@?$Ua+T;H2Jo@M7ueY}C3ry-M
zPdO6tFGBHL#k-Up$6wyPD>(W3)JNwY<Z6lZNvQpQ>A%19eZs!h%Zm@+J-I^o>%0jQ
zK3r}1|G6F1do9~=_SQ>thH2Myau+lSHcaK6zt>bhuI|Dn?PZ;@iV{iT%IhBMuh@7{
z;&YMN=54Z$#z{OH0y<`IxDK9iX1M%Wz>O#Bb;kCr;(Kf7wwpC-Zd$RDaY31BmFwws
z9KprSv8i6i>t`j0=idl5$j)A7Ed2bh;0h^@1HOEnDtmXUPIJy%t9P_>Py2`G_y0e1
zcc1=kw|%FXzR(s1n_n-_FA80~d-6H4``Vkdtsb7Y|Hry<W6WOv^uE`Majv)bMP2?l
z(O~VGg})SH4_FvP7aRB_@2Or^62JC$mM{M`)v#w1RJ50tuFqJkx9()vhN$eNUz5}&
zHglToUg@~bdyV(*ZSB|E&faCe;MG%TAjq`$&Kc%67sc~_`M*9oMW*iZOXFmoT2Wyk
zp?yM2^bWPO+*hw^Vob{P6+R=H^4KrTETpdL$0PZ^y08C@j~`DE@|l!n`lx8dk|{zC
zvNEbhI&0>(zv^{-x=hcc>FVUae~)z^yIIK*m96J;U-`aE`|}v)T;~6)N_vm)F+6vA
zPG{BQn&l>IN;`9}%6SSt{4#|*Va73m&%1vct>3uztHX}QBWn&Fdidza4OShQm;>vM
z9ZD=;!+yx}#Ic<-^p~EKa1FKn_453$^>#bLa{|BLZ?3Ms67r+srxlyt5uWd#)$M<s
z3z+lHR&Rd&r)O_98;+m8K4sq2|3CEQ^yBM)Jv{!sOV`kL%l*G6K~3_1e}3C9@jG07
zR4g^mcE@({`{6$P_J2P-zT7?QT*2vop=*B3-oAXxJc(1R3)GohcO8#&%Go6SDn?&M
zjBnD>A72k{bBg2lQ8sB$C>1>x;&vx&h1Z^WJAF<qY?`tup>C#f#ThMw_tPwYZ9T*F
zB|DNmoPBz+@ABL^Pn}HSbc&C+a>Oy1RXmj8@_cf;d8N!fOX)oim`V%Id27xo-poDU
zyw+0UNwcG!?R@6x3*~Y)Nec-H{oCuI3W^;4H<#MZPH^N-NIi4^-`)H2a&_12<a@5&
zl2~lIJkQZZxo_(=w%b088uJ(z%nH%@87Nsed1t`7Q%bACu7}O@ZCrU<{YI%<{A>9a
z$M(IBOV?0b`#G$tdd{Oidb@l(Hky6UtB>FQUO(*VqPKGep04<DE06t=?je!y<>?~#
zit_GqwJu+twDD6|^<&+DJAZmr%6GY#uTL@Aa#zy$vc@i{>%v<#&G%oPxIlz?g67su
z)AoeT47zLWQGZT)vz~5&?!1Y`-7+6%Z~4A0b7QVc>aUaE|9|n{bW(lw=k4yF-~Z3I
zulV&Kb5{6_2_L2^Xo6}Id7a!BoZGfk3inN)p!{&1->REWzkKLEZ&z7;LQH$@0~v<r
zd#*&S^K#@2?cLm<y?T|_+FU(Gjd@?WOmE8PZ1sG1wZrg|rP)QVj6E~j%|6fl(jVGc
z=^(3>vy3C#?uBTRx9y`%Z7eZoR`cxBjrC<KO7|~Xc~GUHjJx4v;}qtHpQLVgEt*$Q
zSfqS@Z(-jvhDjfacYTlW;XnGtVAG=be_!M8YHzcvDcka^-mdbS{XadSxzYzi?Ee_H
zcF&8u^0WVc(<fI=neHb`*}s2daGA+<VD|RqyMKn-%xKZObW$U_J7}w%e*F5m3o|OL
zeq4O~KCtEG;^X%ho7SXjY&fa;_1kBSjIUQ`b#t0CIXu1-U?hIzkmC2XpTzHcovLD*
zo$g<x$R;#>+tmZFVxF0%aUb9Fm`Qov>jzEk#Sza6&OMC@4hVakv&`kp*{;j#wHd-c
zo3|hAPVL*5@2aDy!kSU2IHy=yZzscyE&2U-6DsrT-=Dkp;?0wilL0xIyCT7JT;CNy
zv!3_)rP8IJGl)N^TDrU7+hu*e(=T6NKi~iB*(SC*&0!OQycii`rmPN1agDSJZGZLT
zf{vCC+unsz;z7@1%;U~DPc8NhU2;QWhoEIk@r(Sr8Q$$O+xV_tQCP35(3a&8xwzIp
zszce#LDl}`hP4IxE4F`6**`60u8Tp!rn`$oH^v+|+PQPfI_cSR&9?uapa1{d!sq|H
z$NutnZol6Dt0C#;$}ab?<(Ipq4qR1V&-mt9_;PX6sn7Qri%b#xb$b6l?OXnnL<NnM
zf<i(=B=hHVn{=-TEf*>I`DgNb9#+wR8QzN*WEz4S4py$B&Zk=0FUacO4YD=t+tUBY
zO*49HVP4?TqJ=UZEd3I59r!b5@y>{uo;Nf8DA!}7{fiEKe{eMD#KmPX9!E6NCvls|
zB`o?V(x}FDQlsA_z3160i*hE0f|!muhO?ObtW?FDrYW{OcUt3h-J(47DtGnI+@Ern
zBO~-I<UKThcj@L%wFq-Mv|n<i%Zv#hejh9fR?BX(tWVT<)3wN@aa)aWu;>L%?x3B=
z%~rp=k+9b5m1K|SREAq8*p@w8=zIRv7tXlvyN@nenit_<vNQW$*^kScgD<-Y3G7nl
z*uCIM>pPL>(V2@{BzUFxf0Vv>W^`d*eE8bMX2<sblmGvhoqv0NZ%top%MG7qH}ypA
zb)WB^xSS`%lJ_k6=z@#)Kd7@tm+0r{JU&*?Ue0`M#W5-7jVl^fe_^cUO`0k&`^@Io
zzw2W4LQEv3WS>{B;n3c@^Ce?c_EI&G`womtW*4jp0gc3)D}l3ew$9pxi53h!hT-bZ
zHeWt$KHtvT=Fb_+cHbMKocs)KJy#jT4UUDV`k0n346G@S-_AFCPUo?ml`Gt0nDv6+
zo{=$_sClu1C-i28W8F8a9V;2n%1t}-^~%4FpO!7FeNz2z3F?{pG&dgE%Xs_V<)UE!
zIa13jqo2PLoe|1#%dv&Q<oLEMbzAn`KYkwHuOR&ErR&_kAC9ZA3Z7P+$;DuM``Y8W
zUz?AgzP?=iD`=!K|K7Z^?frGXjDvQ*;MwxlNO#r8iGFUOC;l!1b;}<=eZ6k~tHbH;
zTU3|6ire`0;ziZHnKF*P#+4_;|3BQkSl;H&GF{)t3$IV=ayup2>0h{r@9)j<Y^h+6
zEh`J(U5Z;B8S|>jdG1b!xW!z|QnJ4K?>n{IS`$h?B+c9{c>Q*@(G-)@KM%3<Y*h=o
zYH_^v0n-=O7uPpU$eF5quHEcvYTWgh4yhH5b2dtunB3AWj+whF_1%G#`@4gUC-1rb
zW$8YveHk{-w@%qrc=GeS2_N!Zf;I>V)qUTP&z~OkT5ZPvpZ4|<adY?9)wLw_urgfg
zN@~>a^L%;A&r3d_%6jL#lLz<|j&*H|Z5OHETIjRiIv{m%U~GEK!?bhOp7AES&%Jfu
z#{TS5a$6twM^{x%po7g-*y^Nf>}9LGU)Pe4RhdUlnY4OeNa&*G-Hab^^lbfk$w%kV
z?nP3v&9hG~eQ(#-ewS(PoGI!?Ys-(nykKZJb<5r>KEVsL8}sxIPx+^E*E_$dq-_3{
zm1&0VCS84R-Sq^F({>9jyZfY}q;yShM=Imh;OpC`maKVL;jp<R2%Kx!HQBWlAC_~S
z+THc$lw56LVTB8$t8$y*?tdq*bTQ5_PfLCKaOc8J4-U=<oY=8!8N&?rH904^6mEoa
za*3#xv0mG>dga3V`6WpPj0v$y-*yMCj*|YlcV@ji_kwTRx0=2d63G%hY<xZ~E%kQ&
zy}Ezb|3BXEKI>C+`)wDmj_Co5rvADb|L^DbdXKji44|Q0zwdI)9Seg?BBsZJ8Y)W5
zE5BTxU%LCxySX(VFJ4qN<*zF#k=(o^@96xp<Sj4eOzHWW*J`*)BkI`8dynO9?x=ke
zi}zUHwO%#r#oOoi>guv19v+;bx$pMQueQ4aaxaBSPxsGlUbCF9%#^8Y`nKg2Wy^ar
zw@&FXt-Z)lVIr6Bu_4!AO6P7BLxNUt*NMxgZiROGth}opr70UPRQ%Ar=PTD$?G4-7
z#F!6A7}tKhc{OL_9xG<mYDwlBPpu0#adLA0*9q!0P<&X<d0Ti*;SQdPpEr-^+tsGM
zt>j(fHC;4g&fIpbcJ=B{i(DQDhAq=t%Rjy8qpj`9DUX;!7?<#Ek`ePUl?&lr)4b3q
z;M~_ehEr`h@9k8m*%1?YUpTy}xAC=+yy>R3-ZJ_I6K1^0=W6QhR5Q4=WyQpR;@3?P
zk<MXxP21-ON6Ry4txHY%^KkQG^L+pM4+XzLbz^zl#z)Wn>qLJ4W3bt={rl<t|5k&7
z>%C>@>lJJJ&%E5u^Rj>XdQI)u*3N>zSyqO8e{e7EO^{94_eI`&S65v8`SL+V#d4u1
z%jN5QUN2mwTD$J}>FYbI=5uv#jPpDh{n6j*$CpLRZauQ_F*cmXvxmE>@(Y(m%=LeJ
z{?;g5^FFRReQu`NI<^(76BHgQ==+*Zsf+H6RMj%Sd$sw&#xt>xj|QC7{qeZ<M9b@H
ze?Og9T4!hKs62L>GIeF+oRi0H9_#k$)ZLJ(cWo)}J>B9!X6LiNx1M@9{pVMs15W}T
zek!?k-hJP`H~pV<>jN#-m%iiV@3{Z6UYnD%{t0+isGNECme9uzZ(jY_ykF7Vnt|o;
z(zu(#+05c6b}n<XXR+oN<>arfQhH+dHa{jiJucF^{lvp(0?wgQ!B0O}CMRUAc=G1R
zgy_{=$!Fi**)U^^NR9V~TP%$EzRG2$9y_8U#BW~R#WLN>!hE%YWXyr3E0d}fwtoHl
z=W+am!mbYPRQFW-KZQU4O#c71{=dD&lb81%Cmq$iz98Zw`}c1v)=xNb4OCuUk@jGp
zJhkiBzaw8h#BH3~S*YUNFITT$CSq6h$2Q;IzW&o<P*XfciZ`P9rDsZP;L2BOMgh#p
zTWSPPDi@13{D|yOPI6xU!ne~@Xzrrs-F=2t@voHb+?<?X^*wI|o6BYiwQQc>S+T8-
zPnS8HE)WRU`g8H^jj#zZ-?$hR)+^pnsJN|tWY-oED=U7{!ldVG_)k1EUixdR%LZ-5
zFx4Wp*AI#+7!sPgCvk5rOyy)yK9;7QSnXf*@VDG)8HQ`BTo1j~UnN{i-8%L9;i_$G
z<xS3csy2zA?3p^@!+NJr392f8^fpL7mzkAd_y0{~N_fh&U8b+MU)J5R(K&b7qm-_-
zr<yJ~KDJ3<NYHBQUgs52ykjNXA+yy>-`<tXUV6iZW!1Ytwr98Gf@(L+&0;S}7e6K?
zw)WwZ3k!^w_ix*na!q!%%=aT=hu&#CjrLnP<DsYZV)@sL7keAo*Jd_p%-w(culV`S
zUT!V9|L>UZ%xS;zR{qbW+v3(TD~&&1d-76mk>_i6{_Xdxf2CiaJA-xEvnr7ilQfXU
zw|`eya9^L>ywLa&+jX^_g2~s~{1ygWT=M+K+?Q?LO=_=LxE1P(*YJMvU7%!iAxz@s
zkAtkyZ&-dQY5rYn|GVCXQDnv}n~OXBm=_8Q{SyTboaf7WXER*4VEFN7bANxloqhBi
z4a=9mSCvHEoLsqrfu-NcXwu4O3m3^>ciJ?^@Yl!FlG!!VPwuW)E!>paHz%9Ln=|6v
zPIKeuhnGJ4dQ<1h&TD(_-Mk#Lz)gF?9qA2oHkxntGTOfO`_{b6Q=`g`ewAtRu{_9}
zq3rCr{?w5fO`>)+WnW%={J!E6dxMGME3qg3?^t#(XwaCI^Z8}Fe7gOgk9#NU8yPN}
z;+N!PV&?w$mHtZup)E;UCvr8-ulrG`^KD{^tLw41r#~IsyVoa^k29%4lHFm++SwLf
z#T6?qxvySYq{tZ(xUx&7=aB14DT&pS`|Eyf{`6CQxwz?CP*s29L^wlIcFLQknlC@9
zyH8nqESNiU^7Vp5MxF-85JvH-yE!C0p2VadI;(X@+Q4Aa!~`i1F=kVl)ON8+p{hF^
z3~$6gTX(A^$~5lDhY)=>nb5A+4_^GfygoE+N^kh-J6EMo#oe5o+PCmpuXyQ#i0IR;
z5!1`Kb<(ePK771q=ba?md$*IcM7e~8{yl}pgi3s+arMVzdwyNLA9GfWLE@b6fmcT#
z@-FSl4qCp#XvLJibt#utEZW!F`@~_v{nMqLn`iDcPD|g+@Zj8x!0ihz-I7X{lJPOQ
zaO<AwdlkjpTRe+XT=&1bvBEJ{@s;;fxk-yR`5nJ`wR?Z>rH~r4TL*SC2|e_#{{CX+
z3hBMK`6X0W|G2jL{)-cJ<#OkC9}NHaAWvG^D!Gy&K>3NT;Og7S9&`Ikjv6g3KYrb|
z`SqoFXPH_pavrWZCYvRpqN!acsbqF{vH3ebhNTh9rWm#zdzs>uIC(l-%z4MISz*-;
zTbDHUnjP62>Xys!;aq_1y04S}|9Db;`uG2bCu`42wk9v#cc`VMUe5ENm&cFX!)E8!
zY8C%GWK;jmJbW|j;hkHju0Hc>$=6uXNl#;*`zS?Z`+i^&*y}Wb!_j_1@9}_>$0m#0
zJy|o=P(yWfUi~(0hHkEzm&(7Z`ue^UU9pCF!`cA$@JRQ|QC$an3j(FKJMF*oBt3n`
zx;5_0o7}c7UvB^B{-X(xv)}Z->-{cv_3Js^(s~2ig$18atV@a$iQJULQ7tpod1tVR
zT)T+TtlrJFk?)>`f7@-pP|#c@^=o|X0maN`Tb@nenZB&oW5?D#KjI3t`1X2FPo9{_
zEgK#BXR?c_&=!sL-ah<`f6l$PC|YJucut__6u#T~TqiFCR~<<`_r$2Z`O2Pk#k_(V
ztaUTvzp}^|?mZ(LdEf2Kb)Wtj@z0djn+txJbk#@vki7l@n{6{>v|cu^ZmiRN%<@7o
zd9}<E@4!RWyf<!E+^gNEX|?^=wB>3p_t#GqSYsTb&Je->b&1~euO3<kYLok>PWYhj
z2p-F|jd0M(Wj%0fvj2XIoN3*uTeoam^ls0lRUL|{UV+>XH_i-8)S1c3B7Q=$dtUh@
zRjG7FfyG?+#DXRji0)Cnx@!Hk^6%>3+jjA-TkTQ$_s#89rm<6}E<7Y4&T#qlrQEEb
zE9y6n?Ql6<vFyvVUHdn+afB^1`M4@~t@h>}7IuBLH7r$oHa#nO|1qKJ)93#Bga4nf
zKXLp2^Z5Q9;W<ZtHr(W|zccO65sN<VB{u%2{@c`lQZ}?*vc<nj_p3@?{IoC2_~+Yw
z&EOPUtDAT7+1HAnv;6#o`I8r9Jc-!9&VPgVLO=a1Es;sL=k5|rE}X!(xX(#y|K!!*
z%RjH)YHa5rUXUJdCRbX(eDhoWv(nh-yVTc-++P}YkoS0z(*~(Ud{<3h3$17roSvX0
ztXJ_m=Jdv2ro5Fb;z^-NQ<IO(m@?}{#r@xArPpq5T2-`;F??#ovlZ)|@5Mfzpn1_E
zyg5zk)`M35yvW!!?NN8C4_v!vnX@sa!r$Y^bMU-aK7ZehsfO*BU*7EYKcC+19KP*M
z*P>M#F}n-T-AY)s@{y#f(HE|ZvQ^H9LLc9;Vc8VKnqS?z%8MmGx7gFS2^8XN&PDfJ
zbi0gn8O3d?S|xbCba+45r6Xa}<X)h@Oi-j-bahUwaqyah6Jxe)uK$1V@9*8si{9Uz
zJyWst)w{pHnIEzVakDc_dB(8d|EuP$#wL&7FJ9~&H0hK4|D$^+>v!0!jQ^{;ah*rT
zsps<(4oAt^{*&{K>HEl=R5hd1@a7(e9xpEjy#^NDDD~P2Ta8V!4BSoz&z*Qq>^CFh
z5%#<GyJeehTv&5s(Tx8;^jAn5?(5=P;5zAxtK6o)i%zg)d0g5Um8Jci{m@-$p3G+t
z)-ZiuCy`O))#!P^RmkUi?aXG0sl{t;UN)besbV`RtueTyns2|*qW6DoS{ANqoMsc)
z{17xVd3Qqju1Pmo_OZXOYK@rwVZ!Z2_g+8rx-+qMzgW~Ml_XV_KgQtJ)pnNMTOJxc
zI3@S*)#T?-zmx{Yx|D6*G_CGx)cV5A+~S>E_AX3mDqL#v_rcMH=8=w@1EMx^6^NYq
z`|ZY>RhyUIJ~!oB&4H!soVV&FuF4c;yUi67#MQtpXS<Zq{E^VXbBY?W3SMDT^0uD2
z9wK&RnGuJ=w4WVOZ~MRg`};e_gT=VEux3vI!@GB<4?1MCK3AQwBr0*u<%=;-{;lR;
zam+ogV$vqNnlhJJ0uv=pmG3CkT+BSTX!)t<^IcwQ%ROc)davQl)bOleVXVWK$HyBj
zZ@9z>W>%M5oL^gMx24yv{MTf@cl&=Sguf5Ixluc{HeI75Sg3dLp3cwW|6gcrTlzI%
zVf^ViCuO#!&E9=2A=P*8X`dZ}*GmgZ`gbt=J;|Cj^+dM7;;3iWk960(i)l!!`@T1S
zPsCG(TTY4h7q3}W6D+)h{dCL)bFF8|7dJbf{VnI@G3%m<X`Vvi!_UpTSAAqU5b(Z!
z>;J7PDm8z>i^lSqrwK`leR~o<-zGk8ZtPhxvpI()lP`WhF{NBDXU@xOX4*$LFHB*m
zYCW}6PU~icqv-r&&E1<yy1XiGdx->HJ8*KLV)vK1hPT6-xYu~QyE>d@|2^wzO!zgH
zd`3slNg<MR+OsC_P03VR$UptgzNq9I!V3#)xYM(Fp2i(Kx<q|lL*uiubpQL(uio08
zTpAY<tk<{RZ|<dE=l=c{?yb|!cHsJA`zj$rNinv2e*GWrRbS@DUAZsGwQ8<wc<}dC
zdn>D>?EWwD&f6e3VZ~<sB893uwYuV$SUuWu*A~BT$Zm3-w|3pjE;Y_e9nLu?#8yUb
z-{iJ?!PZhs$v_i^+tXE_mLA;_%COOLqMF1d!z~8qo&+o`IbqG)JnQzt6upi;+s<t@
z(|LY?_0@wFZPKTYn?C9IIp?+1q~~kdSARUgH6dl{;mPm0e0V}*!*Z3gcrs?RXC1j=
z#4g^%)f7@FXLzk*>dI}p8?`pBeK2S9wo~Qb-|sVLi{YJEIh_kM>B|UOL{sy0Lqpb_
z&EHb2Y_*rKU!TUE9bML%(B{Ro=>6WU$3mtz6&^hCc1cyMMB!Y~r>jJ^EM;gtw$mzP
z*{rNM@#b1Ch1ie>UOsHXTubkq2=p_$s2wmh;#qFw1a5}65zbnEzGqAS@6Wt)s%qK2
z$4*8%QpYuUj1$6(Im~WctJ<`D>6D`m*AGg*ewE-BlmGty{5PJO5BB8Dk=Byvma?$4
zQA=~aqV@k{{Dkr@mgO@JeR%DVby=7D%HyZ6{~WIu{Cu<iL&5A5T&;Et3hL8R9ys}Z
zd#}HylcoF3Pebc@|CYp5c!e%lb~NL|p|d|O=skRGwS7aTu}YDfq}WAA`N=O%E)QL^
z?)fCsbFos3*{#wU9HK(v4x8xSRF#h_h~PR~?4EkZ>BrIwHu<XB^LEdAX*EOebBJl&
zvp$c|Wh_r82C-#6&UqYqE`?A2v)us}QLfjoGao&=)qm9Gan3Rp@g(kjT7ote@7DG8
z9q*UpWjp-q2+LG(m~&=|B|04n=ia>~F_ZV_y1sb3{j+q}{Ny<(kt|@gb3wz_W#Jk}
zjT!bjT}-f8TN&2Cq-Y*}Mmjk+VtU!4@~L(UCa}KxJpaXAi-nR5zJ+eLZ|pzGvGn5^
zALR&x+}_CPCQBkS^Kv&c6{Mdr*{+?PaqPofL+%?_?`W+ppZ~%=**1yCM>?6oM(Xu)
z|M~CieqK{P%dDn+%$+B)@g=wM+?k(m8vkmzSNHGv{r`#U?k)e>Y{LG*`>^H4i8V=1
znhl_C^*v`}UVE!uOHMYwz54r||DL~lW`A`MGrXK|#DPDl%sKqEQjwdLnB~R{PZ7>S
z?M5ZJZ|^)WNLuqY?9wq_D!-#UX0N}Q{;vlNZT|ms;wJxi@ZnAh>n5hp7w&29()==Y
z!EYzl)L#~#1>82geQ=0L`Ix|&j_hpayMIeFpJ(`1M{N|mKDY3}g;l+xs&`B7&8Xx^
z@XGEzTjEruuar15X|>RoPjMH^yk+Li-v8JB|NhG_B^#F7vWWOj^7HVh5C<pr&<!60
zQ^LD%PTcz3rSL`ggiRgG^!ai(?~=Q@`)8|k?xd~8SO47#JXN;d!CyAx>{`RumyCSg
z=df{>AKI5Fwbe6lSD){q`H3vX{E4}}nGXA>FNwW)-cq$hc<ZgJ3|H12dpEgxjqf!5
z%(gFc)uUeK+geKS9^)^cv6JK1!yWrLW~l`UFYlCmDlZ!=G=IbWzno?~%ec9w|N8v?
zzriN$N7|qv=cW7Xt7Il`-xV!&phnx%yLiFex>{>4?Y_!mX$i&V|FqVzNPqgWRP)0Q
zPZs6qLuooDXN#8g`<!|6T>gJko5S&$*Gm3;I5~gqn#yl?)_5sw`5AF!BS%;IvP^Z8
z<A1NQ?dngs6_gX2TsA*rX~g=5%qzF#{Ng3J9;x?<hFkDEYA@T$7X13sw)oizw`+8k
z87<XN+jlo!EHbcZ<r)q5J*lO$4oiIA{{Qd)O{+D{CZC=-VZw)0a9z2&Y0>@JY7(0{
zE`R=cHOJA^k!5pMvh>$gErxAA;VLnjOEq5xX8R@=Gpxy3Fy(~BhZGj;+KV^Rb|t!M
z9Sij1JsEi8?w@%=-AuREnqIeD#GbrO&yl(KMf&U&5$glkg?*W3T>5pf!sgMe68+?E
z!JK7RIzB$E2$-dP%Ij|8v!f61%sRhJ|I9-j1E1Z~AD`d<|FFl}yrt!!!M7hTmfJHs
z+y<4~N)8jhPl&r3Iy-67@5S@~^|*bxaPp#})w0HCGtTkU#xZq9sHk1^%$vDAsV>yU
ze|q3UwXLt}ChTBfh%N0cSre^ZKRy1Q<mRcS`se2N?od;6G0&KM`(9W+V~<4euDx5=
zMu<wLX!~oe{d#Z)<0%t`wNqMF`)Iz>d(-sfQfcN|-p>MIs$beJXdk_{bnCS<($4Wt
zK|!}uSA?DT^XL8l^)J8Vr1HJ=0`2ydRpAeFIuzbE?}La&)t6@lm8E6dEE<g&dX5G>
ze3uZwoSJ%VsqCsk#nf{*0zMy849+_8#OB5(uD3n+e3zNEx}|xY@zXl6>q17{Y&Xxe
z^r<JZvuAs^wLWA1b<(8wM0Sq$0^x%WkKayn>f14=y{pNsS$r>349BcrHc31_0UZt3
ztb&`kzs#NMoZ0HcXLIak#iG-fTW?K&_twT_d0JW4ovSAoX0kA3*`Jy6u2cKeg+(S8
zGZGdxo=Tay=mx9Tu0_WjN`8G+l3XNtzQpI-VfkPYG09a);)*jjsxj|eTUy|^X<Fvh
zojrS7veQ>A@6Snm?qBC3yjaucA!r<L&P(0;&@G7%(^8n%w(G>~{&`h5IzpOloB6K!
z{YHlG+8k|mY`-3!?IpJK$LghTze;Qa%?OE<ZC!h!wC};)ZHpw&E48nm`E2K6X|64w
zRL;ll&tsN)CeZrf%;Sp20IORkS;JJfXo)Y)IJ}WX`TH*;@q@P?v<lRz{#hlGvg;G$
z!QBt$X4t;r^I_BrUiDF_{p>?O-KO~7t&1!#^m?~_Z)v&hvA9S`XrEBTs;Lsi9=Xlk
zHv{K9>N(5vtTH)+kMn1&(NYha8SJ|^H7@(Ww4iFvRyR%FWoCzqFKFNDJk5}CfvY#-
zB_FfegmZ$&-=AT2<GK4er^|8LlnvW<Z!qjX_SmAZHEyaX#|(xFi*@~J(Ob4Y+x+8V
zM%%|r-!eS!c3+yy)_b;MPQ=viN&YT6dtGk_n{U&a*Slm{M4s{mi&=uqysS!v>y~e^
z-m?7JbA~tN^+N8xmnC;?Jydb-$MfA&kG!+W=e<$+oqOZ=FE2~3-COf+$HI#Dd;Zq^
zcyhkX^xWAmC!gFdpZ8+3c$?nK=}lbq$`M+E#|~PTykF$wnr3m}8`sjUagnP_vyUxO
zQuxYwNzRDrn5XrZttQ7Vd{;TwvTq)btj>%@j#sMwd^vcz{r>;U^5R~F%P#+!)6x<R
z9+L=Xy1wIDRj6P5emnajRfcdY20owfbC-6Wi{aQJ8J?3GyR4?SCRMf}^<JCpvx3h8
zT$(4R`q(z_b@JggDnBxH22WMf$JBk-_L%if@X67hAmF;uGjQvvnJfY9!I9fVs*Wt3
zu!(P7Ry1GhrK%MwXHV|5-ovlPby7U_1eb^7DY5f2PtS}#vHNt^&rfF`K7HAv^gz?;
zB&*}CP@lT~1?zT6u<xpkbX!)mdg6pC9{=xh|39t&4{CGNq-!MQT)idfwRcx|&cra#
z2-BmUxg94R#Ddbig8wVLj1WJ5TwB%R$*q}5M?DjnK0MsKxMJJ3?t<cT7XK_iEnipW
zD%<*{p>JVnP}h;QG3%N;&$UEbgsA`bux<My605QC`@d6C>^oPpPc6+o<WxJ?(QEp4
zi8s$bSY9vg5PTz(btLtiz~;N!uUm57E4&PK@_KHtI>3N`S<F?x6Ooax_h_Ve&zK`V
z#guDDs@|`A-dx#VlU6(ttC=2q)_K87>xR=aPdxN)&kgLJYi(;c(dg`XPmdkF;4L~r
zJ3c3GHagxOUsGlp9k4({a>JHuK?~MsEl9OrB5V@J*vGw9i_wC?<Uq@&TaQbgTkR;#
zWHeoU;O+9>U+0#&ylBgku?f_c)~?;m#eB1G-^s-18OhTa3KWZP{8YRADd9{@!b~CM
zYkp2=xC$CuO=k)zAG1AXeI}k|Zm;^Cv+GwC{d&Y3xO(NmLkC%Z6ztrzZ~e6x9}TCu
z%U<j_`LMnI^C!?sN}*#@%63J#1k3&Xl73&3i%WLtOf!@232BdAx+eesp`XEIA}FF{
zDAL_<_QpY<m$_@(xmI7&ZH)=ZeU$N7M%DDfLHqwd#P9A1<O+ze7Jj^b*THwOajx}c
zq1$J?cbNFj<oWX7tr>H6bL=p`vaQJbcUzP6iXhMKg-%(I?<_5K)||d`a`W%j59!_4
z4BQW;sL8mV`(~@ru!=ADn%jp5L7(pg&Zs_fFl&xzlOg}}oYS{vYN$*O+I4E3%jqp!
zl43eqw>EIy<5HX}zFMX=g*%(^ibH4<hh)P+*4?L^&)u*IvAk|-$;}=uY&nr@<`hta
zq-3JcJ5J93emPyu+@EhoC#M~1E3R_V&N#dBdhKO{jO$C6&R<=bd_W{%QnN*Hu4X|&
z;az)=yK#G+7^fvh_3U>xD?6K+o_$17+l1j%`~vpu>=iG2vYPfTy6gPyTXW~Kdxi{a
zrK^IIFYOYIDw`4cn%(*g!;JQ4KR-YJ+`F*h*3+al&Ij}y4llgTHQmC#<06m9$_8J}
zx9t4eSDx&vV#xdrn$)_?y|Uw)?*F43m|vZEE$5~n{ri@b!CXz-q$>)XXKybzTACck
z-0B|i^kd2UkM8c%cUJAsUibbR=c8qZ_s+iV@UGfQL2u3N@EdiOJvXmP*!?uNvi!UF
zFjLw4o=BhR5z`GeKTqiLs=HFQVRf|juP?19xf)gqE}ql6&1lEAwB!lfuRnENrV-7U
zv0HMJQk(vYZMg;!la@`n6r!2e`*v<zc+uAFUmX)R3SUdDmiu*ysp9WX_xW?>rd``-
zJ4g3^<~<*eA8O!j`t|A}343R*6%Vfd;@X$4vHo%Q3g&D%r}Y6xHcq+pq*Kyxn&{J2
zIt&vO+j{i7=WbI<UTfIPr1RG;&q?ylGS)X%3miZDKDqjAxm50^=04txB~gsu63$Gy
z7WG>2`jwOZCq1>ZH!~?8>&<oRS^u@!iT@>UT>Za`pP!#UXmWAu3ufjE;n{5~pNTQt
z+^|8>WagzKy&74KyZ(bx_+j;WvB#nsdlV+kd-v_4fBiMzrHiibJH47Aqwj^i^5q-L
z1mFMKkXPVrqsy7>(w}#8RR{CYA8qT{yX7u>DM>58yZTl;=e@7-Tqzl!FZUY%eSZIc
zf}ez?*jBE4lG$f<?_RD<-qbs>FN61P-ff@d`aDg&A)M0vclQ{2y`E>a;P{aV9XY)-
zuHA{s?szIK?bb6<R$%Q}y#~#s4{vwpb%-+<xZXc`u1cqt^YN#iR@+~XK5ecy;Vz!Z
zvzL=oc8O4;)1m#`-r0K0dT}!Qt*lGMmu?kpHM;P}B7E+J^AFW7cs%VfjpGTuTXD5Q
zu>SlS-h;Qxju{6`WtfxlE=Nr2dE2C~T!yO^BzuoACNK)xFu40aEw#>L6)1f3KxWaV
zwqNJAiSD=~uNBBs61L#u4Cf~s-a3db_?R@c-RaZTx(mkZ_eSrl`uynVZ1Zy$Gi-{B
zGSZsk)26aHY+fp!w_(#gnfh~zjCbzNFF9@#?AjxA<0RA8+Q=YR{_4dE;vo)O?*H}t
zf0aFDOP7nf-LhYI+t2sO>BqZE3p40Fxm!E)Izxig6Vq)HZq=76CB2fR6eU%fQ?9W(
z=10x_lf3R;f8x84&C1_HuZM}cKMi<b9m;=Sc2cvi<V3D+6TXgLEIViTuzk8Dp{K?Z
zTw!<f`pkx6qiIsd6JIs2Dc-yG7H2a<O~Lo~_v5E391iY@JpZ~ecbTo>f@(qK2dy9S
zd5j*q#iT8bf3{+c@S)(d-xsZ_YUPNtc;1qbkTg~Kc@)=vp?!sY`x;HA7*vSwiFo#+
z`u!Yb@9J|iiX)!op568St&w<%M5tUl*SYqu4t(r8AAA)PUOCr5v2lXPnwFM&>lJF7
zwzSWmE2p>q+!|Z;*P+`cioMvdB6gz5()_lLOXn&WPO>dyWVpOScg|hO&2bBtu?0Tw
zkPI)8uyuXY#lqj!v;EkSl)$TAVIneIuNY+xPP)wPuxv`$pS>SnD7~0hY0H21^v#`y
zX-BqR`}N_@92u#6`}=YKGnak2*T{Fn);!nOApHGhzpm7%$j5J=+yDO|zO%|c{QZ6F
zpGx8HCr7Y8D0%<!)U0PmCI5N`zMoQEr~UO+*t%b9w>dJr7V>&M?-IN4U2OyYh^Z2W
zU#9fU68jqdXjzBr)<ZFS<!wU)TX#*Zy1G{GUe)&%`4@vjvRE9}PFZ$vuHj)dqmtCO
z9k2H?|K0!Z{eSoGD%#o0*zWRO*eve$I_%k%S;0~5K{<=c{rPxT#(ZX2^{OdQQt;Z0
zqka;FzRypZ-51DWV_CCRI#=gLC5Osfr7fC^m5)uaTd<_xtX4N-mBR749Az69GiY4x
ziYhtC7;{$axc>b0IXiefHb!uAwk!e765hTj{BO~r+u`R=y#%e1>M;y**yeY?DE00x
zlO?=gxI&pNlbzR!Uo+chdho(IskKvQ$Q*fnX_rFQ=c1QCD*nW+7kGUsZ*`fHzCzRX
zWr3Bss=^JEeHfgKE=W8OynaPJWA@8CKX(5%D*JS~`PlZYOG5cx1)QG3P*A?7*5r0l
z#XgPCpM`_N+_GcCo*mWimwx^$>u&gcrdt-5TX*yA+qGMtotbyX<in3|7=HiZzwf$*
zcslDPv*0QNTgC@(pF_5kILi1|R0X;}uMnA>_fhxLw#nK1tbQIZEwrB;lpX7Fv*Jq1
zlXQ(GbJ%@)PwU*i$v*wpB?*R^5|^ty4jwvYBtQ4PO2EXUc4j<_76%?Z<}7qIV{S#d
z#=|s&MT;YEO<%T)!-@Y%fG6McId{&kXPEGLuJg5>XI@X^*DXHIn$dLj)UJis>=v+X
zxpwCE=1A3@&n<s#eZq9q!F->rm8bDS_n7m}SHH{o@Jx+~*nTF^qjHbHLYtk-k1v_A
zuIR`$r?pEzH!X9qR&UB*ZWpW)eNN(S$+^Ia+cr;@?EG}+sn4f7mhx2#?RV_ft=M}s
z<duexhl5fCC+GjL8%xhhKR+dYo<IIi^UThrvqNvcm@VoX78hyGpE$F-G;`{uE4{4C
zR!B_ObWH5GOisz!e~Ou5)~eguM~-$b>^$vk8ZlkrnThERldI?E@x*zgX>N-)a(_HW
zbMC^wQ&Z1ah9rLY_t){<)@xB-o)dXF%H&o}-8xf9`5s$W(0PSBM;FDIoRf@Bb-lU%
z>ht_FM>`oEHqS3NW!rRC|87{pgvpKf!<c;h)89^<DUt3~tjJiP$bMP)*W_K?($728
zYfDNZ?f$n&h+dSEn(T9O(cN1zTVFC(6y*0y1Z{jY^ODS@Woqe%3RiM#N<Zh)T>0y!
zg}JR=oaV>7-{+m{>ldH8I8|WpVh;|jfS}q9-fuEOCtB{UtoC}{`KBw*^>_)VWT<p!
zq}e%LEt7M)pTDMDD0Vcc-*mL(+!I%Z_9fG6N=kk{xEJ?0X71@m7sbfXL+tKr=dRwv
z(lTXJ{OL_bM-%oRkILJ7=HmLL#eUkyp6=sUTV^K0Y|9^dF6FV~QzwIooZ@>nU*nk)
zH{}6SZ}wy9tOXw*EXvsv^-Qng`5X-uW&?i3=T$SUXL;>A<9g0iaD8fDDW_u1W7kN_
zV@+G556_i*-n?*GpVRD&utVnH<17BG-neC<dd=fIpRb?qzb7L%w`P-0`mq@wuUQ_A
zS-XaHg0b7XX{)w)o%Au(JHfLg?AkeJ*Ed<QNed!El`6P9;u>#0T%OJIo$t%ugCB1A
ztTwn27BT%=`{|Pl#P=|>OkA~RTZ2-Zsp$kBm-nlL7`~aFp7Myva`KVYuO5i0O7X9K
z8t~fblT_l7A2ue|XU;ocu8@d6D_vYF<tMg&NA1>Q2EkFE8J8Yz&YWjmHJ$zYR(_Qk
zxi_x1MY&#>pK@|hK=a){r|Q0|Pbs-7IsvpuxunGN;|rIyo8<rPsQJffD_tzPP5-oY
zvh2&)xvyKDHFz7=IGj@v7S>}conhGD&Tf@nbWh^s(|-}^L2-G@UI^?yb@lh|pI7%L
zX<wR~v2mAZ<yFzEC-zV6EZkB&ae?dnKIf|6l|9K1l9XmnSaUY!#%1P%N2gso8aA)f
z@aF-KFOy>QTJ@H!Y1Ur7O5(-BpubViJ_K#pw5rW3uW{<tkPU{0FRgYQ<DR#BcQYev
z!56M4K9^+gA6d`L;JSUk?YbGq?>g>1<Er>n>-ec#$9DQuW}Ue4W8<w5Goz@dA3m5i
zY(IW=n$tec<4V<GeDUq4EK_n0R$PAFKKG!)WT}GA1l5|!6Kz1bGV6<I^sR)Rh35|5
zOik_K+|8sBc8%%cLZLOLOA;6(rhd_3h^Ur*D6#whluJ{t#y#fS`ZzM1smyfy>q}*B
z0$K~+D&@$AAK0~}a98mgr6o&#2ws=I&HG2Z?4HEh8)}EPD3-AvxRI8+)=*9RR4M0e
zpXuA&wQpXX+f$K!%d#y>nw#O;mg-+wdpD__GgmqC^EiXX4#tLGSL4f!`F7b>UCj8}
z+<yDXsYT*tyb;SBUq=VAvkTW6T-@z{tM5>pVWq{o_QwZT<>aP+KGw72o^8ddt#SJ&
zD}VUhU+?J2yW{1@D)Wqg)0lOnwk`5rGy9FhGvh4Llo>(KnC>2*zIkTYn~M?eWnWZ8
zSxq`q9A&s^)7E<ypF%WUU7go{b$K6@|MEdouY~i4V@YyNa`vGn4`v#P%$U_~P@CBB
z`rZNS<h0o@Uza-TwyZ7+?$+UEFiD!TZ?Xu#Bln3ZLBAUEKAv2-adr-e6sN;~^~56$
z332xJR&_uB{Au-^$jiwI>Z1ISPGIy|e*WLf_xB^J&n@FS_-2lE)wvmLvpt&_LZpu_
z5m>X8&1dbJyD3~7e7P51ymQW3LAf%1t?2n{OShi&D!!3cYL*@QVOb|{CqvGYpk;wt
zRnHlJEnM1mq%yhZuE{s;uNgMW9%wThSr?}2)|1ZK!~2;bdDiBcj0NdyzVm&V{884x
zw|V|;p0$R|mqiwPz2xT!_BwoOUryYqm-ik^2)jB@d|O^G^!Lq;p89uuR~a4lJYT%n
zo9pr`H=ng@6odcYk~(Fop#HDpb!lat$!)2_S!<1#F6ORIJ$w4|^YqQH<7+3af3f!d
z+0GYkaYtubZh2Pz-2XCXat-s_p5N2AJ&)V6{g_Es?4zeSX8ApG_PsLN;qQgZXG+P)
zozr-8O5(Cn(Xt*x<zp|nKWBUj(L8HVyOS?LD>&;&LQmvO=SOE}2F;ylU3PDV&q~4T
z*V^J1zF6Iw&^hO*zIw(Hi{K1pi7C0uuC-0ws(fq{lUL?ZW2?;Njm(Aog6~$}e6;zp
zbMl!@bB|7$@PQY+2j{<)&f3UZAO3v({CN6#@rN@F->x{g#vy$7EtPJj*)yA`v~5i^
zyPncf_l=>nC}UfJS<R>PuJkFgtA6Ndde&~WlQ7OcypGM6zvq*vXBhkAuqi=N@9i#1
zH)<sa7q~j}zqh$~MOVUL;R$VqNXBI8UkcgP7N#ergw>RP@2oSa^qM)DL+qf8&z4J8
z!CVI?NL|rhV5wFWUi{`&_Cqe&w_1J-TkikmoG6qfy`*l}jrH#JK}=!OKL*UWrpvf;
z_FAv~uS*hp4PSq%VH3UXFz5Zv=jWHd_s}}>=h2J9(H%>;q+3^Nmt5r9{pM9zb%MyT
zZ3Shaa{^`luVd@<*Uon5=~wfA-*a0d<dtp8j9Y10y))Yl9=+u`IOCP1@G7IF^LpNA
z<gLwQUG=TuSVZ!Q@5RDytFA3oowhQn(QEt0MT!EY%ool{x~wie>-<7Az=rva4a=-m
zGc}h^xo3T0TV!DT!;Xzd`D(A~HVCdUJa((N`peN%K|0f_z8_Lksd1hFE_kd=&Pf*k
zIQseU;>rF2PF&kFrH`x<nfa+UfAyuCX{r*-BQCP0Dj7^OcAFf1K*aaxLeGMOyEjL&
zCiFf`d0=iiv+Ksyxf|tHIr0YF5M3Ex_(>t5I?F!2DJwCuahv;@o5wDnig`Teno)4p
zkqYZ~6LwZ+yqIz9u!OO-;c;&F-u#f<H`hO{DGFCtvv@AE<3Pa4OAlW?DG1{{d0;7<
zjPBy6HC~_ZTojr6Z8B5R``W5I^<NJzK7L>MWyhSmg=J^?<?=4u%#^peqZc;k*6T-4
z4pu~zB<0@v!mvP~PPOFz;&30fEF0^}%jew>hI3~#e7IvOXPmra!sN!7khiNctRA+X
z?>pi6^zHL6ZuS*B=5`v+d|m!|b<E!X&)&Y=oAmk`hf~*&P51wD@_ySHo)ah}(fB6v
z%G(oDBW~SWbjWDG)Cot$rPFvCrp!tczR4A?|8JVz^}hSEd|8(}_OSU%^qgJZTfBue
zhF7HdZ)GrRzix%WRAWp2s~y?10ySFBKRnPPaL@ALoz4lf?u3L_K43c7-D-RD-s9fz
z-pw<GdW)|dpJhBP*5H+G$+^9@>$Xif@$jYEvz~+3w?#fJ)or;`u?@8HWVQ3v?<)jn
zEtJXMEYdyogRdL7c~=xM%hF=5+}@vGW@>AnN@ZAJy5Gid=BHyOp)VOYjxlm5hC7*@
z6Iz?uEU}>I2Jh~my-sJe1eaSJ7b&{jG4*Ttu{GfwSpwZ}wA>O48TG!qZz%2%oW3MP
zP%>-JiLOnBd_JjOPhu|nD>QU35dY>I8ESd`bHnAEpUsWSGkqS%%zgdhY0){>=L}!C
z3RZ5Nwpu7Z?b)S}$&K$tlVZ2MWH_i?@nuR}wDIZ<o7A@Prg^#ZhgsWhFcvd7C-dgp
zyQ3O@Rt<9=xI{ks<EWR?6LNND#+=DPMTR#L*?cr!&gEWxr*LKX($ic9_xtMmw)9H~
z3EC}+JhCw(Cn;q4x%q!yeEeiu`B2{WpWNP-jQh7F4Q-c9FA)y7`@~?i$w!Zpv;=>)
zy$_Clo2SS2d$kPrYYk;*1D#LHtSrwQxv?<nS@v1>hS+Blg4TI?9F)>GdK$izL1xCO
zhnXEuY(h*OidBA_@_s8z30&VOBiAFveAKAivSwMx{oix4mE!DLWa{Lc_D`EBRQq1j
zs%4VvbHxHhmG1|4r|K0QGg>P3yk)|Z%JOrZimQdZHvXy7DN$t&`2KB+)T_TDHz$1P
zW#yi|lA9B>2zgKCzn|&n?SB8Rudu5>y39|z;iX_s^eM(QrrRPJ18jn2or9w9+!RSt
z+P&yjS8iC>`$?N;^z|!+`M<RAFW7RjB;rlWk9}+VXB&RHc}Cj#@UvSBi&$?hkI2l~
z*eOxFQHy)+*FA=p#oJ1Z7-sHD3l~YOvTbV1X8D~|=GGN;#yWT9y|gYr6U)8U3!cTg
z@Q2k1czS+sh%Q_#_%>^*oMDQW{ew4I>3a^JalKHOony?e&^^htJtejJ`;+C>{VV$_
zvl;KI-nClhedwOA!`{|?QQG^wzwZjySo^hO<Mhpv4epN(LyPKv9xP^vwpd-r&M=j4
z<%;Q=uI7cIO4_HU@_u~nvVHg8xQi(#6U^5YyCqM5RF{(`c<EzF#FlqkTi+bJAg&X(
z@$E6MtJ+sGKK~asyZ3Ed<KZah^V^bZW?YN1ulV)hcm1#Awfw)jmPREWG8X_ZloLOs
zmz(<b<n#T%+BZKIHs5`Pi($*U)2rsHL`(&(yD-+`Q}POvEi-Lj<-FB!6}R4z+mBbN
zHzkyQU=mm+ILY#4OXiVnhj!UKbP6!+3)+;WP{HzCqay>fT_K@~TW8H<rq(IlLL6b)
zGmY7nah+t;WQ~saT=ebO&JD}iKL<!Ar-@34^*)|?i9Olv`JD~gx3yVj$Q4@VpOJ2!
z`n8(9YSa3;({0L?*G+fo+p(s3rqq_id0xeen$sWN?Uv}SHC)Vi;IaMBIPLP%5`~f=
z2gZP%x^}K_yfmj7H*QPPXj;D{!LoQ_EaQ#KcXk$RvsmZ<#EUaZv@^6&ds#b!$MHo@
zPi{$e#dhR_W&l|ACT*@hDyEtHzV4FG%aaG*zsWqgHzE1Jtu#~fGmlqkRxWedKP~qJ
zQ|E$7XAi5k<j8#g_PMk$-e~ukqXF3r+}Z6>=I5q`N@U$#|I{XA%HnN8i$0#1l5gt0
zzNq54Mww_Rr=rc%b!VNgu3viXk|l#)dXDi&w?NsME^AiXxG1>_?l@<;iqq=(v`Xu<
zE7_XLx3nu2v`#4f{c>(`#ICBV!oQubUAO9t;JojkGP&C6(0+le)U)gDD*yfZb3(%T
zP1X_9uRKgIcHPqSR9s}LHd%zln|b!j|Ch^EwHX?&-no=~cBN+kqk_AoZ?+k4j<M?3
zj%yC;@wvZt_V9i_7W4Z~^W64lN-w5w{(SiG^VdQbZEZ~>B84|pr>`kLZWY`WqH*@^
zIyZmWo*i<AXQYesmN9(e`_hpu<?j6UuQFTqnUf0@t$)tls5Za)+Bd&fH^OhoItq)f
ze8(7ntI?ywD9B{0u@9q{5<}Bm^Jh=bDBS2?u()Zd=zG!0-3xjR6Lzg%&#au(#$@JO
zv$JacpEq07%lIyxdVFxpgLlgnWKa9@<*%LjE;;AtlIp7TGajdmj$K&t_M7yft#a}<
zGi#zh=Dc@!{_5{k(`}J!y^mz@3#qyH<Q^+BE?Rkcdv+?bSthrGj%qZ+6;6-b=Fi6@
zk~5C+J({u1cMWsEB~H(dE)(v6uJk4UkLJJW*!aV?zerp&oxyE`cJ`Z#;`-u=m;Ey8
z@Ba1m_z?x(ru1J)XYJ2FFOoOKc|K$au;tSTU%KJjj4z!h7l><XI=l|Z6J^q8NY7>P
z$lD02{dX9rH5HzedEm3y>~rXv%q!cnnw^`EugR+4Y`VRvx7mF?LvQY~<cxw-hvrPM
zn5cPCYMN;D)TFXM{+lN!J7})m%ye1&Sp{R;%ajX~eZEY&#&uczSx%bK((lKAJFEVd
zi)`Fvq~5-)aE{Z%mv47uESH>A`TAVM`!^@|Y+EL%myqh~A7k1Vyufm4-lYsFr!!^h
z3s+nHbm#LlF56XCc4@++2<e<Z(F{_@#izbD6bX9uvCi;9>xW})b6VXGZ{_c8KclxQ
zRqy(@;_#cN!Wvc-TT1?Ly*jV!oZBsllYxd_CT-p8=34Gu;rU|GPn+mh|F$0X>FNA)
zks*g6+TyX7Nndc!?|@Bb1C-6pb7rn*m7cItFL=>}oT<s7Nn6x+X6Ob_Ew!!=DzZ1v
znHm4}NCQuhX+gO`Oy{08T1MWDF<}R^)bk^M_lpE8&M0Quu-7R>b(&?-I_=kUHb?tp
zADtuqIY4l(R939VHjU3~93IQm=C737dnV9h+nLrU`~K!9K0f~HdJsq0)G!XsmO1SQ
zSc}503Cv#TAG+>KGrLvAMdAI^9Mn~6bis=$?faLSo{zV$|NPT_j_iB}2ALV>5{r*z
zJxNV%D%Ufa?8xt6cw@P|<s);G%euj<3cnSIfeztV>&@^kN9<RYZ`lvNtwob8m4XiL
ze&G6nLC5y#;iU<W;~1sF-CdS{)L^q}ueYprw(nIbz3@nS{j=Vgb7j7Bh4Pp*Y%)6N
zY!uyUxN)`Mo9FqfypzB2eQEx^>to9N>zBok=lAnA>}9sS^7db7=cR?4+BbV;^v$)6
z<SCK8bNlhC!aV1->#i1V{Qhs+uKQ~xggz>{xw5F{J+_#&)%bgtZq|$Lv(4K#thnRa
zkPz`y?|Ic0b?N@ox~1iJns;l&a_nOgKIiD6*^++GO*(AZ3gz{xx%}_mUbH^uQGP4v
zzrxvX3wL}goKbXQ-i}8}hg#p5@qfEo^>Ukf^Swof<{Cza+_1PNyP)mTdEV#{zoMJk
z7rxi29_cU)zd2{)G48eOGe4z<PTaooxXe_Msc#A&y8Fxv?cUwI>tk0(AnzQhV2?=V
zUhxE`g5ni|pTB%n?i00{k`$Ub-Lh1rBlVq>{+F~Z45r(IR&Fdb3tkhyJzwy2Ylr|#
z!tBsE{+5K>TjRwS6<+`MecyM}fT>Mv*+It1{nk0XUn(YePTO=>cwZoR8H#ygrtjl9
zhR?d6@3*n~$s)&aqBMlLTL`pf!ge-mQn=SkpP!q!w!b-0`+lwIq@BFGH*JeF?VJ3_
zbal(KZ#iOBkGu@zWZ$J6o)uc)n30#end`>g>ucEDVu~Bjo{-&n>h4NI6V(6@S$Fqz
zuetK_(?UZ%W)|D#pZWMEH17H`!4kiv-`lxTLv>};m`}3)D!twN<D~t+%ZrN2kG=W$
zeV&&lV?gk|?$7Ds$5ZFNZZ;GSx_)r)%$*im?w=j!o<8YtZReid`ghyvCp=+pU|D-(
zhx?JGw-Y?3sRhmG$T_;8UGH5_>0Ix#Z!bz8&XD4n%+R!6B23(oJznvKPiO1W44#KF
zVpD^2_JmEp{q(}ZuJpx93ZZPu$C(^hx{n10S1UL0&zF6<;lYIR?@zz|__wX;$-Z-6
zPyN~#b)I*(aoV!L$&KPY%uNxl>jfsAPOVdoINN$c?}kl?32#B-+TsgU6O4Ci9KI-R
ze6grHe}3ibvm!a0Sl6^fL~?h`U4H-Cn>TB?Rts7Fi46T}yK@c04yzE$<WSD%3M|Fz
zLN_otHuJ4~%wM4Wkny5LdM;?fg5QE6RkTRWQ@T6oTE|4WXE7NptEcZ&5PCo7wB+V%
zKQ8oLD_m!q9BBM_l}rAH&=+@gmsK^XHckC-7P9c1(^gUXdEDM#JLc@0H$P?pgUzc8
zQI%5Z{b#Step#_6clOnJ^HRLluIUz)eAOs#=r;9WoZ~W?^x0Q0eS6m<`Bt$sYv%gt
z+rpnm72nnl`4DC#xJUJT^?HFy+Z)$=c6~W`<joPC)@xB_^D5-3Sh-Fbsfwv`o!la)
zzvk!D*;?z@PWa97?dify8F{a=Of9`9+wPs0^lC4Cxh-^dcBpgQ+!>RPu0Hm0FSG5=
z|MCpxkF3%^m3;g2^z-9mAFY$WeMs0Z@n(K#UhKRb3Tn(dH(&45KDXrA&ePc}{3jnU
zt&97k>-*yLk&3rG46A+x{$PsrnKWh52A2ityQ^QQ%)B%yG2U45yL$58GqLBEh{*Z8
zF({kaUp}Kyuc7l;vzpR;8R-LjOW%1~o3A;&Zo|XkFJ-%bimv@|_V(q=-Lu++E`9s5
zXkU0vph@%g`PS;Hn~x^kxAaJBpTub<=f%0hJD6AXwn-P)r!$&Ir^s+AuD-O6d!ydN
zJ1501H_UF&O6<Lsyp~;9*Z$vf^QcX4_a1y{7#)(gNOftD<@0SvY^49I-`F0sVX8p*
z@^hBES=rQ*t}sn4EIu|zI0m%;ipBEU#+Q3<s3fe^{jn?f+_{I37o54FyW<S&SN{M9
z=H6@5CIxxSDemhIJN}8$)F3HmCbNU=olUzUQhyy}p6fZ&Wlc*yUyzTp)1mdvv9}l=
z{Cxd<ece~{_4DN~#u?SF_foiVI8q_e^JCAg_bcZLT(sM>F)O#{ZLWz%<a!2%Qzi>0
z3p`ykcl)Px1((nGHG3t!w%dA#x#REusO+^{4by%dVt5<v&nvO4>-{XYD^2dDG7EUE
z8FsFpZ5H?N_IvwW<p*2mnLoQ(eE2PB0mSzCSy4Qe%O2mlwK^@z-Y&Cf%jVY_>;=2N
z|G0Nj{NmMrvoFi+sxbfi@V5W)C)eYmZwGfNeRU5h-P#-WN9o1O?(;7gc6oUp{&h>I
zcUxjv`k7tbJ<q=9eR3-IIzRp0`K!NYUSb#K-IB>syu#5lKb>)@$Y!%6x|Oq^9qf@1
zv(7pqaaU(oGs6}+dwZMh&n8p_#-BN-@l`pvXW2~~+ZhZ7X>121J|~r#UED2V?0ug-
zDxzI6a9LZZSMV119_~sLIce!f8xH-*@#%H9&UtX_b6wuDFRl6e8?PNZ(PqY$Bz#9;
z|ABrr|Nrs--~adT%JKN&rl<)zXn{|uZQFVO<>x{>-kgd$5Sg9zdDAKRjJWSyOHIv}
zt@1kE$}qt=Y`e+oR~xjsIc~>%bCFo}agETV$dfV+T0NSFYooW;{x;0$T6AgpPHC;3
zT8X_6nMBTBs5rx1v}4Zeg_D%mtrV@_v_4!_>{-gpW^ThrR@$}k5AS_woM0sU?qu`w
z)iQshxHMa*IJYJ(2;cenVC<Y3dzt^r@B1nrzB792j?XUBr~Z8NbMw@%<$U*kziT#}
zxv?YlnGfIBzssx`g+Ct)xi*vSNpXCMh|;FSiMD$R^L}qSa;)>~(oDYXPLfJ$%=$AW
zyey{_Y1Gcmex|tON*C{!eIoN_an9im+K_5@ZszXG(Um!#5B+^z6MtzZPIJ>gr2hEh
zw@WWHQUkwath={drt;DXkDFcpH)s}Kny7MOqTy>T`TIAgFG=GS)o_jFcqubybAv#W
zZgGmZ563aqU-xv&t~_WeOq|O+<I*nO<zAXw<w8QgzWJH6Xz8TGlaH#_>^=I;NL4ER
z<&3E(b_!TtKmFk3hbtA1cQke<gjX(juQ+q#s!Oxn4D-7W*5yc_c{o=*CFlIjt%Y53
z6S%x*avzNT6!o;{Ab-Z6h8t6Ai;uNTxHe1Iz`&G0RI#>q+vH~Mz%VwQ>WU8vppJwL
z_`t8%3FrIIuDAQS{OMK6%G>T6W`3H!JW*~{vhiGLx5>F3v1?B);_Kr!yj5}grFvGZ
zlV+sxUOoSFX^AuAS>I&EvFq2&nDn@7(?v#A$)@cGyC1C0bcu_8W${{6qVTRocCPCr
z2R{GOr8kT{Qur7)uCDyN=%ZHpCcbrixf$0!9x|*C;`w%7?0ER{yqUXC?Mi+3Yir55
zvwmmR9Qw`wZhv{fd~S}P_vWQ7PfP1L`TAh<--Ffu^UdO>OtqXV>#f<{IPH*rjwyd=
zs_fdzRNm58fdW?oUM-v=B3$3?_x0+o!acit{cAOMGi_M0Va1(*Q*n(+8`dus4^obJ
zepIWxyy|hO;<oH(jj48Wac8b9_@8Afb@knmkaIg5lVVowOV>_pk<sgYmu)=dO;~*5
zmzK42cW#|D<DuX2;~VX>)2DswHOx`iBlkX7XzH~~mg2jAW^CZOAHaR@U*_|LCnRq_
zoOfx-4uNx_=Rz0%tqq?S5VdR4;l<`xU5vFvgmz0A&WlZ0P;t)T>p>@@=dR~e&d06^
zxb6^o$ZYzo>)Rp^#&alUJ}7@IGvmRcoIQ&cY>jJNwd9R<Kyvc3Dg1pELRSS^Kd5(m
zAKX#gG-=C=SPe06j$i+tzZTl-bb*^uC-!|-=iW{(yR+v#f4G59>CkWenpRet<aggL
z`b=L;s$0xC=ch3lbJluUoA*e4VtqXM@87AVOG38VJ7pxxbxL17u*+uS*({?MJ1<;#
zA0x{4CBc~A{j64iRWtu(tsc!Oi3)-?JFb)(JPA-s>{%rLC|N3sJ>f{H;?IvC*Q{A(
zwd0i4q}^JtbBwRnTnPSc>6|Lt@PM&v@s@WL4^=kb_IlI%e)`mx*NuL>n*RSwdAyDJ
z?I^pQ_WQoe|KD6NeYMR5r?PT`61_9dS1qEaGTh?eG?KHuA#cEcnd2nGjcEr<G_IeU
zcjooM0I!3u7G7z(+P!LN%$0ig=%dFLZ(MA@{Z566nDfi?X;Yu<doL-@AROhb`SsA8
zDSCGvHm{!haq&Z`2$ufOx?2zXylh&1E3t;5so3;L$Iht&%FovI$o0l9iHt3|DY`P!
zByqx))$bT5&N=I2DiFu_ZRwPT$w5~7EmNGfY}c6=ko`=C;n$%E`Iaf4w8};1axIv^
ztk|-(ZOLM};&VHFHacY`JM-x;-E->jp;<mFW$F@hrLCeGy2D;9%I_!+GpIA;wiSHV
zmaAzPV&t~VJdTk;X6LdOyF)zO&g3jexnKP}<DQ>1&qd+p2eJwtHy8a6-gxHd8lf}W
zLxMUswJq?A(0ib@x%A~bnVzLfep>2zMsmjIgR*e}w^)I<^cvfrU%C(fd|473=B#@9
z)S5L<51I41=W9u2UJ_39_unCKB}#sI>xK1EPa+Q9{&9QTOtT1YwH(ksrhQS|W_<g$
z?%J&&A;!af@R(kM%NoJyQ*9OJf9iPcQz>=!w%xZT?jDKT(*AR})4LDf^c3B*bmE!G
zwXtE(-fo=9qvrK`PU5{8Neia$RMs}RxnY}V*TH`QC+B$n+VlU*&*R01zwv!p{88dS
zNtsvU%ho5RhZgaeEP8&X^MQ<lit<sX30#-73-S-NT=8D`-E?-(_pM)Q^{=~H?=EOA
zQ!1UCefG7k`Ml@Tt?Ra{-woI_uOxl*f*U*3WBs`pl+}zi861^A7pcu{p7QD6KNg=K
z3C+eaPZ)M2Old1k;yF6~{f0}<AMSj$eJ#4_{Y{%4k)dl>ZLT!Eap<O$^R0cqw|30!
ziM>|pU9tGO&~d4^nO%N6_S>s0Q)N|d+Y-CE@lwNtOUFBJFf93UlXYFuc8y}q8(vRh
zejHnUYF@yFD<>Znwe!9RVVJX8Bc}J5*RE4$=gvC23iD)UCCbK>9lX}cagSGZwlteV
zk^fd3hJ=vNH7~rQr!IIeG(&1tRpQ|=>k9%0Z#$hbt8dA^<?OaPl~1eh`PQRq8mpC)
zBK25iJlk1pcl+XDU-yN-y#96jdi*%-0vfxHjc{Om_vc4}m2LIsm&VfPvfKA<E`2R|
zsQYGQY0-@C8k>7^t3C#No?053$>=4)WX4ziDB@uB<rT+v&f}OHd5&vC@!>~Pc3C|B
zc_}2u$7DU@^QIeNOg`b?E|_SozpZv~YOeY}ratbgJHjlZJNK+Pv-$beZB8G~v2I*{
z^Uac3p)+`vT$+^=6tjS@&q#K~Q67QGwP%;g8S<;0I^`2Daqt!U@5c+Tg)m==5-$o+
z)>)&u`c+9;O=(q9u13rKdykoyg7%&6JetC%zvQ0u{pms0&x_cywV%%^w#Y3gOcI+~
z{ajw^xtOTTs*?-c_hf0@y!7JYjju}<%Ri1|RGcv(io<`Y#v^u<->$D4r(F73b6WY$
z{^hS;DLq+NrOe749OR{UWt#ohfEbPgu{#Wsj3)b?csNa(|JYtv)}-(y;S=w6$sO6V
zdS6tr;mgj|^Ei(5uRLNY`K3i>5|5ej)P#Tk{v8QD_A}-+W95$}Q!IsJERwY}Z5Noy
zoMt*?$s-bdV?n9(f(sR;_4PZ>+<l{>Q~XBn_39XtIDvyq4#ov9W^b7E+azp{V)RsT
z4zu*NlG+uFTb%A1#UJL+_+vbSLD=+3$JWUUqrZ#Vh-Y!%yQR<~pS5>GLEhnw4_2y4
z>{+a}EhuGI5ra=DXS{e2m*aoVfT<V0W><z@T|KYjgF|}Je7mn{vnPBgb(Wesc|rmA
zx3>>-tZc38e#$hYIT(BvVDMUR<bC_})!&Z27r)gUeW=o9wN`1j(`{ppEyh7cNr8Kv
zX0lz`v#g@+Tt@osHAYKUT4c-?4b)@VT&dVlrgCnN;mjZvr-082GZt+-7|YxB%u2#I
zo4I%2OU<mboZip5pA#;wi^;T2^3lHV=fO+v%i0@$KY4KOAY*A)=Gv8}{Ruv|#cyV}
zuM=4<6lJ*f>s<9;=MFNO%1I?Z>)m+l!i*`KXYRaeu%p%Z)zRCBBx9HA&e82%rsw_1
zgHg*%wz>OkhWN6i&k8dxbxBHJOJ(@*B%x%h@w=JV#m~OldvA}eQAF~YTl;o!9OcvA
znq(rpEIi@JmyV0lvFo+UxXYXaGS6T7X0p92z3crw$;}P1w>TLB?$=o@3k%3T%PwEz
z7N*V+)xWjx()3He+UhoM&OB_gEH`sW+wnCTy}52a7d@WEExf*T>$57I^Q&6IWO5Hb
zc-A^w&xxUM8lR${Qa=0j3pHnsU3l=JLZsF<Y_|SMo0d?{r86?8Y-MZMnrN)mZr3`u
zU5W2}pv_|Qt_6(NU#3RH&7FEpZ|%)JVTy_mUw44kez`<UTCaCje}4U^n^$uV7Vy1V
z$P{*M?~J)SFQmMiqs^G$@6V;_-@DT53zuV6=)TO7<+Hp$Tr<-?``RyHH^;qS1)iU$
zsunL3bXnq~9Cpp0ZC+306i2Rv*re#jH=m1F$V_{(y4P9K)ZogWKTEW<omK5`$M~^L
zPUSn5c>m_**OxAq$UchrGUJ-h+rQ3h-FvLX{nC%OrZ}(FEL8$cH=er@W&a?rE-TIS
z)V}r!zpJYPJSyw!54GO7e0b`KaMe?%zU(#Mw#d?;oSoVB9{Z7h{}K!?Ox`#z!|dXw
z<9E)@yZ30aY;C>9-U<9|mzZDd{Qm1!j^!%ONd>9yYvwA&9A?vRNqDl1N!X@5q+z$Y
zM10GJGsmsJF1nzz{%~Y41GlCGYm()2HZQM(Ueoh`8l~51ip=0RceYtGJwJJBL8AYn
zM%mf36moAfYw2k5EDiC>I$3pwSJirU>%uSRIG3jMcZSX8)JyM4ieg;j61Qbi=#`w7
zDU-O>RbP8WPB98Sv2itz;pSs=BW6#|m0_4WnN7)Z{$w##*2Rkyee?46%rI9o6VfZ5
zc->UDG|;g~bBW5&E~&tk&$L@?;u<PB9%_p3W7B%wk}%WZL*DLbuOC^x>0*+4<$L^*
zk@!LDgB5SAO4gmd{*<X9de=R-BS|Mar+zR6&n>cWCuFYv{P=MFj}PMO&-b%P8-%DO
zl)gH3Dr9Nt!pK{48&|78-goWyu^Fqj99fqU!ZG92%q4TQCQYnT;xL=W!|=5AP{?$V
zN>TSm0U6eZON?gBa5K<RYnEG(cJ)~E(TH<jWs+F98*WJZvMpII_57Szn~lk`#%IlO
zjgy@xYySQB*YRvgf!?OoJ4A!{r>a<)CjEK(=1)nP&qg293*X~j7n;@e@2*_f{(ST1
z;9`%RPg<uewmsXb@#MgbWrg3?M8(Xr3ZGk`D5m<(=3=Xi!AA$rZD)>G&sytJ*UrB~
zAm~+SpYF2aAE|z6Yp%tFHhYLlsaIdp?S1FTy<$dXpmWcSW4G7M5IIxnc;t|?!i_*)
z&1$zdd`DJI$X*n=_*SI@*Z#w%r8ia{{AbiIa>kOSye9Qa!kybik5$jPzj^58krTy{
ztik*@`O)48+b@paW(Yf3scS5qyQ67B)y&-w7EWP#ut!0P#p_ecluMqL(=NHXxo$35
z^5tA?Ft4Vns93mC{+@!++Y7dzVLf|nVv}eQg9F#I8oxvLw8}-!?PKLvd-dRv$r;|?
zg59TbAFes2ZoT#S+O_w#vKRAOJY$?teRuLi?FprGU#IeA8N^IqJ9BOo=gaeFC0t(4
z;l6h<XyqgiWeY|N_Va7bxM%%Xpk9&`&KV!RL@3dT$&t|jeC3z9F=N9M?TE>uPj_wE
z7HMexdUdG|+u7R{{!2~SH)u`NG|lH}ik-e-dWx6!zSSbHMHl=CST*&6hG<pt?KM8m
zf(s-Q!mqKs`NnlstZ)ItnS}y5YZ!YaqzmVYrs{|X`7GUf&@7lsaIO@C#f%Jt)`Thh
zY_qn9FNpT8va~io*0W<@vB9$KH!6Sqb>ct1fx)YoHQ~sVYrig5IBg31z3S#d!?e`3
zVLtu*ZasG^FI^M9{;fE9_SV05X53tL<mj?0(e5yj-ColZKPQSZXx+#?RVKy|>%8>q
zJ7Hf_N2|k&4Hs9<536Z^y?U-8L+reB3%cfh*PC^f&p6!r@$tuN=T+?w-w=0k*6#8h
zanr>PGF92+-Crkh|LC%~n`?zGeGA)p;r99SX%BCHmP~N((Of(A(q5)(GX#^p3=L9q
z9X6~MYJDg5{AJt=zl9s_W}P*g&h+{0T}i`rqSqfcpPipmaB$NimV`4oa#9RYg<5<;
zFSL)&TqImETXXBofXv@nv5(K4nl=5k!}?uzI}AVjoy)z$zao!U&|!VRHMUa<ORF|+
zO}u+-^&;UvZRP)CY7_h7?aZGEY?xio$r*31!m6#v*bsZ`)uKZ`&)j&NzWM0FB;mGG
z5e&;Oi}P;%J+aJG@=F_+rL+lOXy(*Oz08rhVb`uXoGmF(-xxaK({9bC#4yfJ8_J(s
zg(vRW)b{7SU5p>w-dxpdOHF%kUCrHY!^!!v-5}q1H|xiq;N(@8*z|LZLv4bq{uhL;
zU31g=f@z8@&)*l<4hH^?%U@g_SQWNR#Bj;NYm;mDX>~7LC4aYGBhzT^&ROlt+$0S5
zmDiWMKf9-!edD^mSgD1qq2|&(_5U50g($MGwlO(3HMF!YNV2#lEVOs?xt*I=W!c1(
zIyHuN3q@_55wEc1UP}I^IUB1Mzc`Z1`%vcByh6n#3FYiQ26K&P8e8%wwFfT{dE9jO
z<=VKl?yYIn*1Mk`UA;NWI$bYaFTJMyx}C{uBOVccIq_c2)mgy?Q!OWlyM0@-drchQ
zjrk@OJCA+1rjW%YzSREl<E_SaFJ`V(GdbFE(LtJHC1cL}lgGF<|D-;!Om3B{+OG2@
z;DoN?_C2pXHRoEhxh#u#?C{l*VP=x@vL3@AFU_YmCNWbi??*qKF;|9vN{-AN@gM^&
zwR6tKk$g1;CK{beHY?UmFYZlP$-HGD(`tF0&#RApJb8fqeaoHmGgTOM(kH1o7Ame_
zZ!$4B6&U?2@5l`!@z+AFDOWsBv@UpJrWG$1#ALFxeUU@V`T*syOV5?ISM%;H`MOH|
zo0`_--z&XleAo(ZTr(V5Wfga0^6H|h$XS)8ijk>00z40DIi^OZyqmL=%Zck4w<Vha
ze<Qc!waGr53`>QTpS=!XeXk;H()h0T;8MdF52et_jr~&{jnXE{cRynevNf!@@vJDE
z-@(!?Cf%!mQHiBBC1vlT#Ny8a>-_u~QfIGxw0o~p$TiQlxwe*%cJZ<KJm=w<_UygT
z?o(41)vWR1eAN2n<0B{0NzZ-sp2R9eTt3w=vETZhSWz~c>cc(JdO4=IE|;?>Yd*dA
zH}?HB%jC}&F3gzX%E>ZY^XmH4WiEy{^i?J|d1}thcGj8O9F{Zr7^eZ_j<vR|YVY{6
z)V_68f3;ox(9P%C24>akOFti+I(7Fc<_AS;>|MfJqoy}<ZwvTcZ}j2#jfekoH-5e+
z%D-yexmk~to@B4xIlK5E!xTM+bCnC1EMMj{&-&omsTaThI9kYIt8+~vXsSWp;xbpq
z#anuvuAd2s=CYjZ<n>(U>FMV&-VYf~&gmZg>$I-*&aQvAPU<Rdm)N#+N~O1lLr+%X
z>Zt;)Q;yB(<@MVpwp!ru7Ma#QFHP~(2=R<L%68B9oOn2Q^SREye;MamNo{WV@WIsb
z<hDCIG*`_zY3af@TeGnARa5>?;jOcR@^e0PtPDSGQ?=p9t}R}kRSwxo-$kE#aUA2;
ze6FaUQq5DS^L&Hp_P=|Zlf0KHUf_)P1{dYit}z&N7ySA3^J)&GLPAqQm}*bvix~@!
zh1f4@%r>dvo!!__!m~ALt@nkNgK>>351yK}FY0oFU7IGuWsR(5mt~d(J~_UqYvH5a
z3^~SgU&Ow8qQq|1@Ur`GY39_d`+c`W8GYinl$xw^J;uLvW@E?eXGXWqM=ZP2k-duj
zqJpKB^xp~MRUT6hUW(Z}>+Ee?>-UTc<}%E)sr_-UBIL!xD0_v@(>JDpHuse|McwN#
z)QT6{#B_}9ZlpQ`i+AbOm)(cAZWZ^9h`zmzi^-(Fe_Nr^;{S)FlK0Ix+cQ)0_Jt5T
z2HVfKZfP?t{L(Hyo&C+EZ~L#$?oTYQC=)bkkUe(b%3{OUE7hYm%7!&*I(*+cQ+sdg
zX2#!>^YjycFZ=enzfX_xg4w$s-Zu62W3vvgVNgwy*u8RPVBn;gmtromvIM-@6TV^1
zjmmyKiv{y-YCX=j%}lBazSMj9FT)w<=7g6k{SNYdzL0U$eT|Cso@<ZSEMOK%=;e*d
zwwjl9DuFBD%3F!s>MPr?NC@i}ReN$XNW^T2FjqOk?r=NmhPKF5CiB%?{MF?QduGfn
zwU*>mzEPPR_xDx)uK0e?RlSp%rcOwZX3zHDS5a3{ck9esZB=!i=jV2AGGg%K^Yv$W
zWm9joZ}C>UyDcIC7EX$h=WZ-wY27*7%_A*VpfY#mlk1x&cgOg!)#XLZ%|4quH~S<@
zg1)co-j=r$TW5q`*we>4YvCk@W%_4suDNydvh&*TYfYV@eNq*lA6+nMQrTF0HbY>2
zLUY_jwb-v+Wh?y7e(9(@CA?fv<Fi26wRJ7>hBGyfw$|s@r0@JK|M_VS+tml3k8ujE
zW2mTi6n}kh?{Z^3m6*(Rd!nbQFI~;^H2PVOr1TC3*Digp6xO0+8e2}@cpxLi-xadp
zr<wH{omjujH*!h!^>55ivoTmm={Vn(J!f-;@4H;r^CgdVti3fov|#GfAkTw(P4^yd
z6~28b-g4vFjRr~O{&9vAWzTMFI6C*<sdlxBCp~AS*)5Wt&+XyYtB8?Y$<}u=@$r+r
zp6^##b<Zmo+W1a1n&F*jw7hKWG2Mo(#>^X+u{l3mJHaYlsG&H|_2qM)ou}5g%rky=
z_tI|F9iKB^91^pB-)Nm<vi3n$kk>241tpiK7FbTcGT}wc(_3$}86r3@e!us#W@`7G
zjiGts+jd=+4xhU8+`&`xGS==Cc$?d_+_Li=r<cOyJKQUGIw(dyc0HH!MqcG@R%}x2
zg<gTpTv0(6Rw!47a-6Cx50PKOq;zx#_rW>EdL|#X3KXsQDi%?ec=FG?Ju$m}o!lpO
z{pLH*<6b|0DG07<$+ys1pr`fy+ttnH^X+~o{qt%2_vGKZ+mE@E?{@rLwWXWOR*>bf
zv*|fW>(yGXyydgbw$6C-`Pk~p6{k*3DD*1Mo-JxH;f&zoRjex%ZdkYr&o$k?*J+RK
zI)<EYSN1y1m64LlyKzoFWzEN=vZ4yJ)%*UI%qhNWar@U^uQPr;4>H2OY!kA)o}cFT
zSxUfg{#@=SG11YYhtj<QSyUzGhpQSrb?@18{?EG#aff|#7&AWn6yV==R7Z<%+Ot(U
z=TA?*G;8~Db#AG)8L?9e9VSev^5;KzWu9D?t;Q1vubrXYrHVxruWsG`v@}09>>T?A
z=I1h|%vSnp)sw22*Z)7*s3|WqW5Ehm&8J~+Z9~reaq!esol!7V@@m90$J1?{YnpB(
zYwtDw_AiO8bp5=a7oPO|zEE(m!<PBRr>CwO*OoGG*n9og#w8{UWo+-~>f60cbo`%H
zE^F_<r|X%}B-Pbr^SU~_xr~$5W;JqVpAp-0=S;ZTevt$Q%_m`*>+(K^H%*AnuH2?w
zzI>|wlLXZ}t8|iftSG*!rN2G<8S?=V@u!EvHZI=acCTWi-l;k7cb*D*p78i+O1%D>
zrCnw-X0&Tn=zL!N>d6ImTVprxZ9=&(cYbWT<nZbNn|RJ$NvnP9C0+ZkG5EHfn-Udq
z<5`e_Udx)sx3}V+X2mL{Ip0V#UDa-)`9iB-=hRo*X-2m%9$skY@kR@Bv-ZA=5w}(q
zzPnWOqrCp_tEUf-KdxCK`fAntRp#rAOcPd}J~Man*RBkfn^W6nE$-zBNKMtK_?#oQ
zW5V=hSu*cCS8MgCGy6Bs=?u};;!#~}q7&?!`EFjO$l)o=p6Yzwnvm=j{@jUY>%6%#
zOgC;`HobLncdz^7#)STR5<!PRC+Fn$eoWN5AtAk9bmy99y_zZ7F@ev1FLF`z)=W9)
zvP>jEXt&aoXIU!h+x0|#ZalZ@ZELNw{p2sz^No+SD!f|u)qyMR3{Qc{x*I?5gg#3t
z+oSPN>-8MwJ1aSz=0wf@ZmT?%k70(AY4@u3AOoY#(?!p259BCoH|F2?^z@~t=M>wb
zJP(%b_;!%V!ccU^jP^jq>q32JA{f5ic(qx(aQSwlqqd>p+`GR$NlEH@zF-Rbyxpae
z?krvh&Njc^`TgC;Zqa>vk5~%7zpMNC>ASrit30kL8H*{hSxuRGCrE>>N^aHDm9gTw
z?6I8FjeC0|uI@dtbZ69w-CHfc=k1*G{`#{T3q5J|BDL_Hs-8RzQ#KVgayuFqi%nZ3
z!?JPv9K-OL#~2H?7^mmRtrAQQ{JzNk<lA@6OCEG~D{wuN)|X1}Y<&8`^s<wMSBY}y
z)SkDeV<wcD@-KY9SAb_9n*-n4)iG<lZ)>}R-g+Cgj(g(uueYBCom8w%f25;Xw0rgL
z@+$?CT-LO#mlQha_2bwfz1*weZ~ok<`SkMh=1=mflQk4>q{Pm<elT{fRj}{M%7R7Z
zx96M}*zzn$D=C~o>bcmpeTE9_s~*ev`Yy~oo4$F1=G((hPyKRbty=T5;n@?06~%iq
z6-De~wB~)~5`5jidGe}o4ln+w={N4nPp+3KXy5EmC85A|a?M%>>yYG46TWlx>{zE*
zeC^uuO-z@?+nr80d1>CTdGe(?{)FT7SVjfo>fCcLpU>a2I^b7$_dz~mlWw=y2ZQd^
z*VNBe?6!Y(E_DB&r`g&4Yk3>2EmsP>IeBtp@u8i<&*u~`dVkZwaKqlee-Ao%3eE4(
z2xwPeNIpDG|MV&?ZFUR(t%tqj9Zb4i*2}&-y=sw?vUt;tOULd$O;g`|=lim=-{#-B
ztaSh5zef_*-(THy-F#_NWBmMl&A;C^otiDHS6g5m!(RSqVN>z!3_a!-b1XIQu0C4R
zS<G-$v+wid#t7q&_r%V5Y+BU)?c&9!FJ15IN~I;${Cgz7{mBpa{~x1MQ`6LsCwO_C
zZB<|O<nE^G)|aQuf7Ga`b61x0yuN$;^W-M3Ze=r#F9o{irQCzL=YIBjeK2xsh8bI#
zrs`CMa3%fKT~2o4i{>sZ4hwZwmA=jwIm`K}k41UV^I6W9bQZ`7JDu|BJhqr`U0M~3
z_g1IpEE~%ht{Cq)6TQkMUgbH<gQjwoq}*jMR;2n}UbcR*`^J-xW2bAhJP*=;v-H>h
z&ENBVCuQH9sqihA;T_)&YnklB%1X+gUGAN4J-W&&Cv0=Y{CAn1*K*fhywP)Z#@S|{
zYnPf+mZjUi3h%j>qBc1$#Y}hanHwIqYZuP)HC=u(X>zE|)zZYdR?~Y<c&EhN-DoZ&
z_pYFvA)252Mq|gBq~J8$HIrTAr?s^lE@Zd)@G$!RuP}SA_s62czvuo@Ix3cs`#3-M
z$GZJdE1&2D-v9kEQ2i={h6M8rUH>kT^zP(o^Z7O0*7AxwI=H^Fu8Uf-l`D%^CH=I+
zm0s4>cM=b?HhL~QlBaWH`odGi)^Z#%YTo@SpQFQ?UHjN4JU+ap<~V!w45k<7k1T6n
zq5pYl+XvfElQ&=98MVn{rnST7*mRMOGZM>Mt0H($l-Ia8PS7tc+NQ!4@n(<5&gBa`
znk*(YADz3ZxYW5>Mq&FCuZB5Sy=^CI#~N-IIm!6x=A)XqdRJfGbXs}Y=KN=y=Zoz9
zezqzV)ZE!yo&RLbqrbZajx1O;Elzj(-LGA~S1<25{^#1&pD&#bzkK@qR7qIXwBK{5
zmQ^LOHhqvxW0bH=N?XMK_`pj}xA{TG?rwemrHDhy@?gdO`Onm1p8u-(R26Cag1h%!
z+DxVo&3a*~8v<VR$e4+6F#KS=q^c32Yr^1hlb`9}0!Nl+>(_}{>#iO4J^wzn>e<i2
zlD?hc-1pwQ+_et;UbD?^{#yHe_D?j=6s^3xf6uAE@&BHFpI+&|T6xAE^%}8|${_uz
zui0bW!Wo1DTJlz{+jVfYp4;j>x>d(MhPR&Mwm07?&6aHcV%;ZR0fR+%OCPVEm)JA)
zd8L1cO?=eTYbPeoj11e!wt>y+wZo|eFBV4q%wD|1|69ye(TVfq`J1jTYggcqk)L~Y
zmHMAEI?fwU^=zy?C_6uYn@!{|qaAD!v*ji&mG7A2sq^~cfeGDj@7}DslBA>OdWXB^
z@p10!Z~9MiAK7O8(5x;oiF?DgkK2vExaqH!iF&TjXTR6W{wP<+gYJD{@mfo#S)ISK
zs{8c8f7K3egWmoK$(TDcHJ$Zas>T+->E7?!4?F!R?YCT2x{T#>=p4N@Md<=3O7$o7
zg-wlr`g4c;i!8f$yL+<_-?Ex$v@U5z=(1gF<~|C)+SRFXG`5<NLFDVxp!tkDrGK9O
za;u}UxM`t)m-JfyP9{lvwok3sg4pB)Co5fivB1lyI&R&EUEPBHn|o@f=2tCpWB6U(
zp7VIhs@2m^mVEm2<@kBOXRkhcKi#2p{&P+59FMv$_paZs{v-D_wS0ct>x3Qh<=6jJ
z?7tuT+g7mWd${%+<J6-w_u07X#(r02U2S)A;%2FL_oo`Y;M~Y$+u<tG;d}nRG)tiU
zjW2%=i0_-=W4G34-t*ITUz-(f?6>AwaXWhU>b%OLRhN!to`3ms5m#8a9qZn$mT_Nh
z3TS_5jXc=f@|b(GQ`pnHp<D{*6;pZ5C6m9p*u73H-zrv=UcU8Q)%UPpl8d!XlGe!Y
zn^j*^e?n#2(oJPm;S6DFGhROZyZ-L#Pchj~1xv$Dd`{&zGM;*}+iV(Z!sq6tHTe=U
z?^)QR&aRTF+&5=-j!2Ki_7zuo93<CTX+K#VwJhw*0Uj~tHAbI4+iYJWs?O`I*d@2_
zT=D(*+$xSaSCcx8!teioboE{H+<Uw3AJuwwU$-hjE_>#+w47~>4R3wtJ-17rzjIIX
z(}tc!JLdB<m@8$!STODLy{fmrOV<_8JoHiU>-%$`RIg><ywCss*d470_kZ)(N#Fi4
zn?EsX>%QAJj@@5&K|1%}m-~NCKi9W+<nnxR{Cj-Or^9Q4f=zE`Y<E_*oTqC&{gURE
zhf(F4a{~g>c)ZuI*)Ug9pP_7yJ449|lN%HH8m#g)WZz{rA3Iii+-xIbR`#2+wMyK#
z6TgVJb27v^e?G9fdHUAR+Dn}jx0tykeJ_m3SUhD?keB3r6S4iWD|emh;c|T|B5?BH
z>Lu1fUMp3M(?0FVySHZbqkX$yyO}O6IbGA}&3duQz&rB1n3ceUJe5;dc3k`#^i8?{
z#=|v1Z?E>bSgk&DLNL$ej@r!WQkM7DXu0*3$F#BMnXTqx@Vf5y+*tYj@jI6U)M7Qa
zX8O%~pm&iWPUU3q^4UFRAJ-IT>M=$zcbrLB{Nzy0HF0^}GwCz;DSERPly9xSWu5Jx
zc;)D0cK>_ze|~fZd1Zh7@p1CAWkL-pGiIObTYT|g=7&x7?_|_(I?60Eyuf}yE_n{~
z*6HtmK3x4QI{B^Hk|}b56Pu*DKS?Hw$831%%vH8VH&}pkt?2Kz4UJm&SM(nb3X0`S
ztZupO&3WM7Hx>EMq0iQue^b5g%WZKZ)j50V>Li`=sIY&lzJD-m;bA>?|HrlbKgYIL
zm3rqKGTZjC@A$NXw~pOEcR@OLij`IE&&q9c5?lAM>(8GnQ~T<N-R-U*`zK=CQdb=>
z|LqleI%KnuHkXS1rD=DZ*t>l?3-8OaEPAQ6<AF=!!f*zw5WY)y9pjnnoa5K-IBv{e
z`*6p^{Kk&R%+QNQ^LaG*WM5wF;#Eof^7+Z6r5_tlFy6nCnmfza_u`I8VcJW)Lzmt7
zBEah86}eW!I6d(H#woQ0%iiP`wq6%MnbT_{a!IPs-u{$N-K^)Qr+XHa)|JiFVQ*BJ
zB*HVN!R=gDn$o#nv2&ku&z-;evXSfCB?=v@nU4Q%jC{H#h`(t!Pv?zqn|*k{T5AV~
zYNv+PZ4Gjmr#GFuwrrW!<oP#i9>vAGG;%cU4wRR!k3Y5VF?VJ8jcDn00#De^+fH2X
z!<z7<!tcCE)t{QF`BmA@)%|*3mhQ2RncW+yc-GgS!(1q>^<$m<mDg*gXKqj8Yunty
zP{q_b|NXja!NrUTbG+0xM!e-aP!wgh<Jg4b;cn}~Z+&lmSXaMr`khrfKe+m{tgaEh
z*mqxg>E`5fSNode8dy}CyGp}t6%SRf&gwifL+d=Njs1&BJu{9ctqo1z>h$>5fw}C)
z$yI$iRh0{GhUl#S6mx39hrj2nPJA}D-?97U!tKYBW^Ojy@BCgos+9BUsq&f-?Jq6|
z*xt?QeCD!H`sd!QY#EnN?^Jb2-<s5ObSY1uz>}Tpwdy_UL|A4TzIi)OQ_w>De!QV?
z>DABHt3#*0J~Z7Y>1X4eUwwIx#g(@)WJ}juE6>;cI9a~t)6bi`7WXSKNjhk5HPQ%P
zb7kYg3WtwUCqrW2o-Ukv_*Gf^?{fB_bLY<)C7;nc@9WyP{pD0|VRohT16xc!#^|oL
zdH-nBkp-vb8l1b^aiu8eg=je6CYF0&*VhOZ`%MsT%6qc<X^hot#-j@b<bJlEGfuvy
z_B|~rE$hWGvnZX2Kv7nw*;DQJY2Tdkx@h4n(WD*v)lM3FW$wQ%R{ei|(fXj$(5L^T
z_BK~cpWGZjXU<%iy%yHCm!dM)N)#8A8J(}aFP^hCV40<~+S#H|w|@n)9+A~yUyo>=
ztyZ2mEA*U2PMXr>`EysU@;M)NtmNA1HK$fz4c!&<jb)S8d(IS5Cw9jIbFNRm=l2`$
znErg1f1~~q@vkfIb2c?zHr`tO`)Ac<V}`J0n{N8_oz_Zvtj(Z&JbGqUs+L7%es0VA
zpN{4W@}}mc9b?&7P-9rMXI_EX-njI=UzqLBe^y9%tto$^nrWu|n{Vu!pI)u;m?!^!
z>E5Rsvb(u1dTgrr@mD{tkV%lC{7Rwgz2$#3IE)roA6j3;;y7<++(Vf#ZI{K&`^|%v
zScfMSE|-d38!`3huGL!Kn-x5!r`tY!{xfK`WVl3u(+2e;H*daN_Pji7^@95Ai+9yc
zm?-vJzd$&n|Bq6u_(JWJu$31kR?f?>3TJw36mIuSmCcgrCjTOb8!n2!S8Uu~?Y#9@
z+S18O^_xZ0Rjwb{G;dyh)xO7zw_i(}`h4g69}&!f0y~uU`}Am^_}=QiXmiuGw9C%v
zt)^$VJ|ElUm~VK^?B3TKRy*?_=SOESNayPL@tKEDcdTM*$l!U<?ab;BF3j+xg5g{r
zOMk(m!@V8GvUB=ouIgOJ#W1~-Q9(9$`$4z2u`#z?CaqfGv+?4(@b!)#H$AB^J9}L0
zL+Yx?w{>%0XU05^pE~`IsC>t&4*quYw`OeNISO)I2hY8~(;YR5BY$@0eLnvzhJ^*|
zKK%Od;ii1u`};og<ZBCOEID~JHL0SVQE8IPrn|PYHn%Xy9Vygknc}o^$rX1mRWHf4
zeN(*BLgyN`tlWDmed<h`aQ)@0d`y+K7?j<sCQ4Xd>P|8jiWBjc+&!`4)04v!m3gl`
zyl4_A?v;D$%8rS8mr~Mr3g$Y?Sl;`(BS7iZ*`6nR5>?oh+hyG+HYh!uS@P7)X;Sj$
zmOX5YoDC}Tvph8KcrZ*l|L|bN`_9F?cg(pTSyEV8zBPaPU-#<w#c3D#-iF_isrvNC
zLsPV?>g-I%iHibQ4#=-({BmB_y2eI|X~&`0LHnM2%ARBt*t*M6+fTO2MZiVHl0RFx
zsg&d0M;^_k(vF&@LYnu5Qn@5@*WRxFVshCy&t^jCs&_(XC)q8&@Wz<^{+3IP*R-mg
zvx1gg*WLT`+>@VoJ7k~y^iyVV$(;FWO_0GWtw<lW>C$VrX&%b`!*m2(m2CUKxYsUt
zzI@%US8t5hFbMX%e;YGrkHO2Ckx#Wv6MwDp`MNB8(@u@c;ei4bE<vYWzI<tP^-OHx
zM+Hkcb1@#5b*mShl4Ve3n<vlwB0zKQ)^*Q9-xUNOIx1vyUSUGaeAWZoPNznf`MJmG
ziF8Mu@iO*gi!tviQ-1Wb{fWJV{mC<%F3j@eW|%#1zx?sg!!KDJ-+0D8{Wtrk<F(D<
zJKuZS+P?nx<>u?uRka7JHQ0W?zuWu!Vbz8<{ui5%%B(J(_xkhX>C$^Q3u>MJe6n-D
zpcYqC>#=3*$N7KAe`VcWy_x?vm)+}|uhkm%&iNW$T{|)K1t-JK$K6?5*MxcdiGMm_
zG@pxwsqymZ=~4MrFM{izwD3-ST5RpK`OvcTQ&-MZe_R(G6BZ~kB`|>Ne)R?M<r1u`
zS5Mq57I@zOa_IR)xzqf+_8FYX|JuTwaM`#|K4kS$`5&|6^1@zpDsZ?6tod|@;l!VW
zt$VBrtb~pA+A~Y~&YqJ!-+wxBSN(I&>V3w$9=&o3+ZQdz?eOrot>P*fxwRI75l^Jk
zHk94@y<A(};T+qN*d8q#ZIQK~d_}(KPY+$cjpJZ}tk0^==eXwlJbT8-GB{AcL-P60
zNy5c3avKBE=h&u5Mm^@8T^x6QTm82`J5J3l_5XHlm-dp!i#-@0e&3MveU0z>XOU;S
zPHGjb+O;wHLf?C9yWbW+0tM#i{YkZ3n6Tkx<JV7j77MK2>VD|s%2S2gW_-W)ck0R3
z54#^$SWIBn+%_-bcE$B%a2G_&<U#(IPd~qe%&(g`_iVzHDUI{b&!0cXV&D2{#S`_9
zm%nGeuv~woeBb+TmzOe5(&(^V6wpz!)90Mu&Uf#P413lw6c{S(-p>5uQPINlHul>-
zoW6Kml`Y3A>gi7}d+u+_@n2;3n<W`&+ebD{ocx}>?|r)dmzzn4xO6wK^1W@jmt%si
znT(L!5yJ`V8TBL|vp=7#{#c{K-{1ex?!OuW4gpW?t}_@N+jZ#m(xUL6ZF`R@TUuE9
zOtpIIw|8!6fZM*vr>{3BEV7%eGdJmHR#DZiYgNl8&Gc%ipYTdvI@rh0U$*`od$U$N
z)6PiNE8@0_zf{VvJ=A*jj<?D8{QcM5W?>vP8#nU_hE_K>d~RU4#>U)r`RS(3Yz&N<
z3-)N3Y2WX4*&4;g!gfmBh$F?HrDti?p^us$T3>ReUpCfxRaCR_wV2W?S4T<C_ack`
zUgWECF8s*V(Dm+3_o4f>2gD*Leh6F9^8V|!wu@6cs&rl%X`kcywRV%m{MlE(u(<s_
z_))2NBf|!{yc=(29W>tcU&`8CoW8ug)NXa^*E4Bz&tJ3NtDW4{dykW;tdu>RL#!@e
z<jbFXE4(&c%{nI9dd)F;<()GjZ>(Sa(B_`?d&{b#`L}<1N`7bAp{sD8x##eur7sgh
zf7lhr&b{iDeR+Ap#HQyTJ9a4*)TXzt)++x|+we8Cavs0Nim*Furxt14Ut?m<lW~u0
zOXSZQe)D|p9MdL;jChNf{1ewAXBzJPD&-(vC$i>~OvKv8J0JfZV86D`{*&9^ed2fi
z%D+B;z--$`bD`*lt-bq0A2L@}RegE%A*1eJ<UD4PH<ss4r&N{JJ^N9Yzc=pNz1^4n
z{jbW@ZO}jd^3s%*Wi^%=yTtXkocmmp`fOjpjE<=9Dal(G=+<t%kbXz2;!b&Ks7O_4
zQiAUKY3c5^RaWo%JC8^uWO5&9eRrfS^tsp5%Ck|X{^d*+mp`6Uz8T4$J$bh1qJ275
z5gD;-Yr>8g>u!|4AN<OIBW3FKFQF1W!KI=n8E+fvOug-}T2?G}?!0*n8o^)Rq*QLL
znVIE2*KYSI{~IBfzjY;^47{&jUB}<RW4F(^th&_AgGuZBUj>1*srnlVYWAydVEgK_
z&#WVTZtD7|Z>uIM?_Kf!N7K%A=c*#tPTM!Z;hk&Y)}S~M=7h`;0iGG5o9=P>>Bg3)
zou2B?7`D{^pw|5jeO-Z9G~J3sPRE(gmHoYUqWn?qjW^`qF3~=_=u<{X!Ine+GSaus
zx_EE#)Ci9I%nptDVN0KGtYWHr@VjyU%b$T0xC2*ja{GI5g}uh7X5M9oKmV)Raq`QV
zkI#FyA6t2+{!x8$-PfnbB9@feMepBr;#l=W?V_7ro5L#Y%>B+sGIqGJuQ{^mUrOBt
z)5?Q-i)|-9x3m!o?K>nMD8RfyB75fdEp}5yDqKS6Y3D9-&{(=i;l$@XQ9tE=#;%#O
zOS><A?Vl`GeW!HU9s7hWR{M1Nu8)~jm(P3LJzcg!uu0*8*+l7htG#n9waoSzY0ozl
zTR(a4rW-dK%@f0Ru6riJGi_zs^fkWug@s*uKl}Xq_D9W}7x|P=IA@Xi9IZ*#u@86m
ziq%Qqt`JXNeel*X{kohjnQc{zeHUK&Jbk_fFGEhnk|{}rq7AY$ZT!ug)RT8IUs?KW
zp+JtwEXl=NFNPl6JAe0%l!ILQ)iY=GmhIlLW%j#(m0Fwpcx}|{zj*8te7!ng+178|
z3Q7+JHJ2{SDw;VX?J0{xC_};qXO*s7mt<X-91f=Ox{2S)SnH%X&!%!!$fTJbUW%za
z2@T&RVs93H(`KKrN_*)ff4iUauD$9${G=c&?AOite_o_4onJU_^;*?O9GO?d<rys`
z>P3$0G)@X!`*r8X_Ey!fJ=>MtXSFY!^6~J;xZ4XBdI;-x?e*C3j*Efcz0bVJM)2ze
zq41AKCO-bS@k5DOs)Xgdu6G?PzE_yo7#%*oSaFkafq80!;MBh-6fb`+DLeCc;)g{S
z9Zjq}d=(eT<+pxcVwcr^Qqe<mkz2Azw==_MYi@=$$4`CHxokYosPfm@{`dU<3;+L`
z^XKut&v)hjKKSwB^m+ThpkqPD!tSNiN=7ta?epL>T5J>?zsX5j({&*)LqChoF(pTp
zsX+@f);b2H^|-Svc2=}eV)z>wwd8}+zS6Ai{GnH?gI?U~QC=NVxK{9rfmTUM_PnUi
z>sB9nweHlGZ#@~)4!P})ivNGg$?#sU>&z?#-7TE03Bf`P2Cu$;x%%hy{xxT|uFC^Q
zG^h&dmX{T>Ny%>W@3(utoKt(oP4$z<LPDL?+a9rRoGkg+?TOnF{w1fMPYvR_@T8(=
zDbLX-Zx}9w9`snzecH5)al=vp%XtjS>Jv6EUfm@6(Y9)r@<p2!Th^(~aZA}*IMFHl
zXk1L#CB6npqn8=iM2sy?_cUHKh-v4q>v?O}H`R+lmh0dYp$6?eX)Z;r*Uvuu6#go;
zsG_8#qUOibj}ND7|9!LD{@1~^#TkF+FtnINy-d%3bL^2p*LR7fu1*YErfdOTu0hMZ
zBpHl_898KI_#(FFH3~+}SjlL3{akdJ#-e$i;aa;6f8ITDMkuSZ-oI(bZ~AsE6n(57
z)}g#7Y{BNcA`Eh$+;{!|B9-6z+f+d~PGw!AY_om$mJWskijtfYB^^#_+_NsQVt7_A
zHgn$AM6uuAd*#0Cp7~X;uT@Yk`}MBB-A}m&_x{5Zl^acrMWY-(aIFt6O<jN8vF3nQ
zwI=I=sYY_0O1@5tKgCk7>!}@^;<cnl^7<y%nOl!O5^sx4-Q*>jd1S|(?to<vZclx0
z8+y@KU;Ch|gn?CwibD+l>CY1$zs#B>>3k%EbC2u3rPd*fzMb;=`7X8HlC4Ac=Dt8L
z$JL8(+FYF#-?gDPchBsNtP!_=9PMvA8059`<dYy(9ic$4mD}VWhxlH4(j+c<TJ{+G
zn>#0rQ>JPKuU*~r`sJ=2CN~~t?dvU*v@Bfn#kg)!#IG!Kv0bx1STy`SzA5tISDQ<X
zcQ2$Sl_jrBypmDTx~z}i!Ixv<M$xC5QPN636Lf9{izt|>eqYk@B%$H^kH%PytQXo#
zr&!wfsGXnu_^^Iu>suv@J1t=hRbNUf?(t{3D4p|YW9r*>!OKWqc`3t-kHV5&j~sQ%
z<ZM4KvY1w#vBx2})HGYxd-lXf^=h#nCxGg0Lz869d5jEq>$}>zS7k0|n_dtR*Y!BV
zEdTs@V-<xh48GPu!81x&8_s&WpL=)WqfOdforfte4hS+ht?gUQ8(dW${yW#K`+d8d
zEpO%#18xT8wetIB{r_zLYkT~khx;q0mfZDdacVnc>DTvGZ|#RthMI>AI`2XzZu#xW
zz?Hjc)lS|8Q-kELK3naz@=VAb8P29Ikw0sXn>IJuEZSr!`_#)>DRBeyg6`A3Wk2UG
z^vM=Ftz5^bu+}%YI%0S4huEwQm*-y<TgD*aVfg3K)uNqj3~m4NTxQO^#pQCW?~iar
z?T;*@op&Zh+^$#-ZVYzI&pnnHqkHA$>BDmS8Uh7Ok39Tx{PUBde}60+ddn79tpB)V
z-DLfYd14GaWi5UT5?wQcHpz&7w)$g|dPc=yW5Q{+-Fu7E1TM6O*)Z>aot=9s#JpP|
z>Wc(JQQ0eZk?A`=KNjAzph($6bGq`uzPDx1XSXPDx$;&oF%s5X8mD%$`|x?4O?z6`
z<=lyV+itRJ_pSGjjCLAZ6j&9m;gexJ5x_Jd$ZFeRp4%1-FKqt=OwtGz<~nMylu`RS
z=ROZU2}{FiObLZkyWT~-<1*hEVq#@6>+Ir-jKY39I4AX;W-$NUkioGwdd2IH85Pp8
zhL1(x8OtB*JQP`Kxtc3kFS~E8<Mtq~_RDcWay{S0o^57ce)z>IcmK5;f8V|O&&&RC
zXS{s<hmU_BzAltMaQu6_{C)X<_H`c~oqqqn`^<`Crglj^tO{+<7O{LQ%y`QC^HG*V
zwaOH)B_cs~Yyla2B_uUdXEyzHbyiH+qLTPmYD&<bxPzXr0yy-WE*(>@lIiA~dgjn3
z^Y&E%W<OtCm{>FYQ;^L&(@uManL8hTTemfF*{eqFA72~9uiP>?rWL&0r!;L*z>;pk
zt35gli~Zu|@;uk%?zs%kebIu_YoD<_O)8qIGli|i<#W*)UB_qV=WkZq>%26yOQ17A
zpqSz9I*UI)Bu~r6in3Yf-kti^{18Xx;@CLd?`IOvhD~<3n8y;`9<luhBg0uKeYtSa
zLbp#whBgyDTKsYaPxhZ&lcdRG+mWo99QNEp(@8P+(%H#Yq2V@9dLBN#cya&V?)m$E
z{QGV(`KZ?F?VG|Bd!Lny)qWJbd)MXTz2EDYHJA1TJNFpO*rddCacg;q&i7xNU9yXm
z=SpT?oZ1>Zr@6bO-R;kjB_en0XEn^&8gy~)@uWI#Eklngoo5ccEd|1UvO7-CFIB&B
z^WLiCtC}67%h(zW<>wzeQ#7UNO(N5T(sFT|g7+yZzaK4TUVizgs&f#-{`!xq41VHu
zkzwUewmVDRoc!nG--jO`PPhMgSjG6)hYyh&bDU;BR`E_hUM}~3;Yan|b}W`Bdbn5_
zPVsQ%+6c|jTeE(80mFu;qMt*(r2AX0SNy#l8d<HS{jynM-`O|1y0W|(XKSZ<L`<D}
zZOta7mm5@XZxxlfoVoVitSK{py-RgE-Miv#iD;LJvCo>qAZykSRli;>pD)XFqbe!)
zv23sIfm_G!PupR2X}LcCYkuBcT&*X1?5u+ndw1<H5!t+LZ{1q;N7F1?ul-37m=UAa
z=DE`7s`T4U=gNEU*3?|NH^+DD6)&B+28IXvCF_pPU)5P!nQ+c0yZP9&Df6}GG`)N@
z>8qa0qTE*!R+qjtR0|a6`7z#7o+-(oHHn!mlz9*PhJ{OmPSt!mxmmvc(_Q<Y7f<Sb
z{k67VOi9K<w!cpyRYSm5Ea=!mjWW5;01?x5i(W6?<FVnz@!l3`PQP^v6<1%-m=hFy
z<KfmHD<}FJ`hM+wF+C{wLE4oG8h4IM_gsm$uTcHB<NvhxI-ROEv!}~g#?^1*UJ>zb
zqeOS-t^g4)NzKO|9bG;e0>`@&uSxAbIoGj&p3$^N#k((^KX<iUa&fT%i+%Y0&{a$m
zW_q34altAy{O1<#gof`nD=ocGI7@b)Ij}oZPWp^Q=2ahw)tj=82QKb5;c5-=irgs|
z9CqbV#?6Pa3jIqKN*r+8%`<Pt@kg$2#3BoiN^m#${@+-!F>&4t-+-g#ImUv!rnN53
z3rRhu5vs|Xv32LJkXIR}6JT+pUz5QwHE{E%&#O}wo}IE>O!J59&(nvyr`Dxun2B)D
zE!e8S&nOb5z0@fAhv8wJD<3W@r%k=EqI~xbk@KtM_s#nLV&!)Yb_Ln9NtFz{w0<<+
zI;v_P7;(Dw*2%dSW*jxHy?U=WbfW0F;=*}L*E6JCH4RFUYh9Ym;Kb<PrM+icf4cvx
zGa5^IUf;a9WW_(OcU%m2?=mz@QEpc{xbK4t%k(lQ#_44W98ETpzV7$vnG@U2*R!+a
z#er)KQIS^1O?OGjD%|l;KW;i9E2vH;{F&Rob#nP<Ka2f)nftK+{lnY>t@(QYUa>B|
z$XJ}iAhvi>BFp-%jx#@6n4j9|q^FxQ<7nP{pVY4}UfSQPme=KwzI0+;%klV{|J5sg
zvxL<Aj-0hN;E0#b>f?{rR;VzZ4f^=hV}rTXbm1uF(z4@QqqG^~rievdKfCj_W`foh
z4@Z@IIo=Em7jBkb&voKfHx1sh$|Kx$rjeklS?rB~V5zk^)zX%!$5Paovh)Iie3NEc
zNmtuGH{14c-4W2ty12y_*~+>_YL|H!f|jihlF(gRGgHM${ZEDR@$N^q3=S#b2e0v@
zu8PcKP3XP5@lIkG=K;6fKYw&YKQgeIxS4@>Mb?fLRu6@aR-2#NBy&D&d9m8ksWNMt
zjVJBi$dK_okatx?=wan;++9JQX}8>C%@a1hT7CMkmu6{6$qY5|+-0xA+{OE>Ki!V+
zJ|4K|_$wubAghnco-Qh#spdS(RXFv#oEs-@wk|4Du1q+y)H9q<cEQ2WX*;s=7afb(
zoSc##w)9*A_m18z_mn38dhP00&G6|ZXG6`dgr1{^Wh-s($=$mV5FC<Gsl%Y0Za%I0
zj$iv!f0;REuCG5ItmvQmeogG=9I?}18NR$&Da-!BW+zL)5{*kI!bF%7RK*yMMPzvB
z9E*_S+?F#T$mCwstnd>F-I?mEw?49-6RlM#sNkV_luK7X$*_8^#*^>tv7$WhEa!Wu
z+zUzC*xpfMVm%`x_YTK2xyzvwB@dtczH-(Awsb*;c@kYFBAOFglvod>$OYAWtK76|
zY38r>wP&|)>|O7Y!+6cC&d_Op?wl+kkC{b=p4b1o&CZOqS<yQ+G_}f1`uxw858Tek
zERhow5RK9f-FTc|3e>3k)3Ra7l!~zMyZrLDuL3gDpMC5tYxS#0GppT@`Zl9LwEk*|
z;+}P4cQi7)g?~n-3O8L#=V)-hVv}?;cT?WX=bmTzlNr2JXI`A{-hRA2KCP;ZJ0$W^
zSNGk0V*S=Zj#dj9Hircrb1@B8^<*%bR^79r<%X=~Jb~R}t{*MRe?87oh<fp=>WJ3@
ziQ_5h<}QIfi}DVw*<Tm2;>qi%{ri=FGZ#+q7ca92<YvedTD)*m#R*%+^@|S{tHlcP
zUZ}|Pyt#4bq1ZrW70shJB%@zOrwJU_>iWK@qTg|w>bi(^*X8&#P3r&I7`OeL9bj?3
z-^fm}+j?qfl5F|1vg<~R8AYNZYiEStSO2qiW$_)~$oPp)lQe=da%Zro2C!av+G11u
z!Xq@W-D};|4OiE%%4+%Cx96s_@L8TKXVyy_S~Dme&yLgWknztxy=Pg2TG9qrhdDDh
z**~@nme{&;*Bm+7+%1{Er{*L}zjw)*duRK{Zoz-jkEKp9rlifC|L?=q!dK7s=KK`0
zv|1gS)-%O#;aTz3^Lu=Dh|3t1n(VV?3&?o<Lp$N9Z;_1si?!EtF3KHuW|%3Y{rAl0
z$2B!7)81Fso!R@G<=DFX6CaC$D}PS;m6)e>-CQZ&>PhtTpjgp5U1zn;t730eu2|g^
zE%Rxj%IQ?`>3c3d{C8LB=&dKV?%y^<F*Cp1AXui+bWrN0ki)x)G0xwQ#QLw}a!oCW
zeERQKP2oHlUw{92d0&P<AFlrKyRug}wp@iLQ>D3Iw*KFRn>~+Y)33f~>u|X+z3+qS
z#P{qh550wK85PTR&)_l?2;^TK<YLS9C-i)0%k$rNuZW2JsA>3fQM^xI#=5D%KKA7<
zh6xKp^CSK<zv{PPdG#><zn#SQ?x$=F{I>&ntxg}kBLAIz?^C7wJ03Uv@iTq#!98w5
z`_WCugulu$z4Y4N&$q8Vlyial9EF0<d!m1ZKh%mzU_9`HgXv=QQ|*Fs*DowPww}BC
zsi-%0Un0W>>3tvgPsKzouJN}x{P%d^gOp{f!e-1qJ+JxP)P(1?>L<!qSxcudG&wvu
zE%RZu&H7Jw7B5gw@n>NW`?cjm=KB49)0dt4TwOl*^is}NjjDv^uj@FfB1HGS;jGX7
zq`)C_|NWZgGdk5uACzX8oWF8v-riDu4wre-@oNn3@UIfBig;@IeQrzS&&Gtw+e{lA
z6&RlE^jDdt+2JzpIYW*8hrh2s&hs!9{_4W;!*(K5rT?016ZY+oI()uZn8h*e+En#+
zX2S`L6Y^Q#->zs+0@Z!u`X#?jx37MF`DI#h>CP86-p_2W#TH5_8N1o6<~h1ctHtlm
z3Z_@Hbaq=zpU8Y8WBOZBhLXaQcMfM*v3$5I|M!4RY1J1Kd3nh<A8u7_^@*44(v82m
zA=qW@qcXLP7CO6<mj5s<D-C4`$yhrzDOl-S7C$Sa=IOjm$8=3aI9Mx#HY+o)zE%I@
z&4;58fBM{g(f$2iN#nF0Td{WjJJx+2%8Ds6_8sngCu2V{dQJ*DR^&fPASwKmJA;f>
zf?j+3)&N-s;gDXBJJWt#Reoq!U)sEQ*VOmBEkB<9b2r|$=5^3^p?cx137%rc5q~Cb
zZa<Sy_-ezw-Zy8s7QRU|TiVJHkg;Q;1ZPgh;iat$HL{W;Du468Iatx}a_8Hx=TANz
zocw*}&7|}-rx(9>FB0Hlche3nEHaRiHRwDlrSnY9*|9Z@fiWawvZNmeSN{!*$DH@~
zdb#WF4LP-D>a|rDx>7@@@$$-^%zK~l$7R;xkc^OqIb~-YTn_Ep{&?04C5CG<t2vnh
zmgT;k^>FSM;|djHVMfor>PoBv=N8284dA}4v~k<U-g4z_JlD+by}WQIZo|>2pFS~#
zYeF_1`FT-%^3R{?uPVy)SADWsUsNF?a%;{iPkzzQJ|CK`|H>X059P}%3eVW=_UOl@
zHAS(ag%g$@``lVQQ$oC3sCjDLwyON#d1~d0-u2hb*jt`{Kl)T*{n2b^<?}+PzsKC?
z(U_;fq7*+X!YbSR)AhBJO1-mx9@_XPEN_*s-JM=m^Xj<8d26DV&GM*n$zIZG=C~nC
z)^11rhl0J)L7PI1m-^Lznmqr^6&{9aLxaLn^;c<s`5NZkkLEXLXu0!}O@dX)bc=TP
znF^JapR|vKGAy}s`m^u6{0=tlkbf`c?T=Hq^Zhz^eF^_-%R{wa+9J+pTfJJVGiC0N
z&4)JyWOS@Z+WGPL>zb&o%S|q4hdlXeeYEKC<;pv+%lFi=AK2hzdP6p>GJC~tw!aHD
zi@DU@G_ACMac0udzBl@xeC=Q8d}?EO&HMLjQ-AkM^EJ=+-T(Uc-Kysap;mUgX3V-8
zC#kziw8nn^(?02+#*e=r(c*9s=<wygQvT@<14mQM75N<%uYMFSTXQN#GI-6#+uyx!
z8b9B>HR6oyE1MnV0n3aPc$t12)QGILI#7Kkmhr(vE%%4(FQwiSwww@N8TaGC?FAQ>
zeW<?EWN=>a-RW~#SA~-%e(REB&@uV+XN`8s)<aL73eO+Zy3Ww_FIJ*<*N4?M>=%xv
zb{^cN&Kzid?B<tG0c#X8E}x#Rd50tIDSy+{RT=B9C<rt=Fjda`oyU@|c8Viy>Uzh0
z`a60TM?TfQe7e|pLf_-=lFUzcT9jg)^)BD}_;((MO#N0fH4dM~2YbIS5%0*&K9sx1
zS<z?plWOVP72!(8;_2P~bw6J5Gu)|vzxe&UKQFhx|9frr;qdkL%m#aZ#mHXwoWIE8
z`yOl8LcRx2H*5dez`Y>;jQG8Rkg{8|XOzDCcHKU*!kOXM=daDnn`iF6IDdV+{F>+S
zC6ylXWnZ7au1SCB|LZXSwRt;uWxqVnzQ@uKcys1XX}Oi=7i;G1l>WVT?xEASd_Ny~
z=e<T=L5$tC(5~3rCdqvJ?~Lc4ex`iC$k%ZCxv<@(>F3wo+C3%h<T{RPha2qQ{k+><
zv-f%V-|u-cb`0lN%i2A4%$<@IC-L#atg90Ic5~ut@5xmf=hc>GPc*;&;eG0pwayG>
zPtNWz{U2W&@OJaT?bB>D&T=n~U0-YYuj@p7zTPL(S)chI+?%{r{mGfTdwz5qGswSr
z$1g8;qWRLgc-h@Gif=fMKmF`keRg+$S%u)ftCcKkS7$C?@RRe#?u=_Yt#8fw<dLJb
zukwf568_z9qeW__m%ZyVTmE+U^!Mk#>Hgi^R+IdB<qggE50yAvxdPIbEi7P6QQMR&
z?78p%?ET;QzscD%Zg{#8))bEA`CDoAGu^%Z9{0b5t<~$xD?dFGaQJIjH*fA+i8-Gp
zr5Z9ns9Nm0u4*3lgQwrc%lF-UTzFl2Yx?5|(bV(BZv_tg><{Cw+nRs;ea&Vwg^crM
z*}G!>tGDi%7d<<pKD)eU`IhU)s-wk><=ozXy>wl>(jZP&d@pl;X^hT<PZvL~oA*D1
z^})gQ-+orSU6$`EfAN&@UMGeJU++jUy-@8B`&cY@i}mgCXyyNP?f=j5{uMRe;(o8^
zTX_Dt`qww!y34IRQ}g*&{(KF)cJ3v$E;Wi`=db^Dzx(aPTI+?ifpK;92gSSf-6PNZ
z>Zs^us3?BRZDPH;kpDja;=g@!FU{k7a4r1X^AFE{ti69|ZS<FGd$ZK<*{wEzpS9<s
zHp2sXzEAJV%8uMs>zp}d<>5``Z836!9{=7wUcdKy_WsM0KZ3duv7Mmq_HOaw=F3Ox
z{~rzqjpy7qKKi@dU72C|X3733zh-*-`=@W-`|8uC+5f)WeDvpob?VofAHCTR^c{3b
zIR3!<>q>K1!*$nx?)x+A|AtfX7B+Kss8>x2`eQn`=9F<oFoQ+!^zz9K?F@Ba)~8wj
zk8D5AD3ZM8I%mVR@{Rr$mv>crz242R!Jpy5oEM&44Ug8J-Tz~YMDw-sO1ZnXPtSg=
zdLzYfX7lBT?D4k^XY5e7ero$ak+tEw<N-N12AlVJfpg*#*d8qZ|H=4X=skai$^WYk
zTh}D3mn+TrveDm~tt@n2xp`mowEK&%F)=j!+4x$1D#N3PaYp+$$bUZmZtrQshRJtV
z)`xsLKKbs;^HN^V|9!go`N2zNhE-LnbAN9)+fwswPPm*+?UCYpQlGZJeY?-{k^Iz;
z1?T%d%$od@_5GgM=z@J2@~3A{JiF{$&IgO*N00jdIP+P+!K{&e?puqW-09^3F>#fu
zda-Y&?O<7O&br3&{+%Nq7H9m6tdi1K|Nr&b_J7;=e_YFcc)I>G;qOl#B>b1I-hc8o
zQ$gcR^KW&ZbKKwhKLMH96~A`|pIl$_$&8)GkENAo^jx_*=lxH1{&w*htdlQRz0ZrR
zowmO~jmhG2&C5zTK5IKch6R)T_iCPVkKb+lyF}t%t=fx~7r*A7{BHc}X2WZ1$pgI;
ze`J5<Unm*L&#ZO~luF(SOpfSTU2$TsDQGVH;^h3g&(Zr8pFZa6)IE0Vn6)5iDzjYw
z8JqfM{rZw$^}n9o{eSoFMURC18#~@^$=-RM-~Q*D&o29`7M>OPwQK!m_bpWm`ImdP
zdzl5?=skLF@o^oA^F4Z7PWc>R<66_VDfQLch?nWx(&PK*RjIJ5ZGU#BUPqm)HB9cL
zY-R4LfMvgo40mr#$v>8;^=#rqzW4U!=UF><D8HV+d{sckR_W7o-6xg)%9!<dlcZ3?
zUG<Hn3(w3Ccd!2_K3^@EL9zGW6L<ChJKpb+`+ob`H07JJmMw1GXL)NpgrfOPbh!(k
zf0X;($^7B?>zZ3zH>pqSDt>ZV_}>hF|38KD3lf(;=i@hAbHI6lZBYGfiL477F4-^#
z=rc{Q^^sm{8m#fkc*5O3-q+?O9j}~do+p}hdgI%5=c3bQ|J-u=c%J{EuhTXeZ}WI6
z*0pliqkF9kzl@EI@3cs}T)6()dFf7Dv%ck9yIEG>W&HBuWuDHZCy)H=N<LZN-}C)T
z^nSsvl20~;${!!zuRV3^oAw^PDQp5K*Kany_W7_p;}prWx+QNGrY_s2c=eV|oy_L7
zD^%QHa^2|;&SsFC7_`b~<7bbyzBRK&t&CQO)T_l!Yjb_~re00<LoIiL`n08CtC~t4
zOtP`I`QqWlZo_*0U&M7@u3XRKN9H8WZ|k4?bH1AOBK9M^TZGKdhV3xj`e*C^5AW@3
z|2OM)ZtX2s-4@z%yW)G2#+DEJ-mI&Su83!ni0f`zVY_I$AVZ^OgV`>vpgmgpyLYG@
zHx}7Gi{FibD>rYUwn<sH%$=>`zVRt{T*BX4R?exmeDATKkdtYGnz*c1c7K3-`I1d9
zx}LubxH++FtyHJW-bwD0*6!rWl`tu=6`$axDH$tzO5~J`tenxVUr{#azVuuaZFpEI
zf1vTQrNs9PwZ*so&%N^^@P)wQ)q$s9nn=l-tXOp4>hP2P-~WEhn{CnXOmVTcdEuXr
zwhecVB+fA@=UBkzn)oh2BjcSwvhu%a*Dcq4xl?<@m_a2bsbuO*DJNcSH-3S2u~$v^
zu^jl((=PL8s^)9WJr8^XT+be_GM{z#*?NunWk0pKSABZUdF!HxP1NeD+rqbcvsQm8
zd+fy5{>p6PwazJb?_Io;(DtbRJ-@N-?@u4N8S+%NnoVzazjjQvqv6D_zD;*CG!|^x
zdU9!+8DH#uUAOdQlb#f8O?W$X>LQ<ot2v#u;~LzJlGkjkUO2U6%c@swPcAv0)IIjo
zGii$ZrVCrWHOg;K>6_M<mRY0G^0hJB@5o8rtFzuFM=WbOmU6m#-EI5Nvo`kT);tBz
zTD&`YN1#D})t~h<(w9yuGQ1fR%cS|`tMTFsR;%Y3o==&5wZg?ADr2tX*?n*GTpSLy
zac8_bzw>s&Z_SKerJtNfj`hT}++X-9aL?&!ORiNqTJ7>N^_Mw4cTHyJiG%BZFbT)?
zuj>ln&{W-B-T#ztGH;O8n`^OG^!gHBw`lL-+rns2S@-4TX8rmfS#{-G&l|k{y5oEt
z&u8my=H<+qjVqK6-;KS~;lv&%V!TTz@V%qV!)2AbN)108|65bV6kuno`S_R&(-Sqm
z1|vU-{HcXU`F|uXZJ(Ma95v17)EQm_lX8tU(JL8tJX`7%*v~Qj-kgk^nr+z&l_e`~
zu2{Z(LYRhzh-vpR>jS^<FhyU#_^B$Q+1BImUunN3InMd9T2B;=jYPO6%yiWGzDMkg
zlIOOm^5!=-`YoybC}@6_ae|uo_t^X&F9aKYetzyXlReo}MeY1)CrJ}IUIwmIv2ra2
z_K%JWebW>)Q@;nw#l1e_nRe_0|H2DVFS(j_w5nFu?+`rkQEgS8;8|a$QzE=kt9A><
zsJU6)N_cK$xNBC7+E?40nXCAwUb@I6{M#zg{YLqf-WB${_c>WFI%ikRoXMc<&JcBK
zi_ghklUaRT>-ppVe%X9@?Z)3`_om)pT9dn{+E7I5z<ax`M+<-TMt=FS)2g;|hew>~
zts8S@T`%!^7jQC`iSdn%kWI6D$QAwuE{B;`If+MWXYJmRa(#78&5;+UE``f#g}T0d
zqEH-O7ClEeG3Ke8==y27br-7sY`OntX2R`S8F^V*uDNfH9kOC;Z8}sH(l7H<p^Tv-
z=*wl<cD^Y=_kZ4dzNdQmUTL!zioJ3>$~P@boi}BpwBSjpSjG*t%pcwQ?cKuflpT}5
zzj=yigV8!Em!*uGE_AKuo*cTv(|TdrwhJn6KBUPB==PLy7v0RyKg$!d{o^^FS7p6x
zQ{O%_YBp;PJI1+qHCyRI-px{zzvpFpX}jLYS{__|?KI==+n*gpS5CYZxi)jlofew{
zmWF`Utv`7VG+usr@v_f(zU9yMy*9jeDJpd;+lr%+Ouh1<|L=6Ri1+d>nX^$-Dt~*~
z=jFR*PI6ZgS-G{O{UUGkQpOggt#8l#-kkls{E9yR_1|}nE?w8<wEAwu21}Xm7er=E
zSy`t1VK?LYRX(QMo1XPq<%P^n({;Np{{F|0p4OOMS~^C0!K=1BoY&u59ky!9rS)-3
z-iEw3+xAhkZ!KH4^!uQk8&Vfv*q@&F_-A?D%lco{i$CYhbi4LSKtcbKYnT4~zyH4c
z)Ly$KNKMGfRH{Qs@zSwu!Btl4%-H?h?JBtzTsoz=FobnU-?D&5p(|I-uk!gR6F%GL
z{JVAE{HxP#w){Le!7Xr)cvfjlZrk&}Kj*SGw5c9ietzGdTbI8pE5=RDnlekH?|)H|
zoz45>-w$>1{7BSP?N3p0UdwPpwm9fx^J#znIesG73k)VHHfZ0~^f@G7dx!Ul%gWv<
zvJCbWWfs3bi?2Wa_xmz#Ii?kD{*V4uiXLT<5o{Nl*TKk>ak!gV-sREhpBh5Gr^OdF
zUJG|&F1!BZ<L&Fs8;_os;B(twX8QZySCJU$#s5u{75LVj`S@0I^Lt*m{bkv|W_Rzg
z{PZp3{iM79_s3~`jXCB1?My~3Yk~>?rGux9Oc@sCG+cZcX~g#=Wo22>oOf}S)0?>t
zyqo&@snyaewV_unpPo{w;XeBJ^M0MPcdy#rz3brhck<87_ixPa|5uolxN4Kn!-wln
zsr%QJe){9Fl9NHxx_85Q4bz9!hlM{jn9cm?6v*Ha%BEndCe(EO@fBHy?lToHC$2i-
z;*`QQ)l)_=+$H(=%&^NI9g8)OJ`!lKdwb@+&JVTc@~yH~N_k=?ePZ>Ne;;Lkv@rA+
zd$##kv{9d!XsmsnjBcIJ`8Ii1hG+Aa?(Z>VSaIs!tC>z3uLbYzy>j84^3RL)Kjznb
zIJmw%>x@_Z+3u~qxqr?c2erTFcJ_wd|G)pA{B$qhuPOh|c4RL9=e=3G?E7B5hRsQ~
zay1`*Ew9%+6)xQK<X5+R{Fi_G;{TnPd1v+$b!$8RuSc)yTfg&PHR&6h&C}Ue&-Kr_
z<#Xa3^MmQK+imR`Zb{!*U*Tn1^Qre}xBPc`tIy8=zD<|?e*ABH;r~DPRGwE>Jo|R*
zjar4^?kGO%kF&3y+dpqlLd6m5%=XJqB>KM{H_o@xdedZYxrJ%&>)vhc-xAheH`Ze4
zNuPI5l6lAbj^{sbD%bz7tb11W{=<~;sXO0u9Ip-#)qg9{AX~Yy?&F5vACCTNEqveo
zNtD0x)Zur|*6Q_Jv?Kf2r+41zy{aH<udgk)vujydhWPe_!9Q-BzZZymzV_&C?JcwG
zH2LIaJf8mO!Nk{hZml-&f6-`J7b<Bdx<_HYfZV4}t?SQgF5Pmzr@hau?1tUX?eES+
ze^<ZHStI^;)8j`M|DBAvUd(q=?(5p>2WNe6+}>Tgef#^LWjAhrx;Me^nN{SSjpk*h
zHIL5l-;a4>f1~n;`%dZa=PTuY7w_1ocUb-0p9|-Xm7BihWw81GY_IjXFCQlVdA;tz
z+RwT7?Tr5f|4rNa*?p(<eeV4~%j+M{{(sY+Q|>6kS>NcUoxQm=Tfy7cuJWIKzw-3+
zSM&eP|Ifc)u5R!DhoAKK7Bdv;yw{UAa1E@j68iUPef-SFN8^{w-~aEjquDO5bmzJI
zR(ZY9U#(%Mv3l}OwgdirabN%bP-a-a>eIf*(_R-dtoq#;Q-833+K&2pI`0@B2+G}3
zXMZfm@J;*oN9pR-^6_6@ygTM8eBt*=&u_hZuWo#OW?zNruLt`ktCh*g9s65+Py2Vs
z@2Skc4i@g6eE&xD+zRXOvgPUJ7CYxKYg~VRGq3$pMf&$!w@QjE>!;SF$xq0T3oko<
z%RiOPpP_=c`NhA`Us(cMb%flOhp+FF4*nPW@YLKO-<X}Buh0CMes-olTiyRQ<E$W^
zWc`X_zj^;Q@B90|{PNOSOCBnmT369AyZx}T=f<8*+Vduf=qJ5TtZp;elXQw}^X@(8
zR(}!g_tAT9z5DQU>F<HdpY*@=-S1_WeSfl<#QHy{824)Er^oMEwD;iOAIkf@@4l1W
zS1|eX&c0LMcZZ#<OMY5&_>3l(o}GQ!-1c{R_g`)AudRJo_WsX<i2tV;r@k((X8*L^
z-s0NZ-}BqQ?b{k$_p~8Sbv`4<qkR3z?;kyTzo;uo-S^#b`Mq9t%zNc6ie}b)PnB@n
z|B%_^zmEL8@0G8QuPY7+Q@?j^Z_=UdwL9&e3#R?PSO4kZ_J5!MhyEy^w<pwidv9({
zDLB^89r>v4e*WL=|DUG+`&X|u&#q?QXZ`rS|K2?Qy?^`t|6$?hWdp6vM59(ddGXo*
z-+txe!CA>?0zbtpUFPR+@#|M^dC<ka#fy($5A*kuWSDs2_*6~%xy76KLCxJYE4oiJ
zFF!o15H#IZxMq^eu}2d&SI<`qp2y+gkjl1jLc^@kswV+y0$E2_I`2Lime)Fo<@u}K
z<xjT1uTJ{2!ouzhL(8OKl`lU!Sqe%jG?p5#K67HC%Y?OCYd@Xb>_5Nu<YUENi>byo
z?=Ro&RZe0s>f@el5_y!jVXk}s{knDcr=8d=`%5irqw;TWS@tPDSDF~#^}YFAVtsi3
z=O(V{5vBJeyPez_d{$N_1-~m#K6QRpp8dX3?d{Jt9^QWL40~y|lDyyV1v@$w=L=ic
zDd#<tw5;Y>a6w$|t>`Z%2K687O4uFNo!zucEd1*B7SA7t7(M)qcI|#xvMs1{%DKtv
z`}58$otkp=W4qF;&|eXmk6jme^onyjh-DtK6P&}lW($*mechU=Y#X%8mRAM2x@j7h
z$;f#M9od-F88<U)DbE5K-MDGb4}aeGdu!sXm@h}&zdiFmGdV)W>6FSuhDQ;pZDw9K
z*7vREJ<so<={kAq@}s}IJFA%z{zX;=9CZ?ys1y9uE$N9z*`^6b+tLLWEVi$dywCmr
z;QwFsh5v8=|GZZ5n%TZb8{fL_=q*?K6eIiJ{(tF@vl$<rgbO`a-<)%c*LwHe_*H+J
z6(T|{wHBxC{CC>FzM^K<y6TWGV$;2gmtB$K__p*hyXLgx+>gE1)a-DSdvn_EWXaAq
zlRc&%`}j3<{q$#*vtrh?&(^qidfH#1X6?ySC8kJdJf6PuQCXdBRn&y{9X64AQ$LD(
zYdo&5o0EL=_Vmhi=e4<9_PNysiWbXOG<r<#?v$)<d@*^((;X|g)Fonc^wiScc&(YZ
z+tDt#bI<kf?|!TgIknr$IO6Av;0Herrnk4Jx+tq_EKV?Qa4+Khuz$Xs+`lL4*I#fk
zluN|$n{PXr(7M!llm6pV>)DFVe`LQCec`+GNi%cdoGR8|k4ra}?6jS4|L*)GSJM|I
zysI43brYIY`}wO*XKgtB_D{vZx(hR{uN42;`JvLHBYgAKUQOHM><5%X-!VtHEZY4|
zd}74fKa*V2OKq2OG@aw}mV9FQV|7&wn@x1JptZlPUbD?{hpwAd``c^u{+JeI`#(Dr
z&%OWokIIB=O7Yx}60bigW`9!tra<?AFV9c;#p|*PLUm&8V!zIOa(TzPlPd3GXFQia
zE-z7>8CGpv{Pf4)n?<%fT~Ccl|LhT0_}+WMRcdD3y#1@vlpih&tlK!F^<%>3%eFR^
zwmeO{%bU+U;m)n8H{9|2AlvMcncrnT{SAKjp0j?7p=3iz=Q*jXSub4|Mg5#TZ~x1$
zXSM8KepwYdL2|!;W8~BHWtYExp4mAkZr=AZTu;6f-4E3Fj(YBP>Hz1raJT&tKV0S3
zWYj#Jde>%O)8yortWEj<COx~HdY^lVbiy^~tUnGJ|Fn)9b-&9iQa>J~`;c|thm%is
zN_{o6jXBf&V0zlU6QybwQ;H=Wf7KOSjb5W@E41@tdZk{~&r?6N7gsyAExO^eZ_oYv
zrH>N!)Gt?7oxuEao8v>XeX};MRl0L*x5&T0?;K>lEM(YoV!!I#-};;M#6`<IbQIJl
zpI)8Pc<i5-=Fw$y8McJ&(kgo_J*oGMME?p+lhvPfvfoH`U4FuJpwH*}4b}@sRjs@0
ze$U>2Xliptk6gu_UB)d=E4s>5)+~IKr&Hm_;NtX}qqn?SH*JrP&#ag)FETY2Y~AI<
zvwniJ*G$Q|YT`jwo4r~WZL4__;I-FcPoi4MgOZ99Of?KE4f#CHQlw0uH)J@NIP8f3
zzgsd!`?!SCe7zkx1uGvvyM6S|!o<|SPMIr{qm6I0<b5-6ioLL0d}2nm_Kg?I)tEo-
z>vgFOR7hFrYj5g3>F<>tD!+`QTozqv>t<o(xHw~CQ^njxk#nA}yt~k6uW{~oo!!i9
zf4OX0x4fo7c<TmdyL&sEJEP9bwx9oV%Wmc0=3d;dr(RobV%mBwT<KNk3iTuF+!Nlj
zO?vWpo%1EdlTXD(CGRcOxaWOFcwfg>UDn>d<>LC%(Jg0|thAD|+`TX%d$mUB@u~Z!
zeNLZxPD>+o)3Nl#v_%0OM^7BcSbFx1i^276^P^@5WixcYFudJ&WgB19lIGPfCY_Gh
zx>C|ERIpQ`x3hC=X2!WhS*Gh=`sPeMaVF6vAGjR;Wr_b5nZ{od$`EyGZom>>+xK6q
z(x#s5DbM#<zba_u*^5X1S*?41^O)JbAkaFviu9zlM?(D9o!O|oq~x9b7kkDJ1~KP(
zKg!?iwVUN*yx?2d9vdsI;M`ath6#@&7d&<g5nq=bn{in5<r|$-CyoW&Ir`Ma_0Pn_
zYsbI0U&{!wnkBFMX6jMHY1TPsZhVW`;lp!&r!@CkWu=3=k33k<yWg(j(HA*8whKp7
zMc#k3ak->ySUJz-V_bch^Xjsl|1Jh5)gG|yTfOUkjK%>^Pfc03@Hmm`E$!0`pG6;D
zc9`u^gJIa|ms)(RZ%_Q&a7iqm@kRgYZR<rF0w#Q2ll3pq-ZZy7fZ+kp`U#396NMPI
zalSau;Z_;sbVA_klr=$rB{z18ByV!i=D0b3;*u@)i_a){2zFH!vZX8(2<qyoec1YB
z;oUE~t1oRR40_<4ma<d%caQAdBa&esj{i+$^_#j)xOIE+;)iyJ{$`Xom?X%vFKX<l
zQce$dV7d`dyL3s9vf?Y9ZBsIf1iKH<j7nFz{6xw4!EeWG<DwSnPNQC~wJujad!(_s
zW*wCia=2l!d)wh|g{3tD$K@uf9DF9|B`q;m(m7;yUt8~nl^)sQtV##X3vRn_zpzSq
z{oA`YU&&t3T^v}Ua{JxxVAGR5oUVJL-UR>4D*tg~BkxKhVN<^^x0EkhB!B#up)RUk
zT)TcoMOEFeO}QOQL1RB(A&rq1Wg%NBUYED$uf+a0YiK)o$!q4xCq>1joDN(Xy=Nrc
z3>JDw+P_?taIeGG#@X>q)W)3}(!r5ijub~poqVK~trzv4P4D`RrwdKf^P_bpJ$`$v
zDZM~B>3z<(<?jMK`^slZES#*T$9t4PG%ETU?}0p>B~w<5a~*hme53S2duNTTuTz8E
zG>@i8vp;8_xqDO9dXFnvqS;sU7R9~ot(?3gsU|_yow-mT;8Fi?Z`tXEOb2`)8TlDC
zFYRmfPL1JRojp-_wm0YZUHg4+KUkIP**b0FRB^@wMp;%HJ$6cDYAzK|^fdB#=o*#D
ztT;We_JL2BuBVrf+{vRJpIuM)Y~-#IG-B3ws&|WHS-Mz2%IVs=&29&0O*OwVy|kVE
zfcx?46HjD{qf+M>Y~Fh1;~htblXnjGz4`p|5u5e)#B09wTy~%HRCM}}UY4DdlQKcl
zrr^DSm*3=W=H<mRCHC3>jOv%MTpf}Y@bdAk;;om?YCe5@iNVI}c6H#MmkO#J2Tlmq
zL<@BqvG%X%>NLD3pZA)jC{Kj_L+!Z>T{{!!DapQ_81pjSCw}YnS*tw5G~;qh#eRL)
zf9@`NG@e<YvvTIEGY6CxZq?kfP|TNmLD#x^@pV<UwpLko7vDa<b<AuZ6F3u3f3){H
z!v)2at9IGFPTcX#e!aMQ`{c}HKOL6_u2{TsmdS^Xh9^db*8=CtoL64BCGm}D*Ogw6
z?8U2_W@;@ImV4eWV_#$H&b;$hiUfmOc>kkV4y|Ba*|n>xT*6t`cg=Tr<8zjmy;56N
z%h)zL`TM@tl`c`*ZpmNQO=9_A<eAfNY3`KvYI&5#m+$jq?<`tB<JE>WJyQ(T+3)wQ
z%R9C4_N2Evo3)E=$|v}X=Nz5sFP<8+Q94~RTt{>}+v{sCU&JOnnzO+D+B>0c4~7J<
zlgbOHeDYw^WYAF-(R6F-5|I>G!n5CF(g&rM3trOtP8=126@v5XB^eu34yLyi`<O}@
zvt5|v!uCn2rLSdzB!|<d3f?_DPVV=Tu1ka)_T}(hP+I#d<U{Gq6|R!%bM*G;eoimB
z>UFehg?Z7QiJMQ0trtt)xXfB)#fc4jt>e$?>GdyP-TYCmVY=s2Hwnv04?CRM<t$UP
zEL9jdWSL3=x1YUO_Uq!sg+Z^RG9`6A&-k!7{*}D_A<OOmrHk&Jnn4B^k52Fo7v6Hu
z>efs9SkYChqNP6gFLw8zJ&jYCLHlpc;=8?Nr`f)~?L74P-H-43h09KU3^I1%y{^Q)
z_Rfk^K1bRgEMYouGUmAj>w?osuV3x!U0(!RMs{!N1%|l&><rRtmlPkibI`nOa;(wp
zvXgA;v79a08g(M&M#-nX99!$B;_PS;qsaQMitT`rXZD*<7XInKUfsU`NA&ZR#21HU
z7f9b)X%^SMU`-Ii%T*83`HRzXD&D$@bcL9&eRRWjo{ayC7T4H374Ef{pCt9(Re$_T
z@qfXemd%aU{jXADPV`qgwua3-_~&PFb&OAZD^uj>nn&#4&bx%~IG9ivEpd!}nkGwz
z0^bgEKK0WU!Qax>KaG~1t~=q;9EI=xYy5YZ)m&9s+G-Z$m3Q9Ac$Q%3wFm0UmPzD@
zY8p*YVa!<SdpUR0F@Hxchx7*bpiE^=*5~^fJT=Q-a4%$>8+xj{gyXI2*9x{(x(}q4
ze@ydR{p|Uz&@P*E=?DD2AOCKVnX#JhW0=<JjkW9stGgyX=Je1PT3grpb!&3<_kh)h
zyZDY(>l~9hnKvuu2FsS9RhN!CgdBdS7I=au?8Sn+6@gxvji*nm1m1F(5aczf;bQMu
zD@LD~z=cy)H{N3VB6eZ><$5;jOBbhiKHRb(=kEiFw3}{^d$yknNzUiImSFl;m?1??
zYfGBYY30CWeYQ8Wg6^*=d&k@9t-sUsp_ghjTefukS@1Zmwj)D^8S@19*|Xy_!{TFA
zxf?aNoa(W(4w}=ztRbQ)>}s=pYRX6Xj{!@%CnkOkoxPqZp}2nA{{{B7doG=yWqSBr
z{KfV>mpcdF8U;$f{lt^IXtfWcz{CeJ{LK1o!KQZ1`z(AI-W+S1c<S+S|Fo%>qFS3Q
zIsMoeSF}$Kdz|>2p*d*bgDneK944uBmT{*qI2yZELMPAEB9OcOPWt!HvNz8!SDz#E
z<@onocc)nLS8&JPb&C3XC1Mkou9kL*N`LWhafYB%W`akrOc761S>YSmy49vTQS4*R
zv{^w`Nh_zR9PIE4nq_Lv&(NPTG3dn!rKPFaHZAAQ@Sg9!w(-Jwt7B!CogZG?A~s*c
zPij$~RPE<2+dbBJ@G*GYOzb!>;eG1Yp;VQdT6?>_HA-W0ofu~sE{-gGe#&TXq+Wu_
z4om*v^3dAbclaE)a-G`tPVh=wKq}XjT1DR14--t1)%sJcz4HQ=|KIU*fv(cUt1X*b
zzMV<0I(FA4H+W_DX90#e!VC^J)A(k!1-6>y9XZV9%jmWAZ2z2-Cu90E-B%t7U9Z0N
zO6>lp65BWS&Myb&x1|!7`55fq-kD`LGbV1&sVy_Ln3rC;IWg(PBeyA%i?<3|C9*8m
z;&Q#kF=JIgMzl?b+V@0hv)dWl%idiKZkBEHE!!sB8PK7`s{Z3biszbF-5vp#IVWyE
zQQFkAZc)Yg-V=JK9aWs!-Q77EG#{^FT5vJTWrn1TfA-g@Q{%oKb=TSJ)jM(1Ggl*J
z#*P)ogI-rL>$0tWnR4g+^6o~96U*0R^*{IXJyWnno25SNzQgq3#vrB$mqqr<3_Rs6
z3L74rToGs+C;rQ@%ZYjIw6mrs8?trH7_R&gt4}?yyDmwnTezu4Z05~Pd!~!OJR_lh
zsWFT}OqqA_8Xs%p2|;(RJ@`L0F@2ko?&>e!kAMH+V`N&Pt@Lna$<fDoOIX)Elzn_L
zpn;o{Q6_x0#=YjC%oM(3)jmmQwU}Sb+M4wsaG~h$1V)azvrYw_N^#T9T_Pf&;+b(y
z-Kc*x@8pP^TXh$)WtfYlcB!rO%4>0q+8_8qf01+QLec%Z44Ri}ZJO!zGKO1h=7q1z
z`zu!(U0tQbzW$khU*O%tc8s1gt>*YYznj2uqu~74?jw6W@BMh1wRH96>FramN!Nak
zyK~HJpA@7jY+B9n-2DHXI2l>LWixw<H5u0KD4MWS^;y@9SxJw)tfU?ZGAs?5`RZs#
z&gImF!f~hc_!7+KZNDhWtROav!9XPbdxhAPP_J;;W$8Tcb;`F&bF+p1IrHNDt0-4L
z{ZALTf-5fPxdeHwHCf=yXr2_~8o4x5GIrSvoBgpiGbZ*VvpSrLI5p$q)Y_u^4PoX>
z!*dJ|@PtXKGZ!9U(qV9j)w&UK<Ep`HvAN7wv{wJ?u@Pzwdb8mGr-f%`T9!n}d#wjo
znaUcr2J>dKD&z=+I!E4Zh}<LBmXx^JtWRmF07J*;RmO|=a!9T?TV1Gh=GA@ii5fvI
zOx}}nUL4Sq$klG&?zHaPmoo<zt=?A6I$e^fLi<e8V>Slm^y_;yE1oYp)E9ECP}x^6
z&Gc>i#5Y=@y)u@oeO9V7>^b(QNw_u6gj@EJ*hPjVB7Y?lvTCOWb$Jx+Pk(S$><UA)
zRs%!qk`GE5OBJ4kmT<;Iw|qF8zv1%H)P+;mc0FC6qpQeJR95w6O1;+|-YBaZJDpd&
z+sMY<AT7AR*P!#MAfv*hU0SEEJdD3^vX5WkxTeFN&%B?mZP!0<w(aBRKJZc#A*lm1
z%Ge$??CxK?`~E}yYtMc@d-X{xQe)|lji)XNC{3KX!e{FR&iy9Mjoh494$X}!IrP(U
zi^R%o$@iOAx%;0DZlBLPb6)9+6vnW2uk3vc3~v4*YdcoGaIU)%G2>`x)Sh$UsT(im
ziM-F<vRgU0$VQ3t$}8=-+s!u^KNziisU5RAE9>RdAkW$RR-I;hy~4Pefn~{*xf35>
zUB+|m{VU$<tDk-J?_DDMZ^wHz!v>>u4Oah4xwB%Db=<X5o!s@@wq9)#zgv^T%~Zn1
zwQzdKZ9N0lt8HOsQe3OjHg8H>bTsH~s&2{VE8>%<oSEFZ$N1~v2xf+o+{IT5GP^xk
zG_yanu2pQ9Fz3-8RsG$iZ)RRJOTO`J;|;^rJ;K{Ao?5PZ{(Hgv-5K%z0$r@{&4Q9|
z740%Db6@%Ju7st@YLkYE%0`d3+DIMP7O^#yQ-0-zRe}F@3P#!+FbFJ>Tp&?9*PNxW
z({1mbAci{}3mGh}nS>iT7cb?T>XUTST)6RMYSN6$LNh*`y>nDGokw@!){A-z)Sp-V
zxRJ;kwz*nV_<Y8?oHQQGdD5YOXR;OKPh#G2X>LFU!@8=q-JYpe!rHd>u1^4$ms1z4
z>0UXHVMUKq^5qQ{zdk;EKGmzHQL|-BK<Cx))Tqnh-`1DqE?y8a^VPI@+so27sJ2{-
znIf<9?~7somlrEV*M80`U6B%`V%#?Wlx>xkZ0^J9y0JRJ_on47SFp@JrncgGyVoR3
ziFGIQzsWRwQ##n%b^LO-&-r)jvgVXT1g;QumiIGY?072oS?T)4jy1OrUg&zcx~A%o
zymZFYf{QW6HQkfu+8Z@}RGN}`<XLE|S>6;e&5r?-G=e_OaL&rL%~>$j)q8uB=NbX?
zO^wq|MY(NHugf->q_eQ(NNVSv1@2q-)m=$Ac&0$k*Gn@&MEi^Fq?#pj9JZuOd@X&L
zYU-bP`eAO_Rr&k74<@9hUwqMEIL%an^?|FQ=c=36mMt>q`rfztY@SZvYTp|w57M%1
zSBp4o_A=}|TCh!R<CY|a&}9t@H-DIFEL}21`S~sF7US!47U>47tvhgXU)4me$<Mpq
z>i&p6eJk$r_O(im*0qac?I!q~H@l^qmv(?LWQToyA&*13|MFE_u3T6CRvIN&{9jtq
zkuuY&X0~Co>4M(1&2xM<zbxLG85)q6EFC`)(hga8*!0+z!)}WiuHAkAX5C`mFV9{T
zCGHDg)3OW^<(zBqE763xLHoz4km7s2p^*%CIHII~FFRMHVIsCT`^`4tf@>>cL$55H
z<SQTX_MO{@j#FF2mQ|?Ce6)#gk;|mLtqd>R^owTBU|TBK&|5ZhpVFi=`!+n7J2hg>
zj#V#Ld+qmG<#XOi>g;=+8T)SavMyms*_|VB;Ka(gx{EI!oZa_UH>33YJI=`wzTS3D
z-mxuiRjGxd#o>C*>YChZZG;3aN$@J3Hu<C!yG$^gW7GMjr3*G)o1l0_SLk%z?wn6;
z8G(^D&8{J@mI*u;i+TK$d+n=<Tc*}bxpYlMOw-e=d&(v+QO%9rEaEv#1)dvzJ1^{Q
zjnv)a`c^h!#!>a$cL8>`JC7zXcC=i7w$ZKcxL#QAnRimjH(eekHdniCieAQc;Kao4
ziO0{fuU)9!aOT6DTWVa49-33kHlJ#7O7T6JQm>X^GJ8+ElR)RL7J;bH5cMOyTEPx?
z4ES7|ms*7wSzrCfb-;zO!I$sWcQJ<PorNDx)vsr6xK@7gSLT+*9{G2K@0|YQRep!V
zOM2;~vsY7{lM-iZ+&LDmwT&TL`n}f`o@)*=A#O7t)iO-KYceyQLBX&sjA6;*>o-y(
z4xi}UyUgjJ9;cHg>ykh1Gt?IeE{P5lu{RF;C7I6l^W?;rl9$7C9-QN!wCmrUiC!yD
zFIl!e;{0xD?sGz))MglPw{Cc~x@@y-M78duKYPNxHvZVFnXA0u+~b_M6(&1>MA@(z
z*t}o-^mg2Q&3EmW4brdoe=y&nowhbg*rXwG>EFWntu8;mv~+4m8(uFHSi3ZQV%oJ;
z7mg`MYJ_IbcC!+3dg&1qwM3w)>&vbS{JdtHb{^W^HdlSO<x<()iDK(c>fT)w9KGb}
z#1u}O=KODWvZgI{&=$*KoDfvCHCa+$BO-{6p{deB<W_;MO11x<&{d{u4)%UK{$1ke
zJ&T7A>t-LH73w<klJED6Rq{rYj%9e<JevBpSt}zfbE%MlZ)(Xzss45Um$W+>?>=^C
z`C6+fQ{tBA^`?EbYMy;zVkGm?A8gw~#3n~8ZV^o|%-IzxapT#R3mLcH_IE6fT|ImM
zt=v@{7ga6?n=5^@Ui)33VDYMRJKyvAJ=<l#DR0!UI_2T(?vTZXOKu!9+ouI?e55Zr
zo51<rn_*SeQvb4jbE<CnOg-k(_*LFaJN<L>pN&4EQruq8Cg0-Cz45#2rM5vxRoKnx
z`3!eT+vl6_Imb8E<x+P+*%F=F(#O%qA1_G>KEGndN~Pn;{N+V94%_E?>@4LJue9xW
zoOa$c;CVw(vXQOzI<uMc&c4^#QDor78g<f9J1716>MJ4#qD#VK3{EWPYIwT()5|7J
z?fRPYYJ2W|N!)%(n?a6KY?ZOYlO@q%nx0OYlO=^Sp1R)3P7q*TX7)Bk@k+FPn|-oX
zROBJCs7v1>PxJAWMLRAoOSi1~t=$&Svsr@MOQ_Xsp@1<ngB~ZxOSjVHHvKM<w`X#h
zsEL@>XRVcq;tV@mIxGLID3gTcJZa9?%}?D}0`C5DOL!tM#i_Z_xzTOX($}jmPl~&F
zw?H|A*Ka!WjL^3J>?bB37cw^VuP~dGCAe2Qwp@7vpN`$-TW^jOEsouHzWztnxz@(F
z8&tx?r`s(5E4cONTC3O>b9m(C7*5=o?xdc&%2>G7$z)j!?-{2%M^m$U_U+hZuHt`W
z^QjxvFEh--xv$RX0~LUf1vADU)tXD|Ov>8lvsyTDNvIkmKApVPBVqG}Y1&&@4%~2k
z-7(+CM!G>y&wUg30X6ou@4AjgJb5!^W!lSi(ic;cc*+!3T=<$@+doe%DpHu8Z@pvD
zp93!*O==9umP^X4nJZ%wm3_3STu<)ItWEX$v!+%Zy7i5D`QeiXA7=?4Kd)_+o4Z@O
zGjj94D<T(9>}rc+pB?-2vE5;v`kH>$h{Il+Ckr$9`7Zhz(0(+csdhup=|cqp6O#-t
zz4;rv-Ag27$%iE=iPZ~zPvmj<@+x#+>lP6ZF$-Ii5ma$~`aaWtYL_;xbYA^5%UH6k
zZrP4KFSaeaH*4zkOs`3<T>6)Cxb~}vomMtpHeX62H0$yWt1AZHnvW-Luob&4u|>D;
z`QBZv8zrJ#7c;AEoW(86%ptquqG`rC9-~R@kM}-)>6)gHI%gu!)>)xyBCTtkSRYKh
ztahp6f<!MDgYv{C?{6Nj+9oq!_$I>J`ryohRUN{usa1C^r5XNT+xPcu{Kob289V0~
zOjsq_tGoUCwqF-MMSNcDRqlWIQ{!A+<6~m2AJ)5hrK{}j6FZX-I#J59DY|3Y%PXPl
zPajpedP8gDwvSW0Ann9oV$DnIxEMSaF=#A}czf`ade+RWqHF!Tc^7HwF*{5tdbmB;
zE<?lbYyjx6k&SCb(m5|igul+Zv`WRj{S^DUwu^GH)&EOYY*FKjk&DP!yfta|>eENo
zY&`vwZ+EQOws!$HI~or}-|1ytu$jw&OF=~A(#6+Cr@y?({`h0ZEuQngk6(S3u;=(5
z*V+4|Eak%aw!OZR$66fzv*e(`X{oLLR~EVMx^nAVneo9{%MDFsk3LYokd?XnpXSkH
z3*|UxTk!-LL_8EalzRHmsqP7qy|=u!$T=oYVcjE}bN7^JTYis8ah6$=`(uv#X=}S~
zsXIk0uT%3d-Oan$C_v+g(yF8Hy)?hAeEw&_(vKG&ysYwHH&@3Zdih0-dpgIG-poGN
z7aTmJ<Y?zzCEqoY><pT+anUP6(+VDxIIW%Hwd2?ZAK$b++fViDJb279Uv=Y~Gcn!i
zuen}tPTX<SsIXOm;blX#@GikkUK_)AZ<Sy@(0M4@Zto7yq>m!j{^HX=znPLN717M<
z5YGDFPcr%8tb<JgQ_S{q%ndR7t-s0ZY|k8@%}%TRUO&lCmbU*6X=6At%nN<pXz{>)
z@u%q<-qpF!EuH*xlJy*kHyhfURNJrJs1<o{v@5A~chD)TozpYl_AoT;zP6!hiM#*J
z(_NceY?u;s%5Nx43^ir5&QmcCe!X<ZUEag%zinQgdN1SUYKh**%niL|)nC5s+uglO
zB`#h~ZHeG@A*QI6r+TyePl&fa&u|H!BmZQ}McrSoer?%1Q7p%OG1sni$u)_g(i3-g
zUcR<1e!aiR+KJL3flF8xOLiBe&%Q8m+x1@eOF`~$s}6ivmhqP@DvLvO+19+Hx{tL@
z<kw!%%D%;KetOc|s1v_0$aI`oSE(~;ZF=$BqgQPnWS2-RoIfj-<=xxKl3&t8wD0sL
zRbLIbBHMUxL+$c21+I!W%Wl2>+waDH@kPUFPWQg&u7=gS&A(^WavDuqc%X3WOw;K*
zj)q*Ey>`y}{mKirZDVRMVx8TW*0v=~r{dDB0~u+;oh%D}rtS0LF_%o)wc6)vA4}8~
zk?A)-?NHtLBU!pAa$WcHl`jq_R*0YT{8z<%;6mH|)>{&%<5(G7l)PBG@33!~8OqiW
zq$T2%z~vx5d8*eom1}1EtiUza^hYZ%vmQ8h_Dlk!z~nhEwf}XW|D+~&v)8qIchRdo
zqC(qFJUW{DIZbZ+g;QyqJ9lkxPMy29V2kuaBSY)2(I%%tw!h+CIkP>YT(oP;+>J%b
zNe9KepOuDwXg=P%WAWw-x2}{e3u@y#C8F%!!W5@c@|k0+X{w~>x7_JLADpLL4ga&E
zujRPV{OIKupDnjgKXFHtU$<5?;LyGBw@(xnuilWk=+K;*Uw5C<I_f)XI`6ckC70Gc
z+ibU}LO9Blcizl^jmmE~@WfB{Z~2??bNPm+EuC!Dcazp`lI|_f^Emh>YcGe6_ao(`
z>1&*%IaW?dI<B9#VU0w$@8)~AqLw$Vx^pw2nEUN(=F{gc-876!+i*u@PpIDuhK`_v
zJYfsvjn|(vT>VOFmy>pX%0lh1-Jh5J-rThExKZnrq$eGbN7Xp4aIi!^3D^F=C+f4`
zszuS~b}`&C|GakA!Krm-(&s<AZQ0yXp<H@GYK`p0$NT1F7=-bzWm$Oc*fk!p?@B4W
zGq0DpKd@V}@>=xV->`^CZ!(Pte-^QF@qD}J6Y)*IG$;MMWOj1vp1|E^6E04jnwWa5
z<<ME5BW16?Hi<BA$hju9JFM#H|7$m7Ps_e*IQ{xgZ|l^~g4)UMlXRumSXqc<&QvlM
z3_SIE``m>WZRX6__v?Y@nRj7&S5M3Je82HED@@Z|sf;V2J^0_Z9k(|8dbRAiI%C1&
zU2hXqY|1vX9N^EF{*$p?j+eP`g1`Tcw+Y_0ta=W+dBqnPPne?A5_W3C<|#`;3eP_~
zc9h}3rln<%CQ7RWXUp);*>%uh>%OhZVLGMUiVIoi-AcSGKWp82^BpFq6Yefre<+PV
zQYkFBQ1JA<%c(4G;cuAE`^25iT)OOT&hxds+_yu*qC1Zot+^uNlf<}5PkqgMj;@v!
zE4In?9!)&$C6bzK619wVg3&Q1hBFL0ZgI-7>gPnGT6ard6_(!hCF(O@#=4Vh{qtQj
zK6>unxS`}dfBf0-Uy}Q$Z83cxmY3@KagL|UWGj}2S8~ybi~);PXHEH=nsn=mK12VJ
zMrDQqQAOj3zmi;QyG8ol51%yA&Qgx-&CPkNEdbu*pqj;d%|S-QYp#6V@w$V1M5KMT
zUk%UQYMu37>GJ%-nL)pn8JZi1uGyDVx%+6ePyENjo!75z2yS}2KH}cF*NMNhM4Voq
z=-m;^?|U_+_wL4$bA2M#b|21MpH*JU!r-?2q^rpLofC8RbObE>uxRGBS=#Jtx6Q58
ziP)~fzVXiPt5;@9PTbygM|wtSziEWGZ1J%RzDciL>wfE)p4{y;TlM5E9@&h`Y;i38
zEv{zMcn|sdmqq^ymp0^BwQ=GCp6g6&%<gT{$i4Z%;M+B))u{{bDn)H7-!efmN<k+_
zMkdcUe|8b8Cf8aC9WSq`eLK!<xY8zk*<e*zY9V`e*_QflF^|ukT&{N+cfEZZr@hf9
zQCg%==6%@YgO!(0TVGja_AKCS$qY6PW1WYa>Ufp6_?$i!vRSD6qUX$46P14%KXfa7
z+HPCZ*E?z3Mz!lgRW4VuI(ZY8&abVh&ybAgi#ye1TvqYC#o9kIdHTU?M}JuwN8I3k
zqVvF9AkKNN#+r5IXB0iFo4@^x={fLO=Ik?u62n0Ab0<NgMm><$z>)uZ>wdcDJu;{*
zE&b9e(4G+_G40@{vf!L1(`nbgTbj#1`LiIjt6){%Os%%auzPvwCwelCG{34bI0$%O
zxo8lSqk4bOi)=k>_5<IRr@p#zy{gFO*_n<dwH>#f-#H=JaE_-}=AoNL$<irFB}a{2
zKFTfWX_;We=bBi3<Lb$SY=KwVc=L}q$e8emJ!cnBowK8p`Gt|6#muYkqHBEPXYLA6
zxUj13$KHu2w51MKT#Ja<>UjQ^js59=HM#Soi*v2G7EfPeZe*2nt#WsHVoC4JWzT+!
z-<+SnVEHr4bq~Y$imW%wU|eRlD!cRM&n21J&g-S#@$P7AlRLSg?$TtHjVh5Ds!LlY
zy!pSb<cQar>?^IiHYY~69libNZQG)EeigYlFQ^pXlo0dPRg@Cy4vl0mnzUwb_V4+c
zp&w7p+`TE%>B<(d9g9<EA1qQn<MYIAmZZ)i5ru6=4_|ns-YvhK$os_XmaNS$)3gou
zUX_%3PfC5hanoAc{ij1)c0}w7Y;E&<X1e7x=S;Cw-f6u~(r%yERq-Ymon?6Ucg=R2
zYi4!tz;z3I(!CjfeZHoc@GrZ#>$fGt4}R?!mf%olnYPH7_1iD5TWHbs{lv$JkiKQ#
zjc4t7xb}Na9qaky?q>|PPL|CJnYwb@i>|phPlu#Wt-74F@$90Vf~`tdeK&8bGSArZ
z|Hgu+vhkP08NMImWWB?&(D^`jWN7!Y{^i=cC(HDSX)b5YWasrf{D*y$arFQH*Mh53
z`*Ig;2u#{oU7g!>^lP)(Mpo?uLR(w@%sME<a%Ig_ccttvk8>B7vDqGbvH8vU|2co>
z{$KZJ{lEN|^*@uoNA4=WILm$e_YJ)#*L<wvIMZWy`rn`ZcmHe#tGRvF{`cmu<zZX0
zR_JV!;O3o>#2B{QzU20@L`lX4iw^45@EsPbOxrv&Mx4{^l+w3fm%KDz_Z+=&c^OBu
z`IT$BTX&!HIh!1-y>V6%i=H}*L6OZlp-o@>PEFCTnwyoA5SDh~QgY^kw&guroEndw
z$gW>KC9x;;+CPz&U1zRcStnyI_*rWA9jk!QkR=}$g=8lew=zxcf6xE1t!=N}lV@e7
zH*WT_)~8wr*d||iIp>Dfp{lu753{BkUEQPG&oVvTs5x}Wbsee2Kc}zaTECCMU}lWj
zT29W}H;$b*1rIwo@XSvCGovhE|MDcQd9wRUHf--XB6B@=>iWBv3WF^sNgkGK+oQF5
z$EGL^we}rv6{0?@c23ne7Iyt;l1|#HWm!pAqwY-Eu=lo0(|mypDW~NpU6-Y|Ju)io
zxhSg@Ej#OV$D>`131KgddS~Qpu&TP}xlWB+b&1GA4Y#!|Q?{m6%-&X;+s+(Ve|w8U
z%!Wg<zL|;ES^aDR+#75r2~4}P@}bCW69bKnFZb)MePznPkXC+ef4xzK;XjpPpZA&n
zj{md|y}DX-?SZ8o8)iB>b$4(@CuF33`T6H_{oUW6Hy@4vGVhu9&&T4A{q4Tj-I?qd
z7MyyeCua4U9JUMHCXd%D9X<16I`?L?^XYD%XMNn3){(fmP3*zryO(QrTE?C@%f-ER
zO`&tzj?yo=)!&Z^b02=itN!;5&%?Ug0^OTi!rE9DC^7G2v<N)$X3eUA$q`@I<j$<u
zU|6#D?uDL>7BiIIo;OKf8r|;4AR$^>c+$yPB6f*j^uuZu$&iga+~y0f>h4iBNHo7&
zQ<D;J_U??Gtewq{6W2e#DM<=rn2}>9vR&q(WU?TG`da=2-BUz&GBX58Ol{!1^E74F
z#?`;wR)aQa&Cdsq%r33?`^QLg8hiKR&s-s5)@+lDd)-o9``lHGCcHJ(mi1NHc|}7+
zbGldIxpQVRInUYGa$L7#+qFZ+`}nKen=|Jc%G}G~Ike8HwE6ZdAq6E*6+PkLa;AlX
z!7;{LpUI}xuL+2`9Iotc)|;_KdZK$@(A2wKQ{xP4pFVzf#$nQfyn|=g?KwJEaemob
zi`VN44u22&{kVFYVc*omSp9sHmM5#M^quCaIj`+V>R-*HvBl7Bb?e2CFZ|OUmd{=H
z_xtgejX%Dzz01G<^F_IW+l#aNt3NHDeph6h$i{O#*QEC@E#_xbe9`@G`sSn4FIL-b
z|DBlrCGqJW{r?{IpZ7<`<$LJz{7*j5VYVZyAoT64cdox`{{8H~TmP$m=db<$KmM9;
zfBz@<y%yEIQM0e*EV%VVd*?3S^jsaK6&Jds#fw-O)VaE+<jw3$dv;4W``r~b$?bZR
zuZ3@$w&9$RnPkdMmEF#3Z<L6vdh{`}cIm6&aFb2n&nOsY&UD*TYVI_1mQiRbm$}kX
zk=Gk<G=Eo9y4bUE&HHsFuZ*1Q@+bDayBPeB)1Wi-`PVON<{EA(=#n{~#`gK6;CTc4
zRo`~?GjQBxOj4UY^WdXDb`x`?q5~6!`~<~TK8bg^xQH{|eSP2QgmjsrX&Wv#%>d<7
zRd7Cy(3$GxYq-vD>+7jISSHL$e0$nK=zvOK&$Jn{PW2>ybxTvyo0{}nTQBU956kxT
zFCSf+Z}&_*um0Bz@$_vc^X~SZP+quo)3Qh(O~Zib9uD5bYx7?o?dWK6zkZJUx}@-8
z>8YMm)WmmhOqiCTvO7;@M(hcmJGV{+ZLGG=&lc<KUA0z{At!^m;bQ37nwH<&PqAGT
z2xlm%ERfps=kRKI8QnUo{l%&CSN<$H=wEAIc;Snd*uBSR7@EW;7I@AnIdi(z^5qQs
zl3mOF<^C<a{MUK;@4IO~)}QNoZ*|xH+-Z9&`F{yv|8CD$30LQ42v_g!mJ&XsQ+M;N
z_yI$Pj*I&`-^z(eZIB5%ojRrYYT&UK{qO(B{oY~Ke7^jSVna7yMe%<7-M{wKd~A;V
z>HggL-u(a9BR|@o(BrS1|Np|x-_2QH9-loOTKY?Twf*mRC+DAgw<dhcu9FFw?^YjP
zD;j8R92o0!D&oRK$yt20ZktsjVi^@Cgs!@4!|tGAu;xbS)C)IKw(Ps-Ugl=<Aj)Zr
zl$V*5b=Qohsgr!&8B<gihxBB*xTZ!qT;b)d>|WS$N8{?QwyK^>QB6*@cg>cIirp+Q
z=v=DRbnz0S^r6U8r>+TovPfr;O#67Gw=B0b_qI9rfup}H8_v{5E|c;o2yR=tRcVs`
zWR?c?BtOBbz+7*}3D-*=ta~kgx6E$JJ)3p8Igj&o>Xx1UY_{!VC|B+4&sTPOKTSXW
zTy~-L#2L5uFNu<F<Xo)1E7CuFnUu=yt@m^EW~qt$y!jyi<z~e@|MJ^x_E)03yp~^Z
z3so_Asbna!{;S^Z^vo)kmCI)73eRK=;+@oLHof!a#MlJ8d^tJet@nCa19Njz<qtbu
zC`g-qaiNd0z?4-IGdG^7oifX3Ra+55L&Up<Q)Q0a{?zOfFFU<)_pf{LZ1rWILSn_3
z;{QFX|ME}x&dn#QZ7&!+Z|=LM@!ELZ>C-1oZcB>W+f8R^I@rB<_1xv(A9eHTtt%>F
zNC-Mte&^wmIqz*`;utG#uzz^_@E~{Tp8z?A7nf4DOZgXlWemOb^K$*aqw}X6_1U-o
zwsmE}|GoWX*Dp$)(apS3W`4}bO=F9r@<G9-rq9pi+4lc)stH<WzrKiN)$gy>^<G!)
z85G`q2{(GyyD#9lxa^I;4+XU+%$rjD{h<Gf|12-2&EA|k@BfFZ@-;R8zb|^>?69~j
zYk>jNg6=8nYB%UGWv|(<lEQeVhmUoShclzYLXlgyOp9wf9||y-7oL(}xTC_BHz6X%
zd&SqO9aU<vM~n0W#lJ<jrz{E0@=2O^<dJ2A$xe>r)eUn(tu7^f3~1x|I_m__sVf3k
zr@dp?DQFzga49t_ZQ{za8?(*}x~gmq3%lI*?7E^X@5dXxS6n8Rd{HgiS#0^eYEz;2
z=XVoDX5?&n)M*p*GF`%2{PpwdyY9s`ul(h+wT=~>_}FvNc<Z|Wosy=Ce&bJG9f8eK
z-62U+Zrnbs9GEdzuy>u_&*Q<8tTs23*5v-FNd}!c>&L=)D=R;^v}UU8d3m40d}i6D
zX&k`?3+JzT@Pg%r>hWMvPxsqeOOA1AUku%PFi(8?9GN>?V%9S(sN61Q^)lehYTnid
zW#$VPOjn!OcTM!#jaIkzQ(N!7y{KoYvtwmy+@Fp~LHm66y-AuhE8>O9lE5Qsp&#c-
z3o2ZCvRUKaX`V|@l2mptY`E-|yCaax<@(ZY6RDGVYA+55W^#F$DXf2P{*vp7?97Lz
zmJAFZj%T}t|EqYb7xypqL;CtT;-4<+HN1N8gFo)S?9T6o4R2h9?0P<??P=dE-hI$k
zEy}J&it)fbHvXrkExSZxjT>jyIN15}|NN9&P@-Tk`<3laUYlR)>*MOn<mFE`|K9yN
zV{Nr!{6q5|<3rQ_JLwknGkkpU`1jx6SKUEs{_pJ<cyRcUbha%shg0$@#wBfa(fimR
zZ2c{k_pH^*E&a!*2%dNQ-n8`H{%N|s`S2h6xl5hDh*i&i*Pbfxw0K(XMDeSK%Dxrz
z2OnG)E2+YAw&xx9&H0MPT343kEi_h^V0F}y&c84<&x<|aSi$5Q@{3)XGJntbuB<Ru
zR+x9oJ|=_Jvo-~LMt!^}>E3M-*0*fR8OG%&r$~9ulu{11o4Iz%Y6r22*A!-@UhG(#
zojz4^Dbs>88ccUO++Ol(SC}s{y6W=DXyrnm_EV>ilq!pI8D_2vTb9oJtKdV%^W>WT
z<q?lg9a`z6ec-9x!nZFi``$eJ^f9p9s^!mx-bpK^xKyOyzB^XEOG|^5VeMHBvF~jq
zM%V2+S93*4EAa+syzo$G>3x3u*TKoM3Rk1JuCDa7sQN5-<Cxh$Ht@34tNgqPi+Pu?
zv-s0Kwg2PClRCxm_ssn(l4fQ#E;^?Z!umV#WBA{h=iEfPw)o8TIFlCf+2%=wrk;cU
z=Z}KUtG}K8?zTIo;e>#%m%4GOuHs>RxyYB1{quGOZgq%RCZ!T_<J^RIZ#+sE7Yfhb
z`sD4m^r+d{t*1Q7S{2H+IGwf1P`b2ho&1wO4>U|Cc<|ml*KGFU-{q3BKNTh{6VCPN
zcBbys{<LnNas7slI)(ec`|qABX2^TKpyq?{6Z!YK4ecNAv6)S0PgrlrSn=a4|L!!U
zWqp$2Nv~X-m|I>xlnbz_7UO#8dY$imhUN0bddDx@Y~(-0!|(2IzvpwWxB6bU@D-n}
z-^|H7E34Ca^zvHAKaAhpmTAAge?QipAuLPNMc-rVT*aP}k51Ks^FEyu5RGC^Q17t)
za{0yQ|Nh^qIOo)VdvwnIM$BJ-i}NQw{{76^aB#0(^7Ze}wbiR+9L0}S2+TB;b9v#B
zv9-gc-p$~Yvf!&-AB!y(yZu|A|E*xjnJa7CKfUa<KQ^05XJ5g!!YfNwPu#qEgW&^F
zv6&y2^oR;Eh|UaH*|S9UoRZ>02b)t`cO~8xPb-+_y6mIc)y=1-teoZA#H+j|Oxaz>
zEpUa9b})bIg@$X(oK-)RES+EZCHnyHW^<LUkHYtF*w>i;H;diwvA=bT#VtQx3G10v
zTgtzuNZiw%_q)wYf%9m^trR)S+Wr~uwR2B5GP*eFxJ}fZU!t6HY>%Gs9-BS)Z^pX$
zBz5ZUx^?Wl;Wnu?xiw##rx;BQ41ZR^zP0V;kEO+bE?vBPc|*;GB4O1#I;ooiuRop1
zalqv&e|^}+sIQ{ku1-=PSDQUntm|ig;1;q(&^!A!*LTx0XQ{Wx7WpJ^Yuw;l!Jtuk
zq{-l(p<cxkMf;N6(uD#kzZo|lntO3!(DnSE44cIEZ`Cei^tgFY`l6rUvn}N@Yz$vc
z_U(TE?PY!ZO!Whd4Zhztl~-;u{`u$Q(bxAY{{H&=^Obqr{(tkIivPcu^|$|A`1|wW
z`)94GGmShN|IhW~&&`c?Qg+pb7u;LTTCJD_mP`n-f5^10HK)hn;)-QXSJ<zvIBh6&
z<?`O<?!EKgZLE5>>;0$m8P`r<&b2?tBf;6F_wCxQ?bmF@JZ`o%pXS)X`IT#}^!0!?
zS*u+)ez&v#Kl=FP-XHgB-@AT)UwZm}Z(n`-ga3K=>NWW5e!iBEvwag^a^U|y<?8vn
zyX#BC{<Qvf()GF48Z{%eMfIdnuh`<K5A$Z;THUE!^5oO9b5_O~v(mF~vwmt5z2<v7
z(E4Ir(znM92fWs(WKQMEGVMO1Vd@oXuxm@&s;x)rj%2<LU|29w`L0LA|8fQ|)rA5<
z$98YbuoC8I=m?5FdC*KFb(OI|be7?|SC1|wEO+;}-)FzYNN4x6>^upE1M}Y7#M-QO
z`YChO_O9*O<L*xCYszK3m%rVgdNlgAxW&xZ)y`5E((Ih-m(AY#Dxkw?ZEo^rua&Ki
zmxHsVZ~v$S-z~XXzDi?4R;|fwe}h@CIH&Xd{H}ldo8_Hl$C#A04`i*eKB;zV&Htu_
z5igGXljYCca_RgUzPa+!3q2&0Tb&MM<Voq4OnNE#{TSnfmoudWCtkR<L-1?*EPJL4
zw{Ex}mD=qjz>t{hx|*rM@D?9qlI^CocC$l|{p#Mf=5}`F?z8+tygJ!O3tsKu`EkFW
z`{&-jUpM#Hzr1;T|Ch$cyDuMJJ^BB>GyiHPhdMs0|9vmtHkS9qTxO2&Suy$^Aqz#$
ze6G&s4NgycUp+6^U4lz-DM!!&jce+nQIYpQr-lTFO$u_*&SgjvX5rprbmv9v;ltVg
zR=UO;re!HEzx^#LW9mO$WozTD`3l`{Z!A+}>AAT0^Sk5U>)+eI{6D$A;zN8*L``zZ
zy}k3#C@uWHtbMQNCu`>U|KH|*c>9pC;o#nXDbEkTKOy)3ao5o`(eb|TKAw5T5xG=B
zV|B>Fklm9Puf34+sB1OXr<ae+`lrriOxVxHuzA_S$$PJvJ&JI=_I5twnJd3`+&ZJR
zc5B`o*@iaWvu9RpU9nAVb(!4KD@h^?8UMOU339Gg{&!>Xa-KaQt_G`o4*os&^RHvV
zrc3AReq`Ni3%hQ8!6^Unvw1dg)jzG&x9ofEFTZ}{YS8-hxVLV*t&MW66fa%X*lDi9
z5WdT}Xvx>98Pf%0RJmit9(pa0Dz9UUyMN5=UmCbZnaWqkK396p)G04_#?&)5v_Aa#
zy7%3@MX3+^!k(waiuz7A4P0hr+*T+Qt+>FoWMj28!#DA_7lZW-Ohmld|A^1&W@p^+
zUG)6PYj)d>6f)+e?@5>ybA$KptDf$*Q?pi@HcWKRQ(?F}pZ86&f3|J#>-FYNiy~Dn
zY!Kg_6z*VTTprWL@m+X=uAGft^Ot$WVzwUdEz?ifPm+J{V|RaM&A)}eKYX8a)7*+*
z&Ts$MimLawzt^3%*E%_u*~D-6VXx<xCm);JSRuaVw1ccuT4?WTg(pT%3$9K|Nw!+*
z?C_G~f|omk;nAag&wD1TbZ$^wDxq$8BWtn7mY6kbs&|*BrMDbewpek&7S3o}mT0%P
z8E=FPjCXlVY~16K?ojWh-1t}j8=u}TcJb=$yV9Mz3<DQjZM;z4pH|+VHsws6Y2;V6
zlkKVh7ux*z_8}p^c&_DMtH>J5H-A@ee)PNWeSOFKIVCTj9@?Q|qiHK7kka~g*_Dhl
z`ZuiPFZ|oVFJK(8J&SYkMaKA3%k&kKb{>y(TBa*IsYco&N~QD3w);K%vxQdg6lH8-
za`Dr+P_QvGdT$@we@2eh1n-Gw3d}PicI&*(zMA(($Kc^2-n^ARKmL7KUjK#p_gY=W
zC-NuUcFSs2o?3gq>D|1t6;G~l=ZcFm2#cCN*;2OU$H|A;49u(6Gv0UoD`}&oG&6LS
z&+>iy7#4`7EuFMh^qN^+A80J{hnIVM(6I<UzL)=kul`L~5xi8(LY*O__WFfm|BgPd
ztKYpbVWMz(T=pHwSIfDRHGhXN9Oy0l@xIQ^F6w3aAKl=jnTDyS&T$J)PRxC;-IkPL
zdQdmm_iVNC`Fv@qNskMSC%it&8hN07mo1;JTX@F2${70}Mm&r%7P5ZIMgCT0=@ORn
zI;xs?)W@*RJ(y?qqhaCI{k`ARtU-mb^~Lsn_Q}V^*Ds#@>Sfji%bzFxF8;h4zW-Bn
z_x)dMk6!27*U|o*IhOD9OHax0H)oFUMQmMZe!^}^_VQ0&g{vPsXzO2U4PtOgFudd#
zHM`)Ga!k-$@vEPHu5rI`DO_~x{0iNY&0nU9UtJSdP+WNKV=v3u9yv}X6Z_QL8fSNP
zM(sa4aY37+@|AlFkC?raG?&RqOPgNCu&t<I!YapAkJ8USN<Y6ZcCEe6@BOpCA5_vk
zpeDRuxf;}^p11pZ`px2VD)UNCsxn5T2ilf*Toh%9<!gMtr~2);+YWclmfwAr5f-~?
z?we%?#ojc9P3(O9{P@+jTKAovsl~q++HcO5U^F&gWTboQ%8fkB)T}!mTO<`?x4*9I
zyLH-^`9(nH#YyY9Zciy*9ILkR&7+)-qjg2!?jCM3Hd*nytudzbe0Ivewj{G`h2zPu
zwl^k8Ii$Xte|ze?hS}^Hk0(a-WQCcQq=c=Mx8I*2l>L$I`S#x2KfcF6hbw5yO3O0d
zc%jT-Usw0>*N>-jl+1-JUjBY>dT5H<$#d&ZWEf1C>YusR;&QuKVQR12^&3w^Si7BW
zm55Y#?Gm(E706^67hQg6C;z#HH!j=?a4x^8TUPkeO@vGQ`ppBc6MkzmEYygQJbd0}
z+kUBe-)H}2Y?!Fb$l)>VnToGfPvKQijqoBXUH;!?`F|zppDKU6^q;?uZQs6%djAiz
zYwg}{+PUJZtZMnaFRCfpI$1#;TUD2(CULEe5sF;;%lMRotJfWwHcLhQOFB-k&Yw_Y
zm#k)tFzb7DP2z4{N>%66w{s3Xp0@X|<jt~kKB=Ob3_(^*20}cmP6#ENdEfXDC*}L<
zg4ykjogQ+D-?WXlN@m{hIJn1yUDGJJNn}!sd(%pb<Sz=L6EECfcWuA%DT$nq`|nHs
z@PB`lAxG`w>BRTd@9up+UcN~#)i#OEd-~CfkCxZp`hVH~aQ(;SfBn_(sWPZ-%-edl
zvn73MgmQYh;jX~PCo_}WyU(~xJGiR2b?P-Kmx!J{Vii)YOg#5zCYCB6^jdT5s@P0J
zmrdQLJG-`oE&LIwRLGmK`0m|^4cc9Qo9=G7@$A&UDS3f=-u745O|RWx5g~R{Z1eVo
zGn8MwWWIGq>+lOLkvqp@Hz!LnG-@)$ib>aAIkDl(?Y<=G+dt&ZK?7s)cW-1|UN}YA
zJU(v4L7mRu&s#lDT}hb#>ALir%Y82juS;b=%M<5sRhYEUqcbIK-C4eCH+qlEdH%#q
zp=b3Q?sDcU9KO3F#S&hw3J^8S4!zhu&9eBVq13*vj^7W2qW*}z-s?2c$acc4T?uEE
z)=qRjEXk0P#&_l4qNvxP)v4c}my7*jdD{MTjS-Jn|LG6s-M`eB%)9q{clyQV#jEEY
zZ$5eM$>-zEC(UhGAH2*ec%XOE_Ly(qlB=x+h1p({ZIX`Kek@+?oAfo9*)eWO;ui7D
zrM#~<?(xu;H55OZ%b4n{)61f{^pKcGQo-SG?7SAt#-28Z<UV#!Ok=cWHu^VBo9m$N
zG*5<GPUhV6E@*7jd4H5U>cr8XE=xt$FXTOEl<btTba5(+XJ#5tz7=z;6Gx!9H=Cx|
z`P~yQ{5F>5jMbfL>1VI=TYkd%%p-ew*PMU#{`|UqGT;7Hi2i$hw}0N(b^qSJ+bo|S
z_wVSd^LFw7ev04K*A_Y;om<SnaIH{;VS!uH`R<n7H@tc(A`X|28g+f=n{&=*t@hd*
zU$yoy{!8!OFn7I?&aCr1Tf+WEc7;{(F`Pg4Ui91bgYg?={!P1m=lJ*b+&@R|i1PoO
zd!+7(_|)#2yL0s`x1Nz<)(C#<7TSI)dhN2Ez86l$7&o^r^SKl>BPKOyNvXN+v|agg
zrM7+i3|=o3Zf>Pwb}NN#PWvp$$vW?rcutZ$U}<S_qrKS3P}`5=)9)?oxXj$1n?}6O
z{dfAb*4<yMb6KB>-2R>){qBfVW!U^FnhF}mZ5t;pwvSJq&KePO=gQ*c=JOL@oM&0U
zn0q~@I?r&Im)?nwPJ!=xc-vVn_TFC{6zZ|~=Umo+)idsP-0`UTR48_F`oqPi-M`=e
z{o&x%g8w|04ab{3OM+_u{(1HKdHBA476sF%ZQAs@MmTNjvKQNKS*_Nc!f}0S#@7iS
z-S7UEjCz@2c*)gk5{t3_v;|51Y}_-}L}+D1{nnK()-+-Isq{3;=IjX`hRG3kUJEoV
z3CTRX;)GFzZe_LYJ*ig$8e1N1vT(~ZO;<}j#+y59?<~o<TouoVfUE1A4BVVO_HBJt
z*~1ysk{vLiXyMcQ!LRo(zXs~27!}zo)EBHje{IbipFfLEh422(_WRzs$#L)hzMJXc
zSGIhYOwsK*bE5WY9<JS%lh7l|rQ=q5YW>9x*EKgBv9XkRf73;=;mQo7wdvZInM!}I
zy}C1-A;!C78e8{XH{G;HOLYGS{m4D|?9C~WZB^gb*1ap^nziMD)b2|c=d^E4$X@?D
z!<L7k;N`m<o9C>z-wK=EdMe+&IRDQjBVohjQ{^kgm(J|mk@V$2ER%-#<edj~E&qcq
zRTBfv(y!0U*)mJI;pV^9rzZ*&{=ZXK{pS*+LP`7ofA5YpZRFnW|4e0W>e^FRZm2U(
zmvl?ocWm7S&)wR&tC=R=SUAPQ<ZhE`RPWUt7Zn-$mxy0|(Hkjmeu?4cH;e2yZXe~|
z^Izt1NitluYsZ?WZ#S_y<WB#2=7GA!oZjbG^LK;L;t5})KR4&=%Dta!|KrxH+48Zx
z-}TRR@1Gp^=kBNY_sjMF=PmrPGim9&ZMJ;<H`>3O7P?mZrgiPOzuEL~4u?(e?9iy5
z)ps;plNa$Os8l@PI^|N?yUqXRT<dOpsbtdh#p(JMG3FO*f)o$znRsd9k}2V)LQ8hc
zF5<n_VJHv~y?XoRx{QQrOE*c_8)|87c{I!YqQmwD-wF*(p9vf}c`A}SWNWs4tHQlR
zwY@hOxf`UH{*}DC(`@l3ufPdy+Ood!`|RWYMpb^io%Qv0)>h-6HpO<V7dYj2%#AO$
z-oNeEUiJkqWM6J#IPi3iOpWYXA@7dZZ0DnPpMtoi?K=LkSmUUPdIR%v<80H3C5z40
zX20FXabem)BRA>Ar?<S*Kl$DAe~m=G;mjjbYZ>mWn0GEoGw!jMeKPMfUNMVBj#0a`
zSk(5I$!+|3@Jub|f%$fMn_8~FtYEM<ay^_-SfS!HF)i)Zofg4{fF*LAORrp9W7jKP
zd!?jSEhkyJelw(H$Ns-?4QGAn6rFb#nMpmzJn!$X@0?usY~P!Jll2`x`D+jJ1RAFp
ze}8?2O<bGPaZa<tYu?@6-oO8>a1PL5`R#V=Tk9XW^U}8_YFzTj-YVUFwBuCBU9F3|
zo}LyxTFtOE{=)2Ej#IAlRxLiW;-UM+d;in!|7G9zonLN!eRcZBxBHF%U&~)N?~Oli
z()R1Me?7l{Rqs!`eD><oyHA^$%PY57tTw+mBcZlZ@al!Gxp%&@MtzxT|L*s%MOR<$
zobT^n@jSBaXoVo};li`Yl6U7{R1z|sdemQkG2<f>8Rtbxtl{Fj0;85pt5mxBK(~KM
z>ZC4_c~f|H&frPjYqS1TnzZh3)!ge+ix06`vPLZwtWG@}zBSDDQmffkX=~4ChktFp
zmE0sa+gCa2Q;*|SUAEQtLXQ<YMZMQPc6{bRC8325_NENKR&s{#x}p}7$#p4S_EGeY
zt8czOe?DK9>G;z_%pZ=QXIsDMKI??badxphM;N7ZQ=0iNXl$J2vM}YE*6LU9T69X=
zR~b!`JZZ8sK~*fBv-0?<FHt(@rF0l-y3KgPv{*PDbFMk4ScPmmt^Z?NzWN)T=G6VH
z1~p$^w6A}bF*DZi?&;4T`cGf`leZ<hXXEwSDQTRu-p${0#cg+3#7XXDX)9JAy4X}L
zxJpHH%SO*XPSTpP3a5A)R$Qo>xQEGM-Ey&IiW^O~ZS38D6SU;8#?;yT%hWW**41&h
z-@Eahw3a@<+xWa6W6qZB$x^w|ENx-m&R+d@bbHu)pZLhHp>}pzr%M^9^uB1fzm>mJ
zMWD1U;7V+o=dCCwlN8Y%j}<u^bZS{&nCwhwv;j?U?wFXNqsY?Wss8pqyW{%<(zdTI
zvfr)#|DpNo@4c1BW798xDBAq{&sF`GC)b}Z;gNMd|2{_1diU;{hBrNeqWt`KmgY`M
zZqH>D%Ui;o>Aua>LFd4s4ZBj^`4}RX{#tqRMyunAH&QK}deSLhgE;0&8VgO4Jp6K-
zlyv6NC&KluY;1-K(LE;F+a#`U^SKvn;xhf|Ns0EtRTq^rmqsZbus#;C%44VYhE393
zmv()+xX7XU!Jhm-x~~uDNIg!xu%?|gu6v^1WsR1!jL?7oZ~We_c6{R(FOe8&2G0hS
z4&F<1>p08%KYZ8|k`kKc89FsoYo-3HJlWTmkEc{!$drDaoNRW@|Fcp3qWWWJPDh+S
zeMZu{_?OT0t#iY+UzrqDdu8daw9vFvw=|ckik98&_dSyG8TK@&G}!XXZ{YmSk$L8r
zmhtJFACXeRj_mmg4mxY{YqLckKAQT+@z&fWMoY@&&F`;}&3VJOR=kuU|M>r=MO}T;
zljE1x9r`_g>0;aBvU!#z+zKHpxt^;Y6Pk9tls6#9@4NxS1nFtfN~>7SzV}3}5J|o7
zV;#ztoth;0bXCd*8wM{?H8t7A1{IfMb1p>%K2u}3eA=|x^=OKN?Y{5#|1PWJPdT+f
zGU-q{kF(L7%{$JETbp-3pLg3;^US4o^`&3F{Ndd-S+Ba|;3V@F!PGTVPM^`4l4?Es
zqMyecrP!&dmlHjc3fE-E#^<eNytb>(xcl|e9IF^Vwuzs5CrCa{*Es*>vH0`rbKUdG
z0v^viVOh%H>F39<BG)nP@B_EoE3X?aFiR5kn>e#tw)w1{uw}XA&Ni{8j9qc_PbX(7
zT&vhNY2Jpb(XO2RDMy`VNr#DPO*)``@9ZI`bFTLm_JqFF=KPR$#cQcRPVl_kXE)5U
z?=yaTayk9-^Z63>pBMj~EdSo$Z`MUaFK(~?$hWV<J|?|0Q(R@e<JGsh+9IKik-MKX
zOzqKQ$oQ=-BF4PBK8yW|Q;(|16ou5FB}>>szwmM`j5(&H8hh+TowyNi_@&ibCN1!3
zuD&9?M8tlV>aG*ddt6>$IlRa>Tw$kZm(p7sUIVd!o|dGe(yR5fUFZ7tGI@U1YuVAf
zy2I>pg?DknG<Tz)2Hdvu+c#=05Y1b>y6Cd!ulsJhe(wGC{@=l;|KI(+`~LsIpZCl+
ztN;J<%8Gyf93L~a@3ZTtS%u#_D61;Qa4wp`s*UwWL&As7gQxZu`3pC(aVqy;HLQ^7
zIp=QflC+a&<Etq~Yp(X)?CZK4s`=8$;Mm?gdu~?EjR9uQ0$$zJRw{9}K6=_lwaB}_
zM4;2v^^53Evlo2IvgMha?5`i+Ox=EAr?A>$<?dq=o`wEzre8~Vsx@Jf(pB5br`yjn
zXNg1|b?uwp7U9Qc<h^sN;xaA$I(C(7Ti5AtyvC3nyMO0ao%$yAgHsYsP8DYfR46hx
z)NBv0mYKbrOPc5Ks`lM4KkA6wn&joRkbUtgjy0=}T@@1MGK|^F>1r}#Np?n_2ru8H
zc{%dG0yeFh%e2Cq!#7|j&t*Ty__wpCy)^XAyBL3RqQ~@?ksD7HsLtlUXLzQaVbfFp
z=RZ^bM*b=@*I##W@2MmEcJJM_H+DU9wC~#7>%lEg&N&=7{mwM@*w;S^Gw&VFPiCDl
zXJ?!J*RI=XN4h)hRaeGF@V#(M7E!eAHZGd4R8n;NkX&Yj9p{p57EBdU+ujB^m~$v@
zc{8_W?SYzCY|DZKLd7&+O4%juo^elTFXKv`b1ODWPS==FF!lX!C$0P6-y4Kx-t^zH
z>U(iK$3vqF!ApZ03-70IXUIskSTI-I@7-IOwh-x;&#v3Owu{J-x@64!IOE>0<#Vqc
z`*6}MXL;$r^p(Qzq*!Z3=LVZ3KJETGRV;a`Xz_fJcM~JtNSaPB-%`J8N<rij(RVDX
z7(Cmu<dS8ZC-Z3Y<gJR|!Z+1v>1EsL$9GwJTvjQ`(72~su_QHOm*)At;wkMBM{jW*
zarpdh@x%R*V(Z>DoVMo95ql!}*tR~o@oa>f={&(^S9_+Tr+QKsCRW*m8BNW)y6)^z
zl^Nfvw(j6#2>y{RlpPyi2hQ2d-g-+<O<fVX=*}ep+ru2oqJQ7rUc9blCrd)_g|c!E
z{+Fj?7$P;6%l2_i-ZDjPZvGcn%e5=t3YQAnDgQc^%G1NG_(jt7nKJ{IG|QxzV>S1e
z|B?Cid#=BelHA&@=C=F2@)$2<Zq@iBuArJ1vj6)3uup6LcmIDHVppqXC!eC;xt?u_
zb>Zp7(`@z$#w+f-bf_#=wAI>_ku%5Fbi%469pe|<iUn##!*h0N8F1@qUvW*j7Er=j
zJh4@7$_b}_AAyz-J?32V>mPy@(^M}z2Xt-;SpRB{4u7PFlg9a0m1!DU40#`)2JD^?
zad_qNJ59@!9&8PYj9g#xMdyCG;?zlo6<RIloudr)PAHJ@s@(m~Z`Xk<$ye)=s@e~J
zsufwcN<$>m>EgGQeA8aF%y|`e?VO?6j{u*qcEP)UJju9M6!pCA`?Tu))kZft%6kj0
zcSPsUGw1)RY`r~KkYNqeA#daIDJ#9?`PXZjxu5b|<D|54mLQ|ZJ;R9O2`rmUnpqXN
zk8U}<H*bGm<Gs(FHJAK&m$4txcCmix+|S1Ov1{jwxNCp+?X=0;ZT+Nq>&N4!Oo>|#
zz1sM6`}<FPlO5M=u=erj=nVRjR<+Y^-wMkYMr`L8dfdGCpZueyt$!6M1J6`l{^4Z2
zNJEIShUKZxS4AGW3LkU5#XQ+`lXpj|3y+wV*y24c8MF2+I%r$VzjFPHS2G1>oVonS
ze6C@4F+)(-Qj?eN^Uv|_Q+&C#=DvC|Q<X=lns@!A%yw<nWXZt)uB+Fdm^pXC%$J6D
zr(3?ydt3k9`Z~Y-z8@FoFP-0h`2S1sAN8N^%Kxl+Fmb!e#qcK&;|iqTm6_h1>9yN9
zWNX;=J35REy4%<lUbCe<_Dero)D>`Vrs|q)2Tmki`CfZT&T*;^AOBX<i(8aAlYj2h
zU*euJDNpKA*!Dj0Lu;Jo9evq)qA~Wp=$3<-Qij}nFJ(2pPS?xX*ik9ew5X_rr&z42
zWkrjQn8w<jl7-#I8iLI0Hcpx%n57cq@j!oa*D?-M$!V=}4L(|T#jc!njM7`&-nD3&
z_T5{1V`pwvUp42~6m$2_*A}bG%hdjOnV6kXkw4eROyj)<1OI`o>S66)1CKomn8@Zm
zd*}S;-qXv!^~@5eT;nji+ha}8b3I16r(U0}s+RYPp18)6I!oBAxvlF&>>`oO%mXgm
zvkqRmtbQf>dVZ+B{Pee9=N|a^A!p7PzZL)e?iNqES9GN*CgJkbM@fg@nXNx~UxMXS
zYT_CZ!-&j8hxJKIla}7;Fz)ea3q6y2XXm=e>$~>-h+<w-`j<`Y{8WV!QyD%z|M`wx
z5t{uc6LytOOJ_*=EA&&3;pp@$@>lf2tePg&MI<kMVdAj=R#*A-OIIf8ByF%-Y_`^+
zEwt5|dFOU*&7yVsD^E5|VTs`1HQQ;`?Z1u}LW@+yz7&Ztxc}dyT-j%{tdcMHzeF|v
zo95s9_srb;<Klc-&gZv3{VMqJ;X=ke<F$PccOUE8w@Ay%eSPwVlF+Sh%XQo)Uo?5m
z_T(bN1O{)WnwULajXdvmB^tgn65%ycWJp=~+~~}X_$?fg<x3bIpA9%`vt!MK6~c3(
zdf!j{VDltZPqBQX)W^$$?OrEMo-7le^pHu=FY`pLoY1sY+oY0<c0FQUwYYxPoSi+s
zw$F^Fd76Gc<#?j6g!{=xqa9iXH?!XLh;LV|4CAvAOD|cWw7UIQ^R6$?7PZJ28*pUK
zbo{D4B{aUR<$LC(&%JiPe|=TqpD(km&F}S}luaK$zPmoZyozzhhiz7Erf-+<)LmE@
zBIBtt?OMXZlnv)TFZDb2#767>jOA+=dOej-YV}(YvQRAE@c6H$7Csr>XcJpL58j<y
z4ov*#r7~@vWX^}hOHEd?#+T;a=#o>uue$i+_iwkqy$E==QqFi!vPk&SBSoS)Ws)a1
zbR6uOdg;qlx!<vVYz#JilX9Y)*Ob<-+{za5`j@Zv&M8V$ZnaA4&!5Zav6N@(E3@_G
zmlFkkSnc5{Vt?AN-?=(QY?j=L<Ln=FrrbKx_-1{{HD22QS;?q|(5ZWO_e9Dq35n{x
z{lJ||x*^3#<6212c6El#s@2BA4C`;tFZ*=r)7mEoL*$qecZbMD-|08ow*2nD(+8Do
ze$W1`-kE>GLnr20y5e<{=BI{JzKAv`c|9%V33Dw8)qk(KXT8?hgRu-&brpy6Ux_Q#
z1aPXZ@z};0WU%h=?*%G}QrACl>#(l7U9hF;*-Vdysa;ofZ)yuPDaN?6MKLUJX)p9u
z2wmw|wMz2)?1h_b1ZHU4KArk(*Q1yciREEwQv(-QbBMfpr?U5Uh}PSqs!J|i@Uma1
z?D~;WG{!Bej$>iNZWS@Zh=!61h6Igek>=_z=Jza{<-6r6@6NOZhm0nruPL7>zpaho
zSM;^de(x)nm(SZd@sn@=PPS8FbC+E2vJvZaJ-+_6%OQguCzmoQq?)o%Hd8jYU8dSF
z@py!z=3cjTF2OD35;}L<!hU^FwEw=-CQafJ)5d_Kg;AdA?S+nRi=`%KvZ`?ISrTI6
z{o>QP%6&@&55>LjO_C4jlJSobTXytA+Q;3BDryHqG(8I=8Lp{Z+qLfrIMXf@U4L=v
z3#OF<)0dxraW6uXL(pft-m#)pcHY%TZzX?zzQw`*BbW5io;X*Q#l9=1Us@8(aWd`Y
zdYAZxIZXl8hGASi%Y)YOY-C{A7kFarHhymF{J&8_T8mg#DI7>uYt&jR(wXWq{k@p|
zM+x(;lnlOeZ<~KBw?4kS|K7g`kFD6eAHM(ktKq|qSDc$CFhAI{^pNFK#_G9yuU`s}
zDhb`0-ZgcqDm#OR@rz0P)6~|ju=l7{-E?TxuLQv@O3&1$GF`d#xptT8<Fi58XBI9A
z>$@+Q@Y60@(yP#(_u+3HHRh9(92UvN>?tzQ%~;O&_=xe;S}S|KWl|?^E=-x7H~qZi
zLXCiVTOQ7id(m<sAZKZ@di0u2!7qQWG3eAtm*zCO6u(t(?W)}CA!5014U^frdki|n
zw^Z+zt`mKBuS@sR*7S4l9zWZDX!gyN{7d)dZQ0Jw|Bcb*RR8BkuEi^Dj%A$i`+e=5
zr0K1b&P(6DIi}%rTeU%B|L^S)3zH4b{k8u7T6=;O+m-3=m3N;0(cL@iTe141t_CC7
z<}J@(YfpRh^hopRcW?Io-NrY0+3(JSMz$u(D|KJrGd%MuxpTGJ#U*Ch8hN%JroY~J
zI`D*jp1QH2bnWZzb*2ZRuIERcTpD8H?so5*p0ie}m{)C}-02fi3u`kN3N|wb)w%37
zjQ#iNP~>fP2j<!@m)D3!U0KVpps$C`{*p&`SoB{9ZAtA=#T5(=&oaflj_zLi!s~x&
zYD&P@boDLEE2bySTlwOub?d8{v*qT?*~}=rFQuts|2zKOe3cTpS7oeom!I^RbZ+9Z
z<2L(Q7o1AS{PpUR<<<30#u?={ceJH+s-D}tW@9+BAX;l(istcYdXKNp3qN@6#HA0N
z%Qz%gYaRTmdvYNcKX;S$727p?wumKM)ZO}7srAy-DI6;2bq`J!=(_Uy$(ePh+!jta
z)vEF~gt1L)yIT6KqgS*7r%Vf-_->WPs}`rAL;g`|;$qW;FMYOLE#aSVYvQf1UitDO
zOj`vadU-D`H)C#AcyohcYy5Gx%m*rQg=(wAzJ<J*x%uAF^L5&LEOU69bN>C2eqX=s
z{`Px$aZl#GJow<u1D)rql0Wnv?3~hN<5v6X^F?2=Q(f)WKK<7;OWr+=Q4BcrLPMNk
zPhPydtj2Q>sXrdSH@v=hH>4|EcrK?w=IgSMTQwydG39cM1%IC}IjE$ma`Es~A2~^F
zzdw<`cU5jo48Om88fVeDmJ{9^7cf1Z(8T6Zq{-;h9iiA+y7u+yRp3|+(_Pwbv~;SK
z8iPm9zVs_SA2%oLVpVvXSTl(^pvH6mttUyW3Ssk3OU!(6K}S(ka>wPbhI7qs6#D5p
z?d|*Kw<Nsg>bhCdVl(xkYD)cfH9K)c86K<L$(Q<m!k)cW_V({S9IGw6_UmotcKe`=
zxqA~fx!E~XyZO%F;CcI<)SXYeWDhGP?moW8z)HQG?QFu^GkjmHD;LW@+Sa~XuWRq>
zPpT}NO^#`;m0a>KK)abupJjnM`^r{#<8qDI<6&VDUQ7`aBU)dJF4vJ->vMf#earl1
zvYkf!?K$gW*w*fIa_#d~Zf`l<A$R$H%Ec)yDXMo}*S!|q=M_0;;ybT$vy<`jA`F9S
zC-T}(S+f6^oi@*w1G5+Iy3Bip+o?iBoqg*G+t<%bR(Vdy*7!2*x$4xfQf%C+4ey+G
zO<sO>`StkdKbuVVWmSB)Ut3@N>%-gEhu{AzTKw(wzXxC5UVM3b@n)UY6MOUa3$Od;
zt@tV8uB<(a3h%lyp8KgCN1QZ0C9NY@ta_*T<!vzcJU;Wh2qS;<-D{2+zin75-QnW?
za-uWuwij|<z78u7UgE!)$M?%=#gmgNd#&zkMztt$lvvN}D-mK|5_HrnsCa&srR0%@
z<Kgr4Uq@Y^9+us(b>00fTF+{OR-Q9X7xH-=v*d{f?~}Lfcb|PdS94oR^cAmKHUo!>
z<~`ky6HILNPR|d?6<S*{U9a_Px9N|GwpwWqCg1A}GZ9u(*qv2-bL|Gl^Zvo!L0p#~
z>oGsq;hy?(<C-EbXUT?$8m&z=Ph<~8RJ-wh&^_0>EB^g;H?H50IHhA=6^nXKeZS;(
zkm!q5mt<-}8(L3^ESx9L;=p?_sATEY@D*y@jC*pp*Uf5fQStb&<f;3TO-xI&e_SYg
zHBV)edpU!q<4m>RUT;D(=FNF@?odQy(cF`tg%|i7I8+fE-8`*I)AP0P)D`K!PDLmF
z+$S7XBlcyr@EoD1TuW<LoU2g#G1V)uhj-Fnmd9H<zpP$h!rF0=lQH=2H}R!^r+GN~
zahK&!4~bk-p)9;<;S8ZywO^}swDw&x$j-m{`g~oD#Sh2pIXdrZvwykW|8KwN^<Vz{
zS^rPu{i}Wc>*0eh2l$$$XKAdlY1J_*me9}Cx@!1GC+1@)gVjPQmabz_v0*)J7u1#B
zn}14N-clOkq4V~$-_o~kCuO|v-+$m9y6$z_$36MiPiAa+HPcCD+O_1s__*R5yVx2R
zot_af(XmZypMq4=7lt)g<}h3fi+KIZD4Tb~wXd^7P2*-<JHl$?klB3X<@`@+MK^bN
zRWJCde5xb$xaGlR^Y%|{o#df&E>g_Pwmam8^YfKqju&$eR-ep>w>7c7z>%os`9o**
zVxL<D-9-z&{+(jdbZJ3jO57ZA#*UJET45UJ9iIQZ^u?g|$G0uF?XB(a+W*`red<}a
za8o>YPMOnFqYb~crz<a7f2ud=MpstPS1b87i!O)E+h*CYMmoebd}`hl6Xq(#MbUYj
zw@v*gi|tSsxuiXbYpP1~1(O)T@F|yl8rV|jDP$IQ>~j)a)f_6aMPU1asqsfdCLOw~
zX*ADS(KgrF>T{T<^$|-^q0-vw)1eC=EbTfb|5)Y5C9#!D&E%zYg0wf>DDe_2v^sNr
zQ*G7r*13PP(lr)@uK9B83B#?OOY|4b^*&W2Bc1X5(z~~><dpIjKl~)QzAN8m-=Y{N
zhVN~;-;ces|9|w!r~5~bJ&v(mvy=Vp+rP)8OaFz*?ha;nay{TpsmO9)m9U=+t=P10
zdNCYbCtX~Bn1AI~-?fKdrLdiC+OcQx<`R#tzS}!Fc<Yv!hVI~9RZ_us_wsuC9iCz-
zCSKpOy>+J;>AX5!eAvoh`xoyilMiXHx4xWwWt0EXO?xhUugwxnyR6tC_5Jqxol&6f
z%mHxAi0x_+SM9o@=SAn|81{GsJ~WbC(SG+?>AgCR2K};x7zTxZD-ZUssCS$<KV^cm
z`Zg=JBhrBl3X>UQn=bQPnJQ=5WPebex!j6DB}{9o(1Tg;7Ku)~bY*i_iS^qLX)4R-
z&UvAItU}^W#a+9<?_RfyGk@6QJjLPrhqZrpe`lC8cmB8a`Y~)(nbXT}T|dif-(#71
z!oqe=Q6PiJz3_`Wuiso8tS=~C65SfPnse)-k0pU0Q!gyi&Jgv!#@m&s-F#ngTXxv7
z?~y#~_Hv#G4tXpgBH@v_yR6}jgvJXtk$z^6uH3TjmV;BHyhZMDc`Q|6*C|R{p586O
zB^vVb*@mK^in*))=G~pz#MqPF*Htw=&v3_vPy4j2o9z}F7R~L*zrAkXxm|O^j~kp^
zdGOuaF!2}brvxl%w_2Si<@!=@!*1VoGA$de+MXIczV~5i?Wffb{`fb1o)vA<@zj9D
zVYk4}UGtxND}Ksqv3%~JSo-|e(k`7t0lZ98Tbs&WY874T<cNCW*XvYw``?rWm)g^l
zxir=!FE}Wy)qCjmxy7@Oey;m_aQFWE7uh#GJKS+ZX;qKJn#*r?DHVxt{B+9G`QfGs
zb{$ic!i;!r8RmRfJ1BBMq)s?#&9&Y`u>Q;0Rc6ZfrB3Lm&NS0ZVAy|p9_wY@E@c(@
zHI1_Z&&<@jeA;t@<DThff~6!>`nCU_XR}W_UfC>fD5-I8=`k5|oy*!KufG&b6rU5k
z^}geCA<gc|p`9Cf)`~0>p7iqR)gNY4zKA;Rz4rZE%ips?v8`t{&MvKx>0#*EzU)>_
z$GMKslF+{!N_MR^ir5t%vD85;vw3xJ#npeSwj6x&R%X_v3(pq)nl^`{>)C?$!H0Z4
zsvlOr;c`n;!%?E+5F=-D;P3TM7D*gs6|>{G)pF>2@Xjh7hVI{fmHx($928yEr98X0
zY*`g+-8$|2kE=W~h2G7fJEzWLtKafJqghY)u+sf9-~Ps*GeZ*gMQXi$*TVm5#hap*
z0Hz|f-Kt$ndHpzDkLsjb#(Jn+Tl6b7Tf#Tq>Mf`2;YH7u%#PQ6EW<EuKl9>-$SHqg
za#yD+=Jm>ogz8+r(i$|$V@-r8PuSL_>(*{CnZ0|;vNOSVxz=wA*0A+ht@-jQ|5j03
zHTCcP|BMzfN8bK9X*u)S_#Zb6-+cWw|B2+OE6=?gnAF8)7DTL?YuXUD-BRZ0Yr#XM
zwW^Sg=E|ia$*of9Up9O0%2CYxAjgpOPI#Bo$%I^^_LhyF?^86-&fCve64;fzL?o)v
zRrr|?n+cmMSIjQ2Wh<9Towv|;%CEVvFYSNSJN)rgulHXc^&UB;%;vpa>xJm%3l}na
zM2<u?T>ba{|IM#O{&l|`<7Id2yphle3tJJg^6a5`fr}?^5tDlT?}6;ssq3tHuF3xD
z+BD~@Zr;t^rx(r$d&XXQbK@@Wq_aET8!b^>zH`r`y2JWsE<5{Gu(qCb614D8u5%G#
zRCwgoZt}WMG4uJ!>t4bhdRzH+%1=KpHt|Ny)8!7yZJS)8VrEatI{jmJ$EvO^S{2<#
zA`Exboe!N}xp$3V^c?Q#isxL*ZtvT~wAmtz)6$jQ?DaDSSD}k+`evoqcqd&fiTP=J
zO=#Dm6@e1*7x@+(s{C2>XYTB@e>$>%zFzMB|HH&C{Ve<Y60!TCd#bM$2KsHDwl`C=
zm8Ha~(lEcm-TH;sLPa~%9i`I?4)d?B=w=Ww*SUM8dh4p!;mo|8Eq7JA>;4`*CV$<e
zd1<)zr3<M6k7x4jfAn|So|zNOHlA7TWIRcAam<U2FY4L^4I>zwBL5r$&C*1H$8wvy
zT!iG>cOR&HrlZcR5xDK`k)R(8D_lKHqwJJcvsRaQoVA<YYQo^*Iz{wiIK!fQ5vv!x
z_!UvE^d%u-t9Qlow^K7Z86`RuXGSi5!8>z~@zNz{f^~OYdZ888su}qEVDmluAMy-7
z@$>fW-BHBe-FGnN8rzefFVv<5D$aViZ=TPUbU`KEEv37UuhBYF75cWWTE+L$T<N;W
zOzf-cRi_8O4UfE}TV*Z6J%wLb+x6Hy)}2$i8z%e7R8_=z3UI5kvYJg_>mX2Q`kCQV
zu72#wj=4%RZNiFHN6!)CvYZ}n5ITWX(k&+HrM1_sGI5O_iBn2PZ51u6+Gox3ayB@&
zmT9Kw`54}BAGJfmT6K4;=IfhJUt<)ZaaQUJhvHeDuzgY?`vn(CWVS|ot+_q%-kj|W
zE|pwA^d7%wZNI<grM-{%*YnS>rSE;VegEW-xeT)V*DbvC^F`WK)exbEZ)M;AUhPy6
z>YnkN<@7s-^#(g6x%wSM1H(#r%y#!Q@djkAjk7lJ_<tphce|0pdKIsCU2Yks7ne@=
z?Kf9`xi(1Ck?(8DHCNlnDdAy}8efm4OX#XM2b}t35LM23{p-`KQVDBcn=gIwXwoB{
zEr<3deSI1Iho!-=?%5~4vgIerw=X)LvNfPmYTl_=JM1!cbXsVA61l1tv8Aa`-)HHh
z%2j(sHx;O=F06jBWD&2_tch3Fi0T>~+bY~rcj8NL+t0Me!toaSzgq9E|Ipq)QD~P}
z>cYaAWok}O801Q;HJtsl+*WSdHI?gi8N<6W(b&Cu-36(umb9~HM)~rupQ_yQ>D!-K
zE`p8~-px;?>N0<DKB^1wI&@Okme=sB#8YOm<d`%1tzn8`U8|KOCrEWL++435G^tI%
zbaUvPV510yiDKzTq9$@XOD#&ca;mzcSZH<a0t0K+qqdhL%@w_}xKch&2(su8xq3!M
zD*wv-z{7d3oL`naKlSWUDBJ6+f=(iT9apYAa<*4=xku2>g){DX_4ce^P<?RKjomY@
zzmNz!zTr}U<-~CNufLAxeO6z7dHQmO4R_D~*ya{){W*8nmJGMmbB)7f_XM%5RobZW
z_)Mc$cK+tmFFsscwAyjODnTojc`NRR^Qta!UOCHy<H4@aE9~Ya&YsLajV&=sW~F}n
zrT{<d$`v9vO+vio^GtpP$nB5QT6&tNRj%P-!31y5G+xs>apCpZvGD=mjHjHn=0e(n
z`}+b;fhS4z`|Wi)Se6EKx-xfdcz*g_kA%+M9%hCU5t^G^FU;jGdaY>I?Z-AZaQ2&p
zo^7tzBEP6?4ASFvovGvWoO9CayfR7FQ$>fBTD$s=s)~9XUtfIKXs=Aenc$1|doA_|
z)C*YrKd$?2zkNlRHTU^{%j#<{_M{w7I3i=RSd6#wV$9xi8ejgdd%5<_`WK;Gcc*$C
zelf>GXRqf!G5hZ;?`Hd5vD6KiKH=QEmW>{5p<E3X3$<r8?-riNA+nBLEd7b{thNm{
zo*i7NClnM_CM5<|Pge-s>2OKA?a)y#;V9R2#ZpB^M}!t}oK(zl;Spn8e5q}UD;L8=
z|5kGrE|!=XIiiKOiXD%bxV$--%3Q)0nh0K1v`l&Jdb7iXZ>8bP_^%lWG2VAqpJd!F
zxhC||W=RQ_>9;KpYH}WU_m;0sylT;AqmT&Aa_Oix*E37KOs`x!cs{bDdfCP5PoA&e
zu>QZ5pS*j0#q@9AK3)@Xm~wjh%+9P=fh+5bZQX3NcP0vQORqk(;rrEX3-Y9131s*t
zoqf7-o1Npz%ZvK<9BND3#*pLMFP`hp*1z7m|K8uf=ltz{ss26rRF?PFmNTm*CFPR$
z+-G9=xwEdW_QCp3%nc8VC#7|TZNI5HYiCUTT#p^=^w)xBMB{69mu9rIw1>pa3iWHs
z)Jj!sO;q%nnRTyjNBWhHgSR-jc*PiWbJR~eGF;iIq4L(~0&_q}`J>F_t)JEuNeW*w
zej@y9`Mz!Ik`t;{ubVLC)EA450V$k{pQi*Qh)H>Q9-V$Rzw+zv?sLt@uHD{$eEa-(
zUiQ7WlB?zit}9v@bFJ^{51V)~@z?tTpB=ntttqx;&s?^z+%ch(BCa|fvJU;ozW$a@
zVUpLEN?S?a)8`cOpSK=8Ap76v)Nkc{|5=hOVT)9!C^ol#x7wOhW6vwtT+rF!ouTT*
z=It-ONMXy8Z&~UW!VamuYhf@sdX!V*B11w{lbMv|l_0H4X@ZM{oL;a_StnrSKWokA
zEi1ga_q<r+v)<KjRf<wPpYf>(QNMQ|v(Md%>F2uCw)E%OqMQQp2PUUIy>_ISEUmw{
zbp5Gat`SDwr#Rn=Zj3Ve!4NY|O7XUF#D^z~Z>igxOWnSC{a*B2?>&pxy2pl{*svl|
z%Vl2csnnwn7A{*Kx7Kli*&l)JrLX0r7niKt<*Uv-m05RMpy{>AVo4tx@++b*KJNc#
zRQ3OE{=LeMx~RN}Ev=mgm9{5NI5+9HM(wjz-}XCoL^PU29KLvJUA^vM*hs+oeb45_
zKkljf@kwB1$l*Ypry6X_H_VEtT=ra;#qikqq<6y2rFSbsJ|<58#p$_yiSo?VN#9nO
zU*js@dnjmDq(<jErENOrjxx=@B6FqB&ZT7H59WrKPgU}li<e5;n7#W^`qo(Q_4Ggc
z9>4#3=9XE*1EthA?KOv;8TqChj9hbU=M<GXc7}=Bzql0S8Lt`Y=w|7qef9jr^WgoK
zLrzNrBW{@ptlM=s;p8Pr?ueJ0-tXdfFqWUd(6LV`CqyDX^gvhWwznsEwRO)pEtGW$
z6bffJQ9k>{YKaR`8eWGKud_|cy=cX7;E%+IN{$I8U6WUfF|6&=fAgT#!7?ghQtnZE
z-FLz{MW>J7<NU06LP9)1=-MgI(4BAcw@T<bf9%rrys|*<s!BiGBf+hyZr1I0vnO5I
zyCr9bwReO>@s8U|ikI7XG3<*E7VzH3GNGF5z`o3<^X=cA)tmjDZ{7Cym;K~UYdov7
zp0sh%->g)wsH_c6(+|G>W?i>8>7H6?Lf<>NMN1a!$=ly|FhC?eB{a|VntBP#OBKgh
zsdm}7F?UqB8Q#C${?(-*v@>*KN+APlbLr{1-mh9iv>b}g`RGp%S`zfOyqkB1*VOi^
z)V~u-*S_Xn0~)9ckMY)=7NoN@%lL`c5ih5cPg5E`x%k<yIp5rrwW{lmXLG6hjfKoC
zo@X4(FIkoHyt%0OGr@JQ;i=UrZ`)k;_`mQJSFUkYDLJ<w%|&bK%Z~;-PC6yuI$3%m
zVU0j1Yw{e86#-irPi*$R{8w)S`yA%!()*sKx9;3_`~}bP^P6N2ILa_LykBhdaOyiA
zJFSz=tX)bQi(kI`8^z}Bz2qzB?`@ZZ4E;Q|n>=<qyRuC0y~!Sq#EjbV&SZvy)69yB
zMk&jmNhPjSiVNU+G?^>f@36X&V$h|l@sTT!Us^qN*#`x;t_cDw?{Z%CwZ7!acw&~|
zb{mb*8FRBrZM>3o8I1X#3JaO&WEcmgxlaDKcaLgw+tkY~b;$uoYaU%mPBHY_u`27D
zgY7kI1MkCOxiweM1@?)tB{wfHSbI3|TiN?_Lc4Bvtlu)#{<Bo*b3SQnJ#XH3v*n5<
z+02_?p1!>Je)aRJ-*ybs%Qqc={5gg(@#wOJ2i#Kb@;PW$t`_&N4ZWw>S}n^gs}}oK
z=+U<pg;Vcr4laAuqqtC`cW*}5gnuqMCW6OGDi*QiHaiGvDY31-z2}<02KV{sR~Nls
zT}fZas1WwaMq{UTsK)*4O5aOsGr=8@uGPnCLn^8bLyf%E51QzzzdboS>(8H^bsft(
zIu1s8O+NJRh2LHIHT$~$bzWEYi&As?X_;7Xn3wxSkX^~whw}4!Up`sL{!8P%WP@9H
zWW<W>(4&bfHvd=`#Q$WnR6*U}o88mxcUS%VY$#u0B5nV!LNA+J@WbVar>&Q-i{><Y
zIQ6?;*bA?vSCn)@-lleccpnfymD{Q&Ak3;+=qZQv&7@vlzlH@;8`&AWCK)njo?rQT
z`ZR{t59|-6os9i%ij{;nH!di&>CNdn^Gqm8aQ4!tAN(967e1M=;33~ZQ=xgnXO!fB
z@;Sd06J%QYvgRen`)M|jFUwn3F=s{nIx*Q{-UAWC#3+rh5}uj<J3jPriaiy2)g{__
zhuis&*Q?EU?%dFwd#Cbhug~v<Oz-b0CVL;Zv94j`?9OlV&^gB-(8;<W`3HkfWok_o
zW5RRC-#NdnxpU&o)=injvMOli<?TMzTjeBDda}jLc=Apd%ndl#CSUev*R$1e(N`s8
zWQDxM4a%QH{(9AF65-;S&0VoqYUS<|0VZOs&7wZrlO!kFPkY%YBQt9vONH^Ijh|NJ
z&+vL`q|8vR?tj|+X!f1>(;KdReR>7doYh{pR3w+}ZdRc<!-Q64+3Q}n)b`eGue@t$
zA=#Vf+W)dW;@Qf|-|72jS+01T?Ah<Sif>nYRQs~&;XMy(8LWgq9Nzq~`gg9?K@;Uj
z(aH;@o2RUru*G|6#q<~2pK6Ym-H(1F|NqI+^Z(!5SN+_)`lsioe;?%L{t4ij+|M{?
z*Q`^F4E=_84{flyX4rMF>zVEs%`krceTQO2(^GplmYny#9ImmeoptIWTPZC)=A-FL
zemWXOtMdvssx)av1-tH0(F=We<*~LsD?`90-zd8P>7<K`IZ7vPT%LDqVTnwrs?luG
zQ(CUU?JS#LEcM<wRVU=_K7XS>pVqg&^4`DBZ}~OhiJ|(O`%9)@m_M=R-dC{#$uzlt
zeS4lqcuJ|LeE(K7M_^-A<?G23dBw@otG1Tw+>Jk+Fg3oabee#}nm@0sc8V~3^X*Qw
z_<PfT_IJKAIgt!CA?~;5U;bwMx_fcVOu5$*b2iNj_{J#ue%3q|haYJgdpvY9cD~4%
z&U0DT^|9>owxuSkn_F%zez@y12SetD8-@QK-rVux==9q9xd#i<l|v-g=uA25!L;P~
z^o@&lPJU#}C0L^P_}XkE@4Kb1KJVIh2Q>axd(`xMjqi<?pkMp!WsYuSl2m%NvSNDq
zmRKhNk>!(5zted+_rjb&<0P?ks|Pa;W=?CkF!#&pb*hnp`7y^GKI)l@Xo<eMUs5~w
zEN}L$hqo9GB_|t+Kb@1BnE2Q)D}K?Y9`@P4tJ%-bxA(W_-)*1F|8Mc`-;>>zZ<!(F
zRH2e4t~4$1VC+vjhGyf2@ZZfQ`pYEOJQDeBmCdGQr5*l4`>xWS9c$lbz6!8;+x*n1
z?)Y3rhQjbG;YGar6&5XVIGOzNqr;;`cY^M@FgI-BTWhba{$thW=66|+=e)YQCDgJ^
z#pDE1qbDr6DSPZ>jkgD@N7o(28Lo4}TpRk8YHl%WF{ifN<2tr_hKY+7tEEdu9k*(h
z%gRSeTHczTZG0&Qqk7LSjBWcDskL!c{Dmz(ic`wgORm3Eozl5Z?;oFPn4!ZeUTX&1
zl&nv@3~b*|8fD$z!ZXt^exBKTS&99hziRDeQJ89J7~!&VmRxb<35otAO}`p^*n+f_
zwp-QB{ndO#dLqc8iq%$jL2F-WUovWK56Z3oV8~#}(|NEef4}cq=TC7frIPL{F7bS_
zE`Ew$*Ot8;cZ2R!uL;pmJCbnp>H9MgiQx840BELT{ick%4ab~)Zyop2Q)gCW*~mR5
znn_`rVB*|_T~||E=l}M5aAUEv(h8sV)BcK1X0&b5+qyGRba~5EjeRE!uO#1daY;0N
z&l{pJ>rC+WN5v;3&O0?1yPb7>+sx|VHTC1$XWRE12c3xXdp}=AMX#{3{YdD}kcRXE
zwU!E*o)@xVznWbq?wMQ3>l+>r{o(x_$(SkJXS?>)R=Kye-u8Pp^?;0=_LSfj)|dsI
z##|n18!bP&sl+;UhMRnUd~fgOgl!qM+mED#S=z5)jX0{2^fEH4j5AfSeU(6htebz3
zL4-=_^>>klCo=^l7gx^;yV<&C8@K1PsjtL&x!r>=m0qmw)~=C0sx0(-YOCDarF|kB
zcQvnT5OtcWdS(*K<~MP{%LNuSZ9kRo&UC>`=T2d1|N4%DtE^04yp7-Yj;l{*hR*-l
z{hY}YCA>KpMYz6aMNgigd-w6RC~byj2iK&H2hvo;ST<)Ym*wiecHJiI>KaB-!`*s!
ze`kMK-2J`#B5%@$j9RNRrk7)CbIx(g>+IXCQq+AxZtK_06PPCzh@KS*&e|>D$Z*d0
zbZqmg(%PNiDo<<mv72|aRRmKmC@k|@9=*Wl-rmnEKG$USHcUF`#5ie3+(SPO1#_L5
z%a1;JP}tY?thUkjP^foi+QNf7^9$v)mwM{&T9sy`Amg?{fhX@v<%*KSQG0DKKR%mZ
z&wk+Z^Yh;)rE6HrR_|qd8S4~U6*N6A`uR_xeT|FkBUnD}TGLVH<{8h(vH0_<cdz<>
zX-Qt?Ob%s@xGL&(jZ1V!=!=aV$1nAGZw=&7<-WHfL?JZwWnQOKfaTfUqKi8It<2Y5
zzlmw7Se3@y)eCZ^PU$I@pV}v9JMG<^vL$|#`VMk_$w*;q2#IVte2VADRqHE%X8sd7
z+CA;;StA4Y%W132vkT)rT+<SQ`**gjpSa?fw$|j;=g)E7h?>-@U6RcPYJJ|8?b6Kq
z`R4t!&-2#12=BUGwWG0&=arDrHckhwla@)TnL?*06m5=9SY!U5(f;;7zcn@uQZ*Cr
z<#!(GD!OVY@pFIhbOW9|pZ<xa%o9UdHopjDY1orDKkULB;mXK`LbcAZPIpaIShoG1
z%%*u~wUo}TC5kD_=J%RR`O;ajUsXT*s=2}2g6WC30)w>PdO!c|<0Yyss(rO}N~Bl3
zRVn9nqcx?qoRBE==5XbUxD{tPn{TNMFEit_*V^a8C!f0{$B<)mTuxT#*d~kEok~1;
zCz=|s#Uunj)AZc5%5Aa6*2pXJO9Y*E-B0z|uk`nh{)UysDKA4;<%LGB@JQo!crxd0
zz4U>f3Dfq@S#;^d2Z`%;x7gA(nk&Bd+?p4t{BmPp9{cI!l2dCgg<lc-W39#>_>w<t
z?j7y=xnV30<q|qC)=djbe=X>AI(nkj0Wafq6TL*!GQ~@E+pjOG*z4DrvqS9YdNtnn
zC7jx|2OjxNk*fT3*2%N~<sR80TU8de4_lZnKJqgEFs&(W_3j;I^V1J_`T8I3xzfkN
z@WDd-Y{0^jyvs>COe?EqUYI&hiDkj!qD^`0V()$}QQVn$(K}@SwX5MT^Io%Uyl0y+
z|DoOb)}!xi-!V9x6J4nB_;UAU?f);n-;><?^_;7pU4$${B**2)f$K8(`-@z4oz5nt
zE?o8Q?fO|K1$$dg-N|c~&lD~TahXtDx#nVMv;0cs$7vsHE|zU!<4z6UC*`%`cTb|G
z!~<*Zvol){`if<}oT|rJP_*28DbIo2%H+an(^s##yW!ebRdDNM^%^r}Ua@IGl4sZp
zDu4NKY<$Svy4XtCa=n?<uCv*{-tLRE+3x01rT*=Oz=L1Ur`}n;x-iW|rQ_7j#s}hm
zJB!^o)<{Zj3|cwmNW#VQzp8)d1|7-H+@N<Us+8yc7m2^2PxgGj=C?QATK>$g<?_O{
zuJ51Rd)RjH<`(0H{}!CSb?e_Nn|Qa@meotvOSm765m_-O`0;A*<pCS-EL`95WQVL#
z#q|XXw`4x}!nrE4#bsT_^ov(NYrQmbSyHQaDJQI9<(J)BzcS9fwdS^+ww)=(_`O(v
z>nWqo`6qm*xP;`r<l~>hwDM}UXKwl`*Vji{1dqB-^YFX*V0Y^)ud-8G>1SWuENL&C
z>Z;4yxQFkt!t?TDEB;RLdidLJMt<S{KK}^O%+Qb1#C8d;xb-1(La}JH%Zl<v=K>Ws
z#(BGi&2A0}oOkK=4r6bQ0~6jP_Dg;``RAmkQOpy5zSQ5_AIq>DUG>-H{&#Ctfh#6g
zyQXQ&-Yd=hk*FdjU3Wr4y(F|(`AYlRWeeUJM0~matAee;B;sm&K;Ob^zL(f{eqLF6
zO|fAWgLk?R!w%Usn$DWpck=JoE-PL8`g%OLCfkv*LOon5Jjp_4&97#!9X*~aw=A}5
z*z<S7m$lb8{{Ct{{jR5GgGIWN27{5mt=YW~hhK3hYNnXJT9xdu@_Z}f;!_;?g50iE
ze7+46#g_bP%YU7_b6%W?<@udY_x=7Ge-9K>m*@JhncVTyHeVdJV#@JF5xw?uY8o%K
ztyYVE5nY@!=W6#otJ%A5Tbr|PeNg&s)u;4F>+-U1ERT43XY0`yYf~9czBrURCnZ8v
zWQVSLQeC@fQt<+bU%MYyEwEId@@LvS-ZhgV)(gd!L_ci*k#ls~N3D6q+HYN@J{B`N
z2(>6B>lQsxI$Qnn`M#>yxw^5LB3^T!EdCgIwVqpp_fDwdUZJ3iYm^io$1GXOz^(Dd
znRCfKPAMhTYU}AwYE#+nZfrayqPX5_^4(*{cdwr`(aJDWbVhQ7)2emxtPYd@O)qwh
z^!Xib@B9CAJyXE@>u&jG@o)I`t#Y>5A3T*V%3xKuS6J(8!pjoBV-~S1Ui`D>+${0f
z>!cOi6|pNS*JOSF2F<FUEhe{=Q|y%9!aALWhP5J-eVY&N`t@{$Pk*}N&RyPf55<X@
zA3go~jN)^?2K}WMIt5=xT^A1rN8FleL05M8v+=ThcGhfP9I<NR`O_SUOH=Hw^u1!}
z`Di@r!oD}>7~Zh-Ss7_*wmfJP?PQJo`oeu;^6ozG=!DIyjxcp?ZawYv$~0wROu}TV
z{wdb0jkyAo^e>-2dC4HGwLRe6+O^I3aVNV=ig*`@^JcEtAaUBL>!vva+j9oKM>Afm
zj((**KRfq#miN>zS9PZcGKl{Cy!y-4``46Qo{FsdFR2~*i(?O;j0j(Zl@;5x?T0*9
zB#1A+B)q1`)%g81M$w58EkzkA{o8bXmBZvTyk}0ivh97u&4ZV}$gA7N7(Y_ke(8+Z
z($^VmtGTv)$xzh1v-MHj>s@zNId#_r9}MK-F=`OEw3*5~wP%^`w(vH-sqwKpnPw(T
zy%&CB*G{!{8IQwnZ4C)>FS-Bi7Q>f>Oi3?~YVYgs-G4G@cSrd>F5T#K=C4|{w1X`3
z<NP|4pboj3*H3<*escTc&9J{$W-%Pnx>tHaV)--E#5JpqOaI*S`ZT|kt_+JS|3|(~
z)9J@O-!HlKW_6m=WvBb=`lt2l?cV&YnZ4wI+Fs#Bn@mnG6|RVpzq%p))sJvT>9?+W
z+{sT<6?M*8qz83HblMg#GtcS$&Gz|vbIa>ri@-}QM8(*j$ZpmAnHcC%Al4)ub~P~8
zIKsNoIr03r7Z<+l*tT4XW!2;B**2Gh+e3bRx1Ba|o2|}yVU`&d2U6!<s$bD#9XM5Q
z@uoeCz28}{4B6Op;%8dD@x3*+_5b$ROMP1;;-DfHw|)JTH$O`I4X0Nw>}V8d5cqj~
zrjwzK$1Cmm=elK+gRQ&6?l;_xt~j%)x;x9S?_Y4~g*94zb*X2U^+{fHY_V#LxSw^w
zA)!0Q%`<<UtYhip^;?!l-Hn`@BB~Y?6uI=Q<JAMQtx|Ktb%nP0cxm4`t#e37LUdcG
zTtlY2(E4Rh!!$J3Iki1q^DgqNP*}R$+EcUj3l|>Iwlem*yYPE3$DxpgA>MA{N9=vC
zSZ)m1F1Sig_QBM73Yr@$UbdMojn2A$>3WH}<?@j0x(!UlljEI3+m?1eQ}Nj4$Mi?9
z+HLZxpPwBg?7Mr9ug%rA$~Z5_cbvUBQP)S+O2^37uX@e5><iNcu9&R(wMV#l>1EFN
zuLn1|w}<Ym+R9O3D;x7QtG}En{m&mu*$uyTT#o3KYV~1T7-Xf@zCqS@?U~DIOHw0b
z9z<$%MxArwOZZ@8-+AoU(&<3~lP;~kog4f5m+3W7mwdHW<l{9~_8bY8k{qEO)fsbF
z%@r+W>G^2PaQQc1tF7?)DPFGP>s8ntU(SB|sqdWQV&R57?K4jcGxf4`HoV+q5a#92
zeaW?W>+y-L1(6j(6$~Qp(-vMiRCf9C-tA>-zrQo4=r3>ciB)^)@3PL|81Fl8P3NUs
z{)#r(E_QphTB-8Gw+*&)Ew*m6t&}@(*U#<iM``WkOGRlLEWX}*$GK0<_($25xobW|
z)Z}DbR*QTqbc}1INco%f8ISy<C0^`Ez59N(kz3)*sSEzD%UiuN?7vDki*L-rP5ao~
zKIXnyCi}2+{iWBzlN7X=))dWM^u_qjq`l%^f2N+cIc3g&;Ct=e`Yk)&OSS$w_3rJY
zlvPK+WxT$7uI|~?`l6zg3pESN4)5|!`uOSh-MxROuC+Lwbm!n*H3h!~G3C>i>cpM@
z&g*Pc#eBqDb+7QTdAqAuJ(DuM{vt<QxbjFs=GukxCWor_)#L_>EsTrgzVVf-D)O-Y
zuIK9Ve}xXro_+gB!osA5PTCn#K9^(LQf3`gnl{U5i6+;rFFSPH1eUxk^fH`P#--uY
z+_Ex6F46ylY4g@~`n$k$434@>l~h^&8uL%o4xF>(=3LYKv;0E4oHTp(zBza2O~boF
z$;p*o_PcMftd7`u#!LCGs9^Z7^7v<(g;7=v%WoAg(U4=9vpJ%2pW@59%coz4h-kAu
z+x@<-KE}BJ|F6li)vu&GM5i6PZ=A(?OPl4KE9b=d&%JrSc0DYffB8wm)maaxEzGzn
znG?VLOYq$!(*uROy*<80X|M5O4*VQ(A&=>5?mMG1JNQ+(q_y6O$Gu&>A^lgzB(?V*
z67Rwuh=(R#iTtR{%H`j($}lwMqtE(v@u7v8Mm$s7lqJnYI6Sqc>{_&=X?4l`K;g_?
z&5;@3^38=!Empt!b$J5!%2R({E}F}G;N9D_yL*f}Bj56S)mrPFN!!V`!`ZNQ>XAkI
zA+PUz^xFICvZKW5PhEF88%_r)7TJ32SoKHbX8qEmKe7&Q+UApErp)$?(RGS2!>8<o
zp51Sz>54V5ntin`<&k)t@+e^H_Ur_gfR}#t6Zp>0PI>t2eEa{jm3=(69xLSic&5D*
zotJif)<LCPJgI(*UVcndu}o)B-`u8jb++NTzY#01DlD8L$9~|tA1}OI6e$efsuiug
z)TjNZeb<zjWlx`66;mwnX)oL;Y_V$2%t=8L_QXERyxP&o5)qicMpXJ%O#8KuH!3uJ
zbwBYo&wgbbFlpAy!lV;lqE9(3-ZA-SZ~op|RsA;q{_;1O`dWWHmcQD3A~~_ZSy%M(
z<DDT39#$%T5vV@?K<xE2-nr2$Ey|aMZ(l7EE6Q_C*3&vpxoH1X4-SX(>n~<SseE~r
zJ<)1L%RP;=i!|o8<<*`m;Yi?l$Fx#Jd;9lE*PJ{Z9XH3c-khrPYPD6qM&9P6R@uX^
zS>A7PdU)}{-73)#k?#j|Z-kdt#MfR5;_E;4Ik~jIq|rj$F5UQ@bx_04LOqkIp>3S^
zk~ZW$xAm^Ax88sCjIQSzrIi8>SJy0jv^%imMc9MgDjQd<dMvnc)1FJR*AukUEq<mH
z{&lXGe17vwWA%GohJQtj3|cG>E6?u|_23D+x{~ogn|zov=hR8f4uSgqpZzr!&wVX1
zfm=2%{OiY@$NkxFES4)c)&8HGc?7g0c2?R34WC$X!&6U8Pn}Ml$rhiqG)By?dGq9q
zIb4Yac1NQcwyx7(49-uXo_FQ1>|;2}q4-#1`Og>IL;Xbsn?+A|l*=zHIebZURhf#|
zRS_qKuJYFsC2qM@3&OfGN^4ZaOy>!5TzD1xmuGhDF<xzM)te3iG0I=9I=A1rcX-F$
z+52qX_P4e9Fs^*gCHZ(s)yLY!l1raFy4>gR<?6gZ<;v=Jng)KK<6YtfcR5uG-L;D8
zZ3{idX8F(W`s(YQcb>jAP5pYs&dh<&C$%q5@qm)PZ};X9twYYUjvMWrAj6xnq0c2y
zZ%J9j_EXd3y{380-YKpV{;>JL={@@&HLfV}UR||P>->`5tr@Ma43xDFyg2t|QRu`Q
z*~0qjd$bR`{}fzhBU$TUAgB4#LvXRd4)=}x$vJik2ZN^?Ug}Hw70sN$);3Q`B_rA>
zLj1|3(2aMePtE@yW|f%ODmpj(IH%`xzJ}j-QdgGAoZY!(??l<>XT>+Szf}6Ye)Eqx
z55LTvXT|hKBRpZ=OT(lMCXS+=thI}@I$huK{F=A$gBfV0uC9mvy#m(&*Y<g7eP4OF
z-rL>cl$nyzlNTR-<&lnD=%q7JZBI8%;Il0%x}LqO=jbV6C!eO3XD-E@Qey!x?|hUE
zTHbjzWbK|)CNtg}EeYarFI;__<wS*M&(&4-5qsC2z0DtcZ{oq^C%YNUZup6L+1_7X
zF<tGXXG5rI_KM^MeLQCqUQWpnvA?!mY3a)Fh~o#oSi}k(KXuy4^U|%Nwd-!($=q9M
zZ?)5Ar=8i;P%YN=PZYIO&wo~(+p<jdbh2PWz{zF{Hir8QHvNW+)=!Zth}&YnCDWQ=
z{pRxB{2U>>>dL3Bj8om1_y6TJj)m`h7*Fi7<Vv)@zPaUWfUmXD)Kjf0$xNr6wOAti
z&)F;qnt0i<af?N<laKAQmZLl5&)#2}>a}HC7`wsaI#;uG>1-3H<-L)5B%m2Omt_xg
z+bTQ3a-}?z-%AyvOpngI$8hq)>J(|MH{z;VLJLgxZjikAEzwCseA9-<N!mB7ud;M|
z=?kBBoh{XH_qJ+=_px_YuFuP?9_qTrmijk{B~QI-XeRmRlPkB#BSSam_0?`?pUyqE
zBKCLYudr)-zUQzh_<r{g^}i#!){{r<?cTt+2@#>%I>&aXZ7ez5=b<xYx?9V%RXv;P
z=9dI~fBWlWsiBBZyGmfBR9)z6zd2JL-HmW7u?fmiVu&>Pm-19hFL~t@9_2MmY#xgg
z;%ldZI&Y!iKESef911B`F@9{56TIx^e-U24(8tu_gWug1p~WeUzIhQQS|0ZIuPBA=
z@z&D*#d<yb#uNRnBBu#AG(#%Z+!kf9FlVa0y3VZpP4n;h{_EBs6L&xKux;zZ_8=|W
zcG1pOJJl3Lr>h?&oGtA+CRX=I-eK3i5<aW_g5KM*d`AQC^PFM38|$O*3dZjJR36Ez
zs?yh@c593IW7*Yb(pDV!v-0?*3A@|)*A#KeYNp-sUuv*-!lRN)6a1|=NfypJ%G@<i
zU({Fm)kIq+^NEw5)Gml!f1oDU*7ZfC$5M9A($9Y0E4^H&TrqAIDQxayt@*m~>H!&9
zjdkAsUW+O|C7BvAXSZcNyErZINr}J6T`oVv`#LYTztT~xOnq5+Na#Tv!>!$l3#wOb
zZdreIr|_{$(>7~3TAtjXu@6+SO<&5EzAz~6gi`<9=cm7wepcwSX4~wx;=Ygd`FgqS
z)!lWI`K+|&e)r*aNfOl;x$}bcQtVM9r6sOUHp(fwR33TB92vb-K>hE<csKKtk20P&
znM4#<uIb}>`@GIw$A0?pBMEP}S>AoTaif^gv<tq|Ru?mfOnv;jhpG6Iz~R6>vjjA&
zJ3yVd5J&{BV$!J<N|HG0mAX)cQS5I<U4@yN`Nda<e<m%j&0sKiDjKjP<%;2sw@gv%
zaw1&XGJkO~?2K+b=xgaz!F}oFCFjs7D=tfJJ>RPklKU|%c~gD;5`&tmJMz5de-wD~
z+73OQW2)?V=|Nw-{rTy#&AZp|dKhJB{A%9IR&)PBPjN{^;@N<b%Dme#f%O6EOaJ87
zT)l5*8KGDFE4}+|s9aH_k#YN;quxELt6y~OY6(~*`O#?Wv0_Ef7MBl7Z0`+@>|Am0
zSd_<+ipuCO^)LTz$<Pt|x0JP6l~qzTz&(hAF?54_MCD3n?#&%4i|?hzuz4>wijAn)
z6y7Fjs_nLU0?W1&FPq;L9#YbBH`n$|S8MrX*&RBO(^tCDZeLp%!vdGECgWc3JCBT~
z-DZhdtij8DqB$gbS9r~axbBU!tT!|Nb*-Mech$>`<ue$zb{zR6aeC()iJJ2(dbclI
zzHYYn)0R`|52rB-bY7kR^1IIhuBGSZo{?(U?WH3ncO|4SbWenC?g#I*{Y$paI{mo3
zet(0zbbZY47y66Y>=%XYTc;J4{XuzG%uWgK>-zV2Vtp6d6fF%|s=KOZD$9h{AP(2x
zK6ARCrYI~?GA?&m_xH~AudU#&fcA=~S{Ha%iX>^}R5CkwEO;{c1mkP)l-t{H&Fl=T
zUZy?%A;b{>=EARypGr3S$1Az-HPq#AQra`~OGNoSA6e$^-z-bE9JV%J<CV3LWx*7a
zy*96UZ|7I9t$(H`7j<s--wk!s_RiUN_bg{!?hi%P=P%UbIU|dn8j5A<E$^s&mQZsm
zu_E>d=hd|WyRKhd{X*-Z_SvvQzpqaIqJ7Qla@H&3jhb)&^qpql&=b&SRn_=vDav%l
zd~W~;t7`B{o%_pgRw#-{lrJ%`^mgV`Rc4*O(c`32e2}0<qV_L!@hSO>|K92TDtVw{
zPIpgKg;NHXSG4+)#EDOWO>@;9t+*^YEW~%UM|?d}q@|>*I5jNkqJ;`W)u%^iCG7SF
ztPyGScISE}Dpp}4UjF>+sjlwzanqusg*vaETe$bGX|rpKQtGo8?t9;P$A9HI{6gm4
zHm>Iki)@z8o$xtJ$j>q0t>o5b3%;pLkNuu>u3l*^$NBhCn{243j@H=((e*)>!dHCx
z{iQK-s(qGbsBClA;tzcu+y{I&-Qb+_{o8HN_bofpAFbQ+X;K|~Uan6d$K5QO)xUOL
z-pwGO-CuD$>p;dv&C7}nOKlEDwQXIezXRN?alC4Ac}~~EgZaOn&tw*H&YCho`SX>h
zvai1AE?xOG;>nYF-wf<drv9CD?Yd*j)q~E}B_8`<n#Q#})LAO|^%#THpAKoUwQDNO
zr+Pih)KRW3i3rUT)aedoJMd`k^NXK*>-Qx7Q@;HtHvRQ`Tdp(lhTs0o`#V$M=%2>{
z6-$yX?pP%(WHNJ$a?2{_iN}p@8E1W{?KiagnVEhuU}<Y+N~UU+d+z7*5K*(6pO)?V
z^UC8^?(C)AmsUDX%PRk8WxU*}hxM88JtNER;y@PGUv+CHa5>K7)6?cy?xQuS(cIbf
z;t9iwB`OEkhqfEGy#K9g9lAi0LH1a3frh8@_o%CKnlq<Wwmv!MnrnSmW6g{{FCz}u
zmvOJB7**z8Tk4u%mwsiUPL2Cc|Cp%uk6HB>zSmCw^UnL7*16WSXOC{HiHk~#T^Ey>
zE$ynL!nwht-}T1yechLoy!#dIRGm9DJ#IapsM5YidyejLJHA0fQ)YX{=0)c|KVA9y
z)yqT7&8E$>_x+yxt<jaY{MWrq`D<O;O<}V8*A*9iQBZW0S*l_f(bA=%)41hr)GI#~
zv9%5}R_0$%{@7p=v1H$+_rdPMS;t<VYg;QO`%=m;v@<=6@$y>DI6iknhW_G)^^$8!
zYZrp&60)*1SxY%r&lKJFo;h*Nl$VK`XCvHxoeJH~(fl&(_b;)<$%Sf5qr5ghzU~?9
z+JC8uf7j%acUGT#xHnY&V~gw&+R~Nk68<D^yT7)8HLJqggB2NDw?8wzZZP}&ZvMJC
zY4h)tEUi;l@nqz(EI6Jp@p#Ss6HfdMFOOa|)Mz%od|4``n<e6kz!v#{)fs`k;(wx9
zLS1j}U3TVH{b~7^vY*1=Uo_6X;;Ev=$}lm)Md(GGV&_JVWj@wMlTVrcu@K)EuOQJB
z>GDconv&V+)+K>QRvwQzW_0SnL%W6AvTtLy)+y}s;s`WhX=TuM`=+yApdv7CVIC_J
z!{@t~+`P}LXGxv;UVAAa^X1fe=}WfAG-aE_dhm$NEaFtGk$ZJ)g#go;?KScH*JS?)
znDndpopz*i)xs?YCT!GNlD~9D?5k%AjT^f5TAy;6d;Lp&=86dhVule5y*iIvO05z*
zxxwZb&q<~Jw~|{U+!j7^aJ=$UC#Frja$EP?eMf6ApVpL_T$Y^0o#i<#Ov5c{VR;3E
z^TSZdHLD_-w`}EVxc)QY?Y|`-AKsji60ybalC>h6_j$p^My@;ZXKYFQs*@<oeEIb3
ziHr@!ZiOvJTGJP@sN9_Dy6VTZPXeaz)(dmO3t!EZ%Q-Y3S6WIg$PlcWb;kXtcFgew
zrxJR%FS{bcvfxKh$DC`5*>hGM)yTB#Sh-92K-Gn{g4wQuEmvfEQkR@(DKTA>ADhNB
zBl=W<6~m^dq7BcQts>`TJ<|#6Dsr`ATWu1f(x7uDv+jdUe9$KaiSL%$O@&rV`c~cF
zxl}AJGG_HWPv20Dq6&EibIzWk5QE4=zopFfZadZ%7Amfky=DPJT(@XUm<cau&cvV_
zsjV_z>tBA}pfy?jm-@NZWxKgvv-^v^(t8%nKXF!JNzk0KCD}jrosSVon&lz-_lSkK
z$<-H5O+URn=brL0s@$c{y*EPC(Y!@n$4PX~(ucAAY)VhKmUjjPr2e$?^%ANn;g$Ca
zXrHnn`Qyx~2XZ@uTpL37&Yo`eD5idAq`5+jt8&Xyo9ykk4qyH7YwEHBS=HvH#higz
zr?Vcao!sEGv}j#NP-!K*K!8!09fP{aw9<RY8TYJz=oowv_Z8>Vm>Q8-u$XggR;Fl3
zDC0`qH7o2R-o9V#Q?l&ouZpsd|Nh*)|9->c7+J|L>eVGH`jfhUzkj>k-bPw}Ms&&f
z$ayb$S657*5)qm4LUwD)hNhzYFFN`x2MiVkI2~rri@&=zJ9hs!@KOoYqtk;fC@7}d
zT#uR^>rpXj8OO`-9}jOfj6E8be6Hzbn2vJ&>K)VWCRndlH_baaqvSyG*MFy`n#6lu
zek$AK7!@mhdd0#1lzF?>uq$x$TIC+mdfYwtbiI7yPwNA63?F|!c;?S7J7eFP^G+*G
zBfL&}=)_!O6VP(~+}z3>Hd8G9gY5;5h?>x;+ndiV-s8`x)3PFE+O`|>BqemWxyvq{
zKllHK^yl^>pXc+fbzJ#==L(_Ny&LA0GCL)Ixv}bC&{g);lP)N|v^OeSUo}yD_KRs7
zFKY!Z-xW|09%`)nXrlz^XqAQ2K5>e6ZZckZpyp6?=83O=oitwAoU;j-?8o6)>ulhc
zu-f^W&=)p^g)R$EgiYG9`s(e);!Dl>J1?1VYaEmeII(TXhKhw|!cG^?DC~?AtQOav
zpy%5!9HyAbI^$H)VX@zFI~``v$}TqIy0mcZ+KJb4``fPbn7n9k3OQ24GH;rkti_%e
zoWJU(y7?YvXgJ!t)=}2s?%TxO^^1Sin6mEdI>;!%X1>d{V^hAz@hS6eEVa*k*Y&Hp
zd`i&OhKsJZZ*s~kns&1=?(gNfr><u0nSO8TOGDq;yi9jZpG?`kp!iOs!qnC)djpgi
zp0sWCQ;C7kSgdgM_#%2=mhHg`pDUO9pRK;do&4ub;CA(O*#>$)lGy)(b`$PXDXM!e
zDe_(@*tPr;llhv6-WPJyTg#WtfA(od*GU^Qwp~XOzB1Q+jZgdJ`SzFop4s-UwKmC{
zmQ{Cz97&kCtYZE34$;_^?QRkbH;l95w!i+VcQO2^cZU38uL;FQvu=G-d-%80YI$_@
zy$K&G7oD4Y@yq#L|1YoqR{!VmU;DqSuhmyX{jbT;<usUjIX!sy^(&wM8+mWeGYEXn
zkiXIPZi`WKp3$s(^R%xtw-@Dn@H%ihc1Eab+Ny2-8@1X#<rzmD)t%|O^yn9P@1;$X
zQr<aCnL00`M3K|xrMOc2+AR~#D+jSUF%{hY)sfvR60k+e^Gztn)z14&EgDr5TP3=G
zn<&mWH7QfuB>L&<DH+LM*M(T>N0xDZWm~y#{;9S5+_Q={nut7IvTxSObz;XtdGE*1
z*6H+eP!yG1RmyYv)2?z4(+qRXTmR&kC!CV0xX4q-w4m}m!_s#fk`0tLa;y@nsJeQj
z<;H#M$I7=-7QVXZz%5l>`(=7yXtV40Y=1G|!lgkIcwQG8TsrTcuNgMa%|p6wg=qvs
zdE^Ttx3`CMOmyN}40lXkzU7!moz}*6rL~3N>hzQs$5K;)Pf3fTXK+qk<974q>Q`}r
z%buS2rTQuMjbGv83A4KD_LYVye2imR-m!Akr39ttdd4d=J<he1C&X;MDcH$snB{Y=
z=*{d`2cM)!)oi=Z|Ks5YMROgisd^h$7BW0vlcI0KATlZVM*PL)!D0-@%((YCUJl>x
zsl#x`hTS1}noEe!m7RC9|NVS2{q#4t-2C(P$(ujA{q17)e?9;InIz8l{d-tnZcMp$
zt<HPfp3LVA6MFPIE-N*)%v(F_rj!0^c?+`!rl^zF2CGk94$1F!JXAE-x`!`Ub#;!2
zkhIBL58amKFPw$ebp~1ZU(!20`A8Atg3>iXhv&8Jk%(ehanj>)?~!SZhxDdi*gPR>
z)%wLnk76d2%~NCH<P_ZEmHKt}?(8e;Cdz2dnpedBU{>Unta7GJD{l0L*%*EKJ4xWf
zjR&)0YOX$fv(D)JT{eMLA+xWQMNIV0_#paY^*-nB-y6B+1~7h@953~Zb=&VFe_eJa
z^<2Kg#-P@0-PsayB=DuTh}EerHCodGd(%zkcpqP2cMY`m-gncBt8&@$I#YtmA8nj;
z@1*SWa;rZpr%Z~}u;hQa)cNW;C*}#NJHDTKE6?Duv_w)^h|!|)x0&-JNA*SP;AL;>
z)s`94rd_TnuMiX2fA@$-=G3--miK1nGrurffBv&-{F?(i4!8=ZrccmYpE~cAd!QbB
z;(AuI$@e(_3hdrfoN#LLfyMLdE<g5RD^HNIRe!F-lV??T_sge<$}WBL-?Qib@v%9%
z&0RicLz`CHH%6|>>wL^E-n{aqtMcUMi*w?vp4l<HTmNG9BCR{2TRY}R<ohzz)IILq
zuJ(MU^5(KH5282s?hh~KKXa@6%&qb?>3aQFe@-v2EV{J6{LYL&T3_w&W%0UiJo8QH
z-tx;I0(`deUpui_;4t^@>wo^Z&GoVW8}DCVVf42ogSX`Ph9_G*q+Rl#IrU41NiODk
z5Roeova5Z&-3q3FBO+qj7a9_GO3nSB!8dWslgyRxyqw?tKKeyztE@8X^$))1IE*Li
z^svcsMrBGo|7^f^ee;B0iY;9yrZ_TNYwX-KXT_e+;r4-!cQ0N`sC&Bp=(mLoO1rm9
zuTENg_uAF9T}H*L`qoR=8Q#rgWAn-rS~@?n`|Vz@?2B7m4Ev(YxK@j<XlF=zm#)ZQ
zqWsi|-{EtJuiw01Kl47{JJ`F4>zdceq@=}<tgD`Lmd>#H``PcB`YCOhkdg_%Uc9^g
z)%CWbyw2I!m&adC6bZU~U)s0#<K~22Q;R#fI+vck!|1WHtKmb@rW>it&IIo&`fRCT
z!0xcFMQ$r-k8lUL;x#o5n{h}#RbYa+v}~BpY`J)^$$E*>e>&>Qk|le$cL|@I7kK#H
zmV^zzj2W68UY=w9829SLg43y4{{v+f`t{B&D{5PK+S;~WnS1G@yVg8nTcwhpYToNT
zqQuRsaAKqV&HeUvzP}egWV?ORsrj6(OS$fblc)F1$cs){$ds}|q%f1c@Xe+R8o5Vi
zUf&U*uU7GMV`=Tp{jX0H%(I++?q%h(IqB!Pzu4c)YTbP--tXSdPkePH@8uYpZKIDZ
zKVVsqw;`S3&%f%*A1P)xv)`JfPu~0E;`M(zrFBz(_22*X_W$*o&mW`r|9!mgN9OO`
zLvhb%PX6%kxw+OIQ@gUdFP~m~@!xCzQ@>Z9f9=5wtF~3YQ;}FC^n{5a&EZi}CzFzc
zv0>rF+p%kYW;xA&w&k4;!-*xA<{k;vRBVYdjN`iElhfA4ChB|6gU>a&r%&W5hvH&G
zhpH309^sCzR}NlzzwOt$RToyRGTZHX@#(L-UcHk!PS0u#l>9tpd!xwRAo;M}mL6-C
z@p7&H-f5Np{6bwCpL#m`p+wivi=oFih^&&nz@G4Nvh89qHgi4iz_^DEg_oPHGv>{m
z|4pLj_xqDJy+;-$7fcdbWaLrtn)_jFh)S4u<~73=TI*h1YVA9y)xBmp2jc|(X|`4>
zXH|dhe4RaSr;V6peg6Kx@0VL@ObB_Rk(oBVb(YQQbF+?|(tfy)U193^)y@(-r?7p_
zzO!FgbK|<wzf$0_<Ef#-O=-1RYke6uo(i;36b*QJc=N{?x$KELJB?W`>oOEcypw&h
zBjeX;u?AlICGu6aIV;xJmVOVA{;=TDo(D$DN~J@M4<F9sP~cWI*Qt6YcvfrI-r1>s
z^-90G4E=88Z9bh+_>w26!}wV3{S!;Oihh<(X0#}cO;{>r$$yUBr{>dxFQ4vB?w>1P
zyWzigQ}FYdpFf7ZUOBVgs`~TFng0*Hz8|w(eRA*QlWy5HrSEF?8P+`F)tR&JG0V1^
zT^r)AyuYt9_eFd@&-^oL)wdtMe3Hbaz&}6daNb+9e*LD!ci(+<HMmpJz2Un4o_|xz
zKdin|pY_cDb4tw*x%$oT_E*+l^Zn=j_&9^f`_RA)U+)=PSI^XK2)6I8m^JCnqtK-j
z+v3Zwe5`1F^y`0DGQ*k~%U>DoES~x&*zf>{<zqYD3DGPw;%_uc{d=|5%XI3#wa++T
zUHY#R9#OmKw#eKo+)p+f@a<L*G(Wy&Dg$4^qrfu<R+#Xro|>0)_4C{*5e)Y~%ua6J
z^5H_Zka^_IWw%4a%TK=!GFkYtu-}tmrHzjL#U05J6Qgq9_6dGlukXe+_wd}FeEkiT
zcDr7-oO=DO{ZO@Tt*f<tDFfq`3b(ZmQ%pn)PtMxCNKvTPHu!3S>R#bXF`FL$`CGpK
z=j8r6&(|hD1V46{A9?R$>>YW7<G4|h(S*vZ!fQQWwlhp`y|bb1Q1{eDE>6++B<4)E
zOMGj(Yn}eOxO}<2>q=`cUQJuFY3djM<WJKb+I==`y1*d9khCG7O#WE(<#Q5upOyCM
zH9To;%+<TXav-20)LSlOO0<!GWF;eS0^gO2v?^(vDTxnPB^X#9c{~4oM5Uhgo=qnv
zZFw$j9{qR1R>m2hPMPW&$Z2b+uD<?sfpOuAm^O{+4(1`+jA4fMFNzta9e2^2{QIM8
zY>Ax7rzQJ&m@=O+sJ(qV`*dKr_Stnt#kbF_i}P6~|MOv<afMKiv~*@s&P@ihTjmYV
zlWyOSTk%?!p+C*vzR+)4U_h`vyTSK+CIYv_PhM|~+VL}*!EVpaPkanA+uGD8_g=C5
z{(k>-^E#fd7hJdw7_80w%pZS7|H6!lnVTnWJ$Rd863>=~i@kRr>u#H_xFC9#=bRPA
z(ov#EGH(k{VhBjxvuaDgpHna2Ufuu8Zuim`TvxkHCTZz~DOl@0{k>42z?qMu+xU~+
z+^RFZ7E@MeU1n_fd6eOScFdL4`R}()@(Ggrmffjgdt&B?eZ7T7yF4oHx3Qj>xcz}f
zk3>#o^F6(Tm&I<Mp72^d^SSnlEH<}mt`iyV-d=i~@95FC`!f3;^|&V4omziosltx)
zODC{>S8UR&E}N(Sy=RudyddV&$?J3~c58&|akuOLU-WNlec#)Cs=?<I3=KH0q)l&S
zEPa}nXKVR>QoGD4yV$*#RBQg<&7X8#LrZV*EjyEgoIev}|KFcxz5o4-Zl%(cGyUDi
zFMZx3Z^O{{?!uW}RsYx~GdAww6<bzReCuJPM&P#pKK=PCo;T>XY_O>l>MdWg<<DWW
z$TgulXRmEt7oWTqG-{)&dYtPPSGJ`j%fW>K8kcn~yzD&xFVWlJ`JzQahvC~_8HT(x
z2D3i}0va=ym-3VpH`ZNv$+f#G@hq>mKVO(L|7zD&WqA?HrVF3h6<EvQ_eH7N^6U?<
zu5%LgrgiKLh7nUHeEyWuv8gwyYM+SFIYI3+W{Xbmx?6d{Ap5rI_k?X9zaKhla^l&?
zpXXnnZ@*Swt7-rDz>m-Kuh{*p{Ji|*m-GMAesrh*d_H~Ul(`Q>(<@B>xT?)t{iRjn
zPsXIy#X2&3Biz<%R0y&ho;>;CNzX5TjtBoVv9j#;JLk6lS7qJ5?x!gicK^6pR`DwK
z$$rs$dopiGA6B0{UH!y=o#=li(f>pmZvHAi+`HXRP5Suz+PCL}p1Xg4_gCxdOM8(s
z*IY|yOyto0G|7N@u}nwv=^dx@c;&)67g=9e^-EnV>)S$+8T+}KPw+UcpWwB0ucAl8
z(psf0_E|sYeoR_;Wy$@@sH*BO=VF)niglUBvds!QvP$)o>9I+Xv-ez|d~wR`<Xd(o
zizoCfyR(Hu@uZF}?>ZB1j@+;7teYFo?5cF_Z{?le)PLe_t!h#I)s>FYNuv1|pSJ{i
zHj7rSQP^@wwBoIVkJr}1w>usOe!h3md$;pFc79$@aiOa#<|IW=TU%+n!n-kqw{FSC
zkgp6EFSoh`e0^)x&}+Xg)OUl)rAteUO669|6koMr;xK7T?uo7ued};(wa%2DLaP;b
zIT(@_9E)7%@>cZi|6SL<zSjC2|95r$*U<g#rGNh?-gGO9bS@G-bEU}2e{JO#ory|w
z&c+-|Tj-O{ynmscjkNro`jE#rjWVwrc7)s$NS?1h<?Wf<b<g9Jf9d2Hd7q6`S(LXu
z*Erd3_7C2v_qb<o4iWa|&WSpC;>eXFB@@s5W#^YZBN=n<wZx1)=HAP~jG|ON^PfK!
z9XB_w<mQPTmv_zTljgph)^;X1b^G`I-(GzD{QUOm4d*Z2vROF!qrwhH)j$>nzthWD
zj~AwON_8<DP?~AJE&3~~%<;g_M;@J<X`N}46~TAYpmai8xct0pX<Pq3H-Gg?#k%&I
z%+c=0o@It|UEMErW}drH^vp9!opTZkL!O=<vtrT7I3L#<CHBn^qB?HX$g#{gTOb&7
zBJ$F+hy#%thP!4-o?Ik&>|;$}|NQocdrYHs9`pAqq`plJ6RzBq+Mjm+`TYQK<LaAV
zqfG_Re~UgBDU<)U?A)V@S-Z`576;xJsQZ+&P5;l5;LqpV6ZhT@J^8flK2N%GUi!=z
zTIpqpbLaNXKQ}9D<CmPn3>z9)a(0zUiZR?}Ix41I%Fie}Rpft6$c@EphnZIY7T><{
zBGZRjv2z#Z-H+OT@InH^1%p2ks(Rk<5={5rT2wQcL8>kG-{k2Ar}g6JUYp2$YGWI(
ziqYnr<V5N26qBv9@~pboK6|#~;SD#FydT>HO(*FF#&3&0#WSlnY(@{$>Y1jKbf0#l
zh(!37AIw=ZZ;wULi~6*LGqZUaxTi&&^{vXWiq89SR$~T}N4jhA9FH|WZI~J=&B80~
z<}*%kvf}febaPYT)}01yn^X7SOyk>bGfkZ{clGSd!`q&7`{?T*y7eyadqQw*ote1v
zvwRn&Pk*;OobmZd*qf^3>o42#?mkd_dP#25tc9l2kH0#R;G{cgXKFcb-t*3jTQ;xU
zll5V7?=hBGwcBhB_u`++&+EN^qrSMT^wR%nxvSeFl{UHLRF<B8m)5!AO<vnIx8zGR
zbs0^n?wyv*w`7$w&#6~DFLrWIqWL7Z#fN7#%vCwL>0qoFQ_al{S8WV!?|tfB8B&w}
zx_7RvLEQJ*JKTEDYfIO}zW%lK>a2rN*TVz6HJ5$heU^S%(Y2YARiQ+(d#aJ`z5{AU
zZ61r>pX<~dx}T*Xm(StZ%Ij~1n_a~>m9WMfUy#+c!kuGtz%j3j@tXsTB4hqtI=y%M
zeYO0c%Ido6&HF8bUR>u1j!S*0Yk$)EsD%2wX2W@{TpX+GnFO9@-D(VY)w{i4ZMyqu
zi}TaMPro?7c>4dH?dfkn@0XZob9#vc3)6&Te^1ezvf5XTd`zYsE&uu@m{vBO{5;=F
z^upZpZj5(16F&V}-e3Rk&C8FQm!Ef8Qy|qL?l_@US@AQU@Wqpn#g$u97ku1NUH@h6
z;fr~X3(J3=p8f3d>-RnO#q#yP+8>L*d+g62?fm`Ya`)YLwN)nnKC7*AxOe-y33GhR
z>Zbk*zjv@OxoDo{^Hp;{{M(t;zvt-rbM-&WPyheQe(?PKYxebT{;fQ7DNBCsrM$@u
zTkfq$=~sz!FAtSe6WX<KQPZWIYpZ8{>1z0E%6ssVPw26(A4{5ccTK%g_4(BUrcldk
z%ky{Y=BHfKWB=Qg;J#~ep8BgfUm}$A7s~$JoR`3MH|W~3?QFUt+n*%5ovu;~^l!Vj
z^+v(=)xs-VrNfqPVe4+wHECvDkUZhb1%uR&Jk#CppOD{S<ez!#aj2x&QS}h-<xvZ5
zgrxth=wp;UvHqMymU_8Rz@+OLS6|J2<@cmGROrE`0Qr}9HY6>L`&hyoaN$zioNiBp
zs~^2O+n0XXu&%V0ackO!YhOh}=BbpjDIO`2@=5Q0n$ocPa<*YC!;yr}(~ou6^EIrC
zVVp5z+xpq7-j&b3eyZ$QzNykp#tU<noV<FXEK0Zi<{5$Mt$UQ6Pe1q;G^@lgx8Syy
z8$*)hPkuS387BpEj;Fa@$~d#mk3pcBb-}*9$;QkM%!0|^6axOvz4s*hKEnjtzs1vL
zd<lOP_-r#L!|lva`vq#@%jW+1r4*;q>V7zFvnWg5>Gp>*e&zgf?Ehl;mOWp5ve>%n
z<E39qH~vw~<`89DzAf1=ea1P5H%u49Ly9YXu78hIVz|JxHTK!38=H1B2#I80woeb2
zmo5-$eeuaCC_-!EdBMYR8iJ=?jf~8TF58&i{n)IWs#4Ldum9}xij6$aKCgIh@_TaX
zlC1An_9!2Tei>u=aJSOTXYbNpuiCh?R_yrAck^!C`Vq3^{HI$c<;m$^itS{g6TfeN
z8u`H~@@(yvWyiM66h1vkCxQ7=s+nN;K6$lH-#-E;XLnc?&Gg+f@$?0arBA=yV0~k4
zG9zuXmX7$|BX{r3Z!2f5JZ8apY3JOhYkiF~+BOF&f1F}`+9llb5YMhtfhS)a%G#Bd
zAYr_0&degIlNtuqZ&O5$+-cjfse1ReL=E>7YjUNzcgdR9Fua}S6=J(JS|FVB{p9)_
z?<m9Vicf?t?s&K*+AA%$LwohhQ)f2yW@PZLJMM4)bsnGed70GPvf6XDNUs%eFyeZ1
zFVBdJVaevD`<8M&7h-4?a;v=6(`1=oT{cOjw=&6M^XJSx=~sl_{Mh%nc$x}l?sgT=
zXIxt=Gd84!tl$35wxq6l&K9}!{4?q5=3kGQd8dU6t$utxciVz`y$zOA!+Ccvl+?Lp
zzcTQ|o4q`1rOtQg=6zVKBlu#I!`{rw$<zNQZ<%}bQnjKmYrgID<Luw%U)IiMxH&ES
zI?uHPsn^YUHy`vKRT5$}jOaajXj|l@1I5oAm~QZH^AI{=^VVQ<;*Z(<-?x>0lQPpe
z?kTk84|t8|l4}gvvGI<so0puMpb~!hWr<bvk?V&u#8f<;c&5!#_gZRSYqf~Yu{XUk
zX~vAcOB2QK-B|l8@@`OC_4~P1BJpO&{qANkb2QJcxVEItX<Em!xd+QQLT1j;oBVA_
z`Q;f`XYSp8IaVj({F98CTdZ$u?J|1d<M4vZVgGK$&sWmiR;<}6QMUVS;^NrbcX?l@
zU!T9bJ#WP`AI+09dW8~l&7!Y9EZv@Cd9%!1M42gqclSiK>brW!s&BtKxObvcow0da
zvUiWq#Orp(HzOyXOt71+vti~rwicb$1zz*>?JBC=?)u!lJm>V|i_Q1*D)iUP{6D$B
zt}_4iv_#E=I)}0<C-tNXX|g!6C>csjQt>>T!pQ!pWKB=+Nsk-SWfJrJXL~F<u%Y*-
zZ?fc;#f?!sw@=N9>GIwB#_a5$v^xK@Q-wD*KHi=cd;R6EbDqB}0y2~9FWmVSQg`vg
zveUPG_Z3t}Zl9SgF^eJQ`>jbT3;U!KUdts}`yX555LkGzRCj~;LAARPIX@F8-6&tN
zk~KhSo9L4)gDb_BX=T>auk>d>_Y`3X+ni>a&E1wFGGp(RtZO|pjc509*w5T6vO&62
z{gwCBm`iiRE^nPBnXr7b#n$M1cg)j#u1pswIw1G|!>5<r|I`;>(@DB5`}*n8(9OF!
z)~o&O+qrg<_n`=HmoOvGiu+z_VMd<mr^~!;<)hEKsN6eyG_-i-xjKPAHJ56tI&*GF
zoJq?sT{Hb%!TlMQY3qG=ZCA}p{S$bzw{P!^bJA?jBAX>2uHjP&@?a=3>lQV8-u=0z
z@_*dfvWiIz9vP2yq(7?!KFmq`_-oDQ$Zz+0cVGW37rAm)%&Lu*qNZtIv$_lBrSKl_
znXLEq{z~tRg4wFZyc;t5*Cso(C$>#XE${5RRdwM;-G(_!W>#(~(AQC$Vr67<>C*0B
zd)MYh-K+aKDen5lO^(U_+c~b<2wHA8>`T4kkY~MCrSauQw%OBprp;C9Jj-*tOegQR
z&ey+hm%WefOFo-;D_io~u6+}twsTzn`gD4ZagVp>?|GKV8bSso;e9!GI;SeuW{0->
z*+pF1GT}YbgvwR!KUXdK@hZ1BeQw}vCflT^6YrG%4oF?NP;|$Yh6M^vc@dyZ{-<Am
zOP%*^dlJJV&%!Ll#@vTSTFN1Z3Ri0-Tu#1fr+m}r%)WnXPyYKmz4_>ur!SAaJa28k
z>&yHzz56Y*_usgBHb836rM??mWWT)ixBK<<_xrlw|DQqQ36C!1{Vo6Z?0vn#-uD;Q
zSZj8CU0|*IVx|tmN)eM^c06AE{+cZz&66j`%f!2@r#tCH2P&lGbxqfr!~S5ulw`W2
zZO^&g6B0}EPS3V2Hj_CTzkgoMVWIwcH4p6mmDvBef6wGi+voI;r`1pY5MhWop0ItF
z$5Q>BYs`38N`}8WWyvu2(qbde*rP?(-nPfxt9u1ji}`nMRQcPPmsy*|YBYJ?9s}Q{
zVz*Cz3I3J8$8=)k>|2#(mYdx3xP?po3U8&XSJ-SKDLQRxh;qK6%+cxRUdG>T(eL+L
z-283*wB^ft>%UKaowp-9faPBL%NI7qZHs>_f4lu;f3iiF{k^P-hR2_qa~(Jyz4rev
zm7gg!|GLjUUc3MMif!AU+i|4beX`}_`zJ?_xy<#yV!72Q^U8v#FItHu>(*{qWMgXE
zSyU$aVef=M8AVo}h`1@OnPFcx?k!_E_T%%5pN|;U=AO=3#vo?7?OyH@TbI2bnPy+r
zdl*n5UDt6*fSW;8=-$jf3k3H|o&Vza>)-DCE-#z6{+{f#-!&{E<=MLO(pM7ao_;*N
ziC4;J`#YQceFYk9krvvY=dD!fJvZT*ZJ1^5fu$j)=dPu_d;O~^>ukc>*WP-ZA8b6;
zKB>$<>GE{tU7>LONh+R`T+Rw#TBO$LtRupiY`o=_#22Uex<{tGy`3lbZsoq)Jx<r=
zEZOXGGiT$LY@L~g)&1*c&&};Wso;G2$dw0XoNH&5u^sqWoi*=Fy72M*RhHf9XLSVi
z-p{#v<jBmgv(z4{ymD00%D?;j_(h%F`+j}c_jQxtRmsiDf~GQ)4Wn0vtUoy;*IIY!
zlNi;Wn@gPK<Kix<hv(Ov{rztH<FmIu#@7q=H(guqf85B^VD>qKso`z5eW_bKo}Iau
zeJip}%rGLv#j*Comz>3hmOK7)nDl<#`+r;6zt2zh7Cip@^jQ6Kv+vXQ-rp40lfKFF
z{*B*<kK6yzHWJ>oW9PEg=HLBwpDyZe-%~Y>qe_d}Z=TAu6!V`wXN7MU9(ZFR{?O-+
z&(R-VeoJ|TuO{%G$hcV$6=-ng_S~Z(H>CCTQx6N&E|+wBo}0?_LL##^D1Y12Lvx;n
zO}jPUD0`lIarB<AFYe_>21zofnP$Jt$u!*z>h_o}+Eg+>@1Vu0XAhV*C4OLi()6)R
zuk-AsjGTiNE8X;8X#CZ9B{wO?)6%I}!X@%5_q$aSy85@P_50h`T>JZB`r7!TvSnF&
z@3^n-?!0<ud2xYNxA4982_@C@EbqEj%(J|FQz>Zc%bdXXW$lZ9-27&<e`fpcW85#h
zn*+1e*X(S&YyA9K(Zs!izT4No37c~)eePQA-KDA9k9lN1Je7Ac_46I^izUDBi0^lK
zu5`ks+&U-U<??!K>8(-Lrf2Tj)Y{AMz542k*u>EHW$V>;O4)Ch-Yzudr8a}Xe0RCc
zlM0UQaZ6tKY}Lf`>86phlV+(n-F$II{K2$zi@x{IPv%_exN5U8x0<ox>}MBVg)_>#
zOD`Bp8#cHt{`_YD95<z%<?nviNv$&TPGi`Uw?DJp`QgvHT@zXzx_YF=Zm3vA%1Yi$
z4LGr3(#(DTo0q3Y^Gp3Jd!4mG{_XtjbAP$ZAKP1=b~}>Sp-O*C+CR7CMV6Vbw1T=X
zJn!CcaY1w6iL}e7oo_CZ=e?a7*nMGJ<jvokrFCM=D#T8yeb4?}x$Eb(d-weA=1;n~
zBGY<*+x=~&YhRbGtp#m@6~FFn<h?UfkbS}D2PyOP{EwtfpX;r&fwNQUe8;VwIWc#Y
z<#qmZNX(rtWuUi3)O77)-Lf4A*;F}0VuDUP>MZ`&%&x~hpKTT2jx#m$^q+sS)N+1y
z)o5>x%BP<E<FmK^dAic}cG5`}HVw;`oI9&uXaDMI<MP#)U$xW9$nMwa760AuTU-6O
zmOQo1{{Q`)oZQ&G7mi&^*())9`pd`L{ls$H_lurT@BUSBt*(e;LA5RCwy2Ve^?Pgo
zfAOy`_;CM^V8iPB|8{-9XMgh-yE#MFS&;+(eji)0S@z!>`>+3BRA1k`w)ySv=y3fG
z&IQp7<zh>?6bu<$`sPHl>CIfDyYysa`J0vBj4O?V+HK_>C;YTKbdmeOnPswSPK}!%
zr0wUfEIYHO?YPd4+i%Z5e!Fi!?|*TIhTN-n_kZ73J8A3s=<SQYnAnHhnbe)889V(J
z*TK||(>5+e?OL%sT$@(;c<k96W%_dcj?HsV&wR5;dWNI<YqqlWm6CJo7WJi<UfgLv
zfAi%ZmEX^QIyZY-ZLw|9?fiJfn{BmI)iUn8b8UH_fA}Bkq{VmNJxTK7Ti4$Eu=Be8
zkq;|pKDOdF51$jZZ4cA0aQ(Vx>HA;ozQ1#l_{rzz-}YKHe0EfvbUWc$Kq^B~@}+R6
zRpKw7NHsjHtNmW?|I~Hc&u5k!)b;JwOFp~&XX4YhTc3p1Uq5towxfx~n=Sr}3fFFZ
z={Jw<L3@r*pyih*Du=xeGaQ(tulT@FOsjOB{`W&sFIKOc#IWgt^1Ss=Ha<vWfAoC$
z>EF+9++MwtUH;#LCc`%@eqSRlTmMO$-s*T!u|IXo^|$R)WK@+IzP%{uQ<}2j?*m4a
zR}yC$zkj>UWwGy$?X8_Z9{x|=x%+-a{{Fif;oIEXXYKg)A;)L;>UI6bPp7?$Nim<*
zdMr&`(Q0Fl$opAY&adz2o(t046p-pQEs}ecvE-AWL+_t?nf!@%W_TTSef?^1L%Pm$
zuZa1M+K-$Jr=7pWoU&0}e8_P_QEa@I@QpRe8;tL>&0}m3I8lDP-R*5;?uut^J<j)1
zTgCla4wWyDe=ParRMM2XcRLf0YOVe#BgByYX|r{H-?GU`k~{ZY?&%TyboSq~OK)xC
z?Y>W|KiS*PAn`b{;Bjep#x@U~EqS{hUw6Jbt#Z|mO@FOp=U9BbfA>ZCrQi2|H?LlP
zeDZ@Q&)Ds&^7r%BUD=lxXV!PHzrmy2Z2EoEZToj_tU3KA=lZsPorQnrJFU*%U$ZaO
zW=5R)S=s8WJN55(&;H8Zf6sjP<jaBC-wTwpZ|=-uHu%WGaKD!M$DZf^ycxbeKRli5
z+~V*1|MUMp$@`gq|KAtCo{Ahe`0a1@gD?K|7yd7P|Mv;I{i5mb{$Kna^WX6QE$I`>
z_3O&ECR-J;cz7@f`7eF2=68tg?Ki*fe6LkyeQamWk>+~pKKJ7b&EoL~@7|lK*mLgx
z%{TI~R?G92w@=PdUh8@9_=*k9UKR3-xldW?q`xaW&Ja0m^2%j*mG}EfeK~w%N2>69
zBROq9i=KB^*j}XOBt0w#4eIkAV>@#|qP$V4RCV#nyU{VZ9eYf(^3NW$O3S^y_dCOn
zlo@}xlbMANxK7S{sdzZY#&EXYEOUl;=IRG7=6(Oc#9+5~2E&V4y>GN`Thv(Y@rt#U
zu$#U6XXV?J+b5@=&#SwTo~bl}LE3H~8_(%uBGV5&s<TcBV&#u0|E$O^Fmry-zdtuW
z9c*pj-IKA@XU;zRgXdpud+Z{9;9W0gNmYj%!;_oboxkQ)>N1{-4)U|e|M*wAvW#cJ
z>+j5k!IMoH7Cvg561Tlx?#>&=phIhVE)@1%c3}K)V}@;}{r}4o>T@^$T)?*AxAC(y
zcJ3K-0}icsPjWoY(O|IS<E~n!XubD5Orcf1ZYy|rcJ0|5vGw<2{`1G)%a!((>E%s2
zdQAR$NWzB4HZSLHxO(0>an@J4zD&peOmEI?KR)A9qTfdg{#{$2ADm<oDJPx2@TleF
zphCUb6XhFc&vSV=J#3EykA#xJ4e<7+Rp1=p__Bhlyz#uTJIBKMuK_KYYbKo*d4K7T
znPCLOgWm>vZ_;-=7js?_d0%vxt8u4IS!H^o@2**KUcuh3FJ|3-Dm$^*eu@v@-n{uN
zk|DQ5ot|m#o%1O6kPmaS;Wf5|e!i*t)@=PnH$T3<`k5=>LX`8vs};+B6n(V5eDm;~
z|95A-tlR!`LD`?l^JD(J{_l}JX-7+(UvM77q<beber9}Mm~f*_cE;AXYwJH)e{|j^
z`Ztd+htbmD&()u21sJB)|K)7x_Md2RUEJSp=l8d7d)s*#<p19OU-|Qk?1SX$ZLABv
zZT%jv)$sG!u`@R+c6|J6&d9RgioM{=*6*KxJZ}D<c6!hCIV|eG?{7ET|G$(u{4n=!
z!5h<O@c#caJ@i>ajXYax+Ue~!iheI{f2h@4Br{il^_bRPz1i2@s;@Ei7CmHhaEw1G
zeKvL>$Fygn|9w2a3*T;05^l>D%gV{fSn7M8vqkj%mbuJ(V;PQwOs#uS#VA~w_kEVp
zYv%s1>}RHhS5A}G|6g#pqR1qFVk1Yx(-Z}!+waxb*WaEgt*l-7@AIv;w_EPdToS(Z
z$)i`7FFsl*TKiAvz=O~Ax9a}AU+3P=u%-ME$4%MPL@%L|1<$lKj|3T1%0%4RzV6<+
zf6sUvCcibgc_p=@s@~M##^;AhcMHwrztqj!D*iDezV{Btet|>ZYZa$0JKmDzKkxpQ
zUD1;hZY7>)P}#wvbDy7eiZ)AO=p1H;H;UX7TZ5X)dBiF^=IJt+#GiTf)#T*wUWIM$
z?cNE8`OhES)NG~ov$;g%Me@Q`eM($ib8a5&|Khk-QfIE~i9XP@!TZnoLd~V@nSp`R
zm^JzevS*)(G~4Z(!1tGfq2tJ**G}nzixf6yOl00*F`q4c<)6IA8?40{H-JX)TCYSk
zZ(XOqf6@;x2G81yCVM};`eGt>{b5epjk2QBeFxMyHAUV}`laR1!w_7_y1+R4*4kY`
zI(LGPE!-}UTQ;vUM@B1q=9bMaCU4k!JGRSAJIBW0<0Jb1QoQ)dn(r6Fip}Qx<}6RW
z5qv3^;evCe``+G@IU78l&E9u(hVAEPYrg9qm)T$Qjd6PB<(c{k%$&(J#rq_DlGop!
ze_m~FS~lbCGZO93LjPVUxnCT`FTHFtvrFLx$v<EB=kEXc@5$!r@851e={RwlYqR{s
zeXIH$_ilV^zhd>`_dBa?FZ(e{nEw1Y-=%xG_{V-buS1s)w%2dtm@g5YXt#%9L+RH=
zwtUX_KRs4_^W>BLnfZT@R;pc|JD>G@U6GL8?=#j_zkmLJ{QphWyOK|g)n;>f>}J(h
zh|Su6K;3+e;^oQWZp*jruHOH3Yrbsl`?t@`<*waYd2z?}dlox>Oi2Iq^J3WVX8HM(
z_}bbeJ!Q=1s;W89wG0sQn8cD$Hc4aZ6XTr;I-B2Ctq9tEapw8OKW#Qi&;5Kxu`<9a
z`qG^ktM0dP#*(bPDW~5oeQR?v+H}(k6>(pNQ!@oyx|&vAEfLpY%-I#|xjw?M&VxJc
zLWp1T!5m|QDaunWXFNaIbaUzLrhs#2cy=9CNiZsuoqqFNWr^wZ$irrD5+48lzI~1V
zo0mBo)6ZGCtvYu8t$~T5(WDBS&G+to>iv1v;KX(@$0F7GpPTp3u-mt(=J&q0dh7h2
zJ=}cpbm6a`KT2|X;=di!uiiaBCenER8{;nyFPyw?Nxz)`Mg11<vdt-0-cCQYPrp02
z&dzMIo8H=(DDKK@#<#RQUnS-AJ$mv==<sRr>4$R77+n0c@7Qms^4+-NeoVa1o|F&g
z&Y$1(`R2jNUz7|I)Qzt^yUfa?T#}Y0cKh738ph*^XEx@Zabn<<ogKK&O2;c9N3W-1
zm+@TNxAlMLw@sFLyg%a{*W$oky{RtA&7W_&EM6Vq_Vul-t*Zz7^tEzg)qQ)iZb`~M
zX0ZEZ@b6ane#4Vbmj5?<QGfc{=jJa0x>uF<vz>l-&wScv1*Qqx<@IizuDEvRjnSoq
zix*Ai`@Yb<>36#L*^}d^rM5laprZKtVrH}HXO+yK|NbqmHZ|dS_WUF`=Rn2@>r}aK
zEOyZGxuSM5pf9N~cJ_JwqZ4Fqd%l>f?k3ATY5VJm@0k+bzS(ym)c)p-S>lf$xILZi
z!}jFTy0>LQQztLFc4UgG@vrP=`Lh!zKivIx3s1D}lL<B=Ce2r?J{U|%xs!0yFZ<Sw
zo7KhjUp`%tU+(2M``jEuX%DN1Z*1br{#2<fvRy28yYDnZ^-a#up!dnea#}a<G4?Dy
zb$s%JnLk|k{@q{xMacT|+o#8-rIx>(v+HK6Zu?EY>`&j!quBqh$u_kz$olkav$(fl
z*}NFrgqyQhZm%}Ansc4+x#_&v-_DQM9@~2V&8j0e)06EIUoAhDm-PND-@CKFil5%E
z|9w%pE~9t-yWHIE#~&U(?<5?oclo&epVjMoZ}->#2%meleR=(_=5x;u_y0d$zc+&?
z|Jn0izo_l^&rG-7Y<J7%*S8N{M}@bSe|W1dUsF<8?5l2>_WokHhMqlNx=xVU?8Doh
z_nmD>SvSctXYs;>XoeF$cV53~y;yla^VRzMUTPOY!q%^xBiW!aBkHX0>Y0q4SJyDw
zNV0|%8%xGMS?jy{<vp>r-U^zrSA=)n=b3eCre->u%3GV6=F(dX%yvjhJc(hwv9$7P
z>)q^So1ZS)yN3H}QFZh)x%qFmJk06+bL*;i)!|QXe$?DzSvK*L`B{_Ce}BAs@+wO%
zCN`$?WOsMBqe1cRZC;anPF-|05>fJ>wA1FDs_gTh>3f$jHyq>UN^TN59^TKM8`gJe
z>f$Q(#PuupbWK%GGoEU3(5K|t*_kq?`t7eiZ9f0xbnDi&b87BbomgA3adY7_-?jNV
zJ$uC$r&!Ko{krn<40X>WhJ<3X;LU7@f112AiSB!R!)zMwib|oUs}4pq8!Emoo_Z#C
znp)xBjkc8^5)PZ`$8X#7@u<zojFy6VDYtg*eSE!f$$_}C<}dF<<#iZ>D_!Szlr21T
z@F9Ol+S%V_cgq<cEardw``z^~$FINAPO8-1Z5>$X+`|~N<8#~H7iG7b9lp+5c5l+h
zHJ;)jX2uyuSQ>KA%O3X>y78?>%;<Jm($XFOkNK7Fe)+p}f9AR^A3!bs&EWP<neNjZ
zMhWGMMUnGV^ykg(UC*p=u*UIo3B!vkPRcE8+tb6<_1=GRioaUB+^lntp3d!x^*bx(
z=!rdU`^4d$+F7v8R$Eec^0uixD<mByyoHnZ&Yd<RRcGebZSPYb*4WLS@Nk;hl%k$W
zD`}lcmG$NE&sJ7_Feu%dDwAZh@}zj#W_AV-u|E6xzFe1G^9<9>CD<HU@^qg>yp{IO
ze<)`%!<7A*|NqMdlV5%*-1qcq?7fY5+Rci~sym<l&`m$LGvC_!rS$!Edw-_I*n71d
zpS=0}<v)vNZv6GR_xJVa+C8QD6O-+3TiUQJd2TW7{LW+xzSm_lpIc;|vp(Cq@1V#Z
zHgBEw-lJ#Us(oFOZLEGU=i0i<m9zeA<S4DJ{XLhz?uY!n`tP&-|DQfDWxf2dXa4iO
z|4*IiXPd9S`Nij)PuAbwIWMlf|25|Qi?^WhA}RYN#`0V1a(X7W9ooaG;yKBp@3B;8
zqV=}TOad&MJC3h<$M!PkW>WR$ml|_**FH<L>OOt(kP`c`tZNLj_bi)qvGwX2!G%+M
z^wvCEHcx{^=&8wzZFAMRkDXm$nSYMqHqWfjY0MM0G)bC7^G@7urdccL#I(Zy)$Do3
zb&m>}->&@p?8V!gPk9-t4Q*#8TXL_>SoSQ!Y>v(x7o|OWcqXY-R{hZtNV2*tx1n>!
zS)nt}KFyF2x7}QspZ)vMv#Q00ifcX`yll*LbIaz7r>y4jtFF6rZC2IHyv;5?lel-O
z`f2$0_@0Trt}f9weg3|VPn}<1RL{O&mi6{|+T!%9c~e5y>qf|#q;+0T_?#fdprF0x
zMw!j=jk!7urv6erj2%TwpJi<L8Iw3KWrJ))Z=3x|yZ@7qt?~2y@YYpmWzEq9)tPtS
zxF(sZEvmTg<z`=N^{W2#IX&jwmeuafl2!6Hw$;gZzQ}Lw*Drs3E%JHzvU?&XkGu-4
zs_HhJN+?Y`y>O!T)Uw^$>;jh`b62u8s2bnSEIh}NeACQZ=#YPB?&b*7m}60O$^Yjn
z-+q{t2i_It2wuiM$@>|nPRH!bM^_{^KNnY*k7u9GePBZ_m&oF;I=iz~m5Y+q?aOD*
zn|Lwf?)!ZO=lQnn3Vf#1q{z5t<(?q-HpN|6l&)-VS=V*p=G(c)4)w4a8?KdBYk2rn
z^&;nygo{1qlG;z=*c(2d{m!_D$L0B_na@*tZi4ol<SaLx?`tORw&c^<u=NK8o)#4+
znmp?WxwpBhS=#VN%h|&>m;W!GaI)k_{<Qc1>e@M<TE8#dy~uRF_U-(lG;8a}d&{%#
z|66NVF7{EQ+VaM(PV1y6>Amls*Omv}jxCyPxifaVCHn)5eQysG&v^Xi2A@p&@wwNl
z{Is_(zjp1M%(lee+s^KIXfk8+twSY~raj&+v;6UUAq9ubAM+BHeZ2Pk|IbAmi>>y|
zNSQwQ^8YV7e>T|fx0x9~&*Vq+vtxVD=6{Lm-)EXwx2f>QkM{bn(cc<<&NJvt`ss9P
zXOU0ZZL`(-&rY|^RWUxl!SMWyTt<!q+as;q^USRhOX6B1)AV&Oe`&H_+&XLes%O7K
z-`aAf-Q9UMdNxB8uVWeiDM6OaQ<QI6K0AA?Nj2?)Nwp@|N#S$G%V%mN*9Fhs(3{z7
zbAh=)tm~=H87EeSpi{9b6N8*MG>_aox-{=wsC0eHG`+laoFR8_-`cl*t8eivc^lis
zn&-ZMyDdMjcl~15(iM~Um;_!5|CIfI#)F)zX`AKEj)gtE9G1Yh<&{Lq?zd+RWX{g=
z^80!0Yw4sCv$}`(w)f0p`F6%FS10wyLX#PHXQTvoMyPTt@Z>2?z3q2;uKLRDlUq4W
z*`<5VidJ0SG4q^~W87N4+Y+<ie%FiNHs|A}t0k9iHvE*n(sNm&n~$AgW6zAe?ri(n
zqu)I;TKIj=7r!+#{dvS#n}6{x?c=#=_U&)xV;kl4z)vL$&aZpgIqTpqA^nrf6!Yz4
z@^U5$F)R&ko~6!PZMjEi%8`J?4H5~z?YxDo!+Le5ynQQsyf7z!|LwO||315wQ3Aa|
zVc(NYizAJuhHvFOBr`eV&zsBQeZ9A8>J%BGb(buW<1&m`V(zY)uwM7KU+yB2#Vcea
zZN432XPD->J2O3}%zt&pYR~hY?^{}AOV35ia9gr$*n8Pe$VJJaa~)_oRnE?o?xlS_
zzm&J^-@LQp@LrBve@~wOyk+~IU7O}!JM~8E<bk!7lMcq2&Dh&;*JIL*-@Mf;pYciW
z){?6IQO~=)|MN5D;$If`&t-k*Ge7fv(VE+lZ)TN+Mce&2e)DkOO{2q6wSxWMj-G8U
z+|=(GU%Knk=ee<M?R{r&bw`~x&o18lboS{sqnzU#Wbds}^}PE%yLdg@y$pkU34Zyz
zU;aMZ_i(n{tbgYmug`4$KAStO_;z8HopGemT+ww=S9fjSdbdWb_MiIWp7g`@o9EPg
zoAgiE{vY@C;>k}XzA+a+zIX81-naWIH*Ng=q`7}ZrpI%8_U}2&r`cY0?@YX%6&G(7
zlDyF1{H!u7_r6J|Rxzz9*|w$ntcF62t3<a@g5H{E&vJfU%DT4ZnVQ1dXKD>4V%e=?
zidvTry-xMotSGWnId8@49SLPmGtDnYDki9>&ba$R#k^T1(&%>3!x~HO<wn89E?#G+
zvTR=8;*l2aKR>qPbpOlX;P&Y?M>&iyrd?rp9lN8x^iu@$O07#ue%`?`Er<2;)=A(0
zoj<?k!^MLKm;UCjJaAuVwtU^)s_740w=U1QdH1Z|>~jw`{Y@|nzLEPoJX6ScXF;m-
z6~E1ydEb3E3#MOYFWezfSGL(Q%jNv+K4q~3K|j2E7|x_mepa^oe#&XKD|hlXR8Cpl
zcd+GzbPnI`IqH($X*0KOug(Ad^yLCuzGyz>mx{Tzr^4M>6HMQ*S?&xj;wuq&_(twj
zg!-$;^&h_ESg+RQe?8e`b?eKo&E>bknbkt39^AF~;H!4tZ2}B8oR-8YJ7-_~AUiEU
zX7^d1V{bfImdv}dThf%h^GMc)3)>>?p6k9V*)CtQ+tzfuZ?){y&Fl1+Tnlb_{cDo<
zHc6I>81b7W9_5^h3_sdrSkz8kc)Fy#>7pX*aU)5A^{GAc7e?O5e<%B7$E}5_hk~vi
zJiJTiLZ~cr58v|zdOp)yTjk8c=AG#_cX;t?@84~Y?*9Co%y9X)-`|6cKmWRMAJ2cc
z^7a3-v)Jsv{bXcF+ThXY#NAV<5R#&=;(DfbI%Auu-MN=D%j+*$)O?qH?Wk0~{G09J
z3XLZ>pUrOnu5|m|EV+hro;Q-?=BCQ(ZMtB-SBW|0d=F#yp2IqP4|9J$H2o~a5MuG(
zE@O4UJwqLa4}woynJ2$Al)tMJ((wPogw(|A>`F_!8tvm#gN}Jk{oHUNE@uC6_K*IH
z?*3(C$jbctWO4kwe=qqNCcM42{M^>x5BReW*#FvFKF_{>-rhau?f-sev!A5(qA7Ku
z|GE?|Cl9`Pm#Tvw##BVN&)vQ90K=q<TaJD+V$rA(4qM#HutG87-c6lc*%$L;?3?fI
z{ynFxe|vGyZcEMWryNq(GEctx^>D`1_H~CUr!0?Lu_%UPrIk+4l<SG-m!2ryvh7*O
zmOTCgzu(t=esb17ZppWc+qbPzcqw1^-u}-6_wox<&93R2>9cFCON$QIU)T6gzIOe;
z+SPm?{)E`QjLW}&e-ZD&uo*j^buay3>3mk4!yt6NBg4}lxu1{bSp9pkc;T+^Z*vna
zCggv=`<`e1Uz<=zaUGt#2B+SA-#_p&Ed6_RZ7x%T#G1Z%eopnhr89K?7a3UbI9@t@
zG<)si4_;QkW8@aUDSBo7T4K-k`}->D-z^N4KhKa;|6$|3ZSUW1zdEr|a0^?v-0J5Y
z|NolQ`@8Y(dnrBX&W351T*T*WE>;N1jZ<^nKkI~?cE8Qr+)Cy>k&k_>YxjJ6ye2#L
zzx{R4TD$2no+i%oVy>!L$j7uFo?iGt$y10S$?Tl4vNwxi#I~x4$2Ws!EnAzx_TlU`
zN26WyHvLV^UiYonqwE6bv4g(yX>&@Jf6A)M&pcbxWpnxenF~s6+TJd6i-UIPZ`gS`
zwq^QZSEJ?AwUg#q#(S*n>rq|*^ZoL*viEZg7OP$Uqhxb7)a26xcco9e{(88`?zi*b
zR5{7?O=|N;)|{@2&2z(=69w)}Si8q>?Yq6U-EvR<yx-n)Zr$$(YZpeYK5Bb>!=C4p
z)r{q4>02F3noxb)IxTr&&bypk+c}#x-`np<`8xOCwp^W>Z8OYy7~<0N&n=VN{EW9b
zEaTQoo5(QpEs4g5)2a?`oSD4&*CDf<>fN{JGPm0<J}aANapq;|pJyd(!og|p1%sV@
zSFg|CfBAK_bZ@7vP35P9?t7c_pC9h7sW2?Pb(j4%_wTxI*RCJh`DofhNzMr>own_{
zCNr{Cvhu9^D(!ORO0tDY_XqC3thB_a+S&L<tdf@G8nu(RJJ&1ahAH1%RPZ$Xt*7_5
z?CCw%*DgE#Hs|1*Nylepwu+TXT#j5*Jn`k<8{IE^JJ@%!`#je=?<A@@wKcqWY7BSi
z?DLad_VA?LI^UTR#IU8~)uU&>9zFY&vHI>`_S0|kw|>4K6j*fWM&SJa&$$_{neP|3
z``z>BOXk_{_x~<?C%5gmqUR%>cl<Fs|JR<j-1G8p*{p`n0FyJ)k3VirEqDH@eONAE
za^mHqE<a4;SvKD+X_Go{68q#+U+2q5Mus_R&pUs(w8?*S-@oggZPnvD9<!$(KACf2
zUtZrOC7IxrUlu%<+Ln8x^~By}1BSlClT==w>GgQ3ayUw6Uuwbcrv}fA_U7xbPuY=U
zzU5r3al|gQz0%RPOL(tt+1z&b!lrUtj~8(frw{Mi>A}9$eeJal&Z+mf?SC2E`}E33
z<Mg{MUNO_HTPOdXZQ6EsUZ*Cn5L1Sw&oj;Ps?9fNs2a~oTDYauG=017-I<B!bDw8C
z%zLxT+Vq{y)114iG1tWdv)Qs^<I4g+P4{V+w*CI&&2uM?T1MaJ8?>5T`4@WMxcp|<
z-MSmOas3Z}MXoKix3_PfEjuB%WNx|GVz1%}@`)i)pL+P7TZFl@g&*aMe>~Id;H5>6
z?^l;!H(pct)9&`|-&-c=t{1ap*m76aMS)#Mq|CZ@Y0A`tpLW%5-&1vM`4cw#RUdQ8
z7@nUvclAa=-kDhs-`Vbn|Nr#K`d99@(`B}|ZvV<_#wR-C`Uk6O^@~3yL?)KhozST)
z)7_)X(4NCnZryY4q?v8UF}Vw^EE9fy-!;KS{O3c{yKa@Y4A*=<bJzH>hQIpUJasOH
zAN$@Lr9Ert=T+e5Jr^owbfAAK!v>LmA+>6vGf&8$efshD`?}(PFXszSFW`-_Ke%`M
zeUI|neLw9Qo^L+BW7kXRb*oh#9Jgd(d{}NHxGMD7`PEZ<UX|$|;JEcfwxLgOOSSK!
z1zLZ?rXJ6_cQBfvZPBTOi5E5U>|5d%%_vy%G)yGml1gECKv&u6h(v)EuflgOT3_I^
zvESub%<}$wc3D@ptzDMAW&3ygKaF3X{eKx~wewt@AOHWS`wi}_=YQPaU-y6h_o+J)
z>*VTdzWn}wCn#yfCfQ=<hMRNu{QY*~ZgARV#tmQ1cJ2P%EPw92OV;^q+ZtF4=JC&J
zjQ{K^apz3xl?U#(+RFL1te^dO?aI^E9}7z?s?zoP8~dgIyh#3Y^y0zyPx;$E>i0c9
zrh7~C)LuW1%iP-EHG<{^#w8wKKVN$J&C|i3FWz4s`|Hg?X0KOk?9R{iXl6CDR@Tw_
z&%(w0o&Wr?_n(!c!ra*!TyNj3`Zz=OIm3-YzvGL)71ZbNuWkQ(sOw<VUCqx|l9LxK
z<5f<XcW8r!v8qX{s`so@$r5j8)*fbVnDuNoXqQMfc$dh^ty4CiDf{Gd?qUwlJo`#J
zpR_|86w{I!c=lTJgw2z1$-c;Q@zJZ+yQ|uJkG<K~mBOxb>{3eSO7+8<l?5}mJpVL9
zZ;I#Jd8&uP_&tTJ=lfP%RyEgIyYl|0SG>Co=ljmsJKgN*r0buKWWU=pQ}y5rlNt8X
zk4hf)>@qWMQ1RQPwv0jQn~_j=+uoRl@}5c)WwX<4f2L1YXDEAiJLj;0Ve8un_tUe|
zul>@DaDJMs<Nq^fx$XIwM5FbAQ?)O4*6-e2XvBW^-IwiuT_0>d+L<N&S(>3Nefl-d
z#l`D4o6mP&qALE)O?=YLYdqVG=llHZGoD{zXFcCcT4&>;nJ*0wr)4@{eSS`6Teb=}
z>)~y($1{ueAAC0NaYf%@39GZb`)B%`)tzmb`~2Rz!pvLuPZVY)RR#FJJ{|b$W^uBP
z-dWGT3zFaLIn8(9^Wj1><Egs?HfM62_vE=fQ{DS%NzYH-TD{#D6}wsXhQD|UYU1v8
zF%lH=O84G8ooUVrok_d0SER7TZk`_YCAzhXdjdCG(uTClf|)ztEzn-|(}wFPi-UzQ
z?=zR33=__zh;>D%GMt-VyY4MphP{g4(|gyxvHPFjw&&NuzwY0@1x~6@{{8;_mwg*^
zx5h4*y!?HB?b}9)10QN%FX+!H|6Gu4mMbm2IrDVT<yn>ILoeTIvCNt5dihpLvYPwZ
zlWWesDqQ`XNw#WI$|wHH>G}6>d+cS~yy4;ROS|WszhRUaUADci@nCLpHRG1obJ!mj
zFPbuA?{dM!X&v9LCLFMN|KegprN7+|kJ&B;)$5<1m(Iz%5p#C(&O<Q^lP9QXzcg8X
z@ysmso6GLjJ-oa7`u!bMA3gP7mdxE@W#BROhMa73NJ&Y?sVvpKUMy!9iOpOXyY0!=
zUlq$Fb*g@}r$4@DZ5(a8<fV5|3eyFtHRq>ionCGJw#Q@5w8I_^u7|!YxtMUWM|Ja(
ziv{9t&pxuv4t}-tXym*D8{XD!7Tx_j?{?<Te?hCZ@?^)xyFk);=;NH9{}bQepC#jd
z*~Ib9PlgE=6Bz6o{H*TIty@s-;x=!y*6vN4c2!sJN-L|Xs@@VdDN5UT?Ma_q6JN5w
z>7TG}%lBQ9x24zSh3(DOJ=>S#cKfpN<;!<NBiq-VS#zqZ=gWtVf?pH;9xxwZW)S*O
z@^KPJn|tZ`g`11>f6S7WVR4+}!N5Q7d*%G^Ihq$GL{49Nv0H#=2KUr=T<>e|AI#)b
z2#l9_5kBu+b5Y%e%#PcfZ&aJV{)~5>SK#?saQm@y9G16A4c9O~P(L8R@N%Qkg1&{X
z_8D@$kE@Sad&FejeMy_YU7Qg!IBQa6_u8hri`)N9`S;|lu;z|M6B$<bC0^~G;FcqH
zbSaC2r<lT)os&B%_SpKm?0H-JWOd>vCS#pbwi;_#F4SCH6TiSb-!sMNlt-ki<CasA
zHt#kUif}h5@3fnIcHiYMj~ORM>j^aMJ}~KZQC0JyH@@c;SQ{AjC~eHR5xm*);nzJW
z0XO)#*Ung}xA>W2f=rOe#DG{o=>@y)G&3x4(fzUXw2O-BgP$C=B@E|N52Z;(S%_bc
z-KgPSE@U6Vci;TCb#YAv^O<?e-T&xxPMpFhaQ4Re(5Dxp(=v~8HB@V)+jg|Bda*)u
z+Nzq(*PcHWw)E)kYt%gOvB=}vhp>m?Nech0b}ef5x$|&S)CL|#k%<fwJoq}QZF=`D
zQ{DAaPOgOiXS95*2g8N!X}$jpc4*w<XXI%*7|d|df7*w;Gjikqz4|xros8-FdA-w>
z>&@<2X5E`N_4BWvzB_f^Xw0*p`00f9RiWM2?m7NFo6jG4xo5uitJIH43m=pe{@r%T
z_fvB3d;a(Hc-8%-&MvnTX8h><Iz{^2e~BLl&jlL)J$QWnzmNPImGZhh{P=CPcWuu!
zY&ht_ejz)xS<uR~^_x=e-<Nk+#BXI^dnomB#nops+rINjWv<$A>06>}d|=M;c%8X-
z!naQQwZdO||2^*omlFOqE{)MKO;?|y#A3kn<Zv>p!xs|<g`Mv=6)IV#2P(zRHe}#d
zG(9KgbmU%h`Blk=ofD)buc`g(OFEp8`$)RC>)@(hR|C(ZZ$ovP)0TWJown^9PvxJ@
z>d)K1%?_En?yL<%$lYAY)}wMypZixVSKPhrg^tYf%4%C9^}I_GuQprFemhZ<`NO9F
z3Re{;ZRuDe$+Jww`1H$D1z|nw^D|bzn7MEBZhM|3sUl}yWE!tu`2BgWJGY_Mx%2$%
zvf|d==CS0tJ0twkYKt?scTRdKY5VJNfHCJ>Nd~=j>vG<i<m+r(8nJH{bM!es%XMef
z^D6UpY}!`*v8^`y#ZHx9U9YasSh&;gy;iW#lGtqxch7Fx@j6FQp8LwwH)agsn{Kb$
z-1_8J@t@5Vduli>CLcWTs@&)FiRJSir{~;C?bg1l-ZNF|>^jTN^qJ=_MN}=D$Yj7P
zwshComV<kC@-am3v^^S<mN|Rt@)=&t%TCTnoVCpII&WTN@!bbv!p4U`>s&UuU75Eb
z<m{_$FZ+&G&Qj}&)BiN{Vxd}|^@}ZwXL;QFxin+>%wo%HZu9nur^Z^=ecL6`-E?B@
zs#E!Q7bnK%g$9Oku=X-vSd(NpJ3Fkm&81=O#OTYV&w5r#J&t5knAOT4bM$nIRLuDl
zv1cC77n(P2GWqr0E~<bz_IKa@iFUV2@BKJ%^V3}2|59oVMn7FYeObQveEy`!yqSkz
ztYcU=?_c?R@29U`TzQ;yQO5nt_4UWqmu`xG_B-`lbD?R`lZby$`pzG_X|uiSwTV<^
z;N=#ZohQY%FPmtdHdFcPiook3cUQfAzv}7Qou9u?p1bj5V*h^8>sBi_-`2UbSEVXR
z<gQzpfv5iUtx3_po}8XvyKLL9cJ=$K_SS0rdsY1NZ@%=a$uo0hB7+QhW-guCV|%_Q
zP3S`Ev>;JM9&K-Kv%c+Bzpo!XVi|j-;-Ki;d5^rzL+^$dUx~CTKHH=bHz`q2MRy(3
zt$C+>eP>r54Rw%OBVD}Bw06r<f3fPo?JE*f8Qel!LiHQxuJPu%=6YHw?X*!>$aUUj
z9-2~OJ3EaOr}K#IytY&`#yEOgHq$8)sq|$lt=->8gdH}qy_x_{_Il7{zlu9?(dVB}
zyiZKHWpllvF>pRtN5+h8=XwI?m%LMuY_IY8{>0+U-a`VfK3#v^Gwby-o-2=M9($b>
z@z{=K#g>55)31%UJ#}?GvU5>L(dw>)t5yjL3(n49=Gd=OHgDVNx9V)%43>I%m4d3?
zGOhe>+Zn2Kl@?8#bD*M{t05$kTakZHTiI!bzUb>|U5hi$T-E;7)%NJmmh;l-lXjO)
zPKikWc;on@_a=?YX2;C;?5mhyB6sWK<b9W!XQhAK@!sTo%I5Xw7Kt9cT3*HQr{cl8
z`5P)87`^o=@;~WRzFYk65!IxZmO3R8!Ledonc5y+JWm5kcgkOH`;_Xw^J!sNgz1+(
zs<{ga&NE--b2BS8oRj#&B2Th#@xouipA+3}SFFC)Xt&=js*mkamA1{k%12gv_rDVl
zn&`vQd95Yv`;zd>Y_EH}?i?tweD`d<+2pwF+8t&~TEoAD>ey=Au|Ck4=OV$yslc&k
zmiqB^R~}^ku@-Op{NioD5hsKCzm#V(_vYO_oAp2KW2g(S{Ll6OQo83eRPfb&e6zfL
z|3%UGXZbwxri;uZvf@0p=FBm6|HyDbkZ+D9Lo{Qv_;kgH`;#9f@aev2z4SGL<$%|Y
z=N{T(g_Ug*ozjeX^P<dM^*vl4-^hCNHM`ex!I{gKUg_8T?E5&qYGKRmA01WhVK&!(
z|32&g|ET=(XUE_D`)>DV|DX5k-?x38)-8T{bJD$B1>U8HJ5E-IE)jj!7rvkS>gR*q
ze!e?j$>qh{PoK9UZAFyb@x*skk0o|+$6EEz37nh~5xTF)_LamkZJ}&+gG{+c8}I$z
z=R8S=YY|UrU6HC^=fPVicg=sKJ$Js!qb<+N84QlC{<%_f;gi$5yszpus(xNs$d(|u
z^Fu<%?bdCfuP^KWe0tn`W9NEpGoEW|`%Vg|OxnOJQhvQWX-bET$8x5XbIz?h&k)k<
zvZ5#BR)rtWwR3sevwgo5%<+5ntMh!6j@Yz|Gy0pOYs1vd11CjizRb9rA<|~KZ{J1U
zxh3J<Hw{c<jLw}5mtA%;VXoG~6V?+YZI4O!zMb2>+`hPX+Vt4T;njT2X{Rq8`YcvB
zLqB@+$t`y;Jc?v=zB-dJrTk%t`Il$CQ_Y|EU1d0+dwOey?QKJz&1<h2MZWS$Yzw;>
zzHsB#Cvzg!CCs|V=Q;Ii>9%!7M^86z=9%|+@(j<u`GONxN$B{mXWns2#DD!|3kK)%
zbB_wI-)y?~<xADEWokib8Eui}XPwSJjhXJ65)m_T=0fAk1`CZ@HanDBZj5wE{d@NQ
z>Q_ZGms*{Eaek(+6vye4>ET{#TaCA#Gr2tN)Dn)auw4Iep6Z8T^Vi-I2~!qvvC~Z}
zel$mGd-<$MpZa{(Zz*y59CB~rl@}9seN0&TPRTDj_UqKtLI%6}u9q`r9WD?NW@y&D
z{Mw}T&8KVU@`}DJ+*N$%&%N7U+uyfT6^M8L`t0AOA9v#Q@rz#z{vR!AyA~%UKW*mY
zw`M+@vm^O?ZYh7?@NMe%ww-UQd+)}s-<n)CFTL>m=ijoYP0q?*{_;vEblc9%Szqs5
zi19X@kvU6^*EcwQ+36Y2><upMOfr~jn0Ix@)c~Hmsq;?z<X(EWYkH~V*`tqVmW37j
ziGKABEzeG?E<9$vYgca7ol`z(t8RE3uJ-ZTe)rd9|8sr+uJcRPzW$KB{pRku+eKwo
zyqui5Iq%FRpY*nKJwC6DE|;41q)RpE=!)d!{G6+q{OZH}*}Iap#6Ip|@c24)$-d{h
z@0`MqUimB*`K#-Xp!S52iCI_vw6_-BTQB}q>Z_89@EYA(r=&Hd$5sdJ4-{2A?Y#Q7
z!nE#J?43uF&dhinleRL^Y|{rn^JhJhM#;r@FGyat*qn4GE$DGu<}Bf(Yi?TQzgfw4
zJ7a^{$8y0_`>Ut<<~+*gzL&B=fJxve6GMsw*9yH2yIAyf3&Qk!1Lq%WZ&bN2doPoL
z<7(yR^$Z)jnWIdu@=waYwo6e=xbd9-s^d4k6gKGz|NK(JlOW;2uw?G4k4$&@7#3+L
z-P|G>GTl@2ov6=5#c7K_9MmY@V<l*_<hT-VnEy&uA-!aVtVL>qObZ^_F}!}IE$`IP
z$uD@4wc*0uvYBEH$~(U)o=e$${#g3Q83mzR8IQ1q2A{~wx_i5G*_@-vD%aOt=05OJ
zIVAr@74P}gj0&H1+4>JYzBei9q1^n>mJMF>zGgR<c;{YYJ7CLqAZ?;~{v45bzw=y2
zcj!L5{#0k(*Q&eL=XV88h`B80_Hn`TbJyIjHFKyh-r7CY%$xVml=2NGhEA?ty6t<M
zFK5nid*`{nC+uJM&GYp~{{1<A%f9~2|C%of*Y9M!I??&q-ZH13?_b@=@Al{Z-^-sX
zUm8-s=+|TgQS<r#w4dIb`Tvpl+KP+$mG`fms=p)iJ^G$~UZ4GqAikQV_rL$Ht$%&O
zI-}C;G5fxJZTH<;><&|=F0PR4m$12-x=P#RI(vZ2CL<eWfz#nZA#o*-t9XBL-_3pY
z>gwrQ`}b#~_Xlpdm!@uAIm_nr^ON04t5lhn$=7GU+P|<+&inf38z(ud7iiU_`=wuA
z{`~P;`!$B&)@&?Z6?HyA<6NSd>lKR}MvX;9xqky=Zkar*C{=5E`1Zmr>m7ynCfyKv
z%EU0Y-{j6GC59!wDqZD~2e=cSntBPpjA2&r7qPnh`ti-``|;1ZR~8AZ4xZ#zQWx~G
zKxB&2lArC3B{wCsg;sa8oK-l*T6o4}|Jipd&nPj}F29*|^~Kki9XF1s?R$5FVU2Ii
zV#8@OlHR%<vRRXNJ|Z^M{`X~xl*=2Wu2lp|>UQ1cWLUrAW0K-KC5FQ69@D#`S6kK}
z|Fv~r*VmgTlBO6nUv+VP&*X7d+EpjUtj{~p<x=2}DkCOsdpCyF(cP<eo#|9**#0Lq
z=}=mS3cEu>s_M$PBTt)lR*Cp=q+OigFtulsowUB=)SO*D%mOD{`4!Lk|MoLx=zqj0
zY2xzn!Mxt*+6p>5*&Hlg*4W5073`~bdR+3kOyun6tW~E&rW;li+ie%U;=;@Dy+V-F
z$*)2*=jbKXn5DaFY*%M!-$^rgFB3oQW_YD#_t_7z&yQLtln3vQ&V9aj;`PNryO}lq
zG$@EXRQLK5s;^zL)o!0trlstP`8@k}-1`0L)xs5j!%Tk8_52oA@X&V82dl%2|1P=N
z#oyCdE3xs}j%CZre|~x7`rUp1AK!_xwmW}q3zPfNwRu;l{Ewb*v-tYfHWaPd=5&9D
zhZjS~=?h^eejN;r73|vdi*e0jZ5`j#hZ<3=t6pTU+Gk|t^7?T38GgTce-G~7+Wapg
zTt04J^!%O9YoEP~`nRv_#Y(;K*d4X!zy7sqv$C(O-@j9M#@D~=MPkH1XGg7PEjQS)
ztXA{rrU2*Csi8ZMosR!;#xm!}U+%*XyVx3xnnUY0Sn01a?)LLM<Q7`rdTQ3xNe!}#
z?Pl#SeR_VCyXLHgM%fBy88~;AZF+h>P5t#H9<%bSJv+E#tKM`rMF;8$ajG?0oY}s|
z!)xKL7!J>4^1<v3VoppFB58`#CbdfKU;p|35w(BcHZn%c{^7PbX~T|ftPDn=awhm?
zaB1$<pPaGvy`hGJoL&qc{0;|3oVe9|(DZ)M#$^Yij{AI>)T!V;`&B0c53^^eUj=We
z;6m4OZU*MZH*T!rZ`(XM{O2l{d)xmM?%B1tV9gaR6T`AdQ=c8vJ$CpoESUZ7*W6II
znXgSZ?3&7?AjExOW=_>bUS8#$9~RAKxRV<7BG3Kfg7e3YxP~s$Uelp1wQtF$H}8tL
zq?1eYzlEQ>dn7Z|W!B=1zC|nbl&7@{C^EeJxFp6&`;dT1-2>x8p)Hd_r*{XeNqb!O
z<lUW!_W3uy&Ch-RPwDo~ipA#n7sD=2Tc2ML8u}}{IpLaj>owayTiSNLW7Yk(I$Hcd
z$?iQdHgcMWUS3PkSoQbr?i~G}KdLXY&s}ih%}0;=s=vpU&)@R@s(t;Blls4#H&1?;
z<KHPG_O;)3%?ZnQg|~d0!mb`Xc#D6rQ&fVD=5(Gg!NPv+hI0O>K!b4oJ+_-1xDWi!
zd1GZOf2g$TOZeHy+LsJjz2CN<x6iQueQ$f4`rGvT)e#zHpI?aYet+io{i446#Z&CQ
z?>H2>{_Bdj#=k1<e=dIT{QvvUMPkm!9%hEw#oCzMi`a9gY}27ljN5<NG6xjI^#&I{
z{dL*nshum6j@8kuJqw;sduk|Xcx{EI($&1ndqRIFeHZ>a`Fx$8|CFwSi&pG;udQBQ
zRk<|&a?X}B+s;coKC=4a%zbwz8uX`Z?r3>6jUk?6XN2M^SC%9nZ%L+_eY5s;`quP9
z(~Ho~?xV5{2Kt(Zm7MNK?LTM3=n<X%WLZS)vZHTR^p>i(pFJFRG$GW|ab1SVypD>S
z#_Y~&O}j6MK218^t>LP1>gcamIqxpGZxDL>?b34Hwxds;O(|^K{p#hNG{NMWzsG`E
z4I@^a?&`SgY+rjyY~fjhGK;VaDT1s=5(6giPu_oRb;{iBD>#-veb>#YsLN1mwvOS{
zujAMM8pq%3nVfyf#_eN5zzpy8?91IxE0(yf)L@vh?@G-4e-bvgDxxEsI1h=5&QTKL
z&Q3`Q;`((W@6kIg_x>hk{TH{lxAhn$J=Tct6use8bT>Ht^~|%?7ZYsPD2Vi4_LfqY
z%bt7i5LeX2+LGFbGu09=EX_~&{>0?Z^3Ceozg48LZvTC`AYAbMmK&Wx_8)FGp5JkN
z=k}Ih&z(N$;u9ALzWexep3y{)eSS&0?n^${*y}kT2##$GZ|>Xt+El~dW5LWHCp`3C
zaIdnlesO!r+*KdKww7COZk}iLdgbP8PyK)OS~HxVBi2`C-0shB^LIkwIff0Fi@pon
zT6@p`V}A4K|0xgdU-|A{Ydxv&ktLV3^_EbdQb_^JQ>&gU+<N?ERffdg``fd3+pC<L
zG3ljAv#!(i_fIl{*G&lE3pc4cbkOU0oUO~nmob_*_3cmDRlRRM|IcKax>xF-+1t)t
zvz59Q+;SvA@YtfuGsCoY`Y(?wEfc;O=E7wdA(h_Xv7|+$`OuM1dzQU-%}KbXw(lOK
zQo5KDFoV4zn0Eos$@$BYR$Xi6;(YJ>WG<Vb+vDfW)jZRl{4<!qvWmm_Y^&Bz|Le8u
zGI|Qtn6AGU;9EUy=Rd(&Mh1QELJV>(`ll}nckY-Hs;KyIx#+j9M+Wb=eo>lKBENNK
zvhkNGQHS17uiqSQ$&<99!&&Kx3orZm<|@-AnkFgh?xiOys!fbJp|RQ|G|*<j?c!FI
zbB{J!*e~r&-}KVTwy^G%m2LH>mUoX#4$sUH>U5oLs1f!^+O=3DSCDnpX_M)O6|+un
zIF@j8^=tO4a+h|z(yN@iB<9eXqg=jvx78;77WsM6davqKRq-Fv!mYxV2e_OrTI`*3
z;^6CV3q_{x$_|(jXfq>k`nBFot8Fid_xzY$`6u<;pZNE-bN8<2zb8_@yR+cNr|Oj}
ztZz+Jb!Sb~_TG@HJ?Ea5#?B*ga=cvUlRKY;<h`t%^5EO5^|OmMEotOF@cpk0L*(4h
z88a+SsB)Ce`m%N3x{zwsNw4jsqy6*b#JC#L{azn>9h37m)^=8B*^~M`Ier^{ci#Ik
z+4H|}>FZOQl|6q4t$W~l)wn$+VsF^AZRck3-#40LpZ{jgwz~flKk;8pf7_5C#(Hz#
zb;BjWkM&Aq*ZVCxRx13b>qykR4chAENgGyFOh~x=`?v2dwHJR<=3ZW;pw+!VLhvfX
zBLj7drMps7`&@5y7puiJssAf-3}TqUz5SRZ@Bd}VyE}dVupeUskHkrRXK-O!6yekK
zQprhasm6Pc)q!WrF54dQdibyE$kKp|X5X@QY!Uj}`ou2t_RdAQl^0`Hl*q16jyQfH
z^|6%xi(UGsT3?sD>=byKWP9+S=-OA$7!};B-)Ce_bX5~%I1x5w<8&#VQ=*p)^p-xp
z$|$bg>Z|m~CF$l5$+)E%k**O7d6`0&98@~5HOp_&oO!cCxYk~{tLCD5q~#c6#l>C=
z`>BC$^2;R|YW}_17A5twuj_MgdQR8p;OUu0r!v|c6RR)s&f}fO{!?W}m~W|!>*9A8
z_gya(IU9Ium3CK0@GAlB8AomTI!Y&sEnKbSB-p7Gbv)wqy-Q9$3<^7^1(x+^uI}?w
zRro#obDPN2Lw~RD^KQ6QT6L|L>A-i}|7YgLOPrYXnR{{Qr(D)=YZH%3HLHh9dpc^~
zUAo4L@$PI+udsZ<{j8eT(%Bd)%!DI?WrS87;z~9QUw+L^^!E$r3yur&3+C=VB=zgh
z?7q&stQX2>eLnd?L(X^B-Pz`DzrMfTT0f!X{``)=`qKArt=<xRc=^}8PmB8)j@(e>
z=X<A-^KzEAZLwR`=ZV>W^G@8KaXlyXwZ)B}xz_o!BD`0ApAsc<=UIkz++4YErJFJ5
zZ>_K1B4n^gcti4)*N-I)Bi?Muk&_MG{pn=7{c``cpCi_@&)r;-<$iGO>+)4??;;8_
zJ5_jhZ@7N$(9aYOhs3yEckRU&eRCeY?%_Rf>xj7w(*f^^TSOur&i<+%`){@V!a|{)
z>5oce`7OEj2kUQ-@V>Hs);GUUevMPC$$OYQm+b30T%#9qe06t`qwSSzs=KyW1?qXU
zdk4Rgm}bFfz<F-f)OA&SdcJEjsy!M+r!#5m8UzMUN#7#*O!01f{LWnc%N^PRk6q0p
znj>9;Lu-UGA1*Esn$B&yXGL3u8OPzDpB3umo|pvf?KyvL{#;d8lV3Gcj;BBW^4;dd
z|5sn0#oxE7mU~kFRsa91$_e5ZU#$1b@5=aMbE3Yks_NU9Ou=5M!Vfklx;>6PO%h}+
z+}UyZq0=1@+w*E_w+6$3upNq;j(^x01i99<c!df)WoG!mvGHP_ThT_H`9ABeKaUrm
z@G$hyv-QmZBHuqXS8ScKNlIPl^?TpC{U7tC-afy7cW1@neRD-8&apaN{emmv)iq9=
zz*G8~uBL06s%z#RycF|kHy3A*(7II)SNDd7RP56F5VMRUTx<39(@y`{gt}GAJ=X1g
zvFuvL>VNgd>8)A13>EyhzD_nPTmJU(o?jo=vzCj5tvm8yi|a?J^Ru$=R(@(a)~vnv
zr@`@Ckq)sIx9z$Q&*^O7;%5{vjL}_EG;RHl$G1(`3%<J-TU~zcv^L~}=(?)8x9>^&
zYDaZnVZD;7=stT%Ld{WE*GW9DTc;$kZcZ?ETkGMvQ6tr3=D}OpKA+XhK5p;nz4$_H
z-@^wE5$kwX9I!}{VBPK@-Ehv|?&q85@*k(a>j=#aRP)PymoSsFYg^Y{Ys)7;&spr;
z^z+inM(tS|;>BAUcYMlV*znY~e4%$F^Q8OhmMD5(a5D*2^E>FD(<hU=D?BKFLR8^`
zU8#Z}HFntA+*?<~+jod<anH}w6XUhdtg8I}{$|A0=fb;`yplO*GqiB8X%juVb&b`O
zONlyqhh4n1mz}d|x@R?+LE)6Ea#nxr&7O!*VFrcVxsl(>Md#?xlVq6obw+vmRhyC(
zrFBW&|6h7nzWKd_IpmzISYgA#Zb2okn)Sjh+xJ~pe6w`tnvNZ7TNotvy!y7KDOf+g
zsNcEEm4#vBYvzJ5)998h8`P@TSIqia*i|jC$|&{6R?S0>iQlf7#J%n<{C1%yKV4pP
zzu<Id_srunR_aBsS@}p~l8~A3wnLMPZ2um=-nO1aFoEZu5|`=p&^uRm3Adl#;-R-9
z@Ou$^_-@Ig&Hru*F0k{s>Mg~`^*P^9Fi331`}qcz9iR5E>3#pFIsIr%VD??5|3zIq
zE5wVge|)q`ZE?r6YT@ZZeAgy~KhfwpCr~K<R&SkX#({TtgXMQf*{_VAAy`qQB*}b4
zF7fF>?=PaJClen%&F{%+xqg1%p3HsUYiqtO|5$y$eADIN%0(ddUb*w%4X(MKuByLk
za#G^!YY8{TnEjidc2<-gy7=<_Cd0RJDzn{t7%x~Tul%ht;}JuIY4yDBg6558AHPCt
zkDVNh8gm}!oiOlH>GGJM(0prKsNB1fT;J&QsX}IgA78zCe{bb<+iQRK{EJ}rV4A;p
z?~@e;vnMXK=9*<a>Fl=iOGGB-d=kxCc`ah*(aTr&pOa<ix+i=6%9TAg_q}TtcU|J4
zsq(#bf?4#fEytIZ{5^MCuVvzr=qx41hBxbT1d~!r*R5Mq7BW3@qnNd~)$&KBtJcUk
zu5NWudd{HYbZL@3zY~K~ONsH~_R{_PMNj3-iMeTE(3hgiWdAI;fMHhm&E|u_M|0)&
z37k*=m;dYW=Az@TZl8{hw&!scY*YNtcUV^V@bc8%eaFq08-|re=h`n2QP1p@*dw~@
zQ@85ER;|s`rmR@?>RO^$qK=f_q6URqw&K6JV)yoK`L}CFxJQNlo0Gz83a(o1I&`W<
zM^C)(65pZdvtK)>tzI-KT2(b(ZR!#a74`&U#Y4Y!WOW<++FltQh_C-%dQEE*_nuvw
z873WbKH^`umZ5vjua8pYp%2$Comvl?3Vo4x`B>y<;iob)KP`5d82wlOw1`;D-iU}o
zrdCTY-ibZ7-xIcOJS*`yyhGPqk9BHZhH}im@7KTme-`~OkMqIH`tLdF6*HH~OesFT
z+Ik|7)A^4_HXU~CuxcxP_hfNd+w>P*a*OI_^B?pNn!~uFKYhWThnbuW7exR5xt4hI
z*R;1Ov05rM_4^~^x5}{YJgI-an=gDzV~47Jk)lTp=YF;?;z^tbWyJ3FPO6jsu&S?R
z_KhRgC8gEB`_?R8D3vyInbVS|PuI6S;`3PU<dd#&JiVyW?1^E6&zzvcMrS{qJaY3{
zpIl$BTdd`p%xB-e{x~I2H?M!~O{b!GZ}&T!TzAe`?)xhC#HE=}{%z8pc{AaR!tUoD
zSt;-RW`{hV<MZUCMAXrchi)%T4`ue=J!ATJ{kBN&($hv~iwxhrGkGj>GiTEBnU5p;
zCNZu%x2YnmAzRFRw&9{6m84lOO%LZhX<KPr5hvHS>)eb)DF%_lbC#Ch*<>8)=ekhh
zw05=HbT!Z7SU=Z^i7ZQ$_V`^k;dwfJxqbhkz>ljV7jZG%%z5-{?#*pMbyNKKH(j&;
z@P#jY^WSN=3w<w}`0ma=#-6pp<y^k^*}z%syRP2I)U9Ga7P$HQgWJ2``)%K`H}$-n
z*td_NKie`*HywT(dpu|B8e7roi9hyu9Jb(jcSvFKV;(QHhfx}vCEq$3^FQCEY3y9u
zdd5cKswB^41E0roovaGJ!RHv$7-lRL&bwK-^FlFaL(CgvGuDI`e$$S&y$e`=NJH*U
z=Iz(t-lbgeJG}p$x!BQWo$i%Bmt43WP2k~;6r9dISM$lj^~S1t;;CknXD-|sxzi<n
z%0J_x%ohTdJaZMNUzj>w`HL#kg_!+k(q_G8PBeah=4+7QyVZ+jT)Wd}&ht*oU@)4Z
zciCu4gsJlq-?N)`Bu7Px-Os%J%SPdUt^a!Wj|VQu`mew2eQWot2PHn;ZYNH~pUmS;
z4?mfgc~b3C&}`kN!jqeou3XyBwbuXZ+MUrZRtY@II6rUO_4yc&0nd_^0>&qGRrQvh
zi%?VC>>*xcG21#$wlS;k)!~;HB<p{C$-Grrl`PJ_t0Zz4OHE|dkxg4_u6Cvg&y)!3
zo+YZLwCvfuk}~0(TbI-;XNhKV%&)xbwnpR0le`m;m3aC2?yvgAf2lM6t13%?<xcgm
z+}msGtL)z8$*-QVEA8NniDz_9cdTGlY_Q-dnexSGwr|M3=d+e&re$&+Ju##3j`h9o
zQfsBmLzQoroW6PFx|}pyw($J|j$h^)KAqQ;xHi7}yWK8$s;q)p_q=#*w$&YP7TKD7
z%er139#=TQS#{GZyXK?ZUaXT?<|*+Nzso-Im*K+C&K7o?!&1$v8@I;ZcdRQ-yO$IH
zy?p}zPAv)cJnib%shf@*J+=LAV1(ewji(<iw`_=8&CHNEqkw;5wCQ{10GCNC7Mwag
z+pWt(NOVhwf|KC%O&y0^l(_O<{&<seH(__-ovc;s*c~3{{L2XYz4nBO4nt?yYS}uI
zzpu*Py<_6&%9-&u^n3X|&F<x<(KFSbp7DR*cXmg~-Nc_Rd<&WUw5@-r%yVPd#GG3;
zZASU;=r>O+_~x<i5qR`cLxC|y=~0KZQ%TOVT-nNhpH-eEOq{~e{bH%v9;J<SXT|kD
zWU;9o@%j>0Xd_zaI;H3JmzXmf+^*^rH?2Rpc9ndn|J+c%=zm(O?cp!Aqe=ztF%?J%
zacWQdc>3+}uFjgw-44#tjH+&-;SAnWr;4%%AD?f<tq@bq?eIb)cg>FzU8@(rjqER#
z>;CfN`=!<|LMOk@%=s2CH~(Ihq*~a!P5-n4ITHAN&)dq(Yu#+kc%yq~^ha~srEG$F
zHP7_-i@vLDdc}Tt!xtOX7U|uG9Pf6j?5UD@cH95<y^rg@%`|ZPYZ+krXi34IUAl*s
ztKR>#NMlFoo?l0A%Fb2WKds}^PtPvnO9_F06WTy^>Zj%Q>TKD6gG(G&*u|OkH!dyd
zmsqp=LhkPypOlVOOkJ~NYj=@jQ2A_EseD})b|<T1Mvc?||6KpK$ZTWhe%~Ak@X)dN
zW~p>9li)+k<J<JT3TsVGpRIK2UoP_Hz^rZOw#;09)@Bydf}{;9-?Q)Mu|4-yUe)_t
z{94Y#S+0|QMOk^3?oHZ|Fhgf<ptV!8uB*w)b;~y`sbt7#TQ4<PzTRxXv~=$qDJLG?
z-LBUZz+2+aU1PlRxf@@~X3n0S&u2b~xtg~{h;y#u<jiSe23OYQr%t*ZSYtjrwdmj(
zx4ao<%oCQaOq}J-)_pJ}Ht}fLLpIkG4YurOTTOH|L}x6!zWwrtb#r3o&yA0XJ(@US
z`OLF(zxBmdU$|p`&+As|y&2|Kzpg%9^K)6a$^Ee2b&KAuI(Gb0Y|K)3M_>M|Uu*I-
zuWj5Jy4qB{L3q!HH?r<?{+=^2UmSDo?K#(3nsZ~O%-k4$WzNcH8eMyjr=6>rRK4tx
z#<EjCt9iF?D-JRL>obd0>fzN)wd0$0<d>^DZ+ny-^r%Q;kCiv?<r%NrvL!W5=f!lJ
z_`c*gbBS}i)e-eTZMRvC8=5Bi+a5n2Rqo^-xn^@%y==4U@$;+hb$i!wJQcc`!kWrv
zY^kMWcb$3B8Y2b|W6c9Rf?P)pVooc}@K9c|?~<g+^pj_TdqNC-6j*(Jy-R!PV|d0@
z<iwfa*o_@2CbO?@TK)5^#pH)Nx@j*{rum6o|9RCU<-(?tycu6L=3L;&wanEGj1=80
z+Ms>&ZRoO+n~!vwn=-H8jIy^o@=fr}qKSIZF$V)TMW<yf+?bj+`R1B^Gn~>d%v^ZK
zDLrzQ7sHW+LVwTTW(ggJU`5}_FC{1anso3K&ooJ%hP+jhufBbhUViwzvj5|x#F;bm
zR$ZAW;4n?>LDcp{zUCQ1-Dh6-9yKlfwx(mnD%r%N3BF>kSGy;)PKufswq|PD5BusO
zzva*GX^Gcl>OcJCJEwZr72dm_#d;)rBpEi{bTNN6@k&QR^2;+DqE@ae4!$bMzQJHq
ztH<trH($v8s`EBJyK_eFEy27;+5NoDTRZbjvgVv^^;zOkdAx6b1>-x-^zTwP*OV(R
z`SGRZ&&6u_gwV<*yL2k;jQ9#<?^GUT6gZL?_n38KvdX%s>j|OnEL@jN;1}B%!_e_`
z)vqFzM<Mexru{W*@blGNKfn30Udi5MwZj)azUui{+8;36_3Gjc!Y6M`ZjhK#Jlpz6
z0&63?J&Oa+HSgy9iq1K!KQQd<Xkw9DxY~f{QBcgByC;QPJ1;%!d%Zu=*sN5=YV((f
zpRA{U)M=c%efj9Tx*uzIzv0e1+sN3E@S)^GGIN964+gb^Grt|!ThVi`Jm>+}0snvU
zwGaO9{Gr8EV0ru6{TKa{j-7lz?^+J)550!nt7p}Ix#s+I`Ny+-G8Tm!9{BouwY$#w
z?J6?w^WvBl&CBC&-tkOP=TVKE)1H{19kKq9^2HmkbFKw_=4*L4=f>_GF)|yDY416=
zY9impup{1k@3Acx+8O>QEHy~}=;^O)59IPzNKC%w+9udN^;BF-Q!xjl(QJ>fw?C%I
zii(D86Ny;$Vvo`!p`hZ-ddcSP9ZgGIB<w_&D~7o}e|>S!#$&gG8N{le{7$jra!*V=
zA$zp@^o2m1tEquMBpP0uE-H9b)w{H_S-a)L1GCs=ou4mXxLfvCP4i~F<%Z|WLku0}
zGd)P0$+A7kPgRiN(WRxcuFVVbZU{ON%B!L8>T&<XjfK0o#N$NzH^q6oN7(6F8Xmkd
z*I2loDS!2=Ma&1jT5scEf2zB1b!p;LEAg%Gjx~jfDjG&GJn3`_i;&;@-SKGD-tV>U
z@?VV$t1fQhNDh_KiSp?C7TdAy|E4R`4bO!~c}_|%bQN~0l4IB?Ak-s%Pp%<|qvE)!
zo|(!uwSC`i7(~qe(Wa~$ykzAJzvbr-JvS|^wUwQ|-Y{a;#yzVA`?oIj*IAr#FGB7-
z<A!zf%{CR@@k=l7HaWFs&e?BUN>~!+nT7hyytq0!g40FnuI<dFM#&68vHnhGqSqNp
z>J~{A%`<x`7-?K{TH{7e|K6MHE#J8csBL<v<8<F=)0qO1WQikfH%~0tv77I<^|o_8
zE>c$AMmuY2%zZaYCR@x^Q9AeJ&_=0cGnwW+kKCfVF4dqy`<Ithm)Z5ynP;;Wailp)
zur*(Nd$;t*%bz=<ujIV@@z;DxbL6|X{cjdpTr9B3**i5`cJ0mI-+V3~O4o0Cn5S58
zC&{~Lb$WQc>52KZrJw!hz4<KO|G#~{k6io<YyG@`Cnx*F`K7<SJO94uqnl^#D~&2^
z3@d92E%r-9R+g16lh@jHF2q*S?xEI{A3mJL(J#L}5?DMpD$1`Y_K3Rk-n=7c7u$-@
zo4N6eia>gy#Fj|4AY;wTRr6*(<C|c#M<n^;hxN}lUNV|}w)Ae`7L&R9s$P+M-hFfV
z61qG#{>(GY_dGYn*DQT{c4u;eVer-xqkHqNsF%)KE2MtWspChWL28@Wc_W6tMX8%q
zcqNYp)#?Ym`?xdjiO_3H$L7jevqdKcrYU-E6>%217*s44GyiO9a*=Z0@`s;SuU7YO
zUcLGJV*`a_AMf3dkTzbtJZab3Bh8si%91fBS0|Y5`BhZ0bYA?wnu?|Ox)N$+-8L6g
z6<5nnyQ!n6TpIG+&yM$QuKuo_muih?&$y}5>-_B1%eNCGXHE1so)Rf$>|8uo&Uej~
zG#2gCI#(pqlvoW!8G;y^GcTPh%0A9Cjc1zA#Mdia4SdY5C(nH8z~+2b#i%(lEzB_D
z)}otnJ)5=^?D^%*mTks(ooBUZ`^Lvt#WR^4%)huCIpQj)`L!$NZhCRt>i%@|XKOOA
z@mBuXEWg*nq9)sW{+jcDZZ1DL&2C}q1UBz$dpDZhD|>BTxbN*X-Ng~<8%mP>lvl0`
z7xH14V|@7wgR<{smvi^-Je{@s@~yj<eZ21HzgX>Cb9Q6m2D6X$(Kig^JWh({c7$s^
z(X{vA5z{ngc#;wSa`~yZQ>p~#t^6FfUWD<@qlQie_L%*~JSv<<D^^Unp?UIe=<Ge8
ze)Z&5K9;DgU3;4SYVYemEq1fFozvOFykN(SU7zjecH4cnWLUK#D$la~g=3@_<FVwR
z&l+nwQnt)WTzN1`TkeTm`UeZ~&X}x{LlcAxSsN~h=3f2%+jqA0(}}g)ZBO^>{Y+eu
zRkS<0dEx4<dKa{x&DgB;XVVOp|G_^#&%gBP{lAP%JHNY|R%@2mo3O|IniO19Wq0f+
zKjRg-TMpkJsb4?9zx2=9qv7*!wg3NlW@YKU>XOpW#gD6BO`CV`>9Y{;+Mhh;(~ke@
zQ~L8k*|kIbD6d=p+6NPhmFBS@Y&;)l{;Fo~@hJU+i?jAE-D!DQ>m66NXz`+qwrA^`
zR?m-rTPrzlCBNKUm5u57+h>{gKJ#5X>wfLkDGUFv`gJPML@4&|?MbWH{ZfxOthwr?
z`;2jt#~N1=9VLc`ca0fa5;Z12c=_OE`1ih7@5|+Swl9CO^4d4s*@k~t<Qq?LuhB`p
z&ML5X<DVn>I!Y~y`Yj$RLYmLheJU6Qc;;7`zF&EEkM%iQ{p*W7-#km;IdJ4-bs&=h
z!}LdOQZ4R9j=NT8)n~GXtvOn8Q~hG<qND`o${quo58@4M1+CpmI->rZhcA7;Bj9az
zzEedqNl^IChu>$@|0gS*y7#fUbY0(%yI+5WU4MV!<Id#^4$YF&=Ir+?)7$@J|MA_w
ztJ*I2-}|kQJGb`R%^gMu7X=-xmJdJX|Ml(nYs)9FbuaqF@~msyy~B}@<rEm+Rq`~k
zKd{c;&zh4Y9ABQpa1As*IQ8|iXH3d1s{YaTrUl@k=8#x5KVF^B99LVn_C4HqUno+1
z2j}5S*;m<`vl%*n9dax_-R*L7O>$(8|K7qA$rCym>e}-@N6+g>nVgu}boHC$g1<k1
z&hRNW`qZTL$4e-`=T~%Gim}-AqSDV1`%*VAm1PzP^kea6zxXty>Ux^@MU#bGS5ut|
z&R;$hS|{KjaZ2HU?~hMmy8cr>O`Ij~^K;>ss<O^?soD&I*3C)(9`FAA=8^jK$T|1^
zeyn}+MP5q$|Mn+LC$Gt0d-pHn-cQGp-^cCu9OZw$|9kUA>Dqm_SvFX_DR}bjV(F(d
z`wxjYy{np2y`s}k^25OwA4BZw1WR(%Sdy>UFE46Qnyyn(6Z>c1J9n>RQ7V&Ho#Fnl
z>fvQ4d*`<E-|MGjZu&X<X0ceW+S&IH<)zokJos)SrFHbXt!`zE-ikZxKkZ(mlGeI0
zL+N98=toD<ZjFQwhd%H)NSr$OqEPA4+%?@(nVZk~-?Fp*Q^xs2eIC=fHzz`Wdh-20
zd`<T20n6^=QE$I2V@b%9|36K>KCnmr?B9p%@_Ss%-ey$@di%G;w<dD5tU5i}L&-+k
zPSjL?PW{uLb|L5dk61kMW!ScZ(OtxENzs2VKgk<PlMM41Lyi?+y=}YHOhbNt#a)-I
zIeqRCRr8n*e4b*U)2N}hRQRN$WqH8yRhb7>EM{Be+dg@|BxU`kmckj#+t=M+c=N<9
z@%7tu)=UU1-8BDAOxNy=*ZkQ(SU$O&^z)y|Ft@)X<o<EZFwbLi*Kpe$4SMp=D9l#C
zGE6;9=2@Tmid>CYb_SgmAB)YB41AKS9<uUfSk{Er9NyUJTT=_3-fnXaNfuhW`-A4;
zxyD&)kG2HGUf69K*E8Stuvf!W=6IIn?<05`PT$}zz4yBGXzY`oS%uki`hVrlzj#@z
z(_ZQ2<<HrD+g?q(GIK`<!_EU<%T-UgUgd0;61-+O<-^*&%~{e6QHn3hnx-aKixz6G
z)^t^Q9HgRDcRF8nb@tw~P6yV<EMQ<!+@K}O<2>`kgbSi~tDoHuUBiEOrOf-k-_G5A
zIxY2jD8mcJ=Gn(BSFY5nE1ElZsc7og?fg+!-pc=<5V!A#c>Nti`*|<k9Xx52xcu5K
zhkX%uSZ*CJn`qgk&L27deDta4^Id@=cMh>TRX98<O24^g!o(|@nh%d}P3S$filO(}
zFMY{1rjJ+O`_gk`&6M&Zsh?JV`#&S|lI11Fzwa`qX)3*4U7r5=+1Xn5HLK>m+q&=K
zLtE{@TWqh4JEmXYQklT?Ktt-_6jzas00t#1?)_S)x-52HTEBg7f7ZMw^Oo;qNZ-7+
zcwt)RdJguidpB>&_4QqSvV8GjtEThKFLQstO8fWx)W&Jqca8pkdiJh2_o;sBhTyH-
z8hiG#G1z?ABiP{D62R%I(ZciUph5ebzBY-48KnXTKL2@s$#%-VSDQ}Rt=;q^s%ov^
zE-u&ETic5`6egW+Hqdx(yHu=gY52s_c|ly3Wfyv01|BS(qEYs6fnV6JkXhk|5#QfD
z&}Pg2+0AuVS6_<R@|KzVCMKURdJ}_AYaLp)^RtYDr_q|~ugymiE}qsd+-1(jkow2u
z^4hGcr9AHsnHff`J}&37>7=C6O_lY5S9KY6TGmvFPh{ghz!sbP=)H!S)EeKK!-*^0
z%mg2nSn;mkf8tWXtSEyNJ(tjE7pXNyp(i-nStMsY+@Gj<@n>x9_UswYa<iql)#u(l
zJ5l(&h-dCRv*nL&8Fk6Lowk&Zd1cb7q^R&&b2dj#@zON95^2>Po$P+=^Ol*N!K<Q9
zT|Q&P#bDs`_6M))UY>c&-`!oAx2iRoKb`felBcFe$)gC~xrq`--nKo9n3e3?$j1<S
z=3UxmpY>bUvPt%;wM8!5Y%<qy!_K@jEejrAy{hAT_}~lIw=;9L+z;w0Y&@2vU&I&l
z_3xg~(qdU{cMBWk9xyb|K7L-m?o()y>Av5ech9ajnq3$8&$s>OePzD7_w2uR*L?c%
z==wfyp>^l13-9g>y`t7o^5wzN{@b?~XJ0odO*8v!(7wDl^Vq4gU)IfiYT$d>WN}FO
z*(nFLHwP5g&Xg>j=JxecfA-Aia31G0&DKPz6d6~uzz_#_E3Fsa*BrgoCm*{!$vp2g
z_v*XBSC+k8*1IMC*yXn`^G&3S468ehuD^b{s`Jc+FEMFFTE_1~_iL|O%j_W9<TmrH
zgwM>zf6sPJkvyfdrt_A_WpT%Xs-K~u7jE6WYyV>Yn=Lc$fAwiLwNYi=R}%DmyUV`4
zGd4U?o3;Au_RTN99lhD_lv}madUwcty}q5Br@xBMymullX32H`_s??b>#8TW7`oJm
zAM=!4zQZeVNy9uv-^+7l?{C`EJS}(Al$pILX|eHh(~1hE@7z3Pqjx#L!(PwHQgd_X
znVy|;A`=yjQzUf#EiZGVF*XFXZ28`tc_}!nxoF@0jH+1So?ubM$VAmrlVqJcE@^$|
zHu)$`zThkBG5O!_#qlw2zMH37X)XJ{isR&LU-M^?Q5E+d>6Pd%p3#-PN=fBFNT}ey
zHz$97n;4h5;f;50?5eMu)t^`2zI#fndlg6dCcV?n45^=I<ZA9(IRC!d?Yvm?lKc5x
zMUF=gEs6=bq}{ApC?TEZoOZ^*=zxK*u){3L<JP;Ly$cY1*mdOAi4A5SOCfVKe1_ta
zT;BPwpXWbsernMfmkG-ySF~wNVK6zo{^KH_ss47qEg2Qgeo!~Mx$flNS$11|jz%w8
zGx6NJDUxe$DtU9~-R!<;;BS2QLa5K?n=TL8Dto1TX1<O|)2!@G+srxpt!T!T3P(0x
zhLXCV1q+k3djEd8m{=I=@78F&&xiNAQKYSRNpR{JhD)Lcq8QkEcAkrf@tP~yuyEHK
zn~qZF_nM|NmYtg*u}s%#hFZhUyepq|^qxLZ^SvzKA<Sv|wvzp6PFmjfkh3X<+^)fe
za+X&52TELX#Xjq&R%&*8pVrQ+o4L;~ruNG{_UYTr{|nxC`SRuCNt0cjpQ|tP*S%-Y
zH{p5P+`nB-{QSIoE7u<NNtrxxs_o`8(<aS3FBZ9XTCDNm1OwCSt7cx>I!kkcN8GXp
zX=yV~Zx;)6IQ89U*{sNCLN2PObaFSBh3IGh?1?sGNXvL~U?)q2_QdK_)?3}gRya<H
z*Xre;5*3`rzW?O?yK<FwSuZoPwyfbw4&|HkdiQ26MW56(4Yw#pow<z%FW>P=o0l|$
zg<Z7yB+shT9LYyhD}+yNu-VI*uA+YHo870fa_PHa+3D_vJ!$33XU=ZTZZYz$Ghe%(
zG0o>{)b*Ug?B@@k?yTK?b*9{0oB1YF{;@}uKYCjmo-O72>+gqCLfhXTQ)Vg6uKGK5
z*3{+V;pMlV?fI@!_xoq1rMA@J&HCx4%ce`(obOKilyUw1foQqx$ZJY(BllVH<gurx
z>v~I`^_rWy-8gpH_0lv3P0gFG8XTvYHr!P5F1}T*e!BDEDW1ch&nQW~UQ-vI=67z!
z!IEQ>s$yKs8g}?RHpn;Mk~l#+>~xMCUvCKSms<~(?^?K?dCk0_<a^VmG;!S3(GBt0
zSXH8!9yr@NB%bNPgAZ$4RiEuy_Ke$n#l`@k4@xp_uIeq@x81dhl{|K9hmq9Y;8Qkd
zJ)WHFxtC<tlK84)b+HUX=I!9v>w?*P))_xlJur*2_WQ+s_qW;j8~Zl!-2$y-fvsI=
z%SySpIl*9$;Q9JW!34!kp0B6120d3l8rl#l=V<!m&9608bys=*2h4PTUj8{sbLafu
zl6=cv>Mv*b5-ICowD|<zJcDUZ{>7}&T2}I3?UBorZczUx>DBMEI(t5s{CxRy2G_)#
zZT81K%06%U{_W1r6}CU0&zxGmU|Ocw$uzxVyA0WX{$(%-nJYQd=cKkrI*-AeIlRk-
z8P42R%5q6pC}FKI;`Y5fhpR$v_UpDYK67WD)e||KQ_&HeEOBPb%<{W63tx9XTX!yN
zm(KACA1|%tY|vVKJo0F4Y2Am9x4*8P{qCVO?_+zu<l55~70zAlrr$Ok+c!&pvUa{n
z&%2L>+m|hw-f_3~<CDlA-h%&%ChWPecQ3#6ZR_~7$DOs-Rx^F>YS*rs-L~FX&U^8+
z)K%PbH{PDP?nS2Ve8Y-a*(Z6#wx7yLGtl=v8kXm8c=nFf%Tv#Htut<wjA|>g^}dnm
zzcS>4VS-m(!Kbz2=f81X{<``8`xnf<9;<V7(u&WnT01!+;<MX&o;${odW&X0OLINK
z8JL;3OrmaPYuCfmJY9++hxL_KN~}5KG*eROVRwSrPCMz?O}jr-n$0dL`|f=`v!=lO
z|A~k%T_3OA&^sk!b5ZbzY{~uIW@&8SLz=h5?|pcAy`EqG;veCQ=l?V4-#6!k`bF0@
z*4=KeW7eKu&bYh!*Xl!aWFM`St}`hqPm?|VH+I_x=hqhba=hwZeHYJKY|Hl1zkKEM
zp4kOuFMfBO)XiVM_3(ioF(#tPHHD>9PZrMAG)i4|^355(G{@G}t4=KJxXHtCXu^f7
z3+7md9scZdG~kiRHA!J(!|S|rH?CZ<I#0<sHeHe@R9q-#H+N~x^N9k7IRCXO_J{gh
z=jKd~OxxU<xRqz~#ItIB;r_hSf2qD`^UI3}E{wHx5o~^1D*e(#nXUWYDhZucGD|Mc
zw+k|iU=Ui()^O&|@7r2ob>XX`PxS~MSXAUBYnU99cE-Slfn|eOTj7(}-09Uv6PC^v
zc?cTo0&Q~eJs#uHYkWt6XS&%tiLb_U?q!>siDzAvDy#jKH(A(mV%!;VU6cK9!v95_
zzL%b%{%Fg|zuV3)O2~Ylx7mXsG&Hc!tkYGi)xB?9N&BSRXYYTx{O|1(p74KfE>1Sn
zG>b9o3%}~w+422XG(!W2!=4c5-FnGW4;-!Ax_0LCpM{#TN7*-&9_iv>*VX?R;G#4w
z@8c?_N_7MMI;*hiRbkh~D>)YYKEcHBtmJ;lmS4B4{--m%TKvCN>3`}2#_8?f`R?wo
z*Lb`<U)x*s)@|Rf#jihj?Prtp{?N|8?>g_H*Xo*{i4F2nn{z+9?f?F}EP1(L%)Ij=
z3<mFyG~TvvkuMf&OVo*NcinbK_@wQ*4a@%apPo={qw!p8hx~P>^nKS{B=xGF7ES4U
zrLbAMXX~2GIdd!LexLZzzL5RtW`60j&jr?9Y?Aunmc3Rsu_%^v!;ICFRdqIO**uAZ
zA-?ovQLW|1PjS;ZR`x#ltWXdU!eJ?KSMAa2-1m!Ti5I<F`SxY1y;1-BIeEG}@BNaw
zYo2mW>b`8}>Bvaw+MC;|w|`!;=H{uhkC%(v|GfFMe(sw07vI%??Z5G8eq2ph#Ia)D
z6Z36@#pM5S{g-FB#k~Dt-SUmwW3$gRRvETm)~&y{XWnKP&GV_xopV3x<mDcB<v6yQ
zbz6U>qMWVkgpSMlE;+^QFIpy~Y}&f-RZz?KE8)#Wd_m%R&67nkzaGr6RMmN0<mdM;
zcYY+#`3C9dYu~-pUy=RXR$C`&L(!JT4s9vP(7@~IbGI`}EVjGKx_|%uDNmZezBY*n
z-Fy1i%lf79?|b8aGOnM$-!!hH=%6z1YQOU;hr15fgw9sl<5kGU@*scZRsX)S`N_hw
zfBXXPX`Ib>DIs?6k5AvUw)L0@d8zrmd>Hy%eOjPM#uY=I3kr7v41Kt5zHHsC;;$N>
zA2s)0-Q#anAx=-iSLVH7l}eHJwFx_ZUrCeAFyf1W|6H!+k8Zu$r88GmX_*K!!`Tm0
z^XC1(CT4x@O`LY<IhkU?=bW>*@J_oNyCm1tXr@}ys|qoOxgQe+UN3v)BD8F0!OSxP
z=hYZwR&U&Qy7+F!u_j41hNBO<5=|E0db;L9WSMZ>%QJJVg3?ZuZuYr6|L=>|uhXi2
z)yt+Eb;_l`Q$Cz^Z3de`?AqFNv2Azumh;swURY9>YJ1@|`{f6#e^o3G_FM^SKfgP<
z_{Gb|&G&yhe);k-{=4p;{rCS&EBtWo(NsRy^2iAhTj$=LoO^$@q>tg6d0lJo${s7{
z|2jA9_}@qR|F(X#Ubp|a^zLmh@A<tybfbICof&?w=4H-Y_EKZsqUBvT&!%!M`Fdx8
zUw7BRl(ZLTe3m3?uRJGk)!Qk&D=~~`+C+ctucA-?&En-$m|D8@!?R_^ffM`Y)_z^7
zbSF)-sO@IS7QZCdnG4$5z8HBwPpMDcY|!rcS6=q$yw@#ehm?#~u{EWf_@^|1uez>k
zRlINW${p`y?K9rau(saaou~Ir{LPZ9Z>na!l~{XtlS$jF_jZ3DbthB^{{PZ_^26rS
z_WQ5$=kNRTy#M{bKaq-JuVWtX&fjCY*JSRBWpjVW|6OYN^J1WG{R-V#w-#0YJpWJS
zXLs<?|K8sfR$DIbx)pN8++F|M_Oko(wRM;OtX=l)VEFS=Tk*v^CutltnQoLmFKNcQ
ziKzxV4u!n&y>!~<?21B_8BxoOv*ZNY9A}-ICXyj>tm@i1QKxW*lFbKaL|X1#p1CTW
z&G<&l`R)w19R|e`tAsrZbon0tS<COXe@Dy}wX?4m=DlEu+3#E;*wA$_>P%Z?)RJB6
z7B3N5nD-*M@b51EY1eaO+MKJyy0`t`SNMICs)k^AD7RBr*TG#Gx}xunBvkHNa&K|T
zCgue?il!P%L==<T5-YfypL&$0WfZ2oli9X++SAHeJB`H`X6?CjB2#$wkK4!F4uTd=
zCh6VzdfhZGh3U(OS=-KSy1c3K$@4|M;j^bd&G`N&{n2X9_a-sNY%OJYrkQiDN?XNq
z=e0!Be3yf|KR)f;x+!UqUu)?*2OVD7RnH%9DK-0gc5!Xw>u(|18~?7TV^;oe<nN|`
zXHyhk_xxvDtlS?m$gF>-;rZqkpOX8>dD9b}T%05heP&pt<RaKA*bu;^$ou}m%5@!u
zB@djRx+h8;YL05V&h-Ahx&7b8hH<~5Tz|>0dN*&~j2)GKL>kV?U47I1)}5UpPtN}C
z|Fe(&=!-E}+5d>BexNtu+4+B}4R2pR*KcUP|NCHf{m0|=cc*_my=jGn4nw|n*38n-
zcPsN&FIQt=>wACXVB`CP-_OlEb^p)Y_s`#Q-bh}s?%KY`wm%AgTrDuzaZhUs^X+mG
z-7;~xKT4Al;#^}ayUa9lj`K3!2=q|nTc3Z%R$iw;T32A@fzXDJmrp-Pxo-cZ>Qp9M
z$Hqg9iCfApUwnGHb&aazNx?3yHxk*&sdqM9W4#m{|4-tu(j!kzE=A`l!9QngcK_G8
zzx2sXu1P^HlRZD54O_c+brkoebLXEt?mm8A{Q1R)kx{lhQ=)psw(CvZ{r9naP~qL~
z{m;5z{CV*Bw*9Y%m;asrc52=JXtwG8?Dl3-@5<%=PJe$sGVt}QWo7H$Z;t=hy+Zgy
z`@c;~6=Yqee%_|t(4Tk6pCRC}$877pVsR|qMpLt1FFAdpjM2e=WvyV(g-f~5R6afJ
zO?Y#Fv1`SVNmVQjj+X8m*WX9*G@Ji=`IC2T#`^uK-5Wm5T4dzB^V6M2U1kxl4Xfm1
zFHZFQY-Y^(A^tC?@6Rtaf9&FnA|kE|ra#(}wBegu@S%t!2{-R_Mkl9VQQ0-C%8Wr_
zvsIQ02ZKOE`q2jNsq$-cTUUJK-F558b$Q6T6p2-WzAZMtkET1ZI7r2(ckg|%*<Uy)
zYQpZSnOYICah-{qdJb)jx_Pws;)>;WPXE<$o$m7PNW#+${b&Iutz{t|GtR2D-JEbU
zsK<$CTGz@ac3YKIs(2!1c}4g4z4CRvD$MZmL(M0VxSQwJr+0Q(@BY1&=VQ*KjvF6!
zx@Q|5Ht2Xf^IBfUs?(*%o_saAoZZ>+HP1PnP2j0PiJ)5BOlB)@Va~Z1k1grCSso$c
zU0zfxdw$(H{lgn0)E<NOJ{<1fS@h#JJA3%~e>Zhp?WFYb>LgXeub0>VTXS(mwDoPT
zwJZx}%inIF6=Y@o|K;y@b$=en-^(hiwc2fb`~JC}z2;KK#Opt<2lWZtKXd<m{Qu2^
z2VC##PuD#Ema_Y#Q>M7@wDz~5(~reZI~xD<S%mcW`&OB6zq$phPODPpQ|ntVwOOOJ
z^oWSM#s1n`H5TiwS1x*|l{J&)o>P~Or%CAA%KYlImCrA|iej5QQ>Sy^sg#>x;xly^
zHQGFL!sadwIllc|LTG4<+i5TN#<Z<h9WtYg*bnrqNoMZ2RFNKbL+7ew(UOLRk*#T4
z#W%BP+m|P8n6XeYE%tn>n)h7InL3%$CmjB7jbA2t_K=cS`(^`np3Q;v@n+w@-Z`<w
z**<E;!FPIZ?t4wLUL49)_~XfDar=K-_1~^uT>1TEy1l(c&FL5O-pI>sUB;gI^W9bZ
zze%+h-@i9sIpYDp&eBV}*0KvuI2r$BrjOh9?I*8(eOmvw_0j*--*fXVtJSwYJCvj8
zsPN|To0HS0&(K*Epb(rDsdl*}X#QEHg|k+63ae}V|MRNq_S)x}(@wgY&9;b?do7vV
zrhDkl7rkZ278>=ZN}e|I-@I*q^zA$mK@XAn%T}HVx@_`%m*VCrXYLqxmaPln2{YI;
z<Gjn~s*GZFZ|-Ngdk-bd<UQ8o#WXc}xrg(kU0RG`s)rX%_B$&+b+WbRt@nlzAx<8v
z6((9Q)Shj<WZ{<*&sheqtX|FXn$>hma1ujz&t2Zb!8~TOJujOymK?u4`=7OMez>Vb
zLe44<^N(APgZhcmy*oKVn|+U5eeiN~!0b&gD|9YB(pnKz>MC{3QoSLRae@E&<9FXC
zpHE`h@UHVtq{hsP-qGnL6VG0pw<T}u%yh}XNx%HUgC@%`tkl_BqO$Y6sP3MqMdndQ
z&tAM;R>->g?v3jqwf~=ey!&AJ^WCY*HI>WHzp9*dGv?w<!SiY%Hdp7(mQ<2hmA7iw
zgge_Ac$Q8)tEL_ne13w&vU5DKEu7zT*qS2EnxFr)-fjJMv9P=0<&Zlq;gc_foZXUG
z#e4b8lFA=1)}DK}?&vLx`mh*Q_V)alId5v~&V*dFNWFaPchRRU`)Xs$^HM7<tLg%K
z+&^E)t^cuqt9(n==8U|R(eAqsFPy5r_{8OV>+(&HUpzl=&g=I@ORaDI`xdsUKjrSH
zJLggoegD2y*`AmEM~|EBey-pC+_!({-+p6jF1fUK<=u|v2(OuOuD{Oe_vJmyx840V
z*6ne~?4@UA9V9igHt#IWkYy|G?@ecvR=pk8Uf#E&`1H4LA9re4@3S(sp0-h8fitJ|
z%*LWt29d*OQg-G>zE=8vG$(IM8KXz=D;~?o6CQ<V99qBk?YA=qXHsV|H%#o)`&v?M
z7MtdA-Bv1P%F)d|4pR@F)tS3HPTBjp$vcfG(@7_b1X;}&o=LH~&U<u*7W=Uu2^Mwb
zbLwr*rYe2=zwue={69ZG&z`;B=kw(UpKd-#4=kJZu5M9w+y3o$ukQH%;mMB`Pfcyl
z>l$re7ktz4IlH{STzt;%ul(ZO|Cg@YI`dO>>D26q_Q=;K@4OUG{Ul!bN!*g*wXW&B
zD(m<EcP^UduzqLh1eRCVW?#-Ut`m!&`ZCtY=+I4_!zxpk9u&FiBOF?>>yEVV|F;^S
zOJ**QTz37+o5rM=*OEnt-=r-{+VC!I^S1Z-np<T~aV5NTNSpPnWF@yrLyp9gK6`1l
zGap|i|L)zAmiqC*OZ9a2+hz@A(~f^yn(T2qD<vo;bK>H=8_ulSFnwM<hm-C_(Ksd@
z^A#H}Z*mH>uP*wxtJ<2CQ77HZe@)c9v@3ld4n7pKH8m8JNK4+EGQ-f>Ya>%c<V>k~
z#~GM=rr#@@Z7y~199Kf_g{x}&o<Vkr1upZJUvuzs`lCmN4I4egs{^-3sD*VLymTqB
zu`TV#^zG)i^X8gF_)VJ~#n`}b%_7hL=3FCYZq=^>OXAFf3-`XCd*-W>cyg<B)w-f{
z>y~pjYqABGB<F3gnG~hR;;7A3*6-6PRd2Q;dxGo5+JBEbExDw&m#ME!&zuNapUbj=
z;l;95MZsyAZu2hI<*t+7${{4kQ))5WI6r%3vAOTugP&_Ie~!I;=jPqavU1VGDnhE0
zF9k;IelNSN@B1&^!%wQMulZ#kH$D6FYpKQTiHtMMJ~Lc#kGmeAQ){U2v^|aO{Iab<
zGi{~q{{D|D`V^e}zw>HoqeR21_xbi8vhK-m`*zQC&*IyQxBu<`@A!P}rJwJj*KR*{
zyZ>8{?0=oRvwXL!`L0X)_2%H=hX<w4)l2=lUu)nxY5J~vx@^gso5Es@#od3de`CFR
zebn)tS8dkaygh6C+c&i<R_tytp1)_I<l$YD`<^X+5t*Z3CA-yNPOQ=*t38QNf?0zZ
z&RkfQmTuS0puuL$t7hDMM|el*>5`+VQlAVnH-zy_<8??skvi)|%^f3Y)g3Q&)E<U;
zGH}mI^$yOO=F20-khCFXO6xnb^XY8nC(~cwGbq{Kt1;`7?CFS<mw#?vy^=WbsO@zV
z<!UoKiC6nKugHylnpFL6f92no-Jkv0KBqgse{_D6yuV&tkF|gH+k?_>-(3vPxWC`?
z`QNnlyL79|S5LUT{NVb!FZutP5`-laYj#AeS)SA$w2oQl`xC*F&)&_|xvcM1`@Hp5
zlu3<77N^D8wVhY*`JLYR-E9BSAAUQ9)+XA{e&#iWy~a@bSw?o7yw>AaCl&oSw<eZ-
zE(tityLsVq%|buEV9tb}cim<+eYWv^8q=Su=6iY0>z!wooIb;&$rCDG+P-q;!4Kyq
z9kiP1)-vasq|WvCDM1FuHs5qPI^`6n_H&W6FDE=_G+bH!Jb9H;kLXe{-3P|ba>9kK
z(*AZ4uh~=fc73?tw~4(ydY)eXqdS+E=Dm1wa*^^hNrsRes}4o99b{1UT{*!Z)}#9W
zTiMHZ0(tYD<8vRqS2APB7QUac!p+4lqIvmwC*MskEe@yDiPcrvy{pOC5HRWJiudM+
zS6^1*)lTb{+}r8%L?I@9!!h&V><jvrHr%_b8SEN+??m-wNez3B!*8lq30#U$JtSeQ
zv~uF-uu@^B3CYWsURe`)&Em)2KO2|*{nNj9(^;3ZAv(J}zeMfsJaYHy)VvpO+m7iP
z`+oIV_V$>_g`KHJ&AjQmgwM*<XLGI!J7}9X%lhm(f1|9tn<_j;zMEff-+5Ky>dtdP
zmlL?1^PlNVPSOgRIm@>=|6uRpwx^NLn$s4ulz(oGeVu=PkKe|pF}wchM@ajcMK9Sl
z^Qrx%ogr7!48L}ttiPwTE@x|YPX5ooSAIV@d+eO`xqsrT-mR(%GWM4;Gr#kATV=%S
z{dcR5TxHLE{6ly5|0D0d^h~o_|NgwmS>M^0-=5pEUEMbAd(^6p0bl<<o3?zm{OYfQ
z4L_gsMm_dcnD+F?m-xOseV1aB`Yq)_oL3ng^6zEdmc49L(tdZI@SVt|+%wPcTuq&+
z!@D`pKy?zgQ<`WqD^o<zbFHRbWzmP@?mjF})YIl?n%Ku=Tl&00^ZU+IBBvuH0$Ca)
zrAj9)Q{1FvvP~yBQ7N@8O(`~1YT3HfDSG{1Lf-gqVqbYNZ^hPitKMsG{dAH)|A+Yh
zOEsUSv=)7@s{YQpy`A^&JFDIIW>@bP%fDJ4Rl3=Xb6MZ{SCZVff9wA{Dlh-%9{;{S
zH`C?!es2`L7q&8jA$Qpy*MF0~cFwc=_xhCJviK>*sdeR+nuXUtB)irMcg&8yQ*!y8
z^_^SO4vKHyxU=Hh`LyU?D*nc2KfIQhv@!kbXT3`Xm$MnJY)Ri#VByPfWYZPiyBno?
zX3lZuko8!e<Yw<J6}hABOi<DWw&1jgwHs?4c)~tcaX6TN={o4;qVqKAZcn*vLZITp
zv$Crk8JGU6Rhz}t5Vgg??eD3t)9<al5L!AZ%r1gqO<wbv&o6`~zCHUja_-X2R|;OH
z^e?&g^@@q=lB0}gz6Pa>y<Kq3^6;60@6l^*ubwdb+C2NmZjPt-ZBMJ2eKb89<I!;4
zBAP?<NeA!xRom~~&!2Q|Z(m}s_+f`h(${|f_C3Xv^5~1h-Dh*%c53bPJAeBYN9Jpu
z12HSETx)ZAw}pj4Pg?y_ZLaZ^>foGrTWZ-75@fbZtUEqE@a=B_>zIot&K`W$_iEFr
z8pVluF<uH?hi&w753mMsX)aaB-ThqGd{XtCDjV+>SCw3`Nu~|2O=Vmciy1P=h%sp>
zWg4sM9S-og>bkqXO#bh*?WIq0Wh>83++F(QWqa-UV4L%wCF}!!&HMkh`PxV4|8M86
z=lk@T{iJdIyeYpv6n#7?Ut1tmtG(-QSjp|#winocn*3RQSe<L-=Fn`$1+`Zmy|;h#
z-hNZHd>MZ|b3@c`zFhxEo2{p}FXOlBewlv#e7#dJgUDz3`!@f!Yy7XjA{4mt|MsJQ
zj=TPQdiOHV0l$Q}`5SkqhrXVk`sIq`*ISQ&t1~#2@6K}d(f$7-RYg3s>%+W~xmBX`
z<Jjg}zWSuMa=mn7!Q#~3J9`dRF=V*bnz0>-nY**fbpGir40Cqw+J9|5r-RP)cuDIj
z{@XvVehYoOUFo?;yln5mvybaf--r*<-Ln46>rK1k%f8<}{^!7VzQ0xt`!0W3G+U-l
zLc-r~eqOx&{}+4tIg)EX-8}#CeHW+~zc*2|yx!#exl6OpZ~74#_N0E3&7!UAqFz_=
z-#Y&Ps9?id`!%&ycK^QA%;~)BawIhE?RRVb7he>g?ml*Y-n2c-uP}U>5~cKLhsdgK
zrKlC}PH#I=xHIMY3b%#JuLzhidT92n{gt$-BxZ3|zoX@Hv7j%ToQxS(h%2t0>&B6`
z_I>Q}2&Gk9N|${Wz7^HwfBtxh;0h`G@WP9mu03Zt;OCvS!B?rAK~I`napR<2>suL)
zB<<z?aykyWm#ATDXTDz6oQacnN8C|ln$a$LXQkeB%UdxQ8Lk*E>Ad>%>*_@!+b72b
zPGX$8(LQF&w5RQ+aUt^;y?!%IoUtS2a@zVBFV~$b*4J#l=3Bxs;jP5wi7thPcfFI_
zc7Np;o4DrIoSQmDB5XTsmp;=tllmquKY?M6b7|$*pyJx|t5q+p-oW~tm*LFS$5(4T
zN+UH+8Sp3`O?-W0@q;sgHs_OPy{?{D|6se#dyhvq(&QNWwdKNhe1CA`ovqIA>gDWz
zFRuI7`v1F?9z)^3Ki_Nu?f$#|eYE`jwRF>!=O!1;oAT)Nt=(T|T3;!%w^tQNt1aEV
zH2%zF@uPpDY)+m%U|~|1Ra^SI;LEj-HTNxk%QzVBXG@r#^F#dRmu~T!;g)~z{^`^?
z7xY|``NFqIoyL@x*L-zCPFdG3?-c#@QPHXVcGvsq2D|;%|4OjhWfHYpKTzZ`gI`%K
zgXQf_pQrEL!nmUG{MC;xd<+-5-j!A@YYjf0D#d?*^NenL+~N=4mTkSfdar)^@p1!^
zt=Ciysy_etdDMR$<D5V8+wcAl_MNBEqEaK@zVE%kw=EVm+e7Caef;s=?s@Z>UTohp
z``ttJ`G4maTyuY4TX1nxU1XT;k&SkKj#KBocJ4aIZ};xcgXGkY4-al%`>c#DZ0rBm
zfuG;+V%ih6F=DQlmHp03>tBlhd-=0M{dL@&yH^cWc*S_<dWZ>zgljMQ`h0?fLyXr|
z-;8rNQdv%4+FkSdVW;8ke_v{RHmqEJ<z4CXHBEk=v#rg_UrF4$v$K8Ox<5B_XI(#T
zJi-3j>UH6vr=MRJe047)>TUSdl@rdr*4VLB=+LU;#t(J7l(>Xnwk^C8!|-6$wp|)`
z>v?X2&t&lw%zGrwx7g>h!j2t0^Oq~JUaFQ_GxzQ^4e``fK1WS&?#O@X-FjSlT2hwl
zq~(*|PhfjpQ+3bewa%1W|HW}FS866ie~Vpl`9#d~)GAreULp3CI+0e)4z*?)`x7Kv
z-<!8LrtDcL!7m)^V*GT04@1gkPo+oS`qmmNPi~syE3?}4y5eSyYZE;y_C1~T$7;^Y
zB(}>eHkVT}xvod8vNADe_R&f)-f_}b*?DGS<=y38+gECcG%ww3;2V7YvrgXC;>G7K
zv3*}0nK8@T>r9}@_02JxXUvkFcYaw-`}56*K5zScV8#3DOXh8L$E&XJ9NxVk=SbsB
z*3TtZL?(YMopf%#nN(TsGTGIp|GFoi6pxpRoRGeA)!ANy^UIE=6udrXdicpN@qg#e
zo^&?*c6Q^@Susar;x^{Ed*5+>>sbG!^pUN7z^=#U)*tUNT5rm?%0HFN{@46l&DpPu
zBcFZJW$2Df+M?sU@z}1FODE4OtXVm8uIrga2Dk2oU20)@c3SI@j@g{ls0$g9XN_cy
zo2Am2GuKHv#xifPjZ87RJL#F%bH2>BtOu*tK1-SwAi8j(|M~|rJYtu{9(-vP9cc5y
zDmwk+k9!skeY5VE`u+Ix>Eprk?p+yQjI)AR7R-o`o>%tybjIn+QS#L{Ha)Ys8l&uQ
zbN~CcO-e6I_^;kxujY1lO>f?*{$2cW91B0&KK-m8B`bd5>D#mS%D$)jpIe)kyzI-n
zO%Ik`Os}kXH225x-_y67UlIKD_Sdv;2ajL8eEj`;bAG=2M*kU(|KFW|_U`M4|357H
zz4z$1?c4v9mt~z3%N2ca$ENPqomo8lSEb+lxohWt-#OtDOT9L}O}X1PKe%*Z-ix!+
z+WF@^pUiu?)%|$1Q$umcv-e#OPw8B34Y8OW$>rR8cur+>p~UIM<;^OyedPDY-jABb
zUX;5xk*{=$SLxg+Mm6K^J1>O<jTdjf(S2<-$6N2S)vG<F*0ic-B!ynzG*54ho_wfi
zqUwS*mljOvP<bof;3ewd+G=8Sapzg30~aKNd6LV&-!Lc-X<q5JO;hC#Xp5Wt8s9(X
zMFd~mi+t=_wrh{=i$5t6C$_EURb&)kpRnzm-S5jk7Js*^h)R0>?a$lCds8RQD7u-Z
zW*AYvklU!0y=aT;&K1fG3a426U1Xl!iT`)y<mA0wU)7&kwQ+9=tq-aJmBktoY?Jif
zPDxGZQGc|0yWg$V{yY3c4h1uCFrE)~3fQE{WbxgCfz7`(@@Q13r6a@ChZZL{Uj4-6
zAjo<Cv(5Q<mxE^>nk7egTdb?U)6Q|{t(|o8f0dr&H6Y^su6(~I3^NYCWR+0n)APzS
zUXj<m_-MYr#O_Jv*1Y#;u>{Y4=qhk4F~Viu;Tz0pU-s6U?6uhXOnu9r&kOE-(3NNU
zJb7}u`mr(rmc;yfQMv!w8P=T<zt(#xd+UWmf(na!c2;c&+OlL%vV*t#^gUZ=GYMP1
zSlMuw_vwxH30-2lZpW24ckU_W)^6w*HSwOPaVKrYE*@haz2&{J+P6Rby=*Y?Ay+ly
zgI4(vcToow@!I8*N14j5_p>`p?kl_hlF>toKlY4P(AxJU)iZkKtJeOKUVO8k!Qt3Z
z8HOuS;xB4H?Ef_{%XYT<(xf|9cjwOD|KZhBpV{*E><Vs&j$ZvXpD|;r|G#QM+b>_=
zsvmWo|2WgZ{=w{b4;{Z>Fj*GA<l47VW#Q%47JS>J&F>!eyPxs@8OPJq2Tw0Qna;3o
z|6jfa^M3~z^Y;Iq&Hw&iw=P5U@BDx;@dJ=q&idT_%m07bf0kj*_hozBUnQj4isqOr
zOna}U<o88_iTNGF3r$m}@JCb2m<$<yB{Fv$-n0D5fkhk|o3rb#oZoln+>!YSGk+Ir
zxbBeiVPRMzQu?Iyor;oLo2IpHY2Bo`dX;NV_ZyWTO?Vi}|2-@I1Y>vEtIMB1&RfcG
zL)v%at>angn*Sr7^X*K$=%VSW)UUy;(eh-D%&Uckd=4VruTqxD3z==~T(57YdC0fs
zu;R7&8$E&EGF3A>FK;`q_3{|#bWXe9I}Kk2Z+Z|RxaaZ?nf22G-eoDD5N16vP5f$w
z<H>_oZ7~;Bz6%Q$yDqko)eWA)-IO#{qvexkNqa=yoxH!d-hQ)D<;u%_H{sW+4i$m5
zzn&aO>o!n%${xMn$GgR0$Cq6trJMHkJUNsa^;GRZ%-!y3yTqo-SITKCt=zd!)Hv%Q
z(}Ay2qPtcVMb|E`RGS^~^e9KHX+;0OUw+2ZE~Qy~dHr6l^q%c2h7Hfuf7kgbsk9r;
zJ{a9~EUR_<_4MTIx<}4m=B6?H%Uiuf^rwJbOX=c0$#ox!q$g!FSZ`*&{xY<_=6{P>
zPnz)&-LOt42F>>uICSM3#P7De`}Qeet*V%3!_g2vg^3z1EEjHUWq48E_{yy!G-7h!
zvPBh3mkPOx@N3<9X0_Kbk?F-sW2uF@o|VfFRz-JzJN(CS^^=F1q6?2sC^X!!_sl+Q
zE$fBVH@l*K_hr?Wf0wt7k@<UA|Nj3MGcViQznW9)@7tffwSK~p{eLe%+Ur06#GCN7
z$Kw9)p7Znld-JXDCEwN?Wc_cx_-K1t@bCF|OZYX5*$>S9|MK7Q>0$A?|Gyo+YhMv!
z|Lgbe|5xoV$?fR>cl!I>z4yP${o9sRA08-|V{a7~Ih(KRXMg7A_&<eH=H7k($Jc0$
z*EWuZRo`Aoy!w-(9&KL8I&)&!w0Cp%+sx~G#MI$^vq#ygmhGtx=v*4nz0xt#D}Q$J
ztd4qcTD78dk6NA5jIHxj@BDRnFY$FxCX<2N-z}wKxs|H?3;rl9Tw%Xqn)#Y)sWo29
zB4a%n3eIInuZdWb%4x~aySY_BNdNp7$Z6NG{YcHs5AS9~2F~~BoXTk(mOJmc)86L~
ze3YAuUfl81Vmp09zKdb2{8R>|n{&9gCT&>sU|Z08k%<bAwRK*u4*#N5@~|)Ea%;x#
znk#RgoaAsgX)4*N%U`%-Pq#qbp-qvugEk7b@4S2|_u3pQVHO4f&z<i$*Y51Obx*f^
zo;ia{k)x90^vv{4H`5=VXcbn}6PF7Q`}Szzt9>rvffkAlu0B4jQd%0u8fV<47<%r=
zG$`h=x31i^wk`kTRU6*>Ro82U54iBfl<8iPdT@Kqv=fyt^1jbiyYNQ+I`8x9`R@Cx
zuGfAx`gMBW<(T-p3k<6L-s{NiKCpV-Yle@ZkG37GQfbfHaPP#~a7l5-1)^8gSI_+I
z_xW?etq*JMq}^RyZpKDUm(p-M%@oJA_MKtGBbz&|D^v=fJnx&TvG3N2f2XAW>BwDf
ze6`Z}*135*OryfAW-Z%u_-f(CB^(Rq>`xRa-?&Vy`^MeNjTSpDl|Ic3J9PhQ);ES1
zo%-2t7l#!zKReF5pz+?^ELrB#D-%}kcHJCWT>Aaykw0_!(<|=(ojZSLtgm6-=`GAP
z;eTGWmsi*C53W{UvHk7d^0oV}zL<9{r}g&n`?oGnV%+_26EBZ<<(fy{x0XB<UaGVI
zl4#1)sZqynZsz{4enw)i@ADJy_W7-NI`747zolGf3!e3FZ@>L!@yZ$=1;LNozF!Qn
zQ%e<<@b_}uZz^Q5YTYB3r#A|_J%pmV4&V9s>gKy%{j*Pg7U`xu^OMr4zB=o|*Xg`m
zOGQqf<63RE@Z@T#HKlCY!pqtG?%j_+_aVDlFmB75y%$B+8%0!Xm1lPQyG!EG`w53+
z4_tZZsuK3?UDD4#hb<0sIA}T_xNosDA2fmmTIhUv<BZtYn3&Gn+OGbWP0k**oGHMx
zq34i+^kqqt!|Iz4o)TZ~aaefuOxBNk!out#mK-cva#nY?PlHFZ>dy0*X6C(Ea_qz*
zYpKq|7J0p&SEshtFFp9`M|6*5U_tb^zO26L((gZ&m)(my={k$S!{trIzH`%R*%DVg
zxg<Az(_C$Xok?eAR~pYsWr#Vp;`xF^DTY%#`=0NaBDr=+_!8bv5$h7Xf@dsDeOdF?
zLRVAF{o<p@u#4f5V#3ExZJJSYesRSGqkCqN1z~4oQx`53)}6~xFy-jl86{^s7g{$T
zDw)!H;#1o^y|+AHmOcEHZhUXkqQJTN&l?~9@>q91W%ES)&oci{|CprW^<?&k%jw<h
z_Zlt#seW0u{5s>Dqsjj3FHg!`u~c~RR&8I;uGN;0(hjpOVCypZx-sw67p<qV=AWN+
zEUtJktF`mXQZe81jM(BMvqG79W-GE6IT%LlGB~CyeBfKk!Pk+xzQxC8KP^3XLMf=H
zbp5Tuph(x5kW+`*F1s;hm_3R3F2Lm!p7p>x*x%i;Hpu<fmu%n9;j8xkSkULwKl|Tj
z{y7(7cl2Lgx%8UfWbsQDyX%b4-SD~Abn;HV%H#U%Ej9;c`tMVIxH7d?A+}`u#>=1k
zrbfNzcoSh<5?bE-E??^Q?X<_{rx&~bR{!Nwecbk1)>+mE3>u%<Kf7cE27gxj`pkcU
z&BV(GYig#=t&{xs!t7_3iS=2j>0f7A+>(<m(e*wWcll1Vgc^g9_1#PAOFS0|w;$=A
z^>Fo!Yd5aREVgtLs}4zy?lQa=mh>$5Y1_+L%MzDZInP?QOmO+iH91l{QqPrM`JAZs
zHS)Sw|L1S)^%1|$FE$bFUH{_B=}Sp_WfwlaDrsDjJa1`h-l}esg_~!exij<PWuE;)
z?WSjUUC&)|xo6Iu?$r+SG_G7>WQg1zqCfp~eP}alL(ct|E@n>(Wt~sf3MH(}(=@(z
z=JKJZu2Txn3B7#Pqsf+Cz$Q`m{lxd^S()dK`j{Fi+yyO%Gd-r}x})>|O(l^9e(9py
z<DdSxHb3TRrxWY#Dd#j|@2B!GOgTOGK+5t|rKD2~VLW1Cr7y4UTD3*lS~oNH%aQ;8
z>VoHKgg8Zbdj7htUat5`V%_!m$CoZ(4H9=s_&H<UuFkWwI%7VuPpnxg*!ApJ;m(Sc
z2a}{aS|@SKKZ@HZ{gg5DS(e1~?$Vc{45xI1y!tfecgnbYd@ymFPi4Tr*E(|Ve_K91
zT6KAI*U^O*-N!xF?Kj=cd-v_USpoCj`Iy}=y&tno_U4Rd3oq^Gk1iLsw3vK0i|eVR
z=3Tcl0t{YnB_B=ul(#5?VbKR~hk#uZR^M{r%iz}Zju8D4nh07Nv1L-k&77GzR(p9E
zPK50gFq*hw`dg07y}?ocs#qQzIdJiG*FGWF$*Yr{!WrauR!)_BWqtPEp0D%2{NKEN
z+aB}w|Lx^z^545B-`9wGKd=7#-CK_KKdR;D$^KDa`<~&)jI;OhzVY?W&$~MN^Apyc
z;XZojx?VpzsFT^Ve)9{<hLcAncSbQVx7uz!z5UmzKL>RR+ix{TCf)lx^T(#L&p+;;
zn=Jn9?)<NL_pA?o-n-n`*kIH8PdOV5=I6bg_tviJ+qK&dqx0KuKYhBmS$*c&8K-u>
zKm7gh<7ww_e)4~=Ze_o4)$RN5ZC3O(AC>ccntc8JlMMf-$x7>l-nkxJdv?P#X%~%Y
zXD5Yj-(`BFc_QC|+w-2YA2_K0cHWy>b=HEX=Y<4?6MbSi7>^|CEnV@7D|Wy0x9~4U
z!ppyNapqsP$)8oC^!iWB+-a|0t$y`%?}}9%tLHral{1x}d#bRS(1D;u?u*<Qx=*lf
zSahkfH#n{SZA4CzaQ$oWKI3K1X&&k2)6dUao+;(WenohLhx2UfefL==oYAQ|vo!tV
zc{|^G_w&PermYm!{GN9!dCKnd8&fu{;%nBKyK=_4%ga(ljwEyz9lRvDJxA&yZ>`ok
z#(?}Cx!d1)L_cghIK`r;^WZFX!RVTkqH^ut9oM!=9agx;d-vljtJ4h34|!rP>$;}(
z-OpW;Icph{!8^kUBV&=~mp+QJ<{M3qIE6YhoSyD;Iv|MMHl_8T@1u2#3Y`8u-ePEC
z{CB5c{|pu9@@E~LOBB6cUI^*fKk;#T+h*-eb2yfNdh}6r_4k_+W#4C>`;}9(^mklI
zvc^t_+PdG}vcl<-Hq)y6>ZcuS_sDObxo*k!!*l)2PqBzT5-eFA_FSWTO8J%&r@mKF
zR)NvoZ5ij5DWBbR^?=uoz`d_ExSh*w_SQ|+tTpR1+Ss>o#W#ca<PE3JM9kqiZvO3X
z^FA{#-^XrgF~>J`ZHVmJbW!bA&oqZ4aa-p`TnwLbB*#pJ!FglHIs*|^Sys2~azpQ+
zdzoHWGa}P+EKePZ+fkBZz4hHW)|_(p&?o2KEC_xrwM+T<`E5Vq)}7l_@-9O+TVq!G
z^*>tox>{79wa+R!wtr6keYe0_Y_3uwYcIBbRC>SM?f<u%dfi`rqTaQy{WUXa^O}V#
z=6>7O-F@!Sk;(tAy}f5rao8`XE_=r2=G^|NGZz~F_war0Hhp#^Tl4IjcV3+Oa?0d#
znt65wTe#0z6RBmHN%}b=vmR!*&F>68D)BtydT{;rsd|^@JzYKd=DUh-hd#ctGPbE_
z_h@C95a#!8=9QJ23(xi3PI&X?+&h`2{}v{eF1#}9>{`#-*{%CUzAbg*kqiurFpAid
zv_ZmnNsNk8+^v1mdzE-sYaIfmuRrtZCHdY)^4dkGZ@AX$wtB%Ig@#R)KCTm8rC$AM
z*|E$2{PDlJq6~A-t&b~~U^tx6d8PE2Nb~D6Av|CD?tl0@Eludvt%C5`KV~H#PRM;E
z&DZ$qU|^e2^i^g(TdvDKQ{Sx;dK<=bEpIaOhb}3@h=}0pL2rZl%6i4W<gDGL>Kk%*
zQcA?@En%YH*SN2r*E@5Cr)I{#S1)C}ubon@td@Dk;e9^1tD{3hT$`=ii-R$M^Wi41
zJlFWO(sqfyVXlwL*JOQMR1g#xzoliVTV3zLL#IL}ADU;gaN<0x)ib1(x;hFrsr&!<
zck(pTuB+*m`4gnpOq#=ODENGNt=Tdq_vx8ut}d=nSgg~<zU9o<tBI~LiK=P-?!iJW
zffH<2SAEx!>;8Os-{m!bTr-<{udTbdsl(etC5P?oXOnZ!_NOq+YiGWx)qP^?b(4tX
z*5!*&hlQv=y=G<Ww(;D?wPOE1#jZ*D>Xtoyb@Jk^tJ&-f3$`4%dho4TzLfR$)g_@t
zZ>{z|^L_k7eC_>Te;#gcQ@6IiTkkISZtK0xGaqfZ(0;)8?}Lr`Hgof@&c3U@d2YA4
z)nVC9jNAv*4~G2vu;yES*}Xr17SEl(@3!Q-6SL>8y7N%+^6#UZ>FRT~AK3A{vVGtB
zPs;tqSD(tI-+9b#7hHM&bItkJ&4pK4pPxxP^Znwt$lK|2ewvl*d|T0%c$T~Q!0O6{
zXCM4dTz_@tjtGVnlPDL}xKp>Mm$M&Vot`M{u(6~2rJ%1$zCq<8f!vsvH?Oz8y;@P)
zBz*fwSm@#C`UvI+x9q~M6n(w6{CV=K_=B?^zc~8pb-*$I%q5rRUu<o<@p@;#%O|nr
z2bd;!Opo~}<}`snOqGrGN4C}XuOEGL9@(pbXGse_v~n<BnDFr2GrvBwNWrjxWs_bA
z?OLSeB=lrmxM18*<Fcq*cUI=yVZ36PqO^cDsL!Eje}~8^p>@|KLoa^2X)xi-$3ow`
zhRcMaLuWPXT@&dGXIDB<>hb6G+WC%x(?VMq4txkQ`@N}RgOgC%b}hD;MXN=R9a?qo
z!dI!noeKn`oSu5eMcQ0gq@}a<9CP4T`G$R$ZMu)Y-Zx9+{jU6V+*fNk6xF8e__9d4
zDAs<m?>XtX66Z>1r#gl`FWQ7!BN(Nce5WkWcKxdAAh?~uW#a-#k3V@Gw>9Lx%uRcd
zcRB9zqKw;@E-sb&@$$u_cLkkCoIEsJvvL_LPVkyV$rTq#=1ya3xOi*s2}i40Nv+>9
z?q|-lE#0=Sa&<)A+Sk)QzRb|-V$FT-!guAs@x2>%i(0JfV`!Lrd)JYTqPZ&{y6)LD
zdF#wev$x;dS9|(jc<kqQdq3>rdtkGoy>a8;B@1@>F-%U-x~R1%P_UP+_3yghYNAC+
zsUO$x(!H9yQZM-KwC6v+usan0WuCuMZ@Xy497`@{2T>uj&pB<g#J@%LbNDfS_<JEu
z?m^`|e^rrr;j(AK=a`9}^OR)U`Q_!^ee<54f6ef0vU#4IzCUmBmHp)vFH#gUTs2Q<
zt(Z~xm*vi1m;JuQuYbNdHUH(GmR&{xd-mU7v*xq*@r$6PvXN7{;-0IAhD}Uco$Mj)
zu-x8N<c`=x0VlPhDIT0ILTkCon3JZeu1Tt`5uW{{zY~14s8q3_&_@U7MSZU%gdVzl
z4v|_@o%+1FD9)^(F~slPkp$0o!oHygbv|Vr@TzjXBYL*jpQ*_-BJ6SN*76H;#J8_q
zTPpqiYkQIV!`D1w2UofXo&R!5*GuV2%>2A9AvPXIu2u!c9d&)U^^-%=2Cc85PrvQ!
zc_OvGO5o3}ioG{JJU#yQ-O=v))+bRNp7VRpr%axqC7EiXbk6^zz0xFZsg-FF>h0$q
z75dK&jp!=4dg3f^q4$^2Q`T!&_yu<yloCl}6$t+pvOXv7A^Q>Ii*IipJN-`E^0rpG
z;zix;&=&9K->>&Ov^K6f8(lqlnr6?nby6Cd?=LuYv@ib2nR%CgmCuq-=PLvnf=yrS
zUOQ=Lm7r&PYYN*ljafZLo!1%{6<oWO{notZ_1(qiHvY2t5hm_%c>4=}Z4uYUo{tqJ
zb7f@gl)8J%`>oj9`@j92(^9qW`iH%I(`1?cOwXA2QR&dCe^)k3hAydUIBxrB*No?Z
z)k@DQ?G7j&wLQ<xl;x_zaOBd$>B|G->KjuIsEMaH%S%snW(nLQ`dy~Il66(g{>dIY
z>e_V_KdfR~{b<Fq_GfqH&tASJc`^Lz&wsmqtxDQZwn^`7uwlge_vYG*^@|Se{vLTU
zJ}GtfMgtw`eW5!`_pmKLvi|2i^NpSV4?q_YJ(g2=Hu>SNBV7lX6i&IeZh5gN^6LEW
zQ*LRQK49`Ve>?o2&pZtumXfbaCR()>FTBLM_mlLRu#l&+8d^Vd`^*;ZJy;MH<HLAl
zt>M`O<0DHM9#nkY5tFuTjt5i666OPG+zcG=z1=istQIcc>K7Aur`cWf(xUx`ZZiu^
z=YHiT(>hsW&It``{#Q;qN8JK#R&W@s5s>CR|HyFf?^9o%%(@^tDf<*#+q-Q7!l!&H
zFZ#ylJt}fk-ohoB8fd|JxKiTW)ennq-P(6~jm~<vJiiNFCT|!xiUfmZo?r|&U8AG2
z!Zgu!L#VXB<2z21eMcTW=)Ssf?P=?yU%E~#eWS>5Y3KT)8hzuc(|=w{l`QU!)Qt(b
z+_~w|DmRJMPT^PYPIrsTODXd8&a=3`aLPNQH+o@xNetIMukBl8TC@7SNjYDi^qTGE
zA9j~;eVLnfKXc#feLoBTA1?2=vNc_}_=`vOMa8457i{>I%aE`x?qPoCg|G#??j%lp
zzEO!ufJu?TJu%vS-gCot*Hx!aUeDUwDJ*fkYjN!5ZHFIw#dtY|GgK`-U-vTZ|4h^U
zzXT5WB}{7VI_IAwxU24XQvU{~P!Z8rOcOdR@_Mz|yyt1?-|p{AkFJkcy~_We?c=y$
z*(!;{Ttz!`dbWxz)DYK`UjMDCbIXh4GMq*S53OF1_4Y?jlJNhH;LZAO$$Cx<lM|wR
z6_?NHI=D(fFxF7JWNTiR2T$13Cvl9MR`n+?J-A9PWwnv>uD<oN*+SQHn!a0FUb#N^
zon$|w#bULK39oC}WLu+h69UdvuRW96<Hg~vxbp1hsx{*L|EzDSF}QpPapZXR;6vb3
zBPnlvp67hrhYR8!uGN^+@i};s8{>+UptzF9M!m=4`JOf%31wNOmEr2RU{!}oFU$F3
z&4>67S88}&O`SS#&G}>1?E9m0H(b1x>stDXO{2?|b#upVrU{?=q%<N<Ul81)q_)N@
zUT^=(MUj~sa@b}&mMWk9%rK*uv7t!pQgn&(a*;%<T{#=#TF+=`dAM+NYHXQO9yd#o
zk-_}PTcP=#(T7@J*DVROS>SSM(su84E2jt@$-jAb-{qLJsL9!zqGG-7gw5Jk9-Z2{
zg(J`J$%6MLtJlw7#<1XK$A@g!(pwv<uTBiKvFJYj<?Jbq#rGqElp-ffuhg3uaauk_
zkh3YwRprkPH%(Wa$(`$61aIos^Ju^0()QzaSf4Al`QW3~FE-ovYKvcXw4C2AXPL8h
zIqOP}a4*Mx!z;N3D{orqUJng(aSiU;P_*d(*53J=TD)PYGr|oc7;@ePC?8tt<iNEq
zqi_Z@<Iesdl}#)O`jeZJm{*y>7twA!a`$dPhhT~Y*ZZtJ3MH;Q(=@~v#e|eze_wl_
zF>c$B6=FiuPp3awIOnAB(qr9SyANq4dvL3Ge}C2NGWBk;{u1^4uDxY{Z*EQ(e_6zJ
zSE9>flHBP)iMv-G2Ys6o6?1F-EF%{NlWj4wskt5-zf3w<W!h0_7`1!Ky+v0f_L@Y6
zl``Ke-S6)4`fk`uc~iy}dfG~y3OvtURSp;YvpKQ2K*&b=w1)E2uo%6L7fOG_OnXkd
zN}S8|@LrG@qu1pjHgO7vb6{Jd!1RTOovx<3*WUgr#opfk<KM}3zin^a^;+t3A?3-l
ztOx7AEfP6;DC2EWsIt^DFYz$Pgw`X8-#<0C?FePwzere!Q+pD(V)x6tIhU9h%vjHG
zbLNMa9%BEryVBMOwoG+A?83)W$o=8!CsyC(PZCa9Z<)()71z4aL&@#7<~e`ut5Sk3
z5et2qB2TZk*cn$;T{ivF`dYKN@U?*x-qyJWm`3zp32%+x=d)p-*J(kY_nQr$=59Q6
zWE)fU+o*RMX96Rp@`n5j6y$7*T%MYz?cr6>KiTtCRaZdi<W<F$f}LIsts6>2o^AS=
zutP+5qNK~z_n&VDHwE{Fs~27TYF%>8;MmJ2s=Q)9cZQ!SJidbe>6=sY6PFgR_sCvV
zxbUWyq}b7;RUZ$o<`Fx4Z2HPdwg-A8OGUgDmrXG1-cmZFwd>%mBk<*-7gHDnRv0q~
zMNgT!dfm2|i>sRU`JX>N|8HhYZ0>A#ckk;jwNBk#%0KNF6N4~g=98HmCs?`{i0V#g
zQ@M2L)x#hBI<0dWr=5O$%Y*5_ypI-v@fLBmf*T)Sy)LOU+qF=0t_R<#ox+~)MJMj&
zUQjaqq|W)(j0H2F8NDrStQ6(F#>{f}!PNuo6;``)raAG}9E#?>wp!|KVB16v-6*?;
zh*LXwa@9D5Y~}B6dVccr#kWdalQw_&`NE7N|8RkyGgAtKR!y7gE-ua}#tSj`6W?sO
zYH4*(J?5#^-q`QeTXtzNcZix!{lph;dMI@53O){{bqo@Zrm?(t;R|JpooYMv|Ee3R
zPv>%J>9R)fB+NS|*?H%thi1Csk`+3exa=8@c!~cvS{U}!b9HOD(VCKyor|<9tad$F
z<z}$)R!*RfT&8YQOoZyEbB_!?FUQs0_ginJa3pi5&uWJoLOE+S)_r`pd&~7BT^F~l
zWsqlU^|YDx`iqwS#;1mni=Srnisr48I9|%uEtHsht#2uJL-ihk^BIfeuiKnca24bf
z<l5tvRJcxLqHWo<8?p>DxKBw}dBu1g`ymz_y1CsnW9s6!6<=GbS(a<@imkkx`^~GD
zgQ4n>r;Cr#x7#N9#+$Q#Z0=k1&X!~C@v7VNO9k3OMNb<{SQRZcG30L1>28U~J}j%E
zCtZK>DAeWatUa~Dvwu`2F>f&Y=-VdY&G3QyvtG+`jw}Ta{vDxi{XrE|8M@ca_F)O_
zx)3;N!q$Z`lh{s6>Ym>G^07<s%Repil5X2EJN!HM{f)%Gn_Jc-d(2<ELA*ie_qi34
z?K{uP9;=XO+tS)uQ)&H5b87`}h2T`zUJmVkwvgDuYd(@19n(DAFCXH+a^U_a9it~X
z(+*|4?Gt?!cZM<U$J4!M#2Kb%F5K;FaQZRV$vLvAo^N@c3N&yW4dCnc*u#+`F|RYA
z<)P%?qp2RwznC9r%zI^Y_N(psvIo2We!s7>X%$oJX{H0B>&=RaPc0Mtv6x38Fl@_N
z7e2|gcMtJ<^q5X$-c-6`N%;3k&2FbxnFN_V);E1)#IyXg_@+>;W$ec^t7mwu=?ilB
z6WQt=cRfM(x>8=|Ij#vS8<o0Pa}_l=YIttv+qt1B^2*#_K4yDgxXqbA>86Vazs5(d
z-KW3%d%Sj&4iJpp`%*F&bTTSS+|;XUPu+aCF=qXdRht$*TBl*ZV%M2K8>NjJf0ys7
zIrlsyAwt;n>eDHbGN%@<b>Jw-zgc~uE57TXu%@kl<}o9^#nbPIR+_ggU92;^MQrnQ
z7oCqs3TL<<%zb*<yUBHxw}*<v<Ez3Vg=a!H^=o9U`6IQeC-G}U>H7cvqF2sL6f5O#
z5p-Go@le;)m4~}MI#V_$ZHP;9XVA|2d_8jmXlhjv+yP7}NZN47;K(e!u6wILHfi75
zE9A6>MQrK?E(fLWww5doJYp+%sC@Ul8THj}byy(lQsvdtOET41w)O1|DU4MWyPmr;
zSFF)WsLJb&k@}>i6FOy>@1*9e<YC_M?BWIy$Nl%V<(n+EQ7{ZmZk?>mu+@B<<Td$~
zz1QAV>76XCI?^q0Mrq=-`YT`d%zFN_eQn?Kud+u!@3H$U`=iG#q;AWs7nd&t?l~NG
z+&=5u{rb-b$~8G7a^Jo-EqR#{6{#EWtKV$)3vC9KwB_%m9ISS!9B|p8o@sN0E9#*s
z>#tWc-k5m@X5L^_c`)y*(CJBR+7tCnBUs*FyDO&gE_1^w(Upyxa^iQ~5?q({<gH(D
zhtuLBk2PM(t5pl`DF{AXu)*`7TSMm$-9<Y*mb<#edkbWiYkvrN`=jdRi@nKC`th2<
z5uYDV2o&3Y>5I^;*Txmwno3vXYR~v=xk09=zdJzZ_I&OKsf;=dH_qs`%2=NXykj(J
z)#@oG!ZAr3?wB4o6>`0J)8))ZquzH{SG%=TOg*^D&sHdZZS(To_Onh4FEu*2YLQU#
z-POK|Y^y(|M)1#_|0Kg$a6;qmvkx{I8b$<ltqcxZ_fAOnSK^dO5$S!P%kp}`{WM8$
zHt#&i=zH3#b)6kAKlz?mmbBrTY~Csn#gk`FvUi(AMC>#WShPy-=@zb875)1bDU=5X
z>SgW8D4!<bm!usSzbDUAYI;#-&DN$!28-tn9ZOj5cs020+?M)R;qp7f2<>05CP}O*
zF=2Ii$ey)eO?AW?_lMJ`oLu!!YR&b6ea_#LSLr<dt-kL1gWu}^0x#QVefvEpApCX4
z{m)tR!mYo&{xa>g>96<uJ)QT}2Prb_QQLC-l;D&ZGfsWVQq8%Zad<;h*rEu5J1Zr0
zoOGuy>6~`Ce3r+$>xKOr-|VaD+{$Kg`mw^%s#z<UqFs^%Z!c&2w=1jn(LA=*QG&9?
zw?Zy;1zwsEpunh~^!OU5?k|g(?Qegdo}Os2K0%E4<jJNtx(pKx*%=ovU7P(idBGZ`
z&d-xxt8Q6TKKb>kvt~Q1L!NC}vXuEk@Z?pOCU3fN#4*T{>7ni~<^W;S=td6K?-ozl
zuUkLK@YXX6v&cRsbl^tf*&CA9TW+u<diGqItE1ui<KFkVr<87p>Ay~2scsU+6Bg5T
z{jD406~iSWiN{wwKl}82LhP}9d}~)#9j^YE&CRJTYulg{w9N0Mg!t1#A74$(iP{(x
zDrlv1C+jA+qUbY5@E+~#en_0|Iay%t^~beJTuH~0Q9)>L&_n)%3rg9#<6ixF(w(s7
z)4?L=Qa`(h*WveeUs%dIsdDA3X|H+29&WhqeK2;v^Mg0ytt=N-YBJ0Ty!?#st`<k;
z*Ql9`H>@hS8z|JW^pNDu9g`KqR1YZiD@ms6T}~)}|5YYtl9l&;HNFFBF}yWif=Qv0
z-><XojsBs!a>>TFE)UJEd$e!gRee=dRr9~)-;1?b+pdfMv&z0&J}X3W`|G;?DV{O6
z7xC^eUoNG=vU7!6cC2@Z<D9OGmXptxmX|7hUQzd9>fQ8tX9HaH7&aRwD7Ri}{a^p>
z-`brMb<YYNxmL#RDbTVzpIu+@;NcG8o)b1Rr*7YslIrXGb?K(;95=J73(l`I<-VvG
z&-C=)^po|o`V#fbd&f+SZuXqKl@?_iY<u}?Z0zOOsi9NnPCvEAM*XpY9uHfy%7*Q`
zn;)<?<lC-!D{sbdY~`yp9m>X%cHGVujt3qV2)sLeefz$xH{@d+7I91OzmO+vQ+`+8
zbl1j%rT<$d_j7!SKGN}qf6^nN*CBrHMbqux+<o-%pkvHli97ow?i4)|I^W+}ov|+Q
zt*YY0+OIullj@~!q<=WPsG045NX6@y=U;6pn7qhF?~cow6Hf6aoolDm|BFr0I;SHg
z*R8a6mD`DgjcTpl%D(9r6IW$rU)Io`<D2<3YS{_pq&u>rn_W-5pPN7TwT$Bw^=!wg
z)(4x~JzgJQ-**3BT~%4Fwi&OYtZG<9%|iM8QeN`~8|n;>#U5p5W!lU5Fl1TV(w(7Y
z241f-6E%yTizqgju03{bt;Cwpz2Bo4*M!#2O#OL(!PRrhyC+xYd9u~B@PB-u?f?7Z
z>W5b6gy){BD6@;@uH8@(c_vLh)~&ocE6U|a^ulv<x3z3a>(dLG=bG{L+0PBvir+=9
z*_5Yt!#6hV)bAMcPz(2-t*gr}o++z*{MD`Ru+XBVnVC0rO9L$R4=7b0c6(SPSjlnl
z^1XZWzh&)A_0IN|t*FbEtKRryF~?O-%_oLO1Ws>{nei=)e^tf1Ipxcb$UM2E(w?Dx
zbhnfFO4j0wum5?R*prvr{xI?6%u@@a*cto;Cmoz5x9Gsx*Z)@7+Q%H5E)b(y_m1I2
z)XmItZ`l_W9!*}qX4fe1Q9bIq$gfRl=kafw@2mc*shRP_YSHVemvao8#hau3$}ffc
zxLI^=UlH_u`}%o-N1r~eI;bZUqsr)&vFDUP#d#yXm{jRM%fC)r(zg6!-W<(I5uQg*
zI(Dm_>QuS5WyThx&}-ACb7f9(oyj!eb;r%~fyTwhy%mfiGS+op_z~-=9xEo+#lP&;
zM783g(j#II=bjf5%GTTc_w1}cWe-0lJv;Z_?*96qSyz5+>3HPRuJnCcSn}&@O*;>T
zu6<Q|l`CQGtJAADp4BM#_<g-_nLba+l|^c?J{vsVZ>|;K%PBSpcwd?BD%?NQb;Z1^
zNf#Vy-1^xV3YS&BP%l4v{6wk2Z*ITuC08y5dM8;u>bKhX)mLVz$$7C(KhOBHesas^
zoj;kF$#v$WQp&2(9i9^Fn%s`LXl&TM*`HTz{k+aOySBzEq)l=QGVYOAmRbLp?f0}f
z`O0Pw5-yz2>)gw*rF@^FLZ|(<?cZOndndl_yR(Yn+pGN1n^W(zt1OyyyZpeKv}88!
z%LPwXyR@}dW{R|H+`cV4aq-U}%Z5FR?dI;Aaa+BYe_EHRJIDQ}&08<XKWWl9*71tz
z<qdJs-HSFIZ!uWCZJ$-}p_dxHMHZ(SnUdKqAKEK%r>G@3OtdP!!13S7{!a{tjFL<r
zX(g@|__Oh3-_uRkqP1>&EEcnFt63P?ZE#&><D)gMY})7k_^q%^%zACpFSWILnwVbj
zVQ%xeakAUjzuUQ7u$}R#Y+&cqXRi~x1$>zepUls2|30_<?&SABxBq|pemP6@X@l3*
z_y2u;`fq_j#Oc1sj9n9+GqWq4GkNPaRn|o2wnEK^BfD<RRO>DXU0pSQD^tYkU!Ssk
z#g(5tKK;4qHv4t)X5*cYT#_H_-DQ+iTypF2!Lzm1Wt<Dv<l4nDZ+P(JT=q2wK6{R(
zAODCxKP)ISC3D|Z&${%#{M*#xUw_~C?bMsSPu2^CvTP32)0JswzwDvdzei}bW@n{X
z>+S;n)6FeM1fGkl8>x6$I0Xu_xVl<4tTK!7m{noNVzqWdsQL=+z$gdj>T3COe$`vw
zSLVJKmKM}r@Um5R&61=bUyfB7Geqv4Iuzs?_lc>w)WAw;ev`njnD71?OJ7|pTlez?
z&-MSFe++xw#Lc!A6}JROB({}sJTq7>@!Is-;?G;ZEPGR;q<?UE{A=5N?)wBcYQMI9
zSGQ;HaY-hIV={vITi<**`rw=E|2MApqtiajx0qb$Xw}fPF~h+t&MQq&uJ(`V+P#)D
zOIjEecD_F@GU+!%$3_93xEC?OZ}swZLyv0A4YUYcmi4+wc4}#$xMqV*|0y5Qb>6ev
zUZ36mWer#2<PQ-GPwX|>o}-|YA^2$)SHlkD3+&vC%a5+#UoG|bfx7*q=|9rmn16cS
zyzh=ApU@}ov(23PtCzL<hVUGC8CP6-h1p<ZS(0wUw6H4)YdzqFVx{S}b<SVEAN{(;
zK`zqrN}GoVL%@v5!8bphJ@&HaV9;&`C-v_p@y|`Q`PUaNTkd)!)r{RTr^Dgi!+CC2
zS(dhC_che^mrs@1@#H`$-{afw)S@M(o=!2->J6Ex=_b&oRKC*ZB7fDgg6~W^eVu3d
zg(e+Y#3}aM*SeCOH%BSYV^Pck(K!eA{J879PI3`b!zx3E_-Q8}>v0&B@4pn2yICmI
zQX%A$25<UgjqlqJygB~h&JjslE1S;qm$%+i=CXVz(^b9PruEyZ>whn93Eyn5(cE%}
z?fIMjQ#1S7W{D+!Hq&KN>)$HSpqp^+`S~yY3>OOHj<0<hke+l?@(dTlYLUOMTmLUk
zK03{6NBIwH7KirxKYty+UF|=w#%Y7;#aM>DMQ0<zuRS}O;4tm==f(RB-H-9;&8zoQ
zH%`u;b^Oz`g?h{x;djpbRb)NQ{WkEWm-mc*iRFv}B~w>)Zq<mLc4l#RV$?LgyGE(?
z6O#AKI4@-D?lg&5#mc5Fl6KNOwyn~YO~J#vE52SRHp3($#53t0yZ4zpf-{6pe{{K2
zv|=-h18<o3>Qj~s!pT|bVPCGUvU6F>aBbDTJyDk{6VvV6gzNX*;fZzYT<4@J#$XhY
z`A}f)f_wR2AI@{Lu3Y$LfywsWwxa2B`Kn$E59aQ+Rehng=b7D#>Q2*(pSc{K8}wOE
z-toESb)9$pCcXuC<bD+%bqI43I=%dn2~U`<wN>{2A61W*n)NbxpZo5+$46c6%Z=nm
ze-?aUSA4P1tn-k@R?jJ$gVpvrF@)w`P|)Hq;?`KVe`m{Fr7b~RUWX+QEep)bG-q8Z
zd?sS`DziPlH5ax{4p%uc^IMjK_m@o-@*muz5_PjTNG-^8-p!@ICFoXM>i)|7b}60x
z^LDO~IeS-v>)bt2AC>4s3tfNpom&$l&asD8!!q9FZ`P9-;cXVn_eM=xe*5;^-}^uE
z+f{w}%@PfYxyOf()Xw>H>~eMGz3QSJzW-PCSMNRA;gEj%))KGSWqbRpUuRxf^ncp5
zX!oZ9H!i4gF4#P|+@)!ozzGSyh?ZU_t_@n(FG&?#JEPo_H?!%Lb~wWcsRf$BEi?XW
zd48O&#I-Y^U$ySto;ANdu2%fH>h`;5+xO4cGhuhuUX#i@EyRLdd9oqThfI@*x~++e
z4j-{f&SDV%yK3t5iKoKBQTKQ46!z6oSAT1qK7B;3<3eBVnbqf`->kb`yx2bSviRZW
z=LNs8*Z*-Zb`Ld*DEVPi5%RzL-_?KoW_l&Zw#YDCR*E{Vv%7cG_oRYt_gr4Iw7>kd
zKKkj0&-Z&DrTcHnD&n$XG_a^NEj(<#clG<&nvg>uKW=tQT*k0O!-+TSp-ceBt}N4;
zwTjJKAKZA^d+TBC28&8@{;I?CzNuHVyDVJ#xz99UpSQ<}ryqrXs@zx=*>-8_i$yP-
zM4lc=EWT=U!(QAYVf`l=)<<g`W^OmpN}go2e$L7%ZI2JX$>D9<V&GnQa`%D_oyO&l
zLe6FEZ~l@{Cha3=7G>hUywyi8UC6%IeRlNm1AWC}C(`&kw(#$IIiI1sZ_VjIo1l*i
z*n3Lu>|#5_Y1FuCYP0(sFLkfppl+x3_u<FaRtS9E9KCn%{k#dOHXSOAoOZ9%=F6M(
z-r8L9t;)4>&eC}OQ;+3#Ww=!TD`OM*zU@k5+41O`*O%}872WJ1)V0fW!(RD5)|SAV
z4d%50H{EVkY+1^+lzEosAs62@xi4d$N3|wCZQWcs<*?{d@8H%$H%)?$|DD}@_iSRa
z$*Gpr1=E-P+!bUsM}Ep5nOomJ`CW{hSMy%&@U>qbxDNRLTai*_BCq_Z;l0h9MK7h*
ztX%6^9-KXvxhOdE`|02x73==jJoxc7{o{`Zb8<7yY>OElpDupj`Icw@Z%_(YE84tu
zmHyfi;Z?gLws4+R?mp(`D6xih_A|+5RvG#E?^gMqRN|T4^YCicq74Gdy03fM=GQBC
zG1mN>e*Nq7$uC#1{@8r0`QWop22=9I#rjrStg(<z^<s=P+08L8bHj;*R?QZh-)7&!
z?;Y;^ogU0@aNb@rdBLYuW|i{}OMZLr6xKP{rT$v}lYmZ^pSp{+|Al2*+`g1!v`Tw!
zepImKb<M9DTW7N}#H89~+%z^<I(*1l^xa2=MZMOoz9l9P)^{xBd!${cJ6}AZ(*1Wp
zUz@5{AJ4TdCzfbR+8oe3<6ZT7C0nen_QsA9=7O|zf4;JurxRW;T6K2IiK1AJhsTAa
z4fYGhEnm7cUeia8T~xJv()RCs_3hh#-+Z5E^K0$eFWbLwzW-ErR%5W4e0ljd>)WM^
zF5FyKUj9wEb?zSHo|5KuU;N$)Z+cnz(7fZO*Q%q9*G#f{FUi>-kKMZ3RO`Cel_iNA
zW@YnFeQK5TC|zphCOxC2r*B!_-M#NaJeT1?uI_2?v~Jih-C24rZU)b1DILA{{}Y<O
z?|B)kyM$M7_4UpbJN`U)W;gvX`-$&SD~u{uHq18VO22Kkne~}_JE&=T9n>`S*8Z)x
z@UU*#akFh-T`Y80vi7*juX$YMGtrviq*BarOa8@EbxXH2r}eBTnqV`hV6OaEcb9O6
zf62wgzyI&D`}=tH_5X)|e2t%T?SIXzZ@W3Hn<svHKD#MyG3&}TE~U#|+j+ul|GztO
zICjmhNZx}BZVD<N6Z+p8z;*f3&Mh43*Fr9=J)`EewM6cdhD+e3lTH`RrwHw=5_%e9
zs>=0-Ypan@&&may%2rF)WnDh?**9D3#mdstK2wjIXZ4*s*mJN;qW`dt>_4v{4(IDd
zjn+?Mm+iHBQa7zz>HD!G9`CkY$=98cpj(nAJ|*Jp>vyNwI69A>Tks}ABezHShS9Nu
zk^Gwt>=_vnPQFmo&{bxwiDfL?lX~Fu7V`%`)RQ$@ufD!kJehaT)A-W{mpU(fP+oB+
zMMl>zcvokScGYy9rN^ssbcIYMm+ETVJ>^v>w{oqn?5@;<I)_3Q7W5^a7tiBr2ySVb
z%<_H7Ppi8P!mWW;hfS<28y76~I4#Pcet);-s>XXKR~#(zUwmO%-oxGUMd$aPi`L`d
zn_Q!NPyhb+%*4k&o{I%GFI%|ZElWo_P5#lUu&?{FST}@yT{@LbVPVli))!XiRvyeU
zWp-c4pj{%fl>eIJmb6EPuWA|IWlr&t?>yPNR=v~jG0){g9?c48uU{{W<2mOv<7-u%
z$Q(QCyY*quFI)WkXn$yu#dPJCjHnia(#*WZHAO{?wvV?|@O|Dpud+n-Xz{9swc58M
z4*2WN`ls-6wckUn=TT9rUcOwhmgXDVEHtlk9bEEoPKsB^hd*mSFEa6OGps+x%*wsA
zI96h|Dp$B9Lynk4YM1}R$Koy*x_^C-{Q1GfD56MzV($mf?C9MynysZAln-nwF8a#0
zFJ@lPqBoOowj1fMUhY#@nVs_d{>+c%ckLGnb+6aX&uw#;cIP>CCv2WN*Riz1u!7}h
zc|(#WUi#f6B-h=R*wX3q``X6|+?V6-u5D!VK5Ovw&*~5PZBZsxi*(;`C2nA_D4)0L
z7?025qP2dOjhiPciLz;*y0XP%(HZUy3pV*o{i0!c<1WhqO{YgoHUvh@y8XE7j9B9G
zIkg>eGfKsqr>fdASpTo+owP>!_Ziz>r=!2-no8Vdc2E?T*F1UQp~A!aA&Q-;|Hb;_
zude#{7aB)T>n|_jPhXO_ec}9Rw-0aZExi)7!@=QqFtc`}Ww&nUi<W-L+cTKDEiO+v
z>CY{lZtPi<nU}wD_SbI}UIKsL_*={WU-NBq=KYRGEB4w~?Jh3(YdLuyYjWteCmf2i
z-l=?gp%8UCWC6diT83a%ci{RZnK`?cKKXO}q+^g>OoYltqirj$9NBehXINVC+66`P
z-uHeyw8rMVaI1^vqD7|^&ld{?ez43o&N$j>V3D13>^4i?%)L(k4NdQe%GaA~Z{ygb
z<5GI1ApEr3;zj$SHux#byq@A}e)YhPwSUh&J;`z4NDK2+u?v^pMBi|F_wDJO!$18U
zlRFE9eneR6ADbl>89KYuaH{hh(YZm#BNKC`WwS0VUYqt#^vCT3I@;TN_r<lJ-emjf
zyMemb<dXeur<@WN9gE#4Q_14+S~vTv#$1LOx4H|ggR{69FEDzCmp)uzv||ycMo8wq
zb7G0VU(I6?)C-xlezT&5*WXjFlSGXoQdZr$ykZ4g`t|MnY_pk`A3Axk{H>i)#Azv=
zNfD1$r>6%x)g{lW&RKM2$q7C7#a7jZPWxsl#awM)A$lO{>h!3~7hRt+Oub!oX!RNC
z-)|OHZ(6oIN3d<3*_sIllPaZFCT=)U$LJCA(493jFEf8dLz>YXp0J9r3;f5b@?WPc
z|GT-W>d)=_Ys#)OY<PI$QM2&eOBsR<DT32xYQ@Z2u}*!X&DqDg+m32nvM$u06ftdk
z_%`+n-#okjPnHl+IrUv%jdj7N813ZEou+GfM7&R~xqoDq_T2DurCjU^=cjJe>-^Md
z7;fdi$mUvCgHME>Ns#U#qesl!SPPd|WLy4hdAn+7iuB$eIty~ot8{#**fn!0dq|Sv
z7QY#s3%5L2B3TkFb>QjAjO_i)3#Oi5u(^fLP^0SSRYrp={N5^IH<l;|MAzJXmiw#F
zsWn1d!**-llv{o)qoX8~i`M?(h%+eqVV<bdwdKntzuk``nbxL6-q_@#yQ?%{s_V^b
zJjteFk=Np${k*kg!Ikx_3W36UM{d5!Jsue%{?Z_HYiU5_l&N=(QoHZG-*tBTLN@I`
zep8>7sq1S!w|$}&-StNEN&m~&tLI&fy!3j{4k3A+m8r)s%I}w|yRyaZ=z7Px)`STg
zHmUYA+jU*B>SOkPkm>z=YVq1N3<5WEa?WOT6@<z!3S-C)jgQLwXTFi`Tu<&9=67v%
zUK^+MMP*m*5=cE9)es~1+C_b>-IdKzGI?6hf0r!2(yG2>L%cwisonao`jgeaUHta?
z?YY%2Sr}B+%427VAO9+`z|^@crvLJx)Kx0GPEASL-S}MV{p%UI3p!b25;k<Ot*{bu
zH1Y49QC_rs$=<1z@;Y1_)L1K;*3a>tYQE-{&V<*Cn(|T&jy;W<xaroGXIjj^dp3s2
z`EiObDhZvvvQ+8CyqBx@ZAjBN;%T<|X)g2BhPHi56Bg~6yE{ohp)-5$^?UC+e4VSS
z?awHQxQbj1txZ)sxFy<R<@N&!f@co;zmiSpJi>MHs#ABtx+!8BmRtLzA9L!y=7^4t
zQsr7}GymnoS(@j{-yS$1acJLb!-Xd$?R7F4xwne(O)J{J@ve=!*J{7F+WNUVuQ{Vz
ztgaVKtKZll{zP>Z_kw6EHbJ*Y&Txm+IZ7-6_vbKW#NSok#kJ01+P#E_9BW$+3ROPU
zEn8W*gf08a$7jt4pB>sBSD~}=z0Di1=>G@L{{7(kGE<#rTD8<guI`$t*RrB4^3*Ta
zsa;d`+PF6UwNhl`>d@L5;E_s6SGIj+&lbg><m}<yT{<UN$CZtncMjKbj>->j_r0xH
z9C*p&tKHl?s<(P}CU3v-Rc)rD;F-0L=S1-eX1#fG^z~xy`IDyQ`0L#l%f7$=wQP!9
z=~Kag3+HVYO%({R;7)1yE~fRm^ORW{&$T@~-c1~-6~0>@CJHQ6-<ZHC@w#Jm)Xcde
z3e8uuT;it|SI!GR=gTFVesqb$yywTP?rkgC7x}>JIj2sd<MJmnMc3L)pLVThlhV8&
zvn^({-qoJa;o-`2$9bZ4In&}IzRT}?9_F3*a5?F))T2-6>EfbAXXpGqm|?v9L8rRw
z{9~K%GOqce6xsAj?8>_t-=$oQ5C71cxw$7r@5G<aF$W{hD@YYD{oGl~;=p3d(OT$z
zB;rF+`JTLttBnVybVpg`JH4LZQn~0){$!2k_xW?La@g8-PQ8<!!`EPvHFe!d?oA$7
zR01bw8ZMltf6k3zk<ZT$pOluKn;WsjWc$Tqf8SM9?fp6JVkpyQF%KPv!ew(dFYMjW
z!+7cfr)fmWh97HV1a{uv$@AdsG0k|}DZ+pD=D$B}#IN&Y^_tScWg3^KbQ?14(dfz%
zOO4pd5&l&XT#KcO@JcB?)qkPilRni&%aK>CUYo7CsfA;6+jqAunIF17sday-`mOP{
zd?}-%tTRjc)(=lvwj0_uKW}*aZSujbQ`Ott_}MQ;MMmp=xOy_-<8~RjC!CxOPV?M!
z!|&|rKDemTv3mDq3ALB6|9LLj^F~e1#Yod_!o^FKOy1YBf?kBNCC6$rm#T=CEE1Vs
zdtZzBci3bm-L*fCoK0GmHT|K*skM_d`Zp&n%;(tSefZv+tCe>QdQ28DdM7=*B>TGJ
zU5~~h?M!aHh}F!y7Jj*~R@c#))%Vr9eMW~8D!=Qqo9KjWce8oHIb*U=i2CJj9lK3?
z_sUxoXR#O<#>AE7rY^o9!M;?|VDj^<B-!m<9!I^eNp0i&Ja?b|t&In}xlC2HIrm(U
zS(_0Z_4B@2((bfx7cRB;l(#jvoH?`c*e3gkondpf_84X^(LB70JKW;F3=2beir2+u
zj0XyI-6ni_v%RMCh!9&h-vMs^$45nfH!SWq|6af0VE)rh!G}K?HT3^loGQP@{^v=~
zdXZX*s|lS>PnTI`adPGSY)fcUn_6@gQGbO+O)D$1xVt*1GI{>pB=cbBtPMYIzpwp$
z{pm#RqU&l}$NT>W>^fN1{Draf$}F+XdP2YZL*jMMh;c4q{;<II)mGDnoIma}_FvL9
z7VQy8OWAN@k?N|`98U3G$7jy1zI^rNCj~|S57Xbxc(H2l-gry@6#@%d{`w~Sct1~>
z7bU57?Sn{O&b3nq_!Jta*E<|exLDu#J=}DuQ?G_~ctKdn>)_W~KW;iZvQ?B%d=%B;
zl9S>ibzti~W$~>NE;kxh?VOQbE!K0<RiXX5MRnnZl8FbI7Sy<J-MhCw(e3iRgvNth
z@$Y2g7MqyP;M_kePJKhlOu@3(H#DBuJ#kTFXVoZw{G{>y7OQ>{!~Zkcs&n%Ge!AT%
zzgIJSve(O%zEhcJh*Y^{Xm%`FzwjwTg12nSs-^N(MIC{D4#`cdKHZ+)&m9gr9y+ws
z$5gq6A%smN>9Rn+T+Ua%_1pWMu4%>CbzBq5elkf}hjSz6)d{Z;ub%w=E=P!vP<G>1
zF4Kth+pQmJ=e=0I=q}IvquFauEev*ZTArA8mZ3Fr*~6|QQCF+ii-KAgA<5Z2hL2Bw
zj(L12Atzbx{k1P!3SyaNmCpJ9<LU&XHP-5#&m?pn983^)*l8=Grt|u3;=eOTRzH{i
zo#SlJo4QT<*UGi(3uT#C-?cwt(>zs7PeM`0VM(fnhrHfgkD?BS?2D7Eb>7CW6XZAj
z;MC3L{b^q1IkjC`FYlaA-P2w@kD;J#-kT!tcGe`pZvwrg1!X}dpPL;Pg{+y{Z=Len
z^N90%0XE&@aNbWJjMN^9Mp-2u6D>Wo(ea~{Yr}zczZbjCv^QCCBe<&Zpwb~}w<+r@
zyDv^SD8#URL0$H@ozp*E5SqKHVBwl~PuA_2^L*deJEw|`B6jL6YjqD7POdT8I{R!a
z*Uyshm#trw8^XA)Bd^3xxpwkf#iLU*3+8eMw@iO0ak6$|qUWp48h03dxWv=Dc&7cI
z92R+5{)j<n>f0!*+D}gwl)vY4Fs~HY_L*h(lJkD2&&SEySYLFVV<Norp~AwLCoM~^
zAFa0*D`!*OxX+>4m96{q>GO-4<PN3=%~^Tj-<GAJrv4ZD)<#c^SY7oVJgPs*t1tdo
zk~+J_v`Z`dH(ao7mo56YM<Vvyp_!Q*0-jtpy_m4{Z=Qw8lIQ$WlIpkieV%YIDJnAK
zxWSHX)0-EIx98@GE>-FDoA^BQq0rne&M>LaVoS?q-{g4D8L1rS-f$-+@*!V>w`_~l
z8nx#0vrRerjl5NQiY%19*2IX{-P{%SEHP}yza^a<K2ugTuw099f46F?BhQ={esXSJ
z=f!4S$T-uyDsY*YK(ouufMYI?Iaj*9ebUMPNbA&w2wtsCZ{OBdJ~sY4^U_*t|7zY^
z*Yj%+1oAxOW;}7JS=G4X<^;xxV#R4GClVTWPx57Q2$-tj)1%nzdN=y%gde=8AFuhK
zvp~Z3&QU+zHIm#Hvc!yRgCiYZhCOR{OJ4h+XZiYB>>XF;2DkWVe|o*r?|sS!29|%C
zb;7oq+B{<NQ*v*--zW99zRTU-`bWHKx~=r2h^e~rp`pLte31_H`G4=_a{I2XgNs*L
zbzD2L;d|_o>*ws_qQka!hcI}5t=g?D?tLjj<iPS-8FOzQi&k%EU3>M&t|$3#Betyy
z{d)=$FIhRU1sdBK^4@GdHO=pP$rY*F5eK`^-anea`~IY-iSfrBe?QETT2oZ5v6h{o
ztn(E=YsBfJi~rA?RdW3D+ItU=yI<%2cg4s_qxaC`e{BNcGqqTA95bs-IGS5`2er?f
zb!m#&6otCk#`g|YUUHfek;C!KxVfRHpiBJ+YX)mVlI-pr;p`0U2^y(A<;4q%yr<Y2
zH*VW}NYU}^mJf3e^%`xx^EPXZ<J;Dm%vKA9yc^ENc=X*<=6E2LmYsL5{>?L&**|Ye
z>G``|Oyg5%l#kt0TvxDA?9Oe?hR4FYO_#WOgqPT*N%EZBtu{xfx_surCD9SupLt#W
zyPQ`zw>GQs0@uwjld#`)ej7R`iL@I(vRttzIBUh8sXzNQ9rl(<ynOwS^WxQ<u#d7l
z*K(}?EMQ|u@d|7<-M;Ym;**z4|DLpX=aY9wU+&NHmoFZlO*L-anI03vY_aZmZ-wBZ
zN^#?IvAK7CrcN(Cw~1r=tHu8^7T(oo=Q^ZwuV{in;J-!PTefUnw>0$WGrKm?+O4bj
z1GO8ruF_ww^MAKYPZ4jlzefLr<~OtEUE1N;vAT7l62sl?bC<bBJh{B}b@=W@2f3p8
zi`U<Ju}f%f(0kSWyN>ETjZE2aZdTmMuiq{<C6!-eIisxLz4%qaZ?|)9t;G&u(bvz2
zH1S+x*i|rjcXSAEuJWC!8zSEbojW;uj$-D)6?1&om$_@7@0_WTc3fyn((Lt1MdjJ1
z`mdR9ex{i9fJC^Z`BuKT9dp}vbS<w8%E;+`x@(1u)zY|(1k*2~EDByv3OsalXLL`x
zbLr!RD3f)G+tSXcJ!7~Mo|fOU_OD-UszAkb^DyqYsf^c*3RSJ=-?>`)KSd%lQ?SjC
z{ea9{m%F0Vty>qD#O_*M__%}dz>Ksy0jq1P3Z-U8hwk|IlvPRVyvJ3w&Z1jcOT5Cj
zW=<)awr=yhgIW7xOp2e+53b%_{Ic)xU;XA5Zyqsw(<xh7Pu;SbsnnvKapyo+^vfMp
zoDc3@zs4vMYB7<$x=izC@S3;t_HS7+tEH`=t03*oy}N(DeGmR_qBM=`=W^Cxul#t$
z_(cvS2l5<P%ozFV)X#3ci!pxhCPl9`ffr;(S~e9QqvrSVEPS}BLZc(}s*~Qs*}a#S
zxz3nb(eolDJeMg>|9t4q%1qYhTPjVSCx-9Xcz50+8*d36yQO#SV+y_}9)A+(dM9W~
zDbr=$Ij+V|9153CMLBG`*C40k&LLjb`d$BnT|{%mh1coZ-%K<WyC=5r$Fhm@C$DEZ
zzxAA|4KH)Jq#5^=bH5Dcw4Kc|aaggyoI56UU-*l=x2G=B6?4CoF~wEEHD;<tkI_-F
zsT(8bt@d1ab7t4<)3u4d5;1o_cq)JVAM?*!z^+qc@<J<y3nr@0{7n187Kupe1U$KC
z5y_)0KdJbxyl%gMo^W`^mGC<r8!yVUxgSh@{^de*m%$X3u;o`6P9!+~`N*8%>htWd
z{Ol+h{wMiU#bREso_M5n^1|Nkm5uVUR||YUOl&qNm|b<DMrlTA&gx)El{x7fEW9mR
zE@duhFuAl!FJ_6U?1mR#rfEG`68!fQ(}9vJQsG}O^81xvUDmMBWqDP;cadPk-;$to
zTLz_LT+2CHS4PRl{e9B!^>q#3LlM<TZj-VV2OlxXZ%!y_ck_K;($CJa_Ui5&3%y@|
z;$Dk(Ze8^~4xFhLUn_~TR(5Pjmfq}MkUan8ilf@z3#W5zcKM?9^}(~c6A26N*_VgE
z%UtrtK|;q=e{F(?U&SBMQt?@g&eIGck^=W8nEx{RXe6C=@ACnF8I46s*QM6_Om#e#
z6ezdYD)xpb?}iUXPrS?5Jib<1@3MEBM9ukgr_MP&oqBkQ1Jmnjw+*4rVS+ZLUbROB
zo!0*0SXla{WWmdfN1xe6r}#~o6qByRb}4-Fq4%~<tNk7x;W!|1{nQ1`81Lto*}{r)
zcrWxyTGh|_S^p`KdvnXfxo_W|E8FPz&F|jU`BPW(hS=$D3d}m0(5Upi?V!`*gH_Yp
zci(@=63RQz`EtY_1LXsGuNdE}l$$o&ea;Oye^2z>cb2K&cbqC#Wis9W;LVm3X}$ZG
zeV5|8m3By{@JyCT`lm~GnCAus*FBfq#Ol5!?UCY|#3hd%*YCZ1Gl+e2UvANS-q)Mv
zweW<6Jeex3w(9m4j=%S7a&vDe>Bzow{d-&9PyfEn+P<s(|2?z++%%8+E19q==cke9
ztk?1k2Oj^vsNVi&?etz>n?nA4OW#Acejd+0lpv_sI!Px|Am-?Wa+CPyA68c-&+j}q
zZ&UMwh6M4lkVR7p<vRMMId-fH-FqC|leJ7$o~pi`q2Qv-sS{eF&93hn=lu8omvPbW
zuOFMY?dr-CWxK@LJeW1u(*3@7tTX!({3&tIlc$a`4b#u?#PRh#;hOPXCg=VMuAAnX
z##7xwb9Ea$isu-tHfBzUdZBY)>&(&_38Hy<lb77O{X3_3qxX#ysU-;??oPQ>u*~!5
z$>(pJjX6wJ`RDfvFbT}9o+6wX*nHJz(Zo$fa}UL)dProhDOmd1@8JWUR`)l~Pj@PL
z9@e|D$Gv&!3wFk)nbn(l_ln#yF)4plSLUt0N}%b?u}5>BS{fy6IAVS8@fH1R38p1c
z>U(l+?i8k0$1FH8_rme?%;#KRe1c~AF4dl^&L#86TjbFigJYf9+P7zIK9?{#;7M<^
zq3|@$c-8%L1E-$4t#`p=+tkVkpPzGUHW?+&Jn$)b(s?EZ(Rb6|MLbhInZW3=IP65i
z$M@Tc)@B-NX5T&{m=?I~hxj=!(LA5<`*s<BlYamDy!WJjclFcrm+Qj!skubyPk!>_
z<-e2R`&y5GUwwD(p^!KGyHCvB(Z2m*>UmJ(k4^i@T5-LNEeDfcSZCk7uNab|Cm%QK
z^r}Tgdf_{+RVzQy{*?tAFmmf%yL#@%$=v@m!p^<65HAqsi<*2SD#CM7aN-6Dmm9AR
z+xD)Q6;vkhD~j<%RrM*pA5Kq}=50Lf9k(y++U?y(^Jc9-ZDGz^C-BASloeyZ%})u{
z&CJaV3A?rwu228d$aZ&i_iUZ}Lek&MD@=audN-eit6cxfGhI&6i`IuFFR2SOvpo;J
zD*Rflb>At4tn#ickJbp7>oAy9J`dJ-<uzs2g!@hRwk%uvIdh_-M{t^W&x-8GZI{;S
zzFaME@XWnilQgvU{MW9wl<MY^<<FdDS{(6}NoR}E>9>jL0`m?|`r_=n^rBI7>GLNK
zF1__k7E3++)A8L~oAc_|Dl?wygy!8bNRFMinrl|o8k_K*+9%&Ghw%7*FEF!Zc)NGg
ztZ4b8YfI)mcM1HW>AUF9s*lFpnkUaPeX)FOB(<n;Uci@=-YFZLc&Gi*)H8@E>0muz
zAj8nQxpbo2B)=D3mzHKODM=2!wQraE=baM{D&4SJcihbMxb^MU;x}u<w~0)w_AqU2
zIx)9?PuuP<Zywrq`c0SAVNvkmQz$Y$`dZJVSwHIejM%W$gj4Dl8Y4M3gu#X$uFMK+
z-TU}o^cyW}gROkIcPdq@o>zQZvuN@+_w32JE7DCvgBKQyGDQ1p>=$6uw%Ji%vV(bn
z#GJR^ro_ZC?~su()N+@b<F;zeU7v6B+%rR1*9V&@x9r>R+`8K!;<U7$rLOP1p8NVn
z9v&M^jrT5LZ9b)0Gf$*NA+F(Qq*G8N&(YZ;GsGgb4{te9k*5~v`gEHA#fx00zCX~o
zmz9~i#yjD8j^Lb+wHq5`8FQZMu(PfIz1X0Djd7Er<579%#N8zx>%Yc1E1#{{z3lol
zb*=zQ^{+{5Z%9s=^yToeg2agjcLg`Jx~hgP@6?szRrZ~s%JyrS<5GsCwKcD{m8Psp
znZ<uj>T#p~?usu$v$yh1&CB%T|8c^@>(P_LXVVwYZR_ybFy;91*7El0N_*0dhi&2z
zX05W``10nDo#v(2MHCyXj2qUj?cOZ*c#iPZvs1b@ZOV(0Ua%=Ik|Dq=Uwg|v9X8|7
zw+%NRn)UbKk7;{5?i_wHhn0=nTUAL{j!P?=tNCkM^0mtNw$v*-KR>N_yJYJoGv+sk
zG&Vl*U7NY}5WEQz)P3#bmxAw=iF<8ZyFWKRHvDMKzuW%D$^IkLEp>MCu^rHK%Gz+F
zB%n-S8^_DSrzQRpI^RtmmA);PE8|?1w=tUM+!6Nd9b((pzq@#H?$j4s*34b<h=JkM
z<9}?QI_`u%++(QRzV=cjS8inU)}?$$swY<(O*w7yu+S_Z$yC*E(mAK6C7~K~U6T)4
zYVJw4p0e!a3Lm~n$7hKYw?%XY31=TT>DYI)@fyD?>)rz)AD`&nTRwI3l@0AzJgR@2
zZAsv`smwIjHL+SaSYl7A?RQUW<5MDYP8O>wc3w%~oO-m=k|D{YDqYt_>74dPw>g`I
zJKXNLZE)IKzsjL&#ZpoBUuN95Laf@JZ21%u)p$%|<Ell1Zr<7!-nG9%+3LBgr!s0i
zUw*7H;K1k5C+pTdD-30=U;aO!fvYNLL6aE&_frd_o8K?mdv{`jyXM;M0vQ{m`q^Tm
z#WPKH)lxRB$THvd&FcGtM{D9vfAeqeE(mktj0}nS$lvzw<<SQ{d<SYiY|7;_jVSUx
zni_aPU!F_u&YkH`S)18R4D!wupHA)E#GtYzec`cdYhOM(7r!<ubbk?eiA~R=tBJ-N
zm>HyhY0BQnwN<R$dO>N5q|!Y#tECBL`&TmtG>iU9xajY_j6?Ilr9f{99l7<D7W<B_
zE1veD&#8jJXX9JHdhI)(OoD1+#jM*LJku}w{A~WAF}FEvjgGBDk^P=_#})>uf(1vH
zG%V=)BARb#AL^beH1*}mPk+@cP8<K~iEHv>n4~(#@w4*ZYmQHK*FHEi_1r}VZN2SA
z**)IV9{GBN@hVTy6N+*D&6%|F^}nhU61q1GAISD`@~)Z5>z9<ad1mqLo<h4>%PuCI
z%u|%e$}bmNF^ko4?}3|7d1u%j6;7}-Sf02cW>G@VW0l8CKJ1n~XY#Xu&eJUl{)f}%
zf7@uOvTM)n>Urm-Y8vbGI&Bu+(z$RdC;5W>txdn%7R;C<`TB>)A`?Daj^?AahwN1j
z9e>a~hw*R7+k-b*r(ZmERzl~#Ov%wX=k}I4<j2f;v({BU_WTC#8>&y5f^M1mC3wAT
zO%yD9C72fdHswkJ=Y%LT+1M4<rV&L96LT{EWxV!W)MfFxxrIB#Jll8m#<SDjgXRhK
zz&+nfr<e?`{VzzKzx8)}(S)fjX*=bn7TPQ6TKdfA(PhZmpmw{!e{tFW2|Uw^w-s!D
zT(qohVQ;mB|L(rxrCSUlUR(69wP)siy#G(r_X%!S9k)zd$>WeKE2-1aq{!Ov*D?Ig
zotF>$e!5J5`cJ{I-5|nPdyN~z5l-u5p4Q-iS#MW)E<74h6LjJ0?7dFh2|*c7+6)1f
z(IMeh?Z&gOg>0QE67?x!$(<`qyP}gBqYSMrefE5OuXG`{)gW6@VtMX)mDjr}mrE@a
zUEaU=+un;7854BBaCNRb<aj96(e~Nf9fxMk&iZ1|%^SeK`9yLlm%HLD!)IR4XJ0!}
zp(a&q^#0kYziU2q8j6<8du})H#O^6KT{JiK&*oKrnjXARc&SH>U}NVUF2-8d^?}<8
zYv*qiJhyx0QdeC?YpMP=-O`3vA}9YI@(tNG(QWOexu%c1gc&A=Z7uEOlF*sj`CWSD
z%|km)II==MB%Qi&GIn1~r>WP=ndgrN@qJ-xPSx+4TfgU<xd#)&ncoR_e==C@VOa1|
z*1Z3E{?yaUS}#WR=E|f^-4w=i4YoY3N6N})jdXmKg{W1>rw{JYIxN>0D>i?99Ddkk
z`BdH1E`##A&{vUr`-+Vs#MKsm6kTnyJu<)Fz1U1u@+h;~XN%pFt5Z1}R_^0ZSY57R
z7m?^$zG;oH;+92g&V*=MT3&Wq?5P-X^Vn{Ycf4F1Yra2pnXS{pBPO8oaK+63OZc72
zzOB0WXv?QLFU{WwNhJ$KBpW+v2MQjXxyo$q8NZ%|j4ZOtk61JG@-wynaV{>~r<tbS
z{(jxwyP5gr9dn+2yKOW3>bCUWiN99NzuWd@>WlddDuI4H2iMqoz5i=>|I0T28m^tI
z)v7YSUE-c|v*}UY!QeW%v`P)n8Atu(8&;OKRL$<SWcpn`nW;hSi}E{(wKn3BQzZ^e
z`m$(!XhOKzYL?<1T?Tx;S0x)xS*q29XYAOLrqJ+?XX<9_%ICZ4H%%_rJCorpYtSL!
zuq<lb4$aq|D-If|C3kNqa!%avfUB^mf3skN&&o~P^1d@N%$av``_%hhXV|$nxz5ly
zGE>M<Y-K8A=)%gbdndouZ}@n0il}!%=nbWmz}+VwN^ER+xPFDqiz(AouKC4XI6Q5C
z?(NMJ?<-cR9*(;Ddo6hIH9*W^E1z+O@{T{TAL5>0T^`e1_xtxtre@R62aS#$NuCt(
zF4N_leemJAdm=s@6bhZ@V6`sTU3aO;)QEFum#n{hUD>niYibsYn_cF^v}3$l+H;)e
zU9WRH$y9Lk%(aKyk~(?Id)aL_7K%%`h1?c$e=%R@^1pKnE++iEth*tyqMyM(@8I$~
zIZ=l<n%3JrYq8f|KdT_=;>Qm?8V73@*6q>gXTMMr?EaMJL|1-w_3q^5u{vL0b|lZ}
zx-7Bs{r0z!%xnK>D}6XPtD<)MI+-YM9<jn32AAd}&iTf6@4=SJyBGMGy4O6s>$Gu$
z)OAkjUBxFA)E_IPE6?Pe=`v$#cD~=WQ)$T!e{`ou^GdCLaJ!yqdCJ^S;|FfdZ7Mg+
zs(w|+TwS33>WN3wrHV&~dRBb??7!NNal%2a<s6^)Fa$(jx7aq_nC-M^l>6mH%n4?J
z919-K&fKfrwEV-SB@%N&`7R&gO!4P-Up2%2)S<ZQyKy4w_jhwju5nq;t;x5hf7*-_
z6F!LL?v~&d{-2r^u~X-q&6+9BN0u}c?zmQ+T+Nb@bN1Al#L)NK!IQX7FZs@tlo%Gx
zKPcHQ>#4Mujr+i?d7oYdpSd%=c0=2Dw|NIo70)RVn7ic{gTvyBhe|^qTz(O6k@azh
zJlCEb59WwgR&#tk^55WM=B)L?Q9nfv-7@S5kej2)`{SI`(;^cdu?GDuh11z2tz+tD
z^l_E5u-#dtH-GcIA~}`N3nq!5_*Y18I(~EKkMyaN*L?c-K;`@$n+)!Zg|Gj4+H{C@
zXg|^_3U_}xfqTYe^Ovvxc^uabie%_pK0C2^L+*JE$Hm*`=1cN!;d(4?*=MfFw7X@Q
z6C1a0lwY#a{mhon2ZG9r8ZSpZST{|)VS}O7ZP$_&vqYk1m2)JBH3nX4P%&9?I%4`6
zW%XNG1v5=VJ6E@wdK4QhIGCpTg6qK4w~1+g3Y(T5d(kwxdXZY^tPi&uc*Nwz(jG@d
zAHV6b<XvF({;O7X#c%CDPWKdgw5p)X;Pu;`YOim-zfra3-KlW9-TRiZX{T&BVSc)w
zY1_2r9PdqvC$2Y^p1ySF+bP%dB0IMksXXPW;g_9u)+2oDs{P)OHbPg3U+eCT0Xq_Q
z@HWidbK{)&e~y>IU+1>pZC$)PMKEpEky_pFlMf2D{?=H#bHdkeT?Zw3r7FI}U#M(h
z<6ZOm?Dp%o-RJPn&lY(m*kBU0M?im0_A-vYKRLbBPbnm9uyEQ@5I0@ZEQtGaa8B5r
z4J{5)9rE)kn}S{5=sXTqk&aj}^}5~lvn)voQgTV_FX&JC@b1*N6E!+L&r)ADUI@3n
zvFMB1w6a@I6;zZt7`=tK84Ft-a{L}?$$d#lDSD{vA)<CI=H;HLks9;2cP!=d&}ocZ
z@YVUpt02YftuoeUSp%o)W^6u@#yQ(!BilKNWT)N`4|mTqW>Q7ozdBFaNM)yPzQ0Y%
zb6N$v%2_MVTHiI9DbfFe#ol@StGFTm<lc<z>uTSPuX1hX(7wJZru**OhJ&AiTU;l%
z=H#A@OY~lzpMT#+jh+9N-1g9If3_|c{+Kc0{pmv6vU6HXAFC<O-umF@$>Wm`F5(aS
z@l_;1by-+qYoI0fexd1smD4WoJ@s=>`o~|N_}B6Nx64l7(4`!6_4hh(2^Q!qw}XqJ
z;KswbPrn6d+Umahp`5a~;Nc5NtDTRpFwMxh|1oN=(W9-KTx1qm&T8*iYqFjFwlu?w
zY|*>x7u}dY+jHK>GlC5pcpPS~ymTkct2MZVCEDxlqQnh0<x}s(=h-jxoHWr!_+qkY
za4GYT1Cu#Ex6E0-zN1UZ{B)$v<_4{+5@~FT<(s!yKUyof;!x@T7GAYNJ%)mpHrYqd
zDXh9Ir_;b_a?&#If1^}T+C7WDR~Nb&KVReNm5OEG_Vw&GPRYeZ46|9)u5}tRzJC0N
z;k%vdf#;rPo8y1}@zz)QtaD&x>v0)>^He>>oqu>f=LQI-c`8qx&%IFeL1KW$illP=
zl(jZRK~lU$O&J@ew0+5ac>PTDu0+Y?6stxiol2GHrHXdn7;hX(Fja23lCbpDdwHXX
zdwf^U)?P1qyMJnwJA>EsZ@zbbz7S+c+#obp?tAs=rlThD&$p*V?TDy)%^R(A^u4&`
z8un)2i!q<VTinYRX|se}o69rJru=7T>f0@0h08APcpLv^HA_dxy@RV<cdYsz0~*NV
zk4w(p#l^t(QFP8p<<}b~znGGCp>Wwx137`p>gAJnXKdhlzQNvU?|HF@PDf{M`nqJ(
z(izkBI5sSqUU+9eW56}fzttblJ`BH<A7}H2KRUlE{QkdxJ{zTH<cmHKNnCs|%g--z
ziRheI55=16A`7QHnc2a8<z1h7%A=E$<{a_)byw!k><jVQ2g{sNFZP5~=mfrUxb&{!
z)Y$^jw~DuJ{|?`#{;?xbcB*su9hp@RC%#TwW!iAAd;1^J?SBsSXk1Od-p{uv<I9^F
z=^NI4jZ6M=G_X=vX}RRFRleTO_uSpWrZt`8wD>cLe0$FCoRT^lb3U9o8{6D+I`hJv
z4XjtDE@d}TuAEV3Y4lS5zI`XVZKos4a@o0=4*VAmtZ~VFxW(z(DgQsFPF8%?YWFOz
zgnc;ob<JwG8LElyYkOW;^6%c&GDGPA*S|Sm-mEo?Q{1fS^#6<GidE@lA7U9wA3F83
zF{I4;%_qOLUhyQm(&4Cv$UoPP%$z0mF5!2ce|=@wUJV2OsanyKT-8E^jt8zxooL;C
zxOvkVeLME}9D!XERxB!tTqN>6h4sog7p-{1h}Bj5!Hw$&TbE5WY*TI#XDeLx`9v41
z#smIqHZ%S`P`7{7fBMP)5A*gb^{+X#Jexu7w#TymDY6Ax9}cDLvs}^E>F4@zpUwd-
zwsafzB|m?CzU%m0KIh%vd$Ug-ul{)aqx=4=wU>806*F0^JO1!!d8E3*-9)LlmtqVH
zN~C^@@dTIN56ww7Jl3-7{V}b}^Wt01Jrm+EzJB7f=7~eUJ~+jg$~stnZ95aT;E~dn
zq~@SY%hvx^zqIK~{z}2vr$?iw=8EK9YPK!9>umbx#l`oZcOLw+I9FoLWVgL`#<|nw
zpFVYZU8Z%1HFj%>-}Wl`MxMK;CX~N>Aj*<lnjNJV$(E+?{zd1;j^ds|(B8Qd<%<0(
zvt1YU$vk0*x{}7QY%k-|%#R9OU;WQqYP%44&MVvB!{S#4$GS(!4|El`+!1OC+<bL`
ziMRLWPE#Z2D)l3K-pv<S++CVA!?j~sy{?o_(*~cFha#(bnKX~HE%phlO3jr`*Xq5}
z`te)f`t|$O!h-5eOI^Y{{SG@i`Q7s4E=o4`)7GinbMNGaFSXCFinqRcWTAES^`6)r
z?s|(XmN@yiecNTS{oGB9&)O%8S>LC$X<b(2LW)5lug~_Sg?lf*sNeQkc9)7zx5tDD
z)l!wxt!avT*1g^LHN873`tvqV^X_=22dbvIv$Usw%=o9rwtDftwr4U0GQ3v%r{>CR
zS@cb^;Lh#*qwI#}3KugM-1zYP<d%XH?RSl~d^_o~|52B8_#GV<8~xK8)_s)|{<7k?
z*rSCXTR*Yzr=I54Qk|)h!hWH<<LaC*(%v7q53G=9+&2I2q)kdIxMY_s>H0JyC3IPo
z+V112;dj0yOuVDcp}gBI=7i%3x9xe(r%qW<G~+5yn$>V{mb4s8jdIV%YiB+zWccI$
zROG<%pN_SuNw&-1NV?tO+_iVF{aaqK8OAIzQn4zxzOA|W@J0Iex-R2{&4(_sOp0JK
zzghD7+P6!x?W@+f9E?zplQ_0Z>jOvbuVyiaj@1ess~V5iFc^F;@MlP_{U9jS7Iel{
znj`PAql@CL74oJ=YxtbzRwq=<exA)BwK!1ZT$=Z%;*%#Q?tdG#y)E0Yq51XsqdcuI
zUd`Ufb@iQ6TL1HO{=+J4EK+OEpYvyI*|IU@gh%Vw@XM~NJVBARXx`ONYs+_q@5oz}
zp_`bw_u<`hT+(&6;c<)M`OY-&S@|75>+{7cY|eezvZ2MXFzodT*A|t@&95S5t4u!l
zT}*HdORjG@;k_(x<LP9LrCTqneVwrAR`eS^g9zpC+??+i672Vy9^ZF;$26Bsj0~r4
zE&W*~wQ|$8jAa~+CZC?~XL48;^0aO3vzSGjdRf`BBimFfYchDwGDm-QOtyLz-SVWu
zY2(BYkNAZX544wrS9o_$?TeoNZpTUHd(+Q;s9xhJ<i2jUc>dbGDsv}`=lhoRpK!j_
zamVjw)XDjrtpPRfFU=GFYdv47Zq=Ql{etI~oVvX%{M^$h-!;F#ny`I1B;!7NbI#+N
zbpaXoI@mVEu1Vgf?vdeoEuhTez54WP+tS~ky1j6heX#CJQ8Se=|1_IM$#Rd<DHg3t
zAL3`2Tw73g@a8eOX&-NkHBRte5<X@7?``=BO<ihpt>nHe+gl~Ucwniu+`Y<x{2ZxM
zRYn`{WKLGoRo9-g_WAd$DVoVQ7s{P_k+gp8di}iq+x{(mO1qT$ReMBUCO&_&rC{OC
zs2Lp%4$WV$-u=z7@AoBvC(KJOi(fN;ynE#`-oGJ}q@6y7Eo*b^7Feqmv+0z{*RnOB
zjT_&iAr)KEjaR<r3qP}L%hzk!s?>b+^_2<hXZ_SPQA{<K*Lh|3s<!qw!-22gCT?Q>
z6uNERnL9?M{GY{mF10CqJ5^Q9UYcK9VOmueZTsHWz5o8dN8g^>e{_C%Nm3{K_=)Fg
z&u-4=JaSAn#Lm=}?REZ?M9){B@64SS^r^Tkqhig4(+*lk{4-QOEKKO$e#CHL@vklB
z3oM@>Qk%HO>9Qn?L4#XS{Oh#=RjxfwdR_|>H4e;bwKBIC+V`TuV_l#B%be>1J09pf
z2>#hqT4xocnH+nyB%}49(}hr%<Zi8y82j0;-8%Q}n}4@W!%O7*k!Y8w)3R<8ju%aK
zHsxVX=SV$aaf<z-MDWT}XZ@Z}J}|8|=*)r}OFnD~)BfDTHC^+S(95ZY%a#@!+B;vC
z`x+afp0U+kXf@+RWtKYmlnpV3lik)%dnO$1l$7e_{`tlty`Wcj9{XJ6>Mnh_jQ#q2
zW&zblr;qF|Is572=hA~gcRZLjvCjRy+a%(Wz?37iQ|JCpD9L{)=a|f`+>-Gkm%(Xi
zx!0C0Y!jwDkpD9`@XP9`tM@m8I!W?-j$LEC#&hdL1Jil-=gHEi1N(WzrdwK`aMnF|
zp<J%)X@j(n;IB=u^BI=>)iaIodg;T<TJS;jkQnpzGu4~!$r;;L%`eNZw~&?fU7cL|
z=lR==Ptm{aZ*asfbqs&CWB)`q(FxbC9Tat#s2OUpeCu|jNt4Wir^)H;X8xe_u5;Sk
zzq+Lf%jNRgLc~lTJi6PkR4pN5^BhU`A8iZ7V*^j#JD<05<@1~6dMnm_bWtrldQQQ(
z{?ntiwO`+?;8fsZ<a()X;v_Gaz`4t6US1bZH($iZ1=lSa-zs0(=58sP9J2Lt)jlSz
z=S8tkWRHq1tdD)o&|@SjWO-I^)23wcV-h=aFW+<EiJhI6s&dz)^HJQ2MK)ze&rW<T
zmM5f?cC+`>rNk{4mpE{2j(Y#XuF%hbvu?_s=f2sqUSvG3O*&k7I`U>-Z|3@a28;7@
zQ}wp)y*qQ}Ox6n<)~K&+I#+r)fl(vt^@@E=mt0C0uTg2fs=0mEZ^i}X)$hJYH0(Io
z$?o-{nRS_f*^bE<vK-sC$}AM_x_Ojq^&PpSOqGJjX4f6t7S4~$mv8;p;rIPg&EA5$
zE}{*rCElwom|s5EVr}R&^b~&D8M^k>U+@NS=jo|_=T6kJWuJNQd1LSOLcb}(>02{4
ztk^w?;}GkN)`o+twyKt^PF<Ne!%0I_t;i#ARqKNEjo;G0{(3(7_4)5+Qn?rYT(tLA
zW(}5i$+lVV5_Q#;Ye7NgvMW&=XJ@O5E#|PfbNGN=<b%jLF5K=vUrf-fdw2JiS&_Vn
zSR}6;ZxWZYeEZD6YdOn!Ky84xK1W-EuWbl$t*lwmrnWo%t*WF>_?;PV<YtK}N{QAm
zd*v95zUVdxH1S`^FZTM9?CTw>Gfz1rpSrqa-<FbdJTtkfHao1Fy7A_bUHA8FZ(im)
zLnG$po?r{6K9y59?WZzYy^Cn{QVf>hukzLY+#(PrvGrS7dgCu4hduNEt~+Js^4722
zxKd*6la*StON-{V8gs5;ShgXy;;KN@8|F!Cn4TqA?M;-F)qUKqv}d;YstbobibIkl
zUV4|G+4gD9#f$|9y%vYPG}tPhvcYI6*L1h4V<8vM%;gDt+U5{-E$@9?=!$nYcdYwp
zaP~U?w)_<<C*{pOvEjwp*G!3#nR(N5rCgd>8<JJIlgjGYSFVgw*f>p6XQrLs@7N@x
zXW5$88fSIS-nsZA!IwKAbmoMeLAyR)zYuYC)xCI7581BdC120nm<0_RcNi$_in<)U
zy!x_|QuEfn;#cdYN$Nx{tNmOT&MXm@FEIDTT%DPAMZ3ahMPEwbHt2rgWz}r+Y#x6`
z*W1wTK37-xPmYUb{_*35RgANgX55=;Gp*JgXTNP8!XjQ56UZo${(S1`qe&*gYbP8$
zB%bz+#dfan%Lvd!(a{@|^LP!dgr2mxW%_OEa%exD^r}~ZFS4qsyeNF#4oQdLlBJt?
zr{?m^ntb-iql#HFs}^1eZ&3>=5&T(L<8R=5Lw?HBw@GtjF3Qv%eds>*cqHT0@ZJqP
zzC!WdtEYw8*Hkn9N?=^<lN+V{*TTY?&HGlO)wh-vEuT+x`&Bbb*6n%MKWpAC&+2M>
zxi8a%OLr6m{kg4UyJ7vjz&HF1C;zf)b@%;Qbo2eS-rM_%dvvo*4Z3|D_U(4jWLkfg
zDKxjpv$sU;{O46>n;VoFFZ7CwH{|6??>DKwE1#P+pUdG<*13!=7ZXm(ZoKDt#pOy1
zx0Fs{KD&bL$9sFV83f~|KKr&~|H0S7nSx38to%&Va$KU`&r-74y^r-n!iLVJN=%Q#
zD;nQ%N$NC){rM=idyYfOzObFPPqxaef8?fVrRuiw`;qj;1$ieDLRVMSZw2k~zmUop
z^F{mIL(P~8p^LJ-vP+k<X}3gj-))?{P&fO!uN=bx&*fe5mzFi|=RG`cY06fcXJ&`n
zR+PM(Gc~98_NxB0Gty^|=0#e^y}5K)cA2zR6Zi93KflfVn0&+I(IIgYF7Y2r4*YnK
zA|ZIHB6O<>?^Dg*1*_urJzbG-S^mqjHxCaixPR!?l+$n6HW_yu=(p3q7nV0itN6*T
zrZY}iPj=aFy8Ld7`Gt=K2L&8Ia0T*JPkA`|tF3;9XURRCf~B=}^2}2;-NYtZH-x<8
zvN|6sakzb1pofg+1uqZHjmIX4ocLDu{M5H2%7@RF>YWYw^w_yHYg4t#w>aKwZ&$m9
z3P`Sb{f{&7jlm@$pAgHCbCSs!DMzn8T;j0kipExkRkyYn{RnhfwSEr&ES9w8{|)ps
zzhq?CJ^HJCdv|rq1}oR1{RNX-KX#wmdT4)2i`#Q8Vfi&1F7Dhe#%R%TZN)(@*$s!D
zn#8BeC^9T9nYCWtCA&Op!?#<nx5xhPjrg`ZDq{17RF~34ySe0b-t4$Psjgi1dF$&>
zOgTxha}twe?@w7ST71%MLwVkTRbgLOgNx3?*KX{Zwdk%;L0h}!LFa3`ELIzv^05D{
z`;&EINqQNNINyTbv2(@Ft=sRoH~;#5flSVIQ3)Hw()OH8IN5(L@|A1eM(NwZehW9N
zDylb6IH+}2_tQfk!5o{JBCll?j@vjpwC)aw<nDEPSS@xdXw3y%M}=@<{mb09>WwSI
z3bH#_b4cA_-Tkir#2uZSdlO$@iaI1}e*Ch_mbXpY8Df_0HOca9&N;}qdhgwf9}T_~
zxHGTac7<_bnEmdhELtu4ou4M^@6=oHDW-ahVq4IWh(y10;ib~w?Nm(+)5KKPB<cR{
znozPkV){o1lLr5)O-t6d++IB6jPmBn?N=8UX}sFvY|3KLVAr92{zK^UrX{SkLXR$(
zCw@yyyx3BFSaOzW$kvxrX6*|*Cc#j!ZRykvReuB<RgX7aTF3e1S+vDM-{~56(s{y)
z<gaEfi@Nqpa^jWq4n7wgwD}HC+v@!IUR*Rc!@3z_dgqlH!spMO+nddB{OgRppDx6g
z+`Kf`tLAx5W?uhmL(es>?KiudVx4rhatNw;es(WRuLuck+9h^kZBg}!6~Ujb`F)@B
zE;UO-*ZY=<$|?utp4k$UHDawdX9=!*_4gWRgX!UGv%;=N)(OOiY6>m&S?enDbWX3a
zt?r^rck*5=H`4tvxBc$?KY^M_I!bHh_e)*c_cJBq;tXLg&83+t8Xnqbug`tgs(Z*~
z`MnhF7aSYTZtVN~_}k=n6<rMnh2CrLcd}xhnYU!>{&(jq<sN;!HRY>IhrCH}3q#-X
z*+o)fzq~eZh~6x-;*@n?|8?Gu@}RdWSqt5Ip04eec&EMg@CEUE);6ZoSh_Fne_FUK
z$|dCZWtIbn@99cuuXik1X4#=V`7ZawmM<orw{K*Iw_H{1EPdJc(`<WpDPzi}TU=FB
z0?)jQ&yGqFdeW}ez|XM2e(t^-i{swCRbR_fpw?N`zn6(${pI#kbu(TInN@aNnZWtc
z`INiFK7mhi%VO0GE}fQRNX=TeV_~oB=d`p*It!(5u6Ww=*eSsLYzX6lnFTfiUjAzq
z_h&4ga$Ue+SIX=9)W1&>n4g&>KH4pzvvxbj0!be|DIJE^%}JH1xx7`qOaU{mO*AxZ
z_<F=Wn6tT+;eLxl`y%bc4GbxJZR57+JXm=9o!ah}l6i+GU9qk4dT~x+f0RQ`=6>l*
z4^QzKnd+Qf8#K$0xANWHPYfO21ydA*epYA-RE4howfhPr5^wC<Xlpeu?~s(n`-grK
zAv^MxZ^>Bl;#H*Xg07?TPtC3*-0W5=f6g!^R(pH<l+f1D9qSr&<+8M+i=Lj6PCuWS
zzxks)@AXI5uG!{?tk`1z!>lx>pMR0*6X)bt6ApGs^(pqI{hc0ARVom#`#FL|Df(5z
z@?s;sEkX;nb<PQXpkK<~@48Z~roM~0@OMUnnC`8OPgor9sRrvRv4!7}IU?F{C}!H%
zC2Q2xc9*YcQ`s1@?clYDjQyGmT5hjN?o>W9*Jw>$+3gLQRof)bm3hVPv^>>y&?wx|
zFP>?}#GfkvBo3^-WoiAh{?Ez!-}S!_|NpfA&(HsV{EyW(J8a$>aPRf$=$+?Ym?*BB
z)jIitymk#YL*J=~3#E2Fdba#;$8VOCwN_yh?kWF?+Az)4AmPT2qgC5H?|j>vxaP?{
zmb(W6vu;|>JzC0s!K+SXm7H7L7Wa&&XVsn^ELkD(SvTQ=MXu0GnGf|RC(Tjz==R^R
zEidy)bS%Tf-{Pm%O`WOKytOBPiC6J&R?qW6uMcY*MZAhU8#?j0(u$;@-8L3q#6^p;
zC8xC(>u$fH8xyWk-1sA9gPJSbkGpZZL&YAhx>pQtw&Z-AVY<&|dhy8{$673srTZ^M
z1~)LCUo?G^{L^XQxtmk3=rJrmp1FVbWO1I}X<~^M;X4i_Y_zaFr@J!e<~#YkeYyU6
z_xD-k<iA*+7x+C&&L#W!SBYh_p9kj3`oyMbW!p#yv!)4tVsQ)hdfyy6m#cH@frN>V
z6>lCct}*cAI3U;i@7LL9r@OCN?k-xmUHx}}uX6y8{ZiI4p&5dA0+W=yKA+q)Pe5T=
zptk-!qsj?p@22cMpS`Pgy6%>kLqQt6D;F=5^m@PgW}Ir(iO)4jyruUK#paqh*2OIF
zJ$(G<!Hb{&|4jdXz5dhU|G)n4{h9uM)&Kv0_V54s{{P#bM>(e*VE2w$-|@D^Z?Z}D
zA)P}mGZ``-ZD;HYT^5_I?_2g>VciYCrE$CtIdQ^OpM>Kjbf0*sycKLr{~N<s{`llO
z-Ytvv6up?BU0Nm?qAO{_;2T_YYjydjK9lLsMcy$mY~xtkeOB|(tyzmEMsTkC<l&uK
zz#1Z!{_5)GMZf;`{wS|oxc{H%=I@^@q-1h_Z@k9p)>U!hw-Lv(n)174=5JjOYC1Kk
zR84E+Dqoq(<}JoDEp*$SqwD^Z$v&Sc#ir}T+rVU(uq=P|+TJM|8^gA)`W_7Kr$=x8
z6Oyi>BRak1?mQ)}sLR1DNuf$w&usO7?|fhWwb1t1$vZz67sSdPZ{4~`U2M8DoA<iZ
zpA~Ttm(P6I>YV*L?)R&Ey|rrh!fOg5)$d(al)rY(HA=DnlE=xDC!^-Nt=%;51B=^U
zzTGXy=Q=BHbPE4|VMXU3FXMA!Ten;1o5laTJl+0(c=Z3n`)~b{w~PM&^x}*4l8d%C
z%zyBA#UJ6NCp+49aHX@=zUrRMuz;JnA-uWNNt?-KA8XQjxtxb;UksN&(eSZjyO87m
z-0zpy8=+gbWv%CK*>0_5;La@GQ>r`HdsWXAmY*Nb#rr=Ne^&9y{$J(Ko2&C(&e$A$
z{{PE<wPl<M?Nj|$7lbR$RespKP%%B3|Jm)VO~)nvoJn1~ox{Y^BV$vw%OZvpqXVBK
zuC9Dj_HHKY?JEA+Qh!A5{fb+7>U~Sk=W9H1Ve<To5}(-a)jQ)6)l#eyR(i_b{^x~@
zK7!jq^n<x7`GUolW!=kRX?U?}P50tyo%6P=tGZtJyz;rhyFX#qeC7V|bZ_HMtW3Z7
zQ)Y?v>J@BN>W@VpZer_ZnW~{VUAXAaJkFl=7Mq0**~Mji3|h5Rbb<EsovW{PZHl_u
zz6La})4tYpt=h2|J~sxTr8%}URr$@P_n+d++o*l~NNUX11qZD<uD$s+yV-s7;f;M!
zi?`^prB7-;Hr4trcQnua${S06ui$Xu+xGoQ)%T0vo<8~X_Up6TF3YE$Zc=bEk$acm
z$5$Gj`qpSEht!sD?_JMr-MsC=iSAuu&)5%#iSRDEP$0M8rg*vavmY<#_XXtG?5*4Q
zcl!Ul?>AE(dn`(O5}D-Ex4`&D$D%aV!##fjWo|Rd91&eO)ga=ZX5OJG5?#HY9_dc{
zz4vF`oFk7pB6ud>P21RDXYcNzV;kMNx#8>PQU-(FxrMj54V3Gne5VSPl%{v8iq`t3
zs-I}O5Uw~~P1EXhbxzpHzl<f*<TeDJ_?xh)I@PxG?CWw{?_0Zb+#ktKbZ^?|^ty{R
z`r?J1brLU+y*gv|WU2Y1tcWv5#S~K`W`3%C!nM<MThUspWbRE17EBP0+I~Cs`?rFc
z%SC~|c^p1fZ20#vw=SQ@!9yoZ!gR-hgu9t0r+&S9S8KI1nNxjX;*yr*Q648E>ula#
zy49ZY?TEJ8#Me%BECQLmH?J;;y>~8j?W=I`#&C9B>uu-!)USPG3_165#nSZUAJll<
zzh5iVb3X>!s{H=;{(ToDkL;PedtYrqP0gxuqtxpjUcTKLuiu_^n0-mHvow9`vu#Ba
zUf+HEdG+tRf4^PbdurO0jDi)b*4!<ue;oL<xadT})UDp!;TFdaZ;a$R@p$Jpjeb|g
z%?Tz_Rt1yxeLDR&@Xp_(%fsuh?Jb{df0v=5_o455%Z8K30#~%ozkJmEOqj<#kByO<
zn&;#U7>{3GlJh}tcS{cUC(UnHH}9!fEpKOf)3%KBzUV*YghIs%!;;@Bo0S9|G8$M*
z?=Ywx+hXL?^YGIWhdDok-!H!`&QSN9|KEd@<+|GSf4<&lV3->0p_irA)Kj3Y^M;pY
z<JlSMYa`sBPI&*t;B8g+#aC0d=gjk3aDk_PmiNXJQ7=S;9Qn_iz1+#-%;5Ou{G7n*
z&(mef*pBk9-nf4f%Mv-y9TRTN2`$x%R+ZARaxgXHmGp_VFxORM$hhmLeRpmlo5J4x
zV(ZVY@#;Ur+VJA1)c(yaVTo537(~Q7#H(8UJt;pm{rYyJ4NlDgcU0d#GB0U7xGVVI
z(dVaD?J%^;GIeLWm*o(im*-}BA>n0t-j`KjU*AIt%BU*e)!qA+*Gm|f|BhY270<Ed
z=RyBEl~>v(A=hHW9{!p=_vOC58moS!E^Cufbh*soa5y1$#?c0$IG@ug!t%$$?+1Q-
z`N2g($HT+#A>X!qDQDO72}>tB$)$P)?o>)V-xPk~2}}FqmWWA-tvS==zhACqxO6T&
z#kMrLc9-!@(OFp!x4q<jtN$Z5ao#?yJ1rMZy<T@ECr7I{O6&H69WtAHx85_lY5V)t
z&ExC6w;j7-R2_cc(V@TGIboOerFC9xTy?8p1NV%ZbD0~AgI}v%dn#L6TFLOhOa9;D
z|9}5K{CK@Sd3X7TC(-|BZ+-vo@7gch?;a6udHeS7h3${J&HHy}Oj9xbzv1u}?H@PH
zxn8!+yR$r&^>*u`*vb5JmDV2eWl82t-<II^>VEZuY}4coQX-;eT!$k(Z=LLN-=E<1
z@lljW0;}1(lnp^!({+VRHRrG~Bu{X65KPzA;9c_IOPe;slFM6Pm84r$*~re1tEn@t
zs;i#yo?(T2|KF*c&+VtJTBEx^UQv?g;r4Cq_y5&#IhaJeiuA2eIpN92{eIG=-`}^h
zoUm9WG;Nme?5fY*E|t?wHDaT!o$TUv`9xh^^$)TzZbNA@+vC&42cp%p*IwA6<-LBw
zL(3Sxt#3C8{nH3;IsP@`zFkIr40FXCJH43cQF<-8?n@b4`Yd#u^q%Yd|Jl3cJi{D5
ze>wZQJNvkTEsPjWEPnjYE%kKsTSp5^js*qlikvO1HDe<E8JxWiTg!$lzZ5+AjLbc~
z-=+!@-@Pxr{k_h-?ylOFu8{BVxBoXbsjOUh!OP2N#^J9&zF)Wyz9r$okK0=c_B>^p
z6|5iJ%WeMhpjX<t^JdYX+)m`{{+Ob)VX1{ydr0<1bFB?(AxF%=ePLK}<kX%&CnOn6
zT9wXqY(CU;=h?N7$A31yz54d}{;R+5|9x)#|L6ag2mgQMfA_cfep-ylOPRurQOgz|
zIvaiCYGUOieOrZQ)!lh|D(nmN&Wf5ZJ?&i5DiC-7X!6C6p-tS3Iuj}n$n2UVfA>n?
zufRnDGYS>tzE~7%an+V7PQN{Og<s&@ASE}p)g|dsi;LF2)nIbq6>G>ejND-Qe#)zd
zn{)ndKF*+L_x^58-Nm1mZNF#!<y|BDeeax)3Q@oJT-zL;y}>UtPg*GZomKng-61h+
z`PaKNyUr2$Wq0RBdZ*v%x31eCwnk+Lc9k03YdoWNjj=3k!K<7lUr#Mg41K>hj3GNz
ze#@HvmHBh(GF&IGOuTk`@}B3WwxX}4vTh5uJmHuMs`2V18Fv(zGfb1zS$S&O!w(Pq
z;!Yf%a_LR#iHkpfJ^EJlc`e7o)3+uavAQ9rVrBd8JJb8sYYe5XURyMErQc1aqFsrc
zqDQ`pJT2V%H8ejmlJDXH9ldrR{)TO~ulL>kwe^+#zpy{`b~1ks_xt~Qct8Jt`rE~s
z-``I2m{pM`y0v<8*XAFkd>dUQbRw=zwg_gZnC+&US$c6_W2`}y$%W$&uO`er&wF3Q
z_Vv-Asreh&y|+%AA<iOk+TXoxrP1aKvmFF<&-Cs+#SjxKcj4GwKlQn9tIE^kkItIw
zxbI`;dcKy7f3BQgZ`|T8KT%Z3y>ip1b>UA1eAfrasjcW<d$@gXv{3}h${4k-ScT6g
zl3RO{m8-93SWR7cJamT6$+ibu!mE#1cKqn{uQsi86r3X%-kBhu!Lw`9Z;=GW4~Js4
z%Y;o=8V6fEm(&UF4rc!0zRrkYZV;oyp9i}yi(kGi@AuyIuSt+WY~tF!*B>XJxVJO;
z=<Aecccg#pT%BH;@z^hC3qz+?^rc=|Z>cq{?Ybv6Ogp@UG2(TG30KBVv(BjJ{_m?m
zL)iDhYqjJz9J|K&t;itDs5W;_Cd1O5TfernM@9RyWO}wPYza!)u%t|e;mwL(AKuo3
zPR>c3PofewX#6QUr?qtY9*GP0BJ<<#O5fg*nVWG==u^_(iHW8>)2_2duQk-Xqw$qT
zER$!FLru*zIhE=2CrT;G9{OxPUA)9QqvqazYs2`xe@}(y@89$}|MuPO(Hv`xrdO^{
zeRgd9ek<05c-zIx85nMLaW#vkd$q}L{<5H8^5HF~dza5uOT6hA^Rd%5LjO|Mk|@Pp
zT=_rEj&9qc(XVO7-zjrYY{vZO;!8|rV;OGTv0{7B*S%x&<7Lw>ygK#mhrzi(E=||p
zrKdh~GIDaYcJJDJG;+#u+X%ZryWJbYcl;4Cu-NIiTIb!;XSrwoAN?gW`>(>|6N?H9
z7MN+j;n7QBY-O-Ia7~%PO77GhC8vqiMe@C^E)UgOXIDP#`#K}g(#SO;P0L56XH#3p
zs#x2*_jTgu{6ByD=gq~vN{kIQ{I1~-Kh>oE(fT+^?d<h*&sM8a|HX?w-LL(|emGso
zxHawF>EIT{yQ({0`DGlv6lLnX#w>7GQJzUf>#=QhzN@D)WQV?oOdD;uw&jy@!nu%h
zq313}n!J8nxaGK|wAMzI$^yoyyOH8t40?BbAMPp)Yd?E%&35$(-V!?NW)@v%EjjA^
z`EUC2vuXa)YCYF?{PX#2<oU?w`n8#AD#xc^Vo*EEqt-U<9Y=@oylqc?{_8m%J-zwY
z>$}^d<)SK!7uOak@=i4kHql?KePH$MldhHijK+RbXNu{a)43Sx5nsuCsYt^r?~qZ}
z!}sA=KbjOC?LDrYH)*Aqb^j!Z-ecVJxUXnEJsREiy-@k5n1-BC4&#{#EKCLqA73je
z`RMuQ<I|HI4n@iB6Xo1)`ReK}3wiTgqx0Sgd%;-71pW5`lcqS$x$?#Q@Vkc$v;3cL
zO5tm>nRaLLQg-9$m2F2BWv_m-bi)xhMS(?Ux4b&68r;k(DQ4Z^GC9K_>_qHCz5R72
zJ1P}B3bOPWT%tnXzv@}JbKkwHO|^S}n$3{XtY{3mIVt4YTq7NpeftDHRy!oUU3GLu
z`b>GAuMvBKC3GC47D`WZ*tq-Xd#(>WyVw{~8bm{1{1JJ$D(q_=_$&>^?3}wLC5F1c
z7=)HCYkkjRIL`;N*wp;n&9=H<d-MPQ>a7(M@{13!KF9uS!ojEs+x5j;!^{HCoCvPS
zsJlC_Ta#zrtVIjg2Zf5At9)?vfKIkfL)fxqt!eW+p2s#uDm_XJjgEiX6cEE5>vZkT
zj6{WXi{v$DB|NsiuTyw<;_F3Le}9yo{P*C9(u1fpW_G9eg`9VVx!Sk07l!Wqm0L7h
zEBQ>tvvn`k7`EJ-WUjrTXT8Q3(aO03_gieHojGF4eXrT!-rfFFnw~mk#p`~nYhBCP
z9(wM5yLY+Izs^gBLiu<8EuQ&zO?&_5e`yMu!k1RFdoNkKskil@=fNXI>JKyLXa;Vc
zc*rKidHz!24-r#N6;7GdVc_%9t-*A5^zSUA%6)Zy_m29hpE>5py#Dq#pC#cB<r!A$
zc#50|(O$BC$s#5Hq90p)^A}%OApD#?GurBY*p9Wo+VvC}&#;tuT^F7^X`j`)<K;o_
zom!`EP2(+5>zulx$w)`4x5)PCH-nbo({t?(hMwMgsN68ZzRpx-p0NL8oA3tTkiea5
zuJ){C<GHpf?CU*nEG52P6n}Bit>ZVDIC!O)WQvpL3#Qc?)|yIG3jS(k;y)WRTVXk?
z<g_gul}|#;9DZr3Y^=!mx8iU2#n;n6y34l9zW?y<?glRg{mCk4ulM?XvSBh>pkXW;
zsr8*PA;@B_Zo;XmLknvcnzu-PnqBf!@>%NjOR6z5_B?KqSo~XF`bMF8w&`Tym13>0
z^zFPDpB%j%Xti+F+>D0pEJ_S-R;PcdRC>vJAmuyTh4tla_800uUcUeD`u|_|^y@DE
zzWDmd&+db|OVifvs5Fh({_T>ed8(~pTF>+2E(v?)MqE(q(pncVN90q)-CJgT&$n$X
z*~G)8^5Iul_P68Cf|q_C?d<q__^(3EY=elX{HI5wgCb=X@jShE#34p3qRlOLn}A*?
zgZAlk_4!8Mm%q64Q|rr14*Lbi?#(?H5%%WC;WEXBC~@(%=l{JtUH^Ig;{T`Xzy9I>
z@B93I%$9SP-+KGmox5$Ap|iA&d+Jl^9eGM>5jt#wes>lW`00sqr8izl*0I{joOZVV
zom<7$H`Skxey+U}dO&B|s>l;w#joeNiR}Dbn7Q9O`eN#gqq~H|b@$(E`YPM_cuU8R
zTJvL*BRY2p``z6C^6i?e(E8ip9hy5fIk$g&y7S3;&(^xvGwc;6rE_h4%dP#k^j=Y8
zWXqL=*4JCUrY!t>d}<4$luJ1Gxpq@IH|g)+uPL>1AKH41SNmbrr57f_UK5g}Rz_Jf
zY^yZQHkCTN<CGcK<MTcX*SOnEdv`@w_l(3QmB*}~w_Ka+s&ahUjnxOO9v(e%XojKY
zG>hO%XU}{#y%abz>t(xt>z7vtg&caW&HH=Cw)%7E?r#^rzI(r~>T~GtXZKck#n;qb
zslRAnebN4O+`d2M=kNX7ot|%R|7yPkPmiV6+-k!ex^s=wVs?JqQgHloO47l5lC|0?
z*`3U76ZgIAH#24C-C82VYV)_m=<)4}`sB)|#sb}+ZtSZ%COdD{Od&s|nVs7sE*+U7
zpINAP(LCn-La9k}xSw;SPjv4o@!`KPA>vR%VdLthb8lQ)zC?O*gq2RmC)tLl`wxD2
zbKdTM>d){0Hq}&Az5l&FF6aN#^w;}kc+SrVc=@~e-mh#!r@&MBinAB;Hiuq&&ama1
z8E;&(Y52-(f^+XM&Udn?`dj3l%d+mp(L*^8tXEA~roKP#lINMK>V+q4dA4&VUY^S{
z?TXQ9)xC#UyW3qHHSd0j)~($jG?h8y_U2RfJEyWCs<30<IL>U1m3mjaOWloA`q#@t
z65ong5+k-`CWtATMsQqtJyS|(d+S{X2AQ93&a*X_UaqmVaX9{M%ihg)5%*iazj*X>
zvCi4Ii&uG-L|yx?nSXu?)27G!>^)yUJ8S4X{hJ3z%thy|)%h=lBG27#vT?fkdv3z(
zbCZ@EJMtfJjQP7oo^i#U8D|~YjdkZZ`F#~-DrYl_n8T=Y^-6PX^y0cJ@9*vIG`qa@
z%j9=|+wZP_-uy65yLscgrwJc&e!aYMYo+b1TZ>n2J+`BJvGVsvt%lEzCVWUT(8%5#
z%BN-i%x<Sl&z{IDb8_w^x!sFkcocd6-oDyxg(v?hyme!ZYQ3fW>db{L&wC`><y)>e
z_50N6oLSlueLA==W_y9Kq$|(cBYclmPnfR6*KI6n#<enh^ZI!|RgNzR)Hv`W>ixvs
zla?q~C472&o!?&9|G41V(AyW6hu4?YzUY1Pw#4Z*yHNQ1rn^aXVq6<8Wr&{*bzG*i
zoa3|)tJ11Vvv=-hI*_n~Jz@K|&-*{zU%;cmw=G}HVu|a*U8fJ-Y&D73H*P(q)Xg*P
zzs@|jhpk~5J>fTv&R&1-64j%>XR@pC@-~;{=XCp4GMNRgDk@#1^s@_mYyo6JW`pU*
zcSQzG=bYKPuLvei*b(X}pg1e%nZCv?6NU@cGndWVC3sqVo%Y`SKD^W2_sxCx(RHU}
zqI6Qu<JaBM41WE&wv1hy${c1rd#yit$);4PP?HReIjmjH8NG?;4<&5fB)8Q!^U5R3
z>W4l193nfNuQqvTv&f$|ZaZyy*=YK4_jTT<S`IC4-m&P5)|Cg{Lf$i^Uu`|=;bdR)
zs%v)E>($?%^$TQsmK1)zcw5~+PPQ)h$3@}ozYpA6Q^a?o(I%_RW&gV6*OslC5tY5h
z_!$So?R1}KiHH5R#VDnwL>_l$5MTRw@x_<7&;Or3UHbps_($#AU;jUy@_5C|)|VG1
zHlCT><0kaduj}9v37#{O|0Rvpo<x<c(~3Q(_k8uiUt0>6o{sm=_x`%R|N5`RThl$>
zwb$`3{Ml-e%-JcLE_T*{k-_TmW>4{Vo>kfbj-_|yZ@zT7VR7Ko+y33wpMLyWm9r;Z
zm!WczyiP6qn>e{To!6k<7`|H9C+Qs8QgHn1j-4wq7WABAh_Q`hUU234+)MvWmrPV^
zR(-OL=Z2ouhSiITij`Y<G^W_?Gl<y2rn|kpC9K6=sUluscKzDzZ_Nud_Z2k${q=Em
zVyd^C@R2ijC9YqxY;bX%x_a{I^J>N!sqgPhULAFHeE@iTI;1Se*lljc*1JNJ*cm2#
z{pM9HeMsR~H`4;P*XP6@x_=f9+G)LAy!EVE$JHB>Jnby{KF1cXRWQriaN&kYOZM#x
z{_D5*znUG)U@G@&g)8H=!vAW$E!7^%YajU>ypsC%R8QjhsCzQ!Urb3{_|Vnmh3$5B
zm)bdUCn~LC4p>il&u#krQk-1+D%Wl`6JNvRV^iXEZ31}$Z#r*TKhJYTaoir=Q^`|v
z1GKlrzJ7SS_t&a9eg1Ekw=<N<=dV}4ZSMar-tNziuWR4C#a9}fDtQ_+F}3+XL35jP
zaQ5k{wv9cK%LBV_PT}4Y8_^;adei={;rjXe_T0RCeZI`!FJ+%Me~o|j!@@`Gk;LKy
z%OrPgn!jJhB72SLH11<EzFh~8em>v1t4DJ33#Nkot8O^GopS!snW*n?(yQ(Nzdjsp
zU;86H-|p9!v(HXnFqoDz?Mr2){rAc_D>D~%Zd}$n`^GZ&$hn)$cl}9wnSO0sy3?)L
z@9t%{-j;rUxBK2Pm#f!TtQTDQuJ*e0{|*PQ#fm=<oj9uh%ewI1Jl)=?xkcg*Ysw8S
zC(JbHHaNfd6mRFcD693$e>2RfZc@9wVCT1n7k6gQVNM9Np6VF&meIp&%KyUkd+W+-
zxgS*h=4hx2<Nn;TY^^^VcO#p2);ck<8kuR{d<si<PF-IkwX$>W(<eL&KK7lQ5xVx(
z`=F2YALIXs|KESFE%fjI)UA)pK7N|6&%QjJL1dlVx1V2488oiyv8DTcH(SitaB0f+
z>*klUi}j<~4fdKT?)vp{^~1e6f8)0OdmMiF+~2idvhLiADt~)+-|lyRz9s&AR<rs3
z>u0&;i}TL7?%sS>bes8huIsOI!^D@XUCi9~aL2sfZ)^?Uo{1H#4f*}-RQ8e`jHh<=
z=Krh9-)>u1T=etVr@G~9Vz%9$Df9ihcJ#knmpRW|OS>l-+3j0ux8v^K-VHl<?yM}X
zy_=W6jd6=n=zWE(@0V7TK7Iu{k0CWWKkrdh;`=*yrep|QJ?kr}$DncQ@i(_wZu34{
z%*(sCvnGV^{c`IXoA8R*nAquT=E0NiMpsu`#?0R#eZ`!uj(HuUMvhx<>El<tqAPPA
z@7fV`v+Z7vyTsPeC0vW;?P`mK?}kr~=ZoSuH@lPSp4;nYf7(j=`|J39H9xv@zgBVo
z?LK$wtZ0Bu#inh$YwPrPzI~IoR6}pI$Ia-N{1w>;{ui5Hf84um_r}M(cNWOz?zmgQ
zl<@i4C%0a<3qRj3*s<|#@6LCZK9(3qz5l=TSNPn&**Sa4z7`~Ze){$6u@irP)!qMO
z7$<1@>dmpq6X#mZSha88rf<J~{rV-PT)lT^q{L>y+&O(Y)q4`zmrLX)6jbG&HBDH*
z{%vn@^X2QSY+2WdnyvoztA_ddeOoi@=Cy}suP}ZY-=80!|9bDYqYCHDuid}n^7GiQ
zv%WX)d4QO2AF!D+gx|?o@#5Ikz-Zs@ZOxgfb6>xHS#dab#f!6U?*n*mYh^9pEVB68
zSJfMLjh8d*ioW?|)vX=pwC?8J-n=y<_wrVG+rPap)>IeI{nP%le%4a!=<v6<KOTR(
zVopr&&o9@$K9%Uc(NwR(eRJZfyGlZPckN?H==Htw<k_!nYp?k3<}D80_3PoalHk{`
zv$vOB-c|ZM;O$4Z-@h)Fo8_%v_LToR8}p1`%hsAr*;RNW!2Hjm_&Kwd%P-uM^>KAc
zb?xSh_a2n<zAcp6divL|XH~akB#vb+(Yk&wqu;wUb@#<zAGCHk?XJI(>Hl~0tS-Y{
zd-r*koLVj)e|JfEGQ;cN-}Tq6d-9Y)#p>xHx9Z1YyA9nwU1gi_;>E7HxeG#8-OXDq
zef{2@JuBDWo;`c^_6^_7uJ7KP@0PoD=gi}|tXnp&3f<cuWf-ygSAeTybLro;ap^$~
z)9>CpU>tw`$_2~p*Pa^N4p>X9N!|K8Mf&h!sfNpsGX*bho3cn-jpx-0*HAJ2-2I<7
zNPpv-a%s;hwt@||*2}J&FF(9h;=rtVOp%MaHkFCY`tv7Jd(SERj%%lUQoSnKPDe6a
zcG|w@$dfs-PAu;NPTO2#n*T8QJ(C-k<1xPsM*aiUQ<zuyowP|^bu=W6q4MLlqVj6{
zpf*(=vFZrB`U|C@+Ya(A-a4JXdi%N^mJ<2)6@O2fU%BkQ|3&TNH`l*u|K9TJ;o*rd
zW#;DYt$JAZ@A$X%^*5@DuilH?U-P5zxBai!&u{Ht)PL%o9vo|VcZTDW%ZH|Hoh??D
zHD5Y<vehE5&g5FB4>JDdKaWj5eV&b(d4X8<$6m#TxSz!rXMEPZV6$jVuW4CSgo)-|
zhMtv-#=0@Z-I<9-sf-ejdkV$R^EOOgxjHLm+1~B#^Y`3)xi9kf*FUT4_2d4l-16D|
zJuj~=m*IiI+b(~dM((d!|MazPa~$Us{p4vV;3<|aw07^5?;Q$lMkS`ZOO9K~<*j6P
z*q+CC+x_zTdv#Cc@BH!mol*56^2#pDIiF|LwtwlmviQfj;D)3751VY5yE>-ZE@F|7
zR!l=7v;3NU{koS<D9c}vT&cExvsX#go%zgen@cr#MJ_3^1kAd<scgy|<{R1TH+y@q
zaes~xnO=E%x$(QZ`#u&KW^FKd-z-{`ut`$sEX#$AyE+UC(V*1f0AAP@5Ncg~as8HC
zeP>^<syp{Ye^&RTb<YF!L*f}1@Lv;Otlf1-a4y4*K<UD;^sI_RBdvm5+mD_WjgEgd
zt@A#2kR!(V{ISw<o7&>DOC~NVij1;g6ZEQX^4Bq*Dz>hGZ^N~!^?_!~XSMrYS~2x$
z^r9aN&3B#XkLXEXc;HU=##NUE(mHv6ooU@;<*K}K)B1TA7tUVi{V3q5_*@a@yh*0p
zR&7kX&JghMTzLAD{;kvhnY|S%>NHm0yT<DAa{1Vj$(z5qf2`7e&FC@ZZhy|L{r}E(
zUtXS`|4Q6mZf{*>O`T~>w>rPs*X#AgZ_WQNzP5dz^{Fo!)-{%;h12}Gr)<^ER#!JZ
zC%VXR!6T!CjS*2bn|@m71RR>W<K?|_Tf1ci!oTcf{3mBVJax;5JG)0aJH2PN*ow7{
z|1CZ}yQVMVcU<xj&*Ce&e||kY`7gh3Ud`LL57pHxV@kK1&-=sH@bRhood%De`)wC#
zt9{U^n9S8A_~TK6NKv6Ys{^a8eb}OuMlYe?z8|e;bza<NdwXV0Y5Jzq_1FKFW^LY+
z;`@HDchy#g3$Fh{uL(bwdbxcT!-}=5w@&WvT~nynVf^2XZEd7U`gTXVea0&+JbqlL
zpLM%iEqbbMruox3Uzyg_{8fk&7F3aUn_GSB`|nD{k8bZY&%QNJKDwj)?}2z5o$L(^
z9#=vmyj~_1T+?`G%5Z7}!-mAWQ{w90!B^2SPnA_<UvNP`*3XQi&g#58<JPd&*Lr98
zyZEQ|2ymU5bUfh6KI6OW1rPHwZu$K#otUVb(`#Y6;A4rO^kKzYDyI@_cZk)0`LXtL
z^0JyKGp%mOF)TQ>WLutN{_EA1M?D`tQO)RJU0^WXMN1}Bc(ZE3felusH}}r`XS-AW
z*Ph_N%0)kPY!$Xxg*b;S|Mf6f$t+txb!)1va4hG#Sz={t{$^&jF?Y>YzJ9f;`;O@L
zBi-9KRnN>46f2PSdj9qI+BM}<U!8i{U%l7ja=@=b)4Tt^q{sh#xj*OcX8X>6Z|C3I
z|1a^*?wZZ_?f(D$pYmgN{O!Nz%ICket(bR2bj#Lr5#N&x1WsB??UcCBktjSdDg95$
z-~Q|0^FM!l{q5N7{L=Wc_wi5WZWr7>=SR+JnJHThuT9;0y4Y0rg-WN#a|gNfRTm#H
zAJDkG*eBn{PN)3c{eRc$H+;N%zxMZw>uUSIUwolmef;F{(DL2l`6<uWevQ8uGw;vl
z!k?NkI}Iyp?tfi&czx%WR|ap}n&<n(KAF-mA%e?tYQz)nmz%YA)d?o+-kx_+uE)4-
zPcQr4*Z*GZvd_vdcUt_d{HKvZf}3;KmFNenj_o<H)#IT{SnG9(vj_4y7p#e0U(KBG
z<yWfRJ*_)$X0?0?jg0oIX5jGDkvumy_E^EvXWw?nwLi4XD0%nD=ohQ%;pT;G*$-U}
z>r;<@Dms5J&cfAV<GP}@<4dE&d(Yji`)CFlIr$A9Ihi08eZ5gKejanc3;#7w1tae)
zuQ+iTbR3A1ZebCBuG9{ht~=4SJo1(DYb;i$&OI7r$0=2C>>eZkrltcaCVMw6t(JN^
z|H$tC?fLV!oU1Z^pUEaDm&0$gRCz7mHuDN6t98e3P6|)d4sKW}r8DViG{dHvZVy)j
zv02L$rHfN~S}G=7bn0JmDse@V`WB@#LZ`JO*Dla53cq}a?~)<I+}EqxoX#B=;LX^3
zbh-Sp*qcI+bc>=Enm%=mxyUEyusmdC5FeY@@>LyM{X*txJehb`Pyf;We;;4y|NC=v
z{=eeypFw3v<lA*G*Z*Jk%l-fDhxv79YkqG(_128R>)cMmEo{AKUq8-NTG)R4vWosa
zHLa=-zb5DJ{WbUZy6<OK|39#|mU)4}be8lSQ>oMU<WJp7Jtg(nC^b8qVM*XT4bxL!
zo?iYPe*e$fhxv7Hzsdi*xBULkdq1T2hn8;t&9^;t|L4=2&*`7%TU==L=Dl70*6)Ar
z*RBq&&wd;D>s8(N&A-3LzW?Xim!{<C#^L;$*CfErW6h1*zs#p)o;q*)LHKzJv(&D<
zofeuWqouaSzuqelTid9Axamum{t;g7*C!8DL`+-Z8S;c7>+Q3e4S&L&8@m14{@Q-$
zak#hT#A%<6)qD?j)ECF}d-^`qdw#yW^@+#CgI>$}?;VXjX=`MoarF1nz#V_IuQ8>I
zg@o<&Ss4FqJHwT8Ye1I_9?$xFmOt|9s=BF{T5XvgY*_oD@~MmggVNHHYtb@~ZeQ@v
ze0)ycnN2%~Z{NLl9LHwmJpOf6ub}W@UfC5khAbVUi01*bZtJ`dVu@^dUX!`hgekp}
z;a%pI&+OKd5A*TWZTo$Z?N#aRd8_89@;^3LHQOh-divkEe_Kr2JhumCWj%S_T5Ks|
zFkdqKg~eh;v4kSQ<bysM)#pFyDy@}Uv(<cW&ld5EL9b7gzWiJn!pCpBC`-{&pu5as
z_EPq}pCmG#9*sWzRzHuyXN|qFP;98__T$H9ZaelWhIs+An0dpM(}9*};%y&)6Iqve
z<E8F5F3xRzMsvmUKR?nn38=pQX_edhJu)d)x;jOvKJzq!Ep2pG2l2|gzxurR^|W*I
zZRPW7iZ~qJmshv*pAxV9eeo&l1^JxZYwl6GmoF;xMy>FTY!2gNFHAjSasGP2HNVAT
zdJpb&2JPNz&GyODP$9zPnMUZ@BT0=+R_6>t`Z*sj>$eMQUfX+5RZ8*VgIi4BK2Ltw
z*Hu3E(sZ>zkEy4MqpwXi$zyz-HRr+l4=n!^w|>3)@$+Z)td1wIIINTlJKix^wd}Y3
zwR=~d{^pYI4&$c@mo&-^_|l?&Y}2vVf7-`>;nA~QF$;e(6^3TKzP~L!-XZ>OX0ms|
zd^g=E)y&0dy{u6ucyyFjXXU162QF0by#MZQT>U?$3Zsqny(j*6E}y8EecCONQ^qCx
z__MN^wX1SwKC_<c9A)w3_0pZHXSB}dN+~S8{Vr|w`>Cx@zfGunw~0;fgPh_zF&X#w
zIU-&c*}Z)dpR9elbJd+QdZy>PJZv+cM6QmyS`Areqjh>ldh5R<!aRbrx;>Z{WIpUV
zD0xwI{^!Txhu_!!zFz-fvV3hL+v$0+%Od%=WHNNDoAYjJ+JvtGJKp(pUbSh>-m&rg
zwbENI=f$R;TYbL1@{Y_Y_1in@ie}nYh?UD7?J%3a?S4(vyO)KLDhJ-i``%rjdtYL=
zc>2{X3=NN~Y6R{wbiGwyv^s2=T5EQy7{i3g#u~3B*J+(mmF(LzFXcmvSn~UfuW=SK
z%VI;EAFW!nh2h2Fk`+<zPYv|%rB!v`V-YjIbn&wq>$!yuFV^)GR<84!zO?n~&bWa1
zpTz~GRVy0W{XN?%t=>Mo>?UBg`?A^1^K&<udc9mS%irX}2b0R?jT4@qVm$Wdn@EEE
zT-m*Q^hEXU^p~&6E8X||_VJU)88*G!Ao~B9XW^!$c1pEdL-`*0RK{^7GaT!A>|<)R
zttRU@AIs|n?^c!EymHHec}iZY#Fkru)%CUcxp(gBg_Q5QdaRbGt|}(He(QFzw{v4l
z=Xy>#)w(&*>bl@8J}D`kLw}n;ZT8tZ;g0^P<fglfUlzW<{cHb8XHy*+i9?SYHq8^L
z)q0Y$VG_pz-Uzda&)zJQQu;ajt?0DZ+&b&UmV|%O-X}ZpDdPpH%Sk~J;TG(?Go&h8
z?BsKwc0N7#_uG>Pf8Lzmf9<uoUYukAgR7LLg7WF;w63h}%^AJC<t(d;?G$p>EdN>c
zG=S&a23`}MY5zm_OS^HkX)``vbxx15#yITsx_9vb;%c{JucSsMm(2UF74FX!^JVqi
zx$0Yu4@}<wH!FPWD*bh#-Jt6rbQ)g1XnJ)~%Xv}AoU7c|CtkfIILSaGWy2vi#h2Q9
zw?CbG=dob}clqQM5)QN0x8LeBFv)dPp05#5vHQ-s2yZ2Z{Q5#}hx$!+Ywj^D=>1(-
zuY7)S=EJ{re2ca|{&&nN>co<QBN2Zl_VqDztnS!Y;V-o{WyJ*Lv<aSvm;F^2&Qki4
zXx%Sax$vfFw^#t*vQ@SJO6Ei+$E@nIxExwMQ9ts^zbSL>q`xW5C_ClyWN}f%4$eux
zkJQ)f+%#{C$)y#pkBb;QTWsvVyjN9!bVj(fvD)d5Wr4rkRCoTzVRAQ5_^tcDy#DpS
zqxFyL{vCgMarX83;-Tfg-TgPq%hbI6^YL)JTEF-7>E{a1E-hYsW$&8Tr>;)@Y|z(x
zZdKbxy=(pL^0t+q=i0ycfB65`=l?(0|NL_O|K#rce}}Wm-+ukM_U~)?UGx7Je!2fQ
z{Nnuo56^wB|EXL2`O6#MMIQoXmQI`{wk>>}s$fy<hQ(5S*E9DXUFjy(!WFFQ_UGSm
zwfIS|46`0Aaph3xE%KUmYL?Bk$!-gML>6UoJr39Pk1<_&m3z$$7RJKEMQ_&@S~|_p
ze7d;Er&eICYm{SVUh$mFb&9*<C5?S;Yh5k(h$k$aXw`9z#b)YejgOJPHWi!c#@vw6
zIh&jD-j8=$O``bT2f3OyVGm6IzYjlM+`7ajaxD-0$E#k=-~(3KvP0z;nI!1_5@l$A
z8zcJsz?Ke`RLhzdheQ;oHF2amMzyeCyw!jGentvI*%h~^%VtVV+I{NE1Ri_-UpA*s
zOXY2xt#SXHHgm=F7hB%&$Nwu3*&bK&{`&WQ-<~|FpR;a%&Cc-e3>S|5y*@AR%C(c6
z%g(%U+{h-Pv|w6R>RwO74ATqUHq%a-C8!-rZgg%vz_4q2B%j14Rd=n`og!a$X*`nF
zy6Gx3Px-+*sb*g%?RoZ}Y^JaER=O`E5*hj;yDEfl^Q$i>N^P@6@5oI(?9AHvN|;Tr
zH)_fBWv=fMR(zZ#HchHH`O{rzYuyQ=r(P+GtdKdTR=>o~eV+UJkl0v;e7BX>yKjZR
zl7EwWyZI5T+}7wqGrdRK8BBaigpSUC%6@si?D5<0R$5B;*!^R<QOo`1?z?S=q7!6q
zyxGQmb#9#O?atXUwfnY<J^b<}(`2p~8^fw@pOuMLzRIE3#Jo0!TCF%<xvatFT-u9M
z@8>+J)88QUG>$7$dsD5^-0RJ*iyM<SdS{$b=v25cZSNtY*&Q7H?|wQ>I{BRGLB+0}
z6K06~llT;BeDtdI4sA&t-zf{@dX-z|9c0Qdsk*cO<eJa*244Pa-~KDhxpn(%Ceu~{
z^#F!QDbduor`k3pEe?=Vd@?y`*O@gL8>CFami(;ZoD+M0(m}15@Wh7QI?nHXN~XoP
z=DO6~{D1G#W9JHU-IzIwy&}DCmd`6}?*54i>-q2`Z#!u1O86?!CTIS@vKI^v3-@1r
zePVCZx=r`y<>gEfo*Q&euej-b(uO5_e4XFgN}o*7wB`_fClewftaESYWO0}9sm@ZN
zf8HMawqVoJs<7|>oMLTv@3WbE!~Z(NgC)#A+EuM|U#nayx*Kc4E$7ynb7zS}z$?Bd
zjjMkfzFa-wxYOgi4aW={>q3P1e6Mv)5oJ=fe)niy2=lU@!oswhGAsh=R-G44Rk?(3
zX4rSY{hz_xNh=?%_c-%mn)HINVf<mQ*QU;E+vd-3NkhGial-Sgs0@a!WtmH~=W=Hn
zN<85?xo*x(u^ZLZpPt13?dSjhWd2h9_0=DbrGEd{{X5>;&#vyb=l^fo(jVXF^4Uus
z-M_u^b9VEq@9S5;`&Rhr;|uNT!|Y&H7rSgDzv};+{k#5``u?x_|6abgso(wg<;lmd
z4n;A)x1Qi`rOtRbT4|oiDb?#7OlxP^I@e7!2%j)b=eNIBL6!j9>xx+u?#&kC^D1tc
ztRHoCU-fBDFOxY-C!`4IZ+TUs_$~Uv)uxyB&jdqfSZd^ZO)~5)UCGp7ksZ43>YMpW
zL3`T{o?<)@X`#*BaPcpf_t9qwQksS7AA-JDZf0XMGx$E)Ud|;ubKkj%;tbVqzowmT
z|8uTxqw((h4;NR@a1W38wP`V9CHLx2Pcq)BU1Kn~#&6-jR{FMO?6Xy2U&|p&tu^j)
z$}CiQuKi?P&V<!oLa(j)cPm~}+7h*3(>}qY6Lb!pXkWTzi+|NMUjB*CUdvpamL0#&
z_zuId6|y<^3p_qtuezPNzxsFk(I@r0cK)kb|MlnbQ|p#aob>Rs375Zd<m9{0CWhVk
z#2|1Xs`B%f*v7C!$v;yby^(Y5-#VQoG*My8pB)JcuI}mXEC@d)6PY}lXYw71?<<y<
ztoyv@I+w}+d%d1pk0g5AC}@0hDw;1g>yE@sjg;i>1AWbPnOD3Pi`|rJEH7MEel_pq
zCY$ied)%g3d*?aMTmO{pSiuPe?JmEQYhFCBKFYs({vYW-AN<Y#zp^NHW;odx?JajS
zd;i{PHoI>9{C(X1|Jj3EL+96j`)vPb^NaKUFZ}(}zqtPQa_jYd^XBcXjEOB-5#D@p
zSI1_XK+6gI!M<+3+kBtTTC`$WxB2qjPLF-AHK%S!mgrGFT_<k(^K8q!JH|=xMANkb
z>sls%TfVnw?W&Ce(JI$cAFk$DSZeU;+w8<TRk4oM3tB>itD`gdHYLVd9f-`HxWe<{
ziG<Xx(k*Jc7#5Uxul^kM?#)rh-~DcjV;+3(h-@+Go!lO}ZAtg~Gko$n`5SsA_`X!m
zW-DI(roqBz-?sOU)_rOIxXI~tALFrShNrJ-Rj%{cvVGH{qMN&qS%$+7u(I<I6?=TT
zxcx2fpU|f=Q+aJxRW-LsO0#S{#C5y%G|#nixw85&e@C|LbhSF8v{R?!6Hj;*OB+RS
ztjkW<bt>*YwnzQq%k%7in={@ntZoXvE$eSBZ#(PN?=6MTg)5aNeh*K3<>$(A_Ky0|
zhA&g^*q_pymKxC#X&d1fmEqUBMj)g!ICZn_WP#X;Qw@)qMoB)|x#`%YcauCa-Z5v1
z#C~!T_}=)cgu(ZHFxQ8dlS)^#m2`#QJmwg4^4OD~QkK#TT$NH!lWo<uIWIUh>AKgN
zl_hmC{leLLS+#fUn+^5Zo4cRSzrO#^&nL_4C))m>f9?O%di$z>n_qoh&d~7q^Z7*w
zKibb;^?dE~|25B_#@A&R{Q9xkyx+e1^Nv|{B70|u8SCkuD>6#ovT)Mg#<r8Ivswy*
zyLPSi*~)h{VA0&efsQu=zep`rytz<HX{WzfS9RM3VTRIo0dnUhU!S{c*^svGg6<JN
zW`X@TCF`;#gsoUpCB922K-HcLH2a^qWp?&dt@kCNI~R&4FkVSe>{kihoMq$1c0EQy
zKs;#1)N{u;MfGi72$je5tBN;jeSQ4TS1L1cL*H-n%!dZ`D}zJo%r3Z3d38^>Z2tmX
z4h1c?(<0aJXDoJ|Vy`Rnd0uCg^-{Ob-}!HBEPY?~3$(l&d|t)(_~h)46(YaV48#MP
ze<o{ZM>cisme5(*oRRyZo2lV|&eAPAxG$N_nsxIjcg(j>Hy&^-Ke%6V_2a5JE5c8$
zlbs)bGOP0d-!}HMo~`R=EN$gh-Mc1i*`qUlT-Q%k<p^@EnJKZ?z;DGS-i<jkp2<5t
znknscTGZSrHSp+B9ZQoVp(}Pjy4mD<PVHL%(dmv1cVf6bOYE<{TqxCc>?6aM!^^*2
zH7*KgFuE$_bIeE1vVQZW6;q#QGj0B%f39TCOPlaw1MTySHeK&!uRosmYmUD@TYB%u
z8E^MkNXgV%{rWt=XP)fL^Z!y}>Q<@!|9kei{mmcd_3Qrq+7x`tFOyMftF-W}6eoXf
z{<RkE@w<A~h<2_HyXLLr!+m|$^7R)CT1CDV&u_Wt)>!BkzU1ALpDT_lDQkRN-ee)1
z+FASgSyz*irQzS{af;hhZe}?ugj>i}sP7Vz^SiYE<fKozes5-OT(?Gdj_;M0preO&
z`m9uYV>w%Fa@ewEM_CxQoO=CtOI_&vb9%Z#TRD;rard+S^K_ckdXOpLr47$2rVXpj
zekQoZYFa;FRD2_$7c%$jyC<u~-bF6hp~Zf7Ls<qxAzQL>osVsV^<0LC%iZy>K$|VV
z2c5BNZex|*rhd&!-uD+%=d5f&y(5Bc1zD9{0tX6h->LBjyuaO@w9A~OYSS}|)l)C^
zJUSS=Gp*BUQJUGiy6Tcs-Uah?tF}L9D5yWbB>a!=?GA=n--8nNb=+2Koo%9?FXqf?
zXusrHm-Hho$0*wfJ%NywIghU<%;o6Wp0v#M<eqr{0~1<<0~Mkdzd7XgEunK-#GGd3
z9*d&b_1p5~Jj2UXzK3oroKd!RuS)vkE@`t9TkbI}*u*Q8uk=OGi1W+O9)Z`#8JuQJ
z)exEadc}I#>&f?+Jh;Po&UGD>tbbE`OZIQ-$7lb8_x+sw>J5M4@53wg^5l+9-}G5|
z^(^!3<*j|2ZF)oIX|Fh*x#Yy<CsC?`TFaw-g*Q*#czbGlRlSMr<qe+db0aHxICWVV
zJoe}^bhy`_@D4S-q|tBuR#ug(yDe{yNVBKmguaCvrmou&c_2vX{2`mEs<Zd@tosqE
za*Cngoy?;P4_|8MKc1%<Y9YQ-?tA*2twOD*wk~6>&^4^F?EKJW<es3nAfPLdJwwgg
z>FUnw+t}mHpXV&Pk@BT{p7np-(^m}6UQf*luxblB{&j{@<Nnl62JdUhYbVR*^>|NN
zJI6D`q#=#x;*DKb6&|j-7rvI^+A2Gbn<tMxO;~av|L3}9x5KeVBXzg0VP(s<G!IzD
z@iTeF!LIYWI<=-;^83zped+CMbK5R5oMXQdcK@Nk!PU9`!E@*3YM01wof6wTeNxLq
z>Bwa@uh!mCm0H6VJJUod>x^d%XYJ9?RVNY@1NLn`7$PCR#YcwCAyxX(DpTDU38Nbp
zMh)x28N8-%)D@MA-{_IFQCy5`?TLdYjwl7X@=sG(qtWkM#(y=Ff7+R82ly7=^f<I|
z;iervxnZkBR<5zQyzzha3E|eQcT)D+PKwwP@M_;%h6lP0_m<y1*S-4Je1B{Gb#n8+
zGaR^LVEQ#W<P`JIEf*@+rPXSO9+_EpCAhL~hMQ$WjQfoHqD{f9$)b5qe^fSJc8b*1
znD3BtN^X&q*OrMX7HtYM;udd8{8u|Q+bBxo&mx|7ri9|K5AVba!oS=Xicx*hRqr>^
z&SP=d(+ir>bL3k#UryR|AmM7>gQ*eoVz(8x1)g7S_-+NS+VhsH9uGE~1Qq=(+)x{Q
zE$^V3$+?io=$~F@6TGKxeXGXL@MhK7`F{c<^Y|+>RnEue{j^=RYVusOdx44fK0mr<
zwmMfrXWH5<l~d;HofrQJUHfV;c&qQ;>GFyxzFZ6|V&r+7uWBw1j1uPD_%3C`5=)I)
z<wg;|V{2pjqibbk?yxuPJ$uS`?&ec`H^gef6ZjfR=NzBuC31+bRiyvWOewbt)&=w4
zS-(x@VrJuZxX82pI=5Yf$w9Bhu?owV%{Vf(yomk6i<L{mI~+dqJxF{n=J>kHf~Rpq
zTGS4U%~=<v4;eMDUlTj$<?@TW&a654)#|!&aa#VQvI{#V9Ij6D+rxLVl(mIXVddeK
zR)0UNT)@bv(GuzOkmHQddYx4h<-Ue2wmN@OEv)KWT-*0Fb<0C*@@Lq*E#IG5^pSD;
zarbN99glbz{+v*eUhEZ|A)2lg64M&BCG+3JH=*`BBjq;#b+2K5f4i?hs-a(8%X`l$
zrx{1Z7k!g=2wfb$E^x2QqQVJ%OP6{JWh+j*xUWj=nQ_`n#{9$mF-?bmmN&88Nc}1H
z)9+-d-{QN0ReH9k(poNDVqaarS1A`M%JwXTPt7WGYoD^sY#qOcnP*ZW{#Cn7tX@&z
zetGk3F+CT*54^v2>Yv-bfBv$+d4?9JtEy|JvDmcpuih+oc;4Z!?W$@^E8Ii%W{XK}
zXz*RU=jxlua&JDC6esc6bhZ{GhOYi~6}*o1o;UN37va%+`wo|BYd5S3cAH@s@o&Mi
z*t^@?7jBDvAo9!*)EqjQFqh}8;})}<$DVfYyLYmnZ0d>`JYv@uOgVRDg7));)3Uuw
zW_|s(gK_HVY5K*k=l`v^m9w>v@k)h87~k2&Mn4{wDfBli+NPK1C-#Pa+Si>15jGDQ
zcLdkAHte1xuefT#GGmQ9qBYS5{JQH;-IAPKd1X<FP~+rm&l2`#RWId7icB1v6f)8-
zaBO=fV|tlcYq#g;8{y}_9A3_nbwnd^%GOh6PG;N=^VV<Obnp1y($+miR_FQ4<(|*~
z^35W7hK5Yedq#_nYkZ5gzPsI5XTY{VzQ;4W^vd-=>>_JJt(LpVIjoI+n06<0-t)@3
z8Cq9QRhgtOe=jZ-b2N4GUFFUbeQ_CGjS2-9oHe}GT+&;0*`PzfceUS<_j@K6FW~rd
zM{<U<i|VzxF5!QEh<p-YZ-319CuYk%E3Nij8{fMdOKbFHd=fuYe#(BK&i$aQ?FQ3R
zYvZmxpL}R0n_fkyXh8IH(+G`2Tjp#k3y)uEHt)}|v+H>oa-JNX_~l6KMcwXAWd@~l
zG7oyb+0ZV>o0z{&Vh(4sXkXif6VroRYWC#Lu>V=rcK`AUi5Gik9zA*glHux2s=OZ<
zO1!j%_HHT@ejT-w6?7=X`lPj4=`L+sFD0*i^*#c$_Ue1wG&$bI2Uo9+YriQaE&pA}
zG@IM6+oNKcK@|^UZR^qrGMew7tSfu#Drgk(C-=p3xv0kll~EQB-D%oqucx2ww$jX=
zF1D^nqR#Gam@)f;$93y1r;B_RzaP2e>K4o2CT8y}CD%;Y7gZ1;aLTKV>#%(K^AiV-
zdt_Zw*)(ypr?P8{c)CZZ<97dT9EXj!=QwC+33-Y!&x<W|{?%n8)*5+9**e5o>g?<{
z^J<?O8ZMlAV(N+Wol)|<#~%IeGX7eaFY7SxJKN1`>u$~My#HDK#AlDMU(9zge}AG7
zCA~wPLGI{xw|C-umS6gL<H^)>n-3KyRIckOoU(Dbm#!IOEMLN``XhHw2zsnKzGLM=
zS+{9&A<^-<e`4E>FSi`Cl|LAuctmB*#{JLd{(2Z(S!yC1<d*b0=5p)R8H|-{Vz*}Z
zPGw8VxG%7cJF{2u+ux)euHk=oe!p$PzBGJ>o32yDhDzJ{Wj^udR;C};@0z;u)WaA5
zjy~A1V$uBjZ{FRvv#YGG51tpat7JwR!vy!shZtQWtq=O7Y?vnfh5felAy?07d<@$-
zuD6HV|6Dw&@oXXkbGD*Z`u|5(f37a-`Mb{XkjwJd7SG>&(6~B>PqC9VA<3kH_s`Xj
zCB;!G(v|tHF>7Ddf{#J@E52KP?!Ch|_ON|eG2MSnxAD{!7E|IpL$`@6Z%bv<j$znv
z|7!oLQ|tDhy!+Fr`Dihx?&H+IhZ1g938iywQ2aB=xn3$XbIB<mPv(usg5|#RZ_7V&
z*fw}+z93V|h9|8q59etpztT4O;MDS#qeDAsWq9*ffy|TZmu!E*ez1#?;T!wfjaR;O
zm>%B3aP*4R^%VD6Yd<yg$f`!o{`BOmic9z%j>w=LoPpCNPIZ4de#<*aKZ#H4zAJ-^
zldDK_#*2*s9(HH#FIBz%U>#q7<+NSbHT7MypSO$b7tU0*R9ao8B)wA3cQb=b&il76
z)54av-!==ZTw6Nt`=^+<tM01%&s!3x7ZW>g{Zu{w#nVo{yXdW?e=lv7>+DXSt$X%5
z-8vhx`qnS4`yAihzCT(1=8Gp=*|MnPbDcw$FWh6>62|>XgO5w6oHKRWhLV}f?Hh~B
z<<jqUPBG|avc4Is=*O!V{=!gO_3={1Ddy+qGP|i6RIJyy^K-fO?Z@*kZ{FU&p{!tm
zz2xz^&u9DJUi(+EA@J1$t_3{ooW@HH`)6J{<k{g=@Y1L{xcrs!{=a`scyjo5w<Lyc
zi@A`X`=g_7<4)(Nd(0Hh-Qc;}ur}uIskglQEce;;Z%)d&8+BoG+*Rmq$Ex$D+w>Y)
z4W#z0ikZ7&MUlYf<C%iK+{fcOE6kM(4(z%-z43cMbE(l5{mCwyI8G)M=6Yq$^Uks5
zJgd-h@KRv6x`AA}<x`c-PmllQH8KAG>d)D)r^PR}h2NM`s&piDiD`h#wAbgEs>>5M
z{L$3yjcT<_yY6<bKEkNjU3qWP$@NYJ$JB%TROYlBu3IC!H2mzltqgH|GECMCQHfK(
z>MUe+h})yOh3owm(58Z44}*guGc@!p)4P}d`Dn6n{k%?N^>7Wb0Ol7ZJAPiSTk<t5
z<D7{z+sVIkk4IPToAx|=|HkjHE9+Ji|Mkq*zf@T^<9gKdy>h-cvez0f%~~se{PvfW
z^Tl61^Yg>sKKuF5rTlZv+V4ANKhKWb_w~i&^(%88NB4g};kS|bM5IUfot&lnYoF|L
z@xIg5{KM-=`ju&&$EThVQky>i;`t=DwO7tOi&OjkgF%w-!7t~_Tt!_gwc1vmo1)rp
zs{3ZA^>qKP0E<7t2W{qXHSaxI@A!_B*VWQ8VyDar_15fEMy^becZaX|3%1pF{e9v1
z<8f}U(J}p~?VVi<j1*c=UFxVWI%>dLyLQjTjD2xwU11SZbuG_Gr)%YIh!u#5&{*1j
zb@n&+d4H>>FVKqlk+#N(H_T#PNBQ+PiCou&K1Kc4Zg&wi|Dk4YR6X_Tx#%-%D%OPl
zy_?OtA?&N%=}$LQrakZM(h4{CdpGME&*Ck9A%U}QA1Za;UwYA-b5|6HD$jC`zYBjZ
zY2K{OqG+?G)vb9${JF?o56;8}xr_c;@xV7B*YriH`EBdhM{j-4-xxjXw$D_CPd2O8
zGH+7RXDFD%yFq4umc*Jpl8P)7s-J&xxT?4HK(g+XjBROYQM}?ugNx>A$GV2>xPEQQ
z*2kIFU#GvUI=e3L)w#HZhRJPP3K-A#i=TVC=GSxej00lN-fF+EHDfjSRr0wmlu=^;
zhYguYf!8Mg4n5lX<ml~9)ujUAUpKO^Y1->HKW~wh>|fW!x2aZlSOUt{f2vtK{r$uX
zw?cQ%|MKn9@yI1BF9*e}F#hhDuYaDub2Z0WK7P~Rweq&V{$4oV-NwzZ?b3ZUxl8=>
z<90oEHofCjf190g=eucE6Zjr%-?YA%+qF>ioRHV<0<|+i&g;FE)q-{ZGnh}*n-`L?
zVxrrs0;z@{7rgcfRPOM*IqAG=%!GOlgD{4*5~o}~&R;p_$DLVW%hX(rZ$3}s3EO^z
zX&2Lm*!_-QUVJKHO0cqv?Kk+7^EZ2PDMNv))XGhI4_x0xZeJoKSZ`X&;8pDX#&voi
zL#NiV#^gEQA9si>F4}dn`StaAf1?&me!nH-`Raf+)!LI!4d#5PtJJ$18>JnWw9)&-
zo$ojHJez;Eq~st!i#Fr6RrlOM-K2ZTnGBQF&Yw=)UG()x)GxK-gyWXGA1n#T<FBj{
z*&8C^r6J+HjN{^ugxLLZ$@Np78#c{Lm~+*}ZFl8P&TfUAs2Mjs>i5;%xxc{T`;PsJ
z_iQ@566Z7TxL32g@3Tp~ehk-v8C5fs&M!W+)WG!9o^5;=Lbh`>yUwki;Uv+=rN_NN
ztRcK2eMNj8Yo`fAPhsh^MM~@e7SA{LCUR^(Vs*!SdDi7k&J3?uop<;~N++}F|4N^n
z<Z!}t<BW(YcUO0^GW^q+v1X0>;;-GEM>GB|Dtg?>n%^rKo@b*Q^YqwM>GD%z@8?JG
zsD;Puhz~v1!`O38UvFMKlR=u!{-^t0)>_W!eAoBYBDA`)&f;3KW5^!gD^mZT{*?MC
z+2CxtN9TXd7jagpDo>wR`ls#HvssJ$PS)~ozH)G(RNSuQ#0`3PKl;Q!wK>M*pmqAU
zk;?b<nU;Bzma?lH{-G?jQmk0|Y4ReW)vFx$2QyT@doJHA$*|#=*W!Yx9f4K%G@}_R
zv_F_TupQWw*6sH8%Qo2=#tQtKS)B5>9rJf#o)z43N$JTM$5KYyO0Mswzi&Rj`uX0~
z&9A>cyxRMcNh6j^q1Wigx_HGOIcaAfY%&RIJb6fVUu5drIh#`^wjOj!@Yxlh*v#6X
za;?yoPq=hn0|V&FmZ{At3Y)w$dXit)GuN{^9N#l(<LTQQR)zlM0xj38>Q6m>ZEFc%
zLQUm7t^O^0SdQ#+anzJe-@3JLa~;E_I&sISb4JZaQ)ep8$=t9ko1tgssnf?Jbsu+~
zuKW1m!?~l8aXWj&cjO4NJ&MuaGE1ya>|y)&d6N!4k}_Hv6ngC%BkOXG!(X|ZuTF?i
z>y3)6Q+V;pk5TO2*#yNG|E7F(dB*<KCbws`@nlWuNY-Sw<~#QfZP}Fc#ba;I-gTQE
zrDXM%o_`gc+;{HoPD6Kn_H{+uPIXORdtKer@keQ@&(mX9C+;*lHsj+|!}ZJ4*H3<~
z|62Lpw0CPt(!-A>n=(w5ZomB|VV%wDa<%<gnG7CRqzogU9?QRe?@ZjeUdDo%7RsU3
zORc@H%yRNu?Q<}6lNm$m_pC23pG|!IY1`+MR|73&YImJ%NSWlruy1N@?U$Eo43D&a
zzmlBvdwu-B(=-3F7+hz`sp?<}jjU3T(otO9a(U4&uQbk;-zT|gb#HF*c`2A1<D69a
zxNYOSx5qpmy1iTy%;)y9#Xr8|8k^7ZoI<H}!IQ3h*}-M^`L;ly@1Z-+>8qc)x;9@u
zBFJ>C!28?koF7G(EPtl_-F*DJ{q9@#xBmOp_v++Nwzv2_n}0di4OO2toNG56I?nNY
ziknuiO?S=Tsx{@C=Kl%gJYd`FB(Jmc>8(pOb2oRLZf+^xWvufjVpjCW^p!Dc8eM;W
z@$Tb1;HR$qgkzaQ7;O0ZUh&$@qsd`UU-`XH^j^KBP+#TJqAPnP<}CC)?C`*J*O>&}
zHUCpG-Y$L{CLiA4Fq!Y0P4C0f*L<1#&P~-97gcQVT{H1pV@474<+)1dL<}VM)^RTQ
zT4A;$#Nznt3Wde|Vx|#~V!YT^D{d@0Hr0dc;ras!7q4|R%{IvjY&-h7BxHq6__37M
z;;qs@p3Jm~KYdnXZM<#mzhBks_4fV$^YwdN?X`bvZduNZ%I<kwx#gDSLuR-8=Ji{@
zzdS$h&u;OPKX+Q|&$G8LfByX4p49f|@AeeA=fCT<`+d(pzW)8y*YEdMmHk+?D!$<J
zx*pFfvlMh0bge^9n13!j&A0wc)M-9#^Sapk)4rxm@#E-u*DHTV+MZ+nq4vVcMZ5nL
zo?SQf9nXQOk;)mu8g6PmORm4Y=lbc9E`yD|T<xw(ONqPhVoUt%tbUak>)*R}{A>KT
z^0@tfraxXE_Vv7fU(EJjH|H0!1liYE{px1As`<aNNozv-!k40ptfsQ0%1?4GzxJu(
z+$Pb4bc2$t3852Rp8R2azfddihO%MLT-&2h8H7@gbeeRqoN#}c5|weF=aQWC@yg48
zy3#M4`mp|}@M<shQ)cxmq&6H5N%yLB{$q3gGt+_4Rs8bq3}uU}K2JZn_~eo`!U|JO
zCm+murX?d)&URn*Oi0Fo=$(?6)uMiEDw{gHyYtS^<JlSOwjS~-(Q4-~k;uEr)h+Jr
z)Npb7!@dnp?5|9nrT0wJ(+V$^VRzUak+~u4>%CQ9&Fxb{SO5CHQrfs<^RbOmWh@f&
z?G`dVp2PYnQ*i0esuc&dwzvC~MP7*ASaH7o*2nA2zf%AIEWi59JLY}=r`P9P1ve^M
z=x(|8>ge<Bmohv`eyQ7Aa)z7g#?1R3m%`S&`Rc)MXAA2e2Znz9CE>6vL}X<a*PMBv
zRVm5|5r4}M$koqwJGEn1;)Z}|@yrt{HOZF~RC<;g9M`fwQPBSVs%l*RL*+ZF)zd%i
zH!4ojJjZ$M<{Oz0TE27S_TQ0TIp25HkJRsuV)d8&<ky#9+sFIB%&w-^<W|Og<_)i}
zct6eic0bSd=7soJ_6=nL@%t)&Mcw6z_mlf~Z1L^QAAjAub@FCK?fVa(w}wZz&z<YP
zjiG1tuPvsM4+`Vnm$Rj7G`!TkwD!o(5KV?;+qlh#wbwA-ICFgI;gerpywbk9ypa1q
z^Y*#5(-UPGgw`mYx8833#!M}1?WNcz-J~_0@0Mk~JvF2A-8$CtI7#NaPyaF`6y*PT
z@b=bhm$i4_#oAcj-g$TRwew~7>P-Lr`1-87|I+=6TDF{jIk&2eZk_1f-pu*wk(O+p
z+|<K0?;ow*UH*P^=JVY*_wLvD5ZhySx90cNhYOi}HB9^%Pq;i>SYD8k7i)fa1AF$`
z`3s)pN`y7LnN4@uaO~L4J0<D!zV{SX#$7npbEDgtb8^y!Q&pWt*IrpTFY<Z%Dd2Y8
zwdaM?mS22#V2<IBO+}S+Bu&oeo}DJzd9~9tz^Wu`hFF8sTG^%Vj-^=lf3&(Eef!q_
zt?IASrfBerWQH(i-k8|4IV0*y>f1{zG_u2IS=I+CF#i$tTg0Uh{ee4Gc^*&L@5Ng$
z%g5SC<S>10dfp@+s9UT2<=(Y_TkP^4%(U@dH&JNrvdWLQlrFy1EpwLs<tJ&{GfQp4
zBX@(d`=4Ha_C{@9!qS6LSAT<-cU9}u&Yiopd(Zv<JPymhf2jD7l@zP3trqtCNBUB2
zbJfyWD-#!1pW@SsUgnYi^<8i6H`d^m^|LJwS-oaVd;K`~bNbVxdzH@Zy6m(#aD&xd
z<~<iPQvdIN^EYzMlZ;ua9dqZK7*9%b`?P%K-J|bK_s{-a%KtsW$jC_PdemmE-J7y6
zbk3CX_22%?^Mbc`|E0|hbrTdE9GurzNtv>2S$6TR25-?S)4Ox;hDK)V2~M0TI8$(*
z+`c<U-@Z#bHh+ok63LVC{PHJHSFd*u4RhgQV)^}G!RJdED!s2C{ybN1_x~1C=<J^>
zvnxMo#P66t>wu1Pcplg6%1fJCO>|2i%ww{czkK^7-mO#b76d!;tL&a4FVgtr*;C$|
zrjksY_In#%Gx&Axcy-fnJIiJxXTMoSEctgY)KAIg)a7?*<C%U#$a9TMyR7QUJx|Y?
zCEU5VTsltu<&Ue;+OPR9-So3!Za8M;^grwlTZ8bgU(fP4{yJX#h)Klcn&RB{mz$~?
z8oboQGq<hZ^y~QNk7j4Xp7Vd?<Kv6DdM9Svt%>Cqg0?CaJd-Vt+1;k7!;tagu*0Jp
zCH`T{F3Tq7X1=^)Z*^&T^}kQX5naUwGWP<rf_LP;@jLGSz=Czlt|uESKHqw~l4pO1
zWq0N0HD2}{8y$LQtc|*Fb3h>A?t|pR8)xi(6|T0mp=|BzbIEL<6Qr26pP$UxX<vPD
z{X*vn*`H7R=-l`xOuBm0NyA%{&x##+Qk(z#UP#ABvvz@G_T&$3kLSL9+Of(Z{6wyO
z=42s%Hcbu79Y?0=?aB5%+xocR`c~QF%s(Cm<ZY4g>Z(xOk;Q!=F^ALWaOkv6F3#8I
z{x0)*q4jY`R&bjXU%I8QE00*`9o4yu)SMHPKQa^?7hE@cp|-0*_3h1b-L6K>u0Gx?
z&YspZ=i?~{{^wl+nOZ73+yc`My%c(}ZL$0LO;c99Key)FQ~fW&nNu&RJdt_5yvZ<6
z^ZM-6Nt+LsnmoJ0AoKPs<7xA=>%SdNUa(wi+FG%Ku4jAN&TC}tSn>4Fw7XGZxoRyY
zkrxGRxUbr3-6?$Dw8za-Y-7iHaqfn#9r5We;!LOJ?T)pzJd*OGN$r?Z6qoXQ1_rfY
zR+Z#C((`VgJ8?dLqG^oF{B;Ml+8^;Uipgn2%wC~B^Tq5P8y(W>cv~m`7M(pO=Hct-
zJYm~qzp~w|-G969k4d(>#imVHjy-u%ke!`v^n&~1_NX&yIVlTg>R(Fh32Iq(DDYAO
z>m}9%P46S-0;?Ghl<hj<xpMZDtG*?neTy`?Z1wd&xoOmC{nmE-Ywlca8^Uuo>i^pX
z$5-hv2IT(XJp1|I{t$*E{`$ts?&WWL^~<N11utMPx&MFq@dMwtCQj=MJQMzWe@*!A
z?Q?m4ZI$IcaOvH{qmy|byu7)8zE1emr?=K}e@pQ+a>|Hm<rg?TEo$p^hL{K6cI=dr
zw*6XmgXPLA^_OqrN|}?F&0qTb;Ju$|tFO+xUlnB8_B}=6m(tw3fmtnc7-p=oVr7u%
z6Fy<d>-o^G^lp+;ZdUCz#t(aB-|}qn>|J%hVB@31CGS%*nHtnT&bqVW_Z*cNlO-0b
z=gnBerLgX+EdR!a^UL%5&b7{1wX&SU;?xZp$%A<gCZb0o^$p{*mpk>uR;PYG@qM;L
zleI;*qPJ$C-E==W`?E{JJJs9F^tr#ql)tdhwV!Gx+|axxSaG?(cA56KHr@1_{&R0g
z|0?4y_&bFsCs}HhnR1%;M~#nfx6Mhl-@D+K<DtaX6S57De{A$%?z|%&{$xFuki#j~
z4S$m&npv}w9r9h&l520Ql4TH3Vl~)Q<Gu0jna2(19E49WgoUq6^I{P4)0Am=dZq8%
zFSYG8vk&TSiJ307=Hdb72ix6uOjLE5_V@keL-u~cd?k~&EZ_2Xj=FM2=)Oy}<==ul
z!}`{732Fc4WZmU-EcWB<*&91t|HLLUZV>yJ&Exo=m#vuL>$TiN5u)cp*mx9|ZkW<@
za8gi8q!Z7{j45k2oN5U+tkZR^$e1<tphvy2D}SwI>5Q;#nF=LGCD_lM_IszcovD-e
z=)z07T=u5B-!3p`VA$bnKlk9uNG_e$of8)St$jSzJj3s*-`^>!K~`N%e9;Fce^|1G
zRpedDOz~sM{!wd;HU>PBPLiHAKkmj0u?w&APjKGjnH_2)DeSd*O4sY6s)AK!`9H5N
zUi$Z)TUF3R7MUx*{>{mi`M2YH(_e-Ir(R1j99xv3|E9h??!+yt{~Lal$Y0(*tJa*Y
z&g3@#?|PH^$7P?xws0xL+OD=*SN&thrkZ#5uC=SL)wjI-_xr2(+3Rh7cf)U;nmI8|
zjHhwRB<tsy8}{V?tN8Ts*WcX?=Vo4y;tl=Uvi|N_zrM%Ib@zyw+?LIIpKp5S@5b$4
zpZ%QIew(4-!P}WUdB<`X8(ONTbSO;s$lv#ZWlyFRUqjD*p~W-In_qdl_Jn1hi!9~n
z?2_PJTREXhU3>CNzx_Lf=FZDyY|q~C@H}TMYf)XRy0G%$!X{huXTCcuHtF^KRBYHM
zQxZ|Fd($9x`MYqRLNVsm9KRUV!q)6^?>8#E=iys=>fyA|02dd*T~2v{)8?11d%We(
zm8~M|cNXg1o$~D|-})OmC#&?1w)S=TsXRDU>1w5$v~xo0gv}y~o(wOysJX~hhbbf)
zP0VKxw_sqLK4Ib8<wcoKS$6E*K7ZQmW~;tOyEJ8HtdTqs_-JXtf?IdKup~rxv73J_
zuDR^}JeKVxC@C`3EKk$i(_nqX^5;sHmZclGF1x+{e3}23=q{W2KOaok<9e#`(2j#e
zM@+Yf=#(1l+0&J8WqM=n(|eznKQ(er+F+6HG~vqp$$5$s3O<$3{-^MwWZE;QFmM?u
z|H}S%`u{iouixI#@qd4(0k5K<U1iaM=f|5Ly6U(nxZZ6ypIV%&q@2ulqfMyoLu~x|
z?aJHv9WI-S3j1D9OAJ|{-FyG%%gN#_9AR&6b;WYO3eC2<E^S(EX1(w5v~cTx`j`1G
z`z&*tsJGOBN13NP%(BWvR{V6##_Y`oQTbb+MXh+@C(-5LV-g!ZOLN~c*ThSqtM;9p
zmgcp@xc6n-!9xKqo0?87*z_$y?{QVF=j<<)wwGfogP10s<?;7A+))zi-uFSMn#m!#
z*TYX~Wu({9`EJ`TYc^}$-4WN97kyXbQ)WeOQj18#S*=8i@0TMkx=JbXI7_ygE*CqV
zd1uzyw<~tZG-n!F&Z~A$um8Gv^Zz=jKM7{x&mQ*V?%(|CPI%hK?6a4<jqBMGK55Lg
z@jCxcg{RJdyUt*?jThsbpPSPkCoVQkUcL1t^Y(x3Q&)(dcYU;Pwb{ML+xwz#1t!<7
zd(pFWhS}7rxf`vLIRko~>?ZG8n14m&b-{f*``Xfd`iqat6{kL3|L0$~+w+P4F9((y
z7{{FuUn3U%rXkw2xbo9dQ;Bmf^4ZyU6S6GMy}hx{SKxeD-qBeWvHLCRzwX)p(l_sC
z^X80sI+;3i6-A>$lXb3UYk!Vb_X;rGVw`=vqb+&0=*?}5w$Ce_aw^{^+c(<q(W<;v
z%WOZDm#jRU@g{1Im~Zpbt4T9nZI!$fvYv59#HSs9Q+B*Py1sJj@0T@m=kh*(nK370
zS<j_?K1?%Z)NB2%t#jq;7O?z#tG>LsIrHxPp5?yte+QZE*{FSd`MZ#ddk*xaC0blx
z@LD2!M)nN9yB=)}YDueJ^c|C6IQ)UZmdANg;!;7aGTU#<_T=>)jkw4=d1s@_+{x0O
zyaq}$7e;%pykIGIk~1N~D|KC_T5)=T&$Dv^$L9GchvZCiyP&|(qUa-b{=xDjt!2|s
ztqLr+j{E=Wj`g?N_uu~bB+onc#Uaax>#%{zm&m!{hXS%D%3ZneMNl#1?M`!F<+-kj
zi(c*&@}B?uozG=c-|JthqLUl!-p}84l*haG6pziMg?DE3yA|(Ucb@Iy8x<9sBZo8_
zA|+<tnbj(&t1_#PF~VoB@-fG~n-=Np-LNchMyP=Ji>`IwBPBc~<MaIe3SYV{mP%xM
zVa4pUXk||G(uL;&W=fpa-X8Y)Yg?klDmlh*oyX!EJI)Jmvt<kJKlWf{%{$JEnX_x(
zB>fO(*f>G>oL0hDW>pQzzma~vOFrITqrCUpo1HWKc*Gdi_!cZ;HJW%yovnL^wx;XC
zgNJz8xCQd&8NFCqyd!ea1*f$IE;@5OX09-clX>t>El!5zl3&b2i>^bF=Mp%N7y4Wi
zVVaX(wz!p{`;y6-cZYg|FDmbE>r$4A_;13t`o-~}cLyg~GM$)lbJo*?{;~-xZp}G!
z(6>!<f?faLu4^$LkEcf;`~35L-NgU1?W?wb=3hNKJ}UN?RWJ{4V)biz{_h`OS=+FD
zcz52u;`vwoJ-<Km=e;X_e0IIrJ~O`Zs9Ra_m!~&yC#cm`9Xe;8x0>r^DFegd5I)gs
zK@680)py>xZ!Q}$MU0_h$6M36KR5i>nVFaHt7KEz4TIl~t>G3ie`KN#GtTp~e-fNf
zd%W+v4a2>#<U4zqAKa1O5|+p6wYu<i<lT^Gj#_o^&%OC^Px3;LDTDs3)eLq_4LwOV
z$L1E-Z^+r2yx2UrW!bwGUnK&rX#G-uE!z3uTT$G~4Qp=Bo>;!%=Ip2U>)!E4X<uIQ
zcW&0-$61es8CEF%?XCVR{!2Z*O?PwE$w-3_S^5&(uWzr~7W!GcNTT{Hv#K1Q?74YM
zh1PLQQ|sT}-)p@~%(&1X_U7ucXVDuAIxSAhu8p~0rt{QD(Y0x<(w`|RVH&A_=iJ%Q
z{IAFRRQa#X1_kBJFC)B;Zr3lky54cCV1;PX1{sd3=nugQnNDj@<!7B1B*{HJ%VJYk
zl=wP>m90JMt&1)8sXkQv=yPdj@KTSBU3t|%FK%9LE*kjr{iJD3@lLk+uk!m(WQW`f
zY)b!UdXCfl%*H<JT^91QZ+EKdM}NDpJ#UujosiU(X3Z0xTy^4^_WIP)+_1uyUblLI
z4_?c6?e!4UdvskG+)$1_(`wzw-0@Gr;JAO#G8wPPhi6heY<7fk`7;`vbk8x<y8J8T
zeei|wA0@ok+ywr-iEC>KDVuX*>mA3CMOWV3EMiGuJ+%BO%c1w%rvIHHSLD3wr%<7*
zkxs>><=f<TPES{;xhbc5O1kucQ{w4{$+o3kVmm!rd46;h&zqWIDCcoxo{D47Qk}}Y
z^Fh;Q+&$}fb!moew0H7$>zmp!dfeqp7cqK&Q(rBzcc!DJdONd$=sTXNPp6d{a(`eF
zI~4t9-tF~&o8P<GA5ebWH}7R~$<pHA+qc#qI#xaT!HUn(`+x7BvHacjy5HYdbrq{@
zoacPE^3|)kQ9Eo6uBXllSrxkUb3$lH+TP{ay63k1ShL^WWak^+bGpmfHq86<rY}}F
zZ}M-ConPAYLhsM!+p(NUH#Vt+kzv{;hR@Ufyh;2Sw#iJwp0Ua>|Cjs;hor3h<JI#f
z%-guK*_8QdU~=v5TQ`^kOhtpIdR%>CBB?g<_^fT^4>m5-b(hO#GWgTHa%<%6_v~w#
z86MwaS>gVvGJCC&P2c0coxR__D8wXMOlB}Rnk*&v^gwdv?&6Kt*{wpK-aqTQY0GS1
zdHJ_z&);KmSj`+@e0%lk(>DyR{Ic8d>Wx#)rnm2W%JS|nTV}@0x&B+y&)2=HR;qly
zJT>nr597CQmS)2I3^QjONC{uV)xZ-wl_kP(<r<x(kCyud+Gl8b9|*dr)jp+iON$fJ
z`{jJ1p$vT|W43RZcBJ`a(x>9`?t1|}ucc>KKR)O6b-nwJI|h>t6@#vZ>9}&Ls5<O?
zy{qr>4hGX|Hiqqz9gGcM-}-%fy~8zpPL{EnMdyl7y=PK>)JiIr7j0)d%MgC4<|bPK
zcf;K%wfCZhcF#`pzFT=O^h23%+tz(|cdAWef53a*GJfIN#{$zH1x)`Eabkw0b9?^k
z%UA4XbkEfi=kD3sab6yrYSXVgeCNN!vQEgPtofn8&oL&?)(dA!LK!(1ywlz%QhWWK
z#k@1CIT&Ziw4SgPI&V_<TzdWbi6;-4&M3{8Rg>1o&lWK8gy}4gS6ou+thFB>{SM*{
zJD5<qcUAn4?@NUGk~g2>xbtM%)N5_apKlG{XK_$*)AaXe4xX6I)S&*BH!Y1POipz6
zRPKEq4X*OBkJsgd9sJ?Beq(mu{5>M!qK}UxFg8dVTezq{;kz4sPQW`zWEub0Z@XS5
zZ?ToKdmBAljbZDqs&9WL->a_rbH)1qyZvAGeV+RH{lAWFiI1c2|NQ*Ew*04c{Ke(k
z3k$DF<XW4Sp4+nYF7K>`8_V7w-2UZfrjaDC)t@_etM6^9h@bc3{h91lf0<Uus#QEO
zI_ax0)tTX*0K0nVycmXnJ7Nu23bogmI@oVr`e<Y0i|v){4z4{>4%`j@yVGC3**^8w
zh577eeupL-R-Y-)KKaLlclz<=^RLW1ua-Xd?!*OVCfW`<cXszVpZ`5=yZC1{w-p@|
z_Y2Ca&*#~`w~{|v^ZI!|-~3q>Nu9b|pB@#8UbHp-MECtPZw9?5M=xEx)Vk+|Zx~zu
z|4kofAG^aZD*M+~*D_+A?SBJXQ^qSIpKW+I?5g?qdCPD1w8vplrlJhR(^eF%PA}8{
z_G|6+T@2IYBMz)Spq{mL_5)iD>Hg>b@_N1-SmqR8N@f+xs<WE9SRg@gomR|+39Hw5
z^tz`DiuwmAvfkM-<KU+i^9nsrbvsyz-d5Gt@i=JNvVy^*Ri!pm!SSu*cIm&1ciir(
z=Q)sK*1+{#!1+?_%pT4YlU_ISR!lqeH{^#|^mc3a>c5M2s=F#ARt0$l=EVdXL};Hn
z5WT(Q;HTTl+&67XqBV9#m%doLa{9Ubt5(R*??3xiamC&AxYCF?J4uemsfsN+FHKz)
zDs^IxKNNLWiTM5}uJ+gWpqEV6`hR~ZiG3{ZVm%ObWWBLXW6C9dsok-quNXzoJ+d`A
zcuP!4c&VQbpQ2UY^WDa6-tud1KIlsJRhj2H|I+VDWsbTq=~DCeyY~jqO1gK#G|I2H
z<lRw`pq;-~uD*4qzOM5B*<Xt*FKrK;A`s>E%=!IpW7lJw69VGzH`GkCnOd-^@&V(9
z++$3ZkLECIq(qd)`>hc(y~7$%C}ArYf2l9v7{ji~wo?kOdbh24baIA=j`zwRWisho
z6Q<ZKayrUos<v_NF@YO7SDOyb&iH@EJ|I1lMMh?Cy_9{e$BtDF;S7BHzu)~dr8j@=
zmiMtd->3XM)%~zj?0n4k-~0C$FArO}U(V#V<L2I8cDd}Z{J%$DcrE_r-11VAsWIiZ
ziN_2k^I8*`2@&($eym-1C~Dr63Cs)y7sGCCl$p2FnL)JdrOL@hMutYU0`C_6N1N+x
z%HCxhe!SwNiR9ks-HX{upZ&bFZ}Aldhm{AVw@;DUW7zO6qfF|2n6|0Rp$})Crl%zs
zb3}e$`o)Gdr@@Ywk;kRJQ=ivG-r?iLw(@hc{Qm3TxyhDfxhmf^@X~rsU)Iv&`==hr
zW&Y4Vwfgeo`u+F+F|XcQR$cMu(Tl(GXXo2oS+31+uI^`E#vEDOZTr96U(0CAkoEQL
zt8JI=g|sqtPJX3+@+JG<wcTRLcY^e^**PBtSUbH+oteBQ$D=G`+vNpSK~uQA86)Jn
z`a5<gv7HW@rl9|7-xtQDh*s^6gFzPB5^FwB-6~tn>u`LlI>Q0xmx9Nb9u%-viYg`q
zT6Sl#ce2}yGU&8^WeZ%G@_yT1ZCySF2DR-cE*vbfD)~5PX|9a#y0gyr!dX*U4@~x*
z-OKdQK4YuBpYXmr58_U5PC6}^`uw}mv0a(p16<;g-Z0GJn!~Iqb3M~iDdSDnWIM0B
zD?r0%!rT*oE7ip+hRp3fT4E%*E$!Ys`&sj3w5(=#XbMQNaX;B%nVhV<_V+~|tK_N`
znJO8)E%~0oM^nVyw99yR9-4dX{2$3)28D^g7iOL_eEp^*#9XQ`Wp3!92`{Z?`@Ott
z!s4{6*DW!q<Bo5h?p5iocP4*7Bo)ofzRkhpfBAs+v_pSQuFTL=mSD4<?-Q=qvOM!b
zYEJH2lPfoZdrBM*Jz3Sxp}4@kA#p=iNMH#o$H6bpc!axW>hLT{HZq?u`B~0M|Hm3O
z>MN@k7Vt~$zE#7a;N->i-KczNc9_V|n-`+GmEzOAmS}be^=cOT27eLYV02IBTGZ0P
z8~$E`gSTbfy^XK5a~8CG-je3)|8KFq9k<@U<A=@j%srp~y98?S-iZ}@bTq@jZeH*5
zPU}LJdfT<RQNEvdOs?AO@X^LHc$HXv!J2EyuCc);E7xzCbZ)wHt*=&gVeUzjDGs8m
ziy}C~qrYwz-`e%$^O1FPuN+#n?p4+5G!Db3R@{tBW@zbbew_c|r%gz*rH`R3_sI}#
z-{%jF{mZvLdwA*9mP5^w-81A|FBceHJGRh#ZrkKX_m8S9SXdNz@Z?dQOH(&5e0M#v
zg!91mRh=GF+2*^?y1$*xxY}xBaBUhx23P(f--2a7+8dKJzlp!uvM6)yu1&EQzf^oV
z6Bl;mhUujVJNN&H4E2nA^OgPi^S}Gw&HwxT^2f)@?|(iHkK3CYK97Huyn0f6-}j(1
zfA^Wo9eJC#Ja7BfZ<7;dEU8)YUHajjh<TyUrLGqlmrJ(pS{VN3M39etuj#T3pTuoq
zt;KVtZ~6SXd?HB8XT5^gjJS_?HlE{U2(;Kfx6QtHQGwja#=MAIK9_eY?)7-CTVIu>
z>Am=tN-x7QXRg@xo=cJ<4(2@?JgG^aD_)-7c=S?dkvAvP_X4Y_ZGxFT{?7V5sjpgn
zu7}P($sv`TC?VA*=;`a0sKV%wd}e-}+1jRcmzJg%xhDVOsWOpeKh&S%eA)DDz{}{e
zyWa1vKUSEl@Mv$ELd~RO%olpyOk~d=;%I0KS-*X0PwmXQi?i9jcEyA$WlZGZp1XTl
zcKGHN(bFqMd6+t%GG_d`6ZFNSEpUmkZ@J7f->oJr3yP1+oe#cT7PYqQXaXqz*wyPu
zta1IbtLeP!wN#}iryu`qvJzs537_R78dS85Yo_MX87JgEr}k<c-EIHBA!7CHFH<e#
z%imsmSX*6VP;j-fePZz>_78{tt$K5%!NygB!#+bdUt?Z=ESJ-+w=V?L7`!<drd6xo
z+A_hp_*&?y#-PAu>Ic*s9Mr_L7(~vU;NSG3j46fZ+L1-iEL+yCsQY)QY}b|Uoo5ne
zK2~SodKvmNeczN*Obq+om#}h{Cg<NM(yQGzW9?Q2qvwUIcRugmX14F;nYtHs=V#ch
z*L{3Y{C<KH-+u+E|Lvcb>)UosthiQw|DDB)ke@&M&!mc3_Z^<Q`|i)28(h}!CO%xY
z`RPrCKa(G<kXjnCb3b#*!aJ+2`LE7>{<-Q>(AJNW=bl~sys<^=n~h0ynN?LaOHWBc
z%TD8nAlrR!KPaTG+o1dGptDZ9=(SIG9u)B#q}lKOFfVFVuKGNdMJ$3-yv4nbCNm^i
zKABfFAx2}3>!NCgCq-q^6}1+IOU^~=7&Co+$<xT4_$$fN&zwQQW>$Gh(D$j0PaSqO
z_F6};pOfQv*z0Q4Tj%gAuU1XCUGeduMXiCCA4A3`zmC2C<j*#qmTOx7zb~}@`Ofv{
z%r{nj`1kUnzuS+KEB?EG{`&uKrmtn@x_j%cg<D%Me;0hlEWo-gi+$Q6hM27rD<1Hr
z7c{&n(Xv?8^kd6HMSsKlZ}v>teVxhSArI4aL&-m1l!SLa=LxHOE<Qy*WLn<JwT27c
ztDUkjU*~PZId3V4=<iL2d{0D+x^`%<u3cHMea@_>p7y_|unV<n#{P`1)8b}ZZKU9Q
z@r_-9?<+s6^3y9n9%lTRAlU5sS;TBPYjf#l&vT+rck6iQuvs(|$gn%)OI(Sbv-)~<
zv2K#!d`mHgY{CEWoo=h^R1&#nraQ1~`n2mv3fJAor+Dle4smf?svNp1qIh%1=cjI5
z9s91=ezxOdd(v#Wag&+(Q=vn%WjTG>Ufl@x(e#@C=Za3WAG1O0YsS4LewmXRTQ-HR
zH9ow5MkJSa>z-#pzD(1(%3magZRCudxY~N=&fN#5>Lgw?uweGDtkK%V<);~*p(%RO
zGy3wP<(i2)=cKDQ?L4;YfIeeRwD0-)q(!=OqvQ8{_K~adpHOs-=~(@Sn(cdzG396e
z>05jtCGOLd`x}LqZ+%>Fv@Gq3k$uUv`1Xpzz{5iG-n^B({rvL+rUMf5+=K7#Tj8*_
zXZGqJk6tCDUrj!na*KJwhQG(oaTjNI7<~1A{!pi&@7>ntZ>MeFvvqOhJBx2^ho)|n
zVAJzqZ{)tLI3s(tnUMb~Gv*zSH!Mo*KKaO@FG!J*!#ePeSZVT`ZTu@g24-!KTcUfw
zRCdSVoQ)!y;?LugK38WIt1=19&y|0#I`#SNu5GKzvrISt)e!mhtui?`I=6-?!ERrA
z+5GrDfBohEF1r5fTc3s8QZM!c?^gfa_^snjwDj7yxBiMh_AO0KpW*lS^1`|==Z!a1
zd^z8I?*GQ5+{}|Q6YLtQY>dKJ?br}Hd-BR%`l|Y0UG*l_vTXKfa1G{4|F5kc#CYh`
zC%fg^do_X!Z;Ge#hCP~iV(#i~tG#Y#N+l;$2_#*1xaT6XBZqfxRnWtL+-Vg;XTDut
z*nHY#*V8M1LZ)pl+FRn+(0O6%(gKU7J#L$C&XG#4aeXN&wPv2tmI<+6r<~R2*~n0_
z>Eu){)d?><Zl74iQO$hziPXalMiaEm+&*2)sNer%h0ZmW1Wi8$#tz<NLU-z_^tAg5
zW^CSYDC)@besFtN{n;a-tuG$c7}X>`YrEL8k!j|PZBgGvv=k-<#Rz+fyjZm%O+mhq
zDMxne>(|Yl7V^<$-RJYBO0L<po+XS)L+$gQKfGHf6i&(9&>8w;;iB7p56r4x)+lqd
zTvogIPBo`Ugy%x&s(m6ow&rE`Ij1zP2nj0)oO!#{;Plgm=bpD2gWtXhI(7Dr@n$xs
z@Q^>QX1q_`ADD(#>l^O+Ss@u7tE4M=TSr^wbL?^rCFLte^Y4HBb8p*KnbUC>x6GJk
zaO4Efw0^UndVFk|R!4FfPQC2@U-0h>|FrkhZmzj@Q$p8>on0dQ?Y6gPnvUg(Zn&B@
zK|-S2ZFSxD^@}&#R?Lgq9JsrX|G=fpY2Ry%ta`uiIUp%5!SEqVKdk;T!@HGNC9cj(
ze)MGW+l$AVUw+ZMmS9r#t?<;G?pdauKN=#dKeWC)FnQXx&bq6!^A-d}?ko*sf1q(&
zrQg$L<*h}J-g+HByX*H(JsDNgvxiMX<JZ5FU%B{ro_XrDL!$SRrtw&=&vP&-zx1JE
zS7`71tZTbMUjKHO9^J6!TK0E_Gfj>A%P+~xTKlm6{vOyh_4n4q_dAchlmD~(zjVU`
z1_r6MZ$B3BH`v8?@~$tszV3##_Wp|@uYaGE>FRr19=5n%;@f|X$J>A9_53d>c`d!H
zT;ytEw*Di(<l8-}7sDGAE}48>@;HQhjnA(=PK8saT099fd2*7kWaZh^#_fN;97_2R
z9AgyGAfO?!W?6sw*;6MXm+gD@tb+4Eqqc7AN;A)iIp>Zhp3;fgmaF#fWROI{WVY2c
zy3<Po?|$2)FX%C4cCg~=pf7xvxAmRYa*|TyIwd{tw%^)>MKdH;-Ja`wdCR%u*?$dg
zEHk<{J!5TNl<|!mmo*|Q-?vVfV|=|;;8Nj&TOF}4Ep#T7$vVWf?dfo>`4_>k#`RCt
zqEnGi`0jST;cxP*maJe=n84cHx@-M`V@JF!gIgzDmsnHef9=q*xlzJmt9KkqxazlS
zb}F0p+TUIbESJ<6T-h!>Wo6<}k&!>~dRrCq*&>M;nOSM(^D{KpC<rlpxm1}o``0no
zH8w`OjV5K4?7sAW-}Q|-J6zYbsK>2Ivp1=p!CI*<e1LV@3gw5sbE_piW<FNea9r)(
z+&1UF7WX|yk(rJ&R-c$@aQ^+;oAG~Jzn{MTCS2}#^E#`ys>doS?96;m%m11^`)zZx
zo9n)g)Lm7-=T1E)@Z;L6_w#nXpLF<lXW6fYyJ{s*pS;c36gyFZfg|T^<L<wUQ-4bL
z?ug6PIeT0@gTWzuZdv2rvYPV$hy1^5-LZ@|visScnf#pf{$0jhpFi$BW;N$Uuc3ep
zkKwxm$G*)fDik|!yERXdrEPKM>dLxJMx5n7#ard)gf)koybWA^%ka=<v4*VjOuKyz
z%x!sRXK{Sjygg;|(x_thneUby%QCfldu`jKh-J5KHrHiKHf*`h;NrRDX1wI@X05tO
zr({J98@koKL_5^&^XKTT?YSGV<LvWCCIA0E_FcTw`t@>C;cx%G%P~hkzyCKdPwty?
zOz+nJvTg3M`(|lbG59!bxA<;XV|eS&$7MAyRP7_KKi>THV6I8Rw4RkKD$Ngc3-{lY
zYJ7S6(#4=XCBIBFrkbt~nP#+}J<Ivrj)K!XVm6x7r%ivBR$agE|19Z=tM@(Luw)7w
z&ypotP7JHgPCCRT$6&)0ol@>J-FPWS^}C)|Wtug!XTCHwE-#sJZNFC0{!iU0lSOrJ
zHtv$YGIbTdOnHg0Tte#VyzIS>A!ov-?Mh#*_w$7CF|7$nj~eGlF>$iTaW7kIcJxXf
z<I&j<mBc>IOjaz4KeAr^S<<N)7wxNq)@Sds_<y8!mFtXy-p-6K99F#0`M6}A_7y&z
zU8fQz9-4d1b8?*A#UFE3O1Ra_#e5H&?|9|5&gy;7EH&xy8rumEY&e|Hme=~u&u+`A
zyfi&?W5lMSZM`S_Qy)rixiaI&WiE!*af~GjpMoQ2Pn^8_ng4NyCx@4OTefVy`qX9V
zH^jRP?c`Ov?60>eJaX(lRI+rTq6)9r&NS_+Y_?{bt;&`g=K6`ud)-!g_VC^d+r%f8
zF_@?v2=Q51S#tK15~G88R^+*k&^;bgp3ga4zU7w8KK<4Di@x(e-&>fxWzG7Y%Uzb-
z%8FW6&3aEyDb&0;c7KLny!5vj<>evvpVRYmY7ZMMkdSFel<11{=LueGy6e@I>7fj<
zrCRR4gI_GpWIga#^mx#t!lV0kpLz9g%XEopLe^2f5>>KOJD02e7599uRCZ<6SD$6a
zi|W2b-&$rf&usNuePh<#&8bq87Ye>#7n=9}$E{x(-tt%8|KfLgyLG)uSg`Bls_41b
z^W5L`E}ms<@Gk0me5FbFtktK~e@!d>Kh<{e)57dsJF64d&D<Zletnr}Zfw5wuUFS2
zp6}<1zWPi%e)h6GuUnqy-0X8M`{=3t_r;m&rc=f1|8Gp3e>`c``?>34Gq#81-&~=7
z?<L2(oD*wT7OpN@z3=?$-^ccTdH90eK4xRK=-$$rx-h#Y>HEL>zW>>{@k(d)d%ar2
zR;eB57d_@bH1++;kf@KpnytONf=cHsjZD@q;b{v?$o84<c{jvhGQ)(5g^729O5App
zKGAq}D>QAhSBr_0m-5s8uFqX=K^K#EJ@xnZ6`N!z&ET<6I;Nw5t+h&UmstMVD=|yg
z+5f(5r6tYVF8%A{zm@a74kz#)XAJ1Id70^WcKI)UgTFOKYtJejHaKL!^2VcW;<>KV
zl`8{Z`ld|_?mpyVF17Eq{#EHz$6Zga=(L_a&>pp7f0Knrdb7r=^II~%342V=ljhWX
zn(39C{q>A&<~t3U^KbR$bKhO<%jhCBYav?_Q^S-utxNVs#EPiRY`Pa9dHMAm*NJ&X
zCq$i-J4(}xG<FF}OF!<a^-gEIu6;~bCrNO=KB#fKf1bzT0u@6kImU{@x-d@tO9CNI
z94|V}WwSF)lah6F&qXL~4l_G?{qHxA>DHbvm~PytaZqhJ%HgZd*1d|O*>l6Uqqjp`
zBN`XJ_L5NIG1Y8MTftCMer!(Pvhy#`aXVbzy0q%BflROK^TRQxR$brxd*8ZiKV#g2
zl6|)F_HHx~`DgNSTYKE{hn|V~2Cl}UTW{VsUn=0cFw6PC+(QOR3l@j`4Y<fV_t(DA
zoL5H$&uyCYK45m{#4C#m=1GW~#&{-9dCl(M<7~XmcK`9~U%Xy1D8#fH9h>F*;^C<;
zTnu-kE}L>6)ZC(dm0|CZfZ6@~&1+3=Rpg&zKUj6*aEh4z?6xu+8NW3LU%Z`Iv-_Ei
z$D&i)D$Om#zQ6aeag~q!+PCO61J7ZD>sAa`q7I*rl07v2YUJ9v=hRIXM;Whr+?A_Y
zxc2I+uZyErhpx}wc(3E@u}O3IMQw%Znm;dHb~kQY<p!y~toZUe;RCBbM%4x}UI<*f
zZOyiAyFT8?Uvhyd^XOUwn~Tz?7k`vrz188W1cT-CzHimDw@#gw$&fLV?eguTJM6EQ
z-Tv76*=O^D5PnAa4}zyZ{+xXM#@k;h7vf%>%J`6$yZ@p251UI`d4JZ}TUA`Vv*O(S
z?yxhr{5I@frh4@P;{roFL3Pn)UCmtE*sPGVsX0C_FMMzM&)@SR<FRr5H}=!z^LyJF
z{w4g(oBi^=&5d<(_WtRUkIT=K{rgqprR3Mo5f^7>=70K?d#?A}+O2vz#|=!DW&2E@
z7`##Wa7N|j%+u@rO1&=zJQqyJFh0rq_i{kUHItLCgqG-)hwc*d+0Kw*tUEzUU|FEq
zREaePEDO3{mR!3#ZFM=Dd;6O9saDasrAoeq{^ylDpIUhz4!GEB^2P7&6lvoPYc}|~
zEzHsu4rEqRxYV*|#<|^ZJMM+fxY)NWdDj|~TDexvFFTjUOp^=}UAWZcl$({*S;jML
zf#$gv!&UOnw#;aq`1A^QLu%3LwQ6BYT%70Yn>(%acE27`VmRmGSH6%H6E>`H-g77O
z=CllxbuWt#8<fobshGvVdw2$i<m)w3FQ;A3Klgm&)=e5JF-O*`PT5~O%~j`7c(+sA
z)usa0og&ZbUN2oDebjWJ^TGoTD_bT6wLE#n_h3aVd#Iq+)!XIPJK4Nf|85kxlp=RB
zv$ouACqsTkoZ3mIhnLetatkJi@+%4KTgtL-_J!r1nzfguw-m&z-=Nk|IpN&#`&UF9
zZU$^*xpBqsNSe+B(V|02C8E_r1v;snEMKQ>GTHc>A!oVZ?md4FcYnE`qRLv3y~|Pf
zu;bbj{PQH*H7xrM#q9lKwVJ`9AZi|m!jsdVKOS{I)<13EuR}W*XO<sY`MLY>@5A4k
z3v#x;+d6Gyj)wKrXI@Jr!qU@s?D^XE^37@MQ#{@bJKl3&_kDBb+@95bA2lzAE?10a
zyO25i>C3%GZfu*i>Mq-bx!d;#{r>gt&Aa=r|1I9yy{-1gv>CVj=317j&)vnaN`GE$
zp+s$A!MCWRx$^t}pL)}L?DPM`P17}g{j~9T?#T7P{Eb}0?!_<Sf|l2t)Rl|;ZM@15
zP%XvkKL1R!$;|oZnnQM5#MRr~J@RvP`s3~MWA1cHP0iT7+Hd>XwDNmpQm=O>rvJTr
zJD+E}{JZ>;=WjXI-||x~yvE2fi8<!!orgN#5{xEtIHoJB2K_($^YZ@q@Ghsuw)3;n
zOTX#FR)_!o^x$gyyHnE5bK7s+%zB^q{>_^=cM9i(9nE9ptJGOp88~mLg>yEarVnf9
z@jx#=rSBfqFK<qg{j%Tin8MY`(Jz>arw9IuSz&hB+>Bj_LE_i6LtR@IFRgG|a7cDC
z<AxAxhRuux>wc_knXMMr_$TLn+O**2+bj;(=RUo%%S*IjX+cV+<L(;<-ibYh2g=T`
z`c)<BP!lQ5wMoKs=hKsE4<n^G{<~gcI51uMQOzCSpb3dDLwT<)H+_Ea>NM-t6LV|c
z%e24qJ0v@8|GEFm=I-ur{lm49A)@=oSC^^AS(BI_9NqA&AjG1(m8GK9STW^pxY5!n
zE+>5?Cv9J+yQ-omP~=fL*S<S1mKJ~4Q4R?yv)SmQ>8g<Xr#G>vJ8#XC#kFr27pa>T
zIjZ<ZY?m#a5%xRr@0;~aA2V(2ZmqVx8C5MdTU>whq+Lx?v9E;L5BNX6@pM{HW#`2V
zzS()c<{W{7bC?87%D#T<u{(6N{q@rwJF9q$CI)$lKmT>AU;zh1)?2w5OC_fB_s=k&
z>lQ4NvcYrhI;~gdw)|~P`y`gW_Mt$tiK*o0@AWoveqE`HS{AM9Rh4S1i4?!S+O*qJ
zdhIS3>x?&H2dzKm?b;ub9Q};PZ|>O`b^*V+H+?Kh!yfYVeD-Q*xR8B1{;u!#t&e$K
z?;5uK$`3JT+%tC$^M;uo%h#Q4NwP2#W;i4H&ZYN$)VAyMb7TG}bj>^0vv8kQbp4Kx
z3o5TE{5gKrC*P|+PeXTdu654l+f}`9-bV|BXj*QIx>INs-S^m{aSC6$W7k80?TfDR
z$M^h8_^Ic(ddp7cguS~9cOPO{vU>adxL==|t>4c$v(q-JXwJ2B=cbo!yFGjDthXjQ
zOM|pNDhFK-%x&Cb>o>t^YwKGRu8hzt<}sF!ELUt=t9RxqPuzZ;CE^#~Gfhzb=dB#W
zz_5Q$ir@L2K@0iVFLnP3`L*(_Ws?7-j#EiDvpWS=E;Y!yx$;)z%q2?=?$5pGa(`~=
zRIR^}+e*%_*JSt*thqEGciSSKLyLD#4$)@*G9^*ck8#7z6O(-p`KWE5A#rQ1*w#}o
zS5Eg|=l@wmcm3U>EVb<~_O6~Wvp8%|cA)2}s{-@4<%@6Zh&O~Z4Q6n2et5L;R_@G=
z6BxAST&-<*vaynL;nELH2N!w0k(_qQVU@u_rC+NQHcl{(ICpB##+zl6UPazAySby_
z0Mnk;+rMfj8eQ6wc7Ez2=Ujz9dDc!Vh5P1v*_$#R$W@znDe=~=dr^PpFTQnoMNs|U
zId?VV3PcaOi8&mN{K>o~Y4Xv&=(1M5tT)`O60Og_HyzV!3a;u9n!}XPH@~G>%6E@n
z65|f73GQ)|J#vH{l~&i4mn>8L9K3Z`*UxZ9mBqh%!(z8eZn&9r_vgeugVOmjEVkdC
zP2YF7{xP?*b@e%(_8k>^>)tvrU0`_jm|H6(r`ohh$Gmx&dDPe4r~Mw?Y&<sCs=HMs
zZnN9oH(oZ5t=6LFzI=<iaAWO!&G`8L*Ye%s|Mk9lW}O{hJmvPwe?lMQ7|uvCnAl7|
zrakAQ*Yj&7$*u>(lmBiNkN<b}#p(FjHM}>XUah&iE@V;|SHo)2TPn9-{kpWWK=0x*
z``p#iCS}{#J`<2}ING+-g2CtL!ep%{A6%?i3l`0l+xpsM-@UVaS4>LpU#Z9y&UFp6
z%>R7O?e)CJA-T&B=HHiWx-YVd*~4btEhUdeo;1sO)@r|2)lBI=;SjpA{gQ82chip5
zAzw1~x-L(D$hBy_kJ#^(Q@pn``1J9K3hvg_|CO`oik`vii4&rC34eFH?(t}8!`fRt
zGnOoR&hMywf9^)@ig#SishVd>?{40*c71qksoeHN-dU_)6c<`0&*8blovrQfm9$~P
z@AHxK{+N`6`kbF_#h^O3T5w&o$fFw@1xlDI!dw1yM;uA-X$7t0n&)x&LrR6xb72P4
z{CtjiC%QXlEL-T2<NS459mlyyQzw<9JXQsoRjztP`_=OGuY{-aipA&cSrNKy_9T~g
z+R64)-ab1szma+2u^UGV*c{exH?6&Azt^PyLP@VAuXpF6fJs$zRg~|3GnjN|ewc~#
z(p@UOhLV|5-G4F<&FMS2;Pt657V~T^H_rN;@^az5oud8C88^&rl%HOjWw^|}Vcl!R
z8S7r?d`pq?6`jk-_8_WD@}vpZ^CiXohklx_+-)ZR>Y5bi1_lwSHA!5&rDreQ-n>y_
zveG`C3GRBm%QsB9(erlV)6-seW#?S)F)#e_VcpO8tiSuBvLmt#xX-@Ih@0#B&(Yvp
z_)UL5w;fj_?j_C=dvs|2vct#x-HvgpeVS^zFXsw3Z&=x$qR?v!bFaO=evvs`d)J~D
zS*LyW6dn!xl682dwCQ4paE2sB(YrfO3Ab#Vknk&LiI!B--oUG6uQ#_#M<wgzTYo&W
z;!^p}MeaWZy3R2yQFJpE+v#)onAa+u+tLi?l1v{YOAKT=4(>RYw(C}xy14P?VD58T
zYs>EJ$ki;=H{_Vnl_BQufA~a3friy34zc9r@4lqGw>A1Lu)b#V-?F`B<&|ufbFXC!
zgr?=><|kZ^Tc;oGciUh~$a+(msh`{O)>hW#Uwt(p!mCQKa+AB!Qr-T&FMZNfrY(|g
z*{{4rWwSPe$;tOC*baTJ*1mnZd9(1Pgu5Y0p8Bg^@?F;c7qZs%xv|38>^CL0CX1JH
zsu^ocWT}37weiTGlasIIWNes_xhK!0G;hv4n`<)`zPKPTY0}>sBh^1kUii3K%=a-p
z=GmaC?3-Mr;@n%fYmJHX*+QR)jN5Cge7?P7STLEbJM`-={o@Re!~?&JsjZ9p(-kS<
zDe0aVbZu?)#U9t-y@8W=3BLTrQ?<l3aE6G{x24g`(pCh{xG+IXwAEAbyUK&5(!E(M
zuH`yMJ5*L`EZnFuVc}t|*ROMr7G(9T-qq-;^Qibp)4`}C>vcO~84Osy&;M^DcBOT~
zaVN1wlRKwgy8UG9cfU#hVv6&pm#>>MKb3dd<Pb)I=FsoW4!10uQ!h)ensVsVy2<}i
zOrkgrb!>gbyGds0QWn-`*Y{6P*gV?2EARBmhj(VMGBb#CF;6JstVlT-%;Fi>al~jQ
zx5X6gEpy#wUeNj+;W}-@4`Z1t0ZE&?6#Q0CmtZ|#cy`H_j<}>ZHCO6BKRUO4PyYLF
z_m~RejtZMDW&RRn{q&9B*T3%$KhNLyIXU-tQtnIpf1=g@T;9Fe_Wz^&xqk)c{=b}k
zSpTGa$iG+j_x-*&Z+>b{x5&eq8b+hk9g63JUdv8A(s-;{%4y%7%-MS`dG7J{jhkq3
zZvVmV`o70YHkocncQKu5e8cBV%dvLzt9O+C>vb*Ns<3VCvfR>x+kB2P7I#f;D!lrQ
z;X>HccIIb4I5*brJ)&W>kk9>%;<pF8-W@Ec_!tv#Zr3mWxo7`xE%>+F;qTsszka(f
zU;F)S?(=uz?->muO(%Rd{`}}f$yVL-Tf;T-qh)j+ZCp2TI)e|ZV^`N*r+YhZrcKT6
zvU*}7?e(1PsP7`}_pTfY*(wuSMH2ov#a)Q6OE=Bf*16AUCbPoFE;Hsk{&$ZsOYo;0
zJH1Rbs6A#)#_kMlu4COx!c9_oCbM-v+f#P(_sTb?J>t)=dzyYvbf-##<g_=}yiWus
zZD5#j=JVai?Q2E7mkI9P8l!gp1na+BPA6ifObeft#BNkur8hBKE^l3}VE2!!$>7Zm
zoQ9LWJYdl*V$x*RTz7NFWVY^Nx61a}ue7UrM7M>$xj9v6Rnx(sJquT#lb)R$K6kxF
zb7$(O)62KIn@1k2I1qnn%T@ooM<!ctxx`R%`|uxS;hijYA6u3hY+9jLq`KwF?RPsZ
zrS81T<5|OZcM4;`<rqaH$*?ykZ%qCvd~WMpwg=PX*O?!Um@fa*mSOLa3%743FAhi*
zy%@Uu3lE>v?uwhi)1GZy`)=(^H=|wo^2>L3-Lsf6<Luht>czJrUR~WDpS^tc+G$HI
zymMam1>Fw1dUvA1pJs>3EuW<MrB+Y<s=t=0BJA(x#p&`w(-w&&hR@yQJXdC$)SBNx
zykh1y3|V`b9k%XWz?O2MOQ2ulL-|ijaSx3gjs<V8RxJH|>F?zF>Cf+<nsz{Y?t+cd
zpHlaJ7Pu*?;mHxvu+!mjnu<riDa(4ZX&hSxe3mSHy=&tMv$jWPYQ+z{-k&U4{^eMf
ztLgPMy1w_`m+ZS4nET_?oq{=?qV{+HeR%xz@#6BjkK)$yUE%XZ-Fx}MxS1Rttuc<)
zwsGPNpUIV!-!Uou%~tt8mr4{Z7ws)!zv*Y5{W&IR2X~{w(pcW>vfE~*RvlwCa$j*K
za;=Ey?d>9Fo7QE>{0&)U#%^OT`ebLm(fOjRy(Q-))cx1^7r#04W|PRZ#Hb3B<<mK`
zFYIA!w(1MldAO9ffZ@j4r}q?IZ8LqsRUWneq@HPnd2z?lq?bwMvEmIUEgmq=v#b-^
z{qD~vndl=*cF8&n*OcmV6(89(OweEbPVm$6%!k*P-&*rdd)3*N33XaJ%fr4LoUkYK
zFss*vS4$mv#2TD0mrQ!~BO$;bqNIDtr6o-VcU9K&tLPjJ%nkFnwfpS~^9<HQ@3*z@
zvsNzc$ogEgbm7U43qsR^qr?AJ)blf3jq8j#R=3$_(}cvxQ<I+eF)nY2+;(}z6SWl#
zZ4Xilr{<QrYd>CRrMf6-i;<e#>I2cP6?Odv2faOY7R}Z8c{OzTw?eK5B8kbmw|jYC
z=U(5n+i1V(7ULD-+}l4sVrYoEzkb%KQ-9C0|9_FqQ1?y!{_mszDw|7pzxq<Qly^bf
z!zZ=<6>FOAFY-{U-Fi<jSh_)ylOZW<a@MLNQ40^vZL05T<t`9DbjGr|(bhYayMa?e
z?ul{!m&^Zk=2y0b*M#4x`FA(=rd~p1>t>@}Tv<2MI#k3w?}RSDdQ2$s$T}-sfio93
z^Z)+6PJ7AgT^kF86}jTi8oy>dymnRPNzE`V?S^pC(D-@s%c>9OuK!i)zdQQ!yK8nc
z|NYqVSZs<!?}Uzh1!?R?F`*lyMR&Wd=zj9{dX&o4{`JqlulT}vz;_XM!}D)8<vG@|
z_In)`7EEul-m`)Ks#)~pYn%3$Hw7~tkNbDmFgN;<=U1VCn<q-c&6lO{Oq<{3kR}uA
zrlWavwUmy6_`J)nc@xx6D>GctI-cKuN=j$KWv@(5we3|e=eWExIqJVuGv@TlgTAL9
zyji+RCq^{QQ&Y(Q+1KWuS`snlvQ6w#@dgpyKmI_wh7T_;=>B{pQlC3bQ^VflvHANw
zU#g-+__O}-^-e#{*?ExZLg=b@ky|(3WOLvVOI6|s6#af7N%yFePg(0hrVG*;cfVbD
z5|)2KX&;wmt?Rc7J%tCnZDvmOS}G*)^<m}NoaEBTPn8oST~B3iPfpsPliE42IK;`r
z;B@ivji<Stx_9&*V&``F{ASXfta+v!QKA*Ae`KUq|6F=U)b`x=wQ3979~tg{C7Wmd
z>wI7P@^5Sn8{hnW_U+o|vJ^wH&uO#E-)Wd0n|FPZy?0YG!^xEFzTM~6{$E@1bawy0
zCwuF^FaQ6mnP2YjZu`I2@BjZ7Z~trZ_kT~%|5LAje>CD-+~)Yq>a9-y`4nb;={(4_
z<&6M~zNU@9(G7=W85k=V_zncUXAHQp;n3V0hZo!bd?)|^{(t@d>;HfL|MmaJ`d{{Q
z{{H;V!~aaR@aSm|?<2LRn8kLU;7Whz!eo$ckj{~%z*6QrL*Arp_p6fg^0t-l;&@^o
zzEgE_;5s17ck%Ad-&5;9{9eEB>+T8PpC5d9^Ko;@t8Bx^o;*3n_E>KHvE?!If^!!b
zw&bxk+^X?;G5Lmz)rHi9)`k(RwXKKB7mKhg=L&GuJT%>m<9xM!x$m@#OI!EM-6b$f
zCoI=TN9OL^u(w-HtU^B=3Uv8rrFhC)d-hJ|@Hk5@=JH?7La(RzY@Z_~s{ZYgWo$o7
z?VpevYg13%y8OC#KG)YhkxMvsXTFwdE}gt-(V66pysX}Ly`MAeTH1EGPAl(Tip1li
zS`v>}`L=z%nZx=k-*Lvo8Qh(&I*&ddWlGR}bX|0&yYq?Z(UtjIxzb(P(vv>!`v2*3
zZ>Qo_PBEUaiz?6OUf<uar)$H71jd4}XM69hTzyV>rx^Eqtz(=#qTd6I>#di%PJMs<
zx`d9@_9+VWzT3IGWhYmLXoc?C8qFo$w`_{44cB{S4wb7WA3yc4x}o!N$NV^kNc}@S
zi%!gIy8gNO%ZWAfW8W=e@)q2pG$Zs_T&vvcTZ_12Ln0NX7RDZbsrBVp)ZZ)jUZ@^>
zTyY`eYWTD5F|YG>Z<PML>~cgn-|awszoy6ky8YA6FYXO3e_9?_|LfYR_`b=@|GzuW
zUjO&=!TA4ws^$N^+-?8=!Q}7XIUH>I*ynDGJQmS>P^NNcyy2&pvG-UMIu5QXc&0vI
zfX7Hut@i!-f6M=G`SwMeU-awNlLtT4|C86(v+LSBU8m`QuwnX%i=jFxfx9HGB)B;p
zsr3>R`x0s>!>|7M@ArFuf3oj=?r)#7fB$wH%R5)y;%D-_v)%Le+ieD|{@b7Tzqs{#
z?i2RcuixK`t4o{owEd(==uZ39YnDAUnowG@Okqa1bi#@|o*NddU-y*#<rItC*84eE
z#c{34UGDg5wS!P>m1n$Tm&2noYh=QXx!-?v_FdNAsq>FpvmbR-bZehBxnjEH;Wr;P
zN<=Dd;BC;jcVx0lZRET!!Je_%U96W<BbG$!w$A3M^ZODtZ%x5!{mtL!EUV0%9XHKq
zCQmDmwm_Z!T>B6%#XY)94Q8fS8qL1HcaesH_Y;xMck92}``Wx*p2w)&>Befn!tSCK
zi!`^;;jLfeToANzhR)%LPIcdBE6u!ZA}@KzvMuE(ygHF_Cui8rg9|+bE_AOy)u(;O
z%SO0iiqqE0mvXNb`otG%Pl#N$ct`Vhwy1+?KUYSIOcsu3U2q}c;w6vl)UW=B4>zyc
zTb8}rGt$XFjVDa;FjIojF89k5mb`oCy)=A^>YitZ0t&)en?qALPrp30NuSxZxs>~1
zStXm4q14{=<IW66lJ}<VH>xTAI`yDL!^u_IjCpgfPCfkX%X#PO`g_MdAA1?({e1h{
z@a=MY9{*qb{>S&-SCoGB-ac#m{qFnv_vil|{`~&pq|<SW_gq@|M<HC>MV3+Hg1_7%
zlXJ`p4E?Q7ruqe(mS}zOSbo2mrAC?*Pxa*Qd%Nad)fQ2iW<Alhm$i1|37()G!kOnL
zB(XPmJW9HLG4JMQ_h&y&UY?)EE7^a)^Hu)MSMPbX_b+$7U4D7(+s}@Re_lSmz3$tO
zsp*TW=ik}&{p8xM2l9S;_ibK#>r9!HwspMu<o`uAMd@-&4j!Clb|>ns^~o1?8^wZF
z#(O_go2Mew{V1w#f?C{fu8Y&It9j|4%60WDe-Sq^=xL1K{$2HZYS)x%|G78k{_N}9
zuYR5;aK&@ik?$+EeXc2J-9Iy8>YuyLE6luY5;Yw^Uc2c1+{O3j$}M~Dyf`v<A<If;
z&Z}wD4!y8FzCH3Tzm*U3gH^?W)|%HsQ`oL=w|+9y{jz5MkHV-aa@ErgF=zfZOYXT}
za_>{|!N|&ul5@*k<&PC|vb3JubMk|@Qsy#!=8xSGNA7Eiot-EC@u%~dR-Jh%F|w`#
z?0L_MQj8ultvbsza{}AyKfWiUTZOungwA(q9ZGt#^x}=2-Fq*)tYchI(|FQGQSa<r
z=1gYs>2v3`-p|$ly)q>Cnf-jl?+3MR=y-EB+&R5`ih!bvgom`n`R6H9Eov0|L~}zw
zeQ#}885VQ>+$?2dK80^f_(LAd&?%PLI)!WV?s>PhuL}hQe0q@<s!$ki*&NEx&ahy6
zI_vUKkttJ8uG;vf^y=x(y+?M$XEHPYx?<<I{ra!2+WZ<D#t#;9CjIq=+jc*yNEbaI
zbedy>*bT=j?uL~Ojz9JnoL`&E@bBB&m7l7mcd33kxkJ?FcD%#}`HsKydEEaWmAyP6
z!#t|=)A6cVET2ERZryk#bM_vlyoz^6%+eiXRj*pAzw2`IImWE1u`>Ofz<Cv!8T|Y2
z<dw1?)^72A^KoVGrbBmE8kZ{tIv)(b=p=J4NwGP~ZK0s~GQK~@KQf%S@O+;At!=a9
zYd$Rc{%3EF_{_Dx=Dt0awKOYl#_Oq<_utrl>}A{Gs5;T5TSCvwE{T}W8{U6>?p?R~
zu-Qo)+O)#-=cJU(zY@Fc>Qd&wCqHcVAJ5^<TXU=D%%!|%?CW2y>neVn^IpbWLMJVo
zcTM%X8!7K3m<pEcVbF^fxRi65_k*vq>GcM=1`XTHcju1y`&&c>aT(03m~`%J-{YNm
zr_b<S^PQHMSml07BhG)7Su^+Pl|LDh?}+3bko91Br1?<s^kl~JhT_SUJGW-W9h>2`
z%|>Z)VByt;T^G(@jN8Baux{(`pQYOcSss0Izg?%L)7n^g$^Fu+uE&QZ^!09hO3H63
zkWv50q5a8$Ye#0iKu$T!g{UnxaYvqaw(?(7va5fQDe>+6JfCHoO*bn}I`!{fLYt>l
zTU#PSmBc(Q#aw5_(~FKwHg2fd5MjY;aQM=W+PE)DD%v_ynsYV=ymHmZSKT}<?s4gT
z`K#`2CqrDXFAsL)J;x=lGr6+1yy;Hp=_k`d^8?OjuD1KKeCnFBn}egXeER&<q{>Z>
zg;uY7QI#te_}b)bK(=x0#YhRK%g?1QKbJBQ^IdM?ADq0^baL;~CyS&E6-;;zr);#J
z`?9T1$I-ZV>$H=1Bzrvg76ojb%l9+kVgW~<!V>ukA<flc$3ItpY_qbp)UR3l<(tmD
zRUb>%ZGZi-M9zP+&dGl(GBvMjpIdRa$gs{k*7Qb!{>;n9_kxn*!@fG~`n>Dni&_C6
zzwoC9Rs|{AS7vKBm&`h5d_{t%SCUshvcLCaN|wv#zWQ|=Hp*JA5sl!F@7#8O&tLvC
zPpg}!{(hZv_wR@0@7?_uPj<K8S7Gw+$8vwWe=jHBWAC^7_hNad>z;d6&eu%KZ8lw=
zc}iI!<V#0t>|3XX;LC0dn``Vtj&o#X%<%bo=)#3ZH&)J*_gP-I`|8`8-}Re+zFi;w
zCLsA2kK$KB_6nV4QB1myQcV{`&21!K#&ONkdsNq3s%-fpcFxVY?e>2k?*D!8|DWT(
ze^=kDjL(|=ZS#ZlM;rFpSDMM?)x5Ue{(rW<>9zYS&5{>PZ#nFlpRltz_hQ`ID{h%d
z7Jf2qZI3g2N_FR1oO^Ok{K<;t8+xZ2U+y}7?C0#FYl@GuZmzg2sA(N_@%$OyjXY0e
zXPsp1zO%XO+T<Ot>cjHfBKgk0z3JEf^4gV9-)C%>3?n7ds!do7z8-lH_M$iJ)%LQf
zt2~}9zj&bZe(2T6GwN&%y)W+=zPe*)YiVt7`P^@L?P|H|po=rjc~eDBacn#IJjz7S
zDcNW1^Um<g5h9)u^Ln;uT#BqS(O#g|uxC>2OSj#|$#Ggth8IhE*R0xN;_>c!vh>5_
zlULXsHn864t;;c0Qg8OOS#HKszurq^r?-7EHCq3OK}qu0J0IgK9_cCdqOB)`Y`-m!
z>`ym3b&2O{klxu{X%|Z8GOjqgsB>vHkHZQ>lR3MhGCxJ%m3qADoR}=9_{W*a2NQH3
zeQw_&<DImwPv=?J9zD$oOw+44{508K?uuk<+&)9%T0x++NwW>V#}QH9(>&J#ch7ry
z=WflT!n>I(*Ui52*Xi%%k3MRjyRJXoA0OV=;P7K#aPfl&mli0#;x)NHH!^p|9VIsH
zkLy<WpLo#DykPnpGgV1Tg{kp!v$sYs+;*$H-}>B!ciMGYff?CLI&9-MMxN|D;h1+J
zfG1kjXf^A@pNF>XvS420<`^*bht>j@n7gsxPEGR=VOg@G**AP{U+Q`fJ~8FkQm*$O
zY?I9*e=jL~eRHCe{mt9Id2{M>?z~!M;A%Rdcyj1Ni_ekmOAa@GoP4(;x8TR-&I}pT
z^sD(SS+-#d88+-*pSMqV=lj2VcFi|iEWSnZ@KiUId2|14`KtU}egEyc%WL!)Ia*d1
zsVy^8IN7+@qSB{<@1g3$kmFa{UK)PB9dFOD;!EMnx8aO()iZ7_viSaJ&w)o5Kg;h=
z+s~9Y+gN4uo}P&x7qIp9b=YTCn`Zy?wy*jB+y3|Y|FbqVetaGO;QY~r%AxOUejPfz
z{_g+2dn+Yh8r?YVpg(hdpPzE}Jd563o<h)0o~^;xO=XI1I2~U1K03GO8{?X)_Rk+(
z!|#4L=HF32t6x9C^z_tuth;u-Yzw#%pQ@Ag?#=xA)w4@4?p(M}$#2RUNuM?)?}&GQ
zl~_V%oAa+e^-Pu_y+7Sl)#|_8w6C|iJWNzy@S3dd_3|?~zk#LWMzPk+=D7~XoU1$4
zL!&+Sn0ySn)cN}zkLt%AQ<Mb${VHoe*6C>^X;b-fmtV`}$XnM8&G{~x<QM9HD*TzL
z@Zn+i#FM(KL-JQwI5XIX|0>J&EPOkKdzBYkfv-ZorRgqF5!TMwrBViQ=cL7ybgw?E
z%$v;~G{b38>y6UYx2?H<WeLt#7n54!TJ!J0%97vB4PD<3A5@u;`Q|1AljaJ4KgGfY
z|2>ie%d#}s8&@fLR}{B7X?1Vyj9xp_M|Vr`QjsOPOAU5@eb_Cy^6}~31l{Ff*Y^Bg
zDam1fUNr6^C+j6MNe}0+KXDEbUC%WhrO&syXub8z)UEq&&x)9H<zVEKPA7(q=3AzC
z=^azrw6O3j*XyX`(&yicK3KWr;Hr4(gQwp}PuM*%aEj^s)&<eIc75~ypLug?)y~8`
z-s$_6&0SMpRBSmpYQK~$Lxy=7`(yRC$7{56!v8Lq_UERJ!u;iWQ|_<Nx9ti4AKtrm
zd%JdT$=7cgsh6yzv;UlPv8(%{{{HXm`rrBU|2)|(E-ufgB)Cp8^(_0#m1#zC3$(d}
z@{c+la|+7|$;qkv{d)HOf4`n>|9@fE?#PELKD-HDFsI;1g~iYC`cLcK{q2n;nH`)N
zp7FdokYBy^_{RtB@9Y0JfB)ZH@pN|noxFXYe~DXU_vWlDI)1w*`}INVqIB^CFAx4s
zum5?Xw&=lo_oy>@{M*dqOL&e)uDxQjIdFmY^|+O0)6P7bcJt0H-zh(TB+c&Kbj-cs
z`FD-zs2dRiMT?lv=B&LEX=$8sqNrnvhR*gc%ZeYTF5|h>owGdKx;NxY&T?sw2(MPX
z?YrtDolaN^@%ts;^}N35{rO4FPoC6&o8CUhK1g}{%jId?*Rl##_Ls?ZraPTr%AfdN
zk5j-hoMG1#ue<p+p5msaG4AV%7ajTgG)FaG>(fD}bqBS+EBz5Y<hN5{q3y#ZK87#P
zc_`11%$=(DInK&_X860AkC)F}XFp-O>SQ%%h1OScPn~Q(YUf{45-YS7;{i1kI9|<*
z$<-+=6`52h|1+0sr4`RRQMT0uuK!JT&+<>+(6zxvlA+eJVA3k%`8$m|FTXLH6|-x5
zb<al!o4u<JMtK!I<S+`gu9RbQpK06gCUq~z)$eS|y(31(Wm^CD)@kV;Wo-NQ=lAd5
zU)Qc(d-7%Fqpw%*+zQ#zWIg9tvVYhcv6-d{+Fy4aN=}{qTT_<pqY&F_xu9)ZY`NJ@
ze{GpAQ9eD=Vn)p4)p>lGGy8wN%lW=xR!E3OR;ej(@b=~Vekr`MH3}D;v4(4b-D{t$
z+VeRKe}Apn9JTUnx~#R;jEI`Dg4N+aCoawY`<z{Bb=&#r<z*ISyZ+tll%KoIUS?j#
zd7p^brN3-s{pQ)$eagvP`PKRAGwHR{wtoKgZGq&cr#mhcT6DM0xTVm%TH&mq*2z=}
z{vwW4ri7WYmP-n{f;gofo~eKK_51vJcJ=>i+s<CoKOVOFAwT1fph><_3KKcX6G9)^
z&(TjT_{FxGS56~z>ArmjcjZ4kd|aEy;)V3RpW&jlNq5Wr!_V}!r*B(oae$*C-7I5%
zh{Zhh*{69wr+@oYwm7`L$c%m4%?y9D?Q56yYBns`=&j7MxoZKl?%osr$=)jsqN{f{
zRu@F6Z8>!$bmxr={`xs7hwM%-uABAK{U_(_jyeVHeOuDkRy%IC{rPnMHNT^^|EHa_
z(ct;;_sSbyrQNJ2{wmg8PSKxGxyN#&`I;L%8+2WNtejp^mT)*|2KS1x1kG;?w={l^
zvr^x3v}l@!K;8D*>;A3`aT@~mS)bk#>DOx#Ve+>wrn>Fs{a5i{1C_2fZmpBZN%wZ{
zyzj*Kz@u~OPv+<AL!Zb8p4gD#S0%stx6@Wj1_u@)wa1M<?^iB9A)9rs)c6T#aT0iY
zz;hm>9>Fhv#5{wKvp6U$JvrlXfZ18y?AtGog-=`X>+M9l8*c05^(5B#i|({m7A?2d
z`P=$?$K@@)%hb5`hna-zn;A6Y&Zpo^&A3N5rwRSIv*?JU{@mXyGmF-mJ8b7=Sap^&
z`RBWyt<!w}Z@RVTi*rlX=lS*5<NuwL+qP}PE*l1)_5IWDPE5{^*cg2zp)&7$*{*rk
z`#Lvjbh*Fuo#A>xOU9?D=P097)Yo?jP9Ih`1kByWY4~>4<q7)bM=iaY@)-E7{6eJT
z=Gbj4@dd38>D%<vpyrurl*5FJo|B{Ie`t~Iu@zon`pUQ>Ejc#V!c+eIzDeioG8e7e
zGvnGhf3>wwYA)Q^wcGT2*7N<p9cJ29ZM2{C@bf9{s#Plve<(4T6~-p<&h+Cy{`<+_
zSNA>XyL@vO_kz`IbyB}CB}z=um5^DdGA}3BG<Wsy(??#c3f{YS8J{gf^31oYzg*|t
zzkQ!uNYNrY)<)cao^Afz-gfQYBo2kxy|%Y(HtxH|wS9HX_H8127OyzlZ6ANoMo``^
zE-_<s*JJxw&ueN~`>So{W%H{qwhQrzy<Q-5OLU9>v$6+~y<2u}<hpTOQ_B1EN7ge}
zr`>0swZ(Ia{F3H~wrwJ3uIcc%tTWQ;z7X2CnrlPp(QFfouPJX%TG&@^Dn6MKU{Pse
zp|*RH49kLBqW8CI&+glD{m=5>FIVmqb<yxk*`8_1C?nkP+UvM2*D2W@n}zRv4$jmJ
zKXLbnac%UhGm}#?lw$6u9Z8T~*ng+zRLTd{KOqrZ@3)z5n*RCj%)3!x=Yw;47;M&^
z*>-8#Hsf6t^Q27*xb@u{Kj=Qcbm4#KRL%!G7B4@nShU>s^|7t*Z=Z2`yKKo^3DeU%
zO|lnU*R`KIO<hgQT+%S2X!%on+v#s}XL;)gwRZo=0S`&fIml_A(#MeVWu3i|WOs4N
z(_PgJOqthf7pCo!+|?bccSwtKL)Q<JlC9UC*oJ!+FmCzzW?J;xnKMMaxmEQ3u5Pi}
zwM*mEys+G7yVML`Z(DqifuW{cjN|`7S>6b?Kg;qj2JgGw_a~&`;~_`&k18_LDuV77
z8N6YR`1{g*@%le?uF<}Qd=_=DqPm!$#Bevf(R@0s=dM0Cr*<Dh+@sFW4`SN4PtQM=
zkm|gz=b-njt0Iw3v2(<Eu6;|LC9--`>#aPIcj}*YW$ais<Zqq+E#5+A?~yx_8Iz;t
zKbw<1+u=UnvEsGN2c#r7zSwiL@nQ6r=LYk_v=%)JtJ`TWcY>|{aov-bU%wt)@vCZ8
z&HjCRYq`}gr<;B}c}7Y>psu0ZwDNh<i#QwcwPNSIX3KhrA2@1Yy{J6&>Ca+2nYtTU
z4>g~63TQp=6gaHN5F(pt^s({C(SWc@rJpg9EDU8+UMc4myw3URVtuthJ$A3{rqokc
zpH1H{l$ZT%7t5nnoheN%Z@1rkJ2heb(_Y@=E6*D{FKgO9`^vAj$3C&;n<Sa@6y8>O
z&wX>x{CMn>U8}`fa!bGe5j>EaFuS)wjq&E`Fq?y!H`mPP;#?}e^P3m9k~g>5i&Y}#
zk~$3AE>DPKv6v$HB}V++jTDZQwsk9<`DP|?Hg@%TW=q~HtoraU?85$STv<Y`YxW03
zl}$Kh{Z^n^mD|4jY8XSB=9$*FyPq>K#obs`edXq{^7OlAC+@4fcBoqF*d(rFyh^EL
zPTw96<?Wt5MLQQwSP-#OE?`c7eEwZ~2cIK1OiNr}e`<SvREX)JaKG!%Q};e_`9IsW
z?y&{KQlnf3wd8jfRy|e{W%A2zTnE|<u<&5ik>@;&3T3ejv87f)72Hi#2Q1I|83x}K
zcio?U)sI_lMS#bYx9ja{3@4R~ByBh)Kk?At3Y7;>4z1bRX*{3t!JLVons1dbC^XCL
z+9>AGYa=D(ALZ3>=a}sD*!jl3XEfXu_cppIeZN%v^wYz~7P7Z)ZGQhh=KYm(x94`M
z3iroLB|Y3^n)q>T(Bt$8ORd+L)g22}EM9m-%X7WjPWv!fhVbegS-yUcSt>L4T-YXl
zE@JyK>D9H*&vskSZr*No{&1}8dY|2opK0yi`_ty;n^gyMcJe0#z4nitK4DtAnMXhC
z1TL<P4sZ7`F9<cvcrx|ikM6~bx5nGtt`poY8L~L~epSZ0`^9DZH-728yEVW5`IogW
zdp`Lxtlj)EHk9$ghnoc*4+VJVw*>cPEuXq}!oSLGAuT&%6B)iZ_qQEZwCW1*i&yEd
zDu~r|efs+Fi`#L3_fLIjANKd#i^@~Gt;?^n-+g!P_Nnmr-n!?rKkfgOfAjsn>;FEz
zX9{?2c!9Yp^V1#AE5ANWE?s&p@69EB)xNH}G*hb?yYw1nnDbwLs2*SQ@n87=NBZyo
zzMO1g<0{ef?fK*G+|`yW>esSn_cFXW`C6ZC-u>pCQ2~>VeoWow7Jrf@LO(b%mFF67
zdwE*+w#yt#CU2KrvO{Oqg3W#$J~|E7j+_53D&KJ?$tmHdb?4jLXFjF;`^Rv-lz+;~
zWAD$jN}ZS@p)+}*L->@|{9j@ZrZWngE<e@ny@^AGDKmOr%}Up%F4fw-uT~X*U*lC4
zlNu8mvn}-idxO4ti>*$Tt4NdI{kfvgqRma;@BBVBR!5$p(Y{DVmH+CUABUL*7(Sn_
ziw6x{T7p}K=>^wo|9!Dh-uUiWxzh2%oaGnq*Vmo>*Y(SmJxpn7{GP~j8$u4d28Mn;
z_Ev4)(i4q=D)Tb`_^Hf`U9F_T8>Yi>pfq;Zd9l1uwa-i+9Bg#+S1;XgWb!K66rO28
zs?y)@IWRBEvu3hsh~~2PC|($$rTg^LS?iB1M%%UZtvonb+m1!v*uOg_lr2xb?~#Ik
zmWQB6-P?~2yO%3$<jCB6<;LDWHs>mJ7_V?S>vZu{@Sd9}&i|^?Ng({{yB%3pRoS~P
zdiUu2EuVLvTk7SvsCuh+|IWnat$6wQ?^)Te4OyI?mu*Y$mL*>|4xC>aAbFf=ZI@xs
z@~Xc=QfZGTpVMysxa1nUS?p7G<BY6Yd55ca3_^E)d;7lS{B-T_?|)T#{aE~><Yx`n
zgT*E{xF*#GuibAuYi04{KD%16UB6@g6xBq_GM&3IN$YuXhu8)AqBWm7YhrHiQ4d@8
z^?u3RJl0)7>(^W>e-|#defa^cc;<lbeK!RxUYx!r!FcDh$A-VhvQkTa9jdrj$?Be#
z(3}|bdizH2(wydx!i%5z#BRTLqj$DCgW5EUNV}QWVzRx1S6d%we|50_*4KX<PP<jx
zZM(_wOzWe6>Z}bOCxrHIt(|&sQKg6P*GqYmwm4+Hs#(jt<%^R2-VJm09t8bWc)%j`
z^xZjzrjQk83HiQJr#B~T(3#K3Hg|%WvqEn0jykL5%!P(iymc88#MMviske=aYPFU~
z`($x7ZQblcpBPUE&FJ2f*k}EE+jM5m$xl^}R2r_YUmUFN+QgmZ@?y=IZ1K7lp1@e)
zj7evwzCXw3d)aFF=d*5c8zb8CrUftVoThI4AmZ1HXNUfV1hrjkTzpF|A$*58w;Z?S
z!R{M-!|&dUT*3hwqj}i8wd1^Gl8)am=5o7z5j(f8(%vkt)M&AtcbW9HgCCQ7HIsF(
z&Uj?emniXhbHLWfqv_(#^<r+b)-eVc&P?p|*>LJuxQX6-jguz@!fQ52Z5CTyRdDrZ
z(9(Hr9EUd?%07G3CwXT5R8h@YhTSQxJf#d7yV9;|H|Qzr&h8C2aqc~|<hGpKq~Djj
zLf4=0Fy>KPd}02iLO$*#d76TiuXMIv<7v(m3^}Ks^h;e{j!DAaF6hnDsp0R>nbh9u
z-g?5chvULwfo+0pMptIcbMsida*^@mR$~c=49zqZ_FajU?`GsoI3?(6XWMr2SI3_9
zTBXMG&Uv>*TzA(>ycgT9FaB8MN$1w-XLMEddN%2(s;yV^?mVup7ZzN5?^0HcD|huF
zgGCQ{Uh=Fwc4bxKnX8?$J?>v`1gz3|GWFG+fK?2KJQoMe4PW&+NoC&kRi1Z^{0~jE
z-uYFpMEF#sQK$RywIA=fFh2M8PJBJ>tm#?9ujkgLeE!Daon1bC(J`*w>(7;!|8%~#
z<LL|SwFx@<m+t<&RCF~{E&F@gG-3Vwvm%QsOHb~3s<Hch&%3-?@xM-m<X*X+r}=8u
zy|PtpL2Gj~i)*zezdE=>_}ca6=lsWx9SeBb)3#c(d$Aj@dRmf&|E$%D(<eSY6aMVg
zl37>Ox7L}iT&($eGvBPDb$%KAUv}l}`FSN`cdVft!`@1TRhLgi%~__rGpqEQlEvo4
z_4{<)f1lpSzt)re+{LWwe<v<x&AYGMZoKKxqQ}uPQnN$%c*JbBWoB5*>2O^q<E+M~
z+%U!8+UMWC+_UtoNxf8L*GqqY&nnYq=9aE=T*pe(zB;r`-dfGvdGcc7V;hs1?te2y
z4l2pcN<5NS&XcO-r(Kn8#H-!L+@dU~mT@P`lJ_2y(*f^27yOqVo6)ntQ0(N1pc4D3
zd)BoSxxEX%@O;{$)n=1tuqPRDE8mSYj(irJdR1Ed@ag!oJ!Qu)sn7mw;c+<lZo}j@
zL58ydoD3JIu6{WyE`RdOu<(^`&h|3r1J_IX+482UI5UXmElqTZzw*U0+2)cGLrQn(
zjXcH~j>}yyb}?l5tS@GEknlQea7n;{<3*R;)$p>lReO^pPRLvr+t{%`8g%kR%{+l(
z#g#MjxSOK(tbC`>q^Whuao)F8zqtfIuhZ^4wJYoa^MZs8r*>&eU1z^7IB(&T=2YWp
zdE1V&Gpn!Mbu~&!iJ{a!AopDw-=`0`$Gq;nej1SbY?qM3#fv*L+&K;uie#;6JHv8%
zS^3|ZK)DARb5!RA<c2*l<lq1OuJ!$j3OoPsmD(O!x~wPuv~N-k-*o@olvxhe=C1EV
zo=yokm=IXBA?1F6KIc9C<qxz9U28-ulS;2@3p37mtbXFh)U@{&TMK^}S6c=I=a^>w
zUVBx6!6D?@mC}b6!Pj}E=Vp}%BpF9%*2?&;`@^zmQyN2hc^doIgAX@<^jjT#dGg-)
zKW=}%%e>bSYvyv-H)K3>f$7298z1GQc9-qh5*N$gnEOCj<NLIIjlZtl`}0Ki67S8T
zCmWw_`a0G4{eq=gk9MZds9IoSw7W=7Cd@ZgZppEL>)#IVuHE}CoZ(8k?A|=p#mo$A
ztClbFxi4k*_n-8XSF1K>)!uWAGxBt1X#P0atW14toommDjkU}EZCmD*U_W!dmdEPF
z&kAmE6{-uz&Ro50{>^y7$ksC}A6l7)3QV4EGi_(q<DdrD<KHjsof~Q+Io(soJMz;V
z!<E05?9t%79orPPWFMEuoFm&`M4wJSuJ*ah==KzEj{~LZpKOHe8p`Yc9o&BJM=V2U
zJ6Cy8xuMjp0~3N;cFwepF)x)lS{gjbVAdTaG0WwXe>3>BEh==fTw%98KJ`cfW5CVA
zL%a0Ps_ZLC($YKKnEX&!u=Mhag%4JHTzBoN7Wvx7rM-l^Z!v?);YW=kd~?p6znjTe
zqjyKEd_#xppV%aDi<Gl(PrrWUe+G^vvejiecO25r26z8i@Yuk+u;m1cWUBhAPd`lG
zuYS??Pj<s`_U4ytF{SU$q|aez2${`YsClS%<(uTBnGMq`E(J*}{m)QxJ1~5u*yjn?
zR9-Ki=5^_@c%0o5{g1(%AC5eJ+G_uMb^rEl+b)G)i^-n#ePZj$M@N`TZ|mA@3|O_)
ze5SW(>-E%#o!UaS8QpFzuDTD;DmD}tC|(S$jELx8q;gfO>(+|{TNPHw-><3Jc=rXf
z!R+YceN`EGLc6%0?74e`HNoOs!k;B;j&rH=?zK}3SXv^=@a(WYbHx20Z?E5aSyuF1
zarftI_l_NVeC_tf`?qaAt(_P+zjV#Sj|EcpbB^Wvou3=_W9s+wx7P^0ue_}O{PE-Z
z{x9~7woCqJ)@%v4-&g*7!Tz;p?%RmyhE{(Lm~ia$J$d^Ot!s%pK6ys(wvd~D?Bt)2
zgDbzS`06E8rk+-NEi-hJSZ{G|aH&!2K2syBzgMUJ)SVpqHi2is4_%S3k;e;UoT7>@
z9|@aoy3a>`uI;?9^WAS1tiHP=eExmK$E)LNXY*a2XRq~UeLz#d(W{L?p%12NU6lUx
z?err@qil_(9Dhx=&1G9UL-d~N{jwOYOA}2i*!XkoGIfP(TPJmTDllH3yW3BH?)eQZ
ze-;TBaUAeHcwv3ft6$}^p(^1lt>Hd)OGPT5E|_jvvrI{Imf`e6<>_@=S3fV?7sslQ
zrQR8%ZJp+MZ;|SR2s!gZldtbmU3Ow>sM=?p*7URN-O^3L+TWvfta{bvX=HIS&c3yL
z7SF~B&whC79Tws)+1hbl0CcY3dFfSeAA0HA@CWT!*VFo1q||k1N23C_>&$~%{o*Ru
zOeb0&jhf7SFYEBj4cjJ%#F}%a94=4qRJY{+W3qep@3L!acjmMNe`j8$^d;DH(v3B;
ziB{|va`_o=2Ss!qeAKdz>+ZTKN@o-2WX0cnbNGGj_ks%<c_H%*|0#7cXEG~IP(G)4
zWa;~ib^V2z_AHk+PU(7U!gaIy=kZDUs<unKx3=gzhR^&rS6{e1R4yoH_9gEgw$pcy
zH7?k=JJ)pcr0wec<-)2GZ_i)5AGI%c*R{KyqO;dnt^CStseZA%_L^mHdTNiWf%e;_
zE$cFFY`9l<w(jZGiq?O*azDMoHPblMd8H;@_<i=}PwT0R?}yF1H>>q?T>iUyqxWCq
zR{QQZ`*#r}Ubp4@voCYn^0)q8&vVRAyQaA6-`c*4*8v&*QFEh$t?%9VdoWs?>s4I)
zwsy1iJijw~kLJzHo_+Pj1Cw)d4_Q`qeL5L@`ORtT-(k0N@@Bqde*35*Hh<mg-K#f+
z{${xyR_t$<yE?mX{)I%XZ;D0rfljfmqKaN_o5D-NRZE`!wB2j#T;TP1dX{dv&fXOU
z84NMrD=YHW$4u~6u=8L%#;los`fH<$QI5`h|JVC=#7H~q&-H(j{N%2^zPRGuh#kcX
zPx*Mgc$77jE!J*l@9sTe8=PLO+7b6c>deI6@0nqXYRXQhB->AAW!rR<<EN}b^6Yi>
z*O?ub>N2dD?rhiZn{&-H^tb8Oux&Rl>D4(sO3X4=HWzr!+kB&RwSKnW{tnkaTHs+C
z8$~yv&h+wIY5Inr_Nh)kcZ*>K%dU!kmX`Tx)7a-9OArn0xbyP-qOO3Akrx7wu``Fy
zweHK>wDIW`CeEO{lXg#ie{NUq+C3@v=U$vUJ?>!lDMu;CX&PtlEi7j94yfBulO1NU
z_f6OA*|IW!Hm<+Jrtou*zJH(4Y5%1H29Z80Vht9?Mw1##e4a6_n;%z^cfPn<|Lc)O
ze-gf3&=nQkXHrrar?56D$uKTN+$>_cQ5(aywQ7nC=ih%7N}k=@bnIPC&Boapy%%2!
zn||FU)w#JYBaQRm4D-$-Oa)Ts*&eH3ebFGmd2mhlcRMb-e}$E+rLNuna=zxr)3&c>
zoBGTxZii;x*l=&%{n=Xl*Vatjc+9N6{>S}2-yWU*UjHL{{oWr&|6lkqzn){xpb^{p
z;pP+J#dZ0+KB+AHcrNU*x99KI`)|~Ke&knMm%jIY$=j*F_G&U(?6q6=Q~%!Y6E#1c
zc7L~9v^ChNz@V;3>ii3F(WA{GqJ7<KZ{G_$oRi18`0sZA?Dg;eKAbIHes9l?&7T8z
z+wFJm$T}}C*rm0=z)}0rMlPq-;pbzfu6xe@nK`ycIMH)e<?UBuCzEbx3Y1R`_pq3f
z$`)Y4x#EV7N9#%j55qMt(m(4lWdAX2XzD&ybMIy?gRxGma?kIbS646QcyeJ?(&jmj
z83hkK5e;9$-8~^l<hu9AUlR?lUDCVD&agm#F<&;b1grWJKL31K*SP+746c3r4Dopi
zuV)tTeaxggXO17YdF39RQ&#1gt*>`|(GHO=d>1e8@p4NI!$j}9YDc`qw|2Px;RUCt
zH-_S6vg<m2|8*=VpUG$=<HMwx6u9qs(ecDpy-i0xeNrmh_227X(fTK|eT!B+=4A+u
zl$Ym`nw_h@lBwV>%ZW`DM-skX+WGX#gE+GzXWxH~E9`#HYf{y%dq;BDTgAPV>hqR7
z6_IN@+FknbtIpmX#ouqay|RA({r^enllGA_YfCRpR~O}TVA&KjAwuLy&r>c=i)ou&
z_CEQP{yg=<VGEfno7&%{Cd(A&?wzg@=GpApu}VUJ6*p(p#8l_RE0QK<XFm$hSe;_)
zAaOb;kMT#w+pFqt-R0l)+}r*8<BcrVBEO}rCST6*hzOmz^g8+T!|Cf2*Hoo%zPIu9
z+_P2b!VZ_Vy^{TXy8Iu*f$VSEU(cl-uBy&_DgErQzW&=k_by#p9slZI?XU0Wo_(72
z_bSsI(I0CP_xf6N9=rZ#{kog+d!_6Cn*N^pwfBDA*}BwEog2S=Tg|fS%dZ*xEWX`m
zF8FxQonhHJZ-b2;n_qD<G(|37_Ve;>enx>^w?45y`{5Zrw=I2b^m9|CKQE$nIi3`B
z1vRp6(OOnf_kPyJ7fX+5m0O=aH~-t31v^%>Z1CW#dUqp8D_LgZi&X_-i~S$_h^zXx
zvTHKDkd{<S`#a%}G4n}g&fp#A**7g%|6x<d5o7bGjeE{m3a!&h^RQWYYtNa4nRh<j
zjdavp_~n=5^z)|d><rqc7U;7x%zCz#GcJNle|B0d_pT_{FjtS0?u9$cp84tsN3NF>
zo6;Iqd3esJZ<pPVb``0HeY7;6`J8X<o<u&e)|T!cJdpGg_;jAS!>zK3Dg2_7KXRo{
zJM?My)CK=FVrNuLI`Lnr%U)A7;=!ufdpY7mKc}?Fn3~-_lCbk?)5VRk*LB~$?<w^9
zGxf0l(yZcLox-PE=bU1?Aic%s+0T_*BEM<LoY2`EU=YE|wAg-qq(R8LTWR5+s*4$p
z$<}_m+8Moe>fWXa`t2S%O_5hz)kTjal%9?}^e8ggKxQY~>s>;te{l7tAC}g1x-%g$
zH~Q=$nQRWe1-BxU?xou8i}=dNc*M>B_uN@Oj(sdy{y*cVe!q|Tgkpuw7Rx6lxw<5Y
zhJ=0i*nj+A_2hpCm%R7=#Xk8TBSWb}f80c8y}<W3eop9<Hy3r6)mrn(m4Velxn>&w
z+e_|cKb4=T%#4#4KBjcQ{O1vd0^V>@`xYq{->kpEm-pP*Wt+E7XxFb;mlKxiUfi=D
ztGB-BS|*&G+Bbi}6d9d@tKM^*ynh~KvQ=ike3R$k%cnCR^mQIwRV|ycR4DvHXkLUW
z!wfE_*~}TnEq`ZDmh4WM&vqv=Yop{erib@ydOa(j-ko1R;r#rm%^^9{%qO2*bb|lV
zR)=$oPI%rg5(#F?yc`|CxJPJbm_bBm%;LF?O&@o8@!wyv_*AEd{nXO<lh-wT*_1X1
z9tt(FPrg&0e#PeF(odDwRr%%}Pug$Kap}y(cdEsDNrLsopdAl()f4_qj{CG@mTd^r
z%+iOtmF|oTcUPXSn&|MO&U*L$*>@IAe|f!L-CO&@+Wj#hKLV8|W^Qoc5o0it+A30Z
z_QKCwv33Ico?bb@o}Dyv+M$}+k$!7ehj2^3R`Rg-t~F-(5<A<_uQaITz>C`Jg%Jw+
z+c$jXUwwD`_e)PKFJ!L`O?#=!Ab<Y5XAi5xjz?0RqUme1<~l8(({*q8#|RN~(}<()
zCVn2@FFlcEa41@x!}&-?^%oa|XUDfc7tN%<3N5z2dAfG?EQV+I3K!Tg1(=Fvdvlks
zdN{pG*gI&QfTfk@^as9Kf1SR``AJRo@7efY?xWigMaCIl_A(VceH^Et+Pi|e{?c~_
zpI_`>`{sUTXOe6Y`r0@5yaV^g6HN0MWEvhkmKQ#!EZ1EU{zSQJwZ!VXH+m~4cdT|e
z$A8w>DSY*X+sEJM|7>0qZOnKf*GwrlW@C5TgWHwiGEaEK64xh%i?eUMGw1ixZ{gEd
z{95&0bgpP!72}SE>`mSQ(Hm>8Po4Qi>u8~p`r6bYv0Q)uq=#xwA_9r;tnPh~(O2#~
zA7*~}Z+Cgzf2WUvOMgFJU3uWrQr$p-ke$(1r+V@|qvA9c8U5XD?LEzX@4c0iL-Oyx
zJ#r}g?dfUvrv+;p^(-mgK4;dW*q6NeOEuz3yQ&<XoGsRSI%DM$SHV5X(w7>nS$8G%
zTrPT1KJ#br#zV5je~#2kazr0ds+$TrT&m`qgr~r?SF3_2nlNsB-k-!T>SyT~xaiJ9
z3!RTF=buKFFmGk^JtZNQzuNx)zo~sKM|<x*xz-#e{p1NBgGP?jgz5hnHJ*qU#O^L#
z>AdJqYnC>{t2&3zzu&F7a41+Yai=6lk;RQdn<wvITu%$gjmt^=-~a65=YKE0urj#b
zeR*eJxs}LiXTBIa(H8B3uXVB>D)Y`1E|>N=UVQwz*Y;CqgYvRA=y`LjS7QlGTFdrr
z&4Gxgmz)-@=g{7wQ1$igyOex?vzPu1&wh9=y)^mF@61m!4N<kXWA>Zpm-Wqg`lvGP
z2@B`R8624t<+SIR7`gxO=Q$M3dSh3``^Ae7+cO({yT5qK(}-_HY1|s6(sRDpI{5rz
zZ=90q5O$9<`M+)HgUgFL)+lALK6tdZKSMg>^9jSL`&=)qVrX5y>}6E><?C^*7bNBi
zRc}A*rrVKgrev}8d4I<L3+^UPzqQ^^o2;lZYi8M3@3lYebvt%zCF{R*P0`3QGPgIM
zv-(!q=|Ty?5QYP?4FB$(4ANTb$e}Rd-JP?1lP4Uu+8uu(<mrR`e<pL4Yu#ZyaDiQ*
z+N`L3B2&0%+uobco}PQz_+hEYQ{LU4)}o@jrv)$8aw;lXU!eUl=*h0qO?3+_xgDn6
zTHD@LmD||&IKZ{w(aATvS8fpTYi)V@UC(~1u=di*75Wdio?Yy1?cun!g`w)Of}qg!
zY41Q|?24ex>GN%06*NmRd=XMMKWNn?qV_m7BlOPnzgPHDnkJpNQk3Px6L{(HG;fY>
zxh9r8ji=kMJyFYMn(=<kg*knT#j-V@h%=s#yWG8DPSm5NNAK>hIQVwkVuv3E=NKa9
z?U{6undxD=+{Z)aJ5x1n_D?mwK2<F_weRf<^~HP3{m;wEe~GUV`Vx`*A-?05=OfL$
zb!?CS1a-QuX*j9ioy)b-<c(Tq>0-sX8g^PiEiOvaCO5M-FYW%!w0;UV=fw1_D`v};
z>aaOnEvWvdk@3dvg+<gEkvVFgb&^GW>-T4^|2wJN<EP(Crzd=TCu%ov><HkNZL)Al
z3Y_;YCw%SPa=m-MS0wJWt-R0daJcEkB702^Kfym*A0!JyzT|)E{3u@id$NBI+v~Q;
zZjsrn3StYUHEpRh$uTcGze?SG+3(A1zu({e`(wMc@}w>EmG1aXx$BZC_y5qxb2gIv
z2fwFGX1Hu7of~~NceQ4m&Fib%x35)eT)1h&-F+qpv&4?CizzEG?owi#awYuc{C79}
zlQ(7SCyFmxuXk56CS0giRAF;k#H60sISySe9XnrdiON&htk$3YZnyqjo>xgOd=bS9
z*@CR@IG3%f3Xj)~cr^2bwUxL>>n`hf?yG4>Rpp=5vG21EzG=vyzS2tjbHtAmahaPQ
z$E9tUuzH!DpYWxflkdI$R8*GU;GV{NLRv&DE!^Z{P{c$xmp<;(orjtZD%t;vI8pW5
zOhKu9#|qV$Bk3KWa}(41Ic195{weG*37w?b;Lx=x>2Q$5N(U{Q-3LX_#&l1d9omwA
zCr`|59!s8CQ^UKpjWUcKkBu45)jZq9dDr=*#g&w}xP`nirVS37r85j8-u0ZCej@wI
z{y<UIhK8tzubbBd-CY(Ra&GgN{~2{(&Nn~*U-tj_|I_yMU-JLCrhh+o<G-x=(}rLZ
z4IdZz6GtLMXLBCZHamTGqu!@W1{2s{E`R%T`X+^^d>713BYuczuW|Bl;pwgT*fYW9
z=K0K*4^H1IIDNl=(u3ny#H#MZ%w4<xX|5^XS>H)}FV8uW#`{>4;nRXeYhvE+G45H;
zveUzR-zODC8MoBh>upOs_j4}k>YsM6&UB8+tMAJgGJedy9sLE=DG^NDk`?vR^xdff
z8K-b{ZT025-M-Fl{2#q*y9oc}7ryUn?Doy**PpTW<L&Y#;<?dh#at8*C><5uU#lJ*
zb5hMK(~%*hKk^BCri=J<))uL?F_xREs>7xw#s<gV^kXjY<~?+JRhjw6ifP$ln<i&%
z+ACvnd-CGLCN^f0qDOXy9@iE2HlN%qqP?7*q2plJ5##HsFWv|2Pd_BT)LcM+!UmCl
znb)O0y|((Ol00YKm!!1QD^-imS{x~1eitd)ko8%q_lMBw3o`0cOW#eOzB%}FaHi4K
zpdY42v(09_NPklGZOzXu*2-Gk4i{ZbJdBg}9lDwFgZtFZ%d_HlT^8He;ol8T5IvP?
zUwZs*>+GM){cu*#Y%ZDOio(TGyVkAw{pgdZar%aJJ3~bSuO$dJmu93TX(ny(ko%ot
z;W+P>a?mON)Qfw>EG<q?&zgR(Fm>I*Q*simxAs=Fi(Q$qb8C+fOU^Hb1Wh@H>%R9Z
z|8ig7w&%ZlvDCNuHkaS!F5hFcQDtI;T6gHZMv+$CX=mn7|Dv+^lEEtR&2x{L9{g=8
z>)*I^UBjv(-9)`Q{dAu7GxsR_rv)B5z12AGLYnBN{1x*x<L~YK{p{%6>#O5`o_c)$
z_o<Bc+~@Zkd$GmPO4dBQKYn-d#?sqc-_4s9z9eUIl>HnNjo65c%?v!p{I!&-cZt`9
zOu3?b=;fom{T?&>Vphbd{AQSNn`zyoM3W_&E$b#U+b{%QSG8t~vUWW1YmM>7CpkOi
z-i7?k^If}d?X|}am0LD{Jy=uYXnC+{x)Oud-i{m{k3!Wr8|~8@0v4^wS?&GcYt+Yt
zK=xZUyOWLuI;@`kYV{=MwZUm?CuIK!i9Rxq=hB*ws+qUb4<=|%ee+1=W6Z?o+n%i}
zHk@J_v&Y2Jqj1|UN$%+qI=%N9Pb<i7TzfRzr|#s9DFREfd-a6#E9^G@IIAjNQlxnD
z)7{8;x68Z|?MGvdh*j%*-m^5Fw8B0lr0O(Ng65xP{w#VYbNc4IP;zvtoZ0gE&9zNW
zQ*Q-a6+6MQxXDB30Nb0%hoA2}H&@@_n$kbNji57hdK~BM?D*{v<}u^<wqgcR>(xp=
zdwlM_vv}gaV!@XxFP>@l<pgUq_eV@OjbLqNby&MrpgEN9cp!JAXhWQpKKF($rCfW_
zC#GG8UY<&uHp$boidlF1skEz}Q}!)c<0Yuf{dCq_f9(?X4d3<uh1%Fz=a<{fh-&97
zkCi;pRp45-TQ5}PnMxSrwix@~e4_|g%fx>x&TR<U*|mXZEm!jQ?2s_QFF(%DkxH49
z*!n=kVWP<9wRxY{1lkAh`|*hTtFfx&-TP^ab?Y}g<>#95G}lybTORvt?bI2e8M8Uk
zF1j2p<aiXb`O6Z8%o;~u!5dor0u{Tqb!i9N8D>1XQu<Opm5uj!rg>TBRm1#$o9<c^
zP3te?DgG;Wfwjtk|INnVr*ka#{xX|bxo&TegTK&<sS`C;uI~~%ceFy^FfXMvu*^vN
zhK`h`dE>sIgcm03ZW_iE|2rdemW$~m^UTcK>hA4t9<|JI^__On=E>;~L5FuLf$lii
zeD!S8K_kVE$?IawvY%`0JR;bkStFn!<i8+g|0CY>p|fwUeR?PDZ&9i2vR3!vRbh-1
zDwa(-<gYsM!qJ?TUXMlp3TKI2=B~XO8qK~wAW1QKo6E;Y#uFLIOGNga<;a=ZD0Om!
z&$ElWr|Gkbe$hE2CH4_=TSR)#&JEl`+%p%h=xFt@d1#?~y|i=3r{L{hZcTq_+!gMb
zv|)|oF~%uhed?K;Lt9p#3!lPRq^)y6@7?KpANEYR{OqP}<i5{hvmNas8b5R%^s<RO
z;}GkTwO~$c+}R}t-XH&5VRN~ge!b_}`lI*u{|N8;`Rdo--A9r|FE@v96v`$u`t6MA
z_E_`i{OMP<3#Lni_<6|R_2G*vQ{SU?Ov|lfx6?`X*PAA6*!Q*~-F4Rlqokv^j3*?T
zGUohjzUxwX*;?K_{P@|c{nz4j6RsWUQV+A)sHYYd^H3vs&8FqdpTgQ&y8b*f(f<DX
zh=$!RGyb9rVe6t@_|7qG(PGnF$7PUi!FSfvwP{`H_1jrK@APKBYz}$Hus5Y{7R&Lu
zQJ)=;Ukq<!*!Akl)^LG2R`(<uwGJ&{NLaT`;Z$lftBRITQ*3Eg^xa42{}^25iV6Le
zdf@7ogvi>d>1^v~#6In?*?z?5=*E|<+st=g=a}_2(%`_$%CrszK@XnxKFvFl)7H)2
zcE6)Hzc&5p7lrpu9qB8xj94o<H)L(zBx`d}O7p1qUz?ppH+_Pqh&*cCQ?ozlrs398
zlC0{pjF}QNdkpz*v{o=!YN)Vv>zix1MySZEhedt4;b|l3@q5<EYRgl{V)adx&!$B9
zTv*5a=3{}8#F^liaq-ub#6E5ZZ(KEYn_A8AtC>OJ;WhzJ;hy4mAu<nlZF;ay=P2Wx
zh#jKmnzT~{PHn%KT{!um7K89}u1{KrHk(xc-73OXk@_O+M}jC%SepF|ZR4WD-w&~S
zpB6vLxWY`i>{Om^JWuMxbe$_-82Fb&|GYhG3M+g4n)0lO*mt`vWR_i7JmJM&hfhYw
zPR(DOw{H>mDFIJ?r^x4WbIPu2+Rot2y!fX+yJ&*jsnC`apPO6Qwyj+z@R%o>VclLQ
z`P8_`gD>aVD{+c6UCv#-^X#T?t@%~Y^Q)e(zQ8DO=9gpkw~P65{~vhU|G#+2-F@!G
zm6?87f4R=z`g3-g@t)9!3%m~hT4Nmj?)o`vm3DW&RWHsf-4B0uc-_|YWs8n_uJt#L
z-<<ls+>ih4*}vxd3jW-h`}q1hh7I~!-&|AwrSG3(V|As^d{+C}(3@Q6qrMuaa?Cp@
zTRioq)Z1xHsncXPWISx`cMhK`RNiW1U(tI|%k<T})WZ*^xN+>AI=d!XwWn$IjxGa%
zuQEJ2Wt)sFtDYadIQQ_$Tk4Y+A5Ik$UZ%NY^>fYt(HBJ<4!^Gz-^s@9aQppkCDsQP
zlCvCFZe4NGh*wPYLDl*B)5KLj<;`ZWNWVEdRLiGKli$Mow(I8SCvt@<eOiy*-4^yC
zNNqb;=u!*$&rjX5wtw1{5cXa4>a#0l7yMWH-CS16YUG@ws=da0nU=Ph&dyzz+ZMlY
zJ1=t}S%Tq%(XC3s?jP1iKzo{v*{<rA-%@k%;kka)a><GUvE4G3<x?wywkBHlIw*;~
zJIl5B)7`ay7<RmD4t05Sl~H77tJ3js5w^JKoW9;Sds5j~%q`ZRTYfw0<Vo98)}md9
zq}t6EKdxw<wt9+}07KHleI?4>7XRP%+x<E@|Jzsgm)hBRT#6}r+gu;T79Lxap1!iZ
z<l3?gFL>tKuJx{c&9$U8zW3lR%hm$Xg^ZW-#H-finpK^xbKO(n_AJ66eEZsG5#P4m
zT#>j}H_oPcr&p88r0VVc_t`ga-Hq-_FW;Jxx%SrERq^J0@A>;>w;p1AkQ1&eWpIAl
z?`tO%I6`@{pU>SKyVW6l>*u~W`R!V67b65Jr%N}b9<}(pIxkDIGVHbgnjcq8n$J$1
zpYtJoc78=PFT<2;EE}Hk-}^LwZ<uF&_8<1uo4&sKqWx#F^^r4zk*<fnDekEFcxG!j
zmlwa~+i;nQpDcEEh~5c}Gz<6RyHs&ah~470wfdKutt*X74;6Ap-<lKs>BSb#pu(Ju
z;&1o9^xJ>y?Weo43?I)q#?KR3zRTQZ{pR2Mxppya__+2xgT*CX2C4mzm(5wlp|5M}
zZMw}>`->9O!}A-K+|p!N7xJQNW@2ZL2baIjtHh_fjTg;lG&vi+<-}=M&uFi}FV$&1
zlOwt$bc~K{niF-<BQS5{nW#vwYoZhF7Imt94w)_5_1mH8*voxScCA~wfh)c5@sUHU
zF^96$+t2^M7Q4N}^-d3@i8#?^;_qNfZie&kpXx2nyVzHdv{db|g!{A862G$p-L%Z-
z-qD<Nb7ofjuI#?1$%iyeCb!)wso8gLlXCh(ueT*Wd0*M29!!mDvN)RdC0088i+0S`
zi4~KcMApAv{QA#sd#~nxlV$@kaV6!A7WJLR)6Te<9p2;n@W-rMvIk#!dP?{d&SW`L
z8ve91mG_#dU-yQ<wBWcMwi7RTm>u8r{FZiXsn^z`nJ=Qvj&CxLs_w1Oc~q2Pxx3TJ
zfV&~8fAurBt09Y)Ze4Km-^;h{=O>@dN`9GsY*Y67Id503+gr3DZ|AozQ=^#brDgXD
z--<d#vA+(xxKg0u{+wTDH3bty`?!wh?3Cv|Gwt`<F2~32C*Q>W`&Yl_&yP%Xo@s|S
z?_RubPw~dm>ib#euiamy@+@!Bjo**;_x-3iEkB`Vm3990^CJH)&iyshse{YXJ$3Ub
zftuO9G1W_18y{TR8va7LY01$#?R`uQ0zor(FKP~b!oGfDgoydtd&Sl!;yYQi#PY(V
z6)!2cPFQp(vVPC<Hz8}TW?c<oFv+uhyj|brElb0EF3qDn-cEe=q9v_!4las1<Y9CB
z?!rUd5^={?iG>JouI(xkc(UHmG-kp%p>X3jbBw(os+8(KI2U<qnaIM;PBzoN23@oQ
z9m)B_TA;c0%mLfaQhU=BCz|ivbn?>N8Co%{9SloEShu<zIPL$E?^);$VUHO;g-aP8
z_&qtql`U9*9Xt$mOo1`1UZ7xW(D#%#?|2U~o-dS9TAg7wd1a%^o3!uCl=gqv?)Gz{
z!g~#dt|{{cUT&SKtGdAGQ&E?}ZHBzqT&wKWs@GlY@6S{E&#}O~!)IEkXl%97({9mD
z_N!$t9)2`8ju5H-|0w=%(f|I-Yu{eBUY&O;^R1J^)=CA(&;*Un>Dp!8uefxVI=Qw^
zKW!AA?`z)myl`5MR_P&4b(U`Zy&^fHF5b_sC>Yx}iLr35J<{!S$lu|W>2J{|Exf9d
zD>v}%oW0YsE+|*WbaFlGH;bA3*}pI>DPmv$PHVkg&Hsb1Hk-#;*1bLFp1q#$+P*FM
zTUYn0GO)iED-wDw{_$t^)c-%kzyF(WfA6P!-6g4&?<U@#^G)col;lyHn|hBnxV7D%
z!(_4B`d7uzizn9w>@QQ<SNP|N`4@HReA)eF>33p4!%+LG{#;yrT)tDB-B0_;_V+oP
zFV6aVRneky(_~%=-EAkBg-+=hmA8K4QMCvSTR!!_PWIJ&U-M1Q0ZAK9y>wa?tFd}(
zk+{kB3|_yf%N^J8nTXEt*>N;AHE90KdmnzZTFm%#bM2uvh6DEl%qny{4q8Rub-A}o
z>;Bc9m7!5FCv4<*))@vB1$YG>W{s^1+2tK`eXBLYUd<ipq28Pf`%gH<WhW_Db|08&
zAurA&=Ee4|mDTy(Nxd$&MRi);LGG6V()J&n8z`&9;QesjYcEY1_d*%=4WDBczBaIv
z^PA*pKlR<Bt^&dO<=}Mktw8<e-D!Dqbvcr*Jf6Py)y2(<)pyTaJinZCLxJel4e6R*
z5*;~G3_D)^>5J9gvqq;?^z)Rt3%VCn=RJ!3Wf8%(_~r`xFRGha8RpFK({D2pI^X4|
zu<C3P+v&N9D-X?$pC|{aUygjt$j;WTP1T)XF5{~jP#`*w<xbO)MLLO&NwL*(_O|CQ
znO~nE=Bk)hzw*}HOHB)w%(#Dh+brjTCfUAnH_5|Ey|=?Y-}tRqJ9nRJ)7njfBDq)B
zF0S65rg-Sfn~w)=JbM4duWEkY+k8@YtDp=Iqlw${gBkJ%giU4F)V^H2HT?V9Z}E0N
zp1EhQe^<M8ZQb1&OSf;ybFF#&<L<k^^Y7JFY%KkKUM}bVmv4t3-M`FU|Ksv){{Q#i
z*SuZ6UH;$G=6t)G720>|&ffg%cJ)`<)sQPybGDRcxbE)!bW~iPxnR#`x7GI!{cqn}
zpI!9CV>8Ra#c2$8WciFkng56h%9R@_=c{PXJh^J!%yrY=O<Of{ZREsBGh-zu*4}CV
ztN3?I{%-x>%-=G@Q^K>ByBCG;T4w$F+STCDnMcp0O!P6T?EbLhfbAc)6{{p=-tCz2
z(ynG1SL;D#BZ)I!n<o8GVM#PFpCQRpW?8lG^IZRl(!c*b{{JWT-&wAF7XM^FnR)tM
zPGuf<i|!o!t~lMgCn8~q&Wc&0)xR9)&b#}`tv|;){ffovLbk={Dw!59a+o3~FIL}^
z_wV$MpQlu``|RZ8YyV{YPj7!U*)aI)<b9q`D|*g~yN7>}SaW^-_Iq{zW4!|ecU}5?
zI_SCMWuMJn`Dv^6`aYepefx@O2G<?e`5rm7WZwP_Jn9US-g-v4?Cf(o=b`p=Qf*0L
zq1l7CYaPBHy=HaOWXCn`X$OTiNPUZ$t-|Ta>7ZqJ%OtHV@|*RIuy^Z}K7C1?cQL^I
zV$z+H8>VgSIWO(qu(fA8H>-mDylGN8Y7Ac#*KAy<B_5P=K_H}hX5_q=mP+4UuROZ3
zY^LvF$ItubMK|AnaP#}MO`J@je3g?MqJ*cpI+uA)PvNmV{Z;0~;u>}@d7Y#U5zOkr
zs^V)m%oAPg|BdbIhSJ4lIaxDbE^TFe7S(;BaZ^;9R*An(vuXVCDY8lTGEbJvuieoZ
z_CkC9(@(NSp5?A8VmE~aX1h)}y*$0+f);CZA@@y1Max_B?|MGBe!jna^Y>?4uZHJ;
zUw7}vj=#rh|GqjH|L5Sr`~ObI|9e`_XSd*!Geh9(Yl?adU-nlrAIR9gvF>e|?XHcr
z@BJiVc@J4X^jl%Re9D93|8~*ce1(qk3$O1fPXBxB?a7<wYU%eG%KYtke|-G^^V>=O
z{H1%Z9n^m@n``5{8LMs=ddKXqPTR#Y!{h3`_61uKJl${FtkhOf5tUL@+^^lIU!l$@
z_Ut#$t@&;T!{cw>-<TxreV5I6@|hRfYS%99Jjio($H`we(*jEBf9~D9y!?Cpw{tHJ
zzI>Hgyjp+8EAs-U)aL2)JI;4V*MCvZ-~a#J#dE$=Q65L%x=meNmwQAdMdn!X*1al+
zm4v*%_jBgle7{u7+lp7r^~7e2gmwP}9p;$4kh*i<+SKi9`|S@sZu-${+e&BlOE~Fy
zlsvuga8aGY^<y(6?}`?g>AsJeqI88}*4x*sUq$Hew$p!H?DpgRzq|W?Z~C4n`TtMS
zqI3FsdTodQW$I^dU)K^d|NZyPhxfjG<+|Y%Tl@0^tPP)ww{F|U_NJ$o>4HSRnzlvV
ztYe<ZC4s+g_UIM4o9_Ldvf=FM_OpRj_n!6U-diwJd$soL#|^#b79V@{>e#oFQMUxn
z9#`kjpR<*5jRD)5Wk-)btJ1ZdZ}~JuD>gEgal<CDbsMf4hK1i)CCL~eJF{E#SlATF
zwO2AXUUZc+w`REXd9y|YyWYP~dyl)ntVn*!<4|lG9~m1N8_i#_>5yCI#!0U~U;aF4
zvW10R)x^ik<ZEKMC!afeHM7n@V$1C_Cdby;HA%+K5DW{~KM=+!qRhY%roZeK%WClh
zw@!R%NGU1Yvnwp=pkUWUjna<v($sS+x0V&MEttLj=)o&{_X<Rt1n#z2{IYbLwIi28
zWagT6M^1Vr=eHEEd3Qwm;ENxckLFf9eyNe+-0Sv|Z}z+132RK|^);m|ow&y2MRWB_
zh0fcB26w&Ltv|Y%$KQ_1es$3=x&G;?o9~ZY-t6|z`e@3IRi<v1kF}pium<?FC6(Me
zGcQhigRxcipPKy{eVh00-;wcWi`;$lqmBaGK6U?l^`-FjOY?ojc5lAh{d=}x{pEMr
zYBi-cHznuh+V6?4pQ1W{?!6gyvZ<3d|Nbv8`sZ%dbM>D$<BYB1{vIvgzN|cd^T7;N
zf#~g%3hU2rDSO15((ZYOZNqfFNrmwXK@MKuX(zRoC40vX3%kVCTV>6b&g@yg=HVmF
zM+qf=N<`iqk^XX&@j{}1m{iG%m+cFZ-yP3Nx3T5VKATZxxA5i7KXa6=WfuBuk+hYX
zDf%F3)%ol>qRLk+wDm-8Xx-52k#b(+(A#+Cfz6I}mARi*<bSh!)c@-1->=s$Ye=ro
z={vT3+2^K1-%kDfc`ZAC3tLt4(>Ht7o2$Q<mE9_-efa2+(sgm==3^3y>EhyU^XK>T
zL_OXry6h`gy7+VZzgxbaI@>GYe6eTK=Vt-3)57n+QhDBcts*x?Z&uP;!?>fbSXRig
zJ@4(kYLVyr#j+|b*SCk~Oa|)(?VQrqwS8w?m$kjOmE2QQoSdrr<m;cWpT8b=-n;AG
zh1+K@y6)Q~#+1<$&N$~w@S3*Xk69Z|HH9uaADX;p*37HFI1aqxlD;_CQAOAI(21Be
zYg$*zB$pnV5s|n{bJDSj6gO?(jb^d3t-oq=b9?tyC%;`Hp}6nUsj8c0#b1kMucp@I
zCZ9~ueVp1P^6tsS2^?;J15QpZSmh&QaDR`2XOPhgZ|=oYK1vH-V(RYY$h;*gG>MIS
z+sY03CSKnx&slGoR6e^>J7Ti4^*vo)*`RYWdzAOHJ-_UnoUz@z?_d|R@4R!0^J|Zb
zt8bM1|DwIQ-a?~5+&X%u;mU`3^J|%d&cD$3xJK>yujf-r)8lqsP-Qut|Kw)J;m;Wg
zY}s?)e*1B!;ojF<#rv3#Ocq@K*u`V&HiNz^AGb#Ce)#k4j;+%3;w|?J&aJJo&g0x@
zx8qmrq~60(CEFIR&_6r(!^O2*zOUc*!)@_YJDIt55>Ia)jH)mF`tkp>uv@d|%-wS;
zjNy-*_53+=SE%=2ne4vs&i>nWdxY(>ocGSW_up)Hb@r?+c6TEu-`{()`g622C&Q{@
zmIpZ@bED=<X#VQ+n!9zOM7*Mq_orOB_hD&4^=|tUE=(-@@Qd%D_qUxhl*Ru>9cr)L
zUv%Mx;KlHN>756e3gm)6DumAKs>`kZdhG1k1qU8}Ja*AC_^V2+u%qwy{@>5ec`wk8
zVB>bU$u>om!6=*Y>GoNQzqCAs9QYr1Y<650v@4?H;4Kf6o7X2E)zG}JIrD{a;_B<K
z()+6zCaIj+x=CzViM^)&?d$8e-*NpF?<e26_UmfhiKh%xIZytXdrS6`(Y3<3mN|mW
zqMvTgRlI)cK4X-{6Q%2FPYxRB6|KFt<z&zEXIifEha^lV{at$fg;AW%G=^+ReMw_R
zF^1m(6E`uO=-zpT|C)<Z>O_^NzB6+;=U?r(U@IiEWzn?rZ|V--j-Fw@f+gT}q6#O&
zSxy(Pbf>AmjC4+kMC7{9{N2I)Oftr`p)ca7L{;AL2QO>pMr~7@bbe`xrrbQ6S__V_
zvcB!>m(2||FgzR)G|_U}<|%8^*c59G!&1-MEn0MZe{;;bih{XsgX#?OzrS4j`^4KF
ztr_Y1vH@Cad0R8uFQv3}o!~m%9?YU?bo+6B+nHzO$#dVnn|W<-6gz|JEruQ2j2TW;
zzJ9dgueEyl`o788tJkmll7IKdj#}-vj2>SE_kL5n74Z7)v-14hrzcH+DZD*ge(m0m
zb3fnLKL7K*?)j5<=Q1bpzuUSur|*9M&7Uh*segZYn{(s6c7^?hheAV)JuY;8yZJxs
zoUcfo)`e%D;w%xV+{&NM-S9Z!zSwc~(>2dh;*QPv@pra&?(VDley3YvZ(jWQPFt~I
zZ#!#4413C(mA{WC=g6Jh9?ulZSn+4E{JyWZq}{E}?^nI;{XUN;AtjBge4~n|L(}q@
z*;@=0`HR*%=@rQus3{5V|DG~o<(5Wa!7D31Y_o}pi*EJ0IKQ~)e5vD2x9++v!hQ$i
z8W^*g1CFQPHMqh1>TXgH-{hQp8LxMf4YZ%k{3@#&C|=y!;>YebNhf)VXuhi;H)CKQ
zf4Wok&9+xxJ?b8$=_O0<SAwL1pC3w!$~Il>yk^31RiC{q$}9M@)p_&asS(dKb$jCP
zJW&&q)Omi<L3h3*Ys1MN!$mX7WST?Q%~!j_RVx{`uD3Si#g>3~dEN?VZuJ&z;)_~S
zR#Nw_@PJdFg~z2sIonSE`YE2iAz}5du8WDEZ}m=YNR?*4`OH9<r}F;!GaGxf=5K0v
z?pU>Redf^}3umvlRO@_ZaZz+**z26>yKX#tyHm^HQb=K#@*0P#g6G_>m+L0mRh33B
zG;-d~|2k*7&Drxl*DEtF#{U+061nSK*~x;}Wp{STZvS*}ul<|<{riiHzR%uYvHGsx
zt$<vOliP)BUD84}+~^Bf(EoXs`@yN7rN5j%TT*ko{HHmmf>2+M?4LuXaXP!6Iu=Y3
z^_~9U((Me@8ZGXNeb#*Y7*2bwKh@{9{L-wR30>zFuQy`8{$o9lTEI3>2N4Brh81UK
zJbb(MyW4Kt=CzBP{i?VWf+qPm$Mx+lD%?{1`D1^b$MOsj6$YiSERDG>Hcxc;Bpj83
z)-!*Q)L|&eJn-6ShNQ&f8;gbN-oAS$&@B2%A?fJ}MZSnMi-~7jZpZDv=k;`lK`lf5
zhQiDDA9iHUmnbgnP?=e-_Ws<d?vNSNf9<*5<l=et!an!d?elx({?}+8jym?dOZ7n1
zG4b9r5lXQuR~nffoTZu7B)ND?W}xNv)#~d*;(zmmJzeQg9%??>@L*)+=`z1}4aplc
zTs7vr)DUxd_e8;qJNUX&R(!`yPO*$TXKqJNS7UnP5ECJ+z`8(f()rT~;Ph-)Ya#K@
z{@tI~@-lbU%HCOP`}g1C<6Ac;@-)m_e4=N?mN}ahx4sFp5{<K*C+ua-&sz0*IiF@=
zMcS^_uPU})=J|av>-nnR*)!$0L^Qs+<$1*SrtNX&pX*<*X^BZ+mXgO2Rco_$T72p3
z?RVQU9-S!>d^>06y3<=<Zn<s4m3MsYv%m>!_j?@?s?@f)Tv<_6bju{tujI4oEYD@H
z-t89N<Gkf_>_1<nhbtT={aE!T%CSWJV?@hp28+<$$xF1_uPon`rkl6w%%X<W4bQkZ
z7t6M9%qcP`<Sn?va&hK1j&R1Lisy2sGBW?(EMG3f&=Xi&yu5L;&(e67SjHQ6HD7+$
ze^~F>xzovS>ZzVJQ>Sj;9Oxc3V?uY@tck47C%B!QKI|&qwZ>}AW07R1v(qKYceh_N
zv^e*5jbOE=&2~+v|K5A$=dkVaf2w20%9^l!vs0^1-orBmPA!6$ukm&tu1ZZ*%=Uae
zanp>f;KwU>7n<I_o|*KrZEKHsKPXa9v&>AG*co^t{pHSC3BjpthRaSFs96+mcXj>$
z^QO7_F(s9dOc#&xtCRNavMygViT~QKvwafr8+pPawWo+p^yZ!>Ta=?775#f{qV4^^
z(uNHY(nU*++@40w`{H}_&6U2RQ;nrqC$PNwY!o5#Xz~6z_bR8b?!WWr@6@;Ti`DOQ
zu$%j(%uSels%z558Kv$Qk7awCA5VU{<@N2|y9-*pTzgzM-rAWpTS6|xK+JcFPw1A*
zZA#8MOV2eJ1#;*sdaz90U(^&PIw$<5@UJUY+J%!&8~<|cp1fJO@UPz2S1---g`J;;
zd0V<K|EqNGoa1V)c6Vd`Fa4d17P}}O)KTB76n*j2>&>%1#}xF0Je^=QJ<w)x-GnH&
zU4Ir7-&lV5#f#NjX6%wLZ`(LUXJJ`mhobfsgJ+WBk*g=avpxFoW0-(yqt(mV!6#Qv
z|19nj&H(Ba8*PjhEcaJE;BsWrniA1x=PYgcX84MqTj#qgF6>C`Dy5Po`9Dj2IXBB3
z%W@4rkugQ-N#m5_npby@zb#$ccYfXW_l*yx$XLs;H5`~ch4o^{PBW(+LQZ+P8%6fW
zEHyNCcxJ~c<hUy6HiN>H-ztY!b?=;g%xK!jIT1{2Qfn@D*q#wF3>3~;m9lQCf0(6o
zdGeI|XP-q%>Ff&LJ#qDGwzo-t)1JQx^f#T?m!i!larYOaoc;}m<wb7-BWIP!9Z4vy
zUVL-Aht%C9r=NV;Wv18H9e=B$s#M&qm8uxmba2sB`FZ=c*FCMX<vR4sZ%?rQVNS1J
zj-s`joGgt!yEm?QvA0Gj+S=<tROmya#A*Idewe*Ww2ydea^9zWYw^~mjR^-QEBDI$
zD2OzkwEeNsT~UT6yDdAbxB9u(^7EJeb*g`&ta!b(?NH~b_gAfUKm0lK<u3kTJdv$i
zKPAq&uOYB)?WWBM0sikMzZ8Gyap_RDv99k1F`jGM4|hHK5matpQ@L9-xsvVOnzl<n
zxTeUId<hL>@Nr>ul(}6Uq2g}7Z|XZ+UAOk=oT`n>8SZdSNqFnCwEp+MgT-ZEKf8Il
z++1zJ#lo9Br$O^cWR%1!4~dNhJ-)l*Jer@b_xj#1DO{2|alegDb?rubMXg(FFJ~{^
z@L=+EmHT{78@K+`u)oRs%GfmJQh^$Gp@Na~+`HcVLGDRAw<u(uxD!<yDt79N62lc2
z1@F7*LEa(KTH!@1FQr1}e=g8_{8|D$qQSv>`&nd5iF)Y5JLPP(8#G$pt@;q-Ci%I2
zcCnPs@$iF6(&p;HEAu6ET^`(@a`{k1oL6ag=j(X4`cfV-h6eM?3>S_h{CxLT>}}@#
zsjChpeEc>|_Wi0Y8ttDv18!a0&(g{I;7x(mNv;HQ>pKDSLwu@S(pY>eU0W4Y!;U=b
z;+h+9OFw1tvBP#XA9?R<&fR{lOusg2iiY~Fg{fYQ(=Mx|wyW-+Dtl>?UCXOy*;|Tk
z-8p`FUjFf?$qLW;`1Pl|?|$1Q$$4<clg0c=GE+kpr)iyZ++H`gZ0@QxHb<_%4_fV*
z((cyp^{nUW9HII+-%`(zCGVy)ZK@Pn&3J}$!-}E{+Sv}h%L5xe&q!@O#pM*fC;oQb
zq@3Q(afVhrc2+00u2P?r#n%4pvHrQ(odtgHf>k{k3__<Yp7ZkB<wH-bny#8Xy=7AH
z$RuJ3+u{#yYF%ntt}14(GnfoHmuJ+Sm}wtZ_5LJ7kX+6hrX7-blMdPXNxf?-Rp`_U
z5>kJ1D?l-Eo$t|()hb)o%-`)CyEE*{#M)JN5BT^kz3}PHma?SOh~oG}$<sSbK&_hj
z)@r9NC|52PZf#=pdb-;tlD)z=<juTT{fNd^K8d^vuk8<9``L~sxGLo>yIMRosAYpt
zLcq(4OA8)3UROMxzF}2)#PgU{m+qd^n3{ZDKuUmjPKaF8F^yMYVnq)fI(1IDDd{^V
z^?uWn+w=F_&4(eiLG3%mxBTO`h%3HyD`s`f>P<_xoE5S%oW{?RD1X)JZP0S3<-ChH
z<^Ng;y>`lp{nD1ctoovKoL5rdgvM%4g+;PKE2j&oo;me;h4@1Lc{Qs#)j0AkGg#Go
zKZz-yv{}l{`N-$oDWA#F3z_3)Y>!%-v+<Dn#fusXr>buB=yClziDh%fub`-Nt+yxe
zFBkjV{B--g{QbP%Ntwo1Klz+CzJ6L#LsD5YnIT1QR<=i`!i}G6s|4hDzRTy!&R1`o
zbwVXbsKMy((PgahDlz(pW_c(bGg6-1!gy%gTkZ2U%geeX+6o_vg!9g^Xx+`dtIQ)(
zbYswGl`ki5nxv%3depiHEaiz^=;yRx-nEmdUjpld)OjyS1Z;OS&nebrEsnL?d3%wQ
z7<*IM4fAgGZ`#+5_s1vgVTgUKlf+UvIcibR<R|lYO^W-PZS&ytkKA5R_4b^X_55PR
zv<a6wCYW5WTD7D8b?1!Dh2fQ=RcEs}Wt*S=d$L2jeV5ZYrh^PEy}6eX0=qUye_0tO
z)9~8WgUx29m#llLcY9i&h22CqrShZ=62eXf4m($Ah?{&W``LYIxm|$%*@U_O{3rkY
z@BQuc|L^zj*Z;nF^Ikwqu55mMXwb5wI<1b!b+w=Q+?!j@Y#LH{@NsPJvhZ{H7W2z1
z1X=_d?yoG^dO0S2w|LU&McS_QMd#l0nM_FPREgB<em={2viS7#Di4i}PR?KPA#Uo=
z$(x1M=alD6j@~I8`*vyb>Ob*1N;^;e+ua#?@A9OGbBFH+7S_E@OwFCTzO8cKhF4Z;
z!Hq0y|D1W?y8hvn@OR5~roQn>VO3hjkm7o?Ylqs6<v-Ud?YFA1Ir^}0-4hqJuP)!t
z%GNyGbkKY8mQ02$Cl{TGm(DnIy?ZTw+3BfOL6`RPC-1uN{5{>fz-N+5*ZKoS5rWE}
zL*K=DPV<vrxba#-HCs^f?UK`si?3XI681Un*HMd9(b(TTQ@_3Y>@1?^GV`y>ANQ;c
zX9B)!Sx!|-o0JkU|96Jo<JVH4V(`55A_F7&d6%SjeU4gqk|#FM%-~7Rs>f%x#jIWP
zBHSoqhI?$Q5yPjC69o?bI@^~r`};E9?K_Xk_PV^WdLE(d7`XA_$MS}KPDdCz_SzkH
zvM4$o&z!m8LlBSH(U<>CEPw3(xBT+{Q&E%mRK9i7Uzd7JL-nrU=B9Gi1{X!Uepeli
zy>mr31$IBGoV?2Q&%#gFX1=^-qpw%x9KVEhedI$>dwz;ViEBdlvdv$^#b++*y7cGg
zlQ7n~Q|9r_WBC`fz^z-iPQU*D>CT68d!#&@+Ki8e-dUt<mi%mwf=buv3D-79Xji;i
zeJS|L`}WDQNosqU7$mf^1tJ$NVwe#+DM^C)(Y38-*$TtwZgENbsn32S_V2BWS1+DB
zyZwIs_ph(lu3V&bNQo;zn6vbIH*=WUO=dO+4mW20T^(UZ?y~u*X|+7R5>V&5IKz`Y
z)$GQpqyLUXtMFB}&5UibJvmpjAfP>W>f}$8wl3m(chAc!;#o?BUo)!$pI6J1JF2IA
zXUzO!_vDPPdrqI0n)#ZgJlT@^;9BXt=&MURzwKPN)TAW8=%$0V^r|E&9fOFvkkuTn
zyFY#0?7sKNoSGQv_hN5b^)eTTBrXd&rF_6BV$0-|AMe5i&#ivI?QoYv|L3wP{R-Px
zZP9rDl4tpm1jZd7-Rp~9eypsUaQ*sY`M(7}HqU3usQb@;e3@9$#BB8y;gJg&s(y8?
z(Gd0wz8k;g=%QQZ@pFAoTE|@bF>?X)#qck$*0{^gP~dPqTK;ZrRnh%}I$p1SK8vVp
zJJ<R8@AmGb0;Nl0{C$W0zeGh%wOg%sDZ3<g>Q#5^0<K?gHr(FhYqorYL;EuAtrJx5
zZ27d(OF4JD^m)NU>&uQ!4QtMOxMjwqFI`{W{5$H(|20P0Smx2S4ST;X`F*1(oG0Ma
zm*2Nv{};I<+VJ+Ud*8vTJMZ0fPH?&Y^zvpXzTO(3nS3V3wOjBlo1aIUYxp$()WE7f
zy<@?_%Q&2KuCdfU-Nbfus>FM?uaRB<FRNd9c~h79wx!6@SzWC$7G+%>Z#x<bX5ZF3
z&|diP(nJ+e(WGOUt~_6^pE*$4$a10G;%%z8<Tax>15h=jFTurdH~0ig*A&Br2_}<&
zJz?<bxb&{yZFOI^;@uVUqHL$@Lw>yAYT(E!S>NI-7!<YenzQ+f;CJV1uXWy07CbNG
z%N;)J$lWiG1a7W+aP)CZOqEDXtKfp3z*RHWz4pB>{^DbQUB%1g^W@)k&;S3p-~PwX
zc+*9EiEGYy+zQM3QPULK$ocw$GNUB>R>{uPWp=wh9h|m!qK3{cskjc^RT@jei<H(j
zbausC?vxgYEA&{MbmC?5O0K0Ya!dA|w)t5U+o4;(Z~u0a`<&bEu!}FxOuBeQZeEeS
z)$c1WPo9|mFi1X7kT0RQM)c3~_qR7&J$YdCsB5#-%Jm$vbC37DIH@6E(3N1)(DVB5
zxtUpaD<|c=o!?VfA+ti$admw6!WZ2w#amW&s%-w{nzVtz<Z_OLdx-Q3-OuJh-(3Tg
zdqj&2k3H+!@%mcwv)c!PH$Oi)_pOY2Fzb}-(?6~YQ{cSwbm#p{CV}YJjqi2~J+R1p
zvUzsT)EX|erDsehsd#X^?l>#oW_p?D!I4tQ#chR&mb!`J3r$p4oM2!O6>>|TFSU3L
zU+CU18)J`MSA;~<A-@$jb)ua(UK??7%s=R5^Zu*FjK@)TIIbRD#D8t+?}ZiiNozN|
z+e-RHegBbnaorR4+JpMJt$7E(J<G1$@XYXL-tzdmI}8Dy2ES+3J&9^DlP%P;OmD7I
zl5PH4)Y|evV)xb0Z_EFEo`31g1a6@hM||4n-0^Ll?{O+8F*WMUodXYV8d+veab^&G
z)PClUtA)aYN!x#m?_J2j-siunW9A(8mG{fo;tUt)+wT{7w8P#q>(thFj2l+<W!k-M
zh_Al!IQhTs!?k^R7i6{zEqbbTdu>g2P~ODKh@+xRa?_vH9P;=b?>NV<yfkys1@YLa
zf^P$!J31ZSDaLSZi%JZuf-zG-e|Eb3y~-W`uL*x`*Zvg0%&{V%htK-u6rJ_2l-NIf
z`~Cjk<%O$3rCY}HWxE-SgDZ6WR!&@WCHlxYkDo_wCb-^Ns_OTv%|PtuN3()Qr+jn%
zF5s)2m|YrD!f-`irq#Ghd+z-SZSDr~uX!0)Jl@>2eeRx1re<eDUhR@(>R@)f7h9-s
z$!VvWy~ac_^{DP^7krpcSFg12a5}eSij8Ldg_TAPG7qh5JM3K6n`@r>*YhZ~|D(}9
zv5hPdrz<)&ZriM`@yea|TkcQMDYme(6{#kAKHM{-W3_CA*F100R{U~R$Z7lOm5*LM
zNbddFsHw&F?pvO8QQF(<>Azx*u>`0I@7`;s>sIZp(pa6ggX5>q+WQAQwT?I3*je2E
zY|k8?)9<-mWNsAv->AA#A$5PqQ|U=NR<3DyY?%|gkY&k5<@PNP3d5iD^7;2XeZXgE
z@?}myrS_Zc;nu#|XQX~yI4-%{I^*eVg&j!_>h|BGWiH9M>*z{Zg`7}#&$#H7Azvfl
z#{H)4d0)=rsXwguoJiie<1AbHvfUbK%NL0CPO)72^#Y6D<#WIN-pyFPRcof>f|Lz8
z9?qZ2)K0wQVpw3*+--gEiS)x$+PyEr>JJr#?v{UG$D8bN@5`mE{o*~_<EoQ?e-MnV
z-N|<KGGo(&zTJ0}mNm}EVwMzEDXM>evf+o;q{OSP@3yn3vS}OJBq<&+-*)uqzbAj#
zKFw5p>C@(__cm|6mh!qh!@@Ft6+Y&vJ8$&gSk`hXbo&9RR8LLkb5pb~Pu##%nRLH)
z-Q7u-xejpIaXj6ZUGTDN<@Aj`=|_bUbRQSD&eC}2#WB}n7w^=Wn^+$jh<yloo*i{*
z$DCuWAKd1MvYkGf#HqOTae-mac?JcG8D#;&g)$q`B^_?-Z8#IWBfFg2%#CsSf{mZ*
zwBGH~=`sHC@k|kqS7E2s+tcYEbM=Ml(qCU+zrF0<&j&9LJAQw%=jUJfFOPVws)B^h
zH^wM)F=%|sQ(axo;1Kk1Bg2H~sr_3rw#`;x>vn1}oqv$&K|uHZ?cJueBBFh#4le8Z
zu;-BX#AU0T>?|j^*e&V{ib~&Y^0rxbdqZ>NjP#>=ou^~ozsz2@`EK{U3#M5rfwOMk
zKI*dkcg4~P8z$+gNZ+{R<vg!3=(*d(Iewn3YEwdX9?SfjSh)E5ExFdOb2sd}yzeVN
zL*H@!P{XT#ANwr*f8_<2T=Um^lO0bOUUgB^Oq%*a>C{%qV-u%*y7cTtU#9Z$i=u_o
zQzEV~S|xq?_~?tr+PKxfV@)16TvYWu$Gv3bqKP+@lHF_BUanUZSK<7i^Q?1?Qa0Q6
z=cyL;I@dLB#W$$cDBnMmpyis@Wydm?&D;BX>Yr2KQHON`4U2@FUQJ&7NpoqS(lXAc
z7h?5wLr!#S2y8paCe!Ttu}1p)_jx>bqLpt6zSh}gqf)sy?XFwwY^O8JIbsExUB3sL
z-{@s_)8zE7D!jpH#e2N-V3BGs?;MN0)5D`xKD0(p38~o+nhRR&|4k}z`492^*SoW;
z|8BAg_7X0#VCz10cFoo5`4u|4=Ovc!>$;G(>&KGgdv4u5dAoDD;+%=wxs8qsCFTlh
z#Y%~OdG_U#iv?rxzi2*&pAzew>~@A)R$iO=^4pIahS7dK@e6lnoLl;7&8g^i+3Ppo
z&3tL~S8`&+yy~N>pUTc0Uek3kLpAn9Wbd4WwIP05D~*)XdiOT^ZsSxqvaD*8<duKY
z3E~GHy0>5cbNJW;8*cgkRcF`DlfT{G|KiE{t*fS5e%!>m{Oo$eP0NZ~rmatrU-{eT
zYpX|yVXw=b1Dvq|UuKE5ow;v*<IJ+gq>a}k%APnUOO)2QN$K2p_Hm_*u;I4bk&ZW2
zChpvKC8YHA<U69f#j*l2TtA*UtvHSCkjbQtD!bwivfr&a`|8g0jXlqySw-rG^aJf^
zPmLKI*{kOk$}fF(YpS`invz&jShUKchMz7|g57ua#I>@_s$cEIm%H1`y@zqa!<1W&
zlUf+q;v|z)U0>YVy(q{1Mcx`6V;fziU%|`L)Y`pf8k$L46h4aCek-QCHNByz`gf6(
z&i?6Z7r)!aFTeNm{Qq0_YBkMXFTTjF?xsetS=O@SNn8w>OBn+KAIALMFK)bj)kC?n
zI|ZWk&ThPw$)uG3{b8y47jN!bJ@(Ig%WF<JHF!R~T)OS{<K*v-m%J}V%Wnytp+0}X
zE`jLP()X9;S{8MOaW}-x<vA^R`L}o2lgOv$TlY+FF1NLdduwvIZuSjpbzkNMpERE-
z-dZ&8O#N%_dv{lx1^eH)xBOK9iQT`oR{EXcWJrFObN|PEW{Y1x9PR$G9C&b<t>Nmu
z-1;jY{bfxjUN|XVBxramuxR$>K6xb(r~KYPmU%By)C#6a2EUwXEdO$)WbOt&$=)E*
zg~=12uQ&1wzdxV%Ew9TSrs%8E>b%D~<~(CqvZOI)=Gia0cjpF9J0Ep%(z;laTFqVk
zQ*V{ci&{IC#b9buQJIk$zxpX1(}>d*bvlgKjN*)(4i<&ZN@y*TpSF9EhOJairR>aq
zrkBt7a|<1dO4;xrvCrd((u%1}m70ti_tX~$GCX<mE~@k5slRHL`!>uyl)mAQY@bU`
z;S7cg2RfxE3U8mXvZUwi1nvI{IooEQPuLst>)9naM~1R{KOO#lt?$W?KX~ri$|bY+
zd2u`2h|Sy+%w(v&VVaep*ri7<r;X*)1h+I@-sx_AuH=eUkM?@$jRngVub#U)Bx=TX
zZhPM{#chAy%$r@AwoCS|LjKx4v1?W(e?KTE|NG7)=3E!$(vri^7iE06+9mic=tta#
zxvT9a6iM)j&9OZ5L_u8agmIO(uSEFO!h7$8y_E0i<$w2{S)ae-R%Y0yX%%kA4}V(w
zwS4Jf-&+3D^ADu5HLhvUo_c~yRmk!3-EZHX+|HM?`*qX)-|yoGF8|$sqVJx(Dcke4
z{&~M6{+kN~t8%KyzB{?{U3yzISG(|;McS*?lAKg}IOiW+bh^EbanIhT|4)hvJ+r#S
zSG9bZ`OeL}39Gz%Sf}Jg@184Ky>;64m-6PP*~1clsT8T7ae1xsaaw8ahq)WmR)3u%
z^<b8R@TE=HB-o7iacPJq?>k?8v90QZXQ0GR9hI14pOZN^O#Ap~A%kIHHG`Rz;K|$E
zjz5heg>{VgT$&{p7yM+?uPVnI`%WZ;M(M0h@^bsI=PsK<Np#1jJ-*x2gPxW7-io(e
zd+nRiW4+ym$s4vPKL38Vk6FpexN=>(&~>gvg$U`-j!{4KUo^L}-q%;JuCKZnBER?Y
zp0dRPrrO5A4N5&vH<>YPa;j8&&)KkLhShtvGt<R>O?%y97IJUX+K$=pUSB$_oc}#@
z)~C)RIf1VVwiaLA=yEOeb=JLizu$=GC~aCSo!7K&_I7RtTRXKIPb}2K6K*_CW}kH8
z=EFth1&Z4^4*wMBx}nD8wk2O_P2``D3uT;>11C+~=W&BS%sTvVopt{A!*#doY6~w)
z3)zZ&yS;AFh5vhN)`ptLJC{W)OR_D!-t^APuhd>!<M8StTWzM$pjH`0@$~EYeVga~
zQQ1H9<%jjRj?RB>{{LaxU-_>3OS2B9w@T~IIj)m*YunK)eMgucNJYH-_tE2Szss9%
z#Y^6;PV(ARc>jqk!^=m;%{8Cj9sTF>BzU*kGvV-aGnbb52pHZpiOJ<;IOlw$HAwL1
zy~=HibaXAgo@tzJ+r#|ud-bwb&X9?xU+lTQVqNDE$8{a7wH>{FAv$+AO#8Sx2^^7%
zYF?)nE);Sy5agQn&FlXDm6HT7Z{v>8)Ms_ze|csK^O2c`Q|Iou!onTLxsW64aK*G4
zr;oe$9b^*NXdHd(toU5-VE^ieWjz7mSyw~u#;uGv#j`vs_m=kcJ=4n6gbHmZhD5y&
zP}Z#6zkok%=~5Hlo-2Q^uit*ey*~2KkLK>ng`a=hYq*^KKK;CDOuqt`=Gn?O24S5)
z^?b#Ju0JY2R~Rx)c?-jh_VyhwWChlWeq(l+m@mdK)t~d!j#bW4lm6}6St!cTC^0=_
z_X}A8v#8MHiS`OxT!fbjWZ7+fzRcW8kl~Pj<?aQX-^B#(Wp}n`T;9ODmh)t3$>BP|
zLwy_M-?aa2*!L#uxmN95FV{rXj+IJo1<!1!$%b`k|Kf~^nouEoJR$I$`Qt~2qwg~q
z+^OGl)%vc5mA{?j(Y<0<AN_sy`0e-mOKZM+++F_dt!L1!6B8M_*U8Fv1nel9AX-?y
z$Z2+j&O%W|U8Oln%L;4WRS8~mIjJZ7h0`*7Y0HG3-6|)SJxo}@l)vJ!$~Ti;#paAr
z{*#t$nXk9-<+9u<;>A%Kzw}tRY=u@kwVgD(lBs_>?%58#$FD`f14y!8Rw&nA{TjvF
zFr`)cWaYe<GqgPWo(HSeUXyj0W0K_mInMCls+@^ZT@%@~b8-%6FnGqf-&8hZo0_H0
z+myYuy*i*T>F+tY4J)+Ef_P3>cW%;I@g+kd`Lofj29wPT)?QG4a+L43ar3rM_w)8L
zJvfzdd~Z<CmA}9H|BEqnZv6IZ&XkmI?^C1~zWA{|bEc?DwSUg8f>-(3&#a~1{*YWZ
z({AOGtd{8g=E0WHEqh|ul+BX8t5AK3d!wt3fUc*;@-sWPa5G%b-nu7t%9=~ZwsSuH
z@?*zhX<NHxDzd7-wK_d+ah|gI_QN7yTufRtL3Pe57TpJCyxxZbFKVA%5}vd+qw-e0
zL*(B57RGlsc~<W4t(2U$n0fclS;A3MJtIIH9}ilrSbFqi$alsR)=f<e?WcGiCb!5O
z{_$+`6Y1tEi#zo)H6<53{flK^{4=am&RaD7Sg-vrVJWjc-%}<?K9*vekllP(#!IsL
z>Y3PW%BAx+Olgjv^2lUSucXSOpVt!J7HSr$FAj{b)4Z-G@MLpBQLNGZ-^Nv2b7nvG
z=8LJ^G<PB6lUeODJD)K;atP6h%`ktl-$KAC<y-cw#Fz;Va&oS+h8~%}1a?2%IW=)*
ztEgVGWGbXi%<>d6&REB&a`KH!(NqftBjra43;*yl9GLOJ^PhL_9a%wFS^kwF2J-Ta
zg$zbtidlAXJ=i1Cu)Dp@rr0jxNP?ik;@L(^*%GQ({N$O|ov6d0dR3LHe>&f-Q}s6g
zcn(~idieVK?HRW^Tg&hLe9%1qzi8)1uE5hh3dK79{jHajPIFB1yDYrd#w2c$K!e}z
zD^?qP@)>9B+$GO2BR!ji;k(Go7j2;#jZeJJnVfc-_Q|I}_{(mA8(kjW=_0y5PbPC@
zC+_zZewwZR?PAa6D9fcH3<iRZ6FWn8rdjXp*u|l!+s3{v<euK4y4{mLD*lyJRhrAQ
zI_b`-$X>qW&Rr2_L-V9;y}E_>&z-n7(BX(zklEwy^X}|pSE$&rz4Qxr(GJ^}(wq8!
z1zivo+i~N}jy*d+E$v#jTH&a~?02)48VNG&49nZckrndg_zVwjQMv8s@);Vkr#heB
z{dIZK8*YvDU(?RcOMGt`))1=3$QaRTy;c6|PPY@8eqo19^94M<9^GR9q^I=~zy6xe
zC2uy^U7oZ^Ekfga>8;Wye~qSnTmx?T_esn+#UrTkSWbQBNoBU{k_JfyoJ{FWPo2K^
z*PgtR#JR9~OUoVxfucgqW>F(2<{Am{;=_^N!KrP#|26OIyYqLJz>Lo``X;*_NprK%
ze)Mi>=*P1mj7oyj7ayC>9bfhI_4&B{_mr5DUGMDOS^n`{{lDOkXWz?~n}rlw^|+>0
z&3t+AmdOD%z0+>mN7r}?w$9h?IcN9uW8Tbrn;zP1)H)W<utu(1Lb>#!b->RlSG+ZT
z>I!~ip1R%U(uz4Jd^%SzQaH&Kxl71EaO2JFdwNRgU(U1iD9v!4(0wJ!OteEN&~TdL
z6&B?gQzeBB*%sgV^>^=no}$o(1r28<e$ACU@ALgo{R-v}Hg{xGU5`)x_iBgsr`ow$
z$uG5CHDiCxTw;`<^mzL`m1u^x-~ELZH9L+o%=yCm(KcSJJane8q1eVx*FWz1J+)r-
zzP0kLl3ihTtGhIF(h7QflSM@TPE9(y)->yz&)TA#TNAyQL{CjTzvi-rVZ^H+YYb}5
zYn1GIBJ`@+A8fdJAm%bhi*>v1{nbZf-<M1-4AuW~J#AHtcIfFeRYB&RiF3bgT6w3s
zQ+ZF%dVg?kvJq*hsi?Yj+G=hU%Mv%nQ#W@!w~{!<dh^(krOaWH)6JB=e4OxKXJtjT
zo6Z`}Y1uz`!a97DHWUS~>Yx2`j)v0I#})#2lahKq{o<c?v)3ZQgy~#})NQUKXU_a7
za(nf!YlnBW(^l`WJbnhdW>xNsm%1z4&x=oWU-|#r+Uxfec78l(EnoBL-}$_|yGlPE
z&yV@jeenT%bFlF#v-YR&7bV_)Ic1{gIT`MKmSV1&2fyWPoTD!GkVpT!qwpc$3HH;3
z6Jo#cbuezaeDwsEq+!Z2mo;Z}4zD=F;Fc;7xbl**>XRld#+i2+18)~pbl7pLWyLT?
zT(%JBY;f6dX2yg|p*~YQPP0dIPspB@#S);jc4w-F)~B*bTZB7Z**^%yvsEs8D*3f&
znS9f%hf{J7@v5I*eQ>Mw;cq;#rI&&$Uc5TZHGj8~WLVg(#VYGo9NND{d6_cXvR_Y>
z9&)j*oKUdHNnKdScf*?8&?t%J2bI`PtFcer%_AvnqSW==o%>0%tFp<X!-p)lE^RuV
z8L0J5GEiw%+N`4oee9z?rO$hG@BJ@PL*Kf0pdsFQ@}PQ5od4mnE0*j6#tT2)HCw^G
zkV#<UhGUYYiw^X>b)Td%wXgAf=G4cR!<Q(j&&+)gsb0O;RAY<&ceV|;BR#oOr$+P@
zXh~0)dN%!u#I)B+)1Aa8cHAv~cfO5v;oj1{o{vk<-{<yRz_#bqo7#0BUaszHwOw=T
z@1o7@57z8oF71CU-(3IQmF4$u$o{+cF|%UAOdH*lC<%w9ZMT;%Se0{Bm5WnASZBdA
z%Xu4rPl;LJ(UM~CC49=a<*&ojU_rhUXAerc9}J#;Ir<d8n`6)+P2=RX4MLn3&qzKo
zI>E)M(7fRMG)>p<f@&|;3nm|qSevQR<e4gTu;{Gvv8<-4+`?xzO;}gt@Mlii4SuaJ
zQ$^o|3oToqckxQ?i%W9n&v{Mxvu+d96SY~{uBDs|@5CP;+-m)L{XLs^Yx%#+m)lvs
zSP~q`k-qnt&iZ{G?G5@8zgKVeRxoB&P+q1LF4+(>`%J_3?HS>*Npqisl}>yfb<w0D
z_+(~^>lWdk34yEUq(yy8UeP105p+&TaJNX}!P`BZUCn#GZf!Cyj^}AVoN**oQAGaO
zMH`*j(gJ>4HyY{8oGF=}Bj&d%Rp&5x1<mu0PSNTvQ(2CQj7o2^D$|bdU{F}PrB+>_
zNHEfTf6l+YaKnRDf2&U4;Z2a&^=nXVl2Vw|(kj!s>(8>xx%^XS3wQ3=(|D>dN@D&0
z1?RSAI~*&wUL3hMF>y|p_LE1K{v_B=+%omn`mNr_mVTM~pR3`nK10c?{XThnWZz1C
z%yQmyt0V4a#NIG_)6{qVT5`s^Q#maQbGn3D{rX+c*GbjvzNTkqr*PhX(xT{ug>P1G
z@b27mS3L3UBJ+@+Q?@)*S|j*jk?Nm63IR;ZjC3Y6Jzk;n(1d+YfS&PbEvqdlAxb8R
zdB<4hR(B>y3K*81+qg`yNY;=|?Zo75PVs>iJYt!LSWdB>Pc6Doyk<r#bLcwui;s+U
znz5{%67sZAgX6c7>7HYz3s<+#x0RXuf79oaj0}+ptCqYM4_&GIx3|?>Fi+ktYX8^f
z>+82KnIg#Duw=`;?GLw`D0td6_$*bCnX_U;qp*=w>DCF-oy#VQI!=DO;QCv+ybXRG
zXA|G7dpAWR{md&rFSgU2cRCLmxqbX5#yk1huQ^fAPF`(#BK2>=`9u2ZXH(9aDEF;u
zI<xE+gJ-X?`r#;VU*8SJZ{D{(-+M1*WpVtmVo0lhVOsEWnQx3UXEI$lmvv|AJqE4|
zDc_V+V+0rK8%4Az#_hjX#21m7-mu9o=1EoGL8b{$N-IP9BVJ554NTEm`f%lHRhES5
znm5->*qoQ&I!(tuRqtSy-k)RV&u+1L&V6iY+Peq^1HUsnt2TB1>VKJ={ruPC-{}l5
z9vr_itM_Hh^vey#(X$+7gpvjOw@&BdQ2Md$jn;fC?$zlx{&1|4S@Ny*-0_7<rL#`I
zjul(_Y3@1g;3~@?sf&%eNgGaWcabnq6!m9Oz9V*1&of)fG;Y$|ozEf_Pp95t-zyvZ
z^GfoDmbn4*7*<D4JoekG?96kPyNyeevlULh;cD=HaLh;eSksQHH&)sId6W3@#olx3
z@@_r8Dzmk29^ic1^mm!({+|V!hqm9ZFRw3smL2LXV#B6!y(O6?_WB-MZLfko(*<)+
ztrF(%Wt~3lca@LkAsKatr1eK;xJ=%nGWYhwwPrRGtLu08&UBnqG--8M+MciCTynmm
zML+HtuKQ@*5Tnh$>QTea!URrv?kQ~V%5o~@Hm@`a)?D$rGV#eJu58Kk;AMx;IdThb
zoi?)yyp+Na)O^0ra=}uaWv?DS`n2<z=OK@`@4}fA<C;SS6LxMdTrc`*6Q}<P!Hq(>
z0#j8Fdv3CIth{_>U&0~7h#A5`8M_S_?s;vE7Gg+lYFWn+VyMb^ZL^Tcmx&X1y`Lwe
zFKZY0P;T}4svU7J*ZY3v_m}Tci)2*bct5B7UO?~iDZ)#3Io;a3#lTE)Y0bgi!pU0~
zR_wXm_G!1sd7F(Z`hDLs1h@nTPE}ecz%XsAR_$hv-jy}4jPiE)MAdm31TN#yowjH@
zU(Q4`?dAfeZ!QKBrdJt+v!$NP1kGIO*4r%9I@7Q-A)#s7$BUwg?$Ns}ZhOx1{AAR<
zvF+T^DNTG6Kkrsb@Uoa%5E|ukIms(;QMcort52TIb~oR*wcbciZm;+X&FgFKw;TPP
zxa<Gy9Md0jKDfTiH*scIF2t~G5?6@qYv=#xzXZ9k{a*ggrS(X{MeUEpB@e9~$~M{;
z?&YwR6)pd}W<@9O*F8l%VY@4*oZ9$K{&JPab=TbDjK+|@)9?6NJDXchaoZ$TzcLP8
zXX|%@)pQY?!#|_EkeQN<8A3amPxLHV_dF``<@0^e!Ft!^MLvabe2-VmvJ6+V`aNZe
zaERm+Z^ws!7HRmWXzFprKdj3XciI*2?i1a9I5Ok?#F;Z~u6Mor{ZF|oiuaJW@Q!II
z3{LZSlip2ZN$4rkTdcHV;-{BCb(*#YiJXso7T%`6wUGJCbB+9O^>!5>mh<<^@B9Aw
z|8sx7|EGVRJNC@=%l(s18mT9m>lV&C@HWo!{#4;h0&|Y<HhFlcj5#Dk&f{!K$h}Q-
zxmu&w-rpp!^lj7TcQqR~Fs<Bl>_T7|%Qgii4X5zDeN%*dQd)Z?PIK*C#yI7LR#Fc~
zNYz=cmy1_--(yU*=j8bHsqBBWPTTod%Mw@iwYl@xY!mjoFST`b%wemH8v@(?CT}iz
zpJ`~tbH#-7N~U7lvt;+5HJx4B;{EksoZC0{b-w{k?@yYk$uQ^mmsFEAyY%=bojQIu
zG+4Yc_Q|RldR=!qi$A?*_^|8fZ^yTDwC&0mejUzb*t7TPHD^2Syt##!G_HPRTpc5*
zn7&g_%k^w#VoSo#u#gg-hCiuy&QDj#S(V(+`HW$ib~Vddwf$lDr=|Y6b_6t!;<`A`
zxMZ%Mt-vE0_iy@E*S$o8FU_&?*O6+jk~}QVRo=WU!?WURcCn)wuihhtZWo{b2G_GD
zw7lJ%XUR|$dm;Mz`U!@HJ>0AK)fZp%y72g5Z*!<&?t>#)AMEd!$MF0*mDli3C*k-z
z@$ma`J1@mo|8cgxwfn`tAI3jD_g&r<Rr5JZ|K16nBHy^rT+@zg&HZ94)csNPP=rg;
z%Pro?h4+QI54h&PVB}Lg)aaf%HR4F<4a+N!a_;E!FK>-n@xC}p`%=hM*VZSW4&2DG
zYcyC96*SB4f{dnWPtp8YQ?;x1%)0GaR8#QTv+e2oG}g?YccxxW*t+mk%L2KHvsXKZ
zGu&)fGM2cP@O$p&zvm>c^6-6^7w^BvZC|C&dBsJ|`SV6$-NeQde}X<-i(bF?;ii_x
zS^K`H7zBOST_NO<c`MQ^En)><*`D_<t$ha<eGpCCkg9s_q^<Fsw)D@TnkjP~q`cm_
zDZG6qnDbX*V#G0xNp;JL-pn?&@SC0z;Sz6o=K0n-@gqfn7Y$Eo+j??)8tE{Yo@-$7
z$ZBHTHGBQmzH5v=Q&w{CeX?=dM(`>O@!n3ODUxr#s<j4bpPG8NkU?pM<YI=2*?vqN
z776R_uzmg;Y?!{oM*JO{Z08Lzc}?}N3<U-OCW?nUdDI2D-`bi~g-ueu9x_)XV#ZPN
z?uANiX;M6onFLyv)&x$OU8Z_pX+^4*=?#V{@11`LIhqNaU!}C>NY@T;gV(QaYp;K2
zn`!%p|LBVim2cU~wxuW<8Gml+VU}e-Q#C<JTOn0G&T`>J<>Rq$momqHcz282%6HPz
zO<YsA*PLpfl*e!)`%32QXNMkYX)l_!`c1LQ<(iA}D_hk$j;3iIikg%n5UlCFK2ZDc
ztW=4$bGi?;2lpj&|86vO57k>+ndvQaJ@-PS@tJ*M@sTd!JKfePE}in#IMTj)m#xi?
zXz6=<%y(`6yyUk;`|*WVB96TmMAt5Vbozl`v%YKFQ*)i=6AyRo&_2r(GBGp#NPuwX
zdxM_V&(qrE*DVZwnq5|{d}b*}>blb==DPWj+I~D?lUP2kzW#Fak%WnC-OqGFM2tS2
zI^J_Q>{^gE+t0MhDG?`(jY<lPbZ$vb+2Qr6WtF7IH}(ei^!a6<O_M_c;|h~Bk43qu
zg690^D?howz%g6;gU^O#sh5Ly9bdU+f{D=AJAXtNls?o>JhF7Q!W_X)i{O>^f9Cd!
z9C)?0YSMLMXDyAnr#5PHZeh}wI;1sQMkm0m=4{OBaQ#M)@+S-dCRe^m3+}7id-Qhn
zksQISkfSC-hKDmx92HO5aDr{orS1#97FEq^FEm~6n*?j_ju)~2m$2jij>@-eA0rqv
zVrPii=-!+gxllM%WY<5-b&Dpbt@zHJW4W;SSkZdE=@V?ac{X-PDjABI#;$8S94f{Z
z`zYXPC|{h>@x4ZdQ;XORDI{dB&6($`=<+mLcSm0HhQI7Dbu-t-B+J-yguHEge#=K&
zbUMSFn^P~x2D)kL`uM7dmp47!KTW*+)$INIH+@<!z5MZiON&2|EBLN|Rcl!Iv!chh
z-s6G!pKrW3_8qafwKKA>?Aa+huMJygT-hxm81yRlTjJ_{U6b9kcQGxvt^3<dsH-VF
zRY~>fr;4RJ{9|S^6iDfuvfQ;s{^BK<+C<0JOjo8eik1vJ`zEO@v@Fw6_n7mb@B^pU
z#(JmKi=3Q(hi^^UFfFhXv;wgE*Z~Fkc}yK?7d%>u<}d3Bd?fjdRZ-ka@>q-Tws^02
z%ZYDZUyq*YB^$+5@N#bU+l~7>cN*VR+8NX`F`M~CqD)|E&^qN^;=Av5Om<tx`!XeQ
zbK_JK-rlLL37(TQR9yq4lY?2dT24HBnyof_{*fZ#ygToO8CV!!3iMw2`*8XH{lDvf
zgSM(X`(E!L!OdV;pP86-D6sR?uKsDAYwcK`?wqqCqkPBn_Y#i6ulr5U-mYDpmEZPv
zn}7UGmYCVUtilfm1&f|;)0(}NW0h9m%$N77SlC24gWfx_^SVYbyY%<lT60d_{<|!n
zRqt|w;EGwM1tqhuDl0L~RdzmF{pRwq$onPXW!aCey4iC1&97m4U?)6>x9wT-_4Et6
z`#aXf?Z5YH^Fwa$oBuKoJ>`;b)!v}8CD*+pM*8^OpsfqeC*1tS?-=B9UAO&H6+@7v
zT#7&An|JRRW@ue;N$OP9Tl%HAwcD1(c&gB@B9&=Dsi&Q$E2(JR<$iZflKsr<&?vJr
zA1!sI=0r@C+<WRwf)=aph4vfX?92E1Ojq{#xSIck(Y`eBvWtrf!fSf{T=W8bZ6CWj
z#TlqgQc>LU@S+pLgltt7Wy@E8WDC}PIw5~nxOwZ06Q38qNjk>I;4$-k*{^@`wrr;-
zu&th%#!wz`X4j7i0&yvAK2sUE<Fztp8Jd~j_L;iPeDj?nn|&9>o7`#BI)D9C(vgI<
z9lKMSyxKEd?<`%SIRDSrzx)3d1n@`iuMR!-yubJ%+uvYUiR^XrY;>0u26%rFY_Mzb
z+r9m!!}7bf#><a0ZMe15%j$b*>6%Ysin>`Y9YUF`PkglwIrjIj+*Kg<`AUd~-h#d(
zp9)1Ea4*-h4a=)5W&G_JzgO=N<4w;4VJ<>vibAfr=4%{ctn3Ptx&2b%$0F@>S?e+z
zmoh9ZO<`!@x{`NqudLgC@5|EVzb^kd?Duzi$+O`3j`fN6bEb8yelD`{-?2k0@7})5
z9r4cWiPQtZ!v<n!LmCYxzuKMea(3>T?^Br+SXOu6(RGl}_LC~FWInL+CF2TCpQ$R!
z@#{AxaCW{(dG>Z@;jyU~`C^;*>^(Ze<Eh1}Bm-IY&8rXWnqic6^zrmNQKic^Y<qvE
z?nsMF1H+R07bdH}4o(gG;8<Hb`Ag+BUhh4BHpU&xPX>)E*}2d8X2g&!HS@Zjva#Cr
zMgbo$&%^t;47|AC<uGW^blmm)S8DTy7g?@%dS`5l>hdm2Yc5T(SQutmcwpCF%j=8o
zp5sV8EaRrj@v_c~)n(_h>g^x$Po10oGl;D_&5cK8m%6~SsMEC*SA0?0^JU{MrFTLP
zFCO5$SpCK8-u;OkH~%L6{%~>e{6CdaF~`sT&sEIw|FqIOdV=+owQFaqa$Qb3^@Ho>
z*6%*uJ<<+!PZ<~{CB}BF?Y?)xJ!+kx;;AQ6ktKYqUvPb%Gdb|=79CEWo!|c6xvSJN
zC&Sj{?f#%{xq7B)n-+D-$umcU9=Ej%tL*+J7I)aiVWQ=#zd5%ncl@sQ?Va;bI)0|6
z><ZU+r(TxcN#1?nuEziOpA&f<>@26upF8JN=Gt=3Ek(~8tc7EqH|U?{I(-zh@X_^-
z?X8`j>3v_XKYb!Cv+)vFx5P@1Bkv{^ep7W)*%{U#;CS}y-z8ni4DPP%##&aL%+Gc)
zGA)>PhGp}E)L%!gRW11KAI^7o(v!B(jYi9!YWvRBPOB)>IMHZw{Z!?_+xw?=e@vOK
z{(2gl9{*9Rb#L3ASHE=N>(yll(yz&w{&8yus8D>~SKP<J!MgheuW_)8Py3z%W!KkN
zBlm4pi_jO@cc^yrS&hS{p(cGT(~7xf1Yh?4WMvq!k5kMw%B5kOYtyasd|NK<>|qwI
z*!^nrnJ0BSC*M}Rl+bF^sH?~#VEFB*q|TZ2j)I$$r&cNN+OF67O0DBsRq6Yai|7CO
z`O0na&%g3dlAb5lJ58DG+!z$7JMU<Onqk$Ic7++P`xd+0wlEZ1V?52cW6?`F`Sq(F
zZ@9m08{ZCr=>4~1rhExH_T<!=r5v_FT?~^i&l8@u$T#5K`TyD{UvW;be1Cy^<JF72
z3@zMmKRvm@H|f>n*%|%);<qh#KlqqeKW|#Wq_Rirlzx|bC*N*JJ;qaZU*!IudnXUy
zyZHat>E{0>`~E-wecb*~#`~tb`gi(6Z(dls;+J>&28Nz1f8QQ<_Y_K5wqY^1#f(WR
zoC`t>OPA=sH4d&l6fx6KQT*v@FGh)rWpDHwu4b1?`aKDhop3nf$*lG{Kl#d7Hp{38
zMZbRaV~zctZ~Y6iy<R+<61#!x)9D$PHy>x*{CCOe?Piltv==feTt8Ns!PpV8^Oj0t
zaB9x33FW_UUfz0q`{QdDC-0r8vPUoOpM(D6?S`PXhQ3_azix(=KI>E(R=TBCtYfZU
z!jVxWZJ6G~dSmy!&9B&FE1YXg^P57GX7)BP-bj$W;rO9AEZcpl)U+t+BMb*5@9s!v
zy{B>ZPIKG!7Z;wsDfwTqIrd#o_TPz;pN~rF#Fvz2)?GMxTei?uIFE5wpNq}0XQ1KR
z51;o-T{-;hdws&boqCTy=WI6Ysh;_gu^}*MriMe9!gaMYBi&`^&UDtea|XR>d#)Gg
zA<)(J=8nn7h<(!PQzakmlzUe^@68PFw?CiFcX908P?olfWtB(L`)iZ8a|cBV-fLow
z-opQGs!4>>8o|v^(~TKd>~Y<(R;W%t{r<~SPdhg?@hvDiTX2FS!CrsT?Q*yBvR~p_
z6+5o~*nRof{kdZG^UUSv#@ALK{hK8!bb9&veKFGCj|SZTvwg!)(=#4AzEev|J{6l4
zEYY94x@Egdz~VR2M@rU-Iw~#uRd{K^5;>jPsX6}3xCAdNw(mY8d!|TLQ}4*s&c{ri
z)ebtd;=Lu<b@*IPv@Sg;vs{&-!q(GLXX9NT<EVS)YtHkW+raPO6)fCysnyeZu2yN`
z6}6_#8mSSly5&El{xJhD$8k;8d47gNuj=sLk_YPLd(Op&+MVRwAhx+h#4hQ>v1Lr#
zLirfXyKHrp7}EQzjOOI9+s=xNTzfeo@P?DqQOU)ExeCv$4qehPOA58-Jg&9#?&%&T
zgJWvF7x!B<Z=E5w#wMOK>S76N)XyG|WtZ6e!j9aXTW+`G$Ho6^tBdURer;cT|9|73
z!~YJxKY3Z(=(f&EsV3HAyVSBO-E!kNq6E(@l99BvpJP1D{_PJf!E4uU>M>NQii(7O
z*cl#iCAG*^^HAh-ckbC7>T6deZ)#NF%9{JOtmN=D-DxYGZ?zj=e*41h-n+|_uDDM8
z#yIVGQPsD;ya{<{7Oi_|bvxE2eChhX1^F8j4EVAbmhIB_-sO~-_RY=A+hXs>XBT_(
zmsa~X%l}{U`DDB2F10Tj@3uaQJI#K4YVWTdr}oylCQSSBLcdmg(#lCY+ulFVyBZjs
zoU^ZPuWRR<T_;=vtY6Na(4DqwiqYnEP1ACf<aL6APA#tTJ)pR4!e$GBBOH@7PAy-+
zz_QD~J@9+K=Q-D1dh?IG`sU`h=jkCy^8o*#kf)#T=sL^G96MpU>{3Q$nDCJV#seJp
zPxcf&u+~nEI2~~;22{btnVbvzHF>k}r;LU`@ubzZ_DP?YgfltF82rEHbK#WwMls77
zUPl-mwG$(&jwhrtoXb}@`MU35kz-bQxU0v*k~PgUs)eGiZ}!=_G2iv%$Kz)YRG;Hy
z2tF%&^lP@J;t{8=3lDYGbJPQ+dT-rbEM@m*bFq{~r^}vOyVuVDW0B5(J>NXq`ug_&
zuQmRduU`JpahqjJprUw?*Y$K>hR52OhN@?+pIM9Q{n~Tc#ohdz&-8m$e(VKvuC?Ac
zb-PuxfBMUvjtUGs{-tSYvZWl81t%~~C{OZooV2XVV@>I{*Uu~>tNQyv>!vh=?$4j8
zcQGf$*s(sSvb)Um&GxxBLzfEk&5WHosm!?bI$OJ%hNZ`(Z)Y?#dtJl-PFOv4cFvz?
z?~iU@Z&Pt$shP!=sQ9aX_IDnw+}E-0Pg`Jo-kOfxtt*~J`A%h;q#-a>(DlU4l}|RP
z%Z0{XnEy)Y{8pvARcRKYe=S#a_T4-BPvzw69aeL+7>*>I4G{dPro(qb&VAzlZ6b}a
zVMlU$Tc4M3&+y#jaJ1QO?Nk0U%e>lT))fjbXjn3BMZi0C&qYsf7VN&kd*0vQ|JD9$
zY#&blcnhxKxQ{;i;nTvb?qB$br<yzX!-p`<j14?udd3HpTGyNx<($|OGevD?+pFJO
ztn8R49G5ZM`9MM^XluIG*Q@X1mYTC)@Y-6M>#)u@YRbuj76N{;(i@A|SIo%1x5Vox
zV{_mUQQwdkR=oR5bWYv=tFHZ#@t?MhPV3K84E0-i=j>5BZ&kstK#n!Bq%<a>hI#oz
z$F!ccjp4z$9zCun^^4A#RJmN3xnj-j9E0$XjL7*xx7YP?t`~^f^+HzQmtu`GPuRV8
zla=h=u&;GG*tL`+sxtrgk=WL~QzUefmcJ=^xcq;|%0-z;8$3F<-rDIkA#d*6b!F><
zCN8K}o!uGwdWL=Xn*wti_btt9ruY?yWlxv6YBJ?_N||cv{96x9f-O>`(_D6aJo|9&
z{j*VX+4*<9Juc6ZRizxVt6rv(EmQQ>N6Y68$s6{S$m?*ii!olY=w@=%Tf8lrFZ+Q?
zl0x(YHa{r~MRx0DzcwUG@7h;U-OUo=Vw%go_iA=2gGI(Q7Nuj1CYMWZv<a*g*|4<U
zuCrA6v=+PZ)W>HQEDBn$BIbN9Mt7n4?Dm+54O@SNGelgTr2d+LVWQ;AIZ_py?%8m4
zyxC&3CRHa{?7UG?%rT>9i}XtVz09nwHI`!j+`l|#b-)Id#B&q--(`DF@>#oFUZp0Z
zG}Cfn^_GeVM@A*C38GIA`{b-;o=_c})50*XrcCO4l=&v1<hutX6rHCrI<z%yXTGan
zA?>i>z4&&iUH6^#vpaaP?9yjgpx$u&;t#iJoC_X2mHio&|0Dfy)4YP;2jtD0Ya8r$
ztvA*7`F|>7`CQrB3CkTNOj7sPhOmD)e)hp<De=FB#y^a;PQT~9|JiI;zW9!7(n0@a
z%s<YZP&;AvkAI3QYmZ+0#lCarHR-OkU;h>CzH9S}e|z)6=D&W%OuyXLZ@gTyN$6Vr
zhQhzsE-sE=b@_YqUjJE+u}qBF{z9A!j+GotT`+m#j5#Z-4xUxJZDXiDJ4e{T-{;k;
z-wr+u1}bcmrdq|jteH3K@r>N1&u07+*}Rth*?Y-J9aoKw8SNGwH8SpukUj9|$eqjt
zwY<dlp=oLxFMq$Qy;gF^{vVnQ?9XO=yZm;;?c2qg*C(CaxhDQWW?I&}(_#1K&6wQb
zdp<<=z?CWe{&(NAuZf+K|J!m>z#X}m{gsz$|9pAaex1L5m-Ln0M^#R2l6I)yao6eZ
z%To`hUi2&Yf3SCLX=%;2`<a*4d-ld>EV8+|Zu`2*^g~<IJ`_E_y58`Xd%FMCztMBU
z)%(A>y<9o%$j6N1wP`}T9c?Qr?4|Oy&%VjNlhJ`)G5BR(K)C<wx6cCC-4YadQD=9j
zuldrP&Dzp`Z@qk*l03my*4FRah39WS6xu(38}PmOmd}RpxI;oW&UT-<$08%^A<O9<
zbSI`L|K!$>29wzi#S}X%YuzciVG6Uy<wHlV72dn~%(WoM`^u9@-NX%RHM2UM<nPuU
zTfd)E;dN?x{Lke-8K-?bZ#l#8^qG%Fb)_Df_y3=aXZZ1KueG4U(mgL^Uu^k(?)<%A
zFRp`=o|LVVw|n*V<S|>%NzvtTmp5rIwW+<KZ(LaP=KjBVrVH2wI)6NyTOG1ErCRXS
zN#&Vur|#VO**uKF=Eu*Y<%d39^L!$fGdJ+*RIvq|+xJxO{-HjrK;rb;%ED-tj6YBB
zYM=e`e8+Wh>sR_xuCM!Uv;Xfduiw7)YyGX0Oq&bSm+rV>^>4SfcK!d$4-(6o4OWPU
zM>8F8%zgaoi%Cw-^T*2Hzb)8SXv;BQ<mXp~5F7DbHG50^*_yUrG5)-a`C$Il&+U^Z
zOk3Fc_qu+OKnCxY{N)|y6MkK0(AxcW|Jmd2UraM^)s{2dHMnt>yTW!>%$N4tz4aSU
zudDz6;^Xe4hbK?m^?&Y!oz48eqiZJWKRw=kK&Sjstz*Qq7q{*6*j!)po&EFfTE<6(
z195g!*T?*Pm&foReyaSFXXj?Gp0R&Iytdgrem}+-+r1VyBKDSi*KqYPpOC%Xd_}t7
z9Fs>2Zhf-!oD?&ALVDqWznhN4NSAM3lxDZBGt6yo$sPgELz7e(C!O&qn%W&PNpj|v
zn^x->Ph5-t`)sdu^!Aqyo#(5!O#gUW%9|})a=!kuQw|0UtLHw`moqbZd|Ha-wYW)K
zNmYl}=I5s#e2T0TzU}WkkwM$|gsJ=y*TT~b{*7Nhrtv&DH(TS2LPbP-d)>NiEC*Tt
ziyrv?uJ(V%f4hda+2#KJY~j&M@4ZM^u&?&{tlM7>I_yk$KA8}?iSg0Y_qm@hPiFjJ
zXsftx+mWl+UWo4y=8=o(TK8b#x>H}e9rWZC?9BTv%+Yl|-M#j!Ucra={L=Tizso;Z
zQeRrT<>3DR_FJ0zxz}C`-qu)>y!mT^(7jd9Z~iJPDJywi#rv{c(4P0OJwt)}ytb3a
zRx>nMbI;}}YhDz+Bb;ISzOVL9#c}n=YU+QUN}s>^oOk%AuJ7NbroV}P`gncS<-ac%
zTi>*5IQ;TQ{8g#+Uz`meQW8F@UJTz;_jp>?T}vr@-YX_)Z+rb5X7Jp*xr5_lt;Hwp
z<c}p)VHc9lJBI7`?UmV8RJ?Du>8*Wb<#XfrGhev4z`|CNVaF}2S8WHkM>jpVzWsfS
z^@?1FTD5&2*{-DjeYkCh(N4L%xLjFTo)5Ro@7G-V|KoP~y3)fBtaq(9PiDBk%Ua{f
zx3dMs$(vap{PJXXxM#I1Z=;>-ww;U)-+ybUz1z3YxBH3zIgJ@j7P^ZTpZL&vZ<!3I
zOO!_tw~FV76$UORgO(qvJjOk3w)pmUYtsMP-q!SBs?lOrk;oI5I3Qsfxa{B2Ta)x2
zZ`YZ|bImAD<;usn|9|KIKmP02v*_>dQVxW8J$R^Yf4}<szx179=C^VucB~2Dm$O;m
z_|SY`{dcY1KNIrb7v}qGZq5i~J|A=7dxM(osqOFf34K!Wj5snul5xeE6Dt>OOcOA+
zxo2Sg`sDp``@N-?`kHw*UDZ|F?I`PDIgw=!C&Mj`a}f$I3qurK7WybiG<>(WzVkMR
zkwZk==YFkLz@z4S%Of(r{rUCv+V*$*CaJvLVz76Az0nOhrW<eXFkg88*!}(AtKRZ4
z{~OC!e!V#L;<?LHoxS+nl=5Uvk529Uz3)|KY|P9R%fBA{Ho?f~%k`U#@2A8snYzEz
z{9aFfykz>`uj>6bPG0WF%wkh=$y|QVZ`&Pq&q=Hf9hEKaQL5ix9kse`Ykc>V_TQ89
z>pEAg`4GNt?WTs`6+i8)_k0QVy6Jc08H+>OgVI9>>z==vlKAzP=h|(tPu7e5y!!Cp
zn)+gs`;#_B`@EPQ<p26fm*@V+o6Bx+Zm5gZxw2mS-;1yH!TwL)-Y(DEBDwPWnr)hA
zOO1`0e=cZODX``0^qIBl_S<c5zV)rNd+gUP$n5f3#q(PGi>>cIUd{g6z5TUQ|GPUc
zC)fpbIILu1=+P_T$;joo8^M)2?}^dW<nqfq%Bq?D9!YYzPh(zSc_iF&YxRjp85PCr
z{BmC&Ijp&pve=#Nf}*&?E{FJqYLiwPO*^tkaPbPxU8Wy9Sk3e9y?&N0J@>^s?e*(a
z-l{fl<q-!J6Z%3-6$-ys-`^+5xL{uGOu?C&Ii>T~eN?HB(3539X|*oX_-vOugHqd-
z&j&v(K9pcNJ<ut<;e=PpvgKS0$~GLBaJ=E)mcI|a-E6beyrksSZP<`!*g4s)$BpaA
zgIhBuq+gIYV#BtjQ!!QPl)B>_6N}_#iA@t2isuJybji=Fk4bTBX+62MW>?XUrro@A
z1R9t3)Vy3&^<7}ENYq4`+=j%KABGM~gPY$zT)<Ep&v0WN4_88U(7A=%<BseV%g^^s
z3G!c3^Zj{N$=1rZ$DZ8IU)6u_prFGu2epZp?qqD<GQ+QQ|1rr44t|dp9Q)B_cG=EF
zdes)ixo_{w>^hYCw54w2y*Zg-%U|A-(u=B%v)a7rkeB3xq!6(g&fypRmTp&gtQ|G)
zV@mAlb*7VjJg)k&c`X0h?!H|;yFC1QS(sp*`s+=b_o{zA{Mf<Q^YYxHGObEx_2itO
zxi1r{e{7lWcDu0UQ+ds_V4;Ppv-c{l+y3v2U%luBQHIoCUzN`HoH{dimcuHQn>*6g
z&h>h6PwsU2-d8Ct*lgJ468P1ntonFz4pSzx`pla`OaZC$=bR}>77>k%ed9kVSHf1<
zTX5H;ptcwVKW}k8O=rENgGrh}Xa1OO$?RM(ZRPoj-BTPx|9o4bbNl@9rH{B=dXGEG
zeacytbY|AfV}%D!y}k6ip_NBS%h+>?Z}0S}7lTu?%9}jc7`|OP_*O#v=~c~tnRP9x
ze>V55cbR(1Xx}fzcD2m-3ngbJNXC3=6VyI0ulac8q)f$?aaK>H8m8Ti3GU|(tux4E
zIBzUfDxvBbv5rwg`>@vgZtar_+n&^ZQMoyJIm4p`r?_@B>g>FGVS|OBB*V%y)_ctr
zVLM}|U7l^Z-sz0_8NsHjE;HLCR{l=T3DTc6%Oh^rihEyO52mucTeoU{T(tY<UrdYE
zYZboqVc1n~V<IoTnCYwMkBimc|9?ANcr80_cj2B|)4MlL_MF_h>tfoI5A#G%Y+dl`
zP@dexgK3NUe+j%gS(_EJBR%Q_-^qiG6T0Fve<(ft5X|Z!`1XrT`B#0REvs+&$nH|=
zI_E30id!Z4?)<Fezw2CNGBo|K=a*f3w{Bb6y7r4#H(1Kf?3b*Tc;G3d!6eqx$>#TH
z$;AVmzZGMS9@{+YNx%im2hMNnxz^qj=Z^Ch@%Wgl%NbKBAA2R`X2B)%5>v^G%Pxg~
zk`47b6s2|H;i9+}h9`k{Pc8IWw^M*IXZnE{cTcAv%k+ImAIL^5`<1fkX-3ig{C<^4
z6TUCXNl&f_EL@{^CQ~xqHp}@2pXE~L;~|oh5`CAR@Coz|k?!)VTYKeFc8O$N%~HJw
zsWqw2j1&G|^;^!TVA>%OX8rig6y-M-7iK!H>owHk6VlqQ#w|JFIq%&!&s^U{T-dpL
znF&Li;Ub3!&gpCuvP~zg4d7rfnltDAfww8l2lU_DPwrS+Z8){YJCxzhEfemFe}e0{
zO&Rw0w|8%n-D-IG(k96(>=TwP_xcyLYnIL}lYs2>=cfansphIpnO1ndNbByZGYdJ>
z9bMv!zdt(oA>Cf~oqgK<U(KJRzdnAu`ZsHN=LQW8?f1)e?Voq9>%4HSi)^XIYSmvH
z7x{i#2&{FT``b?cOjJ+$#~_x@8jteKm|e_IW=Abrz2$bhf;pG&c@18#TuH_hHAN<U
zZ3Yd^rBfyuC26fOj9KY!o?+elVCAnD#tf@wIGqaVEB#+#zT)!Bx63oM`F8o8-7B%~
ztnuEzT?P?)b8HXJGE14wq@e3_<kZK`ptQ|r&4g@^J@dTZv0$6hyM5o6e5n)<oA0t$
zxF_&?X{ENuHuae&FE>4zefi80296@ZUkA@D;hK?Y!sxV4t>0zNslpSI$x7dDJ=ozN
zvd=qhf=c6AX)W{lA|Ew+A4HxA6xQLJv~p5|Qo+lu(=WW<ajhNHg|EJO!)8nD{OKEc
z5;!+Z`<SHo>0#!jeF4v9jM-K{yx<m9@Thc>iYNDE$&8z4eO4Cq&rpkb8uvH!a&Y?@
z&X*UJT$nmK9^Tm$+S1}?!#qKoRpI38?Ea1F{^64v=co6aR!cf{l`HWaUwgwW#m?3D
zANSaKaPsNwSUr8G_p<D!xnE9iy*O!;z>lQbN^QP~yxmvgE=Y##dV2Tp>^>D$x4%F4
z$*hf;I+cBC(V3eKn;lp1H6NZoGydOnlO1~de(rs|b}NsPsg3r7JG@zYaus5;x8=!A
ze67Yg_1h)J^sPI?YM-d3CJG!~6ZvU#;KlBi;+gCYWecXeEp;}2%c*<*aqOa%C;X>=
z6s_P1letoqYoxrHX*s`YD%<I)o1&Y||L^{F_P^k>dl!%Me>(Z_;_?1X=l!Ici<iBc
z!z(9}<oDlP{#@?-Ixi!m9VT<Wboq6z=V&$!`p#?i$L!1$rII)KCua6nY{@h&aH?G@
zWl{OJ>U8>sgsU$K4@6xQou%CWN`v>>PXoJRRikSkRKM)K&M2|?Pus$?TWkVMj(M+j
zu{FN)?RMZ6v;C(Nm%1ySDmql;oSAs3sb;4LugFpp!NVobI5ZgR+G?fJ_G~(O>0VW$
zYuf1uHt;yLkgzJ_1kNxvkIW37>1UU`&aw*rG0k{!Wb^)@&s$C;9PE4l_gnpzPqRF(
zmF)DJ<g(_4hNn={1_puV(oYK>a?HDXuH)U>jb>L4@mwegFU>u1s>f|YO51PSb3x}~
z+>OtCXP@(S)x+RaGa=sTUp1CV$FIqG(Lej(<|u(*b3<$7FWy}crxv}n(RRa^m~#K}
z`KNC2zT)Q+RPXihymzl_Yobq#|K^v6uRd7ZeA)kf|K@o=o_)VkHK{CY>6G6;?^T{T
zm+WR~^lHUqG4&&9uKj%s9{c9LSR5oYPgbb&X^Zdn$6`9CH2<dr_W3*RwRmzV^LyDH
zrENL-vOD^%Rl*-`zq9Q{aP`XrFD}MeO{keIAlm5qOQ$eA$d5<G*6#8lfnBE**mmdl
z?Jws)pkM!m|8Tqfv+C-XhcCW3`0~eG8_RtMZz@cEY~2{uvh&~j7j<W!r*J-0Wq&#4
zN0&jwmA(t2F9NMso;hR589b@4K`^-XRS#c6&)M(uc;>jTJN?u(=Nzla&3OyH<hmc(
zaa$~Q?U97Qh}Eo)41IbsS(oOlypq1*(Go$!ShF03&}n=5FU&po+Q#vZ7=Pu1l7zwo
zj~X;~P5Qfv({1~{GXiJ4l$Lc~J$)_ScI{k-2oG1i$Jw%=MauEA%*~hM*cKFP&-9w9
z;^XDqZxVVYW}a*(YlHgkt*p*Zj(!l;%JcJ8X_%$;_}%7tHcDrG3_~>;k~UmAcVMlG
z?kU4{-zPlYx6@d0)r9T|*-{g&x0GxB%nMMLWWRap@ar#s?oD=7NlM!ALitKZ0L#|o
z&#hXnf7)LxmD15s3EF+MWr5a5)s}THw3mNAen+;Q%}+x3%Qr_wB~{~zIol$IR^-2}
zHTiSX|9E76N&53YVSoQ!o4Wk_Iex2teye`tbt_kIY@E{7KF3#-Z!yO-mPs|g&7>#D
zoeqrDlXt(+_3JHzLPCjk-cHNZcq#sz($8CuhAfDd6Eoh)DpR*8H`GC5@6x@hE;C*%
z&XZeuw8UKJvu~!Ms@gG!ph;7zmZuzBB5kdme^2`E&B_q&_TT;cZ`<emY5#R~zRJ)4
z${GJsUSIQ^y|Ql2**p9fBN+wMdFLNrWe}m~T9F!}E4ZK_balKSn|IBZ=?ostrdOZM
zQ8yJlknQW-;lkl2o-@b9@7FQIh^bp$dr~5HsU=-+dw4Q^xyqJPTa~JKiaG6s7G~G0
z#jaxUES9ZU#`84O|1Qs6)^~rBYVvNc3t;A`oc8*J=_^lesU>HBo%NrpQX*=2h2?b5
z8K0Hj-RJCbWdoP4t=ieVr|15uA6_vBPXFj)H;j;;yW#=|E3=D+#DmRG1G-Ms{O{2U
zUA@n}xtpuxL-f;#4Ab9JiaA&wb+DRd9WIZ(H{0w{TzA-}+b^fTnPO(fy>jk8yYhLT
z7PM=e=lOm&&rE1;ky~GDiZ*}$!A*B26~xcH)zA`Ca_RE<N{jiDqHL>`4m71Msi^Lj
zd%kC{>k-bqRvTCw=6yL*TH4K-pYQ4=)VcL$+Ds3POb>hCj~|{q(yv=%SGD|F&DxLo
z*AM^Cdi!$qs`}{Dn;9ogiENy4Q8<+;Vbz7R%1t`e%UsuQzuC~CB=`QCVvMk}Q;5~u
zAi+iFwFG}DraWtj5naJDTkOx#!^t<^*@j&5DEe*3l6z?H(p<&`e_uLZSXcI$!{JPd
z%#}GWxEdN&T<u+>W+a}u`Srd3zQ1Yb9csV$eRw=SM^^SplxpLwRVJR-^HP@P^>_F)
z?bx&T>f7EI;VV8woV#fnyz=$p?HM;8T(C*y+#kdfcH~A#*XJ2)mxg`uEjXnlx76pz
zg07W|8~uMBQ~l)kt4%6P@HUIW^~TEyzSf}C7EWSSa<<13tBr3zTgbLe;wW!R$^RRS
zn(XJ&OlG`Qd>GX7L8`JfCW5*A)#S%5kzXnw_AwiN6lC5wJ9}jz&$WWx45tEL7)^T!
zUh2F^F-w}!eH~B3GP7Ea`>7!uVul`)%uOC^R#t7;Ar#;I^!vr)wM7-x+yN!ir(9xK
z=R1S5l3@wxCd7Fv^4E`@d$RI&_VU(FW<&q@i5If2eZFc|7Ml0R_s5zw)t^ivK1k@?
zn4_>@#dE2>&!SE`UD?;L`@!a20!$eXj<|<!a4t?32~=b0Ps**R)}H#<N2Nezn$DeK
zRX?d+^?w!!Y~fk?WA{;&pwz4CfsqFvp7(S6FY$I(Rf%%t*R<O5?+@~Lx5eq`J_-4}
zWy0&sdI|sQm+j=r7+w1NWG?MG%Qh*0wfgtl0bjTh^kpx!G~2bT&RE9m-1o6aB!u(g
zro%aZL+6&Q-+p{UMvs?N{$@tSwedDK`_7(nc0bTv5GuWDwtJ@?*VDQvpJcVmNwa3D
ze9Z`FUn}?BcU|cAy>s*4`0V?hzv=9+X7<=kJ6|nXes+tF=#t0wr**0gwu|g6X4AKf
zUT4$AAW-{tj*enr*!uYVpIev4C*Aq6*q-sj4|n?%i?cl^eH>4DXma=O6JlU+<Pl3}
z4)rm=-LoV2(ME<Vu6~P`b+j_H6^a<%?OGS0e<+H7=E>z=&#I=$vpg^rnp-O9TDkq=
zhhr)wH+M0HMooEaHQ(#Z)`jOT7apid+%23Y*y$?0N%vv;tW_PEnyL<bMi%{%X(tLr
z&Mr>aQ2XuFv}w-9lm4pqZ{_I%P5-awQEeA;xaP1hMAwrc#ck~aX7J*|j(tU{W;Qof
zCtClBzPD%3-IsxPSRV4s&JdWXXc+dh&?vHxnXf@YN764(UAl0+XlKRt%kC>T2K>{#
zs<rXKT9%z*`~Ecx_{=ehVYJk{oNzOxUEy8VntqLQa{s<8<zKgX`VpJvtxE6oR+W`C
zTP14VyQj5t^0}~6Gn1D;TH;u-I@%{Gw<AokD^>a8m8|JEbfTEoD7cAUPN_b0=Dx!J
z-9^C<=RRAT|2{vaT4dc6k6RiHD-DB08O}Z0Hf3_6&C-&@Mg4Ubxiuu+4OZ}9Y*#RF
zX;MFA`{mWWqOi}xO{%AldR@vszh%*8|GVsr1!2k_?s_q9{bh%HI!w0nv|e2NwM^I{
zYf{YYD=G|0&*Wwr8Gll^6uFSEDD-hv#p7$QdS_40-(R_cukCmL{-00oo>OO^`(=v7
zR3q1=z4!em7l_K1xn~?abcd_|>5J9ci|2Y3=XL${yX^ig-s)J@pT+XKUe@KBCe4bF
zshFfA-SuzEsgEwLeey-7JB3Asnx|$@>RNup57e#Qb8px2Vp#?b%OYQ{s$Ef;#ZnA`
z=UCc`a*Yn({i_nJ8M`gS<Hh?h%~MMbx-Fcwl8NP@&Zpf<OJ<xZl-Up?u9zNj#^ZZ$
z7gwCu<5>oNQxz?5Nq%Fz;<nx1`q|G3KXN|q?E7p~7Y*L8uAQ{V$Apbx<1$0z*@Bzo
z-FtTSIHY~*NL;n#veku_$qu&17aXj=q2g&2niglOpO6|+@_o*#khaMx>+OHOF?q{l
zG_|Zcmw`cjY2X^q8|!46O+W9}@;TAkzw6Gt3FiI#C)+iry6i66zSQtnh^JO#&CZHz
zi?6(F(&iCMT<meuLSWBcR!vFanI1{GMiZA5z4;(Eb(`QKrJK_XPl@iHnq{oIa!L2*
z#$P*bR`A?>`*iaBXOri*oo#2x%K!V`Z~y1@^X}}M?wIm!V`4}aPv*Vi+$D2LKA*5U
z=(pH?=@re3?&*IOZ8$~F>ReywzAtEAgJ5Lt)`e57ic_XW7fsY+KJbHU?X)=6`(;N&
z8v@>WTXxRmv5fZrouZ=@su=olhrazCs~h#NPrBFNsI7YcXZrmge-}&NuL|L|ul%2W
zc>dolpHH?kv|JGM4w(0C4xemd*!RUhSZ8lE+dWs*KK%9Z?YCr^&Njb&xVb_{{$~Be
zOIPCaeL`;J^`6)v!w@<1^8Ai8Hij>p;id(XvR#!Y7@FO$ddnL%Nn*_e<Jqb@r-iQP
zm$AubTPYT=b@J0ZbgA6ry4n+_2X&IkCuaT^(=cvhh+27c-?bLjzQvQ0HM=LgPFtlB
z>oG~6Yju5ih}rAcTmDQ4ikcen^JZzx>1*+^p~u(f7&16dD4kz5lX-8?{dJ(Nn}1%N
zkYbpSJ@4Sp0<B+@m5Y?bekqlA#Yh|d>So@Sae?8$9bPRD?dwxrWxUVZ)(ERt#<oqq
zX3LkY<DzZuHr2(+u)MhB@hyvloy)8(PH+8KTp-H)f@`%()2wK9#Z{eLj8|M99k_7%
z=+ge6Rq|i>`QApT2_=W9P1+K$B$VAj_}GfO=cYMF&CYepXfU{z7hpft=4p?FxYjl;
z`*SAAkE$XLI|~0_TKGl0>i_xg{QoZhym{}#!T;aa?~r`cvD$ro?dy|k*Dk(&zd~e=
zcdF8|tgH!A&i}7nwmZ3*LB>t+>Em6ME|Hc^)+)Nozr<*KXS{G`i-zR}^}lS9xd+2E
zrEHZRPH9dx@vRYU*!*vTgwDzBFHJAHN6mZNCfRv2c=8YbQyr{9-x5Q_?5nEY|M_$7
z<%+adyLXFk-!HxH?Yy3UA78A$|3B~lgoxE06Yt$*s?}r+NW12FD>*kUWRcABIgVm}
z674@luIm4`{pCBqb=@4c*V?V?Kr4v;C8#Djrg821wtUr*P#5F$HuK93ZjMuY_;)H?
zn!c`&&Ce-r!4@H2jbQr|^O#<g@vq;%X`yeohHBD=8rIr%p;0SNiTYMt7Zup?bgN-W
z|DyXxi^7d#)7I^rTx`j9n)knx9>d!bR<Y8Zy;_>7M+~mEtx567l$u++>z3r|J*)|f
zzA@-{u1N)Vshc)l?96I%Z7|f@;Ko_9dg;RDT37Ec{b=Z4#@I3^R6?g*d9k(f?syA{
zkH5Z#%`tI`TlzHD!j!G8f+?ZAJpIGQHqjkRisncjIoQ**$Vh2=@P|1I=1NbS_0ut0
z_*`(LUf4y~i@g@tje;{-18W_Hb#xMR?CW2p1*HXmPSREVtRhq;Qu$1z%V4tb&fjaU
zrp><Sm6R=>($h6%$rHb{2}`n1y3ak<KimA@!(XTBFI=zt^KZ8NpXbN!|GC@DU&r&n
zX5Y-MRu{AP+wc7OO#jBI#TBw{u0B&&9(H=RYkIb@v8PMm85PxM*DmjyGWm+imsdH0
zE1LALKMSnadD`+|`m}VnewlAY0=x~vtA2NtvR{<xVRpFDCpP2NzSnNkvzgV5`u_H2
zAMK6HeqQzI)!C|&<iEGx9=}^3|Nr-5^J{WUOO1D|RnzX|TErQv`YbQNpF8-{Qcvbh
zXU?Z7-F&-7pgFWLl8?7WJ!Seb4()5%@l{;{VQru@nX0QMs$W+U^AYtI3KQ1(G*3?O
z^5Go{98a~_kKd44`CI#u9<N7{EQ8XMGZIrJ=Vfn=-s0J;+W%?}?*qe#I<8AQS0DJi
ziD}FIi5n(KE@8I&T)h2z%N7OxJxA_|O_}`axM%YzHb%ZBd{Zv#{WMqAOzqvgsXb^>
z#<TR}HKDST&($u{Texr9N7Eiqt^b@uGu3W)d4|-|iAr8mG?|}@g)LngV0qlUxc&M|
z$<GE#*EuZrZ0A~fXzr|SS`0gPay@txn14%Cd~ti^kyB?a%MV&41c|AvD$wL?xV?9;
z%86%OpLwoz?U+4tldO^G_nSwLif=3|Fa1$BZNioCe@QBPnI3#<J`(QgHOZ{QP|z?%
zN%2i;Pm>NqNjK+HpSf1E6~xb~JFgK+Gvwt~%1_A_=eNFoIO^s7j3;*=FYn#|b9XQQ
zKefBnx6Zr2?7jNv{=TmU`~If<{q}fYXY-T5{rqZ5%i1>1d{uXZDSVsf%bizd%<yF1
zIqzF8mz!^?z3!66mO!>of%R`5uidFEkjAn(qv@lldmE!dq3FgillDu~gG6_lJujRo
zIBk_jujQ_-UsZk{J2m(I%KJa;m3}xgIBdWFcmI9i`K%Q}8#l~kd-u@G|Gvk`kUnMA
zPQjk7h3cFuZ*?ADWLfedww2MQz4pfCf6aGn+K&{S__x|2$x?Ef?_ci#{+!fJPOimr
z8ji-iD}FeOUhXh7=ljT};-x9f?$n`U9P@E%_urH6@-o96TkAH2MQ_`2o<UPm?@alg
znT8QtBIcOf2`rVo-DRXW^>NJX36pK(PHqk6GMf6>;sk@-C!UVlBVmELyF7%<raGP8
zob&f;__kBQNm{9~8&bOpBthl;o(i9-fj><|)&#s-WIpHCGJ%lG?ggvVCTSe__V&`X
zorhe1`peqi$u;!M`~5=8lPO!k>roh=!gaNz^y*I~f793*cE=y_Wac#04|bWcwAEKw
zq-~D!nn}zvl>ek#yD_cl2$j@f(0=sJ<8FV1Q;2WtzF$hc9N(?ap4yi2SF~zzw9lL=
zUf=sqrYQ1xv2{PuoauOEOLOG9X=}gKr6ifJ5WcY}^LWC|-h$*QxsSTbqHnbwpQ1Tg
zQ0~}r`COZSm46S<@85jy=knv%-+lNRp8xINgZ;mBc1<k{+w7Fay3A{$^U7+wsY!{)
zigG0;2?a6+O+7xd=kaDH11;T@EFNDA_DTU|(JLM6nLaGJzo7GqN?}0pg)R%@{}*_Z
znICL^ac#M|`S1Kc=i1G;Gngzock=MPkN<x;?*F~tXOq%=nKsv|jrKQhh>2!A?W^Zp
zlJo0^tY?+CL^t1V`^}G*^PlH^9mK|czF@~+p~IRhsy>N0=q743?qHZO_3_Ls(X|{|
zGYws0ObZ^lba_cC-Z0QBl6Cd5me&#AX*T5`yKstU>gwoIPWBN(r`a7!m=z11qE>WF
zb!}BWvuV5RqSDJwo|4BIy}2ce+$2v<NH4tP(XF8?^3&A9AiX_(^X{2{Co`BRRBn=+
zTl=cMd*U1Bq|S}gKuhR<h#DGn9OIZR&7z;g=J0aM?3qC=j{YKwGYnVFTy*TV=cJ5E
zHJ6nM3h!jOFRc31)9ErtrRUw<pi?TGOV<2in5gA+>QC9G!fz|s58UB>q@di)cKz{8
z!@wvBmL@LFxdQ4>4kcRbUvKQ0WE^Z1;m7OzBFptoX_QP=R6u3OiTwe}hrF+vZK?bJ
zz~UFzfxPtWWe1ZOO41k#oTB*B4AqpDWg7lB+<zkBqITy~Wq$tjwikx0ZaA%DZRpte
z<I7_0_kWgrJ$L*!|CfbU7uJ|oy}fty>+Dsf&rkk(+v6;CwR-E%!rym~yV@IH+I~{v
zm>Qo@TZ+W&j+JLRG-iYzI_@aM+7LLUT4{r;8JFSYBbkDlPpwpTi75K+kpHEVXRp`w
z_{X=pO<U*ZEHw|?9QJ(k))zMWcGeq($?x6#dGh|)o4<AL-@bg8|J(l>`Tt8Bo74Wi
z>^R5BG%IuAsm&UhORg#1VGL#unmbE3$%@bANXf~}i7MF#H$8MsiHM(D@%wLiX40ZV
ziT_Vpt-D$Gyz80!!8c8(c&t1bPA6PEV`vt8DS*RGN{8XFg_V{S!>+rBQ>qvFo-XUT
zwsYB}(05<Co)lhEU}jA}@h;nHrFfRgBMX6<jfPz(ivCNz|K`@R@y)sE6;{R`lV*RJ
zJkdhudRH8y0Jq)k49kU+TLSauuHs<aIGwNT@Mout-JLg=P5<~6T*QBNW(>G|$ve2k
zbor)Px2}EkYd$(d;@E?eRjH!uHpxz{essuK%~PoS{WPPpZA(3cI203JPMDg);LtWX
zihl#!yUc|e>JfPw2`goOx3=&lTe5XK2&oEcykuK_u;^&5sGX*<#LD02><q8RcN{#F
z8pHZke$M58+MLk^^CWL8F3;I<x;8|_Fs04eE6dd>wxaXnt+zRXzg;VZf4yT7R8n1A
zJ#CBjOkpW^9>wLav^@QHaCF)K`uS_JyR`hDvlspUJ$JhzdG}`JuV3Zg|GB2W|E_*_
zx4q@>=FNY%*L{3=?cn1@I%YhU*~@w~mZqIzIMS2qda{1Y<*F!;Hc4?IRs{jGtAAsp
znLU4%D9ITGrz|_+v-0~QorjTk{Ct%jhx1MP@Z#9^OVX?FzDP6Q#hEbY>$*MPyL<Wn
zr9D1*@9yK};YPK;D{8)~bN-8ebn7q+`&pIP%`OTZXP7lVzqzOuBc1K6lc_ezV#BQk
zDj(`t<=D??ZoI=8zcDAmr!Tn1wPUsS;+y-{J&ioF<-Nh2NL~ilBZmyl)V03&emZvJ
zOEi1i9*)XqQXKEzy|dsAOPi3LCpi0C&uf#0y!@6)$Av`8V!rpUR9z%n!N8ze_V8|5
z%<4H?3_P6zPU_8P@oRf(5cZVA!Y}qz(kkC0CKGPR-FRB)QT;k9@#fb`a7q7Xm4I;i
z^as7HTWwDF6fU#aV4$7!pyw6W+>=Z(d8yltXG!W+M+!Z6VO6-k^p>*cjSx$=)msiP
z%(GO!d28}=&p377FqtWAr+PTG#FZ)>o#WD5McH*cnU)`!H8uV8MsLAGmlFa*bR=p_
zoYlJGTMjBIepYIBWj}5x@AvEQZPnV;9;e8)k!Q<;rd`-uD*y6ZU%7R(ii(nJePo&_
z$9|EC1zAV7JUwP&lOt6mR^+yJlk$qY{{m~OD)MXmDi42qAmUK+R{G>>^TRK+LZA0;
zpVd<LV(Uz!Nhwz=q#a~b*o1t(JmwS(4{TWDV7)f|%VX)Oq8{9l`%;xWBhtE>&;9nl
z|L3TF-G_gd&Feo-KFiNH>4Wmd!lKu|POg1jeLXvJweh(shHIDHqrcm)uK)4*nzj7)
zUH0kf{<Ew21DYeAM8%vublcJRm(IS4Yj<w?zhyznMzw&aL5(KHPkW?9g=;h3GQaqv
zbN<qq4<0AC80>bu`|?rjofAgWHDY9vYpzP@h=XoGjI59F7MU=?{qP#eZMp6q(<)}?
zZo8@XbW+LBE3SUiML9W@#I!^&Zb<(cHFre@S8~XSSiU~t<lNThCEXVo9yPdxEGlAn
z`RR<0=d^S!UvZ;OhMddgqU@e0B)_hSZwVIJ_v@6u!*#FOdM``fZGRkQ9xwcmA)xdi
zXjKbnnq|L{NW+qoS;mPwPWeoH9VGd7@(0gJYwl%vXZp!|S^V`4*ri`qt)1uj|J(}|
zk>K2#*D3qO*_Sd*Qdw|r{r1VcNgK+#%c5Mi)UDd^x!l(H&Q`h2=BI>2cCoLTeDD{~
zH16<}xM`Cl8E)ikHZ)6B{BYDHVtU;Ko;!(_{Lh8rq+(c=Hcgm*si`}A!(AtrG_U2_
zMaLFNIY&ts8k<a>R=BidO8D1>&DHC+1O=VEaDE9#rtj2SZ{=evpR4!A|9g4!-i8zM
z47(5H+3sRg*!%UFqfnjL-JoUPBTKUFjb>_Q?U^#CeU71uVKT$H<ZbdN*EonIXhpSb
zNn4ub)6gm9p7l;9x9|7%uZL&FR4z5#VYuS|!Sc5CIjL*U#RcBie)V`w`M(!RRx@+<
ze=ooO{olf?e}Crvf6TFJ>Z*=RZn4l4Q_YU4b~5<Qk>|*knpY&oZoF6OtV%|z^tOUS
z5n|7#wXPFnp47m%GB`<h*RJrKUz#$dzphS9yqLG=>n&*>+obFl8dq70Qw(H7%fC)^
zmHGNQ=Yb20>8>>=+-B@?W6aqAYTf-h%eRj2?z5ua3EcaY7bb7;V12)M&d*;nHdSXH
zu{o7B&w6{lWy1vSNxxcU1Q)+McP1}M&($Nl;LB9;BMFiW;(L!>-g^Av&E;D8zEdNO
zB2FIwZQuTB7#PIIdsWVVDyQzTSz4l>nvN~@)6jKcQd#RH^R{-9SdixRyhTwBk1tG}
zZ|E~gCHlqQXF@gGD*Yy^JoGYBSbXuC?Y>Do9ZgC*9vJ(ql-Zr0*!og&&(nO{smWC~
z7JECY_n6+a-EREPYmx)w2cAIt@(aaZ_TQ~KdbW`5-Ir;rlsk88@vh@)RrzQg@xRN%
z>%NY<q89&@#hbP$1`0ksG=b&Fg(a;_6;|^ZE*(2_{V?~T2vx56Yd*ic&(Al1Lg-7C
zYMuvA64y)LmDyYN{!96O>#Dc+F81cXRgY$WR(8UgFYT@SryuVo^Ka)nXFl&sT<*N%
zN>6em?)y&qR;lAuxLwWVgtg*(nHjf_&-T9@oxkqRpYYq?|GnPuui4<&-9vXyHpTaC
zzIW~K`F$Vn-M%g_cmI<8{<>}Ep;=MWHhqdY?Wpreq0mY<{=8q8_NiT$3YRfY5<A6t
zBUf(1WND5CwhhsV8yIHaDCL=UKG`<D;pujX<KYUKKkiLF*lYi5^2A;GycL`LEF;g%
z|GCrb@Qwl#v&bWoIRSQaXFWgAb?36vMgM{<x7F8Q-F>IXH_KG8TFUor_#%t`sa~E7
zxOc_P`I2=owQl2vHkor<CO^NuP57z$-{@(x=S^>UmH9H?$eQD<@^2f{RR%>T6Lc4U
zTP+@W%t)h$Nn(Lsl{3RmKBu@RRtm0`$x5CQ{@2(Wo6fSHk<ek7qO>JvGI!ODnB98?
zSr;vO?x5NCY}<eBW|N(IZm!xlc~UF_msd%bt;kaK>gk+dTz-G^qL8W|oD2*rx4LW(
z|Mqc#w0Y>O-i@BJ{GCn_g5^_RUN*b4_L{`jn_}v<n~SAqY~}oS$8q+Let)}S{)=5t
zR!94kmF|u9dGzVx(vyC7uX(@XYBo*(@gy*E%Jx$(OpEVi=S+DgFa7%7lp@jTNjg&<
zk2I<+3B8%VI(c!=!c$v4ck1Mdi$wLCvs+BOJb8oI{P(@Lo{2wh-_HAOQ?~i1e^odB
zRVu$qS<4sYv2oX-`%*968GeapeSN3du-xKq&X@YY9F?P?4D#FGZ9P1-&MvF{`uQkp
zhHLEYfB*CfoY>`4v)INXot?jp|MRVXXJ0=qW(ek$n96?U-5#&*Pa2*#+ucuCu9SSZ
zYKn1$J-3zG#z`JRDpG6{ScUA>BMz-(71|hb^y2a0b4eyzorlEr75{Y2Fzx97SLAx9
zb&?gwi8FiEggjn8d8?Q%c*woTrcJQC>8#M|)hl;&TUsZjsP!|iO7e^oJ-dACRTcY5
z;q4W>9`ROKEHF}j@kqD(hso=2Zbx>uxeBqr*=Shs;?*X@+1BSwE<a02R_V1^aB<eT
zttYk$Ii<`wI;rx6d8ta+iB|rd9^d*{CQcLJWYFDx`0b*ZY_%Qvt<3={QHiU4WV_-|
z`)}x}@7tKNVcJLSLuJ>SPAxYSX2{`GT#>&zSbt~5j|ZGymaNLLB4^)!3#%!4dY?;D
zc<UxHtA*j&>kkC>>2IypTc^TY8Nyy*eN*D3ScAoh6{mS^^^3JvufM@>b!ndA!(!ue
zhi#Ll8$_5KFHF{cq?i0?N&YLP@})ugf_@F5nT{`(X|8l((!AZjB1a-ga*fumtfj_#
zr^wHI=(ol13|Hr!m3D&D+Lq)Vy=BpBqxaG(JdZnUm*4ueA`FWr7ry?O_UZcTpYdC^
zzqjA}>Eo@tkC)%Kw%zloU;h3j`}q1#8BY%;?)qLb=c`G!>~h<)Q4jrg{9NC6WOezG
z)xA%y&R%?W_2cj`ji=iW<$wDexI=i}*0N>UcPG|eIq&{>jd|6F{=@C^ew&U@{-XbM
zc|SwUUi-U7-)FC!r^UKhXH{6xx1~BozN|lsV-_o~OPFnKX?MPKsn1Ef-YRMJlSxj~
zTC6`WpIO#^WvcM67q8r=G42Ryerg=?<-Pav^$ei7oQ0P+#YeLSc!#9@2=UyX&U}#L
z-o2vf=RNn^<V#ruUfZZ+{M5JEPWyH5%P$-Ze6tj-7o=Wty5-iAy2I@Jk!OecX0^<a
zOy6<KN9w(<P;B10y!Kt8lX$w9w{Cf*?I&C~VQG<18t(>s!^YC*VvgHa7F(}qUb_BP
z!1Aq=Yo^JI=vPdb{xMn-v|jOi_pXH0x(lxjZnQJp<KG$JCAe!7PuQWyce~|`7Bd*A
zO);8xS?IyUKNCA7h1ZF{=`D-)n*4fGhW(QnuOtE#)z2|_eVmYe;?v8eQvUhRrK>-E
z5xd23NvVr@t#0e&xMMAm@6)2=(hHqsIz*1=c^*&Dbz7%e7{051&qNbH)$@~1UpU@<
zbSdY8iM#AKDd$&dxqHo8^6-r7!6Tbrtd5c9RdI}76#w$Tiwla$O2QqI6ErTRr7Bel
z`LBQ7)5RNn&S;{GnCg<&PUb0xa?Q4{J$3!!v6h6LW`f5U?hCvQHl61*tz%~9k_@Nz
znkzxe%sdk8*t~x|{P{DoR%VUM?yVbkT1Wp+l;f?E_!9Q(bw!WqAN{7%H_NIcZ8GwE
zuHF|t+I#i0`s|NOX0N$_Y|R$mrL78!bIWD^$Sz~f=xH)4oRq>eagv8nvT&1T_)L!^
zH&;;`;f}N@?RWBJ^Upu-$xObJv{8@$ncl~UX-8INY^doe)od=kx%0Dh@7u%fCKu<$
zF}x1e<jC8#Zn55*r#cLtmW~w)^CXfO6w|$CO21WLHj7v;EvI(-MF09_pRQP$3Y4+y
zOyjaVZOR_9ea4Xtfg=+)?SJtnX?xL|ptnm~7PKT4Zg016jFM<-Pjz?FQ=0aSv3s)H
zc5~_9t$W`5pZsZ(-{R|ACTvV=aA-2sHJ;}%@2ZgLq4GTWwc3Vve|(kwpZe!>uWIww
zp65rM&a<;BO!*z0TC`U0tFygFz0uZNI$3icE>c=p@Ip)I<1yVLuCF5RS{EHlWO&M6
zx~_N1PlG<UO|ERbNzJ>{)`c9(oy$3K<&;$%r+wLZ;ucOgf6BtdEw}%!z?`JbXI}_f
z^thf7X6N!(E$^Q&z0b>B?h)s<$=hG(O78gGHq9j_)+?T|V4WDloEsU1i<jQ(kcqTp
zK5+BERtuFKK3RFwmA1TC6}7RWa>-M-pF7RYhHMQw;^?%0<>rnvTblxX{}!wY3AWGi
z@1NCgCujG^;P!u&`G3B=c+7tDpL%-r;U|yoK5OpZ{@w28rT2Hv-s{}*s9*m1E4%tX
z5^k<hxp(`+<VDWz_gi;we%$_g?o&RYntRz<ZhGout-a**^Zuc&B|etQykZPSYnK`w
z*l}h{*ejNYUD}q~rcHA;Ha?GWJ8s6;@=jivQ%fc6r?buCt_?+DH7^Ud6boPJ+px{6
zIP3Gt^)bqPCOXrC!e*Cd#>TDVHEokWwt4@l#lJ$9ZtJ`8LyWU7U*=3!?@FJ>kKe8(
z?c{lOvrr(}mjCFZ22agQ)uoxMH1FR2d;XbK)7z`g%_g_x)4M@yJ0{;>Kga80+pA~U
z_sXuMx|c}CPX`U!+30fx-_s7Yiw!-#l96+RSG^AR5)Q@*r;QoB6pYk77Cyc9ET>UR
zO61<XS?%0U-IwUB$bX<?JDq3NOFIUm43DDQg=-9E+`aFsb^OQ##xMM$oUA$0y!GZ<
zDbjD3pDr;8c0C<nJb{a^y0akaSr<cghTXF5Pj2s%zH(Z~>Dh({st)X_tPTpP_MW#g
z6)lq+6s|m-5Vq2M5tH+>+TuVfoprAaba@nYgNi3VyL-E8CqIkPsW_8O9G07R2l6C&
z+*CiE<8ZT`cW+Zn$1KfBCx4tPvDvfV{8x^QX!}<GUH5IRa%zekUS5w`y`~{#nY@3|
z+80w}9<2X*ZT(lZi=LZju3Bj{g>i|ye(oZt%@@Bf%{21R*7g)W<;DGI=A4P4G6pK0
z$3?QrRVJ$}oU_%et!Q!cj$M{oA_YgAr_B6RZU1<FGe79m9!0xf-VEB0^tLLqIBad{
z+<UJx>yvM^hSIS=jvNc;PuSeis+b<EI{D;}D_<BUTwD=5dvjXDjIFjuYj&4kD3#rK
znl-nFv%c;o^S!Ti(~@sRYjXcTe)emQ@VfSv{mdJBW@*n|S-5SQlxAw0#lv$aZ+3F*
z4o+P&|F_Y;eV}o;3cnQ%s*^VhGtBjqby%n3)&E|mL16PS1*Z+AF_$yHKXH6G#ZysV
zCabdV=!&;%S0~6%;C?nU^=`cIb!WX@j`Jnf*haoOd}_<$(l`xiStEu8M&{WUcpRo`
zI>pU$HwfGHGGxse!+a;%Pb>jFoJB4&$&*bsOg(3*FE{<x4r_b2MeEblA{sXw&Hivu
z<DvR@t<;GsxeiaGSeh6$uK1|6e){;VPwV^5HJ>hYw&*;N6n^6E*RQm@XWRLUpEkNE
zEKYhlORR14&E~F{2c9BR#nkvpqE5tUJH?)za<$&-?>Y7LX8)}+|CJU!O*ilCJGFJw
z@|Lcd=eBzB>T5rrxhzC~<#p}8|J^UWZ-w5kSd^lW;<%MB#K5CNOIIZ2;W^dkhXjfq
zE>Jn^KS|+9Phatwpx@4C7&J6@?b>@WFF?J2((~&=VO{L|xm2!QJm<SzdY!DIsL(IF
zh=v09Gn(tJ)HJTWZ@wWWaj)ZEj;DdOt7|)MX1XoRb<FPEz9(_gvUz9oE?Q)UTzMy=
za%R<~aL>cPYfDcs7G%BRDfo1?yIJRYO7NSi;H>TZ>so8l7#3$Wx%vpd%W+;=bA6Wa
zYVW?6M*Gabkr!t%)Ah`Sv)6MzAKY}4we@Jkm#*V~?YuTWYZ6<$apn6g-fH<d`AW~`
z$hxGZr8P4!`2MN+S@U*l%<Shy-&ZtldHCVa&BWrqYx{P}WLfW*VK~Bnac$)He&c3~
zOq<6Cmo)_Cx*g~ilGOh$5)*Pepzgv!(^nUKVm~lAc)g#<aATusA~%ysW@5)~CYMPW
z4o4d0jxae)+EVLiwf5k8!;*^A60WtJve{1+jP`Dom{OL;w)ugr1!pPa27}Xu-l_rT
zwiZ_V8P8ssRcE&A+Uz5Za<)c=3p`z1-%Ovv7NGmrKsP1IL+BvGs;OG@C91>1rX2eB
z;%TDFDXt5<=ZZdBy@J(ChC`+CYSh#VTn_7|nzcEI9=zrx8NtDQjPb3Zl7hPKt3NYt
z-3dM_yjrNlsHXImAn5LCl>=6wb1c#{zBjh$yr0|>d+I{K*@P3V8r7-ijxdNQZa?ti
z$DACgNpC&#)~R_;5<8Vt^6t&12byR9E^GO+IJp10_gm?j-xcC6se#M--SSd66<Zek
z=1Jk+X3sKr>9i1Se}(DtEC!)%QSpb<{C*3pekIWmmgShVv1ffaXlwR7(XL-pPyKu4
zrtZaCcKGw&hYS67FBR$*Pr2`7eo!UR=HAs$J$AESH=JsfdFtG8{G#i+P!3<4iVNjk
z6_<(w1Zr6ouDf|jetIPteNXS{j-)2n4d;Kl98fc2Xqof!U*XHII?vb+G6Y#JV~<vO
zR5^8O!`q@82RK!^ivLcsl&z`>tFk$~TImk&tCcdJzlEKW+`1!j$`<QKTG!PCboe3`
zW<>ZOW6{#Ju${bQNh|C8iQRGUjulK`Z~3UfqA4lduq}79>_yY&u+K+XHuKDyJZ<vi
z=Shjmp3`HJE*9lCPH|o6GbuxpThlUq)znMQe#=~Unmw|}Y!WKockz$$3+;qs8B)I|
zguTnOji3FpUYV~U%h>W}J;Pd&BT*L(<&R`cUBdai=Vxe3VZb)UMSh2u9O;_u7P;?3
zj{5iY9oJvne!stZOY!Hguh+gk?Ec<XvO!fN(S<4DRcf;Ag9mSQn}W}Vq#kRrS$OUA
zI@8NZ;wfHV7TYX6{o2qoXxAg1U8Yw~DC(Z-Yustpz+qXuf0neAV5{3=1^4WoB1@W+
zR42zy$*THP+|SImI#uNW?-9e*|9s-k>^mL8rF;HXK=a$p-r^C*uIqvZuFmrmN&4Or
zSCTPME4j6|sLuJcvC)^=tFF|~Qad^K>bV?&m0c@?j`<7z%6qA|Y38O-AxjZ<&Xw`b
z=O(GF>^xvA&dBglz$$p<jR$#KMKz~ZXILs%WW}XvZrrJtrgCmU|B-Ju-*{-MA9(au
zcezrM&xdbf!L<yVDk=>(&F61w4bxNhJkifHv&=fWrS4C`f+ZWa%%9?_AmreA^07np
zvUnTMEmLFy8WVPk*(o*pUi_8I``F<|n9b)!KPpwP)fg(rFWFRM&Tjs2iMPqy#j7V6
z=Xy1IPkP}P6E%C+ojaSuJZ4vi6<v_ps(tC>new+!Kc0E|f6{_jC4=i;6UCOD;bha%
z^;yW1sPoH}%`h#rg|WEoL*b<cosNT_c1}yr+)!ixJ^k&Ce1(pKRqszGEP1#8>n&*m
zp#vQE6}>i@y<fu6(BVJHY0?S(>z{&V^NUNbQFfkgHn(?8QLaaQX81a1y_nDL>3vU|
z^b)PA&XsY?b-Et5Vc2%xIKgq-)#}DspIc%k1Ut*5gt*R_btCsmq`ZFH>zRxB%v@gl
zFL+hio*=n@??#r0(>rX`PF-Q$tQL`^mAdWUtIJ!DZ{Rz(d7H$k3&Ot^g)_}P^JEUM
zQ=G}=16SXNSl!<3G0ElBr3l5O1=~a!igJ0Sd2K$asXkt<vi@?d<=Y7L@~nf|@1L37
zek#G8Y8oMWaJPc1O@oh$$+@yEjmaDK{3@1M{37>+tI!JlOC1-MYjhm^G+FA4!8w;n
zGiPdxik;cZX4tWN)f4u>qQ)jMZy~MPNAK3heSf`b-u10A9}hp?zU6!Ok5lD*3QuO<
zKF_uBiGRjzo*SYPN%yqx=PT{x*jy*N;?rWCMinPLW1F&b4;YzECModExc)Kio1@a7
zj{I(Bw<%M98=q|~p0~#(tluKxT>9&fqEn*N_|iY??tQrM?RutNOa)ttKfnEcpLK)M
zyC7Y~2GN=ah7qY3I*-qI@vZM@kEqxb-V)7276M1?rq$n%dR@jZ+ss;^sd|^oNw4T^
z?BO{RxZRTkGksmuOojK9RWr+(mR>SfO;o$pcf{-IjgD=Tx9@IUs?(uznx}|&wS4c}
z1M3$EIc>^2xpiZ;Q+4X*w~Y)2TNjyR%}{P%W>B=~j8f+Ip7oxfQc$mdSAuY;*-1`T
zX1RB5&(AJCu(eI#&~|Qdo5=XLJxkxX`!MGo3VO%3<%ZN5Rj$b@j3TA0I4w)f7pDkk
zDa~BPbfIC}pI{mG_GftthfltH=a+j;RgB@1)6}z8E;gG>^yTjz>WT<m*ZV(V^^Y%S
zI1HwqGmbc97-3?3{ne8ExJQ~lbSC<$JYw^E@zQ&0^Pat;nwe+j@XgJT_Ks;(dD0xg
zaLVrA>DZ+%i82Nb7A($xrWLC$t)C%oP<f%Np`+of;<?4zx$%`N<2O!yyU%cq{lkJu
z`Q2WTenGo_#7{0-;J5tgl`o<)pLyI4zMUQ1|MFAmI<o?|{kjX>GLoN7ZkYb1c2U9A
zS#u2YbFGZ;d{x~eJjH&3w_uorPPyHWq&4?zKt1-xwemHux0+|Db#4rr?6Z_1<w$|t
z=8Uo^u6D&Klgctu=IqR#V{+_<%7b?w3XLS+zk7GVLo+FTUZm{73>O|#?&~_+kE9)K
zXjMF}Q~&(#R}WhO&8hweZ#LZ8^?SSWr0Qq8WbUS_ZOc=ZysCOc$Cq1}-9~?=-=n<e
z=bUdhDSb?hNILtl^s8lkW9px2;KMgB1?;QYdVMy7)ACrk_t%7;-Fk3iV_Idj3Y+)J
zO;6fQBJ@1=x?G;>-u-FGf=tDwJ!{|Y*rpZ2{9yvS!>`l7O&L7n-k0&eRK7M#@&m&h
zW4-fRH%jTS+pTh3a7xovBkaiC-XqIRrqpXM*K*o(CHO*l`hv_;{6e)~7YI!8E^=ES
z`mBFJr|P<%%bTtJxxRnC=%aOivw`;{dnXx{W{C!c7Y+ue_09I=9N+nzqt&&?$bQrF
zds}AD2$i)`v`Jf`T58eep}eS-qdjpR!{_ThciQf%R$aAI-g@K1ac<lGb*KC%&X}7p
zIeFEdP|o+F8?rKf#qRm~q2uKigLQMwLPd7@+b-99IQ@po^N;;?9y?y`RcCgO5xkIC
z!LVT3hBFDerq4ghv=!{Akr3Yyy4NHx(Pzh(?bc^g_C7R>D0xt!b^PB;^`$B?sion2
zwp^Ypl=}3PPn&D`)LxhVjh@UZ(<Q&8Mu@On=?P@H!jb8^MnrvyYseuBVbEN9czUwU
zY_<zGllJeJ<8`q;!#TENZCw7MsjImEw*=1reLUir{qqQ>HK{RDhxUtiY+y^+5&L3`
z_0&|S0txX#g*VLpA1cC^-_5!9W9G!E&PTJS9I_41yLWThmb-iUye@i7f1&N2lcBK9
zMvOtFvi6qao$SE9{Bt6kJN|CH*{}Ucr&ayzxe%`^qh0#ai(}%`8F=RiY$}YnaQD0M
z)F4sCxN8ENlA2t9I#w}VEZcqb$j<H6d=aWAK74WDtlj-{{uy8HDKi`{$k(ixn5|{3
z6YJ?2^C&`9{zwJCP5ZhH2a?b71-k0Ho(`x={g!CfTv9eOLOS$doF7wbr5aDD+-fJ6
zP>GdZKaR}a|9{pGb)lAvg2#CNbG{D^<+E@!{1v<AarRFBD;iSQs&v1eTlCySb<Gur
zKgS-ey?zgLQ|qU1zu)is`)_yt-fwNp(ZQ<zsto5QUpn*GCN2Br8;;%2-Z9L|{F$;+
zq13z~aFxurH}BrfxzA9#Z&AnOw!gJgbqfOHyp?=UFS$4C<*R~n(PJ#C-=!@qr_S+M
z<8j32>*+3D?WhHro6J}RiY7|R8oskhyL^aY<rE`6i)X?&uh}1(6!9%3<{&$RTYo)Y
z!m2YL?=_#?RDABbDWu)-+{nxE>^7O^t3J%%ulP?gW6(XnOzen8YRAD-r4FxFyGr{{
zu{ro=ZS@~@PbcMPHS11#&Jqh$d9-^EV}lWQ$@CuPiE-EDj;~e{SvPP0o7c-Pc^p}n
zdlR%4@y^Veq=m<<ue+@b;t=ipaf$1Lc>mEq3s%d^Bu{4ho0|K74zn0T!Lu(8or{+-
z{Meq)ojPSdr`Ob7M!QrlW*$75W9xZKZY}F_+oJV5SaL$QEloUoM&pfh_q69T#Ftl}
zo3r$P%DSoNtAGB;oX7TAy{1d>^NddW-&g-$p2In7*?P;4L|%QS<vTw8otD$_(|s9J
zfJXTI7dZzkPCe_tBhjkj`SixspqMCEj`hDDdM9q^E6cO{cacAS|Gh;n3$49#SNR1m
zHCVZL)6a{SlM0y?Dyp;p-I}lZZt}?;w{7>_n6GlX*z7yYk~!9GhoAB-p5A(xu{g3?
zew9$qV$0dNYE8G4Cd`$;IQ8b+G%MBxSJq53;4_+(cvCju-R(-T&jnG2?nz%2{i-F;
zI5RY#-hArxkGK7x6=j>9GFXcwSs7N?u`ncUkPucp|KjHS?8B2wqgEU<wOQ_XgUj@;
z{kkVRji!G37AA6MN%iEbIh+iI2bSFms4MtaC^wn?fycHK3(rqgYuocK9RDM5ZRJbZ
zU7sxE4($H_tAG2H&0eo(cZFA<kX%{*^6~84GEK|xN`;xrEuS|${WNK#Uaj_>&nH#c
zPbOThKDPRl+M1WsXB=NIRb1&T$bQRKea|+h&g)-U8#PjcT=I5KQCd|qm4Tta|B2ol
zq0|1)TWyMsbZ#v;pC7pIe0$Q2=j$^5o>M!U*T*Ln-w+dVA?n~W0mgz{kxK21<Vh-u
zhq;f(>bBX=dv$3E|5b@eTJ_g<p0GNa&UQ|i`+V94Ew45ABLo(KR(dm4yg%8XcE$X^
z?%%16uVPQVmpAm;s%ZTEj$G0DgaD3%M$G?zOyD`~uXd^`xxAdc!DdyunmXTAt{bPX
zM#ZY-xH}26_gq?YF2O8m`VWcVX#$_OD_<`=wDm?8XjpL3q*;kEr%m}QPtLNI6l>UW
zAtkJ*^3|1<n_{jAU2V&9wp{t!<!z+1PHSq-T+m*RLeB`lXvP)m^O;^{yx+lF+E%-(
zU21WW>{3_F$OZnsSDB6OJkCkzIpt%w`I)y=i;&E+P8S`P)rHTb#CM%*QxrEnx9pVu
z=|%oa@80veCCL!5T<XA<=DUkxtvG&MeSgjNw3VsK3|lGIJz=}w1$WEbR%(8Fea`$Z
z-rKKEzm~A`yioa8rFWOaPX5yh`sAp%!gkT(^)Ej5@BDP&ruJ4*PRl?4l)h=Q@(XiJ
zDOhE4s;J`h&gA2a-OVWi-4-4>|GXzWn!UjJXf0Rqn~*2proFnSZLoUkF_s5&f^PWB
zPMUVJdUm2)=7m+Kz7>>d|BH@o3_hY_*<5O8Z1Si<O_skj<6Ex2$NgurdaNBCzH4u`
zM<i@u$nIuc5U})n%ir#07nv-NFDu$3eMQ{bcJ~jVrQr;7R?Ssg;qT|G!X_Q8y4>sF
zFF}jhZCf<{u71H4bW|i@U2)$AUOmq?*Yu{{eJeuGm0f6Fs{43};3u8yHcuF~%$>o@
z;G}oL?@Xv`OU<>QrgfU3KA8<Am&~862zs|_x=hRUd+NGNbRPt0w9Z!2U3B+cN6nMR
z4ZHo$e-XSsy}(e|H~C`D->aA7#CqixscgJr<octu8<f9P9aUS3G8o>pJ#RR5E&l4K
z9NFv3PNk^%?{Tc0q%!9!gGQ*?@eRG%hb9=m;^DZRUs`uUZ`roYY^~!{D(Cng&xm$(
zOIUs5ZO@}P?uO3?FD-kj&Tc&QQryM8aU9QjSl(JxZ`&Y0angnbUGoY=*|d%4F>e*u
zk9B=v_nIX_xpcluf_m-bqO#?OJsO_Qb5q=JlDw3mMaE0<$%(Hq+S%`SPtTclJd?A3
z>!o|ML>W0=|A@LZP0iC<H_f3XL)W}eb7!k$-KDZwk>4-&^sKK@ofh4B+I!oj8(N23
znv=>5ZTE{l`yiuZBz;7ox%3=MgX5Xlsqx#*f2Hnszv*BvnN>JxMaC_o;Lj6Queu@}
zGi_<=JPB@wq}&rNE2Dkxw0@XV*0R5H$=7Sy`4jY+bNbV#@0_P)8T5VS*%fb&e~5az
zV{=CUU+~|U>G#&It82|;+Os!w<I!6QvJCF7>Pd=!YgWp5&%E7yey;R1`6sttaWb4b
z5#nX7(<Bvpns-i!o7cXd95W9tnqqP$Mxuqo>0Aw)w^!zA&h!mNd^ZgAc)zXrW70S^
z(KUTO(*~7kF|$l-ctKk#3Oy%qhMBN6>@6v{nC+52|5M6ep5F{gV$b5X&+K5l$hYb6
z76ZlgpB&!hwjN}fprWHKqVPgbhvl?s+tbI(*2r&RZD6#G+`V_IMdlin=7Yzh+Jp`F
zJqqV!>~N7OdiL|(1`Cgl(hrhrjxI>DWeQy%pRc=WjS}1ce=i<fKMgvZbDFl|62J3%
z_PU<*TCZ(vFj<wez2c5@Y=@Eh4WH#9cl<m~pJhK~ski6(v`>89OfmlgZ?ML6CUNd`
znm56-<vz=V8>aDU{Mm620wd3f&b_(HFyxGs$86WEj}wfmUSFE>FKMci>++suxmzzu
z%(JxUT>a+Ut1SjIz4l*ycjT5pYN&?r8LoXTwtHM+G-m~HHE`_w#((Zla4;hWi#K<&
z?F-F{BVUq4A1?ivY8YX+OQ}1s>7VIDpQ$y!|CY<%=#tnfupnd~hvgCzy8_8$t*=?$
z=RaDKFB9ndaF*@vpn&v)3=I-GQJ0i*p8b3EYWnHDLWT{yPSl?9+Rl1^&BCS+8KHug
zXI?zlXJK48i6z9=c~i>-?v+aan~x*}PStq8vqhz$H0HH)%Io#}HigEe#w5%;ILlh+
z^pD(LNDku>`>Guax<Iw}ng3FS?M;a>0-J>luN;}%yJY2AkpREPF@Z6<Q9YA>Pnj~w
z=1ykX3BypMq!4=x^#iB2D)l=usPjv$t+qM&dez5}kD~UL6`#*-lQwCe^K(mYXwixE
z+U(>h%%LIb5s?bl4>_9OyBBoy3wK%7vX`g4PQ`VZO@1^lx<yxv$5Qk`aE$6y3B!m@
z>3Jvojw?PXm?^$pf}wlSjM&W|=Ckls+`Q$Sc5;<azThea50^P&g(2$_ChR#U+f@|M
zxnYWb&(736YXu}D*9jNvgjcRo@pRCg`F47$-F>%5s~meAIz#&QUwx;*Imue~jHKa|
z_{fNDTE?cc<c^tscRnL8|L||sAvq;6j%Paz(+ghxda|QnNq4jUyCXjyKg<8}@6Wg2
z?<etaAMm)V?&5l@%*xgM)xK;$uMpo}<8W&y`-5k#Z|~Xbdb2or%KM&#o?AKb_fC7r
zU2QL%m_1MBfBm<AHhz;{Tk*5*m!3H_I@ImXiKt!Dv3D7EIW{S9I<#;E-Psr<;(1`d
z`tQn|{`t@MemryH^=8}i-zqL1+Ncw`Dd+XcNvS6%>7I<zoj9eV^~j}&oedEkQyDI1
zl!TX;ehrGgCoLEnx${_U{hS@@`EF^ZiQY?G7M)vPe=~3*7?|(-J@<Xh@zbTEC9W;o
zgp_vwn;P^u$nw#;n7dbFB))#RV5Ih1LU39Pf66L$9hSp}%k^5`cOJdTcW{m45^l#w
z%Q!j@Zj!Lt#4Wiy=H~Snoi}^-i7~7wzj*4}>$qzS(S7q<_WpR+ZQHO_e0ta0wAIb!
z0@r=Nh&DJWtl*j55Gdf6zanf&7(=A^jR3|uGqoL8e-v`Eo;bBr^3-y+z_2w_Bs>qv
zrZez9@X+vRR(;MS$?Ln})g|?Oj$4<$A951lRhsucNuakPZ>OVE^|#t*yJhWXONcP}
z?wC*@(R*vB?*7b}Th2utA5u8f=Q>vXx8O}nn!9`b?jxdKgC`4hPApw`a?Y&Wo5fqi
z6#c#n9(XrX|LrE$uoI$MQ#V~_t}OZ$$Ce#oeZHV8&*_fS#0C6Dq9Qw@`PM)GdqFvs
zpHp|DR1e=#Q>N4-$Ltb&j(IfNb@*s~eUxLFVe(?prxc5M&mJ3}Jl)JudU5$BMmzh`
ziM9PpE))f3uHyab&*G5xsZgPYJ#GTeYQ4WQho1d9sK3RC>CNrtE$ggJoQ-rAh4bI9
zQ~xXRJIZk7Hp9d!rMbs-dM>sq^oE94xjJ}Dvh}t`lwO}58NGZL!`~21SBZD8uevT2
za9BBoHOMb?<!n_RgOlIqcW(T;Y-WDPJiRF!{Pv&a6)pJ7wY^m>G9qmy>ycHb<Nkz&
zy=OjnD($9G#65*oldsGzd-^SAQ`EYS+8HaKRpo8tQ_x(<#=2m6v*yXjAA5hi>j%Z+
z<i+dV|K&LOJyuu{VI#LZibJw3?t#A2&J|M>|7&SAE#AYUDP6ReDe>eVM;5sZA=&ms
ziRG)rgpXY^xg?nWxcZ>SFNe~|=Bc5AQKi=vKJ{D-StPOCZRM0-nK#O>y)(R!cJsd1
z!NStz)p9pg4;Xpc{FGDQe$QjaxkZ0Y{OFk)@ud9JEA2z<-|lWzch2%Yv~sVb%T<4d
zkD-xaf$DD8SW`3@PXq|9RaF%-J`xdhNHlHdCmo*49k&*BJ={FUM5VoNX5yppWw$1T
zbU*R+D}LHDOKH*RTy^m_xx4EkCZD^#<M+q5RDq`KQ#0NrRkt?g)}A%3<ykx7W1OQz
z+dPrP8xqr3Uw4&Xd|{P9$^|XM=#Z27!R;Pr!eYO0H~RQYyjE}_+PK+OTUfHFYr=1V
z>r3OOxl5+?I)BjLaw$L4RZ%c}r<-lcUA~4-ZP#UIX`WuZa5*E(Lz4v^YghX3YCp2d
zy2Iwz+a1AD+C|qlxE-0Iv_#8EAZz1g-D&2GjIsAzXTEJ;_22F9$u|2qt}x}2FP_sc
zc$N5kwpCOyeq9%#v^Y>xQ_!<{#&(O_ijvn9l<t{id%e)O(|-1laPo|UOb3qCM+kqh
zT(;yyY)<vZyRy{`+4^cO{?hsZ(z8#NEDj5J&M@Vp(bmdk*0#6VZ)_2)@bh;qD68wA
z!!%dSEmcg}G14i*yx@^mQS`CR(ZBWHt#I<scj?Y-E}vn$dRNFI4~?@`CIU+)ygS&F
zVXBjSg#G4Gn?%`t%FkTh$<|K#^<dw@+4EoPoa!%$);+y=&5@+%+2U6X#XL<#e`Pwc
zmArR97G;^Qrm{dWWzwr%>WL+a3@kUZ7@T(Xh_?q9%~&ePDA2}iu<VVT*THRn)=1BN
z@9KBht}~G{ovnn!TX64@D<Y1G?`F-(Vm@H%U!I@YvNyvvoxgH^$ieo7z8618Wb8c}
zx?y_8{0l;hHEwh<G6WrC$SS<=6mj!Ycf-1FyHuW~Q#v*Ilyp^vHXl9EUA8VH%ih(2
zJ?8o;U%^kVFD6CgDLXw|w{Pho9i?>|>#R+RA8)<B+4}Iem+3!N_SvlrnmJ=)OoXAp
zwkW4vzqICln|o)=Gu6DO=dw?n+^e_E@&2s8y4RE5MkF#i#HQq*Vr~lR@{a$hkhxH&
z|C7s#<=lFIHicR!pV?&?wbI0}WGy>`>E`n>-Q11l9Xv1ZwEvb=z4XdSsQGG(%D#B9
zFf%jZt45713ZAdt{pfplw7l{S<C?uQ1cHNHqL^>wZDV`r=pD~{_1cD=_V<OS-qhO^
zwo;FCN#a(=$4olA-Li9cR4Xv7^-T(xvQYbYnX<QF-nljJ)=z5YGmMaUZMA5tWSGIy
zV`@jY7|v?H<#OCsl<C7^A<g+h#y&qUzPNMLCUL!!!VBgivHr=O`5&H{pGzn#`Qo{)
zuw6HENrA5b+eFFK4Q*dU8Jl*gPYy}XTfua5)5Xq>{*{{q7~G=!ygnV1xVrm8_<?u)
zVfT6G)faD-eailF!i~~1QhA#D7%yj^yf~%w)E4<?yCk%pu6h*Q&XH(!N>%8^#9#jt
zDsEpsmpN@+!`ul150^Lvl$@E<RbsZJ<yxJhLsHiq`87w+gj@ce8uZ#{o2x<hz3D#k
z5<V(w9n25A?!<Py&)V?g(*m7`+iwY+zH5J0@>{Hp^Q~`pTD|;uC*DeXZOxFVv4T}t
zd<}!ZE>GU#hBfjwe+#lUvK^RyVa|-ZmOKodGv!=34o|3i+<ZbngqicqRHtiZ?fbG{
z%3b{tlDkjw&M!x`jc(6E*!HPwdS|Q`D{2*J_E1;5ed+ro3)5x|T^7%rnx42)3D@eI
zjH`<;Tim!Em!CgvT@;6I^U~!h908x_OniL5`~9^W%n{XwikoI$`(|Yr!TVzBO5cbB
zN4`4NrEoFcn!WVIm)}(p;WKVN%y2%W7BrP_H%r==NlOyNua!)hxA#nQ>0-`TN^Nh$
z9M7lidz;94;M+{!xS5-`GYXhqUtzgX)a!B<=kj>=hLr({Cw4KO<C$?{$<MZqS=o<&
z9-PO;dFgMVgpP$;L}%Y->j^uowq-g-WQ1}YnmI$m#Qd?PNP_1}1uezNA<idqZZ>$V
zQT(upOZSwD(eJ5~4<9b9UvOo{)MLe$0xm~xopJtYuSv50>C%I)PL?NJ6?(*foIa$n
zF!k=GrHjAwzda^2VcI?EcV%C^FYI|fEq+b6UFW42#?P9HS5EOz6>uxKRw`AzeY^Gb
zKNn+aBVt`PzUu3~ll>?wyz^<Cy2R1w`&}pA{y&zz{`}IlmT@+_L#HTOn!04o{_5s6
z(=+Y6!b<BL`-*kFrOAhL!lpa=y<@rWxi*wTi2G=p;k{s^u-K&{w+{){MdeG){kD5X
zx@6e5qV$>*k_#*Zdhdl?bM;x7QkKO~6W?>@(H#FQ29c!<7sA=Dajn)B=L%v_csbQH
z`$o69H#n=wteV<8ula*Q!_j*>3_+0-CtEkTwB?C4%sIRACr_jLr^QM=N2gw@i+Hqq
z!L_zqF;`fp1bvxQ_vEsT>Bi5lMw7oNPhM@Hl_sFQc)iDur!EKVRy1ok^Y64xxgB{^
zSm)77@A*#ymYhHFlzn^inF|N62G(9K$WL3x8RkBJNjb{`-^=A^ByN5>@X3ntKC{P)
zd!M>XR-eewVm=ncGke8S&!i=(n;90on`odNoW1k*b2&zpWhY*GtV?^|qnNJiE*r2l
zI8?lM-I+_>+iv@3bm}=JAFJ&QzCYbk;ezA_lNn_T<m9Iw@koC5aALr<^Q$g&onR@e
z{WbZ3u%ouuLZNAkq8Ot*8_vv|_Qqc`u={sV(w_<&^+GWdu^9)Gk|$2HJoiU?`t&~i
z8tq5Nve(P22Zw);zc(-T)Wz`B$}hW|jcePZOTW7H9jvwMxu_feYN9yP1)ZY#`{o>*
zcIIl+mgiB|@0u)n-+Je(B!j?C6|q?FIq$U-W2=t-Y<rP3&v4NN-&Ok?l}p>1ZzWw_
zs&}qbk;`#O>8V}*mv%B6TADuD5gJkBHu=mEp0jr*Zv0y0#<0z*THpT1vrbTZ<>!mc
zsZypHf$2OkFVE*ZjuAHESibTP`?STYy4Squ_u^Q&Wx|aGP7@>_ZgAf9dU*`Tf!e<>
zZYGv)QBN$N_u5gg|E1ZAo5sr(*e52IX<aNl?aw)P!*vZ!o3d<Wjg~dtJqwiVI*)E}
zn8vX5MN(Xq>!1F6e{QgxUVDD|;zgO;qn$D)H^12^ck_*0jFzhNgpz*2B6Wc|+A1zj
zZG_H9acySs73t&Ne8*ksq}$amB|Le@0-jxazAe$bGHHKNPRmv9VmrZU^_R+obI&sb
zhbh-eM0U=9o#wnPbjzeob?f5J{t6T6T%Byeno_T(7Ok-7X#F#Xh@=TKlq6Gol};~=
zj+irR#m-|qVX?m~4_eD`yt7*R)MBmu#1n6OmxQ<H?=ODVrTs0|=JTb+A1~|gop8I2
zuURN@j*?935|!`$k2tTkaj^wnTdtU)v%#;st($lA&P_gPnR2!7kCb^^nv^VHieT__
z-5?_?v*ms3o!4`BJpZ~Zk*#}ebL`X9Ia!*r>EAZE2urUG{&n%{qnAz!HU3dTr}<Nv
zme)-vbIzWkR4*AS9^gB_W<$zC2@gq+C2n80Fg}T^``{}IYMB%*WZb~e?X%&Rm)V=z
zzPGbmC$u`SJWLW<&!8y%kio$1@(WL+pz!9<>2(rqCT`Csu`XzO9`bo@ZG!Zzesc-7
z1zdX{srBR(<U3AJS>?T7EL}D><AEPfjlPi6xz<3wWI^AjazY<HS;R8`w>o^-T%7pi
zn(E$<I*P3KEoSa(I%~6u?X<*Oqlnf9i+KxkMW1ie$f$ibJGI1=?~p;ugrg2uY$JJ+
z9K@P;ygPZ3Z7)l@r=5yr@G;iHC9#k9NZg)wE1j9=VF=4|SA{mMbve>PowsY<HXMs;
zO_*`Ic)`>%qn-zUlHUo2z2wiZGmBc$E%`g$;N^+Un~(R1=w7g#vha!f^i7L4yA*U8
ztv&D|^4baOg~j#8ix#!U>?*Jkec`m){-%qfB7d2f?&-z;`@bC8*PYH*WIvsE*ZUI{
z-4#h$KY3c#F|L2_y7k88O#2L-3oTx!)*OAVU=-9A)Fn83*H&4E?)H;k^p-9&DHZ&g
zm?<Esw_u^XlSEsq!l}na+WZV{x4eSleNQs#USC>&cj3%AZpPWcC;sj{x#r+D;SKM1
zvqpP+92b+GzxLmoi7BP61<zDU1*d-el$5w!!F81>gZ|{_ug$+beRH(j6I5PGbj~wd
z8OX4rWy8%qlYBOOi`^;kquXocBAG=R4z~)!7!!_KH8?5sT)d*(GpTLMJ$`SSMrBEs
zNWr$m6NjGfVi4e2q?lAJQz+2J%8+H6-!5w+){#==eo4k?vEppm+m>6yg~PcGO$47>
z>MdR!!=ZSRMO%GoRrA9|8)KB6B6!6vW^UjSTe&~+B6Iqtjxr@*(UvvULQb1%0}TcK
zo;dKL?#XrAFUs;E;rc4YQGO|>w8Ar8-XA?9mObf(fyBWCjq{V<q*`^EwFUIVA6nZ~
z>mPAqs^|6Mn{K<^d}_o_JP`5z`kJY8q1OI~6L;qbua#F?9-vel&&(ik)<>)J21icn
z)|uxdljYf$Ukcd$%4Bz{$nsnLJj$!4Zh3xf>qS@Vb-RCsIjsJ;<K2Zbm-Y7+Jl=X;
zJaxzCKjE3F?|gqwkXauXabL`{eoM{F8H(Y1@5y965K_*~b^PYFmRD>Q!?yD|tFCGm
z<!Z1P89J`n)w-~C<>xf5Qol^TGhRKb)-Y{!{X0+H=vwJZ_q4V)sk*7}RqvQzHadQ#
z@j!p5`4qVZCx@EZ%Um9AxV!51#^Vc`%UwQloz>0Ux9fQ9Ue09xgtxACQ5-87KbLJ*
z-5|O0tz$*O%2LCw@K;Umf^3$#T=^!tEuh2q58K3r$8)@`TUrx6-f_;6ykk{+i{E>T
z+_g)Sw*<#%9+PCW*O@4nt-aI1`bn7OlKZ!G|F#F-)_oW3_InOjXe~?UQ(0AwqT2<3
z%J2Sv;dWHNP+IWPtZ9~VtJeiP$nhx&FF#dygdsxTbzS&94%r+1K86AUS6&%<KMu-{
ztGe@%b+1^$!QE@Biv$+xy{Ju9R1gwuW$2uE=jnuN!2z!X_=<|=JaYfH<uoI+e)((*
zQ<j}F>|6#ul3!munmQ@mNGXNmg~R!Ui$!%WUtPNO>3oi;rBV`JmpzV8^SpfL5ks`|
zf&RR0f_$Hkw|h;vcv;A=Z}tMw>s>+2UKTZ1cQCy6SFt~_yJ+{L3%Bo9y#0Dl^z-fI
z^Ji8VP7djgixWR&W!fOAvn%8k(;OwAlALOBFNPqu1{s%A8@<@CmEC$?b$R;vw%=7H
zQzr>bHj|jXg?Ym?ZRfOG|7KRQUY@nn;8uX+x?H}$uU;;n`(j30pT*TDt5_3KH@{h;
zkhwiDmSy3WKO2^mcvS^lKQp)d-09l2ziv(WSR`EZ`s?)RX4fW~R=erzyWH-6{|VHj
zoY*fl<z4=lZ6}@7ynJpo*lj45a0qOhqr@VxeA#**g)T42>3`an_Pw|#yjt(H@{7o#
z{N-h9OJDw;7vgm^^Feizx<F~;krdk#9%Wn&>T9-~|Cgh7xQl1<e<88;DpqQ#R-c+U
z84?zX^6n_f>6H8wdPU6OcTb;&ouTvVz->1c&c6IVLU&=blgDY(!maYtnvON)@3s|V
z;+^1cmHH@$tLH)BD$BF%DN;I0>m&ld8qcu4vL+;%nLRW`LOV6gN9c@{i{!rrEOTO~
zt@*1otz)9Fwd{n9WqNKx#d;qeEijFFb~5*0jHtL`a#e|^QN#)FHzEcuAtu5eJt_06
zPQ5$$NL|v4Th&o}^CfSEVm{^$kux3I$D+468Q;Fa#O+j}yMIQqX2zrmUc9&eWfX4n
zxuA8@XYa;U^Om)KY;uhItnQR_aqFvhT^B<G+vdI7Ke60X;dhj^8LyzxackB|L9MDc
z<bNMkRZUe%T~TV#F{Lzafjf`bHZ68ztyxb#tE6q$JoG00^mQKZcW>Tkp9rd6sdlJ(
z@65d|cW!%XoMgXzQ9W|f)SY|M><u|z{S}y&v15Ua!ugL1o4PIhcumhFCL4eI`Pug1
zamh9(Q(woqzkj{l%6)pdnc21Ks!pjrb(b&i7QgSik$uCrL%g=z)DG;6xj8p2Qq;6#
zfnLi_F_Sy{B^Rqqh<@?BeQDrz$5@UOQ==vO7DeZ*+hEV*>}4TlzVJwmy|(?KM-x|<
z&3vC!vumb==ZqY^o)hfdi`Iqm+3Ym?(yU=TRpG1X+eWX|ckFgd=Uc@1W1Xk+zG$Z#
zM#ul2a5-zb@As|d@ON{o1ybX}N)1Ie9M0h2Je{z!pP_-l?ecujt|WyQyKj74ta$8H
zQOeBxFvSxN5o$I{4r*Jx^Ab7YHyIy0Z&b~~Uevo=@rg_9oU#&4z3{_)JojgLY98|W
z;&#UGYU{G<<)OiuYd3$H-|FtXw|<(~6C>`^ttnfUo|v@ZrqsQNo;fV%R`(rRvqn(y
z=8p9_WkQmA=lE=MVyC4l`TQ4Nvc6+S(yy)ySN|!y|BABxc+_40?=qhKLdiF{x<8%M
z>lI(Ew^QYYs8GX{MFJhSHe3{DSaEgbx1DmGcQ#*N*7fay+V`Ja3$sMGtz%tu^rq1!
zrUM7Qvur=2nP{7}?qXuemy;D{+Fx_e>s@qT?Bt}?aqgFNo2r&dNlFA`!x5L9$)|b3
z6g?77bIPvFD4S!#?b{=CBggdozE6LChWF18@>;8F>Mp#d`eUpdsI7BnLV?ITrr8W)
z)tejlS1jDcvtYfbq;`OGc$R?RN>M3Oqw4aT52T#Kr?Ms#mN8D4BYz}7_-x$6lKg9}
z4xcu-7O^WZF&)bN_$I7g?8CX!?;4kE=*eNqjS;+BA-V0n>z7ZvF5FqNCbf^h*4^EZ
zSLgDxc2%#)tlXw!O#a=8oW@2e^~w_2dp^wjc7QLUL?V*wWY&g57n0{I<`qtsNXc8b
z(lkk6Qc=BSiD}Ii`3VM*_D0tGRvdGjv`9la>}1OPyBi;<pK$fR*mOE@<D|}oqW0Om
z3MzX~)r$E|n{e1y^V6(5RZa#sqf=@d8B*$2U0V`od@RYjy6DfTe<Cvlc*CYlYxJJD
zP}^$$;nb!WxAb&{$6Gd?x32m#MYZI7TtZJDbKrF?z83juucB1hy6;_C8NxIpa_z@u
z!j?<y)oM#0Z@q53Z2!kwuPbeza&;Ts$<S(xd7OWW<!xzuT9;8vV5wrl+6^nx7CqBA
z(VOG3KH|`??)2>y>lSU@bZ_CTYXTiBW#%YNw06y#^<-Xjl2zjgr6^{W!%`0VSwCk^
zwGn@EDUo4CPMqiE8K<62edoS7>e-qcr_#4MU8xbXQX-m9OmJqgocl{j&ZRVLx3&9;
zY!ksEFU_SJC++#soBH*~#^Z6e|K7A-pKn$8DkOh;--_ywv0~upzOXWIRqC&&OJBP6
z?pR$Lt{0NFduynE(Cu|jO-KDtpPgtaxz^lma{uONCl9-sk!ySVZXRTGP;`*kwJvJr
zlbI|LIz{S%dn{%$IaJNmi~V4p{_4_lBe(6Rd*!EbIi5OH$*^Iq4VR<GhPG$Bc}^d*
z4!Z8T@1P{-=guRKSmwXZ`09H&x8+dvg6rGP6k5&GX-(meSve)`o5Z#?+rG=)7FzyJ
z(bM$lzh4)Zw&gGNF%-CJQLVzZT52XwE=TUv6GD;vM|5OgTB+?YV^^tYx}2%LI=7^+
z_e!j!cACS4U3zUNG?(WW3yQv-c4C8%0;}gDLpMJ$1JCy8OPv!JpNpGyG4xohU>DQd
z%8VjL1)gNK&_oI28&a?Cw=Qu>dB?NmkJ8S)%(r%(cb_5?<5^vo(lSBim{*F&_Wx^Z
zbs5;Le*b*Dndh~?SM$`%e*&jGy|{?yxm04`?jru?c)#iw>EUYe35v01J)KKMoXi(L
zixU6rA~KPA+Tny$kt+&DzxSN|wo|U5Y3c48ig!<$o6dRN^+NUc@)KS5IrrB_@i_cS
z?Bp&gy`{dL<5*=($O-ipuEG<cr`2S5wKo=pSjo<~l&R<uu_b_GQsd|D<i4&cYYe<{
zw%lGm-*)?#E1#E7>$Cm)=<sp5sz<BmTNb{$A~*YMtUM^P+Z;cy+MxF3=gzlj+^ba?
zRxY*mxOr;H6j#5*_}MbSuKp8)nWqc<O4Q_O4bsY5SD2qBEp=oy|3wLJnG7>Y0d;><
zuV*X5@0^iXQ&Gn4aQePVS=UCtl(b(~X&+5<zBM+<&hO3HA<$(iv*B-i@e1v*yVLYn
z`gOeveS1^2Xv?CU)8@7(-^ET2ZpmpCH4$q%=A`7Dy!EQY%GN7oy=Ny1#&{(5O1MmV
zm88(sxAI7ze#aW72}?K*M`#A~BrQGDCm;Ga;dsB0v508opDBM|s&IWinE%kskmdUv
zzs)Hzld>eeev1j2Rhlaqo8Md=^HJ>OyOs0%yH~t&y59QCtb0SOxaIxn7sKzdr0X{&
zNq!QR;tLO-qB!@YifjCGYe$a7<xLype|oO!Hrk@J_RSgL=BM*`w>Jn*xDsXP$2+z4
zS7PVB$nq@4*?Ic{_H_jIT|WBgf3k|$MZYtCJDJwJ<~zIQ-F}$^Gt=eoynU*-XYqBh
zR1U#IeQZYqHota1S3f^Sh+&O6FXN^oEV}lpg`t6My7sn`rP}Pv!@~6VHk|976MHm8
zQ9y6;%U7E0g2lJ@7(O`jV%LoKFP#+H><;YgI^=RZu=s42^qj^djom#73*!_8uKBlw
zU0W;BJ&m<{d(qr|Q_hK9yTsBEY5Dbuw!i)FQ`7es+)tPL^+n#c?*4NBIaXV5|GeM*
z{)<M<tKj*+F59*3{jpi7)oXcW;qvUb(AmoZ-kc2y-}5Sz`)cNbAfMH*f2DY@j<YnK
zt93zaQP9c`gEeZ;oV=YoxGucg!an!gf|!?_il>=m7b_bJhp^B6mQYu<O>MQf{=1M>
zm!EpAzbqHB>hi`e#uEh_uZWsz?pnrspyXEEQsK8VHVMcEo6Y5TF{3Y@V^Z1fJ2!8>
zi_7?aUrB2#*Q>qn($cf;zd5MPes1fmyD#7VZPqNl;_N)7>6TCFt<@bF=?1b@u_1fC
zOjj+xx8Qp0fp1e@`sIgz3k~$NwoX!iaVCT-D?RIB!;NXI32HOr{A(?PH7llW3fKI8
zD$VN4xuw=_)3Seet$zDCgX5RD@7*uAiuKR#t4+Tz`n1mX{q9A3%62`Tcl9;jS)Drp
zH_Kfs?~A6IxNLd<>~!^Wy?w7`jm58@4P3uN_#405)lK=|#HEw0jgD^fT&{YlEd8o^
z@KmSuIlDM^XNaaQ59izMu)as{0_(1zlG_?yf>W(09xU0qIqTNt(2ZHQ<ZhkXGP&re
z^Aexk5id0^f4i_uclXpc8+lcp`bobCv&vM>3Saf-#`=<5Z@skI>-nzt*p)GE*3a6m
zohqohEJ%#OAwKo`7RJc$TAr(3$1Tmi_2ot@OG)gO-3R*eszoEk-|D4Yf8p2?C+X;0
z=DR!lmZNBF$gT|Lx0_a!`0mcuO1<dUv!=T&Nh|fDUi1UjEvMsJ<%BQmb-&rzWA-LF
z=~nFJb=)e;F1gO`GvHlRxAU^AwN|;!fz>6~PA}5Bbu)RTb9(76t!t+jMZZ)@y|}jH
z<*FNc{d3ww+%Ebp4G)VAsZJGr%Tty5IP}z6Bdxc&vQyrQx@zA(z@v3_rS`Jjmv7g&
za;A#qtyrM=qG_Ra`u~KihizQ;e0j!e^OqFcTU`C+#?>Gu_O9mrw~xhpYMwZ3sycex
z-*&Id>-Bq2+He1V;M>Py29Wfhe~Z5+y|+o56~52Qy72L>*?&$&@0+i8<<-l_;`hJu
zt^7SbzLvKkYHQTm1yOIy0`uNa*?TH0MC@*tf#dT-2?l!u#JI0s{q{4dpfvxNjntw|
z>!XDCzH?I5*}CfL%@n)2db!MRx5eKSke*sQH|nv6WA5}>qE)YhwhO=K`EzSaxZu)t
zIxK<{Y*Ve?_Br~N-Of$6(qOq7v^_{nHiT=gpDNGlrFkN6XQV6-b6DMQjp6d&vLJ`m
z-9M9-8fL7HN@ZAZ``wyjcWWJ2#%P|uzKq?`>gvj4*0&i>U3J<SrBSMF{ClsX2*X<o
z(^N$t_Eg11bLVanXgy#&ciYPuaXXW&au+09J&m)>-ePdMH>z%_&V(684d42jR&*Uz
zwoX5nTY4>>FLrajdR5`#;=;|9Nwuk4_usOaz2(~UTXW|g3fUU9wl2Hv+uk3SITL$+
zK8t>Pbhgr#pk>Rao0(lRH=DfT_tmKP8V%X&Vi+#m%h_A=YxQeyUkBmMx-C<JCS+?E
zMogS3xa(!Rs=2O*@z;n_efv%P6TEaUR(<-QQuq3ka`<G`NxWK$OI~K$bOfb6ejGAC
z`WC~5>VEa-P45z4T;KI+t!ll&RlDoa+3CUOTfJ6R+zk8whv&M6X!q}5e?z05>GI3Z
z&71%FqZz|A|2%m+tGzWvv*(qDU*gtgU;bj^wuKY&bzfd@TgvcuXIScDxBf40ZmR2h
zcb~8NEB)%>%g*I>3xlGv%YRmA<R5x}wEeA|R_!87!`0lbaTA*V<@HOaZH~C(&b_lY
zlh^*!JpH|A^w03U?+M>&vo<f~NQ#Dr>!DkfXa7tNi)uS#ocdZfc&Y4R6{CIz<Iqk?
z#ba$*C677Q9=n##+8}4G|6;Dfo7W2$N7sv;WMg=9=Bf4Bw`E4f{!jfD>~8<IQ!c*i
zol$Ja+hW^YZR&BNB7v89Uh&3tOj2WrI($9){gK<3A8mej>f?m`-LKAE_P=P>-tV{l
z{JHg|{+II@r>tL7GVRuv7w^;q!!NkbUZMR$w=br)W&T`KclMWscAS4EUcVK)Yl*m5
z_?l9G{SdF6-822Jd^(x?^V+3}CH231!sFjFeXuP|+bp^I-J_hH0w)Z<?0CmGbMo#@
zF%uQrTaU1q^u$zO{pVD0^wK5Gb*#_MBwSN)Jf2!v;>EYkZ@ya5I=xzD>2|K8P6|_k
z&TI{~^;Hz`TIv$KwI;ILR9u27A#~c)>$O%J40zZDqwHQeX)P5>?)ZE=W7ST#A5Pf?
zwQD~(WfyFaoxl3W7mf$j_kK+~z3kb#{a=-f6FWF#_kVk2ZS+lI&E6A>mX~k)8Wg{B
zn<uxl{`bqL8B7*5Yp*wYTeflA-qMY0r(CLPKcThY?#qw{t_x~Wt9y1_mYcGgRZ)xK
z6;rH3hu4}%3pN%!a9Diz+S{^d4$du^n<I|aMXU-t928z5${^ulFt_aV{h6;i`VJ~+
zG0Og4zv*7j?zeml>y@Hjbo~AM)p+8boMn%fK4+_{Hho!AZgqEVTDJDe%DFd$KQGGR
zVz^Wy?eIHhz34L8`s}|xEzN1aWz15QvhIGp9&D(8?!_{jjBD}!w~N;%WoK;_m|MTq
zvNSSkmh6{3kG6GGJC^_bTJp-{VNYJ(dNqrTuC*(_#bkdL^?l#noK|fr`J!(5Zs-2F
z@BB&<PgjUDNW5oS`>=kCE#IsaSC*B(ITE*|>d`6Lt!*z0em}BWcKGeo;Fl#kVm~c^
z?9^|^SHI|`_Ok7>UhgreJz8S@ZB@JPRdXk<x;qzUe-1x$kxlpRqu#rwx_7e@+rR(u
zou0Wc`pfo9UuUflU79)nV?ku~s!+BIY^P*JQkUO#I~Fz5j8S2|(r&kM#n(>8Kd)$S
ze;uc4_;y#gW;EBSp!heFF8KKzt$ov!vC2zw*Beg;mte<`7pkcl9Fc8)K1<((zBvDl
z@6d)@uF?%gFSZ5DcTOq2wJTtaoLSZC2-}<&DZ8F`$(et7_U7TMwtG)LKd>t@v0ZvC
zfS-N2Z}xAQBU7zst9w5>HTUcF%QLr^T#xrRPu=vh>)GiBd3Qab1F;KicT4k>%-7p+
z=jzno;&biYZ-<<^+I{c#b$y$QFW=4iy>~~MFXI7+?>{frPW`BNLGfaA$f;0cd+V$F
zS8)GJ?f(1DF>uA(9MkK&UFYlZGpsep&F9<po0;MH*Ogv7ciokDS!G^b>E`maH|sD%
zM@YV7*qKk~a`Lruz3;z1vNCh~?YZ9$tUp^N8u)6FK**1pD$y;s-|~6Rw$kYKJH7sV
ziR<w#)^8m{<<^8*H)pi(SiuzKX0}alUDqk4%H6`vs@Y#7Ogj(0+Gfq&yK_@RxZsgF
zj0?)&PYryvNQcERB6#k$DREnpeU`^^EP1)bcllYKY2ln~6BUKDZ{3t+xf<2;w&>>S
zkTAV1*;ln1gqMZ*ELswA&;IG!Z$3df?{`-`Yr0duL00{!kzka1d_}@jC+)ktE_~IJ
z+u)v2yZpFZ^Hy<&3zxG_Cp`b6dH4Q}8$KskRx{4^+5hy_b~Cmp#)?l5*X!obTXodR
zZ|m2di7Na`vdm|dI}aMU{hpV?J6YgU!Qzc~a-LZ|zYsH7LTA>ut4U{hSqv{#sV-Xi
zYUfhk;w#J#^3P3Rc-^&n-OK&ks`4_Mzo-`5Ta+%C$?Nle*Pj(Z%?A%%i+RswI`uAt
z-~NBRb`8HCY?x@vP`s$d%I@s%lH_G|34T+|eb^6#6(ugKOVHk_;J2(q;!(<}_q#vd
zl)L9}VcM69x4w37TfT+e+_TAcw~M>#mnjo%t6p0Mwv}#NuHnV+5WhU}hDXmj?v*?;
zashF>^x5-$*8BFnb8b_Lo?p%KiTg`Le@UR1me>036&k7aOKiN199f_2SYhKhb;*X@
z<?XBuhtC?-{(9k+-hL*5@kZ|{EjA}j;kHHVMJ;!=CeD!v7FViFKD?kh>c^gsGyQws
zy<VWFq~OsL6(7XVb!qwQQ~ciAam9wYYbwtEdQcD)vCO^R*tjUU^vYKApZd15o?rTB
zv9aKJ_4RjElcw?CuY2C;mHqtZ`hA6g;_)}G-+OZX)%(aB(^Gcus$p~3)c8wg60gX@
z-xIQ`vy($N^BXYy+I!{-|CzUw8CKZxiLLs4`EU1aueE-64Xl$pl_ZyU%}`0&kgzvB
z*4;!-GUq4vqbmaLel5Rd2CsIBl)QPvs#dDO@>QtW`X^#mPUidnT>5#&JoMKU#lY8V
zcxFoID6QDBe)D@yx#?@X#bkrF`5bp<&(XGQ+IP*{*Z4XIV~kB{R_XtzK9O&CSG1WF
z?0K`!G9Xjd?$g9<zIOI|Rfik3`_H`jnO>WI44lX&Y`g8cVry9S-$$>K64Q4}EZKXF
zPis?oUHsbzw-!Wv+_m~__*ARwdEA$xrmk+))0)~P;c_g^!ie`;P42F4FW;^*X@}X9
zI!z;Oi(l~wh97$oadhgdlJxZ3C)(Edg|9xe+U4jDrY(EBzG%%264%X*s>@t;x%YBe
z<cYqE+Ht`e%2Pi2$~sgoH;JFgdFj%v)*Zzgw$~eMl`3C<Yt_$Xe*RNkOZNo`+<tmA
zYW=Kh=^>}@doC`%`^m5K-^r-jh1O3?m!<7mlyvCaxo<V8$IL@bxVNz157D0M7_j!|
z$8FbVe(b%m%Od#pOZ(!a;{OR=8q5jt0a3lX*XA!VkE{M0Q(0|#&-8Ke-G!1?PE~ir
zr@VMP<>|#+e*5<XYc1Fs5W9vWf1$N#McjqCvtDsj&3VP)aHiz&O26enoO`#PTC;-T
z>Px<5yMHgg^|mTJO#7-+%(}2usg0XfOgYL}-Z*d3@s~SnuCDYc)w&$(+5gqAdX@IY
zJpb~BZFAmjdAaf4)aI*Yu?y}UOb}&ww(CaR(LE00Q}1a#Gd78SxmIb#qU@92O6xo<
zj>d#cO1pDaW7j30c-3V=;=8)NBDd^b;nRM%Z1)}4r9BV5s@_P&et*)Zziijxx%{Hx
ziw>!Ar(SGZu|#$X>m|OfO}F~bY!Ugp!shDAC38Q8?6$o+HFz%9T=`#{qh5+B8DE_m
z{I+bXl-Q!V8$d)A`_)y~um4_V{r08b(!0lE#NV1p-Svu1P+v8JZ<&?s{^-URQ^K>0
zts8WHO`N$tE=+80m!*Nc>*uSdg|-IX(=2@Xdf$SQt51s)Z|?b$RCITt<gOJjntne1
zvt_-tN%5<fI{NjGw_cAoeXqLQe7<$zx8D6VFHf57-4<kLv}1>5@T*qUoA-Lswb|2G
zhi<ujk8c%|ZtS}933++#Nm_RL-PVeKy-PRl5I$EG?D%SWbZ}~*c6Z)RQ-+k4nG3ep
zymyEUd0etwXI1LO&a$!&XCv7P(ZI=}>w_d+Zmka4<E3_($w6CKc#7+C?wdDnn!RX!
zIQ6ZW($<?9XQpWH)s0nsIW=HeM5f4vte4%~y*tmH`*w3{N_+rQz`7}?r^c;}e#!KC
zYuz5>7hBlt*1ibUN>%JQ$P^%Us3b-4#gttNk4}}`+7uR>{dP*L!CW(e3Q@)d$+zB?
zsjleSv|`)s+`Y}${Hj;qG2dmnSu}RS?{#s$X6dKyS3LUo;KkM7dh>41n|Rw;s9|^7
zYWvc>zkfcjpC8j{eX&?SZ_WHK>HJ%6zqR#yZLR%n6~}iaZMhspnJs1vYj{?DeJeb*
z)|cVrmuMm8$OT-v#vPx;d1imwd9~?_dgYmb3692YvC<ij9dd5|T6|#UDydf=W4brL
zG}_y}-;%lEgdCURCb7~dcO5nd?aDm5>U64m)MPV{V=EVKI=JxfR7Lf?lh!5u6TEh?
zPgD#KReVz8tYWq7l;bb==t<IH=hUpX8^o3RS1;IUI%l1xM0Q(l?b5B6@Aqau+El<J
zbto!_;c7pRz@?_wovgRb*h9q9wcj@+PLy4+@{7OMgr|+sAK728d$}t*vOBPI&S{(K
zv(e(~0^f0YzMA1^u~um6r`K|a49+pkbd=z_l-Zg%W626Pu4^5QDWQ>P6HaXr>)2Pc
z_HB3L2^sdFw`PKiPRz1&tGkx!(e_tEyU@2s=hO!IpE=AAO|GA=lD+0?e^t$hjU#Af
z_@%ftyEtsM9J*&^bL^dZmeW@6ZeL4%&1|;om&6YIwkw%je)P}LrJU2k9~VB}dOg@v
zC-UVa=CJtHeRexn1}(kBds*`J_KqVf7V$7z-WOOW?8W&yWPbMTb?h7T*k;;0Wu4fq
z=-2y~(e2A*c^8SbVlw+;6(^JmZvA6b%ewIu)2|OXY0B=p?k^2qsIx85tNj#Fc%Emk
za_Q{0<Q|_o-agam=S&@*qzemASZY00rfngE5u;da+Prtn3bS7Q*!OE+cRJtwy62yo
z%LP-n{&^6~z<0k+eGU7YPtD~QPaXgNx4-7!Q}w#P``G`T&HwY`{r`i@SH3DZfB&Ji
z>NkdKaS2{-3|S}YwtRgneDX{5t<}A3Uy>5t`F}Con!WajgeL3KKNUPmzkMytYMngX
z*tQ8D-C!%J`{CHeS6qtf-@bjaWjyjk^+k1)u3KWFn7rE@se8t}vz{D!(5`w<k&&q=
z>j78gKaoyG%P$8u9$mCS+P$=KSI3=KzxwQ6?&8|%)|b~G_ub4=PIb*@k?Z{4w@hxb
zbW80o5SX_0=fsJ|b1HL~3~oJAzxt{5{aJD5!0y=+Z(S{z)R-okc9lgPzp!q?%_PU{
z-3v0Fc03hz)Q;M)EbP{{4mr=yyc<Q7wX$uG_HnZ8Tyx**kbz&&A*-eUsk<9rHC<R@
znjv818Q9D7fa{@Pn_TU*x|FsTmowtdeN%p-_wD=P1z!_0Z1r1I()B;duB{F_b*9+V
zFnZo)k=8f27%#l7EG$#>>)l*b^h9rl`HYLo1}@xhF0eiLD_`|IrsK<(U*@jiIY0c~
z&fn?HxNx=j`Mk4j#SX=OQyNdd?N<4F{d&XOhqf2Xua_0~zX@U7w$k`-^Lx%s`f8Ha
zM(#o{s}HF?c01z{UAI@P`pHA41cB6W@z#mFRy^B2ms%hEENSMZ&bIZ&V%y`ZyzgI8
zxU?p8gR<wdPS?*4--UO|2Wv9FSR^t#i}`?$iiWO7M^{keY5sez65L`@J5RUg@1Hzh
zb;H*u>;Igayx!*T>+iN7*8ltKztbc8bV@JZ_ebXwKr1CqNUZT_z7l@&OEj19@%3NQ
zXHAIyy-mA{*Fj`G!$dh_g|x4trn45LzWR0MaE$J^M^`d(W8Q0D=hEOwW@z@feQRdN
zgd+;8F7Gk$^Nm#qi(Ry`E!gL3>X&D2);)c#GUAOB19w;FI3_pVJNi9kPS&i2m(JYg
z{l?L1tX?K=@JeUa?(86OrzpKd+pt(osgoBfaw=9UL?4|gv2MqyQiJrp6S?^wmB>1r
z7H7!%Xrpt~lu2#NDGv32dA)9ItJ;2a&U<?7q)*JoUhB7w8hUF3Ia10(rtuV4Y95Pr
z&Y1c!{mGM=l8x7FzbM=Du=c#mKK%RSj8!6~jtPdhXPmuu*x<=@^~`1Ai!U#lqU5BV
z{O`@t4Xgjwz6)HPq_Aq@so#B4b0w@|YZ<&Qrb;qKcvg4co1$vZ^hq|>Q(yIZ^83|y
z*UjlXSbK!?sqnO2?=GDAZC7&H;az?6ovGOm{3K5APLlI0pUca-VB^Z5)GsD{N%2>I
z?MPE_)>KaASkO{B_0-R{4Xm|auJY7pNG|YX7D|{nbw%<6Po8OVE?eUI<a2fXxEdu~
zR?f}VRLJ(q=n>mBQ(NUpYER-dCxsX6@7h(3innWMrO5}TPfDJBuv*E__w@S9o4>g)
znQBy<%k#|Y*ORaWzHP}2?fLsJi$?8~=$;-Ma5SM??&nSWA3NoLe*G=?<LmCahu{A{
zxP1QJ|Cij}l9x^$ofrRSeib-Ls5!Xs`1qdxzU_MU>U*aaa3%b0S$XE#t=Vfk^-Jz<
zHxD_$Bi8G_q(Ho><-;ro2^pqNhDgN;Wgb(@(hn+}3RyabafxSE>@Vfqf<r;EB7WxW
z-?hHmJz>&3{>y*2{KD=^r7pI`@16%1_s080OyFMLd$IK8=ND%po#GDHO!bc0IY+Gh
zc0&7|=h4r8R9(KJ>-pHCQi@@s;kO^AI&7zQ=ye-&a(z)gb-s<ECST6sbo}uQ`HbpW
zDvUAZyb~0cI32j4o~_?;X75}bhtHKhYfbKI>MFBZ>~u<6YmsZ-5?10FVb*P~aNECV
zfhwD}_r1)cl4tgXMP9m)p)_q$NN~Ku>6V4^KmQ!m6r8CVnUkOM=F$Pf?;g9Cmg{@v
z-`Eu3vDQ&#TB+pH4d0Z`ar*llxI3Y2Hs_NES1N>GbRL|m_b?{&a`+zWFZTq>r*`+*
zd2avq<;$+w?BX|0Y`ah;{rpgzG=sw3FOf++se7IMnH74LmOA+E-kB<VQ0-Mw^6yIi
z!$uL8Zkc2}50z>B%yIkCu`{iLf9r}>!Xo8FcX_I<sF{>`+A-yo(xT3!B&Or@@2anO
zzU8T4LoVBVsr#p=@c1lY-1H(~M(x@n9x<6dt?wM%+w}gvu~`1~&ENTU|2|#6^CmI9
zr}b`ke(@W|HRjb{w#*U?@Ot@=jbW9%j{Lrdt5`2K<T<=P=p(Y;MN9jjazXNJTmFle
zVk8XTd7hb&k|dz!;wF1BZApuR*~h68zZ%?|Ii?*nDiyrAW7!o3oprPBdX?ndcPhFp
zXYqY%McL`2^^a}aR-Vu1eBIeNMTuv#SXasO;?3<Qrt0h0&vEK@%bkAY)r-k*roX)>
ze!P;S{7(7#w>8cJol}!HFt9Y8-K4s}(r~(I^)`hDF(GD$tFFhA1#cXZn8(oZ?4n<q
z?_o9e%!8&`%Tu#{&e^T3lwG*E^OutT>RsyX%fgzz@$i@iNpRiEJ)N~&!r--P49}yi
z93kb^6Q#bzop}-yc}c)2t!fRAlcZN|P;vG-&RhQO@#~%hKb$9H=epCqFG=B-mF&0M
ziuP5DI}a8KIBeN;voTa^W7I{7xu;!%4f+(7w9=o=>y{}i7Y+B{|Anb?{r>mP=8O^Z
zuVfWGT6f3vSMK&VyDOG0_7B=Mp*+vPeJ{uFTVJNW(SEsa!~CtkMcm9hs$6X@#7JCa
zH9WXVZ1Vzved3Ila#K&l7~NbvnT^|FPTKy&?bH1f1)4o3D~d5F?a54Xds|heyfMZs
zcKP334yXMa8|s=%1ur%nHaPv#*urRvKyzyAyi#kqEC!teTBi!`A2kNK%P?Zirdw}c
zCv3`+N&dKb#*xgwU)KNGG~0H`tq*&D1owc}sCx7k<;w?O{=6xGMf1+``<Ei?wsFps
zxO!ddxRc?7jx#)x?Y|N=rz`2OEWG{Tro`OW?~Wgtx;L7=RpHS^<H>WY4!w+TJh~-t
z`oA`=Lk2otva8qKW@r)nb9UyFNzL4hYQHSjY+3MNk0OI6D`);vV^z~lj^CyI1dVbZ
zXl+z8ig>)2TQf26&@TCDyJoD<K6-bPZnDaZgP&|2D+RKio9FyVFG*U`&CE8DWx|XL
z>jLz`66UoXaJhHjvRNtH1c^lgFS1Wf<Fxmj`TlH9rmm)Hg!20|w)P#Tq|=yl<qy1h
zJLC1OkE}g>&I_mTFYXl8@H+J_)m^%+X=3CuXMa--QB@U3?PV{1O*|=}@;N1Js~O*_
zeVU!FH`l#c+uL?RV$ICJsIDNt4KogW-*m3LAn$X}w#>%P6`c?5o*vTA%dLF8^}2uj
z$2_I3ZT|Mfoq98cJ40`sO!_Ty>_x=AP0P7E4I8))Wft7GRbLXW;?L5grgF9<!K;kz
zk;#I(u#nV8)k?=zxBtmWzTfTkJL~g%!-A7(OQ*bVH903Rr|9<M3-hnM>QcH`sIX0_
z+3|^JQ)0R^L*c5Gyysphx_)EWb7$c-*OY#jZ(DC}hzW|<7Tdi3-TpZpD?Uo7uh!eh
z+fY_&`hC??_V-GD-J9C?|BK%Lhui+AzSZ1sM`GU|-LCkJ;o58)lLn8)+bffjjTs9b
ze3-ae?2rKux6zvX`7840GhE23o4YK@>BI@1%C9HRq^#+`!L;DAUFS9Si3{acCrY#}
zR0wSmm=vMiq;GOe^5zHIK=CNe$6t756#q>0=SndYciSgEY27v5%a3`F%{FJ2xVDLt
zLoxQww5K9Hd{W#c_6vSF_H;$Go~RNOihufy{pG$H!uvlr7XCjd(|>exmf{mZ|M_0J
zSEr}F<+xjD<HvlTJ2pgX^U=iJNlwhI4%b<K1qyS>9yw5<&0VMUdSXuQ>J_%ewoH=x
zu9Y&2)-$X%Rr?gt>=9kF%su4op=rEP@ud^)i@tubW=dB{_|+Q@9*-84HBI<;=7-km
zgF&Z``ZzPKnO|EtONv`fb5h7<wy@2rn$IobB6mtRo9ft~iP<V*ee|+oCa**O&RjD_
z`|M?wEH6Gr<h$>i@Zyx>l6Pf_GV4`K=BHj1)fGwIQTuD=+so%K?A+y2Rro0Q{r?Nr
z-9~R-G8O8UB%C>Yu2sX!@x;k<iB92>Pt%Ssd9LQ;SQcEEb?5EL)qBO#L#t~Ru}$o}
zyg)4bw(FGQ)}k*J%ClI1CZ6<rbwFTRQ$vpX{5MJa7W-d%VN$bK_V^aFD@I1z9ow?1
ze(j5B%yRv%>DIo?!c?zWv})I4|5w}ZHU;j>x_{I-#`T3<n|JU>sq)%)6Zf9YnB8i9
z|Mcl8wl(jTPM`Pd)ZhPq=KpP$|GoVG@A`jJ=l{R-m_JuFdH>bDKf;d59jFR+_n)nq
zZo;C#9evb*#o&V0%sTHS+rnpENws8X<lpEj(LJs6*20e(0?|hjRM|{4m&ATL$2on6
zu-8k`wWr+qm5j?)pZiz#x{cw;s@X3r!tPpxTHdvt&3-7bInk$C)I=<%ZnZ#%<B|$c
zL!e6c_{wu9^)>D=-ntsN^ZU_Gqgmd|ITk;QQaZfTK#5n(a;uel23O;Zlkd!>%+ima
z<M3Q%`TXknl!8P<fzpK>Q4<T9b)O$`+mj%vlXada>r#xk^TjGfhL$CtH4m?mG!_is
z>AvV@+p04DZFfHf+?%IckzgpCsBkIvqETSj)f)nvMcd|mExE8qn604C`N)LJix%iz
zoa482d)m48vyCGDo)Briv1+%;$~$v>mR_B(vms(>U4+w%o!Xo`jQgXf|NAkOf&1Fe
z`LC}Zh|-R8-dZRB|Ag-m@Y1hJ`{`o&y`NLe<Em@@#kX;t-{dy$Q}Y*&2ahH#(!5Z0
zc}LfL1C@1ASKnIBpTr!-6mVgEsetAr#tT*vkBq+?mn&#i)@dz$(RwjsS^%HY<kz~v
z94&^oPPi>f4r=~bx!~YI+18f4OH<tX_}U7y9)$#d^qrGd#F}Rm%i;EJ-pt6J2~pCk
zD<|mByE|#U-I-No5klRoISe_&cX~QZKeekxh5Nv*SpR36+OyBw3C_3MS^4hF%B9!k
ztN%Q#|6Bj(g8jcv`JeIj9~=LFnZ5tlhrRQ2-}jWr-ssME1vMV>zigQ`Y0+}t1)ckt
z7*05xx{<-K_wCms8Vawnokjcte{HA{HICpZR5EsU(w=p!aJf^##)UZy1(nt1f3{{_
z<2ZNYRaN9hr5R#vZeht@iBZWuZ2c7v)Yi*Je=+$q$w@$Jab>s(GvDC}eRqy#wJPu)
z5o=!cv8ZI#gyS>Vw9Ac?uj<b2U9eHWM9d>*{)q{jTM8~rec!1&KjgoVgp;JEaZ$(A
zh@%ca?$y|5F$LUWNVLz|RwEW`T5QJR^YBT}%1I%`k_uY|StcyeV0mD#9(7EdsVFxg
z@VWq7YR9RR($=F2X{J`j5z~4vuUT{JqjJ>xDetYkHk|*xA?9KwSAorzAff7$NeV)f
zRexA`?vieHO=Qe)5aSQoJ3}I(yK>sxoe?%&%#L5C+N#fF<Mx`UFVf(0yys%9ZYAH=
z!;SA_bWez1?Y$lsKjmRnaPIuNwhEo9pXQ?5p6A@Y_kUserPXm&CS{lJgv|7i-t1EA
z#rNx$OO|xNP0dM*1UgihX9ZOqm1#a|Z>6-9ZKC4+;uOuyw|PEce1%d!;#_S1P3Zi?
z^g2rX=H68OC0U|NMHFiNFYOLqHpz~)QTL^&=ks$5FYhy~5|BI=vGG-qk-1-QP}eNE
zs0>%0u)C>BKdYw)U0}Z%BK_s<gjKTJ<R)C(u|P|G-J{;Siw?<LI+bg-ui(cwYxcS4
z{_vW=+fsflzBaS^ZB51B-P7OwUVJF@dd;)+x;N!=RS)!QUdYyeUtjac{NL^Ty007A
zfA2NF(v+nCuR57OVeiW3S#xr7)-b=wU4Q=UwZj2RqWHJ`I(8{)wP)FR$G}grwG{^g
zL`~M54PBJ{Qq)rWcod7AN#<`K+4ae<_G;}_%0C=iI4AJNgz2GqS>1uXuYLquvT6JA
z?sc7`rI7vp@RNYp3x!HycXxg9w3A<c>ymDyXrkhYj$MvVE;eZ|J@qv2hHb~YpbZJ#
ztPwT8Rdz%ks=5->JuBNWxzQwI%Hk!-3MHl;JwcjxbK2z1gtPglMr_~-o3l>%j22r{
zhxT?0z2&!-X|N@&=oY$V_+O=6F_uH+pqfM0(xqpXb4Y#`Tp7l(X2qSa*#?q@uKp9J
zuGrc&K|yD#qWYU3ORl+_@@X8Oelf)Pf6#%<e&*<191Z)voO!{>ad?B?mp=VpQCH_q
z)KA}SRb=*t&3T@GRmq{vR=(Zq*aGKYTIS3+%`5q8>1+0vSt<;^*X_5;AB)(^qWj*>
zZs*Szn{2FVkM3stvbE~hQt=p>dd}}#KPrU&zV`C+nUJSS<#YR$mKzwV{4rk>q0ch&
zT&M10kKl|o>9gOrwmRJSBA;d0vGUPeF(yaTHkJJFQ$OY@bg$gi!SV6Jv@0byzb{$L
zbwSs@w13XxmalUv8d4${qUv5PdmLTI<zg1^|8hS=<n(!et{!4yPTRoqjA2hjS@xaj
z2cK?{u-v(1%J1hETcnonJ2o}?H2=M=|M$vA)ql&JqV@0McGc(Sohp*DvYt-VUvg|o
z=B>9Sx7hjqpY#8>+5hj}{r|q#J(2(S#Qxvx?|=T~&$Hb>iO+4p^SD2SN&E?WGo6_^
zoEKgZ`fF=ytIr+1k#RwMyz$y+*A3Tja^?DIm8M1<@eq8K<dfWUFQD<H$!RM?#{S4N
zmKS}O>iylREw#b)vyDLThL`i@CAE$&n*QX+DxV4O9_Z$&y)cTHa%|DQwB`(#jd_ke
zTvFVfeJ9zRI8NSZ{iStCQ}@f^AkNn%n+qe^;~p%(b6fOzWJYb@uSG(Pr-Xt!9b#Q9
z*;Wg1EM>aS{c@WN*Y}<yT(X7}MDv8t6n%9z3+za`Ao0EIy?Yz)inddQ9y55N7_AJS
zELfYkdGfjwbAww#{5TQ>cuQj?d8AzPw`~bvKA|v?Z&Rw`@2FIL^IL1(R1P&;sGN#b
zo0;Of=~A}^f0m<1|AO_QCuHMyoIKcBq}?+yg5j4{YjgHXUtwk8V3h@uGksjQO!_94
ze%jJdz%}i#VdkU<TB{AFJ$QUZp)h0}tH#7oo)>p_?e}JR&wJogbNNoMvSX_6T2GzU
znreG?uAyk>!>*u$^$rT&i;K70X<c8cm?R*%tF2al$>(dI9BV}1@aGCh78NY;a<<;O
zK&*w4UtI6fB9Ai+6G{zp?r4^5*8BUK@2rZ|yQDadlMEk^ZjLT}d3lDLRe6(FUBs%R
zoyw~hNxX6?mgv^5>p1vG=+NQq_ch~dXPECc$)5UIO4j_;A&qHL$5Y<<>DhdIu`t^1
zV$D}qhN(Mi#pl=Txt}M$=jUUCw4Voe^WFW`Y0Q6b>ayn3dB6Sz^VhuFGvin9{}0CY
z-{;poeE;XF^}oaZ|36IM|LbXSy5HNaTK|$E?HYH}-=I^jHf{)>@X$eeX~?cmHkPVl
zOL?RV1^k$e44Zdp>@d<<IGbmig_5zen#mOTHQgP{CT)BrwOCXA#N5PJPLqCh#h?A%
z8T{Bit#Dh<h0llAq)Ia?FgPYJ)N9)mEMB_ty!Gwd7drgS+L})=blsn5C@k>F<CMdM
zU4L&dX*kO_e)D<o>FDO@IWLP}uf4PC*qo4=aZXVc-ygh=ns~4_cJ+dd0?QA7Q+V_4
zarGjB42R%#lUFKhu06iXckk09p?qCIPpn@RMe%JGW!~_Pv%&YkN9CWJC-0i+#j$h3
z1im)@lUI|U*2v$9kz4w<tY+ej#7sqo=?lDqp0_p}4J}X-UTgi*X~I*h>?J#<O^CRi
zm2-?!tLv`Ld(Le|uQMcK#TC@Q@466@H6cs5ZQkmGMf#b2!7G=%pSvm`mc>}_Zn6B#
ze@mRW{v8h&f7?6t)8qWNY>Q16?Ms;YX6MZ2?yFY6R3f9#wrV8aH9N_-vUTF$Ad`tc
z8@_Cta_C#yIVr8^=a!PoyJqZ<@?n(`{Cgoma?QT5meo1@%!{WiYMODFm;YMpjBk={
zij|xCZ{PjMsd#S2QO=fK9S2u&7wk3ux%2r4Ltcd^ybmYkZjnsX2^XE!e7fL;hPcGj
zd8Zbth9&tPiMI54_RhWH-l^p8=J(3hR=s`NTD~`&LuTLKzNzi+ZEJqdb?SK9{?4-G
zwL0&HPZyrEHy>QdZh!sl_j5KS{+o9euDben>-@hT<Nuuf|LL*)ul~CC_WzHI$JLY-
z*EX#_5m$5h26IIIxeK<r+jx}Lp8V$(x9#FIZsxUL7VO{kB`PV%BT!sU!1u92!;u$`
zPcm~O+NMe-y-e%NPk(CNHtX6OqwX_43fnS-Jomm^n|S8K)oZgABwW<a6;(2-zE6<U
ziSpx7yS3wR-|~`#H_Mn0d|%|GVDXhL+m>~KKu<+Xpi(oRo7cgx_j~q!>AJFv*Y8`3
z`Q_Yum(JeW@T*Ji<zX(x#||!MO}Ee3KOvNrYiW>{;*%xn%Cmh=Ov)-Vp5L+2;MA!L
zRf~$t{&e^(kKiyhS-GkIT5SlQG@n&)j8$9Ar3n$L%N-YLe{hOj@NS{h%3bW?pYxV}
z+@M=Nfv1}N;=N7fCvW|@u~@Y}@OYSsb&twbub&%onUBTZ+UWi9L4olrj|>xa^P5Uj
zcL@ajUGVy0--X3Xg05{m&i4JNcH&;W+O}JBT*_oO6iJ*unzHVpb$s~r6S}(!=Ux_S
z5ARbpezZO2it8)GZwdjcPV$H`WM5KyyFGqI_ZpXFVy~yo59-XjR(QJm#qBMkO;&!*
zP6|!OnxgMzJ8ug8r63g)mM9_g*2TcjD{*O{^P*X4CGYryt6ZhrUL3yUVrZDa#_gc!
zQS^1*K6Ng~MIo~m-_A4+*Yp>EUfP}((eSjbyG!F3Ys9->g{QKTX3xE|*z>aU_L@ES
zZ;2hUu730GXtaIhw}sU$&)3>~`gwT!ea_@*{dbv@`R@H;J~rcY-mk{G{qiy2-{q|r
zkF{BTe!lJhSKZ=zb87$0Twn8S-T8la^#49t|L50wx!;TL+ZMk1y0+rboJ4&!P!Tlw
z*tC<g_=RdsH(Tx9ICG16wD>gtz?%uSObxp@wioIJ-7aO!a=CGRBe#oDw9d_jxoa;4
zh_;3IWzTQT%Ui+ljm?=!XKBcln8z;{sH!W?`?cAXY4K{8<3U`Stj^nGwzab8)y!dW
zXA^v6vSAyyGgpgvM7fOV^*NR%I~JN6MhoASFUpIU!rRgswq;JS(4y%Vbf^7#@*^>G
zZ|0v90?n*0mb}wU*)C7{vZni=KuX0VA9u~b8LN5UxVIHGNi+){br6{AtfaoJzi2m0
zpG2e8OPgp{3&%Fs#N1t5E-#qyX@-?&={5bEyMj`my}9PuxG8_#Zq0?-Mx6_m=Sa`j
zS?ej$;doWRM$(r_=fJLA(d*gGi>%7OaW2k3sJctz{E2-l-%K$SPQTu>JTtUy-4w6L
zpVPn2x?;HXSn=tb9Zgc}B~v(U7HjK$e0=>`j;x+6&y{2IADNx!*>>mbi`v&t962S1
z_8lkY=p=9GGTKsPK1=9?V@gz=mwn==SxvsL)!SyPzP2!J=Gdp+aWJWQmxb!4nSRd#
zex?LJnZ%)cUzjC3qxbK_Z1d6+^AGWbP4Z{WX*6b)T`ND0EzmZ8|D0KlF0q9riYJyi
zEYWC?l@t=r$n|J?#<fACNb}Igua;@2V<pyWJh4A6xdv3BwQT?NqBhljiu*mI;tyTL
z{P(MWzAdjhWL<KZe@^ui?nBx#`@RZ0bxd{Nuh0@<S5}_2;oG5LwvK7jPZj_EefQn1
z&U};p{Uv|8`StdH`t@;r?azDa^Z%UR|2cdAr}+P;WA}gT_P77Ob?IaM{O#K|itj!y
zcc99+E!S>^=e@Z5b{maWeA{+?amCWBe|80VX&y~8t4*Dfb?8!{>&xe_-X~q<JrZ^y
zLc(R9VN{i?)SD?rCie~seM))fvwqWeQQ!J)HJ(wtWp$f4H#Hci&S78UEq`LgkB>r!
zbbe~oO1;0+CT$XP=|>#Lp`7iJKPS!MUVMk|=KCqr4R1BJAJYACXom7(7a=CUep$Xx
z2a8fJI=&0JEZb3{)y&bUDaLfgb-zG^%5#B92LjR*Z7)fvH;X);l~?pwMO){QqLISU
zspacJMQ^x^L}iM58&03*Eplt|)SQ!y4YD#E-ia?ZuG3v}=7r&<nSpla7TJ3Ko%(WO
zhjxmD#xV^EuON|B4<?-=J=K*r^Su5Go?e}JS#bli*yWazifxA+m~VV?RQev0CZH&A
zXTEi{{yV1`i`Q%1x#iF$y?*aJ;rf<*i*)meSyKO9@;1C=Ud~!P@$r_5`FR)Jk~Zm8
zTfN;awzw=Zm~DgJqG|SvS|6oxZi_p@&~aIThc)#>=fOu$>^L9JowH-(mFWp*)=pUz
zw!Gk(6GM&un(hv-xI6Nup2yiHp4*vdc{h+brPfclDWhb|&E!=prwKSnR6Ut(8p0<L
z+pQ`2>f+5Z)(8o2!Dok_ZY+v^m-f`)qHXsziOFX7>S}8qrRJ}SS#y<v<A8R>YJZ!e
zAJh7)ojA@LXI8zK=D+0cy@k>CH}Bu%TUWLFV^?t9g6aO9NgHi~*T;SB%UmBbC)(oU
zoAB%Ya^}y^@A-2lT;8(qQ!#(tGx7g-{r`S@|MwPi{g2)2W&R&)%lh_Pd74ahqiw_2
zXFXSVmcPGpcP5W({4F1igM~W`ci)KMIrLItPM8;?R7b}GDHpkq4@>?h@QT@bD=*f4
za;<b?p+U;Tr+&iT5z~VV{8k;Ewfx5yA)|~Fx30IVKFQRI6_?B_j#fK-Dfvu7;pvF#
zIZO_x{oS806S{C@-NkI<lh*@YE1!%!ZriK6=J~BTzF+o~rAAC)cr<Ba;quH~ztdFO
zC9loDJlA>UloG8HHU=K0Eq|0A&sb>vQ=s5UqRY9d3*=`Sh^%2*&plT%X(G2qb>HoU
zO2#`D<#=c$p7oQ>t=^UpKVxIewx@l|Oj)u{b|2h!nX&RnG=J>{uS?g{`QEz)sV0PV
zWt0Z;E#fkm#i(K^w$(^wb&z?fMVDiHwnna3ZW+Ul4QF2dvH0`jT5gO>n}c)MuTvAa
z`D$6ckADh3|8(h29wRkT38SF>d0VERGdE3D*s>*BY4-gqR}&__WZt}?M8a$3PJs;x
zs)m}W`G*58RKCbgRyXkV6QAUw>Fkln8M&J6sMy<m;+H%h-|CxYKPBtnw*w1ripM4d
zaowLBHG5YF+r-Yfb{>UUHTo(dPtT=n`m^Tmq1Y*w$Ikv<cX)%a$>EC|WVa_=49)wt
z^nRH)-_du+5_-Kim1%3$Jf8KvS^i(fo-a`g1ANN2T{@UBVcAMU1B(x}=CL)8CLX$S
zWv9EHzrOwy?Z4N`EGMjv|LwKZ(ELTolhFL^wQ)MSaW^f0@)kewc>nbIv-`g%_CAic
z-@o_q$z$2yYrp+@?d^HD{^3^P{a<hIFMI!Y{{JuQ|Li{h_ucw^KR$UspEiA4c~o=B
zri#aB#i|)z`%3I;-LQOmi1y^|-?v>4pXIr_QY`B0R=&d@K0FP-yli3=<ANzcr<Pkv
z@+;-&b)6AV+&yiH$X1EYw>3^|#v5M=i9ELMn#g;r;m49``k6PR`1%q%cWPJZy9K(L
z%{lh<#f!=7<}@u9*rDO2q#Vw1zH;%sg|j4ex^{iBOy+3F@u>XLGb_84v8Z*Ksf)pr
zUtJ|%Jk5BUCNn%Ty`#c@V3GOOdnR=epOhz`m~bpRyzQvl;%hgYZY7`oz&q!*+cGh!
z;DY2S2a`k=g|NAtu#}y{zD2b4rsa|%r|gb|{@fKuPI2Dzd^m4r+RKI2V!8qPa~F#0
zHk)Nzow|8)5l=hc`xL9J=`Sb0b8{~YQ9jBuZT6?)nOfxwrKA;3cYRadEZh=#KV$8j
zWnC&)UK=;3vPOKcWt_VI_?4|mzq<6&)JpDhH*a}fV%eL$HfA$_-5!?mr@N-s1+zSR
zb@Qh9y}cV=R+x#rij4NrF`r#3sH(QWIBjPB+}aew)7Q1WvbXH@TKw#mf{|WS)7ift
zpN4d2+Rxg1KZ4h?gTZB=__390%&pVr2fb#!eaG$i%3X^LpUz8{(kQv5Zk!TfGV{fZ
zDwj)XF~>NpfBcARTF7GD^frX|xx4$tPr--cTqU2q(0+3+;c3TenR~NVr0?J2tl3c9
zZ-1e8ajkjH{co*Pw_G>Ota^6CcCX3a4{xT6-}`&;%8b)?e>QIBzrVZUGC#wP4@Z=L
z=l_`?@FMqU^z!)pmuY_=Ed1<m6IAzm_lH~8_m<ba|Fm)U^Lt<KJ?jj&^0fKB_x<;@
zrlsufZ{3Pm8~bxw{+<t4v@d>}?l<Spo&5i|HlF``Z}Ip0pRXS8v;F;K`ZWExfQfa+
z#?jRbug{7_X<okUee8u}_Tue(_ub8j+^_A>wP;Fruhn1Cm7+!g9M1hOG{UnSf}P*p
z$uSksd>Lw4=&(lg`cuPn<-JO!7Z-lb`8QL7;iAH&$sy|+Oz$jGoKbd8v*FH?8iC#g
z8uP?{CuXJ=^IlkUTj4ib+J+2~b2AK>x1C$nz3_PEx{Hdcg?FAP-2b4L=y~e#KKW^W
zOFwliZ=Il>d0ygd2)pm8of8VQSErpXGTdOLuDQ_EB9paY#jKCQ78k=Wr<S~u+nM1N
zA-i34N%5`;KEB4cS&fnwOgC<JJ$-Yt(&r0I6PC=X&^O9E@kVaN(gd!mC&>@$>c4c(
z2;C$z`>%!5BoFz?9=nfpoh<v)F+p;=z_Ta+YuGA$O@p5By6NBf$U5VU@RbRItqV>_
zG#z=<$rvIP-7WBFi%|0s$J>z$+rA!LRAVT-Nav;W^{CkACL-xt*)hIP)0W<4WB&8T
zLZMjGaNWHVOF0rf*DRXS!o)6k^7i64PdEaO8Zwlt8JKJjzI()><IL%KUaXco-&7|q
zJNRkFN}+-^ZxUi-ABFHPQt%U<S$JgSx}%;+M*7-iTNWultd#v*w={fBcTer}&krW}
z3wj&eTvwaAndd;+#glAY)59OnRQEOQj9G4HnYV-I+L3w9>jGqV8$?Sy-uOdv`YEj`
zr;gq#sd;7U5b{K(C1TG{cAx#`#XnYYpSRoi{<mGxw~ObikFMNuzxlcBzOR!{AAh&*
zt$mD+=;7^qFTY4z7%;=KqOw-~CSO{sz{`XUA3qfrU%y}R@uhy}qboPv?du;fZ(p<b
z$C;JC|F2MxUSI!kv%1~?ncXiROk8jG@yN;N@^v2?ljHwh(*J*DdEeiM@9%v-!94%x
z`u%^te2%x?U-s@S+cwYyoL*Jb%4s+6oJqTMxp$V)wTafR%NsS~A~V+&$n(saF5V=_
zH)l2{(}cLd^uEYe1Ced>IHIJLr6pVQJ~uCUxh!>(g+&~v4C9XAzZ#dM%K0Dda+1<^
z*O6}H5&I~1>AO={g~I0Q6KkxWOE;AnEjh&#sh#;tr&H2vcSg0i>&~qS1$kY1-Vf!<
zBIVvD&1<VuOyPW;@?hT!--+T&-=DO&nzzv6Zq5|0H_sRGJEaRbs1=<*^)9HEvsX}=
z=bl=6$I*SSwq2ZeyX@GvWHFB*kD0bik@~W3u`R13=I|N(R+VM*p8Ty_!=x%@zSe7_
ztc1OyQ;eSMKTyx)5ft*6L99j4YjYouW81V%ylpnA>$(I^1T`v{voD@`e(`5d1MO_j
z6cfSrE8k}Y<j?xcCBWXa&thqio?=PxnVO45JKDH~Kc9Y6$l9KDYGGy1Qa=4e^I6gU
z(#28}d2c6_CTxj}lzS@FD1JUqv`LLu<f?`8)b5}wNxx3qe6HKKZNb7|LDTDZ@+=wD
zr?<|&(>~d%WwPoP{V5DB6P~oGE?sdnPVvO4cp>2{N&PXef8>-K`kNjK@i?L07`2j5
zDVSr4{IneNdAq7_97*1}cK6PVV=HZa5_=<#dvnCHy%y@ye&dw%cH2Q0<L`gp?_PQT
z*1H^6Zu5QhkC(2mY<+Yyxm-HA%<P)gM62&Vm_NU-iof-~`sgC{4Sz1!w&(9&ymgn*
ze5=4{iwpC$j<4T4XHM*{1y>%GY@O`_IxFdawOz@-)BN$@7T!K@`|m)i{GS)6lE2GW
z{MS|U^|$}9^Y8XMV*5U~eLi2i)m*~nyY>F+efKNhO=Q>Gw*CFzJMuNp#Q%N0es0ds
zt@D39od1u#{_*m<f9d}&=l^?E%WuD7PtCTiqO;$B^LZq9pz3qWvsTA!<^#vlHr>t1
zHIKfaHf7?$91WF*sVjZ8q_-H#_#QGaadS^=^Sb9?#=9v{-1#_*_uGsr_d}smq#WWV
z>uK(sB&^4y#O$)=e<6Q4msrBigUL^S-0kL`_#?l)>C~+A7aTA7aeR-+QExo6rg2H^
zu?#PTPk-iVKQEr<x<%V>{f-od><wajHNO?8XM~C;+|)5xbYxbiVB3=7>+jTy62&h2
z3H#r*Xl4y+igq#D(jX)h_R5bhbY0`2CGOd(v$7<;)|v=UK72N3y>r<LrcK|@d+|>7
zzj{MJ*(oIB+1q(enxWq}Ja%vSZ&WDSc17IL(COzA_2kNZ+?zki@wy#;(R;ZoawAXV
zu0Bz@gI?8tzWsVA<fOKGf@#IM2{LE>UomZbrB@ZaWs_cQc=hoQs^JnRvh((Qs|ug?
zE-p=P>eIN2ypyi)qza7tcPd=t+NQ|3ePUbaj?FyXWp#y`zh{-zI7)PHs<`uy@j`7b
zSL2Dl32%>2KT^5-=<+ppRdTIO6BaCwc;2_>V<k(&EZfQ5QxC_cp7MJq=eS-gaqiT`
zAKy&4p*(kmul}J~&s7=3(l?)ccq(n?67IZ2{RSrm9mjdslJfUvmfU)uJyl!r$UB~E
zM>ZY28@FRNb6DR6tM51d{=WZvqT~MkA1`foQ+WHGukOd<r89qh+iCp!{^x~GI~T>*
zOCFo^|I^`@3b8vL?tOjUR<og5e%ISa;qtTfe3usg=D+*vq?@)~-J>1mp;2`o^zv7w
zu+>gF#(kbS;ZleQ*Yw&K-!^}*t^54r`f`4msOq0PndN`}3;zB$aiQFt>PLoK>oykr
zIKlnB?$;IbxO?}?Kb-#m)&Ab!&-S(7|9=1TcmJP@^`F+)y?Outh5fHH>Gr=*75nFy
zm|M>+TYuYk|89lrpvmagdzN=YbeSI<`n2-iw_OL89Z*}sz!ZOBjme=b&VY61q76<8
zJGA99#Wzhmzo~FVz!RpNy2XwuYMX-RoZH;sbNJ=PR~pl1w9MC8llfCKGHd7WCmSbS
zx6r>HbL&}xJLiG#E~nXZ<R+IrdbM%W^=P*4TddPI#ECn<?0Nm}#l1_vq67`uR&Orc
zsrQNb@fktGh|;w}%=U|_Ls_zuITi?X=+2i|bK+jn3&YjVlnpJ+PaK}{P`K=3f?&Q&
zr)5%u5z|StU7nW&Rj&nDJmq*;?3lZ-^Vg;Wr+00y-t=yx=(J-3e<hANU(@5Zwu|(*
zFiq&%E8EErZ|6^rwh#_dd3r4%I<48b=MQV<(yiCLz0X>N9?m}Oq<uT_mY8kCU&(mC
zKi+}P7ehtZE2|3`y6;U{eNbw>Qpwh^cicNw4_G`enS1%u&AZIG6SOm)PQ9UeL#F%x
z6PZ-G-p&Fqz7<KzhO;+FoHm;`sZX$tWx@=`;u|T=htBA|6qAVk8FxaC>&>@wvZWe1
z?QI>}Can{$ylOaM>>i+c++OYWj_%2t^LewM*{yMwaY+-n`gg;YO_o9VFRGRNOqVOL
z^If&-*mESoi*HG|&Zz<!=0G1Eo|5^ebmN<@%{q~A@$tfFySk4jk6${N@UME``6Vy^
zz5jjURPy)m@3CRG_g8#esV|m&`SNf6si})UetR!ja_eID`YZhMf&xwljF;71HU0P!
zRB`(~)63rTxVr1=tsj{-HNWSk%kTUCz*pY-`;EWtdAk=+cZpkRV;Y>l`s^g0K#hv4
zuilitxBUE~mizs`1%er(=Xo!c{Hrye`{PjhlV16nukY^He|>j<|9fx#KX=~$xpw~l
zH~;_7^8a0{|FN|G^XC8G;^$kICcQnhGjF-^zrOp08`(GPd+q#rQjnUTzyI8N*V1`~
z+c{cIX84NTDt*Wp;H9dT%;zD<%CJF<Q>p6=&vX{Hr0!WKKNMWMu4c$MQ#SqDv9f1g
zS1XU`_A+gHQ+D<GDm@O-i?i9LO_t~MV(FQEVs|-TmAKl0wyMh2uO<Yg+^sND&p2zn
zFwfGGnPcDANztNnWz?45Id^uCeAOxek9$0eWj$WVIe!Vg5>qU>#-(kE>@kgI(+(dA
zH!s#D4_RNFP&JO&T5&jaL(wwk8?Mc+sx2G7b1Jnkv?QLGJ44s;X@o<ikn{4=nYL-O
zbo2OLFc~eHV#1)bqRU7)N>D{fa$3-_rE??BZx?Q9*<Rrt<zF!2hW<I@wEMI6o|++D
z{PoS3DM7x6+(IlT+N!2pYmGXw>w?$M!{=3tO1!rnSb5+G%RbZc2QyY2Eb`RpD?A=`
zefq&%Zr_Tzj#n*|pB}3He%<gQ<LO#X{oT6rD^uO)ujy9wk#v#cyqhyE!1~Knl^;jb
zcysz3=HFd)eQQu#R76qbf?Bb{RX3I$^c9TSFBHt-66<2ZoO0ej>%arPhR4s3Ki!#m
zuj7oI)313NEKOe{@&Y<Fc1{X%6FR3Oc<tf(^;eJR$hqBI*_rj-dq?8V^}Duk)ji$F
z&fw57Ro{*w`P%$?E{Bp&KfbkYzWiHn^;_!;7tg*6+w$>H@YQ2?&OCMBHZ7i=?fJaA
zL!XZv>pp+wTRhtWBkTPa_UzbD_3c9Ld>QqteD}+a-%S3_@lbGT>A%zbyu0pp`_~%Q
z+}<i3Utjiam-PACrw@<Lu7B`GdA@zc<>S}h)mpv%&bTCV*PjP>i|1E-UvS_1@|P>M
z^*^Tm{}LTv`Em96y+1C6%k6x9|L>XoKVH^<?5}?~{r`FQcl%dgpS$MBU%yV<hOe<g
z(UUvnHto685Eq>lSMhh{*|Yb8_FRq2o$~Qhz->MgDetdaPZbq^zOvF+;Z^q32<>8T
zwN&N6B&KbJ-wz9gi$v{M%y=c~Y>9Y|)WrMR7Y(;^hj^)~DcsA*D4Y7%-)OOhpSy{`
zhA9zAZhIItkFf{b)8)NzLZ*aAzCSoqqf*_c=jP*!H3?rP#@3x)yShA9DSk_><&Ddh
z>P7EgOfKnP*}0lC`6BO(Ps%0FUNb#5T4TX`$Uw<fGS$DL&VL=RkJLpkwVarciJPTU
zp9(L%k+9Iy-l!};;X>M4$BCL7Wt2|+>S!vwdSSWA*)3tE{#NgnNrt38amlxBiD5gk
zS~T_Bq<wNxTZFeLSxjX&d0D_*r~KV!LRIZf&#p{fm&6xWZI_>(;Kz6%gQ@Ih=E*aE
zjc4nv(mgR%;F{}`ZxzcF7;J59WRrPb>`+~?rOmkE%LyZ$pzNnH4tM8%DDqqWG-=b_
zxEDeRJLS$#G0{1_*mD(E<C4Fxx5SGJwIsFwooCA65d2tb`uWA3VrTVaCL8cZt4#du
zxohFp+cE~x9%mRhmWbq-J)6d2^ri4=*w;VbnadA}EuZCCB2^%kKD)C|Jl&ch>FUx*
z$<-l0YddFL&z!%m_2aDrcTX&j+2i=~>eAS|XBWKpo%(J6|1gfi2hH0L&G=cqcl)Io
zb9cYHd+$of=c(f5*YwZDlvNgdS=(B)_2((>%vpZ&`@G)WkGH8vRh@G?$}%?djIMpf
z^<(=Eul_uJZtaVEANy=Y+Vg%tx!L~C^Y>rp?KO`M{LW(vSabGEd6fA4UymC3@9p|@
zLL8Ki4K`Q4d3Shwoe*fPklen%=ZufPYdiSptbBaki+Nk0&+nN&J?y+*?v~y0KM%?O
zon8O!`@d`dzdfH{`EBLL&!6WCRWrQat&*nZJ=LV4FH5fe=aXZ{I(yHy?9?_5WjxUK
za@9-it07*zN^3*B7B94JVpw3Q^n2fis<|9mv;GUN(_(2-jhtzJO+(_7dHO4+&YN3O
zZ)`Z4bZJ|g@0xSA$s1<)pDO)blsx@TJBx$v`L@r=>ocax%kA{o%_F8O@T+rH_D&-W
zFTqJ6*J`E1+m4oRbvfRkcWT1BR=vLrdEIHVR1R<Yd9cA$qu^7sNb?CECDG+i{;kUV
z<{vuYN{-6nO**AdyzU;~{q%`Y$Ieeyzpu8udG#)ZM|bMeg%U?1&3dcEtFE!mJG4!<
zq;SS!nRf1NIhW-g9#U)alH8RPzdB)uuhGk*4I$Siw4JHFQI{gbe4qR9h0nH<ZQ-Xr
zaFx|;5Hj%%ZFk;X$+J*Y-$J$J%pSiVjzx6>eQc*&?>-I*)?@DQyqi;0|E#FVk}X?F
z?e^Qdv7+MolO^UGFS#x|>*Y%(!-zAc+iDDKvVIEn*;>81qMD(j(&i^p`QW5QXmiGs
zttqV!s+*Px6lQT-xa}{@x-@62)dI0Nr>NBBM>K*xQcqk_xH|WTprZKlT@yq#Tnf@M
zv}gD3`m%G{gl%Wo9Ok~<xJ1O|isYH6QLH`5E4|bC3q6Ihldd1?{OwbHcXNM@+<~9^
za<8|So5k0yyL;zO#mAXju4}h%{`BJZnFZo=YjS@-I<+v`?&ig_*FL>|d)Bl#(b96a
zij99<sqe-h-6^Mjhn<((_jU5){3(kcowKWXzw`Jz_JVH@tNHJK>fCbuE0^o!WvBV?
zKF-@#^XA>d)4hr{WgnF9-+y27ZSV8HZ|7cXU0=I&>8yJ`f6sWI|99g45AFZo?El>U
ze0upuS&3?f*VZN`4fDcmbEgFDohZNgUFOzpC0V&IZ$8$aZ&Uba<>dAABPzeH<UYOp
z#Y?u5)s<#%w!ZYwo4ahvtq^Iex6EhvUAty!kkaG2qT6MeW3r+^w8@QBqcY>iUU%Dl
zb8@m81cM^wOgA}hT03q2uC=>gIb9Ag?Fh=^c2<^Z*!3l|WB!emZB1qiqNh*fU4CCu
zHpt9=TjYicfs1dZE$A#MpZa3QV*g2^PZXrGaxCKw#N9T&I%nSU`p|^Qi5K@~?ah34
zXyP-KIsPf~Yh2nKlN)6=F!WUK*zNeXW$nbp&ys#PE{W7X<LSSs`$g+f2L{Dsn_AYb
zPvdJ5v`fBmghxzYgxS&dUz^<<%^N4Wd~fGoO}wT(p+Ru_*22>8FE8XyZkWH|WM9pZ
zu(})5tA2DOhFy_Ab>sM=cb=Mo98adY_iYH98)^G%#+1u4Wede7<vcwp)hs5%)}7I>
z0lH9SOY*NS(}*Klg?8ue-k2I{Sm^3+C~->n$&<J5yta!Nia%!UeD`s}DY<L<;Y%Ah
ze%_ol*-!t}iNp;mkqu2!d8^ia>H6~V`s`z3AA(N#|Bj2`2<UeFw`iaE#WRgM&Wr)E
zx;f#`-lRxLES?(Vr!4!{AmY~69m>hySL}3So^s)AjA_qVd$%RbpKTAa+De2@`4#uf
zMkn>qqpI?E-+g0`t4)g7oyvT-^{$kS{~3>EYNjhKe>{AzAC{fD-TSuB*&niZ?^x}(
zdD_m9wtnx!H;2Dp<CkA68_^T`r2f7ygU_tIJ%8@(leXXS=Rs^X`?`iwyJG(2JHM|i
zdHL}6JA-CM8%9MVozmN~5=-~B{aaDC{kh!k`ggme_3VD8J{CRwa83Qu(9G?-|Nk}r
zzHR%y58d{k|NlG~UH94k-|hDn)7xzuzRs1HRWLP3%hcO;t>0W_i?3TB>({@Xx*VJu
z!~5-jzxo*5vRWeQD`>k-oUqXEU8`O)Pc*WTVd*qvnaVbCx`eZl&eEQ*Oa?(}1<!I-
z3MD>$h%+>loFip1D{1?}UdvaEpmFc2<~g%EBeYHhtQ6_!U{;W7;94&vdztsZ_a%n6
z)_S~3n&;-J*8A*`$BSjU2f1_V3@%@~^fy9teyT~Hwyn90JEz}_G)EP&aQ9uh6~&QG
zxAyOLT-P;0DQn$Lk%u2*)ryR-+{hD9+&b4YCvc(6eGAXc+bqS*O$@`FUhK<U_pHw(
zf@ijeh3gzKRrcRzJDpiVpYdhI8$H>zX|tQ=p*IsU@^8+{XxUd}w<a+lRQ%?|%Y4Nj
zx)0twvcRl$u}YZFYhxZQ6>+~y!HE(=`_3FmX_3k;PuFzcH`l_nQD8Bz$>yg=9F8eA
zxaOZaYj@e_KrY*XpBCob(~~zGQ++F=w@7L~N8-=rcRw}je0XFUvi-!-=5;^CPO6Ic
z$GG~>m0og&@q1~2#366q1y8-#@-CG5_I~Lj{^_r}gI)Irtoqj-e7Nn$9G3H5OmmIr
z`+jb_$II}bI*21vpG~)=q*3H%*E1=z;2V4&C;0}5r(3V%>7FiIyJWt^m8AY(3z!9N
z9MC&kaOcF<ntQpwJ6S~C8n5io6?2wn<X5x(ew3YW&wsI~;QfCx_FrZb*!OnVv(x8g
z_q_aV9``4zv}~b4=&Pd>_q>|*<Me#?fK}{&->xl=uMu!MFr#zL)s**_lgpVcs^7>R
z3Z4F8mO5MaD`EZL4{~3hx3~R$PC82b+_JxoR{y^G$;|!#$nc>kgHFwdZTEk)&j0zk
zyza+Mb-DdtBL5nTf=WZZ693s(1hSX<&kwKNx}~b@<<j)|F_m9eK8~;aGSgk&`rDW8
z<?&TzZ*vyS5@mR9c0SKMnnCOLCL7s{;a*Ekeygo1EA%WiynmUMiTAJ1o~e~LM3@$q
z&gEg~EIwuEsrSgz`o(27pNznc$(o=7aIfrXBZbiNj7qsU!&wmv-)734$&Q)f^!t7Q
zqbY~9iy9B>)}E!)(wjM2K?_z^H&45<H*BJW+NQ!uK|2obGPL@>Pe!jf)!1lOp1LL5
z=~~f;9$Jbrb<+%Y9OxB1Y@ig(pr`djM3J+3Q`?bm>hFr2cDg9GKAz1`H|_Mt3|q~a
zE7r>wMzt)`Iom87e<65g-pNUE=jJ!)iZ5LwuX!!<%(;BWOVy{oTx42b@^{xU+dHw-
zvTbx<*!&W%-M}EAe*OBYcd5_bPdhki!?mBCj=JwQyX31Ko5ruO;Y;WNrFpv=zKJ$W
z-E#cjBA32t8{Tc2_eb!C{K35*9Hmc7UO(n^G+)flUiwjP-%8zS+N{ctURmAVcR@AD
zv&~iiW{7l|`7J$<I0=_oa~9pnTVj$FVcxRF<L394Am>9f=6uMQY5nj*!NW2R)#p>z
zXon{=R$2us<qI_Pie5=N%XUI0XOg>V=LWIG-J4}!8_l{H<rnT!HJf8wJ)ddTo|0CD
zGJht&mKfoet3R@evsE5lq#?$C?~h@ihRmM}Z`0*>Ex$B_{XyluYd4qfEU$b2%=<jU
zjt@)Qzu*7SIQQ(<Ur*h)EncyAW7ao$-H_$GJ)ej!=ezrfQ^KLc`^4wtj`wy%WpCPj
zB)6R7ymig*xtZ&;FEt7*T0DE|&DOqqHa{l5XYUEu(YyDD`Om82bicoc>VL1T|H)tf
z;?d594L{Boy<uGQe!+#kfo}ujS6<!heLte=?HBL!^Q=q1-C4OgDxYb>Z7#+`Zo6C5
z8Uh?<pEZihTW=QiJBXoHb?QZjB_XS(G>Ob%&pEHBY9;3q!@eXkd}6Tu4&LGiR~|6T
z`u+Ss;en>$&CY-Jyf&Izt2FmeU!+OM#8p>vKl?BjUWqCGD!emm&Cj+4V&7X8wy0ff
zDsDaDme{$5y^qJ)FwkUj@ahedt!sCEyeOHxp>uPK)>KRH@_Uw@cRKe;iMu~f5|CUK
zrp)f`6ve<1m74eRi|oX-6De~$i|rN_n7m(;UbL}0bwkpnPGPo1U$52Ga_s4N=J;&$
z0>`zPYm~)jHZf_OnlLXlPAj|NMWe4$gU{P@?JU|Wnid}~pBPs7WywyR>+fD@{htxp
zV>OrS<%7*dw@PMwTyo7J#6O_>cZp)9b8Bh9^;4I^1#4yPo^O_6>t4`yy6Vu$cfH#c
z&U}qLRT%QP`as6(7q4@cFOymB%AIGp=JgWSfd0P<8NAgyKFd|@S}?`a?T_`A-3~e1
zt@S6xUCi3Bz*IaoYy0jg(aaqa>&_HNZ#*n?+b3z`WEoY-#L3OEXMz=rIbU~5&J;`I
z)#~x%;c}d07*ST!efigsdqO7CPnzPp_Juu`dp9E`*L2;%Ln<d)V_#^$VT#GHeEVJ8
z{?EnD{P(#P&)NPvaHZ#ce4$9g3@zpx6)$IQxqtirz5XSC?>#$xo?AIg?x$1bsoV8u
zxBhyv`tP+c%Q>cMtUL+lqOOX_pWnf7I@aDV(l_jp{O;RJUVdsW7d<tVJHGOWgt2L`
z|87~uBOoEKrCaXXzkad(MQeZE!{Yj{i_QP7pa0`P{-1yOwts%w=d4@bH~*Er@<#Rz
z|JJliwFPE%%K7iBOG<ux#C%$u_1-^kuJ8Z0Xmc+U!(xHN-8Y3m=i>%#f0p+76kAE|
zRqe-5k1S!B;$<#kCbHmc+o@yMPCc8-5TeDXa*Sb;#F`^KJCs>^+tw5<sAkLDz>tx9
z!=gl>a!u5NS2nx0_83La66SJ!DX4p1ier{t-|p>)8%uW1YmjJ+zg2nk!qnNSUOnRK
z&$uOoHu7Zu@lcyqxU<|&Y2&&rc@aJg8`xHBOh1t-(jZVY{okg;3q%wfEO?m|A6?N9
zHP{&>{%xM)p3?z*riyC25_@mBxY|9GGw(jGq4NBzgZ{!ahHG<*U*x8}UD>w2RJ?-u
z+Mdgb8%@J6emkGI?WC_r-{<0;ruSu%PYUq8xbu<qW@m5260Qv1BH;~6-`^;0aq)Ih
zi*Rt5no+pwu2>7Z)$fNTCl1ayn&Fr1H%B=C&7!3;;#0jjo)xu97>l1(l{~(hJ2ryt
zxY~5*r7Nlx*Q#&GP*0X>IGu2@Ue9LH{XpGi%lxCf4o=^4HQzJk=ii4{J+|q0a?a^%
zX<HDT)W)TI<Nj8|l+;5zohE#j@SmK)v#>ewPl&jL%hxkm-EaD0UryaT$CT@`<aNLK
zeaogij^(&3aoltIy`JPnzq0li6~4Ig;Y6*$vkPbC*)LT5{u-^F&hX;;zPcaVtoN+E
z6mmDd_L=v+kj#J8_xTg~>VACNrx922)b;1I>`lUU5_4*Qd~Pr>bDd!=^yb`Q-S?M{
z9j_I&=d*hI_x;}u0#5TQTt&K$dX;`$*3Y{uR?+5-)AhA;?0(Mv|G)mn<o_@7{~evZ
z?*H@uua5uwyZ?`R|K0lSc}A0er7L}7xK{4=<;$g8KZB*E)%EuLyR!PYe{6)^*HfRL
zpPy^h&9U&zHcnNi&Rq@?ZKaHjzHKpIIhbbOwz%H%=7B7OgBjNig`Eys-FcOQYgr80
zx3TwC+vwVhI$in^G<~1^v>o;eZv-Z9EiqNDWbbl(VtFR)*~1VXmoN!_5vF_r$?Yu@
z*uF2=c;G?L_n1RgD|aVEGG2OcJC@g0>d*uuzDd>#B4@K%1|D`%Z{eSMEXMr0fQi_P
zeW8bTy;gX2@BfRyY}f4t`(ih;X@|uoZCGJ^eutwf`)#-LM;IPd2d1c0?%rHlm((>!
z^^AnhuGN+**DR*CdTOvN=HM}_f4=Ty>gzkTM(sx?mBmS{4_#7V7;G<fXpPNnmM8b5
z7FVACX!=gt_sL$jLoc#&x{OjXZ5FORvOpl^+S`(?;!Gzsoi6b)HOkMlxLOysRKsuS
zB;GsQzinlhux+o}I=Sx$i$bfeEO@rdb1j?UtP`iNh8FC|5!-(H4Clj2@3a#|)ooXk
zpK3||jym{a5zl&SlgTeEP6}N+((1iUsX6zT=iC23o^QTqlJdhZM$r52;}B7%%)+T<
zhGLE;zm#G&b-h@fxLu>Wbj~-^n{pioPwh)tT*+fu@pMl2oFdcCf(O?t46;<t+s}GG
z_31_Lj15J1uI3qJZ_i24`)amNzV6inU-?#nbh#fg4KKdtZ@H$&;=X=f?eXtA^1G}5
zRo}aO@e~gm`@Q<ht@^UUt&eu<%V{t9%bZ{D{^i%|xWZM7&03$!?t9C3fMd4OnWGQy
z?*7{o9Fw<h!n9sb#x*uo_fzF}#qRwz$#9v~_Vv=*%ya%<GGqV^)BJdTNMcX*r|oq=
z*WLg7RsR3~fBXOU%m07=f4%+piT^*WulwPB+;0EYy?bx}{J-s}+<{+w9rMBhpXs*r
zo@F_3<x9<t40)TP7Y`;nhsTt^w%-4JS9#t0QuhA1^0yZcy9RZ$-}JbCJxX&bOGM5U
z>k4<rx!Nm=8T{r1HG6z7S-70rOVj^*{+@4DO`7`i?n<5bTbujJNhK}g`HY`GR<1Lg
z^+Al^bDR0Ajz?W=rs|~}6ZTy3d@H$l=0T<A;4a4yeU{GbCfDZOhPSp^o<6n8*!kkw
z#K|(3=6$}A6{&smQ{<U-Zv}%I%fE@0PTc=^+S9;Wihp1FExT)Jm~%C8-GmEwuU1`d
znYV25(z81P>@#$9&gn6Hu$?*i((|M(Pm90(w2@qP>qzDShKUzM19b!@1+}<X?_}eZ
zXjRDTxhlk$*tI&hXnRUw)wM^*91R}d&}N_aqqbX8SJANT?H%5t-B0b+>(+hTe(r4|
zQ-km(367;9#}0EcOuE$Ml5{huGqJkigoK{r61SE@g;!B)bISTEtd=XQXq9V)JT|g3
zH@3LAS8Vr|9g_Uw-Wz#mXYXhTT-5#l4CfxTeX_BKI}MA?79KZRvvHE3=l`vLuJ{W2
zon&m-XI8k%UDolb!X;h#ZAAu~v&+4+5)1>9Cd^p#>DZ^}n`~R!WVdN7JpZ&M<kE{<
z$1f;|t=20){ZJzC(du>DH_tHYzZ6~B^I-Ngfxg4aU3Vk2eGUZ)d+L1uc<1qPhM0dB
z?b#RHy)(nI#I@wp4h@mhjyryRnXdQyz*>fL^*<Atm;AkVkpEuWsV_6t_b2tNl#LLX
z7p@WRv9Zrk#Db4kT}-s^|I4g<=jC>{vqU`K|06*?=wbBzix+1z9pd?QOoHLhyZj6M
zZ_Y5!xBmO<<NCGhYG1woYyW@w`|8vDe;=&<|8aT!mri&28(wZA75BH@V2-F?ws^U(
zL|d>C56ht_3v=`R?=MUZf4jTr{zC2bmY?Qj=U2Qp4d3_m*y{d0|J%OD6+b;>8lE;u
zfhkmSi_tPY7mX)o=NLVFX3YrERNb&JXl5^y(vnr{rpxtO|9bM<{>wSmi_ib9QTf>R
z`1<i1t<USP^li}NddwGUx&P~~FLxcoI9`_N$ho9ADRVhG<%aF#mdKwkF?&kRy58G~
z#dqa<kLnbCx1CtL?Bq82zBxvQTGJ;kw7wG~q008e&h@Z~|8&h$pEUlS78Tl+A9Bb_
zp}5DD^R;7g;@->z(*hG~dH#7#)=K66D;0Qbm(H3eVmx#447L>|iiaaoTA!ZUkYWGW
z$3O4j(-jf2Pd5cfv|7JeXqva|=#A_covF??ENi(KYsF68G%Z`a_|3gTNeVwU%1i#X
zkc{e87gl$(Hwtm}&~$RR&dSwjyINs}YUX1L(`(0UKW$-QSlTjyWAB2E8zy~gI>t0P
zVva`ttn3(94t9^X%dQ?)z0JMrbu6Qp+sR}fr%d7R(;gqYBjl8}+gwD!$;yvO+f_5>
z&%q~!u{|dF3dULvd`hCL*X3?-UikRRhWE@jw;s-1A(Ni=OQp^!)@4eBy4no$;*Zl_
z3m;_Kv0&qae{z+(JgU_uo#FSsQgh+1j&{zLZ@euVJ&YD!w!7FeBjoaPH@_`MUY84-
zB?>LM(R<{PyYY+3^E%CLSXEc0eeCKLpa0V3)h7NsPC6GKweOcdCbR45<LCRo2;7SP
zY+fVZaP4R6<Yj;7UVC)bUZI6ukz;AE{BswrJads5%eUt3K74oC^;=RUR!<$4?Y;NN
z-Hwgn*q(_ZDd*lciSgb2b#n6Y__|+*j>`YMUH_KdzToWrzo+H@`~UxTe16TJh1b4G
z&zk?I^N8GmU&gY*K@3%H9!-(KpH8)&?mH7F7*`l+QSB<>VOjlj>O9jAk+D^8b2eYO
zUbXY)9gDpz(U*dzY|`7qckg0W-P~E)+P(AiIC3sHY?!9Dq1F4@igW|<w#4|^i)6|)
z87ItWG@NxzgemBV-_Jx%f#@kE&8}uaT;;PL`jz+`3VO7$<kogm$1i)9<bAtyq%L*y
zMD72Q{e@BoPPD0BZ3#N|-RYH?#Bwd3%&J*S4SsB3)J>gn`ox8~%I2P>sS(FA*0C-=
ztk7e6E}_)We{0Jf#r9sG=h8_x`U=F^x|d&B^EOnA_prxG6CP==rSoJeXHL16az||S
z<bzf5t3Q@VJI|3}kz{vxywcgn>*v!Qs%<gChmTt6ZF%MHtH2U+@m6`H$kc}hyMr#K
zZHeJ&sg=2Df9!#>>~j|(?ak6xyKd`T+uL(+(J#kOA*+whymR(D%k3S@jjUsjWvnpC
zS~sy<XL7_E##NoGpC9VbH*<UbPRinz=Dy<H;nQbJmWU@*YR)tEDd{a&o_^(As$A?}
z>;2yG+B0;%-#xnL+Nr6D8yz){@wmU~_fS6l`*XkG6P+lI6NMrT%M_a=bre-@Hb${&
zH(Wh%@x>FN*mvtYUuetuFM1{=92K)ES=yCH&r#)b^-g8x=-JaY_@$p@6@Kh_pzwNg
zp`7;ivw1vWdF?yhlg}t-JF_m(k^XX}b-kS)7b}Bye;wn2naSTJCeQlx<=6Fj0!}{i
zdt{t^t}Z)oTQ%QsrRQa1{u;gIi;M1;n|q2KGq``kP=5FC3%c_)rkn!pUo1HMJ4*a~
z<<XV*zKJzVJ^t?W++Ce3FLlaS%4F-^`_%bZfBy&e{r|l8|JeQitNy*J!{D^|#ddz}
zv(x=0w{mN=llc?=-sHDbJvxQwa6n6<MB5?ZM9Z8CrQNE_qqMhv`86vtQ!T1CH}j=P
z@mH(2*IO-$wg|LbW-aYIvyS0_27_(vg2se{t2f)M*i`v6hGVJ7LTz`CM#m=~uS^IR
znzHfTE2+bdUYFQ}C3TWb&X&tMuX!_J?Jvc`_(ysL5BX1O_T=o?#l_`R+ILx@qRYpO
zH)?fV$`6ItauJ8$oqHz9x$#YN+uge?l4oq6cs+f2=i1xwW2HW^XVTyCd^36f>@<UW
z11tCDpj@s-kF8oPN>`6%cC6l7CZUt%t6{>xv2*qXE(gPmgrG+~VY^&H`dD_gtl^q0
zF#C6uo41eM)z&1PP;rGnPaf>K@W`MkY{A>?jf_saZQFPA^faFMn6ByS5%n*zc|p!C
zhPKGV>s%&WW8rpixFed%DJ-d16<ny0^(gCm><h2FnUAHlW=9xoIaT?$c*l<VJDa(Z
zzir!dX|8e4<V2a4dv`<<^PWqH=3M+1;_Wf%tkAWYmx8Sr4rJO{{P})R`?*BYueglF
z$t+t$lV4i!vP~@W(3#YjRxf1A+907*cj{aDbshfHWiz)u+3f4sBKH1r?#j4s+f7yH
z`3{t^Ew=xA!eCdCyPQyVnW6o;gx=$MB_c;HY>flO?fVRhGT0n4kCmHeubsy}IXdp)
zg|+hyGOK=kvu=H^d+$@>`s=sLAG1HKj831<a^Zfa?!7;S9F9_JyfxMxvp>4jJYmwy
zH;2FT?V8K{ez&FF=Uw$r)@JTE|8DzUGej%R;KbaGo#0e9$2z9$&w*$6KR*nQuX;EA
zUw!>+^}0vx|KHa?TwMQu{okGTKi<5z{eD%yth)YFE&q3hYvo=U4x4<P-A$)P-HM5_
z`uVFfeE#15(%#t@Z@OQbKF!R}{Q7LSDpyTsBR*X(%id;Rk5w`)98-g&7w_D()#PP%
z>D`5r+qdjk?<>)^?e%6hg{2D?`f_;I#<omwGS9qIDBb9q#&#l$huKg^j>RFC<Ge0!
zw(S~0zMR?VLE%d*Kfmrgm{Y%QNB^;XyrMix!BZ43Caa|Bvn*({x8BF$>}bv~iT9$*
z%=b#`E?ccV8IpQ!ZsWNFdvy6rx+OJ>%$H2dU&7<b=l}ZTVcFy(p^5@iijpij^e-se
zzV<ZiP?2P*F#2LUF)e;tOK4K#+O<a9XE{9`oXj?uEfGHGesSC9(<|4jsXch1y6%2e
zUU)&qW~T+R%e+LJ+g67?dzy4XLBX|upZnUTZU!M?rd1l^45~-=q;BTexbw8~d%MO(
zn`?_sL^Z|kZj*QMP}}%wk$p!(v}k9CZS~;|dP+e%f-k>|D3yK`@=j&Kp}G3HO<{9V
z)nxm<GoGGezWelU%UivLM@<`qFL&oSoVs(><>8*)Zz@h@M_oSpN8yrWx$2@L3&bw#
zM;(sa-qZW)()0VfPyEqWY5UABy1eb^#4pOJd_8q{TaLX@5ni!K?ybt+Me`QsWVs4i
zZEw}cnKnUSlJ>nt&TG4!=Ddn1jh$=l+g_yGpRjzQn^{ZVq-{M(bJ9;8JrXkO_pLkY
z?t~`_rJNMBWWQHlT&^wVWU_AOJhK(?buH!w&Hrw;ziSmZ&3{id0W=_EWxDm(nd#~4
zci#KVb?Ec5EMx8dI!A^OR|~%1YI;5E@4ro+SNlRPX#>L+^Xp%;*I)T28&myEJ99br
zg#Wuw%sp_o;CbGyHBUY35}p?DH9W7Zdbzdx?!o$>C+olU|NHg-PqTc*Z~s46`u}~?
z|NrLW_5S~!rZev`N8A@&$awQ|*69~pmj<i**?e8v{=WX@P4)RdZ+%{G|NrZBKh^Yo
z-+rtNp1pkb>of0K7AnZC)^ib_)bf@gVa?QLVeUBD1KsJ{pRDE=Pu;Q5pra(v>y^G(
zThkBS@OW2Yi&rxrn>|^SzIk2TD(_Nfy<Z>A(@SnWH@%ZR)9Qvr(xr(zmr2{sy=r{v
z?8(rQo$c)IuHM?2Yr;=mNSXD3iT6>A%=M!-_v<qL-}zKIC;HRkHScvE8eCditaohh
z<eNM9eX~8d{pv?9Ba29(Q-3$Up8EQi^H0Z=u!t$DU#4U!O^#~T;%d)(yhfs!i}Sd|
znjVu)*;1BIf4gk7ycl)E^?8WAbgY|eV94tS%YG(Iu2ozhBy6<E%jZS)(IXA=k4x@;
zfA)CG`|8gPpY~SnPMf<yJkj*`xx-I8GcV-qoiN3zbwZ*<x?9o{U;hlvt31)d!BdQq
z^<{X)Y(*y=vNc?h?P|2<&Ps(=>v-mMmF)5MJ{gHuSWVKeO}PE}h{+=EzsDyx@2)I-
zddE#o?r_y*i8D{rEmpqDe*9yn_oE3huclv_))yU`W`64N8HH2VgiTNPz5Vi9O*ufR
zH-?$x*W@n|Gy8iOC%=B@7-Y&LYbmNOq?6no$aQ_C;<kIdJ7YQTF4;JJ&#av-AB0?+
zgoD$B&F|@!c6d%ub$^nx=gZbs>3in22^)X*ic971=4+^Xn|klT1>N}!YtEf?IB@&>
z-uL<yt#v}2ihBG1o^z~r=Qyah*pl^XUc33e2bZFhd9LM3HgEm-$e01t@VoZPk~w%q
z2!oD&34cP-^AvNte5Ib~y5zcq=D70z)%KU}-TU<^z5e<3x{vAq|Hbe4t6%%%?f-Z8
z|DL}8Z|VHnpF5ZT+WV*fd+%|%1HU%Q-6{BRq;vVa+8^KL?7tmij^FeCb$s>5J1aLc
z&5qi8@4A*@T5vAMU6Ea$E_;KPE$4PKJG9|q!JY+Ej<ubVnp$wJ!6ahCW1VV~FI!h0
z4bVE>9XLDPNqyt1T}K;CrtRd=c|9T7h^_njWtIN_CEY&~Zt8dmzMIyzI8)>P%pITE
z=A<Y+WopTbst@G!4%paetzGSLaP5}PVB`A&CASrRb=?-+bX2@y*M_`{Mk!ZWb51JW
z7by;|w+gA^wyU`1AN%A&^o#0=#z(v@I-M-y7AL+C<XbEvdiPoAH1>%MY=LY$Hmgj3
z+-4e=8!pauSSPV~&cRbx?<Y>V)|4#&YG2F!6A$;kI2GvkPG2}sQ2A(o%Tk-292$Zt
zv6{^VhJM#yUUZDQsVs2pr4<8HT#4P=+fRdM-dr4Z)7@Z})vE&;OKZIN)H2)FafB;s
z+o?8PNbhI6mcSSo>#|Q>NYd}h#|zu<2Pa;LD0tdD#Z*pQ#Ash(aZ}2d3BmDCdqfRR
z#TF(XlzD&jOQ>J-)aJa$_jvWA@-OWRvyoC!Tfn*S{Y1|-Wwrqpr^QUf);yM-;HdHT
zM(-!DgeeD;&Wq&zalg;UtRJvXUnh~LBJs?NsW)Fll)8r|1!W&DZEaZ*BhPM8_UFsZ
z%177d#lCtm`M2J@gYW)v3yZ$n9cUzcx#My5j1vi8w>{dLKL5tP4UggrLqA^?=f7W<
zZr;~cy;*MFY5seqFSc3RG=5E=-{7D+M^}XVx!lh^AC4@XZO=DZviI4&UqbEX`wTw*
zIB!>?-*B|QX3h4wZ>qnuU-`J6SwP_LTrq}-unO~s;qMOj9k!@@)%&|qatXVA&F!$=
zzdxkcJ@0?F|M&a<`t|ShYacQH|I+^N)ce1$&euO~UC+z*zxJ#E_xxX1mjBxww?X{=
zx(nIf&$n-_`q#Q%N4EY|_xU})ZiUPL{I&D?^=W=SvsW7MB!=w18|L-)MfbAHep56S
zTs>N|*zj?vmiQI7je9MtA6)ozG249h>sSc}uaztLzW7<^ZfCS9f27lJczS%WgJtga
z=cjHT^*X#d<oB+}@A<{Ew$?4(ym|4V4O|QEuFKfVyx?w|;jYVaELR=Z?3((jMA4(X
zWc6jaH}(dX^A^XvY)#yH*DXE0x~e`j-s99%qyFqoVejAYy!Bl6U4G6>uJufpFD?sy
zIm3)$y%Jj>=Tx}{ziQCMC>Lhj(3`%yYITgJ_=QEbA{V%%xKH}!UI=^YwX^#tQ&i%O
z_gmQOB3^}i?YsU#E7zfFYT3%@-ZGvGY+8vCthe5l?atP^x{6`L-l~P%B|a7ztD{l_
z9K1usG9yElX9xL|25_{zT%dEoFsj2Vb^-T=-Tb#!>GWUIYM!<)Vw(H&Q@8c!-Mu9~
zB}39-t<i*scS4ru-s+Vz<IX58+opZ>rQQYBQ%tQ2GA`Dw3oc&@x@J`vHEpY&Z5hv8
zF=zG*+J{R5-Yt;tSh?b^?&2U(-P~27fosbcE~KyV%vX0>-(z$u#3N!#D3ev3>lfw8
zWe-AI!UY-1lwOA3`kQflwaFf@-SU#_l`@*QG;e86GT3y0p>_MrX_IYjUhd_%ST)t`
z0^6awEssNQy^~sS*D!U#T|-@Q<Egu!_GwPH%D!4)cyH&7-^Lu`zQ2=>&hEcl7rR9}
zZg29b(u;mR-QPfW0y&)i;`#G&I=klT%;@EFx#rGEQn*yM`%S3OWxZ}M$+w#vl2QV4
zr`xV9wO*05>hgWP>SfFen6^w7N;#Xo{bgN%r_a(g<)^M@&3Tn)%3^T;<uSFjovW*F
z?78#0vmh@!eP_^Izp|{OUdb22j(RoM?PvD6JZqDUm-y9(r`K1$J96_i-&vkpk`Hc(
zPucaBPxpf9Hhm4oh~C|+3%~DjHs0d!8g#n%vF!EBV=tO*5sp&WQu5<d_OnyBx0qLD
z#Rn{k6<qX4_LK747Bh<-`TsATe#W&3bemMLY{k@93Idt4&-E|kzEpC-mDORX74HJI
zqYAeyuj|O$Jm6Ile*3~;&9Ph$%|~WhqKpgFb}wDGMC-wp++8cCXxmDaz4$J1J3in_
z%<3y?*4DS5diirmuC~a~aXGuyuR36*pMPn<WT)9h;n@s}HhD02Sc{##zU<`(v8>)r
z9VOluA22V7(VendvaTz1dyH<$EoP}HTp3?)88iEofAMULWN7C0{kttXZQG(pU*Gro
z$=cWao_lfS<3D;|)YnViPuRXZqw)HO&3hVutJ{Z36*qt5Yj}B@@BNziC11PE>;JrQ
z{$6oZsl}q^wff@V^|Jfkf=;Orb(&GK;s0IvjD3IZyt%zjn&%g{`JVT8#N|ygxfFLi
zI&XXH<Kbt@XLNU*=4s%&`|aeDFDHL|=}aoMinP1gyY&Ot)StC`*B+UzHM7Lv<L2jn
zP1PZfmrs`y&;R%D$r;VRzg9D_S+AYjyY<0~tH#r1cl~(snEU?keJ`$Ldt5$kSMvI&
zzFou&t)t81^=BMxe4QUr`_eK{V_R}<-P1GAdj9OK|Ge`2ziaE)ExQnCa>Gw*^XEgq
z)kLn!HGf?$b7$Mkq}-PxUvujI>`Siy@xXci=XU%0|BLItylC!UYxDbpTX5wr`|BJh
z-9;SYUpp?jK3gqo-A>{AzFRkz{@OCJcb5QbDPPBp!YS#QS9V>sog{ufZ{y1{iK<S+
zT5HE86M_tCS=;lMzk0o3ql2)Qc164Vv>5g8oq^&rlXvHapLts3yp=^bA@&S=lJ;x<
z70FTaCworaaE@i4z$UTkCpuo!T-R`H+<8L4LBec;rAtde%i9@GF8K+t_l2{{2+o-O
zr7`)#iR1q1X<8wU$qUui@kwz%*6pg*=Q*t5$9(^>yK%>gEna7iJaAtnv&A<`<zm!I
zGo5?#F14DQm$e*8;Prm8MtExN?Eo*{*?Mbwul{|zjO)0b)VI#6vP%`c*VwdH`;_lC
zO~1a7<CH_jnJNK4b+(DOzU}=Pw!v{r*pjfCe7mdKohL1{6gj3%e4aZkaY<_0h6gA1
zYy7&#EOuhkiY+I;geEO{%eO@H^{g364$FQ@RkCrH3a0K<S;V&0YN{85Zn2Dm(&2er
zKTj>}Zr7jmvn?TOrfI}ut<^H#uhbS7N3AZLA2fa0p?5sfewRItbp5`v&#rRUPQ6qP
zhf^B_TCOCW{?4Mkg>7Qy&q>xd0xo(g3ViNeBVkaiD|R?&l4HsWO_c@<UasWpa>Y?g
zLLZbY+}$fb?Zm~9*K20(y?jvCXV1^X&KVnaIVg5ao4s;3`wrWKQrsI~?P?Y}R&)AW
ztj%WWPE)_&-tN_9lO+>{Vy8ULFIwW4`~TpBuGMK`>ofL!s?p>Ct!ud#lKFJvdG3cx
z|NeXAE3dqG<<sPH=?4$a${QbBWB+}Lz@o}OOV{1o^I>Yc(Uiv~)#qd{<=s4B{%Co@
zw;Q%<vO4#^Z2Wib+rIBLE4IBin#@;rO`pl)!@k)DQo<|ub*d-+vH8RQt8r=O+XG51
z8dJ2><@TJGxO~g&-Oru#fBfBT_jmjMeEHw=?F!EJ|C{^&ICH;Tv|TCFZqU%-<(~h4
z9=xxA$6xmsbXxM4_vZEA!|UI3&)@lfWBdF1HxHl7)o1*?l018TtcU;l`Ky00q}{K2
zpzGc*v(t!W!jk3Ye)@AQu3pKUpZ_ZM%;Z-qmUEZBls+T6R3W$e?&II>OaW6K?huuG
zxOQLI=iUnw9Zq&;^4aTLyz#{7!=*y51k=^CUl!NiSlhU~WD>^#193ASnW9JQTfRn=
z7Av%^z5P=+>FQH^?#2gB9<$h^x3aGdZdg*iC%g2p)eOO}GsN2x7YnqkYu&Q?Y!Uat
z9U<1=MHWcfiaZr+`(diXlB9OdA#Tp4vz}2sqCtlpH%Tnrcc>)zyRBnhM9jHozf4t@
zc|2{&dB&%3x`|POQN}Ct)Ojxk-S``i>=Hzr5_QU$;|h$LuUSu-+<fBX+K3X@(=2O)
z_C9i2owS1S%haCBb*sW(_^7Gz9Lh|-)?6@g-xQ0tn;qsEOshR!6E6STHN~z#@6sc=
z^QDZJ5(*b>(7SPFuSZ1LX?KN~hkg&Ll?)wTFnXj^9J&0bEU-9ND<t;J^{qvw5ezF5
zny$Pn$h-0Qrq#8~k5WBf&lieDz4JSExn>=kHv7dQ)6U1!qV`1J=9+A`(q-Dm>5=PG
zo1@kwGERu*iRN(f+kJUj&JsVpBZupn&V7GY)p$waaCL%l`Mlc9+#QE5hU)ZaTW~i9
zoH~E<Xw2!=JPKa(RTxd46v@B)_C~1T+nb<Bx!Ak)mSurE;-e($IDO1E-YckS|8t`^
zq~zDvCvSurc9!pZ{bzdL(!K9M3kTIe2M$eHxVSd1?zQ@K`Cm2+$(J7JzGn!Sv{}vc
zgU#b5ua5N>Y|l|BbL>*?-+ouoCjaTh-wZ3l<FnSzo1R~3-NSg~sJi^~?I+$An6JH>
zmpyI2X<fo?&p%V$-~GQ{|9kzvuh#Z|w*PzNF8}Y6^5fpBZw%Mwr`7ztm%RT={lClV
z@s+<$hR?J6`(gR~`nN|~kN3w`{JHUY{oEZDudcL~@j3*yUBAwiZ&~%kqjsa-f+??7
zuZh~wQW}1q#o<Lxt@vjq>Dt^WL4P$of><3iT}`i+SU=oYv`+9+ZR@q&i!=_DUYWvh
zZFXemrY~Jv1Wd2T6t8*X@y6y->V}eA?_W!l^~|2cEaWjUUc8WDTJ)>~36eJ#7+!fh
z>Gsh*e!l~Yd;h6;hFytyeDT`Xsd+zD-YiI79KDjw>GYb0vJ)!Pti>iPwdBny@iTh3
z`R@hgMHRBUw=1<fO{l4Aop7aogPub`{^{xG{&wEl5_n$VtZdz#CevNst%)oitK`pY
z{kyW;Zfn`eFV5HUL?%mUMg?<h;>~F6l`fk7asK12Ey-8HnVp+gFPt$~C0^JyoMkNo
zhoQeosDR(`msRne3`YZYA6m03U}Drl-Iv}Qd8U2ZTNxhl_)<9EL9y?-<*q_IFL$nT
z>+%avQYn`1oOmQ5vp4FFK1;&hz!aZ#@fEQQTC5MgDaCwQ){*^?DQI5>2k(XV*Gmsb
zrEjn(TF13pRALubRnMg8lqs&c1;;WseB9f+KE1!<lSx|m&WrUEihdkcoW*tcW^0=D
zg(E3nPx#%;`F44+y=V4Ofv$G(?JNJBdM4$&Wyc}$0}7|ip7C+LTfIW*xKqkm)icUm
z<z*)?8RnGhOp5!MGPP=FnA!R_FD9Q(-Z$$8tM-#5nFR^ryI1D_df_{j@lEx2_GM|W
zp6~y&AzbH>*ZOJldtP5yJHMe-_B8)pWv-*l_sffCEjt???{WFyxo5Ksk8Sn7;j22W
z`#+z3YjU~%y+4HvtzM<yWbZRO%&%Blc=l_9WXY|#!dDv}KQIl<Ez6vAzV^?xpN*2q
zan(PM&i}J;|DWl3b=Uj<zW;xFdSC6lZh`L%*XBFr<gKZFdsDss?eqGow>Q_H+xNNc
zDf@e=l3S~P>)$W`c;?l=)0qo34mZD*V0g-sI3;Yo`pQ??YqusnI+eO=r_GHMZk(@e
zV?B6w25K{SF&|hK|K=srK}{*`ZqQZC-#;8OSi{ZelJYFBd!j^3B2R+A%$){ZXM8S1
z7`S~_=|A#*j`I_{3ld)s3hmNtd6KNA{d|!~N~7^CrV|Mq{he3dO*pgur6!Y!>YB?Y
zQ`my#g5ENwG_)mz>B^mJi_KQuweu72rL|l~1Ri=uSaWW7&^-R3P<_+J^|B1t<_7P*
zXYx*0?10Z2S=)Ujc{eWIVr+@o;I@xPC9^;Hspmn1Zpnj3axStzFFNGYY5J_$v~R+j
zrc;T_3a#1V=HFia^uP}re%p0A*{w5T($88gW%69)w?_1HP~olqD_#O=9jnC7ew^iz
z%WW7hF5oz`i(Qtj`^r1XNy*8&;U)1rF7}_g9eJ`3aw)z%zv#<}xeMQ=Uf^mty?DzX
zrQ0S99M0zp4>Jic2!5Sjvj1uDN|i71!r{ATFnp_;H>p|gS*JqbeA)erZj@hHy5?A>
z`t_tK8*WwnWM#L`d8;<_ye#v3_o|~Y$)(nJ7kyTnS)i1!m6SEHROpPKRAmyclJogn
z-_Cbs^KiDOzLoI_(>*bZu{`RiV~yGEQvz>~XgW>*xy(dKtuXn?zXxLc@1E89c)Wf&
zJ?=vO?xTOj)WY^APT_jAgGIe1R$|s&x$R-oK>JQ!8OOiNRb}9-dy;(O4gUngh-+GR
zXOvucKkxS`XEoD<(f23vU795S?*9SZb!RfRt2}Tyu{C;;@pjN20>(G>@7wilK4z5G
z&Hu3HUYSzMj9K$MfA4*o{QbhkzW3?dFCDsj?)>(H{A=#Nul>1D|Ie+x^M8A9-}hR)
z{+0Wz*Q=`;Uf&kU7C&cO_W$nxx5f1z%Ku&B|MSBC&xiAWPuJI-um9bfw`Tcwd7ZH3
z=JA#P-W>kEujJvQSH|&bGhEjvm9Q>Y|MggA>8_po?wvWpQlh-vy2^e2o0m+zrqRp&
zh3|14-f;d;Z=k@#rWc2eLWLL4xM?J*@FIKii~~$l7#tJ@oIHxS&hGqnbArTM@As2F
zxS8$H`*n14^syHfrnR|`N-Pq$3Lcymefhpm=f{})x(^$3ZJbW$$W6Q}Xq#1(bVII6
z;NZQ&)Ha5mXF7`g`?nn4WD_MW;Zkx*C3Blm<=KdI=~>sGWgWaDWUyO)N2W5@A)jyS
z-$X9*-1JVytkgo7`GVKdDGXc7`er4FC9c`^g1P8Lo9cmm%#Zn4g?rZ+oQmz8&eC^F
z@5$ztYY)1Nn>E;)j2gbEcduL1!L=`ZO}CDs#7a?rJ-4jgdLaUOZdt4x%3GZ{cp5sb
zrkJSMiZxiomVfSX;q|+c(3@t%%;5TR=hx=HC)RhLS!`;UaGKL<W{5|IX@n3%B8R%Y
zG^1mz_A||&-s<HHmy&lj*;g%bO4wCzeB}5HnG0{LPHGoht-2Ge=aKZ`$loQ!%ianH
z_dfpV_(mv--%|ZliEz}TO7Xv3OGH`~mZVwO1Qu>?|57ylppt0#HIIEepS7$0Tl>_;
zo-=%-S@$vR>&E#vl1+3BS5D(L%KLCw=xvK(SnMyGPdlQzrPuHM_i}oiMa}QIpRTr_
z{*=*Ib(1%x=eSGaWtVf$E~$jwdcXU&@T{3LLSy!P<&!9W@OS$XwYwERe_mN)o%Q&S
zxV+Er!@G7qy(o9}-LE@`eiXiYwY=cAEQ4OnQ$Ysn*Q<NC$3|;RRXFm-I$ywv!(Ydm
zwP)w{85Psij3Vy)*>3<%NQRxY|26mjr`P*GtpCn%ZM%WqIvx8zKkfhgtpDj>|Kj}p
zZ;$Q22it$w|MxNf{}KDxxT@N(PoJBgpQ9shArkkuZ!`aW>)+?fx0+v1^m+XDx}SUm
zOTh-&<(n+7b2xJ@4Y6P3c7pAK$v>fT*E{>HKkYib`Jqfh5<|y?7pr@hMH#y?tp2T^
z-dY{MdZ*irQ-$U$IUMdvvEAAx+Hf`eY0U4EWoxg!I~7wOcBs{aukOLE(=8KRHDAdU
z?G*jqtG4cWN(|@gJZ-t}6W&zlCiW^V74b^7J;%lP;#%}~hL)L=mrP)Yk!Dpp=^^eU
zFT!9ceKBUv6J?!U?aYj^!pgt0GQvfCi(C3P3$&!H%h=qQaOjLz^n;?PXuCD5=K4)6
z+wykSzK$azuNSTVdSTZ|Q&oBMBe#5$wn~31vh}W9x;vVoXX%@w7prb*o{%!(SS~Dl
zEc5>4?$U)F6P0+UsT&uD3i$oLXVa*gv~5|TfWwk)okv!2pW=V^<n4|Yw(eZ1UshZB
zA`U&$DmpOfNdxPGcc~3iJ$1g%Dqp)HhV6BTR(mf;sz<5Bleh2uIPV<!V!?63Fk;T9
z(@Vc)B+Br=U3RU=+Ir`dsk`TQrU?{$iJoRQ?T=H^rnQVy(>Pq8&G$3vi(8br=+GhG
zaH;pzH~V)9aA;0Gvfzb=vS#k{Wq&-Xj_&A6zVm-kOX;=Dz~dRy-tA=4xyAofrqcFj
zY@!ZF&fDD;&s=ZovQIbBzoFZ>W42RG&Cg3J(=<eurM>#|{9&|xXW)l}+}iwib9*em
zA7xMUc|7a2@2s#xkqlEp=bVu4J#G;<hod~^SNk;m-v@M$X662`w!8A@Soh0>fZO#S
z9!>7AUTS1*`lQV6<B_<ggTKsvH_GmM%Qt!1@_74w>>K{IJihbkQuF-Zf9L<bV*l&m
z{9haQe-zLA|0TKq&VTp+zA`(1U2&fOeoOhc`X7tu&Yin<EBDmD({ttOBP&{iY!_&;
zO<pFx?^A~DEtOwc2UoWXc`fFfn<8wn_nwK#zg@n@UvB3qHnmh_N*RWKkGJb!5Y%Fp
zV4b{kd-X2<g!@O0ZC)?hzWA%F<hfRc0P#b9ubs_!zSuhE?%z>p`?fKyXidTLj~ZdI
zrzAe5txK5WCZAlQ>t-l$=e(%HoLP=j8hN%CvG*k$O5JcGY~HoYDp$*dm1~wUMt)x)
z@;6}XY{NCj+AcdV)J)`ky3W%2`_oic4vqDK(>%62=0B;k3M=dtS5&i#%TIF+3SPKq
zx;T>~Z_Sz}E$;4ho6Mh<^-j@VGFxDGTxHOXUu$>GV4N^Rx2%-ih~?z8chd!y2S4f1
z>nb!cTX*5nuOp5oI+GNVwf40O3Mst^=4tJ;V%fYy%fRn?=RqY~vGq^Bui_S6nY+qR
zh4)&9=)z5p%p&IhvZ<X~%jMn>dL~NXlJ$!>fhTmlGfI+-d}rV8o^Wnwv<HWRpKq+T
zy^-pg&1~KEvbRhUH(H2iO<?GDamp)QIQv;~^~)VW-wOk_&K9iE^R9R)*Sd4=ZvFJH
zQ~f%%W#d$Ij?29N_im0M!w1`iTRnLeJ(J?J+G@5>eABtP=T`mwdX@1?(s}`od9|F1
zv2j_>tg35-%Ux3*9@(xga7_5<M8>o4X1>&p*v8|xKF_LjXOQBq(?XqFcm4lV*m_~<
zboEQdkH1{~@#`*!uCc_K`t#N8`pah>^yIm=Fxu{==yJW^4|0#qI6Hm5@v%2K>!g!A
z4xTCu-+ge+-43hpKC_RH=ls9q{CDoPqs#YMRKMl@c&YR4=k(|Izt(2%Pw$yILut;8
zg}YgrzrOv>Y<x^-z71&A_m0FK%iSfP_MQLtZvW4T^&hU=eXFg1eE$Bonfms1Z{FEg
zJp2C7yLx%)t9|xwN_VTj|9v=axvd0`pZ$lOSC+8n|Et)uCuH}V3l*Zy3!`@LlR9te
z#Nc(2_Xf|RGkpg|%Y~;^eS9NUyZ*|m$bG*I*2d?|*9y40Wk#r!OUy+d+wj{?TaTPc
zXie-`lRD2k!1Gmhpt#^~O@8(rfeUR}qlF4~u-Nt9aysF}u(WM*NSB4_KK00;S;n2m
zG@4o6Jsgu&!+&uo8tbXECW|{ov4oY?1d3a6Eayx+y7J+}8SgZvtXZgheks$+V#&_L
zzk!dA|DAPgsd3M}9UO%MGo2P#iXJ-pwr3sJL4h<&#YE9}TlR{lb~-tzrF93jlwLf~
z#c3GiZ4tO)-)yt&IrpW?6Hj=ac)3v^L+`0_y!JH9<C**QjM%?LnYf=1in{Q<>quar
z(1pp}b9lED^a|ZnsM8M8K2ou>Z1Fd3X`=|s{HvU&jJ@XZT@Bw>>*W@j7SENq#<69Z
zeM#UVA(nrKFFSF}JYg^~@j`?GTen`wqjfRK6YqRKb-h^p^J81CVh?R`y)#K=T-)3v
zl9x6Kzh*yPK7;$-B)+sYV*Z!wzZgH!Tl$CFi)Sf^>6+~ZbFyaaa+tFUbdcw%tn%gS
z1SF2;+6Zh6<(ljK>Z#4fc&3J(21?D++Y1%<dY^mrLx@Rd`*w-e?x<<y`MotSpWc~u
zcSCsJs+lM5R2*G-b@{%I*aY`ucNp69_kTXKad+$Ui*qx+$-ZX_m^rsSbwgX!4{xVq
zA16<+KF6tYEw`M(;>W|=+)F<-e!sux=hx?qKf?E={@uBL)<FjmuG7cg-Sa(c_wV3w
ze{S0_sp2m=8_f>C7SI3p!^rf~$2~sN%Wli2%kMfaF}ck8_x-<&wSQi3&fmRH=3MNz
z#@k_XyXT)f{Lpjhl%0o_S6=%TwugPgKleitn`>Sjc^v=ulK<by`#<lmd9%0vvHqQ3
z?fx|{A6DD%F8dg>{j%lH*7M)0_8yJ4GxT6*&;MC5i;wSqS-a=ug9#fdPOGorVLbLG
z?C6At(u-d?sjQT8iCSK()tCIPt3b5B7Ub()&c>^M>(_sLb8TOD`igC-i&qLby!MrF
zWf5%%5#!(XczJwPqds5ShGkD0UT8}kRg0+2?K*cvgyE~h*{{`VnObui^`|==QJLc+
z9DhooC!O=|Cuj4;CSQ8a^~f=t&fKWeb4lQfTEiOdw<0yX{A<?pw%j>pdE?d51sfgY
zTp$0Mw2L8FD)GjN1yj{m+@JHCai@h%_+D3T<!hhK>^V-%w79&1`N1EBd^dyBZBP22
z=`CqYvaE?cGa)=!F{AOfP?WjOjrIBFKSk6y*A{W-U6^*oEd9;R3)k4v4o2Ge^V#0i
zeCjUI?R)!iNv#yS@0Cne>8%-jhi7kjwAnqiP3_f(bqTRAD^Ew+F#YaGIK5E&xZ;`h
z+`d(&dU9t(a@d||r%A5K{9T&Cuupx)5wE2yyH7~OUeDT_8FoqT?kDlRnvd2!@K`Sq
zDEK2TUn)u=SJn2YE!WTUcfuzK?|H5zQ2AS?`RHR$z3=9F)8xN59N;)1{f+ZM8~Y#Q
zl%FEDlfInVS#>bPyz?yM%}>Eet1|MhbY=@kYO<EzUHDd5cC&8x>}Bn~uLSQEUEf?V
zd7e^M*KV<A&jOFM6gu*A8lSoNW?yBejbQIdliIVk)trpiT$Em2`nkT-MciI2dn>d2
z4py<_PZEybY=8e$mP?WCF5lmsKR*?J(#_A$U&oxld(9%;q)uqx%a8sF8~g6^COp0P
zo9*to*7Y_?ml#_F4mGDfXnEedcvn8x-8;-X{w%Ivx9$IeJ)d;%o7o#o%92d}#&AuV
zck;63{(GvvzyJG7|Nr&)zuMdHeK7z3{PzE=_y1^vw&F;O>wl^KyY%<}71v+O?y2Nh
z?sxycd&|-u!!HrDK6WP9?5P$#{BQMrKgqqje(b-sd)|GGQwD$PjJ>_1pM}2^^QeEw
z$uDq(rJ22_bz7iLfzh!a>={XWW>zSDOmOnq^Xg`Er+Bgwm#@PbPD42#pB^tI6#)&=
zoW$?5=LY#$ThDJ#-&cNr?o3_rJs)Mwc5l94V{QEV>>1<pzxSM9eIWLw&fP0|dRt@o
zw)yxm6*(rIl&YK%z`>W6*n2a_(z5LCZdSqCovKRz&R&=G-|2FsHKbD`AY0>sDr>~X
zn-ix`KN<9Lv%>M?&H2`96`6~~6xSVX&{x=ath`UDsbTAqFO7*YwGpg`7Ykk8c;axe
zhT(&gN(&24Me7E5{A;hP*_xR3(6_U3SM<Hf9>4u&ybw!T)_P1z=jU`m!6Wa#CLYz}
zH=FJithf5riHmD`X5CRpxpq8BC)eEV`2{cY);X%GT*ASdO>QW7tT7PbnRYZHXldNd
zRiC(4@1D4~dD<p!=@jScwLQsMzS|Nuy!zU{Nse!tbKcUXe_JG1Etqf2uGr%5b0lfa
z*9Uec(ffkcyVhjMC9Ti68!r3X(DA4!Z=m7QR9{8WO+wn?8w8)S8|%B@D3#&Li%{68
zkmDYBNIEj&!Ma}=rJE&>_KKddVqp)LX8T_BMdw3A%E~oNovN|x9rCA7_uk}{#d6xR
zd2bNMQgfE~-OH8S-gb!YYAkBz*~Z=Z@`_gJ=PyCI+pSa<mE|OeWleu^a2hk)>EMsM
z=2vRDsZ2cmc!RuXX|>6NE5>fq&nG3UoT%YuwfRq%b4!tiy_rKe&$AEu2U-^v&lL5Y
zWd1?wi0*zKz7;JKPQ`~hP2RL=cSD-WQK7fi1?h#2IU;Kq_OD>Ky1DazW$oWr4z=cj
zmbbS(WGFmeRI$z`YW>^ytdkP18I+ZX?QfModsAVFA(!~-Lo<G6+^_d`@t(dv*y~}h
zO@?$~@cljK>JrN9cg1{TZa#Q*@>k|+t_|C=B6rt5zIC-YzV6?}-}(Cu|K71)f4;8j
z+nMY0YkoZVck0=*v{}oNkNa)8ojI*^g=SmukL?v^Zz>C`J|{Z4oJcY__GU&+n{|QM
z;aitN4I)@eTHiXJn18q_WAf(ZKf)~<pH9DD-)yUCJ|W`6wMnc0Z?5LzICFtxbxH8r
zhgp*)l4Bni3$$_1kxmI>4zw$2j5>Td{l%3VZx(3HSP>l=H;Ly`xOu?ZXKn9S9NO?;
zHP_SITr+!E^X><WxN<#|yC1l1OQeU{h2{nR>0i%WX&1Hsdg?*P(GDpK9+y_}U-M3}
z%CzpC<ap*#uR-3LiY^<`z<J)S=5tGz@<%4Ekj-w3oOb+`m66MnmWiy<iAmQq%q974
zr>^^LYAzrsBU)CWb6YO^Xk1y=i=`dMGo~`SZws5gj3+|Q#o1+Uh+{z5zTp1Z{A-ug
zc6=0@o#6PnPu1b%f+HQXrX9>OGTVCZ<JFHB&aKXSwsYDx0mU0SD>fNSk2ogYwq&Q1
zqM+c~pL0`8j?_Os#?|t6fh9NRs*Kmq)+bbKv0L-wot~?!iptwbJ09^py*0hHUq*h)
z%f>ksiq9(*FHi7V7O8SUB520OB)iPqyXF3YNlj&MD!PO&FSAj&cdRJ8t8H21fdy6x
z^DGQaZWIYQcX9^Z_3}AzvU%#uIu)mu329|Z_mqTd-d$jIF7g?l<F<!8`P6sIe^qpM
zTxH|g@^AaJYT=7d-mE;#&Uf$LCEo6;6P+pNPd&Jvva%yF(bDR5c0C^tpXEC1tImS&
zcbQrgG?lE(l~-8AUjN6-r6A;dz^fatc5fGG2`$Qenis-4>D`}##*S(0W0V%%dZ5rH
z^vv8n`PlOK)UDNwH+{~{Q(CH)_Uz=0lQS!g(jNWTQ*?K4@$#vrtaE2rYW_X&?%yBf
z-;Ig)AF`Ldeq{dhkot4>Zz2hzvrle%T3sLi<5l-&`8~g{-Ff)^p3TRb$J5uv*vOkp
zCttRlV-#stSTj#>>dHmDpX<F-46pFZR_2Hu?)$j+!DKm;_v_7g7Hel{o>HtAI#^eC
z^}L*Q&4ULw_Uqb;EOg=%WXx~eG)=h4^UcJLHNQjZ61{pphMZC1<<O1TxY(wTvt97S
zVdK>zZV#N=&TTrNb+2OK{1?+W{<KbM?mSRFMgHqnqr+BO<|p!Plmpjn4%t=z_2Q}?
zIk7KdKhJKh?#s?zW_7{n+MSLLhj~w%6XrNgF)%Ff>%ZCAwWf(BLcmITkJ9GO<{7MK
ztyOO9Y2CM&VRlsH=BYFD)`q@d6HBvHTw<Tc@aM|b=xG-&9SCiRyuy38$h-Vt!dIaU
zm#3{Y+7~vn>xOTG>4wJvshs=U)Pf@&P2R0c)An|ZbUc<97;^Vh=<1xC49B=_Z{E9G
z{;IRH;ogEZ>4ssBCSe<&YOP__J<n<B!f}D2nmb}wwI7F&wxI|Ym-gmYIW<Sl$lRal
z&NlG@8>3jH%mHIb6|n@dQumqKyc<(2WvA^rJ@3-$y_Zs_m_`VhiAw0aa*TPXePLs)
z@!^b3^A^30INmGzKxOXI&V^-fHq76hyX%49nZCvI7M0q67OB(<(>kIp^!LK@aKj(G
ztNL9Em-h2iTYb5x^*K}^{9)E%F&+;6XHuJVR3F}GiaIy-OX$M8`%3ukF7aC-KG#{5
zYl--Rl~ZI+KHcQJBY%I<oRxB|CCB1s1<#Q*FP_v<wzpu4wZhW7Z}01wZwkLMX;)*&
z!{kb3?q(mE`reLX@pVh<UU>X^F)=8spv0e#@%OLyI(iz%pZ4nTrJk94ly&KqOHV)N
z#sB~GsA=__r$@Q-)^1BtobrX)I8yEJfo0G9{@2$f{AXPH@Y)X3#Fb_Y3=9mOu6{1-
HoD!M<W{h*h

literal 0
HcmV?d00001

-- 
GitLab


From 390959cbb9ab81014ca6f68456ce9058114a7d78 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 14:56:45 -0800
Subject: [PATCH 0909/2345] Internal change.

PiperOrigin-RevId: 229823501
---
 tensorflow/compiler/xla/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/compiler/xla/BUILD b/tensorflow/compiler/xla/BUILD
index aa43fc8dab..636e5ef721 100644
--- a/tensorflow/compiler/xla/BUILD
+++ b/tensorflow/compiler/xla/BUILD
@@ -826,6 +826,7 @@ cc_library(
         "debug_options_parsers.h",
     ],
     hdrs = ["debug_options_flags.h"],
+    visibility = [":friends"],
     deps =
         [
             ":parse_flags_from_env",
-- 
GitLab


From 099b9479b548c2bb1b4460c2fe69ee3c319ed56b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 15:08:23 -0800
Subject: [PATCH 0910/2345] Redirect python.training.distribute and
 python.training.distribution_strategy_context (which are currently just
 forwarding stubs) to python.distribute.*.

PiperOrigin-RevId: 229825818
---
 tensorflow/contrib/distribute/__init__.py     | 4 ++--
 tensorflow/python/eager/tape.py               | 4 ++--
 tensorflow/python/keras/engine/training.py    | 2 +-
 tensorflow/python/training/session_manager.py | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/distribute/__init__.py b/tensorflow/contrib/distribute/__init__.py
index 1bcc453a7e..59d76f5d1c 100644
--- a/tensorflow/contrib/distribute/__init__.py
+++ b/tensorflow/contrib/distribute/__init__.py
@@ -35,8 +35,8 @@ from tensorflow.contrib.distribute.python.tpu_strategy import TPUStrategy
 from tensorflow.python.distribute.cross_device_ops import *
 from tensorflow.python.distribute.distribute_config import DistributeConfig
 from tensorflow.python.distribute.distribute_coordinator import run_standard_tensorflow_server
-from tensorflow.python.training.distribute import *
-from tensorflow.python.training.distribution_strategy_context import *
+from tensorflow.python.distribute.distribute_lib import *
+from tensorflow.python.distribute.distribution_strategy_context import *
 
 from tensorflow.python.util.all_util import remove_undocumented
 
diff --git a/tensorflow/python/eager/tape.py b/tensorflow/python/eager/tape.py
index 56d696bf92..e5d6007b48 100644
--- a/tensorflow/python/eager/tape.py
+++ b/tensorflow/python/eager/tape.py
@@ -27,8 +27,8 @@ from tensorflow.python.util.lazy_loader import LazyLoader
 # distribution_strategy_context.
 # TODO(b/117329403): Remove this circular dependency.
 distribution_strategy_context = LazyLoader(
-    "distribute_lib", globals(),
-    "tensorflow.python.training."
+    "distribution_strategy_context", globals(),
+    "tensorflow.python.distribute."
     "distribution_strategy_context")
 
 
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index e6ede64573..b80b906f67 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -26,6 +26,7 @@ from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.distribute import distribute_coordinator as dc
 from tensorflow.python.distribute import distribute_coordinator_context as dc_context
+from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
@@ -48,7 +49,6 @@ from tensorflow.python.keras.utils.generic_utils import slice_arrays
 from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.training import distribution_strategy_context
 from tensorflow.python.training import optimizer as tf_optimizer_module
 from tensorflow.python.training.checkpointable import base as checkpointable
 from tensorflow.python.training.mode_keys import ModeKeys
diff --git a/tensorflow/python/training/session_manager.py b/tensorflow/python/training/session_manager.py
index 7bd0891e35..104247e60e 100644
--- a/tensorflow/python/training/session_manager.py
+++ b/tensorflow/python/training/session_manager.py
@@ -21,11 +21,11 @@ import time
 import numpy as np
 
 from tensorflow.python.client import session
+from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import checkpoint_management
-from tensorflow.python.training import distribution_strategy_context
 from tensorflow.python.util.tf_export import tf_export
 
 
-- 
GitLab


From 1c73610b59ee174a2d1d54a3624158bb199a6439 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Thu, 17 Jan 2019 15:08:38 -0800
Subject: [PATCH 0911/2345] Internal change.

PiperOrigin-RevId: 229825850
---
 .../tools/ci_build/windows/cpu/pip/build_tf_windows.sh      | 6 +++++-
 .../tools/ci_build/windows/gpu/pip/build_tf_windows.sh      | 6 +++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
index 7ec386a60e..2eeb8dfa1a 100644
--- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
@@ -104,7 +104,11 @@ else
 fi
 
 if [[ "$TF_NIGHTLY" == 1 ]]; then
-  python tensorflow/tools/ci_build/update_version.py --nightly
+  if [[ ${PROJECT_NAME} == *"2.0_preview"* ]]; then
+    python tensorflow/tools/ci_build/update_version.py --version=2.0.0-preview
+  else
+    python tensorflow/tools/ci_build/update_version.py --nightly
+  fi
   if [ -z ${PROJECT_NAME} ]; then
     EXTRA_PIP_FLAGS="--nightly_flag"
   else
diff --git a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
index c076157b08..4cca4f9acf 100644
--- a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
@@ -104,7 +104,11 @@ else
 fi
 
 if [[ "$TF_NIGHTLY" == 1 ]]; then
-  python tensorflow/tools/ci_build/update_version.py --nightly
+  if [[ ${PROJECT_NAME} == *"2.0_preview"* ]]; then
+    python tensorflow/tools/ci_build/update_version.py --version=2.0.0-preview
+  else
+    python tensorflow/tools/ci_build/update_version.py --nightly
+  fi
   if [ -z ${PROJECT_NAME} ]; then
     EXTRA_PIP_FLAGS="--nightly_flag"
   else
-- 
GitLab


From 28401a50f1839f44c76f7b202c8766ecb61fc569 Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne <reedwm@google.com>
Date: Thu, 17 Jan 2019 15:13:50 -0800
Subject: [PATCH 0912/2345] Automated rollback of commit
 2435a1875b574f5d299b1dee431ab5ceccd6132f

PiperOrigin-RevId: 229826749
---
 tensorflow/python/BUILD                  |  6 +-
 tensorflow/python/platform/app.py        | 98 ++++++++++++++++++++++--
 tensorflow/python/platform/googletest.py | 12 +--
 tensorflow/tools/pip_package/BUILD       |  5 --
 4 files changed, 99 insertions(+), 22 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index d17c57b05f..d4e6cbf804 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -213,7 +213,6 @@ py_library(
         ":pywrap_tensorflow",
         ":util",
         "//tensorflow/core:protos_all_py",
-        "@absl_py//absl:app",
         "@absl_py//absl/flags",
         "@six_archive//:six",
     ],
@@ -234,10 +233,7 @@ py_library(
     name = "platform_test",
     srcs = ["platform/googletest.py"],
     srcs_version = "PY2AND3",
-    deps = [
-        ":platform_benchmark",
-        "@absl_py//absl/testing:absltest",
-    ],
+    deps = [":platform_benchmark"],
 )
 
 tf_py_test(
diff --git a/tensorflow/python/platform/app.py b/tensorflow/python/platform/app.py
index 303b70ff57..7b917235c0 100644
--- a/tensorflow/python/platform/app.py
+++ b/tensorflow/python/platform/app.py
@@ -18,23 +18,109 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import errno as _errno
 import sys as _sys
 
-from absl.app import run as _run
-
 from tensorflow.python.platform import flags
 from tensorflow.python.util.tf_export import tf_export
 
 
-def _parse_flags_tolerate_undef(argv):
-  """Parse args, returning any unknown flags (ABSL defaults to crashing)."""
-  return flags.FLAGS(_sys.argv if argv is None else argv, known_only=True)
+def _usage(shorthelp):
+  """Writes __main__'s docstring to stdout with some help text.
+
+  Args:
+    shorthelp: bool, if True, prints only flags from the main module,
+        rather than all flags.
+  """
+  doc = _sys.modules['__main__'].__doc__
+  if not doc:
+    doc = '\nUSAGE: %s [flags]\n' % _sys.argv[0]
+    doc = flags.text_wrap(doc, indent='       ', firstline_indent='')
+  else:
+    # Replace all '%s' with sys.argv[0], and all '%%' with '%'.
+    num_specifiers = doc.count('%') - 2 * doc.count('%%')
+    try:
+      doc %= (_sys.argv[0],) * num_specifiers
+    except (OverflowError, TypeError, ValueError):
+      # Just display the docstring as-is.
+      pass
+  if shorthelp:
+    flag_str = flags.FLAGS.main_module_help()
+  else:
+    flag_str = str(flags.FLAGS)
+  try:
+    _sys.stdout.write(doc)
+    if flag_str:
+      _sys.stdout.write('\nflags:\n')
+      _sys.stdout.write(flag_str)
+    _sys.stdout.write('\n')
+  except IOError as e:
+    # We avoid printing a huge backtrace if we get EPIPE, because
+    # "foo.par --help | less" is a frequent use case.
+    if e.errno != _errno.EPIPE:
+      raise
+
+
+class _HelpFlag(flags.BooleanFlag):
+  """Special boolean flag that displays usage and raises SystemExit."""
+  NAME = 'help'
+  SHORT_NAME = 'h'
+
+  def __init__(self):
+    super(_HelpFlag, self).__init__(
+        self.NAME, False, 'show this help', short_name=self.SHORT_NAME)
+
+  def parse(self, arg):
+    if arg:
+      _usage(shorthelp=True)
+      print()
+      print('Try --helpfull to get a list of all flags.')
+      _sys.exit(1)
+
+
+class _HelpshortFlag(_HelpFlag):
+  """--helpshort is an alias for --help."""
+  NAME = 'helpshort'
+  SHORT_NAME = None
+
+
+class _HelpfullFlag(flags.BooleanFlag):
+  """Display help for flags in main module and all dependent modules."""
+
+  def __init__(self):
+    super(_HelpfullFlag, self).__init__('helpfull', False, 'show full help')
+
+  def parse(self, arg):
+    if arg:
+      _usage(shorthelp=False)
+      _sys.exit(1)
+
+
+_define_help_flags_called = False
+
+
+def _define_help_flags():
+  global _define_help_flags_called
+  if not _define_help_flags_called:
+    flags.DEFINE_flag(_HelpFlag())
+    flags.DEFINE_flag(_HelpfullFlag())
+    flags.DEFINE_flag(_HelpshortFlag())
+    _define_help_flags_called = True
 
 
 @tf_export(v1=['app.run'])
 def run(main=None, argv=None):
   """Runs the program with an optional 'main' function and 'argv' list."""
 
+  # Define help flags.
+  _define_help_flags()
+
+  # Parse known flags.
+  argv = flags.FLAGS(_sys.argv if argv is None else argv, known_only=True)
+
   main = main or _sys.modules['__main__'].main
 
-  _run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
+  # Call the main function, passing through any arguments
+  # to the final program.
+  _sys.exit(main(argv))
+
diff --git a/tensorflow/python/platform/googletest.py b/tensorflow/python/platform/googletest.py
index 802721e34b..fe4b0d0d37 100644
--- a/tensorflow/python/platform/googletest.py
+++ b/tensorflow/python/platform/googletest.py
@@ -13,7 +13,7 @@
 # limitations under the License.
 # ==============================================================================
 
-"""Imports absltest as a replacement for testing.pybase.googletest."""
+"""Imports unittest as a replacement for testing.pybase.googletest."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -26,7 +26,7 @@ import tempfile
 
 # go/tf-wildcard-import
 # pylint: disable=wildcard-import
-from absl.testing.absltest import *
+from unittest import *
 # pylint: enable=wildcard-import
 
 from tensorflow.python.framework import errors
@@ -41,7 +41,7 @@ from tensorflow.python.util.tf_export import tf_export
 
 Benchmark = benchmark.TensorFlowBenchmark  # pylint: disable=invalid-name
 
-absltest_main = main
+unittest_main = main
 
 # We keep a global variable in this module to make sure we create the temporary
 # directory only once per test binary invocation.
@@ -51,7 +51,7 @@ _googletest_temp_dir = ''
 # pylint: disable=invalid-name
 # pylint: disable=undefined-variable
 def g_main(argv):
-  """Delegate to absltest.main after redefining testLoader."""
+  """Delegate to unittest.main after redefining testLoader."""
   if 'TEST_SHARD_STATUS_FILE' in os.environ:
     try:
       f = None
@@ -67,7 +67,7 @@ def g_main(argv):
 
   if ('TEST_TOTAL_SHARDS' not in os.environ or
       'TEST_SHARD_INDEX' not in os.environ):
-    return absltest_main(argv=argv)
+    return unittest_main(argv=argv)
 
   total_shards = int(os.environ['TEST_TOTAL_SHARDS'])
   shard_index = int(os.environ['TEST_SHARD_INDEX'])
@@ -87,7 +87,7 @@ def g_main(argv):
   # Override getTestCaseNames
   base_loader.getTestCaseNames = getShardedTestCaseNames
 
-  absltest_main(argv=argv, testLoader=base_loader)
+  unittest_main(argv=argv, testLoader=base_loader)
 
 
 # Redefine main to allow running benchmarks
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index 3c1a9280a2..c51b45a49c 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -146,11 +146,7 @@ filegroup(
         "//third_party/eigen3:LICENSE",
         "//third_party/fft2d:LICENSE",
         "//third_party/hadoop:LICENSE.txt",
-        "@absl_py//absl:LICENSE",
-        "@absl_py//absl/logging:LICENSE",
         "@absl_py//absl/flags:LICENSE",
-        "@absl_py//absl/testing:LICENSE",
-        "@absl_py//absl/third_party/unittest3_backport:LICENSE",
         "@arm_neon_2_x86_sse//:LICENSE",
         "@astor_archive//:LICENSE",
         "@boringssl//:LICENSE",
@@ -159,7 +155,6 @@ filegroup(
         "@curl//:COPYING",
         "@double_conversion//:LICENSE",
         "@eigen_archive//:COPYING.MPL2",
-        "@enum34_archive//:LICENSE",
         "@farmhash_archive//:COPYING",
         "@fft2d//:fft/readme.txt",
         "@flatbuffers//:LICENSE.txt",
-- 
GitLab


From 097a123794a5ee83b2212991245c9ac0ff8f6c8d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 15:23:45 -0800
Subject: [PATCH 0913/2345] Wrap word2vec_basic inside a main method

PiperOrigin-RevId: 229828434
---
 .../tutorials/word2vec/word2vec_basic.py      | 634 +++++++++---------
 1 file changed, 319 insertions(+), 315 deletions(-)

diff --git a/tensorflow/examples/tutorials/word2vec/word2vec_basic.py b/tensorflow/examples/tutorials/word2vec/word2vec_basic.py
index b09ee99768..805ec203b4 100644
--- a/tensorflow/examples/tutorials/word2vec/word2vec_basic.py
+++ b/tensorflow/examples/tutorials/word2vec/word2vec_basic.py
@@ -18,12 +18,12 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import argparse
 import collections
 import math
 import os
-import sys
-import argparse
 import random
+import sys
 from tempfile import gettempdir
 import zipfile
 
@@ -34,320 +34,324 @@ import tensorflow as tf
 
 from tensorflow.contrib.tensorboard.plugins import projector
 
-# Give a folder path as an argument with '--log_dir' to save
-# TensorBoard summaries. Default is a log folder in current directory.
-current_path = os.path.dirname(os.path.realpath(sys.argv[0]))
-
-parser = argparse.ArgumentParser()
-parser.add_argument(
-    '--log_dir',
-    type=str,
-    default=os.path.join(current_path, 'log'),
-    help='The log directory for TensorBoard summaries.')
-FLAGS, unparsed = parser.parse_known_args()
-
-# Create the directory for TensorBoard variables if there is not.
-if not os.path.exists(FLAGS.log_dir):
-  os.makedirs(FLAGS.log_dir)
-
-# Step 1: Download the data.
-url = 'http://mattmahoney.net/dc/'
-
-
-# pylint: disable=redefined-outer-name
-def maybe_download(filename, expected_bytes):
-  """Download a file if not present, and make sure it's the right size."""
-  local_filename = os.path.join(gettempdir(), filename)
-  if not os.path.exists(local_filename):
-    local_filename, _ = urllib.request.urlretrieve(url + filename,
-                                                   local_filename)
-  statinfo = os.stat(local_filename)
-  if statinfo.st_size == expected_bytes:
-    print('Found and verified', filename)
-  else:
-    print(statinfo.st_size)
-    raise Exception('Failed to verify ' + local_filename +
-                    '. Can you get to it with a browser?')
-  return local_filename
-
-
-filename = maybe_download('text8.zip', 31344016)
-
-
-# Read the data into a list of strings.
-def read_data(filename):
-  """Extract the first file enclosed in a zip file as a list of words."""
-  with zipfile.ZipFile(filename) as f:
-    data = tf.compat.as_str(f.read(f.namelist()[0])).split()
-  return data
-
-
-vocabulary = read_data(filename)
-print('Data size', len(vocabulary))
-
-# Step 2: Build the dictionary and replace rare words with UNK token.
-vocabulary_size = 50000
-
-
-def build_dataset(words, n_words):
-  """Process raw inputs into a dataset."""
-  count = [['UNK', -1]]
-  count.extend(collections.Counter(words).most_common(n_words - 1))
-  dictionary = dict()
-  for word, _ in count:
-    dictionary[word] = len(dictionary)
-  data = list()
-  unk_count = 0
-  for word in words:
-    index = dictionary.get(word, 0)
-    if index == 0:  # dictionary['UNK']
-      unk_count += 1
-    data.append(index)
-  count[0][1] = unk_count
-  reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))
-  return data, count, dictionary, reversed_dictionary
-
-
-# Filling 4 global variables:
-# data - list of codes (integers from 0 to vocabulary_size-1).
-#   This is the original text but words are replaced by their codes
-# count - map of words(strings) to count of occurrences
-# dictionary - map of words(strings) to their codes(integers)
-# reverse_dictionary - maps codes(integers) to words(strings)
-data, count, dictionary, reverse_dictionary = build_dataset(
-    vocabulary, vocabulary_size)
-del vocabulary  # Hint to reduce memory.
-print('Most common words (+UNK)', count[:5])
-print('Sample data', data[:10], [reverse_dictionary[i] for i in data[:10]])
-
 data_index = 0
 
 
-# Step 3: Function to generate a training batch for the skip-gram model.
-def generate_batch(batch_size, num_skips, skip_window):
-  global data_index
-  assert batch_size % num_skips == 0
-  assert num_skips <= 2 * skip_window
-  batch = np.ndarray(shape=(batch_size), dtype=np.int32)
-  labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
-  span = 2 * skip_window + 1  # [ skip_window target skip_window ]
-  buffer = collections.deque(maxlen=span)  # pylint: disable=redefined-builtin
-  if data_index + span > len(data):
-    data_index = 0
-  buffer.extend(data[data_index:data_index + span])
-  data_index += span
-  for i in range(batch_size // num_skips):
-    context_words = [w for w in range(span) if w != skip_window]
-    words_to_use = random.sample(context_words, num_skips)
-    for j, context_word in enumerate(words_to_use):
-      batch[i * num_skips + j] = buffer[skip_window]
-      labels[i * num_skips + j, 0] = buffer[context_word]
-    if data_index == len(data):
-      buffer.extend(data[0:span])
-      data_index = span
+def word2vec_basic(log_dir):
+  """Example of building, training and visualizing a word2vec model."""
+  # Create the directory for TensorBoard variables if there is not.
+  if not os.path.exists(log_dir):
+    os.makedirs(log_dir)
+
+  # Step 1: Download the data.
+  url = 'http://mattmahoney.net/dc/'
+
+  # pylint: disable=redefined-outer-name
+  def maybe_download(filename, expected_bytes):
+    """Download a file if not present, and make sure it's the right size."""
+    local_filename = os.path.join(gettempdir(), filename)
+    if not os.path.exists(local_filename):
+      local_filename, _ = urllib.request.urlretrieve(url + filename,
+                                                     local_filename)
+    statinfo = os.stat(local_filename)
+    if statinfo.st_size == expected_bytes:
+      print('Found and verified', filename)
     else:
-      buffer.append(data[data_index])
-      data_index += 1
-  # Backtrack a little bit to avoid skipping words in the end of a batch
-  data_index = (data_index + len(data) - span) % len(data)
-  return batch, labels
-
-
-batch, labels = generate_batch(batch_size=8, num_skips=2, skip_window=1)
-for i in range(8):
-  print(batch[i], reverse_dictionary[batch[i]], '->', labels[i, 0],
-        reverse_dictionary[labels[i, 0]])
-
-# Step 4: Build and train a skip-gram model.
-
-batch_size = 128
-embedding_size = 128  # Dimension of the embedding vector.
-skip_window = 1  # How many words to consider left and right.
-num_skips = 2  # How many times to reuse an input to generate a label.
-num_sampled = 64  # Number of negative examples to sample.
-
-# We pick a random validation set to sample nearest neighbors. Here we limit the
-# validation samples to the words that have a low numeric ID, which by
-# construction are also the most frequent. These 3 variables are used only for
-# displaying model accuracy, they don't affect calculation.
-valid_size = 16  # Random set of words to evaluate similarity on.
-valid_window = 100  # Only pick dev samples in the head of the distribution.
-valid_examples = np.random.choice(valid_window, valid_size, replace=False)
-
-graph = tf.Graph()
-
-with graph.as_default():
-
-  # Input data.
-  with tf.name_scope('inputs'):
-    train_inputs = tf.placeholder(tf.int32, shape=[batch_size])
-    train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
-    valid_dataset = tf.constant(valid_examples, dtype=tf.int32)
-
-  # Ops and variables pinned to the CPU because of missing GPU implementation
-  with tf.device('/cpu:0'):
-    # Look up embeddings for inputs.
-    with tf.name_scope('embeddings'):
-      embeddings = tf.Variable(
-          tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
-      embed = tf.nn.embedding_lookup(embeddings, train_inputs)
-
-    # Construct the variables for the NCE loss
-    with tf.name_scope('weights'):
-      nce_weights = tf.Variable(
-          tf.truncated_normal(
-              [vocabulary_size, embedding_size],
-              stddev=1.0 / math.sqrt(embedding_size)))
-    with tf.name_scope('biases'):
-      nce_biases = tf.Variable(tf.zeros([vocabulary_size]))
-
-  # Compute the average NCE loss for the batch.
-  # tf.nce_loss automatically draws a new sample of the negative labels each
-  # time we evaluate the loss.
-  # Explanation of the meaning of NCE loss:
-  #   http://mccormickml.com/2016/04/19/word2vec-tutorial-the-skip-gram-model/
-  with tf.name_scope('loss'):
-    loss = tf.reduce_mean(
-        tf.nn.nce_loss(
-            weights=nce_weights,
-            biases=nce_biases,
-            labels=train_labels,
-            inputs=embed,
-            num_sampled=num_sampled,
-            num_classes=vocabulary_size))
-
-  # Add the loss value as a scalar to summary.
-  tf.summary.scalar('loss', loss)
-
-  # Construct the SGD optimizer using a learning rate of 1.0.
-  with tf.name_scope('optimizer'):
-    optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)
-
-  # Compute the cosine similarity between minibatch examples and all embeddings.
-  norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keepdims=True))
-  normalized_embeddings = embeddings / norm
-  valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings,
-                                            valid_dataset)
-  similarity = tf.matmul(
-      valid_embeddings, normalized_embeddings, transpose_b=True)
-
-  # Merge all summaries.
-  merged = tf.summary.merge_all()
-
-  # Add variable initializer.
-  init = tf.global_variables_initializer()
-
-  # Create a saver.
-  saver = tf.train.Saver()
-
-# Step 5: Begin training.
-num_steps = 100001
-
-with tf.Session(graph=graph) as session:
-  # Open a writer to write summaries.
-  writer = tf.summary.FileWriter(FLAGS.log_dir, session.graph)
-
-  # We must initialize all variables before we use them.
-  init.run()
-  print('Initialized')
-
-  average_loss = 0
-  for step in xrange(num_steps):
-    batch_inputs, batch_labels = generate_batch(batch_size, num_skips,
-                                                skip_window)
-    feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels}
-
-    # Define metadata variable.
-    run_metadata = tf.RunMetadata()
-
-    # We perform one update step by evaluating the optimizer op (including it
-    # in the list of returned values for session.run()
-    # Also, evaluate the merged op to get all summaries from the returned "summary" variable.
-    # Feed metadata variable to session for visualizing the graph in TensorBoard.
-    _, summary, loss_val = session.run(
-        [optimizer, merged, loss],
-        feed_dict=feed_dict,
-        run_metadata=run_metadata)
-    average_loss += loss_val
-
-    # Add returned summaries to writer in each step.
-    writer.add_summary(summary, step)
-    # Add metadata to visualize the graph for the last run.
-    if step == (num_steps - 1):
-      writer.add_run_metadata(run_metadata, 'step%d' % step)
-
-    if step % 2000 == 0:
-      if step > 0:
-        average_loss /= 2000
-      # The average loss is an estimate of the loss over the last 2000 batches.
-      print('Average loss at step ', step, ': ', average_loss)
-      average_loss = 0
-
-    # Note that this is expensive (~20% slowdown if computed every 500 steps)
-    if step % 10000 == 0:
-      sim = similarity.eval()
-      for i in xrange(valid_size):
-        valid_word = reverse_dictionary[valid_examples[i]]
-        top_k = 8  # number of nearest neighbors
-        nearest = (-sim[i, :]).argsort()[1:top_k + 1]
-        log_str = 'Nearest to %s:' % valid_word
-        for k in xrange(top_k):
-          close_word = reverse_dictionary[nearest[k]]
-          log_str = '%s %s,' % (log_str, close_word)
-        print(log_str)
-  final_embeddings = normalized_embeddings.eval()
-
-  # Write corresponding labels for the embeddings.
-  with open(FLAGS.log_dir + '/metadata.tsv', 'w') as f:
-    for i in xrange(vocabulary_size):
-      f.write(reverse_dictionary[i] + '\n')
-
-  # Save the model for checkpoints.
-  saver.save(session, os.path.join(FLAGS.log_dir, 'model.ckpt'))
-
-  # Create a configuration for visualizing embeddings with the labels in TensorBoard.
-  config = projector.ProjectorConfig()
-  embedding_conf = config.embeddings.add()
-  embedding_conf.tensor_name = embeddings.name
-  embedding_conf.metadata_path = os.path.join(FLAGS.log_dir, 'metadata.tsv')
-  projector.visualize_embeddings(writer, config)
-
-writer.close()
-
-# Step 6: Visualize the embeddings.
-
-
-# pylint: disable=missing-docstring
-# Function to draw visualization of distance between embeddings.
-def plot_with_labels(low_dim_embs, labels, filename):
-  assert low_dim_embs.shape[0] >= len(labels), 'More labels than embeddings'
-  plt.figure(figsize=(18, 18))  # in inches
-  for i, label in enumerate(labels):
-    x, y = low_dim_embs[i, :]
-    plt.scatter(x, y)
-    plt.annotate(
-        label,
-        xy=(x, y),
-        xytext=(5, 2),
-        textcoords='offset points',
-        ha='right',
-        va='bottom')
-
-  plt.savefig(filename)
-
-
-try:
-  # pylint: disable=g-import-not-at-top
-  from sklearn.manifold import TSNE
-  import matplotlib.pyplot as plt
-
-  tsne = TSNE(
-      perplexity=30, n_components=2, init='pca', n_iter=5000, method='exact')
-  plot_only = 500
-  low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only, :])
-  labels = [reverse_dictionary[i] for i in xrange(plot_only)]
-  plot_with_labels(low_dim_embs, labels, os.path.join(gettempdir(), 'tsne.png'))
-
-except ImportError as ex:
-  print('Please install sklearn, matplotlib, and scipy to show embeddings.')
-  print(ex)
+      print(statinfo.st_size)
+      raise Exception('Failed to verify ' + local_filename +
+                      '. Can you get to it with a browser?')
+    return local_filename
+
+  filename = maybe_download('text8.zip', 31344016)
+
+  # Read the data into a list of strings.
+  def read_data(filename):
+    """Extract the first file enclosed in a zip file as a list of words."""
+    with zipfile.ZipFile(filename) as f:
+      data = tf.compat.as_str(f.read(f.namelist()[0])).split()
+    return data
+
+  vocabulary = read_data(filename)
+  print('Data size', len(vocabulary))
+
+  # Step 2: Build the dictionary and replace rare words with UNK token.
+  vocabulary_size = 50000
+
+  def build_dataset(words, n_words):
+    """Process raw inputs into a dataset."""
+    count = [['UNK', -1]]
+    count.extend(collections.Counter(words).most_common(n_words - 1))
+    dictionary = dict()
+    for word, _ in count:
+      dictionary[word] = len(dictionary)
+    data = list()
+    unk_count = 0
+    for word in words:
+      index = dictionary.get(word, 0)
+      if index == 0:  # dictionary['UNK']
+        unk_count += 1
+      data.append(index)
+    count[0][1] = unk_count
+    reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))
+    return data, count, dictionary, reversed_dictionary
+
+  # Filling 4 global variables:
+  # data - list of codes (integers from 0 to vocabulary_size-1).
+  #   This is the original text but words are replaced by their codes
+  # count - map of words(strings) to count of occurrences
+  # dictionary - map of words(strings) to their codes(integers)
+  # reverse_dictionary - maps codes(integers) to words(strings)
+  data, count, unused_dictionary, reverse_dictionary = build_dataset(
+      vocabulary, vocabulary_size)
+  del vocabulary  # Hint to reduce memory.
+  print('Most common words (+UNK)', count[:5])
+  print('Sample data', data[:10], [reverse_dictionary[i] for i in data[:10]])
+
+  # Step 3: Function to generate a training batch for the skip-gram model.
+  def generate_batch(batch_size, num_skips, skip_window):
+    global data_index
+    assert batch_size % num_skips == 0
+    assert num_skips <= 2 * skip_window
+    batch = np.ndarray(shape=(batch_size), dtype=np.int32)
+    labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
+    span = 2 * skip_window + 1  # [ skip_window target skip_window ]
+    buffer = collections.deque(maxlen=span)  # pylint: disable=redefined-builtin
+    if data_index + span > len(data):
+      data_index = 0
+    buffer.extend(data[data_index:data_index + span])
+    data_index += span
+    for i in range(batch_size // num_skips):
+      context_words = [w for w in range(span) if w != skip_window]
+      words_to_use = random.sample(context_words, num_skips)
+      for j, context_word in enumerate(words_to_use):
+        batch[i * num_skips + j] = buffer[skip_window]
+        labels[i * num_skips + j, 0] = buffer[context_word]
+      if data_index == len(data):
+        buffer.extend(data[0:span])
+        data_index = span
+      else:
+        buffer.append(data[data_index])
+        data_index += 1
+    # Backtrack a little bit to avoid skipping words in the end of a batch
+    data_index = (data_index + len(data) - span) % len(data)
+    return batch, labels
+
+  batch, labels = generate_batch(batch_size=8, num_skips=2, skip_window=1)
+  for i in range(8):
+    print(batch[i], reverse_dictionary[batch[i]], '->', labels[i, 0],
+          reverse_dictionary[labels[i, 0]])
+
+  # Step 4: Build and train a skip-gram model.
+
+  batch_size = 128
+  embedding_size = 128  # Dimension of the embedding vector.
+  skip_window = 1  # How many words to consider left and right.
+  num_skips = 2  # How many times to reuse an input to generate a label.
+  num_sampled = 64  # Number of negative examples to sample.
+
+  # We pick a random validation set to sample nearest neighbors. Here we limit
+  # the validation samples to the words that have a low numeric ID, which by
+  # construction are also the most frequent. These 3 variables are used only for
+  # displaying model accuracy, they don't affect calculation.
+  valid_size = 16  # Random set of words to evaluate similarity on.
+  valid_window = 100  # Only pick dev samples in the head of the distribution.
+  valid_examples = np.random.choice(valid_window, valid_size, replace=False)
+
+  graph = tf.Graph()
+
+  with graph.as_default():
+
+    # Input data.
+    with tf.name_scope('inputs'):
+      train_inputs = tf.placeholder(tf.int32, shape=[batch_size])
+      train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
+      valid_dataset = tf.constant(valid_examples, dtype=tf.int32)
+
+    # Ops and variables pinned to the CPU because of missing GPU implementation
+    with tf.device('/cpu:0'):
+      # Look up embeddings for inputs.
+      with tf.name_scope('embeddings'):
+        embeddings = tf.Variable(
+            tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
+        embed = tf.nn.embedding_lookup(embeddings, train_inputs)
+
+      # Construct the variables for the NCE loss
+      with tf.name_scope('weights'):
+        nce_weights = tf.Variable(
+            tf.truncated_normal([vocabulary_size, embedding_size],
+                                stddev=1.0 / math.sqrt(embedding_size)))
+      with tf.name_scope('biases'):
+        nce_biases = tf.Variable(tf.zeros([vocabulary_size]))
+
+    # Compute the average NCE loss for the batch.
+    # tf.nce_loss automatically draws a new sample of the negative labels each
+    # time we evaluate the loss.
+    # Explanation of the meaning of NCE loss:
+    #   http://mccormickml.com/2016/04/19/word2vec-tutorial-the-skip-gram-model/
+    with tf.name_scope('loss'):
+      loss = tf.reduce_mean(
+          tf.nn.nce_loss(
+              weights=nce_weights,
+              biases=nce_biases,
+              labels=train_labels,
+              inputs=embed,
+              num_sampled=num_sampled,
+              num_classes=vocabulary_size))
+
+    # Add the loss value as a scalar to summary.
+    tf.summary.scalar('loss', loss)
+
+    # Construct the SGD optimizer using a learning rate of 1.0.
+    with tf.name_scope('optimizer'):
+      optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)
+
+    # Compute the cosine similarity between minibatch examples and all
+    # embeddings.
+    norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keepdims=True))
+    normalized_embeddings = embeddings / norm
+    valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings,
+                                              valid_dataset)
+    similarity = tf.matmul(
+        valid_embeddings, normalized_embeddings, transpose_b=True)
+
+    # Merge all summaries.
+    merged = tf.summary.merge_all()
+
+    # Add variable initializer.
+    init = tf.global_variables_initializer()
+
+    # Create a saver.
+    saver = tf.train.Saver()
+
+  # Step 5: Begin training.
+  num_steps = 100001
+
+  with tf.Session(graph=graph) as session:
+    # Open a writer to write summaries.
+    writer = tf.summary.FileWriter(log_dir, session.graph)
+
+    # We must initialize all variables before we use them.
+    init.run()
+    print('Initialized')
+
+    average_loss = 0
+    for step in xrange(num_steps):
+      batch_inputs, batch_labels = generate_batch(batch_size, num_skips,
+                                                  skip_window)
+      feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels}
+
+      # Define metadata variable.
+      run_metadata = tf.RunMetadata()
+
+      # We perform one update step by evaluating the optimizer op (including it
+      # in the list of returned values for session.run()
+      # Also, evaluate the merged op to get all summaries from the returned
+      # "summary" variable. Feed metadata variable to session for visualizing
+      # the graph in TensorBoard.
+      _, summary, loss_val = session.run([optimizer, merged, loss],
+                                         feed_dict=feed_dict,
+                                         run_metadata=run_metadata)
+      average_loss += loss_val
+
+      # Add returned summaries to writer in each step.
+      writer.add_summary(summary, step)
+      # Add metadata to visualize the graph for the last run.
+      if step == (num_steps - 1):
+        writer.add_run_metadata(run_metadata, 'step%d' % step)
+
+      if step % 2000 == 0:
+        if step > 0:
+          average_loss /= 2000
+        # The average loss is an estimate of the loss over the last 2000
+        # batches.
+        print('Average loss at step ', step, ': ', average_loss)
+        average_loss = 0
+
+      # Note that this is expensive (~20% slowdown if computed every 500 steps)
+      if step % 10000 == 0:
+        sim = similarity.eval()
+        for i in xrange(valid_size):
+          valid_word = reverse_dictionary[valid_examples[i]]
+          top_k = 8  # number of nearest neighbors
+          nearest = (-sim[i, :]).argsort()[1:top_k + 1]
+          log_str = 'Nearest to %s:' % valid_word
+          for k in xrange(top_k):
+            close_word = reverse_dictionary[nearest[k]]
+            log_str = '%s %s,' % (log_str, close_word)
+          print(log_str)
+    final_embeddings = normalized_embeddings.eval()
+
+    # Write corresponding labels for the embeddings.
+    with open(log_dir + '/metadata.tsv', 'w') as f:
+      for i in xrange(vocabulary_size):
+        f.write(reverse_dictionary[i] + '\n')
+
+    # Save the model for checkpoints.
+    saver.save(session, os.path.join(log_dir, 'model.ckpt'))
+
+    # Create a configuration for visualizing embeddings with the labels in
+    # TensorBoard.
+    config = projector.ProjectorConfig()
+    embedding_conf = config.embeddings.add()
+    embedding_conf.tensor_name = embeddings.name
+    embedding_conf.metadata_path = os.path.join(log_dir, 'metadata.tsv')
+    projector.visualize_embeddings(writer, config)
+
+  writer.close()
+
+  # Step 6: Visualize the embeddings.
+
+  # pylint: disable=missing-docstring
+  # Function to draw visualization of distance between embeddings.
+  def plot_with_labels(low_dim_embs, labels, filename):
+    assert low_dim_embs.shape[0] >= len(labels), 'More labels than embeddings'
+    plt.figure(figsize=(18, 18))  # in inches
+    for i, label in enumerate(labels):
+      x, y = low_dim_embs[i, :]
+      plt.scatter(x, y)
+      plt.annotate(
+          label,
+          xy=(x, y),
+          xytext=(5, 2),
+          textcoords='offset points',
+          ha='right',
+          va='bottom')
+
+    plt.savefig(filename)
+
+  try:
+    # pylint: disable=g-import-not-at-top
+    from sklearn.manifold import TSNE
+    import matplotlib.pyplot as plt
+
+    tsne = TSNE(
+        perplexity=30, n_components=2, init='pca', n_iter=5000, method='exact')
+    plot_only = 500
+    low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only, :])
+    labels = [reverse_dictionary[i] for i in xrange(plot_only)]
+    plot_with_labels(low_dim_embs, labels, os.path.join(gettempdir(),
+                                                        'tsne.png'))
+
+  except ImportError as ex:
+    print('Please install sklearn, matplotlib, and scipy to show embeddings.')
+    print(ex)
+
+
+# All functionality is run after tf.app.run() (b/122547914). This could be split
+# up but the methods are laid sequentially with their usage for clarity.
+def main(unused_argv):
+  # Give a folder path as an argument with '--log_dir' to save
+  # TensorBoard summaries. Default is a log folder in current directory.
+  current_path = os.path.dirname(os.path.realpath(sys.argv[0]))
+
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+      '--log_dir',
+      type=str,
+      default=os.path.join(current_path, 'log'),
+      help='The log directory for TensorBoard summaries.')
+  flags, unused_flags = parser.parse_known_args()
+  word2vec_basic(flags.log_dir)
+
+if __name__ == '__main__':
+  tf.app.run()
-- 
GitLab


From 47cf89738d3dd09dd4c0ff24df3c5c433c544d0d Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Thu, 17 Jan 2019 15:30:05 -0800
Subject: [PATCH 0914/2345] [TF:XLA] Extract max_num_elements from
 empty_tensor_list.

PiperOrigin-RevId: 229829562
---
 .../compiler/tests/tensor_list_ops_test.py    | 22 +++++++++-----
 .../tf2xla/kernels/tensor_list_ops.cc         | 29 ++++++++++++++++---
 2 files changed, 40 insertions(+), 11 deletions(-)

diff --git a/tensorflow/compiler/tests/tensor_list_ops_test.py b/tensorflow/compiler/tests/tensor_list_ops_test.py
index 3b48aed60c..1ecdb22cd0 100644
--- a/tensorflow/compiler/tests/tensor_list_ops_test.py
+++ b/tensorflow/compiler/tests/tensor_list_ops_test.py
@@ -93,18 +93,26 @@ class ListOpsTest(xla_test.XLATestCase):
       result = sess.run([e11, [e21, e22], [e31, e32]])
       self.assertEqual(result, [1.0, [2.0, 1.0], [3.0, 1.0]])
 
-  def testEmptyTensorList(self):
-    dim = 7
+  def testEmptyTensorListNoMax(self):
     with self.cached_session() as sess, self.test_scope():
-      p = array_ops.placeholder(dtypes.int32)
       l = list_ops.empty_tensor_list(
-          element_shape=(p, 15), element_dtype=dtypes.float32)
+          element_shape=(7, 15), element_dtype=dtypes.float32)
       l = list_ops.tensor_list_push_back(
-          l, constant_op.constant(1.0, shape=(dim, 15)))
+          l, constant_op.constant(1.0, shape=(7, 15)))
       _, e = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
       with self.assertRaisesRegexp(errors.InvalidArgumentError,
-                                   "Use TensorListReserve instead"):
-        self.assertEqual(sess.run(e, {p: dim}), 1.0 * np.ones((dim, 15)))
+                                   "Set the max number of elements"):
+        self.assertEqual(sess.run(e), 1.0 * np.ones((7, 15)))
+
+  def testEmptyTensorListMax(self):
+    with self.cached_session() as sess, self.test_scope():
+      l = list_ops.empty_tensor_list(
+          element_shape=(10, 15), element_dtype=dtypes.float32,
+          max_num_elements=2)
+      l = list_ops.tensor_list_push_back(
+          l, array_ops.fill(value=3.0, dims=(10, 15)))
+      _, e = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
+      self.assertAllEqual(sess.run(e), 3.0 * np.ones((10, 15)))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc b/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc
index ba3d7eb00d..5dda62fd49 100644
--- a/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc
@@ -85,19 +85,40 @@ REGISTER_XLA_OP(Name("TensorListReserve")
 
 class EmptyTensorListOp : public XlaOpKernel {
  public:
-  explicit EmptyTensorListOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {}
+  explicit EmptyTensorListOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("element_dtype", &dtype_));
+  }
 
   void Compile(XlaOpKernelContext* ctx) override {
-    ctx->CtxFailure(
+    TensorShape element_shape;
+    OP_REQUIRES_OK(ctx, ctx->ConstantInputAsShape(0, &element_shape));
+    int64 max_num_elements;
+    OP_REQUIRES_OK(ctx, ctx->ConstantInputAsIntScalar(1, &max_num_elements));
+    OP_REQUIRES(
+        ctx, max_num_elements >= 0,
         errors::InvalidArgument("XLA compilation requires a fixed tensor list "
-                                "size. Use TensorListReserve instead."));
+                                "size. Set the max number of elements."));
+
+    TensorShape tensor_shape;
+    tensor_shape.AddDim(max_num_elements);
+    tensor_shape.AppendShape(element_shape);
+
+    xla::XlaBuilder* b = ctx->builder();
+    ctx->SetOutput(0, xla::Tuple(b, {xla::Broadcast(XlaHelpers::Zero(b, dtype_),
+                                                    tensor_shape.dim_sizes()),
+                                     xla::ConstantR0<int32>(b, 0)}));
   }
 
  private:
+  DataType dtype_;
+
   TF_DISALLOW_COPY_AND_ASSIGN(EmptyTensorListOp);
 };
 
-REGISTER_XLA_OP(Name("EmptyTensorList"), EmptyTensorListOp);
+REGISTER_XLA_OP(Name("EmptyTensorList")
+                    .CompileTimeConstantInput("element_shape")
+                    .CompileTimeConstantInput("max_num_elements"),
+                EmptyTensorListOp);
 
 class TensorListElementShapeOp : public XlaOpKernel {
  public:
-- 
GitLab


From 307b1f368aa36173a799575e93737f150a4db9cd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 15:46:15 -0800
Subject: [PATCH 0915/2345] Introduce a Graph Verifier interface in Grappler.

This can be used to implement specific graph verifiers that can then be plugged into the overall grappler workflow or run standalone.
Some examples of specific verifiers are:
 1. A general structural verifier that verifies that the specified graph has
   a valid structure that meets the specification of what it means to be
      a valid TF graph.
 2. A backend specific verifier that verifies that the specified graph,
     generated after a grappler transformation to convert the input TF graph
     to a corresponding backend graph, is a valid graph in the specification
     of the backend.

PiperOrigin-RevId: 229832249
---
 tensorflow/core/grappler/verifiers/BUILD      | 13 +++++
 .../core/grappler/verifiers/graph_verifier.h  | 53 +++++++++++++++++++
 2 files changed, 66 insertions(+)
 create mode 100644 tensorflow/core/grappler/verifiers/BUILD
 create mode 100644 tensorflow/core/grappler/verifiers/graph_verifier.h

diff --git a/tensorflow/core/grappler/verifiers/BUILD b/tensorflow/core/grappler/verifiers/BUILD
new file mode 100644
index 0000000000..00b1325176
--- /dev/null
+++ b/tensorflow/core/grappler/verifiers/BUILD
@@ -0,0 +1,13 @@
+licenses(["notice"])  # Apache 2.0
+
+cc_library(
+    name = "graph_verifier",
+    hdrs = [
+        "graph_verifier.h",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+    ],
+)
diff --git a/tensorflow/core/grappler/verifiers/graph_verifier.h b/tensorflow/core/grappler/verifiers/graph_verifier.h
new file mode 100644
index 0000000000..f5acd59266
--- /dev/null
+++ b/tensorflow/core/grappler/verifiers/graph_verifier.h
@@ -0,0 +1,53 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_GRAPPLER_VERIFIERS_GRAPH_VERIFIER_H_
+#define TENSORFLOW_CORE_GRAPPLER_VERIFIERS_GRAPH_VERIFIER_H_
+
+#include <string>
+#include <vector>
+#include "tensorflow/core/framework/graph.pb.h"
+#include "tensorflow/core/lib/core/status.h"
+
+namespace tensorflow {
+namespace grappler {
+
+// An abstract interface for verifying a graph.
+// This will be used to implement specific verifiers to verify that a grappler
+// transformed graph is valid.
+// Some examples of specific verifiers are:
+// 1. A general structural verifier that verifies that the specified graph has
+//    a valid structure that meets the specification of what it means to be
+//      a valid TensorFlow graph.
+// 2. A backend specific verifier that verifies that the specified graph,
+//     generated after a grappler transformation to convert the input TensorFlow
+//     graph to a corresponding backend graph, is a valid graph in the
+//     specification of the backend.
+class GraphVerifier {
+ public:
+  GraphVerifier() {}
+  virtual ~GraphVerifier() {}
+
+  // A name for the verifier.
+  virtual string name() const = 0;
+
+  // Implement an algorithm to verify the specified graph.
+  virtual Status Verify(const GraphDef& graph, std::vector<string>* errors) = 0;
+};
+
+}  // end namespace grappler
+}  // end namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_GRAPPLER_VERIFIERS_GRAPH_VERIFIER_H_
-- 
GitLab


From a108131f4ef24cf6e7eecfd22f98f20fd1b6e390 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Thu, 17 Jan 2019 15:47:02 -0800
Subject: [PATCH 0916/2345] Cleanup training_distributed unnecessary scopes
 since we're already in the strategy scope in the entire function

PiperOrigin-RevId: 229832356
---
 .../keras/engine/training_distributed.py      | 65 ++++++++-----------
 1 file changed, 28 insertions(+), 37 deletions(-)

diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 8b7e5bb5cc..9b11c721d4 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -314,10 +314,9 @@ def experimental_tpu_fit_loop(model,
       dtype='int32',
       name='steps_per_run')
 
-  with current_strategy.scope():
-    ctx = current_strategy.extended.experimental_run_steps_on_iterator(
-        step_fn, iterator, iterations=steps_per_run,
-        initial_loop_values=initial_loop_values)
+  ctx = current_strategy.extended.experimental_run_steps_on_iterator(
+      step_fn, iterator, iterations=steps_per_run,
+      initial_loop_values=initial_loop_values)
 
   train_op = ctx.run_op
   output_tensors = ctx.last_step_outputs
@@ -325,9 +324,8 @@ def experimental_tpu_fit_loop(model,
   do_validation = bool(validation_steps)
 
   if model._compile_distribution:
-    with current_strategy.scope():
-      distributed_training_utils._copy_weights_to_distributed_model(
-          model, model._distributed_model_train)
+    distributed_training_utils._copy_weights_to_distributed_model(
+        model, model._distributed_model_train)
 
   callbacks = cbks.configure_callbacks(
       callbacks,
@@ -346,9 +344,8 @@ def experimental_tpu_fit_loop(model,
 
   callbacks.on_train_begin()
   for epoch in range(initial_epoch, epochs):
-    with current_strategy.scope():
-      distributed_training_utils._reset_metrics(
-          model, model._distributed_model_train)
+    distributed_training_utils._reset_metrics(
+        model, model._distributed_model_train)
     callbacks.on_epoch_begin(epoch)
     epoch_logs = {}
     step_index = 0
@@ -382,9 +379,8 @@ def experimental_tpu_fit_loop(model,
       if model._compile_distribution:
         # Since we create a new clone from the original model we need to copy
         # the weights back to the original model before we can run validation.
-        with current_strategy.scope():
-          distributed_training_utils._copy_weights_to_original_model(
-              model, model._distributed_model_train, ModeKeys.TRAIN)
+        distributed_training_utils._copy_weights_to_original_model(
+            model, model._distributed_model_train, ModeKeys.TRAIN)
 
       val_outs = experimental_tpu_test_loop(  # pylint: disable=undefined-variable
           model,
@@ -404,9 +400,8 @@ def experimental_tpu_fit_loop(model,
 
   if model._compile_distribution:
     # Copy the weights back from the replicated model to the original model.
-    with current_strategy.scope():
-      distributed_training_utils._copy_weights_to_original_model(
-          model, model._distributed_model_train, ModeKeys.TRAIN)
+    distributed_training_utils._copy_weights_to_original_model(
+        model, model._distributed_model_train, ModeKeys.TRAIN)
   scope.__exit__(None, None, None)
   return model.history
 
@@ -491,12 +486,11 @@ def experimental_tpu_test_loop(model,
     tensor = model._all_stateful_metrics_tensors[name]
     initial_loop_values[name] = array_ops.zeros(tensor.shape, tensor.dtype)
 
-  with current_strategy.scope():
-    # TODO(priyag): Use steps_per_run when we use new metrics as they will
-    # allow handling metric computation at each step using variables.
-    ctx = current_strategy.extended.experimental_run_steps_on_iterator(
-        step_fn, iterator, iterations=1,
-        initial_loop_values=initial_loop_values)
+  # TODO(priyag): Use steps_per_run when we use new metrics as they will
+  # allow handling metric computation at each step using variables.
+  ctx = current_strategy.extended.experimental_run_steps_on_iterator(
+      step_fn, iterator, iterations=1,
+      initial_loop_values=initial_loop_values)
 
   test_op = ctx.run_op
   output_tensors = ctx.last_step_outputs
@@ -505,13 +499,12 @@ def experimental_tpu_test_loop(model,
     progbar = Progbar(target=steps)
 
   if model._compile_distribution:
-    with current_strategy.scope():
-      distributed_training_utils._copy_weights_to_distributed_model(
-          model, model._distributed_model_test)
-  with current_strategy.scope():
-    distributed_training_utils._reset_metrics(
+    distributed_training_utils._copy_weights_to_distributed_model(
         model, model._distributed_model_test)
 
+  distributed_training_utils._reset_metrics(
+      model, model._distributed_model_test)
+
   assert steps is not None
   outs = [0.] * len(model.metrics_names)
   for step in range(steps):
@@ -609,11 +602,10 @@ def experimental_tpu_predict_loop(model, dataset, verbose=0, steps=None):
     shape.dims = [batch_dimension] + shape.dims[1:]
     initial_loop_values[name] = array_ops.zeros(shape, tensor.dtype)
 
-  with current_strategy.scope():
-    # TODO(priyag, sourabhbajaj): Support steps_per_run if/when we add outfeed.
-    ctx = current_strategy.extended.experimental_run_steps_on_iterator(
-        step_fn, iterator, iterations=1,
-        initial_loop_values=initial_loop_values)
+  # TODO(priyag, sourabhbajaj): Support steps_per_run if/when we add outfeed.
+  ctx = current_strategy.extended.experimental_run_steps_on_iterator(
+      step_fn, iterator, iterations=1,
+      initial_loop_values=initial_loop_values)
 
   predict_op = ctx.run_op
   output_tensors = ctx.last_step_outputs
@@ -622,13 +614,12 @@ def experimental_tpu_predict_loop(model, dataset, verbose=0, steps=None):
     progbar = Progbar(target=steps)
 
   if model._compile_distribution:
-    with current_strategy.scope():
-      distributed_training_utils._copy_weights_to_distributed_model(
-          model, model._distributed_model_predict)
-  with current_strategy.scope():
-    distributed_training_utils._reset_metrics(
+    distributed_training_utils._copy_weights_to_distributed_model(
         model, model._distributed_model_predict)
 
+  distributed_training_utils._reset_metrics(
+      model, model._distributed_model_predict)
+
   assert steps is not None
   # Since we do not know how many samples we will see, we cannot pre-allocate
   # the returned Numpy arrays. Instead, we store one array per batch seen
-- 
GitLab


From 04e8de0b1753ba7c229105d10113ac05fe5aa9d1 Mon Sep 17 00:00:00 2001
From: Blake Hechtman <blakehechtman@google.com>
Date: Thu, 17 Jan 2019 15:49:07 -0800
Subject: [PATCH 0917/2345] [TF] Make behavior of dropout backward
 compatability.

PiperOrigin-RevId: 229832672
---
 tensorflow/python/keras/layers/wrappers_test.py |  4 ++--
 tensorflow/python/ops/nn_ops.py                 | 10 ++++++++--
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/keras/layers/wrappers_test.py b/tensorflow/python/keras/layers/wrappers_test.py
index 313a2193eb..f3aa5c4d68 100644
--- a/tensorflow/python/keras/layers/wrappers_test.py
+++ b/tensorflow/python/keras/layers/wrappers_test.py
@@ -158,8 +158,8 @@ class TimeDistributedTest(test.TestCase):
       # test layers that need learning_phase to be set
       np.random.seed(1234)
       x = keras.layers.Input(shape=(3, 2))
-      y = keras.layers.TimeDistributed(
-          keras.layers.Dropout(.9999))(x, training=True)
+      y = keras.layers.TimeDistributed(keras.layers.Dropout(.999))(
+          x, training=True)
       model = keras.models.Model(x, y)
       y = model.predict(np.random.random((10, 3, 2)))
       self.assertAllClose(np.mean(y), 0., atol=1e-1, rtol=1e-1)
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 25609f9efa..e0ef9e5e34 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -3294,11 +3294,17 @@ def dropout_v2(x, rate, noise_shape=None, seed=None, name=None):  # pylint: disa
     noise_shape = _get_noise_shape(x, noise_shape)
     # Sample a uniform distribution on [0.0, 1.0) and select values larger than
     # rate.
+    #
+    # NOTE: Random uniform actually can only generate 2^23 floats on [1.0, 2.0)
+    # and subtract 1.0.
     random_tensor = random_ops.random_uniform(
         noise_shape, seed=seed, dtype=x.dtype)
     keep_prob = 1 - rate
-    ret = (1 / keep_prob) * math_ops.cast(keep_prob >= random_tensor,
-                                          x.dtype) * x
+    scale = 1 / keep_prob
+    # NOTE: if (1.0 + rate) - 1 is equal to rate, then we want to consider that
+    # float to be selected, hence we use a >= comparison.
+    keep_mask = random_tensor >= rate
+    ret = x * scale * math_ops.cast(keep_mask, x.dtype)
     if not context.executing_eagerly():
       ret.set_shape(x.get_shape())
     return ret
-- 
GitLab


From f8c19045a1ed9e54ee02e8690b5c1a185023c9db Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 15:54:13 -0800
Subject: [PATCH 0918/2345] Improve how the library to link is determined.

PiperOrigin-RevId: 229833513
---
 tensorflow/contrib/tensorrt/BUILD             |  26 +--
 .../tensorrt/custom_plugin_examples/BUILD     |   4 +-
 third_party/gpus/cuda_configure.bzl           | 154 ++++++++++--------
 third_party/nccl/nccl_configure.bzl           |   1 +
 third_party/tensorrt/BUILD.tpl                |   7 +-
 third_party/tensorrt/tensorrt_configure.bzl   |  66 +++-----
 .../preconfig/ubuntu14.04/tensorrt5/BUILD     |   2 +-
 7 files changed, 128 insertions(+), 132 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 80d79164a2..3f10906768 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -49,7 +49,7 @@ tf_cuda_cc_test(
         "//tensorflow/core:test_main",
     ] + if_tensorrt([
         "@local_config_cuda//cuda:cuda_headers",
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]),
 )
 
@@ -63,7 +63,7 @@ tf_custom_op_library(
         ":trt_shape_function",
         "//tensorflow/core:lib_proto_parsing",
     ] + if_tensorrt([
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]),
 )
 
@@ -76,7 +76,7 @@ tf_cuda_library(
         ":trt_logging",
         ":trt_plugins",
     ] + if_tensorrt([
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]) + tf_custom_op_library_additional_deps(),
 )
 
@@ -105,7 +105,7 @@ cc_library(
         "//tensorflow/core:stream_executor_headers_lib",
         "//tensorflow/core/grappler/costs:graph_properties",
     ] + if_tensorrt([
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]) + tf_custom_op_library_additional_deps(),
     # TODO(laigd): fix this by merging header file in cc file.
     alwayslink = 1,  # buildozer: disable=alwayslink-with-hdrs
@@ -149,7 +149,7 @@ tf_cuda_library(
     deps = [
         "//tensorflow/core:lib_proto_parsing",
     ] + if_tensorrt([
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]),
 )
 
@@ -169,7 +169,7 @@ tf_custom_op_py_library(
     dso = [
         ":python/ops/_trt_ops.so",
     ] + if_tensorrt([
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]),
     kernels = [
         ":trt_op_kernels",
@@ -262,7 +262,7 @@ tf_cuda_library(
         "//tensorflow/core:framework_lite",
         "//tensorflow/core:lib_proto_parsing",
     ] + if_tensorrt([
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]),
 )
 
@@ -275,7 +275,7 @@ tf_cuda_library(
         "//tensorflow/core:framework_lite",
         "//tensorflow/core:lib_proto_parsing",
     ] + if_tensorrt([
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]),
 )
 
@@ -347,7 +347,7 @@ tf_cuda_library(
         "//tensorflow/core/grappler/costs:graph_properties",
         "//tensorflow/core/grappler/optimizers:meta_optimizer",
     ] + if_tensorrt([
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]) + tf_custom_op_library_additional_deps(),
 )
 
@@ -377,7 +377,7 @@ tf_cuda_cc_test(
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
     ] + if_tensorrt([
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]),
 )
 
@@ -409,7 +409,7 @@ tf_cuda_cc_test(
         "//tensorflow/core:testlib",
     ] + if_tensorrt([
         "@local_config_cuda//cuda:cuda_headers",
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]),
 )
 
@@ -468,7 +468,7 @@ tf_cuda_library(
         "//tensorflow/core:framework_lite",
         "//tensorflow/core:lib_proto_parsing",
     ] + if_tensorrt([
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]),
 )
 
@@ -488,7 +488,7 @@ tf_cuda_cc_test(
         "//tensorflow/core:test_main",
     ] + if_tensorrt([
         "@local_config_cuda//cuda:cuda_headers",
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]),
 )
 
diff --git a/tensorflow/contrib/tensorrt/custom_plugin_examples/BUILD b/tensorflow/contrib/tensorrt/custom_plugin_examples/BUILD
index 69058c5826..09990fb5e3 100644
--- a/tensorflow/contrib/tensorrt/custom_plugin_examples/BUILD
+++ b/tensorflow/contrib/tensorrt/custom_plugin_examples/BUILD
@@ -48,7 +48,7 @@ tf_custom_op_library(
         "//tensorflow/contrib/tensorrt:trt_plugins",
         "//tensorflow/core:framework_lite",
     ] + if_tensorrt([
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]),
 )
 
@@ -67,7 +67,7 @@ tf_kernel_library(
         "//tensorflow/contrib/tensorrt:trt_plugins",
         "//tensorflow/core:stream_executor_headers_lib",
     ] + if_tensorrt([
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]) + tf_custom_op_library_additional_deps(),
 )
 
diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl
index beacf2f45a..7a6c7f4fe5 100644
--- a/third_party/gpus/cuda_configure.bzl
+++ b/third_party/gpus/cuda_configure.bzl
@@ -643,9 +643,7 @@ def _cudnn_version(repository_ctx, cudnn_install_basedir, cpu_value):
     auto_configure_fail(("cuDNN version detected from %s (%s) does not match " +
                          "TF_CUDNN_VERSION (%s)") %
                         (str(cudnn_h_path), full_version, environ_version),)
-
-  # We only use the major version since we use the libcudnn libraries that are
-  # only versioned with the major version (e.g. libcudnn.so.5).
+  # Only use the major version to match the SONAME of the library.
   version = major_version
   if cpu_value == "Windows":
     version = "64_" + version
@@ -691,11 +689,11 @@ def _is_windows(repository_ctx):
   return get_cpu_value(repository_ctx) == "Windows"
 
 
-def _lib_name(lib, cpu_value, version = "", static = False):
+def lib_name(base_name, cpu_value, version = None, static = False):
   """Constructs the platform-specific name of a library.
 
     Args:
-      lib: The name of the library, such as "cudart"
+      base_name: The name of the library, such as "cudart"
       cpu_value: The name of the host operating system.
       version: The version of the library.
       static: True the library is static or False if it is a shared object.
@@ -703,31 +701,55 @@ def _lib_name(lib, cpu_value, version = "", static = False):
     Returns:
       The platform-specific name of the library.
     """
+  version = "" if not version else "." + version
   if cpu_value in ("Linux", "FreeBSD"):
     if static:
-      return "lib%s.a" % lib
-    else:
-      if version:
-        version = ".%s" % version
-      return "lib%s.so%s" % (lib, version)
+      return "lib%s.a" % base_name
+    return "lib%s.so%s" % (base_name, version)
   elif cpu_value == "Windows":
-    return "%s.lib" % lib
+    return "%s.lib" % base_name
   elif cpu_value == "Darwin":
     if static:
-      return "lib%s.a" % lib
-    elif version:
-      version = ".%s" % version
-    return "lib%s%s.dylib" % (lib, version)
+      return "lib%s.a" % base_name
+    return "lib%s%s.dylib" % (base_name, version)
   else:
     auto_configure_fail("Invalid cpu_value: %s" % cpu_value)
 
+def find_lib(repository_ctx, paths, check_soname = True):
+  """
+    Finds a library among a list of potential paths.
+
+    Args:
+      paths: List of paths to inspect.
+
+    Returns:
+      Returns the first path in paths that exist.
+  """
+  objdump = repository_ctx.which("objdump")
+  mismatches = []
+  for path in [repository_ctx.path(path) for path in paths]:
+    if not path.exists:
+      continue
+    if check_soname and objdump != None:
+      output = repository_ctx.execute([objdump, "-p", str(path)]).stdout
+      output = [line for line in output.splitlines() if "SONAME" in line]
+      sonames = [line.strip().split(" ")[-1] for line in output]
+      if not any([soname == path.basename for soname in sonames]):
+        mismatches.append(str(path))
+        continue
+    return path
+  if mismatches:
+    auto_configure_fail(
+        "None of the libraries match their SONAME: " + ", ".join(mismatches))
+  auto_configure_fail("No library found under: " + ", ".join(paths))
+
 
 def _find_cuda_lib(
         lib,
         repository_ctx,
         cpu_value,
         basedir,
-        version = "",
+        version,
         static = False):
   """Finds the given CUDA or cuDNN library on the system.
 
@@ -740,16 +762,12 @@ def _find_cuda_lib(
       static: True if static library, False if shared object.
 
     Returns:
-      Returns a struct with the following fields:
-        file_name: The basename of the library found on the system.
-        path: The full path to the library.
+      Returns the path to the library.
     """
-  file_name = _lib_name(lib, cpu_value, version, static)
-  for relative_path in CUDA_LIB_PATHS:
-    path = repository_ctx.path("%s/%s%s" % (basedir, relative_path, file_name))
-    if path.exists:
-      return struct(file_name=file_name, path=str(path.realpath))
-  auto_configure_fail("Cannot find cuda library %s" % file_name)
+  file_name = lib_name(lib, cpu_value, version, static)
+  return find_lib(repository_ctx, [
+      "%s/%s%s" % (basedir, path, file_name) for path in CUDA_LIB_PATHS
+  ], check_soname = version and not static)
 
 
 def _find_cupti_header_dir(repository_ctx, cuda_config):
@@ -785,23 +803,17 @@ def _find_cupti_lib(repository_ctx, cuda_config):
       cuda_config: The cuda configuration as returned by _get_cuda_config.
 
     Returns:
-      Returns a struct with the following fields:
-        file_name: The basename of the library found on the system.
-        path: The full path to the library.
+      Returns the path to the library.
     """
-  file_name = _lib_name(
+  file_name = lib_name(
       "cupti",
       cuda_config.cpu_value,
       cuda_config.cuda_version,
   )
-  cuda_toolkit_path = cuda_config.cuda_toolkit_path
-  for relative_path in CUPTI_LIB_PATHS:
-    path = repository_ctx.path(
-        "%s/%s%s" % (cuda_toolkit_path, relative_path, file_name),)
-    if path.exists:
-      return struct(file_name=file_name, path=str(path.realpath))
-
-  auto_configure_fail("Cannot find cupti library %s" % file_name)
+  basedir = cuda_config.cuda_toolkit_path
+  return find_lib(repository_ctx, [
+      "%s/%s%s" % (basedir, path, file_name) for path in CUPTI_LIB_PATHS
+  ])
 
 
 def _find_libs(repository_ctx, cuda_config):
@@ -817,8 +829,12 @@ def _find_libs(repository_ctx, cuda_config):
   cpu_value = cuda_config.cpu_value
   return {
       "cuda":
-          _find_cuda_lib("cuda", repository_ctx, cpu_value,
-                         cuda_config.cuda_toolkit_path),
+          _find_cuda_lib(
+              "cuda",
+              repository_ctx,
+              cpu_value,
+              cuda_config.cuda_toolkit_path,
+              None),
       "cudart":
           _find_cuda_lib(
               "cudart",
@@ -1035,9 +1051,9 @@ def _create_dummy_repository(repository_ctx):
       "cuda:BUILD",
       {
           "%{cuda_driver_lib}":
-              _lib_name("cuda", cpu_value),
+              lib_name("cuda", cpu_value),
           "%{cudart_static_lib}":
-              _lib_name(
+              lib_name(
                   "cudart_static",
                   cpu_value,
                   static=True,
@@ -1045,19 +1061,19 @@ def _create_dummy_repository(repository_ctx):
           "%{cudart_static_linkopt}":
               _cudart_static_linkopt(cpu_value),
           "%{cudart_lib}":
-              _lib_name("cudart", cpu_value),
+              lib_name("cudart", cpu_value),
           "%{cublas_lib}":
-              _lib_name("cublas", cpu_value),
+              lib_name("cublas", cpu_value),
           "%{cusolver_lib}":
-              _lib_name("cusolver", cpu_value),
+              lib_name("cusolver", cpu_value),
           "%{cudnn_lib}":
-              _lib_name("cudnn", cpu_value),
+              lib_name("cudnn", cpu_value),
           "%{cufft_lib}":
-              _lib_name("cufft", cpu_value),
+              lib_name("cufft", cpu_value),
           "%{curand_lib}":
-              _lib_name("curand", cpu_value),
+              lib_name("curand", cpu_value),
           "%{cupti_lib}":
-              _lib_name("cupti", cpu_value),
+              lib_name("cupti", cpu_value),
           "%{copy_rules}":
               "",
           "%{cuda_headers}":
@@ -1071,16 +1087,16 @@ def _create_dummy_repository(repository_ctx):
   repository_ctx.file("cuda/cuda/include/cublas.h")
   repository_ctx.file("cuda/cuda/include/cudnn.h")
   repository_ctx.file("cuda/cuda/extras/CUPTI/include/cupti.h")
-  repository_ctx.file("cuda/cuda/lib/%s" % _lib_name("cuda", cpu_value))
-  repository_ctx.file("cuda/cuda/lib/%s" % _lib_name("cudart", cpu_value))
+  repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cuda", cpu_value))
+  repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cudart", cpu_value))
   repository_ctx.file(
-      "cuda/cuda/lib/%s" % _lib_name("cudart_static", cpu_value))
-  repository_ctx.file("cuda/cuda/lib/%s" % _lib_name("cublas", cpu_value))
-  repository_ctx.file("cuda/cuda/lib/%s" % _lib_name("cusolver", cpu_value))
-  repository_ctx.file("cuda/cuda/lib/%s" % _lib_name("cudnn", cpu_value))
-  repository_ctx.file("cuda/cuda/lib/%s" % _lib_name("curand", cpu_value))
-  repository_ctx.file("cuda/cuda/lib/%s" % _lib_name("cufft", cpu_value))
-  repository_ctx.file("cuda/cuda/lib/%s" % _lib_name("cupti", cpu_value))
+      "cuda/cuda/lib/%s" % lib_name("cudart_static", cpu_value))
+  repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cublas", cpu_value))
+  repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cusolver", cpu_value))
+  repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cudnn", cpu_value))
+  repository_ctx.file("cuda/cuda/lib/%s" % lib_name("curand", cpu_value))
+  repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cufft", cpu_value))
+  repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cupti", cpu_value))
 
   # Set up cuda_config.h, which is used by
   # tensorflow/stream_executor/dso_loader.cc.
@@ -1269,9 +1285,9 @@ def _create_local_cuda_repository(repository_ctx):
   cuda_libs = _find_libs(repository_ctx, cuda_config)
   cuda_lib_srcs = []
   cuda_lib_outs = []
-  for lib in cuda_libs.values():
-    cuda_lib_srcs.append(lib.path)
-    cuda_lib_outs.append("cuda/lib/" + lib.file_name)
+  for path in cuda_libs.values():
+    cuda_lib_srcs.append(str(path))
+    cuda_lib_outs.append("cuda/lib/" + path.basename)
   copy_rules.append(make_copy_files_rule(
       repository_ctx,
       name = "cuda-lib",
@@ -1317,25 +1333,25 @@ def _create_local_cuda_repository(repository_ctx):
       "cuda:BUILD.windows" if _is_windows(repository_ctx) else "cuda:BUILD",
       {
           "%{cuda_driver_lib}":
-              cuda_libs["cuda"].file_name,
+              cuda_libs["cuda"].basename,
           "%{cudart_static_lib}":
-              cuda_libs["cudart_static"].file_name,
+              cuda_libs["cudart_static"].basename,
           "%{cudart_static_linkopt}":
               _cudart_static_linkopt(cuda_config.cpu_value,),
           "%{cudart_lib}":
-              cuda_libs["cudart"].file_name,
+              cuda_libs["cudart"].basename,
           "%{cublas_lib}":
-              cuda_libs["cublas"].file_name,
+              cuda_libs["cublas"].basename,
           "%{cusolver_lib}":
-              cuda_libs["cusolver"].file_name,
+              cuda_libs["cusolver"].basename,
           "%{cudnn_lib}":
-              cuda_libs["cudnn"].file_name,
+              cuda_libs["cudnn"].basename,
           "%{cufft_lib}":
-              cuda_libs["cufft"].file_name,
+              cuda_libs["cufft"].basename,
           "%{curand_lib}":
-              cuda_libs["curand"].file_name,
+              cuda_libs["curand"].basename,
           "%{cupti_lib}":
-              cuda_libs["cupti"].file_name,
+              cuda_libs["cupti"].basename,
           "%{copy_rules}":
               "\n".join(copy_rules),
           "%{cuda_headers}": ('":cuda-include",\n' + '        ":cudnn-include",'
diff --git a/third_party/nccl/nccl_configure.bzl b/third_party/nccl/nccl_configure.bzl
index f7d7d55383..59b5bd3fea 100644
--- a/third_party/nccl/nccl_configure.bzl
+++ b/third_party/nccl/nccl_configure.bzl
@@ -134,6 +134,7 @@ def _nccl_configure_impl(repository_ctx):
     nccl_version = ""
     if _TF_NCCL_VERSION in repository_ctx.os.environ:
         nccl_version = repository_ctx.os.environ[_TF_NCCL_VERSION].strip()
+        nccl_version = nccl_version.split(".")[0]
 
     if nccl_version == "":
         # Alias to open source build from @nccl_archive.
diff --git a/third_party/tensorrt/BUILD.tpl b/third_party/tensorrt/BUILD.tpl
index 17ab2fa119..a41ab808c7 100644
--- a/third_party/tensorrt/BUILD.tpl
+++ b/third_party/tensorrt/BUILD.tpl
@@ -16,9 +16,9 @@ cc_library(
 )
 
 cc_library(
-    name = "nv_infer",
-    srcs = [%{nv_infer}],
-    data = [%{nv_infer}],
+    name = "tensorrt",
+    srcs = %{tensorrt_libs},
+    data = %{tensorrt_libs},
     copts= cuda_default_copts(),
     deps = [
         "@local_config_cuda//cuda:cuda",
@@ -28,6 +28,5 @@ cc_library(
     visibility = ["//visibility:public"],
 )
 
-
 %{copy_rules}
 
diff --git a/third_party/tensorrt/tensorrt_configure.bzl b/third_party/tensorrt/tensorrt_configure.bzl
index 7cf845d5cc..10e243fa84 100644
--- a/third_party/tensorrt/tensorrt_configure.bzl
+++ b/third_party/tensorrt/tensorrt_configure.bzl
@@ -12,6 +12,8 @@ load(
     "auto_configure_fail",
     "get_cpu_value",
     "find_cuda_define",
+    "find_lib",
+    "lib_name",
     "matches_version",
     "make_copy_dir_rule",
     "make_copy_files_rule",
@@ -94,10 +96,11 @@ def _trt_lib_version(repository_ctx, trt_install_path):
         ("TensorRT library version detected from %s/%s (%s) does not match " +
          "TF_TENSORRT_VERSION (%s). To fix this rerun configure again.") %
         (trt_header_dir, "NvInfer.h", full_version, environ_version))
-  return environ_version
+  # Only use the major version to match the SONAME of the library.
+  return major_version
 
 
-def _find_trt_libs(repository_ctx, trt_install_path, trt_lib_version):
+def _find_trt_libs(repository_ctx, cpu_value, trt_install_path, trt_lib_version):
   """Finds the given TensorRT library on the system.
 
   Adapted from code contributed by Sami Kama (https://github.com/samikama).
@@ -109,30 +112,13 @@ def _find_trt_libs(repository_ctx, trt_install_path, trt_lib_version):
       by _trt_lib_version.
 
   Returns:
-    Map of library names to structs with the following fields:
-      src_file_path: The full path to the library found on the system.
-      dst_file_name: The basename of the target library.
+    The path to the library.
   """
-  objdump = repository_ctx.which("objdump")
   result = {}
   for lib in _TF_TENSORRT_LIBS:
-    dst_file_name = "lib%s.so.%s" % (lib, trt_lib_version)
-    src_file_path = repository_ctx.path("%s/%s" % (trt_install_path,
-                                                   dst_file_name))
-    if not src_file_path.exists:
-      auto_configure_fail(
-          "Cannot find TensorRT library %s" % str(src_file_path))
-    if objdump != None:
-      objdump_out = repository_ctx.execute([objdump, "-p", str(src_file_path)])
-      for line in objdump_out.stdout.splitlines():
-        if "SONAME" in line:
-          dst_file_name = line.strip().split(" ")[-1]
-    result.update({
-        lib:
-            struct(
-                dst_file_name=dst_file_name,
-                src_file_path=str(src_file_path.realpath))
-    })
+    file_name = lib_name("nvinfer", cpu_value, trt_lib_version)
+    path = find_lib(repository_ctx, ["%s/%s" % (trt_install_path, file_name)])
+    result[file_name] = path
   return result
 
 
@@ -144,15 +130,12 @@ def _tpl(repository_ctx, tpl, substitutions):
 def _create_dummy_repository(repository_ctx):
   """Create a dummy TensorRT repository."""
   _tpl(repository_ctx, "build_defs.bzl", {"%{if_tensorrt}": "if_false"})
-  substitutions = {
-      "%{tensorrt_genrules}": "",
-      "%{tensorrt_headers}": "",
-  }
-  for lib in _TF_TENSORRT_LIBS:
-    k = "%%{%s}" % lib.replace("nv", "nv_")
-    substitutions.update({k: ""})
-  _tpl(repository_ctx, "BUILD", substitutions)
 
+  _tpl(repository_ctx, "BUILD", {
+      "%{tensorrt_genrules}": "",
+      "%{tensorrt_headers}": "[]",
+      "%{tensorrt_libs}": "[]"
+  })
 
 def _tensorrt_configure_impl(repository_ctx):
   """Implementation of the tensorrt_configure repository rule."""
@@ -168,7 +151,8 @@ def _tensorrt_configure_impl(repository_ctx):
     _create_dummy_repository(repository_ctx)
     return
 
-  if (get_cpu_value(repository_ctx) != "Linux"):
+  cpu_value = get_cpu_value(repository_ctx)
+  if (cpu_value != "Linux"):
     auto_configure_fail("TensorRT is supported only on Linux.")
   if _TF_TENSORRT_VERSION not in repository_ctx.os.environ:
     auto_configure_fail("TensorRT library (libnvinfer) version is not set.")
@@ -179,12 +163,12 @@ def _tensorrt_configure_impl(repository_ctx):
 
   # Copy the library files.
   trt_lib_version = _trt_lib_version(repository_ctx, trt_install_path)
-  trt_libs = _find_trt_libs(repository_ctx, trt_install_path, trt_lib_version)
+  trt_libs = _find_trt_libs(repository_ctx, cpu_value, trt_install_path, trt_lib_version)
   trt_lib_srcs = []
   trt_lib_outs = []
-  for lib in trt_libs.values():
-    trt_lib_srcs.append(lib.src_file_path)
-    trt_lib_outs.append("tensorrt/lib/" + lib.dst_file_name)
+  for path in trt_libs.values():
+    trt_lib_srcs.append(str(path))
+    trt_lib_outs.append("tensorrt/lib/" + path.basename)
   copy_rules = [make_copy_files_rule(
       repository_ctx,
       name = "tensorrt_lib",
@@ -212,15 +196,11 @@ def _tensorrt_configure_impl(repository_ctx):
   _tpl(repository_ctx, "build_defs.bzl", {"%{if_tensorrt}": "if_true"})
 
   # Set up BUILD file.
-  substitutions = {
+  _tpl(repository_ctx, "BUILD", {
       "%{copy_rules}": "\n".join(copy_rules),
       "%{tensorrt_headers}": '":tensorrt_include"',
-  }
-  for lib in _TF_TENSORRT_LIBS:
-    k = "%%{%s}" % lib.replace("nv", "nv_")
-    v = '"tensorrt/lib/%s"' % trt_libs[lib].dst_file_name
-    substitutions.update({k: v})
-  _tpl(repository_ctx, "BUILD", substitutions)
+      "%{tensorrt_libs}": str(trt_lib_outs),
+  })
 
 
 tensorrt_configure = repository_rule(
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/BUILD b/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/BUILD
index 941d99a0db..04442d9c51 100755
--- a/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/BUILD
+++ b/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/BUILD
@@ -17,7 +17,7 @@ cc_library(
 )
 
 cc_library(
-    name = "nv_infer",
+    name = "tensorrt",
     srcs = ["tensorrt/lib/libnvinfer.so.5"],
     copts = cuda_default_copts(),
     data = ["tensorrt/lib/libnvinfer.so.5"],
-- 
GitLab


From 3dedb64c95cb1fed7ce6f521e86c775a098b0525 Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Thu, 17 Jan 2019 16:00:25 -0800
Subject: [PATCH 0919/2345] Update Micro documentation

PiperOrigin-RevId: 229834532
---
 tensorflow/lite/experimental/micro/README.md | 131 +++++++++++++++++++
 1 file changed, 131 insertions(+)

diff --git a/tensorflow/lite/experimental/micro/README.md b/tensorflow/lite/experimental/micro/README.md
index 4c3eab38fa..b47e0feb79 100644
--- a/tensorflow/lite/experimental/micro/README.md
+++ b/tensorflow/lite/experimental/micro/README.md
@@ -509,3 +509,134 @@ Another  badly-formed  format string
 
 If not, you'll need to debug what went wrong, but hopefully with this small
 starting project it should be manageable.
+
+### Troubleshooting
+
+When we've been porting to new platforms, it's often been hard to figure out
+some of the fundamentals like linker settings and other toolchain setup flags.
+If you are having trouble, see if you can find a simple example program for your
+platform, like one that just blinks an LED. If you're able to build and run that
+successfully, then start to swap in parts of the TF Lite Micro codebase to that
+working project, taking it a step at a time and ensuring it's still working
+after every change. For example, a first step might be to paste in your
+`DebugLog()` implementation and call `DebugLog("Hello World!")` from the main
+function.
+
+Another common problem on embedded platforms is the stack size being too small.
+Mbed defaults to 4KB for the main thread's stack, which is too small for most
+models since TensorFlow Lite allocates buffers and other data structures that
+require more memory. The exact size will depend on which model you're running,
+but try increasing it if you are running into strange corruption issues that
+might be related to stack overwriting.
+
+### Optimizing for your Platform
+
+The default reference implementations in TensorFlow Lite Micro are written to be
+portable and easy to understand, not fast, so you'll want to replace performance
+critical parts of the code with versions specifically tailored to your
+architecture. The framework has been designed with this in mind, and we hope the
+combination of small modules and many tests makes it as straightforward as
+possible to swap in your own code a piece at a time, ensuring you have a working
+version at every step.
+
+### Code Module Organization
+
+We have adopted a system of small modules with platform-specific implementations
+to help with portability. Every module is just a standard `.h` header file
+containing the interface (either functions or a class), with an accompanying
+reference implementation in a `.cc` with the same name. The source file
+implements all of the code that's declared in the header. If you have a
+specialized implementation, you can create a folder in the same directory as the
+header and reference source, name it after your platform, and put your
+implementation in a `.cc` file inside that folder. We've already seen one
+example of this, where the Mbed and Bluepill versions of `DebugLog()` are inside
+[mbed](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/experimental/micro/mbed)
+and
+[bluepill](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/experimental/micro/bluepill)
+folders, children of the
+[same directory](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/experimental/micro)
+where the stdio-based
+[`debug_log.cc`](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/experimental/micro/debug_log.cc)
+reference implementation is found.
+
+The advantage of this approach is that we can automatically pick specialized
+implementations based on the current build target, without having to manually
+edit build files for every new platform. It allows incremental optimizations
+from a always-working foundation, without cluttering the reference
+implementations with a lot of variants.
+
+To see why we're doing this, it's worth looking at the alternatives. TensorFlow
+Lite has traditionally used preprocessor macros to separate out some
+platform-specific code within particular files, for example:
+
+```
+#ifndef USE_NEON
+#if defined(__ARM_NEON__) || defined(__ARM_NEON)
+#define USE_NEON
+#include <arm_neon.h>
+#endif
+```
+
+There’s also a tradition in gemmlowp of using file suffixes to indicate
+platform-specific versions of particular headers, with kernel_neon.h being
+included by kernel.h if `USE_NEON` is defined. As a third variation, kernels are
+separated out using a directory structure, with
+tensorflow/lite/kernels/internal/reference containing portable implementations,
+and tensorflow/lite/kernels/internal/optimized holding versions optimized for
+NEON on Arm platforms.
+
+These approaches are hard to extend to multiple platforms. Using macros means
+that platform-specific code is scattered throughout files in a hard-to-find way,
+and can make following the control flow difficult since you need to understand
+the macro state to trace it. For example, I temporarily introduced a bug that
+disabled NEON optimizations for some kernels when I removed
+tensorflow/lite/kernels/internal/common.h from their includes, without realizing
+it was where USE_NEON was defined!
+
+It’s also tough to port to different build systems, since figuring out the right
+combination of macros to use can be hard, especially since some of them are
+automatically defined by the compiler, and others are only set by build scripts,
+often across multiple rules.
+
+The approach we are using extends the file system approach that we use for
+kernel implementations, but with some specific conventions:
+
+-   For each module in TensorFlow Lite, there will be a parent directory that
+    contains tests, interface headers used by other modules, and portable
+    implementations of each part.
+-   Portable means that the code doesn’t include code from any libraries except
+    flatbuffers, or other TF Lite modules. You can include a limited subset of
+    standard C or C++ headers, but you can’t use any functions that require
+    linking against those libraries, including fprintf, etc. You can link
+    against functions in the standard math library, in <math.h>.
+-   Specialized implementations are held inside subfolders of the parent
+    directory, named after the platform or library that they depend on. So, for
+    example if you had my_module/foo.cc, a version that used RISC-V extensions
+    would live in my_module/riscv/foo.cc. If you had a version that used the
+    CMSIS library, it should be in my_module/cmsis/foo.cc.
+-   These specialized implementations should completely replace the top-level
+    implementations. If this involves too much code duplication, the top-level
+    implementation should be split into smaller files, so only the
+    platform-specific code needs to be replaced.
+-   There is a convention about how build systems pick the right implementation
+    file. There will be an ordered list of 'tags' defining the preferred
+    implementations, and to generate the right list of source files, each module
+    will be examined in turn. If a subfolder with a tag’s name contains a .cc
+    file with the same base name as one in the parent folder, then it will
+    replace the parent folder’s version in the list of build files. If there are
+    multiple subfolders with matching tags and file names, then the tag that’s
+    latest in the ordered list will be chosen. This allows us to express “I’d
+    like generically-optimized fixed point if it’s available, but I’d prefer
+    something using the CMSIS library” using the list 'fixed_point cmsis'. These
+    tags are passed in as `TAGS="<foo"` on the command line when you use the
+    main Makefile to build.
+-   There is an implicit “reference” tag at the start of every list, so that
+    it’s possible to support directory structures like the current
+    tensorflow/kernels/internal where portable implementations are held in a
+    “reference” folder that’s a sibling to the NEON-optimized folder.
+-   The headers for each unit in a module should remain platform-agnostic, and
+    be the same for all implementations. Private headers inside a sub-folder can
+    be used as needed, but shouldn’t be referred to by any portable code at the
+    top level.
+-   Tests should be at the parent level, with no platform-specific code.
+-   No platform-specific macros or #ifdef’s should be used in any portable code.
-- 
GitLab


From 07706e401e64577720476e341744ead9eb2bf3e0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 16:00:57 -0800
Subject: [PATCH 0920/2345] Remove alias redirection for RBE python.

PiperOrigin-RevId: 229834629
---
 tensorflow/opensource_only.files    |  1 -
 third_party/py/python_configure.bzl |  4 +---
 third_party/py/remote.BUILD.tpl     | 13 -------------
 3 files changed, 1 insertion(+), 17 deletions(-)
 delete mode 100644 third_party/py/remote.BUILD.tpl

diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index 5f1ec1d673..9c4dd2f4b8 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -217,7 +217,6 @@ tensorflow/third_party/protobuf/BUILD
 tensorflow/third_party/tflite_mobilenet.BUILD
 tensorflow/third_party/py/BUILD
 tensorflow/third_party/py/BUILD.tpl
-tensorflow/third_party/py/remote.BUILD.tpl
 tensorflow/third_party/py/numpy/BUILD
 tensorflow/third_party/py/python_configure.bzl
 tensorflow/third_party/termcolor.BUILD
diff --git a/third_party/py/python_configure.bzl b/third_party/py/python_configure.bzl
index 53264630a1..9a7581c246 100644
--- a/third_party/py/python_configure.bzl
+++ b/third_party/py/python_configure.bzl
@@ -294,9 +294,7 @@ def _create_local_python_repository(repository_ctx):
 def _create_remote_python_repository(repository_ctx, remote_config_repo):
   """Creates pointers to a remotely configured repo set up to build with Python.
   """
-  _tpl(repository_ctx, "remote.BUILD", {
-      "%{REMOTE_PYTHON_REPO}": remote_config_repo,
-  }, "BUILD")
+  repository_ctx.template("BUILD", Label(remote_config_repo + ":BUILD"), {})
 
 
 def _python_autoconf_impl(repository_ctx):
diff --git a/third_party/py/remote.BUILD.tpl b/third_party/py/remote.BUILD.tpl
deleted file mode 100644
index edcac41ec6..0000000000
--- a/third_party/py/remote.BUILD.tpl
+++ /dev/null
@@ -1,13 +0,0 @@
-licenses(["restricted"])
-
-package(default_visibility = ["//visibility:public"])
-
-alias(
-    name = "python_headers",
-    actual = "%{REMOTE_PYTHON_REPO}:python_headers",
-)
-
-alias(
-    name = "numpy_headers",
-    actual = "%{REMOTE_PYTHON_REPO}:numpy_headers",
-)
-- 
GitLab


From fe342218ad5d083b825f9347f5c35d929a8d745b Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Thu, 17 Jan 2019 16:12:06 -0800
Subject: [PATCH 0921/2345] [tf.data] Fix usage of `test_util.run_v1_only` in
 MultiDeviceIterator and temporarily disable a failing test.

PiperOrigin-RevId: 229836608
---
 .../multi_device_iterator_test.py             | 30 +++++++++----------
 1 file changed, 14 insertions(+), 16 deletions(-)

diff --git a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
index 0c88d7533f..2bc0d8f90d 100644
--- a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
+++ b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
@@ -34,15 +34,12 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
 
-# TODO(b/121264236): Once we have a mechanism to have multiple devices in eager
-# / V2 mode, we should remove this annotation and the run_v1_only annotations
-# as well.
 @test_util.run_all_in_graph_and_eager_modes
 class MultiDeviceIteratorTest(test_base.DatasetTestBase,
                               parameterized.TestCase):
 
-  @test_util.run_v1_only
   @parameterized.parameters(0, 1, 42,)
+  @test_util.run_v1_only("b/121264236")
   def testInitOnly(self, num_inits):
     dataset = dataset_ops.Dataset.range(10)
     multi_device_iterator = multi_device_iterator_ops.MultiDeviceIterator(
@@ -53,7 +50,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase,
       for _ in range(num_inits):
         self.evaluate(multi_device_iterator.initializer)
 
-  @test_util.run_v1_only
+  @test_util.run_v1_only("b/121264236")
   def testBasic(self):
     dataset = dataset_ops.Dataset.range(10)
     multi_device_iterator = multi_device_iterator_ops.MultiDeviceIterator(
@@ -71,7 +68,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase,
         self.evaluate(elem_on_1)
         self.evaluate(elem_on_2)
 
-  @test_util.run_v1_only
+  @test_util.run_v1_only("b/121264236")
   def testOneOnSameDevice(self):
     with ops.device("/cpu:0"):
       dataset = dataset_ops.Dataset.range(10)
@@ -90,7 +87,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase,
         self.evaluate(elem_on_1)
         self.evaluate(elem_on_2)
 
-  @test_util.run_v1_only
+  @test_util.run_v1_only("b/121264236")
   def testRepeatDevices(self):
     with ops.device("/cpu:0"):
       dataset = dataset_ops.Dataset.range(20)
@@ -115,7 +112,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase,
         self.evaluate(elem_on_3)
         self.evaluate(elem_on_4)
 
-  @test_util.run_v1_only
+  @test_util.run_v1_only("b/121264236")
   def testNotFullyDivisible(self):
     dataset = dataset_ops.Dataset.range(9)
     multi_device_iterator = multi_device_iterator_ops.MultiDeviceIterator(
@@ -135,7 +132,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase,
         self.evaluate(elem_on_1)
         self.evaluate(elem_on_2)
 
-  @test_util.run_v1_only
+  @test_util.run_v1_only("b/121264236")
   def testGetNextAsOptional(self):
     if context.executing_eagerly():
       return
@@ -172,7 +169,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase,
       with self.assertRaises(errors.InvalidArgumentError):
         self.evaluate(elem_on_2_t)
 
-  @test_util.run_v1_only
+  @test_util.run_v1_only("b/121264236")
   def testUneven(self):
     dataset = dataset_ops.Dataset.range(10)
     multi_device_iterator = multi_device_iterator_ops.MultiDeviceIterator(
@@ -192,7 +189,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase,
         self.evaluate(elem_on_1)
         self.evaluate(elem_on_2)
 
-  @test_util.run_v1_only
+  @test_util.run_v1_only("b/121264236")
   def testMultipleInitializationsGraph(self):
     if context.executing_eagerly():
       return
@@ -214,8 +211,9 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase,
         self.assertEqual([(i, 0), (i, 1)], self.evaluate([elem_on_1,
                                                           elem_on_2]))
 
-  @test_util.run_v1_only
+  @test_util.run_v1_only("b/121264236")
   def testMultipleInitializationsEager(self):
+    self.skipTest("b/123023614")
     if not context.executing_eagerly():
       return
 
@@ -230,7 +228,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase,
       elem_on_1, elem_on_2 = multi_device_iterator.get_next()
       self.assertEqual([(0, 0), (1, 1)], self.evaluate([elem_on_1, elem_on_2]))
 
-  @test_util.run_v1_only
+  @test_util.run_v1_only("b/121264236")
   def testBasicGpu(self):
     if not test_util.is_gpu_available():
       self.skipTest("No GPU available")
@@ -251,7 +249,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase,
         self.evaluate(elem_on_1)
         self.evaluate(elem_on_2)
 
-  @test_util.run_v1_only
+  @test_util.run_v1_only("b/121264236")
   def testUnevenGpu(self):
     if not test_util.is_gpu_available():
       self.skipTest("No GPU available")
@@ -274,7 +272,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase,
         self.evaluate(elem_on_1)
         self.evaluate(elem_on_2)
 
-  @test_util.run_v1_only
+  @test_util.run_v1_only("b/121264236")
   def testGetNextAsOptionalGpu(self):
     if not test_util.is_gpu_available() or context.executing_eagerly():
       self.skipTest("No GPU available")
@@ -311,7 +309,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase,
       with self.assertRaises(errors.InvalidArgumentError):
         self.evaluate(elem_on_2_t)
 
-  @test_util.run_v1_only
+  @test_util.run_v1_only("b/121264236")
   def testOptimization(self):
     dataset = dataset_ops.Dataset.range(10)
     dataset = dataset.apply(optimization.assert_next(["MemoryCacheImpl"]))
-- 
GitLab


From ef10203524cd332afb96ce0fcd3a911654d40284 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Thu, 17 Jan 2019 16:25:57 -0800
Subject: [PATCH 0922/2345] Remove tf.train.ProximalGradientDescentOptimizer
 from the v2 API

I believe this was an oversight during API QA efforts

PiperOrigin-RevId: 229838796
---
 .../training/proximal_gradient_descent.py     |  2 +-
 ...-proximal-gradient-descent-optimizer.pbtxt | 51 -------------------
 .../api/golden/v2/tensorflow.train.pbtxt      |  4 --
 tensorflow/tools/compatibility/renames_v2.py  |  3 +-
 4 files changed, 3 insertions(+), 57 deletions(-)
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-proximal-gradient-descent-optimizer.pbtxt

diff --git a/tensorflow/python/training/proximal_gradient_descent.py b/tensorflow/python/training/proximal_gradient_descent.py
index 369b6cbb50..6eca0e6cb5 100644
--- a/tensorflow/python/training/proximal_gradient_descent.py
+++ b/tensorflow/python/training/proximal_gradient_descent.py
@@ -27,7 +27,7 @@ from tensorflow.python.training import training_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export("train.ProximalGradientDescentOptimizer")
+@tf_export(v1=["train.ProximalGradientDescentOptimizer"])
 class ProximalGradientDescentOptimizer(optimizer.Optimizer):
   # pylint: disable=line-too-long
   """Optimizer that implements the proximal gradient descent algorithm.
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-proximal-gradient-descent-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-proximal-gradient-descent-optimizer.pbtxt
deleted file mode 100644
index eb9a86183e..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-proximal-gradient-descent-optimizer.pbtxt
+++ /dev/null
@@ -1,51 +0,0 @@
-path: "tensorflow.train.ProximalGradientDescentOptimizer"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.proximal_gradient_descent.ProximalGradientDescentOptimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "GATE_GRAPH"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "GATE_NONE"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "GATE_OP"
-    mtype: "<type \'int\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'learning_rate\', \'l1_regularization_strength\', \'l2_regularization_strength\', \'use_locking\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0\', \'0.0\', \'False\', \'ProximalGradientDescent\'], "
-  }
-  member_method {
-    name: "apply_gradients"
-    argspec: "args=[\'self\', \'grads_and_vars\', \'global_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "compute_gradients"
-    argspec: "args=[\'self\', \'loss\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'False\', \'None\'], "
-  }
-  member_method {
-    name: "get_name"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_slot"
-    argspec: "args=[\'self\', \'var\', \'name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_slot_names"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "minimize"
-    argspec: "args=[\'self\', \'loss\', \'global_step\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'name\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'1\', \'None\', \'False\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "variables"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
index a3ace15ca2..8b39086ed1 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
@@ -60,10 +60,6 @@ tf_module {
     name: "JobDef"
     mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
   }
-  member {
-    name: "ProximalGradientDescentOptimizer"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "SequenceExample"
     mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index fb2303e9dd..6695eb4883 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -613,10 +613,10 @@ renames = {
     'tf.summary.FileWriter': 'tf.compat.v1.summary.FileWriter',
     'tf.summary.FileWriterCache': 'tf.compat.v1.summary.FileWriterCache',
     'tf.summary.SessionLog': 'tf.compat.v1.summary.SessionLog',
-    'tf.summary.audio': 'tf.compat.v1.summary.audio',
     'tf.summary.Summary': 'tf.compat.v1.summary.Summary',
     'tf.summary.SummaryDescription': 'tf.compat.v1.summary.SummaryDescription',
     'tf.summary.TaggedRunMetadata': 'tf.compat.v1.summary.TaggedRunMetadata',
+    'tf.summary.audio': 'tf.compat.v1.summary.audio',
     'tf.summary.get_summary_description': 'tf.compat.v1.summary.get_summary_description',
     'tf.summary.histogram': 'tf.compat.v1.summary.histogram',
     'tf.summary.image': 'tf.compat.v1.summary.image',
@@ -664,6 +664,7 @@ renames = {
     'tf.train.Optimizer': 'tf.compat.v1.train.Optimizer',
     'tf.train.ProfilerHook': 'tf.estimator.ProfilerHook',
     'tf.train.ProximalAdagradOptimizer': 'tf.compat.v1.train.ProximalAdagradOptimizer',
+    'tf.train.ProximalGradientDescentOptimizer': 'tf.compat.v1.train.ProximalGradientDescentOptimizer',
     'tf.train.QueueRunner': 'tf.compat.v1.train.QueueRunner',
     'tf.train.RMSPropOptimizer': 'tf.compat.v1.train.RMSPropOptimizer',
     'tf.train.Saver': 'tf.compat.v1.train.Saver',
-- 
GitLab


From c0ea4d61fee3ce6fd563de359bc46dc3260be6c4 Mon Sep 17 00:00:00 2001
From: Edward Loper <edloper@google.com>
Date: Thu, 17 Jan 2019 16:31:45 -0800
Subject: [PATCH 0923/2345] Add CompositeTensor base class.

CompositeTensor is a new base class for tensor-like objects that are composed from tf.Tensors.

This CL:

1. Adds the CompositeTensor base class, which defines two special methods for composing & decomposing CompositeTensors.

2. Adds an "expand_composites" argument to several of the functions in tensorflow nest, which controls whether composite tensors are expanded or treated as atomic values.

3. Updates control_flow_ops.py ops to use composite tensors, rather than special-casing behavior based on type checks.

PiperOrigin-RevId: 229839748
---
 tensorflow/contrib/framework/__init__.py      |   1 +
 tensorflow/python/BUILD                       |  28 ++
 tensorflow/python/eager/function.py           |   3 +-
 .../python/framework/composite_tensor.py      | 100 ++++
 .../python/framework/composite_tensor_test.py | 101 ++++
 tensorflow/python/framework/func_graph.py     |  65 +--
 tensorflow/python/framework/ops.py            | 170 ++++++-
 tensorflow/python/framework/sparse_tensor.py  |  29 +-
 .../kernel_tests/control_flow_ops_py_test.py  | 196 ++++++--
 tensorflow/python/ops/control_flow_ops.py     | 450 +++++++-----------
 tensorflow/python/ops/ragged/ragged_tensor.py |  50 +-
 tensorflow/python/util/nest.py                |  79 ++-
 tensorflow/python/util/util.cc                |  87 +++-
 tensorflow/python/util/util.h                 |  32 +-
 tensorflow/python/util/util.i                 |  30 +-
 .../v1/tensorflow.-indexed-slices.pbtxt       |   1 +
 .../golden/v1/tensorflow.-ragged-tensor.pbtxt |   1 +
 .../golden/v1/tensorflow.-sparse-tensor.pbtxt |   1 +
 .../v1/tensorflow.sparse.-sparse-tensor.pbtxt |   1 +
 .../v2/tensorflow.-indexed-slices.pbtxt       |   1 +
 .../golden/v2/tensorflow.-ragged-tensor.pbtxt |   1 +
 .../golden/v2/tensorflow.-sparse-tensor.pbtxt |   1 +
 .../v2/tensorflow.sparse.-sparse-tensor.pbtxt |   1 +
 23 files changed, 1008 insertions(+), 421 deletions(-)
 create mode 100644 tensorflow/python/framework/composite_tensor.py
 create mode 100644 tensorflow/python/framework/composite_tensor_test.py

diff --git a/tensorflow/contrib/framework/__init__.py b/tensorflow/contrib/framework/__init__.py
index e72e50585a..ff65c1586c 100644
--- a/tensorflow/contrib/framework/__init__.py
+++ b/tensorflow/contrib/framework/__init__.py
@@ -130,6 +130,7 @@ _allowed_symbols = ['nest']
 _nest_allowed_symbols = [
     'assert_same_structure',
     'is_sequence',
+    'is_sequence_or_composite',
     'flatten',
     'flatten_dict_items',
     'pack_sequence_as',
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index d4e6cbf804..e6f8201fc7 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -671,6 +671,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":common_shapes",
+        ":composite_tensor",
         ":cpp_shape_inference_proto_py",
         ":errors",
         ":framework_fast_tensor_util",
@@ -996,12 +997,39 @@ py_library(
     srcs = ["framework/sparse_tensor.py"],
     srcs_version = "PY2AND3",
     deps = [
+        ":composite_tensor",
         ":dtypes",
         ":framework_ops",
         ":tensor_util",
     ],
 )
 
+py_library(
+    name = "composite_tensor",
+    srcs = ["framework/composite_tensor.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":dtypes",
+        ":framework_ops",
+        ":tensor_util",
+    ],
+)
+
+py_test(
+    name = "framework_composite_tensor_test",
+    srcs = ["framework/composite_tensor_test.py"],
+    main = "framework/composite_tensor_test.py",
+    srcs_version = "PY2AND3",
+    deps = [
+        ":composite_tensor",
+        ":framework",
+        ":framework_for_generated_wrappers",
+        ":framework_test_lib",
+        ":platform_test",
+        "//tensorflow/core:protos_all_py",
+    ],
+)
+
 # This target is maintained separately from :util to provide separate visibility
 # for legacy users who were granted visibility when the functions were private
 # members of ops.Graph.
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 9c3e20f936..9642b2ba7c 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -689,7 +689,8 @@ class ConcreteFunction(object):
     # Clear captures, since we pass them in as inputs.
     backwards_graph.captures = {}
     backwards_graph.outputs.extend(
-        grad for grad in func_graph_module.flatten(gradients_wrt_inputs)
+        grad
+        for grad in nest.flatten(gradients_wrt_inputs, expand_composites=True)
         if grad is not None)
     backwards_graph.structured_outputs = gradients_wrt_inputs
     self._backward_graph_function = ConcreteFunction(
diff --git a/tensorflow/python/framework/composite_tensor.py b/tensorflow/python/framework/composite_tensor.py
new file mode 100644
index 0000000000..5c9a292e36
--- /dev/null
+++ b/tensorflow/python/framework/composite_tensor.py
@@ -0,0 +1,100 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tensor-like objects that are composed from tf.Tensors."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import abc
+
+import six
+
+from tensorflow.python import pywrap_tensorflow
+
+
+@six.add_metaclass(abc.ABCMeta)
+class CompositeTensor(object):
+  """Abstract base class for Tensor-like objects that are composed from Tensors.
+
+  Each `CompositeTensor` can be decomposed into a structured collection of
+  component `tf.Tensor`s, and reconstructed from those components.
+
+  The `tensorflow.python.util.nest` module has support for treating composite
+  tensors as structure, which makes it easy to flatten and reconstruct
+  composite tensors (or larger structures that contain composite tensors).
+  E.g.:
+
+  ```python
+  ct = ...  # Create a composite tensor.
+  flat_list_of_tensors = nest.flatten(ct, expand_composites=True)
+  transformed_list_of_tensors = ...  # do something with the flat tensors.
+  result = nest.pack_sequence_as(ct, transformed_list_of_tensors)
+  ```
+  """
+
+  @abc.abstractmethod
+  def _to_components(self):
+    """Decomposes this composite tensor into its components.
+
+    Returns:
+      The components that comprise this composite tensor: a nested structure
+      (as defined by `tf.python.util.nest`) whose values are `tf.Tensor`s or
+      `CompositeTensor`s.
+    """
+    raise NotImplementedError("CompositeTensor._to_components")
+
+  @abc.abstractmethod
+  def _from_components(cls, components):  # pylint: disable=no-self-argument
+    """Creates a composite tensor of type `cls` from components.
+
+    Args:
+      components: The components that should be used to form the
+        composite tensor: a nested structure (as defined by
+        `tf.python.util.nest`) whose values are tf.Tensors or composite
+        tensors.
+
+    Returns:
+      A `CompositeTensor` of type `cls`.
+    """
+    raise NotImplementedError("CompositeTensor._from_components")
+
+  @abc.abstractmethod
+  def _shape_invariant_to_components(self, shape=None):
+    """Converts a shape invariant into invariants for individual components.
+
+    Args:
+      shape: A `tf.TensorShape` object.  The shape invariant for this
+        `CompositeTensor`, or `None` if a default shape invariant should be
+        used (based on the value of this `CompositeTensor`).
+
+    Returns:
+      A nested structure whose values are `tf.TensorShape` objects, specifying
+      the shape invariants for the tensors that comprise this `CompositeTensor`.
+    """
+    raise NotImplementedError("CompositeTensor._shape_invariant_to_components")
+
+  @abc.abstractproperty
+  def _is_graph_tensor(self):
+    """Returns True if this tensor's components belong to a TF graph."""
+    raise NotImplementedError("CompositeTensor._is_symbolic_tensor")
+
+
+pywrap_tensorflow.RegisterType("CompositeTensor", CompositeTensor)
+
+
+# @TODO(edloper): Can we replace convert_to_tensor_or_xyz with just
+# convert_to_tensor_or_composite?  Alternatively, should composite tensors
+# register a dispatch override for tf.convert_to_tensor?
diff --git a/tensorflow/python/framework/composite_tensor_test.py b/tensorflow/python/framework/composite_tensor_test.py
new file mode 100644
index 0000000000..f249faa5d6
--- /dev/null
+++ b/tensorflow/python/framework/composite_tensor_test.py
@@ -0,0 +1,101 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for tensorflow.python.framework.composite_tensor."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import composite_tensor
+from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import googletest
+from tensorflow.python.util import nest
+
+
+class TestCompositeTensor(composite_tensor.CompositeTensor):
+
+  def __init__(self, *components):
+    self._components = components
+
+  def _to_components(self):
+    return self._components
+
+  @classmethod
+  def _from_components(cls, components):
+    return cls(*components)
+
+  def _shape_invariant_to_components(self, shape=None):
+    raise NotImplementedError('CompositeTensor._shape_invariant_to_components')
+
+  def _is_graph_tensor(self):
+    return True
+
+
+class CompositeTensorTest(test_util.TensorFlowTestCase):
+
+  def assertNestEqual(self, a, b, expand_composites=False):
+    if isinstance(a, dict):
+      self.assertIsInstance(b, dict)
+      self.assertEqual(set(a), set(b))
+      for key in a:
+        self.assertNestEqual(a[key], b[key])
+    elif isinstance(a, (list, tuple)):
+      self.assertIsInstance(b, (list, tuple))
+      self.assertEqual(len(a), len(b))
+      for a_val, b_val in zip(a, b):
+        self.assertNestEqual(a_val, b_val)
+    elif expand_composites and isinstance(a, composite_tensor.CompositeTensor):
+      self.assertIsInstance(b, composite_tensor.CompositeTensor)
+      self.assertNestEqual(a._to_components(),
+                           b._to_components())
+
+  def testNestFlatten(self):
+    st1 = sparse_tensor.SparseTensor([[0, 3], [7, 2]], [1, 2], [10, 10])
+    st2 = sparse_tensor.SparseTensor([[1, 2, 3]], ['a'], [10, 10, 10])
+    structure = [[st1], 'foo', {'y': [st2]}]
+    x = nest.flatten(structure, expand_composites=True)
+    self.assertEqual(len(x), 7)
+    self.assertIs(x[0], st1.indices)
+    self.assertIs(x[1], st1.values)
+    self.assertIs(x[2], st1.dense_shape)
+    self.assertEqual(x[3], 'foo')
+    self.assertIs(x[4], st2.indices)
+    self.assertIs(x[5], st2.values)
+    self.assertIs(x[6], st2.dense_shape)
+
+  def testNestPackSequenceAs(self):
+    st1 = sparse_tensor.SparseTensor([[0, 3], [7, 2]], [1, 2], [10, 10])
+    st2 = sparse_tensor.SparseTensor([[1, 2, 3]], ['a'], [10, 10, 10])
+    structure1 = [[st1], 'foo', {'y': [st2]}]
+    flat = [st2.indices, st2.values, st2.dense_shape, 'bar',
+            st1.indices, st1.values, st1.dense_shape]
+    result = nest.pack_sequence_as(structure1, flat, expand_composites=True)
+    expected = [[st2], 'bar', {'y': [st1]}]
+    self.assertNestEqual(expected, result)
+
+  def testAssertSameStructure(self):
+    st1 = sparse_tensor.SparseTensor([[0]], [0], [100])
+    st2 = sparse_tensor.SparseTensor([[0, 3]], ['x'], [100, 100])
+    test = TestCompositeTensor(st1.indices, st1.values, st1.dense_shape)
+    nest.assert_same_structure(st1, st2, expand_composites=False)
+    nest.assert_same_structure(st1, st2, expand_composites=True)
+    nest.assert_same_structure(st1, test, expand_composites=False)
+    with self.assertRaises(TypeError):
+      nest.assert_same_structure(st1, test, expand_composites=True)
+
+
+if __name__ == '__main__':
+  googletest.main()
diff --git a/tensorflow/python/framework/func_graph.py b/tensorflow/python/framework/func_graph.py
index 36f7aa67c7..28d829e89a 100644
--- a/tensorflow/python/framework/func_graph.py
+++ b/tensorflow/python/framework/func_graph.py
@@ -26,7 +26,6 @@ from tensorflow.python.eager import context
 from tensorflow.python.eager import tape
 from tensorflow.python.eager.graph_only_ops import graph_placeholder
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import tensor_spec
 from tensorflow.python.framework.auto_control_deps import AutomaticControlDependencies
 from tensorflow.python.ops import array_ops
@@ -489,7 +488,7 @@ def func_graph_from_py_func(name,
           x = array_ops.identity(op_return_value)
       elif not isinstance(x, tensor_array_ops.TensorArray):
         try:
-          x = ops.convert_to_tensor_or_indexed_slices(x)
+          x = ops.convert_to_tensor_or_composite(x)
         except (ValueError, TypeError):
           raise TypeError(
               "To be compatible with tf.contrib.eager.defun, Python functions "
@@ -623,36 +622,25 @@ def flatten(sequence):
   Flattens non-tensor objects into their constituent tensors.
 
   Args:
-    sequence: A nested structure of Tensors, IndexedSlices, SparseTensors and
+    sequence: A nested structure of Tensors, CompositeTensors, and
       TensorArrays.
 
   Returns:
     A list of tensors.
   """
   # TODO(akshayka): Support `SparseTensor` in a similar fashion.
-  flat_sequence = nest.flatten(sequence)
-  outputs = []
-  for item in flat_sequence:
-    if isinstance(item, ops.IndexedSlices):
-      if item.dense_shape is not None:
-        outputs.extend([item.values, item.indices, item.dense_shape])
-      else:
-        outputs.extend([item.values, item.indices])
-    elif isinstance(item, sparse_tensor.SparseTensor):
-      outputs.extend([item.indices, item.values, item.dense_shape])
-    elif isinstance(item, tensor_array_ops.TensorArray):
-      outputs.append(item.flow)
-    else:
-      outputs.append(item)
-  return outputs
+  flat_sequence = nest.flatten(sequence, expand_composites=True)
+  return [
+      item.flow if isinstance(item, tensor_array_ops.TensorArray) else item
+      for item in flat_sequence]
 
 
 def pack_sequence_as(structure, flat_sequence):
   """Like `nest.pack_sequence_as` but also packs other Tensor-like objects.
 
   Args:
-    structure: The structure to pack into. May contain Tensors, IndexedSlices,
-      TensorArrays or SparseTensors.
+    structure: The structure to pack into. May contain Tensors,
+      CompositeTensors, or TensorArrays.
     flat_sequence: An iterable containing tensors.
 
   Returns:
@@ -661,33 +649,16 @@ def pack_sequence_as(structure, flat_sequence):
   Raises:
     AssertionError if `structure` and `flat_sequence` are not compatible.
   """
-  flattened_structure = nest.flatten(structure)
-  flat_sequence_with_slices_and_tas = []
-  index = 0
-  for t in flattened_structure:
-    if isinstance(t, ops.IndexedSlices):
-      if t.dense_shape is not None:
-        flat_sequence_with_slices_and_tas.append(
-            ops.IndexedSlices(*flat_sequence[index:index + 3]))
-        index += 3
-      else:
-        flat_sequence_with_slices_and_tas.append(
-            ops.IndexedSlices(*flat_sequence[index:index + 2]))
-        index += 2
-    elif isinstance(t, sparse_tensor.SparseTensor):
-      flat_sequence_with_slices_and_tas.append(
-          sparse_tensor.SparseTensor(*flat_sequence[index:index + 3]))
-      index += 3
-    elif isinstance(t, tensor_array_ops.TensorArray):
-      flow = flat_sequence[index]
-      ta = tensor_array_ops.build_ta_with_new_flow(t, flow)
-      flat_sequence_with_slices_and_tas.append(ta)
-      index += 1
-    else:
-      flat_sequence_with_slices_and_tas.append(flat_sequence[index])
-      index += 1
-  assert len(flattened_structure) == len(flat_sequence_with_slices_and_tas)
-  return nest.pack_sequence_as(structure, flat_sequence_with_slices_and_tas)
+  flat_sequence = list(flat_sequence)
+  flattened_structure = nest.flatten(structure, expand_composites=True)
+  if len(flattened_structure) != len(flat_sequence):
+    raise ValueError("Mismatch in element count")
+  for i in range(len(flat_sequence)):
+    if isinstance(flattened_structure[i], tensor_array_ops.TensorArray):
+      flat_sequence[i] = tensor_array_ops.build_ta_with_new_flow(
+          old_ta=flattened_structure[i], flow=flat_sequence[i])
+  return nest.pack_sequence_as(structure, flat_sequence, expand_composites=True)
+
 
 
 def _create_substitute_placeholder(value, name=None, dtype=None):
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 6da6637b0b..e0f61622e4 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -41,6 +41,7 @@ from tensorflow.python.eager import context
 from tensorflow.python.eager import core
 from tensorflow.python.eager import tape
 from tensorflow.python.framework import c_api_util
+from tensorflow.python.framework import composite_tensor
 from tensorflow.python.framework import device as pydev
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -1032,7 +1033,7 @@ def convert_to_tensor(value, dtype=None, name=None, preferred_dtype=None):
       `preferred_dtype` is not possible, this argument has no effect.
 
   Returns:
-    An `Tensor` based on `value`.
+    A `Tensor` based on `value`.
 
   Raises:
     TypeError: If no conversion function is registered for `value` to `dtype`.
@@ -1085,7 +1086,7 @@ def convert_to_tensor_v2(value, dtype=None, dtype_hint=None, name=None):
     name: Optional name to use if a new `Tensor` is created.
 
   Returns:
-    An `Tensor` based on `value`.
+    A `Tensor` based on `value`.
 
   Raises:
     TypeError: If no conversion function is registered for `value` to `dtype`.
@@ -1228,7 +1229,7 @@ def internal_convert_n_to_tensor(values,
       value.
   """
   if not isinstance(values, collections.Sequence):
-    raise TypeError("values must be a list.")
+    raise TypeError("values must be a sequence.")
   ret = []
   if ctx is None: ctx = context.context()
   for i, value in enumerate(values):
@@ -1292,7 +1293,7 @@ def convert_to_tensor_or_indexed_slices(value, dtype=None, name=None):
     name: (Optional.) A name to use if a new `Tensor` is created.
 
   Returns:
-    An `Tensor`, `IndexedSlices`, or `SparseTensor` based on `value`.
+    A `Tensor`, `IndexedSlices`, or `SparseTensor` based on `value`.
 
   Raises:
     ValueError: If `dtype` does not match the element type of `value`.
@@ -1305,7 +1306,7 @@ def internal_convert_to_tensor_or_indexed_slices(value,
                                                  dtype=None,
                                                  name=None,
                                                  as_ref=False):
-  """Converts the given object to an `Tensor` or an `IndexedSlices`.
+  """Converts the given object to a `Tensor` or an `IndexedSlices`.
 
   If `value` is an `IndexedSlices` or `SparseTensor` it is returned
   unmodified. Otherwise, it is converted to a `Tensor` using
@@ -1320,7 +1321,7 @@ def internal_convert_to_tensor_or_indexed_slices(value,
     as_ref: True if the caller wants the results as ref tensors.
 
   Returns:
-    An `Tensor`, `IndexedSlices`, or `SparseTensor` based on `value`.
+    A `Tensor`, `IndexedSlices`, or `SparseTensor` based on `value`.
 
   Raises:
     ValueError: If `dtype` does not match the element type of `value`.
@@ -1351,7 +1352,7 @@ def internal_convert_n_to_tensor_or_indexed_slices(values,
   Args:
     values: A list of `None`, `IndexedSlices`, `SparseTensor`, or objects that
       can be consumed by `convert_to_tensor()`.
-    dtype: (Optional.) The required `DType` of the returned `Tensor`
+    dtype: (Optional.) The required `DType` of the returned `Tensor` or
       `IndexedSlices`.
     name: (Optional.) A name prefix to used when a new `Tensor` is
       created, in which case element `i` will be given the name `name
@@ -1359,7 +1360,7 @@ def internal_convert_n_to_tensor_or_indexed_slices(values,
     as_ref: True if the caller wants the results as ref tensors.
 
   Returns:
-    A list of `Tensor`, `IndexedSlices`, and/or `SparseTensor` objects.
+    A list of `Tensor`, `IndexedSlices`, `SparseTensor` and/or `None` objects.
 
   Raises:
     TypeError: If no conversion function is registered for an element in
@@ -1368,7 +1369,7 @@ def internal_convert_n_to_tensor_or_indexed_slices(values,
       value.
   """
   if not isinstance(values, collections.Sequence):
-    raise TypeError("values must be a list.")
+    raise TypeError("values must be a sequence.")
   ret = []
   for i, value in enumerate(values):
     if value is None:
@@ -1409,6 +1410,132 @@ def convert_n_to_tensor_or_indexed_slices(values, dtype=None, name=None):
       values=values, dtype=dtype, name=name, as_ref=False)
 
 
+def convert_to_tensor_or_composite(value, dtype=None, name=None):
+  """Converts the given object to a `Tensor` or `CompositeTensor`.
+
+  If `value` is a `CompositeTensor` it is returned unmodified. Otherwise, it
+  is converted to a `Tensor` using `convert_to_tensor()`.
+
+  Args:
+    value: A `CompositeTensor` or an object that can be consumed
+      by `convert_to_tensor()`.
+    dtype: (Optional.) The required `DType` of the returned `Tensor` or
+      `CompositeTensor`.
+    name: (Optional.) A name to use if a new `Tensor` is created.
+
+  Returns:
+    A `Tensor` or `CompositeTensor`, based on `value`.
+
+  Raises:
+    ValueError: If `dtype` does not match the element type of `value`.
+  """
+  return internal_convert_to_tensor_or_composite(
+      value=value, dtype=dtype, name=name, as_ref=False)
+
+
+def internal_convert_to_tensor_or_composite(value,
+                                            dtype=None,
+                                            name=None,
+                                            as_ref=False):
+  """Converts the given object to a `Tensor` or `CompositeTensor`.
+
+  If `value` is a `CompositeTensor` it is returned unmodified.  Otherwise, it
+  is converted to a `Tensor` using `convert_to_tensor()`.
+
+  Args:
+    value: A `CompositeTensor`, or an object that can be consumed
+      by `convert_to_tensor()`.
+    dtype: (Optional.) The required `DType` of the returned `Tensor` or
+      `CompositeTensor`.
+    name: (Optional.) A name to use if a new `Tensor` is created.
+    as_ref: True if the caller wants the results as ref tensors.
+
+  Returns:
+    A `Tensor` or `CompositeTensor`, based on `value`.
+
+  Raises:
+    ValueError: If `dtype` does not match the element type of `value`.
+  """
+  if isinstance(value, composite_tensor.CompositeTensor):
+    value_dtype = getattr(value, "dtype", None)
+    if dtype and not dtypes.as_dtype(dtype).is_compatible_with(value_dtype):
+      raise ValueError(
+          "Tensor conversion requested dtype %s for Tensor with dtype %s: %r" %
+          (dtypes.as_dtype(dtype).name, value.dtype.name, str(value)))
+    return value
+  else:
+    return internal_convert_to_tensor(
+        value, dtype=dtype, name=name, as_ref=as_ref)
+
+
+def internal_convert_n_to_tensor_or_composite(values,
+                                              dtype=None,
+                                              name=None,
+                                              as_ref=False):
+  """Converts `values` to a list of `Tensor` or `CompositeTensor` objects.
+
+  Any `CompositeTensor` objects in `values` are returned unmodified.
+
+  Args:
+    values: A list of `None`, `CompositeTensor`, or objects that
+      can be consumed by `convert_to_tensor()`.
+    dtype: (Optional.) The required `DType` of the returned `Tensor`s or
+      `CompositeTensor`s.
+    name: (Optional.) A name prefix to used when a new `Tensor` is
+      created, in which case element `i` will be given the name `name
+      + '_' + i`.
+    as_ref: True if the caller wants the results as ref tensors.
+
+  Returns:
+    A list of `Tensor`, `CompositeTensor`, and/or `None` objects.
+
+  Raises:
+    TypeError: If no conversion function is registered for an element in
+      `values`.
+    RuntimeError: If a registered conversion function returns an invalid
+      value.
+  """
+  if not isinstance(values, collections.Sequence):
+    raise TypeError("values must be a sequence.")
+  ret = []
+  for i, value in enumerate(values):
+    if value is None:
+      ret.append(value)
+    else:
+      n = None if name is None else "%s_%d" % (name, i)
+      ret.append(
+          internal_convert_to_tensor_or_composite(
+              value, dtype=dtype, name=n, as_ref=as_ref))
+  return ret
+
+
+def convert_n_to_tensor_or_composite(values, dtype=None, name=None):
+  """Converts `values` to a list of `Output` or `CompositeTensor` objects.
+
+  Any `CompositeTensor` objects in `values` are returned unmodified.
+
+  Args:
+    values: A list of `None`, `CompositeTensor``, or objects that
+      can be consumed by `convert_to_tensor()`.
+    dtype: (Optional.) The required `DType` of the returned `Tensor`s or
+      `CompositeTensor`s.
+    name: (Optional.) A name prefix to used when a new `Tensor` is
+      created, in which case element `i` will be given the name `name
+      + '_' + i`.
+
+  Returns:
+    A list of `Tensor` and/or `CompositeTensor` objects.
+
+  Raises:
+    TypeError: If no conversion function is registered for an element in
+      `values`.
+    RuntimeError: If a registered conversion function returns an invalid
+      value.
+  """
+  return internal_convert_n_to_tensor_or_composite(
+      values=values, dtype=dtype, name=name, as_ref=False)
+
+
 # TODO(josh11b): Add ctx argument to conversion_func() signature.
 @tf_export("register_tensor_conversion_function")
 def register_tensor_conversion_function(base_type,
@@ -1488,7 +1615,7 @@ def register_tensor_conversion_function(base_type,
 
 
 @tf_export("IndexedSlices")
-class IndexedSlices(_TensorLike):
+class IndexedSlices(_TensorLike, composite_tensor.CompositeTensor):
   """A sparse representation of a set of tensor slices at given indices.
 
   This class is a simple wrapper for a pair of `Tensor` objects:
@@ -1571,6 +1698,29 @@ class IndexedSlices(_TensorLike):
   def __neg__(self):
     return IndexedSlices(-self.values, self.indices, self.dense_shape)
 
+  def _to_components(self):
+    if self._dense_shape is None:
+      return (self._values, self._indices)
+    else:
+      return (self._values, self._indices, self._dense_shape)
+
+  @classmethod
+  def _from_components(cls, components):
+    return cls(*components)
+
+  def _shape_invariant_to_components(self, shape=None):
+    if shape is None:
+      shape = self._values.shape
+    if self._dense_shape is None:
+      return [shape, shape[:1]]  # values, indices
+    else:
+      # values, indices, dense_shape
+      return [shape, shape[:1], tensor_shape.TensorShape([shape.ndims])]
+
+  @property
+  def _is_graph_tensor(self):
+    return hasattr(self._values, 'graph')
+
 
 IndexedSlicesValue = collections.namedtuple(
     "IndexedSlicesValue", ["values", "indices", "dense_shape"])
diff --git a/tensorflow/python/framework/sparse_tensor.py b/tensorflow/python/framework/sparse_tensor.py
index 8546c2299a..c7f505fff3 100644
--- a/tensorflow/python/framework/sparse_tensor.py
+++ b/tensorflow/python/framework/sparse_tensor.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Classes and functions used to construct graphs."""
+"""Sparse tensors."""
 # pylint: disable=g-bad-name
 from __future__ import absolute_import
 from __future__ import division
@@ -21,8 +21,10 @@ from __future__ import print_function
 import collections
 
 from tensorflow.python import pywrap_tensorflow
+from tensorflow.python.framework import composite_tensor
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.util.tf_export import tf_export
 
@@ -34,7 +36,7 @@ _override_helper = ops._override_helper
 
 
 @tf_export("sparse.SparseTensor", "SparseTensor")
-class SparseTensor(_TensorLike):
+class SparseTensor(_TensorLike, composite_tensor.CompositeTensor):
   """Represents a sparse tensor.
 
   TensorFlow represents a sparse tensor as three separate dense tensors:
@@ -237,6 +239,29 @@ class SparseTensor(_TensorLike):
   def _override_operator(operator, func):
     _override_helper(SparseTensor, operator, func)
 
+  def _to_components(self):
+    return (self._indices, self._values, self._dense_shape)
+
+  @classmethod
+  def _from_components(cls, components):
+    return cls(*components)
+
+  def _shape_invariant_to_components(self, shape=None):
+    if shape is None:
+      shape = self.dense_shape.shape
+      # TODO(edloper): What if shape.dense_shape.shape.ndims is None?
+    if shape.ndims != 1:
+      raise ValueError("Shape invariant for SparseTensor must have the form "
+                       "TensorShape([r]), got %r" % shape)
+    rank = tensor_shape.dimension_value(shape[0])
+    return [tensor_shape.TensorShape([None, rank]),  # indices
+            tensor_shape.TensorShape([None]),  # values
+            tensor_shape.TensorShape([rank])]  # dense_shape
+
+  @property
+  def _is_graph_tensor(self):
+    return hasattr(self._values, 'graph')
+
 
 SparseTensorValue = collections.namedtuple(
     "SparseTensorValue", ["indices", "values", "dense_shape"])
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 104e9e82ee..27b3e26c77 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -63,6 +63,7 @@ from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import script_ops
+from tensorflow.python.ops import sparse_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import tensor_array_grad  # pylint: disable=unused-import
 from tensorflow.python.ops import tensor_array_ops
@@ -70,6 +71,7 @@ from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.ops import while_v2  # pylint: disable=unused-import
 # pylint: disable=unused-import
+from tensorflow.python.ops.ragged import ragged_factory_ops
 import tensorflow.python.ops.tensor_array_grad
 # pylint: enable=unused-import
 from tensorflow.python.platform import test
@@ -427,23 +429,31 @@ class ControlFlowTest(test.TestCase):
     self.assertAllEqual(11, val)
     self.assertAllEqual(0, ind)
 
-  @test_util.run_v1_only("b/120545219")
   def testCondSparseTensor(self):
-    with self.cached_session():
-      values = constant_op.constant([2.0, 4.0], name="values")
-      indices = constant_op.constant(
-          [[0], [3]], dtype=dtypes.int64, name="indices")
-      shape = constant_op.constant([10], dtype=dtypes.int64, name="dense_shape")
-      x = sparse_tensor.SparseTensor(indices, values, dense_shape=shape)
-      pred = math_ops.less(1, 2)
-      fn1 = lambda: sparse_tensor.SparseTensor(
-          indices + 1, x.values + 1, dense_shape=shape)
-      fn2 = lambda: sparse_tensor.SparseTensor(
-          indices, x.values - 1, dense_shape=shape)
-      r = control_flow_ops.cond(pred, fn1, fn2)
-      self.assertAllEqual([3.0, 5.0], r.values)
-      self.assertAllEqual([[1], [4]], r.indices)
-      self.assertAllEqual(r.values.get_shape(), (2,))
+    values = constant_op.constant([2.0, 4.0], name="values")
+    indices = constant_op.constant([[0], [3]],
+                                   dtype=dtypes.int64,
+                                   name="indices")
+    shape = constant_op.constant([10], dtype=dtypes.int64, name="dense_shape")
+    x = sparse_tensor.SparseTensor(indices, values, dense_shape=shape)
+    pred = math_ops.less(1, 2)
+    fn1 = lambda: sparse_tensor.SparseTensor(
+        indices + 1, x.values + 1, dense_shape=shape)
+    fn2 = lambda: sparse_tensor.SparseTensor(
+        indices, x.values - 1, dense_shape=shape)
+    r = control_flow_ops.cond(pred, fn1, fn2)
+    self.assertAllEqual([3.0, 5.0], r.values)
+    self.assertAllEqual([[1], [4]], r.indices)
+    self.assertAllEqual(r.values.get_shape(), (2,))
+
+  def testCondRaggedTensor(self):
+    rt = ragged_factory_ops.constant([[1, 2], [3], [4, 5, 6]])
+    pred = math_ops.less(1, 2)
+    fn1 = lambda: array_ops.concat([rt + 2, [[100]]], axis=0)
+    fn2 = lambda: rt[:2] - 2
+    result = control_flow_ops.cond(pred, fn1, fn2)
+    self.assertAllEqual([3, 4, 5, 6, 7, 8, 100], result.values)
+    self.assertAllEqual([0, 2, 3, 6, 7], result.row_splits)
 
   @test_util.run_v1_only("b/120545219")
   def testCondResource(self):
@@ -1519,35 +1529,80 @@ class ControlFlowTest(test.TestCase):
   @test_util.disable_control_flow_v2("b/116328420 (SparseTensor)")
   @test_util.run_v1_only("b/120545219")
   def testWhileShapeInferenceSparseTensor(self):
-    with self.cached_session():
-      values = constant_op.constant([2.0, 4.0], name="values")
-      indices = constant_op.constant(
-          [[0], [3]], dtype=dtypes.int64, name="indices")
-      shape = constant_op.constant([10], dtype=dtypes.int64, name="dense_shape")
-      i = constant_op.constant(0)
-      x = sparse_tensor.SparseTensor(indices, values, dense_shape=shape)
-
-      def c(i, _):
-        return i < 10
+    values = constant_op.constant([2.0, 4.0], name="values")
+    indices = constant_op.constant([[0], [3]],
+                                   dtype=dtypes.int64,
+                                   name="indices")
+    shape = constant_op.constant([10], dtype=dtypes.int64, name="dense_shape")
+    i = constant_op.constant(0)
+    x = sparse_tensor.SparseTensor(indices, values, dense_shape=shape)
+
+    def c(i, _):
+      return i < 10
+
+    def b1(i, x):  # modifies values.  (shape of components is not changed.)
+      return [
+          i + 1,
+          sparse_tensor.SparseTensor(x.indices, x.values * 2.0, x.dense_shape)
+      ]
 
-      def b(i, x):
-        return [
-            i + 1,
-            sparse_tensor.SparseTensor(x.indices, x.values * 2.0, x.dense_shape)
-        ]
+    def b2(i, x):  # adds new values.  (shape of components is changed.)
+      return [
+          i + 1,
+          sparse_ops.sparse_add(
+              x,
+              sparse_tensor.SparseTensor(
+                  indices=math_ops.cast(
+                      array_ops.fill([1, 1], i), dtypes.int64),
+                  values=array_ops.fill([1], 1.0),
+                  dense_shape=x.dense_shape))
+      ]
 
-      _, r = control_flow_ops.while_loop(c, b, [i, x])
-      self.assertEqual(r.dense_shape.get_shape()[0], 1)
+    def b3(i, x):  # modifies rank.  (shape of all components is changed.)
+      return [
+          i + 1,
+          sparse_tensor.SparseTensor(
+              array_ops.concat([x.indices, [[i], [i]]], axis=1), x.values * 2.0,
+              array_ops.concat([x.dense_shape, [10]], axis=0))
+      ]
 
+    # Default shape invariant; b1 only modifies values.
+    _, r = control_flow_ops.while_loop(c, b1, [i, x])
+    self.assertEqual(r.indices.get_shape().as_list(), [None, 1])
+    self.assertEqual(r.values.get_shape().as_list(), [None])
+    self.assertEqual(r.dense_shape.get_shape().as_list(), [1])
+
+    # Default shape invariant; b2 adds new values
+    _, r = control_flow_ops.while_loop(c, b2, [i, x])
+    self.assertEqual(r.indices.get_shape().as_list(), [None, 1])
+    self.assertEqual(r.values.get_shape().as_list(), [None])
+    self.assertEqual(r.dense_shape.get_shape().as_list(), [1])
+
+    # Default shape invariant; b3 modifies rank (which is not allowed).
+    with self.assertRaises(ValueError):
+      _, r = control_flow_ops.while_loop(c, b3, [i, x])
+
+    # Explicit shape invariant, allowing any rank; b1 only modifies values.
+    _, r = control_flow_ops.while_loop(
+        c, b1, [i, x],
+        [i.get_shape(), tensor_shape.TensorShape([None])])
+    self.assertEqual(r.indices.get_shape().as_list(), [None, None])
+    self.assertEqual(r.values.get_shape().as_list(), [None])
+    self.assertEqual(r.dense_shape.get_shape().as_list(), [None])
+
+    # Explicit shape invariant, allowing any rank; b3 modifies rank.
+    _, r = control_flow_ops.while_loop(
+        c, b3, [i, x],
+        [i.get_shape(), tensor_shape.TensorShape([None])])
+    self.assertEqual(r.indices.get_shape().as_list(), [None, None])
+    self.assertEqual(r.values.get_shape().as_list(), [None])
+    self.assertEqual(r.dense_shape.get_shape().as_list(), [None])
+
+    # Explicit shape invariant, with a specific (incompatible) rank.
+    with self.assertRaisesRegexp(ValueError, "is not compatible with"):
       _, r = control_flow_ops.while_loop(
-          c, b, [i, x],
-          [i.get_shape(), tensor_shape.TensorShape([None])])
-      self.assertEqual(r.dense_shape.get_shape().as_list(), [None])
-
-      with self.assertRaisesRegexp(ValueError, "is not compatible with"):
-        _, r = control_flow_ops.while_loop(
-            c, b, [i, x],
-            [i.get_shape(), tensor_shape.TensorShape([5])])
+          c, b1, [i, x],
+          [i.get_shape(), tensor_shape.TensorShape([5])])
 
   @test_util.disable_control_flow_v2("b/116282023 (IndexedSlices)")
   @test_util.run_v1_only("b/120545219")
@@ -1583,6 +1638,65 @@ class ControlFlowTest(test.TestCase):
             c, b, [i, x],
             [i.get_shape(), tensor_shape.TensorShape([None, 5])])
 
+  @test_util.disable_control_flow_v2("b/116328420 (RaggedTensor)")
+  def testWhileShapeInferenceRaggedTensor(self):
+    i = constant_op.constant(0)
+    x = ragged_factory_ops.constant([[1, 2], [3], [4, 5, 6]])
+    c = lambda i, _: i < 10
+
+    def b1(i, x):  # Adds new values to rows (but doesn't create new rows)
+      return [
+          i + 1,
+          array_ops.concat([x, x], axis=1)
+      ]
+
+    def b2(i, x):  # Adds new rows.
+      return [
+          i + 1,
+          array_ops.concat([x, x], axis=0)
+      ]
+
+    # Default shape invariant; b1 adds new values to rows.
+    _, r = control_flow_ops.while_loop(c, b1, [i, x])
+    self.assertEqual(r.row_splits.shape.as_list(), [4])
+
+    self.assertTrue(r.values.shape.as_list() in ([6 * 2**10], [None]))
+
+    # Default shape invariant; b2 adds new rows (not allowed).
+    if not context.executing_eagerly():
+      with self.assertRaises(ValueError):
+        _, r = control_flow_ops.while_loop(c, b2, [i, x])
+
+    # Explicit shape invariant; b1 adds new values to rows.
+    _, r = control_flow_ops.while_loop(
+        c, b1, [i, x],
+        [i.get_shape(), tensor_shape.TensorShape([None, None])])
+    self.assertTrue(r.row_splits.shape.as_list() in ([4], [None]))
+    self.assertTrue(r.values.shape.as_list() in ([6 * 2**10], [None]))
+
+    # Explicit shape invariant; b2 adds new rows.
+    _, r = control_flow_ops.while_loop(
+        c, b2, [i, x],
+        [i.get_shape(), tensor_shape.TensorShape([None, None])])
+    self.assertTrue(r.row_splits.shape.as_list() in ([3 * 2**10 + 1], [None]))
+    self.assertTrue(r.values.shape.as_list() in ([6 * 2**10], [None]))
+
+  @test_util.disable_control_flow_v2("b/116328420 (RaggedTensor)")
+  def testWhileShapeInferenceRaggedTensorRaggedRank2(self):
+    i = constant_op.constant(0)
+    x = ragged_factory_ops.constant([[[1, 2], [3], [4, 5, 6]],
+                                     [[], [8, 9, 10]]])
+    c = lambda i, _: i < 10
+    def b(i, x):
+      return [
+          i + 1,
+          array_ops.concat([x, x[..., i:i+1]], axis=-1)
+      ]
+    _, r = control_flow_ops.while_loop(c, b, [i, x])
+    self.assertEqual(r.row_splits.shape.as_list(), [3])
+    self.assertTrue(r.values.row_splits.shape.as_list() in ([6], [None]))
+    self.assertTrue(r.values.values.shape.as_list() in ([49], [None]))
+
   def _testNestedWhile_1(self, use_gpu):
     with self.cached_session(use_gpu=use_gpu):
       n = constant_op.constant(0)
diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py
index 3b3bfbbfc2..2bc3a9e9b6 100644
--- a/tensorflow/python/ops/control_flow_ops.py
+++ b/tensorflow/python/ops/control_flow_ops.py
@@ -30,11 +30,11 @@ import six
 from tensorflow.core.framework import attr_value_pb2
 from tensorflow.core.protobuf import control_flow_pb2
 from tensorflow.python.eager import context
+from tensorflow.python.framework import composite_tensor
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
@@ -181,47 +181,25 @@ def _Identity(data, name=None):
   Returns:
     A Tensor with the same type and value as the input Tensor.
   """
-  data = ops.internal_convert_to_tensor_or_indexed_slices(data, as_ref=True)
+  data = ops.internal_convert_to_tensor_or_composite(data, as_ref=True)
   if isinstance(data, ops.Tensor):
     if data.dtype._is_ref_dtype:  # pylint: disable=protected-access
       return gen_array_ops.ref_identity(data, name=name)
     else:
       return array_ops.identity(data, name=name)
   else:
-    if not isinstance(data, (ops.IndexedSlices, sparse_tensor.SparseTensor)):
-      raise TypeError("Type %s not supported" % type(data))
-    values = _Identity(data.values, name=name)
-    indices = array_ops.identity(data.indices, name="indices")
-    if isinstance(data, ops.IndexedSlices):
-      dense_shape = data.dense_shape
-      if dense_shape is not None:
-        dense_shape = array_ops.identity(dense_shape, name="dense_shape")
-      return ops.IndexedSlices(values, indices, dense_shape)
-    else:
-      dense_shape = array_ops.identity(data.dense_shape, name="dense_shape")
-      return sparse_tensor.SparseTensor(indices, values, dense_shape)
+    raise TypeError("Type %s not supported" % type(data))
 
 
 def _NextIteration(data, name=None):
-  data = ops.internal_convert_to_tensor_or_indexed_slices(data, as_ref=True)
+  data = ops.internal_convert_to_tensor_or_composite(data, as_ref=True)
   if isinstance(data, ops.Tensor):
     if data.dtype._is_ref_dtype:  # pylint: disable=protected-access
       return ref_next_iteration(data, name=name)
     else:
       return next_iteration(data, name=name)
   else:
-    if not isinstance(data, (ops.IndexedSlices, sparse_tensor.SparseTensor)):
-      raise TypeError("Type %s not supported" % type(data))
-    values = _NextIteration(data.values, name=name)
-    indices = next_iteration(data.indices, name="indices")
-    if isinstance(data, ops.IndexedSlices):
-      dense_shape = data.dense_shape
-      if dense_shape is not None:
-        dense_shape = next_iteration(dense_shape, name="dense_shape")
-      return ops.IndexedSlices(values, indices, dense_shape)
-    else:
-      dense_shape = next_iteration(data.dense_shape, name="dense_shape")
-      return sparse_tensor.SparseTensor(indices, values, dense_shape)
+    raise TypeError("Type %s not supported" % type(data))
 
 
 def _Enter(data,
@@ -244,12 +222,13 @@ def _Enter(data,
     is_constant: If true, the output is constant within the child frame.
     parallel_iterations: The number of iterations allowed to run in parallel.
     use_ref: If true, use ref_enter if data is of ref type.
+    use_input_shape: If true, set the result's shape based on data's shape.
     name: A name for this operation (optional).
 
   Returns:
     The same tensor as `data`.
   """
-  data = ops.internal_convert_to_tensor_or_indexed_slices(data, as_ref=True)
+  data = ops.internal_convert_to_tensor_or_composite(data, as_ref=True)
   if isinstance(data, ops.Tensor):
     if data.dtype._is_ref_dtype and use_ref:  # pylint: disable=protected-access
       result = gen_control_flow_ops.ref_enter(
@@ -261,45 +240,7 @@ def _Enter(data,
       result.set_shape(data.get_shape())
     return result
   else:
-    if not isinstance(data, (ops.IndexedSlices, sparse_tensor.SparseTensor)):
-      raise TypeError("Type %s not supported" % type(data))
-    values = _Enter(
-        data.values,
-        frame_name,
-        is_constant,
-        parallel_iterations=parallel_iterations,
-        use_input_shape=use_input_shape,
-        name=name)
-    indices = gen_control_flow_ops.enter(
-        data.indices,
-        frame_name,
-        is_constant,
-        parallel_iterations,
-        name="indices")
-    if use_input_shape:
-      indices.set_shape(data.indices.get_shape())
-    if isinstance(data, ops.IndexedSlices):
-      dense_shape = data.dense_shape
-      if dense_shape is not None:
-        dense_shape = gen_control_flow_ops.enter(
-            dense_shape,
-            frame_name,
-            is_constant,
-            parallel_iterations,
-            name="dense_shape")
-        if use_input_shape:
-          dense_shape.set_shape(data.dense_shape.get_shape())
-      return ops.IndexedSlices(values, indices, dense_shape)
-    else:
-      dense_shape = gen_control_flow_ops.enter(
-          data.dense_shape,
-          frame_name,
-          is_constant,
-          parallel_iterations,
-          name="dense_shape")
-      if use_input_shape:
-        dense_shape.set_shape(data.dense_shape.get_shape())
-      return sparse_tensor.SparseTensor(indices, values, dense_shape)
+    raise TypeError("Type %s not supported" % type(data))
 
 
 def exit(data, name=None):  # pylint: disable=redefined-builtin
@@ -314,25 +255,14 @@ def exit(data, name=None):  # pylint: disable=redefined-builtin
   Returns:
     The same tensor as `data`.
   """
-  data = ops.internal_convert_to_tensor_or_indexed_slices(data, as_ref=True)
+  data = ops.internal_convert_to_tensor_or_composite(data, as_ref=True)
   if isinstance(data, ops.Tensor):
     if data.dtype._is_ref_dtype:  # pylint: disable=protected-access
       return gen_control_flow_ops.ref_exit(data, name)
     else:
       return gen_control_flow_ops._exit(data, name)
   else:
-    if not isinstance(data, (ops.IndexedSlices, sparse_tensor.SparseTensor)):
-      raise TypeError("Type %s not supported" % type(data))
-    values = exit(data.values, name=name)
-    indices = gen_control_flow_ops._exit(data.indices, name="indices")
-    if isinstance(data, ops.IndexedSlices):
-      dense_shape = data.dense_shape
-      if dense_shape is not None:
-        dense_shape = gen_control_flow_ops._exit(dense_shape, name)
-      return ops.IndexedSlices(values, indices, dense_shape)
-    else:
-      dense_shape = gen_control_flow_ops._exit(data.dense_shape, name)
-      return sparse_tensor.SparseTensor(indices, values, dense_shape)
+    raise TypeError("Type %s not supported" % type(data))
 
 
 def switch(data, pred, dtype=None, name=None):
@@ -355,32 +285,19 @@ def switch(data, pred, dtype=None, name=None):
     to `output_true`, otherwise it goes to `output_false`.
   """
   with ops.name_scope(name, "Switch", [data, pred]) as name:
-    data = ops.internal_convert_to_tensor_or_indexed_slices(
+    data = ops.internal_convert_to_tensor_or_composite(
         data, dtype=dtype, name="data", as_ref=True)
     pred = ops.convert_to_tensor(pred, name="pred")
     if isinstance(data, ops.Tensor):
       return gen_control_flow_ops.switch(data, pred, name=name)
     else:
-      if not isinstance(data, (ops.IndexedSlices, sparse_tensor.SparseTensor)):
+      if not isinstance(data, composite_tensor.CompositeTensor):
         raise TypeError("Type %s not supported" % type(data))
-      val, ind = data.values, data.indices
-      val_f, val_t = gen_control_flow_ops.switch(val, pred, name=name)
-      ind_f, ind_t = gen_control_flow_ops.switch(ind, pred, name="indices")
-      if isinstance(data, ops.IndexedSlices):
-        dense_shape = data.dense_shape
-        if dense_shape is not None:
-          dense_shape_f, dense_shape_t = gen_control_flow_ops.switch(
-              dense_shape, pred, name="dense_shape")
-        else:
-          dense_shape_f, dense_shape_t = None, None
-        return (ops.IndexedSlices(val_f, ind_f, dense_shape_f),
-                ops.IndexedSlices(val_t, ind_t, dense_shape_t))
-      else:
-        dense_shape = data.dense_shape
-        dense_shape_f, dense_shape_t = gen_control_flow_ops.switch(
-            data.dense_shape, pred, name="dense_shape")
-        return (sparse_tensor.SparseTensor(ind_f, val_f, dense_shape_f),
-                sparse_tensor.SparseTensor(ind_t, val_t, dense_shape_t))
+      tensors = nest.flatten(data, expand_composites=True)
+      mapped = [gen_control_flow_ops.switch(tensor, pred) for tensor in tensors]
+      mapped_f, mapped_t = zip(*mapped)
+      return (nest.pack_sequence_as(data, mapped_f, expand_composites=True),
+              nest.pack_sequence_as(data, mapped_t, expand_composites=True))
 
 
 def _SwitchRefOrTensor(data, pred, name="Switch"):
@@ -403,7 +320,7 @@ def _SwitchRefOrTensor(data, pred, name="Switch"):
   Raises:
     TypeError: if data is not a Tensor or IndexedSlices
   """
-  data = ops.convert_to_tensor_or_indexed_slices(data, name="data")
+  data = ops.convert_to_tensor_or_composite(data, name="data")
   # NOTE(vrv): ops.colocate_with(data, ignore_existing=True) below
   # addresses the following scenario.
   #
@@ -456,7 +373,7 @@ def merge(inputs, name=None):
     raise ValueError("At least one of the merge inputs is None: %s" % inputs)
   with ops.name_scope(name, "Merge", inputs) as name:
     inputs = [
-        ops.internal_convert_to_tensor_or_indexed_slices(inp, as_ref=True)
+        ops.internal_convert_to_tensor_or_composite(inp, as_ref=True)
         for inp in inputs
     ]
     if all(isinstance(v, ops.Tensor) for v in inputs):
@@ -464,30 +381,27 @@ def merge(inputs, name=None):
         return gen_control_flow_ops.ref_merge(inputs, name)
       else:
         return gen_control_flow_ops.merge(inputs, name)
-    elif all(isinstance(v, sparse_tensor.SparseTensor) for v in inputs):
-      # Only handle the case when all inputs are SparseTensor.
-      values, _ = merge([inp.values for inp in inputs], name=name)
-      indices, chosen_index = gen_control_flow_ops.merge(
-          [inp.indices for inp in inputs], name="indices")
-      dense_shape, _ = gen_control_flow_ops.merge(
-          [inp.dense_shape for inp in inputs], name="dense_shape")
-      return (sparse_tensor.SparseTensor(indices, values, dense_shape),
-              chosen_index)
     else:
-      # For now convert all the inputs as IndexedSlices.
-      inputs = math_ops._as_indexed_slices_list(inputs, optimize=False)
-      values, _ = merge([inp.values for inp in inputs], name=name)
-      indices, chosen_index = gen_control_flow_ops.merge(
-          [inp.indices for inp in inputs], name="indices")
-      if any(inp.dense_shape is not None for inp in inputs):
-        if any(inp.dense_shape is None for inp in inputs):
-          raise ValueError("Either all merged IndexedSlices must have a "
-                           "dense_shape, or none must have a dense_shape.")
-        dense_shape, _ = gen_control_flow_ops.merge(
-            [inp.dense_shape for inp in inputs], name="dense_shape")
-      else:
-        dense_shape = None
-      return ops.IndexedSlices(values, indices, dense_shape), chosen_index
+      # If there is a mix of tensors and indexed slices, then convert the
+      # tensors to indexed slices.
+      if all(isinstance(v, (ops.IndexedSlices, ops.Tensor)) for v in inputs):
+        inputs = math_ops._as_indexed_slices_list(inputs, optimize=False)
+
+      for v in inputs:
+        if not isinstance(v, composite_tensor.CompositeTensor):
+          raise TypeError("Type %s not supported" % type(v))
+
+      for v in inputs[1:]:
+        nest.assert_same_structure(inputs[0], v, expand_composites=True)
+
+      flat_inputs = [nest.flatten(v, expand_composites=True) for v in inputs]
+      merged_results = [gen_control_flow_ops.merge(component)
+                        for component in zip(*flat_inputs)]
+      flat_merged = [tensor for (tensor, _) in merged_results]
+      chosen_index = merged_results[0][1]
+      merged_inputs = nest.pack_sequence_as(inputs[0], flat_merged,
+                                            expand_composites=True)
+      return (merged_inputs, chosen_index)
 
 
 # pylint: enable=protected-access
@@ -537,6 +451,30 @@ def _ShapeLessThanOrEqual(shape1, shape2):
   return True
 
 
+def _get_shape_invariant(var, shape=None):
+  """Returns a shape invariant for the given variable.
+
+  If `var` is a `CompositeTensor`, then this uses
+  `_shape_invariant_to_components()` to get shape invariants for the
+  component tensors.
+
+  Args:
+    var: The tensor whose shape is described.
+    shape: The shape invariant for the tensor.  If not specified, then a default
+      shape invariant for `var` is returned.
+
+  Returns:
+    The shape invariant for `var` (if it is a `Tensor`), or the shape invariants
+    for the components that comprise `var` (if it is a `CompositeTensor`).
+  """
+  if isinstance(var, composite_tensor.CompositeTensor):
+    return var._shape_invariant_to_components(shape)  # pylint: disable=protected-access
+  elif shape is None:
+    return var.shape
+  else:
+    return shape
+
+
 def _SetShapeInvariants(input_vars, enter_vars, shapes):
   """Set the shapes of the tensors in `enter_vars` to `shapes`.
 
@@ -566,31 +504,7 @@ def _SetShapeInvariants(input_vars, enter_vars, shapes):
             (inp.name, inp.get_shape(), shape))
       var.set_shape(shape)
     else:
-      if not isinstance(var, (ops.IndexedSlices, sparse_tensor.SparseTensor)):
-        raise TypeError("Type %s not supported" % type(var))
-      if isinstance(var, ops.IndexedSlices):
-        if not _ShapeLessThanOrEqual(inp.values.get_shape(), shape):
-          raise ValueError(
-              "The shape invariant specified for %s is not compatible with "
-              "the initial shape of the values tensor of this IndexedSlices. "
-              "It enters the loop with shape %s, but the specified shape "
-              "invariant is %s." % (inp.values.name, inp.values.get_shape(),
-                                    shape))
-        var.values.set_shape(shape)
-        var.indices.set_shape(tensor_shape.TensorShape([shape[0]]))
-        if var.dense_shape is not None:
-          var.dense_shape.set_shape(tensor_shape.TensorShape([shape.ndims]))
-      else:
-        if not _ShapeLessThanOrEqual(inp.dense_shape.get_shape(), shape):
-          raise ValueError(
-              "The shape invariant specified for %s is not compatible with "
-              "the initial shape of the shape tensor of this SparseTensor. "
-              "It enters the loop with shape %s, but the specified shape "
-              "invariant is %s." % (inp.dense_shape.name,
-                                    inp.dense_shape.get_shape(), shape))
-        var.values.set_shape(tensor_shape.TensorShape([None]))
-        var.indices.set_shape(tensor_shape.TensorShape([None, shape.ndims]))
-        var.dense_shape.set_shape(shape)
+      raise TypeError("Type %s not supported" % type(var))
 
 
 def _EnforceShapeInvariant(merge_var, next_var):
@@ -619,49 +533,7 @@ def _EnforceShapeInvariant(merge_var, next_var):
           "use the `shape_invariants` argument of tf.while_loop to specify a "
           "less-specific shape." % (input_t.name, input_t.shape, n_shape))
   else:
-    if not isinstance(merge_var,
-                      (ops.IndexedSlices, sparse_tensor.SparseTensor)):
-      raise TypeError("Type %s not supported" % type(merge_var))
-    if isinstance(merge_var, ops.IndexedSlices):
-      m_values_shape = merge_var.values.get_shape()
-      m_indices_shape = merge_var.indices.get_shape()
-      m_shape_shape = tensor_shape.TensorShape(None)
-      if merge_var.dense_shape is not None:
-        m_shape_shape = merge_var.dense_shape.get_shape()
-      n_values_shape = next_var.values.get_shape()
-      n_indices_shape = next_var.indices.get_shape()
-      n_shape_shape = tensor_shape.TensorShape(None)
-      if next_var.dense_shape is not None:
-        n_shape_shape = next_var.dense_shape.get_shape()
-      if (not _ShapeLessThanOrEqual(n_values_shape, m_values_shape) or
-          not _ShapeLessThanOrEqual(n_indices_shape, m_indices_shape)):
-        if not _ShapeLessThanOrEqual(n_values_shape, m_values_shape):
-          raise ValueError(
-              "The shape for %s is not an invariant for the loop. It enters "
-              "the loop with shape (%s, %s, %s), but has shape (%s, %s, %s) "
-              "after one iteration. Provide shape invariants using either the "
-              "`shape_invariants` argument of tf.while_loop or set_shape() "
-              "on the loop variables." %
-              (merge_var.name, m_values_shape, m_indices_shape, m_shape_shape,
-               n_values_shape, n_indices_shape, n_shape_shape))
-    else:
-      m_values_shape = merge_var.values.get_shape()
-      m_indices_shape = merge_var.indices.get_shape()
-      m_shape_shape = merge_var.dense_shape.get_shape()
-      n_values_shape = next_var.values.get_shape()
-      n_indices_shape = next_var.indices.get_shape()
-      n_shape_shape = next_var.dense_shape.get_shape()
-      if (not _ShapeLessThanOrEqual(n_values_shape, m_values_shape) or
-          not _ShapeLessThanOrEqual(n_indices_shape, m_indices_shape) or
-          not _ShapeLessThanOrEqual(n_shape_shape, m_shape_shape)):
-        raise ValueError(
-            "The shape for %s is not an invariant for the loop. It enters "
-            "the loop with shape (%s, %s, %s), but has shape (%s, %s, %s) "
-            "after one iteration. Provide shape invariants using either "
-            "the `shape_invariants` argument of tf.while_loop or set_shape() "
-            "on the loop variables." %
-            (merge_var.name, m_values_shape, m_indices_shape, m_shape_shape,
-             n_values_shape, n_indices_shape, n_shape_shape))
+    raise TypeError("Type %s not supported" % type(merge_var))
 
 
 def _AddNextAndBackEdge(m, v, enforce_shape_invariant=True):
@@ -676,26 +548,6 @@ def _AddNextAndBackEdge(m, v, enforce_shape_invariant=True):
       # TODO(skyewm): call this for other cases below (needs testing)
       _EnforceShapeInvariant(m, v)
     m.op._update_input(1, v)  # pylint: disable=protected-access
-  elif isinstance(m, ops.IndexedSlices):
-    # pylint: disable=protected-access
-    v = math_ops._as_indexed_slices(v, optimize=False)
-    v = _NextIteration(v)
-    m.values.op._update_input(1, v.values)
-    m.indices.op._update_input(1, v.indices)
-    # pylint: enable=protected-access
-    if m.dense_shape is not None:
-      if v.dense_shape is None:
-        raise ValueError("Must have dense shape: %s" % v.name)
-      m.dense_shape.op._update_input(1, v.dense_shape)
-  elif isinstance(m, sparse_tensor.SparseTensor):
-    if not isinstance(v, sparse_tensor.SparseTensor):
-      raise ValueError("Must be a sparse tensor: %s" % v.name)
-    v = _NextIteration(v)
-    # pylint: disable=protected-access
-    m.values.op._update_input(1, v.values)
-    m.indices.op._update_input(1, v.indices)
-    m.dense_shape.op._update_input(1, v.dense_shape)
-    # pylint: enable=protected-access
   else:
     raise TypeError("Type %s not supported" % type(m))
   return v
@@ -1613,7 +1465,8 @@ class ControlFlowContext(object):
   def ExitResult(self, result):
     """Make a list of tensors available in the outer context."""
     if self._outer_context:
-      nest.map_structure(lambda x: self._outer_context.AddName(x.name), result)
+      nest.map_structure(lambda x: self._outer_context.AddName(x.name), result,
+                         expand_composites=True)
 
   def GetWhileContext(self):
     """Return the while context containing this context."""
@@ -1920,19 +1773,9 @@ class CondContext(ControlFlowContext):
     if isinstance(v, ops.Operation):
       # Use pivot as the proxy for this op.
       return with_dependencies([v], self._pivot)
-    elif isinstance(v, (ops.IndexedSlices, sparse_tensor.SparseTensor)):
-      values = self._ProcessOutputTensor(v.values)
-      indices = self._ProcessOutputTensor(v.indices)
-      if isinstance(v, ops.IndexedSlices):
-        dense_shape = v.dense_shape
-        if dense_shape is not None:
-          dense_shape = self._ProcessOutputTensor(dense_shape)
-        return ops.IndexedSlices(values, indices, dense_shape)
-      else:
-        dense_shape = self._ProcessOutputTensor(v.dense_shape)
-        return sparse_tensor.SparseTensor(indices, values, dense_shape)
     else:
-      v = nest.map_structure(_convert_tensorarray_to_flow, v)
+      v = nest.map_structure(_convert_tensorarray_to_flow, v,
+                             expand_composites=True)
       return self._ProcessOutputTensor(ops.convert_to_tensor(v))
 
   def BuildCondBranch(self, fn):
@@ -1949,11 +1792,13 @@ class CondContext(ControlFlowContext):
           return no_op(), None
         else:
           original_result = nest.map_structure(array_ops.identity,
-                                               original_result)
+                                               original_result,
+                                               expand_composites=True)
     if original_result is None:
       return None, None
 
-    result = nest.map_structure(self._BuildCondTensor, original_result)
+    result = nest.map_structure(self._BuildCondTensor, original_result,
+                                expand_composites=True)
     if not isinstance(result, (list, _basetuple)):
       result = [result]
     return original_result, result
@@ -2120,7 +1965,8 @@ def cond(pred,
 
     # Check that the return values of the two branches have the same structure.
     try:
-      nest.assert_same_structure(orig_res_t, orig_res_f)
+      nest.assert_same_structure(orig_res_t, orig_res_f,
+                                 expand_composites=True)
     except TypeError as e:
       raise TypeError(
           "Incompatible return types of true_fn and false_fn: {}".format(e))
@@ -2132,24 +1978,21 @@ def cond(pred,
     if not res_t:
       raise ValueError("true_fn and false_fn must return at least one result.")
 
-    res_t_flat = nest.flatten(res_t)
-    res_f_flat = nest.flatten(res_f)
-
-    for x, y in zip(res_t_flat, res_f_flat):
-      assert ((isinstance(x, ops.IndexedSlices) and
-               isinstance(y, ops.IndexedSlices)) or
-              (isinstance(x, sparse_tensor.SparseTensor) and
-               isinstance(y, sparse_tensor.SparseTensor)) or
-              (isinstance(x, ops.Tensor) and isinstance(y, ops.Tensor)))
-      val_x = x if isinstance(x, ops.Tensor) else x.values
-      val_y = y if isinstance(y, ops.Tensor) else y.values
-      if val_x.dtype.base_dtype != val_y.dtype.base_dtype:
-        raise ValueError(
-            "Outputs of true_fn and false_fn must have the same type: %s, %s" %
-            (val_x.dtype.name, val_y.dtype.name))
+    res_t_flat = nest.flatten(res_t, expand_composites=True)
+    res_f_flat = nest.flatten(res_f, expand_composites=True)
+
+    for i, (x, y) in enumerate(zip(res_t_flat, res_f_flat)):
+      assert isinstance(x, ops.Tensor) and isinstance(y, ops.Tensor)
+      if x.dtype.base_dtype != y.dtype.base_dtype:
+        _cast_indexed_slice_indices(res_t, res_t_flat, res_f_flat)
+        if res_t_flat[i].dtype.base_dtype != res_f_flat[i].dtype.base_dtype:
+          raise ValueError(
+              "Outputs of true_fn and false_fn must have the same type: "
+              "%s, %s" % (x.dtype.name, y.dtype.name))
 
     merges = [merge(pair)[0] for pair in zip(res_f_flat, res_t_flat)]
-    merges = _convert_flows_to_tensorarrays(nest.flatten(orig_res_t), merges)
+    merges = _convert_flows_to_tensorarrays(
+        nest.flatten(orig_res_t, expand_composites=True), merges)
 
     # Only add non-nested conds to the collection. Any nested control flow will
     # be encapsulated in the root context.
@@ -2158,7 +2001,8 @@ def cond(pred,
       ops.add_to_collection(ops.GraphKeys.COND_CONTEXT, context_t)
       ops.add_to_collection(ops.GraphKeys.COND_CONTEXT, context_f)
 
-    merges = nest.pack_sequence_as(structure=orig_res_t, flat_sequence=merges)
+    merges = nest.pack_sequence_as(structure=orig_res_t, flat_sequence=merges,
+                                   expand_composites=True)
 
     # Singleton lists and tuples are automatically unpacked if strict == False.
     if not strict:
@@ -2166,6 +2010,48 @@ def cond(pred,
     return merges
 
 
+def _cast_indexed_slice_indices(structure, flat_a, flat_b):
+  """Cast IndexedSlice.indices from int32 to int64 where necessary.
+
+  For each `IndexedSlices` in the nested structure `structure`, find its
+  indices `Tensor` in the corresponding flattened lists `flat_a` and `flat_b`
+  (where composites have been expanded); and if those indices tensors have
+  different dtypes (i.e., if one is int64 but the other is int32), then cast
+  them to both be int64.
+
+  Args:
+    structure: The nested structure that was flattened.
+    flat_a: A flattened list of `Tensors` whose structure matches
+        `structure`.  Will be modified in place to cast `IndexedSlices`
+        indices tensors to int64, where necessary.
+    flat_a: A flattened list of `Tensors` whose structure matches
+        `structure`.  Will be modified in place to cast `IndexedSlices`
+        indices tensors to int64, where necessary.
+  """
+  # Find the locations (in flat_a and flat_b) of the IndexedSlices'
+  # indices tensors.
+  indexed_slice_indices = []
+  current_index = 0
+  for item in nest.flatten(structure, expand_composites=False):
+    if isinstance(item, ops.IndexedSlices):
+      # indices is the second component of the composite tensor.
+      indexed_slice_indices.append(current_index + 1)
+    if nest.is_sequence_or_composite(item):
+      current_index += len(nest.flatten(item, expand_composites=True))
+    else:
+      current_index += 1
+  assert current_index == len(flat_a)
+
+  for index in indexed_slice_indices:
+    assert flat_a[index].dtype in (dtypes.int32, dtypes.int64)
+    assert flat_b[index].dtype in (dtypes.int32, dtypes.int64)
+    if flat_a[index].dtype != flat_b[index].dtype:
+      if flat_b[index].dtype == dtypes.int32:
+        flat_b[index] = math_ops.cast(flat_b[index], dtypes.int64)
+      else:
+        flat_a[index] = math_ops.cast(flat_a[index], dtypes.int64)
+
+
 # pylint: enable=g-doc-args
 # pylint: enable=redefined-outer-name
 
@@ -2939,21 +2825,12 @@ class WhileContext(ControlFlowContext):
       if isinstance(x, ops.Tensor):
         self._values.add(x.name)
       else:
-        self._values.add(x.values.name)
-        self._values.add(x.indices.name)
-        if isinstance(x, ops.IndexedSlices):
-          dense_shape = x.dense_shape
-        elif isinstance(x, sparse_tensor.SparseTensor):
-          dense_shape = x.dense_shape
-        else:
-          raise TypeError("Type %s not supported" % type(x))
-        if dense_shape is not None:
-          self._values.add(dense_shape.name)
+        raise TypeError("Type %s not supported" % type(x))
 
   def _BuildLoop(self, pred, body, original_loop_vars, loop_vars,
                  shape_invariants):
     """Core: Add the loop termination condition and body to the graph."""
-    flat_loop_vars = nest.flatten(original_loop_vars)
+    flat_loop_vars = nest.flatten(original_loop_vars, expand_composites=True)
 
     # Let the context know the loop variables so the loop variables
     # would be added in the outer contexts properly.
@@ -3005,7 +2882,8 @@ class WhileContext(ControlFlowContext):
         _convert_flows_to_tensorarrays(flat_loop_vars, merge_vars))
     packed_vars = nest.pack_sequence_as(
         structure=original_loop_vars,
-        flat_sequence=merge_vars_with_tensor_arrays)
+        flat_sequence=merge_vars_with_tensor_arrays,
+        expand_composites=True)
     c = ops.convert_to_tensor(pred(*packed_vars))
     self._pivot = loop_cond(c, name="LoopCond")
     switch_vars = [_SwitchRefOrTensor(x, self._pivot) for x in merge_vars]
@@ -3019,11 +2897,12 @@ class WhileContext(ControlFlowContext):
         _convert_flows_to_tensorarrays(flat_loop_vars, vars_for_body))
     packed_vars_for_body = nest.pack_sequence_as(
         structure=original_loop_vars,
-        flat_sequence=vars_for_body_with_tensor_arrays)
+        flat_sequence=vars_for_body_with_tensor_arrays,
+        expand_composites=True)
     pre_summaries = ops.get_collection(ops.GraphKeys._SUMMARY_COLLECTION)  # pylint: disable=protected-access
     body_result = body(*packed_vars_for_body)
     post_summaries = ops.get_collection(ops.GraphKeys._SUMMARY_COLLECTION)  # pylint: disable=protected-access
-    if not nest.is_sequence(body_result):
+    if not nest.is_sequence_or_composite(body_result):
       body_result = [body_result]
     if len(post_summaries) > len(pre_summaries):
       new_summaries = post_summaries[len(pre_summaries):]
@@ -3037,20 +2916,24 @@ class WhileContext(ControlFlowContext):
             return x
           return array_ops.identity(x)
 
-        body_result = nest.map_structure(map_fn, body_result)
+        body_result = nest.map_structure(map_fn, body_result,
+                                         expand_composites=True)
 
     # Compare the structure types of input and output of body.
     # For backwards compatibility, the first layer is forced to a list
     # during this comparison, because inputs are typically lists and
     # outputs of the body are typically tuples.
-    nest.assert_same_structure(list(packed_vars_for_body), list(body_result))
+    nest.assert_same_structure(list(packed_vars_for_body), list(body_result),
+                               expand_composites=True)
 
     # Store body_result to keep track of TensorArrays returned by body
     original_body_result = body_result
     # Convert TensorArrays returned by body into their flow variables
-    result = nest.map_structure(_convert_tensorarray_to_flow,
-                                nest.flatten(body_result))
-    result = ops.convert_n_to_tensor_or_indexed_slices(result)
+    result = nest.map_structure(
+        _convert_tensorarray_to_flow,
+        nest.flatten(body_result, expand_composites=True),
+        expand_composites=True)
+    result = ops.convert_n_to_tensor_or_composite(result)
 
     # Add NextIteration and the back edges to complete the loop.
     if len(merge_vars) != len(result):
@@ -3076,9 +2959,15 @@ class WhileContext(ControlFlowContext):
     # Keep original_loop_vars to identify which are TensorArrays
     original_loop_vars = loop_vars
     # Convert TensorArrays to their flow variables
-    loop_vars = nest.map_structure(_convert_tensorarray_to_flow,
-                                   nest.flatten(loop_vars))
-    loop_vars = ops.convert_n_to_tensor_or_indexed_slices(loop_vars)
+    loop_vars = nest.map_structure(
+        _convert_tensorarray_to_flow,
+        nest.flatten(loop_vars, expand_composites=False),
+        expand_composites=True)
+    loop_vars = ops.convert_n_to_tensor_or_composite(loop_vars)
+    if shape_invariants is None:
+      shape_invariants = nest.map_structure(
+          _get_shape_invariant, loop_vars, expand_composites=False)
+    loop_vars = nest.flatten(loop_vars, expand_composites=True)
     try:
       self.Enter()
       # _BuildLoop calls _update_input in several places. _mutation_lock()
@@ -3090,14 +2979,15 @@ class WhileContext(ControlFlowContext):
     finally:
       self.Exit()
 
-    flat_result = nest.flatten(original_body_result)
+    flat_result = nest.flatten(original_body_result, expand_composites=True)
     # Convert TensorArray flow variables outside the context back into
     # their associated TensorArrays for returning to caller.
     exit_vars_with_tensor_arrays = (
         _convert_flows_to_tensorarrays(flat_result, exit_vars))
     packed_exit_vars = nest.pack_sequence_as(
         structure=original_body_result,
-        flat_sequence=exit_vars_with_tensor_arrays)
+        flat_sequence=exit_vars_with_tensor_arrays,
+        expand_composites=True)
 
     if return_same_structure:
       return packed_exit_vars
@@ -3111,12 +3001,7 @@ class WhileContext(ControlFlowContext):
       if isinstance(e, ops.Tensor):
         xs = [e]
       else:
-        if not isinstance(e, (ops.IndexedSlices, sparse_tensor.SparseTensor)):
-          raise TypeError("Type %s not supported" % type(e))
-        xs = [e.values, e.indices]
-        shape = e.dense_shape
-        if shape is not None:
-          xs.append(shape)
+        raise TypeError("Type %s not supported" % type(e))
       for x in xs:
         inp_op = x.op.inputs[0].op
         control_inputs = graph._control_dependencies_for_inputs([inp_op])
@@ -3546,7 +3431,12 @@ def while_loop(cond,
     if shape_invariants is not None:
       if maximum_iterations is not None:
         shape_invariants = (tensor_shape.TensorShape([]), shape_invariants)
-      nest.assert_same_structure(loop_vars, shape_invariants)
+
+      nest.assert_same_structure(loop_vars, shape_invariants,
+                                 expand_composites=False)
+      shape_invariants = nest.map_structure(
+          _get_shape_invariant, loop_vars, shape_invariants,
+          expand_composites=False)
 
     loop_context = WhileContext(
         maximum_iterations=maximum_iterations,
@@ -3588,7 +3478,7 @@ def _AsTensorList(x, p):
   for v in x:
     if isinstance(v, ops.Operation):
       v = with_dependencies([v], p)
-    v = ops.convert_to_tensor_or_indexed_slices(v)
+    v = ops.convert_to_tensor_or_composite(v)
     if isinstance(v, ops.Tensor):
       l.append(array_ops.identity(v))
     else:
@@ -3636,7 +3526,7 @@ def with_dependencies(dependencies, output_tensor, name=None):
                       list(dependencies) + [output_tensor]) as name:
     with ops.colocate_with(output_tensor):
       with ops.control_dependencies(dependencies):
-        output_tensor = ops.convert_to_tensor_or_indexed_slices(output_tensor)
+        output_tensor = ops.convert_to_tensor_or_composite(output_tensor)
         if isinstance(output_tensor, ops.Tensor):
           return _Identity(output_tensor, name=name)
         else:
@@ -3687,7 +3577,7 @@ def group(*inputs, **kwargs):
 
     # Sorts *inputs according to their devices.
     ops_on_device = {}  # device -> operations specified on the device.
-    for inp in nest.flatten(inputs):
+    for inp in nest.flatten(inputs, expand_composites=True):
       if not hasattr(inp, "device"):
         raise TypeError("Expected tf.group() expected Tensor arguments not "
                         "'%s' with type '%s'" % (inp, type(inp)))
diff --git a/tensorflow/python/ops/ragged/ragged_tensor.py b/tensorflow/python/ops/ragged/ragged_tensor.py
index fd334e6cc7..8fb3c1f44c 100644
--- a/tensorflow/python/ops/ragged/ragged_tensor.py
+++ b/tensorflow/python/ops/ragged/ragged_tensor.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.client import session
+from tensorflow.python.framework import composite_tensor
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -45,7 +46,7 @@ _eval_using_default_session = ops._eval_using_default_session
 
 
 @tf_export("RaggedTensor")
-class RaggedTensor(object):
+class RaggedTensor(composite_tensor.CompositeTensor):
   """Represents a ragged tensor.
 
   A `RaggedTensor` is a tensor with one or more *ragged dimensions*, which are
@@ -1437,6 +1438,53 @@ class RaggedTensor(object):
       values = values.values
     return values
 
+  #=============================================================================
+  # Composite Tensor
+  #=============================================================================
+
+  def _to_components(self):
+    return (self.flat_values,) + self.nested_row_splits
+
+  @classmethod
+  def _from_components(cls, components):
+    return cls.from_nested_row_splits(components[0], components[1:])
+
+  def _shape_invariant_to_components(self, shape=None):
+    ragged_rank = self.ragged_rank
+    flat_values = self.flat_values
+
+    if shape is None:
+      # Default shape invariant
+      value_shape = flat_values.shape[1:]
+      values_shape = tensor_shape.TensorShape([None]).concatenate(value_shape)
+      return ((values_shape, self._row_splits.shape) +
+              tuple(tensor_shape.TensorShape([None])
+                    for i in range(1, ragged_rank)))
+    else:
+      # Explicitly specified shape invariant
+      if shape.ndims is not None and shape.ndims <= ragged_rank:
+        raise ValueError("Shape invariant %s does not have sufficient rank "
+                         "for a RaggedTensor with %d ragged dimensions." %
+                         (shape, self.ragged_rank))
+      if any(tensor_shape.dimension_value(shape[dim]) is not None
+             for dim in range(1, self.ragged_rank + 1)):
+        raise ValueError("Shape invariant dimension size must be None for "
+                         "ragged dimenions.")
+      nrows = tensor_shape.dimension_value(shape[0])
+      value_shape = shape[self.ragged_rank + 1:]
+      values_shape = tensor_shape.TensorShape([None]).concatenate(value_shape)
+      if nrows is None:
+        outer_splits_shape = tensor_shape.TensorShape([None])
+      else:
+        outer_splits_shape = tensor_shape.TensorShape([nrows + 1])
+      return ((values_shape, outer_splits_shape) +
+              tuple(tensor_shape.TensorShape([None])
+                    for i in range(1, ragged_rank)))
+
+  @property
+  def _is_graph_tensor(self):
+    return hasattr(self._values, 'graph')
+
 
 def is_ragged(value):
   """Returns true if `value` is a ragged tensor or ragged tensor value."""
diff --git a/tensorflow/python/util/nest.py b/tensorflow/python/util/nest.py
index dbae97a373..83fd8bdf60 100644
--- a/tensorflow/python/util/nest.py
+++ b/tensorflow/python/util/nest.py
@@ -74,6 +74,7 @@ def _is_namedtuple(instance, strict=False):
 # See the swig file (util.i) for documentation.
 _is_mapping = _pywrap_tensorflow.IsMapping
 _is_attrs = _pywrap_tensorflow.IsAttrs
+_is_composite_tensor = _pywrap_tensorflow.IsCompositeTensor
 
 
 def _sequence_like(instance, args):
@@ -97,6 +98,8 @@ def _sequence_like(instance, args):
     return type(instance)((key, result[key]) for key in _six.iterkeys(instance))
   elif _is_namedtuple(instance) or _is_attrs(instance):
     return type(instance)(*args)
+  elif _is_composite_tensor(instance):
+    return instance._from_components(args)  # pylint: disable=protected-access
   else:
     # Not a namedtuple
     return type(instance)(args)
@@ -115,6 +118,9 @@ def _yield_value(iterable):
   elif _is_attrs(iterable):
     for value in _get_attrs_values(iterable):
       yield value
+  elif _is_composite_tensor(iterable):
+    for value in _yield_value(iterable._to_components()):  # pylint: disable=protected-access
+      yield value
   else:
     for value in iterable:
       yield value
@@ -124,6 +130,10 @@ def _yield_value(iterable):
 is_sequence = _pywrap_tensorflow.IsSequence
 
 
+# See the swig file (util.i) for documentation.
+is_sequence_or_composite = _pywrap_tensorflow.IsSequenceOrComposite
+
+
 # See the swig file (util.i) for documentation.
 flatten = _pywrap_tensorflow.Flatten
 
@@ -144,7 +154,8 @@ class _DotString(object):
 _DOT = _DotString()
 
 
-def assert_same_structure(nest1, nest2, check_types=True):
+def assert_same_structure(nest1, nest2, check_types=True,
+                          expand_composites=False):
   """Asserts that two structures are nested in the same way.
 
   Note that namedtuples with identical name and fields are always considered
@@ -168,6 +179,8 @@ def assert_same_structure(nest1, nest2, check_types=True):
         considered the same if they are both list subtypes (which allows "list"
         and "_ListWrapper" from checkpointable dependency tracking to compare
         equal).
+    expand_composites: If true, then composite tensors such as `tf.SparseTensor`
+        and `tf.RaggedTensor` are expanded into their component tensors.
 
   Raises:
     ValueError: If the two structures do not have the same number of elements or
@@ -176,7 +189,8 @@ def assert_same_structure(nest1, nest2, check_types=True):
       their substructures. Only possible if `check_types` is `True`.
   """
   try:
-    _pywrap_tensorflow.AssertSameStructure(nest1, nest2, check_types)
+    _pywrap_tensorflow.AssertSameStructure(nest1, nest2, check_types,
+                                           expand_composites)
   except (ValueError, TypeError) as e:
     str1 = str(map_structure(lambda _: _DOT, nest1))
     str2 = str(map_structure(lambda _: _DOT, nest2))
@@ -242,13 +256,14 @@ def flatten_dict_items(dictionary):
   return flat_dictionary
 
 
-def _packed_nest_with_indices(structure, flat, index):
+def _packed_nest_with_indices(structure, flat, index, is_seq):
   """Helper function for pack_sequence_as.
 
   Args:
     structure: Substructure (list / tuple / dict) to mimic.
     flat: Flattened values to output substructure for.
     index: Index at which to start reading from flat.
+    is_seq: Function used to test if a value should be treated as a sequence.
 
   Returns:
     The tuple (new_index, child), where:
@@ -263,8 +278,8 @@ def _packed_nest_with_indices(structure, flat, index):
   """
   packed = []
   for s in _yield_value(structure):
-    if is_sequence(s):
-      new_index, child = _packed_nest_with_indices(s, flat, index)
+    if is_seq(s):
+      new_index, child = _packed_nest_with_indices(s, flat, index, is_seq)
       packed.append(_sequence_like(s, child))
       index = new_index
     else:
@@ -273,7 +288,7 @@ def _packed_nest_with_indices(structure, flat, index):
   return index, packed
 
 
-def pack_sequence_as(structure, flat_sequence):
+def pack_sequence_as(structure, flat_sequence, expand_composites=False):
   """Returns a given flattened sequence packed into a given structure.
 
   If `structure` is a scalar, `flat_sequence` must be a single-element list;
@@ -293,6 +308,8 @@ def pack_sequence_as(structure, flat_sequence):
         tuples, and dicts. Note: numpy arrays and strings are considered
         scalars.
     flat_sequence: flat sequence to pack.
+    expand_composites: If true, then composite tensors such as `tf.SparseTensor`
+        and `tf.RaggedTensor` are expanded into their component tensors.
 
   Returns:
     packed: `flat_sequence` converted to have the same recursive structure as
@@ -303,17 +320,19 @@ def pack_sequence_as(structure, flat_sequence):
       element counts.
     TypeError: `structure` is or contains a dict with non-sortable keys.
   """
-  if not is_sequence(flat_sequence):
+  is_seq = is_sequence_or_composite if expand_composites else is_sequence
+  if not is_seq(flat_sequence):
     raise TypeError("flat_sequence must be a sequence")
 
-  if not is_sequence(structure):
+  if not is_seq(structure):
     if len(flat_sequence) != 1:
       raise ValueError("Structure is a scalar but len(flat_sequence) == %d > 1"
                        % len(flat_sequence))
     return flat_sequence[0]
 
   try:
-    final_index, packed = _packed_nest_with_indices(structure, flat_sequence, 0)
+    final_index, packed = _packed_nest_with_indices(structure, flat_sequence,
+                                                    0, is_seq)
     if final_index < len(flat_sequence):
       raise IndexError
   except IndexError:
@@ -326,7 +345,7 @@ def pack_sequence_as(structure, flat_sequence):
   return _sequence_like(structure, packed)
 
 
-def map_structure(func, *structure, **check_types_dict):
+def map_structure(func, *structure, **kwargs):
   """Applies `func` to each entry in `structure` and returns a new structure.
 
   Applies `func(x[0], x[1], ...)` where x[i] is an entry in
@@ -337,12 +356,18 @@ def map_structure(func, *structure, **check_types_dict):
     func: A callable that accepts as many arguments as there are structures.
     *structure: scalar, or tuple or list of constructed scalars and/or other
       tuples/lists, or scalars.  Note: numpy arrays are considered as scalars.
-    **check_types_dict: only valid keyword argument is `check_types`. If set to
-      `True` (default) the types of iterables within the structures have to be
-      same (e.g. `map_structure(func, [1], (1,))` raises a `TypeError`
-      exception). To allow this set this argument to `False`.
-      Note that namedtuples with identical name and fields are always
-      considered to have the same shallow structure.
+    **kwargs: Valid keyword args are:
+
+      * `check_types`: If set to `True` (default) the types of
+        iterables within the structures have to be same (e.g.
+        `map_structure(func, [1], (1,))` raises a `TypeError`
+        exception). To allow this set this argument to `False`.
+        Note that namedtuples with identical name and fields are always
+        considered to have the same shallow structure.
+      * `expand_composites`: If set to `True`, then composite tensors such
+        as `tf.SparseTensor` and `tf.RaggedTensor` are expanded into their
+        component tensors.  If `False` (the default), then composite tensors
+        are not expanded.
 
   Returns:
     A new structure with the same arity as `structure`, whose values correspond
@@ -364,21 +389,25 @@ def map_structure(func, *structure, **check_types_dict):
   if not structure:
     raise ValueError("Must provide at least one structure")
 
-  if check_types_dict:
-    if "check_types" not in check_types_dict or len(check_types_dict) > 1:
-      raise ValueError("Only valid keyword argument is check_types")
-    check_types = check_types_dict["check_types"]
-  else:
-    check_types = True
+  check_types = True
+  expand_composites = False
+  if kwargs:
+    check_types = kwargs.pop("check_types", check_types)
+    expand_composites = kwargs.pop("expand_composites", expand_composites)
+    if kwargs:
+      raise ValueError("Only valid keyword arguments are check_types "
+                       "and expand_composites")
 
   for other in structure[1:]:
-    assert_same_structure(structure[0], other, check_types=check_types)
+    assert_same_structure(structure[0], other, check_types=check_types,
+                          expand_composites=expand_composites)
 
-  flat_structure = [flatten(s) for s in structure]
+  flat_structure = [flatten(s, expand_composites) for s in structure]
   entries = zip(*flat_structure)
 
   return pack_sequence_as(
-      structure[0], [func(*x) for x in entries])
+      structure[0], [func(*x) for x in entries],
+      expand_composites=expand_composites)
 
 
 def map_structure_with_paths(func, *structure, **kwargs):
diff --git a/tensorflow/python/util/util.cc b/tensorflow/python/util/util.cc
index e69eec73a0..bda0cba82f 100644
--- a/tensorflow/python/util/util.cc
+++ b/tensorflow/python/util/util.cc
@@ -455,6 +455,14 @@ class SparseTensorValueIterator : public ValueIterator {
   Safe_PyObjectPtr tensor_;
 };
 
+// Returns nullptr (to raise an exception) when next() is called.  Caller
+// should have already called PyErr_SetString.
+class ErrorValueIterator : public ValueIterator {
+ public:
+  ErrorValueIterator() {}
+  Safe_PyObjectPtr next() override { return nullptr; }
+};
+
 class AttrsValueIterator : public ValueIterator {
  public:
   explicit AttrsValueIterator(PyObject* nested) : nested_(nested) {
@@ -497,6 +505,35 @@ bool IsSparseTensorValueType(PyObject* o) {
              o, reinterpret_cast<PyTypeObject*>(sparse_tensor_value_type)) == 1;
 }
 
+// Returns 1 if `o` is an instance of CompositeTensor.
+// Returns 0 otherwise.
+// Returns -1 if an error occurred.
+bool IsCompositeTensorHelper(PyObject* o) {
+  static auto* const check_cache = new CachedTypeCheck([](PyObject* to_check) {
+    PyObject* composite_tensor_type = GetRegisteredType("CompositeTensor");
+    if (TF_PREDICT_FALSE(composite_tensor_type == nullptr)) {
+      PyErr_SetString(PyExc_RuntimeError,
+                      tensorflow::strings::StrCat(
+                          "CompositeTensor type has not been set. "
+                          "Please register the type with the identifier "
+                          "\"CompositeTensor\" using RegisterType.")
+                          .c_str());
+      return -1;
+    }
+    int is_instance = PyObject_IsInstance(to_check, composite_tensor_type);
+
+    // Don't cache a failed is_instance check.
+    if (is_instance == -1) return -1;
+
+    return static_cast<int>(is_instance != 0);
+  });
+  return check_cache->CachedLookup(o);
+}
+
+int IsSequenceOrCompositeHelper(PyObject* o) {
+  return IsSequence(o) || IsCompositeTensor(o);
+}
+
 int IsSequenceForDataHelper(PyObject* o) {
   return IsSequenceHelper(o) == 1 && !PyList_Check(o) &&
          !IsSparseTensorValueType(o);
@@ -529,6 +566,18 @@ ValueIteratorPtr GetValueIteratorForData(PyObject* nested) {
   }
 }
 
+// Similar to GetValueIterator above, but expands CompositeTensors.
+ValueIteratorPtr GetValueIteratorForComposite(PyObject* nested) {
+  if (IsCompositeTensor(nested)) {
+    static char expand_method_name[] = "_to_components";
+    nested = PyObject_CallMethod(nested, expand_method_name, nullptr);
+    if (PyErr_Occurred() || nested == nullptr) {
+      return absl::make_unique<ErrorValueIterator>();
+    }
+  }
+  return GetValueIterator(nested);
+}
+
 bool FlattenHelper(
     PyObject* nested, PyObject* list,
     const std::function<int(PyObject*)>& is_sequence_helper,
@@ -596,7 +645,8 @@ void SetDifferentKeysError(PyObject* dict1, PyObject* dict2, string* error_msg,
 bool AssertSameStructureHelper(
     PyObject* o1, PyObject* o2, bool check_types, string* error_msg,
     bool* is_type_error,
-    const std::function<int(PyObject*)>& is_sequence_helper) {
+    const std::function<int(PyObject*)>& is_sequence_helper,
+    const std::function<ValueIteratorPtr(PyObject*)>& value_iterator_getter) {
   DCHECK(error_msg);
   DCHECK(is_type_error);
   const bool is_seq1 = is_sequence_helper(o1);
@@ -702,8 +752,8 @@ bool AssertSameStructureHelper(
     }
   }
 
-  ValueIteratorPtr iter1 = GetValueIterator(o1);
-  ValueIteratorPtr iter2 = GetValueIterator(o2);
+  ValueIteratorPtr iter1 = value_iterator_getter(o1);
+  ValueIteratorPtr iter2 = value_iterator_getter(o2);
 
   if (!iter1->valid() || !iter2->valid()) return false;
 
@@ -714,9 +764,9 @@ bool AssertSameStructureHelper(
       if (Py_EnterRecursiveCall(" in assert_same_structure")) {
         return false;
       }
-      bool no_internal_errors =
-          AssertSameStructureHelper(v1.get(), v2.get(), check_types, error_msg,
-                                    is_type_error, is_sequence_helper);
+      bool no_internal_errors = AssertSameStructureHelper(
+          v1.get(), v2.get(), check_types, error_msg, is_type_error,
+          is_sequence_helper, value_iterator_getter);
       Py_LeaveRecursiveCall();
       if (!no_internal_errors) return false;
       if (!error_msg->empty()) return true;
@@ -742,9 +792,13 @@ bool IsAttrs(PyObject* o) { return IsAttrsHelper(o) == 1; }
 bool IsTensor(PyObject* o) { return IsTensorHelper(o) == 1; }
 bool IsIndexedSlices(PyObject* o) { return IsIndexedSlicesHelper(o) == 1; }
 
-PyObject* Flatten(PyObject* nested) {
+PyObject* Flatten(PyObject* nested, bool expand_composites) {
   PyObject* list = PyList_New(0);
-  if (FlattenHelper(nested, list, IsSequenceHelper, GetValueIterator)) {
+  const std::function<int(PyObject*)>& is_sequence_helper =
+      expand_composites ? IsSequenceOrCompositeHelper : IsSequenceHelper;
+  const std::function<ValueIteratorPtr(PyObject*)>& get_value_iterator =
+      expand_composites ? GetValueIteratorForComposite : GetValueIterator;
+  if (FlattenHelper(nested, list, is_sequence_helper, get_value_iterator)) {
     return list;
   } else {
     Py_DECREF(list);
@@ -752,6 +806,12 @@ PyObject* Flatten(PyObject* nested) {
   }
 }
 
+bool IsSequenceOrComposite(PyObject* o) {
+  return IsSequenceOrCompositeHelper(o) == 1;
+}
+
+bool IsCompositeTensor(PyObject* o) { return IsCompositeTensorHelper(o) == 1; }
+
 bool IsSequenceForData(PyObject* o) { return IsSequenceForDataHelper(o) == 1; }
 
 PyObject* FlattenForData(PyObject* nested) {
@@ -850,11 +910,16 @@ PyObject* SameNamedtuples(PyObject* o1, PyObject* o2) {
   }
 }
 
-PyObject* AssertSameStructure(PyObject* o1, PyObject* o2, bool check_types) {
+PyObject* AssertSameStructure(PyObject* o1, PyObject* o2, bool check_types,
+                              bool expand_composites) {
+  const std::function<int(PyObject*)>& is_sequence_helper =
+      expand_composites ? IsSequenceOrCompositeHelper : IsSequenceHelper;
+  const std::function<ValueIteratorPtr(PyObject*)>& get_value_iterator =
+      expand_composites ? GetValueIteratorForComposite : GetValueIterator;
   string error_msg;
   bool is_type_error = false;
   AssertSameStructureHelper(o1, o2, check_types, &error_msg, &is_type_error,
-                            IsSequenceHelper);
+                            is_sequence_helper, get_value_iterator);
   if (PyErr_Occurred()) {
     // Don't hide Python exceptions while checking (e.g. errors fetching keys
     // from custom mappings).
@@ -878,7 +943,7 @@ PyObject* AssertSameStructureForData(PyObject* o1, PyObject* o2,
   string error_msg;
   bool is_type_error = false;
   AssertSameStructureHelper(o1, o2, check_types, &error_msg, &is_type_error,
-                            IsSequenceForDataHelper);
+                            IsSequenceForDataHelper, GetValueIterator);
   if (PyErr_Occurred()) {
     // Don't hide Python exceptions while checking (e.g. errors fetching keys
     // from custom mappings).
diff --git a/tensorflow/python/util/util.h b/tensorflow/python/util/util.h
index f37cd527d8..4a5db93401 100644
--- a/tensorflow/python/util/util.h
+++ b/tensorflow/python/util/util.h
@@ -33,6 +33,30 @@ namespace swig {
 //   dict.
 bool IsSequence(PyObject* o);
 
+// Implements the same interface as nest.is_sequence_or_composite
+// Returns a true if its input is a collections.Sequence (except strings)
+// or a CompositeTensor.
+//
+// Args:
+//   seq: an input sequence.
+//
+// Returns:
+//   True if the sequence is a not a string and is a collections.Sequence or a
+//   dict or a CompositeTensor.
+bool IsSequenceOrComposite(PyObject* o);
+
+// Implements the same interface as nest.is_sequence_or_composite
+// Returns a true if its input is a collections.Sequence (except strings)
+// or a CompositeTensor.
+//
+// Args:
+//   seq: an input sequence.
+//
+// Returns:
+//   True if the sequence is a not a string and is a collections.Sequence or a
+//   dict or a CompositeTensor.
+bool IsCompositeTensor(PyObject* o);
+
 // Implements the same interface as tensorflow.util.nest._is_namedtuple
 // Returns Py_True iff `instance` should be considered a `namedtuple`.
 //
@@ -118,7 +142,8 @@ PyObject* SameNamedtuples(PyObject* o1, PyObject* o2);
 //
 // Returns:
 //  Py_None on success, nullptr on error.
-PyObject* AssertSameStructure(PyObject* o1, PyObject* o2, bool check_types);
+PyObject* AssertSameStructure(PyObject* o1, PyObject* o2, bool check_types,
+                              bool expand_composites);
 
 // Implements the same interface as tensorflow.util.nest.flatten
 //
@@ -139,6 +164,9 @@ PyObject* AssertSameStructure(PyObject* o1, PyObject* o2, bool check_types);
 // Args:
 //   nest: an arbitrarily nested structure or a scalar object. Note, numpy
 //       arrays are considered scalars.
+//   expand_composites: If true, then composite tensors (such as
+//       `tf.SparseTensor` and `tf.RaggedTensor` are flattened into their
+//       component tensors.
 //
 // Returns:
 //   A Python list, the flattened version of the input.
@@ -146,7 +174,7 @@ PyObject* AssertSameStructure(PyObject* o1, PyObject* o2, bool check_types);
 //
 // Raises:
 //   TypeError: The nest is or contains a dict with non-sortable keys.
-PyObject* Flatten(PyObject* nested);
+PyObject* Flatten(PyObject* nested, bool expand_composites = false);
 
 // The tensorflow.python.data package has its own nest utility that follows very
 // slightly different semantics for its functions than the tensorflow.python
diff --git a/tensorflow/python/util/util.i b/tensorflow/python/util/util.i
index 4d34d61eee..6e2a3d8ccf 100644
--- a/tensorflow/python/util/util.i
+++ b/tensorflow/python/util/util.i
@@ -35,7 +35,7 @@ limitations under the License.
 %noexception tensorflow::swig::IsTensor;
 
 %feature("docstring") tensorflow::swig::IsSequence
-"""Returns a true if its input is a collections.Sequence (except strings).
+"""Returns true if its input is a collections.Sequence (except strings).
 
 Args:
   seq: an input sequence.
@@ -47,6 +47,31 @@ Returns:
 %unignore tensorflow::swig::IsSequence;
 %noexception tensorflow::swig::IsSequence;
 
+%feature("docstring") tensorflow::swig::IsSequenceOrComposite
+"""Returns true if its input is a sequence or a `CompositeTensor`.
+
+Args:
+  seq: an input sequence.
+
+Returns:
+  True if the sequence is a not a string and is a collections.Sequence or a
+  dict or a CompositeTensor.
+"""
+%unignore tensorflow::swig::IsSequenceOrComposite;
+%noexception tensorflow::swig::IsSequenceOrComposite;
+
+%feature("docstring") tensorflow::swig::IsCompositeTensor
+"""Returns true if its input is a `CompositeTensor`.
+
+Args:
+  seq: an input sequence.
+
+Returns:
+  True if the sequence is a CompositeTensor.
+"""
+%unignore tensorflow::swig::IsCompositeTensor;
+%noexception tensorflow::swig::IsCompositeTensor;
+
 %unignore tensorflow::swig::IsNamedtuple;
 %noexception tensorflow::swig::IsNamedtuple;
 
@@ -103,6 +128,8 @@ running.
 Args:
   nest: an arbitrarily nested structure or a scalar object. Note, numpy
       arrays are considered scalars.
+  expand_composites: If true, then composite tensors such as `tf.SparseTensor`
+      and `tf.RaggedTensor` are expanded into their component tensors.
 
 Returns:
   A Python list, the flattened version of the input.
@@ -112,6 +139,7 @@ Raises:
 """
 %unignore tensorflow::swig::Flatten;
 %noexception tensorflow::swig::Flatten;
+%feature("kwargs") tensorflow::swig::Flatten;
 
 %feature("docstring") tensorflow::swig::IsSequenceForData
 """Returns a true if `seq` is a Sequence or dict (except strings/lists).
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.-indexed-slices.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.-indexed-slices.pbtxt
index fee84d8530..5b47c718a5 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.-indexed-slices.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.-indexed-slices.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.IndexedSlices"
 tf_class {
   is_instance: "<class \'tensorflow.python.framework.ops.IndexedSlices\'>"
   is_instance: "<class \'tensorflow.python.framework.ops._TensorLike\'>"
+  is_instance: "<class \'tensorflow.python.framework.composite_tensor.CompositeTensor\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "dense_shape"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.-ragged-tensor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.-ragged-tensor.pbtxt
index c0ed956535..481a8c73ac 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.-ragged-tensor.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.-ragged-tensor.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.RaggedTensor"
 tf_class {
   is_instance: "<class \'tensorflow.python.ops.ragged.ragged_tensor.RaggedTensor\'>"
+  is_instance: "<class \'tensorflow.python.framework.composite_tensor.CompositeTensor\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "dtype"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.-sparse-tensor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.-sparse-tensor.pbtxt
index 3add49e90d..64f7260369 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.-sparse-tensor.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.-sparse-tensor.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.SparseTensor"
 tf_class {
   is_instance: "<class \'tensorflow.python.framework.sparse_tensor.SparseTensor\'>"
   is_instance: "<class \'tensorflow.python.framework.ops._TensorLike\'>"
+  is_instance: "<class \'tensorflow.python.framework.composite_tensor.CompositeTensor\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "dense_shape"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.sparse.-sparse-tensor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.sparse.-sparse-tensor.pbtxt
index 02e59a63e1..a31689a58b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.sparse.-sparse-tensor.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.sparse.-sparse-tensor.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.sparse.SparseTensor"
 tf_class {
   is_instance: "<class \'tensorflow.python.framework.sparse_tensor.SparseTensor\'>"
   is_instance: "<class \'tensorflow.python.framework.ops._TensorLike\'>"
+  is_instance: "<class \'tensorflow.python.framework.composite_tensor.CompositeTensor\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "dense_shape"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-indexed-slices.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-indexed-slices.pbtxt
index fee84d8530..5b47c718a5 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.-indexed-slices.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.-indexed-slices.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.IndexedSlices"
 tf_class {
   is_instance: "<class \'tensorflow.python.framework.ops.IndexedSlices\'>"
   is_instance: "<class \'tensorflow.python.framework.ops._TensorLike\'>"
+  is_instance: "<class \'tensorflow.python.framework.composite_tensor.CompositeTensor\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "dense_shape"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-ragged-tensor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-ragged-tensor.pbtxt
index c0ed956535..481a8c73ac 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.-ragged-tensor.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.-ragged-tensor.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.RaggedTensor"
 tf_class {
   is_instance: "<class \'tensorflow.python.ops.ragged.ragged_tensor.RaggedTensor\'>"
+  is_instance: "<class \'tensorflow.python.framework.composite_tensor.CompositeTensor\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "dtype"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-sparse-tensor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-sparse-tensor.pbtxt
index 3add49e90d..64f7260369 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.-sparse-tensor.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.-sparse-tensor.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.SparseTensor"
 tf_class {
   is_instance: "<class \'tensorflow.python.framework.sparse_tensor.SparseTensor\'>"
   is_instance: "<class \'tensorflow.python.framework.ops._TensorLike\'>"
+  is_instance: "<class \'tensorflow.python.framework.composite_tensor.CompositeTensor\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "dense_shape"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.sparse.-sparse-tensor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.sparse.-sparse-tensor.pbtxt
index 02e59a63e1..a31689a58b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.sparse.-sparse-tensor.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.sparse.-sparse-tensor.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.sparse.SparseTensor"
 tf_class {
   is_instance: "<class \'tensorflow.python.framework.sparse_tensor.SparseTensor\'>"
   is_instance: "<class \'tensorflow.python.framework.ops._TensorLike\'>"
+  is_instance: "<class \'tensorflow.python.framework.composite_tensor.CompositeTensor\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "dense_shape"
-- 
GitLab


From 54e40b2b13ff9cd1067050a505b091a496490f8f Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Thu, 17 Jan 2019 17:01:11 -0800
Subject: [PATCH 0924/2345] [tf.data] C++ test cleanup for map_vectorization

PiperOrigin-RevId: 229844456
---
 .../core/grappler/optimizers/data/BUILD       |   2 +
 .../optimizers/data/map_vectorization_test.cc | 332 +++++++++++-------
 2 files changed, 198 insertions(+), 136 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/data/BUILD b/tensorflow/core/grappler/optimizers/data/BUILD
index fe59dfef15..4ca353d7ba 100644
--- a/tensorflow/core/grappler/optimizers/data/BUILD
+++ b/tensorflow/core/grappler/optimizers/data/BUILD
@@ -497,10 +497,12 @@ tf_cc_test(
         ":graph_utils",
         ":map_vectorization",
         "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
         "//tensorflow/core/grappler:grappler_item",
+        "//tensorflow/core/kernels/data",
     ],
 )
 
diff --git a/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc b/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc
index f4faf41549..9d3d3af9f2 100644
--- a/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc
@@ -21,191 +21,251 @@ limitations under the License.
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/optimizers/data/graph_utils.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/gtl/array_slice.h"
 #include "tensorflow/core/platform/test.h"
 
 namespace tensorflow {
 namespace grappler {
 namespace {
 
-using test::function::GDef;
 using test::function::NDef;
 
-NodeDef MakeMapNodeHelper(StringPiece name, StringPiece input_node_name,
-                          StringPiece function_name, StringPiece map_op_name,
-                          gtl::ArraySlice<PartialTensorShape> output_shapes,
-                          gtl::ArraySlice<DataType> output_types) {
-  return test::function::NDef(
-      name, map_op_name, {string(input_node_name)},
-      {{"f", FunctionDefHelper::FunctionRef(string(function_name))},
-       {"Targuments", {}},
-       {"output_shapes", output_shapes},
-       {"output_types", output_types}});
+NodeDef* AddConstBoolNode(GraphDef* graph, StringPiece name, bool value) {
+  NodeDef* node = graph->add_node();
+  *node = NDef(/*name=*/name, /*op=*/"Const", /*inputs=*/{},
+               /*attrs=*/{{"value", value}, {"dtype", DT_BOOL}});
+
+  return node;
 }
 
-NodeDef MakeMapNode(StringPiece name, StringPiece input_node_name,
-                    StringPiece function_name,
-                    gtl::ArraySlice<PartialTensorShape> output_shapes,
-                    gtl::ArraySlice<DataType> output_types) {
-  return MakeMapNodeHelper(name, input_node_name, function_name, "MapDataset",
-                           output_shapes, output_types);
+NodeDef* AddConstInt32Node(GraphDef* graph, StringPiece name, int value) {
+  NodeDef* node = graph->add_node();
+  *node = NDef(/*name=*/name, /*op=*/"Const", /*inputs=*/{},
+               /*attrs=*/{{"value", value}, {"dtype", DT_INT32}});
+
+  return node;
 }
 
-NodeDef MakeBatchNode(StringPiece name, StringPiece input_node_name,
-                      StringPiece input_batch_size_name,
-                      gtl::ArraySlice<PartialTensorShape> output_shapes,
-                      gtl::ArraySlice<DataType> output_types) {
-  return NDef(
-      name, "BatchDataset",
-      {string(input_node_name), string(input_batch_size_name)},
-      {{"output_types", output_types}, {"output_shapes", output_shapes}});
+NodeDef* AddConstInt64Node(GraphDef* graph, StringPiece name, int64 value) {
+  NodeDef* node = graph->add_node();
+  *node = NDef(/*name=*/name, /*op=*/"Const", /*inputs=*/{},
+               /*attrs=*/{{"value", value}, {"dtype", DT_INT64}});
+
+  return node;
 }
 
-NodeDef MakeBatchV2Node(StringPiece name, StringPiece input_node_name,
-                        StringPiece input_batch_size_name,
-                        StringPiece input_drop_remainder_name,
-                        gtl::ArraySlice<PartialTensorShape> output_shapes,
-                        gtl::ArraySlice<DataType> output_types) {
-  return NDef(
-      name, "BatchDatasetV2",
-      {string(input_node_name), string(input_batch_size_name),
-       string(input_drop_remainder_name)},
-      {{"output_types", output_types}, {"output_shapes", output_shapes}});
+// Adds a simple vectorizable map function that is akin to
+// dataset.map(lambda x: tf.identity(x))
+FunctionDef* AddMapFn(GraphDef* graph) {
+  FunctionDef* map_fn = graph->mutable_library()->add_function();
+  *map_fn = FunctionDefHelper::Create(
+      /*function_name=*/"map_fn",
+      /*in_def=*/{"x: int64"},
+      /*out_def=*/{"res: int64"},
+      /*attr_def=*/{},
+      /*node_def=*/{{{"node"}, "Identity", {"x"}, {{"T", DT_INT64}}}},
+      /*ret_def=*/{{"res", "node:output"}});
+
+  return map_fn;
 }
 
-NodeDef MakeRangeNode(StringPiece name, gtl::ArraySlice<string> inputs) {
-  return NDef(name, "RangeDataset", inputs,
-              {{"output_shapes", gtl::ArraySlice<TensorShape>({{}})},
-               {"output_types", gtl::ArraySlice<DataType>({DT_INT64})}});
+NodeDef* AddMapNode(GraphDef* graph, const string& input_dataset,
+                    bool parallel = false) {
+  NodeDef* map_node = graph->add_node();
+  auto map_fn = AddMapFn(graph);
+
+  if (parallel) {
+    auto num_parallel_calls = AddConstInt32Node(graph, "num_parallel_calls", 8);
+    *map_node =
+        NDef(/*name=*/"map", /*op=*/"ParallelMapDataset",
+             /*inputs=*/{input_dataset, num_parallel_calls->name()},
+             /*attrs=*/
+             {{"f", FunctionDefHelper::FunctionRef(map_fn->signature().name())},
+              {"Targuments", gtl::ArraySlice<DataType>({})},
+              {"output_types", gtl::ArraySlice<DataType>({DT_INT64})},
+              {"output_shapes", gtl::ArraySlice<TensorShape>({{}})},
+              {"use_inter_op_parallelism", false},
+              {"sloppy", true},
+              {"preserve_cardinality", true}});
+  } else {
+    *map_node =
+        NDef(/*name=*/"map", /*op=*/"MapDataset",
+             /*inputs=*/{input_dataset},
+             /*attrs=*/
+             {{"f", FunctionDefHelper::FunctionRef(map_fn->signature().name())},
+              {"Targuments", gtl::ArraySlice<DataType>({})},
+              {"output_types", gtl::ArraySlice<DataType>({DT_INT64})},
+              {"output_shapes", gtl::ArraySlice<TensorShape>({{}})},
+              {"use_inter_op_parallelism", false},
+              {"preserve_cardinality", true}});
+  }
+
+  return map_node;
 }
 
-TEST(MapVectorizationTest, VectorizeMapWithBatch) {
-  GrapplerItem item;
-  item.graph = GDef(
-      {NDef("start", "Const", {}, {{"value", 0}, {"dtype", DT_INT32}}),
-       NDef("stop", "Const", {}, {{"value", 10}, {"dtype", DT_INT32}}),
-       NDef("step", "Const", {}, {{"value", 1}, {"dtype", DT_INT32}}),
-       NDef("batch_size", "Const", {}, {{"value", 1}, {"dtype", DT_INT32}}),
-       MakeRangeNode("range", {"start", "stop", "step"}),
-       MakeMapNode("map", "range", "XTimesTwo", {{}}, {DT_INT32}),
-       MakeBatchNode("batch", "map", "batch_size", {{-1}}, {DT_INT32})},
-      // FunctionLib
-      {
-          test::function::XTimesTwo(),
-      });
-  MapVectorization optimizer;
-  GraphDef output;
-  TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output));
+NodeDef* AddBatchNode(GraphDef* graph, const string& input_dataset,
+                      bool v2 = false) {
+  NodeDef* batch_node = graph->add_node();
+
+  auto batch_size = AddConstInt64Node(graph, "batch_size", 10);
+
+  if (v2) {
+    auto drop_remainder = AddConstBoolNode(graph, "drop_remainder", true);
+    *batch_node = NDef(
+        /*name=*/"batch", /*op=*/"BatchDatasetV2",
+        /*inputs=*/{input_dataset, batch_size->name(), drop_remainder->name()},
+        /*attrs=*/
+        {{"output_types", gtl::ArraySlice<DataType>({DT_INT64})},
+         {"output_shapes", gtl::ArraySlice<TensorShape>({{10, 1}})}});
+  } else {
+    *batch_node = NDef(
+        /*name=*/"batch", /*op=*/"BatchDataset",
+        /*inputs=*/{input_dataset, batch_size->name()},
+        /*attrs=*/
+        {{"output_types", gtl::ArraySlice<DataType>({DT_INT64})},
+         {"output_shapes", gtl::ArraySlice<PartialTensorShape>({{-1, 1}})}});
+  }
+  return batch_node;
+}
+
+NodeDef* AddRangeNode(GraphDef* graph) {
+  auto start = AddConstInt64Node(graph, "start", 0);
+  auto stop = AddConstInt64Node(graph, "stop", 10);
+  auto step = AddConstInt64Node(graph, "step", 1);
+  NodeDef* range_node = graph->add_node();
+  *range_node = NDef(/*name=*/"range", /*op=*/"RangeDataset",
+                     /*inputs=*/{start->name(), stop->name(), step->name()},
+                     /*attrs=*/
+                     {{"output_shapes", gtl::ArraySlice<TensorShape>({{}})},
+                      {"output_types", gtl::ArraySlice<DataType>({DT_INT64})}});
+
+  return range_node;
+}
+
+void CheckNotVectorized(const GraphDef& output, const string& map_op,
+                        const string& batch_op, const string& map_input_name) {
+  ASSERT_EQ(graph_utils::FindAllGraphNodesWithOp(map_op, output).size(), 1);
+  ASSERT_EQ(graph_utils::FindAllGraphNodesWithOp(batch_op, output).size(), 1);
+  const NodeDef& map_node =
+      output.node(graph_utils::FindGraphNodeWithOp(map_op, output));
+  const NodeDef& batch_node =
+      output.node(graph_utils::FindGraphNodeWithOp(batch_op, output));
+  EXPECT_EQ(map_node.input(0), map_input_name);
+  EXPECT_EQ(batch_node.input(0), map_node.name());
+}
 
-  EXPECT_EQ(graph_utils::FindAllGraphNodesWithOp("MapDataset", output).size(),
-            1);
-  EXPECT_EQ(graph_utils::FindAllGraphNodesWithOp("BatchDataset", output).size(),
-            1);
+void CheckVectorized(const GraphDef& output, const string& map_op,
+                     const string& batch_op, const string& map_input_name) {
+  ASSERT_EQ(graph_utils::FindAllGraphNodesWithOp(map_op, output).size(), 1);
+  ASSERT_EQ(graph_utils::FindAllGraphNodesWithOp(batch_op, output).size(), 1);
   const NodeDef& map_node =
-      output.node(graph_utils::FindGraphNodeWithOp("MapDataset", output));
+      output.node(graph_utils::FindGraphNodeWithOp(map_op, output));
   const NodeDef& batch_node =
-      output.node(graph_utils::FindGraphNodeWithOp("BatchDataset", output));
+      output.node(graph_utils::FindGraphNodeWithOp(batch_op, output));
   EXPECT_EQ(map_node.input(0), batch_node.name());
-  EXPECT_EQ(batch_node.input(0), "range");
+  EXPECT_EQ(batch_node.input(0), map_input_name);
+
+  // Check that the function is actually vectorized.
+  // The vectorization of the identity function is itself.
+  string function_name = map_node.attr().at("f").func().name();
+  int found =
+      graph_utils::FindGraphFunctionWithName(function_name, output.library());
+  ASSERT_NE(found, -1);
+  const auto& function = output.library().function(found);
+  EXPECT_EQ(function.node_def(0).op(), "Identity");
 }
 
-TEST(MapVectorizationTest, VectorizeMapWithBatchV2) {
+class MapThenBatchTest
+    : public ::testing::TestWithParam<std::tuple<bool, bool>> {};
+
+TEST_P(MapThenBatchTest, IsVectorized) {
+  bool use_parallel_map = std::get<0>(GetParam());
+  bool use_batch_v2 = std::get<1>(GetParam());
   GrapplerItem item;
-  item.graph = GDef(
-      {NDef("start", "Const", {}, {{"value", 0}, {"dtype", DT_INT32}}),
-       NDef("stop", "Const", {}, {{"value", 10}, {"dtype", DT_INT32}}),
-       NDef("step", "Const", {}, {{"value", 1}, {"dtype", DT_INT32}}),
-       NDef("batch_size", "Const", {}, {{"value", 1}, {"dtype", DT_INT32}}),
-       NDef("drop_remainder", "Const", {},
-            {{"value", false}, {"dtype", DT_BOOL}}),
-       MakeRangeNode("range", {"start", "stop", "step"}),
-       MakeMapNode("map", "range", "XTimesTwo", {{}}, {DT_INT32}),
-       MakeBatchV2Node("batch", "map", "batch_size", "drop_remainder", {{-1}},
-                       {DT_INT32})},
-      // FunctionLib
-      {
-          test::function::XTimesTwo(),
-      });
+  auto range_node = AddRangeNode(&item.graph);
+  auto map_node = AddMapNode(&item.graph, range_node->name(), use_parallel_map);
+  auto batch_node = AddBatchNode(&item.graph, map_node->name(), use_batch_v2);
   MapVectorization optimizer;
   GraphDef output;
   TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output));
+  CheckVectorized(output, map_node->op(), batch_node->op(), range_node->name());
+}
 
-  EXPECT_EQ(graph_utils::FindAllGraphNodesWithOp("MapDataset", output).size(),
-            1);
-  EXPECT_EQ(
-      graph_utils::FindAllGraphNodesWithOp("BatchDatasetV2", output).size(), 1);
-  const NodeDef& map_node =
-      output.node(graph_utils::FindGraphNodeWithOp("MapDataset", output));
-  const NodeDef& batch_node =
-      output.node(graph_utils::FindGraphNodeWithOp("BatchDatasetV2", output));
-  EXPECT_EQ(map_node.input(0), batch_node.name());
-  EXPECT_EQ(batch_node.input(0), "range");
+INSTANTIATE_TEST_SUITE_P(MapThenBatchTest, MapThenBatchTest,
+                         ::testing::Combine(::testing::Bool(),
+                                            ::testing::Bool()));
+
+// Not all dataset types have "output_shapes" and "output_types" attrs defined.
+// Add a generic input node which may not have these attrs defined.
+NodeDef* AddArbitraryInputNode(GraphDef* graph,
+                               std::vector<PartialTensorShape>* output_shapes,
+                               std::vector<DataType>* output_types) {
+  auto input_node = graph->add_node();
+  std::vector<std::pair<string, FunctionDefHelper::AttrValueWrapper>> attrs;
+  if (output_shapes) {
+    attrs.push_back({"output_shapes", *output_shapes});
+  }
+  if (output_types) {
+    attrs.push_back({"output_types", *output_types});
+  }
+
+  *input_node = NDef(/*name=*/"input", /*op=*/"InputDataset", /*inputs=*/{},
+                     /*attrs=*/attrs);
+
+  return input_node;
 }
 
-TEST(MapVectorizationTest, VectorizeWithUndefinedOutputShape) {
+TEST(MapVectorizationTest, VectorizeWithUndefinedOutputShapes) {
+  // Tests that the optimization doesn't break when the input to MapDataset
+  // doesn't have an output_shapes attr defined. In this case, the map and
+  // batch swap does not occur.
   GrapplerItem item;
-  item.graph = GDef(
-      {NDef("batch_size", "Const", {}, {{"value", 1}, {"dtype", DT_INT32}}),
-       NDef("input", "InputDataset", {},
-            {{"output_types", gtl::ArraySlice<DataType>({DT_INT32})}}),
-       MakeMapNode("map", "input", "XTimesTwo", {{}}, {DT_INT32}),
-       MakeBatchNode("batch", "map", "batch_size", {{-1}}, {DT_INT32})},
-      // FunctionLib
-      {
-          test::function::XTimesTwo(),
-      });
+  std::vector<DataType> input_types({DT_INT64});
+  auto input_node = AddArbitraryInputNode(&item.graph, nullptr, &input_types);
+  auto map_node = AddMapNode(&item.graph, input_node->name());
+  auto batch_node = AddBatchNode(&item.graph, map_node->name());
   MapVectorization optimizer;
   GraphDef output;
   TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output));
+  CheckNotVectorized(output, map_node->op(), batch_node->op(),
+                     input_node->name());
 }
 
-TEST(MapVectorizationTest, VectorizeWithUndefinedOutputTypes) {
+TEST(MapVectorizationTest, VectorizeWithUnknownDim) {
+  // Tests that the optimization doesn't break when the input to MapDataset
+  // doesn't has some components with unknown dimensions. In this case, the
+  // optimization does not occur.
   GrapplerItem item;
-  item.graph = GDef(
-      {NDef("batch_size", "Const", {}, {{"value", 1}, {"dtype", DT_INT32}}),
-       NDef("input", "InputDataset", {},
-            {{"output_shapes", gtl::ArraySlice<TensorShape>({{}})}}),
-       MakeMapNode("map", "input", "XTimesTwo", {{}}, {DT_INT32}),
-       MakeBatchNode("batch", "map", "batch_size", {{-1}}, {DT_INT32})},
-      // FunctionLib
-      {
-          test::function::XTimesTwo(),
-      });
+  std::vector<PartialTensorShape> input_shapes({{-1, 2}});
+  std::vector<DataType> input_types({DT_INT64});
+  auto input_node =
+      AddArbitraryInputNode(&item.graph, &input_shapes, &input_types);
+  auto map_node = AddMapNode(&item.graph, input_node->name());
+  auto batch_node = AddBatchNode(&item.graph, map_node->name());
   MapVectorization optimizer;
   GraphDef output;
   TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output));
+  CheckNotVectorized(output, map_node->op(), batch_node->op(),
+                     input_node->name());
 }
 
-TEST(MapVectorizationTest, VectorizeWithFullyDefinedFunction) {
+TEST(MapVectorizationTest, VectorizeWithUndefinedOutputTypes) {
+  // Tests that the optimization doesn't break when the input doesn't have
+  // an output_types attr defined. The output_types of the input node, even
+  // if not present, can be inferred from the map function input signature.
   GrapplerItem item;
-  item.graph = GDef(
-      {NDef("start", "Const", {}, {{"value", 0}, {"dtype", DT_INT32}}),
-       NDef("stop", "Const", {}, {{"value", 10}, {"dtype", DT_INT32}}),
-       NDef("step", "Const", {}, {{"value", 1}, {"dtype", DT_INT32}}),
-       NDef("batch_size", "Const", {}, {{"value", 1}, {"dtype", DT_INT32}}),
-       MakeRangeNode("range", {"start", "stop", "step"}),
-       MakeMapNode("map", "range", "Func", {{}}, {DT_INT32}),
-       MakeBatchNode("batch", "map", "batch_size", {{-1}}, {DT_INT32})},
-      // FunctionLib
-      {FunctionDefHelper::Create(
-          "Func", {"x: int64", "y: int64"}, {"res: int64", "res2: int64"}, {},
-          {{{"o"}, "Mul", {"x", "x"}, {{"T", DT_INT64}}}},
-          {{"res", "o:z"}, {"res2", "o:z"}})});
+  std::vector<PartialTensorShape> input_shapes({{1}});
+  auto input_node = AddArbitraryInputNode(&item.graph, &input_shapes, nullptr);
+  auto map_node = AddMapNode(&item.graph, input_node->name());
+  auto batch_node = AddBatchNode(&item.graph, map_node->name());
   MapVectorization optimizer;
   GraphDef output;
   TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output));
-
-  EXPECT_EQ(graph_utils::FindAllGraphNodesWithOp("MapDataset", output).size(),
-            1);
-  EXPECT_EQ(graph_utils::FindAllGraphNodesWithOp("BatchDataset", output).size(),
-            1);
-  const NodeDef& map_node =
-      output.node(graph_utils::FindGraphNodeWithOp("MapDataset", output));
-  const NodeDef& batch_node =
-      output.node(graph_utils::FindGraphNodeWithOp("BatchDataset", output));
-  EXPECT_EQ(map_node.input(0), batch_node.name());
-  EXPECT_EQ(batch_node.input(0), "range");
+  CheckVectorized(output, map_node->op(), batch_node->op(), input_node->name());
 }
 
+// TODO(rachelim): Add test that has a polymorphic function.
+
 }  // namespace
 }  // namespace grappler
 }  // namespace tensorflow
-- 
GitLab


From 15dca2b2fd1fc8ecab3f605fc810cd5d76fa3851 Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne <reedwm@google.com>
Date: Thu, 17 Jan 2019 17:26:35 -0800
Subject: [PATCH 0925/2345] Automated rollback of commit
 76c2a2512c3fefe065415e338a98c1043acbb131

PiperOrigin-RevId: 229848074
---
 tensorflow/python/module/module_test.py | 66 ++++++++++++-------------
 1 file changed, 33 insertions(+), 33 deletions(-)

diff --git a/tensorflow/python/module/module_test.py b/tensorflow/python/module/module_test.py
index d5d36a7bbc..e7c736696e 100644
--- a/tensorflow/python/module/module_test.py
+++ b/tensorflow/python/module/module_test.py
@@ -143,64 +143,64 @@ class VariableTrackingTest(test.TestCase):
 
   def test_variables(self):
     m = RecursiveModule(3)
-    self.assertCountEqual(m.variables, (m.w, m.child.w, m.child.child.w))
-    self.assertCountEqual(m.child.variables, (m.child.w, m.child.child.w))
-    self.assertCountEqual(m.child.child.variables, (m.child.child.w,))
+    self.assertEqual(m.variables, (m.w, m.child.w, m.child.child.w))
+    self.assertEqual(m.child.variables, (m.child.w, m.child.child.w))
+    self.assertEqual(m.child.child.variables, (m.child.child.w,))
 
   def test_owned_variables(self):
     m = RecursiveModule(3)
-    self.assertCountEqual(m.owned_variables, (m.w,))
-    self.assertCountEqual(m.child.owned_variables, (m.child.w,))
-    self.assertCountEqual(m.child.child.owned_variables, (m.child.child.w,))
+    self.assertEqual(m.owned_variables, (m.w,))
+    self.assertEqual(m.child.owned_variables, (m.child.w,))
+    self.assertEqual(m.child.child.owned_variables, (m.child.child.w,))
 
   def test_trainable_variables(self):
     m = RecursiveModule(3)
-    self.assertCountEqual(m.trainable_variables,
-                          (m.w, m.child.w, m.child.child.w))
-    self.assertCountEqual(m.child.trainable_variables,
-                          (m.child.w, m.child.child.w))
-    self.assertCountEqual(m.child.child.trainable_variables, (m.child.child.w,))
+    self.assertEqual(m.trainable_variables,
+                     (m.w, m.child.w, m.child.child.w))
+    self.assertEqual(m.child.trainable_variables,
+                     (m.child.w, m.child.child.w))
+    self.assertEqual(m.child.child.trainable_variables, (m.child.child.w,))
 
   def test_trainable_variables_ignores_non_trainable(self):
     m = RecursiveModule(3, trainable=False)
-    self.assertEmpty(m.trainable_variables)
-    self.assertEmpty(m.child.trainable_variables)
-    self.assertEmpty(m.child.child.trainable_variables)
+    self.assertEqual(len(m.trainable_variables), 0)
+    self.assertEqual(len(m.child.trainable_variables), 0)
+    self.assertEqual(len(m.child.child.trainable_variables), 0)
 
   def test_owned_trainable_variables(self):
     m = RecursiveModule(3)
-    self.assertCountEqual(m.owned_trainable_variables, (m.w,))
-    self.assertCountEqual(m.child.owned_trainable_variables, (m.child.w,))
-    self.assertCountEqual(m.child.child.owned_trainable_variables,
-                          (m.child.child.w,))
+    self.assertEqual(m.owned_trainable_variables, (m.w,))
+    self.assertEqual(m.child.owned_trainable_variables, (m.child.w,))
+    self.assertEqual(m.child.child.owned_trainable_variables,
+                     (m.child.child.w,))
 
   def test_owned_trainable_variables_ignores_non_trainable(self):
     m = RecursiveModule(3, trainable=False)
-    self.assertEmpty(m.owned_trainable_variables)
-    self.assertEmpty(m.child.owned_trainable_variables)
-    self.assertEmpty(m.child.child.owned_trainable_variables)
+    self.assertEqual(len(m.owned_trainable_variables), 0)
+    self.assertEqual(len(m.child.owned_trainable_variables), 0)
+    self.assertEqual(len(m.child.child.owned_trainable_variables), 0)
 
 
 class ModuleTrackingTest(test.TestCase):
 
   def test_owned_submodules(self):
     m = RecursiveModule(3)
-    self.assertCountEqual(m.owned_submodules, [m.child])
-    self.assertCountEqual(m.child.owned_submodules, [m.child.child])
-    self.assertEmpty(list(m.child.child.owned_submodules))
+    self.assertEqual(list(m.owned_submodules), [m.child])
+    self.assertEqual(list(m.child.owned_submodules), [m.child.child])
+    self.assertEqual(list(m.child.child.owned_submodules), [])
 
   def test_submodules(self):
     m = RecursiveModule(3)
-    self.assertCountEqual(m.submodules, [m.child, m.child.child])
-    self.assertCountEqual(m.child.submodules, [m.child.child])
-    self.assertEmpty(list(m.child.child.submodules))
+    self.assertEqual(list(m.submodules), [m.child, m.child.child])
+    self.assertEqual(list(m.child.submodules), [m.child.child])
+    self.assertEqual(list(m.child.child.submodules), [])
 
   def test_non_ctor_submodule(self):
     m = TreeModule()
     leaf1 = m.new_leaf()
-    self.assertCountEqual(m.submodules, (leaf1,))
+    self.assertEqual(set(m.submodules), {leaf1})
     leaf2 = m.new_leaf()
-    self.assertCountEqual(m.submodules, (leaf1, leaf2))
+    self.assertEqual(set(m.submodules), {leaf1, leaf2})
 
 
 class CommonErrorsTest(test.TestCase):
@@ -323,12 +323,12 @@ class WalkTest(test.TestCase):
     parent = SimpleModule()
     child = parent.c
 
-    self.assertCountEqual(
-        module.walk(parent, predicate=IS_MEMBER),
+    self.assertEqual(
+        list(module.walk(parent, predicate=IS_MEMBER)),
         [parent.a[0], parent.a[1], parent.z])
 
-    self.assertCountEqual(
-        module.walk(parent, recurse_if=IS_MODULE, predicate=IS_MEMBER),
+    self.assertEqual(
+        list(module.walk(parent, recurse_if=IS_MODULE, predicate=IS_MEMBER)),
         [parent.a[0], parent.a[1], parent.z, child.a[0], child.a[1], child.z])
 
 
-- 
GitLab


From 21734ca1e7f61ab8d62119b5a99be2926da58a7c Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Thu, 17 Jan 2019 17:27:48 -0800
Subject: [PATCH 0926/2345] [tf.data] Fixes a bug with how map_vectorization
 checks that the output shapes of the dataset nodes are fully defined.

PiperOrigin-RevId: 229848249
---
 .../optimizers/data/map_vectorization.cc      |  1 +
 .../optimizers/data/map_vectorization_test.cc | 22 +++++++++++++++++--
 .../optimization/map_vectorization_test.py    |  8 +++----
 3 files changed, 25 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
index 4c9e1f31e7..2dbd6e4698 100644
--- a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
@@ -121,6 +121,7 @@ bool IsOutputShapesFullyDefined(const NodeDef& node) {
   const auto& shapes = shapes_attr->list().shape();
 
   for (const TensorShapeProto& shape : shapes) {
+    if (shape.unknown_rank()) return false;
     for (const auto& dim : shape.dim()) {
       if (dim.size() == -1) {
         return false;
diff --git a/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc b/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc
index 9d3d3af9f2..890041f57c 100644
--- a/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc
@@ -231,10 +231,28 @@ TEST(MapVectorizationTest, VectorizeWithUndefinedOutputShapes) {
                      input_node->name());
 }
 
+TEST(MapVectorizationTest, VectorizeWithUnknownRank) {
+  // Tests that the optimization doesn't break when the input to MapDataset
+  // has components with unknown rank. In this case, the optimization does not
+  // occur.
+  GrapplerItem item;
+  std::vector<PartialTensorShape> input_shapes({{}});
+  std::vector<DataType> input_types({DT_INT64});
+  auto input_node =
+      AddArbitraryInputNode(&item.graph, &input_shapes, &input_types);
+  auto map_node = AddMapNode(&item.graph, input_node->name());
+  auto batch_node = AddBatchNode(&item.graph, map_node->name());
+  MapVectorization optimizer;
+  GraphDef output;
+  TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output));
+  CheckNotVectorized(output, map_node->op(), batch_node->op(),
+                     input_node->name());
+}
+
 TEST(MapVectorizationTest, VectorizeWithUnknownDim) {
   // Tests that the optimization doesn't break when the input to MapDataset
-  // doesn't has some components with unknown dimensions. In this case, the
-  // optimization does not occur.
+  // has components with unknown dimensions. In this case, the optimization does
+  // not occur.
   GrapplerItem item;
   std::vector<PartialTensorShape> input_shapes({{-1, 2}});
   std::vector<DataType> input_types({DT_INT64});
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
index ec543aeb76..9ffb8a1825 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
@@ -225,6 +225,10 @@ def _generate_csv_test_case():
 
 
 def _generate_parse_single_example_test_case():
+  # When sparse tensors are used, map_vectorization is not
+  # attempted because the output_shapes of the map dataset are not defined.
+  # TODO(rachelim): Consider being more lax with checking the output_shapes of
+  # the map node.
 
   def parse_example_factory():
 
@@ -243,8 +247,6 @@ def _generate_parse_single_example_test_case():
                     feature={
                         "dense_int": _int64_feature(i),
                         "dense_str": _bytes_feature(str(i)),
-                        "sparse_int": _int64_feature(i, i * 2, i * 4, i * 8),
-                        "sparse_str": _bytes_feature(*["abc"] * i)
                     })).SerializeToString() for i in range(10)
         ]))
 
@@ -252,8 +254,6 @@ def _generate_parse_single_example_test_case():
     features = {
         "dense_int": parsing_ops.FixedLenFeature((), dtypes.int64, 0),
         "dense_str": parsing_ops.FixedLenFeature((), dtypes.string, ""),
-        "sparse_int": parsing_ops.VarLenFeature(dtypes.int64),
-        "sparse_str": parsing_ops.VarLenFeature(dtypes.string),
     }
     return parsing_ops.parse_single_example(x, features)
 
-- 
GitLab


From fd1ab41166633dc5176c792fbd4c843ec2595857 Mon Sep 17 00:00:00 2001
From: Smit Hinsu <hinsu@google.com>
Date: Thu, 17 Jan 2019 17:31:43 -0800
Subject: [PATCH 0927/2345] Fix handling of truncated tensors in TensorRT Const
 op conversion

Currently TensorRT converter manually decodes tensor from protocol buffer and
it does not handle the case where tensor is encoded after truncating trailing
elements that have the same values. Fixed this by simplifying the conversion to
let TensorFlow decode tensor from the protocol buffer and then copying the
values to weights, possibly after casting to an appropriate DataType.

PiperOrigin-RevId: 229848825
---
 .../contrib/tensorrt/convert/convert_nodes.cc | 161 +++++++-----------
 .../contrib/tensorrt/convert/convert_nodes.h  |   4 +
 .../tensorrt/convert/convert_nodes_test.cc    |  61 ++++++-
 3 files changed, 119 insertions(+), 107 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 8b7279ad03..59d295f2e1 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -2712,43 +2712,69 @@ tensorflow::Status ConvertBiasAdd(OpConverterParams* params) {
   return Status::OK();
 }
 
-Status GetTensorDimsWithProtoShape(const Tensor& tensor,
-                                   int tensor_proto_array_len,
-                                   nvinfer1::Dims* dims) {
+void GetTensorDimsWithProtoShape(const Tensor& tensor, nvinfer1::Dims* dims) {
   if (tensor.dims() > 0) {
     *dims = GetTrtDimsForTensor(tensor);
-    if (TrtDimsNumElements(*dims) != tensor_proto_array_len &&
-        tensor_proto_array_len != 1) {
-      return errors::InvalidArgument(
-          "Broadcast on weights only supports kCHANNEL and kUNIFORM");
-    }
   } else {
     dims->nbDims = 1;
     // No dimension provided. Flatten it.
-    dims->d[0] = tensor_proto_array_len;
+    dims->d[0] = tensor.NumElements();
     dims->type[0] = nvinfer1::DimensionType::kSPATIAL;
     for (int i = 1; i < nvinfer1::Dims::MAX_DIMS; ++i) {
       dims->d[i] = 0;
     }
   }
-  return Status::OK();
 }
 
-template <typename CType>
-Status TfTensorToTrtWeights(const DataType dtype, const Tensor& tensor,
-                            const CType* tensor_proto_array,
-                            int tensor_proto_array_len, TrtWeightStore* store,
+Status TfTensorToTrtWeights(const Tensor& tensor, TrtWeightStore* weight_store,
                             TRT_ShapedWeights* weights) {
+  const DataType dtype = tensor.dtype();
+
+  // We always convert the integer constants to INT32, since TRT INT8 is for
+  // quantized inference.
+  //
+  // TODO(aaroey): FP16 will remain in half format and is not converted to
+  // FP32, but the converter currently uses all float weights as FP32. Fix
+  // this.
+  const DataType converted_dtype =
+      (dtype == DT_INT16 || dtype == DT_INT8 || dtype == DT_UINT8 ? DT_INT32
+                                                                  : dtype);
+
+  // Verify that the dtype is supported by TensorRT. Otherwise, return an error.
+  nvinfer1::DataType trt_dtype;
+  TF_RETURN_IF_ERROR(ConvertDType(converted_dtype, &trt_dtype));
+
+  if (tensor.NumElements() == 0) {
+    // Return empty weights having converted dtype.
+    *weights = TRT_ShapedWeights(converted_dtype);
+    return Status::OK();
+  }
+
   nvinfer1::Dims weight_dims;
-  TF_RETURN_IF_ERROR(GetTensorDimsWithProtoShape(tensor, tensor_proto_array_len,
-                                                 &weight_dims));
-  *weights = store->GetTempWeights(dtype, weight_dims);
-  void* dst = const_cast<void*>(weights->GetValues());
-  if (tensor_proto_array_len == 1) {
-    std::fill_n((CType*)dst, TrtDimsNumElements(weight_dims),
-                *tensor_proto_array);
+  GetTensorDimsWithProtoShape(tensor, &weight_dims);
+  *weights = weight_store->GetTempWeights(converted_dtype, weight_dims);
+
+  // Copy the tensor directly if the tensor does not require cast to the
+  // supported type.
+  if (converted_dtype == dtype) {
+    char* dst = static_cast<char*>(const_cast<void*>(weights->GetValues()));
+    memcpy(dst, tensor.tensor_data().data(), tensor.TotalBytes());
+    return Status::OK();
+  }
+
+  // Copy tensor elements after casting them to the converted DataType.
+  int32* dst = static_cast<int32*>(const_cast<void*>(weights->GetValues()));
+  if (dtype == DT_INT16) {
+    const int16* src = tensor.flat<int16>().data();
+    std::copy(src, src + tensor.NumElements(), dst);
+  } else if (dtype == DT_INT8) {
+    const int8* src = tensor.flat<int8>().data();
+    std::copy(src, src + tensor.NumElements(), dst);
   } else {
-    memcpy(dst, tensor_proto_array, weights->size_bytes());
+    // dtype can only be DT_UINT8 at this point.
+    TFTRT_CHECK_EQ_TYPE(dtype, DT_UINT8);
+    const uint8* src = tensor.flat<uint8>().data();
+    std::copy(src, src + tensor.NumElements(), dst);
   }
   return Status::OK();
 }
@@ -2766,15 +2792,6 @@ tensorflow::Status ConvertConst(OpConverterParams* params) {
         "Constant node is expected to have empty input list: ",
         node_def.name());
   }
-  TFAttrs attrs(node_def);
-  const DataType dtype = attrs.get<tensorflow::DataType>("dtype");
-  // We always convert the integer constants to kINT32, since TRT kINT8 is for
-  // quantized inference.
-  const DataType converted_dtype =
-      (dtype == DT_INT16 || dtype == DT_INT8 || dtype == DT_UINT8 ? DT_INT32
-                                                                  : dtype);
-  nvinfer1::DataType trt_dtype;
-  TF_RETURN_IF_ERROR(ConvertDType(converted_dtype, &trt_dtype));
 
   // Create shaped weights as output
   const auto& tensor_proto = node_def.attr().at("value").tensor();
@@ -2784,78 +2801,18 @@ tensorflow::Status ConvertConst(OpConverterParams* params) {
                                         node_def.name());
   }
 
-  TRT_ShapedWeights weights(converted_dtype);
-  if (tensor.NumElements() == 0) {
-    // Do nothing.
-  } else if (!tensor_proto.float_val().empty()) {
-    TF_RETURN_IF_ERROR(TfTensorToTrtWeights(
-        converted_dtype, tensor, tensor_proto.float_val().begin(),
-        tensor_proto.float_val_size(), params->weight_store, &weights));
-  } else if (!tensor_proto.int_val().empty()) {
-    TF_RETURN_IF_ERROR(TfTensorToTrtWeights(
-        converted_dtype, tensor, tensor_proto.int_val().begin(),
-        tensor_proto.int_val_size(), params->weight_store, &weights));
-  } else if (!tensor_proto.half_val().empty()) {
-    // TODO(aaroey): implement fp16 conversion.
-    return errors::Unimplemented("fp16 constant is not supported yet.");
-  } else if (!tensor_proto.tensor_content().empty()) {
-    // TODO(aaroey): fp16 will remain in half format and is not converted to
-    // fp32, but the converter currently uses all float weights as fp32. Fix
-    // this.
-    const auto& content = tensor_proto.tensor_content();
-    if (content.size() > 0) {
-      const int dtype_size = tensorflow::DataTypeSize(dtype);
-      if (content.size() % dtype_size != 0) {
-        return errors::FailedPrecondition("Tensor content size ",
-                                          content.size(),
-                                          " is not a multiple of ", dtype_size);
-      }
-      nvinfer1::Dims weights_dim;
-      TF_RETURN_IF_ERROR(GetTensorDimsWithProtoShape(
-          tensor, content.size() / dtype_size, &weights_dim));
-      const int64_t size_bytes = TrtDimsNumElements(weights_dim) * dtype_size;
-      if (content.size() != size_bytes) {
-        return errors::FailedPrecondition(
-            "Tensor size and TensorProto content size mismatch: ", size_bytes,
-            " vs ", content.size());
-      } else if (tensor.NumElements() != content.size() / dtype_size) {
-        return errors::FailedPrecondition(
-            "Tensor elements count and TensorProto content size mismatch: ",
-            tensor.NumElements(), " vs ", content.size() / dtype_size);
-      }
-      weights =
-          params->weight_store->GetTempWeights(converted_dtype, weights_dim);
-      if (dtype_size == tensorflow::DataTypeSize(converted_dtype)) {
-        port::CopyToArray(content, static_cast<char*>(
-                                       const_cast<void*>(weights.GetValues())));
-      } else {
-        // Copy out the weights as original data type.
-        std::vector<uint8_t> temp_weights(content.size());
-        port::CopyToArray(content,
-                          reinterpret_cast<char*>(temp_weights.data()));
-        int32* dst =
-            static_cast<int32*>(const_cast<void*>(weights.GetValues()));
-        // Copy to the weight store as converted data type.
-        if (dtype == DT_INT16) {
-          int16* data = reinterpret_cast<int16*>(temp_weights.data());
-          std::copy(data, data + tensor.NumElements(), dst);
-        } else if (dtype == DT_INT8) {
-          int8* data = reinterpret_cast<int8*>(temp_weights.data());
-          std::copy(data, data + tensor.NumElements(), dst);
-        } else if (dtype == DT_UINT8) {
-          uint8* data = reinterpret_cast<uint8*>(temp_weights.data());
-          std::copy(data, data + tensor.NumElements(), dst);
-        } else {
-          return errors::FailedPrecondition(
-              "Unexpected data type: ", DataTypeString(dtype),
-              " at: ", node_def.name());
-        }
-      }
-    }
-  } else {
-    return errors::Unimplemented("Not supported constant type, at ",
-                                 node_def.name());
+  TFAttrs attrs(node_def);
+  const DataType dtype = attrs.get<tensorflow::DataType>("dtype");
+  if (dtype != tensor.dtype()) {
+    return errors::InvalidArgument("DataType mismatch between attr (",
+                                   DataTypeString(dtype), ") and tensor (",
+                                   DataTypeString(tensor.dtype()), ")");
   }
+
+  TRT_ShapedWeights weights;
+  TF_RETURN_IF_ERROR(
+      TfTensorToTrtWeights(tensor, params->weight_store, &weights));
+
   if (params->outputs != nullptr) {
     params->outputs->push_back(TRT_TensorOrWeights(weights));
   }
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.h b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
index 4ea5775f04..8495a1a19a 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
@@ -197,6 +197,10 @@ class TRT_ShapedWeights {
   // underlying buffer.
   TRT_ShapedWeights(DataType type, nvinfer1::Dims dims, Tensor tensor);
 
+  // All weights should be stored inside TrtWeightStore to make sure lifetime of
+  // all the underlying tensors are available until the engine is built. For
+  // this reason, tensor_ should never be reassigned to a different value that
+  // is not already present in the TrtWeightStore.
   Tensor tensor_;
 
   friend class TrtWeightStore;
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index c8739b46c1..fa6acdb782 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -1122,6 +1122,30 @@ class OpConverterTest : public ::testing::Test {
   std::unordered_map<string, NodeDef> validator_inputs_;
 };
 
+template <typename T>
+void CopyTensorElements(const Tensor& tensor, proto2::RepeatedField<T>* out) {
+  out->Clear();
+  if (tensor.NumElements() == 0) return;
+
+  // TensorProto does not need to have all the elements present and can truncate
+  // trailing elements with the same value for compressed representation. Such
+  // elements are derived based on the tensor shape.
+  const auto flat = tensor.flat<T>();
+  int64 last_index = 0;
+  for (int64 i = 0; i < tensor.NumElements(); ++i) {
+    if (flat(i) != flat(last_index)) {
+      last_index = i;
+    }
+  }
+
+  int num_out_elements = last_index + 1;
+  out->Reserve(num_out_elements);
+  out->AddNAlreadyReserved(num_out_elements);
+  const T* src = flat.data();
+  T* dst = out->mutable_data();
+  std::copy(src, src + num_out_elements, dst);
+}
+
 template <DataType dtype, typename InputCType, typename OutputCType>
 void TestConvertConst(OpConverterTest* test) {
   NodeDef node_def;
@@ -1134,11 +1158,23 @@ void TestConvertConst(OpConverterTest* test) {
                             const std::vector<OutputCType>& expected_value) {
     test->Reset();
 
-    auto& attr = *node_def.mutable_attr();
+    TensorProto* tensor_attr =
+        (*node_def.mutable_attr())["value"].mutable_tensor();
+    tensor_attr->Clear();
+
     if (as_tensor_content) {
-      tensor.AsProtoTensorContent(attr["value"].mutable_tensor());
+      tensor.AsProtoTensorContent(tensor_attr);
     } else {
-      tensor.AsProtoField(attr["value"].mutable_tensor());
+      tensor.shape().AsProto(tensor_attr->mutable_tensor_shape());
+      tensor_attr->set_dtype(tensor.dtype());
+
+      if (tensor.dtype() == DT_FLOAT) {
+        CopyTensorElements<float>(tensor, tensor_attr->mutable_float_val());
+      } else if (tensor.dtype() == DT_INT32) {
+        CopyTensorElements<int32>(tensor, tensor_attr->mutable_int_val());
+      } else {
+        tensor.AsProtoField(tensor_attr);
+      }
     }
     test->RunValidationAndConversion(node_def);
     TRT_TensorOrWeights output;
@@ -1151,8 +1187,7 @@ void TestConvertConst(OpConverterTest* test) {
   {
     // By default empty tensor will pick DT_FLOAT as data type and we fix it
     // here.
-    attr["value"].mutable_tensor()->set_dtype(dtype);
-    Tensor t;  // Empty tensor.
+    Tensor t(dtype);  // Empty tensor.
     reset_and_test(t, false, {}, {});
   }
   {
@@ -1171,6 +1206,22 @@ void TestConvertConst(OpConverterTest* test) {
     reset_and_test(t, false, {2, 3}, {1, 2, 3, 4, 5, 6});
     reset_and_test(t, true, {2, 3}, {1, 2, 3, 4, 5, 6});
   }
+  {
+    // Set all tensor elements to the same value. Such tensors are encoded
+    // using a single element list in tensor proto.
+    Tensor t = ::tensorflow::test::AsTensor<InputCType>({1, 1, 1, 1, 1, 1},
+                                                        TensorShape({2, 3}));
+    reset_and_test(t, false, {2, 3}, {1, 1, 1, 1, 1, 1});
+    reset_and_test(t, true, {2, 3}, {1, 1, 1, 1, 1, 1});
+  }
+  {
+    // Set trailing tensor elements to the same value. Such tensors are
+    // encoded by truncating all equal elements except the first one.
+    Tensor t = ::tensorflow::test::AsTensor<InputCType>({2, 2, 1, 1, 1, 1},
+                                                        TensorShape({2, 3}));
+    reset_and_test(t, false, {2, 3}, {2, 2, 1, 1, 1, 1});
+    reset_and_test(t, true, {2, 3}, {2, 2, 1, 1, 1, 1});
+  }
 }
 
 TEST_F(OpConverterTest, ConvertConst) {
-- 
GitLab


From 799ec6e59608762d98009db256844ed589b4d6dd Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne <reedwm@google.com>
Date: Thu, 17 Jan 2019 17:38:34 -0800
Subject: [PATCH 0928/2345] Fix flaky conv_ops_test

PiperOrigin-RevId: 229849889
---
 tensorflow/python/kernel_tests/BUILD            | 2 --
 tensorflow/python/kernel_tests/conv_ops_test.py | 9 +++++----
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 40bf4aedd0..aef4354ee0 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -2754,8 +2754,6 @@ cuda_py_test(
     ],
     shard_count = 4,
     tags = [
-        # TODO(b/118887316): Re-enable this test in Kokoro.
-        "no_oss",
         "optonly",  # times out
     ],
     xla_enable_strict_auto_jit = True,
diff --git a/tensorflow/python/kernel_tests/conv_ops_test.py b/tensorflow/python/kernel_tests/conv_ops_test.py
index 7ff1a61e47..390a547fb6 100644
--- a/tensorflow/python/kernel_tests/conv_ops_test.py
+++ b/tensorflow/python/kernel_tests/conv_ops_test.py
@@ -256,7 +256,7 @@ class Conv2DTest(test.TestCase):
       tensors.append(_SetupVal(data_format, use_gpu))
     values = self.evaluate(tensors)
     for i in range(1, len(values)):
-      self.assertAllClose(values[0], values[i], rtol=1e-5, atol=1e-5)
+      self.assertAllClose(values[0], values[i], rtol=1e-3, atol=1e-3)
 
   def _ComputeReferenceDilatedConv(self, tensor_in_sizes, filter_in_sizes,
                                    stride, dilation, padding, data_format,
@@ -297,7 +297,7 @@ class Conv2DTest(test.TestCase):
     return expected, computed
 
   def _VerifyDilatedConvValues(self, tensor_in_sizes, filter_in_sizes, strides,
-                               padding, dilations):
+                               padding, dilations, rtol=1e-4):
     expected_results = []
     computed_results = []
     for data_format, use_gpu in GetTestConfigs():
@@ -313,7 +313,7 @@ class Conv2DTest(test.TestCase):
         tf_logging.debug("expected = %s", e_value)
         tf_logging.debug("actual = %s", c_value)
         self.assertAllClose(
-            e_value.flatten(), c_value.flatten(), atol=tolerance, rtol=1e-4)
+            e_value.flatten(), c_value.flatten(), atol=tolerance, rtol=rtol)
 
   def _VerifyValues(self,
                     tensor_in_sizes,
@@ -2891,7 +2891,8 @@ def GetInceptionFwdDilatedConvTest(input_size, filter_size, stride, padding):
           filter_in_sizes=filter_size,
           strides=[stride, stride],
           dilations=[2, 2],
-          padding=padding)
+          padding=padding,
+          rtol=5e-4)
 
   return Test
 
-- 
GitLab


From 4e8bf7f9ca79e085c37c5d2d3a32c7f409faf999 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 17:40:34 -0800
Subject: [PATCH 0929/2345] Fix for CL 229790895.

PiperOrigin-RevId: 229850165
---
 third_party/gpus/cuda_configure.bzl         | 2 +-
 third_party/tensorrt/tensorrt_configure.bzl | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl
index 7a6c7f4fe5..d1326333d3 100644
--- a/third_party/gpus/cuda_configure.bzl
+++ b/third_party/gpus/cuda_configure.bzl
@@ -1515,7 +1515,7 @@ def _create_remote_cuda_repository(repository_ctx, remote_config_repo):
   )
   repository_ctx.template(
       "crosstool/BUILD",
-      Label(remote_config_repo + "/crosstool/BUILD"),
+      Label(remote_config_repo + "/crosstool:BUILD"),
       {},
   )
 
diff --git a/third_party/tensorrt/tensorrt_configure.bzl b/third_party/tensorrt/tensorrt_configure.bzl
index 10e243fa84..22fd80e91a 100644
--- a/third_party/tensorrt/tensorrt_configure.bzl
+++ b/third_party/tensorrt/tensorrt_configure.bzl
@@ -142,7 +142,7 @@ def _tensorrt_configure_impl(repository_ctx):
   if _TF_TENSORRT_CONFIG_REPO in repository_ctx.os.environ:
     # Forward to the pre-configured remote repository.
     remote_config_repo = repository_ctx.os.environ[_TF_TENSORRT_CONFIG_REPO]
-    repository_ctx.template("BUILD", Label(remote_config_repo + "/BUILD"), {})
+    repository_ctx.template("BUILD", Label(remote_config_repo + ":BUILD"), {})
     # Set up config file.
     _tpl(repository_ctx, "build_defs.bzl", {"%{if_tensorrt}": "if_true"})
     return
-- 
GitLab


From f028498d0d1c2f799a55edf7d47bece78ebdff4d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 18:41:14 -0800
Subject: [PATCH 0930/2345] Fix for CL 229790895.

PiperOrigin-RevId: 229856980
---
 third_party/gpus/cuda_configure.bzl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl
index d1326333d3..5c9873f334 100644
--- a/third_party/gpus/cuda_configure.bzl
+++ b/third_party/gpus/cuda_configure.bzl
@@ -1510,7 +1510,7 @@ def _create_remote_cuda_repository(repository_ctx, remote_config_repo):
   )
   repository_ctx.template(
       "cuda/BUILD",
-      Label(remote_config_repo + "/cuda/BUILD"),
+      Label(remote_config_repo + "/cuda:BUILD"),
       {},
   )
   repository_ctx.template(
-- 
GitLab


From a905c626ca971c4258770b37082b919ba2d9715b Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne <reedwm@google.com>
Date: Thu, 17 Jan 2019 18:46:12 -0800
Subject: [PATCH 0931/2345] Disable broken tests

PiperOrigin-RevId: 229857529
---
 tensorflow/contrib/eager/python/examples/revnet/BUILD  | 1 +
 tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD | 1 +
 2 files changed, 2 insertions(+)

diff --git a/tensorflow/contrib/eager/python/examples/revnet/BUILD b/tensorflow/contrib/eager/python/examples/revnet/BUILD
index 0d85bf63ad..d05270089f 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/BUILD
+++ b/tensorflow/contrib/eager/python/examples/revnet/BUILD
@@ -87,6 +87,7 @@ cuda_py_test(
         "//tensorflow:tensorflow_py",
     ],
     tags = [
+        "no_oss",  # b/123045964
         "optonly",
     ],
 )
diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD b/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD
index f9bf82a7d8..5d48ee4a5f 100644
--- a/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD
+++ b/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD
@@ -25,6 +25,7 @@ cuda_py_test(
         "//tensorflow/contrib/eager/python:tfe",
         "//tensorflow:tensorflow_py",
     ],
+    tags = ["no_oss"],  # b/123045964
 )
 
 cuda_py_test(
-- 
GitLab


From c10447d959d07a51a321b622e7a8da6b61768651 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 19:01:05 -0800
Subject: [PATCH 0932/2345] Allow easier re-running of toco on already
 quantized, mixed-bit-depth tflite models, by:   - fixing a bug whereby
 PropagateArrayDataTypes was brutally propagating     the data type of its
 first input onto all outputs;   - not letting QUANTIZED_UINT8 override
 existing non-8bit quantized data     types.

PiperOrigin-RevId: 229858747
---
 .../graph_transformations/identify_lstm.cc    | 22 +++++++++++++++++--
 .../propagate_array_data_types.cc             |  8 +++++++
 tensorflow/lite/toco/toco_tooling.cc          | 17 ++++++++++++++
 3 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/toco/graph_transformations/identify_lstm.cc b/tensorflow/lite/toco/graph_transformations/identify_lstm.cc
index 089ecee959..65dbb8a176 100644
--- a/tensorflow/lite/toco/graph_transformations/identify_lstm.cc
+++ b/tensorflow/lite/toco/graph_transformations/identify_lstm.cc
@@ -147,12 +147,26 @@ bool MatchOperatorInputs(const Operator& op, const Model& model,
   if (final_output_mul->type != OperatorType::kMul) {
     return ::tensorflow::Status::OK();
   }
+  // final_output_mul->outputs[0] would be one of the two outputs of our
+  // LstmCell. Exit if it does not already have a data type.
+  // We won't be able to propagate data types through a fused LstmCell.
+  if (model->GetArray(final_output_mul->outputs[0]).data_type ==
+      ArrayDataType::kNone) {
+    return ::tensorflow::Status::OK();
+  }
   Operator *state_output_tanh, *fc_output_sig;
   if (!MatchOperatorInputs(*final_output_mul, *model, OperatorType::kTanh,
                            &state_output_tanh, OperatorType::kLogistic,
                            &fc_output_sig)) {
     return ::tensorflow::Status::OK();
   }
+  // state_output_tanh->inputs[0] would be one of the two outputs of our
+  // LstmCell. Exit if it does not already have a data type.
+  // We won't be able to propagate data types through a fused LstmCell.
+  if (model->GetArray(state_output_tanh->inputs[0]).data_type ==
+      ArrayDataType::kNone) {
+    return ::tensorflow::Status::OK();
+  }
 
   // State output TanH
   // (We don't count an operator as ID'd until we verify it has the correct
@@ -262,11 +276,15 @@ bool MatchOperatorInputs(const Operator& op, const Model& model,
       lstm_cell_op->outputs[LstmCellOperator::ACTIV_OUTPUT]));
   const string& concat_temp_array_name =
       AvailableArrayName(*model, base_name + "concat_temp");
-  model->GetOrCreateArray(concat_temp_array_name);
+  auto& concat_temp_array = model->GetOrCreateArray(concat_temp_array_name);
+  concat_temp_array.data_type =
+      model->GetArray(concat_inputs->outputs[0]).data_type;
   lstm_cell_op->outputs[LstmCellOperator::CONCAT_TEMP] = concat_temp_array_name;
   const string& activ_temp_array_name =
       AvailableArrayName(*model, base_name + "activ_temp");
-  model->GetOrCreateArray(activ_temp_array_name);
+  auto& activ_temp_array = model->GetOrCreateArray(activ_temp_array_name);
+  activ_temp_array.data_type =
+      model->GetArray(fully_connected->outputs[0]).data_type;
   lstm_cell_op->outputs[LstmCellOperator::ACTIV_TEMP] = activ_temp_array_name;
   AddMessageF("Created temp outputs %s and %s on operator %s",
               concat_temp_array_name, activ_temp_array_name,
diff --git a/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc b/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
index 7aec6728da..d546ff4ec0 100644
--- a/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
+++ b/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
@@ -266,6 +266,14 @@ void SetDataTypeForAllOutputs(Model* model, Operator* op,
       model->GetArray(op->outputs[1]).data_type = unique_op->idx_out_type;
       break;
     }
+    case OperatorType::kLstmCell: {
+      // It's tricky to propagate data types through a LstmCell, as that has
+      // multiple inputs and outputs, and there are quantized cases with
+      // mixed (8bit vs 16bit) cases. Fortunately, that should never be needed,
+      // as the data formats, such as TFLITE, that have LstmCell nodes, also
+      // have data type fields for all their arrays.
+      break;
+    }
     default: {
       // These operators produce outputs with the same type as their 1st input
       CHECK_GT(op->inputs.size(), 0);
diff --git a/tensorflow/lite/toco/toco_tooling.cc b/tensorflow/lite/toco/toco_tooling.cc
index 55a454e66d..61dca56d23 100644
--- a/tensorflow/lite/toco/toco_tooling.cc
+++ b/tensorflow/lite/toco/toco_tooling.cc
@@ -178,6 +178,23 @@ void SetFinalDataTypeOnInputs(const TocoFlags& toco_flags, Model* model) {
       // Ignore non-real data types.
       continue;
     }
+    // The enum value QUANTIZED_UINT8 for --inference_type and
+    // --inference_input_type has long meant just 'QUANTIZED', being used as
+    // well in mixed 8-bit / 16-bit quantized models. However,
+    // ConvertIODataTypeToArrayDataType still interpretes it as meaning 8bit,
+    // and people have run into issues in the situation where they have an
+    // already mixed 8-bit / 16-bit quantized model in TFLITE format and
+    // want to run it again through toco, without having to re-specify all the
+    // extra array info that was used in the (complicated) process of initially
+    // quantizing that model. In order to have --inference_type=QUANTIZED_UINT8
+    // just work in that case, we implement the logic that when an array is
+    // already quantized, if  --inference_type is quantized (so we're not
+    // asking to dequantize here), no change of quantized data type is to be
+    // recorded.
+    if (array->data_type != toco::ArrayDataType::kFloat &&
+        type != toco::ArrayDataType::kFloat) {
+      continue;
+    }
 
     array->final_data_type = type;
   }
-- 
GitLab


From 2a5abdb77806e06ce7e2820ede6233ca79cf5625 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 19:02:03 -0800
Subject: [PATCH 0933/2345] Nit: Add a missing "\" in lite benchmark
 documentation

PiperOrigin-RevId: 229858857
---
 tensorflow/lite/tools/benchmark/android/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/tools/benchmark/android/README.md b/tensorflow/lite/tools/benchmark/android/README.md
index f5b67e3f79..db82c59acd 100644
--- a/tensorflow/lite/tools/benchmark/android/README.md
+++ b/tensorflow/lite/tools/benchmark/android/README.md
@@ -51,7 +51,7 @@ and can be appended to the `args` string alongside the required `--graph` flag
 args key).
 
 ```
-adb shell am start -S -n
+adb shell am start -S -n \
   org.tensorflow.lite.benchmark/org.tensorflow.lite.benchmark.BenchmarkModelActivity \
   --es args '"--graph=/data/local/tmp/mobilenet_quant_v1_224.tflite --num_threads=4"'
 ```
-- 
GitLab


From 79aeb13673a9107e76622fbc3181a92b398bee28 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 19:16:38 -0800
Subject: [PATCH 0934/2345] Automated rollback of commit
 f132194cf910343d4e1c3aedfad45f7b76fc5f61

PiperOrigin-RevId: 229860333
---
 tensorflow/python/BUILD                 |  1 -
 tensorflow/python/client/session_ref.cc | 31 ++++---------------------
 2 files changed, 4 insertions(+), 28 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index e6f8201fc7..30de1fa440 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -4091,7 +4091,6 @@ cc_library(
         "//tensorflow/core:master_proto_cc",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:replay_log_proto_cc",
-        "@com_google_absl//absl/time",
     ],
 )
 
diff --git a/tensorflow/python/client/session_ref.cc b/tensorflow/python/client/session_ref.cc
index 895624f592..6639cf506e 100644
--- a/tensorflow/python/client/session_ref.cc
+++ b/tensorflow/python/client/session_ref.cc
@@ -18,9 +18,6 @@ limitations under the License.
 #include <memory>
 #include <utility>
 
-#include "absl/time/clock.h"
-#include "absl/time/time.h"
-#include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/io/record_writer.h"
 #include "tensorflow/core/lib/strings/stringprintf.h"
@@ -480,8 +477,6 @@ Status SessionRef::ReleaseCallable(CallableHandle handle) {
   LOG_AND_RUN_OPERATION(ReleaseCallable, handle);
 }
 
-static const absl::Duration kMaxCloseWaitTime = absl::Seconds(60);
-
 Status SessionRef::Close(const RunOptions& run_options) {
   TF_RETURN_IF_ERROR(CheckNotClosed());
   mutex_lock l(run_lock_);
@@ -492,17 +487,8 @@ Status SessionRef::Close(const RunOptions& run_options) {
     status = session_->Close(run_options);
   }
   session_.reset();
-
-  // Wait no more than kMaxCloseWaitTime for all pending operations to finish.
-  absl::Time start_time = absl::Now();
-  absl::Duration wait_time = start_time + kMaxCloseWaitTime - absl::Now();
-  while (run_count_ > 0 && wait_time > absl::ZeroDuration()) {
-    if (run_finished_.wait_for(l, absl::ToChronoMilliseconds(wait_time)) ==
-        std::cv_status::timeout) {
-      status.Update(errors::DeadlineExceeded("Timeout closing session."));
-      return status;
-    }
-    wait_time = start_time + kMaxCloseWaitTime - absl::Now();
+  while (run_count_ > 0) {
+    run_finished_.wait(l);
   }
   return status;
 }
@@ -517,17 +503,8 @@ Status SessionRef::Close() {
     status = session_->Close();
   }
   session_.reset();
-
-  // Wait no more than kMaxCloseWaitTime for all pending operations to finish.
-  absl::Time start_time = absl::Now();
-  absl::Duration wait_time = start_time + kMaxCloseWaitTime - absl::Now();
-  while (run_count_ > 0 && wait_time > absl::ZeroDuration()) {
-    if (run_finished_.wait_for(l, absl::ToChronoMilliseconds(wait_time)) ==
-        std::cv_status::timeout) {
-      status.Update(errors::DeadlineExceeded("Timeout closing session."));
-      return status;
-    }
-    wait_time = start_time + kMaxCloseWaitTime - absl::Now();
+  while (run_count_ > 0) {
+    run_finished_.wait(l);
   }
   return status;
 }
-- 
GitLab


From a33ceb20cfa9af339e79ce814d3eb849b68e8398 Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Thu, 17 Jan 2019 19:38:57 -0800
Subject: [PATCH 0935/2345] Keras Dense layer dtype changes.

* Dense layer now explicitly requires floating point inputs (better error messages than before).
* Dense layer call() now casts inputs to self.dtype.

PiperOrigin-RevId: 229862412
---
 tensorflow/python/keras/engine/base_layer.py | 2 ++
 tensorflow/python/keras/layers/core.py       | 6 ++++++
 tensorflow/python/keras/layers/core_test.py  | 8 ++++++++
 3 files changed, 16 insertions(+)

diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index 7bfc301239..f63fb4e694 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -307,6 +307,8 @@ class Layer(checkpointable.Checkpointable):
     if dtype is None:
       dtype = self.dtype or backend.floatx()
     dtype = dtypes.as_dtype(dtype)
+    if self._dtype is None:
+      self._dtype = dtype.base_dtype.name
     initializer = initializers.get(initializer)
     regularizer = regularizers.get(regularizer)
     constraint = constraints.get(constraint)
diff --git a/tensorflow/python/keras/layers/core.py b/tensorflow/python/keras/layers/core.py
index dfbab80be3..fdc0a76877 100644
--- a/tensorflow/python/keras/layers/core.py
+++ b/tensorflow/python/keras/layers/core.py
@@ -27,6 +27,7 @@ import numpy as np
 
 from tensorflow.python.eager import context
 from tensorflow.python.framework import common_shapes
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.keras import activations
@@ -932,6 +933,10 @@ class Dense(Layer):
     self.input_spec = InputSpec(min_ndim=2)
 
   def build(self, input_shape):
+    dtype = dtypes.as_dtype(self.dtype or K.floatx())
+    if not (dtype.is_floating or dtype.is_complex):
+      raise TypeError('Unable to build `Dense` layer with non-floating point '
+                      'dtype %s' % (dtype,))
     input_shape = tensor_shape.TensorShape(input_shape)
     if tensor_shape.dimension_value(input_shape[-1]) is None:
       raise ValueError('The last dimension of the inputs to `Dense` '
@@ -972,6 +977,7 @@ class Dense(Layer):
         output_shape = shape[:-1] + [self.units]
         outputs.set_shape(output_shape)
     else:
+      inputs = math_ops.cast(inputs, self.dtype)
       outputs = gen_math_ops.mat_mul(inputs, self.kernel)
     if self.use_bias:
       outputs = nn.bias_add(outputs, self.bias)
diff --git a/tensorflow/python/keras/layers/core_test.py b/tensorflow/python/keras/layers/core_test.py
index 9df40f806f..d61c8e3d08 100644
--- a/tensorflow/python/keras/layers/core_test.py
+++ b/tensorflow/python/keras/layers/core_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python import keras
 from tensorflow.python.eager import context
+from tensorflow.python.framework import ops
 from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.ops import math_ops
@@ -283,6 +284,13 @@ class CoreLayersTest(keras_parameterized.TestCase):
     testing_utils.layer_test(
         keras.layers.Dense, kwargs={'units': 3}, input_shape=(3, 4, 5, 2))
 
+  def test_dense_dtype(self):
+    inputs = ops.convert_to_tensor(
+        np.random.randint(low=0, high=7, size=(2, 2)))
+    layer = keras.layers.Dense(5, dtype='float32')
+    outputs = layer(inputs)
+    self.assertEqual(outputs.dtype, 'float32')
+
   def test_dense_regularization(self):
     layer = keras.layers.Dense(
         3,
-- 
GitLab


From 0be3b45806074bc386d3e72113dc48f2a332abe6 Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Thu, 17 Jan 2019 20:14:42 -0800
Subject: [PATCH 0936/2345] Update the code and remove the proto header

---
 tensorflow/core/kernels/data/dataset_utils.cc |  32 +-----
 tensorflow/core/kernels/data/dataset_utils.h  |  17 +--
 .../core/kernels/data/dataset_utils_test.cc   |  20 +---
 tensorflow/core/kernels/data/iterator_ops.cc  | 104 ------------------
 4 files changed, 16 insertions(+), 157 deletions(-)

diff --git a/tensorflow/core/kernels/data/dataset_utils.cc b/tensorflow/core/kernels/data/dataset_utils.cc
index 67a0b1165a..8f3b1f7066 100644
--- a/tensorflow/core/kernels/data/dataset_utils.cc
+++ b/tensorflow/core/kernels/data/dataset_utils.cc
@@ -159,26 +159,6 @@ bool VariantTensorDataReader::Contains(StringPiece key) {
   return map_.find(string(key)) != map_.end();
 }
 
-void VariantTensorDataReader::PreProcess() {
-  string metadata;
-  data_->get_metadata(&metadata);
-  IteratorStateMetadata proto;
-  if (!proto.ParseFromString(metadata)) {
-    status_ = errors::Internal("Error parsing IteratorStateMetadata.");
-    return;
-  }
-  size_t num_entries = proto.keys_size();
-  if (num_entries != data_->tensors_size()) {
-    status_ =
-        errors::InvalidArgument("Unmatched number of keys and tensors: ",
-                                num_entries, " vs. ", data_->tensors_size());
-    return;
-  }
-  for (size_t i = 0; i < num_entries; i++) {
-    map_[proto.keys(i)] = i;
-  }
-}
-
 template <typename T>
 Status VariantTensorDataReader::ReadScalarInternal(StringPiece key, T* val) {
   if (map_.find(string(key)) == map_.end()) {
@@ -213,8 +193,8 @@ Status VariantTensorDataWriter::WriteTensor(StringPiece key,
 
 Status VariantTensorDataWriter::Flush() {
   string metadata;
-  if (!metadata_proto_.SerializeToString(&metadata)) {
-    return errors::Internal("Unable to serialize IteratorStateMetadata.");
+  for (size_t i = 0; i < keys_.size(); ++i) {
+    strings::StrAppend(&metadata, kDelimiter, keys_[i]);
   }
   data_->set_metadata(metadata);
   return Status::OK();
@@ -230,12 +210,8 @@ Status VariantTensorDataWriter::WriteScalarInternal(StringPiece key,
 
 Status VariantTensorDataWriter::WriteTensorInternal(StringPiece key,
                                                     const Tensor& val) {
-  // Write key to the metadata proto. This gets written to `data_`
-  // when `Flush()` is called. We do this lazily to avoid multiple
-  // serialization calls.
-  metadata_proto_.add_keys(string(key));
-
-  // Update tensors.
+  DCHECK_EQ(key.find(kDelimiter), string::npos);
+  keys_.push_back(string(key));
   *(data_->add_tensors()) = val;
   return Status::OK();
 }
diff --git a/tensorflow/core/kernels/data/dataset_utils.h b/tensorflow/core/kernels/data/dataset_utils.h
index b2a19ac869..b960671594 100644
--- a/tensorflow/core/kernels/data/dataset_utils.h
+++ b/tensorflow/core/kernels/data/dataset_utils.h
@@ -16,7 +16,6 @@ limitations under the License.
 #define TENSORFLOW_CORE_KERNELS_DATA_DATASET_UTILS_H_
 
 #include "tensorflow/core/framework/dataset.h"
-#include "tensorflow/core/framework/iterator.pb.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/kernels/data/captured_function.h"
 
@@ -58,16 +57,22 @@ Status VerifyTypesMatch(const DataTypeVector& expected,
 Status VerifyShapesCompatible(const std::vector<PartialTensorShape>& expected,
                               const std::vector<PartialTensorShape>& received);
 
+constexpr char kDelimiter[] = "@@";
+
 // Helper class for reading data from a VariantTensorData object.
 class VariantTensorDataReader : public IteratorStateReader {
  public:
   explicit VariantTensorDataReader(const VariantTensorData* data)
       : data_(data) {
-    PreProcess();
+    string metadata;
+    data_->get_metadata(&metadata);
+    auto keys = str_util::Split(metadata, kDelimiter, str_util::SkipEmpty());
+    for (size_t i = 0; i < keys.size(); ++i) {
+      map_[keys[i]] = i;
+    }
   }
 
-  // Returns OK iff the initialization was successful, i.e.,
-  // pre-processing did not have errors.
+  // Returns OK iff the initialization was successful.
   Status status() const;
   Status ReadScalar(StringPiece key, int64* val) override;
   Status ReadScalar(StringPiece key, string* val) override;
@@ -75,7 +80,6 @@ class VariantTensorDataReader : public IteratorStateReader {
   bool Contains(StringPiece key) override;
 
  private:
-  void PreProcess();
   template <typename T>
   Status ReadScalarInternal(StringPiece key, T* val);
   Status ReadTensorInternal(StringPiece key, Tensor* val);
@@ -102,8 +106,7 @@ class VariantTensorDataWriter : public IteratorStateWriter {
   Status WriteTensorInternal(StringPiece key, const Tensor& val);
   VariantTensorData* data_;
 
-  // TODO(srbs): Set the version string.
-  IteratorStateMetadata metadata_proto_;
+  std::vector<string> keys_;
 };
 
 }  // namespace data
diff --git a/tensorflow/core/kernels/data/dataset_utils_test.cc b/tensorflow/core/kernels/data/dataset_utils_test.cc
index b503e58c1b..ba81760fc0 100644
--- a/tensorflow/core/kernels/data/dataset_utils_test.cc
+++ b/tensorflow/core/kernels/data/dataset_utils_test.cc
@@ -46,7 +46,6 @@ TEST(DatasetUtilsTest, VariantTensorDataRoundtrip) {
   VariantTensorData data;
   VariantTensorDataWriter writer(&data);
   TF_ASSERT_OK(writer.WriteScalar("Int64", 24));
-  TF_ASSERT_OK(writer.WriteScalar("", "Empty_Key"));
   Tensor input_tensor(DT_FLOAT, {1});
   input_tensor.flat<float>()(0) = 2.0f;
   TF_ASSERT_OK(writer.WriteTensor("Tensor", input_tensor));
@@ -57,9 +56,6 @@ TEST(DatasetUtilsTest, VariantTensorDataRoundtrip) {
   int64 val_int64;
   TF_ASSERT_OK(reader.ReadScalar("Int64", &val_int64));
   EXPECT_EQ(val_int64, 24);
-  string val_string;
-  TF_ASSERT_OK(reader.ReadScalar("", &val_string));
-  EXPECT_EQ(val_string, "Empty_Key");
   Tensor val_tensor;
   TF_ASSERT_OK(reader.ReadTensor("Tensor", &val_tensor));
   EXPECT_EQ(input_tensor.NumElements(), val_tensor.NumElements());
@@ -68,6 +64,8 @@ TEST(DatasetUtilsTest, VariantTensorDataRoundtrip) {
 
 TEST(DatasetUtilsTest, VariantTensorDataNonExistentKey) {
   VariantTensorData data;
+  strings::StrAppend(&data.metadata_, "key1", "@@");
+  data.tensors_.push_back(Tensor(DT_INT64, {1}));
   VariantTensorDataReader reader(&data);
   TF_ASSERT_OK(reader.status());
   int64 val_int64;
@@ -81,20 +79,6 @@ TEST(DatasetUtilsTest, VariantTensorDataNonExistentKey) {
             reader.ReadTensor("NonExistentKey", &val_tensor).code());
 }
 
-TEST(DatasetUtilsTest, VariantTensorDataInvalidMetadata) {
-  VariantTensorData data;
-  data.metadata_ = "Invalid Metadata";
-  VariantTensorDataReader reader(&data);
-  EXPECT_EQ(error::INTERNAL, reader.status().code());
-}
-
-TEST(DatasetUtilsTest, VariantTensorDataUnmatchedKeys) {
-  VariantTensorData data;
-  data.tensors_.push_back(Tensor(DT_INT64, {1}));
-  VariantTensorDataReader reader(&data);
-  EXPECT_EQ(error::INVALID_ARGUMENT, reader.status().code());
-}
-
 }  // namespace
 }  // namespace data
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/data/iterator_ops.cc b/tensorflow/core/kernels/data/iterator_ops.cc
index da497a5f72..8f64639a8b 100644
--- a/tensorflow/core/kernels/data/iterator_ops.cc
+++ b/tensorflow/core/kernels/data/iterator_ops.cc
@@ -276,110 +276,6 @@ class IteratorResource : public ResourceBase {
 
 namespace {
 
-constexpr char kDelimiter[] = "@@";
-
-// Helper class for reading data from a VariantTensorData object.
-class VariantTensorDataReader : public IteratorStateReader {
- public:
-  explicit VariantTensorDataReader(const VariantTensorData* data)
-      : data_(data) {
-    string metadata;
-    data_->get_metadata(&metadata);
-    auto keys = str_util::Split(metadata, kDelimiter, str_util::SkipEmpty());
-    for (size_t i = 0; i < keys.size(); ++i) {
-      map_[keys[i]] = i;
-    }
-  }
-
-  // Returns OK iff the initialization was successful, i.e.,
-  // pre-processing did not have errors.
-  Status status() const { return status_; }
-
-  Status ReadScalar(StringPiece key, int64* val) override {
-    return ReadScalarInternal(key, val);
-  }
-
-  Status ReadScalar(StringPiece key, string* val) override {
-    return ReadScalarInternal(key, val);
-  }
-
-  Status ReadTensor(StringPiece key, Tensor* val) override {
-    return ReadTensorInternal(key, val);
-  }
-
-  bool Contains(StringPiece key) override {
-    return map_.find(string(key)) != map_.end();
-  }
-
- private:
-  template <typename T>
-  Status ReadScalarInternal(StringPiece key, T* val) {
-    if (map_.find(string(key)) == map_.end()) {
-      return errors::NotFound(key);
-    }
-    *val = data_->tensors(map_[string(key)]).scalar<T>()();
-    return Status::OK();
-  }
-
-  Status ReadTensorInternal(StringPiece key, Tensor* val) {
-    if (map_.find(string(key)) == map_.end()) {
-      return errors::NotFound(key);
-    }
-    *val = data_->tensors(map_[string(key)]);
-    return Status::OK();
-  }
-
-  std::map<string, size_t> map_;
-  const VariantTensorData* data_;  // Not owned.
-  Status status_;
-};
-
-// Helper class for writing data to a VariantTensorData object.
-class VariantTensorDataWriter : public IteratorStateWriter {
- public:
-  // Does not take ownership of data.
-  explicit VariantTensorDataWriter(VariantTensorData* data) : data_(data) {}
-
-  Status WriteScalar(StringPiece key, const int64 val) override {
-    return WriteScalarInternal(key, val);
-  }
-
-  Status WriteScalar(StringPiece key, const string& val) override {
-    return WriteScalarInternal(key, val);
-  }
-
-  Status WriteTensor(StringPiece key, const Tensor& val) override {
-    return WriteTensorInternal(key, val);
-  }
-
-  Status Flush() {
-    string metadata;
-    for (size_t i = 0; i < keys_.size(); ++i) {
-      strings::StrAppend(&metadata, kDelimiter, keys_[i]);
-    }
-    data_->set_metadata(metadata);
-    return Status::OK();
-  }
-
- private:
-  template <typename T>
-  Status WriteScalarInternal(StringPiece key, const T& val) {
-    Tensor val_t = Tensor(DataTypeToEnum<T>::v(), TensorShape({}));
-    val_t.scalar<T>()() = val;
-    return WriteTensorInternal(key, val_t);
-  }
-
-  Status WriteTensorInternal(StringPiece key, const Tensor& val) {
-    DCHECK_EQ(key.find(kDelimiter), string::npos);
-    keys_.push_back(string(key));
-    *(data_->add_tensors()) = val;
-    return Status::OK();
-  }
-
-  VariantTensorData* data_;
-  std::vector<string> keys_;
-};
-
 // Wrapper for encoding/decoding the iterator state stored in a Variant tensor.
 // The get() method returns an IteratorStateReader which can be used
 // to restore iterator state.
-- 
GitLab


From a45e7253c19ec04cbbc116ceda4052a018d9cc09 Mon Sep 17 00:00:00 2001
From: Blake Hechtman <blakehechtman@google.com>
Date: Thu, 17 Jan 2019 21:16:13 -0800
Subject: [PATCH 0937/2345] [XLA] Support no dimension numbers set in Dot.

PiperOrigin-RevId: 229870615
---
 tensorflow/compiler/xla/service/BUILD         |  1 +
 .../compiler/xla/service/dot_decomposer.cc    |  5 ++
 .../compiler/xla/service/hlo_cost_analysis.cc | 21 ++---
 .../xla/service/hlo_cost_analysis_test.cc     | 82 +++++++++++++++++++
 .../xla/service/hlo_evaluator_typed_visitor.h | 18 ++--
 .../compiler/xla/tests/dot_operation_test.cc  | 30 +++++++
 6 files changed, 137 insertions(+), 20 deletions(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index e39e17c110..2b2c32a681 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -2105,6 +2105,7 @@ tf_cc_test(
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:test_helpers",
+        "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client",
         "//tensorflow/compiler/xla/client:client_library",
         "//tensorflow/compiler/xla/client:local_client",
diff --git a/tensorflow/compiler/xla/service/dot_decomposer.cc b/tensorflow/compiler/xla/service/dot_decomposer.cc
index 855424067d..e8bc6d0571 100644
--- a/tensorflow/compiler/xla/service/dot_decomposer.cc
+++ b/tensorflow/compiler/xla/service/dot_decomposer.cc
@@ -301,6 +301,11 @@ StatusOr<bool> DotDecomposer::Run(HloModule* module) {
         non_canonical_dots.push_back(instruction);
         continue;
       }
+      if (dnums.lhs_batch_dimensions().empty() &&
+          dnums.lhs_contracting_dimensions().empty()) {
+        non_canonical_dots.push_back(instruction);
+        continue;
+      }
       if (dnums.lhs_batch_dimensions().empty()) {
         continue;
       }
diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
index c4b4fa62dd..76fd402b2c 100644
--- a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
+++ b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
@@ -237,24 +237,17 @@ Status HloCostAnalysis::HandleDomain(const HloInstruction* domain) {
 
 Status HloCostAnalysis::HandleDot(const HloInstruction* dot) {
   const Shape& lhs_shape = dot->operand(0)->shape();
-  const Shape& rhs_shape = dot->operand(1)->shape();
+  const Shape& dot_shape = dot->shape();
   const DotDimensionNumbers& dnums = dot->dot_dimension_numbers();
   // Count of elements along the reduction dimension (last dimension for the
   // rhs).
-  int64 reduction_width =
-      lhs_shape.dimensions(dnums.lhs_contracting_dimensions(0));
-  // First divide by reduction width before multiplying by rhs elements to avoid
-  // overflow.
-  int64 fma_count;
-  if (reduction_width == 0) {
-    fma_count = 0;
-  } else {
-    fma_count = (ShapeUtil::ElementsIn(lhs_shape) / reduction_width) *
-                ShapeUtil::ElementsIn(rhs_shape);
+  int64 reduction_width = 1;
+  for (auto dim : dnums.lhs_contracting_dimensions()) {
+    reduction_width *= lhs_shape.dimensions(dim);
   }
-
-  // We count an FMA operation as 2 floating point operations.
-  current_properties_[kFlopsKey] = kFmaFlops * fma_count;
+  // Each output elment requires reduction_widht FMA operations.
+  current_properties_[kFlopsKey] =
+      kFmaFlops * ShapeUtil::ElementsIn(dot_shape) * reduction_width;
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis_test.cc b/tensorflow/compiler/xla/service/hlo_cost_analysis_test.cc
index 82dd57d3c6..4d42770ba7 100644
--- a/tensorflow/compiler/xla/service/hlo_cost_analysis_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_cost_analysis_test.cc
@@ -29,6 +29,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/service.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/platform/logging.h"
 
 #include "tensorflow/compiler/xla/statusor.h"
@@ -157,6 +158,87 @@ TEST_F(HloCostAnalysisTest, MatrixMultiply) {
             sizeof(float) * (10 * 5 + 5 * 30 + 10 * 30));
 }
 
+TEST_F(HloCostAnalysisTest, DotGeneral) {
+  XlaBuilder builder("matrix_multiply");
+  auto lhs =
+      Parameter(&builder, 0, ShapeUtil::MakeShape(F32, {10, 5, 5}), "lhs");
+  auto rhs =
+      Parameter(&builder, 1, ShapeUtil::MakeShape(F32, {5, 5, 30}), "rhs");
+  DotDimensionNumbers dnums;
+  dnums.add_lhs_contracting_dimensions(1);
+  dnums.add_lhs_contracting_dimensions(2);
+  dnums.add_rhs_contracting_dimensions(0);
+  dnums.add_rhs_contracting_dimensions(1);
+  DotGeneral(lhs, rhs, dnums);
+
+  // Run HLO cost analysis.
+  auto hlo_module = BuildHloGraph(&builder);
+  HloCostAnalysis analysis(ShapeSize);
+  ASSERT_IS_OK(
+      hlo_module->entry_computation()->root_instruction()->Accept(&analysis));
+
+  // Check the number of computations returned from the analysis (1500 FMAs).
+  EXPECT_EQ(analysis.flop_count(), 2 * 10 * 30 * 5 * 5);
+
+  EXPECT_EQ(analysis.transcendental_count(), 0);
+
+  // Bytes accessed is sum of inputs and output.
+  EXPECT_EQ(analysis.bytes_accessed(),
+            sizeof(float) * (10 * 5 * 5 + 5 * 5 * 30 + 10 * 30));
+}
+
+TEST_F(HloCostAnalysisTest, DotGeneral2) {
+  XlaBuilder builder("matrix_multiply");
+  auto lhs =
+      Parameter(&builder, 0, ShapeUtil::MakeShape(F32, {10, 5, 5}), "lhs");
+  auto rhs =
+      Parameter(&builder, 1, ShapeUtil::MakeShape(F32, {5, 5, 30}), "rhs");
+  DotDimensionNumbers dnums;
+  dnums.add_lhs_contracting_dimensions(1);
+  dnums.add_lhs_batch_dimensions(2);
+  dnums.add_rhs_contracting_dimensions(0);
+  dnums.add_rhs_batch_dimensions(1);
+  DotGeneral(lhs, rhs, dnums);
+
+  // Run HLO cost analysis.
+  auto hlo_module = BuildHloGraph(&builder);
+  HloCostAnalysis analysis(ShapeSize);
+  ASSERT_IS_OK(
+      hlo_module->entry_computation()->root_instruction()->Accept(&analysis));
+
+  // Check the number of computations returned from the analysis (1500 FMAs).
+  EXPECT_EQ(analysis.flop_count(), 2 * 10 * 30 * 5 * 5);
+
+  EXPECT_EQ(analysis.transcendental_count(), 0);
+
+  // Bytes accessed is sum of inputs and output.
+  EXPECT_EQ(analysis.bytes_accessed(),
+            sizeof(float) * (10 * 5 * 5 + 5 * 5 * 30 + 5 * 10 * 30));
+}
+
+TEST_F(HloCostAnalysisTest, DotGeneral3) {
+  XlaBuilder builder("matrix_multiply");
+  auto lhs = Parameter(&builder, 0, ShapeUtil::MakeShape(F32, {10, 5}), "lhs");
+  auto rhs = Parameter(&builder, 1, ShapeUtil::MakeShape(F32, {5, 30}), "rhs");
+  DotDimensionNumbers dnums;
+  DotGeneral(lhs, rhs, dnums);
+
+  // Run HLO cost analysis.
+  auto hlo_module = BuildHloGraph(&builder);
+  HloCostAnalysis analysis(ShapeSize);
+  ASSERT_IS_OK(
+      hlo_module->entry_computation()->root_instruction()->Accept(&analysis));
+
+  // Check the number of computations returned from the analysis (1500 FMAs).
+  EXPECT_EQ(analysis.flop_count(), 2 * 10 * 30 * 5 * 5);
+
+  EXPECT_EQ(analysis.transcendental_count(), 0);
+
+  // Bytes accessed is sum of inputs and output.
+  EXPECT_EQ(analysis.bytes_accessed(),
+            sizeof(float) * (10 * 5 + 5 * 30 + 5 * 5 * 10 * 30));
+}
+
 TEST_F(HloCostAnalysisTest, Map) {
   XlaBuilder builder("map");
   auto input = Parameter(&builder, 0, ShapeUtil::MakeShape(F32, {10}), "in");
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
index 95a78408b0..648c7d0e67 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
@@ -1187,7 +1187,8 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
   }
 
   Status HandleDot(HloInstruction* dot) override {
-    if (parent_->use_fast_path_) {
+    if (dot->dot_dimension_numbers().rhs_contracting_dimensions_size() == 1 &&
+        parent_->use_fast_path_) {
       return HandleDot<ReturnT>(dot);
     }
     return HandleDotSlowPath(dot);
@@ -1349,12 +1350,17 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
                 static_cast<ElementwiseT>(lhs_literal.Get<ReturnT>(lhs_index)) *
                 static_cast<ElementwiseT>(rhs_literal.Get<ReturnT>(rhs_index));
 
-            for (int64 i = accumulate_index_sizes.size() - 1; i >= 0; --i) {
-              int64 value = ++accumulate_index[i];
-              if (value != accumulate_index_sizes[i]) {
-                break;
+            // If there are no contracting dimension accumulate_index_sizes is
+            // empty, do not try to count down from -1 to 0 since it is and
+            // infinite loop.
+            if (!accumulate_index_sizes.empty()) {
+              for (int64 i = accumulate_index_sizes.size() - 1; i >= 0; --i) {
+                int64 value = ++accumulate_index[i];
+                if (value != accumulate_index_sizes[i]) {
+                  break;
+                }
+                accumulate_index[i] = 0;
               }
-              accumulate_index[i] = 0;
             }
           }
 
diff --git a/tensorflow/compiler/xla/tests/dot_operation_test.cc b/tensorflow/compiler/xla/tests/dot_operation_test.cc
index 33d816292c..4576a04ec3 100644
--- a/tensorflow/compiler/xla/tests/dot_operation_test.cc
+++ b/tensorflow/compiler/xla/tests/dot_operation_test.cc
@@ -1189,5 +1189,35 @@ ENTRY %test {
   EXPECT_TRUE(RunAndCompare(hlo_string, ErrorSpec{1e-3, 1e-3}));
 }
 
+XLA_TEST_F(DotOperationTextTest, DotWithNoDnums) {
+  absl::string_view hlo_string =
+      R"(
+HloModule DotWithNoDnums
+
+ENTRY %test {
+  %lhs = f32[2,3]{1,0} parameter(0)
+  %rhs = f32[4,5]{1,0} parameter(1)
+  ROOT %dot = f32[2,3,4,5]{3,2,1,0} dot(%lhs, %rhs)
+}
+)";
+
+  EXPECT_TRUE(RunAndCompare(hlo_string, ErrorSpec{1e-3, 1e-3}));
+}
+
+XLA_TEST_F(DotOperationTextTest, Einsum) {
+  absl::string_view hlo_string =
+      R"(
+HloModule Einsum
+
+ENTRY %test {
+  %lhs = f32[8,64,96]{2,1,0} parameter(0)
+  %rhs = f32[96,32,4]{2,1,0} parameter(1)
+  ROOT %dot = f32[8,64,32,4]{3,2,1,0}  dot(%lhs, %rhs), lhs_contracting_dims={2}, rhs_contracting_dims={0}
+}
+)";
+
+  EXPECT_TRUE(RunAndCompare(hlo_string, ErrorSpec{4e-3, 4e-3}));
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From a3335403457d1005f5bd556f740a2dd4324edf60 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 23:00:29 -0800
Subject: [PATCH 0938/2345] Split ragged_array_ops.py into multiple targets to
 avoid dependency cycles. Specifically, refactor out ragged gather,
 batch_gather, where, concat and stack ops into their own file to avoid
 dependency cycles.

PiperOrigin-RevId: 229878011
---
 tensorflow/python/ops/ragged/BUILD            |  89 ++-
 tensorflow/python/ops/ragged/__init__.py      |   4 +
 .../python/ops/ragged/ragged_array_ops.py     | 710 ------------------
 .../ops/ragged/ragged_batch_gather_op_test.py |  12 +-
 .../ops/ragged/ragged_batch_gather_ops.py     | 120 +++
 .../ops/ragged/ragged_concat_op_test.py       |  12 +-
 .../python/ops/ragged/ragged_concat_ops.py    | 302 ++++++++
 .../python/ops/ragged/ragged_dispatch.py      |  18 +-
 .../ops/ragged/ragged_gather_nd_op_test.py    |  12 +-
 .../ops/ragged/ragged_gather_op_test.py       |  33 +-
 .../python/ops/ragged/ragged_gather_ops.py    | 258 +++++++
 .../python/ops/ragged/ragged_getitem.py       |   4 +-
 .../python/ops/ragged/ragged_stack_op_test.py |  10 +-
 .../python/ops/ragged/ragged_where_op.py      | 166 ++++
 .../python/ops/ragged/ragged_where_op_test.py |   8 +-
 15 files changed, 989 insertions(+), 769 deletions(-)
 create mode 100644 tensorflow/python/ops/ragged/ragged_batch_gather_ops.py
 create mode 100644 tensorflow/python/ops/ragged/ragged_concat_ops.py
 create mode 100644 tensorflow/python/ops/ragged/ragged_gather_ops.py
 create mode 100644 tensorflow/python/ops/ragged/ragged_where_op.py

diff --git a/tensorflow/python/ops/ragged/BUILD b/tensorflow/python/ops/ragged/BUILD
index 972b14955f..e3bdb74cfe 100644
--- a/tensorflow/python/ops/ragged/BUILD
+++ b/tensorflow/python/ops/ragged/BUILD
@@ -24,10 +24,13 @@ py_library(
     tags = ["nofixdeps"],
     deps = [
         ":ragged_array_ops",
+        ":ragged_batch_gather_ops",
+        ":ragged_concat_ops",
         ":ragged_conversion_ops",
         ":ragged_dispatch",
         ":ragged_factory_ops",
         ":ragged_functional_ops",
+        ":ragged_gather_ops",
         ":ragged_getitem",
         ":ragged_map_ops",
         ":ragged_math_ops",
@@ -37,6 +40,7 @@ py_library(
         ":ragged_tensor_shape",
         ":ragged_tensor_value",
         ":ragged_util",
+        ":ragged_where_op",
         ":segment_id_ops",
         "//tensorflow/python:util",
     ],
@@ -55,6 +59,7 @@ py_library(
         ":segment_id_ops",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:check_ops",
+        "//tensorflow/python:constant_op",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:math_ops",
@@ -64,6 +69,45 @@ py_library(
     ],
 )
 
+py_library(
+    name = "ragged_batch_gather_ops",
+    srcs = ["ragged_batch_gather_ops.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":ragged_array_ops",
+        ":ragged_concat_ops",
+        ":ragged_conversion_ops",
+        ":ragged_gather_ops",
+        ":ragged_tensor",
+        ":ragged_tensor_shape",
+        ":ragged_util",
+        ":ragged_where_op",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:check_ops",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:math_ops",
+    ],
+)
+
+py_library(
+    name = "ragged_concat_ops",
+    srcs = ["ragged_concat_ops.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":ragged_array_ops",
+        ":ragged_conversion_ops",
+        ":ragged_gather_ops",
+        ":ragged_tensor",
+        ":ragged_util",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:check_ops",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:tensor_shape",
+    ],
+)
+
 py_library(
     name = "ragged_conversion_ops",
     srcs = ["ragged_conversion_ops.py"],
@@ -115,13 +159,30 @@ py_library(
     ],
 )
 
+py_library(
+    name = "ragged_gather_ops",
+    srcs = ["ragged_gather_ops.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":ragged_array_ops",
+        ":ragged_conversion_ops",
+        ":ragged_tensor",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:ragged_array_ops_gen",
+        "//tensorflow/python:tensor_shape",
+    ],
+)
+
 py_library(
     name = "ragged_getitem",
     srcs = ["ragged_getitem.py"],
     srcs_version = "PY2AND3",
     deps = [
-        ":ragged_array_ops",
         ":ragged_factory_ops",
+        ":ragged_gather_ops",
         ":ragged_math_ops",
         ":ragged_tensor",
         "//tensorflow/python:array_ops",
@@ -244,6 +305,22 @@ py_library(
     ],
 )
 
+py_library(
+    name = "ragged_where_op",
+    srcs = ["ragged_where_op.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":ragged_concat_ops",
+        ":ragged_functional_ops",
+        ":ragged_gather_ops",
+        ":ragged_tensor",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:math_ops",
+    ],
+)
+
 py_library(
     name = "segment_id_ops",
     srcs = ["segment_id_ops.py"],
@@ -412,6 +489,7 @@ py_test(
     deps = [
         ":ragged_array_ops",
         ":ragged_factory_ops",
+        ":ragged_gather_ops",
         ":ragged_test_util",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:constant_op",
@@ -430,6 +508,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":ragged_array_ops",
+        ":ragged_batch_gather_ops",
         ":ragged_factory_ops",
         ":ragged_tensor",
         ":ragged_test_util",
@@ -448,8 +527,8 @@ py_test(
     srcs = ["ragged_gather_nd_op_test.py"],
     srcs_version = "PY2AND3",
     deps = [
-        ":ragged_array_ops",
         ":ragged_factory_ops",
+        ":ragged_gather_ops",
         ":ragged_test_util",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:dtypes",
@@ -691,7 +770,7 @@ py_test(
     srcs = ["ragged_concat_op_test.py"],
     srcs_version = "PY2AND3",
     deps = [
-        ":ragged_array_ops",
+        ":ragged_concat_ops",
         ":ragged_factory_ops",
         ":ragged_test_util",
         "//tensorflow/python:array_ops",
@@ -710,7 +789,7 @@ py_test(
     srcs = ["ragged_stack_op_test.py"],
     srcs_version = "PY2AND3",
     deps = [
-        ":ragged_array_ops",
+        ":ragged_concat_ops",
         ":ragged_factory_ops",
         ":ragged_test_util",
         "//tensorflow/python:constant_op",
@@ -785,9 +864,9 @@ py_test(
     srcs = ["ragged_where_op_test.py"],
     srcs_version = "PY2AND3",
     deps = [
-        ":ragged_array_ops",
         ":ragged_factory_ops",
         ":ragged_test_util",
+        ":ragged_where_op",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:platform_test",
         "@absl_py//absl/testing:parameterized",
diff --git a/tensorflow/python/ops/ragged/__init__.py b/tensorflow/python/ops/ragged/__init__.py
index 30ed9c53ab..a5ffd8a950 100644
--- a/tensorflow/python/ops/ragged/__init__.py
+++ b/tensorflow/python/ops/ragged/__init__.py
@@ -29,10 +29,13 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.ops.ragged import ragged_array_ops
+from tensorflow.python.ops.ragged import ragged_batch_gather_ops
+from tensorflow.python.ops.ragged import ragged_concat_ops
 from tensorflow.python.ops.ragged import ragged_conversion_ops
 from tensorflow.python.ops.ragged import ragged_dispatch
 from tensorflow.python.ops.ragged import ragged_factory_ops
 from tensorflow.python.ops.ragged import ragged_functional_ops
+from tensorflow.python.ops.ragged import ragged_gather_ops
 from tensorflow.python.ops.ragged import ragged_getitem
 from tensorflow.python.ops.ragged import ragged_map_ops
 from tensorflow.python.ops.ragged import ragged_math_ops
@@ -41,6 +44,7 @@ from tensorflow.python.ops.ragged import ragged_string_ops
 from tensorflow.python.ops.ragged import ragged_tensor
 from tensorflow.python.ops.ragged import ragged_tensor_shape
 from tensorflow.python.ops.ragged import ragged_tensor_value
+from tensorflow.python.ops.ragged import ragged_where_op
 from tensorflow.python.ops.ragged import segment_id_ops
 
 # Add a list of the ops that support Ragged Tensors.
diff --git a/tensorflow/python/ops/ragged/ragged_array_ops.py b/tensorflow/python/ops/ragged/ragged_array_ops.py
index f1e61fd7f6..8c62cc4a72 100644
--- a/tensorflow/python/ops/ragged/ragged_array_ops.py
+++ b/tensorflow/python/ops/ragged/ragged_array_ops.py
@@ -20,11 +20,8 @@ from __future__ import print_function
 
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import check_ops
-from tensorflow.python.ops import gen_ragged_array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops.ragged import ragged_conversion_ops
 from tensorflow.python.ops.ragged import ragged_functional_ops
@@ -34,326 +31,6 @@ from tensorflow.python.ops.ragged import ragged_util
 from tensorflow.python.ops.ragged import segment_id_ops
 
 
-#===============================================================================
-# ragged_gather
-#===============================================================================
-# TODO(edloper): Add an `axis` argument
-def gather(params, indices, validate_indices=None, axis=0, batch_dims=0,
-           name=None):
-  """Gathers ragged slices from `params` axis `0` according to `indices`.
-
-  Returns `RaggedTensor` output, such that:
-
-  ```python
-  output.shape = indices.shape + params.shape[1:]
-  output.ragged_rank = indices.shape.ndims + params.ragged_rank
-  output[i...j, d0...dn] = params[indices[i...j], d0...dn]
-  ```
-
-  `params` may be ragged.  `indices` may be ragged.
-  `indices` must have dtype `int32` or `int64`. If any index is out of bounds,
-  then an error is returned.
-
-  Examples:
-
-  ```python
-  >>> params = tf.constant(['a', 'b', 'c', 'd', 'e'])
-  >>> indices = tf.constant([3, 1, 2, 1, 0])
-  >>> ragged_params = tf.ragged.constant([['a', 'b', 'c'], ['d'], [], ['e']])
-  >>> ragged_indices = tf.ragged.constant([[3, 1, 2], [1], [], [0]])
-
-  >>> print ragged.gather(params, ragged_indices)
-  [['d', 'b', 'c'], ['b'], [], ['a']]
-
-  >>> print ragged.gather(ragged_params, indices)
-  [['e'], ['d'], [], ['d'], ['a', 'b', 'c']]
-
-  >>> print ragged.gather(ragged_params, ragged_indices)
-  [[['e'], ['d'], []], [['d']], [], [['a', 'b', 'c']]]
-  ```
-
-  Args:
-    params: The potentially ragged tensor from which to gather values. Must be
-      at least rank 1.
-    indices: The potentially ragged tensor indicating which values to gather.
-      Must have dtype `int32` or `int64`.  Values must be in the range `[0,
-      params.shape[0]]`.
-    validate_indices: Ignored.
-    axis: Must be zero.
-    batch_dims: Must be zero.
-    name: A name for the operation (optional).
-
-  Returns:
-    A `RaggedTensor`, where `output.dtype=params.dtype` and
-    `output.shape=indices.shape + params.shape[1:]` and
-    `output.ragged_rank=indices.shape.ndims + params.ragged_rank`.
-
-  Raises:
-    ValueError: If indices.shape.ndims is not known statically.
-  """
-  del validate_indices
-  if not isinstance(axis, int) or axis != 0:
-    raise ValueError('axis != 0 is not supported for ragged gather yet.')
-  if not isinstance(batch_dims, int) or batch_dims != 0:
-    raise ValueError('batch_dims != 0 is not supported for ragged gather yet.')
-  with ops.name_scope(name, 'RaggedGather', [params, indices]):
-    params = ragged_tensor.convert_to_tensor_or_ragged_tensor(
-        params, name='params')
-    indices = ragged_tensor.convert_to_tensor_or_ragged_tensor(
-        indices, name='indices')
-
-    if ragged_tensor.is_ragged(indices):
-      return indices.with_values(gather(params, indices.values))
-
-    if not ragged_tensor.is_ragged(params):
-      return array_ops.gather(params, indices)
-
-    indices = ops.convert_to_tensor(indices)
-    if indices.shape.ndims is None:
-      raise ValueError('indices.shape.ndims must be known statically')
-
-    result = gen_ragged_array_ops.ragged_gather(
-        indices=indices,
-        params_dense_values=params.flat_values,
-        params_nested_splits=params.nested_row_splits,
-        OUTPUT_RAGGED_RANK=indices.shape.ndims + len(params.nested_row_splits) -
-        1)
-
-    # Compose the RaggedTensor from splits & values.
-    return ragged_tensor.RaggedTensor.from_nested_row_splits(
-        result.output_dense_values, result.output_nested_splits)
-
-
-#===============================================================================
-# ragged.batch_gather
-#===============================================================================
-def batch_gather(params, indices, name=None):
-  """Gathers slices from `params` according to `indices` with batch dims.
-
-  This operation is similar to `gather`, but it assumes that the leading `N`
-  dimensions of `indices` and `params` are batch dimensions, and performs a
-  gather within each batch.  In particular, when using this operation with `N`
-  batch dimensions `B1...BN`:
-
-  * `indices` has shape `[B1...BN, I]`
-  * `params` has shape `[B1...BN, P1...PM]`.
-  * `result` has shape `[B1...BN, I, P2...PM]`.
-  * `result[b1...bN, i, p2...pM] =
-    params[b1...bN, indices[b1...bN, i], p2...pM]`
-
-  Args:
-    params: A potentially ragged tensor with shape `[B1...BN, P1...PM]` (`N>=0`,
-      `M>0`).
-    indices: A potentially ragged tensor with shape `[B1...BN, I]` (`N>=0`).
-    name: A name for the operation (optional).
-
-  Returns:
-    A potentially ragged tensor with shape `[B1...BN, I, P2...PM]`.
-    `result.ragged_rank = max(indices.ragged_rank, params.ragged_rank)`.
-
-  #### Example:
-    ```python
-    >>> params = tf.ragged.constant([['a', 'b', 'c'], ['d'], [], ['e']])
-    >>> indices = tf.ragged.constant([[1, 2, 0], [], [], [0, 0]])
-    >>> ragged.batch_gather(params, indices)
-    [['b', 'c', 'a'], [], [], ['e', 'e']]
-    ```
-  """
-  if not (ragged_tensor.is_ragged(params) or ragged_tensor.is_ragged(indices)):
-    return array_ops.batch_gather(params, indices, name)
-
-  with ops.name_scope(name, 'RaggedBatchGather', [params, indices]):
-    params = ragged_tensor.convert_to_tensor_or_ragged_tensor(
-        params, name='params')
-    indices = ragged_tensor.convert_to_tensor_or_ragged_tensor(
-        indices, name='indices')
-    indices_ndims = indices.shape.ndims
-    if indices_ndims is None:
-      raise ValueError(
-          'batch_gather does not allow indices with unknown shape.')
-    if indices_ndims == 0:
-      raise ValueError('indices.rank must be at least 1.')
-
-    if ragged_tensor.is_ragged(indices):
-      # If the outermost ragged dimension is a batch dimension, recurse.
-      if indices_ndims > 2:
-        if not ragged_tensor.is_ragged(params):
-          raise ValueError('batch shape from indices does '
-                           'not match params shape')
-        checks = [check_ops.assert_equal(params.row_splits, indices.row_splits)]
-        with ops.control_dependencies(checks):
-          return ragged_tensor.RaggedTensor.from_row_splits(
-              batch_gather(params.values, indices.values), indices.row_splits)
-
-      # Otherwise, indices is a 2D ragged tensor with 1 ragged dimension.
-      else:
-        # Ensure that `params` is ragged and has at least 2 dimensions.
-        if not ragged_tensor.is_ragged(params):
-          if params.shape.ndims is not None and params.shape.ndims < 2:
-            raise ValueError('batch shape from indices does '
-                             'not match params shape')
-          params = ragged_conversion_ops.from_tensor(params, ragged_rank=1)
-
-        # Adjust indices from within-batch to global (in params.values), and
-        # then use ragged.gather to gather them.
-        num_indices = indices.row_lengths()
-        params_starts = params.row_starts()
-        adjustments = ragged_util.repeat(params_starts, num_indices, axis=0)
-        adjusted_index_values = math_ops.to_int64(indices.values) + adjustments
-        return ragged_tensor.RaggedTensor.from_row_splits(
-            gather(params.values, adjusted_index_values), indices.row_splits)
-
-    else:  # params is a RaggedTensor and indices is a Tensor.
-      if indices_ndims == 1:
-        return gather(params, indices)
-      elif indices_ndims == 2:
-        # Adjust indices from batch-local to global (in params.values)
-        adjustments = array_ops.expand_dims(params.row_starts(), 1)
-        adjusted_indices = math_ops.to_int64(indices) + adjustments
-        return gather(params.values, adjusted_indices)
-      else:
-        raise ValueError('batch shape from indices does not match params shape')
-
-
-#===============================================================================
-# ragged.gather_nd
-#===============================================================================
-def gather_nd(params, indices, name=None):
-  """Gather slices from `params` using `n`-dimensional indices.
-
-  This operation is similar to `gather`, but it uses the innermost dimension
-  of `indices` to define a slice into `params`.  In particular, if:
-
-  * `indices` has shape `[A1...AN, I]`
-  * `params` has shape `[B1...BM]`
-
-  Then:
-
-  * `result` has shape `[A1...AN, B_{I+1}...BM]`.
-  * `result[a1...aN] = params[indices[a1...aN, :]]`
-
-  Args:
-    params: A potentially ragged tensor with shape `[A1...AN, I]`.
-    indices: A potentially ragged tensor with shape `[B1...BM]`.
-    name: A name for the operation (optional).
-
-  Returns:
-    A potentially ragged tensor with shape `[A1...AN, B_{I+1}...BM]`.
-
-  #### Examples:
-    ```python
-    >>> params = tf.ragged.constant_value(
-    ...     [ [ ['000', '001'], ['010'              ]          ],
-    ...       [ ['100'       ], ['110', '111', '112'], ['120'] ],
-    ...       [ [            ], ['210'              ]          ] ])
-
-    >>> # Gather 2D slices from a 3D tensor
-    >>> ragged.gather_nd(params, [[2], [0]])
-    [ [ [            ], ['210'] ]
-      [ ['000', '001'], ['010'] ] ]
-
-    >>> # Gather 1D slices from a 3D tensor
-    >>> ragged.gather_nd(params, [[2, 1], [0, 0]])
-    [['210'], ['000', '001']]
-
-    >>> # Gather scalars from a 3D tensor
-    >>> ragged.gather_nd(params, [[0, 0, 1], [1, 1, 2]])
-    ['001', '112']
-    ```
-  """
-  if not (ragged_tensor.is_ragged(params) or ragged_tensor.is_ragged(indices)):
-    return array_ops.gather_nd(params, indices, name)
-
-  with ops.name_scope(name, 'RaggedGatherNd', [params, indices]):
-
-    params = ragged_tensor.convert_to_tensor_or_ragged_tensor(
-        params, name='params')
-    indices = ragged_tensor.convert_to_tensor_or_ragged_tensor(
-        indices, name='indices')
-    indices_shape = indices.shape
-    indices_ndims = indices_shape.ndims
-    if indices_ndims is None:
-      raise ValueError('indices.rank be statically known.')
-    if indices_ndims == 0:
-      raise ValueError('indices.rank must be at least 1.')
-    if (ragged_tensor.is_ragged(indices) and
-        indices_ndims == indices.ragged_rank + 1):
-      raise ValueError('The innermost dimension of indices may not be ragged')
-
-    # `index_size` is the "n" in "gather_nd" -- i.e., the number of dimensions
-    # that each index slices into.
-    index_size = tensor_shape.dimension_value(indices_shape[-1])
-    if index_size is None:
-      raise ValueError('indices.shape[-1] must be statically known.')
-
-    # If `indices` has more than 2 dimensions, then recurse.  If `indices` is
-    # dense, then we convert it to ragged before recursing, and then convert
-    # the result back to `dense` if appropriate.
-    if indices_ndims > 2:
-      indices_is_dense = not ragged_tensor.is_ragged(indices)
-      if indices_is_dense:
-        indices = ragged_conversion_ops.from_tensor(
-            indices, ragged_rank=indices_ndims - 2)
-      result = indices.with_flat_values(gather_nd(params, indices.flat_values))
-      if (indices_is_dense and ragged_tensor.is_ragged(result) and
-          result.ragged_rank == indices_ndims - 2):
-        result = ragged_conversion_ops.to_tensor(result)
-      return result
-
-    # indices_ndims <= 2, and the innermost dimension of indices may not be
-    # ragged, so `indices` must not be ragged.
-    assert not ragged_tensor.is_ragged(indices)
-    assert ragged_tensor.is_ragged(params)
-
-    # Handle corner case: An empty index tuple selects the entire `params`
-    # value.  So if `index_size` is zero, then tile `params`.
-    if index_size == 0:
-      params_ndims = params.ragged_rank + array_ops.rank(params.flat_values)
-      for dim in range(indices_ndims - 1):
-        params = expand_dims(params, axis=0)
-      multiples = array_ops.concat([
-          array_ops.shape(indices)[:-1],
-          array_ops.ones([params_ndims], dtypes.int32)
-      ],
-                                   axis=0)
-      return tile(params, multiples)
-
-    # When index_size=1, we can just flatten the index tuples and use gather.
-    elif index_size == 1:
-      flattened_index_tuples = array_ops.reshape(indices, [-1])
-      return gather(params, flattened_index_tuples)
-
-    # Otherwise, params is a RaggedTensor, and indices is a 1D or 2D Tensor.
-    # Flatten both the index tuples and the params, such that the flattened
-    # index tuples point to the correct values in the flattened params; and
-    # then use ragged.gather on the flattened index tuples & params.
-    else:
-      indices = math_ops.to_int64(indices)
-
-      # Flatten the outermost 2 dimensions of the index tuples & params.
-      flattened_index_tuples = array_ops.gather(params.row_splits,
-                                                indices[..., 0])
-      flattened_index_tuples += indices[..., 1]
-      flattened_params = params.values
-
-      # Flatten any remaining dimensions.
-      for dim in range(2, index_size):
-        if not ragged_tensor.is_ragged(flattened_params):
-          flattened_index_tuples = array_ops.expand_dims(
-              flattened_index_tuples, axis=1)
-          flattened_index_tuples = array_ops.concat(
-              [flattened_index_tuples, indices[..., dim:]], axis=1)
-          return array_ops.gather_nd(flattened_params, flattened_index_tuples)
-
-        flattened_index_tuples = array_ops.gather(
-            flattened_params.row_starts(), flattened_index_tuples)
-        flattened_index_tuples += indices[..., dim]
-        flattened_params = flattened_params.values
-
-      # Gather using the flattened index tuples and params.
-      return gather(flattened_params, flattened_index_tuples)
-
-
 #===============================================================================
 # Masking
 #===============================================================================
@@ -547,260 +224,6 @@ def boolean_mask(data, mask, keepdims=False, name=None):
       return masked_values
 
 
-#===============================================================================
-# Concatenation and Stacking
-#===============================================================================
-def concat(values, axis, name=None):
-  """Concatenates potentially ragged tensors along one dimension.
-
-  Given a list of tensors with the same rank `K` (`K >= axis`), returns a
-  rank-`K` `RaggedTensor` `result` such that `result[i0...iaxis]` is the
-  concatenation of `[rt[i0...iaxis] for rt in values]`.
-
-  Args:
-    values: A list of potentially ragged tensors.  May not be empty. All
-      `values` must have the same rank and the same dtype; but unlike
-      `tf.concat`, they can have arbitrary shapes.
-    axis: A python integer, indicating the dimension along which to concatenate.
-      (Note: Unlike `tf.concat`, the `axis` parameter must be statically known.)
-        Negative values are supported only if the rank of at least one
-        `values` value is statically known.
-    name: A name prefix for the returned tensor (optional).
-
-  Returns:
-    A `RaggedTensor` with rank `K`.
-    `result.ragged_rank=max(axis, max(rt.ragged_rank for rt in values]))`.
-
-  Raises:
-    ValueError: If `values` is empty, if `axis` is out of bounds or if
-      the input tensors have different ranks.
-
-  #### Example:
-    ```python
-    >>> t1 = tf.ragged.constant([[1, 2], [3, 4, 5]])
-    >>> t2 = tf.ragged.constant([[6], [7, 8, 9]])
-    >>> ragged.concat([t1, t2], axis=0)
-    [[1, 2], [3, 4, 5], [6], [7, 8, 9]]
-    >>> ragged.concat([t1, t2], axis=1)
-    [[1, 2, 6], [3, 4, 5, 7, 8, 9]]
-    ```
-  """
-  if not isinstance(values, (list, tuple)):
-    values = [values]
-  with ops.name_scope(name, 'RaggedConcat', values):
-    return _ragged_stack_concat_helper(values, axis, stack_values=False)
-
-
-def stack(values, axis=0, name=None):
-  """Stacks potentially ragged tensors along one dimension.
-
-  Given a list of tensors with the same rank `K` (`K >= axis`), returns a
-  rank-`K+1` `RaggedTensor` `result` such that `result[i0...iaxis]` is the
-  list `[rt[i0...iaxis] for rt in values]`.
-
-  Args:
-    values: A list of potentially ragged tensors.  May not be empty. All
-      `values` must have the same rank and the same dtype; but unlike
-      `tf.concat`, they can have arbitrary shapes.
-    axis: A python integer, indicating the dimension along which to stack.
-      (Note: Unlike `tf.stack`, the `axis` parameter must be statically known.)
-        Negative values are supported only if the rank of at least one
-        `values` value is statically known.
-    name: A name prefix for the returned tensor (optional).
-
-  Returns:
-    A `RaggedTensor` with rank `K+1`.
-    `result.ragged_rank=max(axis, max(rt.ragged_rank for rt in values]))`.
-
-  Raises:
-    ValueError: If `values` is empty, if `axis` is out of bounds or if
-      the input tensors have different ranks.
-
-  #### Example:
-    ```python
-    >>> t1 = tf.ragged.constant([[1, 2], [3, 4, 5]])
-    >>> t2 = tf.ragged.constant([[6], [7, 8, 9]])
-    >>> ragged.stack([t1, t2], axis=0)
-    [[[1, 2], [3, 4, 5]], [[6], [7, 9, 0]]]
-    >>> ragged.stack([t1, t2], axis=1)
-    [[[1, 2], [6]], [[3, 4, 5], [7, 8, 9]]]
-    ```
-  """
-  if not isinstance(values, (list, tuple)):
-    values = [values]
-  with ops.name_scope(name, 'RaggedConcat', values):
-    return _ragged_stack_concat_helper(values, axis, stack_values=True)
-
-
-def _ragged_stack_concat_helper(rt_inputs, axis, stack_values):
-  """Helper function to concatenate or stack ragged tensors.
-
-  Args:
-    rt_inputs: A list of RaggedTensors or Tensors to combine.
-    axis: The axis along which to concatenate or stack.
-    stack_values: A boolean -- if true, then stack values; otherwise,
-      concatenate them.
-
-  Returns:
-    A RaggedTensor.
-  Raises:
-    ValueError: If rt_inputs is empty, or if axis is out of range.
-  """
-  # Validate parameters.
-  if not rt_inputs:
-    raise ValueError('rt_inputs may not be empty.')
-
-  # Convert input tensors.
-  rt_inputs = [
-      ragged_tensor.convert_to_tensor_or_ragged_tensor(
-          rt_input, name='rt_input') for rt_input in rt_inputs
-  ]
-
-  # Special case: if there's only one input, then return it as-is.
-  if len(rt_inputs) == 1:
-    if stack_values:
-      return expand_dims(rt_inputs[0], axis=axis)
-    else:
-      return rt_inputs[0]
-
-  # Check the rank (number of dimensions) of the input tensors.
-  ndims = None
-  for rt in rt_inputs:
-    if ndims is None:
-      ndims = rt.shape.ndims
-    else:
-      rt.shape.assert_has_rank(ndims)
-
-  out_ndims = ndims if (ndims is None or not stack_values) else ndims + 1
-  axis = ragged_util.get_positive_axis(axis, out_ndims)
-
-  # If all the inputs are Tensors, and we're combining the final dimension,
-  # then we can delegate to the tf.stack/tf.concat operation, and return a
-  # Tensor.
-  if all(not ragged_tensor.is_ragged(rt) for rt in rt_inputs):
-    if ndims is not None and (axis == out_ndims - 1 or axis == ndims - 1):
-      if stack_values:
-        return array_ops.stack(rt_inputs, axis)
-      else:
-        return array_ops.concat(rt_inputs, axis)
-
-  # Convert any Tensor inputs to RaggedTensors.  This makes it
-  # possible to concatenate Tensors and RaggedTensors together.
-  for i in range(len(rt_inputs)):
-    if not ragged_tensor.is_ragged(rt_inputs[i]):
-      rt_inputs[i] = ragged_conversion_ops.from_tensor(
-          rt_inputs[i], ragged_rank=1)
-
-  # Convert the input tensors to all have the same ragged_rank.
-  ragged_rank = max(max(rt.ragged_rank for rt in rt_inputs), 1)
-  rt_inputs = [_increase_ragged_rank_to(rt, ragged_rank) for rt in rt_inputs]
-
-  if axis == 0:
-    return _ragged_stack_concat_axis_0(rt_inputs, stack_values)
-  elif axis == 1:
-    return _ragged_stack_concat_axis_1(rt_inputs, stack_values)
-  else:  # axis > 1: recurse.
-    values = [rt.values for rt in rt_inputs]
-    splits = [[rt_input.row_splits] for rt_input in rt_inputs]
-    with ops.control_dependencies(ragged_util.assert_splits_match(splits)):
-      return ragged_tensor.RaggedTensor.from_row_splits(
-          _ragged_stack_concat_helper(values, axis - 1, stack_values),
-          splits[0][0])
-
-
-def _ragged_stack_concat_axis_0(rt_inputs, stack_values):
-  """Helper function to concatenate or stack ragged tensors along axis 0.
-
-  Args:
-    rt_inputs: A list of RaggedTensors, all with the same rank and ragged_rank.
-    stack_values: Boolean.  If true, then stack values; otherwise, concatenate
-      them.
-
-  Returns:
-    A RaggedTensor.
-  """
-  # Concatenate the inner values together.
-  flat_values = [rt.flat_values for rt in rt_inputs]
-  concatenated_flat_values = array_ops.concat(flat_values, axis=0)
-
-  # Concatenate the splits together for each ragged dimension (adjusting
-  # split offsets as necessary).
-  nested_splits = [rt.nested_row_splits for rt in rt_inputs]
-  ragged_rank = rt_inputs[0].ragged_rank
-  concatenated_nested_splits = [
-      _concat_ragged_splits([ns[dim]
-                             for ns in nested_splits])
-      for dim in range(ragged_rank)
-  ]
-
-  # If we are performing a stack operation, then add another splits.
-  if stack_values:
-    stack_lengths = array_ops.stack([_nrows(rt) for rt in rt_inputs])
-    stack_splits = ragged_util.lengths_to_splits(stack_lengths)
-    concatenated_nested_splits.insert(0, stack_splits)
-
-  return ragged_tensor.RaggedTensor.from_nested_row_splits(
-      concatenated_flat_values, concatenated_nested_splits)
-
-
-def _ragged_stack_concat_axis_1(rt_inputs, stack_values):
-  """Helper function to concatenate or stack ragged tensors along axis 1.
-
-  Args:
-    rt_inputs: A list of RaggedTensors, all with the same rank and ragged_rank.
-    stack_values: Boolean.  If true, then stack values; otherwise, concatenate
-      them.
-
-  Returns:
-    A RaggedTensor.
-  """
-  num_inputs = len(rt_inputs)
-
-  rt_nrows = _nrows(rt_inputs[0])
-  nrows_msg = 'Input tensors have incompatible shapes.'
-  nrows_checks = [
-      check_ops.assert_equal(_nrows(rt), rt_nrows, message=nrows_msg)
-      for rt in rt_inputs[1:]
-  ]
-
-  with ops.control_dependencies(nrows_checks):
-    # Concatentate the inputs together to put them in a single ragged tensor.
-    concatenated_rt = _ragged_stack_concat_axis_0(rt_inputs, stack_values=False)
-
-    # Use ragged.gather to permute the rows of concatenated_rt.  In particular,
-    #   permuted_rt = [rt_inputs[0][0], ..., rt_inputs[N][0],
-    #                  rt_inputs[0][1], ..., rt_inputs[N][1],
-    #                      ...,
-    #                  rt_inputs[0][M], ..., rt_input[N][M]]
-    # where `N=num_inputs-1` and `M=rt_nrows-1`.
-    row_indices = math_ops.range(rt_nrows * num_inputs)
-    row_index_matrix = array_ops.reshape(row_indices, [num_inputs, -1])
-    transposed_row_index_matrix = array_ops.transpose(row_index_matrix)
-    row_permutation = array_ops.reshape(transposed_row_index_matrix, [-1])
-    permuted_rt = gather(concatenated_rt, row_permutation)
-
-    if stack_values:
-      # Add a new splits tensor to group together the values.
-      stack_splits = math_ops.range(0, rt_nrows * num_inputs + 1, num_inputs)
-      _copy_row_shape(rt_inputs, stack_splits)
-      return ragged_tensor.RaggedTensor.from_row_splits(permuted_rt,
-                                                        stack_splits)
-    else:
-      # Merge together adjacent rows by dropping the row-split indices that
-      # separate them.
-      concat_splits = permuted_rt.row_splits[::num_inputs]
-      _copy_row_shape(rt_inputs, concat_splits)
-      return ragged_tensor.RaggedTensor.from_row_splits(permuted_rt.values,
-                                                        concat_splits)
-
-
-def _copy_row_shape(rt_inputs, splits):
-  """Sets splits.shape to [rt[shape[0]+1] for each rt in rt_inputs."""
-  for rt in rt_inputs:
-    if rt.shape[0] is not None:
-      splits.set_shape(tensor_shape.TensorShape(rt.shape[0] + 1))
-
-
 #===============================================================================
 # Tiling
 #===============================================================================
@@ -1061,139 +484,6 @@ def expand_dims(input, axis, name=None):  # pylint: disable=redefined-builtin
     return ragged_tensor.RaggedTensor.from_row_splits(values, splits)
 
 
-#===============================================================================
-# ragged.where
-#===============================================================================
-def where(condition, x=None, y=None, name=None):
-  """Return the elements, either from `x` or `y`, depending on the `condition`.
-
-  : If both `x` and `y` are `None`:
-    Returns the coordinates of true elements of `condition`. The coordinates
-    are returned in a 2-D tensor with shape
-    `[num_true_values, dim_size(condition)]`, where `result[i]` is the
-    coordinates of the `i`th true value (in row-major order).
-
-  : If both `x` and `y` are non-`None`:
-    Returns a tensor formed by selecting values from `x` where condition is
-    true, and from `y` when condition is false.  In particular:
-
-    : If `condition`, `x`, and `y` all have the same shape:
-
-      * `result[i1...iN] = x[i1...iN]` if `condition[i1...iN]` is true.
-      * `result[i1...iN] = y[i1...iN]` if `condition[i1...iN]` is false.
-
-    : Otherwise:
-
-      * `condition` must be a vector.
-      * `x` and `y` must have the same number of dimensions.
-      * The outermost dimensions of `condition`, `x`, and `y` must all have the
-        same size.
-      * `result[i] = x[i]` if `condition[i]` is true.
-      * `result[i] = y[i]` if `condition[i]` is false.
-
-  Args:
-    condition: A potentially ragged tensor of type `bool`
-    x: A potentially ragged tensor (optional).
-    y: A potentially ragged tensor (optional).  Must be specified if `x` is
-      specified.  Must have the same rank and type as `x`.
-    name: A name of the operation (optional)
-
-  Returns:
-    : If both `x` and `y` are `None`:
-      A `Tensor` with shape `(num_true, dim_size(condition))`.
-    : Otherwise:
-      A potentially ragged tensor with the same type, rank, and outermost
-      dimension size as `x` and `y`.
-      `result.ragged_rank = max(x.ragged_rank, y.ragged_rank)`.
-
-  Raises:
-    ValueError: When exactly one of `x` or `y` is non-`None`; or when
-      `condition`, `x`, and `y` have incompatible shapes.
-
-  #### Examples:
-    ```python
-    >>> # Coordinates where condition is true.
-    >>> condition = tf.ragged.constant_value(
-    ...     [[True, False, True], [False, True]])
-    >>> ragged.where(condition)
-    [[0, 0], [0, 2], [1, 1]]
-
-    >>> # Elementwise selection between x and y, based on condition.
-    >>> condition = tf.ragged.constant_value(
-    ...     [[True, False, True], [False, True]])
-    >>> x = tf.ragged.constant_value([['A', 'B', 'C'], ['D', 'E']])
-    >>> y = tf.ragged.constant_value([['a', 'b', 'c'], ['d', 'e']])
-    >>> ragged.where(condition, x, y)
-    [['A', 'b', 'C'], ['d', 'E']]
-
-    >>> # Row selection between x and y, based on condition.
-    >>> condition = [True, False]
-    >>> x = tf.ragged.constant_value([['A', 'B', 'C'], ['D', 'E']])
-    >>> y = tf.ragged.constant_value([['a', 'b', 'c'], ['d', 'e']])
-    >>> ragged.where(condition, x, y)
-    [['A', 'B', 'C'], ['d', 'e']]
-    ```
-  """
-  if (x is None) != (y is None):
-    raise ValueError('x and y must be either both None or both non-None')
-  with ops.name_scope('RaggedWhere', name, [condition, x, y]):
-    condition = ragged_tensor.convert_to_tensor_or_ragged_tensor(
-        condition, name='condition')
-    if x is None:
-      return _coordinate_where(condition)
-    else:
-      x = ragged_tensor.convert_to_tensor_or_ragged_tensor(x, name='x')
-      y = ragged_tensor.convert_to_tensor_or_ragged_tensor(y, name='y')
-      return _elementwise_where(condition, x, y)
-
-
-def _elementwise_where(condition, x, y):
-  """Ragged version of tf.where(condition, x, y)."""
-  condition_is_ragged = isinstance(condition, ragged_tensor.RaggedTensor)
-  x_is_ragged = isinstance(x, ragged_tensor.RaggedTensor)
-  y_is_ragged = isinstance(y, ragged_tensor.RaggedTensor)
-
-  if not (condition_is_ragged or x_is_ragged or y_is_ragged):
-    return array_ops.where(condition, x, y)
-
-  elif condition_is_ragged and x_is_ragged and y_is_ragged:
-    return ragged_functional_ops.map_flat_values(array_ops.where, condition, x,
-                                                 y)
-  elif not condition_is_ragged:
-    # Concatenate x and y, and then use `gather` to assemble the selected rows.
-    condition.shape.assert_has_rank(1)
-    x_nrows = _nrows(x)
-    x_and_y = concat([x, y], axis=0)
-    indices = array_ops.where(condition, math_ops.range(x_nrows),
-                              x_nrows + math_ops.range(_nrows(y)))
-    return gather(x_and_y, indices)
-
-  else:
-    raise ValueError('Input shapes do not match.')
-
-
-def _coordinate_where(condition):
-  """Ragged version of tf.where(condition)."""
-  if not isinstance(condition, ragged_tensor.RaggedTensor):
-    return array_ops.where(condition)
-
-  # The coordinate for each `true` value in condition.values.
-  selected_coords = _coordinate_where(condition.values)
-
-  # Convert the first index in each coordinate to a row index and column index.
-  first_index = selected_coords[:, 0]
-  selected_rows = array_ops.gather(condition.value_rowids(), first_index)
-  selected_row_starts = array_ops.gather(condition.row_splits, selected_rows)
-  selected_cols = first_index - selected_row_starts
-
-  # Assemble the row & column index with the indices for inner dimensions.
-  return array_ops.concat([
-      array_ops.expand_dims(selected_rows, 1),
-      array_ops.expand_dims(selected_cols, 1), selected_coords[:, 1:]
-  ],
-                          axis=1)
-
-
 #===============================================================================
 # RaggedTensor Size
 #===============================================================================
diff --git a/tensorflow/python/ops/ragged/ragged_batch_gather_op_test.py b/tensorflow/python/ops/ragged/ragged_batch_gather_op_test.py
index 431d350db8..72692fcef0 100644
--- a/tensorflow/python/ops/ragged/ragged_batch_gather_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_batch_gather_op_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for ragged_array_ops.batch_gather."""
+"""Tests for ragged_batch_gather_ops.batch_gather."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -25,7 +25,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops.ragged import ragged_array_ops
+from tensorflow.python.ops.ragged import ragged_batch_gather_ops
 from tensorflow.python.ops.ragged import ragged_factory_ops
 from tensorflow.python.ops.ragged import ragged_tensor
 from tensorflow.python.ops.ragged import ragged_test_util
@@ -146,7 +146,7 @@ class RaggedBatchGatherOpTest(ragged_test_util.RaggedTensorTestCase,
               [[[[b'c', b'a'], [b'd', b'd']], [[b'f', b'e']]]], ragged_rank=2)),
   ])
   def testRaggedBatchGather(self, descr, params, indices, expected):
-    result = ragged_array_ops.batch_gather(params, indices)
+    result = ragged_batch_gather_ops.batch_gather(params, indices)
     self.assertRaggedEqual(result, expected)
 
   def testRaggedBatchGatherUnknownRankError(self):
@@ -159,11 +159,11 @@ class RaggedBatchGatherOpTest(ragged_test_util.RaggedTensorTestCase,
 
     with self.assertRaisesRegexp(
         ValueError, 'batch_gather does not allow indices with unknown shape.'):
-      ragged_array_ops.batch_gather(params, indices)
+      ragged_batch_gather_ops.batch_gather(params, indices)
 
     with self.assertRaisesRegexp(
         ValueError, 'batch_gather does not allow indices with unknown shape.'):
-      ragged_array_ops.batch_gather(params, ragged_indices)
+      ragged_batch_gather_ops.batch_gather(params, ragged_indices)
 
   @parameterized.parameters(
       [
@@ -208,7 +208,7 @@ class RaggedBatchGatherOpTest(ragged_test_util.RaggedTensorTestCase,
                                        message=None,
                                        error=ValueError):
     with self.assertRaisesRegexp(error, message):
-      ragged_array_ops.batch_gather(params, indices)
+      ragged_batch_gather_ops.batch_gather(params, indices)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/ops/ragged/ragged_batch_gather_ops.py b/tensorflow/python/ops/ragged/ragged_batch_gather_ops.py
new file mode 100644
index 0000000000..9c57aead91
--- /dev/null
+++ b/tensorflow/python/ops/ragged/ragged_batch_gather_ops.py
@@ -0,0 +1,120 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Batch gather operations for RaggedTensors."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import check_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.ragged import ragged_conversion_ops
+from tensorflow.python.ops.ragged import ragged_gather_ops
+from tensorflow.python.ops.ragged import ragged_tensor
+from tensorflow.python.ops.ragged import ragged_util
+
+
+#===============================================================================
+# ragged.batch_gather
+#===============================================================================
+def batch_gather(params, indices, name=None):
+  """Gathers slices from `params` according to `indices` with batch dims.
+
+  This operation is similar to `gather`, but it assumes that the leading `N`
+  dimensions of `indices` and `params` are batch dimensions, and performs a
+  gather within each batch.  In particular, when using this operation with `N`
+  batch dimensions `B1...BN`:
+
+  * `indices` has shape `[B1...BN, I]`
+  * `params` has shape `[B1...BN, P1...PM]`.
+  * `result` has shape `[B1...BN, I, P2...PM]`.
+  * `result[b1...bN, i, p2...pM] =
+    params[b1...bN, indices[b1...bN, i], p2...pM]`
+
+  Args:
+    params: A potentially ragged tensor with shape `[B1...BN, P1...PM]` (`N>=0`,
+      `M>0`).
+    indices: A potentially ragged tensor with shape `[B1...BN, I]` (`N>=0`).
+    name: A name for the operation (optional).
+
+  Returns:
+    A potentially ragged tensor with shape `[B1...BN, I, P2...PM]`.
+    `result.ragged_rank = max(indices.ragged_rank, params.ragged_rank)`.
+
+  #### Example:
+    ```python
+    >>> params = tf.ragged.constant([['a', 'b', 'c'], ['d'], [], ['e']])
+    >>> indices = tf.ragged.constant([[1, 2, 0], [], [], [0, 0]])
+    >>> tf.batch_gather(params, indices)
+    [['b', 'c', 'a'], [], [], ['e', 'e']]
+    ```
+  """
+  if not (ragged_tensor.is_ragged(params) or ragged_tensor.is_ragged(indices)):
+    return array_ops.batch_gather(params, indices, name)
+
+  with ops.name_scope(name, 'RaggedBatchGather', [params, indices]):
+    params = ragged_tensor.convert_to_tensor_or_ragged_tensor(
+        params, name='params')
+    indices = ragged_tensor.convert_to_tensor_or_ragged_tensor(
+        indices, name='indices')
+    indices_ndims = indices.shape.ndims
+    if indices_ndims is None:
+      raise ValueError(
+          'batch_gather does not allow indices with unknown shape.')
+    if indices_ndims == 0:
+      raise ValueError('indices.rank must be at least 1.')
+
+    if ragged_tensor.is_ragged(indices):
+      # If the outermost ragged dimension is a batch dimension, recurse.
+      if indices_ndims > 2:
+        if not ragged_tensor.is_ragged(params):
+          raise ValueError('batch shape from indices does '
+                           'not match params shape')
+        checks = [check_ops.assert_equal(params.row_splits, indices.row_splits)]
+        with ops.control_dependencies(checks):
+          return ragged_tensor.RaggedTensor.from_row_splits(
+              batch_gather(params.values, indices.values), indices.row_splits)
+
+      # Otherwise, indices is a 2D ragged tensor with 1 ragged dimension.
+      else:
+        # Ensure that `params` is ragged and has at least 2 dimensions.
+        if not ragged_tensor.is_ragged(params):
+          if params.shape.ndims is not None and params.shape.ndims < 2:
+            raise ValueError('batch shape from indices does '
+                             'not match params shape')
+          params = ragged_conversion_ops.from_tensor(params, ragged_rank=1)
+
+        # Adjust indices from within-batch to global (in params.values), and
+        # then use ragged.gather to gather them.
+        num_indices = indices.row_lengths()
+        params_starts = params.row_starts()
+        adjustments = ragged_util.repeat(params_starts, num_indices, axis=0)
+        adjusted_index_values = math_ops.to_int64(indices.values) + adjustments
+        return ragged_tensor.RaggedTensor.from_row_splits(
+            ragged_gather_ops.gather(params.values, adjusted_index_values),
+            indices.row_splits)
+
+    else:  # params is a RaggedTensor and indices is a Tensor.
+      if indices_ndims == 1:
+        return ragged_gather_ops.gather(params, indices)
+      elif indices_ndims == 2:
+        # Adjust indices from batch-local to global (in params.values)
+        adjustments = array_ops.expand_dims(params.row_starts(), 1)
+        adjusted_indices = math_ops.to_int64(indices) + adjustments
+        return ragged_gather_ops.gather(params.values, adjusted_indices)
+      else:
+        raise ValueError('batch shape from indices does not match params shape')
diff --git a/tensorflow/python/ops/ragged/ragged_concat_op_test.py b/tensorflow/python/ops/ragged/ragged_concat_op_test.py
index 254afdaa21..62989d3025 100644
--- a/tensorflow/python/ops/ragged/ragged_concat_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_concat_op_test.py
@@ -26,7 +26,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops.ragged import ragged_array_ops
+from tensorflow.python.ops.ragged import ragged_concat_ops
 from tensorflow.python.ops.ragged import ragged_factory_ops
 from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
@@ -235,7 +235,7 @@ class RaggedConcatOpTest(ragged_test_util.RaggedTensorTestCase,
                        expected_ragged_rank=None,
                        expected_shape=None):
     rt_inputs = self._rt_inputs_to_tensors(rt_inputs, ragged_ranks)
-    concatenated = ragged_array_ops.concat(rt_inputs, axis)
+    concatenated = ragged_concat_ops.concat(rt_inputs, axis)
     if expected_ragged_rank is not None:
       self.assertEqual(concatenated.ragged_rank, expected_ragged_rank)
     if expected_shape is not None:
@@ -276,7 +276,7 @@ class RaggedConcatOpTest(ragged_test_util.RaggedTensorTestCase,
                       message=None,
                       ragged_ranks=None):
     rt_inputs = self._rt_inputs_to_tensors(rt_inputs, ragged_ranks)
-    self.assertRaisesRegexp(error, message, ragged_array_ops.concat, rt_inputs,
+    self.assertRaisesRegexp(error, message, ragged_concat_ops.concat, rt_inputs,
                             axis)
 
   @parameterized.parameters([
@@ -294,7 +294,7 @@ class RaggedConcatOpTest(ragged_test_util.RaggedTensorTestCase,
     rt_inputs = [
         array_ops.placeholder_with_default(rt, shape=None) for rt in rt_inputs
     ]
-    concatenated = ragged_array_ops.concat(rt_inputs, axis)
+    concatenated = ragged_concat_ops.concat(rt_inputs, axis)
     with self.assertRaisesRegexp(error, message):
       self.evaluate(concatenated)
 
@@ -307,7 +307,7 @@ class RaggedConcatOpTest(ragged_test_util.RaggedTensorTestCase,
     ]
     self.assertRaisesRegexp(
         ValueError, r'axis may only be negative if ndims is statically known.',
-        ragged_array_ops.concat, rt_inputs, -1)
+        ragged_concat_ops.concat, rt_inputs, -1)
 
   def testSingleTensorInput(self):
     """Tests ragged_concat with a single tensor input.
@@ -317,7 +317,7 @@ class RaggedConcatOpTest(ragged_test_util.RaggedTensorTestCase,
     returns that tensor.  This test exercises that path.
     """
     rt_inputs = ragged_factory_ops.constant([[1, 2], [3, 4]])
-    concatenated = ragged_array_ops.concat(rt_inputs, 0)
+    concatenated = ragged_concat_ops.concat(rt_inputs, 0)
     self.assertRaggedEqual(concatenated, [[1, 2], [3, 4]])
 
 
diff --git a/tensorflow/python/ops/ragged/ragged_concat_ops.py b/tensorflow/python/ops/ragged/ragged_concat_ops.py
new file mode 100644
index 0000000000..0f86b05e17
--- /dev/null
+++ b/tensorflow/python/ops/ragged/ragged_concat_ops.py
@@ -0,0 +1,302 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Concat and stack operations for RaggedTensors."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import check_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.ragged import ragged_array_ops
+from tensorflow.python.ops.ragged import ragged_conversion_ops
+from tensorflow.python.ops.ragged import ragged_gather_ops
+from tensorflow.python.ops.ragged import ragged_tensor
+from tensorflow.python.ops.ragged import ragged_util
+
+
+def concat(values, axis, name=None):
+  """Concatenates potentially ragged tensors along one dimension.
+
+  Given a list of tensors with the same rank `K` (`K >= axis`), returns a
+  rank-`K` `RaggedTensor` `result` such that `result[i0...iaxis]` is the
+  concatenation of `[rt[i0...iaxis] for rt in values]`.
+
+  Args:
+    values: A list of potentially ragged tensors.  May not be empty. All
+      `values` must have the same rank and the same dtype; but unlike
+      `tf.concat`, they can have arbitrary shapes.
+    axis: A python integer, indicating the dimension along which to concatenate.
+      (Note: Unlike `tf.concat`, the `axis` parameter must be statically known.)
+        Negative values are supported only if the rank of at least one
+        `values` value is statically known.
+    name: A name prefix for the returned tensor (optional).
+
+  Returns:
+    A `RaggedTensor` with rank `K`.
+    `result.ragged_rank=max(axis, max(rt.ragged_rank for rt in values]))`.
+
+  Raises:
+    ValueError: If `values` is empty, if `axis` is out of bounds or if
+      the input tensors have different ranks.
+
+  #### Example:
+    ```python
+    >>> t1 = tf.ragged.constant([[1, 2], [3, 4, 5]])
+    >>> t2 = tf.ragged.constant([[6], [7, 8, 9]])
+    >>> ragged.concat([t1, t2], axis=0)
+    [[1, 2], [3, 4, 5], [6], [7, 8, 9]]
+    >>> ragged.concat([t1, t2], axis=1)
+    [[1, 2, 6], [3, 4, 5, 7, 8, 9]]
+    ```
+  """
+  if not isinstance(values, (list, tuple)):
+    values = [values]
+  with ops.name_scope(name, 'RaggedConcat', values):
+    return _ragged_stack_concat_helper(values, axis, stack_values=False)
+
+
+def stack(values, axis=0, name=None):
+  """Stacks potentially ragged tensors along one dimension.
+
+  Given a list of tensors with the same rank `K` (`K >= axis`), returns a
+  rank-`K+1` `RaggedTensor` `result` such that `result[i0...iaxis]` is the
+  list `[rt[i0...iaxis] for rt in values]`.
+
+  Args:
+    values: A list of potentially ragged tensors.  May not be empty. All
+      `values` must have the same rank and the same dtype; but unlike
+      `tf.concat`, they can have arbitrary shapes.
+    axis: A python integer, indicating the dimension along which to stack.
+      (Note: Unlike `tf.stack`, the `axis` parameter must be statically known.)
+        Negative values are supported only if the rank of at least one
+        `values` value is statically known.
+    name: A name prefix for the returned tensor (optional).
+
+  Returns:
+    A `RaggedTensor` with rank `K+1`.
+    `result.ragged_rank=max(axis, max(rt.ragged_rank for rt in values]))`.
+
+  Raises:
+    ValueError: If `values` is empty, if `axis` is out of bounds or if
+      the input tensors have different ranks.
+
+  #### Example:
+    ```python
+    >>> t1 = tf.ragged.constant([[1, 2], [3, 4, 5]])
+    >>> t2 = tf.ragged.constant([[6], [7, 8, 9]])
+    >>> ragged.stack([t1, t2], axis=0)
+    [[[1, 2], [3, 4, 5]], [[6], [7, 9, 0]]]
+    >>> ragged.stack([t1, t2], axis=1)
+    [[[1, 2], [6]], [[3, 4, 5], [7, 8, 9]]]
+    ```
+  """
+  if not isinstance(values, (list, tuple)):
+    values = [values]
+  with ops.name_scope(name, 'RaggedConcat', values):
+    return _ragged_stack_concat_helper(values, axis, stack_values=True)
+
+
+def _ragged_stack_concat_helper(rt_inputs, axis, stack_values):
+  """Helper function to concatenate or stack ragged tensors.
+
+  Args:
+    rt_inputs: A list of RaggedTensors or Tensors to combine.
+    axis: The axis along which to concatenate or stack.
+    stack_values: A boolean -- if true, then stack values; otherwise,
+      concatenate them.
+
+  Returns:
+    A RaggedTensor.
+  Raises:
+    ValueError: If rt_inputs is empty, or if axis is out of range.
+  """
+  # Validate parameters.
+  if not rt_inputs:
+    raise ValueError('rt_inputs may not be empty.')
+
+  # Convert input tensors.
+  rt_inputs = [
+      ragged_tensor.convert_to_tensor_or_ragged_tensor(
+          rt_input, name='rt_input') for rt_input in rt_inputs
+  ]
+
+  # Special case: if there's only one input, then return it as-is.
+  if len(rt_inputs) == 1:
+    if stack_values:
+      return ragged_array_ops.expand_dims(rt_inputs[0], axis=axis)
+    else:
+      return rt_inputs[0]
+
+  # Check the rank (number of dimensions) of the input tensors.
+  ndims = None
+  for rt in rt_inputs:
+    if ndims is None:
+      ndims = rt.shape.ndims
+    else:
+      rt.shape.assert_has_rank(ndims)
+
+  out_ndims = ndims if (ndims is None or not stack_values) else ndims + 1
+  axis = ragged_util.get_positive_axis(axis, out_ndims)
+
+  # If all the inputs are Tensors, and we're combining the final dimension,
+  # then we can delegate to the tf.stack/tf.concat operation, and return a
+  # Tensor.
+  if all(not ragged_tensor.is_ragged(rt) for rt in rt_inputs):
+    if ndims is not None and (axis == out_ndims - 1 or axis == ndims - 1):
+      if stack_values:
+        return array_ops.stack(rt_inputs, axis)
+      else:
+        return array_ops.concat(rt_inputs, axis)
+
+  # Convert any Tensor inputs to RaggedTensors.  This makes it
+  # possible to concatenate Tensors and RaggedTensors together.
+  for i in range(len(rt_inputs)):
+    if not ragged_tensor.is_ragged(rt_inputs[i]):
+      rt_inputs[i] = ragged_conversion_ops.from_tensor(
+          rt_inputs[i], ragged_rank=1)
+
+  # Convert the input tensors to all have the same ragged_rank.
+  ragged_rank = max(max(rt.ragged_rank for rt in rt_inputs), 1)
+  rt_inputs = [_increase_ragged_rank_to(rt, ragged_rank) for rt in rt_inputs]
+
+  if axis == 0:
+    return _ragged_stack_concat_axis_0(rt_inputs, stack_values)
+  elif axis == 1:
+    return _ragged_stack_concat_axis_1(rt_inputs, stack_values)
+  else:  # axis > 1: recurse.
+    values = [rt.values for rt in rt_inputs]
+    splits = [[rt_input.row_splits] for rt_input in rt_inputs]
+    with ops.control_dependencies(ragged_util.assert_splits_match(splits)):
+      return ragged_tensor.RaggedTensor.from_row_splits(
+          _ragged_stack_concat_helper(values, axis - 1, stack_values),
+          splits[0][0])
+
+
+def _ragged_stack_concat_axis_0(rt_inputs, stack_values):
+  """Helper function to concatenate or stack ragged tensors along axis 0.
+
+  Args:
+    rt_inputs: A list of RaggedTensors, all with the same rank and ragged_rank.
+    stack_values: Boolean.  If true, then stack values; otherwise, concatenate
+      them.
+
+  Returns:
+    A RaggedTensor.
+  """
+  # Concatenate the inner values together.
+  flat_values = [rt.flat_values for rt in rt_inputs]
+  concatenated_flat_values = array_ops.concat(flat_values, axis=0)
+
+  # Concatenate the splits together for each ragged dimension (adjusting
+  # split offsets as necessary).
+  nested_splits = [rt.nested_row_splits for rt in rt_inputs]
+  ragged_rank = rt_inputs[0].ragged_rank
+  concatenated_nested_splits = [
+      _concat_ragged_splits([ns[dim]
+                             for ns in nested_splits])
+      for dim in range(ragged_rank)
+  ]
+
+  # If we are performing a stack operation, then add another splits.
+  if stack_values:
+    stack_lengths = array_ops.stack([rt.nrows() for rt in rt_inputs])
+    stack_splits = ragged_util.lengths_to_splits(stack_lengths)
+    concatenated_nested_splits.insert(0, stack_splits)
+
+  return ragged_tensor.RaggedTensor.from_nested_row_splits(
+      concatenated_flat_values, concatenated_nested_splits)
+
+
+def _ragged_stack_concat_axis_1(rt_inputs, stack_values):
+  """Helper function to concatenate or stack ragged tensors along axis 1.
+
+  Args:
+    rt_inputs: A list of RaggedTensors, all with the same rank and ragged_rank.
+    stack_values: Boolean.  If true, then stack values; otherwise, concatenate
+      them.
+
+  Returns:
+    A RaggedTensor.
+  """
+  num_inputs = len(rt_inputs)
+
+  rt_nrows = rt_inputs[0].nrows()
+  nrows_msg = 'Input tensors have incompatible shapes.'
+  nrows_checks = [
+      check_ops.assert_equal(rt.nrows(), rt_nrows, message=nrows_msg)
+      for rt in rt_inputs[1:]
+  ]
+
+  with ops.control_dependencies(nrows_checks):
+    # Concatentate the inputs together to put them in a single ragged tensor.
+    concatenated_rt = _ragged_stack_concat_axis_0(rt_inputs, stack_values=False)
+
+    # Use ragged.gather to permute the rows of concatenated_rt.  In particular,
+    #   permuted_rt = [rt_inputs[0][0], ..., rt_inputs[N][0],
+    #                  rt_inputs[0][1], ..., rt_inputs[N][1],
+    #                      ...,
+    #                  rt_inputs[0][M], ..., rt_input[N][M]]
+    # where `N=num_inputs-1` and `M=rt_nrows-1`.
+    row_indices = math_ops.range(rt_nrows * num_inputs)
+    row_index_matrix = array_ops.reshape(row_indices, [num_inputs, -1])
+    transposed_row_index_matrix = array_ops.transpose(row_index_matrix)
+    row_permutation = array_ops.reshape(transposed_row_index_matrix, [-1])
+    permuted_rt = ragged_gather_ops.gather(concatenated_rt, row_permutation)
+
+    if stack_values:
+      # Add a new splits tensor to group together the values.
+      stack_splits = math_ops.range(0, rt_nrows * num_inputs + 1, num_inputs)
+      _copy_row_shape(rt_inputs, stack_splits)
+      return ragged_tensor.RaggedTensor.from_row_splits(permuted_rt,
+                                                        stack_splits)
+    else:
+      # Merge together adjacent rows by dropping the row-split indices that
+      # separate them.
+      concat_splits = permuted_rt.row_splits[::num_inputs]
+      _copy_row_shape(rt_inputs, concat_splits)
+      return ragged_tensor.RaggedTensor.from_row_splits(permuted_rt.values,
+                                                        concat_splits)
+
+
+def _copy_row_shape(rt_inputs, splits):
+  """Sets splits.shape to [rt[shape[0]+1] for each rt in rt_inputs."""
+  for rt in rt_inputs:
+    if rt.shape[0] is not None:
+      splits.set_shape(tensor_shape.TensorShape(rt.shape[0] + 1))
+
+
+def _increase_ragged_rank_to(rt_input, ragged_rank):
+  """Adds ragged dimensions to `rt_input` so it has the desired ragged rank."""
+  if ragged_rank > 0:
+    if not ragged_tensor.is_ragged(rt_input):
+      rt_input = ragged_conversion_ops.from_tensor(rt_input)
+    if rt_input.ragged_rank < ragged_rank:
+      rt_input = rt_input.with_values(
+          _increase_ragged_rank_to(rt_input.values, ragged_rank - 1))
+  return rt_input
+
+
+def _concat_ragged_splits(splits_list):
+  """Concatenates a list of RaggedTensor splits to form a single splits."""
+  pieces = [splits_list[0]]
+  splits_offset = splits_list[0][-1]
+  for splits in splits_list[1:]:
+    pieces.append(splits[1:] + splits_offset)
+    splits_offset += splits[-1]
+  return array_ops.concat(pieces, axis=0)
diff --git a/tensorflow/python/ops/ragged/ragged_dispatch.py b/tensorflow/python/ops/ragged/ragged_dispatch.py
index 5284750198..0c9e6efe8b 100644
--- a/tensorflow/python/ops/ragged/ragged_dispatch.py
+++ b/tensorflow/python/ops/ragged/ragged_dispatch.py
@@ -32,10 +32,14 @@ from tensorflow.python.ops import parsing_ops
 from tensorflow.python.ops import string_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.ops.ragged import ragged_array_ops
+from tensorflow.python.ops.ragged import ragged_batch_gather_ops
+from tensorflow.python.ops.ragged import ragged_concat_ops
+from tensorflow.python.ops.ragged import ragged_gather_ops
 from tensorflow.python.ops.ragged import ragged_math_ops
 from tensorflow.python.ops.ragged import ragged_tensor
 from tensorflow.python.ops.ragged import ragged_tensor_shape
 from tensorflow.python.ops.ragged import ragged_util
+from tensorflow.python.ops.ragged import ragged_where_op
 from tensorflow.python.util import dispatch
 from tensorflow.python.util import tf_decorator
 from tensorflow.python.util import tf_export
@@ -401,7 +405,7 @@ _V1_OPS_THAT_DELEGATE_TO_V2_OPS = [
 
 def _ragged_gather_v1(params, indices, validate_indices=None, name=None,
                       axis=0, batch_dims=0):
-  return ragged_array_ops.gather(
+  return ragged_gather_ops.gather(
       params=params,
       indices=indices,
       validate_indices=validate_indices,
@@ -422,20 +426,20 @@ def _ragged_size_v1(input, name=None, out_type=dtypes.int32):  # pylint: disable
 
 # (original_op, ragged_op, ragged_args)
 _RAGGED_DISPATCH_OPS = [
-    (array_ops.batch_gather, ragged_array_ops.batch_gather,
+    (array_ops.batch_gather, ragged_batch_gather_ops.batch_gather,
      ['params', 'indices']),
-    (array_ops.concat, ragged_array_ops.concat, ['[values]']),
+    (array_ops.concat, ragged_concat_ops.concat, ['[values]']),
     (array_ops.expand_dims, _ragged_expand_dims_v1, ['input']),
     (array_ops.expand_dims_v2, ragged_array_ops.expand_dims, ['input']),
     (array_ops.gather, _ragged_gather_v1, ['params', 'indices']),
-    (array_ops.gather_v2, ragged_array_ops.gather, ['params', 'indices']),
-    (array_ops.gather_nd, ragged_array_ops.gather_nd, ['params', 'indices']),
+    (array_ops.gather_v2, ragged_gather_ops.gather, ['params', 'indices']),
+    (array_ops.gather_nd, ragged_gather_ops.gather_nd, ['params', 'indices']),
     (array_ops.rank, ragged_array_ops.rank, ['input']),
     (array_ops.size, _ragged_size_v1, ['input']),
     (array_ops.size_v2, ragged_array_ops.size, ['input']),
-    (array_ops.stack, ragged_array_ops.stack, ['[values]']),
+    (array_ops.stack, ragged_concat_ops.stack, ['[values]']),
     (array_ops.tile, ragged_array_ops.tile, ['input']),
-    (array_ops.where, ragged_array_ops.where, ['condition', 'x', 'y']),
+    (array_ops.where, ragged_where_op.where, ['condition', 'x', 'y']),
     (math_ops.unsorted_segment_sum, ragged_math_ops.segment_sum,
      ['data', 'segment_ids']),
     (math_ops.unsorted_segment_prod, ragged_math_ops.segment_prod,
diff --git a/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py b/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py
index d4bffeb401..8e44368d47 100644
--- a/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for ragged_array_ops.gather_nd."""
+"""Tests for ragged_gather_ops.gather_nd."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -26,8 +26,8 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops.ragged import ragged_array_ops
 from tensorflow.python.ops.ragged import ragged_factory_ops
+from tensorflow.python.ops.ragged import ragged_gather_ops
 from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
@@ -201,7 +201,7 @@ class RaggedGatherNdOpTest(ragged_test_util.RaggedTensorTestCase,
           expected=[[b'c', b'd'], [b'a', b'b'], [b'e', b'f']]),
   ])  # pyformat: disable
   def testRaggedGatherNd(self, descr, params, indices, expected):
-    result = ragged_array_ops.gather_nd(params, indices)
+    result = ragged_gather_ops.gather_nd(params, indices)
     self.assertRaggedEqual(result, expected)
 
   def testRaggedGatherNdUnknownRankError(self):
@@ -213,10 +213,10 @@ class RaggedGatherNdOpTest(ragged_test_util.RaggedTensorTestCase,
 
     with self.assertRaisesRegexp(ValueError,
                                  'indices.rank be statically known.'):
-      ragged_array_ops.gather_nd(params, indices1)
+      ragged_gather_ops.gather_nd(params, indices1)
     with self.assertRaisesRegexp(
         ValueError, r'indices.shape\[-1\] must be statically known.'):
-      ragged_array_ops.gather_nd(params, indices2)
+      ragged_gather_ops.gather_nd(params, indices2)
 
   @parameterized.parameters([
       dict(
@@ -238,7 +238,7 @@ class RaggedGatherNdOpTest(ragged_test_util.RaggedTensorTestCase,
                                     message=None,
                                     error=ValueError):
     with self.assertRaisesRegexp(error, message):
-      ragged_array_ops.gather_nd(params, indices)
+      ragged_gather_ops.gather_nd(params, indices)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/ops/ragged/ragged_gather_op_test.py b/tensorflow/python/ops/ragged/ragged_gather_op_test.py
index 9914b56448..eb64bb4ad1 100644
--- a/tensorflow/python/ops/ragged/ragged_gather_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_gather_op_test.py
@@ -17,7 +17,6 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
-
 from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -25,8 +24,8 @@ from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops.ragged import ragged_array_ops
 from tensorflow.python.ops.ragged import ragged_factory_ops
+from tensorflow.python.ops.ragged import ragged_gather_ops
 from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
@@ -41,35 +40,35 @@ class RaggedGatherOpTest(ragged_test_util.RaggedTensorTestCase):
                                                  ['e']])
     ragged_indices = ragged_factory_ops.constant([[3, 1, 2], [1], [], [0]])
     self.assertRaggedEqual(
-        ragged_array_ops.gather(params, ragged_indices),
+        ragged_gather_ops.gather(params, ragged_indices),
         [[b'd', b'b', b'c'], [b'b'], [], [b'a']])
     self.assertRaggedEqual(
-        ragged_array_ops.gather(ragged_params, indices),
+        ragged_gather_ops.gather(ragged_params, indices),
         [[b'e'], [b'd'], [], [b'd'], [b'a', b'b', b'c']])
     self.assertRaggedEqual(
-        ragged_array_ops.gather(ragged_params, ragged_indices),
+        ragged_gather_ops.gather(ragged_params, ragged_indices),
         [[[b'e'], [b'd'], []], [[b'd']], [], [[b'a', b'b', b'c']]])
 
   def testTensorParamsAndTensorIndices(self):
     params = ['a', 'b', 'c', 'd', 'e']
     indices = [2, 0, 2, 1]
     self.assertRaggedEqual(
-        ragged_array_ops.gather(params, indices), [b'c', b'a', b'c', b'b'])
-    self.assertIsInstance(ragged_array_ops.gather(params, indices), ops.Tensor)
+        ragged_gather_ops.gather(params, indices), [b'c', b'a', b'c', b'b'])
+    self.assertIsInstance(ragged_gather_ops.gather(params, indices), ops.Tensor)
 
   def testRaggedParamsAndTensorIndices(self):
     params = ragged_factory_ops.constant([['a', 'b'], ['c', 'd', 'e'], ['f'],
                                           [], ['g']])
     indices = [2, 0, 2, 1]
     self.assertRaggedEqual(
-        ragged_array_ops.gather(params, indices),
+        ragged_gather_ops.gather(params, indices),
         [[b'f'], [b'a', b'b'], [b'f'], [b'c', b'd', b'e']])
 
   def testTensorParamsAndRaggedIndices(self):
     params = ['a', 'b', 'c', 'd', 'e']
     indices = ragged_factory_ops.constant([[2, 1], [1, 2, 0], [3]])
     self.assertRaggedEqual(
-        ragged_array_ops.gather(params, indices),
+        ragged_gather_ops.gather(params, indices),
         [[b'c', b'b'], [b'b', b'c', b'a'], [b'd']])
 
   def testRaggedParamsAndRaggedIndices(self):
@@ -77,7 +76,7 @@ class RaggedGatherOpTest(ragged_test_util.RaggedTensorTestCase):
                                           [], ['g']])
     indices = ragged_factory_ops.constant([[2, 1], [1, 2, 0], [3]])
     self.assertRaggedEqual(
-        ragged_array_ops.gather(params, indices),
+        ragged_gather_ops.gather(params, indices),
         [[[b'f'], [b'c', b'd', b'e']],                # [[p[2], p[1]      ],
          [[b'c', b'd', b'e'], [b'f'], [b'a', b'b']],  #  [p[1], p[2], p[0]],
          [[]]]                                        #  [p[3]            ]]
@@ -88,14 +87,14 @@ class RaggedGatherOpTest(ragged_test_util.RaggedTensorTestCase):
                                           [], ['g']])
     indices = 1
     self.assertRaggedEqual(
-        ragged_array_ops.gather(params, indices), [b'c', b'd', b'e'])
+        ragged_gather_ops.gather(params, indices), [b'c', b'd', b'e'])
 
   def test3DRaggedParamsAnd2DTensorIndices(self):
     params = ragged_factory_ops.constant([[['a', 'b'], []],
                                           [['c', 'd'], ['e'], ['f']], [['g']]])
     indices = [[1, 2], [0, 1], [2, 2]]
     self.assertRaggedEqual(
-        ragged_array_ops.gather(params, indices),
+        ragged_gather_ops.gather(params, indices),
         [[[[b'c', b'd'], [b'e'], [b'f']], [[b'g']]],            # [[p1, p2],
          [[[b'a', b'b'], []], [[b'c', b'd'], [b'e'], [b'f']]],  #  [p0, p1],
          [[[b'g']], [[b'g']]]]                                  #  [p2, p2]]
@@ -109,7 +108,7 @@ class RaggedGatherOpTest(ragged_test_util.RaggedTensorTestCase):
         inner_shape=(2,))
     params = ['a', 'b', 'c', 'd', 'e', 'f', 'g']
     self.assertRaggedEqual(
-        ragged_array_ops.gather(params, indices),
+        ragged_gather_ops.gather(params, indices),
         [[[[b'd', b'e'], [b'a', b'g']], []],
          [[[b'c', b'b'], [b'b', b'a']], [[b'c', b'f']], [[b'c', b'd']]],
          [[[b'b', b'a']]]])  # pyformat: disable
@@ -121,13 +120,13 @@ class RaggedGatherOpTest(ragged_test_util.RaggedTensorTestCase):
     ragged_indices = ragged_factory_ops.constant([[0, 3]])
     with self.assertRaisesRegexp(errors.InvalidArgumentError,
                                  r'indices\[1\] = 3 is not in \[0, 3\)'):
-      self.evaluate(ragged_array_ops.gather(tensor_params, ragged_indices))
+      self.evaluate(ragged_gather_ops.gather(tensor_params, ragged_indices))
     with self.assertRaisesRegexp(errors.InvalidArgumentError,
                                  r'indices\[2\] = 2 is not in \[0, 2\)'):
-      self.evaluate(ragged_array_ops.gather(ragged_params, tensor_indices))
+      self.evaluate(ragged_gather_ops.gather(ragged_params, tensor_indices))
     with self.assertRaisesRegexp(errors.InvalidArgumentError,
                                  r'indices\[1\] = 3 is not in \[0, 2\)'):
-      self.evaluate(ragged_array_ops.gather(ragged_params, ragged_indices))
+      self.evaluate(ragged_gather_ops.gather(ragged_params, ragged_indices))
 
   def testUnknownIndicesRankError(self):
     if context.executing_eagerly():
@@ -137,7 +136,7 @@ class RaggedGatherOpTest(ragged_test_util.RaggedTensorTestCase):
     indices = array_ops.placeholder_with_default(indices, None)
     self.assertRaisesRegexp(ValueError,
                             r'indices\.shape\.ndims must be known statically',
-                            ragged_array_ops.gather, params, indices)
+                            ragged_gather_ops.gather, params, indices)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/ops/ragged/ragged_gather_ops.py b/tensorflow/python/ops/ragged/ragged_gather_ops.py
new file mode 100644
index 0000000000..b49e0e549f
--- /dev/null
+++ b/tensorflow/python/ops/ragged/ragged_gather_ops.py
@@ -0,0 +1,258 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Gather operations for RaggedTensors."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gen_ragged_array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.ragged import ragged_array_ops
+from tensorflow.python.ops.ragged import ragged_conversion_ops
+from tensorflow.python.ops.ragged import ragged_tensor
+
+
+#===============================================================================
+# ragged_gather
+#===============================================================================
+# TODO(edloper): Add an `axis` argument
+def gather(params, indices, validate_indices=None, axis=0, batch_dims=0,
+           name=None):
+  """Gathers ragged slices from `params` axis `0` according to `indices`.
+
+  Returns `RaggedTensor` output, such that:
+
+  ```python
+  output.shape = indices.shape + params.shape[1:]
+  output.ragged_rank = indices.shape.ndims + params.ragged_rank
+  output[i...j, d0...dn] = params[indices[i...j], d0...dn]
+  ```
+
+  `params` may be ragged.  `indices` may be ragged.
+  `indices` must have dtype `int32` or `int64`. If any index is out of bounds,
+  then an error is returned.
+
+  Examples:
+
+  ```python
+  >>> params = tf.constant(['a', 'b', 'c', 'd', 'e'])
+  >>> indices = tf.constant([3, 1, 2, 1, 0])
+  >>> ragged_params = tf.ragged.constant([['a', 'b', 'c'], ['d'], [], ['e']])
+  >>> ragged_indices = tf.ragged.constant([[3, 1, 2], [1], [], [0]])
+
+  >>> print ragged.gather(params, ragged_indices)
+  [['d', 'b', 'c'], ['b'], [], ['a']]
+
+  >>> print ragged.gather(ragged_params, indices)
+  [['e'], ['d'], [], ['d'], ['a', 'b', 'c']]
+
+  >>> print ragged.gather(ragged_params, ragged_indices)
+  [[['e'], ['d'], []], [['d']], [], [['a', 'b', 'c']]]
+  ```
+
+  Args:
+    params: The potentially ragged tensor from which to gather values. Must be
+      at least rank 1.
+    indices: The potentially ragged tensor indicating which values to gather.
+      Must have dtype `int32` or `int64`.  Values must be in the range `[0,
+      params.shape[0]]`.
+    validate_indices: Ignored.
+    axis: Must be zero.
+    batch_dims: Must be zero.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `RaggedTensor`, where `output.dtype=params.dtype` and
+    `output.shape=indices.shape + params.shape[1:]` and
+    `output.ragged_rank=indices.shape.ndims + params.ragged_rank`.
+
+  Raises:
+    ValueError: If indices.shape.ndims is not known statically.
+  """
+  del validate_indices
+  if not isinstance(axis, int) or axis != 0:
+    raise ValueError('axis != 0 is not supported for ragged gather yet.')
+  if not isinstance(batch_dims, int) or batch_dims != 0:
+    raise ValueError('batch_dims != 0 is not supported for ragged gather yet.')
+  with ops.name_scope(name, 'RaggedGather', [params, indices]):
+    params = ragged_tensor.convert_to_tensor_or_ragged_tensor(
+        params, name='params')
+    indices = ragged_tensor.convert_to_tensor_or_ragged_tensor(
+        indices, name='indices')
+
+    if ragged_tensor.is_ragged(indices):
+      return indices.with_values(gather(params, indices.values))
+
+    if not ragged_tensor.is_ragged(params):
+      return array_ops.gather(params, indices)
+
+    indices = ops.convert_to_tensor(indices)
+    if indices.shape.ndims is None:
+      raise ValueError('indices.shape.ndims must be known statically')
+
+    result = gen_ragged_array_ops.ragged_gather(
+        indices=indices,
+        params_dense_values=params.flat_values,
+        params_nested_splits=params.nested_row_splits,
+        OUTPUT_RAGGED_RANK=indices.shape.ndims + len(params.nested_row_splits) -
+        1)
+
+    # Compose the RaggedTensor from splits & values.
+    return ragged_tensor.RaggedTensor.from_nested_row_splits(
+        result.output_dense_values, result.output_nested_splits)
+
+
+#===============================================================================
+# ragged.gather_nd
+#===============================================================================
+def gather_nd(params, indices, name=None):
+  """Gather slices from `params` using `n`-dimensional indices.
+
+  This operation is similar to `gather`, but it uses the innermost dimension
+  of `indices` to define a slice into `params`.  In particular, if:
+
+  * `indices` has shape `[A1...AN, I]`
+  * `params` has shape `[B1...BM]`
+
+  Then:
+
+  * `result` has shape `[A1...AN, B_{I+1}...BM]`.
+  * `result[a1...aN] = params[indices[a1...aN, :]]`
+
+  Args:
+    params: A potentially ragged tensor with shape `[A1...AN, I]`.
+    indices: A potentially ragged tensor with shape `[B1...BM]`.
+    name: A name for the operation (optional).
+
+  Returns:
+    A potentially ragged tensor with shape `[A1...AN, B_{I+1}...BM]`.
+
+  #### Examples:
+    ```python
+    >>> params = tf.ragged.constant_value(
+    ...     [ [ ['000', '001'], ['010'              ]          ],
+    ...       [ ['100'       ], ['110', '111', '112'], ['120'] ],
+    ...       [ [            ], ['210'              ]          ] ])
+
+    >>> # Gather 2D slices from a 3D tensor
+    >>> ragged.gather_nd(params, [[2], [0]])
+    [ [ [            ], ['210'] ]
+      [ ['000', '001'], ['010'] ] ]
+
+    >>> # Gather 1D slices from a 3D tensor
+    >>> ragged.gather_nd(params, [[2, 1], [0, 0]])
+    [['210'], ['000', '001']]
+
+    >>> # Gather scalars from a 3D tensor
+    >>> ragged.gather_nd(params, [[0, 0, 1], [1, 1, 2]])
+    ['001', '112']
+    ```
+  """
+  if not (ragged_tensor.is_ragged(params) or ragged_tensor.is_ragged(indices)):
+    return array_ops.gather_nd(params, indices, name)
+
+  with ops.name_scope(name, 'RaggedGatherNd', [params, indices]):
+
+    params = ragged_tensor.convert_to_tensor_or_ragged_tensor(
+        params, name='params')
+    indices = ragged_tensor.convert_to_tensor_or_ragged_tensor(
+        indices, name='indices')
+    indices_shape = indices.shape
+    indices_ndims = indices_shape.ndims
+    if indices_ndims is None:
+      raise ValueError('indices.rank be statically known.')
+    if indices_ndims == 0:
+      raise ValueError('indices.rank must be at least 1.')
+    if (ragged_tensor.is_ragged(indices) and
+        indices_ndims == indices.ragged_rank + 1):
+      raise ValueError('The innermost dimension of indices may not be ragged')
+
+    # `index_size` is the "n" in "gather_nd" -- i.e., the number of dimensions
+    # that each index slices into.
+    index_size = tensor_shape.dimension_value(indices_shape[-1])
+    if index_size is None:
+      raise ValueError('indices.shape[-1] must be statically known.')
+
+    # If `indices` has more than 2 dimensions, then recurse.  If `indices` is
+    # dense, then we convert it to ragged before recursing, and then convert
+    # the result back to `dense` if appropriate.
+    if indices_ndims > 2:
+      indices_is_dense = not ragged_tensor.is_ragged(indices)
+      if indices_is_dense:
+        indices = ragged_conversion_ops.from_tensor(
+            indices, ragged_rank=indices_ndims - 2)
+      result = indices.with_flat_values(gather_nd(params, indices.flat_values))
+      if (indices_is_dense and ragged_tensor.is_ragged(result) and
+          result.ragged_rank == indices_ndims - 2):
+        result = ragged_conversion_ops.to_tensor(result)
+      return result
+
+    # indices_ndims <= 2, and the innermost dimension of indices may not be
+    # ragged, so `indices` must not be ragged.
+    assert not ragged_tensor.is_ragged(indices)
+    assert ragged_tensor.is_ragged(params)
+
+    # Handle corner case: An empty index tuple selects the entire `params`
+    # value.  So if `index_size` is zero, then tile `params`.
+    if index_size == 0:
+      params_ndims = params.ragged_rank + array_ops.rank(params.flat_values)
+      for dim in range(indices_ndims - 1):
+        params = ragged_array_ops.expand_dims(params, axis=0)
+      multiples = array_ops.concat([
+          array_ops.shape(indices)[:-1],
+          array_ops.ones([params_ndims], dtypes.int32)
+      ],
+                                   axis=0)
+      return ragged_array_ops.tile(params, multiples)
+
+    # When index_size=1, we can just flatten the index tuples and use gather.
+    elif index_size == 1:
+      flattened_index_tuples = array_ops.reshape(indices, [-1])
+      return gather(params, flattened_index_tuples)
+
+    # Otherwise, params is a RaggedTensor, and indices is a 1D or 2D Tensor.
+    # Flatten both the index tuples and the params, such that the flattened
+    # index tuples point to the correct values in the flattened params; and
+    # then use ragged.gather on the flattened index tuples & params.
+    else:
+      indices = math_ops.to_int64(indices)
+
+      # Flatten the outermost 2 dimensions of the index tuples & params.
+      flattened_index_tuples = array_ops.gather(params.row_splits,
+                                                indices[..., 0])
+      flattened_index_tuples += indices[..., 1]
+      flattened_params = params.values
+
+      # Flatten any remaining dimensions.
+      for dim in range(2, index_size):
+        if not ragged_tensor.is_ragged(flattened_params):
+          flattened_index_tuples = array_ops.expand_dims(
+              flattened_index_tuples, axis=1)
+          flattened_index_tuples = array_ops.concat(
+              [flattened_index_tuples, indices[..., dim:]], axis=1)
+          return array_ops.gather_nd(flattened_params, flattened_index_tuples)
+
+        flattened_index_tuples = array_ops.gather(
+            flattened_params.row_starts(), flattened_index_tuples)
+        flattened_index_tuples += indices[..., dim]
+        flattened_params = flattened_params.values
+
+      # Gather using the flattened index tuples and params.
+      return gather(flattened_params, flattened_index_tuples)
diff --git a/tensorflow/python/ops/ragged/ragged_getitem.py b/tensorflow/python/ops/ragged/ragged_getitem.py
index 001a400596..1f1bb51253 100644
--- a/tensorflow/python/ops/ragged/ragged_getitem.py
+++ b/tensorflow/python/ops/ragged/ragged_getitem.py
@@ -23,7 +23,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
-from tensorflow.python.ops.ragged import ragged_array_ops
+from tensorflow.python.ops.ragged import ragged_gather_ops
 from tensorflow.python.ops.ragged import ragged_math_ops
 from tensorflow.python.ops.ragged import ragged_tensor
 
@@ -344,7 +344,7 @@ def _build_ragged_tensor_from_value_ranges(starts, limits, step, values):
 
   # Use `ragged_gather` or `array_ops.gather` to collect the values.
   if isinstance(values, ragged_tensor.RaggedTensor):
-    gathered_values = ragged_array_ops.gather(
+    gathered_values = ragged_gather_ops.gather(
         params=values, indices=value_indices.values)
   else:
     gathered_values = array_ops.gather(
diff --git a/tensorflow/python/ops/ragged/ragged_stack_op_test.py b/tensorflow/python/ops/ragged/ragged_stack_op_test.py
index b2b96a9465..e52ad4de20 100644
--- a/tensorflow/python/ops/ragged/ragged_stack_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_stack_op_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for ragged_array_ops.stack."""
+"""Tests for ragged_concat_ops.stack."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -22,7 +22,7 @@ from absl.testing import parameterized
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import test_util
-from tensorflow.python.ops.ragged import ragged_array_ops
+from tensorflow.python.ops.ragged import ragged_concat_ops
 from tensorflow.python.ops.ragged import ragged_factory_ops
 from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
@@ -330,7 +330,7 @@ class RaggedStackOpTest(ragged_test_util.RaggedTensorTestCase,
         if rrank != 0 else constant_op.constant(rt_input)
         for (rt_input, rrank) in zip(rt_inputs, ragged_ranks)
     ]
-    stacked = ragged_array_ops.stack(rt_inputs, axis)
+    stacked = ragged_concat_ops.stack(rt_inputs, axis)
     if expected_ragged_rank is not None:
       self.assertEqual(stacked.ragged_rank, expected_ragged_rank)
     if expected_shape is not None:
@@ -360,7 +360,7 @@ class RaggedStackOpTest(ragged_test_util.RaggedTensorTestCase,
           message='axis=3 out of bounds: expected -3<=axis<3'),
   )
   def testError(self, rt_inputs, axis, error, message):
-    self.assertRaisesRegexp(error, message, ragged_array_ops.stack, rt_inputs,
+    self.assertRaisesRegexp(error, message, ragged_concat_ops.stack, rt_inputs,
                             axis)
 
   def testSingleTensorInput(self):
@@ -371,7 +371,7 @@ class RaggedStackOpTest(ragged_test_util.RaggedTensorTestCase,
     equivalent to expand_dims(axis=0).  This test exercises that path.
     """
     rt_inputs = ragged_factory_ops.constant([[1, 2], [3, 4]])
-    stacked = ragged_array_ops.stack(rt_inputs, 0)
+    stacked = ragged_concat_ops.stack(rt_inputs, 0)
     self.assertRaggedEqual(stacked, [[[1, 2], [3, 4]]])
 
 
diff --git a/tensorflow/python/ops/ragged/ragged_where_op.py b/tensorflow/python/ops/ragged/ragged_where_op.py
new file mode 100644
index 0000000000..d60ee49ee8
--- /dev/null
+++ b/tensorflow/python/ops/ragged/ragged_where_op.py
@@ -0,0 +1,166 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""where operation for RaggedTensors."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.ragged import ragged_concat_ops
+from tensorflow.python.ops.ragged import ragged_functional_ops
+from tensorflow.python.ops.ragged import ragged_gather_ops
+from tensorflow.python.ops.ragged import ragged_tensor
+
+
+def where(condition, x=None, y=None, name=None):
+  """Return the elements, either from `x` or `y`, depending on the `condition`.
+
+  : If both `x` and `y` are `None`:
+    Returns the coordinates of true elements of `condition`. The coordinates
+    are returned in a 2-D tensor with shape
+    `[num_true_values, dim_size(condition)]`, where `result[i]` is the
+    coordinates of the `i`th true value (in row-major order).
+
+  : If both `x` and `y` are non-`None`:
+    Returns a tensor formed by selecting values from `x` where condition is
+    true, and from `y` when condition is false.  In particular:
+
+    : If `condition`, `x`, and `y` all have the same shape:
+
+      * `result[i1...iN] = x[i1...iN]` if `condition[i1...iN]` is true.
+      * `result[i1...iN] = y[i1...iN]` if `condition[i1...iN]` is false.
+
+    : Otherwise:
+
+      * `condition` must be a vector.
+      * `x` and `y` must have the same number of dimensions.
+      * The outermost dimensions of `condition`, `x`, and `y` must all have the
+        same size.
+      * `result[i] = x[i]` if `condition[i]` is true.
+      * `result[i] = y[i]` if `condition[i]` is false.
+
+  Args:
+    condition: A potentially ragged tensor of type `bool`
+    x: A potentially ragged tensor (optional).
+    y: A potentially ragged tensor (optional).  Must be specified if `x` is
+      specified.  Must have the same rank and type as `x`.
+    name: A name of the operation (optional)
+
+  Returns:
+    : If both `x` and `y` are `None`:
+      A `Tensor` with shape `(num_true, dim_size(condition))`.
+    : Otherwise:
+      A potentially ragged tensor with the same type, rank, and outermost
+      dimension size as `x` and `y`.
+      `result.ragged_rank = max(x.ragged_rank, y.ragged_rank)`.
+
+  Raises:
+    ValueError: When exactly one of `x` or `y` is non-`None`; or when
+      `condition`, `x`, and `y` have incompatible shapes.
+
+  #### Examples:
+    ```python
+    >>> # Coordinates where condition is true.
+    >>> condition = tf.ragged.constant_value(
+    ...     [[True, False, True], [False, True]])
+    >>> ragged.where(condition)
+    [[0, 0], [0, 2], [1, 1]]
+
+    >>> # Elementwise selection between x and y, based on condition.
+    >>> condition = tf.ragged.constant_value(
+    ...     [[True, False, True], [False, True]])
+    >>> x = tf.ragged.constant_value([['A', 'B', 'C'], ['D', 'E']])
+    >>> y = tf.ragged.constant_value([['a', 'b', 'c'], ['d', 'e']])
+    >>> ragged.where(condition, x, y)
+    [['A', 'b', 'C'], ['d', 'E']]
+
+    >>> # Row selection between x and y, based on condition.
+    >>> condition = [True, False]
+    >>> x = tf.ragged.constant_value([['A', 'B', 'C'], ['D', 'E']])
+    >>> y = tf.ragged.constant_value([['a', 'b', 'c'], ['d', 'e']])
+    >>> ragged.where(condition, x, y)
+    [['A', 'B', 'C'], ['d', 'e']]
+    ```
+  """
+  if (x is None) != (y is None):
+    raise ValueError('x and y must be either both None or both non-None')
+  with ops.name_scope('RaggedWhere', name, [condition, x, y]):
+    condition = ragged_tensor.convert_to_tensor_or_ragged_tensor(
+        condition, name='condition')
+    if x is None:
+      return _coordinate_where(condition)
+    else:
+      x = ragged_tensor.convert_to_tensor_or_ragged_tensor(x, name='x')
+      y = ragged_tensor.convert_to_tensor_or_ragged_tensor(y, name='y')
+      return _elementwise_where(condition, x, y)
+
+
+def _elementwise_where(condition, x, y):
+  """Ragged version of tf.where(condition, x, y)."""
+  condition_is_ragged = isinstance(condition, ragged_tensor.RaggedTensor)
+  x_is_ragged = isinstance(x, ragged_tensor.RaggedTensor)
+  y_is_ragged = isinstance(y, ragged_tensor.RaggedTensor)
+
+  if not (condition_is_ragged or x_is_ragged or y_is_ragged):
+    return array_ops.where(condition, x, y)
+
+  elif condition_is_ragged and x_is_ragged and y_is_ragged:
+    return ragged_functional_ops.map_flat_values(array_ops.where, condition, x,
+                                                 y)
+  elif not condition_is_ragged:
+    # Concatenate x and y, and then use `gather` to assemble the selected rows.
+    condition.shape.assert_has_rank(1)
+    x_nrows = _nrows(x)
+    x_and_y = ragged_concat_ops.concat([x, y], axis=0)
+    indices = array_ops.where(condition, math_ops.range(x_nrows),
+                              x_nrows + math_ops.range(_nrows(y)))
+    return ragged_gather_ops.gather(x_and_y, indices)
+
+  else:
+    raise ValueError('Input shapes do not match.')
+
+
+def _coordinate_where(condition):
+  """Ragged version of tf.where(condition)."""
+  if not isinstance(condition, ragged_tensor.RaggedTensor):
+    return array_ops.where(condition)
+
+  # The coordinate for each `true` value in condition.values.
+  selected_coords = _coordinate_where(condition.values)
+
+  # Convert the first index in each coordinate to a row index and column index.
+  first_index = selected_coords[:, 0]
+  selected_rows = array_ops.gather(condition.value_rowids(), first_index)
+  selected_row_starts = array_ops.gather(condition.row_splits, selected_rows)
+  selected_cols = first_index - selected_row_starts
+
+  # Assemble the row & column index with the indices for inner dimensions.
+  return array_ops.concat([
+      array_ops.expand_dims(selected_rows, 1),
+      array_ops.expand_dims(selected_cols, 1), selected_coords[:, 1:]
+  ],
+                          axis=1)
+
+
+def _nrows(rt_input, out_type=dtypes.int64, name=None):
+  if isinstance(rt_input, ragged_tensor.RaggedTensor):
+    return rt_input.nrows(out_type=out_type, name=name)
+  else:
+    with ops.name_scope(name, 'RaggedNRows', [rt_input]):
+      return array_ops.shape(rt_input, out_type=out_type)[0]
diff --git a/tensorflow/python/ops/ragged/ragged_where_op_test.py b/tensorflow/python/ops/ragged/ragged_where_op_test.py
index 3dd9565826..e76a04072a 100644
--- a/tensorflow/python/ops/ragged/ragged_where_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_where_op_test.py
@@ -17,13 +17,11 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
-
 from absl.testing import parameterized
-
 from tensorflow.python.framework import test_util
-from tensorflow.python.ops.ragged import ragged_array_ops
 from tensorflow.python.ops.ragged import ragged_factory_ops
 from tensorflow.python.ops.ragged import ragged_test_util
+from tensorflow.python.ops.ragged import ragged_where_op
 from tensorflow.python.platform import googletest
 
 
@@ -182,7 +180,7 @@ class RaggedWhereOpTest(ragged_test_util.RaggedTensorTestCase,
               [[[[], [b'A']]], [[[b'b']]]])),
   ])   # pyformat: disable
   def testRaggedWhere(self, condition, expected, x=None, y=None):
-    result = ragged_array_ops.where(condition, x, y)
+    result = ragged_where_op.where(condition, x, y)
     self.assertRaggedEqual(result, expected)
 
   @parameterized.parameters([
@@ -201,7 +199,7 @@ class RaggedWhereOpTest(ragged_test_util.RaggedTensorTestCase,
   ])
   def testRaggedWhereErrors(self, condition, error, message, x=None, y=None):
     with self.assertRaisesRegexp(error, message):
-      ragged_array_ops.where(condition, x, y)
+      ragged_where_op.where(condition, x, y)
 
 
 if __name__ == '__main__':
-- 
GitLab


From 2c3c999ef6a8a3045a7c03428f4d11350cd86e35 Mon Sep 17 00:00:00 2001
From: "Li, Guizi" <guizi.li@intel.com>
Date: Fri, 18 Jan 2019 15:07:35 +0800
Subject: [PATCH 0939/2345] smale changes on format

---
 tensorflow/core/graph/mkl_layout_pass_test.cc |  2 +-
 tensorflow/core/kernels/mkl_fused_ops_test.cc | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass_test.cc b/tensorflow/core/graph/mkl_layout_pass_test.cc
index 3ac25cdf02..32c40926ef 100644
--- a/tensorflow/core/graph/mkl_layout_pass_test.cc
+++ b/tensorflow/core/graph/mkl_layout_pass_test.cc
@@ -1542,7 +1542,7 @@ TEST_F(MklLayoutPassTest, Output_ControlEdge_PadWithFusedConv2D_Positive) {
 // Input node pointing to both Pad and _FusedConv2D
 // Output of both Pad and _FusedConv2D feeds one node (G as Output2)
 // A = input(as image), B = input(as paddings), C = Pad(A, B)
-// E = input(as bias), F = _FusedConv2D(C, A, E), G=Output(C, F)
+// E = input(as bias), F = _FusedConv2D(C, A, E), G = Output(C, F)
 // After layout pass - No merging, since Pad and _FusedConv2D both
 // feed to the same node (Z)
 TEST_F(MklLayoutPassTest, NodeMerge_PadWithFusedConv2D_Common_InOutput) {
diff --git a/tensorflow/core/kernels/mkl_fused_ops_test.cc b/tensorflow/core/kernels/mkl_fused_ops_test.cc
index f2527a675d..0d0a152491 100644
--- a/tensorflow/core/kernels/mkl_fused_ops_test.cc
+++ b/tensorflow/core/kernels/mkl_fused_ops_test.cc
@@ -176,14 +176,14 @@ class MklFusedConv2DOpTest : public OpsTestBase {
     int num_args = static_cast<int>(args.size());
 
     TF_EXPECT_OK(NodeDefBuilder("fused_conv_op", "_MklFusedConv2D")
-                     .Attr("T", dtype)
                      .Input(FakeInput(dtype))
                      .Input(FakeInput(dtype))
-                     .Attr("num_args", num_args)
                      .Input(FakeInput(num_args, dtype))
                      .Input(FakeInput(DT_UINT8))
                      .Input(FakeInput(DT_UINT8))
                      .Input(FakeInput(num_args, DT_UINT8))
+                     .Attr("T", dtype)
+                     .Attr("num_args", num_args)
                      .Attr("strides", {1, stride, stride, 1})
                      .Attr("padding", "SAME")
                      .Attr("fused_ops", fused_ops)
@@ -534,16 +534,16 @@ class MklPadWithFusedConv2DOpTest : public OpsTestBase {
                    padding_list_[3], 0, 0});
 
     TF_EXPECT_OK(NodeDefBuilder("pad_fused_conv_op", "_MklPadWithFusedConv2D")
-                     .Attr("T", dtype)
                      .Input(FakeInput(dtype))
                      .Input(FakeInput(dtype))
-                     .Attr("num_args", num_args)
                      .Input(FakeInput(num_args, dtype))
                      .Input(FakeInput(DT_INT32))
                      .Input(FakeInput(DT_UINT8))
                      .Input(FakeInput(DT_UINT8))
                      .Input(FakeInput(num_args, DT_UINT8))
                      .Input(FakeInput(DT_UINT8))
+                     .Attr("T", dtype)
+                     .Attr("num_args", num_args)
                      .Attr("strides", {1, stride, stride, 1})
                      .Attr("padding", "VALID")
                      .Attr("fused_ops", fused_ops)
@@ -558,7 +558,7 @@ class MklPadWithFusedConv2DOpTest : public OpsTestBase {
       AddInputFromArray<T>(arg.shape(), arg.flat<T>());
     AddInputFromArray<int32>(padding.shape(), padding.flat<int32>());
     // Add MKL meta input for input, filter, pad and agrs.
-    for (int i = 0; i < args.size() + 3; i++)
+    for (int i = 0; i < args.size() + 3; ++i)
       AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
     TF_ASSERT_OK(RunOpKernel());
 
-- 
GitLab


From f273e01060b158f09f20f82081e4e6761f729924 Mon Sep 17 00:00:00 2001
From: Denis Khalikov <dennis.khalikov@gmail.com>
Date: Fri, 18 Jan 2019 10:03:51 +0300
Subject: [PATCH 0940/2345] [XLA/AOT] Add check for --help command line option.

---
 tensorflow/compiler/aot/tfcompile_main.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/compiler/aot/tfcompile_main.cc b/tensorflow/compiler/aot/tfcompile_main.cc
index d548de8c44..0b6ab7e723 100644
--- a/tensorflow/compiler/aot/tfcompile_main.cc
+++ b/tensorflow/compiler/aot/tfcompile_main.cc
@@ -136,6 +136,10 @@ int main(int argc, char** argv) {
 
   tensorflow::string usage = tensorflow::tfcompile::kUsageHeader;
   usage += tensorflow::Flags::Usage(argv[0], flag_list);
+  if (argc > 1 && absl::string_view(argv[1]) == "--help") {
+    std::cerr << usage << "\n";
+    return 0;
+  }
   bool parsed_flags_ok = tensorflow::Flags::Parse(&argc, argv, flag_list);
   QCHECK(parsed_flags_ok) << "\n" << usage;
 
-- 
GitLab


From ff088a9e52e9085025c09e50a48fa9be7606f6da Mon Sep 17 00:00:00 2001
From: Dimitris Vardoulakis <dimvar@google.com>
Date: Thu, 17 Jan 2019 23:15:50 -0800
Subject: [PATCH 0941/2345] Fix some Clang-Tidy findings.

PiperOrigin-RevId: 229879297
---
 tensorflow/compiler/xla/service/heap_simulator.h              | 2 +-
 tensorflow/compiler/xla/service/pattern_matcher_gmock_test.cc | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/heap_simulator.h b/tensorflow/compiler/xla/service/heap_simulator.h
index dbbf43082f..3e0631aeb4 100644
--- a/tensorflow/compiler/xla/service/heap_simulator.h
+++ b/tensorflow/compiler/xla/service/heap_simulator.h
@@ -158,7 +158,7 @@ class HeapSimulator {
   void FillDebugTrace(HeapSimulatorTrace::Event::Kind kind,
                       const BufferValue* buffer,
                       const HloInstruction* instruction,
-                      const BufferValue* shared_with_canonical);
+                      const BufferValue* share_with_canonical);
 
   // Counterintuitive: the algorithm_ itself can be a NoFragmentationStatsHeap,
   // in which case we are calculating the same allocs/frees twice in the
diff --git a/tensorflow/compiler/xla/service/pattern_matcher_gmock_test.cc b/tensorflow/compiler/xla/service/pattern_matcher_gmock_test.cc
index 9ca2fb05c1..f51a18b138 100644
--- a/tensorflow/compiler/xla/service/pattern_matcher_gmock_test.cc
+++ b/tensorflow/compiler/xla/service/pattern_matcher_gmock_test.cc
@@ -23,7 +23,6 @@ namespace xla {
 namespace {
 
 namespace m = ::xla::match;
-using ::testing::Eq;
 using ::testing::Not;
 
 template <typename MatchedTy>
-- 
GitLab


From 9168465f29a39bd66cc0dd46f469a13fa33449f5 Mon Sep 17 00:00:00 2001
From: Denis Khalikov <dennis.khalikov@gmail.com>
Date: Tue, 15 Jan 2019 16:22:14 +0300
Subject: [PATCH 0942/2345] [XLA][Rematerialization] Add error message

---
 .../xla/service/hlo_rematerialization.cc      | 22 ++++++++++---------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_rematerialization.cc b/tensorflow/compiler/xla/service/hlo_rematerialization.cc
index ac74e2432f..32b88c09f8 100644
--- a/tensorflow/compiler/xla/service/hlo_rematerialization.cc
+++ b/tensorflow/compiler/xla/service/hlo_rematerialization.cc
@@ -179,7 +179,8 @@ class InstructionList {
   Item* CreateItem(HloInstruction* inst) {
     Item* item = new Item;
     item->instruction = inst;
-    CHECK(item_map_.insert({inst, item}).second) << "inserting inst twice";
+    CHECK(item_map_.insert({inst, item}).second)
+        << "inserting inst twice " << inst->name();
     return item;
   }
 
@@ -418,11 +419,12 @@ class MemoryUsageTracker {
   // the given uses.
   Buffer& RematerializeBuffer(const Buffer& original_buffer, Item* remat_item,
                               ItemList&& rematerialized_uses) {
-    CHECK(original_buffer.defining_instruction->placed);
-    CHECK(!original_buffer.has_indirect_uses);
-    CHECK(!original_buffer.live_out);
+    CHECK(original_buffer.defining_instruction->placed)
+        << original_buffer.defining_instruction->instruction->name();
+    CHECK(!original_buffer.has_indirect_uses) << original_buffer.ToString();
+    CHECK(!original_buffer.live_out) << original_buffer.ToString();
     for (Item* use : rematerialized_uses) {
-      CHECK(!use->placed);
+      CHECK(!use->placed) << use->instruction->name();
     }
     return NewBuffer(remat_item, original_buffer.size,
                      std::move(rematerialized_uses), /*live_out=*/false,
@@ -677,8 +679,8 @@ Status MemoryUsageTracker::AddRematerializedInstruction(Item* original_item,
           << ", remat_instruction = " << remat_item->instruction->name();
 
   TF_RET_CHECK(in_progress_item_ != nullptr);
-  TF_RET_CHECK(original_item->placed);
-  TF_RET_CHECK(!remat_item->placed);
+  TF_RET_CHECK(original_item->placed) << original_item->instruction->name();
+  TF_RET_CHECK(!remat_item->placed) << remat_item->instruction->name();
 
   // Construct the list of buffers used and defined by the rematerialization.
   remat_item->buffers_used = original_item->buffers_used;
@@ -707,7 +709,7 @@ Status MemoryUsageTracker::AddRematerializedInstruction(Item* original_item,
     ItemList unplaced_users;
     for (Item* user : old_buffer.users) {
       if (user->placed) {
-        CHECK(IsFinished(user));
+        CHECK(IsFinished(user)) << user->instruction->name();
         placed_users.push_back(user);
       } else {
         unplaced_users.push_back(user);
@@ -1094,7 +1096,7 @@ StatusOr<bool> HloRematerialization::RematerializeComputation(
         Item* successor_item = instruction_list.GetItem(successor);
         // Assert to make sure we never remat an operation with control
         // successor already placed.
-        CHECK(!successor_item->placed);
+        CHECK(!successor_item->placed) << successor_item->instruction->name();
         place_before.push_back(successor_item);
       }
       instruction_list.InsertBeforeInstructions(remat_item, place_before);
@@ -1164,7 +1166,7 @@ StatusOr<bool> HloRematerialization::RematerializeComputation(
   // Verify some invariants on the memory tracker.
   CHECK_EQ(memory_tracker.memory_usage(), 0);
   for (auto* instruction : computation->instructions()) {
-    CHECK(memory_tracker.IsPlaced(instruction));
+    CHECK(memory_tracker.IsPlaced(instruction)) << instruction->name();
   }
 
   VLOG(1) << "In computation " << computation->name() << " rematerialized "
-- 
GitLab


From dda435ef8413ac47cfd8992bb5eba912ab0cd281 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Fri, 18 Jan 2019 00:49:06 -0800
Subject: [PATCH 0943/2345] Improve key-value sort in the HloEvaluator.

Instead of doing n sorts for n sort operands, do only one (indirect) sort of
indices, and then use them to extract the values from the literals in the
sorted order. We avoid having to use n template arguments by using the
functionality of Literal to abstract away the native type.

PiperOrigin-RevId: 229886785
---
 .../compiler/xla/service/hlo_evaluator.cc     | 296 +++++++++---------
 1 file changed, 148 insertions(+), 148 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index 7589c99202..1d9af45745 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -18,9 +18,9 @@ limitations under the License.
 #include <cmath>
 #include <cstdlib>
 #include <functional>
+#include <iterator>
 #include <string>
 #include <type_traits>
-#include <utility>
 #include <vector>
 
 #include "absl/algorithm/container.h"
@@ -1327,141 +1327,51 @@ Status HloEvaluator::HandleWhile(HloInstruction* while_hlo) {
   return Status::OK();
 }
 
-// Key-value sort is a special snowflake: it's templated on two different
-// element types, one for the keys, and one for the values. Jump through some
-// hoops to make this work.
 namespace {
-template <typename KeyType, typename ValueType>
-StatusOr<Literal> EvaluateSortInternal(HloInstruction* sort,
-                                       const Literal& keys_literal,
-                                       const Literal& values_literal) {
-  auto rank = keys_literal.shape().rank();
-  TF_RET_CHECK(
-      ShapeUtil::SameDimensions(keys_literal.shape(), values_literal.shape()))
-      << "Sort keys and values must have the same dimensions";
-  TF_RET_CHECK(sort->operand_count() >= 2) << "Expected key-value sort";
-  // We need to sort an array of keys and an array of values, where the
-  // sorted order of the values is determined by the keys. The simplest(?)
-  // way to do this is to go to an array-of-pairs representation, sort the
-  // array using the keys, and then go back to pair-of-arrays.
-  VLOG(3) << "HandleSort keys_literal: " << keys_literal.ToString();
-  VLOG(3) << "HandleSort values_literal: " << values_literal.ToString();
-
-  if (rank == 0) {
-    // Nothing to sort.
-    return LiteralUtil::MakeTuple({&keys_literal, &values_literal});
-  }
-
-  Literal keys_result_literal(keys_literal.shape());
-  Literal values_result_literal(values_literal.shape());
-  std::vector<int64> zero_base(rank, 0);
-  std::vector<int64> increment(rank, 1);
-  int64 sort_dim = sort->dimensions(0);
-  int64 sort_dim_elements = keys_literal.shape().dimensions(sort_dim);
-  increment[sort_dim] = sort_dim_elements;
-  // Iterate through each dimension except 'sort_dim'.
-  TF_RETURN_IF_ERROR(ShapeUtil::ForEachIndexWithStatus(
-      keys_literal.shape(), zero_base,
-      AsInt64Slice(keys_literal.shape().dimensions()), increment,
-      [&](absl::Span<const int64> indices) -> StatusOr<bool> {
-        // Extract a slice from the keys and values literals that correspond to
-        // exactly the row in dimension 'sort_dim'.
-        std::vector<int64> limit_indices(indices.begin(), indices.end());
-        absl::c_for_each(limit_indices, [](int64& index) { ++index; });
-        limit_indices[sort_dim] = sort_dim_elements;
-        TF_ASSIGN_OR_RETURN(auto keys_to_sort,
-                            keys_literal.Slice(indices, limit_indices)
-                                .Reshape({sort_dim_elements}));
-        const auto& keys_data = keys_to_sort.data<KeyType>();
-        TF_ASSIGN_OR_RETURN(auto values_to_sort,
-                            values_literal.Slice(indices, limit_indices)
-                                .Reshape({sort_dim_elements}));
-        const auto& values_data = values_to_sort.data<ValueType>();
-        using kv_pair = std::pair<KeyType, ValueType>;
-        std::vector<kv_pair> key_value_vector;
-        key_value_vector.reserve(keys_data.size());
-        for (int i = 0; i < keys_data.size(); ++i) {
-          key_value_vector.push_back(
-              std::make_pair(keys_data[i], values_data[i]));
-        }
-        std::stable_sort(key_value_vector.begin(), key_value_vector.end(),
-                         [](const kv_pair& a, const kv_pair& b) {
-                           return SafeLess<KeyType>(a.first, b.first);
-                         });
-        std::vector<KeyType> result_keys;
-        // We use a InlinedVector here because we need to convert it to an
-        // absl::Span later, and this would not work with std::vector<bool>.
-        absl::InlinedVector<ValueType, 10> result_values;
-        for (const auto& key_value : key_value_vector) {
-          result_keys.push_back(key_value.first);
-          result_values.push_back(key_value.second);
-        }
-        Literal sorted_keys(ShapeUtil::MakeShape(
-            keys_literal.shape().element_type(), {sort_dim_elements}));
-        sorted_keys.PopulateR1(absl::Span<const KeyType>(result_keys));
-        Literal sorted_values(ShapeUtil::MakeShape(
-            values_literal.shape().element_type(), {sort_dim_elements}));
-        sorted_values.PopulateR1(absl::Span<const ValueType>(result_values));
-        std::vector<int64> slice_dimensions(rank, 1);
-        slice_dimensions[sort_dim] = sort_dim_elements;
-        std::vector<int64> start_indices(rank, 0);
-        TF_ASSIGN_OR_RETURN(auto sorted_keys_reshaped,
-                            sorted_keys.Reshape(slice_dimensions));
-        TF_RETURN_IF_ERROR(keys_result_literal.CopySliceFrom(
-            sorted_keys_reshaped, start_indices, indices, slice_dimensions));
-        TF_ASSIGN_OR_RETURN(auto sorted_values_reshaped,
-                            sorted_values.Reshape(slice_dimensions));
-        TF_RETURN_IF_ERROR(values_result_literal.CopySliceFrom(
-            sorted_values_reshaped, start_indices, indices, slice_dimensions));
-        return true;
-      }));
-
-  Literal result_tuple;
-  result_tuple =
-      LiteralUtil::MakeTuple({&keys_result_literal, &values_result_literal});
-  VLOG(3) << "HandleSort result_tuple: " << result_tuple.ToString();
-  return std::move(result_tuple);
-}
-
-template <typename KeyType>
-StatusOr<Literal> EvaluateSortCurried(HloInstruction* sort,
-                                      const Literal& keys_literal,
-                                      const Literal& values_literal) {
-  switch (values_literal.shape().element_type()) {
-    case PRED:
-      return EvaluateSortInternal<KeyType, bool>(sort, keys_literal,
-                                                 values_literal);
-    case F32:
-      return EvaluateSortInternal<KeyType, float>(sort, keys_literal,
-                                                  values_literal);
-    case U32:
-      return EvaluateSortInternal<KeyType, uint32>(sort, keys_literal,
-                                                   values_literal);
-    case S32:
-      return EvaluateSortInternal<KeyType, int32>(sort, keys_literal,
-                                                  values_literal);
-    case BF16:
-      return EvaluateSortInternal<KeyType, bfloat16>(sort, keys_literal,
-                                                     values_literal);
-    default:
-      return InvalidArgument("Unsupported type for Sort");
-  }
-}
-
-StatusOr<Literal> EvaluateSort(HloInstruction* sort,
-                               const Literal& keys_literal,
-                               const Literal& values_literal) {
-  switch (sort->operand(0)->shape().element_type()) {
-    case F32:
-      return EvaluateSortCurried<float>(sort, keys_literal, values_literal);
-    case U32:
-      return EvaluateSortCurried<uint32>(sort, keys_literal, values_literal);
-    case S32:
-      return EvaluateSortCurried<int32>(sort, keys_literal, values_literal);
-    case BF16:
-      return EvaluateSortCurried<bfloat16>(sort, keys_literal, values_literal);
+StatusOr<Literal> ExtractFromIndexPositions(const Literal& from,
+                                            absl::Span<int64 const> indices) {
+  PrimitiveType type = from.shape().element_type();
+  switch (type) {
+    case PRED: {
+      // We use a InlinedVector here because we need to convert it to an
+      // absl::Span later, and this would not work with std::vector<bool>.
+      absl::InlinedVector<bool, 10> values;
+      for (int64 index : indices) {
+        values.push_back(from.Get<bool>({index}));
+      }
+      return LiteralUtil::CreateR1<bool>(values);
+    }
+    case F32: {
+      std::vector<float> values;
+      for (int64 index : indices) {
+        values.push_back(from.Get<float>({index}));
+      }
+      return LiteralUtil::CreateR1<float>(values);
+    }
+    case U32: {
+      std::vector<uint32> values;
+      for (int64 index : indices) {
+        values.push_back(from.Get<uint32>({index}));
+      }
+      return LiteralUtil::CreateR1<uint32>(values);
+    }
+    case S32: {
+      std::vector<int32> values;
+      for (int64 index : indices) {
+        values.push_back(from.Get<int32>({index}));
+      }
+      return LiteralUtil::CreateR1<int32>(values);
+    }
+    case BF16: {
+      std::vector<bfloat16> values;
+      for (int64 index : indices) {
+        values.push_back(from.Get<bfloat16>({index}));
+      }
+      return LiteralUtil::CreateR1<bfloat16>(values);
+    }
     default:
-      return InvalidArgument("Unsupported type for Sort");
+      return InvalidArgument("Unsupported type for Sort: %s",
+                             PrimitiveType_Name(type));
   }
 }
 }  // namespace
@@ -1470,23 +1380,113 @@ Status HloEvaluator::HandleSort(HloInstruction* sort) {
   if (!sort->shape().IsTuple()) {
     return DefaultAction(sort);
   } else {
-    // This is a really stupid work-around for the fact it's hard to support a
-    // multi-value sort directly, due to the fact we need to template the
-    // evaluation function on all of the value types.
-    std::vector<Literal> sort_results_backing;
-    for (int64 i = 0; i < sort->operand_count(); ++i) {
-      auto result = EvaluateSort(sort, GetEvaluatedLiteralFor(sort->operand(0)),
-                                 GetEvaluatedLiteralFor(sort->operand(i)));
-      if (!result.ok()) {
-        return result.status();
+    TF_RET_CHECK(sort->operand_count() >= 2) << "Expected key-value sort";
+    for (int64 i = 1; i < sort->operand_count(); ++i) {
+      TF_RET_CHECK(ShapeUtil::SameDimensions(sort->operand(0)->shape(),
+                                             sort->operand(i)->shape()))
+          << "All Sort operands must have the same dimensions";
+    }
+
+    if (VLOG_IS_ON(3)) {
+      for (int64 i = 0; i < sort->operand_count(); ++i) {
+        VLOG(3) << "HandleSort operand " << i << " literal: "
+                << GetEvaluatedLiteralFor(sort->operand(i)).ToString();
       }
-      sort_results_backing.push_back(
-          std::move(result.ValueOrDie().DecomposeTuple()[1]));
     }
-    std::vector<const Literal*> sort_results;
-    absl::c_transform(sort_results_backing, std::back_inserter(sort_results),
+    Shape key_shape = sort->operand(0)->shape();
+    auto rank = key_shape.rank();
+    PrimitiveType keys_type = key_shape.element_type();
+    if (keys_type != F32 && keys_type != U32 && keys_type != S32 &&
+        keys_type != BF16) {
+      return InvalidArgument("Unsupported type for Sort: %s",
+                             PrimitiveType_Name(keys_type));
+    }
+    std::vector<Literal> result_literals;
+    result_literals.reserve(sort->operand_count());
+    for (int64 i = 0; i < sort->operand_count(); ++i) {
+      result_literals.emplace_back(sort->operand(i)->shape());
+    }
+    std::vector<int64> zero_base(rank, 0);
+    std::vector<int64> increment(rank, 1);
+    int64 sort_dim = sort->dimensions(0);
+    int64 sort_dim_elements = key_shape.dimensions(sort_dim);
+    increment[sort_dim] = sort_dim_elements;
+    // Iterate through each dimension except 'sort_dim'.
+    TF_RETURN_IF_ERROR(ShapeUtil::ForEachIndexWithStatus(
+        key_shape, zero_base, AsInt64Slice(key_shape.dimensions()), increment,
+        [&](absl::Span<const int64> indices) -> StatusOr<bool> {
+          // Extract a slice from each operand literal that corresponds to
+          // exactly the row in dimension 'sort_dim'.
+          std::vector<int64> limit_indices(indices.begin(), indices.end());
+          absl::c_for_each(limit_indices, [](int64& index) { ++index; });
+          limit_indices[sort_dim] = sort_dim_elements;
+          std::vector<Literal> literals_to_sort;
+          literals_to_sort.reserve(sort->operand_count());
+          for (int64 i = 0; i < sort->operand_count(); ++i) {
+            TF_ASSIGN_OR_RETURN(auto literal_to_sort,
+                                GetEvaluatedLiteralFor(sort->operand(i))
+                                    .Slice(indices, limit_indices)
+                                    .Reshape({sort_dim_elements}));
+            literals_to_sort.push_back(std::move(literal_to_sort));
+          }
+          std::vector<int64> indices_to_sort(sort_dim_elements);
+          std::iota(indices_to_sort.begin(), indices_to_sort.end(), 0);
+          std::stable_sort(
+              indices_to_sort.begin(), indices_to_sort.end(),
+              [keys_type, &literals_to_sort](int64 a, int64 b) {
+                switch (keys_type) {
+                  case F32: {
+                    auto key_lhs = literals_to_sort[0].Get<float>({a});
+                    auto key_rhs = literals_to_sort[0].Get<float>({b});
+                    return SafeLess(key_lhs, key_rhs);
+                  }
+                  case U32: {
+                    auto key_lhs = literals_to_sort[0].Get<uint32>({a});
+                    auto key_rhs = literals_to_sort[0].Get<uint32>({b});
+                    return SafeLess(key_lhs, key_rhs);
+                  }
+                  case S32: {
+                    auto key_lhs = literals_to_sort[0].Get<int32>({a});
+                    auto key_rhs = literals_to_sort[0].Get<int32>({b});
+                    return SafeLess(key_lhs, key_rhs);
+                  }
+                  case BF16: {
+                    auto key_lhs = literals_to_sort[0].Get<bfloat16>({a});
+                    auto key_rhs = literals_to_sort[0].Get<bfloat16>({b});
+                    return SafeLess(key_lhs, key_rhs);
+                  }
+                  default:
+                    // We should never reach here, because we checked earlier
+                    // that 'key_type' is one of the cases above.
+                    LOG(FATAL) << "Invalid key type in Sort: %s",
+                        PrimitiveType_Name(keys_type);
+                    return false;
+                }
+              });
+          std::vector<int64> slice_dimensions(rank, 1);
+          slice_dimensions[sort_dim] = sort_dim_elements;
+          std::vector<int64> start_indices(rank, 0);
+          for (int64 i = 0; i < sort->operand_count(); ++i) {
+            TF_ASSIGN_OR_RETURN(Literal sorted_literal,
+                                ExtractFromIndexPositions(literals_to_sort[i],
+                                                          indices_to_sort));
+            TF_ASSIGN_OR_RETURN(auto sorted_literal_reshaped,
+                                sorted_literal.Reshape(slice_dimensions));
+            TF_RETURN_IF_ERROR(result_literals[i].CopySliceFrom(
+                sorted_literal_reshaped, start_indices, indices,
+                slice_dimensions));
+          }
+          return true;
+        }));
+
+    std::vector<const Literal*> literal_ptrs;
+    absl::c_transform(result_literals, std::back_inserter(literal_ptrs),
                       [](const Literal& literal) { return &literal; });
-    evaluated_[sort] = LiteralUtil::MakeTuple(sort_results);
+
+    Literal result_tuple = LiteralUtil::MakeTuple(literal_ptrs);
+    VLOG(3) << "HandleSort result_tuple: " << result_tuple.ToString();
+
+    evaluated_[sort] = std::move(result_tuple);
     return Status::OK();
   }
 }
-- 
GitLab


From 43fb03cc5fb1d61c30e783fa5c771c626d29dd82 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 01:03:34 -0800
Subject: [PATCH 0944/2345] compat: Update forward compatibility horizon to
 2019-01-18

PiperOrigin-RevId: 229888500
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index c558722061..ccc0735b6e 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 17)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 18)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From d29b2faa5fe9c79679171bcafe930b819f74dca5 Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Fri, 18 Jan 2019 01:03:42 -0800
Subject: [PATCH 0945/2345] Test that arg positions are correct where used in
 the conversion script

PiperOrigin-RevId: 229888522
---
 tensorflow/tools/compatibility/tf_upgrade_v2_test.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 23446e30b1..b160d4de08 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -293,6 +293,18 @@ class TestUpgrade(test_util.TensorFlowTestCase):
     visitor.private_map["tf.compat"] = ["v1", "v2"]
     traverse.traverse(tf.compat.v1, visitor)
 
+  def testPositionsMatchArgGiven(self):
+    full_dict = tf_upgrade_v2.TFAPIChangeSpec().function_arg_warnings
+    method_names = full_dict.keys()
+    for method in method_names:
+      # doesn't test methods on objects
+      if not method.startswith("*."):
+        args = full_dict[method].keys()
+        method = get_symbol_for_name(tf, method)
+        arg_spec = tf_inspect.getfullargspec(method)
+        for (arg, pos) in args:
+          self.assertEqual(arg_spec[0][pos], arg)
+
   def testReorderFileNeedsUpdate(self):
     reordered_function_names = (
         tf_upgrade_v2.TFAPIChangeSpec().reordered_function_names)
-- 
GitLab


From 9132586162e08666368eea1eed2f90d8cbb140af Mon Sep 17 00:00:00 2001
From: Andr? Susano Pinto <andresp@google.com>
Date: Fri, 18 Jan 2019 01:53:36 -0800
Subject: [PATCH 0946/2345] Register the gradients of imported functions and
 fix the FunctionDef graphs to point to them.

Note:
 This CL enables computing gradients through a nested imported object based saved
 model function. It assumes that "StatefulPartitionedCall" and "PartionedCall"
 ops always use the default gradient that can be computed by the loader when
 needed.
PiperOrigin-RevId: 229894692
---
 .../saved_model/function_deserialization.py   | 50 +++++++++++++++----
 tensorflow/python/saved_model/load_test.py    | 43 +++++++++++++++-
 2 files changed, 83 insertions(+), 10 deletions(-)

diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py
index ec5214824e..f6427864bb 100644
--- a/tensorflow/python/saved_model/function_deserialization.py
+++ b/tensorflow/python/saved_model/function_deserialization.py
@@ -28,7 +28,9 @@ from tensorflow.python.framework import function_def_to_graph as function_def_li
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_spec
 from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.saved_model import nested_structure_coder
+from tensorflow.python.util import compat
 from tensorflow.python.util import nest
 
 
@@ -160,24 +162,29 @@ def load_function_def_library(library):
   Raises:
     ValueError: if functions dependencies have a cycle.
   """
-  # TODO(andresp): Look into restoring gradient function information.
   functions = {}
-  name_mapping = {}
+
   # Note: Use a new graph to allow function_def_to_graph to help validating
   # that the functions are loaded correctly. This is not possible to do
   # just in eager mode as there is no python API to find if a function has
   # been registered in eager. Note also that despite this the created
   # func_graphs can still be used in eager or in other graphs.
-  with ops.Graph().as_default() as import_graph:
-    for fdef in _sort_function_defs(library):
-      copy = _fix_fdef(fdef, name_mapping)
+  import_graph = ops.Graph()
+
+  for fdef in _sort_function_defs(library):
+    with import_graph.as_default():
+      copy = _fix_fdef(fdef, functions)
 
       func_graph = function_def_lib.function_def_to_graph(copy)
       func = function_lib.ConcreteFunction(func_graph)
       func.add_to_graph(import_graph)
 
-      name_mapping[fdef.signature.name] = func.name
       functions[fdef.signature.name] = func
+
+    # Also register the gradients in the current root context.
+    with ops.init_scope():
+      func._register_gradient()  # pylint: disable=protected-access
+
   return functions
 
 
@@ -215,14 +222,39 @@ def _sort_function_defs(library):
   return [reverse[x] for x in output]
 
 
-def _fix_fdef(orig_fdef, name_map):
+def _fix_fdef(orig_fdef, functions):
+  """Fixes a FunctionDef proto to be loaded in current context.
+
+  In particular, when loading a function library into an eager context, one
+  must rename the functions to avoid conflicts with existent functions.
+
+  Args:
+    orig_fdef: FunctionDef proto to fix. It is not modified.
+    functions: map from function name to a ConcreteFunction instance.
+
+  Returns:
+    A fixed copy of the original FunctionDef.
+  """
   fdef = function_pb2.FunctionDef()
   fdef.CopyFrom(orig_fdef)
-  fdef.signature.name = _clean_function_name(fdef.signature.name)
   for node_def in fdef.node_def:
+    if "_gradient_op_type" in node_def.attr:
+      if node_def.op in ["StatefulPartitionedCall", "PartitionedCall"]:
+        # TODO(andresp): This code assumes that the gradient registered for this
+        # function call is the default gradient for the function and not a
+        # custom one.
+        fname = node_def.attr["f"].func.name
+        node_def.attr["_gradient_op_type"].s = compat.as_bytes(
+            functions[fname]._gradient_name)  # pylint: disable=protected-access
+      else:
+        logging.warning("Importing a function (%s) with ops with custom "
+                        "gradients. Will likely fail if a gradient is "
+                        "requested.", fdef.signature.name)
     for _, attr_value in node_def.attr.items():
       if attr_value.func.name:
-        attr_value.func.name = name_map[attr_value.func.name]
+        attr_value.func.name = functions[attr_value.func.name].name
+
+  fdef.signature.name = _clean_function_name(fdef.signature.name)
   return fdef
 
 
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index 280a26a0a5..1c999a625d 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -361,12 +361,53 @@ class LoadTest(test.TestCase):
     root.bias = bias
     root.g = g
     imported = self.cycle(root)
-    with backprop.GradientTape(watch_accessed_variables=True) as t:
+    with backprop.GradientTape() as t:
       x = constant_op.constant([3.5])
       loss = imported.g(x)
       grad = t.gradient(loss, [imported.weight, imported.bias])
       self.assertAllClose(grad, [3.5, 1.0])
 
+  def test_nested_backprop(self):
+    weight = variables.Variable(1., trainable=True)
+    bias = variables.Variable(0., trainable=True)
+
+    # Note: this function gets called from other function defs via a
+    # "PartitionedCall" op node.
+    @def_function.function(input_signature=[
+        tensor_spec.TensorSpec(None, dtypes.float32),
+        tensor_spec.TensorSpec(None, dtypes.float32)])
+    def mul(x, y):
+      return x * y
+
+    # Note: this function gets called from other function defs via a
+    # "StatefulPartitionedCall" op node.
+    @def_function.function(input_signature=[
+        tensor_spec.TensorSpec(None, dtypes.float32)])
+    def f(x):
+      return mul(weight.read_value(), x)
+
+    @def_function.function(input_signature=[
+        tensor_spec.TensorSpec(None, dtypes.float32)])
+    def g(x):
+      return f(x) + bias,
+
+    @def_function.function(input_signature=[
+        tensor_spec.TensorSpec(None, dtypes.float32)])
+    def h(x):
+      return g(x) + bias,
+
+    root = tracking.AutoCheckpointable()
+    root.weight = weight
+    root.bias = bias
+    root.g = h
+
+    imported = self.cycle(root)
+    with backprop.GradientTape() as t:
+      x = constant_op.constant([3.5])
+      loss = imported.g(x)
+    grad = t.gradient(loss, [imported.weight, imported.bias])
+    self.assertAllClose(grad, [3.5, 2.0])
+
   def test_callable(self):
     class M1(tracking.AutoCheckpointable):
 
-- 
GitLab


From 815d49bb5855f121707b2a5b03656a18e22062f6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 02:02:47 -0800
Subject: [PATCH 0947/2345] Automated rollback of commit
 5c08b1f021e0a40fece750b5ebc0cf85b3281265

PiperOrigin-RevId: 229895719
---
 tensorflow/opensource_only.files            |   3 +
 third_party/gpus/crosstool/remote.BUILD.tpl |  10 ++
 third_party/gpus/cuda/remote.BUILD.tpl      | 110 ++++++++++++++++++++
 third_party/gpus/cuda_configure.bzl         |  17 +--
 third_party/tensorrt/build_defs.bzl.tpl     |   4 +-
 third_party/tensorrt/remote.BUILD.tpl       |   7 ++
 third_party/tensorrt/tensorrt_configure.bzl |  11 +-
 7 files changed, 148 insertions(+), 14 deletions(-)
 create mode 100644 third_party/gpus/crosstool/remote.BUILD.tpl
 create mode 100644 third_party/gpus/cuda/remote.BUILD.tpl
 create mode 100644 third_party/tensorrt/remote.BUILD.tpl

diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index 9c4dd2f4b8..850b5f5ca2 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -82,6 +82,7 @@ tensorflow/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_is_not_
 tensorflow/third_party/gpus/crosstool/CROSSTOOL.tpl
 tensorflow/third_party/gpus/crosstool/CROSSTOOL_hipcc.tpl
 tensorflow/third_party/gpus/crosstool/LICENSE
+tensorflow/third_party/gpus/crosstool/remote.BUILD.tpl
 tensorflow/third_party/gpus/crosstool/windows/msvc_wrapper_for_nvcc.py.tpl
 tensorflow/third_party/gpus/crosstool/BUILD.tpl
 tensorflow/third_party/gpus/crosstool/BUILD
@@ -89,6 +90,7 @@ tensorflow/third_party/gpus/cuda/LICENSE
 tensorflow/third_party/gpus/cuda/BUILD.tpl
 tensorflow/third_party/gpus/cuda/BUILD.windows.tpl
 tensorflow/third_party/gpus/cuda/cuda_config.h.tpl
+tensorflow/third_party/gpus/cuda/remote.BUILD.tpl
 tensorflow/third_party/gpus/cuda/BUILD
 tensorflow/third_party/gpus/cuda/build_defs.bzl.tpl
 tensorflow/third_party/gpus/rocm/rocm_config.h.tpl
@@ -192,6 +194,7 @@ tensorflow/third_party/tensorrt/BUILD
 tensorflow/third_party/tensorrt/build_defs.bzl.tpl
 tensorflow/third_party/tensorrt/BUILD.tpl
 tensorflow/third_party/tensorrt/tensorrt_configure.bzl
+tensorflow/third_party/tensorrt/remote.BUILD.tpl
 tensorflow/third_party/kafka/config.patch
 tensorflow/third_party/kafka/BUILD
 tensorflow/third_party/android/BUILD
diff --git a/third_party/gpus/crosstool/remote.BUILD.tpl b/third_party/gpus/crosstool/remote.BUILD.tpl
new file mode 100644
index 0000000000..b2316331db
--- /dev/null
+++ b/third_party/gpus/crosstool/remote.BUILD.tpl
@@ -0,0 +1,10 @@
+# Description:
+#   Template for crosstool Build file to use a pre-generated config.
+licenses(["restricted"])
+
+package(default_visibility = ["//visibility:public"])
+
+alias(
+    name = "toolchain",
+    actual = "%{remote_cuda_repo}:toolchain",
+)
diff --git a/third_party/gpus/cuda/remote.BUILD.tpl b/third_party/gpus/cuda/remote.BUILD.tpl
new file mode 100644
index 0000000000..100c7bb7c4
--- /dev/null
+++ b/third_party/gpus/cuda/remote.BUILD.tpl
@@ -0,0 +1,110 @@
+# Description:
+#   Template for cuda Build file to use a pre-generated config.
+licenses(["restricted"])  # MPL2, portions GPL v3, LGPL v3, BSD-like
+
+package(default_visibility = ["//visibility:public"])
+
+config_setting(
+    name = "using_nvcc",
+    values = {
+        "define": "using_cuda_nvcc=true",
+    },
+)
+
+config_setting(
+    name = "using_clang",
+    values = {
+        "define": "using_cuda_clang=true",
+    },
+)
+
+# Equivalent to using_clang && -c opt.
+config_setting(
+    name = "using_clang_opt",
+    values = {
+        "define": "using_cuda_clang=true",
+        "compilation_mode": "opt",
+    },
+)
+
+config_setting(
+    name = "darwin",
+    values = {"cpu": "darwin"},
+    visibility = ["//visibility:public"],
+)
+
+config_setting(
+    name = "freebsd",
+    values = {"cpu": "freebsd"},
+    visibility = ["//visibility:public"],
+)
+
+alias(
+    name = "cuda_headers",
+    actual = "%{remote_cuda_repo}/cuda:cuda_headers",
+)
+
+alias(
+    name = "cudart_static",
+    actual = "%{remote_cuda_repo}/cuda:cudart_static",
+)
+
+alias(
+    name = "cuda_driver",
+    actual = "%{remote_cuda_repo}/cuda:cuda_driver",
+)
+
+alias(
+    name = "cudart",
+    actual = "%{remote_cuda_repo}/cuda:cudart",
+)
+
+alias(
+    name = "cublas",
+    actual = "%{remote_cuda_repo}/cuda:cublas",
+)
+
+alias(
+    name = "cusolver",
+    actual = "%{remote_cuda_repo}/cuda:cusolver",
+)
+
+alias(
+    name = "cudnn",
+    actual = "%{remote_cuda_repo}/cuda:cudnn",
+)
+
+alias(
+    name = "cudnn_header",
+    actual = "%{remote_cuda_repo}/cuda:cudnn_header",
+)
+
+alias(
+    name = "cufft",
+    actual = "%{remote_cuda_repo}/cuda:cufft",
+)
+
+alias(
+    name = "curand",
+    actual = "%{remote_cuda_repo}/cuda:curand",
+)
+
+alias(
+    name = "cuda",
+    actual = "%{remote_cuda_repo}/cuda:cuda",
+)
+
+alias(
+    name = "cupti_headers",
+    actual = "%{remote_cuda_repo}/cuda:cupti_headers",
+)
+
+alias(
+    name = "cupti_dsos",
+    actual = "%{remote_cuda_repo}/cuda:cupti_dsos",
+)
+
+alias(
+    name = "libdevice_root",
+    actual = "%{remote_cuda_repo}/cuda:libdevice_root",
+)
diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl
index 5c9873f334..90020b6608 100644
--- a/third_party/gpus/cuda_configure.bzl
+++ b/third_party/gpus/cuda_configure.bzl
@@ -1508,16 +1508,17 @@ def _create_remote_cuda_repository(repository_ctx, remote_config_repo):
               ),
       },
   )
-  repository_ctx.template(
+  _tpl(
+      repository_ctx,
+      "cuda:remote.BUILD",
+      {
+          "%{remote_cuda_repo}": remote_config_repo,
+      },
       "cuda/BUILD",
-      Label(remote_config_repo + "/cuda:BUILD"),
-      {},
-  )
-  repository_ctx.template(
-      "crosstool/BUILD",
-      Label(remote_config_repo + "/crosstool:BUILD"),
-      {},
   )
+  _tpl(repository_ctx, "crosstool:remote.BUILD", {
+      "%{remote_cuda_repo}": remote_config_repo,
+  }, "crosstool/BUILD")
 
 
 def _cuda_autoconf_impl(repository_ctx):
diff --git a/third_party/tensorrt/build_defs.bzl.tpl b/third_party/tensorrt/build_defs.bzl.tpl
index 6d00513827..0dc3a7ba2d 100644
--- a/third_party/tensorrt/build_defs.bzl.tpl
+++ b/third_party/tensorrt/build_defs.bzl.tpl
@@ -2,4 +2,6 @@
 
 def if_tensorrt(if_true, if_false=[]):
   """Tests whether TensorRT was enabled during the configure process."""
-  return %{if_tensorrt}
+  if %{tensorrt_is_configured}:
+    return if_true
+  return if_false
diff --git a/third_party/tensorrt/remote.BUILD.tpl b/third_party/tensorrt/remote.BUILD.tpl
new file mode 100644
index 0000000000..d5afc4a5c1
--- /dev/null
+++ b/third_party/tensorrt/remote.BUILD.tpl
@@ -0,0 +1,7 @@
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//visibility:public"])
+
+alias(name="LICENSE", actual = "%{target}:LICENSE")
+alias(name = "tensorrt_headers", actual = "%{target}:tensorrt_headers")
+alias(name = "tensorrt", actual = "%{target}:tensorrt")
diff --git a/third_party/tensorrt/tensorrt_configure.bzl b/third_party/tensorrt/tensorrt_configure.bzl
index 22fd80e91a..d3bc388346 100644
--- a/third_party/tensorrt/tensorrt_configure.bzl
+++ b/third_party/tensorrt/tensorrt_configure.bzl
@@ -129,7 +129,7 @@ def _tpl(repository_ctx, tpl, substitutions):
 
 def _create_dummy_repository(repository_ctx):
   """Create a dummy TensorRT repository."""
-  _tpl(repository_ctx, "build_defs.bzl", {"%{if_tensorrt}": "if_false"})
+  _tpl(repository_ctx, "build_defs.bzl", {"%{tensorrt_is_configured}": "False"})
 
   _tpl(repository_ctx, "BUILD", {
       "%{tensorrt_genrules}": "",
@@ -141,10 +141,11 @@ def _tensorrt_configure_impl(repository_ctx):
   """Implementation of the tensorrt_configure repository rule."""
   if _TF_TENSORRT_CONFIG_REPO in repository_ctx.os.environ:
     # Forward to the pre-configured remote repository.
-    remote_config_repo = repository_ctx.os.environ[_TF_TENSORRT_CONFIG_REPO]
-    repository_ctx.template("BUILD", Label(remote_config_repo + ":BUILD"), {})
+    repository_ctx.template("BUILD", Label("//third_party/tensorrt:remote.BUILD.tpl"), {
+        "%{target}": repository_ctx.os.environ[_TF_TENSORRT_CONFIG_REPO],
+    })
     # Set up config file.
-    _tpl(repository_ctx, "build_defs.bzl", {"%{if_tensorrt}": "if_true"})
+    _tpl(repository_ctx, "build_defs.bzl", {"%{tensorrt_is_configured}": "True"})
     return
 
   if _TENSORRT_INSTALL_PATH not in repository_ctx.os.environ:
@@ -193,7 +194,7 @@ def _tensorrt_configure_impl(repository_ctx):
   ))
 
   # Set up config file.
-  _tpl(repository_ctx, "build_defs.bzl", {"%{if_tensorrt}": "if_true"})
+  _tpl(repository_ctx, "build_defs.bzl", {"%{tensorrt_is_configured}": "True"})
 
   # Set up BUILD file.
   _tpl(repository_ctx, "BUILD", {
-- 
GitLab


From c188ec5ca9481fe35ce8420fa12d60a62b30c82c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 03:02:37 -0800
Subject: [PATCH 0948/2345] Include device assignment in models exported by
 TPUEstimator

For model parallel usecases the num_cores_per_replica and the
device_assignment attributes are part of the model so we should export
it to get a consistent picture.

PiperOrigin-RevId: 229901092
---
 .../contrib/tpu/python/tpu/tpu_context.py     |  2 +-
 .../contrib/tpu/python/tpu/tpu_estimator.py   | 25 +++++++++++++++++--
 2 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_context.py b/tensorflow/contrib/tpu/python/tpu/tpu_context.py
index 6724624479..9ff0c55187 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_context.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_context.py
@@ -208,7 +208,7 @@ class _InternalTPUContext(object):
         use_tpu and config.tpu_config.num_cores_per_replica)
     self._mode = None
     num_cores_per_replica = config.tpu_config.num_cores_per_replica
-    if num_cores_per_replica:
+    if self._model_parallelism_enabled:
       self._computation_shape = _NUM_CORES_TO_COMPUTATION_SHAPE[
           num_cores_per_replica]
     else:
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 27c8ccf6bd..41d0562513 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -2169,7 +2169,8 @@ class TPUEstimator(estimator_lib.Estimator):
                export_to_tpu=True,
                export_to_cpu=True,
                warm_start_from=None,
-               experimental_exported_model_uses_all_cores=False):
+               experimental_exported_model_uses_all_cores=False,
+               experimental_export_device_assignment=False):
     """Constructs an `TPUEstimator` instance.
 
     Args:
@@ -2228,6 +2229,10 @@ class TPUEstimator(estimator_lib.Estimator):
         cores for inference with TPUPartitionedCall(). Once outside compilation
         is supported in TPUPartitionedCall(), this flag will be enabled by
         default.
+      experimental_export_device_assignment: Whether to include the device
+        assignment in the exported model. Doing so is useful in case of model
+        parallel inference but will tie the exported model to the TPU topology
+        used to export the model.
 
     Raises:
       ValueError: `params` has reserved keys already.
@@ -2295,6 +2300,13 @@ class TPUEstimator(estimator_lib.Estimator):
     self._export_to_tpu = export_to_tpu
     self._experimental_exported_model_uses_all_cores = (
         experimental_exported_model_uses_all_cores)
+    self._experimental_export_device_assignment = (
+        experimental_export_device_assignment)
+    if (experimental_exported_model_uses_all_cores and
+        experimental_export_device_assignment):
+      raise ValueError('experimental_exported_model_uses_all_cores and '
+                       'experimental_export_device_assignment is not supported '
+                       'at the same time.')
 
     self._is_input_fn_invoked = None
     self._rendezvous = {}
@@ -2404,7 +2416,16 @@ class TPUEstimator(estimator_lib.Estimator):
       tpu_computation, tpu_capture = self._build_tpu_computation_for_inference(
           features, labels, mode, config)
 
-      tensors_on_cpu = tpu.rewrite_for_inference(tpu_computation)
+      if self._experimental_export_device_assignment:
+        # Export the device assignment as part of the model. This is useful for
+        # model parallel usecases where the model relies on the mapping between
+        # logical and physical devices.
+        with self._ctx.with_mode(mode) as ctx:
+          device_assignment = ctx.device_assignment
+      else:
+        device_assignment = None
+      tensors_on_cpu = tpu.rewrite_for_inference(
+          tpu_computation, device_assignment=device_assignment)
       (estimator_spec, export_outputs_dict, export_outputs_list,
        predictions_dict) = (
            tpu_capture.get())
-- 
GitLab


From c861dc1dcfb1c9f6182335b3fbd611a1e225e8cd Mon Sep 17 00:00:00 2001
From: Andr? Susano Pinto <andresp@google.com>
Date: Fri, 18 Jan 2019 05:29:33 -0800
Subject: [PATCH 0949/2345] Fix another mismatch between TensorSpec versions
 when toggling tensorflow version.

PiperOrigin-RevId: 229913286
---
 tensorflow/python/saved_model/nested_structure_coder.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/saved_model/nested_structure_coder.py b/tensorflow/python/saved_model/nested_structure_coder.py
index 31ee239f13..e080a6b3de 100644
--- a/tensorflow/python/saved_model/nested_structure_coder.py
+++ b/tensorflow/python/saved_model/nested_structure_coder.py
@@ -83,7 +83,6 @@ class StructureCoder(object):
     """
     return self._map_structure(nested_structure, self._get_encoders())
 
-
   def can_encode(self, nested_structure):
     """Determines whether a nested structure can be encoded into a proto.
 
@@ -361,7 +360,10 @@ class _TensorShapeCodec(object):
   """Codec for `TensorShape`."""
 
   def can_encode(self, pyobj):
-    return isinstance(pyobj, tensor_shape.TensorShape)
+    return isinstance(pyobj, (tensor_shape.TensorShape,
+                              # TODO(b/121255889): Should not need these.
+                              tensor_shape.TensorShapeV1,
+                              tensor_shape.TensorShapeV2))
 
   def do_encode(self, tensor_shape_value, encode_fn):
     del encode_fn
-- 
GitLab


From faad607b60e6bed6e780a2ac93afcdfc96f01429 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Fri, 18 Jan 2019 07:13:43 -0800
Subject: [PATCH 0950/2345] Pass a compare function to the KeyValueSort runtime
 function.

This is in preparation of supporting calling a HloComputation for comparisons.

PiperOrigin-RevId: 229924424
---
 .../compiler/xla/service/cpu/cpu_runtime.cc   |  27 +--
 .../compiler/xla/service/cpu/cpu_runtime.h    |  13 +-
 .../compiler/xla/service/cpu/ir_emitter.cc    | 190 +++++++++--------
 .../xla/service/cpu/runtime_key_value_sort.cc | 197 ++----------------
 .../xla/service/cpu/runtime_key_value_sort.h  |  81 ++-----
 .../xla/service/cpu/simple_orc_jit.cc         |  13 +-
 6 files changed, 140 insertions(+), 381 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc b/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc
index a9febe891b..d8878e622c 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc
@@ -84,31 +84,8 @@ extern const char* const kReleaseOutfeedBufferAfterPopulationSymbolName =
     "__xla_cpu_runtime_ReleaseOutfeedBufferAfterPopulation";
 extern const char* const kParallelForkJoinSymbolName =
     "__xla_cpu_runtime_ParallelForkJoin";
-extern const char* const kKeyValueSortPREDSymbolName =
-    "__xla_cpu_runtime_KeyValueSortPRED";
-extern const char* const kKeyValueSortS8SymbolName =
-    "__xla_cpu_runtime_KeyValueSortS8";
-extern const char* const kKeyValueSortU8SymbolName =
-    "__xla_cpu_runtime_KeyValueSortU8";
-extern const char* const kKeyValueSortS16SymbolName =
-    "__xla_cpu_runtime_KeyValueSortS16";
-extern const char* const kKeyValueSortU16SymbolName =
-    "__xla_cpu_runtime_KeyValueSortU16";
-extern const char* const kKeyValueSortF16SymbolName =
-    "__xla_cpu_runtime_KeyValueSortF16";
-extern const char* const kKeyValueSortS32SymbolName =
-    "__xla_cpu_runtime_KeyValueSortS32";
-extern const char* const kKeyValueSortU32SymbolName =
-    "__xla_cpu_runtime_KeyValueSortU32";
-extern const char* const kKeyValueSortF32SymbolName =
-    "__xla_cpu_runtime_KeyValueSortF32";
-extern const char* const kKeyValueSortS64SymbolName =
-    "__xla_cpu_runtime_KeyValueSortS64";
-extern const char* const kKeyValueSortU64SymbolName =
-    "__xla_cpu_runtime_KeyValueSortU64";
-extern const char* const kKeyValueSortF64SymbolName =
-    "__xla_cpu_runtime_KeyValueSortF64";
-
+extern const char* const kKeyValueSortSymbolName =
+    "__xla_cpu_runtime_KeyValueSort";
 extern const char* const kXlaCpuRuntimeSymbolNamePrefix = "__xla_cpu_runtime_";
 }  // namespace runtime
 }  // namespace cpu
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_runtime.h b/tensorflow/compiler/xla/service/cpu/cpu_runtime.h
index b2e760a224..3a2b44d8c1 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_runtime.h
+++ b/tensorflow/compiler/xla/service/cpu/cpu_runtime.h
@@ -64,18 +64,7 @@ extern const char* const kReleaseInfeedBufferAfterDequeueSymbolName;
 extern const char* const kAcquireOutfeedBufferForPopulationSymbolName;
 extern const char* const kReleaseOutfeedBufferAfterPopulationSymbolName;
 extern const char* const kParallelForkJoinSymbolName;
-extern const char* const kKeyValueSortPREDSymbolName;
-extern const char* const kKeyValueSortS8SymbolName;
-extern const char* const kKeyValueSortU8SymbolName;
-extern const char* const kKeyValueSortS16SymbolName;
-extern const char* const kKeyValueSortU16SymbolName;
-extern const char* const kKeyValueSortF16SymbolName;
-extern const char* const kKeyValueSortS32SymbolName;
-extern const char* const kKeyValueSortU32SymbolName;
-extern const char* const kKeyValueSortF32SymbolName;
-extern const char* const kKeyValueSortS64SymbolName;
-extern const char* const kKeyValueSortU64SymbolName;
-extern const char* const kKeyValueSortF64SymbolName;
+extern const char* const kKeyValueSortSymbolName;
 
 // All symbol names for XLA CPU runtime functions need to start with this
 // prefix.
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index f8a997045a..0226e8275c 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -495,6 +495,26 @@ Status IrEmitter::HandleSort(HloInstruction* hlo) {
   const HloSortInstruction* sort = Cast<HloSortInstruction>(hlo);
   TF_RETURN_IF_ERROR(EmitTargetAddressForOp(sort));
   Shape keys_shape = sort->keys()->shape();
+  PrimitiveType keys_type = keys_shape.element_type();
+  switch (keys_type) {
+    case PRED:
+    case S8:
+    case U8:
+    case S16:
+    case U16:
+    case F16:
+    case S32:
+    case U32:
+    case F32:
+    case S64:
+    case U64:
+    case F64:
+      break;
+    default:
+      return Unimplemented(
+          "Element type %s not supported in the Sort op on CPU.",
+          PrimitiveType_Name(keys_type));
+  }
   std::vector<llvm::Value*> destination_addresses(sort->operand_count());
   for (int64 i = 0; i < sort->operand_count(); ++i) {
     ShapeIndex shape_index =
@@ -542,105 +562,101 @@ Status IrEmitter::HandleSort(HloInstruction* hlo) {
     lower_dimensions *= normalized_keys_shape.dimensions(i);
   }
 
-  PrimitiveType keys_type = keys_shape.element_type();
-  const char* fn_name = nullptr;
-  llvm::Type* keys_native_type = nullptr;
-  switch (keys_type) {
-    case PRED:
-      fn_name = runtime::kKeyValueSortPREDSymbolName;
-      keys_native_type = b_.getInt8PtrTy();
-      break;
-    case S8:
-      fn_name = runtime::kKeyValueSortS8SymbolName;
-      keys_native_type = b_.getInt8PtrTy();
-      break;
-    case U8:
-      fn_name = runtime::kKeyValueSortU8SymbolName;
-      keys_native_type = b_.getInt8PtrTy();
-      break;
-    case S16:
-      fn_name = runtime::kKeyValueSortS16SymbolName;
-      keys_native_type = b_.getInt16Ty()->getPointerTo();
-      break;
-    case U16:
-      fn_name = runtime::kKeyValueSortU16SymbolName;
-      keys_native_type = b_.getInt16Ty()->getPointerTo();
-      break;
-    case F16:
-      fn_name = runtime::kKeyValueSortF16SymbolName;
-      keys_native_type = b_.getHalfTy()->getPointerTo();
-      break;
-    case S32:
-      fn_name = runtime::kKeyValueSortS32SymbolName;
-      keys_native_type = b_.getInt32Ty()->getPointerTo();
-      break;
-    case U32:
-      fn_name = runtime::kKeyValueSortU32SymbolName;
-      keys_native_type = b_.getInt32Ty()->getPointerTo();
-      break;
-    case F32:
-      fn_name = runtime::kKeyValueSortF32SymbolName;
-      keys_native_type = b_.getFloatTy()->getPointerTo();
-      break;
-    case S64:
-      fn_name = runtime::kKeyValueSortS64SymbolName;
-      keys_native_type = b_.getInt64Ty()->getPointerTo();
-      break;
-    case U64:
-      fn_name = runtime::kKeyValueSortU64SymbolName;
-      keys_native_type = b_.getInt64Ty()->getPointerTo();
-      break;
-    case F64:
-      fn_name = runtime::kKeyValueSortF64SymbolName;
-      keys_native_type = b_.getDoubleTy()->getPointerTo();
-      break;
-    default:
-      return Unimplemented(
-          "Element type %s not supported in the Sort op on CPU.",
-          PrimitiveType_Name(keys_type));
+  llvm::FunctionType* less_than_type = llvm::FunctionType::get(
+      b_.getInt1Ty(), {b_.getInt8PtrTy(), b_.getInt8PtrTy()},
+      /*isVarArg=*/false);
+  auto less_than_function = llvm_ir::CreateFunction(
+      less_than_type, llvm::GlobalValue::InternalLinkage,
+      /*enable_fast_math=*/false,
+      /*optimize_for_size=*/true, absl::StrCat(IrName(sort), "_comparator"),
+      module_);
+  // Emit the code for the less_than function.
+  {
+    llvm::IRBuilder<>::InsertPointGuard guard(b_);
+
+    auto* entry_bb =
+        llvm::BasicBlock::Create(b_.getContext(), "entry", less_than_function);
+
+    b_.SetInsertPoint(entry_bb);
+    auto keys_ir_type = llvm_ir::PrimitiveTypeToIrType(keys_type, module_);
+    CHECK_EQ(less_than_function->arg_size(), 2);
+    llvm::Value* keys_lhs_ptr = less_than_function->arg_begin();
+    keys_lhs_ptr = PointerCast(keys_lhs_ptr, keys_ir_type->getPointerTo());
+    llvm::Value* keys_rhs_ptr = less_than_function->arg_begin() + 1;
+    keys_rhs_ptr = PointerCast(keys_rhs_ptr, keys_ir_type->getPointerTo());
+
+    // TODO(b/122298745): Replace the custom compare logic with a call to the
+    // computation specified for the Sort op.
+    llvm::Value* keys_lhs = Load(keys_ir_type, keys_lhs_ptr);
+    llvm::Value* keys_rhs = Load(keys_ir_type, keys_rhs_ptr);
+    bool is_signed_comparison = true;
+    if (primitive_util::IsFloatingPointType(keys_type)) {
+      // We would like a total order of floating point numbers so that the
+      // sort has a predictable behavior in the presence of NaNs. Rather
+      // than using floating point comparison, we use the following trick:
+      // If f is a float, and
+      // x = bit_cast<int32>(f);
+      // y = x < 0 ? 0x7FFFFFFF - x : x;
+      // then y is ordered as an int32 such that finite values have the
+      // obvious order, -0 is ordered before 0, and -NaN and NaN appear at
+      // the beginning and end of the ordering.
+      auto k = b_.getInt(llvm::APInt::getSignedMaxValue(
+          keys_lhs->getType()->getPrimitiveSizeInBits()));
+      auto comparison_type = k->getType();
+      auto zero = llvm::ConstantInt::get(comparison_type, 0);
+      auto maybe_flip = [&](llvm::Value* v) {
+        return b_.CreateSelect(b_.CreateICmp(llvm::ICmpInst::ICMP_SLT, v, zero),
+                               b_.CreateSub(k, v), v);
+      };
+      keys_lhs = b_.CreateBitCast(keys_lhs, comparison_type);
+      keys_rhs = b_.CreateBitCast(keys_rhs, comparison_type);
+      keys_lhs = maybe_flip(keys_lhs);
+      keys_rhs = maybe_flip(keys_rhs);
+    } else if (!primitive_util::IsSignedIntegralType(keys_type)) {
+      is_signed_comparison = false;
+    }
+    llvm::Value* result =
+        b_.CreateICmp(is_signed_comparison ? llvm::ICmpInst::ICMP_SLT
+                                           : llvm::ICmpInst::ICMP_ULT,
+                      keys_lhs, keys_rhs);
+    llvm::ReturnInst::Create(b_.getContext(),
+                             /*retVal=*/result, entry_bb);
   }
 
   llvm::FunctionType* key_value_sort_type = llvm::FunctionType::get(
       b_.getVoidTy(),
-      {keys_native_type, b_.getInt64Ty(), b_.getInt64Ty(), b_.getInt64Ty(),
+      {b_.getInt64Ty(), b_.getInt64Ty(), b_.getInt64Ty(),
        b_.getInt8PtrTy()->getPointerTo(), b_.getInt32Ty(),
-       b_.getInt32Ty()->getPointerTo()},
+       b_.getInt32Ty()->getPointerTo(), less_than_function->getType()},
       /*isVarArg=*/false);
-  auto* key_value_sort_func = llvm::cast<llvm::Function>(
-      module_->getOrInsertFunction(fn_name, key_value_sort_type));
+  auto* key_value_sort_func =
+      llvm::cast<llvm::Function>(module_->getOrInsertFunction(
+          runtime::kKeyValueSortSymbolName, key_value_sort_type));
   key_value_sort_func->setCallingConv(llvm::CallingConv::C);
   key_value_sort_func->setDoesNotThrow();
-  llvm::Value* values;
-  llvm::Value* sizes;
-  if (sort->values_count() == 0) {
-    values = llvm::Constant::getNullValue(b_.getInt8PtrTy()->getPointerTo());
-    sizes = llvm::Constant::getNullValue(b_.getInt32Ty()->getPointerTo());
-  } else {
-    values = llvm_ir::EmitAllocaAtFunctionEntryWithCount(
-        b_.getInt8PtrTy(), b_.getInt32(sort->values_count()),
-        "cc_values_alloca", &b_);
-    sizes = llvm_ir::EmitAllocaAtFunctionEntryWithCount(
-        b_.getInt32Ty(), b_.getInt32(sort->values_count()), "cc_sizes_alloca",
-        &b_);
-    for (int64 i = 0; i < sort->values_count(); ++i) {
-      llvm::Value* value_as_i8ptr =
-          PointerCast(destination_addresses[i + 1], b_.getInt8PtrTy());
-      llvm::Value* slot_in_values_alloca =
-          ConstInBoundsGEP1_32(b_.getInt8PtrTy(), values, i);
-      Store(value_as_i8ptr, slot_in_values_alloca);
-      llvm::Value* slot_in_sizes_alloca =
-          ConstInBoundsGEP1_32(b_.getInt32Ty(), sizes, i);
-      llvm::Value* size = b_.getInt32(ShapeUtil::ByteSizeOfPrimitiveType(
-          sort->operand(i + 1)->shape().element_type()));
-      Store(size, slot_in_sizes_alloca);
-    }
+  llvm::Value* values = llvm_ir::EmitAllocaAtFunctionEntryWithCount(
+      b_.getInt8PtrTy(), b_.getInt32(sort->operand_count()), "cc_values_alloca",
+      &b_);
+  llvm::Value* sizes = llvm_ir::EmitAllocaAtFunctionEntryWithCount(
+      b_.getInt32Ty(), b_.getInt32(sort->operand_count()), "cc_sizes_alloca",
+      &b_);
+  for (int64 i = 0; i < sort->operand_count(); ++i) {
+    llvm::Value* value_as_i8ptr =
+        PointerCast(destination_addresses[i], b_.getInt8PtrTy());
+    llvm::Value* slot_in_values_alloca =
+        ConstInBoundsGEP1_32(b_.getInt8PtrTy(), values, i);
+    Store(value_as_i8ptr, slot_in_values_alloca);
+    llvm::Value* slot_in_sizes_alloca =
+        ConstInBoundsGEP1_32(b_.getInt32Ty(), sizes, i);
+    llvm::Value* size = b_.getInt32(ShapeUtil::ByteSizeOfPrimitiveType(
+        sort->operand(i)->shape().element_type()));
+    Store(size, slot_in_sizes_alloca);
   }
 
   Call(key_value_sort_func,
-       {PointerCast(destination_addresses[0], keys_native_type),
-        b_.getInt64(higher_dimensions), b_.getInt64(sort_dimension_elements),
+       {b_.getInt64(higher_dimensions), b_.getInt64(sort_dimension_elements),
         b_.getInt64(lower_dimensions), values,
-        b_.getInt32(sort->values_count()), sizes});
+        b_.getInt32(sort->operand_count()), sizes, less_than_function});
 
   if (sort->values_count() > 0) {
     llvm_ir::EmitTuple(GetIrArrayFor(sort), destination_addresses, &b_,
diff --git a/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.cc b/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.cc
index 722aa3120e..a0667d0d9d 100644
--- a/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.cc
+++ b/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.cc
@@ -15,12 +15,10 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.h"
 
 #include <algorithm>
-#include <cmath>
 #include <cstring>
-#include <limits>
 #include <memory>
+#include <numeric>
 #include <string>
-#include <utility>
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/platform/dynamic_annotations.h"
@@ -28,80 +26,14 @@ limitations under the License.
 #include "tensorflow/core/platform/types.h"
 
 namespace {
-using tensorflow::int16;
 using tensorflow::int32;
 using tensorflow::int64;
-using tensorflow::int8;
-using tensorflow::uint16;
-using tensorflow::uint32;
-using tensorflow::uint64;
-using tensorflow::uint8;
-
-template <typename KeyType>
-void KeyValueSort(std::pair<KeyType, int64>* row_to_sort, int64 num_elements) {
-  std::sort(row_to_sort, row_to_sort + num_elements);
-}
-
-// We would like a total order of floating point numbers so that the
-// sort has a predictable behavior in the presence of NaNs. Rather
-// than using floating point comparison, we use the following trick:
-// If f is a float, and
-// x = bit_cast<int32>(f);
-// y = x < 0 ? 0x7FFFFFFF - x : x;
-// then y is ordered as an int32 such that finite values have the
-// obvious order, -0 is ordered before 0, and -NaN and NaN appear at
-// the beginning and end of the ordering.
-template <typename CastType, typename UnsignedCastType, typename KeyType>
-CastType Convert(KeyType value) {
-  CastType casted_value;
-  memcpy(&casted_value, &value, sizeof(CastType));
-  if (casted_value < 0) {
-    return static_cast<UnsignedCastType>(std::numeric_limits<CastType>::max()) -
-           casted_value;
-  }
-  return casted_value;
-}
-
-template <typename CastType, typename UnsignedCastType, typename KeyType>
-bool LessThan(KeyType lhs, KeyType rhs) {
-  return Convert<CastType, UnsignedCastType>(lhs) <
-         Convert<CastType, UnsignedCastType>(rhs);
-}
-
-template <>
-void KeyValueSort(std::pair<double, int64>* row_to_sort, int64 num_elements) {
-  std::stable_sort(row_to_sort, row_to_sort + num_elements,
-                   [](const std::pair<double, int64>& lhs,
-                      const std::pair<double, int64>& rhs) -> bool {
-                     return LessThan<int64, uint64>(lhs.first, rhs.first);
-                   });
-}
-
-template <>
-void KeyValueSort(std::pair<float, int64>* row_to_sort, int64 num_elements) {
-  std::stable_sort(row_to_sort, row_to_sort + num_elements,
-                   [](const std::pair<float, int64>& lhs,
-                      const std::pair<float, int64>& rhs) -> bool {
-                     return LessThan<int32, uint32>(lhs.first, rhs.first);
-                   });
-}
-
-template <>
-void KeyValueSort(std::pair<Eigen::half, int64>* row_to_sort,
-                  int64 num_elements) {
-  std::stable_sort(row_to_sort, row_to_sort + num_elements,
-                   [](const std::pair<Eigen::half, int64>& lhs,
-                      const std::pair<Eigen::half, int64>& rhs) -> bool {
-                     return LessThan<int32, uint32>(
-                         Eigen::half_impl::half_to_float(lhs.first),
-                         Eigen::half_impl::half_to_float(rhs.first));
-                   });
-}
+}  // namespace
 
-template <typename KeyType>
-void KeyValueSortImpl(KeyType* keys, int64 a, int64 b, int64 c, char** values,
-                      int32 values_count,
-                      int32* values_primitive_type_size_in_bytes) {
+TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSort(
+    int64 a, int64 b, int64 c, char** values, int32 values_count,
+    int32* values_primitive_type_size_in_bytes,
+    bool (*less_than)(char*, char*)) {
   // 'values' and 'values_primitive_type_size_in_bytes' are managed by the JIT
   // code, so msan can't tell they are initialized.
   TF_ANNOTATE_MEMORY_IS_INITIALIZED(values, values_count * sizeof(char*));
@@ -121,8 +53,8 @@ void KeyValueSortImpl(KeyType* keys, int64 a, int64 b, int64 c, char** values,
   int64 num_iteration_elements = a * c;
   int64 sort_dimension_offset = c;
 
-  std::unique_ptr<std::pair<KeyType, int64>[]> row_to_sort(
-      new std::pair<KeyType, int64>[sort_dimension_elements]);
+  std::unique_ptr<int64[]> indices(new int64[sort_dimension_elements]);
+  std::iota(indices.get(), indices.get() + sort_dimension_elements, 0);
   std::unique_ptr<std::string[]> reordered_values(
       new std::string[sort_dimension_elements]);
   for (int64 index = 0; index < num_iteration_elements; ++index) {
@@ -135,24 +67,22 @@ void KeyValueSortImpl(KeyType* keys, int64 a, int64 b, int64 c, char** values,
     int64 base_offset =
         index % sort_dimension_offset +
         (index - index % sort_dimension_offset) * sort_dimension_elements;
-    // TODO(b/26783907): We could define a custom iterator class that references
-    // all arrays. Then we could avoid the intermediate copy. However this
-    // would become more complicated, and it is not clear if the benefit is high
-    // enough.
-    for (int64 i = 0; i < sort_dimension_elements; ++i) {
-      row_to_sort[i] =
-          std::make_pair(keys[base_offset + i * sort_dimension_offset], i);
-    }
-    KeyValueSort(row_to_sort.get(), sort_dimension_elements);
-    for (int64 i = 0; i < sort_dimension_elements; ++i) {
-      keys[base_offset + i * sort_dimension_offset] = row_to_sort[i].first;
-    }
-
-    // Reorder the values according to the order defined by the keys.
+    std::stable_sort(
+        indices.get(), indices.get() + sort_dimension_elements,
+        [&](int64 a, int64 b) {
+          int64 memory_index_lhs = (base_offset + a * sort_dimension_offset) *
+                                   values_primitive_type_size_in_bytes[0];
+          int64 memory_index_rhs = (base_offset + b * sort_dimension_offset) *
+                                   values_primitive_type_size_in_bytes[0];
+          return less_than(values[0] + memory_index_lhs,
+                           values[0] + memory_index_rhs);
+        });
+
+    // Reorder the values according to the order defined by 'indices'.
     for (int32 idx = 0; idx < values_count; ++idx) {
       for (int64 i = 0; i < sort_dimension_elements; ++i) {
         int64 memory_index =
-            (base_offset + row_to_sort[i].second * sort_dimension_offset) *
+            (base_offset + indices[i] * sort_dimension_offset) *
             values_primitive_type_size_in_bytes[idx];
 
         reordered_values[i] =
@@ -168,88 +98,3 @@ void KeyValueSortImpl(KeyType* keys, int64 a, int64 b, int64 c, char** values,
     }
   }
 }
-}  // namespace
-
-TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSortPRED(
-    bool* keys, int64 a, int64 b, int64 c, char** values, int32 values_count,
-    int32* values_primitive_type_size_in_bytes) {
-  KeyValueSortImpl(keys, a, b, c, values, values_count,
-                   values_primitive_type_size_in_bytes);
-}
-
-TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSortS8(
-    int8* keys, int64 a, int64 b, int64 c, char** values, int32 values_count,
-    int32* values_primitive_type_size_in_bytes) {
-  KeyValueSortImpl(keys, a, b, c, values, values_count,
-                   values_primitive_type_size_in_bytes);
-}
-
-TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSortU8(
-    uint8* keys, int64 a, int64 b, int64 c, char** values, int32 values_count,
-    int32* values_primitive_type_size_in_bytes) {
-  KeyValueSortImpl(keys, a, b, c, values, values_count,
-                   values_primitive_type_size_in_bytes);
-}
-
-TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSortS16(
-    int16* keys, int64 a, int64 b, int64 c, char** values, int32 values_count,
-    int32* values_primitive_type_size_in_bytes) {
-  KeyValueSortImpl(keys, a, b, c, values, values_count,
-                   values_primitive_type_size_in_bytes);
-}
-
-TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSortU16(
-    uint16* keys, int64 a, int64 b, int64 c, char** values, int32 values_count,
-    int32* values_primitive_type_size_in_bytes) {
-  KeyValueSortImpl(keys, a, b, c, values, values_count,
-                   values_primitive_type_size_in_bytes);
-}
-
-TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSortF16(
-    Eigen::half* keys, int64 a, int64 b, int64 c, char** values,
-    int32 values_count, int32* values_primitive_type_size_in_bytes) {
-  KeyValueSortImpl(keys, a, b, c, values, values_count,
-                   values_primitive_type_size_in_bytes);
-}
-
-TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSortS32(
-    int32* keys, int64 a, int64 b, int64 c, char** values, int32 values_count,
-    int32* values_primitive_type_size_in_bytes) {
-  KeyValueSortImpl(keys, a, b, c, values, values_count,
-                   values_primitive_type_size_in_bytes);
-}
-
-TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSortU32(
-    uint32* keys, int64 a, int64 b, int64 c, char** values, int32 values_count,
-    int32* values_primitive_type_size_in_bytes) {
-  KeyValueSortImpl(keys, a, b, c, values, values_count,
-                   values_primitive_type_size_in_bytes);
-}
-
-TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSortF32(
-    float* keys, int64 a, int64 b, int64 c, char** values, int32 values_count,
-    int32* values_primitive_type_size_in_bytes) {
-  KeyValueSortImpl(keys, a, b, c, values, values_count,
-                   values_primitive_type_size_in_bytes);
-}
-
-TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSortS64(
-    int64* keys, int64 a, int64 b, int64 c, char** values, int32 values_count,
-    int32* values_primitive_type_size_in_bytes) {
-  KeyValueSortImpl(keys, a, b, c, values, values_count,
-                   values_primitive_type_size_in_bytes);
-}
-
-TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSortU64(
-    uint64* keys, int64 a, int64 b, int64 c, char** values, int32 values_count,
-    int32* values_primitive_type_size_in_bytes) {
-  KeyValueSortImpl(keys, a, b, c, values, values_count,
-                   values_primitive_type_size_in_bytes);
-}
-
-TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSortF64(
-    double* keys, int64 a, int64 b, int64 c, char** values, int32 values_count,
-    int32* values_primitive_type_size_in_bytes) {
-  KeyValueSortImpl(keys, a, b, c, values, values_count,
-                   values_primitive_type_size_in_bytes);
-}
diff --git a/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.h b/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.h
index 7821099386..5460af3485 100644
--- a/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.h
+++ b/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.h
@@ -21,76 +21,19 @@ limitations under the License.
 
 extern "C" {
 
-// 'keys' represents a 3-dimensional shape with dimensions [a, b, c]. The 'b'
-// dimension of 'keys' is sorted into ascending order. If 'values_count' is <=
-// 0, 'values' and 'values_primitive_type_size_in_bytes' can be nullptr.
-// If 'values_count' > 0, they contain exactly 'values_count' many elements.
-// Each element of 'values' also represents a 3-dimensional shape with
-// dimensions [a, b, c], and the size of the primitive type of the i-th shape
-// has exactly 'values_primitive_type_size_in_bytes[i]' bytes. The elements in
-// each 'values' shape are reordered in such a way that if the element at index
-// 'i' in 'keys' was moved to index 'j', the element at index 'i' in a 'values'
-// shape is also moved to index 'j' (which means that the same elements
-// correspond to each other as before).
-extern void __xla_cpu_runtime_KeyValueSortPRED(
-    bool* keys, tensorflow::int64 a, tensorflow::int64 b, tensorflow::int64 c,
+// Each entry in 'values' represents a 3-dimensional shape with dimensions
+// [a, b, c]. The 'b' dimension of the first shape is sorted into ascending
+// order according to the results of comparisons using the provided 'less_than'
+// function. 'values_count' must be > 0 and specifies the number of entries in
+// 'values' and 'values_primitive_type_size_in_bytes'. The size of the primitive
+// type of the i-th shape has exactly 'values_primitive_type_size_in_bytes[i]'
+// bytes. The elements in each 'values' shape are reordered in the same way
+// according to the comparisons using the first shape.
+extern void __xla_cpu_runtime_KeyValueSort(
+    tensorflow::int64 a, tensorflow::int64 b, tensorflow::int64 c,
     char** values, tensorflow::int32 values_count,
-    tensorflow::int32* values_primitive_type_size_in_bytes);
-
-extern void __xla_cpu_runtime_KeyValueSortS8(
-    tensorflow::int8* keys, tensorflow::int64 a, tensorflow::int64 b,
-    tensorflow::int64 c, char** values, tensorflow::int32 values_count,
-    tensorflow::int32* values_primitive_type_size_in_bytes);
-
-extern void __xla_cpu_runtime_KeyValueSortU8(
-    tensorflow::uint8* keys, tensorflow::int64 a, tensorflow::int64 b,
-    tensorflow::int64 c, char** values, tensorflow::int32 values_count,
-    tensorflow::int32* values_primitive_type_size_in_bytes);
-
-extern void __xla_cpu_runtime_KeyValueSortS16(
-    tensorflow::int16* keys, tensorflow::int64 a, tensorflow::int64 b,
-    tensorflow::int64 c, char** values, tensorflow::int32 values_count,
-    tensorflow::int32* values_primitive_type_size_in_bytes);
-
-extern void __xla_cpu_runtime_KeyValueSortU16(
-    tensorflow::uint16* keys, tensorflow::int64 a, tensorflow::int64 b,
-    tensorflow::int64 c, char** values, tensorflow::int32 values_count,
-    tensorflow::int32* values_primitive_type_size_in_bytes);
-
-extern void __xla_cpu_runtime_KeyValueSortF16(
-    Eigen::half* keys, tensorflow::int64 a, tensorflow::int64 b,
-    tensorflow::int64 c, char** values, tensorflow::int32 values_count,
-    tensorflow::int32* values_primitive_type_size_in_bytes);
-
-extern void __xla_cpu_runtime_KeyValueSortS32(
-    tensorflow::int32* keys, tensorflow::int64 a, tensorflow::int64 b,
-    tensorflow::int64 c, char** values, tensorflow::int32 values_count,
-    tensorflow::int32* values_primitive_type_size_in_bytes);
-
-extern void __xla_cpu_runtime_KeyValueSortU32(
-    tensorflow::uint32* keys, tensorflow::int64 a, tensorflow::int64 b,
-    tensorflow::int64 c, char** values, tensorflow::int32 values_count,
-    tensorflow::int32* values_primitive_type_size_in_bytes);
-
-extern void __xla_cpu_runtime_KeyValueSortF32(
-    float* keys, tensorflow::int64 a, tensorflow::int64 b, tensorflow::int64 c,
-    char** values, tensorflow::int32 values_count,
-    tensorflow::int32* values_primitive_type_size_in_bytes);
-
-extern void __xla_cpu_runtime_KeyValueSortS64(
-    tensorflow::int64* keys, tensorflow::int64 a, tensorflow::int64 b,
-    tensorflow::int64 c, char** values, tensorflow::int32 values_count,
-    tensorflow::int32* values_primitive_type_size_in_bytes);
-
-extern void __xla_cpu_runtime_KeyValueSortU64(
-    tensorflow::uint64* keys, tensorflow::int64 a, tensorflow::int64 b,
-    tensorflow::int64 c, char** values, tensorflow::int32 values_count,
-    tensorflow::int32* values_primitive_type_size_in_bytes);
-
-extern void __xla_cpu_runtime_KeyValueSortF64(
-    double* keys, tensorflow::int64 a, tensorflow::int64 b, tensorflow::int64 c,
-    char** values, tensorflow::int32 values_count,
-    tensorflow::int32* values_primitive_type_size_in_bytes);
+    tensorflow::int32* values_primitive_type_size_in_bytes,
+    bool (*less_than)(char*, char*));
 }
 
 #endif  // TENSORFLOW_COMPILER_XLA_SERVICE_CPU_RUNTIME_KEY_VALUE_SORT_H_
diff --git a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
index 12ab6360c5..9c2685674f 100644
--- a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
+++ b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
@@ -240,18 +240,7 @@ bool RegisterKnownJITSymbols() {
   REGISTER_CPU_RUNTIME_SYMBOL(ParallelForkJoin);
   REGISTER_CPU_RUNTIME_SYMBOL(ReleaseInfeedBufferAfterDequeue);
   REGISTER_CPU_RUNTIME_SYMBOL(ReleaseOutfeedBufferAfterPopulation);
-  REGISTER_CPU_RUNTIME_SYMBOL(KeyValueSortPRED);
-  REGISTER_CPU_RUNTIME_SYMBOL(KeyValueSortS8);
-  REGISTER_CPU_RUNTIME_SYMBOL(KeyValueSortU8);
-  REGISTER_CPU_RUNTIME_SYMBOL(KeyValueSortS16);
-  REGISTER_CPU_RUNTIME_SYMBOL(KeyValueSortU16);
-  REGISTER_CPU_RUNTIME_SYMBOL(KeyValueSortF16);
-  REGISTER_CPU_RUNTIME_SYMBOL(KeyValueSortS32);
-  REGISTER_CPU_RUNTIME_SYMBOL(KeyValueSortU32);
-  REGISTER_CPU_RUNTIME_SYMBOL(KeyValueSortF32);
-  REGISTER_CPU_RUNTIME_SYMBOL(KeyValueSortS64);
-  REGISTER_CPU_RUNTIME_SYMBOL(KeyValueSortU64);
-  REGISTER_CPU_RUNTIME_SYMBOL(KeyValueSortF64);
+  REGISTER_CPU_RUNTIME_SYMBOL(KeyValueSort);
 
   registry->Register("__gnu_f2h_ieee", reinterpret_cast<void*>(__gnu_f2h_ieee));
   registry->Register("__gnu_h2f_ieee", reinterpret_cast<void*>(__gnu_h2f_ieee));
-- 
GitLab


From f9f39e43c8ced8377cbb5e4775815445e825c655 Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Fri, 18 Jan 2019 07:15:12 -0800
Subject: [PATCH 0951/2345] Loosen return type for variables.

PiperOrigin-RevId: 229924537
---
 tensorflow/python/module/module.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/tensorflow/python/module/module.py b/tensorflow/python/module/module.py
index 16db0b0e01..2ba6b39bc0 100644
--- a/tensorflow/python/module/module.py
+++ b/tensorflow/python/module/module.py
@@ -199,23 +199,23 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
 
   @property
   def variables(self):
-    """Returns a tuple of variables owned by this module and it's submodules.
+    """Collection of variables owned by this module and it's submodules.
 
     Returns:
-      A tuple of variables for the current module (sorted by attribute name)
-      followed by variables from all submodules recursively (depth first).
+      A collection of variables for the current module (sorted by attribute
+      name) followed by variables from all submodules recursively (depth first).
     """
     return tuple(walk(self, recurse_if=_IS_MODULE, predicate=_IS_VARIABLE))
 
   @property
   def owned_variables(self):
-    """Returns a tuple of variables that are attributes of the current module.
+    """Collection of variables that are attributes of the current module.
 
     See `variables` for a property which returns all variables from the current
     module and all it's submodules recursively.
 
     Returns:
-      A tuple of variables which are attributes of the current module. Will
+      A collection of variables which are attributes of the current module. Will
       yield variables inside nested structures (lists etc) but not in other
       modules.
     """
@@ -223,24 +223,24 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
 
   @property
   def trainable_variables(self):
-    """Returns a tuple of variables owned by this module and it's submodules.
+    """Collection of variables owned by this module and it's submodules.
 
     Returns:
-      A tuple of variables for the current module (sorted by attribute name)
-      followed by variables from all submodules recursively (depth first).
+      A collection of variables for the current module (sorted by attribute
+      name) followed by variables from all submodules recursively (depth first).
     """
     return tuple(
         walk(self, recurse_if=_IS_MODULE, predicate=_IS_TRAINABLE_VARIABLE))
 
   @property
   def owned_trainable_variables(self):
-    """Returns a tuple of variables that are attributes of the current module.
+    """Collection of variables that are attributes of the current module.
 
     See `variables` for a property which returns all variables from the current
     module and all it's submodules recursively.
 
     Returns:
-      A tuple of variables which are attributes of the current module. Will
+      A collection of variables which are attributes of the current module. Will
       yield variables inside nested structures (lists etc) but not in other
       modules.
     """
-- 
GitLab


From c5d6b96282d6c5ef5a730e9bba7f6133f9e9d671 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 07:32:53 -0800
Subject: [PATCH 0952/2345] Change UNDEFINED to UNSPECIFIED in code handling
 unspecified built-in arguments.

PiperOrigin-RevId: 229926077
---
 .../python/autograph/operators/py_builtins.py | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/tensorflow/python/autograph/operators/py_builtins.py b/tensorflow/python/autograph/operators/py_builtins.py
index ddf05f73f3..9f54cf4fe5 100644
--- a/tensorflow/python/autograph/operators/py_builtins.py
+++ b/tensorflow/python/autograph/operators/py_builtins.py
@@ -37,7 +37,7 @@ from tensorflow.python.ops import list_ops
 from tensorflow.python.ops import math_ops
 
 
-UNDEFINED = object()
+UNSPECIFIED = object()
 
 
 def overload_of(f):
@@ -77,14 +77,14 @@ def _py_float(x):
   return float(x)
 
 
-def int_(x=0, base=UNDEFINED):
+def int_(x=0, base=UNSPECIFIED):
   if tensor_util.is_tensor(x):
     return _tf_int(x, base)
   return _py_int(x, base)
 
 
 def _tf_int(x, base):
-  if base not in (10, UNDEFINED):
+  if base not in (10, UNSPECIFIED):
     raise NotImplementedError('base {} not supported for int'.format(base))
 
   # TODO(mdan): We shouldn't assume int32.
@@ -94,7 +94,7 @@ def _tf_int(x, base):
 
 
 def _py_int(x, base):
-  if base is UNDEFINED:
+  if base is UNSPECIFIED:
     return int(x)
   return int(x, base)
 
@@ -167,7 +167,7 @@ def print_(*objects, **kwargs):
 
 def _tf_py_func_print(objects, kwargs):
   """Overload of print_ as a py_func implementation."""
-  override_kwargs = {k: v for k, v in kwargs.items() if v is not UNDEFINED}
+  override_kwargs = {k: v for k, v in kwargs.items() if v is not UNSPECIFIED}
   if 'flush' not in override_kwargs:
     # Defaulting to flushing the console in graph mode, which helps reduce
     # garbled output in IPython.
@@ -187,7 +187,7 @@ def _tf_py_func_print(objects, kwargs):
       print_wrapper, None, objects, use_dummy_return=True)
 
 
-def range_(start_or_stop, stop=UNDEFINED, step=UNDEFINED):
+def range_(start_or_stop, stop=UNSPECIFIED, step=UNSPECIFIED):
   if any(tensor_util.is_tensor(s) for s in (start_or_stop, stop, step)):
     return _tf_range(start_or_stop, stop, step)
   return _py_range(start_or_stop, stop, step)
@@ -200,10 +200,10 @@ def _tf_range(start_or_stop, stop, step):
   # graph construction error aligns the semantics with Python.
 
   # TODO(mdan): We should optimize this when a full tensor is not required.
-  if step is not UNDEFINED:
+  if step is not UNSPECIFIED:
     # TODO(mdan): Add argument coercion similar to other cases.
     return math_ops.range(start_or_stop, stop, step)
-  if stop is not UNDEFINED:
+  if stop is not UNSPECIFIED:
     stop = math_ops.maximum(start_or_stop, stop)
     return math_ops.range(start_or_stop, stop)
   start_or_stop = math_ops.maximum(start_or_stop, 0)
@@ -211,9 +211,9 @@ def _tf_range(start_or_stop, stop, step):
 
 
 def _py_range(start_or_stop, stop, step):
-  if step is not UNDEFINED:
+  if step is not UNSPECIFIED:
     return range(start_or_stop, stop, step)
-  if stop is not UNDEFINED:
+  if stop is not UNSPECIFIED:
     return range(start_or_stop, stop)
   return range(start_or_stop)
 
-- 
GitLab


From 8690cf7bf33f1e2d96ac1f2dea765d8aaca18f4d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 08:03:30 -0800
Subject: [PATCH 0953/2345] Auto-linting on
 third_party/tensorflow/python/autograph/pyct/transformer.py

PiperOrigin-RevId: 229929152
---
 .../python/autograph/pyct/transformer.py      | 40 +++++++++++--------
 1 file changed, 24 insertions(+), 16 deletions(-)

diff --git a/tensorflow/python/autograph/pyct/transformer.py b/tensorflow/python/autograph/pyct/transformer.py
index 9c776a737a..3578f38c65 100644
--- a/tensorflow/python/autograph/pyct/transformer.py
+++ b/tensorflow/python/autograph/pyct/transformer.py
@@ -35,15 +35,17 @@ class AutoGraphParseError(SyntaxError):
 
 # TODO(mdan): Use namedtuple.
 class EntityInfo(object):
-  """Contains information about a Python entity. Immutable.
+  """Contains information about a Python entity.
+
+  Immutable.
 
   Examples of entities include functions and classes.
 
   Attributes:
     source_code: The entity's source code.
     source_file: The entity's source file.
-    namespace: Dict[str, ], containing symbols visible to the entity
-        (excluding parameters).
+    namespace: Dict[str, ], containing symbols visible to the entity (excluding
+      parameters).
     arg_values: dict[str->*], containing parameter values, if known.
     arg_types: dict[str->*], containing parameter types, if known.
     owner_type: The surrounding class type of the function, if present.
@@ -199,7 +201,9 @@ class Base(gast.NodeTransformer):
   # TODO(mdan): Document all extra features.
 
   def __init__(self, entity_info):
-    """Initialize the transformer. Subclasses should call this.
+    """Initialize the transformer.
+
+    Subclasses should call this.
 
     Args:
       entity_info: An EntityInfo object.
@@ -232,13 +236,15 @@ class Base(gast.NodeTransformer):
     return len(self._local_scope_state)
 
   def enter_local_scope(self, inherit=None):
-    """Deprecated. Use self.state instead.
+    """Deprecated.
+
+    Use self.state instead.
 
     Marks entry into a new local scope.
 
     Args:
-      inherit: Optional enumerable of variable names to copy from the
-          parent scope.
+      inherit: Optional enumerable of variable names to copy from the parent
+        scope.
     """
     scope_entered = {}
     if inherit:
@@ -249,13 +255,15 @@ class Base(gast.NodeTransformer):
     self._local_scope_state.append(scope_entered)
 
   def exit_local_scope(self, keep=None):
-    """Deprecated. Use self.state instead.
+    """Deprecated.
+
+    Use self.state instead.
 
     Marks exit from the current local scope.
 
     Args:
-      keep: Optional enumerable of variable names to copy into the
-          parent scope.
+      keep: Optional enumerable of variable names to copy into the parent scope.
+
     Returns:
       A dict containing the scope that has just been exited.
     """
@@ -390,11 +398,11 @@ class Base(gast.NodeTransformer):
 
     Args:
       targets: list, tuple of or individual AST node. Should be used with the
-          targets field of an ast.Assign node.
+        targets field of an ast.Assign node.
       values: an AST node.
       apply_fn: a function of a single argument, which will be called with the
-          respective nodes of each single assignment. The signature is
-          apply_fn(target, value), no return value.
+        respective nodes of each single assignment. The signature is
+        apply_fn(target, value), no return value.
     """
     if not isinstance(targets, (list, tuple)):
       targets = (targets,)
@@ -429,9 +437,9 @@ class Base(gast.NodeTransformer):
       # call `visit`.  The error needs to be raised before the exception handler
       # below is installed, because said handler will mess up if `node` is not,
       # in fact, a node.
-      msg = (
-          'invalid value for "node": expected "ast.AST", got "{}"; to'
-          ' visit lists of nodes, use "visit_block" instead').format(type(node))
+      msg = ('invalid value for "node": expected "ast.AST", got "{}"; to'
+             ' visit lists of nodes, use "visit_block" instead').format(
+                 type(node))
       raise ValueError(msg)
 
     did_enter_function = False
-- 
GitLab


From 7c65c591fb52b3a343af88835d560196d8290e8c Mon Sep 17 00:00:00 2001
From: Ayush Dubey <ayushd@google.com>
Date: Fri, 18 Jan 2019 08:08:57 -0800
Subject: [PATCH 0954/2345] Invert static ordering of collective ops when
 enabled.

PiperOrigin-RevId: 229929957
---
 tensorflow/core/graph/collective_order.cc      |  4 ++--
 tensorflow/core/graph/collective_order_test.cc | 16 ++++++++--------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/tensorflow/core/graph/collective_order.cc b/tensorflow/core/graph/collective_order.cc
index bbba9264ce..c24166cb8e 100644
--- a/tensorflow/core/graph/collective_order.cc
+++ b/tensorflow/core/graph/collective_order.cc
@@ -92,8 +92,8 @@ Status CreateControlDependencies(
       const auto& deps_j = (*data_dependencies)[collective_nodes[j]];
       if (deps_i.find(instance_keys[j]) == deps_i.end() &&
           deps_j.find(instance_keys[i]) == deps_j.end()) {
-        int src_idx = instance_keys[i] < instance_keys[j] ? i : j;
-        int dst_idx = instance_keys[i] < instance_keys[j] ? j : i;
+        int src_idx = instance_keys[i] > instance_keys[j] ? i : j;
+        int dst_idx = instance_keys[i] > instance_keys[j] ? j : i;
         Node* src_node = collective_nodes[src_idx];
         Node* dst_node = collective_nodes[dst_idx];
         VLOG(1) << "Adding control dependency from node " << src_node->name()
diff --git a/tensorflow/core/graph/collective_order_test.cc b/tensorflow/core/graph/collective_order_test.cc
index 241c98b549..9a158e5c3f 100644
--- a/tensorflow/core/graph/collective_order_test.cc
+++ b/tensorflow/core/graph/collective_order_test.cc
@@ -137,18 +137,18 @@ std::unique_ptr<Graph> InitGraph() {
 }
 
 // Tests that in the graph created by `InitGraph`, exactly 2 control edges are
-// added after calling `OrderCollectives`: c2_0 -> c3_0 and c2_1 -> c3_1.
+// added after calling `OrderCollectives`: c3_0 -> c2_0 and c3_1 -> c2_1.
 TEST(CollectiveOrderTest, SimpleOrder) {
   std::unique_ptr<Graph> graph = InitGraph();
   TF_EXPECT_OK(OrderCollectives(graph.get(), GraphCollectiveOrder::kEdges));
   VerifyGraph(*graph, {"c1_0", "c1_1", "c2_0", "c2_1", "c3_0", "c3_1"},
-              {{"c2_0", "c3_0"}, {"c2_1", "c3_1"}});
+              {{"c3_0", "c2_0"}, {"c3_1", "c2_1"}});
 }
 
 TEST(CollectiveOrderTest, SimpleOrderAttr) {
   std::unique_ptr<Graph> graph = InitGraph();
   TF_EXPECT_OK(OrderCollectives(graph.get(), GraphCollectiveOrder::kAttrs));
-  VerifyAttrs(*graph, {{"c3_0", {2}}, {"c3_1", {2}}});
+  VerifyAttrs(*graph, {{"c2_0", {3}}, {"c2_1", {3}}});
 }
 
 // Initialize the following graph:
@@ -185,12 +185,12 @@ std::unique_ptr<Graph> InitGraph2() {
 }
 
 // Tests that in the graph created by `InitGraph2`, we add the following control
-// edges after calling `OrderCollectives`: c2 -> c3, c3 -> c4.  c2->c4 is
+// edges after calling `OrderCollectives`: c4 -> c3, c3 -> c2.  c4->c2 is
 // pruned because it follows from the other two edges.
 TEST(CollectiveOrderTest, SimpleOrder2) {
   std::unique_ptr<Graph> graph = InitGraph2();
   TF_EXPECT_OK(OrderCollectives(graph.get(), GraphCollectiveOrder::kEdges));
-  VerifyGraph(*graph, {"c1", "c2", "c3", "c4"}, {{"c2", "c3"}, {"c3", "c4"}});
+  VerifyGraph(*graph, {"c1", "c2", "c3", "c4"}, {{"c4", "c3"}, {"c3", "c2"}});
 }
 
 // Initialize the following graph:
@@ -223,12 +223,12 @@ std::unique_ptr<Graph> InitGraphForPruning() {
   return graph;
 }
 
-// Tests that in the graph created by `InitGraphForPruning`, we only add c1 ->
-// c2, c2 -> c3, c3 -> c4, and other edges are pruned away.
+// Tests that in the graph created by `InitGraphForPruning`, we only add c4 ->
+// c3, c3 -> c2, c2 -> c1, and other edges are pruned away.
 TEST(CollectiveOrderTest, Pruning) {
   std::unique_ptr<Graph> graph = InitGraphForPruning();
   TF_EXPECT_OK(OrderCollectives(graph.get(), GraphCollectiveOrder::kAttrs));
-  VerifyAttrs(*graph, {{"c4", {3}}, {"c3", {2}}, {"c2", {1}}});
+  VerifyAttrs(*graph, {{"c3", {4}}, {"c2", {3}}, {"c1", {2}}});
 }
 
 }  // namespace
-- 
GitLab


From 72571e08245505a8ca0a1550ee7f97212b1df113 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 08:09:15 -0800
Subject: [PATCH 0955/2345] Roll CL 229790895 forward with fix.

PiperOrigin-RevId: 229930009
---
 tensorflow/opensource_only.files            |   3 -
 third_party/gpus/crosstool/remote.BUILD.tpl |  10 --
 third_party/gpus/cuda/remote.BUILD.tpl      | 110 --------------------
 third_party/gpus/cuda_configure.bzl         |  22 ++--
 third_party/tensorrt/build_defs.bzl.tpl     |   4 +-
 third_party/tensorrt/remote.BUILD.tpl       |   7 --
 third_party/tensorrt/tensorrt_configure.bzl |  16 +--
 7 files changed, 23 insertions(+), 149 deletions(-)
 delete mode 100644 third_party/gpus/crosstool/remote.BUILD.tpl
 delete mode 100644 third_party/gpus/cuda/remote.BUILD.tpl
 delete mode 100644 third_party/tensorrt/remote.BUILD.tpl

diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index 850b5f5ca2..9c4dd2f4b8 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -82,7 +82,6 @@ tensorflow/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_is_not_
 tensorflow/third_party/gpus/crosstool/CROSSTOOL.tpl
 tensorflow/third_party/gpus/crosstool/CROSSTOOL_hipcc.tpl
 tensorflow/third_party/gpus/crosstool/LICENSE
-tensorflow/third_party/gpus/crosstool/remote.BUILD.tpl
 tensorflow/third_party/gpus/crosstool/windows/msvc_wrapper_for_nvcc.py.tpl
 tensorflow/third_party/gpus/crosstool/BUILD.tpl
 tensorflow/third_party/gpus/crosstool/BUILD
@@ -90,7 +89,6 @@ tensorflow/third_party/gpus/cuda/LICENSE
 tensorflow/third_party/gpus/cuda/BUILD.tpl
 tensorflow/third_party/gpus/cuda/BUILD.windows.tpl
 tensorflow/third_party/gpus/cuda/cuda_config.h.tpl
-tensorflow/third_party/gpus/cuda/remote.BUILD.tpl
 tensorflow/third_party/gpus/cuda/BUILD
 tensorflow/third_party/gpus/cuda/build_defs.bzl.tpl
 tensorflow/third_party/gpus/rocm/rocm_config.h.tpl
@@ -194,7 +192,6 @@ tensorflow/third_party/tensorrt/BUILD
 tensorflow/third_party/tensorrt/build_defs.bzl.tpl
 tensorflow/third_party/tensorrt/BUILD.tpl
 tensorflow/third_party/tensorrt/tensorrt_configure.bzl
-tensorflow/third_party/tensorrt/remote.BUILD.tpl
 tensorflow/third_party/kafka/config.patch
 tensorflow/third_party/kafka/BUILD
 tensorflow/third_party/android/BUILD
diff --git a/third_party/gpus/crosstool/remote.BUILD.tpl b/third_party/gpus/crosstool/remote.BUILD.tpl
deleted file mode 100644
index b2316331db..0000000000
--- a/third_party/gpus/crosstool/remote.BUILD.tpl
+++ /dev/null
@@ -1,10 +0,0 @@
-# Description:
-#   Template for crosstool Build file to use a pre-generated config.
-licenses(["restricted"])
-
-package(default_visibility = ["//visibility:public"])
-
-alias(
-    name = "toolchain",
-    actual = "%{remote_cuda_repo}:toolchain",
-)
diff --git a/third_party/gpus/cuda/remote.BUILD.tpl b/third_party/gpus/cuda/remote.BUILD.tpl
deleted file mode 100644
index 100c7bb7c4..0000000000
--- a/third_party/gpus/cuda/remote.BUILD.tpl
+++ /dev/null
@@ -1,110 +0,0 @@
-# Description:
-#   Template for cuda Build file to use a pre-generated config.
-licenses(["restricted"])  # MPL2, portions GPL v3, LGPL v3, BSD-like
-
-package(default_visibility = ["//visibility:public"])
-
-config_setting(
-    name = "using_nvcc",
-    values = {
-        "define": "using_cuda_nvcc=true",
-    },
-)
-
-config_setting(
-    name = "using_clang",
-    values = {
-        "define": "using_cuda_clang=true",
-    },
-)
-
-# Equivalent to using_clang && -c opt.
-config_setting(
-    name = "using_clang_opt",
-    values = {
-        "define": "using_cuda_clang=true",
-        "compilation_mode": "opt",
-    },
-)
-
-config_setting(
-    name = "darwin",
-    values = {"cpu": "darwin"},
-    visibility = ["//visibility:public"],
-)
-
-config_setting(
-    name = "freebsd",
-    values = {"cpu": "freebsd"},
-    visibility = ["//visibility:public"],
-)
-
-alias(
-    name = "cuda_headers",
-    actual = "%{remote_cuda_repo}/cuda:cuda_headers",
-)
-
-alias(
-    name = "cudart_static",
-    actual = "%{remote_cuda_repo}/cuda:cudart_static",
-)
-
-alias(
-    name = "cuda_driver",
-    actual = "%{remote_cuda_repo}/cuda:cuda_driver",
-)
-
-alias(
-    name = "cudart",
-    actual = "%{remote_cuda_repo}/cuda:cudart",
-)
-
-alias(
-    name = "cublas",
-    actual = "%{remote_cuda_repo}/cuda:cublas",
-)
-
-alias(
-    name = "cusolver",
-    actual = "%{remote_cuda_repo}/cuda:cusolver",
-)
-
-alias(
-    name = "cudnn",
-    actual = "%{remote_cuda_repo}/cuda:cudnn",
-)
-
-alias(
-    name = "cudnn_header",
-    actual = "%{remote_cuda_repo}/cuda:cudnn_header",
-)
-
-alias(
-    name = "cufft",
-    actual = "%{remote_cuda_repo}/cuda:cufft",
-)
-
-alias(
-    name = "curand",
-    actual = "%{remote_cuda_repo}/cuda:curand",
-)
-
-alias(
-    name = "cuda",
-    actual = "%{remote_cuda_repo}/cuda:cuda",
-)
-
-alias(
-    name = "cupti_headers",
-    actual = "%{remote_cuda_repo}/cuda:cupti_headers",
-)
-
-alias(
-    name = "cupti_dsos",
-    actual = "%{remote_cuda_repo}/cuda:cupti_dsos",
-)
-
-alias(
-    name = "libdevice_root",
-    actual = "%{remote_cuda_repo}/cuda:libdevice_root",
-)
diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl
index 90020b6608..76f2f3a07b 100644
--- a/third_party/gpus/cuda_configure.bzl
+++ b/third_party/gpus/cuda_configure.bzl
@@ -1508,17 +1508,21 @@ def _create_remote_cuda_repository(repository_ctx, remote_config_repo):
               ),
       },
   )
-  _tpl(
-      repository_ctx,
-      "cuda:remote.BUILD",
-      {
-          "%{remote_cuda_repo}": remote_config_repo,
-      },
+  repository_ctx.template(
       "cuda/BUILD",
+      Label(remote_config_repo + "/cuda:BUILD"),
+      {},
+  )
+  repository_ctx.template(
+      "cuda/build_defs.bzl",
+      Label(remote_config_repo + "/cuda:build_defs.bzl"),
+      {},
+  )
+  repository_ctx.template(
+      "cuda/cuda/cuda_config.h",
+      Label(remote_config_repo + "/cuda:cuda/cuda_config.h"),
+      {},
   )
-  _tpl(repository_ctx, "crosstool:remote.BUILD", {
-      "%{remote_cuda_repo}": remote_config_repo,
-  }, "crosstool/BUILD")
 
 
 def _cuda_autoconf_impl(repository_ctx):
diff --git a/third_party/tensorrt/build_defs.bzl.tpl b/third_party/tensorrt/build_defs.bzl.tpl
index 0dc3a7ba2d..6d00513827 100644
--- a/third_party/tensorrt/build_defs.bzl.tpl
+++ b/third_party/tensorrt/build_defs.bzl.tpl
@@ -2,6 +2,4 @@
 
 def if_tensorrt(if_true, if_false=[]):
   """Tests whether TensorRT was enabled during the configure process."""
-  if %{tensorrt_is_configured}:
-    return if_true
-  return if_false
+  return %{if_tensorrt}
diff --git a/third_party/tensorrt/remote.BUILD.tpl b/third_party/tensorrt/remote.BUILD.tpl
deleted file mode 100644
index d5afc4a5c1..0000000000
--- a/third_party/tensorrt/remote.BUILD.tpl
+++ /dev/null
@@ -1,7 +0,0 @@
-licenses(["notice"])  # Apache 2.0
-
-package(default_visibility = ["//visibility:public"])
-
-alias(name="LICENSE", actual = "%{target}:LICENSE")
-alias(name = "tensorrt_headers", actual = "%{target}:tensorrt_headers")
-alias(name = "tensorrt", actual = "%{target}:tensorrt")
diff --git a/third_party/tensorrt/tensorrt_configure.bzl b/third_party/tensorrt/tensorrt_configure.bzl
index d3bc388346..c6de25b33e 100644
--- a/third_party/tensorrt/tensorrt_configure.bzl
+++ b/third_party/tensorrt/tensorrt_configure.bzl
@@ -129,7 +129,7 @@ def _tpl(repository_ctx, tpl, substitutions):
 
 def _create_dummy_repository(repository_ctx):
   """Create a dummy TensorRT repository."""
-  _tpl(repository_ctx, "build_defs.bzl", {"%{tensorrt_is_configured}": "False"})
+  _tpl(repository_ctx, "build_defs.bzl", {"%{if_tensorrt}": "if_false"})
 
   _tpl(repository_ctx, "BUILD", {
       "%{tensorrt_genrules}": "",
@@ -141,11 +141,13 @@ def _tensorrt_configure_impl(repository_ctx):
   """Implementation of the tensorrt_configure repository rule."""
   if _TF_TENSORRT_CONFIG_REPO in repository_ctx.os.environ:
     # Forward to the pre-configured remote repository.
-    repository_ctx.template("BUILD", Label("//third_party/tensorrt:remote.BUILD.tpl"), {
-        "%{target}": repository_ctx.os.environ[_TF_TENSORRT_CONFIG_REPO],
-    })
-    # Set up config file.
-    _tpl(repository_ctx, "build_defs.bzl", {"%{tensorrt_is_configured}": "True"})
+    remote_config_repo = repository_ctx.os.environ[_TF_TENSORRT_CONFIG_REPO]
+    repository_ctx.template("BUILD", Label(remote_config_repo + ":BUILD"), {})
+    repository_ctx.template(
+        "build_defs.bzl",
+        Label(remote_config_repo + ":build_defs.bzl"),
+        {},
+    )
     return
 
   if _TENSORRT_INSTALL_PATH not in repository_ctx.os.environ:
@@ -194,7 +196,7 @@ def _tensorrt_configure_impl(repository_ctx):
   ))
 
   # Set up config file.
-  _tpl(repository_ctx, "build_defs.bzl", {"%{tensorrt_is_configured}": "True"})
+  _tpl(repository_ctx, "build_defs.bzl", {"%{if_tensorrt}": "if_true"})
 
   # Set up BUILD file.
   _tpl(repository_ctx, "BUILD", {
-- 
GitLab


From f6b81f458da1f7b68e06c6a4501c5c380d33d590 Mon Sep 17 00:00:00 2001
From: Ayush Dubey <ayushd@google.com>
Date: Fri, 18 Jan 2019 08:15:00 -0800
Subject: [PATCH 0956/2345] Automated rollback of commit
 681f6a6ac9344425549c8a38fcbfae854e7deddf

PiperOrigin-RevId: 229930698
---
 tensorflow/core/BUILD                         |   2 +-
 .../base_collective_executor.cc               |  33 ++
 .../common_runtime/base_collective_executor.h |  18 +
 .../collective_param_resolver_local.cc        | 159 ++++++---
 .../collective_param_resolver_local.h         |  15 +-
 .../core/common_runtime/collective_util.cc    |  31 ++
 .../core/common_runtime/collective_util.h     |  21 ++
 .../hierarchical_tree_broadcaster.h           |   5 +
 .../core/common_runtime/ring_reducer.cc       |  43 +--
 tensorflow/core/common_runtime/ring_reducer.h |  24 +-
 tensorflow/core/distributed_runtime/BUILD     |   1 +
 .../collective_param_resolver_distributed.cc  |  21 +-
 ...lective_param_resolver_distributed_test.cc |   2 +
 tensorflow/core/framework/collective.cc       |  11 +
 tensorflow/core/framework/collective.h        |  32 +-
 tensorflow/core/kernels/BUILD                 |  25 ++
 .../core/kernels/collective_nccl_reducer.cc   | 204 +++++++++++
 .../core/kernels/collective_nccl_reducer.h    |  49 +++
 .../kernels/collective_nccl_reducer_test.cc   | 331 ++++++++++++++++++
 tensorflow/core/kernels/collective_ops.cc     |  23 +-
 tensorflow/core/nccl/BUILD                    |   4 +-
 tensorflow/core/protobuf/worker.proto         |   1 +
 22 files changed, 927 insertions(+), 128 deletions(-)
 create mode 100644 tensorflow/core/kernels/collective_nccl_reducer.cc
 create mode 100644 tensorflow/core/kernels/collective_nccl_reducer.h
 create mode 100644 tensorflow/core/kernels/collective_nccl_reducer_test.cc

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 084db7a0fd..ceec270f99 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -3719,7 +3719,6 @@ tf_cc_tests(
     srcs = [
         "common_runtime/buf_rendezvous_test.cc",
         "common_runtime/collective_executor_mgr_test.cc",
-        "common_runtime/collective_param_resolver_local_test.cc",
         "common_runtime/collective_rma_local_test.cc",
         "common_runtime/device_resolver_local_test.cc",
         "common_runtime/device_set_test.cc",
@@ -3835,6 +3834,7 @@ tf_cc_tests(
     name = "higher_level_tests_needing_kernels",
     size = "small",
     srcs = [
+        "common_runtime/collective_param_resolver_local_test.cc",
         "graph/graph_constructor_test.cc",
     ],
     linkopts = select({
diff --git a/tensorflow/core/common_runtime/base_collective_executor.cc b/tensorflow/core/common_runtime/base_collective_executor.cc
index 92e56df181..03c0e9ce2b 100644
--- a/tensorflow/core/common_runtime/base_collective_executor.cc
+++ b/tensorflow/core/common_runtime/base_collective_executor.cc
@@ -296,4 +296,37 @@ Status BaseCollectiveExecutor::CreateCollective(
   return status;
 }
 
+bool BaseCollectiveExecutor::CheckDependencies(
+    const CollectiveParams& col_params) {
+  for (int32 instance : col_params.instance.impl_details.dependencies) {
+    auto find_iter = launched_.find(instance);
+    if (find_iter == launched_.end() || find_iter->second != 0) {
+      return false;
+    }
+  }
+  return true;
+}
+
+void BaseCollectiveExecutor::WaitForDependencies(
+    const CollectiveParams& col_params) {
+  mutex_lock l(launch_mu_);
+  while (!CheckDependencies(col_params)) {
+    launch_cv_.wait(l);
+  }
+}
+
+void BaseCollectiveExecutor::Launched(const CollectiveParams& col_params) {
+  mutex_lock l(launch_mu_);
+  if (launched_.find(col_params.instance.instance_key) == launched_.end()) {
+    const string& task_name =
+        col_params.instance.task_names[col_params.default_rank];
+    const int32 num_devices =
+        col_params.instance.num_devices_per_task.at(task_name);
+    launched_[col_params.instance.instance_key] = num_devices;
+  }
+  if (--launched_[col_params.instance.instance_key] == 0) {
+    launch_cv_.notify_all();
+  }
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/base_collective_executor.h b/tensorflow/core/common_runtime/base_collective_executor.h
index 09826a8814..b711aa6d50 100644
--- a/tensorflow/core/common_runtime/base_collective_executor.h
+++ b/tensorflow/core/common_runtime/base_collective_executor.h
@@ -135,15 +135,33 @@ class BaseCollectiveExecutor : public CollectiveExecutor {
                                client_locality, done);
   }
 
+  // If we need to enforce an ordering on any portion of collective
+  // implementation, and the ordering is encoded via attribute on the collective
+  // op, this function will block until all dependencies for this collective
+  // have completed.
+  void WaitForDependencies(const CollectiveParams& col_params) override;
+  // Record that this collective has completed the portion of the implementation
+  // that needs to be ordered wrt other collectives, to unblock any of its
+  // dependent ops.
+  void Launched(const CollectiveParams& col_params) override;
+
  protected:
   const int64 step_id_;
   const DeviceMgr* dev_mgr_;  // Not owned.
   std::unique_ptr<PerStepCollectiveRemoteAccess> remote_access_;
   const string* gpu_ring_order_;  // Not owned.
+  mutex launch_mu_;
+  condition_variable launch_cv_;
+  // collective instance key -> number of local devices for which NCCL ops have
+  // been launched.
+  std::unordered_map<int32, int32> launched_ GUARDED_BY(launch_mu_);
 
  private:
   Status CreateCollective(const CollectiveParams& col_params,
                           CollectiveImplementationInterface** col_impl);
+  // Check if all ops on which this collective depends on have launched.
+  bool CheckDependencies(const CollectiveParams& col_params)
+      EXCLUSIVE_LOCKS_REQUIRED(launch_mu_);
 };
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/collective_param_resolver_local.cc b/tensorflow/core/common_runtime/collective_param_resolver_local.cc
index a8e3f4c881..8907f6d56a 100644
--- a/tensorflow/core/common_runtime/collective_param_resolver_local.cc
+++ b/tensorflow/core/common_runtime/collective_param_resolver_local.cc
@@ -26,6 +26,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/gtl/flatmap.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/types.h"
 #include "tensorflow/core/util/device_name_utils.h"
@@ -39,7 +40,10 @@ void CollectiveParamResolverLocal::InstanceRec::WaitForOutMu(mutex_lock& lock) {
 CollectiveParamResolverLocal::CollectiveParamResolverLocal(
     const DeviceMgr* dev_mgr, DeviceResolverInterface* dev_resolver,
     const string& task_name)
-    : dev_mgr_(dev_mgr), dev_resolver_(dev_resolver), task_name_(task_name) {}
+    : nccl_(false),  // (b/111897089): turn on NCCL collectives.
+      dev_mgr_(dev_mgr),
+      dev_resolver_(dev_resolver),
+      task_name_(task_name) {}
 
 void CollectiveParamResolverLocal::CompleteGroupAsync(
     const CompleteGroupRequest* request, CompleteGroupResponse* response,
@@ -316,29 +320,28 @@ GlobalDeviceMap EstablishGlobalRank(
 // cp->same_num_devices_per_task.  Requires cp->instance.task_names
 // be sorted.
 void SetDevPerTask(CollectiveParams* cp) {
-  cp->instance.same_num_devices_per_task = false;
-  if (cp->instance.task_names.empty()) return;
-  int dev_per_task = -1;
-  int count = 0;
+  cp->instance.num_devices_per_task.clear();
   const string* last_task_name = &cp->instance.task_names[0];
+  int count = 0;
   for (const string& task_name : cp->instance.task_names) {
-    if (task_name != *last_task_name) {
-      CHECK_GT(count, 0);
-      if (dev_per_task < 0) {
-        dev_per_task = count;
-      } else {
-        CHECK_GT(dev_per_task, 0);
-        if (count != dev_per_task) return;
-      }
+    if (task_name == *last_task_name) {
+      ++count;
+    } else {
+      cp->instance.num_devices_per_task[*last_task_name] = count;
       count = 1;
       last_task_name = &task_name;
-    } else {
-      ++count;
     }
   }
-  CHECK_GT(count, 0);
-  if ((dev_per_task > 0) && (count != dev_per_task)) {
-    return;
+  cp->instance.num_devices_per_task[*last_task_name] = count;
+
+  cp->instance.same_num_devices_per_task = false;
+  int dev_per_task = -1;
+  for (const auto& task_dev : cp->instance.num_devices_per_task) {
+    if (dev_per_task == -1) {
+      dev_per_task = task_dev.second;
+    } else if (dev_per_task != task_dev.second) {
+      return;
+    }
   }
   cp->instance.same_num_devices_per_task = true;
   CHECK_EQ((cp->group.group_size % cp->group.num_tasks), 0);
@@ -398,7 +401,6 @@ void CollectiveParamResolverLocal::SetDefaultRank(const string& device,
 void CollectiveParamResolverLocal::InitInstanceSharedParams(
     const GroupRec* gr, const CollectiveParams* cp, InstanceRec* ir,
     const StatusCallback& done) {
-  VLOG(1) << "InitInstanceSharedParams " << ir;
   ir->shared.instance = cp->instance;
   {
     mutex_lock gl(gr->mu);
@@ -412,8 +414,8 @@ void CollectiveParamResolverLocal::InitInstanceSharedParams(
   }
   ir->shared.default_rank = -1;
 
-  // Sort devce_names lexicographcally, keeping task_names in
-  // corresponding order.
+  // Sort device_names lexicographically, keeping task_names in corresponding
+  // order.  Also set number of devices per task.
   SortDevicesAndTasks(&ir->shared);
 
   // Get Locality data for all devices.
@@ -605,6 +607,25 @@ void CollectiveParamResolverLocal::CompleteInstanceAsync(
                        "intended only for non-distributed deployment."));
 }
 
+// TODO(b/111897089): we need a better way to pick the collective
+// implementation.  The ideal way would depend upon the topology and link
+// strength before picking a particular implementation.
+void CollectiveParamResolverLocal::AssignCollectiveType(CollectiveParams* cp) {
+  if (cp->instance.type == BROADCAST_COLLECTIVE) {
+    cp->instance.impl_details.collective_name = "HierarchicalTreeBroadcast";
+  } else if (cp->instance.type == REDUCTION_COLLECTIVE) {
+    if (nccl_) {
+      cp->instance.impl_details.collective_name = "NcclReduce";
+    } else {
+      cp->instance.impl_details.collective_name = "RingReduce";
+    }
+  } else {
+    cp->instance.impl_details.collective_name = "undef";
+  }
+  VLOG(1) << "AssignCollectiveType "
+          << cp->instance.impl_details.collective_name;
+}
+
 void CollectiveParamResolverLocal::CompleteInstanceLocal(
     const string& device, const GroupRec* gr, CollectiveParams* cp,
     bool is_source, const StatusCallback& done) {
@@ -641,48 +662,57 @@ void CollectiveParamResolverLocal::CompleteInstanceFromInitializedIRec(
     // custom operator= does a deep copy.
     cp->instance = ir->shared.instance;
   }
-  // Populate the fields common across task, also default_rank.
+  // Populate the fields common across task.
+  AssignCollectiveType(cp);
   SetDefaultRank(device, cp);
   CompleteTaskIsLocal(task_name_, cp);
-  // TODO(b/113171733): we need a better way to pick the collective
-  // implementation.  The ideal way would depend upon the topology and link
-  // strength before picking a particular implementation.
-  cp->instance.impl_details.collective_name =
-      (cp->instance.type == BROADCAST_COLLECTIVE) ? "HierarchicalTreeBroadcast"
-                                                  : "RingReduce";
+
   CollectiveImplementationInterface* col_impl;
-  Status lookup_status = CollectiveRegistry::LookupParamResolverInstance(
+  Status status = CollectiveRegistry::LookupParamResolverInstance(
       cp->instance.impl_details.collective_name, &col_impl);
-  if (!lookup_status.ok()) {
-    done(lookup_status);
+  if (status.ok()) {
+    status = col_impl->InitializeInstanceBeforeGroupDiscovery(cp);
+  }
+  if (!status.ok()) {
+    done(status);
     return;
   }
-  // If broadcast, may need to wait for source discovery.
-  if (cp->instance.type == BROADCAST_COLLECTIVE) {
-    CompleteInstanceSource(ir, cp, is_source,
-                           [col_impl, ir, device, cp, done](InstanceRec* irec) {
-                             CHECK_EQ(ir, irec);
-                             Status s;
-                             {
-                               mutex_lock l(irec->out_mu);
-                               irec->WaitForOutMu(l);
-                               s = irec->status;
-                               cp->source_rank = irec->source_rank;
-                             }
-                             if (s.ok()) {
-                               s = col_impl->InitializeCollectiveParams(cp);
-                             }
-                             done(s);
-                           });
+
+  //  We may need to wait for the group if:
+  //  * this is a broadcast, for source discovery;
+  //  * we are using NCCL with more than 1 worker, for the communicator key from
+  //    rank 0.
+  bool broadcast = cp->instance.type == BROADCAST_COLLECTIVE;
+  bool nccl = cp->instance.type == REDUCTION_COLLECTIVE &&
+              cp->instance.impl_details.collective_name == "NcclReduce" &&
+              cp->group.num_tasks > 1;
+  if (broadcast || nccl) {
+    WaitForGroup(ir, cp, is_source, broadcast, nccl,
+                 [col_impl, ir, device, cp, done](InstanceRec* irec) {
+                   Status s;
+                   if (ir != irec) {
+                     s = errors::Internal("Expected ir ", ir, " and irec ",
+                                          irec, " to be equal");
+                   } else {
+                     mutex_lock l(irec->out_mu);
+                     irec->WaitForOutMu(l);
+                     s = irec->status;
+                     cp->source_rank = irec->source_rank;
+                     cp->instance.communicator_key = irec->communicator_key;
+                   }
+                   if (s.ok()) {
+                     s = col_impl->InitializeCollectiveParams(cp);
+                   }
+                   done(s);
+                 });
   } else {
     done(col_impl->InitializeCollectiveParams(cp));
   }
 }
 
-void CollectiveParamResolverLocal::CompleteInstanceSource(InstanceRec* ir,
-                                                          CollectiveParams* cp,
-                                                          bool is_source,
-                                                          const IRConsumer& f) {
+void CollectiveParamResolverLocal::WaitForGroup(
+    InstanceRec* ir, CollectiveParams* cp, bool is_source, bool init_source,
+    bool init_nccl, const IRConsumer& f) {
   std::vector<IRConsumer> ready_waiters;
   {
     mutex_lock l(ir->out_mu);
@@ -692,7 +722,8 @@ void CollectiveParamResolverLocal::CompleteInstanceSource(InstanceRec* ir,
     if (!ir->known[cp->default_rank]) {
       ir->known[cp->default_rank] = true;
       ++ir->known_count;
-      if (is_source) {
+      if (init_source && is_source) {
+        // Initialize source rank.
         if (ir->source_rank >= 0) {
           ir->status = errors::Internal("Instance ", cp->instance.instance_key,
                                         " already has source ", ir->source_rank,
@@ -702,13 +733,26 @@ void CollectiveParamResolverLocal::CompleteInstanceSource(InstanceRec* ir,
           ir->source_rank = cp->default_rank;
         }
       }
+      if (init_nccl && cp->default_rank == 0) {
+        // Initialize communicator key.
+        if (!ir->communicator_key.empty()) {
+          ir->status =
+              errors::Internal("Instance ", cp->instance.instance_key,
+                               " already has communicator_key ",
+                               str_util::CEscape(ir->communicator_key),
+                               ", received second claim from device ",
+                               cp->instance.device_names[cp->default_rank]);
+        } else {
+          ir->communicator_key = cp->instance.communicator_key;
+        }
+      }
     }
     if (ir->known_count < ir->shared.group.group_size) {
       ir->known_waiters.push_back(f);
       return;
     }
     CHECK_EQ(ir->known_count, ir->shared.group.group_size);
-    if (ir->source_rank < 0) {
+    if (init_source && ir->source_rank < 0) {
       // NOTE(ayushd): changing the error message below would also require
       // updating CompleteParamsBroadcastForgotSend test in
       // CollectiveParamResolverLocalTest.
@@ -718,6 +762,13 @@ void CollectiveParamResolverLocal::CompleteInstanceSource(InstanceRec* ir,
                            "could mean that there were group_size=",
                            ir->known_count, " BcastRecvs but no BcastSend.");
     }
+    if (init_nccl && ir->communicator_key.empty()) {
+      ir->status = errors::Internal(
+          "Instance ", cp->instance.instance_key, " device ",
+          cp->instance.device_names[cp->default_rank],
+          " did not find rank 0 for setting communicator key.  This is an "
+          "internal error in collective param resolution");
+    }
     if (!ir->known_waiters.empty()) {
       ready_waiters = std::move(ir->known_waiters);
     }
diff --git a/tensorflow/core/common_runtime/collective_param_resolver_local.h b/tensorflow/core/common_runtime/collective_param_resolver_local.h
index 365bddc787..fd408e4ef3 100644
--- a/tensorflow/core/common_runtime/collective_param_resolver_local.h
+++ b/tensorflow/core/common_runtime/collective_param_resolver_local.h
@@ -130,8 +130,10 @@ class CollectiveParamResolverLocal : public ParamResolverInterface {
     Status status GUARDED_BY(out_mu);
 
     // These fields are used to count the instances that have called
-    // in and become known while resolving broadcast source identity.
+    // in and become known while resolving broadcast source identity and
+    // communicator key.
     int source_rank GUARDED_BY(out_mu);
+    string communicator_key GUARDED_BY(out_mu);
     int known_count GUARDED_BY(out_mu);
     std::vector<bool> known GUARDED_BY(out_mu);
     std::vector<IRConsumer> known_waiters GUARDED_BY(out_mu);
@@ -197,10 +199,10 @@ class CollectiveParamResolverLocal : public ParamResolverInterface {
                                            const StatusCallback& done)
       LOCKS_EXCLUDED(ir->out_mu);
 
-  // Complete source data for a broadcast instance.
+  // Complete source data and/or nccl communicator key.
   // Precondition: *cp has complete group data and default_rank.
-  void CompleteInstanceSource(InstanceRec* ir, CollectiveParams* cp,
-                              bool is_source, const IRConsumer& f)
+  void WaitForGroup(InstanceRec* ir, CollectiveParams* cp, bool is_source,
+                    bool init_source, bool init_nccl, const IRConsumer& f)
       LOCKS_EXCLUDED(ir->out_mu);
 
   // If cp.device_names contains only devices local to this process
@@ -216,10 +218,15 @@ class CollectiveParamResolverLocal : public ParamResolverInterface {
   // current ordering of cp->instance.device_names.
   void SetDefaultRank(const string& device, CollectiveParams* cp);
 
+  // Sets cp->instance.type based on collective op type, and attempts to assign
+  // best implementation.
+  void AssignCollectiveType(CollectiveParams* cp);
+
   // Helper to grab status under lock, invoke callback out of lock.
   void CallbackWithStatus(const InstanceRecCallback& done, InstanceRec* irec)
       LOCKS_EXCLUDED(irec->out_mu);
 
+  const bool nccl_;
   const DeviceMgr* dev_mgr_;
   DeviceResolverInterface* dev_resolver_;  // Not owned.
   string task_name_;
diff --git a/tensorflow/core/common_runtime/collective_util.cc b/tensorflow/core/common_runtime/collective_util.cc
index 195521a078..bee4a13d18 100644
--- a/tensorflow/core/common_runtime/collective_util.cc
+++ b/tensorflow/core/common_runtime/collective_util.cc
@@ -79,5 +79,36 @@ string SubdivPermDebugString(const CollectiveParams& col_params) {
   return buf;
 }
 
+SubContext::SubContext(OpKernelContext* ctx, OpKernelContext::Params* params,
+                       OpKernel* op, Tensor* output, Tensor* input)
+    : sub_params_(*params),
+      sub_inputs_({output, input}),
+      sub_input_attr_({ctx->input_alloc_attr(0), ctx->input_alloc_attr(0)}),
+      sub_input_dc_(
+          {ctx->input_device_context(0), ctx->input_device_context(0)}) {
+  sub_params_.op_kernel = op;
+  sub_params_.inputs = &sub_inputs_;
+  sub_params_.input_alloc_attrs = &sub_input_attr_;
+  sub_params_.input_device_contexts = &sub_input_dc_;
+  sub_params_.eigen_gpu_device = nullptr;
+  sub_params_.ensure_eigen_gpu_device();
+  sub_params_.forward_from_array = &forward_from_;
+  sub_ctx_.reset(new OpKernelContext(&sub_params_, 1));
+}
+
+Status ComputeBinOp(OpKernelContext* op_ctx, OpKernelContext::Params* params,
+                    Device* device, OpKernel* op, Tensor* output,
+                    Tensor* input) {
+  // Prepare an OpKernelContext that is identical to that of the original Op
+  // (i.e. the collective), except for the input output sizes and identities and
+  // the Op itself.
+  // TODO(ayushd, tucker): Is it possible to cache and reuse these objects?
+  // They're mostly identical inside one device execution.
+  std::unique_ptr<SubContext> sub_ctx(
+      new SubContext(op_ctx, params, op, output, input));
+  device->Compute(op, sub_ctx->sub_ctx_.get());
+  return sub_ctx->sub_ctx_->status();
+}
+
 }  // namespace collective_util
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/collective_util.h b/tensorflow/core/common_runtime/collective_util.h
index ebb5731bec..01fb8b8c81 100644
--- a/tensorflow/core/common_runtime/collective_util.h
+++ b/tensorflow/core/common_runtime/collective_util.h
@@ -32,6 +32,27 @@ Status InitializeDeviceAndLocality(const DeviceMgr* dev_mgr,
                                    DeviceLocality* device_locality);
 string SubdivPermDebugString(const CollectiveParams& col_params);
 
+// Used for executing a sub-operation, e.g. a merge_op instance, with
+// an OpKernelContext based on the one passed into this Op.
+class SubContext {
+ public:
+  OpKernelContext::Params sub_params_;
+  gtl::InlinedVector<TensorValue, 4> sub_inputs_;
+  gtl::InlinedVector<AllocatorAttributes, 4> sub_input_attr_;
+  gtl::InlinedVector<DeviceContext*, 4> sub_input_dc_;
+  // Used only for Binary and Unary Ops for which we require
+  // the calculation to be in-place on the first input.
+  int forward_from_ = 0;
+  std::unique_ptr<OpKernelContext> sub_ctx_;
+  SubContext(OpKernelContext* ctx, OpKernelContext::Params* params,
+             OpKernel* op, Tensor* output, Tensor* input);
+  ~SubContext() = default;
+};
+
+Status ComputeBinOp(OpKernelContext* op_ctx, OpKernelContext::Params* params,
+                    Device* device, OpKernel* op, Tensor* output,
+                    Tensor* input);
+
 }  // namespace collective_util
 }  // namespace tensorflow
 
diff --git a/tensorflow/core/common_runtime/hierarchical_tree_broadcaster.h b/tensorflow/core/common_runtime/hierarchical_tree_broadcaster.h
index ceb9baad30..76392b8e59 100644
--- a/tensorflow/core/common_runtime/hierarchical_tree_broadcaster.h
+++ b/tensorflow/core/common_runtime/hierarchical_tree_broadcaster.h
@@ -41,6 +41,11 @@ class HierarchicalTreeBroadcaster : public CollectiveImplementationInterface {
   // and device_locality.  Also saves the CollectiveContext in this object.
   Status InitializeCollectiveContext(CollectiveContext* col_ctx) override;
 
+  // No-op for hierarchical tree broadcaster.
+  Status InitializeInstanceBeforeGroupDiscovery(CollectiveParams*) override {
+    return Status::OK();
+  }
+
   // Begins async execution of the hierarchical tree broadcast.
   // Must be called in a blockable thread.
   // TODO(b/80529858): remove the previous warning when we have a dedicated
diff --git a/tensorflow/core/common_runtime/ring_reducer.cc b/tensorflow/core/common_runtime/ring_reducer.cc
index 092f15e49e..8ed2fc2f1c 100644
--- a/tensorflow/core/common_runtime/ring_reducer.cc
+++ b/tensorflow/core/common_runtime/ring_reducer.cc
@@ -394,37 +394,6 @@ void RingReducer::Finish(bool ok) {
   done_(s);
 }
 
-RingReducer::SubContext::SubContext(OpKernelContext* ctx,
-                                    OpKernelContext::Params* params,
-                                    OpKernel* op, Tensor* output, Tensor* input)
-    : sub_params_(*params),
-      sub_inputs_({output, input}),
-      sub_input_attr_({ctx->input_alloc_attr(0), ctx->input_alloc_attr(0)}),
-      sub_input_dc_(
-          {ctx->input_device_context(0), ctx->input_device_context(0)}) {
-  sub_params_.op_kernel = op;
-  sub_params_.inputs = &sub_inputs_;
-  sub_params_.input_alloc_attrs = &sub_input_attr_;
-  sub_params_.input_device_contexts = &sub_input_dc_;
-  sub_params_.eigen_gpu_device = nullptr;
-  sub_params_.ensure_eigen_gpu_device();
-  sub_params_.forward_from_array = &forward_from_;
-  sub_ctx_ = new OpKernelContext(&sub_params_, 1);
-}
-
-Status RingReducer::ComputeBinOp(Device* device, OpKernel* op, Tensor* output,
-                                 Tensor* input) {
-  // Prepare an OpKernelContext that is identical to that of the original Op
-  // (i.e. the collective), except for the input output sizes and identities and
-  // the Op itself.
-  // TODO(tucker): Is it possible to cache and reuse these objects?  They're
-  // mostly identical inside one device execution.
-  std::unique_ptr<SubContext> sub_ctx(
-      new SubContext(col_ctx_->op_ctx, col_ctx_->op_params, op, output, input));
-  device->Compute(op, sub_ctx->sub_ctx_);
-  return sub_ctx->sub_ctx_->status();
-}
-
 // At the beginning of the algorithm initialize a RingField struct for
 // every independent field of the tensor.
 void RingReducer::InitRingField(RingField* rf, int chunk_idx, int subdiv_idx,
@@ -632,9 +601,9 @@ bool RingReducer::RunAsyncParts() {
           --recv_pending_count;
           if (!rf->second_pass) {
             rf->action = RF_REDUCE;
-            Status s =
-                ComputeBinOp(col_ctx_->device, col_params_->merge_op.get(),
-                             &rf->chunk, &rf->tmp_chunk);
+            Status s = collective_util::ComputeBinOp(
+                col_ctx_->op_ctx, col_ctx_->op_params, col_ctx_->device,
+                col_params_->merge_op.get(), &rf->chunk, &rf->tmp_chunk);
             if (!s.ok()) {
               aborted = true;
               StartAbort(s);
@@ -647,9 +616,9 @@ bool RingReducer::RunAsyncParts() {
           if (!rf->second_pass && col_params_->final_op.get() && rf->is_final) {
             rf->action = RF_FINALIZE;
             group_size_tensor_ready_.WaitForNotification();
-            Status s =
-                ComputeBinOp(col_ctx_->device, col_params_->final_op.get(),
-                             &rf->chunk, &group_size_tensor_);
+            Status s = collective_util::ComputeBinOp(
+                col_ctx_->op_ctx, col_ctx_->op_params, col_ctx_->device,
+                col_params_->final_op.get(), &rf->chunk, &group_size_tensor_);
             if (!s.ok()) {
               aborted = true;
               StartAbort(s);
diff --git a/tensorflow/core/common_runtime/ring_reducer.h b/tensorflow/core/common_runtime/ring_reducer.h
index 0848e37b52..a5aa8fad70 100644
--- a/tensorflow/core/common_runtime/ring_reducer.h
+++ b/tensorflow/core/common_runtime/ring_reducer.h
@@ -40,6 +40,11 @@ class RingReducer : public CollectiveImplementationInterface {
   // and device_locality.  Also saves the CollectiveContext in this object.
   Status InitializeCollectiveContext(CollectiveContext* col_ctx) override;
 
+  // No-op for ring reducer.
+  Status InitializeInstanceBeforeGroupDiscovery(CollectiveParams*) override {
+    return Status::OK();
+  }
+
   // Begins async execution of the ring reduce algorithm.
   // Must be called in a blockable thread.
   // TODO(b/80529858): remove the previous warning when we have a dedicated
@@ -52,27 +57,8 @@ class RingReducer : public CollectiveImplementationInterface {
   void StartAbort(const Status& s);
   void ContinueAfterInputCopy();
   void Finish(bool ok);
-  Status ComputeBinOp(Device* device, OpKernel* op, Tensor* output,
-                      Tensor* input);
   bool RunAsyncParts();
 
-  // Used for executing a sub-operation, e.g. a merge_op instance, with
-  // an OpKernelContext based on the one passed into this Op.
-  class SubContext {
-   public:
-    OpKernelContext::Params sub_params_;
-    gtl::InlinedVector<TensorValue, 4> sub_inputs_;
-    gtl::InlinedVector<AllocatorAttributes, 4> sub_input_attr_;
-    gtl::InlinedVector<DeviceContext*, 4> sub_input_dc_;
-    // Used only for Binary and Unary Ops for which we require
-    // the calculation to be in-place on the first input.
-    int forward_from_ = 0;
-    OpKernelContext* sub_ctx_;
-    SubContext(OpKernelContext* ctx, OpKernelContext::Params* params,
-               OpKernel* op, Tensor* output, Tensor* input);
-    ~SubContext() { delete sub_ctx_; }
-  };
-
   // Current status of a RingField
   enum RingFieldAction {
     RF_INIT = 0,    // Just initialized for a pass
diff --git a/tensorflow/core/distributed_runtime/BUILD b/tensorflow/core/distributed_runtime/BUILD
index e388d3e6f0..197a3f24d4 100644
--- a/tensorflow/core/distributed_runtime/BUILD
+++ b/tensorflow/core/distributed_runtime/BUILD
@@ -594,6 +594,7 @@ tf_cc_test(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
+        "//tensorflow/core/kernels:collective_ops",
     ],
 )
 
diff --git a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc
index 1dd10d309b..9f94a24fcd 100644
--- a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc
+++ b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc
@@ -181,6 +181,10 @@ void CollectiveParamResolverDistributed::CompleteInstanceAsync(
                           ir->WaitForOutMu(l);
                           response->set_instance_key(cp->instance.instance_key);
                           response->set_source_rank(ir->source_rank);
+                          if (!cp->instance.communicator_key.empty()) {
+                            response->set_communicator_key(
+                                cp->instance.communicator_key);
+                          }
                           done_and_cleanup(fi_status);
                         } else {
                           done_and_cleanup(fi_status);
@@ -283,8 +287,10 @@ void CollectiveParamResolverDistributed::UpdateInstanceCache(
   using InstanceRecPointer = InstanceRec*;
   InstanceRecPointer* irp = new InstanceRecPointer(nullptr);
   int32 source_rank = resp.source_rank();
+  string communicator_key = resp.communicator_key();
 
-  auto continue_with_ir = [this, cp, irp, source_rank, done](const Status& s) {
+  auto continue_with_ir = [cp, irp, source_rank, communicator_key,
+                           done](const Status& s) {
     if (!s.ok()) {
       done(s);
       delete irp;
@@ -306,6 +312,19 @@ void CollectiveParamResolverDistributed::UpdateInstanceCache(
         }
         ir->source_rank = source_rank;
       }
+      if (ir->communicator_key != communicator_key) {
+        if (!ir->communicator_key.empty()) {
+          ir->status = errors::Internal(
+              "UpdateInstanceCache: CompleteInstanceResponse for instance ",
+              cp->instance.instance_key,
+              " gives communicator_key with size =", communicator_key.size(),
+              " but cache already holds communicator_key with size=",
+              ir->communicator_key.size());
+          status = ir->status;
+          break;
+        }
+        ir->communicator_key = communicator_key;
+      }
       if (ir->known_count < cp->group.group_size) {
         ir->known_count = cp->group.group_size;
         if (ir->known.size() != cp->group.group_size) {
diff --git a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc
index 40b18d321a..823d7d5eb9 100644
--- a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc
+++ b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc
@@ -268,6 +268,8 @@ class DeviceResDistTest : public ::testing::Test {
         EXPECT_EQ(cp_[idx].instance.device_names[idx], device_name);
         EXPECT_EQ(cp_[idx].instance.task_names[idx], task_name);
         if (idx > 0) {
+          EXPECT_EQ(cp_[0].instance.communicator_key,
+                    cp_[idx].instance.communicator_key);
           for (int i = 0; i < dev_count; ++i) {
             EXPECT_EQ(cp_[0].instance.device_names[i],
                       cp_[idx].instance.device_names[i]);
diff --git a/tensorflow/core/framework/collective.cc b/tensorflow/core/framework/collective.cc
index 7fa58347f2..b83d183f14 100644
--- a/tensorflow/core/framework/collective.cc
+++ b/tensorflow/core/framework/collective.cc
@@ -17,6 +17,7 @@ limitations under the License.
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/hash/hash.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 
 namespace tensorflow {
@@ -64,7 +65,9 @@ CollInstanceParams& CollInstanceParams::operator=(
     device_names.assign(other.device_names.begin(), other.device_names.end());
     task_names.assign(other.task_names.begin(), other.task_names.end());
     same_num_devices_per_task = other.same_num_devices_per_task;
+    num_devices_per_task = other.num_devices_per_task;
     gpu_ring_order = other.gpu_ring_order;
+    communicator_key = other.communicator_key;
     impl_details.subdiv_offsets.assign(
         other.impl_details.subdiv_offsets.begin(),
         other.impl_details.subdiv_offsets.end());
@@ -76,6 +79,7 @@ CollInstanceParams& CollInstanceParams::operator=(
     impl_details.subdiv_source_rank.assign(
         other.impl_details.subdiv_source_rank.begin(),
         other.impl_details.subdiv_source_rank.end());
+    impl_details.dependencies = other.impl_details.dependencies;
   }
   return *this;
 }
@@ -91,6 +95,13 @@ string CollInstanceParams::ToString() const {
   for (const auto& n : task_names) {
     strings::StrAppend(&v, n, ", ");
   }
+  strings::StrAppend(&v, "} num_devices_per_task={");
+  for (const auto dpt : num_devices_per_task) {
+    strings::StrAppend(&v, dpt.first, ": ", dpt.second, ", ");
+  }
+  strings::StrAppend(&v, "}, collective_name=", impl_details.collective_name,
+                     ", communicator_key=", str_util::CEscape(communicator_key),
+                     ", subdiv_offsets={");
   strings::StrAppend(&v, "}, subdiv_offsets={");
   for (const auto& d : impl_details.subdiv_offsets) {
     strings::StrAppend(&v, d, ",");
diff --git a/tensorflow/core/framework/collective.h b/tensorflow/core/framework/collective.h
index 0321429702..546e3938a8 100644
--- a/tensorflow/core/framework/collective.h
+++ b/tensorflow/core/framework/collective.h
@@ -70,6 +70,8 @@ struct CollImplDetails {
   std::vector<std::vector<int>> subdiv_permutations;
   std::vector<int> subdiv_offsets;
   std::vector<int> subdiv_source_rank;  // rank of source in each subdiv
+  std::vector<int32>
+      dependencies;  // collective instances on which this node depends
 };
 
 // Data common to all members of a collective instance.
@@ -85,9 +87,13 @@ struct CollInstanceParams {
   std::vector<string> task_names;
   // True if every task has the same number of devices.
   bool same_num_devices_per_task = false;
+  // Task -> number of devices on that task.
+  std::unordered_map<string, int32> num_devices_per_task;
   // If passed in to GPUOptions in ConfigProto, defines a good ring order for
   // GPUs.  Assumes same GPU configuration at each worker.
   string gpu_ring_order = "";
+  // Valid when using a communicator-based collective mechanism, e.g. NCCL.
+  string communicator_key;
   CollImplDetails impl_details;
   string ToString() const;
   CollInstanceParams& operator=(const struct CollInstanceParams& other);
@@ -269,6 +275,21 @@ class CollectiveExecutor : public PeerAccessInterface, public core::RefCounted {
 
   virtual PerStepCollectiveRemoteAccess* remote_access() { return nullptr; }
 
+  // `WaitForDependencies` and `Launched` are used for fine-grained control of
+  // execution order between collective instances.  These functions are intended
+  // to be called in `Run` function of collective implementations, and may be
+  // used to make part, or whole, of the collective execution ordered with
+  // respect to other collective instances.
+  //
+  // `WaitForDependencies` will block until it is safe to continue the callee's
+  // execution, where safety is defined as: ordered with respect to the
+  // collective instances defined in the callee's `wait_for` attribute.
+  virtual void WaitForDependencies(const CollectiveParams& col_params) {}
+  // `Launched` unblocks the dependent collective instances by recording that
+  // this callee device has completed the critical portion of the collective
+  // execution.
+  virtual void Launched(const CollectiveParams& col_params) {}
+
   // Used to designate an invalid group or instance key.
   static int64 kInvalidId;
 
@@ -347,7 +368,8 @@ class CollectiveImplementationInterface {
 
   // Initializes the portions of `col_params` specific to this
   // implementation.  Called exactly once for every Collective instance during
-  // the CollectiveParams resolution process when the graph is first executed.
+  // the CollectiveParams resolution process when the graph is first executed,
+  // at the end of `CompleteInstanceLocal()`.
   // NOTE(ayushd): This is effectively a static function because it modifies the
   // `col_params` passed in and should not manipulate any data members.  However
   // because it is virtual and needs to be implemented by every derived class we
@@ -360,6 +382,14 @@ class CollectiveImplementationInterface {
   // object.
   virtual Status InitializeCollectiveContext(CollectiveContext* col_ctx) = 0;
 
+  // Initializes instance params at the beginning of `CompleteInstanceLocal()`,
+  // unlike `InitializeCollectiveParams` which is called at the end.  This
+  // function is called before all devices in the instance are discovered, and
+  // may be used to broadcast data via the shared `InstanceRec` object in
+  // collective param resolution to all devices.
+  virtual Status InitializeInstanceBeforeGroupDiscovery(
+      CollectiveParams* col_params) = 0;
+
   // Processes and moves data according to the logic of this Collective
   // implementation.  Relies on appropriate initialization of op-specific
   // CollectiveParams in InitializeCollectiveParams(), as well as appropriate
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 96536e6945..c26c9af53f 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -60,6 +60,7 @@ load(
     "mkl_deps",
 )
 load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda")
+load("//tensorflow:tensorflow.bzl", "if_nccl")
 
 config_setting(
     # Add "--define tensorflow_xsmm=1" to your build command to use libxsmm for
@@ -180,12 +181,36 @@ tf_cc_test(
 
 tf_kernel_library(
     name = "collective_ops",
+    srcs = if_nccl([
+        "collective_nccl_reducer.h",
+        "collective_nccl_reducer.cc",
+    ]),
     prefix = "collective_ops",
     deps = [
         "//tensorflow/core:collective_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
+    ] + if_nccl([
+        "@local_config_nccl//:nccl",
+        "//tensorflow/core/nccl:nccl_lib",
+    ]),
+)
+
+tf_cuda_cc_test(
+    name = "collective_nccl_reducer_test",
+    size = "small",
+    srcs = ["collective_nccl_reducer_test.cc"],
+    tags = tf_cuda_tests_tags() + ["no_cuda_on_cpu_tap"],
+    deps = [
+        "//tensorflow/core:all_kernels",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
     ],
 )
 
diff --git a/tensorflow/core/kernels/collective_nccl_reducer.cc b/tensorflow/core/kernels/collective_nccl_reducer.cc
new file mode 100644
index 0000000000..113f1487f0
--- /dev/null
+++ b/tensorflow/core/kernels/collective_nccl_reducer.cc
@@ -0,0 +1,204 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/kernels/collective_nccl_reducer.h"
+
+#ifdef GOOGLE_CUDA
+
+#include "tensorflow/core/common_runtime/collective_util.h"
+#include "tensorflow/core/nccl/nccl_manager.h"
+
+namespace tensorflow {
+namespace {
+string NcclCollectiveKey(const string& exec_key, int step_id) {
+  return strings::StrCat(exec_key, ":", step_id);
+}
+}  // namespace
+
+NcclReducer::NcclReducer() : col_ctx_(nullptr), col_params_(nullptr) {}
+
+Status NcclReducer::InitializeCollectiveParams(CollectiveParams* col_params) {
+  if (col_params->instance.type != REDUCTION_COLLECTIVE ||
+      col_params->instance.impl_details.collective_name != "NcclReduce") {
+    return errors::Internal("Unexpected collective type ",
+                            col_params->instance.type, " expected ",
+                            REDUCTION_COLLECTIVE, "; or collective name ",
+                            col_params->instance.impl_details.collective_name,
+                            " expected NcclReduce");
+  } else {
+    return Status::OK();
+  }
+}
+
+Status NcclReducer::InitializeCollectiveContext(CollectiveContext* col_ctx) {
+  col_ctx_ = col_ctx;
+  col_params_ = &col_ctx->col_params;
+  return collective_util::InitializeDeviceAndLocality(
+      col_ctx->dev_mgr, col_ctx->device_name, &col_ctx->device,
+      &col_ctx->device_locality);
+}
+
+Status NcclReducer::InitializeInstanceBeforeGroupDiscovery(
+    CollectiveParams* col_params) {
+  if (col_params->default_rank == 0 && col_params->group.num_tasks > 1) {
+    col_params->instance.communicator_key =
+        NcclManager::instance()->GenerateCommunicatorKey();
+  }
+  return Status::OK();
+}
+
+Status ReductionOp(const string& merge_op, ncclRedOp_t* reduction_op) {
+  if (merge_op == "Add") {
+    *reduction_op = ncclSum;
+    return Status::OK();
+  } else if (merge_op == "Mul") {
+    *reduction_op = ncclProd;
+    return Status::OK();
+  } else {
+    return errors::Internal("Expected merge_op to be either Add or Mul, found ",
+                            merge_op);
+  }
+}
+
+void NcclReducer::Run(StatusCallback done) {
+  ncclRedOp_t reduction_op;
+  Status s = ReductionOp(col_params_->merge_op->type_string(), &reduction_op);
+  if (!s.ok()) {
+    done(s);
+    return;
+  }
+
+  Tensor group_size;
+  Notification group_size_ready;
+  Status group_size_status;
+  if (col_params_->final_op) {
+    // Create an on-device scalar value from group_size_.
+    // TODO(ayushd, tucker): avoid this copy by either reusing across
+    // invocations or providing the scalar to the kernel in host memory.
+    Tensor group_size_val(col_ctx_->output->dtype(), TensorShape({}));
+    switch (col_ctx_->output->dtype()) {
+      case DT_FLOAT:
+        group_size_val.scalar<float>()() = col_params_->group.group_size;
+        break;
+      case DT_DOUBLE:
+        group_size_val.scalar<double>()() = col_params_->group.group_size;
+        break;
+      case DT_INT32:
+        group_size_val.scalar<int32>()() = col_params_->group.group_size;
+        break;
+      case DT_INT64:
+        group_size_val.scalar<int64>()() = col_params_->group.group_size;
+        break;
+      default:
+        done(errors::Internal("Unsupported type ", col_ctx_->output->dtype()));
+        return;
+    }
+    group_size = Tensor(
+        col_ctx_->device->GetAllocator(col_ctx_->op_ctx->input_alloc_attr(0)),
+        col_ctx_->output->dtype(), TensorShape({}));
+    DeviceContext* op_dev_ctx = col_ctx_->op_ctx->op_device_context();
+    // Enqueue copy on gpu stream.
+    op_dev_ctx->CopyCPUTensorToDevice(
+        &group_size_val, col_ctx_->device, &group_size,
+        [&group_size_ready, &group_size_status](const Status& s) {
+          group_size_status = s;
+          group_size_ready.Notify();
+        });
+  } else {
+    group_size_ready.Notify();
+  }
+
+  Notification nccl_done;
+  Status nccl_status;
+  auto* compute_stream = col_ctx_->op_ctx->op_device_context()->stream();
+  auto* gpu_info = col_ctx_->op_ctx->device()->tensorflow_gpu_device_info();
+  // `AddToAllReduce` performs consistency checks for the NCCL call and enqueues
+  // the `Participant` struct locally.  When all local participants with this
+  // `nccl_collective_key` have called `AddToAllReduce` and
+  // `SignalMultiNodeReady`, all devices at this worker are ready to process
+  // this NCCL op.
+  //
+  // The `NcclManager` uses a dedicated CUDA stream for NCCL kernels.  At this
+  // point, it synchronizes the NCCL stream with the compute stream, and then
+  // enqueues the NCCL kernel on the NCCL stream.
+  const int num_global_devices = col_params_->group.group_size;
+  const int num_local_devices = col_params_->instance.num_devices_per_task.at(
+      col_params_->instance.task_names[col_params_->default_rank]);
+  const string nccl_collective_key =
+      NcclCollectiveKey(col_ctx_->exec_key, col_ctx_->step_id);
+  auto done_callback = [&nccl_done, &nccl_status](const Status& s) {
+    nccl_status = s;
+    nccl_done.Notify();
+  };
+  auto participant = absl::make_unique<NcclManager::Participant>(
+      compute_stream->parent(), compute_stream, gpu_info->event_mgr,
+      gpu_info->gpu_id, col_ctx_->input, col_ctx_->output,
+      col_params_->default_rank, std::move(done_callback));
+  VLOG(1) << "NcclReducer calling NcclManager::AddToAllReduce num_tasks "
+          << col_params_->group.num_tasks << " current task "
+          << col_params_->instance.task_names[col_params_->default_rank]
+          << " num local devices " << num_local_devices
+          << " num global devices " << num_global_devices;
+  NcclManager::instance()->AddToAllReduce(
+      std::move(participant),
+      {nccl_collective_key, num_local_devices, num_global_devices,
+       col_params_->instance.communicator_key},
+      reduction_op);
+
+  // NOTE(ayushd): We need to synchronize NCCL launches across nodes to prevent
+  // deadlocks.  In the current implementation, we define a deterministic
+  // sequential launch order between potentially concurrent collective instances
+  // by introducing control information during static graph analysis in
+  // graph/collective_order.cc.  This can be either in the form of explicit
+  // control edges or via `wait_for` attribute on the collective op.
+  //
+  // The other end of the design spectrum would have a distinguished node
+  // dynamically signal the next collective to launch to all other participants.
+  // This has higher degree of runtime coordination, but it may be able to
+  // achieve better performance if the (arbitrary) static execution order
+  // assigned in the first approach turns out to not be good from a scheduling
+  // perspective.  e.g. consider a graph in which c1, c2, and c3 are three
+  // concurrent collective instances, and the static ordering assigns c1 -> c2
+  // -> c3.  In practice, it could turn out that c3 is always ready to execute
+  // before c1 or c2.
+  //
+  // `WaitForDependencies` may block if the collective instances on which this
+  // op depends have not yet launched.  When this function returns, this op is
+  // ready to go.
+  col_ctx_->col_exec->WaitForDependencies(*col_params_);
+  NcclManager::instance()->SignalMultiNodeReady(nccl_collective_key);
+  // When all devices at this worker have called `SignalMultiNodeReady`, the
+  // `NcclManager` will enqueue the NCCL kernel on the NCCL stream.  Thus the
+  // implementation of `Launched` keeps track of the number of devices that have
+  // launched.
+  col_ctx_->col_exec->Launched(*col_params_);
+
+  // Wait for nccl op and group_size copy to succeed, then do final_op.
+  group_size_ready.WaitForNotification();
+  nccl_done.WaitForNotification();
+  Status final_status =
+      group_size_status.ok() ? nccl_status : group_size_status;
+  if (final_status.ok() && col_params_->final_op) {
+    final_status = collective_util::ComputeBinOp(
+        col_ctx_->op_ctx, col_ctx_->op_params, col_ctx_->device,
+        col_params_->final_op.get(), col_ctx_->output, &group_size);
+  }
+  done(final_status);
+}
+
+REGISTER_COLLECTIVE(NcclReduce, NcclReducer);
+
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/collective_nccl_reducer.h b/tensorflow/core/kernels/collective_nccl_reducer.h
new file mode 100644
index 0000000000..dc70b280c5
--- /dev/null
+++ b/tensorflow/core/kernels/collective_nccl_reducer.h
@@ -0,0 +1,49 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_KERNELS_COLLECTIVE_NCCL_REDUCER_H_
+#define TENSORFLOW_CORE_KERNELS_COLLECTIVE_NCCL_REDUCER_H_
+
+#include "tensorflow/core/framework/collective.h"
+
+namespace tensorflow {
+#ifdef GOOGLE_CUDA
+
+class NcclReducer : public CollectiveImplementationInterface {
+ public:
+  NcclReducer();
+  ~NcclReducer() override = default;
+
+  // No-op for this collective implementation.
+  Status InitializeCollectiveParams(CollectiveParams* col_params) override;
+
+  // Initializes the device objects and device localities.
+  Status InitializeCollectiveContext(CollectiveContext* col_ctx) override;
+
+  // Initialize nccl communicator key.
+  Status InitializeInstanceBeforeGroupDiscovery(
+      CollectiveParams* col_params) override;
+
+  // Hands off all reduce to NcclManager.
+  void Run(StatusCallback done) override;
+
+ private:
+  CollectiveContext* col_ctx_;          // Not owned
+  const CollectiveParams* col_params_;  // Not owned
+};
+
+#endif  // GOOGLE_CUDA
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_KERNELS_COLLECTIVE_NCCL_REDUCER_H_
diff --git a/tensorflow/core/kernels/collective_nccl_reducer_test.cc b/tensorflow/core/kernels/collective_nccl_reducer_test.cc
new file mode 100644
index 0000000000..ff33c1322c
--- /dev/null
+++ b/tensorflow/core/kernels/collective_nccl_reducer_test.cc
@@ -0,0 +1,331 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/kernels/collective_nccl_reducer.h"
+
+#include <algorithm>
+#include "absl/memory/memory.h"
+#include "tensorflow/core/common_runtime/base_collective_executor.h"
+#include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/common_runtime/device_factory.h"
+#include "tensorflow/core/common_runtime/device_mgr.h"
+#include "tensorflow/core/common_runtime/device_resolver_local.h"
+#include "tensorflow/core/common_runtime/process_util.h"
+#include "tensorflow/core/common_runtime/test_collective_executor_mgr.h"
+#include "tensorflow/core/framework/collective.h"
+#include "tensorflow/core/framework/fake_input.h"
+#include "tensorflow/core/framework/node_def_builder.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/lib/core/notification.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/public/session_options.h"
+#include "tensorflow/core/public/version.h"
+
+#ifdef GOOGLE_CUDA
+
+namespace tensorflow {
+static constexpr int kStepId = 10;
+
+std::unique_ptr<OpKernel> GetKernel(const NodeDef& node, DeviceBase* device) {
+  Status status;
+  std::unique_ptr<OpKernel> k = CreateOpKernel(
+      DEVICE_GPU, device, device->GetAllocator(AllocatorAttributes()), node,
+      TF_GRAPH_DEF_VERSION, &status);
+  if (!status.ok()) LOG(FATAL) << status;
+  return k;
+}
+
+std::unique_ptr<OpKernel> GetAdd(DeviceBase* device) {
+  NodeDef node_def;
+  NodeDefBuilder builder("add_node", "Add");
+  TF_CHECK_OK(builder.Attr("T", DT_FLOAT)
+                  .Input(FakeInput(DT_FLOAT))
+                  .Input(FakeInput(DT_FLOAT))
+                  .Finalize(&node_def));
+  return GetKernel(node_def, device);
+}
+
+std::unique_ptr<OpKernel> GetDiv(DeviceBase* device) {
+  NodeDef node_def;
+  NodeDefBuilder builder("add_node", "Div");
+  TF_CHECK_OK(builder.Attr("T", DT_FLOAT)
+                  .Input(FakeInput(DT_FLOAT))
+                  .Input(FakeInput(DT_FLOAT))
+                  .Finalize(&node_def));
+  return GetKernel(node_def, device);
+}
+
+class NcclReducerTest : public ::testing::Test {
+ protected:
+  ~NcclReducerTest() override {
+    if (col_exec_) col_exec_->Unref();
+  }
+
+  void InitGPUDevices() {
+    std::vector<std::unique_ptr<Device>> all_devices;
+    SessionOptions session_options;
+    session_options.config.mutable_gpu_options()
+        ->set_per_process_gpu_memory_fraction(0.1);
+    session_options.env = Env::Default();
+    Status s = DeviceFactory::GetFactory(DEVICE_GPU)
+                   ->AddDevices(session_options, "", &all_devices);
+    TF_CHECK_OK(s);
+    for (std::unique_ptr<Device>& d : all_devices) {
+      if (d->device_type() == "GPU") {
+        gpus_.emplace_back(std::move(d));
+      }
+    }
+  }
+
+  void Init(int num_ranks) {
+    setenv("NCCL_DEBUG", "INFO", 1 /* replace */);
+    setenv("NCCL_LAUNCH_MODE", "PARALLEL", 1 /* replace */);
+    InitGPUDevices();
+    std::vector<std::unique_ptr<Device>> local_devices;
+    std::vector<string> device_names;
+    for (int rank = 0; rank < num_ranks; ++rank) {
+      if (rank < gpus_.size()) {
+        local_devices.emplace_back(std::move(gpus_[rank]));
+      }
+    }
+    int num_gpus = local_devices.size();
+    for (const auto& device : local_devices) {
+      device_names.push_back(device->name());
+      VLOG(2) << device->name();
+    }
+    if (!dev_mgr_) dev_mgr_.reset(new DeviceMgr(std::move(local_devices)));
+    col_exec_ = new BaseCollectiveExecutor(
+        &col_exec_mgr_, /*remote_access=*/nullptr, kStepId, dev_mgr_.get(),
+        /*gpu_ring_order=*/nullptr);
+
+    // Initialize collective params.
+    col_params_.name = "test_nccl_collective_op";
+    const int group_key = 5;
+    col_params_.group.group_key = group_key;
+    col_params_.group.device_type = DEVICE_GPU;
+    col_params_.group.group_size = num_ranks;
+    const int instance_key = 23;
+    col_params_.instance.instance_key = instance_key;
+    col_params_.instance.type = REDUCTION_COLLECTIVE;
+    col_params_.instance.data_type = DT_FLOAT;
+    col_params_.instance.impl_details.collective_name = "NcclReduce";
+    const string task_name = "/job:worker/replica:0/task:0";
+    col_params_.instance.num_devices_per_task[task_name] = num_ranks;
+    for (int rank = 0; rank < num_ranks; ++rank) {
+      col_params_.instance.device_names.push_back(
+          device_names[rank % num_gpus]);
+      col_params_.instance.task_names.push_back(task_name);
+    }
+    for (int rank = 0; rank < num_ranks; ++rank) {
+      instances_.push_back(absl::make_unique<DeviceInstance>(
+          rank, col_params_.instance.device_names[rank], this));
+    }
+  }
+
+  void Reduce() {
+    int done = 0;
+    mutex done_mu;
+    condition_variable done_cv;
+    for (const auto& instance : instances_) {
+      DeviceInstance* di = instance.get();
+      SchedClosure([di, &done, &done_mu, &done_cv] {
+        di->DoReduce();
+        mutex_lock l(done_mu);
+        ++done;
+        done_cv.notify_all();
+      });
+    }
+
+    mutex_lock l(done_mu);
+    while (done < instances_.size()) done_cv.wait(l);
+  }
+
+  void RunTest(int num_ranks, int tensor_length) {
+    Init(num_ranks);
+    std::vector<float> expected(tensor_length, 0.0);
+    for (int rank = 0; rank < num_ranks; ++rank) {
+      DeviceInstance* instance = instances_[rank].get();
+      instance->InitTensor(DT_FLOAT, TensorShape({tensor_length}),
+                           [&expected, rank](Tensor* t) {
+                             for (size_t i = 0; i < t->NumElements(); ++i) {
+                               float value = pow(10, rank) * i;
+                               t->flat<float>()(i) = value;
+                               expected[i] += value;
+                             }
+                           });
+    }
+    Reduce();
+    // Confirm that every rank computed the same correct value.
+    for (int i = 0; i < tensor_length; ++i) {
+      expected[i] /= num_ranks;
+    }
+    for (int rank = 0; rank < instances_.size(); ++rank) {
+      TF_ASSERT_OK(instances_[rank]->status_);
+      Tensor* dev_tensor = &instances_[rank]->tensor_;
+      Tensor actual(DT_FLOAT, TensorShape({tensor_length}));
+      Notification note;
+      Device* dev = instances_[rank]->device_;
+      auto* dev_info = dev->tensorflow_gpu_device_info();
+      dev_info->default_context->CopyDeviceTensorToCPU(
+          dev_tensor, /*tensor_name=*/"", dev, &actual,
+          [&note](const Status&) { note.Notify(); });
+      note.WaitForNotification();
+      for (int i = 0; i < tensor_length; ++i) {
+        EXPECT_FLOAT_EQ(expected[i], actual.template flat<float>()(i))
+            << "Mismatch at rank " << rank << " index " << i;
+      }
+    }
+  }
+
+  std::unique_ptr<OpKernel> GetCollectiveReduce(const CollectiveParams& params,
+                                                Tensor* input,
+                                                DeviceBase* device) {
+    mutex_lock l(mu_);
+    NodeDef node_def;
+    NodeDefBuilder builder(
+        strings::StrCat("collective_reduce_", reduce_counter_++),
+        "CollectiveReduce");
+    TF_CHECK_OK(
+        builder.Attr("T", params.instance.data_type)
+            .Attr("merge_op", "Add")
+            .Attr("final_op", "Div")
+            .Attr("group_size", params.group.group_size)
+            .Attr("group_key", params.group.group_key)
+            .Attr("instance_key", params.instance.instance_key)
+            .Attr("subdiv_offsets", params.instance.impl_details.subdiv_offsets)
+            .Input(FakeInput(params.instance.data_type))
+            .Finalize(&node_def));
+    return GetKernel(node_def, device);
+  }
+
+  class DeviceInstance {
+   public:
+    DeviceInstance(int rank, const string& device_name, NcclReducerTest* parent)
+        : parent_(parent), device_name_(device_name), rank_(rank) {
+      TF_CHECK_OK(parent_->dev_mgr_->LookupDevice(device_name_, &device_))
+          << "Could not find device " << device_name_ << " existing devices "
+          << parent_->dev_mgr_->DebugString();
+      col_params_.name = parent_->col_params_.name;
+      col_params_.default_rank = rank;
+      col_params_.group.group_key = parent_->col_params_.group.group_key;
+      col_params_.group.device_type = parent_->col_params_.group.device_type;
+      col_params_.group.group_size = parent_->col_params_.group.group_size;
+      col_params_.instance = parent->col_params_.instance;
+    }
+
+    void InitTensor(DataType dtype, const TensorShape& shape,
+                    const std::function<void(Tensor*)>& init_f) {
+      tensor_ =
+          Tensor(device_->GetAllocator(AllocatorAttributes()), dtype, shape);
+      Tensor cpu_tensor(dtype, shape);
+      init_f(&cpu_tensor);
+      VLOG(2) << "cpu_tensor " << cpu_tensor.DebugString();
+      auto* dev_info = device_->tensorflow_gpu_device_info();
+      Notification note;
+      dev_info->default_context->CopyCPUTensorToDevice(
+          &cpu_tensor, device_, &tensor_,
+          [&note](const Status&) { note.Notify(); });
+      note.WaitForNotification();
+    }
+
+    void DoReduce() {
+      col_params_.merge_op = GetAdd(device_);
+      col_params_.final_op = GetDiv(device_);
+
+      // Prepare an OpKernelContext.
+      OpKernelContext::Params op_params;
+      op_params.step_id = kStepId;
+      op_params.device = device_;
+      gtl::InlinedVector<TensorValue, 4> inputs;
+      inputs.push_back(TensorValue(&tensor_));
+      op_params.inputs = &inputs;
+      gtl::InlinedVector<AllocatorAttributes, 4> input_aa(
+          {AllocatorAttributes()});
+      op_params.input_alloc_attrs = &input_aa;
+      gtl::InlinedVector<DeviceContext*, 4> input_dc;
+      DeviceContext* dev_ctx = nullptr;
+      auto* dev_info = device_->tensorflow_gpu_device_info();
+      if (dev_info) {
+        dev_ctx = dev_info->default_context;
+        dev_ctx->Ref();
+      } else {
+        dev_ctx = new DeviceContext;
+      }
+      input_dc.push_back(dev_ctx);
+      op_params.input_device_contexts = &input_dc;
+      op_params.op_device_context = dev_ctx;
+      int forward_from = 0;
+      op_params.forward_from_array = &forward_from;
+      AllocatorAttributes generic_alloc_attr;
+      op_params.output_attr_array = &generic_alloc_attr;
+      std::unique_ptr<OpKernel> op =
+          parent_->GetCollectiveReduce(col_params_, &tensor_, device_);
+      op_params.op_kernel = op.get();
+      OpKernelContext ctx(&op_params, 1);
+
+      // We never actually execute the kernel, so we need to do the output
+      // allocation it would do, ourselves.
+      Tensor* output_tensor_ptr = nullptr;
+      TF_CHECK_OK(ctx.forward_input_or_allocate_output({0}, 0, tensor_.shape(),
+                                                       &output_tensor_ptr));
+      CHECK_EQ(output_tensor_ptr, ctx.mutable_output(0));
+
+      // Prepare a NcclReducer instance.
+      string exec_key =
+          strings::StrCat(col_params_.instance.instance_key, ":0:0");
+      NcclReducer reducer;
+      CollectiveContext col_ctx(parent_->col_exec_, parent_->dev_mgr_.get(),
+                                &ctx, &op_params, col_params_, exec_key,
+                                kStepId, &tensor_, &tensor_);
+      TF_CHECK_OK(reducer.InitializeCollectiveContext(&col_ctx));
+
+      // Run the all-reduce.
+      reducer.Run([this](Status s) { status_ = s; });
+      if (status_.ok()) {
+        CHECK(tensor_.CopyFrom(*ctx.mutable_output(0), tensor_.shape()));
+      }
+
+      dev_ctx->Unref();
+    }
+
+    NcclReducerTest* parent_;
+    string device_name_;
+    int rank_;
+    Tensor tensor_;
+    Device* device_;
+    CollectiveParams col_params_;
+    Status status_;
+  };
+
+  std::vector<std::unique_ptr<tensorflow::Device>> gpus_;
+  TestCollectiveExecutorMgr col_exec_mgr_;
+  CollectiveExecutor* col_exec_;
+  std::unique_ptr<DeviceMgr> dev_mgr_;
+  std::vector<std::unique_ptr<DeviceInstance>> instances_;
+  CollectiveParams col_params_;
+  mutex mu_;
+  int32 reduce_counter_ GUARDED_BY(mu_) = 0;
+};
+
+TEST_F(NcclReducerTest, Test2Dev16Len) { RunTest(2, 16); }
+TEST_F(NcclReducerTest, Test4Dev16Len) { RunTest(4, 16); }
+TEST_F(NcclReducerTest, Test8Dev16Len) { RunTest(8, 16); }
+TEST_F(NcclReducerTest, Test8Dev128Len) { RunTest(8, 128); }
+TEST_F(NcclReducerTest, Test8Dev1045991Len) { RunTest(8, 1048576); }
+
+}  // namespace tensorflow
+
+#endif
diff --git a/tensorflow/core/kernels/collective_ops.cc b/tensorflow/core/kernels/collective_ops.cc
index 82e2913b64..04e37e8edc 100644
--- a/tensorflow/core/kernels/collective_ops.cc
+++ b/tensorflow/core/kernels/collective_ops.cc
@@ -43,16 +43,17 @@ class CollectiveOpKernel : public AsyncOpKernel {
       // Call in a blockable thread because it's not guaranteed that
       // this call cannot block.
       c->env()->SchedClosure([this, c, done, col_exec]() {
-        col_exec->CompleteParamsAsync(c->device()->name(), &col_params_,
-                                      c->cancellation_manager(),
-                                      [this, c, done](const Status& s) {
-                                        if (s.ok()) {
-                                          ComputeAsync(c, done);
-                                        } else {
-                                          c->SetStatus(s);
-                                          done();
-                                        }
-                                      });
+        col_exec->CompleteParamsAsync(
+            c->device()->name(), &col_params_, c->cancellation_manager(),
+            [this, c, done](const Status& s) {
+              if (s.ok()) {
+                col_params_.instance.impl_details.dependencies = dependencies_;
+                ComputeAsync(c, done);
+              } else {
+                c->SetStatus(s);
+                done();
+              }
+            });
       });
       return false;
     }
@@ -60,6 +61,7 @@ class CollectiveOpKernel : public AsyncOpKernel {
   }
 
   CollectiveParams col_params_;
+  std::vector<int32> dependencies_;
 };
 
 class CollectiveReduceOpKernel : public CollectiveOpKernel {
@@ -87,6 +89,7 @@ class CollectiveReduceOpKernel : public CollectiveOpKernel {
                     "final_op must be one of {\"Id\", \"Div\"} but got ",
                     final_op_name));
     OP_REQUIRES_OK(c, c->GetAttr("T", &col_params_.instance.data_type));
+    OP_REQUIRES_OK(c, c->GetAttr("wait_for", &dependencies_));
 
     const NodeDef& real_node = c->def();
     col_params_.name = strings::StrCat(real_node.name(), ": Reduce(",
diff --git a/tensorflow/core/nccl/BUILD b/tensorflow/core/nccl/BUILD
index 4be33b2a0c..a19e1af888 100644
--- a/tensorflow/core/nccl/BUILD
+++ b/tensorflow/core/nccl/BUILD
@@ -20,9 +20,11 @@ cc_library(
     name = "nccl_lib",
     srcs = if_cuda([
         "nccl_manager.cc",
-        "nccl_manager.h",
         "nccl_rewrite.cc",
     ]),
+    hdrs = if_cuda([
+        "nccl_manager.h",
+    ]),
     copts = tf_copts(),
     deps = if_cuda([
         "@local_config_nccl//:nccl",
diff --git a/tensorflow/core/protobuf/worker.proto b/tensorflow/core/protobuf/worker.proto
index 74058c8465..4284dd119e 100644
--- a/tensorflow/core/protobuf/worker.proto
+++ b/tensorflow/core/protobuf/worker.proto
@@ -535,6 +535,7 @@ message CompleteInstanceRequest {
 message CompleteInstanceResponse {
   int32 instance_key = 1;
   int32 source_rank = 2;
+  bytes communicator_key = 3;
 }
 
 // Request for next agreed-upon step_id for the specified graph_keys.
-- 
GitLab


From 8eafc6c7941e8488db6452af68851a6f531528d9 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Fri, 18 Jan 2019 08:43:06 -0800
Subject: [PATCH 0957/2345] Prevent out-of-bounds memory access in decoding gif
 images.

PiperOrigin-RevId: 229934641
---
 tensorflow/core/lib/gif/gif_io.cc | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tensorflow/core/lib/gif/gif_io.cc b/tensorflow/core/lib/gif/gif_io.cc
index 24aca854eb..ce842e9723 100644
--- a/tensorflow/core/lib/gif/gif_io.cc
+++ b/tensorflow/core/lib/gif/gif_io.cc
@@ -147,6 +147,14 @@ uint8* Decode(const void* srcdata, int datasize,
         GifByteType color_index =
             this_image->RasterBits[(i - img_desc->Top) * (img_desc->Width) +
                                    (j - img_desc->Left)];
+
+        if (color_index >= color_map->ColorCount) {
+          *error_string = strings::StrCat("found color index ", color_index,
+                                          " outside of color map range ",
+                                          color_map->ColorCount);
+          return nullptr;
+        }
+
         const GifColorType& gif_color = color_map->Colors[color_index];
         p_dst[j * channel + 0] = gif_color.Red;
         p_dst[j * channel + 1] = gif_color.Green;
-- 
GitLab


From 44edad5a82d96ceaffa1d67aea3758037805c25f Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Fri, 18 Jan 2019 17:19:55 +0000
Subject: [PATCH 0958/2345] Fix failing test on
 testRaggedTensorGetItemErrorsWithRaggedRank1

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/ops/ragged/ragged_getitem.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/ragged/ragged_getitem.py b/tensorflow/python/ops/ragged/ragged_getitem.py
index a7124bf8e5..212252c661 100644
--- a/tensorflow/python/ops/ragged/ragged_getitem.py
+++ b/tensorflow/python/ops/ragged/ragged_getitem.py
@@ -168,9 +168,9 @@ def _ragged_getitem(rt_input, key_list):
       # TypeError which happens when row_key is not an integer, the exception
       # will simply be ignored as it will be processed later anyway.
       try:
-        if row_key >= len(starts):
+        if int(row_key) >= len(starts):
           raise IndexError('Row key {} out of bounds'.format(row_key))
-      except TypeError:
+      except (TypeError, ValueError):
         pass
     row = rt_input.values[starts[row_key]:limits[row_key]]
     return row.__getitem__(inner_keys)
-- 
GitLab


From 06444e2c956f1aa6265bf467ebb8b3072a280b7f Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Fri, 18 Jan 2019 09:36:09 -0800
Subject: [PATCH 0959/2345] move kDelimiter to an anonymous namespace

---
 tensorflow/core/kernels/data/dataset_utils.cc | 17 +++++++++++++++++
 tensorflow/core/kernels/data/dataset_utils.h  | 12 +-----------
 2 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/tensorflow/core/kernels/data/dataset_utils.cc b/tensorflow/core/kernels/data/dataset_utils.cc
index 8f3b1f7066..dd47ed8e8d 100644
--- a/tensorflow/core/kernels/data/dataset_utils.cc
+++ b/tensorflow/core/kernels/data/dataset_utils.cc
@@ -141,6 +141,23 @@ Status VerifyShapesCompatible(const std::vector<PartialTensorShape>& expected,
   return Status::OK();
 }
 
+namespace {
+
+constexpr char kDelimiter[] = "@@";
+
+}  // namespace
+
+VariantTensorDataReader::VariantTensorDataReader(
+    const tensorflow::VariantTensorData* data)
+    : data_(data) {
+  string metadata;
+  data_->get_metadata(&metadata);
+  auto keys = str_util::Split(metadata, kDelimiter, str_util::SkipEmpty());
+  for (size_t i = 0; i < keys.size(); ++i) {
+    map_[keys[i]] = i;
+  }
+}
+
 Status VariantTensorDataReader::status() const { return status_; }
 
 Status VariantTensorDataReader::ReadScalar(StringPiece key, int64* val) {
diff --git a/tensorflow/core/kernels/data/dataset_utils.h b/tensorflow/core/kernels/data/dataset_utils.h
index b960671594..eae2bc4242 100644
--- a/tensorflow/core/kernels/data/dataset_utils.h
+++ b/tensorflow/core/kernels/data/dataset_utils.h
@@ -57,20 +57,10 @@ Status VerifyTypesMatch(const DataTypeVector& expected,
 Status VerifyShapesCompatible(const std::vector<PartialTensorShape>& expected,
                               const std::vector<PartialTensorShape>& received);
 
-constexpr char kDelimiter[] = "@@";
-
 // Helper class for reading data from a VariantTensorData object.
 class VariantTensorDataReader : public IteratorStateReader {
  public:
-  explicit VariantTensorDataReader(const VariantTensorData* data)
-      : data_(data) {
-    string metadata;
-    data_->get_metadata(&metadata);
-    auto keys = str_util::Split(metadata, kDelimiter, str_util::SkipEmpty());
-    for (size_t i = 0; i < keys.size(); ++i) {
-      map_[keys[i]] = i;
-    }
-  }
+  explicit VariantTensorDataReader(const VariantTensorData* data);
 
   // Returns OK iff the initialization was successful.
   Status status() const;
-- 
GitLab


From 2e2a3ca8e827138a02da1cf129ab28057b9f9ddd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 09:32:48 -0800
Subject: [PATCH 0960/2345] Update Eigen to
 https://bitbucket.org/eigen/eigen/commits/bcc817c0ba985fc27b0c8b9998351cf0f789687b

PiperOrigin-RevId: 229941993
---
 tensorflow/core/kernels/cwise_ops.h | 34 +++--------------------------
 tensorflow/workspace.bzl            |  8 +++----
 2 files changed, 7 insertions(+), 35 deletions(-)

diff --git a/tensorflow/core/kernels/cwise_ops.h b/tensorflow/core/kernels/cwise_ops.h
index a22d76717a..15bd533be0 100644
--- a/tensorflow/core/kernels/cwise_ops.h
+++ b/tensorflow/core/kernels/cwise_ops.h
@@ -51,15 +51,12 @@ struct scalar_arg_op<std::complex<double>> {
 };
 #endif
 
+#if EIGEN_HAS_CXX11_MATH == 0
 template <typename T>
 struct scalar_asinh_op {
   EIGEN_EMPTY_STRUCT_CTOR(scalar_asinh_op)
   EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T operator()(const T& a) const {
-#if EIGEN_HAS_CXX11_MATH
-    return numext::asinh(a);
-#else
     return std::asinh(a);
-#endif  // EIGEN_HAS_CXX11_MATH
   }
 };
 template <typename T>
@@ -71,11 +68,7 @@ template <typename T>
 struct scalar_acosh_op {
   EIGEN_EMPTY_STRUCT_CTOR(scalar_acosh_op)
   EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T operator()(const T& a) const {
-#if EIGEN_HAS_CXX11_MATH
-    return numext::acosh(a);
-#else
     return std::acosh(a);
-#endif  // EIGEN_HAS_CXX11_MATH
   }
 };
 template <typename T>
@@ -87,35 +80,14 @@ template <typename T>
 struct scalar_atanh_op {
   EIGEN_EMPTY_STRUCT_CTOR(scalar_atanh_op)
   EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T operator()(const T& a) const {
-#if EIGEN_HAS_CXX11_MATH
-    return numext::atanh(a);
-#else
     return std::atanh(a);
-#endif  // EIGEN_HAS_CXX11_MATH
   }
 };
 template <typename T>
 struct functor_traits<scalar_atanh_op<T>> {
   enum { Cost = 5 * NumTraits<T>::MulCost, PacketAccess = false };
 };
-
-// TODO(rmlarsen): This is a workaround for upstream change
-// https://bitbucket.org/eigen/eigen/commits/f339468d04d0f87caeb6cab9aef568627e9f6ea9
-// that renamed scalar_binary_pow_op to scalar_pow_op and deleted the unary
-// version of the latter. Remove once we upgrade to Eigen 3.3.
-template <typename Scalar, typename Exponent>
-struct scalar_binary_pow_op_google {
-  EIGEN_EMPTY_STRUCT_CTOR(scalar_binary_pow_op_google)
-  EIGEN_DEVICE_FUNC inline Scalar operator()(const Scalar& a,
-                                             const Exponent& b) const {
-    return numext::pow(a, b);
-  }
-};
-
-template <typename Scalar, typename Exponent>
-struct functor_traits<scalar_binary_pow_op_google<Scalar, Exponent>> {
-  enum { Cost = 5 * NumTraits<Scalar>::MulCost, PacketAccess = false };
-};
+#endif
 
 template <typename Scalar, typename Exponent>
 struct safe_scalar_binary_pow_op {
@@ -865,7 +837,7 @@ template <typename T>
 struct floor_div_real : base<T, Eigen::internal::google_floor_div_real<T>> {};
 
 template <typename T>
-struct pow : base<T, Eigen::internal::scalar_binary_pow_op_google<T, T>> {};
+struct pow : base<T, Eigen::internal::scalar_pow_op<T, T>> {};
 
 template <typename T>
 struct safe_pow : base<T, Eigen::internal::safe_scalar_binary_pow_op<T, T>> {
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index ad73b19ff1..71ecd4d902 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -140,11 +140,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "eigen_archive",
         build_file = clean_dep("//third_party:eigen.BUILD"),
-        sha256 = "df54d805573871d03aaef6d27d45439c18ff6e8db215a3b922b2daefd6914147",
-        strip_prefix = "eigen-eigen-ad3bcd81cc49",
+        sha256 = "48678550a32665331d729be87076e576f2502fff325f5b6c2c78ebf7b1b22c7b",
+        strip_prefix = "eigen-eigen-bcc817c0ba98",
         urls = [
-            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/ad3bcd81cc49.tar.gz",
-            "https://bitbucket.org/eigen/eigen/get/ad3bcd81cc49.tar.gz",
+            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/bcc817c0ba98.tar.gz",
+            "https://bitbucket.org/eigen/eigen/get/bcc817c0ba98.tar.gz",
         ],
     )
 
-- 
GitLab


From cf677e677b11cb503832be9a9cc2e7f6a5076d76 Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Fri, 18 Jan 2019 09:47:51 -0800
Subject: [PATCH 0961/2345] Fix an issue with Sequential manual building when
 in deferred mode.

PiperOrigin-RevId: 229944372
---
 tensorflow/python/keras/engine/sequential_test.py | 14 ++++++++++++++
 tensorflow/python/keras/engine/training.py        |  2 +-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/engine/sequential_test.py b/tensorflow/python/keras/engine/sequential_test.py
index 8e5d6fe93a..a65c200ce6 100644
--- a/tensorflow/python/keras/engine/sequential_test.py
+++ b/tensorflow/python/keras/engine/sequential_test.py
@@ -307,6 +307,20 @@ class TestSequential(keras_parameterized.TestCase):
     self.assertTrue(model.built)
     self.assertEqual(len(model.weights), 8)
 
+  @keras_parameterized.run_all_keras_modes(always_skip_v1=True)
+  def test_sequential_deferred_manual_build(self):
+    model = testing_utils.get_small_sequential_mlp(4, 5)
+    self.assertFalse(model.built)
+    model(array_ops.zeros([1, 2]))
+    self.assertTrue(model.built)
+    self.assertEqual(len(model.outputs), 0)
+    model.compile('rmsprop',
+                  loss='mse',
+                  run_eagerly=testing_utils.should_run_eagerly())
+    self.assertEqual(len(model.outputs), 0)
+    model.train_on_batch(np.zeros((1, 2)), np.zeros((1, 5)))
+    self.assertEqual(len(model.outputs), 1)
+
   @keras_parameterized.run_all_keras_modes
   def test_sequential_nesting(self):
     model = testing_utils.get_small_sequential_mlp(4, 3)
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index b80b906f67..53caac25c9 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -279,7 +279,7 @@ class Model(Network):
     self._distributed_model = None
     # Initialize model metric attributes.
     self._init_metric_attributes()
-    if not self.built:
+    if not self.built or not self.inputs or not self.outputs:
       # Model is not compilable because it does not know its number of inputs
       # and outputs, nor their shapes and names. We will compile after the first
       # time the model gets called on training data.
-- 
GitLab


From 26f4194fe4c26ea723e7828091801846fad44487 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 09:51:54 -0800
Subject: [PATCH 0962/2345] Add some utils for parsing Tensors as simple
 functions at runtime.

Also add some utils for calculating row endpoints of sparse matrices in coordinate list form.

PiperOrigin-RevId: 229945000
---
 tensorflow/core/kernels/BUILD                 |  53 +++
 tensorflow/core/kernels/sparse_utils.cc       | 161 +++++++++
 tensorflow/core/kernels/sparse_utils.h        |  71 ++++
 tensorflow/core/kernels/sparse_utils_test.cc  | 263 ++++++++++++++
 tensorflow/core/kernels/tensor_flag_utils.cc  | 187 ++++++++++
 tensorflow/core/kernels/tensor_flag_utils.h   |  78 +++++
 .../core/kernels/tensor_flag_utils_test.cc    | 322 ++++++++++++++++++
 7 files changed, 1135 insertions(+)
 create mode 100644 tensorflow/core/kernels/sparse_utils.cc
 create mode 100644 tensorflow/core/kernels/sparse_utils.h
 create mode 100644 tensorflow/core/kernels/sparse_utils_test.cc
 create mode 100644 tensorflow/core/kernels/tensor_flag_utils.cc
 create mode 100644 tensorflow/core/kernels/tensor_flag_utils.h
 create mode 100644 tensorflow/core/kernels/tensor_flag_utils_test.cc

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index c26c9af53f..9fcd4106fa 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -367,6 +367,59 @@ tf_kernel_library(
     ]),
 )
 
+cc_library(
+    name = "sparse_utils",
+    srcs = [
+        "sparse_utils.cc",
+    ],
+    hdrs = ["sparse_utils.h"],
+    deps = [
+        "//tensorflow/core:framework",
+        "//tensorflow/core:framework_lite",
+        "//tensorflow/core:lib_internal",
+    ],
+)
+
+tf_cc_test(
+    name = "sparse_utils_test",
+    srcs = ["sparse_utils_test.cc"],
+    deps = [
+        ":sparse_utils",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:framework_lite",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "@com_google_absl//absl/base:core_headers",
+    ],
+)
+
+cc_library(
+    name = "tensor_flag_utils",
+    srcs = [
+        "tensor_flag_utils.cc",
+    ],
+    hdrs = ["tensor_flag_utils.h"],
+    deps = [
+        "//tensorflow/core:framework",
+        "//tensorflow/core:framework_lite",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+tf_cc_test(
+    name = "tensor_flag_utils_test",
+    srcs = ["tensor_flag_utils_test.cc"],
+    deps = [
+        ":tensor_flag_utils",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:framework_lite",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "@com_google_absl//absl/base:core_headers",
+    ],
+)
+
 tf_cuda_library(
     name = "ops_testutil",
     testonly = 1,
diff --git a/tensorflow/core/kernels/sparse_utils.cc b/tensorflow/core/kernels/sparse_utils.cc
new file mode 100644
index 0000000000..198862940d
--- /dev/null
+++ b/tensorflow/core/kernels/sparse_utils.cc
@@ -0,0 +1,161 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/kernels/sparse_utils.h"
+
+#include <cstddef>
+
+#include "tensorflow/core/framework/tensor_shape.h"
+
+namespace tensorflow {
+namespace sparse_utils {
+
+template <typename Tindices>
+Tindices FindNextDenseRowStartIndex(
+    const Tindices sparse_index_begin,
+    const typename TTypes<Tindices>::ConstMatrix& indices_mat) {
+  // Search in the index range [begin, end) of indices_mat.
+  Tindices begin = sparse_index_begin;
+  Tindices end = indices_mat.dimension(0);
+  const Tindices orig_sparse_index_end = end;
+
+  // The first dense row we search.
+  const Tindices orig_dense_index_begin = indices_mat(begin, 0);
+  // Early exit if no next dense row index.
+  if (orig_dense_index_begin == static_cast<int64>(indices_mat(end - 1, 0))) {
+    return orig_sparse_index_end;
+  }
+
+  Tindices increment = 1;
+  while (begin + increment < end &&
+         indices_mat(begin + increment, 0) == orig_dense_index_begin) {
+    increment *= 2;
+  }
+  // Narrow the search space as an optimization.
+  if (begin + increment < end) {
+    end = begin + increment;
+  }
+  begin += increment / 2;
+
+  // Perform a binary search on the interval [begin, end) for
+  // dense_row_index_to_find.
+  const Tindices dense_row_index_to_find = orig_dense_index_begin;
+  while (begin < end) {
+    const Tindices m = begin + (end - begin) / 2;
+    const Tindices m_dense_row_index = static_cast<Tindices>(indices_mat(m, 0));
+    if (m_dense_row_index == dense_row_index_to_find &&
+        (m + 1 == orig_sparse_index_end ||
+         static_cast<Tindices>(indices_mat(m + 1, 0)) !=
+             dense_row_index_to_find)) {
+      return m + 1;
+    } else if (m_dense_row_index <= dense_row_index_to_find) {
+      begin = m + 1;
+    } else {
+      end = m;
+    }
+  }
+
+  // No next dense row index.
+  return orig_sparse_index_end;
+}
+
+template <typename Tindices>
+std::vector<Tindices> GetStartIndicesOfEachDenseRow(
+    const typename TTypes<Tindices>::ConstMatrix& indices_mat,
+    bool* contains_empty_rows) {
+  int64 start_sparse_index_of_cur_dense_row = 0;
+  std::vector<Tindices> segment_indices;
+  const Tindices num_entries_in_sparse_tensor = indices_mat.dimension(0);
+  const Tindices num_dense_rows_in_sparse_tensor =
+      1 + indices_mat(num_entries_in_sparse_tensor - 1, 0) - indices_mat(0, 0);
+  // Reserve an extra slot for the 0 we store in the first entry by convention.
+  segment_indices.reserve(1 + num_dense_rows_in_sparse_tensor);
+  segment_indices.push_back(0);
+  *contains_empty_rows = false;
+  while (true) {
+    const Tindices start_sparse_index_of_next_dense_row =
+        FindNextDenseRowStartIndex<Tindices>(
+            start_sparse_index_of_cur_dense_row, indices_mat);
+    if (start_sparse_index_of_next_dense_row == num_entries_in_sparse_tensor) {
+      segment_indices.push_back(start_sparse_index_of_next_dense_row);
+      break;
+    }
+    // Encode the length of the current dense row as well as the lengths of all
+    // the empty rows until the next dense row,
+    for (Tindices i = 0;
+         i < indices_mat(start_sparse_index_of_next_dense_row, 0) -
+                 indices_mat(start_sparse_index_of_cur_dense_row, 0);
+         ++i) {
+      segment_indices.push_back(start_sparse_index_of_next_dense_row);
+    }
+    // If there is more than one row between the current and next non-empty
+    // rows then those rows are empty.
+    *contains_empty_rows |=
+        indices_mat(start_sparse_index_of_next_dense_row, 0) -
+            indices_mat(start_sparse_index_of_cur_dense_row, 0) >
+        1;
+    start_sparse_index_of_cur_dense_row = start_sparse_index_of_next_dense_row;
+  }
+  return segment_indices;
+}
+
+template <typename Tindices>
+std::vector<Tindices> ParseRowStartIndices(
+    const tensorflow::Tensor& tensor,
+    const Tindices num_nonzero_entries_in_sparse_mat) {
+  std::vector<Tindices> out;
+  auto vec = tensor.vec<Tindices>();
+  out.reserve(vec.size() + 1);
+  for (size_t i = 0; i < vec.dimension(0); ++i) {
+    out.push_back(vec(i));
+  }
+  out.push_back(num_nonzero_entries_in_sparse_mat);
+  return out;
+}
+
+template <typename Tindices>
+bool ContainsEmptyRows(const std::vector<Tindices>& row_start_indices) {
+  // Skip checking the lengths of the first and last dense rows since those are
+  // always non-empty.
+  for (size_t i = 2; i < row_start_indices.size() - 1; ++i) {
+    if (row_start_indices.at(i) - row_start_indices.at(i - 1) == 0) {
+      return true;
+    }
+  }
+  return false;
+}
+
+#define REGISTER_SPARSE_UTIL_FUNCTIONS(TypeIndex)                           \
+  template TypeIndex FindNextDenseRowStartIndex<TypeIndex>(                 \
+      const TypeIndex sparse_index_begin,                                   \
+      const TTypes<TypeIndex>::ConstMatrix& indices_mat);                   \
+  template std::vector<TypeIndex> GetStartIndicesOfEachDenseRow<TypeIndex>( \
+      const TTypes<TypeIndex>::ConstMatrix& indices_mat,                    \
+      bool* contains_empty_rows);                                           \
+  template bool ContainsEmptyRows<TypeIndex>(                               \
+      const std::vector<TypeIndex>& row_start_indices);                     \
+  template std::vector<TypeIndex> ParseRowStartIndices<TypeIndex>(          \
+      const tensorflow::Tensor& tensor,                                     \
+      const TypeIndex num_nonzero_entries_in_sparse_mat);
+
+REGISTER_SPARSE_UTIL_FUNCTIONS(int32);
+REGISTER_SPARSE_UTIL_FUNCTIONS(int64);
+REGISTER_SPARSE_UTIL_FUNCTIONS(uint8);
+REGISTER_SPARSE_UTIL_FUNCTIONS(uint16);
+REGISTER_SPARSE_UTIL_FUNCTIONS(uint32);
+REGISTER_SPARSE_UTIL_FUNCTIONS(uint64);
+
+}  // namespace sparse_utils
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/sparse_utils.h b/tensorflow/core/kernels/sparse_utils.h
new file mode 100644
index 0000000000..9e3c41a496
--- /dev/null
+++ b/tensorflow/core/kernels/sparse_utils.h
@@ -0,0 +1,71 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Helpers for writing OpKernels for sparse tensors.
+#ifndef TENSORFLOW_CORE_KERNELS_SPARSE_UTILS_H_
+#define TENSORFLOW_CORE_KERNELS_SPARSE_UTILS_H_
+
+#include <vector>
+
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace tensorflow {
+namespace sparse_utils {
+
+// Find the index i of the first element for which
+// indices_mat(sparse_index_begin, 0) < indices_mat(i, 0).
+// The search is conducted in the open interval
+// [sparse_index_begin, indices_mat.dimension(0)) and when no such i is found,
+// indices_mat.dimension(0) is returned.
+// indices_mat(k, 0) should be non-decreasing over the interval
+// [begin, indices_mat.dimension(0)).
+// Requires 0 <= sparse_index_begin < indices_mat.dimension(0).
+template <typename Tindices>
+Tindices FindNextDenseRowStartIndex(
+    const Tindices sparse_index_begin,
+    const typename TTypes<Tindices>::ConstMatrix& indices_mat);
+
+// Returns the vector v of indices in indices_mat at which new dense matrix
+// rows begin.
+// v.front() = 0, v.back() = indices_mat.dimension(0), and for i > 0,
+// v[i] - v[i-1] is the length of the ith dense row in indices_mat.
+// *contains_empty_rows = true if and only if indices_mat contains empty rows
+// (rows without values) between its first and last row.
+template <typename Tindices>
+std::vector<Tindices> GetStartIndicesOfEachDenseRow(
+    const typename TTypes<Tindices>::ConstMatrix& indices_mat,
+    bool* contains_empty_rows);
+
+// Converts tensor.vec<Tindices> to an std::vector<Tindices> object, appends
+// the value num_nonzero_entries_in_sparse_mat, and returns the result.
+template <typename Tindices>
+std::vector<Tindices> ParseRowStartIndices(
+    const tensorflow::Tensor& tensor,
+    const Tindices num_nonzero_entries_in_sparse_mat);
+
+// Returns true if and only if the sparse matrix indices_mat whose row start
+// indices are represented by row_start_indices has empty dense rows
+// (between its first and last dense rows).
+// This function satisfies the identity row_start_indices ==
+// GetStartIndicesOfEachDenseRow(indices_mat, &return_value).
+template <typename Tindices>
+bool ContainsEmptyRows(const std::vector<Tindices>& row_start_indices);
+
+}  // namespace sparse_utils
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_KERNELS_SPARSE_UTILS_H_
diff --git a/tensorflow/core/kernels/sparse_utils_test.cc b/tensorflow/core/kernels/sparse_utils_test.cc
new file mode 100644
index 0000000000..5d0adff886
--- /dev/null
+++ b/tensorflow/core/kernels/sparse_utils_test.cc
@@ -0,0 +1,263 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/kernels/sparse_utils.h"
+
+#include <vector>
+
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/framework/types.pb.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace {
+
+using tensorflow::DataType;
+using tensorflow::int32;
+using tensorflow::int64;
+using tensorflow::Tensor;
+using tensorflow::TTypes;
+using tensorflow::uint16;
+using tensorflow::uint32;
+using tensorflow::uint64;
+using tensorflow::sparse_utils::ContainsEmptyRows;
+using tensorflow::sparse_utils::FindNextDenseRowStartIndex;
+using tensorflow::sparse_utils::GetStartIndicesOfEachDenseRow;
+using tensorflow::sparse_utils::ParseRowStartIndices;
+
+TEST(SparseUtilsTest, GetStartIndicesOfEachDenseRow) {
+  {
+    int32 data[] = {0, 0, 1, 0, 4, 0, 6, 0, 7, 0, 8, 0, 10, 0, 12, 0};
+    TTypes<int32>::ConstMatrix indices_mat(data, 8, 2);
+    // indices_list = {0, 1, 4, 6, 7, 8, 10, 12};
+    bool contains_empty_rows;
+    EXPECT_TRUE(GetStartIndicesOfEachDenseRow<int32>(indices_mat,
+                                                     &contains_empty_rows) ==
+                std::vector<int32>({0, 1, 2, 2, 2, 3, 3, 4, 5, 6, 6, 7, 7, 8}));
+    EXPECT_TRUE(contains_empty_rows);
+  }
+  {
+    int32 data[] = {0, 0, 1, 0, 1, 0, 4, 0, 4, 0, 4, 0,  6, 0,  7,
+                    0, 7, 0, 7, 0, 7, 0, 8, 0, 8, 0, 10, 0, 12, 0};
+    TTypes<int32>::ConstMatrix indices_mat(data, 15, 2);
+    // indices_list = {0, 1, 1, 4, 4, 4,  6, 7, 7, 7, 7, 8, 8, 10, 12};
+    bool contains_empty_rows;
+    EXPECT_TRUE(
+        GetStartIndicesOfEachDenseRow<int32>(indices_mat,
+                                             &contains_empty_rows) ==
+        std::vector<int32>({0, 1, 3, 3, 3, 6, 6, 7, 11, 13, 13, 14, 14, 15}));
+    EXPECT_TRUE(contains_empty_rows);
+  }
+  {
+    int64 data[] = {3, 0};
+    TTypes<int64>::ConstMatrix indices_mat(data, 1, 2);
+    bool contains_empty_rows;
+    EXPECT_TRUE(GetStartIndicesOfEachDenseRow<int64>(indices_mat,
+                                                     &contains_empty_rows) ==
+                std::vector<int64>({0, 1}));
+    EXPECT_FALSE(contains_empty_rows);
+  }
+  {
+    uint32 data[] = {3, 0, 3, 0};
+    TTypes<uint32>::ConstMatrix indices_mat(data, 2, 2);
+    bool contains_empty_rows;
+    EXPECT_TRUE(GetStartIndicesOfEachDenseRow<uint32>(indices_mat,
+                                                      &contains_empty_rows) ==
+                std::vector<uint32>({0, 2}));
+    EXPECT_FALSE(contains_empty_rows);
+  }
+  {
+    uint16 data[] = {0, 0, 0, 0, 0, 0, 1, 0};
+    TTypes<uint16>::ConstMatrix indices_mat(data, 4, 2);
+    // indices_list = {0, 0, 0, 1};
+    bool contains_empty_rows;
+    EXPECT_TRUE(GetStartIndicesOfEachDenseRow<uint16>(indices_mat,
+                                                      &contains_empty_rows) ==
+                std::vector<uint16>({0, 3, 4}));
+    EXPECT_FALSE(contains_empty_rows);
+  }
+  {
+    uint64 data[] = {0, 0, 0, 0, 0, 0, 3, 0};
+    TTypes<uint64>::ConstMatrix indices_mat(data, 4, 2);
+    bool contains_empty_rows;
+    // indices_list = {0, 0, 0, 3};
+    EXPECT_TRUE(GetStartIndicesOfEachDenseRow<uint64>(indices_mat,
+                                                      &contains_empty_rows) ==
+                std::vector<uint64>({0, 3, 3, 3, 4}));
+    EXPECT_TRUE(contains_empty_rows);
+  }
+}
+
+TEST(SparseUtilsTest, ParseRowStartIndices) {
+  {
+    Tensor t(DataType::DT_INT32, {1});
+    int indx = 0;
+    for (const int32 v : {0}) {
+      t.flat<int32>()(indx++) = v;
+    }
+    EXPECT_TRUE(ParseRowStartIndices<int32>(t, 1) ==
+                std::vector<int32>({0, 1}));
+  }
+  {
+    Tensor t(DataType::DT_INT64, {1});
+    int indx = 0;
+    for (const int64 v : {0}) {
+      t.flat<int64>()(indx++) = v;
+    }
+    EXPECT_TRUE(ParseRowStartIndices<int64>(t, 2) ==
+                std::vector<int64>({0, 2}));
+  }
+  {
+    Tensor t(DataType::DT_UINT64, {2});
+    int indx = 0;
+    for (const uint64 v : {0, 3}) {
+      t.flat<uint64>()(indx++) = v;
+    }
+    EXPECT_TRUE(ParseRowStartIndices<uint64>(t, 4) ==
+                std::vector<uint64>({0, 3, 4}));
+  }
+  {
+    Tensor t(DataType::DT_UINT16, {2});
+    int indx = 0;
+    for (const uint16 v : {0, 3}) {
+      t.flat<uint16>()(indx++) = v;
+    }
+    EXPECT_TRUE(ParseRowStartIndices<uint16>(t, 4) ==
+                std::vector<uint16>({0, 3, 4}));
+  }
+}
+
+TEST(SparseUtilsTest, ContainsEmptyRows) {
+  {
+    int32 data[] = {0, 0, 1, 0, 4, 0, 6, 0, 7, 0, 8, 0, 10, 0, 12, 0};
+    TTypes<int32>::ConstMatrix indices_mat(data, 8, 2);
+    bool contains_empty_rows;
+    const auto segment_indices =
+        GetStartIndicesOfEachDenseRow<int32>(indices_mat, &contains_empty_rows);
+    // indices_list = {0, 1, 4, 6, 7, 8, 10, 12};
+    EXPECT_TRUE(ContainsEmptyRows(segment_indices));
+  }
+  {
+    int64 data[] = {0, 0, 1, 0, 4, 0, 6, 0, 7, 0, 8, 0, 10, 0, 12, 0};
+    TTypes<int64>::ConstMatrix indices_mat(data, 8, 2);
+    bool contains_empty_rows;
+    const auto segment_indices =
+        GetStartIndicesOfEachDenseRow<int64>(indices_mat, &contains_empty_rows);
+    // indices_list = {0, 1, 4, 6, 7, 8, 10, 12};
+    EXPECT_TRUE(ContainsEmptyRows(segment_indices));
+  }
+  {
+    int32 data[] = {1, 0, 1, 1, 2, 0, 2, 1, 2, 2, 3, 4};
+    TTypes<int32>::ConstMatrix indices_mat(data, 6, 2);
+    bool contains_empty_rows;
+    const auto segment_indices =
+        GetStartIndicesOfEachDenseRow<int32>(indices_mat, &contains_empty_rows);
+    // indices_list = {1, 1, 2, 2, 2, 3};
+    EXPECT_FALSE(ContainsEmptyRows(segment_indices));
+  }
+  {
+    uint16 data[] = {1, 0, 1, 1, 2, 0, 2, 1, 2, 2, 3, 4};
+    TTypes<uint16>::ConstMatrix indices_mat(data, 6, 2);
+    bool contains_empty_rows;
+    const auto segment_indices = GetStartIndicesOfEachDenseRow<uint16>(
+        indices_mat, &contains_empty_rows);
+    // indices_list = {1, 1, 2, 2, 2, 3};
+    EXPECT_FALSE(ContainsEmptyRows(segment_indices));
+  }
+  {
+    int32 data[] = {0, 0, 1, 0, 1, 1, 2, 0, 2, 1, 2, 2, 3, 4};
+    TTypes<int32>::ConstMatrix indices_mat(data, 7, 2);
+    bool contains_empty_rows;
+    const auto segment_indices =
+        GetStartIndicesOfEachDenseRow<int32>(indices_mat, &contains_empty_rows);
+    // indices_list = {0, 1, 1, 2, 2, 2, 3};
+    EXPECT_FALSE(ContainsEmptyRows(segment_indices));
+  }
+  {
+    int64 data[] = {0, 0, 1, 0, 1, 1, 2, 0, 2, 1, 2, 2, 3, 4};
+    TTypes<int64>::ConstMatrix indices_mat(data, 7, 2);
+    bool contains_empty_rows;
+    const auto segment_indices =
+        GetStartIndicesOfEachDenseRow<int64>(indices_mat, &contains_empty_rows);
+    // indices_list = {0, 1, 1, 2, 2, 2, 3};
+    EXPECT_FALSE(ContainsEmptyRows(segment_indices));
+  }
+  {
+    uint32 data[] = {0, 0, 0, 1, 0, 2, 2, 0, 2, 1, 2, 2, 3, 4};
+    TTypes<uint32>::ConstMatrix indices_mat(data, 7, 2);
+    bool contains_empty_rows;
+    const auto segment_indices = GetStartIndicesOfEachDenseRow<uint32>(
+        indices_mat, &contains_empty_rows);
+    // indices_list = {0, 0, 0, 2, 2, 2, 3};
+    EXPECT_TRUE(ContainsEmptyRows(segment_indices));
+  }
+  {
+    int64 data[] = {0, 0, 0, 1, 0, 2, 2, 0, 2, 1, 2, 2, 3, 4};
+    TTypes<int64>::ConstMatrix indices_mat(data, 7, 2);
+    bool contains_empty_rows;
+    const auto segment_indices =
+        GetStartIndicesOfEachDenseRow<int64>(indices_mat, &contains_empty_rows);
+    // indices_list = {0, 0, 0, 2, 2, 2, 3};
+    EXPECT_TRUE(ContainsEmptyRows(segment_indices));
+  }
+  {
+    uint64 data[] = {0, 0, 0, 1, 0, 2, 1, 0, 2, 1, 2, 2, 3, 4};
+    TTypes<uint64>::ConstMatrix indices_mat(data, 7, 2);
+    bool contains_empty_rows;
+    const auto segment_indices = GetStartIndicesOfEachDenseRow<uint64>(
+        indices_mat, &contains_empty_rows);
+    // indices_list = {0, 0, 0, 1, 2, 2, 3};
+    EXPECT_FALSE(ContainsEmptyRows(segment_indices));
+  }
+}
+
+TEST(SparseUtilsTest, FindNextDenseRowStartIndex) {
+  {
+    int32 data[] = {0, 0, 1, 0, 4, 0, 6, 0, 7, 0, 8, 0, 10, 0, 12, 0};
+    TTypes<int32>::ConstMatrix indices_mat(data, 8, 2);
+    // indices_list = {0, 1, 4, 6, 7, 8, 10, 12};
+    for (int32 i = 0; i < 8; ++i) {
+      EXPECT_EQ(i + 1, FindNextDenseRowStartIndex<int32>(i, indices_mat));
+    }
+  }
+  {
+    uint16 data[] = {0, 0, 1, 0, 4, 0, 6, 0, 7, 0, 8, 0, 10, 0, 12, 0};
+    TTypes<uint16>::ConstMatrix indices_mat(data, 8, 2);
+    // indices_list = {0, 1, 4, 6, 7, 8, 10, 12};
+    for (uint16 i = 0; i < 8; ++i) {
+      EXPECT_EQ(i + 1, FindNextDenseRowStartIndex<uint16>(i, indices_mat));
+    }
+  }
+  {
+    int64 data[] = {0, 0, 1, 0, 1, 0, 4, 0, 4, 0, 4, 0,  6, 0,  7,
+                    0, 7, 0, 7, 0, 7, 0, 8, 0, 8, 0, 10, 0, 12, 0};
+    TTypes<int64>::ConstMatrix indices_mat(data, 15, 2);
+    // indices_list = {0, 1, 1, 4, 4, 4,  6, 7, 7, 7, 7, 8, 8, 10, 12};
+    EXPECT_EQ(3, FindNextDenseRowStartIndex<int64>(static_cast<int64>(1),
+                                                   indices_mat));
+    EXPECT_EQ(3, FindNextDenseRowStartIndex<int64>(static_cast<int64>(2),
+                                                   indices_mat));
+    EXPECT_EQ(6, FindNextDenseRowStartIndex<int64>(static_cast<int64>(3),
+                                                   indices_mat));
+    EXPECT_EQ(6, FindNextDenseRowStartIndex<int64>(static_cast<int64>(4),
+                                                   indices_mat));
+    EXPECT_EQ(14, FindNextDenseRowStartIndex<int64>(static_cast<int64>(13),
+                                                    indices_mat));
+    EXPECT_EQ(15, FindNextDenseRowStartIndex<int64>(static_cast<int64>(14),
+                                                    indices_mat));
+  }
+}
+
+}  // namespace
diff --git a/tensorflow/core/kernels/tensor_flag_utils.cc b/tensorflow/core/kernels/tensor_flag_utils.cc
new file mode 100644
index 0000000000..b6f9192729
--- /dev/null
+++ b/tensorflow/core/kernels/tensor_flag_utils.cc
@@ -0,0 +1,187 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/kernels/tensor_flag_utils.h"
+
+#include "absl/strings/str_cat.h"
+#include "tensorflow/core/framework/tensor_shape.h"
+
+namespace tensorflow {
+namespace tensor_flag_utils {
+
+Status ValidateSparseMatrixShardingConfig(const Tensor& config) {
+  if (TensorShapeUtils::IsScalar(config.shape())) {
+    const float scalar_config = config.template scalar<float>()();
+    if (0 < scalar_config && scalar_config <= 1.0) {
+      return Status::OK();
+    }
+    return Status(
+        error::INVALID_ARGUMENT,
+        absl::StrCat("Expected config to be in range (0, 1] but instead found ",
+                     scalar_config));
+  }
+  if (!TensorShapeUtils::IsMatrix(config.shape())) {
+    return Status(error::INVALID_ARGUMENT,
+                  absl::StrCat("Expected config to be either scalar or matrix "
+                               "but instead found tensor of rank ",
+                               config.dims()));
+  }
+  if (config.dim_size(1) != 3) {
+    return Status(
+        error::INVALID_ARGUMENT,
+        absl::StrCat(
+            "Expected config matrix to have dim(1) = 3 but instead found ",
+            config.dim_size(1)));
+  }
+
+  auto config_matrix = config.matrix<float>();
+  for (int i = 0; i < config.dim_size(0); ++i) {
+    if (0 > config_matrix(i, 0)) {
+      return errors::InvalidArgument(
+          "First column of fraction_rows_per_thread_config "
+          "should "
+          "have non-negative values but found ",
+          config_matrix(i, 0), " in row ", i);
+    }
+    if (0 > config_matrix(i, 1)) {
+      return errors::InvalidArgument(
+          "Second column of fraction_rows_per_thread_config "
+          "should "
+          "have non-negative values but found ",
+          config_matrix(i, 1), " in row ", i);
+    }
+    if (!(0 < config_matrix(i, 2) && config_matrix(i, 2) <= 1)) {
+      return errors::InvalidArgument(
+          "Last column of fraction_rows_per_thread_config should "
+          "have values in the range (0, 1] but found ",
+          config_matrix(i, 2), " in row ", i);
+    }
+  }
+  return Status::OK();
+}
+
+template <typename MatrixType, typename K>
+MatrixType FindConfigValueForKey(
+    const typename TTypes<MatrixType>::ConstMatrix& config_mat,
+    const std::pair<K, K>& key) {
+  const int last_row_index = config_mat.dimension(0) - 1;
+  for (int i = 0; i < last_row_index; ++i) {
+    if (key.first >= config_mat(i, 0) && key.second >= config_mat(i, 1)) {
+      return config_mat(i, 2);
+    }
+  }
+  return config_mat(last_row_index, 2);
+}
+
+Status ValidateScalarQuantityShardingConfig(const Tensor& config) {
+  if (TensorShapeUtils::IsScalar(config.shape())) {
+    const float scalar_config = config.template scalar<float>()();
+    if (0 < scalar_config && scalar_config <= 1.0) {
+      return Status::OK();
+    }
+    return Status(
+        error::INVALID_ARGUMENT,
+        absl::StrCat("Expected config to be in range (0, 1] but instead found ",
+                     scalar_config));
+  }
+  if (!TensorShapeUtils::IsMatrix(config.shape())) {
+    return Status(error::INVALID_ARGUMENT,
+                  absl::StrCat("Expected config to be either scalar or matrix "
+                               "but instead found tensor of rank ",
+                               config.dims()));
+  }
+  if (config.dim_size(1) != 2) {
+    return Status(
+        error::INVALID_ARGUMENT,
+        absl::StrCat(
+            "Expected config matrix to have dim(1) = 2 but instead found ",
+            config.dim_size(1)));
+  }
+
+  auto config_matrix = config.matrix<float>();
+  for (int i = 0; i < config.dim_size(0); ++i) {
+    if (0 > config_matrix(i, 0)) {
+      return errors::InvalidArgument(
+          "First column of fraction_rows_per_thread_config "
+          "should "
+          "have non-negative values but found ",
+          config_matrix(i, 0), " in row ", i);
+    }
+    if (!(0 < config_matrix(i, 1) && config_matrix(i, 1) <= 1)) {
+      return errors::InvalidArgument(
+          "Last column of fraction_rows_per_thread_config should "
+          "have values in the range (0, 1] but found ",
+          config_matrix(i, 1), " in row ", i);
+    }
+  }
+  return Status::OK();
+}
+
+template <typename MatrixType, typename K>
+MatrixType FindConfigValueForKey(
+    const typename TTypes<MatrixType>::ConstMatrix& config_mat, const K key) {
+  const int last_row_index = config_mat.dimension(0) - 1;
+  for (int i = 0; i < last_row_index; ++i) {
+    if (key >= config_mat(i, 0)) {
+      return config_mat(i, 1);
+    }
+  }
+  return config_mat(last_row_index, 1);
+}
+
+template <typename Tindices>
+Tindices GetLinearBucket(const Tindices value, const Tindices bucket_size) {
+  const Tindices next_multiple_of_bucket_size =
+      (value + bucket_size - 1) / bucket_size * bucket_size;
+  return next_multiple_of_bucket_size - (bucket_size - 1);
+}
+
+template <typename Tindices>
+Tindices GetPowerBucket(const Tindices value, const Tindices bucket_size) {
+  if (bucket_size == 1) {
+    return 1;
+  }
+  return std::pow(bucket_size, std::floor(std::log(bucket_size * (value - 1)) /
+                                          std::log(bucket_size)) -
+                                   1) +
+         1;
+}
+
+#define REGISTER_SPARSE_UTIL_FUNCTIONS(TypeIndex)                           \
+  template float FindConfigValueForKey<float, TypeIndex>(                   \
+      const TTypes<float>::ConstMatrix& config_mat,                         \
+      const std::pair<TypeIndex, TypeIndex>& key);                          \
+  template float FindConfigValueForKey<float, TypeIndex>(                   \
+      const TTypes<float>::ConstMatrix& config_mat, const TypeIndex key);   \
+  template int64 FindConfigValueForKey<int64, TypeIndex>(                   \
+      const TTypes<int64>::ConstMatrix& config_mat, const TypeIndex key);
+
+REGISTER_SPARSE_UTIL_FUNCTIONS(int32);
+REGISTER_SPARSE_UTIL_FUNCTIONS(int64);
+REGISTER_SPARSE_UTIL_FUNCTIONS(uint8);
+REGISTER_SPARSE_UTIL_FUNCTIONS(uint16);
+REGISTER_SPARSE_UTIL_FUNCTIONS(uint32);
+REGISTER_SPARSE_UTIL_FUNCTIONS(uint64);
+
+template int32 GetLinearBucket(const int32 value, const int32 bucket_size);
+
+template int64 GetLinearBucket(const int64 value, const int64 bucket_size);
+
+template int32 GetPowerBucket(const int32 value, const int32 bucket_size);
+
+template int64 GetPowerBucket(const int64 value, const int64 bucket_size);
+
+}  // namespace tensor_flag_utils
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/tensor_flag_utils.h b/tensorflow/core/kernels/tensor_flag_utils.h
new file mode 100644
index 0000000000..f406c73a29
--- /dev/null
+++ b/tensorflow/core/kernels/tensor_flag_utils.h
@@ -0,0 +1,78 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Helpers for parsing tensors as runtime flags.
+#ifndef TENSORFLOW_CORE_KERNELS_TENSOR_FLAG_UTILS_H_
+#define TENSORFLOW_CORE_KERNELS_TENSOR_FLAG_UTILS_H_
+
+#include <utility>
+#include <vector>
+
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace tensorflow {
+namespace tensor_flag_utils {
+
+// Converts tensor.vec<Tindices> to an std::vector<Tindices> object, appends
+// the value num_nonzero_entries_in_sparse_mat, and returns the result.
+template <typename Tindices>
+std::vector<Tindices> ParseRowStartIndices(
+    const tensorflow::Tensor& tensor,
+    const Tindices num_nonzero_entries_in_sparse_mat);
+
+// Returns Status::OK() if and only if config is a float scalar or a matrix with
+// dimensions M x 3. If config is a scalar then config must be in the range
+// [0, 1.0). If confix is a matrix then config must have shape M x 3, all of
+// its entries must be positive, and entries in the last column may not
+// exceed 1.0. If config is a matrix then it may not be empty.
+Status ValidateSparseMatrixShardingConfig(const Tensor& config);
+
+// Returns Status::OK() if and only if config is a float scalar or a non-empty
+// matrix with dimensions M x 2.
+Status ValidateScalarQuantityShardingConfig(const Tensor& config);
+
+// Returns the last entry of the first row in config_mat for which the first
+// two entries are no smaller than the respective entries in key. If no such
+// row exists then returns the last entry in the last row in config_mat.
+// config_mat may not be empty.
+template <typename MatrixType, typename K>
+MatrixType FindConfigValueForKey(
+    const typename TTypes<MatrixType>::ConstMatrix& config_mat,
+    const std::pair<K, K>& key);
+
+// Returns the last entry of the first row in config_mat for which the first
+// two entries are no smaller than the respective entries in key. If no such
+// row exists then returns the last entry in the last row in config_mat.
+// config_mat may not be empty.
+template <typename MatrixType, typename K>
+MatrixType FindConfigValueForKey(
+    const typename TTypes<MatrixType>::ConstMatrix& config_mat, const K key);
+
+// Returns largest multiple of bucket_size less than value.
+// Expects 1 <= bucket_size <= value.
+template <typename Tindices>
+Tindices GetLinearBucket(const Tindices value, const Tindices bucket_size);
+
+// Returns the largest power of bucket_size less than value.
+// Expects 1 <= bucket_size <= value. If bucket_size = 1, returns 1.
+template <typename Tindices>
+Tindices GetPowerBucket(const Tindices value, const Tindices bucket_size);
+
+}  // namespace tensor_flag_utils
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_KERNELS_TENSOR_FLAG_UTILS_H_
diff --git a/tensorflow/core/kernels/tensor_flag_utils_test.cc b/tensorflow/core/kernels/tensor_flag_utils_test.cc
new file mode 100644
index 0000000000..23ccc7ad7a
--- /dev/null
+++ b/tensorflow/core/kernels/tensor_flag_utils_test.cc
@@ -0,0 +1,322 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/kernels/tensor_flag_utils.h"
+
+#include <vector>
+
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace {
+
+using tensorflow::DataType;
+using tensorflow::int32;
+using tensorflow::int64;
+using tensorflow::Tensor;
+using tensorflow::TTypes;
+using tensorflow::error::INVALID_ARGUMENT;
+using tensorflow::tensor_flag_utils::FindConfigValueForKey;
+using tensorflow::tensor_flag_utils::GetLinearBucket;
+using tensorflow::tensor_flag_utils::GetPowerBucket;
+using tensorflow::tensor_flag_utils::ValidateScalarQuantityShardingConfig;
+using tensorflow::tensor_flag_utils::ValidateSparseMatrixShardingConfig;
+
+TEST(SparseUtilsTest, ValidateSparseMatrixShardingConfig) {
+  // Only a default is specified.
+  {
+    Tensor t(DataType::DT_FLOAT, {});
+    t.scalar<float>()() = 0.7;
+    EXPECT_TRUE(ValidateSparseMatrixShardingConfig(t).ok());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {});
+    t.scalar<float>()() = 1.0;
+    EXPECT_TRUE(ValidateSparseMatrixShardingConfig(t).ok());
+  }
+
+  // Misshapen.
+  {
+    Tensor t(DataType::DT_FLOAT, {1, 1});
+    int indx = 0;
+    for (const float v : {60.0}) {
+      t.flat<float>()(indx++) = v;
+    }
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateSparseMatrixShardingConfig(t).code());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {1, 2});
+    int indx = 0;
+    for (const float v : {
+             60.0,
+             50.0,
+         }) {
+      t.flat<float>()(indx++) = v;
+    }
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateSparseMatrixShardingConfig(t).code());
+  }
+
+  // Only one key is specified.
+  {
+    Tensor t(DataType::DT_FLOAT, {1, 3});
+    int indx = 0;
+    for (const float v : {30.0, 20.0, 1.0}) {
+      t.flat<float>()(indx++) = v;
+    }
+    EXPECT_TRUE(ValidateSparseMatrixShardingConfig(t).ok());
+  }
+
+  // Two keys are specified.
+  {
+    Tensor t(DataType::DT_FLOAT, {2, 3});
+    int indx = 0;
+    for (const float v : {60.0, 50.0, 0.41, 30.0, 20.0, 0.7}) {
+      t.flat<float>()(indx++) = v;
+    }
+    EXPECT_TRUE(ValidateSparseMatrixShardingConfig(t).ok());
+  }
+
+  // Out of range.
+  {
+    Tensor t(DataType::DT_FLOAT, {2, 3});
+    int indx = 0;
+    for (const float v : {60.0, 40.0, 0.41, 30.0, 20.0, 10.7}) {
+      t.flat<float>()(indx++) = v;
+    }
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateSparseMatrixShardingConfig(t).code());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {2, 3});
+    int indx = 0;
+    for (const float v : {60.0, 40.0, 0.41, 30.0, 20.0, -0.7}) {
+      t.flat<float>()(indx++) = v;
+    }
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateSparseMatrixShardingConfig(t).code());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {2, 3});
+    int indx = 0;
+    for (const float v : {60.0, -40.0, 0.41, 30.0, 20.0, 0.7}) {
+      t.flat<float>()(indx++) = v;
+    }
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateSparseMatrixShardingConfig(t).code());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {});
+    t.scalar<float>()() = -0.5;
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateSparseMatrixShardingConfig(t).code());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {});
+    t.scalar<float>()() = 0;
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateSparseMatrixShardingConfig(t).code());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {});
+    t.scalar<float>()() = 1.2;
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateSparseMatrixShardingConfig(t).code());
+  }
+}
+
+TEST(SparseUtilsTest, ValidateScalarQuantityShardingConfig) {
+  // Only a default is specified.
+  {
+    Tensor t(DataType::DT_FLOAT, {});
+    t.scalar<float>()() = 0.7;
+    EXPECT_TRUE(ValidateScalarQuantityShardingConfig(t).ok());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {});
+    t.scalar<float>()() = 1.0;
+    EXPECT_TRUE(ValidateScalarQuantityShardingConfig(t).ok());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {});
+    t.scalar<float>()() = 1.2;
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateScalarQuantityShardingConfig(t).code());
+  }
+
+  // Misshapen.
+  {
+    Tensor t(DataType::DT_FLOAT, {1, 1});
+    int indx = 0;
+    for (const float v : {60.0}) {
+      t.flat<float>()(indx++) = v;
+    }
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateScalarQuantityShardingConfig(t).code());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {1, 2});
+    int indx = 0;
+    for (const float v : {
+             60.0,
+             50.0,
+         }) {
+      t.flat<float>()(indx++) = v;
+    }
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateScalarQuantityShardingConfig(t).code());
+  }
+
+  // Two keys are specified.
+  {
+    Tensor t(DataType::DT_FLOAT, {1, 3});
+    int indx = 0;
+    for (const float v : {30.0, 20.0, 1.0}) {
+      t.flat<float>()(indx++) = v;
+    }
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateScalarQuantityShardingConfig(t).code());
+  }
+
+  // Only one key is specified.
+  {
+    Tensor t(DataType::DT_FLOAT, {2, 2});
+    int indx = 0;
+    for (const float v : {60.0, 0.41, 30.0, 0.7}) {
+      t.flat<float>()(indx++) = v;
+    }
+    EXPECT_TRUE(ValidateScalarQuantityShardingConfig(t).ok());
+  }
+
+  // Out of range.
+  {
+    Tensor t(DataType::DT_FLOAT, {2, 2});
+    int indx = 0;
+    for (const float v : {60.0, 0.41, 30.0, 10.7}) {
+      t.flat<float>()(indx++) = v;
+    }
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateScalarQuantityShardingConfig(t).code());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {2, 2});
+    int indx = 0;
+    for (const float v : {60.0, 0.41, 30.0, -0.7}) {
+      t.flat<float>()(indx++) = v;
+    }
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateScalarQuantityShardingConfig(t).code());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {2, 2});
+    int indx = 0;
+    for (const float v : {-40.0, 0.41, 20.0, 0.7}) {
+      t.flat<float>()(indx++) = v;
+    }
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateScalarQuantityShardingConfig(t).code());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {});
+    t.scalar<float>()() = -0.5;
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateScalarQuantityShardingConfig(t).code());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {});
+    t.scalar<float>()() = 0;
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateScalarQuantityShardingConfig(t).code());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {});
+    t.scalar<float>()() = 1.2;
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateScalarQuantityShardingConfig(t).code());
+  }
+}
+
+TEST(SparseUtils, FindConfigValueForKey) {
+  {
+    float data[] = {60.0, 50.0, 0.41, 30.0, 20.0, 0.1, 0, 0, 0.7};
+    TTypes<float>::ConstMatrix config_mat(data, 3, 3);
+    auto val = FindConfigValueForKey<float, int32>(config_mat, {70, 40});
+    EXPECT_FLOAT_EQ(0.1, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, {60, 50});
+    EXPECT_FLOAT_EQ(0.41, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, {60, 60});
+    EXPECT_FLOAT_EQ(0.41, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, {60, 40});
+    EXPECT_FLOAT_EQ(0.1, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, {50, 60});
+    EXPECT_FLOAT_EQ(0.1, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, {20, 30});
+    EXPECT_FLOAT_EQ(0.7, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, {30, 10});
+    EXPECT_FLOAT_EQ(0.7, val);
+  }
+  {
+    float data[] = {0, 0, 0.7};
+    TTypes<float>::ConstMatrix config_mat(data, 1, 3);
+    auto val = FindConfigValueForKey<float, int64>(config_mat, {70, 40});
+    EXPECT_FLOAT_EQ(0.7, val);
+    val = FindConfigValueForKey<float, int64>(config_mat, {60, 50});
+    EXPECT_FLOAT_EQ(0.7, val);
+    val = FindConfigValueForKey<float, int64>(config_mat, {60, 60});
+    EXPECT_FLOAT_EQ(0.7, val);
+    val = FindConfigValueForKey<float, int64>(config_mat, {60, 40});
+    EXPECT_FLOAT_EQ(0.7, val);
+    val = FindConfigValueForKey<float, int64>(config_mat, {50, 60});
+    EXPECT_FLOAT_EQ(0.7, val);
+    val = FindConfigValueForKey<float, int64>(config_mat, {20, 30});
+    EXPECT_FLOAT_EQ(0.7, val);
+    val = FindConfigValueForKey<float, int64>(config_mat, {30, 10});
+    EXPECT_FLOAT_EQ(0.7, val);
+  }
+  {
+    float data[] = {60.0, 50.0, 0.41, 0, 0, 0.7};
+    TTypes<float>::ConstMatrix config_mat(data, 2, 3);
+    auto val = FindConfigValueForKey<float, int32>(config_mat, {70, 40});
+    EXPECT_FLOAT_EQ(0.7, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, {60, 50});
+    EXPECT_FLOAT_EQ(0.41, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, {60, 60});
+    EXPECT_FLOAT_EQ(0.41, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, {60, 40});
+    EXPECT_FLOAT_EQ(0.7, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, {50, 60});
+    EXPECT_FLOAT_EQ(0.7, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, {20, 30});
+    EXPECT_FLOAT_EQ(0.7, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, {30, 10});
+    EXPECT_FLOAT_EQ(0.7, val);
+  }
+  {
+    float data[] = {60.0, 0.41, 50.0, 0.14, 0, 0.7};
+    TTypes<float>::ConstMatrix config_mat(data, 3, 2);
+    auto val = FindConfigValueForKey<float, int32>(config_mat, 70);
+    EXPECT_FLOAT_EQ(0.41, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, 60);
+    EXPECT_FLOAT_EQ(0.41, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, 55);
+    EXPECT_FLOAT_EQ(0.14, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, 50);
+    EXPECT_FLOAT_EQ(0.14, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, 20);
+    EXPECT_FLOAT_EQ(0.7, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, 30);
+    EXPECT_FLOAT_EQ(0.7, val);
+  }
+}
+
+TEST(SparseUtils, GetLinearBucket) {
+  EXPECT_EQ(11, GetLinearBucket(11, 5));
+  EXPECT_EQ(11, GetLinearBucket(12, 5));
+  EXPECT_EQ(1, GetLinearBucket(4ll, 5ll));
+}
+
+TEST(SparseUtils, GetPowerBucket) {
+  EXPECT_EQ(6, GetPowerBucket(11, 5));
+  EXPECT_EQ(6, GetPowerBucket(12, 5));
+  EXPECT_EQ(1332, GetPowerBucket(1335, 11));
+  EXPECT_EQ(5, GetPowerBucket(5ll, 4ll));
+  EXPECT_EQ(1, GetPowerBucket(4ll, 1ll));
+}
+
+}  // namespace
-- 
GitLab


From 0ed026100272867a270e1766c9ca0ae9804ff802 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Fri, 18 Jan 2019 10:00:49 -0800
Subject: [PATCH 0963/2345] Disable jni on mac since mac doesn't use gnu linker

This does not preclude building Android Java JNI since that uses a different toolchain that does support version-script

PiperOrigin-RevId: 229946385
---
 tensorflow/lite/java/BUILD      | 15 +++++++++++++++
 tensorflow/lite/java/ovic/BUILD | 10 ++++++++--
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/java/BUILD b/tensorflow/lite/java/BUILD
index a539a0cf77..8983079a31 100644
--- a/tensorflow/lite/java/BUILD
+++ b/tensorflow/lite/java/BUILD
@@ -90,6 +90,9 @@ java_test(
     size = "small",
     srcs = ["src/test/java/org/tensorflow/lite/TensorFlowLiteTest.java"],
     javacopts = JAVACOPTS,
+    tags = [
+        "no_mac",  # TODO(b/122888913): libtensorflowlite_test_jni broke on mac.
+    ],
     test_class = "org.tensorflow.lite.TensorFlowLiteTest",
     deps = [
         ":tensorflowlitelib",
@@ -103,6 +106,9 @@ java_test(
     size = "small",
     srcs = ["src/test/java/org/tensorflow/lite/DataTypeTest.java"],
     javacopts = JAVACOPTS,
+    tags = [
+        "no_mac",  # TODO(b/122888913): libtensorflowlite_test_jni broke on mac.
+    ],
     test_class = "org.tensorflow.lite.DataTypeTest",
     deps = [
         ":tensorflowlitelib",
@@ -126,6 +132,9 @@ java_test(
         "src/testdata/with_custom_op.lite",
     ],
     javacopts = JAVACOPTS,
+    tags = [
+        "no_mac",  # TODO(b/122888913): libtensorflowlite_test_jni broke on mac.
+    ],
     test_class = "org.tensorflow.lite.NativeInterpreterWrapperTest",
     deps = [
         ":tensorflowlitelib",
@@ -186,6 +195,9 @@ java_test(
         "src/testdata/add.bin",
     ],
     javacopts = JAVACOPTS,
+    tags = [
+        "no_mac",  # TODO(b/122888913): libtensorflowlite_test_jni broke on mac.
+    ],
     test_class = "org.tensorflow.lite.TensorTest",
     deps = [
         ":tensorflowlitelib",
@@ -199,6 +211,9 @@ filegroup(
     srcs = select({
         "//conditions:default": [":libtensorflowlite_jni.so"],
     }),
+    tags = [
+        "no_mac",  # TODO(b/122888913): libtensorflowlite_test_jni broke on mac.
+    ],
     visibility = ["//visibility:public"],
 )
 
diff --git a/tensorflow/lite/java/ovic/BUILD b/tensorflow/lite/java/ovic/BUILD
index 774320871e..b00c9cd058 100644
--- a/tensorflow/lite/java/ovic/BUILD
+++ b/tensorflow/lite/java/ovic/BUILD
@@ -19,7 +19,10 @@ java_test(
         "//tensorflow/lite/java/ovic/src/testdata:ovic_testdata",
     ],
     javacopts = JAVACOPTS,
-    tags = ["no_oss"],
+    tags = [
+        "no_mac",
+        "no_oss",  # TODO(b/122888913): libtensorflowlite_test_jni broke on mac.
+    ],
     test_class = "org.tensorflow.ovic.OvicClassifierTest",
     visibility = ["//visibility:public"],
     deps = [
@@ -87,7 +90,10 @@ java_test(
         "//tensorflow/lite/java/ovic/src/testdata:ovic_testdata",
     ],
     javacopts = JAVACOPTS,
-    tags = ["no_oss"],
+    tags = [
+        "no_mac",
+        "no_oss",  # TODO(b/122888913): libtensorflowlite_test_jni broke on mac.
+    ],
     test_class = "org.tensorflow.ovic.OvicDetectorTest",
     visibility = ["//visibility:public"],
     deps = [
-- 
GitLab


From e3f2b2067355da6197e8c9500fb74e40357a0f80 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 10:19:29 -0800
Subject: [PATCH 0964/2345] Use google-cloud-cpp v0.5.0 in tensorflow's
 WORKSPACE.

PiperOrigin-RevId: 229949984
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 71ecd4d902..3d8ea6d9e1 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -185,15 +185,15 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
 
     tf_http_archive(
         name = "com_github_googlecloudplatform_google_cloud_cpp",
-        sha256 = "44eee8bd47cbd5ff192e895b45f9f913e2e117f10fdb9af0fd3b1a87a7b53bc3",
-        strip_prefix = "google-cloud-cpp-0.4.0",
+        sha256 = "886bcba3616d5f362838a2d86ae0198dd3670a84a84c82291cda6c30e14779fc",
+        strip_prefix = "google-cloud-cpp-0.5.0",
         system_build_file = clean_dep("//third_party/systemlibs:google_cloud_cpp.BUILD"),
         system_link_files = {
             "//third_party/systemlibs:google_cloud_cpp.google.cloud.bigtable.BUILD": "google/cloud/bigtable/BUILD",
         },
         urls = [
-            "https://mirror.bazel.build/github.com/GoogleCloudPlatform/google-cloud-cpp/archive/v0.4.0.tar.gz",
-            "https://github.com/GoogleCloudPlatform/google-cloud-cpp/archive/v0.4.0.tar.gz",
+            "https://mirror.bazel.build/github.com/GoogleCloudPlatform/google-cloud-cpp/archive/v0.5.0.tar.gz",
+            "https://github.com/GoogleCloudPlatform/google-cloud-cpp/archive/v0.5.0.tar.gz",
         ],
     )
 
-- 
GitLab


From e8d74b48d05e0673d93633ac38e1ab88d71f4dff Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 10:20:52 -0800
Subject: [PATCH 0965/2345] Simplify BufferMap by moving forwardability out of
 it.

PiperOrigin-RevId: 229950204
---
 tensorflow/lite/delegates/flex/buffer_map.h   | 18 -------------
 .../lite/delegates/flex/buffer_map_test.cc    |  9 -------
 tensorflow/lite/delegates/flex/kernel.cc      | 27 +++++++++++++------
 3 files changed, 19 insertions(+), 35 deletions(-)

diff --git a/tensorflow/lite/delegates/flex/buffer_map.h b/tensorflow/lite/delegates/flex/buffer_map.h
index 45bc571f92..b73ed88d37 100644
--- a/tensorflow/lite/delegates/flex/buffer_map.h
+++ b/tensorflow/lite/delegates/flex/buffer_map.h
@@ -56,25 +56,7 @@ class BufferMap {
   // given TfLiteTensor's data.
   void SetFromTfLite(int tensor_index, const TfLiteTensor* tensor);
 
-  // Sets a bit indicating that the tensor associated with 'tensor_index' can
-  // be use by TF's forwarding optimizations.
-  void SetForwardable(int tensor_index) { forwardable_.insert(tensor_index); }
-
-  // Removes all information about which tensors are forwardable.
-  void ClearForwardable() { forwardable_.clear(); }
-
-  // Returns true if this tensor has been explicitly marks as forwardable by
-  // a call to SetForwardable().
-  bool IsForwardable(int tensor_index) const {
-    return forwardable_.count(tensor_index) > 0;
-  }
-
  private:
-  // List of tensors that can be used by TF in its forwarding optimization.
-  // Doing so allows an input tensor to be modified and used as the output
-  // tensor. The delegate takes care of not holding any references to tensors
-  // in this list while Eager is executing the corresponding op.
-  std::set<int> forwardable_;
   // Mapping from TL Lite tensor ID to TensorFlow's Tensor. All tensors that
   // are inputs or outputs of a subgraph will be added here, irrespective of
   // whether their data are managed by TF Lite or TensorFlow.
diff --git a/tensorflow/lite/delegates/flex/buffer_map_test.cc b/tensorflow/lite/delegates/flex/buffer_map_test.cc
index 8edeb280eb..9e8472f1e7 100644
--- a/tensorflow/lite/delegates/flex/buffer_map_test.cc
+++ b/tensorflow/lite/delegates/flex/buffer_map_test.cc
@@ -222,15 +222,6 @@ TEST(BufferMapTest, TensorFlowOverwritesTfLite) {
               ElementsAre(0, 0, 0, 0.123f, 0, 0));
 }
 
-TEST(BufferMapTest, Forwardable) {
-  BufferMap buffer_map;
-  EXPECT_FALSE(buffer_map.IsForwardable(0));
-  buffer_map.SetForwardable(0);
-  EXPECT_TRUE(buffer_map.IsForwardable(0));
-  buffer_map.ClearForwardable();
-  EXPECT_FALSE(buffer_map.IsForwardable(0));
-}
-
 }  // namespace
 }  // namespace flex
 }  // namespace tflite
diff --git a/tensorflow/lite/delegates/flex/kernel.cc b/tensorflow/lite/delegates/flex/kernel.cc
index 2e0fc22ad6..ceb9918f6f 100644
--- a/tensorflow/lite/delegates/flex/kernel.cc
+++ b/tensorflow/lite/delegates/flex/kernel.cc
@@ -67,6 +67,7 @@ class OpInputs {
     for (int index : TfLiteIntArrayView(indexes)) {
       inputs_.push_back(index);
     }
+    forwardable_.resize(inputs_.size());
   }
   ~OpInputs() {}
 
@@ -89,11 +90,21 @@ class OpInputs {
     }
   }
 
+  void SetForwardable(int i, bool v) { forwardable_[i] = v; }
+
+  bool IsForwardable(int i) const { return forwardable_[i]; }
+
   TensorSource GetTensorSource(int i) const { return sources_[i]; }
 
  private:
   std::vector<int> inputs_;
   std::vector<TensorSource> sources_;
+
+  // List of tensors that can be used by TF in its forwarding optimization.
+  // Doing so allows an input tensor to be modified and used as the output
+  // tensor. The delegate takes care of not holding any references to tensors
+  // in this list while Eager is executing the corresponding op.
+  std::vector<int> forwardable_;
 };
 
 // A list of outputs of a given node of the TensorFlow/Eager graph, along with
@@ -279,7 +290,7 @@ class OpNode {
       } else {
         // If this is a forwardable tensor, we will remove it from the previous
         // op's list, giving TF the opportunity to reuse its buffer.
-        bool unref_handle = buffer_map->IsForwardable(input_index);
+        bool unref_handle = inputs_.IsForwardable(i);
         auto* handle =
             s.node->outputs_.GetHandle(s.node_output_index, unref_handle);
         op_->MutableInputs()->push_back(handle);
@@ -485,13 +496,13 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
     }
   }
 
-  buffer_map->ClearForwardable();
-  for (const auto& x : tensor_ref_count) {
-    if (x.second == 1) {
-      // This tensor is referenced once by a single op. We can allow the TF
-      // kernel to "forward" it to the output, meaning its buffer will be
-      // reused and overwritten.
-      buffer_map->SetForwardable(x.first);
+  // All tensors that are referenced exactly once are marked as "forwardable",
+  // meaning that we will allow TensorFlow to reuse its buffer as the output of
+  // an op.
+  for (auto& node_data : op_data->nodes) {
+    for (int i = 0; i < node_data->inputs().Size(); ++i) {
+      bool f = (tensor_ref_count[node_data->inputs().TfLiteIndex(i)] == 1);
+      node_data->mutable_inputs()->SetForwardable(i, f);
     }
   }
 
-- 
GitLab


From d70cb7c2a548a664fbe85d273443412af6a24d80 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Fri, 18 Jan 2019 10:26:13 -0800
Subject: [PATCH 0966/2345] In tf2 tf.dtypes.as_string -> tf.strings.as_string

PiperOrigin-RevId: 229951239
---
 tensorflow/core/api_def/python_api/api_def_AsString.pbtxt | 4 ++++
 tensorflow/tools/api/golden/v1/tensorflow.strings.pbtxt   | 4 ++++
 tensorflow/tools/api/golden/v2/tensorflow.dtypes.pbtxt    | 4 ----
 tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt   | 4 ++++
 4 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/api_def/python_api/api_def_AsString.pbtxt b/tensorflow/core/api_def/python_api/api_def_AsString.pbtxt
index d51defc376..bc8cc309f5 100644
--- a/tensorflow/core/api_def/python_api/api_def_AsString.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_AsString.pbtxt
@@ -2,6 +2,10 @@ op {
   graph_op_name: "AsString"
   endpoint {
     name: "dtypes.as_string"
+    deprecation_version: 2
+  }
+  endpoint {
+    name: "strings.as_string"
   }
   endpoint {
     name: "as_string"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.strings.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.strings.pbtxt
index ad26ded10b..6f4f4c2f31 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.strings.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.strings.pbtxt
@@ -1,5 +1,9 @@
 path: "tensorflow.strings"
 tf_module {
+  member_method {
+    name: "as_string"
+    argspec: "args=[\'input\', \'precision\', \'scientific\', \'shortest\', \'width\', \'fill\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'False\', \'False\', \'-1\', \'\', \'None\'], "
+  }
   member_method {
     name: "format"
     argspec: "args=[\'template\', \'inputs\', \'placeholder\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'{}\', \'3\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.dtypes.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.dtypes.pbtxt
index 01b870a816..956e4d93e5 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.dtypes.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.dtypes.pbtxt
@@ -112,10 +112,6 @@ tf_module {
     name: "as_dtype"
     argspec: "args=[\'type_value\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "as_string"
-    argspec: "args=[\'input\', \'precision\', \'scientific\', \'shortest\', \'width\', \'fill\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'False\', \'False\', \'-1\', \'\', \'None\'], "
-  }
   member_method {
     name: "cast"
     argspec: "args=[\'x\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt
index 962cf9a723..e2da65eee4 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt
@@ -1,5 +1,9 @@
 path: "tensorflow.strings"
 tf_module {
+  member_method {
+    name: "as_string"
+    argspec: "args=[\'input\', \'precision\', \'scientific\', \'shortest\', \'width\', \'fill\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'False\', \'False\', \'-1\', \'\', \'None\'], "
+  }
   member_method {
     name: "format"
     argspec: "args=[\'template\', \'inputs\', \'placeholder\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'{}\', \'3\', \'None\'], "
-- 
GitLab


From 9f3a726641379b3c11c60693e3fc44c40e71d4fd Mon Sep 17 00:00:00 2001
From: Edward Loper <edloper@google.com>
Date: Fri, 18 Jan 2019 10:30:08 -0800
Subject: [PATCH 0967/2345] For SparseTensor shape invariant: accept
 TensorShape(None), and treat is as TensorShape([None]).

PiperOrigin-RevId: 229951972
---
 tensorflow/python/framework/sparse_tensor.py      |  3 ++-
 .../kernel_tests/control_flow_ops_py_test.py      | 15 +++++++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/framework/sparse_tensor.py b/tensorflow/python/framework/sparse_tensor.py
index c7f505fff3..c69fa41677 100644
--- a/tensorflow/python/framework/sparse_tensor.py
+++ b/tensorflow/python/framework/sparse_tensor.py
@@ -249,7 +249,8 @@ class SparseTensor(_TensorLike, composite_tensor.CompositeTensor):
   def _shape_invariant_to_components(self, shape=None):
     if shape is None:
       shape = self.dense_shape.shape
-      # TODO(edloper): What if shape.dense_shape.shape.ndims is None?
+    if shape.ndims is None:
+      shape = tensor_shape.TensorShape([None])
     if shape.ndims != 1:
       raise ValueError("Shape invariant for SparseTensor must have the form "
                        "TensorShape([r]), got %r" % shape)
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 27b3e26c77..d9d7af1f0d 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -1598,6 +1598,21 @@ class ControlFlowTest(test.TestCase):
     self.assertEqual(r.values.get_shape().as_list(), [None])
     self.assertEqual(r.dense_shape.get_shape().as_list(), [None])
 
+    # Shape invariant with ndims=None.  Technically, this isn't supported
+    # according to the docs, but we support it for backwards compatibility.
+    _, r = control_flow_ops.while_loop(
+        c, b1, [i, x],
+        [i.get_shape(), tensor_shape.TensorShape(None)])
+    self.assertEqual(r.indices.get_shape().as_list(), [None, None])
+    self.assertEqual(r.values.get_shape().as_list(), [None])
+    self.assertEqual(r.dense_shape.get_shape().as_list(), [None])
+    _, r = control_flow_ops.while_loop(
+        c, b3, [i, x],
+        [i.get_shape(), tensor_shape.TensorShape(None)])
+    self.assertEqual(r.indices.get_shape().as_list(), [None, None])
+    self.assertEqual(r.values.get_shape().as_list(), [None])
+    self.assertEqual(r.dense_shape.get_shape().as_list(), [None])
+
     # Explicit shape invariant, with a specific (incompatible) rank.
     with self.assertRaisesRegexp(ValueError, "is not compatible with"):
       _, r = control_flow_ops.while_loop(
-- 
GitLab


From 91e843486d1f230554dd0662273d88eb6fa7e295 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 10:48:05 -0800
Subject: [PATCH 0968/2345] Add TFLite NNAPI interface which uses a struct
 instead of using directly function calls. Add an empty implementation for
 NNAPI outside Android.

PiperOrigin-RevId: 229955475
---
 tensorflow/lite/BUILD                         |   2 +-
 tensorflow/lite/delegates/nnapi/BUILD         |   2 +-
 .../lite/delegates/nnapi/nnapi_delegate.cc    | 163 ++---
 tensorflow/lite/nnapi/BUILD                   |  23 +
 tensorflow/lite/nnapi/NeuralNetworksShim.h    |   5 +
 tensorflow/lite/nnapi/nnapi_implementation.cc | 137 +++++
 tensorflow/lite/nnapi/nnapi_implementation.h  | 582 ++++++++++++++++++
 .../lite/nnapi/nnapi_implementation_test.cc   | 120 ++++
 tensorflow/lite/nnapi_delegate.cc             | 137 ++---
 tensorflow/lite/testing/BUILD                 |   2 +-
 tensorflow/lite/testing/nnapi_example.cc      |   2 +-
 11 files changed, 1013 insertions(+), 162 deletions(-)
 create mode 100644 tensorflow/lite/nnapi/nnapi_implementation.cc
 create mode 100644 tensorflow/lite/nnapi/nnapi_implementation.h
 create mode 100644 tensorflow/lite/nnapi/nnapi_implementation_test.cc

diff --git a/tensorflow/lite/BUILD b/tensorflow/lite/BUILD
index 18fd1c1012..2cc661f2ba 100644
--- a/tensorflow/lite/BUILD
+++ b/tensorflow/lite/BUILD
@@ -193,7 +193,7 @@ cc_library(
         "//tensorflow/lite/core/api:api",
         "//tensorflow/lite/kernels:eigen_support",
         "//tensorflow/lite/kernels:gemm_support",
-        "//tensorflow/lite/nnapi:nnapi_lib",
+        "//tensorflow/lite/nnapi:nnapi_implementation",
         "//tensorflow/lite/profiling:profiler",
         "//tensorflow/lite/schema:schema_fbs",
     ] + select({
diff --git a/tensorflow/lite/delegates/nnapi/BUILD b/tensorflow/lite/delegates/nnapi/BUILD
index 63f6da1fa6..99cd6d3f85 100644
--- a/tensorflow/lite/delegates/nnapi/BUILD
+++ b/tensorflow/lite/delegates/nnapi/BUILD
@@ -16,7 +16,7 @@ cc_library(
         "//tensorflow/lite:kernel_api",
         "//tensorflow/lite/c:c_api_internal",
         "//tensorflow/lite/kernels:kernel_util",
-        "//tensorflow/lite/nnapi:nnapi_lib",
+        "//tensorflow/lite/nnapi:nnapi_implementation",
     ],
 )
 
diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
index 94fcc8740b..d14e449b3e 100644
--- a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
+++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
@@ -25,7 +25,7 @@ limitations under the License.
 #include "tensorflow/lite/context_util.h"
 #include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h"
 #include "tensorflow/lite/kernels/kernel_util.h"
-#include "tensorflow/lite/nnapi/NeuralNetworksShim.h"
+#include "tensorflow/lite/nnapi/nnapi_implementation.h"
 
 #ifdef __ANDROID__
 #include <sys/mman.h>
@@ -49,57 +49,39 @@ namespace {
   } while (0)
 
 namespace {
-int32_t GetAndroidSdkVersion() {
-#ifdef __ANDROID__
-  const char* sdkProp = "ro.build.version.sdk";
-  char sdkVersion[PROP_VALUE_MAX];
-  int length = __system_property_get(sdkProp, sdkVersion);
-  if (length != 0) {
-    for (int i = 0; i < length; ++i) {
-      int digit = sdkVersion[i] - '0';
-      if (digit < 0 || digit > 9) {
-        // Non-numeric SDK version, assume it's higher then expected;
-        return std::numeric_limits<int32_t>::max();
-      }
-    }
-    return atoi(sdkVersion);
-  }
-#endif  // __ANDROID__
-  return 0;
-}
-
 constexpr int32_t kMinSdkVersionForNNAPI = 27;
 constexpr int32_t kMinSdkVersionForNNAPI11 = 28;
-static const int32_t kAndroidSdkVersion = GetAndroidSdkVersion();
-
 }  // namespace
 
 // RAII NN API Model Destructor for use with std::unique_ptr
 struct NNFreeModel {
   void operator()(ANeuralNetworksModel* model) {
-    ANeuralNetworksModel_free(model);
+    NnApiImplementation()->ANeuralNetworksModel_free(model);
   }
 };
 // RAII NN API Compilation Destructor for use with std::unique_ptr
 struct NNFreeCompilation {
   void operator()(ANeuralNetworksCompilation* model) {
-    ANeuralNetworksCompilation_free(model);
+    NnApiImplementation()->ANeuralNetworksCompilation_free(model);
   }
 };
 
 // Manage NNAPI shared memory handle
 class NNMemory {
  public:
-  NNMemory(const char* name, size_t size) {
 #ifdef __ANDROID__
+  NNMemory(const NnApi* nnapi, const char* name, size_t size) {
+    nnapi_ = nnapi;
     byte_size_ = size;
-    fd_ = ASharedMemory_create(name, size);
+    fd_ = nnapi_->ASharedMemory_create(name, size);
     data_ptr_ = reinterpret_cast<uint8_t*>(
         mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, 0));
-    ANeuralNetworksMemory_createFromFd(size, PROT_READ | PROT_WRITE, fd_, 0,
-                                       &nn_memory_handle_);
-#endif
+    nnapi_->ANeuralNetworksMemory_createFromFd(size, PROT_READ | PROT_WRITE,
+                                               fd_, 0, &nn_memory_handle_);
   }
+#else
+  NNMemory(const NnApi* /*nnapi*/, const char* /*name*/, size_t /*size*/) {}
+#endif
 
   ~NNMemory() {
 #ifdef __ANDROID__
@@ -107,7 +89,7 @@ class NNMemory {
       munmap(data_ptr_, byte_size_);
     }
     if (nn_memory_handle_) {
-      ANeuralNetworksMemory_free(nn_memory_handle_);
+      nnapi_->ANeuralNetworksMemory_free(nn_memory_handle_);
     }
     if (fd_ > 0) close(fd_);
 #endif
@@ -118,6 +100,7 @@ class NNMemory {
 
  private:
 #ifdef __ANDROID__
+  const NnApi* nnapi_;
   int fd_ = 0;
   size_t byte_size_ = 0;
 #endif
@@ -167,9 +150,10 @@ class OperandMapping {
 // operands for both tensors and parameters, and TFLite separates the two.
 class NNAPIOpBuilder {
  public:
-  NNAPIOpBuilder(TfLiteContext* context, OperandMapping* tensor_mapping,
-                 ANeuralNetworksModel* nn_model)
-      : context_(context),
+  NNAPIOpBuilder(const NnApi* nnapi, TfLiteContext* context,
+                 OperandMapping* tensor_mapping, ANeuralNetworksModel* nn_model)
+      : nnapi_(nnapi),
+        context_(context),
         operand_mapping_(tensor_mapping),
         nn_model_(nn_model) {}
 
@@ -225,7 +209,8 @@ class NNAPIOpBuilder {
         .dimensionCount = dimension_count,
         .dimensions = dims.data()};
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_, ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
+        context_,
+        nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
     int ann_operand = operand_mapping_->add_new_non_tensor_operand();
     augmented_outputs_.push_back(ann_operand);
     return kTfLiteOk;
@@ -242,7 +227,8 @@ class NNAPIOpBuilder {
         reinterpret_cast<uint32_t*>(tensor->dims->data), tensor->params.scale,
         tensor->params.zero_point};
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_, ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
+        context_,
+        nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
     augmented_outputs_.push_back(ann_index);
 
     *ann_tensor_index_out = ann_index;
@@ -299,13 +285,14 @@ class NNAPIOpBuilder {
         nn_type, static_cast<uint32_t>(tensor->dims->size),
         reinterpret_cast<uint32_t*>(tensor->dims->data), scale, zeroPoint};
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_, ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
+        context_,
+        nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
 
     if (tensor->allocation_type == kTfLiteMmapRo) {
       // TODO(b/80630405): Use NNAPIAllocation.
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
           context_,
-          ANeuralNetworksModel_setOperandValue(
+          nnapi_->ANeuralNetworksModel_setOperandValue(
               nn_model_, ann_tensor_index, tensor->data.raw, tensor->bytes));
     }
 
@@ -318,7 +305,7 @@ class NNAPIOpBuilder {
     // Actually add a NN API operation
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
         context_,
-        ANeuralNetworksModel_addOperation(
+        nnapi_->ANeuralNetworksModel_addOperation(
             nn_model_, type, static_cast<uint32_t>(augmented_inputs_.size()),
             augmented_inputs_.data(),
             static_cast<uint32_t>(augmented_outputs_.size()),
@@ -333,11 +320,12 @@ class NNAPIOpBuilder {
   TfLiteStatus AddScalarOperand(T value, int32_t nn_type) {
     ANeuralNetworksOperandType operand_type{.type = nn_type};
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_, ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
+        context_,
+        nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
     int ann_operand = operand_mapping_->add_new_non_tensor_operand();
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_, ANeuralNetworksModel_setOperandValue(nn_model_, ann_operand,
-                                                       &value, sizeof(T)));
+        context_, nnapi_->ANeuralNetworksModel_setOperandValue(
+                      nn_model_, ann_operand, &value, sizeof(T)));
     augmented_inputs_.push_back(ann_operand);
     return kTfLiteOk;
   }
@@ -348,15 +336,19 @@ class NNAPIOpBuilder {
     ANeuralNetworksOperandType operand_type{
         .type = nn_type, .dimensionCount = 1, .dimensions = &num_values};
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_, ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
+        context_,
+        nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
     int ann_operand = operand_mapping_->add_new_non_tensor_operand();
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_, ANeuralNetworksModel_setOperandValue(
+        context_, nnapi_->ANeuralNetworksModel_setOperandValue(
                       nn_model_, ann_operand, values, sizeof(T) * num_values));
     augmented_inputs_.push_back(ann_operand);
     return kTfLiteOk;
   }
 
+  // Access to NNAPI.
+  const NnApi* const nnapi_;
+
   // TfLiteContext for error handling.
   TfLiteContext* const context_;
 
@@ -392,7 +384,7 @@ ANeuralNetworksOperationType BasicMappingFn(
 // The kernel that represents the node sub set of TF Lite being run on NN API.
 class NNAPIDelegateKernel {
  public:
-  NNAPIDelegateKernel() = default;
+  NNAPIDelegateKernel() { nnapi_ = NnApiImplementation(); }
 
   typedef ANeuralNetworksOperationType (*MappingFn)(
       const NNAPIOpMappingArgs& mapping_args);
@@ -401,7 +393,7 @@ class NNAPIDelegateKernel {
   // when called. You can use this function to see if a node is supported
   // (i.e. that MappingFn is not nullptr).
   static MappingFn Map(TfLiteContext* context, int builtin_code, int version,
-                       TfLiteNode* node) {
+                       int android_sdk_version, TfLiteNode* node) {
     switch (builtin_code) {
       case kTfLiteBuiltinAdd:
         if (version == 1) {
@@ -520,7 +512,7 @@ class NNAPIDelegateKernel {
         }
         break;
       case kTfLiteBuiltinSqueeze:
-        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11) {
+        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11) {
           return [](const NNAPIOpMappingArgs& mapping_args)
                      -> ANeuralNetworksOperationType {
             auto builtin = reinterpret_cast<TfLiteSqueezeParams*>(
@@ -636,7 +628,7 @@ class NNAPIDelegateKernel {
         }
         break;
       case kTfLiteBuiltinSub:
-        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11 &&
+        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11 &&
             context->tensors[node->inputs->data[0]].type == kTfLiteFloat32) {
           // NNAPI only support float sub.
           return [](const NNAPIOpMappingArgs& mapping_args)
@@ -649,7 +641,7 @@ class NNAPIDelegateKernel {
         }
         break;
       case kTfLiteBuiltinDiv:
-        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11 &&
+        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11 &&
             context->tensors[node->inputs->data[0]].type == kTfLiteFloat32) {
           // NNAPI only support float div.
           return [](const NNAPIOpMappingArgs& mapping_args)
@@ -662,7 +654,7 @@ class NNAPIDelegateKernel {
         }
         break;
       case kTfLiteBuiltinPad:
-        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11 &&
+        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11 &&
             node->inputs->size == 2 &&
             context->tensors[node->inputs->data[0]].type == kTfLiteFloat32) {
           // NNAPI does not support specifying the padding value.
@@ -672,12 +664,12 @@ class NNAPIDelegateKernel {
         }
         break;
       case kTfLiteBuiltinSpaceToBatchNd:
-        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11) {
+        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11) {
           return BasicMappingFn<ANEURALNETWORKS_SPACE_TO_BATCH_ND>;
         }
         break;
       case kTfLiteBuiltinStridedSlice:
-        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11) {
+        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11) {
           return [](const NNAPIOpMappingArgs& mapping_args)
                      -> ANeuralNetworksOperationType {
             auto builtin = reinterpret_cast<TfLiteStridedSliceParams*>(
@@ -695,7 +687,7 @@ class NNAPIDelegateKernel {
         // dimensions.
         // TODO(b/110888333): Support dynamically-sized tensors in delegates.
         if ((version == 1) &&
-            (kAndroidSdkVersion >= kMinSdkVersionForNNAPI11) &&
+            (android_sdk_version >= kMinSdkVersionForNNAPI11) &&
             (node->inputs->size > 1) &&
             (context->tensors[node->inputs->data[1]].allocation_type ==
              kTfLiteMmapRo)) {
@@ -793,7 +785,7 @@ class NNAPIDelegateKernel {
         break;
       case kTfLiteBuiltinMean:
         // NNAPI does not support generating a scalar as output for MEAN.
-        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11 &&
+        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11 &&
             context->tensors[node->inputs->data[0]].type == kTfLiteFloat32 &&
             context->tensors[node->outputs->data[0]].dims->size > 0) {
           return [](const NNAPIOpMappingArgs& mapping_args)
@@ -837,8 +829,8 @@ class NNAPIDelegateKernel {
 
     if (!nn_model_) {
       ANeuralNetworksModel* model;
-      RETURN_TFLITE_ERROR_IF_NN_ERROR(context,
-                                      ANeuralNetworksModel_create(&model));
+      RETURN_TFLITE_ERROR_IF_NN_ERROR(
+          context, nnapi_->ANeuralNetworksModel_create(&model));
       nn_model_.reset(model);
 
       TF_LITE_ENSURE_STATUS(
@@ -848,10 +840,10 @@ class NNAPIDelegateKernel {
     if (!nn_compilation_) {
       ANeuralNetworksCompilation* compilation;
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
-          context,
-          ANeuralNetworksCompilation_create(nn_model_.get(), &compilation));
+          context, nnapi_->ANeuralNetworksCompilation_create(nn_model_.get(),
+                                                             &compilation));
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
-          context, ANeuralNetworksCompilation_finish(compilation));
+          context, nnapi_->ANeuralNetworksCompilation_finish(compilation));
       nn_compilation_.reset(compilation);
     }
     return kTfLiteOk;
@@ -860,8 +852,8 @@ class NNAPIDelegateKernel {
   TfLiteStatus Invoke(TfLiteContext* context, TfLiteNode* node) {
     ANeuralNetworksExecution* execution = nullptr;
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context,
-        ANeuralNetworksExecution_create(nn_compilation_.get(), &execution));
+        context, nnapi_->ANeuralNetworksExecution_create(nn_compilation_.get(),
+                                                         &execution));
 
     // Set the input tensor buffers. Note: we access tflite tensors using
     // absolute indices but NN api indices inputs by relative indices.
@@ -881,7 +873,7 @@ class NNAPIDelegateKernel {
                tensor->data.raw, tensor->bytes);
         RETURN_TFLITE_ERROR_IF_NN_ERROR(
             context,
-            ANeuralNetworksExecution_setInputFromMemory(
+            nnapi_->ANeuralNetworksExecution_setInputFromMemory(
                 execution, relative_input_index, nullptr,
                 nn_input_memory_->get_handle(), input_offset, tensor->bytes));
         input_offset += tensor->bytes;
@@ -896,7 +888,7 @@ class NNAPIDelegateKernel {
       TfLiteTensor* tensor = &context->tensors[output_index];
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
           context,
-          ANeuralNetworksExecution_setOutputFromMemory(
+          nnapi_->ANeuralNetworksExecution_setOutputFromMemory(
               execution, relative_output_index, nullptr,
               nn_output_memory_->get_handle(), output_offset, tensor->bytes));
       output_offset += tensor->bytes;
@@ -912,7 +904,7 @@ class NNAPIDelegateKernel {
       // reading and writing into the same buffer during a invocation.
       // TODO(110369471): using double shared buffer to minimize the copies.
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
-          context, ANeuralNetworksExecution_setOutput(
+          context, nnapi_->ANeuralNetworksExecution_setOutput(
                        execution, relative_output_index, nullptr,
                        tensor->data.raw, tensor->bytes));
       relative_output_index++;
@@ -920,10 +912,12 @@ class NNAPIDelegateKernel {
     // Invoke ANN in blocking fashion.
     ANeuralNetworksEvent* event = nullptr;
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context, ANeuralNetworksExecution_startCompute(execution, &event));
-    RETURN_TFLITE_ERROR_IF_NN_ERROR(context, ANeuralNetworksEvent_wait(event));
-    ANeuralNetworksEvent_free(event);
-    ANeuralNetworksExecution_free(execution);
+        context,
+        nnapi_->ANeuralNetworksExecution_startCompute(execution, &event));
+    RETURN_TFLITE_ERROR_IF_NN_ERROR(context,
+                                    nnapi_->ANeuralNetworksEvent_wait(event));
+    nnapi_->ANeuralNetworksEvent_free(event);
+    nnapi_->ANeuralNetworksExecution_free(execution);
 
     // copy results from shared memory to the destination.
     output_offset = 0;
@@ -938,6 +932,8 @@ class NNAPIDelegateKernel {
   }
 
  private:
+  // Access to NNApi.
+  const NnApi* nnapi_;
   // ANN API state.
   std::unique_ptr<ANeuralNetworksModel, NNFreeModel> nn_model_;
   std::unique_ptr<ANeuralNetworksCompilation, NNFreeCompilation>
@@ -958,7 +954,7 @@ class NNAPIDelegateKernel {
     // The operand builder allows creating a single op. We create it at this
     // reduced power position rather than in the for loop to avoid reallocating
     // the vectors.
-    NNAPIOpBuilder builder(context, &operand_mapping_, nn_model_.get());
+    NNAPIOpBuilder builder(nnapi_, context, &operand_mapping_, nn_model_.get());
     // Add Tensors
     // allocate outside to avoid realloc
     for (auto node_index : nodes_) {
@@ -981,9 +977,10 @@ class NNAPIDelegateKernel {
         }
       }
       // Get op type and operands
-      int nn_op_type = Map(context, reg->builtin_code, reg->version, node)(
-          {context, &builder, node, &model_state_outputs_,
-           &model_state_tfl_inputs_});
+      int nn_op_type = Map(
+          context, reg->builtin_code, reg->version, nnapi_->android_sdk_version,
+          node)({context, &builder, node, &model_state_outputs_,
+                 &model_state_tfl_inputs_});
       // Map outputs to NN API tensor indices.
       for (auto output_index : TfLiteIntArrayView(node->outputs)) {
         TF_LITE_ENSURE_STATUS(builder.AddTensorOutput(output_index));
@@ -1029,25 +1026,27 @@ class NNAPIDelegateKernel {
 
     // Tell ANN to declare inputs/outputs
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context, ANeuralNetworksModel_identifyInputsAndOutputs(
+        context, nnapi_->ANeuralNetworksModel_identifyInputsAndOutputs(
                      nn_model_.get(), inputs.size(), inputs.data(),
                      outputs.size(), outputs.data()));
 
     // Set relaxed computation mode for fp32 if possible.
-    if (kAndroidSdkVersion >= kMinSdkVersionForNNAPI11) {
+    if (nnapi_->android_sdk_version >= kMinSdkVersionForNNAPI11) {
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
-          context, ANeuralNetworksModel_relaxComputationFloat32toFloat16(
-                       nn_model_.get(), context->allow_fp32_relax_to_fp16));
+          context,
+          nnapi_->ANeuralNetworksModel_relaxComputationFloat32toFloat16(
+              nn_model_.get(), context->allow_fp32_relax_to_fp16));
     }
 
     // Finalize the model
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context, ANeuralNetworksModel_finish(nn_model_.get()));
+        context, nnapi_->ANeuralNetworksModel_finish(nn_model_.get()));
 
     // Create shared memory pool for inputs and outputs.
-    nn_input_memory_.reset(new NNMemory("input_pool", total_input_byte_size));
+    nn_input_memory_.reset(
+        new NNMemory(nnapi_, "input_pool", total_input_byte_size));
     nn_output_memory_.reset(
-        new NNMemory("output_pool", total_output_byte_size));
+        new NNMemory(nnapi_, "output_pool", total_output_byte_size));
 
     return kTfLiteOk;
   }
@@ -1063,7 +1062,9 @@ TfLiteDelegate* NnApiDelegate() {
       .Prepare = [](TfLiteContext* context,
                     TfLiteDelegate* delegate) -> TfLiteStatus {
         // Do not check nodes_ if NN API is unavailable.
-        if (kAndroidSdkVersion < kMinSdkVersionForNNAPI || !NNAPIExists()) {
+        const NnApi* nnapi = NnApiImplementation();
+        if (nnapi->android_sdk_version < kMinSdkVersionForNNAPI ||
+            !nnapi->nnapi_exists) {
           return kTfLiteOk;
         }
 
@@ -1076,6 +1077,7 @@ TfLiteDelegate* NnApiDelegate() {
         TfLiteIntArray* plan;
         TF_LITE_ENSURE_STATUS(context->GetExecutionPlan(context, &plan));
 
+        int android_sdk_version = NnApiImplementation()->android_sdk_version;
         // Check for every node if it is supported
         // TODO(b/80625235): Fix this to do more careful checking of versioning.
         for (int node_index : TfLiteIntArrayView(plan)) {
@@ -1084,7 +1086,8 @@ TfLiteDelegate* NnApiDelegate() {
           TF_LITE_ENSURE_STATUS(context->GetNodeAndRegistration(
               context, node_index, &node, &registration));
           if (NNAPIDelegateKernel::Map(context, registration->builtin_code,
-                                       registration->version, node)) {
+                                       registration->version,
+                                       android_sdk_version, node)) {
             supported_nodes.push_back(node_index);
           }
         }
diff --git a/tensorflow/lite/nnapi/BUILD b/tensorflow/lite/nnapi/BUILD
index bd3a8a69af..172d0c8758 100644
--- a/tensorflow/lite/nnapi/BUILD
+++ b/tensorflow/lite/nnapi/BUILD
@@ -12,3 +12,26 @@ cc_library(
     ],
     linkopts = ["-ldl"],
 )
+
+cc_library(
+    name = "nnapi_implementation",
+    srcs = [
+        "nnapi_implementation.cc",
+    ],
+    hdrs = [
+        "nnapi_implementation.h",
+    ],
+    linkopts = ["-ldl"],
+    deps = [
+        "//tensorflow/lite/nnapi:nnapi_lib",
+    ],
+)
+
+cc_test(
+    name = "nnapi_implementation_test",
+    srcs = ["nnapi_implementation_test.cc"],
+    deps = [
+        "//tensorflow/lite/nnapi:nnapi_implementation",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
diff --git a/tensorflow/lite/nnapi/NeuralNetworksShim.h b/tensorflow/lite/nnapi/NeuralNetworksShim.h
index 2ce6e50de6..3a4e15006e 100644
--- a/tensorflow/lite/nnapi/NeuralNetworksShim.h
+++ b/tensorflow/lite/nnapi/NeuralNetworksShim.h
@@ -22,6 +22,11 @@ limitations under the License.
 
 #include "tensorflow/lite/nnapi/NeuralNetworksTypes.h"
 
+// This interface is now deprecated. You should use instead
+// nnapi_implementation.
+
+// TODO(b/123017568): Update all current usages of this file.
+
 // helpers
 
 #define NNAPI_LOG(format, ...) fprintf(stderr, format "\n", __VA_ARGS__);
diff --git a/tensorflow/lite/nnapi/nnapi_implementation.cc b/tensorflow/lite/nnapi/nnapi_implementation.cc
new file mode 100644
index 0000000000..5c96957f4a
--- /dev/null
+++ b/tensorflow/lite/nnapi/nnapi_implementation.cc
@@ -0,0 +1,137 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/nnapi/nnapi_implementation.h"
+
+#include <cstdlib>
+
+#ifdef __ANDROID__
+#include <dlfcn.h>
+#include <sys/mman.h>
+#include <sys/system_properties.h>
+#include <unistd.h>
+#endif
+
+#define NNAPI_LOG(format, ...) fprintf(stderr, format "\n", __VA_ARGS__);
+
+namespace {
+
+#ifdef __ANDROID__
+int32_t GetAndroidSdkVersion() {
+  const char* sdkProp = "ro.build.version.sdk";
+  char sdkVersion[PROP_VALUE_MAX];
+  int length = __system_property_get(sdkProp, sdkVersion);
+  if (length != 0) {
+    int32_t result = 0;
+    for (int i = 0; i < length; ++i) {
+      int digit = sdkVersion[i] - '0';
+      if (digit < 0 || digit > 9) {
+        // Non-numeric SDK version, assume it's higher than expected;
+        return 0xffff;
+      }
+      result = result * 10 + digit;
+    }
+    return result;
+  }
+  return 0;
+}
+
+void* LoadFunction(void* handle, const char* name) {
+  if (handle == nullptr) {
+    return nullptr;
+  }
+  void* fn = dlsym(handle, name);
+  if (fn == nullptr) {
+    NNAPI_LOG("nnapi error: unable to open function %s", name);
+  }
+  return fn;
+}
+
+#define LOAD_FUNCTION(handle, name) \
+  nnapi.name = reinterpret_cast<name##_fn>(LoadFunction(handle, #name));
+
+#else
+
+#define LOAD_FUNCTION(handle, name) nnapi.name = nullptr;
+
+#endif
+
+const NnApi LoadNnApi() {
+  NnApi nnapi = {};
+
+#ifdef __ANDROID__
+  void* libneuralnetworks = nullptr;
+  void* libandroid = nullptr;
+  nnapi.android_sdk_version = GetAndroidSdkVersion();
+  if (nnapi.android_sdk_version < 27) {
+    NNAPI_LOG("nnapi error: requires android sdk version to be at least %d",
+              27);
+  } else {
+    // TODO: change RTLD_LOCAL? Assumes there can be multiple instances of nn
+    // api RT
+    libneuralnetworks = dlopen("libneuralnetworks.so", RTLD_LAZY | RTLD_LOCAL);
+    if (libneuralnetworks == nullptr) {
+      NNAPI_LOG("nnapi error: unable to open library %s",
+                "libneuralnetworks.so");
+    }
+    libandroid = dlopen("libandroid.so", RTLD_LAZY | RTLD_LOCAL);
+    if (libandroid == nullptr) {
+      NNAPI_LOG("nnapi error: unable to open library %s", "libandroid.so");
+    }
+  }
+  nnapi.nnapi_exists = libneuralnetworks != nullptr;
+#else
+  nnapi.nnapi_exists = false;
+  nnapi.android_sdk_version = 0;
+#endif
+
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksMemory_createFromFd);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksMemory_free);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_create);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_free);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_finish);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_addOperand);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_setOperandValue);
+  LOAD_FUNCTION(libneuralnetworks,
+                ANeuralNetworksModel_setOperandValueFromMemory);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_addOperation);
+  LOAD_FUNCTION(libneuralnetworks,
+                ANeuralNetworksModel_identifyInputsAndOutputs);
+  LOAD_FUNCTION(libneuralnetworks,
+                ANeuralNetworksModel_relaxComputationFloat32toFloat16);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksCompilation_create);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksCompilation_free);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksCompilation_setPreference);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksCompilation_finish);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_create);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_free);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_setInput);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_setInputFromMemory);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_setOutput);
+  LOAD_FUNCTION(libneuralnetworks,
+                ANeuralNetworksExecution_setOutputFromMemory);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_startCompute);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksEvent_wait);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksEvent_free);
+  LOAD_FUNCTION(libandroid, ASharedMemory_create);
+
+  return nnapi;
+}
+
+}  // namespace
+
+const NnApi* NnApiImplementation() {
+  static const NnApi nnapi = LoadNnApi();
+  return &nnapi;
+}
diff --git a/tensorflow/lite/nnapi/nnapi_implementation.h b/tensorflow/lite/nnapi/nnapi_implementation.h
new file mode 100644
index 0000000000..82d7cc75c1
--- /dev/null
+++ b/tensorflow/lite/nnapi/nnapi_implementation.h
@@ -0,0 +1,582 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_NNAPI_NNAPI_IMPLEMENTATION_H_
+#define TENSORFLOW_LITE_NNAPI_NNAPI_IMPLEMENTATION_H_
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "tensorflow/lite/nnapi/NeuralNetworksTypes.h"
+
+struct NnApi {
+  bool nnapi_exists;
+  int32_t android_sdk_version;
+
+  /**
+   * Creates a shared memory object from a file descriptor.
+   *
+   * The shared memory is backed by a file descriptor via mmap.
+   * See {@link ANeuralNetworksMemory} for a description on how to use
+   * this shared memory.
+   *
+   * @param size The requested size in bytes.
+   *             Must not be larger than the file size.
+   * @param prot The desired memory protection for the mapping.
+   *             It is either PROT_NONE or the bitwise OR of one or
+   *             more of the following flags: PROT_READ, PROT_WRITE.
+   * @param fd The requested file descriptor.
+   *           The file descriptor has to be mmap-able. The file
+   *           descriptor will be duplicated.
+   * @param offset The offset to the beginning of the file of the area to map.
+   *               The offset has to be aligned to a page size.
+   * @param memory The memory object to be created.
+   *               Set to NULL if unsuccessful.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if the request completed normally.
+   */
+  int (*ANeuralNetworksMemory_createFromFd)(size_t size, int protect, int fd,
+                                            size_t offset,
+                                            ANeuralNetworksMemory** memory);
+
+  /**
+   * Delete a memory object.
+   *
+   * Destroys the object used by the run time to keep track of the memory.
+   * This will free the underlying actual memory if no other code has open
+   * handles to this memory.
+   *
+   * @param memory The memory object to be freed.
+   */
+  void (*ANeuralNetworksMemory_free)(ANeuralNetworksMemory* memory);
+
+  /**
+   * Create an empty {@link ANeuralNetworksModel}.
+   *
+   * <p>This only creates the object. Computation is performed once
+   * {@link ANeuralNetworksExecution_startCompute} is invoked.
+   *
+   * The model should be constructed with calls to
+   * {@link ANeuralNetworksModel_addOperation} and
+   * {@link ANeuralNetworksModel_addOperand}
+   *
+   * <p>{@link ANeuralNetworksModel_finish} should be called once the model
+   * has been fully constructed.</p>
+   *
+   * <p>{@link ANeuralNetworksModel_free} should be called once the model
+   * is no longer needed.</p>
+   *
+   * @param model The {@link ANeuralNetworksModel} to be created.
+   *              Set to NULL if unsuccessful.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   */
+  int (*ANeuralNetworksModel_create)(ANeuralNetworksModel** model);
+
+  /**
+   * Destroy a model.
+   *
+   * The model need not have been finished by a call to
+   * {@link ANeuralNetworksModel_finish}.
+   *
+   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+   *
+   * @param model The model to be destroyed. Passing NULL is acceptable and
+   *              results in no operation.
+   */
+  void (*ANeuralNetworksModel_free)(ANeuralNetworksModel* model);
+
+  /**
+   * Indicate that we have finished modifying a model. Required before
+   * calling {@link ANeuralNetworksCompilation_compile}.
+   *
+   * An application is responsible to make sure that no other thread uses
+   * the model at the same time.
+   *
+   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+   *
+   * @param model The model to be finished.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   */
+  int (*ANeuralNetworksModel_finish)(ANeuralNetworksModel* model);
+
+  /**
+   * Add an operand to a model.
+   *
+   * The order in which the operands are added is important. The first one added
+   * to a model will have the index value 0, the second 1, etc. These indexes
+   * are used as operand identifiers in
+   * {@link ANeuralNetworksModel_addOperation},
+   * {@link ANeuralNetworksExecution_setInput},
+   * {@link ANeuralNetworksExecution_setInputFromMemory},
+   * {@link ANeuralNetworksExecution_setOutput},
+   * {@link ANeuralNetworksExecution_setOutputFromMemory} and
+   * {@link ANeuralNetworksExecution_setOperandValue}.
+   *
+   * To build a model that can accommodate inputs of various sizes, as you may
+   * want to do for a CNN, set the size of the dimensions that will vary at run
+   * time to 0. If you do so, provide the full dimensions when calling
+   * {@link ANeuralNetworksExecution_setInput} or {@link
+   * ANeuralNetworksExecution_setInputFromMemory}.
+   *
+   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+   * been called will return an error.
+   *
+   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+   *
+   * @param model The model to be modified.
+   * @param type The {@link ANeuralNetworksOperandType} that describes the shape
+   * of the operand.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   */
+  int (*ANeuralNetworksModel_addOperand)(
+      ANeuralNetworksModel* model, const ANeuralNetworksOperandType* type);
+
+  /**
+   * Sets an operand to a constant value.
+   *
+   * For scalar values, the content of buffer is copied into the model.
+   *
+   * For tensor values, a pointer to the buffer is stored within the model.
+   * The application is responsible for not changing the content of this region
+   * until all executions using this model have completed. As the data may
+   * be copied during processing, modifying the data after this call yields
+   * undefined results.
+   *
+   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+   * been called will return an error.
+   *
+   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+   *
+   * @param model The model to be modified.
+   * @param index The index of the model operand we're setting.
+   * @param buffer A pointer to the data to use.
+   * @param length The size in bytes of the data value.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   */
+  int (*ANeuralNetworksModel_setOperandValue)(ANeuralNetworksModel* model,
+                                              int32_t index, const void* buffer,
+                                              size_t length);
+
+  /**
+   * Sets an operand to a value stored in a memory object.
+   *
+   * The content of the memory is not copied. A reference to that memory is
+   * stored inside the model. The application is responsible for not changing
+   * the content of the memory region until all executions using this model have
+   * completed.
+   * As the data may be copied during processing, modifying the data after this
+   * call yields undefined results.
+   *
+   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+   * been called will return an error.
+   *
+   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+   *
+   * @param model The model to be modified.
+   * @param index The index of the model operand we're setting.
+   * @param buffer A pointer to the data to use.
+   * @param memory The memory containing the data.
+   * @param offset This specifies the location of the data within the memory.
+   *               The offset is in bytes from the start of memory.
+   * @param length The size in bytes of the data value.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   */
+  int (*ANeuralNetworksModel_setOperandValueFromMemory)(
+      ANeuralNetworksModel* model, int32_t index,
+      const ANeuralNetworksMemory* memory, size_t offset, size_t length);
+
+  /**
+   * Add an operation to a model.
+   *
+   * @param model The model to be modified.
+   * @param type The type of the operation.
+   * @param inputCount The number of entries in the inputs array.
+   * @param inputs An array of indexes identifying each operand.
+   * @param outputCount The number of entries in the outputs array.
+   * @param outputs An array of indexes identifying each operand.
+   *
+   * The operands specified by inputs and outputs must have been
+   * previously added by calls to {@link ANeuralNetworksModel_addOperand}.
+   *
+   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+   * been called will return an error.
+   *
+   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   */
+  int (*ANeuralNetworksModel_addOperation)(ANeuralNetworksModel* model,
+                                           ANeuralNetworksOperationType type,
+                                           uint32_t inputCount,
+                                           const uint32_t* inputs,
+                                           uint32_t outputCount,
+                                           const uint32_t* outputs);
+
+  /**
+   * Specifies which operands will be the model's inputs and outputs.
+   *
+   * An operand cannot be used for both input and output. Doing so will
+   * return an error.
+   *
+   * @param model The model to be modified.
+   * @param inputCount The number of entries in the inputs array.
+   * @param inputs An array of indexes identifying the input operands.
+   * @param outputCount The number of entries in the outputs array.
+   * @param outputs An array of indexes identifying the output operands.
+   *
+   * The operands specified by inputs and outputs must have been
+   * previously added by calls to {@link ANeuralNetworksModel_addOperand}.
+   *
+   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+   * been called will return an error.
+   *
+   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+   *
+   */
+  int (*ANeuralNetworksModel_identifyInputsAndOutputs)(
+      ANeuralNetworksModel* model, uint32_t inputCount, const uint32_t* inputs,
+      uint32_t outputCount, const uint32_t* outputs);
+
+  /**
+   * Specifies whether {@link ANEURALNETWORKS_TENSOR_FLOAT32} is allowed to be
+   * calculated with range and/or precision as low as that of the
+   * IEEE 754 16-bit floating-point format. By default,
+   * {@link ANEURALNETWORKS_TENSOR_FLOAT32} must be calculated using at least
+   * the range and precision of the IEEE 754 32-bit floating-point format.
+   *
+   * @param model The model to be modified.
+   * @param allow 'true' indicates {@link ANEURALNETWORKS_TENSOR_FLOAT32} may be
+   *              calculated with range and/or precision as low as that of the
+   *              IEEE 754 16-bit floating point format. 'false' indicates
+   *              {@link ANEURALNETWORKS_TENSOR_FLOAT32} must be calculated
+   *              using at least the range and precision of the IEEE 754 32-bit
+   *              floating point format.
+   *
+   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+   * been called will return an error.
+   *
+   * Available since API level 28.
+   *
+   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+   */
+  int (*ANeuralNetworksModel_relaxComputationFloat32toFloat16)(
+      ANeuralNetworksModel* model, bool allow);
+
+  /**
+   * Create a {@link ANeuralNetworksCompilation} to compile the given model.
+   * This only creates the object. Compilation is only performed once
+   * {@link ANeuralNetworksCompilation_start} is invoked.
+   *
+   * <p>The provided model must outlive the compilation.</p>
+   *
+   * The model must already have been finished by a call to
+   * {@link ANeuralNetworksModel_finish}.
+   *
+   * See {@link ANeuralNetworksCompilation} for information on multithreaded
+   * usage.
+   *
+   * @param model The {@link ANeuralNetworksModel} to be compiled.
+   * @param compilation The newly created object or NULL if unsuccessful.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA
+   *         if the model is invalid.
+   */
+  int (*ANeuralNetworksCompilation_create)(
+      ANeuralNetworksModel* model, ANeuralNetworksCompilation** compilation);
+
+  /**
+   * Destroy a compilation.
+   *
+   * <p>If called on a compilation for which
+   * {@link ANeuralNetworksCompilation_start} has been called, the
+   * function will return immediately but will mark the compilation to be
+   * deleted once the compilation completes. The
+   * {@link ANeuralNetworksCompilation_wait} will return ERROR_DELETED.
+   *
+   * See {@link ANeuralNetworksCompilation} for information on multithreaded
+   * usage.
+   *
+   * @param compilation The compilation to be destroyed. Passing NULL is
+   * acceptable and results in no operation.
+   */
+  void (*ANeuralNetworksCompilation_free)(
+      ANeuralNetworksCompilation* compilation);
+
+  /**
+   * Sets the execution preference.
+   *
+   * <p>Provides guidance to the runtime when trade-offs are possible.</p>
+   *
+   * See {@link ANeuralNetworksCompilation} for information on multithreaded
+   * usage.
+   *
+   * @param compilation The compilation to be modified.
+   * @param preference Either {@link PREFER_LOW_POWER},
+   *                  {@link PREFER_SINGLE_FAST_ANSWER}, or
+   *                  {@link PREFER_SUSTAINED_SPEED}.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   */
+  int (*ANeuralNetworksCompilation_setPreference)(
+      ANeuralNetworksCompilation* compilation, int32_t preference);
+
+  /**
+   * Waits until the compilation completes.
+   *
+   * More than one thread can wait on a compilation. When the compilation
+   * completes, all threads will be released.
+   *
+   * See {@link ANeuralNetworksCompilation} for information on multithreaded
+   * usage.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if the compilation completed normally.
+   */
+  int (*ANeuralNetworksCompilation_finish)(
+      ANeuralNetworksCompilation* compilation);
+
+  /**
+   * Create a {@link ANeuralNetworksExecution} to apply the given compilation.
+   * This only creates the object. Computation is only performed once
+   * {@link ANeuralNetworksExecution_startCompute} is invoked.
+   *
+   * <p>The provided compilation must outlive the execution.</p>
+   *
+   * See {@link ANeuralNetworksExecution} for information on multithreaded
+   * usage.
+   *
+   * @param compilation The {@link ANeuralNetworksCompilation} to be evaluated.
+   * @param execution The newly created object or NULL if unsuccessful.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA
+   *         if the compilation is invalid.
+   */
+  int (*ANeuralNetworksExecution_create)(
+      ANeuralNetworksCompilation* compilation,
+      ANeuralNetworksExecution** execution);
+
+  /**
+   * Destroy an execution.
+   *
+   * <p>If called on an execution for which
+   * {@link ANeuralNetworksExecution_startCompute} has been called, the
+   * function will return immediately but will mark the execution to be deleted
+   * once the computation completes.   The {link ANeuralNetworksExecution_wait}
+   * will return ANEURALNETWORKS_ERROR_DELETED.
+   *
+   * See {@link ANeuralNetworksExecution} for information on multithreaded
+   * usage.
+   *
+   * @param execution The execution to be destroyed. Passing NULL is acceptable
+   * and results in no operation.
+   */
+  void (*ANeuralNetworksExecution_free)(ANeuralNetworksExecution* execution);
+
+  /**
+   * Associate a user buffer with an input of the model of the
+   * {@link ANeuralNetworksExecution}.
+   *
+   * <p>The provided buffer must outlive the execution.</p>
+   *
+   * See {@link ANeuralNetworksExecution} for information on multithreaded
+   * usage.
+   *
+   * @param execution The execution to be modified.
+   * @param index The index of the input argument we are setting. It is
+   *              an index into the lists passed to
+   *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is
+   *              not the index associated with {@link
+   * ANeuralNetworksModel_addOperand}.
+   * @param type The type of the operand. This should be used to specify the
+   *             dimensions that were set to 0 when the operand was added to the
+   *             model. All other properties of the type must be the same as
+   *             specified in the model. If the type is the same as specified
+   *             when the model was built, NULL can be passed.
+   * @param buffer The buffer containing the data.
+   * @param length The length in bytes of the buffer.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
+   * the name is not recognized or the buffer is too small for the input.
+   */
+  int (*ANeuralNetworksExecution_setInput)(
+      ANeuralNetworksExecution* execution, int32_t index,
+      const ANeuralNetworksOperandType* type, const void* buffer,
+      size_t length);
+
+  /**
+   * Associate part of a memory object with an input of the model of the
+   * {@link ANeuralNetworksExecution}.
+   *
+   * <p>The provided memory must outlive the execution.</p>
+   *
+   * See {@link ANeuralNetworksExecution} for information on multithreaded
+   * usage.
+   *
+   * @param execution The execution to be modified.
+   * @param index The index of the input argument we are setting. It is
+   *              an index into the lists passed to
+   *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is
+   *              not the index associated with {@link
+   * ANeuralNetworksModel_addOperand}.
+   * @param type The type of the operand. This can be used to specify the
+   *             dimensions that were set to 0 when the operand was added to the
+   *             model. All other values must be the same as specified in the
+   *             model. If the type is the same as specified when the model
+   *             was built, NULL can be passed.
+   * @param memory The memory containing the data.
+   * @param offset This specifies the location of the data within the memory.
+   *               The offset is in bytes from the start of memory.
+   * @param length The size in bytes of the data value.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
+   * the name is not recognized or the buffer is too small for the input.
+   */
+  int (*ANeuralNetworksExecution_setInputFromMemory)(
+      ANeuralNetworksExecution* execution, int32_t index,
+      const ANeuralNetworksOperandType* type,
+      const ANeuralNetworksMemory* memory, size_t offset, size_t length);
+
+  /**
+   * Associate a user buffer with an output of the model of the
+   * {@link ANeuralNetworksExecution}.
+   *
+   * <p>The provided buffer must outlive the execution.</p>
+   *
+   * See {@link ANeuralNetworksExecution} for information on multithreaded
+   * usage.
+   *
+   * @param execution The execution to be modified.
+   * @param index The index of the output argument we are setting. It is
+   *              an index into the lists passed to
+   *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is
+   *              not the index associated with {@link
+   * ANeuralNetworksModel_addOperand}.
+   * @param type The type of the operand. This can be used to specify the
+   *             dimensions that were set to 0 when the operand was added to the
+   *             model. All other values must be the same as specified in the
+   *             model. If the type is the same as specified when the model
+   *             was built, NULL can be passed.
+   * @param buffer The buffer where the data is to be written.
+   * @param length The length in bytes of the buffer.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
+   * the name is not recognized or the buffer is too small for the output.
+   */
+  int (*ANeuralNetworksExecution_setOutput)(
+      ANeuralNetworksExecution* execution, int32_t index,
+      const ANeuralNetworksOperandType* type, void* buffer, size_t length);
+
+  /**
+   * Associate part of a memory object with an output of the model of the
+   * {@link ANeuralNetworksExecution}.
+   *
+   * <p>The provided memory must outlive the execution.</p>
+   *
+   * See {@link ANeuralNetworksExecution} for information on multithreaded
+   * usage.
+   *
+   * @param execution The execution to be modified.
+   * @param index The index of the output argument we are setting. It is
+   *              an index into the lists passed to
+   *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is
+   *              not the index associated with {@link
+   * ANeuralNetworksModel_addOperand}.
+   * @param type The type of the operand. This can be used to specify the
+   *             dimensions that were set to 0 when the operand was added to the
+   *             model. All other values must be the same as specified in the
+   *             model. If the type is the same as specified when the model
+   *             was built, NULL can be passed.
+   * @param memory The memory where the data is to be stored.
+   * @param offset This specifies the location of the data within the memory.
+   *               The offset is in bytes from the start of memory.
+   * @param length The length in bytes of the data value.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
+   * the name is not recognized or the buffer is too small for the output.
+   */
+  int (*ANeuralNetworksExecution_setOutputFromMemory)(
+      ANeuralNetworksExecution* execution, int32_t index,
+      const ANeuralNetworksOperandType* type,
+      const ANeuralNetworksMemory* memory, size_t offset, size_t length);
+
+  /**
+   * Schedule evaluation of the execution.
+   *
+   * <p>Schedules evaluation of the execution. Once the model has been
+   * applied and the outputs are ready to be consumed, the execution will be
+   * signaled. Use {@link ANeuralNetworksExecution_wait} to wait for that
+   * signal.
+   * </p>
+   *
+   * Multiple executions can be scheduled and evaluated concurrently, and
+   * compilations can be performed concurrently with executions. The runtime
+   * makes no guarantee on the ordering of the completion of compilations and
+   * executions. If it's important to the application, the application should
+   * enforce the ordering by using {@link ANeuralNetworksCompilation_wait} and
+   * {@link ANeuralNetworksExecution_wait}.
+   *
+   * ANeuralNetworksExecution_wait must be called to recuperate the resources
+   * used by the execution.
+   *
+   * See {@link ANeuralNetworksExecution} for information on multithreaded
+   * usage.
+   *
+   * @param execution The execution to be scheduled and executed.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   */
+  int (*ANeuralNetworksExecution_startCompute)(
+      ANeuralNetworksExecution* execution, ANeuralNetworksEvent** event);
+
+  /**
+   * Waits until the execution completes.
+   *
+   * More than one thread can wait on an event. When the execution completes,
+   * all threads will be released.
+   *
+   * See {@link ANeuralNetworksExecution} for information on multithreaded
+   * usage.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if the execution completed normally.
+   */
+  int (*ANeuralNetworksEvent_wait)(ANeuralNetworksEvent* event);
+
+  /**
+   * Destroys the event.
+   *
+   * See {@link ANeuralNetworksExecution} for information on multithreaded
+   * usage.
+   */
+  void (*ANeuralNetworksEvent_free)(ANeuralNetworksEvent* event);
+
+  // ASharedMemory_create was added in Android 8.0, so safe to use with NNAPI
+  // which was added in 8.1.
+  int (*ASharedMemory_create)(const char* name, size_t size);
+
+  /**/
+};
+
+/**
+ * Load the NNAPI implementation from the shared libraries.
+ * The NnApi structure is filled with all the pointers. If one function doesn't
+ * exist, a null pointer is stored.
+ */
+const NnApi* NnApiImplementation();
+
+#endif  // TENSORFLOW_LITE_NNAPI_NNAPI_IMPLEMENTATION_H_
diff --git a/tensorflow/lite/nnapi/nnapi_implementation_test.cc b/tensorflow/lite/nnapi/nnapi_implementation_test.cc
new file mode 100644
index 0000000000..35de769bc8
--- /dev/null
+++ b/tensorflow/lite/nnapi/nnapi_implementation_test.cc
@@ -0,0 +1,120 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/nnapi/nnapi_implementation.h"
+#include <gtest/gtest.h>
+
+namespace {
+
+TEST(NnapiLibTest, NnApiImplementation) {
+  const NnApi* nnapi = NnApiImplementation();
+  EXPECT_NE(nnapi, nullptr);
+#ifdef __ANDROID__
+  EXPECT_GT(nnapi->android_sdk_version, 0);
+  if (nnapi.android_sdk_version < 27) {
+    EXPECT_FALSE(nnapi->nnapi_exists);
+    EXPECT_EQ(nnapi->ANeuralNetworksMemory_createFromFd, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksMemory_free, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksModel_create, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksModel_free, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksModel_finish, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksModel_addOperand, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksModel_setOperandValue, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksModel_setOperandValueFromMemory, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksModel_addOperation, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksModel_identifyInputsAndOutputs, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksModel_relaxComputationFloat32toFloat16,
+              nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksCompilation_create, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksCompilation_free, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksCompilation_setPreference, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksCompilation_finish, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksExecution_create, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksExecution_free, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksExecution_setInput, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksExecution_setInputFromMemory, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksExecution_setOutput, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksExecution_setOutputFromMemory, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksExecution_startCompute, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksEvent_wait, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksEvent_free, nullptr);
+    EXPECT_EQ(nnapi->ASharedMemory_create, nullptr);
+  } else {
+    EXPECT_TRUE(nnapi->nnapi_exists);
+    EXPECT_NE(nnapi->ANeuralNetworksMemory_createFromFd, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksMemory_free, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksModel_create, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksModel_free, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksModel_finish, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksModel_addOperand, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksModel_setOperandValue, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksModel_setOperandValueFromMemory, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksModel_addOperation, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksModel_identifyInputsAndOutputs, nullptr);
+    if (nnapi->android_sdk_version >= 28) {
+      // relaxComputationFloat32toFloat16 only available with Android 9.0 (P).
+      EXPECT_NE(nnapi->ANeuralNetworksModel_relaxComputationFloat32toFloat16,
+                nullptr);
+    } else {
+      EXPECT_EQ(nnapi->ANeuralNetworksModel_relaxComputationFloat32toFloat16,
+                nullptr);
+    }
+    EXPECT_NE(nnapi->ANeuralNetworksCompilation_create, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksCompilation_free, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksCompilation_setPreference, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksCompilation_finish, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksExecution_create, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksExecution_free, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksExecution_setInput, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksExecution_setInputFromMemory, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksExecution_setOutput, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksExecution_setOutputFromMemory, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksExecution_startCompute, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksEvent_wait, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksEvent_free, nullptr);
+    EXPECT_NE(nnapi->ASharedMemory_create, nullptr);
+  }
+#else
+  EXPECT_FALSE(nnapi->nnapi_exists);
+  EXPECT_EQ(nnapi->android_sdk_version, 0);
+  EXPECT_EQ(nnapi->ANeuralNetworksMemory_createFromFd, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksMemory_free, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksModel_create, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksModel_free, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksModel_finish, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksModel_addOperand, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksModel_setOperandValue, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksModel_setOperandValueFromMemory, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksModel_addOperation, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksModel_identifyInputsAndOutputs, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksModel_relaxComputationFloat32toFloat16,
+            nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksCompilation_create, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksCompilation_free, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksCompilation_setPreference, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksCompilation_finish, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksExecution_create, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksExecution_free, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksExecution_setInput, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksExecution_setInputFromMemory, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksExecution_setOutput, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksExecution_setOutputFromMemory, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksExecution_startCompute, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksEvent_wait, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksEvent_free, nullptr);
+  EXPECT_EQ(nnapi->ASharedMemory_create, nullptr);
+#endif
+}
+
+}  // namespace
diff --git a/tensorflow/lite/nnapi_delegate.cc b/tensorflow/lite/nnapi_delegate.cc
index be035fe60c..5a3a21258a 100644
--- a/tensorflow/lite/nnapi_delegate.cc
+++ b/tensorflow/lite/nnapi_delegate.cc
@@ -21,7 +21,7 @@ limitations under the License.
 #include "tensorflow/lite/c/builtin_op_data.h"
 #include "tensorflow/lite/core/api/error_reporter.h"
 #include "tensorflow/lite/model.h"
-#include "tensorflow/lite/nnapi/NeuralNetworksShim.h"
+#include "tensorflow/lite/nnapi/nnapi_implementation.h"
 
 #ifdef __ANDROID__
 #include <android/log.h>
@@ -84,56 +84,27 @@ void logError(const char* format, ...) {
 static const int64_t kOperandIdNotSet = -1;
 static const int64_t kOperandNotNeeded = -2;
 
-namespace {
-
-int32_t GetAndroidSdkVersion() {
-#ifdef __ANDROID__
-  const char* sdkProp = "ro.build.version.sdk";
-  char sdkVersion[PROP_VALUE_MAX];
-  int length = __system_property_get(sdkProp, sdkVersion);
-  if (length != 0) {
-    for (int i = 0; i < length; ++i) {
-      int digit = sdkVersion[i] - '0';
-      if (digit < 0 || digit > 9) {
-        // Non-numeric SDK version, assume it's higher then expected;
-        return 0xFFFF;
-      }
-    }
-    return atoi(sdkVersion);
-  }
-  FATAL("No %s prop", sdkProp);
-#endif  // __ANDROID__
-  return 0;
-}
-
-int32_t GetAndroidSdkVersionCached() {
-  static int32_t androidSdkVersion = GetAndroidSdkVersion();
-  return androidSdkVersion;
-}
-
-}  // namespace
-
 NNAPIAllocation::NNAPIAllocation(const char* filename,
                                  ErrorReporter* error_reporter)
     : MMAPAllocation(filename, error_reporter) {
   if (mmapped_buffer_ != MAP_FAILED)
-    CHECK_NN(ANeuralNetworksMemory_createFromFd(buffer_size_bytes_, PROT_READ,
-                                                mmap_fd_, 0, &handle_));
+    CHECK_NN(NnApiImplementation()->ANeuralNetworksMemory_createFromFd(
+        buffer_size_bytes_, PROT_READ, mmap_fd_, 0, &handle_));
 }
 
 NNAPIAllocation::~NNAPIAllocation() {
   if (handle_) {
-    ANeuralNetworksMemory_free(handle_);
+    NnApiImplementation()->ANeuralNetworksMemory_free(handle_);
   }
 }
 
 NNAPIDelegate::~NNAPIDelegate() {
   if (nn_compiled_model_) {
-    ANeuralNetworksCompilation_free(nn_compiled_model_);
+    NnApiImplementation()->ANeuralNetworksCompilation_free(nn_compiled_model_);
     nn_compiled_model_ = nullptr;
   }
   if (nn_model_) {
-    ANeuralNetworksModel_free(nn_model_);
+    NnApiImplementation()->ANeuralNetworksModel_free(nn_model_);
     nn_model_ = nullptr;
     // TODO(aselle): Is this thread-safe and callable multiple times?
   }
@@ -145,6 +116,7 @@ TfLiteStatus addTensorOperands(tflite::Subgraph* subgraph,
                                ANeuralNetworksModel* nn_model,
                                uint32_t* no_of_operands_added,
                                std::vector<int64_t>* nnapi_ids) {
+  const NnApi* nnapi = NnApiImplementation();
   uint32_t next_id = 0;
   for (size_t i = 0; i < subgraph->tensors_size(); i++) {
     // Skip temporaries and RNN back-edges.
@@ -198,24 +170,24 @@ TfLiteStatus addTensorOperands(tflite::Subgraph* subgraph,
         nn_type, static_cast<uint32_t>(tensor->dims->size),
         reinterpret_cast<uint32_t*>(tensor->dims->data), scale, zeroPoint};
     RETURN_ERROR_IF_NN_FAILED(
-        ANeuralNetworksModel_addOperand(nn_model, &operand_type));
+        nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type));
     // TODO(aselle): Based on Michael's suggestion, limiting this to read
     // only memory
     if (tensor->allocation_type == kTfLiteMmapRo) {
       if (const NNAPIAllocation* alloc = dynamic_cast<const NNAPIAllocation*>(
               static_cast<const Allocation*>(tensor->allocation))) {
         RETURN_ERROR_IF_NN_FAILED(
-            ANeuralNetworksModel_setOperandValueFromMemory(
+            nnapi->ANeuralNetworksModel_setOperandValueFromMemory(
                 nn_model, next_id, alloc->memory(),
                 alloc->offset(tensor->data.raw), tensor->bytes));
       } else {
-        RETURN_ERROR_IF_NN_FAILED(ANeuralNetworksModel_setOperandValue(
+        RETURN_ERROR_IF_NN_FAILED(nnapi->ANeuralNetworksModel_setOperandValue(
             nn_model, next_id, tensor->data.raw, tensor->bytes));
       }
     } else if (tensor->bytes == 0) {
       // These size 0 tensors are optional tensors reserved.
-      RETURN_ERROR_IF_NN_FAILED(
-          ANeuralNetworksModel_setOperandValue(nn_model, next_id, nullptr, 0));
+      RETURN_ERROR_IF_NN_FAILED(nnapi->ANeuralNetworksModel_setOperandValue(
+          nn_model, next_id, nullptr, 0));
     }
 
     ++next_id;
@@ -244,6 +216,7 @@ TfLiteStatus AddOpsAndParams(
     uint32_t next_id, std::vector<int>* model_state_inputs,
     std::vector<int>* model_state_outputs,
     const std::vector<int64_t>& tensor_id_to_nnapi_id) {
+  const NnApi* nnapi = NnApiImplementation();
   for (size_t i = 0; i < subgraph->nodes_size(); i++) {
     const auto* node_and_registration = subgraph->node_and_registration(i);
     const TfLiteNode& node = node_and_registration->first;
@@ -258,21 +231,21 @@ TfLiteStatus AddOpsAndParams(
     MapAndAddTensorIds(node.outputs->data, node.outputs->size,
                        &augmented_outputs, tensor_id_to_nnapi_id);
 
-    auto add_scalar_int32 = [&nn_model, &augmented_inputs,
+    auto add_scalar_int32 = [nnapi, &nn_model, &augmented_inputs,
                              &next_id](int value) {
       ANeuralNetworksOperandType operand_type{.type = ANEURALNETWORKS_INT32};
-      CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type))
-      CHECK_NN(ANeuralNetworksModel_setOperandValue(nn_model, next_id, &value,
-                                                    sizeof(int32_t)))
+      CHECK_NN(nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type))
+      CHECK_NN(nnapi->ANeuralNetworksModel_setOperandValue(
+          nn_model, next_id, &value, sizeof(int32_t)))
       augmented_inputs.push_back(next_id++);
     };
 
-    auto add_scalar_float32 = [&nn_model, &augmented_inputs,
+    auto add_scalar_float32 = [nnapi, &nn_model, &augmented_inputs,
                                &next_id](float value) {
       ANeuralNetworksOperandType operand_type{.type = ANEURALNETWORKS_FLOAT32};
-      CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type))
-      CHECK_NN(ANeuralNetworksModel_setOperandValue(nn_model, next_id, &value,
-                                                    sizeof(float)))
+      CHECK_NN(nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type))
+      CHECK_NN(nnapi->ANeuralNetworksModel_setOperandValue(
+          nn_model, next_id, &value, sizeof(float)))
       augmented_inputs.push_back(next_id++);
     };
 
@@ -281,8 +254,8 @@ TfLiteStatus AddOpsAndParams(
           .type = ANEURALNETWORKS_TENSOR_INT32,
           .dimensionCount = 1,
           .dimensions = &num_values};
-      CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type))
-      CHECK_NN(ANeuralNetworksModel_setOperandValue(
+      CHECK_NN(nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type))
+      CHECK_NN(nnapi->ANeuralNetworksModel_setOperandValue(
           nn_model, next_id, values, sizeof(int32_t) * num_values));
       augmented_inputs.push_back(next_id++);
     };
@@ -291,15 +264,16 @@ TfLiteStatus AddOpsAndParams(
     // For each state_out tensor, a corresponding state_in operand needs to be
     // created for NNAPI.
     auto duplicate_state_tensor_float32 =
-        [subgraph, &nn_model, &next_id, &augmented_inputs, &model_state_inputs,
-         &model_state_outputs](int tensor_id) {
+        [nnapi, subgraph, &nn_model, &next_id, &augmented_inputs,
+         &model_state_inputs, &model_state_outputs](int tensor_id) {
           const TfLiteTensor* tensor = subgraph->tensor(tensor_id);
           ANeuralNetworksOperandType operand_type{
               ANEURALNETWORKS_TENSOR_FLOAT32,
               static_cast<uint32_t>(tensor->dims->size),
               reinterpret_cast<uint32_t*>(tensor->dims->data),
               tensor->params.scale, tensor->params.zero_point};
-          CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type));
+          CHECK_NN(
+              nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type));
           augmented_inputs.push_back(next_id);
           model_state_inputs->push_back(next_id);
           model_state_outputs->push_back(tensor_id);
@@ -388,7 +362,7 @@ TfLiteStatus AddOpsAndParams(
     };
 
     // LSTM in NNAPI requires scratch tensor as an output operand.
-    auto add_lstm_scratch_tensor_float32 = [subgraph, &node, &nn_model,
+    auto add_lstm_scratch_tensor_float32 = [nnapi, subgraph, &node, &nn_model,
                                             &next_id, &augmented_outputs]() {
       if (node.temporaries->size == 0) return;
       int scratch_buffer_index = node.temporaries->data[0];
@@ -398,7 +372,7 @@ TfLiteStatus AddOpsAndParams(
           static_cast<uint32_t>(tensor->dims->size),
           reinterpret_cast<uint32_t*>(tensor->dims->data), tensor->params.scale,
           tensor->params.zero_point};
-      CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type));
+      CHECK_NN(nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type));
       augmented_outputs.insert(augmented_outputs.begin(), next_id++);
     };
 
@@ -427,15 +401,16 @@ TfLiteStatus AddOpsAndParams(
     };
 
     // Handle optional input tensors.
-    auto add_optional_tensors = [&nn_model, &augmented_inputs,
+    auto add_optional_tensors = [nnapi, &nn_model, &augmented_inputs,
                                  &next_id](int nn_type) {
       for (size_t idx = 0; idx < augmented_inputs.size(); idx++) {
         if (augmented_inputs[idx] == kOptionalTensor) {
           const std::vector<uint32_t> dim = {0, 0};
           ANeuralNetworksOperandType operand_type{nn_type, 2, dim.data(), 0, 0};
-          CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type))
-          CHECK_NN(ANeuralNetworksModel_setOperandValue(nn_model, next_id,
-                                                        nullptr, 0))
+          CHECK_NN(
+              nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type))
+          CHECK_NN(nnapi->ANeuralNetworksModel_setOperandValue(
+              nn_model, next_id, nullptr, 0))
           augmented_inputs[idx] = next_id++;
         }
       }
@@ -697,13 +672,13 @@ TfLiteStatus AddOpsAndParams(
         break;
     }
 
-    if (nnapi_version == 11 && GetAndroidSdkVersionCached() < 28) {
+    if (nnapi_version == 11 && nnapi->android_sdk_version < 28) {
       logError("Op %d needs NNAPI1.1", builtin);
       return kTfLiteError;
     }
 
     // Add the operation.
-    RETURN_ERROR_IF_NN_FAILED(ANeuralNetworksModel_addOperation(
+    RETURN_ERROR_IF_NN_FAILED(nnapi->ANeuralNetworksModel_addOperation(
         nn_model, nn_op_type, static_cast<uint32_t>(augmented_inputs.size()),
         augmented_inputs.data(),
         static_cast<uint32_t>(augmented_outputs.size()),
@@ -715,9 +690,10 @@ TfLiteStatus AddOpsAndParams(
 TfLiteStatus NNAPIDelegate::BuildGraph(Subgraph* subgraph) {
   if (nn_model_ && nn_compiled_model_) return model_status_;
 
+  const NnApi* nnapi = NnApiImplementation();
   // TODO(aselle): This is not correct. need to handle resize invalidation.
   if (!nn_model_) {
-    CHECK_NN(ANeuralNetworksModel_create(&nn_model_));
+    CHECK_NN(nnapi->ANeuralNetworksModel_create(&nn_model_));
 
     // Find which tensors should be added to NNAPI. TFLite has temporaries
     // and RNN back-edges which are are not valid for NNAPI. We look through all
@@ -764,21 +740,22 @@ TfLiteStatus NNAPIDelegate::BuildGraph(Subgraph* subgraph) {
                        model_states_outputs_.size(), &augmented_outputs,
                        tensor_id_to_nnapi_id);
 
-    CHECK_NN(ANeuralNetworksModel_identifyInputsAndOutputs(
+    CHECK_NN(nnapi->ANeuralNetworksModel_identifyInputsAndOutputs(
         nn_model_, static_cast<uint32_t>(augmented_inputs.size()),
         reinterpret_cast<const uint32_t*>(augmented_inputs.data()),
         static_cast<uint32_t>(augmented_outputs.size()),
         reinterpret_cast<const uint32_t*>(augmented_outputs.data())));
 
-    if (GetAndroidSdkVersionCached() >= 28) {
-      CHECK_NN(ANeuralNetworksModel_relaxComputationFloat32toFloat16(
+    if (nnapi->android_sdk_version >= 28) {
+      CHECK_NN(nnapi->ANeuralNetworksModel_relaxComputationFloat32toFloat16(
           nn_model_, subgraph->GetAllowFp16PrecisionForFp32()));
     }
-    CHECK_NN(ANeuralNetworksModel_finish(nn_model_));
+    CHECK_NN(nnapi->ANeuralNetworksModel_finish(nn_model_));
   }
   if (!nn_compiled_model_) {
-    CHECK_NN(ANeuralNetworksCompilation_create(nn_model_, &nn_compiled_model_));
-    CHECK_NN(ANeuralNetworksCompilation_finish(nn_compiled_model_));
+    CHECK_NN(nnapi->ANeuralNetworksCompilation_create(nn_model_,
+                                                      &nn_compiled_model_));
+    CHECK_NN(nnapi->ANeuralNetworksCompilation_finish(nn_compiled_model_));
   }
   return kTfLiteOk;
 }
@@ -794,8 +771,10 @@ TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
     return model_status_;
   }
 
+  const NnApi* nnapi = NnApiImplementation();
   ANeuralNetworksExecution* execution = nullptr;
-  CHECK_NN(ANeuralNetworksExecution_create(nn_compiled_model_, &execution));
+  CHECK_NN(
+      nnapi->ANeuralNetworksExecution_create(nn_compiled_model_, &execution));
 
   // Currently perform deep copy of input buffer
   for (size_t i = 0; i < subgraph->inputs().size(); i++) {
@@ -803,7 +782,7 @@ TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
     // TODO(aselle): Is this what we want or do we want input instead?
     // TODO(aselle): This should be called setInputValue maybe to be cons.
     TfLiteTensor* tensor = subgraph->tensor(input);
-    CHECK_NN(ANeuralNetworksExecution_setInput(
+    CHECK_NN(nnapi->ANeuralNetworksExecution_setInput(
         execution, i, nullptr, tensor->data.raw, tensor->bytes));
   }
 
@@ -811,7 +790,7 @@ TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
   for (size_t i = 0; i < subgraph->outputs().size(); i++) {
     int output = subgraph->outputs()[i];
     TfLiteTensor* tensor = subgraph->tensor(output);
-    CHECK_NN(ANeuralNetworksExecution_setOutput(
+    CHECK_NN(nnapi->ANeuralNetworksExecution_setOutput(
         execution, i, nullptr, tensor->data.raw, tensor->bytes));
   }
 
@@ -823,21 +802,21 @@ TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
     // Here we are using a deep copy for state_in tensors so that we are not
     // reading and writing into the same buffer during a invocation.
     // TODO(miaowang): using double shared buffer to minimize the copies.
-    CHECK_NN(ANeuralNetworksExecution_setInput(
+    CHECK_NN(nnapi->ANeuralNetworksExecution_setInput(
         execution, i + subgraph->inputs().size(), nullptr, tensor->data.raw,
         tensor->bytes));
     // Tell NNAPI where to output the state_out.
-    CHECK_NN(ANeuralNetworksExecution_setOutput(
+    CHECK_NN(nnapi->ANeuralNetworksExecution_setOutput(
         execution, i + subgraph->outputs().size(), nullptr, tensor->data.raw,
         tensor->bytes));
   }
 
   // Currently use blocking compute.
   ANeuralNetworksEvent* event = nullptr;
-  CHECK_NN(ANeuralNetworksExecution_startCompute(execution, &event));
-  CHECK_NN(ANeuralNetworksEvent_wait(event));
-  ANeuralNetworksEvent_free(event);
-  ANeuralNetworksExecution_free(execution);
+  CHECK_NN(nnapi->ANeuralNetworksExecution_startCompute(execution, &event));
+  CHECK_NN(nnapi->ANeuralNetworksEvent_wait(event));
+  nnapi->ANeuralNetworksEvent_free(event);
+  nnapi->ANeuralNetworksExecution_free(execution);
 
 #if 0
   printf("From the NN API:\n");
@@ -855,6 +834,8 @@ TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
   return kTfLiteOk;
 }
 
-bool NNAPIDelegate::IsSupported() { return NNAPIExists(); }
+bool NNAPIDelegate::IsSupported() {
+  return NnApiImplementation()->nnapi_exists;
+}
 
 }  // namespace tflite
diff --git a/tensorflow/lite/testing/BUILD b/tensorflow/lite/testing/BUILD
index ce85a39385..3840b52a0a 100644
--- a/tensorflow/lite/testing/BUILD
+++ b/tensorflow/lite/testing/BUILD
@@ -237,7 +237,7 @@ tf_cc_binary(
     deps = [
         ":parse_testdata_lib",
         ":tflite_driver",
-        "//tensorflow/lite/nnapi:nnapi_lib",
+        "//tensorflow/lite/nnapi:nnapi_implementation",
     ],
 )
 
diff --git a/tensorflow/lite/testing/nnapi_example.cc b/tensorflow/lite/testing/nnapi_example.cc
index 0ff1fbcd41..309cb19628 100644
--- a/tensorflow/lite/testing/nnapi_example.cc
+++ b/tensorflow/lite/testing/nnapi_example.cc
@@ -26,7 +26,7 @@ limitations under the License.
 #include <iostream>
 #include <sstream>
 #include <string>
-#include "tensorflow/lite/nnapi/NeuralNetworksShim.h"
+#include "tensorflow/lite/nnapi/nnapi_implementation.h"
 #include "tensorflow/lite/testing/parse_testdata.h"
 #include "tensorflow/lite/testing/tflite_driver.h"
 
-- 
GitLab


From 4665e7096e19f18eae8b1830a0d4c1c0ffc8d46e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 10:57:00 -0800
Subject: [PATCH 0969/2345] Add some kernel-specific DepthwiseConv profiling
 labels.

PiperOrigin-RevId: 229957162
---
 .../lite/kernels/internal/optimized/depthwiseconv_uint8.h      | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8.h b/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8.h
index d3dca799a7..0f4226dd15 100644
--- a/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8.h
+++ b/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8.h
@@ -1983,6 +1983,7 @@ inline void DepthwiseConv(
           input_shape, filter_shape, stride_width, stride_height,
           dilation_width_factor, dilation_height_factor, pad_width, pad_height,
           depth_multiplier, output_shape, output_shift)) {
+    gemmlowp::ScopedProfilingLabel specialized_label("DepthwiseConv/8bit/3x3");
     DepthwiseConv3x3Filter(params, input_shape, input_data, filter_shape,
                            filter_data, bias_shape, bias_data, output_shape,
                            output_data);
@@ -1990,6 +1991,8 @@ inline void DepthwiseConv(
   }
 #endif
 
+  gemmlowp::ScopedProfilingLabel specialized_label(
+      "DepthwiseConv/8bit/General");
   DepthwiseConvGeneral(params, input_shape, input_data, filter_shape,
                        filter_data, bias_shape, bias_data, output_shape,
                        output_data);
-- 
GitLab


From cfe98499e525da4f17e8cac494d5d128ecde4da2 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Fri, 18 Jan 2019 11:02:54 -0800
Subject: [PATCH 0970/2345] Fix build

PiperOrigin-RevId: 229958434
---
 tensorflow/tools/compatibility/renames_v2.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 6695eb4883..9d8d25016a 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -169,6 +169,7 @@ renames = {
     'tf.distributions.Uniform': 'tf.compat.v1.distributions.Uniform',
     'tf.distributions.kl_divergence': 'tf.compat.v1.distributions.kl_divergence',
     'tf.div': 'tf.compat.v1.div',
+    'tf.dtypes.as_string': 'tf.strings.as_string',
     'tf.enable_eager_execution': 'tf.compat.v1.enable_eager_execution',
     'tf.enable_resource_variables': 'tf.compat.v1.enable_resource_variables',
     'tf.enable_v2_batch_normalization': 'tf.compat.v1.enable_v2_batch_normalization',
-- 
GitLab


From 691f6c23580c62960ffd59ca27afcf36d075946d Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Fri, 18 Jan 2019 11:09:54 -0800
Subject: [PATCH 0971/2345] Disable smartreply tests by default.

PiperOrigin-RevId: 229959758
---
 tensorflow/lite/models/smartreply/BUILD                       | 4 ++++
 tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/tensorflow/lite/models/smartreply/BUILD b/tensorflow/lite/models/smartreply/BUILD
index 9b2f0da64c..5be2aaff1f 100644
--- a/tensorflow/lite/models/smartreply/BUILD
+++ b/tensorflow/lite/models/smartreply/BUILD
@@ -57,6 +57,7 @@ tf_cc_test(
         "//tensorflow/lite/models:testdata/smartreply_samples.tsv",
         "@tflite_smartreply//:smartreply.tflite",
     ],
+    tags = ["no_oss"],
     deps = [
         ":predictor_lib",
         "//tensorflow/core:test",
@@ -71,6 +72,7 @@ tf_cc_test(
     name = "extract_feature_op_test",
     size = "small",
     srcs = ["ops/extract_feature_test.cc"],
+    tags = ["no_oss"],
     deps = [
         ":custom_ops",
         "//tensorflow/lite:framework",
@@ -85,6 +87,7 @@ tf_cc_test(
     name = "normalize_op_test",
     size = "small",
     srcs = ["ops/normalize_test.cc"],
+    tags = ["no_oss"],
     deps = [
         ":custom_ops",
         "//tensorflow/lite:framework",
@@ -99,6 +102,7 @@ tf_cc_test(
     name = "predict_op_test",
     size = "small",
     srcs = ["ops/predict_test.cc"],
+    tags = ["no_oss"],
     deps = [
         ":custom_ops",
         "//tensorflow/lite:framework",
diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
index 2eeb8dfa1a..d33b732cc5 100644
--- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
@@ -125,6 +125,10 @@ fi
 
 run_configure_for_cpu_build
 
+bazel build --announce_rc --config=opt ${EXTRA_BUILD_FLAGS}  \
+  --build_tag_filters=-no_pip,-no_windows,-no_oss,-gpu \
+  tensorflow/lite:framework tensorflow/lite/examples/minimal:minimal || exit $?
+
 bazel build --announce_rc --config=opt ${EXTRA_BUILD_FLAGS} \
   tensorflow/tools/pip_package:build_pip_package \
   --incompatible_remove_native_http_archive=false || exit $?
-- 
GitLab


From 7fce32e9be8c43a96f0c55063be6ab215d12b0ae Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Fri, 18 Jan 2019 11:24:02 -0800
Subject: [PATCH 0972/2345] [TF] Add version_script option to tf_py_wrap_cc
 Bazel rule. [XLA] Hide all symbols except PyInit_* from the pywrap_xla
 module.

The XLA Python extension includes parts of TensorFlow core. If the TensorFlow symbols are exported by the XLA extension as well as TensorFlow itself, a segfault occurs if both tensorflow and jax are imported into the same Python interpreter.

JAX issue https://github.com/google/jax/issues/120 will be fixed when this change is integrated into the JAX repository.

PiperOrigin-RevId: 229962366
---
 tensorflow/compiler/xla/python/BUILD                 |  5 +++++
 .../xla/python/pywrap_xla_exported_symbols.lds       |  1 +
 .../xla/python/pywrap_xla_version_script.lds         |  6 ++++++
 tensorflow/tensorflow.bzl                            | 12 ++++++++----
 tensorflow/tf_exported_symbols.lds                   |  1 -
 tensorflow/tf_version_script.lds                     |  1 -
 6 files changed, 20 insertions(+), 6 deletions(-)
 create mode 100644 tensorflow/compiler/xla/python/pywrap_xla_exported_symbols.lds
 create mode 100644 tensorflow/compiler/xla/python/pywrap_xla_version_script.lds

diff --git a/tensorflow/compiler/xla/python/BUILD b/tensorflow/compiler/xla/python/BUILD
index ddffafa901..8276ee5a32 100644
--- a/tensorflow/compiler/xla/python/BUILD
+++ b/tensorflow/compiler/xla/python/BUILD
@@ -98,6 +98,11 @@ tf_py_wrap_cc(
         "local_computation_builder.i",
         "//tensorflow/python:platform/base.i",
     ],
+    version_script = select({
+        "//tensorflow:darwin": "pywrap_xla_exported_symbols.lds",
+        "//tensorflow:windows": None,
+        "//conditions:default": "pywrap_xla_version_script.lds",
+    }),
     deps = [
         ":local_computation_builder",
         ":numpy_bridge",
diff --git a/tensorflow/compiler/xla/python/pywrap_xla_exported_symbols.lds b/tensorflow/compiler/xla/python/pywrap_xla_exported_symbols.lds
new file mode 100644
index 0000000000..bce6c1acf8
--- /dev/null
+++ b/tensorflow/compiler/xla/python/pywrap_xla_exported_symbols.lds
@@ -0,0 +1 @@
+_PyInit__pywrap_xla
diff --git a/tensorflow/compiler/xla/python/pywrap_xla_version_script.lds b/tensorflow/compiler/xla/python/pywrap_xla_version_script.lds
new file mode 100644
index 0000000000..d31cfce7be
--- /dev/null
+++ b/tensorflow/compiler/xla/python/pywrap_xla_version_script.lds
@@ -0,0 +1,6 @@
+xla {
+  global:
+    PyInit_*;
+  local:
+    *;
+};
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index 37d52cdb39..874d9e872c 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -1649,7 +1649,9 @@ def tf_py_wrap_cc(
         swig_includes = [],
         deps = [],
         copts = [],
+        version_script = None,
         **kwargs):
+    """Builds a Python extension module."""
     module_name = name.split("/")[-1]
 
     # Convert a rule name such as foo/bar/baz to foo/bar/_baz.so
@@ -1668,6 +1670,11 @@ def tf_py_wrap_cc(
         toolchain_deps = ["@bazel_tools//tools/cpp:current_cc_toolchain"],
         deps = deps + extra_deps,
     )
+    if not version_script:
+        version_script = select({
+            "@local_config_cuda//cuda:darwin": clean_dep("//tensorflow:tf_exported_symbols.lds"),
+            "//conditions:default": clean_dep("//tensorflow:tf_version_script.lds"),
+        })
     vscriptname = name + "_versionscript"
     _append_init_to_versionscript(
         name = vscriptname,
@@ -1676,10 +1683,7 @@ def tf_py_wrap_cc(
             "//conditions:default": True,
         }),
         module_name = module_name,
-        template_file = select({
-            "@local_config_cuda//cuda:darwin": clean_dep("//tensorflow:tf_exported_symbols.lds"),
-            "//conditions:default": clean_dep("//tensorflow:tf_version_script.lds"),
-        }),
+        template_file = version_script,
     )
     extra_linkopts = select({
         "@local_config_cuda//cuda:darwin": [
diff --git a/tensorflow/tf_exported_symbols.lds b/tensorflow/tf_exported_symbols.lds
index 9f6114f503..04632330c5 100644
--- a/tensorflow/tf_exported_symbols.lds
+++ b/tensorflow/tf_exported_symbols.lds
@@ -4,5 +4,4 @@
 *TF_*
 *TFE_*
 *nsync_*
-*pywrap_xla*
 *stream_executor*
diff --git a/tensorflow/tf_version_script.lds b/tensorflow/tf_version_script.lds
index 39d258c3b7..563d178de7 100644
--- a/tensorflow/tf_version_script.lds
+++ b/tensorflow/tf_version_script.lds
@@ -5,7 +5,6 @@ tensorflow {
     *TF_*;
     *TFE_*;
     *nsync_*;
-    *pywrap_xla*;
     *stream_executor*;
   local:
     *;
-- 
GitLab


From b58fa42c78e1441e1230b7c03ffc98a1f389493d Mon Sep 17 00:00:00 2001
From: Edward Loper <edloper@google.com>
Date: Fri, 18 Jan 2019 11:40:42 -0800
Subject: [PATCH 0973/2345] Fix CompositeDelegation handling for
 control_flow_ops such as _Enter.

PiperOrigin-RevId: 229965591
---
 .../kernel_tests/control_flow_ops_py_test.py  | 15 ++++++++++++++
 tensorflow/python/ops/control_flow_ops.py     | 20 +++++++++++++++++++
 2 files changed, 35 insertions(+)

diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index d9d7af1f0d..90d77050e9 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -3760,6 +3760,21 @@ class ControlFlowTest(test.TestCase):
       result = func(qint)
       self.evaluate(result)
 
+  def testSparseIdentity(self):
+    st1 = sparse_tensor.SparseTensor([[0, 5]], ['x'], [10, 10])
+    st2 = control_flow_ops._Identity(st1)
+    self.assertAllEqual(st1.indices, st2.indices)
+    self.assertAllEqual(st1.values, st2.values)
+    self.assertAllEqual(st1.dense_shape, st2.dense_shape)
+
+  def testSparseEnterExit(self):
+    st1 = sparse_tensor.SparseTensor([[0, 5]], ['x'], [10, 10])
+    st2 = control_flow_ops._Enter(st1, "foo_1")
+    st3 = control_flow_ops.exit(st2)
+    self.assertAllEqual(st1.indices, st3.indices)
+    self.assertAllEqual(st1.values, st3.values)
+    self.assertAllEqual(st1.dense_shape, st3.dense_shape)
+
 
 class ControlFlowContextCheckTest(test.TestCase):
 
diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py
index 2bc3a9e9b6..cfdbe63c2f 100644
--- a/tensorflow/python/ops/control_flow_ops.py
+++ b/tensorflow/python/ops/control_flow_ops.py
@@ -187,6 +187,8 @@ def _Identity(data, name=None):
       return gen_array_ops.ref_identity(data, name=name)
     else:
       return array_ops.identity(data, name=name)
+  elif isinstance(data, composite_tensor.CompositeTensor):
+    return nest.map_structure(_Identity, data, expand_composites=True)
   else:
     raise TypeError("Type %s not supported" % type(data))
 
@@ -198,6 +200,8 @@ def _NextIteration(data, name=None):
       return ref_next_iteration(data, name=name)
     else:
       return next_iteration(data, name=name)
+  elif isinstance(data, composite_tensor.CompositeTensor):
+    return nest.map_structure(_NextIteration, data, expand_composites=True)
   else:
     raise TypeError("Type %s not supported" % type(data))
 
@@ -239,6 +243,11 @@ def _Enter(data,
     if use_input_shape:
       result.set_shape(data.get_shape())
     return result
+  elif isinstance(data, composite_tensor.CompositeTensor):
+    def enter_component(t):
+      return _Enter(t, frame_name, is_constant, parallel_iterations,
+                    use_ref, use_input_shape)
+    return nest.map_structure(enter_component, data, expand_composites=True)
   else:
     raise TypeError("Type %s not supported" % type(data))
 
@@ -261,6 +270,8 @@ def exit(data, name=None):  # pylint: disable=redefined-builtin
       return gen_control_flow_ops.ref_exit(data, name)
     else:
       return gen_control_flow_ops._exit(data, name)
+  elif isinstance(data, composite_tensor.CompositeTensor):
+    return nest.map_structure(exit, data, expand_composites=True)
   else:
     raise TypeError("Type %s not supported" % type(data))
 
@@ -548,6 +559,15 @@ def _AddNextAndBackEdge(m, v, enforce_shape_invariant=True):
       # TODO(skyewm): call this for other cases below (needs testing)
       _EnforceShapeInvariant(m, v)
     m.op._update_input(1, v)  # pylint: disable=protected-access
+  elif isinstance(m, composite_tensor.CompositeTensor):
+    # pylint: disable=protected-access
+    def update_component(m_component, v_component):
+      m_component.op._update_input(1, v_component)
+    if isinstance(m, ops.IndexedSlices):
+      v = math_ops._as_indexed_slices(v, optimize=False)
+    # pylint: enable=protected-access
+    v = _NextIteration(v)
+    return nest.map_structure(update_component, m, v, expand_composites=True)
   else:
     raise TypeError("Type %s not supported" % type(m))
   return v
-- 
GitLab


From 216851bb33c348bd8b33973b450d9b4be06815d7 Mon Sep 17 00:00:00 2001
From: Yanan Cao <ycao@google.com>
Date: Fri, 18 Jan 2019 11:44:34 -0800
Subject: [PATCH 0974/2345] Add back a missing arg documentation for
 validate_indices in tf.gather

PiperOrigin-RevId: 229966279
---
 tensorflow/python/ops/array_ops.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 014fdd25cc..b9ff2f9717 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -3266,7 +3266,7 @@ def gather(params,
            validate_indices=None,
            name=None,
            axis=None,
-           batch_dims=0):  # pylint: disable=g-doc-args
+           batch_dims=0):
   r"""Gather slices from params axis axis according to indices.
 
   Gather slices from params axis axis according to indices.  `indices` must be
@@ -3320,13 +3320,14 @@ def gather(params,
       `axis + 1`.
     indices: The index `Tensor`.  Must be one of the following types: `int32`,
       `int64`. Must be in range `[0, params.shape[axis])`.
+    validate_indices: Deprecated, does nothing.
+    name: A name for the operation (optional).
     axis: A `Tensor`. Must be one of the following types: `int32`, `int64`. The
       `axis` in `params` to gather `indices` from. Must be greater than or equal
       to `batch_dims`.  Defaults to the first non-batch dimension. Supports
       negative indexes.
     batch_dims: An `integer`.  The number of batch dimensions.  Must be less
       than `ndims(inices)`.
-    name: A name for the operation (optional).
 
   Returns:
     A `Tensor`. Has the same type as `params`.
-- 
GitLab


From 45679c956cda0d8ffa28834fb917559057fdde86 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 11:48:41 -0800
Subject: [PATCH 0975/2345] Add keras correctness test for stateful LSTM models
 with DeviceAssignment.

PiperOrigin-RevId: 229966996
---
 tensorflow/contrib/distribute/python/BUILD    | 22 +++++
 .../contrib/distribute/python/combinations.py | 22 ++++-
 .../python/keras_correctness_test_base.py     | 23 +++--
 ...as_stateful_lstm_model_correctness_test.py | 99 +++++++++++++++++++
 tensorflow/python/keras/engine/training.py    | 14 +++
 5 files changed, 171 insertions(+), 9 deletions(-)
 create mode 100644 tensorflow/contrib/distribute/python/keras_stateful_lstm_model_correctness_test.py

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 6ae3ec7fb0..7a77dbe29c 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -651,6 +651,7 @@ py_library(
         "keras_embedding_model_correctness_test.py",
         "keras_image_model_correctness_test.py",
         "keras_lstm_model_correctness_test.py",
+        "keras_stateful_lstm_model_correctness_test.py",
     ],
     deps = [
         ":combinations",
@@ -750,6 +751,27 @@ distribute_py_test(
     ],
 )
 
+distribute_py_test(
+    name = "keras_stateful_lstm_model_correctness_test",
+    size = "medium",
+    srcs = ["keras_stateful_lstm_model_correctness_test.py"],
+    full_precision = True,
+    main = "keras_stateful_lstm_model_correctness_test.py",
+    # Shard count is set to an odd number to distribute tasks across
+    # shards more evenly.
+    shard_count = 31,
+    tags = [
+        "multi_and_single_gpu",
+        "no_oss",  # TODO(b/117919883): Fix python error.
+        "no_pip",
+        "no_windows_gpu",
+        "notsan",
+    ],
+    deps = [
+        ":keras_correctness_test_lib",
+    ],
+)
+
 distribute_py_test(
     name = "metrics_v1_test",
     srcs = ["metrics_v1_test.py"],
diff --git a/tensorflow/contrib/distribute/python/combinations.py b/tensorflow/contrib/distribute/python/combinations.py
index 304762e69e..88170d4085 100644
--- a/tensorflow/contrib/distribute/python/combinations.py
+++ b/tensorflow/contrib/distribute/python/combinations.py
@@ -53,6 +53,7 @@ from tensorflow.contrib.distribute.python import tpu_strategy as tpu_lib
 from tensorflow.contrib.optimizer_v2 import adagrad as adagrad_v2
 from tensorflow.contrib.optimizer_v2 import adam as adam_v2
 from tensorflow.contrib.optimizer_v2 import gradient_descent as gradient_descent_v2
+from tensorflow.contrib.tpu.python.tpu import device_assignment as device_assignment_lib
 from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
@@ -321,11 +322,18 @@ class NamedDistribution(object):
     return self._required_tpu
 
 
-def _get_tpu_strategy_creator(steps_per_run, **kwargs):
+def _get_tpu_strategy_creator(steps_per_run, use_single_core=False, **kwargs):
   def _create_tpu_strategy():
     resolver = cluster_resolver.TPUClusterResolver("")
-    tpu_lib.initialize_tpu_system(resolver)
+    topology = tpu_lib.initialize_tpu_system(resolver)
+    device_assignment = None
+    if use_single_core:
+      device_assignment = device_assignment_lib.DeviceAssignment(
+          topology, core_assignment=device_assignment_lib.
+          SINGLE_CORE_ASSIGNMENT)
+
     strategy = tpu_lib.TPUStrategy(resolver, steps_per_run=steps_per_run,
+                                   device_assignment=device_assignment,
                                    **kwargs)
     return strategy
   return _create_tpu_strategy
@@ -345,6 +353,16 @@ tpu_strategy = NamedDistribution(
 tpu_strategy_one_step = NamedDistribution(
     "TPUOneStep", _get_tpu_strategy_creator(steps_per_run=1),
     required_tpu=True)
+tpu_strategy_loop_on_device_one_core = NamedDistribution(
+    "TPULoopOnDeviceOneCore", _get_tpu_strategy_creator(
+        steps_per_run=2, use_single_core=True,
+        _disable_training_loop_on_host=True),
+    required_tpu=True)
+tpu_strategy_one_step_loop_on_device_one_core = NamedDistribution(
+    "TPUOneStepLoopOnDeviceOneCore", _get_tpu_strategy_creator(
+        steps_per_run=1, use_single_core=True,
+        _disable_training_loop_on_host=True),
+    required_tpu=True)
 # TODO(b/122327153): Remove below two NamedDistributions.
 tpu_strategy_loop_on_device = NamedDistribution(
     "TPULoopOnDevice", _get_tpu_strategy_creator(
diff --git a/tensorflow/contrib/distribute/python/keras_correctness_test_base.py b/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
index 08ed933f29..1ceb95cdd6 100644
--- a/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
+++ b/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
@@ -207,8 +207,8 @@ def get_correctness_test_inputs(use_numpy, use_validation_data,
   return training_inputs, eval_inputs, predict_inputs
 
 
-def fit_eval_and_predict(
-    initial_weights, input_fn, model_fn, distribution=None):
+def fit_eval_and_predict(initial_weights, input_fn, model_fn,
+                         distribution=None, is_stateful_model=False):
   """Generates results for fit/predict/evaluate for given model."""
   model = model_fn(initial_weights=initial_weights, distribution=distribution)
   training_inputs, eval_inputs, predict_inputs = input_fn(distribution)
@@ -222,7 +222,15 @@ def fit_eval_and_predict(
   result['weights_1'] = model.get_weights()
 
   if predict_inputs is not None:
-    result['predict_result_1'] = model.predict(**predict_inputs)
+    # Check correctness of the result of predict() invoked
+    # multiple times -- as for stateful models, result of
+    # predict may differ for each batch.
+    predict_length = 1
+    if is_stateful_model:
+      predict_length = 3
+    for i in range(predict_length):
+      result_key = 'predict_result_{}'.format(i)
+      result[result_key] = model.predict(**predict_inputs)
 
   # Train and eval again to mimic user's flow.
 
@@ -334,7 +342,8 @@ class TestDistributionStrategyCorrectnessBase(test.TestCase,
                            distribution,
                            use_numpy,
                            use_validation_data,
-                           with_batch_norm=False):
+                           with_batch_norm=False,
+                           is_stateful_model=False):
     with self.cached_session():
       self.set_up_test_config(use_numpy, use_validation_data, with_batch_norm)
       self.skip_unsupported_test_configuration(distribution)
@@ -355,10 +364,10 @@ class TestDistributionStrategyCorrectnessBase(test.TestCase,
 
       results_with_ds = fit_eval_and_predict(
           initial_weights, input_fn=input_fn, model_fn=self.get_model,
-          distribution=distribution)
+          distribution=distribution, is_stateful_model=is_stateful_model)
       results_without_ds = fit_eval_and_predict(
           initial_weights, input_fn=input_fn, model_fn=self.get_model,
-          distribution=None)
+          distribution=None, is_stateful_model=is_stateful_model)
 
       # First, special case, for multi-replica distributed training, batch norm
       # is not aggregated globally. So it is expected to have different weights.
@@ -448,7 +457,7 @@ class TestDistributionStrategyEmbeddingModelCorrectnessBase(
         features, maxlen=max_words)
     x_train = np.asarray(features, dtype=np.float32)
     y_train = np.asarray(labels, dtype=np.int32).reshape((count, 1))
-    x_predict = x_train
+    x_predict = x_train[:_GLOBAL_BATCH_SIZE]
     return x_train, y_train, x_predict
 
 
diff --git a/tensorflow/contrib/distribute/python/keras_stateful_lstm_model_correctness_test.py b/tensorflow/contrib/distribute/python/keras_stateful_lstm_model_correctness_test.py
new file mode 100644
index 0000000000..54eb07e100
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/keras_stateful_lstm_model_correctness_test.py
@@ -0,0 +1,99 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for stateful tf.keras LSTM models using DistributionStrategy."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.distribute.python import combinations
+from tensorflow.contrib.distribute.python import keras_correctness_test_base
+from tensorflow.python import keras
+from tensorflow.python.eager import test
+from tensorflow.python.training import gradient_descent
+
+
+def strategies_for_stateful_embedding_model():
+  """Returns TPUStrategy with single core device assignment."""
+
+  return [combinations.tpu_strategy_loop_on_device_one_core,
+          combinations.tpu_strategy_one_step_loop_on_device_one_core]
+
+
+def test_combinations_for_stateful_embedding_model():
+  return (
+      combinations.combine(
+          distribution=strategies_for_stateful_embedding_model(),
+          mode='graph',
+          use_numpy=False,
+          use_validation_data=False
+      ))
+
+
+class DistributionStrategyStatefulLstmModelCorrectnessTest(
+    keras_correctness_test_base.
+    TestDistributionStrategyEmbeddingModelCorrectnessBase):
+
+  def get_model(self, max_words=10, initial_weights=None, distribution=None):
+    batch_size = keras_correctness_test_base._GLOBAL_BATCH_SIZE
+
+    with keras_correctness_test_base.MaybeDistributionScope(distribution):
+      word_ids = keras.layers.Input(
+          shape=(max_words,),
+          batch_size=batch_size,
+          dtype=np.int32, name='words')
+      word_embed = keras.layers.Embedding(input_dim=20,
+                                          output_dim=10)(word_ids)
+      lstm_embed = keras.layers.LSTM(units=4,
+                                     return_sequences=False,
+                                     stateful=True)(word_embed)
+
+      preds = keras.layers.Dense(2, activation='softmax')(lstm_embed)
+      model = keras.Model(inputs=[word_ids], outputs=[preds])
+
+      if initial_weights:
+        model.set_weights(initial_weights)
+
+      model.compile(
+          optimizer=gradient_descent.GradientDescentOptimizer(
+              learning_rate=0.1),
+          loss='sparse_categorical_crossentropy',
+          metrics=['sparse_categorical_accuracy'])
+    return model
+
+  @combinations.generate(test_combinations_for_stateful_embedding_model())
+  def test_stateful_lstm_model_correctness(self,
+                                           distribution,
+                                           use_numpy,
+                                           use_validation_data):
+    self.run_correctness_test(distribution, use_numpy, use_validation_data,
+                              is_stateful_model=True)
+
+  @combinations.generate(keras_correctness_test_base.
+                         test_combinations_for_embedding_model())
+  def test_incorrectly_use_multiple_cores_for_stateful_lstm_model(
+      self, distribution, use_numpy, use_validation_data):
+    with self.assertRaisesRegexp(ValueError,
+                                 'Single core must be used for computation '
+                                 'on stateful models. Consider adding '
+                                 '`device_assignment` parameter to '
+                                 'TPUStrategy'):
+      self.run_correctness_test(distribution, use_numpy, use_validation_data,
+                                is_stateful_model=True)
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 53caac25c9..366a96e7a6 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -2214,6 +2214,20 @@ class Model(Network):
       raise NotImplementedError('`sample_weight` is currently not supported '
                                 'when using TPUStrategy.')
 
+    if (self.stateful and distributed_training_utils.is_tpu_strategy(
+        self._distribution_strategy) and self._distribution_strategy.
+        num_replicas_in_sync != 1):
+      raise ValueError('Single core must be used for computation on '
+                       'stateful models. Consider adding `device_assignment` '
+                       'parameter to TPUStrategy using\n'
+                       'topology = tf.contrib.distribute.'
+                       'initialize_tpu_system()\n'
+                       'device_assignment = tf.contrib.tpu.DeviceAssignment('
+                       'topology, core_assignment=tf.contrib.tpu.'
+                       'SINGLE_CORE_ASSIGNMENT)\n'
+                       'tpu_strategy = tf.contrib.distribute.TPUStrategy('
+                       'device_assignment=device_assignment)')
+
     # Validates `steps` and `shuffle` arguments right at the beginning
     # since we use it to construct the dataset object.
     # TODO(anjalisridhar): Remove this check once we refactor the
-- 
GitLab


From b3c326376263f8e1af0d44f29ac1dc6106f699de Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Fri, 18 Jan 2019 11:49:00 -0800
Subject: [PATCH 0976/2345] Fix kokoro test failure by adding necessary
 protobuf dependencies.

PiperOrigin-RevId: 229967055
---
 tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index fa6acdb782..3bd4ff94d1 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -36,6 +36,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
+#include "tensorflow/core/platform/protobuf.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/protobuf/config.pb.h"  // NOLINT
 #include "tensorflow/core/public/session.h"
@@ -1123,7 +1124,7 @@ class OpConverterTest : public ::testing::Test {
 };
 
 template <typename T>
-void CopyTensorElements(const Tensor& tensor, proto2::RepeatedField<T>* out) {
+void CopyTensorElements(const Tensor& tensor, protobuf::RepeatedField<T>* out) {
   out->Clear();
   if (tensor.NumElements() == 0) return;
 
-- 
GitLab


From a206216c854e23d77bd727f039fd194ad831185f Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Fri, 18 Jan 2019 12:06:21 -0800
Subject: [PATCH 0977/2345] Make converter output filenames in logs.

PiperOrigin-RevId: 229970291
---
 tensorflow/tools/compatibility/ast_edits.py   | 11 +++---
 .../tools/compatibility/tf_upgrade_v2_main.py | 34 ++++++++++++++-----
 2 files changed, 32 insertions(+), 13 deletions(-)

diff --git a/tensorflow/tools/compatibility/ast_edits.py b/tensorflow/tools/compatibility/ast_edits.py
index 940be4c23c..17573e4f43 100644
--- a/tensorflow/tools/compatibility/ast_edits.py
+++ b/tensorflow/tools/compatibility/ast_edits.py
@@ -509,7 +509,8 @@ class ASTCodeUpgrader(object):
       in_place: Allow the conversion of an entire directory in place.
 
     Returns:
-      A tuple of files processed, the report string ofr all files, and errors
+      A tuple of files processed, the report string ofr all files, and a dict
+        mapping filenames to errors encountered in that file.
     """
 
     if output_root_directory == root_directory:
@@ -556,7 +557,7 @@ class ASTCodeUpgrader(object):
           files_to_copy.append((fullpath, fullpath_output))
 
     file_count = 0
-    tree_errors = []
+    tree_errors = {}
     report = ""
     report += ("=" * 80) + "\n"
     report += "Input tree: %r\n" % root_directory
@@ -568,7 +569,7 @@ class ASTCodeUpgrader(object):
         os.makedirs(output_directory)
       file_count += 1
       _, l_report, l_errors = self.process_file(input_path, output_path)
-      tree_errors += l_errors
+      tree_errors[input_path] = l_errors
       report += l_report
     for input_path, output_path in files_to_copy:
       output_directory = os.path.dirname(output_path)
@@ -586,7 +587,7 @@ class ASTCodeUpgrader(object):
       files_to_process += py_files
 
     file_count = 0
-    tree_errors = []
+    tree_errors = {}
     report = ""
     report += ("=" * 80) + "\n"
     report += "Input tree: %r\n" % root_directory
@@ -595,7 +596,7 @@ class ASTCodeUpgrader(object):
     for path in files_to_process:
       file_count += 1
       _, l_report, l_errors = self.process_file(path, path)
-      tree_errors += l_errors
+      tree_errors[path] = l_errors
       report += l_report
 
     return file_count, report, tree_errors
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_main.py b/tensorflow/tools/compatibility/tf_upgrade_v2_main.py
index 870bc6f216..871da9ad6b 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_main.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_main.py
@@ -98,15 +98,33 @@ Simple usage:
   else:
     parser.print_help()
   if report_text:
-    open(report_filename, "w").write(report_text)
-    print("TensorFlow 2.0 Upgrade Script")
-    print("-----------------------------")
-    print("Converted %d files\n" % files_processed)
-    print("Detected %d errors that require attention" % len(errors))
-    print("-" * 80)
-    print("\n".join(errors))
-    print("\nMake sure to read the detailed log %r\n" % report_filename)
+    num_errors = 0
+    report = ""
+    for f in errors:
+      if errors[f]:
+        num_errors += len(errors[f])
+        report += "-" * 80 + "\n"
+        report += "File: %s\n" % f
+        report += "-" * 80 + "\n"
+        report += "\n".join(errors[f]) + "\n"
+
+    report = (
+        "TensorFlow 2.0 Upgrade Script\n"
+        "-----------------------------\n"
+        "Converted %d files\n" % files_processed +
+        "Detected %d errors that require attention" % num_errors + "\n" +
+        "-" * 80 + "\n"
+    ) + report
 
+    with open(report_filename, "w") as report_file:
+      report_file.write(report)
+      report_file.write("=" * 80 + "\n")
+      report_file.write("Detailed log follows:\n\n")
+      report_file.write("=" * 80 + "\n")
+      report_file.write(report_text)
+
+    print(report)
+    print("\nMake sure to read the detailed log %r\n" % report_filename)
 
 if __name__ == "__main__":
   main()
-- 
GitLab


From f8b8e49bac488b16b1fe94cc07b8fb96dfb244bb Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Fri, 18 Jan 2019 12:09:53 -0800
Subject: [PATCH 0978/2345] Fix estimator import in tensorflow.compat.v1
 module.

PiperOrigin-RevId: 229970907
---
 tensorflow/compat_template_v1.__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compat_template_v1.__init__.py b/tensorflow/compat_template_v1.__init__.py
index b966c22b23..9549a71c41 100644
--- a/tensorflow/compat_template_v1.__init__.py
+++ b/tensorflow/compat_template_v1.__init__.py
@@ -28,7 +28,8 @@ from tensorflow.python import pywrap_tensorflow  # pylint: disable=unused-import
 from tensorflow.python.tools import component_api_helper as _component_api_helper
 _component_api_helper.package_hook(
     parent_package_str=__name__,
-    child_package_str=('tensorflow_estimator.python.estimator.api.estimator'))
+    child_package_str=(
+        'tensorflow_estimator.python.estimator.api._v1.estimator'))
 _component_api_helper.package_hook(
     parent_package_str=__name__,
     child_package_str=('tensorflow.python.keras.api._v1.keras'))
-- 
GitLab


From c4894a845f6c52740d32f566c862b9b6a99ffccc Mon Sep 17 00:00:00 2001
From: Artem Belevich <tra@google.com>
Date: Fri, 18 Jan 2019 12:14:27 -0800
Subject: [PATCH 0979/2345] Make interactive graphviz buildable in opensource
 TF and add a build test for it.

PiperOrigin-RevId: 229971593
---
 tensorflow/compiler/xla/tools/BUILD           |  6 ++++++
 .../xla/tools/interactive_graphviz.cc         | 16 ++++++++--------
 .../xla/tools/interactive_graphviz_test.sh    | 19 +++++++++++++++++++
 3 files changed, 33 insertions(+), 8 deletions(-)
 create mode 100755 tensorflow/compiler/xla/tools/interactive_graphviz_test.sh

diff --git a/tensorflow/compiler/xla/tools/BUILD b/tensorflow/compiler/xla/tools/BUILD
index 4412a6ec69..52fee4770a 100644
--- a/tensorflow/compiler/xla/tools/BUILD
+++ b/tensorflow/compiler/xla/tools/BUILD
@@ -255,3 +255,9 @@ tf_cc_binary(
         "@com_google_absl//absl/strings",
     ],
 )
+
+sh_test(
+    name = "interactive_graphviz_build_only_test",
+    srcs = ["interactive_graphviz_test.sh"],
+    data = [":interactive_graphviz"],
+)
diff --git a/tensorflow/compiler/xla/tools/interactive_graphviz.cc b/tensorflow/compiler/xla/tools/interactive_graphviz.cc
index 6c90cde5a7..ac865707f8 100644
--- a/tensorflow/compiler/xla/tools/interactive_graphviz.cc
+++ b/tensorflow/compiler/xla/tools/interactive_graphviz.cc
@@ -1,4 +1,4 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -29,8 +29,7 @@ limitations under the License.
 #include "absl/strings/numbers.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_join.h"
-#include "absl/strings/string_view_utils.h"
-#include "absl/strings/util.h"
+#include "absl/strings/str_split.h"
 #include "tensorflow/compiler/xla/client/client_library.h"
 #include "tensorflow/compiler/xla/client/local_client.h"
 #include "tensorflow/compiler/xla/service/compiler.h"
@@ -56,7 +55,8 @@ bool ReadLine(const char *prompt, string *line) {
   return util::ReadLine(prompt, line);
 #else
   std::cout << prompt;
-  return std::getline(std::cin, *line);
+  std::getline(std::cin, *line);
+  return std::cin.good();
 #endif
 }
 
@@ -391,9 +391,9 @@ void DisplayGraphHandle(const Options &opts, const string& handle) {
   std::cout << handle << std::endl;
 
   // If it is a url, try to open it up in the user's browser too.
-  if (strings::StartsWithIgnoreCase(handle, "http://") ||
-      strings::StartsWithIgnoreCase(handle, "https://") ||
-      strings::StartsWithIgnoreCase(handle, "file://")) {
+  if (absl::StartsWithIgnoreCase(handle, "http://") ||
+      absl::StartsWithIgnoreCase(handle, "https://") ||
+      absl::StartsWithIgnoreCase(handle, "file://")) {
     const char* browser_bin = opts.browser.empty() ? "/usr/bin/sensible-browser"
                                                    : opts.browser.c_str();
     tensorflow::SubProcess p;
@@ -515,7 +515,7 @@ void InteractiveDumpGraphs(const Options& opts, const HloModule& module) {
                 << std::endl;
       continue;
     }
-    std::vector<string> tokens = strings::Split(line, ' ');
+    std::vector<string> tokens = absl::StrSplit(line, ' ');
     if (tokens[0] == "quit" || tokens[0] == "exit") {
       break;
     } else if (tokens[0] == "help") {
diff --git a/tensorflow/compiler/xla/tools/interactive_graphviz_test.sh b/tensorflow/compiler/xla/tools/interactive_graphviz_test.sh
new file mode 100755
index 0000000000..b3e43aa7da
--- /dev/null
+++ b/tensorflow/compiler/xla/tools/interactive_graphviz_test.sh
@@ -0,0 +1,19 @@
+#! /bin/bash
+# /* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================*/
+
+# This is a placeholder for a compile-only test for intractive_graphviz tool.
+
+exit 0
-- 
GitLab


From 9cd051ee4b41d7bccaf8ad977644177d55a56baa Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 12:24:53 -0800
Subject: [PATCH 0980/2345] Change the output type of eager profiler result
 from tfprof.ProfileProto to tpu.Trace. Then we can use Tensorboard profile
 plugin to visualize the profiling result.

PiperOrigin-RevId: 229973284
---
 tensorflow/c/eager/BUILD                      |  1 +
 tensorflow/c/eager/c_api_experimental.h       |  5 +-
 tensorflow/c/eager/c_api_experimental_test.cc |  8 +--
 .../contrib/tpu/profiler/trace_events.proto   |  3 +
 tensorflow/core/profiler/lib/BUILD            |  2 +-
 .../core/profiler/lib/eager_profiler.cc       | 56 ++++++++++++++++---
 tensorflow/python/eager/BUILD                 |  2 +-
 tensorflow/python/eager/profiler.py           | 23 +++++---
 tensorflow/python/eager/profiler_test.py      |  4 +-
 9 files changed, 78 insertions(+), 26 deletions(-)

diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
index 51665fb9db..777e2e626a 100644
--- a/tensorflow/c/eager/BUILD
+++ b/tensorflow/c/eager/BUILD
@@ -229,6 +229,7 @@ tf_cuda_cc_test(
         ":c_api_test_util",
         "//tensorflow/c:c_test_util",
         "//tensorflow/cc/profiler",
+        "//tensorflow/contrib/tpu/profiler:trace_events_proto_cc",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
diff --git a/tensorflow/c/eager/c_api_experimental.h b/tensorflow/c/eager/c_api_experimental.h
index 9eb80f5216..5cd5569468 100644
--- a/tensorflow/c/eager/c_api_experimental.h
+++ b/tensorflow/c/eager/c_api_experimental.h
@@ -35,9 +35,8 @@ typedef struct TFE_Profiler TFE_Profiler;
 TF_CAPI_EXPORT extern TFE_Profiler* TFE_NewProfiler(TFE_Context* ctx);
 TF_CAPI_EXPORT extern void TFE_DeleteProfiler(TFE_Profiler* profiler);
 
-// The output string is a binary string of tensorflow.tfprof.ProfileProto.
-// User can write the string to file for offline analysis by tfprof command-line
-// tools or graphical user interface.
+// The output string is a binary string of tensorflow.tpu.Trace. User can write
+// the string to file for offline analysis by tensorboard.
 TF_CAPI_EXPORT extern void TFE_ProfilerSerializeToString(TFE_Context* ctx,
                                                          TFE_Profiler* profiler,
                                                          TF_Buffer* buf,
diff --git a/tensorflow/c/eager/c_api_experimental_test.cc b/tensorflow/c/eager/c_api_experimental_test.cc
index c3d29bd2e0..af55fee66e 100644
--- a/tensorflow/c/eager/c_api_experimental_test.cc
+++ b/tensorflow/c/eager/c_api_experimental_test.cc
@@ -18,12 +18,12 @@ limitations under the License.
 #include <string.h>
 #include "tensorflow/c/eager/c_api_test_util.h"
 #include "tensorflow/cc/profiler/profiler.h"
+#include "tensorflow/contrib/tpu/profiler/trace_events.pb.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/protobuf.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/platform/test_benchmark.h"
-#include "tensorflow/core/profiler/tfprof_log.pb.h"
 
 using tensorflow::string;
 
@@ -70,17 +70,17 @@ void ExecuteWithProfiling(bool async) {
   TFE_ProfilerSerializeToString(ctx, profiler, profiler_result, status);
   TFE_DeleteProfiler(profiler);
   ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
-  tensorflow::tfprof::ProfileProto profile_proto;
+  tensorflow::tpu::Trace profile_proto;
   EXPECT_TRUE(profile_proto.ParseFromString(
       {reinterpret_cast<const char*>(profiler_result->data),
        profiler_result->length}));
   string profile_proto_str = profile_proto.DebugString();
   if (!gpu_device_name.empty()) {
-    EXPECT_TRUE(HasSubstr(profile_proto_str, "gpu:0"));
+    EXPECT_TRUE(HasSubstr(profile_proto_str, "GPU:0"));
     // device name with "stream:all" is collected by Device Tracer.
     EXPECT_TRUE(HasSubstr(profile_proto_str, "stream:all"));
   }
-  EXPECT_TRUE(HasSubstr(profile_proto_str, "cpu:0"));
+  EXPECT_TRUE(HasSubstr(profile_proto_str, "CPU:0"));
   TF_DeleteBuffer(profiler_result);
 
   TF_Tensor* t = TFE_TensorHandleResolve(retvals[0], status);
diff --git a/tensorflow/contrib/tpu/profiler/trace_events.proto b/tensorflow/contrib/tpu/profiler/trace_events.proto
index cb2b916267..96c4784c69 100644
--- a/tensorflow/contrib/tpu/profiler/trace_events.proto
+++ b/tensorflow/contrib/tpu/profiler/trace_events.proto
@@ -56,4 +56,7 @@ message TraceEvent {
   // The duration of the event in picoseconds if applicable.
   // Events without duration are called instant events.
   uint64 duration_ps = 10;
+
+  // Extra arguments that will be displayed in trace view.
+  map<string, string> args = 11;
 }
diff --git a/tensorflow/core/profiler/lib/BUILD b/tensorflow/core/profiler/lib/BUILD
index 3d46606cbd..0320ae1a68 100644
--- a/tensorflow/core/profiler/lib/BUILD
+++ b/tensorflow/core/profiler/lib/BUILD
@@ -23,7 +23,7 @@ tf_cuda_library(
     visibility = ["//tensorflow:internal"],
     deps = [
         "//tensorflow/core/common_runtime/eager:context",
-        "//tensorflow/cc/profiler",
+        "//tensorflow/contrib/tpu/profiler:trace_events_proto_cc",
     ] + select({
         "//tensorflow:android": [
             "//tensorflow/core:android_tensorflow_lib_lite",
diff --git a/tensorflow/core/profiler/lib/eager_profiler.cc b/tensorflow/core/profiler/lib/eager_profiler.cc
index 7702febbc3..6b2e8b9a19 100644
--- a/tensorflow/core/profiler/lib/eager_profiler.cc
+++ b/tensorflow/core/profiler/lib/eager_profiler.cc
@@ -14,7 +14,8 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/profiler/lib/eager_profiler.h"
-#include "tensorflow/cc/profiler/profiler.h"
+#include <string>
+#include "tensorflow/contrib/tpu/profiler/trace_events.pb.h"
 #include "tensorflow/core/common_runtime/eager/context.h"
 #include "tensorflow/core/common_runtime/step_stats_collector.h"
 #include "tensorflow/core/framework/graph.pb.h"
@@ -25,6 +26,47 @@ limitations under the License.
 
 namespace tensorflow {
 
+namespace {
+
+void ConvertRunMetadataToTraceEvent(RunMetadata* run_metadata,
+                                    tpu::Trace* trace) {
+  auto trace_devices = trace->mutable_devices();
+  // TODO(fishx): use a lighter representation instead of GraphDef to insert
+  // python information into trace event.
+
+  for (size_t device_id = 0;
+       device_id < run_metadata->step_stats().dev_stats_size(); ++device_id) {
+    // Create device
+    auto* device_stats =
+        run_metadata->mutable_step_stats()->mutable_dev_stats(device_id);
+    tensorflow::tpu::Device device;
+    device.set_name(device_stats->device());
+    device.set_device_id(device_id);
+    tensorflow::tpu::Resource resource;
+    resource.set_name("0");
+    resource.set_resource_id(0);
+    (*device.mutable_resources())[0] = resource;
+    (*trace_devices)[device_id] = device;
+
+    // Emit events.
+    for (auto node :
+         run_metadata->step_stats().dev_stats(device_id).node_stats()) {
+      auto* event = trace->add_trace_events();
+      auto* args = event->mutable_args();
+      event->set_device_id(device_id);
+      event->set_resource_id(0);
+      event->set_name(node.node_name());
+      event->set_timestamp_ps(node.all_start_micros() * 1000000000);
+      event->set_duration_ps(node.all_end_rel_micros() * 1000000000);
+      (*args)["label"] = node.timeline_label();
+    }
+  }
+
+  // TODO(fishx): Convert allocation data as well.
+}
+
+}  // namespace
+
 /*static*/ std::unique_ptr<EagerProfiler> EagerProfiler::Create(
     EagerContext* const context) {
   return absl::WrapUnique(new EagerProfiler(context));
@@ -58,12 +100,12 @@ Status EagerProfiler::SerializeToString(string* content) {
     step_stats_collector->Finalize();
   }
 
-  // TODO(fishx): update tfprof to use a lighter representation instead of
-  // GraphDef.
-  GraphDef graph;
-  std::unique_ptr<tfprof::Profiler> tfprof(new tfprof::Profiler(graph));
-  tfprof->AddStep(0, run_metadata_);
-  return tfprof->SerializeToString(content);
+  tpu::Trace trace;
+
+  ConvertRunMetadataToTraceEvent(&run_metadata_, &trace);
+
+  trace.SerializeToString(content);
+  return Status::OK();
 }
 
 EagerProfiler::EagerProfiler(EagerContext* const context) : context_(context) {
diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD
index 3a35734051..6f798fcdb0 100644
--- a/tensorflow/python/eager/BUILD
+++ b/tensorflow/python/eager/BUILD
@@ -108,8 +108,8 @@ cuda_py_test(
     additional_deps = [
         ":profiler",
         ":test",
-        "//tensorflow/core/profiler:protos_all_py",
         "//tensorflow/python:constant_op",
+        "//tensorflow/contrib/tpu/profiler:trace_events_proto_py",
     ],
 )
 
diff --git a/tensorflow/python/eager/profiler.py b/tensorflow/python/eager/profiler.py
index f1d4373085..de96ae6ce9 100644
--- a/tensorflow/python/eager/profiler.py
+++ b/tensorflow/python/eager/profiler.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import os
 import threading
 
 from tensorflow.python import pywrap_tensorflow
@@ -25,8 +26,11 @@ from tensorflow.python.eager import context
 from tensorflow.python.framework import c_api_util
 from tensorflow.python.platform import gfile
 
+LOGDIR_PLUGIN = 'plugins/profile'
+
 _profiler = None
 _profiler_lock = threading.Lock()
+_run_num = 0
 
 
 def start():
@@ -48,14 +52,14 @@ def stop():
   """Stop current profiling session and return its result.
 
   Returns:
-    A binary string of tensorflow.tfprof.ProfileProto. User can write the string
-    to file for offline analysis by tfprof command-line tools or graphical user
-    interface.
+    A binary string of tensorflow.tpu.Trace. User can write the string
+    to file for offline analysis by tensorboard.
 
   Raises:
     AssertionError: If there is no active profiling session.
   """
   global _profiler
+  global _run_num
   if _profiler is None:
     raise AssertionError('Cannot stop profiling. No profiler is running.')
   with c_api_util.tf_buffer() as buffer_:
@@ -67,6 +71,7 @@ def stop():
   with _profiler_lock:
     pywrap_tensorflow.TFE_DeleteProfiler(_profiler)
     _profiler = None
+    _run_num += 1
   return result
 
 
@@ -75,19 +80,21 @@ class Profiler(object):
 
   Example usage:
   ```python
-  with Profiler("/path/to/save/result"):
+  with Profiler("/path/to/logdir"):
     # do some work
   ```
   """
 
-  def __init__(self, filename):
-    self._filename = filename
+  def __init__(self, logdir):
+    self._logdir = logdir
 
   def __enter__(self):
     start()
 
   def __exit__(self, typ, value, tb):
     result = stop()
-    with gfile.Open(self._filename, 'wb') as f:
+    plugin_dir = os.path.join(self._logdir, LOGDIR_PLUGIN,
+                              'run{}'.format(_run_num))
+    gfile.MakeDirs(plugin_dir)
+    with gfile.Open(os.path.join(plugin_dir, 'local.trace'), 'wb') as f:
       f.write(result)
-
diff --git a/tensorflow/python/eager/profiler_test.py b/tensorflow/python/eager/profiler_test.py
index b3fe2efabe..319d063990 100644
--- a/tensorflow/python/eager/profiler_test.py
+++ b/tensorflow/python/eager/profiler_test.py
@@ -18,7 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.core.profiler import tfprof_log_pb2
+from tensorflow.contrib.tpu.profiler import trace_events_pb2
 from tensorflow.python.eager import profiler
 from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
@@ -34,7 +34,7 @@ class ProfilerTest(test_util.TensorFlowTestCase):
     product = three * five
     self.assertAllEqual(15, product)
     profile_result = profiler.stop()
-    profile_pb = tfprof_log_pb2.ProfileProto()
+    profile_pb = trace_events_pb2.Trace()
     profile_pb.ParseFromString(profile_result)
     profile_pb_str = '%s' % profile_pb
     self.assertTrue('Mul' in profile_pb_str)
-- 
GitLab


From 6fb73564d4d831fdb3699225235931c132cc3eb5 Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Fri, 18 Jan 2019 12:27:23 -0800
Subject: [PATCH 0981/2345] Enable more tests.

PiperOrigin-RevId: 229973629
---
 tensorflow/lite/kernels/internal/BUILD | 22 ++++------------------
 1 file changed, 4 insertions(+), 18 deletions(-)

diff --git a/tensorflow/lite/kernels/internal/BUILD b/tensorflow/lite/kernels/internal/BUILD
index b734b2d6cc..1ff5164d2c 100644
--- a/tensorflow/lite/kernels/internal/BUILD
+++ b/tensorflow/lite/kernels/internal/BUILD
@@ -254,9 +254,6 @@ cc_library(
 cc_test(
     name = "tensor_test",
     srcs = ["tensor_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",  # TODO(b/117786830)
-    ],
     deps = [
         ":tensor",
         "@com_google_googletest//:gtest",
@@ -286,9 +283,6 @@ cc_library(
 cc_test(
     name = "quantization_util_test",
     srcs = ["quantization_util_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",  # TODO(b/117786830)
-    ],
     deps = [
         ":quantization_util",
         "@com_google_googletest//:gtest",
@@ -568,9 +562,6 @@ tf_cc_test(
         "//conditions:default": [],
     }),
     linkstatic = 1,
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":tensor_utils",
         "//tensorflow/lite/c:c_api_internal",
@@ -595,9 +586,6 @@ cc_test(
     name = "depthwiseconv_quantized_test",
     srcs = ["depthwiseconv_quantized_test.cc"],
     shard_count = 2,
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":optimized_base",
         ":reference_base",
@@ -610,9 +598,6 @@ cc_test(
 cc_test(
     name = "resize_bilinear_test",
     srcs = ["resize_bilinear_test.cc"],
-    tags = [
-        "tflite_not_portable",
-    ],
     deps = [
         ":optimized_base",
         ":reference_base",
@@ -625,9 +610,6 @@ cc_test(
 cc_test(
     name = "resize_nearest_neighbor_test",
     srcs = ["resize_nearest_neighbor_test.cc"],
-    tags = [
-        "tflite_not_portable",
-    ],
     deps = [
         ":optimized_base",
         ":reference_base",
@@ -679,6 +661,10 @@ cc_test(
 cc_test(
     name = "log_quantized_test",
     srcs = ["log_quantized_test.cc"],
+    linkopts = select({
+        "//tensorflow:windows": [],
+        "//conditions:default": ["-lm"],
+    }),
     deps = [
         ":optimized_base",
         ":reference_base",
-- 
GitLab


From 2bd14a830f8143cee455d5abe8540e8923b25f63 Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Fri, 18 Jan 2019 12:27:52 -0800
Subject: [PATCH 0982/2345] Fix to make sure correctness tests run in all modes

PiperOrigin-RevId: 229973693
---
 tensorflow/python/keras/engine/correctness_test.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/engine/correctness_test.py b/tensorflow/python/keras/engine/correctness_test.py
index c2f3b040de..68634235d1 100644
--- a/tensorflow/python/keras/engine/correctness_test.py
+++ b/tensorflow/python/keras/engine/correctness_test.py
@@ -66,7 +66,10 @@ class SimpleBiasTest(keras_parameterized.TestCase):
 
   def _get_simple_bias_model(self):
     model = testing_utils.get_model_from_layers([Bias()], input_shape=(1,))
-    model.compile(keras.optimizer_v2.gradient_descent.SGD(0.1), 'mae')
+    model.compile(
+        keras.optimizer_v2.gradient_descent.SGD(0.1),
+        'mae',
+        run_eagerly=testing_utils.should_run_eagerly())
     return model
 
   def test_simple_bias_fit(self):
@@ -101,7 +104,10 @@ class MultipleInputTest(keras_parameterized.TestCase):
       model = MultiInputSubclassed()
     else:
       model = multi_input_functional()
-    model.compile(keras.optimizer_v2.gradient_descent.SGD(0.1), 'mae')
+    model.compile(
+        keras.optimizer_v2.gradient_descent.SGD(0.1),
+        'mae',
+        run_eagerly=testing_utils.should_run_eagerly())
     return model
 
   @parameterized.named_parameters(('subclassed', True), ('functional', False))
-- 
GitLab


From 49cd875b9ae1bc9da96871c321010c25e7ae3b32 Mon Sep 17 00:00:00 2001
From: Ayush Dubey <ayushd@google.com>
Date: Fri, 18 Jan 2019 12:28:08 -0800
Subject: [PATCH 0983/2345] Build fix for collective_nccl_reducer_test_gpu for
 non-CUDA builds.

PiperOrigin-RevId: 229973734
---
 tensorflow/core/kernels/collective_nccl_reducer_test.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/kernels/collective_nccl_reducer_test.cc b/tensorflow/core/kernels/collective_nccl_reducer_test.cc
index ff33c1322c..26c92f1f74 100644
--- a/tensorflow/core/kernels/collective_nccl_reducer_test.cc
+++ b/tensorflow/core/kernels/collective_nccl_reducer_test.cc
@@ -12,6 +12,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
+
+#ifdef GOOGLE_CUDA
+
 #include "tensorflow/core/kernels/collective_nccl_reducer.h"
 
 #include <algorithm>
@@ -34,8 +37,6 @@ limitations under the License.
 #include "tensorflow/core/public/session_options.h"
 #include "tensorflow/core/public/version.h"
 
-#ifdef GOOGLE_CUDA
-
 namespace tensorflow {
 static constexpr int kStepId = 10;
 
-- 
GitLab


From 603afda1e43adef70bbd6764cc7416417e7baba7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 12:38:36 -0800
Subject: [PATCH 0984/2345] [TF:XLA] Make HloComputation equality comparison
 iterative.

When comparing HLOs with large subcomputations (like fusion), the previous recursive implementation can cause a stack overflow.

PiperOrigin-RevId: 229975468
---
 .../compiler/xla/service/hlo_computation.cc   | 43 ++++++++++++-------
 .../xla/service/hlo_computation_test.cc       | 23 ++++++++++
 2 files changed, 51 insertions(+), 15 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc
index f9b64d12ae..98164752b9 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation.cc
@@ -695,21 +695,34 @@ bool HloComputation::operator==(const HloComputation& other) const {
   }
   absl::flat_hash_set<std::pair<const HloInstruction*, const HloInstruction*>>
       visited;
-  std::function<bool(const HloInstruction*, const HloInstruction*)> eq =
-      [&visited, &eq](const HloInstruction* a, const HloInstruction* b) {
-        // If <a,b> are visited but not identical, the recursion should have
-        // been aborted. So, if <a,b> are visited at this point, they must be
-        // identical.
-        if (visited.contains(std::make_pair(a, b))) {
-          return true;
-        }
-        visited.emplace(a, b);
-        return a->Identical(
-            *b, eq, [](const HloComputation* a, const HloComputation* b) {
-              return *a == *b;
-            });
-      };
-  return eq(root_instruction(), other.root_instruction());
+  std::vector<std::pair<const HloInstruction*, const HloInstruction*>> worklist;
+
+  worklist.push_back({root_instruction(), other.root_instruction()});
+
+  while (!worklist.empty()) {
+    auto pair = worklist.back();
+    worklist.pop_back();
+
+    if (visited.contains(pair)) {
+      continue;
+    }
+    visited.emplace(pair);
+    // TODO(b/123082518): Avoid recursively invoking == becasue it may
+    // cause a stack overflow with deeply nested subcomputations.
+    bool identical_ignoring_operands = pair.first->Identical(
+        *pair.second,
+        [](const HloInstruction*, const HloInstruction*) { return true; },
+        [](const HloComputation* a, const HloComputation* b) {
+          return *a == *b;
+        });
+    if (!identical_ignoring_operands) {
+      return false;
+    }
+    for (size_t i = 0; i < pair.first->operands().size(); ++i) {
+      worklist.push_back({pair.first->operand(i), pair.second->operand(i)});
+    }
+  }
+  return true;
 }
 
 Status HloComputation::ReplaceWithNewInstruction(
diff --git a/tensorflow/compiler/xla/service/hlo_computation_test.cc b/tensorflow/compiler/xla/service/hlo_computation_test.cc
index 251c7bbec4..35ee532704 100644
--- a/tensorflow/compiler/xla/service/hlo_computation_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation_test.cc
@@ -645,5 +645,28 @@ TEST_F(HloComputationTest, StringificationCanonical) {
   EXPECT_EQ(computation->ToString(options), expected_computation2);
 }
 
+std::unique_ptr<HloComputation> MakeAddNComputation(int n) {
+  auto builder = HloComputation::Builder("add_n");
+  auto result = builder.AddInstruction(HloInstruction::CreateParameter(
+      0, ShapeUtil::MakeShape(F32, {}), "x_value"));
+  auto one = builder.AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<float>(1.0)));
+  for (int i = 0; i < n; ++i) {
+    result = builder.AddInstruction(HloInstruction::CreateBinary(
+        one->shape(), HloOpcode::kAdd, result, one));
+  }
+  return builder.Build();
+}
+
+TEST_F(HloComputationTest, DeepEquality) {
+  auto computation_a = MakeAddNComputation(200000);
+  auto computation_b = MakeAddNComputation(200000);
+  EXPECT_TRUE(*computation_a == *computation_b);
+
+  auto computation_c = MakeAddNComputation(199999);
+  EXPECT_FALSE(*computation_a == *computation_c);
+  EXPECT_FALSE(*computation_c == *computation_b);
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From fec674bdcdc5fbd3ee277031e100ba3c438d4bcc Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Fri, 18 Jan 2019 12:50:23 -0800
Subject: [PATCH 0985/2345] EncapsulateSubgraphs: skip duplicate control edge
 checking.

This check should be unnecessary (as we are reconstructing the edges only as
needed).  This speeds up startup of large mesh-tf models by over a minute.

PiperOrigin-RevId: 229977388
---
 .../jit/encapsulate_subgraphs_pass.cc         | 32 ++++++++++++-------
 1 file changed, 21 insertions(+), 11 deletions(-)

diff --git a/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc b/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc
index a21e083131..d0d7a3f378 100644
--- a/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc
+++ b/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc
@@ -1008,13 +1008,15 @@ Status Encapsulator::Subgraph::AddHostComputes(
       // subgraph.
       for (const auto& src_node : oc_subgraph.control_inputs) {
         Node* src_image = node_images.at(src_node);
-        graph_->AddControlEdge(src_image, host_compute);
+        graph_->AddControlEdge(src_image, host_compute,
+                               /* allow_duplicates= */ true);
       }
 
       // Connect the _HostCompute node to its ancestor host compute nodes.
       for (const auto& ancestor_name : host_compute_ancestors) {
         Node* ancestor = host_compute_node[ancestor_name];
-        graph_->AddControlEdge(ancestor, host_compute);
+        graph_->AddControlEdge(ancestor, host_compute,
+                               /* allow_duplicates= */ true);
       }
 
       // Connect the consumers in the subgraph to the _HostCompute node.
@@ -1031,7 +1033,8 @@ Status Encapsulator::Subgraph::AddHostComputes(
       // node.
       for (const auto& dst_node : oc_subgraph.control_outputs) {
         Node* dst_image = node_images.at(dst_node);
-        graph_->AddControlEdge(host_compute, dst_image);
+        graph_->AddControlEdge(host_compute, dst_image,
+                               /* allow_duplicates= */ true);
       }
     }
   }
@@ -1059,7 +1062,8 @@ Status Encapsulator::Subgraph::MakeSequencingNode(const string& subgraph_name,
 void Encapsulator::Subgraph::ConnectSequencerToCallNode(Graph* graph_out) {
   if (sequencer_ != nullptr) {
     VLOG(2) << "ConnectSequencerToCallNode";
-    graph_out->AddControlEdge(sequencer_, call_node_);
+    graph_out->AddControlEdge(sequencer_, call_node_,
+                              /* allow_duplicates= */ true);
   }
 }
 
@@ -1279,7 +1283,8 @@ Status Encapsulator::Subgraph::AddRecvAtHostNode(
   // completes. This has no effect on execution order but prevents the
   // RecvAtHost being pruned.
   TF_RETURN_IF_ERROR(MakeSequencingNode(subgraph_name, graph_out));
-  graph_out->AddControlEdge(oc_subgraph->recv_at_host, sequencer_);
+  graph_out->AddControlEdge(oc_subgraph->recv_at_host, sequencer_,
+                            true /* skip duplicates check */);
 
   return Status::OK();
 }
@@ -1336,7 +1341,8 @@ Status Encapsulator::Subgraph::AddSendFromHostNode(
   // subgraph completes. This has no effect on execution order but prevents the
   // RecvAtHost being pruned.
   TF_RETURN_IF_ERROR(MakeSequencingNode(subgraph_name, graph_out));
-  graph_out->AddControlEdge(oc_subgraph->send_from_host, sequencer_);
+  graph_out->AddControlEdge(oc_subgraph->send_from_host, sequencer_,
+                            /* allow_duplicates= */ true);
 
   return Status::OK();
 }
@@ -1446,7 +1452,8 @@ Status Encapsulator::CopySubgraphEdges(
         src_func_id == dst_func_id) {
       Graph* g = subgraphs_[src_func_id].GetGraph();
       if (edge->IsControlEdge()) {
-        g->AddControlEdge(src_image, dst_image);
+        g->AddControlEdge(src_image, dst_image,
+                          /* allow_duplicates= */ true);
       } else {
         g->AddEdge(src_image, edge->src_output(), dst_image, edge->dst_input());
       }
@@ -1732,7 +1739,8 @@ Status Encapsulator::CopyEdgeToOutputGraph(
     if (edges_added
             ->emplace(OutputTensor(src_image, -1), InputTensor(dst_image, -1))
             .second) {
-      graph_out->AddControlEdge(src_image, dst_image);
+      graph_out->AddControlEdge(src_image, dst_image,
+                                /* allow_duplicates= */ true);
     }
 
     return Status::OK();
@@ -1761,7 +1769,8 @@ Status Encapsulator::AddCallNodeDependencies(Graph* graph_out) {
     const string& subgraph = ancestors.first;
     for (const string& ancestor : ancestors.second) {
       graph_out->AddControlEdge(subgraphs_[ancestor].GetCallNode(),
-                                subgraphs_[subgraph].GetCallNode());
+                                subgraphs_[subgraph].GetCallNode(),
+                                /* allow_duplicates= */ true);
     }
   }
   return Status::OK();
@@ -2129,7 +2138,8 @@ Status CheckClusterDependencyForCycles(
     const string& ancestor, const string& successor,
     const std::unordered_map<string, std::unordered_set<string>>& ancestors,
     const std::unordered_map<Node*, PathDetails>& node_ancestors_map,
-    GraphCycles* cycle_detector, std::map<string, int>* cycle_detector_map) {
+    GraphCycles* cycle_detector,
+    std::unordered_map<string, int>* cycle_detector_map) {
   if (cycle_detector_map->find(ancestor) == cycle_detector_map->end()) {
     (*cycle_detector_map)[ancestor] = cycle_detector->NewNode();
   }
@@ -2173,7 +2183,7 @@ Status Encapsulator::FindClusterDependencies() {
   // We check that clusters are acyclic using this cycle detector.
   GraphCycles cycle_detector;
   // Map from cluster name to cycle detector node id.
-  std::map<string, int> cycle_detector_map;
+  std::unordered_map<string, int> cycle_detector_map;
   // Process the nodes in topologically-sorted order.
   std::vector<Node*> nodes;
   GetReversePostOrder(*graph_in_, &nodes);
-- 
GitLab


From 2515feb397b22822e012a916798ccb5c3e1cd997 Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Fri, 18 Jan 2019 12:54:05 -0800
Subject: [PATCH 0986/2345] [Grappler] Check regular input port upper bound
 before returning fanin node. Have GraphView also store max regular input port
 per node.

PiperOrigin-RevId: 229977962
---
 tensorflow/core/grappler/graph_view.h         | 102 ++++++---
 tensorflow/core/grappler/graph_view_test.cc   | 193 ++++++++++--------
 .../core/grappler/mutable_graph_view.cc       |  99 ++++-----
 3 files changed, 231 insertions(+), 163 deletions(-)

diff --git a/tensorflow/core/grappler/graph_view.h b/tensorflow/core/grappler/graph_view.h
index dc4ab93894..88b1e975c1 100644
--- a/tensorflow/core/grappler/graph_view.h
+++ b/tensorflow/core/grappler/graph_view.h
@@ -143,13 +143,20 @@ class GraphViewInternal {
 
   // Gets the output port(s) in the immediate fanin of an input port.
   absl::flat_hash_set<OutputPort> GetFanin(const InputPort& port) const {
-    if (port.port_id >= 0) return {GetRegularFanin(port)};
+    if (port.port_id >= 0) {
+      OutputPort regular_fanin = GetRegularFanin(port);
+      if (regular_fanin.node == nullptr) {
+        return {};
+      }
+      return {regular_fanin};
+    }
 
     // Collect fanin for the control input.
     absl::flat_hash_set<OutputPort> result;
-    for (int i = port.node->input_size() - 1; i >= 0; --i) {
+    const int first_control_port =
+        gtl::FindWithDefault(max_regular_input_port_, port.node, -1) + 1;
+    for (int i = first_control_port; i < port.node->input_size(); ++i) {
       TensorId tensor_id = ParseTensorName(port.node->input(i));
-      if (tensor_id.index() >= 0) break;  // we reached regular inputs
 
       auto it = nodes_.find(tensor_id.node());
       if (it != nodes_.end()) result.emplace(it->second, tensor_id.index());
@@ -158,22 +165,36 @@ class GraphViewInternal {
   }
 
   // Special case: regular (i.e. non-control) input ports can only have one
-  // fanin.
+  // fanin. If port.port_id is out of range or is a control dependency, then an
+  // empty OutputPort is returned.
   const OutputPort GetRegularFanin(const InputPort& port) const {
-    DCHECK_GE(port.port_id, 0);
-    if (port.port_id < 0) return OutputPort();
+    if (port.port_id < 0 ||
+        port.port_id >
+            gtl::FindWithDefault(max_regular_input_port_, port.node, -1)) {
+      return OutputPort();
+    }
 
     TensorId tensor_id = ParseTensorName(port.node->input(port.port_id));
     return GetOutputPort(tensor_id.node(), tensor_id.index());
   }
 
   // Checks if a tensor id is a fanin of the node.
-  bool HasFanin(const NodeDef& node, const TensorId& fanin) const {
-    if (fanin.index() < -1) {
+  bool HasFanin(const NodeDefT& node, const TensorId& fanin) const {
+    int end = node.input_size();
+    if (end == 0 || fanin.index() < -1) {
       return false;
     }
-    for (const string& input : node.input()) {
-      if (ParseTensorName(input) == fanin) {
+
+    const int num_regular_fanins =
+        gtl::FindWithDefault(max_regular_input_port_, &node, -1) + 1;
+    int start = 0;
+    if (fanin.index() > -1) {
+      end = num_regular_fanins;
+    } else {
+      start = num_regular_fanins;
+    }
+    for (int i = start; i < end; ++i) {
+      if (ParseTensorName(node.input(i)) == fanin) {
         return true;
       }
     }
@@ -183,14 +204,14 @@ class GraphViewInternal {
   // Gets all the input ports in the immediate fanout of a node. Include the
   // controlled nodes iff include_controlled_nodes is true.
   absl::flat_hash_set<InputPort> GetFanouts(
-      const NodeDef& node, bool include_controlled_nodes) const {
+      const NodeDefT& node, bool include_controlled_nodes) const {
     absl::flat_hash_set<InputPort> result;
 
     OutputPort port;
     port.node = const_cast<NodeDefT*>(&node);
     const int first_port_id = include_controlled_nodes ? -1 : 0;
     const int last_port_id =
-        gtl::FindWithDefault(max_regular_output_port_, port.node, -1);
+        gtl::FindWithDefault(max_regular_output_port_, &node, -1);
 
     for (int i = first_port_id; i <= last_port_id; ++i) {
       port.port_id = i;
@@ -205,11 +226,14 @@ class GraphViewInternal {
   // Gets all the output ports in the immediate fanin of a node. Include the
   // controlling nodes iff include_controlling_nodes is true.
   absl::flat_hash_set<OutputPort> GetFanins(
-      const NodeDef& node, bool include_controlling_nodes) const {
+      const NodeDefT& node, bool include_controlling_nodes) const {
     absl::flat_hash_set<OutputPort> result;
-    for (int i = 0; i < node.input_size(); ++i) {
+    const int max_input_port =
+        include_controlling_nodes
+            ? node.input_size() - 1
+            : gtl::FindWithDefault(max_regular_input_port_, &node, -1);
+    for (int i = 0; i <= max_input_port; ++i) {
       TensorId tensor_id = ParseTensorName(node.input(i));
-      if (tensor_id.index() < 0 && !include_controlling_nodes) break;
 
       auto it = nodes_.find(tensor_id.node());
       if (it != nodes_.end()) result.emplace(it->second, tensor_id.index());
@@ -219,27 +243,23 @@ class GraphViewInternal {
 
   // Gets the number of ports in the immediate fanin of a node. Count the
   // controlling nodes iff include_controlling_nodes is true.
-  int NumFanins(const NodeDef& node, bool include_controlling_nodes) const {
-    int count = 0;
-    for (const string& input : node.input()) {
-      if (!include_controlling_nodes && IsControlInput(input)) {
-        break;
-      }
-      count += 1;
+  int NumFanins(const NodeDefT& node, bool include_controlling_nodes) const {
+    if (include_controlling_nodes) {
+      return node.input_size();
     }
-    return count;
+    return gtl::FindWithDefault(max_regular_input_port_, &node, -1) + 1;
   }
 
   // Gets the number of ports in the immediate fanout of a node. Count the
   // controlled nodes iff include_controlled_nodes is true.
-  int NumFanouts(const NodeDef& node, bool include_controlled_nodes) const {
+  int NumFanouts(const NodeDefT& node, bool include_controlled_nodes) const {
     int count = 0;
 
     OutputPort port;
     port.node = const_cast<NodeDefT*>(&node);
     const int first_port_id = include_controlled_nodes ? -1 : 0;
     const int last_port_id =
-        gtl::FindWithDefault(max_regular_output_port_, port.node, -1);
+        gtl::FindWithDefault(max_regular_output_port_, &node, -1);
 
     for (int i = first_port_id; i <= last_port_id; ++i) {
       port.port_id = i;
@@ -253,7 +273,7 @@ class GraphViewInternal {
   // Gets all the edges in the immediate fanout of a node. Include the
   // controlled edges iff include_controlled_edges is true.
   absl::flat_hash_set<Edge> GetFanoutEdges(
-      const NodeDef& node, bool include_controlled_edges) const {
+      const NodeDefT& node, bool include_controlled_edges) const {
     absl::flat_hash_set<Edge> result;
 
     OutputPort port;
@@ -267,8 +287,7 @@ class GraphViewInternal {
       auto it = fanouts_.find(port);
       if (it != fanouts_.end()) {
         for (auto itr = it->second.begin(); itr != it->second.end(); ++itr) {
-          result.emplace(/*src=*/OutputPort(const_cast<NodeDefT*>(&node), i),
-                         /*dst=*/*itr);
+          result.emplace(/*src=*/port, /*dst=*/*itr);
         }
       }
     }
@@ -278,11 +297,14 @@ class GraphViewInternal {
   // Gets all the edges in the immediate fanin of a node. Include the
   // controlling edges iff include_controlling_edges is true.
   absl::flat_hash_set<Edge> GetFaninEdges(
-      const NodeDef& node, bool include_controlling_edges) const {
+      const NodeDefT& node, bool include_controlling_edges) const {
     absl::flat_hash_set<Edge> result;
-    for (int i = 0; i < node.input_size(); ++i) {
+    const int max_input_port =
+        include_controlling_edges
+            ? node.input_size() - 1
+            : gtl::FindWithDefault(max_regular_input_port_, &node, -1);
+    for (int i = 0; i <= max_input_port; ++i) {
       TensorId tensor_id = ParseTensorName(node.input(i));
-      if (tensor_id.index() < 0 && !include_controlling_edges) break;
 
       auto it = nodes_.find(tensor_id.node());
       if (it != nodes_.end()) {
@@ -303,7 +325,10 @@ class GraphViewInternal {
     CHECK(result.second) << "Non unique node name detected: " << node->name();
   }
 
+  // TODO(lyandy): Checks for self loops, Switch control dependencies, fanins
+  // exist, and all regular fanins come before controlling fanins.
   void AddFanouts(NodeDefT* node) {
+    int max_input_port = -1;
     for (int i = 0; i < node->input_size(); ++i) {
       TensorId tensor_id = ParseTensorName(node->input(i));
       OutputPort output(nodes_[tensor_id.node()], tensor_id.index());
@@ -311,11 +336,15 @@ class GraphViewInternal {
       if (output.port_id < 0) {
         fanouts_[output].emplace(node, -1);
       } else {
+        max_input_port = i;
         max_regular_output_port_[output.node] =
             std::max(max_regular_output_port_[output.node], output.port_id);
         fanouts_[output].emplace(node, i);
       }
     }
+    if (max_input_port > -1) {
+      max_regular_input_port_[node] = max_input_port;
+    }
   }
 
   // Access to the mutable internal state for MutableGraphView.
@@ -325,7 +354,11 @@ class GraphViewInternal {
     return fanouts_;
   }
 
-  absl::flat_hash_map<const NodeDef*, int>& max_regular_output_port() {
+  absl::flat_hash_map<const NodeDefT*, int>& max_regular_input_port() {
+    return max_regular_input_port_;
+  }
+
+  absl::flat_hash_map<const NodeDefT*, int>& max_regular_output_port() {
     return max_regular_output_port_;
   }
 
@@ -338,10 +371,13 @@ class GraphViewInternal {
   // A mapping from the output port to all inputs that read from it.
   absl::flat_hash_map<OutputPort, absl::flat_hash_set<InputPort>> fanouts_;
 
+  // Keep a maximum index of input tensors of the node.
+  absl::flat_hash_map<const NodeDefT*, int> max_regular_input_port_;
+
   // Keep a maximum index of tensor fetched from the node. It doesn't guarantee
   // that all tensors in the [0, max_regular_output_port] range are actually
   // fetched by other nodes.
-  absl::flat_hash_map<const NodeDef*, int> max_regular_output_port_;
+  absl::flat_hash_map<const NodeDefT*, int> max_regular_output_port_;
 
   // If the node has no fanouts at given output port (output tensor consumers)
   // we return a reference to this set from `GetFanout` (we can't construct new
diff --git a/tensorflow/core/grappler/graph_view_test.cc b/tensorflow/core/grappler/graph_view_test.cc
index 404dcd30c1..839057065b 100644
--- a/tensorflow/core/grappler/graph_view_test.cc
+++ b/tensorflow/core/grappler/graph_view_test.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/inputs/trivial_test_graph_input_yielder.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/platform/test.h"
 
 namespace tensorflow {
@@ -42,26 +43,24 @@ TEST_F(GraphViewTest, OpPortIdToArgIdShapeN) {
 
   const OpDef* a_op_def = nullptr;
   const OpDef* b_op_def = nullptr;
-  EXPECT_TRUE(
-      OpRegistry::Global()->LookUpOpDef(a_node_def.op(), &a_op_def).ok());
-  EXPECT_TRUE(
-      OpRegistry::Global()->LookUpOpDef(b_node_def.op(), &b_op_def).ok());
+  TF_EXPECT_OK(OpRegistry::Global()->LookUpOpDef(a_node_def.op(), &a_op_def));
+  TF_EXPECT_OK(OpRegistry::Global()->LookUpOpDef(b_node_def.op(), &b_op_def));
 
   // Const has 0 inputs, 1 output.
-  EXPECT_EQ(-1, OpInputPortIdToArgId(a_node_def, *a_op_def, 0));
-  EXPECT_EQ(0, OpOutputPortIdToArgId(a_node_def, *a_op_def, 0));
-  EXPECT_EQ(-1, OpOutputPortIdToArgId(a_node_def, *a_op_def, 1));
+  EXPECT_EQ(OpInputPortIdToArgId(a_node_def, *a_op_def, 0), -1);
+  EXPECT_EQ(OpOutputPortIdToArgId(a_node_def, *a_op_def, 0), 0);
+  EXPECT_EQ(OpOutputPortIdToArgId(a_node_def, *a_op_def, 1), -1);
 
   // ShapeN has N=3 inputs and outputs.
-  EXPECT_EQ(0, OpInputPortIdToArgId(b_node_def, *b_op_def, 0));
-  EXPECT_EQ(0, OpInputPortIdToArgId(b_node_def, *b_op_def, 1));
-  EXPECT_EQ(0, OpInputPortIdToArgId(b_node_def, *b_op_def, 2));
-  EXPECT_EQ(-1, OpInputPortIdToArgId(b_node_def, *b_op_def, 3));
-  EXPECT_EQ(0, OpOutputPortIdToArgId(b_node_def, *b_op_def, 0));
-  EXPECT_EQ(0, OpOutputPortIdToArgId(b_node_def, *b_op_def, 1));
-  EXPECT_EQ(0, OpOutputPortIdToArgId(b_node_def, *b_op_def, 2));
-  EXPECT_EQ(-1, OpOutputPortIdToArgId(b_node_def, *b_op_def, 3));
-  EXPECT_EQ(-1, OpOutputPortIdToArgId(b_node_def, *b_op_def, 4));
+  EXPECT_EQ(OpInputPortIdToArgId(b_node_def, *b_op_def, 0), 0);
+  EXPECT_EQ(OpInputPortIdToArgId(b_node_def, *b_op_def, 1), 0);
+  EXPECT_EQ(OpInputPortIdToArgId(b_node_def, *b_op_def, 2), 0);
+  EXPECT_EQ(OpInputPortIdToArgId(b_node_def, *b_op_def, 3), -1);
+  EXPECT_EQ(OpOutputPortIdToArgId(b_node_def, *b_op_def, 0), 0);
+  EXPECT_EQ(OpOutputPortIdToArgId(b_node_def, *b_op_def, 1), 0);
+  EXPECT_EQ(OpOutputPortIdToArgId(b_node_def, *b_op_def, 2), 0);
+  EXPECT_EQ(OpOutputPortIdToArgId(b_node_def, *b_op_def, 3), -1);
+  EXPECT_EQ(OpOutputPortIdToArgId(b_node_def, *b_op_def, 4), -1);
 }
 
 TEST_F(GraphViewTest, OpPortIdToArgIdSparseSplit) {
@@ -76,22 +75,21 @@ TEST_F(GraphViewTest, OpPortIdToArgIdSparseSplit) {
 
     const NodeDef& b_node_def = *graph_view.GetNode("b");
     const OpDef* b_op_def = nullptr;
-    EXPECT_TRUE(
-        OpRegistry::Global()->LookUpOpDef(b_node_def.op(), &b_op_def).ok());
+    TF_EXPECT_OK(OpRegistry::Global()->LookUpOpDef(b_node_def.op(), &b_op_def));
 
     // We have 4 inputs.
-    EXPECT_EQ(0, OpInputPortIdToArgId(b_node_def, *b_op_def, 0));
-    EXPECT_EQ(1, OpInputPortIdToArgId(b_node_def, *b_op_def, 1));
-    EXPECT_EQ(2, OpInputPortIdToArgId(b_node_def, *b_op_def, 2));
-    EXPECT_EQ(3, OpInputPortIdToArgId(b_node_def, *b_op_def, 3));
-    EXPECT_EQ(-1, OpInputPortIdToArgId(b_node_def, *b_op_def, 4));
+    EXPECT_EQ(OpInputPortIdToArgId(b_node_def, *b_op_def, 0), 0);
+    EXPECT_EQ(OpInputPortIdToArgId(b_node_def, *b_op_def, 1), 1);
+    EXPECT_EQ(OpInputPortIdToArgId(b_node_def, *b_op_def, 2), 2);
+    EXPECT_EQ(OpInputPortIdToArgId(b_node_def, *b_op_def, 3), 3);
+    EXPECT_EQ(OpInputPortIdToArgId(b_node_def, *b_op_def, 4), -1);
 
     for (int port_id = 0; port_id <= num_splits * 3; ++port_id) {
       int arg_id = -1;
       if (port_id < num_splits * 3) {
         arg_id = port_id / num_splits;
       }
-      EXPECT_EQ(arg_id, OpOutputPortIdToArgId(b_node_def, *b_op_def, port_id));
+      EXPECT_EQ(OpOutputPortIdToArgId(b_node_def, *b_op_def, port_id), arg_id);
     }
   }
 }
@@ -110,18 +108,17 @@ TEST_F(GraphViewTest, ParseSingleExample) {
   const NodeDef& c_node_def = *graph_view.GetNode("c");
 
   const OpDef* c_op_def = nullptr;
-  EXPECT_TRUE(
-      OpRegistry::Global()->LookUpOpDef(c_node_def.op(), &c_op_def).ok());
-
-  EXPECT_EQ(0, OpOutputPortIdToArgId(c_node_def, *c_op_def, 0));
-  EXPECT_EQ(0, OpOutputPortIdToArgId(c_node_def, *c_op_def, 1));
-  EXPECT_EQ(1, OpOutputPortIdToArgId(c_node_def, *c_op_def, 2));
-  EXPECT_EQ(1, OpOutputPortIdToArgId(c_node_def, *c_op_def, 3));
-  EXPECT_EQ(2, OpOutputPortIdToArgId(c_node_def, *c_op_def, 4));
-  EXPECT_EQ(2, OpOutputPortIdToArgId(c_node_def, *c_op_def, 5));
-  EXPECT_EQ(3, OpOutputPortIdToArgId(c_node_def, *c_op_def, 6));
-  EXPECT_EQ(3, OpOutputPortIdToArgId(c_node_def, *c_op_def, 7));
-  EXPECT_EQ(-1, OpOutputPortIdToArgId(c_node_def, *c_op_def, 8));
+  TF_EXPECT_OK(OpRegistry::Global()->LookUpOpDef(c_node_def.op(), &c_op_def));
+
+  EXPECT_EQ(OpOutputPortIdToArgId(c_node_def, *c_op_def, 0), 0);
+  EXPECT_EQ(OpOutputPortIdToArgId(c_node_def, *c_op_def, 1), 0);
+  EXPECT_EQ(OpOutputPortIdToArgId(c_node_def, *c_op_def, 2), 1);
+  EXPECT_EQ(OpOutputPortIdToArgId(c_node_def, *c_op_def, 3), 1);
+  EXPECT_EQ(OpOutputPortIdToArgId(c_node_def, *c_op_def, 4), 2);
+  EXPECT_EQ(OpOutputPortIdToArgId(c_node_def, *c_op_def, 5), 2);
+  EXPECT_EQ(OpOutputPortIdToArgId(c_node_def, *c_op_def, 6), 3);
+  EXPECT_EQ(OpOutputPortIdToArgId(c_node_def, *c_op_def, 7), 3);
+  EXPECT_EQ(OpOutputPortIdToArgId(c_node_def, *c_op_def, 8), -1);
 }
 
 TEST_F(GraphViewTest, BasicGraph) {
@@ -132,26 +129,26 @@ TEST_F(GraphViewTest, BasicGraph) {
   GraphView graph(&item.graph);
 
   GraphView::InputPort input = graph.GetInputPort("AddN", 0);
-  EXPECT_EQ("AddN", input.node->name());
-  EXPECT_EQ(0, input.port_id);
+  EXPECT_EQ(input.node->name(), "AddN");
+  EXPECT_EQ(input.port_id, 0);
   GraphView::OutputPort fanin = graph.GetRegularFanin(input);
-  EXPECT_EQ("Square", fanin.node->name());
-  EXPECT_EQ(0, fanin.port_id);
+  EXPECT_EQ(fanin.node->name(), "Square");
+  EXPECT_EQ(fanin.port_id, 0);
 
   input = graph.GetInputPort("AddN", 1);
-  EXPECT_EQ("AddN", input.node->name());
-  EXPECT_EQ(1, input.port_id);
+  EXPECT_EQ(input.node->name(), "AddN");
+  EXPECT_EQ(input.port_id, 1);
   fanin = graph.GetRegularFanin(input);
-  EXPECT_EQ("Square_1", fanin.node->name());
-  EXPECT_EQ(0, fanin.port_id);
+  EXPECT_EQ(fanin.node->name(), "Square_1");
+  EXPECT_EQ(fanin.port_id, 0);
 
   GraphView::OutputPort output = graph.GetOutputPort("AddN", 0);
-  EXPECT_EQ("AddN", output.node->name());
-  EXPECT_EQ(0, output.port_id);
-  EXPECT_EQ(2, graph.GetFanout(output).size());
+  EXPECT_EQ(output.node->name(), "AddN");
+  EXPECT_EQ(output.port_id, 0);
+  EXPECT_EQ(graph.GetFanout(output).size(), 2);
   for (auto fanout : graph.GetFanout(output)) {
     if (fanout.node->name() == "AddN_2" || fanout.node->name() == "AddN_3") {
-      EXPECT_EQ(0, fanout.port_id);
+      EXPECT_EQ(fanout.port_id, 0);
     } else {
       // Invalid fanout
       EXPECT_FALSE(true);
@@ -159,7 +156,7 @@ TEST_F(GraphViewTest, BasicGraph) {
   }
 
   const NodeDef* add_node = graph.GetNode("AddN");
-  EXPECT_NE(nullptr, add_node);
+  EXPECT_NE(add_node, nullptr);
 
   absl::flat_hash_set<string> fanouts;
   absl::flat_hash_set<string> expected_fanouts = {"AddN_2:0", "AddN_3:0"};
@@ -190,44 +187,44 @@ TEST_F(GraphViewTest, ControlDependencies) {
   GraphView graph(&item.graph);
 
   GraphView::OutputPort output = graph.GetOutputPort("a", -1);
-  EXPECT_EQ("a", output.node->name());
-  EXPECT_EQ(-1, output.port_id);
+  EXPECT_EQ(output.node->name(), "a");
+  EXPECT_EQ(output.port_id, -1);
   auto fanout = graph.GetFanout(output);
-  EXPECT_EQ(1, fanout.size());
-  EXPECT_EQ("d", (*fanout.begin()).node->name());
-  EXPECT_EQ(-1, (*fanout.begin()).port_id);
+  EXPECT_EQ(fanout.size(), 1);
+  EXPECT_EQ((*fanout.begin()).node->name(), "d");
+  EXPECT_EQ((*fanout.begin()).port_id, -1);
 
   output = graph.GetOutputPort("a", 0);
-  EXPECT_EQ("a", output.node->name());
-  EXPECT_EQ(0, output.port_id);
+  EXPECT_EQ(output.node->name(), "a");
+  EXPECT_EQ(output.port_id, 0);
   fanout = graph.GetFanout(output);
-  EXPECT_EQ(1, fanout.size());
-  EXPECT_EQ("b", (*fanout.begin()).node->name());
-  EXPECT_EQ(0, (*fanout.begin()).port_id);
+  EXPECT_EQ(fanout.size(), 1);
+  EXPECT_EQ((*fanout.begin()).node->name(), "b");
+  EXPECT_EQ((*fanout.begin()).port_id, 0);
 
   GraphView::InputPort input = graph.GetInputPort("d", -1);
-  EXPECT_EQ("d", input.node->name());
-  EXPECT_EQ(-1, input.port_id);
+  EXPECT_EQ(input.node->name(), "d");
+  EXPECT_EQ(input.port_id, -1);
   auto fanin = graph.GetFanin(input);
-  EXPECT_EQ(1, fanin.size());
-  EXPECT_EQ("a", (*fanin.begin()).node->name());
-  EXPECT_EQ(-1, (*fanin.begin()).port_id);
+  EXPECT_EQ(fanin.size(), 1);
+  EXPECT_EQ((*fanin.begin()).node->name(), "a");
+  EXPECT_EQ((*fanin.begin()).port_id, -1);
 
   input = graph.GetInputPort("d", 0);
-  EXPECT_EQ("d", input.node->name());
-  EXPECT_EQ(0, input.port_id);
+  EXPECT_EQ(input.node->name(), "d");
+  EXPECT_EQ(input.port_id, 0);
   fanin = graph.GetFanin(input);
-  EXPECT_EQ(1, fanin.size());
-  EXPECT_EQ("b", (*fanin.begin()).node->name());
-  EXPECT_EQ(0, (*fanin.begin()).port_id);
+  EXPECT_EQ(fanin.size(), 1);
+  EXPECT_EQ((*fanin.begin()).node->name(), "b");
+  EXPECT_EQ((*fanin.begin()).port_id, 0);
 
   input = graph.GetInputPort("d", 1);
-  EXPECT_EQ("d", input.node->name());
-  EXPECT_EQ(1, input.port_id);
+  EXPECT_EQ(input.node->name(), "d");
+  EXPECT_EQ(input.port_id, 1);
   fanin = graph.GetFanin(input);
-  EXPECT_EQ(1, fanin.size());
-  EXPECT_EQ("c", (*fanin.begin()).node->name());
-  EXPECT_EQ(0, (*fanin.begin()).port_id);
+  EXPECT_EQ(fanin.size(), 1);
+  EXPECT_EQ((*fanin.begin()).node->name(), "c");
+  EXPECT_EQ((*fanin.begin()).port_id, 0);
 }
 
 TEST_F(GraphViewTest, HasNode) {
@@ -238,8 +235,8 @@ TEST_F(GraphViewTest, HasNode) {
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
   GraphView graph(&item.graph);
 
-  EXPECT_EQ(true, graph.HasNode("a"));
-  EXPECT_EQ(false, graph.HasNode("b"));
+  EXPECT_EQ(graph.HasNode("a"), true);
+  EXPECT_EQ(graph.HasNode("b"), false);
 }
 
 TEST_F(GraphViewTest, HasFanin) {
@@ -254,14 +251,42 @@ TEST_F(GraphViewTest, HasFanin) {
   GraphView graph(&item.graph);
 
   const NodeDef* d_node = graph.GetNode("d");
-  EXPECT_NE(nullptr, d_node);
-
-  EXPECT_EQ(true, graph.HasFanin(*d_node, {"a", Graph::kControlSlot}));
-  EXPECT_EQ(false, graph.HasFanin(*d_node, {"a", 0}));
-  EXPECT_EQ(true, graph.HasFanin(*d_node, {"b", 0}));
-  EXPECT_EQ(false, graph.HasFanin(*d_node, {"b", Graph::kControlSlot}));
-  EXPECT_EQ(true, graph.HasFanin(*d_node, {"c", 0}));
-  EXPECT_EQ(false, graph.HasFanin(*d_node, {"c", Graph::kControlSlot}));
+  EXPECT_NE(d_node, nullptr);
+
+  EXPECT_EQ(graph.HasFanin(*d_node, {"a", Graph::kControlSlot}), true);
+  EXPECT_EQ(graph.HasFanin(*d_node, {"a", 0}), false);
+  EXPECT_EQ(graph.HasFanin(*d_node, {"b", 0}), true);
+  EXPECT_EQ(graph.HasFanin(*d_node, {"b", Graph::kControlSlot}), false);
+  EXPECT_EQ(graph.HasFanin(*d_node, {"c", 0}), true);
+  EXPECT_EQ(graph.HasFanin(*d_node, {"c", Graph::kControlSlot}), false);
+}
+
+TEST_F(GraphViewTest, GetRegularFaninPortOutOfBounds) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+  Output a = ops::Const(s.WithOpName("a"), 0.0f, {10, 10});
+  Output b = ops::Square(s.WithOpName("b"), {});
+  Output c = ops::Sqrt(s.WithOpName("c"), {b});
+  Output d = ops::AddN(s.WithOpName("d").WithControlDependencies(a), {b, c});
+
+  GrapplerItem item;
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+  GraphView graph(&item.graph);
+
+  const NodeDef* b_node = graph.GetNode("b");
+  EXPECT_NE(b_node, nullptr);
+  const NodeDef* c_node = graph.GetNode("c");
+  EXPECT_NE(c_node, nullptr);
+  const NodeDef* d_node = graph.GetNode("d");
+  EXPECT_NE(d_node, nullptr);
+
+  auto d_output_0 = graph.GetRegularFanin({d_node, 0});
+  EXPECT_EQ(d_output_0, GraphView::OutputPort(b_node, 0));
+  auto d_output_1 = graph.GetRegularFanin({d_node, 1});
+  EXPECT_EQ(d_output_1, GraphView::OutputPort(c_node, 0));
+  auto d_output_2 = graph.GetRegularFanin({d_node, 2});
+  EXPECT_EQ(d_output_2, GraphView::OutputPort());
+  auto d_output_control = graph.GetRegularFanin({d_node, Graph::kControlSlot});
+  EXPECT_EQ(d_output_control, GraphView::OutputPort());
 }
 
 }  // namespace
diff --git a/tensorflow/core/grappler/mutable_graph_view.cc b/tensorflow/core/grappler/mutable_graph_view.cc
index e31eedf166..43dec07c1d 100644
--- a/tensorflow/core/grappler/mutable_graph_view.cc
+++ b/tensorflow/core/grappler/mutable_graph_view.cc
@@ -117,10 +117,11 @@ bool CanDedupControlWithRegularInput(const MutableGraphView& graph,
 }  // namespace
 
 void MutableGraphView::AddAndDedupFanouts(NodeDef* node) {
-  // TODO(lyandy): Checks for self loops, Switch control dependencies and if
-  // fanins exist.
+  // TODO(lyandy): Checks for self loops, Switch control dependencies, fanins
+  // exist, and all regular fanins come before controlling fanins.
   absl::flat_hash_set<absl::string_view> fanins;
   absl::flat_hash_set<absl::string_view> controlling_fanins;
+  int max_input_port = -1;
   int pos = 0;
   const int last_idx = node->input_size() - 1;
   int last_pos = last_idx;
@@ -144,6 +145,7 @@ void MutableGraphView::AddAndDedupFanouts(NodeDef* node) {
       if (is_control_input) {
         fanouts()[output].emplace(node, Graph::kControlSlot);
       } else {
+        max_input_port = pos;
         max_regular_output_port()[output.node] =
             std::max(max_regular_output_port()[output.node], output.port_id);
         fanouts()[output].emplace(node, pos);
@@ -158,6 +160,10 @@ void MutableGraphView::AddAndDedupFanouts(NodeDef* node) {
   if (last_pos < last_idx) {
     node->mutable_input()->DeleteSubrange(last_pos + 1, last_idx - last_pos);
   }
+
+  if (max_input_port > -1) {
+    max_regular_input_port()[node] = max_input_port;
+  }
 }
 
 void MutableGraphView::UpdateMaxRegularOutputPortForRemovedFanin(
@@ -303,9 +309,7 @@ Status MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
     // Update input at destination node.
     input_port.node->set_input(
         input_port.port_id,
-        output_port.port_id == 0
-            ? to_node->name()
-            : absl::StrCat(to_node->name(), ":", output_port.port_id));
+        TensorIdToString({to_node->name(), output_port.port_id}));
 
     // Remove old edge between the `from_node` and the fanout node.
     remove_edge(output_port, input_port);
@@ -333,7 +337,7 @@ Status MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
 
 bool MutableGraphView::AddFaninInternal(NodeDef* node,
                                         const OutputPort& fanin) {
-  int num_non_controlling_fanins =
+  int num_regular_fanins =
       NumFanins(*node, /*include_controlling_nodes=*/false);
   bool input_is_control = IsOutputPortControlling(fanin);
   bool can_dedup_control_with_regular_input =
@@ -341,7 +345,7 @@ bool MutableGraphView::AddFaninInternal(NodeDef* node,
   // Don't add duplicate control dependencies.
   if (input_is_control) {
     const int start =
-        can_dedup_control_with_regular_input ? 0 : num_non_controlling_fanins;
+        can_dedup_control_with_regular_input ? 0 : num_regular_fanins;
     for (int i = start; i < node->input_size(); ++i) {
       if (ParseTensorName(node->input(i)).node() == fanin.node->name()) {
         return false;
@@ -351,17 +355,15 @@ bool MutableGraphView::AddFaninInternal(NodeDef* node,
 
   InputPort input;
   input.node = node;
-  input.port_id =
-      input_is_control ? Graph::kControlSlot : num_non_controlling_fanins;
+  input.port_id = input_is_control ? Graph::kControlSlot : num_regular_fanins;
 
   node->add_input(TensorIdToString({fanin.node->name(), fanin.port_id}));
-  if (IsOutputPortRegular(fanin)) {
+  if (!input_is_control) {
     int last_node_input = node->input_size() - 1;
     // If there are control dependencies in node, move newly inserted fanin to
     // be before such control dependencies.
-    if (num_non_controlling_fanins < last_node_input) {
-      node->mutable_input()->SwapElements(last_node_input,
-                                          num_non_controlling_fanins);
+    if (num_regular_fanins < last_node_input) {
+      node->mutable_input()->SwapElements(last_node_input, num_regular_fanins);
     }
   }
 
@@ -370,9 +372,12 @@ bool MutableGraphView::AddFaninInternal(NodeDef* node,
     max_regular_output_port()[fanin.node] = fanin.port_id;
   }
 
-  // Dedup control dependencies.
-  if (!input_is_control && can_dedup_control_with_regular_input) {
-    RemoveControllingFaninInternal(node, fanin.node);
+  // Update max input port and dedup control dependencies.
+  if (!input_is_control) {
+    max_regular_input_port()[node] = num_regular_fanins;
+    if (can_dedup_control_with_regular_input) {
+      RemoveControllingFaninInternal(node, fanin.node);
+    }
   }
 
   return true;
@@ -491,14 +496,12 @@ bool MutableGraphView::RemoveRegularFaninInternal(NodeDef* node,
 
   auto mutable_inputs = node->mutable_input();
   bool modified = false;
-  const int num_inputs = node->input_size();
+  const int num_regular_fanins =
+      NumFanins(*node, /*include_controlling_nodes=*/false);
   int i;
   int curr_pos = 0;
-  for (i = 0; i < num_inputs; ++i) {
+  for (i = 0; i < num_regular_fanins; ++i) {
     TensorId tensor_id = ParseTensorName(node->input(i));
-    if (IsTensorIdControlling(tensor_id)) {
-      break;
-    }
     if (tensor_id.node() == fanin.node->name() &&
         tensor_id.index() == fanin.port_id) {
       remove_input(fanin, i, /*update_max_port=*/true);
@@ -517,9 +520,17 @@ bool MutableGraphView::RemoveRegularFaninInternal(NodeDef* node,
     }
   }
 
-  if (modified && curr_pos < i) {
-    // Remove fanins from node inputs.
-    mutable_inputs->DeleteSubrange(curr_pos, i - curr_pos);
+  if (modified) {
+    const int last_regular_input_port = curr_pos - 1;
+    if (last_regular_input_port < 0) {
+      max_regular_input_port().erase(node);
+    } else {
+      max_regular_input_port()[node] = last_regular_input_port;
+    }
+    if (curr_pos < i) {
+      // Remove fanins from node inputs.
+      mutable_inputs->DeleteSubrange(curr_pos, i - curr_pos);
+    }
   }
 
   return modified;
@@ -601,14 +612,14 @@ Status MutableGraphView::RemoveAllFanins(absl::string_view node_name,
     return Status::OK();
   }
 
+  const int num_regular_fanins =
+      NumFanins(*node, /*include_controlling_nodes=*/false);
   RemoveFaninsInternal(node, keep_controlling_fanins);
   if (keep_controlling_fanins) {
-    int num_non_controlling_fanins =
-        NumFanins(*node, /*include_controlling_nodes=*/false);
-    if (num_non_controlling_fanins == 0) {
+    if (num_regular_fanins == 0) {
       return Status::OK();
-    } else if (num_non_controlling_fanins < node->input_size()) {
-      node->mutable_input()->DeleteSubrange(0, num_non_controlling_fanins);
+    } else if (num_regular_fanins < node->input_size()) {
+      node->mutable_input()->DeleteSubrange(0, num_regular_fanins);
     } else {
       node->clear_input();
     }
@@ -673,33 +684,27 @@ Status MutableGraphView::UpdateFanin(absl::string_view node_name,
     return Status::OK();
   }
 
-  // In place mutation, requires no shifting of ports.
+  // In place mutation of regular fanins, requires no shifting of ports.
   string to_fanin_string = TensorIdToString(to_fanin);
-  int num_inputs = node->input_size();
+  const int num_regular_fanins =
+      NumFanins(*node, /*include_controlling_nodes=*/false);
   bool modified = false;
   absl::flat_hash_set<InputPort>* from_fanin_port_fanouts = nullptr;
   absl::flat_hash_set<InputPort>* to_fanin_port_fanouts = nullptr;
-  for (int i = 0; i < num_inputs; ++i) {
+  for (int i = 0; i < num_regular_fanins; ++i) {
     if (ParseTensorName(node->input(i)) == from_fanin) {
-      InputPort old_input;
-      old_input.node = node;
-      old_input.port_id =
-          IsTensorIdControlling(from_fanin) ? Graph::kControlSlot : i;
+      InputPort input(node, i);
       if (from_fanin_port_fanouts == nullptr) {
         OutputPort from_fanin_port(from_fanin_node, from_fanin.index());
         from_fanin_port_fanouts = &fanouts()[from_fanin_port];
       }
-      from_fanin_port_fanouts->erase(old_input);
+      from_fanin_port_fanouts->erase(input);
 
-      InputPort new_input;
-      new_input.node = node;
-      new_input.port_id =
-          IsTensorIdControlling(to_fanin) ? Graph::kControlSlot : i;
       if (to_fanin_port_fanouts == nullptr) {
         OutputPort to_fanin_port(to_fanin_node, to_fanin.index());
         to_fanin_port_fanouts = &fanouts()[to_fanin_port];
       }
-      to_fanin_port_fanouts->insert(new_input);
+      to_fanin_port_fanouts->insert(input);
 
       node->set_input(i, to_fanin_string);
       modified = true;
@@ -824,14 +829,15 @@ void MutableGraphView::RemoveFaninsInternal(NodeDef* deleted_node,
                                             bool keep_controlling_fanins) {
   for (int i = 0; i < deleted_node->input_size(); ++i) {
     TensorId tensor_id = ParseTensorName(deleted_node->input(i));
-    if (keep_controlling_fanins && IsTensorIdControlling(tensor_id)) {
+    bool is_control = IsTensorIdControlling(tensor_id);
+    if (keep_controlling_fanins && is_control) {
       break;
     }
     OutputPort fanin(nodes()[tensor_id.node()], tensor_id.index());
 
     InputPort input;
     input.node = deleted_node;
-    input.port_id = IsTensorIdControlling(tensor_id) ? Graph::kControlSlot : i;
+    input.port_id = is_control ? Graph::kControlSlot : i;
 
     auto it = fanouts().find(fanin);
     if (it != fanouts().end()) {
@@ -840,11 +846,12 @@ void MutableGraphView::RemoveFaninsInternal(NodeDef* deleted_node,
       UpdateMaxRegularOutputPortForRemovedFanin(fanin, *fanouts_set);
     }
   }
+  max_regular_input_port().erase(deleted_node);
 }
 
 void MutableGraphView::RemoveFanoutsInternal(NodeDef* deleted_node) {
-  const int max_port = gtl::FindWithDefault(max_regular_output_port(),
-                                            deleted_node, Graph::kControlSlot);
+  const int max_port =
+      gtl::FindWithDefault(max_regular_output_port(), deleted_node, -1);
   for (int i = Graph::kControlSlot; i <= max_port; ++i) {
     fanouts().erase({deleted_node, i});
   }
-- 
GitLab


From cb40852a6ee804b12ab71aadb12babf96b090991 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 13:09:17 -0800
Subject: [PATCH 0987/2345] Adds a layer normalization layer to core (So users
 can switch away from tf.contrib.layers.layer_norm)

PiperOrigin-RevId: 229980536
---
 tensorflow/python/keras/layers/__init__.py    |   1 +
 .../python/keras/layers/normalization.py      | 224 ++++++++++++++++-
 .../python/keras/layers/normalization_test.py | 229 ++++++++++++++++++
 .../tools/api/generator/api_init_files.bzl    |   1 +
 .../tools/api/generator/api_init_files_v1.bzl |   1 +
 ...rs.experimental.-layer-normalization.pbtxt | 183 ++++++++++++++
 ...tensorflow.keras.layers.experimental.pbtxt |   7 +
 .../golden/v1/tensorflow.keras.layers.pbtxt   |   4 +
 ...rs.experimental.-layer-normalization.pbtxt | 183 ++++++++++++++
 ...tensorflow.keras.layers.experimental.pbtxt |   7 +
 .../golden/v2/tensorflow.keras.layers.pbtxt   |   4 +
 11 files changed, 842 insertions(+), 2 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.-layer-normalization.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.-layer-normalization.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.pbtxt

diff --git a/tensorflow/python/keras/layers/__init__.py b/tensorflow/python/keras/layers/__init__.py
index 285388f340..dec9673b5b 100644
--- a/tensorflow/python/keras/layers/__init__.py
+++ b/tensorflow/python/keras/layers/__init__.py
@@ -110,6 +110,7 @@ from tensorflow.python.keras.layers.noise import GaussianDropout
 
 # Normalization layers.
 from tensorflow.python.keras.layers.normalization import BatchNormalization
+from tensorflow.python.keras.layers.normalization import LayerNormalization
 
 # Pooling layers.
 from tensorflow.python.keras.layers.pooling import MaxPooling1D
diff --git a/tensorflow/python/keras/layers/normalization.py b/tensorflow/python/keras/layers/normalization.py
index 171e1e231e..5008bd7714 100644
--- a/tensorflow/python/keras/layers/normalization.py
+++ b/tensorflow/python/keras/layers/normalization.py
@@ -90,8 +90,7 @@ class BatchNormalizationV2(Layer):
       if the fused implementation cannot be used. If `None`, use the faster
       implementation if possible. If False, do not used the fused
       implementation.
-    trainable: Boolean, if `True` also add variables to the graph collection
-      `GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable).
+    trainable: Boolean, if `True` the variables will be marked as trainable.
     virtual_batch_size: An `int`. By default, `virtual_batch_size` is `None`,
       which means batch normalization is performed across the whole batch. When
       `virtual_batch_size` is not `None`, instead perform "Ghost Batch
@@ -798,3 +797,224 @@ if tf2.enabled():
   enable_v2_batch_normalization()
 else:
   disable_v2_batch_normalization()
+
+
+@keras_export('keras.layers.experimental.LayerNormalization')
+class LayerNormalization(Layer):
+  """Layer normalization layer (Ba et al., 2016).
+
+  Normalize the activations of the previous layer for each given example in a
+  batch independently, rather than across a batch like Batch Normalization.
+  i.e. applies a transformation that maintains the mean activation within each
+  example close to 0 and the activation standard deviation close to 1.
+
+  Given a tensor `inputs` of rank `R`, moments are calculated and normalization
+  is performed over all axes in norm_axis.  Scaling and centering,
+  if requested, is performed over all axes in params_axis.
+
+  By default, normalization is performed over all but the first axis
+  (the `HWC` if `inputs` is `NHWC`), while the `beta` and `gamma` trainable
+  parameters are calculated for the rightmost axis (the `C` if `inputs` is
+  `NHWC`).  Scaling and recentering is performed via broadcast of the
+  `beta` and `gamma` parameters with the normalized tensor.
+
+  The shapes of `beta` and `gamma` are
+  `[inputs.shape[i] for i in (param axes)]`,
+  and this part of the inputs' shape must be fully defined.
+
+  Arguments:
+    norm_axis: Integer or List. normalization will be
+      performed along these dimensions. If unspecified (None), it will default
+      to the dimensions `begin_norm_axis : rank(inputs)`
+    params_axis: Integer or List. The (beta, gamma) dimensions: scale
+      and centering parameters will have take their shapes from these axes and
+      will be broadcast with the normalized inputs accordingly. If unspecified
+      (None), it will default to the last dimension
+    epsilon: Small float added to variance to avoid dividing by zero.
+    center: If True, add offset of `beta` to normalized tensor.
+        If False, `beta` is ignored.
+    scale: If True, multiply by `gamma`.
+        If False, `gamma` is not used.
+        When the next layer is linear (also e.g. `nn.relu`),
+        this can be disabled since the scaling
+        will be done by the next layer.
+    beta_initializer: Initializer for the beta weight.
+    gamma_initializer: Initializer for the gamma weight.
+    beta_regularizer: Optional regularizer for the beta weight.
+    gamma_regularizer: Optional regularizer for the gamma weight.
+    beta_constraint: Optional constraint for the beta weight.
+    gamma_constraint: Optional constraint for the gamma weight.
+    trainable: Boolean, if `True` the variables will be marked as trainable.
+
+  Input shape:
+      Arbitrary. Use the keyword argument `input_shape`
+      (tuple of integers, does not include the samples axis)
+      when using this layer as the first layer in a model.
+
+  Output shape:
+      Same shape as input.
+
+  References:
+      - [Layer Normalization](https://arxiv.org/abs/1607.06450)
+  """
+
+  def __init__(self,
+               norm_axis=None,
+               params_axis=-1,
+               epsilon=1e-12,
+               center=True,
+               scale=True,
+               beta_initializer='zeros',
+               gamma_initializer='ones',
+               beta_regularizer=None,
+               gamma_regularizer=None,
+               beta_constraint=None,
+               gamma_constraint=None,
+               trainable=True,
+               name=None,
+               **kwargs):
+    super(LayerNormalization, self).__init__(
+        name=name, trainable=trainable, **kwargs)
+    if isinstance(norm_axis, list):
+      self.norm_axis = norm_axis[:]
+    elif isinstance(norm_axis, int):
+      self.norm_axis = norm_axis
+    elif norm_axis is None:
+      self.norm_axis = None
+    else:
+      raise TypeError('norm_axis must be int or list or None, type given: %s'
+                      % type(norm_axis))
+
+    if isinstance(params_axis, list):
+      self.params_axis = params_axis[:]
+    elif isinstance(params_axis, int):
+      self.params_axis = params_axis
+    else:
+      raise TypeError('params_axis must be int or list, type given: %s'
+                      % type(params_axis))
+
+    self.epsilon = epsilon
+    self.center = center
+    self.scale = scale
+    self.beta_initializer = initializers.get(beta_initializer)
+    self.gamma_initializer = initializers.get(gamma_initializer)
+    self.beta_regularizer = regularizers.get(beta_regularizer)
+    self.gamma_regularizer = regularizers.get(gamma_regularizer)
+    self.beta_constraint = constraints.get(beta_constraint)
+    self.gamma_constraint = constraints.get(gamma_constraint)
+
+    self.supports_masking = True
+
+  def build(self, input_shape):
+    ndims = len(input_shape)
+    if ndims is None:
+      raise ValueError('Input shape %s has undefined rank.' % input_shape)
+
+    # Handle an unspecified norm_axis
+    if self.norm_axis is None:
+      self.norm_axis = list(range(1, ndims))
+
+    # Convert axes to lists and resolve negatives
+    if isinstance(self.norm_axis, int):
+      self.norm_axis = [self.norm_axis]
+    for idx, x in enumerate(self.norm_axis):
+      if x < 0:
+        self.norm_axis[idx] = ndims + x
+
+    if isinstance(self.params_axis, int):
+      self.params_axis = [self.params_axis]
+    for idx, x in enumerate(self.params_axis):
+      if x < 0:
+        self.params_axis[idx] = ndims + x
+
+    # Validate axes
+    for x in self.norm_axis:
+      if x < 0 or x >= ndims:
+        raise ValueError('Invalid axis: %d' % x)
+    if len(self.norm_axis) != len(set(self.norm_axis)):
+      raise ValueError('Duplicate axis: %s' % self.norm_axis)
+
+    for x in self.params_axis:
+      if x < 0 or x >= ndims:
+        raise ValueError('Invalid axis: %d' % x)
+    if len(self.params_axis) != len(set(self.params_axis)):
+      raise ValueError('Duplicate axis: %s' % self.params_axis)
+
+    param_shape = [input_shape[dim] for dim in self.params_axis]
+
+    if self.scale:
+      self.gamma = self.add_weight(
+          name='gamma',
+          shape=param_shape,
+          initializer=self.gamma_initializer,
+          regularizer=self.gamma_regularizer,
+          constraint=self.gamma_constraint,
+          trainable=True)
+    else:
+      self.gamma = None
+
+    if self.center:
+      self.beta = self.add_weight(
+          name='beta',
+          shape=param_shape,
+          initializer=self.beta_initializer,
+          regularizer=self.beta_regularizer,
+          constraint=self.beta_constraint,
+          trainable=True)
+    else:
+      self.beta = None
+
+  def call(self, inputs):
+    # Compute the axes along which to reduce the mean / variance
+    input_shape = inputs.get_shape()
+    ndims = len(input_shape)
+
+    # Calculate the moments on the last axis (layer activations).
+    mean, variance = nn.moments(inputs, self.norm_axis, keep_dims=True)
+
+    # Broadcasting only necessary for norm where the params axes aren't just
+    # the last dimension
+    broadcast_shape = [1] * ndims
+    for dim in self.params_axis:
+      broadcast_shape[dim] = input_shape.dims[dim].value
+    def _broadcast(v):
+      if (v is not None and
+          len(v.get_shape()) != ndims and
+          self.params_axis != [ndims - 1]):
+        return array_ops.reshape(v, broadcast_shape)
+      return v
+    scale, offset = _broadcast(self.gamma), _broadcast(self.beta)
+
+    # Compute layer normalization using the batch_normalization function.
+    outputs = nn.batch_normalization(
+        inputs,
+        mean,
+        variance,
+        offset=offset,
+        scale=scale,
+        variance_epsilon=self.epsilon)
+
+    # If some components of the shape got lost due to adjustments, fix that.
+    outputs.set_shape(input_shape)
+
+    return outputs
+
+  def compute_output_shape(self, input_shape):
+    return input_shape
+
+  def get_config(self):
+    config = {
+        'norm_axis': self.norm_axis,
+        'params_axis': self.params_axis,
+        'epsilon': self.epsilon,
+        'center': self.center,
+        'scale': self.scale,
+        'beta_initializer': initializers.serialize(self.beta_initializer),
+        'gamma_initializer': initializers.serialize(self.gamma_initializer),
+        'beta_regularizer': regularizers.serialize(self.beta_regularizer),
+        'gamma_regularizer': regularizers.serialize(self.gamma_regularizer),
+        'beta_constraint': constraints.serialize(self.beta_constraint),
+        'gamma_constraint': constraints.serialize(self.gamma_constraint)
+    }
+    base_config = super(LayerNormalization, self).get_config()
+    return dict(list(base_config.items()) + list(config.items()))
diff --git a/tensorflow/python/keras/layers/normalization_test.py b/tensorflow/python/keras/layers/normalization_test.py
index f81ddcecb4..4308e0f724 100644
--- a/tensorflow/python/keras/layers/normalization_test.py
+++ b/tensorflow/python/keras/layers/normalization_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python import keras
+from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import test_util as tf_test_util
 from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import testing_utils
@@ -363,5 +364,233 @@ class NormalizationLayersGraphModeOnlyTest(test.TestCase):
           (val_a - np.mean(val_a)) / np.std(val_a), out, atol=1e-3)
 
 
+def _run_layernorm_correctness_test(layer, dtype='float32'):
+  model = keras.models.Sequential()
+  norm = layer(input_shape=(2, 2, 2))
+  model.add(norm)
+  model.compile(loss='mse',
+                optimizer=gradient_descent.GradientDescentOptimizer(0.01),
+                run_eagerly=testing_utils.should_run_eagerly())
+
+  # centered on 5.0, variance 10.0
+  x = (np.random.normal(loc=5.0, scale=10.0, size=(1000, 2, 2, 2))
+       .astype(dtype))
+  model.fit(x, x, epochs=4, verbose=0)
+  out = model.predict(x)
+  out -= keras.backend.eval(norm.beta)
+  out /= keras.backend.eval(norm.gamma)
+
+  np.testing.assert_allclose(out.mean(), 0.0, atol=1e-1)
+  np.testing.assert_allclose(out.std(), 1.0, atol=1e-1)
+
+
+class LayerNormalizationTest(keras_parameterized.TestCase):
+
+  @keras_parameterized.run_all_keras_modes
+  def test_basic_layernorm(self):
+    testing_utils.layer_test(
+        keras.layers.LayerNormalization,
+        kwargs={
+            'gamma_regularizer': keras.regularizers.l2(0.01),
+            'beta_regularizer': keras.regularizers.l2(0.01)
+        },
+        input_shape=(3, 4, 2))
+    testing_utils.layer_test(
+        keras.layers.LayerNormalization,
+        kwargs={
+            'gamma_initializer': 'ones',
+            'beta_initializer': 'ones',
+        },
+        input_shape=(3, 4, 2))
+    testing_utils.layer_test(
+        keras.layers.LayerNormalization,
+        kwargs={'scale': False,
+                'center': False},
+        input_shape=(3, 3))
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def test_layernorm_weights(self):
+    layer = keras.layers.LayerNormalization(scale=False, center=False)
+    layer.build((None, 3, 4))
+    self.assertEqual(len(layer.trainable_weights), 0)
+    self.assertEqual(len(layer.weights), 0)
+
+    layer = keras.layers.LayerNormalization()
+    layer.build((None, 3, 4))
+    self.assertEqual(len(layer.trainable_weights), 2)
+    self.assertEqual(len(layer.weights), 2)
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def test_layernorm_regularization(self):
+    layer = keras.layers.LayerNormalization(
+        gamma_regularizer='l1', beta_regularizer='l1')
+    layer.build((None, 3, 4))
+    self.assertEqual(len(layer.losses), 2)
+    max_norm = keras.constraints.max_norm
+    layer = keras.layers.LayerNormalization(
+        gamma_constraint=max_norm, beta_constraint=max_norm)
+    layer.build((None, 3, 4))
+    self.assertEqual(layer.gamma.constraint, max_norm)
+    self.assertEqual(layer.beta.constraint, max_norm)
+
+  @keras_parameterized.run_all_keras_modes
+  def test_layernorm_convnet(self):
+    if test.is_gpu_available(cuda_only=True):
+      with self.session(use_gpu=True):
+        model = keras.models.Sequential()
+        norm = keras.layers.LayerNormalization(input_shape=(3, 4, 4))
+        model.add(norm)
+        model.compile(loss='mse',
+                      optimizer=gradient_descent.GradientDescentOptimizer(0.01),
+                      run_eagerly=testing_utils.should_run_eagerly())
+
+        # centered on 5.0, variance 10.0
+        x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 3, 4, 4))
+        model.fit(x, x, epochs=4, verbose=0)
+        out = model.predict(x)
+        out -= np.reshape(keras.backend.eval(norm.beta), (1, 3, 1, 1))
+        out /= np.reshape(keras.backend.eval(norm.gamma), (1, 3, 1, 1))
+
+        np.testing.assert_allclose(np.mean(out, axis=(0, 2, 3)), 0.0, atol=1e-1)
+        np.testing.assert_allclose(np.std(out, axis=(0, 2, 3)), 1.0, atol=1e-1)
+
+  @keras_parameterized.run_all_keras_modes
+  def test_layernorm_convnet_channel_last(self):
+    model = keras.models.Sequential()
+    norm = keras.layers.LayerNormalization(input_shape=(4, 4, 3))
+    model.add(norm)
+    model.compile(loss='mse',
+                  optimizer=gradient_descent.GradientDescentOptimizer(0.01),
+                  run_eagerly=testing_utils.should_run_eagerly())
+
+    # centered on 5.0, variance 10.0
+    x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 4, 4, 3))
+    model.fit(x, x, epochs=4, verbose=0)
+    out = model.predict(x)
+    out -= np.reshape(keras.backend.eval(norm.beta), (1, 1, 1, 3))
+    out /= np.reshape(keras.backend.eval(norm.gamma), (1, 1, 1, 3))
+
+    np.testing.assert_allclose(np.mean(out, axis=(0, 1, 2)), 0.0, atol=1e-1)
+    np.testing.assert_allclose(np.std(out, axis=(0, 1, 2)), 1.0, atol=1e-1)
+
+  @keras_parameterized.run_all_keras_modes
+  def test_layernorm_correctness(self):
+    _run_layernorm_correctness_test(
+        normalization.LayerNormalization, dtype='float32')
+
+  @keras_parameterized.run_all_keras_modes
+  def test_layernorm_mixed_precision(self):
+    _run_layernorm_correctness_test(
+        normalization.LayerNormalization, dtype='float16')
+
+  def doOutputTest(self,
+                   input_shape,
+                   tol=1e-5,
+                   norm_axis=None,
+                   params_axis=-1,
+                   dtype=None):
+    ndim = len(input_shape)
+    if norm_axis is None:
+      moments_axis = range(1, ndim)
+    elif isinstance(norm_axis, int):
+      if norm_axis < 0:
+        moments_axis = [norm_axis + ndim]
+      else:
+        moments_axis = [norm_axis]
+    else:
+      moments_axis = []
+      for dim in norm_axis:
+        if dim < 0:
+          dim = dim + ndim
+        moments_axis.append(dim)
+
+    moments_axis = tuple(moments_axis)
+    expected_shape = []
+    for i in range(ndim):
+      if i not in moments_axis:
+        expected_shape.append(input_shape[i])
+
+    expected_mean = np.zeros(expected_shape)
+    expected_var = np.ones(expected_shape)
+    for mu in [0.0, 1e2]:
+      for sigma in [1.0, 0.1]:
+        inputs = np.random.randn(*input_shape) * sigma + mu
+        inputs_t = constant_op.constant(inputs, shape=input_shape)
+        layer = normalization.LayerNormalization(
+            norm_axis=norm_axis, params_axis=params_axis, dtype=dtype)
+        outputs = layer(inputs_t)
+        beta = layer.beta
+        gamma = layer.gamma
+        for weight in layer.weights:
+          self.evaluate(weight.initializer)
+        outputs = self.evaluate(outputs)
+        beta = self.evaluate(beta)
+        gamma = self.evaluate(gamma)
+
+        # The mean and variance of the output should be close to 0 and 1
+        # respectively.
+
+        # Make sure that there are no NaNs
+        self.assertFalse(np.isnan(outputs).any())
+        mean = np.mean(outputs, axis=moments_axis)
+        var = np.var(outputs, axis=moments_axis)
+        # Layer-norm implemented in numpy
+        eps = 1e-12
+        expected_out = (
+            (gamma * (inputs - np.mean(
+                inputs, axis=moments_axis, keepdims=True)) /
+             np.sqrt(eps + np.var(
+                 inputs, axis=moments_axis, keepdims=True))) + beta)
+        self.assertAllClose(expected_mean, mean, atol=tol, rtol=tol)
+        self.assertAllClose(expected_var, var, atol=tol)
+        # The full computation gets a bigger tolerance
+        self.assertAllClose(expected_out, outputs, atol=5 * tol)
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def testOutput2DInput(self):
+    self.doOutputTest((10, 300))
+    self.doOutputTest((10, 300), norm_axis=[0])
+    self.doOutputTest((10, 300), params_axis=[0, 1])
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def testOutput2DInputDegenerateNormAxis(self):
+    with self.assertRaisesRegexp(ValueError, r'Invalid axis: 2'):
+      self.doOutputTest((10, 300), norm_axis=2)
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def testOutput4DInput(self):
+    self.doOutputTest((100, 10, 10, 3))
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def testOutput4DInputNormOnInnermostAxis(self):
+    # Equivalent tests
+    shape = (100, 10, 10, 3)
+    self.doOutputTest(
+        shape, norm_axis=list(range(3, len(shape))), tol=1e-4, dtype='float64')
+    self.doOutputTest(shape, norm_axis=-1, tol=1e-4, dtype='float64')
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def testOutputSmallInput(self):
+    self.doOutputTest((10, 10, 10, 30))
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def testOutputSmallInputNormOnInnermostAxis(self):
+    self.doOutputTest((10, 10, 10, 30), norm_axis=3)
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def testOutputSmallInputNormOnMixedAxes(self):
+    self.doOutputTest((10, 10, 10, 30), norm_axis=[0, 3])
+    self.doOutputTest((10, 10, 10, 30), params_axis=[-2, -1])
+    self.doOutputTest((10, 10, 10, 30), norm_axis=[0, 3],
+                      params_axis=[-3, -2, -1])
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def testOutputBigInput(self):
+    self.doOutputTest((1, 100, 100, 1))
+    self.doOutputTest((1, 100, 100, 1), norm_axis=[1, 2])
+    self.doOutputTest((1, 100, 100, 1), norm_axis=[1, 2],
+                      params_axis=[-2, -1])
+
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/tools/api/generator/api_init_files.bzl b/tensorflow/python/tools/api/generator/api_init_files.bzl
index 5182f53beb..7637cbd584 100644
--- a/tensorflow/python/tools/api/generator/api_init_files.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files.bzl
@@ -77,6 +77,7 @@ KERAS_API_INIT_FILES = [
     "keras/experimental/__init__.py",
     "keras/initializers/__init__.py",
     "keras/layers/__init__.py",
+    "keras/layers/experimental/__init__.py",
     "keras/losses/__init__.py",
     "keras/metrics/__init__.py",
     "keras/models/__init__.py",
diff --git a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
index 3e2c00c87f..5db84b1d0b 100644
--- a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
@@ -100,6 +100,7 @@ KERAS_API_INIT_FILES_V1 = [
     "keras/experimental/__init__.py",
     "keras/initializers/__init__.py",
     "keras/layers/__init__.py",
+    "keras/layers/experimental/__init__.py",
     "keras/losses/__init__.py",
     "keras/metrics/__init__.py",
     "keras/models/__init__.py",
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.-layer-normalization.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.-layer-normalization.pbtxt
new file mode 100644
index 0000000000..476c597fe6
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.-layer-normalization.pbtxt
@@ -0,0 +1,183 @@
+path: "tensorflow.keras.layers.experimental.LayerNormalization"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.layers.normalization.LayerNormalization\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'norm_axis\', \'params_axis\', \'epsilon\', \'center\', \'scale\', \'beta_initializer\', \'gamma_initializer\', \'beta_regularizer\', \'gamma_regularizer\', \'beta_constraint\', \'gamma_constraint\', \'trainable\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'-1\', \'1e-12\', \'True\', \'True\', \'zeros\', \'ones\', \'None\', \'None\', \'None\', \'None\', \'True\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.pbtxt
new file mode 100644
index 0000000000..0f22961546
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.pbtxt
@@ -0,0 +1,7 @@
+path: "tensorflow.keras.layers.experimental"
+tf_module {
+  member {
+    name: "LayerNormalization"
+    mtype: "<type \'type\'>"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.pbtxt
index 9d7e5bb8c7..2c5fd644d4 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.pbtxt
@@ -396,6 +396,10 @@ tf_module {
     name: "ZeroPadding3D"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "experimental"
+    mtype: "<type \'module\'>"
+  }
   member_method {
     name: "Input"
     argspec: "args=[\'shape\', \'batch_size\', \'name\', \'dtype\', \'sparse\', \'tensor\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'False\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.-layer-normalization.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.-layer-normalization.pbtxt
new file mode 100644
index 0000000000..476c597fe6
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.-layer-normalization.pbtxt
@@ -0,0 +1,183 @@
+path: "tensorflow.keras.layers.experimental.LayerNormalization"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.layers.normalization.LayerNormalization\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'norm_axis\', \'params_axis\', \'epsilon\', \'center\', \'scale\', \'beta_initializer\', \'gamma_initializer\', \'beta_regularizer\', \'gamma_regularizer\', \'beta_constraint\', \'gamma_constraint\', \'trainable\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'-1\', \'1e-12\', \'True\', \'True\', \'zeros\', \'ones\', \'None\', \'None\', \'None\', \'None\', \'True\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.pbtxt
new file mode 100644
index 0000000000..0f22961546
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.pbtxt
@@ -0,0 +1,7 @@
+path: "tensorflow.keras.layers.experimental"
+tf_module {
+  member {
+    name: "LayerNormalization"
+    mtype: "<type \'type\'>"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.pbtxt
index e84c9a2a8f..5cf80b97ae 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.pbtxt
@@ -396,6 +396,10 @@ tf_module {
     name: "ZeroPadding3D"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "experimental"
+    mtype: "<type \'module\'>"
+  }
   member_method {
     name: "Input"
     argspec: "args=[\'shape\', \'batch_size\', \'name\', \'dtype\', \'sparse\', \'tensor\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'False\', \'None\'], "
-- 
GitLab


From 07f552d779abdb4b90ed66e13e2eeef465f2c47a Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Fri, 18 Jan 2019 13:09:34 -0800
Subject: [PATCH 0988/2345] Add a way to check if a function exists in the
 eager context

Simplifies SavedModel loading slightly

PiperOrigin-RevId: 229980590
---
 tensorflow/c/eager/c_api.cc                    |  4 ++++
 tensorflow/c/eager/c_api.h                     |  4 ++++
 tensorflow/python/eager/context.py             |  4 ++++
 tensorflow/python/eager/pywrap_tfe_src.cc      |  8 ++++----
 tensorflow/python/eager/pywrap_tfe_test.py     | 11 +++++++++++
 .../python/framework/function_def_to_graph.py  | 13 +++++++++++--
 tensorflow/python/pywrap_tfe.i                 |  8 ++++++++
 .../saved_model/function_deserialization.py    | 18 +++++-------------
 8 files changed, 51 insertions(+), 19 deletions(-)

diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index f0408677d4..e2574d5311 100755
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -754,6 +754,10 @@ void TFE_ContextAddFunction(TFE_Context* ctx, TF_Function* function,
   status->status = ctx->context.AddFunctionDef(function->fdef);
 }
 
+unsigned char TFE_ContextHasFunction(TFE_Context* ctx, const char* name) {
+  return ctx->context.FindFunctionDef(name) != nullptr;
+}
+
 void TFE_ContextEnableRunMetadata(TFE_Context* ctx) {
   ctx->context.SetShouldStoreMetadata(true);
 }
diff --git a/tensorflow/c/eager/c_api.h b/tensorflow/c/eager/c_api.h
index 120748ab76..044dfb7415 100755
--- a/tensorflow/c/eager/c_api.h
+++ b/tensorflow/c/eager/c_api.h
@@ -393,6 +393,10 @@ TF_CAPI_EXPORT extern void TFE_ContextAddFunction(TFE_Context* ctx,
                                                   TF_Function* function,
                                                   TF_Status* status);
 
+// Checks whether a function is registered under `name`.
+TF_CAPI_EXPORT unsigned char TFE_ContextHasFunction(TFE_Context* ctx,
+                                                    const char* name);
+
 // Enables tracing of RunMetadata on the ops executed from this context.
 TF_CAPI_EXPORT extern void TFE_ContextEnableRunMetadata(TFE_Context* ctx);
 
diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py
index 6c5b191cf9..33551fccd5 100644
--- a/tensorflow/python/eager/context.py
+++ b/tensorflow/python/eager/context.py
@@ -643,6 +643,10 @@ class Context(object):
     pywrap_tensorflow.TFE_ContextAddFunctionDef(
         self._handle, fdef_string, len(fdef_string))
 
+  def has_function(self, name):
+    """Check if a function `name` is registered."""
+    return bool(pywrap_tensorflow.TFE_ContextHasFunction(self._handle, name))
+
   def add_post_execution_callback(self, callback):
     """Add a post-execution callback to the context.
 
diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc
index 9ce500bc08..da1bb24fc7 100644
--- a/tensorflow/python/eager/pywrap_tfe_src.cc
+++ b/tensorflow/python/eager/pywrap_tfe_src.cc
@@ -835,15 +835,15 @@ int MaybeRaiseExceptionFromStatus(const tensorflow::Status& status,
 }
 
 const char* TFE_GetPythonString(PyObject* o) {
+#if PY_MAJOR_VERSION >= 3
   if (PyBytes_Check(o)) {
     return PyBytes_AsString(o);
-  }
-#if PY_MAJOR_VERSION >= 3
-  if (PyUnicode_Check(o)) {
+  } else {
     return PyUnicode_AsUTF8(o);
   }
+#else
+  return PyBytes_AsString(o);
 #endif
-  return nullptr;
 }
 
 int64_t get_uid() {
diff --git a/tensorflow/python/eager/pywrap_tfe_test.py b/tensorflow/python/eager/pywrap_tfe_test.py
index 2abd4678af..445ffbc532 100644
--- a/tensorflow/python/eager/pywrap_tfe_test.py
+++ b/tensorflow/python/eager/pywrap_tfe_test.py
@@ -22,6 +22,7 @@ from tensorflow.python import pywrap_tensorflow
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
 from tensorflow.python.eager import core
+from tensorflow.python.eager import def_function
 from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -259,6 +260,16 @@ class Tests(test.TestCase):
         "Value for attr 'num_split' of 0 must be at least minimum 1"):
       array_ops.split(value=[1, 2, 3], num_or_size_splits=0)
 
+  def testIsFunction(self):
+    ctx = context.context()
+    self.assertFalse(ctx.has_function("not_a_function"))
+
+    @def_function.function
+    def f():
+      return 1.
+
+    self.assertTrue(ctx.has_function(f.get_concrete_function().name))
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/framework/function_def_to_graph.py b/tensorflow/python/framework/function_def_to_graph.py
index 10ad7ad541..aa670f1e37 100644
--- a/tensorflow/python/framework/function_def_to_graph.py
+++ b/tensorflow/python/framework/function_def_to_graph.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 from tensorflow.core.framework import graph_pb2
 from tensorflow.core.framework import types_pb2
 from tensorflow.core.framework import versions_pb2
+from tensorflow.python.eager import context
 from tensorflow.python.framework import importer
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import versions
@@ -76,6 +77,14 @@ def function_def_to_graph(fdef, input_shapes=None):
   return func_graph
 
 
+def _is_function(fname):
+  """Checks for a function definition with `fname` in the current context."""
+  if context.executing_eagerly():
+    return context.context().has_function(fname)
+  else:
+    return ops.get_default_graph()._is_function(fname)  # pylint: disable=protected-access
+
+
 def function_def_to_graph_def(fdef, input_shapes=None):
   """Convert a FunctionDef to a GraphDef.
 
@@ -147,12 +156,12 @@ def function_def_to_graph_def(fdef, input_shapes=None):
     for attr in op_def.attr:
       if attr.type == "func":
         fname = node_def.attr[attr.name].func.name
-        if not ops.get_default_graph()._is_function(fname):  # pylint: disable=protected-access
+        if not _is_function(fname):
           raise ValueError("%s function not found." % fname)
       elif attr.type == "list(func)":
         for fn in node_def.attr[attr.name].list.func:
           fname = fn.name
-          if not ops.get_default_graph()._is_function(fname):  # pylint: disable=protected-access
+          if not _is_function(fname):
             raise ValueError("%s function not found." % fname)
 
     # Iterate over output_args in op_def to build the map.
diff --git a/tensorflow/python/pywrap_tfe.i b/tensorflow/python/pywrap_tfe.i
index d0ba244853..5f6fbdcb5e 100755
--- a/tensorflow/python/pywrap_tfe.i
+++ b/tensorflow/python/pywrap_tfe.i
@@ -22,6 +22,7 @@ limitations under the License.
 %rename("%s") TFE_ContextListDevices;
 %rename("%s") TFE_ContextAddFunction;
 %rename("%s") TFE_ContextAddFunctionDef;
+%rename("%s") TFE_ContextHasFunction;
 %rename("%s") TFE_ContextEnableRunMetadata;
 %rename("%s") TFE_ContextDisableRunMetadata;
 %rename("%s") TFE_ContextExportRunMetadata;
@@ -140,6 +141,13 @@ limitations under the License.
   $1 = const_cast<char*>(TFE_GetPythonString($input));
 }
 
+// For const parameters in a function, SWIG pretty much ignores the const.
+// See: http://www.swig.org/Doc2.0/SWIG.html#SWIG_nn13
+// Hence the 'const_cast'.
+%typemap(in) const char* name {
+  $1 = const_cast<char*>(TFE_GetPythonString($input));
+}
+
 %typemap(in) (TFE_Context*) {
   $1 = (TFE_Context*)PyCapsule_GetPointer($input, nullptr);
 
diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py
index f6427864bb..934e189f78 100644
--- a/tensorflow/python/saved_model/function_deserialization.py
+++ b/tensorflow/python/saved_model/function_deserialization.py
@@ -164,22 +164,14 @@ def load_function_def_library(library):
   """
   functions = {}
 
-  # Note: Use a new graph to allow function_def_to_graph to help validating
-  # that the functions are loaded correctly. This is not possible to do
-  # just in eager mode as there is no python API to find if a function has
-  # been registered in eager. Note also that despite this the created
-  # func_graphs can still be used in eager or in other graphs.
-  import_graph = ops.Graph()
-
   for fdef in _sort_function_defs(library):
-    with import_graph.as_default():
-      copy = _fix_fdef(fdef, functions)
+    copy = _fix_fdef(fdef, functions)
 
-      func_graph = function_def_lib.function_def_to_graph(copy)
-      func = function_lib.ConcreteFunction(func_graph)
-      func.add_to_graph(import_graph)
+    func_graph = function_def_lib.function_def_to_graph(copy)
+    func = function_lib.ConcreteFunction(func_graph)
+    func.add_to_graph()
 
-      functions[fdef.signature.name] = func
+    functions[fdef.signature.name] = func
 
     # Also register the gradients in the current root context.
     with ops.init_scope():
-- 
GitLab


From 7a283b835b80808a9cdc6043998f29c8188342a5 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Fri, 18 Jan 2019 13:12:18 -0800
Subject: [PATCH 0989/2345] [XLA] Fix invalid return type for Real() of C128 in
 HLO evaluator. Add a test.

PiperOrigin-RevId: 229981105
---
 tensorflow/compiler/xla/service/hlo_evaluator.cc    |  2 +-
 .../compiler/xla/service/hlo_evaluator_test.cc      | 13 +++++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index 1d9af45745..a40ded76cf 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -542,7 +542,7 @@ Status HloEvaluator::HandleReal(HloInstruction* real) {
       break;
     }
     case C128: {
-      auto result_or = ElementWiseUnaryOpImpl<float, complex128>(
+      auto result_or = ElementWiseUnaryOpImpl<double, complex128>(
           real, [](complex128 elem_operand) { return std::real(elem_operand); },
           GetEvaluatedLiteralFor(operand));
       TF_ASSIGN_OR_RETURN(evaluated_[real], std::move(result_or));
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
index d34fa48efb..590f76f472 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
@@ -308,6 +308,19 @@ TEST_F(HloEvaluatorTest, DoesNotR2) {
                                     {0, std::numeric_limits<int>::min()}});
   TestUnaryOp(HloOpcode::kNot, std::move(expected), std::move(operand));
 }
+
+TEST_F(HloEvaluatorTest, DoesRealC128) {
+  auto x = LiteralUtil::CreateR1<complex128>({{1, 0}, {-100, 4}});
+  auto expected_real = LiteralUtil::CreateR1<double>({1, -100});
+  TestUnaryOp(HloOpcode::kReal, std::move(expected_real), std::move(x));
+}
+
+TEST_F(HloEvaluatorTest, DoesImagC128) {
+  auto x = LiteralUtil::CreateR1<complex128>({{1, 0}, {-100, 4}});
+  auto expected_imag = LiteralUtil::CreateR1<double>({0, 4});
+  TestUnaryOp(HloOpcode::kImag, std::move(expected_imag), std::move(x));
+}
+
 // Verifies that HloEvaluator evaluates a HLO Computation with non-parameter nor
 // constant operands.
 TEST_F(HloEvaluatorTest, DoesTraverseInstructions) {
-- 
GitLab


From 5ee41c266b389c54fddb73a94db22c9f5259374e Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Fri, 18 Jan 2019 13:36:30 -0800
Subject: [PATCH 0990/2345] Fix check for loggable buffers.

PiperOrigin-RevId: 229985133
---
 tensorflow/lite/tools/optimize/calibrator.cc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/tools/optimize/calibrator.cc b/tensorflow/lite/tools/optimize/calibrator.cc
index 0e817f9346..ccb55c3081 100644
--- a/tensorflow/lite/tools/optimize/calibrator.cc
+++ b/tensorflow/lite/tools/optimize/calibrator.cc
@@ -199,8 +199,10 @@ std::vector<int> GetLoggableTensorIndices(
   for (auto tensor_index : tensor_indices) {
     auto tensor = tensors->Get(tensor_index);
     auto buffer_index = tensor->buffer();
-    bool has_no_buffer =
-        buffer_index == 0 || (tensor_buffers->Get(buffer_index) == nullptr);
+    const bool has_no_buffer =
+        (tensor_buffers->Get(buffer_index) == nullptr) ||
+        (tensor_buffers->Get(buffer_index)->data() == nullptr) ||
+        (tensor_buffers->Get(buffer_index)->data()->size() == 0);
     if (has_no_buffer && tensor->type() == tflite::TensorType_FLOAT32) {
       loggable.push_back(tensor_index);
     }
-- 
GitLab


From 8c9dfb62e40bbee2dfeab6c3c9ca633c95002ffd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 13:45:43 -0800
Subject: [PATCH 0991/2345] Update sequence input layer to new FeatureColumn
 API.

PiperOrigin-RevId: 229986930
---
 tensorflow/contrib/feature_column/BUILD       |  18 +-
 .../sequence_feature_column_v2.py             | 232 +++++++-------
 ...ence_feature_column_v2_integration_test.py | 283 ++++++++++++++++++
 .../sequence_feature_column_v2_test.py        | 201 +++++++------
 tensorflow/python/feature_column/BUILD        |   1 +
 .../feature_column/feature_column_v2.py       | 163 ++++++----
 .../feature_column/feature_column_v2_test.py  |   4 +-
 ...sorflow.keras.layers.-dense-features.pbtxt |   1 +
 8 files changed, 645 insertions(+), 258 deletions(-)
 create mode 100644 tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_integration_test.py

diff --git a/tensorflow/contrib/feature_column/BUILD b/tensorflow/contrib/feature_column/BUILD
index 4e29e25599..8fc5f1cfe7 100644
--- a/tensorflow/contrib/feature_column/BUILD
+++ b/tensorflow/contrib/feature_column/BUILD
@@ -96,7 +96,6 @@ tf_py_test(
     name = "sequence_feature_column_v2_test",
     srcs = ["python/feature_column/sequence_feature_column_v2_test.py"],
     additional_deps = [
-        ":sequence_feature_column",
         ":sequence_feature_column_v2",
         "@absl_py//absl/testing:parameterized",
         "//third_party/py/numpy",
@@ -113,3 +112,20 @@ tf_py_test(
     ],
     tags = ["no_pip"],
 )
+
+py_test(
+    name = "sequence_feature_column_v2_integration_test",
+    srcs = ["python/feature_column/sequence_feature_column_v2_integration_test.py"],
+    srcs_version = "PY2AND3",
+    tags = ["no_pip"],
+    deps = [
+        ":sequence_feature_column_v2",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:parsing_ops",
+        "//tensorflow/python:training",
+        "//tensorflow/python:util",
+        "//tensorflow/python/feature_column:feature_column_py",
+        "//tensorflow/python/keras:layers",
+    ],
+)
diff --git a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2.py b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2.py
index 83b93ec332..2f4bda194a 100644
--- a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2.py
+++ b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2.py
@@ -27,6 +27,7 @@ import collections
 
 from tensorflow.python.feature_column import feature_column as fc_old
 from tensorflow.python.feature_column import feature_column_lib as fc
+from tensorflow.python.feature_column import feature_column_v2 as fc_v2
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
@@ -34,107 +35,115 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import parsing_ops
 from tensorflow.python.ops import sparse_ops
-from tensorflow.python.ops import variable_scope
 
 # pylint: disable=protected-access
 
 
-def sequence_input_layer(
-    features,
-    feature_columns,
-    weight_collections=None,
-    trainable=True):
-  """"Builds input layer for sequence input.
+class SequenceFeatures(fc_v2._BaseFeaturesLayer):
+  """A layer for sequence input.
 
-  All `feature_columns` must be sequence dense columns with the same
-  `sequence_length`. The output of this method can be fed into sequence
-  networks, such as RNN.
+    All `feature_columns` must be sequence dense columns with the same
+    `sequence_length`. The output of this method can be fed into sequence
+    networks, such as RNN.
 
-  The output of this method is a 3D `Tensor` of shape `[batch_size, T, D]`.
-  `T` is the maximum sequence length for this batch, which could differ from
-  batch to batch.
+    The output of this method is a 3D `Tensor` of shape `[batch_size, T, D]`.
+    `T` is the maximum sequence length for this batch, which could differ from
+    batch to batch.
 
-  If multiple `feature_columns` are given with `Di` `num_elements` each, their
-  outputs are concatenated. So, the final `Tensor` has shape
-  `[batch_size, T, D0 + D1 + ... + Dn]`.
+    If multiple `feature_columns` are given with `Di` `num_elements` each, their
+    outputs are concatenated. So, the final `Tensor` has shape
+    `[batch_size, T, D0 + D1 + ... + Dn]`.
 
-  Example:
+    Example:
 
-  ```python
-  rating = sequence_numeric_column('rating')
-  watches = sequence_categorical_column_with_identity(
-      'watches', num_buckets=1000)
-  watches_embedding = embedding_column(watches, dimension=10)
-  columns = [rating, watches]
+    ```python
+    rating = sequence_numeric_column('rating')
+    watches = sequence_categorical_column_with_identity(
+        'watches', num_buckets=1000)
+    watches_embedding = embedding_column(watches, dimension=10)
+    columns = [rating, watches]
 
-  features = tf.parse_example(..., features=make_parse_example_spec(columns))
-  input_layer, sequence_length = sequence_input_layer(features, columns)
+    features = tf.parse_example(..., features=make_parse_example_spec(columns))
+    sequence_input_layer = SequenceFeatures(columns)
+    sequence_input, sequence_length = sequence_input_layer(features)
+    sequence_length_mask = tf.sequence_mask(sequence_length)
 
-  rnn_cell = tf.nn.rnn_cell.BasicRNNCell(hidden_size)
-  outputs, state = tf.nn.dynamic_rnn(
-      rnn_cell, inputs=input_layer, sequence_length=sequence_length)
-  ```
+    rnn_cell = tf.keras.layers.SimpleRNNCell(hidden_size)
+    rnn_layer = tf.keras.layers.RNN(rnn_cell)
+    outputs, state = rnn_layer(sequence_input, mask=sequence_length_mask)
+    ```
+  """
 
-  Args:
-    features: A dict mapping keys to tensors.
-    feature_columns: An iterable of dense sequence columns. Valid columns are
-      - `embedding_column` that wraps a `sequence_categorical_column_with_*`
-      - `sequence_numeric_column`.
-    weight_collections: A list of collection names to which the Variable will be
-      added. Note that variables will also be added to collections
-      `tf.GraphKeys.GLOBAL_VARIABLES` and `ops.GraphKeys.MODEL_VARIABLES`.
-    trainable: If `True` also add the variable to the graph collection
-      `GraphKeys.TRAINABLE_VARIABLES`.
+  def __init__(
+      self,
+      feature_columns,
+      trainable=True,
+      name=None,
+      **kwargs):
+    """"Constructs a SequenceFeatures layer.
 
-  Returns:
-    An `(input_layer, sequence_length)` tuple where:
-    - input_layer: A float `Tensor` of shape `[batch_size, T, D]`.
-        `T` is the maximum sequence length for this batch, which could differ
-        from batch to batch. `D` is the sum of `num_elements` for all
-        `feature_columns`.
-    - sequence_length: An int `Tensor` of shape `[batch_size]`. The sequence
-        length for each example.
+    Args:
+      feature_columns: An iterable of dense sequence columns. Valid columns are
+        - `embedding_column` that wraps a `sequence_categorical_column_with_*`
+        - `sequence_numeric_column`.
+      trainable: Boolean, whether the layer's variables will be updated via
+        gradient descent during training.
+      name: Name to give to the SequenceFeatures.
+      **kwargs: Keyword arguments to construct a layer.
+
+    Raises:
+      ValueError: If any of the `feature_columns` is not a
+        `SequenceDenseColumn`.
+    """
+    super(SequenceFeatures, self).__init__(
+        feature_columns=feature_columns,
+        trainable=trainable,
+        name=name,
+        expected_column_type=fc_v2.SequenceDenseColumn,
+        **kwargs)
 
-  Raises:
-    ValueError: If any of the `feature_columns` is the wrong type.
-  """
-  feature_columns = fc_old._normalize_feature_columns(feature_columns)
-  for c in feature_columns:
-    if not isinstance(c, fc_old._SequenceDenseColumn):
-      raise ValueError(
-          'All feature_columns must be of type _SequenceDenseColumn. '
-          'You can wrap a sequence_categorical_column with an embedding_column '
-          'or indicator_column. '
-          'Given (type {}): {}'.format(type(c), c))
-
-  with variable_scope.variable_scope(
-      None, default_name='sequence_input_layer', values=features.values()):
-    builder = fc_old._LazyBuilder(features)
+  def _target_shape(self, input_shape, total_elements):
+    return (input_shape[0], input_shape[1], total_elements)
+
+  def call(self, features):
+    """Returns sequence input corresponding to the `feature_columns`.
+
+    Args:
+      features: A dict mapping keys to tensors.
+
+    Returns:
+      An `(input_layer, sequence_length)` tuple where:
+      - input_layer: A float `Tensor` of shape `[batch_size, T, D]`.
+          `T` is the maximum sequence length for this batch, which could differ
+          from batch to batch. `D` is the sum of `num_elements` for all
+          `feature_columns`.
+      - sequence_length: An int `Tensor` of shape `[batch_size]`. The sequence
+          length for each example.
+
+    Raises:
+      ValueError: If features are not a dictionary.
+    """
+    if not isinstance(features, dict):
+      raise ValueError('We expected a dictionary here. Instead we got: ',
+                       features)
+    transformation_cache = fc.FeatureTransformationCache(features)
     output_tensors = []
     sequence_lengths = []
-    ordered_columns = []
-
-    for column in sorted(feature_columns, key=lambda x: x.name):
-      ordered_columns.append(column)
-      with variable_scope.variable_scope(
-          None, default_name=column._var_scope_name):
-        dense_tensor, sequence_length = column._get_sequence_dense_tensor(
-            builder,
-            weight_collections=weight_collections,
-            trainable=trainable)
+
+    for column in self._feature_columns:
+      with ops.name_scope(column.name):
+        dense_tensor, sequence_length = column.get_sequence_dense_tensor(
+            transformation_cache, self._state_manager)
         # Flattens the final dimension to produce a 3D Tensor.
-        num_elements = column._variable_shape.num_elements()
-        shape = array_ops.shape(dense_tensor)
-        target_shape = [shape[0], shape[1], num_elements]
-        output_tensors.append(
-            array_ops.reshape(dense_tensor, shape=target_shape))
+        output_tensors.append(self._process_dense_tensor(column, dense_tensor))
         sequence_lengths.append(sequence_length)
 
-    fc_old._verify_static_batch_size_equality(output_tensors, ordered_columns)
-    fc_old._verify_static_batch_size_equality(sequence_lengths, ordered_columns)
+    # Check and process sequence lengths.
+    fc_v2._verify_static_batch_size_equality(sequence_lengths,
+                                             self._feature_columns)
     sequence_length = _assert_all_equal_and_return(sequence_lengths)
 
-    return array_ops.concat(output_tensors, -1), sequence_length
+    return self._verify_and_concat_tensors(output_tensors), sequence_length
 
 
 def concatenate_context_input(context_input, sequence_input):
@@ -203,12 +212,13 @@ def sequence_categorical_column_with_identity(
   columns = [watches_embedding]
 
   features = tf.parse_example(..., features=make_parse_example_spec(columns))
-  sequence_feature_layer = SequenceFeatureLayer(columns)
-  input_layer, sequence_length = sequence_feature_layer(features)
+  sequence_feature_layer = SequenceFeatures(columns)
+  sequence_input, sequence_length = sequence_feature_layer(features)
+  sequence_length_mask = tf.sequence_mask(sequence_length)
 
-  rnn_cell = tf.nn.rnn_cell.BasicRNNCell(hidden_size)
-  outputs, state = tf.nn.dynamic_rnn(
-      rnn_cell, inputs=input_layer, sequence_length=sequence_length)
+  rnn_cell = tf.keras.layers.SimpleRNNCell(hidden_size)
+  rnn_layer = tf.keras.layers.RNN(rnn_cell)
+  outputs, state = rnn_layer(sequence_input, mask=sequence_length_mask)
   ```
 
   Args:
@@ -250,12 +260,13 @@ def sequence_categorical_column_with_hash_bucket(
   columns = [tokens_embedding]
 
   features = tf.parse_example(..., features=make_parse_example_spec(columns))
-  sequence_feature_layer = SequenceFeatureLayer(columns)
-  input_layer, sequence_length = sequence_feature_layer(features)
+  sequence_feature_layer = SequenceFeatures(columns)
+  sequence_input, sequence_length = sequence_feature_layer(features)
+  sequence_length_mask = tf.sequence_mask(sequence_length)
 
-  rnn_cell = tf.nn.rnn_cell.BasicRNNCell(hidden_size)
-  outputs, state = tf.nn.dynamic_rnn(
-      rnn_cell, inputs=input_layer, sequence_length=sequence_length)
+  rnn_cell = tf.keras.layers.SimpleRNNCell(hidden_size)
+  rnn_layer = tf.keras.layers.RNN(rnn_cell)
+  outputs, state = rnn_layer(sequence_input, mask=sequence_length_mask)
   ```
 
   Args:
@@ -296,12 +307,13 @@ def sequence_categorical_column_with_vocabulary_file(
   columns = [states_embedding]
 
   features = tf.parse_example(..., features=make_parse_example_spec(columns))
-  sequence_feature_layer = SequenceFeatureLayer(columns)
-  input_layer, sequence_length = sequence_feature_layer(features)
+  sequence_feature_layer = SequenceFeatures(columns)
+  sequence_input, sequence_length = sequence_feature_layer(features)
+  sequence_length_mask = tf.sequence_mask(sequence_length)
 
-  rnn_cell = tf.nn.rnn_cell.BasicRNNCell(hidden_size)
-  outputs, state = tf.nn.dynamic_rnn(
-      rnn_cell, inputs=input_layer, sequence_length=sequence_length)
+  rnn_cell = tf.keras.layers.SimpleRNNCell(hidden_size)
+  rnn_layer = tf.keras.layers.RNN(rnn_cell)
+  outputs, state = rnn_layer(sequence_input, mask=sequence_length_mask)
   ```
 
   Args:
@@ -358,12 +370,13 @@ def sequence_categorical_column_with_vocabulary_list(
   columns = [colors_embedding]
 
   features = tf.parse_example(..., features=make_parse_example_spec(columns))
-  sequence_feature_layer = SequenceFeatureLayer(columns)
-  input_layer, sequence_length = sequence_feature_layer(features)
+  sequence_feature_layer = SequenceFeatures(columns)
+  sequence_input, sequence_length = sequence_feature_layer(features)
+  sequence_length_mask = tf.sequence_mask(sequence_length)
 
-  rnn_cell = tf.nn.rnn_cell.BasicRNNCell(hidden_size)
-  outputs, state = tf.nn.dynamic_rnn(
-      rnn_cell, inputs=input_layer, sequence_length=sequence_length)
+  rnn_cell = tf.keras.layers.SimpleRNNCell(hidden_size)
+  rnn_layer = tf.keras.layers.RNN(rnn_cell)
+  outputs, state = rnn_layer(sequence_input, mask=sequence_length_mask)
   ```
 
   Args:
@@ -415,12 +428,13 @@ def sequence_numeric_column(
   columns = [temperature]
 
   features = tf.parse_example(..., features=make_parse_example_spec(columns))
-  sequence_feature_layer = SequenceFeatureLayer(columns)
-  input_layer, sequence_length = sequence_feature_layer(features)
+  sequence_feature_layer = SequenceFeatures(columns)
+  sequence_input, sequence_length = sequence_feature_layer(features)
+  sequence_length_mask = tf.sequence_mask(sequence_length)
 
-  rnn_cell = tf.nn.rnn_cell.BasicRNNCell(hidden_size)
-  outputs, state = tf.nn.dynamic_rnn(
-      rnn_cell, inputs=input_layer, sequence_length=sequence_length)
+  rnn_cell = tf.keras.layers.SimpleRNNCell(hidden_size)
+  rnn_layer = tf.keras.layers.RNN(rnn_cell)
+  outputs, state = rnn_layer(sequence_input, mask=sequence_length_mask)
   ```
 
   Args:
@@ -445,7 +459,7 @@ def sequence_numeric_column(
     ValueError: if any dimension in shape is not a positive integer.
     ValueError: if `dtype` is not convertible to `tf.float32`.
   """
-  shape = fc_old._check_shape(shape=shape, key=key)
+  shape = fc_v2._check_shape(shape=shape, key=key)
   if not (dtype.is_integer or dtype.is_floating):
     raise ValueError('dtype must be convertible to float. '
                      'dtype: {}, key: {}'.format(dtype, key))
@@ -540,8 +554,10 @@ class SequenceNumericColumn(
     # For the 2D case, the raw values are grouped according to num_elements;
     # for the 3D case, the grouping happens in the third dimension, and
     # sequence length is not affected.
-    num_elements = (self.variable_shape.num_elements()
-                    if sp_tensor.shape.ndims == 2 else 1)
+    if sp_tensor.shape.ndims == 2:
+      num_elements = self.variable_shape.num_elements()
+    else:
+      num_elements = 1
     seq_length = fc_old._sequence_length_from_sparse_tensor(
         sp_tensor, num_elements=num_elements)
 
diff --git a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_integration_test.py b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_integration_test.py
new file mode 100644
index 0000000000..1b165a620a
--- /dev/null
+++ b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_integration_test.py
@@ -0,0 +1,283 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Integration test for sequence feature columns with SequenceExamples."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import string
+import tempfile
+
+from google.protobuf import text_format
+
+from tensorflow.contrib.feature_column.python.feature_column import sequence_feature_column_v2 as sfc
+from tensorflow.core.example import example_pb2
+from tensorflow.core.example import feature_pb2
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.feature_column import feature_column_v2 as fc
+from tensorflow.python.keras.layers import recurrent
+from tensorflow.python.ops import parsing_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+from tensorflow.python.util import compat
+
+
+class SequenceFeatureColumnIntegrationTest(test.TestCase):
+
+  def _make_sequence_example(self):
+    example = example_pb2.SequenceExample()
+    example.context.feature['int_ctx'].int64_list.value.extend([5])
+    example.context.feature['float_ctx'].float_list.value.extend([123.6])
+    for val in range(0, 10, 2):
+      feat = feature_pb2.Feature()
+      feat.int64_list.value.extend([val] * val)
+      example.feature_lists.feature_list['int_list'].feature.extend([feat])
+    for val in range(1, 11, 2):
+      feat = feature_pb2.Feature()
+      feat.bytes_list.value.extend([compat.as_bytes(str(val))] * val)
+      example.feature_lists.feature_list['str_list'].feature.extend([feat])
+
+    return example
+
+  def _build_feature_columns(self):
+    col = fc.categorical_column_with_identity('int_ctx', num_buckets=100)
+    ctx_cols = [
+        fc.embedding_column(col, dimension=10),
+        fc.numeric_column('float_ctx')
+    ]
+
+    identity_col = sfc.sequence_categorical_column_with_identity(
+        'int_list', num_buckets=10)
+    bucket_col = sfc.sequence_categorical_column_with_hash_bucket(
+        'bytes_list', hash_bucket_size=100)
+    seq_cols = [
+        fc.embedding_column(identity_col, dimension=10),
+        fc.embedding_column(bucket_col, dimension=20)
+    ]
+
+    return ctx_cols, seq_cols
+
+  def test_sequence_example_into_input_layer(self):
+    examples = [_make_sequence_example().SerializeToString()] * 100
+    ctx_cols, seq_cols = self._build_feature_columns()
+
+    def _parse_example(example):
+      ctx, seq = parsing_ops.parse_single_sequence_example(
+          example,
+          context_features=fc.make_parse_example_spec_v2(ctx_cols),
+          sequence_features=fc.make_parse_example_spec_v2(seq_cols))
+      ctx.update(seq)
+      return ctx
+
+    ds = dataset_ops.Dataset.from_tensor_slices(examples)
+    ds = ds.map(_parse_example)
+    ds = ds.batch(20)
+
+    # Test on a single batch
+    features = ds.make_one_shot_iterator().get_next()
+
+    # Tile the context features across the sequence features
+    sequence_input_layer = sfc.SequenceFeatures(seq_cols)
+    seq_layer, _ = sequence_input_layer(features)
+    input_layer = fc.DenseFeatures(ctx_cols)
+    ctx_layer = input_layer(features)
+    input_layer = sfc.concatenate_context_input(ctx_layer, seq_layer)
+
+    rnn_layer = recurrent.RNN(recurrent.SimpleRNNCell(10))
+    output = rnn_layer(input_layer)
+
+    with self.cached_session() as sess:
+      sess.run(variables.global_variables_initializer())
+      features_r = sess.run(features)
+      self.assertAllEqual(features_r['int_list'].dense_shape, [20, 3, 6])
+
+      output_r = sess.run(output)
+      self.assertAllEqual(output_r.shape, [20, 10])
+
+
+class SequenceExampleParsingTest(test.TestCase):
+
+  def test_seq_ex_in_sequence_categorical_column_with_identity(self):
+    self._test_parsed_sequence_example(
+        'int_list', sfc.sequence_categorical_column_with_identity,
+        10, [3, 6], [2, 4, 6])
+
+  def test_seq_ex_in_sequence_categorical_column_with_hash_bucket(self):
+    self._test_parsed_sequence_example(
+        'bytes_list', sfc.sequence_categorical_column_with_hash_bucket,
+        10, [3, 4], [compat.as_bytes(x) for x in 'acg'])
+
+  def test_seq_ex_in_sequence_categorical_column_with_vocabulary_list(self):
+    self._test_parsed_sequence_example(
+        'bytes_list', sfc.sequence_categorical_column_with_vocabulary_list,
+        list(string.ascii_lowercase), [3, 4],
+        [compat.as_bytes(x) for x in 'acg'])
+
+  def test_seq_ex_in_sequence_categorical_column_with_vocabulary_file(self):
+    _, fname = tempfile.mkstemp()
+    with open(fname, 'w') as f:
+      f.write(string.ascii_lowercase)
+    self._test_parsed_sequence_example(
+        'bytes_list', sfc.sequence_categorical_column_with_vocabulary_file,
+        fname, [3, 4], [compat.as_bytes(x) for x in 'acg'])
+
+  def _test_parsed_sequence_example(
+      self, col_name, col_fn, col_arg, shape, values):
+    """Helper function to check that each FeatureColumn parses correctly.
+
+    Args:
+      col_name: string, name to give to the feature column. Should match
+        the name that the column will parse out of the features dict.
+      col_fn: function used to create the feature column. For example,
+        sequence_numeric_column.
+      col_arg: second arg that the target feature column is expecting.
+      shape: the expected dense_shape of the feature after parsing into
+        a SparseTensor.
+      values: the expected values at index [0, 2, 6] of the feature
+        after parsing into a SparseTensor.
+    """
+    example = _make_sequence_example()
+    columns = [
+        fc.categorical_column_with_identity('int_ctx', num_buckets=100),
+        fc.numeric_column('float_ctx'),
+        col_fn(col_name, col_arg)
+    ]
+    context, seq_features = parsing_ops.parse_single_sequence_example(
+        example.SerializeToString(),
+        context_features=fc.make_parse_example_spec_v2(columns[:2]),
+        sequence_features=fc.make_parse_example_spec_v2(columns[2:]))
+
+    with self.cached_session() as sess:
+      ctx_result, seq_result = sess.run([context, seq_features])
+      self.assertEqual(list(seq_result[col_name].dense_shape), shape)
+      self.assertEqual(
+          list(seq_result[col_name].values[[0, 2, 6]]), values)
+      self.assertEqual(list(ctx_result['int_ctx'].dense_shape), [1])
+      self.assertEqual(ctx_result['int_ctx'].values[0], 5)
+      self.assertEqual(list(ctx_result['float_ctx'].shape), [1])
+      self.assertAlmostEqual(ctx_result['float_ctx'][0], 123.6, places=1)
+
+
+_SEQ_EX_PROTO = """
+context {
+  feature {
+    key: "float_ctx"
+    value {
+      float_list {
+        value: 123.6
+      }
+    }
+  }
+  feature {
+    key: "int_ctx"
+    value {
+      int64_list {
+        value: 5
+      }
+    }
+  }
+}
+feature_lists {
+  feature_list {
+    key: "bytes_list"
+    value {
+      feature {
+        bytes_list {
+          value: "a"
+        }
+      }
+      feature {
+        bytes_list {
+          value: "b"
+          value: "c"
+        }
+      }
+      feature {
+        bytes_list {
+          value: "d"
+          value: "e"
+          value: "f"
+          value: "g"
+        }
+      }
+    }
+  }
+  feature_list {
+    key: "float_list"
+    value {
+      feature {
+        float_list {
+          value: 1.0
+        }
+      }
+      feature {
+        float_list {
+          value: 3.0
+          value: 3.0
+          value: 3.0
+        }
+      }
+      feature {
+        float_list {
+          value: 5.0
+          value: 5.0
+          value: 5.0
+          value: 5.0
+          value: 5.0
+        }
+      }
+    }
+  }
+  feature_list {
+    key: "int_list"
+    value {
+      feature {
+        int64_list {
+          value: 2
+          value: 2
+        }
+      }
+      feature {
+        int64_list {
+          value: 4
+          value: 4
+          value: 4
+          value: 4
+        }
+      }
+      feature {
+        int64_list {
+          value: 6
+          value: 6
+          value: 6
+          value: 6
+          value: 6
+          value: 6
+        }
+      }
+    }
+  }
+}
+"""
+
+
+def _make_sequence_example():
+  example = example_pb2.SequenceExample()
+  return text_format.Parse(_SEQ_EX_PROTO, example)
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py
index be012a8769..661eba4741 100644
--- a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py
+++ b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py
@@ -22,9 +22,7 @@ import os
 from absl.testing import parameterized
 import numpy as np
 
-from tensorflow.contrib.feature_column.python.feature_column import sequence_feature_column as sfc_old
 from tensorflow.contrib.feature_column.python.feature_column import sequence_feature_column_v2 as sfc
-from tensorflow.python.feature_column import feature_column as fc_old
 from tensorflow.python.feature_column import feature_column_lib as fc
 from tensorflow.python.feature_column.feature_column_v2_test import _TestStateManager
 from tensorflow.python.framework import dtypes
@@ -32,13 +30,15 @@ from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import lookup_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import sparse_ops
+from tensorflow.python.ops import variables as variables_lib
 from tensorflow.python.platform import test
 from tensorflow.python.training import monitored_session
 
 
-class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
+class SequenceFeaturesTest(test.TestCase, parameterized.TestCase):
 
   @parameterized.named_parameters(
       {'testcase_name': '2D',
@@ -111,29 +111,27 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
 
     categorical_column_a = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column_a = fc_old._embedding_column(
+    embedding_column_a = fc.embedding_column(
         categorical_column_a,
         dimension=embedding_dimension_a,
         initializer=_get_initializer(embedding_dimension_a, embedding_values_a))
     categorical_column_b = sfc.sequence_categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size)
-    embedding_column_b = fc_old._embedding_column(
+    embedding_column_b = fc.embedding_column(
         categorical_column_b,
         dimension=embedding_dimension_b,
         initializer=_get_initializer(embedding_dimension_b, embedding_values_b))
 
-    input_layer, sequence_length = sfc.sequence_input_layer(
-        features={
-            'aaa': sparse_input_a,
-            'bbb': sparse_input_b,
-        },
-        # Test that columns are reordered alphabetically.
-        feature_columns=[embedding_column_b, embedding_column_a])
+    # Test that columns are reordered alphabetically.
+    sequence_input_layer = sfc.SequenceFeatures(
+        [embedding_column_b, embedding_column_a])
+    input_layer, sequence_length = sequence_input_layer({
+        'aaa': sparse_input_a, 'bbb': sparse_input_b,})
 
     global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)
     self.assertCountEqual(
-        ('sequence_input_layer/aaa_embedding/embedding_weights:0',
-         'sequence_input_layer/bbb_embedding/embedding_weights:0'),
+        ('sequence_features/aaa_embedding/embedding_weights:0',
+         'sequence_features/bbb_embedding/embedding_weights:0'),
         tuple([v.name for v in global_vars]))
     with monitored_session.MonitoredSession() as sess:
       self.assertAllEqual(embedding_values_a, global_vars[0].eval(session=sess))
@@ -152,18 +150,17 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
         values=(2, 0, 1),
         dense_shape=(2, 2))
 
-    categorical_column_a = fc_old._categorical_column_with_identity(
+    categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column_a = fc_old._embedding_column(
+    embedding_column_a = fc.embedding_column(
         categorical_column_a, dimension=2)
 
     with self.assertRaisesRegexp(
         ValueError,
         r'In embedding_column: aaa_embedding\. categorical_column must be of '
-        r'type _SequenceCategoricalColumn to use sequence_input_layer\.'):
-      _, _ = sfc.sequence_input_layer(
-          features={'aaa': sparse_input},
-          feature_columns=[embedding_column_a])
+        r'type SequenceCategoricalColumn to use SequenceFeatures\.'):
+      sequence_input_layer = sfc.SequenceFeatures([embedding_column_a])
+      _, _ = sequence_input_layer({'aaa': sparse_input})
 
   def test_shared_embedding_column(self):
     vocabulary_size = 3
@@ -210,21 +207,18 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
     categorical_column_b = sfc.sequence_categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size)
     # Test that columns are reordered alphabetically.
-    shared_embedding_columns = fc.shared_embedding_columns(
+    shared_embedding_columns = fc.shared_embedding_columns_v2(
         [categorical_column_b, categorical_column_a],
         dimension=embedding_dimension,
         initializer=_get_initializer(embedding_dimension, embedding_values))
 
-    input_layer, sequence_length = sfc.sequence_input_layer(
-        features={
-            'aaa': sparse_input_a,
-            'bbb': sparse_input_b,
-        },
-        feature_columns=shared_embedding_columns)
+    sequence_input_layer = sfc.SequenceFeatures(shared_embedding_columns)
+    input_layer, sequence_length = sequence_input_layer({
+        'aaa': sparse_input_a, 'bbb': sparse_input_b})
 
     global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)
     self.assertCountEqual(
-        ('sequence_input_layer/aaa_bbb_shared_embedding/embedding_weights:0',),
+        ('aaa_bbb_shared_embedding:0',),
         tuple([v.name for v in global_vars]))
     with monitored_session.MonitoredSession() as sess:
       self.assertAllEqual(embedding_values, global_vars[0].eval(session=sess))
@@ -248,23 +242,20 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
         values=(2, 0, 1),
         dense_shape=(2, 2))
 
-    categorical_column_a = fc_old._categorical_column_with_identity(
+    categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    categorical_column_b = fc_old._categorical_column_with_identity(
+    categorical_column_b = fc.categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size)
-    shared_embedding_columns = fc.shared_embedding_columns(
+    shared_embedding_columns = fc.shared_embedding_columns_v2(
         [categorical_column_a, categorical_column_b], dimension=2)
 
     with self.assertRaisesRegexp(
         ValueError,
         r'In embedding_column: aaa_shared_embedding\. categorical_column must '
-        r'be of type _SequenceCategoricalColumn to use sequence_input_layer\.'):
-      _, _ = sfc.sequence_input_layer(
-          features={
-              'aaa': sparse_input_a,
-              'bbb': sparse_input_b
-          },
-          feature_columns=shared_embedding_columns)
+        r'be of type SequenceCategoricalColumn to use SequenceFeatures\.'):
+      sequence_input_layer = sfc.SequenceFeatures(shared_embedding_columns)
+      _, _ = sequence_input_layer({'aaa': sparse_input_a,
+                                   'bbb': sparse_input_b})
 
   @parameterized.named_parameters(
       {'testcase_name': '2D',
@@ -319,17 +310,15 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
 
     categorical_column_a = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size_a)
-    indicator_column_a = fc_old._indicator_column(categorical_column_a)
+    indicator_column_a = fc.indicator_column(categorical_column_a)
     categorical_column_b = sfc.sequence_categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size_b)
-    indicator_column_b = fc_old._indicator_column(categorical_column_b)
-    input_layer, sequence_length = sfc.sequence_input_layer(
-        features={
-            'aaa': sparse_input_a,
-            'bbb': sparse_input_b,
-        },
-        # Test that columns are reordered alphabetically.
-        feature_columns=[indicator_column_b, indicator_column_a])
+    indicator_column_b = fc.indicator_column(categorical_column_b)
+    # Test that columns are reordered alphabetically.
+    sequence_input_layer = sfc.SequenceFeatures(
+        [indicator_column_b, indicator_column_a])
+    input_layer, sequence_length = sequence_input_layer({
+        'aaa': sparse_input_a, 'bbb': sparse_input_b})
 
     with monitored_session.MonitoredSession() as sess:
       self.assertAllEqual(expected_input_layer, input_layer.eval(session=sess))
@@ -346,17 +335,16 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
         values=(2, 0, 1),
         dense_shape=(2, 2))
 
-    categorical_column_a = fc_old._categorical_column_with_identity(
+    categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    indicator_column_a = fc_old._indicator_column(categorical_column_a)
+    indicator_column_a = fc.indicator_column(categorical_column_a)
 
     with self.assertRaisesRegexp(
         ValueError,
         r'In indicator_column: aaa_indicator\. categorical_column must be of '
-        r'type _SequenceCategoricalColumn to use sequence_input_layer\.'):
-      _, _ = sfc.sequence_input_layer(
-          features={'aaa': sparse_input},
-          feature_columns=[indicator_column_a])
+        r'type SequenceCategoricalColumn to use SequenceFeatures\.'):
+      sequence_input_layer = sfc.SequenceFeatures([indicator_column_a])
+      _, _ = sequence_input_layer({'aaa': sparse_input})
 
   @parameterized.named_parameters(
       {'testcase_name': '2D',
@@ -386,11 +374,10 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
       self, sparse_input_args, expected_input_layer, expected_sequence_length):
     sparse_input = sparse_tensor.SparseTensorValue(**sparse_input_args)
 
-    numeric_column = sfc_old.sequence_numeric_column('aaa')
+    numeric_column = sfc.sequence_numeric_column('aaa')
 
-    input_layer, sequence_length = sfc.sequence_input_layer(
-        features={'aaa': sparse_input},
-        feature_columns=[numeric_column])
+    sequence_input_layer = sfc.SequenceFeatures([numeric_column])
+    input_layer, sequence_length = sequence_input_layer({'aaa': sparse_input})
 
     with monitored_session.MonitoredSession() as sess:
       self.assertAllEqual(expected_input_layer, input_layer.eval(session=sess))
@@ -428,14 +415,13 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
       )
   def test_numeric_column_multi_dim(
       self, sparse_input_args, expected_input_layer, expected_sequence_length):
-    """Tests sequence_input_layer for multi-dimensional numeric_column."""
+    """Tests SequenceFeatures for multi-dimensional numeric_column."""
     sparse_input = sparse_tensor.SparseTensorValue(**sparse_input_args)
 
-    numeric_column = sfc_old.sequence_numeric_column('aaa', shape=(2, 2))
+    numeric_column = sfc.sequence_numeric_column('aaa', shape=(2, 2))
 
-    input_layer, sequence_length = sfc.sequence_input_layer(
-        features={'aaa': sparse_input},
-        feature_columns=[numeric_column])
+    sequence_input_layer = sfc.SequenceFeatures([numeric_column])
+    input_layer, sequence_length = sequence_input_layer({'aaa': sparse_input})
 
     with monitored_session.MonitoredSession() as sess:
       self.assertAllEqual(expected_input_layer, input_layer.eval(session=sess))
@@ -454,22 +440,20 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
         indices=((0, 0), (1, 0)),
         values=(1., 10.),
         dense_shape=(2, 2))
-    numeric_column_a = sfc_old.sequence_numeric_column('aaa')
-    numeric_column_b = sfc_old.sequence_numeric_column('bbb')
+    numeric_column_a = sfc.sequence_numeric_column('aaa')
+    numeric_column_b = sfc.sequence_numeric_column('bbb')
 
-    _, sequence_length = sfc.sequence_input_layer(
-        features={
-            'aaa': sparse_input_a,
-            'bbb': sparse_input_b,
-        },
-        feature_columns=[numeric_column_a, numeric_column_b])
+    sequence_input_layer = sfc.SequenceFeatures(
+        [numeric_column_a, numeric_column_b])
+    _, sequence_length = sequence_input_layer({
+        'aaa': sparse_input_a, 'bbb': sparse_input_b})
 
     with monitored_session.MonitoredSession() as sess:
       with self.assertRaisesRegexp(
           errors.InvalidArgumentError,
           r'\[Condition x == y did not hold element-wise:\] '
-          r'\[x \(sequence_input_layer/aaa/sequence_length:0\) = \] \[2 1\] '
-          r'\[y \(sequence_input_layer/bbb/sequence_length:0\) = \] \[1 1\]'):
+          r'\[x \(sequence_features/aaa/sequence_length:0\) = \] \[2 1\] '
+          r'\[y \(sequence_features/bbb/sequence_length:0\) = \] \[1 1\]'):
         sess.run(sequence_length)
 
   @parameterized.named_parameters(
@@ -497,11 +481,10 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
       self, sparse_input_args, expected_shape):
     """Tests that we return a known static shape when we have one."""
     sparse_input = sparse_tensor.SparseTensorValue(**sparse_input_args)
-    numeric_column = sfc_old.sequence_numeric_column('aaa', shape=(2, 2))
+    numeric_column = sfc.sequence_numeric_column('aaa', shape=(2, 2))
 
-    input_layer, _ = sfc.sequence_input_layer(
-        features={'aaa': sparse_input},
-        feature_columns=[numeric_column])
+    sequence_input_layer = sfc.SequenceFeatures([numeric_column])
+    input_layer, _ = sequence_input_layer({'aaa': sparse_input})
     shape = input_layer.get_shape()
     self.assertEqual(shape, expected_shape)
 
@@ -534,13 +517,49 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
     sparse_input = sparse_tensor.SparseTensorValue(**sparse_input_args)
     categorical_column = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=3)
-    indicator_column = fc_old._indicator_column(categorical_column)
+    indicator_column = fc.indicator_column(categorical_column)
 
-    input_layer, _ = sfc.sequence_input_layer(
-        features={'aaa': sparse_input}, feature_columns=[indicator_column])
+    sequence_input_layer = sfc.SequenceFeatures([indicator_column])
+    input_layer, _ = sequence_input_layer({'aaa': sparse_input})
     shape = input_layer.get_shape()
     self.assertEqual(shape, expected_shape)
 
+  def test_compute_output_shape(self):
+    price1 = sfc.sequence_numeric_column('price1', shape=2)
+    price2 = sfc.sequence_numeric_column('price2')
+    with ops.Graph().as_default():
+      features = {
+          'price1': sparse_tensor.SparseTensor(
+              indices=[[0, 0, 0], [0, 0, 1],
+                       [0, 1, 0], [0, 1, 1],
+                       [1, 0, 0], [1, 0, 1],
+                       [2, 0, 0], [2, 0, 1],
+                       [3, 0, 0], [3, 0, 1]],
+              values=[0., 1., 10., 11., 100., 101., 200., 201., 300., 301.],
+              dense_shape=(4, 3, 2)),
+          'price2': sparse_tensor.SparseTensor(
+              indices=[[0, 0],
+                       [0, 1],
+                       [1, 0],
+                       [2, 0],
+                       [3, 0]],
+              values=[10., 11., 20., 30., 40.],
+              dense_shape=(4, 3))}
+      sequence_features = sfc.SequenceFeatures([price1, price2])
+      seq_input, seq_len = sequence_features(features)
+      self.assertEqual(
+          sequence_features.compute_output_shape((None, None)),
+          (None, None, 3))
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose([[[0., 1., 10.], [10., 11., 11.], [0., 0., 0.]],
+                           [[100., 101., 20.], [0., 0., 0.], [0., 0., 0.]],
+                           [[200., 201., 30.], [0., 0., 0.], [0., 0., 0.]],
+                           [[300., 301., 40.], [0., 0., 0.], [0., 0., 0.]]],
+                          self.evaluate(seq_input))
+      self.assertAllClose([2, 1, 1, 1], self.evaluate(seq_len))
+
 
 class ConcatenateContextInputTest(test.TestCase, parameterized.TestCase):
   """Tests the utility fn concatenate_context_input."""
@@ -605,8 +624,8 @@ class ConcatenateContextInputTest(test.TestCase, parameterized.TestCase):
       sfc.concatenate_context_input(context_input, seq_input)
 
 
-class InputLayerTest(test.TestCase):
-  """Tests input_layer with sequence feature columns."""
+class DenseFeaturesTest(test.TestCase):
+  """Tests DenseFeatures with sequence feature columns."""
 
   def test_embedding_column(self):
     """Tests that error is raised for sequence embedding column."""
@@ -620,16 +639,15 @@ class InputLayerTest(test.TestCase):
 
     categorical_column_a = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column_a = fc_old._embedding_column(
+    embedding_column_a = fc.embedding_column(
         categorical_column_a, dimension=2)
 
     with self.assertRaisesRegexp(
         ValueError,
         r'In embedding_column: aaa_embedding\. categorical_column must not be '
-        r'of type _SequenceCategoricalColumn\.'):
-      _ = fc_old.input_layer(
-          features={'aaa': sparse_input},
-          feature_columns=[embedding_column_a])
+        r'of type SequenceCategoricalColumn\.'):
+      input_layer = fc.DenseFeatures([embedding_column_a])
+      _ = input_layer({'aaa': sparse_input})
 
   def test_indicator_column(self):
     """Tests that error is raised for sequence indicator column."""
@@ -643,15 +661,14 @@ class InputLayerTest(test.TestCase):
 
     categorical_column_a = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    indicator_column_a = fc_old._indicator_column(categorical_column_a)
+    indicator_column_a = fc.indicator_column(categorical_column_a)
 
     with self.assertRaisesRegexp(
         ValueError,
         r'In indicator_column: aaa_indicator\. categorical_column must not be '
-        r'of type _SequenceCategoricalColumn\.'):
-      _ = fc_old.input_layer(
-          features={'aaa': sparse_input},
-          feature_columns=[indicator_column_a])
+        r'of type SequenceCategoricalColumn\.'):
+      input_layer = fc.DenseFeatures([indicator_column_a])
+      _ = input_layer({'aaa': sparse_input})
 
 
 def _assert_sparse_tensor_value(test_case, expected, actual):
@@ -946,7 +963,7 @@ class SequenceEmbeddingColumnTest(
         embedding_column, {'aaa': inputs})
 
     global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)
-    self.assertItemsEqual(
+    self.assertCountEqual(
         ('embedding_weights:0',), tuple([v.name for v in global_vars]))
     with monitored_session.MonitoredSession() as sess:
       self.assertAllEqual(embedding_values, global_vars[0].eval(session=sess))
diff --git a/tensorflow/python/feature_column/BUILD b/tensorflow/python/feature_column/BUILD
index 2b986348b7..789887e537 100644
--- a/tensorflow/python/feature_column/BUILD
+++ b/tensorflow/python/feature_column/BUILD
@@ -29,6 +29,7 @@ py_library(
         "//tensorflow/python:embedding_ops",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:init_ops",
+        "//tensorflow/python:layers_base",
         "//tensorflow/python:lookup_ops",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:nn_ops",
diff --git a/tensorflow/python/feature_column/feature_column_v2.py b/tensorflow/python/feature_column/feature_column_v2.py
index 5976f647af..7c28fe2467 100644
--- a/tensorflow/python/feature_column/feature_column_v2.py
+++ b/tensorflow/python/feature_column/feature_column_v2.py
@@ -304,8 +304,84 @@ class _StateManagerImpl(StateManager):
     raise ValueError('Variable does not exist.')
 
 
+class _BaseFeaturesLayer(Layer):
+  """Base class for DenseFeatures and SequenceFeatures.
+
+  Defines common methods and helpers.
+
+  Args:
+    feature_columns: An iterable containing the FeatureColumns to use as
+      inputs to your model.
+    expected_column_type: Expected class for provided feature columns.
+    trainable:  Boolean, whether the layer's variables will be updated via
+      gradient descent during training.
+    name: Name to give to the DenseFeatures.
+    **kwargs: Keyword arguments to construct a layer.
+
+  Raises:
+    ValueError: if an item in `feature_columns` doesn't match
+      `expected_column_type`.
+  """
+  def __init__(self, feature_columns, expected_column_type, trainable, name,
+               **kwargs):
+    super(_BaseFeaturesLayer, self).__init__(
+        name=name, trainable=trainable, **kwargs)
+    self._feature_columns = _normalize_feature_columns(feature_columns)
+    self._state_manager = _StateManagerImpl(self, self.trainable)
+    for column in self._feature_columns:
+      if not isinstance(column, expected_column_type):
+        raise ValueError(
+            'Items of feature_columns must be a {}. '
+            'You can wrap a categorical column with an '
+            'embedding_column or indicator_column. Given: {}'.format(
+                expected_column_type, column))
+
+  def build(self, _):
+    for column in self._feature_columns:
+      with variable_scope._pure_variable_scope(self.name):  # pylint: disable=protected-access
+        with variable_scope._pure_variable_scope(column.name):  # pylint: disable=protected-access
+          column.create_state(self._state_manager)
+    super(_BaseFeaturesLayer, self).build(None)
+
+  def _output_shape(self, input_shape, num_elements):
+    """Computes expected output shape of the layer or a column's dense tensor.
+
+    Args:
+      input_shape: Tensor or array with batch shape.
+      num_elements: Size of the last dimension of the output.
+
+    Returns:
+      Tuple with output shape.
+    """
+    raise NotImplementedError('Calling an abstract method.')
+
+  def compute_output_shape(self, input_shape):
+    total_elements = 0
+    for column in self._feature_columns:
+      total_elements += column.variable_shape.num_elements()
+    return self._target_shape(input_shape, total_elements)
+
+  def _process_dense_tensor(self, column, tensor):
+    """Reshapes the dense tensor output of a column based on expected shape.
+
+    Args:
+      column: A DenseColumn or SequenceDenseColumn object.
+      tensor: A dense tensor obtained from the same column.
+
+    Returns:
+      Reshaped dense tensor."""
+    num_elements = column.variable_shape.num_elements()
+    target_shape = self._target_shape(array_ops.shape(tensor), num_elements)
+    return array_ops.reshape(tensor, shape=target_shape)
+
+  def _verify_and_concat_tensors(self, output_tensors):
+    """Verifies and concatenates the dense output of several columns."""
+    _verify_static_batch_size_equality(output_tensors, self._feature_columns)
+    return array_ops.concat(output_tensors, -1)
+
+
 @keras_export('keras.layers.DenseFeatures', v1=[])
-class DenseFeatures(Layer):
+class DenseFeatures(_BaseFeaturesLayer):
   """A layer that produces a dense `Tensor` based on given `feature_columns`.
 
   Generally a single example in training data is described with FeatureColumns.
@@ -345,8 +421,8 @@ class DenseFeatures(Layer):
         `bucketized_column`, `indicator_column`. If you have categorical
         features, you can wrap them with an `embedding_column` or
         `indicator_column`.
-      trainable: If `True` also add the variable to the graph collection
-        `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`).
+      trainable:  Boolean, whether the layer's variables will be updated via
+        gradient descent during training.
       name: Name to give to the DenseFeatures.
       **kwargs: Keyword arguments to construct a layer.
 
@@ -354,28 +430,18 @@ class DenseFeatures(Layer):
       ValueError: if an item in `feature_columns` is not a `DenseColumn`.
     """
     super(DenseFeatures, self).__init__(
-        name=name, trainable=trainable, **kwargs)
-
-    self._feature_columns = _normalize_feature_columns(feature_columns)
-    self._feature_columns = sorted(self._feature_columns, key=lambda x: x.name)
-    self._state_manager = _StateManagerImpl(self, self.trainable)
-    for column in self._feature_columns:
-      if not isinstance(column, DenseColumn):
-        raise ValueError(
-            'Items of feature_columns must be a DenseColumn. '
-            'You can wrap a categorical column with an '
-            'embedding_column or indicator_column. Given: {}'.format(column))
+        feature_columns=feature_columns,
+        trainable=trainable,
+        name=name,
+        expected_column_type=DenseColumn,
+        **kwargs)
 
   @property
   def _is_feature_layer(self):
     return True
 
-  def build(self, _):
-    for column in self._feature_columns:
-      with variable_scope._pure_variable_scope(self.name):  # pylint: disable=protected-access
-        with variable_scope._pure_variable_scope(column.name):  # pylint: disable=protected-access
-          column.create_state(self._state_manager)
-      super(DenseFeatures, self).build(None)
+  def _target_shape(self, input_shape, total_elements):
+    return (input_shape[0], total_elements)
 
   def call(self, features, cols_to_output_tensors=None):
     """Returns a dense tensor corresponding to the `feature_columns`.
@@ -401,27 +467,15 @@ class DenseFeatures(Layer):
                        features)
     transformation_cache = FeatureTransformationCache(features)
     output_tensors = []
-    ordered_columns = []
     for column in self._feature_columns:
       with ops.name_scope(column.name):
-        ordered_columns.append(column)
         tensor = column.get_dense_tensor(transformation_cache,
                                          self._state_manager)
-        num_elements = column.variable_shape.num_elements()
-        batch_size = array_ops.shape(tensor)[0]
-        tensor = array_ops.reshape(tensor, shape=(batch_size, num_elements))
-        output_tensors.append(tensor)
+        processed_tensors = self._process_dense_tensor(column, tensor)
         if cols_to_output_tensors is not None:
-          cols_to_output_tensors[column] = tensor
-
-    _verify_static_batch_size_equality(output_tensors, ordered_columns)
-    return array_ops.concat(output_tensors, 1)
-
-  def compute_output_shape(self, input_shape):
-    total_elements = 0
-    for column in self._feature_columns:
-      total_elements += column.variable_shape.num_elements()
-    return (input_shape[0], total_elements)
+          cols_to_output_tensors[column] = processed_tensors
+        output_tensors.append(processed_tensors)
+    return self._verify_and_concat_tensors(output_tensors)
 
 
 class _LinearModelLayer(Layer):
@@ -438,7 +492,6 @@ class _LinearModelLayer(Layer):
         name=name, trainable=trainable, **kwargs)
 
     self._feature_columns = _normalize_feature_columns(feature_columns)
-    self._feature_columns = sorted(self._feature_columns, key=lambda x: x.name)
     for column in self._feature_columns:
       if not isinstance(column, (DenseColumn, CategoricalColumn)):
         raise ValueError(
@@ -694,7 +747,7 @@ def _transform_features_v2(features, feature_columns, state_manager):
   with ops.name_scope(
       None, default_name='transform_features', values=features.values()):
     transformation_cache = FeatureTransformationCache(features)
-    for column in sorted(feature_columns, key=lambda x: x.name):
+    for column in feature_columns:
       with ops.name_scope(None, default_name=column.name):
         outputs[column] = transformation_cache.get(column, state_manager)
   return outputs
@@ -2660,7 +2713,7 @@ def _normalize_feature_columns(feature_columns):
                                                name_to_column[column.name]))
     name_to_column[column.name] = column
 
-  return feature_columns
+  return sorted(feature_columns, key=lambda x: x.name)
 
 
 class NumericColumn(
@@ -3068,10 +3121,10 @@ class EmbeddingColumn(
       raise ValueError(
           'In embedding_column: {}. '
           'categorical_column must not be of type SequenceCategoricalColumn. '
-          'Suggested fix A: If you wish to use input_layer, use a '
+          'Suggested fix A: If you wish to use DenseFeatures, use a '
           'non-sequence categorical_column_with_*. '
           'Suggested fix B: If you wish to create sequence input, use '
-          'sequence_input_layer instead of input_layer. '
+          'SequenceFeatures instead of DenseFeatures. '
           'Given (type {}): {}'.format(self.name, type(self.categorical_column),
                                        self.categorical_column))
     # Get sparse IDs and weights.
@@ -3088,10 +3141,10 @@ class EmbeddingColumn(
       raise ValueError(
           'In embedding_column: {}. '
           'categorical_column must not be of type _SequenceCategoricalColumn. '
-          'Suggested fix A: If you wish to use input_layer, use a '
+          'Suggested fix A: If you wish to use DenseFeatures, use a '
           'non-sequence categorical_column_with_*. '
           'Suggested fix B: If you wish to create sequence input, use '
-          'sequence_input_layer instead of input_layer. '
+          'SequenceFeatures instead of DenseFeatures. '
           'Given (type {}): {}'.format(self.name, type(self.categorical_column),
                                        self.categorical_column))
     sparse_tensors = self.categorical_column._get_sparse_tensors(  # pylint: disable=protected-access
@@ -3105,7 +3158,7 @@ class EmbeddingColumn(
       raise ValueError(
           'In embedding_column: {}. '
           'categorical_column must be of type SequenceCategoricalColumn '
-          'to use sequence_input_layer. '
+          'to use SequenceFeatures. '
           'Suggested fix: Use one of sequence_categorical_column_with_*. '
           'Given (type {}): {}'.format(self.name, type(self.categorical_column),
                                        self.categorical_column))
@@ -3129,8 +3182,8 @@ class EmbeddingColumn(
         (SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn)):  # pylint: disable=protected-access
       raise ValueError(
           'In embedding_column: {}. '
-          'categorical_column must be of type _SequenceCategoricalColumn '
-          'to use sequence_input_layer. '
+          'categorical_column must be of type SequenceCategoricalColumn '
+          'to use SequenceFeatures. '
           'Suggested fix: Use one of sequence_categorical_column_with_*. '
           'Given (type {}): {}'.format(self.name, type(self.categorical_column),
                                        self.categorical_column))
@@ -3298,10 +3351,10 @@ class SharedEmbeddingColumn(
       raise ValueError(
           'In embedding_column: {}. '
           'categorical_column must not be of type SequenceCategoricalColumn. '
-          'Suggested fix A: If you wish to use input_layer, use a '
+          'Suggested fix A: If you wish to use DenseFeatures, use a '
           'non-sequence categorical_column_with_*. '
           'Suggested fix B: If you wish to create sequence input, use '
-          'SequenceFeatureLayer instead of FeatureLayer. '
+          'SequenceFeatures instead of DenseFeatures. '
           'Given (type {}): {}'.format(self.name, type(self.categorical_column),
                                        self.categorical_column))
     return self._get_dense_tensor_internal(transformation_cache, state_manager)
@@ -3315,7 +3368,7 @@ class SharedEmbeddingColumn(
       raise ValueError(
           'In embedding_column: {}. '
           'categorical_column must be of type SequenceCategoricalColumn '
-          'to use SequenceFeatureLayer. '
+          'to use SequenceFeatures. '
           'Suggested fix: Use one of sequence_categorical_column_with_*. '
           'Given (type {}): {}'.format(self.name, type(self.categorical_column),
                                        self.categorical_column))
@@ -4333,10 +4386,10 @@ class IndicatorColumn(
       raise ValueError(
           'In indicator_column: {}. '
           'categorical_column must not be of type SequenceCategoricalColumn. '
-          'Suggested fix A: If you wish to use input_layer, use a '
+          'Suggested fix A: If you wish to use DenseFeatures, use a '
           'non-sequence categorical_column_with_*. '
           'Suggested fix B: If you wish to create sequence input, use '
-          'sequence_input_layer instead of input_layer. '
+          'SequenceFeatures instead of DenseFeatures. '
           'Given (type {}): {}'.format(self.name, type(self.categorical_column),
                                        self.categorical_column))
     # Feature has been already transformed. Return the intermediate
@@ -4354,10 +4407,10 @@ class IndicatorColumn(
       raise ValueError(
           'In indicator_column: {}. '
           'categorical_column must not be of type _SequenceCategoricalColumn. '
-          'Suggested fix A: If you wish to use input_layer, use a '
+          'Suggested fix A: If you wish to use DenseFeatures, use a '
           'non-sequence categorical_column_with_*. '
           'Suggested fix B: If you wish to create sequence input, use '
-          'sequence_input_layer instead of input_layer. '
+          'SequenceFeatures instead of DenseFeatures. '
           'Given (type {}): {}'.format(self.name, type(self.categorical_column),
                                        self.categorical_column))
     # Feature has been already transformed. Return the intermediate
@@ -4370,7 +4423,7 @@ class IndicatorColumn(
       raise ValueError(
           'In indicator_column: {}. '
           'categorical_column must be of type SequenceCategoricalColumn '
-          'to use sequence_input_layer. '
+          'to use SequenceFeatures. '
           'Suggested fix: Use one of sequence_categorical_column_with_*. '
           'Given (type {}): {}'.format(self.name, type(self.categorical_column),
                                        self.categorical_column))
@@ -4400,7 +4453,7 @@ class IndicatorColumn(
       raise ValueError(
           'In indicator_column: {}. '
           'categorical_column must be of type _SequenceCategoricalColumn '
-          'to use sequence_input_layer. '
+          'to use SequenceFeatures. '
           'Suggested fix: Use one of sequence_categorical_column_with_*. '
           'Given (type {}): {}'.format(self.name, type(self.categorical_column),
                                        self.categorical_column))
diff --git a/tensorflow/python/feature_column/feature_column_v2_test.py b/tensorflow/python/feature_column/feature_column_v2_test.py
index 137d5e0a8c..2b150790c1 100644
--- a/tensorflow/python/feature_column/feature_column_v2_test.py
+++ b/tensorflow/python/feature_column/feature_column_v2_test.py
@@ -3262,7 +3262,7 @@ class DenseFeaturesTest(test.TestCase):
       fc.DenseFeatures(feature_columns=[])(features={})
 
   def test_should_be_dense_column(self):
-    with self.assertRaisesRegexp(ValueError, 'must be a DenseColumn'):
+    with self.assertRaisesRegexp(ValueError, 'must be a .*DenseColumn'):
       fc.DenseFeatures(feature_columns=[
           fc.categorical_column_with_hash_bucket('wire_cast', 4)
       ])(
@@ -3423,7 +3423,7 @@ class DenseFeaturesTest(test.TestCase):
               sparse_tensor.SparseTensor(
                   indices=[[0, 0], [0, 1]], values=[1, 2], dense_shape=[1, 2])
       }
-      with self.assertRaisesRegexp(Exception, 'must be a DenseColumn'):
+      with self.assertRaisesRegexp(Exception, 'must be a .*DenseColumn'):
         fc.DenseFeatures([animal])(features)
 
   def test_static_batch_size_mismatch(self):
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt
index 28c1926bd7..3caa3ff4ed 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.keras.layers.DenseFeatures"
 tf_class {
   is_instance: "<class \'tensorflow.python.feature_column.feature_column_v2.DenseFeatures\'>"
+  is_instance: "<class \'tensorflow.python.feature_column.feature_column_v2._BaseFeaturesLayer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
-- 
GitLab


From 36d3a359562f89819013465e736e351a9782c9be Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Fri, 18 Jan 2019 13:52:59 -0800
Subject: [PATCH 0992/2345] [TF:XLA] Bump open source llvm revision to r351568

PiperOrigin-RevId: 229988232
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 3d8ea6d9e1..466262ab3f 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -512,11 +512,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "986122dee6053db98d4cf8a60efedd6d94e20c557cc97299a0c7d33b9efae12d",
-        strip_prefix = "llvm-329f768b5fc380a4bfa327396f108a8d8f33e77b",
+        sha256 = "d78389159aee7c3465b49843b46a8ac1b37db28db09dcf7522f6d31f32e5c97c",
+        strip_prefix = "llvm-316e6ae08e262dee261d1ca53a685b5ca6757a7f",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/329f768b5fc380a4bfa327396f108a8d8f33e77b.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/329f768b5fc380a4bfa327396f108a8d8f33e77b.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/316e6ae08e262dee261d1ca53a685b5ca6757a7f.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/316e6ae08e262dee261d1ca53a685b5ca6757a7f.tar.gz",
         ],
     )
 
-- 
GitLab


From 04b0afd11dc4ab680793ae865917ac8c8f0b910c Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Fri, 18 Jan 2019 13:53:02 -0800
Subject: [PATCH 0993/2345] New Equality Check API

The new equality check API allows users to configure different options when comparing two shapes. Eventually this will subsume other equality check functions (Compatible, CompatibleIgnoringElementType, CompatibleIgnoringFpPrecision, etc).

PiperOrigin-RevId: 229988246
---
 tensorflow/compiler/xla/shape.cc      |  73 ++++++++++++++++++
 tensorflow/compiler/xla/shape.h       |  39 ++++++++++
 tensorflow/compiler/xla/shape_util.cc | 105 ++------------------------
 tensorflow/compiler/xla/shape_util.h  |   8 +-
 4 files changed, 121 insertions(+), 104 deletions(-)

diff --git a/tensorflow/compiler/xla/shape.cc b/tensorflow/compiler/xla/shape.cc
index 1a029efe85..a36d3547a0 100644
--- a/tensorflow/compiler/xla/shape.cc
+++ b/tensorflow/compiler/xla/shape.cc
@@ -112,6 +112,79 @@ void Shape::DeleteDimension(int64 dim_to_delete) {
   }
 }
 
+bool Shape::Equal::operator()(const Shape& lhs, const Shape& rhs) {
+  if (lhs.IsTuple()) {
+    return rhs.IsTuple() &&
+           absl::c_equal(
+               lhs.tuple_shapes(), rhs.tuple_shapes(),
+               [=](const Shape& l, const Shape& r) { return (*this)(l, r); });
+  } else if (!lhs.IsArray()) {
+    // Non-tuple, non-array tupes such as opaque and token types are trivially
+    // the same.
+    return lhs.element_type() == rhs.element_type();
+  }
+
+  if (!rhs.IsArray()) {
+    return false;
+  }
+
+  if (!ignore_element_type_) {
+    if ((ignore_fp_precision_ &&
+         !ShapeUtil::SameElementTypeIgnoringFpPrecision(lhs, rhs)) ||
+        (!ignore_fp_precision_ && !ShapeUtil::SameElementType(lhs, rhs))) {
+      VLOG(3) << "CompareShapes: lhs element type != rhs element type";
+      return false;
+    }
+  }
+
+  if (!ignore_layout_) {
+    if (lhs.layout().format() != rhs.layout().format()) {
+      VLOG(3) << "CompareShapes: lhs layout format != rhs layout format";
+      return false;
+    }
+    if (LayoutUtil::IsDenseArray(lhs)) {
+      if (!absl::c_equal(LayoutUtil::MinorToMajor(lhs),
+                         LayoutUtil::MinorToMajor(rhs))) {
+        VLOG(3) << "CompareShapes: lhs layout != rhs layout";
+        return false;
+      }
+
+      const auto& lhs_tiles = lhs.layout().tiles();
+      const auto& rhs_tiles = rhs.layout().tiles();
+      if (lhs_tiles.size() != rhs_tiles.size()) {
+        return false;
+      }
+      for (int64 i = 0; i < lhs_tiles.size(); i++) {
+        if (!absl::c_equal(lhs_tiles[i].dimensions(),
+                           rhs_tiles[i].dimensions())) {
+          return false;
+        }
+      }
+
+      if (lhs.layout().element_size_in_bits() !=
+          rhs.layout().element_size_in_bits()) {
+        return false;
+      }
+    }
+  }
+
+  if (!ShapeUtil::SameDimensions(lhs, rhs)) {
+    VLOG(3) << "CompareShapes: lhs dimensions != rhs dimensions";
+    return false;
+  }
+
+  if (!ignore_dynamic_dimension_) {
+    for (int i = 0; i < lhs.rank(); ++i) {
+      if (lhs.is_dynamic_dimension(i) != rhs.is_dynamic_dimension(i)) {
+        VLOG(3)
+            << "CompareShapes: lhs and rhs have different dynamic dimensions.";
+        return false;
+      }
+    }
+  }
+  return true;
+}
+
 std::ostream& operator<<(std::ostream& out, const Shape& shape) {
   out << shape.ToString(/*print_layout=*/true);
   return out;
diff --git a/tensorflow/compiler/xla/shape.h b/tensorflow/compiler/xla/shape.h
index dc4cdc31a7..e40e5ca37a 100644
--- a/tensorflow/compiler/xla/shape.h
+++ b/tensorflow/compiler/xla/shape.h
@@ -138,6 +138,45 @@ class Shape {
   string ShortDebugString() const { return ToProto().ShortDebugString(); }
   string DebugString() const { return ToProto().DebugString(); }
 
+  // Equal is a configurable functor to check the equality of two shapes.
+  //
+  // Examples:
+  //
+  // - Comparing two shapes ignoring they layout difference:
+  //   Equal().IgnoreLayout()(shape1, shape2);
+  //
+  // - Comparing two shapes ignoring they layout and element type difference:
+  //   Equal().IgnoreLayout().IgnoreElementType()(shape1, shape2);
+  class Equal {
+   public:
+    Equal() = default;
+
+    bool operator()(const Shape& lhs, const Shape& rhs);
+
+    Equal& IgnoreLayout() {
+      ignore_layout_ = true;
+      return *this;
+    }
+    Equal& IgnoreElementType() {
+      ignore_element_type_ = true;
+      return *this;
+    }
+    Equal& IgnoreFpPrecision() {
+      ignore_fp_precision_ = true;
+      return *this;
+    }
+    Equal& IgnoreDynamicDimension() {
+      ignore_dynamic_dimension_ = true;
+      return *this;
+    }
+
+   public:
+    bool ignore_layout_ = false;
+    bool ignore_element_type_ = false;
+    bool ignore_fp_precision_ = false;
+    bool ignore_dynamic_dimension_ = false;
+  };
+
  private:
   // The element type of this shape (tuple, array, etc).
   PrimitiveType element_type_ = PRIMITIVE_TYPE_INVALID;
diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc
index f1ef202da0..799a4483df 100644
--- a/tensorflow/compiler/xla/shape_util.cc
+++ b/tensorflow/compiler/xla/shape_util.cc
@@ -85,77 +85,6 @@ bool ShapeIndexView::StartsWith(ShapeIndexView prefix) const {
 }
 
 namespace {
-
-// Recursive helper for comparing the equality of two shapes. Returns true if
-// the shapes are the same. If compare_layouts is true, then layouts must also
-// match.
-bool CompareShapes(const Shape& lhs, const Shape& rhs, bool compare_layouts,
-                   bool ignore_fp_precision) {
-  if ((ignore_fp_precision &&
-       !ShapeUtil::SameElementTypeIgnoringFpPrecision(lhs, rhs)) ||
-      (!ignore_fp_precision && !ShapeUtil::SameElementType(lhs, rhs))) {
-    VLOG(3) << "CompareShapes: lhs element type != rhs element type";
-    return false;
-  }
-
-  if (lhs.IsTuple()) {
-    return absl::c_equal(lhs.tuple_shapes(), rhs.tuple_shapes(),
-                         [=](const Shape& l, const Shape& r) {
-                           return CompareShapes(l, r, compare_layouts,
-                                                ignore_fp_precision);
-                         });
-  } else if (!lhs.IsArray()) {
-    // Non-tuple, non-array tupes such as opaque and token types are trivially
-    // the same.
-    return true;
-  }
-
-  if (compare_layouts) {
-    if (lhs.layout().format() != rhs.layout().format()) {
-      VLOG(3) << "CompareShapes: lhs layout format != rhs layout format";
-      return false;
-    }
-    if (LayoutUtil::IsDenseArray(lhs)) {
-      if (!absl::c_equal(LayoutUtil::MinorToMajor(lhs),
-                         LayoutUtil::MinorToMajor(rhs))) {
-        VLOG(3) << "CompareShapes: lhs layout != rhs layout";
-        return false;
-      }
-
-      const auto& lhs_tiles = lhs.layout().tiles();
-      const auto& rhs_tiles = rhs.layout().tiles();
-      if (lhs_tiles.size() != rhs_tiles.size()) {
-        return false;
-      }
-      for (int64 i = 0; i < lhs_tiles.size(); i++) {
-        if (!absl::c_equal(lhs_tiles[i].dimensions(),
-                           rhs_tiles[i].dimensions())) {
-          return false;
-        }
-      }
-
-      if (lhs.layout().element_size_in_bits() !=
-          rhs.layout().element_size_in_bits()) {
-        return false;
-      }
-    }
-  }
-
-  if (!ShapeUtil::SameDimensions(lhs, rhs)) {
-    VLOG(3) << "CompareShapes: lhs dimensions != rhs dimensions";
-    return false;
-  }
-
-  for (int i = 0; i < lhs.rank(); ++i) {
-    if (lhs.is_dynamic_dimension(i) != rhs.is_dynamic_dimension(i)) {
-      VLOG(3)
-          << "CompareShapes: lhs and rhs have different dynamic dimensions.";
-      return false;
-    }
-  }
-  return true;
-}
-
 // Constructs and returns the new shape with the given minor_to_major order in
 // its Layout.
 StatusOr<Shape> MakeShapeWithLayoutInternal(
@@ -182,12 +111,11 @@ StatusOr<Shape> MakeShapeWithLayoutInternal(
   TF_RETURN_IF_ERROR(ShapeUtil::ValidateShape(shape));
   return shape;
 }
-
 }  // namespace
 
 /* static */ bool ShapeUtil::Equal(const Shape& lhs, const Shape& rhs) {
-  bool equal = CompareShapes(lhs, rhs, /*compare_layouts=*/true,
-                             /*ignore_fp_precision=*/false);
+  bool equal = Shape::Equal()(lhs, rhs);
+
   if (!equal && VLOG_IS_ON(3)) {
     VLOG(3) << "ShapeUtil::Equal differ: lhs = " << lhs.ShortDebugString()
             << ", rhs = " << rhs.ShortDebugString();
@@ -198,8 +126,7 @@ StatusOr<Shape> MakeShapeWithLayoutInternal(
 
 /* static */ bool ShapeUtil::EqualIgnoringFpPrecision(const Shape& lhs,
                                                       const Shape& rhs) {
-  bool equal = CompareShapes(lhs, rhs, /*compare_layouts=*/true,
-                             /*ignore_fp_precision=*/true);
+  bool equal = Shape::Equal().IgnoreFpPrecision()(lhs, rhs);
   if (!equal && VLOG_IS_ON(3)) {
     VLOG(3) << "ShapeUtil::EqualIgnoringFpPrecision differ: lhs = "
             << lhs.ShortDebugString() << ", rhs = " << rhs.ShortDebugString();
@@ -563,37 +490,17 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
 }
 
 /* static */ bool ShapeUtil::Compatible(const Shape& lhs, const Shape& rhs) {
-  return CompareShapes(lhs, rhs, /*compare_layouts=*/false,
-                       /*ignore_fp_precision=*/false);
+  return Shape::Equal().IgnoreLayout()(lhs, rhs);
 }
 
 /* static */ bool ShapeUtil::CompatibleIgnoringElementType(const Shape& lhs,
                                                            const Shape& rhs) {
-  if (lhs.IsArray()) {
-    return rhs.IsArray() && SameDimensions(lhs, rhs);
-  } else if (lhs.element_type() == TUPLE) {
-    return rhs.element_type() == TUPLE &&
-           absl::c_equal(lhs.tuple_shapes(), rhs.tuple_shapes(),
-                         CompatibleIgnoringElementType);
-  } else {
-    // Opaque, token, etc types are vacuously compatible.
-    return lhs.element_type() == rhs.element_type();
-  }
+  return Shape::Equal().IgnoreElementType().IgnoreLayout()(lhs, rhs);
 }
 
 /* static */ bool ShapeUtil::CompatibleIgnoringFpPrecision(const Shape& lhs,
                                                            const Shape& rhs) {
-  if (lhs.IsArray()) {
-    return rhs.IsArray() && SameElementTypeIgnoringFpPrecision(lhs, rhs) &&
-           CompatibleIgnoringElementType(lhs, rhs);
-  } else if (lhs.element_type() == TUPLE) {
-    return rhs.element_type() == TUPLE &&
-           absl::c_equal(lhs.tuple_shapes(), rhs.tuple_shapes(),
-                         CompatibleIgnoringFpPrecision);
-  } else {
-    // Opaque, token, etc types are vacuously compatible.
-    return lhs.element_type() == rhs.element_type();
-  }
+  return Shape::Equal().IgnoreFpPrecision().IgnoreLayout()(lhs, rhs);
 }
 
 /* static */ int64 ShapeUtil::GetDimension(const Shape& shape,
diff --git a/tensorflow/compiler/xla/shape_util.h b/tensorflow/compiler/xla/shape_util.h
index e98c6e024b..fb6da7460e 100644
--- a/tensorflow/compiler/xla/shape_util.h
+++ b/tensorflow/compiler/xla/shape_util.h
@@ -675,11 +675,9 @@ class ShapeUtil {
 
   template <typename FnType>
   static void ForEachIndex(const Shape& shape, const FnType& visitor_function) {
-    ForEachIndexWithStatus(shape,
-                           [&](absl::Span<const int64> indices) {
-                             return StatusOr<bool>(visitor_function(indices));
-                           })
-        .IgnoreError();
+    ForEachIndexWithStatus(shape, [&](absl::Span<const int64> indices) {
+      return StatusOr<bool>(visitor_function(indices));
+    }).IgnoreError();
   }
 
   // A parallel version of ForEachIndex(WithStatus). This can only be used if
-- 
GitLab


From 916df403fbfc799ac5b6af49a0c1cdf3846e66aa Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Fri, 18 Jan 2019 13:55:00 -0800
Subject: [PATCH 0994/2345] Extend tensor casting to graph mode in Layer's
 __call__ method.

PiperOrigin-RevId: 229988566
---
 tensorflow/python/keras/engine/base_layer.py      |  9 ++++-----
 tensorflow/python/keras/engine/base_layer_test.py | 15 ++++++++++++++-
 2 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index f63fb4e694..93849e771b 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -513,11 +513,10 @@ class Layer(checkpointable.Checkpointable):
       ValueError: if the layer's `call` method returns None (an invalid value).
     """
     input_list = nest.flatten(inputs)
-    if context.executing_eagerly():
-      # Accept NumPy inputs by converting to Tensors when executing eagerly.
-      if all(isinstance(x, (np.ndarray, float, int)) for x in input_list):
-        inputs = nest.map_structure(ops.convert_to_tensor, inputs)
-        input_list = nest.flatten(inputs)
+    # Accept NumPy inputs by converting to Tensors.
+    if all(isinstance(x, (np.ndarray, float, int)) for x in input_list):
+      inputs = nest.map_structure(ops.convert_to_tensor, inputs)
+      input_list = nest.flatten(inputs)
 
     # We will attempt to build a TF graph if & only if all inputs are symbolic.
     # This is always the case in graph mode. It can also be the case in eager
diff --git a/tensorflow/python/keras/engine/base_layer_test.py b/tensorflow/python/keras/engine/base_layer_test.py
index ebee4a3043..c925a8ca8a 100644
--- a/tensorflow/python/keras/engine/base_layer_test.py
+++ b/tensorflow/python/keras/engine/base_layer_test.py
@@ -26,6 +26,8 @@ from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import test_util
+from tensorflow.python.keras import keras_parameterized
+from tensorflow.python.keras import testing_utils
 from tensorflow.python.keras.engine import base_layer
 from tensorflow.python.keras.optimizer_v2 import rmsprop
 from tensorflow.python.ops import array_ops
@@ -71,7 +73,7 @@ class InvalidLayer(base_layer.Layer):
     raise ValueError('You did something wrong!')
 
 
-class BaseLayerTest(test.TestCase, parameterized.TestCase):
+class BaseLayerTest(keras_parameterized.TestCase):
 
   @parameterized.parameters(DynamicLayer1, DynamicLayer2)
   def test_dynamic_layer_in_functional_model_in_graph_mode(self, layer_class):
@@ -210,6 +212,17 @@ class BaseLayerTest(test.TestCase, parameterized.TestCase):
     with self.assertRaisesRegexp(ValueError, 'You did something wrong!'):
       _ = InvalidLayer()(inputs)
 
+  @keras_parameterized.run_with_all_model_types
+  @test_util.run_in_graph_and_eager_modes
+  def test_build_with_numpy_data(self):
+    model_layers = [
+        keras.layers.Dense(3, activation='relu', kernel_initializer='ones'),
+        keras.layers.Dense(1, activation='sigmoid', kernel_initializer='ones')
+    ]
+    model = testing_utils.get_model_from_layers(model_layers, input_shape=(4,))
+    model(np.zeros((2, 4), dtype='float32'))
+    self.assertTrue(model.built)
+
   def test_using_symbolic_tensors_with_tf_ops(self):
     # Single-input.
     x = keras.Input((3,))
-- 
GitLab


From e1607105febedade60326ac475116d2c6b0310a4 Mon Sep 17 00:00:00 2001
From: Bhavani Subramanian <bhavani1.subramanian@intel.com>
Date: Fri, 18 Jan 2019 14:16:54 -0800
Subject: [PATCH 0995/2345] Addressed review comments.

---
 tensorflow/core/graph/mkl_layout_pass.cc      | 109 +++++++---------
 tensorflow/core/graph/mkl_layout_pass_test.cc |  81 +++++-------
 tensorflow/core/kernels/mkl_conv_ops.cc       | 122 ++++++++----------
 tensorflow/core/kernels/mkl_fused_ops_test.cc |  16 ++-
 4 files changed, 138 insertions(+), 190 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 035b3a46ba..b82be1e1ad 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -1391,13 +1391,13 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
                               bool change_format = false);
   static void CopyAttrsConcatV2(const Node* orig_node, NodeBuilder* nb,
                                 bool change_format = false);
-  static void CopyAttrsConvCheckConstFilter(const Node* orig_node,
-                                            NodeBuilder* nb,
-                                            bool change_format = false);
   static void CopyAttrsConv(const Node* orig_node, NodeBuilder* nb,
                             bool change_format = false);
   static void CopyAttrsConv2DDepthwise(const Node* orig_node, NodeBuilder* nb,
                                        bool change_format = false);
+  static void CopyAttrsConvCheckConstFilter(const Node* orig_node,
+                                            NodeBuilder* nb,
+                                            bool change_format = false);
   static void CopyAttrsDataType(const Node* orig_node, NodeBuilder* nb,
                                 bool change_format = false);
   static void CopyAttrsFusedBatchNorm(const Node* orig_node, NodeBuilder* nb,
@@ -1936,45 +1936,6 @@ void MklLayoutRewritePass::AddWorkSpaceEdgeIfNeeded(
 // Op-specific functions to copy attributes from old node to new node
 //////////////////////////////////////////////////////////////////////////
 
-void MklLayoutRewritePass::CopyFormatAttrsConv(
-    const Node* orig_node, NodeBuilder* nb, const std::vector<int32>& strides,
-    const std::vector<int32>& dilations, bool change_format) {
-  string data_format;
-
-  if (!change_format) {
-    nb->Attr("strides", strides);
-    nb->Attr("dilations", dilations);
-
-    TF_CHECK_OK(GetNodeAttr(orig_node->def(), "data_format", &data_format));
-    nb->Attr("data_format", data_format);
-  } else {
-    std::vector<int32> new_strides;
-    std::vector<int32> new_dilations;
-    if (strides.size() == 5) {
-      // `strides` and `dilations` also need to be changed according to
-      // `data_format`. In this case, from `NDHWC` to `NCDHW`.
-      new_strides = {strides[NDHWC::dim::N], strides[NDHWC::dim::C],
-                     strides[NDHWC::dim::D], strides[NDHWC::dim::H],
-                     strides[NDHWC::dim::W]};
-
-      new_dilations = {dilations[NDHWC::dim::N], dilations[NDHWC::dim::C],
-                       dilations[NDHWC::dim::D], dilations[NDHWC::dim::H],
-                       dilations[NDHWC::dim::W]};
-    } else {
-      // `strides` and `dilations` also need to be changed according to
-      // `data_format`. In this case, from `NHWC` to `NCHW`.
-
-      new_strides = {strides[NHWC::dim::N], strides[NHWC::dim::C],
-                     strides[NHWC::dim::H], strides[NHWC::dim::W]};
-
-      new_dilations = {dilations[NHWC::dim::N], dilations[NHWC::dim::C],
-                       dilations[NHWC::dim::H], dilations[NHWC::dim::W]};
-    }
-    nb->Attr("strides", new_strides);
-    nb->Attr("dilations", new_dilations);
-  }
-}
-
 void MklLayoutRewritePass::CopyAttrsConvCheckConstFilter(const Node* orig_node,
                                                          NodeBuilder* nb,
                                                          bool change_format) {
@@ -1982,7 +1943,6 @@ void MklLayoutRewritePass::CopyAttrsConvCheckConstFilter(const Node* orig_node,
   string padding;
   std::vector<int32> strides;
   std::vector<int32> dilations;
-  bool is_filter_const;
 
   // Get all attributes from old node.
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "T", &T));
@@ -1990,15 +1950,13 @@ void MklLayoutRewritePass::CopyAttrsConvCheckConstFilter(const Node* orig_node,
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "dilations", &dilations));
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "padding", &padding));
 
-  // Check if filter is a constant.
   Node* filter_node = nullptr;
   orig_node->input_node(1, &filter_node);
-  is_filter_const = filter_node->IsConstant();
 
   // Add attributes to new node.
   nb->Attr("T", T);
   nb->Attr("padding", padding);
-  nb->Attr("is_filter_const", is_filter_const);
+  nb->Attr("is_filter_const", filter_node->IsConstant());
 
   // Add attributes related to `data_format`.
   CopyFormatAttrsConv(orig_node, nb, strides, dilations, change_format);
@@ -2035,7 +1993,6 @@ void MklLayoutRewritePass::CopyAttrsPadWithConv2D(const Node* orig_node,
   string padding;
   std::vector<int32> strides;
   std::vector<int32> dilations;
-  bool is_filter_const;
   bool use_cudnn_on_gpu;
 
   // Get all attributes from old node.
@@ -2048,17 +2005,15 @@ void MklLayoutRewritePass::CopyAttrsPadWithConv2D(const Node* orig_node,
       GetNodeAttr(orig_node->def(), "use_cudnn_on_gpu", &use_cudnn_on_gpu));
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "Tpaddings", &Tpaddings));
 
-  // Check if filter is a constant.
   Node* filter_node = nullptr;
   orig_node->input_node(1, &filter_node);
-  is_filter_const = filter_node->IsConstant();
 
   // Add attributes to new node.
   nb->Attr("T", T);
-  nb->Attr("is_filter_const", is_filter_const);
   nb->Attr("strides", strides);
   nb->Attr("dilations", dilations);
   nb->Attr("padding", padding);
+  nb->Attr("is_filter_const", filter_node->IsConstant());
   nb->Attr("data_format", data_format);
   nb->Attr("use_cudnn_on_gpu", use_cudnn_on_gpu);
   nb->Attr("Tpaddings", Tpaddings);
@@ -2106,7 +2061,6 @@ void MklLayoutRewritePass::CopyAttrsConv2DDepthwise(const Node* orig_node,
   string padding;
   std::vector<int32> strides;
   std::vector<int32> dilations;
-  bool is_filter_const;
 
   // Get all attributes from old node.
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "T", &T));
@@ -2115,18 +2069,16 @@ void MklLayoutRewritePass::CopyAttrsConv2DDepthwise(const Node* orig_node,
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "padding", &padding));
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "data_format", &data_format));
 
-  // Check if filter is a constant.
   Node* filter_node = nullptr;
   orig_node->input_node(1, &filter_node);
-  is_filter_const = filter_node->IsConstant();
 
   // Add attributes to new node.
   nb->Attr("T", T);
   nb->Attr("strides", strides);
   nb->Attr("dilations", dilations);
   nb->Attr("padding", padding);
+  nb->Attr("is_filter_const", filter_node->IsConstant());
   nb->Attr("data_format", data_format);
-  nb->Attr("is_filter_const", is_filter_const);
 }
 
 void MklLayoutRewritePass::CopyAttrsAddN(const Node* orig_node, NodeBuilder* nb,
@@ -2262,7 +2214,6 @@ void MklLayoutRewritePass::CopyAttrsQuantizedConv2D(const Node* orig_node,
   string padding;
   string data_format("NHWC");
   std::vector<int32> strides, dilations;
-  bool is_filter_const;
 
   // Get all attributes from old node.
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "Tinput", &Tinput));
@@ -2272,21 +2223,19 @@ void MklLayoutRewritePass::CopyAttrsQuantizedConv2D(const Node* orig_node,
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "strides", &strides));
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "dilations", &dilations));
 
-  // Check if filter is a constant.
   Node* filter_node = nullptr;
   orig_node->input_node(1, &filter_node);
-  is_filter_const = filter_node->IsConstant();
 
   // Add attributes to new node.
   nb->Attr("Tinput", Tinput);
   nb->Attr("Tfilter", Tfilter);
   nb->Attr("out_type", out_type);
   nb->Attr("padding", padding);
+  nb->Attr("is_filter_const", filter_node->IsConstant());
   nb->Attr("strides", strides);
   nb->Attr("dilations", dilations);
   nb->Attr("T", out_type);  // added "T" for facilitating MklToTf conversion.
   nb->Attr("data_format", data_format);
-  nb->Attr("is_filter_const", is_filter_const);
 
   // Requantization attr Tbias.
   DataType Tbias;
@@ -2354,6 +2303,45 @@ void MklLayoutRewritePass::CopyAttrsSplit(const Node* orig_node,
   nb->Attr("data_format", data_format);
 }
 
+void MklLayoutRewritePass::CopyFormatAttrsConv(
+    const Node* orig_node, NodeBuilder* nb, const std::vector<int32>& strides,
+    const std::vector<int32>& dilations, bool change_format) {
+  string data_format;
+
+  if (!change_format) {
+    nb->Attr("strides", strides);
+    nb->Attr("dilations", dilations);
+
+    TF_CHECK_OK(GetNodeAttr(orig_node->def(), "data_format", &data_format));
+    nb->Attr("data_format", data_format);
+  } else {
+    std::vector<int32> new_strides;
+    std::vector<int32> new_dilations;
+    if (strides.size() == 5) {
+      // `strides` and `dilations` also need to be changed according to
+      // `data_format`. In this case, from `NDHWC` to `NCDHW`.
+      new_strides = {strides[NDHWC::dim::N], strides[NDHWC::dim::C],
+                     strides[NDHWC::dim::D], strides[NDHWC::dim::H],
+                     strides[NDHWC::dim::W]};
+
+      new_dilations = {dilations[NDHWC::dim::N], dilations[NDHWC::dim::C],
+                       dilations[NDHWC::dim::D], dilations[NDHWC::dim::H],
+                       dilations[NDHWC::dim::W]};
+    } else {
+      // `strides` and `dilations` also need to be changed according to
+      // `data_format`. In this case, from `NHWC` to `NCHW`.
+
+      new_strides = {strides[NHWC::dim::N], strides[NHWC::dim::C],
+                     strides[NHWC::dim::H], strides[NHWC::dim::W]};
+
+      new_dilations = {dilations[NHWC::dim::N], dilations[NHWC::dim::C],
+                       dilations[NHWC::dim::H], dilations[NHWC::dim::W]};
+    }
+    nb->Attr("strides", new_strides);
+    nb->Attr("dilations", new_dilations);
+  }
+}
+
 void MklLayoutRewritePass::CopyAttrsConcat(const Node* orig_node,
                                            NodeBuilder* nb,
                                            bool change_format) {
@@ -2419,7 +2407,6 @@ void MklLayoutRewritePass::CopyAttrsFusedConv2D(const Node* orig_node,
   std::vector<int32> strides;
   std::vector<int32> dilations;
   std::vector<string> fused_ops;
-  bool is_filter_const;
 
   // Get all attributes from old node.
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "T", &T));
@@ -2431,21 +2418,19 @@ void MklLayoutRewritePass::CopyAttrsFusedConv2D(const Node* orig_node,
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "fused_ops", &fused_ops));
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "epsilon", &epsilon));
 
-  // Check if filter is a constant.
   Node* filter_node = nullptr;
   orig_node->input_node(1, &filter_node);
-  is_filter_const = filter_node->IsConstant();
 
   // Add attributes to new node.
   nb->Attr("T", T);
   nb->Attr("num_args", num_args);
   nb->Attr("strides", strides);
   nb->Attr("padding", padding);
+  nb->Attr("is_filter_const", filter_node->IsConstant());
   nb->Attr("data_format", data_format);
   nb->Attr("dilations", dilations);
   nb->Attr("fused_ops", fused_ops);
   nb->Attr("epsilon", epsilon);
-  nb->Attr("is_filter_const", is_filter_const);
 }
 
 //////////////////////////////////////////////////////////////////////////
diff --git a/tensorflow/core/graph/mkl_layout_pass_test.cc b/tensorflow/core/graph/mkl_layout_pass_test.cc
index 02b8e23927..f79b6b0c66 100644
--- a/tensorflow/core/graph/mkl_layout_pass_test.cc
+++ b/tensorflow/core/graph/mkl_layout_pass_test.cc
@@ -127,20 +127,12 @@ class MklLayoutPassTest : public ::testing::Test {
   template <typename T>
   T DoMklLayoutOptimizationPassGetAttrVal(const string& attr,
                                           const string& node_name) {
-    string before = CanonicalGraphString(&graph_);
-    LOG(ERROR) << "Before MKL layout rewrite pass: " << before;
-
-    std::unique_ptr<Graph>* ug = new std::unique_ptr<Graph>(&graph_);
-    RunMklLayoutRewritePass(ug);
-
-    string result = CanonicalGraphString(&graph_);
-    LOG(ERROR) << "After MKL layout rewrite pass:  " << result;
-
-    T attr_val{};
+    DoMklLayoutOptimizationPass();
+    T attr_val;
     for (const Node* n : graph_.nodes()) {
       if (IncludeNode(n) && n->type_string() == node_name) {
         TF_CHECK_OK(GetNodeAttr(n->def(), attr, &attr_val));
-        break;
+        return attr_val;
       }
     }
     return attr_val;
@@ -2553,6 +2545,7 @@ TEST_F(MklLayoutPassTest, NodeRewrite_Slice_DeviceTest) {
 
 /////////////////////////////////////////////////////////////////////
 //         Post-rewrite fixup pass test
+/////////////////////////////////////////////////////////////////////
 
 TEST_F(MklLayoutPassTest, PostRewriteFixUpPass) {
   InitGraph(
@@ -2583,13 +2576,12 @@ TEST_F(MklLayoutPassTest, PostRewriteFixUpPass) {
 }
 
 /////////////////////////////////////////////////////////////////////
-
-/////////////////////////////////////////////////////////////////////
-// Unit tests related to filter caching.
+//         Unit tests related to filter caching.
 //
 // These tests check if the attribute `is_filter_const` is set to true
 // when filter is a constant and false otherwise for various operators
 // such as Conv2D, Conv2DWithBias, Conv3D etc.
+/////////////////////////////////////////////////////////////////////
 
 // Conv2D op where filter is a constant.
 TEST_F(MklLayoutPassTest, Conv2D_FilterCaching_Positive) {
@@ -2610,9 +2602,8 @@ TEST_F(MklLayoutPassTest, Conv2D_FilterCaching_Positive) {
       " input: ['A', 'B']}"
       "node { name: 'D' op: 'Zeta' attr { key: 'T' value { type: DT_FLOAT } }"
       " input: ['B', 'C'] }");
-  EXPECT_EQ(DoMklLayoutOptimizationPassGetAttrVal<bool>("is_filter_const",
-                                                        "_MklConv2D"),
-            true);
+  EXPECT_TRUE(DoMklLayoutOptimizationPassGetAttrVal<bool>("is_filter_const",
+                                                          "_MklConv2D"));
 }
 
 // Conv2D op where filter is NOT a constant.
@@ -2630,9 +2621,8 @@ TEST_F(MklLayoutPassTest, Conv2D_FilterCaching_Negative) {
       " input: ['A', 'B']}"
       "node { name: 'D' op: 'Zeta' attr { key: 'T' value { type: DT_FLOAT } }"
       " input: ['B', 'C'] }");
-  EXPECT_EQ(DoMklLayoutOptimizationPassGetAttrVal<bool>("is_filter_const",
-                                                        "_MklConv2D"),
-            false);
+  EXPECT_FALSE(DoMklLayoutOptimizationPassGetAttrVal<bool>("is_filter_const",
+                                                           "_MklConv2D"));
 }
 
 // Conv2D + BiasAdd fusion where filter is a constant.
@@ -2661,9 +2651,8 @@ TEST_F(MklLayoutPassTest, Conv2DWithBias_FilterCaching_Positive) {
       "node { name: 'Z' op: 'Zeta'"
       " attr {key: 'T'                 value { type: DT_FLOAT } }"
       " input: ['E', 'Y']}");
-  EXPECT_EQ(DoMklLayoutOptimizationPassGetAttrVal<bool>("is_filter_const",
-                                                        "_MklConv2DWithBias"),
-            true);
+  EXPECT_TRUE(DoMklLayoutOptimizationPassGetAttrVal<bool>(
+      "is_filter_const", "_MklConv2DWithBias"));
 }
 
 // Conv2D + BiasAdd fusion where filter is NOT a constant.
@@ -2688,9 +2677,8 @@ TEST_F(MklLayoutPassTest, Conv2DWithBias_FilterCaching_Negative) {
       "node { name: 'Z' op: 'Zeta'"
       " attr {key: 'T'                 value { type: DT_FLOAT } }"
       " input: ['E', 'Y']}");
-  EXPECT_EQ(DoMklLayoutOptimizationPassGetAttrVal<bool>("is_filter_const",
-                                                        "_MklConv2DWithBias"),
-            false);
+  EXPECT_FALSE(DoMklLayoutOptimizationPassGetAttrVal<bool>(
+      "is_filter_const", "_MklConv2DWithBias"));
 }
 
 // Conv3D op where filter is a constant.
@@ -2714,9 +2702,8 @@ TEST_F(MklLayoutPassTest, Conv3D_FilterCaching_Positive) {
       " input: ['A', 'B']}"
       "node { name: 'D' op: 'Zeta' attr { key: 'T' value { type: DT_FLOAT } }"
       " input: ['B', 'C'] }");
-  EXPECT_EQ(DoMklLayoutOptimizationPassGetAttrVal<bool>("is_filter_const",
-                                                        "_MklConv3D"),
-            true);
+  EXPECT_TRUE(DoMklLayoutOptimizationPassGetAttrVal<bool>("is_filter_const",
+                                                          "_MklConv3D"));
 }
 
 // Conv3D op where filter is NOT a constant.
@@ -2736,9 +2723,8 @@ TEST_F(MklLayoutPassTest, Conv3D_FilterCaching_Negative) {
       " input: ['A', 'B']}"
       "node { name: 'D' op: 'Zeta' attr { key: 'T' value { type: DT_FLOAT } }"
       " input: ['B', 'C'] }");
-  EXPECT_EQ(DoMklLayoutOptimizationPassGetAttrVal<bool>("is_filter_const",
-                                                        "_MklConv3D"),
-            false);
+  EXPECT_FALSE(DoMklLayoutOptimizationPassGetAttrVal<bool>("is_filter_const",
+                                                           "_MklConv3D"));
 }
 
 // Pad + Conv2D fusion where filter is a constant.
@@ -2768,9 +2754,8 @@ TEST_F(MklLayoutPassTest, PadWithConv2D_FilterCaching_Positive) {
       "node { name: 'Z' op: 'Zeta'"
       " attr {key: 'T'                 value { type: DT_FLOAT } }"
       " input: ['E', 'Y']}");
-  EXPECT_EQ(DoMklLayoutOptimizationPassGetAttrVal<bool>("is_filter_const",
-                                                        "_MklPadWithConv2D"),
-            true);
+  EXPECT_TRUE(DoMklLayoutOptimizationPassGetAttrVal<bool>("is_filter_const",
+                                                          "_MklPadWithConv2D"));
 }
 
 // Pad + Conv2D fusion where filter is NOT a constant.
@@ -2796,9 +2781,8 @@ TEST_F(MklLayoutPassTest, PadWithConv2D_FilterCaching_Negative) {
       "node { name: 'Z' op: 'Zeta'"
       " attr {key: 'T'                 value { type: DT_FLOAT } }"
       " input: ['E', 'Y']}");
-  EXPECT_EQ(DoMklLayoutOptimizationPassGetAttrVal<bool>("is_filter_const",
-                                                        "_MklPadWithConv2D"),
-            false);
+  EXPECT_FALSE(DoMklLayoutOptimizationPassGetAttrVal<bool>(
+      "is_filter_const", "_MklPadWithConv2D"));
 }
 
 // _FusedConv2D + BiasAdd fusion where filter is a constant.
@@ -2823,9 +2807,8 @@ TEST_F(MklLayoutPassTest, FusedConv2DWithBias_FilterCaching_Positive) {
       " input: ['A', 'B', 'C']}"
       "node { name: 'E' op: 'Zeta' attr { key: 'T' value { type: DT_FLOAT } }"
       " input: ['D', 'C'] }");
-  EXPECT_EQ(DoMklLayoutOptimizationPassGetAttrVal<bool>("is_filter_const",
-                                                        "_MklFusedConv2D"),
-            true);
+  EXPECT_TRUE(DoMklLayoutOptimizationPassGetAttrVal<bool>("is_filter_const",
+                                                          "_MklFusedConv2D"));
 }
 
 // _FusedConv2D + BiasAdd fusion where filter is NOT a constant.
@@ -2846,9 +2829,8 @@ TEST_F(MklLayoutPassTest, FusedConv2DWithBias_FilterCaching_Negative) {
       " input: ['A', 'B', 'C']}"
       "node { name: 'E' op: 'Zeta' attr { key: 'T' value { type: DT_FLOAT } }"
       " input: ['D', 'C'] }");
-  EXPECT_EQ(DoMklLayoutOptimizationPassGetAttrVal<bool>("is_filter_const",
-                                                        "_MklFusedConv2D"),
-            false);
+  EXPECT_FALSE(DoMklLayoutOptimizationPassGetAttrVal<bool>("is_filter_const",
+                                                           "_MklFusedConv2D"));
 }
 
 // Depthwise Conv2D op where filter is a constant.
@@ -2869,9 +2851,8 @@ TEST_F(MklLayoutPassTest, DepthwiseConv2dNative_FilterCaching_Positive) {
       " input: ['A', 'B']}"
       "node { name: 'D' op: 'Zeta' attr { key: 'T' value { type: DT_FLOAT } }"
       " input: ['B', 'C'] }");
-  EXPECT_EQ(DoMklLayoutOptimizationPassGetAttrVal<bool>(
-                "is_filter_const", "_MklDepthwiseConv2dNative"),
-            true);
+  EXPECT_TRUE(DoMklLayoutOptimizationPassGetAttrVal<bool>(
+      "is_filter_const", "_MklDepthwiseConv2dNative"));
 }
 
 // Depthwise Conv2D op where filter is NOT a constant.
@@ -2888,11 +2869,9 @@ TEST_F(MklLayoutPassTest, DepthwiseConv2dNative_FilterCaching_Negative) {
       " input: ['A', 'B']}"
       "node { name: 'D' op: 'Zeta' attr { key: 'T' value { type: DT_FLOAT } }"
       " input: ['B', 'C'] }");
-  EXPECT_EQ(DoMklLayoutOptimizationPassGetAttrVal<bool>(
-                "is_filter_const", "_MklDepthwiseConv2dNative"),
-            false);
+  EXPECT_FALSE(DoMklLayoutOptimizationPassGetAttrVal<bool>(
+      "is_filter_const", "_MklDepthwiseConv2dNative"));
 }
-/////////////////////////////////////////////////////////////////////
 
 static void BM_MklLayoutRewritePass(int iters, int op_nodes) {
   testing::StopTiming();
diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc
index ae3ba50133..9869556a8f 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_ops.cc
@@ -1027,7 +1027,7 @@ class MklConvOp : public OpKernel {
       filter.SetUsrMem(filter_md, &filter_tensor);
 
       // MKLDNN dilations start from 0.
-      for (int i = 0; i < dilations.size(); i++) --dilations[i];
+      for (int i = 0; i < dilations.size(); ++i) --dilations[i];
 
       // In some cases, primitive descriptor could potentially contain
       // large buffers. As a result, we don't cache these primitives if the
@@ -1074,7 +1074,7 @@ class MklConvOp : public OpKernel {
       Ttemp_output* dst_data =
           reinterpret_cast<Ttemp_output*>(dst_tensor->flat<Toutput>().data());
 
-      // Check whether src and filter needs to be reordered
+      // Check whether src and filter need to be reordered
       Tinput* src_data = nullptr;
       if (src_md.data.format != conv_fwd->GetSrcMemoryFormat()) {
         // Reorder src
@@ -1270,10 +1270,10 @@ class MklConvOp : public OpKernel {
 
   // Allocate persistent tensors for cached filter data and
   // cached filter memory descriptor (data format)
-  void AllocatePersistTensor(OpKernelContext* context,
-                             const ConvFwdPd& conv_prim_desc,
-                             Tensor** filter_tensor) {
-    CHECK_NOTNULL(filter_tensor);
+  void AllocatePersistentTensor(OpKernelContext* context,
+                                const ConvFwdPd& conv_prim_desc,
+                                Tensor** filter_tensor) {
+    DCHECK(filter_tensor);
     TensorShape filter_tf_shape;
     filter_tf_shape.AddDim(
         (conv_prim_desc.weights_primitive_desc().get_size() / sizeof(Tfilter)));
@@ -1356,13 +1356,11 @@ class MklConvOp : public OpKernel {
     stream(stream::kind::eager).submit(net).wait();
   }
 
-  inline bool IsFilterCacheEmpty(OpKernelContext* context) {
-    tf_shared_lock lock(mu_);
-    SHARED_LOCKS_REQUIRED(mu_) {
-      const Tensor& cached_filter_data_tensor =
-          *cached_filter_data_ptensor_.AccessTensor(context);
-      return (cached_filter_data_tensor.NumElements() == 0);
-    }
+  inline bool IsFilterCacheEmpty(OpKernelContext* context)
+      SHARED_LOCKS_REQUIRED(mu_) {
+    const Tensor& cached_filter_data_tensor =
+        *cached_filter_data_ptensor_.AccessTensor(context);
+    return (cached_filter_data_tensor.NumElements() == 0);
   }
 
   // Cache the converted filter in a persistent tensor.
@@ -1370,51 +1368,47 @@ class MklConvOp : public OpKernel {
   void CacheFilter(OpKernelContext* context,
                    const std::shared_ptr<ConvFwdPd>& conv_fwd_pd,
                    Tfilter* filter_data, const Tensor& filter_tensor,
-                   MklDnnData<Tfilter>& filter, const memory::desc& filter_md) {
-    mutex_lock lock(mu_);
-    EXCLUSIVE_LOCKS_REQUIRED(mu_) {
-      const Tensor& cached_filter_data_tensor =
-          *cached_filter_data_ptensor_.AccessTensor(context);
+                   MklDnnData<Tfilter>& filter, const memory::desc& filter_md)
+      EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+    const Tensor& cached_filter_data_tensor =
+        *cached_filter_data_ptensor_.AccessTensor(context);
 
-      // If filter is already cached, there's nothing to do.
-      if (cached_filter_data_tensor.NumElements() > 0) {
-        return;
-      }
-
-      // Otherwise, cache filter
-      filter.SetUsrMem(filter_md, &filter_tensor);
-      filter.CheckReorderToOpMem(conv_fwd_pd.get()->weights_primitive_desc());
-      filter_data = static_cast<Tfilter*>(filter.GetOpMem().get_data_handle());
-
-      Tensor* filter_tensor_ptr = nullptr;
-      AllocatePersistTensor(context, *conv_fwd_pd, &filter_tensor_ptr);
-      void* cached_filter_data = filter.GetTensorBuffer(filter_tensor_ptr);
-      size_t cached_filter_data_size =
-          filter.GetOpMem().get_primitive_desc().get_size();
-      memcpy(cached_filter_data, filter_data, cached_filter_data_size);
+    // If filter is already cached, there's nothing to do.
+    if (cached_filter_data_tensor.NumElements() > 0) {
+      return;
     }
+
+    // Otherwise, cache filter
+    filter.SetUsrMem(filter_md, &filter_tensor);
+    filter.CheckReorderToOpMem(conv_fwd_pd.get()->weights_primitive_desc());
+    filter_data = static_cast<Tfilter*>(filter.GetOpMem().get_data_handle());
+
+    Tensor* filter_tensor_ptr = nullptr;
+    AllocatePersistentTensor(context, *conv_fwd_pd, &filter_tensor_ptr);
+    void* cached_filter_data = filter.GetTensorBuffer(filter_tensor_ptr);
+    size_t cached_filter_data_size =
+        filter.GetOpMem().get_primitive_desc().get_size();
+    memcpy(cached_filter_data, filter_data, cached_filter_data_size);
   }
 
   Tfilter* GetCachedFilter(OpKernelContext* context,
-                           const memory::format& filter_mf) {
-    tf_shared_lock lock(mu_);
-    SHARED_LOCKS_REQUIRED(mu_) {
-      const Tensor& cached_filter_data =
-          *cached_filter_data_ptensor_.AccessTensor(context);
-      const Tensor& cached_filter_md =
-          *cached_filter_md_ptensor_.AccessTensor(context);
-
-      // Check if the memory descriptor of the cached weights is same as
-      // filter_mf. If so, we can used the cached weights; otherwise
-      // return NULL.
-      // TODO (bhavanis): Do we need to cast filter_mf before the check?
-      if (cached_filter_md.scalar<int32>().size() &&
-          cached_filter_md.scalar<int32>()() == filter_mf) {
-        return static_cast<Tfilter*>(
-            const_cast<Tfilter*>(cached_filter_data.flat<Tfilter>().data()));
-      }
-      return nullptr;
+                           const memory::format& filter_mf)
+      SHARED_LOCKS_REQUIRED(mu_) {
+    const Tensor& cached_filter_data =
+        *cached_filter_data_ptensor_.AccessTensor(context);
+    const Tensor& cached_filter_md =
+        *cached_filter_md_ptensor_.AccessTensor(context);
+
+    // Check if the memory descriptor of the cached weights is same as
+    // filter_mf. If so, we can used the cached weights; otherwise
+    // return NULL.
+    // TODO (bhavanis): Do we need to cast filter_mf before the check?
+    if (cached_filter_md.scalar<int32>().size() &&
+        cached_filter_md.scalar<int32>()() == filter_mf) {
+      return static_cast<Tfilter*>(
+          const_cast<Tfilter*>(cached_filter_data.flat<Tfilter>().data()));
     }
+    return nullptr;
   }
 };
 
@@ -1488,7 +1482,7 @@ class MklQuantizedConv2DOp
     bool is_filter_const;
     OP_REQUIRES_OK(context,
                    context->GetAttr("is_filter_const", &is_filter_const));
-    OP_REQUIRES(context, is_filter_const == true,
+    OP_REQUIRES(context, is_filter_const,
                 errors::InvalidArgument("Filter must be a constant"));
   }
 
@@ -1514,9 +1508,9 @@ class MklQuantizedConv2DOp
     float max_output_value;
     if (std::is_same<Toutput, quint8>::value ||
         std::is_same<Toutput, qint8>::value) {
-      // This is the case when convolution and requantization is fused.
-      // min_freezed_output and max_freezed_output are the actual ranges
-      // for the output.
+      // This is the case when convolution and requantization are fused.
+      // min_freezed_output and max_freezed_output are the actual range
+      // of the output.
       min_output_value = context->input(6 + bias_index_offset).flat<float>()(0);
       max_output_value = context->input(7 + bias_index_offset).flat<float>()(0);
     } else {
@@ -1645,13 +1639,7 @@ class MklQuantizedConv2DReluOp
 
   explicit MklQuantizedConv2DReluOp(OpKernelConstruction* context)
       : MklQuantizedConv2DOp<Device, Tbias, Toutput, Ttemp_output,
-                             bias_enabled>(context) {
-    bool is_filter_const;
-    OP_REQUIRES_OK(context,
-                   context->GetAttr("is_filter_const", &is_filter_const));
-    OP_REQUIRES(context, is_filter_const == true,
-                errors::InvalidArgument("Filter must be a constant"));
-  }
+                             bias_enabled>(context) {}
 
  protected:
   void ExtendConvFwdParams(OpKernelContext* context,
@@ -1682,13 +1670,7 @@ class MklQuantizedConv2DSumReluOp
 
   explicit MklQuantizedConv2DSumReluOp(OpKernelConstruction* context)
       : MklQuantizedConv2DOp<Device, Tbias, Toutput, Ttemp_output,
-                             bias_enabled>(context) {
-    bool is_filter_const;
-    OP_REQUIRES_OK(context,
-                   context->GetAttr("is_filter_const", &is_filter_const));
-    OP_REQUIRES(context, is_filter_const == true,
-                errors::InvalidArgument("Filter must be a constant"));
-  }
+                             bias_enabled>(context) {}
 
  protected:
   void ExtendConvFwdParams(OpKernelContext* context,
diff --git a/tensorflow/core/kernels/mkl_fused_ops_test.cc b/tensorflow/core/kernels/mkl_fused_ops_test.cc
index 6d2e4fae7c..0fdec670b6 100644
--- a/tensorflow/core/kernels/mkl_fused_ops_test.cc
+++ b/tensorflow/core/kernels/mkl_fused_ops_test.cc
@@ -431,20 +431,22 @@ class FilterCacheTest : public OpsTestBase {
 
     TF_ASSERT_OK(RunOpKernel());
 
-    // Compare output to expected results
-    const Tensor& first = *GetOutput(0);
-    const Tensor& second = *GetOutput(2);
+    // Compare outputs to expected results
+    const Tensor& output = *GetOutput(0);
+    const Tensor& output_layout = *GetOutput(2);
     ConvMklToTF<T> conv_comp;
-    conv_comp.ConvertAndCompare(dtype, first, second, expected);
+    conv_comp.ConvertAndCompare(dtype, output, output_layout, expected);
 
     // Run kernel again to ensure cached data is being reused
+    // TODO: Check if data is actually cached and reused
     TF_ASSERT_OK(RunOpKernel());
 
     // Compare output to expected results
-    const Tensor& first_new = *GetOutput(0);
-    const Tensor& second_new = *GetOutput(2);
+    const Tensor& output_new = *GetOutput(0);
+    const Tensor& output_layout_new = *GetOutput(2);
     ConvMklToTF<T> conv_comp_new;
-    conv_comp_new.ConvertAndCompare(dtype, first_new, second_new, expected);
+    conv_comp_new.ConvertAndCompare(dtype, output_new, output_layout_new,
+                                    expected);
   }
 };
 
-- 
GitLab


From 5c5102a3c925748b8919c85a515955443a7594cd Mon Sep 17 00:00:00 2001
From: David Majnemer <majnemer@google.com>
Date: Fri, 18 Jan 2019 14:34:28 -0800
Subject: [PATCH 0996/2345] [XLA] Don't rely on ctype for classifying
 characters

Instead, use the ascii routines in ABSL. This makes the code do the same thing
regardless of how the locale is configured.

PiperOrigin-RevId: 229995801
---
 tensorflow/compiler/xla/metric_table_report.cc           | 4 ++--
 tensorflow/compiler/xla/parse_flags_from_env.cc          | 6 ++++--
 tensorflow/compiler/xla/service/BUILD                    | 3 ++-
 .../service/cpu/tests/cpu_eigen_dot_operation_test.cc    | 1 -
 .../compiler/xla/service/cpu/tests/cpu_intrinsic_test.cc | 7 ++++---
 tensorflow/compiler/xla/service/hlo_lexer.cc             | 9 +++++----
 .../compiler/xla/service/indexed_array_analysis_test.cc  | 5 ++---
 tensorflow/compiler/xla/service/name_uniquer.cc          | 5 +++--
 8 files changed, 22 insertions(+), 18 deletions(-)

diff --git a/tensorflow/compiler/xla/metric_table_report.cc b/tensorflow/compiler/xla/metric_table_report.cc
index ad1699a1ae..bad65ac320 100644
--- a/tensorflow/compiler/xla/metric_table_report.cc
+++ b/tensorflow/compiler/xla/metric_table_report.cc
@@ -15,9 +15,9 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/metric_table_report.h"
 
-#include <cctype>
 #include <unordered_map>
 
+#include "absl/strings/ascii.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_format.h"
 #include "tensorflow/core/platform/logging.h"
@@ -249,7 +249,7 @@ string MetricTableReport::MetricString(double metric) {
   string output;
   // Copy leading non-digit characters unconditionally.
   // This picks up the leading sign.
-  while (!sp1.empty() && !isdigit(sp1[0])) {
+  while (!sp1.empty() && !absl::ascii_isdigit(sp1[0])) {
     output.push_back(sp1[0]);
     sp1.remove_prefix(1);
   }
diff --git a/tensorflow/compiler/xla/parse_flags_from_env.cc b/tensorflow/compiler/xla/parse_flags_from_env.cc
index 5b568888d1..91e71f5d1d 100644
--- a/tensorflow/compiler/xla/parse_flags_from_env.cc
+++ b/tensorflow/compiler/xla/parse_flags_from_env.cc
@@ -24,6 +24,7 @@ limitations under the License.
 #include <unordered_map>
 #include <vector>
 
+#include "absl/strings/ascii.h"
 #include "absl/strings/str_format.h"
 #include "absl/strings/str_join.h"
 #include "absl/types/span.h"
@@ -37,7 +38,7 @@ limitations under the License.
 
 namespace xla {
 
-static const char kWS[] = " \t\r\n";           // whitespace
+static const char kWS[] = " \t\r\n";  // whitespace
 
 // The following struct represents an argv[]-style array, parsed
 // from data gleaned from the environment.
@@ -104,7 +105,8 @@ static void ParseArgvFromString(const string& flag_str, EnvArgv* a) {
     // Set e to the index just past the end of the flag.
     size_t e = b;
     while (e != flag_str.size() && isascii(flag_str[e]) &&
-           (strchr("-_", flag_str[e]) != nullptr || isalnum(flag_str[e]))) {
+           (strchr("-_", flag_str[e]) != nullptr ||
+            absl::ascii_isalnum(flag_str[e]))) {
       e++;
     }
     if (e != flag_str.size() && flag_str[e] == '=' &&
diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 2b2c32a681..3c123fd5c6 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -3656,12 +3656,13 @@ tf_cc_test(
     extra_copts = ["-Wno-string-plus-int"],
     deps = [
         ":hlo_matchers",
+        ":hlo_parser",
         ":indexed_array_analysis",
         "//tensorflow/compiler/xla:test",
-        "//tensorflow/compiler/xla/service:hlo_parser",
         "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:test_utils",
         "//tensorflow/core:test",
+        "@com_google_absl//absl/strings",
     ],
 )
 
diff --git a/tensorflow/compiler/xla/service/cpu/tests/cpu_eigen_dot_operation_test.cc b/tensorflow/compiler/xla/service/cpu/tests/cpu_eigen_dot_operation_test.cc
index 0b4ac9dc29..8b7f843582 100644
--- a/tensorflow/compiler/xla/service/cpu/tests/cpu_eigen_dot_operation_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/tests/cpu_eigen_dot_operation_test.cc
@@ -16,7 +16,6 @@ limitations under the License.
 // Tests that we call into Eigen for dot operations as needed.
 
 #include <algorithm>
-#include <cctype>
 #include <string>
 
 #include "absl/strings/str_cat.h"
diff --git a/tensorflow/compiler/xla/service/cpu/tests/cpu_intrinsic_test.cc b/tensorflow/compiler/xla/service/cpu/tests/cpu_intrinsic_test.cc
index 3fb0e3cd91..9078b8fd1f 100644
--- a/tensorflow/compiler/xla/service/cpu/tests/cpu_intrinsic_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/tests/cpu_intrinsic_test.cc
@@ -14,9 +14,9 @@ limitations under the License.
 ==============================================================================*/
 
 #include <algorithm>
-#include <cctype>
 #include <string>
 
+#include "absl/strings/ascii.h"
 #include "absl/strings/str_cat.h"
 #include "tensorflow/compiler/xla/service/cpu/cpu_compiler.h"
 #include "tensorflow/compiler/xla/service/cpu/tests/cpu_codegen_test.h"
@@ -59,8 +59,9 @@ class CpuUnaryIntrinsicTest
 
     string features{spec.features.data(), spec.features.size()};
     if (!features.empty()) {
-      std::replace_if(features.begin(), features.end(),
-                      [](char c) { return c != '_' && !isalnum(c); }, '_');
+      std::replace_if(
+          features.begin(), features.end(),
+          [](char c) { return c != '_' && !absl::ascii_isalnum(c); }, '_');
     } else {
       features = "";
     }
diff --git a/tensorflow/compiler/xla/service/hlo_lexer.cc b/tensorflow/compiler/xla/service/hlo_lexer.cc
index 5e81515134..798760885d 100644
--- a/tensorflow/compiler/xla/service/hlo_lexer.cc
+++ b/tensorflow/compiler/xla/service/hlo_lexer.cc
@@ -17,6 +17,7 @@ limitations under the License.
 
 #include <unordered_map>
 
+#include "absl/strings/ascii.h"
 #include "absl/strings/escaping.h"
 #include "absl/strings/numbers.h"
 #include "absl/strings/str_split.h"
@@ -37,8 +38,8 @@ constexpr int kError = -2;
 
 // [a-zA-Z0-9_.-]
 bool IsIdentifierChar(char c) {
-  return isalnum(static_cast<unsigned char>(c)) || c == '-' || c == '.' ||
-         c == '_';
+  return absl::ascii_isalnum(static_cast<unsigned char>(c)) || c == '-' ||
+         c == '.' || c == '_';
 }
 
 }  // namespace
@@ -105,7 +106,7 @@ TokKind HloLexer::LexToken() {
     switch (current_char) {
       default:
         // [a-zA-Z_]
-        if (isalpha(static_cast<unsigned char>(current_char)) ||
+        if (absl::ascii_isalpha(static_cast<unsigned char>(current_char)) ||
             current_char == '_') {
           return LexIdentifier();
         }
@@ -300,7 +301,7 @@ TokKind HloLexer::LexIdentifier() {
 // name ::= [a-zA-Z_][a-zA-Z0-9_.-]*
 TokKind HloLexer::LexPercent() {
   const char* name_start = current_ptr_;
-  if (isalpha(static_cast<unsigned char>(PeekCurrentChar())) ||
+  if (absl::ascii_isalpha(static_cast<unsigned char>(PeekCurrentChar())) ||
       PeekCurrentChar() == '_') {
     current_ptr_++;
     while (IsIdentifierChar(PeekCurrentChar())) {
diff --git a/tensorflow/compiler/xla/service/indexed_array_analysis_test.cc b/tensorflow/compiler/xla/service/indexed_array_analysis_test.cc
index 295465c848..62107b5a88 100644
--- a/tensorflow/compiler/xla/service/indexed_array_analysis_test.cc
+++ b/tensorflow/compiler/xla/service/indexed_array_analysis_test.cc
@@ -13,9 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include <ctype.h>
-
 #include "tensorflow/compiler/xla/service/indexed_array_analysis.h"
+#include "absl/strings/ascii.h"
 #include "tensorflow/compiler/xla/tests/hlo_test_base.h"
 #include "tensorflow/compiler/xla/tests/test_utils.h"
 
@@ -43,7 +42,7 @@ class IndexedArrayAnalysisTest : public HloTestBase {
     string result;
 
     for (char c : text) {
-      if (!isspace(c)) {
+      if (!absl::ascii_isspace(c)) {
         result.push_back(c);
       } else if (!result.empty() && result.back() != ' ') {
         result.push_back(' ');
diff --git a/tensorflow/compiler/xla/service/name_uniquer.cc b/tensorflow/compiler/xla/service/name_uniquer.cc
index f6feed2993..e55b83d17e 100644
--- a/tensorflow/compiler/xla/service/name_uniquer.cc
+++ b/tensorflow/compiler/xla/service/name_uniquer.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/service/name_uniquer.h"
 
+#include "absl/strings/ascii.h"
 #include "absl/strings/numbers.h"
 #include "absl/strings/str_cat.h"
 #include "tensorflow/compiler/xla/primitive_util.h"
@@ -28,7 +29,7 @@ namespace {
 
 bool IsAllowed(char character) {
   auto c = static_cast<unsigned char>(character);
-  return (isalnum(c) != 0) || c == '_' || c == '.' || c == '-';
+  return (absl::ascii_isalnum(c) != 0) || c == '_' || c == '.' || c == '-';
 }
 
 }  // namespace
@@ -46,7 +47,7 @@ NameUniquer::NameUniquer(const string& separator) {
 
   string result = name;
   char c = static_cast<unsigned char>(result[0]);
-  if (!isalpha(c) && c != '_') {
+  if (!absl::ascii_isalpha(c) && c != '_') {
     result[0] = '_';
   }
   for (int i = 1; i < result.length(); i++) {
-- 
GitLab


From f6fde51408b1a474ee32f7c1ce48b9358775babe Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Fri, 18 Jan 2019 14:38:26 -0800
Subject: [PATCH 0997/2345] Small cleanup of backend tests.

PiperOrigin-RevId: 229996461
---
 tensorflow/python/keras/backend_test.py | 709 +++++++++++-------------
 1 file changed, 337 insertions(+), 372 deletions(-)

diff --git a/tensorflow/python/keras/backend_test.py b/tensorflow/python/keras/backend_test.py
index e3f86ee2c0..d25c214622 100644
--- a/tensorflow/python/keras/backend_test.py
+++ b/tensorflow/python/keras/backend_test.py
@@ -122,20 +122,19 @@ class BackendUtilsTest(test.TestCase):
         sess.run(y, feed_dict={x: np.random.random((2, 3))})
 
   def test_learning_phase_scope(self):
-    with self.cached_session():
-      initial_learning_phase = keras.backend.learning_phase()
-      with keras.backend.learning_phase_scope(1) as lp:
-        self.assertEqual(lp, 1)
-        self.assertEqual(keras.backend.learning_phase(), 1)
-      self.assertEqual(keras.backend.learning_phase(), initial_learning_phase)
-      with keras.backend.learning_phase_scope(0) as lp:
-        self.assertEqual(lp, 0)
-        self.assertEqual(keras.backend.learning_phase(), 0)
-      self.assertEqual(keras.backend.learning_phase(), initial_learning_phase)
-      with self.assertRaises(ValueError):
-        with keras.backend.learning_phase_scope(None):
-          pass
-      self.assertEqual(keras.backend.learning_phase(), initial_learning_phase)
+    initial_learning_phase = keras.backend.learning_phase()
+    with keras.backend.learning_phase_scope(1) as lp:
+      self.assertEqual(lp, 1)
+      self.assertEqual(keras.backend.learning_phase(), 1)
+    self.assertEqual(keras.backend.learning_phase(), initial_learning_phase)
+    with keras.backend.learning_phase_scope(0) as lp:
+      self.assertEqual(lp, 0)
+      self.assertEqual(keras.backend.learning_phase(), 0)
+    self.assertEqual(keras.backend.learning_phase(), initial_learning_phase)
+    with self.assertRaises(ValueError):
+      with keras.backend.learning_phase_scope(None):
+        pass
+    self.assertEqual(keras.backend.learning_phase(), initial_learning_phase)
 
   def test_int_shape(self):
     x = keras.backend.ones(shape=(3, 4))
@@ -146,21 +145,20 @@ class BackendUtilsTest(test.TestCase):
       self.assertEqual(keras.backend.int_shape(x), (None, 4))
 
   def test_in_train_phase(self):
-    with self.cached_session():
-      y1 = keras.backend.variable(1)
-      y2 = keras.backend.variable(2)
-      if context.executing_eagerly():
-        with keras.backend.learning_phase_scope(0):
-          y_val_test = keras.backend.in_train_phase(y1, y2).numpy()
-        with keras.backend.learning_phase_scope(1):
-          y_val_train = keras.backend.in_train_phase(y1, y2).numpy()
-      else:
-        y = keras.backend.in_train_phase(y1, y2)
-        f = keras.backend.function([keras.backend.learning_phase()], [y])
-        y_val_test = f([0])[0]
-        y_val_train = f([1])[0]
-      self.assertAllClose(y_val_test, 2)
-      self.assertAllClose(y_val_train, 1)
+    y1 = keras.backend.variable(1)
+    y2 = keras.backend.variable(2)
+    if context.executing_eagerly():
+      with keras.backend.learning_phase_scope(0):
+        y_val_test = keras.backend.in_train_phase(y1, y2).numpy()
+      with keras.backend.learning_phase_scope(1):
+        y_val_train = keras.backend.in_train_phase(y1, y2).numpy()
+    else:
+      y = keras.backend.in_train_phase(y1, y2)
+      f = keras.backend.function([keras.backend.learning_phase()], [y])
+      y_val_test = f([0])[0]
+      y_val_train = f([1])[0]
+    self.assertAllClose(y_val_test, 2)
+    self.assertAllClose(y_val_train, 1)
 
   def test_is_keras_tensor(self):
     x = keras.backend.variable(1)
@@ -187,74 +185,63 @@ class BackendUtilsTest(test.TestCase):
 class BackendVariableTest(test.TestCase):
 
   def test_zeros(self):
-    with self.cached_session():
-      x = keras.backend.zeros((3, 4))
-      val = keras.backend.eval(x)
-      self.assertAllClose(val, np.zeros((3, 4)))
+    x = keras.backend.zeros((3, 4))
+    val = keras.backend.eval(x)
+    self.assertAllClose(val, np.zeros((3, 4)))
 
   def test_ones(self):
-    with self.cached_session():
-      x = keras.backend.ones((3, 4))
-      val = keras.backend.eval(x)
-      self.assertAllClose(val, np.ones((3, 4)))
+    x = keras.backend.ones((3, 4))
+    val = keras.backend.eval(x)
+    self.assertAllClose(val, np.ones((3, 4)))
 
   def test_eye(self):
-    with self.cached_session():
-      x = keras.backend.eye(4)
-      val = keras.backend.eval(x)
-      self.assertAllClose(val, np.eye(4))
+    x = keras.backend.eye(4)
+    val = keras.backend.eval(x)
+    self.assertAllClose(val, np.eye(4))
 
   def test_zeros_like(self):
-    with self.cached_session():
-      x = keras.backend.zeros((3, 4))
-      y = keras.backend.zeros_like(x)
-      val = keras.backend.eval(y)
-      self.assertAllClose(val, np.zeros((3, 4)))
+    x = keras.backend.zeros((3, 4))
+    y = keras.backend.zeros_like(x)
+    val = keras.backend.eval(y)
+    self.assertAllClose(val, np.zeros((3, 4)))
 
   def test_ones_like(self):
-    with self.cached_session():
-      x = keras.backend.zeros((3, 4))
-      y = keras.backend.ones_like(x)
-      val = keras.backend.eval(y)
-      self.assertAllClose(val, np.ones((3, 4)))
+    x = keras.backend.zeros((3, 4))
+    y = keras.backend.ones_like(x)
+    val = keras.backend.eval(y)
+    self.assertAllClose(val, np.ones((3, 4)))
 
   def test_random_uniform_variable(self):
-    with self.cached_session():
-      x = keras.backend.random_uniform_variable((30, 20), low=1, high=2, seed=0)
-      val = keras.backend.eval(x)
-      self.assertAllClose(val.mean(), 1.5, atol=1e-1)
-      self.assertAllClose(val.max(), 2., atol=1e-1)
-      self.assertAllClose(val.min(), 1., atol=1e-1)
+    x = keras.backend.random_uniform_variable((30, 20), low=1, high=2, seed=0)
+    val = keras.backend.eval(x)
+    self.assertAllClose(val.mean(), 1.5, atol=1e-1)
+    self.assertAllClose(val.max(), 2., atol=1e-1)
+    self.assertAllClose(val.min(), 1., atol=1e-1)
 
   def test_random_normal_variable(self):
-    with self.cached_session():
-      x = keras.backend.random_normal_variable((30, 20), 1., 0.5,
-                                               seed=0)
-      val = keras.backend.eval(x)
-      self.assertAllClose(val.mean(), 1., atol=1e-1)
-      self.assertAllClose(val.std(), 0.5, atol=1e-1)
+    x = keras.backend.random_normal_variable((30, 20), 1., 0.5, seed=0)
+    val = keras.backend.eval(x)
+    self.assertAllClose(val.mean(), 1., atol=1e-1)
+    self.assertAllClose(val.std(), 0.5, atol=1e-1)
 
   def test_count_params(self):
-    with self.cached_session():
-      x = keras.backend.zeros((4, 5))
-      val = keras.backend.count_params(x)
-      self.assertAllClose(val, 20)
+    x = keras.backend.zeros((4, 5))
+    val = keras.backend.count_params(x)
+    self.assertAllClose(val, 20)
 
   def test_constant(self):
-    with self.cached_session():
-      ref_val = np.random.random((3, 4)).astype('float32')
-      x = keras.backend.constant(ref_val)
-      val = keras.backend.eval(x)
-      self.assertAllClose(val, ref_val)
+    ref_val = np.random.random((3, 4)).astype('float32')
+    x = keras.backend.constant(ref_val)
+    val = keras.backend.eval(x)
+    self.assertAllClose(val, ref_val)
 
   def test_sparse_variable(self):
-    with self.cached_session():
-      val = scipy.sparse.eye(10)
-      x = keras.backend.variable(val)
-      self.assertTrue(isinstance(x, sparse_tensor.SparseTensor))
+    val = scipy.sparse.eye(10)
+    x = keras.backend.variable(val)
+    self.assertTrue(isinstance(x, sparse_tensor.SparseTensor))
 
-      y = keras.backend.to_dense(x)
-      self.assertFalse(keras.backend.is_sparse(y))
+    y = keras.backend.to_dense(x)
+    self.assertFalse(keras.backend.is_sparse(y))
 
 
 @test_util.run_all_in_graph_and_eager_modes
@@ -292,20 +279,19 @@ class BackendLinearAlgebraTest(test.TestCase):
         (keras.backend.argmax, np.argmax),
     ]
     for keras_op, np_op in ops_to_test:
-      with self.cached_session():
-        compare_single_input_op_to_numpy(keras_op, np_op, input_shape=(4, 7, 5),
-                                         keras_kwargs={'axis': 1},
-                                         np_kwargs={'axis': 1})
-        compare_single_input_op_to_numpy(keras_op, np_op, input_shape=(4, 7, 5),
-                                         keras_kwargs={'axis': -1},
-                                         np_kwargs={'axis': -1})
-        if 'keepdims' in tf_inspect.getargspec(keras_op).args:
-          compare_single_input_op_to_numpy(keras_op, np_op,
-                                           input_shape=(4, 7, 5),
-                                           keras_kwargs={'axis': 1,
-                                                         'keepdims': True},
-                                           np_kwargs={'axis': 1,
-                                                      'keepdims': True})
+      compare_single_input_op_to_numpy(keras_op, np_op, input_shape=(4, 7, 5),
+                                       keras_kwargs={'axis': 1},
+                                       np_kwargs={'axis': 1})
+      compare_single_input_op_to_numpy(keras_op, np_op, input_shape=(4, 7, 5),
+                                       keras_kwargs={'axis': -1},
+                                       np_kwargs={'axis': -1})
+      if 'keepdims' in tf_inspect.getargspec(keras_op).args:
+        compare_single_input_op_to_numpy(keras_op, np_op,
+                                         input_shape=(4, 7, 5),
+                                         keras_kwargs={'axis': 1,
+                                                       'keepdims': True},
+                                         np_kwargs={'axis': 1,
+                                                    'keepdims': True})
 
   def test_elementwise_ops(self):
     ops_to_test = [
@@ -318,32 +304,28 @@ class BackendLinearAlgebraTest(test.TestCase):
         (keras.backend.exp, np.exp),
     ]
     for keras_op, np_op in ops_to_test:
-      with self.cached_session():
-        compare_single_input_op_to_numpy(keras_op, np_op, input_shape=(4, 7))
+      compare_single_input_op_to_numpy(keras_op, np_op, input_shape=(4, 7))
 
     ops_to_test = [
         (keras.backend.sqrt, np.sqrt),
         (keras.backend.log, np.log),
     ]
     for keras_op, np_op in ops_to_test:
-      with self.cached_session():
-        compare_single_input_op_to_numpy(keras_op, np_op,
-                                         input_shape=(4, 7),
-                                         negative_values=False)
+      compare_single_input_op_to_numpy(keras_op, np_op,
+                                       input_shape=(4, 7),
+                                       negative_values=False)
 
-    with self.cached_session():
-      compare_single_input_op_to_numpy(
-          keras.backend.clip, np.clip,
-          input_shape=(6, 4),
-          keras_kwargs={'min_value': 0.1, 'max_value': 2.4},
-          np_kwargs={'a_min': 0.1, 'a_max': 1.4})
+    compare_single_input_op_to_numpy(
+        keras.backend.clip, np.clip,
+        input_shape=(6, 4),
+        keras_kwargs={'min_value': 0.1, 'max_value': 2.4},
+        np_kwargs={'a_min': 0.1, 'a_max': 1.4})
 
-    with self.cached_session():
-      compare_single_input_op_to_numpy(
-          keras.backend.pow, np.power,
-          input_shape=(6, 4),
-          keras_args=[3],
-          np_args=[3])
+    compare_single_input_op_to_numpy(
+        keras.backend.pow, np.power,
+        input_shape=(6, 4),
+        keras_args=[3],
+        np_args=[3])
 
   def test_two_tensor_ops(self):
     ops_to_test = [
@@ -357,84 +339,82 @@ class BackendLinearAlgebraTest(test.TestCase):
         (keras.backend.minimum, np.minimum),
     ]
     for keras_op, np_op in ops_to_test:
-      with self.cached_session():
-        compare_two_inputs_op_to_numpy(keras_op, np_op,
-                                       input_shape_a=(4, 7),
-                                       input_shape_b=(4, 7))
+      compare_two_inputs_op_to_numpy(keras_op, np_op,
+                                     input_shape_a=(4, 7),
+                                     input_shape_b=(4, 7))
 
   def test_relu(self):
     x = ops.convert_to_tensor([[-4, 0], [2, 7]], 'float32')
-    with self.cached_session():
-      # standard relu
-      relu_op = keras.backend.relu(x)
-      self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 7]])
 
-      # alpha (leaky relu used)
-      relu_op = keras.backend.relu(x, alpha=0.5)
-      if not context.executing_eagerly():
-        self.assertTrue('LeakyRelu' in relu_op.name)
-      self.assertAllClose(keras.backend.eval(relu_op), [[-2, 0], [2, 7]])
+    # standard relu
+    relu_op = keras.backend.relu(x)
+    self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 7]])
 
-      # max_value < some elements
-      relu_op = keras.backend.relu(x, max_value=5)
-      self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 5]])
+    # alpha (leaky relu used)
+    relu_op = keras.backend.relu(x, alpha=0.5)
+    if not context.executing_eagerly():
+      self.assertTrue('LeakyRelu' in relu_op.name)
+    self.assertAllClose(keras.backend.eval(relu_op), [[-2, 0], [2, 7]])
 
-      # nn.relu6 used
-      relu_op = keras.backend.relu(x, max_value=6)
-      if not context.executing_eagerly():
-        self.assertTrue('Relu6' in relu_op.name)  # uses tf.nn.relu6
-      self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 6]])
+    # max_value < some elements
+    relu_op = keras.backend.relu(x, max_value=5)
+    self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 5]])
+
+    # nn.relu6 used
+    relu_op = keras.backend.relu(x, max_value=6)
+    if not context.executing_eagerly():
+      self.assertTrue('Relu6' in relu_op.name)  # uses tf.nn.relu6
+    self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 6]])
 
-      # max value > 6
-      relu_op = keras.backend.relu(x, max_value=10)
-      self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 7]])
+    # max value > 6
+    relu_op = keras.backend.relu(x, max_value=10)
+    self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 7]])
 
-      # max value is float
-      relu_op = keras.backend.relu(x, max_value=4.3)
-      self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 4.3]])
+    # max value is float
+    relu_op = keras.backend.relu(x, max_value=4.3)
+    self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 4.3]])
 
-      # max value == 0
-      relu_op = keras.backend.relu(x, max_value=0)
-      self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [0, 0]])
+    # max value == 0
+    relu_op = keras.backend.relu(x, max_value=0)
+    self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [0, 0]])
 
-      # alpha and max_value
-      relu_op = keras.backend.relu(x, alpha=0.25, max_value=3)
-      self.assertAllClose(keras.backend.eval(relu_op), [[-1, 0], [2, 3]])
+    # alpha and max_value
+    relu_op = keras.backend.relu(x, alpha=0.25, max_value=3)
+    self.assertAllClose(keras.backend.eval(relu_op), [[-1, 0], [2, 3]])
 
-      # threshold
-      relu_op = keras.backend.relu(x, threshold=3)
-      self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [0, 7]])
+    # threshold
+    relu_op = keras.backend.relu(x, threshold=3)
+    self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [0, 7]])
 
-      # threshold is float
-      relu_op = keras.backend.relu(x, threshold=1.5)
-      self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 7]])
+    # threshold is float
+    relu_op = keras.backend.relu(x, threshold=1.5)
+    self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 7]])
 
-      # threshold is negative
-      relu_op = keras.backend.relu(x, threshold=-5)
-      self.assertAllClose(keras.backend.eval(relu_op), [[-4, 0], [2, 7]])
+    # threshold is negative
+    relu_op = keras.backend.relu(x, threshold=-5)
+    self.assertAllClose(keras.backend.eval(relu_op), [[-4, 0], [2, 7]])
 
-      # threshold and max_value
-      relu_op = keras.backend.relu(x, threshold=3, max_value=5)
-      self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [0, 5]])
+    # threshold and max_value
+    relu_op = keras.backend.relu(x, threshold=3, max_value=5)
+    self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [0, 5]])
 
-      # threshold and alpha
-      relu_op = keras.backend.relu(x, alpha=0.25, threshold=4)
-      self.assertAllClose(keras.backend.eval(relu_op), [[-2, -1], [-0.5, 7]])
+    # threshold and alpha
+    relu_op = keras.backend.relu(x, alpha=0.25, threshold=4)
+    self.assertAllClose(keras.backend.eval(relu_op), [[-2, -1], [-0.5, 7]])
 
-      # threshold, alpha, and max_value
-      relu_op = keras.backend.relu(x, alpha=0.25, threshold=4, max_value=5)
-      self.assertAllClose(keras.backend.eval(relu_op), [[-2, -1], [-0.5, 5]])
+    # threshold, alpha, and max_value
+    relu_op = keras.backend.relu(x, alpha=0.25, threshold=4, max_value=5)
+    self.assertAllClose(keras.backend.eval(relu_op), [[-2, -1], [-0.5, 5]])
 
 
 @test_util.run_all_in_graph_and_eager_modes
 class BackendShapeOpsTest(test.TestCase):
 
   def test_reshape(self):
-    with self.cached_session():
-      compare_single_input_op_to_numpy(keras.backend.reshape, np.reshape,
-                                       input_shape=(4, 7),
-                                       keras_args=[(2, 14)],
-                                       np_args=[(2, 14)])
+    compare_single_input_op_to_numpy(keras.backend.reshape, np.reshape,
+                                     input_shape=(4, 7),
+                                     keras_args=[(2, 14)],
+                                     np_args=[(2, 14)])
 
   def test_concatenate(self):
     a = keras.backend.variable(np.ones((1, 2, 3)))
@@ -443,12 +423,11 @@ class BackendShapeOpsTest(test.TestCase):
     self.assertEqual(y.get_shape().as_list(), [1, 2, 5])
 
   def test_permute_dimensions(self):
-    with self.cached_session():
-      compare_single_input_op_to_numpy(keras.backend.permute_dimensions,
-                                       np.transpose,
-                                       input_shape=(4, 7),
-                                       keras_args=[(1, 0)],
-                                       np_args=[(1, 0)])
+    compare_single_input_op_to_numpy(keras.backend.permute_dimensions,
+                                     np.transpose,
+                                     input_shape=(4, 7),
+                                     keras_args=[(1, 0)],
+                                     np_args=[(1, 0)])
 
   def test_resize_images(self):
     height_factor = 2
@@ -523,18 +502,16 @@ class BackendShapeOpsTest(test.TestCase):
     self.assertEqual(y.get_shape().as_list(), [1, 2, 3])
 
   def test_flatten(self):
-    with self.cached_session():
-      compare_single_input_op_to_numpy(keras.backend.flatten,
-                                       np.reshape,
-                                       input_shape=(4, 7, 6),
-                                       np_args=[(4 * 7 * 6,)])
+    compare_single_input_op_to_numpy(keras.backend.flatten,
+                                     np.reshape,
+                                     input_shape=(4, 7, 6),
+                                     np_args=[(4 * 7 * 6,)])
 
   def test_batch_flatten(self):
-    with self.cached_session():
-      compare_single_input_op_to_numpy(keras.backend.batch_flatten,
-                                       np.reshape,
-                                       input_shape=(4, 7, 6),
-                                       np_args=[(4, 7 * 6)])
+    compare_single_input_op_to_numpy(keras.backend.batch_flatten,
+                                     np.reshape,
+                                     input_shape=(4, 7, 6),
+                                     np_args=[(4, 7 * 6)])
 
   def test_temporal_padding(self):
 
@@ -545,12 +522,11 @@ class BackendShapeOpsTest(test.TestCase):
       y[:, padding[0]:-padding[1], :] = x
       return y
 
-    with self.cached_session():
-      compare_single_input_op_to_numpy(keras.backend.temporal_padding,
-                                       ref_op,
-                                       input_shape=(4, 7, 6),
-                                       keras_args=[(2, 3)],
-                                       np_args=[(2, 3)])
+    compare_single_input_op_to_numpy(keras.backend.temporal_padding,
+                                     ref_op,
+                                     input_shape=(4, 7, 6),
+                                     keras_args=[(2, 3)],
+                                     np_args=[(2, 3)])
 
   def test_spatial_2d_padding(self):
 
@@ -568,23 +544,22 @@ class BackendShapeOpsTest(test.TestCase):
         y[:, :, padding[0][0]:-padding[0][1], padding[1][0]:-padding[1][1]] = x
       return y
 
-    with self.cached_session():
-      compare_single_input_op_to_numpy(
-          keras.backend.spatial_2d_padding,
-          ref_op,
-          input_shape=(2, 3, 2, 3),
-          keras_args=[((2, 3), (1, 2))],
-          keras_kwargs={'data_format': 'channels_last'},
-          np_args=[((2, 3), (1, 2))],
-          np_kwargs={'data_format': 'channels_last'})
-      compare_single_input_op_to_numpy(
-          keras.backend.spatial_2d_padding,
-          ref_op,
-          input_shape=(2, 3, 2, 3),
-          keras_args=[((2, 3), (1, 2))],
-          keras_kwargs={'data_format': 'channels_first'},
-          np_args=[((2, 3), (1, 2))],
-          np_kwargs={'data_format': 'channels_first'})
+    compare_single_input_op_to_numpy(
+        keras.backend.spatial_2d_padding,
+        ref_op,
+        input_shape=(2, 3, 2, 3),
+        keras_args=[((2, 3), (1, 2))],
+        keras_kwargs={'data_format': 'channels_last'},
+        np_args=[((2, 3), (1, 2))],
+        np_kwargs={'data_format': 'channels_last'})
+    compare_single_input_op_to_numpy(
+        keras.backend.spatial_2d_padding,
+        ref_op,
+        input_shape=(2, 3, 2, 3),
+        keras_args=[((2, 3), (1, 2))],
+        keras_kwargs={'data_format': 'channels_first'},
+        np_args=[((2, 3), (1, 2))],
+        np_kwargs={'data_format': 'channels_first'})
 
   def test_spatial_3d_padding(self):
 
@@ -611,73 +586,70 @@ class BackendShapeOpsTest(test.TestCase):
           padding[2][0]:-padding[2][1]] = x
       return y
 
-    with self.cached_session():
-      compare_single_input_op_to_numpy(
-          keras.backend.spatial_3d_padding,
-          ref_op,
-          input_shape=(2, 3, 2, 3, 2),
-          keras_args=[((2, 3), (1, 2), (2, 3))],
-          keras_kwargs={'data_format': 'channels_last'},
-          np_args=[((2, 3), (1, 2), (2, 3))],
-          np_kwargs={'data_format': 'channels_last'})
-      compare_single_input_op_to_numpy(
-          keras.backend.spatial_3d_padding,
-          ref_op,
-          input_shape=(2, 3, 2, 3, 2),
-          keras_args=[((2, 3), (1, 2), (2, 3))],
-          keras_kwargs={'data_format': 'channels_first'},
-          np_args=[((2, 3), (1, 2), (2, 3))],
-          np_kwargs={'data_format': 'channels_first'})
+    compare_single_input_op_to_numpy(
+        keras.backend.spatial_3d_padding,
+        ref_op,
+        input_shape=(2, 3, 2, 3, 2),
+        keras_args=[((2, 3), (1, 2), (2, 3))],
+        keras_kwargs={'data_format': 'channels_last'},
+        np_args=[((2, 3), (1, 2), (2, 3))],
+        np_kwargs={'data_format': 'channels_last'})
+    compare_single_input_op_to_numpy(
+        keras.backend.spatial_3d_padding,
+        ref_op,
+        input_shape=(2, 3, 2, 3, 2),
+        keras_args=[((2, 3), (1, 2), (2, 3))],
+        keras_kwargs={'data_format': 'channels_first'},
+        np_args=[((2, 3), (1, 2), (2, 3))],
+        np_kwargs={'data_format': 'channels_first'})
 
 
 @test_util.run_all_in_graph_and_eager_modes
 class BackendNNOpsTest(test.TestCase, parameterized.TestCase):
 
   def test_bias_add(self):
-    with self.cached_session():
-      keras_op = keras.backend.bias_add
-      np_op = np.add
-      compare_two_inputs_op_to_numpy(keras_op, np_op,
-                                     input_shape_a=(4, 7),
-                                     input_shape_b=(7,))
-      compare_two_inputs_op_to_numpy(keras_op, np_op,
-                                     input_shape_a=(4, 3, 7),
-                                     input_shape_b=(7,))
-      compare_two_inputs_op_to_numpy(keras_op, np_op,
-                                     input_shape_a=(4, 3, 5, 7),
-                                     input_shape_b=(7,))
-      compare_two_inputs_op_to_numpy(keras_op, np_op,
-                                     input_shape_a=(4, 3, 5, 2, 7),
-                                     input_shape_b=(7,))
-
-      with self.assertRaises((ValueError, errors_impl.InvalidArgumentError)):
-        x = keras.backend.variable((3, 4))
-        b = keras.backend.variable((3, 4))
-        keras.backend.bias_add(x, b)
-      with self.assertRaises(ValueError):
-        x = keras.backend.variable((3, 4))
-        b = keras.backend.variable((4,))
-        keras.backend.bias_add(x, b, data_format='unknown')
+    keras_op = keras.backend.bias_add
+    np_op = np.add
+    compare_two_inputs_op_to_numpy(keras_op, np_op,
+                                   input_shape_a=(4, 7),
+                                   input_shape_b=(7,))
+    compare_two_inputs_op_to_numpy(keras_op, np_op,
+                                   input_shape_a=(4, 3, 7),
+                                   input_shape_b=(7,))
+    compare_two_inputs_op_to_numpy(keras_op, np_op,
+                                   input_shape_a=(4, 3, 5, 7),
+                                   input_shape_b=(7,))
+    compare_two_inputs_op_to_numpy(keras_op, np_op,
+                                   input_shape_a=(4, 3, 5, 2, 7),
+                                   input_shape_b=(7,))
+
+    with self.assertRaises((ValueError, errors_impl.InvalidArgumentError)):
+      x = keras.backend.variable((3, 4))
+      b = keras.backend.variable((3, 4))
+      keras.backend.bias_add(x, b)
+    with self.assertRaises(ValueError):
+      x = keras.backend.variable((3, 4))
+      b = keras.backend.variable((4,))
+      keras.backend.bias_add(x, b, data_format='unknown')
 
   def test_bias_add_channels_first(self):
-    with self.cached_session():
 
-      def keras_op(x, b):
-        return keras.backend.bias_add(x, b, data_format='channels_first')
+    def keras_op(x, b):
+      return keras.backend.bias_add(x, b, data_format='channels_first')
 
-      def np_op(x, b):
-        if x.ndim == 3:
-          b = b.reshape((1, b.shape[0], 1))
-        if x.ndim == 4:
-          b = b.reshape((1, b.shape[0], 1, 1))
-        return x + b
+    def np_op(x, b):
+      if x.ndim == 3:
+        b = b.reshape((1, b.shape[0], 1))
+      if x.ndim == 4:
+        b = b.reshape((1, b.shape[0], 1, 1))
+      return x + b
 
-      compare_two_inputs_op_to_numpy(keras_op, np_op,
-                                     input_shape_a=(4, 3, 7),
-                                     input_shape_b=(3,))
-      compare_two_inputs_op_to_numpy(keras_op, np_op,
-                                     input_shape_a=(4, 3, 5, 7),
-                                     input_shape_b=(3,))
+    compare_two_inputs_op_to_numpy(keras_op, np_op,
+                                   input_shape_a=(4, 3, 7),
+                                   input_shape_b=(3,))
+    compare_two_inputs_op_to_numpy(keras_op, np_op,
+                                   input_shape_a=(4, 3, 5, 7),
+                                   input_shape_b=(3,))
 
   def test_pool2d(self):
     val = np.random.random((10, 3, 10, 10))
@@ -837,9 +809,9 @@ class BackendNNOpsTest(test.TestCase, parameterized.TestCase):
                                              strides,
                                              output_shape,
                                              'channels_last')
-          with self.cached_session():
-            conv_cf = keras.backend.eval(conv_cf)
-            conv_cl = keras.backend.eval(conv_cl)
+
+          conv_cf = keras.backend.eval(conv_cf)
+          conv_cl = keras.backend.eval(conv_cl)
 
           self.assertAllCloseAccordingToType(
               conv_cf,
@@ -887,9 +859,8 @@ class BackendNNOpsTest(test.TestCase, parameterized.TestCase):
                                                   output_shape,
                                                   'channels_last')
 
-    with self.cached_session():
-      local_conv = keras.backend.eval(local_conv)
-      local_conv_dim = keras.backend.eval(local_conv_dim)
+    local_conv = keras.backend.eval(local_conv)
+    local_conv_dim = keras.backend.eval(local_conv_dim)
 
     self.assertAllCloseAccordingToType(local_conv, local_conv_dim)
 
@@ -1045,24 +1016,23 @@ class BackendNNOpsTest(test.TestCase, parameterized.TestCase):
         {'go_backwards': False, 'mask': mask},
         {'go_backwards': False, 'mask': mask, 'unroll': True},
     ]
-    with self.cached_session():
-      for i, kwargs in enumerate(kwargs_list):
-        last_output, outputs, new_states = keras.backend.rnn(rnn_fn, inputs,
-                                                             initial_states,
-                                                             **kwargs)
-        # check static shape inference
-        self.assertEqual(last_output.get_shape().as_list(),
+    for i, kwargs in enumerate(kwargs_list):
+      last_output, outputs, new_states = keras.backend.rnn(rnn_fn, inputs,
+                                                           initial_states,
+                                                           **kwargs)
+      # check static shape inference
+      self.assertEqual(last_output.get_shape().as_list(),
+                       [num_samples, output_dim])
+      self.assertEqual(outputs.get_shape().as_list(),
+                       [num_samples, timesteps, output_dim])
+      for state in new_states:
+        self.assertEqual(state.get_shape().as_list(),
                          [num_samples, output_dim])
-        self.assertEqual(outputs.get_shape().as_list(),
-                         [num_samples, timesteps, output_dim])
-        for state in new_states:
-          self.assertEqual(state.get_shape().as_list(),
-                           [num_samples, output_dim])
 
-        last_output_list[i].append(keras.backend.eval(last_output))
-        outputs_list[i].append(keras.backend.eval(outputs))
-        self.assertEqual(len(new_states), 1)
-        state_list[i].append(keras.backend.eval(new_states[0]))
+      last_output_list[i].append(keras.backend.eval(last_output))
+      outputs_list[i].append(keras.backend.eval(outputs))
+      self.assertLen(new_states, 1)
+      state_list[i].append(keras.backend.eval(new_states[0]))
 
       def assert_list_pairwise(z_list, atol=1e-05):
         for (z1, z2) in zip(z_list[1:], z_list[:-1]):
@@ -1144,29 +1114,28 @@ class BackendNNOpsTest(test.TestCase, parameterized.TestCase):
         {'go_backwards': False, 'mask': mask},
         {'go_backwards': False, 'mask': mask, 'unroll': True},
     ]
-    with self.cached_session():
-      for i, kwargs in enumerate(kwargs_list):
-        last_output, outputs, new_states = keras.backend.rnn(rnn_fn, inputs,
-                                                             initial_states,
-                                                             **kwargs)
-        # check static shape inference
-        self.assertEqual(last_output.get_shape().as_list(),
-                         [num_samples, output_dim])
-        self.assertEqual(outputs.get_shape().as_list(),
-                         [num_samples, timesteps, output_dim])
-        # for state in new_states:
-        #   self.assertEqual(state.get_shape().as_list(),
-        #                     [num_samples, output_dim])
-        self.assertEqual(new_states[0].get_shape().as_list(),
-                         [num_samples, output_dim])
-        self.assertEqual(new_states[1].get_shape().as_list(),
-                         [num_samples, 2 * output_dim])
-
-        last_output_list[i].append(keras.backend.eval(last_output))
-        outputs_list[i].append(keras.backend.eval(outputs))
-        self.assertEqual(len(new_states), 2)
-        state_list[i].append(keras.backend.eval(new_states[0]))
-        additional_state_list[i].append(keras.backend.eval(new_states[1]))
+    for i, kwargs in enumerate(kwargs_list):
+      last_output, outputs, new_states = keras.backend.rnn(rnn_fn, inputs,
+                                                           initial_states,
+                                                           **kwargs)
+      # check static shape inference
+      self.assertEqual(last_output.get_shape().as_list(),
+                       [num_samples, output_dim])
+      self.assertEqual(outputs.get_shape().as_list(),
+                       [num_samples, timesteps, output_dim])
+      # for state in new_states:
+      #   self.assertEqual(state.get_shape().as_list(),
+      #                     [num_samples, output_dim])
+      self.assertEqual(new_states[0].get_shape().as_list(),
+                       [num_samples, output_dim])
+      self.assertEqual(new_states[1].get_shape().as_list(),
+                       [num_samples, 2 * output_dim])
+
+      last_output_list[i].append(keras.backend.eval(last_output))
+      outputs_list[i].append(keras.backend.eval(outputs))
+      self.assertLen(new_states, 2)
+      state_list[i].append(keras.backend.eval(new_states[0]))
+      additional_state_list[i].append(keras.backend.eval(new_states[1]))
 
       def assert_list_pairwise(z_list, atol=1e-05):
         for (z1, z2) in zip(z_list[1:], z_list[:-1]):
@@ -1356,53 +1325,52 @@ class BackendNNOpsTest(test.TestCase, parameterized.TestCase):
 class TestCTC(test.TestCase):
 
   def test_ctc_decode(self):
-    with self.cached_session():
-      depth = 6
-      seq_len_0 = 5
-      input_prob_matrix_0 = np.asarray(
-          [[0.30999, 0.309938, 0.0679938, 0.0673362, 0.0708352, 0.173908],
-           [0.215136, 0.439699, 0.0370931, 0.0393967, 0.0381581, 0.230517],
-           [0.199959, 0.489485, 0.0233221, 0.0251417, 0.0233289, 0.238763],
-           [0.279611, 0.452966, 0.0204795, 0.0209126, 0.0194803, 0.20655],
-           [0.51286, 0.288951, 0.0243026, 0.0220788, 0.0219297, 0.129878],
-           # Random entry added in at time=5
-           [0.155251, 0.164444, 0.173517, 0.176138, 0.169979, 0.160671]],
-          dtype=np.float32)
-
-      # len max_time_steps array of batch_size x depth matrices
-      inputs = ([input_prob_matrix_0[t, :][np.newaxis, :]
-                 for t in range(seq_len_0)] +  # Pad to max_time_steps = 8
-                2 * [np.zeros((1, depth), dtype=np.float32)])
-
-      inputs = keras.backend.variable(np.asarray(inputs).transpose((1, 0, 2)))
-
-      # batch_size length vector of sequence_lengths
-      input_length = keras.backend.variable(
-          np.array([seq_len_0], dtype=np.int32))
-      # batch_size length vector of negative log probabilities
-      log_prob_truth = np.array([
-          -3.5821197,  # output beam 0
-          -3.777835    # output beam 1
-      ], np.float32)[np.newaxis, :]
-
-      decode_truth = [np.array([1, 0]), np.array([0, 1, 0])]
-      beam_width = 2
-      top_paths = 2
-
-      decode_pred_tf, log_prob_pred_tf = keras.backend.ctc_decode(
-          inputs,
-          input_length,
-          greedy=False,
-          beam_width=beam_width,
-          top_paths=top_paths)
-
-      self.assertEqual(len(decode_pred_tf), top_paths)
-      log_prob_pred = keras.backend.eval(log_prob_pred_tf)
-      for i in range(top_paths):
-        self.assertTrue(
-            np.alltrue(
-                decode_truth[i] == keras.backend.eval(decode_pred_tf[i])))
-      self.assertAllClose(log_prob_truth, log_prob_pred)
+    depth = 6
+    seq_len_0 = 5
+    input_prob_matrix_0 = np.asarray(
+        [[0.30999, 0.309938, 0.0679938, 0.0673362, 0.0708352, 0.173908],
+         [0.215136, 0.439699, 0.0370931, 0.0393967, 0.0381581, 0.230517],
+         [0.199959, 0.489485, 0.0233221, 0.0251417, 0.0233289, 0.238763],
+         [0.279611, 0.452966, 0.0204795, 0.0209126, 0.0194803, 0.20655],
+         [0.51286, 0.288951, 0.0243026, 0.0220788, 0.0219297, 0.129878],
+         # Random entry added in at time=5
+         [0.155251, 0.164444, 0.173517, 0.176138, 0.169979, 0.160671]],
+        dtype=np.float32)
+
+    # len max_time_steps array of batch_size x depth matrices
+    inputs = ([input_prob_matrix_0[t, :][np.newaxis, :]
+               for t in range(seq_len_0)] +  # Pad to max_time_steps = 8
+              2 * [np.zeros((1, depth), dtype=np.float32)])
+
+    inputs = keras.backend.variable(np.asarray(inputs).transpose((1, 0, 2)))
+
+    # batch_size length vector of sequence_lengths
+    input_length = keras.backend.variable(
+        np.array([seq_len_0], dtype=np.int32))
+    # batch_size length vector of negative log probabilities
+    log_prob_truth = np.array([
+        -3.5821197,  # output beam 0
+        -3.777835    # output beam 1
+    ], np.float32)[np.newaxis, :]
+
+    decode_truth = [np.array([1, 0]), np.array([0, 1, 0])]
+    beam_width = 2
+    top_paths = 2
+
+    decode_pred_tf, log_prob_pred_tf = keras.backend.ctc_decode(
+        inputs,
+        input_length,
+        greedy=False,
+        beam_width=beam_width,
+        top_paths=top_paths)
+
+    self.assertEqual(len(decode_pred_tf), top_paths)
+    log_prob_pred = keras.backend.eval(log_prob_pred_tf)
+    for i in range(top_paths):
+      self.assertTrue(
+          np.alltrue(
+              decode_truth[i] == keras.backend.eval(decode_pred_tf[i])))
+    self.assertAllClose(log_prob_truth, log_prob_pred)
 
   @test_util.run_v1_only('b/120545219')
   def test_ctc_batch_cost(self):
@@ -1463,29 +1431,26 @@ class TestCTC(test.TestCase):
 class TestRandomOps(test.TestCase):
 
   def test_random_binomial(self):
-    with self.cached_session():
-      np.random.seed(123)
-      x = keras.backend.random_binomial((1000, 1000), p=0.5)
-      self.assertAllClose(np.mean(keras.backend.eval(x)), 0.5, atol=0.1)
+    np.random.seed(123)
+    x = keras.backend.random_binomial((1000, 1000), p=0.5)
+    self.assertAllClose(np.mean(keras.backend.eval(x)), 0.5, atol=0.1)
 
   def test_truncated_normal(self):
-    with self.cached_session():
-      np.random.seed(123)
-      x = keras.backend.truncated_normal((1000, 1000), mean=0.0, stddev=1.0)
-      y = keras.backend.eval(x)
-      self.assertAllClose(np.mean(y), 0., atol=0.1)
-      self.assertAllClose(np.std(y), 0.88, atol=0.1)
-      self.assertAllClose(np.max(y), 2., atol=0.1)
-      self.assertAllClose(np.min(y), -2., atol=0.1)
+    np.random.seed(123)
+    x = keras.backend.truncated_normal((1000, 1000), mean=0.0, stddev=1.0)
+    y = keras.backend.eval(x)
+    self.assertAllClose(np.mean(y), 0., atol=0.1)
+    self.assertAllClose(np.std(y), 0.88, atol=0.1)
+    self.assertAllClose(np.max(y), 2., atol=0.1)
+    self.assertAllClose(np.min(y), -2., atol=0.1)
 
   def test_string_input(self):
-    with self.cached_session():
-      seq = keras.Sequential([
-          keras.layers.InputLayer(input_shape=(1,), dtype=dtypes.string),
-          keras.layers.Lambda(lambda x: x[0])
-      ])
-      preds = seq.predict([['tensorflow eager']])
-      self.assertEqual(preds.shape, (1,))
+    seq = keras.Sequential([
+        keras.layers.InputLayer(input_shape=(1,), dtype=dtypes.string),
+        keras.layers.Lambda(lambda x: x[0])
+    ])
+    preds = seq.predict([['tensorflow eager']])
+    self.assertEqual(preds.shape, (1,))
 
 
 class BackendGraphTests(test.TestCase):
-- 
GitLab


From 84edc28bcf8cb73f17acd1d267c17d76c417ccfe Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 14:39:56 -0800
Subject: [PATCH 0998/2345] Properly handle errors while parsing op data.

PiperOrigin-RevId: 229996694
---
 tensorflow/lite/core/api/BUILD                |  1 +
 .../lite/core/api/flatbuffer_conversions.cc   | 25 +++--
 .../core/api/flatbuffer_conversions_test.cc   | 99 ++++++++++++-------
 tensorflow/lite/kernels/reshape_test.cc       | 10 --
 4 files changed, 79 insertions(+), 56 deletions(-)

diff --git a/tensorflow/lite/core/api/BUILD b/tensorflow/lite/core/api/BUILD
index 6a43b0322d..db6b4a2d18 100644
--- a/tensorflow/lite/core/api/BUILD
+++ b/tensorflow/lite/core/api/BUILD
@@ -51,6 +51,7 @@ cc_test(
     srcs = ["flatbuffer_conversions_test.cc"],
     deps = [
         ":api",
+        "//tensorflow/lite:string",
         "//tensorflow/lite/c:c_api_internal",
         "@com_google_googletest//:gtest",
     ],
diff --git a/tensorflow/lite/core/api/flatbuffer_conversions.cc b/tensorflow/lite/core/api/flatbuffer_conversions.cc
index 7f27f1f503..bcf7680290 100644
--- a/tensorflow/lite/core/api/flatbuffer_conversions.cc
+++ b/tensorflow/lite/core/api/flatbuffer_conversions.cc
@@ -28,22 +28,27 @@ namespace {
 // Copies the contents from the flatbuffer int vector `flatbuffer` into the
 // int array `buffer`. `flat_vector` and `buffer` represent the same
 // configuration operation for a given operation.
-void FlatBufferIntVectorToArray(int max_size_of_buffer,
-                                const flatbuffers::Vector<int32_t>* flat_vector,
-                                int* buffer, ErrorReporter* error_reporter) {
+TfLiteStatus FlatBufferIntVectorToArray(
+    int max_size_of_buffer, const flatbuffers::Vector<int32_t>* flat_vector,
+    int* buffer, ErrorReporter* error_reporter, const char* op_name) {
   if (!flat_vector) {
-    error_reporter->Report("Input array not provided for operation.\n");
+    error_reporter->Report("Input array not provided for operation '%s'.\n",
+                           op_name);
+    return kTfLiteError;
   } else {
     int num_dimensions = flat_vector->Length();
     if (num_dimensions > max_size_of_buffer / sizeof(int)) {
       error_reporter->Report(
-          "Found too many dimensions in the operation's input array.\n");
+          "Found too many dimensions in the input array of operation '%s'.\n",
+          op_name);
+      return kTfLiteError;
     } else {
       for (int i = 0; i < num_dimensions; ++i) {
         buffer[i] = flat_vector->Get(i);
       }
     }
   }
+  return kTfLiteOk;
 }
 
 }  // namespace
@@ -452,8 +457,9 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type,
       auto* params = allocator->AllocatePOD<TfLiteReshapeParams>();
       if (auto* schema_params = op->builtin_options_as_ReshapeOptions()) {
         auto* new_shape = schema_params->new_shape();
-        FlatBufferIntVectorToArray(sizeof(params->shape), new_shape,
-                                   params->shape, error_reporter);
+        TF_LITE_ENSURE_STATUS(FlatBufferIntVectorToArray(
+            sizeof(params->shape), new_shape, params->shape, error_reporter,
+            "reshape"));
         params->num_dimensions = new_shape->Length();
       }
       *builtin_data = reinterpret_cast<void*>(params);
@@ -521,8 +527,9 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type,
       auto* params = allocator->AllocatePOD<TfLiteSqueezeParams>();
       if (auto* schema_params = op->builtin_options_as_SqueezeOptions()) {
         const auto& squeeze_dims = schema_params->squeeze_dims();
-        FlatBufferIntVectorToArray(sizeof(params->squeeze_dims), squeeze_dims,
-                                   params->squeeze_dims, error_reporter);
+        TF_LITE_ENSURE_STATUS(FlatBufferIntVectorToArray(
+            sizeof(params->squeeze_dims), squeeze_dims, params->squeeze_dims,
+            error_reporter, "squeeze"));
         params->num_squeeze_dims = squeeze_dims->Length();
       }
       *builtin_data = reinterpret_cast<void*>(params);
diff --git a/tensorflow/lite/core/api/flatbuffer_conversions_test.cc b/tensorflow/lite/core/api/flatbuffer_conversions_test.cc
index 4d1d1b21fd..4a5de48302 100644
--- a/tensorflow/lite/core/api/flatbuffer_conversions_test.cc
+++ b/tensorflow/lite/core/api/flatbuffer_conversions_test.cc
@@ -17,8 +17,10 @@ limitations under the License.
 
 #include <cstring>
 
+#include <gmock/gmock.h>
 #include <gtest/gtest.h>
 #include "tensorflow/lite/c/builtin_op_data.h"
+#include "tensorflow/lite/string.h"
 
 namespace tflite {
 namespace {
@@ -33,6 +35,8 @@ class MockErrorReporter : public ErrorReporter {
   char* GetBuffer() { return buffer_; }
   int GetBufferSize() { return buffer_size_; }
 
+  string GetAsString() const { return string(buffer_, buffer_size_); }
+
  private:
   static constexpr int kBufferSize = 256;
   char buffer_[kBufferSize];
@@ -60,25 +64,56 @@ class MockDataAllocator : public BuiltinDataAllocator {
 
 }  // namespace
 
-TEST(FlatbufferConversions, TestParseOpDataConv) {
-  MockErrorReporter mock_reporter;
-  ErrorReporter* reporter = &mock_reporter;
-  MockDataAllocator mock_allocator;
-
-  flatbuffers::FlatBufferBuilder builder;
-  flatbuffers::Offset<void> conv_options =
-      CreateConv2DOptions(builder, Padding_SAME, 1, 2,
-                          ActivationFunctionType_RELU, 3, 4)
-          .Union();
-  flatbuffers::Offset<Operator> conv_offset = CreateOperatorDirect(
-      builder, 0, nullptr, nullptr, BuiltinOptions_Conv2DOptions, conv_options,
-      nullptr, CustomOptionsFormat_FLEXBUFFERS, nullptr);
-  builder.Finish(conv_offset);
-  void* conv_pointer = builder.GetBufferPointer();
-  const Operator* conv_op = flatbuffers::GetRoot<Operator>(conv_pointer);
+class FlatbufferConversionsTest : public ::testing::Test {
+ public:
+  const Operator* BuildTestOperator(BuiltinOptions op_type,
+                                    flatbuffers::Offset<void> options) {
+    flatbuffers::Offset<Operator> offset =
+        CreateOperatorDirect(builder_, 0, nullptr, nullptr, op_type, options,
+                             nullptr, CustomOptionsFormat_FLEXBUFFERS, nullptr);
+    builder_.Finish(offset);
+    void* pointer = builder_.GetBufferPointer();
+    return flatbuffers::GetRoot<Operator>(pointer);
+  }
+
+ protected:
+  MockErrorReporter mock_reporter_;
+  MockDataAllocator mock_allocator_;
+  flatbuffers::FlatBufferBuilder builder_;
+};
+
+TEST_F(FlatbufferConversionsTest, ParseBadSqueeze) {
+  const Operator* op = BuildTestOperator(
+      BuiltinOptions_SqueezeOptions, CreateSqueezeOptions(builder_).Union());
+  void* output_data = nullptr;
+  EXPECT_NE(kTfLiteOk, ParseOpData(op, BuiltinOperator_SQUEEZE, &mock_reporter_,
+                                   &mock_allocator_, &output_data));
+  EXPECT_THAT(mock_reporter_.GetAsString(),
+              ::testing::ContainsRegex(
+                  "Input array not provided for operation 'squeeze'"));
+}
+
+TEST_F(FlatbufferConversionsTest, ParseBadReshape) {
+  const Operator* op = BuildTestOperator(
+      BuiltinOptions_ReshapeOptions, CreateSqueezeOptions(builder_).Union());
+  void* output_data = nullptr;
+  EXPECT_NE(kTfLiteOk, ParseOpData(op, BuiltinOperator_RESHAPE, &mock_reporter_,
+                                   &mock_allocator_, &output_data));
+  EXPECT_THAT(mock_reporter_.GetAsString(),
+              ::testing::ContainsRegex(
+                  "Input array not provided for operation 'reshape'"));
+}
+
+TEST_F(FlatbufferConversionsTest, TestParseOpDataConv) {
+  const Operator* conv_op =
+      BuildTestOperator(BuiltinOptions_Conv2DOptions,
+                        CreateConv2DOptions(builder_, Padding_SAME, 1, 2,
+                                            ActivationFunctionType_RELU, 3, 4)
+                            .Union());
   void* output_data = nullptr;
-  EXPECT_EQ(kTfLiteOk, ParseOpData(conv_op, BuiltinOperator_CONV_2D, reporter,
-                                   &mock_allocator, &output_data));
+  EXPECT_EQ(kTfLiteOk,
+            ParseOpData(conv_op, BuiltinOperator_CONV_2D, &mock_reporter_,
+                        &mock_allocator_, &output_data));
   EXPECT_NE(nullptr, output_data);
   TfLiteConvParams* params = reinterpret_cast<TfLiteConvParams*>(output_data);
   EXPECT_EQ(kTfLitePaddingSame, params->padding);
@@ -89,30 +124,20 @@ TEST(FlatbufferConversions, TestParseOpDataConv) {
   EXPECT_EQ(4, params->dilation_height_factor);
 }
 
-TEST(FlatbufferConversions, TestParseOpDataCustom) {
-  MockErrorReporter mock_reporter;
-  ErrorReporter* reporter = &mock_reporter;
-  MockDataAllocator mock_allocator;
-
-  flatbuffers::FlatBufferBuilder builder;
-  flatbuffers::Offset<void> null_options;
-  flatbuffers::Offset<Operator> custom_offset = CreateOperatorDirect(
-      builder, 0, nullptr, nullptr, BuiltinOptions_NONE, null_options, nullptr,
-      CustomOptionsFormat_FLEXBUFFERS, nullptr);
-  builder.Finish(custom_offset);
-  void* custom_pointer = builder.GetBufferPointer();
-  const Operator* custom_op = flatbuffers::GetRoot<Operator>(custom_pointer);
+TEST_F(FlatbufferConversionsTest, TestParseOpDataCustom) {
+  const Operator* custom_op =
+      BuildTestOperator(BuiltinOptions_NONE, flatbuffers::Offset<void>());
   void* output_data = nullptr;
-  EXPECT_EQ(kTfLiteOk, ParseOpData(custom_op, BuiltinOperator_CUSTOM, reporter,
-                                   &mock_allocator, &output_data));
+  EXPECT_EQ(kTfLiteOk,
+            ParseOpData(custom_op, BuiltinOperator_CUSTOM, &mock_reporter_,
+                        &mock_allocator_, &output_data));
   EXPECT_EQ(nullptr, output_data);
 }
 
-TEST(FlatbufferConversions, TestConvertTensorType) {
-  MockErrorReporter mock_reporter;
-  ErrorReporter* reporter = &mock_reporter;
+TEST_F(FlatbufferConversionsTest, TestConvertTensorType) {
   TfLiteType type;
-  EXPECT_EQ(kTfLiteOk, ConvertTensorType(TensorType_FLOAT32, &type, reporter));
+  EXPECT_EQ(kTfLiteOk,
+            ConvertTensorType(TensorType_FLOAT32, &type, &mock_reporter_));
   EXPECT_EQ(kTfLiteFloat32, type);
 }
 
diff --git a/tensorflow/lite/kernels/reshape_test.cc b/tensorflow/lite/kernels/reshape_test.cc
index e2210aeaf0..e9d12a9def 100644
--- a/tensorflow/lite/kernels/reshape_test.cc
+++ b/tensorflow/lite/kernels/reshape_test.cc
@@ -137,21 +137,11 @@ TEST_P(ReshapeOpTest, MismatchedDimensions) {
 #endif
 
 TEST_P(ReshapeOpTest, TooManyDimensions) {
-  if (GetParam() == kAsReshapeOption) {
 #ifdef GTEST_HAS_DEATH_TEST
     EXPECT_DEATH(ReshapeOpModel<float>({1, 1, 2, 1, 1, 1, 1, 1, 1}, {9},
                                        {1, 1, 1, 1, 1, 1, 1, 1, 2}, GetParam()),
                  "Found too many dimensions");
 #endif
-  } else {
-    ReshapeOpModel<float> m({1, 1, 2, 1, 1, 1, 1, 1, 1}, {9},
-                            {1, 1, 1, 1, 1, 1, 1, 1, 2}, GetParam());
-    m.SetInput({3, 4});
-    m.Invoke();
-    EXPECT_THAT(m.GetOutput(), ElementsAreArray({3, 4}));
-    EXPECT_THAT(m.GetOutputShape(),
-                ElementsAreArray({1, 1, 1, 1, 1, 1, 1, 1, 2}));
-  }
 }
 
 #ifdef GTEST_HAS_DEATH_TEST
-- 
GitLab


From fe7fb35c9cafc37e731074c7ca5b9f7e04649c3c Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Fri, 18 Jan 2019 14:45:01 -0800
Subject: [PATCH 0999/2345] Best-effort lift initializers out of tf.functions
 to avoid racy initialization.

PiperOrigin-RevId: 229997649
---
 tensorflow/python/eager/def_function.py | 58 ++++++++++++++++++++++++-
 1 file changed, 56 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py
index e5950f5454..2e8b82c68d 100644
--- a/tensorflow/python/eager/def_function.py
+++ b/tensorflow/python/eager/def_function.py
@@ -25,6 +25,7 @@ import weakref
 from tensorflow.python.eager import context
 from tensorflow.python.eager import function as function_lib
 from tensorflow.python.eager import lift_to_graph
+from tensorflow.python.eager import tape
 from tensorflow.python.framework import func_graph as func_graph_module
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import control_flow_ops
@@ -56,6 +57,9 @@ class UnliftedInitializerVariable(resource_variable_ops.ResourceVariable):
                dtype=None,
                constraint=None,
                add_initializers_to=None,
+               lifted_initializer_graph=None,
+               lifted_all_initializers=None,
+               lifted_placeholders=None,
                **unused_kwargs):
     """Creates a variable.
 
@@ -89,6 +93,11 @@ class UnliftedInitializerVariable(resource_variable_ops.ResourceVariable):
       add_initializers_to: if not None and not in legacy graph mode, the
         initializer tensor will be added to this map instead of adding the
         assignment to the function.
+      lifted_initializer_graph: FuncGraph to try to lift initializers to.
+      lifted_all_initializers: list with one boolean element, which will be
+        set to False if we cannot lift this initializer to the above graph.
+      lifted_placeholders: placeholders for resource handles lifted out of
+        this graph.
 
     Raises:
       ValueError: If the initial value is not specified, or does not have a
@@ -167,6 +176,7 @@ class UnliftedInitializerVariable(resource_variable_ops.ResourceVariable):
             with ops.name_scope("Assign") as n, ops.colocate_with(self._handle):
               self._initializer_op = resource_variable_ops.assign_variable_op(
                   self._handle, lifted_initializer, name=n)
+              assign = self._initializer_op
           with ops.name_scope("Read"), ops.colocate_with(self._handle):
             # Manually assign reads to the handle's device to avoid log
             # messages.
@@ -177,6 +187,7 @@ class UnliftedInitializerVariable(resource_variable_ops.ResourceVariable):
       else:
         if add_initializers_to is not None:
           add_initializers_to[self] = initial_value
+          assign = None
         else:
           def assign_fn():
             with ops.name_scope("Assign") as n, ops.colocate_with(self._handle):
@@ -190,9 +201,18 @@ class UnliftedInitializerVariable(resource_variable_ops.ResourceVariable):
             return ops.convert_to_tensor(0)
           # Note: this cond is always guaranteed to run because we're inside a
           # defun which will insert automatic control dependencies.
-          control_flow_ops.cond(
+          assign = control_flow_ops.cond(
               resource_variable_ops.var_is_initialized_op(self._handle),
               not_assign_fn, assign_fn)
+      if lifted_initializer_graph is not None and assign is not None:
+        try:
+          handle_placeholder = ops.convert_to_tensor(self._handle)
+          op_map = lift_to_graph.lift_to_graph(
+              assign, lifted_initializer_graph,
+              sources=[handle_placeholder])
+          lifted_placeholders.append((self._handle, op_map[handle_placeholder]))
+        except ValueError:
+          lifted_all_initializers[0] = False
 
     # After the handle has been created, set up a way to clean it up when
     # executing eagerly. We'll hold the only reference to the deleter, so that
@@ -205,6 +225,19 @@ class UnliftedInitializerVariable(resource_variable_ops.ResourceVariable):
     self._cached_shape_as_list = None
 
 
+class FunctionDeleter(object):
+
+  def __init__(self, func_graph):
+    self.func_graph = func_graph
+
+  def __del__(self):
+    try:
+      func_graph_module.dismantle_func_graph(self.func_graph)
+    except:  # pylint: disable=bare-except
+      # Note: bare except here because this can be noisy at shutdown time.
+      pass
+
+
 class Function(object):
   """Wrapper class for the graph functions defined for a Python function.
 
@@ -296,11 +329,17 @@ class Function(object):
     """
 
     created_variables = []
+    lifted_initializer_graph = func_graph_module.FuncGraph("initializer")
+    lifted_all_initializers = [True]
+    lifted_placeholders = []
 
     def variable_capturing_scope(unused_next_creator, **kwds):
       """Creates UnliftedInitializerVariables and saves references to them."""
       v = UnliftedInitializerVariable(
-          add_initializers_to=add_initializers_to, **kwds)
+          add_initializers_to=add_initializers_to,
+          lifted_initializer_graph=lifted_initializer_graph,
+          lifted_all_initializers=lifted_all_initializers,
+          lifted_placeholders=lifted_placeholders, **kwds)
       created_variables.append(weakref.ref(v))
       return v
 
@@ -308,9 +347,13 @@ class Function(object):
     self._stateful_fn = self._defun_with_scope(variable_capturing_scope)
     self._stateful_fn._name = self._name  # pylint: disable=protected-access
     # Force the definition of the function for these arguments
+    self._lifted_initializer_graph = lifted_initializer_graph
+    self._graph_deleter = FunctionDeleter(self._lifted_initializer_graph)
+    self._lifted_placeholders = lifted_placeholders
     self._concrete_stateful_fn = (
         self._stateful_fn._get_concrete_function_internal_garbage_collected(  # pylint: disable=protected-access
             *args, **kwds))
+    self._lifted_all_initializers = lifted_all_initializers[0]
 
     def invalid_creator_scope(*unused_args, **unused_kwds):
       """Disables variable creation."""
@@ -338,6 +381,17 @@ class Function(object):
 
     # This is the first call of __call__, so we have to initialize.
     self._initialize(args, kwds)
+    if self._lifted_all_initializers and self._lifted_placeholders:
+      with ops.init_scope():
+        handles, placeholders = zip(*self._lifted_placeholders)
+        if context.executing_eagerly():
+          lifted_fn = function_lib._EagerDefinedFunction(  # pylint: disable=protected-access
+              "initializer" + str(ops.uid()),
+              self._lifted_initializer_graph,
+              placeholders, [], {})
+          with tape.stop_recording():
+            lifted_fn.call(context.context(), list(handles))
+      return self._stateless_fn(*args, **kwds)
     canon_args, canon_kwds = self._canonicalize_function_inputs(args, kwds)
 
     if not self._created_variables:
-- 
GitLab


From a8e0f21a3770b5fb16330cf8617e0c2bb1558300 Mon Sep 17 00:00:00 2001
From: Mingsheng Hong <hongm@google.com>
Date: Fri, 18 Jan 2019 14:48:30 -0800
Subject: [PATCH 1000/2345] Added various debugging support to C++ eager layer:
 tensor handle and eager operation. Added experimental C APIs for debug
 prints.

Added some VLOG statements for to help memory/pointer debugging.

PiperOrigin-RevId: 229998268
---
 tensorflow/c/c_api_experimental.cc            | 13 ++++++++++++
 tensorflow/c/c_api_experimental.h             | 12 +++++++++++
 tensorflow/c/c_api_experimental_test.cc       | 20 +++++++++++++++++++
 tensorflow/c/eager/c_api.cc                   |  1 +
 .../common_runtime/eager/eager_operation.cc   | 19 ++++++++++++++++++
 .../common_runtime/eager/eager_operation.h    |  2 ++
 .../common_runtime/eager/tensor_handle.cc     | 11 ++++++++++
 .../core/common_runtime/eager/tensor_handle.h |  3 +++
 8 files changed, 81 insertions(+)

diff --git a/tensorflow/c/c_api_experimental.cc b/tensorflow/c/c_api_experimental.cc
index f04b285037..92f7da7de0 100644
--- a/tensorflow/c/c_api_experimental.cc
+++ b/tensorflow/c/c_api_experimental.cc
@@ -128,6 +128,14 @@ const char* TF_GraphDebugString(TF_Graph* graph, size_t* len) {
   return ret;
 }
 
+char* TF_FunctionDebugString(TF_Function* func, size_t* len) {
+  const auto& debug_str = func->fdef.DebugString();
+  *len = debug_str.size();
+  char* ret = static_cast<char*>(malloc(*len + 1));
+  memcpy(ret, debug_str.c_str(), *len + 1);
+  return ret;
+}
+
 // On success, returns a set of TF_Function instances from `text_proto` of
 // GraphDef type. These functions must be deleted by calling TF_DeleteFunction.
 //
@@ -8748,6 +8756,11 @@ void TFE_TensorHandlePrintDebugString(TFE_TensorHandle* handle) {
   TF_DeleteStatus(status);
 }
 
+void TFE_OpPrintDebugString(TFE_Op* op) {
+  VLOG(1) << "TFE_OpPrintDebugString() over " << op;
+  LOG(INFO) << op->operation.DebugString();
+}
+
 struct TFE_ExecuteOpNotification {
   TFE_ExecuteOpNotification() : status(TF_NewStatus(), TF_DeleteStatus) {}
   tensorflow::Notification n;
diff --git a/tensorflow/c/c_api_experimental.h b/tensorflow/c/c_api_experimental.h
index e6d04d0c2b..e8da4099b8 100644
--- a/tensorflow/c/c_api_experimental.h
+++ b/tensorflow/c/c_api_experimental.h
@@ -84,6 +84,15 @@ TF_CAPI_EXPORT extern TF_Buffer* TF_CreateRunOptions(
 TF_CAPI_EXPORT extern const char* TF_GraphDebugString(TF_Graph* graph,
                                                       size_t* len);
 
+// Returns the function content in a human-readable format, with length set in
+// `len`. The format is subject to change in the future.
+// The returned string is heap-allocated, and caller should call free() on it.
+//
+// Do not return const char*, because some foreign language binding
+// (e.g. swift) cannot then call free() on the returned pointer.
+TF_CAPI_EXPORT extern char* TF_FunctionDebugString(TF_Function* func,
+                                                   size_t* len);
+
 // Creates a stack of data set + iterator nodes, currently hard-coded to return
 // a sequence of 3 float values <42.0, 43.0, 44.0> over 3 calls. On success,
 // returns the IteratorGetNext node, which caller can run or feed into an node.
@@ -181,6 +190,8 @@ TF_CAPI_EXPORT extern TFE_TensorHandle* TFE_DequeueVariantTensor(
 TF_CAPI_EXPORT extern void TFE_TensorHandlePrintDebugString(
     TFE_TensorHandle* handle);
 
+TF_CAPI_EXPORT extern void TFE_OpPrintDebugString(TFE_Op* op);
+
 typedef struct TFE_ExecuteOpNotification TFE_ExecuteOpNotification;
 
 // Allows invoking a kernel asynchronously, and explicitly returns a
@@ -255,6 +266,7 @@ TF_CAPI_EXPORT extern void TFE_EnableCollectiveOps(TFE_Context* ctx,
                                                    const void* proto,
                                                    size_t proto_len,
                                                    TF_Status* status);
+
 #ifdef __cplusplus
 } /* end extern "C" */
 #endif
diff --git a/tensorflow/c/c_api_experimental_test.cc b/tensorflow/c/c_api_experimental_test.cc
index daa7701b7f..65a4e10b26 100644
--- a/tensorflow/c/c_api_experimental_test.cc
+++ b/tensorflow/c/c_api_experimental_test.cc
@@ -296,5 +296,25 @@ TEST(CAPI_EXPERIMENTAL, TFE_ExecuteOpInNewThreadTest_Blocking) {
   TF_DeleteStatus(status);
 }
 
+TEST(CAPI_EXPERIMENTAL, DebugPrint) {
+  TF_Status* status = TF_NewStatus();
+  TFE_ContextOptions* opts = TFE_NewContextOptions();
+  CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  TFE_Context* ctx = TFE_NewContext(opts, status);
+  CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  TFE_DeleteContextOptions(opts);
+
+  TFE_TensorHandle* m = TestMatrixTensorHandle();
+  TFE_Op* op = MatMulOp(ctx, m, m);
+
+  CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  TFE_OpPrintDebugString(op);
+
+  TFE_DeleteTensorHandle(m);
+  TFE_DeleteOp(op);
+  TFE_DeleteContext(ctx);
+  TF_DeleteStatus(status);
+}
+
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index e2574d5311..a92ba1f0f3 100755
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -712,6 +712,7 @@ void TFE_OpSetAttrFunctionList(TFE_Op* op, const char* attr_name,
 
 void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
                  TF_Status* status) {
+  VLOG(1) << "Calling TFE_Execute() on op " << op;
   tensorflow::gtl::InlinedVector<tensorflow::TensorHandle*, 2> handle_retvals(
       *num_retvals);
   status->status =
diff --git a/tensorflow/core/common_runtime/eager/eager_operation.cc b/tensorflow/core/common_runtime/eager/eager_operation.cc
index 0718e68932..b10320ca30 100644
--- a/tensorflow/core/common_runtime/eager/eager_operation.cc
+++ b/tensorflow/core/common_runtime/eager/eager_operation.cc
@@ -35,4 +35,23 @@ void EagerOperation::ConsumeInput(tensorflow::TensorHandle* h) {
   inputs_.push_back(h);
   attrs_.NumInputs(static_cast<int>(inputs_.size()));
 }
+
+string EagerOperation::DebugString() const {
+  string out;
+  VLOG(1) << "EagerOperation::DebugString() over " << this;
+
+  strings::StrAppend(&out, "Name: ", name_, "\n");
+  strings::StrAppend(
+      &out, "Device: ", Device() ? Device()->DebugString() : "[]", "\n");
+  for (const auto& input : inputs_) {
+    VLOG(1) << "Input ptr: " << input;
+    strings::StrAppend(&out, "Input: ", input->DebugString(), "\n");
+  }
+
+  NodeDef ndef;
+  Attrs().FillAttrValueMap(ndef.mutable_attr());
+  strings::StrAppend(&out, "Attrs: ", ndef.DebugString(), "\n");
+  return out;
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/eager/eager_operation.h b/tensorflow/core/common_runtime/eager/eager_operation.h
index 5a9e1f0292..23a2d1bf98 100644
--- a/tensorflow/core/common_runtime/eager/eager_operation.h
+++ b/tensorflow/core/common_runtime/eager/eager_operation.h
@@ -64,6 +64,8 @@ class EagerOperation {
 
   void SetUseXla(bool use_xla) { use_xla_ = use_xla; }
 
+  string DebugString() const;
+
  private:
   tensorflow::EagerContext* ctx_;  // Must outlive the EagerOperation.
   const tensorflow::string name_;
diff --git a/tensorflow/core/common_runtime/eager/tensor_handle.cc b/tensorflow/core/common_runtime/eager/tensor_handle.cc
index eaca6d7b32..5e5936f1cf 100644
--- a/tensorflow/core/common_runtime/eager/tensor_handle.cc
+++ b/tensorflow/core/common_runtime/eager/tensor_handle.cc
@@ -312,4 +312,15 @@ Device* GetResourceDevice(const Tensor& t, EagerContext* ctx) {
   return it->second;
 }
 
+string TensorHandle::DebugString() const {
+  VLOG(1) << "Calling TensorHandle::DebugString() on " << this;
+
+  string out;
+  strings::StrAppend(&out, "Device: ", device_ ? device_->DebugString() : "[]");
+  // Consider supporting non-CPU tensors (when device_ is non-NULL) if needed.
+  strings::StrAppend(&out, ", Tensor: ", device_ ? "?" : tensor_.DebugString(),
+                     "\n");
+  return out;
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/eager/tensor_handle.h b/tensorflow/core/common_runtime/eager/tensor_handle.h
index a4727d6f76..0ecd9e6ddc 100644
--- a/tensorflow/core/common_runtime/eager/tensor_handle.h
+++ b/tensorflow/core/common_runtime/eager/tensor_handle.h
@@ -59,6 +59,7 @@ class TensorHandle : public core::RefCounted {
                Device* op_device, Device* resource_device, EagerContext* ctx);
 
   ~TensorHandle() override {
+    VLOG(1) << "Deleting TensorHandle " << this;
     if (call_on_destroy_) {
       call_on_destroy_();
     }
@@ -114,6 +115,8 @@ class TensorHandle : public core::RefCounted {
 
   bool IsRemote();
 
+  string DebugString() const;
+
  private:
   // If the contents of the Tensor pointed to by this handle is yet to be
   // computed by a EagerNode, this function will block till that compuatation is
-- 
GitLab


From d319d8d716b58d71ee0794025a7679e47ed79622 Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Fri, 18 Jan 2019 15:11:35 -0800
Subject: [PATCH 1001/2345] Differentiate calls to colocate_with from private
 vs public API.

colocate_with has been deprecated from public API. There are still TF internal uses of colocate_with but previously calls to private and public were treated the same and resulted in deprecation warnings for internal uses.

PiperOrigin-RevId: 230002109
---
 tensorflow/python/framework/ops.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index e0f61622e4..63b7108f66 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -5145,12 +5145,19 @@ def _colocate_with_for_gradient(op, gradient_uid, ignore_existing=False):
         op, gradient_uid=gradient_uid, ignore_existing=ignore_existing)
 
 
+# Internal interface to colocate_with. colocate_with has been deprecated from
+# public API. There are still a few internal uses of colocate_with. Add internal
+# only API for those uses to avoid deprecation warning.
+def colocate_with(op, ignore_existing=False):
+  return _colocate_with_for_gradient(op, None, ignore_existing=ignore_existing)
+
+
 @deprecation.deprecated(
     date=None,
     instructions="Colocations handled automatically by placer.")
 @tf_export(v1=["colocate_with"])
-def colocate_with(op, ignore_existing=False):
-  return _colocate_with_for_gradient(op, None, ignore_existing=ignore_existing)
+def _colocate_with(op, ignore_existing=False):
+  return colocate_with(op, ignore_existing)
 
 
 @tf_export("control_dependencies")
-- 
GitLab


From fb4381d0b58ff000db720356d3fa62b2a6ba25d3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 15:16:37 -0800
Subject: [PATCH 1002/2345] Automated rollback of commit
 916df403fbfc799ac5b6af49a0c1cdf3846e66aa

PiperOrigin-RevId: 230002892
---
 tensorflow/python/keras/engine/base_layer.py      |  9 +++++----
 tensorflow/python/keras/engine/base_layer_test.py | 15 +--------------
 2 files changed, 6 insertions(+), 18 deletions(-)

diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index 93849e771b..f63fb4e694 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -513,10 +513,11 @@ class Layer(checkpointable.Checkpointable):
       ValueError: if the layer's `call` method returns None (an invalid value).
     """
     input_list = nest.flatten(inputs)
-    # Accept NumPy inputs by converting to Tensors.
-    if all(isinstance(x, (np.ndarray, float, int)) for x in input_list):
-      inputs = nest.map_structure(ops.convert_to_tensor, inputs)
-      input_list = nest.flatten(inputs)
+    if context.executing_eagerly():
+      # Accept NumPy inputs by converting to Tensors when executing eagerly.
+      if all(isinstance(x, (np.ndarray, float, int)) for x in input_list):
+        inputs = nest.map_structure(ops.convert_to_tensor, inputs)
+        input_list = nest.flatten(inputs)
 
     # We will attempt to build a TF graph if & only if all inputs are symbolic.
     # This is always the case in graph mode. It can also be the case in eager
diff --git a/tensorflow/python/keras/engine/base_layer_test.py b/tensorflow/python/keras/engine/base_layer_test.py
index c925a8ca8a..ebee4a3043 100644
--- a/tensorflow/python/keras/engine/base_layer_test.py
+++ b/tensorflow/python/keras/engine/base_layer_test.py
@@ -26,8 +26,6 @@ from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import test_util
-from tensorflow.python.keras import keras_parameterized
-from tensorflow.python.keras import testing_utils
 from tensorflow.python.keras.engine import base_layer
 from tensorflow.python.keras.optimizer_v2 import rmsprop
 from tensorflow.python.ops import array_ops
@@ -73,7 +71,7 @@ class InvalidLayer(base_layer.Layer):
     raise ValueError('You did something wrong!')
 
 
-class BaseLayerTest(keras_parameterized.TestCase):
+class BaseLayerTest(test.TestCase, parameterized.TestCase):
 
   @parameterized.parameters(DynamicLayer1, DynamicLayer2)
   def test_dynamic_layer_in_functional_model_in_graph_mode(self, layer_class):
@@ -212,17 +210,6 @@ class BaseLayerTest(keras_parameterized.TestCase):
     with self.assertRaisesRegexp(ValueError, 'You did something wrong!'):
       _ = InvalidLayer()(inputs)
 
-  @keras_parameterized.run_with_all_model_types
-  @test_util.run_in_graph_and_eager_modes
-  def test_build_with_numpy_data(self):
-    model_layers = [
-        keras.layers.Dense(3, activation='relu', kernel_initializer='ones'),
-        keras.layers.Dense(1, activation='sigmoid', kernel_initializer='ones')
-    ]
-    model = testing_utils.get_model_from_layers(model_layers, input_shape=(4,))
-    model(np.zeros((2, 4), dtype='float32'))
-    self.assertTrue(model.built)
-
   def test_using_symbolic_tensors_with_tf_ops(self):
     # Single-input.
     x = keras.Input((3,))
-- 
GitLab


From eefbb37190dc4634fa651a47840bce7a7ea16ba4 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Fri, 18 Jan 2019 15:23:08 -0800
Subject: [PATCH 1003/2345] Copy clustering ops to core.

PiperOrigin-RevId: 230003911
---
 tensorflow/python/BUILD                      |  25 +
 tensorflow/python/ops/clustering_ops.py      | 770 +++++++++++++++++++
 tensorflow/python/ops/clustering_ops_test.py | 212 +++++
 3 files changed, 1007 insertions(+)
 create mode 100644 tensorflow/python/ops/clustering_ops.py
 create mode 100644 tensorflow/python/ops/clustering_ops_test.py

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 30de1fa440..4690bed62b 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -118,6 +118,7 @@ py_library(
         ":check_ops",
         ":client",
         ":client_testlib",
+        ":clustering_ops",
         ":collective_ops",
         ":cond_v2",
         ":confusion_matrix",
@@ -2112,6 +2113,30 @@ tf_py_test(
     ],
 )
 
+py_library(
+    name = "clustering_ops",
+    srcs = ["ops/clustering_ops.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":clustering_ops_gen",
+        ":framework",
+        ":ops",
+        ":training",
+    ],
+)
+
+tf_py_test(
+    name = "clustering_ops_test",
+    size = "medium",
+    srcs = ["ops/clustering_ops_test.py"],
+    additional_deps = [
+        ":client_testlib",
+        ":clustering_ops",
+        ":framework_for_generated_wrappers",
+        "//third_party/py/numpy",
+    ],
+)
+
 py_library(
     name = "collective_ops",
     srcs = ["ops/collective_ops.py"],
diff --git a/tensorflow/python/ops/clustering_ops.py b/tensorflow/python/ops/clustering_ops.py
new file mode 100644
index 0000000000..d48b89cbac
--- /dev/null
+++ b/tensorflow/python/ops/clustering_ops.py
@@ -0,0 +1,770 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Clustering Operations."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import check_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import gen_clustering_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn_impl
+from tensorflow.python.ops import random_ops
+from tensorflow.python.ops import state_ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.ops.embedding_ops import embedding_lookup
+# go/tf-wildcard-import
+# pylint: disable=wildcard-import
+from tensorflow.python.ops.gen_clustering_ops import *
+# pylint: enable=wildcard-import
+
+# Euclidean distance between vectors U and V is defined as \\(||U - V||_F\\)
+# which is the square root of the sum of the absolute squares of the elements
+# difference.
+SQUARED_EUCLIDEAN_DISTANCE = 'squared_euclidean'
+# Cosine distance between vectors U and V is defined as
+# \\(1 - (U \dot V) / (||U||_F ||V||_F)\\)
+COSINE_DISTANCE = 'cosine'
+
+RANDOM_INIT = 'random'
+KMEANS_PLUS_PLUS_INIT = 'kmeans_plus_plus'
+KMC2_INIT = 'kmc2'
+
+# The name of the variable holding the cluster centers. Used by the Estimator.
+CLUSTERS_VAR_NAME = 'clusters'
+
+
+class KMeans(object):
+  """Creates the graph for k-means clustering."""
+
+  def __init__(self,
+               inputs,
+               num_clusters,
+               initial_clusters=RANDOM_INIT,
+               distance_metric=SQUARED_EUCLIDEAN_DISTANCE,
+               use_mini_batch=False,
+               mini_batch_steps_per_iteration=1,
+               random_seed=0,
+               kmeans_plus_plus_num_retries=2,
+               kmc2_chain_length=200):
+    """Creates an object for generating KMeans clustering graph.
+
+    This class implements the following variants of K-means algorithm:
+
+    If use_mini_batch is False, it runs standard full batch K-means. Each step
+    runs a single iteration of K-Means. This step can be run sharded across
+    multiple workers by passing a list of sharded inputs to this class. Note
+    however that a single step needs to process the full input at once.
+
+    If use_mini_batch is True, it runs a generalization of the mini-batch
+    K-means algorithm. It runs multiple iterations, where each iteration is
+    composed of mini_batch_steps_per_iteration steps. Two copies of cluster
+    centers are maintained: one that is updated at the end of each iteration,
+    and one that is updated every step. The first copy is used to compute
+    cluster allocations for each step, and for inference, while the second copy
+    is the one updated each step using the mini-batch update rule. After each
+    iteration is complete, this second copy is copied back the first copy.
+
+    Note that for use_mini_batch=True, when mini_batch_steps_per_iteration=1,
+    the algorithm reduces to the standard mini-batch algorithm. Also by setting
+    mini_batch_steps_per_iteration = num_inputs / batch_size, the algorithm
+    becomes an asynchronous version of the full-batch algorithm. Note however
+    that there is no guarantee by this implementation that each input is seen
+    exactly once per iteration. Also, different updates are applied
+    asynchronously without locking. So this asynchronous version may not behave
+    exactly like a full-batch version.
+
+    Args:
+      inputs: An input tensor or list of input tensors. It is assumed that the
+        data points have been previously randomly permuted.
+      num_clusters: An integer tensor specifying the number of clusters. This
+        argument is ignored if initial_clusters is a tensor or numpy array.
+      initial_clusters: Specifies the clusters used during initialization. One
+        of the following:
+        - a tensor or numpy array with the initial cluster centers.
+        - a function f(inputs, k) that returns up to k centers from `inputs`.
+        - "random": Choose centers randomly from `inputs`.
+        - "kmeans_plus_plus": Use kmeans++ to choose centers from `inputs`.
+        - "kmc2": Use the fast k-MC2 algorithm to choose centers from `inputs`.
+        In the last three cases, one batch of `inputs` may not yield
+        `num_clusters` centers, in which case initialization will require
+        multiple batches until enough centers are chosen. In the case of
+        "random" or "kmeans_plus_plus", if the input size is <= `num_clusters`
+        then the entire batch is chosen to be cluster centers.
+      distance_metric: Distance metric used for clustering. Supported options:
+        "squared_euclidean", "cosine".
+      use_mini_batch: If true, use the mini-batch k-means algorithm. Else assume
+        full batch.
+      mini_batch_steps_per_iteration: Number of steps after which the updated
+        cluster centers are synced back to a master copy.
+      random_seed: Seed for PRNG used to initialize seeds.
+      kmeans_plus_plus_num_retries: For each point that is sampled during
+        kmeans++ initialization, this parameter specifies the number of
+        additional points to draw from the current distribution before selecting
+        the best. If a negative value is specified, a heuristic is used to
+        sample O(log(num_to_sample)) additional points.
+      kmc2_chain_length: Determines how many candidate points are used by the
+        k-MC2 algorithm to produce one new cluster centers. If a (mini-)batch
+        contains less points, one new cluster center is generated from the
+        (mini-)batch.
+
+    Raises:
+      ValueError: An invalid argument was passed to initial_clusters or
+        distance_metric.
+    """
+    if isinstance(initial_clusters, str) and initial_clusters not in [
+        RANDOM_INIT, KMEANS_PLUS_PLUS_INIT, KMC2_INIT
+    ]:
+      raise ValueError(
+          "Unsupported initialization algorithm '%s'" % initial_clusters)
+    if distance_metric not in [SQUARED_EUCLIDEAN_DISTANCE, COSINE_DISTANCE]:
+      raise ValueError("Unsupported distance metric '%s'" % distance_metric)
+    self._inputs = inputs if isinstance(inputs, list) else [inputs]
+    self._num_clusters = num_clusters
+    self._initial_clusters = initial_clusters
+    self._distance_metric = distance_metric
+    self._use_mini_batch = use_mini_batch
+    self._mini_batch_steps_per_iteration = int(mini_batch_steps_per_iteration)
+    self._random_seed = random_seed
+    self._kmeans_plus_plus_num_retries = kmeans_plus_plus_num_retries
+    self._kmc2_chain_length = kmc2_chain_length
+
+  @classmethod
+  def _distance_graph(cls, inputs, clusters, distance_metric):
+    """Computes distance between each input and each cluster center.
+
+    Args:
+      inputs: list of input Tensors.
+      clusters: cluster Tensor.
+      distance_metric: distance metric used for clustering
+
+    Returns:
+      list of Tensors, where each element corresponds to each element in inputs.
+      The value is the distance of each row to all the cluster centers.
+      Currently only Euclidean distance and cosine distance are supported.
+    """
+    assert isinstance(inputs, list)
+    if distance_metric == SQUARED_EUCLIDEAN_DISTANCE:
+      return cls._compute_euclidean_distance(inputs, clusters)
+    elif distance_metric == COSINE_DISTANCE:
+      return cls._compute_cosine_distance(
+          inputs, clusters, inputs_normalized=True)
+    else:
+      assert False, str(distance_metric)
+
+  @classmethod
+  def _compute_euclidean_distance(cls, inputs, clusters):
+    """Computes Euclidean distance between each input and each cluster center.
+
+    Args:
+      inputs: list of input Tensors.
+      clusters: cluster Tensor.
+
+    Returns:
+      list of Tensors, where each element corresponds to each element in inputs.
+      The value is the distance of each row to all the cluster centers.
+    """
+    output = []
+    for inp in inputs:
+      with ops.colocate_with(inp, ignore_existing=True):
+        # Computes Euclidean distance. Note the first and third terms are
+        # broadcast additions.
+        squared_distance = (
+            math_ops.reduce_sum(math_ops.square(inp), 1, keepdims=True) -
+            2 * math_ops.matmul(inp, clusters, transpose_b=True) +
+            array_ops.transpose(
+                math_ops.reduce_sum(
+                    math_ops.square(clusters), 1, keepdims=True)))
+        output.append(squared_distance)
+
+    return output
+
+  @classmethod
+  def _compute_cosine_distance(cls, inputs, clusters, inputs_normalized=True):
+    """Computes cosine distance between each input and each cluster center.
+
+    Args:
+      inputs: list of input Tensor.
+      clusters: cluster Tensor
+      inputs_normalized: if True, it assumes that inp and clusters are
+      normalized and computes the dot product which is equivalent to the cosine
+      distance. Else it L2 normalizes the inputs first.
+
+    Returns:
+      list of Tensors, where each element corresponds to each element in inp.
+      The value is the distance of each row to all the cluster centers.
+    """
+    output = []
+    if not inputs_normalized:
+      with ops.colocate_with(clusters, ignore_existing=True):
+        clusters = nn_impl.l2_normalize(clusters, dim=1)
+    for inp in inputs:
+      with ops.colocate_with(inp, ignore_existing=True):
+        if not inputs_normalized:
+          inp = nn_impl.l2_normalize(inp, dim=1)
+        output.append(1 - math_ops.matmul(inp, clusters, transpose_b=True))
+    return output
+
+  def _infer_graph(self, inputs, clusters):
+    """Maps input to closest cluster and the score.
+
+    Args:
+      inputs: list of input Tensors.
+      clusters: Tensor of cluster centers.
+
+    Returns:
+      List of tuple, where each value in tuple corresponds to a value in inp.
+      The tuple has following three elements:
+      all_scores: distance of each input to each cluster center.
+      score: distance of each input to closest cluster center.
+      cluster_idx: index of cluster center closest to the corresponding input.
+    """
+    assert isinstance(inputs, list)
+    # Pairwise distances are used only by transform(). In all other cases, this
+    # sub-graph is not evaluated.
+    scores = self._distance_graph(inputs, clusters, self._distance_metric)
+    output = []
+    if (self._distance_metric == COSINE_DISTANCE and
+        not self._clusters_l2_normalized()):
+      # The cosine distance between normalized vectors x and y is the same as
+      # 2 * squared_euclidean_distance. We are using this fact and reusing the
+      # nearest_neighbors op.
+      # TODO(ands): Support COSINE distance in nearest_neighbors and remove
+      # this.
+      with ops.colocate_with(clusters, ignore_existing=True):
+        clusters = nn_impl.l2_normalize(clusters, dim=1)
+    for inp, score in zip(inputs, scores):
+      with ops.colocate_with(inp, ignore_existing=True):
+        (indices, distances) = gen_clustering_ops.nearest_neighbors(
+            inp, clusters, 1)
+        if self._distance_metric == COSINE_DISTANCE:
+          distances *= 0.5
+        output.append((score, array_ops.squeeze(distances, [-1]),
+                       array_ops.squeeze(indices, [-1])))
+    return zip(*output)
+
+  def _clusters_l2_normalized(self):
+    """Returns True if clusters centers are kept normalized."""
+    return (self._distance_metric == COSINE_DISTANCE and
+            (not self._use_mini_batch or
+             self._mini_batch_steps_per_iteration > 1))
+
+  def _create_variables(self, num_clusters):
+    """Creates variables.
+
+    Args:
+      num_clusters: an integer Tensor providing the number of clusters.
+
+    Returns:
+      Tuple with following elements:
+      - cluster_centers: a Tensor for storing cluster centers
+      - cluster_centers_initialized: bool Variable indicating whether clusters
+            are initialized.
+      - cluster_counts: a Tensor for storing counts of points assigned to this
+            cluster. This is used by mini-batch training.
+      - cluster_centers_updated: Tensor representing copy of cluster centers
+            that are updated every step.
+      - update_in_steps: numbers of steps left before we sync
+            cluster_centers_updated back to cluster_centers.
+    """
+    init_value = array_ops.constant([], dtype=dtypes.float32)
+    cluster_centers = variable_scope.variable(
+        init_value, name=CLUSTERS_VAR_NAME, validate_shape=False)
+    cluster_centers_initialized = variable_scope.variable(
+        False, dtype=dtypes.bool, name='initialized')
+
+    if self._use_mini_batch and self._mini_batch_steps_per_iteration > 1:
+      # Copy of cluster centers actively updated each step according to
+      # mini-batch update rule.
+      cluster_centers_updated = variable_scope.variable(
+          init_value, name='clusters_updated', validate_shape=False)
+      # How many steps till we copy the updated clusters to cluster_centers.
+      update_in_steps = variable_scope.variable(
+          self._mini_batch_steps_per_iteration,
+          dtype=dtypes.int64,
+          name='update_in_steps')
+      # Count of points assigned to cluster_centers_updated.
+      cluster_counts = variable_scope.variable(
+          array_ops.zeros([num_clusters], dtype=dtypes.int64))
+    else:
+      cluster_centers_updated = cluster_centers
+      update_in_steps = None
+      cluster_counts = (
+          variable_scope.variable(
+              array_ops.ones([num_clusters], dtype=dtypes.int64))
+          if self._use_mini_batch else None)
+    return (cluster_centers, cluster_centers_initialized, cluster_counts,
+            cluster_centers_updated, update_in_steps)
+
+  @classmethod
+  def _l2_normalize_data(cls, inputs):
+    """Normalized the input data."""
+    output = []
+    for inp in inputs:
+      with ops.colocate_with(inp, ignore_existing=True):
+        output.append(nn_impl.l2_normalize(inp, dim=1))
+    return output
+
+  def training_graph(self):
+    """Generate a training graph for kmeans algorithm.
+
+    This returns, among other things, an op that chooses initial centers
+    (init_op), a boolean variable that is set to True when the initial centers
+    are chosen (cluster_centers_initialized), and an op to perform either an
+    entire Lloyd iteration or a mini-batch of a Lloyd iteration (training_op).
+    The caller should use these components as follows. A single worker should
+    execute init_op multiple times until cluster_centers_initialized becomes
+    True. Then multiple workers may execute training_op any number of times.
+
+    Returns:
+      A tuple consisting of:
+      all_scores: A matrix (or list of matrices) of dimensions (num_input,
+        num_clusters) where the value is the distance of an input vector and a
+        cluster center.
+      cluster_idx: A vector (or list of vectors). Each element in the vector
+        corresponds to an input row in 'inp' and specifies the cluster id
+        corresponding to the input.
+      scores: Similar to cluster_idx but specifies the distance to the
+        assigned cluster instead.
+      cluster_centers_initialized: scalar indicating whether clusters have been
+        initialized.
+      init_op: an op to initialize the clusters.
+      training_op: an op that runs an iteration of training.
+    """
+    # Implementation of kmeans.
+    if (isinstance(self._initial_clusters, str) or
+        callable(self._initial_clusters)):
+      initial_clusters = self._initial_clusters
+      num_clusters = ops.convert_to_tensor(self._num_clusters)
+    else:
+      initial_clusters = ops.convert_to_tensor(self._initial_clusters)
+      num_clusters = array_ops.shape(initial_clusters)[0]
+
+    inputs = self._inputs
+    (cluster_centers_var, cluster_centers_initialized, total_counts,
+     cluster_centers_updated,
+     update_in_steps) = self._create_variables(num_clusters)
+    init_op = _InitializeClustersOpFactory(
+        self._inputs, num_clusters, initial_clusters, self._distance_metric,
+        self._random_seed, self._kmeans_plus_plus_num_retries,
+        self._kmc2_chain_length, cluster_centers_var, cluster_centers_updated,
+        cluster_centers_initialized).op()
+    cluster_centers = cluster_centers_var
+
+    if self._distance_metric == COSINE_DISTANCE:
+      inputs = self._l2_normalize_data(inputs)
+      if not self._clusters_l2_normalized():
+        cluster_centers = nn_impl.l2_normalize(cluster_centers, dim=1)
+
+    all_scores, scores, cluster_idx = self._infer_graph(inputs, cluster_centers)
+    if self._use_mini_batch:
+      sync_updates_op = self._mini_batch_sync_updates_op(
+          update_in_steps, cluster_centers_var, cluster_centers_updated,
+          total_counts)
+      assert sync_updates_op is not None
+      with ops.control_dependencies([sync_updates_op]):
+        training_op = self._mini_batch_training_op(
+            inputs, cluster_idx, cluster_centers_updated, total_counts)
+    else:
+      assert cluster_centers == cluster_centers_var
+      training_op = self._full_batch_training_op(
+          inputs, num_clusters, cluster_idx, cluster_centers_var)
+
+    return (all_scores, cluster_idx, scores, cluster_centers_initialized,
+            init_op, training_op)
+
+  def _mini_batch_sync_updates_op(self, update_in_steps, cluster_centers_var,
+                                  cluster_centers_updated, total_counts):
+    if self._use_mini_batch and self._mini_batch_steps_per_iteration > 1:
+      assert update_in_steps is not None
+      with ops.colocate_with(update_in_steps, ignore_existing=True):
+
+        def _f():
+          # Note that there is a race condition here, so we do a best effort
+          # updates here. We reset update_in_steps first so that other workers
+          # don't duplicate the updates. Also we update cluster_center_vars
+          # before resetting total_counts to avoid large updates to
+          # cluster_centers_updated based on partially updated
+          # cluster_center_vars.
+          with ops.control_dependencies([
+              state_ops.assign(update_in_steps,
+                               self._mini_batch_steps_per_iteration - 1)
+          ]):
+            with ops.colocate_with(
+                cluster_centers_updated, ignore_existing=True):
+              if self._distance_metric == COSINE_DISTANCE:
+                cluster_centers = nn_impl.l2_normalize(
+                    cluster_centers_updated, dim=1)
+              else:
+                cluster_centers = cluster_centers_updated
+            with ops.colocate_with(cluster_centers_var, ignore_existing=True):
+              with ops.control_dependencies(
+                  [state_ops.assign(cluster_centers_var, cluster_centers)]):
+                with ops.colocate_with(None, ignore_existing=True):
+                  with ops.control_dependencies([
+                      state_ops.assign(total_counts,
+                                       array_ops.zeros_like(total_counts))
+                  ]):
+                    return array_ops.identity(update_in_steps)
+
+        return control_flow_ops.cond(
+            update_in_steps <= 0, _f,
+            lambda: state_ops.assign_sub(update_in_steps, 1))
+    else:
+      return control_flow_ops.no_op()
+
+  def _mini_batch_training_op(self, inputs, cluster_idx_list, cluster_centers,
+                              total_counts):
+    """Creates an op for training for mini batch case.
+
+    Args:
+      inputs: list of input Tensors.
+      cluster_idx_list: A vector (or list of vectors). Each element in the
+        vector corresponds to an input row in 'inp' and specifies the cluster id
+        corresponding to the input.
+      cluster_centers: Tensor Ref of cluster centers.
+      total_counts: Tensor Ref of cluster counts.
+
+    Returns:
+      An op for doing an update of mini-batch k-means.
+    """
+    update_ops = []
+    for inp, cluster_idx in zip(inputs, cluster_idx_list):
+      with ops.colocate_with(inp, ignore_existing=True):
+        assert total_counts is not None
+        cluster_idx = array_ops.reshape(cluster_idx, [-1])
+        # Dedupe the unique ids of cluster_centers being updated so that updates
+        # can be locally aggregated.
+        unique_ids, unique_idx = array_ops.unique(cluster_idx)
+        num_unique_cluster_idx = array_ops.size(unique_ids)
+        # Fetch the old values of counts and cluster_centers.
+        with ops.colocate_with(total_counts, ignore_existing=True):
+          old_counts = array_ops.gather(total_counts, unique_ids)
+        # TODO(agarwal): This colocation seems to run into problems. Fix it.
+        with ops.colocate_with(cluster_centers, ignore_existing=True):
+          old_cluster_centers = array_ops.gather(cluster_centers, unique_ids)
+        # Locally aggregate the increment to counts.
+        count_updates = math_ops.unsorted_segment_sum(
+            array_ops.ones_like(unique_idx, dtype=total_counts.dtype),
+            unique_idx, num_unique_cluster_idx)
+        # Locally compute the sum of inputs mapped to each id.
+        # For a cluster with old cluster value x, old count n, and with data
+        # d_1,...d_k newly assigned to it, we recompute the new value as
+        # \\(x += (sum_i(d_i) - k * x) / (n + k)\\).
+        # Compute \\(sum_i(d_i)\\), see comment above.
+        cluster_center_updates = math_ops.unsorted_segment_sum(
+            inp, unique_idx, num_unique_cluster_idx)
+        # Shape to enable broadcasting count_updates and learning_rate to inp.
+        # It extends the shape with 1's to match the rank of inp.
+        broadcast_shape = array_ops.concat([
+            array_ops.reshape(num_unique_cluster_idx, [1]),
+            array_ops.ones(
+                array_ops.reshape(array_ops.rank(inp) - 1, [1]),
+                dtype=dtypes.int32)
+        ], 0)
+        # Subtract k * x, see comment above.
+        cluster_center_updates -= math_ops.cast(
+            array_ops.reshape(count_updates, broadcast_shape),
+            inp.dtype) * old_cluster_centers
+        learning_rate = math_ops.reciprocal(
+            math_ops.cast(old_counts + count_updates, inp.dtype))
+        learning_rate = array_ops.reshape(learning_rate, broadcast_shape)
+        # scale by 1 / (n + k), see comment above.
+        cluster_center_updates *= learning_rate
+        # Apply the updates.
+      update_counts = state_ops.scatter_add(total_counts, unique_ids,
+                                            count_updates)
+      update_cluster_centers = state_ops.scatter_add(
+          cluster_centers, unique_ids, cluster_center_updates)
+      update_ops.extend([update_counts, update_cluster_centers])
+    return control_flow_ops.group(*update_ops)
+
+  def _full_batch_training_op(self, inputs, num_clusters, cluster_idx_list,
+                              cluster_centers):
+    """Creates an op for training for full batch case.
+
+    Args:
+      inputs: list of input Tensors.
+      num_clusters: an integer Tensor providing the number of clusters.
+      cluster_idx_list: A vector (or list of vectors). Each element in the
+        vector corresponds to an input row in 'inp' and specifies the cluster id
+        corresponding to the input.
+      cluster_centers: Tensor Ref of cluster centers.
+
+    Returns:
+      An op for doing an update of mini-batch k-means.
+    """
+    cluster_sums = []
+    cluster_counts = []
+    epsilon = constant_op.constant(1e-6, dtype=inputs[0].dtype)
+    for inp, cluster_idx in zip(inputs, cluster_idx_list):
+      with ops.colocate_with(inp, ignore_existing=True):
+        cluster_sums.append(
+            math_ops.unsorted_segment_sum(inp, cluster_idx, num_clusters))
+        cluster_counts.append(
+            math_ops.unsorted_segment_sum(
+                array_ops.reshape(
+                    array_ops.ones(
+                        array_ops.reshape(array_ops.shape(inp)[0], [-1])),
+                    [-1, 1]), cluster_idx, num_clusters))
+    with ops.colocate_with(cluster_centers, ignore_existing=True):
+      new_clusters_centers = math_ops.add_n(cluster_sums) / (
+          math_ops.cast(math_ops.add_n(cluster_counts), cluster_sums[0].dtype) +
+          epsilon)
+      if self._clusters_l2_normalized():
+        new_clusters_centers = nn_impl.l2_normalize(new_clusters_centers, dim=1)
+    return state_ops.assign(cluster_centers, new_clusters_centers)
+
+
+class _InitializeClustersOpFactory(object):
+  """Internal class to create the op to initialize the clusters.
+
+    The op performs this algorithm (see constructor args):
+
+    num_remaining = num_clusters - length(cluster_centers)
+    if num_remaining == 0:
+      assert that cluster_centers_initialized is true
+    else:
+      assert that num_remaining > 0
+      new_centers = choose up to num_remaining initial centers
+      l2-normalize new_centers if using cosine distance
+      all_centers = concat(cluster_centers, new_centers)
+      cluster_centers := all_centers
+      if there is a cluster_centers_updated variable:
+        cluster_centers_updated := cluster_centers
+      num_now_remaining = num_clusters - length(cluster_centers)
+      if num_now_remaining == 0:
+        cluster_centers_initialized := true
+  """
+
+  # TODO(ccolby): Refactor this class so that kmc2 isn't so much a special case.
+
+  def __init__(self, inputs, num_clusters, initial_clusters, distance_metric,
+               random_seed, kmeans_plus_plus_num_retries, kmc2_chain_length,
+               cluster_centers, cluster_centers_updated,
+               cluster_centers_initialized):
+    """Creates an op factory.
+
+    Args:
+      inputs: See KMeans constructor.
+      num_clusters: An integer Tensor providing the number of clusters.
+      initial_clusters: See KMeans constructor.
+      distance_metric: See KMeans constructor.
+      random_seed: See KMeans constructor.
+      kmeans_plus_plus_num_retries: See KMeans constructor.
+      kmc2_chain_length: See KMeans constructor.
+      cluster_centers: The TF variable holding the initial centers. It may
+          already contain some centers when the op is executed.
+      cluster_centers_updated: A second TF variable to hold a copy of the
+          initial centers, used for full-batch mode. In mini-batch mode,
+          cluster_centers_updated is the same variable as cluster_centers.
+      cluster_centers_initialized: A boolean TF variable that will be set
+          to true when all the initial centers have been chosen.
+    """
+    # All of these instance variables are constants.
+    self._inputs = inputs
+    self._num_clusters = num_clusters
+    self._initial_clusters = initial_clusters
+    self._distance_metric = distance_metric
+    self._random_seed = random_seed
+    self._kmeans_plus_plus_num_retries = kmeans_plus_plus_num_retries
+    self._kmc2_chain_length = kmc2_chain_length
+    self._cluster_centers = cluster_centers
+    self._cluster_centers_updated = cluster_centers_updated
+    self._cluster_centers_initialized = cluster_centers_initialized
+
+    self._num_selected = array_ops.shape(self._cluster_centers)[0]
+    self._num_remaining = self._num_clusters - self._num_selected
+    self._num_data = math_ops.add_n(
+        [array_ops.shape(i)[0] for i in self._inputs])
+
+  def _random(self):
+    indices = random_ops.random_uniform(
+        array_ops.reshape(self._num_remaining, [-1]),
+        minval=0,
+        maxval=math_ops.cast(self._num_data, dtypes.int64),
+        seed=self._random_seed,
+        dtype=dtypes.int64)
+    return embedding_lookup(self._inputs, indices, partition_strategy='div')
+
+  def _kmeans_plus_plus(self):
+    # Points from only the first shard are used for initializing centers.
+    # TODO(ands): Use all points.
+    inp = self._inputs[0]
+    if self._distance_metric == COSINE_DISTANCE:
+      inp = nn_impl.l2_normalize(inp, dim=1)
+    return gen_clustering_ops.kmeans_plus_plus_initialization(
+        inp,
+        math_ops.to_int64(self._num_remaining), self._random_seed,
+        self._kmeans_plus_plus_num_retries)
+
+  def _kmc2_multiple_centers(self):
+    """Adds new initial cluster centers using the k-MC2 algorithm.
+
+    In each call to the op, the provided batch is split into subsets based on
+    the specified `kmc2_chain_length`. On each subset, a single Markov chain of
+    the k-MC2 algorithm is used to add *one* new center cluster center. If there
+    are less than `kmc2_chain_length` points in the subset, a single center is
+    added using one Markov chain on the full input. It is assumed that the
+    provided batch has previously been randomly permuted. Otherwise, k-MC2 may
+    return suboptimal centers.
+
+    Returns:
+      An op that adds new cluster centers.
+    """
+    # The op only operates on the first shard of data.
+    first_shard = self._inputs[0]
+    # Number of points in the input that can be used.
+    batch_size = array_ops.shape(first_shard)[0]
+    # Maximum number of subsets such that the size of each subset is at least
+    # `kmc2_chain_length`. Final subsets may be larger.
+    max_to_sample = math_ops.cast(
+        batch_size / self._kmc2_chain_length, dtype=dtypes.int32)
+    # We sample at least one new center and at most all remaining centers.
+    num_to_sample = math_ops.maximum(
+        math_ops.minimum(self._num_remaining, max_to_sample), 1)
+
+    def _cond(i, _):
+      """Stopping condition for the while loop."""
+      return math_ops.less(i, num_to_sample)
+
+    def _body(i, _):
+      """Body that adds a single new center based on a subset."""
+
+      def _sample_random():
+        """Returns a random point as a cluster center."""
+        # By assumption the batch is reshuffled and _sample_random is always
+        # called for i=0. Hence, we simply return the first point.
+        new_center = array_ops.reshape(first_shard[0], [1, -1])
+        if self._distance_metric == COSINE_DISTANCE:
+          new_center = nn_impl.l2_normalize(new_center, dim=1)
+        return new_center
+
+      def _sample_kmc2_chain():
+        """Returns previous centers as well as a new center sampled using k-MC2.
+        """
+        # Extract the subset from the underlying batch.
+        start = i * self._kmc2_chain_length
+        end = start + self._kmc2_chain_length
+        subset = first_shard[start:end]
+        # Compute the distances from points in the subset to previous centers.
+        _, distances = gen_clustering_ops.nearest_neighbors(
+            subset, self._cluster_centers, 1)
+        # Sample index of new center using k-MC2 Markov chain.
+        new_center_index = gen_clustering_ops.kmc2_chain_initialization(
+            array_ops.squeeze(distances), self._random_seed)
+        # Extract actual new center.
+        newly_sampled_center = array_ops.reshape(subset[new_center_index],
+                                                 [1, -1])
+        # Return concatenation with previously sampled centers.
+        if self._distance_metric == COSINE_DISTANCE:
+          newly_sampled_center = nn_impl.l2_normalize(
+              newly_sampled_center, dim=1)
+        return array_ops.concat([self._cluster_centers, newly_sampled_center],
+                                0)
+
+      # Obtain a random point if there are no previously sampled centers.
+      # Otherwise, construct a k-MC2 Markov chain.
+      new_centers = control_flow_ops.cond(
+          math_ops.equal(self._num_selected, 0), _sample_random,
+          _sample_kmc2_chain)
+      # Assign new cluster centers to underlying variable.
+      assigned_centers = state_ops.assign(
+          self._cluster_centers, new_centers, validate_shape=False)
+      if self._cluster_centers_updated is not self._cluster_centers:
+        assigned_centers = state_ops.assign(
+            self._cluster_centers_updated,
+            assigned_centers,
+            validate_shape=False)
+      return i + 1, self._num_clusters - array_ops.shape(assigned_centers)[0]
+
+    # Add num_to_sample new data points.
+    _, num_remaining = control_flow_ops.while_loop(_cond, _body, [0, 0])
+    return num_remaining
+
+  def _greedy_batch_sampler(self, sampler):
+    # If the input dataset size is smaller than the number of centers
+    # remaining, choose the entire input dataset as centers. This can happen
+    # with mini-batch. Otherwise, sample the batch according to the provided
+    # sampler.
+    return control_flow_ops.cond(self._num_data <= self._num_remaining,
+                                 lambda: array_ops.concat(self._inputs, 0),
+                                 sampler)
+
+  def _single_batch_sampler(self, sampler):
+    # Enforce that there are at least as many data points as centers
+    # remaining. This gives the provided sampler the chance to select all
+    # remaining centers from a single batch.
+    with ops.control_dependencies(
+        [check_ops.assert_greater_equal(self._num_data, self._num_remaining)]):
+      return sampler()
+
+  def _choose_initial_centers(self):
+    if isinstance(self._initial_clusters, str):
+      if self._initial_clusters == RANDOM_INIT:
+        return self._greedy_batch_sampler(self._random)
+      else:  # self._initial_clusters == KMEANS_PLUS_PLUS_INIT
+        return self._single_batch_sampler(self._kmeans_plus_plus)
+    elif callable(self._initial_clusters):
+      return self._initial_clusters(self._inputs, self._num_remaining)
+    else:
+      with ops.control_dependencies([
+          check_ops.assert_equal(self._num_remaining,
+                                 array_ops.shape(self._initial_clusters)[0])
+      ]):
+        return self._initial_clusters
+
+  def _add_new_centers(self):
+    """Adds some centers and returns the number of centers remaining."""
+    new_centers = self._choose_initial_centers()
+    if self._distance_metric == COSINE_DISTANCE:
+      new_centers = nn_impl.l2_normalize(new_centers, dim=1)
+    # If cluster_centers is empty, it doesn't have the right shape for concat.
+    all_centers = control_flow_ops.cond(
+        math_ops.equal(self._num_selected, 0), lambda: new_centers,
+        lambda: array_ops.concat([self._cluster_centers, new_centers], 0))
+    # TODO(ccolby): De-dupe all_centers?
+    a = state_ops.assign(
+        self._cluster_centers, all_centers, validate_shape=False)
+    if self._cluster_centers_updated is not self._cluster_centers:
+      a = state_ops.assign(
+          self._cluster_centers_updated, a, validate_shape=False)
+    return self._num_clusters - array_ops.shape(a)[0]
+
+  def _initialize(self):
+    with ops.control_dependencies([
+        check_ops.assert_positive(self._num_remaining),
+    ]):
+      if self._initial_clusters == KMC2_INIT:
+        num_now_remaining = self._kmc2_multiple_centers()
+      else:
+        num_now_remaining = self._add_new_centers()
+      return control_flow_ops.cond(
+          math_ops.equal(num_now_remaining, 0),
+          lambda: state_ops.assign(self._cluster_centers_initialized, True),
+          control_flow_ops.no_op)
+
+  def op(self):
+    """Returns the cluster initializer op."""
+    return control_flow_ops.cond(
+        math_ops.equal(self._num_remaining, 0),
+        lambda: check_ops.assert_equal(self._cluster_centers_initialized, True),
+        self._initialize)
diff --git a/tensorflow/python/ops/clustering_ops_test.py b/tensorflow/python/ops/clustering_ops_test.py
new file mode 100644
index 0000000000..5804c660e6
--- /dev/null
+++ b/tensorflow/python/ops/clustering_ops_test.py
@@ -0,0 +1,212 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.  You may obtain a copy of
+# the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations under
+# the License.
+# ==============================================================================
+"""Tests for clustering_ops."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import clustering_ops
+from tensorflow.python.platform import test
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class KmeansPlusPlusInitializationTest(test.TestCase):
+
+  # All but one input point are close to (101, 1). With uniform random sampling,
+  # it is highly improbable for (-1, -1) to be selected.
+  def setUp(self):
+    self._points = np.array([[100., 0.],
+                             [101., 2.],
+                             [102., 0.],
+                             [100., 1.],
+                             [100., 2.],
+                             [101., 0.],
+                             [101., 0.],
+                             [101., 1.],
+                             [102., 0.],
+                             [-1., -1.]]).astype(np.float32)
+
+  def runTestWithSeed(self, seed):
+    with self.cached_session():
+      sampled_points = clustering_ops.kmeans_plus_plus_initialization(
+          self._points, 3, seed, (seed % 5) - 1)
+      self.assertAllClose(
+          sorted(self.evaluate(sampled_points).tolist()),
+          [[-1., -1.], [101., 1.], [101., 1.]],
+          atol=1.0)
+
+  def testBasic(self):
+    for seed in range(100):
+      self.runTestWithSeed(seed)
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class KMC2InitializationTest(test.TestCase):
+
+  def runTestWithSeed(self, seed):
+    with self.cached_session():
+      distances = np.zeros(1000).astype(np.float32)
+      distances[6] = 10e7
+      distances[4] = 10e3
+
+      sampled_point = clustering_ops.kmc2_chain_initialization(distances, seed)
+      self.assertAllEqual(sampled_point, 6)
+      distances[6] = 0.0
+      sampled_point = clustering_ops.kmc2_chain_initialization(distances, seed)
+      self.assertAllEqual(sampled_point, 4)
+
+  def testBasic(self):
+    for seed in range(100):
+      self.runTestWithSeed(seed)
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class KMC2InitializationLargeTest(test.TestCase):
+
+  def setUp(self):
+    self._distances = np.zeros(1001)
+    self._distances[500] = 100.0
+    self._distances[1000] = 50.0
+
+  def testBasic(self):
+    with self.cached_session():
+      counts = {}
+      seed = 0
+      for i in range(50):
+        sample = self.evaluate(
+            clustering_ops.kmc2_chain_initialization(self._distances, seed + i))
+        counts[sample] = counts.get(sample, 0) + 1
+      self.assertEquals(len(counts), 2)
+      self.assertTrue(500 in counts)
+      self.assertTrue(1000 in counts)
+      self.assertGreaterEqual(counts[500], 5)
+      self.assertGreaterEqual(counts[1000], 5)
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class KMC2InitializationCornercaseTest(test.TestCase):
+
+  def setUp(self):
+    self._distances = np.zeros(10)
+
+  def runTestWithSeed(self, seed):
+    with self.cached_session():
+      sampled_point = clustering_ops.kmc2_chain_initialization(
+          self._distances, seed)
+      self.assertAllEqual(sampled_point, 0)
+
+  def testBasic(self):
+    for seed in range(100):
+      self.runTestWithSeed(seed)
+
+
+@test_util.run_all_in_graph_and_eager_modes
+# A simple test that can be verified by hand.
+class NearestCentersTest(test.TestCase):
+
+  def setUp(self):
+    self._points = np.array([[100., 0.],
+                             [101., 2.],
+                             [99., 2.],
+                             [1., 1.]]).astype(np.float32)
+
+    self._centers = np.array([[100., 0.],
+                              [99., 1.],
+                              [50., 50.],
+                              [0., 0.],
+                              [1., 1.]]).astype(np.float32)
+
+  def testNearest1(self):
+    with self.cached_session():
+      [indices, distances] = clustering_ops.nearest_neighbors(self._points,
+                                                              self._centers, 1)
+      self.assertAllClose(indices, [[0], [0], [1], [4]])
+      self.assertAllClose(distances, [[0.], [5.], [1.], [0.]])
+
+  def testNearest2(self):
+    with self.cached_session():
+      [indices, distances] = clustering_ops.nearest_neighbors(self._points,
+                                                              self._centers, 2)
+      self.assertAllClose(indices, [[0, 1], [0, 1], [1, 0], [4, 3]])
+      self.assertAllClose(distances, [[0., 2.], [5., 5.], [1., 5.], [0., 2.]])
+
+
+@test_util.run_all_in_graph_and_eager_modes
+# A test with large inputs.
+class NearestCentersLargeTest(test.TestCase):
+
+  def setUp(self):
+    num_points = 1000
+    num_centers = 2000
+    num_dim = 100
+    max_k = 5
+    # Construct a small number of random points and later tile them.
+    points_per_tile = 10
+    assert num_points % points_per_tile == 0
+    points = np.random.standard_normal(
+        [points_per_tile, num_dim]).astype(np.float32)
+    # Construct random centers.
+    self._centers = np.random.standard_normal(
+        [num_centers, num_dim]).astype(np.float32)
+
+    # Exhaustively compute expected nearest neighbors.
+    def squared_distance(x, y):
+      return np.linalg.norm(x - y, ord=2)**2
+
+    nearest_neighbors = [
+        sorted([(squared_distance(point, self._centers[j]), j)
+                for j in range(num_centers)])[:max_k] for point in points
+    ]
+    expected_nearest_neighbor_indices = np.array(
+        [[i for _, i in nn] for nn in nearest_neighbors])
+    expected_nearest_neighbor_squared_distances = np.array(
+        [[dist for dist, _ in nn] for nn in nearest_neighbors])
+    # Tile points and expected results to reach requested size (num_points)
+    (self._points, self._expected_nearest_neighbor_indices,
+     self._expected_nearest_neighbor_squared_distances) = (
+         np.tile(x, (int(num_points / points_per_tile), 1))
+         for x in (points, expected_nearest_neighbor_indices,
+                   expected_nearest_neighbor_squared_distances))
+
+  def testNearest1(self):
+    with self.cached_session():
+      [indices, distances] = clustering_ops.nearest_neighbors(self._points,
+                                                              self._centers, 1)
+      self.assertAllClose(
+          indices,
+          self._expected_nearest_neighbor_indices[:, [0]])
+      self.assertAllClose(
+          distances,
+          self._expected_nearest_neighbor_squared_distances[:, [0]])
+
+  def testNearest5(self):
+    with self.cached_session():
+      [indices, distances] = clustering_ops.nearest_neighbors(self._points,
+                                                              self._centers, 5)
+      self.assertAllClose(
+          indices,
+          self._expected_nearest_neighbor_indices[:, 0:5])
+      self.assertAllClose(
+          distances,
+          self._expected_nearest_neighbor_squared_distances[:, 0:5])
+
+
+if __name__ == "__main__":
+  np.random.seed(0)
+  test.main()
-- 
GitLab


From d86e3ae60853df703baa664c6ccfb872fbd67893 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Fri, 18 Jan 2019 15:39:23 -0800
Subject: [PATCH 1004/2345] [tf.data] Add logging to the performance modeling
 framework.

PiperOrigin-RevId: 230006654
---
 tensorflow/core/framework/model.cc | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/framework/model.cc b/tensorflow/core/framework/model.cc
index 47c8dbe61b..a1c87a3f42 100644
--- a/tensorflow/core/framework/model.cc
+++ b/tensorflow/core/framework/model.cc
@@ -380,7 +380,12 @@ std::shared_ptr<Node> Model::AddNode(Node::Factory factory, const string& name,
     output_ = node;
   }
   if (output) {
+    VLOG(3) << "Adding " << node->name() << "(id:" << node->id()
+            << ") as input for " << output->name() << "(id:" << output->id()
+            << ")";
     output->add_input(node);
+  } else {
+    VLOG(3) << "Adding " << node->name() << "(id:" << node->id() << ")";
   }
   collect_resource_usage_ =
       collect_resource_usage_ || node->has_tunable_parameters();
@@ -493,10 +498,13 @@ void Model::RecordStop(const string& name, bool start_output) {
 void Model::RemoveNode(const string& name) {
   mutex_lock l(mu_);
   auto node = gtl::FindOrNull(lookup_table_, name);
-  if (node && (*node)->output()) {
-    (*node)->output()->remove_input(*node);
+  if (node) {
+    if ((*node)->output()) {
+      (*node)->output()->remove_input(*node);
+    }
+    VLOG(3) << "Removing " << (*node)->name() << "(id:" << (*node)->id() << ")";
+    remove_node_hook_(*node);
   }
-  remove_node_hook_(*node);
   lookup_table_.erase(name);
 }
 
-- 
GitLab


From 708090d48995456bfa66615398d8c56dadebe018 Mon Sep 17 00:00:00 2001
From: James Ring <sjr@google.com>
Date: Fri, 18 Jan 2019 15:59:16 -0800
Subject: [PATCH 1005/2345] Make TF_DeleteKernelBuilder not crash on nullptr.

After this change, TF_DeleteKernelBuilder will comply with the
conventions established in c_api.h, namely that *Delete* functions are
safe with nullptr parameters.

PiperOrigin-RevId: 230009727
---
 tensorflow/c/kernels.cc      | 7 ++++---
 tensorflow/c/kernels_test.cc | 4 ++++
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/tensorflow/c/kernels.cc b/tensorflow/c/kernels.cc
index c45efd7fae..9505bf9dda 100644
--- a/tensorflow/c/kernels.cc
+++ b/tensorflow/c/kernels.cc
@@ -48,9 +48,10 @@ TF_KernelBuilder* TF_NewKernelBuilder(
 }
 
 void TF_DeleteKernelBuilder(TF_KernelBuilder* builder) {
-  DCHECK_NE(builder, nullptr);
-  delete builder->cc_builder;
-  delete builder;
+  if (builder != nullptr) {
+    delete builder->cc_builder;
+    delete builder;
+  }
 }
 
 namespace tensorflow {
diff --git a/tensorflow/c/kernels_test.cc b/tensorflow/c/kernels_test.cc
index 531eb598fa..0d2954717e 100644
--- a/tensorflow/c/kernels_test.cc
+++ b/tensorflow/c/kernels_test.cc
@@ -224,4 +224,8 @@ TEST(TestKernel, TestInputAndOutputCount) {
   }
 }
 
+TEST(TestKernel, DeleteKernelBuilderIsOkOnNull) {
+  TF_DeleteKernelBuilder(nullptr);
+}
+
 }  // namespace tensorflow
-- 
GitLab


From f21f60fb993b89d7ea3a136c78a2491b1f1c4783 Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Fri, 18 Jan 2019 16:02:12 -0800
Subject: [PATCH 1006/2345] Make tf2 doc-generator aware of tf_export,
 tf_inspect. Fix tf.__all__ for doc generation. Add site_path flag.

PiperOrigin-RevId: 230010213
---
 tensorflow/tools/docs/BUILD        |  5 +++-
 tensorflow/tools/docs/generate2.py | 39 ++++++++++++++++++++++++++++--
 2 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/tensorflow/tools/docs/BUILD b/tensorflow/tools/docs/BUILD
index b072853a4e..4bb452f256 100644
--- a/tensorflow/tools/docs/BUILD
+++ b/tensorflow/tools/docs/BUILD
@@ -163,7 +163,10 @@ py_binary(
     name = "generate2",
     srcs = ["generate2.py"],
     srcs_version = "PY2AND3",
-    deps = ["//tensorflow:tensorflow_py"],
+    deps = [
+        "//tensorflow:tensorflow_py",
+        "//tensorflow/python:util",
+    ],
 )
 
 py_library(
diff --git a/tensorflow/tools/docs/generate2.py b/tensorflow/tools/docs/generate2.py
index fba909d26d..0a50eb6c23 100644
--- a/tensorflow/tools/docs/generate2.py
+++ b/tensorflow/tools/docs/generate2.py
@@ -31,10 +31,22 @@ from os import path
 
 from absl import app
 from absl import flags
-
 import tensorflow as tf
 
+from tensorflow_docs.api_generator import doc_generator_visitor
 from tensorflow_docs.api_generator import generate_lib
+from tensorflow_docs.api_generator import parser
+
+from tensorflow.python.util import tf_export
+from tensorflow.python.util import tf_inspect
+
+# Use tensorflow's `tf_inspect`, which is aware of `tf_decorator`.
+parser.tf_inspect = tf_inspect
+
+# `tf` has an `__all__` that doesn't list important things like `keras`.
+# The doc generator recognizes `__all__` as the list of public symbols.
+# So patch `tf.__all__` to list everything.
+tf.__all__ = [item_name for item_name, value in tf_inspect.getmembers(tf)]
 
 FLAGS = flags.FLAGS
 
@@ -50,6 +62,28 @@ flags.DEFINE_string(
 flags.DEFINE_bool("search_hints", True,
                   "Include meta-data search hints at the top of each file.")
 
+flags.DEFINE_string("site_path", "",
+                    "The prefix ({site-path}/api_docs/python/...) used in the "
+                    "`_toc.yaml` and `_redirects.yaml` files")
+
+
+# The doc generator isn't aware of tf_export.
+# So prefix the score tuples with -1 when this is the canonical name, +1
+# otherwise. The generator chooses the name with the lowest score.
+class TfExportAwareDocGeneratorVisitor(
+    doc_generator_visitor.DocGeneratorVisitor):
+  """A `tf_export` aware doc_visitor."""
+
+  def _score_name(self, name):
+    canonical = tf_export.get_canonical_name_for_symbol(self._index[name])
+
+    canonical_score = 1
+    if canonical is not None and name == "tf." + canonical:
+      canonical_score = -1
+
+    scores = super(TfExportAwareDocGeneratorVisitor, self)._score_name(name)
+    return (canonical_score,) + scores
+
 
 def build_docs(output_dir, code_url_prefix, search_hints=True):
   """Build api docs for tensorflow v2.
@@ -66,7 +100,8 @@ def build_docs(output_dir, code_url_prefix, search_hints=True):
       base_dir=base_dir,
       search_hints=search_hints,
       code_url_prefix=code_url_prefix,
-      site_path="api_docs/")
+      site_path=FLAGS.site_path,
+      visitor_cls=TfExportAwareDocGeneratorVisitor)
 
   doc_generator.build(output_dir)
 
-- 
GitLab


From e38ca6bfaf6ef77ca97ab76d71ec8b17ad61da29 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 16:09:38 -0800
Subject: [PATCH 1007/2345] Automated rollback of commit
 2e2a3ca8e827138a02da1cf129ab28057b9f9ddd

PiperOrigin-RevId: 230011598
---
 tensorflow/core/kernels/cwise_ops.h | 34 ++++++++++++++++++++++++++---
 tensorflow/workspace.bzl            |  8 +++----
 2 files changed, 35 insertions(+), 7 deletions(-)

diff --git a/tensorflow/core/kernels/cwise_ops.h b/tensorflow/core/kernels/cwise_ops.h
index 15bd533be0..a22d76717a 100644
--- a/tensorflow/core/kernels/cwise_ops.h
+++ b/tensorflow/core/kernels/cwise_ops.h
@@ -51,12 +51,15 @@ struct scalar_arg_op<std::complex<double>> {
 };
 #endif
 
-#if EIGEN_HAS_CXX11_MATH == 0
 template <typename T>
 struct scalar_asinh_op {
   EIGEN_EMPTY_STRUCT_CTOR(scalar_asinh_op)
   EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T operator()(const T& a) const {
+#if EIGEN_HAS_CXX11_MATH
+    return numext::asinh(a);
+#else
     return std::asinh(a);
+#endif  // EIGEN_HAS_CXX11_MATH
   }
 };
 template <typename T>
@@ -68,7 +71,11 @@ template <typename T>
 struct scalar_acosh_op {
   EIGEN_EMPTY_STRUCT_CTOR(scalar_acosh_op)
   EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T operator()(const T& a) const {
+#if EIGEN_HAS_CXX11_MATH
+    return numext::acosh(a);
+#else
     return std::acosh(a);
+#endif  // EIGEN_HAS_CXX11_MATH
   }
 };
 template <typename T>
@@ -80,14 +87,35 @@ template <typename T>
 struct scalar_atanh_op {
   EIGEN_EMPTY_STRUCT_CTOR(scalar_atanh_op)
   EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T operator()(const T& a) const {
+#if EIGEN_HAS_CXX11_MATH
+    return numext::atanh(a);
+#else
     return std::atanh(a);
+#endif  // EIGEN_HAS_CXX11_MATH
   }
 };
 template <typename T>
 struct functor_traits<scalar_atanh_op<T>> {
   enum { Cost = 5 * NumTraits<T>::MulCost, PacketAccess = false };
 };
-#endif
+
+// TODO(rmlarsen): This is a workaround for upstream change
+// https://bitbucket.org/eigen/eigen/commits/f339468d04d0f87caeb6cab9aef568627e9f6ea9
+// that renamed scalar_binary_pow_op to scalar_pow_op and deleted the unary
+// version of the latter. Remove once we upgrade to Eigen 3.3.
+template <typename Scalar, typename Exponent>
+struct scalar_binary_pow_op_google {
+  EIGEN_EMPTY_STRUCT_CTOR(scalar_binary_pow_op_google)
+  EIGEN_DEVICE_FUNC inline Scalar operator()(const Scalar& a,
+                                             const Exponent& b) const {
+    return numext::pow(a, b);
+  }
+};
+
+template <typename Scalar, typename Exponent>
+struct functor_traits<scalar_binary_pow_op_google<Scalar, Exponent>> {
+  enum { Cost = 5 * NumTraits<Scalar>::MulCost, PacketAccess = false };
+};
 
 template <typename Scalar, typename Exponent>
 struct safe_scalar_binary_pow_op {
@@ -837,7 +865,7 @@ template <typename T>
 struct floor_div_real : base<T, Eigen::internal::google_floor_div_real<T>> {};
 
 template <typename T>
-struct pow : base<T, Eigen::internal::scalar_pow_op<T, T>> {};
+struct pow : base<T, Eigen::internal::scalar_binary_pow_op_google<T, T>> {};
 
 template <typename T>
 struct safe_pow : base<T, Eigen::internal::safe_scalar_binary_pow_op<T, T>> {
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 466262ab3f..49c68b6ec0 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -140,11 +140,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "eigen_archive",
         build_file = clean_dep("//third_party:eigen.BUILD"),
-        sha256 = "48678550a32665331d729be87076e576f2502fff325f5b6c2c78ebf7b1b22c7b",
-        strip_prefix = "eigen-eigen-bcc817c0ba98",
+        sha256 = "df54d805573871d03aaef6d27d45439c18ff6e8db215a3b922b2daefd6914147",
+        strip_prefix = "eigen-eigen-ad3bcd81cc49",
         urls = [
-            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/bcc817c0ba98.tar.gz",
-            "https://bitbucket.org/eigen/eigen/get/bcc817c0ba98.tar.gz",
+            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/ad3bcd81cc49.tar.gz",
+            "https://bitbucket.org/eigen/eigen/get/ad3bcd81cc49.tar.gz",
         ],
     )
 
-- 
GitLab


From 6108a72eafbbec6a161f70691b9dcef1798dbbbe Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Fri, 18 Jan 2019 16:16:17 -0800
Subject: [PATCH 1008/2345] Put the calibration resource into the device's
 resource manager as well. This helps to make the calibration process work in
 both the old workflow and the new one.

PiperOrigin-RevId: 230012540
---
 .../contrib/tensorrt/convert/convert_graph.cc |  6 ---
 .../contrib/tensorrt/kernels/trt_engine_op.cc | 43 +++++++++++++------
 .../contrib/tensorrt/kernels/trt_engine_op.h  |  4 +-
 3 files changed, 33 insertions(+), 20 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index e2350f69a6..15cf97a2ee 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -666,14 +666,8 @@ tensorflow::Status CreateTRTNode(const std::vector<EngineInfo>& infos, int pos,
     segment_string = info.segment_graph_def.SerializeAsString();
   }
 
-  // TODO(aaroey): use enum instead, and add a helper method to do the
-  // conversion.
   string prec_string;
   TF_RETURN_IF_ERROR(GetPrecisionModeName(info.precision_mode, &prec_string));
-  if (info.precision_mode == INT8MODE && calibrate_int8 &&
-      !TRTResourceManager::instance()->getManager("TRTCalibration")) {
-    LOG(ERROR) << "Failed to construct calibration storage";
-  }
   tensorflow::NodeDefBuilder node_builder(info.engine_name, "TRTEngineOp");
   if (!info.device.empty()) node_builder.Device(info.device);
   if (VLOG_IS_ON(1)) {
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
index 7548c8ccda..d28d88fa3a 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
@@ -195,18 +195,37 @@ void TRTEngineOp::ExecuteCalibration(OpKernelContext* ctx,
   VLOG(1) << "Executing TRT calibration: " << name();
   helper->Ref();
   tensorflow::core::ScopedUnref sc(helper);
-  // TODO(aaroey): remove the ResourceMgr singleton.
-  auto trt_rm = TRTResourceManager::instance();
-  auto res_mgr = trt_rm->getManager("TRTCalibration");
+  auto res_mgr = ctx->resource_manager();
   TRTCalibrationResource* calib_res = nullptr;
-  auto status = res_mgr->LookupOrCreate(
-      funcdef_name_, "Calibrator", &calib_res,
-      {[ctx, this](TRTCalibrationResource** cr) -> tensorflow::Status {
-        return this->AllocateCalibrationResources(ctx, cr);
-      }});
-  if (!status.ok()) {
-    ctx->SetStatus(status);
-    return;
+  OP_REQUIRES_OK(
+      ctx,
+      res_mgr->LookupOrCreate(
+          "TF_TRT_Calibration", name(),
+          reinterpret_cast<SerializableResourceBase**>(&calib_res),
+          {[ctx, this](SerializableResourceBase** cr) -> tensorflow::Status {
+            return this->AllocateCalibrationResources(ctx, cr);
+          }}));
+  tensorflow::core::ScopedUnref calib_sc(calib_res);
+  // TODO(aaroey): here we also add the resource to the ResourceMgr singleton.
+  // This is needed before we migrate all uses of calib_graph_to_infer_graph()
+  // to the new calibration workflow. After that we'll remove this block.
+  {
+    auto deprecated_rm =
+        TRTResourceManager::instance()->getManager("TRTCalibration");
+    TRTCalibrationResource* copied_resource = nullptr;
+    // Check whether the resource exists, and create it if not.
+    if (deprecated_rm->Lookup(funcdef_name_, "Calibrator", &copied_resource)
+            .ok()) {
+      // Do nothing if the resource exists.
+      copied_resource->Unref();
+    } else {
+      copied_resource = calib_res;
+      // Increase the refcount by 1 then transfer the ownership of that refcount
+      // to the ResourceMgr singleton.
+      copied_resource->Ref();
+      OP_REQUIRES_OK(ctx, deprecated_rm->Create(funcdef_name_, "Calibrator",
+                                                copied_resource));
+    }
   }
   int num_inputs = ctx->num_inputs();
   // Pass input data to calibrator
@@ -542,7 +561,7 @@ EngineContext* TRTEngineOp::GetEngine(
 }
 
 tensorflow::Status TRTEngineOp::AllocateCalibrationResources(
-    OpKernelContext* ctx, TRTCalibrationResource** cr) {
+    OpKernelContext* ctx, SerializableResourceBase** cr) {
   auto cres = new TRTCalibrationResource();
   *cr = cres;
   // Get the allocator.
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
index 7f0f05aa0a..76c6020078 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
@@ -23,6 +23,7 @@ limitations under the License.
 #include "tensorflow/contrib/tensorrt/log/trt_logger.h"
 #include "tensorflow/contrib/tensorrt/resources/trt_allocator.h"
 #include "tensorflow/contrib/tensorrt/resources/trt_lru_cache.h"
+#include "tensorflow/contrib/tensorrt/resources/trt_resources.h"
 #include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/framework/op.h"
@@ -38,7 +39,6 @@ limitations under the License.
 namespace tensorflow {
 namespace tensorrt {
 struct TRTInt8Calibrator;
-class TRTCalibrationResource;
 class AsyncHelper;
 //  TODO(Sami): Remove this file?
 
@@ -69,7 +69,7 @@ class TRTEngineOp : public AsyncOpKernel {
 
   // Allocate necessary resources for calibration
   Status AllocateCalibrationResources(OpKernelContext* ctx,
-                                      TRTCalibrationResource** cr);
+                                      SerializableResourceBase** cr);
 
   // Get engine for the input shape
   EngineContext* GetEngine(const std::vector<TensorShape>& input_shapes,
-- 
GitLab


From fe77e94dbf3d5633be38aacee5f90e1c1302506e Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Fri, 18 Jan 2019 16:25:12 -0800
Subject: [PATCH 1009/2345] Add unit test and fix bugs in AttentionMechanismV2.

PiperOrigin-RevId: 230013805
---
 tensorflow/contrib/seq2seq/BUILD              | 15 +++
 .../kernel_tests/attention_wrapper_v2_test.py | 94 +++++++++++++++++++
 .../seq2seq/python/ops/attention_wrapper.py   | 71 +++++++++-----
 3 files changed, 159 insertions(+), 21 deletions(-)
 create mode 100644 tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_v2_test.py

diff --git a/tensorflow/contrib/seq2seq/BUILD b/tensorflow/contrib/seq2seq/BUILD
index 89176180ae..7d5ba90ded 100644
--- a/tensorflow/contrib/seq2seq/BUILD
+++ b/tensorflow/contrib/seq2seq/BUILD
@@ -213,3 +213,18 @@ cuda_py_test(
         "//tensorflow/python:variables",
     ],
 )
+
+cuda_py_test(
+    name = "attention_wrapper_v2_test",
+    size = "medium",
+    srcs = ["python/kernel_tests/attention_wrapper_v2_test.py"],
+    additional_deps = [
+        ":seq2seq_py",
+        "@absl_py//absl/testing:parameterized",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:init_ops",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:variables",
+    ],
+)
diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_v2_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_v2_test.py
new file mode 100644
index 0000000000..7ff04e1780
--- /dev/null
+++ b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_v2_test.py
@@ -0,0 +1,94 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for contrib.seq2seq.python.ops.attention_wrapper."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+import numpy as np
+
+from tensorflow.contrib.seq2seq.python.ops import attention_wrapper as wrapper
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class AttentionMechanismTest(test.TestCase, parameterized.TestCase):
+
+  def setUp(self):
+    super(AttentionMechanismTest, self).setUp()
+    self.batch = 10
+    self.timestep = 5
+    self.memory_size = 6
+    self.units = 8
+
+    self.memory = ops.convert_to_tensor(
+        np.random.random((self.batch, self.timestep, self.memory_size)),
+        dtype=np.float32)
+    self.query = ops.convert_to_tensor(
+        np.random.random((self.batch, self.units)), dtype=np.float32)
+    self.state = ops.convert_to_tensor(
+        np.random.random((self.batch, self.timestep)), dtype=np.float32)
+
+  @parameterized.named_parameters(
+      ("luong", wrapper.LuongAttentionV2),
+      ("luong_monotonic", wrapper.LuongMonotonicAttentionV2),
+      ("bahdanau", wrapper.BahdanauAttentionV2),
+      ("bahdanau_monotonic", wrapper.BahdanauMonotonicAttentionV2),
+  )
+  def test_attention_shape_inference(self, attention_cls):
+    attention = attention_cls(self.units)
+    attention_score = attention([self.query, self.state, self.memory])
+    self.assertLen(attention_score, 2)
+    self.assertEqual(attention_score[0].shape, (self.batch, self.timestep))
+    self.assertEqual(attention_score[1].shape, (self.batch, self.timestep))
+
+  @parameterized.named_parameters(
+      ("luong", wrapper.LuongAttentionV2),
+      ("luong_monotonic", wrapper.LuongMonotonicAttentionV2),
+      ("bahdanau", wrapper.BahdanauAttentionV2),
+      ("bahdanau_monotonic", wrapper.BahdanauMonotonicAttentionV2),
+  )
+  def test_get_config(self, attention_cls):
+    attention = attention_cls(self.units)
+    config = attention.get_config()
+
+    attention_from_config = attention_cls.from_config(config)
+    config_from_clone = attention_from_config.get_config()
+
+    self.assertDictEqual(config, config_from_clone)
+
+  @parameterized.named_parameters(
+      ("luong", wrapper.LuongAttentionV2),
+      ("luong_monotonic", wrapper.LuongMonotonicAttentionV2),
+      ("bahdanau", wrapper.BahdanauAttentionV2),
+      ("bahdanau_monotonic", wrapper.BahdanauMonotonicAttentionV2),
+  )
+  def test_layer_output(self, attention_cls):
+    attention = attention_cls(self.units)
+
+    score = attention([self.query, self.state, self.memory])
+    self.evaluate(variables.variables_initializer(attention.variables))
+
+    score_val = self.evaluate(score)
+    self.assertLen(score_val, 2)
+    self.assertEqual(score_val[0].shape, (self.batch, self.timestep))
+    self.assertEqual(score_val[1].shape, (self.batch, self.timestep))
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
index 31c62d5849..ae3e7f1b5d 100644
--- a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
+++ b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
@@ -28,7 +28,7 @@ from tensorflow.contrib.framework.python.framework import tensor_util
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
-from tensorflow.python.keras.engine.base_layer import Layer
+from tensorflow.python.keras import layers
 from tensorflow.python.layers import base as layers_base
 from tensorflow.python.layers import core as layers_core
 from tensorflow.python.ops import array_ops
@@ -218,7 +218,7 @@ class _BaseAttentionMechanism(AttentionMechanism):
     return self.initial_alignments(batch_size, dtype)
 
 
-class _BaseAttentionMechanismV2(AttentionMechanism, Layer):
+class _BaseAttentionMechanismV2(AttentionMechanism, layers.Layer):
   """A base AttentionMechanism class providing common functionality.
 
   Common functionality includes:
@@ -251,11 +251,11 @@ class _BaseAttentionMechanismV2(AttentionMechanism, Layer):
         creation.
     """
     if (query_layer is not None
-        and not isinstance(query_layer, layers_base.Layer)):
+        and not isinstance(query_layer, layers.Layer)):
       raise TypeError(
           "query_layer is not a Layer: %s" % type(query_layer).__name__)
     if (memory_layer is not None
-        and not isinstance(memory_layer, layers_base.Layer)):
+        and not isinstance(memory_layer, layers.Layer)):
       raise TypeError(
           "memory_layer is not a Layer: %s" % type(memory_layer).__name__)
     self.query_layer = query_layer
@@ -275,10 +275,12 @@ class _BaseAttentionMechanismV2(AttentionMechanism, Layer):
     self._check_inner_dims_defined = True
 
   def build(self, input_shape):
+    # The layer suppose to take 3 inputs, [query, state, memory].
+    query_input_shape, _, memory_input_shape = input_shape
     if self.query_layer is not None:
-      self.query_layer.build(input_shape)
+      self.query_layer.build(query_input_shape)
     if self.memory_layer is not None:
-      self.memory_layer.build(input_shape)
+      self.memory_layer.build(memory_input_shape)
     # dtype of the layer is known at this moment, create the score_mask_value if
     # needed.
     self.score_mask_value = dtypes.as_dtype(self.dtype).as_numpy_dtype(-np.inf)
@@ -310,8 +312,8 @@ class _BaseAttentionMechanismV2(AttentionMechanism, Layer):
       self.batch_size = (
           tensor_shape.dimension_value(self.keys.shape[0]) or
           array_ops.shape(self.keys)[0])
-      self.alignments_size = (tensor_shape.dimension_value(self.keys.shape[1])
-                              or array_ops.shape(self.keys)[1])
+      self._alignments_size = (tensor_shape.dimension_value(self.keys.shape[1])
+                               or array_ops.shape(self.keys)[1])
       if memory_mask is not None:
         self.probability_fn = lambda score, prev: (  # pylint:disable=g-long-lambda
             self.probability_fn(_maybe_mask_score(
@@ -398,6 +400,8 @@ class _BaseAttentionMechanismV2(AttentionMechanism, Layer):
     """
     # Reconstruct the query and memory layer for parent class.
     from tensorflow.python.keras.layers import deserialize as deserialize_layer  # pylint: disable=g-import-not-at-top
+    # Instead of updating the input, create a copy and use that.
+    config = config.copy()
     query_layer_config = config.pop("query_layer", None)
     if query_layer_config:
       query_layer = deserialize_layer(query_layer_config,
@@ -410,6 +414,10 @@ class _BaseAttentionMechanismV2(AttentionMechanism, Layer):
       config["memory_layer"] = memory_layer
     return config
 
+  @property
+  def alignments_size(self):
+    return self._alignments_size
+
 
 def _luong_score(query, keys, scale):
   """Implements Luong-style (multiplicative) scoring function.
@@ -606,10 +614,13 @@ class LuongAttentionV2(_BaseAttentionMechanismV2):
     wrapped_probability_fn = lambda score, _: probability_fn(score)
     if dtype is None:
       dtype = dtypes.float32
+    memory_layer = kwargs.pop("memory_layer", None)
+    if not memory_layer:
+      memory_layer = layers.Dense(
+          units, name="memory_layer", use_bias=False, dtype=dtype)
     super(LuongAttentionV2, self).__init__(
         query_layer=None,
-        memory_layer=layers_core.Dense(
-            units, name="memory_layer", use_bias=False, dtype=dtype),
+        memory_layer=memory_layer,
         probability_fn=wrapped_probability_fn,
         name=name,
         dtype=dtype,
@@ -640,6 +651,7 @@ class LuongAttentionV2(_BaseAttentionMechanismV2):
       alignments: Tensor of dtype matching `self.values` and shape
         `[batch_size, alignments_size]` (`alignments_size` is memory's
         `max_time`).
+      next_state: Same as the alignments.
     """
     score = _luong_score(query, self.keys, self.scale_weight)
     alignments = self.probability_fn(score, state)
@@ -865,11 +877,17 @@ class BahdanauAttentionV2(_BaseAttentionMechanismV2):
     wrapped_probability_fn = lambda score, _: probability_fn(score)
     if dtype is None:
       dtype = dtypes.float32
+    query_layer = kwargs.pop("query_layer", None)
+    if not query_layer:
+      query_layer = layers.Dense(
+          units, name="query_layer", use_bias=False, dtype=dtype)
+    memory_layer = kwargs.pop("memory_layer", None)
+    if not memory_layer:
+      memory_layer = layers.Dense(
+          units, name="memory_layer", use_bias=False, dtype=dtype)
     super(BahdanauAttentionV2, self).__init__(
-        query_layer=layers_core.Dense(
-            units, name="query_layer", use_bias=False, dtype=dtype),
-        memory_layer=layers_core.Dense(
-            units, name="memory_layer", use_bias=False, dtype=dtype),
+        query_layer=query_layer,
+        memory_layer=memory_layer,
         probability_fn=wrapped_probability_fn,
         name=name,
         dtype=dtype,
@@ -907,6 +925,7 @@ class BahdanauAttentionV2(_BaseAttentionMechanismV2):
       alignments: Tensor of dtype matching `self.values` and shape
         `[batch_size, alignments_size]` (`alignments_size` is memory's
         `max_time`).
+      next_state: same as alignments.
     """
     processed_query = self.query_layer(query) if self.query_layer else query
     score = _bahdanau_score(processed_query, self.keys, self.attention_v,
@@ -1314,11 +1333,17 @@ class BahdanauMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
     wrapped_probability_fn = functools.partial(
         _monotonic_probability_fn, sigmoid_noise=sigmoid_noise, mode=mode,
         seed=sigmoid_noise_seed)
+    query_layer = kwargs.pop("query_layer", None)
+    if not query_layer:
+      query_layer = layers.Dense(
+          units, name="query_layer", use_bias=False, dtype=dtype)
+    memory_layer = kwargs.pop("memory_layer", None)
+    if not memory_layer:
+      memory_layer = layers.Dense(
+          units, name="memory_layer", use_bias=False, dtype=dtype)
     super(BahdanauMonotonicAttentionV2, self).__init__(
-        query_layer=layers_core.Dense(
-            units, name="query_layer", use_bias=False, dtype=dtype),
-        memory_layer=layers_core.Dense(
-            units, name="memory_layer", use_bias=False, dtype=dtype),
+        query_layer=query_layer,
+        memory_layer=memory_layer,
         probability_fn=wrapped_probability_fn,
         name=name,
         dtype=dtype,
@@ -1551,10 +1576,13 @@ class LuongMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
     wrapped_probability_fn = functools.partial(
         _monotonic_probability_fn, sigmoid_noise=sigmoid_noise, mode=mode,
         seed=sigmoid_noise_seed)
+    memory_layer = kwargs.pop("memory_layer", None)
+    if not memory_layer:
+      memory_layer = layers.Dense(
+          units, name="memory_layer", use_bias=False, dtype=dtype)
     super(LuongMonotonicAttentionV2, self).__init__(
         query_layer=None,
-        memory_layer=layers_core.Dense(
-            units, name="memory_layer", use_bias=False, dtype=dtype),
+        memory_layer=memory_layer,
         probability_fn=wrapped_probability_fn,
         name=name,
         dtype=dtype,
@@ -1593,6 +1621,7 @@ class LuongMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
       alignments: Tensor of dtype matching `self.values` and shape
         `[batch_size, alignments_size]` (`alignments_size` is memory's
         `max_time`).
+      next_state: Same as alignments
     """
     score = _luong_score(query, self.keys, self.attention_g)
     score += self.attention_score_bias
@@ -1603,7 +1632,7 @@ class LuongMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
   def get_config(self):
     config = {
         "units": self.units,
-        "normalize": self.normalize,
+        "scale": self.scale,
         "sigmoid_noise": self.sigmoid_noise,
         "sigmoid_noise_seed": self.sigmoid_noise_seed,
         "score_bias_init": self.score_bias_init,
-- 
GitLab


From c2e7173f8f61c74529cfd879becb762b48db37ad Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Fri, 18 Jan 2019 16:38:54 -0800
Subject: [PATCH 1010/2345] Updated Micro documentation on porting to a new
 platform

PiperOrigin-RevId: 230015624
---
 tensorflow/lite/experimental/micro/README.md | 175 ++++++++++++++++++-
 1 file changed, 173 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/experimental/micro/README.md b/tensorflow/lite/experimental/micro/README.md
index b47e0feb79..88f3cd71f5 100644
--- a/tensorflow/lite/experimental/micro/README.md
+++ b/tensorflow/lite/experimental/micro/README.md
@@ -537,7 +537,9 @@ critical parts of the code with versions specifically tailored to your
 architecture. The framework has been designed with this in mind, and we hope the
 combination of small modules and many tests makes it as straightforward as
 possible to swap in your own code a piece at a time, ensuring you have a working
-version at every step.
+version at every step. To write specialized implementations for a platform, it's
+useful to understand how optional components are handled inside the build
+system.
 
 ### Code Module Organization
 
@@ -628,7 +630,7 @@ kernel implementations, but with some specific conventions:
     latest in the ordered list will be chosen. This allows us to express “I’d
     like generically-optimized fixed point if it’s available, but I’d prefer
     something using the CMSIS library” using the list 'fixed_point cmsis'. These
-    tags are passed in as `TAGS="<foo"` on the command line when you use the
+    tags are passed in as `TAGS="<foo>"` on the command line when you use the
     main Makefile to build.
 -   There is an implicit “reference” tag at the start of every list, so that
     it’s possible to support directory structures like the current
@@ -640,3 +642,172 @@ kernel implementations, but with some specific conventions:
     top level.
 -   Tests should be at the parent level, with no platform-specific code.
 -   No platform-specific macros or #ifdef’s should be used in any portable code.
+
+The implementation of these rules is handled inside the Makefile, with a
+[`specialize` function](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/experimental/micro/tools/make/helper_functions.inc#L42)
+that takes a list of reference source file paths as an input, and returns the
+equivalent list with specialized versions of those files swapped in if they
+exist.
+
+### Working with Generated Projects
+
+So far, I've recommended that you use the standalone generated projects for your
+system. You might be wondering why you're not just checking out the full
+[TensorFlow codebase from Github](https://github.com/tensorflow/tensorflow/)?
+The main reason is that there is a lot more diversity of architectures, IDEs,
+support libraries, and operating systems in the embedded world. Many of the
+toolchains require their own copy of source files, or a list of sources to be
+written to a project file. When a developer working on TensorFlow adds a new
+source file or changes its location, we can't expect her to update multiple
+different project files, many of which she may not have the right software to
+verify the change was correct. That means we have to rely on a central listing
+of source files (which in our case is held in the makefile), and then call a
+tool to generate other project files from those. We could ask embedded
+developers to do this process themselves after downloading the main source, but
+running the makefile requires a Linux system which may not be available, takes
+time, and involves downloading a lot of dependencies. That is why we've opted to
+make regular snapshots of the results of generating these projects for popular
+IDEs and platforms, so that embedded developers have a fast and friendly way to
+start using TensorFlow Lite for Microcontrollers.
+
+This does have the disadvantage that you're no longer working directly on the
+main repository, instead you have a copy that's outside of source control. We've
+tried to make the copy as similar to the main repo as possible, for example by
+keeping the paths of all source files the same, and ensuring that there are no
+changes between the copied files and the originals, but it still makes it
+tougher to sync as the main repository is updated. There are also multiple
+copies of the source tree, one for each target, so any change you make to one
+copy has to be manually propagated across all the other projects you care about.
+This doesn't matter so much if you're just using the projects as they are to
+build products, but if you want to support a new platform and have the changes
+reflected in the main code base, you'll have to do some extra work.
+
+As an example, think about the `DebugLog()` implementation we discussed adding
+for a new platform earlier. At this point, you have a new version of
+`debug_log.cc` that does what's required, but how can you share that with the
+wider community? The first step is to pick a tag name for your platform. This
+can either be the operating system (for example 'mbed'), the name of a device
+('bluepill'), or some other text that describes it. This should be a short
+string with no spaces or special characters. Log in or create an account on
+Github, fork the full
+[TensorFlow codebase](https://github.com/tensorflow/tensorflow/) using the
+'Fork' button on the top left, and then grab your fork by using a command like
+`git clone https://github.com/<your user name>/tensorflow`.
+
+You'll either need Linux, MacOS, or Windows with something like CygWin installed
+to run the next steps, since they involve building a makefile. Run the following
+commands from a terminal, inside the root of the source folder:
+
+```
+tensorflow/lite/experimental/micro/tools/make/download_dependencies.sh
+make -f tensorflow/lite/experimental/micro/tools/make/Makefile generate_projects
+```
+
+This will take a few minutes, since it has to download some large toolchains for
+the dependencies. Once it has finished, you should see some folders created
+inside a path like
+`tensorflow/lite/experimental/micro/tools/make/gen/linux_x86_64/prj/`. The exact
+path depends on your host operating system, but you should be able to figure it
+out from all the copy commands. These folders contain the generated project and
+source files, with
+`tensorflow/lite/experimental/micro/tools/make/gen/linux_x86_64/prj/keil`
+containing the Keil uVision targets,
+`tensorflow/lite/experimental/micro/tools/make/gen/linux_x86_64/prj/mbed` with
+the Mbed versions, and so on.
+
+If you've got this far, you've successfully set up the project generation flow.
+Now you need to add your specialized implementation of `DebugLog()`. Start by
+creating a folder inside `tensorflow/lite/experimental/micro/` named after the
+tag you picked earlier. Put your `debug_log.cc` file inside this folder, and
+then run this command, with '<your tag>' replaced by the actual folder name:
+
+```
+make -f tensorflow/lite/experimental/micro/tools/make/Makefile TAGS="<your tag>" generate_projects
+```
+
+If your tag name actually refers to a whole target architecture, then you'll use
+TARGET or TARGET_ARCH instead. For example, here's how a simple RISC-V set of
+projects is generated:
+
+```
+make -f tensorflow/lite/experimental/micro/tools/make/Makefile TARGET="riscv32_mcu" generate_projects
+```
+
+The way it works is the same as TAGS though, it just looks for specialized
+implementations with the same containing folder name.
+
+If you look inside the projects that have been created, you should see that the
+default `DebugLog()` implementation is no longer present at
+`tensorflow/lite/experimental/micro/debug_log.cc`, and instead
+`tensorflow/lite/experimental/micro/<your tag>/debug_log.cc` is being used. Copy
+over the generated project files and try building them in your own IDE. If
+everything works, then you're ready to submit your change.
+
+To do this, run something like:
+
+```
+git add tensorflow/lite/experimental/micro/<your tag>/debug_log.cc
+git commit -a -m "Added DebugLog() support for <your platform>"
+git push origin master
+```
+
+Then go back to https://github.com/<your account>/tensorflow, and choose "New
+Pull Request" near the top. You should then be able to go through the standard
+TensorFlow PR process to get your change added to the main repository, and
+available to the rest of the community!
+
+### Supporting a Platform with Makefiles
+
+The changes you've made so far will enable other developers using the generated
+projects to use your platform, but TensorFlow's continuous integration process
+uses makefiles to build frequently and ensure changes haven't broken the build
+process for different systems. If you are able to convert your build procedure
+into something that can be expressed by a makefile, then we can integrate your
+platform into our CI builds and make sure it continues to work.
+
+Fully describing how to do this is beyond the scope of this documentation, but
+the biggest needs are:
+
+-   A command-line compiler that can be called for every source file.
+-   A list of the arguments to pass into the compiler to build and link all
+    files.
+-   The correct linker map files and startup assembler to ensure `main()` gets
+    called.
+
+### Supporting a Platform with Emulation Testing
+
+Integrating your platform into the makefile process should help us make sure
+that it continues to build, but it doesn't guarantee that the results of the
+build process will run correctly. Running tests is something we require to be
+able to say that TensorFlow officially supports a platform, since otherwise we
+can't guarantee that users will have a good experience when they try using it.
+Since physically maintaining a full set of all supported hardware devices isn't
+feasible, we rely on software emulation to run these tests. A good example is
+our
+[STM32F4 'Bluepill' support](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/experimental/micro/testing/test_bluepill_binary.sh),
+which uses [Docker](https://www.docker.com/) and [Renode](https://renode.io/) to
+run built binaries in an emulator. You can use whatever technologies you want,
+the only requirements are that they capture the debug log output of the tests
+being run in the emulator, and parse them for the string that indicates the test
+was successful. These scripts need to run on Ubuntu 18.04, in a bash
+environment, though Docker is available if you need to install extra software or
+have other dependencies.
+
+### Optimizing for your Platform
+
+Clearly, getting debug logging support is only the beginning of the work you'll
+need to do on a particular platform. It's very likely that you'll want to
+optimize the core deep learning operations that take up the most time when
+running models you care about. The good news is that the process for providing
+optimized implementations is the same as the one you just went through to
+provide your own logging. You'll need to identify parts of the code that are
+bottlenecks, and then add specialized implementations in their own folders.
+These don't need to be platform specific, they can also be broken out by which
+library they rely on for example. [Here's where we do that for the CMSIS
+implementation of integer fast-fourier
+transforms](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/preprocessor.cc).
+This more complex case shows that you can also add helper source files alongside
+the main implementation, as long as you
+[mention them in the platform-specific makefile](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/Makefile.inc).
+You can also do things like update the list of libraries that need to be linked
+in, or add include paths to required headers.
-- 
GitLab


From b99c63abbd331c4d0e459cc3542e3d6a49764d23 Mon Sep 17 00:00:00 2001
From: nammbash <niroopshankar.ammbashankar@intel.com>
Date: Fri, 18 Jan 2019 16:53:38 -0800
Subject: [PATCH 1011/2345] requantization_op_perchannel_support

---
 tensorflow/core/BUILD                         |   1 +
 .../api_def_QuantizedConv2DAndRelu.pbtxt      |   4 +
 ..._QuantizedConv2DAndReluAndRequantize.pbtxt |   4 +
 ...api_def_QuantizedConv2DAndRequantize.pbtxt |   4 +
 .../api_def_QuantizedConv2DWithBias.pbtxt     |   4 +
 ...i_def_QuantizedConv2DWithBiasAndRelu.pbtxt |   4 +
 ...edConv2DWithBiasAndReluAndRequantize.pbtxt |   4 +
 ...QuantizedConv2DWithBiasAndRequantize.pbtxt |   4 +
 ...ithBiasSignedSumAndReluAndRequantize.pbtxt |   4 +
 ...ef_QuantizedConv2DWithBiasSumAndRelu.pbtxt |   4 +
 ...onv2DWithBiasSumAndReluAndRequantize.pbtxt |   4 +
 ...pi_def_RequantizationRangePerChannel.pbtxt |   4 +
 .../api_def_RequantizePerChannel.pbtxt        |   4 +
 tensorflow/core/kernels/BUILD                 |  65 +++-
 .../core/kernels/mkl_quantized_conv_ops.h     |  37 ++-
 ...mkl_requantization_range_per_channel_op.cc | 110 +++++++
 .../core/kernels/mkl_requantize_ops_test.cc   | 297 ++++++++++++++++++
 .../kernels/mkl_requantize_per_channel_op.cc  | 171 ++++++++++
 tensorflow/core/kernels/requantize.cc         |   8 +-
 tensorflow/core/ops/math_ops.cc               |  47 ++-
 20 files changed, 775 insertions(+), 9 deletions(-)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_QuantizedConv2DAndRelu.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_QuantizedConv2DAndReluAndRequantize.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_QuantizedConv2DAndRequantize.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBias.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasAndRelu.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasAndReluAndRequantize.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasAndRequantize.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasSignedSumAndReluAndRequantize.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasSumAndRelu.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasSumAndReluAndRequantize.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_RequantizationRangePerChannel.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_RequantizePerChannel.pbtxt
 create mode 100644 tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc
 create mode 100644 tensorflow/core/kernels/mkl_requantize_ops_test.cc
 create mode 100644 tensorflow/core/kernels/mkl_requantize_per_channel_op.cc

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 084db7a0fd..4be4f105e6 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1449,6 +1449,7 @@ cc_library(
         "//tensorflow/core/kernels:mkl_identity_op",
         "//tensorflow/core/kernels:mkl_input_conversion_op",
         "//tensorflow/core/kernels:mkl_lrn_op",
+        "//tensorflow/core/kernels:mkl_requantize_ops",
         "//tensorflow/core/kernels:mkl_pooling_ops",
         "//tensorflow/core/kernels:mkl_relu_op",
         "//tensorflow/core/kernels:mkl_reshape_op",
diff --git a/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DAndRelu.pbtxt b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DAndRelu.pbtxt
new file mode 100644
index 0000000000..17ff15378c
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DAndRelu.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "QuantizedConv2DAndRelu"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DAndReluAndRequantize.pbtxt b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DAndReluAndRequantize.pbtxt
new file mode 100644
index 0000000000..b3ab3eba2c
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DAndReluAndRequantize.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "QuantizedConv2DAndReluAndRequantize"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DAndRequantize.pbtxt b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DAndRequantize.pbtxt
new file mode 100644
index 0000000000..8b00c2b7f6
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DAndRequantize.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "QuantizedConv2DAndRequantize"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBias.pbtxt b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBias.pbtxt
new file mode 100644
index 0000000000..f309f648ca
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBias.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "QuantizedConv2DWithBias"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasAndRelu.pbtxt b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasAndRelu.pbtxt
new file mode 100644
index 0000000000..b6b73eaae3
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasAndRelu.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "QuantizedConv2DWithBiasAndRelu"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasAndReluAndRequantize.pbtxt b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasAndReluAndRequantize.pbtxt
new file mode 100644
index 0000000000..101f72708a
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasAndReluAndRequantize.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "QuantizedConv2DWithBiasAndReluAndRequantize"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasAndRequantize.pbtxt b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasAndRequantize.pbtxt
new file mode 100644
index 0000000000..697e268415
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasAndRequantize.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "QuantizedConv2DWithBiasAndRequantize"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasSignedSumAndReluAndRequantize.pbtxt b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasSignedSumAndReluAndRequantize.pbtxt
new file mode 100644
index 0000000000..0cf52d6c89
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasSignedSumAndReluAndRequantize.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "QuantizedConv2DWithBiasSignedSumAndReluAndRequantize"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasSumAndRelu.pbtxt b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasSumAndRelu.pbtxt
new file mode 100644
index 0000000000..e91a2b8dc0
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasSumAndRelu.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "QuantizedConv2DWithBiasSumAndRelu"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasSumAndReluAndRequantize.pbtxt b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasSumAndReluAndRequantize.pbtxt
new file mode 100644
index 0000000000..fe3ec528bf
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2DWithBiasSumAndReluAndRequantize.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "QuantizedConv2DWithBiasSumAndReluAndRequantize"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_RequantizationRangePerChannel.pbtxt b/tensorflow/core/api_def/base_api/api_def_RequantizationRangePerChannel.pbtxt
new file mode 100644
index 0000000000..8874a53ec1
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_RequantizationRangePerChannel.pbtxt
@@ -0,0 +1,4 @@
+op {
+graph_op_name:
+  "RequantizationRangePerChannel" visibility : HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_RequantizePerChannel.pbtxt b/tensorflow/core/api_def/base_api/api_def_RequantizePerChannel.pbtxt
new file mode 100644
index 0000000000..0e4cd40a36
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_RequantizePerChannel.pbtxt
@@ -0,0 +1,4 @@
+op {
+graph_op_name:
+  "RequantizePerChannel" visibility : HIDDEN
+}
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 96536e6945..3a6c7119ff 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -6016,7 +6016,9 @@ tf_kernel_library(
 tf_cc_test(
     name = "requantization_range_op_test",
     size = "small",
-    srcs = ["requantization_range_op_test.cc"],
+    srcs = [
+        "requantization_range_op_test.cc",
+    ],
     deps = [
         ":ops_testutil",
         ":ops_util",
@@ -6893,6 +6895,67 @@ tf_mkl_kernel_library(
     deps = NN_DEPS + mkl_deps() + [":cwise_op"],
 )
 
+tf_mkl_kernel_library(
+    name = "mkl_requantize_ops",
+    srcs = [
+        "mkl_requantization_range_per_channel_op.cc",
+        "mkl_requantize_per_channel_op.cc",
+    ],
+    hdrs = [
+        "meta_support.h",
+        "no_op.h",
+        "reference_gemm.h",
+    ],
+    deps = if_mkl(
+        [
+            ":concat_lib_hdrs",
+            ":conv_ops",
+            ":cwise_op",
+            ":eigen_helpers",
+            ":image_resizer_state",
+            ":ops_util",
+            ":pooling_ops",
+            ":quantization_utils",
+            "//tensorflow/core:array_ops_op_lib",
+            "//tensorflow/core:core_cpu",
+            "//tensorflow/core:framework",
+            "//tensorflow/core:lib",
+            "//tensorflow/core:math_ops_op_lib",
+            "//tensorflow/core:nn_ops_op_lib",
+            "//third_party/eigen3",
+            "@gemmlowp",
+            ":transpose_functor",
+            "@mkl_dnn",
+            "//third_party/mkl:intel_binary_blob",
+        ],
+    ),
+)
+
+tf_cc_test_mkl(
+    name = "mkl_requantize_ops_test",
+    size = "small",
+    srcs = ["mkl_requantize_ops_test.cc"],
+    deps = [
+        ":mkl_requantize_ops",
+        ":ops_testutil",
+        ":ops_util",
+        ":quantization_utils",
+        ":quantized_ops",
+        "//tensorflow/cc:cc_ops",
+        "//tensorflow/core:array_ops_op_lib",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:framework_internal",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:tensorflow",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+    ],
+)
+
 tf_cc_test_mkl(
     name = "mkl_fused_ops_test",
     size = "small",
diff --git a/tensorflow/core/kernels/mkl_quantized_conv_ops.h b/tensorflow/core/kernels/mkl_quantized_conv_ops.h
index 10825f6962..84a1ccb4c2 100644
--- a/tensorflow/core/kernels/mkl_quantized_conv_ops.h
+++ b/tensorflow/core/kernels/mkl_quantized_conv_ops.h
@@ -16,16 +16,18 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_KERNELS_MKL_QUANTIZED_CONV_OPS_H_
 #define TENSORFLOW_CORE_KERNELS_MKL_QUANTIZED_CONV_OPS_H_
 
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/tensor.h"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
 #ifdef INTEL_MKL
 
 namespace tensorflow {
 template <class T>
 float MklFloatForOneQuantizedLevel(float range_min, float range_max) {
-  const int64 highest = static_cast<int64>(Eigen::NumTraits<T>::highest());
-  const int64 lowest = static_cast<int64>(Eigen::NumTraits<T>::lowest());
+  int64 highest = static_cast<int64>(Eigen::NumTraits<T>::highest());
+  int64 lowest = static_cast<int64>(Eigen::NumTraits<T>::lowest());
+  if (lowest < -highest) lowest += 1;
+
   const float float_for_one_quantized_level =
       (range_max - range_min) / (highest - lowest);
   return float_for_one_quantized_level;
@@ -48,6 +50,35 @@ void MklQuantizationRangeForMultiplication(float min_a, float max_a,
   *min_c = c_float_for_one_quant_level * c_lowest;
   *max_c = c_float_for_one_quant_level * c_highest;
 }
+
+template <class T1, class T2, class T3>
+void MklQuantizationRangeForMultiplication(float min_a, float max_a,
+                                           const Tensor& min_b_vector,
+                                           const Tensor& max_b_vector,
+                                           Tensor** min_c_vector,
+                                           Tensor** max_c_vector) {
+  CHECK(min_b_vector.NumElements() == (*min_c_vector)->NumElements());
+  CHECK(max_b_vector.NumElements() == (*max_c_vector)->NumElements());
+  size_t n_channel = min_b_vector.NumElements();
+  const int64 c_highest = static_cast<int64>(Eigen::NumTraits<T3>::highest());
+  const int64 c_lowest = static_cast<int64>(Eigen::NumTraits<T3>::lowest());
+  const float* min_b = min_b_vector.flat<float>().data();
+  const float* max_b = max_b_vector.flat<float>().data();
+  float* min_c = (*min_c_vector)->flat<float>().data();
+  float* max_c = (*max_c_vector)->flat<float>().data();
+#pragma omp parallel for
+  for (size_t n = 0; n < n_channel; n++) {
+    float a_float_for_one_quant_level =
+        MklFloatForOneQuantizedLevel<T1>(min_a, max_a);
+    float b_float_for_one_quant_level =
+        MklFloatForOneQuantizedLevel<T2>(min_b[n], max_b[n]);
+    float c_float_for_one_quant_level =
+        a_float_for_one_quant_level * b_float_for_one_quant_level;
+    min_c[n] = c_float_for_one_quant_level * c_lowest;
+    max_c[n] = c_float_for_one_quant_level * c_highest;
+  }
+}
+
 }  // namespace tensorflow
 
 #endif  // INTEL_MKL
diff --git a/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc b/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc
new file mode 100644
index 0000000000..d23a2d320a
--- /dev/null
+++ b/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc
@@ -0,0 +1,110 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// See docs in ../ops/array_ops.cc.
+#ifdef INTEL_MKL
+#define EIGEN_USE_THREADS
+
+#include <math.h>
+#include <limits>
+
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/type_traits.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/kernels/meta_support.h"
+#include "tensorflow/core/kernels/no_op.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/util/mkl_util.h"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+
+namespace tensorflow {
+
+typedef Eigen::ThreadPoolDevice CPUDevice;
+
+class MklRequantizationRangePerChannelOp : public OpKernel {
+ public:
+  explicit MklRequantizationRangePerChannelOp(OpKernelConstruction* ctx)
+      : OpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("clip_value_max", &clip_value_max_));
+  }
+
+  void Compute(OpKernelContext* ctx) override {
+    const Tensor& input = ctx->input(kInputTensorIndex);
+    const Tensor& input_min = ctx->input(kInputMin);
+    const Tensor& input_max = ctx->input(kInputMax);
+
+    size_t depth = input_max.NumElements();
+    OP_REQUIRES(ctx, input_min.dim_size(0) == depth,
+                errors::InvalidArgument("min has incorrect size, expected ",
+                                        depth, " was ", input_min.dim_size(0)));
+    OP_REQUIRES(ctx, input_max.dim_size(0) == depth,
+                errors::InvalidArgument("max has incorrect size, expected ",
+                                        depth, " was ", input_max.dim_size(0)));
+
+    const float* input_min_data = input_min.flat<float>().data();
+    const float* input_max_data = input_max.flat<float>().data();
+    std::vector<float> ranges(depth);
+    bool is_non_negative = true;
+    Eigen::array<int, 2> shuffling({1, 0});
+    auto input_matrix = input.flat_inner_dims<qint32>();
+    auto transposed_input = input_matrix.shuffle(shuffling);
+
+#pragma omp parallel for
+    for (size_t i = 0; i < depth; i++) {
+      Eigen::Tensor<qint32, 0, Eigen::RowMajor> min =
+          transposed_input.chip<0>(i).minimum();
+      Eigen::Tensor<qint32, 0, Eigen::RowMajor> max =
+          transposed_input.chip<0>(i).maximum();
+      int32_t min_per_channel = min();
+      int32_t max_per_channel = max();
+      int32_t abs_max =
+          std::max(std::abs(min_per_channel), std::abs(max_per_channel));
+      float scale =
+          std::max(std::abs(input_min_data[i]), std::abs(input_max_data[i]));
+      ranges[i] = (scale * (float)abs_max / (float)(1L << 31));
+      if (min_per_channel < 0) is_non_negative = false;
+    }
+
+    float out_min_max = std::numeric_limits<float>::min();
+    for (size_t i = 0; i < depth; i++) {
+      if (out_min_max < ranges[i]) out_min_max = ranges[i];
+    }
+    // Fixing max to clip_value_max_ (example 6.0 to support relu6)
+    if (out_min_max > clip_value_max_) out_min_max = clip_value_max_;
+
+    Tensor* output_min = nullptr;
+    Tensor* output_max = nullptr;
+    OP_REQUIRES_OK(ctx, ctx->allocate_output(kOutputMin, {}, &output_min));
+    OP_REQUIRES_OK(ctx, ctx->allocate_output(kOutputMax, {}, &output_max));
+    output_min->flat<float>()(0) = is_non_negative ? 0.0f : out_min_max * -1.0f;
+    output_max->flat<float>()(0) = out_min_max;
+  }
+
+ private:
+  float clip_value_max_ = std::numeric_limits<float>::infinity();
+  const int kInputTensorIndex = 0;
+  const int kInputMin = 1;
+  const int kInputMax = 2;
+  const int kOutputMin = 0;
+  const int kOutputMax = 1;
+};
+
+REGISTER_KERNEL_BUILDER(Name("RequantizationRangePerChannel")
+                            .Device(DEVICE_CPU)
+                            .TypeConstraint<qint32>("T"),
+                        MklRequantizationRangePerChannelOp);
+}  // namespace tensorflow
+#endif  // INTEL_MKL
diff --git a/tensorflow/core/kernels/mkl_requantize_ops_test.cc b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
new file mode 100644
index 0000000000..44ab936471
--- /dev/null
+++ b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
@@ -0,0 +1,297 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/framework/allocator.h"
+#include "tensorflow/core/framework/fake_input.h"
+#include "tensorflow/core/framework/node_def_builder.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_testutil.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/framework/types.pb.h"
+#include "tensorflow/core/kernels/ops_testutil.h"
+#include "tensorflow/core/kernels/ops_util.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/platform/test_benchmark.h"
+
+#include <cmath>
+
+namespace tensorflow {
+
+class MklRequantizatedOpsTest : public OpsTestBase {};
+
+class MklRequantizatedOpsTestHelper : public OpsTestBase {
+ public:
+  void Setup(Tensor &input_tensor_qint32, float &range_weights_ch1,
+             float &range_weights_ch2);
+  void TestBody(){};
+};
+
+void MklRequantizatedOpsTestHelper::Setup(Tensor &input_tensor_qint32,
+                                          float &range_weights_ch1,
+                                          float &range_weights_ch2) {
+  // Step 1: Assumption of inputs
+  // ----------------------------
+  // Assume input Tensor T (NHWC) in FP32 has range [0, 5.0]   size nt*ht*wt*ct
+  // Assume input Filter W (NHWC) with 2 output channels of    size nw*ht**wt*2
+  // logically,   Filter W has 2 channels W1 and W2 each of    size nw*ht**wt*1
+  // Assume input Filter W1(NHWC) in FP32 has range [-2.0, 2.0]size nw*ht**wt*1
+  // Assume input Filter W2(NHWC) in FP32 has range [-3.0, 3.0]size nw*ht**wt*1
+
+  // Step 2: Assumption of Quantizing inputs and weights (per channel)
+  // ------------------------------------------------------------------
+  // When these 2 Tensors, T and W are quantized using a Quantize Op.
+  // When  the input Tensor T (NHWC) is quantized to unsigned int8.
+  // While the input Filter W (NHWC) is qunatized to signed int8.
+  // hence T max value is mapped to ((2^8-1) = 255) while W to ((2^7)-1 = 127))
+
+  // Range of Quantized T  in int8[0   , 255] maps to orig T  in FP32[0   , 5.0]
+  // Range of Quantized W1 in int8[-127, 127] maps to orig W1 in FP32[-2.0, 2.0]
+  // Range of Quantized W2 in int8[-127, 127] maps to orig W2 in FP32[-3.0, 3.0]
+
+  // Hence the resolution of Quantized T will be 5.0/255
+  // Hence the resolution of Quantized W1 will be 2.0/127
+  // Hence the resolution of Quantized W2 will be 3.0/127
+
+  // Step 3: Assumption of quantizedconv on quantized input&weights(per channel)
+  // ---------------------------------------------------------------------------
+  // The input T and weights W1 (or W2) will be convolved (and multipled)
+  // The output Tensor T is in int32 whose range is [-2^31, 2^31]
+  // The Range of the Convolved T*W1 is 2^31 * 5.0/255 * 2.0/127 = 663110.59
+  // So Range of Convolved T*W1 in int32[-2^31, 22^31] that maps to
+  // orig T Range in FP32[0,5.0] * [-2.0, 2.0] is [-663110.59, 663110.59]
+
+  // The Range of the Convolved T*W2 is 2^31 * 5.0/255 * 3.0/127 = 994665.88
+  // So Range of Convolved T*W2 in int32[-2^31, 22^31] that maps to
+  // orig T Range in FP32[0,5.0] * [-3.0, 3.0]  is [-994665.88, 994665.88]
+
+  // Step 4: Assumption output above is fed to Requantization_range_perchannel
+  // --------------------------------------------------------------------------
+  // Here we recalculate the new Range for Convolved T*W so that we
+  // make good use in int8 qunatization from int32 to int8.
+
+  // We assume the above operations are performed and use these values above
+  // as ranges for Requantization_range_perchannel_op.
+  range_weights_ch1 = 663110.59;  // For W1 channel
+  range_weights_ch2 = 994665.88;  // For W2 Channel
+
+  // We Fill the inputs Tensor T qint32 with arbitrary int32 values
+  test::FillValues<qint32>(
+      &input_tensor_qint32,
+      {-1000, -2000,  2000,   4000,   -3000,  -6000,  4000,   8000,
+       5000,  10000,  -6000,  -12000, 7000,   14000,  8000,   16000,
+       9000,  -18000, -10000, -20000, 11000,  22000,  -12000, -24000,
+       13000, 26000,  14000,  28000,  -15000, -30000, 16000,  32000});
+
+  // Step 5: Define and run requantization_range_perchannel
+  // -------------------------------------------------------
+  // See test RequantizationRangePerChannelTest_Basic and/or
+  // test RequantizationRangePerChannelTest_ClipMax
+}
+
+// Following tests the RequantizationRangePerChannel Op wherein the range
+// of the weights is calculated per channel.
+TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_Basic) {
+  // Let us setup the tensor and inputs before we run this op.
+  float clip_max_value = pow(2, 31);
+  float range_weights_ch1 = 0.0;
+  float range_weights_ch2 = 0.0;
+
+  // Create the input tensor
+  const int input_height = 4;
+  const int input_width = 4;
+  const int input_channels = 2;
+
+  // define and input tensor T shape.
+  Tensor input_tensor_qint32(DT_QINT32,
+                             {1, input_height, input_width, input_channels});
+
+  // Explanation and setup prior to this Op. Fill T and populate range values.
+  MklRequantizatedOpsTestHelper helper;
+  helper.Setup(input_tensor_qint32, range_weights_ch1, range_weights_ch2);
+
+  // Step 5: Define and run requantization_range_perchannel
+  // -------------------------------------------------------
+  // Define, Create and initalize the OP in question.
+  TF_ASSERT_OK(NodeDefBuilder("requantization_range_per_channel",
+                              "RequantizationRangePerChannel")
+                   .Input(FakeInput(DT_QINT32))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Attr("T", DataTypeToEnum<qint32>::v())
+                   .Attr("clip_value_max", clip_max_value)
+                   .Finalize(node_def()));
+  TF_ASSERT_OK(InitOp());
+
+  // Add the Input Nodes to the Op.
+  AddInputFromArray<qint32>(input_tensor_qint32.shape(),
+                            input_tensor_qint32.flat<qint32>());
+
+  // Calculate the Min and max from the ranges
+  float ch1_min = -1.0 * range_weights_ch1;
+  float ch1_max = range_weights_ch1;
+  float ch2_min = -1.0 * range_weights_ch2;
+  float ch2_max = range_weights_ch2;
+
+  // Add the Perchannel range Nodes to the Op.
+  AddInputFromArray<float>(TensorShape({input_channels}), {ch1_min, ch2_min});
+  AddInputFromArray<float>(TensorShape({input_channels}), {ch1_max, ch2_max});
+
+  // Run the Kernel
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Step 6: Verify Output and Store values to test Requantize_perchannel
+  // --------------------------------------------------------------------
+
+  // Verify the Expected Outputs
+  const float output_min = GetOutput(0)->flat<float>()(0);
+  const float output_max = GetOutput(1)->flat<float>()(0);
+  EXPECT_NEAR(-14.8217, output_min, 0.002);
+  EXPECT_NEAR(14.8217, output_max, 0.002);
+
+  // output range is made use in RequantizePerChannelTest_Basic
+}
+
+TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_ClipMax) {
+  // Let us setup the tensor and inputs before we run this op.
+  float clip_max_value = 6;  // Can be used as 6 for Relu 6 activations.
+  float range_weights_ch1 = 0.0;
+  float range_weights_ch2 = 0.0;
+
+  // Create the input tensor
+  const int input_height = 4;
+  const int input_width = 4;
+  const int input_channels = 2;
+
+  // define and input tensor T shape.
+  Tensor input_tensor_qint32(DT_QINT32,
+                             {1, input_height, input_width, input_channels});
+
+  // Explanation and setup prior to this Op. Fill T and populate range values.
+  MklRequantizatedOpsTestHelper helper;
+  helper.Setup(input_tensor_qint32, range_weights_ch1, range_weights_ch2);
+
+  // Step 5: Define and run requantization_range_perchannel
+  // -------------------------------------------------------
+  // Define, Create and initalize the OP in question.
+  TF_ASSERT_OK(NodeDefBuilder("requantization_range_per_channel",
+                              "RequantizationRangePerChannel")
+                   .Input(FakeInput(DT_QINT32))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Attr("T", DataTypeToEnum<qint32>::v())
+                   .Attr("clip_value_max", clip_max_value)
+                   .Finalize(node_def()));
+  TF_ASSERT_OK(InitOp());
+
+  // Add the Input Nodes to the Op.
+  AddInputFromArray<qint32>(input_tensor_qint32.shape(),
+                            input_tensor_qint32.flat<qint32>());
+
+  // Calculate the Min and max from the ranges
+  float ch1_min = -1.0 * range_weights_ch1;
+  float ch1_max = range_weights_ch1;
+  float ch2_min = -1.0 * range_weights_ch2;
+  float ch2_max = range_weights_ch2;
+
+  // Add the Perchannel range Nodes to the Op.
+  AddInputFromArray<float>(TensorShape({input_channels}), {ch1_min, ch2_min});
+  AddInputFromArray<float>(TensorShape({input_channels}), {ch1_max, ch2_max});
+
+  // Run the Kernel
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Step 6: Verify Output and Store values to test Requantize_perchannel
+  // --------------------------------------------------------------------
+
+  // Verify the Expected Outputs
+  const float output_min = GetOutput(0)->flat<float>()(0);
+  const float output_max = GetOutput(1)->flat<float>()(0);
+  EXPECT_NEAR(-6.0, output_min, 0.002);  // Values are Max as with clip_value
+  EXPECT_NEAR(6.0, output_max, 0.002);   // Values are Max as with clip_value
+}
+
+TEST_F(MklRequantizatedOpsTest, RequantizePerChannelTest_Basic) {
+  // Let us setup the tensor and inputs before we run this op.
+  float clip_max_value = pow(2, 31);
+  float range_weights_ch1 = 0.0;
+  float range_weights_ch2 = 0.0;
+
+  // Create the input tensor
+  const int input_height = 4;
+  const int input_width = 4;
+  const int input_channels = 2;
+
+  // define an input tensor T shape.
+  Tensor input_tensor_qint32(DT_QINT32,
+                             {1, input_height, input_width, input_channels});
+
+  // Explanation and setup prior to this Op. Fill T and populate range values.
+  MklRequantizatedOpsTestHelper helper;
+  helper.Setup(input_tensor_qint32, range_weights_ch1, range_weights_ch2);
+
+  // Step 7: Define and run requantize_perchannel
+  // --------------------------------------------
+  // The Output of Requantization_range_op_per_channel which calculated the
+  // new ranges of int8 is fed to the requantize per channel op.
+  // Here the Values of Convolved T*W is converted from int32 to int8.
+
+  TF_ASSERT_OK(NodeDefBuilder("requantize_per_channel", "RequantizePerChannel")
+                   .Input(FakeInput(DT_QINT32))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Attr("T", DataTypeToEnum<qint32>::v())
+                   .Attr("out_type", DataTypeToEnum<qint8>::v())
+                   .Finalize(node_def()));
+  TF_ASSERT_OK(InitOp());
+
+  // Add the Input Nodes to the Op.
+  AddInputFromArray<qint32>(input_tensor_qint32.shape(),
+                            input_tensor_qint32.flat<qint32>());
+
+  // Calculate the Min and max from the ranges
+  float ch1_min = -1.0 * range_weights_ch1;
+  float ch1_max = range_weights_ch1;
+  float ch2_min = -1.0 * range_weights_ch2;
+  float ch2_max = range_weights_ch2;
+
+  // Add the Perchannel range Nodes to the Op.
+  AddInputFromArray<float>(TensorShape({input_channels}), {ch1_min, ch2_min});
+  AddInputFromArray<float>(TensorShape({input_channels}), {ch1_max, ch2_max});
+
+  // Calculate the Min and max from Step 6 above
+  // in RequantizationRangePerChannelTest_Basic
+  float range_op_output_min = -14.8217;
+  float range_op_output_max = 14.8217;
+
+  // Add the Requested_min and requested_max stored from Step 6.
+  AddInputFromArray<float>(TensorShape({1}), {range_op_output_min});
+  AddInputFromArray<float>(TensorShape({1}), {range_op_output_max});
+
+  // Run the kernel
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Verify the output with the expected output
+  Tensor output = *GetOutput(0);
+  const float output_min = GetOutput(1)->flat<float>()(0);
+  const float output_max = GetOutput(2)->flat<float>()(0);
+  EXPECT_NEAR(range_op_output_min, output_min, 0.002);
+  EXPECT_NEAR(range_op_output_max, output_max, 0.002);
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc b/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc
new file mode 100644
index 0000000000..f9aa550866
--- /dev/null
+++ b/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc
@@ -0,0 +1,171 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// See docs in ../ops/array_ops.cc.
+
+#define EIGEN_USE_THREADS
+#ifdef INTEL_MKL
+#include <math.h>
+
+#include "mkldnn.hpp"
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/type_traits.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/kernels/meta_support.h"
+#include "tensorflow/core/kernels/no_op.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/util/mkl_util.h"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+
+namespace tensorflow {
+
+typedef Eigen::ThreadPoolDevice CPUDevice;
+
+template <typename Device, typename Toutput>
+class MklRequantizePerChannelOp : public OpKernel {
+ public:
+  explicit MklRequantizePerChannelOp(OpKernelConstruction* ctx)
+      : OpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("out_type", &out_type_));
+    OP_REQUIRES(ctx, out_type_ == DT_QINT8 || out_type_ == DT_QUINT8,
+                errors::InvalidArgument(
+                    "out_type must be qint8 or quint8, but got: " + out_type_));
+  }
+  virtual ~MklRequantizePerChannelOp() {}
+  void Compute(OpKernelContext* ctx) override {
+    try {
+      const Tensor& input = ctx->input(kInputTensorIndex);
+      const Tensor& input_min_vec = ctx->input(kInputMinVec);
+      float* input_min_vec_data = (float*)const_cast<void*>(
+          static_cast<const void*>(input_min_vec.flat<float>().data()));
+      const Tensor& input_max_vec = ctx->input(kInputMaxVec);
+      float* input_max_vec_data = (float*)const_cast<void*>(
+          static_cast<const void*>(input_max_vec.flat<float>().data()));
+
+      const Tensor& input_requested_min = ctx->input(this->kRequestMin);
+      const float input_requested_min_float =
+          input_requested_min.flat<float>()(0);
+      const Tensor& input_requested_max = ctx->input(this->kRequestMax);
+      const float input_requested_max_float =
+          input_requested_max.flat<float>()(0);
+
+      size_t depth = input_min_vec.NumElements();
+      OP_REQUIRES(
+          ctx, input_min_vec.dim_size(0) == depth,
+          errors::InvalidArgument("min has incorrect size, expected ", depth,
+                                  " was ", input_min_vec.dim_size(0)));
+      OP_REQUIRES(
+          ctx, input_max_vec.dim_size(0) == depth,
+          errors::InvalidArgument("max has incorrect size, expected ", depth,
+                                  " was ", input_max_vec.dim_size(0)));
+
+      if (out_type_ == DT_QINT8) CHECK(input_requested_min_float < 0.0f);
+
+      const float factor = (out_type_ == DT_QINT8) ? 127.0f : 255.0f;
+      float requested_min_max = std::max(std::abs(input_requested_min_float),
+                                         std::abs(input_requested_max_float));
+      Tensor* output = nullptr;
+      OP_REQUIRES_OK(ctx, ctx->allocate_output(kOutputTensorIndex,
+                                               input.shape(), &output));
+
+      std::vector<float> scales(depth);
+      for (int i = 0; i < depth; i++) {
+        float min_max_from_vec = std::max(std::abs(input_min_vec_data[i]),
+                                          std::abs(input_max_vec_data[i]));
+        float scale =
+            factor * (min_max_from_vec / requested_min_max / (float)(1L << 31));
+        scales[i] = scale;
+      }
+
+      mkldnn::primitive_attr reorder_attr;
+      reorder_attr.set_output_scales(2, scales);
+
+      memory::dims dims_mkl_order =
+          TFShapeToMklDnnDimsInNCHW(input.shape(), FORMAT_NHWC);
+      memory::desc input_md = memory::desc(dims_mkl_order, MklDnnType<qint32>(),
+                                           memory::format::nhwc);
+      memory::desc output_md =
+          (out_type_ == DT_QINT8)
+              ? memory::desc(dims_mkl_order, MklDnnType<qint8>(),
+                             memory::format::nhwc)
+              : memory::desc(dims_mkl_order, MklDnnType<quint8>(),
+                             memory::format::nhwc);
+
+      memory::primitive_desc input_pd =
+          memory::primitive_desc(input_md, cpu_engine_);
+      memory::primitive_desc output_pd =
+          memory::primitive_desc(output_md, cpu_engine_);
+
+      void* input_buf =
+          static_cast<void*>(const_cast<qint32*>(input.flat<qint32>().data()));
+      void* output_buf;
+      if (out_type_ == DT_QINT8) {
+        output_buf = static_cast<void*>(
+            const_cast<qint8*>(output->flat<qint8>().data()));
+      } else {
+        output_buf = static_cast<void*>(
+            const_cast<quint8*>(output->flat<quint8>().data()));
+      }
+
+      std::unique_ptr<memory> input_mem_prim_(new memory(input_pd, input_buf));
+      std::unique_ptr<memory> output_mem_prim_(
+          new memory(output_pd, output_buf));
+
+      mkldnn::reorder::primitive_desc reorder_pd =
+          mkldnn::reorder::primitive_desc(input_pd, output_pd, reorder_attr);
+      std::vector<mkldnn::primitive> net;
+      net.push_back(
+          mkldnn::reorder(reorder_pd, *input_mem_prim_, *output_mem_prim_));
+      stream(stream::kind::eager).submit(net).wait();
+
+      Tensor* output_min = nullptr;
+      Tensor* output_max = nullptr;
+      OP_REQUIRES_OK(ctx, ctx->allocate_output(kOutputMin, {}, &output_min));
+      OP_REQUIRES_OK(ctx, ctx->allocate_output(kOutputMax, {}, &output_max));
+
+      output_min->flat<float>()(0) = input_requested_min_float;
+      output_max->flat<float>()(0) = input_requested_max_float;
+
+    } catch (mkldnn::error& e) {
+      string error_msg = "Status: " + std::to_string(e.status) + ", message: " +
+                         std::string(e.message) + ", in file " +
+                         std::string(__FILE__) + ":" + std::to_string(__LINE__);
+      OP_REQUIRES_OK(
+          ctx, errors::Aborted("Operation received an exception:", error_msg));
+    }
+  }
+
+ private:
+  const int kInputTensorIndex = 0;
+  const int kInputMinVec = 1;
+  const int kInputMaxVec = 2;
+  const int kRequestMin = 3;
+  const int kRequestMax = 4;
+  const int kOutputTensorIndex = 0;
+  const int kOutputMin = 1;
+  const int kOutputMax = 2;
+  DataType out_type_;
+  engine cpu_engine_ = engine(engine::cpu, 0);
+};
+
+REGISTER_KERNEL_BUILDER(Name("RequantizePerChannel")
+                            .Device(DEVICE_CPU)
+                            .TypeConstraint<qint32>("T")
+                            .TypeConstraint<qint8>("out_type"),
+                        MklRequantizePerChannelOp<CPUDevice, qint8>);
+
+}  // namespace tensorflow
+#endif  // INTEL_MKL
diff --git a/tensorflow/core/kernels/requantize.cc b/tensorflow/core/kernels/requantize.cc
index dce6f1a185..ab057d5a67 100644
--- a/tensorflow/core/kernels/requantize.cc
+++ b/tensorflow/core/kernels/requantize.cc
@@ -19,7 +19,6 @@ limitations under the License.
 
 #include <math.h>
 
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/type_traits.h"
@@ -27,6 +26,7 @@ limitations under the License.
 #include "tensorflow/core/kernels/meta_support.h"
 #include "tensorflow/core/kernels/quantization_utils.h"
 #include "tensorflow/core/lib/core/errors.h"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
 namespace tensorflow {
 
@@ -100,4 +100,10 @@ REGISTER_KERNEL_BUILDER(Name("Requantize")
                             .TypeConstraint<quint8>("out_type"),
                         RequantizeOp<qint32, quint8>);
 
+REGISTER_KERNEL_BUILDER(Name("Requantize")
+                            .Device(DEVICE_CPU)
+                            .TypeConstraint<qint32>("Tinput")
+                            .TypeConstraint<qint8>("out_type"),
+                        RequantizeOp<qint32, qint8>);
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc
index eb9cbd3225..52243f2fcf 100644
--- a/tensorflow/core/ops/math_ops.cc
+++ b/tensorflow/core/ops/math_ops.cc
@@ -1247,12 +1247,12 @@ Status RangeSize(const Tensor* start_t, const Tensor* limit_t,
   T limit = limit_t->scalar<T>()();
   T delta = delta_t->scalar<T>()();
   if (start > limit && delta > 0) {
-    return errors::InvalidArgument(
-        "Requires start <= limit when delta > 0: ", start, "/", limit);
+    return errors::InvalidArgument("Requires start <= limit when delta > 0: ",
+                                   start, "/", limit);
   }
   if (start < limit && delta < 0) {
-    return errors::InvalidArgument(
-        "Requires start >= limit when delta < 0: ", start, "/", limit);
+    return errors::InvalidArgument("Requires start >= limit when delta < 0: ",
+                                   start, "/", limit);
   }
   if (delta == 0) {
     return errors::InvalidArgument("Requires delta != 0");
@@ -1686,6 +1686,45 @@ Add two input tensors element wise using mkl kernel sum.
 inputs: Must all be the same size and shape.
 )doc");
 
+REGISTER_OP("RequantizePerChannel")
+    .Input("input: T")
+    .Input("input_min: float")
+    .Input("input_max: float")
+    .Input("requested_output_min: float")
+    .Input("requested_output_max: float")
+    .Output("output: out_type")
+    .Output("output_min: float")
+    .Output("output_max: float")
+    .Attr("T: quantizedtype = DT_QINT32")
+    .Attr("out_type: quantizedtype = DT_QUINT8")
+    .SetShapeFn([](InferenceContext* c) {
+      TF_RETURN_IF_ERROR(shape_inference::UnchangedShape(c));
+      ShapeHandle unused;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &unused));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 1, &unused));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 0, &unused));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(4), 0, &unused));
+      c->set_output(1, c->Scalar());
+      c->set_output(2, c->Scalar());
+      return Status::OK();
+    });
+REGISTER_OP("RequantizationRangePerChannel")
+    .Input("input: T")
+    .Input("input_min: float")
+    .Input("input_max: float")
+    .Output("output_min: float")
+    .Output("output_max: float")
+    .Attr("T: quantizedtype = DT_QINT32")
+    .Attr("clip_value_max: float")
+    .SetShapeFn([](InferenceContext* c) {
+      ShapeHandle unused;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &unused));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 1, &unused));
+      c->set_output(0, c->Scalar());
+      c->set_output(1, c->Scalar());
+      return Status::OK();
+    });
+
 #endif  // INTEL_MKL
 
 REGISTER_OP("NextAfter")
-- 
GitLab


From a6924e6affd935f537cdaf8977094df0e15a7957 Mon Sep 17 00:00:00 2001
From: Mingsheng Hong <hongm@google.com>
Date: Fri, 18 Jan 2019 17:00:33 -0800
Subject: [PATCH 1012/2345] Added tracing support to the eager C++ layer, where
 we can create a graph by symbolically executing a sequence of eager ops. This
 involves the following changes:

1. Extended the eager C++ layer with symbolic tensor support, where a
tensorflow::TensorHandle represents a graph node, whose execution yields the
"concrete" tensor value. In contrast, the existing tensor handles are conrete
tensors.

Symbolic tensors can be created from a graph node (represented as TF_Output) via
TFE_NewTensorHandleFromTFOutput(). The associated graph node can be retrieved
from a symbolic tensor via TFE_GetTFOutputFromTensorHandle().

2. Added another experimental C API TFE_AddEagerOpToGraph() for symbolic
execution of a TFE_Op, which converts the op to a graph node, and the graph is
associated with a newly added TFE_TraceContext object to be passed into the C
API call.

3. If an eager op OP takes any concrete tensors as input,
TFE_AddEagerOpToGraph() will create symbolic tensors for them, to serve as input
to the converted graph node from OP. When executing that graph, those concrete
tensors will be fed as input.

Here's an example tracing use case illustrated by Swift code:
let x = Tensor<Float>(1.0)
let y = x + x

The first statement is executed on the host, yielding a concrete tensor value
for x.

The second statement is first symbolically executed to create a
graph (containing the "Add" node). Next, the graph is executed to compute y.

During this symbolic execution, TFE_AddEagerOpToGraph() is called on the "Add"
eager op for x + x, to insert an "Add" graph node. The node takes as input a
symbolic tensor (a PlaceHolder graph node) corresponding to x. When executing
the graph, the concrete tensor of x is then fed as the input.

TFE_AddEagerOpToGraph() does not yet support TF op attributes.

An example call-site code of this feature is Swift for TensorFlow at
https://github.com/apple/swift/pull/21589 (where some of the above C APIs are
subject to name changes).

PiperOrigin-RevId: 230018312
---
 tensorflow/c/BUILD                            |   2 +
 tensorflow/c/c_api_experimental.cc            | 165 ++++++++++++++++++
 tensorflow/c/c_api_experimental.h             |  48 +++++
 tensorflow/c/c_api_experimental_test.cc       |  51 +++++-
 tensorflow/c/c_api_internal.h                 |   2 +
 tensorflow/c/eager/c_api.cc                   |   2 +
 tensorflow/c/eager/c_api_internal.h           |  26 +++
 tensorflow/core/common_runtime/eager/BUILD    |   1 +
 .../common_runtime/eager/tensor_handle.cc     |  19 ++
 .../core/common_runtime/eager/tensor_handle.h |  21 ++-
 10 files changed, 335 insertions(+), 2 deletions(-)

diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index 4e2fe34d28..a48778d612 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -129,6 +129,7 @@ tf_cuda_library(
         "//tensorflow/core:lib_platform",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/common_runtime/eager:attr_builder",
+        "@com_google_absl//absl/strings",
     ],
 )
 
@@ -318,6 +319,7 @@ tf_cc_test(
     deps = [
         ":c_api",
         ":c_api_experimental",
+        ":c_api_internal",
         ":c_test_util",
         "//tensorflow/c/eager:c_api",
         "//tensorflow/c/eager:c_api_test_util",
diff --git a/tensorflow/c/c_api_experimental.cc b/tensorflow/c/c_api_experimental.cc
index 92f7da7de0..6cc74cfb32 100644
--- a/tensorflow/c/c_api_experimental.cc
+++ b/tensorflow/c/c_api_experimental.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/c/c_api_experimental.h"
 
+#include "absl/strings/substitute.h"
 #include "tensorflow/c/c_api.h"
 #include "tensorflow/c/c_api_internal.h"
 #include "tensorflow/c/eager/c_api.h"
@@ -8745,6 +8746,12 @@ static void CheckOk(TF_Status* status) {
 
 void TFE_TensorHandlePrintDebugString(TFE_TensorHandle* handle) {
   auto* status = TF_NewStatus();
+  if (!TFE_TensorHandleIsConcrete(handle)) {
+    VLOG(1) << "Symbolic tensor: " << handle;
+    TF_DeleteStatus(status);
+    return;
+  }
+
   TF_Tensor* t = TFE_TensorHandleResolve(handle, status);
   CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
 
@@ -8954,3 +8961,161 @@ TF_CAPI_EXPORT extern void TFE_EnableCollectiveOps(TFE_Context* ctx,
   }
   status->status = EnableCollectiveOps(server_def, ctx);
 }
+
+std::string tensorflow::getTF_OutputDebugString(TF_Output node) {
+  return absl::Substitute("TF_Output($0, $1)", node.oper, node.index);
+}
+
+using tensorflow::getTF_OutputDebugString;
+
+TFE_TensorHandle* TFE_NewTensorHandleFromTFOutput(TF_Output t,
+                                                  TF_DataType dtype) {
+  auto ret = new TFE_TensorHandle(t, dtype);
+  VLOG(1) << "Storing TFOutput " << getTF_OutputDebugString(t)
+          << " into tensor handle " << ret << " with internal handle "
+          << ret->handle;
+  return ret;
+}
+
+unsigned char TFE_TensorHandleIsConcrete(TFE_TensorHandle* handle) {
+  assert(handle->handle != nullptr);
+  return handle->handle->getSymbolicTensor() == nullptr;
+}
+
+TF_Output TFE_GetTFOutputFromTensorHandle(TFE_TensorHandle* handle,
+                                          TF_Status* status) {
+  if (TFE_TensorHandleIsConcrete(handle)) {
+    status->status =
+        tensorflow::errors::Internal("Not a symbolic tensor: ", handle);
+    return TF_Output{nullptr, -1};
+  }
+
+  auto* sym_tensor = handle->handle->getSymbolicTensor();
+  CHECK(sym_tensor != nullptr);
+  auto ret = TF_Output{sym_tensor->oper, sym_tensor->index};
+  VLOG(1) << "Retrieving " << getTF_OutputDebugString(ret)
+          << " from tensor handle " << handle;
+  CHECK_GE(sym_tensor->index, 0);
+  return ret;
+}
+
+TFE_TraceContext* TFE_NewTraceContext(TF_Graph* graph) {
+  return new TFE_TraceContext(graph);
+}
+
+void TFE_DeleteTraceContext(TFE_TraceContext* trace_ctx) { delete trace_ctx; }
+
+// If `handle` is already symbolic, return it. Otherwise map it to a new
+// symbolic tensor (a PlaceHolder op) and return that.
+static TF_Output getOrCreateSymbolicTensor(TFE_TraceContext* trace_ctx,
+                                           tensorflow::TensorHandle* handle,
+                                           TF_Status* status) {
+  VLOG(1) << "Getting symbolic tensor for input tensor handle " << handle
+          << ": " << handle->DebugString();
+
+  auto* sym_tensor = handle->getSymbolicTensor();
+  if (sym_tensor != nullptr) {
+    auto ret = TF_Output{sym_tensor->oper, sym_tensor->index};
+    VLOG(1) << "This handle is a symbolic tensor " << sym_tensor << ": "
+            << getTF_OutputDebugString(ret);
+    return ret;
+  }
+
+  auto find_it = trace_ctx->input_tensor_map.find(handle);
+  if (find_it != trace_ctx->input_tensor_map.end()) {
+    VLOG(1) << "There exists a map entry from this concrete tensor to: "
+            << getTF_OutputDebugString(find_it->second);
+    return find_it->second;
+  }
+
+  auto node_name = tensorflow::strings::StrCat("additional_input_",
+                                               trace_ctx->node_counter++);
+  VLOG(1) << "Adding a place holder node named " << node_name;
+  auto* desc =
+      TF_NewOperation(trace_ctx->graph, "Placeholder", node_name.c_str());
+  TF_SetAttrType(desc, "dtype",
+                 static_cast<TF_DataType>(handle->dtype) /*TF_FLOAT*/);
+  auto* result = TF_FinishOperation(desc, status);
+  if (!status->status.ok()) {
+    return TF_Output{nullptr, -1};
+  }
+
+  auto ret = TF_Output{result, 0};
+  VLOG(1) << "Creating a new map entry to map to: "
+          << getTF_OutputDebugString(ret);
+  trace_ctx->input_tensor_map[handle] = ret;
+  // `handle` could be destroyed before it's read from `input_tensor_map` (say
+  // during a subsequent TFE_FinalizeInputTensorsFromTraceContext() call), so we
+  // increment its ref count to extend its life span to that of `trace_ctx`.
+  handle->Ref();
+  VLOG(1) << "Ref count for handle " << handle
+          << " is 1?: " << handle->RefCountIsOne();
+  return ret;
+}
+
+void TFE_AddEagerOpToGraph(TFE_Op* op, TFE_TraceContext* trace_ctx,
+                           TFE_TensorHandle** retvals, int* num_retvals,
+                           TF_Status* status) {
+  VLOG(1) << "Calling TFE_AddEagerOpToGraph() with op " << op << ": "
+          << op->operation.DebugString();
+
+  const auto& op_type = op->operation.Name();
+  auto op_name =
+      tensorflow::strings::StrCat(op_type, "_", trace_ctx->node_counter++);
+  auto* desc =
+      TF_NewOperation(trace_ctx->graph, op_type.c_str(), op_name.c_str());
+  for (auto* input : op->operation.Inputs()) {
+    auto symbolic_input = getOrCreateSymbolicTensor(trace_ctx, input, status);
+    if (!status->status.ok()) return;
+    TF_AddInput(desc, symbolic_input);
+  }
+
+  VLOG(1) << "Adding attrs.";
+  // TODO(hongm): add attrs
+
+  auto* graph_op = TF_FinishOperation(desc, status);
+  if (!status->status.ok()) return;
+
+  VLOG(1) << "Op finalized; setting return tensors.";
+  *num_retvals = TF_OperationNumOutputs(graph_op);
+  VLOG(1) << "This op has " << *num_retvals << " outputs.";
+  for (int i = 0; i < *num_retvals; ++i) {
+    auto output = TF_Output{graph_op, i};
+    auto dtype = TF_OperationOutputType(output);
+    retvals[i] = TFE_NewTensorHandleFromTFOutput(output, dtype);
+  }
+}
+
+int TFE_FinalizeInputTensorsFromTraceContext(TFE_TraceContext* trace_ctx) {
+  if (trace_ctx->input_tensors == nullptr) {
+    trace_ctx->input_tensors =
+        new std::vector<std::pair<tensorflow::TensorHandle*, TF_Output>>();
+    trace_ctx->input_tensors->reserve(trace_ctx->input_tensor_map.size());
+
+    for (auto input : trace_ctx->input_tensor_map) {
+      trace_ctx->input_tensors->emplace_back(input.first, input.second);
+    }
+  }
+  return trace_ctx->input_tensor_map.size();
+}
+
+TF_Output TFE_GetInputGraphNodeFromTraceContext(TFE_TraceContext* trace_ctx,
+                                                unsigned int idx) {
+  CHECK(trace_ctx->input_tensors != nullptr);
+  CHECK(trace_ctx->input_tensors->size() > idx);
+  return trace_ctx->input_tensors->at(idx).second;
+}
+
+TFE_TensorHandle* TFE_ConsumeInputConcreteTensorFromTraceContext(
+    TFE_TraceContext* trace_ctx, unsigned int idx) {
+  CHECK(trace_ctx->input_tensors != nullptr);
+  CHECK(trace_ctx->input_tensors->size() > idx);
+  auto* handle = trace_ctx->input_tensors->at(idx).first;
+  VLOG(1) << "Ref count for internal handle " << handle
+          << " is 1?: " << handle->RefCountIsOne();
+  handle->Ref();
+  auto* ret = new TFE_TensorHandle(handle);
+  VLOG(1) << "Returning a new tensor handle " << ret << ": "
+          << handle->DebugString();
+  return ret;
+}
diff --git a/tensorflow/c/c_api_experimental.h b/tensorflow/c/c_api_experimental.h
index e8da4099b8..48ea0ec1ed 100644
--- a/tensorflow/c/c_api_experimental.h
+++ b/tensorflow/c/c_api_experimental.h
@@ -267,6 +267,54 @@ TF_CAPI_EXPORT extern void TFE_EnableCollectiveOps(TFE_Context* ctx,
                                                    size_t proto_len,
                                                    TF_Status* status);
 
+// Create a symbolic tensor from the input graph node.
+TF_CAPI_EXPORT extern TFE_TensorHandle* TFE_NewTensorHandleFromTFOutput(
+    TF_Output t, TF_DataType data_type);
+
+// Returns 0 if the input tensor handle represents a symbolic tensor (i.e., a
+// graph node). Otherwise returns non-0.
+TF_CAPI_EXPORT extern unsigned char TFE_TensorHandleIsConcrete(
+    TFE_TensorHandle* handle);
+
+// If `handle` is a symbolic tensor, return the corresponding graph node
+// represented by TF_Output. Otherwise, return an error status.
+TF_CAPI_EXPORT extern TF_Output TFE_GetTFOutputFromTensorHandle(
+    TFE_TensorHandle* handle, TF_Status* status);
+
+typedef struct TFE_TraceContext TFE_TraceContext;
+
+// A trace context contains a trace graph, to which TFE_AddEagerOpToGraph()
+// calls add graph nodes as a way to symbolically execute the eager ops.
+//
+// It also contains a hash map from concrete input tensors to symbolic
+// tensors. That map will be used to create input tensors to the trace graph.
+TF_CAPI_EXPORT extern TFE_TraceContext* TFE_NewTraceContext(TF_Graph* graph);
+
+TF_CAPI_EXPORT extern void TFE_DeleteTraceContext(TFE_TraceContext* trace_ctx);
+
+// Symbolically executes `op`, by adding a corresponding node to the graph
+// associated with `trace_ctx`. This graph node outputs a set of symbolic
+// tensors in `retvals` and `num_retvals`.
+TF_CAPI_EXPORT extern void TFE_AddEagerOpToGraph(TFE_Op* op,
+                                                 TFE_TraceContext* trace_ctx,
+                                                 TFE_TensorHandle** retvals,
+                                                 int* num_retvals,
+                                                 TF_Status* status);
+
+// Finalizes the trace graph and its inputs, and returns the number of inputs.
+// After this call, the next two APIs can be called to iterate over the input
+// tensors.
+TF_CAPI_EXPORT extern int TFE_FinalizeInputTensorsFromTraceContext(
+    TFE_TraceContext* trace_ctx);
+
+TF_CAPI_EXPORT extern TF_Output TFE_GetInputGraphNodeFromTraceContext(
+    TFE_TraceContext* trace_ctx, unsigned int idx);
+
+// Each input tensor should be consumed at most once.
+TF_CAPI_EXPORT extern TFE_TensorHandle*
+TFE_ConsumeInputConcreteTensorFromTraceContext(TFE_TraceContext* trace_ctx,
+                                               unsigned int idx);
+
 #ifdef __cplusplus
 } /* end extern "C" */
 #endif
diff --git a/tensorflow/c/c_api_experimental_test.cc b/tensorflow/c/c_api_experimental_test.cc
index 65a4e10b26..4cfcf2ef3b 100644
--- a/tensorflow/c/c_api_experimental_test.cc
+++ b/tensorflow/c/c_api_experimental_test.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/c/c_api_experimental.h"
+#include "tensorflow/c/c_api_internal.h"
 #include "tensorflow/c/c_test_util.h"
 #include "tensorflow/c/eager/c_api.h"
 #include "tensorflow/c/eager/c_api_test_util.h"
@@ -296,7 +297,29 @@ TEST(CAPI_EXPERIMENTAL, TFE_ExecuteOpInNewThreadTest_Blocking) {
   TF_DeleteStatus(status);
 }
 
-TEST(CAPI_EXPERIMENTAL, DebugPrint) {
+TEST(CAPI_EXPERIMENTAL, SymbolicTensor) {
+  TF_Status* status = TF_NewStatus();
+  auto node = TF_Output{nullptr, 1};
+  auto* sym_handle = TFE_NewTensorHandleFromTFOutput(node, TF_FLOAT);
+  TFE_TensorHandlePrintDebugString(sym_handle);
+  CHECK_EQ(TFE_TensorHandleDataType(sym_handle), TF_FLOAT);
+  ASSERT_FALSE(TFE_TensorHandleIsConcrete(sym_handle));
+  auto same_node = TFE_GetTFOutputFromTensorHandle(sym_handle, status);
+  CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  ASSERT_EQ(same_node.oper, node.oper);
+  ASSERT_EQ(same_node.index, node.index);
+  TFE_DeleteTensorHandle(sym_handle);
+
+  TFE_TensorHandle* m = TestMatrixTensorHandle();
+  ASSERT_TRUE(TFE_TensorHandleIsConcrete(m));
+  (void)TFE_GetTFOutputFromTensorHandle(m, status);
+  CHECK_EQ(TF_INTERNAL, TF_GetCode(status)) << TF_Message(status);
+  TFE_DeleteTensorHandle(m);
+
+  TF_DeleteStatus(status);
+}
+
+TEST(CAPI_EXPERIMENTAL, DebugPrintAndSymbolicExecution) {
   TF_Status* status = TF_NewStatus();
   TFE_ContextOptions* opts = TFE_NewContextOptions();
   CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
@@ -310,6 +333,32 @@ TEST(CAPI_EXPERIMENTAL, DebugPrint) {
   CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
   TFE_OpPrintDebugString(op);
 
+  auto* graph = TF_NewGraph();
+  auto* trace_ctx = TFE_NewTraceContext(graph);
+  TFE_TensorHandle* retvals[5];
+  int num_retvals = 5;
+  // Symbolically execute this op, which adds a graph node to `trace_ctx`.
+  TFE_AddEagerOpToGraph(op, trace_ctx, retvals, &num_retvals, status);
+  CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+
+  int num_inputs = TFE_FinalizeInputTensorsFromTraceContext(trace_ctx);
+  CHECK_EQ(num_inputs, 1);
+  auto input_sym_tensor = TFE_GetInputGraphNodeFromTraceContext(trace_ctx,
+                                                                /*idx*/ 0);
+
+  LOG(INFO) << tensorflow::getTF_OutputDebugString(input_sym_tensor);
+  auto handle = TFE_ConsumeInputConcreteTensorFromTraceContext(trace_ctx,
+                                                               /*idx*/ 0);
+  TFE_TensorHandlePrintDebugString(handle);
+  TFE_DeleteTensorHandle(handle);
+
+  CHECK_EQ(num_retvals, 1);
+  CHECK_EQ(TFE_TensorHandleDataType(retvals[0]), TF_FLOAT);
+  TFE_DeleteTensorHandle(retvals[0]);
+
+  TFE_DeleteTraceContext(trace_ctx);
+  TF_DeleteGraph(graph);
+
   TFE_DeleteTensorHandle(m);
   TFE_DeleteOp(op);
   TFE_DeleteContext(ctx);
diff --git a/tensorflow/c/c_api_internal.h b/tensorflow/c/c_api_internal.h
index 5ba26d3c58..73283d7756 100644
--- a/tensorflow/c/c_api_internal.h
+++ b/tensorflow/c/c_api_internal.h
@@ -228,6 +228,8 @@ void RecordMutation(TF_Graph* graph, const TF_Operation& op,
 bool ExtendSessionGraphHelper(TF_Session* session, TF_Status* status)
     LOCKS_EXCLUDED(session->graph->mu, session->mu);
 
+std::string getTF_OutputDebugString(TF_Output node);
+
 }  // end namespace tensorflow
 
 #endif  // TENSORFLOW_C_C_API_INTERNAL_H_
diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index a92ba1f0f3..af13f487af 100755
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -356,6 +356,8 @@ TFE_TensorHandle* TFE_NewTensorHandle(TF_Tensor* t, TF_Status* status) {
 
 void TFE_DeleteTensorHandle(TFE_TensorHandle* h) {
   if (h == nullptr) return;
+  VLOG(1) << "Deleting tensor handle " << h << " with internal handle "
+          << h->handle;
   if (h->handle) {
     h->handle->Unref();
   }
diff --git a/tensorflow/c/eager/c_api_internal.h b/tensorflow/c/eager/c_api_internal.h
index d330aa4290..3b9e681194 100644
--- a/tensorflow/c/eager/c_api_internal.h
+++ b/tensorflow/c/eager/c_api_internal.h
@@ -83,6 +83,12 @@ struct TFE_TensorHandle {
   TFE_TensorHandle(tensorflow::TensorHandle* handle) : handle(handle) {}
 
   tensorflow::TensorHandle* handle;
+
+  // Create a symbolic tensor.
+  TFE_TensorHandle(TF_Output t, TF_DataType dtype)
+      : handle(new tensorflow::TensorHandle(
+            tensorflow::OutputGraphNode{t.oper, t.index},
+            static_cast<tensorflow::DataType>(dtype))) {}
 };
 
 struct TFE_TensorDebugInfo {
@@ -115,4 +121,24 @@ void SetOpAttrValueScalar(TFE_Context* ctx, TFE_Op* op,
                           const char* attr_name, TF_Status* status);
 }  // namespace tensorflow
 
+struct TFE_TraceContext {
+  TF_Graph* const graph;
+
+  unsigned int node_counter = 0;
+  // Each tensor handle will have its ref count incremented when it's added as a
+  // map key, and decremented when this object is destroyed.
+  std::map<tensorflow::TensorHandle*, TF_Output> input_tensor_map;
+  std::vector<std::pair<tensorflow::TensorHandle*, TF_Output>>* input_tensors =
+      nullptr;
+
+  TFE_TraceContext(TF_Graph* graph) : graph(graph) {}
+
+  ~TFE_TraceContext() {
+    delete input_tensors;
+    for (auto input : input_tensor_map) {
+      input.first->Unref();
+    }
+  }
+};
+
 #endif  // TENSORFLOW_C_EAGER_C_API_INTERNAL_H_
diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD
index 06cbe8049f..0a16f80c8f 100644
--- a/tensorflow/core/common_runtime/eager/BUILD
+++ b/tensorflow/core/common_runtime/eager/BUILD
@@ -106,6 +106,7 @@ tf_cuda_library(
             "//tensorflow/core:android_tensorflow_lib_lite",
         ],
         "//conditions:default": [
+            "@com_google_absl//absl/strings",
             "//tensorflow/core:core_cpu_lib",
             "//tensorflow/core:framework",
             "//tensorflow/core:framework_internal",
diff --git a/tensorflow/core/common_runtime/eager/tensor_handle.cc b/tensorflow/core/common_runtime/eager/tensor_handle.cc
index 5e5936f1cf..e44a97b265 100644
--- a/tensorflow/core/common_runtime/eager/tensor_handle.cc
+++ b/tensorflow/core/common_runtime/eager/tensor_handle.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include <string>
 #include <vector>
 
+#include "absl/strings/substitute.h"
 #include "tensorflow/core/common_runtime/copy_tensor.h"
 #include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
@@ -99,6 +100,19 @@ TensorHandle::TensorHandle(int64 op_id, int32 output_num,
                               : resource_device_ == nullptr);
 }
 
+TensorHandle::TensorHandle(OutputGraphNode symbolic_tensor, DataType dtype)
+    : dtype(dtype),
+      node_id_(0),
+      device_(nullptr),
+      op_device_(nullptr),
+      resource_device_(nullptr),
+      remote_op_id_(-1),
+      remote_output_num_(-1),
+      remote_shape_node_id_(-1),
+      ctx_(nullptr),
+      is_ready_(true),
+      symbolic_tensor(new OutputGraphNode(symbolic_tensor)) {}
+
 bool TensorHandle::IsReady() {
   if (node_id_ == 0) return true;
   mutex_lock l(ctx_mutex_);
@@ -315,6 +329,11 @@ Device* GetResourceDevice(const Tensor& t, EagerContext* ctx) {
 string TensorHandle::DebugString() const {
   VLOG(1) << "Calling TensorHandle::DebugString() on " << this;
 
+  if (symbolic_tensor) {
+    return absl::Substitute("TF_Output($0, $1)", symbolic_tensor->oper,
+                            symbolic_tensor->index);
+  }
+
   string out;
   strings::StrAppend(&out, "Device: ", device_ ? device_->DebugString() : "[]");
   // Consider supporting non-CPU tensors (when device_ is non-NULL) if needed.
diff --git a/tensorflow/core/common_runtime/eager/tensor_handle.h b/tensorflow/core/common_runtime/eager/tensor_handle.h
index 0ecd9e6ddc..e4ccb11dba 100644
--- a/tensorflow/core/common_runtime/eager/tensor_handle.h
+++ b/tensorflow/core/common_runtime/eager/tensor_handle.h
@@ -41,8 +41,17 @@ limitations under the License.
 #include "tensorflow/core/public/session_options.h"
 #include "tensorflow/core/public/version.h"
 
+struct TF_Operation;
+
 namespace tensorflow {
 
+// This struct is isomorphic to TF_Output, but we cannot use the latter here due
+// to layering concerns (TF_Output is defined at the C API layer).
+struct OutputGraphNode {
+  TF_Operation* oper;
+  int index;  // The index of the output within oper.
+};
+
 // Associates a Tensor and a Device, used in the eager runtime. Internal version
 // of the TFE_TensorHandle struct and the python EagerTensor class
 // (unrelated to python TensorHandle).
@@ -58,8 +67,11 @@ class TensorHandle : public core::RefCounted {
                DataType dtype, std::function<void()> call_on_destroy, Device* d,
                Device* op_device, Device* resource_device, EagerContext* ctx);
 
+  // Symbolic tensor constructor.
+  TensorHandle(OutputGraphNode symbolic_tensor, DataType dtype);
+
   ~TensorHandle() override {
-    VLOG(1) << "Deleting TensorHandle " << this;
+    VLOG(1) << "Deleting internal TensorHandle " << this;
     if (call_on_destroy_) {
       call_on_destroy_();
     }
@@ -115,6 +127,8 @@ class TensorHandle : public core::RefCounted {
 
   bool IsRemote();
 
+  OutputGraphNode* getSymbolicTensor() const { return symbolic_tensor.get(); }
+
   string DebugString() const;
 
  private:
@@ -168,6 +182,11 @@ class TensorHandle : public core::RefCounted {
   // `ctx` object is not owned and should outlive this handle.
   EagerContext* ctx_ GUARDED_BY(ctx_mutex_);
   bool is_ready_ GUARDED_BY(ctx_mutex_);
+
+  // When non-NULL, this tensor handle instance represents a symbolic tensor
+  // (corresponding to a graph node), whose concrete value is to be produced by
+  // executing that graph node.
+  std::unique_ptr<OutputGraphNode> symbolic_tensor;
 };
 
 // If tensor's dtype is DT_RESOURCE, returns the device backing the resource.
-- 
GitLab


From 502ca3eed27ea4b096a707dd88b56d339482a4f6 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Fri, 18 Jan 2019 17:35:42 -0800
Subject: [PATCH 1013/2345] Fix a typo in op_hint.py comment

PiperOrigin-RevId: 230022859
---
 tensorflow/lite/python/op_hint.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tensorflow/lite/python/op_hint.py b/tensorflow/lite/python/op_hint.py
index 6ec050171f..8df37c111b 100644
--- a/tensorflow/lite/python/op_hint.py
+++ b/tensorflow/lite/python/op_hint.py
@@ -36,9 +36,7 @@ Example:
   session = tf.Session()
 
   graphdef_to_convert = tf.lite.convert_op_hints_to_stubs(session)
-  tflite_graph = tf.lite.toco_convert(graphdef_to_convert,
-                                              [image], [output])
-                                              [image], [output])
+  tflite_graph = tf.lite.toco_convert(graphdef_to_convert, [image], [output])
   with open("/tmp/graph.fb", "wb") as fp:
     fp.write(tflite_graph)
 
-- 
GitLab


From 698d685ca1217dbb83b16599c44f65c0e937beba Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Fri, 18 Jan 2019 17:47:55 -0800
Subject: [PATCH 1014/2345] Added table of contents and getting started guide
 to TFL micro documentation

PiperOrigin-RevId: 230023991
---
 tensorflow/lite/experimental/micro/README.md | 290 +++++++++++--------
 1 file changed, 175 insertions(+), 115 deletions(-)

diff --git a/tensorflow/lite/experimental/micro/README.md b/tensorflow/lite/experimental/micro/README.md
index 88f3cd71f5..e85b9660e3 100644
--- a/tensorflow/lite/experimental/micro/README.md
+++ b/tensorflow/lite/experimental/micro/README.md
@@ -7,118 +7,66 @@ so it's designed to be portable even to 'bare metal' systems. The core runtime
 fits in 16KB on a Cortex M3, and with enough operators to run a speech keyword
 detection model, takes up a total of 22KB.
 
-The design goals are for the framework to be:
-
--   **Readable**: We want embedded software engineers to be able to understand
-    what's required to run ML inference without having to study research papers.
-    We've tried to keep the code base small, modular, and have reference
-    implementations of all operations to help with this.
-
--   **Easy to modify**: We know that there are a lot of different platforms and
-    requirements in the embedded world, and we don't expect to cover all of them
-    in one framework. Instead, we're hoping that it can be a good starting point
-    for developers to build on top of to meet their own needs. For example, we
-    tried to make it easy to replace the implementations of key computational
-    operators that are often crucial for performance, without having to touch
-    the data flow and other runtime code. We want it to make more sense to use
-    our workflow to handle things like model import and less-important
-    operations, and customize the parts that matter, rather than having to
-    reimplement everything in your own engine.
-
--   **Well-tested**: If you're modifying code, you need to know if your changes
-    are correct. Having an easy way to test lets you develop much faster. To
-    help there, we've written tests for all the components, and we've made sure
-    that the tests can be run on almost any platform, with no dependencies apart
-    from the ability to log text to a debug console somewhere. We also provide
-    an easy way to run all the tests on-device as part of an automated test
-    framework, and we use qemu/Renode emulation so that tests can be run even
-    without physical devices present.
-
--   **Easy to integrate**: We want to be as open a system as possible, and use
-    the best code available for each platform. To do that, we're going to rely
-    on projects like
-    [CMSIS-NN](https://www.keil.com/pack/doc/CMSIS/NN/html/index.html),
-    [uTensor](https://github.com/uTensor/uTensor), and other vendor libraries to
-    handle as much performance-critical code as possible. We know that there are
-    an increasing number of options to accelerate neural networks on
-    microcontrollers, so we're aiming to be a good host for deploying those
-    hardware technologies too.
-
--   **Compatible**: We're using the same file schema, interpreter API, and
-    kernel interface as regular TensorFlow Lite, so we leverage the large
-    existing set of tools, documentation, and examples for the project. The
-    biggest barrier to deploying ML models is getting them from a training
-    environment into a form that's easy to run inference on, so we see reusing
-    this rich ecosystem as being crucial to being easily usable. We also hope to
-    integrate this experimental work back into the main codebase in the future.
-
-To meet those goals, we've made some tradeoffs:
-
--   **Simple C++**: To help with readability, our code is written in a modern
-    version of C++, but we generally treat it as a "better C", rather relying on
-    more complex features such as template meta-programming. As mentioned
-    earlier, we avoid any use of dynamic memory allocation (new/delete) or the
-    standard C/C++ libraries, so we believe this should still be fairly
-    portable. It does mean that some older devices with C-only toolchains won't
-    be supported, but we're hoping that the reference operator implementations
-    (which are simple C-like functions) can still be useful in those cases. The
-    interfaces are also designed to be C-only, so it should be possible to
-    integrate the resulting library with pure C projects.
-
--   **Interpreted**: Code generation is a popular pattern for embedded code,
-    because it gives standalone code that's easy to modify and step through, but
-    we've chosen to go with an interpreted approach. In our internal
-    microcontroller work we've found that using an extremely stripped-down
-    interpreter with almost no dependencies gives us a lot of the same
-    advantages, but is easier to maintain. For example, when new updates come
-    out for the underlying library, you can just merge your local modifications
-    in a single step, rather than having to regenerate new code and then patch
-    in any changes you subsequently made. The coarse granularity of the
-    interpreted primitives means that each operation call typically takes
-    hundreds of thousands of instruction cycles at least, so we don't see
-    noticeable performance gains from avoiding what's essentially a single
-    switch statement at the interpreter level to call each operation. We're
-    still working on improving the packaging though, for example we're
-    considering having the ability to snapshot all the source files and headers
-    used for a particular model, being able to compile the code and data
-    together as a library, and then access it through a minimal set of C
-    interface calls which hide the underlying complexity.
-
--   **Flatbuffers**: We represent our models using
-    [the standard flatbuffer schema used by the rest of TensorFlow Lite](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/schema/schema.fbs),
-    with the difference that we always keep it in read-only program memory
-    (typically flash) rather than relying on having a file system to read it
-    from. This is a good fit because flatbuffer's serialized format is designed
-    to be mapped into memory without requiring any extra memory allocations or
-    modifications to access it. All of the functions to read model values work
-    directly on the serialized bytes, and large sections of data like weights
-    are directly accessible as sequential C-style arrays of their data type,
-    with no strides or unpacking needed. We do get a lot of value from using
-    flatbuffers, but there is a cost in complexity. The flat buffer library code
-    is all inline
-    [inside the main headers](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/schema/schema_generated.h),
-    but it isn't straightforward to inspect their implementations, and the model
-    data structures aren't easy to comprehend from the debugger. The header for
-    the schema itself also has to be periodically updated when new information
-    is added to the file format, though we try to handle that transparently for
-    most developers by checking in a pre-generated version.
-
--   **Code Duplication**: Some of the code in this prototype largely duplicates
-    the logic in other parts of the TensorFlow Lite code base, for example the
-    operator wrappers. We've tried to keep share as much as we can between the
-    two interpreters, but there are some assumptions built into the original
-    runtime that make this difficult. We'll be working on modularizing the main
-    interpreter so that we can move to an entirely shared system.
-
-This initial preview release is designed to get early feedback, and is not
-intended to be a final product. It only includes enough operations to run a
-simple keyword recognition model, and the implementations are not optimized.
-We're hoping this will be a good way to get feedback and collaborate to improve
-the framework.
-
-## Getting Started with Make
-
-Building requires a Linux or OS X machine.
+## Table of Contents
+
+-   [Getting Started](#getting-started)
+    *   [Getting Started with Portable Reference Code](#getting-started-with-portable-reference-code)
+    *   [Building Portable Reference Code using Make](#building-portable-reference-code-using-make)
+    *   [Building for the "Blue Pill" STM32F103 using Make](#building-for-the-blue-pill-stm32f103-using-make)
+    *   [Building for "Hifive1" SiFive FE310 development board using Make](#building-for-hifive1-sifive-fe310-development-board-using-make)
+    *   [Building for Ambiq Micro Apollo3Blue EVB using Make](#building-for-ambiq-micro-apollo3blue-evb-using-make)
+        *   [Additional Apollo3 Instructions](#additional-apollo3-instructions)
+-   [Goals](#goals)
+-   [Generating Project Files](#generating-project-#files)
+-   [How to Port TensorFlow Lite Micro to a New Platform](#how-to-port-tensorflow-lite-micro-to-a-new-platform)
+    *   [Requirements](#requirements)
+    *   [Getting Started](getting-started)
+    *   [Troubleshooting](#troubleshooting)
+    *   [Optimizing for your Platform](#optimizing-for-your-platform)
+    *   [Code Module Organization](#code-module-organization)
+    *   [Working with Generated Projects](#working-with-generated-projects)
+    *   [Supporting a Platform with Makefiles](#supporting-a-platform-with-makefiles)
+    *   [Supporting a Platform with Emulation Testing](#supporting-a-platform-with-emulation-testing)
+    *   [Implementing More Optimizations](#implementing-more-optimizations)
+
+# Getting Started
+
+One of the challenges of embedded software development is that there are a lot
+of different architectures, devices, operating systems, and build systems. We
+aim to support as many of the popular combinations as we can, and make it as
+easy as possible to add support for others.
+
+If you're a product developer, we have build instructions or pre-generated
+project files that you can download for the following platforms:
+
+| Device | Mbed | Keil | Make/GCC
+-------- | ---- | ---- | --------
+[STM32F746G Discovery Board](https://www.st.com/en/evaluation-tools/32f746gdiscovery.html) | [Download](https://drive.google.com/open?id=1OtgVkytQBrEYIpJPsE8F6GUKHPBS3Xeb) | - | [Download](https://drive.google.com/open?id=1u46mTtAMZ7Y1aD-He1u3R8AE4ZyEpnOl) | -
+["Blue Pill" STM32F103-compatible development board](https://github.com/google/stm32_bare_lib) | - | - | [Instructions](#building-for-the-blue-pill-stm32f103-using-make)
+[Ambiq Micro Apollo3Blue EVB using Make](https://ambiqmicro.com/apollo-ultra-low-power-mcus/) | - | - | [Instructions](#building-for-ambiq-micro-apollo3blue-evb-using-make)
+[Generic Keil uVision Projects](http://www2.keil.com/mdk5/uvision/) | - | [Download](https://drive.google.com/open?id=1Lw9rsdquNKObozClLPoE5CTJLuhfh5mV) | -
+
+If your device is not yet supported, it may not be too hard to add support. You
+can learn about that process
+[here](#how-to-port-tensorflow-lite-micro-to-a-new-platform). We're looking
+forward to getting your help expanding this table!
+
+## Getting Started with Portable Reference Code
+
+If you don't have a particular microcontroller platform in mind yet, or just
+want to try out the code before beginning porting, the easiest way to begin is
+by
+[downloading the platform-agnostic reference code](https://drive.google.com/open?id=1cawEQAkqquK_SO4crReDYqf_v7yAwOY8).
+You'll see a series of folders inside the archive, with each one containing just
+the source files you need to build one binary. There is a simple Makefile for
+each folder, but you should be able to load the files into almost any IDE and
+build them. There's also a [Visual Studio Code](https://code.visualstudio.com/) project file already set up, so
+you can easily explore the code in a cross-platform IDE.
+
+## Building Portable Reference Code using Make
+
+It's easy to build portable reference code directly from Github using make if
+you're on a Linux or OS X machine.
 
 -   Open a terminal
 -   Download the TensorFlow source with `git clone
@@ -177,7 +125,7 @@ building binaries that run locally on the Mac OS or Linux machine you're
 building on, but this approach becomes important when we're targeting simple
 micro controller devices.
 
-## Building for the "Blue Pill" STM32F103
+## Building for the "Blue Pill" STM32F103 using Make
 
 The goal of this library is to enable machine learning on resource-constrained
 micro controllers and DSPs, and as part of that we've targeted the
@@ -365,6 +313,117 @@ file =
 tensorflow/lite/experimental/micro/tools/make/gen/apollo3evb_cortex-m4/bin/pushbutton_cmsis_speech_test.bin
 5. Prog Addr = 0x0000C000
 
+## Goals
+
+The design goals are for the framework to be:
+
+-   **Readable**: We want embedded software engineers to be able to understand
+    what's required to run ML inference without having to study research papers.
+    We've tried to keep the code base small, modular, and have reference
+    implementations of all operations to help with this.
+
+-   **Easy to modify**: We know that there are a lot of different platforms and
+    requirements in the embedded world, and we don't expect to cover all of them
+    in one framework. Instead, we're hoping that it can be a good starting point
+    for developers to build on top of to meet their own needs. For example, we
+    tried to make it easy to replace the implementations of key computational
+    operators that are often crucial for performance, without having to touch
+    the data flow and other runtime code. We want it to make more sense to use
+    our workflow to handle things like model import and less-important
+    operations, and customize the parts that matter, rather than having to
+    reimplement everything in your own engine.
+
+-   **Well-tested**: If you're modifying code, you need to know if your changes
+    are correct. Having an easy way to test lets you develop much faster. To
+    help there, we've written tests for all the components, and we've made sure
+    that the tests can be run on almost any platform, with no dependencies apart
+    from the ability to log text to a debug console somewhere. We also provide
+    an easy way to run all the tests on-device as part of an automated test
+    framework, and we use qemu/Renode emulation so that tests can be run even
+    without physical devices present.
+
+-   **Easy to integrate**: We want to be as open a system as possible, and use
+    the best code available for each platform. To do that, we're going to rely
+    on projects like
+    [CMSIS-NN](https://www.keil.com/pack/doc/CMSIS/NN/html/index.html),
+    [uTensor](https://github.com/uTensor/uTensor), and other vendor libraries to
+    handle as much performance-critical code as possible. We know that there are
+    an increasing number of options to accelerate neural networks on
+    microcontrollers, so we're aiming to be a good host for deploying those
+    hardware technologies too.
+
+-   **Compatible**: We're using the same file schema, interpreter API, and
+    kernel interface as regular TensorFlow Lite, so we leverage the large
+    existing set of tools, documentation, and examples for the project. The
+    biggest barrier to deploying ML models is getting them from a training
+    environment into a form that's easy to run inference on, so we see reusing
+    this rich ecosystem as being crucial to being easily usable. We also hope to
+    integrate this experimental work back into the main codebase in the future.
+
+To meet those goals, we've made some tradeoffs:
+
+-   **Simple C++**: To help with readability, our code is written in a modern
+    version of C++, but we generally treat it as a "better C", rather relying on
+    more complex features such as template meta-programming. As mentioned
+    earlier, we avoid any use of dynamic memory allocation (new/delete) or the
+    standard C/C++ libraries, so we believe this should still be fairly
+    portable. It does mean that some older devices with C-only toolchains won't
+    be supported, but we're hoping that the reference operator implementations
+    (which are simple C-like functions) can still be useful in those cases. The
+    interfaces are also designed to be C-only, so it should be possible to
+    integrate the resulting library with pure C projects.
+
+-   **Interpreted**: Code generation is a popular pattern for embedded code,
+    because it gives standalone code that's easy to modify and step through, but
+    we've chosen to go with an interpreted approach. In our internal
+    microcontroller work we've found that using an extremely stripped-down
+    interpreter with almost no dependencies gives us a lot of the same
+    advantages, but is easier to maintain. For example, when new updates come
+    out for the underlying library, you can just merge your local modifications
+    in a single step, rather than having to regenerate new code and then patch
+    in any changes you subsequently made. The coarse granularity of the
+    interpreted primitives means that each operation call typically takes
+    hundreds of thousands of instruction cycles at least, so we don't see
+    noticeable performance gains from avoiding what's essentially a single
+    switch statement at the interpreter level to call each operation. We're
+    still working on improving the packaging though, for example we're
+    considering having the ability to snapshot all the source files and headers
+    used for a particular model, being able to compile the code and data
+    together as a library, and then access it through a minimal set of C
+    interface calls which hide the underlying complexity.
+
+-   **Flatbuffers**: We represent our models using
+    [the standard flatbuffer schema used by the rest of TensorFlow Lite](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/schema/schema.fbs),
+    with the difference that we always keep it in read-only program memory
+    (typically flash) rather than relying on having a file system to read it
+    from. This is a good fit because flatbuffer's serialized format is designed
+    to be mapped into memory without requiring any extra memory allocations or
+    modifications to access it. All of the functions to read model values work
+    directly on the serialized bytes, and large sections of data like weights
+    are directly accessible as sequential C-style arrays of their data type,
+    with no strides or unpacking needed. We do get a lot of value from using
+    flatbuffers, but there is a cost in complexity. The flat buffer library code
+    is all inline
+    [inside the main headers](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/schema/schema_generated.h),
+    but it isn't straightforward to inspect their implementations, and the model
+    data structures aren't easy to comprehend from the debugger. The header for
+    the schema itself also has to be periodically updated when new information
+    is added to the file format, though we try to handle that transparently for
+    most developers by checking in a pre-generated version.
+
+-   **Code Duplication**: Some of the code in this prototype largely duplicates
+    the logic in other parts of the TensorFlow Lite code base, for example the
+    operator wrappers. We've tried to keep share as much as we can between the
+    two interpreters, but there are some assumptions built into the original
+    runtime that make this difficult. We'll be working on modularizing the main
+    interpreter so that we can move to an entirely shared system.
+
+This initial preview release is designed to get early feedback, and is not
+intended to be a final product. It only includes enough operations to run a
+simple keyword recognition model, and the implementations are not optimized.
+We're hoping this will be a good way to get feedback and collaborate to improve
+the framework.
+
 ## Generating Project Files
 
 It's not always easy or convenient to use a makefile-based build process,
@@ -384,7 +443,8 @@ This will create a folder in
 `tensorflow/lite/experimental/micro/tools/make/gen/mbed_cortex-m4/prj/micro_speech_main_test/mbed`
 that contains the source and header files, some Mbed configuration files, and a
 README. You should then be able to copy this directory to another machine, and
-use it just like any other Mbed project.
+use it just like any other Mbed project. There's more information about project
+files [below](#working-with-generated-projects).
 
 ## How to Port TensorFlow Lite Micro to a New Platform
 
@@ -793,7 +853,7 @@ was successful. These scripts need to run on Ubuntu 18.04, in a bash
 environment, though Docker is available if you need to install extra software or
 have other dependencies.
 
-### Optimizing for your Platform
+### Implementing More Optimizations
 
 Clearly, getting debug logging support is only the beginning of the work you'll
 need to do on a particular platform. It's very likely that you'll want to
-- 
GitLab


From 1790a016fc49e1760078682e17c42168f1fbfa77 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 18:04:14 -0800
Subject: [PATCH 1015/2345] Recreate preconfigs with nvcc binaries.

PiperOrigin-RevId: 230025587
---
 .../ubuntu14.04/cuda10.0-cudnn7/cuda/BUILD    | 44 +++++++++++-------
 .../ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD     | 45 ++++++++++++-------
 .../preconfig/ubuntu14.04/tensorrt5/BUILD     |  8 +---
 .../ubuntu14.04/tensorrt5/build_defs.bzl      |  4 +-
 4 files changed, 62 insertions(+), 39 deletions(-)

diff --git a/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/BUILD b/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/BUILD
index c813efccf9..13429d7b88 100755
--- a/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/BUILD
+++ b/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/BUILD
@@ -1192,9 +1192,7 @@ genrule(
         "cuda/include/vector_functions.hpp",
         "cuda/include/vector_types.h",
     ],
-    cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/local/cuda-10.0/include/CL/cl.h" "$(@D)/cuda/include/CL/cl.h" && cp -f "/usr/local/cuda-10.0/include/CL/cl.hpp" "$(@D)/cuda/include/CL/cl.hpp" && cp -f "/usr/local/cuda-10.0/include/CL/cl_egl.h" "$(@D)/cuda/include/CL/cl_egl.h" && cp -f "/usr/local/cuda-10.0/include/CL/cl_ext.h" "$(@D)/cuda/include/CL/cl_ext.h" && cp -f "/usr/local/cuda-10.0/include/CL/cl_gl.h" "$(@D)/cuda/include/CL/cl_gl.h" && cp -f "/usr/local/cuda-10.0/include/CL/cl_gl_ext.h" "$(@D)/cuda/include/CL/cl_gl_ext.h" && cp -f "/usr/local/cuda-10.0/include/CL/cl_platform.h" "$(@D)/cuda/include/CL/cl_platform.h" && cp -f "/usr/local/cuda-10.0/include/CL/opencl.h" "$(@D)/cuda/include/CL/opencl.h" && cp -f "/usr/local/cuda-10.0/include/builtin_types.h" "$(@D)/cuda/include/builtin_types.h" && cp -f "/usr/local/cuda-10.0/include/channel_descriptor.h" "$(@D)/cuda/include/channel_descriptor.h" && cp -f "/usr/local/cuda-10.0/include/common_functions.h" "$(@D)/cuda/include/common_functions.h" && cp -f "/usr/local/cuda-10.0/include/cooperative_groups.h" "$(@D)/cuda/include/cooperative_groups.h" && cp -f "/usr/local/cuda-10.0/include/cooperative_groups_helpers.h" "$(@D)/cuda/include/cooperative_groups_helpers.h" && cp -f "/usr/local/cuda-10.0/include/crt/common_functions.h" "$(@D)/cuda/include/crt/common_functions.h" && cp -f "/usr/local/cuda-10.0/include/crt/device_double_functions.h" "$(@D)/cuda/include/crt/device_double_functions.h" && cp -f "/usr/local/cuda-10.0/include/crt/device_double_functions.hpp" "$(@D)/cuda/include/crt/device_double_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/crt/device_functions.h" "$(@D)/cuda/include/crt/device_functions.h" && cp -f "/usr/local/cuda-10.0/include/crt/device_functions.hpp" "$(@D)/cuda/include/crt/device_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/crt/func_macro.h" "$(@D)/cuda/include/crt/func_macro.h" && cp -f "/usr/local/cuda-10.0/include/crt/host_config.h" "$(@D)/cuda/include/crt/host_config.h" && cp -f "/usr/local/cuda-10.0/include/crt/host_defines.h" "$(@D)/cuda/include/crt/host_defines.h" && cp -f "/usr/local/cuda-10.0/include/crt/host_runtime.h" "$(@D)/cuda/include/crt/host_runtime.h" && cp -f "/usr/local/cuda-10.0/include/crt/math_functions.h" "$(@D)/cuda/include/crt/math_functions.h" && cp -f "/usr/local/cuda-10.0/include/crt/math_functions.hpp" "$(@D)/cuda/include/crt/math_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/crt/mma.h" "$(@D)/cuda/include/crt/mma.h" && cp -f "/usr/local/cuda-10.0/include/crt/mma.hpp" "$(@D)/cuda/include/crt/mma.hpp" && cp -f "/usr/local/cuda-10.0/include/crt/nvfunctional" "$(@D)/cuda/include/crt/nvfunctional" && cp -f "/usr/local/cuda-10.0/include/crt/sm_70_rt.h" "$(@D)/cuda/include/crt/sm_70_rt.h" && cp -f "/usr/local/cuda-10.0/include/crt/sm_70_rt.hpp" "$(@D)/cuda/include/crt/sm_70_rt.hpp" && cp -f "/usr/local/cuda-10.0/include/crt/storage_class.h" "$(@D)/cuda/include/crt/storage_class.h" && cp -f "/usr/local/cuda-10.0/include/cuComplex.h" "$(@D)/cuda/include/cuComplex.h" && cp -f "/usr/local/cuda-10.0/include/cublas.h" "$(@D)/cuda/include/cublas.h" && cp -f "/usr/local/cuda-10.0/include/cublasXt.h" "$(@D)/cuda/include/cublasXt.h" && cp -f "/usr/local/cuda-10.0/include/cublas_api.h" "$(@D)/cuda/include/cublas_api.h" && cp -f "/usr/local/cuda-10.0/include/cublas_v2.h" "$(@D)/cuda/include/cublas_v2.h" && cp -f "/usr/local/cuda-10.0/include/cuda.h" "$(@D)/cuda/include/cuda.h" && cp -f "/usr/local/cuda-10.0/include/cudaEGL.h" "$(@D)/cuda/include/cudaEGL.h" && cp -f "/usr/local/cuda-10.0/include/cudaGL.h" "$(@D)/cuda/include/cudaGL.h" && cp -f "/usr/local/cuda-10.0/include/cudaProfiler.h" "$(@D)/cuda/include/cudaProfiler.h" && cp -f "/usr/local/cuda-10.0/include/cudaVDPAU.h" "$(@D)/cuda/include/cudaVDPAU.h" && cp -f "/usr/local/cuda-10.0/include/cuda_device_runtime_api.h" "$(@D)/cuda/include/cuda_device_runtime_api.h" && cp -f "/usr/local/cuda-10.0/include/cuda_egl_interop.h" "$(@D)/cuda/include/cuda_egl_interop.h" && cp -f "/usr/local/cuda-10.0/include/cuda_fp16.h" "$(@D)/cuda/include/cuda_fp16.h" && cp -f "/usr/local/cuda-10.0/include/cuda_fp16.hpp" "$(@D)/cuda/include/cuda_fp16.hpp" && cp -f "/usr/local/cuda-10.0/include/cuda_gl_interop.h" "$(@D)/cuda/include/cuda_gl_interop.h" && cp -f "/usr/local/cuda-10.0/include/cuda_occupancy.h" "$(@D)/cuda/include/cuda_occupancy.h" && cp -f "/usr/local/cuda-10.0/include/cuda_profiler_api.h" "$(@D)/cuda/include/cuda_profiler_api.h" && cp -f "/usr/local/cuda-10.0/include/cuda_runtime.h" "$(@D)/cuda/include/cuda_runtime.h" && cp -f "/usr/local/cuda-10.0/include/cuda_runtime_api.h" "$(@D)/cuda/include/cuda_runtime_api.h" && cp -f "/usr/local/cuda-10.0/include/cuda_surface_types.h" "$(@D)/cuda/include/cuda_surface_types.h" && cp -f "/usr/local/cuda-10.0/include/cuda_texture_types.h" "$(@D)/cuda/include/cuda_texture_types.h" && cp -f "/usr/local/cuda-10.0/include/cuda_vdpau_interop.h" "$(@D)/cuda/include/cuda_vdpau_interop.h" && cp -f "/usr/local/cuda-10.0/include/cudalibxt.h" "$(@D)/cuda/include/cudalibxt.h" && cp -f "/usr/local/cuda-10.0/include/cudart_platform.h" "$(@D)/cuda/include/cudart_platform.h" && cp -f "/usr/local/cuda-10.0/include/cufft.h" "$(@D)/cuda/include/cufft.h" && cp -f "/usr/local/cuda-10.0/include/cufftXt.h" "$(@D)/cuda/include/cufftXt.h" && cp -f "/usr/local/cuda-10.0/include/cufftw.h" "$(@D)/cuda/include/cufftw.h" && cp -f "/usr/local/cuda-10.0/include/curand.h" "$(@D)/cuda/include/curand.h" && cp -f "/usr/local/cuda-10.0/include/curand_discrete.h" "$(@D)/cuda/include/curand_discrete.h" && cp -f "/usr/local/cuda-10.0/include/curand_discrete2.h" "$(@D)/cuda/include/curand_discrete2.h" && cp -f "/usr/local/cuda-10.0/include/curand_globals.h" "$(@D)/cuda/include/curand_globals.h" && cp -f "/usr/local/cuda-10.0/include/curand_kernel.h" "$(@D)/cuda/include/curand_kernel.h" && cp -f "/usr/local/cuda-10.0/include/curand_lognormal.h" "$(@D)/cuda/include/curand_lognormal.h" && cp -f "/usr/local/cuda-10.0/include/curand_mrg32k3a.h" "$(@D)/cuda/include/curand_mrg32k3a.h" && cp -f "/usr/local/cuda-10.0/include/curand_mtgp32.h" "$(@D)/cuda/include/curand_mtgp32.h" && cp -f "/usr/local/cuda-10.0/include/curand_mtgp32_host.h" "$(@D)/cuda/include/curand_mtgp32_host.h" && cp -f "/usr/local/cuda-10.0/include/curand_mtgp32_kernel.h" "$(@D)/cuda/include/curand_mtgp32_kernel.h" && cp -f "/usr/local/cuda-10.0/include/curand_mtgp32dc_p_11213.h" "$(@D)/cuda/include/curand_mtgp32dc_p_11213.h" && cp -f "/usr/local/cuda-10.0/include/curand_normal.h" "$(@D)/cuda/include/curand_normal.h" && cp -f "/usr/local/cuda-10.0/include/curand_normal_static.h" "$(@D)/cuda/include/curand_normal_static.h" && cp -f "/usr/local/cuda-10.0/include/curand_philox4x32_x.h" "$(@D)/cuda/include/curand_philox4x32_x.h" && cp -f "/usr/local/cuda-10.0/include/curand_poisson.h" "$(@D)/cuda/include/curand_poisson.h" && cp -f "/usr/local/cuda-10.0/include/curand_precalc.h" "$(@D)/cuda/include/curand_precalc.h" && cp -f "/usr/local/cuda-10.0/include/curand_uniform.h" "$(@D)/cuda/include/curand_uniform.h" && cp -f "/usr/local/cuda-10.0/include/cusolverDn.h" "$(@D)/cuda/include/cusolverDn.h" && cp -f "/usr/local/cuda-10.0/include/cusolverRf.h" "$(@D)/cuda/include/cusolverRf.h" && cp -f "/usr/local/cuda-10.0/include/cusolverSp.h" "$(@D)/cuda/include/cusolverSp.h" && cp -f "/usr/local/cuda-10.0/include/cusolverSp_LOWLEVEL_PREVIEW.h" "$(@D)/cuda/include/cusolverSp_LOWLEVEL_PREVIEW.h" && cp -f "/usr/local/cuda-10.0/include/cusolver_common.h" "$(@D)/cuda/include/cusolver_common.h" && cp -f "/usr/local/cuda-10.0/include/cusparse.h" "$(@D)/cuda/include/cusparse.h" && cp -f "/usr/local/cuda-10.0/include/cusparse_v2.h" "$(@D)/cuda/include/cusparse_v2.h" && cp -f "/usr/local/cuda-10.0/include/device_atomic_functions.h" "$(@D)/cuda/include/device_atomic_functions.h" && cp -f "/usr/local/cuda-10.0/include/device_atomic_functions.hpp" "$(@D)/cuda/include/device_atomic_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/device_double_functions.h" "$(@D)/cuda/include/device_double_functions.h" && cp -f "/usr/local/cuda-10.0/include/device_functions.h" "$(@D)/cuda/include/device_functions.h" && cp -f "/usr/local/cuda-10.0/include/device_launch_parameters.h" "$(@D)/cuda/include/device_launch_parameters.h" && cp -f "/usr/local/cuda-10.0/include/device_types.h" "$(@D)/cuda/include/device_types.h" && cp -f "/usr/local/cuda-10.0/include/driver_functions.h" "$(@D)/cuda/include/driver_functions.h" && cp -f "/usr/local/cuda-10.0/include/driver_types.h" "$(@D)/cuda/include/driver_types.h" && cp -f "/usr/local/cuda-10.0/include/fatBinaryCtl.h" "$(@D)/cuda/include/fatBinaryCtl.h" && cp -f "/usr/local/cuda-10.0/include/fatbinary.h" "$(@D)/cuda/include/fatbinary.h" && cp -f "/usr/local/cuda-10.0/include/host_config.h" "$(@D)/cuda/include/host_config.h" && cp -f "/usr/local/cuda-10.0/include/host_defines.h" "$(@D)/cuda/include/host_defines.h" && cp -f "/usr/local/cuda-10.0/include/library_types.h" "$(@D)/cuda/include/library_types.h" && cp -f "/usr/local/cuda-10.0/include/math_constants.h" "$(@D)/cuda/include/math_constants.h" && cp -f "/usr/local/cuda-10.0/include/math_functions.h" "$(@D)/cuda/include/math_functions.h" && cp -f "/usr/local/cuda-10.0/include/mma.h" "$(@D)/cuda/include/mma.h" && cp -f "/usr/local/cuda-10.0/include/npp.h" "$(@D)/cuda/include/npp.h" && cp -f "/usr/local/cuda-10.0/include/nppcore.h" "$(@D)/cuda/include/nppcore.h" && cp -f "/usr/local/cuda-10.0/include/nppdefs.h" "$(@D)/cuda/include/nppdefs.h" && cp -f "/usr/local/cuda-10.0/include/nppi.h" "$(@D)/cuda/include/nppi.h" && cp -f "/usr/local/cuda-10.0/include/nppi_arithmetic_and_logical_operations.h" "$(@D)/cuda/include/nppi_arithmetic_and_logical_operations.h" && cp -f "/usr/local/cuda-10.0/include/nppi_color_conversion.h" "$(@D)/cuda/include/nppi_color_conversion.h" && cp -f "/usr/local/cuda-10.0/include/nppi_compression_functions.h" "$(@D)/cuda/include/nppi_compression_functions.h" && cp -f "/usr/local/cuda-10.0/include/nppi_computer_vision.h" "$(@D)/cuda/include/nppi_computer_vision.h" && cp -f "/usr/local/cuda-10.0/include/nppi_data_exchange_and_initialization.h" "$(@D)/cuda/include/nppi_data_exchange_and_initialization.h" && cp -f "/usr/local/cuda-10.0/include/nppi_filtering_functions.h" "$(@D)/cuda/include/nppi_filtering_functions.h" && cp -f "/usr/local/cuda-10.0/include/nppi_geometry_transforms.h" "$(@D)/cuda/include/nppi_geometry_transforms.h" && cp -f "/usr/local/cuda-10.0/include/nppi_linear_transforms.h" "$(@D)/cuda/include/nppi_linear_transforms.h" && cp -f "/usr/local/cuda-10.0/include/nppi_morphological_operations.h" "$(@D)/cuda/include/nppi_morphological_operations.h" && cp -f "/usr/local/cuda-10.0/include/nppi_statistics_functions.h" "$(@D)/cuda/include/nppi_statistics_functions.h" && cp -f "/usr/local/cuda-10.0/include/nppi_support_functions.h" "$(@D)/cuda/include/nppi_support_functions.h" && cp -f "/usr/local/cuda-10.0/include/nppi_threshold_and_compare_operations.h" "$(@D)/cuda/include/nppi_threshold_and_compare_operations.h" && cp -f "/usr/local/cuda-10.0/include/npps.h" "$(@D)/cuda/include/npps.h" && cp -f "/usr/local/cuda-10.0/include/npps_arithmetic_and_logical_operations.h" "$(@D)/cuda/include/npps_arithmetic_and_logical_operations.h" && cp -f "/usr/local/cuda-10.0/include/npps_conversion_functions.h" "$(@D)/cuda/include/npps_conversion_functions.h" && cp -f "/usr/local/cuda-10.0/include/npps_filtering_functions.h" "$(@D)/cuda/include/npps_filtering_functions.h" && cp -f "/usr/local/cuda-10.0/include/npps_initialization.h" "$(@D)/cuda/include/npps_initialization.h" && cp -f "/usr/local/cuda-10.0/include/npps_statistics_functions.h" "$(@D)/cuda/include/npps_statistics_functions.h" && cp -f "/usr/local/cuda-10.0/include/npps_support_functions.h" "$(@D)/cuda/include/npps_support_functions.h" && cp -f "/usr/local/cuda-10.0/include/nppversion.h" "$(@D)/cuda/include/nppversion.h" && cp -f "/usr/local/cuda-10.0/include/nvToolsExt.h" "$(@D)/cuda/include/nvToolsExt.h" && cp -f "/usr/local/cuda-10.0/include/nvToolsExtCuda.h" "$(@D)/cuda/include/nvToolsExtCuda.h" && cp -f "/usr/local/cuda-10.0/include/nvToolsExtCudaRt.h" "$(@D)/cuda/include/nvToolsExtCudaRt.h" && cp -f "/usr/local/cuda-10.0/include/nvToolsExtMeta.h" "$(@D)/cuda/include/nvToolsExtMeta.h" && cp -f "/usr/local/cuda-10.0/include/nvToolsExtSync.h" "$(@D)/cuda/include/nvToolsExtSync.h" && cp -f "/usr/local/cuda-10.0/include/nvblas.h" "$(@D)/cuda/include/nvblas.h" && cp -f "/usr/local/cuda-10.0/include/nvfunctional" "$(@D)/cuda/include/nvfunctional" && cp -f "/usr/local/cuda-10.0/include/nvgraph.h" "$(@D)/cuda/include/nvgraph.h" && cp -f "/usr/local/cuda-10.0/include/nvjpeg.h" "$(@D)/cuda/include/nvjpeg.h" && cp -f "/usr/local/cuda-10.0/include/nvml.h" "$(@D)/cuda/include/nvml.h" && cp -f "/usr/local/cuda-10.0/include/nvrtc.h" "$(@D)/cuda/include/nvrtc.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvToolsExt.h" "$(@D)/cuda/include/nvtx3/nvToolsExt.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvToolsExtCuda.h" "$(@D)/cuda/include/nvtx3/nvToolsExtCuda.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvToolsExtCudaRt.h" "$(@D)/cuda/include/nvtx3/nvToolsExtCudaRt.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvToolsExtOpenCL.h" "$(@D)/cuda/include/nvtx3/nvToolsExtOpenCL.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvToolsExtSync.h" "$(@D)/cuda/include/nvtx3/nvToolsExtSync.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxImpl.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxImpl.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxImplCore.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxImplCore.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxImplCudaRt_v3.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxImplCudaRt_v3.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxImplCuda_v3.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxImplCuda_v3.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxImplOpenCL_v3.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxImplOpenCL_v3.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxImplSync_v3.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxImplSync_v3.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxInit.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxInit.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxInitDecls.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxInitDecls.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxInitDefs.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxInitDefs.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxLinkOnce.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxLinkOnce.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxTypes.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxTypes.h" && cp -f "/usr/local/cuda-10.0/include/sm_20_atomic_functions.h" "$(@D)/cuda/include/sm_20_atomic_functions.h" && cp -f "/usr/local/cuda-10.0/include/sm_20_atomic_functions.hpp" "$(@D)/cuda/include/sm_20_atomic_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/sm_20_intrinsics.h" "$(@D)/cuda/include/sm_20_intrinsics.h" && cp -f "/usr/local/cuda-10.0/include/sm_20_intrinsics.hpp" "$(@D)/cuda/include/sm_20_intrinsics.hpp" && cp -f "/usr/local/cuda-10.0/include/sm_30_intrinsics.h" "$(@D)/cuda/include/sm_30_intrinsics.h" && cp -f "/usr/local/cuda-10.0/include/sm_30_intrinsics.hpp" "$(@D)/cuda/include/sm_30_intrinsics.hpp" && cp -f "/usr/local/cuda-10.0/include/sm_32_atomic_functions.h" "$(@D)/cuda/include/sm_32_atomic_functions.h" && cp -f "/usr/local/cuda-10.0/include/sm_32_atomic_functions.hpp" "$(@D)/cuda/include/sm_32_atomic_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/sm_32_intrinsics.h" "$(@D)/cuda/include/sm_32_intrinsics.h" && cp -f "/usr/local/cuda-10.0/include/sm_32_intrinsics.hpp" "$(@D)/cuda/include/sm_32_intrinsics.hpp" && cp -f "/usr/local/cuda-10.0/include/sm_35_atomic_functions.h" "$(@D)/cuda/include/sm_35_atomic_functions.h" && cp -f "/usr/local/cuda-10.0/include/sm_35_intrinsics.h" "$(@D)/cuda/include/sm_35_intrinsics.h" && cp -f "/usr/local/cuda-10.0/include/sm_60_atomic_functions.h" "$(@D)/cuda/include/sm_60_atomic_functions.h" && cp -f "/usr/local/cuda-10.0/include/sm_60_atomic_functions.hpp" "$(@D)/cuda/include/sm_60_atomic_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/sm_61_intrinsics.h" "$(@D)/cuda/include/sm_61_intrinsics.h" && cp -f "/usr/local/cuda-10.0/include/sm_61_intrinsics.hpp" "$(@D)/cuda/include/sm_61_intrinsics.hpp" && cp -f "/usr/local/cuda-10.0/include/sobol_direction_vectors.h" "$(@D)/cuda/include/sobol_direction_vectors.h" && cp -f "/usr/local/cuda-10.0/include/surface_functions.h" "$(@D)/cuda/include/surface_functions.h" && cp -f "/usr/local/cuda-10.0/include/surface_functions.hpp" "$(@D)/cuda/include/surface_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/surface_indirect_functions.h" "$(@D)/cuda/include/surface_indirect_functions.h" && cp -f "/usr/local/cuda-10.0/include/surface_indirect_functions.hpp" "$(@D)/cuda/include/surface_indirect_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/surface_types.h" "$(@D)/cuda/include/surface_types.h" && cp -f "/usr/local/cuda-10.0/include/texture_fetch_functions.h" "$(@D)/cuda/include/texture_fetch_functions.h" && cp -f "/usr/local/cuda-10.0/include/texture_fetch_functions.hpp" "$(@D)/cuda/include/texture_fetch_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/texture_indirect_functions.h" "$(@D)/cuda/include/texture_indirect_functions.h" && cp -f "/usr/local/cuda-10.0/include/texture_indirect_functions.hpp" "$(@D)/cuda/include/texture_indirect_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/texture_types.h" "$(@D)/cuda/include/texture_types.h" && cp -f "/usr/local/cuda-10.0/include/thrust/adjacent_difference.h" "$(@D)/cuda/include/thrust/adjacent_difference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/advance.h" "$(@D)/cuda/include/thrust/advance.h" && cp -f "/usr/local/cuda-10.0/include/thrust/binary_search.h" "$(@D)/cuda/include/thrust/binary_search.h" && cp -f "/usr/local/cuda-10.0/include/thrust/complex.h" "$(@D)/cuda/include/thrust/complex.h" && cp -f "/usr/local/cuda-10.0/include/thrust/copy.h" "$(@D)/cuda/include/thrust/copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/count.h" "$(@D)/cuda/include/thrust/count.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/adjacent_difference.inl" "$(@D)/cuda/include/thrust/detail/adjacent_difference.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/advance.inl" "$(@D)/cuda/include/thrust/detail/advance.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/alignment.h" "$(@D)/cuda/include/thrust/detail/alignment.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/allocator_traits.h" "$(@D)/cuda/include/thrust/detail/allocator/allocator_traits.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/allocator_traits.inl" "$(@D)/cuda/include/thrust/detail/allocator/allocator_traits.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/copy_construct_range.h" "$(@D)/cuda/include/thrust/detail/allocator/copy_construct_range.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/copy_construct_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/copy_construct_range.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/default_construct_range.h" "$(@D)/cuda/include/thrust/detail/allocator/default_construct_range.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/default_construct_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/default_construct_range.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/destroy_range.h" "$(@D)/cuda/include/thrust/detail/allocator/destroy_range.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/destroy_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/destroy_range.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/fill_construct_range.h" "$(@D)/cuda/include/thrust/detail/allocator/fill_construct_range.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/fill_construct_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/fill_construct_range.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/malloc_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/malloc_allocator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/malloc_allocator.inl" "$(@D)/cuda/include/thrust/detail/allocator/malloc_allocator.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/no_throw_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/no_throw_allocator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/tagged_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/tagged_allocator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/tagged_allocator.inl" "$(@D)/cuda/include/thrust/detail/allocator/tagged_allocator.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/temporary_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/temporary_allocator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/temporary_allocator.inl" "$(@D)/cuda/include/thrust/detail/allocator/temporary_allocator.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/binary_search.inl" "$(@D)/cuda/include/thrust/detail/binary_search.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/arithmetic.h" "$(@D)/cuda/include/thrust/detail/complex/arithmetic.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/c99math.h" "$(@D)/cuda/include/thrust/detail/complex/c99math.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/catrig.h" "$(@D)/cuda/include/thrust/detail/complex/catrig.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/catrigf.h" "$(@D)/cuda/include/thrust/detail/complex/catrigf.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/ccosh.h" "$(@D)/cuda/include/thrust/detail/complex/ccosh.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/ccoshf.h" "$(@D)/cuda/include/thrust/detail/complex/ccoshf.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/cexp.h" "$(@D)/cuda/include/thrust/detail/complex/cexp.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/cexpf.h" "$(@D)/cuda/include/thrust/detail/complex/cexpf.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/clog.h" "$(@D)/cuda/include/thrust/detail/complex/clog.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/clogf.h" "$(@D)/cuda/include/thrust/detail/complex/clogf.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/complex.inl" "$(@D)/cuda/include/thrust/detail/complex/complex.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/cpow.h" "$(@D)/cuda/include/thrust/detail/complex/cpow.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/cproj.h" "$(@D)/cuda/include/thrust/detail/complex/cproj.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/csinh.h" "$(@D)/cuda/include/thrust/detail/complex/csinh.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/csinhf.h" "$(@D)/cuda/include/thrust/detail/complex/csinhf.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/csqrt.h" "$(@D)/cuda/include/thrust/detail/complex/csqrt.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/csqrtf.h" "$(@D)/cuda/include/thrust/detail/complex/csqrtf.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/ctanh.h" "$(@D)/cuda/include/thrust/detail/complex/ctanh.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/ctanhf.h" "$(@D)/cuda/include/thrust/detail/complex/ctanhf.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/math_private.h" "$(@D)/cuda/include/thrust/detail/complex/math_private.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/stream.h" "$(@D)/cuda/include/thrust/detail/complex/stream.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config.h" "$(@D)/cuda/include/thrust/detail/config.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/compiler.h" "$(@D)/cuda/include/thrust/detail/config/compiler.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/compiler_fence.h" "$(@D)/cuda/include/thrust/detail/config/compiler_fence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/config.h" "$(@D)/cuda/include/thrust/detail/config/config.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/debug.h" "$(@D)/cuda/include/thrust/detail/config/debug.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/device_system.h" "$(@D)/cuda/include/thrust/detail/config/device_system.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/exec_check_disable.h" "$(@D)/cuda/include/thrust/detail/config/exec_check_disable.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/forceinline.h" "$(@D)/cuda/include/thrust/detail/config/forceinline.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/global_workarounds.h" "$(@D)/cuda/include/thrust/detail/config/global_workarounds.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/host_device.h" "$(@D)/cuda/include/thrust/detail/config/host_device.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/host_system.h" "$(@D)/cuda/include/thrust/detail/config/host_system.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/simple_defines.h" "$(@D)/cuda/include/thrust/detail/config/simple_defines.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/contiguous_storage.h" "$(@D)/cuda/include/thrust/detail/contiguous_storage.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/contiguous_storage.inl" "$(@D)/cuda/include/thrust/detail/contiguous_storage.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/copy.h" "$(@D)/cuda/include/thrust/detail/copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/copy.inl" "$(@D)/cuda/include/thrust/detail/copy.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/copy_if.h" "$(@D)/cuda/include/thrust/detail/copy_if.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/copy_if.inl" "$(@D)/cuda/include/thrust/detail/copy_if.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/count.inl" "$(@D)/cuda/include/thrust/detail/count.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/cstdint.h" "$(@D)/cuda/include/thrust/detail/cstdint.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/device_delete.inl" "$(@D)/cuda/include/thrust/detail/device_delete.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/device_free.inl" "$(@D)/cuda/include/thrust/detail/device_free.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/device_malloc.inl" "$(@D)/cuda/include/thrust/detail/device_malloc.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/device_new.inl" "$(@D)/cuda/include/thrust/detail/device_new.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/device_ptr.inl" "$(@D)/cuda/include/thrust/detail/device_ptr.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/device_reference.inl" "$(@D)/cuda/include/thrust/detail/device_reference.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/device_vector.inl" "$(@D)/cuda/include/thrust/detail/device_vector.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/dispatch/is_trivial_copy.h" "$(@D)/cuda/include/thrust/detail/dispatch/is_trivial_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/distance.inl" "$(@D)/cuda/include/thrust/detail/distance.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/equal.inl" "$(@D)/cuda/include/thrust/detail/equal.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/execute_with_allocator.h" "$(@D)/cuda/include/thrust/detail/execute_with_allocator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/execution_policy.h" "$(@D)/cuda/include/thrust/detail/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/extrema.inl" "$(@D)/cuda/include/thrust/detail/extrema.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/fill.inl" "$(@D)/cuda/include/thrust/detail/fill.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/find.inl" "$(@D)/cuda/include/thrust/detail/find.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/for_each.inl" "$(@D)/cuda/include/thrust/detail/for_each.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/function.h" "$(@D)/cuda/include/thrust/detail/function.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional.inl" "$(@D)/cuda/include/thrust/detail/functional.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/actor.h" "$(@D)/cuda/include/thrust/detail/functional/actor.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/actor.inl" "$(@D)/cuda/include/thrust/detail/functional/actor.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/argument.h" "$(@D)/cuda/include/thrust/detail/functional/argument.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/composite.h" "$(@D)/cuda/include/thrust/detail/functional/composite.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/operators/arithmetic_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/arithmetic_operators.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/operators/assignment_operator.h" "$(@D)/cuda/include/thrust/detail/functional/operators/assignment_operator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/operators/bitwise_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/bitwise_operators.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/operators/compound_assignment_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/compound_assignment_operators.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/operators/logical_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/logical_operators.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/operators/operator_adaptors.h" "$(@D)/cuda/include/thrust/detail/functional/operators/operator_adaptors.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/operators/relational_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/relational_operators.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/placeholder.h" "$(@D)/cuda/include/thrust/detail/functional/placeholder.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/value.h" "$(@D)/cuda/include/thrust/detail/functional/value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/gather.inl" "$(@D)/cuda/include/thrust/detail/gather.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/generate.inl" "$(@D)/cuda/include/thrust/detail/generate.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/get_iterator_value.h" "$(@D)/cuda/include/thrust/detail/get_iterator_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/host_vector.inl" "$(@D)/cuda/include/thrust/detail/host_vector.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/inner_product.inl" "$(@D)/cuda/include/thrust/detail/inner_product.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/integer_math.h" "$(@D)/cuda/include/thrust/detail/integer_math.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/integer_traits.h" "$(@D)/cuda/include/thrust/detail/integer_traits.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/internal_functional.h" "$(@D)/cuda/include/thrust/detail/internal_functional.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/logical.inl" "$(@D)/cuda/include/thrust/detail/logical.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/merge.inl" "$(@D)/cuda/include/thrust/detail/merge.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/minmax.h" "$(@D)/cuda/include/thrust/detail/minmax.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/mismatch.inl" "$(@D)/cuda/include/thrust/detail/mismatch.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/mpl/math.h" "$(@D)/cuda/include/thrust/detail/mpl/math.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/numeric_traits.h" "$(@D)/cuda/include/thrust/detail/numeric_traits.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/overlapped_copy.h" "$(@D)/cuda/include/thrust/detail/overlapped_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/pair.inl" "$(@D)/cuda/include/thrust/detail/pair.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/partition.inl" "$(@D)/cuda/include/thrust/detail/partition.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/pointer.h" "$(@D)/cuda/include/thrust/detail/pointer.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/pointer.inl" "$(@D)/cuda/include/thrust/detail/pointer.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/preprocessor.h" "$(@D)/cuda/include/thrust/detail/preprocessor.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/range/head_flags.h" "$(@D)/cuda/include/thrust/detail/range/head_flags.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/range/tail_flags.h" "$(@D)/cuda/include/thrust/detail/range/tail_flags.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/raw_pointer_cast.h" "$(@D)/cuda/include/thrust/detail/raw_pointer_cast.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/raw_reference_cast.h" "$(@D)/cuda/include/thrust/detail/raw_reference_cast.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/reduce.inl" "$(@D)/cuda/include/thrust/detail/reduce.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/reference.h" "$(@D)/cuda/include/thrust/detail/reference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/reference.inl" "$(@D)/cuda/include/thrust/detail/reference.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/reference_forward_declaration.h" "$(@D)/cuda/include/thrust/detail/reference_forward_declaration.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/remove.inl" "$(@D)/cuda/include/thrust/detail/remove.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/replace.inl" "$(@D)/cuda/include/thrust/detail/replace.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/reverse.inl" "$(@D)/cuda/include/thrust/detail/reverse.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/scan.inl" "$(@D)/cuda/include/thrust/detail/scan.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/scatter.inl" "$(@D)/cuda/include/thrust/detail/scatter.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/seq.h" "$(@D)/cuda/include/thrust/detail/seq.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/sequence.inl" "$(@D)/cuda/include/thrust/detail/sequence.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/set_operations.inl" "$(@D)/cuda/include/thrust/detail/set_operations.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/sort.inl" "$(@D)/cuda/include/thrust/detail/sort.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/static_assert.h" "$(@D)/cuda/include/thrust/detail/static_assert.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/static_map.h" "$(@D)/cuda/include/thrust/detail/static_map.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/swap.h" "$(@D)/cuda/include/thrust/detail/swap.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/swap.inl" "$(@D)/cuda/include/thrust/detail/swap.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/swap_ranges.inl" "$(@D)/cuda/include/thrust/detail/swap_ranges.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/tabulate.inl" "$(@D)/cuda/include/thrust/detail/tabulate.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/temporary_array.h" "$(@D)/cuda/include/thrust/detail/temporary_array.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/temporary_array.inl" "$(@D)/cuda/include/thrust/detail/temporary_array.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/transform.inl" "$(@D)/cuda/include/thrust/detail/transform.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/transform_reduce.inl" "$(@D)/cuda/include/thrust/detail/transform_reduce.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/transform_scan.inl" "$(@D)/cuda/include/thrust/detail/transform_scan.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/trivial_sequence.h" "$(@D)/cuda/include/thrust/detail/trivial_sequence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/tuple.inl" "$(@D)/cuda/include/thrust/detail/tuple.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/tuple_meta_transform.h" "$(@D)/cuda/include/thrust/detail/tuple_meta_transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/tuple_transform.h" "$(@D)/cuda/include/thrust/detail/tuple_transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits.h" "$(@D)/cuda/include/thrust/detail/type_traits.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/algorithm/intermediate_type_from_function_and_iterators.h" "$(@D)/cuda/include/thrust/detail/type_traits/algorithm/intermediate_type_from_function_and_iterators.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/function_traits.h" "$(@D)/cuda/include/thrust/detail/type_traits/function_traits.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/has_member_function.h" "$(@D)/cuda/include/thrust/detail/type_traits/has_member_function.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/has_nested_type.h" "$(@D)/cuda/include/thrust/detail/type_traits/has_nested_type.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/has_trivial_assign.h" "$(@D)/cuda/include/thrust/detail/type_traits/has_trivial_assign.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/is_call_possible.h" "$(@D)/cuda/include/thrust/detail/type_traits/is_call_possible.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/is_metafunction_defined.h" "$(@D)/cuda/include/thrust/detail/type_traits/is_metafunction_defined.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/iterator/is_discard_iterator.h" "$(@D)/cuda/include/thrust/detail/type_traits/iterator/is_discard_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/iterator/is_output_iterator.h" "$(@D)/cuda/include/thrust/detail/type_traits/iterator/is_output_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/minimum_type.h" "$(@D)/cuda/include/thrust/detail/type_traits/minimum_type.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/pointer_traits.h" "$(@D)/cuda/include/thrust/detail/type_traits/pointer_traits.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/result_of_adaptable_function.h" "$(@D)/cuda/include/thrust/detail/type_traits/result_of_adaptable_function.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/uninitialized_copy.inl" "$(@D)/cuda/include/thrust/detail/uninitialized_copy.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/uninitialized_fill.inl" "$(@D)/cuda/include/thrust/detail/uninitialized_fill.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/unique.inl" "$(@D)/cuda/include/thrust/detail/unique.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/use_default.h" "$(@D)/cuda/include/thrust/detail/use_default.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/util/align.h" "$(@D)/cuda/include/thrust/detail/util/align.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/util/blocking.h" "$(@D)/cuda/include/thrust/detail/util/blocking.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/vector_base.h" "$(@D)/cuda/include/thrust/detail/vector_base.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/vector_base.inl" "$(@D)/cuda/include/thrust/detail/vector_base.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/device_allocator.h" "$(@D)/cuda/include/thrust/device_allocator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/device_delete.h" "$(@D)/cuda/include/thrust/device_delete.h" && cp -f "/usr/local/cuda-10.0/include/thrust/device_free.h" "$(@D)/cuda/include/thrust/device_free.h" && cp -f "/usr/local/cuda-10.0/include/thrust/device_malloc.h" "$(@D)/cuda/include/thrust/device_malloc.h" && cp -f "/usr/local/cuda-10.0/include/thrust/device_malloc_allocator.h" "$(@D)/cuda/include/thrust/device_malloc_allocator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/device_new.h" "$(@D)/cuda/include/thrust/device_new.h" && cp -f "/usr/local/cuda-10.0/include/thrust/device_new_allocator.h" "$(@D)/cuda/include/thrust/device_new_allocator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/device_ptr.h" "$(@D)/cuda/include/thrust/device_ptr.h" && cp -f "/usr/local/cuda-10.0/include/thrust/device_reference.h" "$(@D)/cuda/include/thrust/device_reference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/device_vector.h" "$(@D)/cuda/include/thrust/device_vector.h" && cp -f "/usr/local/cuda-10.0/include/thrust/distance.h" "$(@D)/cuda/include/thrust/distance.h" && cp -f "/usr/local/cuda-10.0/include/thrust/equal.h" "$(@D)/cuda/include/thrust/equal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/execution_policy.h" "$(@D)/cuda/include/thrust/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/extrema.h" "$(@D)/cuda/include/thrust/extrema.h" && cp -f "/usr/local/cuda-10.0/include/thrust/fill.h" "$(@D)/cuda/include/thrust/fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/find.h" "$(@D)/cuda/include/thrust/find.h" && cp -f "/usr/local/cuda-10.0/include/thrust/for_each.h" "$(@D)/cuda/include/thrust/for_each.h" && cp -f "/usr/local/cuda-10.0/include/thrust/functional.h" "$(@D)/cuda/include/thrust/functional.h" && cp -f "/usr/local/cuda-10.0/include/thrust/gather.h" "$(@D)/cuda/include/thrust/gather.h" && cp -f "/usr/local/cuda-10.0/include/thrust/generate.h" "$(@D)/cuda/include/thrust/generate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/host_vector.h" "$(@D)/cuda/include/thrust/host_vector.h" && cp -f "/usr/local/cuda-10.0/include/thrust/inner_product.h" "$(@D)/cuda/include/thrust/inner_product.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/constant_iterator.h" "$(@D)/cuda/include/thrust/iterator/constant_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/counting_iterator.h" "$(@D)/cuda/include/thrust/iterator/counting_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/any_assign.h" "$(@D)/cuda/include/thrust/iterator/detail/any_assign.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/any_system_tag.h" "$(@D)/cuda/include/thrust/iterator/detail/any_system_tag.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/constant_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/constant_iterator_base.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/counting_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/counting_iterator.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/device_system_tag.h" "$(@D)/cuda/include/thrust/iterator/detail/device_system_tag.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/discard_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/discard_iterator_base.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/distance_from_result.h" "$(@D)/cuda/include/thrust/iterator/detail/distance_from_result.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/host_system_tag.h" "$(@D)/cuda/include/thrust/iterator/detail/host_system_tag.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/is_iterator_category.h" "$(@D)/cuda/include/thrust/iterator/detail/is_iterator_category.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/is_trivial_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/is_trivial_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/iterator_adaptor_base.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_adaptor_base.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/iterator_category_to_system.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_category_to_system.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/iterator_category_to_traversal.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_category_to_traversal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/iterator_category_with_system_and_traversal.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_category_with_system_and_traversal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/iterator_facade_category.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_facade_category.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/iterator_traits.inl" "$(@D)/cuda/include/thrust/iterator/detail/iterator_traits.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/iterator_traversal_tags.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_traversal_tags.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/join_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/join_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/minimum_category.h" "$(@D)/cuda/include/thrust/iterator/detail/minimum_category.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/minimum_system.h" "$(@D)/cuda/include/thrust/iterator/detail/minimum_system.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/normal_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/normal_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/permutation_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/permutation_iterator_base.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/retag.h" "$(@D)/cuda/include/thrust/iterator/detail/retag.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/reverse_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/reverse_iterator.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/reverse_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/reverse_iterator_base.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/tagged_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/tagged_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/transform_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/transform_iterator.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/transform_output_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/transform_output_iterator.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/tuple_of_iterator_references.h" "$(@D)/cuda/include/thrust/iterator/detail/tuple_of_iterator_references.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/universal_categories.h" "$(@D)/cuda/include/thrust/iterator/detail/universal_categories.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/zip_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/zip_iterator.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/zip_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/zip_iterator_base.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/discard_iterator.h" "$(@D)/cuda/include/thrust/iterator/discard_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/iterator_adaptor.h" "$(@D)/cuda/include/thrust/iterator/iterator_adaptor.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/iterator_categories.h" "$(@D)/cuda/include/thrust/iterator/iterator_categories.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/iterator_facade.h" "$(@D)/cuda/include/thrust/iterator/iterator_facade.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/iterator_traits.h" "$(@D)/cuda/include/thrust/iterator/iterator_traits.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/permutation_iterator.h" "$(@D)/cuda/include/thrust/iterator/permutation_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/retag.h" "$(@D)/cuda/include/thrust/iterator/retag.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/reverse_iterator.h" "$(@D)/cuda/include/thrust/iterator/reverse_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/transform_iterator.h" "$(@D)/cuda/include/thrust/iterator/transform_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/transform_output_iterator.h" "$(@D)/cuda/include/thrust/iterator/transform_output_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/zip_iterator.h" "$(@D)/cuda/include/thrust/iterator/zip_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/logical.h" "$(@D)/cuda/include/thrust/logical.h" && cp -f "/usr/local/cuda-10.0/include/thrust/memory.h" "$(@D)/cuda/include/thrust/memory.h" && cp -f "/usr/local/cuda-10.0/include/thrust/merge.h" "$(@D)/cuda/include/thrust/merge.h" && cp -f "/usr/local/cuda-10.0/include/thrust/mismatch.h" "$(@D)/cuda/include/thrust/mismatch.h" && cp -f "/usr/local/cuda-10.0/include/thrust/pair.h" "$(@D)/cuda/include/thrust/pair.h" && cp -f "/usr/local/cuda-10.0/include/thrust/partition.h" "$(@D)/cuda/include/thrust/partition.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random.h" "$(@D)/cuda/include/thrust/random.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/discard_block_engine.inl" "$(@D)/cuda/include/thrust/random/detail/discard_block_engine.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/linear_congruential_engine.inl" "$(@D)/cuda/include/thrust/random/detail/linear_congruential_engine.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/linear_congruential_engine_discard.h" "$(@D)/cuda/include/thrust/random/detail/linear_congruential_engine_discard.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/linear_feedback_shift_engine.inl" "$(@D)/cuda/include/thrust/random/detail/linear_feedback_shift_engine.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/linear_feedback_shift_engine_wordmask.h" "$(@D)/cuda/include/thrust/random/detail/linear_feedback_shift_engine_wordmask.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/mod.h" "$(@D)/cuda/include/thrust/random/detail/mod.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/normal_distribution.inl" "$(@D)/cuda/include/thrust/random/detail/normal_distribution.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/normal_distribution_base.h" "$(@D)/cuda/include/thrust/random/detail/normal_distribution_base.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/random_core_access.h" "$(@D)/cuda/include/thrust/random/detail/random_core_access.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/subtract_with_carry_engine.inl" "$(@D)/cuda/include/thrust/random/detail/subtract_with_carry_engine.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/uniform_int_distribution.inl" "$(@D)/cuda/include/thrust/random/detail/uniform_int_distribution.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/uniform_real_distribution.inl" "$(@D)/cuda/include/thrust/random/detail/uniform_real_distribution.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/xor_combine_engine.inl" "$(@D)/cuda/include/thrust/random/detail/xor_combine_engine.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/xor_combine_engine_max.h" "$(@D)/cuda/include/thrust/random/detail/xor_combine_engine_max.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/discard_block_engine.h" "$(@D)/cuda/include/thrust/random/discard_block_engine.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/linear_congruential_engine.h" "$(@D)/cuda/include/thrust/random/linear_congruential_engine.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/linear_feedback_shift_engine.h" "$(@D)/cuda/include/thrust/random/linear_feedback_shift_engine.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/normal_distribution.h" "$(@D)/cuda/include/thrust/random/normal_distribution.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/subtract_with_carry_engine.h" "$(@D)/cuda/include/thrust/random/subtract_with_carry_engine.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/uniform_int_distribution.h" "$(@D)/cuda/include/thrust/random/uniform_int_distribution.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/uniform_real_distribution.h" "$(@D)/cuda/include/thrust/random/uniform_real_distribution.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/xor_combine_engine.h" "$(@D)/cuda/include/thrust/random/xor_combine_engine.h" && cp -f "/usr/local/cuda-10.0/include/thrust/reduce.h" "$(@D)/cuda/include/thrust/reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/remove.h" "$(@D)/cuda/include/thrust/remove.h" && cp -f "/usr/local/cuda-10.0/include/thrust/replace.h" "$(@D)/cuda/include/thrust/replace.h" && cp -f "/usr/local/cuda-10.0/include/thrust/reverse.h" "$(@D)/cuda/include/thrust/reverse.h" && cp -f "/usr/local/cuda-10.0/include/thrust/scan.h" "$(@D)/cuda/include/thrust/scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/scatter.h" "$(@D)/cuda/include/thrust/scatter.h" && cp -f "/usr/local/cuda-10.0/include/thrust/sequence.h" "$(@D)/cuda/include/thrust/sequence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/set_operations.h" "$(@D)/cuda/include/thrust/set_operations.h" && cp -f "/usr/local/cuda-10.0/include/thrust/sort.h" "$(@D)/cuda/include/thrust/sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/swap.h" "$(@D)/cuda/include/thrust/swap.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/cpp/detail/adjacent_difference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/cpp/detail/assign_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/cpp/detail/binary_search.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/copy.h" "$(@D)/cuda/include/thrust/system/cpp/detail/copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/cpp/detail/copy_if.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/count.h" "$(@D)/cuda/include/thrust/system/cpp/detail/count.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/equal.h" "$(@D)/cuda/include/thrust/system/cpp/detail/equal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/cpp/detail/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/extrema.h" "$(@D)/cuda/include/thrust/system/cpp/detail/extrema.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/fill.h" "$(@D)/cuda/include/thrust/system/cpp/detail/fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/find.h" "$(@D)/cuda/include/thrust/system/cpp/detail/find.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/for_each.h" "$(@D)/cuda/include/thrust/system/cpp/detail/for_each.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/gather.h" "$(@D)/cuda/include/thrust/system/cpp/detail/gather.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/generate.h" "$(@D)/cuda/include/thrust/system/cpp/detail/generate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/get_value.h" "$(@D)/cuda/include/thrust/system/cpp/detail/get_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/cpp/detail/inner_product.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/cpp/detail/iter_swap.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/logical.h" "$(@D)/cuda/include/thrust/system/cpp/detail/logical.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/cpp/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/memory.inl" "$(@D)/cuda/include/thrust/system/cpp/detail/memory.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/merge.h" "$(@D)/cuda/include/thrust/system/cpp/detail/merge.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/cpp/detail/mismatch.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/par.h" "$(@D)/cuda/include/thrust/system/cpp/detail/par.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/partition.h" "$(@D)/cuda/include/thrust/system/cpp/detail/partition.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/reduce.h" "$(@D)/cuda/include/thrust/system/cpp/detail/reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/cpp/detail/reduce_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/remove.h" "$(@D)/cuda/include/thrust/system/cpp/detail/remove.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/replace.h" "$(@D)/cuda/include/thrust/system/cpp/detail/replace.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/reverse.h" "$(@D)/cuda/include/thrust/system/cpp/detail/reverse.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/scan.h" "$(@D)/cuda/include/thrust/system/cpp/detail/scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/cpp/detail/scan_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/scatter.h" "$(@D)/cuda/include/thrust/system/cpp/detail/scatter.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/sequence.h" "$(@D)/cuda/include/thrust/system/cpp/detail/sequence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/cpp/detail/set_operations.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/sort.h" "$(@D)/cuda/include/thrust/system/cpp/detail/sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/cpp/detail/swap_ranges.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/cpp/detail/tabulate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/cpp/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/transform.h" "$(@D)/cuda/include/thrust/system/cpp/detail/transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/cpp/detail/transform_reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/cpp/detail/transform_scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/cpp/detail/uninitialized_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/cpp/detail/uninitialized_fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/unique.h" "$(@D)/cuda/include/thrust/system/cpp/detail/unique.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/cpp/detail/unique_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/vector.inl" "$(@D)/cuda/include/thrust/system/cpp/detail/vector.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/execution_policy.h" "$(@D)/cuda/include/thrust/system/cpp/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/memory.h" "$(@D)/cuda/include/thrust/system/cpp/memory.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/vector.h" "$(@D)/cuda/include/thrust/system/cpp/vector.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/config.h" "$(@D)/cuda/include/thrust/system/cuda/config.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/cuda/detail/adjacent_difference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/cuda/detail/assign_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/cuda/detail/binary_search.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/copy.h" "$(@D)/cuda/include/thrust/system/cuda/detail/copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/cuda/detail/copy_if.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/core/agent_launcher.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/agent_launcher.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/core/alignment.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/alignment.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/core/triple_chevron_launch.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/triple_chevron_launch.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/core/util.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/util.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/count.h" "$(@D)/cuda/include/thrust/system/cuda/detail/count.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cross_system.h" "$(@D)/cuda/include/thrust/system/cuda/detail/cross_system.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_histogram.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_downsweep.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_downsweep.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_upsweep.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_upsweep.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_reduce.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_reduce_by_key.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_reduce_by_key.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_rle.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_rle.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_scan.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_segment_fixup.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_segment_fixup.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_select_if.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_select_if.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_spmv_orig.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_spmv_orig.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/single_pass_scan_operators.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/single_pass_scan_operators.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_adjacent_difference.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_adjacent_difference.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_discontinuity.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_discontinuity.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_exchange.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_exchange.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_histogram.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_load.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_load.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_radix_rank.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_radix_rank.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_radix_sort.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_raking_layout.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_raking_layout.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_reduce.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_scan.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_shuffle.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_shuffle.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_store.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_store.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_atomic.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_atomic.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_sort.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking_commutative_only.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking_commutative_only.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_warp_reductions.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_warp_reductions.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_raking.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_raking.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans2.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans2.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans3.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans3.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/cub.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/cub.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_histogram.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_partition.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_partition.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_radix_sort.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_reduce.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_run_length_encode.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_run_length_encode.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_scan.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_segmented_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_segmented_radix_sort.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_segmented_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_segmented_reduce.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_select.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_select.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_spmv.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_spmv.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_histogram.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_radix_sort.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce_by_key.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce_by_key.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_rle.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_rle.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_scan.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_select_if.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_select_if.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_orig.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_orig.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/grid/grid_barrier.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_barrier.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/grid/grid_even_share.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_even_share.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/grid/grid_mapping.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_mapping.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/grid/grid_queue.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_queue.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/host/mutex.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/host/mutex.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/arg_index_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/arg_index_input_iterator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/cache_modified_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/cache_modified_input_iterator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/cache_modified_output_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/cache_modified_output_iterator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/constant_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/constant_input_iterator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/counting_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/counting_input_iterator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/discard_output_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/discard_output_iterator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/tex_obj_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/tex_obj_input_iterator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/tex_ref_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/tex_ref_input_iterator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/transform_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/transform_input_iterator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/thread/thread_load.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_load.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/thread/thread_operators.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_operators.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/thread/thread_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_reduce.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/thread/thread_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_scan.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/thread/thread_search.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_search.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/thread/thread_store.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_store.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_allocator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_allocator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_arch.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_arch.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_debug.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_debug.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_device.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_device.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_macro.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_macro.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_namespace.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_namespace.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_ptx.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_ptx.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_type.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_type.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_shfl.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_shfl.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_smem.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_smem.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_shfl.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_shfl.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_smem.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_smem.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/warp/warp_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/warp_reduce.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/warp/warp_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/warp_scan.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/equal.h" "$(@D)/cuda/include/thrust/system/cuda/detail/equal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/error.inl" "$(@D)/cuda/include/thrust/system/cuda/detail/error.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/cuda/detail/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/extrema.h" "$(@D)/cuda/include/thrust/system/cuda/detail/extrema.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/fill.h" "$(@D)/cuda/include/thrust/system/cuda/detail/fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/find.h" "$(@D)/cuda/include/thrust/system/cuda/detail/find.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/for_each.h" "$(@D)/cuda/include/thrust/system/cuda/detail/for_each.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/gather.h" "$(@D)/cuda/include/thrust/system/cuda/detail/gather.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/generate.h" "$(@D)/cuda/include/thrust/system/cuda/detail/generate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/get_value.h" "$(@D)/cuda/include/thrust/system/cuda/detail/get_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/guarded_cuda_runtime_api.h" "$(@D)/cuda/include/thrust/system/cuda/detail/guarded_cuda_runtime_api.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/guarded_driver_types.h" "$(@D)/cuda/include/thrust/system/cuda/detail/guarded_driver_types.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/cuda/detail/inner_product.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/internal/copy_cross_system.h" "$(@D)/cuda/include/thrust/system/cuda/detail/internal/copy_cross_system.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/internal/copy_device_to_device.h" "$(@D)/cuda/include/thrust/system/cuda/detail/internal/copy_device_to_device.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/cuda/detail/iter_swap.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/logical.h" "$(@D)/cuda/include/thrust/system/cuda/detail/logical.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/cuda/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/memory.inl" "$(@D)/cuda/include/thrust/system/cuda/detail/memory.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/merge.h" "$(@D)/cuda/include/thrust/system/cuda/detail/merge.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/cuda/detail/mismatch.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/par.h" "$(@D)/cuda/include/thrust/system/cuda/detail/par.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/par_to_seq.h" "$(@D)/cuda/include/thrust/system/cuda/detail/par_to_seq.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/parallel_for.h" "$(@D)/cuda/include/thrust/system/cuda/detail/parallel_for.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/partition.h" "$(@D)/cuda/include/thrust/system/cuda/detail/partition.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/reduce.h" "$(@D)/cuda/include/thrust/system/cuda/detail/reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/cuda/detail/reduce_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/remove.h" "$(@D)/cuda/include/thrust/system/cuda/detail/remove.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/replace.h" "$(@D)/cuda/include/thrust/system/cuda/detail/replace.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/reverse.h" "$(@D)/cuda/include/thrust/system/cuda/detail/reverse.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/scan.h" "$(@D)/cuda/include/thrust/system/cuda/detail/scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/cuda/detail/scan_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/scatter.h" "$(@D)/cuda/include/thrust/system/cuda/detail/scatter.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/sequence.h" "$(@D)/cuda/include/thrust/system/cuda/detail/sequence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/cuda/detail/set_operations.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/sort.h" "$(@D)/cuda/include/thrust/system/cuda/detail/sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/cuda/detail/swap_ranges.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/cuda/detail/tabulate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/cuda/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/terminate.h" "$(@D)/cuda/include/thrust/system/cuda/detail/terminate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/transform.h" "$(@D)/cuda/include/thrust/system/cuda/detail/transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/cuda/detail/transform_reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/cuda/detail/transform_scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/cuda/detail/uninitialized_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/cuda/detail/uninitialized_fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/unique.h" "$(@D)/cuda/include/thrust/system/cuda/detail/unique.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/cuda/detail/unique_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/util.h" "$(@D)/cuda/include/thrust/system/cuda/detail/util.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/vector.inl" "$(@D)/cuda/include/thrust/system/cuda/detail/vector.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/error.h" "$(@D)/cuda/include/thrust/system/cuda/error.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/execution_policy.h" "$(@D)/cuda/include/thrust/system/cuda/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/experimental/pinned_allocator.h" "$(@D)/cuda/include/thrust/system/cuda/experimental/pinned_allocator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/memory.h" "$(@D)/cuda/include/thrust/system/cuda/memory.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/vector.h" "$(@D)/cuda/include/thrust/system/cuda/vector.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/detail/adl/adjacent_difference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/assign_value.h" "$(@D)/cuda/include/thrust/system/detail/adl/assign_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/adl/binary_search.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/copy.h" "$(@D)/cuda/include/thrust/system/detail/adl/copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/copy_if.h" "$(@D)/cuda/include/thrust/system/detail/adl/copy_if.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/count.h" "$(@D)/cuda/include/thrust/system/detail/adl/count.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/equal.h" "$(@D)/cuda/include/thrust/system/detail/adl/equal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/extrema.h" "$(@D)/cuda/include/thrust/system/detail/adl/extrema.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/fill.h" "$(@D)/cuda/include/thrust/system/detail/adl/fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/find.h" "$(@D)/cuda/include/thrust/system/detail/adl/find.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/for_each.h" "$(@D)/cuda/include/thrust/system/detail/adl/for_each.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/gather.h" "$(@D)/cuda/include/thrust/system/detail/adl/gather.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/generate.h" "$(@D)/cuda/include/thrust/system/detail/adl/generate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/get_value.h" "$(@D)/cuda/include/thrust/system/detail/adl/get_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/inner_product.h" "$(@D)/cuda/include/thrust/system/detail/adl/inner_product.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/iter_swap.h" "$(@D)/cuda/include/thrust/system/detail/adl/iter_swap.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/logical.h" "$(@D)/cuda/include/thrust/system/detail/adl/logical.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/detail/adl/malloc_and_free.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/merge.h" "$(@D)/cuda/include/thrust/system/detail/adl/merge.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/mismatch.h" "$(@D)/cuda/include/thrust/system/detail/adl/mismatch.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/partition.h" "$(@D)/cuda/include/thrust/system/detail/adl/partition.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/reduce.h" "$(@D)/cuda/include/thrust/system/detail/adl/reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/detail/adl/reduce_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/remove.h" "$(@D)/cuda/include/thrust/system/detail/adl/remove.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/replace.h" "$(@D)/cuda/include/thrust/system/detail/adl/replace.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/reverse.h" "$(@D)/cuda/include/thrust/system/detail/adl/reverse.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/scan.h" "$(@D)/cuda/include/thrust/system/detail/adl/scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/scan_by_key.h" "$(@D)/cuda/include/thrust/system/detail/adl/scan_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/scatter.h" "$(@D)/cuda/include/thrust/system/detail/adl/scatter.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/sequence.h" "$(@D)/cuda/include/thrust/system/detail/adl/sequence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/set_operations.h" "$(@D)/cuda/include/thrust/system/detail/adl/set_operations.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/sort.h" "$(@D)/cuda/include/thrust/system/detail/adl/sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/swap_ranges.h" "$(@D)/cuda/include/thrust/system/detail/adl/swap_ranges.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/tabulate.h" "$(@D)/cuda/include/thrust/system/detail/adl/tabulate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/detail/adl/temporary_buffer.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/transform.h" "$(@D)/cuda/include/thrust/system/detail/adl/transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/transform_reduce.h" "$(@D)/cuda/include/thrust/system/detail/adl/transform_reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/transform_scan.h" "$(@D)/cuda/include/thrust/system/detail/adl/transform_scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/detail/adl/uninitialized_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/detail/adl/uninitialized_fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/unique.h" "$(@D)/cuda/include/thrust/system/detail/adl/unique.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/unique_by_key.h" "$(@D)/cuda/include/thrust/system/detail/adl/unique_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/bad_alloc.h" "$(@D)/cuda/include/thrust/system/detail/bad_alloc.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/errno.h" "$(@D)/cuda/include/thrust/system/detail/errno.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/error_category.inl" "$(@D)/cuda/include/thrust/system/detail/error_category.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/error_code.inl" "$(@D)/cuda/include/thrust/system/detail/error_code.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/error_condition.inl" "$(@D)/cuda/include/thrust/system/detail/error_condition.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/detail/generic/adjacent_difference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/adjacent_difference.inl" "$(@D)/cuda/include/thrust/system/detail/generic/adjacent_difference.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/advance.h" "$(@D)/cuda/include/thrust/system/detail/generic/advance.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/advance.inl" "$(@D)/cuda/include/thrust/system/detail/generic/advance.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/generic/binary_search.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/binary_search.inl" "$(@D)/cuda/include/thrust/system/detail/generic/binary_search.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/copy.h" "$(@D)/cuda/include/thrust/system/detail/generic/copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/copy.inl" "$(@D)/cuda/include/thrust/system/detail/generic/copy.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/copy_if.h" "$(@D)/cuda/include/thrust/system/detail/generic/copy_if.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/copy_if.inl" "$(@D)/cuda/include/thrust/system/detail/generic/copy_if.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/count.h" "$(@D)/cuda/include/thrust/system/detail/generic/count.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/count.inl" "$(@D)/cuda/include/thrust/system/detail/generic/count.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/distance.h" "$(@D)/cuda/include/thrust/system/detail/generic/distance.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/distance.inl" "$(@D)/cuda/include/thrust/system/detail/generic/distance.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/equal.h" "$(@D)/cuda/include/thrust/system/detail/generic/equal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/equal.inl" "$(@D)/cuda/include/thrust/system/detail/generic/equal.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/extrema.h" "$(@D)/cuda/include/thrust/system/detail/generic/extrema.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/extrema.inl" "$(@D)/cuda/include/thrust/system/detail/generic/extrema.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/fill.h" "$(@D)/cuda/include/thrust/system/detail/generic/fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/find.h" "$(@D)/cuda/include/thrust/system/detail/generic/find.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/find.inl" "$(@D)/cuda/include/thrust/system/detail/generic/find.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/for_each.h" "$(@D)/cuda/include/thrust/system/detail/generic/for_each.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/gather.h" "$(@D)/cuda/include/thrust/system/detail/generic/gather.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/gather.inl" "$(@D)/cuda/include/thrust/system/detail/generic/gather.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/generate.h" "$(@D)/cuda/include/thrust/system/detail/generic/generate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/generate.inl" "$(@D)/cuda/include/thrust/system/detail/generic/generate.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/inner_product.h" "$(@D)/cuda/include/thrust/system/detail/generic/inner_product.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/inner_product.inl" "$(@D)/cuda/include/thrust/system/detail/generic/inner_product.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/logical.h" "$(@D)/cuda/include/thrust/system/detail/generic/logical.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/memory.h" "$(@D)/cuda/include/thrust/system/detail/generic/memory.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/memory.inl" "$(@D)/cuda/include/thrust/system/detail/generic/memory.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/merge.h" "$(@D)/cuda/include/thrust/system/detail/generic/merge.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/merge.inl" "$(@D)/cuda/include/thrust/system/detail/generic/merge.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/mismatch.h" "$(@D)/cuda/include/thrust/system/detail/generic/mismatch.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/mismatch.inl" "$(@D)/cuda/include/thrust/system/detail/generic/mismatch.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/partition.h" "$(@D)/cuda/include/thrust/system/detail/generic/partition.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/partition.inl" "$(@D)/cuda/include/thrust/system/detail/generic/partition.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/reduce.h" "$(@D)/cuda/include/thrust/system/detail/generic/reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/reduce.inl" "$(@D)/cuda/include/thrust/system/detail/generic/reduce.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/detail/generic/reduce_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/reduce_by_key.inl" "$(@D)/cuda/include/thrust/system/detail/generic/reduce_by_key.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/remove.h" "$(@D)/cuda/include/thrust/system/detail/generic/remove.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/remove.inl" "$(@D)/cuda/include/thrust/system/detail/generic/remove.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/replace.h" "$(@D)/cuda/include/thrust/system/detail/generic/replace.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/replace.inl" "$(@D)/cuda/include/thrust/system/detail/generic/replace.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/reverse.h" "$(@D)/cuda/include/thrust/system/detail/generic/reverse.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/reverse.inl" "$(@D)/cuda/include/thrust/system/detail/generic/reverse.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scalar/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/generic/scalar/binary_search.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scalar/binary_search.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scalar/binary_search.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scan.h" "$(@D)/cuda/include/thrust/system/detail/generic/scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scan.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scan.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scan_by_key.h" "$(@D)/cuda/include/thrust/system/detail/generic/scan_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scan_by_key.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scan_by_key.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scatter.h" "$(@D)/cuda/include/thrust/system/detail/generic/scatter.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scatter.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scatter.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/select_system.h" "$(@D)/cuda/include/thrust/system/detail/generic/select_system.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/sequence.h" "$(@D)/cuda/include/thrust/system/detail/generic/sequence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/sequence.inl" "$(@D)/cuda/include/thrust/system/detail/generic/sequence.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/set_operations.h" "$(@D)/cuda/include/thrust/system/detail/generic/set_operations.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/set_operations.inl" "$(@D)/cuda/include/thrust/system/detail/generic/set_operations.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/sort.h" "$(@D)/cuda/include/thrust/system/detail/generic/sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/sort.inl" "$(@D)/cuda/include/thrust/system/detail/generic/sort.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/swap_ranges.h" "$(@D)/cuda/include/thrust/system/detail/generic/swap_ranges.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/swap_ranges.inl" "$(@D)/cuda/include/thrust/system/detail/generic/swap_ranges.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/tabulate.h" "$(@D)/cuda/include/thrust/system/detail/generic/tabulate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/tabulate.inl" "$(@D)/cuda/include/thrust/system/detail/generic/tabulate.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/tag.h" "$(@D)/cuda/include/thrust/system/detail/generic/tag.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/detail/generic/temporary_buffer.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/temporary_buffer.inl" "$(@D)/cuda/include/thrust/system/detail/generic/temporary_buffer.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/transform.h" "$(@D)/cuda/include/thrust/system/detail/generic/transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/transform.inl" "$(@D)/cuda/include/thrust/system/detail/generic/transform.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/transform_reduce.h" "$(@D)/cuda/include/thrust/system/detail/generic/transform_reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/transform_reduce.inl" "$(@D)/cuda/include/thrust/system/detail/generic/transform_reduce.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/transform_scan.h" "$(@D)/cuda/include/thrust/system/detail/generic/transform_scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/transform_scan.inl" "$(@D)/cuda/include/thrust/system/detail/generic/transform_scan.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/type_traits.h" "$(@D)/cuda/include/thrust/system/detail/generic/type_traits.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/uninitialized_copy.inl" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_copy.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/uninitialized_fill.inl" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_fill.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/unique.h" "$(@D)/cuda/include/thrust/system/detail/generic/unique.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/unique.inl" "$(@D)/cuda/include/thrust/system/detail/generic/unique.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/unique_by_key.h" "$(@D)/cuda/include/thrust/system/detail/generic/unique_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/unique_by_key.inl" "$(@D)/cuda/include/thrust/system/detail/generic/unique_by_key.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/internal/decompose.h" "$(@D)/cuda/include/thrust/system/detail/internal/decompose.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/detail/sequential/adjacent_difference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/assign_value.h" "$(@D)/cuda/include/thrust/system/detail/sequential/assign_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/sequential/binary_search.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/copy.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/copy.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/copy_backward.h" "$(@D)/cuda/include/thrust/system/detail/sequential/copy_backward.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/copy_if.h" "$(@D)/cuda/include/thrust/system/detail/sequential/copy_if.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/count.h" "$(@D)/cuda/include/thrust/system/detail/sequential/count.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/equal.h" "$(@D)/cuda/include/thrust/system/detail/sequential/equal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/execution_policy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/extrema.h" "$(@D)/cuda/include/thrust/system/detail/sequential/extrema.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/fill.h" "$(@D)/cuda/include/thrust/system/detail/sequential/fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/find.h" "$(@D)/cuda/include/thrust/system/detail/sequential/find.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/for_each.h" "$(@D)/cuda/include/thrust/system/detail/sequential/for_each.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/gather.h" "$(@D)/cuda/include/thrust/system/detail/sequential/gather.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/general_copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/general_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/generate.h" "$(@D)/cuda/include/thrust/system/detail/sequential/generate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/get_value.h" "$(@D)/cuda/include/thrust/system/detail/sequential/get_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/inner_product.h" "$(@D)/cuda/include/thrust/system/detail/sequential/inner_product.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/insertion_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/insertion_sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/iter_swap.h" "$(@D)/cuda/include/thrust/system/detail/sequential/iter_swap.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/logical.h" "$(@D)/cuda/include/thrust/system/detail/sequential/logical.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/detail/sequential/malloc_and_free.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/merge.h" "$(@D)/cuda/include/thrust/system/detail/sequential/merge.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/merge.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/merge.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/mismatch.h" "$(@D)/cuda/include/thrust/system/detail/sequential/mismatch.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/partition.h" "$(@D)/cuda/include/thrust/system/detail/sequential/partition.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/reduce.h" "$(@D)/cuda/include/thrust/system/detail/sequential/reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/detail/sequential/reduce_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/remove.h" "$(@D)/cuda/include/thrust/system/detail/sequential/remove.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/replace.h" "$(@D)/cuda/include/thrust/system/detail/sequential/replace.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/reverse.h" "$(@D)/cuda/include/thrust/system/detail/sequential/reverse.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/scan.h" "$(@D)/cuda/include/thrust/system/detail/sequential/scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/scan_by_key.h" "$(@D)/cuda/include/thrust/system/detail/sequential/scan_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/scatter.h" "$(@D)/cuda/include/thrust/system/detail/sequential/scatter.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/sequence.h" "$(@D)/cuda/include/thrust/system/detail/sequential/sequence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/set_operations.h" "$(@D)/cuda/include/thrust/system/detail/sequential/set_operations.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/sort.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/stable_merge_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_merge_sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/stable_merge_sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_merge_sort.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/stable_primitive_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_primitive_sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/stable_primitive_sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_primitive_sort.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/stable_radix_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_radix_sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/stable_radix_sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_radix_sort.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/swap_ranges.h" "$(@D)/cuda/include/thrust/system/detail/sequential/swap_ranges.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/tabulate.h" "$(@D)/cuda/include/thrust/system/detail/sequential/tabulate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/detail/sequential/temporary_buffer.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/transform.h" "$(@D)/cuda/include/thrust/system/detail/sequential/transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/transform_reduce.h" "$(@D)/cuda/include/thrust/system/detail/sequential/transform_reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/transform_scan.h" "$(@D)/cuda/include/thrust/system/detail/sequential/transform_scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/trivial_copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/trivial_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/uninitialized_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/detail/sequential/uninitialized_fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/unique.h" "$(@D)/cuda/include/thrust/system/detail/sequential/unique.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/unique_by_key.h" "$(@D)/cuda/include/thrust/system/detail/sequential/unique_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/system_error.inl" "$(@D)/cuda/include/thrust/system/detail/system_error.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/error_code.h" "$(@D)/cuda/include/thrust/system/error_code.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/omp/detail/adjacent_difference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/omp/detail/assign_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/omp/detail/binary_search.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/copy.h" "$(@D)/cuda/include/thrust/system/omp/detail/copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/copy.inl" "$(@D)/cuda/include/thrust/system/omp/detail/copy.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/omp/detail/copy_if.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/copy_if.inl" "$(@D)/cuda/include/thrust/system/omp/detail/copy_if.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/count.h" "$(@D)/cuda/include/thrust/system/omp/detail/count.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/default_decomposition.h" "$(@D)/cuda/include/thrust/system/omp/detail/default_decomposition.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/default_decomposition.inl" "$(@D)/cuda/include/thrust/system/omp/detail/default_decomposition.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/equal.h" "$(@D)/cuda/include/thrust/system/omp/detail/equal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/omp/detail/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/extrema.h" "$(@D)/cuda/include/thrust/system/omp/detail/extrema.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/fill.h" "$(@D)/cuda/include/thrust/system/omp/detail/fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/find.h" "$(@D)/cuda/include/thrust/system/omp/detail/find.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/for_each.h" "$(@D)/cuda/include/thrust/system/omp/detail/for_each.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/for_each.inl" "$(@D)/cuda/include/thrust/system/omp/detail/for_each.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/gather.h" "$(@D)/cuda/include/thrust/system/omp/detail/gather.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/generate.h" "$(@D)/cuda/include/thrust/system/omp/detail/generate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/get_value.h" "$(@D)/cuda/include/thrust/system/omp/detail/get_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/omp/detail/inner_product.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/omp/detail/iter_swap.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/logical.h" "$(@D)/cuda/include/thrust/system/omp/detail/logical.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/omp/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/memory.inl" "$(@D)/cuda/include/thrust/system/omp/detail/memory.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/merge.h" "$(@D)/cuda/include/thrust/system/omp/detail/merge.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/omp/detail/mismatch.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/par.h" "$(@D)/cuda/include/thrust/system/omp/detail/par.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/partition.h" "$(@D)/cuda/include/thrust/system/omp/detail/partition.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/partition.inl" "$(@D)/cuda/include/thrust/system/omp/detail/partition.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/reduce.h" "$(@D)/cuda/include/thrust/system/omp/detail/reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/reduce.inl" "$(@D)/cuda/include/thrust/system/omp/detail/reduce.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/reduce_by_key.inl" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_by_key.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/reduce_intervals.h" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_intervals.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/reduce_intervals.inl" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_intervals.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/remove.h" "$(@D)/cuda/include/thrust/system/omp/detail/remove.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/remove.inl" "$(@D)/cuda/include/thrust/system/omp/detail/remove.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/replace.h" "$(@D)/cuda/include/thrust/system/omp/detail/replace.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/reverse.h" "$(@D)/cuda/include/thrust/system/omp/detail/reverse.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/scan.h" "$(@D)/cuda/include/thrust/system/omp/detail/scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/omp/detail/scan_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/scatter.h" "$(@D)/cuda/include/thrust/system/omp/detail/scatter.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/sequence.h" "$(@D)/cuda/include/thrust/system/omp/detail/sequence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/omp/detail/set_operations.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/sort.h" "$(@D)/cuda/include/thrust/system/omp/detail/sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/sort.inl" "$(@D)/cuda/include/thrust/system/omp/detail/sort.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/omp/detail/swap_ranges.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/omp/detail/tabulate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/omp/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/transform.h" "$(@D)/cuda/include/thrust/system/omp/detail/transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/omp/detail/transform_reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/omp/detail/transform_scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/omp/detail/uninitialized_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/omp/detail/uninitialized_fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/unique.h" "$(@D)/cuda/include/thrust/system/omp/detail/unique.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/unique.inl" "$(@D)/cuda/include/thrust/system/omp/detail/unique.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/omp/detail/unique_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/unique_by_key.inl" "$(@D)/cuda/include/thrust/system/omp/detail/unique_by_key.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/vector.inl" "$(@D)/cuda/include/thrust/system/omp/detail/vector.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/execution_policy.h" "$(@D)/cuda/include/thrust/system/omp/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/memory.h" "$(@D)/cuda/include/thrust/system/omp/memory.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/vector.h" "$(@D)/cuda/include/thrust/system/omp/vector.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/system_error.h" "$(@D)/cuda/include/thrust/system/system_error.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/tbb/detail/adjacent_difference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/tbb/detail/assign_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/tbb/detail/binary_search.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/copy.h" "$(@D)/cuda/include/thrust/system/tbb/detail/copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/copy.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/copy.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/tbb/detail/copy_if.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/copy_if.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/copy_if.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/count.h" "$(@D)/cuda/include/thrust/system/tbb/detail/count.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/equal.h" "$(@D)/cuda/include/thrust/system/tbb/detail/equal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/tbb/detail/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/extrema.h" "$(@D)/cuda/include/thrust/system/tbb/detail/extrema.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/fill.h" "$(@D)/cuda/include/thrust/system/tbb/detail/fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/find.h" "$(@D)/cuda/include/thrust/system/tbb/detail/find.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/for_each.h" "$(@D)/cuda/include/thrust/system/tbb/detail/for_each.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/for_each.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/for_each.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/gather.h" "$(@D)/cuda/include/thrust/system/tbb/detail/gather.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/generate.h" "$(@D)/cuda/include/thrust/system/tbb/detail/generate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/get_value.h" "$(@D)/cuda/include/thrust/system/tbb/detail/get_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/tbb/detail/inner_product.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/tbb/detail/iter_swap.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/logical.h" "$(@D)/cuda/include/thrust/system/tbb/detail/logical.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/tbb/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/memory.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/memory.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/merge.h" "$(@D)/cuda/include/thrust/system/tbb/detail/merge.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/merge.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/merge.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/tbb/detail/mismatch.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/par.h" "$(@D)/cuda/include/thrust/system/tbb/detail/par.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/partition.h" "$(@D)/cuda/include/thrust/system/tbb/detail/partition.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/partition.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/partition.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/reduce.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/reduce.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/reduce_by_key.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce_by_key.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/reduce_intervals.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce_intervals.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/remove.h" "$(@D)/cuda/include/thrust/system/tbb/detail/remove.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/remove.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/remove.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/replace.h" "$(@D)/cuda/include/thrust/system/tbb/detail/replace.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/reverse.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reverse.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/scan.h" "$(@D)/cuda/include/thrust/system/tbb/detail/scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/scan.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/scan.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/tbb/detail/scan_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/scatter.h" "$(@D)/cuda/include/thrust/system/tbb/detail/scatter.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/sequence.h" "$(@D)/cuda/include/thrust/system/tbb/detail/sequence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/tbb/detail/set_operations.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/sort.h" "$(@D)/cuda/include/thrust/system/tbb/detail/sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/sort.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/sort.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/tbb/detail/swap_ranges.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/tbb/detail/tabulate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/tbb/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/transform.h" "$(@D)/cuda/include/thrust/system/tbb/detail/transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/tbb/detail/transform_reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/tbb/detail/transform_scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/tbb/detail/uninitialized_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/tbb/detail/uninitialized_fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/unique.h" "$(@D)/cuda/include/thrust/system/tbb/detail/unique.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/unique.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/unique.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/tbb/detail/unique_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/unique_by_key.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/unique_by_key.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/vector.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/vector.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/execution_policy.h" "$(@D)/cuda/include/thrust/system/tbb/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/memory.h" "$(@D)/cuda/include/thrust/system/tbb/memory.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/vector.h" "$(@D)/cuda/include/thrust/system/tbb/vector.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system_error.h" "$(@D)/cuda/include/thrust/system_error.h" && cp -f "/usr/local/cuda-10.0/include/thrust/tabulate.h" "$(@D)/cuda/include/thrust/tabulate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/transform.h" "$(@D)/cuda/include/thrust/transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/transform_reduce.h" "$(@D)/cuda/include/thrust/transform_reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/transform_scan.h" "$(@D)/cuda/include/thrust/transform_scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/tuple.h" "$(@D)/cuda/include/thrust/tuple.h" && cp -f "/usr/local/cuda-10.0/include/thrust/uninitialized_copy.h" "$(@D)/cuda/include/thrust/uninitialized_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/uninitialized_fill.h" "$(@D)/cuda/include/thrust/uninitialized_fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/unique.h" "$(@D)/cuda/include/thrust/unique.h" && cp -f "/usr/local/cuda-10.0/include/thrust/version.h" "$(@D)/cuda/include/thrust/version.h" && cp -f "/usr/local/cuda-10.0/include/vector_functions.h" "$(@D)/cuda/include/vector_functions.h" && cp -f "/usr/local/cuda-10.0/include/vector_functions.hpp" "$(@D)/cuda/include/vector_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/vector_types.h" "$(@D)/cuda/include/vector_types.h"
-   """,
+    cmd = """cp -rf "/usr/local/cuda-10.0/include/." "$(@D)/cuda/include/" """,
 )
 
 genrule(
@@ -1202,9 +1200,7 @@ genrule(
     outs = [
         "cuda/nvvm/libdevice/libdevice.10.bc",
     ],
-    cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/local/cuda-10.0/nvvm/libdevice/libdevice.10.bc" "$(@D)//libdevice.10.bc"
-   """,
+    cmd = """cp -rf "/usr/local/cuda-10.0/nvvm/libdevice/." "$(@D)/" """,
 )
 
 genrule(
@@ -1241,9 +1237,7 @@ genrule(
         "cuda/extras/CUPTI/include/openmp/cupti_openmp.h",
         "cuda/extras/CUPTI/include/openmp/ompt.h",
     ],
-    cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/GL/gl.h" "$(@D)/cuda/extras/CUPTI/include/GL/gl.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/GL/glew.h" "$(@D)/cuda/extras/CUPTI/include/GL/glew.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/GL/glext.h" "$(@D)/cuda/extras/CUPTI/include/GL/glext.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/GL/glu.h" "$(@D)/cuda/extras/CUPTI/include/GL/glu.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/GL/glut.h" "$(@D)/cuda/extras/CUPTI/include/GL/glut.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/GL/glx.h" "$(@D)/cuda/extras/CUPTI/include/GL/glx.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/GL/glxext.h" "$(@D)/cuda/extras/CUPTI/include/GL/glxext.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/GL/wglew.h" "$(@D)/cuda/extras/CUPTI/include/GL/wglew.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/GL/wglext.h" "$(@D)/cuda/extras/CUPTI/include/GL/wglext.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cuda_stdint.h" "$(@D)/cuda/extras/CUPTI/include/cuda_stdint.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti.h" "$(@D)/cuda/extras/CUPTI/include/cupti.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_activity.h" "$(@D)/cuda/extras/CUPTI/include/cupti_activity.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_callbacks.h" "$(@D)/cuda/extras/CUPTI/include/cupti_callbacks.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_driver_cbid.h" "$(@D)/cuda/extras/CUPTI/include/cupti_driver_cbid.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_events.h" "$(@D)/cuda/extras/CUPTI/include/cupti_events.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_metrics.h" "$(@D)/cuda/extras/CUPTI/include/cupti_metrics.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_nvtx_cbid.h" "$(@D)/cuda/extras/CUPTI/include/cupti_nvtx_cbid.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_result.h" "$(@D)/cuda/extras/CUPTI/include/cupti_result.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_runtime_cbid.h" "$(@D)/cuda/extras/CUPTI/include/cupti_runtime_cbid.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_version.h" "$(@D)/cuda/extras/CUPTI/include/cupti_version.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/generated_cudaGL_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cudaGL_meta.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/generated_cudaVDPAU_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cudaVDPAU_meta.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/generated_cuda_gl_interop_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_gl_interop_meta.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/generated_cuda_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_meta.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/generated_cuda_runtime_api_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_runtime_api_meta.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/generated_cuda_vdpau_interop_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_vdpau_interop_meta.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/generated_nvtx_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_nvtx_meta.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/openacc/cupti_openacc.h" "$(@D)/cuda/extras/CUPTI/include/openacc/cupti_openacc.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/openmp/cupti_openmp.h" "$(@D)/cuda/extras/CUPTI/include/openmp/cupti_openmp.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/openmp/ompt.h" "$(@D)/cuda/extras/CUPTI/include/openmp/ompt.h"
-   """,
+    cmd = """cp -rf "/usr/local/cuda-10.0/extras/CUPTI/include/." "$(@D)/cuda/extras/CUPTI/include/" """,
 )
 
 genrule(
@@ -1259,9 +1253,31 @@ genrule(
         "cuda/lib/libcudnn.so.7",
         "cuda/lib/libcupti.so.10.0",
     ],
-    cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/local/cuda-10.0/targets/x86_64-linux/lib/stubs/libcuda.so" "$(@D)/cuda/lib/libcuda.so" && cp -f "/usr/local/cuda-10.0/targets/x86_64-linux/lib/libcudart.so.10.0.130" "$(@D)/cuda/lib/libcudart.so.10.0" && cp -f "/usr/local/cuda-10.0/targets/x86_64-linux/lib/libcudart_static.a" "$(@D)/cuda/lib/libcudart_static.a" && cp -f "/usr/local/cuda-10.0/targets/x86_64-linux/lib/libcublas.so.10.0.130" "$(@D)/cuda/lib/libcublas.so.10.0" && cp -f "/usr/local/cuda-10.0/targets/x86_64-linux/lib/libcusolver.so.10.0.130" "$(@D)/cuda/lib/libcusolver.so.10.0" && cp -f "/usr/local/cuda-10.0/targets/x86_64-linux/lib/libcurand.so.10.0.130" "$(@D)/cuda/lib/libcurand.so.10.0" && cp -f "/usr/local/cuda-10.0/targets/x86_64-linux/lib/libcufft.so.10.0.145" "$(@D)/cuda/lib/libcufft.so.10.0" && cp -f "/usr/lib/x86_64-linux-gnu/libcudnn.so.7.3.1" "$(@D)/cuda/lib/libcudnn.so.7" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/lib64/libcupti.so.10.0.130" "$(@D)/cuda/lib/libcupti.so.10.0"
-   """,
+    cmd = """cp -f "/usr/local/cuda-10.0/lib64/stubs/libcuda.so" $(location cuda/lib/libcuda.so) && cp -f "/usr/local/cuda-10.0/lib64/libcudart.so.10.0" $(location cuda/lib/libcudart.so.10.0) && cp -f "/usr/local/cuda-10.0/lib64/libcudart_static.a" $(location cuda/lib/libcudart_static.a) && cp -f "/usr/local/cuda-10.0/lib64/libcublas.so.10.0" $(location cuda/lib/libcublas.so.10.0) && cp -f "/usr/local/cuda-10.0/lib64/libcusolver.so.10.0" $(location cuda/lib/libcusolver.so.10.0) && cp -f "/usr/local/cuda-10.0/lib64/libcurand.so.10.0" $(location cuda/lib/libcurand.so.10.0) && cp -f "/usr/local/cuda-10.0/lib64/libcufft.so.10.0" $(location cuda/lib/libcufft.so.10.0) && cp -f "/usr/lib/x86_64-linux-gnu/libcudnn.so.7" $(location cuda/lib/libcudnn.so.7) && cp -f "/usr/local/cuda-10.0/extras/CUPTI/lib64/libcupti.so.10.0" $(location cuda/lib/libcupti.so.10.0) """,
+)
+
+genrule(
+    name = "cuda-bin",
+    outs = [
+        "cuda/bin/bin2c",
+        "cuda/bin/crt/link.stub",
+        "cuda/bin/crt/prelink.stub",
+        "cuda/bin/cuda-gdb",
+        "cuda/bin/cuda-gdbserver",
+        "cuda/bin/cuda-memcheck",
+        "cuda/bin/cudafe++",
+        "cuda/bin/cuobjdump",
+        "cuda/bin/fatbinary",
+        "cuda/bin/gpu-library-advisor",
+        "cuda/bin/nvcc",
+        "cuda/bin/nvcc.profile",
+        "cuda/bin/nvdisasm",
+        "cuda/bin/nvlink",
+        "cuda/bin/nvprof",
+        "cuda/bin/nvprune",
+        "cuda/bin/ptxas",
+    ],
+    cmd = """cp -rf "/usr/local/cuda-10.0/bin/." "$(@D)/cuda/bin/" """,
 )
 
 genrule(
@@ -1269,7 +1285,5 @@ genrule(
     outs = [
         "cuda/include/cudnn.h",
     ],
-    cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/include/cudnn.h" "$(@D)/cudnn.h"
-   """,
+    cmd = """cp -f "/usr/include/cudnn.h" $(location cuda/include/cudnn.h) """,
 )
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD b/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD
index c6930904b5..cfd0a08e93 100755
--- a/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD
+++ b/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD
@@ -1187,9 +1187,7 @@ genrule(
         "cuda/include/vector_functions.hpp",
         "cuda/include/vector_types.h",
     ],
-    cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/local/cuda-9.0/include/CL/cl.h" "$(@D)/cuda/include/CL/cl.h" && cp -f "/usr/local/cuda-9.0/include/CL/cl.hpp" "$(@D)/cuda/include/CL/cl.hpp" && cp -f "/usr/local/cuda-9.0/include/CL/cl_egl.h" "$(@D)/cuda/include/CL/cl_egl.h" && cp -f "/usr/local/cuda-9.0/include/CL/cl_ext.h" "$(@D)/cuda/include/CL/cl_ext.h" && cp -f "/usr/local/cuda-9.0/include/CL/cl_gl.h" "$(@D)/cuda/include/CL/cl_gl.h" && cp -f "/usr/local/cuda-9.0/include/CL/cl_gl_ext.h" "$(@D)/cuda/include/CL/cl_gl_ext.h" && cp -f "/usr/local/cuda-9.0/include/CL/cl_platform.h" "$(@D)/cuda/include/CL/cl_platform.h" && cp -f "/usr/local/cuda-9.0/include/CL/opencl.h" "$(@D)/cuda/include/CL/opencl.h" && cp -f "/usr/local/cuda-9.0/include/builtin_types.h" "$(@D)/cuda/include/builtin_types.h" && cp -f "/usr/local/cuda-9.0/include/channel_descriptor.h" "$(@D)/cuda/include/channel_descriptor.h" && cp -f "/usr/local/cuda-9.0/include/common_functions.h" "$(@D)/cuda/include/common_functions.h" && cp -f "/usr/local/cuda-9.0/include/cooperative_groups.h" "$(@D)/cuda/include/cooperative_groups.h" && cp -f "/usr/local/cuda-9.0/include/cooperative_groups_helpers.h" "$(@D)/cuda/include/cooperative_groups_helpers.h" && cp -f "/usr/local/cuda-9.0/include/crt/common_functions.h" "$(@D)/cuda/include/crt/common_functions.h" && cp -f "/usr/local/cuda-9.0/include/crt/device_double_functions.h" "$(@D)/cuda/include/crt/device_double_functions.h" && cp -f "/usr/local/cuda-9.0/include/crt/device_double_functions.hpp" "$(@D)/cuda/include/crt/device_double_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/crt/device_functions.h" "$(@D)/cuda/include/crt/device_functions.h" && cp -f "/usr/local/cuda-9.0/include/crt/device_functions.hpp" "$(@D)/cuda/include/crt/device_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/crt/func_macro.h" "$(@D)/cuda/include/crt/func_macro.h" && cp -f "/usr/local/cuda-9.0/include/crt/host_config.h" "$(@D)/cuda/include/crt/host_config.h" && cp -f "/usr/local/cuda-9.0/include/crt/host_defines.h" "$(@D)/cuda/include/crt/host_defines.h" && cp -f "/usr/local/cuda-9.0/include/crt/host_runtime.h" "$(@D)/cuda/include/crt/host_runtime.h" && cp -f "/usr/local/cuda-9.0/include/crt/math_functions.h" "$(@D)/cuda/include/crt/math_functions.h" && cp -f "/usr/local/cuda-9.0/include/crt/math_functions.hpp" "$(@D)/cuda/include/crt/math_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/crt/mma.h" "$(@D)/cuda/include/crt/mma.h" && cp -f "/usr/local/cuda-9.0/include/crt/mma.hpp" "$(@D)/cuda/include/crt/mma.hpp" && cp -f "/usr/local/cuda-9.0/include/crt/nvfunctional" "$(@D)/cuda/include/crt/nvfunctional" && cp -f "/usr/local/cuda-9.0/include/crt/sm_70_rt.h" "$(@D)/cuda/include/crt/sm_70_rt.h" && cp -f "/usr/local/cuda-9.0/include/crt/sm_70_rt.hpp" "$(@D)/cuda/include/crt/sm_70_rt.hpp" && cp -f "/usr/local/cuda-9.0/include/crt/storage_class.h" "$(@D)/cuda/include/crt/storage_class.h" && cp -f "/usr/local/cuda-9.0/include/cuComplex.h" "$(@D)/cuda/include/cuComplex.h" && cp -f "/usr/local/cuda-9.0/include/cublas.h" "$(@D)/cuda/include/cublas.h" && cp -f "/usr/local/cuda-9.0/include/cublasXt.h" "$(@D)/cuda/include/cublasXt.h" && cp -f "/usr/local/cuda-9.0/include/cublas_api.h" "$(@D)/cuda/include/cublas_api.h" && cp -f "/usr/local/cuda-9.0/include/cublas_v2.h" "$(@D)/cuda/include/cublas_v2.h" && cp -f "/usr/local/cuda-9.0/include/cuda.h" "$(@D)/cuda/include/cuda.h" && cp -f "/usr/local/cuda-9.0/include/cudaEGL.h" "$(@D)/cuda/include/cudaEGL.h" && cp -f "/usr/local/cuda-9.0/include/cudaGL.h" "$(@D)/cuda/include/cudaGL.h" && cp -f "/usr/local/cuda-9.0/include/cudaProfiler.h" "$(@D)/cuda/include/cudaProfiler.h" && cp -f "/usr/local/cuda-9.0/include/cudaVDPAU.h" "$(@D)/cuda/include/cudaVDPAU.h" && cp -f "/usr/local/cuda-9.0/include/cuda_device_runtime_api.h" "$(@D)/cuda/include/cuda_device_runtime_api.h" && cp -f "/usr/local/cuda-9.0/include/cuda_fp16.h" "$(@D)/cuda/include/cuda_fp16.h" && cp -f "/usr/local/cuda-9.0/include/cuda_fp16.hpp" "$(@D)/cuda/include/cuda_fp16.hpp" && cp -f "/usr/local/cuda-9.0/include/cuda_gl_interop.h" "$(@D)/cuda/include/cuda_gl_interop.h" && cp -f "/usr/local/cuda-9.0/include/cuda_occupancy.h" "$(@D)/cuda/include/cuda_occupancy.h" && cp -f "/usr/local/cuda-9.0/include/cuda_profiler_api.h" "$(@D)/cuda/include/cuda_profiler_api.h" && cp -f "/usr/local/cuda-9.0/include/cuda_runtime.h" "$(@D)/cuda/include/cuda_runtime.h" && cp -f "/usr/local/cuda-9.0/include/cuda_runtime_api.h" "$(@D)/cuda/include/cuda_runtime_api.h" && cp -f "/usr/local/cuda-9.0/include/cuda_surface_types.h" "$(@D)/cuda/include/cuda_surface_types.h" && cp -f "/usr/local/cuda-9.0/include/cuda_texture_types.h" "$(@D)/cuda/include/cuda_texture_types.h" && cp -f "/usr/local/cuda-9.0/include/cuda_vdpau_interop.h" "$(@D)/cuda/include/cuda_vdpau_interop.h" && cp -f "/usr/local/cuda-9.0/include/cudalibxt.h" "$(@D)/cuda/include/cudalibxt.h" && cp -f "/usr/local/cuda-9.0/include/cufft.h" "$(@D)/cuda/include/cufft.h" && cp -f "/usr/local/cuda-9.0/include/cufftXt.h" "$(@D)/cuda/include/cufftXt.h" && cp -f "/usr/local/cuda-9.0/include/cufftw.h" "$(@D)/cuda/include/cufftw.h" && cp -f "/usr/local/cuda-9.0/include/curand.h" "$(@D)/cuda/include/curand.h" && cp -f "/usr/local/cuda-9.0/include/curand_discrete.h" "$(@D)/cuda/include/curand_discrete.h" && cp -f "/usr/local/cuda-9.0/include/curand_discrete2.h" "$(@D)/cuda/include/curand_discrete2.h" && cp -f "/usr/local/cuda-9.0/include/curand_globals.h" "$(@D)/cuda/include/curand_globals.h" && cp -f "/usr/local/cuda-9.0/include/curand_kernel.h" "$(@D)/cuda/include/curand_kernel.h" && cp -f "/usr/local/cuda-9.0/include/curand_lognormal.h" "$(@D)/cuda/include/curand_lognormal.h" && cp -f "/usr/local/cuda-9.0/include/curand_mrg32k3a.h" "$(@D)/cuda/include/curand_mrg32k3a.h" && cp -f "/usr/local/cuda-9.0/include/curand_mtgp32.h" "$(@D)/cuda/include/curand_mtgp32.h" && cp -f "/usr/local/cuda-9.0/include/curand_mtgp32_host.h" "$(@D)/cuda/include/curand_mtgp32_host.h" && cp -f "/usr/local/cuda-9.0/include/curand_mtgp32_kernel.h" "$(@D)/cuda/include/curand_mtgp32_kernel.h" && cp -f "/usr/local/cuda-9.0/include/curand_mtgp32dc_p_11213.h" "$(@D)/cuda/include/curand_mtgp32dc_p_11213.h" && cp -f "/usr/local/cuda-9.0/include/curand_normal.h" "$(@D)/cuda/include/curand_normal.h" && cp -f "/usr/local/cuda-9.0/include/curand_normal_static.h" "$(@D)/cuda/include/curand_normal_static.h" && cp -f "/usr/local/cuda-9.0/include/curand_philox4x32_x.h" "$(@D)/cuda/include/curand_philox4x32_x.h" && cp -f "/usr/local/cuda-9.0/include/curand_poisson.h" "$(@D)/cuda/include/curand_poisson.h" && cp -f "/usr/local/cuda-9.0/include/curand_precalc.h" "$(@D)/cuda/include/curand_precalc.h" && cp -f "/usr/local/cuda-9.0/include/curand_uniform.h" "$(@D)/cuda/include/curand_uniform.h" && cp -f "/usr/local/cuda-9.0/include/cusolverDn.h" "$(@D)/cuda/include/cusolverDn.h" && cp -f "/usr/local/cuda-9.0/include/cusolverRf.h" "$(@D)/cuda/include/cusolverRf.h" && cp -f "/usr/local/cuda-9.0/include/cusolverSp.h" "$(@D)/cuda/include/cusolverSp.h" && cp -f "/usr/local/cuda-9.0/include/cusolverSp_LOWLEVEL_PREVIEW.h" "$(@D)/cuda/include/cusolverSp_LOWLEVEL_PREVIEW.h" && cp -f "/usr/local/cuda-9.0/include/cusolver_common.h" "$(@D)/cuda/include/cusolver_common.h" && cp -f "/usr/local/cuda-9.0/include/cusparse.h" "$(@D)/cuda/include/cusparse.h" && cp -f "/usr/local/cuda-9.0/include/cusparse_v2.h" "$(@D)/cuda/include/cusparse_v2.h" && cp -f "/usr/local/cuda-9.0/include/device_atomic_functions.h" "$(@D)/cuda/include/device_atomic_functions.h" && cp -f "/usr/local/cuda-9.0/include/device_atomic_functions.hpp" "$(@D)/cuda/include/device_atomic_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/device_double_functions.h" "$(@D)/cuda/include/device_double_functions.h" && cp -f "/usr/local/cuda-9.0/include/device_double_functions.hpp" "$(@D)/cuda/include/device_double_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/device_functions.h" "$(@D)/cuda/include/device_functions.h" && cp -f "/usr/local/cuda-9.0/include/device_functions.hpp" "$(@D)/cuda/include/device_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/device_functions_decls.h" "$(@D)/cuda/include/device_functions_decls.h" && cp -f "/usr/local/cuda-9.0/include/device_launch_parameters.h" "$(@D)/cuda/include/device_launch_parameters.h" && cp -f "/usr/local/cuda-9.0/include/device_types.h" "$(@D)/cuda/include/device_types.h" && cp -f "/usr/local/cuda-9.0/include/driver_functions.h" "$(@D)/cuda/include/driver_functions.h" && cp -f "/usr/local/cuda-9.0/include/driver_types.h" "$(@D)/cuda/include/driver_types.h" && cp -f "/usr/local/cuda-9.0/include/dynlink_cuda.h" "$(@D)/cuda/include/dynlink_cuda.h" && cp -f "/usr/local/cuda-9.0/include/dynlink_cuda_cuda.h" "$(@D)/cuda/include/dynlink_cuda_cuda.h" && cp -f "/usr/local/cuda-9.0/include/dynlink_cuviddec.h" "$(@D)/cuda/include/dynlink_cuviddec.h" && cp -f "/usr/local/cuda-9.0/include/dynlink_nvcuvid.h" "$(@D)/cuda/include/dynlink_nvcuvid.h" && cp -f "/usr/local/cuda-9.0/include/fatBinaryCtl.h" "$(@D)/cuda/include/fatBinaryCtl.h" && cp -f "/usr/local/cuda-9.0/include/fatbinary.h" "$(@D)/cuda/include/fatbinary.h" && cp -f "/usr/local/cuda-9.0/include/host_config.h" "$(@D)/cuda/include/host_config.h" && cp -f "/usr/local/cuda-9.0/include/host_defines.h" "$(@D)/cuda/include/host_defines.h" && cp -f "/usr/local/cuda-9.0/include/library_types.h" "$(@D)/cuda/include/library_types.h" && cp -f "/usr/local/cuda-9.0/include/math_constants.h" "$(@D)/cuda/include/math_constants.h" && cp -f "/usr/local/cuda-9.0/include/math_functions.h" "$(@D)/cuda/include/math_functions.h" && cp -f "/usr/local/cuda-9.0/include/math_functions.hpp" "$(@D)/cuda/include/math_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/math_functions_dbl_ptx3.h" "$(@D)/cuda/include/math_functions_dbl_ptx3.h" && cp -f "/usr/local/cuda-9.0/include/math_functions_dbl_ptx3.hpp" "$(@D)/cuda/include/math_functions_dbl_ptx3.hpp" && cp -f "/usr/local/cuda-9.0/include/mma.h" "$(@D)/cuda/include/mma.h" && cp -f "/usr/local/cuda-9.0/include/npp.h" "$(@D)/cuda/include/npp.h" && cp -f "/usr/local/cuda-9.0/include/nppcore.h" "$(@D)/cuda/include/nppcore.h" && cp -f "/usr/local/cuda-9.0/include/nppdefs.h" "$(@D)/cuda/include/nppdefs.h" && cp -f "/usr/local/cuda-9.0/include/nppi.h" "$(@D)/cuda/include/nppi.h" && cp -f "/usr/local/cuda-9.0/include/nppi_arithmetic_and_logical_operations.h" "$(@D)/cuda/include/nppi_arithmetic_and_logical_operations.h" && cp -f "/usr/local/cuda-9.0/include/nppi_color_conversion.h" "$(@D)/cuda/include/nppi_color_conversion.h" && cp -f "/usr/local/cuda-9.0/include/nppi_compression_functions.h" "$(@D)/cuda/include/nppi_compression_functions.h" && cp -f "/usr/local/cuda-9.0/include/nppi_computer_vision.h" "$(@D)/cuda/include/nppi_computer_vision.h" && cp -f "/usr/local/cuda-9.0/include/nppi_data_exchange_and_initialization.h" "$(@D)/cuda/include/nppi_data_exchange_and_initialization.h" && cp -f "/usr/local/cuda-9.0/include/nppi_filtering_functions.h" "$(@D)/cuda/include/nppi_filtering_functions.h" && cp -f "/usr/local/cuda-9.0/include/nppi_geometry_transforms.h" "$(@D)/cuda/include/nppi_geometry_transforms.h" && cp -f "/usr/local/cuda-9.0/include/nppi_linear_transforms.h" "$(@D)/cuda/include/nppi_linear_transforms.h" && cp -f "/usr/local/cuda-9.0/include/nppi_morphological_operations.h" "$(@D)/cuda/include/nppi_morphological_operations.h" && cp -f "/usr/local/cuda-9.0/include/nppi_statistics_functions.h" "$(@D)/cuda/include/nppi_statistics_functions.h" && cp -f "/usr/local/cuda-9.0/include/nppi_support_functions.h" "$(@D)/cuda/include/nppi_support_functions.h" && cp -f "/usr/local/cuda-9.0/include/nppi_threshold_and_compare_operations.h" "$(@D)/cuda/include/nppi_threshold_and_compare_operations.h" && cp -f "/usr/local/cuda-9.0/include/npps.h" "$(@D)/cuda/include/npps.h" && cp -f "/usr/local/cuda-9.0/include/npps_arithmetic_and_logical_operations.h" "$(@D)/cuda/include/npps_arithmetic_and_logical_operations.h" && cp -f "/usr/local/cuda-9.0/include/npps_conversion_functions.h" "$(@D)/cuda/include/npps_conversion_functions.h" && cp -f "/usr/local/cuda-9.0/include/npps_filtering_functions.h" "$(@D)/cuda/include/npps_filtering_functions.h" && cp -f "/usr/local/cuda-9.0/include/npps_initialization.h" "$(@D)/cuda/include/npps_initialization.h" && cp -f "/usr/local/cuda-9.0/include/npps_statistics_functions.h" "$(@D)/cuda/include/npps_statistics_functions.h" && cp -f "/usr/local/cuda-9.0/include/npps_support_functions.h" "$(@D)/cuda/include/npps_support_functions.h" && cp -f "/usr/local/cuda-9.0/include/nppversion.h" "$(@D)/cuda/include/nppversion.h" && cp -f "/usr/local/cuda-9.0/include/nvToolsExt.h" "$(@D)/cuda/include/nvToolsExt.h" && cp -f "/usr/local/cuda-9.0/include/nvToolsExtCuda.h" "$(@D)/cuda/include/nvToolsExtCuda.h" && cp -f "/usr/local/cuda-9.0/include/nvToolsExtCudaRt.h" "$(@D)/cuda/include/nvToolsExtCudaRt.h" && cp -f "/usr/local/cuda-9.0/include/nvToolsExtMeta.h" "$(@D)/cuda/include/nvToolsExtMeta.h" && cp -f "/usr/local/cuda-9.0/include/nvToolsExtSync.h" "$(@D)/cuda/include/nvToolsExtSync.h" && cp -f "/usr/local/cuda-9.0/include/nvblas.h" "$(@D)/cuda/include/nvblas.h" && cp -f "/usr/local/cuda-9.0/include/nvfunctional" "$(@D)/cuda/include/nvfunctional" && cp -f "/usr/local/cuda-9.0/include/nvgraph.h" "$(@D)/cuda/include/nvgraph.h" && cp -f "/usr/local/cuda-9.0/include/nvml.h" "$(@D)/cuda/include/nvml.h" && cp -f "/usr/local/cuda-9.0/include/nvrtc.h" "$(@D)/cuda/include/nvrtc.h" && cp -f "/usr/local/cuda-9.0/include/sm_20_atomic_functions.h" "$(@D)/cuda/include/sm_20_atomic_functions.h" && cp -f "/usr/local/cuda-9.0/include/sm_20_atomic_functions.hpp" "$(@D)/cuda/include/sm_20_atomic_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/sm_20_intrinsics.h" "$(@D)/cuda/include/sm_20_intrinsics.h" && cp -f "/usr/local/cuda-9.0/include/sm_20_intrinsics.hpp" "$(@D)/cuda/include/sm_20_intrinsics.hpp" && cp -f "/usr/local/cuda-9.0/include/sm_30_intrinsics.h" "$(@D)/cuda/include/sm_30_intrinsics.h" && cp -f "/usr/local/cuda-9.0/include/sm_30_intrinsics.hpp" "$(@D)/cuda/include/sm_30_intrinsics.hpp" && cp -f "/usr/local/cuda-9.0/include/sm_32_atomic_functions.h" "$(@D)/cuda/include/sm_32_atomic_functions.h" && cp -f "/usr/local/cuda-9.0/include/sm_32_atomic_functions.hpp" "$(@D)/cuda/include/sm_32_atomic_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/sm_32_intrinsics.h" "$(@D)/cuda/include/sm_32_intrinsics.h" && cp -f "/usr/local/cuda-9.0/include/sm_32_intrinsics.hpp" "$(@D)/cuda/include/sm_32_intrinsics.hpp" && cp -f "/usr/local/cuda-9.0/include/sm_35_atomic_functions.h" "$(@D)/cuda/include/sm_35_atomic_functions.h" && cp -f "/usr/local/cuda-9.0/include/sm_35_intrinsics.h" "$(@D)/cuda/include/sm_35_intrinsics.h" && cp -f "/usr/local/cuda-9.0/include/sm_60_atomic_functions.h" "$(@D)/cuda/include/sm_60_atomic_functions.h" && cp -f "/usr/local/cuda-9.0/include/sm_60_atomic_functions.hpp" "$(@D)/cuda/include/sm_60_atomic_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/sm_61_intrinsics.h" "$(@D)/cuda/include/sm_61_intrinsics.h" && cp -f "/usr/local/cuda-9.0/include/sm_61_intrinsics.hpp" "$(@D)/cuda/include/sm_61_intrinsics.hpp" && cp -f "/usr/local/cuda-9.0/include/sobol_direction_vectors.h" "$(@D)/cuda/include/sobol_direction_vectors.h" && cp -f "/usr/local/cuda-9.0/include/surface_functions.h" "$(@D)/cuda/include/surface_functions.h" && cp -f "/usr/local/cuda-9.0/include/surface_functions.hpp" "$(@D)/cuda/include/surface_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/surface_indirect_functions.h" "$(@D)/cuda/include/surface_indirect_functions.h" && cp -f "/usr/local/cuda-9.0/include/surface_indirect_functions.hpp" "$(@D)/cuda/include/surface_indirect_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/surface_types.h" "$(@D)/cuda/include/surface_types.h" && cp -f "/usr/local/cuda-9.0/include/texture_fetch_functions.h" "$(@D)/cuda/include/texture_fetch_functions.h" && cp -f "/usr/local/cuda-9.0/include/texture_fetch_functions.hpp" "$(@D)/cuda/include/texture_fetch_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/texture_indirect_functions.h" "$(@D)/cuda/include/texture_indirect_functions.h" && cp -f "/usr/local/cuda-9.0/include/texture_indirect_functions.hpp" "$(@D)/cuda/include/texture_indirect_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/texture_types.h" "$(@D)/cuda/include/texture_types.h" && cp -f "/usr/local/cuda-9.0/include/thrust/adjacent_difference.h" "$(@D)/cuda/include/thrust/adjacent_difference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/advance.h" "$(@D)/cuda/include/thrust/advance.h" && cp -f "/usr/local/cuda-9.0/include/thrust/binary_search.h" "$(@D)/cuda/include/thrust/binary_search.h" && cp -f "/usr/local/cuda-9.0/include/thrust/complex.h" "$(@D)/cuda/include/thrust/complex.h" && cp -f "/usr/local/cuda-9.0/include/thrust/copy.h" "$(@D)/cuda/include/thrust/copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/count.h" "$(@D)/cuda/include/thrust/count.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/adjacent_difference.inl" "$(@D)/cuda/include/thrust/detail/adjacent_difference.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/advance.inl" "$(@D)/cuda/include/thrust/detail/advance.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/allocator_traits.h" "$(@D)/cuda/include/thrust/detail/allocator/allocator_traits.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/allocator_traits.inl" "$(@D)/cuda/include/thrust/detail/allocator/allocator_traits.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/copy_construct_range.h" "$(@D)/cuda/include/thrust/detail/allocator/copy_construct_range.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/copy_construct_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/copy_construct_range.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/default_construct_range.h" "$(@D)/cuda/include/thrust/detail/allocator/default_construct_range.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/default_construct_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/default_construct_range.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/destroy_range.h" "$(@D)/cuda/include/thrust/detail/allocator/destroy_range.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/destroy_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/destroy_range.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/fill_construct_range.h" "$(@D)/cuda/include/thrust/detail/allocator/fill_construct_range.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/fill_construct_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/fill_construct_range.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/malloc_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/malloc_allocator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/malloc_allocator.inl" "$(@D)/cuda/include/thrust/detail/allocator/malloc_allocator.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/no_throw_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/no_throw_allocator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/tagged_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/tagged_allocator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/tagged_allocator.inl" "$(@D)/cuda/include/thrust/detail/allocator/tagged_allocator.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/temporary_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/temporary_allocator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/temporary_allocator.inl" "$(@D)/cuda/include/thrust/detail/allocator/temporary_allocator.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/binary_search.inl" "$(@D)/cuda/include/thrust/detail/binary_search.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/arithmetic.h" "$(@D)/cuda/include/thrust/detail/complex/arithmetic.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/c99math.h" "$(@D)/cuda/include/thrust/detail/complex/c99math.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/catrig.h" "$(@D)/cuda/include/thrust/detail/complex/catrig.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/catrigf.h" "$(@D)/cuda/include/thrust/detail/complex/catrigf.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/ccosh.h" "$(@D)/cuda/include/thrust/detail/complex/ccosh.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/ccoshf.h" "$(@D)/cuda/include/thrust/detail/complex/ccoshf.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/cexp.h" "$(@D)/cuda/include/thrust/detail/complex/cexp.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/cexpf.h" "$(@D)/cuda/include/thrust/detail/complex/cexpf.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/clog.h" "$(@D)/cuda/include/thrust/detail/complex/clog.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/clogf.h" "$(@D)/cuda/include/thrust/detail/complex/clogf.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/complex.inl" "$(@D)/cuda/include/thrust/detail/complex/complex.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/cpow.h" "$(@D)/cuda/include/thrust/detail/complex/cpow.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/cpowf.h" "$(@D)/cuda/include/thrust/detail/complex/cpowf.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/cproj.h" "$(@D)/cuda/include/thrust/detail/complex/cproj.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/csinh.h" "$(@D)/cuda/include/thrust/detail/complex/csinh.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/csinhf.h" "$(@D)/cuda/include/thrust/detail/complex/csinhf.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/csqrt.h" "$(@D)/cuda/include/thrust/detail/complex/csqrt.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/csqrtf.h" "$(@D)/cuda/include/thrust/detail/complex/csqrtf.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/ctanh.h" "$(@D)/cuda/include/thrust/detail/complex/ctanh.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/ctanhf.h" "$(@D)/cuda/include/thrust/detail/complex/ctanhf.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/math_private.h" "$(@D)/cuda/include/thrust/detail/complex/math_private.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/stream.h" "$(@D)/cuda/include/thrust/detail/complex/stream.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config.h" "$(@D)/cuda/include/thrust/detail/config.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/compiler.h" "$(@D)/cuda/include/thrust/detail/config/compiler.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/compiler_fence.h" "$(@D)/cuda/include/thrust/detail/config/compiler_fence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/config.h" "$(@D)/cuda/include/thrust/detail/config/config.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/debug.h" "$(@D)/cuda/include/thrust/detail/config/debug.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/device_system.h" "$(@D)/cuda/include/thrust/detail/config/device_system.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/exec_check_disable.h" "$(@D)/cuda/include/thrust/detail/config/exec_check_disable.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/forceinline.h" "$(@D)/cuda/include/thrust/detail/config/forceinline.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/global_workarounds.h" "$(@D)/cuda/include/thrust/detail/config/global_workarounds.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/host_device.h" "$(@D)/cuda/include/thrust/detail/config/host_device.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/host_system.h" "$(@D)/cuda/include/thrust/detail/config/host_system.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/simple_defines.h" "$(@D)/cuda/include/thrust/detail/config/simple_defines.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/contiguous_storage.h" "$(@D)/cuda/include/thrust/detail/contiguous_storage.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/contiguous_storage.inl" "$(@D)/cuda/include/thrust/detail/contiguous_storage.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/copy.h" "$(@D)/cuda/include/thrust/detail/copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/copy.inl" "$(@D)/cuda/include/thrust/detail/copy.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/copy_if.h" "$(@D)/cuda/include/thrust/detail/copy_if.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/copy_if.inl" "$(@D)/cuda/include/thrust/detail/copy_if.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/count.inl" "$(@D)/cuda/include/thrust/detail/count.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/cstdint.h" "$(@D)/cuda/include/thrust/detail/cstdint.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/device_delete.inl" "$(@D)/cuda/include/thrust/detail/device_delete.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/device_free.inl" "$(@D)/cuda/include/thrust/detail/device_free.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/device_malloc.inl" "$(@D)/cuda/include/thrust/detail/device_malloc.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/device_new.inl" "$(@D)/cuda/include/thrust/detail/device_new.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/device_ptr.inl" "$(@D)/cuda/include/thrust/detail/device_ptr.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/device_reference.inl" "$(@D)/cuda/include/thrust/detail/device_reference.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/device_vector.inl" "$(@D)/cuda/include/thrust/detail/device_vector.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/dispatch/is_trivial_copy.h" "$(@D)/cuda/include/thrust/detail/dispatch/is_trivial_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/distance.inl" "$(@D)/cuda/include/thrust/detail/distance.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/equal.inl" "$(@D)/cuda/include/thrust/detail/equal.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/execute_with_allocator.h" "$(@D)/cuda/include/thrust/detail/execute_with_allocator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/execution_policy.h" "$(@D)/cuda/include/thrust/detail/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/extrema.inl" "$(@D)/cuda/include/thrust/detail/extrema.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/fill.inl" "$(@D)/cuda/include/thrust/detail/fill.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/find.inl" "$(@D)/cuda/include/thrust/detail/find.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/for_each.inl" "$(@D)/cuda/include/thrust/detail/for_each.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/function.h" "$(@D)/cuda/include/thrust/detail/function.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional.inl" "$(@D)/cuda/include/thrust/detail/functional.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/actor.h" "$(@D)/cuda/include/thrust/detail/functional/actor.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/actor.inl" "$(@D)/cuda/include/thrust/detail/functional/actor.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/argument.h" "$(@D)/cuda/include/thrust/detail/functional/argument.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/composite.h" "$(@D)/cuda/include/thrust/detail/functional/composite.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/arithmetic_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/arithmetic_operators.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/assignment_operator.h" "$(@D)/cuda/include/thrust/detail/functional/operators/assignment_operator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/bitwise_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/bitwise_operators.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/compound_assignment_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/compound_assignment_operators.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/logical_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/logical_operators.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/operator_adaptors.h" "$(@D)/cuda/include/thrust/detail/functional/operators/operator_adaptors.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/relational_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/relational_operators.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/placeholder.h" "$(@D)/cuda/include/thrust/detail/functional/placeholder.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/value.h" "$(@D)/cuda/include/thrust/detail/functional/value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/gather.inl" "$(@D)/cuda/include/thrust/detail/gather.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/generate.inl" "$(@D)/cuda/include/thrust/detail/generate.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/get_iterator_value.h" "$(@D)/cuda/include/thrust/detail/get_iterator_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/host_vector.inl" "$(@D)/cuda/include/thrust/detail/host_vector.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/inner_product.inl" "$(@D)/cuda/include/thrust/detail/inner_product.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/integer_math.h" "$(@D)/cuda/include/thrust/detail/integer_math.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/integer_traits.h" "$(@D)/cuda/include/thrust/detail/integer_traits.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/internal_functional.h" "$(@D)/cuda/include/thrust/detail/internal_functional.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/logical.inl" "$(@D)/cuda/include/thrust/detail/logical.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/merge.inl" "$(@D)/cuda/include/thrust/detail/merge.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/minmax.h" "$(@D)/cuda/include/thrust/detail/minmax.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/mismatch.inl" "$(@D)/cuda/include/thrust/detail/mismatch.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/mpl/math.h" "$(@D)/cuda/include/thrust/detail/mpl/math.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/numeric_traits.h" "$(@D)/cuda/include/thrust/detail/numeric_traits.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/overlapped_copy.h" "$(@D)/cuda/include/thrust/detail/overlapped_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/pair.inl" "$(@D)/cuda/include/thrust/detail/pair.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/partition.inl" "$(@D)/cuda/include/thrust/detail/partition.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/pointer.h" "$(@D)/cuda/include/thrust/detail/pointer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/pointer.inl" "$(@D)/cuda/include/thrust/detail/pointer.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/range/head_flags.h" "$(@D)/cuda/include/thrust/detail/range/head_flags.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/range/tail_flags.h" "$(@D)/cuda/include/thrust/detail/range/tail_flags.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/raw_pointer_cast.h" "$(@D)/cuda/include/thrust/detail/raw_pointer_cast.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/raw_reference_cast.h" "$(@D)/cuda/include/thrust/detail/raw_reference_cast.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/reduce.inl" "$(@D)/cuda/include/thrust/detail/reduce.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/reference.h" "$(@D)/cuda/include/thrust/detail/reference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/reference.inl" "$(@D)/cuda/include/thrust/detail/reference.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/reference_forward_declaration.h" "$(@D)/cuda/include/thrust/detail/reference_forward_declaration.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/remove.inl" "$(@D)/cuda/include/thrust/detail/remove.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/replace.inl" "$(@D)/cuda/include/thrust/detail/replace.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/reverse.inl" "$(@D)/cuda/include/thrust/detail/reverse.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/scan.inl" "$(@D)/cuda/include/thrust/detail/scan.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/scatter.inl" "$(@D)/cuda/include/thrust/detail/scatter.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/seq.h" "$(@D)/cuda/include/thrust/detail/seq.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/sequence.inl" "$(@D)/cuda/include/thrust/detail/sequence.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/set_operations.inl" "$(@D)/cuda/include/thrust/detail/set_operations.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/sort.inl" "$(@D)/cuda/include/thrust/detail/sort.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/static_assert.h" "$(@D)/cuda/include/thrust/detail/static_assert.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/static_map.h" "$(@D)/cuda/include/thrust/detail/static_map.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/swap.h" "$(@D)/cuda/include/thrust/detail/swap.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/swap.inl" "$(@D)/cuda/include/thrust/detail/swap.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/swap_ranges.inl" "$(@D)/cuda/include/thrust/detail/swap_ranges.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/tabulate.inl" "$(@D)/cuda/include/thrust/detail/tabulate.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/temporary_array.h" "$(@D)/cuda/include/thrust/detail/temporary_array.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/temporary_array.inl" "$(@D)/cuda/include/thrust/detail/temporary_array.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/transform.inl" "$(@D)/cuda/include/thrust/detail/transform.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/transform_reduce.inl" "$(@D)/cuda/include/thrust/detail/transform_reduce.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/transform_scan.inl" "$(@D)/cuda/include/thrust/detail/transform_scan.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/trivial_sequence.h" "$(@D)/cuda/include/thrust/detail/trivial_sequence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/tuple.inl" "$(@D)/cuda/include/thrust/detail/tuple.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/tuple_meta_transform.h" "$(@D)/cuda/include/thrust/detail/tuple_meta_transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/tuple_transform.h" "$(@D)/cuda/include/thrust/detail/tuple_transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits.h" "$(@D)/cuda/include/thrust/detail/type_traits.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/algorithm/intermediate_type_from_function_and_iterators.h" "$(@D)/cuda/include/thrust/detail/type_traits/algorithm/intermediate_type_from_function_and_iterators.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/function_traits.h" "$(@D)/cuda/include/thrust/detail/type_traits/function_traits.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/has_member_function.h" "$(@D)/cuda/include/thrust/detail/type_traits/has_member_function.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/has_nested_type.h" "$(@D)/cuda/include/thrust/detail/type_traits/has_nested_type.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/has_trivial_assign.h" "$(@D)/cuda/include/thrust/detail/type_traits/has_trivial_assign.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/is_call_possible.h" "$(@D)/cuda/include/thrust/detail/type_traits/is_call_possible.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/is_metafunction_defined.h" "$(@D)/cuda/include/thrust/detail/type_traits/is_metafunction_defined.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/iterator/is_discard_iterator.h" "$(@D)/cuda/include/thrust/detail/type_traits/iterator/is_discard_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/iterator/is_output_iterator.h" "$(@D)/cuda/include/thrust/detail/type_traits/iterator/is_output_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/minimum_type.h" "$(@D)/cuda/include/thrust/detail/type_traits/minimum_type.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/pointer_traits.h" "$(@D)/cuda/include/thrust/detail/type_traits/pointer_traits.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/result_of_adaptable_function.h" "$(@D)/cuda/include/thrust/detail/type_traits/result_of_adaptable_function.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/uninitialized_copy.inl" "$(@D)/cuda/include/thrust/detail/uninitialized_copy.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/uninitialized_fill.inl" "$(@D)/cuda/include/thrust/detail/uninitialized_fill.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/unique.inl" "$(@D)/cuda/include/thrust/detail/unique.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/use_default.h" "$(@D)/cuda/include/thrust/detail/use_default.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/util/align.h" "$(@D)/cuda/include/thrust/detail/util/align.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/util/blocking.h" "$(@D)/cuda/include/thrust/detail/util/blocking.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/vector_base.h" "$(@D)/cuda/include/thrust/detail/vector_base.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/vector_base.inl" "$(@D)/cuda/include/thrust/detail/vector_base.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/device_allocator.h" "$(@D)/cuda/include/thrust/device_allocator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/device_delete.h" "$(@D)/cuda/include/thrust/device_delete.h" && cp -f "/usr/local/cuda-9.0/include/thrust/device_free.h" "$(@D)/cuda/include/thrust/device_free.h" && cp -f "/usr/local/cuda-9.0/include/thrust/device_malloc.h" "$(@D)/cuda/include/thrust/device_malloc.h" && cp -f "/usr/local/cuda-9.0/include/thrust/device_malloc_allocator.h" "$(@D)/cuda/include/thrust/device_malloc_allocator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/device_new.h" "$(@D)/cuda/include/thrust/device_new.h" && cp -f "/usr/local/cuda-9.0/include/thrust/device_new_allocator.h" "$(@D)/cuda/include/thrust/device_new_allocator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/device_ptr.h" "$(@D)/cuda/include/thrust/device_ptr.h" && cp -f "/usr/local/cuda-9.0/include/thrust/device_reference.h" "$(@D)/cuda/include/thrust/device_reference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/device_vector.h" "$(@D)/cuda/include/thrust/device_vector.h" && cp -f "/usr/local/cuda-9.0/include/thrust/distance.h" "$(@D)/cuda/include/thrust/distance.h" && cp -f "/usr/local/cuda-9.0/include/thrust/equal.h" "$(@D)/cuda/include/thrust/equal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/execution_policy.h" "$(@D)/cuda/include/thrust/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/extrema.h" "$(@D)/cuda/include/thrust/extrema.h" && cp -f "/usr/local/cuda-9.0/include/thrust/fill.h" "$(@D)/cuda/include/thrust/fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/find.h" "$(@D)/cuda/include/thrust/find.h" && cp -f "/usr/local/cuda-9.0/include/thrust/for_each.h" "$(@D)/cuda/include/thrust/for_each.h" && cp -f "/usr/local/cuda-9.0/include/thrust/functional.h" "$(@D)/cuda/include/thrust/functional.h" && cp -f "/usr/local/cuda-9.0/include/thrust/gather.h" "$(@D)/cuda/include/thrust/gather.h" && cp -f "/usr/local/cuda-9.0/include/thrust/generate.h" "$(@D)/cuda/include/thrust/generate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/host_vector.h" "$(@D)/cuda/include/thrust/host_vector.h" && cp -f "/usr/local/cuda-9.0/include/thrust/inner_product.h" "$(@D)/cuda/include/thrust/inner_product.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/constant_iterator.h" "$(@D)/cuda/include/thrust/iterator/constant_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/counting_iterator.h" "$(@D)/cuda/include/thrust/iterator/counting_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/any_assign.h" "$(@D)/cuda/include/thrust/iterator/detail/any_assign.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/any_system_tag.h" "$(@D)/cuda/include/thrust/iterator/detail/any_system_tag.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/constant_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/constant_iterator_base.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/counting_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/counting_iterator.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/device_system_tag.h" "$(@D)/cuda/include/thrust/iterator/detail/device_system_tag.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/discard_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/discard_iterator_base.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/distance_from_result.h" "$(@D)/cuda/include/thrust/iterator/detail/distance_from_result.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/host_system_tag.h" "$(@D)/cuda/include/thrust/iterator/detail/host_system_tag.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/is_iterator_category.h" "$(@D)/cuda/include/thrust/iterator/detail/is_iterator_category.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/is_trivial_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/is_trivial_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_adaptor_base.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_adaptor_base.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_category_to_system.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_category_to_system.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_category_to_traversal.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_category_to_traversal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_category_with_system_and_traversal.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_category_with_system_and_traversal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_facade_category.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_facade_category.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_traits.inl" "$(@D)/cuda/include/thrust/iterator/detail/iterator_traits.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_traversal_tags.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_traversal_tags.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/join_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/join_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/minimum_category.h" "$(@D)/cuda/include/thrust/iterator/detail/minimum_category.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/minimum_system.h" "$(@D)/cuda/include/thrust/iterator/detail/minimum_system.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/normal_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/normal_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/permutation_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/permutation_iterator_base.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/retag.h" "$(@D)/cuda/include/thrust/iterator/detail/retag.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/reverse_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/reverse_iterator.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/reverse_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/reverse_iterator_base.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/tagged_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/tagged_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/transform_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/transform_iterator.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/transform_output_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/transform_output_iterator.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/tuple_of_iterator_references.h" "$(@D)/cuda/include/thrust/iterator/detail/tuple_of_iterator_references.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/universal_categories.h" "$(@D)/cuda/include/thrust/iterator/detail/universal_categories.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/zip_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/zip_iterator.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/zip_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/zip_iterator_base.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/discard_iterator.h" "$(@D)/cuda/include/thrust/iterator/discard_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/iterator_adaptor.h" "$(@D)/cuda/include/thrust/iterator/iterator_adaptor.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/iterator_categories.h" "$(@D)/cuda/include/thrust/iterator/iterator_categories.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/iterator_facade.h" "$(@D)/cuda/include/thrust/iterator/iterator_facade.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/iterator_traits.h" "$(@D)/cuda/include/thrust/iterator/iterator_traits.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/permutation_iterator.h" "$(@D)/cuda/include/thrust/iterator/permutation_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/retag.h" "$(@D)/cuda/include/thrust/iterator/retag.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/reverse_iterator.h" "$(@D)/cuda/include/thrust/iterator/reverse_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/transform_iterator.h" "$(@D)/cuda/include/thrust/iterator/transform_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/transform_output_iterator.h" "$(@D)/cuda/include/thrust/iterator/transform_output_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/zip_iterator.h" "$(@D)/cuda/include/thrust/iterator/zip_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/logical.h" "$(@D)/cuda/include/thrust/logical.h" && cp -f "/usr/local/cuda-9.0/include/thrust/memory.h" "$(@D)/cuda/include/thrust/memory.h" && cp -f "/usr/local/cuda-9.0/include/thrust/merge.h" "$(@D)/cuda/include/thrust/merge.h" && cp -f "/usr/local/cuda-9.0/include/thrust/mismatch.h" "$(@D)/cuda/include/thrust/mismatch.h" && cp -f "/usr/local/cuda-9.0/include/thrust/pair.h" "$(@D)/cuda/include/thrust/pair.h" && cp -f "/usr/local/cuda-9.0/include/thrust/partition.h" "$(@D)/cuda/include/thrust/partition.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random.h" "$(@D)/cuda/include/thrust/random.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/discard_block_engine.inl" "$(@D)/cuda/include/thrust/random/detail/discard_block_engine.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/linear_congruential_engine.inl" "$(@D)/cuda/include/thrust/random/detail/linear_congruential_engine.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/linear_congruential_engine_discard.h" "$(@D)/cuda/include/thrust/random/detail/linear_congruential_engine_discard.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/linear_feedback_shift_engine.inl" "$(@D)/cuda/include/thrust/random/detail/linear_feedback_shift_engine.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/linear_feedback_shift_engine_wordmask.h" "$(@D)/cuda/include/thrust/random/detail/linear_feedback_shift_engine_wordmask.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/mod.h" "$(@D)/cuda/include/thrust/random/detail/mod.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/normal_distribution.inl" "$(@D)/cuda/include/thrust/random/detail/normal_distribution.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/normal_distribution_base.h" "$(@D)/cuda/include/thrust/random/detail/normal_distribution_base.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/random_core_access.h" "$(@D)/cuda/include/thrust/random/detail/random_core_access.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/subtract_with_carry_engine.inl" "$(@D)/cuda/include/thrust/random/detail/subtract_with_carry_engine.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/uniform_int_distribution.inl" "$(@D)/cuda/include/thrust/random/detail/uniform_int_distribution.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/uniform_real_distribution.inl" "$(@D)/cuda/include/thrust/random/detail/uniform_real_distribution.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/xor_combine_engine.inl" "$(@D)/cuda/include/thrust/random/detail/xor_combine_engine.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/xor_combine_engine_max.h" "$(@D)/cuda/include/thrust/random/detail/xor_combine_engine_max.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/discard_block_engine.h" "$(@D)/cuda/include/thrust/random/discard_block_engine.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/linear_congruential_engine.h" "$(@D)/cuda/include/thrust/random/linear_congruential_engine.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/linear_feedback_shift_engine.h" "$(@D)/cuda/include/thrust/random/linear_feedback_shift_engine.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/normal_distribution.h" "$(@D)/cuda/include/thrust/random/normal_distribution.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/subtract_with_carry_engine.h" "$(@D)/cuda/include/thrust/random/subtract_with_carry_engine.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/uniform_int_distribution.h" "$(@D)/cuda/include/thrust/random/uniform_int_distribution.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/uniform_real_distribution.h" "$(@D)/cuda/include/thrust/random/uniform_real_distribution.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/xor_combine_engine.h" "$(@D)/cuda/include/thrust/random/xor_combine_engine.h" && cp -f "/usr/local/cuda-9.0/include/thrust/reduce.h" "$(@D)/cuda/include/thrust/reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/remove.h" "$(@D)/cuda/include/thrust/remove.h" && cp -f "/usr/local/cuda-9.0/include/thrust/replace.h" "$(@D)/cuda/include/thrust/replace.h" && cp -f "/usr/local/cuda-9.0/include/thrust/reverse.h" "$(@D)/cuda/include/thrust/reverse.h" && cp -f "/usr/local/cuda-9.0/include/thrust/scan.h" "$(@D)/cuda/include/thrust/scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/scatter.h" "$(@D)/cuda/include/thrust/scatter.h" && cp -f "/usr/local/cuda-9.0/include/thrust/sequence.h" "$(@D)/cuda/include/thrust/sequence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/set_operations.h" "$(@D)/cuda/include/thrust/set_operations.h" && cp -f "/usr/local/cuda-9.0/include/thrust/sort.h" "$(@D)/cuda/include/thrust/sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/swap.h" "$(@D)/cuda/include/thrust/swap.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/cpp/detail/adjacent_difference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/cpp/detail/assign_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/cpp/detail/binary_search.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/copy.h" "$(@D)/cuda/include/thrust/system/cpp/detail/copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/cpp/detail/copy_if.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/count.h" "$(@D)/cuda/include/thrust/system/cpp/detail/count.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/equal.h" "$(@D)/cuda/include/thrust/system/cpp/detail/equal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/cpp/detail/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/extrema.h" "$(@D)/cuda/include/thrust/system/cpp/detail/extrema.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/fill.h" "$(@D)/cuda/include/thrust/system/cpp/detail/fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/find.h" "$(@D)/cuda/include/thrust/system/cpp/detail/find.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/for_each.h" "$(@D)/cuda/include/thrust/system/cpp/detail/for_each.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/gather.h" "$(@D)/cuda/include/thrust/system/cpp/detail/gather.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/generate.h" "$(@D)/cuda/include/thrust/system/cpp/detail/generate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/get_value.h" "$(@D)/cuda/include/thrust/system/cpp/detail/get_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/cpp/detail/inner_product.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/cpp/detail/iter_swap.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/logical.h" "$(@D)/cuda/include/thrust/system/cpp/detail/logical.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/cpp/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/memory.inl" "$(@D)/cuda/include/thrust/system/cpp/detail/memory.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/merge.h" "$(@D)/cuda/include/thrust/system/cpp/detail/merge.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/cpp/detail/mismatch.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/par.h" "$(@D)/cuda/include/thrust/system/cpp/detail/par.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/partition.h" "$(@D)/cuda/include/thrust/system/cpp/detail/partition.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/reduce.h" "$(@D)/cuda/include/thrust/system/cpp/detail/reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/cpp/detail/reduce_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/remove.h" "$(@D)/cuda/include/thrust/system/cpp/detail/remove.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/replace.h" "$(@D)/cuda/include/thrust/system/cpp/detail/replace.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/reverse.h" "$(@D)/cuda/include/thrust/system/cpp/detail/reverse.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/scan.h" "$(@D)/cuda/include/thrust/system/cpp/detail/scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/cpp/detail/scan_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/scatter.h" "$(@D)/cuda/include/thrust/system/cpp/detail/scatter.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/sequence.h" "$(@D)/cuda/include/thrust/system/cpp/detail/sequence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/cpp/detail/set_operations.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/sort.h" "$(@D)/cuda/include/thrust/system/cpp/detail/sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/cpp/detail/swap_ranges.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/cpp/detail/tabulate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/cpp/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/transform.h" "$(@D)/cuda/include/thrust/system/cpp/detail/transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/cpp/detail/transform_reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/cpp/detail/transform_scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/cpp/detail/uninitialized_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/cpp/detail/uninitialized_fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/unique.h" "$(@D)/cuda/include/thrust/system/cpp/detail/unique.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/cpp/detail/unique_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/vector.inl" "$(@D)/cuda/include/thrust/system/cpp/detail/vector.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/execution_policy.h" "$(@D)/cuda/include/thrust/system/cpp/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/memory.h" "$(@D)/cuda/include/thrust/system/cpp/memory.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/vector.h" "$(@D)/cuda/include/thrust/system/cpp/vector.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/config.h" "$(@D)/cuda/include/thrust/system/cuda/config.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/cuda/detail/adjacent_difference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/cuda/detail/assign_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/cuda/detail/binary_search.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/copy.h" "$(@D)/cuda/include/thrust/system/cuda/detail/copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/cuda/detail/copy_if.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/core/agent_launcher.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/agent_launcher.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/core/alignment.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/alignment.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/core/triple_chevron_launch.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/triple_chevron_launch.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/core/util.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/util.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/count.h" "$(@D)/cuda/include/thrust/system/cuda/detail/count.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cross_system.h" "$(@D)/cuda/include/thrust/system/cuda/detail/cross_system.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_histogram.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_downsweep.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_downsweep.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_upsweep.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_upsweep.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_reduce.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_reduce_by_key.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_reduce_by_key.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_rle.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_rle.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_scan.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_segment_fixup.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_segment_fixup.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_select_if.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_select_if.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_spmv_csrt.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_spmv_csrt.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_spmv_orig.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_spmv_orig.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_spmv_row_based.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_spmv_row_based.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/single_pass_scan_operators.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/single_pass_scan_operators.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_adjacent_difference.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_adjacent_difference.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_discontinuity.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_discontinuity.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_exchange.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_exchange.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_histogram.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_load.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_load.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_radix_rank.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_radix_rank.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_radix_sort.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_raking_layout.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_raking_layout.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_reduce.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_scan.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_shuffle.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_shuffle.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_store.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_store.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_atomic.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_atomic.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_sort.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking_commutative_only.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking_commutative_only.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_warp_reductions.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_warp_reductions.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_raking.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_raking.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans2.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans2.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans3.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans3.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/cub.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/cub.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_histogram.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_partition.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_partition.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_radix_sort.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_reduce.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_run_length_encode.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_run_length_encode.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_scan.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_segmented_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_segmented_radix_sort.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_segmented_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_segmented_reduce.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_select.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_select.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_spmv.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_spmv.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_histogram.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_radix_sort.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce_by_key.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce_by_key.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_rle.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_rle.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_scan.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_select_if.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_select_if.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_csrt.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_csrt.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_orig.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_orig.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_row_based.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_row_based.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/grid/grid_barrier.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_barrier.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/grid/grid_even_share.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_even_share.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/grid/grid_mapping.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_mapping.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/grid/grid_queue.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_queue.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/host/mutex.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/host/mutex.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/arg_index_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/arg_index_input_iterator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/cache_modified_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/cache_modified_input_iterator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/cache_modified_output_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/cache_modified_output_iterator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/constant_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/constant_input_iterator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/counting_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/counting_input_iterator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/discard_output_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/discard_output_iterator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/tex_obj_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/tex_obj_input_iterator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/tex_ref_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/tex_ref_input_iterator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/transform_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/transform_input_iterator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_load.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_load.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_operators.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_operators.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_reduce.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_scan.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_search.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_search.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_store.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_store.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_allocator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_allocator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_arch.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_arch.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_debug.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_debug.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_device.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_device.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_macro.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_macro.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_namespace.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_namespace.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_ptx.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_ptx.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_type.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_type.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_shfl.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_shfl.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_smem.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_smem.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_shfl.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_shfl.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_smem.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_smem.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/warp_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/warp_reduce.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/warp_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/warp_scan.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/equal.h" "$(@D)/cuda/include/thrust/system/cuda/detail/equal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/error.inl" "$(@D)/cuda/include/thrust/system/cuda/detail/error.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/cuda/detail/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/extrema.h" "$(@D)/cuda/include/thrust/system/cuda/detail/extrema.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/fill.h" "$(@D)/cuda/include/thrust/system/cuda/detail/fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/find.h" "$(@D)/cuda/include/thrust/system/cuda/detail/find.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/for_each.h" "$(@D)/cuda/include/thrust/system/cuda/detail/for_each.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/gather.h" "$(@D)/cuda/include/thrust/system/cuda/detail/gather.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/generate.h" "$(@D)/cuda/include/thrust/system/cuda/detail/generate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/get_value.h" "$(@D)/cuda/include/thrust/system/cuda/detail/get_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/guarded_cuda_runtime_api.h" "$(@D)/cuda/include/thrust/system/cuda/detail/guarded_cuda_runtime_api.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/guarded_driver_types.h" "$(@D)/cuda/include/thrust/system/cuda/detail/guarded_driver_types.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/cuda/detail/inner_product.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/internal/copy_cross_system.h" "$(@D)/cuda/include/thrust/system/cuda/detail/internal/copy_cross_system.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/internal/copy_device_to_device.h" "$(@D)/cuda/include/thrust/system/cuda/detail/internal/copy_device_to_device.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/cuda/detail/iter_swap.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/logical.h" "$(@D)/cuda/include/thrust/system/cuda/detail/logical.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/cuda/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/memory.inl" "$(@D)/cuda/include/thrust/system/cuda/detail/memory.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/memory_buffer.h" "$(@D)/cuda/include/thrust/system/cuda/detail/memory_buffer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/merge.h" "$(@D)/cuda/include/thrust/system/cuda/detail/merge.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/cuda/detail/mismatch.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/par.h" "$(@D)/cuda/include/thrust/system/cuda/detail/par.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/par_to_seq.h" "$(@D)/cuda/include/thrust/system/cuda/detail/par_to_seq.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/parallel_for.h" "$(@D)/cuda/include/thrust/system/cuda/detail/parallel_for.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/partition.h" "$(@D)/cuda/include/thrust/system/cuda/detail/partition.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/reduce.h" "$(@D)/cuda/include/thrust/system/cuda/detail/reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/cuda/detail/reduce_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/remove.h" "$(@D)/cuda/include/thrust/system/cuda/detail/remove.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/replace.h" "$(@D)/cuda/include/thrust/system/cuda/detail/replace.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/reverse.h" "$(@D)/cuda/include/thrust/system/cuda/detail/reverse.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/scan.h" "$(@D)/cuda/include/thrust/system/cuda/detail/scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/cuda/detail/scan_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/scatter.h" "$(@D)/cuda/include/thrust/system/cuda/detail/scatter.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/sequence.h" "$(@D)/cuda/include/thrust/system/cuda/detail/sequence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/cuda/detail/set_operations.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/sort.h" "$(@D)/cuda/include/thrust/system/cuda/detail/sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/cuda/detail/swap_ranges.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/cuda/detail/tabulate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/cuda/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/terminate.h" "$(@D)/cuda/include/thrust/system/cuda/detail/terminate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/transform.h" "$(@D)/cuda/include/thrust/system/cuda/detail/transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/cuda/detail/transform_reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/cuda/detail/transform_scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/cuda/detail/uninitialized_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/cuda/detail/uninitialized_fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/unique.h" "$(@D)/cuda/include/thrust/system/cuda/detail/unique.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/cuda/detail/unique_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/util.h" "$(@D)/cuda/include/thrust/system/cuda/detail/util.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/vector.inl" "$(@D)/cuda/include/thrust/system/cuda/detail/vector.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/error.h" "$(@D)/cuda/include/thrust/system/cuda/error.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/execution_policy.h" "$(@D)/cuda/include/thrust/system/cuda/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/experimental/pinned_allocator.h" "$(@D)/cuda/include/thrust/system/cuda/experimental/pinned_allocator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/memory.h" "$(@D)/cuda/include/thrust/system/cuda/memory.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/vector.h" "$(@D)/cuda/include/thrust/system/cuda/vector.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/detail/adl/adjacent_difference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/assign_value.h" "$(@D)/cuda/include/thrust/system/detail/adl/assign_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/adl/binary_search.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/copy.h" "$(@D)/cuda/include/thrust/system/detail/adl/copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/copy_if.h" "$(@D)/cuda/include/thrust/system/detail/adl/copy_if.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/count.h" "$(@D)/cuda/include/thrust/system/detail/adl/count.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/equal.h" "$(@D)/cuda/include/thrust/system/detail/adl/equal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/extrema.h" "$(@D)/cuda/include/thrust/system/detail/adl/extrema.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/fill.h" "$(@D)/cuda/include/thrust/system/detail/adl/fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/find.h" "$(@D)/cuda/include/thrust/system/detail/adl/find.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/for_each.h" "$(@D)/cuda/include/thrust/system/detail/adl/for_each.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/gather.h" "$(@D)/cuda/include/thrust/system/detail/adl/gather.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/generate.h" "$(@D)/cuda/include/thrust/system/detail/adl/generate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/get_value.h" "$(@D)/cuda/include/thrust/system/detail/adl/get_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/inner_product.h" "$(@D)/cuda/include/thrust/system/detail/adl/inner_product.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/iter_swap.h" "$(@D)/cuda/include/thrust/system/detail/adl/iter_swap.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/logical.h" "$(@D)/cuda/include/thrust/system/detail/adl/logical.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/detail/adl/malloc_and_free.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/merge.h" "$(@D)/cuda/include/thrust/system/detail/adl/merge.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/mismatch.h" "$(@D)/cuda/include/thrust/system/detail/adl/mismatch.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/partition.h" "$(@D)/cuda/include/thrust/system/detail/adl/partition.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/reduce.h" "$(@D)/cuda/include/thrust/system/detail/adl/reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/detail/adl/reduce_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/remove.h" "$(@D)/cuda/include/thrust/system/detail/adl/remove.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/replace.h" "$(@D)/cuda/include/thrust/system/detail/adl/replace.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/reverse.h" "$(@D)/cuda/include/thrust/system/detail/adl/reverse.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/scan.h" "$(@D)/cuda/include/thrust/system/detail/adl/scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/scan_by_key.h" "$(@D)/cuda/include/thrust/system/detail/adl/scan_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/scatter.h" "$(@D)/cuda/include/thrust/system/detail/adl/scatter.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/sequence.h" "$(@D)/cuda/include/thrust/system/detail/adl/sequence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/set_operations.h" "$(@D)/cuda/include/thrust/system/detail/adl/set_operations.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/sort.h" "$(@D)/cuda/include/thrust/system/detail/adl/sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/swap_ranges.h" "$(@D)/cuda/include/thrust/system/detail/adl/swap_ranges.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/tabulate.h" "$(@D)/cuda/include/thrust/system/detail/adl/tabulate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/detail/adl/temporary_buffer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/transform.h" "$(@D)/cuda/include/thrust/system/detail/adl/transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/transform_reduce.h" "$(@D)/cuda/include/thrust/system/detail/adl/transform_reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/transform_scan.h" "$(@D)/cuda/include/thrust/system/detail/adl/transform_scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/detail/adl/uninitialized_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/detail/adl/uninitialized_fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/unique.h" "$(@D)/cuda/include/thrust/system/detail/adl/unique.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/unique_by_key.h" "$(@D)/cuda/include/thrust/system/detail/adl/unique_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/bad_alloc.h" "$(@D)/cuda/include/thrust/system/detail/bad_alloc.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/errno.h" "$(@D)/cuda/include/thrust/system/detail/errno.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/error_category.inl" "$(@D)/cuda/include/thrust/system/detail/error_category.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/error_code.inl" "$(@D)/cuda/include/thrust/system/detail/error_code.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/error_condition.inl" "$(@D)/cuda/include/thrust/system/detail/error_condition.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/detail/generic/adjacent_difference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/adjacent_difference.inl" "$(@D)/cuda/include/thrust/system/detail/generic/adjacent_difference.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/advance.h" "$(@D)/cuda/include/thrust/system/detail/generic/advance.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/advance.inl" "$(@D)/cuda/include/thrust/system/detail/generic/advance.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/generic/binary_search.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/binary_search.inl" "$(@D)/cuda/include/thrust/system/detail/generic/binary_search.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/copy.h" "$(@D)/cuda/include/thrust/system/detail/generic/copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/copy.inl" "$(@D)/cuda/include/thrust/system/detail/generic/copy.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/copy_if.h" "$(@D)/cuda/include/thrust/system/detail/generic/copy_if.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/copy_if.inl" "$(@D)/cuda/include/thrust/system/detail/generic/copy_if.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/count.h" "$(@D)/cuda/include/thrust/system/detail/generic/count.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/count.inl" "$(@D)/cuda/include/thrust/system/detail/generic/count.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/distance.h" "$(@D)/cuda/include/thrust/system/detail/generic/distance.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/distance.inl" "$(@D)/cuda/include/thrust/system/detail/generic/distance.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/equal.h" "$(@D)/cuda/include/thrust/system/detail/generic/equal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/equal.inl" "$(@D)/cuda/include/thrust/system/detail/generic/equal.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/extrema.h" "$(@D)/cuda/include/thrust/system/detail/generic/extrema.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/extrema.inl" "$(@D)/cuda/include/thrust/system/detail/generic/extrema.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/fill.h" "$(@D)/cuda/include/thrust/system/detail/generic/fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/find.h" "$(@D)/cuda/include/thrust/system/detail/generic/find.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/find.inl" "$(@D)/cuda/include/thrust/system/detail/generic/find.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/for_each.h" "$(@D)/cuda/include/thrust/system/detail/generic/for_each.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/gather.h" "$(@D)/cuda/include/thrust/system/detail/generic/gather.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/gather.inl" "$(@D)/cuda/include/thrust/system/detail/generic/gather.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/generate.h" "$(@D)/cuda/include/thrust/system/detail/generic/generate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/generate.inl" "$(@D)/cuda/include/thrust/system/detail/generic/generate.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/inner_product.h" "$(@D)/cuda/include/thrust/system/detail/generic/inner_product.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/inner_product.inl" "$(@D)/cuda/include/thrust/system/detail/generic/inner_product.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/logical.h" "$(@D)/cuda/include/thrust/system/detail/generic/logical.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/memory.h" "$(@D)/cuda/include/thrust/system/detail/generic/memory.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/memory.inl" "$(@D)/cuda/include/thrust/system/detail/generic/memory.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/merge.h" "$(@D)/cuda/include/thrust/system/detail/generic/merge.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/merge.inl" "$(@D)/cuda/include/thrust/system/detail/generic/merge.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/mismatch.h" "$(@D)/cuda/include/thrust/system/detail/generic/mismatch.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/mismatch.inl" "$(@D)/cuda/include/thrust/system/detail/generic/mismatch.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/partition.h" "$(@D)/cuda/include/thrust/system/detail/generic/partition.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/partition.inl" "$(@D)/cuda/include/thrust/system/detail/generic/partition.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reduce.h" "$(@D)/cuda/include/thrust/system/detail/generic/reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reduce.inl" "$(@D)/cuda/include/thrust/system/detail/generic/reduce.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/detail/generic/reduce_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reduce_by_key.inl" "$(@D)/cuda/include/thrust/system/detail/generic/reduce_by_key.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/remove.h" "$(@D)/cuda/include/thrust/system/detail/generic/remove.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/remove.inl" "$(@D)/cuda/include/thrust/system/detail/generic/remove.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/replace.h" "$(@D)/cuda/include/thrust/system/detail/generic/replace.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/replace.inl" "$(@D)/cuda/include/thrust/system/detail/generic/replace.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reverse.h" "$(@D)/cuda/include/thrust/system/detail/generic/reverse.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reverse.inl" "$(@D)/cuda/include/thrust/system/detail/generic/reverse.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scalar/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/generic/scalar/binary_search.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scalar/binary_search.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scalar/binary_search.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scan.h" "$(@D)/cuda/include/thrust/system/detail/generic/scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scan.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scan.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scan_by_key.h" "$(@D)/cuda/include/thrust/system/detail/generic/scan_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scan_by_key.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scan_by_key.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scatter.h" "$(@D)/cuda/include/thrust/system/detail/generic/scatter.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scatter.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scatter.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/select_system.h" "$(@D)/cuda/include/thrust/system/detail/generic/select_system.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/sequence.h" "$(@D)/cuda/include/thrust/system/detail/generic/sequence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/sequence.inl" "$(@D)/cuda/include/thrust/system/detail/generic/sequence.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/set_operations.h" "$(@D)/cuda/include/thrust/system/detail/generic/set_operations.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/set_operations.inl" "$(@D)/cuda/include/thrust/system/detail/generic/set_operations.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/sort.h" "$(@D)/cuda/include/thrust/system/detail/generic/sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/sort.inl" "$(@D)/cuda/include/thrust/system/detail/generic/sort.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/swap_ranges.h" "$(@D)/cuda/include/thrust/system/detail/generic/swap_ranges.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/swap_ranges.inl" "$(@D)/cuda/include/thrust/system/detail/generic/swap_ranges.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/tabulate.h" "$(@D)/cuda/include/thrust/system/detail/generic/tabulate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/tabulate.inl" "$(@D)/cuda/include/thrust/system/detail/generic/tabulate.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/tag.h" "$(@D)/cuda/include/thrust/system/detail/generic/tag.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/detail/generic/temporary_buffer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/temporary_buffer.inl" "$(@D)/cuda/include/thrust/system/detail/generic/temporary_buffer.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform.h" "$(@D)/cuda/include/thrust/system/detail/generic/transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform.inl" "$(@D)/cuda/include/thrust/system/detail/generic/transform.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform_reduce.h" "$(@D)/cuda/include/thrust/system/detail/generic/transform_reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform_reduce.inl" "$(@D)/cuda/include/thrust/system/detail/generic/transform_reduce.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform_scan.h" "$(@D)/cuda/include/thrust/system/detail/generic/transform_scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform_scan.inl" "$(@D)/cuda/include/thrust/system/detail/generic/transform_scan.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/type_traits.h" "$(@D)/cuda/include/thrust/system/detail/generic/type_traits.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/uninitialized_copy.inl" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_copy.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/uninitialized_fill.inl" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_fill.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/unique.h" "$(@D)/cuda/include/thrust/system/detail/generic/unique.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/unique.inl" "$(@D)/cuda/include/thrust/system/detail/generic/unique.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/unique_by_key.h" "$(@D)/cuda/include/thrust/system/detail/generic/unique_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/unique_by_key.inl" "$(@D)/cuda/include/thrust/system/detail/generic/unique_by_key.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/internal/decompose.h" "$(@D)/cuda/include/thrust/system/detail/internal/decompose.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/detail/sequential/adjacent_difference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/assign_value.h" "$(@D)/cuda/include/thrust/system/detail/sequential/assign_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/sequential/binary_search.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/copy.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/copy.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/copy_backward.h" "$(@D)/cuda/include/thrust/system/detail/sequential/copy_backward.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/copy_if.h" "$(@D)/cuda/include/thrust/system/detail/sequential/copy_if.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/count.h" "$(@D)/cuda/include/thrust/system/detail/sequential/count.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/equal.h" "$(@D)/cuda/include/thrust/system/detail/sequential/equal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/execution_policy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/extrema.h" "$(@D)/cuda/include/thrust/system/detail/sequential/extrema.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/fill.h" "$(@D)/cuda/include/thrust/system/detail/sequential/fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/find.h" "$(@D)/cuda/include/thrust/system/detail/sequential/find.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/for_each.h" "$(@D)/cuda/include/thrust/system/detail/sequential/for_each.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/gather.h" "$(@D)/cuda/include/thrust/system/detail/sequential/gather.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/general_copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/general_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/generate.h" "$(@D)/cuda/include/thrust/system/detail/sequential/generate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/get_value.h" "$(@D)/cuda/include/thrust/system/detail/sequential/get_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/inner_product.h" "$(@D)/cuda/include/thrust/system/detail/sequential/inner_product.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/insertion_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/insertion_sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/iter_swap.h" "$(@D)/cuda/include/thrust/system/detail/sequential/iter_swap.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/logical.h" "$(@D)/cuda/include/thrust/system/detail/sequential/logical.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/detail/sequential/malloc_and_free.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/merge.h" "$(@D)/cuda/include/thrust/system/detail/sequential/merge.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/merge.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/merge.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/mismatch.h" "$(@D)/cuda/include/thrust/system/detail/sequential/mismatch.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/partition.h" "$(@D)/cuda/include/thrust/system/detail/sequential/partition.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/reduce.h" "$(@D)/cuda/include/thrust/system/detail/sequential/reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/detail/sequential/reduce_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/remove.h" "$(@D)/cuda/include/thrust/system/detail/sequential/remove.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/replace.h" "$(@D)/cuda/include/thrust/system/detail/sequential/replace.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/reverse.h" "$(@D)/cuda/include/thrust/system/detail/sequential/reverse.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/scan.h" "$(@D)/cuda/include/thrust/system/detail/sequential/scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/scan_by_key.h" "$(@D)/cuda/include/thrust/system/detail/sequential/scan_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/scatter.h" "$(@D)/cuda/include/thrust/system/detail/sequential/scatter.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/sequence.h" "$(@D)/cuda/include/thrust/system/detail/sequential/sequence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/set_operations.h" "$(@D)/cuda/include/thrust/system/detail/sequential/set_operations.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/sort.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_merge_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_merge_sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_merge_sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_merge_sort.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_primitive_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_primitive_sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_primitive_sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_primitive_sort.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_radix_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_radix_sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_radix_sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_radix_sort.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/swap_ranges.h" "$(@D)/cuda/include/thrust/system/detail/sequential/swap_ranges.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/tabulate.h" "$(@D)/cuda/include/thrust/system/detail/sequential/tabulate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/detail/sequential/temporary_buffer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/transform.h" "$(@D)/cuda/include/thrust/system/detail/sequential/transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/transform_reduce.h" "$(@D)/cuda/include/thrust/system/detail/sequential/transform_reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/transform_scan.h" "$(@D)/cuda/include/thrust/system/detail/sequential/transform_scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/trivial_copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/trivial_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/uninitialized_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/detail/sequential/uninitialized_fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/unique.h" "$(@D)/cuda/include/thrust/system/detail/sequential/unique.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/unique_by_key.h" "$(@D)/cuda/include/thrust/system/detail/sequential/unique_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/system_error.inl" "$(@D)/cuda/include/thrust/system/detail/system_error.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/error_code.h" "$(@D)/cuda/include/thrust/system/error_code.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/omp/detail/adjacent_difference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/omp/detail/assign_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/omp/detail/binary_search.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/copy.h" "$(@D)/cuda/include/thrust/system/omp/detail/copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/copy.inl" "$(@D)/cuda/include/thrust/system/omp/detail/copy.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/omp/detail/copy_if.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/copy_if.inl" "$(@D)/cuda/include/thrust/system/omp/detail/copy_if.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/count.h" "$(@D)/cuda/include/thrust/system/omp/detail/count.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/default_decomposition.h" "$(@D)/cuda/include/thrust/system/omp/detail/default_decomposition.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/default_decomposition.inl" "$(@D)/cuda/include/thrust/system/omp/detail/default_decomposition.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/equal.h" "$(@D)/cuda/include/thrust/system/omp/detail/equal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/omp/detail/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/extrema.h" "$(@D)/cuda/include/thrust/system/omp/detail/extrema.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/fill.h" "$(@D)/cuda/include/thrust/system/omp/detail/fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/find.h" "$(@D)/cuda/include/thrust/system/omp/detail/find.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/for_each.h" "$(@D)/cuda/include/thrust/system/omp/detail/for_each.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/for_each.inl" "$(@D)/cuda/include/thrust/system/omp/detail/for_each.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/gather.h" "$(@D)/cuda/include/thrust/system/omp/detail/gather.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/generate.h" "$(@D)/cuda/include/thrust/system/omp/detail/generate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/get_value.h" "$(@D)/cuda/include/thrust/system/omp/detail/get_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/omp/detail/inner_product.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/omp/detail/iter_swap.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/logical.h" "$(@D)/cuda/include/thrust/system/omp/detail/logical.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/omp/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/memory.inl" "$(@D)/cuda/include/thrust/system/omp/detail/memory.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/merge.h" "$(@D)/cuda/include/thrust/system/omp/detail/merge.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/omp/detail/mismatch.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/par.h" "$(@D)/cuda/include/thrust/system/omp/detail/par.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/partition.h" "$(@D)/cuda/include/thrust/system/omp/detail/partition.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/partition.inl" "$(@D)/cuda/include/thrust/system/omp/detail/partition.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce.h" "$(@D)/cuda/include/thrust/system/omp/detail/reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce.inl" "$(@D)/cuda/include/thrust/system/omp/detail/reduce.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce_by_key.inl" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_by_key.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce_intervals.h" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_intervals.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce_intervals.inl" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_intervals.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/remove.h" "$(@D)/cuda/include/thrust/system/omp/detail/remove.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/remove.inl" "$(@D)/cuda/include/thrust/system/omp/detail/remove.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/replace.h" "$(@D)/cuda/include/thrust/system/omp/detail/replace.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reverse.h" "$(@D)/cuda/include/thrust/system/omp/detail/reverse.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/scan.h" "$(@D)/cuda/include/thrust/system/omp/detail/scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/omp/detail/scan_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/scatter.h" "$(@D)/cuda/include/thrust/system/omp/detail/scatter.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/sequence.h" "$(@D)/cuda/include/thrust/system/omp/detail/sequence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/omp/detail/set_operations.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/sort.h" "$(@D)/cuda/include/thrust/system/omp/detail/sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/sort.inl" "$(@D)/cuda/include/thrust/system/omp/detail/sort.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/omp/detail/swap_ranges.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/omp/detail/tabulate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/omp/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/transform.h" "$(@D)/cuda/include/thrust/system/omp/detail/transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/omp/detail/transform_reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/omp/detail/transform_scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/omp/detail/uninitialized_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/omp/detail/uninitialized_fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/unique.h" "$(@D)/cuda/include/thrust/system/omp/detail/unique.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/unique.inl" "$(@D)/cuda/include/thrust/system/omp/detail/unique.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/omp/detail/unique_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/unique_by_key.inl" "$(@D)/cuda/include/thrust/system/omp/detail/unique_by_key.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/vector.inl" "$(@D)/cuda/include/thrust/system/omp/detail/vector.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/execution_policy.h" "$(@D)/cuda/include/thrust/system/omp/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/memory.h" "$(@D)/cuda/include/thrust/system/omp/memory.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/vector.h" "$(@D)/cuda/include/thrust/system/omp/vector.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/system_error.h" "$(@D)/cuda/include/thrust/system/system_error.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/tbb/detail/adjacent_difference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/tbb/detail/assign_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/tbb/detail/binary_search.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/copy.h" "$(@D)/cuda/include/thrust/system/tbb/detail/copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/copy.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/copy.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/tbb/detail/copy_if.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/copy_if.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/copy_if.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/count.h" "$(@D)/cuda/include/thrust/system/tbb/detail/count.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/equal.h" "$(@D)/cuda/include/thrust/system/tbb/detail/equal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/tbb/detail/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/extrema.h" "$(@D)/cuda/include/thrust/system/tbb/detail/extrema.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/fill.h" "$(@D)/cuda/include/thrust/system/tbb/detail/fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/find.h" "$(@D)/cuda/include/thrust/system/tbb/detail/find.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/for_each.h" "$(@D)/cuda/include/thrust/system/tbb/detail/for_each.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/for_each.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/for_each.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/gather.h" "$(@D)/cuda/include/thrust/system/tbb/detail/gather.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/generate.h" "$(@D)/cuda/include/thrust/system/tbb/detail/generate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/get_value.h" "$(@D)/cuda/include/thrust/system/tbb/detail/get_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/tbb/detail/inner_product.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/tbb/detail/iter_swap.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/logical.h" "$(@D)/cuda/include/thrust/system/tbb/detail/logical.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/tbb/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/memory.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/memory.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/merge.h" "$(@D)/cuda/include/thrust/system/tbb/detail/merge.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/merge.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/merge.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/tbb/detail/mismatch.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/par.h" "$(@D)/cuda/include/thrust/system/tbb/detail/par.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/partition.h" "$(@D)/cuda/include/thrust/system/tbb/detail/partition.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/partition.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/partition.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reduce.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reduce.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reduce_by_key.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce_by_key.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reduce_intervals.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce_intervals.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/remove.h" "$(@D)/cuda/include/thrust/system/tbb/detail/remove.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/remove.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/remove.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/replace.h" "$(@D)/cuda/include/thrust/system/tbb/detail/replace.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reverse.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reverse.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/scan.h" "$(@D)/cuda/include/thrust/system/tbb/detail/scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/scan.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/scan.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/tbb/detail/scan_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/scatter.h" "$(@D)/cuda/include/thrust/system/tbb/detail/scatter.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/sequence.h" "$(@D)/cuda/include/thrust/system/tbb/detail/sequence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/tbb/detail/set_operations.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/sort.h" "$(@D)/cuda/include/thrust/system/tbb/detail/sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/sort.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/sort.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/tbb/detail/swap_ranges.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/tbb/detail/tabulate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/tbb/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/transform.h" "$(@D)/cuda/include/thrust/system/tbb/detail/transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/tbb/detail/transform_reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/tbb/detail/transform_scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/tbb/detail/uninitialized_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/tbb/detail/uninitialized_fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/unique.h" "$(@D)/cuda/include/thrust/system/tbb/detail/unique.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/unique.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/unique.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/tbb/detail/unique_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/unique_by_key.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/unique_by_key.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/vector.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/vector.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/execution_policy.h" "$(@D)/cuda/include/thrust/system/tbb/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/memory.h" "$(@D)/cuda/include/thrust/system/tbb/memory.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/vector.h" "$(@D)/cuda/include/thrust/system/tbb/vector.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system_error.h" "$(@D)/cuda/include/thrust/system_error.h" && cp -f "/usr/local/cuda-9.0/include/thrust/tabulate.h" "$(@D)/cuda/include/thrust/tabulate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/transform.h" "$(@D)/cuda/include/thrust/transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/transform_reduce.h" "$(@D)/cuda/include/thrust/transform_reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/transform_scan.h" "$(@D)/cuda/include/thrust/transform_scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/tuple.h" "$(@D)/cuda/include/thrust/tuple.h" && cp -f "/usr/local/cuda-9.0/include/thrust/uninitialized_copy.h" "$(@D)/cuda/include/thrust/uninitialized_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/uninitialized_fill.h" "$(@D)/cuda/include/thrust/uninitialized_fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/unique.h" "$(@D)/cuda/include/thrust/unique.h" && cp -f "/usr/local/cuda-9.0/include/thrust/version.h" "$(@D)/cuda/include/thrust/version.h" && cp -f "/usr/local/cuda-9.0/include/vector_functions.h" "$(@D)/cuda/include/vector_functions.h" && cp -f "/usr/local/cuda-9.0/include/vector_functions.hpp" "$(@D)/cuda/include/vector_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/vector_types.h" "$(@D)/cuda/include/vector_types.h"
-   """,
+    cmd = """cp -rf "/usr/local/cuda-9.0/include/." "$(@D)/cuda/include/" """,
 )
 
 genrule(
@@ -1197,9 +1195,7 @@ genrule(
     outs = [
         "cuda/nvvm/libdevice/libdevice.10.bc",
     ],
-    cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/local/cuda-9.0/nvvm/libdevice/libdevice.10.bc" "$(@D)//libdevice.10.bc"
-   """,
+    cmd = """cp -rf "/usr/local/cuda-9.0/nvvm/libdevice/." "$(@D)/" """,
 )
 
 genrule(
@@ -1234,9 +1230,7 @@ genrule(
         "cuda/extras/CUPTI/include/generated_nvtx_meta.h",
         "cuda/extras/CUPTI/include/openacc/cupti_openacc.h",
     ],
-    cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/GL/gl.h" "$(@D)/cuda/extras/CUPTI/include/GL/gl.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glew.h" "$(@D)/cuda/extras/CUPTI/include/GL/glew.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glext.h" "$(@D)/cuda/extras/CUPTI/include/GL/glext.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glu.h" "$(@D)/cuda/extras/CUPTI/include/GL/glu.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glut.h" "$(@D)/cuda/extras/CUPTI/include/GL/glut.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glx.h" "$(@D)/cuda/extras/CUPTI/include/GL/glx.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glxext.h" "$(@D)/cuda/extras/CUPTI/include/GL/glxext.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/GL/wglew.h" "$(@D)/cuda/extras/CUPTI/include/GL/wglew.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/GL/wglext.h" "$(@D)/cuda/extras/CUPTI/include/GL/wglext.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cuda_stdint.h" "$(@D)/cuda/extras/CUPTI/include/cuda_stdint.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti.h" "$(@D)/cuda/extras/CUPTI/include/cupti.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_activity.h" "$(@D)/cuda/extras/CUPTI/include/cupti_activity.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_callbacks.h" "$(@D)/cuda/extras/CUPTI/include/cupti_callbacks.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_driver_cbid.h" "$(@D)/cuda/extras/CUPTI/include/cupti_driver_cbid.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_events.h" "$(@D)/cuda/extras/CUPTI/include/cupti_events.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_metrics.h" "$(@D)/cuda/extras/CUPTI/include/cupti_metrics.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_nvtx_cbid.h" "$(@D)/cuda/extras/CUPTI/include/cupti_nvtx_cbid.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_result.h" "$(@D)/cuda/extras/CUPTI/include/cupti_result.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_runtime_cbid.h" "$(@D)/cuda/extras/CUPTI/include/cupti_runtime_cbid.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_version.h" "$(@D)/cuda/extras/CUPTI/include/cupti_version.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cudaGL_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cudaGL_meta.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cudaVDPAU_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cudaVDPAU_meta.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cuda_gl_interop_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_gl_interop_meta.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cuda_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_meta.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cuda_runtime_api_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_runtime_api_meta.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cuda_vdpau_interop_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_vdpau_interop_meta.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/generated_nvtx_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_nvtx_meta.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/openacc/cupti_openacc.h" "$(@D)/cuda/extras/CUPTI/include/openacc/cupti_openacc.h"
-   """,
+    cmd = """cp -rf "/usr/local/cuda-9.0/extras/CUPTI/include/." "$(@D)/cuda/extras/CUPTI/include/" """,
 )
 
 genrule(
@@ -1252,9 +1246,32 @@ genrule(
         "cuda/lib/libcudnn.so.7",
         "cuda/lib/libcupti.so.9.0",
     ],
-    cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/local/cuda-9.0/targets/x86_64-linux/lib/stubs/libcuda.so" "$(@D)/cuda/lib/libcuda.so" && cp -f "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcudart.so.9.0.176" "$(@D)/cuda/lib/libcudart.so.9.0" && cp -f "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcudart_static.a" "$(@D)/cuda/lib/libcudart_static.a" && cp -f "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcublas.so.9.0.480" "$(@D)/cuda/lib/libcublas.so.9.0" && cp -f "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcusolver.so.9.0.176" "$(@D)/cuda/lib/libcusolver.so.9.0" && cp -f "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcurand.so.9.0.176" "$(@D)/cuda/lib/libcurand.so.9.0" && cp -f "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcufft.so.9.0.176" "$(@D)/cuda/lib/libcufft.so.9.0" && cp -f "/usr/lib/x86_64-linux-gnu/libcudnn.so.7.1.4" "$(@D)/cuda/lib/libcudnn.so.7" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/lib64/libcupti.so.9.0.176" "$(@D)/cuda/lib/libcupti.so.9.0"
-   """,
+    cmd = """cp -f "/usr/local/cuda-9.0/lib64/stubs/libcuda.so" $(location cuda/lib/libcuda.so) && cp -f "/usr/local/cuda-9.0/lib64/libcudart.so.9.0" $(location cuda/lib/libcudart.so.9.0) && cp -f "/usr/local/cuda-9.0/lib64/libcudart_static.a" $(location cuda/lib/libcudart_static.a) && cp -f "/usr/local/cuda-9.0/lib64/libcublas.so.9.0" $(location cuda/lib/libcublas.so.9.0) && cp -f "/usr/local/cuda-9.0/lib64/libcusolver.so.9.0" $(location cuda/lib/libcusolver.so.9.0) && cp -f "/usr/local/cuda-9.0/lib64/libcurand.so.9.0" $(location cuda/lib/libcurand.so.9.0) && cp -f "/usr/local/cuda-9.0/lib64/libcufft.so.9.0" $(location cuda/lib/libcufft.so.9.0) && cp -f "/usr/lib/x86_64-linux-gnu/libcudnn.so.7" $(location cuda/lib/libcudnn.so.7) && cp -f "/usr/local/cuda-9.0/extras/CUPTI/lib64/libcupti.so.9.0" $(location cuda/lib/libcupti.so.9.0) """,
+)
+
+genrule(
+    name = "cuda-bin",
+    outs = [
+        "cuda/bin/bin2c",
+        "cuda/bin/crt/link.stub",
+        "cuda/bin/crt/prelink.stub",
+        "cuda/bin/cuda-gdb",
+        "cuda/bin/cuda-gdbserver",
+        "cuda/bin/cuda-memcheck",
+        "cuda/bin/cudafe",
+        "cuda/bin/cudafe++",
+        "cuda/bin/cuobjdump",
+        "cuda/bin/fatbinary",
+        "cuda/bin/gpu-library-advisor",
+        "cuda/bin/nvcc",
+        "cuda/bin/nvcc.profile",
+        "cuda/bin/nvdisasm",
+        "cuda/bin/nvlink",
+        "cuda/bin/nvprof",
+        "cuda/bin/nvprune",
+        "cuda/bin/ptxas",
+    ],
+    cmd = """cp -rf "/usr/local/cuda-9.0/bin/." "$(@D)/cuda/bin/" """,
 )
 
 genrule(
@@ -1262,7 +1279,5 @@ genrule(
     outs = [
         "cuda/include/cudnn.h",
     ],
-    cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/include/cudnn.h" "$(@D)/cudnn.h"
-   """,
+    cmd = """cp -f "/usr/include/cudnn.h" $(location cuda/include/cudnn.h) """,
 )
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/BUILD b/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/BUILD
index 04442d9c51..75e2b7fd33 100755
--- a/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/BUILD
+++ b/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/BUILD
@@ -35,9 +35,7 @@ genrule(
     outs = [
         "tensorrt/lib/libnvinfer.so.5",
     ],
-    cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/lib/x86_64-linux-gnu/libnvinfer.so.5.0.2" "$(@D)/libnvinfer.so.5"
-   """,
+    cmd = """cp -f "/usr/lib/x86_64-linux-gnu/libnvinfer.so.5" $(location tensorrt/lib/libnvinfer.so.5) """,
 )
 
 genrule(
@@ -46,7 +44,5 @@ genrule(
         "tensorrt/include/NvInfer.h",
         "tensorrt/include/NvUtils.h",
     ],
-    cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/include/x86_64-linux-gnu/NvInfer.h" "$(@D)/tensorrt/include/NvInfer.h" && cp -f "/usr/include/x86_64-linux-gnu/NvUtils.h" "$(@D)/tensorrt/include/NvUtils.h"
-   """,
+    cmd = """cp -f "/usr/include/x86_64-linux-gnu/NvInfer.h" $(location tensorrt/include/NvInfer.h) && cp -f "/usr/include/x86_64-linux-gnu/NvUtils.h" $(location tensorrt/include/NvUtils.h) """,
 )
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/build_defs.bzl b/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/build_defs.bzl
index 5c1c40361d..527be93834 100755
--- a/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/build_defs.bzl
+++ b/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/build_defs.bzl
@@ -2,6 +2,4 @@
 
 def if_tensorrt(if_true, if_false = []):
     """Tests whether TensorRT was enabled during the configure process."""
-    if True:
-        return if_true
-    return if_false
+    return if_true
-- 
GitLab


From e2968144a0a1d0344166a7996afc3cedd4381a9d Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Fri, 18 Jan 2019 18:08:42 -0800
Subject: [PATCH 1016/2345] [Grappler] AddSubgraph in MutableGraphView

PiperOrigin-RevId: 230026030
---
 tensorflow/core/grappler/graph_view.h         | 15 ++++--
 .../core/grappler/mutable_graph_view.cc       | 25 +++++++++
 tensorflow/core/grappler/mutable_graph_view.h | 10 ++++
 .../core/grappler/mutable_graph_view_test.cc  | 52 +++++++++++++++++++
 4 files changed, 98 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/grappler/graph_view.h b/tensorflow/core/grappler/graph_view.h
index 88b1e975c1..63c58a0aed 100644
--- a/tensorflow/core/grappler/graph_view.h
+++ b/tensorflow/core/grappler/graph_view.h
@@ -318,11 +318,18 @@ class GraphViewInternal {
  protected:
   explicit GraphViewInternal(GraphDefT* graph) : graph_(graph) {}
 
+  Status AddUniqueNode(NodeDefT* node) {
+    auto inserted = nodes_.emplace(node->name(), node);
+    return inserted.second
+               ? Status::OK()
+               : errors::InvalidArgument("Non unique node name detected: ",
+                                         node->name());
+  }
+
+  // TODO(ezhulenev): Remove this function.
   void AddUniqueNodeOrDie(NodeDefT* node) {
-    auto result = nodes_.emplace(node->name(), node);
-    // TODO(ezhulenev): Replace CHECK with factory method returning
-    // absl::StatusOr (when available).
-    CHECK(result.second) << "Non unique node name detected: " << node->name();
+    Status st = AddUniqueNode(node);
+    CHECK(st.ok()) << st.error_message();
   }
 
   // TODO(lyandy): Checks for self loops, Switch control dependencies, fanins
diff --git a/tensorflow/core/grappler/mutable_graph_view.cc b/tensorflow/core/grappler/mutable_graph_view.cc
index 43dec07c1d..0f171f7656 100644
--- a/tensorflow/core/grappler/mutable_graph_view.cc
+++ b/tensorflow/core/grappler/mutable_graph_view.cc
@@ -215,6 +215,31 @@ NodeDef* MutableGraphView::AddNode(NodeDef&& node) {
   return node_in_graph;
 }
 
+Status MutableGraphView::AddSubgraph(GraphDef&& subgraph) {
+  if (subgraph.library().function_size() != 0) {
+    return errors::InvalidArgument(
+        "Can't add a subgraph with non-empty function library");
+  }
+
+  int node_size_before = graph()->node_size();
+
+  for (NodeDef& node : *subgraph.mutable_node()) {
+    auto* node_in_graph = graph()->add_node();
+    *node_in_graph = std::move(node);
+    TF_RETURN_IF_ERROR(AddUniqueNode(node_in_graph));
+  }
+
+  // TODO(ezhulenev, lyandy): Right now AddAndDedupFanouts do not check that
+  // fanins actually exists in the graph, and there is already TODO for that.
+
+  for (int i = node_size_before; i < graph()->node_size(); ++i) {
+    NodeDef* node = graph()->mutable_node(i);
+    AddAndDedupFanouts(node);
+  }
+
+  return Status::OK();
+}
+
 Status MutableGraphView::UpdateFanouts(absl::string_view from_node,
                                        absl::string_view to_node) {
   NodeDef* from_node_ptr = GetNode(from_node);
diff --git a/tensorflow/core/grappler/mutable_graph_view.h b/tensorflow/core/grappler/mutable_graph_view.h
index c62129bcad..f08b5dde08 100644
--- a/tensorflow/core/grappler/mutable_graph_view.h
+++ b/tensorflow/core/grappler/mutable_graph_view.h
@@ -63,6 +63,16 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // node in graph.
   NodeDef* AddNode(NodeDef&& node);
 
+  // Adds all nodes from the `subgraph` to the underlying graph and updates the
+  // view. `subgraph` doesn't have to be a valid graph definition on it's own,
+  // it can have edges to the nodes that are not in it, however after adding
+  // it to the underlying graph, final graph must be valid.
+  //
+  // TODO(ezhulenev): Currently it will fail if subgraph has non-empty function
+  // library. Add support for adding new functions from the subgraph function
+  // library into the underlying graph.
+  Status AddSubgraph(GraphDef&& subgraph);
+
   // Updates all fanouts (input ports fetching output tensors) from `from_node`
   // to the `to_node`, including control dependencies.
   //
diff --git a/tensorflow/core/grappler/mutable_graph_view_test.cc b/tensorflow/core/grappler/mutable_graph_view_test.cc
index acfaba5ddd..ea88359e2e 100644
--- a/tensorflow/core/grappler/mutable_graph_view_test.cc
+++ b/tensorflow/core/grappler/mutable_graph_view_test.cc
@@ -141,6 +141,58 @@ void CheckGraph(const MutableGraphView& mutable_graph) {
   }
 }
 
+TEST(MutableGraphViewTest, AddSubgraph) {
+  GraphDef graph_def = test::function::GDef(
+      {
+          NDef("foo", "NotImportant", {}, {}),
+          NDef("bar", "NotImportant", {}, {}),
+          NDef("baz", "NotImportant", {"foo", "bar"}),
+      },
+      /*funcs=*/{});
+  MutableGraphView graph(&graph_def);
+
+  // `s/bar` node has inputs that are valid only if we add subgraph into the
+  // original graph.
+  GraphDef subgraph = test::function::GDef(
+      {
+          NDef("s/n0", "NotImportant", {}, {}),
+          NDef("s/n1", "NotImportant", {"bar", "s/n0"}, {}),
+      },
+      /*funcs=*/{});
+
+  TF_EXPECT_OK(graph.AddSubgraph(std::move(subgraph)));
+
+  // Fanins and fanouts must be updated for the nodes of the original graph, and
+  // added subgraph.
+  CheckNode(graph, "bar", "NotImportant", "", {}, {}, {"baz:1", "s/n1"});
+  CheckNode(graph, "s/n1", "NotImportant", "", {}, {"bar", "s/n0"}, {});
+  CheckGraph(graph);
+}
+
+// TODO(ezhulenev): Add support for adding a subgraph and merging function
+// libraries.
+TEST(MutableGraphViewTest, AddSubgraphWithFunctionLibrary) {
+  GraphDef graph_def = test::function::GDef(
+      {
+          NDef("foo", "NotImportant", {}, {}),
+          NDef("bar", "NotImportant", {}, {}),
+          NDef("baz", "NotImportant", {"foo", "bar"}),
+      },
+      /*funcs=*/{});
+  MutableGraphView graph(&graph_def);
+
+  FunctionDef x_times_two = test::function::XTimesTwo();
+  GraphDef subgraph = test::function::GDef(
+      {
+          NDef("s/n0", "NotImportant", {}, {}),
+          NDef("s/n1", "NotImportant", {"bar", "s/n0"}, {}),
+      },
+      /*funcs=*/{x_times_two});
+
+  Status status = graph.AddSubgraph(std::move(subgraph));
+  EXPECT_FALSE(status.ok());
+}
+
 TEST(MutableGraphViewTest, AddAndUpdateFanouts) {
   // Actual node.op() is not important in this test.
   GraphDef graph_def = test::function::GDef(
-- 
GitLab


From fdd4979d7be91c8b17bdc0afa56cfe012d8a7913 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 18 Jan 2019 19:15:05 -0800
Subject: [PATCH 1017/2345] Add direct dependencies to tf op libraries.

PiperOrigin-RevId: 230030481
---
 tensorflow/contrib/batching/BUILD                   |  1 +
 .../contrib/hvx/hvx_ops_support_checker/BUILD       | 13 +++++++++++++
 tensorflow/contrib/util/BUILD                       |  3 +++
 3 files changed, 17 insertions(+)

diff --git a/tensorflow/contrib/batching/BUILD b/tensorflow/contrib/batching/BUILD
index 648f3ebb05..5174afe0a6 100644
--- a/tensorflow/contrib/batching/BUILD
+++ b/tensorflow/contrib/batching/BUILD
@@ -37,6 +37,7 @@ py_library(
 cc_library(
     name = "batch_ops_kernels",
     deps = [
+        "//tensorflow/core:batch_ops_op_lib",
         "//tensorflow/core/kernels:batch_kernels",
     ],
     alwayslink = 1,
diff --git a/tensorflow/contrib/hvx/hvx_ops_support_checker/BUILD b/tensorflow/contrib/hvx/hvx_ops_support_checker/BUILD
index 0081fb6177..d319aa7986 100644
--- a/tensorflow/contrib/hvx/hvx_ops_support_checker/BUILD
+++ b/tensorflow/contrib/hvx/hvx_ops_support_checker/BUILD
@@ -16,9 +16,22 @@ tf_cc_binary(
     srcs = ["hvx_ops_support_checker_main.cc"],
     visibility = ["//visibility:public"],
     deps = [
+        "//tensorflow/core:array_ops_op_lib",
+        "//tensorflow/core:candidate_sampling_ops_op_lib",
+        "//tensorflow/core:control_flow_ops_op_lib",
         "//tensorflow/core:framework_internal",
+        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:lib",
+        "//tensorflow/core:list_ops_op_lib",
+        "//tensorflow/core:manip_ops_op_lib",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:random_ops_op_lib",
+        "//tensorflow/core:remote_fused_graph_ops_op_lib",
+        "//tensorflow/core:string_ops_op_lib",
+        "//tensorflow/core:training_ops_op_lib",
+        "//tensorflow/core:user_ops_op_lib",
         "//tensorflow/core/kernels:remote_fused_graph_execute_utils",
         "//tensorflow/core/kernels/hexagon:graph_transferer",
         "//tensorflow/tools/graph_transforms:file_utils",
diff --git a/tensorflow/contrib/util/BUILD b/tensorflow/contrib/util/BUILD
index d9ccda8e89..07dbd5ca8d 100644
--- a/tensorflow/contrib/util/BUILD
+++ b/tensorflow/contrib/util/BUILD
@@ -16,9 +16,12 @@ cc_library(
     srcs = ["convert_graphdef_memmapped_format_lib.cc"],
     hdrs = ["convert_graphdef_memmapped_format_lib.h"],
     deps = [
+        "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:framework_internal",
+        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:tensorflow",
         "//tensorflow/core/kernels:immutable_constant_op",
-- 
GitLab


From 467c1c2e215c5481372ba4e685869ae504b9e74f Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 18 Jan 2019 20:17:09 -0800
Subject: [PATCH 1018/2345] Add direct dependencies to tf op libraries.

PiperOrigin-RevId: 230034058
---
 tensorflow/c/BUILD                              |  7 +++++++
 tensorflow/compiler/jit/BUILD                   |  6 ++++++
 tensorflow/compiler/tf2xla/kernels/BUILD        |  9 +++++++++
 tensorflow/core/common_runtime/eager/BUILD      |  4 ++++
 tensorflow/core/debug/BUILD                     |  4 ++++
 tensorflow/core/distributed_runtime/BUILD       |  6 ++++++
 tensorflow/core/distributed_runtime/rpc/BUILD   |  5 +++++
 tensorflow/core/kernels/data/BUILD              |  7 +++++++
 tensorflow/core/kernels/data/experimental/BUILD |  5 +++++
 tensorflow/core/kernels/hexagon/BUILD           |  7 +++++++
 tensorflow/tools/graph_transforms/BUILD         | 14 ++++++++++++++
 11 files changed, 74 insertions(+)

diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index a48778d612..6e50a09bfc 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -289,13 +289,20 @@ tf_cuda_cc_test(
         "//tensorflow/cc/saved_model:signature_constants",
         "//tensorflow/cc/saved_model:tag_constants",
         "//tensorflow/compiler/jit",
+        "//tensorflow/core:array_ops_op_lib",
+        "//tensorflow/core:bitwise_ops_op_lib",
+        "//tensorflow/core:control_flow_ops_op_lib",
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:direct_session",
         "//tensorflow/core:framework",
         "//tensorflow/core:framework_internal",
+        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:lib",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:proto_text",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:spectral_ops_op_lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core/kernels:array",
diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index 1d72d8c2d8..55e2e6d11f 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -175,12 +175,18 @@ cc_library(
         "//tensorflow/compiler/xla/client:global_data",
         "//tensorflow/compiler/xla/client:local_client",
         "//tensorflow/compiler/xla/service:stream_pool",
+        "//tensorflow/core:array_ops_op_lib",
+        "//tensorflow/core:control_flow_ops_op_lib",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
+        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:resource_variable_ops_op_lib",
         "//tensorflow/core:stream_executor_no_cuda",
         "//tensorflow/core/kernels:cast_op",
         "//tensorflow/core/kernels:constant_op",
diff --git a/tensorflow/compiler/tf2xla/kernels/BUILD b/tensorflow/compiler/tf2xla/kernels/BUILD
index 47209d285f..52d2901e73 100644
--- a/tensorflow/compiler/tf2xla/kernels/BUILD
+++ b/tensorflow/compiler/tf2xla/kernels/BUILD
@@ -144,13 +144,22 @@ tf_kernel_library(
         "//tensorflow/compiler/xla/client/lib:quantize",
         "//tensorflow/compiler/xla/client/lib:sorting",
         "//tensorflow/compiler/xla/client/lib:triangular_solve",
+        "//tensorflow/core:bitwise_ops_op_lib",
+        "//tensorflow/core:control_flow_ops_op_lib",
         "//tensorflow/core:framework",
+        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:image_ops_op_lib",
         "//tensorflow/core:lib",
         "//tensorflow/core:linalg_ops_op_lib",
+        "//tensorflow/core:list_ops_op_lib",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:random_ops_op_lib",
+        "//tensorflow/core:resource_variable_ops_op_lib",
         "//tensorflow/core:spectral_ops_op_lib",
         "//tensorflow/core:stateless_random_ops_op_lib",
+        "//tensorflow/core:training_ops_op_lib",
         "//tensorflow/core/kernels:bounds_check",
         "//tensorflow/core/kernels:concat_lib",
         "//tensorflow/core/kernels:constant_op",
diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD
index 0a16f80c8f..370ec529be 100644
--- a/tensorflow/core/common_runtime/eager/BUILD
+++ b/tensorflow/core/common_runtime/eager/BUILD
@@ -181,11 +181,15 @@ tf_cc_test(
         "//tensorflow/cc:client_session",
         "//tensorflow/cc:ops",
         "//tensorflow/cc:scope",
+        "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:direct_session",
         "//tensorflow/core:framework",
+        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:lib",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
diff --git a/tensorflow/core/debug/BUILD b/tensorflow/core/debug/BUILD
index 591c22b8f6..e1961b822b 100644
--- a/tensorflow/core/debug/BUILD
+++ b/tensorflow/core/debug/BUILD
@@ -221,10 +221,14 @@ tf_cc_test(
     deps = [
         ":debug_grpc_testlib",
         ":debug_io_utils",
+        "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
+        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:lib",
         "//tensorflow/core:master_proto_cc",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
diff --git a/tensorflow/core/distributed_runtime/BUILD b/tensorflow/core/distributed_runtime/BUILD
index 197a3f24d4..6f09bef5ea 100644
--- a/tensorflow/core/distributed_runtime/BUILD
+++ b/tensorflow/core/distributed_runtime/BUILD
@@ -648,6 +648,7 @@ tf_cuda_cc_test(
         ":remote_device",
         ":worker_interface",
         "//tensorflow:grpc++",
+        "//tensorflow/core:control_flow_ops_op_lib",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
@@ -713,9 +714,14 @@ tf_cuda_cc_test(
     tags = tf_cuda_tests_tags(),
     deps = [
         "//tensorflow/cc:cc_ops",
+        "//tensorflow/core:array_ops_op_lib",
+        "//tensorflow/core:bitwise_ops_op_lib",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
+        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:lib",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:tensorflow",
         "//tensorflow/core:test",
diff --git a/tensorflow/core/distributed_runtime/rpc/BUILD b/tensorflow/core/distributed_runtime/rpc/BUILD
index 273709a01f..a081ec7b67 100644
--- a/tensorflow/core/distributed_runtime/rpc/BUILD
+++ b/tensorflow/core/distributed_runtime/rpc/BUILD
@@ -314,7 +314,9 @@ tf_cc_binary(
         "//tensorflow:grpc++",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework_internal",
+        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:lib",
+        "//tensorflow/core:math_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/distributed_runtime:server_lib",
         "//tensorflow/core/kernels:data_flow",
@@ -330,9 +332,12 @@ tf_cc_binary(
     deps = [
         ":grpc_server_lib",
         "//tensorflow:grpc++",
+        "//tensorflow/core:array_ops_op_lib",
+        "//tensorflow/core:bitwise_ops_op_lib",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework_internal",
         "//tensorflow/core:lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:testlib",
         "//tensorflow/core/distributed_runtime:server_lib",
diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD
index dcbdb74691..3cadb55a21 100644
--- a/tensorflow/core/kernels/data/BUILD
+++ b/tensorflow/core/kernels/data/BUILD
@@ -76,13 +76,18 @@ tf_cc_test(
     srcs = ["single_threaded_executor_test.cc"],
     deps = [
         ":single_threaded_executor",
+        "//tensorflow/core:bitwise_ops_op_lib",
+        "//tensorflow/core:control_flow_ops_op_lib",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
         "//tensorflow/core:framework_internal",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:math_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:random_ops_op_lib",
+        "//tensorflow/core:spectral_ops_op_lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
@@ -566,6 +571,8 @@ tf_kernel_library(
         ":tensor_slice_dataset_op",
         ":window_dataset_op",
         ":zip_dataset_op",
+        "//tensorflow/core:array_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core/kernels/data/experimental:dataset_kernels",
     ],
 )
diff --git a/tensorflow/core/kernels/data/experimental/BUILD b/tensorflow/core/kernels/data/experimental/BUILD
index 2c3ffad375..4f7c8f156c 100644
--- a/tensorflow/core/kernels/data/experimental/BUILD
+++ b/tensorflow/core/kernels/data/experimental/BUILD
@@ -119,12 +119,14 @@ tf_kernel_library(
     name = "map_and_batch_dataset_op",
     srcs = ["map_and_batch_dataset_op.cc"],
     deps = [
+        "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:dataset_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:metrics",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core/kernels:inplace_ops",
         "//tensorflow/core/kernels/data:captured_function",
         "//tensorflow/core/kernels/data:dataset_utils",
@@ -169,11 +171,13 @@ tf_kernel_library(
     name = "numa_map_and_batch_dataset_op",
     srcs = ["numa_map_and_batch_dataset_op.cc"],
     deps = [
+        "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:experimental_dataset_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core/kernels:inplace_ops",
         "//tensorflow/core/kernels/data:captured_function",
         "@com_google_absl//absl/memory",
@@ -200,6 +204,7 @@ tf_kernel_library(
     deps = [
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
+        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core/kernels/data:parallel_map_iterator",
     ],
 )
diff --git a/tensorflow/core/kernels/hexagon/BUILD b/tensorflow/core/kernels/hexagon/BUILD
index 87d36f22d7..a85de34ac2 100644
--- a/tensorflow/core/kernels/hexagon/BUILD
+++ b/tensorflow/core/kernels/hexagon/BUILD
@@ -24,11 +24,18 @@ tf_cc_test(
     deps = [
         ":graph_transferer",
         "//tensorflow/cc:cc_ops",
+        "//tensorflow/core:array_ops_op_lib",
+        "//tensorflow/core:bitwise_ops_op_lib",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:direct_session",
+        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:lib",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:mkl_nn_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:ops",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:remote_fused_graph_ops_op_lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
diff --git a/tensorflow/tools/graph_transforms/BUILD b/tensorflow/tools/graph_transforms/BUILD
index f229099e49..41ed31a5c1 100644
--- a/tensorflow/tools/graph_transforms/BUILD
+++ b/tensorflow/tools/graph_transforms/BUILD
@@ -136,6 +136,19 @@ cc_library(
     ] + if_not_windows([
         "//tensorflow/core/kernels:remote_fused_graph_rewriter_transform",
         "//tensorflow/core/kernels/hexagon:hexagon_rewriter_transform",
+        "//tensorflow/core:user_ops_op_lib",
+        "//tensorflow/core:training_ops_op_lib",
+        "//tensorflow/core:string_ops_op_lib",
+        "//tensorflow/core:remote_fused_graph_ops_op_lib",
+        "//tensorflow/core:random_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:manip_ops_op_lib",
+        "//tensorflow/core:list_ops_op_lib",
+        "//tensorflow/core:functional_ops_op_lib",
+        "//tensorflow/core:control_flow_ops_op_lib",
+        "//tensorflow/core:candidate_sampling_ops_op_lib",
+        "//tensorflow/core:array_ops_op_lib",
     ]) + if_not_v2([
         "//tensorflow/contrib/rnn:gru_ops_op_lib",
         "//tensorflow/contrib/rnn:lstm_ops_op_lib",
@@ -175,6 +188,7 @@ tf_cc_test(
         ":transforms_lib",
         "//tensorflow/cc:cc_ops",
         "//tensorflow/cc:sendrecv_ops",
+        "//tensorflow/core:bitwise_ops_op_lib",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
-- 
GitLab


From f52351444551016d7dd949a5aa599da489a97045 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 19 Jan 2019 01:03:16 -0800
Subject: [PATCH 1019/2345] compat: Update forward compatibility horizon to
 2019-01-19

PiperOrigin-RevId: 230049974
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index ccc0735b6e..14b9a562cd 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 18)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 19)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From ceaf6feb124c11f1188e5a5739cc099d52658c41 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sat, 19 Jan 2019 20:56:44 +0000
Subject: [PATCH 1020/2345] Fix sparse_to_dense warning in 25043

This fix fixes sparse_to_dense warning raised in 25043.

This fix fixes 25043.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/ops/nn_impl.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/nn_impl.py b/tensorflow/python/ops/nn_impl.py
index 7abfde5149..17174a313d 100644
--- a/tensorflow/python/ops/nn_impl.py
+++ b/tensorflow/python/ops/nn_impl.py
@@ -32,7 +32,7 @@ from tensorflow.python.ops import gen_array_ops  # pylint: disable=unused-import
 from tensorflow.python.ops import gen_nn_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn_ops
-from tensorflow.python.ops import sparse_ops
+from tensorflow.python.ops import gen_sparse_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.util.deprecation import deprecated_args
 from tensorflow.python.util.deprecation import deprecated_argument_lookup
@@ -1440,7 +1440,7 @@ def _compute_sampled_logits(weights,
            array_ops.expand_dims(num_sampled, 0)], 0)
       if sampled_logits.dtype != acc_weights.dtype:
         acc_weights = math_ops.cast(acc_weights, sampled_logits.dtype)
-      sampled_logits += sparse_ops.sparse_to_dense(
+      sampled_logits += gen_sparse_ops.sparse_to_dense(
           sparse_indices,
           sampled_logits_shape,
           acc_weights,
-- 
GitLab


From d28aabb83228524df1b906c66b35bdb5657a59b5 Mon Sep 17 00:00:00 2001
From: David Majnemer <majnemer@google.com>
Date: Sat, 19 Jan 2019 14:18:03 -0800
Subject: [PATCH 1021/2345] [XLA] Optimize division/remainder by a power of 2

X / Y => X >> log_2(y)
X % Y => X & (Y-1)

PiperOrigin-RevId: 230081985
---
 .../xla/service/algebraic_simplifier.cc       | 262 ++++++++++++++++++
 .../xla/service/algebraic_simplifier_test.cc  |  80 ++++++
 .../compiler/xla/service/gather_expander.cc   |  17 +-
 .../xla/service/hlo_creation_utils.cc         |  12 +-
 .../compiler/xla/service/hlo_creation_utils.h |  12 +-
 .../xla/service/hlo_creation_utils_test.cc    |  10 +-
 .../compiler/xla/service/scatter_expander.cc  |  10 +-
 7 files changed, 369 insertions(+), 34 deletions(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index 8a02c48146..da15ff7d7a 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -36,6 +36,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/layout_util.h"
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/compiler/xla/primitive_util.h"
 #include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h"
 #include "tensorflow/compiler/xla/service/hlo_casting_utils.h"
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
@@ -52,6 +53,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/util.h"
 #include "tensorflow/compiler/xla/window_util.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
+#include "tensorflow/core/lib/core/bits.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/logging.h"
@@ -219,6 +221,8 @@ class AlgebraicSimplifierVisitor : public DfsHloVisitorWithDefault {
 
   Status HandlePower(HloInstruction* power) override;
 
+  Status HandleRemainder(HloInstruction* remainder) override;
+
   Status HandleReshape(HloInstruction* reshape) override;
 
   Status HandleReduce(HloInstruction* reduce) override;
@@ -812,6 +816,79 @@ Status InvertConstant(const HloInstruction& constant, Literal* result) {
     return T{1.0} / constant.literal().Get<T>(indices);
   });
 }
+
+template <typename T>
+std::unique_ptr<HloInstruction> TryDivideToShift(HloInstruction* divide,
+                                                 HloComputation* computation) {
+  HloInstruction *a, *b, *c;
+  CHECK(Match(divide, m::Divide(m::Op(&a), m::Op(&b))));
+
+  if (ShapeUtil::ElementIsIntegral(divide->shape()) &&
+      !Match(b, m::ConstantEffectiveScalar(&c)) &&
+      !Match(b, m::Broadcast(m::ConstantEffectiveScalar(&c)))) {
+    return nullptr;
+  }
+
+  if (ShapeUtil::ElementIsSigned(divide->shape())) {
+    int64 b_value = c->literal().GetFirstElement<T>();
+    if (b_value > 0 && IsPowerOfTwo(static_cast<uint64>(b_value))) {
+      // Handle negative dividends by negating the result of the division.
+      HloInstruction* zero_like_a = BroadcastZeros(
+          computation, a->shape().element_type(), a->shape().dimensions());
+
+      auto* dividend_is_negative =
+          computation->AddInstruction(HloInstruction::CreateBinary(
+              ShapeUtil::ChangeElementType(a->shape(), PRED), HloOpcode::kLt, a,
+              zero_like_a));
+
+      auto* negated_dividend = computation->AddInstruction(
+          HloInstruction::CreateUnary(a->shape(), HloOpcode::kNegate, a));
+
+      auto* abs_dividend =
+          computation->AddInstruction(HloInstruction::CreateTernary(
+              a->shape(), HloOpcode::kSelect, dividend_is_negative,
+              negated_dividend, a));
+
+      int log2_abs_b_value = tensorflow::Log2Floor64(b_value);
+
+      auto* shift_amount =
+          computation->AddInstruction(HloInstruction::CreateConstant(
+              LiteralUtil::CreateR0<T>(log2_abs_b_value)));
+      if (!ShapeUtil::IsScalar(b->shape())) {
+        shift_amount = computation->AddInstruction(
+            HloInstruction::CreateBroadcast(b->shape(), shift_amount, {}));
+      }
+
+      auto* quotient = computation->AddInstruction(HloInstruction::CreateBinary(
+          divide->shape(), HloOpcode::kShiftRightLogical, abs_dividend,
+          shift_amount));
+
+      auto* neqated_quotient =
+          computation->AddInstruction(HloInstruction::CreateUnary(
+              quotient->shape(), HloOpcode::kNegate, quotient));
+
+      return HloInstruction::CreateTernary(divide->shape(), HloOpcode::kSelect,
+                                           dividend_is_negative,
+                                           neqated_quotient, quotient);
+    }
+  } else {
+    uint64 b_value = c->literal().GetFirstElement<T>();
+    if (IsPowerOfTwo(b_value)) {
+      int log2_abs_b_value = tensorflow::Log2Floor64(b_value);
+      HloInstruction* shift_amount =
+          computation->AddInstruction(HloInstruction::CreateConstant(
+              LiteralUtil::CreateR0<T>(log2_abs_b_value)));
+      if (!ShapeUtil::IsScalar(b->shape())) {
+        shift_amount = computation->AddInstruction(
+            HloInstruction::CreateBroadcast(b->shape(), shift_amount, {}));
+      }
+      return HloInstruction::CreateBinary(
+          divide->shape(), HloOpcode::kShiftRightLogical, a, shift_amount);
+    }
+  }
+
+  return nullptr;
+}
 }  // namespace
 
 Status AlgebraicSimplifierVisitor::HandleDivide(HloInstruction* divide) {
@@ -824,6 +901,60 @@ Status AlgebraicSimplifierVisitor::HandleDivide(HloInstruction* divide) {
     return Status::OK();
   }
 
+  // A / B => A >> log2(B) if B is a power of 2.
+  switch (divide->shape().element_type()) {
+    case S8:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryDivideToShift<int8>(divide, computation_)) {
+        return ReplaceWithNewInstruction(divide, std::move(shift));
+      }
+      break;
+    case S16:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryDivideToShift<int16>(divide, computation_)) {
+        return ReplaceWithNewInstruction(divide, std::move(shift));
+      }
+      break;
+    case S32:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryDivideToShift<int32>(divide, computation_)) {
+        return ReplaceWithNewInstruction(divide, std::move(shift));
+      }
+      break;
+    case S64:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryDivideToShift<int64>(divide, computation_)) {
+        return ReplaceWithNewInstruction(divide, std::move(shift));
+      }
+      break;
+    case U8:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryDivideToShift<uint8>(divide, computation_)) {
+        return ReplaceWithNewInstruction(divide, std::move(shift));
+      }
+      break;
+    case U16:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryDivideToShift<uint16>(divide, computation_)) {
+        return ReplaceWithNewInstruction(divide, std::move(shift));
+      }
+      break;
+    case U32:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryDivideToShift<uint32>(divide, computation_)) {
+        return ReplaceWithNewInstruction(divide, std::move(shift));
+      }
+      break;
+    case U64:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryDivideToShift<uint64>(divide, computation_)) {
+        return ReplaceWithNewInstruction(divide, std::move(shift));
+      }
+      break;
+    default:
+      break;
+  }
+
   // exp(A)/exp(B) => exp(A-B)
   if (Match(divide, m::Divide(m::Exp(m::Op(&a)), m::Exp(m::Op(&b)))
                         .WithShape(m::Shape(&shape)))) {
@@ -2159,6 +2290,137 @@ AlgebraicSimplifierVisitor::TryToSinkBroadcastAfterOpWithUniqueNonScalarOperand(
   return changed;
 }
 
+namespace {
+template <typename T>
+std::unique_ptr<HloInstruction> TryRemainderToAnd(HloInstruction* remainder,
+                                                  HloComputation* computation) {
+  HloInstruction *a, *b, *c;
+  CHECK(Match(remainder, m::Remainder(m::Op(&a), m::Op(&b))));
+
+  if (ShapeUtil::ElementIsIntegral(remainder->shape()) &&
+      !Match(b, m::ConstantEffectiveScalar(&c)) &&
+      !Match(b, m::Broadcast(m::ConstantEffectiveScalar(&c)))) {
+    return nullptr;
+  }
+
+  if (ShapeUtil::ElementIsSigned(remainder->shape())) {
+    int64 b_value = c->literal().GetFirstElement<T>();
+    if (b_value > 0 && IsPowerOfTwo(static_cast<uint64>(b_value))) {
+      // Handle negative dividends by negating the result of the division.
+      HloInstruction* zero_like_a = BroadcastZeros(
+          computation, a->shape().element_type(), a->shape().dimensions());
+
+      auto* dividend_is_negative =
+          computation->AddInstruction(HloInstruction::CreateBinary(
+              ShapeUtil::ChangeElementType(a->shape(), PRED), HloOpcode::kLt, a,
+              zero_like_a));
+
+      auto* negated_dividend = computation->AddInstruction(
+          HloInstruction::CreateUnary(a->shape(), HloOpcode::kNegate, a));
+
+      auto* abs_dividend =
+          computation->AddInstruction(HloInstruction::CreateTernary(
+              a->shape(), HloOpcode::kSelect, dividend_is_negative,
+              negated_dividend, a));
+
+      auto* mask_amount =
+          computation->AddInstruction(HloInstruction::CreateConstant(
+              LiteralUtil::CreateR0<T>(b_value - 1)));
+      if (!ShapeUtil::IsScalar(b->shape())) {
+        mask_amount = computation->AddInstruction(
+            HloInstruction::CreateBroadcast(b->shape(), mask_amount, {}));
+      }
+
+      auto* quotient = computation->AddInstruction(HloInstruction::CreateBinary(
+          remainder->shape(), HloOpcode::kAnd, abs_dividend, mask_amount));
+
+      auto* neqated_quotient =
+          computation->AddInstruction(HloInstruction::CreateUnary(
+              quotient->shape(), HloOpcode::kNegate, quotient));
+
+      return HloInstruction::CreateTernary(
+          remainder->shape(), HloOpcode::kSelect, dividend_is_negative,
+          neqated_quotient, quotient);
+    }
+  } else {
+    uint64 b_value = c->literal().GetFirstElement<T>();
+    if (IsPowerOfTwo(b_value)) {
+      HloInstruction* mask_amount =
+          computation->AddInstruction(HloInstruction::CreateConstant(
+              LiteralUtil::CreateR0<T>(b_value - 1)));
+      if (!ShapeUtil::IsScalar(b->shape())) {
+        mask_amount = computation->AddInstruction(
+            HloInstruction::CreateBroadcast(b->shape(), mask_amount, {}));
+      }
+      return HloInstruction::CreateBinary(remainder->shape(), HloOpcode::kAnd,
+                                          a, mask_amount);
+    }
+  }
+  return nullptr;
+}
+}  // namespace
+
+Status AlgebraicSimplifierVisitor::HandleRemainder(HloInstruction* remainder) {
+  HloInstruction *a, *b;
+  CHECK(Match(remainder, m::Remainder(m::Op(&a), m::Op(&b))));
+
+  // A % B => A & (B - 1) if B is a power of 2.
+  switch (remainder->shape().element_type()) {
+    case S8:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryRemainderToAnd<int8>(remainder, computation_)) {
+        return ReplaceWithNewInstruction(remainder, std::move(shift));
+      }
+      break;
+    case S16:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryRemainderToAnd<int16>(remainder, computation_)) {
+        return ReplaceWithNewInstruction(remainder, std::move(shift));
+      }
+      break;
+    case S32:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryRemainderToAnd<int32>(remainder, computation_)) {
+        return ReplaceWithNewInstruction(remainder, std::move(shift));
+      }
+      break;
+    case S64:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryRemainderToAnd<int64>(remainder, computation_)) {
+        return ReplaceWithNewInstruction(remainder, std::move(shift));
+      }
+      break;
+    case U8:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryRemainderToAnd<uint8>(remainder, computation_)) {
+        return ReplaceWithNewInstruction(remainder, std::move(shift));
+      }
+      break;
+    case U16:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryRemainderToAnd<uint16>(remainder, computation_)) {
+        return ReplaceWithNewInstruction(remainder, std::move(shift));
+      }
+      break;
+    case U32:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryRemainderToAnd<uint32>(remainder, computation_)) {
+        return ReplaceWithNewInstruction(remainder, std::move(shift));
+      }
+      break;
+    case U64:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryRemainderToAnd<uint64>(remainder, computation_)) {
+        return ReplaceWithNewInstruction(remainder, std::move(shift));
+      }
+      break;
+    default:
+      break;
+  }
+
+  return Status::OK();
+}
+
 Status AlgebraicSimplifierVisitor::HandleReshape(HloInstruction* reshape) {
   auto operand = reshape->mutable_operand(0);
 
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index 0dea498456..96354e6f6e 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -194,6 +194,86 @@ TEST_F(AlgebraicSimplifierTest, FactorFpAdditionBfloat16) {
                   m::Broadcast(m::ConstantScalar(0.125)))));
 }
 
+TEST_F(AlgebraicSimplifierTest, UnsignedDivideByPowerOf2) {
+  const char* kModuleStr = R"(
+    HloModule m
+    test {
+      p = u32[4] parameter(0)
+      c = u32[] constant(8)
+      b = u32[4] broadcast(c), dimensions={}
+      ROOT d = u32[4] divide(p, b)
+    }
+  )";
+  TF_ASSERT_OK_AND_ASSIGN(auto m, ParseAndReturnVerifiedModule(kModuleStr));
+  ASSERT_TRUE(AlgebraicSimplifier(default_options_).Run(m.get()).ValueOrDie());
+  EXPECT_THAT(m->entry_computation()->root_instruction(),
+              GmockMatch(m::ShiftRightLogical(
+                  m::Parameter(0), m::Broadcast(m::ConstantScalar(3)))));
+}
+
+TEST_F(AlgebraicSimplifierTest, SignedDivideByPowerOf2) {
+  const char* kModuleStr = R"(
+    HloModule m
+    test {
+      p = s32[4] parameter(0)
+      c = s32[] constant(8)
+      b = s32[4] broadcast(c), dimensions={}
+      ROOT d = s32[4] divide(p, b)
+    }
+  )";
+  TF_ASSERT_OK_AND_ASSIGN(auto m, ParseAndReturnVerifiedModule(kModuleStr));
+  ASSERT_TRUE(AlgebraicSimplifier(default_options_).Run(m.get()).ValueOrDie());
+  auto match_dividend_is_negative =
+      m::Lt(m::Parameter(0), m::Broadcast(m::ConstantScalar(0)));
+  auto match_abs = m::Select(match_dividend_is_negative,
+                             m::Negate(m::Parameter(0)), m::Parameter(0));
+  auto match_shift =
+      m::ShiftRightLogical(match_abs, m::Broadcast(m::ConstantScalar(3)));
+  EXPECT_THAT(m->entry_computation()->root_instruction(),
+              GmockMatch(m::Select(match_dividend_is_negative,
+                                   m::Negate(match_shift), match_shift)));
+}
+
+TEST_F(AlgebraicSimplifierTest, UnsignedRemainderByPowerOf2) {
+  const char* kModuleStr = R"(
+    HloModule m
+    test {
+      p = u32[4] parameter(0)
+      c = u32[] constant(8)
+      b = u32[4] broadcast(c), dimensions={}
+      ROOT r = u32[4] remainder(p, b)
+    }
+  )";
+  TF_ASSERT_OK_AND_ASSIGN(auto m, ParseAndReturnVerifiedModule(kModuleStr));
+  ASSERT_TRUE(AlgebraicSimplifier(default_options_).Run(m.get()).ValueOrDie());
+  EXPECT_THAT(m->entry_computation()->root_instruction(),
+              GmockMatch(m::AndAnyOrder(m::Parameter(0),
+                                        m::Broadcast(m::ConstantScalar(7)))));
+}
+
+TEST_F(AlgebraicSimplifierTest, SignedRemainderByPowerOf2) {
+  const char* kModuleStr = R"(
+    HloModule m
+    test {
+      p = s32[4] parameter(0)
+      c = s32[] constant(8)
+      b = s32[4] broadcast(c), dimensions={}
+      ROOT r = s32[4] remainder(p, b)
+    }
+  )";
+  TF_ASSERT_OK_AND_ASSIGN(auto m, ParseAndReturnVerifiedModule(kModuleStr));
+  ASSERT_TRUE(AlgebraicSimplifier(default_options_).Run(m.get()).ValueOrDie());
+  auto match_dividend_is_negative =
+      m::Lt(m::Parameter(0), m::Broadcast(m::ConstantScalar(0)));
+  auto match_abs = m::Select(match_dividend_is_negative,
+                             m::Negate(m::Parameter(0)), m::Parameter(0));
+  auto match_and =
+      m::AndAnyOrder(match_abs, m::Broadcast(m::ConstantScalar(7)));
+  EXPECT_THAT(m->entry_computation()->root_instruction(),
+              GmockMatch(m::Select(match_dividend_is_negative,
+                                   m::Negate(match_and), match_and)));
+}
+
 // Test that A * 0 is simplified to 0
 TEST_F(AlgebraicSimplifierTest, MulZero) {
   auto m = CreateNewVerifiedModule();
diff --git a/tensorflow/compiler/xla/service/gather_expander.cc b/tensorflow/compiler/xla/service/gather_expander.cc
index 01cef49966..590942cddc 100644
--- a/tensorflow/compiler/xla/service/gather_expander.cc
+++ b/tensorflow/compiler/xla/service/gather_expander.cc
@@ -153,10 +153,9 @@ static StatusOr<std::vector<HloInstruction*>> GatherLoopBody(
            dim_numbers.index_vector_dim() ==
                gather.operand(1)->shape().dimensions_size());
 
-  TF_ASSIGN_OR_RETURN(
-      HloInstruction * induction_var_as_vector,
+  HloInstruction* induction_var_as_vector =
       MakeBroadcastHlo(induction_var, /*broadcast_dimensions=*/{},
-                       /*result_shape_bounds=*/{1}));
+                       /*result_shape_bounds=*/{1});
 
   HloInstruction* index_vector;
 
@@ -222,7 +221,7 @@ static StatusOr<std::vector<HloInstruction*>> GatherLoopBody(
       {operand, start_indices, updated_accumulator}};
 }
 
-static StatusOr<HloInstruction*> CreateGatherLoopAccumulatorInitValue(
+static HloInstruction* CreateGatherLoopAccumulatorInitValue(
     HloComputation* computation, PrimitiveType element_type,
     absl::Span<const int64> slice_sizes, int64 gather_loop_trip_count,
     const GatherDimensionNumbers& dim_numbers) {
@@ -332,12 +331,10 @@ StatusOr<HloInstruction*> GatherExpander::ExpandGather(
   CHECK_EQ(gather_loop_trip_count,
            canonical_start_indices->shape().dimensions(0));
 
-  TF_ASSIGN_OR_RETURN(
-      HloInstruction * accumulator_init,
-      CreateGatherLoopAccumulatorInitValue(
-          computation, output_shape.element_type(),
-          gather_instr->gather_slice_sizes(), gather_loop_trip_count,
-          gather_instr->gather_dimension_numbers()));
+  HloInstruction* accumulator_init = CreateGatherLoopAccumulatorInitValue(
+      computation, output_shape.element_type(),
+      gather_instr->gather_slice_sizes(), gather_loop_trip_count,
+      gather_instr->gather_dimension_numbers());
 
   StatusOr<std::vector<HloInstruction*>> gather_loop_result_or_error =
       WhileUtil::MakeCountedLoop(
diff --git a/tensorflow/compiler/xla/service/hlo_creation_utils.cc b/tensorflow/compiler/xla/service/hlo_creation_utils.cc
index bb5d21c654..d56f673455 100644
--- a/tensorflow/compiler/xla/service/hlo_creation_utils.cc
+++ b/tensorflow/compiler/xla/service/hlo_creation_utils.cc
@@ -156,9 +156,9 @@ StatusOr<HloInstruction*> MakeDynamicUpdateSliceHlo(
       dynamic_update_slice_shape, operand, update, scalar_start_indices));
 }
 
-StatusOr<HloInstruction*> MakeBroadcastHlo(
-    HloInstruction* operand, absl::Span<const int64> broadcast_dimensions,
-    absl::Span<const int64> result_shape_bounds) {
+HloInstruction* MakeBroadcastHlo(HloInstruction* operand,
+                                 absl::Span<const int64> broadcast_dimensions,
+                                 absl::Span<const int64> result_shape_bounds) {
   HloComputation* computation = operand->parent();
   Shape broadcast_shape = ShapeUtil::MakeShape(operand->shape().element_type(),
                                                result_shape_bounds);
@@ -394,9 +394,9 @@ StatusOr<HloInstruction*> PadVectorWithZeros(HloInstruction* operand,
   return MakePadHlo(operand, zero, padding_config);
 }
 
-StatusOr<HloInstruction*> BroadcastZeros(
-    HloComputation* computation, PrimitiveType element_type,
-    absl::Span<const int64> broadcast_dimensions) {
+HloInstruction* BroadcastZeros(HloComputation* computation,
+                               PrimitiveType element_type,
+                               absl::Span<const int64> broadcast_dimensions) {
   HloInstruction* zero = computation->AddInstruction(
       HloInstruction::CreateConstant(LiteralUtil::Zero(element_type)));
   return MakeBroadcastHlo(zero, /*broadcast_dimensions=*/{},
diff --git a/tensorflow/compiler/xla/service/hlo_creation_utils.h b/tensorflow/compiler/xla/service/hlo_creation_utils.h
index 8e5ddbbd50..1c3174e9c8 100644
--- a/tensorflow/compiler/xla/service/hlo_creation_utils.h
+++ b/tensorflow/compiler/xla/service/hlo_creation_utils.h
@@ -82,9 +82,9 @@ StatusOr<HloInstruction*> MakeDynamicUpdateSliceHlo(
 
 // Creates a broadcast HLO instruction and adds it to the computation containing
 // `operand`.
-StatusOr<HloInstruction*> MakeBroadcastHlo(
-    HloInstruction* operand, absl::Span<const int64> broadcast_dimensions,
-    absl::Span<const int64> result_shape_bounds);
+HloInstruction* MakeBroadcastHlo(HloInstruction* operand,
+                                 absl::Span<const int64> broadcast_dimensions,
+                                 absl::Span<const int64> result_shape_bounds);
 
 // Creates a GetTupleElement HLO instruction and adds it to the computation
 // containing `operand`.
@@ -198,9 +198,9 @@ StatusOr<HloInstruction*> PadVectorWithZeros(HloInstruction* operand,
 // Broadcasts a zero value of type `element_type` into a tensor with element
 // type `element_type` and dimension bounds `broadcast_dimensions`.  The
 // broadcast instruction is emitted into `computation`.
-StatusOr<HloInstruction*> BroadcastZeros(
-    HloComputation* computation, PrimitiveType element_type,
-    absl::Span<const int64> broadcast_dimensions);
+HloInstruction* BroadcastZeros(HloComputation* computation,
+                               PrimitiveType element_type,
+                               absl::Span<const int64> broadcast_dimensions);
 
 // Creates a HLO computation that takes arguments of type `domain` and produces
 // a value of type `range`.
diff --git a/tensorflow/compiler/xla/service/hlo_creation_utils_test.cc b/tensorflow/compiler/xla/service/hlo_creation_utils_test.cc
index 3715e12b4e..6025e6a779 100644
--- a/tensorflow/compiler/xla/service/hlo_creation_utils_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_creation_utils_test.cc
@@ -191,9 +191,8 @@ TEST_F(HloCreationUtilsTest, BroadcastZeros_S32) {
                                              /*output_shape_dims=*/{2, 2},
                                              &param, &entry_computation);
 
-  TF_ASSERT_OK_AND_ASSIGN(
-      HloInstruction * zeros,
-      BroadcastZeros(module->entry_computation(), S32, {2, 2}));
+  HloInstruction* zeros =
+      BroadcastZeros(module->entry_computation(), S32, {2, 2});
   entry_computation->set_root_instruction(zeros);
 
   HloEvaluator evaluator;
@@ -211,9 +210,8 @@ TEST_F(HloCreationUtilsTest, BroadcastZeros_F32) {
                                              /*output_shape_dims=*/{2, 2},
                                              &param, &entry_computation);
 
-  TF_ASSERT_OK_AND_ASSIGN(
-      HloInstruction * zeros,
-      BroadcastZeros(module->entry_computation(), F32, {2, 2}));
+  HloInstruction* zeros =
+      BroadcastZeros(module->entry_computation(), F32, {2, 2});
   entry_computation->set_root_instruction(zeros);
 
   HloEvaluator evaluator;
diff --git a/tensorflow/compiler/xla/service/scatter_expander.cc b/tensorflow/compiler/xla/service/scatter_expander.cc
index 036c3c36f6..acad871c4d 100644
--- a/tensorflow/compiler/xla/service/scatter_expander.cc
+++ b/tensorflow/compiler/xla/service/scatter_expander.cc
@@ -171,10 +171,9 @@ static StatusOr<HloInstruction*> CheckIndexValidity(
   // Valid range for the index: [0, operand_dims - window_sizes]
 
   // Check if the index has any negative values.
-  TF_ASSIGN_OR_RETURN(
-      HloInstruction * zero_index,
+  HloInstruction* zero_index =
       BroadcastZeros(computation, index->shape().element_type(),
-                     AsInt64Slice(index->shape().dimensions())));
+                     AsInt64Slice(index->shape().dimensions()));
   TF_ASSIGN_OR_RETURN(HloInstruction * negative_index_check,
                       MakeBinaryHlo(HloOpcode::kLe, zero_index, index));
 
@@ -222,10 +221,9 @@ static StatusOr<std::vector<HloInstruction*>> ScatterLoopBody(
   bool has_scalar_indices = scatter_indices->shape().dimensions_size() == 1;
 
   // Build a vector form of the induction variable of the while loop.
-  TF_ASSIGN_OR_RETURN(
-      HloInstruction * induction_var_as_vector,
+  HloInstruction* induction_var_as_vector =
       MakeBroadcastHlo(induction_var, /*broadcast_dimensions=*/{},
-                       /*result_shape_bounds=*/{1}));
+                       /*result_shape_bounds=*/{1});
 
   // Pick the index to scatter from scatter_indices based on the induction_var
   // and transform that to an index into the `operand` space.
-- 
GitLab


From 12e86468e2a9b57e636c1d0afcdf3f657f6df0b6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 20 Jan 2019 01:03:06 -0800
Subject: [PATCH 1022/2345] compat: Update forward compatibility horizon to
 2019-01-20

PiperOrigin-RevId: 230110846
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 14b9a562cd..16fe3db26e 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 19)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 20)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 87be20d81482b86d8c73575eb8e9b702cf86e0e0 Mon Sep 17 00:00:00 2001
From: Gautam <gautamrbharadwaj@gmail.com>
Date: Sun, 20 Jan 2019 21:44:12 +0530
Subject: [PATCH 1023/2345] Update Adam.py

---
 tensorflow/python/training/adam.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tensorflow/python/training/adam.py b/tensorflow/python/training/adam.py
index b80fb03111..2932851d21 100644
--- a/tensorflow/python/training/adam.py
+++ b/tensorflow/python/training/adam.py
@@ -102,9 +102,6 @@ class AdamOptimizer(optimizer.Optimizer):
     self._beta2_t = None
     self._epsilon_t = None
 
-    # Created in SparseApply if needed.
-    self._updated_lr = None
-
   def _get_beta_accumulators(self):
     with ops.init_scope():
       if context.executing_eagerly():
-- 
GitLab


From 329bd829d234165d235d2a7d47c3b2d8c8ca114f Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Sun, 20 Jan 2019 14:48:27 -0800
Subject: [PATCH 1024/2345] Fix typos in TFL Micro documentation

PiperOrigin-RevId: 230148789
---
 tensorflow/lite/experimental/micro/README.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/lite/experimental/micro/README.md b/tensorflow/lite/experimental/micro/README.md
index e85b9660e3..6568ecd0fa 100644
--- a/tensorflow/lite/experimental/micro/README.md
+++ b/tensorflow/lite/experimental/micro/README.md
@@ -65,7 +65,7 @@ you can easily explore the code in a cross-platform IDE.
 
 ## Building Portable Reference Code using Make
 
-It's easy to build portable reference code directly from Github using make if
+It's easy to build portable reference code directly from GitHub using make if
 you're on a Linux or OS X machine.
 
 -   Open a terminal
@@ -261,7 +261,7 @@ You should see the following log with the magic string `~~~ALL TEST PASSED~~~`:
 02:25:22.4253 [DEBUG] uart0: [+0.16ms host +0s virt 0.28s virt from start]   Progam has exited with code:0x00000000
 ```
 
-## Building for Ambiq Micro Apollo3Blue EVB
+## Building for Ambiq Micro Apollo3Blue EVB using Make
 
 Follow these steps to get the pushbutton yes/no example working on Apollo 3:
 
@@ -713,7 +713,7 @@ exist.
 
 So far, I've recommended that you use the standalone generated projects for your
 system. You might be wondering why you're not just checking out the full
-[TensorFlow codebase from Github](https://github.com/tensorflow/tensorflow/)?
+[TensorFlow codebase from GitHub](https://github.com/tensorflow/tensorflow/)?
 The main reason is that there is a lot more diversity of architectures, IDEs,
 support libraries, and operating systems in the embedded world. Many of the
 toolchains require their own copy of source files, or a list of sources to be
@@ -749,7 +749,7 @@ wider community? The first step is to pick a tag name for your platform. This
 can either be the operating system (for example 'mbed'), the name of a device
 ('bluepill'), or some other text that describes it. This should be a short
 string with no spaces or special characters. Log in or create an account on
-Github, fork the full
+GitHub, fork the full
 [TensorFlow codebase](https://github.com/tensorflow/tensorflow/) using the
 'Fork' button on the top left, and then grab your fork by using a command like
 `git clone https://github.com/<your user name>/tensorflow`.
-- 
GitLab


From eb17145a7e8f8d50418d0238e8dbd445ea2bd1d7 Mon Sep 17 00:00:00 2001
From: Alan Chiao <alanchiao@google.com>
Date: Sun, 20 Jan 2019 14:53:09 -0800
Subject: [PATCH 1025/2345] Refresh open-source perturbed test acoustic model
 to be compatible with today's kernels.

Add equivalent model with post-training quantization.

PiperOrigin-RevId: 230148964
---
 tensorflow/lite/models/speech_test.cc         | 15 +++-
 .../lite/models/testdata/g3doc/README.md      | 90 ++++++++-----------
 2 files changed, 50 insertions(+), 55 deletions(-)

diff --git a/tensorflow/lite/models/speech_test.cc b/tensorflow/lite/models/speech_test.cc
index f3509d1ece..a3713c5531 100644
--- a/tensorflow/lite/models/speech_test.cc
+++ b/tensorflow/lite/models/speech_test.cc
@@ -139,7 +139,7 @@ TEST_P(SpeechTest, DISABLED_SpeakerIdOkGoogleTest) {
       << test_driver.GetErrorMessage();
 }
 
-TEST_P(SpeechTest, DISABLED_AsrAmTest) {
+TEST_P(SpeechTest, AsrAmTest) {
   std::stringstream os;
   ASSERT_TRUE(
       ConvertCsvData("speech_asr_am_model.tflite", "speech_asr_am_model_in.csv",
@@ -152,6 +152,19 @@ TEST_P(SpeechTest, DISABLED_AsrAmTest) {
       << test_driver.GetErrorMessage();
 }
 
+TEST_P(SpeechTest, AsrAmQuantizedTest) {
+  std::stringstream os;
+  ASSERT_TRUE(ConvertCsvData(
+      "speech_asr_am_model_int8.tflite", "speech_asr_am_model_in.csv",
+      "speech_asr_am_model_int8_out.csv", /*input_tensor=*/"0",
+      /*output_tensor=*/"104",
+      /*persistent_tensors=*/"18,19,38,39,58,59,78,79,98,99",
+      /*sequence_size=*/320, &os));
+  testing::TfLiteDriver test_driver(/*use_nnapi=*/false);
+  ASSERT_TRUE(testing::ParseAndRunTests(&os, &test_driver, GetMaxInvocations()))
+      << test_driver.GetErrorMessage();
+}
+
 // The original version of speech_asr_lm_model_test.cc ran a few sequences
 // through the interpreter and stored the sum of all the output, which was them
 // compared for correctness. In this test we are comparing all the intermediate
diff --git a/tensorflow/lite/models/testdata/g3doc/README.md b/tensorflow/lite/models/testdata/g3doc/README.md
index 2a4f1c143a..afe5f16b38 100644
--- a/tensorflow/lite/models/testdata/g3doc/README.md
+++ b/tensorflow/lite/models/testdata/g3doc/README.md
@@ -3,6 +3,42 @@
 Sample test data has been provided for speech related models in Tensorflow Lite
 to help users working with speech models to verify and test their models.
 
+### Models and Inputs and Outputs:
+
+[ASR AM model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_asr_am_model.tflite)
+
+[ASR AM quantized model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_asr_am_model_int8.tflite)
+
+[ASR AM test inputs](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_asr_am_model_in.csv)
+
+[ASR AM test outputs](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_asr_am_model_out.csv)
+
+[ASR AM int8 test outputs](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_asr_am_model_int8_out.csv)
+
+The models below are not maintained.
+
+[Speech hotword model (Svdf
+rank=1)](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_hotword_model_rank1_2017_11_14.tflite)
+
+[Speech hotword model (Svdf
+rank=2)](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_hotword_model_rank2_2017_11_14.tflite)
+
+[Speaker-id
+model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_speakerid_model_2017_11_14.tflite)
+
+[TTS
+model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_tts_model_2017_11_14.tflite)
+
+### Test Bench
+
+[Model tests](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/models/speech_test.cc)
+
+Download the ASR AM test models and inputs and output files to the
+models/testdata directory to run the tests.
+
+
+## Speech Model Architectures
+
 For the hotword, speaker-id and automatic speech recognition sample models, the
 architecture assumes that the models receive their input from a speech
 pre-processing module. The speech pre-processing module receives the audio
@@ -87,57 +123,3 @@ The model consists of a convolutional layer, followed by a fully-connected
 layer, two LSTM layers, and two additional fully-connected layers.
 The corresponding parameters as shown in the figure.
 ![endpointer_model](endpointer.svg "Endpointer model")
-
-
-## Speech models test input/output generation
-
-As mentioned above the input to models are generated from a pre-processing
-module (output of a log-mel filterbank, or linguistic features), and the outputs
-are generated by running the equivalent TensorFlow model by feeding them the
-same input.
-
-## Link to the open source code
-
-### Models:
-
-[Speech hotword model (Svdf
-rank=1)](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_hotword_model_rank1_2017_11_14.tflite)
-
-[Speech hotword model (Svdf
-rank=2)](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_hotword_model_rank2_2017_11_14.tflite)
-
-[Speaker-id
-model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_speakerid_model_2017_11_14.tflite)
-
-[TTS
-model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_tts_model_2017_11_14.tflite)
-
-[ASR AM
-model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_terse_am_model_2017_11_14.tflite)
-
-### Test benches
-
-[Speech hotword model
-test](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/models/speech_hotword_model_test.cc)
-
-[Speaker-id model
-test](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/models/speech_speakerid_model_test.cc)
-
-[TTS model
-test](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/models/speech_tts_model_test.cc)
-
-[ASR AM model
-test](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/models/speech_asr_am_model_test.cc)
-
-[ASR LM model
-test](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/models/speech_asr_lm_model_test.cc)
-
-[Endpointer model
-test](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/models/speech_endpointer_model_test.cc)
-
-## Android Support
-The models have been tested on Android phones, using the following tests:
-
-[Hotword] (https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/android/BUILD?rcl=172930882&l=25)
-
-[Speaker-id] (https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/android/BUILD?rcl=172930882&l=36)
-- 
GitLab


From 4f1b0ee65a652743ede11d95a33513588ba7f772 Mon Sep 17 00:00:00 2001
From: Haoyu Zhang <haoyuzhang@google.com>
Date: Sun, 20 Jan 2019 16:37:03 -0800
Subject: [PATCH 1026/2345] Prevent repeatedly calling iterator.get_next() in
 Keras training loop.

Calling iterator.get_next() repeatedly causes execution slowdown and eventually leads to OOM. This change avoids this call on (1) training dataset iterator when the dataset needs to be reset after an epoch; (2) validation dataset iterator when multiple validation passes need to be invoked during training.

PiperOrigin-RevId: 230154198
---
 .../python/keras/engine/training_arrays.py    | 38 +++++++++++++------
 1 file changed, 26 insertions(+), 12 deletions(-)

diff --git a/tensorflow/python/keras/engine/training_arrays.py b/tensorflow/python/keras/engine/training_arrays.py
index d0881c11a5..cefd4d57fd 100644
--- a/tensorflow/python/keras/engine/training_arrays.py
+++ b/tensorflow/python/keras/engine/training_arrays.py
@@ -60,6 +60,7 @@ def model_iteration(model,
                     validation_freq=1,
                     mode=ModeKeys.TRAIN,
                     validation_in_fit=False,
+                    prepared_feed_values_from_dataset=False,
                     steps_name='steps',
                     **kwargs):
   """Loop function for arrays of data with modes TRAIN/TEST/PREDICT.
@@ -94,11 +95,14 @@ def model_iteration(model,
         which to run validation, e.g. `validation_freq=[1, 2, 10]` runs
         validation at the end of the 1st, 2nd, and 10th epochs.
       mode: One of ModeKeys.TRAIN/ModeKeys.TEST/ModeKeys.PREDICT.
-      validation_in_fit: DEPRECATED: if true, then this method is invoked from
-        within training iteration (for validation). In this case, do not copy
-        weights when using a tf.distribute.Strategy. The input is deprecated as
-        it is not required if the user creates a distributed model under the
-        distribution strategy scope rather than passing it to compile.
+      validation_in_fit: if true, then this method is invoked from within
+        training iteration (for validation). In the case where `val_inputs` is a
+        dataset, this flag indicates that its iterator and feed values are
+        already created so should properly reuse resources.
+      prepared_feed_values_from_dataset: if True, `inputs` is a list of feed
+        tensors returned from `_prepare_feed_values` call on the validation
+        dataset, so do not call it again on `inputs`. Should only be used for
+        inline validation (i.e., only if `validation_in_fit` is also True).
       steps_name: The string name of the steps argument, either `steps`,
         `validation_steps`, or `steps_per_epoch`. Only used for error message
         formatting.
@@ -133,10 +137,6 @@ def model_iteration(model,
           inputs, steps_per_epoch, epochs=epochs, steps_name=steps_name)
     input_iterator = _get_iterator(inputs, model._distribution_strategy)
 
-  val_iterator = None
-  if isinstance(val_inputs, (dataset_ops.DatasetV1, dataset_ops.DatasetV2)):
-    val_iterator = _get_iterator(val_inputs, model._distribution_strategy)
-
   if mode == ModeKeys.TRAIN:
     _print_train_info(inputs, val_inputs, steps_per_epoch, verbose)
 
@@ -156,13 +156,28 @@ def model_iteration(model,
 
   # Prepare input data.
   inputs = input_iterator or inputs
-  ins = _prepare_feed_values(model, inputs, targets, sample_weights, mode)
+  if validation_in_fit and prepared_feed_values_from_dataset:
+    # When invoking validation in training loop, avoid creating iterator and
+    # list of feed values for the same validation dataset multiple times (which
+    # essentially would call `iterator.get_next()` that slows down execution and
+    # leads to OOM errors eventually.
+    ins = inputs
+  else:
+    ins = _prepare_feed_values(model, inputs, targets, sample_weights, mode)
   if not is_dataset:
     num_samples_or_steps = _get_num_samples_or_steps(ins, batch_size,
                                                      steps_per_epoch)
   else:
     num_samples_or_steps = steps_per_epoch
 
+  # Prepare validation data. Hold references to the iterator and the input list
+  # to properly reinitialize and reuse in multiple validation passes.
+  val_iterator = None
+  if isinstance(val_inputs, (dataset_ops.DatasetV1, dataset_ops.DatasetV2)):
+    val_iterator = _get_iterator(val_inputs, model._distribution_strategy)
+    val_inputs = _prepare_feed_values(
+        model, val_iterator, val_targets, val_sample_weights, ModeKeys.TEST)
+
   # Configure callbacks.
   count_mode = 'steps' if use_steps else 'samples'
   callbacks = cbks.configure_callbacks(
@@ -338,7 +353,6 @@ def model_iteration(model,
     if (do_validation and
         training_utils.should_run_validation(validation_freq, epoch) and
         not callbacks.model.stop_training):
-      val_inputs = val_iterator or val_inputs
       val_results = model_iteration(
           model,
           val_inputs,
@@ -350,6 +364,7 @@ def model_iteration(model,
           verbose=0,
           mode=ModeKeys.TEST,
           validation_in_fit=True,
+          prepared_feed_values_from_dataset=(val_iterator is not None),
           steps_name='validation_steps')
       if not isinstance(val_results, list):
         val_results = [val_results]
@@ -366,7 +381,6 @@ def model_iteration(model,
     # Reinitialize dataset iterator for the next epoch.
     if reset_dataset_after_each_epoch and epoch < epochs - 1:
       _reinitialize_iterator(input_iterator, model._distribution_strategy)
-      ins = _prepare_feed_values(model, input_iterator, None, None, mode)
 
   callbacks._call_end_hook(mode)
 
-- 
GitLab


From 48998029e313b9206daa2d49f55aeb810242361e Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Sun, 20 Jan 2019 17:03:28 -0800
Subject: [PATCH 1027/2345] Fix formatting on Apollo3 instructions

PiperOrigin-RevId: 230155453
---
 tensorflow/lite/experimental/micro/README.md | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/tensorflow/lite/experimental/micro/README.md b/tensorflow/lite/experimental/micro/README.md
index 6568ecd0fa..30070bf008 100644
--- a/tensorflow/lite/experimental/micro/README.md
+++ b/tensorflow/lite/experimental/micro/README.md
@@ -307,10 +307,12 @@ Follow these steps to get the pushbutton yes/no example working on Apollo 3:
 
 ### Additional Apollo3 Instructions
 
-To flash a part with JFlash Lite, do the following: 1. At the command line:
-JFlashLiteExe 2. Device = AMA3B1KK-KBR 3. Interface = SWD at 1000 kHz 4. Data
-file =
-tensorflow/lite/experimental/micro/tools/make/gen/apollo3evb_cortex-m4/bin/pushbutton_cmsis_speech_test.bin
+To flash a part with JFlash Lite, do the following: 
+
+1. At the command line: JFlashLiteExe 
+2. Device = AMA3B1KK-KBR 
+3. Interface = SWD at 1000 kHz 
+4. Data file = `tensorflow/lite/experimental/micro/tools/make/gen/apollo3evb_cortex-m4/bin/pushbutton_cmsis_speech_test.bin`
 5. Prog Addr = 0x0000C000
 
 ## Goals
-- 
GitLab


From e6dbfa1790419a6bd857bdb14240a7f67165fc42 Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Sun, 20 Jan 2019 22:24:52 -0800
Subject: [PATCH 1028/2345] Factor out PerChannelQuantization.

PiperOrigin-RevId: 230173465
---
 .../lite/tools/optimize/quantization_utils.cc | 83 +++++++++++++++++++
 .../lite/tools/optimize/quantization_utils.h  | 18 ++++
 .../tools/optimize/quantization_utils_test.cc | 30 +++++++
 .../lite/tools/optimize/subgraph_quantizer.cc | 83 +++----------------
 4 files changed, 144 insertions(+), 70 deletions(-)

diff --git a/tensorflow/lite/tools/optimize/quantization_utils.cc b/tensorflow/lite/tools/optimize/quantization_utils.cc
index ebac65ed10..7239ebcc7a 100644
--- a/tensorflow/lite/tools/optimize/quantization_utils.cc
+++ b/tensorflow/lite/tools/optimize/quantization_utils.cc
@@ -13,6 +13,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 #include "tensorflow/lite/tools/optimize/quantization_utils.h"
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/kernels/internal/round.h"
+#include "tensorflow/lite/kernels/internal/types.h"
 
 #include <cmath>
 #include <cstdint>
@@ -21,6 +24,11 @@ namespace tflite {
 namespace optimize {
 namespace utils {
 
+namespace {
+const int8_t kMinQuantizedValue = -127;
+const int8_t kMaxQuantizedValue = 127;
+}  // namespace
+
 TfLiteStatus NumElements(const TensorT& tensor, uint64_t* num_elements) {
   if (tensor.shape.empty()) {
     return kTfLiteError;
@@ -62,6 +70,81 @@ void GetAsymmetricQuantizationParams(
   quantization_params->zero_point = std::vector<int64_t>(1, zero_point);
 }
 
+// Per-channel quantize a tensor at the given index and returns both scales and
+// quantized values.
+void SymmetricPerChannelQuantization(const float* const input,
+                                     const std::vector<int>& dimension,
+                                     int32_t channel_dim_index,
+                                     std::vector<float>* output_scales,
+                                     std::vector<int8_t>* output_value) {
+  const int32_t channel_dim_size = dimension[channel_dim_index];
+  std::vector<float> min_vals(channel_dim_size);
+  std::vector<float> max_vals(channel_dim_size);
+  std::vector<bool> has_min_max_value(channel_dim_size, false);
+  int indices[4];
+  RuntimeShape tensor_dims{dimension[0], dimension[1], dimension[2],
+                           dimension[3]};
+
+  // Compute min max ranges per channel
+  for (indices[0] = 0; indices[0] < dimension[0]; indices[0]++) {
+    for (indices[1] = 0; indices[1] < dimension[1]; indices[1]++) {
+      for (indices[2] = 0; indices[2] < dimension[2]; indices[2]++) {
+        for (indices[3] = 0; indices[3] < dimension[3]; indices[3]++) {
+          int channel_idx = indices[channel_dim_index];
+          const float val = input[Offset(tensor_dims, indices)];
+          if (has_min_max_value[channel_idx]) {
+            if (min_vals[channel_idx] > val) {
+              min_vals[channel_idx] = val;
+            } else if (max_vals[channel_idx] < val) {
+              max_vals[channel_idx] = val;
+            }
+          } else {
+            min_vals[channel_idx] = val;
+            max_vals[channel_idx] = val;
+            has_min_max_value[channel_idx] = true;
+          }
+        }
+      }
+    }
+  }
+
+  // Calculate scales per channel
+  std::vector<float> scale_invs(channel_dim_size);
+  const float half_scale = kMaxQuantizedValue;
+  for (size_t channel_idx = 0; channel_idx < channel_dim_size; channel_idx++) {
+    const float half_range = std::max(std::abs(min_vals[channel_idx]),
+                                      std::abs(max_vals[channel_idx]));
+    output_scales->at(channel_idx) = half_range / half_scale;
+    if (half_range == 0) {
+      scale_invs[channel_idx] = 0;
+    } else {
+      scale_invs[channel_idx] = half_scale / half_range;
+    }
+  }
+
+  // Quantize the values.
+  const uint64_t num_elements_per_channel =
+      output_scales->size() / channel_dim_size;
+  std::vector<int8_t> quantized_buffer(num_elements_per_channel);
+  memset(indices, 0, 4 * sizeof(int));
+  for (indices[0] = 0; indices[0] < dimension[0]; indices[0]++) {
+    for (indices[1] = 0; indices[1] < dimension[1]; indices[1]++) {
+      for (indices[2] = 0; indices[2] < dimension[2]; indices[2]++) {
+        for (indices[3] = 0; indices[3] < dimension[3]; indices[3]++) {
+          int channel_idx = indices[channel_dim_index];
+          int index = Offset(tensor_dims, indices);
+          const float val = input[index];
+          const int32_t quantized_value =
+              static_cast<int32_t>(TfLiteRound(val * scale_invs[channel_idx]));
+          output_value->at(index) = std::min<int8_t>(
+              kMaxQuantizedValue,
+              std::max<int8_t>(kMinQuantizedValue, quantized_value));
+        }
+      }
+    }
+  }
+}
+
 }  // namespace utils
 }  // namespace optimize
 }  // namespace tflite
diff --git a/tensorflow/lite/tools/optimize/quantization_utils.h b/tensorflow/lite/tools/optimize/quantization_utils.h
index 8e05c69a40..c4afb0c224 100644
--- a/tensorflow/lite/tools/optimize/quantization_utils.h
+++ b/tensorflow/lite/tools/optimize/quantization_utils.h
@@ -35,6 +35,24 @@ void GetAsymmetricQuantizationParams(
     float min, float max, const int quant_min, const int quant_max,
     QuantizationParametersT* quantization_params);
 
+// Per-channel quantize a tensor at the given index and returns both scales and
+// quantized values.
+// Parameters:
+// - input is the float input data to be quantized.
+// - dimension is the dimension of the input data. Only supports dimension of
+//   size 4.
+// - channel_dim_index is the channel index within "dimension".
+//   dimension[channel_dim_index] gives the number of channels.
+// - output_scale is the output scale, the size of which equals the number of
+//   channels.
+// - output_value is the output data, the size of which equals the number of
+//   inputs.
+void SymmetricPerChannelQuantization(const float* const input,
+                                     const std::vector<int>& dimension,
+                                     int32_t channel_dim_index,
+                                     std::vector<float>* output_scales,
+                                     std::vector<int8_t>* output_value);
+
 }  // namespace utils
 }  // namespace optimize
 }  // namespace tflite
diff --git a/tensorflow/lite/tools/optimize/quantization_utils_test.cc b/tensorflow/lite/tools/optimize/quantization_utils_test.cc
index 482ec70470..0602d82a1c 100644
--- a/tensorflow/lite/tools/optimize/quantization_utils_test.cc
+++ b/tensorflow/lite/tools/optimize/quantization_utils_test.cc
@@ -21,6 +21,8 @@ namespace optimize {
 namespace utils {
 namespace {
 
+using ::testing::ElementsAreArray;
+
 TEST(QuantizationUtilsTest, NumElements) {
   TensorT tensor;
   tensor.shape = {1, 2, 3, 4};
@@ -124,6 +126,34 @@ TEST(QuantizationUtilsTest, AsymmetricQuantizationParamsWithZeroInRange) {
   EXPECT_LT(zero_point, quant_max);
 }
 
+TEST(QuantizationUtilsTest, SymmetricPerChannelQuantization) {
+  // Set up an input with [3, 2, 2, 2] size and 0 is the channel index.
+  const std::vector<float> input = {
+      3.0, 2.0, 5.0,  -2.0, 3.0,  2.0,  5.0,  -2.0,  // Channel 1.
+      1.0, 2.0, 3.0,  4.0,  5.0,  6.0,  7.0,  8.0,   // Channel 2.
+      1.0, 0.0, -1.0, -2.0, -3.0, -4.0, -5.0, -6.0,  // Channel 3.
+  };
+  const std::vector<int32_t> dimension = {3, 2, 2, 2};
+  const int channel_index = 0;
+
+  // Create holder for output scale and data.
+  std::vector<float> output_scales(3);
+  std::vector<int8_t> output_data(3 * 2 * 2 * 2);
+
+  // Call SymmetricPerChannelQuantization and verify the result.
+  SymmetricPerChannelQuantization(input.data(), dimension, channel_index,
+                                  &output_scales, &output_data);
+  const std::vector<float> expected_output_scales = {0.0393700786, 0.0629921257,
+                                                     0.0472440943};
+  const std::vector<int8_t> expected_output_data = {
+      76, 51, 127, -51, 76,  51,  127,  -51,   // Channel 1.
+      16, 32, 48,  64,  79,  95,  111,  127,   // Channel 2.
+      21, 0,  -21, -42, -64, -85, -106, -127,  // Channel 3.
+  };
+  EXPECT_THAT(output_scales, ElementsAreArray(expected_output_scales));
+  EXPECT_THAT(output_data, ElementsAreArray(expected_output_data));
+}
+
 }  // namespace
 }  // namespace utils
 }  // namespace optimize
diff --git a/tensorflow/lite/tools/optimize/subgraph_quantizer.cc b/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
index bd6e6e81b2..3bab365c82 100644
--- a/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
+++ b/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
@@ -33,9 +33,6 @@ namespace optimize {
 namespace internal {
 
 namespace {
-const int8_t kMinQuantizedValue = -127;
-const int8_t kMaxQuantizedValue = 127;
-
 TfLiteStatus AddQuantizationParams(const std::vector<float>& scales,
                                    const std::vector<int64_t>& zero_point,
                                    int quantized_dimension,
@@ -91,88 +88,34 @@ const OpWithBiasTensors* GetInfoForOpWithBiasTensor(BuiltinOperator op_code) {
 TfLiteStatus SymmetricPerChannelQuantizeTensor(ModelT* model, TensorT* tensor,
                                                int32_t channel_dim_index,
                                                ErrorReporter* error_reporter) {
-  int32_t channel_dim_size = tensor->shape[channel_dim_index];
   if (tensor->shape.size() != 4) {
     error_reporter->Report("Only dims=4 is supported, tensor dims: %d",
                            tensor->shape.size());
     return kTfLiteError;
   }
 
+  // Get dimensions.
   uint64_t num_elements;
   TF_LITE_ENSURE_STATUS(utils::NumElements(*tensor, &num_elements));
-  const uint64_t num_elements_per_channel = num_elements / channel_dim_size;
-
-  std::vector<float> min_vals(channel_dim_size);
-  std::vector<float> max_vals(channel_dim_size);
-  std::vector<bool> has_min_max_value(channel_dim_size, false);
-  int indices[4];
-  RuntimeShape tensor_dims{tensor->shape[0], tensor->shape[1], tensor->shape[2],
-                           tensor->shape[3]};
-  BufferT* buffer = model->buffers[tensor->buffer].get();
-  float* float_data = reinterpret_cast<float*>(buffer->data.data());
+  const int32_t channel_dim_size = tensor->shape[channel_dim_index];
 
-  // Compute min max ranges per channel
-  for (indices[0] = 0; indices[0] < tensor->shape[0]; indices[0]++) {
-    for (indices[1] = 0; indices[1] < tensor->shape[1]; indices[1]++) {
-      for (indices[2] = 0; indices[2] < tensor->shape[2]; indices[2]++) {
-        for (indices[3] = 0; indices[3] < tensor->shape[3]; indices[3]++) {
-          int channel_idx = indices[channel_dim_index];
-          const float val = float_data[Offset(tensor_dims, indices)];
-          if (has_min_max_value[channel_idx]) {
-            if (min_vals[channel_idx] > val) {
-              min_vals[channel_idx] = val;
-            } else if (max_vals[channel_idx] < val) {
-              max_vals[channel_idx] = val;
-            }
-          } else {
-            min_vals[channel_idx] = val;
-            max_vals[channel_idx] = val;
-            has_min_max_value[channel_idx] = true;
-          }
-        }
-      }
-    }
-  }
+  // Get input float data.
+  BufferT* buffer = model->buffers[tensor->buffer].get();
+  float* float_input_data = reinterpret_cast<float*>(buffer->data.data());
 
-  // Calculate scales per channel
+  // Create container for output scale and output data.
   std::vector<float> scales(channel_dim_size);
-  std::vector<float> scale_invs(channel_dim_size);
-  const float half_scale = kMaxQuantizedValue;
-  for (size_t channel_idx = 0; channel_idx < channel_dim_size; channel_idx++) {
-    const float half_range = std::max(std::abs(min_vals[channel_idx]),
-                                      std::abs(max_vals[channel_idx]));
-    scales[channel_idx] = half_range / half_scale;
-    if (half_range == 0) {
-      scale_invs[channel_idx] = 0;
-    } else {
-      scale_invs[channel_idx] = half_scale / half_range;
-    }
-  }
-
-  // Quantize the values.
-  std::vector<int8_t> quantized_buffer(num_elements_per_channel);
   std::vector<int8_t> final_buffer(num_elements);
-  memset(indices, 0, 4 * sizeof(int));
-  for (indices[0] = 0; indices[0] < tensor->shape[0]; indices[0]++) {
-    for (indices[1] = 0; indices[1] < tensor->shape[1]; indices[1]++) {
-      for (indices[2] = 0; indices[2] < tensor->shape[2]; indices[2]++) {
-        for (indices[3] = 0; indices[3] < tensor->shape[3]; indices[3]++) {
-          int channel_idx = indices[channel_dim_index];
-          int index = Offset(tensor_dims, indices);
-          const float val = float_data[index];
-          const int32_t quantized_value =
-              static_cast<int32_t>(TfLiteRound(val * scale_invs[channel_idx]));
-          final_buffer[index] = std::min<int8_t>(
-              kMaxQuantizedValue,
-              std::max<int8_t>(kMinQuantizedValue, quantized_value));
-        }
-      }
-    }
-  }
+
+  // Quantize the input data with respect to channel_dim_index.
+  const std::vector<int> tensor_dims = {tensor->shape[0], tensor->shape[1],
+                                        tensor->shape[2], tensor->shape[3]};
+  utils::SymmetricPerChannelQuantization(
+      float_input_data, tensor_dims, channel_dim_index, &scales, &final_buffer);
 
   // Set the buffers and output type.
   uint8_t* uint8_buffer = reinterpret_cast<uint8_t*>(final_buffer.data());
-  size_t buffer_size = num_elements * sizeof(int8_t);
+  const size_t buffer_size = num_elements * sizeof(int8_t);
   std::vector<int64_t> zero_point(scales.size(), 0);
   return AddQuantizationParams(scales, zero_point, channel_dim_index,
                                uint8_buffer, buffer_size, TensorType_INT8,
-- 
GitLab


From 858a402b9386ead2026945f0553f95fa84d5c101 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 21 Jan 2019 01:03:22 -0800
Subject: [PATCH 1029/2345] compat: Update forward compatibility horizon to
 2019-01-21

PiperOrigin-RevId: 230186900
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 16fe3db26e..0bd20a1f97 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 20)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 21)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 70cd49ec7d2b89706ab7eb081dce787facf3d4f1 Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Mon, 21 Jan 2019 01:44:23 -0800
Subject: [PATCH 1030/2345] Fix doc.

PiperOrigin-RevId: 230191900
---
 tensorflow/python/eager/function.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 9642b2ba7c..ac596654d5 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -1082,12 +1082,12 @@ class Function(object):
 
   @property
   def _input_signature(self):
-    """Returns the wrapped Python function."""
+    """Returns the input signature."""
     return self._function_spec.input_signature  # pylint: disable=protected-access
 
   @property
   def _flat_input_signature(self):
-    """Returns the wrapped Python function."""
+    """Returns the flattened input signature."""
     return self._function_spec.flat_input_signature  # pylint: disable=protected-access
 
   def _get_concrete_function_internal_garbage_collected(self, *args, **kwargs):
-- 
GitLab


From 0136a548c70fa231cef55bdce404df38a30ed805 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 21 Jan 2019 02:19:15 -0800
Subject: [PATCH 1031/2345] Update cuda10 image to latest version.

PiperOrigin-RevId: 230195624
---
 third_party/toolchains/preconfig/generate/containers.bzl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/third_party/toolchains/preconfig/generate/containers.bzl b/third_party/toolchains/preconfig/generate/containers.bzl
index 428208523b..a86261328e 100644
--- a/third_party/toolchains/preconfig/generate/containers.bzl
+++ b/third_party/toolchains/preconfig/generate/containers.bzl
@@ -1,5 +1,5 @@
 container_digests = {
     "ubuntu16.04": "sha256:d0d98c53111c3ec071aa81632a2b0d6f210e5c2411c5172e31f99002125ec4de",
     "cuda9.0-cudnn7-ubuntu14.04": "sha256:006a76ee1838122ff7f21ebac85f24c1ef350d4dd79b3ceff0e4fe649ed90d33",
-    "cuda10.0-cudnn7-ubuntu14.04": "sha256:e36f05f1ff39e39ddf07122e37f2b1895948bb6f7acc3db37a3c496be5e66228",
+    "cuda10.0-cudnn7-ubuntu14.04": "sha256:d433e1221f802dac393bc8652fabcc63aa46896cd920bb888ae0e2002fe6b756",
 }
-- 
GitLab


From f7dd78dcc8e4855fff383cd4ac49135e0cf52687 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 21 Jan 2019 02:25:31 -0800
Subject: [PATCH 1032/2345] Internal change.

PiperOrigin-RevId: 230196231
---
 .../Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04        | 11 +++++++----
 .../tools/ci_build/install/install_pip_packages.sh    |  6 +++---
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04 b/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04
index d08d31d913..c6099c9e45 100644
--- a/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04
+++ b/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04
@@ -3,7 +3,7 @@
 #       --tag "gcr.io/tensorflow-testing/nosla-cuda10.0-cudnn7-ubuntu14.04" .
 # $ docker push gcr.io/tensorflow-testing/nosla-cuda10.0-cudnn7-ubuntu14.04
 
-FROM ubuntu:14.04
+FROM gcr.io/clang-docker-builder/clang-ubuntu14_04
 LABEL maintainer="Manuel Klimek <klimek@google.com>"
 
 RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates apt-transport-https gnupg-curl && \
@@ -59,13 +59,16 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 # https://github.com/NVIDIA/nvidia-docker/issues/775
 RUN ln -s libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1
 
-# Install a newer version of libstdc++, as new clang versions do not work
-# with the stock ubuntu 14.04 libstdc++.
+# Install a newer version of g++:
+# - we need a new libstdc++, because new clang versions do not work with a stock
+#   ubuntu 14.04 libstdc++.
+# - we want to compile with g++-7 to get ahead of LLVM dropping support for
+#   gcc 4.8.
 RUN apt-get update && \
     apt-get install -y software-properties-common && \
     add-apt-repository ppa:ubuntu-toolchain-r/test -y && \
     apt-get update && \
-    apt-get install -y libstdc++-7-dev && \
+    apt-get install -y --no-install-recommends g++-7 && \
     rm -rf /var/lib/apt/lists/*
 
 # Copy and run the install scripts.
diff --git a/tensorflow/tools/ci_build/install/install_pip_packages.sh b/tensorflow/tools/ci_build/install/install_pip_packages.sh
index 5ae840cfa0..3878452b69 100755
--- a/tensorflow/tools/ci_build/install/install_pip_packages.sh
+++ b/tensorflow/tools/ci_build/install/install_pip_packages.sh
@@ -18,10 +18,10 @@ set -e
 
 # We don't apt-get install so that we can install a newer version of pip.
 # Only needed for Ubuntu 14.04 and 16.04; not needed for 18.04 and Debian 8,9?
-# Run easy_install before easy_install3, so that the default pip points to pip2,
+# Run easy_install after easy_install3, so that the default pip points to pip2,
 # to match the default python version of 2.7.
-easy_install3 -U pip==9.0.3
-easy_install -U pip==9.0.3
+easy_install3 -U pip==18.1
+easy_install -U pip==18.1
 
 # Install pip packages from whl files to avoid the time-consuming process of
 # building from source.
-- 
GitLab


From 90b5434409684dce13714117ef3c953cd917be21 Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Mon, 21 Jan 2019 02:45:23 -0800
Subject: [PATCH 1033/2345] Fixed positional test for object test cases and a
 lint error in the upgrade script

PiperOrigin-RevId: 230198038
---
 .../tools/compatibility/tf_upgrade_v2.py      |  4 ----
 .../tools/compatibility/tf_upgrade_v2_test.py | 22 ++++++++++++-------
 2 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 3f9032bb33..7479c693b1 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -896,14 +896,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             make_initializable_iterator_deprecation,
         "*.make_one_shot_iterator":
             make_one_shot_iterator_deprecation,
-        "tf.assert_greater":
-            assert_return_type_comment,
         "tf.assert_equal":
             assert_return_type_comment,
         "tf.assert_none_equal":
             assert_return_type_comment,
-        "tf.assert_less":
-            assert_return_type_comment,
         "tf.assert_negative":
             assert_return_type_comment,
         "tf.assert_positive":
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index b160d4de08..79f0a465b5 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -296,14 +296,20 @@ class TestUpgrade(test_util.TensorFlowTestCase):
   def testPositionsMatchArgGiven(self):
     full_dict = tf_upgrade_v2.TFAPIChangeSpec().function_arg_warnings
     method_names = full_dict.keys()
-    for method in method_names:
-      # doesn't test methods on objects
-      if not method.startswith("*."):
-        args = full_dict[method].keys()
-        method = get_symbol_for_name(tf, method)
-        arg_spec = tf_inspect.getfullargspec(method)
-        for (arg, pos) in args:
-          self.assertEqual(arg_spec[0][pos], arg)
+    for method_name in method_names:
+      args = full_dict[method_name].keys()
+      # special case for optimizer methods
+      if method_name.startswith("*."):
+        method = method_name.replace("*", "tf.train.Optimizer")
+      else:
+        method = method_name
+      method = get_symbol_for_name(tf, method)
+      arg_spec = tf_inspect.getfullargspec(method)
+      for (arg, pos) in args:
+        # to deal with the self argument on methods on objects
+        if method_name.startswith("*."):
+          pos += 1
+        self.assertEqual(arg_spec[0][pos], arg)
 
   def testReorderFileNeedsUpdate(self):
     reordered_function_names = (
-- 
GitLab


From 9b271631359c61b3c979cac1c79801b4067ee6b4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 21 Jan 2019 02:49:05 -0800
Subject: [PATCH 1034/2345] Support tf.nn.static_bidirectional_rnn using
 Ophints.

PiperOrigin-RevId: 230198367
---
 .../lite/experimental/examples/lstm/BUILD     |  22 ++
 .../lstm/bidirectional_sequence_lstm_test.py  | 191 ++++++++++
 tensorflow/lite/toco/BUILD                    |   1 +
 .../graph_transformations.h                   |   1 +
 .../group_bidirectional_sequence_lstm.cc      | 346 ++++++++++++++++++
 .../propagate_array_data_types.cc             |   6 +
 .../propagate_fixed_sizes.cc                  |  60 +++
 tensorflow/lite/toco/model.h                  |   9 +-
 tensorflow/lite/toco/tflite/operator.cc       |  50 +++
 tensorflow/lite/toco/toco_tooling.cc          |   7 +
 tensorflow/lite/toco/tooling_util.cc          |   1 +
 11 files changed, 693 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/lite/experimental/examples/lstm/bidirectional_sequence_lstm_test.py
 create mode 100644 tensorflow/lite/toco/graph_transformations/group_bidirectional_sequence_lstm.cc

diff --git a/tensorflow/lite/experimental/examples/lstm/BUILD b/tensorflow/lite/experimental/examples/lstm/BUILD
index f39673c028..dd44c1b67b 100644
--- a/tensorflow/lite/experimental/examples/lstm/BUILD
+++ b/tensorflow/lite/experimental/examples/lstm/BUILD
@@ -73,3 +73,25 @@ py_test(
         "@six_archive//:six",
     ],
 )
+
+py_test(
+    name = "bidirectional_sequence_lstm_test",
+    size = "large",
+    srcs = ["bidirectional_sequence_lstm_test.py"],
+    srcs_version = "PY2AND3",
+    tags = [
+        "no_oss",
+        "no_pip",
+    ],
+    deps = [
+        ":tflite_lstm",
+        "//tensorflow:tensorflow_py",
+        "//tensorflow/examples/tutorials/mnist:input_data",
+        "//tensorflow/lite/python:lite",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:platform",
+        "//tensorflow/python/tools:optimize_for_inference",
+        "//third_party/py/numpy",
+        "@six_archive//:six",
+    ],
+)
diff --git a/tensorflow/lite/experimental/examples/lstm/bidirectional_sequence_lstm_test.py b/tensorflow/lite/experimental/examples/lstm/bidirectional_sequence_lstm_test.py
new file mode 100644
index 0000000000..99f4beda59
--- /dev/null
+++ b/tensorflow/lite/experimental/examples/lstm/bidirectional_sequence_lstm_test.py
@@ -0,0 +1,191 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+import tempfile
+import numpy as np
+import tensorflow as tf
+
+from tensorflow.examples.tutorials.mnist import input_data
+from tensorflow.lite.experimental.examples.lstm.tflite_lstm import TFLiteLSTMCell
+from tensorflow.lite.python.op_hint import convert_op_hints_to_stubs
+from tensorflow.lite.python.op_hint import find_all_hinted_output_nodes
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import test
+from tensorflow.python.tools import optimize_for_inference_lib
+
+# Number of steps to train model.
+TRAIN_STEPS = 1
+
+CONFIG = tf.ConfigProto(device_count={"GPU": 0})
+
+
+class BidirectionalSequenceLstmTest(test_util.TensorFlowTestCase):
+
+  def setUp(self):
+    tf.reset_default_graph()
+    # Import MNIST dataset
+    self.mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)
+
+    # Define constants
+    # Unrolled through 28 time steps
+    self.time_steps = 28
+    # Rows of 28 pixels
+    self.n_input = 28
+    # Learning rate for Adam optimizer
+    self.learning_rate = 0.001
+    # MNIST is meant to be classified in 10 classes(0-9).
+    self.n_classes = 10
+    # Batch size
+    self.batch_size = 16
+    # Lstm Units.
+    self.num_units = 16
+
+  def buildLstmLayer(self):
+    return tf.nn.rnn_cell.MultiRNNCell([
+        TFLiteLSTMCell(
+            self.num_units, use_peepholes=True, forget_bias=0, name="rnn1"),
+        TFLiteLSTMCell(self.num_units, num_proj=8, forget_bias=0, name="rnn2"),
+        TFLiteLSTMCell(
+            self.num_units // 2,
+            use_peepholes=True,
+            num_proj=8,
+            forget_bias=0,
+            name="rnn3"),
+        TFLiteLSTMCell(self.num_units, forget_bias=0, name="rnn4")
+    ])
+
+  def buildModel(self, fw_lstm_layer, bw_lstm_layer):
+    # Weights and biases for output softmax layer.
+    out_weights = tf.Variable(
+        tf.random_normal([self.num_units * 2, self.n_classes]))
+    out_bias = tf.Variable(tf.random_normal([self.n_classes]))
+
+    # input image placeholder
+    x = tf.placeholder(
+        "float", [None, self.time_steps, self.n_input], name="INPUT_IMAGE")
+
+    lstm_input = tf.unstack(x, self.time_steps, 1)
+    outputs, _, _ = tf.nn.static_bidirectional_rnn(
+        fw_lstm_layer, bw_lstm_layer, lstm_input, dtype="float32")
+
+    # Compute logits by multiplying outputs[-1] of shape [batch_size,num_units]
+    # by the softmax layer's out_weight of shape [num_units,n_classes]
+    # plus out_bias
+    prediction = tf.matmul(outputs[-1], out_weights) + out_bias
+    output_class = tf.nn.softmax(prediction, name="OUTPUT_CLASS")
+
+    return x, prediction, output_class
+
+  def trainModel(self, x, prediction, output_class, sess):
+    # input label placeholder
+    y = tf.placeholder("float", [None, self.n_classes])
+    # Loss function
+    loss = tf.reduce_mean(
+        tf.nn.softmax_cross_entropy_with_logits(logits=prediction, labels=y))
+    # Optimization
+    opt = tf.train.AdamOptimizer(
+        learning_rate=self.learning_rate).minimize(loss)
+
+    # Initialize variables
+    init = tf.global_variables_initializer()
+    sess.run(init)
+    for _ in range(TRAIN_STEPS):
+      batch_x, batch_y = self.mnist.train.next_batch(
+          batch_size=self.batch_size, shuffle=False)
+
+      batch_x = batch_x.reshape((self.batch_size, self.time_steps,
+                                 self.n_input))
+      sess.run(opt, feed_dict={x: batch_x, y: batch_y})
+
+  def saveAndRestoreModel(self, fw_lstm_layer, bw_lstm_layer, sess, saver):
+    model_dir = tempfile.mkdtemp()
+    saver.save(sess, model_dir)
+
+    # Reset the graph.
+    tf.reset_default_graph()
+    x, prediction, output_class = self.buildModel(fw_lstm_layer, bw_lstm_layer)
+
+    new_sess = tf.Session(config=CONFIG)
+    saver = tf.train.Saver()
+    saver.restore(new_sess, model_dir)
+    return x, prediction, output_class, new_sess
+
+  def getInferenceResult(self, x, output_class, sess):
+    b1, _ = self.mnist.train.next_batch(batch_size=1)
+    sample_input = np.reshape(b1, (1, self.time_steps, self.n_input))
+
+    expected_output = sess.run(output_class, feed_dict={x: sample_input})
+    # It is important to keep all the ophint output nodes.
+    hinted_outputs_nodes = find_all_hinted_output_nodes(sess)
+    hinted_outputs_nodes.append(output_class.op.name)
+    frozen_graph = tf.graph_util.convert_variables_to_constants(
+        sess, sess.graph_def, hinted_outputs_nodes)
+    return sample_input, expected_output, frozen_graph
+
+  def tfliteInvoke(self, graph, test_inputs, outputs):
+    tf.reset_default_graph()
+    # Turn the input into placeholder of shape 1
+    tflite_input = tf.placeholder(
+        "float", [1, self.time_steps, self.n_input], name="INPUT_IMAGE_LITE")
+    tf.import_graph_def(graph, name="", input_map={"INPUT_IMAGE": tflite_input})
+    with tf.Session() as sess:
+      curr = sess.graph_def
+      curr = convert_op_hints_to_stubs(graph_def=curr)
+
+    curr = optimize_for_inference_lib.optimize_for_inference(
+        curr, ["INPUT_IMAGE_LITE"], ["OUTPUT_CLASS"],
+        [tf.float32.as_datatype_enum])
+
+    tflite = tf.lite.toco_convert(
+        curr, [tflite_input], [outputs], allow_custom_ops=False)
+
+    interpreter = tf.lite.Interpreter(model_content=tflite)
+
+    try:
+      interpreter.allocate_tensors()
+    except ValueError:
+      assert False
+
+    input_index = (interpreter.get_input_details()[0]["index"])
+    interpreter.set_tensor(input_index, test_inputs)
+    interpreter.invoke()
+    output_index = (interpreter.get_output_details()[0]["index"])
+    result = interpreter.get_tensor(output_index)
+    # Reset all variables so it will not pollute other inferences.
+    interpreter.reset_all_variables()
+    return result
+
+  def testStaticRnnMultiRnnCell(self):
+    sess = tf.Session(config=CONFIG)
+
+    x, prediction, output_class = self.buildModel(self.buildLstmLayer(),
+                                                  self.buildLstmLayer())
+    self.trainModel(x, prediction, output_class, sess)
+
+    saver = tf.train.Saver()
+    x, prediction, output_class, new_sess = self.saveAndRestoreModel(
+        self.buildLstmLayer(), self.buildLstmLayer(), sess, saver)
+
+    test_inputs, expected_output, frozen_graph = self.getInferenceResult(
+        x, output_class, new_sess)
+
+    result = self.tfliteInvoke(frozen_graph, test_inputs, output_class)
+    self.assertTrue(np.allclose(expected_output, result, rtol=1e-6, atol=1e-2))
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/lite/toco/BUILD b/tensorflow/lite/toco/BUILD
index 40bceedd6a..3b5b08fe1a 100644
--- a/tensorflow/lite/toco/BUILD
+++ b/tensorflow/lite/toco/BUILD
@@ -192,6 +192,7 @@ cc_library(
         "graph_transformations/fuse_binary_into_preceding_affine.cc",
         "graph_transformations/fuse_broadcast_into_following_binary.cc",
         "graph_transformations/graph_transformations.cc",
+        "graph_transformations/group_bidirectional_sequence_lstm.cc",
         "graph_transformations/hardcode_min_max.cc",
         "graph_transformations/identify_dilated_conv.cc",
         "graph_transformations/identify_l2_normalization.cc",
diff --git a/tensorflow/lite/toco/graph_transformations/graph_transformations.h b/tensorflow/lite/toco/graph_transformations/graph_transformations.h
index 187b584b69..3554dda516 100644
--- a/tensorflow/lite/toco/graph_transformations/graph_transformations.h
+++ b/tensorflow/lite/toco/graph_transformations/graph_transformations.h
@@ -127,6 +127,7 @@ DECLARE_GRAPH_TRANSFORMATION(FuseActivationFunctions)
 DECLARE_GRAPH_TRANSFORMATION(FuseBinaryIntoFollowingAffine)
 DECLARE_GRAPH_TRANSFORMATION(FuseBinaryIntoPrecedingAffine)
 DECLARE_GRAPH_TRANSFORMATION(FuseBroadcastIntoFollowingBinary)
+DECLARE_GRAPH_TRANSFORMATION(GroupBidirectionalSequenceLstm)
 DECLARE_GRAPH_TRANSFORMATION(IdentifyL2Normalization)
 DECLARE_GRAPH_TRANSFORMATION(IdentifyL2Pool)
 DECLARE_GRAPH_TRANSFORMATION(IdentifyLstmCell)
diff --git a/tensorflow/lite/toco/graph_transformations/group_bidirectional_sequence_lstm.cc b/tensorflow/lite/toco/graph_transformations/group_bidirectional_sequence_lstm.cc
new file mode 100644
index 0000000000..48d1fc0138
--- /dev/null
+++ b/tensorflow/lite/toco/graph_transformations/group_bidirectional_sequence_lstm.cc
@@ -0,0 +1,346 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <cstdio>
+#include <iterator>
+#include <memory>
+#include <stack>
+#include <string>
+#include <vector>
+
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/lite/toco/graph_transformations/graph_transformations.h"
+#include "tensorflow/lite/toco/model.h"
+#include "tensorflow/lite/toco/tooling_util.h"
+
+namespace toco {
+namespace {
+
+std::vector<std::unique_ptr<Operator>>::iterator FindOperator(
+    Model* model, const Operator& op) {
+  return std::find_if(
+      model->operators.begin(), model->operators.end(),
+      [&op](const std::unique_ptr<Operator>& ptr) { return ptr.get() == &op; });
+}
+
+bool MatchTwoUnpackOps(const Operator& op, const Model& model,
+                       Operator** fw_output, Operator** bw_output) {
+  if (op.inputs.size() != 2) {
+    return false;
+  }
+
+  *fw_output = GetOpWithOutput(model, op.inputs[0]);
+  *bw_output = GetOpWithOutput(model, op.inputs[1]);
+  if (*fw_output == nullptr || *bw_output == nullptr) {
+    return false;
+  }
+
+  if ((*fw_output)->type != OperatorType::kUnpack ||
+      (*bw_output)->type != OperatorType::kUnpack) {
+    return false;
+  }
+
+  // TODO(renjieliu): Check the shapes are matching.
+
+  return true;
+}
+
+bool FindUnidirectionalSequenceLstmOp(const Model& model,
+                                      const Operator& output_op,
+                                      std::stack<Operator*>* lstm_ops,
+                                      Operator** input_op) {
+  Operator* op_it = nullptr;
+  op_it = GetOpWithOutput(model, output_op.inputs[0]);
+  if (op_it == nullptr) {
+    return false;
+  }
+
+  while (op_it->type == OperatorType::kUnidirectionalSequenceLstm) {
+    lstm_ops->push(op_it);
+    // Check the first input of the unidirectional squence lstm op.
+    op_it = GetOpWithOutput(model, op_it->inputs[0]);
+    if (op_it == nullptr) {
+      return false;
+    }
+  }
+
+  *input_op = op_it;
+  return true;
+}
+
+bool ConstructBidirectionalSequenceLstmOp(
+    const Operator& fw_lstm_op, const Operator& bw_lstm_op, Model* model,
+    BidirectionalSequenceLstmOperator** bi_op) {
+  // TODO(renjieliu): Check the shapes & configurations are equal.
+  constexpr int kBidirectionalSequenceLstmInputsCount = 47;
+  constexpr int kFwLstmInputsStartIndex = 1;
+  constexpr int kBwLstmInputsStartIndex = 18;
+  constexpr int kFwInputActivationStartIndex = 35;
+  constexpr int kBwInputActivationStartIndex = 37;
+  constexpr int kAuxInputStartIndex = 39;
+  (*bi_op)->inputs.reserve(kBidirectionalSequenceLstmInputsCount);
+  const string& input_array_name =
+      AvailableArrayName(*model, "bidirectional_sequence_lstm_input_0");
+  model->GetOrCreateArray(input_array_name);
+  // The input will be changed later.
+  (*bi_op)->inputs.push_back(input_array_name);
+  int i = 1;
+  // Fill in the fw_lstm weights.
+  for (; i < kBwLstmInputsStartIndex; ++i) {
+    (*bi_op)->inputs.push_back(fw_lstm_op.inputs[i]);
+  }
+
+  // Fill in the bw_lstm weights. bidirectional lstm backward weights start
+  // from 18.
+  for (; i < kFwInputActivationStartIndex; ++i) {
+    (*bi_op)->inputs.push_back(
+        bw_lstm_op
+            .inputs[i - (kBwLstmInputsStartIndex - kFwLstmInputsStartIndex)]);
+  }
+
+  // Fill in fw_lstm previous states.
+  for (; i < kBwInputActivationStartIndex; ++i) {
+    (*bi_op)->inputs.push_back(
+        fw_lstm_op.inputs[i - (kFwInputActivationStartIndex -
+                               kBwLstmInputsStartIndex)]);
+  }
+
+  // Fill in bw_lstm previous states.
+  for (; i < kAuxInputStartIndex; ++i) {
+    (*bi_op)->inputs.push_back(
+        bw_lstm_op.inputs[i - (kBwInputActivationStartIndex -
+                               kBwLstmInputsStartIndex)]);
+  }
+
+  // TODO(renjieliu): Deal with Auxiliary input and weights for 39 - 47.
+  for (; i <= kBidirectionalSequenceLstmInputsCount; ++i) {
+    const string& temp_array_name = AvailableArrayName(
+        *model, "bidirectional_sequence_lstm_temp_" + std::to_string(i));
+    model->CreateOptionalArray(temp_array_name);
+    (*bi_op)->inputs.push_back(temp_array_name);
+  }
+
+  // Deal with outputs.
+  (*bi_op)->outputs.reserve(2);
+  const string& fw_output_array_name =
+      AvailableArrayName(*model, "bidirectional_sequence_lstm_fw_output_0");
+  const string& bw_output_array_name =
+      AvailableArrayName(*model, "bidirectional_sequence_lstm_bw_output_0");
+  model->GetOrCreateArray(fw_output_array_name);
+  model->GetOrCreateArray(bw_output_array_name);
+  (*bi_op)->outputs.push_back(fw_output_array_name);
+  (*bi_op)->outputs.push_back(bw_output_array_name);
+  (*bi_op)->merge_outputs = false;
+  return true;
+}
+
+bool GroupFwBwLstmOps(
+    Model* model, std::stack<Operator*> fw_lstm_ops,
+    std::stack<Operator*> bw_lstm_ops,
+    std::vector<BidirectionalSequenceLstmOperator*>* bidirectional_lstm_ops) {
+  while (!fw_lstm_ops.empty()) {
+    Operator* fw_lstm_op = fw_lstm_ops.top();
+    Operator* bw_lstm_op = bw_lstm_ops.top();
+    BidirectionalSequenceLstmOperator* bidirectional_sequence_lstm_op =
+        new BidirectionalSequenceLstmOperator;
+    if (!ConstructBidirectionalSequenceLstmOp(
+            *fw_lstm_op, *bw_lstm_op, model, &bidirectional_sequence_lstm_op)) {
+      return false;
+    }
+
+    bidirectional_lstm_ops->push_back(bidirectional_sequence_lstm_op);
+    fw_lstm_ops.pop();
+    bw_lstm_ops.pop();
+  }
+  return true;
+}
+
+void RewireBidirectionalSequenceLstmOpsConnections(
+    const string& input_array_name,
+    const std::vector<BidirectionalSequenceLstmOperator*>&
+        bidirectional_sequence_lstm_ops,
+    std::vector<std::unique_ptr<Operator>>::iterator* op_it, Model* model) {
+  string cur_fw_input = input_array_name;
+  string cur_bw_input = input_array_name;
+  for (int i = 0; i < bidirectional_sequence_lstm_ops.size(); ++i) {
+    DeleteArrayIfUsedOnce(bidirectional_sequence_lstm_ops[i]->inputs[0], model);
+    bidirectional_sequence_lstm_ops[i]->inputs[0] = cur_fw_input;
+    if (i != 0) {
+      DeleteArrayIfUsedOnce(bidirectional_sequence_lstm_ops[i]->inputs[39],
+                            model);
+      bidirectional_sequence_lstm_ops[i]->inputs[39] = cur_bw_input;
+    }
+    cur_fw_input = bidirectional_sequence_lstm_ops[i]->outputs[0];
+    cur_bw_input = bidirectional_sequence_lstm_ops[i]->outputs[1];
+    if (i != (bidirectional_sequence_lstm_ops.size() - 1)) {
+      bidirectional_sequence_lstm_ops[i]->merge_outputs = false;
+    } else {
+      // TODO(renjieliu): We need to check whether the outputs of the last bidi
+      // lstms needs merged outputs or not.
+      bidirectional_sequence_lstm_ops[i]->merge_outputs = true;
+      DeleteArrayIfUnused(bidirectional_sequence_lstm_ops[i]->outputs[1],
+                          model);
+      bidirectional_sequence_lstm_ops[i]->outputs.pop_back();
+    }
+    model->operators.emplace(*op_it, bidirectional_sequence_lstm_ops[i]);
+    *op_it += 1;
+  }
+}
+
+void RewireFinalUnpackOutputs(
+    const UnpackOperator& original_unpack_operator,
+    UnpackOperator** final_unpack_operator,
+    BidirectionalSequenceLstmOperator** final_bidi_lstm_operator,
+    Model* model) {
+  (*final_unpack_operator)
+      ->inputs.push_back((*final_bidi_lstm_operator)->outputs[0]);
+  (*final_unpack_operator)->axis = original_unpack_operator.axis;
+  (*final_unpack_operator)->num = original_unpack_operator.num;
+
+  for (int i = 0; i < original_unpack_operator.outputs.size(); ++i) {
+    const string& output_array_name = original_unpack_operator.outputs[i];
+    const string& final_unpack_output_array_name = AvailableArrayName(
+        *model, "bidirectional_sequence_lstm_unpack_" + std::to_string(i));
+    model->GetOrCreateArray(final_unpack_output_array_name);
+    (*final_unpack_operator)->outputs.push_back(final_unpack_output_array_name);
+    Operator* unpack_following_op = GetOpWithInput(*model, output_array_name);
+    if (unpack_following_op != nullptr) {
+      // If there's a following op after the unpack, it must be a concat op.
+      DCHECK(unpack_following_op->type == OperatorType::kConcatenation);
+      // For every output of the concat, rewire the outputs.
+      for (const string& concat_output : unpack_following_op->outputs) {
+        (*final_unpack_operator)->outputs[i] = concat_output;
+      }
+      // Remove the concat op.
+      model->operators.erase(FindOperator(model, *unpack_following_op));
+    }
+  }
+}
+
+void RemoveUnpackOperator(const Operator& unpack_op, Model* model) {
+  for (const string& output_array_name : unpack_op.outputs) {
+    DeleteArrayIfUnused(output_array_name, model);
+  }
+  model->operators.erase(FindOperator(model, unpack_op));
+}
+
+void RemoveUnidirectionalSequenceLstmOps(std::stack<Operator*> uni_lstm_ops,
+                                         Model* model) {
+  while (!uni_lstm_ops.empty()) {
+    Operator* uni_lstm_op = uni_lstm_ops.top();
+    DeleteArrayIfUnused(uni_lstm_op->outputs[0], model);
+    model->operators.erase(FindOperator(model, *uni_lstm_op));
+    uni_lstm_ops.pop();
+  }
+}
+
+}  // namespace
+
+// TODO(renjieliu): Support graph generated by dynamic rnn as well.
+::tensorflow::Status GroupBidirectionalSequenceLstm::Run(Model* model,
+                                                         std::size_t op_index,
+                                                         bool* modified) {
+  *modified = false;
+  // Bidirectional sequence lstm will generate two separate unidirectional
+  // sequence lstm ops, for static bidirectional sequence lstm, there will be
+  // a concatenation op at very end; for dynamic bidirectional squence lstm,
+  // it is not guaranteed, but currently we do not support that.
+  auto op_it = model->operators.begin() + op_index;
+  Operator* final_concat_op = op_it->get();
+  if (final_concat_op->type != OperatorType::kConcatenation &&
+      final_concat_op->type != OperatorType::kConcat &&
+      final_concat_op->type != OperatorType::kConcatV2) {
+    return ::tensorflow::Status::OK();
+  }
+
+  // Match fw unidirectional lstm outputs and bw unidirectional lstm outputs:
+  // should be two unstack ops.
+  Operator *fw_lstm_output, *bw_lstm_output;
+  if (!MatchTwoUnpackOps(*final_concat_op, *model, &fw_lstm_output,
+                         &bw_lstm_output)) {
+    return ::tensorflow::Status::OK();
+  }
+
+  // Find all upstream unidirectional lstm ops.
+  std::stack<Operator*> fw_unidirectional_sequence_lstm_ops,
+      bw_unidirectional_sequence_lstm_ops;
+  Operator *first_fw_lstm_input, *first_bw_lstm_input;
+  if (!FindUnidirectionalSequenceLstmOp(*model, *fw_lstm_output,
+                                        &fw_unidirectional_sequence_lstm_ops,
+                                        &first_fw_lstm_input) ||
+      !FindUnidirectionalSequenceLstmOp(*model, *bw_lstm_output,
+                                        &bw_unidirectional_sequence_lstm_ops,
+                                        &first_bw_lstm_input)) {
+    return ::tensorflow::Status::OK();
+  }
+
+  if (fw_unidirectional_sequence_lstm_ops.size() !=
+          bw_unidirectional_sequence_lstm_ops.size() ||
+      fw_unidirectional_sequence_lstm_ops.empty()) {
+    return ::tensorflow::Status::OK();
+  }
+
+  // For static bidirectional sequence lstm, we should have two pack ops.
+  if (first_fw_lstm_input->type != OperatorType::kPack ||
+      first_bw_lstm_input->type != OperatorType::kPack) {
+    return ::tensorflow::Status::OK();
+  }
+
+  // fw_lstm & bw_lstm should point to the same input, but reversed sequence.
+  for (int i = 0; i < first_fw_lstm_input->inputs.size(); ++i) {
+    if (first_fw_lstm_input->inputs[i] !=
+        first_bw_lstm_input
+            ->inputs[first_fw_lstm_input->inputs.size() - i - 1]) {
+      return ::tensorflow::Status::OK();
+    }
+  }
+
+  std::vector<BidirectionalSequenceLstmOperator*>
+      bidirectional_sequence_lstm_ops;
+  if (!GroupFwBwLstmOps(model, fw_unidirectional_sequence_lstm_ops,
+                        bw_unidirectional_sequence_lstm_ops,
+                        &bidirectional_sequence_lstm_ops)) {
+    return ::tensorflow::Status::OK();
+  }
+
+  // Rewire the inputs & outputs.
+  string current_input = first_fw_lstm_input->outputs[0];
+  RewireBidirectionalSequenceLstmOpsConnections(
+      current_input, bidirectional_sequence_lstm_ops, &op_it, model);
+
+  // Insert a unpack op for the output.
+  UnpackOperator* unpack_operator = new UnpackOperator;
+
+  RewireFinalUnpackOutputs(
+      static_cast<const UnpackOperator&>(*fw_lstm_output), &unpack_operator,
+      &bidirectional_sequence_lstm_ops[bidirectional_sequence_lstm_ops.size() -
+                                       1],
+      model);
+  model->operators.emplace(op_it, unpack_operator);
+
+  // Delete unused ops.
+  RemoveUnpackOperator(*fw_lstm_output, model);
+  RemoveUnpackOperator(*bw_lstm_output, model);
+  RemoveUnidirectionalSequenceLstmOps(fw_unidirectional_sequence_lstm_ops,
+                                      model);
+  RemoveUnidirectionalSequenceLstmOps(bw_unidirectional_sequence_lstm_ops,
+                                      model);
+  // Only keep the fw lstm's pack input.
+  DeleteArrayIfUnused(first_bw_lstm_input->outputs[0], model);
+  model->operators.erase(FindOperator(model, *first_bw_lstm_input));
+  *modified = true;
+  return ::tensorflow::Status::OK();
+}
+
+}  // namespace toco
diff --git a/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc b/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
index d546ff4ec0..d8e0135350 100644
--- a/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
+++ b/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
@@ -266,6 +266,12 @@ void SetDataTypeForAllOutputs(Model* model, Operator* op,
       model->GetArray(op->outputs[1]).data_type = unique_op->idx_out_type;
       break;
     }
+    case OperatorType::kBidirectionalSequenceLstm: {
+      const ArrayDataType data_type = model->GetArray(op->inputs[0]).data_type;
+      if (data_type != ArrayDataType::kFloat) return ::tensorflow::Status::OK();
+      SetDataTypeForAllOutputs(model, op, data_type);
+      break;
+    }
     case OperatorType::kLstmCell: {
       // It's tricky to propagate data types through a LstmCell, as that has
       // multiple inputs and outputs, and there are quantized cases with
diff --git a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
index 67c8c13059..e33922429b 100644
--- a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
+++ b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
@@ -1149,6 +1149,62 @@ void ProcessUnidirectionalSequenceRnnOperator(
   output_shape->ReplaceDims({timestamp, batch_size, output_size});
 }
 
+void ProcessBidirectionalSequenceLstmOperator(
+    Model* model, BidirectionalSequenceLstmOperator* op) {
+  // We assume time major.
+  auto& fw_output_array = model->GetArray(op->outputs[0]);
+  auto& bw_output_array = model->GetArray(op->outputs[1]);
+  if (fw_output_array.has_shape()) {
+    // Shape already propagated
+    return;
+  }
+
+  if (fw_output_array.data_type == ArrayDataType::kNone) {
+    // Yield until the output type has been set by PropagateArrayDataTypes
+    return;
+  }
+
+  // TODO(renjieliu): check the inputs, as well as all kinds of weights.
+  const auto& input_array = model->GetArray(op->inputs[0]);
+  // Yield until input dims have been resolved.
+  if (!input_array.has_shape()) {
+    return;
+  }
+  const auto& input_shape = input_array.shape();
+  const int batch_size = input_shape.dims(1);
+  const int timestamp = input_shape.dims(0);
+
+  constexpr int kBwRecurrentToOutputWeightsTensor = 25;
+  const auto& recurrent_to_output_weights_array =
+      model->GetArray(op->inputs[kBwRecurrentToOutputWeightsTensor]);
+  // Yield until input dims have been resolved.
+  if (!recurrent_to_output_weights_array.has_shape()) {
+    return;
+  }
+
+  constexpr int kFwInputActivationStateTensor = 35;
+  constexpr int kFwInputCellStateTensor = 36;
+  constexpr int kBwInputActivationStateTensor = 37;
+  constexpr int kBwInputCellStateTensor = 38;
+  // b(115961645): This is a hack to work around.
+  model->GetArray(op->inputs[kFwInputActivationStateTensor]).buffer.reset();
+  model->GetArray(op->inputs[kFwInputCellStateTensor]).buffer.reset();
+  model->GetArray(op->inputs[kBwInputActivationStateTensor]).buffer.reset();
+  model->GetArray(op->inputs[kBwInputCellStateTensor]).buffer.reset();
+
+  const auto& output_weights_shape = recurrent_to_output_weights_array.shape();
+  const int output_size = output_weights_shape.dims(1);
+
+  Shape* fw_output_shape = fw_output_array.mutable_shape();
+  if (op->merge_outputs) {
+    fw_output_shape->ReplaceDims({timestamp, batch_size, 2 * output_size});
+  } else {
+    fw_output_shape->ReplaceDims({timestamp, batch_size, output_size});
+    Shape* bw_output_shape = bw_output_array.mutable_shape();
+    bw_output_shape->ReplaceDims({timestamp, batch_size, output_size});
+  }
+}
+
 void ProcessSpaceToBatchNDOperator(Model* model, SpaceToBatchNDOperator* op) {
   const auto& input_array = model->GetArray(op->inputs[0]);
   // Yield until input dims have been resolved.
@@ -2082,6 +2138,10 @@ void ProcessUniqueOperator(Model* model, UniqueOperator* op) {
       ProcessUnidirectionalSequenceRnnOperator(
           model, static_cast<UnidirectionalSequenceRnnOperator*>(op));
       break;
+    case OperatorType::kBidirectionalSequenceLstm:
+      ProcessBidirectionalSequenceLstmOperator(
+          model, static_cast<BidirectionalSequenceLstmOperator*>(op));
+      break;
     case OperatorType::kLstmCell:
       ProcessLstmCellOperator(model, static_cast<LstmCellOperator*>(op));
       break;
diff --git a/tensorflow/lite/toco/model.h b/tensorflow/lite/toco/model.h
index 3ca5e1646e..f448e3a897 100644
--- a/tensorflow/lite/toco/model.h
+++ b/tensorflow/lite/toco/model.h
@@ -160,7 +160,8 @@ enum class OperatorType : uint8 {
   kAbs,
   kMirrorPad,
   kUnique,
-  kUnidirectionalSequenceRnn
+  kUnidirectionalSequenceRnn,
+  kBidirectionalSequenceLstm
 };
 
 // Helper to deal with TensorFlow arrays using a different ordering of
@@ -650,6 +651,12 @@ struct UnidirectionalSequenceLstmOperator : Operator {
       : Operator(OperatorType::kUnidirectionalSequenceLstm) {}
 };
 
+struct BidirectionalSequenceLstmOperator : Operator {
+  BidirectionalSequenceLstmOperator()
+      : Operator(OperatorType::kBidirectionalSequenceLstm) {}
+  bool merge_outputs;
+};
+
 // Element-wise multiplication operator.
 //
 // Inputs:
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index d97f04b3f4..cd11ba3779 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -915,6 +915,53 @@ class UnidirectionalSequenceLstm
   }
 };
 
+class BidirectionalSequenceLstm
+    : public BuiltinOperator<
+          BidirectionalSequenceLstmOperator,
+          ::tflite::BidirectionalSequenceLSTMOptions,
+          ::tflite::BuiltinOptions_BidirectionalSequenceLSTMOptions> {
+ public:
+  using BuiltinOperator::BuiltinOperator;
+  flatbuffers::Offset<TfLiteOptions> WriteOptions(
+      const TocoOperator& op,
+      flatbuffers::FlatBufferBuilder* builder) const override {
+    // Current toco converter only supports tanh, no clip.
+    return ::tflite::CreateBidirectionalSequenceLSTMOptions(
+        *builder, /*fused_activation_function=*/
+        ::tflite::ActivationFunctionType_TANH,
+        /*cell_clip=*/0.0,
+        /*proj_clip=*/0.0,
+        /*merge_outputs=*/op.merge_outputs,
+        /*time_major=*/true);
+  }
+
+  void ReadOptions(const TfLiteOptions& options,
+                   TocoOperator* op) const override {
+    // Only support tanh activation, so check that tflite type is tanh.
+    DCHECK(options.fused_activation_function() ==
+           ::tflite::ActivationFunctionType_TANH);
+    op->merge_outputs = options.merge_outputs();
+  }
+
+  int GetVersion(const OperatorSignature& op_signature) const override {
+    return 1;
+  }
+
+  std::vector<bool> GetMutatingInputVariables(
+      const Operator& op) const override {
+    std::vector<bool> mutating_input_variables(op.inputs.size(), false);
+    // Forward input activation state.
+    mutating_input_variables[35] = true;
+    // Forward input cell state.
+    mutating_input_variables[36] = true;
+    // Backward input activation state.
+    mutating_input_variables[37] = true;
+    // Backward input cell state.
+    mutating_input_variables[38] = true;
+    return mutating_input_variables;
+  }
+};
+
 class Mean : public BuiltinOperator<MeanOperator, ::tflite::ReducerOptions,
                                     ::tflite::BuiltinOptions_ReducerOptions> {
  public:
@@ -1882,6 +1929,9 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
   ops.emplace_back(MakeUnique<UnidirectionalSequenceLstm>(
       ::tflite::BuiltinOperator_UNIDIRECTIONAL_SEQUENCE_LSTM,
       OperatorType::kUnidirectionalSequenceLstm));
+  ops.emplace_back(MakeUnique<BidirectionalSequenceLstm>(
+      ::tflite::BuiltinOperator_BIDIRECTIONAL_SEQUENCE_LSTM,
+      OperatorType::kBidirectionalSequenceLstm));
   ops.push_back(MakeUnique<OneHot>(::tflite::BuiltinOperator_ONE_HOT,
                                    OperatorType::kOneHot));
   ops.push_back(MakeUnique<Unpack>(::tflite::BuiltinOperator_UNPACK,
diff --git a/tensorflow/lite/toco/toco_tooling.cc b/tensorflow/lite/toco/toco_tooling.cc
index 61dca56d23..1a6b3c6f34 100644
--- a/tensorflow/lite/toco/toco_tooling.cc
+++ b/tensorflow/lite/toco/toco_tooling.cc
@@ -323,6 +323,13 @@ void Transform(const TocoFlags& toco_flags, Model* model) {
                             });
   }
 
+  // Try to merge bidirectional sequence lstm if present.
+  GraphTransformationsSet bidirectional_transformations;
+  bidirectional_transformations.Add(new RemoveUnusedOp);
+  bidirectional_transformations.Add(new toco::GroupBidirectionalSequenceLstm);
+  RunGraphTransformations(model, "Group bidirectional sequence lstm",
+                          bidirectional_transformations);
+
   // Fix any issues with IO edges. This must happen after any transform that
   // may modify the structure of the edges.
   FixEdgeArrays(model);
diff --git a/tensorflow/lite/toco/tooling_util.cc b/tensorflow/lite/toco/tooling_util.cc
index da6b3bd8e8..ad83fe4fb0 100644
--- a/tensorflow/lite/toco/tooling_util.cc
+++ b/tensorflow/lite/toco/tooling_util.cc
@@ -413,6 +413,7 @@ const char* OperatorTypeName(OperatorType type) {
     HANDLE_OPERATORTYPENAME_CASE(Unpack)
     HANDLE_OPERATORTYPENAME_CASE(ZerosLike)
     HANDLE_OPERATORTYPENAME_CASE(UnidirectionalSequenceLstm)
+    HANDLE_OPERATORTYPENAME_CASE(BidirectionalSequenceLstm)
     HANDLE_OPERATORTYPENAME_CASE(ResizeNearestNeighbor)
     HANDLE_OPERATORTYPENAME_CASE(LeakyRelu)
     HANDLE_OPERATORTYPENAME_CASE(SquaredDifference)
-- 
GitLab


From d9bdbbdc1a6c94524650549189d76dba5bc5ddc3 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Mon, 21 Jan 2019 05:27:04 -0800
Subject: [PATCH 1035/2345] Delete unimplemented function declaration.

PiperOrigin-RevId: 230214093
---
 tensorflow/compiler/xla/service/cpu/ir_emitter.h | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.h b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
index a6fb11dcbf..974dd7cd3f 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.h
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
@@ -250,14 +250,6 @@ class IrEmitter : public DfsHloVisitorWithDefault,
   llvm::Value* EmitBufferPointer(const BufferAllocation::Slice& slice,
                                  const Shape& target_shape);
 
-  // Emits a function into the current module. This can be used for
-  // computations embedded inside other computations, such as the
-  // function that a map operation applies.
-  StatusOr<llvm::Function*> EmitFunction(
-      HloComputation* function,  // The function to emit.
-      absl::string_view
-          function_name_suffix);  // Used for LLVM IR register names.
-
   // Emits a call to a thread local function (e.g. to the computation nested
   // within a reduce or a map).  Thread local callees (by definition) only write
   // to and read from thread local allocations.
-- 
GitLab


From 187091a7f9d1390a557d39949134ab08cf497bae Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 21 Jan 2019 06:03:24 -0800
Subject: [PATCH 1036/2345] Allow specifying gcc versions for the remote
 toolchain configuration.

PiperOrigin-RevId: 230217386
---
 third_party/toolchains/preconfig/generate/BUILD        | 9 +++++++++
 third_party/toolchains/preconfig/generate/generate.bzl | 3 +++
 third_party/toolchains/preconfig/generate/generate.sh  | 4 ++--
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/third_party/toolchains/preconfig/generate/BUILD b/third_party/toolchains/preconfig/generate/BUILD
index ad79255251..7bc5f2bb60 100644
--- a/third_party/toolchains/preconfig/generate/BUILD
+++ b/third_party/toolchains/preconfig/generate/BUILD
@@ -26,6 +26,15 @@ tensorflow_rbe_config(
     tensorrt_version = "5",
 )
 
+tensorflow_rbe_config(
+    name = "ubuntu14.04-py3-gcc7-cuda10.0-cudnn7-tensorrt5",
+    compiler = "gcc-7",
+    cuda_version = "10.0",
+    cudnn_version = "7",
+    python_version = "3",
+    tensorrt_version = "5",
+)
+
 tensorflow_rbe_config(
     name = "ubuntu14.04-py3-gcc-cuda10.0-cudnn7-tensorrt5",
     compiler = "gcc",
diff --git a/third_party/toolchains/preconfig/generate/generate.bzl b/third_party/toolchains/preconfig/generate/generate.bzl
index fc485d43d2..40e0957cf2 100644
--- a/third_party/toolchains/preconfig/generate/generate.bzl
+++ b/third_party/toolchains/preconfig/generate/generate.bzl
@@ -21,6 +21,8 @@ def _tensorflow_rbe_config(name, compiler, python_version, cuda_version = None,
         "CC": compiler,
         "PYTHON_BIN_PATH": "/usr/bin/python%s" % python_version,
         "CLEAR_CACHE": "1",
+        "HOST_CXX_COMPILER": compiler,
+        "HOST_C_COMPILER": compiler,
     }
 
     if cuda_version != None:
@@ -43,6 +45,7 @@ def _tensorflow_rbe_config(name, compiler, python_version, cuda_version = None,
             "TF_NEED_TENSORRT" : "1",
             "TF_TENSORRT_VERSION": tensorrt_version,
             "TENSORRT_INSTALL_PATH": "/usr/lib/x86_64-linux-gnu",
+            "GCC_HOST_COMPILER_PATH": compiler if compiler != "clang" else "",
         })
 
     docker_toolchain_autoconfig(
diff --git a/third_party/toolchains/preconfig/generate/generate.sh b/third_party/toolchains/preconfig/generate/generate.sh
index c05a4de6fb..e39daa6a30 100755
--- a/third_party/toolchains/preconfig/generate/generate.sh
+++ b/third_party/toolchains/preconfig/generate/generate.sh
@@ -38,8 +38,8 @@ TENSORRT_VERSION="${PLATFORM[5]}"
 # TODO(klimek): Put this into the name.
 
 if [[ -n "${CUDA_VERSION}" ]]; then
-  if [[ "${COMPILER}" == "gcc" ]]; then
-    COMPILER="gcc-nvcc-${CUDA_VERSION}"
+  if [[ "${COMPILER}" == gcc* ]]; then
+    COMPILER="${COMPILER}-nvcc-${CUDA_VERSION}"
   fi
   # Currently we create a special toolchain for clang when compiling with
   # cuda enabled. We can get rid of this once the default toolchain bazel
-- 
GitLab


From 4bfa2359152e9d106c2c20e9fff67643c8578c81 Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Mon, 21 Jan 2019 07:42:02 -0800
Subject: [PATCH 1037/2345] Add performance note for recursive methods.

Addresses comments from tensorflow/community#56.

PiperOrigin-RevId: 230225963
---
 tensorflow/python/module/module.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tensorflow/python/module/module.py b/tensorflow/python/module/module.py
index 2ba6b39bc0..f9b8260025 100644
--- a/tensorflow/python/module/module.py
+++ b/tensorflow/python/module/module.py
@@ -201,6 +201,10 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
   def variables(self):
     """Collection of variables owned by this module and it's submodules.
 
+    Note: this method uses reflection to find variables on the current instance
+    and submodules. For performance reasons you may wish to cache the result
+    of calling this method if you don't expect the return value to change.
+
     Returns:
       A collection of variables for the current module (sorted by attribute
       name) followed by variables from all submodules recursively (depth first).
@@ -225,6 +229,10 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
   def trainable_variables(self):
     """Collection of variables owned by this module and it's submodules.
 
+    Note: this method uses reflection to find variables on the current instance
+    and submodules. For performance reasons you may wish to cache the result
+    of calling this method if you don't expect the return value to change.
+
     Returns:
       A collection of variables for the current module (sorted by attribute
       name) followed by variables from all submodules recursively (depth first).
-- 
GitLab


From dad786f9638bc7be6d79e0d4bb087a955d973a6a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 21 Jan 2019 08:21:47 -0800
Subject: [PATCH 1038/2345] Deflake TPUGANEstimator test.

PiperOrigin-RevId: 230229729
---
 .../gan/python/estimator/python/tpu_gan_estimator_test.py      | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/contrib/gan/python/estimator/python/tpu_gan_estimator_test.py b/tensorflow/contrib/gan/python/estimator/python/tpu_gan_estimator_test.py
index 9fdcc08334..0d9e6489bd 100644
--- a/tensorflow/contrib/gan/python/estimator/python/tpu_gan_estimator_test.py
+++ b/tensorflow/contrib/gan/python/estimator/python/tpu_gan_estimator_test.py
@@ -184,8 +184,7 @@ class TPUGANEstimatorIntegrationTest(test.TestCase, parameterized.TestCase):
     # Evaluate.
     num_steps_eval = 2
     scores = est.evaluate(eval_input_fn, steps=num_steps_eval)
-    self.assertEqual(num_steps_train + num_steps_eval,
-                     scores[ops.GraphKeys.GLOBAL_STEP])
+    self.assertIn(ops.GraphKeys.GLOBAL_STEP, six.iterkeys(scores))
     self.assertIn('loss', six.iterkeys(scores))
     self.assertEqual(scores['discriminator_loss'] + scores['generator_loss'],
                      scores['loss'])
-- 
GitLab


From a088c1019e83d4c5b7a71370a6e2bd70c89d0bf7 Mon Sep 17 00:00:00 2001
From: "Wen-Heng (Jack) Chung" <whchung@gmail.com>
Date: Mon, 21 Jan 2019 11:16:41 -0600
Subject: [PATCH 1039/2345] Fix copy rules for ROCm path

Fix incorrect targets for ROCm headers, and how ROCm headers are copied into
execroot.
---
 third_party/gpus/cuda_configure.bzl | 2 +-
 third_party/gpus/rocm_configure.bzl | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl
index 76f2f3a07b..40c5e373ed 100644
--- a/third_party/gpus/cuda_configure.bzl
+++ b/third_party/gpus/cuda_configure.bzl
@@ -1193,7 +1193,7 @@ def make_copy_dir_rule(repository_ctx, name, src_dir, out_dir):
     outs = [
 %s
     ],
-    cmd = \"""cp -rf "%s/." "%s/" \""",
+    cmd = \"""cp -rLf "%s/." "%s/" \""",
 )""" % (name, "\n".join(outs), src_dir, out_dir)
 
 def _read_dir(repository_ctx, src_dir):
diff --git a/third_party/gpus/rocm_configure.bzl b/third_party/gpus/rocm_configure.bzl
index a57ec9f7a6..f67fb8ae38 100644
--- a/third_party/gpus/rocm_configure.bzl
+++ b/third_party/gpus/rocm_configure.bzl
@@ -580,13 +580,13 @@ def _create_local_rocm_repository(repository_ctx):
         ),
         make_copy_dir_rule(
             repository_ctx,
-            name = "rocm-blas",
+            name = "rocblas-include",
             src_dir = rocm_toolkit_path + "/rocblas/include",
             out_dir = "rocm/include/rocblas",
         ),
         make_copy_dir_rule(
             repository_ctx,
-            name = "rocm-miopen",
+            name = "miopen-include",
             src_dir = rocm_toolkit_path + "/miopen/include",
             out_dir = "rocm/include/miopen",
         ),
-- 
GitLab


From ff91cd691027076e6128afbd902d3d38e3672787 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 21 Jan 2019 11:45:07 -0800
Subject: [PATCH 1040/2345] Use CUDA tools from @local_config_cuda when
 building NCCL.

PiperOrigin-RevId: 230245994
---
 third_party/nccl/build_defs.bzl.tpl |  8 ++++----
 third_party/nccl/nccl_configure.bzl | 16 ----------------
 2 files changed, 4 insertions(+), 20 deletions(-)

diff --git a/third_party/nccl/build_defs.bzl.tpl b/third_party/nccl/build_defs.bzl.tpl
index 1fb90eea22..245f180a91 100644
--- a/third_party/nccl/build_defs.bzl.tpl
+++ b/third_party/nccl/build_defs.bzl.tpl
@@ -202,25 +202,25 @@ _device_link = rule(
         "gpu_archs": attr.string_list(),
         "nvlink_args": attr.string_list(),
         "_nvlink": attr.label(
-            default = Label("@local_config_nccl//:nvlink"),
+            default = Label("@local_config_cuda//cuda:cuda/bin/nvlink"),
             allow_single_file = True,
             executable = True,
             cfg = "host",
         ),
         "_fatbinary": attr.label(
-            default = Label("@local_config_nccl//:cuda/bin/fatbinary"),
+            default = Label("@local_config_cuda//cuda:cuda/bin/fatbinary"),
             allow_single_file = True,
             executable = True,
             cfg = "host",
         ),
         "_bin2c": attr.label(
-            default = Label("@local_config_nccl//:cuda/bin/bin2c"),
+            default = Label("@local_config_cuda//cuda:cuda/bin/bin2c"),
             allow_single_file = True,
             executable = True,
             cfg = "host",
         ),
         "_link_stub": attr.label(
-            default = Label("@local_config_nccl//:cuda/bin/crt/link.stub"),
+            default = Label("@local_config_cuda//cuda:cuda/bin/crt/link.stub"),
             allow_single_file = True,
         ),
     },
diff --git a/third_party/nccl/nccl_configure.bzl b/third_party/nccl/nccl_configure.bzl
index 59b5bd3fea..3a836fadc3 100644
--- a/third_party/nccl/nccl_configure.bzl
+++ b/third_party/nccl/nccl_configure.bzl
@@ -43,13 +43,6 @@ cc_library(
 """
 
 _NCCL_ARCHIVE_BUILD_CONTENT = """
-exports_files([
-    "cuda/bin/crt/link.stub",
-    "cuda/bin/fatbinary",
-    "cuda/bin/bin2c",
-    "nvlink",
-])
-
 filegroup(
   name = "LICENSE",
   data = ["@nccl_archive//:LICENSE.txt"],
@@ -151,15 +144,6 @@ def _nccl_configure_impl(repository_ctx):
         repository_ctx.template("build_defs.bzl", _label("build_defs.bzl.tpl"), {
             "%{gpu_architectures}": str(gpu_architectures),
         })
-
-        repository_ctx.symlink(cuda_toolkit_path(repository_ctx), "cuda")
-
-        # Temporary work-around for setups which symlink ptxas to a newer
-        # version. The versions of nvlink and ptxas need to agree, so we find
-        # nvlink next to the real location of ptxas. This is only temporary and
-        # will be removed again soon.
-        nvlink_dir = repository_ctx.path("cuda/bin/ptxas").realpath.dirname
-        repository_ctx.symlink(nvlink_dir.get_child("nvlink"), "nvlink")
     else:
         # Create target for locally installed NCCL.
         nccl_install_path = repository_ctx.os.environ[_NCCL_INSTALL_PATH].strip()
-- 
GitLab


From 125ecf68ccb6d94dfc2f05d68017e37232ad59c2 Mon Sep 17 00:00:00 2001
From: Shivani Agrawal <shivaniagrawal@google.com>
Date: Mon, 21 Jan 2019 19:53:54 -0800
Subject: [PATCH 1041/2345] [data.optimization] Adds more checks for stateful
 "If" and "While" ops.

PiperOrigin-RevId: 230274771
---
 .../core/grappler/optimizers/data/BUILD       |   5 +-
 .../optimizers/data/function_utils.cc         |  51 ++
 .../grappler/optimizers/data/function_utils.h |  22 +-
 .../optimizers/data/function_utils_test.cc    | 688 ++++++++++++++++++
 .../optimizers/data/hoist_random_uniform.cc   |   6 +-
 .../optimizers/data/map_parallelization.cc    |  21 +-
 .../optimizers/data/map_vectorization.cc      |  13 +-
 7 files changed, 769 insertions(+), 37 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/data/BUILD b/tensorflow/core/grappler/optimizers/data/BUILD
index 4ca353d7ba..0a645d5cf4 100644
--- a/tensorflow/core/grappler/optimizers/data/BUILD
+++ b/tensorflow/core/grappler/optimizers/data/BUILD
@@ -125,7 +125,9 @@ tf_cc_test(
     srcs = ["function_utils_test.cc"],
     deps = [
         ":function_utils",
+        ":graph_utils",
         "//tensorflow/core:framework",
+        "//tensorflow/core:ops",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
@@ -133,7 +135,7 @@ tf_cc_test(
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
         "//tensorflow/core/kernels:cast_op",
         "//tensorflow/tools/graph_transforms:transform_utils",
-    ],
+    ] + tf_protos_all(),
 )
 
 cc_library(
@@ -438,6 +440,7 @@ cc_library(
         "map_parallelization.h",
     ],
     deps = [
+        ":function_utils",
         ":graph_utils",
         ":optimizer_base",
         "@com_google_absl//absl/container:flat_hash_set",
diff --git a/tensorflow/core/grappler/optimizers/data/function_utils.cc b/tensorflow/core/grappler/optimizers/data/function_utils.cc
index 351209f92e..20536910db 100644
--- a/tensorflow/core/grappler/optimizers/data/function_utils.cc
+++ b/tensorflow/core/grappler/optimizers/data/function_utils.cc
@@ -171,6 +171,57 @@ void SetUniqueFunctionNodeName(StringPiece prefix, FunctionDef* function,
   node->set_name(std::move(name));
 }
 
+bool IsFunctionStateful(const FunctionLibraryDefinition& library,
+                        const FunctionDef& function_def, bool skip_assert) {
+  if (!function_def.signature().is_stateful()) return false;
+
+  for (const NodeDef& node_def : function_def.node_def()) {
+    if (IsNodeStateful(library, node_def, skip_assert)) return true;
+  }
+  return false;
+}
+
+bool IsNodeStateful(const FunctionLibraryDefinition& library,
+                    const NodeDef& node, bool skip_assert) {
+  const OpDef* op_def;
+  Status s = OpRegistry::Global()->LookUpOpDef(node.op(), &op_def);
+
+  if (!s.ok()) return true;
+
+  if (!op_def->is_stateful()) return false;
+
+  if (skip_assert && op_def->name() == "Assert") {
+    return false;
+  }
+
+  if (op_def->name() == "If") {
+    const FunctionDef* then_func =
+        library.Find(node.attr().at("then_branch").func().name());
+    const FunctionDef* else_func =
+        library.Find(node.attr().at("else_branch").func().name());
+    if ((then_func != nullptr &&
+         !IsFunctionStateful(library, *then_func, skip_assert)) &&
+        (else_func != nullptr &&
+         !IsFunctionStateful(library, *else_func, skip_assert))) {
+      return false;
+    }
+  }
+
+  if (op_def->name() == "While") {
+    const FunctionDef* cond_func =
+        library.Find(node.attr().at("cond").func().name());
+    const FunctionDef* body_func =
+        library.Find(node.attr().at("body").func().name());
+    if ((cond_func != nullptr &&
+         !IsFunctionStateful(library, *cond_func, skip_assert)) &&
+        (body_func != nullptr &&
+         !IsFunctionStateful(library, *body_func, skip_assert))) {
+      return false;
+    }
+  }
+  return true;
+}
+
 }  // namespace function_utils
 }  // namespace grappler
 }  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/function_utils.h b/tensorflow/core/grappler/optimizers/data/function_utils.h
index ec08c11dce..79271e8ad0 100644
--- a/tensorflow/core/grappler/optimizers/data/function_utils.h
+++ b/tensorflow/core/grappler/optimizers/data/function_utils.h
@@ -101,8 +101,24 @@ int FindFunctionNodeWithOp(StringPiece op, const FunctionDef& function);
 void SetUniqueFunctionNodeName(StringPiece prefix, FunctionDef* function,
                                NodeDef* node);
 
-}  // namespace function_utils
-}  // namespace grappler
-}  // namespace tensorflow
+// Checks if the function is stateful by checking the function graph for
+// stateful ops. Because the "If" and "While" ops are conservatively marked as
+// stateful, the check recurses into their graph to determine whether they are
+// actually stateful. The `skip_assert` argument determines whether the "Assert"
+// op should be treated as stateful or not.
+bool IsFunctionStateful(const FunctionLibraryDefinition& library,
+                        const FunctionDef& function_def,
+                        bool skip_assert = false);
+
+// Checks if the node is stateful. Because the "If" or "While" ops are
+// conservatively marked as stateful, the check recurses into their graph to
+// determine whether they are actually stateful. The `skip_assert` argument
+// determines whether the "Assert" op  should be treated as stateful or not.
+bool IsNodeStateful(const FunctionLibraryDefinition& library,
+                    const NodeDef& node, bool skip_assert = false);
+
+}  // end namespace function_utils
+}  // end namespace grappler
+}  // end namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_FUNCTION_UTILS_H_
diff --git a/tensorflow/core/grappler/optimizers/data/function_utils_test.cc b/tensorflow/core/grappler/optimizers/data/function_utils_test.cc
index 3739e20eb1..8ae0cde4cd 100644
--- a/tensorflow/core/grappler/optimizers/data/function_utils_test.cc
+++ b/tensorflow/core/grappler/optimizers/data/function_utils_test.cc
@@ -16,6 +16,8 @@ limitations under the License.
 #include "tensorflow/core/grappler/optimizers/data/function_utils.h"
 
 #include "tensorflow/core/framework/function_testlib.h"
+#include "tensorflow/core/framework/graph.pb.h"
+#include "tensorflow/core/grappler/optimizers/data/graph_utils.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/tools/graph_transforms/transform_utils.h"
@@ -158,6 +160,692 @@ TEST(FunctionUtilsTest, AddNodeToFunctionDef) {
   }
 }
 
+// Graph containing function with "If" and "Assert" Op.
+/*
+  @eager_function.defun
+  def test_function():
+    pred = constant_op.constant(True)
+
+    def fn1():
+      return control_flow_ops.no_op()
+
+    def fn2():
+      return control_flow_ops.Assert(False, ["Wrong branch!!!"])
+
+    return control_flow_ops.cond(pred, fn1, fn2)
+
+  r = test_function()
+*/
+// Following proto is generated in python using the above code block, to
+// regenerate get the graph_def from the default graph/specified graph for the
+// code block (e.g ops.get_default_graph.as_graph_def()).
+constexpr char kCondGraphProto[] = R"proto(
+  node {
+    name: "StatefulPartitionedCall"
+    op: "StatefulPartitionedCall"
+    attr {
+      key: "Tin"
+      value { list {} }
+    }
+    attr {
+      key: "Tout"
+      value { list { type: DT_BOOL } }
+    }
+    attr {
+      key: "_gradient_op_type"
+      value { s: "PartitionedCall-20" }
+    }
+    attr {
+      key: "config"
+      value { s: "" }
+    }
+    attr {
+      key: "config_proto"
+      value { s: "" }
+    }
+    attr {
+      key: "executor_type"
+      value { s: "" }
+    }
+    attr {
+      key: "f"
+      value { func { name: "__inference_test_function_19" } }
+    }
+  }
+  library {
+    function {
+      signature {
+        name: "cond_true_3"
+        input_arg { name: "identity_const" type: DT_BOOL }
+        output_arg { name: "identity_1" type: DT_BOOL }
+      }
+      node_def { name: "NoOp" op: "NoOp" }
+      node_def {
+        name: "Identity"
+        op: "Identity"
+        input: "identity_const"
+        input: "^NoOp"
+        attr {
+          key: "T"
+          value { type: DT_BOOL }
+        }
+      }
+      node_def {
+        name: "Identity_1"
+        op: "Identity"
+        input: "Identity:output:0"
+        attr {
+          key: "T"
+          value { type: DT_BOOL }
+        }
+      }
+      ret { key: "identity_1" value: "Identity_1:output:0" }
+    }
+    function {
+      signature {
+        name: "cond_false_4"
+        input_arg { name: "identity_const" type: DT_BOOL }
+        output_arg { name: "identity_1" type: DT_BOOL }
+        is_stateful: true
+      }
+      node_def {
+        name: "Assert/Const"
+        op: "Const"
+        attr {
+          key: "dtype"
+          value { type: DT_STRING }
+        }
+        attr {
+          key: "value"
+          value {
+            tensor {
+              dtype: DT_STRING
+              tensor_shape {}
+              string_val: "Wrong branch!!!"
+            }
+          }
+        }
+      }
+      node_def {
+        name: "Assert/Assert/condition"
+        op: "Const"
+        attr {
+          key: "dtype"
+          value { type: DT_BOOL }
+        }
+        attr {
+          key: "value"
+          value {
+            tensor {
+              dtype: DT_BOOL
+              tensor_shape {}
+              bool_val: false
+            }
+          }
+        }
+      }
+      node_def {
+        name: "Assert/Assert/data_0"
+        op: "Const"
+        attr {
+          key: "dtype"
+          value { type: DT_STRING }
+        }
+        attr {
+          key: "value"
+          value {
+            tensor {
+              dtype: DT_STRING
+              tensor_shape {}
+              string_val: "Wrong branch!!!"
+            }
+          }
+        }
+      }
+      node_def {
+        name: "Assert/Assert"
+        op: "Assert"
+        input: "Assert/Assert/condition:output:0"
+        input: "Assert/Assert/data_0:output:0"
+        attr {
+          key: "T"
+          value { list { type: DT_STRING } }
+        }
+        attr {
+          key: "summarize"
+          value { i: 3 }
+        }
+      }
+      node_def {
+        name: "Identity"
+        op: "Identity"
+        input: "identity_const"
+        input: "^Assert/Assert"
+        attr {
+          key: "T"
+          value { type: DT_BOOL }
+        }
+      }
+      node_def {
+        name: "Identity_1"
+        op: "Identity"
+        input: "Identity:output:0"
+        input: "^Assert/Assert"
+        attr {
+          key: "T"
+          value { type: DT_BOOL }
+        }
+      }
+      ret { key: "identity_1" value: "Identity_1:output:0" }
+    }
+    function {
+      signature {
+        name: "__inference_test_function_19"
+        output_arg { name: "identity" type: DT_BOOL }
+        is_stateful: true
+      }
+      node_def {
+        name: "Const"
+        op: "Const"
+        attr {
+          key: "dtype"
+          value { type: DT_BOOL }
+        }
+        attr {
+          key: "value"
+          value {
+            tensor {
+              dtype: DT_BOOL
+              tensor_shape {}
+              bool_val: true
+            }
+          }
+        }
+      }
+      node_def {
+        name: "cond"
+        op: "If"
+        input: "Const:output:0"
+        input: "Const:output:0"
+        attr {
+          key: "Tcond"
+          value { type: DT_BOOL }
+        }
+        attr {
+          key: "Tin"
+          value { list { type: DT_BOOL } }
+        }
+        attr {
+          key: "Tout"
+          value { list { type: DT_BOOL } }
+        }
+        attr {
+          key: "_lower_using_switch_merge"
+          value { b: true }
+        }
+        attr {
+          key: "else_branch"
+          value { func { name: "cond_false_4" } }
+        }
+        attr {
+          key: "output_shapes"
+          value { list { shape {} } }
+        }
+        attr {
+          key: "then_branch"
+          value { func { name: "cond_true_3" } }
+        }
+      }
+      node_def {
+        name: "cond/Identity"
+        op: "Identity"
+        input: "cond:output:0"
+        attr {
+          key: "T"
+          value { type: DT_BOOL }
+        }
+      }
+      node_def {
+        name: "Identity"
+        op: "Identity"
+        input: "cond/Identity:output:0"
+        input: "^cond"
+        attr {
+          key: "T"
+          value { type: DT_BOOL }
+        }
+      }
+      ret { key: "identity" value: "Identity:output:0" }
+    }
+  }
+  versions { producer: 27 min_consumer: 12 })proto";
+
+// Graph containing function with "While" Op in python.
+/*
+  @eager_function.defun
+  def test_function():
+    return control_flow_ops.while_loop(
+        lambda i: i < 3, lambda i: i + 1, [0], maximum_iterations=1)
+
+  r = test_function()
+*/
+// Following proto is generated in python using the above code block, to
+// regenerate get the graph_def from the default graph/specified graph for the
+// code block (e.g ops.get_default_graph.as_graph_def()).
+constexpr char kWhileGraphProto[] = R"proto(
+  node {
+    name: "StatefulPartitionedCall"
+    op: "StatefulPartitionedCall"
+    attr {
+      key: "Tin"
+      value { list {} }
+    }
+    attr {
+      key: "Tout"
+      value { list { type: DT_INT32 } }
+    }
+    attr {
+      key: "_gradient_op_type"
+      value { s: "PartitionedCall-35" }
+    }
+    attr {
+      key: "config"
+      value { s: "" }
+    }
+    attr {
+      key: "config_proto"
+      value { s: "" }
+    }
+    attr {
+      key: "executor_type"
+      value { s: "" }
+    }
+    attr {
+      key: "f"
+      value { func { name: "__inference_test_function_34" } }
+    }
+  }
+  library {
+    function {
+      signature {
+        name: "while_body_5"
+        input_arg { name: "while_loop_counter" type: DT_INT32 }
+        input_arg { name: "const" type: DT_INT32 }
+        input_arg { name: "maximum_iterations" type: DT_INT32 }
+        output_arg { name: "identity" type: DT_INT32 }
+        output_arg { name: "identity_1" type: DT_INT32 }
+        output_arg { name: "identity_2" type: DT_INT32 }
+      }
+      node_def {
+        name: "add/y"
+        op: "Const"
+        attr {
+          key: "dtype"
+          value { type: DT_INT32 }
+        }
+        attr {
+          key: "value"
+          value {
+            tensor {
+              dtype: DT_INT32
+              tensor_shape {}
+              int_val: 1
+            }
+          }
+        }
+      }
+      node_def {
+        name: "add"
+        op: "Add"
+        input: "const"
+        input: "add/y:output:0"
+        attr {
+          key: "T"
+          value { type: DT_INT32 }
+        }
+      }
+      node_def {
+        name: "add_1/y"
+        op: "Const"
+        attr {
+          key: "dtype"
+          value { type: DT_INT32 }
+        }
+        attr {
+          key: "value"
+          value {
+            tensor {
+              dtype: DT_INT32
+              tensor_shape {}
+              int_val: 1
+            }
+          }
+        }
+      }
+      node_def {
+        name: "add_1"
+        op: "Add"
+        input: "while_loop_counter"
+        input: "add_1/y:output:0"
+        attr {
+          key: "T"
+          value { type: DT_INT32 }
+        }
+      }
+      node_def {
+        name: "Identity"
+        op: "Identity"
+        input: "add_1:z:0"
+        attr {
+          key: "T"
+          value { type: DT_INT32 }
+        }
+      }
+      node_def {
+        name: "Identity_1"
+        op: "Identity"
+        input: "add:z:0"
+        attr {
+          key: "T"
+          value { type: DT_INT32 }
+        }
+      }
+      node_def {
+        name: "Identity_2"
+        op: "Identity"
+        input: "maximum_iterations"
+        attr {
+          key: "T"
+          value { type: DT_INT32 }
+        }
+      }
+      ret { key: "identity" value: "Identity:output:0" }
+      ret { key: "identity_1" value: "Identity_1:output:0" }
+      ret { key: "identity_2" value: "Identity_2:output:0" }
+    }
+    function {
+      signature {
+        name: "__inference_test_function_34"
+        output_arg { name: "identity" type: DT_INT32 }
+        is_stateful: true
+      }
+      node_def {
+        name: "maximum_iterations"
+        op: "Const"
+        attr {
+          key: "dtype"
+          value { type: DT_INT32 }
+        }
+        attr {
+          key: "value"
+          value {
+            tensor {
+              dtype: DT_INT32
+              tensor_shape {}
+              int_val: 1
+            }
+          }
+        }
+      }
+      node_def {
+        name: "Const"
+        op: "Const"
+        attr {
+          key: "dtype"
+          value { type: DT_INT32 }
+        }
+        attr {
+          key: "value"
+          value {
+            tensor {
+              dtype: DT_INT32
+              tensor_shape {}
+              int_val: 0
+            }
+          }
+        }
+      }
+      node_def {
+        name: "while/loop_counter"
+        op: "Const"
+        attr {
+          key: "dtype"
+          value { type: DT_INT32 }
+        }
+        attr {
+          key: "value"
+          value {
+            tensor {
+              dtype: DT_INT32
+              tensor_shape {}
+              int_val: 0
+            }
+          }
+        }
+      }
+      node_def {
+        name: "while"
+        op: "While"
+        input: "while/loop_counter:output:0"
+        input: "Const:output:0"
+        input: "maximum_iterations:output:0"
+        attr {
+          key: "T"
+          value { list { type: DT_INT32 type: DT_INT32 type: DT_INT32 } }
+        }
+        attr {
+          key: "_lower_using_switch_merge"
+          value { b: true }
+        }
+        attr {
+          key: "body"
+          value { func { name: "while_body_5" } }
+        }
+        attr {
+          key: "cond"
+          value { func { name: "while_cond_4" } }
+        }
+        attr {
+          key: "output_shapes"
+          value {
+            list {
+              shape {}
+              shape {}
+              shape {}
+            }
+          }
+        }
+      }
+      node_def {
+        name: "while/Identity"
+        op: "Identity"
+        input: "while:output:0"
+        attr {
+          key: "T"
+          value { type: DT_INT32 }
+        }
+      }
+      node_def {
+        name: "while/Identity_1"
+        op: "Identity"
+        input: "while:output:1"
+        attr {
+          key: "T"
+          value { type: DT_INT32 }
+        }
+      }
+      node_def {
+        name: "while/Identity_2"
+        op: "Identity"
+        input: "while:output:2"
+        attr {
+          key: "T"
+          value { type: DT_INT32 }
+        }
+      }
+      node_def {
+        name: "Identity"
+        op: "Identity"
+        input: "while/Identity_1:output:0"
+        input: "^while"
+        attr {
+          key: "T"
+          value { type: DT_INT32 }
+        }
+      }
+      ret { key: "identity" value: "Identity:output:0" }
+    }
+    function {
+      signature {
+        name: "while_cond_4"
+        input_arg { name: "while_loop_counter" type: DT_INT32 }
+        input_arg { name: "const" type: DT_INT32 }
+        input_arg { name: "less_maximum_iterations" type: DT_INT32 }
+        output_arg { name: "identity" type: DT_BOOL }
+      }
+      node_def {
+        name: "Less"
+        op: "Less"
+        input: "while_loop_counter"
+        input: "less_maximum_iterations"
+        attr {
+          key: "T"
+          value { type: DT_INT32 }
+        }
+      }
+      node_def {
+        name: "Less_1/y"
+        op: "Const"
+        attr {
+          key: "dtype"
+          value { type: DT_INT32 }
+        }
+        attr {
+          key: "value"
+          value {
+            tensor {
+              dtype: DT_INT32
+              tensor_shape {}
+              int_val: 3
+            }
+          }
+        }
+      }
+      node_def {
+        name: "Less_1"
+        op: "Less"
+        input: "const"
+        input: "Less_1/y:output:0"
+        attr {
+          key: "T"
+          value { type: DT_INT32 }
+        }
+      }
+      node_def {
+        name: "LogicalAnd"
+        op: "LogicalAnd"
+        input: "Less:z:0"
+        input: "Less_1:z:0"
+      }
+      node_def {
+        name: "Identity"
+        op: "Identity"
+        input: "LogicalAnd:z:0"
+        attr {
+          key: "T"
+          value { type: DT_BOOL }
+        }
+      }
+      ret { key: "identity" value: "Identity:output:0" }
+    }
+  }
+  versions { producer: 27 min_consumer: 12 })proto";
+
+// TODO(shivaniagrawal): split the test into multiple tests for better
+// readability and add full coverage i.e. add/separate out the tests for all
+// branches of IsNodeStateful and IsFunctionStateful:
+// - test for IsNodeStateful for Cond that has a stateful branch
+// - test for IsNodeStateful for Cond that does not have a stateful branches
+// - test for IsNodeStateful for While that has a stateful branch
+// - test for IsNodeStateful for While that does not have a stateful branches
+// - test for IsNodeStateful for Assert
+// - test for IsNodeStateful for a stateful op
+// - test for IsNodeStateful for a stateless op
+//
+// - test for IsFunctionStateful for a function that contains a Cond
+// - test for IsFunctionStateful for a function that contains a While
+// - test for IsFunctionStateful for a function that contains an Assert (and no
+//   other stateful op)
+// - test for IsFunctionStateful for a function that contains a stateful op
+//   other than Assert
+// - test for IsFunctionStateful for a function that does not contain a stateful
+//   op
+
+TEST(FunctionUtilsTest, IsFunctionStateful) {
+  GraphDef graph_def;
+  MutableGraphView graph(&graph_def);
+
+  NodeDef* nodeA = graph_utils::AddNode("", "A", {}, {}, &graph);
+  FunctionDef* function = graph_def.mutable_library()->add_function();
+  *function = test::function::XTimesTwo();
+
+  FunctionLibraryDefinition lib_def(OpRegistry::Global(),
+                                    *graph_def.mutable_library());
+
+  EXPECT_FALSE(IsFunctionStateful(lib_def, *function));
+
+  // Op "A" is not a registered Op.
+  EXPECT_TRUE(IsNodeStateful(lib_def, *nodeA));
+
+  // Get graph_def for the graph `kCondGraphProto`, graph with function
+  // containing "If" and "Assert" Op.
+
+  GraphDef graph_def_cond;
+  protobuf::TextFormat::ParseFromString(kCondGraphProto, &graph_def_cond);
+  FunctionLibraryDefinition cond_lib(OpRegistry::Global(),
+                                     graph_def_cond.library());
+
+  const FunctionDef* no_op_fnc = cond_lib.Find("cond_true_3");
+
+  EXPECT_FALSE(IsFunctionStateful(cond_lib, *no_op_fnc));
+  EXPECT_FALSE(IsFunctionStateful(cond_lib, *no_op_fnc, true));
+
+  const FunctionDef* assert_func = cond_lib.Find("cond_false_4");
+
+  EXPECT_TRUE(IsFunctionStateful(cond_lib, *assert_func));
+  EXPECT_FALSE(IsFunctionStateful(cond_lib, *assert_func, true));
+
+  EXPECT_TRUE(ContainsFunctionNodeWithOp("Const", *assert_func));
+  EXPECT_TRUE(ContainsFunctionNodeWithOp("Assert", *assert_func));
+
+  for (auto node : assert_func->node_def()) {
+    if (node.op() == "Const") {
+      EXPECT_FALSE(IsNodeStateful(lib_def, node));
+    }
+    if (node.op() == "Assert") {
+      EXPECT_TRUE(IsNodeStateful(lib_def, node));
+      EXPECT_FALSE(IsNodeStateful(lib_def, node, true));
+    }
+  }
+
+  const FunctionDef* cond_func = cond_lib.Find("__inference_test_function_19");
+
+  EXPECT_TRUE(IsFunctionStateful(cond_lib, *cond_func));
+  EXPECT_FALSE(IsFunctionStateful(cond_lib, *cond_func, true));
+
+  // Get graph def for the graph `kWhileGraphProto`, graph with function
+  // containing "While" Op.
+
+  GraphDef graph_def_while;
+  protobuf::TextFormat::ParseFromString(kWhileGraphProto, &graph_def_while);
+
+  FunctionLibraryDefinition while_lib(OpRegistry::Global(),
+                                      graph_def_while.library());
+  const FunctionDef* while_function =
+      while_lib.Find("__inference_test_function_34");
+  EXPECT_FALSE(IsFunctionStateful(while_lib, *while_function));
+  EXPECT_FALSE(IsFunctionStateful(while_lib, *while_function, true));
+}
 }  // namespace
 }  // namespace function_utils
 }  // namespace grappler
diff --git a/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc b/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc
index 496277780f..e29b620140 100644
--- a/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc
+++ b/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc
@@ -174,7 +174,7 @@ const FunctionDef* MakeLessStatefulFunction(const FunctionDef& map_function,
   return stateless_function;
 }
 // This function returns true if function is stateful and has single
-// RandomUniform op and no other stateful ops except Assert.
+// RandomUniform op and no other stateful ops except Assert and If/While.
 // `is_stateful_after_hoisting` is set to true if RandomUniform is the only
 // stateful op and hoisting can be performed.
 bool CanHoistRandomUniform(const FunctionDef& map_function,
@@ -189,10 +189,10 @@ bool CanHoistRandomUniform(const FunctionDef& map_function,
   for (const auto& node : map_function.node_def()) {
     const OpDef* op_def;
     TF_CHECK_OK(library.LookUpOpDef(node.op(), &op_def));
-    // Skip stateless nodes and assert, as it does not actually have a state.
     if (!op_def->is_stateful()) continue;
 
-    if (op_def->name() == "Assert") {
+    if (!function_utils::IsNodeStateful(library, node, true)) {
+      // Skip ops that are marked stateful but are in fact not stateful.
       have_other_stateful_ops = true;
       continue;
     }
diff --git a/tensorflow/core/grappler/optimizers/data/map_parallelization.cc b/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
index 57e541cde6..90dd885c7f 100644
--- a/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/mutable_graph_view.h"
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.h"
+#include "tensorflow/core/grappler/optimizers/data/function_utils.h"
 #include "tensorflow/core/grappler/optimizers/data/graph_utils.h"
 #include "tensorflow/core/grappler/utils.h"
 
@@ -33,23 +34,6 @@ constexpr char kMapDataset[] = "MapDataset";
 constexpr char kParallelMapDataset[] = "ParallelMapDataset";
 constexpr int kAutotune = -1;
 
-bool CanParallelize(const FunctionDef& function,
-                    const FunctionLibraryDefinition& library) {
-  if (!function.signature().is_stateful()) return true;
-
-  for (const auto& node : function.node_def()) {
-    const OpDef* op_def;
-    TF_CHECK_OK(library.LookUpOpDef(node.op(), &op_def));
-    // Assert is marked as stateful, but it does not have any state (except
-    // changing io).  Similarly to CUDA, we do not give guarantee that the
-    // assert operation that would fail would be the first one, so that we can
-    // parallelize it.
-    if (op_def->is_stateful() && op_def->name() != "Assert") return false;
-  }
-
-  return true;
-}
-
 NodeDef MakeParallelMap(const string& name, MutableGraphView* graph) {
   // The inputs of the node to be parallelized could be changed by the
   // optimization pass, so we need to look it up in the modified graph.
@@ -88,7 +72,8 @@ Status MapParallelization::OptimizeAndCollectStats(Cluster* cluster,
 
     auto* function =
         function_library.Find(map_node->attr().at("f").func().name());
-    if (!CanParallelize(*function, function_library)) continue;
+    if (function_utils::IsFunctionStateful(function_library, *function, true))
+      continue;
 
     auto* parallel_map =
         graph.AddNode(MakeParallelMap(map_node->name(), &graph));
diff --git a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
index 2dbd6e4698..db64cfff02 100644
--- a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
@@ -131,17 +131,6 @@ bool IsOutputShapesFullyDefined(const NodeDef& node) {
   return true;
 }
 
-bool IsStatefulFn(const FunctionLibraryDefinition& library,
-                  const FunctionDef& function_def) {
-  for (const NodeDef& node_def : function_def.node_def()) {
-    const OpDef* op_def;
-    Status s = library.LookUpOpDef(node_def.op(), &op_def);
-    if (!s.ok() || op_def->is_stateful()) {
-      return true;
-    }
-  }
-  return false;
-}
 
 NodeDef MakeNewBatchNode(const NodeDef& old_batch_node,
                          const NodeDef& input_node,
@@ -250,7 +239,7 @@ Status MapVectorization::OptimizeAndCollectStats(Cluster* cluster,
     // Check that this is a valid optimization.
     if (!IsOutputShapesFullyDefined(*input_node) ||
         !IsOutputShapesFullyDefined(*map_node) ||
-        IsStatefulFn(function_library, *orig_func)) {
+        function_utils::IsFunctionStateful(function_library, *orig_func)) {
       // 1. If any of the inputs have an unknown shape, don't optimize, since
       // inputs might not be batchable.
       // 2. If any of the map func outputs have an unknown shape, don't
-- 
GitLab


From 58d76bb520d0f10e7008a96789252de52dac18e7 Mon Sep 17 00:00:00 2001
From: Haoyu Zhang <haoyuzhang@google.com>
Date: Mon, 21 Jan 2019 22:26:53 -0800
Subject: [PATCH 1042/2345] Pass the `validation_freq` argument to enable
 custom validation frequency in distribution strategy cases.

PiperOrigin-RevId: 230284959
---
 tensorflow/python/keras/engine/training.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 366a96e7a6..41abd3c4ad 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -812,7 +812,8 @@ class Model(Network):
               sample_weight=sample_weight,
               initial_epoch=initial_epoch,
               steps_per_epoch=steps_per_epoch,
-              validation_steps=validation_steps)
+              validation_steps=validation_steps,
+              validation_freq=validation_freq)
 
         # Independent worker only for now.
         return dc.run_distribute_coordinator(
@@ -836,7 +837,8 @@ class Model(Network):
             sample_weight=sample_weight,
             initial_epoch=initial_epoch,
             steps_per_epoch=steps_per_epoch,
-            validation_steps=validation_steps)
+            validation_steps=validation_steps,
+            validation_freq=validation_freq)
 
     batch_size = self._validate_or_infer_batch_size(
         batch_size, steps_per_epoch, x)
-- 
GitLab


From 66da1dc0fdc29c371561bb381d5d93faba0f42da Mon Sep 17 00:00:00 2001
From: Gautam <gautamrbharadwaj@gmail.com>
Date: Tue, 22 Jan 2019 12:37:09 +0530
Subject: [PATCH 1043/2345] Update adam_gs_optimizer.py

---
 tensorflow/contrib/opt/python/training/adam_gs_optimizer.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tensorflow/contrib/opt/python/training/adam_gs_optimizer.py b/tensorflow/contrib/opt/python/training/adam_gs_optimizer.py
index c5c9fc74de..c30d7abbee 100644
--- a/tensorflow/contrib/opt/python/training/adam_gs_optimizer.py
+++ b/tensorflow/contrib/opt/python/training/adam_gs_optimizer.py
@@ -112,8 +112,6 @@ class AdamGSOptimizer(optimizer.Optimizer):
     self._beta2_t = None
     self._epsilon_t = None
 
-    # Created in SparseApply if needed.
-    self._updated_lr = None
 
   def _get_beta_accumulators(self):
     return (math_ops.pow(self._beta1_t, self._global_step_on_worker),
-- 
GitLab


From 3de2f6dbedcbb6560c4349badea203cee48a5b38 Mon Sep 17 00:00:00 2001
From: Chao Liu <yijinliu@gmail.com>
Date: Mon, 21 Jan 2019 23:52:50 -0800
Subject: [PATCH 1044/2345] Fix a memory leak in MklDnnData. This commit fixes
 leak caused by field user_memory_, which I can repro and verify. From reading
 the code, I think the same leak may happen to fields like reorder_memory_.
 Since I cannot repro or verify the later, I'll leave it the authors.

---
 tensorflow/core/util/mkl_util.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/core/util/mkl_util.h b/tensorflow/core/util/mkl_util.h
index 125a2e6b23..548f54883c 100644
--- a/tensorflow/core/util/mkl_util.h
+++ b/tensorflow/core/util/mkl_util.h
@@ -1766,6 +1766,7 @@ class MklDnnData {
   inline void SetUsrMem(const memory::primitive_desc& pd,
                         void* data_buffer = nullptr) {
     CHECK_NOTNULL(cpu_engine_);
+    if (user_memory_) delete user_memory_;
     // TODO(nhasabni): can we remove dynamic memory allocation?
     if (data_buffer) {
       user_memory_ = new memory(pd, data_buffer);
-- 
GitLab


From f44c86305911b1066d916d2c289a7a429b6f2a69 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 01:02:44 -0800
Subject: [PATCH 1045/2345] compat: Update forward compatibility horizon to
 2019-01-22

PiperOrigin-RevId: 230297751
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 0bd20a1f97..95940550b4 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 21)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 22)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 4b45dcc63dd043acbe6bb90475d356d2f3149b0c Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Tue, 22 Jan 2019 05:15:47 -0800
Subject: [PATCH 1046/2345] Add more debug information when deserializing.

PiperOrigin-RevId: 230322487
---
 .../saved_model/function_deserialization.py   | 21 ++++++++++++-------
 tensorflow/python/saved_model/load_test.py    |  2 +-
 2 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py
index 934e189f78..22d5ff4cf4 100644
--- a/tensorflow/python/saved_model/function_deserialization.py
+++ b/tensorflow/python/saved_model/function_deserialization.py
@@ -122,16 +122,20 @@ def recreate_function(saved_function, concrete_functions):
     """Calls a restored function."""
     # TODO(allenl): Functions saved with input_signatures should revive with
     # input_signatures.
+    try:
+      canonicalized_inputs = function_spec.canonicalize_function_inputs(
+          *args, **kwargs)
+    except ValueError as e:
+      raise ValueError(
+          "Cannot canonicalize input args %r and kwargs %r. Error: %r." %
+          (args, kwargs, e))
+
+    debug_considered_signatures = []
     for concrete_function in saved_function.concrete_function:
       function_obj = concrete_functions[concrete_function.name]
       canonicalized_original_inputs = coder.decode_proto(
           concrete_function.canonicalized_input_signature)
-
-      try:
-        canonicalized_inputs = function_spec.canonicalize_function_inputs(
-            *args, **kwargs)
-      except ValueError:
-        continue
+      debug_considered_signatures.append(canonicalized_original_inputs)
 
       if _inputs_compatible(canonicalized_inputs,
                             canonicalized_original_inputs):
@@ -140,8 +144,9 @@ def recreate_function(saved_function, concrete_functions):
         return function_obj._call_flat(filtered_inputs)  # pylint: disable=protected-access
 
     raise AssertionError(
-        "Could not find matching function to call for args %r and kwargs %r" %
-        (args, kwargs))
+        "Could not find matching function to call for canonicalized inputs %r. "
+        "Only existing signatures are %r."
+        % (canonicalized_inputs, debug_considered_signatures))
 
   return restored_function
 
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index 1c999a625d..b1dc0bd128 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -469,7 +469,7 @@ class LoadTest(test.TestCase):
 
     imported = self.cycle(root)
 
-    with self.assertRaises(AssertionError):
+    with self.assertRaisesRegexp(ValueError, "Cannot canonicalize"):
       # We cannot call the function with a constant of shape ().
       self.assertEqual(7, imported.f(constant_op.constant(2)).numpy())
 
-- 
GitLab


From 39656b348b83b5fe4d723e9345b499c2f181062e Mon Sep 17 00:00:00 2001
From: Felix Lemke <flemke.dev@gmail.com>
Date: Tue, 22 Jan 2019 16:02:43 +0100
Subject: [PATCH 1047/2345] docs(gan): fix tutorial link for ipythonnotebook

Closes #25101
---
 tensorflow/contrib/gan/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/gan/README.md b/tensorflow/contrib/gan/README.md
index db7dc51daa..068ccb3f8b 100644
--- a/tensorflow/contrib/gan/README.md
+++ b/tensorflow/contrib/gan/README.md
@@ -57,7 +57,7 @@ These include the following main pieces (explained in detail below).
     generative models.
 
 *   [examples](https://github.com/tensorflow/models/tree/master/research/gan/)
-    and [tutorial](http://https://github.com/tensorflow/models/tree/master/research/gan/tutorial.ipynb): See examples of how to use TF-GAN
+    and [tutorial](https://github.com/tensorflow/models/tree/master/research/gan/tutorial.ipynb): See examples of how to use TF-GAN
     to make GAN training easier, or use the more complicated examples to
     jumpstart your own project. These include unconditional and conditional
     GANs, InfoGANs, adversarial losses on existing networks, and image-to-image
-- 
GitLab


From c38ed2f020920176fbb412e4419ffe4d68885853 Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Tue, 22 Jan 2019 08:57:02 -0800
Subject: [PATCH 1048/2345] Add back computed bias scales to the quantization
 params.

PiperOrigin-RevId: 230348592
---
 .../lite/tools/optimize/subgraph_quantizer.cc | 12 +++-----
 .../tools/optimize/subgraph_quantizer_test.cc | 30 +++++++++++--------
 2 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/tensorflow/lite/tools/optimize/subgraph_quantizer.cc b/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
index 3bab365c82..99cc8584c6 100644
--- a/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
+++ b/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
@@ -186,14 +186,10 @@ TfLiteStatus SymmetricPerChannelBiasQuantize(const TensorT* input_tensor,
   // Set the buffers and output type.
   uint8_t* uint8_buffer = reinterpret_cast<uint8_t*>(final_buffer.data());
   size_t buffer_size = num_elements * sizeof(int32_t);
-  // For Bias we only set the quantized values, the scale and quantized
-  // dimension is implicit.
-  tensor->quantization = nullptr;
-  model->buffers[tensor->buffer]->data.assign(uint8_buffer,
-                                              uint8_buffer + buffer_size);
-
-  tensor->type = TensorType_INT32;
-  return kTfLiteOk;
+  std::vector<int64_t> zero_point(scales.size(), 0);
+  return AddQuantizationParams(scales, zero_point, channel_dim_index,
+                               uint8_buffer, buffer_size, TensorType_INT32,
+                               model, tensor);
 }
 }  // namespace
 
diff --git a/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc b/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
index 1ea49da899..580ed5eb26 100644
--- a/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
+++ b/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
@@ -90,16 +90,16 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantizationWithUnitScale) {
 
   ASSERT_TRUE(weights_tensor->quantization);
   const int out_channel_size = weights_tensor->shape[0];
-
-  // Bias tensor doesn't contain quantization info.
-  ASSERT_FALSE(bias_tensor->quantization);
-
+  ASSERT_TRUE(bias_tensor->quantization);
+  ASSERT_TRUE(weights_tensor->quantization);
+  const std::vector<float>& bias_scales = bias_tensor->quantization->scale;
   const std::vector<float>& weights_scales =
       weights_tensor->quantization->scale;
 
   const std::vector<int64_t>& weights_zero_points =
       weights_tensor->quantization->zero_point;
 
+  ASSERT_EQ(bias_scales.size(), out_channel_size);
   ASSERT_EQ(weights_scales.size(), out_channel_size);
   ASSERT_EQ(weights_zero_points.size(), out_channel_size);
   ASSERT_EQ(input_tensor->quantization->scale.size(), 1);
@@ -108,6 +108,7 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantizationWithUnitScale) {
 
   for (size_t i = 0; i < out_channel_size; i++) {
     EXPECT_EQ(weights_scales[i], 1);
+    EXPECT_EQ(bias_scales[i], 1);
     EXPECT_EQ(weights_zero_points[i], 0);
   }
 
@@ -188,21 +189,28 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantization) {
 
   ASSERT_TRUE(weights_tensor->quantization);
   const int out_channel_size = weights_tensor->shape[0];
-
-  // Bias tensor doesn't contain quantization info.
-  ASSERT_FALSE(bias_tensor->quantization);
-
+  ASSERT_TRUE(bias_tensor->quantization);
   ASSERT_TRUE(weights_tensor->quantization);
+  const std::vector<float>& bias_scales = bias_tensor->quantization->scale;
   const std::vector<float>& weights_scales =
       weights_tensor->quantization->scale;
   const std::vector<int64_t>& weights_zero_points =
       weights_tensor->quantization->zero_point;
 
+  ASSERT_EQ(bias_scales.size(), out_channel_size);
   ASSERT_EQ(weights_scales.size(), out_channel_size);
   ASSERT_EQ(weights_zero_points.size(), out_channel_size);
   ASSERT_EQ(input_tensor->quantization->scale.size(), 1);
   ASSERT_EQ(output_tensor->quantization->scale.size(), 1);
 
+  const float eps = 1e-7;
+
+  // Bias scale should be input * per_channel_weight_scale.
+  for (size_t i = 0; i < out_channel_size; i++) {
+    EXPECT_NEAR(bias_scales[i],
+                input_tensor->quantization->scale[0] * weights_scales[i], eps);
+  }
+
   const auto bias_buffer = model.buffers[bias_tensor->buffer].get();
   ASSERT_EQ(bias_buffer->data.size(), sizeof(int32_t) * bias_tensor->shape[0]);
   const int32_t* bias_values =
@@ -213,10 +221,8 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantization) {
       reinterpret_cast<const float*>(original_bias_buffer->data()->data());
 
   for (size_t i = 0; i < out_channel_size; i++) {
-    const float bias_scale =
-        input_tensor->quantization->scale[0] * weights_scales[i];
-    auto dequantized_value = bias_values[i] * bias_scale;
-    EXPECT_NEAR(dequantized_value, bias_float_buffer[i], bias_scale / 2);
+    auto dequantized_value = bias_values[i] * bias_scales[i];
+    EXPECT_NEAR(dequantized_value, bias_float_buffer[i], bias_scales[i] / 2);
   }
 
   const auto weights_buffer = model.buffers[weights_tensor->buffer].get();
-- 
GitLab


From 05677b7c9127967c685af444a6f5ac7291fb3b86 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 09:00:42 -0800
Subject: [PATCH 1049/2345] Delete deprecated and no-longer-used
 tf.Strategy.distribute_dataset().

PiperOrigin-RevId: 230349129
---
 tensorflow/contrib/distribute/python/BUILD    |   2 -
 .../python/collective_all_reduce_strategy.py  |  11 +-
 .../distribute/python/input_lib_test.py       | 234 ------------------
 .../distribute/python/mirrored_strategy.py    |  12 -
 .../distribute/python/one_device_strategy.py  |   4 -
 .../contrib/distribute/python/tpu_strategy.py |   9 +-
 tensorflow/python/distribute/BUILD            |   3 -
 .../python/distribute/distribute_lib.py       |  51 +---
 tensorflow/python/distribute/input_lib.py     | 234 ------------------
 .../python/distribute/mirrored_strategy.py    |  15 +-
 .../distribute/parameter_server_strategy.py   |  11 +-
 ...orflow.distribute.-mirrored-strategy.pbtxt |   4 -
 .../v1/tensorflow.distribute.-strategy.pbtxt  |   4 -
 ...orflow.distribute.-mirrored-strategy.pbtxt |   4 -
 .../v2/tensorflow.distribute.-strategy.pbtxt  |   4 -
 15 files changed, 5 insertions(+), 597 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 7a77dbe29c..fb8732d591 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -51,9 +51,7 @@ cuda_py_test(
         ":mirrored_strategy",
         ":multi_worker_test_base",
         "@absl_py//absl/testing:parameterized",
-        "//tensorflow/core:protos_all_py",
         "//tensorflow/python:errors",
-        "//tensorflow/python:framework_test_lib",
         "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python/distribute:input_lib",
         "//tensorflow/python/distribute:values",
diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
index eee0754325..aa4d82b4d0 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
@@ -251,14 +251,6 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
         self._container_strategy(), device_map, logical_device,
         _real_mirrored_creator, *args, **kwargs)
 
-  def _distribute_dataset(self, dataset_fn):
-    """Distributes the dataset to each local GPU."""
-    # TODO(yuefengz): shard the dataset.
-    worker_index = 0
-    return input_lib.PerReplicaDataset(
-        self._call_dataset_fn(dataset_fn), self._input_workers, worker_index,
-        prefetch_on_device=True)
-
   def _make_dataset_iterator(self, dataset):
     return input_lib.DatasetIterator(dataset, self._input_workers,
                                      self._num_replicas_in_sync)
@@ -374,8 +366,7 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
   def _global_batch_size(self):
     """`make_dataset_iterator` and `make_numpy_iterator` use global batch size.
 
-    `distribute_dataset` and `make_input_fn_iterator` assume per-replica
-    batching.
+    `make_input_fn_iterator` assumes per-replica batching.
 
     Returns:
       Boolean.
diff --git a/tensorflow/contrib/distribute/python/input_lib_test.py b/tensorflow/contrib/distribute/python/input_lib_test.py
index f589cd6ad5..10a58316ec 100644
--- a/tensorflow/contrib/distribute/python/input_lib_test.py
+++ b/tensorflow/contrib/distribute/python/input_lib_test.py
@@ -22,7 +22,6 @@ from absl.testing import parameterized
 
 from tensorflow.contrib.distribute.python import combinations
 from tensorflow.contrib.distribute.python import multi_worker_test_base
-from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.data.experimental.ops import batching
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import distribute_lib
@@ -31,243 +30,10 @@ from tensorflow.python.distribute import values
 from tensorflow.python.eager import context
 from tensorflow.python.eager import test
 from tensorflow.python.framework import errors
-from tensorflow.python.framework import test_util
 from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.ops import random_ops
 from tensorflow.python.util import nest
 
 
-class PerReplicaDatasetTest(test.TestCase):
-
-  config = config_pb2.ConfigProto()
-  config.allow_soft_placement = True
-
-  def _test_iterator(self, devices, dataset, expected_values):
-    device_map = values.ReplicaDeviceMap(devices)
-    input_workers = input_lib.InputWorkers(device_map)
-    per_replica_dataset = input_lib.PerReplicaDataset(dataset, input_workers, 0)
-    if context.executing_eagerly():
-      iterator = per_replica_dataset.make_one_shot_iterator()
-    else:
-      iterator = per_replica_dataset.make_initializable_iterator()
-      self.evaluate([iterator.initializer])
-
-    for expected_value in expected_values:
-      next_element = iterator.get_next_as_list()
-      computed_value = self.evaluate(next_element)
-      self.assertEqual(expected_value, computed_value)
-
-    with self.assertRaises(errors.OutOfRangeError):
-      next_element = iterator.get_next_as_list()
-      self.evaluate(next_element)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testOneDevice(self):
-    devices = ["/device:CPU:0"]
-    dataset = dataset_ops.Dataset.range(10)
-
-    expected_values = [[i] for i in range(10)]
-
-    self._test_iterator(devices, dataset, expected_values)
-
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testMultipleDevices(self):
-    if context.num_gpus() < 1 and context.executing_eagerly():
-      self.skipTest("A GPU is not available for this test in eager mode.")
-
-    devices = ["/device:CPU:0", "/device:GPU:0"]
-    dataset = dataset_ops.Dataset.range(10)
-
-    expected_values = [[i, i+1] for i in range(0, 10, 2)]
-
-    self._test_iterator(devices, dataset, expected_values)
-
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testTupleDataset(self):
-    if context.num_gpus() < 1 and context.executing_eagerly():
-      self.skipTest("A GPU is not available for this test in eager mode.")
-
-    devices = ["/device:CPU:0", "/device:GPU:0"]
-    dataset1 = dataset_ops.Dataset.range(10)
-    dataset2 = dataset_ops.Dataset.range(10).map(lambda x: x**2)
-    dataset = dataset_ops.Dataset.zip((dataset1, dataset2))
-
-    expected_values = [[(i, i**2), (i+1, (i+1)**2)] for i in range(0, 10, 2)]
-
-    self._test_iterator(devices, dataset, expected_values)
-
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testUnevenDatasetBatches(self):
-    if context.num_gpus() < 1 and context.executing_eagerly():
-      self.skipTest("A GPU is not available for this test in eager mode.")
-
-    devices = ["/device:CPU:0", "/device:GPU:0"]
-    dataset = dataset_ops.Dataset.range(11)
-
-    expected_values = [[i, i+1] for i in range(0, 10, 2)]
-    self._test_iterator(devices, dataset, expected_values)
-
-  def testInitializableIterator(self):
-    with context.graph_mode():
-      devices = ["/device:CPU:0"]
-      # Using random input since that is only allowed with initializable
-      # iterator.
-      dataset = dataset_ops.Dataset.from_tensor_slices(
-          random_ops.random_uniform((10,)))
-
-      device_map = values.ReplicaDeviceMap(devices)
-      input_workers = input_lib.InputWorkers(device_map)
-      per_replica_dataset = input_lib.PerReplicaDataset(
-          dataset, input_workers, 0)
-      iterator = per_replica_dataset.make_initializable_iterator()
-
-      self.evaluate(iterator.initializer)
-      next_element = iterator.get_next_as_list()
-      for _ in range(10):
-        self.evaluate(next_element)
-
-      # Should fail after the input is finished.
-      with self.assertRaises(errors.OutOfRangeError):
-        self.evaluate(next_element)
-
-      # After re-initializing the iterator, should be able to iterate again.
-      self.evaluate(iterator.initializer)
-      for _ in range(10):
-        self.evaluate(next_element)
-
-
-class MultiWorkerDatasetTest(multi_worker_test_base.MultiWorkerTestBase):
-
-  def _test_iterator(self, sess, iterator, devices, expected_values):
-    next_element = iterator.get_next()
-    for r, device in enumerate(devices):
-      v = values.select_replica(r, next_element)
-      # The `v` here can be a tuple.
-      for element in nest.flatten(v):
-        self.assertTrue(element.device in device)
-
-    for expected_value in expected_values:
-      t = [values.select_replica(r, next_element) for r in range(len(devices))]
-      actual = sess.run(t)
-      self.assertEqual(expected_value, actual)
-
-    with self.assertRaises(errors.OutOfRangeError):
-      sess.run([values.select_replica(r, next_element)
-                for r in range(len(devices))])
-
-  def _test_dataset(self, dataset_fn, worker_devices, devices,
-                    expected_values):
-    device_map = values.ReplicaDeviceMap(devices)
-    input_workers = input_lib.InputWorkers(device_map, worker_devices)
-    multi_worker_dataset = input_lib.MultiWorkerDataset(
-        dataset_fn, input_workers)
-    multi_worker_iterator = multi_worker_dataset.make_initializable_iterator()
-    with self.cached_session() as sess:
-      sess.run(multi_worker_iterator.initializer)
-      self._test_iterator(sess, multi_worker_iterator, devices, expected_values)
-
-  def _cpu_devices(self):
-    worker_devices = (
-        ("/job:worker/replica:0/task:0",
-         ["/job:worker/replica:0/task:0/device:CPU:0"]),
-        ("/job:worker/replica:0/task:1",
-         ["/job:worker/replica:0/task:1/device:CPU:0"])
-    )
-    devices = [
-        "/job:worker/replica:0/task:0/device:CPU:0",
-        "/job:worker/replica:0/task:1/device:CPU:0"
-    ]
-    return worker_devices, devices
-
-  def _cpu_and_one_gpu_devices(self):
-    worker_devices = (
-        ("/job:worker/replica:0/task:0", (
-            "/job:worker/replica:0/task:0/device:GPU:0",
-            "/job:worker/replica:0/task:0/device:CPU:0"
-        )),
-        ("/job:worker/replica:0/task:1", (
-            "/job:worker/replica:0/task:1/device:GPU:0",
-            "/job:worker/replica:0/task:1/device:CPU:0"
-        ))
-    )
-    devices = [
-        "/job:worker/replica:0/task:0/device:GPU:0",
-        "/job:worker/replica:0/task:0/device:CPU:0",
-        "/job:worker/replica:0/task:1/device:GPU:0",
-        "/job:worker/replica:0/task:1/device:CPU:0"
-    ]
-    return worker_devices, devices
-
-  def testDataDistributionOneDevicePerWorker(self):
-    worker_devices, devices = self._cpu_devices()
-    with context.graph_mode():
-      dataset_fn = lambda: dataset_ops.Dataset.range(8)
-      self._test_dataset(
-          dataset_fn, worker_devices, devices,
-          [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5], [6, 6], [7, 7]])
-
-  def testDataDistributionTwoDevicePerWorker(self):
-    if context.num_gpus() < 1:
-      self.skipTest("A GPU is not available for this test.")
-    worker_devices, devices = self._cpu_and_one_gpu_devices()
-    with context.graph_mode():
-      dataset_fn = lambda: dataset_ops.Dataset.range(8)
-      self._test_dataset(
-          dataset_fn, worker_devices, devices,
-          [[0, 1, 0, 1], [2, 3, 2, 3], [4, 5, 4, 5], [6, 7, 6, 7]])
-
-  def testTupleDataset(self):
-    worker_devices, devices = self._cpu_devices()
-
-    with context.graph_mode():
-
-      def dataset_fn():
-        dataset1 = dataset_ops.Dataset.range(8)
-        dataset2 = dataset_ops.Dataset.range(8).map(lambda x: x**2)
-        return dataset_ops.Dataset.zip((dataset1, dataset2))
-
-      expected_values = [[(i, i**2), (i, i**2)] for i in range(8)]
-      self._test_dataset(dataset_fn, worker_devices, devices,
-                         expected_values)
-
-  def testInitializableIterator(self):
-    worker_devices, devices = self._cpu_devices()
-    with context.graph_mode(), self.cached_session() as sess:
-      dataset_fn = lambda: dataset_ops.Dataset.range(8)
-      device_map = values.ReplicaDeviceMap(devices)
-      input_workers = input_lib.InputWorkers(device_map, worker_devices)
-      multi_worker_dataset = input_lib.MultiWorkerDataset(
-          dataset_fn, input_workers)
-      multi_worker_iterator = multi_worker_dataset.make_initializable_iterator()
-
-      sess.run(multi_worker_iterator.initializer)
-      self._test_iterator(
-          sess, multi_worker_iterator, devices,
-          [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5], [6, 6], [7, 7]])
-
-      # After re-initializing the iterator, should be able to iterate again.
-      sess.run(multi_worker_iterator.initializer)
-      self._test_iterator(
-          sess, multi_worker_iterator, devices,
-          [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5], [6, 6], [7, 7]])
-
-  def testValueErrorForIterator(self):
-    # Incompatiable arguments.
-    d1 = "/device:GPU:0"
-    d2 = "/device:GPU:1"
-    device_map = values.ReplicaDeviceMap([d1, d2])
-    input_workers = input_lib.InputWorkers(
-        device_map, (("w1", (d1,)), ("w2", (d2,))))
-    with self.assertRaises(ValueError):
-      input_lib.MultiWorkerDataIterator([("w1", None)], input_workers)
-
-  def testDuplicateDevices(self):
-    _, devices = self._cpu_devices()
-    devices.append("/job:worker/replica:0/task:0/device:CPU:0")
-    with self.assertRaises(ValueError):
-      _ = values.ReplicaDeviceMap(devices)
-
-
 class InputIteratorTestBase(test.TestCase):
 
   def _test_iterator(self, input_type, dataset_fn, worker_device_pairs,
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index 2e23a51ee5..5391e083fc 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -18,8 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import functools
-
 from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import input_lib
 from tensorflow.python.distribute import mirrored_strategy
@@ -192,16 +190,6 @@ class MirroredExtended(CoreMirroredExtended):
     """
     return input_lib.DatasetIterator(dataset, self._input_workers)
 
-  def _distribute_dataset(self, dataset_fn):
-    if self._local_mode:
-      return input_lib.PerReplicaDataset(
-          self._call_dataset_fn(dataset_fn), self._input_workers, 0)
-    else:
-      return input_lib.MultiWorkerDataset(
-          functools.partial(self._call_dataset_fn, dataset_fn),
-          self._input_workers,
-          auto_shard=self._auto_shard_dataset)
-
   # TODO(priyag): Delete this once all strategies use global batch size.
   @property
   def _global_batch_size(self):
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py
index 836cb7cc41..24d6a443fe 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy.py
@@ -72,10 +72,6 @@ class OneDeviceExtended(distribute_lib.DistributionStrategyExtended):
     """Make iterator from dataset without splitting the batch."""
     return input_lib.DatasetIterator(dataset, self._input_workers)
 
-  def _distribute_dataset(self, dataset_fn):
-    return input_lib.PerReplicaDataset(
-        self._call_dataset_fn(dataset_fn), self._input_workers, 0)
-
   def _make_input_fn_iterator(
       self,
       input_fn,
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index f55e6f0b20..5a494a6ae7 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -22,7 +22,6 @@ from __future__ import division
 from __future__ import print_function
 
 import copy
-import functools
 
 from tensorflow.contrib.tpu.python.ops import tpu_ops
 from tensorflow.contrib.tpu.python.tpu import device_assignment as device_assignment_lib
@@ -323,11 +322,6 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
     return input_lib.DatasetIterator(dataset, self._input_workers,
                                      self._num_replicas_in_sync)
 
-  def _distribute_dataset(self, dataset_fn):
-    return input_lib.MultiWorkerDataset(
-        functools.partial(self._call_dataset_fn, dataset_fn),
-        self._input_workers)
-
   def _make_input_fn_iterator(
       self,
       input_fn,
@@ -726,8 +720,7 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
   def _global_batch_size(self):
     """`make_dataset_iterator` and `make_numpy_iterator` use global batch size.
 
-    `distribute_dataset` and `make_input_fn_iterator` assume per-replica
-    batching.
+    `make_input_fn_iterator` assumes per-replica batching.
 
     Returns:
       Boolean.
diff --git a/tensorflow/python/distribute/BUILD b/tensorflow/python/distribute/BUILD
index a6a1c470b4..24a2a84b7f 100644
--- a/tensorflow/python/distribute/BUILD
+++ b/tensorflow/python/distribute/BUILD
@@ -316,10 +316,7 @@ py_library(
         ":distribute_lib",
         ":input_ops",
         ":values",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:framework_ops",
-        "//tensorflow/python:util",
         "//tensorflow/python/data/ops:multi_device_iterator_ops",
         "//tensorflow/python/eager:context",
     ],
diff --git a/tensorflow/python/distribute/distribute_lib.py b/tensorflow/python/distribute/distribute_lib.py
index e7d52c6c98..2cc99b3db4 100644
--- a/tensorflow/python/distribute/distribute_lib.py
+++ b/tensorflow/python/distribute/distribute_lib.py
@@ -333,36 +333,6 @@ class DistributionStrategy(object):
     """DEPRECATED: use extended.colocate_vars_with() instead."""
     return self._extended.colocate_vars_with(colocate_with_variable)
 
-  @doc_controls.do_not_generate_docs  # DEPRECATED
-  def distribute_dataset(self, dataset_fn):
-    """Return a `dataset` split across all replicas.  DEPRECATED.
-
-    DEPRECATED: Please use `make_dataset_iterator` or
-    `make_input_fn_iterator` instead.
-
-    Suitable for providing input to `extended.call_for_each_replica()` by
-    creating an iterator:
-
-    ```
-    def dataset_fn():
-      return tf.data.Dataset.from_tensors([[1.]]).repeat()
-
-    with strategy.scope():
-      distributed_dataset = strategy.distribute_dataset(dataset_fn)
-      iterator = distributed_dataset.make_initializable_iterator()
-      replica_results = strategy.extended.call_for_each_replica(
-          replica_fn, args=(iterator.get_next(),))
-    ```
-
-    Args:
-      dataset_fn: A function that returns a `tf.data.Dataset` with per-replica
-        batching.
-
-    Returns:
-      A `PerReplicaDataset` that will produce data for each replica.
-    """
-    return self._extended._distribute_dataset(dataset_fn)  # pylint: disable=protected-access
-
   def make_dataset_iterator(self, dataset):
     """Makes an iterator for input provided via `dataset`.
 
@@ -756,8 +726,7 @@ class DistributionStrategyExtended(object):
     a variable (which by definition will have locality V(`v`), though
     will match another locality if inside a `colocate_vars_with`
     scope).
-  * `d.make_dataset_iterator(dataset)` (or the deprecated
-    `d.distribute_dataset(dataset).make_one_shot_iterator()`): in cross-replica
+  * `d.make_dataset_iterator(dataset)`: in cross-replica
     context, produces an iterator with locality T
   * `d.broadcast(t)`: in cross-replica context, produces a value with locality M
   * `d.extended.broadcast_to(t, v)`: in cross-replica context, produces a value
@@ -965,21 +934,6 @@ class DistributionStrategyExtended(object):
     """Validate `colocate_with_variable` argument to `colocate_vars_with`."""
     pass
 
-  def _call_dataset_fn(self, dataset_fn):
-    """Call the `dataset_fn` with `input_context` as argument."""
-    result = dataset_fn()
-    if not isinstance(result, dataset_ops.DatasetV2):
-      raise ValueError(
-          "dataset_fn() must return a tf.data.Dataset when using a "
-          "tf.distribute.Strategy.")
-    return result
-
-  # TODO(josh11b): `PerReplicaDataset` currently only implements a few methods of
-  # Dataset API such as make_one_shot_iterator and make_initializable_iterator.
-  # Extend to implement more functionality of datasets.
-  def _distribute_dataset(self, dataset_fn):
-    raise NotImplementedError("must be implemented in descendants")
-
   def _make_dataset_iterator(self, dataset):
     raise NotImplementedError("must be implemented in descendants")
 
@@ -1549,9 +1503,6 @@ class _DefaultDistributionExtended(DistributionStrategyExtended):
   def variable_created_in_scope(self, v):
     return v._distribute_strategy is None  # pylint: disable=protected-access
 
-  def _distribute_dataset(self, dataset_fn):
-    return self._call_dataset_fn(dataset_fn)
-
   def _make_dataset_iterator(self, dataset):
     return _DefaultDistributionExtended.DefaultInputIterator(dataset)
 
diff --git a/tensorflow/python/distribute/input_lib.py b/tensorflow/python/distribute/input_lib.py
index 3ba80c9287..43915917d6 100644
--- a/tensorflow/python/distribute/input_lib.py
+++ b/tensorflow/python/distribute/input_lib.py
@@ -29,9 +29,6 @@ from tensorflow.python.eager import context
 from tensorflow.python.framework import device as tf_device
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.util import nest
 
 
 class InputWorkers(object):
@@ -91,237 +88,6 @@ class InputWorkers(object):
         self.__class__.__name__, debug_repr, self._device_map)
 
 
-class PerReplicaDataIterator(object):
-  """An iterator (like `tf.data.Iterator`) into a `PerReplicaDataset`."""
-
-  def __init__(self, iterator, input_workers, worker_index, prefetch_on_device):
-    assert isinstance(input_workers, InputWorkers)
-    self._iterator = iterator
-    self._input_workers = input_workers
-    self._worker_index = worker_index
-    self._prefetch_on_device = prefetch_on_device
-
-  @property
-  def initializer(self):
-    return self._iterator.initializer
-
-  def get_next_as_list(self, name=None):
-    """Scatter the input across devices."""
-    if self._prefetch_on_device:
-      data_list = self._iterator.get_next()
-    else:
-      batch = self._iterator.get_next(name=name)
-      data_list = []
-      def get_ith(i):
-        return lambda x: x[i]
-
-      devices = self._input_workers.compute_devices_for_worker(
-          self._worker_index)
-      for i, d in enumerate(devices):
-        v = nest.map_structure(get_ith(i), batch)
-        if context.executing_eagerly():
-          with ops.device(d):
-            v = nest.map_structure(array_ops.identity, v)
-        data_list.append(v)
-
-    return data_list
-
-  def get_next(self, name=None):
-    assert self._input_workers.num_workers == 1
-    data_list = self.get_next_as_list(name)
-    return values.regroup(self._input_workers.device_map, data_list)
-
-  @property
-  def output_classes(self):
-    return self._iterator.output_classes
-
-  @property
-  def output_shapes(self):
-    return self._iterator.output_shapes
-
-  @property
-  def output_types(self):
-    return self._iterator.output_types
-
-
-class PerReplicaDataset(object):
-  """Like `tf.data.Dataset` split devices, producing `PerReplica` data."""
-
-  def __init__(self, dataset, input_workers, worker_index,
-               prefetch_on_device=None):
-    assert isinstance(input_workers, InputWorkers)
-    assert worker_index is not None
-    assert worker_index is not True  # pylint: disable=g-bool-id-comparison
-    assert worker_index is not False  # pylint: disable=g-bool-id-comparison
-    self._input_workers = input_workers
-    self._worker_index = worker_index
-
-    # Default to using prefetching, unless specified.
-    self._prefetch_on_device = prefetch_on_device
-    if self._prefetch_on_device is None:
-      self._prefetch_on_device = True
-
-    self._dataset = dataset
-    if not self._prefetch_on_device:
-      # TODO(priyag): If dropping remainder is not appropriate, find another
-      # approach to distributing the dataset when not possible to divide evenly.
-      # Possibly not an issue when we start using PartitionedDataset.
-      num_replicas = len(
-          self._input_workers.compute_devices_for_worker(self._worker_index))
-      self._dataset = self._dataset.batch(num_replicas, drop_remainder=True)
-    else:
-      self._replica_devices = self._input_workers.compute_devices_for_worker(
-          self._worker_index)
-
-  def make_one_shot_iterator(self):
-    """Get a one time use iterator for the distributed PerReplicaDataset."""
-    # Graph mode with one shot iterator is disabled.
-    if not context.executing_eagerly():
-      raise ValueError("Cannot create a one shot iterator. Please use "
-                       "`make_initializable_iterator()` instead.")
-    if self._prefetch_on_device:
-      dataset_iterator = multi_device_iterator_ops.MultiDeviceIterator(
-          self._dataset, self._replica_devices)
-    else:
-      dataset_iterator = dataset_ops.make_one_shot_iterator(self._dataset)
-    return PerReplicaDataIterator(
-        dataset_iterator,
-        self._input_workers,
-        self._worker_index,
-        prefetch_on_device=self._prefetch_on_device)
-
-  def make_initializable_iterator(self):
-    """Get an initializable iterator for the distributed PerReplicaDataset."""
-    # Eager mode generates already initialized iterators. Hence we cannot create
-    # an initializable iterator.
-    if context.executing_eagerly():
-      raise ValueError("Cannot create initializable iterator in Eager mode. "
-                       "Please use `make_one_shot_iterator` instead.")
-    if self._prefetch_on_device:
-      dataset_iterator = multi_device_iterator_ops.MultiDeviceIterator(
-          self._dataset, self._replica_devices)
-    else:
-      dataset_iterator = dataset_ops.make_initializable_iterator(self._dataset)
-    return PerReplicaDataIterator(
-        dataset_iterator, self._input_workers, self._worker_index,
-        prefetch_on_device=self._prefetch_on_device)
-
-
-class MultiWorkerDataIterator(object):
-  """An iterator (like `tf.data.Iterator`) into a `MultiWorkerDataset`."""
-
-  def __init__(self, iterators, input_workers):
-    """Initialize the `MultiWorkerDataIterator` object.
-
-    Args:
-      iterators: a list of worker, iterator pairs.
-      input_workers: an `InputWorkers` object.
-
-    Raises:
-      ValueError: if iterators and input_workers are not compatible.
-    """
-    assert isinstance(input_workers, InputWorkers)
-    workers = tuple(d for d, _ in iterators)
-    if workers != input_workers.worker_devices:
-      raise ValueError("iterators and input_workers are not compatible. "
-                       "iterator workers: %r input_workers devices: %r" %
-                       (workers, input_workers.worker_devices))
-    self._iterators = tuple(i for _, i in iterators)
-    self._input_workers = input_workers
-
-  @property
-  def initializer(self):
-    return control_flow_ops.group(
-        tuple(iterator.initializer for iterator in self._iterators))
-
-  def get_iterator(self, worker):
-    for i, w in enumerate(self._input_workers.worker_devices):
-      if worker == w:
-        return self._iterators[i]
-    return None
-
-  @property
-  def output_shapes(self):
-    return self._iterators[0].output_shapes
-
-  @property
-  def output_types(self):
-    return self._iterators[0].output_types
-
-  def get_next(self, name=None):
-    """Scatter the input across hosts and devices."""
-    replicas = []
-    for worker, iterator in zip(self._input_workers.worker_devices,
-                                self._iterators):
-      if name is not None:
-        d = tf_device.DeviceSpec.from_string(worker)
-        new_name = "%s_%s_%d" % (name, d.job, d.task)
-      else:
-        new_name = None
-      with ops.device(worker):
-        data_per_worker = iterator.get_next_as_list(name=new_name)
-        # Append to replicas to get a flat list of values indexed by replica.
-        replicas.extend(data_per_worker)
-
-    return values.regroup(self._input_workers.device_map, replicas)
-
-
-class MultiWorkerDataset(object):
-  """Like a `tf.data.Dataset` that distributes data to different workers.
-
-  Each worker gets one shard of the input dataset. This currently does not work
-  in eager mode.
-  """
-
-  def __init__(self, dataset_fn, input_workers, prefetch_on_device=None,
-               auto_shard=False):
-    """Initialize the MultiWorkerDataset object.
-
-    Args:
-      dataset_fn: a function or a list of functions that returns a
-        `tf.data.Dataset`.
-      input_workers: an `InputWorkers` object.
-      prefetch_on_device: whether to prefetch to devices.
-      auto_shard: whether to auto-shard the dataset.
-    """
-    assert isinstance(input_workers, InputWorkers)
-    if isinstance(dataset_fn, (list, tuple)):
-      if len(dataset_fn) != input_workers.num_workers:
-        raise ValueError("If `dataset_fn` is a list, it must have one entry "
-                         "per worker")
-    # TODO(rohanj): b/120673685 to track re-enabling auto sharding.
-    if auto_shard:
-      raise ValueError("Currently autosharding is not supported.")
-    self._input_workers = input_workers
-    self._datasets = []
-    # TODO(yuefengz, priyag): support different set of jobs for input
-    # processing.
-    for i, worker in enumerate(input_workers.worker_devices):
-      with ops.device(worker):
-        if isinstance(dataset_fn, (list, tuple)):
-          worker_input = dataset_fn[i]()
-        else:
-          worker_input = dataset_fn()
-        dataset = PerReplicaDataset(worker_input, input_workers, i,
-                                    prefetch_on_device=prefetch_on_device)
-        self._datasets.append((worker, dataset))
-
-  def make_one_shot_iterator(self):
-    iterators = []
-    for worker, dataset in self._datasets:
-      with ops.device(worker):
-        iterators.append((worker, dataset_ops.make_one_shot_iterator(dataset)))
-    return MultiWorkerDataIterator(iterators, self._input_workers)
-
-  def make_initializable_iterator(self):
-    iterators = []
-    for worker, dataset in self._datasets:
-      with ops.device(worker):
-        iterators.append(
-            (worker, dataset_ops.make_initializable_iterator(dataset)))
-    return MultiWorkerDataIterator(iterators, self._input_workers)
-
-
 class InputIterator(object):
   """An input iterator, intended to be passed to `DistributionStrategy.run`."""
 
diff --git a/tensorflow/python/distribute/mirrored_strategy.py b/tensorflow/python/distribute/mirrored_strategy.py
index f279c3ba1b..c5314e386e 100644
--- a/tensorflow/python/distribute/mirrored_strategy.py
+++ b/tensorflow/python/distribute/mirrored_strategy.py
@@ -20,7 +20,6 @@ from __future__ import print_function
 
 import contextlib
 import copy
-import functools
 import threading
 
 from tensorflow.python import pywrap_tensorflow
@@ -548,17 +547,6 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
   def _validate_colocate_with_variable(self, colocate_with_variable):
     values.validate_colocate_distributed_variable(colocate_with_variable, self)
 
-  def _distribute_dataset(self, dataset_fn):
-    if self._local_mode:
-      worker_index = 0
-      return input_lib.PerReplicaDataset(
-          self._call_dataset_fn(dataset_fn), self._input_workers, worker_index)
-    else:
-      return input_lib.MultiWorkerDataset(
-          functools.partial(self._call_dataset_fn, dataset_fn),
-          self._input_workers,
-          auto_shard=False)
-
   def _make_dataset_iterator(self, dataset):
     return input_lib.DatasetIterator(
         dataset, self._input_workers, self._num_replicas_in_sync)
@@ -784,8 +772,7 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
   def _global_batch_size(self):
     """`make_dataset_iterator` and `make_numpy_iterator` use global batch size.
 
-    `distribute_dataset` and `make_input_fn_iterator` assume per-replica
-    batching.
+    `make_input_fn_iterator` assumes per-replica batching.
 
     Returns:
       Boolean.
diff --git a/tensorflow/python/distribute/parameter_server_strategy.py b/tensorflow/python/distribute/parameter_server_strategy.py
index 8093f8ccb0..6cc5865eed 100644
--- a/tensorflow/python/distribute/parameter_server_strategy.py
+++ b/tensorflow/python/distribute/parameter_server_strategy.py
@@ -233,14 +233,6 @@ class ParameterServerStrategyExtended(
   def _validate_colocate_with_variable(self, colocate_with_variable):
     values.validate_colocate(colocate_with_variable, self)
 
-  def _distribute_dataset(self, dataset_fn):
-    """Distributes the dataset to each local GPU."""
-    return input_lib.PerReplicaDataset(
-        self._call_dataset_fn(dataset_fn),
-        self._input_workers,
-        0,
-        prefetch_on_device=True)
-
   def _make_dataset_iterator(self, dataset):
     return input_lib.DatasetIterator(dataset, self._input_workers,
                                      self._num_replicas_in_sync)
@@ -538,8 +530,7 @@ class ParameterServerStrategyExtended(
   def _global_batch_size(self):
     """`make_dataset_iterator` and `make_numpy_iterator` use global batch size.
 
-    `distribute_dataset` and `make_input_fn_iterator` assume per-replica
-    batching.
+    `make_input_fn_iterator` assumes per-replica batching.
 
     Returns:
       Boolean.
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
index 4cb78b08f8..89748f7713 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
@@ -27,10 +27,6 @@ tf_class {
     name: "configure"
     argspec: "args=[\'self\', \'session_config\', \'cluster_spec\', \'task_type\', \'task_id\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
-  member_method {
-    name: "distribute_dataset"
-    argspec: "args=[\'self\', \'dataset_fn\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "experimental_make_numpy_iterator"
     argspec: "args=[\'self\', \'numpy_input\', \'batch_size\', \'num_epochs\', \'shuffle\', \'session\'], varargs=None, keywords=None, defaults=[\'1\', \'1024\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
index 11c1479b5b..6ed49d339d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
@@ -26,10 +26,6 @@ tf_class {
     name: "configure"
     argspec: "args=[\'self\', \'session_config\', \'cluster_spec\', \'task_type\', \'task_id\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
-  member_method {
-    name: "distribute_dataset"
-    argspec: "args=[\'self\', \'dataset_fn\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "experimental_make_numpy_iterator"
     argspec: "args=[\'self\', \'numpy_input\', \'batch_size\', \'num_epochs\', \'shuffle\', \'session\'], varargs=None, keywords=None, defaults=[\'1\', \'1024\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
index 4cb78b08f8..89748f7713 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
@@ -27,10 +27,6 @@ tf_class {
     name: "configure"
     argspec: "args=[\'self\', \'session_config\', \'cluster_spec\', \'task_type\', \'task_id\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
-  member_method {
-    name: "distribute_dataset"
-    argspec: "args=[\'self\', \'dataset_fn\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "experimental_make_numpy_iterator"
     argspec: "args=[\'self\', \'numpy_input\', \'batch_size\', \'num_epochs\', \'shuffle\', \'session\'], varargs=None, keywords=None, defaults=[\'1\', \'1024\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
index 11c1479b5b..6ed49d339d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
@@ -26,10 +26,6 @@ tf_class {
     name: "configure"
     argspec: "args=[\'self\', \'session_config\', \'cluster_spec\', \'task_type\', \'task_id\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
-  member_method {
-    name: "distribute_dataset"
-    argspec: "args=[\'self\', \'dataset_fn\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "experimental_make_numpy_iterator"
     argspec: "args=[\'self\', \'numpy_input\', \'batch_size\', \'num_epochs\', \'shuffle\', \'session\'], varargs=None, keywords=None, defaults=[\'1\', \'1024\', \'None\'], "
-- 
GitLab


From 0b65aa379fc46b40f963edcf58545d99b4cf2f63 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 09:03:17 -0800
Subject: [PATCH 1050/2345] Adding transformer.TransformerContext to 1. wrap
 the immutable EntityContext and 2. track the current OriginInfo, so that we
 can move error AutoGraphParseError handling to api.to_graph().

Removing the finally block in visit() in transformer.Base error handling.

PiperOrigin-RevId: 230349773
---
 .../pyct/static_analysis/live_values.py       | 11 ++---
 .../pyct/static_analysis/type_info.py         |  5 ++-
 .../python/autograph/pyct/transformer.py      | 43 +++++++++++++------
 3 files changed, 39 insertions(+), 20 deletions(-)

diff --git a/tensorflow/python/autograph/pyct/static_analysis/live_values.py b/tensorflow/python/autograph/pyct/static_analysis/live_values.py
index e8e3d229be..14504fbe1b 100644
--- a/tensorflow/python/autograph/pyct/static_analysis/live_values.py
+++ b/tensorflow/python/autograph/pyct/static_analysis/live_values.py
@@ -39,7 +39,8 @@ class LiveValueResolver(transformer.Base):
 
   def visit_ClassDef(self, node):
     self.generic_visit(node)
-    anno.setanno(node, 'live_val', self.entity_info.namespace[node.name])
+    anno.setanno(
+        node, 'live_val', self.transformer_ctx.entity_info.namespace[node.name])
     return node
 
   def visit_Name(self, node):
@@ -53,8 +54,8 @@ class LiveValueResolver(transformer.Base):
       if not is_defined:
         if node.id in self.literals:
           anno.setanno(node, 'live_val', self.literals[node.id])
-        elif node.id in self.entity_info.namespace:
-          obj = self.entity_info.namespace[node.id]
+        elif node.id in self.transformer_ctx.entity_info.namespace:
+          obj = self.transformer_ctx.entity_info.namespace[node.id]
           anno.setanno(node, 'live_val', obj)
           if hasattr(obj, '__name__'):
             anno.setanno(node, 'fqn', (obj.__name__,))
@@ -86,8 +87,8 @@ class LiveValueResolver(transformer.Base):
         def_, = defs
         # Note: param_of is a weakref.
         if def_.param_of and def_.param_of() is self.enclosing_entities[0]:
-          if node.id in self.entity_info.arg_values:
-            obj = self.entity_info.arg_values[node.id]
+          if node.id in self.transformer_ctx.entity_info.arg_values:
+            obj = self.transformer_ctx.entity_info.arg_values[node.id]
             anno.setanno(node, 'live_val', obj)
             anno.setanno(node, 'fqn', (obj.__class__.__name__,))
     return node
diff --git a/tensorflow/python/autograph/pyct/static_analysis/type_info.py b/tensorflow/python/autograph/pyct/static_analysis/type_info.py
index edb2ef0e27..9230457b2e 100644
--- a/tensorflow/python/autograph/pyct/static_analysis/type_info.py
+++ b/tensorflow/python/autograph/pyct/static_analysis/type_info.py
@@ -141,10 +141,11 @@ class TypeInfoResolver(transformer.Base):
     arg_name = str(qn)
     self.scope.setval(qn, arg_node)
     if (len(self.enclosing_entities) == 1 and
-        arg_name in self.entity_info.arg_types):
+        arg_name in self.transformer_ctx.entity_info.arg_types):
       # Forge a node to hold the type information, so that method calls on
       # it can resolve the type.
-      type_string, type_obj = self.entity_info.arg_types[arg_name]
+      type_string, type_obj = self.transformer_ctx.entity_info.arg_types[
+          arg_name]
       anno.setanno(arg_node, 'type', type_obj)
       anno.setanno(arg_node, 'type_fqn', tuple(type_string.split('.')))
 
diff --git a/tensorflow/python/autograph/pyct/transformer.py b/tensorflow/python/autograph/pyct/transformer.py
index 3578f38c65..8c019dd8f6 100644
--- a/tensorflow/python/autograph/pyct/transformer.py
+++ b/tensorflow/python/autograph/pyct/transformer.py
@@ -33,6 +33,23 @@ class AutoGraphParseError(SyntaxError):
   pass
 
 
+# TODO(znado): Use namedtuple.
+class Context(object):
+  """Contains information about a source code transformation.
+
+  This object is mutable, and is updated during conversion. Not thread safe.
+
+  Attributes:
+    entity_info: EntityInfo, immutable.
+    current_origin: origin_info.OriginInfo, holds the OriginInfo of the last
+      AST node to be processed successfully. Useful for error handling.
+  """
+
+  def __init__(self, entity_info):
+    self.entity_info = entity_info
+    self.current_origin = None
+
+
 # TODO(mdan): Use namedtuple.
 class EntityInfo(object):
   """Contains information about a Python entity.
@@ -208,11 +225,11 @@ class Base(gast.NodeTransformer):
     Args:
       entity_info: An EntityInfo object.
     """
-    self._current_origin = None
     self._lineno = 0
     self._col_offset = 0
-    # TODO(znado): remove this from the constructor of all Transformers.
-    self.entity_info = entity_info
+    # TODO(znado): make the constructor take a Context instead of an EntityInfo.
+    # TODO(mdan): Rename to self.ctx.
+    self.transformer_ctx = Context(entity_info)
     self._enclosing_entities = []
 
     # A stack that allows keeping mutable, scope-local state where scopes may be
@@ -446,8 +463,8 @@ class Base(gast.NodeTransformer):
     local_scope_size_at_entry = len(self._local_scope_state)
     processing_expr_node = False
 
+    parent_origin = self.transformer_ctx.current_origin
     try:
-      parent_origin = self._current_origin
       if isinstance(node, (gast.FunctionDef, gast.ClassDef, gast.Lambda)):
         did_enter_function = True
       elif isinstance(node, gast.Expr):
@@ -457,14 +474,15 @@ class Base(gast.NodeTransformer):
         self._enclosing_entities.append(node)
 
       if anno.hasanno(node, anno.Basic.ORIGIN):
-        self._current_origin = anno.getanno(node, anno.Basic.ORIGIN)
+        self.transformer_ctx.current_origin = anno.getanno(node,
+                                                           anno.Basic.ORIGIN)
 
       if processing_expr_node:
         entry_expr_value = node.value
 
       if not anno.hasanno(node, anno.Basic.SKIP_PROCESSING):
         result = super(Base, self).visit(node)
-      self._current_origin = parent_origin
+      self.transformer_ctx.current_origin = parent_origin
 
       # Adjust for consistency: replacing the value of an Expr with
       # an Assign node removes the need for the Expr node.
@@ -491,12 +509,13 @@ class Base(gast.NodeTransformer):
       return result
 
     except (ValueError, AttributeError, KeyError, NotImplementedError) as e:
-      if not self._current_origin:
+      if not self.transformer_ctx.current_origin:
         raise e
-      original_file_path = self._current_origin.loc.filename
-      original_line_number = self._current_origin.loc.lineno
-      original_col_offset = self._current_origin.loc.col_offset
-      original_source_line = self._current_origin.source_code_line
+      original_file_path = self.transformer_ctx.current_origin.loc.filename
+      original_line_number = self.transformer_ctx.current_origin.loc.lineno
+      original_col_offset = self.transformer_ctx.current_origin.loc.col_offset
+      original_source_line = (
+          self.transformer_ctx.current_origin.source_code_line)
       msg = '%s: %s.' % (e.__class__.__name__, str(e))
 
       # TODO(mdan): Avoid the printing of the original exception.
@@ -507,5 +526,3 @@ class Base(gast.NodeTransformer):
           AutoGraphParseError(msg, (original_file_path, original_line_number,
                                     original_col_offset, original_source_line)),
           sys.exc_info()[2])
-    finally:
-      self._current_origin = parent_origin
-- 
GitLab


From 80f47caee4bca375b8285e3b3185c7d29113b9e8 Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <kramerb@google.com>
Date: Tue, 22 Jan 2019 09:19:46 -0800
Subject: [PATCH 1051/2345] [TF:XLA] Drop unused types.h include from
 executable_run_options.h

This is part of the tfcompile deps, and we don't want to have a dependency on
half of TensorFlow for AOT users, even though it's header-only.

PiperOrigin-RevId: 230352690
---
 tensorflow/compiler/xla/executable_run_options.h | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/tensorflow/compiler/xla/executable_run_options.h b/tensorflow/compiler/xla/executable_run_options.h
index ba3217f31b..6f36d11dfb 100644
--- a/tensorflow/compiler/xla/executable_run_options.h
+++ b/tensorflow/compiler/xla/executable_run_options.h
@@ -16,9 +16,6 @@ limitations under the License.
 #ifndef TENSORFLOW_COMPILER_XLA_EXECUTABLE_RUN_OPTIONS_H_
 #define TENSORFLOW_COMPILER_XLA_EXECUTABLE_RUN_OPTIONS_H_
 
-// Pulls in the ::stream_executor -> ::xla::se namespace alias.
-#include "tensorflow/compiler/xla/types.h"
-
 // These classes are forward declared so that ExecutableRunOptions can be linked
 // into an XLA-compiled binary without having to link all of the pointed-to
 // objects (e.g., for an ahead-of-time compiled CPU binary, the gpu tools don't
@@ -28,12 +25,6 @@ class Stream;
 class Platform;
 }  // namespace stream_executor
 
-namespace tensorflow {
-namespace thread {
-class ThreadPool;
-}  // namespace thread
-}  // namespace tensorflow
-
 namespace Eigen {
 struct ThreadPoolDevice;
 }  // namespace Eigen
-- 
GitLab


From 5d99fe9b657f34f22abf696e3d20d01f491af143 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 22 Jan 2019 09:44:08 -0800
Subject: [PATCH 1052/2345] Canonicalize Not(...) more aggressively in deadness
 analysis

This change makes deadness analysis canonicalize Not expressions more
aggressively using DeMorgan's law.  It uses this added power to more
aggressively detect And and Or expressions whose operands include a predicate
and its negation.

The main motivation for doing this is to simplify ~#true to #false and ~#false
to #true, but I can't test that directly before another related change lands.

PiperOrigin-RevId: 230356857
---
 tensorflow/compiler/jit/deadness_analysis.cc  | 131 ++++++++++++++++--
 .../compiler/jit/deadness_analysis_test.cc    |  39 ++++++
 2 files changed, 155 insertions(+), 15 deletions(-)

diff --git a/tensorflow/compiler/jit/deadness_analysis.cc b/tensorflow/compiler/jit/deadness_analysis.cc
index 0562838f62..9a2ada5b42 100644
--- a/tensorflow/compiler/jit/deadness_analysis.cc
+++ b/tensorflow/compiler/jit/deadness_analysis.cc
@@ -333,16 +333,19 @@ class PredicateFactory {
   }
 
   Predicate* MakeNotPredicate(Predicate* pred) {
-    SignatureForNot signature = pred;
-    auto it = interned_not_instances_.find(signature);
-    if (it == interned_not_instances_.end()) {
-      std::unique_ptr<Predicate> new_pred = Make<NotPredicate>(pred);
-      Predicate* new_pred_ptr = new_pred.get();
-      interned_not_instances_.emplace(signature, std::move(new_pred));
-      return new_pred_ptr;
-    } else {
-      return it->second.get();
+    auto it = make_not_predicate_cache_.find(pred);
+    if (it != make_not_predicate_cache_.end()) {
+      return it->second;
     }
+
+    Predicate* result = MakeNotPredicateImpl(pred);
+
+    bool insert_successful =
+        make_not_predicate_cache_.insert({pred, result}).second;
+    (void)insert_successful;
+    DCHECK(insert_successful);
+
+    return result;
   }
 
   Predicate* MakeAndRecurrencePredicate(Predicate* start, Predicate* step) {
@@ -378,7 +381,52 @@ class PredicateFactory {
   Predicate* MakeTrue() { return MakeAndPredicate({}); }
   Predicate* MakeFalse() { return MakeOrPredicate({}); }
 
+  ~PredicateFactory() {
+    DCHECK_EQ(stack_depth_, 0) << "Unnested IncrementStackDepth?";
+  }
+
  private:
+  Predicate* MakeNotPredicateImpl(Predicate* pred) {
+    IncrementStackDepth stack_frame(this);
+    if (!stack_frame.HasOverflowed()) {
+      if (Predicate* simplified = SimplifyUsingDeMorgan(pred)) {
+        return simplified;
+      }
+
+      // ~~A => A
+      if (auto* not_pred = dynamic_cast<NotPredicate*>(pred)) {
+        return not_pred->operand();
+      }
+    }
+
+    SignatureForNot signature = pred;
+    auto it = interned_not_instances_.find(signature);
+    if (it == interned_not_instances_.end()) {
+      std::unique_ptr<Predicate> new_pred = Make<NotPredicate>(pred);
+      Predicate* new_pred_ptr = new_pred.get();
+      interned_not_instances_.emplace(signature, std::move(new_pred));
+      return new_pred_ptr;
+    } else {
+      return it->second.get();
+    }
+  }
+
+  Predicate* SimplifyUsingDeMorgan(Predicate* pred) {
+    // ~(A & B & C & ...) => ~A | ~B | ~C | ~...
+    // ~(A | B | C | ...) -> ~A & ~B & ~C & ~...
+    Predicate::Kind kind = pred->kind();
+
+    if (kind == Predicate::Kind::kAnd || kind == Predicate::Kind::kOr) {
+      std::vector<Predicate*> new_operands;
+      absl::c_transform(pred->GetOperands(), std::back_inserter(new_operands),
+                        [&](Predicate* p) { return MakeNotPredicate(p); });
+      return kind == Predicate::Kind::kOr ? MakeAndPredicate(new_operands)
+                                          : MakeOrPredicate(new_operands);
+    }
+
+    return nullptr;
+  }
+
   template <typename PredicateT, typename... Args>
   std::unique_ptr<Predicate> Make(Args&&... args) {
     return std::unique_ptr<PredicateT>(
@@ -422,6 +470,36 @@ class PredicateFactory {
     }
   };
 
+  // Used to limit recursion to avoid blowing up the stack and cap compile time.
+  class IncrementStackDepth {
+   public:
+    explicit IncrementStackDepth(PredicateFactory* parent) : parent_(parent) {
+      parent_->stack_depth_++;
+    }
+
+    bool HasOverflowed() const {
+      const int kMaxStackDepth = 8;
+      return parent_->stack_depth_ >= kMaxStackDepth;
+    }
+
+    ~IncrementStackDepth() { parent_->stack_depth_--; }
+
+   private:
+    PredicateFactory* parent_;
+  };
+
+  // A cache for the MakeNotPredicate function.
+  //
+  // NB! This is *not* the same as `interned_not_instances_`.
+  // `interned_not_instances_` maps ensures pointer identity for `NotPredicate`
+  // instances, i.e., it ensures there at most one instance of Not(predicate)
+  // for any given predicate whereas `make_not_predicate_cache_` simply caches
+  // the result of the `MakeNotPredicate` function.  The values in
+  // `interned_not_instances_` are always instance of `NotPredicate` whereas the
+  // values in `make_not_predicate_cache_` may not be (for instance it will map
+  // Not(Not(A)) to A).
+  absl::flat_hash_map<Predicate*, Predicate*> make_not_predicate_cache_;
+
   absl::flat_hash_map<SignatureForAndOr, std::unique_ptr<Predicate>,
                       HashSignatureForAndOr>
       interned_and_or_instances_;
@@ -432,6 +510,7 @@ class PredicateFactory {
   absl::flat_hash_map<SignatureForSymbol, std::unique_ptr<Predicate>,
                       HashSignatureForSymbol>
       interned_symbol_instances_;
+  int stack_depth_ = 0;
 };
 
 Predicate* PredicateFactory::MakeInternedAndOr(
@@ -466,6 +545,13 @@ Predicate* PredicateFactory::MakeAndOrImpl(
     absl::Span<Predicate* const> operands, bool is_and) {
   Predicate::Kind pred_kind =
       is_and ? Predicate::Kind::kAnd : Predicate::Kind::kOr;
+
+  IncrementStackDepth stack_frame(this);
+  if (stack_frame.HasOverflowed()) {
+    return MakeInternedAndOr(
+        std::vector<Predicate*>(operands.begin(), operands.end()), pred_kind);
+  }
+
   Predicate::Kind other_pred_kind =
       is_and ? Predicate::Kind::kOr : Predicate::Kind::kAnd;
   absl::flat_hash_set<Predicate*> simplified_ops_set;
@@ -494,16 +580,31 @@ Predicate* PredicateFactory::MakeAndOrImpl(
 
   // Simplify "A&~A=>False" and "A|~A=>True".
   absl::flat_hash_set<Predicate*> negated_ops;
-  for (Predicate* op : simplified_ops) {
-    if (op->kind() == Predicate::Kind::kNot) {
-      negated_ops.insert(dynamic_cast<NotPredicate&>(*op).operand());
-    }
-  }
-
   for (Predicate* op : simplified_ops) {
     if (negated_ops.count(op)) {
+      // Simple case:
+      //
+      //   A & ~A & ... == False
+      //   A | ~A | ... == True
       return is_and ? MakeFalse() : MakeTrue();
     }
+
+    Predicate* negated_op = MakeNotPredicate(op);
+    if (negated_op->kind() == pred_kind) {
+      // Slightly more complicated case:
+      //
+      //   (~A | ~B | ~C) & A & B & C & ... ==
+      //   ~(A & B & C) & (A & B & C) & ... == False
+      //
+      //   (~A & ~B & ~C) | A | B | C | ... ==
+      //   ~(A | B | C) | (A | B | C) | ... == True
+      if (absl::c_all_of(negated_op->GetOperands(), [&](Predicate* p) {
+            return simplified_ops_set.contains(p);
+          })) {
+        return is_and ? MakeFalse() : MakeTrue();
+      }
+    }
+    negated_ops.insert(negated_op);
   }
 
   // If all ops contain the same subop, then factor it out thanks to the
diff --git a/tensorflow/compiler/jit/deadness_analysis_test.cc b/tensorflow/compiler/jit/deadness_analysis_test.cc
index 8a73101c18..11ee00ac6b 100644
--- a/tensorflow/compiler/jit/deadness_analysis_test.cc
+++ b/tensorflow/compiler/jit/deadness_analysis_test.cc
@@ -818,5 +818,44 @@ TEST(DeadnessAnalysisTest, RecvVsSwitchText) {
   EXPECT_EQ(predicate_map[logical_and_output_0], "(recv:0 & *recv:0)");
 }
 
+TEST(DeadnessAnalysisTest, DeMorgan) {
+  Scope root = Scope::NewRootScope().ExitOnError();
+
+  Output cond_0 = ops::Placeholder(root.WithOpName("cond_0"), DT_BOOL);
+  Output cond_1 = ops::Placeholder(root.WithOpName("cond_1"), DT_BOOL);
+  Output value = ops::Placeholder(root.WithOpName("value"), DT_FLOAT);
+
+  ops::Switch sw_0(root.WithOpName("switch_0"), value, cond_0);
+  ops::Switch sw_1(root.WithOpName("switch_1"), value, cond_1);
+
+  Output and_0_1 =
+      ops::Add(root.WithOpName("and_0_1"), sw_0.output_true, sw_1.output_true);
+
+  Output or_not0_not1 = ops::Merge(root.WithOpName("or_not0_not1"),
+                                   {sw_0.output_false, sw_1.output_false})
+                            .output;
+
+  // Predicate(should_always_be_dead) =
+  // (A & B) & (~A | ~B) = (A & B) & ~(A & B) = False
+  Output should_always_be_dead =
+      ops::Add(root.WithOpName("should_always_be_dead"), and_0_1, or_not0_not1);
+
+  // Predicate(should_always_be_dead) =
+  // (A & B) | (~A | ~B) = (A & B) | ~(A & B) = True
+  Output should_always_be_alive =
+      ops::Merge(root.WithOpName("should_always_be_alive"),
+                 {and_0_1, or_not0_not1})
+          .output;
+
+  std::unique_ptr<DeadnessAnalysis> result;
+  TF_ASSERT_OK(AnalyzeDeadness(root.graph(), &result));
+
+  PredicateMapTy predicate_map;
+  TF_ASSERT_OK(ComputePredicates(*root.graph(), &predicate_map));
+
+  EXPECT_EQ(predicate_map[ControlOutputFor(should_always_be_dead)], "#false");
+  EXPECT_EQ(predicate_map[ControlOutputFor(should_always_be_alive)], "#true");
+}
+
 }  // namespace
 }  // namespace tensorflow
-- 
GitLab


From 5731073b59c56e4d7bfc81b153153d4665f76bf5 Mon Sep 17 00:00:00 2001
From: Shanqing Cai <cais@google.com>
Date: Tue, 22 Jan 2019 09:44:23 -0800
Subject: [PATCH 1053/2345] Eager SPINN example: Replace py_binary with
 py_library

PiperOrigin-RevId: 230356890
---
 third_party/examples/eager/spinn/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/third_party/examples/eager/spinn/BUILD b/third_party/examples/eager/spinn/BUILD
index 0e39d4696f..640bcb230c 100644
--- a/third_party/examples/eager/spinn/BUILD
+++ b/third_party/examples/eager/spinn/BUILD
@@ -1,6 +1,6 @@
 licenses(["notice"])  # 3-clause BSD.
 
-py_binary(
+py_library(
     name = "spinn",
     srcs = ["spinn.py"],
     srcs_version = "PY2AND3",
-- 
GitLab


From 4aaca394a3c86ac2c0db0e749a569c1395376fd2 Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Tue, 22 Jan 2019 09:51:03 -0800
Subject: [PATCH 1054/2345] Remove unused dependencies.

PiperOrigin-RevId: 230357984
---
 tensorflow/lite/experimental/c/BUILD                      | 2 --
 tensorflow/lite/experimental/kernels/BUILD                | 8 --------
 tensorflow/lite/experimental/micro/BUILD                  | 1 -
 .../lite/experimental/micro/examples/micro_speech/BUILD   | 3 ---
 tensorflow/lite/experimental/micro/kernels/BUILD          | 2 --
 tensorflow/lite/experimental/writer/BUILD                 | 1 -
 tensorflow/lite/kernels/BUILD                             | 2 --
 tensorflow/lite/kernels/internal/BUILD                    | 3 ---
 tensorflow/lite/profiling/BUILD                           | 1 -
 tensorflow/lite/tools/benchmark/BUILD                     | 4 ----
 10 files changed, 27 deletions(-)

diff --git a/tensorflow/lite/experimental/c/BUILD b/tensorflow/lite/experimental/c/BUILD
index cde53e2838..2f0f4327b7 100644
--- a/tensorflow/lite/experimental/c/BUILD
+++ b/tensorflow/lite/experimental/c/BUILD
@@ -66,7 +66,6 @@ cc_library(
         ":c_api_internal",
         "//tensorflow/lite:context",
         "//tensorflow/lite:framework",
-        "//tensorflow/lite:schema_fbs_version",
         "//tensorflow/lite/kernels:builtin_ops",
     ],
 )
@@ -94,7 +93,6 @@ cc_test(
     deps = [
         ":c_api",
         "//tensorflow/lite:context",
-        "//tensorflow/lite:kernel_api",
         "//tensorflow/lite/testing:util",
         "@com_google_googletest//:gtest",
     ],
diff --git a/tensorflow/lite/experimental/kernels/BUILD b/tensorflow/lite/experimental/kernels/BUILD
index dd314545cb..5d2337f2e2 100644
--- a/tensorflow/lite/experimental/kernels/BUILD
+++ b/tensorflow/lite/experimental/kernels/BUILD
@@ -50,21 +50,13 @@ cc_library(
     }),
     deps = [
         ":ctc_utils",
-        "//tensorflow/lite:builtin_op_data",
         "//tensorflow/lite:framework",
-        "//tensorflow/lite:string_util",
         "//tensorflow/lite/c:c_api_internal",
-        "//tensorflow/lite/kernels:builtin_ops",
-        "//tensorflow/lite/kernels:gemm_support",
         "//tensorflow/lite/kernels:kernel_util",
         "//tensorflow/lite/kernels:op_macros",
-        "//tensorflow/lite/kernels/internal:kernel_utils",
         "//tensorflow/lite/kernels/internal:optimized",
         "//tensorflow/lite/kernels/internal:optimized_base",
-        "//tensorflow/lite/kernels/internal:quantization_util",
-        "//tensorflow/lite/kernels/internal:reference_base",
         "//tensorflow/lite/kernels/internal:tensor",
-        "//tensorflow/lite/kernels/internal:tensor_utils",
         "@flatbuffers",
     ],
 )
diff --git a/tensorflow/lite/experimental/micro/BUILD b/tensorflow/lite/experimental/micro/BUILD
index 2d00ef76f4..b16b8b49f8 100644
--- a/tensorflow/lite/experimental/micro/BUILD
+++ b/tensorflow/lite/experimental/micro/BUILD
@@ -29,7 +29,6 @@ cc_library(
         "simple_tensor_allocator.h",
     ],
     deps = [
-        "//tensorflow/lite:schema_fbs_version",
         "//tensorflow/lite/c:c_api_internal",
         "//tensorflow/lite/core/api",
         "//tensorflow/lite/schema:schema_fbs",
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD b/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD
index 171d07cf56..51ba2976a2 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD
@@ -235,15 +235,12 @@ cc_binary(
     deps = [
         ":audio_provider",
         ":feature_provider",
-        ":features_test_data",
         ":model_settings",
-        ":preprocessor_reference",
         ":recognize_commands",
         ":tiny_conv_model_data",
         "//tensorflow/lite:schema_fbs_version",
         "//tensorflow/lite/experimental/micro:micro_framework",
         "//tensorflow/lite/experimental/micro/kernels:all_ops_resolver",
-        "//tensorflow/lite/experimental/micro/kernels:micro_ops",
         "//tensorflow/lite/schema:schema_fbs",
     ],
 )
diff --git a/tensorflow/lite/experimental/micro/kernels/BUILD b/tensorflow/lite/experimental/micro/kernels/BUILD
index 47ac85c605..e2d3164d4c 100644
--- a/tensorflow/lite/experimental/micro/kernels/BUILD
+++ b/tensorflow/lite/experimental/micro/kernels/BUILD
@@ -22,7 +22,6 @@ cc_library(
     copts = tflite_copts(),
     deps = [
         "//tensorflow/lite/c:c_api_internal",
-        "//tensorflow/lite/experimental/micro:micro_framework",
         "//tensorflow/lite/kernels:kernel_util",
         "//tensorflow/lite/kernels:op_macros",
         "//tensorflow/lite/kernels:padding",
@@ -43,7 +42,6 @@ cc_library(
     copts = tflite_copts(),
     deps = [
         ":micro_ops",
-        "//tensorflow/lite/c:c_api_internal",
         "//tensorflow/lite/experimental/micro:micro_framework",
     ],
 )
diff --git a/tensorflow/lite/experimental/writer/BUILD b/tensorflow/lite/experimental/writer/BUILD
index 57ce636367..6aa81ff173 100644
--- a/tensorflow/lite/experimental/writer/BUILD
+++ b/tensorflow/lite/experimental/writer/BUILD
@@ -33,7 +33,6 @@ cc_library(
         "//tensorflow/lite:builtin_op_data",
         "//tensorflow/lite:framework",
         "//tensorflow/lite:schema_fbs_version",
-        "//tensorflow/lite/kernels:builtin_ops",
         "//tensorflow/lite/schema:schema_fbs_with_reflection",
     ],
 )
diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index be6e5d9ba5..765d7a6731 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -111,7 +111,6 @@ cc_library(
     deps = [
         "//tensorflow/lite/c:c_api_internal",
         "//tensorflow/lite/kernels/internal:round",
-        "//tensorflow/lite/kernels/internal:types",
     ],
 )
 
@@ -286,7 +285,6 @@ cc_library(
     srcs = ["register_ref.cc"],
     hdrs = ["register_ref.h"],
     deps = [
-        ":builtin_op_kernels",
         "//tensorflow/lite:framework",
         "//tensorflow/lite:util",
         "//tensorflow/lite/c:c_api_internal",
diff --git a/tensorflow/lite/kernels/internal/BUILD b/tensorflow/lite/kernels/internal/BUILD
index 1ff5164d2c..c736e5d640 100644
--- a/tensorflow/lite/kernels/internal/BUILD
+++ b/tensorflow/lite/kernels/internal/BUILD
@@ -46,7 +46,6 @@ cc_library(
     ],
     deps = [
         "//tensorflow/lite/kernels:op_macros",
-        "@com_google_absl//absl/base:core_headers",
     ],
 )
 
@@ -60,7 +59,6 @@ cc_library(
     ],
     deps = [
         "//tensorflow/lite/kernels:op_macros",
-        "@com_google_absl//absl/base:core_headers",
     ],
 )
 
@@ -546,7 +544,6 @@ cc_library(
     }),
     deps = [
         ":types",
-        "//tensorflow/lite:string",
     ],
 )
 
diff --git a/tensorflow/lite/profiling/BUILD b/tensorflow/lite/profiling/BUILD
index 64723ba385..bbc252045b 100644
--- a/tensorflow/lite/profiling/BUILD
+++ b/tensorflow/lite/profiling/BUILD
@@ -74,7 +74,6 @@ tf_cc_test(
         ":profile_summarizer",
         "//tensorflow/lite:framework",
         "//tensorflow/lite:schema_fbs_version",
-        "//tensorflow/lite/kernels:builtin_ops",
         "//tensorflow/lite/kernels:kernel_util",
         "//tensorflow/lite/kernels:test_util",
         "//tensorflow/lite/testing:util",
diff --git a/tensorflow/lite/tools/benchmark/BUILD b/tensorflow/lite/tools/benchmark/BUILD
index e68997a00b..ce31eaf42f 100644
--- a/tensorflow/lite/tools/benchmark/BUILD
+++ b/tensorflow/lite/tools/benchmark/BUILD
@@ -141,10 +141,6 @@ cc_library(
         ":logging",
         "//tensorflow/core:stats_calculator_portable",
         "//tensorflow/lite:framework",
-        "//tensorflow/lite:string_util",
-        "//tensorflow/lite/kernels:builtin_ops",
-        "//tensorflow/lite/profiling:profile_summarizer",
-        "//tensorflow/lite/profiling:profiler",
         "//tensorflow/lite/profiling:time",
     ],
 )
-- 
GitLab


From 7d11353dbc2d7f83f154d01eeb2f127eeeb785bf Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 09:59:01 -0800
Subject: [PATCH 1055/2345]  Update documentation for
 GraphOptimizer::Optimize's return value.

PiperOrigin-RevId: 230359499
---
 tensorflow/core/grappler/optimizers/graph_optimizer.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/grappler/optimizers/graph_optimizer.h b/tensorflow/core/grappler/optimizers/graph_optimizer.h
index e587a2b2af..44dfe0de78 100644
--- a/tensorflow/core/grappler/optimizers/graph_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/graph_optimizer.h
@@ -39,7 +39,7 @@ class GraphOptimizer {
   // Routine called to allow an algorithm to propose a rewritten graph
   // for the graph, feeds and fetches in "item" to run more efficiently
   // on "cluster".
-  // Returns true iff it managed to generate a solution, false otherwise.
+  // Returns an error status if it failed to generate a solution.
   virtual Status Optimize(Cluster* cluster, const GrapplerItem& item,
                           GraphDef* optimized_graph) = 0;
 
-- 
GitLab


From 7df1985f651268cd02ba9011b4295412e7d635cf Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Tue, 22 Jan 2019 10:13:12 -0800
Subject: [PATCH 1056/2345] Handle fetching uninitialized tensors from
 TensorLists.

This is needed to support use-cases of the form:

c = constant([1., 2.])
l = tensor_list_from_tensor(c, [])
t = tensor_list_get_item(l, 0)
dt = gradients(t, c)

In the above example gradient of the TensorListFromTensor op, a TensorListStack op, would attempt to stack a list with DT_INVALID type tensors. Since we only read index 0, the gradient list has only index 0 set.

This change replaces the DT_INVALID tensors with zeros of the appropriate shape and type. This change adds an `element_shape` input to TensorListGetItem, TensorListPopBack, TensorListStack and TensorListGather ops. This is the shape used to build a zeros tensor for a DT_INVALID tensor in the list. Returning zeros for DT_VARIANT (nested lists) type tensors is not supported yet.

I will add support for TensorListConcat in a follow-up change.

PiperOrigin-RevId: 230362520
---
 tensorflow/core/kernels/fill_functor.cu.cc    |   6 +
 tensorflow/core/kernels/list_kernels.cc       | 194 +++---------
 tensorflow/core/kernels/list_kernels.cu.cc    |  65 ++--
 tensorflow/core/kernels/list_kernels.h        | 289 ++++++++++++------
 .../core/ops/compat/ops_history.v1.pbtxt      |  16 +
 tensorflow/core/ops/list_ops.cc               |   4 +
 tensorflow/core/ops/ops.pbtxt                 |  16 +
 .../python/kernel_tests/list_ops_test.py      | 178 ++++++++++-
 .../kernel_tests/tensor_array_ops_test.py     |  47 ++-
 tensorflow/python/ops/list_ops.py             |  74 ++++-
 10 files changed, 590 insertions(+), 299 deletions(-)

diff --git a/tensorflow/core/kernels/fill_functor.cu.cc b/tensorflow/core/kernels/fill_functor.cu.cc
index 050c95cf40..946d5c9b02 100644
--- a/tensorflow/core/kernels/fill_functor.cu.cc
+++ b/tensorflow/core/kernels/fill_functor.cu.cc
@@ -88,6 +88,12 @@ struct SetZeroFunctor<GPUDevice, T> {
   }
 };
 
+template <>
+void SetZeroFunctor<GPUDevice, Variant>::operator()(
+    const GPUDevice& d, typename TTypes<Variant>::Flat out) {
+  // TODO(b/123028789): Implement this.
+}
+
 #define DEFINE_SETZERO_GPU(T) template struct SetZeroFunctor<GPUDevice, T>;
 TF_CALL_NUMBER_TYPES(DEFINE_SETZERO_GPU);
 TF_CALL_bool(DEFINE_SETZERO_GPU);
diff --git a/tensorflow/core/kernels/list_kernels.cc b/tensorflow/core/kernels/list_kernels.cc
index 5859c20d89..7c0b70d651 100644
--- a/tensorflow/core/kernels/list_kernels.cc
+++ b/tensorflow/core/kernels/list_kernels.cc
@@ -173,6 +173,17 @@ Status TensorShapeFromTensor(const Tensor& t, PartialTensorShape* out) {
       DataTypeString(t.dtype()));
 }
 
+Status GetElementShapeFromInput(OpKernelContext* c,
+                                const TensorList& tensor_list, int index,
+                                PartialTensorShape* element_shape) {
+  TF_RETURN_IF_ERROR(TensorShapeFromTensor(c->input(index), element_shape));
+  // Check that `element_shape` and `tensor_list.element_shape` are
+  // compatible and store the merged shape in `element_shape`.
+  PartialTensorShape tmp = *element_shape;
+  TF_RETURN_IF_ERROR(tmp.MergeWith(tensor_list.element_shape, element_shape));
+  return Status::OK();
+}
+
 class EmptyTensorList : public OpKernel {
  public:
   explicit EmptyTensorList(OpKernelConstruction* ctx) : OpKernel(ctx) {
@@ -358,69 +369,6 @@ class TensorListElementShape : public OpKernel {
 REGISTER_KERNEL_BUILDER(Name("TensorListElementShape").Device(DEVICE_CPU),
                         TensorListElementShape);
 
-#if GOOGLE_CUDA
-
-REGISTER_KERNEL_BUILDER(Name("TensorListElementShape")
-                            .Device(DEVICE_GPU)
-                            .HostMemory("element_shape"),
-                        TensorListElementShape);
-
-#endif  // GOOGLE_CUDA
-
-class TensorListPopBack : public OpKernel {
- public:
-  explicit TensorListPopBack(OpKernelConstruction* c) : OpKernel(c) {
-    OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_));
-  }
-
-  ~TensorListPopBack() override {}
-
-  void Compute(OpKernelContext* c) override {
-    const TensorList* l = c->input(0).scalar<Variant>()().get<TensorList>();
-    OP_REQUIRES(c, l != nullptr,
-                errors::InvalidArgument(
-                    "Input handle is not a list. Saw: '",
-                    c->input(0).scalar<Variant>()().DebugString(), "'"));
-    OP_REQUIRES(c, element_dtype_ == l->element_dtype,
-                errors::InvalidArgument("Invalid data types; op elements ",
-                                        DataTypeString(element_dtype_),
-                                        " but list elements ",
-                                        DataTypeString(l->element_dtype)));
-
-    OP_REQUIRES(c, !l->tensors.empty(),
-                errors::InvalidArgument("Trying to pop from an empty list."));
-
-    c->set_output(1, l->tensors.back());
-    AllocatorAttributes attr;
-    attr.set_on_host(true);
-    std::unique_ptr<Tensor> maybe_result = c->forward_input(
-        0, 0, DT_VARIANT, TensorShape{}, c->input_memory_type(0), attr);
-    if (maybe_result != nullptr) {
-      maybe_result->scalar<Variant>()().get<TensorList>()->tensors.pop_back();
-    } else {
-      TensorList output;
-      output = *l;
-      output.tensors.pop_back();
-      Tensor* result;
-      OP_REQUIRES_OK(c, c->allocate_output(0, TensorShape{}, &result, attr));
-      result->scalar<Variant>()() = std::move(output);
-    }
-  }
-
- private:
-  DataType element_dtype_;
-};
-
-REGISTER_KERNEL_BUILDER(Name("TensorListPopBack").Device(DEVICE_CPU),
-                        TensorListPopBack);
-
-#if GOOGLE_CUDA
-
-REGISTER_KERNEL_BUILDER(Name("TensorListPopBack").Device(DEVICE_GPU),
-                        TensorListPopBack);
-
-#endif  // GOOGLE_CUDA
-
 class TensorListReserve : public OpKernel {
  public:
   explicit TensorListReserve(OpKernelConstruction* c) : OpKernel(c) {
@@ -458,61 +406,6 @@ REGISTER_KERNEL_BUILDER(Name("TensorListReserve")
                         TensorListReserve);
 
 #endif  // GOOGLE_CUDA
-
-class TensorListGetItem : public OpKernel {
- public:
-  explicit TensorListGetItem(OpKernelConstruction* c) : OpKernel(c) {
-    OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_));
-  }
-
-  void Compute(OpKernelContext* c) override {
-    OP_REQUIRES(
-        c, c->input(0).shape().num_elements() == 1,
-        errors::InvalidArgument("List tensors are supposed to be scalars."));
-    const TensorList* l = c->input(0).scalar<Variant>()().get<TensorList>();
-    OP_REQUIRES(c, l != nullptr,
-                errors::InvalidArgument(
-                    "Input handle is not a list. Saw: '",
-                    c->input(0).scalar<Variant>()().DebugString(), "'"));
-    OP_REQUIRES(c, element_dtype_ == l->element_dtype,
-                errors::InvalidArgument("Invalid data types; op elements ",
-                                        DataTypeString(element_dtype_),
-                                        " but list elements ",
-                                        DataTypeString(l->element_dtype)));
-    int32 index = c->input(1).scalar<int32>()();
-    OP_REQUIRES(c, index < l->tensors.size(),
-                errors::InvalidArgument("Trying to access element ", index,
-                                        " in a list with ", l->tensors.size(),
-                                        " elements."));
-    c->set_output(0, l->tensors[index]);
-  }
-
- private:
-  DataType element_dtype_;
-};
-
-REGISTER_KERNEL_BUILDER(Name("TensorListGetItem").Device(DEVICE_CPU),
-                        TensorListGetItem);
-
-#if GOOGLE_CUDA
-
-#define REGISTER_TENSOR_LIST_GET_ITEM_GPU(T)                      \
-  REGISTER_KERNEL_BUILDER(Name("TensorListGetItem")               \
-                              .TypeConstraint<T>("element_dtype") \
-                              .Device(DEVICE_GPU)                 \
-                              .HostMemory("index"),               \
-                          TensorListGetItem);
-
-TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_GET_ITEM_GPU);
-TF_CALL_complex64(REGISTER_TENSOR_LIST_GET_ITEM_GPU);
-TF_CALL_complex128(REGISTER_TENSOR_LIST_GET_ITEM_GPU);
-TF_CALL_int64(REGISTER_TENSOR_LIST_GET_ITEM_GPU);
-REGISTER_TENSOR_LIST_GET_ITEM_GPU(bfloat16)
-REGISTER_TENSOR_LIST_GET_ITEM_GPU(bool)
-#undef REGISTER_TENSOR_LIST_GET_ITEM_GPU
-
-#endif  // GOOGLE_CUDA
-
 class TensorListResize : public OpKernel {
  public:
   explicit TensorListResize(OpKernelConstruction* c) : OpKernel(c) {}
@@ -736,23 +629,7 @@ REGISTER_KERNEL_BUILDER(Name("TensorListConcatLists").Device(DEVICE_GPU),
 
 #endif  // GOOGLE_CUDA
 
-#define REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(T)               \
-  REGISTER_KERNEL_BUILDER(Name("TensorListPushBackBatch")         \
-                              .TypeConstraint<T>("element_dtype") \
-                              .Device(DEVICE_CPU),                \
-                          TensorListPushBackBatch<CPUDevice, T>)
-
-TF_CALL_ALL_TYPES(REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU);
-REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(quint8);
-REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(qint8);
-REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(quint16);
-REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(qint16);
-REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(qint32);
-REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(bfloat16);
-
-#undef REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU
-
-#define REGISTER_TENSOR_LIST_STACK_CPU(T)                         \
+#define REGISTER_TENSOR_LIST_OPS_CPU(T)                           \
   REGISTER_KERNEL_BUILDER(Name("TensorListStack")                 \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_CPU),                \
@@ -764,19 +641,15 @@ REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(bfloat16);
   REGISTER_KERNEL_BUILDER(Name("TensorListConcat")                \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_CPU),                \
-                          TensorListConcat<CPUDevice, T>)
-
-TF_CALL_POD_STRING_TYPES(REGISTER_TENSOR_LIST_STACK_CPU);
-REGISTER_TENSOR_LIST_STACK_CPU(quint8);
-REGISTER_TENSOR_LIST_STACK_CPU(qint8);
-REGISTER_TENSOR_LIST_STACK_CPU(quint16);
-REGISTER_TENSOR_LIST_STACK_CPU(qint16);
-REGISTER_TENSOR_LIST_STACK_CPU(qint32);
-REGISTER_TENSOR_LIST_STACK_CPU(bfloat16);
-
-#undef REGISTER_TENSOR_LIST_STACK_CPU
-
-#define REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(T)                   \
+                          TensorListConcat<CPUDevice, T>)         \
+  REGISTER_KERNEL_BUILDER(Name("TensorListGetItem")               \
+                              .TypeConstraint<T>("element_dtype") \
+                              .Device(DEVICE_CPU),                \
+                          TensorListGetItem<CPUDevice, T>)        \
+  REGISTER_KERNEL_BUILDER(Name("TensorListPopBack")               \
+                              .TypeConstraint<T>("element_dtype") \
+                              .Device(DEVICE_CPU),                \
+                          TensorListPopBack<CPUDevice, T>)        \
   REGISTER_KERNEL_BUILDER(Name("TensorListFromTensor")            \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_CPU),                \
@@ -788,17 +661,24 @@ REGISTER_TENSOR_LIST_STACK_CPU(bfloat16);
   REGISTER_KERNEL_BUILDER(Name("TensorListSplit")                 \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_CPU),                \
-                          TensorListSplit<CPUDevice, T>)
+                          TensorListSplit<CPUDevice, T>)          \
+  REGISTER_KERNEL_BUILDER(Name("TensorListPushBackBatch")         \
+                              .TypeConstraint<T>("element_dtype") \
+                              .Device(DEVICE_CPU),                \
+                          TensorListPushBackBatch<CPUDevice, T>)
+
+TF_CALL_POD_STRING_TYPES(REGISTER_TENSOR_LIST_OPS_CPU);
+REGISTER_TENSOR_LIST_OPS_CPU(quint8);
+REGISTER_TENSOR_LIST_OPS_CPU(qint8);
+REGISTER_TENSOR_LIST_OPS_CPU(quint16);
+REGISTER_TENSOR_LIST_OPS_CPU(qint16);
+REGISTER_TENSOR_LIST_OPS_CPU(qint32);
+REGISTER_TENSOR_LIST_OPS_CPU(bfloat16);
+REGISTER_TENSOR_LIST_OPS_CPU(Variant);
 
-TF_CALL_POD_STRING_TYPES(REGISTER_TENSOR_LIST_FROM_TENSOR_CPU);
-REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(quint8);
-REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(qint8);
-REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(quint16);
-REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(qint16);
-REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(qint32);
-REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(bfloat16);
+#undef REGISTER_TENSOR_LIST_OPS_CPU
 
-#undef REGISTER_TENSOR_LIST_FROM_TENSOR_CPU
+#define REGISTER_TENSOR_LIST_OPS_CPU(T)
 
 REGISTER_UNARY_VARIANT_BINARY_OP_FUNCTION(ADD_VARIANT_BINARY_OP, DEVICE_CPU,
                                           TensorList,
diff --git a/tensorflow/core/kernels/list_kernels.cu.cc b/tensorflow/core/kernels/list_kernels.cu.cc
index 23f552642c..c3a5041b00 100644
--- a/tensorflow/core/kernels/list_kernels.cu.cc
+++ b/tensorflow/core/kernels/list_kernels.cu.cc
@@ -36,47 +36,38 @@ namespace tensorflow {
 
 typedef Eigen::GpuDevice GPUDevice;
 
-#define REGISTER_TENSOR_LIST_STACK_GPU(T)                         \
+#define REGISTER_TENSOR_LIST_OPS_GPU(T)                           \
   REGISTER_KERNEL_BUILDER(Name("TensorListStack")                 \
                               .TypeConstraint<T>("element_dtype") \
-                              .Device(DEVICE_GPU),                \
+                              .Device(DEVICE_GPU)                 \
+                              .HostMemory("element_shape"),       \
                           TensorListStack<GPUDevice, T>)          \
   REGISTER_KERNEL_BUILDER(Name("TensorListGather")                \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_GPU)                 \
-                              .HostMemory("indices"),             \
+                              .HostMemory("indices")              \
+                              .HostMemory("element_shape"),       \
                           TensorListGather<GPUDevice, T>)         \
+  REGISTER_KERNEL_BUILDER(Name("TensorListGetItem")               \
+                              .TypeConstraint<T>("element_dtype") \
+                              .Device(DEVICE_GPU)                 \
+                              .HostMemory("index")                \
+                              .HostMemory("element_shape"),       \
+                          TensorListGetItem<GPUDevice, T>)        \
+  REGISTER_KERNEL_BUILDER(Name("TensorListPopBack")               \
+                              .TypeConstraint<T>("element_dtype") \
+                              .Device(DEVICE_GPU)                 \
+                              .HostMemory("element_shape"),       \
+                          TensorListPopBack<GPUDevice, T>)        \
   REGISTER_KERNEL_BUILDER(Name("TensorListConcat")                \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_GPU)                 \
                               .HostMemory("lengths"),             \
-                          TensorListConcat<GPUDevice, T>)
-
-TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_STACK_GPU);
-REGISTER_TENSOR_LIST_STACK_GPU(bfloat16);
-TF_CALL_complex64(REGISTER_TENSOR_LIST_STACK_GPU);
-TF_CALL_complex128(REGISTER_TENSOR_LIST_STACK_GPU);
-TF_CALL_int64(REGISTER_TENSOR_LIST_STACK_GPU);
-REGISTER_TENSOR_LIST_STACK_GPU(bool);
-
-#undef REGISTER_TENSOR_LIST_STACK_GPU
-
-#define REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU(T)               \
+                          TensorListConcat<GPUDevice, T>)         \
   REGISTER_KERNEL_BUILDER(Name("TensorListPushBackBatch")         \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_GPU),                \
-                          TensorListPushBackBatch<GPUDevice, T>)
-
-TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU);
-REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU(bfloat16);
-TF_CALL_complex64(REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU);
-TF_CALL_complex128(REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU);
-TF_CALL_int64(REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU);
-REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU(bool);
-
-#undef REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU
-
-#define REGISTER_TENSOR_LIST_FROM_TENSOR_GPU(T)                   \
+                          TensorListPushBackBatch<GPUDevice, T>)  \
   REGISTER_KERNEL_BUILDER(Name("TensorListFromTensor")            \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_GPU)                 \
@@ -95,14 +86,20 @@ REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU(bool);
                               .HostMemory("lengths"),             \
                           TensorListSplit<GPUDevice, T>)
 
-TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_FROM_TENSOR_GPU);
-REGISTER_TENSOR_LIST_FROM_TENSOR_GPU(bfloat16);
-TF_CALL_complex64(REGISTER_TENSOR_LIST_FROM_TENSOR_GPU);
-TF_CALL_complex128(REGISTER_TENSOR_LIST_FROM_TENSOR_GPU);
-TF_CALL_int64(REGISTER_TENSOR_LIST_FROM_TENSOR_GPU);
-REGISTER_TENSOR_LIST_FROM_TENSOR_GPU(bool);
+TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_OPS_GPU);
+REGISTER_TENSOR_LIST_OPS_GPU(bfloat16);
+TF_CALL_complex64(REGISTER_TENSOR_LIST_OPS_GPU);
+TF_CALL_complex128(REGISTER_TENSOR_LIST_OPS_GPU);
+TF_CALL_int64(REGISTER_TENSOR_LIST_OPS_GPU);
+REGISTER_TENSOR_LIST_OPS_GPU(bool);
+
+#undef REGISTER_TENSOR_LIST_OPS_GPU
 
-#undef REGISTER_TENSOR_LIST_FROM_TENSOR_GPU
+REGISTER_KERNEL_BUILDER(Name("TensorListPopBack")
+                            .TypeConstraint<Variant>("element_dtype")
+                            .Device(DEVICE_GPU)
+                            .HostMemory("element_shape"),
+                        TensorListPopBack<GPUDevice, Variant>)
 
 REGISTER_UNARY_VARIANT_BINARY_OP_FUNCTION(ADD_VARIANT_BINARY_OP, DEVICE_GPU,
                                           TensorList,
diff --git a/tensorflow/core/kernels/list_kernels.h b/tensorflow/core/kernels/list_kernels.h
index fd1be80f11..f74a88a768 100644
--- a/tensorflow/core/kernels/list_kernels.h
+++ b/tensorflow/core/kernels/list_kernels.h
@@ -66,14 +66,9 @@ struct TensorList {
 
 Status TensorShapeFromTensor(const Tensor& t, PartialTensorShape* out);
 
-// Allocates a Tensor of requested shape and dtype and fills it with zeros.
-template <typename Device, typename T>
-void BuildZerosTensor(OpKernelContext* c, DataType dtype,
-                      const TensorShape& shape, Tensor* zeros) {
-  OP_REQUIRES_OK(c, c->allocate_temp(dtype, shape, zeros));
-  functor::SetZeroFunctor<Device, T> f;
-  f(c->eigen_device<Device>(), zeros->flat<T>());
-}
+Status GetElementShapeFromInput(OpKernelContext* c,
+                                const TensorList& tensor_list, int index,
+                                PartialTensorShape* element_shape);
 
 template <typename Device, typename T>
 class TensorListStack : public OpKernel {
@@ -85,8 +80,6 @@ class TensorListStack : public OpKernel {
     OP_REQUIRES_OK(c, c->GetAttr("num_elements", &num_elements_));
   }
 
-  ~TensorListStack() {}
-
   void Compute(OpKernelContext* c) override {
     const TensorList* tensor_list =
         c->input(0).scalar<Variant>()().get<TensorList>();
@@ -99,13 +92,6 @@ class TensorListStack : public OpKernel {
         errors::InvalidArgument(
             "Invalid data types; op elements ", DataTypeString(element_dtype_),
             " but list elements ", DataTypeString(tensor_list->element_dtype)));
-    OP_REQUIRES(
-        c,
-        !tensor_list->tensors.empty() ||
-            tensor_list->element_shape.IsFullyDefined(),
-        errors::InvalidArgument("Tried to stack elements of a empty ",
-                                "list with non-fully-defined element_shape: ",
-                                tensor_list->element_shape.DebugString()));
     if (num_elements_ != -1) {
       OP_REQUIRES(c, tensor_list->tensors.size() == num_elements_,
                   errors::InvalidArgument(
@@ -113,44 +99,40 @@ class TensorListStack : public OpKernel {
                       " elements but got a list with ",
                       tensor_list->tensors.size(), " elements."));
     }
-    // Compute the shape of the output tensor.
-    // If `element_shape` is fully-defined it gets used. It is assumed that all
-    // element tensors have the same shape.
-    // If `element_shape` is not fully-defined the shape of the first
-    // initialized element tensor is used and it is checked that all other
-    // initialized tensors have the same shape. An error is thrown if the list
-    // only contains DT_INVALID type tensors.
-    TensorShape resulting_element_shape;
-    if (!tensor_list->element_shape.AsTensorShape(&resulting_element_shape)) {
-      bool resulting_element_shape_initialized = false;
+    PartialTensorShape partial_element_shape;
+    OP_REQUIRES_OK(c, GetElementShapeFromInput(c, *tensor_list, 1,
+                                               &partial_element_shape));
+    OP_REQUIRES(
+        c,
+        partial_element_shape.IsFullyDefined() || !tensor_list->tensors.empty(),
+        errors::InvalidArgument("Tried to stack elements of an empty ",
+                                "list with non-fully-defined element_shape: ",
+                                partial_element_shape.DebugString()));
+
+    // Check that `element_shape` input tensor is compatible with the shapes of
+    // element tensors.
+    if (!tensor_list->element_shape.IsFullyDefined()) {
       for (int i = 0; i < tensor_list->tensors.size(); ++i) {
         const Tensor& t = tensor_list->tensors[i];
-        if (!resulting_element_shape_initialized) {
-          if (t.dtype() == DT_INVALID) {
-            continue;
-          }
-          resulting_element_shape = t.shape();
-          resulting_element_shape_initialized = true;
-          continue;
+        if (t.dtype() != DT_INVALID) {
+          PartialTensorShape tmp = partial_element_shape;
+          OP_REQUIRES_OK(c, tmp.MergeWith(t.shape(), &partial_element_shape));
         }
-        OP_REQUIRES(
-            c, t.dtype() == DT_INVALID || t.shape() == resulting_element_shape,
-            errors::InvalidArgument(
-                "Tried to stack tensors with unequal shapes: ",
-                resulting_element_shape.DebugString(), " vs ",
-                t.shape().DebugString()));
       }
-      OP_REQUIRES(
-          c, resulting_element_shape_initialized,
-          errors::InvalidArgument("Tried to stack list which only contains ",
-                                  "uninitialized tensors and has a ",
-                                  "non-fully-defined element_shape: ",
-                                  tensor_list->element_shape.DebugString()));
     }
-    TensorShape output_tensor_shape = resulting_element_shape;
-    output_tensor_shape.InsertDim(0, tensor_list->tensors.size());
+
+    // Compute the shape of the output tensor by pre-pending the leading dim to
+    // the element_shape.
+    TensorShape element_shape;
+    OP_REQUIRES(c, partial_element_shape.AsTensorShape(&element_shape),
+                errors::InvalidArgument(
+                    "Tried to stack list which only contains uninitialized ",
+                    "tensors and has a non-fully-defined element_shape: ",
+                    partial_element_shape.DebugString()));
+    TensorShape output_shape = element_shape;
+    output_shape.InsertDim(0, tensor_list->tensors.size());
     Tensor* output;
-    OP_REQUIRES_OK(c, c->allocate_output(0, output_tensor_shape, &output));
+    OP_REQUIRES_OK(c, c->allocate_output(0, output_shape, &output));
     if (output->NumElements() == 0) {
       return;
     }
@@ -158,13 +140,21 @@ class TensorListStack : public OpKernel {
     ConstMatrixVector inputs_flat;
     inputs_flat.reserve(tensor_list->tensors.size());
     Tensor zeros;
-    BuildZerosTensor<Device, T>(c, element_dtype_, resulting_element_shape,
-                                &zeros);
     for (const auto& t : tensor_list->tensors) {
       if (t.dtype() != DT_INVALID) {
         inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
             t.shaped<T, 2>({1, t.NumElements()})));
       } else {
+        if (!zeros.NumElements()) {
+          AllocatorAttributes attr;
+          if (element_dtype_ == DT_VARIANT) {
+            attr.set_on_host(true);
+          }
+          OP_REQUIRES_OK(
+              c, c->allocate_temp(element_dtype_, element_shape, &zeros, attr));
+          functor::SetZeroFunctor<Device, T>()(c->eigen_device<Device>(),
+                                               zeros.flat<T>());
+        }
         inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
             const_cast<const Tensor&>(zeros).shaped<T, 2>(
                 {1, zeros.NumElements()})));
@@ -186,6 +176,123 @@ class TensorListStack : public OpKernel {
   DataType element_dtype_;
 };
 
+template <typename Device, typename T>
+class TensorListGetItem : public OpKernel {
+ public:
+  explicit TensorListGetItem(OpKernelConstruction* c) : OpKernel(c) {
+    OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_));
+  }
+
+  void Compute(OpKernelContext* c) override {
+    OP_REQUIRES(
+        c, c->input(0).shape().num_elements() == 1,
+        errors::InvalidArgument("List tensors are supposed to be scalars."));
+    const TensorList* l = c->input(0).scalar<Variant>()().get<TensorList>();
+    OP_REQUIRES(c, l != nullptr,
+                errors::InvalidArgument(
+                    "Input handle is not a list. Saw: '",
+                    c->input(0).scalar<Variant>()().DebugString(), "'"));
+    OP_REQUIRES(c, element_dtype_ == l->element_dtype,
+                errors::InvalidArgument("Invalid data types; op elements ",
+                                        DataTypeString(element_dtype_),
+                                        " but list elements ",
+                                        DataTypeString(l->element_dtype)));
+    int32 index = c->input(1).scalar<int32>()();
+    OP_REQUIRES(c, index < l->tensors.size(),
+                errors::InvalidArgument("Trying to access element ", index,
+                                        " in a list with ", l->tensors.size(),
+                                        " elements."));
+    if (l->tensors[index].dtype() != DT_INVALID) {
+      c->set_output(0, l->tensors[index]);
+    } else {
+      PartialTensorShape partial_element_shape;
+      OP_REQUIRES_OK(
+          c, GetElementShapeFromInput(c, *l, 2, &partial_element_shape));
+      TensorShape element_shape;
+      OP_REQUIRES(
+          c, partial_element_shape.AsTensorShape(&element_shape),
+          errors::InvalidArgument("Trying to read an uninitialized tensor but ",
+                                  "element_shape is not fully defined.",
+                                  partial_element_shape.DebugString()));
+      Tensor* result;
+      AllocatorAttributes attr;
+      if (element_dtype_ == DT_VARIANT) {
+        attr.set_on_host(true);
+      }
+      OP_REQUIRES_OK(c, c->allocate_output(0, element_shape, &result, attr));
+      functor::SetZeroFunctor<Device, T>()(c->eigen_device<Device>(),
+                                           result->flat<T>());
+    }
+  }
+
+ private:
+  DataType element_dtype_;
+};
+
+template <typename Device, typename T>
+class TensorListPopBack : public OpKernel {
+ public:
+  explicit TensorListPopBack(OpKernelConstruction* c) : OpKernel(c) {
+    OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_));
+  }
+
+  void Compute(OpKernelContext* c) override {
+    const TensorList* l = c->input(0).scalar<Variant>()().get<TensorList>();
+    OP_REQUIRES(c, l != nullptr,
+                errors::InvalidArgument(
+                    "Input handle is not a list. Saw: '",
+                    c->input(0).scalar<Variant>()().DebugString(), "'"));
+    OP_REQUIRES(c, element_dtype_ == l->element_dtype,
+                errors::InvalidArgument("Invalid data types; op elements ",
+                                        DataTypeString(element_dtype_),
+                                        " but list elements ",
+                                        DataTypeString(l->element_dtype)));
+
+    OP_REQUIRES(c, !l->tensors.empty(),
+                errors::InvalidArgument("Trying to pop from an empty list."));
+
+    const Tensor& t = l->tensors.back();
+    if (t.dtype() != DT_INVALID) {
+      c->set_output(1, t);
+    } else {
+      PartialTensorShape partial_element_shape;
+      OP_REQUIRES_OK(
+          c, GetElementShapeFromInput(c, *l, 1, &partial_element_shape));
+      TensorShape element_shape;
+      OP_REQUIRES(
+          c, partial_element_shape.AsTensorShape(&element_shape),
+          errors::InvalidArgument("Trying to read an uninitialized tensor but ",
+                                  "element_shape is not fully defined.",
+                                  partial_element_shape.DebugString()));
+      Tensor* result;
+      AllocatorAttributes attr;
+      if (element_dtype_ == DT_VARIANT) {
+        attr.set_on_host(true);
+      }
+      OP_REQUIRES_OK(c, c->allocate_output(1, element_shape, &result, attr));
+      functor::SetZeroFunctor<Device, T>()(c->eigen_device<Device>(),
+                                           result->flat<T>());
+    }
+    AllocatorAttributes attr;
+    attr.set_on_host(true);
+    std::unique_ptr<Tensor> maybe_result = c->forward_input(
+        0, 0, DT_VARIANT, TensorShape{}, c->input_memory_type(0), attr);
+    if (maybe_result != nullptr) {
+      maybe_result->scalar<Variant>()().get<TensorList>()->tensors.pop_back();
+    } else {
+      TensorList output;
+      output = *l;
+      output.tensors.pop_back();
+      Tensor* result;
+      OP_REQUIRES_OK(c, c->allocate_output(0, TensorShape{}, &result, attr));
+      result->scalar<Variant>()() = std::move(output);
+    }
+  }
+
+ private:
+  DataType element_dtype_;
+};
+
 template <typename Device, typename T>
 class TensorListConcat : public OpKernel {
  public:
@@ -206,8 +313,6 @@ class TensorListConcat : public OpKernel {
     }
   }
 
-  ~TensorListConcat() {}
-
   void Compute(OpKernelContext* c) override {
     // Check that the input Variant tensor is indeed a TensorList and has the
     // correct element type.
@@ -418,47 +523,47 @@ class TensorListGather : public OpKernel {
             "Invalid data types; op elements ", DataTypeString(element_dtype_),
             " but list elements ", DataTypeString(tensor_list->element_dtype)));
     Tensor indices = c->input(1);
+    PartialTensorShape partial_element_shape;
+    OP_REQUIRES_OK(c, GetElementShapeFromInput(c, *tensor_list, 2,
+                                               &partial_element_shape));
     OP_REQUIRES(
-        c,
-        indices.NumElements() > 0 ||
-            tensor_list->element_shape.IsFullyDefined(),
+        c, partial_element_shape.IsFullyDefined() || indices.NumElements() > 0,
         errors::InvalidArgument("Tried to gather 0-elements from "
                                 "a list with non-fully-defined shape: ",
-                                tensor_list->element_shape.DebugString()));
-    // Compute the shape of the output tensor.
-    // If `element_shape` is fully-defined it gets used. It is assumed that all
-    // requested tensors have the same shape.
-    // If `element_shape` is not fully-defined the shape of the first requested
-    // tensor is used and it is checked that all other tensors have the same
-    // shape.
-    TensorShape resulting_shape;
-    if (!tensor_list->element_shape.AsTensorShape(&resulting_shape)) {
-      const int i = indices.flat<int32>()(0);
-      OP_REQUIRES(
-          c, i < tensor_list->tensors.size(),
-          errors::InvalidArgument("Index ", i, " out o range; list only has ",
-                                  tensor_list->tensors.size(), " elements."));
-      const Tensor& t = tensor_list->tensors[i];
-      resulting_shape = t.shape();
-      for (int index = 1; index < indices.NumElements(); ++index) {
+                                partial_element_shape.DebugString()));
+
+    // Check that `element_shape` input tensor is compatible with the shapes of
+    // element tensors.
+    if (!tensor_list->element_shape.IsFullyDefined()) {
+      for (int index = 0; index < indices.NumElements(); ++index) {
         const int i = indices.flat<int32>()(index);
         const Tensor& t = tensor_list->tensors[i];
-        OP_REQUIRES(c, t.shape() == resulting_shape,
-                    errors::InvalidArgument(
-                        "Tried to gather elements with unequal shapes: ",
-                        resulting_shape.DebugString(), " vs ",
-                        t.shape().DebugString()));
+        if (t.dtype() != DT_INVALID) {
+          PartialTensorShape tmp = partial_element_shape;
+          OP_REQUIRES_OK(c, tmp.MergeWith(t.shape(), &partial_element_shape));
+        }
       }
     }
-    resulting_shape.InsertDim(0, indices.NumElements());
+
+    // Compute the shape of the output tensor by pre-pending the leading dim to
+    // the element_shape.
+    TensorShape element_shape;
+    OP_REQUIRES(
+        c, partial_element_shape.AsTensorShape(&element_shape),
+        errors::InvalidArgument("Tried to gather uninitialized tensors from a ",
+                                "list with non-fully-defined element_shape: ",
+                                partial_element_shape.DebugString()));
+    TensorShape output_shape = element_shape;
+    output_shape.InsertDim(0, indices.NumElements());
     Tensor* output;
-    OP_REQUIRES_OK(c, c->allocate_output(0, resulting_shape, &output));
+    OP_REQUIRES_OK(c, c->allocate_output(0, output_shape, &output));
     if (output->NumElements() == 0) {
       return;
     }
 
     ConstMatrixVector inputs_flat;
-    inputs_flat.reserve(tensor_list->tensors.size());
+    inputs_flat.reserve(indices.NumElements());
+    Tensor zeros;
     for (int index = 0; index < indices.NumElements(); ++index) {
       const int i = indices.flat<int32>()(index);
       OP_REQUIRES(
@@ -466,8 +571,24 @@ class TensorListGather : public OpKernel {
           errors::InvalidArgument("Index ", i, " out o range; list only has ",
                                   tensor_list->tensors.size(), " elements."));
       const Tensor& t = tensor_list->tensors[i];
-      inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
-          t.shaped<T, 2>({1, t.NumElements()})));
+      if (t.dtype() != DT_INVALID) {
+        inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
+            t.shaped<T, 2>({1, t.NumElements()})));
+      } else {
+        if (!zeros.NumElements()) {
+          AllocatorAttributes attr;
+          if (element_dtype_ == DT_VARIANT) {
+            attr.set_on_host(true);
+          }
+          OP_REQUIRES_OK(
+              c, c->allocate_temp(element_dtype_, element_shape, &zeros, attr));
+          functor::SetZeroFunctor<Device, T>()(c->eigen_device<Device>(),
+                                               zeros.flat<T>());
+        }
+        inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
+            const_cast<const Tensor&>(zeros).shaped<T, 2>(
+                {1, zeros.NumElements()})));
+      }
     }
     auto output_flat = output->shaped<T, 2>({1, output->NumElements()});
 
@@ -657,8 +778,6 @@ class TensorListPushBackBatch : public OpKernel {
     OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_));
   }
 
-  ~TensorListPushBackBatch() override {}
-
   void Compute(OpKernelContext* c) override {
     const Tensor& input = c->input(1);
     OP_REQUIRES(c, element_dtype_ == input.dtype(),
diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 3b7a7b812a..eb251fc517 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -77304,6 +77304,10 @@ op {
     name: "indices"
     type: DT_INT32
   }
+  input_arg {
+    name: "element_shape"
+    type: DT_INT32
+  }
   output_arg {
     name: "values"
     type_attr: "element_dtype"
@@ -77323,6 +77327,10 @@ op {
     name: "index"
     type: DT_INT32
   }
+  input_arg {
+    name: "element_shape"
+    type: DT_INT32
+  }
   output_arg {
     name: "item"
     type_attr: "element_dtype"
@@ -77349,6 +77357,10 @@ op {
     name: "input_handle"
     type: DT_VARIANT
   }
+  input_arg {
+    name: "element_shape"
+    type: DT_INT32
+  }
   output_arg {
     name: "output_handle"
     type: DT_VARIANT
@@ -77540,6 +77552,10 @@ op {
     name: "input_handle"
     type: DT_VARIANT
   }
+  input_arg {
+    name: "element_shape"
+    type: DT_INT32
+  }
   output_arg {
     name: "tensor"
     type_attr: "element_dtype"
diff --git a/tensorflow/core/ops/list_ops.cc b/tensorflow/core/ops/list_ops.cc
index cbc9c7a2f4..a16fabe2ab 100644
--- a/tensorflow/core/ops/list_ops.cc
+++ b/tensorflow/core/ops/list_ops.cc
@@ -130,6 +130,7 @@ REGISTER_OP("TensorListLength")
 
 REGISTER_OP("TensorListPopBack")
     .Input("input_handle: variant")
+    .Input("element_shape: int32")
     .Output("output_handle: variant")
     .Output("tensor: element_dtype")
     .Attr("element_dtype: type")
@@ -166,6 +167,7 @@ REGISTER_OP("TensorListPopBack")
 
 REGISTER_OP("TensorListStack")
     .Input("input_handle: variant")
+    .Input("element_shape: int32")
     .Output("tensor: element_dtype")
     .Attr("element_dtype: type")
     .Attr("num_elements: int = -1")
@@ -351,6 +353,7 @@ REGISTER_OP("TensorListReserve")
 REGISTER_OP("TensorListGetItem")
     .Input("input_handle: variant")
     .Input("index: int32")
+    .Input("element_shape: int32")
     .Output("item: element_dtype")
     .Attr("element_dtype: type")
     .SetShapeFn([](shape_inference::InferenceContext* c) {
@@ -418,6 +421,7 @@ REGISTER_OP("TensorListSetItem")
 REGISTER_OP("TensorListGather")
     .Input("input_handle: variant")
     .Input("indices: int32")
+    .Input("element_shape: int32")
     .Output("values: element_dtype")
     .Attr("element_dtype: type")
     .SetShapeFn([](shape_inference::InferenceContext* c) {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 92a7bb9a34..f43a8c4744 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -37082,6 +37082,10 @@ op {
     name: "indices"
     type: DT_INT32
   }
+  input_arg {
+    name: "element_shape"
+    type: DT_INT32
+  }
   output_arg {
     name: "values"
     type_attr: "element_dtype"
@@ -37101,6 +37105,10 @@ op {
     name: "index"
     type: DT_INT32
   }
+  input_arg {
+    name: "element_shape"
+    type: DT_INT32
+  }
   output_arg {
     name: "item"
     type_attr: "element_dtype"
@@ -37127,6 +37135,10 @@ op {
     name: "input_handle"
     type: DT_VARIANT
   }
+  input_arg {
+    name: "element_shape"
+    type: DT_INT32
+  }
   output_arg {
     name: "output_handle"
     type: DT_VARIANT
@@ -37318,6 +37330,10 @@ op {
     name: "input_handle"
     type: DT_VARIANT
   }
+  input_arg {
+    name: "element_shape"
+    type: DT_INT32
+  }
   output_arg {
     name: "tensor"
     type_attr: "element_dtype"
diff --git a/tensorflow/python/kernel_tests/list_ops_test.py b/tensorflow/python/kernel_tests/list_ops_test.py
index ec6906f20c..3f340aa07d 100644
--- a/tensorflow/python/kernel_tests/list_ops_test.py
+++ b/tensorflow/python/kernel_tests/list_ops_test.py
@@ -33,6 +33,7 @@ from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import gen_list_ops
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import list_ops
 from tensorflow.python.ops import math_ops
@@ -89,6 +90,58 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       l = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
       self.evaluate(l)
 
+  def testPopUninitializedTensorUseListElementShape(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=[2, 3], num_elements=3)
+    _, e = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
+    self.assertAllEqual(e, np.zeros((2, 3)))
+
+  def testPopUninitializedTensorUseSpecifiedElementShape(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=[None, 3], num_elements=3)
+    _, e = gen_list_ops.tensor_list_pop_back(
+        l, element_dtype=dtypes.float32, element_shape=[4, 3])
+    self.assertAllEqual(e, np.zeros((4, 3)))
+
+  def testPopUninitializedTensorWithInvalidElementShapeFails(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        "Trying to read an uninitialized tensor but "
+        "element_shape is not fully defined"):
+      _, e = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
+      self.evaluate(e)
+
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=[None, 2], num_elements=3)
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        r"Incompatible shapes during merge: \[1,3\] vs. \[\?,2\]"):
+      _, e = gen_list_ops.tensor_list_pop_back(
+          l, element_dtype=dtypes.float32, element_shape=[1, 3])
+      self.evaluate(e)
+
+  def testPushGetGrad(self):
+    with backprop.GradientTape() as tape:
+      l = list_ops.empty_tensor_list(
+          element_dtype=dtypes.float32, element_shape=None)
+      c0 = constant_op.constant(5.0)
+      c1 = constant_op.constant([10.0, 20.0])
+      tape.watch(c0)
+      tape.watch(c1)
+      l = list_ops.tensor_list_push_back(l, c0)
+      l = list_ops.tensor_list_push_back(l, c1)
+      t1 = list_ops.tensor_list_get_item(l, 1, element_dtype=dtypes.float32)
+      self.assertAllEqual(self.evaluate(t1), [10.0, 20.0])
+      # t1 == c1 so the gradient should be [0., [1., 1.]]
+      # This tests that the gradient of push_back correctly converts DT_INVALID
+      # tensors to zeros. The list returned by the gradient of GetItem will
+      # have only have tensor at index 1 set and others set to DT_INVALID.
+      dt0, dt1 = tape.gradient(t1, [c0, c1])
+      self.assertAllEqual(self.evaluate(dt1), [1.0, 1.0])
+      self.assertEqual(self.evaluate(dt0), 0.0)
+
   def _testStack(self, max_num_elements):
     l = list_ops.empty_tensor_list(
         element_dtype=dtypes.float32,
@@ -130,7 +183,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
     # Should raise an error when the element tensors do not all have the same
     # shape.
-    with self.assertRaisesRegexp(errors.InvalidArgumentError, "unequal shapes"):
+    with self.assertRaisesRegexp(errors.InvalidArgumentError,
+                                 "Incompatible ranks during merge: 0 vs. 1"):
       l = list_ops.tensor_list_push_back(l, constant_op.constant([3.0, 4.0]))
       t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
       self.evaluate(t)
@@ -151,7 +205,9 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
     # Should raise an error when the element tensors do not all have the same
     # shape.
-    with self.assertRaisesRegexp(errors.InvalidArgumentError, "unequal shapes"):
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        r"Incompatible shapes during merge: \[1\] vs. \[2\]"):
       l = list_ops.tensor_list_push_back(l, constant_op.constant([2.0, 3.0]))
       t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
       self.evaluate(t)
@@ -230,6 +286,13 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
       self.evaluate(t)
 
+  def testStackUsingSpecifiedElementShape(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
+    t = gen_list_ops.tensor_list_stack(
+        l, element_dtype=dtypes.float32, element_shape=[])
+    self.assertAllEqual(self.evaluate(t), np.zeros((3,)))
+
   @parameterized.named_parameters(("NoMaxNumElements", None),
                                   ("WithMaxNumElements", 2))
   def testGatherGrad(self, max_num_elements):
@@ -268,7 +331,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
     # Should raise an error when the requested tensors do not all have the same
     # shape.
-    with self.assertRaisesRegexp(errors.InvalidArgumentError, "unequal shapes"):
+    with self.assertRaisesRegexp(errors.InvalidArgumentError,
+                                 "Incompatible ranks during merge: 0 vs. 1"):
       t = list_ops.tensor_list_gather(l, [0, 2], element_dtype=dtypes.float32)
       self.evaluate(t)
 
@@ -292,7 +356,9 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
     # Should raise an error when the requested tensors do not all have the same
     # shape.
-    with self.assertRaisesRegexp(errors.InvalidArgumentError, "unequal shapes"):
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        r"Incompatible shapes during merge: \[1\] vs. \[2\]"):
       t = list_ops.tensor_list_gather(l, [0, 2], element_dtype=dtypes.float32)
       self.evaluate(t)
 
@@ -347,6 +413,54 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     dl_length = list_ops.tensor_list_length(dl)
     self.assertAllEqual(self.evaluate(dl_length), 3)
 
+  def _testGatherWithUninitializedTensors(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=[], num_elements=3)
+    t = list_ops.tensor_list_gather(l, [0, 2], element_dtype=dtypes.float32)
+    self.assertAllEqual(self.evaluate(t), [0., 0.])
+
+  def testGatherWithUninitializedTensors(self):
+    self._testGatherWithUninitializedTensors()
+
+  def testGatherWithUninitializedTensorsGpu(self):
+    if not context.num_gpus():
+      return
+    with context.device("gpu:0"):
+      self._testGatherWithUninitializedTensors()
+
+  def _testGatherWithUninitializedTensorsInferShape(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
+    l = list_ops.tensor_list_set_item(l, 1, [1., 2.])
+    t = list_ops.tensor_list_gather(l, [1, 2], element_dtype=dtypes.float32)
+    self.assertAllEqual(self.evaluate(t), [[1., 2.], [0., 0.]])
+
+  def testGatherWithUninitializedTensorsInferShape(self):
+    self._testGatherWithUninitializedTensorsInferShape()
+
+  def testGatherWithUninitializedTensorsInferShapeGpu(self):
+    if not context.num_gpus():
+      return
+    with context.device("gpu:0"):
+      self._testGatherWithUninitializedTensorsInferShape()
+
+  def testGatherReservedListWithNoElementsAndPartialElementShapeFails(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        "Tried to gather uninitialized tensors from a"
+        " list with non-fully-defined element_shape"):
+      t = list_ops.tensor_list_gather(l, [0], element_dtype=dtypes.float32)
+      self.evaluate(t)
+
+  def testGatherUsingSpecifiedElementShape(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
+    t = gen_list_ops.tensor_list_gather(
+        l, [0, 1, 2], element_dtype=dtypes.float32, element_shape=[])
+    self.assertAllEqual(self.evaluate(t), np.zeros((3,)))
+
   def testScatterOutputListSize(self):
     c0 = constant_op.constant([1.0, 2.0])
     l = list_ops.tensor_list_scatter(
@@ -386,6 +500,18 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     dt = tape.gradient(loss, c0)
     self.assertAllEqual(self.evaluate(dt), [2., 4.])
 
+  def testScatterWithPartialReadGrad(self):
+    with backprop.GradientTape() as tape:
+      c0 = constant_op.constant([1.0, 2.0])
+      tape.watch(c0)
+      l = list_ops.tensor_list_scatter(
+          c0, [1, 0], ops.convert_to_tensor([], dtype=dtypes.int32))
+      t0 = list_ops.tensor_list_get_item(l, 0, element_dtype=dtypes.float32)
+      self.assertAllEqual(self.evaluate(t0), 2.0)
+      loss = t0 * t0
+    dt = tape.gradient(loss, c0)
+    self.assertAllEqual(self.evaluate(dt), [0., 4.])
+
   def testTensorListFromTensor(self):
     t = constant_op.constant([1.0, 2.0])
     l = list_ops.tensor_list_from_tensor(t, element_shape=[])
@@ -427,6 +553,50 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       self.assertAllEqual(self.evaluate(e), 10.0)
     self.assertAllEqual(self.evaluate(tape.gradient(e, t)), 2.0)
 
+  def testGetUninitializedTensorUseListElementShape(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=[], num_elements=3)
+    l = list_ops.tensor_list_set_item(l, 0, 5.)
+    e1 = list_ops.tensor_list_get_item(l, 1, element_dtype=dtypes.float32)
+    e2 = list_ops.tensor_list_get_item(l, 2, element_dtype=dtypes.float32)
+    self.assertEqual(self.evaluate(e1), 0.)
+    self.assertEqual(self.evaluate(e2), 0.)
+
+  def testGetUninitializedTensorUseSpecifiedElementShape(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
+    e0 = gen_list_ops.tensor_list_get_item(
+        l, 0, element_shape=[], element_dtype=dtypes.float32)
+    e1 = gen_list_ops.tensor_list_get_item(
+        l, 1, element_shape=[2, 3], element_dtype=dtypes.float32)
+    self.assertEqual(self.evaluate(e0), 0.)
+    self.assertAllEqual(self.evaluate(e1), np.zeros((2, 3)))
+
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=[None, 3], num_elements=3)
+    e1 = gen_list_ops.tensor_list_get_item(
+        l, 1, element_shape=[2, 3], element_dtype=dtypes.float32)
+    self.assertAllEqual(self.evaluate(e1), np.zeros((2, 3)))
+
+  def testGetUninitializedTensorWithInvalidElementShapeFails(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        "Trying to read an uninitialized tensor but "
+        "element_shape is not fully defined"):
+      e0 = list_ops.tensor_list_get_item(l, 0, element_dtype=dtypes.float32)
+      self.evaluate(e0)
+
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=[None, 2], num_elements=3)
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        r"Incompatible shapes during merge: \[1,3\] vs. \[\?,2\]"):
+      e0 = gen_list_ops.tensor_list_get_item(
+          l, 0, element_dtype=dtypes.float32, element_shape=[1, 3])
+      self.evaluate(e0)
+
   @test_util.run_deprecated_v1
   @test_util.enable_control_flow_v2
   def testSkipEagerSetItemIndexOutOfBounds(self):
diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
index e8af998a70..d0efb47a38 100644
--- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py
+++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
@@ -1281,13 +1281,23 @@ class TensorArrayTest(test.TestCase):
       grad_r0_vals = session.run(grad_r0)[0]
       self.assertAllEqual(grad_r0_vals, [1.0, 0.0])
 
-  # TODO(srbs): Figure out how to enable this. This is probably failing
-  # because we are trying to stack a TensorList with invalid tensors.
-  # That is because we do not receive gradients for all list indices.
-  # Figure out how TensorArray handles this.
-  def disabletestGradientWhenNotAllComponentsRead(self):
+  @test_util.run_deprecated_v1
+  def testSkipEagerGradientWhenNotAllComponentsRead(self):
     self._testGradientWhenNotAllComponentsRead()
 
+  @test_util.run_deprecated_v1
+  def testSkipEagerWriteButNotAllComponentsReadGrad(self):
+    with self.cached_session(use_gpu=True) as session:
+      x0 = constant_op.constant(5.0)
+      x1 = constant_op.constant(10.0)
+      ta = tensor_array_ops.TensorArray(
+          dtype=dtypes.float32, size=2).write(0, x0).write(1, x1)
+      r0 = ta.read(0)
+      # calculate (dr0/dx0, dr0/dx1).  since r0 = x0, gradients are (1, 0).
+      grad_r0_x1 = gradients_impl.gradients(ys=[r0], xs=[x0, x1], grad_ys=[1.0])
+      grad_r0_x1_vals = session.run(grad_r0_x1)
+      self.assertAllEqual(grad_r0_x1_vals, [1.0, 0.0])
+
   def _testTensorArrayUnpackDynamic(self):
     with self.cached_session(use_gpu=True) as sess:
       ta = tensor_array_ops.TensorArray(
@@ -1321,8 +1331,8 @@ class TensorArrayTest(test.TestCase):
     with self.cached_session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
           dtype=dtypes.float32, size=0, dynamic_size=False, infer_shape=False)
-      v2_msg = ("Tried to stack elements of a empty list with non-fully-defined"
-                " element_shape")
+      v2_msg = ("Tried to stack elements of an empty list with "
+                "non-fully-defined element_shape")
       v1_msg = (
           "TensorArray has size zero, but element shape <unknown> is not "
           "fully defined. Currently only static shapes are supported when "
@@ -1385,6 +1395,29 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual([10.0, -10.0], read_vals[1])
       self.assertAllEqual([[2.0, 3.0], [4.0, 5.0]], grad_vals[0])
 
+  @test_util.run_v1_only("b/117943489")
+  def testSkipEagerTensorArrayScatterPartialReadAndGradients(self):
+    with self.session(use_gpu=True) as session:
+      ta = tensor_array_ops.TensorArray(
+          dtype=dtypes.float32,
+          tensor_array_name="foo",
+          size=0,
+          dynamic_size=True)
+
+      indices = constant_op.constant([1, 8])
+      value = constant_op.constant([[1.0, -1.0], [10.0, -10.0]])
+
+      w = ta.scatter(indices, value)
+      r0 = w.read(1)
+
+      # Test combined gradients + aggregation of read(0)
+      grad = gradients_impl.gradients(
+          ys=[r0], xs=[value], grad_ys=[[2.0, 3.0]])[0]
+      read_val, grad_val = session.run([r0, grad])
+
+      self.assertAllEqual([1.0, -1.0], read_val)
+      self.assertAllEqual([[2.0, 3.0], [0.0, 0.0]], grad_val)
+
   @test_util.run_v1_only("b/118890905")
   def testTensorArrayWriteGatherAndGradients(self):
     with self.session(use_gpu=True) as session:
diff --git a/tensorflow/python/ops/list_ops.py b/tensorflow/python/ops/list_ops.py
index df928ea85d..af059a541e 100644
--- a/tensorflow/python/ops/list_ops.py
+++ b/tensorflow/python/ops/list_ops.py
@@ -71,6 +71,41 @@ def tensor_list_from_tensor(tensor, element_shape, name=None):
       name=name)
 
 
+def tensor_list_get_item(input_handle, index, element_dtype, name=None):
+  return gen_list_ops.tensor_list_get_item(
+      input_handle=input_handle,
+      index=index,
+      element_shape=-1,
+      element_dtype=element_dtype,
+      name=name)
+
+
+def tensor_list_pop_back(input_handle, element_dtype, name=None):
+  return gen_list_ops.tensor_list_pop_back(
+      input_handle=input_handle,
+      element_shape=-1,
+      element_dtype=element_dtype,
+      name=name)
+
+
+def tensor_list_gather(input_handle, indices, element_dtype, name=None):
+  return gen_list_ops.tensor_list_gather(
+      input_handle=input_handle,
+      indices=indices,
+      element_shape=-1,
+      element_dtype=element_dtype,
+      name=name)
+
+
+def tensor_list_stack(input_handle, element_dtype, num_elements=-1, name=None):
+  return gen_list_ops.tensor_list_stack(
+      input_handle=input_handle,
+      element_shape=-1,
+      element_dtype=element_dtype,
+      num_elements=num_elements,
+      name=name)
+
+
 def tensor_list_concat(input_handle, element_dtype, element_shape=None,
                        name=None):
   # Ignore the lengths output of TensorListConcat. It is only used during
@@ -110,7 +145,9 @@ def tensor_list_set_item(input_handle,
 @ops.RegisterGradient("TensorListPushBack")
 def _PushBackGrad(op, dresult):
   return gen_list_ops.tensor_list_pop_back(
-      dresult, element_dtype=op.get_attr("element_dtype"))
+      dresult,
+      element_shape=array_ops.shape(op.inputs[1]),
+      element_dtype=op.get_attr("element_dtype"))
 
 
 @ops.RegisterGradient("TensorListPopBack")
@@ -120,12 +157,12 @@ def _PopBackGrad(op, dlist, delement):
         element_dtype=delement.dtype,
         element_shape=gen_list_ops.tensor_list_element_shape(
             op.outputs[0], shape_type=dtypes.int32))
-  return gen_list_ops.tensor_list_push_back(dlist, delement)
+  return gen_list_ops.tensor_list_push_back(dlist, delement), None
 
 
 @ops.RegisterGradient("TensorListStack")
 def _TensorListStackGrad(unused_op, dtensor):
-  return tensor_list_from_tensor(dtensor, element_shape=dtensor.shape[1:])
+  return tensor_list_from_tensor(dtensor, element_shape=dtensor.shape[1:]), None
 
 
 @ops.RegisterGradient("TensorListConcat")
@@ -150,17 +187,21 @@ def _TensorListSplitGrad(op, dlist):
 @ops.RegisterGradient("TensorListFromTensor")
 def _TensorListFromTensorGrad(op, dlist):
   """Gradient for TensorListFromTensor."""
-  if op.inputs[0].shape.dims and op.inputs[0].shape.dims[0].value is not None:
-    num_elements = op.inputs[0].shape.dims[0].value
+  t = op.inputs[0]
+  if t.shape.dims and t.shape.dims[0].value is not None:
+    num_elements = t.shape.dims[0].value
   else:
     num_elements = None
   if dlist is None:
     dlist = empty_tensor_list(
-        element_dtype=op.inputs[0].dtype,
+        element_dtype=t.dtype,
         element_shape=gen_list_ops.tensor_list_element_shape(
             op.outputs[0], shape_type=dtypes.int32))
   tensor_grad = gen_list_ops.tensor_list_stack(
-      dlist, element_dtype=op.inputs[0].dtype, num_elements=num_elements)
+      dlist,
+      element_shape=array_ops.slice(array_ops.shape(t), [1], [-1]),
+      element_dtype=t.dtype,
+      num_elements=num_elements)
   shape_grad = None
   return tensor_grad, shape_grad
 
@@ -177,17 +218,22 @@ def _TensorListGetItemGrad(op, ditem):
       index=op.inputs[1],
       item=ditem)
   index_grad = None
-  return list_grad, index_grad
+  element_shape_grad = None
+  return list_grad, index_grad, element_shape_grad
 
 
 @ops.RegisterGradient("TensorListSetItem")
 def _TensorListSetItemGrad(op, dlist):
+  """Gradient function for TensorListSetItem."""
   _, index, item = op.inputs
   list_grad = gen_list_ops.tensor_list_set_item(
       dlist, index=index, item=array_ops.zeros_like(item))
   index_grad = None
   element_grad = gen_list_ops.tensor_list_get_item(
-      dlist, index, element_dtype=item.dtype)
+      dlist,
+      index,
+      element_shape=array_ops.shape(item),
+      element_dtype=item.dtype)
   return list_grad, index_grad, element_grad
 
 
@@ -200,7 +246,8 @@ def _TensorListResizeGrad(op, dlist):
 
 @ops.RegisterGradient("TensorListGather")
 def _TensorListGatherGrad(op, dtensor):
-  input_list, indices = op.inputs
+  """Gradient function for TensorListGather."""
+  input_list, indices, _ = op.inputs
   dlist = gen_list_ops.tensor_list_scatter(
       tensor=dtensor,
       indices=indices,
@@ -209,14 +256,17 @@ def _TensorListGatherGrad(op, dtensor):
   # so we manually resize it to match the size of the input list.
   input_list_size = gen_list_ops.tensor_list_length(input_list)
   dlist = gen_list_ops.tensor_list_resize(dlist, input_list_size)
-  return dlist, None
+  return dlist, None, None
 
 
 @ops.RegisterGradient("TensorListScatter")
 def _TensorListScatterGrad(op, dlist):
   t, indices, _ = op.inputs
   return gen_list_ops.tensor_list_gather(
-      dlist, indices, element_dtype=t.dtype), None, None
+      dlist,
+      indices,
+      element_shape=array_ops.slice(array_ops.shape(t), [1], [-1]),
+      element_dtype=t.dtype), None, None
 
 
 def _build_element_shape(shape):
-- 
GitLab


From 0404ca1401ef04904ae4cf5be4619491dca32adf Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 10:14:17 -0800
Subject: [PATCH 1057/2345] Automated rollback of commit
 80f47caee4bca375b8285e3b3185c7d29113b9e8

PiperOrigin-RevId: 230362704
---
 tensorflow/compiler/xla/executable_run_options.h | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tensorflow/compiler/xla/executable_run_options.h b/tensorflow/compiler/xla/executable_run_options.h
index 6f36d11dfb..ba3217f31b 100644
--- a/tensorflow/compiler/xla/executable_run_options.h
+++ b/tensorflow/compiler/xla/executable_run_options.h
@@ -16,6 +16,9 @@ limitations under the License.
 #ifndef TENSORFLOW_COMPILER_XLA_EXECUTABLE_RUN_OPTIONS_H_
 #define TENSORFLOW_COMPILER_XLA_EXECUTABLE_RUN_OPTIONS_H_
 
+// Pulls in the ::stream_executor -> ::xla::se namespace alias.
+#include "tensorflow/compiler/xla/types.h"
+
 // These classes are forward declared so that ExecutableRunOptions can be linked
 // into an XLA-compiled binary without having to link all of the pointed-to
 // objects (e.g., for an ahead-of-time compiled CPU binary, the gpu tools don't
@@ -25,6 +28,12 @@ class Stream;
 class Platform;
 }  // namespace stream_executor
 
+namespace tensorflow {
+namespace thread {
+class ThreadPool;
+}  // namespace thread
+}  // namespace tensorflow
+
 namespace Eigen {
 struct ThreadPoolDevice;
 }  // namespace Eigen
-- 
GitLab


From 6624e34a94088a96a1ac5a6fafeb1055c96580b6 Mon Sep 17 00:00:00 2001
From: Peter Ma <pcma@google.com>
Date: Tue, 22 Jan 2019 10:25:42 -0800
Subject: [PATCH 1058/2345] Modification for internal usage.

PiperOrigin-RevId: 230364779
---
 tensorflow/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 29d71c323a..0be7920d11 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -385,6 +385,7 @@ package_group(
         "-//third_party/tensorflow/python/estimator",
         "//learning/deepmind/...",
         "//learning/meta_rank/...",
+        "//platforms/performance/autograppler/...",
         "//tensorflow/...",
         "//tensorflow_estimator/contrib/...",
         "//tensorflow_fold/llgtm/...",
-- 
GitLab


From 951a5f1531bb65e90aaa76cec2e56230c11ccdab Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 10:27:52 -0800
Subject: [PATCH 1059/2345] Trivial wording change for tf.Variable
 documentation.

PiperOrigin-RevId: 230365183
---
 tensorflow/python/ops/variables.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index 3392b1aad9..236a1cd537 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -304,7 +304,7 @@ class Variable(six.with_metaclass(VariableMetaclass,
   tf.cond(v, lambda: v.assign(False), my_false_fn)  # Note: this is broken.
   ```
 
-  Here replacing adding `use_resource=True` when constructing the variable will
+  Here, adding `use_resource=True` when constructing the variable will
   fix any nondeterminism issues:
 
   ```
@@ -1239,7 +1239,7 @@ class VariableV1(Variable):
   tf.cond(v, lambda: v.assign(False), my_false_fn)  # Note: this is broken.
   ```
 
-  Here replacing adding `use_resource=True` when constructing the variable will
+  Here, adding `use_resource=True` when constructing the variable will
   fix any nondeterminism issues:
   ```
   v = tf.Variable(True, use_resource=True)
-- 
GitLab


From 94ea59c903e00a0ffde07d541aea4a1aee3c563d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 10:30:50 -0800
Subject: [PATCH 1060/2345] Automated rollback of commit
 80f47caee4bca375b8285e3b3185c7d29113b9e8

PiperOrigin-RevId: 230365804
---
 tensorflow/compiler/xla/executable_run_options.h | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/tensorflow/compiler/xla/executable_run_options.h b/tensorflow/compiler/xla/executable_run_options.h
index ba3217f31b..6f36d11dfb 100644
--- a/tensorflow/compiler/xla/executable_run_options.h
+++ b/tensorflow/compiler/xla/executable_run_options.h
@@ -16,9 +16,6 @@ limitations under the License.
 #ifndef TENSORFLOW_COMPILER_XLA_EXECUTABLE_RUN_OPTIONS_H_
 #define TENSORFLOW_COMPILER_XLA_EXECUTABLE_RUN_OPTIONS_H_
 
-// Pulls in the ::stream_executor -> ::xla::se namespace alias.
-#include "tensorflow/compiler/xla/types.h"
-
 // These classes are forward declared so that ExecutableRunOptions can be linked
 // into an XLA-compiled binary without having to link all of the pointed-to
 // objects (e.g., for an ahead-of-time compiled CPU binary, the gpu tools don't
@@ -28,12 +25,6 @@ class Stream;
 class Platform;
 }  // namespace stream_executor
 
-namespace tensorflow {
-namespace thread {
-class ThreadPool;
-}  // namespace thread
-}  // namespace tensorflow
-
 namespace Eigen {
 struct ThreadPoolDevice;
 }  // namespace Eigen
-- 
GitLab


From 13ab1b065de437efc3c1ba1d7c5c5bfe3edb7f99 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 10:43:22 -0800
Subject: [PATCH 1061/2345] Fixed typo; see also changes to
 tag_python/saved_model/tag_constants.py and
 tools/api/golden/v2/tensorflow.saved_model.pbtxt

PiperOrigin-RevId: 230368577
---
 tensorflow/tools/compatibility/renames_v2.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 9d8d25016a..9c02c20f0f 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -497,7 +497,7 @@ renames = {
     'tf.saved_model.Builder': 'tf.compat.v1.saved_model.Builder',
     'tf.saved_model.LEGACY_INIT_OP_KEY': 'tf.compat.v1.saved_model.LEGACY_INIT_OP_KEY',
     'tf.saved_model.MAIN_OP_KEY': 'tf.compat.v1.saved_model.MAIN_OP_KEY',
-    'tf.saved_model.TRAINING': 'tf.saved_model.TRANING',
+    'tf.saved_model.TRAINING': 'tf.saved_model.TRAINING',
     'tf.saved_model.build_tensor_info': 'tf.compat.v1.saved_model.build_tensor_info',
     'tf.saved_model.builder.SavedModelBuilder': 'tf.compat.v1.saved_model.builder.SavedModelBuilder',
     'tf.saved_model.constants.ASSETS_DIRECTORY': 'tf.saved_model.ASSETS_DIRECTORY',
@@ -538,7 +538,7 @@ renames = {
     'tf.saved_model.tag_constants.GPU': 'tf.saved_model.GPU',
     'tf.saved_model.tag_constants.SERVING': 'tf.saved_model.SERVING',
     'tf.saved_model.tag_constants.TPU': 'tf.saved_model.TPU',
-    'tf.saved_model.tag_constants.TRAINING': 'tf.saved_model.TRANING',
+    'tf.saved_model.tag_constants.TRAINING': 'tf.saved_model.TRAINING',
     'tf.saved_model.utils.build_tensor_info': 'tf.compat.v1.saved_model.utils.build_tensor_info',
     'tf.saved_model.utils.get_tensor_from_tensor_info': 'tf.compat.v1.saved_model.utils.get_tensor_from_tensor_info',
     'tf.scatter_add': 'tf.compat.v1.scatter_add',
-- 
GitLab


From 73cfd7b0c3f88deaf0a5e84ec84206dbcdd98e5e Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Tue, 22 Jan 2019 10:56:24 -0800
Subject: [PATCH 1062/2345] [XLA] Fix zero sized HLO elimination to handle
 instructions without layouts.

The zero sized HLO elimination pass is run before layout assignment and may
encounter instructions without layouts. Fix the pass to use a default layout
to create the resulting literal when the instruction doesn't have a layout.

Add a test case.

PiperOrigin-RevId: 230371248
---
 .../xla/service/zero_sized_hlo_elimination.cc       | 10 ++++++++--
 .../xla/service/zero_sized_hlo_elimination_test.cc  | 13 +++++++++++++
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/zero_sized_hlo_elimination.cc b/tensorflow/compiler/xla/service/zero_sized_hlo_elimination.cc
index a76fbf3f66..661b7aa7d9 100644
--- a/tensorflow/compiler/xla/service/zero_sized_hlo_elimination.cc
+++ b/tensorflow/compiler/xla/service/zero_sized_hlo_elimination.cc
@@ -37,9 +37,15 @@ StatusOr<bool> ZeroSizedHloElimination::Run(HloModule* module) {
       }
       if (comp->IsRemovable(instruction) &&
           ShapeUtil::IsZeroElementArray(instruction->shape())) {
+        // If the instruction doesn't have a layout, use a default layout for
+        // the literal.
+        Shape shape = instruction->shape();
+        if (!LayoutUtil::HasLayout(shape)) {
+          LayoutUtil::SetToDefaultLayout(&shape);
+        }
         TF_RETURN_IF_ERROR(comp->ReplaceWithNewInstruction(
-            instruction, HloInstruction::CreateConstant(
-                             Literal::CreateFromShape(instruction->shape()))));
+            instruction,
+            HloInstruction::CreateConstant(Literal::CreateFromShape(shape))));
         changed = true;
       }
     }
diff --git a/tensorflow/compiler/xla/service/zero_sized_hlo_elimination_test.cc b/tensorflow/compiler/xla/service/zero_sized_hlo_elimination_test.cc
index a546a6d39c..572a79609e 100644
--- a/tensorflow/compiler/xla/service/zero_sized_hlo_elimination_test.cc
+++ b/tensorflow/compiler/xla/service/zero_sized_hlo_elimination_test.cc
@@ -82,5 +82,18 @@ TEST_F(ZeroSizedHloEliminationTest, DoesNotEliminateConstant) {
   EXPECT_FALSE(changed);
 }
 
+TEST_F(ZeroSizedHloEliminationTest, ZeroSizedInstructionWithoutLayoutFolded) {
+  Shape op_shape = ShapeUtil::MakeShape(F32, {4, 0});
+  op_shape.clear_layout();
+  HloInstruction* param1 = builder_.AddInstruction(
+      HloInstruction::CreateParameter(1, op_shape, "zero sized param 1"));
+  HloInstruction* param2 = builder_.AddInstruction(
+      HloInstruction::CreateParameter(2, op_shape, "zero sized param 2"));
+  builder_.AddInstruction(
+      HloInstruction::CreateBinary(op_shape, HloOpcode::kAdd, param1, param2));
+  TF_ASSERT_OK_AND_ASSIGN(bool changed, RunZeroSizedElimination());
+  EXPECT_TRUE(changed);
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From e1e77eacb92e6bfcfc3f554dcdfed2de59798d41 Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Tue, 22 Jan 2019 11:03:20 -0800
Subject: [PATCH 1063/2345] [Grappler] Improve error messages in
 MutableGraphView to make them easily distinguishable from one another.

PiperOrigin-RevId: 230372780
---
 .../core/grappler/mutable_graph_view.cc       | 299 ++++++++++--------
 tensorflow/core/grappler/mutable_graph_view.h |   7 +-
 .../core/grappler/mutable_graph_view_test.cc  | 205 ++++++++----
 3 files changed, 314 insertions(+), 197 deletions(-)

diff --git a/tensorflow/core/grappler/mutable_graph_view.cc b/tensorflow/core/grappler/mutable_graph_view.cc
index 0f171f7656..dc9be9ebe7 100644
--- a/tensorflow/core/grappler/mutable_graph_view.cc
+++ b/tensorflow/core/grappler/mutable_graph_view.cc
@@ -38,26 +38,6 @@ namespace grappler {
 
 namespace {
 
-const char kMissingMsg[] = "missing";
-const char kInvalidMsg[] = "invalid";
-const char kNoErrMsg[] = "";
-
-string FaninError(bool tensor_id_valid, bool node_missing) {
-  string s;
-  if (!tensor_id_valid && node_missing) {
-    s = absl::StrCat(" ", kInvalidMsg, "/", kMissingMsg);
-  } else if (!tensor_id_valid) {
-    s = absl::StrCat(" ", kInvalidMsg);
-  } else if (node_missing) {
-    s = absl::StrCat(" ", kMissingMsg);
-  }
-  return s;
-}
-
-string NodeError(bool node_missing) {
-  return node_missing ? absl::StrCat(" ", kMissingMsg) : kNoErrMsg;
-}
-
 bool IsTensorIdPortValid(const TensorId& tensor_id) {
   return tensor_id.index() >= Graph::kControlSlot;
 }
@@ -114,6 +94,32 @@ bool CanDedupControlWithRegularInput(const MutableGraphView& graph,
   return CanDedupControlWithRegularInput(graph, *control_node);
 }
 
+Status MutationError(absl::string_view function_name, absl::string_view params,
+                     absl::string_view msg) {
+  return errors::InvalidArgument(absl::Substitute(
+      "MutableGraphView::$0($1) error: $2.", function_name, params, msg));
+}
+
+string NodeMissingErrorMsg(absl::string_view node_name) {
+  return absl::Substitute("node '$0' was not found", node_name);
+}
+
+string InvalidFaninErrorMsg(absl::string_view fanin) {
+  return absl::Substitute("fanin '$0' must be a valid tensor id", fanin);
+}
+
+string RegularFaninErrorMsg(absl::string_view fanin) {
+  return absl::Substitute("fanin '$0' must be a regular tensor id", fanin);
+}
+
+Status UpdateFanoutsError(absl::string_view from_node_name,
+                          absl::string_view to_node_name,
+                          absl::string_view msg) {
+  string params = absl::Substitute("from_node_name='$0', to_node_name='$1'",
+                                   from_node_name, to_node_name);
+  return MutationError("UpdateFanouts", params, msg);
+}
+
 }  // namespace
 
 void MutableGraphView::AddAndDedupFanouts(NodeDef* node) {
@@ -217,8 +223,20 @@ NodeDef* MutableGraphView::AddNode(NodeDef&& node) {
 
 Status MutableGraphView::AddSubgraph(GraphDef&& subgraph) {
   if (subgraph.library().function_size() != 0) {
-    return errors::InvalidArgument(
-        "Can't add a subgraph with non-empty function library");
+    constexpr int kMaxNumNodesForDebugStr = 100;
+    string subgraph_str = "too large to display";
+    if (subgraph.node_size() < kMaxNumNodesForDebugStr) {
+      constexpr int kMaxLenSubgraphStr = 1000;
+      subgraph_str = subgraph.ShortDebugString();
+      if (subgraph_str.length() > kMaxLenSubgraphStr) {
+        subgraph_str =
+            absl::StrCat(subgraph_str.substr(0, kMaxLenSubgraphStr), "...");
+      }
+    }
+    string params = absl::Substitute("subgraph='$0'", subgraph_str);
+    return MutationError(
+        "AddSubgraph", params,
+        "can't add a subgraph with non-empty function library");
   }
 
   int node_size_before = graph()->node_size();
@@ -240,27 +258,20 @@ Status MutableGraphView::AddSubgraph(GraphDef&& subgraph) {
   return Status::OK();
 }
 
-Status MutableGraphView::UpdateFanouts(absl::string_view from_node,
-                                       absl::string_view to_node) {
-  NodeDef* from_node_ptr = GetNode(from_node);
-  NodeDef* to_node_ptr = GetNode(to_node);
-  if (from_node_ptr && to_node_ptr) {
-    return UpdateFanoutsInternal(from_node_ptr, to_node_ptr);
-  } else if (!from_node_ptr) {
-    return errors::Internal(absl::Substitute(
-        "Can't update fanouts from '$0' to '$1', from node was not found.",
-        from_node, to_node));
-  } else if (!to_node_ptr) {
-    return errors::Internal(absl::Substitute(
-        "Can't update fanouts from '$0' to '$1', to node was not found.",
-        from_node, to_node));
-  } else {
-    return errors::Internal(
-        absl::Substitute("Can't update fanouts from '$0' to '$1', from and to "
-                         "nodes were not found.",
-                         from_node, to_node));
+Status MutableGraphView::UpdateFanouts(absl::string_view from_node_name,
+                                       absl::string_view to_node_name) {
+  NodeDef* from_node = GetNode(from_node_name);
+  if (from_node == nullptr) {
+    return UpdateFanoutsError(from_node_name, to_node_name,
+                              NodeMissingErrorMsg(from_node_name));
   }
-  return Status::OK();
+  NodeDef* to_node = GetNode(to_node_name);
+  if (to_node == nullptr) {
+    return UpdateFanoutsError(from_node_name, to_node_name,
+                              NodeMissingErrorMsg(to_node_name));
+  }
+
+  return UpdateFanoutsInternal(from_node, to_node);
 }
 
 Status MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
@@ -298,10 +309,11 @@ Status MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
     if (to_node_is_switch) {
       // Trying to add a Switch as a control dependency, which if allowed will
       // make the graph invalid.
-      return errors::Internal(
-          absl::Substitute("Can't update fanouts from '$0' to '$1', to node is "
-                           "being added as a Switch control dependency.",
-                           from_node->name(), to_node->name()));
+      return UpdateFanoutsError(
+          from_node->name(), to_node->name(),
+          absl::Substitute("can't update fanouts to node '$0' as it will "
+                           "become a Switch control dependency",
+                           to_node->name()));
     }
 
     NodeDef* node = control_port.node;
@@ -410,20 +422,26 @@ bool MutableGraphView::AddFaninInternal(NodeDef* node,
 
 Status MutableGraphView::AddRegularFanin(absl::string_view node_name,
                                          const TensorId& fanin) {
-  NodeDef* node = GetNode(node_name);
-  NodeDef* fanin_node = GetNode(fanin.node());
+  auto error_status = [node_name, fanin](absl::string_view msg) {
+    string params = absl::Substitute("node_name='$0', fanin='$1'", node_name,
+                                     fanin.ToString());
+    return MutationError("AddRegularFanin", params, msg);
+  };
 
-  string node_err = NodeError(/*node_missing=*/node == nullptr);
-  string fanin_err = FaninError(IsTensorIdRegular(fanin),
-                                /*node_missing=*/fanin_node == nullptr);
-  if (!node_err.empty() || !fanin_err.empty()) {
-    return errors::Internal(absl::Substitute(
-        "Can't add$0 fanin '$1' as regular fanin to$2 node '$3'.", fanin_err,
-        fanin.ToString(), node_err, node_name));
+  if (!IsTensorIdRegular(fanin)) {
+    return error_status(RegularFaninErrorMsg(fanin.ToString()));
   }
   if (node_name == fanin.node()) {
-    return errors::Internal(absl::Substitute(
-        "Can't add fanin '$0' as regular fanin to self.", fanin.ToString()));
+    return error_status(absl::Substitute("can't add regular fanin '$0' to self",
+                                         fanin.ToString()));
+  }
+  NodeDef* node = GetNode(node_name);
+  if (node == nullptr) {
+    return error_status(NodeMissingErrorMsg(node_name));
+  }
+  NodeDef* fanin_node = GetNode(fanin.node());
+  if (fanin_node == nullptr) {
+    return error_status(NodeMissingErrorMsg(fanin.node()));
   }
 
   AddFaninInternal(node, {fanin_node, fanin.index()});
@@ -432,20 +450,26 @@ Status MutableGraphView::AddRegularFanin(absl::string_view node_name,
 
 Status MutableGraphView::AddControllingFanin(absl::string_view node_name,
                                              const TensorId& fanin) {
-  NodeDef* node = GetNode(node_name);
-  NodeDef* fanin_node = GetNode(fanin.node());
+  auto error_status = [node_name, fanin](absl::string_view msg) {
+    string params = absl::Substitute("node_name='$0', fanin='$1'", node_name,
+                                     fanin.ToString());
+    return MutationError("AddControllingFanin", params, msg);
+  };
 
-  string node_err = NodeError(/*node_missing=*/node == nullptr);
-  string fanin_err = FaninError(IsTensorIdPortValid(fanin),
-                                /*node_missing=*/fanin_node == nullptr);
-  if (!node_err.empty() || !fanin_err.empty()) {
-    return errors::Internal(
-        absl::Substitute("Can't add$0 controlling fanin '$1' to$2 node '$3'.",
-                         fanin_err, fanin.ToString(), node_err, node_name));
+  if (!IsTensorIdPortValid(fanin)) {
+    return error_status(InvalidFaninErrorMsg(fanin.ToString()));
   }
   if (node_name == fanin.node()) {
-    return errors::Internal(absl::Substitute(
-        "Can't add controlling fanin '$0' to self.", fanin.ToString()));
+    return error_status(absl::Substitute(
+        "can't add controlling fanin '$0' to self", fanin.ToString()));
+  }
+  NodeDef* node = GetNode(node_name);
+  if (node == nullptr) {
+    return error_status(NodeMissingErrorMsg(node_name));
+  }
+  NodeDef* fanin_node = GetNode(fanin.node());
+  if (fanin_node == nullptr) {
+    return error_status(NodeMissingErrorMsg(fanin.node()));
   }
 
   if (!IsSwitch(*fanin_node)) {
@@ -453,9 +477,10 @@ Status MutableGraphView::AddControllingFanin(absl::string_view node_name,
   } else {
     if (IsTensorIdControlling(fanin)) {
       // Can't add a Switch node control dependency.
-      return errors::Internal(absl::Substitute(
-          "Can't add Switch as controlling fanin '$0' to node '$1'.",
-          fanin.ToString(), node_name));
+      return error_status(
+          absl::Substitute("can't add controlling fanin '$0' as it will become "
+                           "a Switch control dependency",
+                           fanin.ToString()));
     }
     // We can't anchor control dependencies directly on the switch node: unlike
     // other nodes only one of the outputs of the switch node will be generated
@@ -468,10 +493,9 @@ Status MutableGraphView::AddControllingFanin(absl::string_view node_name,
       if (IsIdentity(*fanout.node) || IsIdentityNSingleInput(*fanout.node)) {
         if (ParseTensorName(fanout.node->input(0)) == fanin) {
           if (fanout.node->name() == node_name) {
-            return errors::Internal(absl::Substitute(
-                "Can't add found controlling fanin '$0' from fanin '$1' to "
-                "self.",
-                AsControlDependency(fanout.node->name()), fanin.ToString()));
+            return error_status(absl::Substitute(
+                "can't add found controlling fanin '$0' to self",
+                AsControlDependency(fanout.node->name())));
           }
           AddFaninInternal(node, {fanout.node, Graph::kControlSlot});
           return Status::OK();
@@ -483,9 +507,9 @@ Status MutableGraphView::AddControllingFanin(absl::string_view node_name,
     string ctrl_dep_name = AddPrefixToNodeName(
         absl::StrCat(fanin.node(), "_", fanin.index()), kMutableGraphViewCtrl);
     if (node_name == ctrl_dep_name) {
-      return errors::Internal(absl::Substitute(
-          "Can't add generated controlling fanin '$0' from fanin '$1' to self.",
-          AsControlDependency(ctrl_dep_name), fanin.ToString()));
+      return error_status(
+          absl::Substitute("can't add generated controlling fanin '$0' to self",
+                           AsControlDependency(ctrl_dep_name)));
     }
 
     // Reuse a previously created node, if possible.
@@ -563,21 +587,26 @@ bool MutableGraphView::RemoveRegularFaninInternal(NodeDef* node,
 
 Status MutableGraphView::RemoveRegularFanin(absl::string_view node_name,
                                             const TensorId& fanin) {
-  NodeDef* node = GetNode(node_name);
-  NodeDef* fanin_node = GetNode(fanin.node());
+  auto error_status = [node_name, fanin](absl::string_view msg) {
+    string params = absl::Substitute("node_name='$0', fanin='$1'", node_name,
+                                     fanin.ToString());
+    return MutationError("RemoveRegularFanin", params, msg);
+  };
 
-  string node_err = NodeError(/*node_missing=*/node == nullptr);
-  string fanin_err = FaninError(IsTensorIdRegular(fanin),
-                                /*node_missing=*/fanin_node == nullptr);
-  if (!node_err.empty() || !fanin_err.empty()) {
-    return errors::Internal(absl::Substitute(
-        "Can't remove$0 fanin '$1' as regular fanin from$2 node '$3'.",
-        fanin_err, fanin.ToString(), node_err, node_name));
+  if (!IsTensorIdRegular(fanin)) {
+    return error_status(RegularFaninErrorMsg(fanin.ToString()));
   }
   if (node_name == fanin.node()) {
-    return errors::Internal(
-        absl::Substitute("Can't remove fanin '$0' as regular fanin from self.",
-                         fanin.ToString()));
+    return error_status(absl::Substitute(
+        "can't remove regular fanin '$0' from self", fanin.ToString()));
+  }
+  NodeDef* node = GetNode(node_name);
+  if (node == nullptr) {
+    return error_status(NodeMissingErrorMsg(node_name));
+  }
+  NodeDef* fanin_node = GetNode(fanin.node());
+  if (fanin_node == nullptr) {
+    return error_status(NodeMissingErrorMsg(fanin.node()));
   }
 
   RemoveRegularFaninInternal(node, {fanin_node, fanin.index()});
@@ -604,21 +633,25 @@ bool MutableGraphView::RemoveControllingFaninInternal(NodeDef* node,
 
 Status MutableGraphView::RemoveControllingFanin(
     absl::string_view node_name, absl::string_view fanin_node_name) {
-  NodeDef* node = GetNode(node_name);
-  NodeDef* fanin_node = GetNode(fanin_node_name);
+  auto error_status = [node_name, fanin_node_name](absl::string_view msg) {
+    string params = absl::Substitute("node_name='$0', fanin_node_name='$1'",
+                                     node_name, fanin_node_name);
+    return MutationError("RemoveControllingFanin", params, msg);
+  };
 
-  string node_err = NodeError(/*node_missing=*/node == nullptr);
-  string fanin_err = NodeError(/*node_missing=*/fanin_node == nullptr);
-  if (!node_err.empty() || !fanin_err.empty()) {
-    return errors::Internal(absl::Substitute(
-        "Can't remove$0 controlling fanin '$1' from$2 node '$3'.", fanin_err,
-        AsControlDependency(string(fanin_node_name)), node_err, node_name));
-  }
   if (node_name == fanin_node_name) {
-    return errors::Internal(
-        absl::Substitute("Can't remove controlling fanin '$0' from self.",
+    return error_status(
+        absl::Substitute("can't remove controlling fanin '$0' from self",
                          AsControlDependency(string(fanin_node_name))));
   }
+  NodeDef* node = GetNode(node_name);
+  if (node == nullptr) {
+    return error_status(NodeMissingErrorMsg(node_name));
+  }
+  NodeDef* fanin_node = GetNode(fanin_node_name);
+  if (fanin_node == nullptr) {
+    return error_status(NodeMissingErrorMsg(fanin_node_name));
+  }
 
   RemoveControllingFaninInternal(node, fanin_node);
   return Status::OK();
@@ -629,8 +662,11 @@ Status MutableGraphView::RemoveAllFanins(absl::string_view node_name,
   NodeDef* node = GetNode(node_name);
 
   if (node == nullptr) {
-    return errors::Internal(absl::Substitute(
-        "Can't remove all fanins from missing node '$0'.", node_name));
+    string params =
+        absl::Substitute("node_name='$0', keep_controlling_fanins=$1",
+                         node_name, keep_controlling_fanins);
+    return MutationError("RemoveAllFanins", params,
+                         NodeMissingErrorMsg(node_name));
   }
 
   if (node->input().empty()) {
@@ -657,21 +693,30 @@ Status MutableGraphView::RemoveAllFanins(absl::string_view node_name,
 Status MutableGraphView::UpdateFanin(absl::string_view node_name,
                                      const TensorId& from_fanin,
                                      const TensorId& to_fanin) {
+  auto error_status = [node_name, from_fanin, to_fanin](absl::string_view msg) {
+    string params =
+        absl::Substitute("node_name='$0', from_fanin='$1', to_fanin='$2'",
+                         node_name, from_fanin.ToString(), to_fanin.ToString());
+    return MutationError("UpdateFanin", params, msg);
+  };
+
+  if (!IsTensorIdPortValid(from_fanin)) {
+    return error_status(InvalidFaninErrorMsg(from_fanin.ToString()));
+  }
+  if (!IsTensorIdPortValid(to_fanin)) {
+    return error_status(InvalidFaninErrorMsg(to_fanin.ToString()));
+  }
   NodeDef* node = GetNode(node_name);
+  if (node == nullptr) {
+    return error_status(NodeMissingErrorMsg(node_name));
+  }
   NodeDef* from_fanin_node = GetNode(from_fanin.node());
+  if (from_fanin_node == nullptr) {
+    return error_status(NodeMissingErrorMsg(from_fanin.node()));
+  }
   NodeDef* to_fanin_node = GetNode(to_fanin.node());
-
-  string node_err = NodeError(/*node_missing=*/node == nullptr);
-  string from_fanin_err =
-      FaninError(IsTensorIdPortValid(from_fanin),
-                 /*node_missing=*/from_fanin_node == nullptr);
-  string to_fanin_err = FaninError(IsTensorIdPortValid(to_fanin),
-                                   /*node_missing=*/to_fanin_node == nullptr);
-  if (!node_err.empty() || !from_fanin_err.empty() || !to_fanin_err.empty()) {
-    return errors::Internal(absl::Substitute(
-        "Can't update$0 fanin '$1' to$2 fanin '$3' in$4 node '$5'.",
-        from_fanin_err, from_fanin.ToString(), to_fanin_err,
-        to_fanin.ToString(), node_err, node_name));
+  if (to_fanin_node == nullptr) {
+    return error_status(NodeMissingErrorMsg(from_fanin.node()));
   }
 
   // When replacing a non control dependency fanin with a control dependency, or
@@ -679,15 +724,13 @@ Status MutableGraphView::UpdateFanin(absl::string_view node_name,
   bool to_fanin_is_control = IsTensorIdControlling(to_fanin);
   if (to_fanin_is_control && IsSwitch(*to_fanin_node)) {
     // Can't add Switch node as a control dependency.
-    return errors::Internal(absl::Substitute(
-        "Can't update fanin '$0' to fanin '$1' in node '$2', to fanin is a "
-        "Switch control dependency.",
-        from_fanin.ToString(), to_fanin.ToString(), node_name));
+    return error_status(
+        absl::Substitute("can't update to fanin '$0' as it will become a "
+                         "Switch control dependency",
+                         to_fanin.ToString()));
   }
   if (node_name == from_fanin.node() || node_name == to_fanin.node()) {
-    return errors::Internal(absl::Substitute(
-        "Can't update fanin '$0' to fanin '$1' in self '$2'.",
-        from_fanin.ToString(), to_fanin.ToString(), node_name));
+    return error_status("can't update fanin to or from self");
   }
 
   if (from_fanin == to_fanin) {
@@ -797,16 +840,18 @@ Status MutableGraphView::CheckNodesCanBeDeleted(
   };
 
   if (!missing_nodes.empty()) {
-    VLOG(1) << absl::Substitute("Attempting to delete missing node(s) [$0]",
+    VLOG(2) << absl::Substitute("Attempting to delete missing node(s) [$0].",
                                 sort_and_sample(&missing_nodes));
   }
   if (!nodes_with_fanouts.empty()) {
     std::vector<string> input_node_names(nodes_to_delete.begin(),
                                          nodes_to_delete.end());
-    return errors::Internal(absl::Substitute(
-        "Can't delete node(s) with retained fanout(s) [$0] from node(s) [$1].",
-        sort_and_sample(&nodes_with_fanouts),
-        sort_and_sample(&input_node_names)));
+    string params = absl::Substitute("nodes_to_delete={$0}",
+                                     sort_and_sample(&input_node_names));
+    string error_msg =
+        absl::Substitute("can't delete node(s) with retained fanouts(s) [$0]",
+                         sort_and_sample(&nodes_with_fanouts));
+    return MutationError("DeleteNodes", params, error_msg);
   }
 
   return Status::OK();
diff --git a/tensorflow/core/grappler/mutable_graph_view.h b/tensorflow/core/grappler/mutable_graph_view.h
index f08b5dde08..73c32f4e55 100644
--- a/tensorflow/core/grappler/mutable_graph_view.h
+++ b/tensorflow/core/grappler/mutable_graph_view.h
@@ -73,8 +73,8 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // library into the underlying graph.
   Status AddSubgraph(GraphDef&& subgraph);
 
-  // Updates all fanouts (input ports fetching output tensors) from `from_node`
-  // to the `to_node`, including control dependencies.
+  // Updates all fanouts (input ports fetching output tensors) from
+  // `from_node_name` to the `to_node_name`, including control dependencies.
   //
   // Example: We have 3 nodes that use `bar` node output tensors as inputs:
   //   1. foo1(bar:0, bar:1, other:0)
@@ -85,7 +85,8 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   //   1. foo1(new_bar:0, new_bar:1, other:0)
   //   2. foo2(new_bar:1, other:1)
   //   3. foo3(other:2, ^new_bar)
-  Status UpdateFanouts(absl::string_view from_node, absl::string_view to_node);
+  Status UpdateFanouts(absl::string_view from_node_name,
+                       absl::string_view to_node_name);
 
   // Adds regular fanin `fanin` to node `node_name`. If the node or fanin do not
   // exist in the graph, nothing will be modified in the graph. Otherwise fanin
diff --git a/tensorflow/core/grappler/mutable_graph_view_test.cc b/tensorflow/core/grappler/mutable_graph_view_test.cc
index ea88359e2e..8b94364d54 100644
--- a/tensorflow/core/grappler/mutable_graph_view_test.cc
+++ b/tensorflow/core/grappler/mutable_graph_view_test.cc
@@ -189,8 +189,14 @@ TEST(MutableGraphViewTest, AddSubgraphWithFunctionLibrary) {
       },
       /*funcs=*/{x_times_two});
 
+  string subgraph_str = subgraph.ShortDebugString();
   Status status = graph.AddSubgraph(std::move(subgraph));
   EXPECT_FALSE(status.ok());
+  string expected_msg = absl::Substitute(
+      "MutableGraphView::AddSubgraph(subgraph='$0') error: can't add a "
+      "subgraph with non-empty function library.",
+      subgraph_str);
+  EXPECT_EQ(status.error_message(), expected_msg);
 }
 
 TEST(MutableGraphViewTest, AddAndUpdateFanouts) {
@@ -289,14 +295,16 @@ TEST(MutableGraphViewTest, UpdateFanoutsToSwitchWithControlFromSwitch) {
   Status s = graph.UpdateFanouts("a", "b");
   EXPECT_FALSE(s.ok());
   string expected_msg =
-      "Can't update fanouts from 'a' to 'b', to node is being added as a "
-      "Switch control dependency.";
+      "MutableGraphView::UpdateFanouts(from_node_name='a', to_node_name='b') "
+      "error: can't update fanouts to node 'b' as it will become a Switch "
+      "control dependency.";
   EXPECT_EQ(s.error_message(), expected_msg);
   s = graph.UpdateFanouts("d", "b");
   EXPECT_FALSE(s.ok());
   expected_msg =
-      "Can't update fanouts from 'd' to 'b', to node is being added as a "
-      "Switch control dependency.";
+      "MutableGraphView::UpdateFanouts(from_node_name='d', to_node_name='b') "
+      "error: can't update fanouts to node 'b' as it will become a Switch "
+      "control dependency.";
   EXPECT_EQ(s.error_message(), expected_msg);
 
   EXPECT_EQ(graph.graph()->node_size(), 5);
@@ -431,62 +439,80 @@ TEST(MutableGraphViewTest, AddRegularFanin) {
                       error_msg, {"c:1", "^b", "^a"});
 
   // Add control to node with 1 input 0 controls.
-  error_msg = "Can't add invalid fanin '^b' as regular fanin to node 'foo_1'.";
+  error_msg =
+      "MutableGraphView::AddRegularFanin(node_name='foo_1', fanin='^b') error: "
+      "fanin '^b' must be a regular tensor id.";
   TestAddRegularFanin("foo_1", /*node_exists=*/true, {"b", Graph::kControlSlot},
                       /*success=*/false, error_msg, {"a"});
   // Add control to node with multiple inputs and 0 controls.
-  error_msg = "Can't add invalid fanin '^c' as regular fanin to node 'foo_3'.";
+  error_msg =
+      "MutableGraphView::AddRegularFanin(node_name='foo_3', fanin='^c') error: "
+      "fanin '^c' must be a regular tensor id.";
   TestAddRegularFanin("foo_3", /*node_exists=*/true, {"c", Graph::kControlSlot},
                       /*success=*/false, error_msg, {"b", "a:1", "a:1"});
   // Add control to node with 1 input multiple controls.
-  error_msg = "Can't add invalid fanin '^d' as regular fanin to node 'foo_2'.";
+  error_msg =
+      "MutableGraphView::AddRegularFanin(node_name='foo_2', fanin='^d') error: "
+      "fanin '^d' must be a regular tensor id.";
   TestAddRegularFanin("foo_2", /*node_exists=*/true, {"d", Graph::kControlSlot},
                       /*success=*/false, error_msg, {"b", "^a", "^c"});
   // Add control to node with multiple input multiple controls.
-  error_msg = "Can't add invalid fanin '^a' as regular fanin to node 'foo_4'.";
+  error_msg =
+      "MutableGraphView::AddRegularFanin(node_name='foo_4', fanin='^a') error: "
+      "fanin '^a' must be a regular tensor id.";
   TestAddRegularFanin("foo_4", /*node_exists=*/true, {"a", Graph::kControlSlot},
                       /*success=*/false, error_msg,
                       {"a", "b:2", "b:2", "^c", "^d"});
   // Add control to node with 0 inputs 0 controls.
-  error_msg = "Can't add invalid fanin '^a' as regular fanin to node 'foo_5'.";
+  error_msg =
+      "MutableGraphView::AddRegularFanin(node_name='foo_5', fanin='^a') error: "
+      "fanin '^a' must be a regular tensor id.";
   TestAddRegularFanin("foo_5", /*node_exists=*/true, {"a", Graph::kControlSlot},
                       /*success=*/false, error_msg, {});
   // Add control to node with 0 inputs multiple controls.
-  error_msg = "Can't add invalid fanin '^c' as regular fanin to node 'foo_6'.";
+  error_msg =
+      "MutableGraphView::AddRegularFanin(node_name='foo_6', fanin='^c') error: "
+      "fanin '^c' must be a regular tensor id.";
   TestAddRegularFanin("foo_6", /*node_exists=*/true, {"c", Graph::kControlSlot},
                       /*success=*/false, error_msg, {"^a", "^b"});
   // Add control to node with control that already exists.
-  error_msg = "Can't add invalid fanin '^a' as regular fanin to node 'foo_2'.";
+  error_msg =
+      "MutableGraphView::AddRegularFanin(node_name='foo_2', fanin='^a') error: "
+      "fanin '^a' must be a regular tensor id.";
   TestAddRegularFanin("foo_2", /*node_exists=*/true, {"a", Graph::kControlSlot},
                       /*success=*/false, error_msg, {"b", "^a", "^c"});
 
   // Add fanin to node where node is missing.
   error_msg =
-      "Can't add fanin 'a:0' as regular fanin to missing node 'foo_missing'.";
+      "MutableGraphView::AddRegularFanin(node_name='foo_missing', fanin='a:0') "
+      "error: node 'foo_missing' was not found.";
   TestAddRegularFanin("foo_missing", /*node_exists=*/false, {"a", 0},
                       /*success=*/false, error_msg, {});
   // Add fanin to node where fanin is missing.
   error_msg =
-      "Can't add missing fanin 'bar_missing:0' as regular fanin to node "
-      "'foo_1'.";
+      "MutableGraphView::AddRegularFanin(node_name='foo_1', "
+      "fanin='bar_missing:0') error: node 'bar_missing' was not found.";
   TestAddRegularFanin("foo_1", /*node_exists=*/true, {"bar_missing", 0},
                       /*success=*/false, error_msg, {"a"});
   // Add fanin to node where node and fanin are missing.
   error_msg =
-      "Can't add missing fanin 'bar_missing:0' as regular fanin to missing "
-      "node 'foo_missing'.";
+      "MutableGraphView::AddRegularFanin(node_name='foo_missing', "
+      "fanin='bar_missing:0') error: node 'foo_missing' was not found.";
   TestAddRegularFanin("foo_missing", /*node_exists=*/false, {"bar_missing", 0},
                       /*success=*/false, error_msg, {});
   // Add control fanin to node where node and fanin are missing.
   error_msg =
-      "Can't add invalid/missing fanin '^bar_missing' as regular fanin to "
-      "missing node 'foo_missing'.";
+      "MutableGraphView::AddRegularFanin(node_name='foo_missing', "
+      "fanin='^bar_missing') error: fanin '^bar_missing' must be a regular "
+      "tensor id.";
   TestAddRegularFanin("foo_missing", /*node_exists=*/false,
                       {"bar_missing", Graph::kControlSlot},
                       /*success=*/false, error_msg, {});
 
   // Add self to create cycle.
-  error_msg = "Can't add fanin 'foo_6:2' as regular fanin to self.";
+  error_msg =
+      "MutableGraphView::AddRegularFanin(node_name='foo_6', fanin='foo_6:2') "
+      "error: can't add regular fanin 'foo_6:2' to self.";
   TestAddRegularFanin("foo_6", /*node_exists=*/true, {"foo_6", 2},
                       /*success=*/false, error_msg, {"^a", "^b"});
 }
@@ -558,19 +584,22 @@ TEST(MutableGraphViewTest, RemoveRegularFanin) {
 
   // Remove control from node with 1 input multiple controls.
   error_msg =
-      "Can't remove invalid fanin '^a' as regular fanin from node 'foo_2'.";
+      "MutableGraphView::RemoveRegularFanin(node_name='foo_2', fanin='^a') "
+      "error: fanin '^a' must be a regular tensor id.";
   TestRemoveRegularFanin("foo_2", /*node_exists=*/true,
                          {"a", Graph::kControlSlot},
                          /*success=*/false, error_msg, {"b", "^a", "^c"});
   // Remove control from node with multiple input multiple controls.
   error_msg =
-      "Can't remove invalid fanin '^d' as regular fanin from node 'foo_4'.";
+      "MutableGraphView::RemoveRegularFanin(node_name='foo_4', fanin='^d') "
+      "error: fanin '^d' must be a regular tensor id.";
   TestRemoveRegularFanin(
       "foo_4", /*node_exists=*/true, {"d", Graph::kControlSlot},
       /*success=*/false, error_msg, {"a", "b:2", "b:2", "^c", "^d"});
   // Remove control from node with 0 inputs multiple controls.
   error_msg =
-      "Can't remove invalid fanin '^a' as regular fanin from node 'foo_6'.";
+      "MutableGraphView::RemoveRegularFanin(node_name='foo_6', fanin='^a') "
+      "error: fanin '^a' must be a regular tensor id.";
   TestRemoveRegularFanin("foo_6", /*node_exists=*/true,
                          {"a", Graph::kControlSlot},
                          /*success=*/false, error_msg, {"^a", "^b"});
@@ -585,51 +614,57 @@ TEST(MutableGraphViewTest, RemoveRegularFanin) {
 
   // Remove control from node with 1 input 0 controls.
   error_msg =
-      "Can't remove invalid fanin '^b' as regular fanin from node 'foo_1'.";
+      "MutableGraphView::RemoveRegularFanin(node_name='foo_1', fanin='^b') "
+      "error: fanin '^b' must be a regular tensor id.";
   TestRemoveRegularFanin("foo_1", /*node_exists=*/true,
                          {"b", Graph::kControlSlot},
                          /*success=*/false, error_msg, {"a"});
   // Remove control from node with multiple inputs and 0 controls.
   error_msg =
-      "Can't remove invalid fanin '^c' as regular fanin from node 'foo_3'.";
+      "MutableGraphView::RemoveRegularFanin(node_name='foo_3', fanin='^c') "
+      "error: fanin '^c' must be a regular tensor id.";
   TestRemoveRegularFanin("foo_3", /*node_exists=*/true,
                          {"c", Graph::kControlSlot},
                          /*success=*/false, error_msg, {"b", "a:1", "a:1"});
   // Remove control from node with 0 inputs 0 controls.
   error_msg =
-      "Can't remove invalid fanin '^a' as regular fanin from node 'foo_5'.";
+      "MutableGraphView::RemoveRegularFanin(node_name='foo_5', fanin='^a') "
+      "error: fanin '^a' must be a regular tensor id.";
   TestRemoveRegularFanin("foo_5", /*node_exists=*/true,
                          {"a", Graph::kControlSlot},
                          /*success=*/false, error_msg, {});
 
   // Remove fanin from node where node is missing.
   error_msg =
-      "Can't remove fanin 'a:0' as regular fanin from missing node "
-      "'foo_missing'.";
+      "MutableGraphView::RemoveRegularFanin(node_name='foo_missing', "
+      "fanin='a:0') error: node 'foo_missing' was not found.";
   TestRemoveRegularFanin("foo_missing", /*node_exists=*/false, {"a", 0},
                          /*success=*/false, error_msg, {});
   // Remove fanin from node where fanin is missing.
   error_msg =
-      "Can't remove missing fanin 'bar_missing:0' as regular fanin from node "
-      "'foo_1'.";
+      "MutableGraphView::RemoveRegularFanin(node_name='foo_1', "
+      "fanin='bar_missing:0') error: node 'bar_missing' was not found.";
   TestRemoveRegularFanin("foo_1", /*node_exists=*/true, {"bar_missing", 0},
                          /*success=*/false, error_msg, {"a"});
   // Remove fanin from node where node and fanin are missing.
   error_msg =
-      "Can't remove missing fanin 'bar_missing:0' as regular fanin from "
-      "missing node 'foo_missing'.";
+      "MutableGraphView::RemoveRegularFanin(node_name='foo_missing', "
+      "fanin='bar_missing:0') error: node 'foo_missing' was not found.";
   TestRemoveRegularFanin("foo_missing", /*node_exists=*/false,
                          {"bar_missing", 0}, /*success=*/false, error_msg, {});
   // Remove control from node where node and fanin are missing.
   error_msg =
-      "Can't remove invalid/missing fanin '^bar_missing' as regular fanin from "
-      "missing node 'foo_missing'.";
+      "MutableGraphView::RemoveRegularFanin(node_name='foo_missing', "
+      "fanin='^bar_missing') error: fanin '^bar_missing' must be a regular "
+      "tensor id.";
   TestRemoveRegularFanin("foo_missing", /*node_exists=*/false,
                          {"bar_missing", Graph::kControlSlot},
                          /*success=*/false, error_msg, {});
 
   // Remove self.
-  error_msg = "Can't remove fanin 'foo_6:2' as regular fanin from self.";
+  error_msg =
+      "MutableGraphView::RemoveRegularFanin(node_name='foo_6', "
+      "fanin='foo_6:2') error: can't remove regular fanin 'foo_6:2' from self.";
   TestRemoveRegularFanin("foo_6", /*node_exists=*/true, {"foo_6", 2},
                          /*success=*/false, error_msg, {"^a", "^b"});
 }
@@ -718,10 +753,15 @@ TEST(MutableGraphViewTest, RemoveAllFanins) {
                       /*success=*/true, error_msg, {"^a", "^b"});
 
   // Remove all fanins from node where node is missing.
-  error_msg = "Can't remove all fanins from missing node 'foo_missing'.";
+  error_msg =
+      "MutableGraphView::RemoveAllFanins(node_name='foo_missing', "
+      "keep_controlling_fanins=false) error: node 'foo_missing' was not found.";
   TestRemoveAllFanins("foo_missing", /*node_exists=*/false,
                       /*keep_controlling_nodes=*/false,
                       /*success=*/false, error_msg, {});
+  error_msg =
+      "MutableGraphView::RemoveAllFanins(node_name='foo_missing', "
+      "keep_controlling_fanins=true) error: node 'foo_missing' was not found.";
   TestRemoveAllFanins("foo_missing", /*node_exists=*/false,
                       /*keep_controlling_nodes=*/true,
                       /*success=*/false, error_msg, {});
@@ -794,63 +834,77 @@ TEST(MutableGraphViewTest, UpdateFanin) {
 
   // Update fanin of node where node is missing.
   error_msg =
-      "Can't update fanin 'a:0' to fanin 'a:1' in missing node 'foo_missing'.";
+      "MutableGraphView::UpdateFanin(node_name='foo_missing', "
+      "from_fanin='a:0', to_fanin='a:1') error: node 'foo_missing' was not "
+      "found.";
   TestUpdateFanin("foo_missing", /*node_exists=*/false, {"a", 0}, {"a", 1},
                   /*success=*/false, error_msg, {});
   // Update fanin of node where from fanin is missing.
   error_msg =
-      "Can't update missing fanin 'from_bar_missing:0' to fanin 'a:1' in node "
-      "'foo_1'.";
+      "MutableGraphView::UpdateFanin(node_name='foo_1', "
+      "from_fanin='from_bar_missing:0', to_fanin='a:1') error: node "
+      "'from_bar_missing' was not found.";
   TestUpdateFanin("foo_1", /*node_exists=*/true, {"from_bar_missing", 0},
                   {"a", 1},
                   /*success=*/false, error_msg, {"a"});
   // Update fanin of node where to fanin is missing.
   error_msg =
-      "Can't update fanin 'a:0' to missing fanin 'to_bar_missing:1' in node "
-      "'foo_1'.";
+      "MutableGraphView::UpdateFanin(node_name='foo_1', from_fanin='a:0', "
+      "to_fanin='to_bar_missing:1') error: node 'a' was not found.";
   TestUpdateFanin("foo_1", /*node_exists=*/true, {"a", 0},
                   {"to_bar_missing", 1}, /*success=*/false, error_msg, {"a"});
   // Update fanin of node where from/to fanins and node are missing.
   error_msg =
-      "Can't update missing fanin 'from_bar_missing:0' to missing fanin "
-      "'to_bar_missing:1' in missing node 'foo_missing'.";
+      "MutableGraphView::UpdateFanin(node_name='foo_missing', "
+      "from_fanin='from_bar_missing:0', to_fanin='to_bar_missing:1') error: "
+      "node 'foo_missing' was not found.";
   TestUpdateFanin("foo_missing", /*node_exists=*/false, {"from_bar_missing", 0},
                   {"to_bar_missing", 1},
                   /*success=*/false, error_msg, {});
   // Update fanin of node where from fanin is invalid.
   error_msg =
-      "Can't update invalid fanin 'a:-2' to fanin 'a:0' in node 'foo_1'.";
+      "MutableGraphView::UpdateFanin(node_name='foo_1', from_fanin='a:-2', "
+      "to_fanin='a:0') error: fanin 'a:-2' must be a valid tensor id.";
   TestUpdateFanin("foo_1", /*node_exists=*/true, {"a", -2}, {"a", 0},
                   /*success=*/false, error_msg, {"a"});
   // Update fanin of node where to fanin is invalid.
   error_msg =
-      "Can't update fanin 'a:0' to invalid fanin 'a:-2' in node 'foo_1'.";
+      "MutableGraphView::UpdateFanin(node_name='foo_1', from_fanin='a:0', "
+      "to_fanin='a:-2') error: fanin 'a:-2' must be a valid tensor id.";
   TestUpdateFanin("foo_1", /*node_exists=*/true, {"a", 0}, {"a", -2},
                   /*success=*/false, error_msg, {"a"});
   // Update fanin of node where from/to fanins are invalid and missing and node
   // is missing.
   error_msg =
-      "Can't update invalid/missing fanin 'from_bar_missing:-2' to "
-      "invalid/missing fanin 'to_bar_missing:-3' in missing node "
-      "'foo_missing'.";
+      "MutableGraphView::UpdateFanin(node_name='foo_missing', "
+      "from_fanin='from_bar_missing:-2', to_fanin='to_bar_missing:-3') error: "
+      "fanin 'from_bar_missing:-2' must be a valid tensor id.";
   TestUpdateFanin("foo_missing", /*node_exists=*/false,
                   {"from_bar_missing", -2}, {"to_bar_missing", -3},
                   /*success=*/false, error_msg, {});
 
   // Update to self to create cycle.
-  error_msg = "Can't update fanin 'b:2' to fanin 'foo_4:3' in self 'foo_4'.";
+  error_msg =
+      "MutableGraphView::UpdateFanin(node_name='foo_4', from_fanin='b:2', "
+      "to_fanin='foo_4:3') error: can't update fanin to or from self.";
   TestUpdateFanin("foo_4", /*node_exists=*/true, {"b", 2}, {"foo_4", 3},
                   /*success=*/false, error_msg,
                   {"a", "b:2", "b:2", "^c", "^d"});
-  error_msg = "Can't update fanin 'b:2' to fanin '^foo_4' in self 'foo_4'.";
+  error_msg =
+      "MutableGraphView::UpdateFanin(node_name='foo_4', from_fanin='b:2', "
+      "to_fanin='^foo_4') error: can't update fanin to or from self.";
   TestUpdateFanin(
       "foo_4", /*node_exists=*/true, {"b", 2}, {"foo_4", Graph::kControlSlot},
       /*success=*/false, error_msg, {"a", "b:2", "b:2", "^c", "^d"});
-  error_msg = "Can't update fanin '^c' to fanin 'foo_4:4' in self 'foo_4'.";
+  error_msg =
+      "MutableGraphView::UpdateFanin(node_name='foo_4', from_fanin='^c', "
+      "to_fanin='foo_4:4') error: can't update fanin to or from self.";
   TestUpdateFanin(
       "foo_4", /*node_exists=*/true, {"c", Graph::kControlSlot}, {"foo_4", 4},
       /*success=*/false, error_msg, {"a", "b:2", "b:2", "^c", "^d"});
-  error_msg = "Can't update fanin '^c' to fanin '^foo_4' in self 'foo_4'.";
+  error_msg =
+      "MutableGraphView::UpdateFanin(node_name='foo_4', from_fanin='^c', "
+      "to_fanin='^foo_4') error: can't update fanin to or from self.";
   TestUpdateFanin("foo_4", /*node_exists=*/true, {"c", Graph::kControlSlot},
                   {"foo_4", Graph::kControlSlot}, /*success=*/false, error_msg,
                   {"a", "b:2", "b:2", "^c", "^d"});
@@ -868,8 +922,9 @@ void TestUpdateFaninFromFaninToNodeAsSwitchControl(const TensorId& fanin) {
   Status s = graph.UpdateFanin("c", fanin, {"b", Graph::kControlSlot});
   EXPECT_FALSE(s.ok());
   string expected_msg = absl::Substitute(
-      "Can't update fanin '$0' to fanin '^b' in node 'c', to fanin is a Switch "
-      "control dependency.",
+      "MutableGraphView::UpdateFanin(node_name='c', from_fanin='$0', "
+      "to_fanin='^b') error: can't update to fanin '^b' as it will become a "
+      "Switch control dependency.",
       fanin.ToString());
   EXPECT_EQ(s.error_message(), expected_msg);
 
@@ -1090,18 +1145,23 @@ TEST(MutableGraphViewTest, AddControllingFaninMissing) {
   // Missing fanin.
   Status s = graph.AddControllingFanin("a", {"c", Graph::kControlSlot});
   EXPECT_FALSE(s.ok());
-  string expected_msg = "Can't add missing controlling fanin '^c' to node 'a'.";
+  string expected_msg =
+      "MutableGraphView::AddControllingFanin(node_name='a', fanin='^c') error: "
+      "node 'c' was not found.";
   EXPECT_EQ(s.error_message(), expected_msg);
   // Missing node.
   s = graph.AddControllingFanin("d", {"a", Graph::kControlSlot});
   EXPECT_FALSE(s.ok());
-  expected_msg = "Can't add controlling fanin '^a' to missing node 'd'.";
+  expected_msg =
+      "MutableGraphView::AddControllingFanin(node_name='d', fanin='^a') error: "
+      "node 'd' was not found.";
   EXPECT_EQ(s.error_message(), expected_msg);
   // Missing node and fanin.
   s = graph.AddControllingFanin("c", {"d", Graph::kControlSlot});
   EXPECT_FALSE(s.ok());
   expected_msg =
-      "Can't add missing controlling fanin '^d' to missing node 'c'.";
+      "MutableGraphView::AddControllingFanin(node_name='c', fanin='^d') error: "
+      "node 'c' was not found.";
   EXPECT_EQ(s.error_message(), expected_msg);
 
   ASSERT_EQ(graph.graph()->node_size(), 2);
@@ -1158,7 +1218,9 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitch) {
   Status s = graph.AddControllingFanin("a", {"b", Graph::kControlSlot});
   EXPECT_FALSE(s.ok());
   string expected_msg =
-      "Can't add Switch as controlling fanin '^b' to node 'a'.";
+      "MutableGraphView::AddControllingFanin(node_name='a', fanin='^b') error: "
+      "can't add controlling fanin '^b' as it will become a Switch control "
+      "dependency.";
   EXPECT_EQ(s.error_message(), expected_msg);
 
   ASSERT_EQ(graph.graph()->node_size(), 2);
@@ -1264,7 +1326,9 @@ void TestAddControllingFaninSelfLoops(absl::string_view node_name,
 }
 
 TEST(MutableGraphViewTest, AddControllingFaninSelfLoops) {
-  string error_msg = "Can't add controlling fanin '^a' to self.";
+  string error_msg =
+      "MutableGraphView::AddControllingFanin(node_name='a', fanin='^a') error: "
+      "can't add controlling fanin '^a' to self.";
   TestAddControllingFaninSelfLoops("a", {"a", Graph::kControlSlot}, error_msg);
 
   // Adding Switch control dependency to Identity consumer. Node `c` is
@@ -1273,7 +1337,8 @@ TEST(MutableGraphViewTest, AddControllingFaninSelfLoops) {
   // Identity, this will introduce a self loop, so no control dependency should
   // be added.
   error_msg =
-      "Can't add found controlling fanin '^c' from fanin 'b:0' to self.";
+      "MutableGraphView::AddControllingFanin(node_name='c', fanin='b:0') "
+      "error: can't add found controlling fanin '^c' to self.";
   TestAddControllingFaninSelfLoops("c", {"b", 0}, error_msg);
 
   // Adding Switch control dependency to Identity consumer. Node `d` is
@@ -1282,7 +1347,8 @@ TEST(MutableGraphViewTest, AddControllingFaninSelfLoops) {
   // Identity, this will introduce a self loop, so no control dependency should
   // be added.
   error_msg =
-      "Can't add found controlling fanin '^d' from fanin 'b:1' to self.";
+      "MutableGraphView::AddControllingFanin(node_name='d', fanin='b:1') "
+      "error: can't add found controlling fanin '^d' to self.";
   TestAddControllingFaninSelfLoops("d", {"b", 1}, error_msg);
 }
 
@@ -1305,8 +1371,9 @@ TEST(MutableGraphViewTest, AddControllingFaninSelfLoopsGeneratedIdentity) {
   Status s = graph.AddControllingFanin("ConstantFoldingCtrl/b_1", {"b", 1});
   EXPECT_FALSE(s.ok());
   string expected_msg =
-      "Can't add generated controlling fanin '^ConstantFoldingCtrl/b_1' from "
-      "fanin 'b:1' to self.";
+      "MutableGraphView::AddControllingFanin(node_name='ConstantFoldingCtrl/"
+      "b_1', fanin='b:1') error: can't add generated controlling fanin "
+      "'^ConstantFoldingCtrl/b_1' to self.";
   EXPECT_EQ(s.error_message(), expected_msg);
 
   EXPECT_EQ(graph.graph()->node_size(), 4);
@@ -1396,7 +1463,10 @@ TEST(MutableGraphViewTest, RemoveControllingFaninSelfLoop) {
 
   Status s = graph.RemoveControllingFanin("c", "c");
   EXPECT_FALSE(s.ok());
-  string expected_msg = "Can't remove controlling fanin '^c' from self.";
+  string expected_msg =
+      "MutableGraphView::RemoveControllingFanin(node_name='c', "
+      "fanin_node_name='c') error: can't remove controlling fanin '^c' from "
+      "self.";
   EXPECT_EQ(s.error_message(), expected_msg);
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
@@ -1517,8 +1587,8 @@ TEST(MutableGraphViewTest, DeleteNodesWithError) {
   Status s = graph.DeleteNodes({"b", "a"});
   EXPECT_FALSE(s.ok());
   string error_msg =
-      "Can't delete node(s) with retained fanout(s) [a, b] from node(s) [a, "
-      "b].";
+      "MutableGraphView::DeleteNodes(nodes_to_delete={a, b}) error: can't "
+      "delete node(s) with retained fanouts(s) [a, b].";
   EXPECT_EQ(s.error_message(), error_msg);
 
   EXPECT_EQ(graph.graph()->node_size(), 6);
@@ -1551,8 +1621,9 @@ TEST(MutableGraphViewTest, DeleteNodesWithLargeError) {
   Status s = graph.DeleteNodes({"a", "b", "c", "d", "e", "f"});
   EXPECT_FALSE(s.ok());
   string error_msg =
-      "Can't delete node(s) with retained fanout(s) [a, b, c, d, e, ...] from "
-      "node(s) [a, b, c, d, e, ...].";
+      "MutableGraphView::DeleteNodes(nodes_to_delete={a, b, c, d, e, ...}) "
+      "error: can't delete node(s) with retained fanouts(s) [a, b, c, d, e, "
+      "...].";
   EXPECT_EQ(s.error_message(), error_msg);
 
   EXPECT_EQ(graph.graph()->node_size(), 13);
-- 
GitLab


From 6da1788b4ad78b714f7918042381e2aba36a3c67 Mon Sep 17 00:00:00 2001
From: Yunlu Li <yunluli@google.com>
Date: Tue, 22 Jan 2019 11:23:10 -0800
Subject: [PATCH 1064/2345] Add support for joining data with default
 precision.

PiperOrigin-RevId: 230376960
---
 tensorflow/lite/testing/join.h       | 16 +++++++++++++++-
 tensorflow/lite/testing/join_test.cc |  5 +++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/testing/join.h b/tensorflow/lite/testing/join.h
index d1c3146086..d10d2909b5 100644
--- a/tensorflow/lite/testing/join.h
+++ b/tensorflow/lite/testing/join.h
@@ -24,7 +24,21 @@ limitations under the License.
 namespace tflite {
 namespace testing {
 
-// Join a list of data separated by delimiter.
+// Join a list of data with default precision separated by delimiter.
+template <typename T>
+string JoinDefault(T* data, size_t len, const string& delimiter) {
+  if (len == 0 || data == nullptr) {
+    return "";
+  }
+  std::stringstream result;
+  result << data[0];
+  for (int i = 1; i < len; i++) {
+    result << delimiter << data[i];
+  }
+  return result.str();
+}
+
+// Join a list of data with fixed precision separated by delimiter.
 template <typename T>
 string Join(T* data, size_t len, const string& delimiter) {
   if (len == 0 || data == nullptr) {
diff --git a/tensorflow/lite/testing/join_test.cc b/tensorflow/lite/testing/join_test.cc
index 0b3c07f37e..476a7f2059 100644
--- a/tensorflow/lite/testing/join_test.cc
+++ b/tensorflow/lite/testing/join_test.cc
@@ -26,6 +26,11 @@ TEST(JoinTest, JoinInt) {
   EXPECT_EQ(Join(data.data(), data.size(), ","), "1,2,3");
 }
 
+TEST(JoinDefaultTest, JoinFloat) {
+  float data[] = {1.0, -3, 2.3, 1e-5};
+  EXPECT_EQ(JoinDefault(data, 4, " "), "1 -3 2.3 1e-05");
+}
+
 TEST(JoinTest, JoinFloat) {
   float data[] = {1.0, -3, 2.3, 1e-5};
   EXPECT_EQ(Join(data, 4, " "), "1 -3 2.29999995 9.99999975e-06");
-- 
GitLab


From 79e040fd8a7e09bc4754f4635aed7a3e1a91ce40 Mon Sep 17 00:00:00 2001
From: Cong Liu <congliu@google.com>
Date: Tue, 22 Jan 2019 11:54:24 -0800
Subject: [PATCH 1065/2345] [XLA] Propagate is_dynamic_dimensions for some
 operations.

PiperOrigin-RevId: 230382946
---
 tensorflow/compiler/xla/client/BUILD          |   1 +
 tensorflow/compiler/xla/client/xla_builder.cc |  60 ++-
 tensorflow/compiler/xla/client/xla_builder.h  |  14 +-
 .../compiler/xla/client/xla_builder_test.cc   | 406 +++++++++++++++++-
 .../compiler/xla/service/shape_inference.cc   | 125 +++++-
 tensorflow/compiler/xla/shape.h               |   4 +
 tensorflow/compiler/xla/shape_util.cc         |   4 +
 tensorflow/compiler/xla/shape_util_test.cc    |  20 +
 8 files changed, 586 insertions(+), 48 deletions(-)

diff --git a/tensorflow/compiler/xla/client/BUILD b/tensorflow/compiler/xla/client/BUILD
index 27c075e8f1..f5d56e8a9e 100644
--- a/tensorflow/compiler/xla/client/BUILD
+++ b/tensorflow/compiler/xla/client/BUILD
@@ -246,6 +246,7 @@ tf_cc_test(
         "//tensorflow/compiler/xla:status_macros",
         "//tensorflow/compiler/xla:test",
         "//tensorflow/compiler/xla:test_helpers",
+        "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/service:hlo",
         "//tensorflow/compiler/xla/service:hlo_matchers",
diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc
index 20298d175d..5c9f9f7088 100644
--- a/tensorflow/compiler/xla/client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_builder.cc
@@ -299,23 +299,25 @@ XlaComputation XlaBuilder::BuildAndNoteError() {
   return build_status.ConsumeValueOrDie();
 }
 
-StatusOr<XlaComputation> XlaBuilder::Build() {
+StatusOr<XlaComputation> XlaBuilder::Build(bool remove_dynamic_dimensions) {
   if (!first_error_.ok()) {
     string backtrace;
     first_error_backtrace_.Dump(tensorflow::DebugWriteToString, &backtrace);
     return AppendStatus(first_error_, backtrace);
   }
-  return Build(instructions_.back().id());
+  return Build(instructions_.back().id(), remove_dynamic_dimensions);
 }
 
-StatusOr<XlaComputation> XlaBuilder::Build(XlaOp root) {
+StatusOr<XlaComputation> XlaBuilder::Build(XlaOp root,
+                                           bool remove_dynamic_dimensions) {
   if (root.builder_ != this) {
     return InvalidArgument("Given root operation is not in this computation.");
   }
-  return Build(root.handle());
+  return Build(root.handle(), remove_dynamic_dimensions);
 }
 
-StatusOr<XlaComputation> XlaBuilder::Build(int64 root_id) {
+StatusOr<XlaComputation> XlaBuilder::Build(int64 root_id,
+                                           bool remove_dynamic_dimensions) {
   if (!first_error_.ok()) {
     string backtrace;
     first_error_backtrace_.Dump(tensorflow::DebugWriteToString, &backtrace);
@@ -325,20 +327,22 @@ StatusOr<XlaComputation> XlaBuilder::Build(int64 root_id) {
   // TODO(b/121223198): XLA backend cannot handle dynamic dimensions yet, remove
   // all dynamic dimensions before building xla program until we have support in
   // the backend.
-  std::function<void(ShapeProto*)> remove_dynamic_dimension =
-      [&](ShapeProto* shape) {
-        if (shape->tuple_shapes_size() != 0) {
-          for (int64 i = 0; i < shape->tuple_shapes_size(); ++i) {
-            remove_dynamic_dimension(shape->mutable_tuple_shapes(i));
+  if (remove_dynamic_dimensions) {
+    std::function<void(ShapeProto*)> remove_dynamic_dimension =
+        [&](ShapeProto* shape) {
+          if (shape->tuple_shapes_size() != 0) {
+            for (int64 i = 0; i < shape->tuple_shapes_size(); ++i) {
+              remove_dynamic_dimension(shape->mutable_tuple_shapes(i));
+            }
           }
-        }
-        for (int64 i = 0; i < shape->dimensions_size(); ++i) {
-          shape->set_is_dynamic_dimension(i, false);
-        }
-      };
+          for (int64 i = 0; i < shape->dimensions_size(); ++i) {
+            shape->set_is_dynamic_dimension(i, false);
+          }
+        };
 
-  for (auto& instruction : instructions_) {
-    remove_dynamic_dimension(instruction.mutable_shape());
+    for (auto& instruction : instructions_) {
+      remove_dynamic_dimension(instruction.mutable_shape());
+    }
   }
 
   HloComputationProto entry;
@@ -497,16 +501,19 @@ XlaOp XlaBuilder::BinaryOp(HloOpcode binop, const XlaOp& lhs, const XlaOp& rhs,
       const Shape& from_shape = should_broadcast_lhs ? lhs_shape : rhs_shape;
 
       std::vector<int64> to_size;
-      for (int64 size : shape.dimensions()) {
-        to_size.push_back(size);
+      std::vector<bool> to_size_is_dynamic;
+      for (int i = 0; i < shape.rank(); i++) {
+        to_size.push_back(shape.dimensions(i));
+        to_size_is_dynamic.push_back(shape.is_dynamic_dimension(i));
       }
       for (int64 from_dim = 0; from_dim < from_shape.rank(); from_dim++) {
         int64 to_dim = broadcast_dimensions[from_dim];
         to_size[to_dim] = from_shape.dimensions(from_dim);
+        to_size_is_dynamic[to_dim] = from_shape.is_dynamic_dimension(from_dim);
       }
 
-      const Shape& broadcasted_shape =
-          ShapeUtil::MakeShape(from_shape.element_type(), to_size);
+      const Shape& broadcasted_shape = ShapeUtil::MakeShape(
+          from_shape.element_type(), to_size, to_size_is_dynamic);
       TF_ASSIGN_OR_RETURN(
           XlaOp broadcasted_operand,
           InDimBroadcast(broadcasted_shape, from, broadcast_dimensions));
@@ -665,8 +672,17 @@ XlaOp XlaBuilder::BroadcastInDim(
     TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand));
     // Output shape, in the case of degenerate broadcast, the out_dim_size is
     // not necessarily the same as the dimension sizes of the output shape.
-    const auto& output_shape =
+    auto output_shape =
         ShapeUtil::MakeShape(operand_shape.element_type(), out_dim_size);
+    for (int i = 0; i < broadcast_dimensions.size(); i++) {
+      if (broadcast_dimensions[i] < 0 ||
+          broadcast_dimensions[i] > out_dim_size.size()) {
+        return InvalidArgument("Broadcast dimension %lld is out of bound",
+                               broadcast_dimensions[i]);
+      }
+      output_shape.set_dynamic_dimension(broadcast_dimensions[i],
+                                         operand_shape.is_dynamic_dimension(i));
+    }
 
     TF_RETURN_IF_ERROR(ShapeInference::InferBroadcastShape(
                            operand_shape, output_shape, broadcast_dimensions)
diff --git a/tensorflow/compiler/xla/client/xla_builder.h b/tensorflow/compiler/xla/client/xla_builder.h
index 8908d172fa..7df1efe6b8 100644
--- a/tensorflow/compiler/xla/client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_builder.h
@@ -197,11 +197,19 @@ class XlaBuilder {
   // status. Note that all ops that have been enqueued will be moved to the
   // computation being returned. The root of the computation will be the last
   // added operation.
-  StatusOr<XlaComputation> Build();
+  //
+  // `remove_dynamic_dimensions` tells the builder whether to remove the
+  // dyanmic dimensions information in all ops.
+  //
+  // TODO(b/121223198): Delete `remove_dynamic_dimensions` and keeps the
+  // dynamic dimensions information when XLA backend can handle dynamic
+  // dimensions.
+  StatusOr<XlaComputation> Build(bool remove_dynamic_dimensions = true);
 
   // Overload of Build which specifies a particular root instruction for the
   // computation.
-  StatusOr<XlaComputation> Build(XlaOp root);
+  StatusOr<XlaComputation> Build(XlaOp root,
+                                 bool remove_dynamic_dimensions = true);
 
   // Builds the computation with the requested operations, or notes an error in
   // the parent XlaBuilder and returns an empty computation if building failed.
@@ -293,7 +301,7 @@ class XlaBuilder {
   };
 
   // Build helper which takes the id of the root operation..
-  StatusOr<XlaComputation> Build(int64 root_id);
+  StatusOr<XlaComputation> Build(int64 root_id, bool remove_dynamic_dimensions);
 
   // Description for the methods below can be found in the corresponding public
   // functions section in this file.
diff --git a/tensorflow/compiler/xla/client/xla_builder_test.cc b/tensorflow/compiler/xla/client/xla_builder_test.cc
index feee8187c7..098165000a 100644
--- a/tensorflow/compiler/xla/client/xla_builder_test.cc
+++ b/tensorflow/compiler/xla/client/xla_builder_test.cc
@@ -25,6 +25,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/test.h"
 #include "tensorflow/compiler/xla/test_helpers.h"
+#include "tensorflow/compiler/xla/util.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
 
 namespace xla {
@@ -39,7 +40,8 @@ using ::testing::HasSubstr;
 class XlaBuilderTest : public ::testing::Test {
  protected:
   StatusOr<std::unique_ptr<HloModule>> BuildHloModule(XlaBuilder* b) {
-    TF_ASSIGN_OR_RETURN(XlaComputation computation, b->Build());
+    TF_ASSIGN_OR_RETURN(XlaComputation computation,
+                        b->Build(/*remove_dynamic_dimensions=*/false));
     const HloModuleProto& proto = computation.proto();
     TF_ASSIGN_OR_RETURN(const auto& config,
                         HloModule::CreateModuleConfigFromProto(
@@ -50,7 +52,8 @@ class XlaBuilderTest : public ::testing::Test {
   // Overload which explicitly specifies the root instruction.
   StatusOr<std::unique_ptr<HloModule>> BuildHloModule(XlaBuilder* b,
                                                       XlaOp root) {
-    TF_ASSIGN_OR_RETURN(XlaComputation computation, b->Build(root));
+    TF_ASSIGN_OR_RETURN(XlaComputation computation,
+                        b->Build(root, /*remove_dynamic_dimensions=*/false));
     const HloModuleProto& proto = computation.proto();
     TF_ASSIGN_OR_RETURN(const auto& config,
                         HloModule::CreateModuleConfigFromProto(
@@ -447,8 +450,7 @@ TEST_F(XlaBuilderTest, ProtoMatches) {
 }
 
 TEST_F(XlaBuilderTest, DynamicParameter) {
-  std::vector<XlaComputation> computations;
-  XlaBuilder b("builder");
+  XlaBuilder b(TestName());
   Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
       {ShapeUtil::MakeShape(F32, {5}), ShapeUtil::MakeShape(F32, {6})});
   auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
@@ -463,9 +465,399 @@ TEST_F(XlaBuilderTest, DynamicParameter) {
                                  ->parameter_instruction(0)
                                  ->shape()
                                  .tuple_shapes(1);
-  // TODO(b/121223198): The dynamic dimension should be set once we enable
-  // dynamic dimensions in xla builder.
-  EXPECT_FALSE(param_shape.is_dynamic_dimension(0));
+  EXPECT_TRUE(param_shape.is_dynamic_dimension(0));
+}
+
+TEST_F(XlaBuilderTest, DynamicUnary) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {5}), ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{1},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/0));
+  auto gte = GetTupleElement(p0, 0);
+  Neg(gte);
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  const Shape& result_shape =
+      module->entry_computation()->root_instruction()->shape();
+  EXPECT_TRUE(result_shape.is_dynamic_dimension(0));
+}
+
+TEST_F(XlaBuilderTest, DynamicBinary) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {5}), ShapeUtil::MakeShape(F32, {5}),
+       ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{2},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/0));
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{2},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{1},
+                                   /*target_dim_num=*/0));
+  auto gte0 = GetTupleElement(p0, 0);
+  auto gte1 = GetTupleElement(p0, 1);
+  Add(gte0, gte1);
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  const Shape& result_shape =
+      module->entry_computation()->root_instruction()->shape();
+  EXPECT_TRUE(result_shape.is_dynamic_dimension(0));
+}
+
+TEST_F(XlaBuilderTest, DynamicBinaryHasBroadcast) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {5, 4}), ShapeUtil::MakeShape(F32, {5}),
+       ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{2},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/0));
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{2},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{1},
+                                   /*target_dim_num=*/0));
+  auto gte0 = GetTupleElement(p0, 0);
+  auto gte1 = GetTupleElement(p0, 1);
+  Add(gte0, gte1, {0});
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  const Shape& result_shape =
+      module->entry_computation()->root_instruction()->shape();
+  EXPECT_TRUE(ContainersEqual(result_shape.dynamic_dimensions(), {true, false}))
+      << result_shape;
+}
+
+TEST_F(XlaBuilderTest, DynamicBroadcast) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {5, 4}), ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{1},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/0));
+  auto gte = GetTupleElement(p0, 0);
+  BroadcastInDim(gte, /*out_dim_size=*/{3, 5, 4},
+                 /*broadcast_dimensions=*/{1, 2});
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  const Shape& result_shape =
+      module->entry_computation()->root_instruction()->shape();
+  EXPECT_TRUE(
+      ContainersEqual(result_shape.dynamic_dimensions(), {false, true, false}))
+      << result_shape;
+}
+
+TEST_F(XlaBuilderTest, DynamicPad) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {5, 4}), ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  auto pad_val = ConstantR0<float>(&b, -1);
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{1},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/0));
+  auto gte = GetTupleElement(p0, 0);
+  PaddingConfig padding_config;
+  for (int i = 0; i < 2; i++) {
+    auto dimension = padding_config.add_dimensions();
+    dimension->set_edge_padding_low(0);
+    dimension->set_edge_padding_high(0);
+    dimension->set_interior_padding(0);
+  }
+  Pad(gte, pad_val, padding_config);
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  const Shape& result_shape =
+      module->entry_computation()->root_instruction()->shape();
+  EXPECT_TRUE(ContainersEqual(result_shape.dynamic_dimensions(), {true, false}))
+      << result_shape;
+}
+
+TEST_F(XlaBuilderTest, DynamicConvolution) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {1, 2, 2, 128}),
+       ShapeUtil::MakeShape(F32, {2, 2, 128, 8}), ShapeUtil::MakeShape(U32, {}),
+       ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{2},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/0));
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{3},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{1},
+                                   /*target_dim_num=*/2));
+  auto input = GetTupleElement(p0, 0);
+  auto filter = GetTupleElement(p0, 1);
+  ConvolutionDimensionNumbers dnums;
+  dnums.set_input_batch_dimension(0);
+  dnums.set_output_batch_dimension(0);
+  dnums.add_input_spatial_dimensions(1);
+  dnums.add_output_spatial_dimensions(1);
+  dnums.add_input_spatial_dimensions(2);
+  dnums.add_output_spatial_dimensions(2);
+  dnums.set_input_feature_dimension(3);
+  dnums.set_output_feature_dimension(3);
+  dnums.add_kernel_spatial_dimensions(0);
+  dnums.add_kernel_spatial_dimensions(1);
+  dnums.set_kernel_input_feature_dimension(2);
+  dnums.set_kernel_output_feature_dimension(3);
+  ConvWithGeneralDimensions(input, filter, {1, 1}, Padding::kValid, dnums,
+                            /*feature_group_count=*/1);
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  const Shape& result_shape =
+      module->entry_computation()->root_instruction()->shape();
+  EXPECT_TRUE(ContainersEqual(result_shape.dynamic_dimensions(),
+                              {true, false, false, false}))
+      << result_shape;
+}
+
+TEST_F(XlaBuilderTest, DynamicDot) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {2, 3, 4}),
+       ShapeUtil::MakeShape(F32, {2, 4, 5}), ShapeUtil::MakeShape(U32, {}),
+       ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{2},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/0));
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{2},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{1},
+                                   /*target_dim_num=*/0));
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{3},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/1));
+
+  auto lhs = GetTupleElement(p0, 0);
+  auto rhs = GetTupleElement(p0, 1);
+  DotDimensionNumbers dnums;
+  dnums.add_lhs_contracting_dimensions(2);
+  dnums.add_rhs_contracting_dimensions(1);
+  dnums.add_lhs_batch_dimensions(0);
+  dnums.add_rhs_batch_dimensions(0);
+  DotGeneral(lhs, rhs, dnums);
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  const Shape& result_shape =
+      module->entry_computation()->root_instruction()->shape();
+  EXPECT_TRUE(
+      ContainersEqual(result_shape.dynamic_dimensions(), {true, true, false}))
+      << result_shape;
+}
+
+TEST_F(XlaBuilderTest, DynamicReduce) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {5, 4, 3}), ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  auto init = ConstantR0<float>(&b, 0);
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{1},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/1));
+  auto gte = GetTupleElement(p0, 0);
+  XlaBuilder bsum(TestName());
+  Add(Parameter(&bsum, 0, ShapeUtil::MakeShape(F32, {}), "x"),
+      Parameter(&bsum, 1, ShapeUtil::MakeShape(F32, {}), "y"));
+  TF_ASSERT_OK_AND_ASSIGN(auto sum, bsum.Build());
+  Reduce(gte, init, sum, {0});
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  const Shape& result_shape =
+      module->entry_computation()->root_instruction()->shape();
+  EXPECT_TRUE(ContainersEqual(result_shape.dynamic_dimensions(), {true, false}))
+      << result_shape;
+}
+
+TEST_F(XlaBuilderTest, DynamicReduceWindow) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {2, 4, 8}), ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  auto init = ConstantR0<float>(&b, 0.f);
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{1},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/0));
+  auto gte = GetTupleElement(p0, 0);
+  XlaBuilder bsum(TestName());
+  Add(Parameter(&bsum, 0, ShapeUtil::MakeShape(F32, {}), "x"),
+      Parameter(&bsum, 1, ShapeUtil::MakeShape(F32, {}), "y"));
+  TF_ASSERT_OK_AND_ASSIGN(auto sum, bsum.Build());
+  ReduceWindow(gte, init, sum, /*window_dimensions=*/{1, 2, 4},
+               /*window_strides=*/{1, 1, 1}, Padding::kValid);
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  const Shape& result_shape =
+      module->entry_computation()->root_instruction()->shape();
+  EXPECT_TRUE(
+      ContainersEqual(result_shape.dynamic_dimensions(), {true, false, false}))
+      << result_shape;
+}
+
+TEST_F(XlaBuilderTest, DynamicSelectAndScatter) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {2, 4, 8}),
+       ShapeUtil::MakeShape(F32, {2, 2, 2}), ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  auto init = ConstantR0<float>(&b, 0.f);
+  XlaBuilder bsum(TestName());
+  Add(Parameter(&bsum, 0, ShapeUtil::MakeShape(F32, {}), "x"),
+      Parameter(&bsum, 1, ShapeUtil::MakeShape(F32, {}), "y"));
+  TF_ASSERT_OK_AND_ASSIGN(auto sum, bsum.Build());
+  XlaBuilder bge(TestName());
+  Ge(Parameter(&bge, 0, ShapeUtil::MakeShape(F32, {}), "x"),
+     Parameter(&bge, 1, ShapeUtil::MakeShape(F32, {}), "y"));
+  TF_ASSERT_OK_AND_ASSIGN(auto ge, bge.Build());
+
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{2},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/0));
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{2},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{1},
+                                   /*target_dim_num=*/0));
+  auto gte0 = GetTupleElement(p0, 0);
+  auto source = GetTupleElement(p0, 1);
+  SelectAndScatter(gte0, ge, {1, 2, 4}, {1, 2, 4}, Padding::kValid, source,
+                   init, sum);
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  const Shape& result_shape =
+      module->entry_computation()->root_instruction()->shape();
+  EXPECT_TRUE(
+      ContainersEqual(result_shape.dynamic_dimensions(), {true, false, false}))
+      << result_shape;
+}
+
+TEST_F(XlaBuilderTest, DynamicReshape) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {2, 3, 4, 5, 6}),
+       ShapeUtil::MakeShape(U32, {}), ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{1},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/2));
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{2},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/3));
+  auto gte = GetTupleElement(p0, 0);  // f32[2, 3, <=4, <=5, 6]
+  Reshape(gte, /*new_sizes=*/{6, 4, 1, 5, 2, 3});
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  const Shape& result_shape =
+      module->entry_computation()->root_instruction()->shape();
+  EXPECT_TRUE(result_shape.is_dynamic_dimension(1));
+  EXPECT_TRUE(result_shape.is_dynamic_dimension(3));
+  EXPECT_TRUE(ContainersEqual(result_shape.dynamic_dimensions(),
+                              {false, true, false, true, false, false}))
+      << result_shape;
+}
+
+TEST_F(XlaBuilderTest, DynamicSelect) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {4, 5, 6}),
+       ShapeUtil::MakeShape(F32, {4, 5, 6}), ShapeUtil::MakeShape(U32, {}),
+       ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  auto pred = Parameter(&b, 1, ShapeUtil::MakeShape(PRED, {}), "pred");
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{2},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/1));
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{3},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{1},
+                                   /*target_dim_num=*/1));
+  auto gte0 = GetTupleElement(p0, 0);
+  auto gte1 = GetTupleElement(p0, 1);
+  Select(pred, gte0, gte1);
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  const Shape& result_shape =
+      module->entry_computation()->root_instruction()->shape();
+  EXPECT_TRUE(result_shape.is_dynamic_dimension(1));
+  EXPECT_FALSE(result_shape.is_dynamic_dimension(2));
+  EXPECT_TRUE(
+      ContainersEqual(result_shape.dynamic_dimensions(), {false, true, false}))
+      << result_shape;
+}
+
+TEST_F(XlaBuilderTest, DynamicSelectNotCompatible) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {4, 5, 6}),
+       ShapeUtil::MakeShape(F32, {4, 5, 6}), ShapeUtil::MakeShape(U32, {}),
+       ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  auto pred = Parameter(&b, 1, ShapeUtil::MakeShape(PRED, {}), "pred");
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{2},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/1));
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{3},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{1},
+                                   /*target_dim_num=*/2));
+  auto gte0 = GetTupleElement(p0, 0);  // f32[4,<=5,6]
+  auto gte1 = GetTupleElement(p0, 1);  // f32[4,5,<=6]
+  Select(pred, gte0, gte1);
+  Status status = BuildHloModule(&b).status();
+  ASSERT_IS_NOT_OK(status);
+  EXPECT_THAT(status.error_message(),
+              ::testing::HasSubstr("Operands to select must be the same shape; "
+                                   "got f32[4,<=5,6] and f32[4,5,<=6]"));
+}
+
+TEST_F(XlaBuilderTest, DynamicTranspose) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {3, 5}), ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{1},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/0));
+  auto gte = GetTupleElement(p0, 0);
+  Transpose(gte, /*permutation=*/{1, 0});
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  const Shape& result_shape =
+      module->entry_computation()->root_instruction()->shape();
+  EXPECT_TRUE(ContainersEqual(result_shape.dynamic_dimensions(), {false, true}))
+      << result_shape;
 }
 
 TEST_F(XlaBuilderTest, AfterAllWithNonTokenOperands) {
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index fad7afefaf..4200f09abd 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -156,6 +156,14 @@ Status VerifyReducerShape(const ProgramShape& reducer_shape,
   return Status::OK();
 }
 
+bool IsTrivialWindowDimension(const WindowDimension& window_dimension) {
+  return window_dimension.size() == 1 && window_dimension.stride() == 1 &&
+         window_dimension.padding_low() == 0 &&
+         window_dimension.padding_high() == 0 &&
+         window_dimension.window_dilation() == 1 &&
+         window_dimension.base_dilation() == 1;
+}
+
 StatusOr<Shape> InferWindowOutputShape(const Shape& base_shape,
                                        const Window& window,
                                        PrimitiveType element_type,
@@ -167,6 +175,7 @@ StatusOr<Shape> InferWindowOutputShape(const Shape& base_shape,
   }
 
   std::vector<int64> output_dimensions(window.dimensions_size());
+  std::vector<bool> output_is_dynamic(window.dimensions_size());
   for (int64 i = 0; i < window.dimensions_size(); ++i) {
     const auto& dim = window.dimensions(i);
     if (dim.size() <= 0) {
@@ -196,6 +205,12 @@ StatusOr<Shape> InferWindowOutputShape(const Shape& base_shape,
           window.DebugString());
     }
 
+    if (base_shape.is_dynamic_dimension(i) && !IsTrivialWindowDimension(dim)) {
+      return Unimplemented(
+          "Dynamic shape is not supported for non trivial window: %s",
+          window_util::ToString(window));
+    }
+
     const int64 dilated_base = window_util::DilatedBound(
         ShapeUtil::GetDimension(base_shape, i), dim.base_dilation());
     const int64 padded_dilated_base =
@@ -205,9 +220,11 @@ StatusOr<Shape> InferWindowOutputShape(const Shape& base_shape,
 
     output_dimensions[i] = window_util::StridedBound(
         padded_dilated_base, dilated_window, dim.stride());
+    output_is_dynamic[i] = base_shape.is_dynamic_dimension(i);
   }
 
-  return ShapeUtil::MakeValidatedShape(element_type, output_dimensions);
+  return ShapeUtil::MakeValidatedShape(element_type, output_dimensions,
+                                       output_is_dynamic);
 }
 
 }  // namespace
@@ -500,17 +517,29 @@ StatusOr<Shape> InferWindowOutputShape(const Shape& base_shape,
                            padding_config.ShortDebugString());
   }
 
+  if (!padding_value_shape.is_static()) {
+    return InvalidArgument("Dynamic padding value is not supported");
+  }
+
   std::vector<int64> dimensions(operand_shape.rank());
+  std::vector<bool> is_dynamic(operand_shape.rank());
   for (int64 i = 0; i < operand_shape.dimensions_size(); ++i) {
     const auto& p = padding_config.dimensions(i);
+    if (operand_shape.is_dynamic_dimension(i) && p.edge_padding_high() != 0 &&
+        p.edge_padding_low() != 0 && p.interior_padding() != 0) {
+      return InvalidArgument(
+          "Dynamic dimension on padding dimension is not supported.");
+    }
     dimensions[i] = operand_shape.dimensions(i) + p.edge_padding_low() +
                     p.edge_padding_high() +
                     std::max<int64>(operand_shape.dimensions(i) - 1, 0LL) *
                         p.interior_padding();
+    is_dynamic[i] = operand_shape.is_dynamic_dimension(i);
   }
+
   return ShapeUtil::MakeShape(
       ShapeUtil::HigherPrecisionElementType(operand_shape, padding_value_shape),
-      dimensions);
+      dimensions, is_dynamic);
 }
 
 // Current DotDimensionNumbers Requirements:
@@ -620,7 +649,9 @@ Status ValidateDotDimensionNumbers(
     const int64 rhs_contracting_dimension =
         dimension_numbers.rhs_contracting_dimensions(i);
     if (lhs.dimensions(lhs_contracting_dimension) !=
-        rhs.dimensions(rhs_contracting_dimension)) {
+            rhs.dimensions(rhs_contracting_dimension) ||
+        lhs.is_dynamic_dimension(lhs_contracting_dimension) !=
+            rhs.is_dynamic_dimension(rhs_contracting_dimension)) {
       return fail("Contracting dimension sizes do not match.");
     }
   }
@@ -634,7 +665,10 @@ Status ValidateDotDimensionNumbers(
   // Check that batch dimension numbers and sizes match.
   for (int64 i = 0; i < dimension_numbers.lhs_batch_dimensions_size(); ++i) {
     if (lhs.dimensions(dimension_numbers.lhs_batch_dimensions(i)) !=
-        rhs.dimensions(dimension_numbers.rhs_batch_dimensions(i))) {
+            rhs.dimensions(dimension_numbers.rhs_batch_dimensions(i)) ||
+        lhs.is_dynamic_dimension(dimension_numbers.lhs_batch_dimensions(i)) !=
+            rhs.is_dynamic_dimension(
+                dimension_numbers.rhs_batch_dimensions(i))) {
       return fail("Batch dimension sizes must match for lhs/rhs.");
     }
   }
@@ -645,14 +679,17 @@ Status ValidateDotDimensionNumbers(
   // Generate the result dimensions in order, rhs dimensions followed by lhs
   // dimensions except the contracted and batch dimensions.
   std::vector<int64> dimensions;
+  std::vector<bool> is_dynamic;
   for (int64 lhs_dim : dimension_numbers.lhs_batch_dimensions()) {
     dimensions.push_back(lhs.dimensions(lhs_dim));
+    is_dynamic.push_back(lhs.is_dynamic_dimension(lhs_dim));
   }
   for (int64 i = 0; i < lhs.rank(); i++) {
     if (!absl::c_linear_search(dimension_numbers.lhs_contracting_dimensions(),
                                i) &&
         !absl::c_linear_search(dimension_numbers.lhs_batch_dimensions(), i)) {
       dimensions.push_back(lhs.dimensions(i));
+      is_dynamic.push_back(lhs.is_dynamic_dimension(i));
     }
   }
   for (int64 i = 0; i < rhs.rank(); i++) {
@@ -660,10 +697,11 @@ Status ValidateDotDimensionNumbers(
                                i) &&
         !absl::c_linear_search(dimension_numbers.rhs_batch_dimensions(), i)) {
       dimensions.push_back(rhs.dimensions(i));
+      is_dynamic.push_back(rhs.is_dynamic_dimension(i));
     }
   }
   Shape result = ShapeUtil::MakeShape(
-      ShapeUtil::HigherPrecisionElementType(lhs, rhs), dimensions);
+      ShapeUtil::HigherPrecisionElementType(lhs, rhs), dimensions, is_dynamic);
 
   TF_DCHECK_OK(ShapeUtil::ValidateShapeWithOptionalLayout(result));
   VLOG(2) << "inferred dot shape: " << ShapeUtil::HumanString(result);
@@ -681,13 +719,17 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
   // dimension). In that case, the output shape has the non-1 dimension size
   // from the lhs/rhs pair in every index.
   std::vector<int64> output_dimensions(lhs.rank());
+  std::vector<bool> output_dimensions_is_dynamic(lhs.rank());
   for (int64 i = 0; i < lhs.rank(); ++i) {
     if (lhs.dimensions(i) == rhs.dimensions(i)) {
       output_dimensions[i] = lhs.dimensions(i);
+      output_dimensions_is_dynamic[i] = lhs.is_dynamic_dimension(i);
     } else if (lhs.dimensions(i) == 1) {
       output_dimensions[i] = rhs.dimensions(i);
+      output_dimensions_is_dynamic[i] = rhs.is_dynamic_dimension(i);
     } else if (rhs.dimensions(i) == 1) {
       output_dimensions[i] = lhs.dimensions(i);
+      output_dimensions_is_dynamic[i] = lhs.is_dynamic_dimension(i);
     } else {
       return InvalidArgument(
           "Binary op %s with incompatible shapes: %s and %s.",
@@ -696,7 +738,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
     }
   }
   return ShapeUtil::MakeShape(ShapeUtil::HigherPrecisionElementType(lhs, rhs),
-                              output_dimensions);
+                              output_dimensions, output_dimensions_is_dynamic);
 }
 
 /* static */ StatusOr<Shape> ShapeInference::InferInDimBroadcastShape(
@@ -775,6 +817,9 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
     }
     int64 small_dimension_size = smaller_shape.dimensions(i);
     int64 large_dimension_size = larger_shape.dimensions(dimension_to_match);
+    bool small_is_dynamic = smaller_shape.is_dynamic_dimension(i);
+    bool large_is_dynamic =
+        larger_shape.is_dynamic_dimension(dimension_to_match);
     // Dimension sizes must be compatible: match or be degenerate (degenerate
     // case is handled by degenerate dimension broadcasting which occurs after
     // InDim broadcasting).
@@ -786,6 +831,12 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
           ShapeUtil::HumanString(smaller_shape),
           ShapeUtil::HumanString(larger_shape));
     }
+    if (small_is_dynamic != large_is_dynamic) {
+      return InvalidArgument(
+          "Broadcast dimension %d dynamism mismatch: %s and %s.", i,
+          ShapeUtil::HumanString(smaller_shape),
+          ShapeUtil::HumanString(larger_shape));
+    }
     // Make sure the broadcast dimensions are listed in a strictly increasing
     // order.
     if (i > 0 && broadcast_dimensions.at(i - 1) >= dimension_to_match) {
@@ -795,6 +846,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
     }
 
     output_shape.set_dimensions(dimension_to_match, small_dimension_size);
+    output_shape.set_dynamic_dimension(dimension_to_match, small_is_dynamic);
   }
 
   return output_shape;
@@ -1731,8 +1783,33 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
     dimensions[dnums.output_spatial_dimensions(i)] =
         window_output_shape.dimensions(i);
   }
+  std::vector<bool> is_dynamic(num_dims);
+  for (int i = 0; i < num_dims; i++) {
+    if (lhs.is_dynamic_dimension(i)) {
+      if (i == dnums.input_batch_dimension()) {
+        is_dynamic[dnums.output_batch_dimension()] = true;
+      } else if (i == dnums.input_feature_dimension()) {
+        // Input feature dimension is a contracting dimension, which does not
+        // affect the output dimension size. So we need to do nothing.
+      } else {
+        return InvalidArgument(
+            "Dynamic Spatial Convolution is not supported: lhs shape is %s ",
+            lhs.ToString());
+      }
+    }
+    if (rhs.is_dynamic_dimension(i)) {
+      if (i == dnums.kernel_input_feature_dimension()) {
+        // Kernel feature dimension does not affect the output dimension size.
+        // So we need to do nothing.
+      } else {
+        return InvalidArgument(
+            "Dynamic Spatial Convolution is not supported: rhs shape is %s ",
+            rhs.ToString());
+      }
+    }
+  }
   return ShapeUtil::MakeShape(ShapeUtil::HigherPrecisionElementType(lhs, rhs),
-                              dimensions);
+                              dimensions, is_dynamic);
 }
 
 /* static */ StatusOr<Shape> ShapeInference::InferFftShape(
@@ -1904,7 +1981,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
   for (int64 i = 1; i < num_reduced_args; ++i) {
     if (!ShapeUtil::SameDimensions(*reduced_args[0], *reduced_args[i])) {
       return InvalidArgument(
-          "All reduced tensors must have the sime dimension. Tensor 0 has "
+          "All reduced tensors must have the same dimension. Tensor 0 has "
           "shape %s, Tensor %d has shape %s",
           ShapeUtil::HumanString(*reduced_args[0]), i,
           ShapeUtil::HumanString(*reduced_args[i]));
@@ -1933,20 +2010,22 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
   std::set<int64> dimensions_to_reduce_set(dimensions_to_reduce.begin(),
                                            dimensions_to_reduce.end());
   std::vector<int64> new_dimensions;
+  std::vector<bool> new_is_dynamic;
   for (int i = 0; i < arg.rank(); ++i) {
     if (dimensions_to_reduce_set.find(i) == dimensions_to_reduce_set.end()) {
       new_dimensions.push_back(arg.dimensions(i));
+      new_is_dynamic.push_back(arg.is_dynamic_dimension(i));
     }
   }
 
   if (ShapeUtil::IsScalar(to_apply.result())) {
     return ShapeUtil::MakeShape(to_apply.result().element_type(),
-                                new_dimensions);
+                                new_dimensions, new_is_dynamic);
   } else {
     std::vector<Shape> result_subshapes;
     for (const Shape& subshape : to_apply.result().tuple_shapes()) {
-      result_subshapes.push_back(
-          ShapeUtil::MakeShape(subshape.element_type(), new_dimensions));
+      result_subshapes.push_back(ShapeUtil::MakeShape(
+          subshape.element_type(), new_dimensions, new_is_dynamic));
     }
     return ShapeUtil::MakeTupleShape(result_subshapes);
   }
@@ -2020,6 +2099,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
         ShapeUtil::HumanString(source_shape),
         ShapeUtil::HumanString(window_result_shape));
   }
+
   return operand_shape;
 }
 
@@ -2502,6 +2582,12 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
           i, operand_shape.dimensions(i), broadcast_dimensions[i],
           output_shape.dimensions(broadcast_dimensions[i]));
     }
+    if (operand_shape.is_dynamic_dimension(i) !=
+        output_shape.is_dynamic_dimension(broadcast_dimensions[i])) {
+      return InvalidArgument(
+          "Broadcast input and output dynamism mismatch: %s and %s",
+          operand_shape.ToString(), output_shape.ToString());
+    }
     // Make sure the broadcast dimensions are listed in a strictly increasing
     // order.
     if (i > 0 && broadcast_dimensions[i - 1] >= broadcast_dimensions[i]) {
@@ -2544,6 +2630,14 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
         StrJoin(dimensions, ","), ShapeUtil::HumanString(operand));
   }
 
+  std::vector<std::pair<int64, int64>> unmodified_dims =
+      ShapeUtil::DimensionsUnmodifiedByReshape(operand, inferred_shape);
+  for (auto& unmodified : unmodified_dims) {
+    if (operand.is_dynamic_dimension(unmodified.first)) {
+      inferred_shape.set_dynamic_dimension(unmodified.second, true);
+    }
+  }
+
   return inferred_shape;
 }
 
@@ -2624,12 +2718,11 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
     // dimensions as on_true and on_false.
     return ShapeUtil::ChangeElementType(
         on_true, ShapeUtil::HigherPrecisionElementType(on_true, on_false));
-  } else {
-    return InvalidArgument(
-        "Select operation with non-scalar predicate with dimensionality "
-        " different from the other operands: %s.",
-        ShapeUtil::HumanString(pred));
   }
+  return InvalidArgument(
+      "Select operation with non-scalar predicate with dimensionality "
+      " different from the other operands: %s.",
+      ShapeUtil::HumanString(pred));
 }
 
 /* static */ StatusOr<Shape> ShapeInference::InferTupleSelectShape(
diff --git a/tensorflow/compiler/xla/shape.h b/tensorflow/compiler/xla/shape.h
index e40e5ca37a..e6b4e872f6 100644
--- a/tensorflow/compiler/xla/shape.h
+++ b/tensorflow/compiler/xla/shape.h
@@ -72,6 +72,10 @@ class Shape {
     dynamic_dimensions_[dimension] = is_dynamic;
   }
 
+  const std::vector<bool>& dynamic_dimensions() const {
+    return dynamic_dimensions_;
+  }
+
   // Add dimension_upper_bound().
 
   // Removes the given dimension form the shape. Layout, if it exists, is
diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc
index 799a4483df..1ada4bc036 100644
--- a/tensorflow/compiler/xla/shape_util.cc
+++ b/tensorflow/compiler/xla/shape_util.cc
@@ -929,6 +929,10 @@ Status ForEachMutableSubshapeHelper(
   for (auto dim : Permute(permutation, shape.dimensions())) {
     new_shape.add_dimensions(dim);
   }
+  for (int64 i = 0; i < shape.rank(); i++) {
+    new_shape.set_dynamic_dimension(permutation[i],
+                                    shape.is_dynamic_dimension(i));
+  }
 
   // If `shape` has a layout, by contract we choose a new layout such that the
   // transpose defined by this permutation is a bitcast.
diff --git a/tensorflow/compiler/xla/shape_util_test.cc b/tensorflow/compiler/xla/shape_util_test.cc
index 61b4e73e06..126ae58293 100644
--- a/tensorflow/compiler/xla/shape_util_test.cc
+++ b/tensorflow/compiler/xla/shape_util_test.cc
@@ -710,6 +710,26 @@ TEST(ShapeUtilTest, PermuteDimensionsLayout) {
   } while (std::next_permutation(layout.begin(), layout.end()));
 }
 
+TEST(ShapeUtilTest, PermuteDynamicDimensions) {
+  Shape shape =
+      ShapeUtil::MakeShape(F32, {10, 100, 1000},
+                           /*dynamic_dimensions*/ {false, true, true});
+  SCOPED_TRACE(absl::StrCat("shape=", shape.ToString()));
+
+  std::vector<int64> permutation(3);
+  std::iota(permutation.begin(), permutation.end(), 0);
+  do {
+    SCOPED_TRACE(absl::StrCat("permutation=", absl::StrJoin(permutation, ",")));
+
+    auto permuted = ShapeUtil::PermuteDimensions(permutation, shape);
+    for (int i = 0; i < shape.rank(); i++) {
+      EXPECT_EQ(permuted.dimensions(permutation[i]), shape.dimensions(i));
+      EXPECT_EQ(permuted.is_dynamic_dimension(permutation[i]),
+                shape.is_dynamic_dimension(i));
+    }
+  } while (std::next_permutation(permutation.begin(), permutation.end()));
+}
+
 TEST(AlgebraicSimplifierTest, ReshapeIsBitcast_3x2x2_6x2_Dim0IsMostMinor) {
   EXPECT_FALSE(ShapeUtil::ReshapeIsBitcast(
       ShapeUtil::MakeShapeWithLayout(F32, {3, 2, 2}, {0, 1, 2}),
-- 
GitLab


From 0b8ac6e1b9e85b3d16c8180cb2291c38ff66a1ba Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Tue, 22 Jan 2019 12:15:34 -0800
Subject: [PATCH 1066/2345] [Grappler] Cleanup grappler function inlining rules
 wrt stateful and dataset ops.

Grappler doesn't try to be the "automatic control dependency tracking framework", if side-effects after inlining are not guaranteed to have well defined execution order, we just do not inline it, and don't try to fix it. This is a graph-construction problem that should be fixed in tf-eager auto-control dependencies tracking.

In Agressive mode function optimizer skips this check and always inline.

PiperOrigin-RevId: 230386480
---
 .../core/grappler/graph_topology_view.cc      |  60 ++-
 .../core/grappler/graph_topology_view.h       |   6 +
 tensorflow/core/grappler/op_types.cc          |  16 +
 tensorflow/core/grappler/op_types.h           |   5 +
 .../grappler/optimizers/function_optimizer.cc | 450 ++++++++++++------
 .../optimizers/function_optimizer_test.cc     |  38 +-
 6 files changed, 395 insertions(+), 180 deletions(-)

diff --git a/tensorflow/core/grappler/graph_topology_view.cc b/tensorflow/core/grappler/graph_topology_view.cc
index 38ccfbaeb8..79e2f9a92f 100644
--- a/tensorflow/core/grappler/graph_topology_view.cc
+++ b/tensorflow/core/grappler/graph_topology_view.cc
@@ -62,19 +62,37 @@ Status GraphTopologyView::InitializeFromGraph(
   // 1. Add ephemeral edges to the adjacency lists.
   for (const GraphView::Edge& edge : ephemeral_edges) {
     const auto src = node_name_to_index_.find(edge.src.node->name());
-    if (src == node_name_to_index_.end()) {
-      return errors::InvalidArgument("Non-existent src node: ",
-                                     edge.src.node->name());
+    const bool valid_src = src != node_name_to_index_.end();
+
+    if (!valid_src) {
+      const string error_message =
+          absl::StrCat("Non-existent src node: ", edge.src.node->name());
+      if (skip_invalid_edges_) {
+        VLOG(0) << "Skip error: " << error_message;
+      } else {
+        return errors::InvalidArgument(error_message);
+      }
     }
+
     const auto dst = node_name_to_index_.find(edge.dst.node->name());
-    if (dst == node_name_to_index_.end()) {
-      return errors::InvalidArgument("Non-existent dst node: ",
-                                     edge.dst.node->name());
+    const bool valid_dst = dst != node_name_to_index_.end();
+
+    if (!valid_dst) {
+      const string error_message =
+          absl::StrCat("Non-existent dst node: ", edge.dst.node->name());
+      if (skip_invalid_edges_) {
+        VLOG(0) << "Skip error: " << error_message;
+      } else {
+        return errors::InvalidArgument(error_message);
+      }
+    }
+
+    if (valid_dst && valid_src) {
+      const int src_idx = src->second;
+      const int dst_idx = dst->second;
+      fanins_[dst_idx].push_back(src_idx);
+      fanouts_[src_idx].push_back(dst_idx);
     }
-    const int src_idx = src->second;
-    const int dst_idx = dst->second;
-    fanins_[dst_idx].push_back(src_idx);
-    fanouts_[src_idx].push_back(dst_idx);
   }
 
   // 2. Add graph edges to the adjacency lists.
@@ -85,13 +103,23 @@ Status GraphTopologyView::InitializeFromGraph(
     for (const string& input : node.input()) {
       TensorId tensor = ParseTensorName(input);
       const auto it = node_name_to_index_.find(tensor.node());
-      if (it == node_name_to_index_.end()) {
-        return errors::InvalidArgument("Non-existent input ", input,
-                                       " for node ", node.name());
+      const bool valid_input = it != node_name_to_index_.end();
+
+      if (!valid_input) {
+        const string error_message = absl::StrCat("Non-existent input ", input,
+                                                  " in node ", node.name());
+        if (skip_invalid_edges_) {
+          VLOG(3) << "Skip error: " << error_message;
+        } else {
+          return errors::InvalidArgument(error_message);
+        }
+      }
+
+      if (valid_input) {
+        const int input_idx = it->second;
+        fanins_[node_idx].push_back(input_idx);
+        fanouts_[input_idx].push_back(node_idx);
       }
-      const int input_idx = it->second;
-      fanins_[node_idx].push_back(input_idx);
-      fanouts_[input_idx].push_back(node_idx);
     }
 
     // Dedup the input list while it's still hot in cache.
diff --git a/tensorflow/core/grappler/graph_topology_view.h b/tensorflow/core/grappler/graph_topology_view.h
index 1c222df4b6..c40d0093b9 100644
--- a/tensorflow/core/grappler/graph_topology_view.h
+++ b/tensorflow/core/grappler/graph_topology_view.h
@@ -47,6 +47,8 @@ namespace grappler {
 class GraphTopologyView {
  public:
   GraphTopologyView() = default;
+  explicit GraphTopologyView(bool skip_invalid_edges)
+      : skip_invalid_edges_(skip_invalid_edges) {}
 
   // Initialize graph topology view from the graph. It's possible to pass
   // additional edges that do not exist in a graph, but must be respected when
@@ -84,6 +86,10 @@ class GraphTopologyView {
   const absl::InlinedVector<int, 2>& GetFanout(int node_idx) const;
 
  private:
+  // If true, all invalid edges and inputs (srd, dst or input node not found in
+  // a graph) will be skipped, otherwise initialization will fail with error.
+  bool skip_invalid_edges_ = false;
+
   // WARN: `graph_` must outlive this object and graph nodes must not be
   // destructed, because node names captured with absl::string_view.
   const GraphDef* graph_ = nullptr;  // do not own
diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc
index f7931c615c..e9bf36ab29 100644
--- a/tensorflow/core/grappler/op_types.cc
+++ b/tensorflow/core/grappler/op_types.cc
@@ -568,6 +568,22 @@ bool IsDataset(const NodeDef& node) {
          op == "DatasetToSingleElement" || op == "ReduceDataset";
 }
 
+bool IsStateful(const NodeDef node, const OpRegistryInterface* op_registry) {
+  const OpDef* op_def = nullptr;
+  const string& op_name = node.op();
+  Status status = op_registry->LookUpOpDef(op_name, &op_def);
+  if (!status.ok()) {
+    LOG(WARNING) << "Failed to lookup OpDef for " << op_name
+                 << ". Error: " << status.error_message();
+    return false;
+  }
+  return op_def->is_stateful();
+}
+
+bool IsStateful(const NodeDef node) {
+  return IsStateful(node, OpRegistry::Global());
+}
+
 bool IsFreeOfSideEffect(const NodeDef& node,
                         const OpRegistryInterface* op_registry) {
   // Placeholders must be preserved to keep the graph feedable.
diff --git a/tensorflow/core/grappler/op_types.h b/tensorflow/core/grappler/op_types.h
index 092748f4ff..4f48ec4767 100644
--- a/tensorflow/core/grappler/op_types.h
+++ b/tensorflow/core/grappler/op_types.h
@@ -186,6 +186,11 @@ bool IsPersistent(const NodeDef& node);
 // Returns true if the node belongs to the NC_DATASET class (see graph/graph.h).
 bool IsDataset(const NodeDef& node);
 
+// Returns true if the node op is marked as stateful, or if it was not found in
+// op_registry.
+bool IsStateful(const NodeDef node, const OpRegistryInterface* op_registry);
+bool IsStateful(const NodeDef node);  // use OpRegistry::Global()
+
 bool IsFreeOfSideEffect(const NodeDef& node,
                         const OpRegistryInterface* op_registry);
 bool IsFreeOfSideEffect(const NodeDef& node);  // use OpRegistry::Global()
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index bd69405b1a..da8ad4a778 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -68,6 +68,11 @@ constexpr char kFuncAttrName[] = "f";
 
 constexpr char kNoInlineAttr[] = "_noinline";
 
+// Names of the nodes that used to anchor incoming/outgoing control edges for
+// inlined function calls (see InlineIndirectFunctionCall).
+constexpr char kControlInputNodeName[] = "control_input";
+constexpr char kControlOutputNodeName[] = "control_output";
+
 bool AttrIsTrue(const FunctionDef& func, const string& attr) {
   return func.attr().count(attr) != 0 && func.attr().at(attr).b();
 }
@@ -1237,12 +1242,26 @@ Status InlineSymbolicGradient(const NodeDef& node,
 // dependency tracking via input/output control edges, and we relax some of the
 // constraints that we have for direct function call inlining.
 //
-// "When a `PartitionedCallOp` function has a resource (DT_RESOURCE data type)
-// input argument it "captures" the mutable resource.  This is implemented by
-// automatically adding a incoming control edge from the previous side-effectful
-// op touching that resource, and an outgoing control edge to the next
-// side-effectful op using the same resource. This serializes the mutations of
-// the resource to make graph execution deterministic.
+// Automatic control dependency rules:
+//
+// 1) "When a `PartitionedCallOp` function has a resource (DT_RESOURCE data
+//    type) input argument it "captures" the mutable resource.  This is
+//    implemented by automatically adding a incoming control edge from the
+//    previous side-effectful op touching that resource, and an outgoing control
+//    edge to the next side-effectful op using the same resource. This
+//    serializes the mutations of the resource to make graph execution
+//    deterministic.
+//
+// 2) All stateful ops inside a function body are guaranteed to execute in
+//    program order, this is achieved by adding control edges between stateful
+//    ops at graph construction time.
+//
+// 3) Furthermore, all ops accepting the same resource as an input are
+//    guaranteed to run in program order. This is also done by adding control
+//    edges at graph construction time. The last op touching the resource
+//    will have an outgoing control edge to all function return nodes, which
+//    will guarantee that all side effects to the resource will happen before
+//    function completion.
 //
 // Function call inlining must preserve side effect visibility:
 //
@@ -1251,17 +1270,31 @@ Status InlineSymbolicGradient(const NodeDef& node,
 // 2) All side effects to the captured resources, that happened inside function
 //    body, must be visible to every op/function using that resource after the
 //    function call completed.
-
-// To guarantee that these properties are preserved after inlining we do:
 //
-// 1) Forward all input control dependencies from the function call node to the
-//    inlined function inputs (Identity nodes).
-// 2) Each side-effectful op inside function body adds itself as a control
-//    dependency to all the nodes in output control set of function call node.
+// To guarantee that these properties are preserved after inlining we:
+//
+// 1) Create "input_control" NoOp. Function call node incoming control edges
+//    will be forwarded *to* this node. Function inputs (Identity nodes) will
+//    have a control edge *from* this node. If function has no inputs, by
+//    construction it must have nodes without inputs in the function body, and
+//    in this case these nodes will have a control edge *from* this node.
+
+// 2) Create "output_control" NoOp. All nodes that have incoming control edge
+//    *from* the function call node, will be forwarded to this node. Function
+//    outputs (Identity nodes) will have a control edge *to* this node. This
+//    will guarantee that nodes that have control dependency on the function
+//    call, will observe all side-effects (guaranteed by graph construction with
+//    automatic control dependencies tracking).
 //
-// We do not add any other control dependencies to/from function body nodes,
-// because they are pure functions of input tensors, and can be freely
-// reordered.
+// If after function instantiation we find a stateful or a dataset op inside
+// the function body, that is not reachable from any of the function outputs (or
+// if the function has no outputs), we do not inline it, because we can't
+// guarantee that these nodes will be executed in correct order (or executed at
+// all) after inlining.
+//
+// We do not try to add any extra control edges to make sure that all
+// side-effectful nodes will be executed, that should be handled at graph
+// construction time.
 
 struct MaybeDeadOutput {
   const NodeDef* dead_tensor_src;
@@ -1382,6 +1415,131 @@ Status IsInlinableIndirectFunctionCall(const FunctionOptimizerContext& ctx,
   return Status::OK();
 }
 
+// Checks that all side-effects will be executed in well defined order. We do it
+// by checking if there is a path from stateful/dataset ops to one of the output
+// nodes.
+Status CheckThatSideEffectsWillExecute(
+    const FunctionOptimizerContext& ctx,
+    const GraphTopologyView& graph_topo_view,
+    const absl::flat_hash_set<string> output_nodes) {
+  // We ignore side-effects safety check in aggressive mode.
+  const bool aggressive = ctx.opt_level() == RewriterConfig::AGGRESSIVE;
+
+  for (const NodeDef& func_body_node : graph_topo_view.graph()->node()) {
+    const bool node_must_execute =
+        IsDataset(func_body_node) ||
+        IsStateful(func_body_node, &ctx.function_library());
+    if (!node_must_execute) continue;
+
+    VLOG(3) << "Check that node " << func_body_node.name()
+            << " will execute after inlining.";
+    bool will_execute = false;
+
+    // Check if we reached one of the output nodes.
+    const auto callbacks = DfsCallbacks::PreOrder([&](const NodeDef* node) {
+      if (output_nodes.count(node->name())) will_execute = true;
+    });
+
+    // Stop if we already proved that node will execute.
+    const auto predicates = DfsPredicates::Enter(
+        [&](const NodeDef* node) { return !will_execute; });
+
+    DfsTraversal(graph_topo_view, {&func_body_node},
+                 TraversalDirection::kFollowOutputs, predicates, callbacks);
+
+    if (!will_execute && !aggressive) {
+      return errors::Internal(
+          "Can't guarantee execution of a side-effectful node, that is not "
+          "reachable from function outputs. Function body node: ",
+          SummarizeNodeDef(func_body_node));
+    }
+
+    if (!will_execute && aggressive) {
+      LOG(WARNING)
+          << "Can't guarantee execution of a side-effectful node, that is not "
+             "reachable from function outputs. Function body node: "
+          << SummarizeNodeDef(func_body_node);
+    }
+  }
+
+  return Status::OK();
+}
+
+Status PlaceInlinedFunctionBody(
+    const FunctionOptimizerContext& ctx, const NodeDef& func_node,
+    const GrapplerFunctionItem& item,
+    const absl::flat_hash_map<absl::string_view, int>& input_placeholders_idx,
+    GraphDef* placed_graph_def) {
+  // Control flow lowering and Placer works with a Graph object.
+  std::unique_ptr<Graph> func_body_graph =
+      absl::make_unique<Graph>(ctx.function_library());
+
+  GraphConstructorOptions opts;
+  TF_RETURN_IF_ERROR(
+      ConvertGraphDefToGraph(opts, item.graph, func_body_graph.get()));
+
+  // TODO(ezhulenev): Lower If/While ops.
+
+  // ------------------------------------------------------------------------ //
+  // Before placing the function body nodes we pin input placeholders to the
+  // same device as their corresponding input nodes.
+
+  for (Node* func_body_node : func_body_graph->nodes()) {
+    const auto input_placeholder_idx =
+        input_placeholders_idx.find(func_body_node->name());
+
+    if (input_placeholder_idx != input_placeholders_idx.end()) {
+      const int input_idx = input_placeholder_idx->second;
+      const GraphView::OutputPort output_port =
+          ctx.graph_view().GetRegularFanin({&func_node, input_idx});
+
+      VLOG(3) << "Pin inlined function input node '" << func_body_node->name()
+              << "' to the '" << output_port.node->device() << "' device.";
+      func_body_node->set_requested_device(output_port.node->device());
+    }
+  }
+
+  // ------------------------------------------------------------------------ //
+  // After placing nodes corresponding to the function inputs, we need to assign
+  // device placements to all other function body nodes.
+
+  const DeviceSet* devices = ctx.devices();
+
+  if (devices->devices().empty()) {
+    // If there are no devices available for placer, we just put all nodes to
+    // the same device as a function caller node. This can happen if Grappler is
+    // running "offline", without active runtime session, for example as a part
+    // of a batch job for graph analysis/optimization.
+    VLOG(3) << "Assign function call node device to all function body nodes. "
+            << "Device: " << func_node.device();
+    for (Node* func_body_node : func_body_graph->nodes()) {
+      func_body_node->set_requested_device(func_node.device());
+    }
+  } else {
+    // If we are running in an active runtime session, Grappler will get the
+    // graph after initial placing is done, and we should have devices for the
+    // placer.
+    VLOG(3) << "Run placer for instantiated function body. Devices: ["
+            << absl::StrJoin(
+                   devices->devices(), ", ",
+                   [](string* out, const Device* d) { out->append(d->name()); })
+            << "]";
+
+    // Use function caller node device as a default for placer.
+    const Device* default_device =
+        devices->FindDeviceByName(func_node.device());
+
+    Placer placer(func_body_graph.get(), devices,
+                  nullptr /* No session options */, default_device);
+    TF_RETURN_IF_ERROR(placer.Run());
+  }
+
+  // Convert Graph back to the placed GraphDef.
+  func_body_graph->ToGraphDef(placed_graph_def);
+
+  return Status::OK();
+}
+
 Status InlineIndirectFunctionCall(const NodeDef& func_node,
                                   const FunctionDef& func,
                                   const int graph_def_version,
@@ -1457,26 +1615,6 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
     inputs.push_back(tensor_id);
   }
 
-  // If we have a node inside the function body without inputs (e.g. Const), we
-  // must attach a control dependency to it, to make sure that if a function
-  // call happens inside a loop, the node will be evaluated in correct frame.
-  //
-  // If the function call node has no inputs and no control dependencies, it
-  // means that it can't be a function call inside a loop, and we can safely
-  // insert that node without inputs into the main graph.
-  //
-  // TODO(ezhulenev): Use FrameMap (see grappler/utils/frame.h) to find out if
-  // the function is called inside a loop.
-  std::vector<string> empty_inputs_hook;
-  if (!item.inputs().empty()) {
-    const InputArgExpansion& arg0 = item.inputs()[0];
-    DCHECK(!arg0.placeholders.empty());
-    empty_inputs_hook.push_back(AsControlDependency(AddPrefixToNodeName(
-        arg0.placeholders[0], /*prefix=*/func_node.name())));
-  } else if (!happens_before.empty()) {
-    empty_inputs_hook.push_back(AsControlDependency(happens_before[0]));
-  }
-
   // Mapping from input placeholder name to function input position.
   absl::flat_hash_map<absl::string_view, int> input_placeholders_idx;
   for (const InputArgExpansion& input_arg : item.inputs()) {
@@ -1489,70 +1627,88 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
   const string prefix = strings::StrCat(func_node.name(), "/");
 
   // ------------------------------------------------------------------------ //
-  // Before placing the function body nodes we pin input placeholders to the
-  // same device as their corresponding input nodes.
+  // For each function output value we added an identity node that reads the
+  // tensor from one of the function body nodes. When we inline function into
+  // the main graph we want to bypass these nodes, so we keep a mapping from
+  // 'output node name' -> 'output tensor name'.
+  absl::flat_hash_map<string, string> output_tensors;
 
-  for (NodeDef& func_body_node : *item.graph.mutable_node()) {
-    const auto input_placeholder_idx =
-        input_placeholders_idx.find(func_body_node.name());
+  // Unique names of nodes producing tensors in `output_tensors`.
+  absl::flat_hash_set<string> output_tensors_nodes;
 
-    if (input_placeholder_idx != input_placeholders_idx.end()) {
-      const int input_idx = input_placeholder_idx->second;
-      const GraphView::OutputPort output_port =
-          ctx->graph_view().GetRegularFanin({&func_node, input_idx});
+  // Identity nodes added to the function body in place of function outputs.
+  absl::flat_hash_set<string> output_nodes;
+  for (const OutputArgExpansion& output_arg : item.outputs()) {
+    for (const string& output_node : output_arg.output_nodes) {
+      output_nodes.insert(output_node);
+    }
+  }
 
-      VLOG(3) << "Pin inlined function input node '" << func_body_node.name()
-              << "' to the '" << output_port.node->device() << "' device.";
-      func_body_node.set_device(output_port.node->device());
+  for (const NodeDef& func_body_node : item.graph.node()) {
+    const string& node_name = func_body_node.name();
+
+    if (IsIdentity(func_body_node) && output_nodes.count(node_name)) {
+      const string& output_tensor = func_body_node.input(0);
+      output_tensors.emplace(node_name, output_tensor);
+
+      SafeTensorId tensor_id = ParseTensorName(output_tensor);
+      output_tensors_nodes.insert(tensor_id.node());
     }
   }
 
   // ------------------------------------------------------------------------ //
-  // After placing nodes corresponding to the function inputs, we need to assign
-  // device placements to all other function body nodes.
+  // To guarantee side-effects execution order we add NoOp control_input and
+  // control_output nodes:
+  // 1) 'control_input' node will have incoming control edges from all nodes in
+  //    'happens_before' set.
+  // 2) 'control_output' node will have outgoing control edges to all nodes in
+  //    'happens_after' set.
 
-  GraphDef placed_graph_def;
+  NodeDef* control_input = nullptr;
+  NodeDef* control_output = nullptr;
 
-  const DeviceSet* devices = ctx->devices();
+  // IMPORTANT: Actual control inputs will be added to these nodes at the very
+  // last stage, because we don't want to have invalid edges in a function body
+  // graph (control edges depend on the nodes in the "outer" optimized graph).
 
-  if (devices->devices().empty()) {
-    // If there are no devices available for placer, we just put all nodes to
-    // the same device as a function caller node. This can happen if Grappler is
-    // running "offline", without active runtime session, for example as a part
-    // of a batch job for graph analysis/optimization.
-    VLOG(3) << "Assign function call node device to all function body nodes. "
-            << "Device: " << func_node.device();
-    placed_graph_def = item.mutable_function_body();
-    for (NodeDef& node : *placed_graph_def.mutable_node()) {
-      node.set_device(func_node.device());
-    }
-  } else {
-    // If we are running in an active runtime session, Grappler will get the
-    // graph after initial placing is done, and we should have devices for the
-    // placer.
-    VLOG(3) << "Run placer for instantiated function body. Devices: ["
-            << absl::StrJoin(
-                   devices->devices(), ", ",
-                   [](string* out, const Device* d) { out->append(d->name()); })
-            << "]";
+  if (!happens_before.empty()) {
+    control_input = item.graph.add_node();
+    control_input->set_op("NoOp");
+    control_input->set_name(kControlInputNodeName);
+  }
 
-    // Construct a Graph object from the instantiated function body.
-    GraphConstructorOptions opts;
-    Graph graph(ctx->function_library());
-    TF_RETURN_IF_ERROR(
-        ConvertGraphDefToGraph(opts, item.function_body(), &graph));
+  if (!happens_after.empty()) {
+    control_output = item.graph.add_node();
+    control_output->set_op("NoOp");
+    control_output->set_name(kControlOutputNodeName);
+  }
 
-    // Use function caller node device as a default for placer.
-    const Device* default_device =
-        devices->FindDeviceByName(func_node.device());
+  // ------------------------------------------------------------------------ //
+  // If we have a node inside the function body without inputs (e.g. Const), we
+  // must attach a control dependency to it, to make sure that if a function
+  // call happens inside a loop, the node will be evaluated in correct frame.
+  //
+  // If the function call node has no inputs and no control dependencies, it
+  // means that it can't be a function call inside a loop, and we can safely
+  // insert that node without inputs into the main graph.
+  //
+  // TODO(ezhulenev): Use FrameMap (see grappler/utils/frame.h) to find out if
+  // the function is called inside a loop.
+  std::vector<string> empty_inputs_hook;
+  if (!item.inputs().empty()) {
+    const InputArgExpansion& arg0 = item.inputs()[0];
+    empty_inputs_hook.push_back(arg0.placeholders[0]);
+  } else if (control_input != nullptr) {
+    empty_inputs_hook.push_back(control_input->name());
+  }
 
-    Placer placer(&graph, devices, nullptr, /* No session options */
-                  default_device);
-    TF_RETURN_IF_ERROR(placer.Run());
+  // ------------------------------------------------------------------------ //
+  // Grappler called after PRE_PLACEMENT and PLACEMENT passes, so we have to
+  // make sure that after inlining all nodes will have valid device assignment.
 
-    // Convert Graph back to the GraphDef.
-    graph.ToGraphDef(&placed_graph_def);
-  }
+  GraphDef placed_graph_def;
+  TF_RETURN_IF_ERROR(PlaceInlinedFunctionBody(
+      *ctx, func_node, item, input_placeholders_idx, &placed_graph_def));
 
   // ------------------------------------------------------------------------ //
   // After all nodes placed we need to prepare them for inlining into the
@@ -1578,17 +1734,21 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
       func_body_node.add_input(inputs[input_idx].ToString());
 
       // All side effects must happen before inputs can start executing.
-      for (const string& hb_node : happens_before) {
-        func_body_node.add_input(AsControlDependency(hb_node));
+      if (control_input) {
+        func_body_node.add_input(
+            AsControlDependency(inlined_node_name(control_input->name())));
       }
-
     } else {
       // Update inputs of the regular function body nodes.
       for (string& input : *func_body_node.mutable_input()) {
         input = inlined_node_name(input);
       }
-      if (func_body_node.input_size() == 0 && !empty_inputs_hook.empty()) {
-        *func_body_node.add_input() = empty_inputs_hook[0];
+      // Add control input to ensure node executed in correct frame.
+      if (func_body_node.input_size() == 0 && !empty_inputs_hook.empty() &&
+          func_body_node.name() != kControlInputNodeName &&
+          func_body_node.name() != kControlOutputNodeName) {
+        *func_body_node.add_input() =
+            AsControlDependency(inlined_node_name(empty_inputs_hook[0]));
       }
     }
 
@@ -1606,97 +1766,85 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
     AddDefaultsToNodeDef(*op_def, &func_body_node);
   }
 
-  // Construct a graph view for the preprocessed function body graph.
-  GraphView placed_graph_view(&placed_graph_def);
+  // ------------------------------------------------------------------------ //
+  // Check that after inlining all side-effects will be executed in well defined
+  // order. We do it by checking if there is a path from stateful/dataset ops to
+  // one of the output nodes.
+
+  // Because we rename all the nodes before inlining, we need a copy of
+  // output_nodes with a new names.
+  absl::flat_hash_set<string> inlined_output_nodes;
+  for (const string& output_node : output_nodes) {
+    inlined_output_nodes.insert(inlined_node_name(output_node));
+  }
+  const auto is_inlined_output_node = [&](const NodeDef& node) -> bool {
+    return inlined_output_nodes.find(node.name()) != inlined_output_nodes.end();
+  };
 
-  // Keep track of side-effectful ops inside function body. Each outgoing
-  // control edge from the function call node, must be replaced with control
-  // edges from inlined side-effectful ops.
-  std::vector<string> side_effectful_nodes;
+  // Construct a graph topology view for DFS traversals (skip invalid edges for
+  // input nodes connected to nodes in the optimized graph).
+  GraphTopologyView placed_topo_view(/*skip_invalid_edges=*/true);
+  TF_RETURN_IF_ERROR(placed_topo_view.InitializeFromGraph(placed_graph_def));
+  TF_RETURN_IF_ERROR(CheckThatSideEffectsWillExecute(*ctx, placed_topo_view,
+                                                     inlined_output_nodes));
 
-  // We have to make sure that all side-effectful and dataset-output nodes
-  // inside a function body will be executed after function inlining.
-  for (NodeDef& func_body_node : *placed_graph_def.mutable_node()) {
-    const bool node_must_execute =
-        !IsFreeOfSideEffect(func_body_node, &ctx->function_library()) ||
-        IsDataset(func_body_node);
-
-    if (node_must_execute) {
-      int num_fanouts = placed_graph_view.NumFanouts(
-          func_body_node, /*include_controlled_nodes=*/true);
-
-      // If the node doesn't have any outgoing edges and we do not have any
-      // nodes in the `happens_after` set, we can't inline a function and
-      // guarantee that it will be executed. The only exception if we do
-      // function library optimization, and the GrapplerItem was instantiated
-      // for the function body, because functions do not prune these ops.
-
-      if (num_fanouts == 0 && happens_after.empty() &&
-          !ctx->optimization_options().is_function_instantiation) {
-        return errors::Internal(
-            "Can't inline a function with a side-effectful op with empty "
-            "fanouts and empty output control edge set. Function body node: ",
-            SummarizeNodeDef(func_body_node));
-      }
+  // ------------------------------------------------------------------------ //
+  // Move all the nodes to the optimized graph after successful preprocessing.
 
-      side_effectful_nodes.push_back(func_body_node.name());
+  if (control_input != nullptr) {
+    string inlined_node = inlined_node_name(control_input->name());
+    absl::optional<int> node_idx = placed_topo_view.GetNodeIndex(inlined_node);
+
+    for (const string& node_name : happens_before) {
+      placed_graph_def.mutable_node(*node_idx)->add_input(
+          AsControlDependency(node_name));
     }
   }
 
-  // Identity nodes added to the function body in place of function outputs.
-  absl::flat_hash_set<string> output_nodes;
-  for (const OutputArgExpansion& output_arg : item.outputs()) {
-    for (const string& output_node : output_arg.output_nodes) {
-      output_nodes.insert(inlined_node_name(output_node));
+  if (control_output != nullptr) {
+    string inlined_node = inlined_node_name(control_output->name());
+    absl::optional<int> node_idx = placed_topo_view.GetNodeIndex(inlined_node);
+
+    // Add control edges from all nodes producing output tensors.
+    for (const string& node_name : output_tensors_nodes) {
+      placed_graph_def.mutable_node(*node_idx)->add_input(
+          AsControlDependency(inlined_node_name(node_name)));
     }
-  }
 
-  // For each function output value we added an identity node that reads the
-  // tensor from one of the function body nodes. When we inline function into
-  // the main graph we want to bypass these nodes, so we keep a mapping from
-  // 'output node name' -> 'output tensor name'.
-  absl::flat_hash_map<string, string> output_tensors;
+    // Forward all control dependencies in the optimized graph to the new node.
+    ctx->AddControlOverrides(func_node, {inlined_node});
+  }
 
-  // Move all the nodes to the optimized graph after successful preprocessing.
   for (NodeDef& func_body_node : *placed_graph_def.mutable_node()) {
-    const string& node_name = func_body_node.name();
-
-    // Skip output identity node, and add a mapping to the output tensor.
-    if (IsIdentity(func_body_node) && output_nodes.count(node_name)) {
-      output_tensors.emplace(node_name, func_body_node.input(0));
+    // Skip output identity nodes.
+    if (IsIdentity(func_body_node) && is_inlined_output_node(func_body_node))
       continue;
-    }
 
     optimized_graph->add_node()->Swap(&func_body_node);
   }
 
-  DCHECK(output_tensors.size() == item.output_size())
-      << "Each function output must be mapped to an output tensor";
-
-  // TODO(ezhulenev): Inline nested indirect function calls.
-
   // Indirect function call is fully inlined into the optimized graph, and we do
   // not copy the original function call node, so we have to setup tensor
   // mapping from old output tensors, to the outputs of inlined nodes.
   int output_idx = 0;
   for (const OutputArgExpansion& output : item.outputs()) {
     for (const string& output_node : output.output_nodes) {
-      const string inlined_output = inlined_node_name(output_node);
-      const string& output_tensor = output_tensors.at(inlined_output);
+      const string& output_tensor = output_tensors.at(output_node);
 
       const SafeTensorId from_tensor(func_node.name(), output_idx++);
       const SafeTensorId to_tensor = ParseTensorName(output_tensor);
 
-      ctx->AddTensorMapping(from_tensor, to_tensor);
+      const SafeTensorId inlined_to_tensor =
+          SafeTensorId(absl::StrCat(func_node.name(), "/", to_tensor.node()),
+                       to_tensor.index());
+
+      ctx->AddTensorMapping(from_tensor, inlined_to_tensor);
     }
   }
 
-  // After inlining we'll have to forward all control dependencies from function
-  // call node to all side-effectful ops inside function body.
-  ctx->AddControlOverrides(func_node, side_effectful_nodes);
-
   VLOG(3) << "Successfully inlined indirect function call: "
           << SummarizeNodeDef(func_node);
+
   return Status::OK();
 }
 
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
index 827b0658c5..a67acffd03 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
@@ -831,36 +831,43 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithControlDependencies) {
             kDevice),
 
        // Function body of a first function call inlined into the graph.
-       NDef("f1/x", "Identity", {"a:0", "^init_v"}, {{"T", DT_FLOAT}}, kDevice),
-       NDef("f1/y", "Identity", {"b:0", "^init_v"}, {{"T", DT_FLOAT}}, kDevice),
-       NDef("f1/v", "Identity", {"v:0", "^init_v"}, {{"T", DT_RESOURCE}},
+       NDef("f1/control_input", "NoOp", {"^init_v"}, {}, kDevice),
+       NDef("f1/x", "Identity", {"a:0", "^f1/control_input"}, {{"T", DT_FLOAT}},
             kDevice),
+       NDef("f1/y", "Identity", {"b:0", "^f1/control_input"}, {{"T", DT_FLOAT}},
+            kDevice),
+       NDef("f1/v", "Identity", {"v:0", "^f1/control_input"},
+            {{"T", DT_RESOURCE}}, kDevice),
        NDef("f1/one", "Const", {"^f1/x"},
             {{"dtype", DT_FLOAT}, {"value", kOne}}, kDevice),
        NDef("f1/add", "AssignAddVariableOp", {"f1/v", "f1/one"},
             {{"dtype", DT_FLOAT}}, kDevice),
        NDef("f1/mul", "Mul", {"f1/x", "f1/y", "^f1/add"}, {{"T", DT_FLOAT}},
             kDevice),
+       NDef("f1/control_output", "NoOp", {"^f1/mul"}, {}, kDevice),
 
        // Function body of a second function call also inlined into the graph,
        // and input nodes read directly from the inlined nodes of the first
        // function call.
-       NDef("f2/x", "Identity", {"f1/mul:0", "^f1/add"}, {{"T", DT_FLOAT}},
-            kDevice),
-       NDef("f2/y", "Identity", {"f1/mul:0", "^f1/add"}, {{"T", DT_FLOAT}},
-            kDevice),
-       NDef("f2/v", "Identity", {"v:0", "^f1/add"}, {{"T", DT_RESOURCE}},
-            kDevice),
+       NDef("f2/control_input", "NoOp", {"^f1/control_output"}, {}, kDevice),
+       NDef("f2/x", "Identity", {"f1/mul:0", "^f2/control_input"},
+            {{"T", DT_FLOAT}}, kDevice),
+       NDef("f2/y", "Identity", {"f1/mul:0", "^f2/control_input"},
+            {{"T", DT_FLOAT}}, kDevice),
+       NDef("f2/v", "Identity", {"v:0", "^f2/control_input"},
+            {{"T", DT_RESOURCE}}, kDevice),
        NDef("f2/one", "Const", {"^f2/x"},
             {{"dtype", DT_FLOAT}, {"value", kOne}}, kDevice),
        NDef("f2/add", "AssignAddVariableOp", {"f2/v", "f2/one"},
             {{"dtype", DT_FLOAT}}, kDevice),
        NDef("f2/mul", "Mul", {"f2/x", "f2/y", "^f2/add"}, {{"T", DT_FLOAT}},
             kDevice),
+       NDef("f2/control_output", "NoOp", {"^f2/mul"}, {}, kDevice),
 
        // Return values read directly from inlined nodes.
        NDef("out_1", "Identity", {"f2/mul:0"}, {{"T", DT_FLOAT}}, kDevice),
-       NDef("out_2", "ReadVariableOp", {"v", "^f1/add", "^f2/add"},
+       NDef("out_2", "ReadVariableOp",
+            {"v", "^f1/control_output", "^f2/control_output"},
             {{"dtype", DT_FLOAT}}, kDevice)},
 
       // Function library.
@@ -937,7 +944,8 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithDevicePlacement) {
   CompareGraphs(expected, optimized_graph);
 }
 
-TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithoutSideEffects) {
+TEST_F(FunctionOptimizerTest,
+       InlineIndirectFunctionWithControlDependencyAndNoSideEffects) {
   using test::function::NDef;
   using FDH = FunctionDefHelper;
 
@@ -998,12 +1006,16 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithoutSideEffects) {
        NDef("f1/x", "Identity", {"a:0"}, {{"T", DT_FLOAT}}, kDevice),
        NDef("f1/y", "Identity", {"b:0"}, {{"T", DT_FLOAT}}, kDevice),
        NDef("f1/mul", "Mul", {"f1/x", "f1/y"}, {{"T", DT_FLOAT}}, kDevice),
+       NDef("f1/control_output", "NoOp", {"^f1/mul"}, {}, kDevice),
 
        // Function body of a second function call also inlined into the graph,
        // and input nodes read directly from the inlined nodes of the first
        // function call, and control dependency edge removed.
-       NDef("f2/x", "Identity", {"f1/mul:0"}, {{"T", DT_FLOAT}}, kDevice),
-       NDef("f2/y", "Identity", {"f1/mul:0"}, {{"T", DT_FLOAT}}, kDevice),
+       NDef("f2/control_input", "NoOp", {"^f1/control_output"}, {}, kDevice),
+       NDef("f2/x", "Identity", {"f1/mul:0", "^f2/control_input"},
+            {{"T", DT_FLOAT}}, kDevice),
+       NDef("f2/y", "Identity", {"f1/mul:0", "^f2/control_input"},
+            {{"T", DT_FLOAT}}, kDevice),
        NDef("f2/mul", "Mul", {"f2/x", "f2/y"}, {{"T", DT_FLOAT}}, kDevice),
 
        // Return directly from inlined node of f2.
-- 
GitLab


From bfcddf733e85c790921afccec4ca80849e9eb9ab Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Tue, 22 Jan 2019 12:22:53 -0800
Subject: [PATCH 1067/2345] Refactor Python utility functions from Python
 wrapper. - Moves ErrorReporter to a separate class. - Moves some utility
 functions to a separate class.

PiperOrigin-RevId: 230387612
---
 .../lite/python/interpreter_wrapper/BUILD     | 23 +++++
 .../interpreter_wrapper.cc                    | 95 ++-----------------
 .../interpreter_wrapper/interpreter_wrapper.i |  1 +
 .../python_error_reporter.cc                  | 43 +++++++++
 .../python_error_reporter.h                   | 49 ++++++++++
 .../interpreter_wrapper/python_utils.cc       | 77 +++++++++++++++
 .../python/interpreter_wrapper/python_utils.h | 38 ++++++++
 7 files changed, 237 insertions(+), 89 deletions(-)
 create mode 100644 tensorflow/lite/python/interpreter_wrapper/python_error_reporter.cc
 create mode 100644 tensorflow/lite/python/interpreter_wrapper/python_error_reporter.h
 create mode 100644 tensorflow/lite/python/interpreter_wrapper/python_utils.cc
 create mode 100644 tensorflow/lite/python/interpreter_wrapper/python_utils.h

diff --git a/tensorflow/lite/python/interpreter_wrapper/BUILD b/tensorflow/lite/python/interpreter_wrapper/BUILD
index 767a9fc476..6de6fb48f7 100644
--- a/tensorflow/lite/python/interpreter_wrapper/BUILD
+++ b/tensorflow/lite/python/interpreter_wrapper/BUILD
@@ -11,6 +11,8 @@ cc_library(
     srcs = ["interpreter_wrapper.cc"],
     hdrs = ["interpreter_wrapper.h"],
     deps = [
+        ":python_error_reporter",
+        ":python_utils",
         "//tensorflow/lite:framework",
         "//tensorflow/lite/kernels:builtin_ops",
         "//third_party/py/numpy:headers",
@@ -19,6 +21,27 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "python_error_reporter",
+    srcs = ["python_error_reporter.cc"],
+    hdrs = ["python_error_reporter.h"],
+    deps = [
+        "//tensorflow/lite/core/api",
+        "//third_party/python_runtime:headers",
+    ],
+)
+
+cc_library(
+    name = "python_utils",
+    srcs = ["python_utils.cc"],
+    hdrs = ["python_utils.h"],
+    deps = [
+        "//tensorflow/lite:framework",
+        "//third_party/py/numpy:headers",
+        "//third_party/python_runtime:headers",
+    ],
+)
+
 tf_py_wrap_cc(
     name = "tensorflow_wrap_interpreter_wrapper",
     srcs = [
diff --git a/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.cc b/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.cc
index d14af439ec..9ccaabbfe9 100644
--- a/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.cc
+++ b/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.cc
@@ -21,6 +21,8 @@ limitations under the License.
 #include "tensorflow/lite/interpreter.h"
 #include "tensorflow/lite/kernels/register.h"
 #include "tensorflow/lite/model.h"
+#include "tensorflow/lite/python/interpreter_wrapper/python_error_reporter.h"
+#include "tensorflow/lite/python/interpreter_wrapper/python_utils.h"
 
 // Disallow Numpy 1.7 deprecated symbols.
 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
@@ -60,36 +62,6 @@ limitations under the License.
 namespace tflite {
 namespace interpreter_wrapper {
 
-class PythonErrorReporter : public tflite::ErrorReporter {
- public:
-  PythonErrorReporter() {}
-
-  // Report an error message
-  int Report(const char* format, va_list args) override {
-    char buf[1024];
-    int formatted = vsnprintf(buf, sizeof(buf), format, args);
-    buffer_ << buf;
-    return formatted;
-  }
-
-  // Set's a Python runtime exception with the last error.
-  PyObject* exception() {
-    std::string last_message = message();
-    PyErr_SetString(PyExc_RuntimeError, last_message.c_str());
-    return nullptr;
-  }
-
-  // Gets the last error message and clears the buffer.
-  std::string message() {
-    std::string value = buffer_.str();
-    buffer_.clear();
-    return value;
-  }
-
- private:
-  std::stringstream buffer_;
-};
-
 namespace {
 
 // Calls PyArray's initialization to initialize all the API pointers. Note that
@@ -114,61 +86,6 @@ std::unique_ptr<tflite::Interpreter> CreateInterpreter(
   return interpreter;
 }
 
-int TfLiteTypeToPyArrayType(TfLiteType tf_lite_type) {
-  switch (tf_lite_type) {
-    case kTfLiteFloat32:
-      return NPY_FLOAT32;
-    case kTfLiteInt32:
-      return NPY_INT32;
-    case kTfLiteInt16:
-      return NPY_INT16;
-    case kTfLiteUInt8:
-      return NPY_UINT8;
-    case kTfLiteInt8:
-      return NPY_INT8;
-    case kTfLiteInt64:
-      return NPY_INT64;
-    case kTfLiteString:
-      return NPY_OBJECT;
-    case kTfLiteBool:
-      return NPY_BOOL;
-    case kTfLiteComplex64:
-      return NPY_COMPLEX64;
-    case kTfLiteNoType:
-      return NPY_NOTYPE;
-      // Avoid default so compiler errors created when new types are made.
-  }
-  return NPY_NOTYPE;
-}
-
-TfLiteType TfLiteTypeFromPyArray(PyArrayObject* array) {
-  int pyarray_type = PyArray_TYPE(array);
-  switch (pyarray_type) {
-    case NPY_FLOAT32:
-      return kTfLiteFloat32;
-    case NPY_INT32:
-      return kTfLiteInt32;
-    case NPY_INT16:
-      return kTfLiteInt16;
-    case NPY_UINT8:
-      return kTfLiteUInt8;
-    case NPY_INT8:
-      return kTfLiteInt8;
-    case NPY_INT64:
-      return kTfLiteInt64;
-    case NPY_BOOL:
-      return kTfLiteBool;
-    case NPY_OBJECT:
-    case NPY_STRING:
-    case NPY_UNICODE:
-      return kTfLiteString;
-    case NPY_COMPLEX64:
-      return kTfLiteComplex64;
-      // Avoid default so compiler errors created when new types are made.
-  }
-  return kTfLiteNoType;
-}
-
 struct PyDecrefDeleter {
   void operator()(PyObject* p) const { Py_DECREF(p); }
 };
@@ -307,7 +224,7 @@ PyObject* InterpreterWrapper::TensorType(int i) const {
     return nullptr;
   }
 
-  int code = TfLiteTypeToPyArrayType(tensor->type);
+  int code = python_utils::TfLiteTypeToPyArrayType(tensor->type);
   if (code == -1) {
     PyErr_Format(PyExc_ValueError, "Invalid tflite type code %d", code);
     return nullptr;
@@ -352,12 +269,12 @@ PyObject* InterpreterWrapper::SetTensor(int i, PyObject* value) {
   PyArrayObject* array = reinterpret_cast<PyArrayObject*>(array_safe.get());
   const TfLiteTensor* tensor = interpreter_->tensor(i);
 
-  if (TfLiteTypeFromPyArray(array) != tensor->type) {
+  if (python_utils::TfLiteTypeFromPyArray(array) != tensor->type) {
     PyErr_Format(PyExc_ValueError,
                  "Cannot set tensor:"
                  " Got tensor of type %d"
                  " but expected type %d for input %d ",
-                 TfLiteTypeFromPyArray(array), tensor->type, i);
+                 python_utils::TfLiteTypeFromPyArray(array), tensor->type, i);
     return nullptr;
   }
 
@@ -400,7 +317,7 @@ PyObject* CheckGetTensorArgs(Interpreter* interpreter_, int tensor_index,
     return nullptr;
   }
 
-  *type_num = TfLiteTypeToPyArrayType((*tensor)->type);
+  *type_num = python_utils::TfLiteTypeToPyArrayType((*tensor)->type);
   if (*type_num == -1) {
     PyErr_SetString(PyExc_ValueError, "Unknown tensor type.");
     return nullptr;
diff --git a/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.i b/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.i
index f52ef1eeca..ef4b28f047 100644
--- a/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.i
+++ b/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.i
@@ -21,6 +21,7 @@ limitations under the License.
 #include "tensorflow/lite/interpreter.h"
 #include "tensorflow/lite/model.h"
 #include "tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.h"
+#include "tensorflow/lite/python/interpreter_wrapper/python_error_reporter.h"
 %}
 
 
diff --git a/tensorflow/lite/python/interpreter_wrapper/python_error_reporter.cc b/tensorflow/lite/python/interpreter_wrapper/python_error_reporter.cc
new file mode 100644
index 0000000000..803a4c2934
--- /dev/null
+++ b/tensorflow/lite/python/interpreter_wrapper/python_error_reporter.cc
@@ -0,0 +1,43 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/python/interpreter_wrapper/python_error_reporter.h"
+
+namespace tflite {
+namespace interpreter_wrapper {
+
+// Report an error message
+int PythonErrorReporter::Report(const char* format, va_list args) {
+  char buf[1024];
+  int formatted = vsnprintf(buf, sizeof(buf), format, args);
+  buffer_ << buf;
+  return formatted;
+}
+
+// Set's a Python runtime exception with the last error.
+PyObject* PythonErrorReporter::exception() {
+  std::string last_message = message();
+  PyErr_SetString(PyExc_RuntimeError, last_message.c_str());
+  return nullptr;
+}
+
+// Gets the last error message and clears the buffer.
+std::string PythonErrorReporter::message() {
+  std::string value = buffer_.str();
+  buffer_.clear();
+  return value;
+}
+}  // namespace interpreter_wrapper
+}  // namespace tflite
diff --git a/tensorflow/lite/python/interpreter_wrapper/python_error_reporter.h b/tensorflow/lite/python/interpreter_wrapper/python_error_reporter.h
new file mode 100644
index 0000000000..7d4e308834
--- /dev/null
+++ b/tensorflow/lite/python/interpreter_wrapper/python_error_reporter.h
@@ -0,0 +1,49 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_PYTHON_INTERPRETER_WRAPPER_PYTHON_ERROR_REPORTER_H_
+#define TENSORFLOW_LITE_PYTHON_INTERPRETER_WRAPPER_PYTHON_ERROR_REPORTER_H_
+
+#include <Python.h>
+
+#include <sstream>
+#include <string>
+
+#include "tensorflow/lite/core/api/error_reporter.h"
+
+namespace tflite {
+namespace interpreter_wrapper {
+
+class PythonErrorReporter : public tflite::ErrorReporter {
+ public:
+  PythonErrorReporter() {}
+
+  // Report an error message
+  int Report(const char* format, va_list args) override;
+
+  // Sets a Python runtime exception with the last error and
+  // clears the error message buffer.
+  PyObject* exception();
+
+  // Gets the last error message and clears the buffer.
+  std::string message();
+
+ private:
+  std::stringstream buffer_;
+};
+
+}  // namespace interpreter_wrapper
+}  // namespace tflite
+#endif  // TENSORFLOW_LITE_PYTHON_INTERPRETER_WRAPPER_PYTHON_ERROR_REPORTER_H_
diff --git a/tensorflow/lite/python/interpreter_wrapper/python_utils.cc b/tensorflow/lite/python/interpreter_wrapper/python_utils.cc
new file mode 100644
index 0000000000..2dc604356a
--- /dev/null
+++ b/tensorflow/lite/python/interpreter_wrapper/python_utils.cc
@@ -0,0 +1,77 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/python/interpreter_wrapper/python_utils.h"
+
+namespace tflite {
+namespace python_utils {
+
+int TfLiteTypeToPyArrayType(TfLiteType tf_lite_type) {
+  switch (tf_lite_type) {
+    case kTfLiteFloat32:
+      return NPY_FLOAT32;
+    case kTfLiteInt32:
+      return NPY_INT32;
+    case kTfLiteInt16:
+      return NPY_INT16;
+    case kTfLiteUInt8:
+      return NPY_UINT8;
+    case kTfLiteInt8:
+      return NPY_INT8;
+    case kTfLiteInt64:
+      return NPY_INT64;
+    case kTfLiteString:
+      return NPY_OBJECT;
+    case kTfLiteBool:
+      return NPY_BOOL;
+    case kTfLiteComplex64:
+      return NPY_COMPLEX64;
+    case kTfLiteNoType:
+      return NPY_NOTYPE;
+      // Avoid default so compiler errors created when new types are made.
+  }
+  return NPY_NOTYPE;
+}
+
+TfLiteType TfLiteTypeFromPyArray(PyArrayObject* array) {
+  int pyarray_type = PyArray_TYPE(array);
+  switch (pyarray_type) {
+    case NPY_FLOAT32:
+      return kTfLiteFloat32;
+    case NPY_INT32:
+      return kTfLiteInt32;
+    case NPY_INT16:
+      return kTfLiteInt16;
+    case NPY_UINT8:
+      return kTfLiteUInt8;
+    case NPY_INT8:
+      return kTfLiteInt8;
+    case NPY_INT64:
+      return kTfLiteInt64;
+    case NPY_BOOL:
+      return kTfLiteBool;
+    case NPY_OBJECT:
+    case NPY_STRING:
+    case NPY_UNICODE:
+      return kTfLiteString;
+    case NPY_COMPLEX64:
+      return kTfLiteComplex64;
+      // Avoid default so compiler errors created when new types are made.
+  }
+  return kTfLiteNoType;
+}
+
+}  // namespace python_utils
+}  // namespace tflite
diff --git a/tensorflow/lite/python/interpreter_wrapper/python_utils.h b/tensorflow/lite/python/interpreter_wrapper/python_utils.h
new file mode 100644
index 0000000000..30a44226b8
--- /dev/null
+++ b/tensorflow/lite/python/interpreter_wrapper/python_utils.h
@@ -0,0 +1,38 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_PYTHON_INTERPRETER_WRAPPER_PYTHON_UTILS_H_
+#define TENSORFLOW_LITE_PYTHON_INTERPRETER_WRAPPER_PYTHON_UTILS_H_
+
+#include "tensorflow/lite/context.h"
+
+// Disallow Numpy 1.7 deprecated symbols.
+#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
+
+#include <Python.h>
+
+#include "numpy/arrayobject.h"
+#include "numpy/ufuncobject.h"
+
+namespace tflite {
+namespace python_utils {
+
+int TfLiteTypeToPyArrayType(TfLiteType tf_lite_type);
+
+TfLiteType TfLiteTypeFromPyArray(PyArrayObject* array);
+
+}  // namespace python_utils
+}  // namespace tflite
+#endif  // TENSORFLOW_LITE_PYTHON_INTERPRETER_WRAPPER_PYTHON_UTILS_H_
-- 
GitLab


From 27c2f0e7e52d0919ad9c526447f036c49a4a601e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 12:41:06 -0800
Subject: [PATCH 1068/2345] Remove NCCL2 remote config repo, it is no longer
 being used because NCCL is built from source.

PiperOrigin-RevId: 230390673
---
 tensorflow/opensource_only.files              |  1 -
 .../preconfig/ubuntu14.04/nccl2/BUILD         | 25 -------------------
 .../preconfig/ubuntu14.04/nccl2/WORKSPACE     |  2 --
 3 files changed, 28 deletions(-)
 delete mode 100755 third_party/toolchains/preconfig/ubuntu14.04/nccl2/BUILD
 delete mode 100644 third_party/toolchains/preconfig/ubuntu14.04/nccl2/WORKSPACE

diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index 9c4dd2f4b8..c546a34cf8 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -47,7 +47,6 @@ tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/BUI
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/build_defs.bzl
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu14.04/nccl2/BUILD
 tensorflow/third_party/toolchains/preconfig/generate/workspace.bzl
 tensorflow/third_party/toolchains/preconfig/generate/containers.bzl
 tensorflow/third_party/toolchains/preconfig/generate/generate.bzl
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/nccl2/BUILD b/third_party/toolchains/preconfig/ubuntu14.04/nccl2/BUILD
deleted file mode 100755
index 96ed60d3cf..0000000000
--- a/third_party/toolchains/preconfig/ubuntu14.04/nccl2/BUILD
+++ /dev/null
@@ -1,25 +0,0 @@
-filegroup(
-    name = "LICENSE",
-    visibility = ["//visibility:public"],
-)
-
-cc_library(
-    name = "nccl",
-    srcs = ["libnccl.so.2"],
-    hdrs = ["nccl.h"],
-    include_prefix = "third_party/nccl",
-    visibility = ["//visibility:public"],
-    deps = [
-        "@local_config_cuda//cuda:cuda_headers",
-    ],
-)
-
-genrule(
-    name = "nccl-files",
-    outs = [
-        "libnccl.so.2",
-        "nccl.h",
-    ],
-    cmd = """cp "/usr/include/nccl.h" "$(@D)/nccl.h" &&
-           cp "/usr/lib/libnccl.so.2" "$(@D)/libnccl.so.2" """,
-)
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/nccl2/WORKSPACE b/third_party/toolchains/preconfig/ubuntu14.04/nccl2/WORKSPACE
deleted file mode 100644
index 1e6662ac91..0000000000
--- a/third_party/toolchains/preconfig/ubuntu14.04/nccl2/WORKSPACE
+++ /dev/null
@@ -1,2 +0,0 @@
-# DO NOT EDIT: automatically generated WORKSPACE file for nccl_configure rule
-workspace(name = "local_config_nccl")
-- 
GitLab


From e9106d123c6d75f76a86d406d036e3650371cf97 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 22 Jan 2019 12:59:02 -0800
Subject: [PATCH 1069/2345] [TF:XLA] Include the frame name in the identity of
 and recurrences

This is a correctness fix that is important for users outside the loop
(i.e. users that use that "last value" of the and recurrence).  The last value
of the and recurrence depends on how many times the loop executes, which isn't
_just_ a function of the init and step deadness predicate.

For instance if we have a loop like:

  Init = ... // Could be live or dead
  Step = ... // Could be live or dead
  V = Init
  for (int i = 0; i < N; i++) {
    V = V * Step
  }
  X = V

Then if N is 0 then X (the "loop exit" value of V) is alive iff Init was alive
(i.e. Predicate(X) == Predicate(Init)).  But if N is 1 then X is alive iff both
Init and Step are alive (i.e. Predicate(X) == Predicate(Init) &
Predicate(Step)).  We can't infer this dependence of Predicate(X) on N if
Predicate(X) == {Predicate(Init),&,Predicate(Step)} since that does not denote
the dependence on the loop trip count.  So we make the frame name (i.e. the
loop) part of the and recurrence identity.

Once I added the control flow inference logic I also noticed that quite a few of
the tests were violating frame invariants.  I've fixed those tests.

Also worth pointing out: I've not yet seen this bug cause any problems in the
wild.

PiperOrigin-RevId: 230393971
---
 tensorflow/compiler/jit/deadness_analysis.cc  | 112 +++++++--
 .../compiler/jit/deadness_analysis_test.cc    | 222 +++++++++++-------
 2 files changed, 232 insertions(+), 102 deletions(-)

diff --git a/tensorflow/compiler/jit/deadness_analysis.cc b/tensorflow/compiler/jit/deadness_analysis.cc
index 9a2ada5b42..22c265adcb 100644
--- a/tensorflow/compiler/jit/deadness_analysis.cc
+++ b/tensorflow/compiler/jit/deadness_analysis.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include "tensorflow/compiler/jit/deadness_analysis_internal.h"
 #include "tensorflow/compiler/jit/xla_cluster_util.h"
 #include "tensorflow/core/graph/algorithm.h"
+#include "tensorflow/core/graph/control_flow.h"
 #include "tensorflow/core/graph/tensor_id.h"
 #include "tensorflow/core/lib/hash/hash.h"
 
@@ -222,29 +223,40 @@ class NotPredicate : public Predicate {
   std::array<Predicate*, 1> operands_;
 };
 
-// Represents an infinite list of predicates.
+// Represents the liveness of an induction variable.  For users inside the loop
+// this represents the "current" liveness of the induction variable.  For users
+// outside the loop it represents the "last" liveness of the induction variable.
 //
-// An AndRecurrence with start = S and step = X is printed as {S,&,X} and stands
-// for the list of predicates:
+// More concretely, an and recurrence {S,&,X}<loop> represents the liveness of V
+// in the following graph:
 //
-//   S, S & GenSym(X,1), S & GenSym(X,1) & GenSym(X,2), ...
+//   V = Merge(S', V_NextIt)
+//   V = Op(V, X')
+//   V_NextIt = NextIteration(V)
 //
-// where GenSym(<expression>, <id>) renames every SymbolPredicate in
-// <expression> by appending <id> to it, in effect creating a "fresh" symbol.
-// This means {P,&,Q} is not equal to "P on the first iteration; P&Q on
-// subsequent iterations".
+// where Predicate(S') = S and Predicate(X') = X.
+//
+// `X` may contain symbolic predicates and the operations corresponding to these
+// symbolic predicates are either in frame `loop` or outside it.  The symbols
+// that are inside frame `loop` are loop variant (i.e. can have different
+// liveness in each loop iteration) and the symbols that are outside frame
+// `loop` are loop invariant (i.e. have the same liveness across all
+// iterations).
 class AndRecurrencePredicate : public Predicate {
  public:
-  explicit AndRecurrencePredicate(Predicate* start, Predicate* step)
-      : Predicate(HashPredicateSequence(Kind::kAndRecurrence, {start, step})),
-        operands_({start, step}) {}
+  explicit AndRecurrencePredicate(Predicate* start, Predicate* step,
+                                  string frame)
+      : Predicate(Hash(start, step, frame)),
+        operands_({start, step}),
+        frame_(std::move(frame)) {}
 
   Predicate* start() const { return operands_[0]; }
   Predicate* step() const { return operands_[1]; }
+  absl::string_view frame() const { return frame_; }
 
   string ToString() const override {
     return absl::StrCat("{", start()->ToString(), ",&,", step()->ToString(),
-                        "}");
+                        "}<", frame(), ">");
   }
 
   Kind kind() const override { return Kind::kAndRecurrence; }
@@ -255,6 +267,13 @@ class AndRecurrencePredicate : public Predicate {
 
  private:
   std::array<Predicate*, 2> operands_;
+  string frame_;
+
+  static int64 Hash(Predicate* start, Predicate* step, const string& frame) {
+    return Hash64Combine(
+        HashPredicateSequence(Kind::kAndRecurrence, {start, step}),
+        Hash64(frame));
+  }
 };
 
 // Represents an uninterpreted symbol in a logical predicate.
@@ -348,18 +367,22 @@ class PredicateFactory {
     return result;
   }
 
-  Predicate* MakeAndRecurrencePredicate(Predicate* start, Predicate* step) {
-    auto it = interned_and_rec_instances_.find({start, step});
+  Predicate* MakeAndRecurrencePredicate(Predicate* start, Predicate* step,
+                                        string frame) {
+    SignatureForAndRec signature(start, step, std::move(frame));
+    auto it = interned_and_rec_instances_.find(signature);
     if (it != interned_and_rec_instances_.end()) {
       return it->second.get();
     }
 
-    std::unique_ptr<Predicate> new_pred =
-        Make<AndRecurrencePredicate>(start, step);
+    std::unique_ptr<Predicate> new_pred = Make<AndRecurrencePredicate>(
+        std::get<0>(signature), std::get<1>(signature), std::get<2>(signature));
     Predicate* new_pred_ptr = new_pred.get();
-    CHECK(interned_and_rec_instances_
-              .emplace(SignatureForAndRec(start, step), std::move(new_pred))
-              .second);
+    bool inserted =
+        interned_and_rec_instances_.emplace(signature, std::move(new_pred))
+            .second;
+    (void)inserted;
+    DCHECK(inserted);
     return new_pred_ptr;
   }
 
@@ -450,7 +473,7 @@ class PredicateFactory {
   using SignatureForAndOr =
       std::pair<Predicate::Kind, absl::Span<Predicate* const>>;
   using SignatureForNot = Predicate*;
-  using SignatureForAndRec = std::pair<Predicate*, Predicate*>;
+  using SignatureForAndRec = std::tuple<Predicate*, Predicate*, string>;
   using SignatureForSymbol = std::pair<SafeTensorId, bool>;
 
   struct HashSignatureForAndOr {
@@ -720,6 +743,7 @@ class DeadnessAnalysisImpl : public DeadnessAnalysis {
   const Graph& graph_;
   absl::flat_hash_map<TensorId, Predicate*, TensorId::Hasher> predicate_map_;
   PredicateFactory predicate_factory_;
+  std::vector<ControlFlowInfo> control_flow_info_;
   bool vlog_;
 };
 
@@ -862,6 +886,28 @@ Predicate* DeduceStepPredicate(PredicateFactory* predicate_factory,
 
   return found_sym ? predicate_factory->MakeAndPredicate(and_ops) : nullptr;
 }
+
+Status GetFullFrameName(const Node* n,
+                        absl::Span<const ControlFlowInfo> cfi_infos,
+                        string* full_frame_name) {
+  int depth = 0;
+  for (const ControlFlowInfo* cfi_iter = &cfi_infos[n->id()]; !n->IsSource();
+       n = cfi_iter->parent_frame, cfi_iter = &cfi_infos[n->id()]) {
+    if (depth > 0) {
+      absl::StrAppend(full_frame_name, ";");
+    }
+
+    absl::StrAppend(full_frame_name, cfi_iter->frame_name);
+
+    if (depth++ > 5000) {
+      return errors::Internal(
+          "Frame of depth > 5000:  Probably malformed graph or a bug in "
+          "BuildControlFlowInfo");
+    }
+  }
+
+  return Status::OK();
+}
 }  // namespace
 
 Status DeadnessAnalysisImpl::HandleMerge(Node* n,
@@ -926,8 +972,11 @@ Status DeadnessAnalysisImpl::HandleMerge(Node* n,
 
         Predicate* start =
             predicate_factory_.MakeOrPredicate(non_recurrent_inputs);
-        Predicate* and_rec =
-            predicate_factory_.MakeAndRecurrencePredicate(start, step);
+        string frame_name;
+        TF_RETURN_IF_ERROR(
+            GetFullFrameName(n, control_flow_info_, &frame_name));
+        Predicate* and_rec = predicate_factory_.MakeAndRecurrencePredicate(
+            start, step, std::move(frame_name));
         SetPredicate(n, {0, 1, Graph::kControlSlot}, and_rec, should_revisit);
         return Status::OK();
       }
@@ -993,6 +1042,25 @@ Status DeadnessAnalysisImpl::Populate() {
 
 Status DeadnessAnalysisImpl::PopulateWithReversePostOrder(
     absl::Span<Node* const> rpo) {
+  std::vector<string> unreachable_nodes;
+  // Compute the loop structure of the graph.
+  std::vector<ControlFlowInfo> control_flow_info;
+  TF_RETURN_IF_ERROR(
+      BuildControlFlowInfo(&graph_, &control_flow_info_, &unreachable_nodes));
+
+  // Do some opportunistic error checking:
+  if (!unreachable_nodes.empty()) {
+    if (unreachable_nodes.size() > 5) {
+      unreachable_nodes.erase(unreachable_nodes.begin() + 5,
+                              unreachable_nodes.end());
+    }
+
+    return errors::InvalidArgument(
+        "Found unreachable nodes, most likely source and sink nodes not "
+        "connected: ",
+        absl::StrJoin(unreachable_nodes, ", "));
+  }
+
   // This an abstract interpretation over the deadness propagation semantics of
   // the graph executor.
   //
diff --git a/tensorflow/compiler/jit/deadness_analysis_test.cc b/tensorflow/compiler/jit/deadness_analysis_test.cc
index 11ee00ac6b..1adc32cf67 100644
--- a/tensorflow/compiler/jit/deadness_analysis_test.cc
+++ b/tensorflow/compiler/jit/deadness_analysis_test.cc
@@ -123,10 +123,9 @@ InductionVarInfo CreateInductionVariable(const Scope& root,
   Output increment_by = ops::Const(root.WithOpName(prefix + "/incr"), 1);
   Output final_value = ops::Const(root.WithOpName(prefix + "/final"), 10);
   Output loop_cond_expr =
-      ops::Less(root.WithOpName(prefix + "/less"), iv.output, final_value);
-  Output loop_cond =
-      ops::LoopCond(root.WithOpName(prefix + "/cond"), loop_cond_expr);
-  ops::Switch latch(root.WithOpName(prefix + "/latch"), iv.output, loop_cond);
+      ops::Less(root.WithOpName(prefix + "/cond"), iv.output, final_value);
+  ops::Switch latch(root.WithOpName(prefix + "/latch"), iv.output,
+                    loop_cond_expr);
   ops::internal::Exit exit(root.WithOpName(prefix + "/exit"),
                            latch.output_false);
   Output iv_next = ops::Add(root.WithOpName(prefix + "/ivnext"),
@@ -140,7 +139,7 @@ InductionVarInfo CreateInductionVariable(const Scope& root,
   root.graph()->AddControlEdge(iv.output.node(), increment_by.node());
   root.graph()->AddControlEdge(iv.output.node(), final_value.node());
 
-  return {iv.output, loop_cond};
+  return {iv.output, loop_cond_expr};
 }
 
 InductionVarInfo CreateInductionVariable(const Scope& root,
@@ -515,24 +514,27 @@ TEST(DeadnessAnalysisTest, Loop) {
 
     // In theory we should be able to tell that iv0/cond:0 and iv1/cond:0
     // produce the same deadness.  But we're not that smart today.
-    EXPECT_EQ(predicate_map[ControlOutputFor(iv0)], "{#true,&,*iv0/cond:0}");
-    EXPECT_EQ(predicate_map[ControlOutputFor(iv1)], "{#true,&,*iv1/cond:0}");
-    EXPECT_EQ(predicate_map[ControlOutputFor(iv2)], "{#true,&,*iv2/cond:0}");
+    EXPECT_EQ(predicate_map[ControlOutputFor(iv0)],
+              "{#true,&,*iv0/cond:0}<fr0>");
+    EXPECT_EQ(predicate_map[ControlOutputFor(iv1)],
+              "{#true,&,*iv1/cond:0}<fr0>");
+    EXPECT_EQ(predicate_map[ControlOutputFor(iv2)],
+              "{#true,&,*iv2/cond:0}<fr0>");
     EXPECT_EQ(predicate_map[ControlOutputFor(add0)],
-              "({#true,&,*iv1/cond:0} & {#true,&,*iv0/cond:0})");
+              "({#true,&,*iv1/cond:0}<fr0> & {#true,&,*iv0/cond:0}<fr0>)");
     EXPECT_EQ(predicate_map[ControlOutputFor(add1)],
-              "({#true,&,*iv1/cond:0} & {#true,&,*iv2/cond:0})");
+              "({#true,&,*iv2/cond:0}<fr0> & {#true,&,*iv1/cond:0}<fr0>)");
   }
 }
 
 TEST(DeadnessAnalysisTest, ControlEquivalentLoopBodies) {
   Scope root = Scope::NewRootScope().ExitOnError();
-  InductionVarInfo iv = CreateInductionVariable(root, "iv0", "frame", 0);
+  InductionVarInfo iv = CreateInductionVariable(root, "iv0", "loop", 0);
   Output dependent_iv0 =
-      CreateDependentLoopInvariantValue(root, "div0", "frame", iv.loop_cond, 0)
+      CreateDependentLoopInvariantValue(root, "div0", "loop", iv.loop_cond, 0)
           .induction_var;
   Output dependent_iv1 =
-      CreateDependentLoopInvariantValue(root, "div1", "frame", iv.loop_cond, 0)
+      CreateDependentLoopInvariantValue(root, "div1", "loop", iv.loop_cond, 0)
           .induction_var;
   Output add0 = ops::Add(root.WithOpName("add0"), dependent_iv0, dependent_iv1);
 
@@ -549,13 +551,13 @@ TEST(DeadnessAnalysisTest, ControlEquivalentLoopBodies) {
     TF_ASSERT_OK(ComputePredicates(*root.graph(), &predicate_map));
 
     EXPECT_EQ(predicate_map[ControlOutputFor(iv.induction_var)],
-              "{#true,&,*iv0/cond:0}");
+              "{#true,&,*iv0/cond:0}<loop>");
     EXPECT_EQ(predicate_map[ControlOutputFor(dependent_iv0)],
-              "{#true,&,(*iv0/cond:0 & iv0/iv:0)}");
+              "{#true,&,(*iv0/cond:0 & iv0/iv:0)}<loop>");
     EXPECT_EQ(predicate_map[ControlOutputFor(dependent_iv1)],
-              "{#true,&,(*iv0/cond:0 & iv0/iv:0)}");
+              "{#true,&,(*iv0/cond:0 & iv0/iv:0)}<loop>");
     EXPECT_EQ(predicate_map[ControlOutputFor(add0)],
-              "{#true,&,(*iv0/cond:0 & iv0/iv:0)}");
+              "{#true,&,(*iv0/cond:0 & iv0/iv:0)}<loop>");
   }
 }
 
@@ -595,32 +597,33 @@ TEST(DeadnessAnalysisTest, LoopInvariantPredicateOnBackedge) {
 TEST(DeadnessAnalysisTest, ControlEquivalentNestedLoopBodies) {
   Scope root = Scope::NewRootScope().ExitOnError();
   InductionVarInfo iv_outer =
-      CreateInductionVariable(root, "iv_outer", "frame", 0);
+      CreateInductionVariable(root, "iv_outer", "outer_loop", 0);
+  Output enter_constant_outer_loop = ops::internal::Enter(
+      root.WithOpName("constant_enter_outer_loop"),
+      ops::Const(root.WithOpName("constant"), 5), "outer_loop",
+      ops::internal::Enter::Attrs().IsConstant(true));
   ops::Switch inner_value(root.WithOpName("outer_is_live"),
-                          ops::Const(root.WithOpName("constant"), 5),
-                          iv_outer.loop_cond);
+                          enter_constant_outer_loop, iv_outer.loop_cond);
   InductionVarInfo iv_inner = CreateInductionVariable(
-      root, "iv_inner", "frame",
-      ops::internal::Enter(root.WithOpName("iv_inner/enter"),
-                           inner_value.output_true, "frame_inner"));
+      root, "iv_inner", "inner_loop", inner_value.output_true);
 
   Output dependent_outer_iv0 =
-      CreateDependentLoopInvariantValue(root, "dependent_outer_iv0", "frame",
-                                        iv_outer.loop_cond, 0)
+      CreateDependentLoopInvariantValue(root, "dependent_outer_iv0",
+                                        "outer_loop", iv_outer.loop_cond, 0)
           .induction_var;
   Output dependent_outer_iv1 =
-      CreateDependentLoopInvariantValue(root, "dependent_outer_iv1", "frame",
-                                        iv_outer.loop_cond, 0)
+      CreateDependentLoopInvariantValue(root, "dependent_outer_iv1",
+                                        "outer_loop", iv_outer.loop_cond, 0)
           .induction_var;
 
-  Output dependent_inner_iv0 =
-      CreateDependentLoopInvariantValue(root, "dependent_inner_iv0", "frame",
-                                        iv_inner.loop_cond, dependent_outer_iv0)
-          .induction_var;
-  Output dependent_inner_iv1 =
-      CreateDependentLoopInvariantValue(root, "dependent_inner_iv1", "frame",
-                                        iv_inner.loop_cond, dependent_outer_iv1)
-          .induction_var;
+  Output dependent_inner_iv0 = CreateDependentLoopInvariantValue(
+                                   root, "dependent_inner_iv0", "inner_loop",
+                                   iv_inner.loop_cond, dependent_outer_iv0)
+                                   .induction_var;
+  Output dependent_inner_iv1 = CreateDependentLoopInvariantValue(
+                                   root, "dependent_inner_iv1", "inner_loop",
+                                   iv_inner.loop_cond, dependent_outer_iv1)
+                                   .induction_var;
 
   Output add0 = ops::Add(root.WithOpName("add0"), dependent_inner_iv0,
                          dependent_inner_iv1);
@@ -638,46 +641,50 @@ TEST(DeadnessAnalysisTest, ControlEquivalentNestedLoopBodies) {
     TF_ASSERT_OK(ComputePredicates(*root.graph(), &predicate_map));
 
     EXPECT_EQ(predicate_map[ControlOutputFor(iv_outer.induction_var)],
-              "{#true,&,*iv_outer/cond:0}");
+              "{#true,&,*iv_outer/cond:0}<outer_loop>");
     EXPECT_EQ(predicate_map[ControlOutputFor(iv_inner.induction_var)],
-              "{(*iv_outer/cond:0 & {#true,&,*iv_outer/cond:0}),&,"
-              "*iv_inner/cond:0}");
+              "{({#true,&,*iv_outer/cond:0}<outer_loop> & "
+              "*iv_outer/cond:0),&,*iv_inner/cond:0}<inner_loop;outer_loop>");
 
     EXPECT_EQ(predicate_map[ControlOutputFor(dependent_inner_iv0)],
-              "{{#true,&,(iv_outer/iv:0 & *iv_outer/cond:0)},&,"
-              "(*iv_inner/cond:0 & iv_inner/iv:0)}");
+              "{{#true,&,(iv_outer/iv:0 & "
+              "*iv_outer/cond:0)}<outer_loop>,&,(*iv_inner/cond:0 & "
+              "iv_inner/iv:0)}<inner_loop;outer_loop>");
+
     EXPECT_EQ(predicate_map[ControlOutputFor(dependent_inner_iv1)],
-              "{{#true,&,(iv_outer/iv:0 & *iv_outer/cond:0)},&,"
-              "(*iv_inner/cond:0 & iv_inner/iv:0)}");
+              "{{#true,&,(iv_outer/iv:0 & "
+              "*iv_outer/cond:0)}<outer_loop>,&,(*iv_inner/cond:0 & "
+              "iv_inner/iv:0)}<inner_loop;outer_loop>");
     EXPECT_EQ(predicate_map[ControlOutputFor(add0)],
-              "{{#true,&,(iv_outer/iv:0 & *iv_outer/cond:0)},&,"
-              "(*iv_inner/cond:0 & iv_inner/iv:0)}");
+              "{{#true,&,(iv_outer/iv:0 & "
+              "*iv_outer/cond:0)}<outer_loop>,&,(*iv_inner/cond:0 & "
+              "iv_inner/iv:0)}<inner_loop;outer_loop>");
   }
 }
 
 TEST(DeadnessAnalysisTest, ControlNonEquivalentNestedLoopBodies) {
   Scope root = Scope::NewRootScope().ExitOnError();
-  InductionVarInfo iv_outer_0 =
-      CreateInductionVariable(root, "iv_outer_0", "frame", 0);
-  ops::Switch inner_value_0(root.WithOpName("outer_0_is_live"),
-                            ops::Const(root.WithOpName("constant"), 5),
-                            iv_outer_0.loop_cond);
-  InductionVarInfo iv_inner_0 = CreateInductionVariable(
-      root, "iv_inner_0", "frame",
-      ops::internal::Enter(root.WithOpName("iv_inner_0/enter"),
-                           inner_value_0.output_true, "frame_inner"));
-
-  InductionVarInfo iv_outer_1 =
-      CreateInductionVariable(root, "iv_outer_1", "frame", 1);
-  ops::Switch inner_init_value_1(root.WithOpName("outer_1_is_live"),
-                                 ops::Const(root.WithOpName("constant"), 5),
-                                 iv_outer_1.loop_cond);
-  InductionVarInfo iv_inner_1 = CreateInductionVariable(
-      root, "iv_inner_1", "frame",
-      ops::internal::Enter(root.WithOpName("iv_inner_1/enter"),
-                           inner_init_value_1.output_true, "frame_inner"));
-  Output add0 = ops::Add(root.WithOpName("add0"), iv_inner_0.induction_var,
-                         iv_inner_1.induction_var);
+
+  std::array<Output, 2> outer_iv;
+  std::array<Output, 2> inner_iv;
+
+  for (int i : {0, 1}) {
+    InductionVarInfo iv_outer =
+        CreateInductionVariable(root, "iv_outer", "outer_loop", 0);
+    Output enter_constant_outer_loop = ops::internal::Enter(
+        root.WithOpName("constant_enter_outer_loop"),
+        ops::Const(root.WithOpName("constant"), 5), "outer_loop",
+        ops::internal::Enter::Attrs().IsConstant(true));
+    ops::Switch inner_value(root.WithOpName("outer_is_live"),
+                            enter_constant_outer_loop, iv_outer.loop_cond);
+    InductionVarInfo iv_inner = CreateInductionVariable(
+        root, "iv_inner", "inner_loop", inner_value.output_true);
+
+    outer_iv[i] = iv_outer.induction_var;
+    inner_iv[i] = iv_inner.induction_var;
+  }
+
+  Output add0 = ops::Add(root.WithOpName("add0"), inner_iv[0], inner_iv[1]);
 
   VLogGraphIfAsked(*root.graph());
 
@@ -692,21 +699,76 @@ TEST(DeadnessAnalysisTest, ControlNonEquivalentNestedLoopBodies) {
     PredicateMapTy predicate_map;
     TF_ASSERT_OK(ComputePredicates(*root.graph(), &predicate_map));
 
-    EXPECT_EQ(predicate_map[ControlOutputFor(iv_outer_0.induction_var)],
-              "{#true,&,*iv_outer_0/cond:0}");
-    EXPECT_EQ(predicate_map[ControlOutputFor(iv_inner_0.induction_var)],
-              "{(*iv_outer_0/cond:0 & {#true,&,*iv_outer_0/cond:0}),&,"
-              "*iv_inner_0/cond:0}");
-    EXPECT_EQ(predicate_map[ControlOutputFor(iv_outer_1.induction_var)],
-              "{#true,&,*iv_outer_1/cond:0}");
-    EXPECT_EQ(predicate_map[ControlOutputFor(iv_inner_1.induction_var)],
-              "{(*iv_outer_1/cond:0 & {#true,&,*iv_outer_1/cond:0}),&,"
-              "*iv_inner_1/cond:0}");
-    EXPECT_EQ(predicate_map[ControlOutputFor(add0)],
-              "({(*iv_outer_1/cond:0 & {#true,&,*iv_outer_1/cond:0}),&,"
-              "*iv_inner_1/cond:0} & "
-              "{(*iv_outer_0/cond:0 & {#true,&,*iv_outer_0/cond:0}),&,"
-              "*iv_inner_0/cond:0})");
+    EXPECT_EQ(predicate_map[ControlOutputFor(outer_iv[0])],
+              "{#true,&,*iv_outer/cond:0}<outer_loop>");
+    EXPECT_EQ(predicate_map[ControlOutputFor(inner_iv[0])],
+              "{({#true,&,*iv_outer/cond:0}<outer_loop> & "
+              "*iv_outer/cond:0),&,*iv_inner/cond:0}<inner_loop;outer_loop>");
+    EXPECT_EQ(predicate_map[ControlOutputFor(outer_iv[1])],
+              "{#true,&,*iv_outer/cond_1:0}<outer_loop>");
+    EXPECT_EQ(
+        predicate_map[ControlOutputFor(inner_iv[1])],
+        "{({#true,&,*iv_outer/cond_1:0}<outer_loop> & "
+        "*iv_outer/cond_1:0),&,*iv_inner/cond_1:0}<inner_loop;outer_loop>");
+    EXPECT_EQ(
+        predicate_map[ControlOutputFor(add0)],
+        "({({#true,&,*iv_outer/cond_1:0}<outer_loop> & "
+        "*iv_outer/cond_1:0),&,*iv_inner/cond_1:0}<inner_loop;outer_loop> & "
+        "{({#true,&,*iv_outer/cond:0}<outer_loop> & "
+        "*iv_outer/cond:0),&,*iv_inner/cond:0}<inner_loop;outer_loop>)");
+  }
+}
+
+TEST(DeadnessAnalysisTest, AndRecurrenceNeedsFrameName) {
+  Scope root = Scope::NewRootScope().ExitOnError();
+  InductionVarInfo iv_0 = CreateInductionVariable(root, "iv_0", "frame_0", 10);
+  InductionVarInfo iv_1 = CreateInductionVariable(root, "iv_1", "frame_1", 9);
+
+  Output init = CreateSwitch(root, "init").output_true;
+  Output step = CreateSwitch(root, "step").output_true;
+
+  std::array<Output, 2> exits;
+  std::array<Output, 2> next_iterations;
+
+  for (int i : {0, 1}) {
+    Output init_enter = ops::internal::Enter(
+        root.WithOpName(absl::StrCat("init_enter_frame_", i)), init,
+        absl::StrCat("frame_", i),
+        ops::internal::Enter::Attrs().IsConstant(true));
+    Output step_enter = ops::internal::Enter(
+        root.WithOpName(absl::StrCat("step_enter_frame_", i)), step,
+        absl::StrCat("frame_", i),
+        ops::internal::Enter::Attrs().IsConstant(true));
+
+    ops::Merge iv(root.WithOpName(absl::StrCat("expr_", i)),
+                  {init_enter, init_enter});
+    Output add = ops::Add(root.WithOpName(absl::StrCat("add_", i)), iv.output,
+                          step_enter);
+    next_iterations[i] = ops::NextIteration(
+        root.WithOpName(absl::StrCat("expr_", i, "_next_iteration")), add);
+    EXPECT_TRUE(
+        root.graph()
+            ->UpdateEdge(next_iterations[i].node(), 0, iv.output.node(), 1)
+            .ok());
+    exits[i] = ops::internal::Exit(root.WithOpName(absl::StrCat("exit_", i)),
+                                   iv.output);
+  }
+
+  FixupSourceAndSinkEdges(root.graph());
+
+  {
+    PredicateMapTy predicate_map;
+    TF_ASSERT_OK(ComputePredicates(*root.graph(), &predicate_map));
+
+    EXPECT_NE(predicate_map[ControlOutputFor(exits[0])],
+              predicate_map[ControlOutputFor(exits[1])]);
+    EXPECT_NE(predicate_map[ControlOutputFor(exits[0])], "");
+    EXPECT_NE(predicate_map[ControlOutputFor(exits[1])], "");
+
+    EXPECT_NE(predicate_map[ControlOutputFor(next_iterations[0])],
+              predicate_map[ControlOutputFor(next_iterations[1])]);
+    EXPECT_NE(predicate_map[ControlOutputFor(next_iterations[0])], "");
+    EXPECT_NE(predicate_map[ControlOutputFor(next_iterations[1])], "");
   }
 }
 
-- 
GitLab


From 794b7c6713c7e576063ad10c34048933f92f4085 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Tue, 22 Jan 2019 13:13:33 -0800
Subject: [PATCH 1070/2345] Internal change.

PiperOrigin-RevId: 230396943
---
 tensorflow/cc/profiler/BUILD                | 2 +-
 tensorflow/contrib/distribute/python/BUILD  | 2 ++
 tensorflow/contrib/fused_conv/BUILD         | 1 +
 tensorflow/python/kernel_tests/BUILD        | 4 ----
 tensorflow/python/kernel_tests/linalg/BUILD | 1 -
 tensorflow/python/kernel_tests/signal/BUILD | 2 --
 tensorflow/python/ops/parallel_for/BUILD    | 1 -
 7 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/tensorflow/cc/profiler/BUILD b/tensorflow/cc/profiler/BUILD
index cf65fe1ab9..e9838d9aba 100644
--- a/tensorflow/cc/profiler/BUILD
+++ b/tensorflow/cc/profiler/BUILD
@@ -10,7 +10,7 @@ tf_cuda_cc_test(
     name = "profiler_test",
     srcs = ["profiler_test.cc"],
     tags = [
-        "noguitar",  # b/77649654
+        "nogpu",  # b/77649654
     ],
     deps = [
         ":profiler",
diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index fb8732d591..c663e32026 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -679,6 +679,7 @@ distribute_py_test(
         "no_oss",  # TODO(b/117919883): Fix python error.
         "no_pip",
         "no_windows_gpu",
+        "notap",  # TODO(b/123097760): Fix and re-enable the test.
         "notsan",
     ],
     deps = [
@@ -763,6 +764,7 @@ distribute_py_test(
         "no_oss",  # TODO(b/117919883): Fix python error.
         "no_pip",
         "no_windows_gpu",
+        "notap",  # TODO(b/123097760): Fix and re-enable the test.
         "notsan",
     ],
     deps = [
diff --git a/tensorflow/contrib/fused_conv/BUILD b/tensorflow/contrib/fused_conv/BUILD
index 57a5bfbf43..f65f450eba 100644
--- a/tensorflow/contrib/fused_conv/BUILD
+++ b/tensorflow/contrib/fused_conv/BUILD
@@ -171,6 +171,7 @@ cuda_py_test(
     main = "python/ops/fused_conv2d_bias_activation_benchmark.py",
     tags = [
         "manual",  # TODO(b/117128481): re-enable after fixing OSS build
+        "nogpu",
         "requires-gpu-sm70",
     ],
 )
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index aef4354ee0..2271d1f9f7 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -1435,7 +1435,6 @@ cuda_py_test(
         "noasan",  # times out
         "optonly",  # times out
     ],
-    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1525,7 +1524,6 @@ cuda_py_test(
         "//tensorflow/python:nn_grad",
         "//tensorflow/python:nn_ops",
     ],
-    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2756,7 +2754,6 @@ cuda_py_test(
     tags = [
         "optonly",  # times out
     ],
-    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2774,7 +2771,6 @@ cuda_py_test(
     ],
     # TODO(b/118842098): Re-enable this test in Kokoro.
     tags = ["no_oss"],
-    xla_enable_strict_auto_jit = True,
 )
 
 tf_py_test(
diff --git a/tensorflow/python/kernel_tests/linalg/BUILD b/tensorflow/python/kernel_tests/linalg/BUILD
index c1aa99cc37..b4608aa38c 100644
--- a/tensorflow/python/kernel_tests/linalg/BUILD
+++ b/tensorflow/python/kernel_tests/linalg/BUILD
@@ -150,7 +150,6 @@ cuda_py_test(
         "noasan",  # times out, b/63678675
         "optonly",  # times out, b/79171797
     ],
-    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
diff --git a/tensorflow/python/kernel_tests/signal/BUILD b/tensorflow/python/kernel_tests/signal/BUILD
index 554bf38029..3e37d230ef 100644
--- a/tensorflow/python/kernel_tests/signal/BUILD
+++ b/tensorflow/python/kernel_tests/signal/BUILD
@@ -29,7 +29,6 @@ cuda_py_tests(
         "//tensorflow/python:spectral_ops_test_util",
         "//tensorflow/python/ops/signal",
     ],
-    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_tests(
@@ -46,7 +45,6 @@ cuda_py_tests(
     ],
     shard_count = 4,
     tags = ["optonly"],
-    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_tests(
diff --git a/tensorflow/python/ops/parallel_for/BUILD b/tensorflow/python/ops/parallel_for/BUILD
index f54ca558f4..fd89d8ad45 100644
--- a/tensorflow/python/ops/parallel_for/BUILD
+++ b/tensorflow/python/ops/parallel_for/BUILD
@@ -114,7 +114,6 @@ cuda_py_test(
         "//tensorflow/python:random_ops",
         "//tensorflow/python:util",
     ],
-    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
-- 
GitLab


From da5ba1b58296512efe961dbeb76120cd933f203b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 13:27:05 -0800
Subject: [PATCH 1071/2345] Fix spelling/typo.

PiperOrigin-RevId: 230399257
---
 tensorflow/tools/api/tests/api_compatibility_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/api/tests/api_compatibility_test.py b/tensorflow/tools/api/tests/api_compatibility_test.py
index 723fceef41..6b9fc7ddce 100644
--- a/tensorflow/tools/api/tests/api_compatibility_test.py
+++ b/tensorflow/tools/api/tests/api_compatibility_test.py
@@ -59,7 +59,7 @@ _UPDATE_GOLDENS_HELP = """
 # DEFINE_boolean, only_test_core_api, default False:
 _ONLY_TEST_CORE_API_HELP = """
     Some TF APIs are being moved outside of the tensorflow/ directory. There is
-    no garuntee which versions of these APIs will be present when running this
+    no guarantee which versions of these APIs will be present when running this
     test. Therefore, do not error out on API changes in non-core TF code
     if this flag is set.
 """
-- 
GitLab


From a4d4392677f58eb26eed7e7e8ec7604d3d21335b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 13:49:06 -0800
Subject: [PATCH 1072/2345] Minor fix in comments

PiperOrigin-RevId: 230403343
---
 tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc b/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
index 5b4f863f74..61a8933208 100644
--- a/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
+++ b/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
@@ -441,12 +441,6 @@ xla::StatusOr<xla::XlaOp> MakeXlaBackpropFilterConvOp(
   std::vector<int64> window_strides(attrs.num_spatial_dims);
   std::vector<int64> ones(attrs.num_spatial_dims, 1);
 
-  // The activations (inputs) form the LHS of the convolution.
-  // Activations have shape: [batch, in_rows, in_cols, ..., in_depth]
-  // For the gradient computation, we flip the roles of the batch and
-  // feature dimensions.
-  // Each spatial entry has size in_depth * batch
-
   // Swap n_dim and c_dim in the activations.
   dnums.set_input_batch_dimension(c_dim);
   dnums.set_input_feature_dimension(n_dim);
@@ -480,7 +474,7 @@ xla::StatusOr<xla::XlaOp> MakeXlaBackpropFilterConvOp(
     // convolution, we get the right size for the filter.
     // The padded_in_rows should be such that when we convolve this with the
     // expanded_out_rows as a filter, we should get filter_rows back.
-    //
+
     const int64 padded_in_size =
         dims.spatial_dims[i].expanded_output_size +
         (dims.spatial_dims[i].filter_size - 1) * attrs.dilations[dim];
-- 
GitLab


From ea1d491e6702a191e748160be30ebadcdffc7fdd Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Tue, 22 Jan 2019 14:11:24 -0800
Subject: [PATCH 1073/2345] Move TF-TRT C++ implementations to
 compiler/tf2tensorrt.

PiperOrigin-RevId: 230407626
---
 tensorflow/compiler/tf2tensorrt/BUILD         | 440 +++++++++++++++++
 .../tf2tensorrt}/convert/convert_graph.cc     |  23 +-
 .../tf2tensorrt}/convert/convert_graph.h      |   8 +-
 .../convert/convert_graph_test.cc             |   4 +-
 .../tf2tensorrt}/convert/convert_nodes.cc     |  22 +-
 .../tf2tensorrt}/convert/convert_nodes.h      |  16 +-
 .../convert/convert_nodes_test.cc             |  11 +-
 .../convert/trt_optimization_pass.cc          |  14 +-
 .../convert/trt_optimization_pass.h           |   6 +-
 .../tf2tensorrt}/convert/utils.cc             |   2 +-
 .../tf2tensorrt}/convert/utils.h              |   6 +-
 .../kernels/get_serialized_resource_op.cc     |   8 +-
 .../get_serialized_resource_op_test.cc        |   2 +-
 .../tf2tensorrt}/kernels/trt_engine_op.cc     |  21 +-
 .../tf2tensorrt}/kernels/trt_engine_op.h      |  16 +-
 .../ops/get_serialized_resource_op.cc         |   0
 .../tf2tensorrt}/ops/trt_engine_op.cc         |   0
 .../tf2tensorrt}/plugin/trt_plugin.cc         |   6 +-
 .../tf2tensorrt}/plugin/trt_plugin.h          |   6 +-
 .../tf2tensorrt}/plugin/trt_plugin_factory.cc |   2 +-
 .../tf2tensorrt}/plugin/trt_plugin_factory.h  |  10 +-
 .../plugin/trt_plugin_factory_test.cc         |   4 +-
 .../tf2tensorrt}/plugin/trt_plugin_utils.cc   |   2 +-
 .../tf2tensorrt}/plugin/trt_plugin_utils.h    |   8 +-
 .../tf2tensorrt}/python/ops/trt_ops.py        |   6 +-
 .../tf2tensorrt}/segment/segment.cc           |   9 +-
 .../tf2tensorrt}/segment/segment.h            |   6 +-
 .../tf2tensorrt}/segment/segment_test.cc      |   2 +-
 .../tf2tensorrt}/segment/union_find.h         |   6 +-
 .../tf2tensorrt}/tensorrt_test.cc             |   0
 .../tf2tensorrt/utils/test_utils.cc}          |   2 +-
 .../tf2tensorrt/utils/test_utils.h}           |   6 +-
 .../tf2tensorrt/utils}/trt_allocator.cc       |   2 +-
 .../tf2tensorrt/utils}/trt_allocator.h        |   6 +-
 .../tf2tensorrt/utils}/trt_allocator_test.cc  |   2 +-
 .../tf2tensorrt/utils}/trt_int8_calibrator.cc |   2 +-
 .../tf2tensorrt/utils}/trt_int8_calibrator.h  |   6 +-
 .../tf2tensorrt/utils}/trt_logger.cc          |   2 +-
 .../tf2tensorrt/utils}/trt_logger.h           |   6 +-
 .../tf2tensorrt/utils}/trt_lru_cache.h        |  10 +-
 .../tf2tensorrt/utils}/trt_lru_cache_test.cc  |   2 +-
 .../utils}/trt_resource_manager.cc            |   2 +-
 .../tf2tensorrt/utils}/trt_resource_manager.h |   6 +-
 .../tf2tensorrt/utils}/trt_resources.cc       |   2 +-
 .../tf2tensorrt/utils}/trt_resources.h        |  14 +-
 tensorflow/contrib/tensorrt/BUILD             | 441 +-----------------
 .../tensorrt/custom_plugin_examples/BUILD     |   4 +-
 .../custom_plugin_examples/inc_op_plugin.cc   |   2 +-
 .../custom_plugin_examples/inc_op_plugin.h    |   2 +-
 .../contrib/tensorrt/python/__init__.py       |   2 +-
 .../tensorrt/python/trt_convert_test.py       |   4 +-
 .../contrib/tensorrt/shape_fn/trt_shfn.cc     |   4 +-
 .../tensorrt/test/quantization_mnist_test.py  |   4 +-
 .../test/tf_trt_integration_test_base.py      |   4 +-
 tensorflow/contrib/tensorrt/trt_conversion.i  |   6 +-
 55 files changed, 630 insertions(+), 579 deletions(-)
 create mode 100644 tensorflow/compiler/tf2tensorrt/BUILD
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/convert/convert_graph.cc (98%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/convert/convert_graph.h (94%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/convert/convert_graph_test.cc (98%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/convert/convert_nodes.cc (99%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/convert/convert_nodes.h (97%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/convert/convert_nodes_test.cc (99%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/convert/trt_optimization_pass.cc (96%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/convert/trt_optimization_pass.h (91%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/convert/utils.cc (97%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/convert/utils.h (88%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/kernels/get_serialized_resource_op.cc (88%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/kernels/get_serialized_resource_op_test.cc (97%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/kernels/trt_engine_op.cc (97%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/kernels/trt_engine_op.h (88%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/ops/get_serialized_resource_op.cc (100%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/ops/trt_engine_op.cc (100%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/plugin/trt_plugin.cc (96%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/plugin/trt_plugin.h (92%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/plugin/trt_plugin_factory.cc (96%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/plugin/trt_plugin_factory.h (91%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/plugin/trt_plugin_factory_test.cc (96%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/plugin/trt_plugin_utils.cc (94%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/plugin/trt_plugin_utils.h (82%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/python/ops/trt_ops.py (88%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/segment/segment.cc (99%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/segment/segment.h (92%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/segment/segment_test.cc (99%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/segment/union_find.h (92%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/tensorrt_test.cc (100%)
 rename tensorflow/{contrib/tensorrt/test/utils.cc => compiler/tf2tensorrt/utils/test_utils.cc} (97%)
 rename tensorflow/{contrib/tensorrt/test/utils.h => compiler/tf2tensorrt/utils/test_utils.h} (89%)
 rename tensorflow/{contrib/tensorrt/resources => compiler/tf2tensorrt/utils}/trt_allocator.cc (98%)
 rename tensorflow/{contrib/tensorrt/resources => compiler/tf2tensorrt/utils}/trt_allocator.h (93%)
 rename tensorflow/{contrib/tensorrt/resources => compiler/tf2tensorrt/utils}/trt_allocator_test.cc (98%)
 rename tensorflow/{contrib/tensorrt/resources => compiler/tf2tensorrt/utils}/trt_int8_calibrator.cc (98%)
 rename tensorflow/{contrib/tensorrt/resources => compiler/tf2tensorrt/utils}/trt_int8_calibrator.h (93%)
 rename tensorflow/{contrib/tensorrt/log => compiler/tf2tensorrt/utils}/trt_logger.cc (96%)
 rename tensorflow/{contrib/tensorrt/log => compiler/tf2tensorrt/utils}/trt_logger.h (86%)
 rename tensorflow/{contrib/tensorrt/resources => compiler/tf2tensorrt/utils}/trt_lru_cache.h (94%)
 rename tensorflow/{contrib/tensorrt/resources => compiler/tf2tensorrt/utils}/trt_lru_cache_test.cc (96%)
 rename tensorflow/{contrib/tensorrt/resources => compiler/tf2tensorrt/utils}/trt_resource_manager.cc (96%)
 rename tensorflow/{contrib/tensorrt/resources => compiler/tf2tensorrt/utils}/trt_resource_manager.h (87%)
 rename tensorflow/{contrib/tensorrt/resources => compiler/tf2tensorrt/utils}/trt_resources.cc (96%)
 rename tensorflow/{contrib/tensorrt/resources => compiler/tf2tensorrt/utils}/trt_resources.h (82%)

diff --git a/tensorflow/compiler/tf2tensorrt/BUILD b/tensorflow/compiler/tf2tensorrt/BUILD
new file mode 100644
index 0000000000..a67e511826
--- /dev/null
+++ b/tensorflow/compiler/tf2tensorrt/BUILD
@@ -0,0 +1,440 @@
+# Description:
+#   Wrap NVIDIA TensorRT (http://developer.nvidia.com/tensorrt) with tensorflow
+#   and provide TensorRT operators and converter package.
+#   APIs are meant to change over time.
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])  # Apache 2.0
+
+exports_files(["LICENSE"])
+
+load(
+    "//tensorflow:tensorflow.bzl",
+    "tf_cc_test",
+    "tf_copts",
+    "tf_cuda_library",
+    "tf_custom_op_library",
+    "tf_custom_op_library_additional_deps",
+    "tf_gen_op_libs",
+    "tf_gen_op_wrapper_py",
+)
+load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_test")
+load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library")
+load(
+    "@local_config_tensorrt//:build_defs.bzl",
+    "if_tensorrt",
+)
+
+tf_cuda_cc_test(
+    name = "tensorrt_test_cc",
+    size = "small",
+    srcs = ["tensorrt_test.cc"],
+    tags = [
+        "no_windows",
+        "nomac",
+    ],
+    deps = [
+        "//tensorflow/core:gpu_init",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:stream_executor",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ] + if_tensorrt([
+        "@local_config_cuda//cuda:cuda_headers",
+        "@local_config_tensorrt//:tensorrt",
+    ]),
+)
+
+tf_custom_op_library(
+    name = "python/ops/_trt_ops.so",
+    srcs = [
+        "ops/get_serialized_resource_op.cc",
+        "ops/trt_engine_op.cc",
+    ],
+    deps = [
+        "//tensorflow/core:lib_proto_parsing",
+    ] + if_tensorrt([
+        "@local_config_tensorrt//:tensorrt",
+    ]),
+)
+
+cc_library(
+    name = "trt_op_kernels",
+    srcs = [
+        "kernels/get_serialized_resource_op.cc",
+        "kernels/trt_engine_op.cc",
+    ],
+    hdrs = [
+        "kernels/trt_engine_op.h",
+    ],
+    copts = tf_copts(),
+    visibility = ["//visibility:public"],
+    deps = [
+        ":test_utils",
+        ":trt_allocator",
+        ":trt_conversion",
+        ":trt_logging",
+        ":trt_plugins",
+        ":trt_resources",
+        ":utils",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/strings",
+        "//tensorflow/core:gpu_headers_lib",
+        "//tensorflow/core:lib_proto_parsing",
+        "//tensorflow/core:stream_executor_headers_lib",
+        "//tensorflow/core/grappler/costs:graph_properties",
+    ] + if_tensorrt([
+        "@local_config_tensorrt//:tensorrt",
+    ]) + tf_custom_op_library_additional_deps(),
+    # TODO(laigd): fix this by merging header file in cc file.
+    alwayslink = 1,  # buildozer: disable=alwayslink-with-hdrs
+)
+
+tf_cuda_cc_test(
+    name = "get_serialized_resource_op_test",
+    size = "small",
+    srcs = ["kernels/get_serialized_resource_op_test.cc"],
+    tags = [
+        "no_cuda_on_cpu_tap",
+        "no_windows",
+        "nomac",
+    ],
+    deps = [
+        ":get_serialized_resource_op_op_lib",
+        ":trt_op_kernels",
+        ":trt_resources",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+        "//tensorflow/core/kernels:ops_testutil",
+    ],
+)
+
+tf_gen_op_libs(
+    op_lib_names = [
+        "trt_engine_op",
+        "get_serialized_resource_op",
+    ],
+)
+
+tf_cuda_library(
+    name = "trt_logging",
+    srcs = ["utils/trt_logger.cc"],
+    hdrs = ["utils/trt_logger.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tensorflow/core:lib_proto_parsing",
+    ] + if_tensorrt([
+        "@local_config_tensorrt//:tensorrt",
+    ]),
+)
+
+tf_gen_op_wrapper_py(
+    name = "trt_ops",
+    deps = [
+        ":get_serialized_resource_op_op_lib",
+        ":trt_engine_op_op_lib",
+        ":trt_logging",
+    ],
+)
+
+tf_custom_op_py_library(
+    name = "trt_ops_loader",
+    srcs = ["python/ops/trt_ops.py"],
+    dso = [
+        "python/ops/_trt_ops.so",
+    ] + if_tensorrt([
+        "@local_config_tensorrt//:tensorrt",
+    ]),
+    kernels = [
+        ":trt_op_kernels",
+        ":trt_engine_op_op_lib",
+        ":get_serialized_resource_op_op_lib",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:resources",
+    ],
+)
+
+tf_cuda_library(
+    name = "trt_resources",
+    srcs = [
+        "utils/trt_int8_calibrator.cc",
+        "utils/trt_resource_manager.cc",
+        "utils/trt_resources.cc",
+    ],
+    hdrs = [
+        "utils/trt_int8_calibrator.h",
+        "utils/trt_lru_cache.h",
+        "utils/trt_resource_manager.h",
+        "utils/trt_resources.h",
+    ],
+    deps = [
+        ":trt_allocator",
+        ":trt_logging",
+        ":utils",
+        "//tensorflow/core:framework_headers_lib",
+        "//tensorflow/core:framework_lite",
+        "//tensorflow/core:lib_proto_parsing",
+    ] + if_tensorrt([
+        "@local_config_tensorrt//:tensorrt",
+    ]),
+)
+
+tf_cuda_library(
+    name = "trt_allocator",
+    srcs = ["utils/trt_allocator.cc"],
+    hdrs = ["utils/trt_allocator.h"],
+    deps = [
+        "//tensorflow/core:framework_headers_lib",
+        "//tensorflow/core:framework_lite",
+        "//tensorflow/core:lib_proto_parsing",
+    ] + if_tensorrt([
+        "@local_config_tensorrt//:tensorrt",
+    ]),
+)
+
+tf_cc_test(
+    name = "trt_allocator_test",
+    size = "small",
+    srcs = ["utils/trt_allocator_test.cc"],
+    tags = [
+        "no_windows",
+        "nomac",
+    ],
+    deps = [
+        ":trt_allocator",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
+tf_cc_test(
+    name = "trt_lru_cache_test",
+    size = "small",
+    srcs = ["utils/trt_lru_cache_test.cc"],
+    tags = [
+        "no_windows",
+        "nomac",
+    ],
+    deps = [
+        ":trt_resources",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
+# Library for the node-level conversion portion of TensorRT operation creation
+tf_cuda_library(
+    name = "trt_conversion",
+    srcs = [
+        "convert/convert_graph.cc",
+        "convert/convert_nodes.cc",
+        "convert/trt_optimization_pass.cc",
+    ],
+    hdrs = [
+        "convert/convert_graph.h",
+        "convert/convert_nodes.h",
+        "convert/trt_optimization_pass.h",
+    ],
+    deps = [
+        ":segment",
+        ":test_utils",
+        ":trt_allocator",
+        ":trt_plugins",
+        ":trt_logging",
+        ":trt_resources",
+        ":utils",
+        "@com_google_absl//absl/strings",
+        "//tensorflow/core/grappler/clusters:cluster",
+        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
+        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
+        "//tensorflow/core/grappler:grappler_item",
+        "//tensorflow/core/grappler:utils",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:framework_lite",
+        "//tensorflow/core:gpu_runtime",
+        "//tensorflow/core:graph",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core/grappler:devices",
+        "//tensorflow/core/grappler/clusters:virtual_cluster",
+        "//tensorflow/core/grappler/costs:graph_properties",
+        "//tensorflow/core/grappler/optimizers:meta_optimizer",
+    ] + if_tensorrt([
+        "@local_config_tensorrt//:tensorrt",
+    ]) + tf_custom_op_library_additional_deps(),
+)
+
+tf_cuda_cc_test(
+    name = "convert_graph_test",
+    size = "medium",
+    srcs = ["convert/convert_graph_test.cc"],
+    tags = [
+        "no_cuda_on_cpu_tap",
+        "no_windows",
+        "nomac",
+    ],
+    deps = [
+        ":trt_conversion",
+        "@com_google_googletest//:gtest",
+        "@com_google_absl//absl/strings",
+        "//tensorflow/cc:cc_ops",
+        "//tensorflow/cc:scope",
+        "//tensorflow/core/grappler:grappler_item",
+        "//tensorflow/core/grappler/clusters:cluster",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:core_cpu_base",
+        "//tensorflow/core:direct_session",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+    ] + if_tensorrt([
+        "@local_config_tensorrt//:tensorrt",
+    ]),
+)
+
+tf_cuda_cc_test(
+    name = "convert_nodes_test",
+    size = "medium",
+    srcs = ["convert/convert_nodes_test.cc"],
+    tags = [
+        "no_cuda_on_cpu_tap",
+        "no_windows",
+        "nomac",
+    ],
+    deps = [
+        ":trt_logging",
+        ":trt_conversion",
+        ":trt_plugins",
+        "@com_google_googletest//:gtest",
+        "@com_google_absl//absl/strings",
+        "//tensorflow/cc:cc_ops",
+        "//tensorflow/cc:ops",
+        "//tensorflow/cc:scope",
+        "//tensorflow/core/grappler/costs:graph_properties",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:core_cpu_base",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+    ] + if_tensorrt([
+        "@local_config_cuda//cuda:cuda_headers",
+        "@local_config_tensorrt//:tensorrt",
+    ]),
+)
+
+# Library for the segmenting portion of TensorRT operation creation
+cc_library(
+    name = "segment",
+    srcs = ["segment/segment.cc"],
+    hdrs = [
+        "segment/segment.h",
+        "segment/union_find.h",
+    ],
+    deps = [
+        "//tensorflow/core:graph",
+        "//tensorflow/core:lib_proto_parsing",
+        "//tensorflow/core:protos_all_cc",
+        "@com_google_absl//absl/strings",
+        "@protobuf_archive//:protobuf_headers",
+    ],
+)
+
+tf_cc_test(
+    name = "segment_test",
+    size = "small",
+    srcs = ["segment/segment_test.cc"],
+    tags = [
+        "no_windows",
+        "nomac",
+    ],
+    deps = [
+        ":segment",
+        "//tensorflow/cc:cc_ops",
+        "//tensorflow/cc:scope",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:ops",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+    ],
+)
+
+# Library for the plugin factory
+tf_cuda_library(
+    name = "trt_plugins",
+    srcs = [
+        "plugin/trt_plugin.cc",
+        "plugin/trt_plugin_factory.cc",
+        "plugin/trt_plugin_utils.cc",
+    ],
+    hdrs = [
+        "plugin/trt_plugin.h",
+        "plugin/trt_plugin_factory.h",
+        "plugin/trt_plugin_utils.h",
+    ],
+    deps = [
+        "//tensorflow/core:framework_lite",
+        "//tensorflow/core:lib_proto_parsing",
+    ] + if_tensorrt([
+        "@local_config_tensorrt//:tensorrt",
+    ]),
+)
+
+tf_cuda_cc_test(
+    name = "trt_plugin_factory_test",
+    size = "small",
+    srcs = ["plugin/trt_plugin_factory_test.cc"],
+    tags = [
+        "no_cuda_on_cpu_tap",
+        "no_windows",
+        "nomac",
+    ],
+    deps = [
+        ":trt_plugins",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ] + if_tensorrt([
+        "@local_config_cuda//cuda:cuda_headers",
+        "@local_config_tensorrt//:tensorrt",
+    ]),
+)
+
+cc_library(
+    name = "utils",
+    srcs = ["convert/utils.cc"],
+    hdrs = ["convert/utils.h"],
+    copts = tf_copts(),
+    deps = [
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+    ],
+)
+
+cc_library(
+    name = "test_utils",
+    srcs = ["utils/test_utils.cc"],
+    hdrs = ["utils/test_utils.h"],
+    deps = [
+        "//tensorflow/core:lib",
+        "@com_googlesource_code_re2//:re2",
+    ],
+)
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc
similarity index 98%
rename from tensorflow/contrib/tensorrt/convert/convert_graph.cc
rename to tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc
index 15cf97a2ee..1fdb099cc1 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/convert/convert_graph.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/convert_graph.h"
 
 #include <fstream>
 #include <list>
@@ -24,13 +24,14 @@ limitations under the License.
 #include <utility>
 #include <vector>
 
-#include "tensorflow/contrib/tensorrt/convert/convert_nodes.h"
-#include "tensorflow/contrib/tensorrt/convert/utils.h"
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_resource_manager.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_resources.h"
-#include "tensorflow/contrib/tensorrt/segment/segment.h"
-#include "tensorflow/contrib/tensorrt/test/utils.h"
+#include "absl/strings/str_cat.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/utils.h"
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.h"
+#include "tensorflow/compiler/tf2tensorrt/segment/segment.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/test_utils.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_resource_manager.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_resources.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_id.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_id_manager.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_process_state.h"
@@ -63,8 +64,8 @@ limitations under the License.
 namespace tensorflow {
 namespace tensorrt {
 namespace convert {
-using ::tensorflow::strings::StrAppend;
-using ::tensorflow::strings::StrCat;
+using absl::StrAppend;
+using absl::StrCat;
 
 // Returns compiled TRT version information {Maj, Min, Patch}
 std::vector<int> GetLinkedTensorRTVersion() {
@@ -151,7 +152,7 @@ Status TrtCandidateSelector::IsTensorRTCandidate(const tensorflow::Node* node) {
   if (precision_mode_ == INT8MODE && quantize_ops.count(node->type_string())) {
     is_supported_op_type = true;
   }
-  // LINT.ThenChange(//tensorflow/contrib/tensorrt/convert/convert_nodes.cc)
+  // LINT.ThenChange(//tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc)
   if (!is_supported_op_type) {
     return errors::Unimplemented("Op type ", node->type_string(),
                                  " is not supported");
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.h b/tensorflow/compiler/tf2tensorrt/convert/convert_graph.h
similarity index 94%
rename from tensorflow/contrib/tensorrt/convert/convert_graph.h
rename to tensorflow/compiler/tf2tensorrt/convert/convert_graph.h
index 1f39f56f63..fb82a430c6 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.h
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_graph.h
@@ -12,12 +12,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_CONVERT_CONVERT_GRAPH_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_CONVERT_CONVERT_GRAPH_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_CONVERT_CONVERT_GRAPH_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_CONVERT_CONVERT_GRAPH_H_
 
 #include <vector>
 
-#include "tensorflow/contrib/tensorrt/convert/convert_nodes.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h"
 #include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/grappler/clusters/cluster.h"
 #include "tensorflow/core/grappler/costs/graph_properties.h"
@@ -123,4 +123,4 @@ std::pair<int, tensorflow::Allocator*> GetDeviceAndAllocator(
 #endif  // GOOGLE_TENSORRT
 #endif  // GOOGLE_CUDA
 
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_CONVERT_CONVERT_GRAPH_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_CONVERT_CONVERT_GRAPH_H_
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph_test.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_graph_test.cc
similarity index 98%
rename from tensorflow/contrib/tensorrt/convert/convert_graph_test.cc
rename to tensorflow/compiler/tf2tensorrt/convert/convert_graph_test.cc
index 2d2bfeb192..a3c3a8ac65 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_graph_test.cc
@@ -13,13 +13,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/convert/convert_graph.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/convert_graph.h"
 
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
 #include "tensorflow/cc/framework/scope.h"
 #include "tensorflow/cc/ops/standard_ops.h"
-#include "tensorflow/contrib/tensorrt/convert/convert_nodes.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h"
 #include "tensorflow/core/common_runtime/device_mgr.h"
 #include "tensorflow/core/common_runtime/device_set.h"
 #include "tensorflow/core/framework/tensor_shape.h"
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
similarity index 99%
rename from tensorflow/contrib/tensorrt/convert/convert_nodes.cc
rename to tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
index 59d295f2e1..c08582a42e 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/convert/convert_nodes.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h"
 
 #include <algorithm>
 #include <cstring>
@@ -24,11 +24,12 @@ limitations under the License.
 #include <utility>
 #include <vector>
 
-#include "tensorflow/contrib/tensorrt/convert/utils.h"
-#include "tensorflow/contrib/tensorrt/log/trt_logger.h"
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_resource_manager.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_resources.h"
+#include "absl/strings/str_cat.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/utils.h"
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_logger.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_resource_manager.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_resources.h"
 #include "tensorflow/core/framework/node_def.pb.h"  // NOLINT
 #include "tensorflow/core/framework/node_def_builder.h"
 #include "tensorflow/core/framework/tensor.pb.h"        // NOLINT
@@ -43,6 +44,7 @@ limitations under the License.
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/protobuf.h"
 #include "tensorflow/core/platform/tensor_coding.h"
 #include "tensorflow/core/platform/types.h"
 
@@ -81,9 +83,9 @@ const char* const kInputPHName = "TensorRTInputPH_";
 const char* const kOutputPHName = "TensorRTOutputPH_";
 
 namespace convert {
+using absl::StrAppend;
+using absl::StrCat;
 using ::tensorflow::str_util::Split;
-using ::tensorflow::strings::StrAppend;
-using ::tensorflow::strings::StrCat;
 
 inline tensorflow::Status ConvertDType(tensorflow::DataType tf_dtype,
                                        nvinfer1::DataType* trt_dtype) {
@@ -3692,7 +3694,7 @@ tensorflow::Status ConvertGraphDefToEngine(
     if (tensorflow::str_util::StartsWith(node_name, kInputPHName) &&
         (node_def.op() == "Placeholder")) {
       int32 slot_number = -1;
-      if (!tensorflow::strings::safe_strto32(
+      if (!tensorflow::strings::safe_strto32(  // non-absl ok
               node_name.c_str() + strlen(kInputPHName), &slot_number)) {
         return tensorflow::errors::InvalidArgument(
             "Failed to parse slot number from ", node_name);
@@ -3721,7 +3723,7 @@ tensorflow::Status ConvertGraphDefToEngine(
     } else if (tensorflow::str_util::StartsWith(node_name, kOutputPHName) &&
                (node_def.op() == "Identity")) {
       int32 slot_number = -1;
-      if (!tensorflow::strings::safe_strto32(
+      if (!tensorflow::strings::safe_strto32(  // non-absl ok
               node_name.c_str() + strlen(kOutputPHName), &slot_number)) {
         return tensorflow::errors::InvalidArgument(
             "Failed to parse slot number from ", node_name);
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.h b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
similarity index 97%
rename from tensorflow/contrib/tensorrt/convert/convert_nodes.h
rename to tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
index 8495a1a19a..aebc0ca38d 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.h
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_CONVERT_CONVERT_NODES_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_CONVERT_CONVERT_NODES_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_CONVERT_CONVERT_NODES_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_CONVERT_CONVERT_NODES_H_
 
 #include <set>
 #include <string>
@@ -22,11 +22,11 @@ limitations under the License.
 #include <utility>
 #include <vector>
 
-#include "tensorflow/contrib/tensorrt/convert/utils.h"
-#include "tensorflow/contrib/tensorrt/log/trt_logger.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_allocator.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_resources.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/utils.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_allocator.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_int8_calibrator.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_logger.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_resources.h"
 #include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/graph/graph.h"
 #include "tensorflow/core/grappler/costs/graph_properties.h"
@@ -555,4 +555,4 @@ class Converter {
 #endif  // GOOGLE_TENSORRT
 #endif  // GOOGLE_CUDA
 
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_CONVERT_CONVERT_NODES_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_CONVERT_CONVERT_NODES_H_
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
similarity index 99%
rename from tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
rename to tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
index 3bd4ff94d1..3a70423d12 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/convert/convert_nodes.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h"
 
 #include <memory>
 #include <unordered_map>
@@ -21,11 +21,12 @@ limitations under the License.
 
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
+#include "absl/strings/str_cat.h"
 #include "tensorflow/cc/framework/ops.h"
 #include "tensorflow/cc/framework/scope.h"
 #include "tensorflow/cc/ops/standard_ops.h"
-#include "tensorflow/contrib/tensorrt/log/trt_logger.h"
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h"
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_logger.h"
 #include "tensorflow/core/framework/node_def.pb.h"  // NOLINT
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor.pb.h"  // NOLINT
@@ -51,7 +52,7 @@ namespace tensorflow {
 namespace tensorrt {
 namespace convert {
 
-using ::tensorflow::strings::StrCat;
+using absl::StrCat;
 using ::testing::ElementsAre;
 using ::testing::ElementsAreArray;
 
@@ -2316,7 +2317,7 @@ TEST_F(OpConverterTest, ConvertSqueeze) {
     Scope s = Scope::NewRootScope();
     auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
     ops::Squeeze::Attrs squeeze_attrs;
-    squeeze_attrs.axis_ = gtl::ArraySlice<int>(axis);
+    squeeze_attrs.axis_ = gtl::ArraySlice<int>(axis);  // non-absl ok
     auto squeeze =
         ops::Squeeze(s.WithOpName("my_squeeze"), input, squeeze_attrs);
     return squeeze.operation.node()->def();
diff --git a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc b/tensorflow/compiler/tf2tensorrt/convert/trt_optimization_pass.cc
similarity index 96%
rename from tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc
rename to tensorflow/compiler/tf2tensorrt/convert/trt_optimization_pass.cc
index d57f2300f8..ebf8df1349 100644
--- a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/trt_optimization_pass.cc
@@ -12,9 +12,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/convert/trt_optimization_pass.h"
-#include "tensorflow/contrib/tensorrt/convert/convert_graph.h"
-#include "tensorflow/contrib/tensorrt/convert/utils.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/trt_optimization_pass.h"
+
+#include "absl/strings/str_cat.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/convert_graph.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/utils.h"
 #include "tensorflow/core/grappler/clusters/cluster.h"
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.h"
@@ -30,9 +32,9 @@ namespace tensorflow {
 namespace tensorrt {
 namespace convert {
 // TODO(sami): Remove VLOG messages once the code matures
+using absl::StrAppend;
+using absl::StrCat;
 using tensorflow::str_util::Uppercase;
-using tensorflow::strings::StrAppend;
-using tensorflow::strings::StrCat;
 
 tensorflow::Status TRTOptimizationPass::Init(
     const tensorflow::RewriterConfig_CustomGraphOptimizer* config) {
@@ -243,7 +245,7 @@ tensorflow::Status TRTOptimizationPass::Optimize(
     // If the last token is not an integer, it must be part of the name.
     // Otherwise it is port number.
     if (tokens.size() > 1 &&
-        !strings::safe_strto32(tokens.back(), &dumm_port)) {
+        !strings::safe_strto32(tokens.back(), &dumm_port)) {  // non-absl ok
       StrAppend(&s, ":", tokens.back());
     }
     nodes_to_preserve.push_back(s);
diff --git a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.h b/tensorflow/compiler/tf2tensorrt/convert/trt_optimization_pass.h
similarity index 91%
rename from tensorflow/contrib/tensorrt/convert/trt_optimization_pass.h
rename to tensorflow/compiler/tf2tensorrt/convert/trt_optimization_pass.h
index 3e8dc0978e..bd6c6dbce1 100644
--- a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.h
+++ b/tensorflow/compiler/tf2tensorrt/convert/trt_optimization_pass.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_CONVERT_TRT_OPTIMIZATION_PASS_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_CONVERT_TRT_OPTIMIZATION_PASS_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_CONVERT_TRT_OPTIMIZATION_PASS_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_CONVERT_TRT_OPTIMIZATION_PASS_H_
 
 #include <string>
 
@@ -77,4 +77,4 @@ class TRTOptimizationPass : public tensorflow::grappler::CustomGraphOptimizer {
 
 #endif  // GOOGLE_CUDA
 #endif  // GOOGLE_TENSORRT
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_CONVERT_TRT_OPTIMIZATION_PASS_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_CONVERT_TRT_OPTIMIZATION_PASS_H_
diff --git a/tensorflow/contrib/tensorrt/convert/utils.cc b/tensorflow/compiler/tf2tensorrt/convert/utils.cc
similarity index 97%
rename from tensorflow/contrib/tensorrt/convert/utils.cc
rename to tensorflow/compiler/tf2tensorrt/convert/utils.cc
index e7a1febb8c..62a0f62ad6 100644
--- a/tensorflow/contrib/tensorrt/convert/utils.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/utils.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/convert/utils.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/utils.h"
 
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
diff --git a/tensorflow/contrib/tensorrt/convert/utils.h b/tensorflow/compiler/tf2tensorrt/convert/utils.h
similarity index 88%
rename from tensorflow/contrib/tensorrt/convert/utils.h
rename to tensorflow/compiler/tf2tensorrt/convert/utils.h
index 0592f31462..9f9ee59087 100644
--- a/tensorflow/contrib/tensorrt/convert/utils.h
+++ b/tensorflow/compiler/tf2tensorrt/convert/utils.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_CONVERT_UTILS_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_CONVERT_UTILS_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_CONVERT_UTILS_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_CONVERT_UTILS_H_
 
 #include <memory>
 
@@ -47,4 +47,4 @@ Status GetPrecisionMode(const string& name, int* precision_mode);
 }  // namespace tensorrt
 }  // namespace tensorflow
 
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_CONVERT_UTILS_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_CONVERT_UTILS_H_
diff --git a/tensorflow/contrib/tensorrt/kernels/get_serialized_resource_op.cc b/tensorflow/compiler/tf2tensorrt/kernels/get_serialized_resource_op.cc
similarity index 88%
rename from tensorflow/contrib/tensorrt/kernels/get_serialized_resource_op.cc
rename to tensorflow/compiler/tf2tensorrt/kernels/get_serialized_resource_op.cc
index f68bc2b485..eae1f8e752 100644
--- a/tensorflow/contrib/tensorrt/kernels/get_serialized_resource_op.cc
+++ b/tensorflow/compiler/tf2tensorrt/kernels/get_serialized_resource_op.cc
@@ -13,13 +13,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_KERNELS_GET_SERIALIZED_RESOURCE_OP_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_KERNELS_GET_SERIALIZED_RESOURCE_OP_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_KERNELS_GET_SERIALIZED_RESOURCE_OP_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_KERNELS_GET_SERIALIZED_RESOURCE_OP_H_
 
 #include <memory>
 #include <vector>
 
-#include "tensorflow/contrib/tensorrt/resources/trt_resources.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_resources.h"
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/resource_mgr.h"
@@ -70,4 +70,4 @@ REGISTER_KERNEL_BUILDER(Name("GetSerializedResourceOp").Device(DEVICE_GPU),
 
 #endif  // GOOGLE_TENSORRT
 #endif  // GOOGLE_CUDA
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_KERNELS_GET_SERIALIZED_RESOURCE_OP_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_KERNELS_GET_SERIALIZED_RESOURCE_OP_H_
diff --git a/tensorflow/contrib/tensorrt/kernels/get_serialized_resource_op_test.cc b/tensorflow/compiler/tf2tensorrt/kernels/get_serialized_resource_op_test.cc
similarity index 97%
rename from tensorflow/contrib/tensorrt/kernels/get_serialized_resource_op_test.cc
rename to tensorflow/compiler/tf2tensorrt/kernels/get_serialized_resource_op_test.cc
index a91228e4c4..ec038ebda0 100644
--- a/tensorflow/contrib/tensorrt/kernels/get_serialized_resource_op_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/kernels/get_serialized_resource_op_test.cc
@@ -18,7 +18,7 @@ limitations under the License.
 #include <fstream>
 #include <vector>
 
-#include "tensorflow/contrib/tensorrt/resources/trt_resources.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_resources.h"
 #include "tensorflow/core/framework/fake_input.h"
 #include "tensorflow/core/framework/node_def_builder.h"
 #include "tensorflow/core/framework/tensor.h"
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.cc
similarity index 97%
rename from tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
rename to tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.cc
index d28d88fa3a..198d68b609 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.cc
@@ -12,18 +12,19 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tensorflow/contrib/tensorrt/kernels/trt_engine_op.h"
+#include "tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.h"
 
 #include <algorithm>
 
 #include "absl/memory/memory.h"
-#include "tensorflow/contrib/tensorrt/convert/convert_nodes.h"
-#include "tensorflow/contrib/tensorrt/convert/utils.h"
-#include "tensorflow/contrib/tensorrt/log/trt_logger.h"
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_resource_manager.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_resources.h"
-#include "tensorflow/contrib/tensorrt/test/utils.h"
+#include "absl/strings/str_cat.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/utils.h"
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/test_utils.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_logger.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_resource_manager.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_resources.h"
 #include "tensorflow/core/framework/graph_to_functiondef.h"
 #include "tensorflow/core/lib/core/refcount.h"
 #include "tensorflow/core/lib/strings/str_util.h"
@@ -39,9 +40,9 @@ limitations under the License.
 namespace tensorflow {
 namespace tensorrt {
 static Logger logger;
+using absl::StrAppend;
+using absl::StrCat;
 using ::nvinfer1::IRuntime;
-using ::tensorflow::strings::StrAppend;
-using ::tensorflow::strings::StrCat;
 
 // A helper class to call done() when destructed for asynchronous execution.
 // Helps simultaneous execution of native and TRT engines.
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h b/tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.h
similarity index 88%
rename from tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
rename to tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.h
index 76c6020078..64f8c97a74 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
+++ b/tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.h
@@ -13,17 +13,17 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_KERNELS_TRT_ENGINE_OP_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_KERNELS_TRT_ENGINE_OP_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_KERNELS_TRT_ENGINE_OP_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_KERNELS_TRT_ENGINE_OP_H_
 
 #include <memory>
 #include <vector>
 
-#include "tensorflow/contrib/tensorrt/convert/utils.h"
-#include "tensorflow/contrib/tensorrt/log/trt_logger.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_allocator.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_lru_cache.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_resources.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/utils.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_allocator.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_logger.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_lru_cache.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_resources.h"
 #include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/framework/op.h"
@@ -127,4 +127,4 @@ class TRTEngineOp : public AsyncOpKernel {
 #endif  // GOOGLE_TENSORRT
 #endif  // GOOGLE_CUDA
 
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_KERNELS_TRT_ENGINE_OP_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_KERNELS_TRT_ENGINE_OP_H_
diff --git a/tensorflow/contrib/tensorrt/ops/get_serialized_resource_op.cc b/tensorflow/compiler/tf2tensorrt/ops/get_serialized_resource_op.cc
similarity index 100%
rename from tensorflow/contrib/tensorrt/ops/get_serialized_resource_op.cc
rename to tensorflow/compiler/tf2tensorrt/ops/get_serialized_resource_op.cc
diff --git a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc b/tensorflow/compiler/tf2tensorrt/ops/trt_engine_op.cc
similarity index 100%
rename from tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
rename to tensorflow/compiler/tf2tensorrt/ops/trt_engine_op.cc
diff --git a/tensorflow/contrib/tensorrt/plugin/trt_plugin.cc b/tensorflow/compiler/tf2tensorrt/plugin/trt_plugin.cc
similarity index 96%
rename from tensorflow/contrib/tensorrt/plugin/trt_plugin.cc
rename to tensorflow/compiler/tf2tensorrt/plugin/trt_plugin.cc
index 062f86e8bb..a4341c530f 100644
--- a/tensorflow/contrib/tensorrt/plugin/trt_plugin.cc
+++ b/tensorflow/compiler/tf2tensorrt/plugin/trt_plugin.cc
@@ -13,10 +13,12 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin.h"
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin.h"
+
 #include <cassert>
 #include <cstring>
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin_utils.h"
+
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_utils.h"
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
diff --git a/tensorflow/contrib/tensorrt/plugin/trt_plugin.h b/tensorflow/compiler/tf2tensorrt/plugin/trt_plugin.h
similarity index 92%
rename from tensorflow/contrib/tensorrt/plugin/trt_plugin.h
rename to tensorflow/compiler/tf2tensorrt/plugin/trt_plugin.h
index 754920b60c..f495d85703 100644
--- a/tensorflow/contrib/tensorrt/plugin/trt_plugin.h
+++ b/tensorflow/compiler/tf2tensorrt/plugin/trt_plugin.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_PLUGIN_TRT_PLUGIN_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_PLUGIN_TRT_PLUGIN_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_PLUGIN_TRT_PLUGIN_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_PLUGIN_TRT_PLUGIN_H_
 
 #include <iostream>
 #include <unordered_map>
@@ -71,4 +71,4 @@ class PluginTensorRT : public nvinfer1::IPlugin {
 #endif  // GOOGLE_TENSORRT
 #endif  // GOOGLE_CUDA
 
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_PLUGIN_TRT_PLUGIN_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_PLUGIN_TRT_PLUGIN_H_
diff --git a/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.cc b/tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.cc
similarity index 96%
rename from tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.cc
rename to tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.cc
index cccc912262..871fb1210b 100644
--- a/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.cc
+++ b/tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h"
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.h"
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
diff --git a/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h b/tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.h
similarity index 91%
rename from tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h
rename to tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.h
index bbae9fb65c..9aa99a40b8 100644
--- a/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h
+++ b/tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.h
@@ -13,14 +13,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_PLUGIN_TRT_PLUGIN_FACTORY_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_PLUGIN_TRT_PLUGIN_FACTORY_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_PLUGIN_TRT_PLUGIN_FACTORY_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_PLUGIN_TRT_PLUGIN_FACTORY_H_
 
 #include <memory>
 #include <unordered_map>
 
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin.h"
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin_utils.h"
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin.h"
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_utils.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/macros.h"
 #include "tensorflow/core/platform/mutex.h"
@@ -99,4 +99,4 @@ class TrtPluginRegistrar {
 #endif  // GOOGLE_TENSORRT
 #endif  // GOOGLE_CUDA
 
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_PLUGIN_TRT_PLUGIN_FACTORY_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_PLUGIN_TRT_PLUGIN_FACTORY_H_
diff --git a/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory_test.cc b/tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory_test.cc
similarity index 96%
rename from tensorflow/contrib/tensorrt/plugin/trt_plugin_factory_test.cc
rename to tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory_test.cc
index 129bdcdbc2..7d9c465c22 100644
--- a/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory_test.cc
@@ -13,9 +13,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h"
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.h"
 
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin.h"
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/test.h"
diff --git a/tensorflow/contrib/tensorrt/plugin/trt_plugin_utils.cc b/tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_utils.cc
similarity index 94%
rename from tensorflow/contrib/tensorrt/plugin/trt_plugin_utils.cc
rename to tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_utils.cc
index a8f60886c0..f3d6b4ff47 100644
--- a/tensorflow/contrib/tensorrt/plugin/trt_plugin_utils.cc
+++ b/tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_utils.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin_utils.h"
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_utils.h"
 #include <cassert>
 
 #if GOOGLE_CUDA
diff --git a/tensorflow/contrib/tensorrt/plugin/trt_plugin_utils.h b/tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_utils.h
similarity index 82%
rename from tensorflow/contrib/tensorrt/plugin/trt_plugin_utils.h
rename to tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_utils.h
index 274ce42fec..e5eff15c19 100644
--- a/tensorflow/contrib/tensorrt/plugin/trt_plugin_utils.h
+++ b/tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_utils.h
@@ -13,12 +13,12 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_PLUGIN_TRT_PLUGIN_UTILS_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_PLUGIN_TRT_PLUGIN_UTILS_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_PLUGIN_TRT_PLUGIN_UTILS_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_PLUGIN_TRT_PLUGIN_UTILS_H_
 
 #include <functional>
 
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin.h"
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin.h"
 #include "tensorflow/core/platform/types.h"
 
 #if GOOGLE_CUDA
@@ -43,4 +43,4 @@ string ExtractOpName(const void* serial_data, size_t serial_length,
 #endif  // GOOGLE_TENSORRT
 #endif  // GOOGLE_CUDA
 
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_PLUGIN_TRT_PLUGIN_UTILS_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_PLUGIN_TRT_PLUGIN_UTILS_H_
diff --git a/tensorflow/contrib/tensorrt/python/ops/trt_ops.py b/tensorflow/compiler/tf2tensorrt/python/ops/trt_ops.py
similarity index 88%
rename from tensorflow/contrib/tensorrt/python/ops/trt_ops.py
rename to tensorflow/compiler/tf2tensorrt/python/ops/trt_ops.py
index 1fee06854f..86bfabf99e 100644
--- a/tensorflow/contrib/tensorrt/python/ops/trt_ops.py
+++ b/tensorflow/compiler/tf2tensorrt/python/ops/trt_ops.py
@@ -22,13 +22,13 @@ import platform
 
 if platform.system() != "Windows":
   # pylint: disable=wildcard-import,unused-import,g-import-not-at-top
-  from tensorflow.contrib.tensorrt.ops.gen_trt_ops import *
+  from tensorflow.compiler.tf2tensorrt.ops.gen_trt_ops import *
 
-  from tensorflow.contrib.util import loader
+  from tensorflow.python.framework import load_library
   from tensorflow.python.platform import resource_loader
   # pylint: enable=wildcard-import,unused-import,g-import-not-at-top
 
-  _trt_ops = loader.load_op_library(
+  _trt_ops = load_library.load_op_library(
       resource_loader.get_path_to_datafile("_trt_ops.so"))
 else:
   raise RuntimeError("Windows platforms are not supported")
diff --git a/tensorflow/contrib/tensorrt/segment/segment.cc b/tensorflow/compiler/tf2tensorrt/segment/segment.cc
similarity index 99%
rename from tensorflow/contrib/tensorrt/segment/segment.cc
rename to tensorflow/compiler/tf2tensorrt/segment/segment.cc
index ecaffa3023..4a8a4ac758 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.cc
+++ b/tensorflow/compiler/tf2tensorrt/segment/segment.cc
@@ -13,14 +13,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/segment/segment.h"
+#include "tensorflow/compiler/tf2tensorrt/segment/segment.h"
 
 #include <queue>
 #include <set>
 #include <unordered_map>
 #include <vector>
 
-#include "tensorflow/contrib/tensorrt/segment/union_find.h"
+#include "absl/strings/str_cat.h"
+#include "tensorflow/compiler/tf2tensorrt/segment/union_find.h"
 #include "tensorflow/core/graph/algorithm.h"
 #include "tensorflow/core/graph/graph.h"
 #include "tensorflow/core/graph/graph_constructor.h"
@@ -32,8 +33,8 @@ limitations under the License.
 namespace tensorflow {
 namespace tensorrt {
 namespace segment {
-using ::tensorflow::strings::StrAppend;
-using ::tensorflow::strings::StrCat;
+using absl::StrAppend;
+using absl::StrCat;
 
 // A simple graph representation to mirror tensorflow::Graph. This structure
 // helps saving memory since segmenter modifies the graph in place, preventing
diff --git a/tensorflow/contrib/tensorrt/segment/segment.h b/tensorflow/compiler/tf2tensorrt/segment/segment.h
similarity index 92%
rename from tensorflow/contrib/tensorrt/segment/segment.h
rename to tensorflow/compiler/tf2tensorrt/segment/segment.h
index 6cc92cdb5d..9a0ccc9aef 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.h
+++ b/tensorflow/compiler/tf2tensorrt/segment/segment.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_SEGMENT_SEGMENT_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_SEGMENT_SEGMENT_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_SEGMENT_SEGMENT_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_SEGMENT_SEGMENT_H_
 
 #include <set>
 #include <vector>
@@ -60,4 +60,4 @@ tensorflow::Status SegmentGraph(
 }  // namespace tensorrt
 }  // namespace tensorflow
 
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_SEGMENT_SEGMENT_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_SEGMENT_SEGMENT_H_
diff --git a/tensorflow/contrib/tensorrt/segment/segment_test.cc b/tensorflow/compiler/tf2tensorrt/segment/segment_test.cc
similarity index 99%
rename from tensorflow/contrib/tensorrt/segment/segment_test.cc
rename to tensorflow/compiler/tf2tensorrt/segment/segment_test.cc
index 4ac02327ae..58512d3b09 100644
--- a/tensorflow/contrib/tensorrt/segment/segment_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/segment/segment_test.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/segment/segment.h"
+#include "tensorflow/compiler/tf2tensorrt/segment/segment.h"
 
 #include "tensorflow/cc/framework/scope.h"
 #include "tensorflow/cc/ops/standard_ops.h"
diff --git a/tensorflow/contrib/tensorrt/segment/union_find.h b/tensorflow/compiler/tf2tensorrt/segment/union_find.h
similarity index 92%
rename from tensorflow/contrib/tensorrt/segment/union_find.h
rename to tensorflow/compiler/tf2tensorrt/segment/union_find.h
index 1c64ebbb0a..6458ae692f 100644
--- a/tensorflow/contrib/tensorrt/segment/union_find.h
+++ b/tensorflow/compiler/tf2tensorrt/segment/union_find.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_SEGMENT_UNION_FIND_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_SEGMENT_UNION_FIND_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_SEGMENT_UNION_FIND_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_SEGMENT_UNION_FIND_H_
 
 namespace tensorflow {
 namespace tensorrt {
@@ -76,4 +76,4 @@ UnionFind<T>* UnionFind<T>::FindRoot() {
 }  // namespace tensorrt
 }  // namespace tensorflow
 
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_SEGMENT_UNION_FIND_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_SEGMENT_UNION_FIND_H_
diff --git a/tensorflow/contrib/tensorrt/tensorrt_test.cc b/tensorflow/compiler/tf2tensorrt/tensorrt_test.cc
similarity index 100%
rename from tensorflow/contrib/tensorrt/tensorrt_test.cc
rename to tensorflow/compiler/tf2tensorrt/tensorrt_test.cc
diff --git a/tensorflow/contrib/tensorrt/test/utils.cc b/tensorflow/compiler/tf2tensorrt/utils/test_utils.cc
similarity index 97%
rename from tensorflow/contrib/tensorrt/test/utils.cc
rename to tensorflow/compiler/tf2tensorrt/utils/test_utils.cc
index 276308b3a0..3bcca99afb 100644
--- a/tensorflow/contrib/tensorrt/test/utils.cc
+++ b/tensorflow/compiler/tf2tensorrt/utils/test_utils.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/test/utils.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/test_utils.h"
 
 #include <unordered_map>
 #include <vector>
diff --git a/tensorflow/contrib/tensorrt/test/utils.h b/tensorflow/compiler/tf2tensorrt/utils/test_utils.h
similarity index 89%
rename from tensorflow/contrib/tensorrt/test/utils.h
rename to tensorflow/compiler/tf2tensorrt/utils/test_utils.h
index 4bb4120206..bcd628b62f 100644
--- a/tensorflow/contrib/tensorrt/test/utils.h
+++ b/tensorflow/compiler/tf2tensorrt/utils/test_utils.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_TEST_UTILS_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_TEST_UTILS_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TEST_UTILS_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TEST_UTILS_H_
 
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
@@ -41,4 +41,4 @@ string GetTestValue(const string& label);
 }  // namespace tensorrt
 }  // namespace tensorflow
 
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_TEST_UTILS_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TEST_UTILS_H_
diff --git a/tensorflow/contrib/tensorrt/resources/trt_allocator.cc b/tensorflow/compiler/tf2tensorrt/utils/trt_allocator.cc
similarity index 98%
rename from tensorflow/contrib/tensorrt/resources/trt_allocator.cc
rename to tensorflow/compiler/tf2tensorrt/utils/trt_allocator.cc
index 7a2e93414a..1636cdc30c 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_allocator.cc
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_allocator.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/resources/trt_allocator.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_allocator.h"
 
 #include "tensorflow/core/platform/logging.h"
 
diff --git a/tensorflow/contrib/tensorrt/resources/trt_allocator.h b/tensorflow/compiler/tf2tensorrt/utils/trt_allocator.h
similarity index 93%
rename from tensorflow/contrib/tensorrt/resources/trt_allocator.h
rename to tensorflow/compiler/tf2tensorrt/utils/trt_allocator.h
index f857a9de05..59ffb42bad 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_allocator.h
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_allocator.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_ALLOCATOR_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_ALLOCATOR_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_ALLOCATOR_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_ALLOCATOR_H_
 
 #include <unordered_map>
 
@@ -81,4 +81,4 @@ class TRTDeviceAllocator : public TRTBaseAllocator {
 
 #endif  // GOOGLE_TENSORRT
 #endif  // GOOGLE_CUDA
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_ALLOCATOR_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_ALLOCATOR_H_
diff --git a/tensorflow/contrib/tensorrt/resources/trt_allocator_test.cc b/tensorflow/compiler/tf2tensorrt/utils/trt_allocator_test.cc
similarity index 98%
rename from tensorflow/contrib/tensorrt/resources/trt_allocator_test.cc
rename to tensorflow/compiler/tf2tensorrt/utils/trt_allocator_test.cc
index beb1284208..e457c64928 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_allocator_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_allocator_test.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/resources/trt_allocator.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_allocator.h"
 
 #include "tensorflow/core/platform/test.h"
 
diff --git a/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.cc b/tensorflow/compiler/tf2tensorrt/utils/trt_int8_calibrator.cc
similarity index 98%
rename from tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.cc
rename to tensorflow/compiler/tf2tensorrt/utils/trt_int8_calibrator.cc
index dab1dd9343..bf111d3a2e 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.cc
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_int8_calibrator.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_int8_calibrator.h"
 
 #include <atomic>
 #include <unordered_map>
diff --git a/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h b/tensorflow/compiler/tf2tensorrt/utils/trt_int8_calibrator.h
similarity index 93%
rename from tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h
rename to tensorflow/compiler/tf2tensorrt/utils/trt_int8_calibrator.h
index 65466c9741..10587e9962 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_int8_calibrator.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_INT8_CALIBRATOR_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_INT8_CALIBRATOR_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_INT8_CALIBRATOR_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_INT8_CALIBRATOR_H_
 
 #include <atomic>
 #include <string>
@@ -96,4 +96,4 @@ struct TRTInt8Calibrator : public nvinfer1::IInt8EntropyCalibrator {
 
 #endif
 #endif
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_INT8_CALIBRATOR_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_INT8_CALIBRATOR_H_
diff --git a/tensorflow/contrib/tensorrt/log/trt_logger.cc b/tensorflow/compiler/tf2tensorrt/utils/trt_logger.cc
similarity index 96%
rename from tensorflow/contrib/tensorrt/log/trt_logger.cc
rename to tensorflow/compiler/tf2tensorrt/utils/trt_logger.cc
index dda0dc9e71..c48bd6bf77 100644
--- a/tensorflow/contrib/tensorrt/log/trt_logger.cc
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_logger.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/log/trt_logger.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_logger.h"
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
diff --git a/tensorflow/contrib/tensorrt/log/trt_logger.h b/tensorflow/compiler/tf2tensorrt/utils/trt_logger.h
similarity index 86%
rename from tensorflow/contrib/tensorrt/log/trt_logger.h
rename to tensorflow/compiler/tf2tensorrt/utils/trt_logger.h
index 96ccacb791..22f4de970a 100644
--- a/tensorflow/contrib/tensorrt/log/trt_logger.h
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_logger.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_LOG_TRT_LOGGER_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_LOG_TRT_LOGGER_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_LOGGER_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_LOGGER_H_
 
 #include "tensorflow/core/platform/types.h"
 
@@ -41,4 +41,4 @@ class Logger : public nvinfer1::ILogger {
 #endif  // GOOGLE_TENSORRT
 #endif  // GOOGLE_CUDA
 
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_LOG_TRT_LOGGER_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_LOGGER_H_
diff --git a/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h b/tensorflow/compiler/tf2tensorrt/utils/trt_lru_cache.h
similarity index 94%
rename from tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
rename to tensorflow/compiler/tf2tensorrt/utils/trt_lru_cache.h
index afd1b83e74..09c47b36b0 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_lru_cache.h
@@ -13,14 +13,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_LRU_CACHE_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_LRU_CACHE_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_LRU_CACHE_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_LRU_CACHE_H_
 
 #include <list>
 #include <unordered_map>
 
-#include "tensorflow/contrib/tensorrt/convert/utils.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_allocator.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/utils.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_allocator.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/lib/core/errors.h"
 
@@ -189,4 +189,4 @@ class TRTEngineCacheResource : public tensorflow::ResourceBase {
 }  // namespace tensorrt
 }  // namespace tensorflow
 
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_LRU_CACHE_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_LRU_CACHE_H_
diff --git a/tensorflow/contrib/tensorrt/resources/trt_lru_cache_test.cc b/tensorflow/compiler/tf2tensorrt/utils/trt_lru_cache_test.cc
similarity index 96%
rename from tensorflow/contrib/tensorrt/resources/trt_lru_cache_test.cc
rename to tensorflow/compiler/tf2tensorrt/utils/trt_lru_cache_test.cc
index a0959479ce..0aa5eb8f7d 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_lru_cache_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_lru_cache_test.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/resources/trt_lru_cache.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_lru_cache.h"
 
 #include "tensorflow/core/platform/test.h"
 
diff --git a/tensorflow/contrib/tensorrt/resources/trt_resource_manager.cc b/tensorflow/compiler/tf2tensorrt/utils/trt_resource_manager.cc
similarity index 96%
rename from tensorflow/contrib/tensorrt/resources/trt_resource_manager.cc
rename to tensorflow/compiler/tf2tensorrt/utils/trt_resource_manager.cc
index 9c3698e5d1..0a72a88bc7 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_resource_manager.cc
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_resource_manager.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/resources/trt_resource_manager.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_resource_manager.h"
 #include "tensorflow/core/platform/logging.h"
 
 namespace tensorflow {
diff --git a/tensorflow/contrib/tensorrt/resources/trt_resource_manager.h b/tensorflow/compiler/tf2tensorrt/utils/trt_resource_manager.h
similarity index 87%
rename from tensorflow/contrib/tensorrt/resources/trt_resource_manager.h
rename to tensorflow/compiler/tf2tensorrt/utils/trt_resource_manager.h
index 19f39e6d3d..03879ffff2 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_resource_manager.h
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_resource_manager.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_RESOURCE_MANAGER_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_RESOURCE_MANAGER_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_RESOURCE_MANAGER_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_RESOURCE_MANAGER_H_
 #include <memory>
 
 #include <string>
@@ -42,4 +42,4 @@ class TRTResourceManager {
 }  // namespace tensorrt
 }  // namespace tensorflow
 
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_RESOURCE_MANAGER_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_RESOURCE_MANAGER_H_
diff --git a/tensorflow/contrib/tensorrt/resources/trt_resources.cc b/tensorflow/compiler/tf2tensorrt/utils/trt_resources.cc
similarity index 96%
rename from tensorflow/contrib/tensorrt/resources/trt_resources.cc
rename to tensorflow/compiler/tf2tensorrt/utils/trt_resources.cc
index c19eb34dab..37f7fe99fb 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_resources.cc
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_resources.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/resources/trt_resources.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_resources.h"
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
diff --git a/tensorflow/contrib/tensorrt/resources/trt_resources.h b/tensorflow/compiler/tf2tensorrt/utils/trt_resources.h
similarity index 82%
rename from tensorflow/contrib/tensorrt/resources/trt_resources.h
rename to tensorflow/compiler/tf2tensorrt/utils/trt_resources.h
index d0a87f2c31..5e8d4b3b73 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_resources.h
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_resources.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_RESOURCES_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_RESOURCES_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_RESOURCES_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_RESOURCES_H_
 
 #include <list>
 #include <sstream>
@@ -23,10 +23,10 @@ limitations under the License.
 #include <unordered_map>
 #include <vector>
 
-#include "tensorflow/contrib/tensorrt/convert/utils.h"
-#include "tensorflow/contrib/tensorrt/log/trt_logger.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_allocator.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/utils.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_allocator.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_int8_calibrator.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_logger.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 
@@ -70,4 +70,4 @@ class TRTCalibrationResource : public SerializableResourceBase {
 
 #endif  // GOOGLE_TENSORRT
 #endif  // GOOGLE_CUDA
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_RESOURCES_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_RESOURCES_H_
diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 3f10906768..67461450f8 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -11,18 +11,12 @@ exports_files(["LICENSE"])
 
 load(
     "//tensorflow:tensorflow.bzl",
-    "tf_cc_test",
     "tf_copts",
     "tf_cuda_library",
-    "tf_custom_op_library",
     "tf_custom_op_library_additional_deps",
-    "tf_gen_op_libs",
-    "tf_gen_op_wrapper_py",
 )
 load("//tensorflow:tensorflow.bzl", "cuda_py_test")
 load("//tensorflow:tensorflow.bzl", "cuda_py_tests")
-load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_test")
-load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library")
 load("//tensorflow:tensorflow.bzl", "tf_py_wrap_cc")
 load(
     "@local_config_tensorrt//:build_defs.bzl",
@@ -33,158 +27,19 @@ exports_files(glob([
     "test/testdata/*",
 ]))
 
-tf_cuda_cc_test(
-    name = "tensorrt_test_cc",
-    size = "small",
-    srcs = ["tensorrt_test.cc"],
-    tags = [
-        "no_windows",
-        "nomac",
-    ],
-    deps = [
-        "//tensorflow/core:gpu_init",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:stream_executor",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-    ] + if_tensorrt([
-        "@local_config_cuda//cuda:cuda_headers",
-        "@local_config_tensorrt//:tensorrt",
-    ]),
-)
-
-tf_custom_op_library(
-    name = "python/ops/_trt_ops.so",
-    srcs = [
-        "ops/get_serialized_resource_op.cc",
-        "ops/trt_engine_op.cc",
-    ],
-    deps = [
-        ":trt_shape_function",
-        "//tensorflow/core:lib_proto_parsing",
-    ] + if_tensorrt([
-        "@local_config_tensorrt//:tensorrt",
-    ]),
-)
-
 tf_cuda_library(
     name = "trt_shape_function",
     srcs = ["shape_fn/trt_shfn.cc"],
     hdrs = ["shape_fn/trt_shfn.h"],
     visibility = ["//visibility:public"],
     deps = [
-        ":trt_logging",
-        ":trt_plugins",
+        "//tensorflow/compiler/tf2tensorrt:trt_logging",
+        "//tensorflow/compiler/tf2tensorrt:trt_plugins",
     ] + if_tensorrt([
         "@local_config_tensorrt//:tensorrt",
     ]) + tf_custom_op_library_additional_deps(),
 )
 
-cc_library(
-    name = "trt_op_kernels",
-    srcs = [
-        "kernels/get_serialized_resource_op.cc",
-        "kernels/trt_engine_op.cc",
-    ],
-    hdrs = [
-        "kernels/trt_engine_op.h",
-    ],
-    copts = tf_copts(),
-    visibility = ["//visibility:public"],
-    deps = [
-        ":test_utils",
-        ":trt_allocator",
-        ":trt_conversion",
-        ":trt_logging",
-        ":trt_plugins",
-        ":trt_resources",
-        ":utils",
-        "@com_google_absl//absl/memory",
-        "//tensorflow/core:gpu_headers_lib",
-        "//tensorflow/core:lib_proto_parsing",
-        "//tensorflow/core:stream_executor_headers_lib",
-        "//tensorflow/core/grappler/costs:graph_properties",
-    ] + if_tensorrt([
-        "@local_config_tensorrt//:tensorrt",
-    ]) + tf_custom_op_library_additional_deps(),
-    # TODO(laigd): fix this by merging header file in cc file.
-    alwayslink = 1,  # buildozer: disable=alwayslink-with-hdrs
-)
-
-tf_cuda_cc_test(
-    name = "get_serialized_resource_op_test",
-    size = "small",
-    srcs = ["kernels/get_serialized_resource_op_test.cc"],
-    tags = [
-        "no_cuda_on_cpu_tap",
-        "no_windows",
-        "nomac",
-    ],
-    deps = [
-        ":get_serialized_resource_op_op_lib",
-        ":trt_op_kernels",
-        ":trt_resources",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:protos_all_cc",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-        "//tensorflow/core:testlib",
-        "//tensorflow/core/kernels:ops_testutil",
-    ],
-)
-
-tf_gen_op_libs(
-    op_lib_names = [
-        "trt_engine_op",
-        "get_serialized_resource_op",
-    ],
-)
-
-tf_cuda_library(
-    name = "trt_logging",
-    srcs = ["log/trt_logger.cc"],
-    hdrs = ["log/trt_logger.h"],
-    visibility = ["//visibility:public"],
-    deps = [
-        "//tensorflow/core:lib_proto_parsing",
-    ] + if_tensorrt([
-        "@local_config_tensorrt//:tensorrt",
-    ]),
-)
-
-tf_gen_op_wrapper_py(
-    name = "trt_ops",
-    deps = [
-        ":get_serialized_resource_op_op_lib",
-        ":trt_engine_op_op_lib",
-        ":trt_logging",
-        ":trt_shape_function",
-    ],
-)
-
-tf_custom_op_py_library(
-    name = "trt_ops_loader",
-    srcs = ["python/ops/trt_ops.py"],
-    dso = [
-        ":python/ops/_trt_ops.so",
-    ] + if_tensorrt([
-        "@local_config_tensorrt//:tensorrt",
-    ]),
-    kernels = [
-        ":trt_op_kernels",
-        ":trt_engine_op_op_lib",
-        ":get_serialized_resource_op_op_lib",
-        ":trt_shape_function",
-    ],
-    srcs_version = "PY2AND3",
-    deps = [
-        "//tensorflow/contrib/util:util_py",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:resources",
-    ],
-)
-
 py_library(
     name = "init_py",
     srcs = [
@@ -204,8 +59,8 @@ py_library(
     name = "trt_ops_py",
     srcs_version = "PY2AND3",
     deps = [
-        ":trt_ops",
-        ":trt_ops_loader",
+        "//tensorflow/compiler/tf2tensorrt:trt_ops",
+        "//tensorflow/compiler/tf2tensorrt:trt_ops_loader",
     ],
 )
 
@@ -234,264 +89,13 @@ tf_py_wrap_cc(
         "//tensorflow/python:platform/base.i",
     ],
     deps = [
-        ":test_utils",
-        ":trt_conversion",
-        ":trt_op_kernels",
+        "//tensorflow/compiler/tf2tensorrt:test_utils",
+        "//tensorflow/compiler/tf2tensorrt:trt_conversion",
+        "//tensorflow/compiler/tf2tensorrt:trt_op_kernels",
         "//third_party/python_runtime:headers",
     ],
 )
 
-tf_cuda_library(
-    name = "trt_resources",
-    srcs = [
-        "resources/trt_int8_calibrator.cc",
-        "resources/trt_resource_manager.cc",
-        "resources/trt_resources.cc",
-    ],
-    hdrs = [
-        "resources/trt_int8_calibrator.h",
-        "resources/trt_lru_cache.h",
-        "resources/trt_resource_manager.h",
-        "resources/trt_resources.h",
-    ],
-    deps = [
-        ":trt_allocator",
-        ":trt_logging",
-        ":utils",
-        "//tensorflow/core:framework_headers_lib",
-        "//tensorflow/core:framework_lite",
-        "//tensorflow/core:lib_proto_parsing",
-    ] + if_tensorrt([
-        "@local_config_tensorrt//:tensorrt",
-    ]),
-)
-
-tf_cuda_library(
-    name = "trt_allocator",
-    srcs = ["resources/trt_allocator.cc"],
-    hdrs = ["resources/trt_allocator.h"],
-    deps = [
-        "//tensorflow/core:framework_headers_lib",
-        "//tensorflow/core:framework_lite",
-        "//tensorflow/core:lib_proto_parsing",
-    ] + if_tensorrt([
-        "@local_config_tensorrt//:tensorrt",
-    ]),
-)
-
-tf_cc_test(
-    name = "trt_allocator_test",
-    size = "small",
-    srcs = ["resources/trt_allocator_test.cc"],
-    tags = [
-        "no_windows",
-        "nomac",
-    ],
-    deps = [
-        ":trt_allocator",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-    ],
-)
-
-tf_cc_test(
-    name = "trt_lru_cache_test",
-    size = "small",
-    srcs = ["resources/trt_lru_cache_test.cc"],
-    tags = [
-        "no_windows",
-        "nomac",
-    ],
-    deps = [
-        ":trt_resources",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-    ],
-)
-
-# Library for the node-level conversion portion of TensorRT operation creation
-tf_cuda_library(
-    name = "trt_conversion",
-    srcs = [
-        "convert/convert_graph.cc",
-        "convert/convert_nodes.cc",
-        "convert/trt_optimization_pass.cc",
-    ],
-    hdrs = [
-        "convert/convert_graph.h",
-        "convert/convert_nodes.h",
-        "convert/trt_optimization_pass.h",
-    ],
-    deps = [
-        ":segment",
-        ":test_utils",
-        ":trt_allocator",
-        ":trt_plugins",
-        ":trt_logging",
-        ":trt_resources",
-        ":utils",
-        "//tensorflow/core/grappler/clusters:cluster",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
-        "//tensorflow/core/grappler:grappler_item",
-        "//tensorflow/core/grappler:utils",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:framework_lite",
-        "//tensorflow/core:gpu_runtime",
-        "//tensorflow/core:graph",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:lib_internal",
-        "//tensorflow/core:protos_all_cc",
-        "//tensorflow/core/grappler:devices",
-        "//tensorflow/core/grappler/clusters:virtual_cluster",
-        "//tensorflow/core/grappler/costs:graph_properties",
-        "//tensorflow/core/grappler/optimizers:meta_optimizer",
-    ] + if_tensorrt([
-        "@local_config_tensorrt//:tensorrt",
-    ]) + tf_custom_op_library_additional_deps(),
-)
-
-tf_cuda_cc_test(
-    name = "convert_graph_test",
-    size = "medium",
-    srcs = ["convert/convert_graph_test.cc"],
-    tags = [
-        "no_cuda_on_cpu_tap",
-        "no_windows",
-        "nomac",
-    ],
-    deps = [
-        ":trt_conversion",
-        "@com_google_googletest//:gtest",
-        "//tensorflow/cc:cc_ops",
-        "//tensorflow/cc:scope",
-        "//tensorflow/core/grappler:grappler_item",
-        "//tensorflow/core/grappler/clusters:cluster",
-        "//tensorflow/core:core_cpu",
-        "//tensorflow/core:core_cpu_base",
-        "//tensorflow/core:direct_session",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:protos_all_cc",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-        "//tensorflow/core:testlib",
-    ] + if_tensorrt([
-        "@local_config_tensorrt//:tensorrt",
-    ]),
-)
-
-tf_cuda_cc_test(
-    name = "convert_nodes_test",
-    size = "medium",
-    srcs = ["convert/convert_nodes_test.cc"],
-    tags = [
-        "no_cuda_on_cpu_tap",
-        "no_windows",
-        "nomac",
-    ],
-    deps = [
-        ":trt_logging",
-        ":trt_conversion",
-        ":trt_plugins",
-        "@com_google_googletest//:gtest",
-        "//tensorflow/cc:cc_ops",
-        "//tensorflow/cc:ops",
-        "//tensorflow/cc:scope",
-        "//tensorflow/core/grappler/costs:graph_properties",
-        "//tensorflow/core:core_cpu",
-        "//tensorflow/core:core_cpu_base",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:protos_all_cc",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-        "//tensorflow/core:testlib",
-    ] + if_tensorrt([
-        "@local_config_cuda//cuda:cuda_headers",
-        "@local_config_tensorrt//:tensorrt",
-    ]),
-)
-
-# Library for the segmenting portion of TensorRT operation creation
-cc_library(
-    name = "segment",
-    srcs = ["segment/segment.cc"],
-    hdrs = [
-        "segment/segment.h",
-        "segment/union_find.h",
-    ],
-    deps = [
-        "//tensorflow/core:graph",
-        "//tensorflow/core:lib_proto_parsing",
-        "//tensorflow/core:protos_all_cc",
-        "@protobuf_archive//:protobuf_headers",
-    ],
-)
-
-tf_cc_test(
-    name = "segment_test",
-    size = "small",
-    srcs = ["segment/segment_test.cc"],
-    tags = [
-        "no_windows",
-        "nomac",
-    ],
-    deps = [
-        ":segment",
-        "//tensorflow/cc:cc_ops",
-        "//tensorflow/cc:scope",
-        "//tensorflow/core:core_cpu",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:ops",
-        "//tensorflow/core:protos_all_cc",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-        "//tensorflow/core:testlib",
-    ],
-)
-
-# Library for the plugin factory
-tf_cuda_library(
-    name = "trt_plugins",
-    srcs = [
-        "plugin/trt_plugin.cc",
-        "plugin/trt_plugin_factory.cc",
-        "plugin/trt_plugin_utils.cc",
-    ],
-    hdrs = [
-        "plugin/trt_plugin.h",
-        "plugin/trt_plugin_factory.h",
-        "plugin/trt_plugin_utils.h",
-    ],
-    deps = [
-        "//tensorflow/core:framework_lite",
-        "//tensorflow/core:lib_proto_parsing",
-    ] + if_tensorrt([
-        "@local_config_tensorrt//:tensorrt",
-    ]),
-)
-
-tf_cuda_cc_test(
-    name = "trt_plugin_factory_test",
-    size = "small",
-    srcs = ["plugin/trt_plugin_factory_test.cc"],
-    tags = [
-        "no_cuda_on_cpu_tap",
-        "no_windows",
-        "nomac",
-    ],
-    deps = [
-        ":trt_plugins",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-    ] + if_tensorrt([
-        "@local_config_cuda//cuda:cuda_headers",
-        "@local_config_tensorrt//:tensorrt",
-    ]),
-)
-
 py_library(
     name = "tf_trt_integration_test_base",
     srcs = ["test/tf_trt_integration_test_base.py"],
@@ -590,23 +194,20 @@ cuda_py_test(
     ],
 )
 
-cc_library(
-    name = "utils",
-    srcs = ["convert/utils.cc"],
-    hdrs = ["convert/utils.h"],
-    copts = tf_copts(),
-    deps = [
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-    ],
+# The following rules forward the libraries that were moved in order to not
+# break other internal targets.
+
+alias(
+    name = "trt_conversion",
+    actual = "//tensorflow/compiler/tf2tensorrt:trt_conversion",
 )
 
-cc_library(
-    name = "test_utils",
-    srcs = ["test/utils.cc"],
-    hdrs = ["test/utils.h"],
-    deps = [
-        "//tensorflow/core:lib",
-        "@com_googlesource_code_re2//:re2",
-    ],
+alias(
+    name = "trt_op_kernels",
+    actual = "//tensorflow/compiler/tf2tensorrt:trt_op_kernels",
+)
+
+alias(
+    name = "trt_engine_op_op_lib",
+    actual = "//tensorflow/compiler/tf2tensorrt:trt_engine_op_op_lib",
 )
diff --git a/tensorflow/contrib/tensorrt/custom_plugin_examples/BUILD b/tensorflow/contrib/tensorrt/custom_plugin_examples/BUILD
index 09990fb5e3..0a2cf105ba 100644
--- a/tensorflow/contrib/tensorrt/custom_plugin_examples/BUILD
+++ b/tensorflow/contrib/tensorrt/custom_plugin_examples/BUILD
@@ -45,7 +45,7 @@ tf_custom_op_library(
         "inc_op_kernel.cu.cc",
     ],
     deps = [
-        "//tensorflow/contrib/tensorrt:trt_plugins",
+        "//tensorflow/compiler/tf2tensorrt:trt_plugins",
         "//tensorflow/core:framework_lite",
     ] + if_tensorrt([
         "@local_config_tensorrt//:tensorrt",
@@ -64,7 +64,7 @@ tf_kernel_library(
         "inc_op_kernel.cu.cc",
     ],
     deps = [
-        "//tensorflow/contrib/tensorrt:trt_plugins",
+        "//tensorflow/compiler/tf2tensorrt:trt_plugins",
         "//tensorflow/core:stream_executor_headers_lib",
     ] + if_tensorrt([
         "@local_config_tensorrt//:tensorrt",
diff --git a/tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_plugin.cc b/tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_plugin.cc
index 8d4c893af5..7c9075142a 100644
--- a/tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_plugin.cc
+++ b/tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_plugin.cc
@@ -15,8 +15,8 @@ limitations under the License.
 
 #include "tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_plugin.h"
 
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.h"
 #include "tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_kernel.h"
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h"
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
diff --git a/tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_plugin.h b/tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_plugin.h
index 189e9c939b..fb048d7b19 100644
--- a/tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_plugin.h
+++ b/tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_plugin.h
@@ -19,7 +19,7 @@ limitations under the License.
 #include <cassert>
 #include <cstring>
 
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin.h"
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin.h"
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
diff --git a/tensorflow/contrib/tensorrt/python/__init__.py b/tensorflow/contrib/tensorrt/python/__init__.py
index e2cf253ca0..75490aecfb 100644
--- a/tensorflow/contrib/tensorrt/python/__init__.py
+++ b/tensorflow/contrib/tensorrt/python/__init__.py
@@ -19,7 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 # pylint: disable=unused-import,line-too-long
-from tensorflow.contrib.tensorrt.python.ops import trt_ops
+from tensorflow.compiler.tf2tensorrt.python.ops import trt_ops
 from tensorflow.contrib.tensorrt.python.trt_convert import add_test_value
 from tensorflow.contrib.tensorrt.python.trt_convert import calib_graph_to_infer_graph
 from tensorflow.contrib.tensorrt.python.trt_convert import clear_test_values
diff --git a/tensorflow/contrib/tensorrt/python/trt_convert_test.py b/tensorflow/contrib/tensorrt/python/trt_convert_test.py
index 3ef18e3e15..abd822c7b7 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert_test.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert_test.py
@@ -20,10 +20,10 @@ from __future__ import print_function
 
 import os
 
-from tensorflow.contrib.tensorrt.python import trt_convert
 # pylint: disable=unused-import
-from tensorflow.contrib.tensorrt.python.ops import trt_ops
+from tensorflow.compiler.tf2tensorrt.python.ops import trt_ops
 # pylint: enable=unused-import
+from tensorflow.contrib.tensorrt.python import trt_convert
 from tensorflow.core.framework import graph_pb2
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import rewriter_config_pb2
diff --git a/tensorflow/contrib/tensorrt/shape_fn/trt_shfn.cc b/tensorflow/contrib/tensorrt/shape_fn/trt_shfn.cc
index f30dba59ad..5c60d6b589 100644
--- a/tensorflow/contrib/tensorrt/shape_fn/trt_shfn.cc
+++ b/tensorflow/contrib/tensorrt/shape_fn/trt_shfn.cc
@@ -14,14 +14,14 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/contrib/tensorrt/shape_fn/trt_shfn.h"
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h"
 
 #include <string>
 #include <vector>
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-#include "tensorflow/contrib/tensorrt/log/trt_logger.h"
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_logger.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorrt/include/NvInfer.h"
 
diff --git a/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py b/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
index 753b47f4d9..d68211a7ee 100644
--- a/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
+++ b/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
@@ -18,10 +18,10 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.tensorrt.python import trt_convert
 # pylint: disable=unused-import
-from tensorflow.contrib.tensorrt.python.ops import trt_ops
+from tensorflow.compiler.tf2tensorrt.python.ops import trt_ops
 # pylint: enable=unused-import
+from tensorflow.contrib.tensorrt.python import trt_convert
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python import data
 from tensorflow.python import keras
diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
index db43581404..9a00cdb11a 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
@@ -25,10 +25,10 @@ import warnings
 import numpy as np
 import six
 
-from tensorflow.contrib.tensorrt.python import trt_convert
 # pylint: disable=unused-import
-from tensorflow.contrib.tensorrt.python.ops import trt_ops
+from tensorflow.compiler.tf2tensorrt.python.ops import trt_ops
 # pylint: enable=unused-import
+from tensorflow.contrib.tensorrt.python import trt_convert
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python.framework import dtypes
diff --git a/tensorflow/contrib/tensorrt/trt_conversion.i b/tensorflow/contrib/tensorrt/trt_conversion.i
index 6ea15fb8ef..c12895c730 100644
--- a/tensorflow/contrib/tensorrt/trt_conversion.i
+++ b/tensorflow/contrib/tensorrt/trt_conversion.i
@@ -99,9 +99,9 @@ _LIST_OUTPUT_TYPEMAP(int, PyLong_FromLong);
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/util/stat_summarizer.h"
-#include "tensorflow/contrib/tensorrt/convert/convert_graph.h"
-#include "tensorflow/contrib/tensorrt/convert/utils.h"
-#include "tensorflow/contrib/tensorrt/test/utils.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/convert_graph.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/utils.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/test_utils.h"
 %}
 
 %ignoreall
-- 
GitLab


From f5fc7af1b0584672ca29bd4c584481e3dedaacd6 Mon Sep 17 00:00:00 2001
From: Blake Hechtman <blakehechtman@google.com>
Date: Tue, 22 Jan 2019 14:11:54 -0800
Subject: [PATCH 1074/2345] [XLA:CLIENT] Make a simple einsum interface to dot
 general.

PiperOrigin-RevId: 230407712
---
 tensorflow/compiler/xla/client/lib/BUILD      |  21 +-
 tensorflow/compiler/xla/client/lib/matrix.cc  | 259 ++++++++++++++----
 tensorflow/compiler/xla/client/lib/matrix.h   |  38 +++
 .../compiler/xla/client/lib/matrix_test.cc    |  76 +++++
 tensorflow/compiler/xla/tests/BUILD           |   2 +
 .../compiler/xla/tests/dot_operation_test.cc  |  38 +++
 6 files changed, 369 insertions(+), 65 deletions(-)

diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD
index 27f9c61848..ded072faf7 100644
--- a/tensorflow/compiler/xla/client/lib/BUILD
+++ b/tensorflow/compiler/xla/client/lib/BUILD
@@ -214,12 +214,16 @@ cc_library(
         ":arithmetic",
         ":constants",
         "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:status",
         "//tensorflow/compiler/xla:status_macros",
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:types",
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:xla_builder",
+        "@com_google_absl//absl/algorithm:container",
+        "@com_google_absl//absl/container:flat_hash_set",
+        "@com_google_absl//absl/strings",
         "@com_google_absl//absl/types:span",
     ],
 )
@@ -230,12 +234,16 @@ xla_test(
     deps = [
         ":matrix",
         ":slicing",
+        "//tensorflow/compiler/xla:status",
+        "//tensorflow/compiler/xla:status_macros",
+        "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:test",
         "//tensorflow/compiler/xla:types",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:xla_builder",
         "//tensorflow/compiler/xla/tests:client_library_test_base",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+        "@com_google_absl//absl/strings",
     ],
 )
 
@@ -274,7 +282,6 @@ cc_library(
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:xla_builder",
-        "//tensorflow/core:lib",
         "@com_google_absl//absl/base",
     ],
 )
@@ -326,12 +333,7 @@ cc_library(
     srcs = ["slicing.cc"],
     hdrs = ["slicing.h"],
     deps = [
-        "//tensorflow/compiler/xla:shape_util",
-        "//tensorflow/compiler/xla:status_macros",
-        "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:types",
-        "//tensorflow/compiler/xla:util",
-        "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:xla_builder",
         "@com_google_absl//absl/types:span",
     ],
@@ -345,7 +347,6 @@ xla_test(
         "//tensorflow/compiler/xla:literal_util",
         "//tensorflow/compiler/xla:test",
         "//tensorflow/compiler/xla:types",
-        "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:xla_builder",
         "//tensorflow/compiler/xla/tests:client_library_test_base",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
@@ -372,7 +373,6 @@ xla_test(
         ":sorting",
         "//tensorflow/compiler/xla:test",
         "//tensorflow/compiler/xla:types",
-        "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:xla_builder",
         "//tensorflow/compiler/xla/tests:client_library_test_base",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
@@ -466,18 +466,13 @@ xla_test(
         ":triangular_solve",
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:literal",
-        "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:test",
         "//tensorflow/compiler/xla:types",
-        "//tensorflow/compiler/xla:xla_data_proto",
-        "//tensorflow/compiler/xla/client:global_data",
-        "//tensorflow/compiler/xla/client:local_client",
         "//tensorflow/compiler/xla/client:xla_builder",
         "//tensorflow/compiler/xla/tests:client_library_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
-        "//tensorflow/core:lib",
         "//tensorflow/core:test",
     ],
 )
diff --git a/tensorflow/compiler/xla/client/lib/matrix.cc b/tensorflow/compiler/xla/client/lib/matrix.cc
index dcec2139e4..a5aea96090 100644
--- a/tensorflow/compiler/xla/client/lib/matrix.cc
+++ b/tensorflow/compiler/xla/client/lib/matrix.cc
@@ -15,17 +15,25 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/client/lib/matrix.h"
 
+#include <array>
 #include <numeric>
 #include <vector>
 
+#include "absl/algorithm/container.h"
+#include "absl/container/flat_hash_set.h"
+#include "absl/strings/ascii.h"
+#include "absl/strings/str_split.h"
+#include "absl/strings/string_view.h"
 #include "absl/types/span.h"
 #include "tensorflow/compiler/xla/client/lib/arithmetic.h"
 #include "tensorflow/compiler/xla/client/lib/constants.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/status.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/util.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
 
 namespace xla {
 
@@ -91,77 +99,224 @@ XlaOp UpperTriangle(XlaOp x) { return Triangle(x, false); }
 
 XlaOp LowerTriangle(XlaOp x) { return Triangle(x, true); }
 
-XlaOp BatchDot(XlaOp x, XlaOp y, PrecisionConfig::Precision precision) {
+Status ValidateEinsumNumericDimensions(absl::Span<const int64> x_config,
+                                       absl::Span<const int64> y_config,
+                                       absl::Span<const int64> output_config) {
+  for (auto dim : output_config) {
+    if (absl::c_linear_search(x_config, dim) ||
+        absl::c_linear_search(y_config, dim)) {
+      if (absl::c_count(output_config, dim) > 1) {
+        return InvalidArgument("Einsum has repeated output dimension.");
+      }
+      continue;
+    }
+    return InvalidArgument(
+        "Einsum has output dimension without corresponding input dimension.");
+  }
+  for (auto dim : x_config) {
+    if (absl::c_linear_search(y_config, dim) ||
+        absl::c_linear_search(output_config, dim)) {
+      if (absl::c_count(x_config, dim) > 1) {
+        return InvalidArgument("Einsum has repeated lhs dimension.");
+      }
+      continue;
+    }
+    return InvalidArgument(
+        "Einsum has lhs dimension without corresponding rhs or output "
+        "dimension.");
+  }
+  for (auto dim : y_config) {
+    if (absl::c_linear_search(x_config, dim) ||
+        absl::c_linear_search(output_config, dim)) {
+      if (absl::c_count(y_config, dim) > 1) {
+        return InvalidArgument("Einsum has repeated rhs dimension.");
+      }
+      continue;
+    }
+    return InvalidArgument(
+        "Einsum has rhs dimension without corresponding lhs or output "
+        "dimension.");
+  }
+  return Status::OK();
+}
+
+xla::XlaOp Einsum(xla::XlaOp x, absl::Span<const int64> x_config, xla::XlaOp y,
+                  absl::Span<const int64> y_config,
+                  absl::Span<const int64> output_config,
+                  xla::PrecisionConfig::Precision precision) {
   XlaBuilder* builder = x.builder();
   return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
-    TF_ASSIGN_OR_RETURN(Shape x_shape, builder->GetShape(x));
-    TF_ASSIGN_OR_RETURN(Shape y_shape, builder->GetShape(y));
+    TF_RETURN_IF_ERROR(
+        ValidateEinsumNumericDimensions(x_config, y_config, output_config));
+    const int64 x_rank = x_config.size();
+    const int64 y_rank = y_config.size();
+    const int64 output_rank = output_config.size();
+    absl::flat_hash_set<int64> x_map;
+    absl::flat_hash_set<int64> y_map;
+    absl::flat_hash_set<int64> output_map;
 
-    // Check that both tensors have the same number of dimensions. There must be
-    // at least two (the batch dimensions can be empty).
-    if (x_shape.rank() != y_shape.rank()) {
-      return InvalidArgument(
-          "Arguments to BatchDot have different ranks: %s vs. %s",
-          ShapeUtil::HumanString(x_shape), ShapeUtil::HumanString(y_shape));
+    auto find = [&](const absl::flat_hash_set<int64>& map, int64 d) {
+      return map.count(d) != 0;
+    };
+
+    auto insert = [&](absl::flat_hash_set<int64>& map, char d) {
+      CHECK(!find(map, d));
+      map.insert(d);
+    };
+
+    for (auto d : x_config) {
+      insert(x_map, d);
     }
-    const int ndims = x_shape.rank();
-    if (ndims < 2) {
-      return InvalidArgument(
-          "Arguments to BatchDot must have rank >= 2: got %d", ndims);
+
+    for (auto d : y_config) {
+      insert(y_map, d);
     }
 
-    // The batch dimensions must be equal and the matrix dimensions must be
-    // valid.
-    std::vector<int64> batch_dimension_numbers;
-    for (int i = 0; i < ndims - 2; ++i) {
-      if (x_shape.dimensions(i) != y_shape.dimensions(i)) {
-        return InvalidArgument(
-            "Dimension %d of inputs to BatchDot must be equal: shapes %s vs %s",
-            i, ShapeUtil::HumanString(x_shape),
-            ShapeUtil::HumanString(y_shape));
-      }
-      batch_dimension_numbers.push_back(i);
+    for (auto d : output_config) {
+      insert(output_map, d);
     }
 
-    int x_inner_dim = ndims - 1;
-    int y_inner_dim = ndims - 2;
-    if (x_shape.dimensions(x_inner_dim) != y_shape.dimensions(y_inner_dim)) {
-      return InvalidArgument(
-          "Dimensions %d and %d of arguments to BatchDot must be equal: "
-          "shapes %s vs %s",
-          x_inner_dim, y_inner_dim, ShapeUtil::HumanString(x_shape),
-          ShapeUtil::HumanString(y_shape));
+    DotDimensionNumbers dnums;
+    std::vector<int64> lhs_outer_dims;
+    auto is_batch_dim = [&](int64 d) {
+      return find(x_map, d) && find(y_map, d) && find(output_map, d);
+    };
+    auto is_contracting = [&](int64 d) {
+      return find(x_map, d) && find(y_map, d);
+    };
+    auto rhs_dimension_number = [&](int64 d) {
+      return absl::c_find(y_config, d) - y_config.begin();
+    };
+    for (int64 i = 0; i < x_rank; ++i) {
+      auto dim_name = x_config[i];
+      if (is_batch_dim(dim_name)) {
+        dnums.add_lhs_batch_dimensions(i);
+        dnums.add_rhs_batch_dimensions(rhs_dimension_number(dim_name));
+      } else if (is_contracting(dim_name)) {
+        dnums.add_lhs_contracting_dimensions(i);
+        dnums.add_rhs_contracting_dimensions(rhs_dimension_number(dim_name));
+      } else {
+        lhs_outer_dims.push_back(i);
+      }
     }
 
-    // Check for zero lhs/rhs dim size.
-    if (ShapeUtil::IsZeroElementArray(x_shape) ||
-        ShapeUtil::IsZeroElementArray(y_shape)) {
-      std::vector<int64> dimensions(batch_dimension_numbers.size());
-      for (int i = 0; i < batch_dimension_numbers.size(); ++i) {
-        dimensions[i] = x_shape.dimensions(batch_dimension_numbers[i]);
+    std::vector<int64> rhs_outer_dims;
+    for (int64 i = 0; i < y_rank; ++i) {
+      auto dim_name = y_config[i];
+      if (!is_batch_dim(dim_name) && !is_contracting(dim_name)) {
+        rhs_outer_dims.push_back(i);
       }
-      int x_outer_dim = ndims - 2;
-      int y_outer_dim = ndims - 1;
-      dimensions.push_back(x_shape.dimensions(x_outer_dim));
-      dimensions.push_back(y_shape.dimensions(y_outer_dim));
-      return Broadcast(
-          ConstantLiteral(builder, LiteralUtil::Zero(x_shape.element_type())),
-          dimensions);
+    }
+
+    auto output_dimension_number = [&](char d) {
+      return absl::c_find(output_config, d) - output_config.begin();
+    };
+
+    std::vector<int64> output_dims;
+    output_dims.reserve(output_rank);
+    for (auto d : dnums.lhs_batch_dimensions()) {
+      output_dims.push_back(output_dimension_number(x_config[d]));
+    }
+    for (auto d : lhs_outer_dims) {
+      output_dims.push_back(output_dimension_number(x_config[d]));
+    }
+    for (auto d : rhs_outer_dims) {
+      output_dims.push_back(output_dimension_number(y_config[d]));
+    }
+
+    std::vector<int64> transpose_dims(output_rank);
+    for (int64 i = 0; i < output_rank; ++i) {
+      transpose_dims[output_dims[i]] = i;
     }
 
     PrecisionConfig precision_proto;
     precision_proto.add_operand_precision(precision);
     precision_proto.add_operand_precision(precision);
+    return Transpose(DotGeneral(x, y, dnums, &precision_proto), transpose_dims);
+  });
+}
+
+XlaOp BatchDot(XlaOp x, XlaOp y, PrecisionConfig::Precision precision) {
+  XlaBuilder* builder = x.builder();
+  return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    TF_ASSIGN_OR_RETURN(Shape x_shape, builder->GetShape(x));
+    TF_ASSIGN_OR_RETURN(Shape y_shape, builder->GetShape(y));
 
-    DotDimensionNumbers dot_dnums;
-    dot_dnums.add_lhs_contracting_dimensions(x_inner_dim);
-    dot_dnums.add_rhs_contracting_dimensions(y_inner_dim);
-    for (auto batch_dimension_number : batch_dimension_numbers) {
-      dot_dnums.add_lhs_batch_dimensions(batch_dimension_number);
-      dot_dnums.add_rhs_batch_dimensions(batch_dimension_number);
+    // The batch dimensions must be equal and the matrix dimensions must be
+    // valid.
+    std::vector<int64> batch_dimension_numbers;
+    const int ndims = x_shape.rank();
+    batch_dimension_numbers.reserve(ndims - 2);
+    for (int i = 0; i < ndims - 2; ++i) {
+      batch_dimension_numbers.push_back(i);
     }
+    std::vector<int64> x_config = batch_dimension_numbers;
+    x_config.push_back(ndims - 2);
+    x_config.push_back(ndims);
+    std::vector<int64> y_config = batch_dimension_numbers;
+    y_config.push_back(ndims);
+    y_config.push_back(ndims - 1);
+    std::vector<int64> output_config = batch_dimension_numbers;
+    output_config.push_back(ndims - 2);
+    output_config.push_back(ndims - 1);
+    return Einsum(x, x_config, y, y_config, output_config, precision);
+  });
+}
 
-    return DotGeneral(x, y, dot_dnums, &precision_proto);
+StatusOr<std::array<std::vector<int64>, 3>> ParseEinsumString(
+    absl::string_view einsum_config) {
+  std::array<std::vector<int64>, 3> einsum_config_numeric;
+  std::vector<absl::string_view> main_split =
+      absl::StrSplit(einsum_config, ',');
+
+  if (main_split.size() != 2) {
+    return InvalidArgument("Expected one \",\" in einsum_config.");
+  }
+
+  auto maybe_invalid_character = [](char d) {
+    if (absl::ascii_isalpha(d)) {
+      return Status::OK();
+    }
+    if (d == '.') {
+      return InvalidArgument("Unsupported \"...\" or \".\" in einsum config.");
+    }
+    return InvalidArgument("Unexpected character in einsum config.");
+  };
+
+  auto& x_config = einsum_config_numeric[0];
+  x_config.reserve(main_split[0].size());
+  for (auto d : main_split[0]) {
+    TF_RETURN_IF_ERROR(maybe_invalid_character(d));
+    x_config.push_back(static_cast<int64>(d));
+  }
+  std::vector<absl::string_view> y_output_split =
+      absl::StrSplit(main_split[1], "->");
+  if (y_output_split.size() != 2) {
+    return InvalidArgument("Expected one \"->\" in einsum_config.");
+  }
+  auto& y_config = einsum_config_numeric[1];
+  y_config.reserve(y_output_split[0].size());
+  for (auto d : y_output_split[0]) {
+    TF_RETURN_IF_ERROR(maybe_invalid_character(d));
+    y_config.push_back(static_cast<int64>(d));
+  }
+  auto& output_config = einsum_config_numeric[2];
+  output_config.reserve(y_output_split[1].size());
+  for (auto d : y_output_split[1]) {
+    TF_RETURN_IF_ERROR(maybe_invalid_character(d));
+    output_config.push_back(static_cast<int64>(d));
+  }
+  return einsum_config_numeric;
+}
+
+XlaOp Einsum(XlaOp x, XlaOp y, absl::string_view einsum_config,
+             PrecisionConfig::Precision precision) {
+  XlaBuilder* builder = x.builder();
+  return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    TF_ASSIGN_OR_RETURN(auto einsum_config_numeric,
+                        ParseEinsumString(einsum_config));
+    return Einsum(x, einsum_config_numeric[0], y, einsum_config_numeric[1],
+                  einsum_config_numeric[2], precision);
   });
 }
 
diff --git a/tensorflow/compiler/xla/client/lib/matrix.h b/tensorflow/compiler/xla/client/lib/matrix.h
index 916cd83748..491f1eab4c 100644
--- a/tensorflow/compiler/xla/client/lib/matrix.h
+++ b/tensorflow/compiler/xla/client/lib/matrix.h
@@ -16,7 +16,11 @@ limitations under the License.
 #ifndef TENSORFLOW_COMPILER_XLA_CLIENT_LIB_MATRIX_H_
 #define TENSORFLOW_COMPILER_XLA_CLIENT_LIB_MATRIX_H_
 
+#include <array>
+#include "absl/strings/string_view.h"
+#include "absl/types/span.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
 
@@ -65,6 +69,40 @@ xla::XlaOp BatchDot(
     xla::XlaOp x, xla::XlaOp y,
     xla::PrecisionConfig::Precision precision = xla::PrecisionConfig::DEFAULT);
 
+// Parse an einsum string into dimension numbers:
+//   "ab,cb->ac"
+// becomes:
+//   {{0, 1},{2, 1},{0, 2}}
+//
+// NOTE: This function is meant for testing, there is no need to call it
+// directly.
+
+StatusOr<std::array<std::vector<int64>, 3>> ParseEinsumString(
+    absl::string_view einsum_config);
+
+// Determine if each dimension label is in at least two inputs.
+//
+// NOTE: This function is meant for testing, there is no need to call it
+// directly.
+Status ValidateEinsumNumericDimensions(absl::Span<const int64> x_config,
+                                       absl::Span<const int64> y_config,
+                                       absl::Span<const int64> output_config);
+
+// Supports two operand einsum notation like "ab,cb->ac".
+xla::XlaOp Einsum(
+    xla::XlaOp x, xla::XlaOp y, absl::string_view einsum_config,
+    xla::PrecisionConfig::Precision precision = xla::PrecisionConfig::DEFAULT);
+
+// Same as above but supporting numeric labels on dimensins. So "ab,cb->ac"
+// becomes:
+//   x_config = {0, 1}
+//   y_config = {2, 1}
+//   output_config = {0, 2}
+xla::XlaOp Einsum(
+    xla::XlaOp x, absl::Span<const int64> x_config, xla::XlaOp y,
+    absl::Span<const int64> y_config, absl::Span<const int64> output_config,
+    xla::PrecisionConfig::Precision precision = xla::PrecisionConfig::DEFAULT);
+
 // Transposes a stack of matrices `x` by swapping the last two dimensions.
 xla::XlaOp TransposeInMinorDims(xla::XlaOp x);
 
diff --git a/tensorflow/compiler/xla/client/lib/matrix_test.cc b/tensorflow/compiler/xla/client/lib/matrix_test.cc
index 0593a7517a..79cf529ee9 100644
--- a/tensorflow/compiler/xla/client/lib/matrix_test.cc
+++ b/tensorflow/compiler/xla/client/lib/matrix_test.cc
@@ -15,8 +15,11 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/client/lib/matrix.h"
 
+#include "absl/strings/string_view.h"
 #include "tensorflow/compiler/xla/client/lib/slicing.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/compiler/xla/status.h"
+#include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/test.h"
 #include "tensorflow/compiler/xla/tests/client_library_test_base.h"
 #include "tensorflow/compiler/xla/tests/test_macros.h"
@@ -101,5 +104,78 @@ XLA_TEST_F(MatrixTest, RowBatchDot) {
   ComputeAndCompareR3<float>(&builder, {{{33}}, {{292}}},
                              {a_data.get(), row_data.get(), index_data.get()});
 }
+
+XLA_TEST_F(MatrixTest, Einsum) {
+  XlaBuilder builder(TestName());
+
+  int n = 4;
+
+  XlaOp a, row, index;
+  auto a_data =
+      CreateR3Parameter<float>(BatchedAValsFull(), 0, "a", &builder, &a);
+  auto row_data = CreateR3Parameter<float>({{{9, 1, 0, 0}}, {{2, 4, 0, 0}}}, 1,
+                                           "row", &builder, &row);
+  // Select {{3, 6, 0, 1}, {24, 61,  82,  48}} out of BatchedAValsFull().
+  auto index_data = CreateR0Parameter<int>(1, 2, "index", &builder, &index);
+
+  auto l_index = DynamicSliceInMinorDims(
+      a, {index, ConstantR0<int32>(&builder, 0)}, {1, n});
+  Einsum(l_index, row, "abc,adc->abd");
+
+  ComputeAndCompareR3<float>(&builder, {{{33}}, {{292}}},
+                             {a_data.get(), row_data.get(), index_data.get()});
+}
+
+XLA_TEST_F(MatrixTest, ParseEinsumString) {
+  auto to_vec = [](absl::string_view s) {
+    std::vector<int64> v;
+    v.reserve(s.size());
+    for (auto c : s) {
+      v.push_back(int64{c});
+    }
+    return v;
+  };
+
+  auto to_string = [&](absl::string_view x, absl::string_view y,
+                       absl::string_view o) {
+    return absl::StrCat(x, ",", y, "->", o);
+  };
+
+  std::vector<std::vector<string>> good_test_cases = {{"ab", "bc", "ac"},
+                                                      {"Bab", "Bbc", "Bac"},
+                                                      {"ab", "cd", "dcba"},
+                                                      {"abc", "abd", "cbd"}};
+  for (auto test_case : good_test_cases) {
+    auto parse_result_or_status =
+        ParseEinsumString(to_string(test_case[0], test_case[1], test_case[2]));
+    EXPECT_TRUE(parse_result_or_status.status().ok());
+    auto parse_result = parse_result_or_status.ValueOrDie();
+    for (int i = 0; i < 3; ++i) {
+      EXPECT_EQ(parse_result[i], to_vec(test_case[i]));
+    }
+    EXPECT_TRUE(ValidateEinsumNumericDimensions(
+                    parse_result[0], parse_result[1], parse_result[2])
+                    .ok());
+  }
+
+  std::vector<string> einsum_strings_that_fail_parsing = {
+      "", "a", "ab->ba", "ab,bc,cd->ad", "a...b,bc->a...c"};
+  for (auto test_case : einsum_strings_that_fail_parsing) {
+    auto parse_result_or_status = ParseEinsumString(test_case);
+    EXPECT_FALSE(parse_result_or_status.status().ok());
+  }
+
+  std::vector<string> einsum_strings_that_fail_numeric_validation = {
+      "a,b->c", "ab,bc->acd", "abz,bc->ac", "ab,bcz->ac"};
+  for (auto test_case : einsum_strings_that_fail_numeric_validation) {
+    auto parse_result_or_status = ParseEinsumString(test_case);
+    EXPECT_TRUE(parse_result_or_status.status().ok());
+    auto parse_result = parse_result_or_status.ValueOrDie();
+    EXPECT_FALSE(ValidateEinsumNumericDimensions(
+                     parse_result[0], parse_result[1], parse_result[2])
+                     .ok());
+  }
+}
+
 }  // namespace
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 4e7480c9af..7e05a7956e 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -730,6 +730,7 @@ xla_test(
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla/client:local_client",
         "//tensorflow/compiler/xla/client:xla_builder",
+        "//tensorflow/compiler/xla/client/lib:matrix",
         "//tensorflow/compiler/xla/tests:client_library_test_base",
         "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
@@ -788,6 +789,7 @@ xla_test(
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla/client:local_client",
         "//tensorflow/compiler/xla/client:xla_builder",
+        "//tensorflow/compiler/xla/client/lib:matrix",
         "//tensorflow/compiler/xla/tests:client_library_test_base",
         "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
diff --git a/tensorflow/compiler/xla/tests/dot_operation_test.cc b/tensorflow/compiler/xla/tests/dot_operation_test.cc
index 4576a04ec3..f740f48158 100644
--- a/tensorflow/compiler/xla/tests/dot_operation_test.cc
+++ b/tensorflow/compiler/xla/tests/dot_operation_test.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include "absl/strings/str_cat.h"
 #include "tensorflow/compiler/xla/array2d.h"
 #include "tensorflow/compiler/xla/array3d.h"
+#include "tensorflow/compiler/xla/client/lib/matrix.h"
 #include "tensorflow/compiler/xla/client/local_client.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/primitive_util.h"
@@ -1157,6 +1158,43 @@ XLA_TEST_F(DotOperationTest, DotRank2AndRank2NonDefaultContractionDims) {
   ComputeAndCompareR2<float>(&builder, expected, {}, error_spec_);
 }
 
+using EinsumParamType =
+    std::tuple<std::vector<int64>, std::vector<int64>, string>;
+class EinsumTest : public DotOperationTest,
+                   public ::testing::WithParamInterface<EinsumParamType> {};
+XLA_TEST_P(EinsumTest, SimpleEinsumTest) {
+  XlaBuilder builder(TestName());
+  auto x = AddParam(
+      MakeFakeLiteral(ShapeUtil::MakeShape(F32, std::get<0>(GetParam())))
+          .ValueOrDie(),
+      &builder);
+  auto y = AddParam(
+      MakeFakeLiteral(ShapeUtil::MakeShape(F32, std::get<1>(GetParam())))
+          .ValueOrDie(),
+      &builder);
+  Einsum(x, y, std::get<2>(GetParam()));
+  ComputeAndCompare(&builder, {}, ErrorSpec{1e-3, 1e-3});
+}
+
+std::vector<EinsumParamType> GetEinsumTestCases() {
+  using v = std::vector<int64>;
+  using p = EinsumParamType;
+  std::vector<p> test_cases = {
+      p{v{5, 6}, v{6, 7}, "mk,kn->mn"},
+      p{v{5, 6}, v{6, 7}, "mk,kn->nm"},
+      p{v{5, 6, 11}, v{6, 11, 7}, "mkB,kBn->nmB"},
+      p{v{31, 55, 11}, v{55, 11, 29}, "mkB,kBn->nmB"},
+      p{v{31, 55, 11}, v{55, 11, 29}, "mkB,kBn->Bnm"},
+      p{v{8, 55, 11, 3}, v{55, 11, 3, 29}, "mkBC,kBCn->BCnm"},
+      p{v{5, 6}, v{6, 7}, "ab,cd->dcba"},
+      p{v{6}, v{6, 7}, "b,bc->c"},
+  };
+  return test_cases;
+}
+
+INSTANTIATE_TEST_CASE_P(Einsum, EinsumTest,
+                        ::testing::ValuesIn(GetEinsumTestCases()));
+
 class DotOperationTextTest : public HloTestBase {};
 
 XLA_TEST_F(DotOperationTextTest, DotReorderedDotDims) {
-- 
GitLab


From 4e15557ac4c92f219f51ed94883d1e47dca88417 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 14:16:37 -0800
Subject: [PATCH 1075/2345] Update references to bounds_checks.h

PiperOrigin-RevId: 230408522
---
 .../compiler/jit/mark_for_compilation_pass.cc |  2 +-
 tensorflow/compiler/jit/xla_cluster_util.cc   |  2 +-
 .../compiler/tf2xla/kernels/concat_op.cc      |  2 +-
 .../tf2xla/kernels/conv_op_helpers.cc         |  2 +-
 .../compiler/tf2xla/kernels/conv_ops.cc       |  2 +-
 .../tf2xla/kernels/dynamic_stitch_op.cc       |  2 +-
 tensorflow/compiler/tf2xla/kernels/fft_ops.cc |  2 +-
 .../compiler/tf2xla/kernels/index_ops.cc      |  2 +-
 .../compiler/tf2xla/kernels/index_ops_cpu.cc  |  2 +-
 tensorflow/compiler/tf2xla/kernels/pack_op.cc |  2 +-
 .../compiler/tf2xla/kernels/pooling_ops.cc    |  2 +-
 .../compiler/tf2xla/kernels/scan_ops.cc       |  2 +-
 .../compiler/tf2xla/kernels/select_op.cc      |  2 +-
 .../compiler/tf2xla/kernels/shape_op.cc       |  2 +-
 .../compiler/tf2xla/kernels/shape_util.cc     |  2 +-
 .../compiler/tf2xla/kernels/stack_ops.cc      |  2 +-
 .../tf2xla/kernels/tensor_array_ops.cc        |  2 +-
 .../tf2xla/kernels/tensor_list_ops.cc         |  2 +-
 .../compiler/tf2xla/kernels/transpose_op.cc   |  2 +-
 .../compiler/tf2xla/kernels/unpack_op.cc      |  2 +-
 .../kernels/masked_matmul_ops.cc              |  2 +-
 .../fused_conv2d_bias_activation_op.cc        |  2 +-
 .../tensor_forest/kernels/tree_utils.h        |  2 +-
 .../core/common_runtime/eval_const_tensor.cc  |  2 +-
 .../core/common_runtime/shape_refiner.cc      |  2 +-
 tensorflow/core/framework/shape_inference.cc  |  2 +-
 tensorflow/core/framework/tensor_shape.cc     |  2 +-
 tensorflow/core/kernels/BUILD                 |  9 ++------
 tensorflow/core/kernels/argmax_op.cc          |  2 +-
 tensorflow/core/kernels/bias_op.cc            |  2 +-
 tensorflow/core/kernels/bounds_check.h        | 21 -------------------
 tensorflow/core/kernels/concat_op.cc          |  4 ++--
 tensorflow/core/kernels/constant_op.cc        |  2 +-
 tensorflow/core/kernels/conv_ops.cc           |  2 +-
 tensorflow/core/kernels/conv_ops_fused.cc     |  2 +-
 .../kernels/conv_ops_fused_image_transform.cc |  2 +-
 .../core/kernels/conv_ops_using_gemm.cc       |  2 +-
 tensorflow/core/kernels/crop_and_resize_op.cc |  2 +-
 tensorflow/core/kernels/ctc_loss_op.cc        |  2 +-
 tensorflow/core/kernels/cwise_op_select.cc    |  2 +-
 tensorflow/core/kernels/cwise_ops.h           |  2 +-
 tensorflow/core/kernels/decode_bmp_op.cc      |  2 +-
 .../core/kernels/depthwise_conv_grad_op.cc    |  2 +-
 tensorflow/core/kernels/depthwise_conv_op.cc  |  2 +-
 .../core/kernels/dynamic_partition_op.cc      |  2 +-
 .../kernels/dynamic_partition_op_gpu.cu.cc    |  2 +-
 tensorflow/core/kernels/dynamic_stitch_op.cc  |  2 +-
 tensorflow/core/kernels/encode_jpeg_op.cc     |  2 +-
 tensorflow/core/kernels/encode_png_op.cc      |  2 +-
 tensorflow/core/kernels/encode_wav_op.cc      |  2 +-
 .../core/kernels/extract_image_patches_op.cc  |  2 +-
 .../core/kernels/extract_jpeg_shape_op.cc     |  2 +-
 .../core/kernels/extract_volume_patches_op.cc |  2 +-
 tensorflow/core/kernels/gather_functor.h      |  2 +-
 tensorflow/core/kernels/gather_nd_op.cc       |  2 +-
 tensorflow/core/kernels/gather_nd_op.h        |  2 +-
 .../core/kernels/gather_nd_op_cpu_impl.h      |  2 +-
 tensorflow/core/kernels/gather_op.cc          |  2 +-
 tensorflow/core/kernels/image_resizer_state.h |  2 +-
 tensorflow/core/kernels/in_topk_op.cc         |  2 +-
 tensorflow/core/kernels/lookup_table_op.h     |  2 +-
 tensorflow/core/kernels/lrn_op.cc             |  2 +-
 tensorflow/core/kernels/maxpooling_op.cc      |  2 +-
 tensorflow/core/kernels/mkl_concat_op.cc      |  2 +-
 tensorflow/core/kernels/mkl_conv_ops.cc       |  2 +-
 tensorflow/core/kernels/mkl_conv_ops.h        |  2 +-
 tensorflow/core/kernels/mkl_lrn_op.cc         |  2 +-
 .../core/kernels/mkl_pooling_ops_common.cc    |  2 +-
 .../kernels/neon/neon_depthwise_conv_op.cc    |  2 +-
 .../core/kernels/non_max_suppression_op.cc    |  2 +-
 .../core/kernels/resource_variable_ops.cc     |  2 +-
 tensorflow/core/kernels/reverse_op.cc         |  2 +-
 tensorflow/core/kernels/roll_op.cc            |  2 +-
 .../sample_distorted_bounding_box_op.cc       |  2 +-
 .../core/kernels/save_restore_tensor.cc       |  2 +-
 .../core/kernels/save_restore_v2_ops.cc       |  2 +-
 .../core/kernels/scale_and_translate_op.cc    |  2 +-
 tensorflow/core/kernels/scan_ops.cc           |  2 +-
 tensorflow/core/kernels/scatter_functor.h     |  2 +-
 tensorflow/core/kernels/scatter_nd_op.cc      |  2 +-
 tensorflow/core/kernels/scatter_nd_op.h       |  2 +-
 .../core/kernels/scatter_nd_op_cpu_impl.h     |  2 +-
 tensorflow/core/kernels/searchsorted_op.cc    |  2 +-
 .../core/kernels/segment_reduction_ops.cc     |  4 +++-
 tensorflow/core/kernels/shape_ops.h           |  2 +-
 .../core/kernels/spacetobatch_functor.h       |  2 +-
 .../kernels/sparse_tensor_dense_matmul_op.cc  |  2 +-
 .../sparse_tensor_dense_matmul_op_gpu.cu.cc   |  2 +-
 tensorflow/core/kernels/sparse_xent_op.h      |  2 +-
 tensorflow/core/kernels/split_op.cc           |  2 +-
 tensorflow/core/kernels/split_v_op.cc         |  2 +-
 .../core/kernels/stateful_random_ops.cc       |  2 +-
 .../core/kernels/stateless_random_ops.cc      |  2 +-
 tensorflow/core/kernels/strided_slice_op.cc   |  2 +-
 .../core/kernels/strided_slice_op_impl.h      |  2 +-
 tensorflow/core/kernels/substr_op.cc          |  2 +-
 tensorflow/core/kernels/tensor_array_ops.cc   |  2 +-
 tensorflow/core/kernels/training_ops.cc       |  2 +-
 tensorflow/core/kernels/transpose_op.cc       |  2 +-
 tensorflow/core/kernels/unicode_ops.cc        |  2 +-
 tensorflow/core/kernels/unique_op.cc          |  2 +-
 tensorflow/core/kernels/unpack_op.cc          |  2 +-
 tensorflow/core/kernels/where_op.cc           |  2 +-
 tensorflow/core/kernels/where_op_gpu.cu.h     |  2 +-
 tensorflow/core/util/sparse/dim_comparator.h  |  2 +-
 tensorflow/core/util/sparse/sparse_tensor.h   |  2 +-
 tensorflow/core/util/strided_slice_op.cc      |  2 +-
 107 files changed, 110 insertions(+), 134 deletions(-)
 delete mode 100644 tensorflow/core/kernels/bounds_check.h

diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass.cc b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
index cb01690845..234e8052d1 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
@@ -34,6 +34,7 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/util.h"
 #include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/graph_def_util.h"
 #include "tensorflow/core/framework/memory_types.h"
 #include "tensorflow/core/framework/node_def.pb.h"
@@ -42,7 +43,6 @@ limitations under the License.
 #include "tensorflow/core/graph/algorithm.h"
 #include "tensorflow/core/graph/control_flow.h"
 #include "tensorflow/core/graph/graph_constructor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/gtl/cleanup.h"
 #include "tensorflow/core/lib/strings/stringprintf.h"
 #include "tensorflow/core/public/version.h"
diff --git a/tensorflow/compiler/jit/xla_cluster_util.cc b/tensorflow/compiler/jit/xla_cluster_util.cc
index fef28fc810..80993861ab 100644
--- a/tensorflow/compiler/jit/xla_cluster_util.cc
+++ b/tensorflow/compiler/jit/xla_cluster_util.cc
@@ -19,9 +19,9 @@ limitations under the License.
 
 #include "absl/strings/str_cat.h"
 #include "tensorflow/compiler/jit/resource_operation_safety_analysis.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/graph/control_flow.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/util/device_name_utils.h"
 
 namespace tensorflow {
diff --git a/tensorflow/compiler/tf2xla/kernels/concat_op.cc b/tensorflow/compiler/tf2xla/kernels/concat_op.cc
index cd7c7f4a82..91e4d9cea7 100644
--- a/tensorflow/compiler/tf2xla/kernels/concat_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/concat_op.cc
@@ -24,13 +24,13 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/concat_lib.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
diff --git a/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc b/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
index 61a8933208..5f99b24e22 100644
--- a/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
+++ b/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
@@ -26,13 +26,13 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/lib/constants.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/node_def_util.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_slice.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/conv_grad_ops.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/util/padding.h"
diff --git a/tensorflow/compiler/tf2xla/kernels/conv_ops.cc b/tensorflow/compiler/tf2xla/kernels/conv_ops.cc
index eafdba876a..52c3c2c4a9 100644
--- a/tensorflow/compiler/tf2xla/kernels/conv_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/conv_ops.cc
@@ -25,13 +25,13 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/lib/matrix.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/node_def_util.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_slice.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/conv_grad_ops.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/util/padding.h"
diff --git a/tensorflow/compiler/tf2xla/kernels/dynamic_stitch_op.cc b/tensorflow/compiler/tf2xla/kernels/dynamic_stitch_op.cc
index 6e6ba21daf..b119997cf3 100644
--- a/tensorflow/compiler/tf2xla/kernels/dynamic_stitch_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/dynamic_stitch_op.cc
@@ -22,10 +22,10 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 
 namespace tensorflow {
 namespace {
diff --git a/tensorflow/compiler/tf2xla/kernels/fft_ops.cc b/tensorflow/compiler/tf2xla/kernels/fft_ops.cc
index 6df8b5367d..a623585aad 100644
--- a/tensorflow/compiler/tf2xla/kernels/fft_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/fft_ops.cc
@@ -21,12 +21,12 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal_util.h"
 #include "tensorflow/compiler/xla/util.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_slice.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/conv_grad_ops.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/util/padding.h"
diff --git a/tensorflow/compiler/tf2xla/kernels/index_ops.cc b/tensorflow/compiler/tf2xla/kernels/index_ops.cc
index 978e9480ea..c1539f48d4 100644
--- a/tensorflow/compiler/tf2xla/kernels/index_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/index_ops.cc
@@ -22,12 +22,12 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/lib/arithmetic.h"
 #include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 
 namespace tensorflow {
 XlaArgMinMaxOp::XlaArgMinMaxOp(OpKernelConstruction* ctx, bool is_min)
diff --git a/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc b/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc
index 30b993045c..e4bbdef648 100644
--- a/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc
+++ b/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc
@@ -20,12 +20,12 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/lib/arithmetic.h"
 #include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 
 namespace tensorflow {
 namespace {
diff --git a/tensorflow/compiler/tf2xla/kernels/pack_op.cc b/tensorflow/compiler/tf2xla/kernels/pack_op.cc
index a9b519d892..426a0941df 100644
--- a/tensorflow/compiler/tf2xla/kernels/pack_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/pack_op.cc
@@ -24,12 +24,12 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/concat_lib.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
diff --git a/tensorflow/compiler/tf2xla/kernels/pooling_ops.cc b/tensorflow/compiler/tf2xla/kernels/pooling_ops.cc
index 06c6cc37ec..23bb050a34 100644
--- a/tensorflow/compiler/tf2xla/kernels/pooling_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/pooling_ops.cc
@@ -26,10 +26,10 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/xla_computation.h"
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/util.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/conv_grad_ops.h"
 #include "tensorflow/core/kernels/pooling_ops_common.h"
 
diff --git a/tensorflow/compiler/tf2xla/kernels/scan_ops.cc b/tensorflow/compiler/tf2xla/kernels/scan_ops.cc
index 4b9e1a578b..daefdfc58a 100644
--- a/tensorflow/compiler/tf2xla/kernels/scan_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/scan_ops.cc
@@ -23,13 +23,13 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/client/xla_computation.h"
 #include "tensorflow/compiler/xla/literal.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/concat_lib.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
diff --git a/tensorflow/compiler/tf2xla/kernels/select_op.cc b/tensorflow/compiler/tf2xla/kernels/select_op.cc
index 9e4c57c9bf..aaf8c6075d 100644
--- a/tensorflow/compiler/tf2xla/kernels/select_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/select_op.cc
@@ -20,8 +20,8 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 
 namespace tensorflow {
 namespace {
diff --git a/tensorflow/compiler/tf2xla/kernels/shape_op.cc b/tensorflow/compiler/tf2xla/kernels/shape_op.cc
index 85b0367f73..31d4cc1316 100644
--- a/tensorflow/compiler/tf2xla/kernels/shape_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/shape_op.cc
@@ -22,9 +22,9 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/lib/constants.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 
 namespace tensorflow {
 namespace {
diff --git a/tensorflow/compiler/tf2xla/kernels/shape_util.cc b/tensorflow/compiler/tf2xla/kernels/shape_util.cc
index 76ea5f5255..b18e3f965c 100644
--- a/tensorflow/compiler/tf2xla/kernels/shape_util.cc
+++ b/tensorflow/compiler/tf2xla/kernels/shape_util.cc
@@ -17,7 +17,7 @@ limitations under the License.
 
 #include <limits>
 
-#include "tensorflow/core/kernels/bounds_check.h"
+#include "tensorflow/core/framework/bounds_check.h"
 
 namespace tensorflow {
 
diff --git a/tensorflow/compiler/tf2xla/kernels/stack_ops.cc b/tensorflow/compiler/tf2xla/kernels/stack_ops.cc
index d0c5231e84..b6c96b1f58 100644
--- a/tensorflow/compiler/tf2xla/kernels/stack_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/stack_ops.cc
@@ -24,13 +24,13 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/literal.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/concat_lib.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
diff --git a/tensorflow/compiler/tf2xla/kernels/tensor_array_ops.cc b/tensorflow/compiler/tf2xla/kernels/tensor_array_ops.cc
index 22e64789a3..77a3e5c001 100644
--- a/tensorflow/compiler/tf2xla/kernels/tensor_array_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/tensor_array_ops.cc
@@ -27,13 +27,13 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_resource.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/concat_lib.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
diff --git a/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc b/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc
index 5dda62fd49..6757a21190 100644
--- a/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc
@@ -24,13 +24,13 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/literal.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/concat_lib.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
diff --git a/tensorflow/compiler/tf2xla/kernels/transpose_op.cc b/tensorflow/compiler/tf2xla/kernels/transpose_op.cc
index c9b324a243..76793d677b 100644
--- a/tensorflow/compiler/tf2xla/kernels/transpose_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/transpose_op.cc
@@ -24,9 +24,9 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
 #include "tensorflow/core/framework/register_types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 
 namespace tensorflow {
 namespace {
diff --git a/tensorflow/compiler/tf2xla/kernels/unpack_op.cc b/tensorflow/compiler/tf2xla/kernels/unpack_op.cc
index 8671632976..2fc5619de7 100644
--- a/tensorflow/compiler/tf2xla/kernels/unpack_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/unpack_op.cc
@@ -24,12 +24,12 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/concat_lib.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
diff --git a/tensorflow/contrib/factorization/kernels/masked_matmul_ops.cc b/tensorflow/contrib/factorization/kernels/masked_matmul_ops.cc
index a8c5d0763c..68078ba8bb 100644
--- a/tensorflow/contrib/factorization/kernels/masked_matmul_ops.cc
+++ b/tensorflow/contrib/factorization/kernels/masked_matmul_ops.cc
@@ -19,12 +19,12 @@
 #include <numeric>
 #include <vector>
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/threadpool.h"
 
diff --git a/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc b/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc
index c541c71f99..b6b75ffa24 100644
--- a/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc
+++ b/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc
@@ -19,13 +19,13 @@ limitations under the License.
 
 #include "tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.h"
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_slice.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/conv_2d.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/lib/core/errors.h"
diff --git a/tensorflow/contrib/tensor_forest/kernels/tree_utils.h b/tensorflow/contrib/tensor_forest/kernels/tree_utils.h
index e04eb60f9b..774da472f1 100644
--- a/tensorflow/contrib/tensor_forest/kernels/tree_utils.h
+++ b/tensorflow/contrib/tensor_forest/kernels/tree_utils.h
@@ -18,10 +18,10 @@
 #include <limits>
 
 #include "tensorflow/contrib/tensor_forest/kernels/data_spec.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/random/distribution_sampler.h"
 #include "tensorflow/core/lib/random/simple_philox.h"
 #include "tensorflow/core/lib/strings/strcat.h"
diff --git a/tensorflow/core/common_runtime/eval_const_tensor.cc b/tensorflow/core/common_runtime/eval_const_tensor.cc
index 87749da7af..fb51e2dec3 100644
--- a/tensorflow/core/common_runtime/eval_const_tensor.cc
+++ b/tensorflow/core/common_runtime/eval_const_tensor.cc
@@ -19,11 +19,11 @@ limitations under the License.
 
 #include "tensorflow/core/common_runtime/graph_runner.h"
 #include "tensorflow/core/common_runtime/shape_refiner.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/versions.pb.h"
 #include "tensorflow/core/graph/graph.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 
 namespace tensorflow {
 
diff --git a/tensorflow/core/common_runtime/shape_refiner.cc b/tensorflow/core/common_runtime/shape_refiner.cc
index 9488a44778..8f28d27903 100644
--- a/tensorflow/core/common_runtime/shape_refiner.cc
+++ b/tensorflow/core/common_runtime/shape_refiner.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include <vector>
 
 #include "tensorflow/core/common_runtime/eval_const_tensor.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/common_shape_fns.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/tensor.h"
@@ -27,7 +28,6 @@ limitations under the License.
 #include "tensorflow/core/framework/versions.pb.h"
 #include "tensorflow/core/graph/algorithm.h"
 #include "tensorflow/core/graph/graph_constructor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/gtl/stl_util.h"
 #include "tensorflow/core/public/session.h"
diff --git a/tensorflow/core/framework/shape_inference.cc b/tensorflow/core/framework/shape_inference.cc
index 4dcc80680f..bffbee50b5 100644
--- a/tensorflow/core/framework/shape_inference.cc
+++ b/tensorflow/core/framework/shape_inference.cc
@@ -14,10 +14,10 @@ limitations under the License.
 ==============================================================================*/
 #include "tensorflow/core/framework/shape_inference.h"
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/node_def.pb_text.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/tensor_shape.pb.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/strings/numbers.h"
 #include "tensorflow/core/lib/strings/scanner.h"
diff --git a/tensorflow/core/framework/tensor_shape.cc b/tensorflow/core/framework/tensor_shape.cc
index 5e0b976e17..be451d1e31 100644
--- a/tensorflow/core/framework/tensor_shape.cc
+++ b/tensorflow/core/framework/tensor_shape.cc
@@ -15,8 +15,8 @@ limitations under the License.
 
 #include "tensorflow/core/framework/tensor_shape.h"
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/tensor_shape.pb.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 9fcd4106fa..049d429d36 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -649,13 +649,10 @@ cc_library(
     ],
 )
 
-cc_library(
+alias(
     name = "bounds_check",
-    hdrs = ["bounds_check.h"],
+    actual = "//tensorflow/core:framework_bounds_check",
     visibility = [":friends"],
-    deps = [
-        "//tensorflow/core:framework_bounds_check",
-    ],
 )
 
 # Private support libraries ---------------------------------------------------
@@ -5439,7 +5436,6 @@ filegroup(
     srcs = [
         "avgpooling_op.h",
         "batch_util.h",
-        "bounds_check.h",
         "cwise_ops.h",
         "cwise_ops_common.h",
         "cwise_ops_gradients.h",
@@ -5480,7 +5476,6 @@ filegroup(
         "assign_op.h",
         "bias_op.cc",
         "bias_op.h",
-        "bounds_check.h",
         "cast_op.cc",
         "cast_op.h",
         "cast_op_impl.h",
diff --git a/tensorflow/core/kernels/argmax_op.cc b/tensorflow/core/kernels/argmax_op.cc
index c731b64993..778f818a61 100644
--- a/tensorflow/core/kernels/argmax_op.cc
+++ b/tensorflow/core/kernels/argmax_op.cc
@@ -25,13 +25,13 @@ limitations under the License.
 
 #include <memory>
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/macros.h"
 
diff --git a/tensorflow/core/kernels/bias_op.cc b/tensorflow/core/kernels/bias_op.cc
index 9006fb46fd..074f64a634 100644
--- a/tensorflow/core/kernels/bias_op.cc
+++ b/tensorflow/core/kernels/bias_op.cc
@@ -19,11 +19,11 @@ limitations under the License.
 
 #include "tensorflow/core/kernels/bias_op.h"
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/util/tensor_format.h"
 
 #if GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/bounds_check.h b/tensorflow/core/kernels/bounds_check.h
deleted file mode 100644
index ce6ec1012d..0000000000
--- a/tensorflow/core/kernels/bounds_check.h
+++ /dev/null
@@ -1,21 +0,0 @@
-/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_CORE_KERNELS_BOUNDS_CHECK_H_
-#define TENSORFLOW_CORE_KERNELS_BOUNDS_CHECK_H_
-
-#include "tensorflow/core/framework/bounds_check.h"
-
-#endif  // TENSORFLOW_CORE_KERNELS_BOUNDS_CHECK_H_
diff --git a/tensorflow/core/kernels/concat_op.cc b/tensorflow/core/kernels/concat_op.cc
index ff62983517..72d8b45dd9 100644
--- a/tensorflow/core/kernels/concat_op.cc
+++ b/tensorflow/core/kernels/concat_op.cc
@@ -18,16 +18,16 @@ limitations under the License.
 #include <limits>
 #include <vector>
 
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/concat_lib.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
 namespace tensorflow {
 
diff --git a/tensorflow/core/kernels/constant_op.cc b/tensorflow/core/kernels/constant_op.cc
index 75ca77fad5..5ff428dd31 100644
--- a/tensorflow/core/kernels/constant_op.cc
+++ b/tensorflow/core/kernels/constant_op.cc
@@ -24,6 +24,7 @@ limitations under the License.
 #include "tensorflow/core/kernels/constant_op.h"
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
@@ -32,7 +33,6 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/framework/variant_op_registry.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/fill_functor.h"
 #include "tensorflow/core/platform/macros.h"
 
diff --git a/tensorflow/core/kernels/conv_ops.cc b/tensorflow/core/kernels/conv_ops.cc
index a8138fd0a7..979c76dc3c 100644
--- a/tensorflow/core/kernels/conv_ops.cc
+++ b/tensorflow/core/kernels/conv_ops.cc
@@ -28,13 +28,13 @@ limitations under the License.
 #include <map>
 #include <vector>
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_slice.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/conv_2d.h"
 #include "tensorflow/core/kernels/deep_conv2d.h"
 #include "tensorflow/core/kernels/ops_util.h"
diff --git a/tensorflow/core/kernels/conv_ops_fused.cc b/tensorflow/core/kernels/conv_ops_fused.cc
index 551782c488..9c807c3375 100644
--- a/tensorflow/core/kernels/conv_ops_fused.cc
+++ b/tensorflow/core/kernels/conv_ops_fused.cc
@@ -41,11 +41,11 @@ limitations under the License.
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_join.h"
 #include "absl/strings/substitute.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/conv_2d.h"
 #include "tensorflow/core/kernels/conv_ops.h"
 #include "tensorflow/core/kernels/ops_util.h"
diff --git a/tensorflow/core/kernels/conv_ops_fused_image_transform.cc b/tensorflow/core/kernels/conv_ops_fused_image_transform.cc
index 7be1de29c9..0542216a23 100644
--- a/tensorflow/core/kernels/conv_ops_fused_image_transform.cc
+++ b/tensorflow/core/kernels/conv_ops_fused_image_transform.cc
@@ -21,6 +21,7 @@ limitations under the License.
 
 #include <string>
 #include <vector>
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/common_shape_fns.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
@@ -29,7 +30,6 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_slice.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/conv_2d.h"
 #include "tensorflow/core/kernels/conv_ops.h"
 #include "tensorflow/core/kernels/gemm_functors.h"
diff --git a/tensorflow/core/kernels/conv_ops_using_gemm.cc b/tensorflow/core/kernels/conv_ops_using_gemm.cc
index af0a9fa82e..05df9e0207 100644
--- a/tensorflow/core/kernels/conv_ops_using_gemm.cc
+++ b/tensorflow/core/kernels/conv_ops_using_gemm.cc
@@ -49,6 +49,7 @@ limitations under the License.
 #include <string.h>
 #include <map>
 #include <vector>
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/common_shape_fns.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
@@ -57,7 +58,6 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_slice.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/conv_ops.h"
 #include "tensorflow/core/kernels/gemm_functors.h"
 #include "tensorflow/core/kernels/image_resizer_state.h"
diff --git a/tensorflow/core/kernels/crop_and_resize_op.cc b/tensorflow/core/kernels/crop_and_resize_op.cc
index 99d01b4db6..838cedd7a4 100644
--- a/tensorflow/core/kernels/crop_and_resize_op.cc
+++ b/tensorflow/core/kernels/crop_and_resize_op.cc
@@ -23,11 +23,11 @@ limitations under the License.
 #include <string>
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/logging.h"
diff --git a/tensorflow/core/kernels/ctc_loss_op.cc b/tensorflow/core/kernels/ctc_loss_op.cc
index fb375ee4b3..aa68e105ad 100644
--- a/tensorflow/core/kernels/ctc_loss_op.cc
+++ b/tensorflow/core/kernels/ctc_loss_op.cc
@@ -15,10 +15,10 @@ limitations under the License.
 
 // See docs in ../ops/ctc_ops.cc.
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/macros.h"
 #include "tensorflow/core/util/ctc/ctc_loss_calculator.h"
diff --git a/tensorflow/core/kernels/cwise_op_select.cc b/tensorflow/core/kernels/cwise_op_select.cc
index dd4e4ea547..3b51563ca2 100644
--- a/tensorflow/core/kernels/cwise_op_select.cc
+++ b/tensorflow/core/kernels/cwise_op_select.cc
@@ -19,8 +19,8 @@ limitations under the License.
 #define EIGEN_USE_GPU
 #endif  // GOOGLE_CUDA
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/register_types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/cwise_ops_common.h"
 #include "tensorflow/core/platform/prefetch.h"
 
diff --git a/tensorflow/core/kernels/cwise_ops.h b/tensorflow/core/kernels/cwise_ops.h
index a22d76717a..e7bc7f86cc 100644
--- a/tensorflow/core/kernels/cwise_ops.h
+++ b/tensorflow/core/kernels/cwise_ops.h
@@ -22,9 +22,9 @@ limitations under the License.
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_types.h"
 #include "tensorflow/core/framework/tensor_types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 
 namespace Eigen {
 namespace internal {
diff --git a/tensorflow/core/kernels/decode_bmp_op.cc b/tensorflow/core/kernels/decode_bmp_op.cc
index ae451be7e2..c75fc94bc1 100644
--- a/tensorflow/core/kernels/decode_bmp_op.cc
+++ b/tensorflow/core/kernels/decode_bmp_op.cc
@@ -16,13 +16,13 @@ limitations under the License.
 // See docs in ../ops/image_ops.cc
 
 #include <memory>
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/framework/types.pb.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/logging.h"
 
diff --git a/tensorflow/core/kernels/depthwise_conv_grad_op.cc b/tensorflow/core/kernels/depthwise_conv_grad_op.cc
index c152f2b7e4..ab98cacd1a 100644
--- a/tensorflow/core/kernels/depthwise_conv_grad_op.cc
+++ b/tensorflow/core/kernels/depthwise_conv_grad_op.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include <algorithm>
 #include <cmath>
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
@@ -25,7 +26,6 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/conv_grad_ops.h"
 #include "tensorflow/core/kernels/depthwise_conv_op.h"
 #include "tensorflow/core/kernels/ops_util.h"
diff --git a/tensorflow/core/kernels/depthwise_conv_op.cc b/tensorflow/core/kernels/depthwise_conv_op.cc
index dacd3cfea8..11c2b31633 100644
--- a/tensorflow/core/kernels/depthwise_conv_op.cc
+++ b/tensorflow/core/kernels/depthwise_conv_op.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include <cmath>
 #include <type_traits>
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
@@ -26,7 +27,6 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/conv_ops.h"
 #include "tensorflow/core/kernels/depthwise_conv_op.h"
 #include "tensorflow/core/kernels/ops_util.h"
diff --git a/tensorflow/core/kernels/dynamic_partition_op.cc b/tensorflow/core/kernels/dynamic_partition_op.cc
index 572d04ae2c..95af19c4c4 100644
--- a/tensorflow/core/kernels/dynamic_partition_op.cc
+++ b/tensorflow/core/kernels/dynamic_partition_op.cc
@@ -16,11 +16,11 @@ limitations under the License.
 // See docs in ../ops/data_flow_ops.cc.
 
 #include <vector>
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/gtl/inlined_vector.h"
 #include "tensorflow/core/util/util.h"
 
diff --git a/tensorflow/core/kernels/dynamic_partition_op_gpu.cu.cc b/tensorflow/core/kernels/dynamic_partition_op_gpu.cu.cc
index e7882acc80..59f687bf9c 100644
--- a/tensorflow/core/kernels/dynamic_partition_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/dynamic_partition_op_gpu.cu.cc
@@ -40,11 +40,11 @@ limitations under the License.
 #include "third_party/cub/iterator/constant_input_iterator.cuh"
 #include "third_party/cub/thread/thread_operators.cuh"
 #include "tensorflow/core/common_runtime/gpu/gpu_event_mgr.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/fill_functor.h"
 #include "tensorflow/core/kernels/gather_functor_gpu.cu.h"
 #include "tensorflow/core/util/cuda_kernel_helper.h"
diff --git a/tensorflow/core/kernels/dynamic_stitch_op.cc b/tensorflow/core/kernels/dynamic_stitch_op.cc
index f21f2acf26..5b8845b675 100644
--- a/tensorflow/core/kernels/dynamic_stitch_op.cc
+++ b/tensorflow/core/kernels/dynamic_stitch_op.cc
@@ -15,10 +15,10 @@ limitations under the License.
 
 // See docs in ../ops/data_flow_ops.cc.
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/threadpool.h"
 
 #ifdef GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/encode_jpeg_op.cc b/tensorflow/core/kernels/encode_jpeg_op.cc
index 1a5b0f2b67..e80404a437 100644
--- a/tensorflow/core/kernels/encode_jpeg_op.cc
+++ b/tensorflow/core/kernels/encode_jpeg_op.cc
@@ -16,12 +16,12 @@ limitations under the License.
 // See docs in ../ops/image_ops.cc
 
 #include <memory>
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/jpeg/jpeg_mem.h"
 #include "tensorflow/core/platform/logging.h"
diff --git a/tensorflow/core/kernels/encode_png_op.cc b/tensorflow/core/kernels/encode_png_op.cc
index 8fcda25e69..cb9a1660a7 100644
--- a/tensorflow/core/kernels/encode_png_op.cc
+++ b/tensorflow/core/kernels/encode_png_op.cc
@@ -16,12 +16,12 @@ limitations under the License.
 // See docs in ../ops/image_ops.cc
 
 #include <memory>
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/png/png_io.h"
 #include "tensorflow/core/platform/logging.h"
diff --git a/tensorflow/core/kernels/encode_wav_op.cc b/tensorflow/core/kernels/encode_wav_op.cc
index aed095076b..082f9a74ae 100644
--- a/tensorflow/core/kernels/encode_wav_op.cc
+++ b/tensorflow/core/kernels/encode_wav_op.cc
@@ -15,12 +15,12 @@ limitations under the License.
 
 // See docs in ../ops/audio_ops.cc
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/wav/wav_io.h"
 
diff --git a/tensorflow/core/kernels/extract_image_patches_op.cc b/tensorflow/core/kernels/extract_image_patches_op.cc
index 68631d14db..9306eccf9f 100644
--- a/tensorflow/core/kernels/extract_image_patches_op.cc
+++ b/tensorflow/core/kernels/extract_image_patches_op.cc
@@ -20,11 +20,11 @@ limitations under the License.
 
 #include "tensorflow/core/kernels/extract_image_patches_op.h"
 #include <vector>
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/platform/logging.h"
diff --git a/tensorflow/core/kernels/extract_jpeg_shape_op.cc b/tensorflow/core/kernels/extract_jpeg_shape_op.cc
index 60d798af56..ab424595c1 100644
--- a/tensorflow/core/kernels/extract_jpeg_shape_op.cc
+++ b/tensorflow/core/kernels/extract_jpeg_shape_op.cc
@@ -16,12 +16,12 @@ limitations under the License.
 // See docs in ../ops/image_ops.cc
 
 #include <memory>
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/jpeg/jpeg_mem.h"
 #include "tensorflow/core/platform/logging.h"
diff --git a/tensorflow/core/kernels/extract_volume_patches_op.cc b/tensorflow/core/kernels/extract_volume_patches_op.cc
index 52cd078a35..8107bca7d1 100644
--- a/tensorflow/core/kernels/extract_volume_patches_op.cc
+++ b/tensorflow/core/kernels/extract_volume_patches_op.cc
@@ -26,11 +26,11 @@ when rates are to be added.
 
 #include "tensorflow/core/kernels/extract_volume_patches_op.h"
 #include <vector>
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/platform/logging.h"
diff --git a/tensorflow/core/kernels/gather_functor.h b/tensorflow/core/kernels/gather_functor.h
index 7710cf93d6..93bdebc00e 100644
--- a/tensorflow/core/kernels/gather_functor.h
+++ b/tensorflow/core/kernels/gather_functor.h
@@ -18,11 +18,11 @@ limitations under the License.
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/type_traits.h"
 #include "tensorflow/core/framework/variant.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/platform/prefetch.h"
 #include "tensorflow/core/platform/types.h"
 #include "tensorflow/core/util/work_sharder.h"
diff --git a/tensorflow/core/kernels/gather_nd_op.cc b/tensorflow/core/kernels/gather_nd_op.cc
index e50b7fe3bf..58867a34bc 100644
--- a/tensorflow/core/kernels/gather_nd_op.cc
+++ b/tensorflow/core/kernels/gather_nd_op.cc
@@ -17,10 +17,10 @@ limitations under the License.
 #define EIGEN_USE_THREADS
 
 #include "tensorflow/core/kernels/gather_nd_op.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/mem.h"
diff --git a/tensorflow/core/kernels/gather_nd_op.h b/tensorflow/core/kernels/gather_nd_op.h
index 003badb74d..77c0d7717e 100644
--- a/tensorflow/core/kernels/gather_nd_op.h
+++ b/tensorflow/core/kernels/gather_nd_op.h
@@ -18,8 +18,8 @@ limitations under the License.
 // Functor definition for GatherOp, must be compilable by nvcc.
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/tensor_types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
diff --git a/tensorflow/core/kernels/gather_nd_op_cpu_impl.h b/tensorflow/core/kernels/gather_nd_op_cpu_impl.h
index 1c78de253e..cf9817dc30 100644
--- a/tensorflow/core/kernels/gather_nd_op_cpu_impl.h
+++ b/tensorflow/core/kernels/gather_nd_op_cpu_impl.h
@@ -22,10 +22,10 @@ limitations under the License.
 
 #include <atomic>
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/gather_nd_op.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/mem.h"
diff --git a/tensorflow/core/kernels/gather_op.cc b/tensorflow/core/kernels/gather_op.cc
index 5795f68889..b26f0a7528 100644
--- a/tensorflow/core/kernels/gather_op.cc
+++ b/tensorflow/core/kernels/gather_op.cc
@@ -15,12 +15,12 @@ limitations under the License.
 
 // See docs in ../ops/array_ops.cc.
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/variant.h"
 #include "tensorflow/core/framework/variant_encode_decode.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/gather_functor.h"
 #include "tensorflow/core/platform/mem.h"
 #include "tensorflow/core/platform/types.h"
diff --git a/tensorflow/core/kernels/image_resizer_state.h b/tensorflow/core/kernels/image_resizer_state.h
index 1d4fa1a7db..8078c7036a 100644
--- a/tensorflow/core/kernels/image_resizer_state.h
+++ b/tensorflow/core/kernels/image_resizer_state.h
@@ -28,12 +28,12 @@ limitations under the License.
 #include <array>
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 
 namespace tensorflow {
 
diff --git a/tensorflow/core/kernels/in_topk_op.cc b/tensorflow/core/kernels/in_topk_op.cc
index c37055239c..506091f76e 100644
--- a/tensorflow/core/kernels/in_topk_op.cc
+++ b/tensorflow/core/kernels/in_topk_op.cc
@@ -18,10 +18,10 @@ limitations under the License.
 #define EIGEN_USE_THREADS
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 
 namespace tensorflow {
 
diff --git a/tensorflow/core/kernels/lookup_table_op.h b/tensorflow/core/kernels/lookup_table_op.h
index 9451247f26..b046401c0a 100644
--- a/tensorflow/core/kernels/lookup_table_op.h
+++ b/tensorflow/core/kernels/lookup_table_op.h
@@ -16,12 +16,12 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_KERNELS_LOOKUP_TABLE_OP_H_
 #define TENSORFLOW_CORE_KERNELS_LOOKUP_TABLE_OP_H_
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/lookup_interface.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/lookup_util.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
diff --git a/tensorflow/core/kernels/lrn_op.cc b/tensorflow/core/kernels/lrn_op.cc
index f405ca3c58..ba30432e21 100644
--- a/tensorflow/core/kernels/lrn_op.cc
+++ b/tensorflow/core/kernels/lrn_op.cc
@@ -19,10 +19,10 @@ limitations under the License.
 #define EIGEN_USE_THREADS
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/lib/core/errors.h"
 
diff --git a/tensorflow/core/kernels/maxpooling_op.cc b/tensorflow/core/kernels/maxpooling_op.cc
index ab235843f7..d24cb1cc92 100644
--- a/tensorflow/core/kernels/maxpooling_op.cc
+++ b/tensorflow/core/kernels/maxpooling_op.cc
@@ -22,13 +22,13 @@ limitations under the License.
 #include <vector>
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_slice.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/conv_2d.h"
 #include "tensorflow/core/kernels/eigen_pooling.h"
 #include "tensorflow/core/kernels/ops_util.h"
diff --git a/tensorflow/core/kernels/mkl_concat_op.cc b/tensorflow/core/kernels/mkl_concat_op.cc
index f0278caee6..3a5c87485c 100644
--- a/tensorflow/core/kernels/mkl_concat_op.cc
+++ b/tensorflow/core/kernels/mkl_concat_op.cc
@@ -18,12 +18,12 @@ limitations under the License.
 
 #include "mkldnn.hpp"
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/concat_lib.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc
index 8c585fb48c..6578f97ef4 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_ops.cc
@@ -21,13 +21,13 @@ limitations under the License.
 #include <map>
 #include <vector>
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_slice.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/mkl_conv_ops.h"
 #include "tensorflow/core/kernels/mkl_quantized_conv_ops.h"
 #include "tensorflow/core/kernels/no_op.h"
diff --git a/tensorflow/core/kernels/mkl_conv_ops.h b/tensorflow/core/kernels/mkl_conv_ops.h
index fd279ead9d..0881e8ebcd 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.h
+++ b/tensorflow/core/kernels/mkl_conv_ops.h
@@ -21,13 +21,13 @@ limitations under the License.
 #include <vector>
 
 #include "mkldnn.hpp"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_slice.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/conv_grad_ops.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/lib/core/errors.h"
diff --git a/tensorflow/core/kernels/mkl_lrn_op.cc b/tensorflow/core/kernels/mkl_lrn_op.cc
index 4d46abb0a4..bc52127b94 100644
--- a/tensorflow/core/kernels/mkl_lrn_op.cc
+++ b/tensorflow/core/kernels/mkl_lrn_op.cc
@@ -24,10 +24,10 @@ limitations under the License.
 #include <vector>
 #include "mkldnn.hpp"
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/util/mkl_util.h"
diff --git a/tensorflow/core/kernels/mkl_pooling_ops_common.cc b/tensorflow/core/kernels/mkl_pooling_ops_common.cc
index dc84d3941e..a8d1dffd4e 100644
--- a/tensorflow/core/kernels/mkl_pooling_ops_common.cc
+++ b/tensorflow/core/kernels/mkl_pooling_ops_common.cc
@@ -19,8 +19,8 @@ limitations under the License.
 #include <limits>
 #include <vector>
 #include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/common_shape_fns.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 
 namespace tensorflow {
 
diff --git a/tensorflow/core/kernels/neon/neon_depthwise_conv_op.cc b/tensorflow/core/kernels/neon/neon_depthwise_conv_op.cc
index 0e820bbb62..b218f62ddd 100644
--- a/tensorflow/core/kernels/neon/neon_depthwise_conv_op.cc
+++ b/tensorflow/core/kernels/neon/neon_depthwise_conv_op.cc
@@ -19,6 +19,7 @@ limitations under the License.
 
 #define GEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK
 #include "public/gemmlowp.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
@@ -26,7 +27,6 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/neon/depthwiseconv_float.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/lib/core/status.h"
diff --git a/tensorflow/core/kernels/non_max_suppression_op.cc b/tensorflow/core/kernels/non_max_suppression_op.cc
index 37f615abd9..2f4e51272f 100644
--- a/tensorflow/core/kernels/non_max_suppression_op.cc
+++ b/tensorflow/core/kernels/non_max_suppression_op.cc
@@ -24,12 +24,12 @@ limitations under the License.
 #include <vector>
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/gtl/stl_util.h"
 #include "tensorflow/core/platform/logging.h"
diff --git a/tensorflow/core/kernels/resource_variable_ops.cc b/tensorflow/core/kernels/resource_variable_ops.cc
index 4167b60051..8e3c52ba5b 100644
--- a/tensorflow/core/kernels/resource_variable_ops.cc
+++ b/tensorflow/core/kernels/resource_variable_ops.cc
@@ -56,12 +56,12 @@ limitations under the License.
 
 #include "absl/strings/str_join.h"
 #include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/variant_op_registry.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/dense_update_functor.h"
 #include "tensorflow/core/kernels/gather_functor.h"
 #include "tensorflow/core/kernels/resource_variable_ops.h"
diff --git a/tensorflow/core/kernels/reverse_op.cc b/tensorflow/core/kernels/reverse_op.cc
index 1c4d0bc1ae..aa2434da03 100644
--- a/tensorflow/core/kernels/reverse_op.cc
+++ b/tensorflow/core/kernels/reverse_op.cc
@@ -19,13 +19,13 @@ limitations under the License.
 #include "tensorflow/core/kernels/reverse_op.h"
 #include <memory>
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/type_traits.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/util/work_sharder.h"
diff --git a/tensorflow/core/kernels/roll_op.cc b/tensorflow/core/kernels/roll_op.cc
index efa30438d9..494a846ff5 100644
--- a/tensorflow/core/kernels/roll_op.cc
+++ b/tensorflow/core/kernels/roll_op.cc
@@ -13,13 +13,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/common_shape_fns.h"
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/register_types_traits.h"
 #include "tensorflow/core/framework/shape_inference.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/gtl/array_slice.h"
 #include "tensorflow/core/platform/types.h"
 #include "tensorflow/core/util/work_sharder.h"
diff --git a/tensorflow/core/kernels/sample_distorted_bounding_box_op.cc b/tensorflow/core/kernels/sample_distorted_bounding_box_op.cc
index c0fde8042e..0e68af867b 100644
--- a/tensorflow/core/kernels/sample_distorted_bounding_box_op.cc
+++ b/tensorflow/core/kernels/sample_distorted_bounding_box_op.cc
@@ -14,11 +14,11 @@ limitations under the License.
 ==============================================================================*/
 // See docs in ../ops/image_ops.cc.
 #include <math.h>
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/random/simple_philox.h"
 #include "tensorflow/core/util/guarded_philox_random.h"
 
diff --git a/tensorflow/core/kernels/save_restore_tensor.cc b/tensorflow/core/kernels/save_restore_tensor.cc
index 82546d581a..8580891fc0 100644
--- a/tensorflow/core/kernels/save_restore_tensor.cc
+++ b/tensorflow/core/kernels/save_restore_tensor.cc
@@ -19,10 +19,10 @@ limitations under the License.
 #include <utility>
 #include <vector>
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/threadpool.h"
 #include "tensorflow/core/lib/gtl/array_slice.h"
 #include "tensorflow/core/lib/strings/str_util.h"
diff --git a/tensorflow/core/kernels/save_restore_v2_ops.cc b/tensorflow/core/kernels/save_restore_v2_ops.cc
index 180eb3ca34..ed1195c053 100644
--- a/tensorflow/core/kernels/save_restore_v2_ops.cc
+++ b/tensorflow/core/kernels/save_restore_v2_ops.cc
@@ -18,11 +18,11 @@ limitations under the License.
 #include <string>
 #include <vector>
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/framework/types.pb.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/save_restore_tensor.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/io/path.h"
diff --git a/tensorflow/core/kernels/scale_and_translate_op.cc b/tensorflow/core/kernels/scale_and_translate_op.cc
index 149c5526ae..34fef536df 100644
--- a/tensorflow/core/kernels/scale_and_translate_op.cc
+++ b/tensorflow/core/kernels/scale_and_translate_op.cc
@@ -20,13 +20,13 @@ limitations under the License.
 
 #include <memory>
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/framework/types.pb.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/sampling_kernels.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/strings/stringprintf.h"
diff --git a/tensorflow/core/kernels/scan_ops.cc b/tensorflow/core/kernels/scan_ops.cc
index 0a6848361a..ea42fdefb4 100644
--- a/tensorflow/core/kernels/scan_ops.cc
+++ b/tensorflow/core/kernels/scan_ops.cc
@@ -18,12 +18,12 @@ limitations under the License.
 #define EIGEN_USE_GPU
 #endif  // GOOGLE_CUDA
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 
 #include "third_party/eigen3/Eigen/Core"
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
diff --git a/tensorflow/core/kernels/scatter_functor.h b/tensorflow/core/kernels/scatter_functor.h
index 2d43bde23f..755f8f8dc5 100644
--- a/tensorflow/core/kernels/scatter_functor.h
+++ b/tensorflow/core/kernels/scatter_functor.h
@@ -20,10 +20,10 @@ limitations under the License.
 
 #include "third_party/eigen3/Eigen/Core"
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/variant_op_registry.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/dense_update_functor.h"
 #include "tensorflow/core/platform/types.h"
 
diff --git a/tensorflow/core/kernels/scatter_nd_op.cc b/tensorflow/core/kernels/scatter_nd_op.cc
index 50e4c66b7e..9c51d4e3a7 100644
--- a/tensorflow/core/kernels/scatter_nd_op.cc
+++ b/tensorflow/core/kernels/scatter_nd_op.cc
@@ -22,11 +22,11 @@ limitations under the License.
 
 #include "tensorflow/core/kernels/scatter_nd_op.h"
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/dense_update_functor.h"
 #include "tensorflow/core/kernels/fill_functor.h"
 #include "tensorflow/core/kernels/inplace_ops_functor.h"
diff --git a/tensorflow/core/kernels/scatter_nd_op.h b/tensorflow/core/kernels/scatter_nd_op.h
index 8d04731aae..eec70ba69e 100644
--- a/tensorflow/core/kernels/scatter_nd_op.h
+++ b/tensorflow/core/kernels/scatter_nd_op.h
@@ -18,11 +18,11 @@ limitations under the License.
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/fill_functor.h"
 #include "tensorflow/core/kernels/scatter_nd_op.h"
 #include "tensorflow/core/platform/mutex.h"
diff --git a/tensorflow/core/kernels/scatter_nd_op_cpu_impl.h b/tensorflow/core/kernels/scatter_nd_op_cpu_impl.h
index 472f5a3547..01e4656eab 100644
--- a/tensorflow/core/kernels/scatter_nd_op_cpu_impl.h
+++ b/tensorflow/core/kernels/scatter_nd_op_cpu_impl.h
@@ -24,11 +24,11 @@ limitations under the License.
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/fill_functor.h"
 #include "tensorflow/core/kernels/scatter_nd_op.h"
 #include "tensorflow/core/platform/mutex.h"
diff --git a/tensorflow/core/kernels/searchsorted_op.cc b/tensorflow/core/kernels/searchsorted_op.cc
index dc627ac77a..06b2d81837 100644
--- a/tensorflow/core/kernels/searchsorted_op.cc
+++ b/tensorflow/core/kernels/searchsorted_op.cc
@@ -17,11 +17,11 @@ limitations under the License.
 
 #include "tensorflow/core/kernels/searchsorted_op.h"
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/types.h"
 
diff --git a/tensorflow/core/kernels/segment_reduction_ops.cc b/tensorflow/core/kernels/segment_reduction_ops.cc
index 2328fc6afd..6e1a0d57a1 100644
--- a/tensorflow/core/kernels/segment_reduction_ops.cc
+++ b/tensorflow/core/kernels/segment_reduction_ops.cc
@@ -22,15 +22,17 @@ limitations under the License.
 
 #include "third_party/eigen3/Eigen/Core"
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+
 #include "tensorflow/core/kernels/segment_reduction_ops.h"
 #include <vector>
+
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/util/util.h"
diff --git a/tensorflow/core/kernels/shape_ops.h b/tensorflow/core/kernels/shape_ops.h
index 7a50f158af..dbd82d2fe7 100644
--- a/tensorflow/core/kernels/shape_ops.h
+++ b/tensorflow/core/kernels/shape_ops.h
@@ -20,11 +20,11 @@ limitations under the License.
 #include <unordered_set>
 #include <vector>
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/variant_op_registry.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 
 namespace tensorflow {
 
diff --git a/tensorflow/core/kernels/spacetobatch_functor.h b/tensorflow/core/kernels/spacetobatch_functor.h
index f46a84da1e..459f20b0ae 100644
--- a/tensorflow/core/kernels/spacetobatch_functor.h
+++ b/tensorflow/core/kernels/spacetobatch_functor.h
@@ -19,9 +19,9 @@ limitations under the License.
 #include <type_traits>
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
diff --git a/tensorflow/core/kernels/sparse_tensor_dense_matmul_op.cc b/tensorflow/core/kernels/sparse_tensor_dense_matmul_op.cc
index 30c57ef287..0a97c6b6a5 100644
--- a/tensorflow/core/kernels/sparse_tensor_dense_matmul_op.cc
+++ b/tensorflow/core/kernels/sparse_tensor_dense_matmul_op.cc
@@ -19,9 +19,9 @@ limitations under the License.
 
 #include "tensorflow/core/kernels/sparse_tensor_dense_matmul_op.h"
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/op_kernel.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/fill_functor.h"
 
 namespace tensorflow {
diff --git a/tensorflow/core/kernels/sparse_tensor_dense_matmul_op_gpu.cu.cc b/tensorflow/core/kernels/sparse_tensor_dense_matmul_op_gpu.cu.cc
index e261e42e0d..ea95a882b1 100644
--- a/tensorflow/core/kernels/sparse_tensor_dense_matmul_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/sparse_tensor_dense_matmul_op_gpu.cu.cc
@@ -19,8 +19,8 @@ limitations under the License.
 
 #include "tensorflow/core/kernels/sparse_tensor_dense_matmul_op.h"
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/register_types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/util/cuda_kernel_helper.h"
 
 namespace tensorflow {
diff --git a/tensorflow/core/kernels/sparse_xent_op.h b/tensorflow/core/kernels/sparse_xent_op.h
index 5e462424ed..c94597f297 100644
--- a/tensorflow/core/kernels/sparse_xent_op.h
+++ b/tensorflow/core/kernels/sparse_xent_op.h
@@ -18,9 +18,9 @@ limitations under the License.
 // Functor definition for SparseXentOp, must be compilable by nvcc.
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor_types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/platform/macros.h"
 #include "tensorflow/core/platform/types.h"
 
diff --git a/tensorflow/core/kernels/split_op.cc b/tensorflow/core/kernels/split_op.cc
index 11db72bfa3..ed3429ff5c 100644
--- a/tensorflow/core/kernels/split_op.cc
+++ b/tensorflow/core/kernels/split_op.cc
@@ -18,10 +18,10 @@ limitations under the License.
 #define EIGEN_USE_THREADS
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/kernels/split_lib.h"
 #include "tensorflow/core/lib/core/status.h"
diff --git a/tensorflow/core/kernels/split_v_op.cc b/tensorflow/core/kernels/split_v_op.cc
index 5c19a45fb1..0324ce9bab 100644
--- a/tensorflow/core/kernels/split_v_op.cc
+++ b/tensorflow/core/kernels/split_v_op.cc
@@ -24,10 +24,10 @@ limitations under the License.
 #include <numeric>
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/kernels/split_lib.h"
 #include "tensorflow/core/lib/core/status.h"
diff --git a/tensorflow/core/kernels/stateful_random_ops.cc b/tensorflow/core/kernels/stateful_random_ops.cc
index e5c1c25864..7cbbfd490e 100644
--- a/tensorflow/core/kernels/stateful_random_ops.cc
+++ b/tensorflow/core/kernels/stateful_random_ops.cc
@@ -22,13 +22,13 @@ limitations under the License.
 #include "absl/strings/str_join.h"
 #include "absl/types/variant.h"
 #include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/variant_op_registry.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/dense_update_functor.h"
 #include "tensorflow/core/kernels/gather_functor.h"
 #include "tensorflow/core/kernels/random_op.h"
diff --git a/tensorflow/core/kernels/stateless_random_ops.cc b/tensorflow/core/kernels/stateless_random_ops.cc
index 925f5291a6..959334abc8 100644
--- a/tensorflow/core/kernels/stateless_random_ops.cc
+++ b/tensorflow/core/kernels/stateless_random_ops.cc
@@ -13,11 +13,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/random_op.h"
 #include "tensorflow/core/lib/random/random_distributions.h"
 #include "tensorflow/core/platform/logging.h"
diff --git a/tensorflow/core/kernels/strided_slice_op.cc b/tensorflow/core/kernels/strided_slice_op.cc
index 6db68f937d..20bf42ccaa 100644
--- a/tensorflow/core/kernels/strided_slice_op.cc
+++ b/tensorflow/core/kernels/strided_slice_op.cc
@@ -27,10 +27,10 @@ limitations under the License.
 #include "tensorflow/core/kernels/strided_slice_op_impl.h"
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/kernels/training_op_helpers.h"
 #include "tensorflow/core/kernels/variable_ops.h"
diff --git a/tensorflow/core/kernels/strided_slice_op_impl.h b/tensorflow/core/kernels/strided_slice_op_impl.h
index c4205159c3..d9b62d4c75 100644
--- a/tensorflow/core/kernels/strided_slice_op_impl.h
+++ b/tensorflow/core/kernels/strided_slice_op_impl.h
@@ -22,13 +22,13 @@ limitations under the License.
 #include "tensorflow/core/kernels/strided_slice_op.h"
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/register_types_traits.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/variant.h"
 #include "tensorflow/core/framework/variant_encode_decode.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/dense_update_functor.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/lib/core/status.h"
diff --git a/tensorflow/core/kernels/substr_op.cc b/tensorflow/core/kernels/substr_op.cc
index 93c427039d..77b16b9384 100644
--- a/tensorflow/core/kernels/substr_op.cc
+++ b/tensorflow/core/kernels/substr_op.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include <string>
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/op_kernel.h"
@@ -25,7 +26,6 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/string_util.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
diff --git a/tensorflow/core/kernels/tensor_array_ops.cc b/tensorflow/core/kernels/tensor_array_ops.cc
index aa85f546a8..129035638a 100644
--- a/tensorflow/core/kernels/tensor_array_ops.cc
+++ b/tensorflow/core/kernels/tensor_array_ops.cc
@@ -23,13 +23,13 @@ limitations under the License.
 #include <numeric>  // clang-format off
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/concat_lib.h"
 #include "tensorflow/core/kernels/split_lib.h"
 #include "tensorflow/core/kernels/tensor_array.h"
diff --git a/tensorflow/core/kernels/training_ops.cc b/tensorflow/core/kernels/training_ops.cc
index b2239ab5c3..5594c998dd 100644
--- a/tensorflow/core/kernels/training_ops.cc
+++ b/tensorflow/core/kernels/training_ops.cc
@@ -18,9 +18,9 @@ limitations under the License.
 
 #include <algorithm>
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/training_op_helpers.h"
 #include "tensorflow/core/kernels/training_ops.h"
 #include "tensorflow/core/kernels/variable_ops.h"
diff --git a/tensorflow/core/kernels/transpose_op.cc b/tensorflow/core/kernels/transpose_op.cc
index 48e392c070..1c0d70c333 100644
--- a/tensorflow/core/kernels/transpose_op.cc
+++ b/tensorflow/core/kernels/transpose_op.cc
@@ -19,11 +19,11 @@ limitations under the License.
 
 #include "tensorflow/core/kernels/transpose_op.h"
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/transpose_functor.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/strings/str_util.h"
diff --git a/tensorflow/core/kernels/unicode_ops.cc b/tensorflow/core/kernels/unicode_ops.cc
index c9c2ac1e69..c071db6064 100644
--- a/tensorflow/core/kernels/unicode_ops.cc
+++ b/tensorflow/core/kernels/unicode_ops.cc
@@ -31,6 +31,7 @@ limitations under the License.
 #include "unicode/unistr.h"  // TF:icu
 #include "unicode/uset.h"  // TF:icu
 #include "unicode/utypes.h"  // TF:icu
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/op_kernel.h"
@@ -39,7 +40,6 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/string_util.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
diff --git a/tensorflow/core/kernels/unique_op.cc b/tensorflow/core/kernels/unique_op.cc
index 3bdcfc90b8..adf84bae49 100644
--- a/tensorflow/core/kernels/unique_op.cc
+++ b/tensorflow/core/kernels/unique_op.cc
@@ -17,11 +17,11 @@ limitations under the License.
 #include <unordered_map>
 #include <utility>
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/hash/hash.h"
 
diff --git a/tensorflow/core/kernels/unpack_op.cc b/tensorflow/core/kernels/unpack_op.cc
index 1e1647db5c..8577ce7bf7 100644
--- a/tensorflow/core/kernels/unpack_op.cc
+++ b/tensorflow/core/kernels/unpack_op.cc
@@ -18,10 +18,10 @@ limitations under the License.
 #define EIGEN_USE_THREADS
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/kernels/split_lib.h"
 #include "tensorflow/core/lib/core/status.h"
diff --git a/tensorflow/core/kernels/where_op.cc b/tensorflow/core/kernels/where_op.cc
index aa65fa6b63..374257d176 100644
--- a/tensorflow/core/kernels/where_op.cc
+++ b/tensorflow/core/kernels/where_op.cc
@@ -26,13 +26,13 @@ limitations under the License.
 #include <memory>
 #include <numeric>
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/macros.h"
 #include "tensorflow/core/platform/types.h"
diff --git a/tensorflow/core/kernels/where_op_gpu.cu.h b/tensorflow/core/kernels/where_op_gpu.cu.h
index 2255597651..54b22d230a 100644
--- a/tensorflow/core/kernels/where_op_gpu.cu.h
+++ b/tensorflow/core/kernels/where_op_gpu.cu.h
@@ -25,9 +25,9 @@ limitations under the License.
 #include "third_party/cub/device/device_select.cuh"
 #include "third_party/cub/iterator/counting_input_iterator.cuh"
 #include "third_party/cub/iterator/transform_input_iterator.cuh"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor_types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/where_op.h"
 #include "tensorflow/core/platform/macros.h"
 #include "tensorflow/core/platform/types.h"
diff --git a/tensorflow/core/util/sparse/dim_comparator.h b/tensorflow/core/util/sparse/dim_comparator.h
index 0782e7e1a8..498df7a021 100644
--- a/tensorflow/core/util/sparse/dim_comparator.h
+++ b/tensorflow/core/util/sparse/dim_comparator.h
@@ -17,7 +17,7 @@ limitations under the License.
 #define TENSORFLOW_CORE_UTIL_SPARSE_DIM_COMPARATOR_H_
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
-#include "tensorflow/core/kernels/bounds_check.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/lib/gtl/array_slice.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/types.h"
diff --git a/tensorflow/core/util/sparse/sparse_tensor.h b/tensorflow/core/util/sparse/sparse_tensor.h
index 89c163aa51..cd168f6597 100644
--- a/tensorflow/core/util/sparse/sparse_tensor.h
+++ b/tensorflow/core/util/sparse/sparse_tensor.h
@@ -22,11 +22,11 @@ limitations under the License.
 
 #include "absl/base/macros.h"
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/framework/types.pb.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/strings/str_util.h"
diff --git a/tensorflow/core/util/strided_slice_op.cc b/tensorflow/core/util/strided_slice_op.cc
index 55688e5808..9db35da6a2 100644
--- a/tensorflow/core/util/strided_slice_op.cc
+++ b/tensorflow/core/util/strided_slice_op.cc
@@ -16,7 +16,7 @@ limitations under the License.
 #include "tensorflow/core/util/strided_slice_op.h"
 
 #include <array>
-#include "tensorflow/core/kernels/bounds_check.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/lib/core/status.h"
 
 namespace tensorflow {
-- 
GitLab


From 89d7b76d3a58364e3021a70e88c58896584f44ee Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Tue, 22 Jan 2019 14:53:12 -0800
Subject: [PATCH 1076/2345] Automated rollback of commit
 74a6cca5d867d37e79ec9d780f2c57b926f07a80

PiperOrigin-RevId: 230415975
---
 tensorflow/python/framework/dtypes.py | 110 ++++++++++++--------------
 1 file changed, 52 insertions(+), 58 deletions(-)

diff --git a/tensorflow/python/framework/dtypes.py b/tensorflow/python/framework/dtypes.py
index 465ea4229b..574d82ea93 100644
--- a/tensorflow/python/framework/dtypes.py
+++ b/tensorflow/python/framework/dtypes.py
@@ -535,42 +535,29 @@ _np_qint32 = np.dtype([("qint32", np.int32, 1)])
 np_resource = np.dtype([("resource", np.ubyte, 1)])
 
 # Standard mappings between types_pb2.DataType values and numpy.dtypes.
-_NP_TO_TF = {
-    np.float16: float16,
-    np.float32: float32,
-    np.float64: float64,
-    np.int32: int32,
-    np.int64: int64,
-    np.uint8: uint8,
-    np.uint16: uint16,
-    np.uint32: uint32,
-    np.uint64: uint64,
-    np.int16: int16,
-    np.int8: int8,
-    np.complex64: complex64,
-    np.complex128: complex128,
-    np.object_: string,
-    np.string_: string,
-    np.unicode_: string,
-    np.bool_: bool,
-    _np_qint8: qint8,
-    _np_quint8: quint8,
-    _np_qint16: qint16,
-    _np_quint16: quint16,
-    _np_qint32: qint32,
-    _np_bfloat16: bfloat16,
-}
-
-# On Python 2.X `np.longlong` could be a distinct type used for long
-# integers e.g. 42L. See numpy/numpy#9799.
-for pdt in [
-    np.longlong,
-    np.ulonglong,
-]:
-  if pdt not in _NP_TO_TF:
-    _NP_TO_TF[pdt] = next(
-        _NP_TO_TF[dt] for dt in _NP_TO_TF if dt == pdt().dtype)
-
+_NP_TO_TF = frozenset([
+    (np.float16, float16),
+    (np.float32, float32),
+    (np.float64, float64),
+    (np.int32, int32),
+    (np.int64, int64),
+    (np.uint8, uint8),
+    (np.uint16, uint16),
+    (np.uint32, uint32),
+    (np.uint64, uint64),
+    (np.int16, int16),
+    (np.int8, int8),
+    (np.complex64, complex64),
+    (np.complex128, complex128),
+    (np.object_, string),
+    (np.bool_, bool),
+    (_np_qint8, qint8),
+    (_np_quint8, quint8),
+    (_np_qint16, qint16),
+    (_np_quint16, quint16),
+    (_np_qint32, qint32),
+    (_np_bfloat16, bfloat16),
+])
 _TF_TO_NP = {
     types_pb2.DT_HALF:
         np.float16,
@@ -677,20 +664,6 @@ _PYTHON_TO_TF = {
     builtins.object: string
 }
 
-_ANY_TO_TF = {}
-_ANY_TO_TF.update(_INTERN_TABLE)
-_ANY_TO_TF.update(_STRING_TO_TF)
-_ANY_TO_TF.update(_PYTHON_TO_TF)
-_ANY_TO_TF.update(_NP_TO_TF)
-
-# Ensure no collisions.
-assert len(_ANY_TO_TF) == sum(len(d) for d in [
-    _INTERN_TABLE,
-    _STRING_TO_TF,
-    _PYTHON_TO_TF,
-    _NP_TO_TF
-])
-
 
 @tf_export("dtypes.as_dtype", "as_dtype")
 def as_dtype(type_value):
@@ -711,16 +684,37 @@ def as_dtype(type_value):
   if isinstance(type_value, DType):
     return type_value
 
-  if isinstance(type_value, np.dtype):
-    try:
-      return _NP_TO_TF[type_value.type]
-    except KeyError:
-      pass
+  try:
+    return _INTERN_TABLE[type_value]
+  except KeyError:
+    pass
+
+  try:
+    return _STRING_TO_TF[type_value]
+  except KeyError:
+    pass
 
   try:
-    return _ANY_TO_TF[type_value]
+    return _PYTHON_TO_TF[type_value]
   except KeyError:
     pass
 
-  raise TypeError(
-      "Cannot convert value %r to a TensorFlow DType." % type_value)
+  if isinstance(type_value, np.dtype):
+    # The numpy dtype for strings is variable length. We can not compare
+    # dtype with a single constant (np.string does not exist) to decide
+    # dtype is a "string" type. We need to compare the dtype.type to be
+    # sure it's a string type.
+    if type_value.type == np.string_ or type_value.type == np.unicode_:
+      return string
+
+  if isinstance(type_value, (type, np.dtype)):
+    for key, val in _NP_TO_TF:
+      try:
+        if key == type_value:
+          return val
+      except TypeError as e:
+        raise TypeError("Cannot convert {} to a dtype. {}".format(
+            type_value, e))
+
+  raise TypeError("Cannot convert value %r to a TensorFlow DType." % type_value)
+
-- 
GitLab


From 46ddf3105bfd2a8eebabfff50c0cbf3e16d911dd Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Tue, 22 Jan 2019 14:56:56 -0800
Subject: [PATCH 1077/2345] [XLA] HloPassFix should implement RunOnModuleGroup

The default implementation just forwards to the underlying pass' RunOnModuleGroup, which means we silently run the underlying pass once, instead of running it to a fixed point.

PiperOrigin-RevId: 230416670
---
 .../compiler/xla/service/hlo_pass_fix.h       | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/tensorflow/compiler/xla/service/hlo_pass_fix.h b/tensorflow/compiler/xla/service/hlo_pass_fix.h
index 791b1a97b0..35dc9c0029 100644
--- a/tensorflow/compiler/xla/service/hlo_pass_fix.h
+++ b/tensorflow/compiler/xla/service/hlo_pass_fix.h
@@ -19,6 +19,7 @@ limitations under the License.
 #include <algorithm>
 
 #include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/service/hlo_module_group.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/types.h"
@@ -39,9 +40,36 @@ class HloPassFix : public Pass {
     int64 iteration_count = 0;
     int64 limit =
         std::max(static_cast<int64>(1000), module->instruction_count());
+    VLOG(3) << "Running HloPassFix.";
     while (changed_this_iteration) {
       TF_ASSIGN_OR_RETURN(changed_this_iteration, Pass::Run(module));
       changed |= changed_this_iteration;
+      VLOG(3) << "changed_this_iteration: " << changed_this_iteration;
+      ++iteration_count;
+      if (iteration_count == limit) {
+        LOG(ERROR)
+            << "Unexpectedly high number of iterations in HLO passes ("
+            << iteration_count
+            << ")\nIf compilation hangs here, please file a bug with XLA.";
+      }
+    }
+    return changed;
+  }
+
+  StatusOr<bool> RunOnModuleGroup(HloModuleGroup* module_group) override {
+    bool changed = false;
+    bool changed_this_iteration = true;
+    int64 iteration_count = 0;
+    int64 limit = 1000;
+    for (const HloModule* module : module_group->modules()) {
+      limit = std::max<int64>(limit, module->instruction_count());
+    }
+    VLOG(3) << "Running HloPassFix.";
+    while (changed_this_iteration) {
+      TF_ASSIGN_OR_RETURN(changed_this_iteration,
+                          Pass::RunOnModuleGroup(module_group));
+      changed |= changed_this_iteration;
+      VLOG(3) << "changed_this_iteration: " << changed_this_iteration;
       ++iteration_count;
       if (iteration_count == limit) {
         LOG(ERROR)
-- 
GitLab


From f7d933a71fa022f1f3cddc3fec2a9db7b520473f Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Tue, 22 Jan 2019 14:58:31 -0800
Subject: [PATCH 1078/2345] Reapply change "cast inputs to Layer's __call__ to
 tensors" after fixing the failing unit test.

PiperOrigin-RevId: 230416943
---
 .../sequence_feature_column_v2_test.py            |  2 +-
 tensorflow/python/keras/engine/base_layer.py      |  9 ++++-----
 tensorflow/python/keras/engine/base_layer_test.py | 15 ++++++++++++++-
 3 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py
index 661eba4741..95d8ee278c 100644
--- a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py
+++ b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py
@@ -370,7 +370,7 @@ class SequenceFeaturesTest(test.TestCase, parameterized.TestCase):
            [[3.], [0.], [8.], [0.]]],
        'expected_sequence_length': [2, 2]},
       )
-  def test_numeric_column(
+  def DISABLED_test_numeric_column(  # See b/123240957
       self, sparse_input_args, expected_input_layer, expected_sequence_length):
     sparse_input = sparse_tensor.SparseTensorValue(**sparse_input_args)
 
diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index f63fb4e694..93849e771b 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -513,11 +513,10 @@ class Layer(checkpointable.Checkpointable):
       ValueError: if the layer's `call` method returns None (an invalid value).
     """
     input_list = nest.flatten(inputs)
-    if context.executing_eagerly():
-      # Accept NumPy inputs by converting to Tensors when executing eagerly.
-      if all(isinstance(x, (np.ndarray, float, int)) for x in input_list):
-        inputs = nest.map_structure(ops.convert_to_tensor, inputs)
-        input_list = nest.flatten(inputs)
+    # Accept NumPy inputs by converting to Tensors.
+    if all(isinstance(x, (np.ndarray, float, int)) for x in input_list):
+      inputs = nest.map_structure(ops.convert_to_tensor, inputs)
+      input_list = nest.flatten(inputs)
 
     # We will attempt to build a TF graph if & only if all inputs are symbolic.
     # This is always the case in graph mode. It can also be the case in eager
diff --git a/tensorflow/python/keras/engine/base_layer_test.py b/tensorflow/python/keras/engine/base_layer_test.py
index ebee4a3043..c925a8ca8a 100644
--- a/tensorflow/python/keras/engine/base_layer_test.py
+++ b/tensorflow/python/keras/engine/base_layer_test.py
@@ -26,6 +26,8 @@ from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import test_util
+from tensorflow.python.keras import keras_parameterized
+from tensorflow.python.keras import testing_utils
 from tensorflow.python.keras.engine import base_layer
 from tensorflow.python.keras.optimizer_v2 import rmsprop
 from tensorflow.python.ops import array_ops
@@ -71,7 +73,7 @@ class InvalidLayer(base_layer.Layer):
     raise ValueError('You did something wrong!')
 
 
-class BaseLayerTest(test.TestCase, parameterized.TestCase):
+class BaseLayerTest(keras_parameterized.TestCase):
 
   @parameterized.parameters(DynamicLayer1, DynamicLayer2)
   def test_dynamic_layer_in_functional_model_in_graph_mode(self, layer_class):
@@ -210,6 +212,17 @@ class BaseLayerTest(test.TestCase, parameterized.TestCase):
     with self.assertRaisesRegexp(ValueError, 'You did something wrong!'):
       _ = InvalidLayer()(inputs)
 
+  @keras_parameterized.run_with_all_model_types
+  @test_util.run_in_graph_and_eager_modes
+  def test_build_with_numpy_data(self):
+    model_layers = [
+        keras.layers.Dense(3, activation='relu', kernel_initializer='ones'),
+        keras.layers.Dense(1, activation='sigmoid', kernel_initializer='ones')
+    ]
+    model = testing_utils.get_model_from_layers(model_layers, input_shape=(4,))
+    model(np.zeros((2, 4), dtype='float32'))
+    self.assertTrue(model.built)
+
   def test_using_symbolic_tensors_with_tf_ops(self):
     # Single-input.
     x = keras.Input((3,))
-- 
GitLab


From 29970780c7e2aaf55040f09e10e5c1f6649cfd32 Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Tue, 22 Jan 2019 15:06:11 -0800
Subject: [PATCH 1079/2345] Add tests for dynamic parameter binding's
 serialization and deserialization behaviors.

PiperOrigin-RevId: 230418554
---
 .../xla/service/dynamic_parameter_binding.cc  |   3 +-
 .../xla/service/dynamic_parameter_binding.h   |   2 +-
 .../service/dynamic_parameter_binding_test.cc | 107 +++++++++++-------
 3 files changed, 70 insertions(+), 42 deletions(-)

diff --git a/tensorflow/compiler/xla/service/dynamic_parameter_binding.cc b/tensorflow/compiler/xla/service/dynamic_parameter_binding.cc
index 5549cccfa8..e9c8aa03e2 100644
--- a/tensorflow/compiler/xla/service/dynamic_parameter_binding.cc
+++ b/tensorflow/compiler/xla/service/dynamic_parameter_binding.cc
@@ -29,7 +29,8 @@ Status DynamicParameterBinding::Bind(
 }
 
 absl::optional<DynamicParameterBinding::DynamicParameter>
-DynamicParameterBinding::GetBinding(const DynamicDimension& dynamic_dimension) {
+DynamicParameterBinding::GetBinding(
+    const DynamicDimension& dynamic_dimension) const {
   auto param_iter = bindings_.find(dynamic_dimension);
   if (param_iter == bindings_.end()) {
     return absl::nullopt;
diff --git a/tensorflow/compiler/xla/service/dynamic_parameter_binding.h b/tensorflow/compiler/xla/service/dynamic_parameter_binding.h
index dd474d8eed..57af2c43d3 100644
--- a/tensorflow/compiler/xla/service/dynamic_parameter_binding.h
+++ b/tensorflow/compiler/xla/service/dynamic_parameter_binding.h
@@ -89,7 +89,7 @@ class DynamicParameterBinding {
   //
   // Returns nullopt if the binding is not set.
   absl::optional<DynamicParameter> GetBinding(
-      const DynamicDimension& dynamic_dimension);
+      const DynamicDimension& dynamic_dimension) const;
 
   using BindingFn =
       std::function<Status(const DynamicParameter& dynamic_parameter,
diff --git a/tensorflow/compiler/xla/service/dynamic_parameter_binding_test.cc b/tensorflow/compiler/xla/service/dynamic_parameter_binding_test.cc
index 83a6d83dff..b5d57cda4f 100644
--- a/tensorflow/compiler/xla/service/dynamic_parameter_binding_test.cc
+++ b/tensorflow/compiler/xla/service/dynamic_parameter_binding_test.cc
@@ -33,7 +33,15 @@ limitations under the License.
 
 namespace xla {
 namespace {
-class DynamicParameterBindingTest : public HloTestBase {};
+class DynamicParameterBindingTest : public HloTestBase {
+ protected:
+  // Serialize and then deserialize a binding.
+  void SerializeAndDeserialize(DynamicParameterBinding* binding) {
+    DynamicParameterBindingProto proto = binding->ToProto();
+    TF_ASSERT_OK_AND_ASSIGN(*binding,
+                            DynamicParameterBinding::CreateFromProto(proto));
+  }
+};
 
 TEST_F(DynamicParameterBindingTest, SimpleBinding) {
   // 'b' is a dynamic shape; 'a' represents the real size of b's first
@@ -56,15 +64,20 @@ ENTRY main {
       binding.Bind(DynamicParameterBinding::DynamicParameter{0, {}},
                    DynamicParameterBinding::DynamicDimension{1, {}, 0}));
 
-  absl::optional<DynamicParameterBinding::DynamicParameter> param =
-      binding.GetBinding(
-          DynamicParameterBinding::DynamicDimension{/*parameter_num=*/1,
-                                                    /*parameter_index=*/{},
-                                                    /*dimension=*/0});
-  EXPECT_TRUE(param);
-  EXPECT_EQ(param->parameter_num, 0);
-  EXPECT_EQ(param->parameter_index, ShapeIndex({}));
-  TF_EXPECT_OK(binding.Verify(*module));
+  auto test = [&](const DynamicParameterBinding& binding) {
+    absl::optional<DynamicParameterBinding::DynamicParameter> param =
+        binding.GetBinding(
+            DynamicParameterBinding::DynamicDimension{/*parameter_num=*/1,
+                                                      /*parameter_index=*/{},
+                                                      /*dimension=*/0});
+    EXPECT_TRUE(param);
+    EXPECT_EQ(param->parameter_num, 0);
+    EXPECT_EQ(param->parameter_index, ShapeIndex({}));
+    TF_EXPECT_OK(binding.Verify(*module));
+  };
+  test(binding);
+  SerializeAndDeserialize(&binding);
+  test(binding);
 }
 
 TEST_F(DynamicParameterBindingTest, TupleBinding) {
@@ -89,16 +102,21 @@ ENTRY main {
       binding.Bind(DynamicParameterBinding::DynamicParameter{0, {0}},
                    DynamicParameterBinding::DynamicDimension{0, {1}, 0}));
 
-  absl::optional<DynamicParameterBinding::DynamicParameter> param =
-      binding.GetBinding(
-          DynamicParameterBinding::DynamicDimension{/*parameter_num=*/0,
-                                                    /*parameter_index=*/{1},
-                                                    /*dimension=*/0});
-
-  EXPECT_TRUE(param);
-  EXPECT_EQ(param->parameter_num, 0);
-  EXPECT_EQ(param->parameter_index, ShapeIndex({0}));
-  TF_EXPECT_OK(binding.Verify(*module));
+  auto test = [&](const DynamicParameterBinding& binding) {
+    absl::optional<DynamicParameterBinding::DynamicParameter> param =
+        binding.GetBinding(
+            DynamicParameterBinding::DynamicDimension{/*parameter_num=*/0,
+                                                      /*parameter_index=*/{1},
+                                                      /*dimension=*/0});
+
+    EXPECT_TRUE(param);
+    EXPECT_EQ(param->parameter_num, 0);
+    EXPECT_EQ(param->parameter_index, ShapeIndex({0}));
+    TF_EXPECT_OK(binding.Verify(*module));
+  };
+  test(binding);
+  SerializeAndDeserialize(&binding);
+  test(binding);
 }
 
 TEST_F(DynamicParameterBindingTest, TupleBindingWithMultiDimension) {
@@ -127,26 +145,35 @@ ENTRY main {
       binding.Bind(DynamicParameterBinding::DynamicParameter{0, {0}},
                    DynamicParameterBinding::DynamicDimension{0, {1}, 1}));
 
-  absl::optional<DynamicParameterBinding::DynamicParameter> param =
-      binding.GetBinding(
-          DynamicParameterBinding::DynamicDimension{/*parameter_num=*/0,
-                                                    /*parameter_index=*/{1},
-                                                    /*dimension=*/0});
-
-  EXPECT_TRUE(param);
-  EXPECT_EQ(param->parameter_num, 0);
-  EXPECT_EQ(param->parameter_index, ShapeIndex({0}));
-
-  absl::optional<DynamicParameterBinding::DynamicParameter> param2 =
-      binding.GetBinding(
-          DynamicParameterBinding::DynamicDimension{/*parameter_num=*/0,
-                                                    /*parameter_index=*/{1},
-                                                    /*dimension=*/0});
-  EXPECT_TRUE(param2);
-  EXPECT_EQ(param2->parameter_num, 0);
-  EXPECT_EQ(param2->parameter_index, ShapeIndex({0}));
-
-  TF_EXPECT_OK(binding.Verify(*module));
+  auto test = [&](const DynamicParameterBinding& binding) {
+    absl::optional<DynamicParameterBinding::DynamicParameter> param =
+        binding.GetBinding(
+            DynamicParameterBinding::DynamicDimension{/*parameter_num=*/0,
+                                                      /*parameter_index=*/{1},
+                                                      /*dimension=*/0});
+
+    EXPECT_TRUE(param);
+    EXPECT_EQ(param->parameter_num, 0);
+    EXPECT_EQ(param->parameter_index, ShapeIndex({0}));
+
+    absl::optional<DynamicParameterBinding::DynamicParameter> param2 =
+
+        binding.GetBinding(
+            DynamicParameterBinding::DynamicDimension{/*parameter_num=*/0,
+                                                      /*parameter_index=*/{1},
+                                                      /*dimension=*/0});
+    EXPECT_TRUE(param2);
+    EXPECT_EQ(param2->parameter_num, 0);
+    EXPECT_EQ(param2->parameter_index, ShapeIndex({0}));
+    TF_EXPECT_OK(binding.Verify(*module));
+  };
+
+  test(binding);
+
+  SerializeAndDeserialize(&binding);
+
+  // Test the binding again after deserialization.
+  test(binding);
 }
 
 }  // namespace
-- 
GitLab


From 522066c826bbfb73cba6c07adcb62e00f4542017 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Tue, 22 Jan 2019 15:13:33 -0800
Subject: [PATCH 1080/2345] Fix TFLite Makefile

Do not build with NNAPI for microcontroller or iOS

PiperOrigin-RevId: 230420008
---
 tensorflow/lite/tools/make/Makefile | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/tensorflow/lite/tools/make/Makefile b/tensorflow/lite/tools/make/Makefile
index 994f660dba..e98ba9b2e0 100644
--- a/tensorflow/lite/tools/make/Makefile
+++ b/tensorflow/lite/tools/make/Makefile
@@ -111,15 +111,27 @@ $(wildcard tensorflow/lite/*/*/*test.cc) \
 $(wildcard tensorflow/lite/*/*/*/*test.cc) \
 $(wildcard tensorflow/lite/kernels/test_util.cc) \
 $(MINIMAL_SRCS)
+
 ifeq ($(BUILD_TYPE),micro)
-CORE_CC_EXCLUDE_SRCS += \
-tensorflow/lite/mmap_allocation.cc \
-tensorflow/lite/nnapi_delegate.cc
+	CORE_CC_EXCLUDE_SRCS += tensorflow/lite/mmap_allocation.cc
 else
-CORE_CC_EXCLUDE_SRCS += \
-tensorflow/lite/mmap_allocation_disabled.cc \
-tensorflow/lite/nnapi_delegate_disabled.cc
+	CORE_CC_EXCLUDE_SRCS += tensorflow/lite/mmap_allocation_disabled.cc
+endif
+
+BUILD_WITH_NNAPI=true
+ifeq ($(BUILD_TYPE),micro)
+	BUILD_WITH_NNAPI=false
+endif
+ifeq ($(TARGET),ios)
+	BUILD_WITH_NNAPI=false
 endif
+ifeq ($(BUILD_WITH_NNAPI),true)
+	CORE_CC_EXCLUDE_SRCS += tensorflow/lite/nnapi_delegate_disabled.cc
+else
+	CORE_CC_EXCLUDE_SRCS += tensorflow/lite/nnapi_delegate.cc
+endif
+
+
 # Filter out all the excluded files.
 TF_LITE_CC_SRCS := $(filter-out $(CORE_CC_EXCLUDE_SRCS), $(CORE_CC_ALL_SRCS))
 
-- 
GitLab


From 93d5185f1668c390224224972f166fd9afb786ee Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Tue, 22 Jan 2019 15:26:48 -0800
Subject: [PATCH 1081/2345] [tf.data] Make map_vectorization optimization work
 with ExperimentalMapAndBatchDataset as well. i.e.
 dataset.apply(tf.data.experimental.map_and_batch(map_fn, ...)) ->
 dataset.batch(...).map(vectorized_map_fn)

PiperOrigin-RevId: 230422450
---
 .../core/grappler/optimizers/data/BUILD       |   2 +
 .../optimizers/data/map_vectorization.cc      | 231 ++++++++++++-----
 .../optimizers/data/map_vectorization.h       |   6 +
 .../optimizers/data/map_vectorization_test.cc | 235 ++++++++++++++----
 .../kernel_tests/optimization/BUILD           |   3 +-
 .../optimization/map_vectorization_test.py    |  81 +++++-
 6 files changed, 443 insertions(+), 115 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/data/BUILD b/tensorflow/core/grappler/optimizers/data/BUILD
index 0a645d5cf4..93e8405d18 100644
--- a/tensorflow/core/grappler/optimizers/data/BUILD
+++ b/tensorflow/core/grappler/optimizers/data/BUILD
@@ -481,6 +481,7 @@ cc_library(
         ":optimizer_base",
         ":vectorization_utils",
         "@com_google_absl//absl/container:flat_hash_set",
+        "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
@@ -505,6 +506,7 @@ tf_cc_test(
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
         "//tensorflow/core/grappler:grappler_item",
+        "//tensorflow/core/grappler/utils:topological_sort",
         "//tensorflow/core/kernels/data",
     ],
 )
diff --git a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
index db64cfff02..bc935bbd0a 100644
--- a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
+#include "tensorflow/core/framework/node_def_util.h"
 #include "tensorflow/core/framework/tensor.pb.h"  // NOLINT
 #include "tensorflow/core/framework/tensor_shape.pb.h"
 #include "tensorflow/core/grappler/clusters/cluster.h"
@@ -29,6 +30,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/optimizers/data/function_utils.h"
 #include "tensorflow/core/grappler/optimizers/data/graph_utils.h"
 #include "tensorflow/core/grappler/utils.h"
+#include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/gtl/map_util.h"
 #include "tensorflow/core/platform/protobuf.h"
 
@@ -36,6 +38,12 @@ namespace tensorflow {
 namespace grappler {
 namespace {
 
+constexpr char kBatchOp[] = "BatchDataset";
+constexpr char kBatchV2Op[] = "BatchDatasetV2";
+constexpr char kExperimentalMapAndBatchOp[] = "ExperimentalMapAndBatchDataset";
+constexpr char kMapOp[] = "MapDataset";
+constexpr char kParallelMapOp[] = "ParallelMapDataset";
+
 // Returns a FunctionDef containing a MapDefun op that wraps the original
 // function.
 FunctionDef* CreateMapDefunWrapper(const NodeDef& map_node,
@@ -101,7 +109,6 @@ FunctionDef* AddVectorizedFunction(const NodeDef& map_node,
   const NodeDef& map_defun_node = vectorized_func->node_def(0);
   DCHECK_EQ(map_defun_node.op(), "MapDefun");
 
-  // TODO(b/116285210): Unreferenced functions should get cleaned up later
   FunctionDef* result;
   Status s = vectorization_utils::VectorizeMapDefun(
       *vectorized_func, map_defun_node, library, &result);
@@ -131,23 +138,53 @@ bool IsOutputShapesFullyDefined(const NodeDef& node) {
   return true;
 }
 
+// Returns a mapping from input names to the [start, end) indices of the input
+// in the node's input list.
+Status GetInputMap(const NodeDef& node, NameRangeMap* result) {
+  const OpRegistrationData* op_reg_data;  // Owned by global op registry
+  TF_RETURN_IF_ERROR(OpRegistry::Global()->LookUp(node.op(), &op_reg_data));
+
+  return NameRangesForNode(node, op_reg_data->op_def, result,
+                           /*outputs=*/nullptr);
+}
+
+Status CopyInputs(StringPiece input_name, const NameRangeMap& input_map,
+                  const NodeDef& from, NodeDef* to) {
+  const auto* range = gtl::FindOrNull(input_map, input_name);
+  if (range == nullptr) {
+    return errors::Internal(
+        "Failed to copy inputs: did not find inputs with name: ", input_name,
+        ", in node with name: ", from.name());
+  }
+  for (int i = range->first; i < range->second; ++i) {
+    to->add_input(from.input(i));
+  }
+
+  return Status::OK();
+}
 
-NodeDef MakeNewBatchNode(const NodeDef& old_batch_node,
-                         const NodeDef& input_node,
-                         const FunctionDef& vectorized_func,
-                         MutableGraphView* graph) {
+Status AddNewBatchNode(const NodeDef& old_batch_node, const NodeDef& input_node,
+                       const FunctionDef& vectorized_func,
+                       MutableGraphView* graph, NodeDef** new_batch_node) {
   NodeDef batch_node;
-  batch_node.set_op(old_batch_node.op());
+  batch_node.set_op(old_batch_node.op() == kBatchOp ? kBatchOp : kBatchV2Op);
   graph_utils::SetUniqueGraphNodeName(batch_node.op(), graph->graph(),
                                       &batch_node);
 
   // Set the `input_dataset` input argument
   batch_node.add_input(input_node.name());
-  // Set the `batch_size` input_argument
-  batch_node.add_input(old_batch_node.input(1));
-  if (batch_node.op() == "BatchDatasetV2") {
-    // Set the `drop_remainder` input argument
-    batch_node.add_input(old_batch_node.input(2));
+
+  NameRangeMap input_map;
+  TF_RETURN_IF_ERROR(GetInputMap(old_batch_node, &input_map));
+
+  // Set the `batch_size` input argument
+  TF_RETURN_IF_ERROR(
+      CopyInputs("batch_size", input_map, old_batch_node, &batch_node));
+
+  // Set the `drop_remainder` input argument
+  if (batch_node.op() != kBatchOp) {
+    TF_RETURN_IF_ERROR(
+        CopyInputs("drop_remainder", input_map, old_batch_node, &batch_node));
   }
 
   // Set attrs
@@ -157,6 +194,9 @@ NodeDef MakeNewBatchNode(const NodeDef& old_batch_node,
   }
   (*batch_node.mutable_attr())["output_types"] = output_types;
 
+  // It is safe to assume that input_node has the "output_shapes" attr here,
+  // because earlier we checked that the input node has fully defined output
+  // shapes.
   auto& output_shapes_attr = (*batch_node.mutable_attr())["output_shapes"];
   const auto& input_shapes =
       input_node.attr().at("output_shapes").list().shape();
@@ -168,23 +208,54 @@ NodeDef MakeNewBatchNode(const NodeDef& old_batch_node,
     dim->set_size(batch_size);
     shape->MergeFrom(input_shapes.Get(i));
   }
-  return batch_node;
+
+  *new_batch_node = graph->AddNode(std::move(batch_node));
+  return Status::OK();
 }
 
-NodeDef MakeNewMapNode(const NodeDef& old_map_node,
-                       const NodeDef& old_batch_node,
-                       const NodeDef& new_batch_node,
-                       const FunctionDef& vectorized_func,
-                       MutableGraphView* graph) {
+Status AddNewMapNode(const NodeDef& old_map_node, const NodeDef& old_batch_node,
+                     const NodeDef& new_batch_node,
+                     const FunctionDef& vectorized_func,
+                     MutableGraphView* graph, NodeDef** new_map_node) {
   NodeDef map_node;
-  map_node.set_op(old_map_node.op());
+  map_node.set_op(old_map_node.op() == kMapOp ? kMapOp : kParallelMapOp);
   graph_utils::SetUniqueGraphNodeName(map_node.op(), graph->graph(), &map_node);
 
   // Set the `input_dataset` input argument
   map_node.add_input(new_batch_node.name());
-  for (int i = 1; i < old_map_node.input_size(); i++) {
-    // Set the `other_arguments` and `num_parallel_calls` input arguments
-    map_node.add_input(old_map_node.input(i));
+
+  NameRangeMap input_map;
+  TF_RETURN_IF_ERROR(GetInputMap(old_map_node, &input_map));
+
+  // Set the `other_arguments` input argument
+  TF_RETURN_IF_ERROR(
+      CopyInputs("other_arguments", input_map, old_map_node, &map_node));
+
+  // Set the `num_parallel_calls` input argument
+  // TODO(rachelim): Evaluate the performance of potential transformations to
+  // the new `num_parallel_calls`:
+  //   1) dividing by the `batch_size`, since the new map will be operating
+  //      over larger elements
+  //   2) use the autotune value (i.e. -1)
+  //   3) use the original value
+  if (old_map_node.op() == kExperimentalMapAndBatchOp) {
+    // This `Cast` op is necessary because the `num_parallel_calls` input for
+    // ExperimentalMapAndBatch has type DT_INT64, but the input for ParallelMap
+    // expects type DT_INT32
+    NodeDef cast_node;
+    cast_node.set_op("Cast");
+    graph_utils::SetUniqueGraphNodeName(cast_node.op(), graph->graph(),
+                                        &cast_node);
+    AddNodeAttr("SrcT", DT_INT64, &cast_node);
+    AddNodeAttr("DstT", DT_INT32, &cast_node);
+    TF_RETURN_IF_ERROR(
+        CopyInputs("num_parallel_calls", input_map, old_map_node, &cast_node));
+    auto added_cast_node = graph->AddNode(std::move(cast_node));
+    map_node.add_input(added_cast_node->name());
+
+  } else if (old_map_node.op() == kParallelMapOp) {
+    TF_RETURN_IF_ERROR(
+        CopyInputs("num_parallel_calls", input_map, old_map_node, &map_node));
   }
 
   // Set attrs
@@ -197,8 +268,65 @@ NodeDef MakeNewMapNode(const NodeDef& old_map_node,
   }
 
   (*map_node.mutable_attr())["use_inter_op_parallelism"].set_b(true);
+  *new_map_node = graph->AddNode(std::move(map_node));
+  return Status::OK();
+}
 
-  return map_node;
+// Given an input pipeline graph and a query node, tries to the node to the
+// 'batch' node in a input_dataset->map->batch pattern, or the 'map_and_batch'
+// node in an input_dataset->map_and_batch pattern.
+bool FindMapAndBatchPattern(const MutableGraphView& graph, const NodeDef& node,
+                            const FunctionLibraryDefinition& function_library,
+                            const NodeDef** batch_node_output,
+                            const NodeDef** map_node_output,
+                            const NodeDef** input_node_output,
+                            const FunctionDef** map_fn_output) {
+  const FunctionDef*& map_fn = *map_fn_output;
+  const NodeDef*& batch_node = *batch_node_output;
+  const NodeDef*& map_node = *map_node_output;
+  const NodeDef*& input_node = *input_node_output;
+
+  if (node.op() == kExperimentalMapAndBatchOp) {
+    batch_node = &node;
+    map_node = &node;
+  } else if (node.op() == kBatchOp || node.op() == kBatchV2Op) {
+    batch_node = &node;
+    map_node = graph_utils::GetInputNode(*batch_node, graph);
+    if (map_node->op() != kMapOp && map_node->op() != kParallelMapOp) {
+      return false;
+    }
+    if (!IsOutputShapesFullyDefined(*map_node)) {
+      // If any of the map func outputs have an unknown shape, don't
+      // optimize, so that batching errors surface as before.
+      VLOG(1) << "Cannot vectorize dataset.map().batch() because the map "
+                 "dataset does not have fully defined output shapes.";
+      return false;
+    }
+  } else {
+    return false;
+  }
+
+  // Input to the map node
+  input_node = graph_utils::GetInputNode(*map_node, graph);
+  DCHECK_NE(input_node, nullptr);
+
+  if (!IsOutputShapesFullyDefined(*input_node)) {
+    // If any of the inputs have an unknown shape, don't optimize, since
+    // inputs might not be batchable.
+    VLOG(1) << "Cannot vectorize dataset.map().batch() because the input "
+               "dataset does not have fully defined output shapes.";
+    return false;
+  }
+
+  map_fn = function_library.Find(map_node->attr().at("f").func().name());
+
+  if (function_utils::IsFunctionStateful(function_library, *map_fn)) {
+    VLOG(1) << "Cannot vectorize dataset.map().batch() because the map "
+               "function is stateful.";
+    return false;
+  }
+
+  return true;
 }
 
 }  // namespace
@@ -211,58 +339,37 @@ Status MapVectorization::OptimizeAndCollectStats(Cluster* cluster,
   MutableGraphView graph(output);
   absl::flat_hash_set<string> nodes_to_delete;
 
-  for (const NodeDef& node : item.graph.node()) {
-    // Find Map->Batch nodes.
-    // TODO(rachelim): Optimize MapAndBatchDataset[V2] as well.
-    if (node.op() != "BatchDataset" && node.op() != "BatchDatasetV2") {
-      continue;
-    }
-
-    const NodeDef& batch_node(node);
-    NodeDef* node2 = graph_utils::GetInputNode(batch_node, graph);
-    if (node2->op() != "MapDataset" && node2->op() != "ParallelMapDataset") {
-      continue;
-    }
-
-    // Use a more descriptive variable name now that we know the node type.
-    NodeDef* map_node = node2;
-    // Input to the map node
-    NodeDef* input_node = graph_utils::GetInputNode(*map_node, graph);
-    CHECK_NOTNULL(input_node);
-
-    FunctionDefLibrary* library = output->mutable_library();
+  FunctionDefLibrary* library = output->mutable_library();
 
+  for (const NodeDef& node : item.graph.node()) {
     FunctionLibraryDefinition function_library(OpRegistry::Global(), *library);
-    const FunctionDef* orig_func =
-        function_library.Find(map_node->attr().at("f").func().name());
-
-    // Check that this is a valid optimization.
-    if (!IsOutputShapesFullyDefined(*input_node) ||
-        !IsOutputShapesFullyDefined(*map_node) ||
-        function_utils::IsFunctionStateful(function_library, *orig_func)) {
-      // 1. If any of the inputs have an unknown shape, don't optimize, since
-      // inputs might not be batchable.
-      // 2. If any of the map func outputs have an unknown shape, don't
-      // optimize, so that batching errors surface as before.
-      // 3. If the function is stateful, don't vectorize it.
+    const NodeDef* map_node;
+    const NodeDef* batch_node;
+    const NodeDef* input_node;
+    const FunctionDef* map_func;
+    if (!FindMapAndBatchPattern(graph, node, function_library, &batch_node,
+                                &map_node, &input_node, &map_func)) {
       continue;
     }
 
     FunctionDef* vectorized_func =
-        AddVectorizedFunction(*map_node, *orig_func, library);
+        AddVectorizedFunction(*map_node, *map_func, library);
     CHECK_NOTNULL(vectorized_func);
 
-    auto* new_batch_node = graph.AddNode(
-        MakeNewBatchNode(batch_node, *input_node, *vectorized_func, &graph));
+    NodeDef* new_batch_node;
+    TF_RETURN_IF_ERROR(AddNewBatchNode(
+        *batch_node, *input_node, *vectorized_func, &graph, &new_batch_node));
 
-    auto* new_map_node = graph.AddNode(MakeNewMapNode(
-        *map_node, batch_node, *new_batch_node, *vectorized_func, &graph));
-    TF_RETURN_IF_ERROR(
-        graph.UpdateFanouts(batch_node.name(), new_map_node->name()));
+    NodeDef* new_map_node;
+    TF_RETURN_IF_ERROR(AddNewMapNode(*map_node, *batch_node, *new_batch_node,
+                                     *vectorized_func, &graph, &new_map_node));
 
+    // Make output of Batch point to Map instead.
+    TF_RETURN_IF_ERROR(
+        graph.UpdateFanouts(batch_node->name(), new_map_node->name()));
     // Mark the `Map` and `Batch` nodes for removal.
     nodes_to_delete.insert(map_node->name());
-    nodes_to_delete.insert(batch_node.name());
+    nodes_to_delete.insert(batch_node->name());
     stats->num_changes++;
   }
   TF_RETURN_IF_ERROR(graph.DeleteNodes(nodes_to_delete));
diff --git a/tensorflow/core/grappler/optimizers/data/map_vectorization.h b/tensorflow/core/grappler/optimizers/data/map_vectorization.h
index cde6ef6839..3e170aae27 100644
--- a/tensorflow/core/grappler/optimizers/data/map_vectorization.h
+++ b/tensorflow/core/grappler/optimizers/data/map_vectorization.h
@@ -21,6 +21,12 @@ limitations under the License.
 namespace tensorflow {
 namespace grappler {
 
+// This optimizer rewrites dataset.map(map_fn, ...).batch(...) and
+// dataset.apply(tf.data.experimental.map_and_batch(map_fn, ...)) patterns in an
+// input pipeline. It vectorizes the map_fn, such that this segment can be
+// rewritten as dataset.batch().map(vectorized_map_fn). This is more performant
+// when the map_fn is cheap, because it amortizes the cost of running a map
+// function over a larger batch.
 class MapVectorization : public TFDataOptimizerBase {
  public:
   MapVectorization() = default;
diff --git a/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc b/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc
index 890041f57c..8a6df49758 100644
--- a/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_testutil.h"
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/optimizers/data/graph_utils.h"
+#include "tensorflow/core/grappler/utils/topological_sort.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/gtl/array_slice.h"
 #include "tensorflow/core/platform/test.h"
@@ -28,28 +29,45 @@ namespace tensorflow {
 namespace grappler {
 namespace {
 
+constexpr char kConstOp[] = "Const";
+constexpr char kRangeOp[] = "RangeDataset";
+constexpr char kBatchOp[] = "BatchDataset";
+constexpr char kBatchV2Op[] = "BatchDatasetV2";
+constexpr char kExperimentalMapAndBatchOp[] = "ExperimentalMapAndBatchDataset";
+constexpr char kMapOp[] = "MapDataset";
+constexpr char kParallelMapOp[] = "ParallelMapDataset";
+constexpr char kAttrNameF[] = "f";
+constexpr char kAttrNameTarguments[] = "Targuments";
+constexpr char kAttrNameOutputTypes[] = "output_types";
+constexpr char kAttrNameOutputShapes[] = "output_shapes";
+constexpr char kAttrNameInterOpParallelism[] = "use_inter_op_parallelism";
+constexpr char kAttrNamePreserveCardinality[] = "preserve_cardinality";
+constexpr char kAttrNameSloppy[] = "sloppy";
+constexpr char kAttrNameValue[] = "value";
+constexpr char kAttrNameDtype[] = "dtype";
+
 using test::function::NDef;
 
 NodeDef* AddConstBoolNode(GraphDef* graph, StringPiece name, bool value) {
   NodeDef* node = graph->add_node();
-  *node = NDef(/*name=*/name, /*op=*/"Const", /*inputs=*/{},
-               /*attrs=*/{{"value", value}, {"dtype", DT_BOOL}});
+  *node = NDef(/*name=*/name, /*op=*/kConstOp, /*inputs=*/{},
+               /*attrs=*/{{kAttrNameValue, value}, {kAttrNameDtype, DT_BOOL}});
 
   return node;
 }
 
 NodeDef* AddConstInt32Node(GraphDef* graph, StringPiece name, int value) {
   NodeDef* node = graph->add_node();
-  *node = NDef(/*name=*/name, /*op=*/"Const", /*inputs=*/{},
-               /*attrs=*/{{"value", value}, {"dtype", DT_INT32}});
+  *node = NDef(/*name=*/name, /*op=*/kConstOp, /*inputs=*/{},
+               /*attrs=*/{{kAttrNameValue, value}, {kAttrNameDtype, DT_INT32}});
 
   return node;
 }
 
 NodeDef* AddConstInt64Node(GraphDef* graph, StringPiece name, int64 value) {
   NodeDef* node = graph->add_node();
-  *node = NDef(/*name=*/name, /*op=*/"Const", /*inputs=*/{},
-               /*attrs=*/{{"value", value}, {"dtype", DT_INT64}});
+  *node = NDef(/*name=*/name, /*op=*/kConstOp, /*inputs=*/{},
+               /*attrs=*/{{kAttrNameValue, value}, {kAttrNameDtype, DT_INT64}});
 
   return node;
 }
@@ -70,60 +88,64 @@ FunctionDef* AddMapFn(GraphDef* graph) {
 }
 
 NodeDef* AddMapNode(GraphDef* graph, const string& input_dataset,
-                    bool parallel = false) {
+                    const string& map_fn, bool parallel = false,
+                    int suffix = 0) {
   NodeDef* map_node = graph->add_node();
-  auto map_fn = AddMapFn(graph);
 
   if (parallel) {
-    auto num_parallel_calls = AddConstInt32Node(graph, "num_parallel_calls", 8);
-    *map_node =
-        NDef(/*name=*/"map", /*op=*/"ParallelMapDataset",
-             /*inputs=*/{input_dataset, num_parallel_calls->name()},
-             /*attrs=*/
-             {{"f", FunctionDefHelper::FunctionRef(map_fn->signature().name())},
-              {"Targuments", gtl::ArraySlice<DataType>({})},
-              {"output_types", gtl::ArraySlice<DataType>({DT_INT64})},
-              {"output_shapes", gtl::ArraySlice<TensorShape>({{}})},
-              {"use_inter_op_parallelism", false},
-              {"sloppy", true},
-              {"preserve_cardinality", true}});
+    auto num_parallel_calls = AddConstInt32Node(
+        graph, strings::StrCat("num_parallel_calls", suffix), 8);
+    *map_node = NDef(
+        /*name=*/strings::StrCat("map", suffix), /*op=*/kParallelMapOp,
+        /*inputs=*/{input_dataset, num_parallel_calls->name()},
+        /*attrs=*/
+        {{kAttrNameF, FunctionDefHelper::FunctionRef(map_fn)},
+         {kAttrNameTarguments, gtl::ArraySlice<DataType>({})},
+         {kAttrNameOutputTypes, gtl::ArraySlice<DataType>({DT_INT64})},
+         {kAttrNameOutputShapes, gtl::ArraySlice<TensorShape>({{}})},
+         {kAttrNameInterOpParallelism, false},
+         {kAttrNameSloppy, true},
+         {kAttrNamePreserveCardinality, true}});
   } else {
     *map_node =
-        NDef(/*name=*/"map", /*op=*/"MapDataset",
+        NDef(/*name=*/strings::StrCat("map", suffix), /*op=*/kMapOp,
              /*inputs=*/{input_dataset},
              /*attrs=*/
-             {{"f", FunctionDefHelper::FunctionRef(map_fn->signature().name())},
-              {"Targuments", gtl::ArraySlice<DataType>({})},
-              {"output_types", gtl::ArraySlice<DataType>({DT_INT64})},
-              {"output_shapes", gtl::ArraySlice<TensorShape>({{}})},
-              {"use_inter_op_parallelism", false},
-              {"preserve_cardinality", true}});
+             {{kAttrNameF, FunctionDefHelper::FunctionRef(map_fn)},
+              {kAttrNameTarguments, gtl::ArraySlice<DataType>({})},
+              {kAttrNameOutputTypes, gtl::ArraySlice<DataType>({DT_INT64})},
+              {kAttrNameOutputShapes, gtl::ArraySlice<TensorShape>({{}})},
+              {kAttrNameInterOpParallelism, false},
+              {kAttrNamePreserveCardinality, true}});
   }
 
   return map_node;
 }
 
 NodeDef* AddBatchNode(GraphDef* graph, const string& input_dataset,
-                      bool v2 = false) {
+                      bool v2 = false, int suffix = 0) {
   NodeDef* batch_node = graph->add_node();
 
-  auto batch_size = AddConstInt64Node(graph, "batch_size", 10);
+  auto batch_size =
+      AddConstInt64Node(graph, strings::StrCat("batch_size", suffix), 10);
 
   if (v2) {
-    auto drop_remainder = AddConstBoolNode(graph, "drop_remainder", true);
+    auto drop_remainder = AddConstBoolNode(
+        graph, strings::StrCat("drop_remainder", suffix), true);
     *batch_node = NDef(
-        /*name=*/"batch", /*op=*/"BatchDatasetV2",
+        /*name=*/strings::StrCat("batch", suffix), /*op=*/kBatchV2Op,
         /*inputs=*/{input_dataset, batch_size->name(), drop_remainder->name()},
         /*attrs=*/
-        {{"output_types", gtl::ArraySlice<DataType>({DT_INT64})},
-         {"output_shapes", gtl::ArraySlice<TensorShape>({{10, 1}})}});
+        {{kAttrNameOutputTypes, gtl::ArraySlice<DataType>({DT_INT64})},
+         {kAttrNameOutputShapes, gtl::ArraySlice<TensorShape>({{10, 1}})}});
   } else {
     *batch_node = NDef(
-        /*name=*/"batch", /*op=*/"BatchDataset",
+        /*name=*/strings::StrCat("batch", suffix), /*op=*/kBatchOp,
         /*inputs=*/{input_dataset, batch_size->name()},
         /*attrs=*/
-        {{"output_types", gtl::ArraySlice<DataType>({DT_INT64})},
-         {"output_shapes", gtl::ArraySlice<PartialTensorShape>({{-1, 1}})}});
+        {{kAttrNameOutputTypes, gtl::ArraySlice<DataType>({DT_INT64})},
+         {kAttrNameOutputShapes,
+          gtl::ArraySlice<PartialTensorShape>({{v2 ? 10 : -1, 1}})}});
   }
   return batch_node;
 }
@@ -133,11 +155,12 @@ NodeDef* AddRangeNode(GraphDef* graph) {
   auto stop = AddConstInt64Node(graph, "stop", 10);
   auto step = AddConstInt64Node(graph, "step", 1);
   NodeDef* range_node = graph->add_node();
-  *range_node = NDef(/*name=*/"range", /*op=*/"RangeDataset",
-                     /*inputs=*/{start->name(), stop->name(), step->name()},
-                     /*attrs=*/
-                     {{"output_shapes", gtl::ArraySlice<TensorShape>({{}})},
-                      {"output_types", gtl::ArraySlice<DataType>({DT_INT64})}});
+  *range_node =
+      NDef(/*name=*/"range", /*op=*/kRangeOp,
+           /*inputs=*/{start->name(), stop->name(), step->name()},
+           /*attrs=*/
+           {{kAttrNameOutputShapes, gtl::ArraySlice<TensorShape>({{}})},
+            {kAttrNameOutputTypes, gtl::ArraySlice<DataType>({DT_INT64})}});
 
   return range_node;
 }
@@ -167,7 +190,7 @@ void CheckVectorized(const GraphDef& output, const string& map_op,
 
   // Check that the function is actually vectorized.
   // The vectorization of the identity function is itself.
-  string function_name = map_node.attr().at("f").func().name();
+  string function_name = map_node.attr().at(kAttrNameF).func().name();
   int found =
       graph_utils::FindGraphFunctionWithName(function_name, output.library());
   ASSERT_NE(found, -1);
@@ -183,7 +206,9 @@ TEST_P(MapThenBatchTest, IsVectorized) {
   bool use_batch_v2 = std::get<1>(GetParam());
   GrapplerItem item;
   auto range_node = AddRangeNode(&item.graph);
-  auto map_node = AddMapNode(&item.graph, range_node->name(), use_parallel_map);
+  auto map_fn = AddMapFn(&item.graph);
+  auto map_node = AddMapNode(&item.graph, range_node->name(),
+                             map_fn->signature().name(), use_parallel_map);
   auto batch_node = AddBatchNode(&item.graph, map_node->name(), use_batch_v2);
   MapVectorization optimizer;
   GraphDef output;
@@ -195,18 +220,118 @@ INSTANTIATE_TEST_SUITE_P(MapThenBatchTest, MapThenBatchTest,
                          ::testing::Combine(::testing::Bool(),
                                             ::testing::Bool()));
 
-// Not all dataset types have "output_shapes" and "output_types" attrs defined.
-// Add a generic input node which may not have these attrs defined.
+NodeDef* AddMapAndBatchNode(GraphDef* graph, const string& input_dataset,
+                            const string& map_fn, int suffix = 0) {
+  auto batch_size =
+      AddConstInt64Node(graph, strings::StrCat("batch_size", suffix), 10);
+  auto num_parallel_calls = AddConstInt64Node(
+      graph, strings::StrCat("num_parallel_calls", suffix), 8);
+  auto drop_remainder =
+      AddConstBoolNode(graph, strings::StrCat("drop_remainder", suffix), true);
+
+  auto map_and_batch_node = graph->add_node();
+  *map_and_batch_node =
+      NDef(/*name=*/strings::StrCat("map_and_batch", suffix),
+           /*op=*/kExperimentalMapAndBatchOp,
+           /*inputs=*/
+           {input_dataset, batch_size->name(), num_parallel_calls->name(),
+            drop_remainder->name()},
+           /*attrs=*/
+           {{kAttrNameF, FunctionDefHelper::FunctionRef(map_fn)},
+            {kAttrNameTarguments, gtl::ArraySlice<DataType>({})},
+            {kAttrNameOutputTypes, gtl::ArraySlice<DataType>({DT_INT64})},
+            {kAttrNameOutputShapes,
+             gtl::ArraySlice<PartialTensorShape>({{10, 1}})}});
+
+  return map_and_batch_node;
+}
+
+TEST(MapVectorizationTest, VectorizeExperimentalMapAndBatch) {
+  GrapplerItem item;
+  auto range_node = AddRangeNode(&item.graph);
+  auto map_fn = AddMapFn(&item.graph);
+  auto map_and_batch_node = AddMapAndBatchNode(&item.graph, range_node->name(),
+                                               map_fn->signature().name());
+  ASSERT_NE(map_and_batch_node, nullptr);
+
+  MapVectorization optimizer;
+  GraphDef output;
+  TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output));
+  CheckVectorized(output, kParallelMapOp, kBatchV2Op, "range");
+}
+
+class ChainedMapAndBatchTest
+    : public ::testing::TestWithParam<std::tuple<bool, bool>> {};
+
+// Tests:
+// 1) map.batch.map.batch
+// 2) map.batch.map_and_batch
+// 3) map_and_batch.map.batch
+// 4) map_and_batch.map_and_batch
+TEST_P(ChainedMapAndBatchTest, IsVectorized) {
+  GrapplerItem item;
+  auto input_node = AddRangeNode(&item.graph);
+
+  auto map_fn = AddMapFn(&item.graph);
+
+  auto make_map_and_batch = [&item, map_fn](NodeDef* input, bool fuse,
+                                            int suffix) {
+    if (fuse) {
+      return AddMapAndBatchNode(&item.graph, input->name(),
+                                map_fn->signature().name(), suffix);
+    }
+    auto map_node = AddMapNode(&item.graph, input->name(),
+                               map_fn->signature().name(), true, suffix);
+    auto batch_node = AddBatchNode(&item.graph, map_node->name(), true, suffix);
+    return batch_node;
+  };
+
+  auto map_and_batch_0 =
+      make_map_and_batch(input_node, std::get<0>(GetParam()), 0);
+  auto map_and_batch_1 =
+      make_map_and_batch(map_and_batch_0, std::get<1>(GetParam()), 1);
+  ASSERT_NE(map_and_batch_1, nullptr);
+
+  MapVectorization optimizer;
+  GraphDef output;
+  TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output));
+  TF_ASSERT_OK(TopologicalSort(&output));
+  std::vector<int> map_nodes =
+      graph_utils::FindAllGraphNodesWithOp(kParallelMapOp, output);
+  std::vector<int> batch_nodes =
+      graph_utils::FindAllGraphNodesWithOp(kBatchV2Op, output);
+  ASSERT_EQ(map_nodes.size(), 2);
+  ASSERT_EQ(batch_nodes.size(), 2);
+  const NodeDef& range_node =
+      output.node(graph_utils::FindGraphNodeWithOp(kRangeOp, output));
+
+  const NodeDef& batch_node_0 = output.node(batch_nodes[0]);
+  EXPECT_EQ(batch_node_0.input(0), range_node.name());
+  const NodeDef& map_node_0 = output.node(map_nodes[0]);
+  EXPECT_EQ(map_node_0.input(0), batch_node_0.name());
+  const NodeDef& batch_node_1 = output.node(batch_nodes[1]);
+  EXPECT_EQ(batch_node_1.input(0), map_node_0.name());
+  const NodeDef& map_node_1 = output.node(map_nodes[1]);
+  EXPECT_EQ(map_node_1.input(0), batch_node_1.name());
+}
+
+INSTANTIATE_TEST_SUITE_P(ChainedMapAndBatchTest, ChainedMapAndBatchTest,
+                         ::testing::Combine(::testing::Bool(),
+                                            ::testing::Bool()));
+
+// Not all dataset types have kAttrNameOutputShapes and kAttrNameOutputTypes
+// attrs defined. Add a generic input node which may not have these attrs
+// defined.
 NodeDef* AddArbitraryInputNode(GraphDef* graph,
                                std::vector<PartialTensorShape>* output_shapes,
                                std::vector<DataType>* output_types) {
   auto input_node = graph->add_node();
   std::vector<std::pair<string, FunctionDefHelper::AttrValueWrapper>> attrs;
   if (output_shapes) {
-    attrs.push_back({"output_shapes", *output_shapes});
+    attrs.push_back({kAttrNameOutputShapes, *output_shapes});
   }
   if (output_types) {
-    attrs.push_back({"output_types", *output_types});
+    attrs.push_back({kAttrNameOutputTypes, *output_types});
   }
 
   *input_node = NDef(/*name=*/"input", /*op=*/"InputDataset", /*inputs=*/{},
@@ -222,7 +347,9 @@ TEST(MapVectorizationTest, VectorizeWithUndefinedOutputShapes) {
   GrapplerItem item;
   std::vector<DataType> input_types({DT_INT64});
   auto input_node = AddArbitraryInputNode(&item.graph, nullptr, &input_types);
-  auto map_node = AddMapNode(&item.graph, input_node->name());
+  auto map_fn = AddMapFn(&item.graph);
+  auto map_node =
+      AddMapNode(&item.graph, input_node->name(), map_fn->signature().name());
   auto batch_node = AddBatchNode(&item.graph, map_node->name());
   MapVectorization optimizer;
   GraphDef output;
@@ -240,7 +367,9 @@ TEST(MapVectorizationTest, VectorizeWithUnknownRank) {
   std::vector<DataType> input_types({DT_INT64});
   auto input_node =
       AddArbitraryInputNode(&item.graph, &input_shapes, &input_types);
-  auto map_node = AddMapNode(&item.graph, input_node->name());
+  auto map_fn = AddMapFn(&item.graph);
+  auto map_node =
+      AddMapNode(&item.graph, input_node->name(), map_fn->signature().name());
   auto batch_node = AddBatchNode(&item.graph, map_node->name());
   MapVectorization optimizer;
   GraphDef output;
@@ -258,7 +387,9 @@ TEST(MapVectorizationTest, VectorizeWithUnknownDim) {
   std::vector<DataType> input_types({DT_INT64});
   auto input_node =
       AddArbitraryInputNode(&item.graph, &input_shapes, &input_types);
-  auto map_node = AddMapNode(&item.graph, input_node->name());
+  auto map_fn = AddMapFn(&item.graph);
+  auto map_node =
+      AddMapNode(&item.graph, input_node->name(), map_fn->signature().name());
   auto batch_node = AddBatchNode(&item.graph, map_node->name());
   MapVectorization optimizer;
   GraphDef output;
@@ -274,7 +405,9 @@ TEST(MapVectorizationTest, VectorizeWithUndefinedOutputTypes) {
   GrapplerItem item;
   std::vector<PartialTensorShape> input_shapes({{1}});
   auto input_node = AddArbitraryInputNode(&item.graph, &input_shapes, nullptr);
-  auto map_node = AddMapNode(&item.graph, input_node->name());
+  auto map_fn = AddMapFn(&item.graph);
+  auto map_node =
+      AddMapNode(&item.graph, input_node->name(), map_fn->signature().name());
   auto batch_node = AddBatchNode(&item.graph, map_node->name());
   MapVectorization optimizer;
   GraphDef output;
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD b/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
index 703d3350db..3bfe55244e 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
@@ -207,7 +207,7 @@ py_test(
 
 py_test(
     name = "map_vectorization_test",
-    size = "medium",
+    size = "small",
     srcs = ["map_vectorization_test.py"],
     shard_count = 8,
     srcs_version = "PY2AND3",
@@ -233,6 +233,7 @@ py_test(
         "//tensorflow/python:nn",
         "//tensorflow/python:parsing_ops",
         "//tensorflow/python:sparse_tensor",
+        "//tensorflow/python/data/experimental/ops:batching",
         "//tensorflow/python/data/experimental/ops:optimization",
         "//tensorflow/python/data/experimental/ops:optimization_options",
         "//tensorflow/python/data/kernel_tests:test_base",
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
index 9ffb8a1825..5e748a78ef 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.core.example import example_pb2
 from tensorflow.core.example import feature_pb2
+from tensorflow.python.data.experimental.ops import batching
 from tensorflow.python.data.experimental.ops import optimization
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
@@ -384,7 +385,7 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
       self.evaluate(nxt)
 
   def testOptimizationWithCapturedInputs(self):
-    # Tests that vectorization works with captured inputs
+    # Tests that vectorization works with captured inputs.
     y = constant_op.constant(1, shape=(2,))
     z = constant_op.constant(2, shape=(2,))
 
@@ -397,6 +398,84 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
         base_dataset, map_fn, expect_optimized=True)
     self.assertDatasetsEqual(optimized, unoptimized)
 
+  def testOptimizationWithMapAndBatchFusion(self):
+    # Tests that vectorization works on fused map and batch.
+    y = constant_op.constant(1, shape=(2,))
+    z = constant_op.constant(2, shape=(2,))
+
+    def map_fn(x):
+      return x, y, z
+
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    base_dataset = dataset_ops.Dataset.from_tensor_slices([[1, 2],
+                                                           [3, 4]]).repeat(5)
+    base_dataset = base_dataset.with_options(options)
+
+    def _make_dataset(node_names):
+      dataset = base_dataset.apply(optimization.assert_next(node_names))
+      dataset = dataset.apply(batching.map_and_batch(map_fn, 100))
+      return dataset
+
+    unoptimized = _make_dataset(["MapAndBatch"])
+    optimized = _make_dataset(["Batch", "ParallelMap"])
+    options = dataset_ops.Options()
+    options.experimental_optimization.map_vectorization = True
+    optimized = optimized.with_options(options)
+    self.assertDatasetsEqual(optimized, unoptimized)
+
+  @parameterized.named_parameters(
+      ("1", True, True),
+      ("2", True, False),
+      ("3", False, True),
+      ("4", False, False),
+  )
+  def testOptimizationWithChainedMapAndBatch(self, fuse_first, fuse_second):
+    # Tests that vectorization works on chained map and batch functions.
+    def map_fn(x):
+      return x * 2
+
+    unoptimized_seq = []
+
+    def make_apply_fn(is_fused):
+      if is_fused:
+        unoptimized_seq.append("MapAndBatch")
+
+        def apply_fn(dataset):
+          return dataset.apply(
+              batching.map_and_batch(map_fn, 2, 12, drop_remainder=True))
+
+        return apply_fn
+      else:
+        unoptimized_seq.extend(["ParallelMap", "Batch"])
+
+        def apply_fn(dataset):
+          return dataset.map(map_fn, 12).batch(2, drop_remainder=True)
+
+        return apply_fn
+
+    base_dataset = dataset_ops.Dataset.range(1000)
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    base_dataset = base_dataset.with_options(options)
+
+    apply_fn_1 = make_apply_fn(fuse_first)
+    apply_fn_2 = make_apply_fn(fuse_second)
+
+    def make_dataset(node_names):
+      dataset = base_dataset.apply(optimization.assert_next(node_names))
+      dataset = apply_fn_1(dataset)
+      dataset = apply_fn_2(dataset)
+      return dataset
+
+    unoptimized = make_dataset(unoptimized_seq)
+    optimized = make_dataset(["Batch", "ParallelMap", "Batch", "ParallelMap"])
+    options = dataset_ops.Options()
+    options.experimental_optimization.map_vectorization = True
+    optimized = optimized.with_options(options)
+
+    self.assertDatasetsEqual(optimized, unoptimized)
+
   # TODO(b/117581999): Add eager coverage for the following tests.
   def testSkipEagerOptimizationIgnoreStateful(self):
 
-- 
GitLab


From 96a2610cc56910f27219c64d037245a96f07ef89 Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Tue, 22 Jan 2019 15:37:10 -0800
Subject: [PATCH 1082/2345] Remove binary from micro tests

PiperOrigin-RevId: 230424415
---
 tensorflow/lite/experimental/micro/tools/make/Makefile       | 5 +----
 .../lite/experimental/micro/tools/make/helper_functions.inc  | 4 +++-
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/tensorflow/lite/experimental/micro/tools/make/Makefile b/tensorflow/lite/experimental/micro/tools/make/Makefile
index fde195118b..8be57cc18d 100644
--- a/tensorflow/lite/experimental/micro/tools/make/Makefile
+++ b/tensorflow/lite/experimental/micro/tools/make/Makefile
@@ -156,9 +156,6 @@ include $(wildcard tensorflow/lite/experimental/micro/examples/*/Makefile.inc)
 MICROLITE_LIB_OBJS := $(addprefix $(OBJDIR), \
 $(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(MICROLITE_CC_SRCS))))
 
-MICROLITE_TEST_TARGETS := $(addprefix $(BINDIR), \
-$(patsubst %_test.cc,%.test_target,$(MICROLITE_TEST_SRCS)))
-
 # For normal manually-created TensorFlow C++ source files.
 $(OBJDIR)%.o: %.cc
 	@mkdir -p $(dir $@)
@@ -201,7 +198,7 @@ $(BINDIR)%.test_target: $(BINDIR)%_test
 $(foreach TEST_TARGET,$(MICROLITE_TEST_SRCS),\
 $(eval $(call microlite_test,$(notdir $(basename $(TEST_TARGET))),$(TEST_TARGET))))
 
-test: test_micro_speech $(MICROLITE_TEST_TARGETS)
+test: $(MICROLITE_TEST_TARGETS)
 
 # Gets rid of all generated files.
 clean:
diff --git a/tensorflow/lite/experimental/micro/tools/make/helper_functions.inc b/tensorflow/lite/experimental/micro/tools/make/helper_functions.inc
index 87c002635f..c20cc5cfb3 100644
--- a/tensorflow/lite/experimental/micro/tools/make/helper_functions.inc
+++ b/tensorflow/lite/experimental/micro/tools/make/helper_functions.inc
@@ -103,7 +103,6 @@ $(1)_LOCAL_HDRS := $(3)
 $(1)_LOCAL_OBJS := $$(addprefix $$(OBJDIR), \
 $$(patsubst %.cc,%.o,$$(patsubst %.c,%.o,$$($(1)_LOCAL_SRCS))))
 $(1)_BINARY := $$(BINDIR)$(1)
-ALL_BINARIES += $$($(1)_BINARY)
 $$($(1)_BINARY): $$($(1)_LOCAL_OBJS) $$(MICROLITE_LIB_PATH)
 	@mkdir -p $$(dir $$@)
 	$$(CXX) $$(CXXFLAGS) $$(INCLUDES) \
@@ -113,5 +112,8 @@ $(1): $$($(1)_BINARY)
 $(1)_bin: $$($(1)_BINARY).bin
 test_$(1): $$($(1)_BINARY)
 	$$(TEST_SCRIPT) $$($(1)_BINARY) '~~~ALL TESTS PASSED~~~'
+ifneq (,$(findstring _test,$(1)))
+  MICROLITE_TEST_TARGETS += test_$(1)
+endif
 $(eval $(call generate_microlite_projects,$(1),$(call specialize,$(2)),$(3)))
 endef
-- 
GitLab


From 824c73a1533fed37222ed5d071e04bae1fc0c3b1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 15:40:52 -0800
Subject: [PATCH 1083/2345] Fix test size to enable running during presubmits.

PiperOrigin-RevId: 230425144
---
 tensorflow/contrib/eager/python/examples/densenet/BUILD | 6 ++++--
 tensorflow/contrib/eager/python/examples/l2hmc/BUILD    | 3 ++-
 tensorflow/contrib/eager/python/examples/resnet50/BUILD | 6 ++++--
 tensorflow/contrib/eager/python/examples/revnet/BUILD   | 9 ++++++---
 4 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/eager/python/examples/densenet/BUILD b/tensorflow/contrib/eager/python/examples/densenet/BUILD
index 56a682ec55..6e10d25acf 100644
--- a/tensorflow/contrib/eager/python/examples/densenet/BUILD
+++ b/tensorflow/contrib/eager/python/examples/densenet/BUILD
@@ -17,13 +17,14 @@ py_binary(
 
 cuda_py_test(
     name = "densenet_test",
-    size = "large",
+    size = "medium",
     srcs = ["densenet_test.py"],
     additional_deps = [
         ":densenet",
         "//tensorflow/contrib/eager/python:tfe",
         "//tensorflow:tensorflow_py",
     ],
+    shard_count = 4,
     tags = [
         "no_pip",
         "optonly",
@@ -33,13 +34,14 @@ cuda_py_test(
 
 cuda_py_test(
     name = "densenet_graph_test",
-    size = "large",
+    size = "medium",
     srcs = ["densenet_graph_test.py"],
     additional_deps = [
         ":densenet",
         "//third_party/py/numpy",
         "//tensorflow:tensorflow_py",
     ],
+    shard_count = 4,
     tags = [
         "no_pip",
         "noasan",
diff --git a/tensorflow/contrib/eager/python/examples/l2hmc/BUILD b/tensorflow/contrib/eager/python/examples/l2hmc/BUILD
index 78548c51c9..35d5099042 100644
--- a/tensorflow/contrib/eager/python/examples/l2hmc/BUILD
+++ b/tensorflow/contrib/eager/python/examples/l2hmc/BUILD
@@ -28,7 +28,7 @@ py_library(
 
 cuda_py_test(
     name = "l2hmc_test",
-    size = "large",
+    size = "medium",
     srcs = ["l2hmc_test.py"],
     additional_deps = [
         ":l2hmc",
@@ -36,6 +36,7 @@ cuda_py_test(
         "//tensorflow/contrib/eager/python:tfe",
         "//third_party/py/numpy",
     ],
+    shard_count = 4,
     tags = [
         "oss_serial",
     ],
diff --git a/tensorflow/contrib/eager/python/examples/resnet50/BUILD b/tensorflow/contrib/eager/python/examples/resnet50/BUILD
index 04ac78a2d3..f2851d9722 100644
--- a/tensorflow/contrib/eager/python/examples/resnet50/BUILD
+++ b/tensorflow/contrib/eager/python/examples/resnet50/BUILD
@@ -27,7 +27,7 @@ py_library(
 
 cuda_py_test(
     name = "resnet50_test",
-    size = "large",
+    size = "medium",
     srcs = ["resnet50_test.py"],
     additional_deps = [
         ":resnet50",
@@ -35,6 +35,7 @@ cuda_py_test(
         "//tensorflow/contrib/eager/python:tfe",
         "//tensorflow:tensorflow_py",
     ],
+    shard_count = 4,
     tags = [
         "noasan",  # Fix b/118130911
         "nomsan",  # Fix b/118130911
@@ -46,7 +47,7 @@ cuda_py_test(
 
 cuda_py_test(
     name = "resnet50_graph_test",
-    size = "large",
+    size = "medium",
     srcs = ["resnet50_graph_test.py"],
     additional_deps = [
         ":resnet50",
@@ -54,6 +55,7 @@ cuda_py_test(
         "//third_party/py/numpy",
         "//tensorflow:tensorflow_py",
     ],
+    shard_count = 4,
     tags = [
         "noasan",
         "nomsan",
diff --git a/tensorflow/contrib/eager/python/examples/revnet/BUILD b/tensorflow/contrib/eager/python/examples/revnet/BUILD
index d05270089f..1948c1d8ff 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/BUILD
+++ b/tensorflow/contrib/eager/python/examples/revnet/BUILD
@@ -67,12 +67,13 @@ py_library(
 # Tests
 cuda_py_test(
     name = "ops_test",
-    size = "large",
+    size = "medium",
     srcs = ["ops_test.py"],
     additional_deps = [
         ":ops",
         "//tensorflow:tensorflow_py",
     ],
+    shard_count = 4,
     tags = [
         "oss_serial",
     ],
@@ -80,12 +81,13 @@ cuda_py_test(
 
 cuda_py_test(
     name = "blocks_test",
-    size = "large",
+    size = "medium",
     srcs = ["blocks_test.py"],
     additional_deps = [
         ":blocks",
         "//tensorflow:tensorflow_py",
     ],
+    shard_count = 4,
     tags = [
         "no_oss",  # b/123045964
         "optonly",
@@ -94,7 +96,7 @@ cuda_py_test(
 
 cuda_py_test(
     name = "revnet_test",
-    size = "large",
+    size = "medium",
     srcs = ["revnet_test.py"],
     additional_deps = [
         ":blocks_test",
@@ -102,6 +104,7 @@ cuda_py_test(
         ":revnet",
         "//tensorflow:tensorflow_py",
     ],
+    shard_count = 4,
     tags = [
         "no_pip",  # depends on blocks_test, which is not available in pip package
         "optonly",
-- 
GitLab


From fea5ff8e3dc49feef809d28ef0ac1fd48f6a7009 Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Tue, 22 Jan 2019 15:53:15 -0800
Subject: [PATCH 1084/2345] Remove unused vector.

PiperOrigin-RevId: 230427369
---
 tensorflow/lite/tools/optimize/quantization_utils.cc | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tensorflow/lite/tools/optimize/quantization_utils.cc b/tensorflow/lite/tools/optimize/quantization_utils.cc
index 7239ebcc7a..c2405ca389 100644
--- a/tensorflow/lite/tools/optimize/quantization_utils.cc
+++ b/tensorflow/lite/tools/optimize/quantization_utils.cc
@@ -123,9 +123,6 @@ void SymmetricPerChannelQuantization(const float* const input,
   }
 
   // Quantize the values.
-  const uint64_t num_elements_per_channel =
-      output_scales->size() / channel_dim_size;
-  std::vector<int8_t> quantized_buffer(num_elements_per_channel);
   memset(indices, 0, 4 * sizeof(int));
   for (indices[0] = 0; indices[0] < dimension[0]; indices[0]++) {
     for (indices[1] = 0; indices[1] < dimension[1]; indices[1]++) {
-- 
GitLab


From 632658950c5f3acb1e05bc375bbcff90479934f0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 15:54:34 -0800
Subject: [PATCH 1085/2345] Print the model name as part of
 layer_utils.print_summary (aka model.summary)

CURRENT:
_________________________________________________________________
Layer (type)                 Output Shape              Param #
=================================================================
model_1_input (InputLayer)   [(None, 128)]             0

PROPOSED
_________________________________________________________________
Model: "part_of_speech_tagger"
Layer (type)                 Output Shape              Param #
=================================================================
model_1_input (InputLayer)   [(None, 128)]             0
PiperOrigin-RevId: 230427583
---
 tensorflow/python/keras/utils/layer_utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/keras/utils/layer_utils.py b/tensorflow/python/keras/utils/layer_utils.py
index 1d85e8a25f..640462d5c6 100644
--- a/tensorflow/python/keras/utils/layer_utils.py
+++ b/tensorflow/python/keras/utils/layer_utils.py
@@ -158,6 +158,7 @@ def print_summary(model, line_length=None, positions=None, print_fn=None):
       line += ' ' * (positions[i] - len(line))
     print_fn(line)
 
+  print_fn('Model: "{}"'.format(model.name))
   print_fn('_' * line_length)
   print_row(to_display, positions)
   print_fn('=' * line_length)
-- 
GitLab


From 24baec1b8c2556c8c1ffe1fdf6f67b0324f4389c Mon Sep 17 00:00:00 2001
From: Youlong Cheng <ylc@google.com>
Date: Tue, 22 Jan 2019 16:00:46 -0800
Subject: [PATCH 1086/2345]   Internal change.

PiperOrigin-RevId: 230428660
---
 tensorflow/contrib/tpu/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index d898f05405..b3d5c41e7b 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -171,6 +171,7 @@ tf_custom_op_py_library(
         ":tpu_ordinal_selector_op_op_lib",
     ],
     srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
     deps = [
         ":tpu_ordinal_selector_op",
     ],
-- 
GitLab


From b084482d79d28bb79f26c9b86f7ba654fcf25238 Mon Sep 17 00:00:00 2001
From: James Keeling <jtkeeling@google.com>
Date: Tue, 22 Jan 2019 16:12:11 -0800
Subject: [PATCH 1087/2345] Add bounds checking when looking at the stack in
 deprecation messages.

This was breaking integration with R, fixes #25067.

PiperOrigin-RevId: 230431304
---
 tensorflow/python/util/deprecation.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/util/deprecation.py b/tensorflow/python/util/deprecation.py
index 9aaf0c2de9..f8e8d4c28a 100644
--- a/tensorflow/python/util/deprecation.py
+++ b/tensorflow/python/util/deprecation.py
@@ -100,7 +100,13 @@ def _validate_deprecation_args(date, instructions):
 def _call_location(outer=False):
   """Returns call location given level up from current call."""
   stack = tf_stack.extract_stack()
-  frame = stack[-4 if outer else -3]
+  length = len(stack)
+  if length == 0:  # should never happen as we're in a function
+    return 'UNKNOWN'
+  index = length-4 if outer else length-3
+  if index < 0:
+    index = 0
+  frame = stack[index]
   return '{filename}:{lineno}'.format(filename=frame[0], lineno=frame[1])
 
 
-- 
GitLab


From d09e5afc188343492877ab815d4f7cf46c81ff4f Mon Sep 17 00:00:00 2001
From: James Keeling <jtkeeling@google.com>
Date: Tue, 22 Jan 2019 16:13:25 -0800
Subject: [PATCH 1088/2345] Split strided slice GPU code into multiple files

This file was a bottleneck during compilation, often taking many minutes to compile. In local testing this change reduces the wall-clock build time for the affected kernels from >3 mins to approx 1 min.

PiperOrigin-RevId: 230431569
---
 tensorflow/core/kernels/BUILD                 |  6 +++-
 .../kernels/strided_slice_op_gpu_bool.cu.cc   | 27 +++++++++++++++++
 .../strided_slice_op_gpu_complex.cu.cc        | 28 ++++++++++++++++++
 ..._gpu.cu.cc => strided_slice_op_gpu_impl.h} | 13 +++------
 .../kernels/strided_slice_op_gpu_int.cu.cc    | 29 +++++++++++++++++++
 .../strided_slice_op_gpu_number_types.cu.cc   | 27 +++++++++++++++++
 6 files changed, 120 insertions(+), 10 deletions(-)
 create mode 100644 tensorflow/core/kernels/strided_slice_op_gpu_bool.cu.cc
 create mode 100644 tensorflow/core/kernels/strided_slice_op_gpu_complex.cu.cc
 rename tensorflow/core/kernels/{strided_slice_op_gpu.cu.cc => strided_slice_op_gpu_impl.h} (90%)
 create mode 100644 tensorflow/core/kernels/strided_slice_op_gpu_int.cu.cc
 create mode 100644 tensorflow/core/kernels/strided_slice_op_gpu_number_types.cu.cc

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 049d429d36..ce6504e0b6 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -138,7 +138,11 @@ tf_kernel_library(
         "slice_op.h",
         "strided_slice_op.h",
         "strided_slice_op_impl.h",
-        "strided_slice_op_gpu.cu.cc",
+        "strided_slice_op_gpu_impl.h",
+        "strided_slice_op_gpu_int.cu.cc",
+        "strided_slice_op_gpu_complex.cu.cc",
+        "strided_slice_op_gpu_bool.cu.cc",
+        "strided_slice_op_gpu_number_types.cu.cc",
     ],
     deps = [
         ":bounds_check",
diff --git a/tensorflow/core/kernels/strided_slice_op_gpu_bool.cu.cc b/tensorflow/core/kernels/strided_slice_op_gpu_bool.cu.cc
new file mode 100644
index 0000000000..8c3f8f2ad3
--- /dev/null
+++ b/tensorflow/core/kernels/strided_slice_op_gpu_bool.cu.cc
@@ -0,0 +1,27 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/strided_slice_op.h"
+#include "tensorflow/core/kernels/strided_slice_op_gpu_impl.h"
+
+namespace tensorflow {
+TF_CALL_bool(DEFINE_GPU_KERNELS);
+}  // end namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/strided_slice_op_gpu_complex.cu.cc b/tensorflow/core/kernels/strided_slice_op_gpu_complex.cu.cc
new file mode 100644
index 0000000000..f695192465
--- /dev/null
+++ b/tensorflow/core/kernels/strided_slice_op_gpu_complex.cu.cc
@@ -0,0 +1,28 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/strided_slice_op.h"
+#include "tensorflow/core/kernels/strided_slice_op_gpu_impl.h"
+
+namespace tensorflow {
+TF_CALL_complex64(DEFINE_GPU_KERNELS);
+TF_CALL_complex128(DEFINE_GPU_KERNELS);
+}  // end namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/strided_slice_op_gpu.cu.cc b/tensorflow/core/kernels/strided_slice_op_gpu_impl.h
similarity index 90%
rename from tensorflow/core/kernels/strided_slice_op_gpu.cu.cc
rename to tensorflow/core/kernels/strided_slice_op_gpu_impl.h
index cce1d2fddd..d70f369ac0 100644
--- a/tensorflow/core/kernels/strided_slice_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/strided_slice_op_gpu_impl.h
@@ -13,6 +13,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#ifndef TENSORFLOW_CORE_KERNELS_STRIDED_SLICE_OP_GPU_IMPL_H_
+#define TENSORFLOW_CORE_KERNELS_STRIDED_SLICE_OP_GPU_IMPL_H_
+
 #if GOOGLE_CUDA
 
 #define EIGEN_USE_GPU
@@ -50,16 +53,8 @@ typedef Eigen::GpuDevice GPUDevice;
   template struct functor::StridedSliceAssign<GPUDevice, T, 6>; \
   template struct functor::StridedSliceAssign<GPUDevice, T, 7>; \
   template struct functor::StridedSliceAssignScalar<GPUDevice, T>;
-TF_CALL_GPU_NUMBER_TYPES(DEFINE_GPU_KERNELS);
-TF_CALL_complex64(DEFINE_GPU_KERNELS);
-TF_CALL_complex128(DEFINE_GPU_KERNELS);
-TF_CALL_int64(DEFINE_GPU_KERNELS);
-TF_CALL_bool(DEFINE_GPU_KERNELS);
-TF_CALL_int8(DEFINE_GPU_KERNELS);
-DEFINE_GPU_KERNELS(int32);
-
-#undef DEFINE_GPU_KERNELS
 
 }  // end namespace tensorflow
 
 #endif  // GOOGLE_CUDA
+#endif  // TENSORFLOW_CORE_KERNELS_STRIDED_SLICE_OP_GPU_IMPL_H_
diff --git a/tensorflow/core/kernels/strided_slice_op_gpu_int.cu.cc b/tensorflow/core/kernels/strided_slice_op_gpu_int.cu.cc
new file mode 100644
index 0000000000..07dd0130ad
--- /dev/null
+++ b/tensorflow/core/kernels/strided_slice_op_gpu_int.cu.cc
@@ -0,0 +1,29 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/strided_slice_op.h"
+#include "tensorflow/core/kernels/strided_slice_op_gpu_impl.h"
+
+namespace tensorflow {
+TF_CALL_int8(DEFINE_GPU_KERNELS);
+TF_CALL_int32(DEFINE_GPU_KERNELS);
+TF_CALL_int64(DEFINE_GPU_KERNELS);
+}  // end namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/strided_slice_op_gpu_number_types.cu.cc b/tensorflow/core/kernels/strided_slice_op_gpu_number_types.cu.cc
new file mode 100644
index 0000000000..149886308c
--- /dev/null
+++ b/tensorflow/core/kernels/strided_slice_op_gpu_number_types.cu.cc
@@ -0,0 +1,27 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/strided_slice_op.h"
+#include "tensorflow/core/kernels/strided_slice_op_gpu_impl.h"
+
+namespace tensorflow {
+TF_CALL_GPU_NUMBER_TYPES(DEFINE_GPU_KERNELS);
+}  // end namespace tensorflow
+
+#endif  // GOOGLE_CUDA
-- 
GitLab


From a6fe0dfd38a7b5f61e6259471baf2a44cc590570 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 16:20:46 -0800
Subject: [PATCH 1089/2345] Fix the test size

PiperOrigin-RevId: 230433114
---
 tensorflow/python/keras/BUILD | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index c4c403119c..7705491c85 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -770,7 +770,7 @@ tf_py_test(
 
 tf_py_test(
     name = "data_utils_test",
-    size = "large",
+    size = "medium",
     srcs = ["utils/data_utils_test.py"],
     additional_deps = [
         ":keras",
@@ -778,6 +778,7 @@ tf_py_test(
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
     ],
+    shard_count = 6,
     tags = [
         "no_oss",
         "no_windows",
@@ -967,7 +968,7 @@ tf_py_test(
 
 tf_py_test(
     name = "training_generator_test",
-    size = "large",
+    size = "medium",
     srcs = ["engine/training_generator_test.py"],
     additional_deps = [
         ":keras",
@@ -975,7 +976,7 @@ tf_py_test(
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
     ],
-    shard_count = 3,
+    shard_count = 6,
     tags = [
         "no_oss",
         "notsan",
-- 
GitLab


From 6fa83a643f148c78a9bf4e50a8a7572accde5e09 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 16:40:27 -0800
Subject: [PATCH 1090/2345] Fix the test size

PiperOrigin-RevId: 230436567
---
 tensorflow/python/BUILD | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 4690bed62b..bc109c09bc 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -5451,7 +5451,7 @@ tf_py_test(
 
 cuda_py_test(
     name = "accumulate_n_benchmark",
-    size = "large",
+    size = "medium",
     srcs = ["ops/accumulate_n_benchmark.py"],
     additional_deps = [
         ":array_ops",
@@ -5466,6 +5466,7 @@ cuda_py_test(
         ":state_ops_gen",
     ],
     main = "ops/accumulate_n_benchmark.py",
+    shard_count = 6,
 )
 
 cuda_py_test(
-- 
GitLab


From 09bdeffbcd2c4f8f9c0da8d101eaf9edc1faf54d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 16:50:38 -0800
Subject: [PATCH 1091/2345] n/a

PiperOrigin-RevId: 230438132
---
 tensorflow/contrib/distribute/python/examples/BUILD |  6 ++++++
 tensorflow/contrib/eager/python/examples/BUILD      | 10 +++++-----
 .../contrib/eager/python/examples/densenet/BUILD    |  7 +++++++
 tensorflow/contrib/eager/python/examples/gan/BUILD  |  7 +++++++
 .../eager/python/examples/linear_regression/BUILD   |  7 +++++++
 .../contrib/eager/python/examples/revnet/BUILD      | 13 ++++++++++---
 .../eager/python/examples/rnn_colorbot/BUILD        |  7 +++++++
 .../contrib/eager/python/examples/rnn_ptb/BUILD     |  7 +++++++
 tensorflow/contrib/rnn/BUILD                        |  9 ++++++++-
 9 files changed, 64 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/examples/BUILD b/tensorflow/contrib/distribute/python/examples/BUILD
index 84b106545e..75d9da6a96 100644
--- a/tensorflow/contrib/distribute/python/examples/BUILD
+++ b/tensorflow/contrib/distribute/python/examples/BUILD
@@ -31,6 +31,12 @@ py_binary(
 
 py_binary(
     name = "keras_mnist",
+    srcs = ["keras_mnist.py"],
+    deps = [":keras_mnist_lib"],
+)
+
+py_library(
+    name = "keras_mnist_lib",
     srcs = [
         "keras_mnist.py",
     ],
diff --git a/tensorflow/contrib/eager/python/examples/BUILD b/tensorflow/contrib/eager/python/examples/BUILD
index 97c299a911..3e0881754c 100644
--- a/tensorflow/contrib/eager/python/examples/BUILD
+++ b/tensorflow/contrib/eager/python/examples/BUILD
@@ -6,16 +6,16 @@ package(default_visibility = ["//tensorflow:internal"])
 py_library(
     name = "examples_pip",
     deps = [
-        "//tensorflow/contrib/eager/python/examples/densenet",
-        "//tensorflow/contrib/eager/python/examples/gan:mnist",
+        "//tensorflow/contrib/eager/python/examples/densenet:densenet_lib",
+        "//tensorflow/contrib/eager/python/examples/gan:mnist_lib",
         "//tensorflow/contrib/eager/python/examples/l2hmc",
         "//tensorflow/contrib/eager/python/examples/l2hmc:neural_nets",
-        "//tensorflow/contrib/eager/python/examples/linear_regression",
+        "//tensorflow/contrib/eager/python/examples/linear_regression:linear_regression_lib",
         "//tensorflow/contrib/eager/python/examples/resnet50",
         "//tensorflow/contrib/eager/python/examples/revnet",
         "//tensorflow/contrib/eager/python/examples/revnet:config",
-        "//tensorflow/contrib/eager/python/examples/rnn_colorbot",
-        "//tensorflow/contrib/eager/python/examples/rnn_ptb",
+        "//tensorflow/contrib/eager/python/examples/rnn_colorbot:rnn_colorbot_lib",
+        "//tensorflow/contrib/eager/python/examples/rnn_ptb:rnn_ptb_lib",
         "//tensorflow/contrib/eager/python/examples/spinn:data",
     ],
 )
diff --git a/tensorflow/contrib/eager/python/examples/densenet/BUILD b/tensorflow/contrib/eager/python/examples/densenet/BUILD
index 6e10d25acf..fbb5daf230 100644
--- a/tensorflow/contrib/eager/python/examples/densenet/BUILD
+++ b/tensorflow/contrib/eager/python/examples/densenet/BUILD
@@ -9,6 +9,13 @@ py_binary(
     name = "densenet",
     srcs = ["densenet.py"],
     srcs_version = "PY2AND3",
+    deps = [":densenet_lib"],
+)
+
+py_library(
+    name = "densenet_lib",
+    srcs = ["densenet.py"],
+    srcs_version = "PY2AND3",
     deps = [
         "//tensorflow:tensorflow_py",
         "//tensorflow/contrib/eager/python:tfe",
diff --git a/tensorflow/contrib/eager/python/examples/gan/BUILD b/tensorflow/contrib/eager/python/examples/gan/BUILD
index d64c8eb9ce..74b742515f 100644
--- a/tensorflow/contrib/eager/python/examples/gan/BUILD
+++ b/tensorflow/contrib/eager/python/examples/gan/BUILD
@@ -9,6 +9,13 @@ py_binary(
     name = "mnist",
     srcs = ["mnist.py"],
     srcs_version = "PY2AND3",
+    deps = [":mnist_lib"],
+)
+
+py_library(
+    name = "mnist_lib",
+    srcs = ["mnist.py"],
+    srcs_version = "PY2AND3",
     deps = [
         "//tensorflow:tensorflow_py",
         "//tensorflow/contrib/eager/python:tfe",
diff --git a/tensorflow/contrib/eager/python/examples/linear_regression/BUILD b/tensorflow/contrib/eager/python/examples/linear_regression/BUILD
index 6a178ddcec..2e7b8a013b 100644
--- a/tensorflow/contrib/eager/python/examples/linear_regression/BUILD
+++ b/tensorflow/contrib/eager/python/examples/linear_regression/BUILD
@@ -9,6 +9,13 @@ py_binary(
     name = "linear_regression",
     srcs = ["linear_regression.py"],
     srcs_version = "PY2AND3",
+    deps = [":linear_regression_lib"],
+)
+
+py_library(
+    name = "linear_regression_lib",
+    srcs = ["linear_regression.py"],
+    srcs_version = "PY2AND3",
     deps = [
         "//tensorflow:tensorflow_py",
         "//tensorflow/contrib/eager/python:tfe",
diff --git a/tensorflow/contrib/eager/python/examples/revnet/BUILD b/tensorflow/contrib/eager/python/examples/revnet/BUILD
index 1948c1d8ff..cb207b8ddf 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/BUILD
+++ b/tensorflow/contrib/eager/python/examples/revnet/BUILD
@@ -135,6 +135,13 @@ py_binary(
     name = "main",
     srcs = ["main.py"],
     srcs_version = "PY2AND3",
+    deps = [":main_lib"],
+)
+
+py_library(
+    name = "main_lib",
+    srcs = ["main.py"],
+    srcs_version = "PY2AND3",
     deps = [
         ":cifar_input",
         ":config",
@@ -149,7 +156,7 @@ py_binary(
     srcs_version = "PY2AND3",
     deps = [
         ":cifar_input",
-        ":main",
+        ":main_lib",
         ":revnet",
         "//tensorflow:tensorflow_py",
     ],
@@ -161,7 +168,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":cifar_input",
-        ":main",
+        ":main_lib",
         ":revnet",
         "//tensorflow:tensorflow_py",
     ],
@@ -173,7 +180,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":cifar_input",
-        ":main",
+        ":main_lib",
         ":revnet",
         "//tensorflow:tensorflow_py",
     ],
diff --git a/tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD b/tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD
index 576f60396e..d4a854fb53 100644
--- a/tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD
+++ b/tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD
@@ -9,6 +9,13 @@ py_binary(
     name = "rnn_colorbot",
     srcs = ["rnn_colorbot.py"],
     srcs_version = "PY2AND3",
+    deps = [":rnn_colorbot_lib"],
+)
+
+py_library(
+    name = "rnn_colorbot_lib",
+    srcs = ["rnn_colorbot.py"],
+    srcs_version = "PY2AND3",
     deps = [
         "//tensorflow:tensorflow_py",
         "//tensorflow/contrib/eager/python:tfe",
diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD b/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD
index 5d48ee4a5f..1c5a3842d5 100644
--- a/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD
+++ b/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD
@@ -9,6 +9,13 @@ py_binary(
     name = "rnn_ptb",
     srcs = ["rnn_ptb.py"],
     srcs_version = "PY2AND3",
+    deps = [":rnn_ptb_lib"],
+)
+
+py_library(
+    name = "rnn_ptb_lib",
+    srcs = ["rnn_ptb.py"],
+    srcs_version = "PY2AND3",
     deps = [
         "//tensorflow:tensorflow_py",
         "//tensorflow/contrib/cudnn_rnn:cudnn_rnn_py",
diff --git a/tensorflow/contrib/rnn/BUILD b/tensorflow/contrib/rnn/BUILD
index 39b6885968..d65d80df80 100644
--- a/tensorflow/contrib/rnn/BUILD
+++ b/tensorflow/contrib/rnn/BUILD
@@ -388,6 +388,13 @@ py_binary(
     name = "checkpoint_convert",
     srcs = ["python/tools/checkpoint_convert.py"],
     srcs_version = "PY2AND3",
+    deps = [":checkpoint_convert_lib"],
+)
+
+py_library(
+    name = "checkpoint_convert_lib",
+    srcs = ["python/tools/checkpoint_convert.py"],
+    srcs_version = "PY2AND3",
     deps = [
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:framework_ops",
@@ -406,7 +413,7 @@ py_test(
     srcs_version = "PY2AND3",
     tags = ["no_pip"],
     deps = [
-        ":checkpoint_convert",
+        ":checkpoint_convert_lib",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:session",
-- 
GitLab


From 701ad58880ae7b4d317d0bda5f1497346273b2af Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 17:32:56 -0800
Subject: [PATCH 1092/2345] Merged commit includes the following changes:
 230445097  by A. Unique TensorFlower<gardener@tensorflow.org>:     Automated
 rollback of changelist 230438242.

230444163  by A. Unique TensorFlower<gardener@tensorflow.org>:

    Add missing right parenthesis in tf.Session doc.

--
230443190  by A. Unique TensorFlower<gardener@tensorflow.org>:

    Adding a capture_resource_var_by_value argument to Defun in which resource variable could be captured as var_handle instead of var value. This is useful for TPU inference where we don't want copy variable from CPU to TPU.

--
230442585  by A. Unique TensorFlower<gardener@tensorflow.org>:

    Update docstring to reflect MultiDeviceIterator now supported
    in eager execution.

--
230441663  by A. Unique TensorFlower<gardener@tensorflow.org>:

    Internal changes.

--
230439505  by A. Unique TensorFlower<gardener@tensorflow.org>:

    Interpreter conversion of new schema quantization to new CAPI quantization

--
230438242  by A. Unique TensorFlower<gardener@tensorflow.org>:

    n/a

--

PiperOrigin-RevId: 230445097
---
 tensorflow/lite/c/c_api_internal.c            |  12 +-
 tensorflow/lite/c/c_api_internal.h            |   3 +
 tensorflow/lite/core/subgraph.cc              |  52 ++-
 tensorflow/lite/core/subgraph.h               |  18 +-
 tensorflow/lite/interpreter.cc                |  63 ++-
 tensorflow/lite/interpreter.h                 |  13 +
 tensorflow/lite/interpreter_test.cc           |  52 ++-
 tensorflow/lite/model.cc                      |  86 ++--
 tensorflow/lite/model.h                       |   3 +
 tensorflow/lite/string_util_test.cc           |   5 +-
 tensorflow/opensource_only.files              | 418 +++++++++---------
 tensorflow/python/client/session.py           |   4 +-
 .../data/ops/multi_device_iterator_ops.py     |   8 +-
 tensorflow/python/framework/function.py       |  23 +-
 14 files changed, 474 insertions(+), 286 deletions(-)

diff --git a/tensorflow/lite/c/c_api_internal.c b/tensorflow/lite/c/c_api_internal.c
index 29dba15c63..f20ee23bd8 100644
--- a/tensorflow/lite/c/c_api_internal.c
+++ b/tensorflow/lite/c/c_api_internal.c
@@ -91,10 +91,10 @@ void TfLiteTensorDataFree(TfLiteTensor* t) {
   t->data.raw = NULL;
 }
 
-void TfLiteQuantizationFree(TfLiteTensor* t) {
-  if (t->quantization.type == kTfLiteAffineQuantization) {
+void TfLiteQuantizationFree(TfLiteQuantization* quantization) {
+  if (quantization->type == kTfLiteAffineQuantization) {
     TfLiteAffineQuantization* q_params =
-        (TfLiteAffineQuantization*)(t->quantization.params);
+        (TfLiteAffineQuantization*)(quantization->params);
     if (q_params->scale) {
       TfLiteFloatArrayFree(q_params->scale);
       q_params->scale = NULL;
@@ -105,8 +105,8 @@ void TfLiteQuantizationFree(TfLiteTensor* t) {
     }
     free(q_params);
   }
-  t->quantization.params = NULL;
-  t->quantization.type = kTfLiteNoQuantization;
+  quantization->params = NULL;
+  quantization->type = kTfLiteNoQuantization;
 }
 
 void TfLiteTensorFree(TfLiteTensor* t) {
@@ -114,7 +114,7 @@ void TfLiteTensorFree(TfLiteTensor* t) {
   if (t->dims) TfLiteIntArrayFree(t->dims);
   t->dims = NULL;
 
-  TfLiteQuantizationFree(t);
+  TfLiteQuantizationFree(&t->quantization);
 }
 
 void TfLiteTensorReset(TfLiteType type, const char* name, TfLiteIntArray* dims,
diff --git a/tensorflow/lite/c/c_api_internal.h b/tensorflow/lite/c/c_api_internal.h
index 31f483370c..83e2be6907 100644
--- a/tensorflow/lite/c/c_api_internal.h
+++ b/tensorflow/lite/c/c_api_internal.h
@@ -340,6 +340,9 @@ typedef struct {
 // Free data memory of tensor `t`.
 void TfLiteTensorDataFree(TfLiteTensor* t);
 
+// Free quantization data.
+void TfLiteQuantizationFree(TfLiteQuantization* quantization);
+
 // Free memory of tensor `t`.
 void TfLiteTensorFree(TfLiteTensor* t);
 
diff --git a/tensorflow/lite/core/subgraph.cc b/tensorflow/lite/core/subgraph.cc
index 4be80d143e..ab83456229 100644
--- a/tensorflow/lite/core/subgraph.cc
+++ b/tensorflow/lite/core/subgraph.cc
@@ -72,6 +72,34 @@ bool HasDynamicTensor(const TfLiteContext& context,
   return HasDynamicTensorImpl(context, TfLiteIntArrayView{int_array});
 }
 
+// Gets the legacy TfLiteQuantizationParams from the current TfLiteQuantization.
+TfLiteQuantizationParams GetLegacyQuantization(
+    const TfLiteQuantization& quantization) {
+  TfLiteQuantizationParams legacy_quantization;
+  legacy_quantization.scale = 0;
+  legacy_quantization.zero_point = 0;
+
+  // If the quantization type isn't affine, return the empty
+  // legacy_quantization.
+  if (quantization.type != kTfLiteAffineQuantization) {
+    return legacy_quantization;
+  }
+
+  auto* affine_quantization =
+      reinterpret_cast<TfLiteAffineQuantization*>(quantization.params);
+  if (!affine_quantization || !affine_quantization->scale ||
+      !affine_quantization->zero_point ||
+      affine_quantization->scale->size != 1 ||
+      affine_quantization->zero_point->size != 1) {
+    return legacy_quantization;
+  }
+
+  // We know its per-layer quantization now.
+  legacy_quantization.scale = affine_quantization->scale->data[0];
+  legacy_quantization.zero_point = affine_quantization->zero_point->data[0];
+  return legacy_quantization;
+}
+
 }  // namespace
 
 // A trivial implementation of GraphInfo around the Interpreter.
@@ -779,7 +807,7 @@ TfLiteStatus Subgraph::GetNodeAndRegistration(
 
 TfLiteStatus Subgraph::SetTensorParametersReadOnly(
     int tensor_index, TfLiteType type, const char* name, const size_t rank,
-    const int* dims, TfLiteQuantizationParams quantization, const char* buffer,
+    const int* dims, TfLiteQuantization quantization, const char* buffer,
     size_t bytes, const Allocation* allocation) {
   if (state_ == kStateInvokableAndImmutable) {
     ReportError(
@@ -804,16 +832,22 @@ TfLiteStatus Subgraph::SetTensorParametersReadOnly(
       EqualArrayAndTfLiteIntArray(tensor.dims, rank, dims)) {
     // Fast path which does not invalidate the invokable property.
     TfLiteTensorDataFree(&tensor);
+    TfLiteQuantizationFree(&tensor.quantization);
     tensor.data.raw = const_cast<char*>(buffer);
     if (!tensor.dims) tensor.dims = ConvertArrayToTfLiteIntArray(rank, dims);
-    tensor.params = quantization;
+    tensor.params = GetLegacyQuantization(quantization);
+    tensor.quantization = quantization;
     tensor.allocation_type = kTfLiteMmapRo;
     tensor.allocation = allocation;
   } else {
     state_ = kStateUninvokable;
     TfLiteTensorReset(type, name, ConvertArrayToTfLiteIntArray(rank, dims),
-                      quantization, const_cast<char*>(buffer), bytes,
-                      kTfLiteMmapRo, allocation, false, &tensor);
+                      GetLegacyQuantization(quantization),
+                      const_cast<char*>(buffer), bytes, kTfLiteMmapRo,
+                      allocation, false, &tensor);
+    // TODO(suharshs): Update TfLiteTensorReset to include the new quantization
+    // if there are other required callers.
+    tensor.quantization = quantization;
   }
   return kTfLiteOk;
 }
@@ -824,7 +858,7 @@ TfLiteStatus Subgraph::SetTensorParametersReadOnly(
 // to Interpreter.
 TfLiteStatus Subgraph::SetTensorParametersReadWrite(
     int tensor_index, TfLiteType type, const char* name, const size_t rank,
-    const int* dims, TfLiteQuantizationParams quantization, bool is_variable) {
+    const int* dims, TfLiteQuantization quantization, bool is_variable) {
   if (state_ == kStateInvokableAndImmutable) {
     ReportError(
         "SetTensorParametersReadWrite is disallowed when graph is immutable.");
@@ -854,10 +888,14 @@ TfLiteStatus Subgraph::SetTensorParametersReadWrite(
     allocation_type = kTfLiteArenaRwPersistent;
   }
 
+  TfLiteTensor& tensor = context_->tensors[tensor_index];
   TfLiteTensorReset(type, name, ConvertArrayToTfLiteIntArray(rank, dims),
-                    quantization,
+                    GetLegacyQuantization(quantization),
                     /*buffer=*/nullptr, required_bytes, allocation_type,
-                    nullptr, is_variable, &context_->tensors[tensor_index]);
+                    nullptr, is_variable, &tensor);
+  // TODO(suharshs): Update TfLiteTensorReset to include the new quantization
+  // if there are other required callers.
+  tensor.quantization = quantization;
   return kTfLiteOk;
 }
 
diff --git a/tensorflow/lite/core/subgraph.h b/tensorflow/lite/core/subgraph.h
index 0bec218d89..5ca2977e0c 100644
--- a/tensorflow/lite/core/subgraph.h
+++ b/tensorflow/lite/core/subgraph.h
@@ -79,18 +79,22 @@ class Subgraph {
   // This variant assumes an external buffer has been allocated of size
   // bytes. The lifetime of buffer must be ensured to be greater or equal
   // to Interpreter.
-  TfLiteStatus SetTensorParametersReadOnly(
-      int tensor_index, TfLiteType type, const char* name, const size_t rank,
-      const int* dims, TfLiteQuantizationParams quantization,
-      const char* buffer, size_t bytes, const Allocation* allocation);
+  TfLiteStatus SetTensorParametersReadOnly(int tensor_index, TfLiteType type,
+                                           const char* name, const size_t rank,
+                                           const int* dims,
+                                           TfLiteQuantization quantization,
+                                           const char* buffer, size_t bytes,
+                                           const Allocation* allocation);
 
   // Set description of inputs/outputs/data/fptrs for node `node_index`.
   // This variant assumes an external buffer has been allocated of size
   // bytes. The lifetime of buffer must be ensured to be greater or equal
   // to Interpreter.
-  TfLiteStatus SetTensorParametersReadWrite(
-      int tensor_index, TfLiteType type, const char* name, const size_t rank,
-      const int* dims, TfLiteQuantizationParams quantization, bool is_variable);
+  TfLiteStatus SetTensorParametersReadWrite(int tensor_index, TfLiteType type,
+                                            const char* name, const size_t rank,
+                                            const int* dims,
+                                            TfLiteQuantization quantization,
+                                            bool is_variable);
 
   // WARNING: Experimental interface, subject to change
   // Overrides execution plan. This bounds checks indices sent in.
diff --git a/tensorflow/lite/interpreter.cc b/tensorflow/lite/interpreter.cc
index 60fa2130fa..7a6a074ded 100644
--- a/tensorflow/lite/interpreter.cc
+++ b/tensorflow/lite/interpreter.cc
@@ -32,6 +32,26 @@ limitations under the License.
 
 namespace tflite {
 
+namespace {
+
+// Gets the current TfLiteQuantization from the legacy fLiteQuantizationParams.
+TfLiteQuantization GetQuantizationFromLegacy(
+    const TfLiteQuantizationParams& legacy_quantization) {
+  TfLiteQuantization quantization;
+  quantization.type = kTfLiteAffineQuantization;
+  auto* affine_quantization = reinterpret_cast<TfLiteAffineQuantization*>(
+      malloc(sizeof(TfLiteAffineQuantization)));
+  affine_quantization->scale = TfLiteFloatArrayCreate(1);
+  affine_quantization->zero_point = TfLiteIntArrayCreate(1);
+  affine_quantization->scale->data[0] = legacy_quantization.scale;
+  affine_quantization->zero_point->data[0] = legacy_quantization.zero_point;
+  quantization.params = affine_quantization;
+
+  return quantization;
+}
+
+}  // namespace
+
 Interpreter::Interpreter(ErrorReporter* error_reporter)
     : error_reporter_(error_reporter ? error_reporter
                                      : DefaultErrorReporter()) {
@@ -123,24 +143,49 @@ TfLiteStatus Interpreter::ResetVariableTensors() {
   return primary_subgraph().ResetVariableTensors();
 }
 
+TfLiteStatus Interpreter::SetTensorParametersReadOnly(
+    int tensor_index, TfLiteType type, const char* name,
+    const std::vector<int>& dims, TfLiteQuantization quantization,
+    const char* buffer, size_t bytes, const Allocation* allocation) {
+  return primary_subgraph().SetTensorParametersReadOnly(
+      tensor_index, type, name, dims.size(), dims.data(), quantization, buffer,
+      bytes, allocation);
+}
+
+TfLiteStatus Interpreter::SetTensorParametersReadWrite(
+    int tensor_index, TfLiteType type, const char* name,
+    const std::vector<int>& dims, TfLiteQuantization quantization,
+    bool is_variable) {
+  return primary_subgraph().SetTensorParametersReadWrite(
+      tensor_index, type, name, dims.size(), dims.data(), quantization,
+      is_variable);
+}
+
 TfLiteStatus Interpreter::SetTensorParametersReadOnly(
     int tensor_index, TfLiteType type, const char* name, const size_t rank,
     const int* dims, TfLiteQuantizationParams quantization, const char* buffer,
     size_t bytes, const Allocation* allocation) {
-  return primary_subgraph().SetTensorParametersReadOnly(
-      tensor_index, type, name, rank, dims, quantization, buffer, bytes,
-      allocation);
+  TfLiteQuantization new_quantization = GetQuantizationFromLegacy(quantization);
+  if (primary_subgraph().SetTensorParametersReadOnly(
+          tensor_index, type, name, rank, dims, new_quantization, buffer, bytes,
+          allocation) != kTfLiteOk) {
+    TfLiteQuantizationFree(&new_quantization);
+    return kTfLiteError;
+  }
+  return kTfLiteOk;
 }
 
-// Set description of inputs/outputs/data/fptrs for node `node_index`.
-// This variant assumes an external buffer has been allocated of size
-// bytes. The lifetime of buffer must be ensured to be greater or equal
-// to Interpreter.
 TfLiteStatus Interpreter::SetTensorParametersReadWrite(
     int tensor_index, TfLiteType type, const char* name, const size_t rank,
     const int* dims, TfLiteQuantizationParams quantization, bool is_variable) {
-  return primary_subgraph().SetTensorParametersReadWrite(
-      tensor_index, type, name, rank, dims, quantization, is_variable);
+  TfLiteQuantization new_quantization = GetQuantizationFromLegacy(quantization);
+  if (primary_subgraph().SetTensorParametersReadWrite(
+          tensor_index, type, name, rank, dims, new_quantization,
+          is_variable) != kTfLiteOk) {
+    TfLiteQuantizationFree(&new_quantization);
+    return kTfLiteError;
+  }
+  return kTfLiteOk;
 }
 
 TfLiteStatus Interpreter::SetExecutionPlan(const std::vector<int>& new_plan) {
diff --git a/tensorflow/lite/interpreter.h b/tensorflow/lite/interpreter.h
index 754439c966..806b66c12a 100644
--- a/tensorflow/lite/interpreter.h
+++ b/tensorflow/lite/interpreter.h
@@ -160,6 +160,12 @@ class Interpreter {
   // This variant assumes an external buffer has been allocated of size
   // bytes. The lifetime of buffer must be ensured to be greater or equal
   // to Interpreter.
+  TfLiteStatus SetTensorParametersReadOnly(
+      int tensor_index, TfLiteType type, const char* name,
+      const std::vector<int>& dims, TfLiteQuantization quantization,
+      const char* buffer, size_t bytes, const Allocation* allocation = nullptr);
+
+  // Legacy. Deprecated in favor of above.
   inline TfLiteStatus SetTensorParametersReadOnly(
       int tensor_index, TfLiteType type, const char* name,
       const std::vector<int>& dims, TfLiteQuantizationParams quantization,
@@ -179,6 +185,13 @@ class Interpreter {
   // This variant assumes an external buffer has been allocated of size
   // bytes. The lifetime of buffer must be ensured to be greater or equal
   // to Interpreter.
+  TfLiteStatus SetTensorParametersReadWrite(int tensor_index, TfLiteType type,
+                                            const char* name,
+                                            const std::vector<int>& dims,
+                                            TfLiteQuantization quantization,
+                                            bool is_variable = false);
+
+  // Legacy. Deprecated in favor of above.
   inline TfLiteStatus SetTensorParametersReadWrite(
       int tensor_index, TfLiteType type, const char* name,
       const std::vector<int>& dims, TfLiteQuantizationParams quantization,
diff --git a/tensorflow/lite/interpreter_test.cc b/tensorflow/lite/interpreter_test.cc
index 1ee993c6f9..e1aedfe65c 100644
--- a/tensorflow/lite/interpreter_test.cc
+++ b/tensorflow/lite/interpreter_test.cc
@@ -73,8 +73,9 @@ TEST(BasicInterpreter, TestAllocateTensorsResetVariableTensors) {
   int tensor_index;
   ASSERT_EQ(interpreter.AddTensors(1, &tensor_index), kTfLiteOk);
   constexpr int kTensorSize = 16;
+  TfLiteQuantizationParams quant;
   interpreter.SetTensorParametersReadWrite(tensor_index, kTfLiteFloat32, "",
-                                           {kTensorSize}, {}, true);
+                                           {kTensorSize}, quant, true);
   interpreter.SetVariables({tensor_index});
   ASSERT_EQ(interpreter.AllocateTensors(), kTfLiteOk);
   TfLiteTensor* tensor = interpreter.tensor(tensor_index);
@@ -170,6 +171,55 @@ TEST(BasicInterpreter, CheckAllocate) {
   }
 }
 
+TEST(BasicInterpreter, CheckQuantization) {
+  Interpreter interpreter;
+  ASSERT_EQ(interpreter.AddTensors(2), kTfLiteOk);
+  interpreter.SetInputs({0, 1});
+  interpreter.SetOutputs({});
+  TfLiteType tensor_type = kTfLiteInt8;
+  const uint8_t int8s[] = {3, 4};
+  float scale = 0.5f;
+  int32_t zero_point = 12;
+
+  TfLiteQuantization rw_quantization;
+  rw_quantization.type = kTfLiteAffineQuantization;
+  auto* rw_affine_quantization = reinterpret_cast<TfLiteAffineQuantization*>(
+      malloc(sizeof(TfLiteAffineQuantization)));
+  rw_affine_quantization->scale = TfLiteFloatArrayCreate(1);
+  rw_affine_quantization->zero_point = TfLiteIntArrayCreate(1);
+  rw_affine_quantization->scale->data[0] = scale;
+  rw_affine_quantization->zero_point->data[0] = zero_point;
+  rw_quantization.params = rw_affine_quantization;
+
+  TfLiteQuantization ro_quantization;
+  ro_quantization.type = kTfLiteAffineQuantization;
+  auto* ro_affine_quantization = reinterpret_cast<TfLiteAffineQuantization*>(
+      malloc(sizeof(TfLiteAffineQuantization)));
+  ro_affine_quantization->scale = TfLiteFloatArrayCreate(1);
+  ro_affine_quantization->zero_point = TfLiteIntArrayCreate(1);
+  ro_affine_quantization->scale->data[0] = scale;
+  ro_affine_quantization->zero_point->data[0] = zero_point;
+  ro_quantization.params = ro_affine_quantization;
+
+  ASSERT_EQ(interpreter.SetTensorParametersReadWrite(0, tensor_type, "", {3},
+                                                     rw_quantization),
+            kTfLiteOk);
+  ASSERT_EQ(interpreter.SetTensorParametersReadOnly(
+                1, tensor_type, "", {2}, ro_quantization,
+                reinterpret_cast<const char*>(int8s), 2),
+            kTfLiteOk);
+  ASSERT_EQ(interpreter.AllocateTensors(), kTfLiteOk);
+  // Check that the legacy scale and zero_point are set correctly.
+  ASSERT_EQ(interpreter.tensor(0)->params.scale, scale);
+  ASSERT_EQ(interpreter.tensor(0)->params.zero_point, zero_point);
+  ASSERT_EQ(interpreter.tensor(0)->quantization.type, rw_quantization.type);
+  ASSERT_EQ(interpreter.tensor(0)->quantization.type, rw_quantization.type);
+  ASSERT_EQ(interpreter.tensor(1)->params.scale, scale);
+  ASSERT_EQ(interpreter.tensor(1)->params.zero_point, zero_point);
+  ASSERT_EQ(interpreter.tensor(1)->quantization.type, ro_quantization.type);
+  ASSERT_EQ(interpreter.tensor(1)->quantization.type, ro_quantization.type);
+}
+
 TEST(BasicInterpreter, CheckResize) {
   const float floats[] = {-3., -4.};
   const int32_t int32s[] = {-3, -4};
diff --git a/tensorflow/lite/model.cc b/tensorflow/lite/model.cc
index 663ee38280..c736685a98 100644
--- a/tensorflow/lite/model.cc
+++ b/tensorflow/lite/model.cc
@@ -21,6 +21,7 @@ limitations under the License.
 
 #include "tensorflow/lite/allocation.h"
 #include "tensorflow/lite/c/builtin_op_data.h"
+#include "tensorflow/lite/c/c_api_internal.h"
 #include "tensorflow/lite/core/api/error_reporter.h"
 #include "tensorflow/lite/core/api/flatbuffer_conversions.h"
 #include "tensorflow/lite/model.h"
@@ -299,6 +300,56 @@ TfLiteStatus InterpreterBuilder::ParseNodes(
   return status;
 }
 
+TfLiteStatus InterpreterBuilder::ParseQuantization(
+    const QuantizationParameters* src_quantization,
+    TfLiteQuantization* quantization) {
+  quantization->type = kTfLiteNoQuantization;
+  if (!src_quantization || !src_quantization->scale() ||
+      src_quantization->scale()->size() == 0) {
+    return kTfLiteOk;
+  }
+  if (!src_quantization->zero_point()) {
+    error_reporter_->Report(
+        "Quantization parameters has non-null scale but null zero_point.");
+    return kTfLiteError;
+  }
+
+  // Ensure that the number of scales matches the number of zero_points.
+  if (src_quantization->scale()->size() !=
+      src_quantization->zero_point()->size()) {
+    error_reporter_->Report(
+        "QuantizationParam has %d zero_point values and %d scale values. Must "
+        "have same number.",
+        src_quantization->zero_point()->size(),
+        src_quantization->scale()->size());
+    return kTfLiteError;
+  }
+
+  // Affine-quantization.
+  quantization->type = kTfLiteAffineQuantization;
+  auto* affine_quantization = reinterpret_cast<TfLiteAffineQuantization*>(
+      malloc(sizeof(TfLiteAffineQuantization)));
+  const size_t num_scales = src_quantization->scale()->size();
+  affine_quantization->scale = TfLiteFloatArrayCreate(num_scales);
+  affine_quantization->zero_point = TfLiteIntArrayCreate(num_scales);
+  for (size_t i = 0; i < num_scales; ++i) {
+    affine_quantization->scale->data[i] = src_quantization->scale()->Get(i);
+    affine_quantization->zero_point->data[i] =
+        src_quantization->zero_point()->Get(i);
+  }
+  if (src_quantization->quantized_dimension() < 0 ||
+      src_quantization->quantized_dimension() >= num_scales) {
+    error_reporter_->Report(
+        "quantized_dimension must be in range [0, %d). Was %d.", num_scales,
+        src_quantization->quantized_dimension());
+    return kTfLiteError;
+  }
+  affine_quantization->quantized_dimension =
+      src_quantization->quantized_dimension();
+  quantization->params = reinterpret_cast<void*>(affine_quantization);
+  return kTfLiteOk;
+}
+
 TfLiteStatus InterpreterBuilder::ParseTensors(
     const flatbuffers::Vector<flatbuffers::Offset<Buffer>>* buffers,
     const flatbuffers::Vector<flatbuffers::Offset<Tensor>>* tensors,
@@ -317,36 +368,11 @@ TfLiteStatus InterpreterBuilder::ParseTensors(
     const auto* tensor = tensors->Get(i);
     std::vector<int> dims = FlatBufferIntArrayToVector(tensor->shape());
 
-    TfLiteQuantizationParams quantization;
-    quantization.scale = 0;
-    quantization.zero_point = 0;
-    auto* q_params = tensor->quantization();
-    if (q_params) {
-      // Note that the schema could hold per-channel quantization parameters
-      // but we really only support one value for the whole tensor.
-      // TODO(aselle): This breaks as well if these are nullptr's.
-      // TODO(aselle): This assumes non per-channel quantization.
-
-      if (q_params->scale()) {
-        if (q_params->scale()->size() != 1) {
-          error_reporter_->Report(
-              "QuantizationParam has %d scale values (only 1 is supported).",
-              q_params->scale()->size());
-          return kTfLiteError;
-        }
-        quantization.scale = q_params->scale()->Get(0);
-      }
-
-      if (q_params->zero_point()) {
-        if (q_params->zero_point()->size() != 1) {
-          error_reporter_->Report(
-              "QuantizationParam has %d zero_point values"
-              " (only 1 is supported).",
-              q_params->zero_point()->size());
-          return kTfLiteError;
-        }
-        quantization.zero_point = q_params->zero_point()->Get(0);
-      }
+    const auto* src_quantization = tensor->quantization();
+    TfLiteQuantization quantization;
+    if (ParseQuantization(src_quantization, &quantization) != kTfLiteOk) {
+      status = kTfLiteError;
+      continue;
     }
 
     TfLiteType type;
diff --git a/tensorflow/lite/model.h b/tensorflow/lite/model.h
index 069cefabf9..a9bd4c9c09 100644
--- a/tensorflow/lite/model.h
+++ b/tensorflow/lite/model.h
@@ -35,6 +35,7 @@ limitations under the License.
 #define TENSORFLOW_LITE_MODEL_H_
 
 #include <memory>
+#include "tensorflow/lite/c/c_api_internal.h"
 #include "tensorflow/lite/core/api/error_reporter.h"
 #include "tensorflow/lite/core/api/op_resolver.h"
 #include "tensorflow/lite/interpreter.h"
@@ -203,6 +204,8 @@ class InterpreterBuilder {
       const flatbuffers::Vector<flatbuffers::Offset<Tensor>>* tensors,
       Interpreter* interpreter);
   TfLiteStatus ApplyDelegates(Interpreter* interpreter);
+  TfLiteStatus ParseQuantization(const QuantizationParameters* src_quantization,
+                                 TfLiteQuantization* quantization);
 
   const ::tflite::Model* model_;
   const OpResolver& op_resolver_;
diff --git a/tensorflow/lite/string_util_test.cc b/tensorflow/lite/string_util_test.cc
index cbf1d7b226..90ae553cab 100644
--- a/tensorflow/lite/string_util_test.cc
+++ b/tensorflow/lite/string_util_test.cc
@@ -35,8 +35,9 @@ TEST(StringUtil, TestStringUtil) {
 
   char data[] = {1, 0, 0, 0, 12, 0, 0, 0, 15, 0, 0, 0, 'X', 'Y', 'Z'};
 
-  interpreter.SetTensorParametersReadOnly(2, kTfLiteString, "", {1}, {}, data,
-                                          15);
+  TfLiteQuantization quant;
+  interpreter.SetTensorParametersReadOnly(2, kTfLiteString, "", {1}, quant,
+                                          data, 15);
   TfLiteTensor* t2 = interpreter.tensor(2);
   interpreter.AllocateTensors();
 
diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index c546a34cf8..a5e51d7276 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -1,248 +1,248 @@
-tensorflow/contrib/tpu/profiler/pip_package/BUILD
-tensorflow/contrib/tpu/profiler/pip_package/setup.py
-tensorflow/contrib/tpu/profiler/pip_package/README
-tensorflow/contrib/tpu/profiler/pip_package/build_pip_package.sh
-tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/main.py
-tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/__init__.py
-tensorflow/contrib/mpi/BUILD
-tensorflow/tools/ci_build/remote/BUILD
-tensorflow/tools/pip_package/README
-tensorflow/tools/pip_package/MANIFEST.in
-tensorflow/tools/pip_package/simple_console.py
-tensorflow/tools/pip_package/build_pip_package.sh
-tensorflow/tools/pip_package/check_load_py_test.py
-tensorflow/tools/pip_package/pip_smoke_test.py
-tensorflow/tools/pip_package/simple_console_for_windows.py
-tensorflow/tools/pip_package/setup.py
-tensorflow/tools/pip_package/BUILD
-tensorflow/tools/lib_package/concat_licenses.sh
-tensorflow/tools/lib_package/libtensorflow_test.c
-tensorflow/tools/lib_package/LibTensorFlowTest.java
-tensorflow/tools/lib_package/BUILD
-tensorflow/tools/lib_package/libtensorflow_test.sh
-tensorflow/tools/lib_package/README.md
-tensorflow/tools/lib_package/libtensorflow_java_test.sh
-tensorflow/tools/def_file_filter/def_file_filter_configure.bzl
-tensorflow/tools/def_file_filter/BUILD
-tensorflow/tools/def_file_filter/BUILD.tpl
-tensorflow/tools/def_file_filter/def_file_filter.py.tpl
-tensorflow/third_party/mkl/MKL_LICENSE
-tensorflow/third_party/mkl/LICENSE
-tensorflow/third_party/mkl/BUILD
-tensorflow/third_party/mkl/mkl.BUILD
-tensorflow/third_party/mkl/build_defs.bzl
-tensorflow/third_party/backports_weakref.BUILD
-tensorflow/third_party/toolchains/clang6/BUILD
+tensorflow/api_template_v1.__init__.py
+tensorflow/compat_template_v1.__init__.py
+tensorflow/stream_executor/build_defs.bzl
+tensorflow/third_party/curl.BUILD
+tensorflow/third_party/snappy.BUILD
+tensorflow/third_party/tflite_mobilenet_float.BUILD
+tensorflow/third_party/libxsmm.BUILD
+tensorflow/third_party/cub.BUILD
+tensorflow/third_party/repo.bzl
+tensorflow/third_party/systemlibs/curl.BUILD
+tensorflow/third_party/systemlibs/snappy.BUILD
+tensorflow/third_party/systemlibs/protobuf.BUILD
+tensorflow/third_party/systemlibs/absl_py.BUILD
+tensorflow/third_party/systemlibs/google_cloud_cpp.google.cloud.bigtable.BUILD
+tensorflow/third_party/systemlibs/astor.BUILD
+tensorflow/third_party/systemlibs/absl_py.absl.flags.BUILD
+tensorflow/third_party/systemlibs/png.BUILD
+tensorflow/third_party/systemlibs/enum34.BUILD
+tensorflow/third_party/systemlibs/BUILD.tpl
+tensorflow/third_party/systemlibs/boringssl.BUILD
+tensorflow/third_party/systemlibs/absl_py.absl.testing.BUILD
+tensorflow/third_party/systemlibs/swig.BUILD
+tensorflow/third_party/systemlibs/double_conversion.BUILD
+tensorflow/third_party/systemlibs/gif.BUILD
+tensorflow/third_party/systemlibs/cython.BUILD
+tensorflow/third_party/systemlibs/google_cloud_cpp.BUILD
+tensorflow/third_party/systemlibs/zlib.BUILD
+tensorflow/third_party/systemlibs/grpc.BUILD
+tensorflow/third_party/systemlibs/re2.BUILD
+tensorflow/third_party/systemlibs/build_defs.bzl.tpl
+tensorflow/third_party/systemlibs/protobuf.bzl
+tensorflow/third_party/systemlibs/gast.BUILD
+tensorflow/third_party/systemlibs/syslibs_configure.bzl
+tensorflow/third_party/systemlibs/nsync.BUILD
+tensorflow/third_party/systemlibs/jsoncpp.BUILD
+tensorflow/third_party/systemlibs/six.BUILD
+tensorflow/third_party/systemlibs/lmdb.BUILD
+tensorflow/third_party/systemlibs/BUILD
+tensorflow/third_party/systemlibs/googleapis.BUILD
+tensorflow/third_party/systemlibs/sqlite.BUILD
+tensorflow/third_party/systemlibs/termcolor.BUILD
+tensorflow/third_party/systemlibs/pcre.BUILD
+tensorflow/third_party/tflite_mobilenet_quant.BUILD
+tensorflow/third_party/common.bzl
+tensorflow/third_party/boringssl/BUILD
+tensorflow/third_party/kafka/config.patch
+tensorflow/third_party/kafka/BUILD
+tensorflow/third_party/android/android_configure.bzl
+tensorflow/third_party/android/android_configure.BUILD.tpl
+tensorflow/third_party/android/android.bzl.tpl
+tensorflow/third_party/android/BUILD
+tensorflow/third_party/astor.BUILD
+tensorflow/third_party/icu/udata.patch
+tensorflow/third_party/tflite_mobilenet.BUILD
+tensorflow/third_party/png.BUILD
+tensorflow/third_party/mpi/.gitignore
+tensorflow/third_party/mpi/BUILD
+tensorflow/third_party/nanopb.BUILD
+tensorflow/third_party/arm_neon_2_x86_sse.BUILD
+tensorflow/third_party/linenoise.BUILD
+tensorflow/third_party/llvm/llvm.autogenerated.BUILD
+tensorflow/third_party/llvm/llvm.bzl
+tensorflow/third_party/llvm/BUILD
+tensorflow/third_party/llvm/expand_cmake_vars.py
+tensorflow/third_party/swig.BUILD
+tensorflow/third_party/png_fix_rpi.patch
+tensorflow/third_party/double_conversion.BUILD
+tensorflow/third_party/gif.BUILD
+tensorflow/third_party/grpc/BUILD
+tensorflow/third_party/cython.BUILD
+tensorflow/third_party/eigen.BUILD
+tensorflow/third_party/codegen.BUILD
+tensorflow/third_party/zlib.BUILD
+tensorflow/third_party/tensorrt/BUILD.tpl
+tensorflow/third_party/tensorrt/tensorrt_configure.bzl
+tensorflow/third_party/tensorrt/build_defs.bzl.tpl
+tensorflow/third_party/tensorrt/LICENSE
+tensorflow/third_party/tensorrt/BUILD
+tensorflow/third_party/tflite_ovic_testdata.BUILD
+tensorflow/third_party/fft2d/fft.h
+tensorflow/third_party/fft2d/fft2d.BUILD
+tensorflow/third_party/fft2d/LICENSE
+tensorflow/third_party/fft2d/BUILD
 tensorflow/third_party/toolchains/clang6/README.md
 tensorflow/third_party/toolchains/clang6/repo.bzl
-tensorflow/third_party/toolchains/clang6/CROSSTOOL.tpl
 tensorflow/third_party/toolchains/clang6/clang.BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/BUILD
+tensorflow/third_party/toolchains/clang6/CROSSTOOL.tpl
+tensorflow/third_party/toolchains/clang6/BUILD
+tensorflow/third_party/toolchains/remote/BUILD.tpl
+tensorflow/third_party/toolchains/remote/configure.bzl
+tensorflow/third_party/toolchains/remote/execution.bzl.tpl
+tensorflow/third_party/toolchains/remote/BUILD
+tensorflow/third_party/toolchains/preconfig/generate/archives.bzl
+tensorflow/third_party/toolchains/preconfig/generate/workspace.bzl
+tensorflow/third_party/toolchains/preconfig/generate/generate.bzl
+tensorflow/third_party/toolchains/preconfig/generate/containers.bzl
+tensorflow/third_party/toolchains/preconfig/generate/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/build_defs.bzl
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/py3/BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/build_defs.bzl
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/build_defs.bzl
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/build_defs.bzl
-tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD
-tensorflow/third_party/toolchains/preconfig/generate/workspace.bzl
-tensorflow/third_party/toolchains/preconfig/generate/containers.bzl
-tensorflow/third_party/toolchains/preconfig/generate/generate.bzl
-tensorflow/third_party/toolchains/preconfig/generate/archives.bzl
-tensorflow/third_party/toolchains/preconfig/generate/BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu16.04/clang/BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu16.04/clang/dummy_toolchain.bzl
-tensorflow/third_party/toolchains/preconfig/ubuntu16.04/py3/BUILD
-tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/BUILD
-tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/dummy_toolchain.bzl
 tensorflow/third_party/toolchains/preconfig/win_1803/py36/BUILD
 tensorflow/third_party/toolchains/preconfig/win_1803/BUILD
-tensorflow/third_party/toolchains/gpus/cuda/build_defs.bzl
-tensorflow/third_party/toolchains/gpus/cuda/BUILD
-tensorflow/third_party/toolchains/gpus/cuda/cuda/cuda_config.h
-tensorflow/third_party/toolchains/gpus/crosstool/BUILD
-tensorflow/third_party/toolchains/gpus/crosstool/CROSSTOOL
-tensorflow/third_party/toolchains/gpus/py/BUILD
-tensorflow/third_party/toolchains/cpus/arm/arm_compiler_configure.bzl
+tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/dummy_toolchain.bzl
+tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu16.04/py3/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu16.04/clang/dummy_toolchain.bzl
+tensorflow/third_party/toolchains/preconfig/ubuntu16.04/clang/BUILD
+tensorflow/third_party/toolchains/cpus/py3/BUILD
 tensorflow/third_party/toolchains/cpus/arm/CROSSTOOL.tpl
 tensorflow/third_party/toolchains/cpus/arm/BUILD
-tensorflow/third_party/toolchains/cpus/py3/BUILD
+tensorflow/third_party/toolchains/cpus/arm/arm_compiler_configure.bzl
 tensorflow/third_party/toolchains/cpus/py/BUILD
-tensorflow/third_party/toolchains/remote/configure.bzl
-tensorflow/third_party/toolchains/remote/BUILD.tpl
-tensorflow/third_party/toolchains/remote/BUILD
-tensorflow/third_party/toolchains/remote/execution.bzl.tpl
+tensorflow/third_party/toolchains/gpus/crosstool/CROSSTOOL
+tensorflow/third_party/toolchains/gpus/crosstool/BUILD
+tensorflow/third_party/toolchains/gpus/cuda/build_defs.bzl
+tensorflow/third_party/toolchains/gpus/cuda/cuda/cuda_config.h
+tensorflow/third_party/toolchains/gpus/cuda/BUILD
+tensorflow/third_party/toolchains/gpus/py/BUILD
 tensorflow/third_party/toolchains/BUILD
-tensorflow/third_party/gpus/BUILD
+tensorflow/third_party/nccl/archive.BUILD
+tensorflow/third_party/nccl/nccl_configure.bzl
+tensorflow/third_party/nccl/system.BUILD.tpl
+tensorflow/third_party/nccl/build_defs.bzl.tpl
+tensorflow/third_party/nccl/LICENSE
+tensorflow/third_party/nccl/BUILD
+tensorflow/third_party/farmhash.BUILD
+tensorflow/third_party/sycl/crosstool/BUILD
+tensorflow/third_party/backports_weakref.BUILD
+tensorflow/third_party/gast.BUILD
+tensorflow/third_party/clang_toolchain/cc_configure_clang.bzl
+tensorflow/third_party/clang_toolchain/download_clang.bzl
+tensorflow/third_party/clang_toolchain/BUILD
+tensorflow/third_party/mpi_collectives/BUILD
+tensorflow/third_party/jsoncpp.BUILD
+tensorflow/third_party/mkl_dnn/mkldnn.BUILD
+tensorflow/third_party/mkl_dnn/LICENSE
+tensorflow/third_party/python_runtime/BUILD
+tensorflow/third_party/ngraph/ngraph.BUILD
+tensorflow/third_party/ngraph/nlohmann_json.BUILD
+tensorflow/third_party/ngraph/build_defs.bzl
+tensorflow/third_party/ngraph/ngraph_tf.BUILD
+tensorflow/third_party/ngraph/NGRAPH_LICENSE
+tensorflow/third_party/ngraph/tbb.BUILD
+tensorflow/third_party/ngraph/LICENSE
+tensorflow/third_party/ngraph/BUILD
+tensorflow/third_party/gpus/crosstool/windows/msvc_wrapper_for_nvcc.py.tpl
+tensorflow/third_party/gpus/crosstool/CROSSTOOL.tpl
+tensorflow/third_party/gpus/crosstool/BUILD.tpl
 tensorflow/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_rocm.tpl
 tensorflow/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_is_not_gcc.tpl
-tensorflow/third_party/gpus/crosstool/CROSSTOOL.tpl
 tensorflow/third_party/gpus/crosstool/CROSSTOOL_hipcc.tpl
 tensorflow/third_party/gpus/crosstool/LICENSE
-tensorflow/third_party/gpus/crosstool/windows/msvc_wrapper_for_nvcc.py.tpl
-tensorflow/third_party/gpus/crosstool/BUILD.tpl
 tensorflow/third_party/gpus/crosstool/BUILD
-tensorflow/third_party/gpus/cuda/LICENSE
+tensorflow/third_party/gpus/rocm/BUILD.tpl
+tensorflow/third_party/gpus/rocm/rocm_config.h.tpl
+tensorflow/third_party/gpus/rocm/build_defs.bzl.tpl
+tensorflow/third_party/gpus/rocm/BUILD
 tensorflow/third_party/gpus/cuda/BUILD.tpl
+tensorflow/third_party/gpus/cuda/build_defs.bzl.tpl
 tensorflow/third_party/gpus/cuda/BUILD.windows.tpl
 tensorflow/third_party/gpus/cuda/cuda_config.h.tpl
+tensorflow/third_party/gpus/cuda/LICENSE
 tensorflow/third_party/gpus/cuda/BUILD
-tensorflow/third_party/gpus/cuda/build_defs.bzl.tpl
-tensorflow/third_party/gpus/rocm/rocm_config.h.tpl
-tensorflow/third_party/gpus/rocm/BUILD
-tensorflow/third_party/gpus/rocm/BUILD.tpl
-tensorflow/third_party/gpus/rocm/build_defs.bzl.tpl
 tensorflow/third_party/gpus/cuda_configure.bzl
 tensorflow/third_party/gpus/rocm_configure.bzl
-tensorflow/third_party/snappy.BUILD
-tensorflow/third_party/cython.BUILD
-tensorflow/third_party/farmhash.BUILD
-tensorflow/third_party/eigen3/Eigen/Cholesky
-tensorflow/third_party/eigen3/Eigen/QR
-tensorflow/third_party/eigen3/Eigen/LU
+tensorflow/third_party/gpus/BUILD
+tensorflow/third_party/py/python_configure.bzl
+tensorflow/third_party/py/BUILD.tpl
+tensorflow/third_party/py/numpy/BUILD
+tensorflow/third_party/py/BUILD
+tensorflow/third_party/mkl/build_defs.bzl
+tensorflow/third_party/mkl/mkl.BUILD
+tensorflow/third_party/mkl/MKL_LICENSE
+tensorflow/third_party/mkl/LICENSE
+tensorflow/third_party/mkl/BUILD
+tensorflow/third_party/com_google_absl.BUILD
+tensorflow/third_party/six.BUILD
+tensorflow/third_party/lmdb.BUILD
+tensorflow/third_party/BUILD
+tensorflow/third_party/googleapis.BUILD
+tensorflow/third_party/sqlite.BUILD
+tensorflow/third_party/termcolor.BUILD
+tensorflow/third_party/protobuf/BUILD
+tensorflow/third_party/pcre.BUILD
+tensorflow/third_party/git/BUILD.tpl
+tensorflow/third_party/git/git_configure.bzl
+tensorflow/third_party/git/BUILD
+tensorflow/third_party/pprof.BUILD
+tensorflow/third_party/tflite_smartreply.BUILD
 tensorflow/third_party/eigen3/Eigen/Core
-tensorflow/third_party/eigen3/Eigen/SVD
+tensorflow/third_party/eigen3/Eigen/Cholesky
 tensorflow/third_party/eigen3/Eigen/Eigenvalues
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/FixedPoint
+tensorflow/third_party/eigen3/Eigen/SVD
+tensorflow/third_party/eigen3/Eigen/LU
+tensorflow/third_party/eigen3/Eigen/QR
+tensorflow/third_party/eigen3/unsupported/Eigen/MatrixFunctions
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/Tensor
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX512.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/ThreadPool
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/TypeCastingAVX512.h
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX2.h
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatMatProduct.h
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/FixedPointTypes.h
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatVecProduct.h
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatMatProductNEON.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX512.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatVecProduct.h
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/TypeCastingAVX2.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/FixedPointTypes.h
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatMatProductAVX2.h
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/ThreadPool
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX2.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/FixedPoint
 tensorflow/third_party/eigen3/unsupported/Eigen/SpecialFunctions
-tensorflow/third_party/eigen3/unsupported/Eigen/MatrixFunctions
 tensorflow/third_party/eigen3/LICENSE
 tensorflow/third_party/eigen3/BUILD
-tensorflow/third_party/systemlibs/build_defs.bzl.tpl
-tensorflow/third_party/systemlibs/absl_py.BUILD
-tensorflow/third_party/systemlibs/curl.BUILD
-tensorflow/third_party/systemlibs/termcolor.BUILD
-tensorflow/third_party/systemlibs/absl_py.absl.flags.BUILD
-tensorflow/third_party/systemlibs/grpc.BUILD
-tensorflow/third_party/systemlibs/swig.BUILD
-tensorflow/third_party/systemlibs/protobuf.bzl
-tensorflow/third_party/systemlibs/protobuf.BUILD
-tensorflow/third_party/systemlibs/BUILD
-tensorflow/third_party/systemlibs/google_cloud_cpp.BUILD
-tensorflow/third_party/systemlibs/astor.BUILD
-tensorflow/third_party/systemlibs/six.BUILD
-tensorflow/third_party/systemlibs/absl_py.absl.testing.BUILD
-tensorflow/third_party/systemlibs/boringssl.BUILD
-tensorflow/third_party/systemlibs/nsync.BUILD
-tensorflow/third_party/systemlibs/google_cloud_cpp.google.cloud.bigtable.BUILD
-tensorflow/third_party/systemlibs/gif.BUILD
-tensorflow/third_party/systemlibs/pcre.BUILD
-tensorflow/third_party/systemlibs/BUILD.tpl
-tensorflow/third_party/systemlibs/snappy.BUILD
-tensorflow/third_party/systemlibs/gast.BUILD
-tensorflow/third_party/systemlibs/cython.BUILD
-tensorflow/third_party/systemlibs/double_conversion.BUILD
-tensorflow/third_party/systemlibs/zlib.BUILD
-tensorflow/third_party/systemlibs/jsoncpp.BUILD
-tensorflow/third_party/systemlibs/enum34.BUILD
-tensorflow/third_party/systemlibs/re2.BUILD
-tensorflow/third_party/systemlibs/lmdb.BUILD
-tensorflow/third_party/systemlibs/googleapis.BUILD
-tensorflow/third_party/systemlibs/png.BUILD
-tensorflow/third_party/systemlibs/syslibs_configure.bzl
-tensorflow/third_party/systemlibs/sqlite.BUILD
-tensorflow/third_party/python_runtime/BUILD
-tensorflow/third_party/sycl/crosstool/BUILD
-tensorflow/third_party/ngraph/LICENSE
-tensorflow/third_party/ngraph/tbb.BUILD
-tensorflow/third_party/ngraph/BUILD
-tensorflow/third_party/ngraph/ngraph.BUILD
-tensorflow/third_party/ngraph/build_defs.bzl
-tensorflow/third_party/ngraph/NGRAPH_LICENSE
-tensorflow/third_party/ngraph/ngraph_tf.BUILD
-tensorflow/third_party/ngraph/nlohmann_json.BUILD
-tensorflow/third_party/clang_toolchain/download_clang.bzl
-tensorflow/third_party/clang_toolchain/BUILD
-tensorflow/third_party/clang_toolchain/cc_configure_clang.bzl
-tensorflow/third_party/gast.BUILD
-tensorflow/third_party/llvm/BUILD
-tensorflow/third_party/llvm/expand_cmake_vars.py
-tensorflow/third_party/llvm/llvm.autogenerated.BUILD
-tensorflow/third_party/llvm/llvm.bzl
-tensorflow/third_party/icu/udata.patch
-tensorflow/third_party/nccl/archive.BUILD
-tensorflow/third_party/nccl/LICENSE
-tensorflow/third_party/nccl/system.BUILD.tpl
-tensorflow/third_party/nccl/nccl_configure.bzl
-tensorflow/third_party/nccl/build_defs.bzl.tpl
-tensorflow/third_party/nccl/BUILD
-tensorflow/third_party/fft2d/BUILD
-tensorflow/third_party/fft2d/fft.h
-tensorflow/third_party/fft2d/LICENSE
-tensorflow/third_party/fft2d/fft2d.BUILD
-tensorflow/third_party/boringssl/BUILD
-tensorflow/third_party/mpi/.gitignore
-tensorflow/third_party/mpi/BUILD
-tensorflow/third_party/tensorrt/LICENSE
-tensorflow/third_party/tensorrt/BUILD
-tensorflow/third_party/tensorrt/build_defs.bzl.tpl
-tensorflow/third_party/tensorrt/BUILD.tpl
-tensorflow/third_party/tensorrt/tensorrt_configure.bzl
-tensorflow/third_party/kafka/config.patch
-tensorflow/third_party/kafka/BUILD
-tensorflow/third_party/android/BUILD
-tensorflow/third_party/android/android.bzl.tpl
-tensorflow/third_party/android/android_configure.bzl
-tensorflow/third_party/android/android_configure.BUILD.tpl
-tensorflow/third_party/tflite_smartreply.BUILD
-tensorflow/third_party/mkl_dnn/LICENSE
-tensorflow/third_party/mkl_dnn/mkldnn.BUILD
-tensorflow/third_party/pcre.BUILD
-tensorflow/third_party/linenoise.BUILD
-tensorflow/third_party/sqlite.BUILD
-tensorflow/third_party/common.bzl
-tensorflow/third_party/com_google_absl.BUILD
-tensorflow/third_party/pprof.BUILD
-tensorflow/third_party/BUILD
-tensorflow/third_party/tflite_mobilenet_quant.BUILD
-tensorflow/third_party/lmdb.BUILD
-tensorflow/third_party/git/BUILD.tpl
-tensorflow/third_party/git/BUILD
-tensorflow/third_party/git/git_configure.bzl
-tensorflow/third_party/protobuf/BUILD
-tensorflow/third_party/tflite_mobilenet.BUILD
-tensorflow/third_party/py/BUILD
-tensorflow/third_party/py/BUILD.tpl
-tensorflow/third_party/py/numpy/BUILD
-tensorflow/third_party/py/python_configure.bzl
-tensorflow/third_party/termcolor.BUILD
-tensorflow/third_party/png_fix_rpi.patch
-tensorflow/third_party/swig.BUILD
-tensorflow/third_party/astor.BUILD
-tensorflow/third_party/grpc/BUILD
-tensorflow/third_party/curl.BUILD
-tensorflow/third_party/arm_neon_2_x86_sse.BUILD
-tensorflow/third_party/png.BUILD
-tensorflow/third_party/googleapis.BUILD
-tensorflow/third_party/mpi_collectives/BUILD
-tensorflow/third_party/nanopb.BUILD
-tensorflow/third_party/gif.BUILD
-tensorflow/third_party/double_conversion.BUILD
-tensorflow/third_party/six.BUILD
-tensorflow/third_party/tflite_mobilenet_float.BUILD
-tensorflow/third_party/repo.bzl
-tensorflow/third_party/codegen.BUILD
-tensorflow/third_party/cub.BUILD
-tensorflow/third_party/jsoncpp.BUILD
-tensorflow/third_party/tflite_ovic_testdata.BUILD
-tensorflow/third_party/libxsmm.BUILD
-tensorflow/third_party/zlib.BUILD
-tensorflow/third_party/eigen.BUILD
-tensorflow/stream_executor/build_defs.bzl
-tensorflow/api_template_v1.__init__.py
-tensorflow/compat_template_v1.__init__.py
+tensorflow/tools/pip_package/simple_console.py
+tensorflow/tools/pip_package/setup.py
+tensorflow/tools/pip_package/MANIFEST.in
+tensorflow/tools/pip_package/README
+tensorflow/tools/pip_package/check_load_py_test.py
+tensorflow/tools/pip_package/pip_smoke_test.py
+tensorflow/tools/pip_package/BUILD
+tensorflow/tools/pip_package/build_pip_package.sh
+tensorflow/tools/pip_package/simple_console_for_windows.py
+tensorflow/tools/lib_package/LibTensorFlowTest.java
+tensorflow/tools/lib_package/README.md
+tensorflow/tools/lib_package/libtensorflow_test.sh
+tensorflow/tools/lib_package/libtensorflow_java_test.sh
+tensorflow/tools/lib_package/libtensorflow_test.c
+tensorflow/tools/lib_package/concat_licenses.sh
+tensorflow/tools/lib_package/BUILD
+tensorflow/tools/ci_build/remote/BUILD
+tensorflow/tools/def_file_filter/BUILD.tpl
+tensorflow/tools/def_file_filter/def_file_filter_configure.bzl
+tensorflow/tools/def_file_filter/def_file_filter.py.tpl
+tensorflow/tools/def_file_filter/BUILD
+tensorflow/contrib/mpi/BUILD
+tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/main.py
+tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/__init__.py
+tensorflow/contrib/tpu/profiler/pip_package/setup.py
+tensorflow/contrib/tpu/profiler/pip_package/README
+tensorflow/contrib/tpu/profiler/pip_package/BUILD
+tensorflow/contrib/tpu/profiler/pip_package/build_pip_package.sh
 tensorflow/api_template.__init__.py
 tensorflow/__init__.py
\ No newline at end of file
diff --git a/tensorflow/python/client/session.py b/tensorflow/python/client/session.py
index b97eb884b3..bdca7dee55 100644
--- a/tensorflow/python/client/session.py
+++ b/tensorflow/python/client/session.py
@@ -1532,7 +1532,7 @@ class Session(BaseSession):
 
     If no `graph` argument is specified when constructing the session,
     the default graph will be launched in the session. If you are
-    using more than one graph (created with `tf.Graph()` in the same
+    using more than one graph (created with `tf.Graph()`) in the same
     process, you will have to use different sessions for each graph,
     but each graph can be used in multiple sessions. In this case, it
     is often clearer to pass the graph to be launched explicitly to
@@ -1675,7 +1675,7 @@ class InteractiveSession(BaseSession):
 
     If no `graph` argument is specified when constructing the session,
     the default graph will be launched in the session. If you are
-    using more than one graph (created with `tf.Graph()` in the same
+    using more than one graph (created with `tf.Graph()`) in the same
     process, you will have to use different sessions for each graph,
     but each graph can be used in multiple sessions. In this case, it
     is often clearer to pass the graph to be launched explicitly to
diff --git a/tensorflow/python/data/ops/multi_device_iterator_ops.py b/tensorflow/python/data/ops/multi_device_iterator_ops.py
index 44aebb60cf..8f5c550216 100644
--- a/tensorflow/python/data/ops/multi_device_iterator_ops.py
+++ b/tensorflow/python/data/ops/multi_device_iterator_ops.py
@@ -126,13 +126,7 @@ class _PerDeviceGenerator(dataset_ops.DatasetV2):
 
 
 class MultiDeviceIterator(object):
-  """An iterator over multiple devices.
-
-  @compatibility(eager)
-  MultiDeviceIterator isn't currently supported in Eager mode but support is
-  coming soon.
-  @end_compatibility
-  """
+  """An iterator over multiple devices."""
 
   def __init__(self,
                dataset,
diff --git a/tensorflow/python/framework/function.py b/tensorflow/python/framework/function.py
index afc11b17bf..291986a2c4 100644
--- a/tensorflow/python/framework/function.py
+++ b/tensorflow/python/framework/function.py
@@ -210,6 +210,7 @@ class _DefinedFunction(object):
                shape_func=None,
                capture_by_value=False,
                whitelisted_stateful_ops=None,
+               capture_resource_var_by_value=True,
                **kwargs):
     """Creates _DefinedFunction.
 
@@ -232,6 +233,8 @@ class _DefinedFunction(object):
         will be copied into the function body.
       whitelisted_stateful_ops: A set of ops that if stateful we ignore and
         copy into the function body, when `capture_by_value` is True.
+      capture_resource_var_by_value: Boolean (defaults to True). If False,
+        captured resource variable returns the handle instead of value.
       **kwargs: The keyword arguments. **kwargs is passed to every call
         site of this function.
 
@@ -250,6 +253,7 @@ class _DefinedFunction(object):
     self._whitelisted_stateful_ops = whitelisted_stateful_ops
     if self._whitelisted_stateful_ops is None:
       self._whitelisted_stateful_ops = set()
+    self._capture_resource_var_by_value = capture_resource_var_by_value
     self._extra_kwargs = kwargs
     # Constructed only when C API is disabled, lazily
     self._definition = None
@@ -352,7 +356,8 @@ class _DefinedFunction(object):
         self._func_name,
         self._capture_by_value,
         self._caller_device,
-        whitelisted_stateful_ops=self._whitelisted_stateful_ops)
+        whitelisted_stateful_ops=self._whitelisted_stateful_ops,
+        capture_resource_var_by_value=self._capture_resource_var_by_value)
 
     self._extra_inputs = temp_graph.extra_inputs
     # pylint: disable=protected-access
@@ -636,11 +641,12 @@ class _FuncGraph(ops.Graph):
   function argument and the caller passes in the captured tensor.
   """
 
-  def __init__(self, name, capture_by_value, whitelisted_stateful_ops, *args,
-               **kwargs):
+  def __init__(self, name, capture_by_value, whitelisted_stateful_ops,
+               capture_resource_var_by_value, *args, **kwargs):
     super(_FuncGraph, self).__init__(*args, **kwargs)
     self._capture_by_value = capture_by_value
     self._whitelisted_stateful_ops = whitelisted_stateful_ops
+    self._capture_resource_var_by_value = capture_resource_var_by_value
     self._building_function = True
     self._outer_graph = ops.get_default_graph()
     self._vscope = vs.get_variable_scope()
@@ -735,7 +741,8 @@ class _FuncGraph(ops.Graph):
           collections=collections,
           use_resource=use_resource)
       self.extra_vars.append(var)
-      if isinstance(var, resource_variable_ops.ResourceVariable):
+      if (isinstance(var, resource_variable_ops.ResourceVariable) and
+          self._capture_resource_var_by_value):
         # For resource-based variables read the variable outside the function
         # and pass in the value. This ensures that the function is pure and
         # differentiable. TODO(apassos) this may have performance problems if
@@ -830,7 +837,8 @@ def func_graph_from_py_func(func,
                             container=None,
                             collections_ref=None,
                             arg_shapes=None,
-                            whitelisted_stateful_ops=None):
+                            whitelisted_stateful_ops=None,
+                            capture_resource_var_by_value=True):
   """Returns a _FuncGraph generated from `func`.
 
   Args:
@@ -850,6 +858,8 @@ def func_graph_from_py_func(func,
     arg_shapes: A sequence of the function's argument shapes.
     whitelisted_stateful_ops: A set of ops that if stateful we ignore and
       re-create.
+    capture_resource_var_by_value: Boolean (defaults to True). If False,
+      captured resource variable returns the handle instead of value.
 
   Returns:
     A _FuncGraph.
@@ -859,7 +869,8 @@ def func_graph_from_py_func(func,
   """
   if not name:
     name = function_utils.get_func_name(func)
-  func_graph = _FuncGraph(name, capture_by_value, whitelisted_stateful_ops)
+  func_graph = _FuncGraph(name, capture_by_value, whitelisted_stateful_ops,
+                          capture_resource_var_by_value)
 
   with func_graph.as_default(), ops.device(device):
     # pylint: disable=protected-access
-- 
GitLab


From eabe5a7d8030c87fb7250410c20ab55ec37e30a2 Mon Sep 17 00:00:00 2001
From: Yanan Cao <ycao@google.com>
Date: Tue, 22 Jan 2019 17:36:01 -0800
Subject: [PATCH 1093/2345] Hide an internal TODO item from public doc

PiperOrigin-RevId: 230445548
---
 tensorflow/contrib/tpu/python/tpu/tpu.py |   5 +-
 tensorflow/opensource_only.files         | 418 +++++++++++------------
 2 files changed, 211 insertions(+), 212 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu.py b/tensorflow/contrib/tpu/python/tpu/tpu.py
index b04baebfe6..f8762e86e5 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu.py
@@ -915,9 +915,6 @@ def split_compile_and_shard(computation,
     return x + 3
   ... = shard(computation, ...)
 
-  TODO(phawkins): consider adding support for broadcasting Tensors passed
-  as inputs.
-
   If `outputs_from_all_shards` is true, the outputs from all shards of
   `computation` are concatenated back together along their `output_shards_axes`.
   Otherwise, each output is taken from an arbitrary shard.
@@ -959,6 +956,8 @@ def split_compile_and_shard(computation,
     ValueError: If len(input_shard_axes) != len(inputs)
     ValueError: If len(output_shard_axes) != len(outputs from `computation`)
   """
+  # TODO(phawkins): consider adding support for broadcasting Tensors passed as
+  # inputs.
 
   if num_shards <= 0:
     raise ValueError("num_shards must be a positive integer.")
diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index a5e51d7276..c546a34cf8 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -1,248 +1,248 @@
-tensorflow/api_template_v1.__init__.py
-tensorflow/compat_template_v1.__init__.py
-tensorflow/stream_executor/build_defs.bzl
-tensorflow/third_party/curl.BUILD
-tensorflow/third_party/snappy.BUILD
-tensorflow/third_party/tflite_mobilenet_float.BUILD
-tensorflow/third_party/libxsmm.BUILD
-tensorflow/third_party/cub.BUILD
-tensorflow/third_party/repo.bzl
-tensorflow/third_party/systemlibs/curl.BUILD
-tensorflow/third_party/systemlibs/snappy.BUILD
-tensorflow/third_party/systemlibs/protobuf.BUILD
-tensorflow/third_party/systemlibs/absl_py.BUILD
-tensorflow/third_party/systemlibs/google_cloud_cpp.google.cloud.bigtable.BUILD
-tensorflow/third_party/systemlibs/astor.BUILD
-tensorflow/third_party/systemlibs/absl_py.absl.flags.BUILD
-tensorflow/third_party/systemlibs/png.BUILD
-tensorflow/third_party/systemlibs/enum34.BUILD
-tensorflow/third_party/systemlibs/BUILD.tpl
-tensorflow/third_party/systemlibs/boringssl.BUILD
-tensorflow/third_party/systemlibs/absl_py.absl.testing.BUILD
-tensorflow/third_party/systemlibs/swig.BUILD
-tensorflow/third_party/systemlibs/double_conversion.BUILD
-tensorflow/third_party/systemlibs/gif.BUILD
-tensorflow/third_party/systemlibs/cython.BUILD
-tensorflow/third_party/systemlibs/google_cloud_cpp.BUILD
-tensorflow/third_party/systemlibs/zlib.BUILD
-tensorflow/third_party/systemlibs/grpc.BUILD
-tensorflow/third_party/systemlibs/re2.BUILD
-tensorflow/third_party/systemlibs/build_defs.bzl.tpl
-tensorflow/third_party/systemlibs/protobuf.bzl
-tensorflow/third_party/systemlibs/gast.BUILD
-tensorflow/third_party/systemlibs/syslibs_configure.bzl
-tensorflow/third_party/systemlibs/nsync.BUILD
-tensorflow/third_party/systemlibs/jsoncpp.BUILD
-tensorflow/third_party/systemlibs/six.BUILD
-tensorflow/third_party/systemlibs/lmdb.BUILD
-tensorflow/third_party/systemlibs/BUILD
-tensorflow/third_party/systemlibs/googleapis.BUILD
-tensorflow/third_party/systemlibs/sqlite.BUILD
-tensorflow/third_party/systemlibs/termcolor.BUILD
-tensorflow/third_party/systemlibs/pcre.BUILD
-tensorflow/third_party/tflite_mobilenet_quant.BUILD
-tensorflow/third_party/common.bzl
-tensorflow/third_party/boringssl/BUILD
-tensorflow/third_party/kafka/config.patch
-tensorflow/third_party/kafka/BUILD
-tensorflow/third_party/android/android_configure.bzl
-tensorflow/third_party/android/android_configure.BUILD.tpl
-tensorflow/third_party/android/android.bzl.tpl
-tensorflow/third_party/android/BUILD
-tensorflow/third_party/astor.BUILD
-tensorflow/third_party/icu/udata.patch
-tensorflow/third_party/tflite_mobilenet.BUILD
-tensorflow/third_party/png.BUILD
-tensorflow/third_party/mpi/.gitignore
-tensorflow/third_party/mpi/BUILD
-tensorflow/third_party/nanopb.BUILD
-tensorflow/third_party/arm_neon_2_x86_sse.BUILD
-tensorflow/third_party/linenoise.BUILD
-tensorflow/third_party/llvm/llvm.autogenerated.BUILD
-tensorflow/third_party/llvm/llvm.bzl
-tensorflow/third_party/llvm/BUILD
-tensorflow/third_party/llvm/expand_cmake_vars.py
-tensorflow/third_party/swig.BUILD
-tensorflow/third_party/png_fix_rpi.patch
-tensorflow/third_party/double_conversion.BUILD
-tensorflow/third_party/gif.BUILD
-tensorflow/third_party/grpc/BUILD
-tensorflow/third_party/cython.BUILD
-tensorflow/third_party/eigen.BUILD
-tensorflow/third_party/codegen.BUILD
-tensorflow/third_party/zlib.BUILD
-tensorflow/third_party/tensorrt/BUILD.tpl
-tensorflow/third_party/tensorrt/tensorrt_configure.bzl
-tensorflow/third_party/tensorrt/build_defs.bzl.tpl
-tensorflow/third_party/tensorrt/LICENSE
-tensorflow/third_party/tensorrt/BUILD
-tensorflow/third_party/tflite_ovic_testdata.BUILD
-tensorflow/third_party/fft2d/fft.h
-tensorflow/third_party/fft2d/fft2d.BUILD
-tensorflow/third_party/fft2d/LICENSE
-tensorflow/third_party/fft2d/BUILD
+tensorflow/contrib/tpu/profiler/pip_package/BUILD
+tensorflow/contrib/tpu/profiler/pip_package/setup.py
+tensorflow/contrib/tpu/profiler/pip_package/README
+tensorflow/contrib/tpu/profiler/pip_package/build_pip_package.sh
+tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/main.py
+tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/__init__.py
+tensorflow/contrib/mpi/BUILD
+tensorflow/tools/ci_build/remote/BUILD
+tensorflow/tools/pip_package/README
+tensorflow/tools/pip_package/MANIFEST.in
+tensorflow/tools/pip_package/simple_console.py
+tensorflow/tools/pip_package/build_pip_package.sh
+tensorflow/tools/pip_package/check_load_py_test.py
+tensorflow/tools/pip_package/pip_smoke_test.py
+tensorflow/tools/pip_package/simple_console_for_windows.py
+tensorflow/tools/pip_package/setup.py
+tensorflow/tools/pip_package/BUILD
+tensorflow/tools/lib_package/concat_licenses.sh
+tensorflow/tools/lib_package/libtensorflow_test.c
+tensorflow/tools/lib_package/LibTensorFlowTest.java
+tensorflow/tools/lib_package/BUILD
+tensorflow/tools/lib_package/libtensorflow_test.sh
+tensorflow/tools/lib_package/README.md
+tensorflow/tools/lib_package/libtensorflow_java_test.sh
+tensorflow/tools/def_file_filter/def_file_filter_configure.bzl
+tensorflow/tools/def_file_filter/BUILD
+tensorflow/tools/def_file_filter/BUILD.tpl
+tensorflow/tools/def_file_filter/def_file_filter.py.tpl
+tensorflow/third_party/mkl/MKL_LICENSE
+tensorflow/third_party/mkl/LICENSE
+tensorflow/third_party/mkl/BUILD
+tensorflow/third_party/mkl/mkl.BUILD
+tensorflow/third_party/mkl/build_defs.bzl
+tensorflow/third_party/backports_weakref.BUILD
+tensorflow/third_party/toolchains/clang6/BUILD
 tensorflow/third_party/toolchains/clang6/README.md
 tensorflow/third_party/toolchains/clang6/repo.bzl
-tensorflow/third_party/toolchains/clang6/clang.BUILD
 tensorflow/third_party/toolchains/clang6/CROSSTOOL.tpl
-tensorflow/third_party/toolchains/clang6/BUILD
-tensorflow/third_party/toolchains/remote/BUILD.tpl
-tensorflow/third_party/toolchains/remote/configure.bzl
-tensorflow/third_party/toolchains/remote/execution.bzl.tpl
-tensorflow/third_party/toolchains/remote/BUILD
-tensorflow/third_party/toolchains/preconfig/generate/archives.bzl
-tensorflow/third_party/toolchains/preconfig/generate/workspace.bzl
-tensorflow/third_party/toolchains/preconfig/generate/generate.bzl
-tensorflow/third_party/toolchains/preconfig/generate/containers.bzl
-tensorflow/third_party/toolchains/preconfig/generate/BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/build_defs.bzl
+tensorflow/third_party/toolchains/clang6/clang.BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/build_defs.bzl
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/py3/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc/BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/build_defs.bzl
-tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/build_defs.bzl
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/build_defs.bzl
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD
+tensorflow/third_party/toolchains/preconfig/generate/workspace.bzl
+tensorflow/third_party/toolchains/preconfig/generate/containers.bzl
+tensorflow/third_party/toolchains/preconfig/generate/generate.bzl
+tensorflow/third_party/toolchains/preconfig/generate/archives.bzl
+tensorflow/third_party/toolchains/preconfig/generate/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu16.04/clang/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu16.04/clang/dummy_toolchain.bzl
+tensorflow/third_party/toolchains/preconfig/ubuntu16.04/py3/BUILD
+tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/BUILD
+tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/dummy_toolchain.bzl
 tensorflow/third_party/toolchains/preconfig/win_1803/py36/BUILD
 tensorflow/third_party/toolchains/preconfig/win_1803/BUILD
-tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/dummy_toolchain.bzl
-tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu16.04/py3/BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu16.04/clang/dummy_toolchain.bzl
-tensorflow/third_party/toolchains/preconfig/ubuntu16.04/clang/BUILD
-tensorflow/third_party/toolchains/cpus/py3/BUILD
-tensorflow/third_party/toolchains/cpus/arm/CROSSTOOL.tpl
-tensorflow/third_party/toolchains/cpus/arm/BUILD
-tensorflow/third_party/toolchains/cpus/arm/arm_compiler_configure.bzl
-tensorflow/third_party/toolchains/cpus/py/BUILD
-tensorflow/third_party/toolchains/gpus/crosstool/CROSSTOOL
-tensorflow/third_party/toolchains/gpus/crosstool/BUILD
 tensorflow/third_party/toolchains/gpus/cuda/build_defs.bzl
-tensorflow/third_party/toolchains/gpus/cuda/cuda/cuda_config.h
 tensorflow/third_party/toolchains/gpus/cuda/BUILD
+tensorflow/third_party/toolchains/gpus/cuda/cuda/cuda_config.h
+tensorflow/third_party/toolchains/gpus/crosstool/BUILD
+tensorflow/third_party/toolchains/gpus/crosstool/CROSSTOOL
 tensorflow/third_party/toolchains/gpus/py/BUILD
+tensorflow/third_party/toolchains/cpus/arm/arm_compiler_configure.bzl
+tensorflow/third_party/toolchains/cpus/arm/CROSSTOOL.tpl
+tensorflow/third_party/toolchains/cpus/arm/BUILD
+tensorflow/third_party/toolchains/cpus/py3/BUILD
+tensorflow/third_party/toolchains/cpus/py/BUILD
+tensorflow/third_party/toolchains/remote/configure.bzl
+tensorflow/third_party/toolchains/remote/BUILD.tpl
+tensorflow/third_party/toolchains/remote/BUILD
+tensorflow/third_party/toolchains/remote/execution.bzl.tpl
 tensorflow/third_party/toolchains/BUILD
-tensorflow/third_party/nccl/archive.BUILD
-tensorflow/third_party/nccl/nccl_configure.bzl
-tensorflow/third_party/nccl/system.BUILD.tpl
-tensorflow/third_party/nccl/build_defs.bzl.tpl
-tensorflow/third_party/nccl/LICENSE
-tensorflow/third_party/nccl/BUILD
-tensorflow/third_party/farmhash.BUILD
-tensorflow/third_party/sycl/crosstool/BUILD
-tensorflow/third_party/backports_weakref.BUILD
-tensorflow/third_party/gast.BUILD
-tensorflow/third_party/clang_toolchain/cc_configure_clang.bzl
-tensorflow/third_party/clang_toolchain/download_clang.bzl
-tensorflow/third_party/clang_toolchain/BUILD
-tensorflow/third_party/mpi_collectives/BUILD
-tensorflow/third_party/jsoncpp.BUILD
-tensorflow/third_party/mkl_dnn/mkldnn.BUILD
-tensorflow/third_party/mkl_dnn/LICENSE
-tensorflow/third_party/python_runtime/BUILD
-tensorflow/third_party/ngraph/ngraph.BUILD
-tensorflow/third_party/ngraph/nlohmann_json.BUILD
-tensorflow/third_party/ngraph/build_defs.bzl
-tensorflow/third_party/ngraph/ngraph_tf.BUILD
-tensorflow/third_party/ngraph/NGRAPH_LICENSE
-tensorflow/third_party/ngraph/tbb.BUILD
-tensorflow/third_party/ngraph/LICENSE
-tensorflow/third_party/ngraph/BUILD
-tensorflow/third_party/gpus/crosstool/windows/msvc_wrapper_for_nvcc.py.tpl
-tensorflow/third_party/gpus/crosstool/CROSSTOOL.tpl
-tensorflow/third_party/gpus/crosstool/BUILD.tpl
+tensorflow/third_party/gpus/BUILD
 tensorflow/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_rocm.tpl
 tensorflow/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_is_not_gcc.tpl
+tensorflow/third_party/gpus/crosstool/CROSSTOOL.tpl
 tensorflow/third_party/gpus/crosstool/CROSSTOOL_hipcc.tpl
 tensorflow/third_party/gpus/crosstool/LICENSE
+tensorflow/third_party/gpus/crosstool/windows/msvc_wrapper_for_nvcc.py.tpl
+tensorflow/third_party/gpus/crosstool/BUILD.tpl
 tensorflow/third_party/gpus/crosstool/BUILD
-tensorflow/third_party/gpus/rocm/BUILD.tpl
-tensorflow/third_party/gpus/rocm/rocm_config.h.tpl
-tensorflow/third_party/gpus/rocm/build_defs.bzl.tpl
-tensorflow/third_party/gpus/rocm/BUILD
+tensorflow/third_party/gpus/cuda/LICENSE
 tensorflow/third_party/gpus/cuda/BUILD.tpl
-tensorflow/third_party/gpus/cuda/build_defs.bzl.tpl
 tensorflow/third_party/gpus/cuda/BUILD.windows.tpl
 tensorflow/third_party/gpus/cuda/cuda_config.h.tpl
-tensorflow/third_party/gpus/cuda/LICENSE
 tensorflow/third_party/gpus/cuda/BUILD
+tensorflow/third_party/gpus/cuda/build_defs.bzl.tpl
+tensorflow/third_party/gpus/rocm/rocm_config.h.tpl
+tensorflow/third_party/gpus/rocm/BUILD
+tensorflow/third_party/gpus/rocm/BUILD.tpl
+tensorflow/third_party/gpus/rocm/build_defs.bzl.tpl
 tensorflow/third_party/gpus/cuda_configure.bzl
 tensorflow/third_party/gpus/rocm_configure.bzl
-tensorflow/third_party/gpus/BUILD
-tensorflow/third_party/py/python_configure.bzl
-tensorflow/third_party/py/BUILD.tpl
-tensorflow/third_party/py/numpy/BUILD
-tensorflow/third_party/py/BUILD
-tensorflow/third_party/mkl/build_defs.bzl
-tensorflow/third_party/mkl/mkl.BUILD
-tensorflow/third_party/mkl/MKL_LICENSE
-tensorflow/third_party/mkl/LICENSE
-tensorflow/third_party/mkl/BUILD
-tensorflow/third_party/com_google_absl.BUILD
-tensorflow/third_party/six.BUILD
-tensorflow/third_party/lmdb.BUILD
-tensorflow/third_party/BUILD
-tensorflow/third_party/googleapis.BUILD
-tensorflow/third_party/sqlite.BUILD
-tensorflow/third_party/termcolor.BUILD
-tensorflow/third_party/protobuf/BUILD
-tensorflow/third_party/pcre.BUILD
-tensorflow/third_party/git/BUILD.tpl
-tensorflow/third_party/git/git_configure.bzl
-tensorflow/third_party/git/BUILD
-tensorflow/third_party/pprof.BUILD
-tensorflow/third_party/tflite_smartreply.BUILD
-tensorflow/third_party/eigen3/Eigen/Core
+tensorflow/third_party/snappy.BUILD
+tensorflow/third_party/cython.BUILD
+tensorflow/third_party/farmhash.BUILD
 tensorflow/third_party/eigen3/Eigen/Cholesky
-tensorflow/third_party/eigen3/Eigen/Eigenvalues
-tensorflow/third_party/eigen3/Eigen/SVD
-tensorflow/third_party/eigen3/Eigen/LU
 tensorflow/third_party/eigen3/Eigen/QR
-tensorflow/third_party/eigen3/unsupported/Eigen/MatrixFunctions
+tensorflow/third_party/eigen3/Eigen/LU
+tensorflow/third_party/eigen3/Eigen/Core
+tensorflow/third_party/eigen3/Eigen/SVD
+tensorflow/third_party/eigen3/Eigen/Eigenvalues
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/FixedPoint
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/Tensor
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/ThreadPool
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX512.h
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/TypeCastingAVX512.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX2.h
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatMatProduct.h
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatMatProductNEON.h
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX512.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/FixedPointTypes.h
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatVecProduct.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatMatProductNEON.h
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/TypeCastingAVX2.h
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/FixedPointTypes.h
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatMatProductAVX2.h
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX2.h
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/FixedPoint
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/ThreadPool
 tensorflow/third_party/eigen3/unsupported/Eigen/SpecialFunctions
+tensorflow/third_party/eigen3/unsupported/Eigen/MatrixFunctions
 tensorflow/third_party/eigen3/LICENSE
 tensorflow/third_party/eigen3/BUILD
-tensorflow/tools/pip_package/simple_console.py
-tensorflow/tools/pip_package/setup.py
-tensorflow/tools/pip_package/MANIFEST.in
-tensorflow/tools/pip_package/README
-tensorflow/tools/pip_package/check_load_py_test.py
-tensorflow/tools/pip_package/pip_smoke_test.py
-tensorflow/tools/pip_package/BUILD
-tensorflow/tools/pip_package/build_pip_package.sh
-tensorflow/tools/pip_package/simple_console_for_windows.py
-tensorflow/tools/lib_package/LibTensorFlowTest.java
-tensorflow/tools/lib_package/README.md
-tensorflow/tools/lib_package/libtensorflow_test.sh
-tensorflow/tools/lib_package/libtensorflow_java_test.sh
-tensorflow/tools/lib_package/libtensorflow_test.c
-tensorflow/tools/lib_package/concat_licenses.sh
-tensorflow/tools/lib_package/BUILD
-tensorflow/tools/ci_build/remote/BUILD
-tensorflow/tools/def_file_filter/BUILD.tpl
-tensorflow/tools/def_file_filter/def_file_filter_configure.bzl
-tensorflow/tools/def_file_filter/def_file_filter.py.tpl
-tensorflow/tools/def_file_filter/BUILD
-tensorflow/contrib/mpi/BUILD
-tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/main.py
-tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/__init__.py
-tensorflow/contrib/tpu/profiler/pip_package/setup.py
-tensorflow/contrib/tpu/profiler/pip_package/README
-tensorflow/contrib/tpu/profiler/pip_package/BUILD
-tensorflow/contrib/tpu/profiler/pip_package/build_pip_package.sh
+tensorflow/third_party/systemlibs/build_defs.bzl.tpl
+tensorflow/third_party/systemlibs/absl_py.BUILD
+tensorflow/third_party/systemlibs/curl.BUILD
+tensorflow/third_party/systemlibs/termcolor.BUILD
+tensorflow/third_party/systemlibs/absl_py.absl.flags.BUILD
+tensorflow/third_party/systemlibs/grpc.BUILD
+tensorflow/third_party/systemlibs/swig.BUILD
+tensorflow/third_party/systemlibs/protobuf.bzl
+tensorflow/third_party/systemlibs/protobuf.BUILD
+tensorflow/third_party/systemlibs/BUILD
+tensorflow/third_party/systemlibs/google_cloud_cpp.BUILD
+tensorflow/third_party/systemlibs/astor.BUILD
+tensorflow/third_party/systemlibs/six.BUILD
+tensorflow/third_party/systemlibs/absl_py.absl.testing.BUILD
+tensorflow/third_party/systemlibs/boringssl.BUILD
+tensorflow/third_party/systemlibs/nsync.BUILD
+tensorflow/third_party/systemlibs/google_cloud_cpp.google.cloud.bigtable.BUILD
+tensorflow/third_party/systemlibs/gif.BUILD
+tensorflow/third_party/systemlibs/pcre.BUILD
+tensorflow/third_party/systemlibs/BUILD.tpl
+tensorflow/third_party/systemlibs/snappy.BUILD
+tensorflow/third_party/systemlibs/gast.BUILD
+tensorflow/third_party/systemlibs/cython.BUILD
+tensorflow/third_party/systemlibs/double_conversion.BUILD
+tensorflow/third_party/systemlibs/zlib.BUILD
+tensorflow/third_party/systemlibs/jsoncpp.BUILD
+tensorflow/third_party/systemlibs/enum34.BUILD
+tensorflow/third_party/systemlibs/re2.BUILD
+tensorflow/third_party/systemlibs/lmdb.BUILD
+tensorflow/third_party/systemlibs/googleapis.BUILD
+tensorflow/third_party/systemlibs/png.BUILD
+tensorflow/third_party/systemlibs/syslibs_configure.bzl
+tensorflow/third_party/systemlibs/sqlite.BUILD
+tensorflow/third_party/python_runtime/BUILD
+tensorflow/third_party/sycl/crosstool/BUILD
+tensorflow/third_party/ngraph/LICENSE
+tensorflow/third_party/ngraph/tbb.BUILD
+tensorflow/third_party/ngraph/BUILD
+tensorflow/third_party/ngraph/ngraph.BUILD
+tensorflow/third_party/ngraph/build_defs.bzl
+tensorflow/third_party/ngraph/NGRAPH_LICENSE
+tensorflow/third_party/ngraph/ngraph_tf.BUILD
+tensorflow/third_party/ngraph/nlohmann_json.BUILD
+tensorflow/third_party/clang_toolchain/download_clang.bzl
+tensorflow/third_party/clang_toolchain/BUILD
+tensorflow/third_party/clang_toolchain/cc_configure_clang.bzl
+tensorflow/third_party/gast.BUILD
+tensorflow/third_party/llvm/BUILD
+tensorflow/third_party/llvm/expand_cmake_vars.py
+tensorflow/third_party/llvm/llvm.autogenerated.BUILD
+tensorflow/third_party/llvm/llvm.bzl
+tensorflow/third_party/icu/udata.patch
+tensorflow/third_party/nccl/archive.BUILD
+tensorflow/third_party/nccl/LICENSE
+tensorflow/third_party/nccl/system.BUILD.tpl
+tensorflow/third_party/nccl/nccl_configure.bzl
+tensorflow/third_party/nccl/build_defs.bzl.tpl
+tensorflow/third_party/nccl/BUILD
+tensorflow/third_party/fft2d/BUILD
+tensorflow/third_party/fft2d/fft.h
+tensorflow/third_party/fft2d/LICENSE
+tensorflow/third_party/fft2d/fft2d.BUILD
+tensorflow/third_party/boringssl/BUILD
+tensorflow/third_party/mpi/.gitignore
+tensorflow/third_party/mpi/BUILD
+tensorflow/third_party/tensorrt/LICENSE
+tensorflow/third_party/tensorrt/BUILD
+tensorflow/third_party/tensorrt/build_defs.bzl.tpl
+tensorflow/third_party/tensorrt/BUILD.tpl
+tensorflow/third_party/tensorrt/tensorrt_configure.bzl
+tensorflow/third_party/kafka/config.patch
+tensorflow/third_party/kafka/BUILD
+tensorflow/third_party/android/BUILD
+tensorflow/third_party/android/android.bzl.tpl
+tensorflow/third_party/android/android_configure.bzl
+tensorflow/third_party/android/android_configure.BUILD.tpl
+tensorflow/third_party/tflite_smartreply.BUILD
+tensorflow/third_party/mkl_dnn/LICENSE
+tensorflow/third_party/mkl_dnn/mkldnn.BUILD
+tensorflow/third_party/pcre.BUILD
+tensorflow/third_party/linenoise.BUILD
+tensorflow/third_party/sqlite.BUILD
+tensorflow/third_party/common.bzl
+tensorflow/third_party/com_google_absl.BUILD
+tensorflow/third_party/pprof.BUILD
+tensorflow/third_party/BUILD
+tensorflow/third_party/tflite_mobilenet_quant.BUILD
+tensorflow/third_party/lmdb.BUILD
+tensorflow/third_party/git/BUILD.tpl
+tensorflow/third_party/git/BUILD
+tensorflow/third_party/git/git_configure.bzl
+tensorflow/third_party/protobuf/BUILD
+tensorflow/third_party/tflite_mobilenet.BUILD
+tensorflow/third_party/py/BUILD
+tensorflow/third_party/py/BUILD.tpl
+tensorflow/third_party/py/numpy/BUILD
+tensorflow/third_party/py/python_configure.bzl
+tensorflow/third_party/termcolor.BUILD
+tensorflow/third_party/png_fix_rpi.patch
+tensorflow/third_party/swig.BUILD
+tensorflow/third_party/astor.BUILD
+tensorflow/third_party/grpc/BUILD
+tensorflow/third_party/curl.BUILD
+tensorflow/third_party/arm_neon_2_x86_sse.BUILD
+tensorflow/third_party/png.BUILD
+tensorflow/third_party/googleapis.BUILD
+tensorflow/third_party/mpi_collectives/BUILD
+tensorflow/third_party/nanopb.BUILD
+tensorflow/third_party/gif.BUILD
+tensorflow/third_party/double_conversion.BUILD
+tensorflow/third_party/six.BUILD
+tensorflow/third_party/tflite_mobilenet_float.BUILD
+tensorflow/third_party/repo.bzl
+tensorflow/third_party/codegen.BUILD
+tensorflow/third_party/cub.BUILD
+tensorflow/third_party/jsoncpp.BUILD
+tensorflow/third_party/tflite_ovic_testdata.BUILD
+tensorflow/third_party/libxsmm.BUILD
+tensorflow/third_party/zlib.BUILD
+tensorflow/third_party/eigen.BUILD
+tensorflow/stream_executor/build_defs.bzl
+tensorflow/api_template_v1.__init__.py
+tensorflow/compat_template_v1.__init__.py
 tensorflow/api_template.__init__.py
 tensorflow/__init__.py
\ No newline at end of file
-- 
GitLab


From 25d117af60aba6b92a89de8d69642c12bfc1eae1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 17:52:35 -0800
Subject: [PATCH 1094/2345] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 230447675

---
 tensorflow/go/op/wrappers.go | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 666ebe33df..8e2c98fb07 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -16317,7 +16317,7 @@ func TensorListStackNumElements(value int64) TensorListStackAttr {
 // tensor: the gathered result
 // num_elements: optional. If not -1, the number of elements in the list.
 //
-func TensorListStack(scope *Scope, input_handle tf.Output, element_dtype tf.DataType, optional ...TensorListStackAttr) (tensor tf.Output) {
+func TensorListStack(scope *Scope, input_handle tf.Output, element_shape tf.Output, element_dtype tf.DataType, optional ...TensorListStackAttr) (tensor tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -16328,7 +16328,7 @@ func TensorListStack(scope *Scope, input_handle tf.Output, element_dtype tf.Data
 	opspec := tf.OpSpec{
 		Type: "TensorListStack",
 		Input: []tf.Input{
-			input_handle,
+			input_handle, element_shape,
 		},
 		Attrs: attrs,
 	}
@@ -17359,7 +17359,7 @@ func StringToHashBucketFast(scope *Scope, input tf.Output, num_buckets int64) (o
 // tensor: the withdrawn last element of the list
 // element_dtype: the type of elements in the list
 // element_shape: the shape of the output tensor
-func TensorListPopBack(scope *Scope, input_handle tf.Output, element_dtype tf.DataType) (output_handle tf.Output, tensor tf.Output) {
+func TensorListPopBack(scope *Scope, input_handle tf.Output, element_shape tf.Output, element_dtype tf.DataType) (output_handle tf.Output, tensor tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -17367,7 +17367,7 @@ func TensorListPopBack(scope *Scope, input_handle tf.Output, element_dtype tf.Da
 	opspec := tf.OpSpec{
 		Type: "TensorListPopBack",
 		Input: []tf.Input{
-			input_handle,
+			input_handle, element_shape,
 		},
 		Attrs: attrs,
 	}
@@ -28621,7 +28621,7 @@ func TensorListElementShape(scope *Scope, input_handle tf.Output, shape_type tf.
 // item: the element at that position
 //
 //
-func TensorListGetItem(scope *Scope, input_handle tf.Output, index tf.Output, element_dtype tf.DataType) (item tf.Output) {
+func TensorListGetItem(scope *Scope, input_handle tf.Output, index tf.Output, element_shape tf.Output, element_dtype tf.DataType) (item tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -28629,7 +28629,7 @@ func TensorListGetItem(scope *Scope, input_handle tf.Output, index tf.Output, el
 	opspec := tf.OpSpec{
 		Type: "TensorListGetItem",
 		Input: []tf.Input{
-			input_handle, index,
+			input_handle, index, element_shape,
 		},
 		Attrs: attrs,
 	}
@@ -28781,7 +28781,7 @@ func TensorListSetItem(scope *Scope, input_handle tf.Output, index tf.Output, it
 // input_handle: The input tensor list.
 // indices: The indices used to index into the list.
 // values: The tensor.
-func TensorListGather(scope *Scope, input_handle tf.Output, indices tf.Output, element_dtype tf.DataType) (values tf.Output) {
+func TensorListGather(scope *Scope, input_handle tf.Output, indices tf.Output, element_shape tf.Output, element_dtype tf.DataType) (values tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -28789,7 +28789,7 @@ func TensorListGather(scope *Scope, input_handle tf.Output, indices tf.Output, e
 	opspec := tf.OpSpec{
 		Type: "TensorListGather",
 		Input: []tf.Input{
-			input_handle, indices,
+			input_handle, indices, element_shape,
 		},
 		Attrs: attrs,
 	}
-- 
GitLab


From 1d0552e94d4e091ce03248675ac92c1e1e0accae Mon Sep 17 00:00:00 2001
From: Alan Chiao <alanchiao@google.com>
Date: Tue, 22 Jan 2019 18:09:32 -0800
Subject: [PATCH 1095/2345] Match TFLite argmin/argmax shape propagation to
 Tensorflow. - output shape is now input shape - 1 - handle axis values other
 than axis = 3

PiperOrigin-RevId: 230450280
---
 tensorflow/lite/kernels/arg_min_max.cc        | 12 +++++--
 tensorflow/lite/kernels/arg_min_max_test.cc   | 20 ++++++------
 .../internal/reference/legacy_reference_ops.h | 12 ++++++-
 .../internal/reference/reference_ops.h        | 10 ++----
 .../propagate_fixed_sizes.cc                  | 31 ++++++++++++++++---
 tensorflow/lite/toco/model.h                  |  2 ++
 6 files changed, 62 insertions(+), 25 deletions(-)

diff --git a/tensorflow/lite/kernels/arg_min_max.cc b/tensorflow/lite/kernels/arg_min_max.cc
index eea2de27f7..f9adf6bc4e 100644
--- a/tensorflow/lite/kernels/arg_min_max.cc
+++ b/tensorflow/lite/kernels/arg_min_max.cc
@@ -36,9 +36,15 @@ TfLiteStatus ResizeOutput(TfLiteContext* context, const TfLiteTensor* input,
     axis_value += NumDimensions(input);
   }
 
-  // Copy the input dimensions to output except make the axis dimension 1.
-  TfLiteIntArray* output_dims = TfLiteIntArrayCopy(input->dims);
-  output_dims->data[axis_value] = 1;
+  // Copy the input dimensions to output except the axis dimension.
+  TfLiteIntArray* output_dims = TfLiteIntArrayCreate(NumDimensions(input) - 1);
+  int j = 0;
+  for (int i = 0; i < NumDimensions(input); ++i) {
+    if (i != axis_value) {
+      output_dims->data[j] = SizeOfDimension(input, i);
+      ++j;
+    }
+  }
   return context->ResizeTensor(context, output, output_dims);
 }
 
diff --git a/tensorflow/lite/kernels/arg_min_max_test.cc b/tensorflow/lite/kernels/arg_min_max_test.cc
index dcdff74cc6..1b1000f29e 100644
--- a/tensorflow/lite/kernels/arg_min_max_test.cc
+++ b/tensorflow/lite/kernels/arg_min_max_test.cc
@@ -83,7 +83,7 @@ TEST(ArgMaxOpTest, GetMaxArgFloat) {
   model.Invoke();
 
   EXPECT_THAT(model.GetOutput(), ElementsAreArray({1}));
-  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 1, 1}));
+  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 1}));
 }
 
 TEST(ArgMaxOpTest, GetMaxArgInt) {
@@ -94,7 +94,7 @@ TEST(ArgMaxOpTest, GetMaxArgInt) {
   model.Invoke();
 
   EXPECT_THAT(model.GetOutput(), ElementsAreArray({1}));
-  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 1, 1}));
+  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 1}));
 }
 
 TEST(ArgMaxOpTest, GetMaxArgMulDimensions) {
@@ -105,7 +105,7 @@ TEST(ArgMaxOpTest, GetMaxArgMulDimensions) {
   model.Invoke();
 
   EXPECT_THAT(model.GetOutput(), ElementsAreArray({3, 1}));
-  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 2, 1}));
+  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 2}));
 }
 
 TEST(ArgMaxOpTest, GetMaxArgNegativeAxis) {
@@ -116,7 +116,7 @@ TEST(ArgMaxOpTest, GetMaxArgNegativeAxis) {
   model.Invoke();
 
   EXPECT_THAT(model.GetOutput(), ElementsAreArray({0, 1, 0, 0}));
-  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 1, 4}));
+  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 4}));
 }
 
 TEST(ArgMaxOpTest, GetMaxArgOutput64) {
@@ -127,7 +127,7 @@ TEST(ArgMaxOpTest, GetMaxArgOutput64) {
   model.Invoke();
 
   EXPECT_THAT(model.GetOutput(), ElementsAreArray({0, 1}));
-  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 2, 1}));
+  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 2}));
 }
 
 TEST(ArgMinOpTest, GetMinArgFloat) {
@@ -138,7 +138,7 @@ TEST(ArgMinOpTest, GetMinArgFloat) {
   model.Invoke();
 
   EXPECT_THAT(model.GetOutput(), ElementsAreArray({0}));
-  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 1, 1}));
+  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 1}));
 }
 
 TEST(ArgMinOpTest, GetMinArgInt) {
@@ -149,7 +149,7 @@ TEST(ArgMinOpTest, GetMinArgInt) {
   model.Invoke();
 
   EXPECT_THAT(model.GetOutput(), ElementsAreArray({0}));
-  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 1, 1}));
+  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 1}));
 }
 
 TEST(ArgMinOpTest, GetMinArgMulDimensions) {
@@ -160,7 +160,7 @@ TEST(ArgMinOpTest, GetMinArgMulDimensions) {
   model.Invoke();
 
   EXPECT_THAT(model.GetOutput(), ElementsAreArray({0, 0}));
-  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 2, 1}));
+  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 2}));
 }
 
 TEST(ArgMinOpTest, GetMinArgNegativeAxis) {
@@ -171,7 +171,7 @@ TEST(ArgMinOpTest, GetMinArgNegativeAxis) {
   model.Invoke();
 
   EXPECT_THAT(model.GetOutput(), ElementsAreArray({0, 0, 0, 1}));
-  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 1, 4}));
+  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 4}));
 }
 
 TEST(ArgMinOpTest, GetMinArgOutput64) {
@@ -182,7 +182,7 @@ TEST(ArgMinOpTest, GetMinArgOutput64) {
   model.Invoke();
 
   EXPECT_THAT(model.GetOutput(), ElementsAreArray({1, 0}));
-  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 2, 1}));
+  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 2}));
 }
 
 }  // namespace
diff --git a/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h b/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h
index 380fc8f98e..390bf08e30 100644
--- a/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h
+++ b/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h
@@ -23,6 +23,7 @@ limitations under the License.
 #include "tensorflow/lite/kernels/internal/reference/depthwiseconv_float.h"
 #include "tensorflow/lite/kernels/internal/reference/depthwiseconv_uint8.h"
 #include "tensorflow/lite/kernels/internal/reference/reference_ops.h"
+#include "tensorflow/lite/kernels/internal/types.h"
 
 namespace tflite {
 
@@ -2033,7 +2034,16 @@ template <typename T1, typename T2, typename T3>
 void ArgMax(const T3* axis, const T1* input_data,
             const tflite::Dims<4>& input_dims, T2* output_data,
             const tflite::Dims<4>& output_dims) {
-  ArgMinMax(DimsToShape(input_dims), input_data, axis, DimsToShape(output_dims),
+  // Assumes the input always has 4 dimensions, and therefore,
+  // output always has three dimensions.
+  auto output_shape = RuntimeShape(
+      {output_dims.sizes[2], output_dims.sizes[1], output_dims.sizes[0]});
+  // Another way to interpret this is that output_dims.sizes[4] is always 1.
+  TFLITE_DCHECK_EQ(output_shape.FlatSize(),
+                   DimsToShape(output_dims).FlatSize());
+  // Legacy path only supported this.
+  TFLITE_DCHECK_EQ(axis[0], 3);
+  ArgMinMax(DimsToShape(input_dims), input_data, axis, output_shape,
             output_data, std::greater<T1>());
 }
 
diff --git a/tensorflow/lite/kernels/internal/reference/reference_ops.h b/tensorflow/lite/kernels/internal/reference/reference_ops.h
index 1e2596c852..44772d3003 100644
--- a/tensorflow/lite/kernels/internal/reference/reference_ops.h
+++ b/tensorflow/lite/kernels/internal/reference/reference_ops.h
@@ -3960,11 +3960,8 @@ void ArgMinMax(const RuntimeShape& input1_shape, const T1* input1_data,
                const T3* input2_data, const RuntimeShape& output_shape,
                T2* output_data, const Cmp& cmp) {
   gemmlowp::ScopedProfilingLabel label("ArgMinMax");
-  // For ArgMax, the number of output dimensions = (number of input dimensions -
-  // 1). For the sake of simplicity, the output dimensions are equal to the
-  // input dimensions here. We enforce the constraint that the axis dimension
-  // must always be 1.
-  TFLITE_DCHECK_EQ(input1_shape.DimensionsCount(),
+  TFLITE_DCHECK_GT(input1_shape.DimensionsCount(), 0);
+  TFLITE_DCHECK_EQ(input1_shape.DimensionsCount() - 1,
                    output_shape.DimensionsCount());
 
   int axis = input2_data[0];
@@ -3973,7 +3970,6 @@ void ArgMinMax(const RuntimeShape& input1_shape, const T1* input1_data,
   }
 
   const int axis_size = input1_shape.Dims(axis);
-  TFLITE_DCHECK_EQ(output_shape.Dims(axis), 1);
 
   int outer_size = 1;
   for (int i = 0; i < axis; ++i) {
@@ -3984,7 +3980,7 @@ void ArgMinMax(const RuntimeShape& input1_shape, const T1* input1_data,
   int inner_size = 1;
   const int dims_count = input1_shape.DimensionsCount();
   for (int i = axis + 1; i < dims_count; ++i) {
-    TFLITE_DCHECK_EQ(input1_shape.Dims(i), output_shape.Dims(i));
+    TFLITE_DCHECK_EQ(input1_shape.Dims(i), output_shape.Dims(i - 1));
     inner_size *= input1_shape.Dims(i);
   }
 
diff --git a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
index e33922429b..7077c6c205 100644
--- a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
+++ b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
@@ -1712,14 +1712,37 @@ void ProcessArgMinMaxOperator(Model* model, Op* op) {
     return;
   }
 
+  const Array& axis_array = model->GetArray(op->inputs[1]);
+  // Yield until input axis array shape has been resolved.
+  if (!axis_array.has_shape()) {
+    return;
+  }
+
   const std::vector<int>& input_dims = input_array.shape().dims();
+
+  CHECK(axis_array.data_type == ArrayDataType::kInt32 ||
+        axis_array.data_type == ArrayDataType::kInt64)
+      << "axis_array must be int32, int64";
+
+  CHECK_EQ(RequiredBufferSizeForShape(axis_array.shape()), 1)
+      << "Axis array must be scalar.";
+
+  int64 axis;
+  if (axis_array.data_type == ArrayDataType::kInt32) {
+    axis = axis_array.GetBuffer<ArrayDataType::kInt32>().data[0];
+  } else {
+    axis = axis_array.GetBuffer<ArrayDataType::kInt64>().data[0];
+  }
+
   std::vector<int> output_dims;
 
-  output_dims.reserve(input_dims.size());
-  for (int i = 0; i < input_dims.size() - 1; ++i) {
-    output_dims.push_back(input_dims[i]);
+  output_dims.reserve(input_dims.size() - 1);
+  for (int i = 0; i < input_dims.size(); ++i) {
+    if (i != axis) {
+      output_dims.push_back(input_dims[i]);
+    }
   }
-  output_dims.push_back(1);
+
   const string& output_name = op->outputs[0];
   auto& output_array = model->GetArray(output_name);
   if (output_array.has_shape()) {
diff --git a/tensorflow/lite/toco/model.h b/tensorflow/lite/toco/model.h
index f448e3a897..51087ef92e 100644
--- a/tensorflow/lite/toco/model.h
+++ b/tensorflow/lite/toco/model.h
@@ -1703,6 +1703,7 @@ struct GatherOperator : Operator {
 //
 // Inputs:
 //   inputs[0]: required: the input tensor
+//   inputs[1]: optional: 0-D (scalar) axis
 //
 // TensorFlow equivalent: ArgMax
 struct ArgMaxOperator : Operator {
@@ -1714,6 +1715,7 @@ struct ArgMaxOperator : Operator {
 //
 // Inputs:
 //   inputs[0]: required: the input tensor
+//   inputs[1]: optional: 0-D (scalar) axis
 //
 // TensorFlow equivalent: ArgMin
 struct ArgMinOperator : Operator {
-- 
GitLab


From 81607606202eb57d444b2f6b63ee0798d144d3ad Mon Sep 17 00:00:00 2001
From: Yifei Feng <yifeif@google.com>
Date: Tue, 22 Jan 2019 19:31:22 -0800
Subject: [PATCH 1096/2345] Fix pip_smoke failure in sanity.

PiperOrigin-RevId: 230458127
---
 tensorflow/contrib/eager/python/examples/gan/BUILD            | 4 ++--
 .../contrib/eager/python/examples/linear_regression/BUILD     | 4 ++--
 tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD   | 2 +-
 tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD        | 4 ++--
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/eager/python/examples/gan/BUILD b/tensorflow/contrib/eager/python/examples/gan/BUILD
index 74b742515f..d99a519112 100644
--- a/tensorflow/contrib/eager/python/examples/gan/BUILD
+++ b/tensorflow/contrib/eager/python/examples/gan/BUILD
@@ -27,7 +27,7 @@ cuda_py_test(
     name = "mnist_test",
     srcs = ["mnist_test.py"],
     additional_deps = [
-        ":mnist",
+        ":mnist_lib",
         "//tensorflow/contrib/eager/python:tfe",
         "//tensorflow:tensorflow_py",
     ],
@@ -37,7 +37,7 @@ cuda_py_test(
     name = "mnist_graph_test",
     srcs = ["mnist_graph_test.py"],
     additional_deps = [
-        ":mnist",
+        ":mnist_lib",
         "//third_party/py/numpy",
         "//tensorflow:tensorflow_py",
     ],
diff --git a/tensorflow/contrib/eager/python/examples/linear_regression/BUILD b/tensorflow/contrib/eager/python/examples/linear_regression/BUILD
index 2e7b8a013b..30afef83bc 100644
--- a/tensorflow/contrib/eager/python/examples/linear_regression/BUILD
+++ b/tensorflow/contrib/eager/python/examples/linear_regression/BUILD
@@ -27,7 +27,7 @@ cuda_py_test(
     size = "small",
     srcs = ["linear_regression_test.py"],
     additional_deps = [
-        ":linear_regression",
+        ":linear_regression_lib",
         "//tensorflow:tensorflow_py",
     ],
     tags = [
@@ -41,7 +41,7 @@ cuda_py_test(
     size = "small",
     srcs = ["linear_regression_graph_test.py"],
     additional_deps = [
-        ":linear_regression",
+        ":linear_regression_lib",
         "//tensorflow:tensorflow_py",
     ],
 )
diff --git a/tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD b/tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD
index d4a854fb53..f4dbe7ac16 100644
--- a/tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD
+++ b/tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD
@@ -28,7 +28,7 @@ cuda_py_test(
     name = "rnn_colorbot_test",
     srcs = ["rnn_colorbot_test.py"],
     additional_deps = [
-        ":rnn_colorbot",
+        ":rnn_colorbot_lib",
         "//tensorflow/contrib/eager/python:tfe",
         "//tensorflow:tensorflow_py",
     ],
diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD b/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD
index 1c5a3842d5..43a6ca526d 100644
--- a/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD
+++ b/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD
@@ -28,7 +28,7 @@ cuda_py_test(
     name = "rnn_ptb_test",
     srcs = ["rnn_ptb_test.py"],
     additional_deps = [
-        ":rnn_ptb",
+        ":rnn_ptb_lib",
         "//tensorflow/contrib/eager/python:tfe",
         "//tensorflow:tensorflow_py",
     ],
@@ -39,7 +39,7 @@ cuda_py_test(
     name = "rnn_ptb_graph_test",
     srcs = ["rnn_ptb_graph_test.py"],
     additional_deps = [
-        ":rnn_ptb",
+        ":rnn_ptb_lib",
         "//third_party/py/numpy",
         "//tensorflow:tensorflow_py",
     ],
-- 
GitLab


From 7c2ca5b472908a067980a61c0ac1026003a76b75 Mon Sep 17 00:00:00 2001
From: Dayananda-V <dayanandasiet@gmail.com>
Date: Tue, 22 Jan 2019 10:50:31 +0530
Subject: [PATCH 1097/2345] TF Keras saving_utils_test missing test cases add
 1- extract_model_metrics api test case

---
 .../python/keras/saving/saving_utils_test.py  | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/tensorflow/python/keras/saving/saving_utils_test.py b/tensorflow/python/keras/saving/saving_utils_test.py
index ae267e283c..c307333747 100644
--- a/tensorflow/python/keras/saving/saving_utils_test.py
+++ b/tensorflow/python/keras/saving/saving_utils_test.py
@@ -40,6 +40,7 @@ from tensorflow.python.saved_model import loader
 from tensorflow.python.saved_model import save as save_lib
 from tensorflow.python.saved_model import signature_constants
 from tensorflow.python.saved_model import tag_constants
+from tensorflow.python.training import rmsprop
 
 
 class TraceModelCallTest(keras_parameterized.TestCase):
@@ -205,5 +206,32 @@ class ModelSaveTest(keras_parameterized.TestCase):
         {model.output_names[0]: model.predict_on_batch(inputs)},
         _import_and_infer(save_dir, {model.input_names[0]: np.ones((8, 5))}))
 
+
+class ExtractModelMetricsTest(test.TestCase):
+
+  def test_extract_model_metrics(self):
+    a = keras.layers.Input(shape=(3,), name='input_a')
+    b = keras.layers.Input(shape=(3,), name='input_b')
+
+    dense = keras.layers.Dense(4, name='dense')
+    c = dense(a)
+    d = dense(b)
+    e = keras.layers.Dropout(0.5, name='dropout')(c)
+
+    model = keras.models.Model([a, b], [d, e])
+    extract_metrics = saving_utils.extract_model_metrics(model)
+    self.assertEqual(None, extract_metrics)
+
+    extract_metric_names = ['dense_loss', 'dropout_loss',
+                            'dense_binary_accuracy', 'dropout_binary_accuracy']
+    model_metric_names = ['loss'] + extract_metric_names
+    model.compile(loss='mae',
+                  metrics=[keras.metrics.BinaryAccuracy()],
+                  optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.01),
+                  run_eagerly=None)
+    extract_metrics = saving_utils.extract_model_metrics(model)
+    self.assertEqual(set(model_metric_names), set(model.metrics_names))
+    self.assertEqual(set(extract_metric_names), set(extract_metrics.keys()))
+
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From d5ee710a7f82b6f68a17e0921542b26ad26193f4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 20:19:06 -0800
Subject: [PATCH 1098/2345] Support Reverse_v2

PiperOrigin-RevId: 230461952
---
 tensorflow/lite/build_def.bzl                 |   1 +
 tensorflow/lite/builtin_ops.h                 |   1 +
 .../lite/core/api/flatbuffer_conversions.cc   |   1 +
 tensorflow/lite/kernels/BUILD                 |  13 ++
 .../internal/reference/reference_ops.h        |  27 +++
 tensorflow/lite/kernels/register.cc           |   2 +
 tensorflow/lite/kernels/reverse.cc            | 127 +++++++++++
 tensorflow/lite/kernels/reverse_test.cc       | 199 ++++++++++++++++++
 tensorflow/lite/nnapi_delegate.cc             |   1 +
 tensorflow/lite/schema/schema.fbs             |   5 +
 tensorflow/lite/schema/schema_generated.h     | 124 ++++++++++-
 tensorflow/lite/testing/generate_examples.py  |  31 +++
 tensorflow/lite/toco/export_tensorflow.cc     |  18 ++
 .../propagate_fixed_sizes.cc                  |   1 +
 tensorflow/lite/toco/import_tensorflow.cc     |   1 +
 tensorflow/lite/toco/model.h                  |  13 +-
 tensorflow/lite/toco/tflite/operator.cc       |   2 +
 tensorflow/lite/toco/tflite/operator_test.cc  |   2 +
 tensorflow/lite/toco/tooling_util.cc          |   1 +
 19 files changed, 563 insertions(+), 7 deletions(-)
 create mode 100644 tensorflow/lite/kernels/reverse.cc
 create mode 100644 tensorflow/lite/kernels/reverse_test.cc

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index 43d8e30143..bfac073d87 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -291,6 +291,7 @@ def generated_test_models():
         "relu6",
         "reshape",
         "resize_bilinear",
+        "reverse_v2",
         "rsqrt",
         "shape",
         "sigmoid",
diff --git a/tensorflow/lite/builtin_ops.h b/tensorflow/lite/builtin_ops.h
index 79a2655617..b6ffb826f5 100644
--- a/tensorflow/lite/builtin_ops.h
+++ b/tensorflow/lite/builtin_ops.h
@@ -130,6 +130,7 @@ typedef enum {
   kTfLiteBuiltinSplitV = 102,
   kTfLiteBuiltinUnique = 103,
   kTfLiteBuiltinCeil = 104,
+  kTfLiteBuiltinReverseV2 = 105,
 } TfLiteBuiltinOperator;
 
 #ifdef __cplusplus
diff --git a/tensorflow/lite/core/api/flatbuffer_conversions.cc b/tensorflow/lite/core/api/flatbuffer_conversions.cc
index bcf7680290..3a74b1e6c9 100644
--- a/tensorflow/lite/core/api/flatbuffer_conversions.cc
+++ b/tensorflow/lite/core/api/flatbuffer_conversions.cc
@@ -726,6 +726,7 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type,
     case BuiltinOperator_FLOOR_MOD:
     case BuiltinOperator_RANGE:
     case BuiltinOperator_SQUARED_DIFFERENCE:
+    case BuiltinOperator_REVERSE_V2:
       break;
   }
   return kTfLiteOk;
diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index 765d7a6731..b93dda7382 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -200,6 +200,7 @@ cc_library(
         "reshape.cc",
         "resize_bilinear.cc",
         "resize_nearest_neighbor.cc",
+        "reverse.cc",
         "select.cc",
         "shape.cc",
         "skip_gram.cc",
@@ -1258,6 +1259,18 @@ tf_cc_test(
     ],
 )
 
+tf_cc_test(
+    name = "reverse_test",
+    size = "small",
+    srcs = ["reverse_test.cc"],
+    deps = [
+        ":builtin_ops",
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite/kernels:test_util",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
 filegroup(
     name = "all_files",
     srcs = glob(
diff --git a/tensorflow/lite/kernels/internal/reference/reference_ops.h b/tensorflow/lite/kernels/internal/reference/reference_ops.h
index 44772d3003..125fa95674 100644
--- a/tensorflow/lite/kernels/internal/reference/reference_ops.h
+++ b/tensorflow/lite/kernels/internal/reference/reference_ops.h
@@ -4720,6 +4720,33 @@ void Fill(const RuntimeShape& value_shape, const T* value_data,
   }
 }
 
+template <typename Scalar>
+void Reverse(int axis, const RuntimeShape& input_shape,
+             const Scalar* input_data, const RuntimeShape& output_shape,
+             Scalar* output_data) {
+  gemmlowp::ScopedProfilingLabel label("Reverse");
+
+  int outer_size = 1;
+  for (int i = 0; i < axis; ++i) {
+    outer_size *= input_shape.Dims(i);
+  }
+
+  int copy_size = 1;
+  for (int i = axis + 1; i < input_shape.DimensionsCount(); ++i) {
+    copy_size *= input_shape.Dims(i);
+  }
+
+  const int dims_at_axis = input_shape.Dims(axis);
+  for (int i = 0; i < outer_size; ++i) {
+    for (int j = 0; j < dims_at_axis; ++j) {
+      const int start_pos = (i * dims_at_axis + j) * copy_size;
+      Scalar* output_ptr = output_data + start_pos;
+      int loc = (i * dims_at_axis + dims_at_axis - j - 1) * copy_size;
+      memcpy(output_ptr, input_data + loc, copy_size * sizeof(Scalar));
+    }
+  }
+}
+
 }  // namespace reference_ops
 }  // namespace tflite
 
diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index df4e431fb3..fcfe0b25bb 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -131,6 +131,7 @@ TfLiteRegistration* Register_SQUARED_DIFFERENCE();
 TfLiteRegistration* Register_FILL();
 TfLiteRegistration* Register_MIRROR_PAD();
 TfLiteRegistration* Register_UNIQUE();
+TfLiteRegistration* Register_REVERSE_V2();
 
 TfLiteStatus UnsupportedTensorFlowOp(TfLiteContext* context, TfLiteNode* node) {
   context->ReportError(
@@ -288,6 +289,7 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_FILL, Register_FILL());
   AddBuiltin(BuiltinOperator_MIRROR_PAD, Register_MIRROR_PAD());
   AddBuiltin(BuiltinOperator_UNIQUE, Register_UNIQUE());
+  AddBuiltin(BuiltinOperator_REVERSE_V2, Register_REVERSE_V2());
 
   // TODO(andrewharp, ahentz): Move these somewhere more appropriate so that
   // custom ops aren't always included by default.
diff --git a/tensorflow/lite/kernels/reverse.cc b/tensorflow/lite/kernels/reverse.cc
new file mode 100644
index 0000000000..855aee8df1
--- /dev/null
+++ b/tensorflow/lite/kernels/reverse.cc
@@ -0,0 +1,127 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/c/builtin_op_data.h"
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/kernels/internal/reference/reference_ops.h"
+#include "tensorflow/lite/kernels/internal/tensor.h"
+#include "tensorflow/lite/kernels/kernel_util.h"
+
+namespace tflite {
+namespace ops {
+namespace builtin {
+namespace reverse {
+namespace {
+
+constexpr int kInputTensor = 0;
+constexpr int kAxisTensor = 1;
+constexpr int kOutputTensor = 0;
+
+TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
+  TF_LITE_ENSURE_EQ(context, NumInputs(node), 2);
+  TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);
+
+  const TfLiteTensor* input = GetInput(context, node, kInputTensor);
+  const TfLiteTensor* axis = GetInput(context, node, kAxisTensor);
+  TF_LITE_ENSURE_EQ(context, NumDimensions(axis), 1);
+  TF_LITE_ENSURE(context, NumDimensions(input) >= NumElements(axis));
+
+  if (input->type != kTfLiteInt32 && input->type != kTfLiteFloat32 &&
+      input->type != kTfLiteUInt8 && input->type != kTfLiteInt16 &&
+      input->type != kTfLiteInt64) {
+    context->ReportError(context, "Type '%s' is not supported by reverse.",
+                         TfLiteTypeGetName(input->type));
+    return kTfLiteError;
+  }
+
+  if (axis->type != kTfLiteInt32) {
+    context->ReportError(context, "Axis Type '%s' is not supported by reverse.",
+                         TfLiteTypeGetName(axis->type));
+    return kTfLiteError;
+  }
+
+  // TODO(renjieliu): support multi-axis case.
+  if (NumElements(axis) > 1) {
+    context->ReportError(context, "Current does not support more than 1 axis.");
+  }
+
+  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+  TfLiteIntArray* output_shape = TfLiteIntArrayCopy(input->dims);
+  TF_LITE_ENSURE_EQ(context, output->type, input->type);
+
+  return context->ResizeTensor(context, output, output_shape);
+}
+
+TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
+  const TfLiteTensor* input = GetInput(context, node, kInputTensor);
+  const TfLiteTensor* axis_tensor = GetInput(context, node, kAxisTensor);
+  int axis = GetTensorData<int32_t>(axis_tensor)[0];
+
+  TF_LITE_ENSURE(context, axis >= 0 && axis < NumDimensions(input));
+  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+
+  switch (output->type) {
+    case kTfLiteFloat32: {
+      reference_ops::Reverse<float>(
+          axis, GetTensorShape(input), GetTensorData<float>(input),
+          GetTensorShape(output), GetTensorData<float>(output));
+      break;
+    }
+    case kTfLiteUInt8: {
+      reference_ops::Reverse<uint8_t>(
+          axis, GetTensorShape(input), GetTensorData<uint8_t>(input),
+          GetTensorShape(output), GetTensorData<uint8_t>(output));
+      break;
+    }
+    case kTfLiteInt16: {
+      reference_ops::Reverse<int16_t>(
+          axis, GetTensorShape(input), GetTensorData<int16_t>(input),
+          GetTensorShape(output), GetTensorData<int16_t>(output));
+      break;
+    }
+    case kTfLiteInt32: {
+      reference_ops::Reverse<int32_t>(
+          axis, GetTensorShape(input), GetTensorData<int32_t>(input),
+          GetTensorShape(output), GetTensorData<int32_t>(output));
+      break;
+    }
+    case kTfLiteInt64: {
+      reference_ops::Reverse<int64_t>(
+          axis, GetTensorShape(input), GetTensorData<int64_t>(input),
+          GetTensorShape(output), GetTensorData<int64_t>(output));
+      break;
+    }
+    default: {
+      context->ReportError(context, "Type '%s' is not supported by reverse.",
+                           TfLiteTypeGetName(output->type));
+      return kTfLiteError;
+    }
+  }
+
+  return kTfLiteOk;
+}
+
+}  // namespace
+}  // namespace reverse
+
+TfLiteRegistration* Register_REVERSE_V2() {
+  static TfLiteRegistration r = {nullptr, nullptr, reverse::Prepare,
+                                 reverse::Eval};
+  return &r;
+}
+
+}  // namespace builtin
+}  // namespace ops
+}  // namespace tflite
diff --git a/tensorflow/lite/kernels/reverse_test.cc b/tensorflow/lite/kernels/reverse_test.cc
new file mode 100644
index 0000000000..9bc0c24b64
--- /dev/null
+++ b/tensorflow/lite/kernels/reverse_test.cc
@@ -0,0 +1,199 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/kernels/test_util.h"
+#include "tensorflow/lite/model.h"
+
+namespace tflite {
+namespace {
+
+using ::testing::ElementsAre;
+using ::testing::ElementsAreArray;
+
+template <typename T>
+class ReverseOpModel : public SingleOpModel {
+ public:
+  ReverseOpModel(const TensorData& input, const TensorData& axis) {
+    input_ = AddInput(input);
+    axis_ = AddInput(axis);
+
+    output_ = AddOutput({input.type, {}});
+
+    SetBuiltinOp(BuiltinOperator_REVERSE_V2, BuiltinOptions_ReverseV2Options,
+                 CreateReverseV2Options(builder_).Union());
+    BuildInterpreter({GetShape(input_)});
+  }
+
+  int input() { return input_; }
+  int axis() { return axis_; }
+
+  std::vector<T> GetOutput() { return ExtractVector<T>(output_); }
+  std::vector<int> GetOutputShape() { return GetTensorShape(output_); }
+
+ private:
+  int input_;
+  int axis_;
+  int output_;
+};
+
+// float32 tests.
+TEST(ReverseOpTest, FloatOneDimension) {
+  ReverseOpModel<float> model({TensorType_FLOAT32, {4}},
+                              {TensorType_INT32, {1}});
+  model.PopulateTensor<float>(model.input(), {1, 2, 3, 4});
+  model.PopulateTensor<int32_t>(model.axis(), {0});
+  model.Invoke();
+
+  EXPECT_THAT(model.GetOutputShape(), ElementsAre(4));
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({4, 3, 2, 1}));
+}
+
+TEST(ReverseOpTest, FloatMultiDimensions) {
+  ReverseOpModel<float> model({TensorType_FLOAT32, {4, 3, 2}},
+                              {TensorType_INT32, {1}});
+  model.PopulateTensor<float>(model.input(),
+                              {1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12,
+                               13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24});
+  model.PopulateTensor<int32_t>(model.axis(), {1});
+  model.Invoke();
+
+  EXPECT_THAT(model.GetOutputShape(), ElementsAre(4, 3, 2));
+  EXPECT_THAT(
+      model.GetOutput(),
+      ElementsAreArray({5,  6,  3,  4,  1,  2,  11, 12, 9,  10, 7,  8,
+                        17, 18, 15, 16, 13, 14, 23, 24, 21, 22, 19, 20}));
+}
+
+// int32 tests
+TEST(ReverseOpTest, Int32OneDimension) {
+  ReverseOpModel<int32_t> model({TensorType_INT32, {4}},
+                                {TensorType_INT32, {1}});
+  model.PopulateTensor<int32_t>(model.input(), {1, 2, 3, 4});
+  model.PopulateTensor<int32_t>(model.axis(), {0});
+  model.Invoke();
+
+  EXPECT_THAT(model.GetOutputShape(), ElementsAre(4));
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({4, 3, 2, 1}));
+}
+
+TEST(ReverseOpTest, Int32MultiDimensions) {
+  ReverseOpModel<int32_t> model({TensorType_INT32, {4, 3, 2}},
+                                {TensorType_INT32, {1}});
+  model.PopulateTensor<int32_t>(
+      model.input(), {1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12,
+                      13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24});
+  model.PopulateTensor<int32_t>(model.axis(), {1});
+  model.Invoke();
+
+  EXPECT_THAT(model.GetOutputShape(), ElementsAre(4, 3, 2));
+  EXPECT_THAT(
+      model.GetOutput(),
+      ElementsAreArray({5,  6,  3,  4,  1,  2,  11, 12, 9,  10, 7,  8,
+                        17, 18, 15, 16, 13, 14, 23, 24, 21, 22, 19, 20}));
+}
+
+// int64 tests
+TEST(ReverseOpTest, Int64OneDimension) {
+  ReverseOpModel<int64_t> model({TensorType_INT64, {4}},
+                                {TensorType_INT32, {1}});
+  model.PopulateTensor<int64_t>(model.input(), {1, 2, 3, 4});
+  model.PopulateTensor<int32_t>(model.axis(), {0});
+  model.Invoke();
+
+  EXPECT_THAT(model.GetOutputShape(), ElementsAre(4));
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({4, 3, 2, 1}));
+}
+
+TEST(ReverseOpTest, Int64MultiDimensions) {
+  ReverseOpModel<int64_t> model({TensorType_INT64, {4, 3, 2}},
+                                {TensorType_INT32, {1}});
+  model.PopulateTensor<int64_t>(
+      model.input(), {1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12,
+                      13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24});
+  model.PopulateTensor<int32_t>(model.axis(), {1});
+  model.Invoke();
+
+  EXPECT_THAT(model.GetOutputShape(), ElementsAre(4, 3, 2));
+  EXPECT_THAT(
+      model.GetOutput(),
+      ElementsAreArray({5,  6,  3,  4,  1,  2,  11, 12, 9,  10, 7,  8,
+                        17, 18, 15, 16, 13, 14, 23, 24, 21, 22, 19, 20}));
+}
+
+// uint8 tests
+TEST(ReverseOpTest, Uint8OneDimension) {
+  ReverseOpModel<uint8_t> model({TensorType_UINT8, {4}},
+                                {TensorType_INT32, {1}});
+  model.PopulateTensor<uint8_t>(model.input(), {1, 2, 3, 4});
+  model.PopulateTensor<int32_t>(model.axis(), {0});
+  model.Invoke();
+
+  EXPECT_THAT(model.GetOutputShape(), ElementsAre(4));
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({4, 3, 2, 1}));
+}
+
+TEST(ReverseOpTest, Uint8MultiDimensions) {
+  ReverseOpModel<uint8_t> model({TensorType_UINT8, {4, 3, 2}},
+                                {TensorType_INT32, {1}});
+  model.PopulateTensor<uint8_t>(
+      model.input(), {1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12,
+                      13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24});
+  model.PopulateTensor<int32_t>(model.axis(), {1});
+  model.Invoke();
+
+  EXPECT_THAT(model.GetOutputShape(), ElementsAre(4, 3, 2));
+  EXPECT_THAT(
+      model.GetOutput(),
+      ElementsAreArray({5,  6,  3,  4,  1,  2,  11, 12, 9,  10, 7,  8,
+                        17, 18, 15, 16, 13, 14, 23, 24, 21, 22, 19, 20}));
+}
+
+// int16 tests
+TEST(ReverseOpTest, Int16OneDimension) {
+  ReverseOpModel<int16_t> model({TensorType_INT16, {4}},
+                                {TensorType_INT32, {1}});
+  model.PopulateTensor<int16_t>(model.input(), {1, 2, 3, 4});
+  model.PopulateTensor<int32_t>(model.axis(), {0});
+  model.Invoke();
+
+  EXPECT_THAT(model.GetOutputShape(), ElementsAre(4));
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({4, 3, 2, 1}));
+}
+
+TEST(ReverseOpTest, Int16MultiDimensions) {
+  ReverseOpModel<int16_t> model({TensorType_INT16, {4, 3, 2}},
+                                {TensorType_INT32, {1}});
+  model.PopulateTensor<int16_t>(
+      model.input(), {1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12,
+                      13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24});
+  model.PopulateTensor<int32_t>(model.axis(), {1});
+  model.Invoke();
+
+  EXPECT_THAT(model.GetOutputShape(), ElementsAre(4, 3, 2));
+  EXPECT_THAT(
+      model.GetOutput(),
+      ElementsAreArray({5,  6,  3,  4,  1,  2,  11, 12, 9,  10, 7,  8,
+                        17, 18, 15, 16, 13, 14, 23, 24, 21, 22, 19, 20}));
+}
+
+}  // namespace
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/lite/nnapi_delegate.cc b/tensorflow/lite/nnapi_delegate.cc
index 5a3a21258a..f7cb1588e1 100644
--- a/tensorflow/lite/nnapi_delegate.cc
+++ b/tensorflow/lite/nnapi_delegate.cc
@@ -663,6 +663,7 @@ TfLiteStatus AddOpsAndParams(
       case tflite::BuiltinOperator_SPLIT_V:
       case tflite::BuiltinOperator_UNIQUE:
       case tflite::BuiltinOperator_CEIL:
+      case tflite::BuiltinOperator_REVERSE_V2:
         logError("Op code %d is currently not delegated to NNAPI", builtin);
         return kTfLiteError;
         break;
diff --git a/tensorflow/lite/schema/schema.fbs b/tensorflow/lite/schema/schema.fbs
index 087a87dc81..fbcb18fbeb 100644
--- a/tensorflow/lite/schema/schema.fbs
+++ b/tensorflow/lite/schema/schema.fbs
@@ -218,6 +218,7 @@ enum BuiltinOperator : byte {
   SPLIT_V = 102,
   UNIQUE = 103,
   CEIL = 104,
+  REVERSE_V2 = 105,
 }
 
 // Options for the builtin operators.
@@ -302,6 +303,7 @@ union BuiltinOptions {
   AbsOptions,
   SplitVOptions,
   UniqueOptions,
+  ReverseV2Options,
 }
 
 enum Padding : byte { SAME, VALID }
@@ -719,6 +721,9 @@ table UniqueOptions {
   idx_out_type:TensorType = INT32;
 }
 
+table ReverseV2Options {
+}
+
 // An OperatorCode can be an enum value (BuiltinOperator) if the operator is a
 // builtin, or a string if the operator is custom.
 table OperatorCode {
diff --git a/tensorflow/lite/schema/schema_generated.h b/tensorflow/lite/schema/schema_generated.h
index cfde6c9104..6ad7df050e 100755
--- a/tensorflow/lite/schema/schema_generated.h
+++ b/tensorflow/lite/schema/schema_generated.h
@@ -271,6 +271,9 @@ struct MirrorPadOptionsT;
 struct UniqueOptions;
 struct UniqueOptionsT;
 
+struct ReverseV2Options;
+struct ReverseV2OptionsT;
+
 struct OperatorCode;
 struct OperatorCodeT;
 
@@ -525,11 +528,12 @@ enum BuiltinOperator {
   BuiltinOperator_SPLIT_V = 102,
   BuiltinOperator_UNIQUE = 103,
   BuiltinOperator_CEIL = 104,
+  BuiltinOperator_REVERSE_V2 = 105,
   BuiltinOperator_MIN = BuiltinOperator_ADD,
-  BuiltinOperator_MAX = BuiltinOperator_CEIL
+  BuiltinOperator_MAX = BuiltinOperator_REVERSE_V2
 };
 
-inline const BuiltinOperator (&EnumValuesBuiltinOperator())[104] {
+inline const BuiltinOperator (&EnumValuesBuiltinOperator())[105] {
   static const BuiltinOperator values[] = {
     BuiltinOperator_ADD,
     BuiltinOperator_AVERAGE_POOL_2D,
@@ -634,7 +638,8 @@ inline const BuiltinOperator (&EnumValuesBuiltinOperator())[104] {
     BuiltinOperator_ABS,
     BuiltinOperator_SPLIT_V,
     BuiltinOperator_UNIQUE,
-    BuiltinOperator_CEIL
+    BuiltinOperator_CEIL,
+    BuiltinOperator_REVERSE_V2
   };
   return values;
 }
@@ -746,6 +751,7 @@ inline const char * const *EnumNamesBuiltinOperator() {
     "SPLIT_V",
     "UNIQUE",
     "CEIL",
+    "REVERSE_V2",
     nullptr
   };
   return names;
@@ -838,11 +844,12 @@ enum BuiltinOptions {
   BuiltinOptions_AbsOptions = 78,
   BuiltinOptions_SplitVOptions = 79,
   BuiltinOptions_UniqueOptions = 80,
+  BuiltinOptions_ReverseV2Options = 81,
   BuiltinOptions_MIN = BuiltinOptions_NONE,
-  BuiltinOptions_MAX = BuiltinOptions_UniqueOptions
+  BuiltinOptions_MAX = BuiltinOptions_ReverseV2Options
 };
 
-inline const BuiltinOptions (&EnumValuesBuiltinOptions())[81] {
+inline const BuiltinOptions (&EnumValuesBuiltinOptions())[82] {
   static const BuiltinOptions values[] = {
     BuiltinOptions_NONE,
     BuiltinOptions_Conv2DOptions,
@@ -924,7 +931,8 @@ inline const BuiltinOptions (&EnumValuesBuiltinOptions())[81] {
     BuiltinOptions_MirrorPadOptions,
     BuiltinOptions_AbsOptions,
     BuiltinOptions_SplitVOptions,
-    BuiltinOptions_UniqueOptions
+    BuiltinOptions_UniqueOptions,
+    BuiltinOptions_ReverseV2Options
   };
   return values;
 }
@@ -1012,6 +1020,7 @@ inline const char * const *EnumNamesBuiltinOptions() {
     "AbsOptions",
     "SplitVOptions",
     "UniqueOptions",
+    "ReverseV2Options",
     nullptr
   };
   return names;
@@ -1346,6 +1355,10 @@ template<> struct BuiltinOptionsTraits<UniqueOptions> {
   static const BuiltinOptions enum_value = BuiltinOptions_UniqueOptions;
 };
 
+template<> struct BuiltinOptionsTraits<ReverseV2Options> {
+  static const BuiltinOptions enum_value = BuiltinOptions_ReverseV2Options;
+};
+
 struct BuiltinOptionsUnion {
   BuiltinOptions type;
   void *value;
@@ -2017,6 +2030,14 @@ struct BuiltinOptionsUnion {
     return type == BuiltinOptions_UniqueOptions ?
       reinterpret_cast<const UniqueOptionsT *>(value) : nullptr;
   }
+  ReverseV2OptionsT *AsReverseV2Options() {
+    return type == BuiltinOptions_ReverseV2Options ?
+      reinterpret_cast<ReverseV2OptionsT *>(value) : nullptr;
+  }
+  const ReverseV2OptionsT *AsReverseV2Options() const {
+    return type == BuiltinOptions_ReverseV2Options ?
+      reinterpret_cast<const ReverseV2OptionsT *>(value) : nullptr;
+  }
 };
 
 bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *obj, BuiltinOptions type);
@@ -7113,6 +7134,46 @@ inline flatbuffers::Offset<UniqueOptions> CreateUniqueOptions(
 
 flatbuffers::Offset<UniqueOptions> CreateUniqueOptions(flatbuffers::FlatBufferBuilder &_fbb, const UniqueOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
 
+struct ReverseV2OptionsT : public flatbuffers::NativeTable {
+  typedef ReverseV2Options TableType;
+  ReverseV2OptionsT() {
+  }
+};
+
+struct ReverseV2Options FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef ReverseV2OptionsT NativeTableType;
+  bool Verify(flatbuffers::Verifier &verifier) const {
+    return VerifyTableStart(verifier) &&
+           verifier.EndTable();
+  }
+  ReverseV2OptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const;
+  void UnPackTo(ReverseV2OptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const;
+  static flatbuffers::Offset<ReverseV2Options> Pack(flatbuffers::FlatBufferBuilder &_fbb, const ReverseV2OptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
+};
+
+struct ReverseV2OptionsBuilder {
+  flatbuffers::FlatBufferBuilder &fbb_;
+  flatbuffers::uoffset_t start_;
+  explicit ReverseV2OptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb)
+        : fbb_(_fbb) {
+    start_ = fbb_.StartTable();
+  }
+  ReverseV2OptionsBuilder &operator=(const ReverseV2OptionsBuilder &);
+  flatbuffers::Offset<ReverseV2Options> Finish() {
+    const auto end = fbb_.EndTable(start_);
+    auto o = flatbuffers::Offset<ReverseV2Options>(end);
+    return o;
+  }
+};
+
+inline flatbuffers::Offset<ReverseV2Options> CreateReverseV2Options(
+    flatbuffers::FlatBufferBuilder &_fbb) {
+  ReverseV2OptionsBuilder builder_(_fbb);
+  return builder_.Finish();
+}
+
+flatbuffers::Offset<ReverseV2Options> CreateReverseV2Options(flatbuffers::FlatBufferBuilder &_fbb, const ReverseV2OptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
+
 struct OperatorCodeT : public flatbuffers::NativeTable {
   typedef OperatorCode TableType;
   BuiltinOperator builtin_code;
@@ -7486,6 +7547,9 @@ struct Operator FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
   const UniqueOptions *builtin_options_as_UniqueOptions() const {
     return builtin_options_type() == BuiltinOptions_UniqueOptions ? static_cast<const UniqueOptions *>(builtin_options()) : nullptr;
   }
+  const ReverseV2Options *builtin_options_as_ReverseV2Options() const {
+    return builtin_options_type() == BuiltinOptions_ReverseV2Options ? static_cast<const ReverseV2Options *>(builtin_options()) : nullptr;
+  }
   const flatbuffers::Vector<uint8_t> *custom_options() const {
     return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_CUSTOM_OPTIONS);
   }
@@ -7837,6 +7901,10 @@ template<> inline const UniqueOptions *Operator::builtin_options_as<UniqueOption
   return builtin_options_as_UniqueOptions();
 }
 
+template<> inline const ReverseV2Options *Operator::builtin_options_as<ReverseV2Options>() const {
+  return builtin_options_as_ReverseV2Options();
+}
+
 struct OperatorBuilder {
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
@@ -10484,6 +10552,29 @@ inline flatbuffers::Offset<UniqueOptions> CreateUniqueOptions(flatbuffers::FlatB
       _idx_out_type);
 }
 
+inline ReverseV2OptionsT *ReverseV2Options::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
+  auto _o = new ReverseV2OptionsT();
+  UnPackTo(_o, _resolver);
+  return _o;
+}
+
+inline void ReverseV2Options::UnPackTo(ReverseV2OptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const {
+  (void)_o;
+  (void)_resolver;
+}
+
+inline flatbuffers::Offset<ReverseV2Options> ReverseV2Options::Pack(flatbuffers::FlatBufferBuilder &_fbb, const ReverseV2OptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) {
+  return CreateReverseV2Options(_fbb, _o, _rehasher);
+}
+
+inline flatbuffers::Offset<ReverseV2Options> CreateReverseV2Options(flatbuffers::FlatBufferBuilder &_fbb, const ReverseV2OptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) {
+  (void)_rehasher;
+  (void)_o;
+  struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const ReverseV2OptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va;
+  return tflite::CreateReverseV2Options(
+      _fbb);
+}
+
 inline OperatorCodeT *OperatorCode::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
   auto _o = new OperatorCodeT();
   UnPackTo(_o, _resolver);
@@ -11062,6 +11153,10 @@ inline bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *ob
       auto ptr = reinterpret_cast<const UniqueOptions *>(obj);
       return verifier.VerifyTable(ptr);
     }
+    case BuiltinOptions_ReverseV2Options: {
+      auto ptr = reinterpret_cast<const ReverseV2Options *>(obj);
+      return verifier.VerifyTable(ptr);
+    }
     default: return false;
   }
 }
@@ -11400,6 +11495,10 @@ inline void *BuiltinOptionsUnion::UnPack(const void *obj, BuiltinOptions type, c
       auto ptr = reinterpret_cast<const UniqueOptions *>(obj);
       return ptr->UnPack(resolver);
     }
+    case BuiltinOptions_ReverseV2Options: {
+      auto ptr = reinterpret_cast<const ReverseV2Options *>(obj);
+      return ptr->UnPack(resolver);
+    }
     default: return nullptr;
   }
 }
@@ -11726,6 +11825,10 @@ inline flatbuffers::Offset<void> BuiltinOptionsUnion::Pack(flatbuffers::FlatBuff
       auto ptr = reinterpret_cast<const UniqueOptionsT *>(value);
       return CreateUniqueOptions(_fbb, ptr, _rehasher).Union();
     }
+    case BuiltinOptions_ReverseV2Options: {
+      auto ptr = reinterpret_cast<const ReverseV2OptionsT *>(value);
+      return CreateReverseV2Options(_fbb, ptr, _rehasher).Union();
+    }
     default: return 0;
   }
 }
@@ -12052,6 +12155,10 @@ inline BuiltinOptionsUnion::BuiltinOptionsUnion(const BuiltinOptionsUnion &u) FL
       value = new UniqueOptionsT(*reinterpret_cast<UniqueOptionsT *>(u.value));
       break;
     }
+    case BuiltinOptions_ReverseV2Options: {
+      value = new ReverseV2OptionsT(*reinterpret_cast<ReverseV2OptionsT *>(u.value));
+      break;
+    }
     default:
       break;
   }
@@ -12459,6 +12566,11 @@ inline void BuiltinOptionsUnion::Reset() {
       delete ptr;
       break;
     }
+    case BuiltinOptions_ReverseV2Options: {
+      auto ptr = reinterpret_cast<ReverseV2OptionsT *>(value);
+      delete ptr;
+      break;
+    }
     default: break;
   }
   value = nullptr;
diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index 99aa843afb..3aaaf545df 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -3895,6 +3895,37 @@ def make_unique_tests(zip_path):
       build_inputs,
       expected_tf_success=9)
 
+
+def make_reverse_v2_tests(zip_path):
+  """Make a set of tests to do reverse_v2."""
+
+  test_parameters = [{
+      "base_shape": [[3, 4, 3], [3, 4], [5, 6, 7, 8]],
+      "axis": [0, 1, 2, 3],
+  }]
+
+  def get_valid_axis(parameters):
+    """Return a tweaked version of 'axis'."""
+    axis = parameters["axis"]
+    shape = parameters["base_shape"][:]
+    while axis > len(shape) - 1:
+      axis -= 1
+    return axis
+
+  def build_graph(parameters):
+    input_tensor = tf.placeholder(
+        dtype=tf.float32, name=("input"), shape=parameters["base_shape"])
+    outs = tf.reverse(input_tensor, axis=[get_valid_axis(parameters)])
+    return [input_tensor], [outs]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_value = create_tensor_data(np.float32, shape=parameters["base_shape"])
+    return [input_value], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_value])))
+
+  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+
+
 # Toco binary path provided by the generate rule.
 bin_path = None
 
diff --git a/tensorflow/lite/toco/export_tensorflow.cc b/tensorflow/lite/toco/export_tensorflow.cc
index 816cb9bc28..50a30f5162 100644
--- a/tensorflow/lite/toco/export_tensorflow.cc
+++ b/tensorflow/lite/toco/export_tensorflow.cc
@@ -2062,6 +2062,20 @@ void ConvertZerosLikeOperator(const Model& model,
   (*zeros_like_op->mutable_attr())["T"].set_type(data_type);
 }
 
+void ConvertReverseV2Operator(const Model& model,
+                              const ReverseV2Operator& src_op,
+                              const char* op_name, GraphDef* tensorflow_graph) {
+  tensorflow::NodeDef* reverse_v2_op = tensorflow_graph->add_node();
+  reverse_v2_op->set_op(op_name);
+  reverse_v2_op->set_name(src_op.outputs[0]);
+  DCHECK_EQ(src_op.inputs.size(), 2);
+  *reverse_v2_op->add_input() = src_op.inputs[0];
+  *reverse_v2_op->add_input() = src_op.inputs[1];
+  const tensorflow::DataType data_type =
+      GetTensorFlowDataType(model, src_op.inputs[0]);
+  (*reverse_v2_op->mutable_attr())["T"].set_type(data_type);
+}
+
 void ConvertOperator(const Model& model, const Operator& src_op,
                      GraphDef* tensorflow_graph) {
   if (src_op.fused_activation_function != FusedActivationFunctionType::kNone) {
@@ -2341,6 +2355,10 @@ void ConvertOperator(const Model& model, const Operator& src_op,
     ConvertZerosLikeOperator(
         model, static_cast<const TensorFlowZerosLikeOperator&>(src_op),
         "ZerosLike", tensorflow_graph);
+  } else if (src_op.type == OperatorType::kReverseV2) {
+    ConvertReverseV2Operator(model,
+                             static_cast<const ReverseV2Operator&>(src_op),
+                             "Reverse_V2", tensorflow_graph);
   } else {
     LOG(FATAL) << "Unhandled operator type " << OperatorTypeName(src_op.type);
   }
diff --git a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
index 7077c6c205..eba03dcc14 100644
--- a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
+++ b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
@@ -2009,6 +2009,7 @@ void ProcessUniqueOperator(Model* model, UniqueOperator* op) {
     case OperatorType::kLogicalNot:
     case OperatorType::kLogicalOr:
     case OperatorType::kZerosLike:
+    case OperatorType::kReverseV2:
       ProcessSimpleOperator(model, op, 0);
       break;
     case OperatorType::kGather:
diff --git a/tensorflow/lite/toco/import_tensorflow.cc b/tensorflow/lite/toco/import_tensorflow.cc
index 0f107f8d08..b1b04949ba 100644
--- a/tensorflow/lite/toco/import_tensorflow.cc
+++ b/tensorflow/lite/toco/import_tensorflow.cc
@@ -2459,6 +2459,7 @@ ConverterMapType GetTensorFlowNodeConverterMap() {
       {"Reshape", ConvertSimpleOperator<TensorFlowReshapeOperator, 2, 1>},
       {"ResizeBilinear", ConvertResizeBilinearOperator},
       {"ResizeNearestNeighbor", ConvertResizeNearestNeighborOperator},
+      {"ReverseV2", ConvertSimpleOperator<ReverseV2Operator, 2, 1>},
       {"Rsqrt", ConvertSimpleOperator<TensorFlowRsqrtOperator, 1, 1>},
       {"Select", ConvertSimpleOperator<SelectOperator, 3, 1>},
       {"Shape", ConvertShapeOperator},
diff --git a/tensorflow/lite/toco/model.h b/tensorflow/lite/toco/model.h
index 51087ef92e..84efda111b 100644
--- a/tensorflow/lite/toco/model.h
+++ b/tensorflow/lite/toco/model.h
@@ -161,7 +161,8 @@ enum class OperatorType : uint8 {
   kMirrorPad,
   kUnique,
   kUnidirectionalSequenceRnn,
-  kBidirectionalSequenceLstm
+  kBidirectionalSequenceLstm,
+  kReverseV2
 };
 
 // Helper to deal with TensorFlow arrays using a different ordering of
@@ -1958,6 +1959,16 @@ struct TensorFlowZerosLikeOperator : Operator {
   TensorFlowZerosLikeOperator() : Operator(OperatorType::kZerosLike) {}
 };
 
+// ReverseV2 operator:
+//
+// Inputs:
+// Inputs[0]: required: the input array.
+//
+// TensorFlow equivalent: ReverseV2.
+struct ReverseV2Operator : Operator {
+  ReverseV2Operator() : Operator(OperatorType::kReverseV2) {}
+};
+
 enum class MirrorPadMode { kNone, kSymmetric, kReflect };
 
 // MirrorPad Operator:
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index cd11ba3779..e26aeea010 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -2036,6 +2036,8 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
       MakeUnique<SimpleOperator<AbsOperator>>("ABS", OperatorType::kAbs));
   ops.push_back(
       MakeUnique<SimpleOperator<FillOperator>>("FILL", OperatorType::kFill));
+  ops.push_back(MakeUnique<SimpleOperator<ReverseV2Operator>>(
+      "REVERSE_V2", OperatorType::kReverseV2));
   return ops;
 }
 }  // namespace
diff --git a/tensorflow/lite/toco/tflite/operator_test.cc b/tensorflow/lite/toco/tflite/operator_test.cc
index 760f6b79c5..52f10c2032 100644
--- a/tensorflow/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/lite/toco/tflite/operator_test.cc
@@ -151,6 +151,8 @@ TEST_F(OperatorTest, SimpleOperators) {
   CheckSimpleOperator<FloorModOperator>("FLOOR_MOD", OperatorType::kFloorMod);
   CheckSimpleOperator<RangeOperator>("RANGE", OperatorType::kRange);
   CheckSimpleOperator<FillOperator>("FILL", OperatorType::kFill);
+  CheckSimpleOperator<ReverseV2Operator>("REVERSE_V2",
+                                         OperatorType::kReverseV2);
 }
 
 TEST_F(OperatorTest, BuiltinAdd) {
diff --git a/tensorflow/lite/toco/tooling_util.cc b/tensorflow/lite/toco/tooling_util.cc
index ad83fe4fb0..b2c8e0e132 100644
--- a/tensorflow/lite/toco/tooling_util.cc
+++ b/tensorflow/lite/toco/tooling_util.cc
@@ -420,6 +420,7 @@ const char* OperatorTypeName(OperatorType type) {
     HANDLE_OPERATORTYPENAME_CASE(MirrorPad)
     HANDLE_OPERATORTYPENAME_CASE(Unique)
     HANDLE_OPERATORTYPENAME_CASE(UnidirectionalSequenceRnn)
+    HANDLE_OPERATORTYPENAME_CASE(ReverseV2)
     default:
       LOG(FATAL) << "Unhandled op type";
 #undef HANDLE_OPERATORTYPENAME_CASE
-- 
GitLab


From 59931a8956263293be02884f195745afa2125246 Mon Sep 17 00:00:00 2001
From: Ruoxin Sang <rxsang@google.com>
Date: Tue, 22 Jan 2019 20:35:24 -0800
Subject: [PATCH 1099/2345] Support dynamic padding in tpu.replicate API.

PiperOrigin-RevId: 230463203
---
 tensorflow/compiler/tf2xla/xla_compiler.cc    |  67 +++++--
 tensorflow/compiler/tf2xla/xla_compiler.h     |   6 +-
 tensorflow/contrib/tpu/BUILD                  |   1 +
 tensorflow/contrib/tpu/ops/replication_ops.cc |   2 +
 tensorflow/contrib/tpu/proto/BUILD            |   9 +
 .../contrib/tpu/proto/dynamic_padding.proto   |  19 ++
 tensorflow/contrib/tpu/python/tpu/tpu.py      | 172 ++++++++++++++++--
 7 files changed, 245 insertions(+), 31 deletions(-)
 create mode 100644 tensorflow/contrib/tpu/proto/dynamic_padding.proto

diff --git a/tensorflow/compiler/tf2xla/xla_compiler.cc b/tensorflow/compiler/tf2xla/xla_compiler.cc
index 15fd265686..c5c5b80aa1 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler.cc
@@ -646,14 +646,15 @@ Status XlaCompiler::BuildArguments(
     bool use_tuple_arg, xla::XlaBuilder* builder, XlaContext* context,
     const std::map<int, int>& arg_cores,
     std::vector<XlaExpression>* arg_expressions,
-    std::vector<int>* input_mapping, std::vector<xla::Shape>* input_shapes,
+    std::vector<int>* input_to_args, std::vector<xla::Shape>* input_shapes,
     bool is_entry_computation) {
   arg_expressions->resize(args.size());
 
   // Argument numbers of arguments and resources that are to be passed to the
-  // XLA computation as runtime parameters.
-  input_mapping->clear();
-  input_mapping->reserve(args.size());
+  // XLA computation as runtime parameters. `input_to_args[a] = b` means that
+  // the a'th XLA input corresponds to the b'th original arg indexes.
+  input_to_args->clear();
+  input_to_args->reserve(args.size());
 
   // Fills in constant arguments, and computes non-constant argument order.
   for (std::vector<XlaCompiler::Argument>::size_type i = 0; i < args.size();
@@ -674,13 +675,13 @@ Status XlaCompiler::BuildArguments(
                 /*tensor_array_multiple_writes_aggregate=*/true));
         arg_expression = XlaExpression::Resource(resource);
         if (arg.initialized) {
-          input_mapping->push_back(i);
+          input_to_args->push_back(i);
         }
         break;
       }
       case XlaCompiler::Argument::kParameter:
       case XlaCompiler::Argument::kToken: {
-        input_mapping->push_back(i);
+        input_to_args->push_back(i);
         break;
       }
       case XlaCompiler::Argument::kConstant:
@@ -692,15 +693,23 @@ Status XlaCompiler::BuildArguments(
     }
   }
 
-  if (input_mapping->empty()) {
+  if (input_to_args->empty()) {
     return Status::OK();
   }
 
-  std::vector<xla::Shape> arg_shapes(input_mapping->size());
-  for (std::vector<int>::size_type i = 0; i < input_mapping->size(); ++i) {
+  // `arg_to_inputs[c] = d` means that the c'th original arg index corresponds
+  // to the d'th XLA input. Note that the value -1 corresponds to constants, or
+  // other args that don't correspond to an input.
+  std::vector<int> arg_to_inputs(args.size(), -1);
+  for (int i = 0; i < input_to_args->size(); i++) {
+    arg_to_inputs[input_to_args->at(i)] = i;
+  }
+
+  std::vector<xla::Shape> arg_shapes(input_to_args->size());
+  for (std::vector<int>::size_type i = 0; i < input_to_args->size(); ++i) {
     // Computes the shapes of non-constant arguments.
     TF_RETURN_IF_ERROR(XLAShapeForArgument(
-        args[(*input_mapping)[i]], is_entry_computation, &arg_shapes[i]));
+        args[(*input_to_args)[i]], is_entry_computation, &arg_shapes[i]));
   }
 
   if (use_tuple_arg) {
@@ -717,13 +726,13 @@ Status XlaCompiler::BuildArguments(
   builder->SetOpMetadata(arg_metadata);
 
   // Build parameter handles for non-constant arguments.
-  std::vector<xla::XlaOp> arg_handles(input_mapping->size());
+  std::vector<xla::XlaOp> arg_handles(input_to_args->size());
   if (use_tuple_arg) {
     xla::XlaOp tuple;
     if (is_entry_computation) {
       xla::OpSharding tuple_sharding;
       tuple_sharding.set_type(xla::OpSharding::Type::OpSharding_Type_TUPLE);
-      for (int64 parameter : *input_mapping) {
+      for (int64 parameter : *input_to_args) {
         auto it = arg_cores.find(parameter);
         const int core = it == arg_cores.end() ? 0 : it->second;
         *tuple_sharding.add_tuple_shardings() =
@@ -735,7 +744,7 @@ Status XlaCompiler::BuildArguments(
     } else {
       tuple = xla::Parameter(builder, 0, (*input_shapes)[0], "arg_tuple");
     }
-    for (std::vector<int>::size_type i = 0; i < input_mapping->size(); ++i) {
+    for (std::vector<int>::size_type i = 0; i < input_to_args->size(); ++i) {
       auto it = arg_cores.find(i);
       const int core = it == arg_cores.end() ? -1 : it->second;
       xla::XlaScopedShardingAssignment assign_sharding(
@@ -743,8 +752,19 @@ Status XlaCompiler::BuildArguments(
                               : xla::sharding_builder::AssignDevice(core));
       arg_handles[i] = xla::GetTupleElement(tuple, i);
     }
+
+    for (int i = 0; i < input_to_args->size(); ++i) {
+      const XlaCompiler::Argument& arg = args[input_to_args->at(i)];
+      for (const auto& dim_and_arg_num : arg.dynamic_dim_to_arg_num_map) {
+        int dynamic_size_param_index = arg_to_inputs.at(dim_and_arg_num.second);
+        TF_RETURN_IF_ERROR(builder->SetDynamicBinding(
+            /*dynamic_size_param_num=*/0, {dynamic_size_param_index},
+            /*target_param_num=*/0, /*target_param_index=*/{i},
+            dim_and_arg_num.first));
+      }
+    }
   } else {
-    for (std::vector<int>::size_type i = 0; i < input_mapping->size(); ++i) {
+    for (std::vector<int>::size_type i = 0; i < input_to_args->size(); ++i) {
       auto it = arg_cores.find(i);
       const int core = it == arg_cores.end() ? -1 : it->second;
       xla::XlaScopedShardingAssignment assign_sharding(
@@ -753,6 +773,17 @@ Status XlaCompiler::BuildArguments(
       arg_handles[i] = xla::Parameter(builder, i, (*input_shapes)[i],
                                       absl::StrCat("arg", i));
     }
+
+    for (int i = 0; i < input_to_args->size(); ++i) {
+      const XlaCompiler::Argument& arg = args[input_to_args->at(i)];
+      for (const auto& dim_and_arg_num : arg.dynamic_dim_to_arg_num_map) {
+        int dynamic_size_param_index = arg_to_inputs.at(dim_and_arg_num.second);
+        TF_RETURN_IF_ERROR(builder->SetDynamicBinding(
+            /*dynamic_size_param_num=*/dynamic_size_param_index, {},
+            /*target_param_num=*/i, /*target_param_index=*/{},
+            dim_and_arg_num.first));
+      }
+    }
   }
 
   builder->ClearOpMetadata();
@@ -760,12 +791,12 @@ Status XlaCompiler::BuildArguments(
   // Fill in the handles in non-constant arguments, and reshape parameters
   // back to their correct shapes.
   VLOG(2) << "XLA computation inputs:";
-  for (std::vector<int>::size_type i = 0; i < input_mapping->size(); ++i) {
-    const XlaCompiler::Argument& arg = args[input_mapping->at(i)];
+  for (std::vector<int>::size_type i = 0; i < input_to_args->size(); ++i) {
+    const XlaCompiler::Argument& arg = args[input_to_args->at(i)];
     VLOG(2) << "  XLA arg " << i
             << " shape: " << xla::ShapeUtil::HumanString(arg_shapes[i])
-            << " name: " << arg.name << " TF arg " << input_mapping->at(i);
-    XlaExpression& arg_expression = (*arg_expressions)[input_mapping->at(i)];
+            << " name: " << arg.name << " TF arg " << input_to_args->at(i);
+    XlaExpression& arg_expression = (*arg_expressions)[input_to_args->at(i)];
     switch (arg.kind) {
       case XlaCompiler::Argument::kResource: {
         TF_RET_CHECK(arg.initialized);
diff --git a/tensorflow/compiler/tf2xla/xla_compiler.h b/tensorflow/compiler/tf2xla/xla_compiler.h
index 0d801b73a8..2d4cb0bd99 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.h
+++ b/tensorflow/compiler/tf2xla/xla_compiler.h
@@ -157,6 +157,10 @@ class XlaCompiler {
     // as `tensor_array_gradients`.
     std::set<string> tensor_array_gradients;
 
+    // dynamic dims to arg number map. Empty if no dynamic shapes.
+    std::map<int32, int32> dynamic_dim_to_arg_num_map;
+    bool is_pad_arg = false;
+
     bool operator==(const Argument& other) const;
 
     // Returns a human-readable summary of the argument.
@@ -420,7 +424,7 @@ class XlaCompiler {
                         XlaContext* context,
                         const std::map<int, int>& arg_cores,
                         std::vector<XlaExpression>* arg_expressions,
-                        std::vector<int>* input_mapping,
+                        std::vector<int>* input_to_args,
                         std::vector<xla::Shape>* input_shapes,
                         bool is_entry_computation);
 
diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index b3d5c41e7b..faf11afb21 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -319,6 +319,7 @@ py_library(
         "//tensorflow/contrib/cluster_resolver:cluster_resolver_py",
         "//tensorflow/contrib/compiler:xla",
         "//tensorflow/contrib/tpu/proto:compilation_result_proto_py",
+        "//tensorflow/contrib/tpu/proto:dynamic_padding_proto_py",
         "//tensorflow/contrib/tpu/proto:optimization_parameters_proto_py",
         "//tensorflow/contrib/tpu/proto:topology_proto_py",
         "//tensorflow/contrib/tpu/proto:tpu_embedding_configuration_proto_py",
diff --git a/tensorflow/contrib/tpu/ops/replication_ops.cc b/tensorflow/contrib/tpu/ops/replication_ops.cc
index 285e11d92d..d4180d1a20 100644
--- a/tensorflow/contrib/tpu/ops/replication_ops.cc
+++ b/tensorflow/contrib/tpu/ops/replication_ops.cc
@@ -31,6 +31,7 @@ REGISTER_OP("TPUReplicateMetadata")
     // Deprecated. Use num_cores_per_replica instead.
     .Attr("computation_shape: list(int) = []")
     .Attr("host_compute_core: list(string) = []")
+    .Attr("padding_map: list(string) = []")
     .SetShapeFn(shape_inference::UnknownShape);
 
 REGISTER_OP("TPUReplicatedInput")
@@ -105,6 +106,7 @@ REGISTER_OP("TPUReplicate")
     .Attr("NumVariables: int >= 0")
     .Attr("Tguaranteed_constants: list(type) >= 0")
     .Attr("output_types: list(type) >= 0")
+    .Attr("padding_map: list(string) = []")
     .Input("inputs: Tinputs")
     .Input("broadcast_inputs: Tbroadcast_inputs")
     .Input("variables: NumVariables * resource")
diff --git a/tensorflow/contrib/tpu/proto/BUILD b/tensorflow/contrib/tpu/proto/BUILD
index c20cab844c..ea98ee25c8 100644
--- a/tensorflow/contrib/tpu/proto/BUILD
+++ b/tensorflow/contrib/tpu/proto/BUILD
@@ -49,6 +49,15 @@ tf_proto_library(
     visibility = ["//visibility:public"],
 )
 
+tf_proto_library(
+    name = "dynamic_padding_proto",
+    srcs = [
+        "dynamic_padding.proto",
+    ],
+    cc_api_version = 2,
+    visibility = ["//visibility:public"],
+)
+
 tf_proto_library_py(
     name = "compilation_result_proto",
     srcs = [
diff --git a/tensorflow/contrib/tpu/proto/dynamic_padding.proto b/tensorflow/contrib/tpu/proto/dynamic_padding.proto
new file mode 100644
index 0000000000..c9ebf18116
--- /dev/null
+++ b/tensorflow/contrib/tpu/proto/dynamic_padding.proto
@@ -0,0 +1,19 @@
+syntax = "proto3";
+
+option cc_enable_arenas = true;
+
+package tensorflow.tpu;
+
+// A mapping between the dynamic shape dimension of an input and the arg that
+// represents the real shape.
+message PaddingMap {
+  // Input arg index with dynamic shapes.
+  int32 arg_index = 1;
+
+  // The dynamic shape dimension index.
+  int32 shape_index = 2;
+
+  // The arg index that dynamic dimension maps to, which represents the value
+  // of the real shape.
+  int32 padding_arg_index = 3;
+}
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu.py b/tensorflow/contrib/tpu/python/tpu/tpu.py
index f8762e86e5..4945e5be07 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu.py
@@ -24,16 +24,20 @@ from six.moves import xrange  # pylint: disable=redefined-builtin
 
 from tensorflow.contrib.compiler import xla
 from tensorflow.contrib.framework.python.framework import experimental
+from tensorflow.contrib.tpu.proto import dynamic_padding_pb2 as dynamic_padding
 from tensorflow.contrib.tpu.python.ops import tpu_ops
 from tensorflow.contrib.tpu.python.tpu import tpu_function
 
 from tensorflow.core.framework import attr_value_pb2
 from tensorflow.python.compat import compat as api_compat
 from tensorflow.python.framework import device as pydev
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import compat
@@ -482,7 +486,8 @@ def replicate(computation,
               inputs=None,
               infeed_queue=None,
               device_assignment=None,
-              name=None):
+              name=None,
+              maximum_shapes=None):
   """Builds a graph operator that runs a replicated TPU computation.
 
   Args:
@@ -503,6 +508,14 @@ def replicate(computation,
       only one core, and there is either only one replica, or the number of
       replicas is equal to the number of cores in the TPU system.
     name: (Deprecated) Does nothing.
+    maximum_shapes: A nested structure of tf.TensorShape representing the shape
+      to which the respective component of each input element in each replica
+      should be padded. Any unknown dimensions (e.g. tf.Dimension(None) in a
+      tf.TensorShape or -1 in a tensor-like object) will be padded to the
+      maximum size of that dimension over all replicas. Note that if the input
+      dimension is already static, we won't do padding on it and we require the
+      maximum_shapes to have the same value or None on that dimension. The
+      structure of `maximum_shapes` needs to be the same as `inputs[0]`.
   Returns:
     A list of outputs, indexed by `[replica_num]` each output can be a nested
     structure same as what computation() returns with a few exceptions.
@@ -519,9 +532,101 @@ def replicate(computation,
     ValueError: If all replicas do not have equal numbers of input tensors.
     ValueError: If the number of inputs per replica does not match
       the number of formal parameters to `computation`.
+    ValueError: If the static `inputs` dimensions don't match with the values
+      given in `maximum_shapes`.
+    ValueError: If the structure of inputs per replica does not match
+      the structure of `maximum_shapes`.
   """
-  return split_compile_and_replicate(computation, inputs, infeed_queue,
-                                     device_assignment, name)[1]
+  return split_compile_and_replicate(
+      computation,
+      inputs,
+      infeed_queue,
+      device_assignment,
+      name,
+      maximum_shapes=maximum_shapes)[1]
+
+
+def _pad_all_input(inputs, padded_shapes):
+  """Pad all input tensors given padded_shapes.
+
+  The real shape tensors will be concatenated with the padded original inputs.
+
+  Args:
+    inputs: The original inputs.
+    padded_shapes: A list of padded shapes for each input.
+
+  Returns:
+    The padded inputs and a PaddingMap list which maps the padded input
+    dimension to the real shape argument index.
+  """
+  input_shape_tensors = []
+  for core_idx, inputs_per_core in enumerate(inputs):
+    for idx, input_tensor in enumerate(inputs_per_core):
+      if core_idx == 0:
+        input_shape_tensors.append([])
+      input_shape_tensors[idx].append(array_ops.shape(input_tensor))
+
+  maximum_shapes = []
+  for shapes_per_input in input_shape_tensors:
+    maximum_shapes.append(
+        math_ops.reduce_max(array_ops.stack(shapes_per_input), axis=0))
+
+  padded_inputs = []
+  real_shapes = []
+  padding_maps = []
+  for core_idx, inputs_per_core in enumerate(inputs):
+    padded_inputs.append([])
+    real_shapes.append([])
+    real_shape_idx = len(inputs_per_core) - 1
+    for idx, input_tensor in enumerate(inputs_per_core):
+      input_shape_tensor = input_shape_tensors[idx][core_idx]
+      input_shape = input_tensor.get_shape()
+      padded_shape = padded_shapes[idx]
+
+      # The static shape of inputs should be compatible with the given padded
+      # shapes.
+      input_shape.assert_is_compatible_with(padded_shape)
+
+      if input_shape.is_fully_defined():
+        # Do nothing if the shape of the whole tensor is already static.
+        padded_inputs[core_idx].append(input_tensor)
+      else:
+        # Only pad the non static shape dimension.
+        for i, s in enumerate(input_shape):
+          if s.value is None:
+            if core_idx == 0:
+              real_shape_idx += 1
+              padding_map = dynamic_padding.PaddingMap()
+              padding_map.arg_index = idx
+              padding_map.shape_index = i
+              padding_map.padding_arg_index = real_shape_idx
+              padding_maps.append(padding_map)
+            real_shapes[core_idx].append(
+                math_ops.cast(input_shape_tensor[i], dtypes.uint32))
+
+        paddings = []
+        for i, s in enumerate(padded_shape):
+          if input_shape[i].value:
+            # Don't pad if input shape is already static.
+            padding = [0, 0]
+          else:
+            if s.value:
+              # Pad to the given maximum value.
+              padding = [0, s.value - input_shape_tensor[i]]
+            else:
+              # If maximum value is not given, then pad to the maximum dimension
+              # among all the cores.
+              padding = [0, maximum_shapes[idx][i] - input_shape_tensor[i]]
+          paddings.append(padding)
+
+        padded_input = array_ops.pad(input_tensor, paddings)
+        padded_inputs[core_idx].append(padded_input)
+
+  num_replicas = len(padded_inputs)
+  for i in range(num_replicas):
+    padded_inputs[i].extend(real_shapes[i])
+
+  return padded_inputs, padding_maps
 
 
 def split_compile_and_replicate(computation,
@@ -529,7 +634,8 @@ def split_compile_and_replicate(computation,
                                 infeed_queue=None,
                                 device_assignment=None,
                                 name=None,
-                                use_tpu=True):
+                                use_tpu=True,
+                                maximum_shapes=None):
   """Builds graph operators that runs compilation and replicated computation.
 
   This is a lower level interface than replicate that returns a separate compile
@@ -559,6 +665,15 @@ def split_compile_and_replicate(computation,
     use_tpu: When false, the input `computation` is executed on the XLA CPU/GPU
       backends. Currently, only supports a default placement (computation is
       placed on GPU if one is available, and on CPU if not).
+    maximum_shapes: A nested structure of tf.TensorShape representing the shape
+      to which the respective component of each input element in each replica
+      should be padded. Any unknown dimensions (e.g. tf.Dimension(None) in a
+      tf.TensorShape or -1 in a tensor-like object) will be padded to the
+      maximum size of that dimension over all replicas. Note that if the input
+      dimension is already static, we won't do padding on it and we require the
+      maximum_shapes to have the same value or None on that dimension. The
+      structure of `maximum_shapes` needs to be the same as `inputs[0]`.
+
   Returns:
     A list of lists with the first list corresponding to the compile op and the
     second a list of output tensors, indexed by `[replica_num][output_num]`.
@@ -566,6 +681,10 @@ def split_compile_and_replicate(computation,
     ValueError: If all replicas do not have equal numbers of input tensors.
     ValueError: If the number of inputs per replica does not match
       the number of formal parameters to `computation`.
+    ValueError: If the static `inputs` dimensions don't match with the values
+      given in `maximum_shapes`.
+    ValueError: If the structure of inputs per replica does not match
+      the structure of `maximum_shapes`.
   """
   del name
   inputs = [[]] if inputs is None else inputs
@@ -644,13 +763,34 @@ def split_compile_and_replicate(computation,
                for i in inputs[0]]), infeed_queue.number_of_tuple_elements,
                                              arg_error))
 
+  if maximum_shapes:
+    if infeed_queue:
+      raise ValueError(
+          "Dynamic input shapes are not supported with infeed queues")
+
+    # Make sure maximum_shapes has the same structure as inputs.
+    nest.assert_same_structure(inputs[0], maximum_shapes)
+
+    # Flatten padded shapes.
+    flat_maximum_shapes = nest.flatten(maximum_shapes)
+    flat_maximum_shapes = [
+        tensor_shape.TensorShape(s) for s in flat_maximum_shapes
+    ]
+
+    flat_inputs, padding_maps = _pad_all_input(flat_inputs, flat_maximum_shapes)
+
+    serialized_padding_maps = []
+    for padding_map in padding_maps:
+      serialized_padding_maps.append(padding_map.SerializeToString())
+    metadata_kwargs["padding_map"] = serialized_padding_maps
+
   graph = ops.get_default_graph()
 
   # Fan-in: Builds a TPUReplicatedInput node for each input.
-  computation_inputs = []
-  for i in range(0, flat_input_arity):
+  flat_replicated_inputs = []
+  for i in range(0, len(flat_inputs[0])):
     replicas = [flat_inputs[replica][i] for replica in xrange(num_replicas)]
-    computation_inputs.append(
+    flat_replicated_inputs.append(
         tpu_ops.tpu_replicated_input(replicas, name="input{}".format(i)))
 
   cluster_name = graph.unique_name("cluster")
@@ -670,18 +810,26 @@ def split_compile_and_replicate(computation,
       # computation. This is to avoid orphaned TPUReplicatedInput nodes.
       # TODO(phawkins): consider instead pruning unused TPUReplicatedInput
       # and eliding trivial TPUReplicatedInput/TPUReplicatedOutput pairs.
-      computation_inputs = [
+      flat_replicated_inputs = [
           array_ops.identity(x, name="replicated_input_{}".format(i))
-          for i, x in enumerate(computation_inputs)
+          for i, x in enumerate(flat_replicated_inputs)
       ]
-      for i in computation_inputs:
+      for i in flat_replicated_inputs:
         # pylint: disable=protected-access
-        i.op._set_attr("_tpu_input_identity", attr_value_pb2.AttrValue(b=True))
+        # Add an attribute to the identity node so that they could be removed in
+        # encapsulate TPU computation pass if unused. However we don't remove
+        # inputs when dynamic padding is enabled.
+        # TODO(rxsang): Use other ways except argument index in padding_map so
+        # outside compilation can work with dynamic padding correctly.
+        if maximum_shapes is None:
+          i.op._set_attr("_tpu_input_identity",
+                         attr_value_pb2.AttrValue(b=True))
         # pylint: enable=protected-access
 
       # Unflatten the computation inputs to match original input structure.
       computation_inputs = nest.pack_sequence_as(
-          structure=inputs[0], flat_sequence=computation_inputs)
+          structure=inputs[0],
+          flat_sequence=flat_replicated_inputs[:flat_input_arity])
 
       # If there is an infeed queue, adds the dequeued values to the
       # computation's inputs.
-- 
GitLab


From 324866ba4ddde9ebfbe544023471f9daed1ec5e4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 20:46:51 -0800
Subject: [PATCH 1100/2345] Automated rollback of commit
 2e2a3ca8e827138a02da1cf129ab28057b9f9ddd

PiperOrigin-RevId: 230464029
---
 tensorflow/core/kernels/cwise_ops.h | 34 +++--------------------------
 tensorflow/workspace.bzl            |  8 +++----
 2 files changed, 7 insertions(+), 35 deletions(-)

diff --git a/tensorflow/core/kernels/cwise_ops.h b/tensorflow/core/kernels/cwise_ops.h
index e7bc7f86cc..b03ee937ca 100644
--- a/tensorflow/core/kernels/cwise_ops.h
+++ b/tensorflow/core/kernels/cwise_ops.h
@@ -51,15 +51,12 @@ struct scalar_arg_op<std::complex<double>> {
 };
 #endif
 
+#if EIGEN_HAS_CXX11_MATH == 0
 template <typename T>
 struct scalar_asinh_op {
   EIGEN_EMPTY_STRUCT_CTOR(scalar_asinh_op)
   EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T operator()(const T& a) const {
-#if EIGEN_HAS_CXX11_MATH
-    return numext::asinh(a);
-#else
     return std::asinh(a);
-#endif  // EIGEN_HAS_CXX11_MATH
   }
 };
 template <typename T>
@@ -71,11 +68,7 @@ template <typename T>
 struct scalar_acosh_op {
   EIGEN_EMPTY_STRUCT_CTOR(scalar_acosh_op)
   EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T operator()(const T& a) const {
-#if EIGEN_HAS_CXX11_MATH
-    return numext::acosh(a);
-#else
     return std::acosh(a);
-#endif  // EIGEN_HAS_CXX11_MATH
   }
 };
 template <typename T>
@@ -87,35 +80,14 @@ template <typename T>
 struct scalar_atanh_op {
   EIGEN_EMPTY_STRUCT_CTOR(scalar_atanh_op)
   EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T operator()(const T& a) const {
-#if EIGEN_HAS_CXX11_MATH
-    return numext::atanh(a);
-#else
     return std::atanh(a);
-#endif  // EIGEN_HAS_CXX11_MATH
   }
 };
 template <typename T>
 struct functor_traits<scalar_atanh_op<T>> {
   enum { Cost = 5 * NumTraits<T>::MulCost, PacketAccess = false };
 };
-
-// TODO(rmlarsen): This is a workaround for upstream change
-// https://bitbucket.org/eigen/eigen/commits/f339468d04d0f87caeb6cab9aef568627e9f6ea9
-// that renamed scalar_binary_pow_op to scalar_pow_op and deleted the unary
-// version of the latter. Remove once we upgrade to Eigen 3.3.
-template <typename Scalar, typename Exponent>
-struct scalar_binary_pow_op_google {
-  EIGEN_EMPTY_STRUCT_CTOR(scalar_binary_pow_op_google)
-  EIGEN_DEVICE_FUNC inline Scalar operator()(const Scalar& a,
-                                             const Exponent& b) const {
-    return numext::pow(a, b);
-  }
-};
-
-template <typename Scalar, typename Exponent>
-struct functor_traits<scalar_binary_pow_op_google<Scalar, Exponent>> {
-  enum { Cost = 5 * NumTraits<Scalar>::MulCost, PacketAccess = false };
-};
+#endif
 
 template <typename Scalar, typename Exponent>
 struct safe_scalar_binary_pow_op {
@@ -865,7 +837,7 @@ template <typename T>
 struct floor_div_real : base<T, Eigen::internal::google_floor_div_real<T>> {};
 
 template <typename T>
-struct pow : base<T, Eigen::internal::scalar_binary_pow_op_google<T, T>> {};
+struct pow : base<T, Eigen::internal::scalar_pow_op<T, T>> {};
 
 template <typename T>
 struct safe_pow : base<T, Eigen::internal::safe_scalar_binary_pow_op<T, T>> {
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 49c68b6ec0..466262ab3f 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -140,11 +140,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "eigen_archive",
         build_file = clean_dep("//third_party:eigen.BUILD"),
-        sha256 = "df54d805573871d03aaef6d27d45439c18ff6e8db215a3b922b2daefd6914147",
-        strip_prefix = "eigen-eigen-ad3bcd81cc49",
+        sha256 = "48678550a32665331d729be87076e576f2502fff325f5b6c2c78ebf7b1b22c7b",
+        strip_prefix = "eigen-eigen-bcc817c0ba98",
         urls = [
-            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/ad3bcd81cc49.tar.gz",
-            "https://bitbucket.org/eigen/eigen/get/ad3bcd81cc49.tar.gz",
+            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/bcc817c0ba98.tar.gz",
+            "https://bitbucket.org/eigen/eigen/get/bcc817c0ba98.tar.gz",
         ],
     )
 
-- 
GitLab


From 2b0dafbfd2ee325080bf267cc953b9ec64939614 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 21:08:05 -0800
Subject: [PATCH 1101/2345] Fix the test size

PiperOrigin-RevId: 230465931
---
 tensorflow/contrib/learn/BUILD    | 4 ++--
 tensorflow/contrib/quantize/BUILD | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/learn/BUILD b/tensorflow/contrib/learn/BUILD
index 14065fcee5..4749371248 100644
--- a/tensorflow/contrib/learn/BUILD
+++ b/tensorflow/contrib/learn/BUILD
@@ -357,9 +357,9 @@ py_test(
 
 py_test(
     name = "dnn_linear_combined_test",
-    size = "large",
+    size = "medium",
     srcs = ["python/learn/estimators/dnn_linear_combined_test.py"],
-    shard_count = 4,
+    shard_count = 8,
     srcs_version = "PY2AND3",
     tags = ["no_oss"],  # flaky b/70524820
     deps = [
diff --git a/tensorflow/contrib/quantize/BUILD b/tensorflow/contrib/quantize/BUILD
index b35c4fde1a..b67e68ea96 100644
--- a/tensorflow/contrib/quantize/BUILD
+++ b/tensorflow/contrib/quantize/BUILD
@@ -202,8 +202,9 @@ py_test(
 
 py_test(
     name = "quantize_parameterized_test",
-    size = "large",
+    size = "medium",
     srcs = ["python/quantize_parameterized_test.py"],
+    shard_count = 4,
     srcs_version = "PY2AND3",
     # TODO(b/118839526): Re-enable msan test.
     tags = [
-- 
GitLab


From bba3f6a4e733dfd5865dfb14943624c13a7ba9fd Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Tue, 22 Jan 2019 21:36:00 -0800
Subject: [PATCH 1102/2345] Refuse a v2 argument to tf_export, since that is
 always user error.

PiperOrigin-RevId: 230468088
---
 tensorflow/python/util/tf_export.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/util/tf_export.py b/tensorflow/python/util/tf_export.py
index 90c9c4b5b3..ce1a1a5c30 100644
--- a/tensorflow/python/util/tf_export.py
+++ b/tensorflow/python/util/tf_export.py
@@ -251,7 +251,7 @@ def get_v2_constants(module):
 class api_export(object):  # pylint: disable=invalid-name
   """Provides ways to export symbols to the TensorFlow API."""
 
-  def __init__(self, *args, **kwargs):
+  def __init__(self, *args, **kwargs):  # pylint: disable=g-doc-args
     """Export under the names *args (first one is considered canonical).
 
     Args:
@@ -269,6 +269,10 @@ class api_export(object):  # pylint: disable=invalid-name
     """
     self._names = args
     self._names_v1 = kwargs.get('v1', args)
+    if 'v2' in kwargs:
+      raise ValueError('You passed a "v2" argument to tf_export. This is not '
+                       'what you want. Pass v2 names directly as positional '
+                       'arguments instead.')
     self._api_name = kwargs.get('api_name', TENSORFLOW_API_NAME)
     self._overrides = kwargs.get('overrides', [])
     self._allow_multiple_exports = kwargs.get('allow_multiple_exports', False)
-- 
GitLab


From 617506c8e04eeade1fc5cd75d4006ccbb6f6e55f Mon Sep 17 00:00:00 2001
From: Gautam <gautamrbharadwaj@gmail.com>
Date: Wed, 23 Jan 2019 11:47:33 +0530
Subject: [PATCH 1103/2345] Update gradient_descent.py

removing grad.indices and replacing back grad
---
 tensorflow/python/training/gradient_descent.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/training/gradient_descent.py b/tensorflow/python/training/gradient_descent.py
index b6cceb028d..1a527345ef 100644
--- a/tensorflow/python/training/gradient_descent.py
+++ b/tensorflow/python/training/gradient_descent.py
@@ -56,7 +56,7 @@ class GradientDescentOptimizer(optimizer.Optimizer):
     return training_ops.apply_gradient_descent(
         var,
         math_ops.cast(self._learning_rate_tensor, var.dtype.base_dtype),
-        grad.indices,
+        grad,
         use_locking=self._use_locking).op
 
   def _resource_apply_dense(self, grad, handle):
-- 
GitLab


From 76aa6cf917d719183a82d47970a6f98333740420 Mon Sep 17 00:00:00 2001
From: Dimitris Vardoulakis <dimvar@google.com>
Date: Tue, 22 Jan 2019 22:34:07 -0800
Subject: [PATCH 1104/2345] [TF:XLA] Fix the AR/CRS combiner to avoid crashing
 when two cross-module AllReduces lead to the same cross-replica AllReduce.
 Also, add tests RewriteMultipleAdds and RewriteArSubtractCrs to document
 existing behavior of the pass.

This graph:

A
|
AR    B   C
 \   /    |
   +     AR
    \   /
      +
      |
     CRS

gets rewritten to:

    B  const  C
     \   /    |
 A    Div    AR  const
  \   /      |  /
    +       Div
     \     /
        +
        |
       CRS

Ideally, we would remove both cross-module AllReduces. It's not not straightforward to do that, so I'm leaving it for a separate CL.

PiperOrigin-RevId: 230472380
---
 .../compiler/xla/service/ar_crs_combiner.cc   |  30 +-
 .../compiler/xla/service/ar_crs_combiner.h    |   5 +
 .../xla/service/ar_crs_combiner_test.cc       | 335 ++++++++++++++++++
 3 files changed, 360 insertions(+), 10 deletions(-)

diff --git a/tensorflow/compiler/xla/service/ar_crs_combiner.cc b/tensorflow/compiler/xla/service/ar_crs_combiner.cc
index 8e327ee84b..d01393a748 100644
--- a/tensorflow/compiler/xla/service/ar_crs_combiner.cc
+++ b/tensorflow/compiler/xla/service/ar_crs_combiner.cc
@@ -37,10 +37,12 @@ namespace {
 
 namespace m = match;
 
-// Returns true iff the argument instruction is an AllReduce, followed by a
-// certain sequence of instructions and then a CRS. It must be possible to move
-// the AR past each instruction in the sequence.
-bool MatchesArCrsPattern(HloInstruction* instruction) {
+// Checks if the argument instruction is an AllReduce, followed by a certain
+// sequence of instructions and then a CRS. It must be possible to move
+// the AR past each instruction in the sequence. Returns the CRS, which is the
+// last instruction in the sequence.
+absl::optional<HloInstruction*> MatchesArCrsPattern(
+    HloInstruction* instruction) {
   auto can_ar_move_past_instruction = [](HloInstruction* instruction) -> bool {
     if (instruction->user_count() != 1) {
       return false;
@@ -73,17 +75,19 @@ bool MatchesArCrsPattern(HloInstruction* instruction) {
   if (!instruction->IsCrossModuleAllReduce() ||
       !computation_is_addition(instruction->called_computations()[0]) ||
       instruction->user_count() != 1) {
-    return false;
+    return absl::nullopt;
   }
   auto next = instruction->users()[0];
   while (!next->IsCrossReplicaAllReduce()) {
     if (can_ar_move_past_instruction(next)) {
       next = next->users()[0];
     } else {
-      return false;
+      return absl::nullopt;
     }
   }
-  return computation_is_addition(next->called_computations()[0]);
+  return computation_is_addition(next->called_computations()[0])
+             ? absl::optional<HloInstruction*>(next)
+             : absl::nullopt;
 }
 
 }  // namespace
@@ -99,7 +103,7 @@ absl::optional<HloInstruction*> ArCrsCombiner::WhileFromBodyParameter(
       return caller_instruction;
     }
   }
-  return absl::optional<HloInstruction*>();
+  return absl::nullopt;
 }
 
 std::vector<HloInstruction*> ArCrsCombiner::GetAllTuples(
@@ -231,8 +235,14 @@ bool ArCrsCombiner::InstructionsComputeSameValue(
 void ArCrsCombiner::GroupAllReducesById(HloModule* module) {
   for (HloComputation* computation : module->MakeNonfusionComputations()) {
     for (HloInstruction* instruction : computation->instructions()) {
-      if (MatchesArCrsPattern(instruction)) {
-        all_reduce_map_[*(instruction->all_reduce_id())].push_back(instruction);
+      auto maybe_crs = MatchesArCrsPattern(instruction);
+      if (maybe_crs) {
+        auto crs = *maybe_crs;
+        int64 ar_id = *(instruction->all_reduce_id());
+        if (crs_reserved_map_.find(crs) == crs_reserved_map_.end()) {
+          all_reduce_map_[ar_id].push_back(instruction);
+          crs_reserved_map_[crs] = ar_id;
+        }
       }
     }
   }
diff --git a/tensorflow/compiler/xla/service/ar_crs_combiner.h b/tensorflow/compiler/xla/service/ar_crs_combiner.h
index 6f54b97615..e61ef5d4f9 100644
--- a/tensorflow/compiler/xla/service/ar_crs_combiner.h
+++ b/tensorflow/compiler/xla/service/ar_crs_combiner.h
@@ -83,6 +83,11 @@ class ArCrsCombiner : public HloModulePass {
   // Map from all-reduce ids to the all reduce instructions.
   absl::flat_hash_map<int64, std::vector<HloInstruction*>> all_reduce_map_;
 
+  // Map from a CRS instruction to the all-reduce ID of the AR paired with the
+  // CRS. Sometimes, several ARs in the code could be paired with the same CRS.
+  // We use this map to pick a single AR/CRS path to rewrite.
+  absl::flat_hash_map<HloInstruction*, int64> crs_reserved_map_;
+
   std::unique_ptr<CallGraph> call_graph_;
 };
 
diff --git a/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc b/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc
index 08eb77f032..ea28bba959 100644
--- a/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc
+++ b/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc
@@ -773,5 +773,340 @@ ENTRY %entrycomp (p: f32[]) -> (f32[], f32[]) {
   CompareReplicaGroups(replica_groups_before, replica_groups_after);
 }
 
+TEST_F(ArCrsCombinerTest, RewriteMultipleAdds) {
+  const char* module_str = R"(
+HloModule foobar
+
+%sum (x: f32[], y: f32[]) -> f32[] {
+  %x = f32[] parameter(0)
+  %y = f32[] parameter(1)
+  ROOT %add = f32[] add(%x, %y)
+}
+
+ENTRY %entrycomp (p: f32[]) -> (f32[], f32[]) {
+  %p = f32[] parameter(0)
+  %constant.1 = f32[] constant(1)
+  %constant.2 = f32[] constant(2)
+
+  %all-reduce.ar.1 = f32[]
+      all-reduce(%p),
+      replica_groups={{0},{1}},
+      all_reduce_id=1,
+      to_apply=%sum,
+      sharding={maximal device=0}
+  %add.11 = f32[]
+      add(%constant.1, %all-reduce.ar.1),
+      sharding={maximal device=0}
+  %add.12 = f32[]
+      add(%constant.2, %add.11),
+      sharding={maximal device=0}
+  %all-reduce.1 = f32[]
+      all-reduce(%add.12),
+      replica_groups={{0,1}},
+      to_apply=%sum,
+      sharding={maximal device=0}
+
+  %all-reduce.ar.2 = f32[]
+      all-reduce(%p),
+      replica_groups={{0},{1}},
+      all_reduce_id=1,
+      to_apply=%sum,
+      sharding={maximal device=0}
+  %add.21 = f32[]
+      add(%constant.1, %all-reduce.ar.2),
+      sharding={maximal device=0}
+  %add.22 = f32[]
+      add(%constant.2, %add.21),
+      sharding={maximal device=0}
+  %all-reduce.2 = f32[]
+      all-reduce(%add.22),
+      replica_groups={{0,1}},
+      to_apply=%sum,
+      sharding={maximal device=0}
+
+  ROOT %tuple = (f32[], f32[])
+      tuple(%all-reduce.1, %all-reduce.2),
+      sharding={{maximal device=0}, {maximal device=1}}
+}
+)";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseAndReturnVerifiedModule(module_str));
+  auto crs_before =
+      module->entry_computation()->root_instruction()->operands()[0];
+  auto replica_groups_before = crs_before->replica_groups();
+  ArCrsCombiner combiner(2);
+  auto changed = combiner.Run(module.get()).ValueOrDie();
+  EXPECT_TRUE(changed);
+  EXPECT_THAT(module->entry_computation()->root_instruction(),
+              op::Tuple(op::AllReduce(op::Add(
+                            op::Divide(op::Constant(), op::Constant()),
+                            op::Add(op::Divide(op::Constant(), op::Constant()),
+                                    op::Parameter()))),
+                        op::AllReduce(op::Add(
+                            op::Divide(op::Constant(), op::Constant()),
+                            op::Add(op::Divide(op::Constant(), op::Constant()),
+                                    op::Parameter())))));
+  auto crs_after =
+      module->entry_computation()->root_instruction()->operands()[0];
+  auto replica_groups_after = crs_after->replica_groups();
+  CompareReplicaGroups(replica_groups_before, replica_groups_after);
+}
+
+TEST_F(ArCrsCombinerTest, RewriteArSubtractCrs) {
+  const char* module_str = R"(
+HloModule foobar
+
+%sum.f32 (x: f32[], y: f32[]) -> f32[] {
+  %x = f32[] parameter(0)
+  %y = f32[] parameter(1)
+  ROOT %add = f32[] add(%x, %y)
+}
+
+ENTRY %entrycomp (p: f32[]) -> (f32[], f32[]) {
+  %p = f32[] parameter(0)
+  %constant.f32 = f32[] constant(123)
+
+  %all-reduce.ar.1 = f32[]
+      all-reduce(%p),
+      replica_groups={{0},{1}},
+      all_reduce_id=1,
+      to_apply=%sum.f32,
+      sharding={maximal device=0}
+  %sub.1 = f32[]
+      subtract(%constant.f32, %all-reduce.ar.1),
+      sharding={maximal device=0}
+  %all-reduce.1 = f32[]
+      all-reduce(%sub.1),
+      replica_groups={{0,1}},
+      to_apply=%sum.f32,
+      sharding={maximal device=0}
+
+  %all-reduce.ar.2 = f32[]
+      all-reduce(%p),
+      replica_groups={{0},{1}},
+      all_reduce_id=1,
+      to_apply=%sum.f32,
+      sharding={maximal device=1}
+  %sub.2 = f32[]
+      subtract(%constant.f32, %all-reduce.ar.2),
+      sharding={maximal device=1}
+  %all-reduce.2 = f32[]
+      all-reduce(%sub.2),
+      replica_groups={{0,1}},
+      to_apply=%sum.f32,
+      sharding={maximal device=1}
+
+  ROOT %tuple = (f32[], f32[])
+      tuple(%all-reduce.1, %all-reduce.2),
+      sharding={{maximal device=0}, {maximal device=1}}
+}
+)";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseAndReturnVerifiedModule(module_str));
+  auto crs_before =
+      module->entry_computation()->root_instruction()->operands()[0];
+  auto replica_groups_before = crs_before->replica_groups();
+  ArCrsCombiner combiner(2);
+  auto changed = combiner.Run(module.get()).ValueOrDie();
+  EXPECT_TRUE(changed);
+  EXPECT_THAT(
+      module->entry_computation()->root_instruction(),
+      op::Tuple(
+          op::AllReduce(op::Subtract(op::Divide(op::Constant(), op::Constant()),
+                                     op::Parameter())),
+          op::AllReduce(op::Subtract(op::Divide(op::Constant(), op::Constant()),
+                                     op::Parameter()))));
+  auto crs_after =
+      module->entry_computation()->root_instruction()->operands()[0];
+  auto replica_groups_after = crs_after->replica_groups();
+  CompareReplicaGroups(replica_groups_before, replica_groups_after);
+}
+
+TEST_F(ArCrsCombinerTest, RewriteMultipleARsLeft) {
+  const char* module_str = R"(
+HloModule foobar
+
+%sum (x: f32[], y: f32[]) -> f32[] {
+  %x = f32[] parameter(0)
+  %y = f32[] parameter(1)
+  ROOT %add = f32[] add(%x, %y)
+}
+
+ENTRY %entrycomp (p: f32[]) -> (f32[], f32[]) {
+  %p = f32[] parameter(0)
+  %const1 = f32[] constant(1)
+  %const2 = f32[] constant(2)
+
+  %ar11 = f32[]
+      all-reduce(%p),
+      replica_groups={{0},{1}},
+      all_reduce_id=1,
+      to_apply=%sum,
+      sharding={maximal device=0}
+  %add11 = f32[]
+      add(%ar11, %const1),
+      sharding={maximal device=0}
+  %ar12 = f32[]
+      all-reduce(%p),
+      replica_groups={{0},{1}},
+      all_reduce_id=2,
+      to_apply=%sum,
+      sharding={maximal device=0}
+  %add12 = f32[]
+      add(%add11, %ar12),
+      sharding={maximal device=0}
+  %crs1 = f32[]
+      all-reduce(%add12),
+      replica_groups={{0,1}},
+      to_apply=%sum,
+      sharding={maximal device=0}
+
+  %ar21 = f32[]
+      all-reduce(%p),
+      replica_groups={{0},{1}},
+      all_reduce_id=1,
+      to_apply=%sum,
+      sharding={maximal device=1}
+  %add21 = f32[]
+      add(%ar21, %const1),
+      sharding={maximal device=1}
+  %ar22 = f32[]
+      all-reduce(%p),
+      replica_groups={{0},{1}},
+      all_reduce_id=2,
+      to_apply=%sum,
+      sharding={maximal device=1}
+  %add22 = f32[]
+      add(%add21, %ar22),
+      sharding={maximal device=1}
+  %crs2 = f32[]
+      all-reduce(%add22),
+      replica_groups={{0,1}},
+      to_apply=%sum,
+      sharding={maximal device=1}
+
+  ROOT %tuple = (f32[], f32[])
+      tuple(%crs1, %crs2),
+      sharding={{maximal device=0}, {maximal device=1}}
+}
+)";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseAndReturnVerifiedModule(module_str));
+  auto crs_before =
+      module->entry_computation()->root_instruction()->operands()[0];
+  auto replica_groups_before = crs_before->replica_groups();
+  ArCrsCombiner combiner(2);
+  auto changed = combiner.Run(module.get()).ValueOrDie();
+  EXPECT_TRUE(changed);
+  EXPECT_THAT(module->entry_computation()->root_instruction(),
+              op::Tuple(op::AllReduce(op::Add(
+                            op::Add(op::Parameter(),
+                                    op::Divide(op::Constant(), op::Constant())),
+                            op::Divide(op::AllReduce(), op::Constant()))),
+                        op::AllReduce(op::Add(
+                            op::Add(op::Parameter(),
+                                    op::Divide(op::Constant(), op::Constant())),
+                            op::Divide(op::AllReduce(), op::Constant())))));
+  auto crs_after =
+      module->entry_computation()->root_instruction()->operands()[0];
+  auto replica_groups_after = crs_after->replica_groups();
+  CompareReplicaGroups(replica_groups_before, replica_groups_after);
+}
+
+TEST_F(ArCrsCombinerTest, RewriteMultipleARsRight) {
+  const char* module_str = R"(
+HloModule foobar
+
+%sum (x: f32[], y: f32[]) -> f32[] {
+  %x = f32[] parameter(0)
+  %y = f32[] parameter(1)
+  ROOT %add = f32[] add(%x, %y)
+}
+
+ENTRY %entrycomp (p: f32[]) -> (f32[], f32[]) {
+  %p = f32[] parameter(0)
+  %const1 = f32[] constant(1)
+  %const2 = f32[] constant(2)
+
+  %ar11 = f32[]
+      all-reduce(%p),
+      replica_groups={{0},{1}},
+      all_reduce_id=1,
+      to_apply=%sum,
+      sharding={maximal device=0}
+  %ar12 = f32[]
+      all-reduce(%p),
+      replica_groups={{0},{1}},
+      all_reduce_id=2,
+      to_apply=%sum,
+      sharding={maximal device=0}
+  %add11 = f32[]
+      add(%ar12, %const1),
+      sharding={maximal device=0}
+  %add12 = f32[]
+      add(%ar11, %add11),
+      sharding={maximal device=0}
+  %crs1 = f32[]
+      all-reduce(%add12),
+      replica_groups={{0,1}},
+      to_apply=%sum,
+      sharding={maximal device=0}
+
+  %ar21 = f32[]
+      all-reduce(%p),
+      replica_groups={{0},{1}},
+      all_reduce_id=1,
+      to_apply=%sum,
+      sharding={maximal device=1}
+  %ar22 = f32[]
+      all-reduce(%p),
+      replica_groups={{0},{1}},
+      all_reduce_id=2,
+      to_apply=%sum,
+      sharding={maximal device=1}
+  %add21 = f32[]
+      add(%ar22, %const1),
+      sharding={maximal device=1}
+  %add22 = f32[]
+      add(%ar21, %add21),
+      sharding={maximal device=1}
+  %crs2 = f32[]
+      all-reduce(%add22),
+      replica_groups={{0,1}},
+      to_apply=%sum,
+      sharding={maximal device=1}
+
+  ROOT %tuple = (f32[], f32[])
+      tuple(%crs1, %crs2),
+      sharding={{maximal device=0}, {maximal device=1}}
+}
+)";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseAndReturnVerifiedModule(module_str));
+  auto crs_before =
+      module->entry_computation()->root_instruction()->operands()[0];
+  auto replica_groups_before = crs_before->replica_groups();
+  ArCrsCombiner combiner(2);
+  auto changed = combiner.Run(module.get()).ValueOrDie();
+  EXPECT_TRUE(changed);
+  EXPECT_THAT(module->entry_computation()->root_instruction(),
+              op::Tuple(op::AllReduce(op::Add(
+                            op::Parameter(),
+                            op::Divide(op::Add(op::AllReduce(), op::Constant()),
+                                       op::Constant()))),
+                        op::AllReduce(op::Add(
+                            op::Parameter(),
+                            op::Divide(op::Add(op::AllReduce(), op::Constant()),
+                                       op::Constant())))));
+  auto crs_after =
+      module->entry_computation()->root_instruction()->operands()[0];
+  auto replica_groups_after = crs_after->replica_groups();
+  CompareReplicaGroups(replica_groups_before, replica_groups_after);
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From bba989aa5bb64a2ebfa58007c23c71d644cdbdeb Mon Sep 17 00:00:00 2001
From: Koan-Sin Tan <koansin.tan@gmail.com>
Date: Wed, 23 Jan 2019 15:31:04 +0800
Subject: [PATCH 1105/2345] fix the location of build_ios_universal_lib.sh

---
 tensorflow/lite/tools/benchmark/ios/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/tools/benchmark/ios/README.md b/tensorflow/lite/tools/benchmark/ios/README.md
index fed9e7ea7e..9b1a5ea908 100644
--- a/tensorflow/lite/tools/benchmark/ios/README.md
+++ b/tensorflow/lite/tools/benchmark/ios/README.md
@@ -24,7 +24,7 @@ to build TFLite.
 Running
 
 ```bash
-tensorflow/lite/build_ios_universal_lib.sh
+tensorflow/lite/tools/make/build_ios_universal_lib.sh
 ```
 will also build `tensorflow/lite/gen/lib/benchmark-lib.a` .
 
-- 
GitLab


From 7cfe43a11d95af0df88bf6e84479b6297df7117f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 23:57:28 -0800
Subject: [PATCH 1106/2345] Fix typo.

PiperOrigin-RevId: 230478673
---
 tensorflow/python/distribute/mirrored_strategy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/distribute/mirrored_strategy.py b/tensorflow/python/distribute/mirrored_strategy.py
index c5314e386e..df96f83a58 100644
--- a/tensorflow/python/distribute/mirrored_strategy.py
+++ b/tensorflow/python/distribute/mirrored_strategy.py
@@ -413,7 +413,7 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
   This strategy uses one replica per device and sync replication for its
   multi-GPU version.
 
-  The multi-worker version will be added in the fture.
+  The multi-worker version will be added in the future.
 
   Args:
     devices: a list of device strings.
-- 
GitLab


From d072d4d3a638c3764a05cfefa68df5d0d0a80e57 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 23 Jan 2019 00:28:13 -0800
Subject: [PATCH 1107/2345] Add support for bidirectional_sequence_rnn for
 tf.nn.static_bidirectional_rnn case

PiperOrigin-RevId: 230482907
---
 .../kernels/bidirectional_sequence_rnn.cc     |  99 +++++----
 .../bidirectional_sequence_rnn_test.cc        | 190 ++++++++++++++++--
 2 files changed, 231 insertions(+), 58 deletions(-)

diff --git a/tensorflow/lite/kernels/bidirectional_sequence_rnn.cc b/tensorflow/lite/kernels/bidirectional_sequence_rnn.cc
index e807626108..0adf574bb0 100644
--- a/tensorflow/lite/kernels/bidirectional_sequence_rnn.cc
+++ b/tensorflow/lite/kernels/bidirectional_sequence_rnn.cc
@@ -53,7 +53,10 @@ constexpr int kBwWeightsTensor = 5;
 constexpr int kBwRecurrentWeightsTensor = 6;
 constexpr int kBwBiasTensor = 7;
 constexpr int kBwHiddenStateTensor = 8;
-// Auxiliary inputs.
+// Used as auxiliary input and weights when stacking for
+// tf.contrib.rnn.stack_bidirectional_rnn case (with cross links); Used as input
+// to the backward cell when stacking for tf.nn.static_bidirectional_rnn case
+// (without cross links).
 constexpr int kAuxInputTensor = 9;       // Optional.
 constexpr int kFwAuxWeightsTensor = 10;  // Optional.
 constexpr int kBwAuxWeightsTensor = 11;  // Optional.
@@ -113,13 +116,12 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   const TfLiteTensor* bw_aux_input_weights =
       GetOptionalInputTensor(context, node, kBwAuxWeightsTensor);
 
-  const bool aux_inputs_all_or_none =
-      ((aux_input != nullptr) && (fw_aux_input_weights != nullptr) &&
+  const bool aux_inputs_weights_or_none =
+      ((fw_aux_input_weights != nullptr) &&
        (bw_aux_input_weights != nullptr)) ||
-      ((aux_input == nullptr) && (fw_aux_input_weights == nullptr) &&
-       (bw_aux_input_weights == nullptr));
-  TF_LITE_ENSURE(context, aux_inputs_all_or_none);
-  const bool has_aux_input = (aux_input != nullptr);
+      ((fw_aux_input_weights == nullptr) && (bw_aux_input_weights == nullptr));
+  TF_LITE_ENSURE(context, aux_inputs_weights_or_none);
+  const bool has_aux_input = (fw_aux_input_weights != nullptr);
 
   // Check all the parameters of tensor match within themselves and match the
   // input configuration.
@@ -277,16 +279,19 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   return kTfLiteOk;
 }
 
-TfLiteStatus EvalFloat(
-    const TfLiteTensor* input, const TfLiteTensor* fw_input_weights,
-    const TfLiteTensor* fw_recurrent_weights, const TfLiteTensor* fw_bias,
-    const TfLiteTensor* bw_input_weights,
-    const TfLiteTensor* bw_recurrent_weights, const TfLiteTensor* bw_bias,
-    const TfLiteTensor* aux_input, const TfLiteTensor* fw_aux_input_weights,
-    const TfLiteTensor* bw_aux_input_weights,
-    const TfLiteBidirectionalSequenceRNNParams* params,
-    TfLiteTensor* fw_hidden_state, TfLiteTensor* fw_output,
-    TfLiteTensor* bw_hidden_state, TfLiteTensor* bw_output) {
+TfLiteStatus EvalFloat(const TfLiteTensor* input, const TfLiteTensor* bw_input,
+                       const TfLiteTensor* fw_input_weights,
+                       const TfLiteTensor* fw_recurrent_weights,
+                       const TfLiteTensor* fw_bias,
+                       const TfLiteTensor* bw_input_weights,
+                       const TfLiteTensor* bw_recurrent_weights,
+                       const TfLiteTensor* bw_bias,
+                       const TfLiteTensor* aux_input,
+                       const TfLiteTensor* fw_aux_input_weights,
+                       const TfLiteTensor* bw_aux_input_weights,
+                       const TfLiteBidirectionalSequenceRNNParams* params,
+                       TfLiteTensor* fw_hidden_state, TfLiteTensor* fw_output,
+                       TfLiteTensor* bw_hidden_state, TfLiteTensor* bw_output) {
   const bool time_major = params->time_major;
   const int batch_size =
       (time_major) ? input->dims->data[1] : input->dims->data[0];
@@ -339,7 +344,7 @@ TfLiteStatus EvalFloat(
     float* bw_hidden_state_ptr_batch = bw_hidden_state->data.f;
     for (int s = max_time - 1; s >= 0; s--) {
       const float* input_ptr_batch =
-          input->data.f + s * input_size * batch_size;
+          bw_input->data.f + s * input_size * batch_size;
       const float* aux_input_ptr_batch =
           (aux_input != nullptr)
               ? aux_input->data.f + s * input_size * batch_size
@@ -407,7 +412,8 @@ TfLiteStatus EvalFloat(
 }
 
 TfLiteStatus EvalHybrid(
-    const TfLiteTensor* input, const TfLiteTensor* fw_input_weights,
+    const TfLiteTensor* input, const TfLiteTensor* bw_input,
+    const TfLiteTensor* fw_input_weights,
     const TfLiteTensor* fw_recurrent_weights, const TfLiteTensor* fw_bias,
     const TfLiteTensor* bw_input_weights,
     const TfLiteTensor* bw_recurrent_weights, const TfLiteTensor* bw_bias,
@@ -504,7 +510,7 @@ TfLiteStatus EvalHybrid(
       float* bw_hidden_state_ptr_batch = bw_hidden_state->data.f;
       for (int s = max_time - 1; s >= 0; s--) {
         const float* input_ptr_batch =
-            input->data.f + s * input_size * batch_size;
+            bw_input->data.f + s * input_size * batch_size;
         const float* aux_input_ptr_batch =
             (aux_input != nullptr)
                 ? aux_input->data.f + s * input_size * batch_size
@@ -616,13 +622,35 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
                                 ? nullptr
                                 : GetOutput(context, node, kBwOutputTensor);
 
+  const bool has_previous_bw_output = (aux_input != nullptr);
+  const bool use_aux_input = (fw_aux_input_weights != nullptr);
+
+  // We want to cover the following cases:
+  //
+  // If not stacking (not connected after other bidi lstms):
+  //   both fw & bw will just use `input`; aux_input will be null.
+  //
+  // If stacking with cross_links, TensorFlow equivalent
+  // (tf.contrib.rnn.stack_bidirectional_rnn):
+  //   both fw & bw will use `input`, but aux_input will be none null.
+  //   Note, this time, whether connected after other bidi lstms both works.
+  //
+  // If stacking without cross_links, but connected after other bidi lstms,
+  // TensorFlow equivalent (tf.nn.static_bidirectional_rnn):
+  //   fw will use `input`, bw will use aux_input, and the `real aux_input`
+  //   will be null.
+
+  const bool non_stacking_mode = !use_aux_input && has_previous_bw_output;
+  const TfLiteTensor* bw_input = non_stacking_mode ? aux_input : input;
+  const TfLiteTensor* real_aux_input = non_stacking_mode ? nullptr : aux_input;
+
   switch (fw_input_weights->type) {
     case kTfLiteFloat32:
-      return EvalFloat(input, fw_input_weights, fw_recurrent_weights, fw_bias,
-                       bw_input_weights, bw_recurrent_weights, bw_bias,
-                       aux_input, fw_aux_input_weights, bw_aux_input_weights,
-                       params, fw_hidden_state, fw_output, bw_hidden_state,
-                       bw_output);
+      return EvalFloat(input, bw_input, fw_input_weights, fw_recurrent_weights,
+                       fw_bias, bw_input_weights, bw_recurrent_weights, bw_bias,
+                       real_aux_input, fw_aux_input_weights,
+                       bw_aux_input_weights, params, fw_hidden_state, fw_output,
+                       bw_hidden_state, bw_output);
     case kTfLiteUInt8:
     case kTfLiteInt8: {
       TfLiteTensor* input_quantized =
@@ -634,17 +662,16 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
       TfLiteTensor* scaling_factors =
           GetTemporary(context, node, kScalingFactors);
       TfLiteTensor* aux_input_quantized =
-          (aux_input != nullptr)
-              ? GetTemporary(context, node, kAuxInputQuantized)
-              : nullptr;
-
-      return EvalHybrid(input, fw_input_weights, fw_recurrent_weights, fw_bias,
-                        bw_input_weights, bw_recurrent_weights, bw_bias,
-                        aux_input, fw_aux_input_weights, bw_aux_input_weights,
-                        params, scaling_factors, input_quantized,
-                        aux_input_quantized, fw_hidden_state_quantized,
-                        fw_hidden_state, fw_output, bw_hidden_state_quantized,
-                        bw_hidden_state, bw_output);
+          use_aux_input ? GetTemporary(context, node, kAuxInputQuantized)
+                        : nullptr;
+
+      return EvalHybrid(input, bw_input, fw_input_weights, fw_recurrent_weights,
+                        fw_bias, bw_input_weights, bw_recurrent_weights,
+                        bw_bias, real_aux_input, fw_aux_input_weights,
+                        bw_aux_input_weights, params, scaling_factors,
+                        input_quantized, aux_input_quantized,
+                        fw_hidden_state_quantized, fw_hidden_state, fw_output,
+                        bw_hidden_state_quantized, bw_hidden_state, bw_output);
     }
     default:
       context->ReportError(context, "Type not currently supported.");
diff --git a/tensorflow/lite/kernels/bidirectional_sequence_rnn_test.cc b/tensorflow/lite/kernels/bidirectional_sequence_rnn_test.cc
index 5bad8e02c2..9b61f8238b 100644
--- a/tensorflow/lite/kernels/bidirectional_sequence_rnn_test.cc
+++ b/tensorflow/lite/kernels/bidirectional_sequence_rnn_test.cc
@@ -654,8 +654,8 @@ const std::initializer_list<float> recurrent_weights = {
 class BidirectionalRNNOpModel : public SingleOpModel {
  public:
   BidirectionalRNNOpModel(int batches, int sequence_len, int fw_units,
-                          int bw_units, int input_size, bool time_major,
-                          bool merge_outputs)
+                          int bw_units, int input_size, bool use_aux_input,
+                          bool time_major, bool merge_outputs)
       : batches_(batches),
         sequence_len_(sequence_len),
         fw_units_(fw_units),
@@ -671,7 +671,13 @@ class BidirectionalRNNOpModel : public SingleOpModel {
     bw_bias_ = AddInput(TensorType_FLOAT32);
     bw_hidden_state_ = AddInput(TensorType_FLOAT32, true);
 
-    aux_input_ = AddNullInput();
+    int aux_input_size = 0;
+    if (use_aux_input) {
+      aux_input_ = AddInput(TensorType_FLOAT32);
+      aux_input_size = input_size_;
+    } else {
+      aux_input_ = AddNullInput();
+    }
     aux_fw_weights_ = AddNullInput();
     aux_bw_weights_ = AddNullInput();
 
@@ -691,18 +697,18 @@ class BidirectionalRNNOpModel : public SingleOpModel {
                      : std::vector<int>({batches_, sequence_len_, input_size_});
 
     BuildInterpreter({
-        input_shape,                   // input
-        {fw_units_, input_size_},      // fw_weights
-        {fw_units_, fw_units_},        // fw_recurrent_weights
-        {fw_units_},                   // fw_bias
-        {batches_, fw_units_},         // fw_hidden_state
-        {bw_units_, input_size_},      // bw_weights
-        {bw_units_, bw_units_},        // bw_recurrent_weights
-        {bw_units_},                   // bw_bias
-        {batches_, bw_units_},         // bw_hidden_state
-        {batches_, sequence_len_, 0},  // aux_input
-        {fw_units_, 0},                // aux_fw_weights
-        {bw_units_, 0},                // aux_bw_weights
+        input_shape,                                // input
+        {fw_units_, input_size_},                   // fw_weights
+        {fw_units_, fw_units_},                     // fw_recurrent_weights
+        {fw_units_},                                // fw_bias
+        {batches_, fw_units_},                      // fw_hidden_state
+        {bw_units_, input_size_},                   // bw_weights
+        {bw_units_, bw_units_},                     // bw_recurrent_weights
+        {bw_units_},                                // bw_bias
+        {batches_, bw_units_},                      // bw_hidden_state
+        {batches_, sequence_len_, aux_input_size},  // aux_input
+        {fw_units_, 0},                             // aux_fw_weights
+        {bw_units_, 0},                             // aux_bw_weights
     });
   }
 
@@ -738,6 +744,10 @@ class BidirectionalRNNOpModel : public SingleOpModel {
     PopulateTensor(input_, offset, begin, end);
   }
 
+  void SetAuxInput(int offset, float* begin, float* end) {
+    PopulateTensor(aux_input_, offset, begin, end);
+  }
+
   std::vector<float> GetFwOutput() { return ExtractVector<float>(fw_output_); }
   std::vector<float> GetBwOutput() { return ExtractVector<float>(bw_output_); }
 
@@ -775,7 +785,8 @@ class BidirectionalRNNOpModel : public SingleOpModel {
 TEST(BidirectionalRNNOpTest, BlackBoxTest) {
   BidirectionalRNNOpModel rnn(/*batches=*/2, /*sequence_len=*/16,
                               /*fw_units=*/16, /*bw_units=*/16,
-                              /*input_size=*/8, /*time_major=*/false,
+                              /*input_size=*/8, /*use_aux_input=*/false,
+                              /*time_major=*/false,
                               /*merge_outputs=*/false);
   rnn.SetFwWeights(weights);
   rnn.SetBwWeights(weights);
@@ -813,7 +824,8 @@ TEST(BidirectionalRNNOpTest, BlackBoxTest) {
 TEST(BidirectionalRNNOpTest, BlackBoxTestTimeMajor) {
   BidirectionalRNNOpModel rnn(/*batches=*/2, /*sequence_len=*/16,
                               /*fw_units=*/16, /*bw_units=*/16,
-                              /*input_size=*/8, /*time_major=*/true,
+                              /*input_size=*/8, /*use_aux_input=*/false,
+                              /*time_major=*/true,
                               /*merge_outputs=*/false);
   rnn.SetFwWeights(weights);
   rnn.SetBwWeights(weights);
@@ -822,7 +834,6 @@ TEST(BidirectionalRNNOpTest, BlackBoxTestTimeMajor) {
   rnn.SetFwRecurrentWeights(recurrent_weights);
   rnn.SetBwRecurrentWeights(recurrent_weights);
 
-  // const int input_sequence_size = rnn.input_size() * rnn.sequence_len();
   // Insert the inputs in time_major format. The batch_major format is:
   // [b0t0, b0t1, ..., b0t15, b1t0, b1t1, ..., b1t15]. This is reshuffled as:
   // [b0t0, b1t0, b0t1, b1t1, ..., b0t15, b1t15].
@@ -850,7 +861,8 @@ TEST(BidirectionalRNNOpTest, BlackBoxTestTimeMajor) {
 TEST(BidirectionalRNNOpTest, BlackBoxTestMergeOutputs) {
   BidirectionalRNNOpModel rnn(/*batches=*/2, /*sequence_len=*/16,
                               /*fw_units=*/16, /*bw_units=*/16,
-                              /*input_size=*/8, /*time_major=*/false,
+                              /*input_size=*/8, /*use_aux_input=*/false,
+                              /*time_major=*/false,
                               /*merge_outputs=*/true);
   rnn.SetFwWeights(weights);
   rnn.SetBwWeights(weights);
@@ -888,7 +900,8 @@ TEST(BidirectionalRNNOpTest, BlackBoxTestMergeOutputs) {
 TEST(BidirectionalRNNOpTest, BlackBoxTestTimeMajorMergeOutputs) {
   BidirectionalRNNOpModel rnn(/*batches=*/2, /*sequence_len=*/16,
                               /*fw_units=*/16, /*bw_units=*/16,
-                              /*input_size=*/8, /*time_major=*/true,
+                              /*input_size=*/8, /*use_aux_input=*/false,
+                              /*time_major=*/true,
                               /*merge_outputs=*/true);
   rnn.SetFwWeights(weights);
   rnn.SetBwWeights(weights);
@@ -932,7 +945,8 @@ TEST(BidirectionalRNNOpTest, BlackBoxTestTimeMajorMergeOutputs) {
 TEST(BidirectionalRNNOpTest, BlackBoxTestReverseInputs) {
   BidirectionalRNNOpModel rnn(/*batches=*/2, /*sequence_len=*/16,
                               /*fw_units=*/16, /*bw_units=*/16,
-                              /*input_size=*/8, /*time_major=*/false,
+                              /*input_size=*/8, /*use_aux_input=*/false,
+                              /*time_major=*/false,
                               /*merge_outputs=*/false);
   rnn.SetFwWeights(weights);
   rnn.SetBwWeights(weights);
@@ -979,7 +993,8 @@ TEST(BidirectionalRNNOpTest, BlackBoxTestReverseInputs) {
 TEST(BidirectionalRNNOpTest, EndToEndTest) {
   BidirectionalRNNOpModel rnn(/*batches=*/1, /*sequence_len=*/4,
                               /*fw_units=*/16, /*bw_units=*/16,
-                              /*input_size=*/8, /*time_major=*/false,
+                              /*input_size=*/8, /*use_aux_input=*/false,
+                              /*time_major=*/false,
                               /*merge_outputs=*/false);
   const int output_size = 4;
   float dnn_weights[] = {
@@ -1046,6 +1061,137 @@ TEST(BidirectionalRNNOpTest, EndToEndTest) {
   }
 }
 
+// Same as BlackBox test, but has aux input.
+TEST(BidirectionalRNNOpTest, BlackBoxTestAuxInput) {
+  BidirectionalRNNOpModel rnn(/*batches=*/2, /*sequence_len=*/16,
+                              /*fw_units=*/16, /*bw_units=*/16,
+                              /*input_size=*/8, /*use_aux_input=*/true,
+                              /*time_major=*/true,
+                              /*merge_outputs=*/false);
+  rnn.SetFwWeights(weights);
+  rnn.SetBwWeights(weights);
+  rnn.SetFwBias(biases);
+  rnn.SetBwBias(biases);
+  rnn.SetFwRecurrentWeights(recurrent_weights);
+  rnn.SetBwRecurrentWeights(recurrent_weights);
+
+  // Insert the inputs in time_major format. The batch_major format is:
+  // [b0t0, b0t1, ..., b0t15, b1t0, b1t1, ..., b1t15]. This is reshuffled as:
+  // [b0t0, b1t0, b0t1, b1t1, ..., b0t15, b1t15].
+  for (int i = 0; i < rnn.sequence_len(); i++) {
+    float* batch_start = rnn_input + i * rnn.input_size();
+    float* batch_end = batch_start + rnn.input_size();
+    // The two batches are identical.
+    // Also make aux input the same as input.
+    rnn.SetInput(2 * i * rnn.input_size(), batch_start, batch_end);
+    rnn.SetAuxInput(2 * i * rnn.input_size(), batch_start, batch_end);
+    rnn.SetInput((2 * i + 1) * rnn.input_size(), batch_start, batch_end);
+    rnn.SetAuxInput((2 * i + 1) * rnn.input_size(), batch_start, batch_end);
+  }
+
+  rnn.Invoke();
+
+  std::vector<float> fw_expected;
+  for (int i = 0; i < rnn.sequence_len(); i++) {
+    float* golden_fw_start = rnn_golden_fw_output + i * rnn.num_fw_units();
+    float* golden_fw_end = golden_fw_start + rnn.num_fw_units();
+    fw_expected.insert(fw_expected.end(), golden_fw_start, golden_fw_end);
+    fw_expected.insert(fw_expected.end(), golden_fw_start, golden_fw_end);
+  }
+  EXPECT_THAT(rnn.GetFwOutput(), ElementsAreArray(ArrayFloatNear(fw_expected)));
+}
+
+// Same as previous test, but has aux input is all zeros.
+TEST(BidirectionalRNNOpTest, BlackBoxTestAuxInputZeros) {
+  BidirectionalRNNOpModel rnn(/*batches=*/2, /*sequence_len=*/16,
+                              /*fw_units=*/16, /*bw_units=*/16,
+                              /*input_size=*/8, /*use_aux_input=*/true,
+                              /*time_major=*/true,
+                              /*merge_outputs=*/false);
+  rnn.SetFwWeights(weights);
+  rnn.SetBwWeights(weights);
+  rnn.SetFwBias(biases);
+  rnn.SetBwBias(biases);
+  rnn.SetFwRecurrentWeights(recurrent_weights);
+  rnn.SetBwRecurrentWeights(recurrent_weights);
+
+  // Initialize bw inputs with zeros.
+  std::vector<float> bw_inputs(rnn.sequence_len(), 0);
+
+  // Insert the inputs in time_major format. The batch_major format is:
+  // [b0t0, b0t1, ..., b0t15, b1t0, b1t1, ..., b1t15]. This is reshuffled as:
+  // [b0t0, b1t0, b0t1, b1t1, ..., b0t15, b1t15].
+  for (int i = 0; i < rnn.sequence_len(); i++) {
+    float* batch_start = rnn_input + i * rnn.input_size();
+    float* batch_end = batch_start + rnn.input_size();
+    // The two batches are identical.
+    // Also make aux input the same as input.
+    rnn.SetInput(2 * i * rnn.input_size(), batch_start, batch_end);
+    rnn.SetAuxInput(2 * i * rnn.input_size(), &bw_inputs[0],
+                    &bw_inputs[bw_inputs.size() - 1]);
+    rnn.SetInput((2 * i + 1) * rnn.input_size(), batch_start, batch_end);
+    rnn.SetAuxInput((2 * i + 1) * rnn.input_size(), &bw_inputs[0],
+                    &bw_inputs[bw_inputs.size() - 1]);
+  }
+
+  rnn.Invoke();
+
+  std::vector<float> fw_expected;
+  for (int i = 0; i < rnn.sequence_len(); i++) {
+    float* golden_fw_start = rnn_golden_fw_output + i * rnn.num_fw_units();
+    float* golden_fw_end = golden_fw_start + rnn.num_fw_units();
+    fw_expected.insert(fw_expected.end(), golden_fw_start, golden_fw_end);
+    fw_expected.insert(fw_expected.end(), golden_fw_start, golden_fw_end);
+  }
+  EXPECT_THAT(rnn.GetFwOutput(), ElementsAreArray(ArrayFloatNear(fw_expected)));
+}
+
+// Same as previous test, but has input is all zeros, and aux input is the real
+// input. This is testing the bw path is functional.
+TEST(BidirectionalRNNOpTest, BlackBoxTestAuxInputInputZeros) {
+  BidirectionalRNNOpModel rnn(/*batches=*/2, /*sequence_len=*/16,
+                              /*fw_units=*/16, /*bw_units=*/16,
+                              /*input_size=*/8, /*use_aux_input=*/true,
+                              /*time_major=*/true,
+                              /*merge_outputs=*/false);
+  rnn.SetFwWeights(weights);
+  rnn.SetBwWeights(weights);
+  rnn.SetFwBias(biases);
+  rnn.SetBwBias(biases);
+  rnn.SetFwRecurrentWeights(recurrent_weights);
+  rnn.SetBwRecurrentWeights(recurrent_weights);
+
+  // Initialize bw inputs with zeros.
+  std::vector<float> fw_inputs(rnn.sequence_len(), 0);
+
+  // Insert the inputs in time_major format. The batch_major format is:
+  // [b0t0, b0t1, ..., b0t15, b1t0, b1t1, ..., b1t15]. This is reshuffled as:
+  // [b0t0, b1t0, b0t1, b1t1, ..., b0t15, b1t15].
+  for (int i = 0; i < rnn.sequence_len(); i++) {
+    float* batch_start = rnn_input + i * rnn.input_size();
+    float* batch_end = batch_start + rnn.input_size();
+    // The two batches are identical.
+    // Also make aux input the same as input.
+    rnn.SetAuxInput(2 * i * rnn.input_size(), batch_start, batch_end);
+    rnn.SetInput(2 * i * rnn.input_size(), &fw_inputs[0],
+                 &fw_inputs[fw_inputs.size() - 1]);
+    rnn.SetAuxInput((2 * i + 1) * rnn.input_size(), batch_start, batch_end);
+    rnn.SetInput((2 * i + 1) * rnn.input_size(), &fw_inputs[0],
+                 &fw_inputs[fw_inputs.size() - 1]);
+  }
+
+  rnn.Invoke();
+
+  std::vector<float> bw_expected;
+  for (int i = 0; i < rnn.sequence_len(); i++) {
+    float* golden_bw_start = rnn_golden_bw_output + i * rnn.num_fw_units();
+    float* golden_bw_end = golden_bw_start + rnn.num_fw_units();
+    bw_expected.insert(bw_expected.end(), golden_bw_start, golden_bw_end);
+    bw_expected.insert(bw_expected.end(), golden_bw_start, golden_bw_end);
+  }
+  EXPECT_THAT(rnn.GetBwOutput(), ElementsAreArray(ArrayFloatNear(bw_expected)));
+}
+
 }  // namespace
 }  // namespace tflite
 
-- 
GitLab


From 2283fefab6fbd77da2388f54ebaf860311742fe2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 23 Jan 2019 01:02:31 -0800
Subject: [PATCH 1108/2345] compat: Update forward compatibility horizon to
 2019-01-23

PiperOrigin-RevId: 230486363
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 95940550b4..c8ac0bee44 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 22)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 23)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From fc1a15743702eab5937f26d17f7215de7a81c0cf Mon Sep 17 00:00:00 2001
From: Dayananda-V <dayanandasiet@gmail.com>
Date: Wed, 23 Jan 2019 16:06:12 +0530
Subject: [PATCH 1109/2345] TF Eager profiler_test added test to validate error
 case

---
 tensorflow/python/eager/profiler_test.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tensorflow/python/eager/profiler_test.py b/tensorflow/python/eager/profiler_test.py
index 319d063990..6940f56a98 100644
--- a/tensorflow/python/eager/profiler_test.py
+++ b/tensorflow/python/eager/profiler_test.py
@@ -33,11 +33,16 @@ class ProfilerTest(test_util.TensorFlowTestCase):
     five = constant_op.constant(5)
     product = three * five
     self.assertAllEqual(15, product)
+    with self.assertRaises(AssertionError):
+      profiler.start()
+
     profile_result = profiler.stop()
     profile_pb = trace_events_pb2.Trace()
     profile_pb.ParseFromString(profile_result)
     profile_pb_str = '%s' % profile_pb
     self.assertTrue('Mul' in profile_pb_str)
+    with self.assertRaises(AssertionError):
+      profiler.stop()
 
 
 if __name__ == '__main__':
-- 
GitLab


From 939f9651a7c6e1c8e8dd02380909fce22fa34a66 Mon Sep 17 00:00:00 2001
From: Mark Ryan <mark.d.ryan@intel.com>
Date: Tue, 22 Jan 2019 15:43:50 +0100
Subject: [PATCH 1110/2345] Fix init_ops_test on AVX512 builds

The init_ops_tests unit test is failing on AVX512 builds due
to a hard coded packet size in eigen_contraction_kernel.h.
This commit fixes the issue by determining the appropriate
packet size using internal::packet_traits.  It maintains
the minimum packet size of 8 used in the existing code and
so should have no effect on non AVX512 builds.

Fixes: https://github.com/tensorflow/tensorflow/issues/25127

Signed-off-by: Mark Ryan <mark.d.ryan@intel.com>
---
 tensorflow/core/kernels/eigen_contraction_kernel.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/eigen_contraction_kernel.h b/tensorflow/core/kernels/eigen_contraction_kernel.h
index 05a3ae07a1..4089eec59e 100644
--- a/tensorflow/core/kernels/eigen_contraction_kernel.h
+++ b/tensorflow/core/kernels/eigen_contraction_kernel.h
@@ -197,7 +197,8 @@ class TensorContractionBlocking<float, float, float, StorageIndex,
     // We split Kth dimensions in roughly equal slices.
     StorageIndex target_k_slices =
         (std::max)(StorageIndex(1), Eigen::divup(k, kc_));
-    StorageIndex packet_size = 8;
+    StorageIndex packet_size = internal::packet_traits<Scalar>::size;
+    if (packet_size < 8) packet_size = 8;
     StorageIndex target_bk =
         Eigen::divup(k / target_k_slices, packet_size) * packet_size;
     kc_ = (std::min)(k, target_bk);
-- 
GitLab


From 2691e6746fd96583802a821b202e7fa6316f32c2 Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Wed, 23 Jan 2019 03:59:07 -0800
Subject: [PATCH 1111/2345] Support Nones and DTypes as arguments for
 serialization.

Motivation: https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/layers/LSTMCell#get_initial_state
PiperOrigin-RevId: 230504341
---
 tensorflow/python/framework/func_graph.py  | 10 +++++-
 tensorflow/python/saved_model/load_test.py | 41 ++++++++++++++++++++++
 2 files changed, 50 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/framework/func_graph.py b/tensorflow/python/framework/func_graph.py
index 28d829e89a..fd0163a6a1 100644
--- a/tensorflow/python/framework/func_graph.py
+++ b/tensorflow/python/framework/func_graph.py
@@ -25,6 +25,7 @@ from tensorflow.core.framework import attr_value_pb2
 from tensorflow.python.eager import context
 from tensorflow.python.eager import tape
 from tensorflow.python.eager.graph_only_ops import graph_placeholder
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_spec
 from tensorflow.python.framework.auto_control_deps import AutomaticControlDependencies
@@ -86,7 +87,14 @@ def convert_structure_to_signature(structure, arg_names=None):
     """A representation for this argument, for converting into signatures."""
     if isinstance(arg, ops.Tensor):
       return tensor_spec.TensorSpec(arg.shape, arg.dtype, name)
-    if isinstance(arg, (int, float, bool, tensor_spec.TensorSpec)):
+    if isinstance(arg, (
+        int,
+        float,
+        bool,
+        type(None),
+        dtypes.DType,
+        tensor_spec.TensorSpec,
+    )):
       return arg
     return UnknownArgument()
 
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index b1dc0bd128..da6f568c4b 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -29,6 +29,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import tensor_spec
 from tensorflow.python.lib.io import file_io
+from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.saved_model import load
 from tensorflow.python.saved_model import save
@@ -198,6 +199,46 @@ class LoadTest(test.TestCase):
     self.assertEqual(4, imported.f(constant_op.constant(2), True).numpy())
     self.assertEqual(7, imported.f(constant_op.constant(2)).numpy())
 
+  def test_function_with_default_none_input(self):
+
+    def func(x, dtype=None):
+      if dtype:
+        return array_ops.zeros(shape=x.shape, dtype=dtype)
+      else:
+        return array_ops.zeros(shape=x.shape, dtype=dtypes.float32)
+
+    root = tracking.AutoCheckpointable()
+    root.f = def_function.function(func)
+
+    self.assertAllEqual([0.0, 0.0, 0.0],
+                        root.f(constant_op.constant([1, 2, 3])).numpy())
+    self.assertAllEqual([0.0, 0.0, 0.0],
+                        root.f(constant_op.constant([1.0, 2.0, 3.0])).numpy())
+    self.assertAllEqual([0.0, 0.0, 0.0, 0.0],
+                        root.f(constant_op.constant([1, 2, 3, 4])).numpy())
+    self.assertAllEqual([0, 0, 0],
+                        root.f(
+                            constant_op.constant([1.0, 2.0, 3.0]),
+                            dtype=dtypes.int32).numpy())
+
+    concrete_functions = root.f._list_all_concrete_functions_for_serialization()  # pylint: disable=protected-access
+    self.assertEqual(4, len(concrete_functions))
+
+    imported = self.cycle(root)
+
+    self.assertAllEqual([0.0, 0.0, 0.0],
+                        imported.f(constant_op.constant([1, 2, 3]),
+                                   None).numpy())
+    self.assertAllEqual([0.0, 0.0, 0.0],
+                        imported.f(constant_op.constant([1.0, 2.0,
+                                                         3.0])).numpy())
+    self.assertAllEqual([0.0, 0.0, 0.0, 0.0],
+                        imported.f(constant_op.constant([1, 2, 3, 4])).numpy())
+    self.assertAllEqual([0, 0, 0],
+                        imported.f(
+                            constant_op.constant([1.0, 2.0, 3.0]),
+                            dtype=dtypes.int32).numpy())
+
   def test_structured_inputs(self):
 
     def func(x, training=True):
-- 
GitLab


From bf6c9500beb57d61a5800c74c008ed19098614be Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Wed, 23 Jan 2019 05:40:09 -0800
Subject: [PATCH 1112/2345] Use a logging mechanism that allows greater output
 control.

Requirements that this mechanism meets:
 * users should be able to change the verbosity programmatically (ag.set_verbosity)
 * users should be able to change the verbosity without touching the code (AUTOGRAPH_VERBOSITY env variable)
 * different levels of verbosity should be supported, ala vlog
 * the mechanism should work in interactive prompts like Jupyter by mirroring the log messages to console
All these controls are global (at least for now).

PiperOrigin-RevId: 230514195
---
 tensorflow/python/autograph/utils/BUILD       |  3 +
 .../python/autograph/utils/ag_logging.py      | 92 +++++++++++++++++++
 2 files changed, 95 insertions(+)
 create mode 100644 tensorflow/python/autograph/utils/ag_logging.py

diff --git a/tensorflow/python/autograph/utils/BUILD b/tensorflow/python/autograph/utils/BUILD
index 790c661661..f5e0dbf00b 100644
--- a/tensorflow/python/autograph/utils/BUILD
+++ b/tensorflow/python/autograph/utils/BUILD
@@ -20,6 +20,7 @@ py_library(
     name = "utils",
     srcs = [
         "__init__.py",
+        "ag_logging.py",
         "context_managers.py",
         "misc.py",
         "py_func.py",
@@ -33,7 +34,9 @@ py_library(
     deps = [
         "//tensorflow/python:dtypes",
         "//tensorflow/python:list_ops",
+        "//tensorflow/python:platform",
         "//tensorflow/python:script_ops",
+        "//tensorflow/python:util",
         "//tensorflow/python/autograph/pyct",
         "//tensorflow/python/data/ops:dataset_ops",
         "@six_archive//:six",
diff --git a/tensorflow/python/autograph/utils/ag_logging.py b/tensorflow/python/autograph/utils/ag_logging.py
new file mode 100644
index 0000000000..dd0c271a1a
--- /dev/null
+++ b/tensorflow/python/autograph/utils/ag_logging.py
@@ -0,0 +1,92 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Logging and debugging utilities."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import sys
+
+# TODO(mdan): Use a custom logger class.
+from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.util.tf_export import tf_export
+
+VERBOSITY_VAR_NAME = 'AUTOGRAPH_VERBOSITY'
+DEFAULT_VERBOSITY = 0
+
+verbosity_level = None  # vlog-like. Takes precedence over the env variable.
+echo_log_to_stdout = False
+
+# In interactive Python, logging echo is enabled by default.
+if hasattr(sys, 'ps1') or hasattr(sys, 'ps2'):
+  echo_log_to_stdout = True
+
+
+@tf_export('autograph.set_verbosity')
+def set_verbosity(level, alsologtostdout=False):
+  """Sets the AutoGraph verbosity level.
+
+  # Debug logging in AutoGraph
+
+  More verbose logging is useful to enable when filing bug reports or debugging
+  things.
+
+  There are two controls that control the logging verbosity:
+   * The `set_verbosity` function
+   * The `AUTOGRAPH_VERBOSITY` environment variable
+  `set_verbosity` takes precedence over the environment variable.
+
+  By default, logs are output to absl's default logger, with INFO level. They
+  can be mirrored to stdout by using the `alsologtostdout` argument. Mirroring
+  is enabled by default when Python runs in interactive mode.
+
+  Args:
+    level: int, the verbosity level; uses the same scale as vlog
+    alsologtostdout: bool, whether to ech log messages to stdout
+  """
+  global verbosity_level
+  global echo_log_to_stdout
+  verbosity_level = level
+  echo_log_to_stdout = alsologtostdout
+
+
+@tf_export('autograph.trace')
+def trace(*args):
+  """Traces argument information at compilation time."""
+  print(*args)
+
+
+def get_verbosity():
+  global verbosity_level
+  if verbosity_level is not None:
+    return verbosity_level
+  return os.getenv(VERBOSITY_VAR_NAME, DEFAULT_VERBOSITY)
+
+
+def has_verbosity(level):
+  return get_verbosity() >= level
+
+
+def log(level, msg, *args, **kwargs):
+  if has_verbosity(level):
+    logging.info(msg, *args, **kwargs)
+    if echo_log_to_stdout:
+      print(msg % args)
+
+
+def warn_first_n(msg, *args, **kwargs):
+  logging.log_first_n(logging.WARNING, msg, *args, **kwargs)
-- 
GitLab


From f3919b99b1e0f8f3efe7634a43731e1dd9c220a0 Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Wed, 23 Jan 2019 06:06:14 -0800
Subject: [PATCH 1113/2345] Add test for repeated saving/loading and make it
 work. Also adding one more test for kwargs and concrete_functions.

Note that cycling (repeated save/restore) does not work for nested functions yet, but I am submitting this test to prevent more damage in the future: every new feature should ensure that cycling works properly with it.

PiperOrigin-RevId: 230516944
---
 tensorflow/python/framework/func_graph.py     |   4 +-
 tensorflow/python/saved_model/BUILD           |   1 +
 .../saved_model/function_deserialization.py   |  36 +++-
 tensorflow/python/saved_model/load.py         |   6 +-
 tensorflow/python/saved_model/load_test.py    | 173 +++++++++++-------
 .../saved_model/nested_structure_coder.py     |   3 +-
 6 files changed, 145 insertions(+), 78 deletions(-)

diff --git a/tensorflow/python/framework/func_graph.py b/tensorflow/python/framework/func_graph.py
index fd0163a6a1..e7718cfef1 100644
--- a/tensorflow/python/framework/func_graph.py
+++ b/tensorflow/python/framework/func_graph.py
@@ -107,7 +107,9 @@ def convert_structure_to_signature(structure, arg_names=None):
           "Passed in arg_names don't match actual signature (%s)." % arg_names)
     # Replace all top-level names with their actual arg_names. If a path before
     # was "(2,'a',1)", it will become "(arg_names[2],'a',1)".
-    flattened = [((arg_names[0],) + path[1:], arg) for path, arg in flattened]
+    flattened = [
+        ((arg_names[path[0]],) + path[1:], arg) for path, arg in flattened
+    ]
 
   mapped = [
       encode_arg(arg, "/".join([str(p) for p in path]))
diff --git a/tensorflow/python/saved_model/BUILD b/tensorflow/python/saved_model/BUILD
index ebbefd7cde..5d08a40d67 100644
--- a/tensorflow/python/saved_model/BUILD
+++ b/tensorflow/python/saved_model/BUILD
@@ -337,6 +337,7 @@ tf_py_test(
     additional_deps = [
         ":load",
         ":save",
+        "@absl_py//absl/testing:parameterized",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:lib",
diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py
index 22d5ff4cf4..903331cee0 100644
--- a/tensorflow/python/saved_model/function_deserialization.py
+++ b/tensorflow/python/saved_model/function_deserialization.py
@@ -97,6 +97,30 @@ def recreate_concrete_function(saved_concrete_function, concrete_functions):
   return concrete_function
 
 
+class RestoredFunction(def_function.Function):
+  """Wrapper class for a function that has been restored from saved state.
+
+  See `def_function.Function`.
+  """
+
+  def __init__(self, python_function, name, function_spec, concrete_functions):
+    # TODO(mdan): We may enable autograph once exceptions are supported.
+    super(RestoredFunction, self).__init__(
+        python_function, name, autograph=False)
+    self._concrete_functions = concrete_functions
+    # TODO(vbardiovsky): This does not propagate to stateful and stateless
+    # functions of the RestoredFunction, which will have seen only defunned
+    # restored_function_body(*args, **kwargs). Therefore get_concrete_function()
+    # called on RestoredFunction will not work properly.
+    self._function_spec = function_spec
+
+  def _list_all_concrete_functions_for_serialization(self):
+    return self._concrete_functions
+
+  def get_concrete_function(self, *args, **kwargs):
+    raise NotImplementedError()
+
+
 def recreate_function(saved_function, concrete_functions):
   """Creates a `Function` from a `SavedFunction`.
 
@@ -116,9 +140,7 @@ def recreate_function(saved_function, concrete_functions):
   function_spec = _deserialize_function_spec(saved_function.function_spec,
                                              coder)
 
-  # TODO(mdan): We may enable autograph once exceptions are supported.
-  @def_function.function(autograph=False)
-  def restored_function(*args, **kwargs):
+  def restored_function_body(*args, **kwargs):
     """Calls a restored function."""
     # TODO(allenl): Functions saved with input_signatures should revive with
     # input_signatures.
@@ -148,7 +170,10 @@ def recreate_function(saved_function, concrete_functions):
         "Only existing signatures are %r."
         % (canonicalized_inputs, debug_considered_signatures))
 
-  return restored_function
+  cfs = [concrete_functions[f.name] for f in saved_function.concrete_function]
+
+  return RestoredFunction(restored_function_body,
+                          restored_function_body.__name__, function_spec, cfs)
 
 
 def load_function_def_library(library):
@@ -210,8 +235,7 @@ def _sort_function_defs(library):
         ready.append(dest)
 
   if len(output) != len(library.function):
-    loaded = set([x.signature.name for x in output])
-    failed_to_resolve = sorted(set(in_count.keys()) - loaded)
+    failed_to_resolve = sorted(set(in_count.keys()) - set(output))
     raise ValueError("There is a cyclic-dependency between functions. ",
                      "Could not resolve %r." % (failed_to_resolve,))
 
diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
index 2aa4924b29..12b79d07e7 100644
--- a/tensorflow/python/saved_model/load.py
+++ b/tensorflow/python/saved_model/load.py
@@ -88,9 +88,11 @@ class _Loader(object):
       # exactly the same. Fortunately the repacking logic cares only about
       # the structure.
       # TODO(vbardiovsky): Should we just replicate the structures, with
-      # Nones instead of real objects? Decide when we start solving
-      # idempotency.
+      # Nones instead of real objects?
       function._func_graph.structured_outputs = original_outputs  # pylint: disable=protected-access
+      input_signature = coder.decode_proto(
+          concrete_function.canonicalized_input_signature)
+      function._func_graph.structured_input_signature = input_signature  # pylint: disable=protected-access
 
   def _setup_functions(self):
 
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index da6f568c4b..6eb4b13603 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -22,6 +22,8 @@ import collections
 import os
 import tempfile
 
+from absl.testing import parameterized
+
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import def_function
 from tensorflow.python.eager import test
@@ -36,40 +38,51 @@ from tensorflow.python.saved_model import save
 from tensorflow.python.training.checkpointable import tracking
 
 
-class LoadTest(test.TestCase):
+@parameterized.named_parameters(
+    dict(testcase_name="ReloadOnce", cycles=1),
+    dict(testcase_name="ReloadTwice", cycles=2),
+    dict(testcase_name="ReloadThrice", cycles=3))
+class LoadTest(test.TestCase, parameterized.TestCase):
+
+  def cycle(self, obj, cycles=1, signatures=None):
+    to_save = obj
 
-  def cycle(self, obj, signatures=None):
-    path = tempfile.mkdtemp(prefix=self.get_temp_dir())
-    save.save(obj, path, signatures=signatures or {})
-    return load.load(path)
+    # TODO(vbardiovsky): It would be nice if exported protos reached a fixed
+    # point w.r.t. saving/restoring, ideally after 2nd saving.
+    for _ in range(cycles):
+      path = tempfile.mkdtemp(prefix=self.get_temp_dir())
+      save.save(to_save, path, signatures=signatures or {})
+      loaded = load.load(path)
+      to_save = loaded
+    return loaded
 
-  def test_structure_import(self):
+  def test_structure_import(self, cycles):
     root = tracking.AutoCheckpointable()
     root.dep_one = tracking.AutoCheckpointable()
     root.dep_two = tracking.AutoCheckpointable()
     root.dep_two.dep = tracking.AutoCheckpointable()
     root.dep_three = root.dep_two.dep
-    imported = self.cycle(root)
+    imported = self.cycle(root, cycles)
     self.assertIs(imported.dep_three, imported.dep_two.dep)
     self.assertIsNot(imported.dep_one, imported.dep_two)
 
-  def test_variables(self):
+  def test_variables(self, cycles):
     root = tracking.AutoCheckpointable()
     root.v1 = variables.Variable(1., trainable=True)
     root.v2 = variables.Variable(2., trainable=False)
-    imported = self.cycle(root)
-    self.assertEquals(imported.v1.numpy(), 1.0)
+    imported = self.cycle(root, cycles)
+    self.assertEqual(imported.v1.numpy(), 1.0)
     self.assertTrue(imported.v1.trainable)
-    self.assertEquals(imported.v2.numpy(), 2.0)
+    self.assertEqual(imported.v2.numpy(), 2.0)
     self.assertFalse(imported.v2.trainable)
 
-  def test_capture_variables(self):
+  def test_capture_variables(self, cycles):
     root = tracking.AutoCheckpointable()
     root.weights = variables.Variable(2.)
     root.f = def_function.function(
         lambda x: root.weights * x,
         input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
-    imported = self.cycle(root)
+    imported = self.cycle(root, cycles)
     self.assertEqual(4., imported.f(constant_op.constant(2.)).numpy())
     imported.weights.assign(4.0)
     self.assertEqual(8., imported.f(constant_op.constant(2.)).numpy())
@@ -80,7 +93,7 @@ class LoadTest(test.TestCase):
       f.write(contents)
     return filename
 
-  def test_assets(self):
+  def test_assets(self, cycles):
     file1 = self._make_asset("contents 1")
     file2 = self._make_asset("contents 2")
 
@@ -98,33 +111,33 @@ class LoadTest(test.TestCase):
 
     imported = load.load(load_dir)
     with open(imported.asset1.asset_path.numpy(), "r") as f:
-      self.assertEquals("contents 1", f.read())
+      self.assertEqual("contents 1", f.read())
     with open(imported.asset2.asset_path.numpy(), "r") as f:
-      self.assertEquals("contents 2", f.read())
+      self.assertEqual("contents 2", f.read())
 
-  def test_capture_assets(self):
+  def test_capture_assets(self, cycles):
     root = tracking.AutoCheckpointable()
     root.vocab = tracking.TrackableAsset(self._make_asset("contents"))
     root.f = def_function.function(
         lambda: root.vocab.asset_path,
         input_signature=[])
-    imported = self.cycle(root)
+    imported = self.cycle(root, cycles)
     origin_output = root.f().numpy()
     imported_output = imported.f().numpy()
     self.assertNotEqual(origin_output, imported_output)
     with open(imported_output, "r") as f:
-      self.assertEquals("contents", f.read())
+      self.assertEqual("contents", f.read())
 
-  def test_dedup_assets(self):
+  def test_dedup_assets(self, cycles):
     vocab = self._make_asset("contents")
     root = tracking.AutoCheckpointable()
     root.asset1 = tracking.TrackableAsset(vocab)
     root.asset2 = tracking.TrackableAsset(vocab)
-    imported = self.cycle(root)
+    imported = self.cycle(root, cycles)
     self.assertEqual(imported.asset1.asset_path.numpy(),
                      imported.asset2.asset_path.numpy())
 
-  def test_implicit_input_signature(self):
+  def test_implicit_input_signature(self, cycles):
     @def_function.function
     def func(x):
       return 2 * x
@@ -136,12 +149,12 @@ class LoadTest(test.TestCase):
     root.f(constant_op.constant(1.))
     root.f(constant_op.constant(1))
 
-    imported = self.cycle(root)
+    imported = self.cycle(root, cycles)
 
     self.assertEqual(4., imported.f(constant_op.constant(2.)).numpy())
     self.assertEqual(14, imported.f(constant_op.constant(7)).numpy())
 
-  def test_explicit_input_signature(self):
+  def test_explicit_input_signature(self, cycles):
     @def_function.function(
         input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
     def func(x):
@@ -150,10 +163,10 @@ class LoadTest(test.TestCase):
     root = tracking.AutoCheckpointable()
     root.f = func
 
-    imported = self.cycle(root)
+    imported = self.cycle(root, cycles)
     self.assertEqual(4., imported.f(constant_op.constant(2.0)).numpy())
 
-  def test_explicit_save_signature(self):
+  def test_explicit_save_signature(self, cycles):
     @def_function.function
     def func(x):
       return 2 * x
@@ -162,11 +175,14 @@ class LoadTest(test.TestCase):
     root.f = func
 
     imported = self.cycle(
-        root, {"f": root.f.get_concrete_function(
-            tensor_spec.TensorSpec(None, dtypes.float32))})
+        root, cycles, {
+            "f":
+                root.f.get_concrete_function(
+                    tensor_spec.TensorSpec(None, dtypes.float32))
+        })
     self.assertEqual(4., imported.f(constant_op.constant(2.0)).numpy())
 
-  def test_nested_functions(self):
+  def test_nested_functions(self, cycles):
     f = def_function.function(
         lambda x: x*2.0,
         input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
@@ -176,10 +192,13 @@ class LoadTest(test.TestCase):
 
     root = tracking.AutoCheckpointable()
     root.g = g
-    imported = self.cycle(root)
+    # TODO(vbardiovsky): Enable this test. For this to work, we must ensure that
+    # concrete_function._inference_function._graph._functions contains all
+    # functions that were on the graph before saving.
+    imported = self.cycle(root, 1)
     imported.g(constant_op.constant([1.0]))
 
-  def test_function_with_default_bool_input(self):
+  def test_function_with_default_bool_input(self, cycles):
 
     def func(x, training=False):
       if training:
@@ -194,12 +213,12 @@ class LoadTest(test.TestCase):
     self.assertEqual(7, root.f(constant_op.constant(1)).numpy())
     self.assertEqual(2, root.f(constant_op.constant(1), True).numpy())
 
-    imported = self.cycle(root)
+    imported = self.cycle(root, cycles)
 
     self.assertEqual(4, imported.f(constant_op.constant(2), True).numpy())
     self.assertEqual(7, imported.f(constant_op.constant(2)).numpy())
 
-  def test_function_with_default_none_input(self):
+  def test_function_with_default_none_input(self, cycles):
 
     def func(x, dtype=None):
       if dtype:
@@ -224,7 +243,7 @@ class LoadTest(test.TestCase):
     concrete_functions = root.f._list_all_concrete_functions_for_serialization()  # pylint: disable=protected-access
     self.assertEqual(4, len(concrete_functions))
 
-    imported = self.cycle(root)
+    imported = self.cycle(root, cycles)
 
     self.assertAllEqual([0.0, 0.0, 0.0],
                         imported.f(constant_op.constant([1, 2, 3]),
@@ -239,7 +258,7 @@ class LoadTest(test.TestCase):
                             constant_op.constant([1.0, 2.0, 3.0]),
                             dtype=dtypes.int32).numpy())
 
-  def test_structured_inputs(self):
+  def test_structured_inputs(self, cycles):
 
     def func(x, training=True):
       # x is a nested structure, we care about one particular tensor.
@@ -263,7 +282,7 @@ class LoadTest(test.TestCase):
     # matching signature will be valid on the loaded model.
     self.assertEqual(31, root.f(input1).numpy())
 
-    imported = self.cycle(root)
+    imported = self.cycle(root, cycles)
 
     with self.assertRaisesRegexp(AssertionError,
                                  "Could not find matching function to call.*"):
@@ -272,7 +291,7 @@ class LoadTest(test.TestCase):
     self.assertEqual(31, imported.f(input1).numpy())
     self.assertEqual(32, imported.f(input3).numpy())
 
-  def test_structured_output(self):
+  def test_structured_output(self, cycles):
 
     # Use fields with non-alphabetical order
     named_tuple_type = collections.namedtuple("NamedTupleHello", ["b", "a"])
@@ -292,7 +311,7 @@ class LoadTest(test.TestCase):
     self.assertEqual(3, result[1].numpy())
     self.assertEqual(0.5, result[2]["x"].numpy())
 
-    imported = self.cycle(root)
+    imported = self.cycle(root, cycles)
 
     result = imported.f(constant_op.constant(2), constant_op.constant(5))
     self.assertEqual(7, result[0].a.numpy())
@@ -301,7 +320,7 @@ class LoadTest(test.TestCase):
     self.assertEqual(5, result[1].numpy())
     self.assertEqual(0.5, result[2]["x"].numpy())
 
-  def test_positional_arguments(self):
+  def test_positional_arguments(self, cycles):
     def func(x, training=False, abc=7.1, defg=7.7):
       del abc
       if training:
@@ -319,13 +338,13 @@ class LoadTest(test.TestCase):
     self.assertEqual(2, root.f(constant_op.constant(1), True).numpy())
     self.assertEqual(6, root.f(constant_op.constant(1), defg=7.0).numpy())
 
-    imported = self.cycle(root)
+    imported = self.cycle(root, cycles)
 
     self.assertEqual(4, imported.f(constant_op.constant(2), True).numpy())
     self.assertEqual(7, imported.f(constant_op.constant(2)).numpy())
     self.assertEqual(6, imported.f(constant_op.constant(1), defg=7.0).numpy())
 
-  def test_additional_kwargs(self):
+  def test_additional_kwargs(self, cycles):
     def func(x, training=False, **options):
       del options
       if training:
@@ -339,7 +358,7 @@ class LoadTest(test.TestCase):
     x = constant_op.constant(10)
     self.assertEqual(7, root.f(x, learning_rate=0.5, epochs=3).numpy())
 
-    imported = self.cycle(root)
+    imported = self.cycle(root, cycles)
 
     with self.assertRaisesRegexp(AssertionError,
                                  "Could not find matching function to call.*"):
@@ -347,7 +366,7 @@ class LoadTest(test.TestCase):
 
     self.assertEqual(7, imported.f(x, learning_rate=0.5, epochs=3).numpy())
 
-  def test_member_function(self):
+  def test_member_function(self, cycles):
     class CheckpointableWithMember(tracking.AutoCheckpointable):
 
       def __init__(self):
@@ -367,12 +386,12 @@ class LoadTest(test.TestCase):
     self.assertEqual(27, root.f(constant_op.constant(1)).numpy())
     self.assertEqual(2, root.f(constant_op.constant(1), True).numpy())
 
-    imported = self.cycle(root)
+    imported = self.cycle(root, cycles)
 
     self.assertEqual(4, imported.f(constant_op.constant(2), True).numpy())
     self.assertEqual(27, imported.f(constant_op.constant(2)).numpy())
 
-  def test_side_effect_listing(self):
+  def test_side_effect_listing(self, cycles):
     class M(tracking.AutoCheckpointable):
 
       def __init__(self):
@@ -388,9 +407,9 @@ class LoadTest(test.TestCase):
 
     m = M()
     self.cycle(m)
-    self.assertEquals(4.0, m.f(constant_op.constant(2.0)).numpy())
+    self.assertEqual(4.0, m.f(constant_op.constant(2.0)).numpy())
 
-  def test_basic_backprop(self):
+  def test_basic_backprop(self, cycles):
     weight = variables.Variable(1., trainable=True)
     bias = variables.Variable(0., trainable=True)
     g = def_function.function(
@@ -401,14 +420,14 @@ class LoadTest(test.TestCase):
     root.weight = weight
     root.bias = bias
     root.g = g
-    imported = self.cycle(root)
+    imported = self.cycle(root, cycles)
     with backprop.GradientTape() as t:
       x = constant_op.constant([3.5])
       loss = imported.g(x)
       grad = t.gradient(loss, [imported.weight, imported.bias])
       self.assertAllClose(grad, [3.5, 1.0])
 
-  def test_nested_backprop(self):
+  def test_nested_backprop(self, cycles):
     weight = variables.Variable(1., trainable=True)
     bias = variables.Variable(0., trainable=True)
 
@@ -442,14 +461,17 @@ class LoadTest(test.TestCase):
     root.bias = bias
     root.g = h
 
-    imported = self.cycle(root)
+    # TODO(vbardiovsky): Enable this test. For this to work, we must ensure that
+    # concrete_function._inference_function._graph._functions contains all
+    # functions that were on the graph before saving.
+    imported = self.cycle(root, 1)
     with backprop.GradientTape() as t:
       x = constant_op.constant([3.5])
       loss = imported.g(x)
     grad = t.gradient(loss, [imported.weight, imported.bias])
     self.assertAllClose(grad, [3.5, 2.0])
 
-  def test_callable(self):
+  def test_callable(self, cycles):
     class M1(tracking.AutoCheckpointable):
 
       @def_function.function(
@@ -463,7 +485,7 @@ class LoadTest(test.TestCase):
     root.m2.__call__ = def_function.function(
         input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])(
             lambda x: x*3.0)
-    imported = self.cycle(root)
+    imported = self.cycle(root, cycles)
     x = constant_op.constant(1.0)
 
     self.assertTrue(callable(imported.m1))
@@ -478,7 +500,7 @@ class LoadTest(test.TestCase):
     # Verify that user objects without `__call__` attribute are not callable.
     self.assertFalse(callable(imported))
 
-  def test_chain_callable(self):
+  def test_chain_callable(self, cycles):
     func = def_function.function(
         input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])(
             lambda x: x*3.0)
@@ -487,12 +509,12 @@ class LoadTest(test.TestCase):
     root.__call__.__call__ = tracking.AutoCheckpointable()
     root.__call__.__call__.__call__ = func
 
-    imported = self.cycle(root)
+    imported = self.cycle(root, cycles)
     self.assertTrue(callable(imported))
     x = constant_op.constant(1.0)
     self.assertAllEqual(imported(x).numpy(), 3.0)
 
-  def test_soft_matching(self):
+  def test_soft_matching(self, cycles):
 
     @def_function.function(
         input_signature=[tensor_spec.TensorSpec([None], dtypes.int32)])
@@ -508,7 +530,7 @@ class LoadTest(test.TestCase):
     concrete_functions = root.f._list_all_concrete_functions_for_serialization()  # pylint: disable=protected-access
     self.assertEqual(1, len(concrete_functions))
 
-    imported = self.cycle(root)
+    imported = self.cycle(root, cycles)
 
     with self.assertRaisesRegexp(ValueError, "Cannot canonicalize"):
       # We cannot call the function with a constant of shape ().
@@ -522,7 +544,7 @@ class LoadTest(test.TestCase):
     self.assertAllEqual([2, 4, 6],
                         imported.f(constant_op.constant([1, 2, 3])).numpy())
 
-  def test_concrete_function(self):
+  def test_concrete_function(self, cycles):
 
     @def_function.function(
         input_signature=[tensor_spec.TensorSpec([None], dtypes.int32)])
@@ -535,14 +557,14 @@ class LoadTest(test.TestCase):
     self.assertAllEqual([2], root.f(constant_op.constant([1])).numpy())
     self.assertAllEqual([2, 4], root.f(constant_op.constant([1, 2])).numpy())
 
-    imported = self.cycle(root)
+    imported = self.cycle(root, cycles)
 
     self.assertAllEqual([2, 4, 6, 8],
                         imported.f(constant_op.constant([1, 2, 3, 4])).numpy())
     self.assertAllEqual([2, 4, 6],
                         imported.f(constant_op.constant([1, 2, 3])).numpy())
 
-  def test_concrete_function_arg_names(self):
+  def test_concrete_function_arg_names(self, cycles):
 
     @def_function.function(
         input_signature=[tensor_spec.TensorSpec([None], dtypes.int32)])
@@ -554,12 +576,12 @@ class LoadTest(test.TestCase):
 
     self.assertAllEqual([2], root.f(constant_op.constant([1])).numpy())
 
-    imported = self.cycle(root)
+    imported = self.cycle(root, cycles)
 
     self.assertAllEqual([2, 4, 6],
                         imported.f(x=constant_op.constant([1, 2, 3])).numpy())
 
-  def test_concrete_function_no_signature(self):
+  def test_concrete_function_no_signature(self, cycles):
     @def_function.function
     def func(x):
       return 2 * x
@@ -567,11 +589,11 @@ class LoadTest(test.TestCase):
     root = tracking.AutoCheckpointable()
     root.f = func.get_concrete_function(constant_op.constant([1]))
     self.assertAllEqual([4], root.f(constant_op.constant([2])).numpy())
-    imported = self.cycle(root)
+    imported = self.cycle(root, cycles)
     self.assertAllEqual([6],
                         imported.f(constant_op.constant([3])).numpy())
 
-  def test_concrete_function_backprop(self):
+  def test_concrete_function_backprop(self, cycles):
     @def_function.function(
         input_signature=[tensor_spec.TensorSpec([None], dtypes.float32)])
     def func(x):
@@ -587,25 +609,40 @@ class LoadTest(test.TestCase):
       return tape.gradient(output, inp)
 
     self.assertEqual(2., _compute_gradient(root.f).numpy())
-    imported = self.cycle(root)
+    imported = self.cycle(root, cycles)
     self.assertEqual(2., _compute_gradient(imported.f).numpy())
 
-  def test_dict(self):
+  def test_revived_concrete_function_kwargs(self, cycles):
+
+    @def_function.function
+    def func(x, y):
+      return x * (y + 1.)
+    root = tracking.AutoCheckpointable()
+    root.f = func.get_concrete_function(
+        tensor_spec.TensorSpec([], dtypes.float32),
+        tensor_spec.TensorSpec([], dtypes.float32))
+    self.assertEqual(8., root.f(y=constant_op.constant(3.),
+                                x=constant_op.constant(2.)).numpy())
+    imported = self.cycle(root, cycles)
+    self.assertEqual(8., imported.f(y=constant_op.constant(3.),
+                                    x=constant_op.constant(2.)).numpy())
+
+  def test_dict(self, cycles):
     root = tracking.AutoCheckpointable()
     root.variables = dict(a=variables.Variable(1.))
     root.variables["b"] = variables.Variable(2.)
     root.variables["c"] = 1
-    imported = self.cycle(root)
+    imported = self.cycle(root, cycles)
     self.assertEqual(1., imported.variables["a"].numpy())
     self.assertEqual(2., imported.variables["b"].numpy())
     self.assertEqual(set(["a", "b"]), set(imported.variables.keys()))
 
-  def test_list(self):
+  def test_list(self, cycles):
     root = tracking.AutoCheckpointable()
     root.variables = [variables.Variable(1.)]
     root.variables.append(1)
     root.variables.append(variables.Variable(3.))
-    imported = self.cycle(root)
+    imported = self.cycle(root, cycles)
     self.assertEqual(1., imported.variables[0].numpy())
     self.assertEqual(3., imported.variables[2].numpy())
     self.assertIs(None, imported.variables[1])
diff --git a/tensorflow/python/saved_model/nested_structure_coder.py b/tensorflow/python/saved_model/nested_structure_coder.py
index e080a6b3de..5cf9a5b155 100644
--- a/tensorflow/python/saved_model/nested_structure_coder.py
+++ b/tensorflow/python/saved_model/nested_structure_coder.py
@@ -38,6 +38,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import tensor_spec
 from tensorflow.python.saved_model import struct_pb2
+from tensorflow.python.util import compat
 
 
 class NotEncodableError(Exception):
@@ -304,7 +305,7 @@ class _StringCodec(object):
 
   def do_decode(self, value, decode_fn):
     del decode_fn
-    return value.string_value
+    return compat.as_str(value.string_value)
 
 
 StructureCoder.register_codec(_StringCodec())
-- 
GitLab


From 4a9ebc33bc2e72a8efc8b360277856f38fd7692d Mon Sep 17 00:00:00 2001
From: Andr? Susano Pinto <andresp@google.com>
Date: Wed, 23 Jan 2019 07:32:50 -0800
Subject: [PATCH 1114/2345] Don't throw an error if a checkpointable does not
 has an implicit signature.

Note:
 - Exporting objects without signatures is a valid use case.
 - Exporting objects with multiple tf.functions is also a use case.
 - Users should be explicit about which signatures to export using the 3 arg of `tf.saved_model.save` or have exactly one tf.function which can be made into a signature.
PiperOrigin-RevId: 230526504
---
 tensorflow/python/saved_model/load_test.py | 20 +++--
 tensorflow/python/saved_model/save.py      | 88 +++++++++++-----------
 tensorflow/python/saved_model/save_test.py | 61 ++-------------
 3 files changed, 62 insertions(+), 107 deletions(-)

diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index 6eb4b13603..d4742dde1c 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -46,12 +46,11 @@ class LoadTest(test.TestCase, parameterized.TestCase):
 
   def cycle(self, obj, cycles=1, signatures=None):
     to_save = obj
-
     # TODO(vbardiovsky): It would be nice if exported protos reached a fixed
     # point w.r.t. saving/restoring, ideally after 2nd saving.
     for _ in range(cycles):
       path = tempfile.mkdtemp(prefix=self.get_temp_dir())
-      save.save(to_save, path, signatures=signatures or {})
+      save.save(to_save, path, signatures)
       loaded = load.load(path)
       to_save = loaded
     return loaded
@@ -102,7 +101,7 @@ class LoadTest(test.TestCase, parameterized.TestCase):
     root.asset2 = tracking.TrackableAsset(file2)
 
     save_dir = os.path.join(self.get_temp_dir(), "save_dir")
-    save.save(root, save_dir, signatures={})
+    save.save(root, save_dir)
 
     file_io.delete_file(file1)
     file_io.delete_file(file2)
@@ -557,7 +556,8 @@ class LoadTest(test.TestCase, parameterized.TestCase):
     self.assertAllEqual([2], root.f(constant_op.constant([1])).numpy())
     self.assertAllEqual([2, 4], root.f(constant_op.constant([1, 2])).numpy())
 
-    imported = self.cycle(root, cycles)
+    # TODO(andresp): Fix exporting of loaded concrete functions as signatures.
+    imported = self.cycle(root, cycles, signatures={})
 
     self.assertAllEqual([2, 4, 6, 8],
                         imported.f(constant_op.constant([1, 2, 3, 4])).numpy())
@@ -576,7 +576,8 @@ class LoadTest(test.TestCase, parameterized.TestCase):
 
     self.assertAllEqual([2], root.f(constant_op.constant([1])).numpy())
 
-    imported = self.cycle(root, cycles)
+    # TODO(andresp): Fix exporting of loaded concrete functions as signatures.
+    imported = self.cycle(root, cycles, signatures={})
 
     self.assertAllEqual([2, 4, 6],
                         imported.f(x=constant_op.constant([1, 2, 3])).numpy())
@@ -589,7 +590,8 @@ class LoadTest(test.TestCase, parameterized.TestCase):
     root = tracking.AutoCheckpointable()
     root.f = func.get_concrete_function(constant_op.constant([1]))
     self.assertAllEqual([4], root.f(constant_op.constant([2])).numpy())
-    imported = self.cycle(root, cycles)
+    # TODO(andresp): Fix exporting of loaded concrete functions as signatures.
+    imported = self.cycle(root, cycles, signatures={})
     self.assertAllEqual([6],
                         imported.f(constant_op.constant([3])).numpy())
 
@@ -609,7 +611,8 @@ class LoadTest(test.TestCase, parameterized.TestCase):
       return tape.gradient(output, inp)
 
     self.assertEqual(2., _compute_gradient(root.f).numpy())
-    imported = self.cycle(root, cycles)
+    # TODO(andresp): Fix exporting of loaded concrete functions as signatures.
+    imported = self.cycle(root, cycles, signatures={})
     self.assertEqual(2., _compute_gradient(imported.f).numpy())
 
   def test_revived_concrete_function_kwargs(self, cycles):
@@ -623,7 +626,8 @@ class LoadTest(test.TestCase, parameterized.TestCase):
         tensor_spec.TensorSpec([], dtypes.float32))
     self.assertEqual(8., root.f(y=constant_op.constant(3.),
                                 x=constant_op.constant(2.)).numpy())
-    imported = self.cycle(root, cycles)
+    # TODO(andresp): Fix exporting of loaded concrete functions as signatures.
+    imported = self.cycle(root, cycles, signatures={})
     self.assertEqual(8., imported.f(y=constant_op.constant(3.),
                                     x=constant_op.constant(2.)).numpy())
 
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index 9c3ace7325..339f5587f3 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -129,63 +129,60 @@ class _SaveableView(object):
     return functions
 
 
+def _get_signature(function):
+  if (isinstance(function, (defun.Function, def_function.Function)) and
+      function._input_signature is not None):  # pylint: disable=protected-access
+    function = function.get_concrete_function()
+  if not isinstance(function, defun.ConcreteFunction):
+    return None
+  return function
+
+
+def _valid_signature(concrete_function):
+  """Returns whether concrete function can be converted to a signature."""
+  try:
+    _normalize_outputs(concrete_function.structured_outputs, "unused", "unused")
+  except ValueError:
+    return False
+  return True
+
+
 def _find_function_to_export(saveable_view):
-  """Iterate over `root`'s attributes, finding traced functions."""
-  exported_function = None
-  previous_attribute_name = None
+  """Function to export, None if no suitable function was found."""
+  # If the user did not specify signatures, check the root object for a function
+  # that can be made into a signature.
   functions = saveable_view.functions[saveable_view.root]
-  for name, value in sorted(functions.items()):
-    if exported_function is not None:
-      raise ValueError(
-          ("Exporting an object with no "
-           "tf.saved_model.save(..., signatures=...) "
-           "argument specified, and with more than one "
-           "@tf.function-decorated method attached to it: {}. The signature "
-           "keys for these functions are ambiguous. Specify signature "
-           "functions explicitly.").format(
-               [previous_attribute_name, name]))
-    exported_function = value
-    previous_attribute_name = name
-  if exported_function is None:
-    exported_function = functions.get(DEFAULT_SIGNATURE_ATTR, None)
-  if exported_function is None:
-    raise ValueError(
-        ("Exporting an object with no tf.saved_model.save(..., signatures=...) "
-         "argument specified, and with no @tf.function-decorated methods "
-         "attached to it. In the future this will be a supported use-case for "
-         "Python re-import, but at the moment saving a SavedModel without "
-         "signatures does not make sense, as the only consumers will expect "
-         "signatures. Either decorate a method or specify a signature function "
-         "explicitly."))
-  return exported_function
+  signature = functions.get(DEFAULT_SIGNATURE_ATTR, None)
+  if signature is not None:
+    return signature
+
+  # TODO(andresp): Discuss removing this behaviour. It can lead to WTFs when a
+  # user decides to annotate more functions with tf.function and suddenly
+  # serving that model way later in the process stops working.
+  if len(functions) == 1:
+    single_function = list(functions.values())[0]
+    signature = _get_signature(single_function)
+    if signature and  _valid_signature(signature):
+      return signature
+  return None
 
 
 def _canonicalize_signatures(signatures):
   """Converts `signatures` into a dictionary of concrete functions."""
+  if signatures is None:
+    return {}
   if not isinstance(signatures, collections.Mapping):
     signatures = {
         signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signatures}
   concrete_signatures = {}
-  for serving_key, signature_function in signatures.items():
-    if isinstance(signature_function, (defun.Function, def_function.Function)):
-      input_signature = signature_function._input_signature  # pylint: disable=protected-access
-      if input_signature is None:
-        raise ValueError(
-            ("Unable to use the function {} as a signature directly. Functions "
-             "used to generate serving signatures must either have an "
-             "`input_signature=` specified when constructed, or must be "
-             "converted to concrete functions using "
-             "`f.get_concrete_function(...)`.").format(signature_function))
-      signature_function = signature_function.get_concrete_function()
-    elif not isinstance(signature_function, defun.ConcreteFunction):
+  for signature_key, function in signatures.items():
+    signature_function = _get_signature(function)
+    if signature_function is None:
       raise ValueError(
           ("Expected a TensorFlow function to generate a signature for, but "
-           "got {}. Python functions may be decorated with "
-           "`@tf.function(input_signature=...)` and passed as signatures "
-           "directly, or created without a signature using `@tf.function` "
-           "and then converted to a concrete TensorFlow function using "
-           "`f.get_concrete_function(...)`.").format(signature_function))
-    concrete_signatures[serving_key] = signature_function
+           "got {}. Only `tf.functions` with an input signature or "
+           "concrete functions can be used as a signature.").format(function))
+    concrete_signatures[signature_key] = signature_function
   return concrete_signatures
 
 
@@ -820,6 +817,7 @@ def save(obj, export_dir, signatures=None):
 
   if signatures is None:
     signatures = _find_function_to_export(saveable_view)
+
   signatures = _canonicalize_signatures(signatures)
 
   # TODO(allenl): Factor out some subset of SavedModelBuilder which is 2.x
diff --git a/tensorflow/python/saved_model/save_test.py b/tensorflow/python/saved_model/save_test.py
index 5f9dbe2c25..fa79e45bf1 100644
--- a/tensorflow/python/saved_model/save_test.py
+++ b/tensorflow/python/saved_model/save_test.py
@@ -23,16 +23,16 @@ import sys
 
 from tensorflow.python.client import session as session_lib
 from tensorflow.python.eager import backprop
-from tensorflow.python.eager import function
 from tensorflow.python.eager import def_function
+from tensorflow.python.eager import function
 from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_spec
 from tensorflow.python.framework import test_util
-from tensorflow.python.keras.engine import training
 from tensorflow.python.keras.layers import core
+from tensorflow.python.keras.optimizer_v2 import adam
 from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import lookup_ops
 from tensorflow.python.ops import math_ops
@@ -41,7 +41,6 @@ from tensorflow.python.saved_model import loader
 from tensorflow.python.saved_model import save
 from tensorflow.python.saved_model import signature_constants
 from tensorflow.python.saved_model import tag_constants
-from tensorflow.python.keras.optimizer_v2 import adam
 from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.training.checkpointable import util
 
@@ -120,7 +119,7 @@ class SaveTest(test.TestCase):
     root.f(constant_op.constant(1.))
     save_dir = os.path.join(self.get_temp_dir(), "saved_model")
     with self.assertRaisesRegexp(
-        ValueError, "must be converted to concrete functions"):
+        ValueError, "Expected a TensorFlow function"):
       save.save(root, save_dir, root.f)
 
   def test_nested_inputs(self):
@@ -183,11 +182,6 @@ class SaveTest(test.TestCase):
         second_loss,
         _import_and_infer(save_dir, {"x": [[3., 4.]], "y": [2.]}))
 
-  def test_trivial_save_exception(self):
-    save_dir = os.path.join(self.get_temp_dir(), "saved_model")
-    with self.assertRaisesRegexp(ValueError, "signature"):
-      save.save(tracking.AutoCheckpointable(), save_dir)
-
   def test_single_method_default_signature(self):
     model = _ModelWithOptimizer()
     x = constant_op.constant([[3., 4.]])
@@ -208,28 +202,11 @@ class SaveTest(test.TestCase):
     self.assertAllClose({"output_0": 3.},
                         _import_and_infer(save_dir, {}))
 
-  def test_ambiguous_signatures(self):
-    model = _ModelWithOptimizer()
-    x = constant_op.constant([[3., 4.]])
-    y = constant_op.constant([2.])
-    model.call(x, y)
-    model.second_function = def_function.function(lambda: 1.)
-    save_dir = os.path.join(self.get_temp_dir(), "saved_model")
-    with self.assertRaisesRegexp(ValueError, "call.*second_function"):
-      save.save(model, save_dir)
-
-  def test_no_signature(self):
-
-    class Model(util.Checkpoint):
-
-      def call(self, inputs):
-        return inputs * 2.
-
+  def test_single_function_no_signature(self):
+    model = tracking.AutoCheckpointable()
+    model.f = def_function.function(lambda: 3.)
     save_dir = os.path.join(self.get_temp_dir(), "saved_model")
-    model = Model()
-    with self.assertRaisesRegexp(
-        ValueError, "no @tf.function-decorated methods"):
-      save.save(model, save_dir)
+    save.save(model, save_dir)
 
   def test_find_default_save_function(self):
 
@@ -307,30 +284,6 @@ class SaveTest(test.TestCase):
       self.assertNotIn("T", complex_node.attr)
       self.assertNotIn("Tout", complex_node.attr)
 
-  def test_subclassed_no_signature(self):
-
-    class Subclassed(training.Model):
-
-      def call(self, inputs):
-        return inputs * 2.
-
-    save_dir = os.path.join(self.get_temp_dir(), "saved_model")
-    model = Subclassed()
-    with self.assertRaisesRegexp(
-        ValueError, "no @tf.function-decorated methods"):
-      save.save(model, save_dir)
-
-    traced_call = def_function.function(
-        model.call,
-        input_signature=(tensor_spec.TensorSpec(
-            (None, None),
-            dtype=dtypes.float32),))
-    save.save(model, save_dir, traced_call)
-    self.assertAllClose({"output_0": [[8., 10.], [10., 12.]]},
-                        _import_and_infer(
-                            save_dir,
-                            {"inputs": [[4., 5.], [5., 6.]]}))
-
 
 class AssetTests(test.TestCase):
 
-- 
GitLab


From 48bae8da4d68b6aa8de0d9dd373b7f7cfd669123 Mon Sep 17 00:00:00 2001
From: James Keeling <jtkeeling@google.com>
Date: Wed, 23 Jan 2019 08:33:14 -0800
Subject: [PATCH 1115/2345] Collect metrics for graph build time

This is analogous to how TF already collects metrics about graph execution counts and time. It allows users to monitor how long they spend building and optimizing graphs.

PiperOrigin-RevId: 230534860
---
 tensorflow/core/BUILD                         |  1 +
 .../common_runtime/graph_execution_state.cc   |  4 +++-
 tensorflow/core/common_runtime/metrics.cc     | 24 +++++++++++++++++++
 tensorflow/core/common_runtime/metrics.h      | 15 ++++++++++++
 4 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index ceec270f99..87ac325387 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -3036,6 +3036,7 @@ tf_cuda_library(
         ":framework",
         ":graph",
         ":lib",
+        ":metrics",
         ":proto_text",
         ":protos_all_cc",
         "//tensorflow/core/grappler:grappler_item",
diff --git a/tensorflow/core/common_runtime/graph_execution_state.cc b/tensorflow/core/common_runtime/graph_execution_state.cc
index 8875883f62..b185ea1fa5 100644
--- a/tensorflow/core/common_runtime/graph_execution_state.cc
+++ b/tensorflow/core/common_runtime/graph_execution_state.cc
@@ -23,6 +23,7 @@ limitations under the License.
 #include <vector>
 
 #include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/common_runtime/metrics.h"
 #include "tensorflow/core/common_runtime/optimization_registry.h"
 #include "tensorflow/core/common_runtime/placer.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
@@ -732,6 +733,7 @@ Status GraphExecutionState::OptimizeGraph(
 Status GraphExecutionState::BuildGraph(const BuildGraphOptions& options,
                                        std::unique_ptr<ClientGraph>* out) {
   VLOG(1) << "BuildGraph";
+  const uint64 start_time_usecs = Env::Default()->NowMicros();
   if (!graph_) {
     // It is only valid to call this method directly when the original graph
     // was created with the option `place_pruned_graph == false`.
@@ -835,7 +837,7 @@ Status GraphExecutionState::BuildGraph(const BuildGraphOptions& options,
   CopyGraph(*optimized_graph, &dense_copy->graph);
 
   // TODO(vrv): We should check invariants of the graph here.
-
+  metrics::UpdateGraphBuildTime(Env::Default()->NowMicros() - start_time_usecs);
   *out = std::move(dense_copy);
   return Status::OK();
 }
diff --git a/tensorflow/core/common_runtime/metrics.cc b/tensorflow/core/common_runtime/metrics.cc
index 362d3ceca8..a34a580c2a 100644
--- a/tensorflow/core/common_runtime/metrics.cc
+++ b/tensorflow/core/common_runtime/metrics.cc
@@ -38,6 +38,23 @@ auto* tf_data_elements_counter = monitoring::Counter<1>::New(
 auto* tf_data_optimization_counter = monitoring::Counter<1>::New(
     "/tensorflow/data/optimization", "tf.data optimization", "name");
 
+auto* build_graph_calls = monitoring::Counter<0>::New(
+    "/tensorflow/core/graph_build_calls",
+    "The number of times TensorFlow has created a new client graph. "
+    "A client graph is a sub-graph of the full graph, induced by a set of "
+    "options, including the requested feeds and fetches. It includes time "
+    "spent optimizing the graph with Grappler, and time spent pruning the "
+    "sub-graph.");
+
+auto* build_graph_time_usecs = monitoring::Counter<0>::New(
+    "/tensorflow/core/graph_build_time_usecs",
+    "The amount of time TensorFlow has spent creating new client graphs in "
+    "microseconds. "
+    "A client graph is a sub-graph of the full graph, induced by a set of "
+    "options, including the requested feeds and fetches. It includes time "
+    "spent optimizing the graph with Grappler, and time spent pruning the "
+    "sub-graph.");
+
 }  // namespace
 
 void RecordTFDataAutotune(const string& name) {
@@ -59,5 +76,12 @@ void UpdateGraphExecTime(const uint64 running_time_usecs) {
   }
 }
 
+void UpdateGraphBuildTime(const uint64 running_time_usecs) {
+  if (running_time_usecs > 0) {
+    build_graph_calls->GetCell()->IncrementBy(1);
+    build_graph_time_usecs->GetCell()->IncrementBy(running_time_usecs);
+  }
+}
+
 }  // namespace metrics
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/metrics.h b/tensorflow/core/common_runtime/metrics.h
index 9641265362..49dbddd911 100644
--- a/tensorflow/core/common_runtime/metrics.h
+++ b/tensorflow/core/common_runtime/metrics.h
@@ -39,6 +39,21 @@ void RecordTFDataOptimization(const string& name, int64 num_changes);
 
 void UpdateGraphExecTime(const uint64 running_time_usecs);
 
+// Updates the metrics stored about time spent building graphs.
+//
+// By "GraphBuild", we refer to building a client graph, which is a sub-graph of
+// the full graph, induced by a set of options. In particular, these options
+// include the feeds and fetches requested.
+//
+// This includes time spent:
+//   * optimizing the graphs with Grappler
+//   * pruning the sub-graph (unless the place_pruned_graph option is set)
+//
+// When executing eagerly, this will not record any activity.
+//
+// TODO(jtkeeling): Should we record building/optimizing tf.functions?
+void UpdateGraphBuildTime(const uint64 running_time_usecs);
+
 }  // namespace metrics
 }  // namespace tensorflow
 
-- 
GitLab


From a6cf0c6829ea8ee80335636c6754506c72fbd911 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 23 Jan 2019 08:55:34 -0800
Subject: [PATCH 1116/2345] Fix casting issue in 
 sequence_feature_column_v2_test.test_numeric_column.

PiperOrigin-RevId: 230538290
---
 .../python/feature_column/sequence_feature_column_v2_test.py  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py
index 95d8ee278c..a1feaddcc0 100644
--- a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py
+++ b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py
@@ -363,14 +363,14 @@ class SequenceFeaturesTest(test.TestCase, parameterized.TestCase):
            # feature 0, ids [[20, 3], [5]]
            # feature 1, ids [[3], [8]]
            'indices': ((0, 0, 0), (0, 0, 1), (0, 1, 0), (1, 0, 0), (1, 1, 0)),
-           'values': (20, 3, 5., 3., 8.),
+           'values': (20., 3., 5., 3., 8.),
            'dense_shape': (2, 2, 2)},
        'expected_input_layer': [
            [[20.], [3.], [5.], [0.]],
            [[3.], [0.], [8.], [0.]]],
        'expected_sequence_length': [2, 2]},
       )
-  def DISABLED_test_numeric_column(  # See b/123240957
+  def test_numeric_column(
       self, sparse_input_args, expected_input_layer, expected_sequence_length):
     sparse_input = sparse_tensor.SparseTensorValue(**sparse_input_args)
 
-- 
GitLab


From 7ed644962ae016c838fc88cf474ded7c817ea9c9 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Wed, 23 Jan 2019 09:20:56 -0800
Subject: [PATCH 1117/2345] Use std::vector<string> to represent frame nesting
 in and recurrences; NFC

This is a bit cleaner than concatenating the frame names when we create the
AndRec node.

The change should be NFC, but because the hash for the and recurrence nodes have
changed, the order of the and-rec nodes in some of the tests have also changed.

PiperOrigin-RevId: 230542562
---
 tensorflow/compiler/jit/deadness_analysis.cc  | 41 +++++++++----------
 .../compiler/jit/deadness_analysis_test.cc    | 10 ++---
 2 files changed, 25 insertions(+), 26 deletions(-)

diff --git a/tensorflow/compiler/jit/deadness_analysis.cc b/tensorflow/compiler/jit/deadness_analysis.cc
index 22c265adcb..f5ec9af1aa 100644
--- a/tensorflow/compiler/jit/deadness_analysis.cc
+++ b/tensorflow/compiler/jit/deadness_analysis.cc
@@ -245,18 +245,18 @@ class NotPredicate : public Predicate {
 class AndRecurrencePredicate : public Predicate {
  public:
   explicit AndRecurrencePredicate(Predicate* start, Predicate* step,
-                                  string frame)
+                                  std::vector<string> frame)
       : Predicate(Hash(start, step, frame)),
         operands_({start, step}),
         frame_(std::move(frame)) {}
 
   Predicate* start() const { return operands_[0]; }
   Predicate* step() const { return operands_[1]; }
-  absl::string_view frame() const { return frame_; }
+  absl::Span<const string> frame() const { return frame_; }
 
   string ToString() const override {
     return absl::StrCat("{", start()->ToString(), ",&,", step()->ToString(),
-                        "}<", frame(), ">");
+                        "}<", absl::StrJoin(frame(), ";"), ">");
   }
 
   Kind kind() const override { return Kind::kAndRecurrence; }
@@ -267,12 +267,16 @@ class AndRecurrencePredicate : public Predicate {
 
  private:
   std::array<Predicate*, 2> operands_;
-  string frame_;
+  std::vector<string> frame_;
 
-  static int64 Hash(Predicate* start, Predicate* step, const string& frame) {
+  static int64 Hash(Predicate* start, Predicate* step,
+                    const std::vector<string>& frame) {
+    uint64 frame_hash = 0;
+    for (const string& sub_frame : frame) {
+      frame_hash = Hash64Combine(Hash64(sub_frame), frame_hash);
+    }
     return Hash64Combine(
-        HashPredicateSequence(Kind::kAndRecurrence, {start, step}),
-        Hash64(frame));
+        HashPredicateSequence(Kind::kAndRecurrence, {start, step}), frame_hash);
   }
 };
 
@@ -368,7 +372,7 @@ class PredicateFactory {
   }
 
   Predicate* MakeAndRecurrencePredicate(Predicate* start, Predicate* step,
-                                        string frame) {
+                                        std::vector<string> frame) {
     SignatureForAndRec signature(start, step, std::move(frame));
     auto it = interned_and_rec_instances_.find(signature);
     if (it != interned_and_rec_instances_.end()) {
@@ -473,7 +477,8 @@ class PredicateFactory {
   using SignatureForAndOr =
       std::pair<Predicate::Kind, absl::Span<Predicate* const>>;
   using SignatureForNot = Predicate*;
-  using SignatureForAndRec = std::tuple<Predicate*, Predicate*, string>;
+  using SignatureForAndRec =
+      std::tuple<Predicate*, Predicate*, std::vector<string>>;
   using SignatureForSymbol = std::pair<SafeTensorId, bool>;
 
   struct HashSignatureForAndOr {
@@ -887,17 +892,12 @@ Predicate* DeduceStepPredicate(PredicateFactory* predicate_factory,
   return found_sym ? predicate_factory->MakeAndPredicate(and_ops) : nullptr;
 }
 
-Status GetFullFrameName(const Node* n,
-                        absl::Span<const ControlFlowInfo> cfi_infos,
-                        string* full_frame_name) {
+Status GetFullFrame(const Node* n, absl::Span<const ControlFlowInfo> cfi_infos,
+                    std::vector<string>* frame) {
   int depth = 0;
   for (const ControlFlowInfo* cfi_iter = &cfi_infos[n->id()]; !n->IsSource();
        n = cfi_iter->parent_frame, cfi_iter = &cfi_infos[n->id()]) {
-    if (depth > 0) {
-      absl::StrAppend(full_frame_name, ";");
-    }
-
-    absl::StrAppend(full_frame_name, cfi_iter->frame_name);
+    frame->push_back(cfi_iter->frame_name);
 
     if (depth++ > 5000) {
       return errors::Internal(
@@ -972,11 +972,10 @@ Status DeadnessAnalysisImpl::HandleMerge(Node* n,
 
         Predicate* start =
             predicate_factory_.MakeOrPredicate(non_recurrent_inputs);
-        string frame_name;
-        TF_RETURN_IF_ERROR(
-            GetFullFrameName(n, control_flow_info_, &frame_name));
+        std::vector<string> frame;
+        TF_RETURN_IF_ERROR(GetFullFrame(n, control_flow_info_, &frame));
         Predicate* and_rec = predicate_factory_.MakeAndRecurrencePredicate(
-            start, step, std::move(frame_name));
+            start, step, std::move(frame));
         SetPredicate(n, {0, 1, Graph::kControlSlot}, and_rec, should_revisit);
         return Status::OK();
       }
diff --git a/tensorflow/compiler/jit/deadness_analysis_test.cc b/tensorflow/compiler/jit/deadness_analysis_test.cc
index 1adc32cf67..a6a87631bf 100644
--- a/tensorflow/compiler/jit/deadness_analysis_test.cc
+++ b/tensorflow/compiler/jit/deadness_analysis_test.cc
@@ -523,7 +523,7 @@ TEST(DeadnessAnalysisTest, Loop) {
     EXPECT_EQ(predicate_map[ControlOutputFor(add0)],
               "({#true,&,*iv1/cond:0}<fr0> & {#true,&,*iv0/cond:0}<fr0>)");
     EXPECT_EQ(predicate_map[ControlOutputFor(add1)],
-              "({#true,&,*iv2/cond:0}<fr0> & {#true,&,*iv1/cond:0}<fr0>)");
+              "({#true,&,*iv1/cond:0}<fr0> & {#true,&,*iv2/cond:0}<fr0>)");
   }
 }
 
@@ -712,10 +712,10 @@ TEST(DeadnessAnalysisTest, ControlNonEquivalentNestedLoopBodies) {
         "*iv_outer/cond_1:0),&,*iv_inner/cond_1:0}<inner_loop;outer_loop>");
     EXPECT_EQ(
         predicate_map[ControlOutputFor(add0)],
-        "({({#true,&,*iv_outer/cond_1:0}<outer_loop> & "
-        "*iv_outer/cond_1:0),&,*iv_inner/cond_1:0}<inner_loop;outer_loop> & "
-        "{({#true,&,*iv_outer/cond:0}<outer_loop> & "
-        "*iv_outer/cond:0),&,*iv_inner/cond:0}<inner_loop;outer_loop>)");
+        "({({#true,&,*iv_outer/cond:0}<outer_loop> & "
+        "*iv_outer/cond:0),&,*iv_inner/cond:0}<inner_loop;outer_loop> & "
+        "{({#true,&,*iv_outer/cond_1:0}<outer_loop> & "
+        "*iv_outer/cond_1:0),&,*iv_inner/cond_1:0}<inner_loop;outer_loop>)");
   }
 }
 
-- 
GitLab


From 677ffec245c1a27515d6ef7b1b5ae7491414e500 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Wed, 23 Jan 2019 09:34:06 -0800
Subject: [PATCH 1118/2345] [tf.data] Adding `shared_name` argument to
 `tf.data.make_initializable_iterator` and fixing its documentation.

PiperOrigin-RevId: 230544635
---
 tensorflow/python/data/ops/dataset_ops.py            | 11 +++++++----
 tensorflow/tools/api/golden/v1/tensorflow.data.pbtxt |  2 +-
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index 45e732be0d..104f99a3d4 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -1733,7 +1733,7 @@ def make_one_shot_iterator(dataset):
 
 
 @tf_export(v1=["data.make_initializable_iterator"])
-def make_initializable_iterator(dataset):
+def make_initializable_iterator(dataset, shared_name=None):
   """Creates a `tf.data.Iterator` for enumerating the elements of a dataset.
 
   Note: The returned iterator will be in an uninitialized state,
@@ -1741,13 +1741,16 @@ def make_initializable_iterator(dataset):
 
   ```python
   dataset = ...
-  iterator = dataset.make_initializable_iterator()
+  iterator = tf.data.make_initializable_iterator(dataset)
   # ...
   sess.run(iterator.initializer)
   ```
 
   Args:
     dataset: A `tf.data.Dataset`.
+    shared_name: (Optional.) If non-empty, the returned iterator will be
+      shared under the given name across multiple sessions that share the
+      same devices (e.g. when using a remote server).
 
   Returns:
     A `tf.data.Iterator` over the elements of `dataset`.
@@ -1758,9 +1761,9 @@ def make_initializable_iterator(dataset):
   try:
     # Call the defined `make_initializable_iterator()` if there is one, because
     # some datasets (e.g. for prefetching) override its behavior.
-    return dataset.make_initializable_iterator()
+    return dataset.make_initializable_iterator(shared_name)
   except AttributeError:
-    return DatasetV1Adapter(dataset).make_initializable_iterator()
+    return DatasetV1Adapter(dataset).make_initializable_iterator(shared_name)
 
 
 @tf_export("data.Options")
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.pbtxt
index aa47468059..272963382a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.pbtxt
@@ -30,7 +30,7 @@ tf_module {
   }
   member_method {
     name: "make_initializable_iterator"
-    argspec: "args=[\'dataset\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'dataset\', \'shared_name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "make_one_shot_iterator"
-- 
GitLab


From f1df907bbb41d96a6a220bee09586b0915571335 Mon Sep 17 00:00:00 2001
From: mbhuiyan <mohammad.ashraf.bhuiyan@intel.com>
Date: Wed, 23 Jan 2019 09:55:56 -0800
Subject: [PATCH 1119/2345] cleaning up mkl relu (removing MKL ML related code)

---
 tensorflow/core/kernels/mkl_relu_op.cc | 359 +------------------------
 1 file changed, 9 insertions(+), 350 deletions(-)

diff --git a/tensorflow/core/kernels/mkl_relu_op.cc b/tensorflow/core/kernels/mkl_relu_op.cc
index d8ab1cd25b..e6cbd69820 100644
--- a/tensorflow/core/kernels/mkl_relu_op.cc
+++ b/tensorflow/core/kernels/mkl_relu_op.cc
@@ -16,15 +16,14 @@ limitations under the License.
 // See docs in ../ops/nn_ops.cc.
 #ifdef INTEL_MKL
 
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "mkldnn.hpp"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/lib/core/errors.h"
-
-#ifndef INTEL_MKL_ML_ONLY
-#include "mkldnn.hpp"
+#include "tensorflow/core/util/mkl_util.h"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
 using mkldnn::algorithm;
 using mkldnn::eltwise_bounded_relu;
@@ -36,16 +35,9 @@ using mkldnn::prop_kind;
 using mkldnn::relu_backward;
 using mkldnn::relu_forward;
 using mkldnn::stream;
-#else
-#include "mkl_dnn.h"
-#include "mkl_dnn_types.h"
-#endif
-#include "tensorflow/core/util/mkl_util.h"
 
 namespace tensorflow {
 
-#ifndef INTEL_MKL_ML_ONLY
-
 template <typename T>
 class MklEltwiseFwdParams {
  public:
@@ -451,335 +443,8 @@ class MklEltwiseBwdPrimitiveFactory : public MklPrimitiveFactory<T> {
   }
 };
 
-#endif
-
 typedef Eigen::ThreadPoolDevice CPUDevice;
 
-struct MklReluHelpers {
-  static void ValidateSameSizeHelper(OpKernelContext* context, const Tensor& g,
-                                     const Tensor& a) {
-    OP_REQUIRES(context, a.IsSameSize(g),
-                errors::InvalidArgument("g and a must be the same size"));
-  }
-  static bool ValidateSameSize(OpKernelContext* context, const Tensor& g,
-                               const Tensor& a) {
-    ValidateSameSizeHelper(context, g, a);
-    return context->status().ok();
-  }
-};
-
-#ifdef INTEL_MKL_ML_ONLY
-
-template <typename Device, typename T>
-class MklReluOp : public OpKernel {
- public:
-  ~MklReluOp() {}
-
-  explicit MklReluOp(OpKernelConstruction* context) : OpKernel(context) {}
-
-  void Compute(OpKernelContext* context) override {
-    MklReluOpContext mkl_context;
-
-    const Tensor& input = MklGetInput(context, 0);
-    GetMklShape(context, 0, &mkl_context.input_shape);
-    void* user_i = static_cast<void*>(const_cast<T*>(input.flat<T>().data()));
-    bool input_in_mkl_format = mkl_context.input_shape.IsMklTensor();
-
-    if (!input_in_mkl_format && !input.dims()) {  // handle the case of a scalar
-      const TensorShape& o_shape = input.shape();
-      Tensor* out_tensor = nullptr;
-      mkl_context.output_shape.SetMklTensor(false);
-      AllocateOutputSetMklShape(context, 0, &out_tensor, o_shape,
-                                mkl_context.output_shape);
-      void* out_o = static_cast<void*>(out_tensor->flat<T>().data());
-      (static_cast<T*>(out_o))[0] =
-          std::max((static_cast<T*>(user_i))[0], static_cast<T>(0));
-      return;
-    }
-
-    // Generate size, stride for input if input is in MKL format.
-    if (input_in_mkl_format) {
-      mkl_context.in_dims = mkl_context.input_shape.GetDimension();
-      mkl_context.in_sizes = new size_t[mkl_context.in_dims];
-      mkl_context.in_strides = new size_t[mkl_context.in_dims];
-      for (int i = 0; i < mkl_context.in_dims; i++) {
-        mkl_context.in_sizes[i] = mkl_context.input_shape.GetSizes()[i];
-        mkl_context.in_strides[i] = mkl_context.input_shape.GetStrides()[i];
-      }
-    } else {
-      mkl_context.in_dims = input.dims();
-      mkl_context.in_sizes = new size_t[mkl_context.in_dims];
-      mkl_context.in_strides = new size_t[mkl_context.in_dims];
-      for (int i = 0; i < mkl_context.in_dims; i++) {
-        mkl_context.in_sizes[i] = input.dim_size((mkl_context.in_dims - 1) - i);
-      }
-      mkl_context.in_strides[0] = 1;
-      for (int i = 1; i < mkl_context.in_dims; i++) {
-        mkl_context.in_strides[i] =
-            mkl_context.in_strides[i - 1] * mkl_context.in_sizes[i - 1];
-      }
-    }
-
-    float negative_slope = 0.0;
-    mkl_context.MklCreateInputLayouts(context);
-    CHECK_EQ(dnnReLUCreateForward_F32(&mkl_context.prim_relu_fwd, NULL,
-                                      mkl_context.lt_input, negative_slope),
-             E_SUCCESS);
-
-    Tensor* output = nullptr;
-
-    if (input_in_mkl_format) {
-      TensorShape tf_shape;
-      mkl_context.output_shape.SetMklTensor(true);
-      mkl_context.output_shape.SetMklLayout(mkl_context.prim_relu_fwd,
-                                            dnnResourceDst);
-      mkl_context.output_shape.SetTfLayout(
-          mkl_context.in_dims, mkl_context.in_sizes, mkl_context.in_strides);
-      mkl_context.output_shape.SetTfDimOrder(
-          mkl_context.in_dims, mkl_context.input_shape.GetTfToMklDimMap());
-      tf_shape.AddDim(dnnLayoutGetMemorySize_F32(static_cast<dnnLayout_t>(
-                          mkl_context.output_shape.GetMklLayout())) /
-                      sizeof(T));
-      AllocateOutputSetMklShape(context, 0, &output, tf_shape,
-                                mkl_context.output_shape);
-    } else {
-      const TensorShape& o_shape = input.shape();
-      mkl_context.output_shape.SetMklTensor(false);
-      AllocateOutputSetMklShape(context, 0, &output, o_shape,
-                                mkl_context.output_shape);
-    }
-
-    void* user_o = static_cast<void*>(const_cast<T*>(output->flat<T>().data()));
-
-    mkl_context.relu_res[dnnResourceDst] = user_o;
-    mkl_context.relu_res[dnnResourceSrc] = user_i;
-    CHECK_EQ(dnnExecute_F32(mkl_context.prim_relu_fwd, mkl_context.relu_res),
-             E_SUCCESS);
-    mkl_context.MklCleanup();
-  }
-
- private:
-  typedef struct {
-    int in_dims;
-    size_t* in_sizes;
-    size_t* in_strides;
-    MklShape input_shape, output_shape;
-    dnnPrimitive_t prim_relu_fwd = nullptr;
-    void* relu_res[dnnResourceNumber];
-    dnnLayout_t lt_input = nullptr;
-
-    void MklCleanup() {
-      bool input_in_mkl_format = input_shape.IsMklTensor();
-      if (!input_in_mkl_format) {
-        dnnLayoutDelete_F32(lt_input);
-        free(in_sizes);
-        free(in_strides);
-      }
-      dnnDelete_F32(prim_relu_fwd);
-    }
-
-    void MklCreateInputLayouts(OpKernelContext* context) {
-      bool input_in_mkl_format = input_shape.IsMklTensor();
-      if (!input_in_mkl_format) {
-        CHECK_EQ(dnnLayoutCreate_F32(&lt_input, in_dims, in_sizes, in_strides),
-                 E_SUCCESS);
-      } else {
-        lt_input = static_cast<dnnLayout_t>(input_shape.GetCurLayout());
-      }
-    }
-  } MklReluOpContext;
-};
-
-template <typename Device, typename T>
-class MklReluGradOp : public OpKernel {
- public:
-  ~MklReluGradOp() {}
-
-  explicit MklReluGradOp(OpKernelConstruction* context) : OpKernel(context) {}
-
-  void Compute(OpKernelContext* context) override;
-
- private:
-  typedef struct {
-    int in_dims;
-    size_t* in_sizes;
-    size_t* in_strides;
-    MklShape input_shape, grad_shape, output_shape;
-    void* relu_res[dnnResourceNumber];
-    dnnPrimitive_t prim_relu_bwd;
-    dnnLayout_t lt_input, lt_grad;
-
-    void MklPrepareReluGradInputs(OpKernelContext* context,
-                                  Tensor* mkl_tmp_input_buf_tensor) {
-      const Tensor& g = MklGetInput(context, 0);
-      const Tensor& a = MklGetInput(context, 1);
-      void* buf_input = static_cast<void*>(const_cast<T*>(a.flat<T>().data()));
-      void* mkl_buffer_convert = nullptr;
-
-      dnnPrimitive_t cv_input_to_grad = nullptr;
-
-      // if input and grad are not in the same layout,
-      // do a conversion between them.
-      if (!dnnLayoutCompare_F32(lt_input, lt_grad)) {
-        AllocTmpBuffer(context, mkl_tmp_input_buf_tensor, lt_grad,
-                       &mkl_buffer_convert);
-        CHECK_EQ(dnnConversionCreate_F32(&cv_input_to_grad, lt_input, lt_grad),
-                 E_SUCCESS);
-        CHECK_EQ(dnnConversionExecute_F32(cv_input_to_grad, buf_input,
-                                          mkl_buffer_convert),
-                 E_SUCCESS);
-        relu_res[dnnResourceSrc] = mkl_buffer_convert;
-        dnnDelete_F32(cv_input_to_grad);
-      } else {
-        relu_res[dnnResourceSrc] = buf_input;
-      }
-
-      void* buf_grad = static_cast<void*>(const_cast<T*>(g.flat<T>().data()));
-      relu_res[dnnResourceDiffDst] = buf_grad;
-    }
-
-    void MklCreateInputLayouts(OpKernelContext* context) {
-      bool grad_is_mkl = grad_shape.IsMklTensor();
-      bool input_is_mkl = input_shape.IsMklTensor();
-      if (!input_is_mkl) {
-        CHECK_EQ(dnnLayoutCreate_F32(&lt_input, in_dims, in_sizes, in_strides),
-                 E_SUCCESS);
-      } else {
-        lt_input = static_cast<dnnLayout_t>(input_shape.GetCurLayout());
-      }
-
-      if (!grad_is_mkl) {
-        CHECK_EQ(dnnLayoutCreate_F32(&lt_grad, in_dims, in_sizes, in_strides),
-                 E_SUCCESS);
-      } else {
-        lt_grad = static_cast<dnnLayout_t>(grad_shape.GetCurLayout());
-      }
-    }
-
-    void MklCleanup() {
-      bool grad_is_mkl = grad_shape.IsMklTensor();
-      bool input_is_mkl = input_shape.IsMklTensor();
-      dnnDelete_F32(prim_relu_bwd);
-      if (!input_is_mkl) {
-        dnnLayoutDelete_F32(lt_input);
-        free(in_sizes);
-        free(in_strides);
-      }
-      if (!grad_is_mkl) {
-        dnnLayoutDelete_F32(lt_grad);
-      }
-    }
-  } MklReluGradOpContext;
-};
-
-template <typename Device, typename T>
-void MklReluGradOp<Device, T>::Compute(OpKernelContext* context) {
-  MklReluGradOpContext mkl_context;
-  const Tensor& g = MklGetInput(context, 0);
-  const Tensor& a = MklGetInput(context, 1);
-
-  void* user_i = static_cast<void*>(const_cast<T*>(a.flat<T>().data()));
-  void* user_g = static_cast<void*>(const_cast<T*>(g.flat<T>().data()));
-
-  GetMklShape(context, 0, &mkl_context.grad_shape);
-  GetMklShape(context, 1, &mkl_context.input_shape);
-
-  bool grad_is_mkl = mkl_context.grad_shape.IsMklTensor();
-  bool input_is_mkl = mkl_context.input_shape.IsMklTensor();
-  if (!input_is_mkl && !grad_is_mkl &&
-      !MklReluHelpers::ValidateSameSize(context, g, a))
-    return;
-  Tensor* output = nullptr;
-
-  if (!input_is_mkl && !grad_is_mkl && !a.dims()) {
-    // handle the scalar case
-    const TensorShape& g_shape = g.shape();
-    mkl_context.output_shape.SetMklTensor(false);
-    AllocateOutputSetMklShape(context, 0, &output, g_shape,
-                              mkl_context.output_shape);
-
-    void* out_o = static_cast<void*>(output->flat<T>().data());
-    (static_cast<T*>(out_o))[0] =
-        (static_cast<T*>(user_g))[0] * ((static_cast<T*>(user_i))[0] > 0);
-    return;
-  }
-
-  // generate size, stride for input if input/grad is in mkl format.
-  if (grad_is_mkl || input_is_mkl) {
-    const MklShape* tmp_mkl_shape =
-        (grad_is_mkl) ? &mkl_context.grad_shape : &mkl_context.input_shape;
-
-    mkl_context.in_dims = tmp_mkl_shape->GetDimension();
-    mkl_context.in_strides = new size_t[mkl_context.in_dims];
-    mkl_context.in_sizes = new size_t[mkl_context.in_dims];
-    for (int i = 0; i < mkl_context.in_dims; i++) {
-      mkl_context.in_sizes[i] = tmp_mkl_shape->GetSizes()[i];
-      mkl_context.in_strides[i] = tmp_mkl_shape->GetStrides()[i];
-    }
-  } else {
-    mkl_context.in_dims = g.dims();
-    mkl_context.in_strides = new size_t[mkl_context.in_dims];
-    mkl_context.in_sizes = new size_t[mkl_context.in_dims];
-
-    for (int i = 0; i < mkl_context.in_dims; i++) {
-      mkl_context.in_sizes[i] = g.dim_size((mkl_context.in_dims - 1) - i);
-    }
-    mkl_context.in_strides[0] = 1;
-    for (int i = 1; i < mkl_context.in_dims; i++) {
-      mkl_context.in_strides[i] =
-          mkl_context.in_strides[i - 1] * mkl_context.in_sizes[i - 1];
-    }
-  }
-
-  mkl_context.MklCreateInputLayouts(context);
-  float negative_slope = 0.0;
-  CHECK_EQ(dnnReLUCreateBackward_F32(&mkl_context.prim_relu_bwd, NULL,
-                                     mkl_context.lt_grad, mkl_context.lt_grad,
-                                     negative_slope),
-           E_SUCCESS);
-  Tensor mkl_tmp_input_buf_tensor;
-  mkl_context.MklPrepareReluGradInputs(context, &mkl_tmp_input_buf_tensor);
-
-  if (input_is_mkl ||
-      grad_is_mkl) { /*if  grad or input are mkl leave it in mkl*/
-    TensorShape tf_shape;
-    mkl_context.output_shape.SetMklTensor(true);
-    mkl_context.output_shape.SetMklLayout(mkl_context.prim_relu_bwd,
-                                          dnnResourceDiffSrc);
-    mkl_context.output_shape.SetTfLayout(
-        mkl_context.in_dims, mkl_context.in_sizes, mkl_context.in_strides);
-    // if input_is_mkl or grad_is_mkl, then we copy strides and sizes from mkl
-    // shape of one that is in mkl layout.
-    if (grad_is_mkl == true) {
-      mkl_context.output_shape.SetTfDimOrder(
-          mkl_context.in_dims, mkl_context.grad_shape.GetTfToMklDimMap());
-    } else {
-      mkl_context.output_shape.SetTfDimOrder(
-          mkl_context.in_dims, mkl_context.input_shape.GetTfToMklDimMap());
-    }
-
-    tf_shape.AddDim(dnnLayoutGetMemorySize_F32(static_cast<dnnLayout_t>(
-                        mkl_context.output_shape.GetMklLayout())) /
-                    sizeof(T));
-    AllocateOutputSetMklShape(context, 0, &output, tf_shape,
-                              mkl_context.output_shape);
-  } else {
-    const TensorShape& o_shape = g.shape();
-    mkl_context.output_shape.SetMklTensor(false);
-    AllocateOutputSetMklShape(context, 0, &output, o_shape,
-                              mkl_context.output_shape);
-  }
-
-  mkl_context.relu_res[dnnResourceDiffSrc] =
-      static_cast<void*>(output->flat<T>().data());
-
-  CHECK_EQ(dnnExecute_F32(mkl_context.prim_relu_bwd, mkl_context.relu_res),
-           E_SUCCESS);
-  mkl_context.MklCleanup();
-}
-
-#else  // INTEL_MKL_ML_ONLY
-
 template <typename Device, typename T, algorithm alg_kind>
 class MklReluOpBase : public OpKernel {
  public:
@@ -866,9 +531,9 @@ class MklReluOpBase : public OpKernel {
       // execute eltwise
       eltwise_fwd->Execute(src_data, dst_data);
     } catch (mkldnn::error& e) {
-      string error_msg = "Status: " + std::to_string(e.status) +
-                         ", message: " + string(e.message) + ", in file " +
-                         string(__FILE__) + ":" + std::to_string(__LINE__);
+      string error_msg = "Status: " + std::to_string(e.status) + ", message: " +
+                         string(e.message) + ", in file " + string(__FILE__) +
+                         ":" + std::to_string(__LINE__);
       OP_REQUIRES_OK(
           context,
           errors::Aborted("Operation received an exception:", error_msg));
@@ -1034,9 +699,9 @@ class MklReluGradOpBase : public OpKernel {
       // execute eltwise bwd
       eltwise_bwd->Execute(src_data, diff_dst_data, diff_src_data);
     } catch (mkldnn::error& e) {
-      string error_msg = "Status: " + std::to_string(e.status) +
-                         ", message: " + string(e.message) + ", in file " +
-                         string(__FILE__) + ":" + std::to_string(__LINE__);
+      string error_msg = "Status: " + std::to_string(e.status) + ", message: " +
+                         string(e.message) + ", in file " + string(__FILE__) +
+                         ":" + std::to_string(__LINE__);
       OP_REQUIRES_OK(
           context,
           errors::Aborted("Operation received an exception:", error_msg));
@@ -1399,8 +1064,6 @@ class MklLeakyReluGradOp : public MklReluGradOpBase<Device, T, eltwise_relu> {
   }
 };
 
-#endif
-
 // register dnn kernels for supported operations and supported types
 #define REGISTER_RELU_MKL_SUPPORTED_KERNELS_TYPES(type)             \
   REGISTER_KERNEL_BUILDER(Name("_MklRelu")                          \
@@ -1415,8 +1078,6 @@ class MklLeakyReluGradOp : public MklReluGradOpBase<Device, T, eltwise_relu> {
                           MklReluGradOp<CPUDevice, type>);
 TF_CALL_float(REGISTER_RELU_MKL_SUPPORTED_KERNELS_TYPES);
 
-#ifndef INTEL_MKL_ML_ONLY
-
 // register dnn kernels for supported operations and supported types
 #define REGISTER_ELU_MKL_SUPPORTED_KERNELS_TYPES(type)              \
   REGISTER_KERNEL_BUILDER(Name("_MklElu")                           \
@@ -1470,8 +1131,6 @@ TF_CALL_float(REGISTER_RELU6_MKL_SUPPORTED_KERNELS_TYPES);
                           MklLeakyReluGradOp<CPUDevice, type>);
 TF_CALL_float(REGISTER_LeakyRelu_MKL_SUPPORTED_KERNELS_TYPES);
 
-#endif
-
 }  // namespace tensorflow
 
 #endif  // INTEL_MKL
-- 
GitLab


From cab236de006379ffd9e148fa0594f8ab949155fa Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Wed, 23 Jan 2019 09:50:58 -0800
Subject: [PATCH 1120/2345] Teach deadness analysis to peek into constant
 Switch conditions

PiperOrigin-RevId: 230547549
---
 tensorflow/compiler/jit/deadness_analysis.cc  | 50 +++++++++++++++----
 .../compiler/jit/deadness_analysis_test.cc    | 38 ++++++++++++++
 2 files changed, 77 insertions(+), 11 deletions(-)

diff --git a/tensorflow/compiler/jit/deadness_analysis.cc b/tensorflow/compiler/jit/deadness_analysis.cc
index f5ec9af1aa..08fabb3e30 100644
--- a/tensorflow/compiler/jit/deadness_analysis.cc
+++ b/tensorflow/compiler/jit/deadness_analysis.cc
@@ -20,6 +20,8 @@ limitations under the License.
 #include "absl/strings/str_join.h"
 #include "tensorflow/compiler/jit/deadness_analysis_internal.h"
 #include "tensorflow/compiler/jit/xla_cluster_util.h"
+#include "tensorflow/compiler/xla/status_macros.h"
+#include "tensorflow/core/framework/tensor.pb.h"
 #include "tensorflow/core/graph/algorithm.h"
 #include "tensorflow/core/graph/control_flow.h"
 #include "tensorflow/core/graph/tensor_id.h"
@@ -304,7 +306,7 @@ class SymbolPredicate : public Predicate {
   // "tensor_id() is live and evaluates to true".
   //
   // If `must_be_true()` is false then this SymbolPredicate represents the
-  // proposition "tensor_id() is live (and may evalutate to any value)"
+  // proposition "tensor_id() is live (and may evaluate to any value)"
   TensorId tensor_id() const { return tensor_id_; }
   bool must_be_true() const { return must_be_true_; }
 
@@ -390,7 +392,24 @@ class PredicateFactory {
     return new_pred_ptr;
   }
 
-  Predicate* MakeSymbolPredicate(TensorId tensor_id, bool must_be_true) {
+  Status MakeSymbolPredicate(Node* node, int output_idx, bool must_be_true,
+                             Predicate** predicate) {
+    TensorId tensor_id(node->name(), output_idx);
+
+    bool is_boolean_tensor = node->output_type(tensor_id.index()) == DT_BOOL;
+    TF_RET_CHECK(!must_be_true || is_boolean_tensor);
+
+    if (node->type_string() == "Const" && must_be_true) {
+      const TensorProto* proto = nullptr;
+      TF_RETURN_IF_ERROR(GetNodeAttr(node->def(), "value", &proto));
+
+      Tensor tensor(proto->dtype());
+      TF_RET_CHECK(tensor.FromProto(*proto));
+
+      *predicate = tensor.scalar<bool>()() ? MakeTrue() : MakeFalse();
+      return Status::OK();
+    }
+
     SignatureForSymbol signature = {tensor_id, must_be_true};
     auto it = interned_symbol_instances_.find(signature);
     if (it == interned_symbol_instances_.end()) {
@@ -399,10 +418,12 @@ class PredicateFactory {
       Predicate* new_pred_ptr = new_pred.get();
       interned_symbol_instances_.emplace(std::move(signature),
                                          std::move(new_pred));
-      return new_pred_ptr;
+      *predicate = new_pred_ptr;
     } else {
-      return it->second.get();
+      *predicate = it->second.get();
     }
+
+    return Status::OK();
   }
 
   Predicate* MakeTrue() { return MakeAndPredicate({}); }
@@ -791,9 +812,12 @@ Status DeadnessAnalysisImpl::HandleSwitch(Node* n,
   TF_RETURN_IF_ERROR(GetInputPreds(n, EdgeKind::kDataAndControl, &input_preds));
   const Edge* pred_edge;
   TF_RETURN_IF_ERROR(n->input_edge(1, &pred_edge));
-  Predicate* true_switch = predicate_factory_.MakeSymbolPredicate(
-      TensorId(pred_edge->src()->name(), pred_edge->src_output()),
-      /*must_be_true=*/true);
+
+  Predicate* true_switch;
+  TF_RETURN_IF_ERROR(predicate_factory_.MakeSymbolPredicate(
+      pred_edge->src(), pred_edge->src_output(),
+      /*must_be_true=*/true, &true_switch));
+
   Predicate* false_switch = predicate_factory_.MakeNotPredicate(true_switch);
 
   // Output 0 is alive iff all inputs are alive and the condition is false.
@@ -930,8 +954,10 @@ Status DeadnessAnalysisImpl::HandleMerge(Node* n,
     if (has_unvisited_backedge) {
       // We're visiting this merge for the first time and it has an unvisited
       // backedge.
-      Predicate* input_data_pred = predicate_factory_.MakeSymbolPredicate(
-          TensorId(n->name(), 0), /*must_be_true=*/false);
+      Predicate* input_data_pred;
+      TF_RETURN_IF_ERROR(predicate_factory_.MakeSymbolPredicate(
+          n, /*output_idx=*/0, /*must_be_true=*/false, &input_data_pred));
+
       SetPredicate(n, {0, 1, Graph::kControlSlot}, input_data_pred,
                    should_revisit);
       return Status::OK();
@@ -990,8 +1016,10 @@ Status DeadnessAnalysisImpl::HandleRecv(Node* n,
   // acquire a dead signal from a _Send.
   std::vector<Predicate*> input_preds;
   TF_RETURN_IF_ERROR(GetInputPreds(n, EdgeKind::kDataAndControl, &input_preds));
-  input_preds.push_back(predicate_factory_.MakeSymbolPredicate(
-      TensorId(n->name(), 0), /*must_be_true=*/false));
+  Predicate* signal_is_alive;
+  TF_RETURN_IF_ERROR(predicate_factory_.MakeSymbolPredicate(
+      n, /*output_idx=*/0, /*must_be_true=*/false, &signal_is_alive));
+  input_preds.push_back(signal_is_alive);
   SetPredicate(n, {0, Graph::kControlSlot},
                predicate_factory_.MakeAndPredicate(input_preds),
                should_revisit);
diff --git a/tensorflow/compiler/jit/deadness_analysis_test.cc b/tensorflow/compiler/jit/deadness_analysis_test.cc
index a6a87631bf..16ee8f86d5 100644
--- a/tensorflow/compiler/jit/deadness_analysis_test.cc
+++ b/tensorflow/compiler/jit/deadness_analysis_test.cc
@@ -919,5 +919,43 @@ TEST(DeadnessAnalysisTest, DeMorgan) {
   EXPECT_EQ(predicate_map[ControlOutputFor(should_always_be_alive)], "#true");
 }
 
+TEST(DeadnessAnalysisTest, ConstantTrueSwitchCondition) {
+  Scope root = Scope::NewRootScope().ExitOnError();
+
+  Output constant_true = ops::Const(root.WithOpName("const_true"), true);
+  Output value = ops::Placeholder(root.WithOpName("value"), DT_FLOAT);
+  ops::Switch sw(root.WithOpName("switch"), value, constant_true);
+
+  Output id_false = ops::Identity(root.WithOpName("id_false"), sw.output_false);
+  Output id_true = ops::Identity(root.WithOpName("id_true"), sw.output_true);
+
+  FixupSourceAndSinkEdges(root.graph());
+
+  PredicateMapTy predicate_map;
+  TF_ASSERT_OK(ComputePredicates(*root.graph(), &predicate_map));
+
+  EXPECT_EQ(predicate_map[ControlOutputFor(id_false)], "#false");
+  EXPECT_EQ(predicate_map[ControlOutputFor(id_true)], "#true");
+}
+
+TEST(DeadnessAnalysisTest, ConstantFalseSwitchCondition) {
+  Scope root = Scope::NewRootScope().ExitOnError();
+
+  Output constant_false = ops::Const(root.WithOpName("const_false"), false);
+  Output value = ops::Placeholder(root.WithOpName("value"), DT_FLOAT);
+  ops::Switch sw(root.WithOpName("switch"), value, constant_false);
+
+  Output id_false = ops::Identity(root.WithOpName("id_false"), sw.output_false);
+  Output id_true = ops::Identity(root.WithOpName("id_true"), sw.output_true);
+
+  FixupSourceAndSinkEdges(root.graph());
+
+  PredicateMapTy predicate_map;
+  TF_ASSERT_OK(ComputePredicates(*root.graph(), &predicate_map));
+
+  EXPECT_EQ(predicate_map[ControlOutputFor(id_false)], "#true");
+  EXPECT_EQ(predicate_map[ControlOutputFor(id_true)], "#false");
+}
+
 }  // namespace
 }  // namespace tensorflow
-- 
GitLab


From eac1f2732f67f95f9fb7bfca3fede93e59fd6822 Mon Sep 17 00:00:00 2001
From: Alan Chiao <alanchiao@google.com>
Date: Wed, 23 Jan 2019 10:09:39 -0800
Subject: [PATCH 1121/2345] Internalize custom op.

PiperOrigin-RevId: 230551643
---
 tensorflow/lite/kernels/BUILD                 |  15 --
 .../kernels/sparse_output_fully_connected.cc  | 252 ------------------
 .../sparse_output_fully_connected_test.cc     | 192 -------------
 3 files changed, 459 deletions(-)
 delete mode 100644 tensorflow/lite/kernels/sparse_output_fully_connected.cc
 delete mode 100644 tensorflow/lite/kernels/sparse_output_fully_connected_test.cc

diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index b93dda7382..cb0d1f8746 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -207,7 +207,6 @@ cc_library(
         "slice.cc",
         "space_to_batch_nd.cc",
         "space_to_depth.cc",
-        "sparse_output_fully_connected.cc",
         "sparse_to_dense.cc",
         "split.cc",
         "split_v.cc",
@@ -344,20 +343,6 @@ tf_cc_test(
     ],
 )
 
-tf_cc_test(
-    name = "sparse_output_fully_connected_test",
-    size = "small",
-    srcs = ["sparse_output_fully_connected_test.cc"],
-    deps = [
-        ":builtin_ops",
-        "//tensorflow/lite:framework",
-        "//tensorflow/lite/kernels:test_util",
-        "//tensorflow/lite/kernels/internal:types",
-        "@com_google_googletest//:gtest",
-        "@flatbuffers",
-    ],
-)
-
 tf_cc_test(
     name = "activations_test",
     size = "small",
diff --git a/tensorflow/lite/kernels/sparse_output_fully_connected.cc b/tensorflow/lite/kernels/sparse_output_fully_connected.cc
deleted file mode 100644
index 248969535c..0000000000
--- a/tensorflow/lite/kernels/sparse_output_fully_connected.cc
+++ /dev/null
@@ -1,252 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-// SparseOutputFullyConnected is a fully connected layer that uses a single
-// row in the weights and bias via a lookup.
-#include "tensorflow/lite/context.h"
-#include "tensorflow/lite/kernels/internal/tensor.h"
-#include "tensorflow/lite/kernels/internal/tensor_utils.h"
-#include "tensorflow/lite/kernels/kernel_util.h"
-
-namespace tflite {
-namespace ops {
-namespace custom {
-namespace sparse_output_fully_connected {
-
-// Input tensors of size {n_batch, n_input}
-constexpr int kInputTensor = 0;
-// Auxiliary input tensor of size { 1 }
-constexpr int kInputLookupTensor = 1;
-
-// Weights tensor of size { n_embeddings , n_input }
-constexpr int kWeightsTensor = 2;
-// Bias tensor of size { n_embeddings }
-constexpr int kBiasTensor = 3;
-
-// Output tensor.
-constexpr int kOutputTensor = 0;
-
-// Temporary tensors.
-enum TemporaryTensor {
-  kInputQuantized = 0,
-  kScalingFactors = 1,
-  kNumTemporaryTensors = 2
-};
-
-// Struct to hold op data.
-struct OpData {
-  int scratch_tensor_index;
-};
-
-void* Init(TfLiteContext* context, const char* buffer, size_t length) {
-  auto* data = new OpData;
-  context->AddTensors(context, /*tensors_to_add=*/kNumTemporaryTensors,
-                      &data->scratch_tensor_index);
-  return data;
-}
-
-void Free(TfLiteContext* context, void* buffer) {
-  delete reinterpret_cast<OpData*>(buffer);
-}
-
-TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
-  OpData* op_data = reinterpret_cast<OpData*>(node->user_data);
-
-  TF_LITE_ENSURE_EQ(context, node->inputs->size, 4);
-  TF_LITE_ENSURE_EQ(context, node->outputs->size, 1);
-
-  const TfLiteTensor* input = GetInput(context, node, kInputTensor);
-  TF_LITE_ENSURE_EQ(context, input->type, kTfLiteFloat32);
-  TF_LITE_ENSURE_EQ(context, NumDimensions(input), 2);
-  const int n_batch = SizeOfDimension(input, 0);
-  const int n_input = SizeOfDimension(input, 1);
-
-  const TfLiteTensor* lookup = GetInput(context, node, kInputLookupTensor);
-  TF_LITE_ENSURE_EQ(context, lookup->type, kTfLiteInt32);
-  TF_LITE_ENSURE_EQ(context, NumDimensions(lookup), 1);
-  // Only support single lookup.
-  TF_LITE_ENSURE_EQ(context, SizeOfDimension(lookup, 0), 1);
-
-  const TfLiteTensor* weights = GetInput(context, node, kWeightsTensor);
-  TF_LITE_ENSURE_EQ(context, NumDimensions(weights), 2);
-  TF_LITE_ENSURE_EQ(context, SizeOfDimension(weights, 1), n_input);
-
-  const TfLiteTensor* bias = GetInput(context, node, kBiasTensor);
-  TF_LITE_ENSURE_EQ(context, NumElements(bias), SizeOfDimension(weights, 0));
-
-  const bool is_hybrid_op =
-      ((weights->type == kTfLiteUInt8 || weights->type == kTfLiteInt8) &&
-       input->type == kTfLiteFloat32);
-
-  // Resize output.
-  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
-  TfLiteIntArray* output_size_array = TfLiteIntArrayCreate(1);
-  output_size_array->data[0] = 1;
-  TF_LITE_ENSURE_OK(context,
-                    context->ResizeTensor(context, output, output_size_array));
-
-  if (is_hybrid_op) {
-    TfLiteIntArrayFree(node->temporaries);
-    node->temporaries = TfLiteIntArrayCreate(kNumTemporaryTensors);
-
-    // Allocate temporary tensors to store quantized values of input.
-    node->temporaries->data[kInputQuantized] = op_data->scratch_tensor_index;
-    TfLiteTensor* input_quantized =
-        GetTemporary(context, node, /*index=*/kInputQuantized);
-    input_quantized->type = weights->type;
-    input_quantized->allocation_type = kTfLiteArenaRw;
-    if (!TfLiteIntArrayEqual(input_quantized->dims, input->dims)) {
-      TfLiteIntArray* input_quantized_size = TfLiteIntArrayCopy(input->dims);
-      TF_LITE_ENSURE_OK(context, context->ResizeTensor(context, input_quantized,
-                                                       input_quantized_size));
-    }
-
-    // Tell interpreter to allocate temporary tensors to store scaling factors.
-    node->temporaries->data[kScalingFactors] =
-        op_data->scratch_tensor_index + kScalingFactors;
-    TfLiteTensor* scaling_factors =
-        GetTemporary(context, node, /*index=*/kScalingFactors);
-    scaling_factors->type = kTfLiteFloat32;
-    scaling_factors->allocation_type = kTfLiteArenaRw;
-    int scaling_dims[1] = {n_batch};
-    if (!TfLiteIntArrayEqualsArray(scaling_factors->dims, 1, scaling_dims)) {
-      TfLiteIntArray* scaling_factors_size = TfLiteIntArrayCreate(1);
-      scaling_factors_size->data[0] = n_batch;
-      TF_LITE_ENSURE_OK(context, context->ResizeTensor(context, scaling_factors,
-                                                       scaling_factors_size));
-    }
-  }
-  return kTfLiteOk;
-}
-
-TfLiteStatus EvalFloat(const TfLiteTensor* input, const TfLiteTensor* lookup,
-                       const TfLiteTensor* weights, const TfLiteTensor* bias,
-                       TfLiteTensor* output) {
-  const int n_batch = SizeOfDimension(input, 0);
-  const int n_input = SizeOfDimension(input, 1);
-
-  const float* input_ptr_batch = input->data.f;
-
-  // Initialize pointer to right row according to lookup value.
-  int32 lookup_index = lookup->data.i32[0];
-  const float* weights_ptr = weights->data.f + lookup_index * n_input;
-
-  // Initialize output to bias.
-  if (bias) {
-    float* bias_ptr = bias->data.f + lookup_index;
-    tensor_utils::VectorBatchVectorAssign(bias_ptr, 1, n_batch, output->data.f);
-  } else {
-    tensor_utils::ZeroVector(output->data.f, n_batch * 1);
-  }
-
-  tensor_utils::MatrixBatchVectorMultiplyAccumulate(
-      weights_ptr, /*m_rows=*/1, n_input, input_ptr_batch, n_batch,
-      output->data.f, /*result_stride=*/1);
-
-  return kTfLiteOk;
-}
-
-TfLiteStatus EvalHybrid(const TfLiteTensor* input, const TfLiteTensor* lookup,
-                        const TfLiteTensor* weights, const TfLiteTensor* bias,
-                        TfLiteTensor* scaling_factors,
-                        TfLiteTensor* input_quantized, TfLiteTensor* output) {
-  const int n_batch = SizeOfDimension(input, 0);
-  const int n_input = SizeOfDimension(input, 1);
-
-  const float* input_ptr_batch = input->data.f;
-
-  // Initialize the pointer to storage for scaling factors.
-  float* scaling_factors_ptr = scaling_factors->data.f;
-  int32 lookup_index = lookup->data.i32[0];
-
-  // Initialize the pointer to storage for quantized values and a pointer to
-  // the row according to lookup value.
-  int8_t *quantized_input_ptr_batch, *weights_ptr;
-  if (weights->type == kTfLiteUInt8) {
-    quantized_input_ptr_batch =
-        reinterpret_cast<int8_t*>(input_quantized->data.uint8);
-    weights_ptr =
-        reinterpret_cast<int8_t*>(weights->data.uint8) + lookup_index * n_input;
-  } else {
-    quantized_input_ptr_batch = input_quantized->data.int8;
-    weights_ptr = weights->data.int8 + lookup_index * n_input;
-  }
-
-  // Initialize output to bias.
-  if (bias) {
-    float* bias_ptr = bias->data.f + lookup_index;
-    tensor_utils::VectorBatchVectorAssign(bias_ptr, 1, n_batch, output->data.f);
-  } else {
-    tensor_utils::ZeroVector(output->data.f, n_batch * 1);
-  }
-
-  if (!tensor_utils::IsZeroVector(input_ptr_batch, n_batch * n_input)) {
-    // Quantize input from float to int8.
-    float unused_min, unused_max;
-    for (int b = 0; b < n_batch; ++b) {
-      const int offset = b * n_input;
-      tensor_utils::SymmetricQuantizeFloats(
-          input_ptr_batch + offset, n_input, quantized_input_ptr_batch + offset,
-          &unused_min, &unused_max, &scaling_factors_ptr[b]);
-      scaling_factors_ptr[b] *= weights->params.scale;
-    }
-
-    tensor_utils::MatrixBatchVectorMultiplyAccumulate(
-        weights_ptr, /*m_rows=*/1, n_input, quantized_input_ptr_batch,
-        scaling_factors_ptr, n_batch, output->data.f, /*result_stride=*/1);
-  }
-  return kTfLiteOk;
-}
-
-TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
-  const TfLiteTensor* input = GetInput(context, node, kInputTensor);
-  const TfLiteTensor* lookup = GetInput(context, node, kInputLookupTensor);
-  const TfLiteTensor* weights = GetInput(context, node, kWeightsTensor);
-  const TfLiteTensor* bias = GetInput(context, node, kBiasTensor);
-  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
-
-  switch (weights->type) {
-    case kTfLiteFloat32: {
-      return EvalFloat(input, lookup, weights, bias, output);
-    }
-    case kTfLiteUInt8:
-    case kTfLiteInt8: {
-      TfLiteTensor* input_quantized =
-          GetTemporary(context, node, /*index=*/kInputQuantized);
-      TfLiteTensor* scaling_factors =
-          GetTemporary(context, node, /*index=*/kScalingFactors);
-      return EvalHybrid(input, lookup, weights, bias, scaling_factors,
-                        input_quantized, output);
-    }
-    default:
-      context->ReportError(context, "Type %d is not currently supported.",
-                           weights->type);
-      return kTfLiteError;
-  }
-  return kTfLiteOk;
-}
-
-}  // namespace sparse_output_fully_connected
-
-TfLiteRegistration* Register_SPARSE_OUTPUT_FULLY_CONNECTED() {
-  static TfLiteRegistration r = {sparse_output_fully_connected::Init,
-                                 sparse_output_fully_connected::Free,
-                                 sparse_output_fully_connected::Prepare,
-                                 sparse_output_fully_connected::Eval};
-  return &r;
-}
-
-}  // namespace custom
-}  // namespace ops
-}  // namespace tflite
diff --git a/tensorflow/lite/kernels/sparse_output_fully_connected_test.cc b/tensorflow/lite/kernels/sparse_output_fully_connected_test.cc
deleted file mode 100644
index 7d5fec192c..0000000000
--- a/tensorflow/lite/kernels/sparse_output_fully_connected_test.cc
+++ /dev/null
@@ -1,192 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-// Unit test for TFLite sparse output fully connected op.
-#include <iomanip>
-#include <random>
-#include <vector>
-
-#include <gtest/gtest.h>
-#include "flatbuffers/flexbuffers.h"  // TF:flatbuffers
-#include "tensorflow/lite/kernels/internal/types.h"
-#include "tensorflow/lite/kernels/register.h"
-#include "tensorflow/lite/kernels/test_util.h"
-
-namespace tflite {
-
-namespace ops {
-namespace custom {
-
-TfLiteRegistration* Register_SPARSE_OUTPUT_FULLY_CONNECTED();
-
-namespace {
-
-using ::testing::ElementsAreArray;
-
-class BaseSparseOutputFullyConnectedOpModel : public SingleOpModel {
- public:
-  BaseSparseOutputFullyConnectedOpModel(const TensorData& input,
-                                        const TensorData& weights,
-                                        const TensorData& output = {
-                                            TensorType_FLOAT32}) {
-    input_ = AddInput(input);
-    lookup_ = AddInput({TensorType_INT32, {1}});
-    weights_ = AddInput(weights);
-    int bias_size = GetShape(weights_)[0];
-    bias_ = AddInput({TensorType_FLOAT32, {bias_size}});
-    output_ = AddOutput(output);
-
-    // Create empty (required) options map.
-    flexbuffers::Builder fbb;
-    fbb.Map([&]() {});
-    fbb.Finish();
-
-    SetCustomOp("SPARSE_OUTPUT_FULLY_CONNECTED", fbb.GetBuffer(),
-                Register_SPARSE_OUTPUT_FULLY_CONNECTED);
-    BuildInterpreter({GetShape(input_), GetShape(lookup_), GetShape(weights_),
-                      GetShape(bias_)});
-  }
-
-  void SetInput(const std::vector<float>& data) {
-    PopulateTensor(input_, data);
-  }
-
-  void SetLookup(const std::vector<int32_t>& f) { PopulateTensor(lookup_, f); }
-
-  void SetBias(const std::vector<float>& f) { PopulateTensor(bias_, f); }
-
-  std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
-
- protected:
-  int input_;
-  int lookup_;
-  int weights_;
-  int bias_;
-  int output_;
-};
-
-class FloatSparseOutputFullyConnectedOpModel
-    : public BaseSparseOutputFullyConnectedOpModel {
- public:
-  using BaseSparseOutputFullyConnectedOpModel::
-      BaseSparseOutputFullyConnectedOpModel;
-
-  void SetWeights(const std::vector<float>& f) { PopulateTensor(weights_, f); }
-};
-
-class HybridSparseOutputFullyConnectedOpModel
-    : public BaseSparseOutputFullyConnectedOpModel {
- public:
-  using BaseSparseOutputFullyConnectedOpModel::
-      BaseSparseOutputFullyConnectedOpModel;
-
-  void SetWeights(const std::vector<float>& f) {
-    SymmetricQuantizeAndPopulate(weights_, f);
-  }
-
-  void SetSignedWeights(const std::vector<float>& f) {
-    SignedSymmetricQuantizeAndPopulate(weights_, f);
-  }
-};
-
-TEST(SparseOutputFullyConnectedOpTest, SimpleTestFloat) {
-  FloatSparseOutputFullyConnectedOpModel m({TensorType_FLOAT32, {1, 5}},
-                                           {TensorType_FLOAT32, {3, 5}},
-                                           {TensorType_FLOAT32, {}});
-
-  m.SetInput({-1.0, 0.0, 1.0, 2.0, 3.0});
-
-  m.SetLookup({2});
-
-  m.SetWeights({
-      -1.0, 0.0, 1.0, 2.0, 3.0,  //
-      0.0, 1.0, 2.0, 3.0, 4.0,   //
-      1.0, 2.0, 3.0, 4.0, 5.0,   //
-  });
-
-  m.SetBias({1.0, 2.0, 3.0});
-
-  m.Invoke();
-
-  EXPECT_THAT(m.GetOutput(), ElementsAreArray({28}));
-}
-
-TEST(SparseOutputFullyConnectedOpTest, SimpleTestHybridUint8) {
-  HybridSparseOutputFullyConnectedOpModel m({TensorType_FLOAT32, {1, 5}},
-                                            {TensorType_UINT8, {3, 5}},
-                                            {TensorType_FLOAT32, {}});
-
-  m.SetInput({-1.0, 0.0, 1.0, 2.0, 3.0});
-
-  m.SetLookup({2});
-
-  m.SetWeights({
-      -1.0, 0.0, 1.0, 2.0, 3.0,  //
-      0.0, 1.0, 2.0, 3.0, 4.0,   //
-      1.0, 2.0, 3.0, 4.0, 5.0,   //
-  });
-
-  m.SetBias({1.0, 2.0, 3.0});
-
-  m.Invoke();
-
-  // We get 28.0552 instead of 28.
-  //
-  // Input -> -42, 0, 42, 85, 127 with scale factor of 127/3.
-  // Looked up weights ->  25, 51, 76, 102, 127 with scale factor of 127/5.
-  //
-  // (-42 * 25 + 0 * 51 + 42 * 76 + 85 * 102 + 127 * 127) * (3*5/127^2) + 3.0
-  // gives us the expected result.
-  EXPECT_THAT(m.GetOutput(), ElementsAreArray(ArrayFloatNear({28}, 0.0553)));
-}
-
-TEST(SparseOutputFullyConnectedOpTest, SimpleTestHybridInt8) {
-  HybridSparseOutputFullyConnectedOpModel m({TensorType_FLOAT32, {1, 5}},
-                                            {TensorType_INT8, {3, 5}},
-                                            {TensorType_FLOAT32, {}});
-
-  m.SetInput({-1.0, 0.0, 1.0, 2.0, 3.0});
-
-  m.SetLookup({2});
-
-  m.SetSignedWeights({
-      -1.0, 0.0, 1.0, 2.0, 3.0,  //
-      0.0, 1.0, 2.0, 3.0, 4.0,   //
-      1.0, 2.0, 3.0, 4.0, 5.0,   //
-  });
-
-  m.SetBias({1.0, 2.0, 3.0});
-
-  m.Invoke();
-
-  // We get 28.0552 instead of 28.
-  //
-  // Input -> -42, 0, 42, 85, 127 with scale factor of 127/3.
-  // Looked up weights ->  25, 51, 76, 102, 127 with scale factor of 127/5.
-  //
-  // (-42 * 25 + 0 * 51 + 42 * 76 + 85 * 102 + 127 * 127) * (3*5/127^2) + 3.0
-  // gives us the expected result.
-  EXPECT_THAT(m.GetOutput(), ElementsAreArray(ArrayFloatNear({28}, 0.0553)));
-}
-
-}  // namespace
-}  // namespace custom
-}  // namespace ops
-}  // namespace tflite
-
-int main(int argc, char** argv) {
-  ::tflite::LogToStderr();
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}
-- 
GitLab


From a2ffb71698e331bff46baced54765ac54ced402a Mon Sep 17 00:00:00 2001
From: Pulkit Bhuwalka <pulkitb@google.com>
Date: Wed, 23 Jan 2019 10:14:54 -0800
Subject: [PATCH 1122/2345] Fix build breakage introduced by removing contrib
 visibility changes.

PiperOrigin-RevId: 230552714
---
 tensorflow/contrib/framework/BUILD | 1 +
 tensorflow/contrib/layers/BUILD    | 1 +
 2 files changed, 2 insertions(+)

diff --git a/tensorflow/contrib/framework/BUILD b/tensorflow/contrib/framework/BUILD
index 88a14a2a94..3f6dbe0cbd 100644
--- a/tensorflow/contrib/framework/BUILD
+++ b/tensorflow/contrib/framework/BUILD
@@ -51,6 +51,7 @@ tf_custom_op_py_library(
         "//learning/brain:__subpackages__",
         "//tensorflow:__subpackages__",
         "//tensorflow_estimator:__subpackages__",
+        "//tensorflow_model_optimization:__subpackages__",
         "//video/youtube/personalization:__subpackages__",
     ],
     deps = [
diff --git a/tensorflow/contrib/layers/BUILD b/tensorflow/contrib/layers/BUILD
index 9ca6f8df5d..69d5496f8a 100644
--- a/tensorflow/contrib/layers/BUILD
+++ b/tensorflow/contrib/layers/BUILD
@@ -81,6 +81,7 @@ tf_custom_op_py_library(
     visibility = [
         "//learning/brain:__subpackages__",
         "//tensorflow:__subpackages__",
+        "//tensorflow_model_optimization:__subpackages__",
         "//video/youtube/personalization:__subpackages__",
     ],
     deps = [
-- 
GitLab


From edd036fb973795d9605f3bd6b927121b377e1c8a Mon Sep 17 00:00:00 2001
From: Doe Hyun Yoon <dyoon@google.com>
Date: Wed, 23 Jan 2019 10:36:54 -0800
Subject: [PATCH 1123/2345] In op level cost estimator, use PredictMetadata()
 for ShapeN operation.

PiperOrigin-RevId: 230557345
---
 tensorflow/core/grappler/costs/op_level_cost_estimator.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
index 96bac8d0cb..1da8a98498 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
@@ -54,6 +54,7 @@ constexpr char kVariable[] = "Variable";
 constexpr char kVariableV2[] = "VariableV2";
 constexpr char kRank[] = "Rank";
 constexpr char kShape[] = "Shape";
+constexpr char kShapeN[] = "ShapeN";
 constexpr char kSize[] = "Size";
 constexpr char kStopGradient[] = "StopGradient";
 constexpr char kPreventGradient[] = "PreventGradient";
@@ -264,6 +265,7 @@ OpLevelCostEstimator::OpLevelCostEstimator() {
 
       {kRank, wrap(&OpLevelCostEstimator::PredictMetadata)},
       {kShape, wrap(&OpLevelCostEstimator::PredictMetadata)},
+      {kShapeN, wrap(&OpLevelCostEstimator::PredictMetadata)},
       {kSize, wrap(&OpLevelCostEstimator::PredictMetadata)},
       {kMaxPool, wrap(&OpLevelCostEstimator::PredictMaxPool)},
       {kMaxPoolGrad, wrap(&OpLevelCostEstimator::PredictMaxPoolGrad)},
-- 
GitLab


From d94beab8f904946cc5694c4e328de9247eaf0ee8 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Wed, 23 Jan 2019 10:40:15 -0800
Subject: [PATCH 1124/2345] Minor fixes to process_function_library_runtime

PiperOrigin-RevId: 230558130
---
 .../common_runtime/process_function_library_runtime.cc     | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/common_runtime/process_function_library_runtime.cc b/tensorflow/core/common_runtime/process_function_library_runtime.cc
index b236343a0f..82f07b44ec 100644
--- a/tensorflow/core/common_runtime/process_function_library_runtime.cc
+++ b/tensorflow/core/common_runtime/process_function_library_runtime.cc
@@ -350,10 +350,13 @@ Status ProcessFunctionLibraryRuntime::PinArgsAndRets(
               }
               std::vector<Device*> matching_devices;
               device_set.FindMatchingDevices(parsed, &matching_devices);
-              if (matching_devices.size() != 1) {
+              if (matching_devices.empty()) {
+                return errors::InvalidArgument(
+                    "Unable to find any devices for spec ", *src_device);
+              } else if (matching_devices.size() != 1) {
                 // Convert a vector of devices to a string.
                 // Using absl::StrJoin did not work in Android builds.
-                string devices = "]";
+                string devices = "[";
                 for (Device* device : matching_devices) {
                   devices.append(device->name());
                   devices.append(", ");
-- 
GitLab


From 4f4c9547bce7113e3c3814f71af52da152c15d10 Mon Sep 17 00:00:00 2001
From: Matej Rizman <matejr@google.com>
Date: Wed, 23 Jan 2019 10:41:19 -0800
Subject: [PATCH 1125/2345] Add SYCL/CUDA kernels for tf.math.nextafter.

PiperOrigin-RevId: 230558354
---
 tensorflow/core/kernels/nextafter_op.cc       | 32 +++++++--------
 tensorflow/core/kernels/nextafter_op.h        | 40 +++++++++++++++++++
 .../core/kernels/nextafter_op_gpu.cu.cc       | 29 ++++++++++++++
 3 files changed, 84 insertions(+), 17 deletions(-)
 create mode 100644 tensorflow/core/kernels/nextafter_op.h
 create mode 100644 tensorflow/core/kernels/nextafter_op_gpu.cu.cc

diff --git a/tensorflow/core/kernels/nextafter_op.cc b/tensorflow/core/kernels/nextafter_op.cc
index bcc9d4ee1f..6166a1053f 100644
--- a/tensorflow/core/kernels/nextafter_op.cc
+++ b/tensorflow/core/kernels/nextafter_op.cc
@@ -1,4 +1,3 @@
-
 /* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,26 +13,25 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include "tensorflow/core/kernels/nextafter_op.h"
 #include "tensorflow/core/kernels/cwise_ops_common.h"
 
 namespace tensorflow {
 
-namespace functor {
-
-template <typename T>
-struct nextafter_op {
-  EIGEN_EMPTY_STRUCT_CTOR(nextafter_op)
-  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T operator()(const T& x1,
-                                                           const T& x2) const {
-    return std::nextafter(x1, x2);
-  }
-};
-
-template <typename T>
-struct nextafter : base<T, nextafter_op<T>> {};
-
-}  // namespace functor
-
 REGISTER2(BinaryOp, CPU, "NextAfter", functor::nextafter, float, double);
 
+#if TENSORFLOW_USE_SYCL
+#define REGISTER_SYCL_KERNEL(TYPE)                                     \
+  REGISTER_KERNEL_BUILDER(                                             \
+      Name("NextAfter").Device(DEVICE_SYCL).TypeConstraint<TYPE>("T"), \
+      BinaryOp<SYCLDevice, functor::nextafter<TYPE>>);
+REGISTER_SYCL_KERNEL(float);
+REGISTER_SYCL_KERNEL(double);
+#undef REGISTER_SYCL_KERNEL
+#endif  // TENSORFLOW_USE_SYCL
+
+#if GOOGLE_CUDA
+REGISTER2(BinaryOp, GPU, "NextAfter", functor::nextafter, float, double);
+#endif  // GOOGLE_CUDA
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/nextafter_op.h b/tensorflow/core/kernels/nextafter_op.h
new file mode 100644
index 0000000000..64374980f2
--- /dev/null
+++ b/tensorflow/core/kernels/nextafter_op.h
@@ -0,0 +1,40 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_KERNELS_NEXTAFTER_OP_H_
+#define TENSORFLOW_CORE_KERNELS_NEXTAFTER_OP_H_
+
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/kernels/cwise_ops.h"
+
+namespace tensorflow {
+namespace functor {
+
+template <typename T>
+struct nextafter_op {
+  EIGEN_EMPTY_STRUCT_CTOR(nextafter_op)
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T operator()(const T& x1,
+                                                           const T& x2) const {
+    return std::nextafter(x1, x2);
+  }
+};
+
+template <typename T>
+struct nextafter : base<T, nextafter_op<T>> {};
+
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_KERNELS_NEXTAFTER_OP_H_
diff --git a/tensorflow/core/kernels/nextafter_op_gpu.cu.cc b/tensorflow/core/kernels/nextafter_op_gpu.cu.cc
new file mode 100644
index 0000000000..d2321c6a88
--- /dev/null
+++ b/tensorflow/core/kernels/nextafter_op_gpu.cu.cc
@@ -0,0 +1,29 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#include "tensorflow/core/kernels/cwise_ops_gpu_common.cu.h"
+#include "tensorflow/core/kernels/nextafter_op.h"
+
+namespace tensorflow {
+namespace functor {
+
+DEFINE_BINARY2(nextafter, float, double);
+
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
-- 
GitLab


From 4fac1ce6b5fa89a68e4566fd446929cb7f453a17 Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Wed, 23 Jan 2019 10:43:23 -0800
Subject: [PATCH 1126/2345] n/a

PiperOrigin-RevId: 230558777
---
 tensorflow/tools/docs/BUILD | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/docs/BUILD b/tensorflow/tools/docs/BUILD
index 4bb452f256..cc106b5955 100644
--- a/tensorflow/tools/docs/BUILD
+++ b/tensorflow/tools/docs/BUILD
@@ -81,7 +81,7 @@ py_library(
     srcs_version = "PY2AND3",
 )
 
-py_binary(
+py_library(
     name = "generate_lib",
     srcs = ["generate_lib.py"],
     srcs_version = "PY2AND3",
@@ -155,7 +155,7 @@ py_test(
         "optonly",
     ],
     deps = [
-        ":generate2",
+        ":generate2_lib",
     ],
 )
 
@@ -163,6 +163,13 @@ py_binary(
     name = "generate2",
     srcs = ["generate2.py"],
     srcs_version = "PY2AND3",
+    deps = [":generate2_lib"],
+)
+
+py_library(
+    name = "generate2_lib",
+    srcs = ["generate2.py"],
+    srcs_version = "PY2AND3",
     deps = [
         "//tensorflow:tensorflow_py",
         "//tensorflow/python:util",
-- 
GitLab


From 014bb07ebd28bf8ac696642697aa7dd42e89d504 Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Wed, 23 Jan 2019 11:13:12 -0800
Subject: [PATCH 1127/2345] Fix/workaround for issue with whitelisting TF ops.

PiperOrigin-RevId: 230565860
---
 tensorflow/python/keras/engine/base_layer_utils.py    | 11 ++++++++++-
 .../python/keras/layers/advanced_activations_test.py  |  9 +++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/engine/base_layer_utils.py b/tensorflow/python/keras/engine/base_layer_utils.py
index 0d0257a1dd..5d39038ddd 100644
--- a/tensorflow/python/keras/engine/base_layer_utils.py
+++ b/tensorflow/python/keras/engine/base_layer_utils.py
@@ -248,7 +248,16 @@ def create_keras_history(tensors):
       operations and need to have Keras metadata assigned to them.
   """
 
-  _create_keras_history_helper(tensors, set())
+  try:
+    _create_keras_history_helper(tensors, set())
+  except AttributeError:
+    # This can happen with sublayers inside of layers in the Functional API.
+    # The error occurs when a Functional Model is running in V2 function
+    # mode and a non-Keras Tensor is passed as an input to a sublayer inside
+    # of another layer.
+    # TODO(omalleyt): Only run `create_keras_history` during Functional API
+    # creation phase.
+    pass
 
 
 def _create_keras_history_helper(tensors, processed_ops=None):
diff --git a/tensorflow/python/keras/layers/advanced_activations_test.py b/tensorflow/python/keras/layers/advanced_activations_test.py
index f32bb457c8..f04185417e 100644
--- a/tensorflow/python/keras/layers/advanced_activations_test.py
+++ b/tensorflow/python/keras/layers/advanced_activations_test.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import numpy as np
+
 from tensorflow.python import keras
 from tensorflow.python.eager import context
 from tensorflow.python.keras import keras_parameterized
@@ -88,6 +90,13 @@ class AdvancedActivationsTest(keras_parameterized.TestCase):
             kwargs={'negative_slope': -2},
             input_shape=(2, 3, 4))
 
+  @keras_parameterized.run_with_all_model_types
+  def test_layer_as_activation(self):
+    layer = keras.layers.Dense(1, activation=keras.layers.ReLU())
+    model = testing_utils.get_model_from_layers([layer], input_shape=(10,))
+    model.compile('sgd', 'mse', run_eagerly=testing_utils.should_run_eagerly())
+    model.fit(np.ones((10, 10)), np.ones((10, 1)), batch_size=2)
+
 
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 584c2e123cd41fb164bb7388229fc26a06824d4e Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Wed, 23 Jan 2019 11:28:23 -0800
Subject: [PATCH 1128/2345] Better error message when an eager op sees a graph
 tensor.

PiperOrigin-RevId: 230569140
---
 tensorflow/python/eager/def_function_test.py  | 27 +++++++++++++++++++
 tensorflow/python/framework/ops_test.py       |  2 +-
 .../python/keras/engine/base_layer_test.py    |  8 ++----
 tensorflow/python/pywrap_tfe.i                | 19 +++++++++++++
 4 files changed, 49 insertions(+), 7 deletions(-)

diff --git a/tensorflow/python/eager/def_function_test.py b/tensorflow/python/eager/def_function_test.py
index e0a82bb0be..b777ead913 100644
--- a/tensorflow/python/eager/def_function_test.py
+++ b/tensorflow/python/eager/def_function_test.py
@@ -61,6 +61,21 @@ class _HasDecoratedMethod(object):
   def f(self, x):
     return x * 3.
 
+# pylint: disable=bad-continuation,anomalous-backslash-in-string
+MIXING_GRAPH_EAGER_TENSORS_ERROR = (
+"""An op outside of the function building code is being passed
+a "Graph" tensor. It is possible to have Graph tensors
+leak out of the function building context by including a
+tf.init_scope in your function building code.
+For example, the following function will fail:
+  @tf.function
+  def has_init_scope\(\):
+    my_constant = tf.constant\(1.\)
+    with tf.init_scope\(\):
+      added = my_constant \* 2
+The graph tensor has name: Const:0""")
+# pylint: enable=bad-continuation,anomalous-backslash-in-string
+
 
 class DefFunctionTest(test.TestCase):
 
@@ -296,6 +311,18 @@ class DefFunctionTest(test.TestCase):
     # function itself is not involved in a reference cycle.
     self.assertIs(None, weak_fn())
 
+  def testErrorMessageWhenGraphTensorIsPassedToEager(self):
+
+    @def_function.function
+    def failing_function():
+      a = constant_op.constant(1.)
+
+      with ops.init_scope():
+        _ = a + a
+
+    with self.assertRaisesRegexp(TypeError, MIXING_GRAPH_EAGER_TENSORS_ERROR):
+      failing_function()
+
 
 if __name__ == '__main__':
   ops.enable_eager_execution()
diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py
index 58d311fe4e..8347e9d1eb 100644
--- a/tensorflow/python/framework/ops_test.py
+++ b/tensorflow/python/framework/ops_test.py
@@ -2348,7 +2348,7 @@ class InitScopeTest(test_util.TensorFlowTestCase):
           math_ops.add(c, c)
         c2 = constant_op.constant(2.0)
       with self.assertRaisesRegexp(
-          TypeError, "contains objects other than 'EagerTensor'"):
+          TypeError, "Graph tensors"):
         math_ops.add(c2, c2)
 
   def testPreservesNameScopeInEagerExecution(self):
diff --git a/tensorflow/python/keras/engine/base_layer_test.py b/tensorflow/python/keras/engine/base_layer_test.py
index c925a8ca8a..8e9de8b7cc 100644
--- a/tensorflow/python/keras/engine/base_layer_test.py
+++ b/tensorflow/python/keras/engine/base_layer_test.py
@@ -252,18 +252,14 @@ class BaseLayerTest(keras_parameterized.TestCase):
       x1 = array_ops.ones((3, 3))
     x2 = array_ops.ones((3, 3))
     self.assertIsInstance(x2, ops.EagerTensor)
-    with self.assertRaisesRegexp(TypeError,
-                                 'provided list of inputs contains '
-                                 'objects other than \'EagerTensor\''):
+    with self.assertRaisesRegexp(TypeError, 'Graph tensors'):
       math_ops.matmul(x1, x2)
 
   def test_mixing_numpy_arrays_and_graph_tensors(self):
     with ops.Graph().as_default():
       x1 = array_ops.ones((3, 3))
     x2 = np.ones((3, 3), dtype='float32')
-    with self.assertRaisesRegexp(TypeError,
-                                 'provided list of inputs contains '
-                                 'objects other than \'EagerTensor\''):
+    with self.assertRaisesRegexp(TypeError, 'Graph tensors'):
       math_ops.matmul(x1, x2)
 
   @test_util.run_in_graph_and_eager_modes
diff --git a/tensorflow/python/pywrap_tfe.i b/tensorflow/python/pywrap_tfe.i
index 5f6fbdcb5e..19254359cc 100755
--- a/tensorflow/python/pywrap_tfe.i
+++ b/tensorflow/python/pywrap_tfe.i
@@ -184,6 +184,25 @@ limitations under the License.
       }
       if (EagerTensor_CheckExact(elem)) {
         (*$1)[i] = EagerTensor_Handle(elem);
+      } else if (tensorflow::swig::IsTensor(elem)) {
+        // If it isnt an EagerTensor, but is still a Tensor, it must be a graph
+        // tensor.
+        SWIG_exception_fail(
+            SWIG_TypeError,
+            tensorflow::strings::StrCat(
+                "An op outside of the function building code is being passed\n"
+                "a \"Graph\" tensor. It is possible to have Graph tensors\n"
+                "leak out of the function building context by including a\n"
+                "tf.init_scope in your function building code.\n"
+                "For example, the following function will fail:\n",
+                "  @tf.function\n",
+                "  def has_init_scope():\n",
+                "    my_constant = tf.constant(1.)\n",
+                "    with tf.init_scope():\n",
+                "      added = my_constant * 2\n",
+                "The graph tensor has name: ",
+                TFE_GetPythonString(PyObject_GetAttrString(elem, "name")))
+                .c_str());
       } else {
         SWIG_exception_fail(
             SWIG_TypeError,
-- 
GitLab


From 9c62b6be387deb214f2ae4dd55a8e7e630fe2d28 Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Wed, 23 Jan 2019 12:15:51 -0800
Subject: [PATCH 1129/2345] Add renames for tf.contrib.saved_model -->
 tf.keras.experimental

PiperOrigin-RevId: 230578299
---
 tensorflow/tools/compatibility/tf_upgrade_v2.py      |  4 ++++
 tensorflow/tools/compatibility/tf_upgrade_v2_test.py | 10 ++++++++++
 2 files changed, 14 insertions(+)

diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 7479c693b1..9a761c3497 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -508,6 +508,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.data.experimental.unbatch",
         "tf.contrib.data.unique":
             "tf.data.experimental.unique",
+        "tf.contrib.saved_model.load_keras_model":
+            "tf.keras.experimental.load_from_saved_model",
+        "tf.contrib.saved_model.save_keras_model":
+            "tf.keras.experimental.export",
         "tf.contrib.rnn.RNNCell":
             "tf.nn.rnn_cell.RNNCell",
         "tf.contrib.rnn.LSTMStateTuple":
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 79f0a465b5..f3ecf1c269 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -712,6 +712,16 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
     _, unused_report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, expected_text)
 
+  def testKerasSavedModel(self):
+    text = (
+        "tf.contrib.saved_model.save_keras_model(model, './saved_models')\n"
+        "tf.contrib.saved_model.load_keras_model(saved_model_path)\n")
+    expected_text = (
+        "tf.keras.experimental.export(model, './saved_models')\n"
+        "tf.keras.experimental.load_from_saved_model(saved_model_path)\n")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
   def testStatelessMultinomial(self):
     text = (
         "tf.random.stateless_multinomial(logits, num_samples, seed, "
-- 
GitLab


From 9d92b41bca1b3bfdb8d67dcf85c298462de13004 Mon Sep 17 00:00:00 2001
From: Petros Mol <pmol@google.com>
Date: Wed, 23 Jan 2019 12:17:51 -0800
Subject: [PATCH 1130/2345] Adds a keras layer that implements explicit feature
 maps that approximates shift-invariant kernels (Gaussian, Laplace and more)
 using random Fourier features.

PiperOrigin-RevId: 230578634
---
 tensorflow/python/keras/BUILD                 |  50 ++-
 tensorflow/python/keras/layers/__init__.py    |   3 +
 tensorflow/python/keras/layers/kernelized.py  | 258 ++++++++++++
 .../python/keras/layers/kernelized_test.py    | 391 ++++++++++++++++++
 .../python/keras/utils/kernelized_utils.py    | 117 ++++++
 .../keras/utils/kernelized_utils_test.py      | 116 ++++++
 6 files changed, 931 insertions(+), 4 deletions(-)
 create mode 100644 tensorflow/python/keras/layers/kernelized.py
 create mode 100644 tensorflow/python/keras/layers/kernelized_test.py
 create mode 100644 tensorflow/python/keras/utils/kernelized_utils.py
 create mode 100644 tensorflow/python/keras/utils/kernelized_utils_test.py

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 7705491c85..cfbe56d49f 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -298,6 +298,7 @@ py_library(
         "layers/core.py",
         "layers/cudnn_recurrent.py",
         "layers/embeddings.py",
+        "layers/kernelized.py",
         "layers/local.py",
         "layers/merge.py",
         "layers/noise.py",
@@ -307,6 +308,7 @@ py_library(
         "layers/serialization.py",
         "layers/wrappers.py",
         "utils/generic_utils.py",
+        "utils/kernelized_utils.py",
         "utils/layer_utils.py",
         "utils/tf_utils.py",
     ],
@@ -318,17 +320,19 @@ py_library(
         "//tensorflow/python:dtypes",
         "//tensorflow/python:embedding_ops",
         "//tensorflow/python:framework_ops",
-        "//tensorflow/python:logging_ops",
+        "//tensorflow/python:init_ops",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:nn",
-        "//tensorflow/python:random_ops",
-        "//tensorflow/python:sparse_ops",
+        "//tensorflow/python:nn_ops",
+        "//tensorflow/python:platform",
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python:standard_ops",
-        "//tensorflow/python:tensor_array_ops",
         "//tensorflow/python:tensor_shape",
+        "//tensorflow/python:tensor_util",
         "//tensorflow/python:util",
+        "//tensorflow/python:variables",
         "//tensorflow/python/distribute:distribute_lib",
+        "//third_party/py/numpy",
     ],
 )
 
@@ -738,6 +742,32 @@ tf_py_test(
     ],
 )
 
+tf_py_test(
+    name = "kernelized_test",
+    size = "small",
+    srcs = ["layers/kernelized_test.py"],
+    additional_deps = [
+        ":backend",
+        ":initializers",
+        ":keras",
+        ":layers",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:init_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:random_ops",
+        "//tensorflow/python:random_seed",
+        "//tensorflow/python:tensor_shape",
+        "//tensorflow/python/eager:context",
+    ],
+)
+
 tf_py_test(
     name = "wrappers_test",
     size = "medium",
@@ -837,6 +867,18 @@ tf_py_test(
     ],
 )
 
+tf_py_test(
+    name = "kernelized_utils_test",
+    size = "small",
+    srcs = ["utils/kernelized_utils_test.py"],
+    additional_deps = [
+        ":layers",
+        "@absl_py//absl/testing:parameterized",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:constant_op",
+    ],
+)
+
 cuda_py_test(
     name = "multi_gpu_utils_test",
     srcs = ["utils/multi_gpu_utils_test.py"],
diff --git a/tensorflow/python/keras/layers/__init__.py b/tensorflow/python/keras/layers/__init__.py
index dec9673b5b..88fbaca3ea 100644
--- a/tensorflow/python/keras/layers/__init__.py
+++ b/tensorflow/python/keras/layers/__init__.py
@@ -112,6 +112,9 @@ from tensorflow.python.keras.layers.noise import GaussianDropout
 from tensorflow.python.keras.layers.normalization import BatchNormalization
 from tensorflow.python.keras.layers.normalization import LayerNormalization
 
+# Kernelized layers.
+from tensorflow.python.keras.layers.kernelized import RandomFourierFeatures
+
 # Pooling layers.
 from tensorflow.python.keras.layers.pooling import MaxPooling1D
 from tensorflow.python.keras.layers.pooling import MaxPooling2D
diff --git a/tensorflow/python/keras/layers/kernelized.py b/tensorflow/python/keras/layers/kernelized.py
new file mode 100644
index 0000000000..9753fc66de
--- /dev/null
+++ b/tensorflow/python/keras/layers/kernelized.py
@@ -0,0 +1,258 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Keras layers that implement explicit (approximate) kernel feature maps."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+import six
+
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.keras import initializers
+from tensorflow.python.keras.engine import base_layer
+from tensorflow.python.keras.engine import input_spec
+from tensorflow.python.ops import gen_math_ops
+from tensorflow.python.ops import init_ops
+from tensorflow.python.ops import nn
+
+_SUPPORTED_RBF_KERNEL_TYPES = ['gaussian', 'laplacian']
+
+
+class RandomFourierFeatures(base_layer.Layer):
+  r"""Layer that maps its inputs using random Fourier features.
+
+  This layer implements a feature map \\(\phi: \mathbb{R}^d \rightarrow
+  \mathbb{R}^D\\) which approximates shift-invariant kernels. A kernel function
+  K(x, y) defined over \\(\mathbb{R}^d x \mathbb{R}^d\\) is shift-invariant if
+  K(x, y) = k(x-y) for some function defined over \\(\mathbb{R}^d\\). Many
+  popular Radial Basis Functions (in short RBF), including gaussian and
+  laplacian kernels are shift-invariant.
+
+  The layer approximates a (shift invariant) kernel K in the following sense:
+    up to a scaling factor, for all inputs \\(x, y \in \mathbb{R}^d\\)
+        \\(\phi(x)^T \cdot \phi(y) \approx K(x, y)\\)
+
+  The implementation of this layer is based on the following paper:
+  "Random Features for Large-Scale Kernel Machines" by Ali Rahimi and Ben Recht.
+  (link: https://people.eecs.berkeley.edu/~brecht/papers/07.rah.rec.nips.pdf)
+
+  The distribution from which the parameters of the random features map (layer)
+  are sampled, determines which shift-invariant kernel the layer approximates
+  (see paper for more details). The users can use the distribution of their
+  choice. Due to their popularity, the layer supports the out-of-the-box
+  approximation of the following RBF kernels:
+  - Gaussian: \\(K(x, y) = e^{-\frac{\|x-y\|_2^2}{2 \cdot scale^2}}\\)
+  - Laplacian: \\(K(x, y) = e^{-\frac{\|x-y\|_1}{scale}}\\)
+
+  NOTE: Unlike the map described in the paper and the scikit-learn
+  implementation, the output of this layer does not apply the sqrt(2/D)
+  normalization factor.
+
+  Usage for ML: Typically, this layer is used to "kernelize" linear models by
+  applying a non-linear transformation (this layer) to the input features and
+  then training a linear model on top of the transformed features. Depending on
+  the loss function of the linear model, the composition of this layer and the
+  linear model results to models that are equivalent (up to approximation) to
+  kernel SVMs (for hinge loss), kernel logistic regression (for logistic loss),
+  kernel linear regression (for squared loss) etc.
+
+  Example of building a kernel multinomial logistic regression model with
+  Gaussian kernel in keras:
+  ```python
+  random_features_layer = RandomFourierFeatures(
+      output_dim=500,
+      kernel_initializer='gaussian',
+      scale=5.0,
+      ...)
+
+  model = tf.keras.models.Sequential()
+  model.add(random_features_layer)
+  model.add(tf.keras.layers.Dense(units=num_classes, activation='softmax')
+
+  model.compile(elif isinstance(identifier, six.string_types):
+    loss=tf.keras.losses.categorical_crossentropy, optimizer=..., metrics=...)
+  ```
+
+  To use another kernel, replace the layer creation command with:
+  ```python
+  random_features_layer = RandomFourierFeatures(
+      output_dim=500,
+      kernel_initializer=<my_initializer>,
+      scale=...,
+      ...)
+  ```
+
+  Arguments:
+    output_dim: Positive integer, the dimension of the layer's output, i.e., the
+      number of random features used to approximate the kernel.
+    kernel_initializer: Determines the distribution of the parameters of the
+      random features map (and therefore the kernel approximated by the layer).
+      It can be either a string or an instance of TensorFlow's Initializer
+      class. Currently only 'gaussian' and 'laplacian' are supported as string
+      initializers (case insensitive). Note that these parameters are not
+      trainable.
+    scale: For gaussian and laplacian kernels, this corresponds to a scaling
+      factor of the corresponding kernel approximated by the layer (see concrete
+      definitions above). When provided, it should be a positive float. If None,
+      the implementation chooses a default value (1.0 typically). Both the
+      approximation error of the kernel and the classification quality are
+      sensitive to this parameter. If trainable is set to True, this paramater
+      is learned end-to-end during training and the provided value serves as an
+      initialization value.
+      NOTE: When this layer is used to map the initial features and then the
+        transformed features are fed to a linear model, by making `scale`
+        trainable, the resulting optimization problem is no longer convex (even
+        if the loss function used by the linear model is convex).
+    trainable: Whether the scaling parameter of th layer is trainable. Defaults
+      to False.
+    name: name for the RandomFourierFeatures layer.
+
+  Raises:
+    ValueError: if output_dim or stddev are not positive or if the provided
+      kernel_initializer is not supported.
+  """
+
+  def __init__(self,
+               output_dim,
+               kernel_initializer='gaussian',
+               scale=None,
+               trainable=False,
+               name=None,
+               **kwargs):
+    if output_dim <= 0:
+      raise ValueError(
+          '`output_dim` should be a positive integer. Given: {}.'.format(
+              output_dim))
+    if isinstance(kernel_initializer, six.string_types):
+      if kernel_initializer.lower() not in _SUPPORTED_RBF_KERNEL_TYPES:
+        raise ValueError(
+            'Unsupported kernel type: \'{}\'. Supported kernel types: {}.'
+            .format(kernel_initializer, _SUPPORTED_RBF_KERNEL_TYPES))
+    if scale is not None and scale <= 0.0:
+      raise ValueError('When provided, `scale` should be a positive float. '
+                       'Given: {}.'.format(scale))
+    super(RandomFourierFeatures, self).__init__(
+        trainable=trainable, name=name, **kwargs)
+    self.output_dim = output_dim
+    self.kernel_initializer = kernel_initializer
+    self.scale = scale
+
+  def build(self, input_shape):
+    input_shape = tensor_shape.TensorShape(input_shape)
+    # TODO(sibyl-vie3Poto): Allow higher dimension inputs. Currently the input is expected
+    # to have shape [batch_size, dimension].
+    if input_shape.rank != 2:
+      raise ValueError(
+          'The rank of the input tensor should be 2. Got {} instead.'.format(
+              input_shape.ndims))
+    if input_shape.dims[1].value is None:
+      raise ValueError(
+          'The last dimension of the inputs to `RandomFourierFeatures` '
+          'should be defined. Found `None`.')
+    self.input_spec = input_spec.InputSpec(
+        ndim=2, axes={1: input_shape.dims[1].value})
+    input_dim = input_shape.dims[1].value
+
+    kernel_initializer = _get_random_features_initializer(
+        self.kernel_initializer, shape=(input_dim, self.output_dim))
+
+    unscaled_kernel = self.add_weight(
+        name='unscaled_random_features',
+        shape=(input_dim, self.output_dim),
+        dtype=dtypes.float32,
+        initializer=kernel_initializer,
+        trainable=False)
+
+    self.bias = self.add_weight(
+        name='random_features_bias',
+        shape=(self.output_dim,),
+        dtype=dtypes.float32,
+        initializer=init_ops.random_uniform_initializer(
+            minval=0.0, maxval=2 * np.pi, dtype=dtypes.float32),
+        trainable=False)
+
+    if self.scale is None:
+      self.scale = _get_default_scale(self.kernel_initializer, input_dim)
+    scale = self.add_weight(
+        name='random_features_scale',
+        shape=(1,),
+        dtype=dtypes.float32,
+        initializer=init_ops.constant_initializer(self.scale),
+        trainable=True,
+        constraint='NonNeg')
+    self.kernel = (1.0 / scale) * unscaled_kernel
+    super(RandomFourierFeatures, self).build(input_shape)
+
+  def call(self, inputs):
+    inputs = ops.convert_to_tensor(inputs, dtype=self.dtype)
+    inputs = gen_math_ops.cast(inputs, dtypes.float32)
+    outputs = gen_math_ops.mat_mul(inputs, self.kernel)
+    outputs = nn.bias_add(outputs, self.bias)
+    return gen_math_ops.cos(outputs)
+
+  def compute_output_shape(self, input_shape):
+    input_shape = tensor_shape.TensorShape(input_shape)
+    input_shape = input_shape.with_rank(2)
+    if input_shape.dims[-1].value is None:
+      raise ValueError(
+          'The innermost dimension of input shape must be defined. Given: %s' %
+          input_shape)
+    return input_shape[:-1].concatenate(self.output_dim)
+
+  def get_config(self):
+    kernel_initializer = self.kernel_initializer
+    if isinstance(self.kernel_initializer, init_ops.Initializer):
+      kernel_initializer = initializers.serialize(self.kernel_initializer)
+    config = {
+        'output_dim': self.output_dim,
+        'kernel_initializer': kernel_initializer,
+        'scale': self.scale,
+    }
+    base_config = super(RandomFourierFeatures, self).get_config()
+    return dict(list(base_config.items()) + list(config.items()))
+
+
+def _get_random_features_initializer(initializer, shape):
+  """Returns Initializer object for random features."""
+
+  def _get_cauchy_samples(loc, scale, shape):
+    probs = np.random.uniform(low=0., high=1., size=shape)
+    return loc + scale * np.tan(np.pi * (probs - 0.5))
+
+  random_features_initializer = initializer
+  if isinstance(initializer, six.string_types):
+    if initializer.lower() == 'gaussian':
+      random_features_initializer = init_ops.random_normal_initializer(
+          stddev=1.0)
+    elif initializer.lower() == 'laplacian':
+      random_features_initializer = init_ops.constant_initializer(
+          _get_cauchy_samples(loc=0.0, scale=1.0, shape=shape))
+
+    else:
+      raise ValueError(
+          'Unsupported kernel type: \'{}\'. Supported kernel types: {}.'.format(
+              random_features_initializer, _SUPPORTED_RBF_KERNEL_TYPES))
+  return random_features_initializer
+
+
+def _get_default_scale(initializer, input_dim):
+  if (isinstance(initializer, six.string_types) and
+      initializer.lower() == 'gaussian'):
+    return np.sqrt(input_dim / 2.0)
+  return 1.0
diff --git a/tensorflow/python/keras/layers/kernelized_test.py b/tensorflow/python/keras/layers/kernelized_test.py
new file mode 100644
index 0000000000..300b1b8155
--- /dev/null
+++ b/tensorflow/python/keras/layers/kernelized_test.py
@@ -0,0 +1,391 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for kernelized.py."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import functools
+import math
+
+from absl.testing import parameterized
+import numpy as np
+
+from tensorflow.python.eager import context
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import random_seed
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
+from tensorflow.python.keras import backend as keras_backend
+from tensorflow.python.keras import initializers
+from tensorflow.python.keras.layers import kernelized as kernel_layers
+from tensorflow.python.keras.utils import kernelized_utils
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import init_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import random_ops
+from tensorflow.python.platform import test
+
+
+def _exact_gaussian(stddev):
+  return functools.partial(
+      kernelized_utils.exact_gaussian_kernel, stddev=stddev)
+
+
+def _exact_laplacian(stddev):
+  return functools.partial(
+      kernelized_utils.exact_laplacian_kernel, stddev=stddev)
+
+
+class RandomFourierFeaturesTest(test.TestCase, parameterized.TestCase):
+
+  def _assert_all_close(self, expected, actual, atol=0.001):
+    if not context.executing_eagerly():
+      with self.cached_session() as sess:
+        keras_backend._initialize_variables(sess)
+        self.assertAllClose(expected, actual, atol=atol)
+    else:
+      self.assertAllClose(expected, actual, atol=atol)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def test_invalid_output_dim(self):
+    with self.assertRaisesRegexp(
+        ValueError, r'`output_dim` should be a positive integer. Given: -3.'):
+      _ = kernel_layers.RandomFourierFeatures(output_dim=-3, scale=2.0)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def test_unsupported_kernel_type(self):
+    with self.assertRaisesRegexp(
+        ValueError, r'Unsupported kernel type: \'unsupported_kernel\'.'):
+      _ = kernel_layers.RandomFourierFeatures(
+          3, 'unsupported_kernel', stddev=2.0)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def test_invalid_scale(self):
+    with self.assertRaisesRegexp(
+        ValueError,
+        r'When provided, `scale` should be a positive float. Given: 0.0.'):
+      _ = kernel_layers.RandomFourierFeatures(output_dim=10, scale=0.0)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def test_invalid_input_shape(self):
+    inputs = random_ops.random_uniform((3, 2, 4), seed=1)
+    rff_layer = kernel_layers.RandomFourierFeatures(output_dim=10, scale=3.0)
+    with self.assertRaisesRegexp(
+        ValueError,
+        r'The rank of the input tensor should be 2. Got 3 instead.'):
+      _ = rff_layer.apply(inputs)
+
+  @parameterized.named_parameters(
+      ('gaussian', 'gaussian', 10.0, False),
+      ('random', init_ops.random_uniform_initializer, 1.0, True))
+  @test_util.run_in_graph_and_eager_modes()
+  def test_random_features_properties(self, initializer, scale, trainable):
+    rff_layer = kernel_layers.RandomFourierFeatures(
+        output_dim=10,
+        kernel_initializer=initializer,
+        scale=scale,
+        trainable=trainable)
+    self.assertEqual(rff_layer.output_dim, 10)
+    self.assertEqual(rff_layer.kernel_initializer, initializer)
+    self.assertEqual(rff_layer.scale, scale)
+    self.assertEqual(rff_layer.trainable, trainable)
+
+  @parameterized.named_parameters(('gaussian', 'gaussian', False),
+                                  ('laplacian', 'laplacian', True),
+                                  ('other', init_ops.ones_initializer, True))
+  @test_util.run_in_graph_and_eager_modes()
+  def test_call(self, initializer, trainable):
+    rff_layer = kernel_layers.RandomFourierFeatures(
+        output_dim=10,
+        kernel_initializer=initializer,
+        scale=1.0,
+        trainable=trainable,
+        name='random_fourier_features')
+    inputs = random_ops.random_uniform((3, 2), seed=1)
+    outputs = rff_layer(inputs)
+    self.assertListEqual([3, 10], outputs.get_shape().as_list())
+    num_trainable_vars = 1 if trainable else 0
+    self.assertLen(rff_layer.non_trainable_variables, 3 - num_trainable_vars)
+    if not context.executing_eagerly():
+      self.assertLen(
+          ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES),
+          num_trainable_vars)
+
+  @test_util.assert_no_new_pyobjects_executing_eagerly
+  def test_no_eager_Leak(self):
+    # Tests that repeatedly constructing and building a Layer does not leak
+    # Python objects.
+    inputs = random_ops.random_uniform((5, 4), seed=1)
+    kernel_layers.RandomFourierFeatures(output_dim=4, name='rff')(inputs)
+    kernel_layers.RandomFourierFeatures(output_dim=10, scale=2.0)(inputs)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def test_output_shape(self):
+    inputs = random_ops.random_uniform((3, 2), seed=1)
+    rff_layer = kernel_layers.RandomFourierFeatures(
+        output_dim=7, name='random_fourier_features', trainable=True)
+    outputs = rff_layer(inputs)
+    self.assertEqual([3, 7], outputs.get_shape().as_list())
+
+  @parameterized.named_parameters(
+      ('gaussian', 'gaussian'), ('laplacian', 'laplacian'),
+      ('other', init_ops.random_uniform_initializer))
+  @test_util.run_deprecated_v1
+  def test_call_on_placeholder(self, initializer):
+    inputs = array_ops.placeholder(dtype=dtypes.float32, shape=[None, None])
+    rff_layer = kernel_layers.RandomFourierFeatures(
+        output_dim=5,
+        kernel_initializer=initializer,
+        name='random_fourier_features')
+    with self.assertRaisesRegexp(
+        ValueError, r'The last dimension of the inputs to '
+        '`RandomFourierFeatures` should be defined. Found `None`.'):
+      rff_layer(inputs)
+
+    inputs = array_ops.placeholder(dtype=dtypes.float32, shape=[2, None])
+    rff_layer = kernel_layers.RandomFourierFeatures(
+        output_dim=5,
+        kernel_initializer=initializer,
+        name='random_fourier_features')
+    with self.assertRaisesRegexp(
+        ValueError, r'The last dimension of the inputs to '
+        '`RandomFourierFeatures` should be defined. Found `None`.'):
+      rff_layer(inputs)
+
+    inputs = array_ops.placeholder(dtype=dtypes.float32, shape=[None, 3])
+    rff_layer = kernel_layers.RandomFourierFeatures(
+        output_dim=5, name='random_fourier_features')
+    rff_layer(inputs)
+
+  @parameterized.named_parameters(('gaussian', 10, 'gaussian', 2.0),
+                                  ('laplacian', 5, 'laplacian', None),
+                                  ('other', 10, init_ops.ones_initializer, 1.0))
+  @test_util.run_in_graph_and_eager_modes()
+  def test_compute_output_shape(self, output_dim, initializer, scale):
+    rff_layer = kernel_layers.RandomFourierFeatures(
+        output_dim, initializer, scale=scale, name='rff')
+    with self.assertRaises(ValueError):
+      rff_layer.compute_output_shape(tensor_shape.TensorShape(None))
+    with self.assertRaises(ValueError):
+      rff_layer.compute_output_shape(tensor_shape.TensorShape([]))
+    with self.assertRaises(ValueError):
+      rff_layer.compute_output_shape(tensor_shape.TensorShape([3]))
+    with self.assertRaises(ValueError):
+      rff_layer.compute_output_shape(tensor_shape.TensorShape([3, 2, 3]))
+
+    with self.assertRaisesRegexp(
+        ValueError, r'The innermost dimension of input shape must be defined.'):
+      rff_layer.compute_output_shape(tensor_shape.TensorShape([3, None]))
+
+    self.assertEqual([None, output_dim],
+                     rff_layer.compute_output_shape((None, 3)).as_list())
+    self.assertEqual([None, output_dim],
+                     rff_layer.compute_output_shape(
+                         tensor_shape.TensorShape([None, 2])).as_list())
+    self.assertEqual([4, output_dim],
+                     rff_layer.compute_output_shape((4, 1)).as_list())
+
+  @parameterized.named_parameters(
+      ('gaussian', 10, 'gaussian', 3.0, False),
+      ('laplacian', 5, 'laplacian', 5.5, True),
+      ('other', 7, init_ops.random_uniform_initializer(), None, True))
+  @test_util.run_in_graph_and_eager_modes()
+  def test_get_config(self, output_dim, initializer, scale, trainable):
+    rff_layer = kernel_layers.RandomFourierFeatures(
+        output_dim,
+        initializer,
+        scale=scale,
+        trainable=trainable,
+        name='random_fourier_features',
+    )
+    expected_initializer = initializer
+    if isinstance(initializer, init_ops.Initializer):
+      expected_initializer = initializers.serialize(initializer)
+
+    expected_config = {
+        'output_dim': output_dim,
+        'kernel_initializer': expected_initializer,
+        'scale': scale,
+        'name': 'random_fourier_features',
+        'trainable': trainable,
+        'dtype': None,
+    }
+    self.assertLen(expected_config, len(rff_layer.get_config()))
+    self.assertSameElements(
+        list(expected_config.items()), list(rff_layer.get_config().items()))
+
+  @parameterized.named_parameters(
+      ('gaussian', 5, 'gaussian', None, True),
+      ('laplacian', 5, 'laplacian', 5.5, False),
+      ('other', 7, init_ops.ones_initializer(), 2.0, True))
+  @test_util.run_in_graph_and_eager_modes()
+  def test_from_config(self, output_dim, initializer, scale, trainable):
+    model_config = {
+        'output_dim': output_dim,
+        'kernel_initializer': initializer,
+        'scale': scale,
+        'trainable': trainable,
+        'name': 'random_fourier_features',
+    }
+    rff_layer = kernel_layers.RandomFourierFeatures.from_config(model_config)
+    self.assertEqual(rff_layer.output_dim, output_dim)
+    self.assertEqual(rff_layer.kernel_initializer, initializer)
+    self.assertEqual(rff_layer.scale, scale)
+    self.assertEqual(rff_layer.trainable, trainable)
+
+    inputs = random_ops.random_uniform((3, 2), seed=1)
+    outputs = rff_layer(inputs)
+    self.assertListEqual([3, output_dim], outputs.get_shape().as_list())
+    num_trainable_vars = 1 if trainable else 0
+    self.assertLen(rff_layer.trainable_variables, num_trainable_vars)
+    if trainable:
+      self.assertEqual('random_fourier_features/random_features_scale:0',
+                       rff_layer.trainable_variables[0].name)
+    self.assertLen(rff_layer.non_trainable_variables, 3 - num_trainable_vars)
+    if not context.executing_eagerly():
+      self.assertLen(
+          ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES),
+          num_trainable_vars)
+
+  @parameterized.named_parameters(
+      ('gaussian', 10, 'gaussian', 3.0, True),
+      ('laplacian', 5, 'laplacian', 5.5, False),
+      ('other', 10, init_ops.random_uniform_initializer(), None, True))
+  @test_util.run_in_graph_and_eager_modes()
+  def test_same_random_features_params_reused(self, output_dim, initializer,
+                                              scale, trainable):
+    """Applying the layer on the same input twice gives the same output."""
+    rff_layer = kernel_layers.RandomFourierFeatures(
+        output_dim=output_dim,
+        kernel_initializer=initializer,
+        scale=scale,
+        trainable=trainable,
+        name='random_fourier_features')
+    inputs = constant_op.constant(
+        np.random.uniform(low=-1.0, high=1.0, size=(2, 4)))
+    output1 = rff_layer.apply(inputs)
+    output2 = rff_layer.apply(inputs)
+    self._assert_all_close(output1, output2)
+
+  @parameterized.named_parameters(
+      ('gaussian', 'gaussian', 5.0), ('laplacian', 'laplacian', 3.0),
+      ('other', init_ops.random_uniform_initializer(), 5.0))
+  @test_util.run_in_graph_and_eager_modes()
+  def test_different_params_similar_approximation(self, initializer, scale):
+    # Layers initialized using different randomness (seed).
+    rff_layer1 = kernel_layers.RandomFourierFeatures(
+        output_dim=3000,
+        kernel_initializer=initializer,
+        scale=scale,
+        name='rff1')
+    rff_layer2 = kernel_layers.RandomFourierFeatures(
+        output_dim=2000,
+        kernel_initializer=initializer,
+        scale=scale,
+        name='rff2')
+    # Two distinct inputs.
+    x = constant_op.constant([[1.0, -1.0, 0.5]])
+    y = constant_op.constant([[-1.0, 1.0, 1.0]])
+
+    # Apply both layers to both inputs.
+    output_x1 = math.sqrt(2.0 / 3000.0) * rff_layer1.apply(x)
+    output_y1 = math.sqrt(2.0 / 3000.0) * rff_layer1.apply(y)
+    output_x2 = math.sqrt(2.0 / 2000.0) * rff_layer2.apply(x)
+    output_y2 = math.sqrt(2.0 / 2000.0) * rff_layer2.apply(y)
+
+    # Compute the inner products of the outputs (on inputs x and y) for both
+    # layers. For any fixed random features layer rff_layer, and inputs x, y,
+    # rff_layer(x)^T * rff_layer(y) ~= K(x,y) up to a normalization factor.
+    approx_kernel1 = kernelized_utils.inner_product(output_x1, output_y1)
+    approx_kernel2 = kernelized_utils.inner_product(output_x2, output_y2)
+    self._assert_all_close(approx_kernel1, approx_kernel2, atol=0.05)
+
+  @parameterized.named_parameters(
+      ('gaussian', 'gaussian', 5.0, _exact_gaussian(stddev=5.0)),
+      ('laplacian', 'laplacian', 20.0, _exact_laplacian(stddev=20.0)))
+  @test_util.run_in_graph_and_eager_modes()
+  def test_bad_kernel_approximation(self, initializer, scale, exact_kernel_fn):
+    """Approximation is bad when output dimension is small."""
+    # Two distinct inputs.
+    x = constant_op.constant([[1.0, -1.0, 0.5]])
+    y = constant_op.constant([[-1.0, 1.0, 1.0]])
+
+    small_output_dim = 10
+    random_seed.set_random_seed(1234)
+    # Initialize layer.
+    rff_layer = kernel_layers.RandomFourierFeatures(
+        output_dim=small_output_dim,
+        kernel_initializer=initializer,
+        scale=scale,
+        name='random_fourier_features')
+
+    # Apply layer to both inputs.
+    output_x = math.sqrt(2.0 / small_output_dim) * rff_layer.apply(x)
+    output_y = math.sqrt(2.0 / small_output_dim) * rff_layer.apply(y)
+
+    # The inner products of the outputs (on inputs x and y) approximates the
+    # real value of the RBF kernel but poorly since the output dimension of the
+    # layer is small.
+    exact_kernel_value = exact_kernel_fn(x, y)
+    approx_kernel_value = kernelized_utils.inner_product(output_x, output_y)
+    abs_error = math_ops.abs(exact_kernel_value - approx_kernel_value)
+    if not context.executing_eagerly():
+      with self.cached_session() as sess:
+        keras_backend._initialize_variables(sess)
+        abs_error_eval = sess.run([abs_error])
+        self.assertGreater(abs_error_eval[0][0], 0.05)
+        self.assertLess(abs_error_eval[0][0], 0.5)
+    else:
+      self.assertGreater(abs_error, 0.05)
+      self.assertLess(abs_error, 0.5)
+
+  @parameterized.named_parameters(
+      ('gaussian', 'gaussian', 10.0, _exact_gaussian(stddev=10.0)),
+      ('laplacian', 'laplacian', 50.0, _exact_laplacian(stddev=50.0)))
+  @test_util.run_in_graph_and_eager_modes()
+  def test_good_kernel_approximation_multiple_inputs(self, initializer, scale,
+                                                     exact_kernel_fn):
+    # Parameters.
+    input_dim = 5
+    output_dim = 5000
+    x_rows = 20
+    y_rows = 30
+
+    random_seed.set_random_seed(1234)
+    x = random_ops.random_uniform(shape=(x_rows, input_dim), maxval=1.0)
+    y = random_ops.random_uniform(shape=(y_rows, input_dim), maxval=1.0)
+
+    rff_layer = kernel_layers.RandomFourierFeatures(
+        output_dim=output_dim,
+        kernel_initializer=initializer,
+        scale=scale,
+        name='random_fourier_features')
+
+    # The shapes of output_x and output_y are (x_rows, output_dim) and
+    # (y_rows, output_dim) respectively.
+    output_x = math.sqrt(2.0 / output_dim) * rff_layer.apply(x)
+    output_y = math.sqrt(2.0 / output_dim) * rff_layer.apply(y)
+
+    approx_kernel_matrix = kernelized_utils.inner_product(output_x, output_y)
+    exact_kernel_matrix = exact_kernel_fn(x, y)
+    self._assert_all_close(approx_kernel_matrix, exact_kernel_matrix, atol=0.1)
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/python/keras/utils/kernelized_utils.py b/tensorflow/python/keras/utils/kernelized_utils.py
new file mode 100644
index 0000000000..26ad59c9f2
--- /dev/null
+++ b/tensorflow/python/keras/utils/kernelized_utils.py
@@ -0,0 +1,117 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Utility methods related to kernelized layers."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+
+
+def _to_matrix(u):
+  """If input tensor is a vector (i.e., has rank 1), converts it to matrix."""
+  u_rank = len(u.shape)
+  if u_rank not in [1, 2]:
+    raise ValueError('The input tensor should have rank 1 or 2. Given rank: {}'
+                     .format(u_rank))
+  if u_rank == 1:
+    return array_ops.expand_dims(u, 0)
+  return u
+
+
+def _align_matrices(x, y):
+  """Aligns x and y tensors to allow computations over pairs of their rows."""
+  x_matrix = _to_matrix(x)
+  y_matrix = _to_matrix(y)
+  x_shape = x_matrix.shape
+  y_shape = y_matrix.shape
+  if y_shape[1] != x_shape[1]:  # dimensions do not match.
+    raise ValueError(
+        'The outermost dimensions of the input tensors should match. Given: {} '
+        'vs {}.'.format(y_shape[1], x_shape[1]))
+
+  x_tile = array_ops.tile(
+      array_ops.expand_dims(x_matrix, 1), [1, y_shape[0], 1])
+  y_tile = array_ops.tile(
+      array_ops.expand_dims(y_matrix, 0), [x_shape[0], 1, 1])
+  return x_tile, y_tile
+
+
+def inner_product(u, v):
+  u = _to_matrix(u)
+  v = _to_matrix(v)
+  return math_ops.matmul(u, v, transpose_b=True)
+
+
+def exact_gaussian_kernel(x, y, stddev):
+  """Computes exact Gaussian kernel value(s) for tensors x and y and stddev.
+
+  The Gaussian kernel for vectors u, v is defined as follows:
+       K(u, v) = exp(-||u-v||^2 / (2* stddev^2))
+  where the norm is the l2-norm. x, y can be either vectors or matrices. If they
+  are vectors, they must have the same dimension. If they are matrices, they
+  must have the same number of columns. In the latter case, the method returns
+  (as a matrix) K(u, v) values for all pairs (u, v) where u is a row from x and
+  v is a row from y.
+
+  Args:
+    x: a tensor of rank 1 or 2. It's shape should be either [dim] or [m, dim].
+    y: a tensor of rank 1 or 2. It's shape should be either [dim] or [n, dim].
+    stddev: The width of the Gaussian kernel.
+
+  Returns:
+    A single value (scalar) with shape (1, 1) (if x, y are vectors) or a matrix
+      of shape (m, n) with entries K(u, v) (where K is the Gaussian kernel) for
+      all (u,v) pairs where u, v are rows from x and y respectively.
+
+  Raises:
+    InvalidShapeError: if the shapes of x, y are not compatible.
+  """
+  x_aligned, y_aligned = _align_matrices(x, y)
+  diff_squared_l2_norm = math_ops.reduce_sum(
+      math_ops.square(math_ops.subtract(x_aligned, y_aligned)), 2)
+  return math_ops.exp(-diff_squared_l2_norm / (2 * stddev * stddev))
+
+
+def exact_laplacian_kernel(x, y, stddev):
+  """Computes exact Laplacian kernel value(s) for tensors x and y using stddev.
+
+  The Laplacian kernel for vectors u, v is defined as follows:
+       K(u, v) = exp(-||u-v|| / stddev)
+  where the norm is the l1-norm. x, y can be either vectors or matrices. If they
+  are vectors, they must have the same dimension. If they are matrices, they
+  must have the same number of columns. In the latter case, the method returns
+  (as a matrix) K(u, v) values for all pairs (u, v) where u is a row from x and
+  v is a row from y.
+
+  Args:
+    x: a tensor of rank 1 or 2. It's shape should be either [dim] or [m, dim].
+    y: a tensor of rank 1 or 2. It's shape should be either [dim] or [n, dim].
+    stddev: The width of the Gaussian kernel.
+
+  Returns:
+    A single value (scalar) with shape (1, 1)  if x, y are vectors or a matrix
+    of shape (m, n) with entries K(u, v) (where K is the Laplacian kernel) for
+    all (u,v) pairs where u, v are rows from x and y respectively.
+
+  Raises:
+    InvalidShapeError: if the shapes of x, y are not compatible.
+  """
+  x_aligned, y_aligned = _align_matrices(x, y)
+  diff_l1_norm = math_ops.reduce_sum(
+      math_ops.abs(math_ops.subtract(x_aligned, y_aligned)), 2)
+  return math_ops.exp(-diff_l1_norm / stddev)
diff --git a/tensorflow/python/keras/utils/kernelized_utils_test.py b/tensorflow/python/keras/utils/kernelized_utils_test.py
new file mode 100644
index 0000000000..7e9a72493d
--- /dev/null
+++ b/tensorflow/python/keras/utils/kernelized_utils_test.py
@@ -0,0 +1,116 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for kernelized_utils.py."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import functools
+
+from absl.testing import parameterized
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.keras.utils import kernelized_utils
+from tensorflow.python.platform import test
+
+
+def _exact_gaussian(stddev):
+  return functools.partial(
+      kernelized_utils.exact_gaussian_kernel, stddev=stddev)
+
+
+def _exact_laplacian(stddev):
+  return functools.partial(
+      kernelized_utils.exact_laplacian_kernel, stddev=stddev)
+
+
+class KernelizedUtilsTest(test.TestCase, parameterized.TestCase):
+
+  @parameterized.named_parameters(
+      ('gaussian', _exact_gaussian(stddev=10.0), [[1.0]]),
+      ('laplacian', _exact_laplacian(stddev=50.0), [[1.0]]))
+  def test_equal_vectors(self, exact_kernel_fn, expected_values):
+    """Identical vectors give exactly the identity kernel value."""
+    x = constant_op.constant([0.5, -0.5, -0.5, 0.5])
+    y = constant_op.constant([0.5, -0.5, -0.5, 0.5])
+    exact_kernel = exact_kernel_fn(x, y)
+    shape = exact_kernel.get_shape().as_list()
+    self.assertLen(shape, 2)
+    # x and y are identical and therefore K(x, y) will be precisely equal to
+    # the identity value of the kernel.
+    self.assertAllClose(expected_values, exact_kernel, atol=1e-6)
+
+  @parameterized.named_parameters(
+      ('gaussian', _exact_gaussian(stddev=10.0), [[1.0]]),
+      ('laplacian', _exact_laplacian(stddev=50.0), [[1.0]]))
+  def test_almost_identical_vectors(self, exact_kernel_fn, expected_values):
+    """Almost identical vectors give the identity kernel value."""
+    x = constant_op.constant([1.0, 0.4, -2.1, -1.1])
+    y = constant_op.constant([1.01, 0.39, -2.099, -1.101])
+    exact_kernel = exact_kernel_fn(x, y)
+    shape = exact_kernel.get_shape().as_list()
+    self.assertLen(shape, 2)
+    # x and y are almost identical and therefore K(x, y) will be almost equal to
+    # the identity value of the kernel.
+    self.assertAllClose(expected_values, exact_kernel, atol=1e-3)
+
+  @parameterized.named_parameters(
+      ('gaussian', _exact_gaussian(stddev=1.0), [[0.99], [0.977]]),
+      ('laplacian', _exact_laplacian(stddev=5.0), [[0.96], [0.94]]))
+  def test_similar_matrices(self, exact_kernel_fn, expected_values):
+    """Pairwise "close" vectors give high kernel values (similarity scores)."""
+    x = constant_op.constant([1.0, 3.4, -2.1, 0.9, 3.3, -2.0], shape=[2, 3])
+    y = constant_op.constant([1.1, 3.35, -2.05])
+    exact_kernel = exact_kernel_fn(x, y)
+    shape = exact_kernel.get_shape().as_list()
+    self.assertLen(shape, 2)
+    # The 2 rows of x are close to y. The pairwise kernel values (similarity
+    # scores) are somewhat close to the identity value of the kernel.
+    self.assertAllClose(expected_values, exact_kernel, atol=1e-2)
+
+  @parameterized.named_parameters(
+      ('gaussian', _exact_gaussian(stddev=2.0), [[.997, .279], [.251, 1.],
+                                                 [.164, 0.019]]),
+      ('laplacian', _exact_laplacian(stddev=2.0), [[.904, .128], [.116, 1.],
+                                                   [.07, 0.027]]))
+  def test_matrices_varying_similarity(self, exact_kernel_fn, expected_values):
+    """Test matrices with row vectors of varying pairwise similarity."""
+    x = constant_op.constant([1.0, 2., -2., 0.9, 3.3, -1.0], shape=[3, 2])
+    y = constant_op.constant([1.1, 2.1, -2., 0.9], shape=[2, 2])
+    exact_kernel = exact_kernel_fn(x, y)
+
+    shape = exact_kernel.get_shape().as_list()
+    self.assertLen(shape, 2)
+    self.assertAllClose(expected_values, exact_kernel, atol=1e-2)
+
+  @parameterized.named_parameters(
+      ('gaussian', _exact_gaussian(stddev=1.0), [[0.0]]),
+      ('laplacian', _exact_laplacian(stddev=1.0), [[0.0]]))
+  def test_completely_dissimilar_vectors(self, exact_kernel_fn,
+                                         expected_values):
+    """Very dissimilar vectors give very low similarity scores."""
+    x = constant_op.constant([1.0, 3.4, -2.1, -5.1])
+    y = constant_op.constant([0.5, 2.1, 1.0, 3.0])
+    exact_kernel = exact_kernel_fn(x, y)
+    shape = exact_kernel.get_shape().as_list()
+    self.assertLen(shape, 2)
+    # x and y are very "far" from each other and so the corresponding kernel
+    # value will be very low.
+    self.assertAllClose(expected_values, exact_kernel, atol=1e-2)
+
+
+if __name__ == '__main__':
+  test.main()
-- 
GitLab


From dbf08f7e2087f427e101bc178db4df672b01fab3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 23 Jan 2019 12:45:26 -0800
Subject: [PATCH 1131/2345] Implements Profiler server that can perform
 on-demand profiling on TF eager program. It reuse service defined for TPU
 since we want both TPU and GPU user cand use the same capture_tpu_profile and
 TensorBoard profile plugin following this instruction:
 https://cloud.google.com/tpu/docs/cloud-tpu-tools#capture_profile

I will rename TPUProfiler and capture_tpu_profile to more generic names.

PiperOrigin-RevId: 230583138
---
 tensorflow/c/eager/BUILD                      |  1 +
 tensorflow/c/eager/c_api_experimental.cc      |  6 ++
 tensorflow/c/eager/c_api_experimental.h       |  9 +++
 .../core/common_runtime/eager/context.cc      |  8 ++
 .../core/common_runtime/eager/context.h       |  5 ++
 .../core/profiler/lib/eager_profiler.cc       |  4 +-
 tensorflow/core/profiler/rpc/BUILD            | 35 +++++++++
 .../core/profiler/rpc/profiler_server.cc      | 43 +++++++++++
 .../core/profiler/rpc/profiler_server.h       | 24 ++++++
 .../profiler/rpc/profiler_service_impl.cc     | 74 +++++++++++++++++++
 .../core/profiler/rpc/profiler_service_impl.h | 30 ++++++++
 tensorflow/python/eager/profiler.py           | 17 +++++
 tensorflow/python/pywrap_tfe.i                |  1 +
 13 files changed, 255 insertions(+), 2 deletions(-)
 create mode 100644 tensorflow/core/profiler/rpc/BUILD
 create mode 100644 tensorflow/core/profiler/rpc/profiler_server.cc
 create mode 100644 tensorflow/core/profiler/rpc/profiler_server.h
 create mode 100644 tensorflow/core/profiler/rpc/profiler_service_impl.cc
 create mode 100644 tensorflow/core/profiler/rpc/profiler_service_impl.h

diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
index 777e2e626a..04dfefa6da 100644
--- a/tensorflow/c/eager/BUILD
+++ b/tensorflow/c/eager/BUILD
@@ -210,6 +210,7 @@ tf_cuda_library(
         "//tensorflow/core/distributed_runtime:remote_device",
         "//tensorflow/core/distributed_runtime:server_lib",
         "//tensorflow/core/distributed_runtime:worker_env",
+        "//tensorflow/core/profiler/rpc:profiler_server",
         "//tensorflow/core:gpu_runtime",
     ],
 )
diff --git a/tensorflow/c/eager/c_api_experimental.cc b/tensorflow/c/eager/c_api_experimental.cc
index 1ce03fb226..dab1750564 100644
--- a/tensorflow/c/eager/c_api_experimental.cc
+++ b/tensorflow/c/eager/c_api_experimental.cc
@@ -17,6 +17,7 @@ limitations under the License.
 
 #include "tensorflow/c/c_api.h"
 #include "tensorflow/c/eager/c_api_internal.h"
+#include "tensorflow/core/profiler/rpc/profiler_server.h"
 
 using tensorflow::string;
 
@@ -44,3 +45,8 @@ void TFE_ProfilerSerializeToString(TFE_Context* ctx, TFE_Profiler* profiler,
     tensorflow::port::Free(data);
   };
 }
+
+void TFE_StartProfilerServer(TFE_Context* ctx, int port) {
+  auto server_thread = tensorflow::StartProfilerServer(&ctx->context, port);
+  ctx->context.AddChildThread(std::move(server_thread));
+}
diff --git a/tensorflow/c/eager/c_api_experimental.h b/tensorflow/c/eager/c_api_experimental.h
index 5cd5569468..8c85d0e516 100644
--- a/tensorflow/c/eager/c_api_experimental.h
+++ b/tensorflow/c/eager/c_api_experimental.h
@@ -42,6 +42,15 @@ TF_CAPI_EXPORT extern void TFE_ProfilerSerializeToString(TFE_Context* ctx,
                                                          TF_Buffer* buf,
                                                          TF_Status* status);
 
+// Start a profiler grpc server which listens to specified port. It will start
+// the server on its own thread. It can be shutdown by destructing TFE_Context.
+// Creating multiple profiler server is allowed. The service defined in
+// tensorflow/contrib/tpu/profiler/tpu_profiler.proto. Please use
+// tensorflow/contrib/tpu/profiler/capture_tpu_profile to capture tracable
+// file following
+// https://cloud.google.com/tpu/docs/cloud-tpu-tools#capture_trace.
+TF_CAPI_EXPORT extern void TFE_StartProfilerServer(TFE_Context* ctx, int port);
+
 #ifdef __cplusplus
 } /* end extern "C" */
 #endif
diff --git a/tensorflow/core/common_runtime/eager/context.cc b/tensorflow/core/common_runtime/eager/context.cc
index 1d93d2bbe6..4bc3843ee8 100644
--- a/tensorflow/core/common_runtime/eager/context.cc
+++ b/tensorflow/core/common_runtime/eager/context.cc
@@ -207,6 +207,14 @@ EagerContext::~EagerContext() {
   executor_.WaitForAllPendingNodes().IgnoreError();
   ClearCaches();
   rendezvous_->Unref();
+
+  for (auto& thread : child_threads_) {
+    thread.reset();
+  }
+}
+
+void EagerContext::AddChildThread(std::unique_ptr<Thread> thread) {
+  child_threads_.push_back(std::move(thread));
 }
 
 bool EagerContext::FindFunctionByName(const string& name) {
diff --git a/tensorflow/core/common_runtime/eager/context.h b/tensorflow/core/common_runtime/eager/context.h
index d8807cf839..1f24109a76 100644
--- a/tensorflow/core/common_runtime/eager/context.h
+++ b/tensorflow/core/common_runtime/eager/context.h
@@ -30,6 +30,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/common_runtime/rendezvous_mgr.h"
 #include "tensorflow/core/example/example.pb.h"
+#include "tensorflow/core/platform/env.h"
 #ifndef __ANDROID__
 #include "tensorflow/core/distributed_runtime/eager/eager_client.h"
 #include "tensorflow/core/distributed_runtime/server_lib.h"
@@ -238,6 +239,9 @@ class EagerContext {
 
   tensorflow::Env* TFEnv() const { return env_; }
 
+  // All child threads will be reset() when destructing EagerContext.
+  void AddChildThread(std::unique_ptr<Thread> thread);
+
  private:
   void InitDeviceMapAndAsync();
   Status MaybeRegisterFunctionRemotely(const FunctionDef& fdef);
@@ -334,6 +338,7 @@ class EagerContext {
 
   bool use_send_tensor_rpc_;
   const bool pin_small_ops_to_cpu_;
+  std::vector<std::unique_ptr<tensorflow::Thread>> child_threads_;
 };
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/profiler/lib/eager_profiler.cc b/tensorflow/core/profiler/lib/eager_profiler.cc
index 6b2e8b9a19..ab97682cd8 100644
--- a/tensorflow/core/profiler/lib/eager_profiler.cc
+++ b/tensorflow/core/profiler/lib/eager_profiler.cc
@@ -56,8 +56,8 @@ void ConvertRunMetadataToTraceEvent(RunMetadata* run_metadata,
       event->set_device_id(device_id);
       event->set_resource_id(0);
       event->set_name(node.node_name());
-      event->set_timestamp_ps(node.all_start_micros() * 1000000000);
-      event->set_duration_ps(node.all_end_rel_micros() * 1000000000);
+      event->set_timestamp_ps(node.all_start_micros() * 1000000);
+      event->set_duration_ps(node.all_end_rel_micros() * 1000000);
       (*args)["label"] = node.timeline_label();
     }
   }
diff --git a/tensorflow/core/profiler/rpc/BUILD b/tensorflow/core/profiler/rpc/BUILD
new file mode 100644
index 0000000000..83ec75dfb4
--- /dev/null
+++ b/tensorflow/core/profiler/rpc/BUILD
@@ -0,0 +1,35 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])  # Apache 2.0
+
+load("//tensorflow:tensorflow.bzl", "tf_cuda_library")
+
+tf_cuda_library(
+    name = "profiler_service_impl",
+    srcs = ["profiler_service_impl.cc"],
+    hdrs = ["profiler_service_impl.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tensorflow:grpc++",
+        "//tensorflow/contrib/tpu/profiler:tpu_profiler_proto_cc",
+        "//tensorflow/core:framework",
+        "//tensorflow/core/common_runtime/eager:context",
+        "//tensorflow/core/profiler/lib:eager_profiler",
+    ],
+    alwayslink = 1,
+)
+
+tf_cuda_library(
+    name = "profiler_server",
+    srcs = ["profiler_server.cc"],
+    hdrs = ["profiler_server.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":profiler_service_impl",
+        "//tensorflow:grpc++",
+        "//tensorflow/contrib/tpu/profiler:tpu_profiler_proto_cc",
+        "//tensorflow/core:framework",
+        "//tensorflow/core/common_runtime/eager:context",
+    ],
+    alwayslink = 1,
+)
diff --git a/tensorflow/core/profiler/rpc/profiler_server.cc b/tensorflow/core/profiler/rpc/profiler_server.cc
new file mode 100644
index 0000000000..08affffb2d
--- /dev/null
+++ b/tensorflow/core/profiler/rpc/profiler_server.cc
@@ -0,0 +1,43 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/profiler/rpc/profiler_server.h"
+#include <memory>
+#include <utility>
+#include "grpcpp/grpcpp.h"
+#include "tensorflow/contrib/tpu/profiler/tpu_profiler.grpc.pb.h"
+#include "tensorflow/core/profiler/rpc/profiler_service_impl.h"
+#include "tensorflow/core/util/ptr_util.h"
+
+namespace tensorflow {
+
+std::unique_ptr<Thread> StartProfilerServer(EagerContext* const eager_context,
+                                            int32 port) {
+  return WrapUnique(eager_context->TFEnv()->StartThread(
+      {}, "profiler server", [eager_context, port]() {
+        string server_address = strings::StrCat("0.0.0.0:", port);
+        std::unique_ptr<TPUProfiler::Service> service =
+            CreateProfilerService(eager_context);
+        ::grpc::ServerBuilder builder;
+        builder.AddListeningPort(server_address,
+                                 ::grpc::InsecureServerCredentials());
+        builder.RegisterService(service.get());
+        std::unique_ptr<::grpc::Server> server(builder.BuildAndStart());
+        LOG(INFO) << "Profiling Server listening on " << server_address;
+        server->Wait();
+      }));
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/profiler/rpc/profiler_server.h b/tensorflow/core/profiler/rpc/profiler_server.h
new file mode 100644
index 0000000000..ebefdd6cb0
--- /dev/null
+++ b/tensorflow/core/profiler/rpc/profiler_server.h
@@ -0,0 +1,24 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_PROFILER_RPC_PROFILER_SERVER_H_
+#define TENSORFLOW_CORE_PROFILER_RPC_PROFILER_SERVER_H_
+
+#include "tensorflow/core/common_runtime/eager/context.h"
+
+namespace tensorflow {
+std::unique_ptr<Thread> StartProfilerServer(EagerContext* const eager_context,
+                                            int32 port);
+}  // namespace tensorflow
+#endif  // TENSORFLOW_CORE_PROFILER_RPC_PROFILER_SERVER_H_
diff --git a/tensorflow/core/profiler/rpc/profiler_service_impl.cc b/tensorflow/core/profiler/rpc/profiler_service_impl.cc
new file mode 100644
index 0000000000..bde2ff2f5f
--- /dev/null
+++ b/tensorflow/core/profiler/rpc/profiler_service_impl.cc
@@ -0,0 +1,74 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/profiler/rpc/profiler_service_impl.h"
+#include "grpcpp/support/status.h"
+#include "tensorflow/contrib/tpu/profiler/tpu_profiler.grpc.pb.h"
+#include "tensorflow/core/common_runtime/eager/context.h"
+#include "tensorflow/core/profiler/lib/eager_profiler.h"
+#include "tensorflow/core/util/ptr_util.h"
+
+namespace tensorflow {
+namespace {
+
+// TODO(fishx): Rename TPUProfiler to something more generic.
+class ProfilerServiceImpl : public TPUProfiler::Service {
+ public:
+  explicit ProfilerServiceImpl(EagerContext* const eager_context)
+      : eager_context_(eager_context) {}
+  ~ProfilerServiceImpl() override {}
+
+  ::grpc::Status Monitor(::grpc::ServerContext* ctx, const MonitorRequest* req,
+                         MonitorResponse* response) override {
+    return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "unimplemented.");
+  }
+
+  ::grpc::Status Profile(::grpc::ServerContext* ctx, const ProfileRequest* req,
+                         ProfileResponse* response) override {
+    LOG(INFO) << "Received a profile request.";
+    std::unique_ptr<EagerProfiler> profiler =
+        EagerProfiler::Create(eager_context_);
+    if (!profiler->Status().ok()) {
+      return ::grpc::Status(::grpc::StatusCode::INTERNAL,
+                            profiler->Status().error_message());
+    }
+
+    Env* env = eager_context_->TFEnv();
+    for (size_t i = 0; i < req->duration_ms(); ++i) {
+      env->SleepForMicroseconds(1000);
+      if (ctx->IsCancelled()) {
+        return ::grpc::Status::CANCELLED;
+      }
+    }
+
+    Status s = profiler->SerializeToString(response->mutable_encoded_trace());
+    if (!s.ok()) {
+      return ::grpc::Status(::grpc::StatusCode::INTERNAL, s.error_message());
+    }
+
+    return ::grpc::Status::OK;
+  }
+
+ private:
+  EagerContext* const eager_context_;
+};
+}  // namespace
+
+std::unique_ptr<TPUProfiler::Service> CreateProfilerService(
+    EagerContext* const eager_context) {
+  return MakeUnique<ProfilerServiceImpl>(eager_context);
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/profiler/rpc/profiler_service_impl.h b/tensorflow/core/profiler/rpc/profiler_service_impl.h
new file mode 100644
index 0000000000..79dc767eb1
--- /dev/null
+++ b/tensorflow/core/profiler/rpc/profiler_service_impl.h
@@ -0,0 +1,30 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_PROFILER_RPC_PROFILER_SERVICE_IMPL_H_
+#define TENSORFLOW_CORE_PROFILER_RPC_PROFILER_SERVICE_IMPL_H_
+
+#include "grpcpp/grpcpp.h"
+#include "grpcpp/server_context.h"
+#include "grpcpp/support/status.h"
+#include "tensorflow/contrib/tpu/profiler/tpu_profiler.grpc.pb.h"
+#include "tensorflow/core/common_runtime/eager/context.h"
+#include "tensorflow/core/profiler/lib/eager_profiler.h"
+
+namespace tensorflow {
+std::unique_ptr<TPUProfiler::Service> CreateProfilerService(
+    EagerContext* const eager_context);
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_PROFILER_RPC_PROFILER_SERVICE_IMPL_H_
diff --git a/tensorflow/python/eager/profiler.py b/tensorflow/python/eager/profiler.py
index de96ae6ce9..a883122643 100644
--- a/tensorflow/python/eager/profiler.py
+++ b/tensorflow/python/eager/profiler.py
@@ -75,6 +75,23 @@ def stop():
   return result
 
 
+def start_profiler_server(port):
+  """Start a profiler grpc server that listens to given port.
+
+  The profiler server will keep the program running even the training finishes.
+  Please shutdown the server with CTRL-C. The service defined in
+  tensorflow/contrib/tpu/profiler/tpu_profiler.proto. Please use
+  tensorflow/contrib/tpu/profiler/capture_tpu_profile to capture tracable
+  file following https://cloud.google.com/tpu/docs/cloud-tpu-tools#capture_trace
+
+  Args:
+    port: port profiler server listens to.
+  """
+  pywrap_tensorflow.TFE_StartProfilerServer(
+      context.context()._handle,  # pylint: disable=protected-access
+      port)
+
+
 class Profiler(object):
   """Context-manager eager profiler api.
 
diff --git a/tensorflow/python/pywrap_tfe.i b/tensorflow/python/pywrap_tfe.i
index 19254359cc..bae20aca50 100755
--- a/tensorflow/python/pywrap_tfe.i
+++ b/tensorflow/python/pywrap_tfe.i
@@ -36,6 +36,7 @@ limitations under the License.
 %rename("%s") TFE_NewProfiler;
 %rename("%s") TFE_DeleteProfiler;
 %rename("%s") TFE_ProfilerSerializeToString;
+%rename("%s") TFE_StartProfilerServer;
 %rename("%s") TFE_OpNameGetAttrType;
 %rename("%s") TFE_Py_InitEagerTensor;
 %rename("%s") TFE_Py_SetEagerTensorProfiler;
-- 
GitLab


From 5215b322c15ef29832ee734dc588dc50ce50d38b Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Wed, 23 Jan 2019 12:49:34 -0800
Subject: [PATCH 1132/2345] Fix division by zero when float min == max.

PiperOrigin-RevId: 230583756
---
 .../lite/tools/optimize/quantization_utils.cc |  6 ++++-
 .../tools/optimize/quantization_utils_test.cc | 22 +++++++++++++++++++
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/tools/optimize/quantization_utils.cc b/tensorflow/lite/tools/optimize/quantization_utils.cc
index c2405ca389..bcddfe1950 100644
--- a/tensorflow/lite/tools/optimize/quantization_utils.cc
+++ b/tensorflow/lite/tools/optimize/quantization_utils.cc
@@ -55,7 +55,11 @@ void GetAsymmetricQuantizationParams(
   min = std::min(static_cast<float>(min), 0.0f);
   max = std::max(static_cast<float>(max), 0.0f);
   const float scale = (max - min) / (quant_max_float - quant_min_float);
-  const float zero_point_from_min = quant_min_float - min / scale;
+  // Scale can be zero if min and max are exactly 0.0f.
+  float zero_point_from_min = quant_min_float;
+  if (scale != 0) {
+    zero_point_from_min = quant_min_float - min / scale;
+  }
   int64_t zero_point;
   if (zero_point_from_min < quant_min_float) {
     zero_point = static_cast<int64_t>(quant_min);
diff --git a/tensorflow/lite/tools/optimize/quantization_utils_test.cc b/tensorflow/lite/tools/optimize/quantization_utils_test.cc
index 0602d82a1c..a3f2467c74 100644
--- a/tensorflow/lite/tools/optimize/quantization_utils_test.cc
+++ b/tensorflow/lite/tools/optimize/quantization_utils_test.cc
@@ -126,6 +126,28 @@ TEST(QuantizationUtilsTest, AsymmetricQuantizationParamsWithZeroInRange) {
   EXPECT_LT(zero_point, quant_max);
 }
 
+TEST(QuantizationUtilsTest, AsymmetricQuantizationParamsWithZeroMinMax) {
+  const float float_min = 0;
+  const float float_max = 0;
+  const int quant_min = -128;
+  const int quant_max = 127;
+  QuantizationParametersT params;
+  GetAsymmetricQuantizationParams(float_min, float_max, quant_min, quant_max,
+                                  &params);
+  ASSERT_EQ(params.max.size(), 1);
+  ASSERT_EQ(params.min.size(), 1);
+  ASSERT_EQ(params.scale.size(), 1);
+  ASSERT_EQ(params.zero_point.size(), 1);
+  EXPECT_EQ(params.max[0], float_max);
+  EXPECT_EQ(params.min[0], float_min);
+  int64_t zero_point = params.zero_point[0];
+  float scale = params.scale[0];
+  const float eps = 1e-7f;
+  EXPECT_NEAR(scale, 0, eps);
+  EXPECT_NEAR(zero_point, quant_min, eps);
+  EXPECT_LT(zero_point, quant_max);
+}
+
 TEST(QuantizationUtilsTest, SymmetricPerChannelQuantization) {
   // Set up an input with [3, 2, 2, 2] size and 0 is the channel index.
   const std::vector<float> input = {
-- 
GitLab


From 87cc788e1cc04036e89d87bbe71df7f449d0e934 Mon Sep 17 00:00:00 2001
From: Revan Sopher <rsopher@google.com>
Date: Wed, 23 Jan 2019 13:14:20 -0800
Subject: [PATCH 1133/2345] Automated rollback of commit
 2435a1875b574f5d299b1dee431ab5ceccd6132f

PiperOrigin-RevId: 230588224
---
 tensorflow/opensource_only.files          |  2 +-
 tensorflow/python/BUILD                   |  6 +-
 tensorflow/python/platform/app.py         | 98 ++---------------------
 tensorflow/python/platform/googletest.py  | 12 +--
 tensorflow/tools/pip_package/BUILD        |  5 ++
 tensorflow/workspace.bzl                  |  3 +-
 third_party/{systemlibs => }/enum34.BUILD |  2 +-
 7 files changed, 26 insertions(+), 102 deletions(-)
 rename third_party/{systemlibs => }/enum34.BUILD (84%)

diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index c546a34cf8..75976cd8fc 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -147,7 +147,6 @@ tensorflow/third_party/systemlibs/cython.BUILD
 tensorflow/third_party/systemlibs/double_conversion.BUILD
 tensorflow/third_party/systemlibs/zlib.BUILD
 tensorflow/third_party/systemlibs/jsoncpp.BUILD
-tensorflow/third_party/systemlibs/enum34.BUILD
 tensorflow/third_party/systemlibs/re2.BUILD
 tensorflow/third_party/systemlibs/lmdb.BUILD
 tensorflow/third_party/systemlibs/googleapis.BUILD
@@ -213,6 +212,7 @@ tensorflow/third_party/git/BUILD.tpl
 tensorflow/third_party/git/BUILD
 tensorflow/third_party/git/git_configure.bzl
 tensorflow/third_party/protobuf/BUILD
+tensorflow/third_party/enum34.BUILD
 tensorflow/third_party/tflite_mobilenet.BUILD
 tensorflow/third_party/py/BUILD
 tensorflow/third_party/py/BUILD.tpl
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index bc109c09bc..c89027582c 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -214,6 +214,7 @@ py_library(
         ":pywrap_tensorflow",
         ":util",
         "//tensorflow/core:protos_all_py",
+        "@absl_py//absl:app",
         "@absl_py//absl/flags",
         "@six_archive//:six",
     ],
@@ -234,7 +235,10 @@ py_library(
     name = "platform_test",
     srcs = ["platform/googletest.py"],
     srcs_version = "PY2AND3",
-    deps = [":platform_benchmark"],
+    deps = [
+        ":platform_benchmark",
+        "@absl_py//absl/testing:absltest",
+    ],
 )
 
 tf_py_test(
diff --git a/tensorflow/python/platform/app.py b/tensorflow/python/platform/app.py
index 7b917235c0..303b70ff57 100644
--- a/tensorflow/python/platform/app.py
+++ b/tensorflow/python/platform/app.py
@@ -18,109 +18,23 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import errno as _errno
 import sys as _sys
 
+from absl.app import run as _run
+
 from tensorflow.python.platform import flags
 from tensorflow.python.util.tf_export import tf_export
 
 
-def _usage(shorthelp):
-  """Writes __main__'s docstring to stdout with some help text.
-
-  Args:
-    shorthelp: bool, if True, prints only flags from the main module,
-        rather than all flags.
-  """
-  doc = _sys.modules['__main__'].__doc__
-  if not doc:
-    doc = '\nUSAGE: %s [flags]\n' % _sys.argv[0]
-    doc = flags.text_wrap(doc, indent='       ', firstline_indent='')
-  else:
-    # Replace all '%s' with sys.argv[0], and all '%%' with '%'.
-    num_specifiers = doc.count('%') - 2 * doc.count('%%')
-    try:
-      doc %= (_sys.argv[0],) * num_specifiers
-    except (OverflowError, TypeError, ValueError):
-      # Just display the docstring as-is.
-      pass
-  if shorthelp:
-    flag_str = flags.FLAGS.main_module_help()
-  else:
-    flag_str = str(flags.FLAGS)
-  try:
-    _sys.stdout.write(doc)
-    if flag_str:
-      _sys.stdout.write('\nflags:\n')
-      _sys.stdout.write(flag_str)
-    _sys.stdout.write('\n')
-  except IOError as e:
-    # We avoid printing a huge backtrace if we get EPIPE, because
-    # "foo.par --help | less" is a frequent use case.
-    if e.errno != _errno.EPIPE:
-      raise
-
-
-class _HelpFlag(flags.BooleanFlag):
-  """Special boolean flag that displays usage and raises SystemExit."""
-  NAME = 'help'
-  SHORT_NAME = 'h'
-
-  def __init__(self):
-    super(_HelpFlag, self).__init__(
-        self.NAME, False, 'show this help', short_name=self.SHORT_NAME)
-
-  def parse(self, arg):
-    if arg:
-      _usage(shorthelp=True)
-      print()
-      print('Try --helpfull to get a list of all flags.')
-      _sys.exit(1)
-
-
-class _HelpshortFlag(_HelpFlag):
-  """--helpshort is an alias for --help."""
-  NAME = 'helpshort'
-  SHORT_NAME = None
-
-
-class _HelpfullFlag(flags.BooleanFlag):
-  """Display help for flags in main module and all dependent modules."""
-
-  def __init__(self):
-    super(_HelpfullFlag, self).__init__('helpfull', False, 'show full help')
-
-  def parse(self, arg):
-    if arg:
-      _usage(shorthelp=False)
-      _sys.exit(1)
-
-
-_define_help_flags_called = False
-
-
-def _define_help_flags():
-  global _define_help_flags_called
-  if not _define_help_flags_called:
-    flags.DEFINE_flag(_HelpFlag())
-    flags.DEFINE_flag(_HelpfullFlag())
-    flags.DEFINE_flag(_HelpshortFlag())
-    _define_help_flags_called = True
+def _parse_flags_tolerate_undef(argv):
+  """Parse args, returning any unknown flags (ABSL defaults to crashing)."""
+  return flags.FLAGS(_sys.argv if argv is None else argv, known_only=True)
 
 
 @tf_export(v1=['app.run'])
 def run(main=None, argv=None):
   """Runs the program with an optional 'main' function and 'argv' list."""
 
-  # Define help flags.
-  _define_help_flags()
-
-  # Parse known flags.
-  argv = flags.FLAGS(_sys.argv if argv is None else argv, known_only=True)
-
   main = main or _sys.modules['__main__'].main
 
-  # Call the main function, passing through any arguments
-  # to the final program.
-  _sys.exit(main(argv))
-
+  _run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
diff --git a/tensorflow/python/platform/googletest.py b/tensorflow/python/platform/googletest.py
index fe4b0d0d37..802721e34b 100644
--- a/tensorflow/python/platform/googletest.py
+++ b/tensorflow/python/platform/googletest.py
@@ -13,7 +13,7 @@
 # limitations under the License.
 # ==============================================================================
 
-"""Imports unittest as a replacement for testing.pybase.googletest."""
+"""Imports absltest as a replacement for testing.pybase.googletest."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -26,7 +26,7 @@ import tempfile
 
 # go/tf-wildcard-import
 # pylint: disable=wildcard-import
-from unittest import *
+from absl.testing.absltest import *
 # pylint: enable=wildcard-import
 
 from tensorflow.python.framework import errors
@@ -41,7 +41,7 @@ from tensorflow.python.util.tf_export import tf_export
 
 Benchmark = benchmark.TensorFlowBenchmark  # pylint: disable=invalid-name
 
-unittest_main = main
+absltest_main = main
 
 # We keep a global variable in this module to make sure we create the temporary
 # directory only once per test binary invocation.
@@ -51,7 +51,7 @@ _googletest_temp_dir = ''
 # pylint: disable=invalid-name
 # pylint: disable=undefined-variable
 def g_main(argv):
-  """Delegate to unittest.main after redefining testLoader."""
+  """Delegate to absltest.main after redefining testLoader."""
   if 'TEST_SHARD_STATUS_FILE' in os.environ:
     try:
       f = None
@@ -67,7 +67,7 @@ def g_main(argv):
 
   if ('TEST_TOTAL_SHARDS' not in os.environ or
       'TEST_SHARD_INDEX' not in os.environ):
-    return unittest_main(argv=argv)
+    return absltest_main(argv=argv)
 
   total_shards = int(os.environ['TEST_TOTAL_SHARDS'])
   shard_index = int(os.environ['TEST_SHARD_INDEX'])
@@ -87,7 +87,7 @@ def g_main(argv):
   # Override getTestCaseNames
   base_loader.getTestCaseNames = getShardedTestCaseNames
 
-  unittest_main(argv=argv, testLoader=base_loader)
+  absltest_main(argv=argv, testLoader=base_loader)
 
 
 # Redefine main to allow running benchmarks
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index c51b45a49c..3c1a9280a2 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -146,7 +146,11 @@ filegroup(
         "//third_party/eigen3:LICENSE",
         "//third_party/fft2d:LICENSE",
         "//third_party/hadoop:LICENSE.txt",
+        "@absl_py//absl:LICENSE",
+        "@absl_py//absl/logging:LICENSE",
         "@absl_py//absl/flags:LICENSE",
+        "@absl_py//absl/testing:LICENSE",
+        "@absl_py//absl/third_party/unittest3_backport:LICENSE",
         "@arm_neon_2_x86_sse//:LICENSE",
         "@astor_archive//:LICENSE",
         "@boringssl//:LICENSE",
@@ -155,6 +159,7 @@ filegroup(
         "@curl//:COPYING",
         "@double_conversion//:LICENSE",
         "@eigen_archive//:COPYING.MPL2",
+        "@enum34_archive//:LICENSE",
         "@farmhash_archive//:COPYING",
         "@fft2d//:fft/readme.txt",
         "@flatbuffers//:LICENSE.txt",
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 466262ab3f..b55b9c4c57 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -337,7 +337,8 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
             "https://pypi.python.org/packages/bf/3e/31d502c25302814a7c2f1d3959d2a3b3f78e509002ba91aea64993936876/enum34-1.1.6.tar.gz",
         ],
         sha256 = "8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1",
-        build_file = clean_dep("//third_party/systemlibs:enum34.BUILD"),
+        build_file = clean_dep("//third_party:enum34.BUILD"),
+        strip_prefix = "enum34-1.1.6/enum",
     )
 
     tf_http_archive(
diff --git a/third_party/systemlibs/enum34.BUILD b/third_party/enum34.BUILD
similarity index 84%
rename from third_party/systemlibs/enum34.BUILD
rename to third_party/enum34.BUILD
index a5ac25592b..85262b07f6 100644
--- a/third_party/systemlibs/enum34.BUILD
+++ b/third_party/enum34.BUILD
@@ -7,7 +7,7 @@ exports_files(["LICENSE"])
 
 py_library(
     name = "enum",
-    srcs = ["enum34-1.1.6/enum/__init__.py"],
+    srcs = ["__init__.py"],
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
 )
-- 
GitLab


From e37ff18eb289a4da97bd681816e6f68756b6c19d Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Wed, 23 Jan 2019 13:16:15 -0800
Subject: [PATCH 1134/2345] Adding public APIs for `cumsum` and `cumprod` keras
 backend functions.

PiperOrigin-RevId: 230588562
---
 tensorflow/python/keras/backend.py                        | 2 ++
 .../tools/api/golden/v1/tensorflow.keras.backend.pbtxt    | 8 ++++++++
 .../tools/api/golden/v2/tensorflow.keras.backend.pbtxt    | 8 ++++++++
 3 files changed, 18 insertions(+)

diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index 9677f12d34..f7c0d4741a 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -1558,6 +1558,7 @@ def prod(x, axis=None, keepdims=False):
   return math_ops.reduce_prod(x, axis, keepdims)
 
 
+@keras_export('keras.backend.cumsum')
 def cumsum(x, axis=0):
   """Cumulative sum of the values in a tensor, alongside the specified axis.
 
@@ -1571,6 +1572,7 @@ def cumsum(x, axis=0):
   return math_ops.cumsum(x, axis=axis)
 
 
+@keras_export('keras.backend.cumprod')
 def cumprod(x, axis=0):
   """Cumulative product of the values in a tensor, alongside the specified axis.
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.backend.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.backend.pbtxt
index 8cd0c6ea5f..3a31024817 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.backend.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.backend.pbtxt
@@ -124,6 +124,14 @@ tf_module {
     name: "ctc_label_dense_to_sparse"
     argspec: "args=[\'labels\', \'label_lengths\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "cumprod"
+    argspec: "args=[\'x\', \'axis\'], varargs=None, keywords=None, defaults=[\'0\'], "
+  }
+  member_method {
+    name: "cumsum"
+    argspec: "args=[\'x\', \'axis\'], varargs=None, keywords=None, defaults=[\'0\'], "
+  }
   member_method {
     name: "dot"
     argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.backend.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.backend.pbtxt
index d200d3d26d..01a3fde258 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.backend.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.backend.pbtxt
@@ -124,6 +124,14 @@ tf_module {
     name: "ctc_label_dense_to_sparse"
     argspec: "args=[\'labels\', \'label_lengths\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "cumprod"
+    argspec: "args=[\'x\', \'axis\'], varargs=None, keywords=None, defaults=[\'0\'], "
+  }
+  member_method {
+    name: "cumsum"
+    argspec: "args=[\'x\', \'axis\'], varargs=None, keywords=None, defaults=[\'0\'], "
+  }
   member_method {
     name: "dot"
     argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-- 
GitLab


From d411188eb53b3a9f2eab6d42f88f2ad780e91f38 Mon Sep 17 00:00:00 2001
From: Ruoxin Sang <rxsang@google.com>
Date: Wed, 23 Jan 2019 13:19:19 -0800
Subject: [PATCH 1135/2345] Don't check types when comparing structure with
 inputs and maximum_shapes. So lists and tuples are regarded as the same.

PiperOrigin-RevId: 230589066
---
 tensorflow/contrib/tpu/python/tpu/tpu.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu.py b/tensorflow/contrib/tpu/python/tpu/tpu.py
index 4945e5be07..ebbccea02c 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu.py
@@ -769,7 +769,7 @@ def split_compile_and_replicate(computation,
           "Dynamic input shapes are not supported with infeed queues")
 
     # Make sure maximum_shapes has the same structure as inputs.
-    nest.assert_same_structure(inputs[0], maximum_shapes)
+    nest.assert_same_structure(inputs[0], maximum_shapes, check_types=False)
 
     # Flatten padded shapes.
     flat_maximum_shapes = nest.flatten(maximum_shapes)
-- 
GitLab


From 9883edb26c9d7293dac2c2977d6606f3a0cb3e5d Mon Sep 17 00:00:00 2001
From: Mahmoud Abuzaina <mahmoud.abuzaina@intel.com>
Date: Wed, 23 Jan 2019 13:33:40 -0800
Subject: [PATCH 1136/2345] Adding support for MKL Quantized Concat

---
 tensorflow/core/BUILD                         |  14 +-
 tensorflow/core/api_def/excluded_ops.cc       |   6 +-
 tensorflow/core/framework/common_shape_fns.cc |  20 +-
 tensorflow/core/framework/common_shape_fns.h  |   2 +
 tensorflow/core/kernels/BUILD                 |  24 ++
 tensorflow/core/kernels/mkl_concat_op.cc      | 103 ++++++--
 .../kernels/mkl_quantized_concat_op_test.cc   | 229 ++++++++++++++++++
 .../core/kernels/quantized_concat_op.cc       |  25 +-
 tensorflow/core/ops/mkl_array_ops.cc          |  92 +++++++
 9 files changed, 480 insertions(+), 35 deletions(-)
 create mode 100644 tensorflow/core/kernels/mkl_quantized_concat_op_test.cc
 create mode 100644 tensorflow/core/ops/mkl_array_ops.cc

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index ceec270f99..2966a8fcdd 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1138,6 +1138,13 @@ tf_gen_op_libs(
     deps = [":protos_all_cc"],
 )
 
+tf_gen_op_libs(
+    op_lib_names = [
+        "mkl_array_ops",
+    ],
+    deps = [":protos_all_cc"],
+)
+
 tf_gen_op_libs(
     op_lib_names = [
         "audio_ops",
@@ -1277,7 +1284,10 @@ cc_library(
         ":training_ops_op_lib",
         ":user_ops_op_lib",
         ":word2vec_ops",
-    ] + if_mkl([":mkl_nn_ops_op_lib"]) + tf_additional_cloud_op_deps(),
+    ] + if_mkl([
+        ":mkl_array_ops_op_lib",
+        ":mkl_nn_ops_op_lib",
+    ]) + tf_additional_cloud_op_deps(),
     alwayslink = 1,
 )
 
@@ -4472,7 +4482,7 @@ tf_cc_test(
         "//tensorflow/cc:scope",
         "//tensorflow/core/kernels:cwise_op",
         "//third_party/eigen3",
-    ],
+    ] + if_mkl([":mkl_array_ops_op_lib"]),
 )
 
 tf_cc_test(
diff --git a/tensorflow/core/api_def/excluded_ops.cc b/tensorflow/core/api_def/excluded_ops.cc
index 02026e94ab..65d2102ac8 100644
--- a/tensorflow/core/api_def/excluded_ops.cc
+++ b/tensorflow/core/api_def/excluded_ops.cc
@@ -24,9 +24,9 @@ const std::unordered_set<std::string>* GetExcludedOps() {
            "GcsConfigureBlockCache", "GcsConfigureCredentials",
 #ifdef INTEL_MKL
            // QuantizedFusedOps for Intel CPU
-           "QuantizedConv2DAndRequantize", "QuantizedConv2DWithBias",
-           "QuantizedConv2DWithBiasAndRequantize", "QuantizedConv2DAndRelu",
-           "QuantizedConv2DAndReluAndRequantize",
+           "QuantizedConcatV2", "QuantizedConv2DAndRequantize",
+           "QuantizedConv2DWithBias", "QuantizedConv2DWithBiasAndRequantize",
+           "QuantizedConv2DAndRelu", "QuantizedConv2DAndReluAndRequantize",
            "QuantizedConv2DWithBiasAndRelu",
            "QuantizedConv2DWithBiasAndReluAndRequantize",
            "QuantizedConv2DWithBiasSumAndRelu",
diff --git a/tensorflow/core/framework/common_shape_fns.cc b/tensorflow/core/framework/common_shape_fns.cc
index 876ac188ac..b168a7207f 100644
--- a/tensorflow/core/framework/common_shape_fns.cc
+++ b/tensorflow/core/framework/common_shape_fns.cc
@@ -57,9 +57,8 @@ Status GetWindowedOutputSizeVerboseV2(int64 input_size, int64 filter_size,
   if (*output_size < 0) {
     return errors::InvalidArgument(
         "Computed output size would be negative: ", *output_size,
-        " [input_size: ", input_size,
-        ", effective_filter_size: ", effective_filter_size,
-        ", stride: ", stride, "]");
+        " [input_size: ", input_size, ", effective_filter_size: ",
+        effective_filter_size, ", stride: ", stride, "]");
   }
   return Status::OK();
 }
@@ -1299,6 +1298,12 @@ Status ConcatV2Shape(InferenceContext* c) {
                            c->num_inputs() - 1 /* dim_index */);
 }
 
+Status QuantizedConcatV2Shape(InferenceContext* c, int num_inputs_to_concat) {
+  return ConcatShapeHelper(c, 0 /* start_value_index */,
+                           num_inputs_to_concat /* end_value_index */,
+                           num_inputs_to_concat /* dim_index */);
+}
+
 Status BroadcastBinaryOpOutputShapeFnHelper(InferenceContext* c,
                                             ShapeHandle shape_x,
                                             ShapeHandle shape_y,
@@ -1562,11 +1567,10 @@ Status ScatterNdUpdateShape(InferenceContext* c) {
       Status s = c->Merge(prefix_indices, prefix_updates, &unused);
       if (!s.ok()) {
         return errors::InvalidArgument(
-            "The outer ", num_outer_dims,
-            " dimensions of indices.shape=", c->DebugString(indices_shape),
-            " must match the outer ", num_outer_dims,
-            " dimensions of updates.shape=", c->DebugString(updates_shape),
-            ": ", s.error_message());
+            "The outer ", num_outer_dims, " dimensions of indices.shape=",
+            c->DebugString(indices_shape), " must match the outer ",
+            num_outer_dims, " dimensions of updates.shape=",
+            c->DebugString(updates_shape), ": ", s.error_message());
       }
 
       ShapeHandle input_suffix;
diff --git a/tensorflow/core/framework/common_shape_fns.h b/tensorflow/core/framework/common_shape_fns.h
index 14b9688bdc..d421844ee6 100644
--- a/tensorflow/core/framework/common_shape_fns.h
+++ b/tensorflow/core/framework/common_shape_fns.h
@@ -279,6 +279,8 @@ Status ConcatShape(shape_inference::InferenceContext* c,
 // Shape function for concat operations.
 Status ConcatV2Shape(shape_inference::InferenceContext* c);
 
+Status QuantizedConcatV2Shape(InferenceContext* c, int num_inputs_to_concat);
+
 // Shape function for binary operators that broadcast their inputs
 // and with output to output_index.
 // Note: out cannot be NULL.
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index ce6504e0b6..85a5c990d4 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -6570,6 +6570,30 @@ tf_cc_test(
     ],
 )
 
+tf_cc_test_mkl(
+    name = "mkl_quantized_concat_op_test",
+    size = "small",
+    srcs = ["mkl_quantized_concat_op_test.cc"],
+    deps = [
+        ":mkl_concat_op",
+        ":ops_testutil",
+        ":ops_util",
+        ":quantization_utils",
+        ":quantized_ops",
+        "//tensorflow/core:array_ops_op_lib",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:mkl_array_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+    ],
+)
+
 tf_cc_test(
     name = "quantized_batch_norm_op_test",
     size = "small",
diff --git a/tensorflow/core/kernels/mkl_concat_op.cc b/tensorflow/core/kernels/mkl_concat_op.cc
index 3a5c87485c..a955a90990 100644
--- a/tensorflow/core/kernels/mkl_concat_op.cc
+++ b/tensorflow/core/kernels/mkl_concat_op.cc
@@ -17,7 +17,6 @@ limitations under the License.
 #include <vector>
 
 #include "mkldnn.hpp"
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
@@ -25,12 +24,15 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/kernels/concat_lib.h"
+#include "tensorflow/core/kernels/concat_lib_cpu.h"
+#include "tensorflow/core/kernels/quantization_utils.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
+#include "tensorflow/core/util/mkl_util.h"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
 using mkldnn::concat;
 using mkldnn::stream;
-#include "tensorflow/core/util/mkl_util.h"
 
 namespace tensorflow {
 typedef Eigen::ThreadPoolDevice CPUDevice;
@@ -78,9 +80,8 @@ class EigenConcatBaseOp : public OpKernel {
     const TensorShape& input_shape = input_shapes[0];
 
     int32 axis = concat_dim < 0 ? concat_dim + input_dims : concat_dim;
-    OP_REQUIRES(c,
-                (0 <= axis && axis < input_dims) ||
-                    (allow_legacy_scalars() && concat_dim == 0),
+    OP_REQUIRES(c, (0 <= axis && axis < input_dims) ||
+                       (allow_legacy_scalars() && concat_dim == 0),
                 errors::InvalidArgument(
                     "ConcatOp : Expected concatenating dimensions in the range "
                     "[",
@@ -102,13 +103,12 @@ class EigenConcatBaseOp : public OpKernel {
       const auto in = values[i];
       const bool in_is_scalar = IsLegacyScalar(input_shapes[i]);
       OP_REQUIRES(
-          c,
-          (input_shapes[i].dims() == input_dims) ||
-              (input_is_scalar && in_is_scalar),
+          c, (input_shapes[i].dims() == input_dims) ||
+                 (input_is_scalar && in_is_scalar),
           errors::InvalidArgument(
               "ConcatOp : Ranks of all input tensors should match: shape[0] = ",
-              input_shape.DebugString(), " vs. shape[", i,
-              "] = ", input_shapes[i].DebugString()));
+              input_shape.DebugString(), " vs. shape[", i, "] = ",
+              input_shapes[i].DebugString()));
       if (in.NumElements() > 0) {
         int64 inputs_flat_dim1 = in.NumElements() / inputs_flat_dim0;
         inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
@@ -226,9 +226,53 @@ class MklConcatOp : public OpKernel {
       // format and avoid calling eigen version.
       if (!are_all_tf_inputs && !are_all_mkl_inputs) invoke_eigen = true;
 
+      OpInputList input_mins, input_maxes;
+      if (std::is_same<T, qint8>::value || std::is_same<T, quint8>::value) {
+        // MKL DNN concat does not support input tensors that have different
+        // ranges, check if the ranges of the all input tensors are the same
+        // if not, forward it to Eigen implementation.
+
+        OP_REQUIRES_OK(context, context->input_list("input_mins", &input_mins));
+        OP_REQUIRES(context, (input_mins.size() == N),
+                    errors::InvalidArgument(
+                        "QuantizedConcatOp : Expected mins input list length ",
+                        input_mins.size(), " to equal values length ", N));
+
+        OP_REQUIRES_OK(context,
+                       context->input_list("input_maxes", &input_maxes));
+        OP_REQUIRES(context, (input_maxes.size() == N),
+                    errors::InvalidArgument(
+                        "QuantizedConcatOp : Expected maxes input list length ",
+                        input_maxes.size(), " to equal values length ", N));
+        float input_min = input_mins[0].flat<float>()(0);
+        float input_max = input_maxes[0].flat<float>()(0);
+        const float eps = 1.0e-6;
+        for (int i = 1; i < N; i++) {
+          float min = input_mins[i].flat<float>()(0);
+          float max = input_maxes[i].flat<float>()(0);
+
+          if (fabs(input_min - min) > eps || fabs(input_max - max) > eps) {
+            invoke_eigen = true;
+            break;
+          }
+        }
+      }
+
       // Call Eigen library
       if (invoke_eigen) {
-        CallEigenVersion(context, input_tensors, mkl_input_shapes);
+        if (std::is_same<T, qint8>::value || std::is_same<T, quint8>::value) {
+          // MKL DNN quantized concat does not support input tensors with
+          // different ranges.
+          // TODO (mabuzain): Add quantized version of CallEigen() to support
+          // this case.
+          OP_REQUIRES(context, false,
+                      errors::Unimplemented("MKL DNN quantized concat does not "
+                                            "support input tensors that have "
+                                            "different ranges"));
+        } else {
+          CallEigenVersion(context, input_tensors, mkl_input_shapes);
+        }
+
         return;
       }
 
@@ -374,10 +418,27 @@ class MklConcatOp : public OpKernel {
       std::vector<primitive> net;
       net.push_back(concat_op);
       stream(stream::kind::eager).submit(net).wait();
+
+      // For quantized concat, min and max outputs are also computed.
+      if (std::is_same<T, qint8>::value || std::is_same<T, quint8>::value) {
+        Tensor* output_min = nullptr;
+        Tensor* output_max = nullptr;
+        MklDnnShape output_min_mkl_shape, output_max_mkl_shape;
+        output_min_mkl_shape.SetMklTensor(false);
+        output_max_mkl_shape.SetMklTensor(false);
+        AllocateOutputSetMklShape(context, 1, &output_min, {},
+                                  output_min_mkl_shape);
+        AllocateOutputSetMklShape(context, 2, &output_max, {},
+                                  output_max_mkl_shape);
+        // All input tensors should have the same range, just use the
+        // first one
+        output_min->flat<float>()(0) = input_mins[0].flat<float>()(0);
+        output_max->flat<float>()(0) = input_maxes[0].flat<float>()(0);
+      }
     } catch (mkldnn::error& e) {
-      string error_msg = "Status: " + std::to_string(e.status) +
-                         ", message: " + string(e.message) + ", in file " +
-                         string(__FILE__) + ":" + std::to_string(__LINE__);
+      string error_msg = "Status: " + std::to_string(e.status) + ", message: " +
+                         string(e.message) + ", in file " + string(__FILE__) +
+                         ":" + std::to_string(__LINE__);
       OP_REQUIRES_OK(
           context,
           errors::Aborted("Operation received an exception:", error_msg));
@@ -490,6 +551,20 @@ class MklConcatOp : public OpKernel {
 
 TF_CALL_float(REGISTER_MKL_CPU);
 
+REGISTER_KERNEL_BUILDER(Name("_MklQuantizedConcatV2")
+                            .Device(DEVICE_CPU)
+                            .TypeConstraint<quint8>("T")
+                            .HostMemory("axis")
+                            .Label(mkl_op_registry::kMklQuantizedOpLabel),
+                        MklConcatOp<CPUDevice, quint8, NAME_IS_AXIS>)
+
+REGISTER_KERNEL_BUILDER(Name("_MklQuantizedConcatV2")
+                            .Device(DEVICE_CPU)
+                            .TypeConstraint<qint8>("T")
+                            .HostMemory("axis")
+                            .Label(mkl_op_registry::kMklQuantizedOpLabel),
+                        MklConcatOp<CPUDevice, qint8, NAME_IS_AXIS>)
+
 #undef REGISTER_CONCAT_MKL
 }  // namespace tensorflow
 
diff --git a/tensorflow/core/kernels/mkl_quantized_concat_op_test.cc b/tensorflow/core/kernels/mkl_quantized_concat_op_test.cc
new file mode 100644
index 0000000000..f9098f960a
--- /dev/null
+++ b/tensorflow/core/kernels/mkl_quantized_concat_op_test.cc
@@ -0,0 +1,229 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#define EIGEN_USE_THREADS
+
+#include <functional>
+#include <memory>
+#include <vector>
+
+#include "tensorflow/core/common_runtime/kernel_benchmark_testlib.h"
+#include "tensorflow/core/framework/allocator.h"
+#include "tensorflow/core/framework/fake_input.h"
+#include "tensorflow/core/framework/node_def_builder.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_testutil.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/framework/types.pb.h"
+#include "tensorflow/core/graph/node_builder.h"
+#include "tensorflow/core/kernels/ops_testutil.h"
+#include "tensorflow/core/kernels/ops_util.h"
+#include "tensorflow/core/kernels/quantization_utils.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/platform/test_benchmark.h"
+
+namespace tensorflow {
+
+using test::graph::Constant;
+
+// Helper class for converting MKL tesnors to TF tensors and comparing to
+// expected values
+
+static const uint8 dummy_tensor[] = {0, 0, 0, 0, 0, 0, 0, 0};
+static const TensorShape dummy_shape({8});
+
+class ConvMklToTF : public OpsTestBase {
+ public:
+  template <typename T>
+  void ConvertMKL2TF(DataType dtype, const Tensor& first, const Tensor& second,
+                     Tensor& output) {
+    // Create an MKL to TF conversion node and execute it
+    TF_EXPECT_OK(NodeDefBuilder("mkl_to_tf_op", "_MklToTf")
+                     .Input(FakeInput(dtype))     // Input
+                     .Input(FakeInput(DT_UINT8))  // Mkl second tensor
+                     .Attr("T", dtype)
+                     .Attr("_kernel", "MklOp")
+                     .Finalize(node_def()));
+    TF_EXPECT_OK(InitOp());
+    AddInputFromArray<T>(first.shape(), first.flat<T>());
+    AddInputFromArray<uint8>(second.shape(), second.flat<uint8>());
+    TF_ASSERT_OK(RunOpKernel());
+
+    output = *GetOutput(0);
+  }
+  void TestBody(){};
+};
+
+class QuantizedConcatTest : public OpsTestBase {
+ protected:
+  QuantizedConcatTest() {}
+
+  void TestSmall8Bit(float first_min, float first_max, float second_min,
+                     float second_max);
+  void TestSecondDim8Bit(float first_min, float first_max, float second_min,
+                         float second_max);
+};
+
+TEST_F(QuantizedConcatTest, Small8BitSameRange) {
+  // Range for both is the same, so impl can use memcpy.
+  TestSmall8Bit(0.0f, 255.0f, 0.0f, 255.0f);
+}
+
+void QuantizedConcatTest::TestSmall8Bit(float first_min, float first_max,
+                                        float second_min, float second_max) {
+  TF_ASSERT_OK(NodeDefBuilder("quantized_concat_op", "_MklQuantizedConcatV2")
+                   .Input(FakeInput(2, DT_QUINT8))
+                   .Input(FakeInput(DT_INT32))
+                   .Input(FakeInput(2, DT_FLOAT))
+                   .Input(FakeInput(2, DT_FLOAT))
+                   .Input(FakeInput(2, DT_UINT8))  // MKl second tensor
+                   .Input(FakeInput(DT_UINT8))     // MKl second tensor
+                   .Input(FakeInput(2, DT_UINT8))  // MKl second tensor
+                   .Input(FakeInput(2, DT_UINT8))  // MKl second tensor
+                   .Attr("N", 2)
+                   .Attr("T", DataTypeToEnum<quint8>::v())
+                   .Attr("Tidx", DT_INT32)
+                   .Attr("_kernel", "QuantizedMklOp")
+                   .Finalize(node_def()));
+  TF_ASSERT_OK(InitOp());
+  const int first_batch = 2;
+  const int first_height = 2;
+  const int first_width = 3;
+  const int first_depth = 1;
+  Tensor first_float(DT_FLOAT,
+                     {first_batch, first_height, first_width, first_depth});
+  test::FillValues<float>(&first_float,
+                          {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
+  Tensor first_quantized =
+      FloatTensorToQuantized<quint8>(first_float, first_min, first_max);
+
+  const int second_batch = 2;
+  const int second_height = 2;
+  const int second_width = 3;
+  const int second_depth = 1;
+  Tensor second_float(
+      DT_FLOAT, {second_batch, second_height, second_width, second_depth});
+  test::FillValues<float>(&second_float,
+                          {13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24});
+  Tensor second_quantized =
+      FloatTensorToQuantized<quint8>(second_float, second_min, second_max);
+
+  const int expected_batch = first_batch + second_batch;
+  Tensor expected_float(
+      DT_FLOAT, {expected_batch, first_height, first_width, first_depth});
+  test::FillValues<float>(&expected_float,
+                          {1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12,
+                           13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24});
+
+  AddInputFromArray<quint8>(first_quantized.shape(),
+                            first_quantized.flat<quint8>());
+  AddInputFromArray<quint8>(second_quantized.shape(),
+                            second_quantized.flat<quint8>());
+  AddInputFromArray<int32>(TensorShape({}), {0});
+  AddInputFromArray<float>(TensorShape({}), {first_min});
+  AddInputFromArray<float>(TensorShape({}), {second_min});
+  AddInputFromArray<float>(TensorShape({}), {first_max});
+  AddInputFromArray<float>(TensorShape({}), {second_max});
+  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+  TF_ASSERT_OK(RunOpKernel());
+  const Tensor& output_quantized = *GetOutput(0);
+  const float output_min = GetOutput(1)->flat<float>()(0);
+  const float output_max = GetOutput(2)->flat<float>()(0);
+  Tensor output_float =
+      QuantizedTensorToFloat<quint8>(output_quantized, output_min, output_max);
+  test::ExpectTensorNear<float>(expected_float, output_float, 0.2);
+}
+
+TEST_F(QuantizedConcatTest, SecondDim8BitSameRange) {
+  TestSecondDim8Bit(-10.0f, 150.0f, -10.0f, 150.0f);
+}
+
+void QuantizedConcatTest::TestSecondDim8Bit(float first_min, float first_max,
+                                            float second_min,
+                                            float second_max) {
+  TF_ASSERT_OK(NodeDefBuilder("quantized_concat_op", "_MklQuantizedConcatV2")
+                   .Input(FakeInput(2, DT_QUINT8))
+                   .Input(FakeInput(DT_INT32))
+                   .Input(FakeInput(2, DT_FLOAT))
+                   .Input(FakeInput(2, DT_FLOAT))
+                   .Input(FakeInput(2, DT_UINT8))  // MKl second tensor
+                   .Input(FakeInput(DT_UINT8))     // MKl second tensor
+                   .Input(FakeInput(2, DT_UINT8))  // MKl second tensor
+                   .Input(FakeInput(2, DT_UINT8))  // MKl second tensor
+                   .Attr("N", 2)
+                   .Attr("T", DataTypeToEnum<quint8>::v())
+                   .Attr("Tidx", DT_INT32)
+                   .Attr("_kernel", "QuantizedMklOp")
+                   .Finalize(node_def()));
+  TF_ASSERT_OK(InitOp());
+  const int first_batch = 2;
+  const int first_height = 2;
+  const int first_width = 3;
+  const int first_depth = 1;
+  Tensor first_float(DT_FLOAT,
+                     {first_batch, first_height, first_width, first_depth});
+  test::FillValues<float>(&first_float,
+                          {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
+  Tensor first_quantized =
+      FloatTensorToQuantized<quint8>(first_float, first_min, first_max);
+
+  const int second_batch = 2;
+  const int second_height = 2;
+  const int second_width = 3;
+  const int second_depth = 1;
+
+  Tensor second_float(
+      DT_FLOAT, {second_batch, second_height, second_width, second_depth});
+  test::FillValues<float>(&second_float,
+                          {13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24});
+  Tensor second_quantized =
+      FloatTensorToQuantized<quint8>(second_float, second_min, second_max);
+
+  const int expected_height = first_height + second_height;
+  Tensor expected_float(
+      DT_FLOAT, {first_batch, expected_height, first_width, first_depth});
+  test::FillValues<float>(&expected_float,
+                          {1, 2, 3, 4,  5,  6,  13, 14, 15, 16, 17, 18,
+                           7, 8, 9, 10, 11, 12, 19, 20, 21, 22, 23, 24});
+
+  AddInputFromArray<quint8>(first_quantized.shape(),
+                            first_quantized.flat<quint8>());
+  AddInputFromArray<quint8>(second_quantized.shape(),
+                            second_quantized.flat<quint8>());
+  AddInputFromArray<int32>(TensorShape({}), {1});
+  AddInputFromArray<float>(TensorShape({}), {first_min});
+  AddInputFromArray<float>(TensorShape({}), {second_min});
+  AddInputFromArray<float>(TensorShape({}), {first_max});
+  AddInputFromArray<float>(TensorShape({}), {second_max});
+  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+  TF_ASSERT_OK(RunOpKernel());
+  const Tensor& output_quantized = *GetOutput(0);
+  const float output_min = GetOutput(1)->flat<float>()(0);
+  const float output_max = GetOutput(2)->flat<float>()(0);
+  Tensor output_float =
+      QuantizedTensorToFloat<quint8>(output_quantized, output_min, output_max);
+  test::ExpectTensorNear<float>(expected_float, output_float, 1.0);
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/quantized_concat_op.cc b/tensorflow/core/kernels/quantized_concat_op.cc
index b03ac8e87d..1715849011 100644
--- a/tensorflow/core/kernels/quantized_concat_op.cc
+++ b/tensorflow/core/kernels/quantized_concat_op.cc
@@ -17,13 +17,13 @@ limitations under the License.
 
 #include <vector>
 
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/kernels/concat_lib_cpu.h"
 #include "tensorflow/core/kernels/quantization_utils.h"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
 namespace tensorflow {
 
@@ -135,8 +135,8 @@ class QuantizedConcatOp : public OpKernel {
           context, in.dims() == input_dims || (input_is_scalar && in_is_scalar),
           errors::InvalidArgument(
               "ConcatOp : Ranks of all input tensors should match: shape[0] = ",
-              input_shape.DebugString(), " vs. shape[", i,
-              "] = ", in.shape().DebugString()));
+              input_shape.DebugString(), " vs. shape[", i, "] = ",
+              in.shape().DebugString()));
       for (int j = 0; j < input_dims; ++j) {
         if (j == concat_dim) {
           continue;
@@ -145,8 +145,8 @@ class QuantizedConcatOp : public OpKernel {
             context, in.dim_size(j) == input_shape.dim_size(j),
             errors::InvalidArgument(
                 "ConcatOp : Dimensions of inputs should match: shape[0] = ",
-                input_shape.DebugString(), " vs. shape[", i,
-                "] = ", in.shape().DebugString()));
+                input_shape.DebugString(), " vs. shape[", i, "] = ",
+                in.shape().DebugString()));
       }
       if (in.NumElements() > 0) {
         int64 inputs_flat_dim1 = in.NumElements() / inputs_flat_dim0;
@@ -184,9 +184,8 @@ class QuantizedConcatOp : public OpKernel {
     const int input_dims = values[0].dims();
     const TensorShape& input_shape = values[0].shape();
     OP_REQUIRES(
-        context,
-        (0 <= concat_dim && concat_dim < input_dims) ||
-            (allow_legacy_scalars() && concat_dim == 0),
+        context, (0 <= concat_dim && concat_dim < input_dims) ||
+                     (allow_legacy_scalars() && concat_dim == 0),
         errors::InvalidArgument(
             "ConcatOp : Expected concatenating dimensions in the range [", 0,
             ", ", input_dims, "), but got ", concat_dim));
@@ -246,4 +245,14 @@ REGISTER_QUANTIZED_CONCAT(qint32);
 
 #undef REGISTER_QUANTIZED_CONCAT
 
+#define REGISTER_QUANTIZED_CONCATV2(type)                \
+  REGISTER_KERNEL_BUILDER(Name("QuantizedConcatV2")      \
+                              .Device(DEVICE_CPU)        \
+                              .TypeConstraint<type>("T") \
+                              .HostMemory("axis"),       \
+                          QuantizedConcatOp<type>)
+
+REGISTER_QUANTIZED_CONCATV2(quint8);
+REGISTER_QUANTIZED_CONCATV2(qint32);
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/ops/mkl_array_ops.cc b/tensorflow/core/ops/mkl_array_ops.cc
new file mode 100644
index 0000000000..ad15d0bf88
--- /dev/null
+++ b/tensorflow/core/ops/mkl_array_ops.cc
@@ -0,0 +1,92 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifdef INTEL_MKL
+
+// This file contains the registration of MKL-DNN array ops.
+
+#include "tensorflow/core/framework/common_shape_fns.h"
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/shape_inference.h"
+#include "tensorflow/core/framework/tensor.pb.h"
+#include "tensorflow/core/util/mirror_pad_mode.h"
+#include "tensorflow/core/util/padding.h"
+#include "tensorflow/core/util/strided_slice_op.h"
+#include "tensorflow/core/util/tensor_format.h"
+
+namespace tensorflow {
+
+using shape_inference::DimensionHandle;
+using shape_inference::InferenceContext;
+using shape_inference::ShapeHandle;
+using shape_inference::UnchangedShape;
+
+// Adding QuantizedConcatV2 which is similar to the existing QuantizedConcat
+// op to be able to replace it by _MklQuantizedConcatV2 in the graph rewrite.
+REGISTER_OP("QuantizedConcatV2")
+    .Input("values: N * T")
+    .Input("axis: Tidx")
+    .Input("input_mins: N * float32")
+    .Input("input_maxes: N * float32")
+    .Output("output: T")
+    .Output("output_min: float")
+    .Output("output_max: float")
+    .Attr("N: int >= 2")
+    .Attr("T: type")
+    .Attr("Tidx: {int32, int64} = DT_INT32")
+    .SetShapeFn([](InferenceContext* c) {
+      const int n = (c->num_inputs() - 1) / 3;
+      TF_RETURN_IF_ERROR(shape_inference::QuantizedConcatV2Shape(c, n));
+      ShapeHandle unused;
+      for (int i = n + 1; i < c->num_inputs(); ++i) {
+        TF_RETURN_IF_ERROR(c->WithRank(c->input(i), 0, &unused));
+      }
+      c->set_output(1, c->Scalar());
+      c->set_output(2, c->Scalar());
+      return Status::OK();
+    });
+
+REGISTER_OP("_MklQuantizedConcatV2")
+    .Input("values: N * T")
+    .Input("axis: Tidx")
+    .Input("input_mins:  N * float32")
+    .Input("input_maxes: N * float32")
+    .Input("mkl_values: N * uint8")
+    .Input("mkl_axis: uint8")
+    .Input("mkl_input_mins:  N * uint8")
+    .Input("mkl_input_maxes: N * uint8")
+    .Output("output: T")
+    .Output("output_min: float")
+    .Output("output_max: float")
+    .Output("mkl_output: uint8")
+    .Output("mkl_output_min: uint8")
+    .Output("mkl_output_max: uint8")
+    .Attr("N: int >= 2")
+    .Attr("T: type")
+    .Attr("Tidx: {int32, int64} = DT_INT32")
+    .SetShapeFn([](InferenceContext* c) {
+      const int n = (c->num_inputs() / 2 - 1) / 3;
+      TF_RETURN_IF_ERROR(shape_inference::QuantizedConcatV2Shape(c, n));
+      ShapeHandle unused;
+      for (int i = n + 1; i < c->num_inputs() / 2; ++i) {
+        TF_RETURN_IF_ERROR(c->WithRank(c->input(i), 0, &unused));
+      }
+      c->set_output(1, c->Scalar());
+      c->set_output(2, c->Scalar());
+      return Status::OK();
+    });
+}
+
+#endif
\ No newline at end of file
-- 
GitLab


From 2f2503d5d0c27104848fcee418ca9f97607cc39c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 23 Jan 2019 13:38:59 -0800
Subject: [PATCH 1137/2345] Add templates and interfaces for creating lookup
 tables

PiperOrigin-RevId: 230593171
---
 tensorflow/core/kernels/lookup_tables/BUILD   |  43 ++++
 .../lookup_tables/lookup_table_interface.h    | 150 +++++++++++
 .../kernels/lookup_tables/table_op_utils.h    | 239 ++++++++++++++++++
 .../lookup_tables/table_resource_utils.h      | 101 ++++++++
 4 files changed, 533 insertions(+)
 create mode 100644 tensorflow/core/kernels/lookup_tables/BUILD
 create mode 100644 tensorflow/core/kernels/lookup_tables/lookup_table_interface.h
 create mode 100644 tensorflow/core/kernels/lookup_tables/table_op_utils.h
 create mode 100644 tensorflow/core/kernels/lookup_tables/table_resource_utils.h

diff --git a/tensorflow/core/kernels/lookup_tables/BUILD b/tensorflow/core/kernels/lookup_tables/BUILD
new file mode 100644
index 0000000000..53074f0819
--- /dev/null
+++ b/tensorflow/core/kernels/lookup_tables/BUILD
@@ -0,0 +1,43 @@
+# Description:
+#   OpKernels and resource templates for lookup tables.
+
+package(
+    default_visibility = [
+        "//tensorflow:__subpackages__",
+        "//tensorflow:internal",
+    ],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+cc_library(
+    name = "lookup_table_interface",
+    hdrs = ["lookup_table_interface.h"],
+    deps = [
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "@com_google_absl//absl/types:span",
+    ],
+)
+
+cc_library(
+    name = "table_resource_utils",
+    hdrs = ["table_resource_utils.h"],
+    deps = [
+        ":lookup_table_interface",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+    ],
+)
+
+cc_library(
+    name = "table_op_utils",
+    hdrs = ["table_op_utils.h"],
+    deps = [
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "@com_google_absl//absl/base:core_headers",
+    ],
+)
diff --git a/tensorflow/core/kernels/lookup_tables/lookup_table_interface.h b/tensorflow/core/kernels/lookup_tables/lookup_table_interface.h
new file mode 100644
index 0000000000..62121de618
--- /dev/null
+++ b/tensorflow/core/kernels/lookup_tables/lookup_table_interface.h
@@ -0,0 +1,150 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_KERNELS_LOOKUP_TABLES_LOOKUP_TABLE_INTERFACE_H_
+#define TENSORFLOW_CORE_KERNELS_LOOKUP_TABLES_LOOKUP_TABLE_INTERFACE_H_
+
+#include <cstddef>
+#include <string>
+
+#include "absl/types/span.h"
+#include "tensorflow/core/framework/resource_mgr.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/platform/mutex.h"
+
+namespace tensorflow {
+namespace tables {
+
+// Interface for key-value pair lookups with support for heterogeneous keys.
+// One of the key high performance abstractions introduced here is the
+// hierarchy of a primary and secondary (or fallback) table.
+// The primary table can be considered the main data structure into which
+// key-value pairs may be inserted.
+// When a key is requested, LookupTableInterface::LookupKey semantics expect
+// that the primary table be queried first. The secondary table is queried
+// when the primary table lookup fails (eg. does not contain a requested key).
+// Lookups in the secondary table must always return well defined values in
+// this case.
+// LookupTableInterface features batch and serial table insert/lookup
+// methods. Serial methods have corresponding status methods which say
+// whether the table supports serial inserts/lookups which are guaranteed to
+// succeed.
+// For example, an implementation may only support a one-time batch population
+// after which further insertions are undefined. Another may not permit lookups
+// before its size is non zero.
+// In each of these cases, the success or failure of inserts/lookups is not
+// dependent on the parameter values. On the other hand, once the table is in
+// ready state, LookupTableInterface semantics guarantee that all lookups and
+// inserts will succeed.
+// Implementations must guarantee thread-safety when GetMutex is used to
+// synchronize method access.
+template <typename HeterogeneousKeyType, typename ValueType>
+class LookupTableInterface : public ResourceBase {
+ public:
+  using heterogeneous_key_type = HeterogeneousKeyType;
+  using value_type = ValueType;
+  using key_type = heterogeneous_key_type;
+
+  // Return value should be used to synchronize read/write access to
+  // all public methods. If null, no synchronization is needed.
+  virtual mutex* GetMutex() const = 0;
+
+  // Insert the KV pair into the underlying table. If a key equivalent to key
+  // already exists in the underlying table, its corresponding value is
+  // overridden. Returns true only if the key was inserted for the first time.
+  // Undefined if TableInsertStatus() != OK.
+  virtual bool InsertOrAssign(const HeterogeneousKeyType& key,
+                              const ValueType& value) = 0;
+
+  // Returns OK if it is safe to call InsertOrAssign.
+  // Once OK is returned, it is safe to call InsertOrAssign for the rest of the
+  // program.
+  virtual Status TableInsertStatus() const TF_MUST_USE_RESULT = 0;
+
+  // Stores each KV pair {keys[i], values[i]} in the underlying map, overriding
+  // pre-existing pairs with equivalent keys.
+  // keys and values should have the same size.
+  virtual Status BatchInsertOrAssign(
+      absl::Span<const HeterogeneousKeyType> keys,
+      absl::Span<const ValueType> values) = 0;
+
+  // Prefetch key_to_find into implementation defined data caches.
+  // Implementations are free to leave this a no-op.
+  // Undefined if TableLookupStatus() != OK.
+  virtual void PrefetchKey(const HeterogeneousKeyType& key_to_find) const {}
+
+  // Returns true if and only if the primary table contains key_to_find.
+  // What constitutes the primary table is implementation defined.
+  // Undefined if TableLookupStatus() != OK.
+  virtual bool ContainsKey(const HeterogeneousKeyType& key_to_find) const = 0;
+
+  // Lookup the value for key_to_find in the primary. If this lookup fails
+  // (for eg. the key does not exist), the value in the secondary table is
+  // returned.
+  // Undefined if TableLookupStatus() != OK.
+  virtual ValueType LookupKey(
+      const HeterogeneousKeyType& key_to_find) const = 0;
+
+  // Lookup the value for key_to_find in the primary table and return it.
+  // If the lookup fails, the return value is undefined unless
+  // PrimaryTableDefaultValue() != NULL in which case
+  // *PrimaryTableDefaultValue() is returned.
+  // Undefined if TableLookupStatus() != OK.
+  virtual ValueType LookupKeyInPrimaryTable(
+      const HeterogeneousKeyType& key_to_find) const = 0;
+
+  // Lookup the value for key_to_find in the secondary table and return it.
+  // This value may not be well defined if key_to_find is in the primary table.
+  // Undefined if TableLookupStatus() != OK.
+  virtual ValueType LookupKeyInSecondaryTable(
+      const HeterogeneousKeyType& key_to_find) const = 0;
+
+  // Lookup the values for keys in the primary or secondary table and store
+  // them in values. prefetch_lookahead is used to prefetch the key at index
+  // i + prefetch_lookahead at the ith iteration of the implemented loop.
+  // Undefined if TableLookupStatus() != OK.
+  virtual Status BatchLookup(absl::Span<const HeterogeneousKeyType> keys,
+                             absl::Span<ValueType> values,
+                             int64 prefetch_lookahead) const = 0;
+
+  // Returns OK if it is safe to call PrefetchKey, ContainsKey, LookupKey,
+  // LookupKeyInPrimaryTable, and LookupKeyInSecondaryTable.
+  // Once OK is returned, it is safe to call these methods until the next
+  // non-const method of this class is called.
+  virtual Status TableLookupStatus() const TF_MUST_USE_RESULT = 0;
+
+  // If non-null value is returned, *value is guaranteed to not be a value in
+  // the primary table. LookupKeyInPrimaryTable returns *value for keys
+  // which are not in the primary table.
+  virtual const ValueType* PrimaryTableDefaultValue() const = 0;
+
+  // Returns the number of elements in the primary table.
+  // Undefined if SizeStatus() != OK.
+  virtual size_t size() const = 0;
+
+  // Returns OK if the return value of size() is well defined.
+  virtual Status SizeStatus() const TF_MUST_USE_RESULT = 0;
+
+  string DebugString() const override {
+    return strings::StrCat("A lookup table of size: ", size());
+  }
+
+  ~LookupTableInterface() override = default;
+};
+
+}  // namespace tables
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_KERNELS_LOOKUP_TABLES_LOOKUP_TABLE_INTERFACE_H_
diff --git a/tensorflow/core/kernels/lookup_tables/table_op_utils.h b/tensorflow/core/kernels/lookup_tables/table_op_utils.h
new file mode 100644
index 0000000000..fd850574b5
--- /dev/null
+++ b/tensorflow/core/kernels/lookup_tables/table_op_utils.h
@@ -0,0 +1,239 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_KERNELS_LOOKUP_TABLES_TABLE_OP_UTILS_H_
+#define TENSORFLOW_CORE_KERNELS_LOOKUP_TABLES_TABLE_OP_UTILS_H_
+
+#include <cstddef>
+#include <string>
+#include <type_traits>
+
+#include "absl/base/thread_annotations.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/resource_mgr.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/platform/macros.h"
+#include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace tensorflow {
+namespace tables {
+
+// Lookup or create resources of type Container but treat them as having type
+// ContainerBase for the purpose of dynamic dispatching.
+// Container must have constructor Container(OpKernelContext*, OpKernel*)
+template <class ContainerBase, class ContainerChild>
+class ResourceConstructionOp : public OpKernel {
+ public:
+  explicit ResourceConstructionOp(OpKernelConstruction* ctx)
+      : OpKernel(ctx), table_handle_set_(false) {
+    static_assert(std::is_base_of<ContainerBase, ContainerChild>::value,
+                  "ContainerChild is not derived from ContainerBase");
+    OP_REQUIRES_OK(
+        ctx, ctx->GetAttr("use_node_name_sharing", &use_node_name_sharing_));
+  }
+
+  void Compute(OpKernelContext* ctx) override {
+    mutex_lock l(mu_);
+
+    if (!table_handle_set_) {
+      OP_REQUIRES_OK(ctx, cinfo_.Init(ctx->resource_manager(), def(),
+                                      use_node_name_sharing_));
+    }
+
+    auto creator = [ctx,
+                    this](ContainerBase** ret) EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+      ContainerBase* container = new ContainerChild(ctx, this);
+      if (!ctx->status().ok()) {
+        container->Unref();
+        return ctx->status();
+      }
+      if (ctx->track_allocations()) {
+        ctx->record_persistent_memory_allocation(container->MemoryUsed());
+      }
+      *ret = container;
+      return Status::OK();
+    };
+
+    ContainerBase* container_base = nullptr;
+    OP_REQUIRES_OK(
+        ctx, cinfo_.resource_manager()->template LookupOrCreate<ContainerBase>(
+                 cinfo_.container(), cinfo_.name(), &container_base, creator));
+    core::ScopedUnref unref_me(container_base);
+
+    Tensor* handle;
+    OP_REQUIRES_OK(ctx, ctx->allocate_output(0, TensorShape({}), &handle));
+    handle->scalar<ResourceHandle>()() = MakeResourceHandle<ContainerBase>(
+        ctx, cinfo_.container(), cinfo_.name());
+    table_handle_set_ = true;
+  }
+
+  ~ResourceConstructionOp() override {
+    // If the table object was not shared, delete it.
+    if (table_handle_set_ && cinfo_.resource_is_private_to_kernel()) {
+      if (!cinfo_.resource_manager()
+               ->template Delete<ContainerBase>(cinfo_.container(),
+                                                cinfo_.name())
+               .ok()) {
+        // Do nothing; the resource may have been deleted by session resets.
+      }
+    }
+  }
+
+ private:
+  mutex mu_;
+  PersistentTensor table_handle_ GUARDED_BY(mu_);
+  bool table_handle_set_ GUARDED_BY(mu_);
+  ContainerInfo cinfo_;
+  bool use_node_name_sharing_;
+
+  TF_DISALLOW_COPY_AND_ASSIGN(ResourceConstructionOp);
+};
+
+// Lookup or create resources of type Container but treat them as having type
+// ContainerBase for the purpose of dynamic dispatching.
+// Container must have constructor
+// Container(OpKernelContext*, OpKernel*, FallbackTableBaseType*)
+// Container must decrease the reference count of the FallbackTableBaseType*
+// constructor argument before its destructor completes.
+template <class ContainerBase, class ContainerChild,
+          class FallbackTableBaseType = ContainerBase>
+class TableWithFallbackConstructionOp : public OpKernel {
+ public:
+  explicit TableWithFallbackConstructionOp(OpKernelConstruction* ctx)
+      : OpKernel(ctx), table_handle_set_(false) {
+    static_assert(std::is_base_of<ContainerBase, ContainerChild>::value,
+                  "ContainerChild is not derived from ContainerBase");
+    OP_REQUIRES_OK(ctx, ctx->allocate_persistent(tensorflow::DT_STRING,
+                                                 tensorflow::TensorShape({2}),
+                                                 &table_handle_, nullptr));
+    OP_REQUIRES_OK(
+        ctx, ctx->GetAttr("use_node_name_sharing", &use_node_name_sharing_));
+  }
+
+  void Compute(OpKernelContext* ctx) override {
+    OpInputList table_int64_args;
+    OP_REQUIRES_OK(ctx, ctx->input_list("table_int64_args", &table_int64_args));
+    if (ctx->num_inputs() == table_int64_args.size()) {
+      ctx->SetStatus(errors::InvalidArgument(
+          "Expected op to have a resource input after the table_int64_args "
+          "input but no such input found."));
+      return;
+    }
+
+    FallbackTableBaseType* fallback_table = nullptr;
+    {
+      const Tensor& table_handle = ctx->input(table_int64_args.size());
+      ResourceHandle handle(table_handle.scalar<ResourceHandle>()());
+      OP_REQUIRES_OK(
+          ctx, ctx->resource_manager()->Lookup(handle.container(),
+                                               handle.name(), &fallback_table));
+    }
+    mutex_lock l(mu_);
+
+    if (!table_handle_set_) {
+      OP_REQUIRES_OK(ctx, cinfo_.Init(ctx->resource_manager(), def(),
+                                      use_node_name_sharing_));
+    }
+
+    auto creator = [ctx, this, fallback_table](
+                       ContainerBase** ret) EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+      // container construction logic can't be merged with
+      // ResourceConstructionOp because Container constructor requires an
+      // input which can only be constructed if the resource manager
+      // internal lock is not already held.
+      ContainerBase* container = new ContainerChild(ctx, this, fallback_table);
+      if (!ctx->status().ok()) {
+        container->Unref();
+        return ctx->status();
+      }
+      if (ctx->track_allocations()) {
+        ctx->record_persistent_memory_allocation(container->MemoryUsed());
+      }
+      *ret = container;
+      return Status::OK();
+    };
+
+    ContainerBase* table = nullptr;
+    OP_REQUIRES_OK(
+        ctx, cinfo_.resource_manager()->template LookupOrCreate<ContainerBase>(
+                 cinfo_.container(), cinfo_.name(), &table, creator));
+    core::ScopedUnref unref_me(table);
+
+    Tensor* handle;
+    OP_REQUIRES_OK(ctx, ctx->allocate_output(0, TensorShape({}), &handle));
+    handle->scalar<ResourceHandle>()() = MakeResourceHandle<ContainerBase>(
+        ctx, cinfo_.container(), cinfo_.name());
+    table_handle_set_ = true;
+  }
+
+  ~TableWithFallbackConstructionOp() override {
+    // If the table object was not shared, delete it.
+    if (table_handle_set_ && cinfo_.resource_is_private_to_kernel()) {
+      if (!cinfo_.resource_manager()
+               ->template Delete<ContainerBase>(cinfo_.container(),
+                                                cinfo_.name())
+               .ok()) {
+        // Do nothing; the resource may have been deleted by session resets.
+      }
+    }
+  }
+
+ private:
+  mutex mu_;
+  PersistentTensor table_handle_ GUARDED_BY(mu_);
+  bool table_handle_set_ GUARDED_BY(mu_);
+  ContainerInfo cinfo_;
+  bool use_node_name_sharing_;
+
+  TF_DISALLOW_COPY_AND_ASSIGN(TableWithFallbackConstructionOp);
+};
+
+// Op that returns the size of a container.
+template <class Container>
+class ContainerSizeOp : public OpKernel {
+ public:
+  explicit ContainerSizeOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}
+
+  void Compute(OpKernelContext* ctx) override {
+    const Tensor& container_handle = ctx->input(0);
+    ResourceHandle handle(container_handle.scalar<ResourceHandle>()());
+    Container* container;
+    OP_REQUIRES_OK(ctx, ctx->resource_manager()->Lookup(
+                            handle.container(), handle.name(), &container));
+    core::ScopedUnref unref_me(container);
+    OP_REQUIRES_OK(ctx, container->SizeStatus());
+
+    Tensor* out;
+    OP_REQUIRES_OK(ctx, ctx->allocate_output(0, TensorShape({}), &out));
+
+    auto* mutex = container->GetMutex();
+    if (mutex != nullptr) {
+      tf_shared_lock lock(*mutex);
+      out->scalar<int64>()() = container->size();
+    } else {
+      out->scalar<int64>()() = container->size();
+    }
+  }
+};
+
+}  // namespace tables
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_KERNELS_LOOKUP_TABLES_TABLE_OP_UTILS_H_
diff --git a/tensorflow/core/kernels/lookup_tables/table_resource_utils.h b/tensorflow/core/kernels/lookup_tables/table_resource_utils.h
new file mode 100644
index 0000000000..2065904d61
--- /dev/null
+++ b/tensorflow/core/kernels/lookup_tables/table_resource_utils.h
@@ -0,0 +1,101 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_KERNELS_LOOKUP_TABLES_TABLE_RESOURCE_UTILS_H_
+#define TENSORFLOW_CORE_KERNELS_LOOKUP_TABLES_TABLE_RESOURCE_UTILS_H_
+
+#include <memory>
+
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/kernels/lookup_tables/lookup_table_interface.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/platform/mutex.h"
+
+namespace tensorflow {
+namespace tables {
+
+// Parent class for tables with support for multithreaded synchronization.
+template <typename HeterogeneousKeyType, typename ValueType>
+class LookupTableWithSynchronization
+    : public LookupTableInterface<HeterogeneousKeyType, ValueType> {
+ public:
+  // By convention, it is assumed that the OpKernel which creates this
+  // resource is bound to an op whose first input is an in64 tensor list with a
+  // first element whose boolean value indicates whether a mutex should be
+  // exported to synchronize access to state. If the tensor list is empty,
+  // the status in ctx is set to InvalidArgument and this object is in an
+  // undefined state.
+  LookupTableWithSynchronization(OpKernelContext* ctx, OpKernel* kernel) {
+    OpInputList table_int64_args;
+    OP_REQUIRES_OK(ctx, ctx->input_list("table_int64_args", &table_int64_args));
+    if (table_int64_args.size() == 0) {
+      ctx->SetStatus(errors::InvalidArgument(
+          "table_int64_args should not be empty. Set the first element "
+          "to 1 to enable synchronized table use and to 0 otherwise."));
+      return;
+    }
+    if (ctx->input(0).scalar<int64>()() != 0) {
+      mutex_ = absl::make_unique<mutex>();
+    }
+  }
+
+  // Mutex for synchronizing access to unsynchronized methods.
+  mutex* GetMutex() const override { return mutex_.get(); }
+
+ private:
+  // Use this for locking.
+  mutable std::unique_ptr<mutex> mutex_;
+};
+
+// Parent class for tables which can be constructed with arbitrary
+// lookup fallbacks.
+// Since LookupTableInterface::LookupKey assumes that all keys can be mapped
+// to values, LookupTableWithFallbackInterface allows clients to implement
+// two-stage lookups. If the first key lookup fails, clients can choose
+// to perform a fallback lookup using an externally supplied table.
+template <typename HeterogeneousKeyType, typename ValueType,
+          typename FallbackTableRegisteredType =
+              LookupTableInterface<HeterogeneousKeyType, ValueType>>
+class LookupTableWithFallbackInterface
+    : public LookupTableWithSynchronization<HeterogeneousKeyType, ValueType> {
+ public:
+  LookupTableWithFallbackInterface(OpKernelContext* ctx, OpKernel* kernel,
+                                   FallbackTableRegisteredType* fallback_table)
+      : LookupTableWithSynchronization<HeterogeneousKeyType, ValueType>(ctx,
+                                                                        kernel),
+        fallback_table_(fallback_table) {}
+
+  // Clients are required to fail when ctx is set to a not-OK status in
+  // the constructor so this dereference is safe.
+  const FallbackTableRegisteredType& fallback_table() const {
+    return *fallback_table_;
+  }
+
+  ~LookupTableWithFallbackInterface() override {
+    if (fallback_table_ != nullptr) {
+      fallback_table_->Unref();
+    }
+  }
+
+ private:
+  FallbackTableRegisteredType* fallback_table_;
+};
+
+}  // namespace tables
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_KERNELS_LOOKUP_TABLES_TABLE_RESOURCE_UTILS_H_
-- 
GitLab


From b322b7993bcadc91fa1afa6d1ca116b0cfcf5a5b Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Wed, 23 Jan 2019 14:28:38 -0800
Subject: [PATCH 1138/2345] - Fix freezing of the learning phase in V2 graph
 functions mode for Keras. - Make learning phase value thread-local in eager,
 to prevent `set_learning_phase` from affecting other threads. In graph mode
 it is graph-local. - Publicly expose `learning_phase_scope`, which was meant
 to be public but has been forgotten in API exports.

PiperOrigin-RevId: 230603630
---
 tensorflow/python/keras/backend.py            | 58 +++++++++++++++----
 tensorflow/python/keras/backend_test.py       | 20 +++----
 .../python/keras/engine/base_layer_test.py    | 32 ++++++++++
 .../python/keras/engine/training_eager.py     |  2 +-
 .../python/keras/engine/training_generator.py |  6 +-
 .../python/keras/layers/normalization_test.py | 12 ++--
 tensorflow/python/keras/saving/saved_model.py |  5 +-
 .../keras/utils/multi_gpu_utils_test.py       |  1 -
 .../golden/v1/tensorflow.keras.backend.pbtxt  |  4 ++
 .../golden/v2/tensorflow.keras.backend.pbtxt  |  4 ++
 10 files changed, 107 insertions(+), 37 deletions(-)

diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index f7c0d4741a..a3369ef43a 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -87,11 +87,8 @@ _GRAPH_LEARNING_PHASES = weakref.WeakKeyDictionary()
 
 # _DUMMY_EAGER_GRAPH is used as a key in _GRAPH_LEARNING_PHASES.
 # We keep a separate reference to it to make sure it does not get removed from
-# _GRAPH_LEARNING_PHASES. We use a dummy class instead of something like a
-# string because strings are not weakly-referencable.
-class _DummyEagerGraph(object):
-  pass
-_DUMMY_EAGER_GRAPH = _DummyEagerGraph()
+# _GRAPH_LEARNING_PHASES.
+_DUMMY_EAGER_GRAPH = threading.local()
 
 # This boolean flag can be set to True to leave variable initialization
 # up to the user.
@@ -287,11 +284,25 @@ def set_learning_phase(value):
     raise ValueError('Expected learning phase to be 0 or 1.')
   with ops.init_scope():
     if context.executing_eagerly():
+      # In an eager context, the learning phase values applies to both the eager
+      # context and the internal Keras graph.
       _GRAPH_LEARNING_PHASES[_DUMMY_EAGER_GRAPH] = value
-    else:
-      _GRAPH_LEARNING_PHASES[get_graph()] = value
+    _GRAPH_LEARNING_PHASES[get_graph()] = value
+
+
+def set_eager_learning_phase(value):
+  """Internal utility that sets the learning phase in eager execution only.
+
+  Arguments:
+      value: Learning phase value, either 0 or 1 (integers).
+  """
+  global _GRAPH_LEARNING_PHASES  # pylint: disable=global-variable-not-assigned
+  assert value in {0, 1}
+  assert context.executing_eagerly()
+  _GRAPH_LEARNING_PHASES[_DUMMY_EAGER_GRAPH] = value
 
 
+@keras_export('keras.backend.learning_phase_scope')
 @tf_contextlib.contextmanager
 def learning_phase_scope(value):
   """Provides a scope within which the learning phase is equal to `value`.
@@ -302,24 +313,49 @@ def learning_phase_scope(value):
      value: Learning phase value, either 0 or 1 (integers).
 
   Yields:
-    The provided value.
+    None.
 
   Raises:
      ValueError: if `value` is neither `0` nor `1`.
   """
+  global _GRAPH_LEARNING_PHASES  # pylint: disable=global-variable-not-assigned
   if value not in {0, 1}:
     raise ValueError('Expected learning phase to be 0 or 1.')
   previous_value = learning_phase()
   try:
     set_learning_phase(value)
-    yield value
+    yield
   finally:
     # Restore learning phase to initial value.
     with ops.init_scope():
       if context.executing_eagerly():
         _GRAPH_LEARNING_PHASES[_DUMMY_EAGER_GRAPH] = previous_value
-      else:
-        _GRAPH_LEARNING_PHASES[get_graph()] = previous_value
+      _GRAPH_LEARNING_PHASES[get_graph()] = previous_value
+
+
+@tf_contextlib.contextmanager
+def eager_learning_phase_scope(value):
+  """Internal scope that sets the learning phase in eager execution only.
+
+  Arguments:
+      value: Learning phase value, either 0 or 1 (integers).
+
+  Yields:
+    None.
+
+  Raises:
+     ValueError: if `value` is neither `0` nor `1`.
+  """
+  global _GRAPH_LEARNING_PHASES  # pylint: disable=global-variable-not-assigned
+  assert value in {0, 1}
+  assert context.executing_eagerly()
+  previous_value = learning_phase()
+  try:
+    _GRAPH_LEARNING_PHASES[_DUMMY_EAGER_GRAPH] = value
+    yield
+  finally:
+    # Restore learning phase to initial value.
+    _GRAPH_LEARNING_PHASES[_DUMMY_EAGER_GRAPH] = previous_value
 
 
 def _get_session():
diff --git a/tensorflow/python/keras/backend_test.py b/tensorflow/python/keras/backend_test.py
index d25c214622..27657aeef6 100644
--- a/tensorflow/python/keras/backend_test.py
+++ b/tensorflow/python/keras/backend_test.py
@@ -108,27 +108,23 @@ class BackendUtilsTest(test.TestCase):
 
   def test_learning_phase(self):
     with self.cached_session() as sess:
-      keras.backend.set_learning_phase(1)
-      self.assertEqual(keras.backend.learning_phase(), 1)
       with self.assertRaises(ValueError):
         keras.backend.set_learning_phase(2)
 
       # Test running with a learning-phase-consuming layer
-      keras.backend.set_learning_phase(0)
-      x = keras.Input((3,))
-      y = keras.layers.BatchNormalization()(x)
-      if not context.executing_eagerly():
-        self.evaluate(variables.global_variables_initializer())
-        sess.run(y, feed_dict={x: np.random.random((2, 3))})
+      with keras.backend.learning_phase_scope(0):
+        x = keras.Input((3,))
+        y = keras.layers.BatchNormalization()(x)
+        if not context.executing_eagerly():
+          self.evaluate(variables.global_variables_initializer())
+          sess.run(y, feed_dict={x: np.random.random((2, 3))})
 
   def test_learning_phase_scope(self):
     initial_learning_phase = keras.backend.learning_phase()
-    with keras.backend.learning_phase_scope(1) as lp:
-      self.assertEqual(lp, 1)
+    with keras.backend.learning_phase_scope(1):
       self.assertEqual(keras.backend.learning_phase(), 1)
     self.assertEqual(keras.backend.learning_phase(), initial_learning_phase)
-    with keras.backend.learning_phase_scope(0) as lp:
-      self.assertEqual(lp, 0)
+    with keras.backend.learning_phase_scope(0):
       self.assertEqual(keras.backend.learning_phase(), 0)
     self.assertEqual(keras.backend.learning_phase(), initial_learning_phase)
     with self.assertRaises(ValueError):
diff --git a/tensorflow/python/keras/engine/base_layer_test.py b/tensorflow/python/keras/engine/base_layer_test.py
index 8e9de8b7cc..93ca10ee4e 100644
--- a/tensorflow/python/keras/engine/base_layer_test.py
+++ b/tensorflow/python/keras/engine/base_layer_test.py
@@ -223,6 +223,38 @@ class BaseLayerTest(keras_parameterized.TestCase):
     model(np.zeros((2, 4), dtype='float32'))
     self.assertTrue(model.built)
 
+  def test_learning_phase_freezing_for_layers(self):
+    # This test is only meant to run in graph functions mode (ambient eager).
+    # In forced eager, `model.predict` ignores the global learning phase
+    # and just uses training=False. TODO(fchollet): consider unifying the
+    # behaviors.
+
+    class LearningPhaseLayer(keras.layers.Layer):
+
+      def call(self, inputs):
+        return keras.backend.in_train_phase(
+            lambda: array_ops.ones_like(inputs),
+            lambda: array_ops.zeros_like(inputs))
+
+    def get_learning_phase_value():
+      model = keras.models.Sequential([LearningPhaseLayer(input_shape=(1,))])
+      return np.sum(model.predict(np.ones((1, 1))))
+
+    self.assertEqual(get_learning_phase_value(), 0)
+
+    # Test scope.
+    with keras.backend.learning_phase_scope(1):
+      self.assertEqual(get_learning_phase_value(), 1)
+
+    # The effects of the scope end after exiting it.
+    self.assertEqual(get_learning_phase_value(), 0)
+
+    # Test setting.
+    keras.backend.set_learning_phase(1)
+    self.assertEqual(get_learning_phase_value(), 1)
+    keras.backend.set_learning_phase(0)
+    self.assertEqual(get_learning_phase_value(), 0)
+
   def test_using_symbolic_tensors_with_tf_ops(self):
     # Single-input.
     x = keras.Input((3,))
diff --git a/tensorflow/python/keras/engine/training_eager.py b/tensorflow/python/keras/engine/training_eager.py
index e62dd1ffa8..e64c0c5eba 100644
--- a/tensorflow/python/keras/engine/training_eager.py
+++ b/tensorflow/python/keras/engine/training_eager.py
@@ -202,7 +202,7 @@ def _process_single_batch(model,
   Raises:
       ValueError: If the model has no loss to optimize.
   """
-  with backend.learning_phase_scope(1 if training else 0):
+  with backend.eager_learning_phase_scope(1 if training else 0):
     with GradientTape() as tape:
       outs, loss, loss_metrics, aggregated_loss_metrics, masks\
         = _model_loss(
diff --git a/tensorflow/python/keras/engine/training_generator.py b/tensorflow/python/keras/engine/training_generator.py
index b8defad8b2..da460ee205 100644
--- a/tensorflow/python/keras/engine/training_generator.py
+++ b/tensorflow/python/keras/engine/training_generator.py
@@ -140,7 +140,6 @@ def model_iteration(model,
       shuffle=shuffle)
 
   do_validation = validation_data is not None
-  should_set_learning_phase = context.executing_eagerly() and model.run_eagerly
   is_sequence = isinstance(generator, data_utils.Sequence)
   _validate_arguments(is_sequence, is_dataset, use_multiprocessing, workers,
                       steps_per_epoch, validation_data, validation_steps, mode,
@@ -181,9 +180,10 @@ def model_iteration(model,
   else:
     aggregator = training_utils.MetricsAggregator(True, steps_per_epoch)
 
+  should_set_learning_phase = context.executing_eagerly() and model.run_eagerly
   if should_set_learning_phase:
     old_learning_phase = backend.learning_phase()
-    backend.set_learning_phase(1 if mode == ModeKeys.TRAIN else 0)
+    backend.set_eager_learning_phase(1 if mode == ModeKeys.TRAIN else 0)
 
   callbacks.model.stop_training = False
   callbacks._call_begin_hook(mode)
@@ -302,7 +302,7 @@ def model_iteration(model,
     enqueuer.stop()
 
   if should_set_learning_phase:
-    backend.set_learning_phase(old_learning_phase)
+    backend.set_eager_learning_phase(old_learning_phase)
 
   if mode == ModeKeys.TRAIN:
     return model.history
diff --git a/tensorflow/python/keras/layers/normalization_test.py b/tensorflow/python/keras/layers/normalization_test.py
index 4308e0f724..bd5951b845 100644
--- a/tensorflow/python/keras/layers/normalization_test.py
+++ b/tensorflow/python/keras/layers/normalization_test.py
@@ -356,12 +356,12 @@ class NormalizationLayersGraphModeOnlyTest(test.TestCase):
 
       # Simulates training-mode with trainable layer.
       # Should use mini-batch statistics.
-      keras.backend.set_learning_phase(1)
-      model = get_model(bn_mean, bn_std)
-      model.compile(loss='mse', optimizer='rmsprop')
-      out = model.predict(val_a)
-      self.assertAllClose(
-          (val_a - np.mean(val_a)) / np.std(val_a), out, atol=1e-3)
+      with keras.backend.learning_phase_scope(1):
+        model = get_model(bn_mean, bn_std)
+        model.compile(loss='mse', optimizer='rmsprop')
+        out = model.predict(val_a)
+        self.assertAllClose(
+            (val_a - np.mean(val_a)) / np.std(val_a), out, atol=1e-3)
 
 
 def _run_layernorm_correctness_test(layer, dtype='float32'):
diff --git a/tensorflow/python/keras/saving/saved_model.py b/tensorflow/python/keras/saving/saved_model.py
index 06e02acf32..fbf0bf68ef 100644
--- a/tensorflow/python/keras/saving/saved_model.py
+++ b/tensorflow/python/keras/saving/saved_model.py
@@ -255,9 +255,8 @@ def _export_mode(
         'Model does not have an optimizer. Cannot export mode %s' % mode)
 
   model_graph = ops.get_default_graph()
-  with ops.Graph().as_default() as g:
-
-    K.set_learning_phase(mode == mode_keys.ModeKeys.TRAIN)
+  with ops.Graph().as_default() as g, K.learning_phase_scope(
+      mode == mode_keys.ModeKeys.TRAIN):
 
     if input_signature is None:
       input_tensors = None
diff --git a/tensorflow/python/keras/utils/multi_gpu_utils_test.py b/tensorflow/python/keras/utils/multi_gpu_utils_test.py
index 8c1abd6324..9c711bd2a2 100644
--- a/tensorflow/python/keras/utils/multi_gpu_utils_test.py
+++ b/tensorflow/python/keras/utils/multi_gpu_utils_test.py
@@ -148,7 +148,6 @@ class TestMultiGPUModel(test.TestCase):
       input_shape = (num_samples,) + shape
       x_train = np.random.randint(0, 255, input_shape)
       y_train = np.random.randint(0, num_classes, (input_shape[0],))
-      keras.backend.set_learning_phase(True)
 
       y_train = keras.utils.to_categorical(y_train, num_classes)
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.backend.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.backend.pbtxt
index 3a31024817..712e3ad6fc 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.backend.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.backend.pbtxt
@@ -256,6 +256,10 @@ tf_module {
     name: "learning_phase"
     argspec: "args=[], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "learning_phase_scope"
+    argspec: "args=[\'value\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "less"
     argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.backend.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.backend.pbtxt
index 01a3fde258..49e3d1155c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.backend.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.backend.pbtxt
@@ -252,6 +252,10 @@ tf_module {
     name: "learning_phase"
     argspec: "args=[], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "learning_phase_scope"
+    argspec: "args=[\'value\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "less"
     argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-- 
GitLab


From 952ad4877a8ca8720ea2e763e36c11fd5761162c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 23 Jan 2019 14:33:08 -0800
Subject: [PATCH 1139/2345] Add benchmark for DivNoNan. Use wallclock to
 measure throughput in benchmarks.

PiperOrigin-RevId: 230604519
---
 tensorflow/core/kernels/cwise_ops_test.cc | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/tensorflow/core/kernels/cwise_ops_test.cc b/tensorflow/core/kernels/cwise_ops_test.cc
index 696d5840e8..acf7cc2899 100644
--- a/tensorflow/core/kernels/cwise_ops_test.cc
+++ b/tensorflow/core/kernels/cwise_ops_test.cc
@@ -45,6 +45,7 @@ int ColsFromArg(int arg) { return (arg % kRows); }
 #define BM_UNARY(DEVICE, FUNC, T, TYPE)                              \
   void BM_##DEVICE##_##FUNC##_##TYPE(int iters, int num) {           \
     const int64 tot = static_cast<int64>(iters) * num;               \
+    testing::UseRealTime();                                          \
     testing::ItemsProcessed(tot);                                    \
     testing::BytesProcessed(tot * sizeof(T));                        \
     test::Benchmark(#DEVICE, Unary<T>(#FUNC, num, TYPE)).Run(iters); \
@@ -100,6 +101,7 @@ Graph* BinaryScalar(int num, const string& func) {
 #define BM_BINARY_SCALAR(DEVICE, FUNC)                             \
   void BM_##DEVICE##_##FUNC##_scalar(int iters, int num) {         \
     const int64 tot = static_cast<int64>(iters) * num;             \
+    testing::UseRealTime();                                        \
     testing::ItemsProcessed(tot);                                  \
     testing::BytesProcessed(tot * sizeof(float));                  \
     test::Benchmark(#DEVICE, BinaryScalar(num, #FUNC)).Run(iters); \
@@ -125,6 +127,15 @@ BM_BINARY_SCALAR(gpu, Add);
 #ifdef TENSORFLOW_USE_SYCL
 BM_BINARY_SCALAR(sycl, Add);
 #endif  // TENSORFLOW_USE_SYCL
+
+BM_BINARY_SCALAR(cpu, DivNoNan);
+#if GOOGLE_CUDA
+BM_BINARY_SCALAR(gpu, DivNoNan);
+#endif  // GOOGLE_CUDA
+#ifdef TENSORFLOW_USE_SYCL
+BM_BINARY_SCALAR(sycl, DivNoNan);
+#endif  // TENSORFLOW_USE_SYCL
+
 #undef BM_BINARY_SCALAR
 
 template <class T>
@@ -146,6 +157,7 @@ Graph* BiasAdd(int rows, int cols, DataType type) {
     const int rows = RowsFromArg(arg);                                         \
     const int cols = ColsFromArg(arg);                                         \
     const int64 tot = static_cast<int64>(iters) * rows * cols;                 \
+    testing::UseRealTime();                                                    \
     testing::ItemsProcessed(tot);                                              \
     testing::BytesProcessed(tot * sizeof(C_TYPE));                             \
     test::Benchmark(#DEVICE, BiasAdd<C_TYPE>(rows, cols, TF_TYPE)).Run(iters); \
@@ -197,6 +209,7 @@ Graph* BiasAddGrad(int rows, int cols, int channels, DataType type,
     const int rows = RowsFromArg(arg);                                         \
     const int cols = ColsFromArg(arg);                                         \
     const int64 tot = static_cast<int64>(iters) * rows * cols * channels;      \
+    testing::UseRealTime();                                                    \
     testing::ItemsProcessed(tot);                                              \
     testing::BytesProcessed(tot * sizeof(C_TYPE));                             \
     test::Benchmark(#DEVICE, BiasAddGrad<C_TYPE>(rows, cols, channels,         \
@@ -259,6 +272,7 @@ Graph* BcastAdd(int rows, int cols, int dim) {
     const int rows = RowsFromArg(arg);                             \
     const int cols = ColsFromArg(arg);                             \
     const int64 tot = static_cast<int64>(iters) * rows * cols;     \
+    testing::UseRealTime();                                        \
     testing::ItemsProcessed(tot);                                  \
     testing::BytesProcessed(tot * sizeof(float));                  \
     test::Benchmark(#DEVICE, BcastAdd(rows, cols, 0)).Run(iters);  \
@@ -285,6 +299,7 @@ BM_BCAST_ADD_ROW_ALL(sycl);
     const int rows = RowsFromArg(arg);                             \
     const int cols = ColsFromArg(arg);                             \
     const int64 tot = static_cast<int64>(iters) * rows * cols;     \
+    testing::UseRealTime();                                        \
     testing::ItemsProcessed(tot);                                  \
     testing::BytesProcessed(tot * sizeof(float));                  \
     test::Benchmark(#DEVICE, BcastAdd(rows, cols, 1)).Run(iters);  \
@@ -311,6 +326,7 @@ BM_BCAST_ADD_COL_ALL(sycl);
     const int rows = RowsFromArg(arg);                                 \
     const int cols = ColsFromArg(arg);                                 \
     const int64 tot = static_cast<int64>(iters) * rows * cols;         \
+    testing::UseRealTime();                                            \
     testing::ItemsProcessed(tot);                                      \
     testing::BytesProcessed(tot * sizeof(float));                      \
     test::Benchmark(#DEVICE, BcastAdd(rows, cols, 2)).Run(iters);      \
@@ -338,6 +354,7 @@ BM_BCAST_ADD_CROSS_RC_ALL(sycl);
     const int rows = RowsFromArg(arg);                                 \
     const int cols = ColsFromArg(arg);                                 \
     const int64 tot = static_cast<int64>(iters) * rows * cols;         \
+    testing::UseRealTime();                                            \
     testing::ItemsProcessed(tot);                                      \
     testing::BytesProcessed(tot * sizeof(float));                      \
     test::Benchmark(#DEVICE, BcastAdd(rows, cols, 3)).Run(iters);      \
-- 
GitLab


From fca403f459388cf73fcc1b615a1c2b653a950408 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Wed, 23 Jan 2019 14:44:41 -0800
Subject: [PATCH 1140/2345] [Grappler] Add functions from the subgraph to the
 main graph

PiperOrigin-RevId: 230606789
---
 tensorflow/core/grappler/BUILD                |  2 +
 .../core/grappler/mutable_graph_view.cc       | 42 ++++++++++-----
 tensorflow/core/grappler/mutable_graph_view.h |  9 ++--
 .../core/grappler/mutable_graph_view_test.cc  | 54 +++++++++++--------
 4 files changed, 68 insertions(+), 39 deletions(-)

diff --git a/tensorflow/core/grappler/BUILD b/tensorflow/core/grappler/BUILD
index d5266247cf..9fe699360f 100644
--- a/tensorflow/core/grappler/BUILD
+++ b/tensorflow/core/grappler/BUILD
@@ -209,10 +209,12 @@ cc_library(
         ":graph_view",
         ":op_types",
         ":utils",
+        "//tensorflow/core:framework",
         "//tensorflow/core:graph",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
+        "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/types:span",
diff --git a/tensorflow/core/grappler/mutable_graph_view.cc b/tensorflow/core/grappler/mutable_graph_view.cc
index dc9be9ebe7..3b203661c5 100644
--- a/tensorflow/core/grappler/mutable_graph_view.cc
+++ b/tensorflow/core/grappler/mutable_graph_view.cc
@@ -18,10 +18,12 @@ limitations under the License.
 #include <algorithm>
 #include <utility>
 
+#include "absl/container/flat_hash_map.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_join.h"
 #include "absl/strings/string_view.h"
 #include "absl/strings/substitute.h"
+#include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/graph/graph.h"
@@ -222,28 +224,40 @@ NodeDef* MutableGraphView::AddNode(NodeDef&& node) {
 }
 
 Status MutableGraphView::AddSubgraph(GraphDef&& subgraph) {
-  if (subgraph.library().function_size() != 0) {
-    constexpr int kMaxNumNodesForDebugStr = 100;
-    string subgraph_str = "too large to display";
-    if (subgraph.node_size() < kMaxNumNodesForDebugStr) {
-      constexpr int kMaxLenSubgraphStr = 1000;
-      subgraph_str = subgraph.ShortDebugString();
-      if (subgraph_str.length() > kMaxLenSubgraphStr) {
-        subgraph_str =
-            absl::StrCat(subgraph_str.substr(0, kMaxLenSubgraphStr), "...");
+  // 1. Add all new functions and check that functions with the same name
+  // have identical definition.
+  const int function_size = subgraph.library().function_size();
+  if (function_size > 0) {
+    absl::flat_hash_map<absl::string_view, const FunctionDef*> graph_fdefs;
+    for (const FunctionDef& fdef : graph()->library().function()) {
+      graph_fdefs.emplace(fdef.signature().name(), &fdef);
+    }
+
+    for (FunctionDef& fdef : *subgraph.mutable_library()->mutable_function()) {
+      const auto graph_fdef = graph_fdefs.find(fdef.signature().name());
+
+      if (graph_fdef == graph_fdefs.end()) {
+        VLOG(3) << "Add new function definition: " << fdef.signature().name();
+        graph()->mutable_library()->add_function()->Swap(&fdef);
+      } else {
+        if (!FunctionDefsEqual(fdef, *graph_fdef->second)) {
+          return MutationError(
+              "AddSubgraph",
+              absl::Substitute("function_size=$0", function_size),
+              absl::StrCat(
+                  "Found different function definition with the same name: ",
+                  fdef.signature().name()));
+        }
       }
     }
-    string params = absl::Substitute("subgraph='$0'", subgraph_str);
-    return MutationError(
-        "AddSubgraph", params,
-        "can't add a subgraph with non-empty function library");
   }
 
+  // 2. Add all nodes to the underlying graph.
   int node_size_before = graph()->node_size();
 
   for (NodeDef& node : *subgraph.mutable_node()) {
     auto* node_in_graph = graph()->add_node();
-    *node_in_graph = std::move(node);
+    node_in_graph->Swap(&node);
     TF_RETURN_IF_ERROR(AddUniqueNode(node_in_graph));
   }
 
diff --git a/tensorflow/core/grappler/mutable_graph_view.h b/tensorflow/core/grappler/mutable_graph_view.h
index 73c32f4e55..b4d07bbd57 100644
--- a/tensorflow/core/grappler/mutable_graph_view.h
+++ b/tensorflow/core/grappler/mutable_graph_view.h
@@ -68,9 +68,12 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // it can have edges to the nodes that are not in it, however after adding
   // it to the underlying graph, final graph must be valid.
   //
-  // TODO(ezhulenev): Currently it will fail if subgraph has non-empty function
-  // library. Add support for adding new functions from the subgraph function
-  // library into the underlying graph.
+  // If subgraph function library is not empty, all new functions will be added
+  // to the graph. Functions that appear with the same name in both subgraph and
+  // the graph represented by *this, must have identical function definitions.
+  //
+  // IMPORTANT: All nodes and functions of the given subgraph moved into the
+  // underlying graph, which leaves subgraph in valid but undefined state.
   Status AddSubgraph(GraphDef&& subgraph);
 
   // Updates all fanouts (input ports fetching output tensors) from
diff --git a/tensorflow/core/grappler/mutable_graph_view_test.cc b/tensorflow/core/grappler/mutable_graph_view_test.cc
index 8b94364d54..7c7dff07ae 100644
--- a/tensorflow/core/grappler/mutable_graph_view_test.cc
+++ b/tensorflow/core/grappler/mutable_graph_view_test.cc
@@ -169,34 +169,44 @@ TEST(MutableGraphViewTest, AddSubgraph) {
   CheckGraph(graph);
 }
 
-// TODO(ezhulenev): Add support for adding a subgraph and merging function
-// libraries.
-TEST(MutableGraphViewTest, AddSubgraphWithFunctionLibrary) {
-  GraphDef graph_def = test::function::GDef(
-      {
-          NDef("foo", "NotImportant", {}, {}),
-          NDef("bar", "NotImportant", {}, {}),
-          NDef("baz", "NotImportant", {"foo", "bar"}),
-      },
-      /*funcs=*/{});
+TEST(MutableGraphViewTest, AddSubgraphAndAddFunction) {
+  GraphDef graph_def;
   MutableGraphView graph(&graph_def);
 
   FunctionDef x_times_two = test::function::XTimesTwo();
-  GraphDef subgraph = test::function::GDef(
-      {
-          NDef("s/n0", "NotImportant", {}, {}),
-          NDef("s/n1", "NotImportant", {"bar", "s/n0"}, {}),
-      },
-      /*funcs=*/{x_times_two});
+  GraphDef subgraph = test::function::GDef({}, {x_times_two});
+
+  TF_EXPECT_OK(graph.AddSubgraph(std::move(subgraph)));
+  EXPECT_EQ(graph_def.library().function_size(), 1);
+}
+
+TEST(MutableGraphViewTest, AddSubgraphAndSkipSameFunction) {
+  FunctionDef x_times_two = test::function::XTimesTwo();
+
+  GraphDef graph_def = test::function::GDef({}, {x_times_two});
+  MutableGraphView graph(&graph_def);
+
+  GraphDef subgraph = test::function::GDef({}, {x_times_two});
+
+  TF_EXPECT_OK(graph.AddSubgraph(std::move(subgraph)));
+  EXPECT_EQ(graph_def.library().function_size(), 1);
+}
+
+TEST(MutableGraphViewTest, AddSubgraphAndFailIfFunctionDifferent) {
+  FunctionDef x_times_four = test::function::XTimesFour();
+  x_times_four.mutable_signature()->set_name("XTimesTwo");
+
+  GraphDef graph_def = test::function::GDef({}, {x_times_four});
+  MutableGraphView graph(&graph_def);
+
+  FunctionDef x_times_two = test::function::XTimesTwo();
+  GraphDef subgraph = test::function::GDef({}, {x_times_two});
 
-  string subgraph_str = subgraph.ShortDebugString();
   Status status = graph.AddSubgraph(std::move(subgraph));
   EXPECT_FALSE(status.ok());
-  string expected_msg = absl::Substitute(
-      "MutableGraphView::AddSubgraph(subgraph='$0') error: can't add a "
-      "subgraph with non-empty function library.",
-      subgraph_str);
-  EXPECT_EQ(status.error_message(), expected_msg);
+  EXPECT_EQ(status.error_message(),
+            "MutableGraphView::AddSubgraph(function_size=1) error: Found "
+            "different function definition with the same name: XTimesTwo.");
 }
 
 TEST(MutableGraphViewTest, AddAndUpdateFanouts) {
-- 
GitLab


From 7e939d752a0da122834662cb0bb0f1ac2c473f8a Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Wed, 23 Jan 2019 14:51:43 -0800
Subject: [PATCH 1141/2345] Configure grappler optimization options for
 TF-Eager functions called via function library runtime.

PiperOrigin-RevId: 230608127
---
 tensorflow/core/common_runtime/eager/BUILD    |  1 +
 .../common_runtime/eager/kernel_and_device.cc | 30 ++++++++++++++++---
 .../process_function_library_runtime.cc       | 10 ++++---
 tensorflow/core/grappler/grappler_item.cc     |  8 +++--
 .../grappler/optimizers/meta_optimizer.cc     | 12 ++++----
 .../core/grappler/optimizers/meta_optimizer.h | 13 ++++----
 tensorflow/core/kernels/BUILD                 |  1 +
 .../core/kernels/partitioned_function_ops.cc  | 29 +++++++++++++++---
 tensorflow/python/eager/wrap_function.py      |  5 +++-
 tensorflow/python/eager/wrap_function_test.py | 25 ++++++++++++++++
 10 files changed, 109 insertions(+), 25 deletions(-)

diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD
index 370ec529be..1df55d9c88 100644
--- a/tensorflow/core/common_runtime/eager/BUILD
+++ b/tensorflow/core/common_runtime/eager/BUILD
@@ -155,6 +155,7 @@ tf_cuda_library(
     deps = [
         ":attr_builder",
         "@farmhash_archive//:farmhash",
+        "@com_google_absl//absl/strings",
     ] + select({
         "//tensorflow:android": [
             "//tensorflow/core:android_tensorflow_lib_lite",
diff --git a/tensorflow/core/common_runtime/eager/kernel_and_device.cc b/tensorflow/core/common_runtime/eager/kernel_and_device.cc
index 943b183d63..09f60a7e91 100644
--- a/tensorflow/core/common_runtime/eager/kernel_and_device.cc
+++ b/tensorflow/core/common_runtime/eager/kernel_and_device.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/common_runtime/eager/kernel_and_device.h"
 
+#include "absl/strings/match.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/common_runtime/eager/attr_builder.h"
 #include "tensorflow/core/common_runtime/process_function_library_runtime.h"
@@ -95,10 +96,31 @@ Status KernelAndDeviceFunc::Init(const NodeDef& ndef,
           "Failed to parse config_proto attribute as tensorflow::ConfigProto "
           "proto.");
     }
-    options.optimize_graph_fn =
-        std::bind(grappler::OptimizeGraph, std::placeholders::_1,
-                  std::placeholders::_2, std::placeholders::_3,
-                  std::placeholders::_4, config_proto, std::placeholders::_5);
+    // We are going to execute the graph via function library runtime, and
+    // because function execution semantics is slightly different from the
+    // regular tensorlow graph, we need to make sure that Grappler respects it
+    // when doing it's optimization passes (e.g. do not prune stateful and
+    // dataset ops).
+    grappler::GrapplerItem::OptimizationOptions optimization_options;
+    optimization_options.is_function_instantiation = true;
+
+    // Keras graphs expected to be executed with regular graph execution
+    // semantics (it's allowed to prune stateful and dataset ops).
+    if (absl::StrContains(function_def->signature().name(), "keras_graph")) {
+      optimization_options.is_function_instantiation = false;
+    }
+
+    // Wrapped function expects execution semantics to be the same as
+    // `session.run`, so we should prune unreachable stateful and dataset ops.
+    if (absl::StrContains(function_def->signature().name(),
+                          "wrapped_function")) {
+      optimization_options.is_function_instantiation = false;
+    }
+
+    options.optimize_graph_fn = std::bind(
+        grappler::OptimizeGraph, std::placeholders::_1, std::placeholders::_2,
+        std::placeholders::_3, std::placeholders::_4, config_proto,
+        optimization_options, std::placeholders::_5);
   }
 #endif
   options.graph_collector = graph_collector;
diff --git a/tensorflow/core/common_runtime/process_function_library_runtime.cc b/tensorflow/core/common_runtime/process_function_library_runtime.cc
index 82f07b44ec..950a93671c 100644
--- a/tensorflow/core/common_runtime/process_function_library_runtime.cc
+++ b/tensorflow/core/common_runtime/process_function_library_runtime.cc
@@ -571,15 +571,12 @@ Status ProcessFunctionLibraryRuntime::InstantiateMultiDevice(
   DumpGraph("Before running POST_PLACEMENT passes", graph.get());
   TF_RETURN_IF_ERROR(OptimizationPassRegistry::Global()->RunGrouping(
       OptimizationPassRegistry::POST_PLACEMENT, optimization_options));
-  DumpGraph("Before running POST_REWRITE_FOR_EXEC passes", graph.get());
-  TF_RETURN_IF_ERROR(OptimizationPassRegistry::Global()->RunGrouping(
-      OptimizationPassRegistry::POST_REWRITE_FOR_EXEC, optimization_options));
-  DumpGraph("After all optimization passes", graph.get());
 
   Device* cpu_device;
   TF_RETURN_IF_ERROR(device_mgr_->LookupDevice("CPU:0", &cpu_device));
 
   if (options.optimize_graph_fn) {
+    DumpGraph("Before running graph optimization fn", graph.get());
     Status status = options.optimize_graph_fn(std::move(ret_node_names),
                                               &data->overlay_lib_, device_set,
                                               cpu_device, &graph);
@@ -590,6 +587,11 @@ Status ProcessFunctionLibraryRuntime::InstantiateMultiDevice(
     DumpGraph("After optimization", graph.get());
   }
 
+  DumpGraph("Before running POST_REWRITE_FOR_EXEC passes", graph.get());
+  TF_RETURN_IF_ERROR(OptimizationPassRegistry::Global()->RunGrouping(
+      OptimizationPassRegistry::POST_REWRITE_FOR_EXEC, optimization_options));
+  DumpGraph("After all optimization passes", graph.get());
+
   std::unordered_map<string, std::unique_ptr<Graph>> subgraphs;
   TF_RETURN_IF_ERROR(
       PartitionFunctionGraph(device_set, std::move(graph), &subgraphs));
diff --git a/tensorflow/core/grappler/grappler_item.cc b/tensorflow/core/grappler/grappler_item.cc
index 1323dd9a64..e27d2b049c 100644
--- a/tensorflow/core/grappler/grappler_item.cc
+++ b/tensorflow/core/grappler/grappler_item.cc
@@ -115,11 +115,15 @@ std::unordered_set<string> GrapplerItem::NodesToPreserve() const {
     }
   }
 
-  // Tensorflow functions do not prune side effects, or dataset-output ops from
+  // Tensorflow functions do not prune stateful or dataset-output ops from
   // the function body (see PruneFunctionBody in common_runtime/function.cc).
+  //
+  // We also keep placeholders in the functions body, because it's a bug to have
+  // placeholders inside functions, and we want to catch such invalid graphs
+  // early.
   if (optimization_options_.is_function_instantiation) {
     for (const NodeDef& node : graph.node()) {
-      if (!IsFreeOfSideEffect(node) || IsDataset(node)) {
+      if (IsStateful(node) || IsDataset(node) || IsPlaceholder(node)) {
         result.insert(node.name());
       }
     }
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
index c200b7db07..e4b2e4c55a 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
@@ -631,16 +631,18 @@ Status RunMetaOptimizer(const GrapplerItem& item, const ConfigProto& cfg,
   return status;
 }
 
-Status OptimizeGraph(std::vector<string> ret_node_names,
-                     FunctionLibraryDefinition* flib,
-                     const DeviceSet& device_set, Device* cpu_device,
-                     const ConfigProto& config_proto,
-                     std::unique_ptr<tensorflow::Graph>* g) {
+Status OptimizeGraph(
+    std::vector<string> ret_node_names, FunctionLibraryDefinition* flib,
+    const DeviceSet& device_set, Device* cpu_device,
+    const ConfigProto& config_proto,
+    const GrapplerItem::OptimizationOptions& optimization_options,
+    std::unique_ptr<tensorflow::Graph>* g) {
   if (!tensorflow::grappler::MetaOptimizerEnabled(config_proto)) {
     return Status::OK();
   }
 
   tensorflow::grappler::GrapplerItem item;
+  item.optimization_options() = optimization_options;
 
   // Add all available devices so that inlined function can be placed.
   for (const Device* d : device_set.devices()) {
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.h b/tensorflow/core/grappler/optimizers/meta_optimizer.h
index c972fe3202..aaf8ef938f 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer.h
@@ -112,16 +112,19 @@ Status RunMetaOptimizer(const GrapplerItem& item, const ConfigProto& cfg,
 // `device_set`: the set of devices that graph can refer to.
 // `cpu_device`: the CPU device.
 // `config_proto`: Grapper configuration.
+// `optimization_options`: Grappler optimization constraints that are known only
+//    at runtime.
 //
 // **g is a graph constructed based on the runtime library 'lib'.
 // OptimizeGraph mutates **g extensively and replaces '*g' with a
 // complete copy. Therefore, the caller should not keep any references
 // to nodes *g.
-Status OptimizeGraph(std::vector<string> ret_node_names,
-                     FunctionLibraryDefinition* lib,
-                     const DeviceSet& device_set, Device* cpu_device,
-                     const ConfigProto& config_proto,
-                     std::unique_ptr<tensorflow::Graph>* g);
+Status OptimizeGraph(
+    std::vector<string> ret_node_names, FunctionLibraryDefinition* lib,
+    const DeviceSet& device_set, Device* cpu_device,
+    const ConfigProto& config_proto,
+    const GrapplerItem::OptimizationOptions& optimization_options,
+    std::unique_ptr<tensorflow::Graph>* g);
 
 }  // namespace grappler
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index ce6504e0b6..9476bc5718 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -2417,6 +2417,7 @@ tf_kernel_library(
         "//tensorflow/core/grappler/optimizers:meta_optimizer",
         "//tensorflow/core/grappler/utils:functions",
         "//tensorflow/stream_executor:stream",
+        "@com_google_absl//absl/strings",
     ],
 )
 
diff --git a/tensorflow/core/kernels/partitioned_function_ops.cc b/tensorflow/core/kernels/partitioned_function_ops.cc
index cadb83d8cf..5d26265aaa 100644
--- a/tensorflow/core/kernels/partitioned_function_ops.cc
+++ b/tensorflow/core/kernels/partitioned_function_ops.cc
@@ -12,6 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
+#include "absl/strings/match.h"
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/common_runtime/rendezvous_mgr.h"
 #include "tensorflow/core/framework/function.h"
@@ -156,13 +157,33 @@ class PartitionedCallOp : public AsyncOpKernel {
   Status Instantiate(FunctionLibraryRuntime* lib, OpKernelContext* ctx,
                      std::vector<Tensor>* inputs,
                      FunctionLibraryRuntime::Handle* handle) {
+    // We are going to execute the graph via function library runtime, and
+    // because function execution semantics is slightly different from the
+    // regular tensorlow graph, we need to make sure that Grappler respects it
+    // when doing it's optimization passes (e.g. do not prune stateful and
+    // dataset ops).
+    grappler::GrapplerItem::OptimizationOptions optimization_options;
+    optimization_options.is_function_instantiation = true;
+
+    // Keras graphs expected to be executed with regular graph execution
+    // semantics (it's allowed to prune stateful and dataset ops).
+    if (absl::StrContains(func_.name(), "keras_graph")) {
+      optimization_options.is_function_instantiation = false;
+    }
+
+    // Wrapped function expects execution semantics to be the same as
+    // `session.run`, so we should prune unreachable stateful and dataset ops.
+    if (absl::StrContains(func_.name(), "wrapped_function")) {
+      optimization_options.is_function_instantiation = false;
+    }
+
     FunctionLibraryRuntime::InstantiateOptions opts;
     opts.target = lib->device()->name();
     opts.is_multi_device_function = true;
-    opts.optimize_graph_fn =
-        std::bind(grappler::OptimizeGraph, std::placeholders::_1,
-                  std::placeholders::_2, std::placeholders::_3,
-                  std::placeholders::_4, config_proto_, std::placeholders::_5);
+    opts.optimize_graph_fn = std::bind(
+        grappler::OptimizeGraph, std::placeholders::_1, std::placeholders::_2,
+        std::placeholders::_3, std::placeholders::_4, config_proto_,
+        optimization_options, std::placeholders::_5);
     opts.graph_collector = ctx->graph_collector();
     opts.executor_type = executor_type_;
 
diff --git a/tensorflow/python/eager/wrap_function.py b/tensorflow/python/eager/wrap_function.py
index 61b4876830..f8fbda861c 100644
--- a/tensorflow/python/eager/wrap_function.py
+++ b/tensorflow/python/eager/wrap_function.py
@@ -192,9 +192,12 @@ def wrap_function(fn, signature, name=None):
     the wrapped graph function.
   """
   holder = VariableHolder(fn)
+  func_graph_name = "wrapped_function"
+  if name is not None:
+    func_graph_name = "wrapped_function_" + name
   return WrappedFunction(
       func_graph.func_graph_from_py_func(
-          name,
+          func_graph_name,
           holder,
           args=None, kwargs=None, signature=signature,
           add_control_dependencies=False,
diff --git a/tensorflow/python/eager/wrap_function_test.py b/tensorflow/python/eager/wrap_function_test.py
index a6e1931fcd..bb2b28861f 100644
--- a/tensorflow/python/eager/wrap_function_test.py
+++ b/tensorflow/python/eager/wrap_function_test.py
@@ -217,6 +217,31 @@ class WrapFunctionTest(test.TestCase):
     self.assertEqual(1, does_not_increment(constant_op.constant(3)).numpy())
     self.assertEqual(3, v.numpy())
 
+  def testPruneStatefulOpsFromWrappedFunc(self):
+
+    v0 = variables.Variable(0)
+    v1 = variables.Variable(0)
+
+    # When we wrap a function, we expect it to be executed with 'tf.Graph`
+    # rules: it's allowed to prune all ops that are not in transitive fanin of
+    # the fetches.
+    def f(x):
+      v0.assign_add(1, name='increment_v0')
+      v1.assign_add(1, name='increment_v1')
+      return x
+
+    f_wrapped = wrap_function.wrap_function(f, [1])
+
+    self.assertEqual(1, f_wrapped().numpy())
+    self.assertEqual(0, v0.numpy())
+    self.assertEqual(0, v1.numpy())
+
+    f_wrapped_with_name = wrap_function.wrap_function(f, [2], name='func')
+
+    self.assertEqual(2, f_wrapped_with_name().numpy())
+    self.assertEqual(0, v0.numpy())
+    self.assertEqual(0, v1.numpy())
+
 
 if __name__ == '__main__':
   ops.enable_eager_execution()
-- 
GitLab


From 9204f23763461d117c380233c2e8b997dd5dc667 Mon Sep 17 00:00:00 2001
From: Shivani Agrawal <shivaniagrawal@google.com>
Date: Wed, 23 Jan 2019 15:03:08 -0800
Subject: [PATCH 1142/2345] Removing eager test for dataset from large tensors.

PiperOrigin-RevId: 230610452
---
 .../optimization/optimize_dataset_test.py     | 71 ++++++++-----------
 1 file changed, 29 insertions(+), 42 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
index f44ade049b..bcd027ebbb 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
@@ -29,7 +29,6 @@ from tensorflow.python.data.experimental.ops import scan_ops
 from tensorflow.python.data.experimental.ops import threadpool
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
@@ -113,47 +112,35 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = self.getNext(dataset)
     self.evaluate(get_next())
 
-  def testOptimizationLargeInputFromTensor(self):
-    def dataset_fn(input_t):
-      dataset = dataset_ops.Dataset.from_tensors(input_t)
-      options = dataset_ops.Options()
-      options.experimental_optimization.apply_default_optimizations = False
-      return dataset.with_options(options)
-
-    if context.executing_eagerly():
-      input_t = np.ones([512, 1024, 1025], np.int32)
-      get_next = self.getNext(dataset_fn(input_t))
-      self.evaluate(get_next())
-    else:
-      input_t = array_ops.placeholder(dtypes.int32, (None, None, None))
-      iterator = dataset_ops.make_initializable_iterator(dataset_fn(input_t))
-      init_op = iterator.initializer
-      get_next = iterator.get_next()
-
-      with self.cached_session() as sess:
-        sess.run(init_op, {input_t: np.ones([512, 1024, 1025], np.int32)})
-        self.evaluate(get_next)
-
-  def testOptimizationLargeInputFromTensorSlices(self):
-    def dataset_fn(input_t):
-      dataset = dataset_ops.Dataset.from_tensor_slices(input_t)
-      options = dataset_ops.Options()
-      options.experimental_optimization.apply_default_optimizations = False
-      return dataset.with_options(options)
-
-    if context.executing_eagerly():
-      input_t = np.ones([1, 512, 1024, 1025], np.int32)
-      get_next = self.getNext(dataset_fn(input_t))
-      self.evaluate(get_next())
-    else:
-      input_t = array_ops.placeholder(dtypes.int32, (None, None, None, None))
-      iterator = dataset_ops.make_initializable_iterator(dataset_fn(input_t))
-      init_op = iterator.initializer
-      get_next = iterator.get_next()
-
-      with self.cached_session() as sess:
-        sess.run(init_op, {input_t: np.ones([1, 512, 1024, 1025], np.int32)})
-        self.evaluate(get_next)
+  # TODO(b/123300735): Add eager coverage for the following tests.
+  def testSkipEagerOptimizationLargeInputFromTensor(self):
+    input_t = array_ops.placeholder(dtypes.int32, (None, None, None))
+    dataset = dataset_ops.Dataset.from_tensors(input_t)
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
+    iterator = dataset_ops.make_initializable_iterator(dataset)
+    init_op = iterator.initializer
+    get_next = iterator.get_next()
+
+    with self.cached_session() as sess:
+      sess.run(init_op, {input_t: np.ones([512, 1024, 1025], np.int32)})
+      self.evaluate(get_next)
+
+  # TODO(b/117581999): Add eager coverage for the following tests.
+  def testSkipEagerOptimizationLargeInputFromTensorSlices(self):
+    input_t = array_ops.placeholder(dtypes.int32, (None, None, None, None))
+    dataset = dataset_ops.Dataset.from_tensor_slices(input_t)
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
+    iterator = dataset_ops.make_initializable_iterator(dataset)
+    init_op = iterator.initializer
+    get_next = iterator.get_next()
+
+    with self.cached_session() as sess:
+      sess.run(init_op, {input_t: np.ones([1, 512, 1024, 1025], np.int32)})
+      self.evaluate(get_next)
 
   def testOptimizationNestedDataset(self):
 
-- 
GitLab


From 5f8ea841b0d509e05e008aac3b5930aaf4b52846 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Wed, 23 Jan 2019 15:07:28 -0800
Subject: [PATCH 1143/2345] [TF:XLA] Bump open source llvm revision to r351969

PiperOrigin-RevId: 230611280
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index b55b9c4c57..b2814fcb31 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -513,11 +513,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "d78389159aee7c3465b49843b46a8ac1b37db28db09dcf7522f6d31f32e5c97c",
-        strip_prefix = "llvm-316e6ae08e262dee261d1ca53a685b5ca6757a7f",
+        sha256 = "eff37dd30ccd52a76ca32428b2d20663402a1f75b55a808aba56fa6aa9979895",
+        strip_prefix = "llvm-0c8614fd4d276a31d2b5907b1c75d0003251cbf0",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/316e6ae08e262dee261d1ca53a685b5ca6757a7f.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/316e6ae08e262dee261d1ca53a685b5ca6757a7f.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/0c8614fd4d276a31d2b5907b1c75d0003251cbf0.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/0c8614fd4d276a31d2b5907b1c75d0003251cbf0.tar.gz",
         ],
     )
 
-- 
GitLab


From de1fb8d19f3e5a8062fd54dcc99e02ab4b4c4090 Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Wed, 23 Jan 2019 15:32:39 -0800
Subject: [PATCH 1144/2345] Cleanup the DnnSupport::DoConvolve API. It's
 similar to PrepareForConvolution: * Turn the public interface into template,
 non-virtual public function, which calls into * public, type-erased, virtual
 function.

Number of overloads is shrinked by the number of primitive types.

PiperOrigin-RevId: 230615784
---
 tensorflow/stream_executor/cuda/cuda_dnn.cc | 652 ++++++--------------
 tensorflow/stream_executor/cuda/cuda_dnn.h  | 153 +----
 tensorflow/stream_executor/dnn.cc           |  10 +
 tensorflow/stream_executor/dnn.h            | 155 ++---
 4 files changed, 261 insertions(+), 709 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index 62df7e6eff..ef2e13d314 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -82,17 +82,6 @@ static_assert(CUDNN_VERSION >= 6000, "cuDNN needs to be version 6.0 or higher");
     }                                                                    \
   } while (false)
 
-// Returns whether status is 'ok', and potentially logs the error.
-bool IsStatusOk(const port::Status& status, bool report_error) {
-  if (status.ok()) {
-    return true;
-  }
-  if (report_error) {
-    LOG(ERROR) << status.error_message();
-  }
-  return false;
-}
-
 // Converts (via narrowing) a type T value to a type U, and checks that the
 // value has no value change due to the conversion.
 template <typename WideT, typename NarrowT>
@@ -2778,17 +2767,15 @@ bool ShouldIncludeWinogradNonfusedAlgo(
 }
 #endif
 
-template <class ElementType>
 dnn::ConvolutionProto GenerateConvProto(
-    dnn::ConvolutionKind kind, const dnn::BatchDescriptor& input_descriptor,
+    dnn::ConvolutionKind kind, dnn::DataType element_type,
+    const dnn::BatchDescriptor& input_descriptor,
     const dnn::FilterDescriptor& filter_descriptor,
     const dnn::BatchDescriptor& output_descriptor, dnn::AlgorithmDesc algorithm,
     const dnn::ConvolutionDescriptor& convolution_descriptor, double conv_scale,
     double side_value_scale, dnn::DataType acc_type,
     dnn::ActivationMode activation) {
   dnn::ConvolutionProto conv_config;
-  auto element_type = dnn::ToDataType<ElementType>::value;
-
   conv_config.set_kind(kind);
   *conv_config.mutable_input() = input_descriptor.ToProto(element_type);
   *conv_config.mutable_filter() = filter_descriptor.ToProto(element_type);
@@ -2897,21 +2884,20 @@ port::Status CudnnSupport::DoPrepareForConvolution(
   return port::Status::OK();
 }
 
-template <class T>
-port::Status CudnnSupport::DoConvolveImpl(
-    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-    const DeviceMemory<T>& input_data,
+port::Status CudnnSupport::DoConvolve(
+    dnn::ConvolutionKind kind, dnn::DataType element_type, Stream* stream,
+    const dnn::BatchDescriptor& input_descriptor, DeviceMemoryBase input_data,
     const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<T>& filter_data,
+    DeviceMemoryBase filter_data, const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemoryBase output_data,
     const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& output_descriptor, DeviceMemory<T>* output_data,
-    dnn::DataType accumulator_type, const dnn::AlgorithmDesc& algorithm_desc,
-    DeviceMemory<uint8>* scratch_memory,
+    dnn::AlgorithmDesc algorithm_desc, DeviceMemory<uint8> scratch_memory,
     dnn::ProfileResult* output_profile_result) {
-  cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
+  cudnnDataType_t cudnn_type = ToCudnnDataType(element_type);
   CudnnTensorDescriptor input_nd(input_descriptor, cudnn_type);
   CudnnTensorDescriptor output_nd(output_descriptor, cudnn_type);
-  CudnnFilterDescriptor filter(filter_descriptor, cudnn_type);
+  CudnnFilterDescriptor filter_nd(filter_descriptor, cudnn_type);
+  auto accumulator_type = GetConvAccumulatorType(element_type);
   CudnnConvolutionDescriptor conv(convolution_descriptor,
                                   ToCudnnDataType(accumulator_type));
 
@@ -2940,10 +2926,10 @@ port::Status CudnnSupport::DoConvolveImpl(
     }
   }
 
-  // Report an error if we might be hitting a cuDNN bug that accesses illegal
-  // memory. See nvbugs/2138754, b/80018418.
-  if (CUDNN_VERSION < 7300) {
-    SE_RETURN_IF_ERROR([&] {
+  auto get_fwd_bugs = [&]() -> port::Status {
+    // Report an error if we might be hitting a cuDNN bug that accesses illegal
+    // memory. See nvbugs/2138754, b/80018418.
+    if (CUDNN_VERSION < 7300) {
       if (algorithm_desc.algo_id() != CUDNN_CONVOLUTION_FWD_ALGO_FFT_TILING) {
         return port::Status::OK();
       }
@@ -2966,26 +2952,162 @@ port::Status CudnnSupport::DoConvolveImpl(
       SE_RETURN_IF_ERROR(check_sizes(input_descriptor.count(),
                                      output_descriptor.feature_map_count()));
       return port::Status::OK();
-    }());
-  }
+    }
+    if (algorithm_desc.algo_id() ==
+            CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
+        !ShouldIncludeWinogradNonfusedAlgo(input_descriptor,
+                                           output_descriptor)) {
+      return port::Status(
+          port::error::FAILED_PRECONDITION,
+          "This configuration has potential integer overflow in "
+          "cuDNNv5 and cuDNNv6. See b/68264959.");
+    }
+    return port::Status::OK();
+  };
 
-  if (algorithm_desc.algo_id() ==
-          CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
-      !ShouldIncludeWinogradNonfusedAlgo(input_descriptor, output_descriptor)) {
-    return port::Status(port::error::FAILED_PRECONDITION,
-                        "This configuration has potential integer overflow in "
-                        "cuDNNv5 and cuDNNv6. See b/68264959.");
-  }
+  auto get_bwd_data_bugs = [&]() -> port::Status {
+    if (algorithm_desc.algo_id() ==
+            CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
+        !ShouldIncludeWinogradNonfusedAlgo(input_descriptor,
+                                           output_descriptor)) {
+      return port::Status(
+          port::error::FAILED_PRECONDITION,
+          "This configuration has potential integer overflow in "
+          "cuDNNv5 and cuDNNv6. See b/68264959.");
+    }
 
-  RETURN_IF_CUDNN_ERROR(cudnnConvolutionForward(
-      cudnn.handle(),
-      /*alpha=*/alpha, /*srcDesc=*/input_nd.handle(),
-      /*srcData=*/input_data.opaque(), /*filterDesc=*/filter.handle(),
-      /*filterData=*/filter_data.opaque(), /*convDesc=*/conv.handle(),
-      /*algo=*/ToConvForwardAlgo(algorithm_desc),
-      /*workSpace=*/scratch_memory->opaque(),
-      /*workSpaceSizeInBytes=*/scratch_memory->size(), /*beta=*/beta,
-      /*yDesc=*/output_nd.handle(), /*y=*/output_data->opaque()));
+    // Cudnn 7.1.4 has a bug if the workspace of the following convolution is
+    // not zero-initialized, nvbugs/2254619.
+    if (CUDNN_VERSION >= 7000 && CUDNN_VERSION < 7300 &&
+        algorithm_desc.algo_id() == CUDNN_CONVOLUTION_BWD_DATA_ALGO_1 &&
+        cudnn_type == CUDNN_DATA_HALF && algorithm_desc.tensor_ops_enabled() &&
+        input_descriptor.layout() == dnn::DataLayout::kBatchYXDepth &&
+        filter_descriptor.layout() == dnn::FilterLayout::kOutputInputYX &&
+        output_descriptor.layout() == dnn::DataLayout::kBatchDepthYX &&
+        (convolution_descriptor.vertical_filter_stride() > 1 ||
+         convolution_descriptor.horizontal_filter_stride() > 1)) {
+      stream->ThenMemZero(&scratch_memory, scratch_memory.size());
+    }
+    return port::Status::OK();
+  };
+
+  auto get_bwd_filter_bugs = [&]() -> port::Status {
+    // Report an error if we might be hitting a cuDNN bug that produces
+    // incorrect results. See nvbugs/2072856
+    if (CUDNN_VERSION < 7300) {
+      SE_RETURN_IF_ERROR([&] {
+        if (algorithm_desc.algo_id() !=
+            CUDNN_CONVOLUTION_BWD_FILTER_ALGO_FFT_TILING) {
+          return port::Status::OK();
+        }
+        if (output_descriptor.height() > 1 && output_descriptor.width() > 1) {
+          return port::Status::OK();
+        }
+        int convolution_size = output_descriptor.height() > 1
+                                   ? filter_descriptor.input_filter_height()
+                                   : filter_descriptor.input_filter_width();
+        if (convolution_size <= 32) {
+          return port::Status::OK();
+        }
+        cudnnConvolutionMode_t convolution_mode;
+        cudnnDataType_t compute_type;
+        RETURN_IF_CUDNN_ERROR(cudnnGetConvolutionNdDescriptor(
+            conv.handle(), 0, nullptr, nullptr, nullptr, nullptr,
+            &convolution_mode, &compute_type));
+        if (convolution_mode != CUDNN_CONVOLUTION) {
+          return port::Status::OK();
+        }
+        return port::Status(
+            port::error::FAILED_PRECONDITION,
+            "This configuration potentially produces incorrect results.");
+      }());
+    }
+
+    if (algorithm_desc.algo_id() ==
+            CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
+        !ShouldIncludeWinogradNonfusedAlgo(input_descriptor,
+                                           output_descriptor)) {
+      return port::Status(
+          port::error::FAILED_PRECONDITION,
+          "This configuration has potential integer overflow in "
+          "cuDNNv5 and cuDNNv6. See b/68264959.");
+    }
+
+    // Zero out the result buffer for strided conv backward filter for NHWC
+    // layouts. cuDNN 7.1.4 and 7.2 has non-determinisic bug if the buffer is
+    // not zeroed.
+    //
+    // This wrong result caused by the bug is very flaky. It needs to be run for
+    // up to 20 times to produce a mismatch.
+    //
+    // See nvbugs/2379553.
+    if (CUDNN_VERSION >= 7100 && CUDNN_VERSION < 7300 &&
+        algorithm_desc.algo_id() == CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1 &&
+        cudnn_type == CUDNN_DATA_HALF &&
+        input_descriptor.layout() == dnn::DataLayout::kBatchYXDepth &&
+        filter_descriptor.layout() == dnn::FilterLayout::kOutputYXInput &&
+        output_descriptor.layout() == dnn::DataLayout::kBatchYXDepth &&
+        (convolution_descriptor.vertical_filter_stride() > 1 ||
+         convolution_descriptor.horizontal_filter_stride() > 1)) {
+      stream->ThenMemZero(&filter_data, filter_data.size());
+    }
+    return port::Status::OK();
+  };
+
+  switch (kind) {
+    case dnn::ConvolutionKind::FORWARD: {
+      SE_RETURN_IF_ERROR(get_fwd_bugs());
+      RETURN_IF_CUDNN_ERROR(cudnnConvolutionForward(
+          cudnn.handle(),
+          /*alpha=*/alpha, /*srcDesc=*/input_nd.handle(),
+          /*srcData=*/input_data.opaque(), /*filterDesc=*/filter_nd.handle(),
+          /*filterData=*/filter_data.opaque(), /*convDesc=*/conv.handle(),
+          /*algo=*/ToConvForwardAlgo(algorithm_desc),
+          /*workSpace=*/scratch_memory.opaque(),
+          /*workSpaceSizeInBytes=*/scratch_memory.size(), /*beta=*/beta,
+          /*yDesc=*/output_nd.handle(), /*y=*/output_data.opaque()));
+      break;
+    }
+    case dnn::ConvolutionKind::BACKWARD_DATA: {
+      SE_RETURN_IF_ERROR(get_bwd_data_bugs());
+      RETURN_IF_CUDNN_ERROR(cudnnConvolutionBackwardData(
+          cudnn.handle(),
+          /*alpha=*/alpha,
+          /*wDesc=*/filter_nd.handle(),
+          /*w=*/filter_data.opaque(),
+          /*dyDesc=*/output_nd.handle(),
+          /*dy=*/output_data.opaque(),
+          /*convDesc=*/conv.handle(),
+          /*algo=*/ToConvBackwardDataAlgo(algorithm_desc),
+          /*workSpace=*/scratch_memory.opaque(),
+          /*workSpaceSizeInBytes=*/scratch_memory.size(),
+          /*beta=*/beta,
+          /*dxDesc=*/input_nd.handle(),
+          /*dx=*/input_data.opaque()));
+      break;
+    }
+    case dnn::ConvolutionKind::BACKWARD_FILTER: {
+      SE_RETURN_IF_ERROR(get_bwd_filter_bugs());
+      RETURN_IF_CUDNN_ERROR(cudnnConvolutionBackwardFilter(
+          cudnn.handle(),
+          /*alpha=*/alpha,
+          /*srcDesc=*/input_nd.handle(),
+          /*srcData=*/input_data.opaque(),
+          /*diffDesc=*/output_nd.handle(),
+          /*diffData=*/output_data.opaque(),
+          /*convDesc=*/conv.handle(),
+          /*algo=*/ToConvBackwardFilterAlgo(algorithm_desc),
+          /*workSpace=*/scratch_memory.opaque(),
+          /*workSpaceSizeInBytes=*/scratch_memory.size(),
+          /*beta=*/beta,
+          /*gradDesc=*/filter_nd.handle(),
+          /*dw=*/filter_data.opaque()));
+      break;
+    }
+    default:
+      return port::InternalError(
+          absl::StrCat("Unexpected convolution kind ", static_cast<int>(kind)));
+  }
 
   if (is_profiling) {
     if (!timer->Stop(AsCUDAStream(stream))) {
@@ -2994,13 +3116,13 @@ port::Status CudnnSupport::DoConvolveImpl(
     output_profile_result->set_algorithm(algorithm_desc);
     output_profile_result->set_elapsed_time_in_ms(
         timer->GetElapsedMilliseconds());
-    output_profile_result->set_scratch_size(scratch_memory->size());
+    output_profile_result->set_scratch_size(scratch_memory.size());
 
     LogCudaProto(
-        GenerateConvProto<T>(
-            dnn::ConvolutionKind::FORWARD, input_descriptor, filter_descriptor,
-            output_descriptor, algorithm_desc, convolution_descriptor, dalpha,
-            dbeta, accumulator_type, dnn::ActivationMode::kNone),
+        GenerateConvProto(kind, element_type, input_descriptor,
+                          filter_descriptor, output_descriptor, algorithm_desc,
+                          convolution_descriptor, dalpha, dbeta,
+                          accumulator_type, dnn::ActivationMode::kNone),
         output_profile_result->elapsed_time_in_ms(), stream->parent());
   }
 
@@ -3120,12 +3242,13 @@ port::Status CudnnSupport::DoFusedConvolveImpl(
         timer->GetElapsedMilliseconds());
     output_profile_result->set_scratch_size(scratch.size());
 
-    LogCudaProto(GenerateConvProto<ElementType>(
-                     dnn::ConvolutionKind::FORWARD, conv_input_descriptor,
-                     filter_descriptor, output_descriptor, algo_desc,
-                     convolution_descriptor, conv_input_scale, side_input_scale,
-                     accumulator_type, activation_mode),
-                 output_profile_result->elapsed_time_in_ms(), stream->parent());
+    LogCudaProto(
+        GenerateConvProto(
+            dnn::ConvolutionKind::FORWARD, dnn::ToDataType<ElementType>::value,
+            conv_input_descriptor, filter_descriptor, output_descriptor,
+            algo_desc, convolution_descriptor, conv_input_scale,
+            side_input_scale, accumulator_type, activation_mode),
+        output_profile_result->elapsed_time_in_ms(), stream->parent());
   }
 
   return port::Status::OK();
@@ -3436,62 +3559,6 @@ port::Status CudnnSupport::DoBatchNormalizationBackwardImpl(
   return port::Status::OK();
 }
 
-bool CudnnSupport::DoConvolve(
-    Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
-    const DeviceMemory<float>& input_data,
-    const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<float>& filter_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<float>* output_data, const dnn::AlgorithmDesc& algorithm_desc,
-    DeviceMemory<uint8>* scratch_memory,
-    dnn::ProfileResult* output_profile_result) {
-  return IsStatusOk(
-      DoConvolveImpl(stream, batch_descriptor, input_data, filter_descriptor,
-                     filter_data, convolution_descriptor, output_descriptor,
-                     output_data, GetConvAccumulatorType(dnn::DataType::kFloat),
-                     algorithm_desc, scratch_memory, output_profile_result),
-      /*report_error=*/!output_profile_result);
-}
-
-bool CudnnSupport::DoConvolve(
-    Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
-    const DeviceMemory<double>& input_data,
-    const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<double>& filter_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<double>* output_data, const dnn::AlgorithmDesc& algorithm_desc,
-    DeviceMemory<uint8>* scratch_memory,
-    dnn::ProfileResult* output_profile_result) {
-  return IsStatusOk(
-      DoConvolveImpl(stream, batch_descriptor, input_data, filter_descriptor,
-                     filter_data, convolution_descriptor, output_descriptor,
-                     output_data,
-                     GetConvAccumulatorType(dnn::DataType::kDouble),
-                     algorithm_desc, scratch_memory, output_profile_result),
-      /*report_error=*/!output_profile_result);
-}
-
-bool CudnnSupport::DoConvolve(
-    Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
-    const DeviceMemory<Eigen::half>& input_data,
-    const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<Eigen::half>& filter_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<Eigen::half>* output_data,
-    const dnn::AlgorithmDesc& algorithm_desc,
-    DeviceMemory<uint8>* scratch_memory,
-    dnn::ProfileResult* output_profile_result) {
-  return IsStatusOk(
-      DoConvolveImpl(stream, batch_descriptor, input_data, filter_descriptor,
-                     filter_data, convolution_descriptor, output_descriptor,
-                     output_data, GetConvAccumulatorType(dnn::DataType::kHalf),
-                     algorithm_desc, scratch_memory, output_profile_result),
-      /*report_error=*/!output_profile_result);
-}
-
 bool CudnnSupport::DoFusedConvolve(
     Stream* stream, const dnn::BatchDescriptor& conv_input_descriptor,
     const DeviceMemory<double>& conv_input_data, double conv_input_scale,
@@ -3619,363 +3686,6 @@ bool CudnnSupport::DoTransformTensor(Stream* stream,
   return IsStatusOk(status, /*report_error=*/true);
 }
 
-template <class T>
-port::Status CudnnSupport::DoConvolveBackwardDataImpl(
-    Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<T>& filter_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<T> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& input_descriptor,
-    DeviceMemory<T>* backward_input_data, dnn::DataType accumulator_type,
-    const dnn::AlgorithmDesc& algorithm_desc,
-    DeviceMemory<uint8>* scratch_memory,
-    dnn::ProfileResult* output_profile_result) {
-  cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
-  // Alpha is the scaling factor for input.
-  float falpha = 1.0;
-  double dalpha = 1.0;
-  void* alpha = cudnn_type == CUDNN_DATA_DOUBLE ? static_cast<void*>(&dalpha)
-                                                : static_cast<void*>(&falpha);
-  // Beta is the scaling factor for output.
-  float fbeta = 0.0;
-  double dbeta = 0.0;
-  void* beta = cudnn_type == CUDNN_DATA_DOUBLE ? static_cast<void*>(&dbeta)
-                                               : static_cast<void*>(&fbeta);
-
-  auto cudnn = cudnn_->GetHandle(parent_, stream);
-
-  CudnnTensorDescriptor out_back_nd(output_descriptor, cudnn_type);
-  CudnnTensorDescriptor in_back_nd(input_descriptor, cudnn_type);
-  CudnnFilterDescriptor filter(filter_descriptor, cudnn_type);
-  CudnnConvolutionDescriptor conv(convolution_descriptor,
-                                  ToCudnnDataType(accumulator_type));
-
-  const bool is_profiling = output_profile_result != nullptr;
-
-  std::unique_ptr<CUDATimer, TimerDeleter> timer;
-  if (is_profiling) {
-    timer.reset(new CUDATimer(parent_));  // NOLINT
-    // The start and stop of the timer should be as close to the Cudnn call as
-    // possible. It is still possible for other threads to issue workload on
-    // to this stream. So it could take multiple profiling measurements.
-    if (!timer->Init() || !timer->Start(AsCUDAStream(stream))) {
-      return port::Status(port::error::INTERNAL, "Failed to start timer");
-    }
-  }
-
-  if (algorithm_desc.algo_id() ==
-          CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
-      !ShouldIncludeWinogradNonfusedAlgo(input_descriptor, output_descriptor)) {
-    return port::Status(port::error::FAILED_PRECONDITION,
-                        "This configuration has potential integer overflow in "
-                        "cuDNNv5 and cuDNNv6. See b/68264959.");
-  }
-
-  // Cudnn 7.1.4 has a bug if the workspace of the following convolution is not
-  // zero-initialized, nvbugs/2254619.
-  if (CUDNN_VERSION >= 7000 && CUDNN_VERSION < 7300 &&
-      algorithm_desc.algo_id() == CUDNN_CONVOLUTION_BWD_DATA_ALGO_1 &&
-      cudnn_type == CUDNN_DATA_HALF && algorithm_desc.tensor_ops_enabled() &&
-      input_descriptor.layout() == dnn::DataLayout::kBatchYXDepth &&
-      filter_descriptor.layout() == dnn::FilterLayout::kOutputInputYX &&
-      output_descriptor.layout() == dnn::DataLayout::kBatchDepthYX &&
-      (convolution_descriptor.vertical_filter_stride() > 1 ||
-       convolution_descriptor.horizontal_filter_stride() > 1)) {
-    stream->ThenMemZero(scratch_memory, scratch_memory->size());
-  }
-
-  RETURN_IF_CUDNN_ERROR(cudnnConvolutionBackwardData(
-      cudnn.handle(),
-      /*alpha=*/alpha,
-      /*wDesc=*/filter.handle(),
-      /*w=*/filter_data.opaque(),
-      /*dyDesc=*/out_back_nd.handle(),
-      /*dy=*/backward_output_data.opaque(),
-      /*convDesc=*/conv.handle(),
-      /*algo=*/ToConvBackwardDataAlgo(algorithm_desc),
-      /*workSpace=*/scratch_memory->opaque(),
-      /*workSpaceSizeInBytes=*/scratch_memory->size(),
-      /*beta=*/beta,
-      /*dxDesc=*/in_back_nd.handle(),
-      /*dx=*/backward_input_data->opaque()));
-  if (is_profiling) {
-    if (!timer->Stop(AsCUDAStream(stream))) {
-      return port::Status(port::error::INTERNAL, "Failed to stop timer");
-    }
-    output_profile_result->set_algorithm(algorithm_desc);
-    output_profile_result->set_elapsed_time_in_ms(
-        timer->GetElapsedMilliseconds());
-    output_profile_result->set_scratch_size(scratch_memory->size());
-
-    LogCudaProto(GenerateConvProto<T>(
-                     dnn::ConvolutionKind::BACKWARD_DATA, input_descriptor,
-                     filter_descriptor, output_descriptor, algorithm_desc,
-                     convolution_descriptor, dalpha, dbeta, accumulator_type,
-                     dnn::ActivationMode::kNone),
-                 output_profile_result->elapsed_time_in_ms(), stream->parent());
-  }
-
-  return port::Status::OK();
-}
-
-bool CudnnSupport::DoConvolveBackwardData(
-    Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<double>& filter_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<double> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& input_descriptor,
-    DeviceMemory<double>* backward_input_data,
-    const dnn::AlgorithmDesc& algorithm_desc,
-    DeviceMemory<uint8>* scratch_memory,
-    dnn::ProfileResult* output_profile_result) {
-  return IsStatusOk(
-      DoConvolveBackwardDataImpl(
-          stream, filter_descriptor, filter_data, output_descriptor,
-          backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, GetConvAccumulatorType(dnn::DataType::kDouble),
-          algorithm_desc, scratch_memory, output_profile_result),
-      /*report_error=*/!output_profile_result);
-}
-
-bool CudnnSupport::DoConvolveBackwardData(
-    Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<float>& filter_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<float> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& input_descriptor,
-    DeviceMemory<float>* backward_input_data,
-    const dnn::AlgorithmDesc& algorithm_desc,
-    DeviceMemory<uint8>* scratch_memory,
-    dnn::ProfileResult* output_profile_result) {
-  return IsStatusOk(
-      DoConvolveBackwardDataImpl(
-          stream, filter_descriptor, filter_data, output_descriptor,
-          backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, GetConvAccumulatorType(dnn::DataType::kFloat),
-          algorithm_desc, scratch_memory, output_profile_result),
-      /*report_error=*/!output_profile_result);
-}
-
-bool CudnnSupport::DoConvolveBackwardData(
-    Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<Eigen::half>& filter_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<Eigen::half> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& input_descriptor,
-    DeviceMemory<Eigen::half>* backward_input_data,
-    const dnn::AlgorithmDesc& algorithm_desc,
-    DeviceMemory<uint8>* scratch_memory,
-    dnn::ProfileResult* output_profile_result) {
-  return IsStatusOk(
-      DoConvolveBackwardDataImpl(
-          stream, filter_descriptor, filter_data, output_descriptor,
-          backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, GetConvAccumulatorType(dnn::DataType::kHalf),
-          algorithm_desc, scratch_memory, output_profile_result),
-      /*report_error=*/!output_profile_result);
-}
-
-template <class T>
-port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
-    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-    const DeviceMemory<T>& input_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<T> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::FilterDescriptor& filter_descriptor,
-    DeviceMemory<T>* backward_filter_data, dnn::DataType accumulator_type,
-    const dnn::AlgorithmDesc& algorithm_desc,
-    DeviceMemory<uint8>* scratch_memory,
-    dnn::ProfileResult* output_profile_result) {
-  cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
-  // Alpha is the scaling factor for input.
-  float falpha = 1.0;
-  double dalpha = 1.0;
-  void* alpha = cudnn_type == CUDNN_DATA_DOUBLE ? static_cast<void*>(&dalpha)
-                                                : static_cast<void*>(&falpha);
-  // Beta is the scaling factor for output.
-  float fbeta = 0.0;
-  double dbeta = 0.0;
-  void* beta = cudnn_type == CUDNN_DATA_DOUBLE ? static_cast<void*>(&dbeta)
-                                               : static_cast<void*>(&fbeta);
-
-  auto cudnn = cudnn_->GetHandle(parent_, stream);
-
-  CudnnTensorDescriptor out_back_nd(output_descriptor, cudnn_type);
-  CudnnTensorDescriptor input_nd(input_descriptor, cudnn_type);
-  CudnnFilterDescriptor filter(filter_descriptor, cudnn_type);
-  CudnnConvolutionDescriptor conv(convolution_descriptor,
-                                  ToCudnnDataType(accumulator_type));
-
-  const bool is_profiling = output_profile_result != nullptr;
-
-  std::unique_ptr<CUDATimer, TimerDeleter> timer;
-  if (is_profiling) {
-    timer.reset(new CUDATimer(parent_));  // NOLINT
-    // The start and stop of the timer should be as close to the Cudnn call as
-    // possible. It is still possible for other threads to issue workload on
-    // to this stream. So it could take multiple profiling measurements.
-    if (!timer->Init() || !timer->Start(AsCUDAStream(stream))) {
-      return port::Status(port::error::INTERNAL, "Failed to start timer");
-    }
-  }
-
-  // Report an error if we might be hitting a cuDNN bug that produces incorrect
-  // results. See nvbugs/2072856
-  if (CUDNN_VERSION < 7300) {
-    SE_RETURN_IF_ERROR([&] {
-      if (algorithm_desc.algo_id() !=
-          CUDNN_CONVOLUTION_BWD_FILTER_ALGO_FFT_TILING) {
-        return port::Status::OK();
-      }
-      if (output_descriptor.height() > 1 && output_descriptor.width() > 1) {
-        return port::Status::OK();
-      }
-      int convolution_size = output_descriptor.height() > 1
-                                 ? filter_descriptor.input_filter_height()
-                                 : filter_descriptor.input_filter_width();
-      if (convolution_size <= 32) {
-        return port::Status::OK();
-      }
-      cudnnConvolutionMode_t convolution_mode;
-      cudnnDataType_t compute_type;
-      RETURN_IF_CUDNN_ERROR(cudnnGetConvolutionNdDescriptor(
-          conv.handle(), 0, nullptr, nullptr, nullptr, nullptr,
-          &convolution_mode, &compute_type));
-      if (convolution_mode != CUDNN_CONVOLUTION) {
-        return port::Status::OK();
-      }
-      return port::Status(
-          port::error::FAILED_PRECONDITION,
-          "This configuration potentially produces incorrect results.");
-    }());
-  }
-
-  if (algorithm_desc.algo_id() ==
-          CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
-      !ShouldIncludeWinogradNonfusedAlgo(input_descriptor, output_descriptor)) {
-    return port::Status(port::error::FAILED_PRECONDITION,
-                        "This configuration has potential integer overflow in "
-                        "cuDNNv5 and cuDNNv6. See b/68264959.");
-  }
-
-  // Zero out the result buffer for strided conv backward filter for NHWC
-  // layouts. cuDNN 7.1.4 and 7.2 has non-determinisic bug if the buffer is not
-  // zeroed.
-  //
-  // This wrong result caused by the bug is very flaky. It needs to be run for
-  // up to 20 times to produce a mismatch.
-  //
-  // See nvbugs/2379553.
-  if (CUDNN_VERSION >= 7100 && CUDNN_VERSION < 7300 &&
-      algorithm_desc.algo_id() == CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1 &&
-      cudnn_type == CUDNN_DATA_HALF &&
-      input_descriptor.layout() == dnn::DataLayout::kBatchYXDepth &&
-      filter_descriptor.layout() == dnn::FilterLayout::kOutputYXInput &&
-      output_descriptor.layout() == dnn::DataLayout::kBatchYXDepth &&
-      (convolution_descriptor.vertical_filter_stride() > 1 ||
-       convolution_descriptor.horizontal_filter_stride() > 1)) {
-    stream->ThenMemZero(backward_filter_data, backward_filter_data->size());
-  }
-
-  RETURN_IF_CUDNN_ERROR(cudnnConvolutionBackwardFilter(
-      cudnn.handle(),
-      /*alpha=*/alpha,
-      /*srcDesc=*/input_nd.handle(),
-      /*srcData=*/input_data.opaque(),
-      /*diffDesc=*/out_back_nd.handle(),
-      /*diffData=*/backward_output_data.opaque(),
-      /*convDesc=*/conv.handle(),
-      /*algo=*/ToConvBackwardFilterAlgo(algorithm_desc),
-      /*workSpace=*/scratch_memory->opaque(),
-      /*workSpaceSizeInBytes=*/scratch_memory->size(),
-      /*beta=*/beta,
-      /*gradDesc=*/filter.handle(),
-      /*dw=*/backward_filter_data->opaque()));
-  if (is_profiling) {
-    if (!timer->Stop(AsCUDAStream(stream))) {
-      return port::Status(port::error::INTERNAL, "Failed to stop timer");
-    }
-    output_profile_result->set_algorithm(algorithm_desc);
-    output_profile_result->set_elapsed_time_in_ms(
-        timer->GetElapsedMilliseconds());
-    output_profile_result->set_scratch_size(scratch_memory->size());
-
-    LogCudaProto(GenerateConvProto<T>(
-                     dnn::ConvolutionKind::BACKWARD_FILTER, input_descriptor,
-                     filter_descriptor, output_descriptor, algorithm_desc,
-                     convolution_descriptor, dalpha, dbeta, accumulator_type,
-                     dnn::ActivationMode::kNone),
-                 output_profile_result->elapsed_time_in_ms(), stream->parent());
-  }
-
-  return port::Status::OK();
-}
-
-bool CudnnSupport::DoConvolveBackwardFilter(
-    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-    const DeviceMemory<double>& input_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<double> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::FilterDescriptor& filter_descriptor,
-    DeviceMemory<double>* backward_filter_data,
-    const dnn::AlgorithmDesc& algorithm_desc,
-    DeviceMemory<uint8>* scratch_memory,
-    dnn::ProfileResult* output_profile_result) {
-  return IsStatusOk(
-      DoConvolveBackwardFilterImpl(
-          stream, input_descriptor, input_data, output_descriptor,
-          backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, GetConvAccumulatorType(dnn::DataType::kDouble),
-          algorithm_desc, scratch_memory, output_profile_result),
-      /*report_error=*/!output_profile_result);
-}
-
-bool CudnnSupport::DoConvolveBackwardFilter(
-    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-    const DeviceMemory<float>& input_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<float> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::FilterDescriptor& filter_descriptor,
-    DeviceMemory<float>* backward_filter_data,
-    const dnn::AlgorithmDesc& algorithm_desc,
-    DeviceMemory<uint8>* scratch_memory,
-    dnn::ProfileResult* output_profile_result) {
-  return IsStatusOk(
-      DoConvolveBackwardFilterImpl(
-          stream, input_descriptor, input_data, output_descriptor,
-          backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, GetConvAccumulatorType(dnn::DataType::kFloat),
-          algorithm_desc, scratch_memory, output_profile_result),
-      /*report_error=*/!output_profile_result);
-}
-
-bool CudnnSupport::DoConvolveBackwardFilter(
-    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-    const DeviceMemory<Eigen::half>& input_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<Eigen::half> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::FilterDescriptor& filter_descriptor,
-    DeviceMemory<Eigen::half>* backward_filter_data,
-    const dnn::AlgorithmDesc& algorithm_desc,
-    DeviceMemory<uint8>* scratch_memory,
-    dnn::ProfileResult* output_profile_result) {
-  return IsStatusOk(
-      DoConvolveBackwardFilterImpl(
-          stream, input_descriptor, input_data, output_descriptor,
-          backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, GetConvAccumulatorType(dnn::DataType::kHalf),
-          algorithm_desc, scratch_memory, output_profile_result),
-      /*report_error=*/!output_profile_result);
-}
-
 template <class T>
 port::Status CudnnSupport::DoConvolveBackwardBiasImpl(
     Stream* stream, const dnn::BatchDescriptor& input_descriptor,
diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.h b/tensorflow/stream_executor/cuda/cuda_dnn.h
index 383a99e118..2fd311e887 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.h
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.h
@@ -258,38 +258,16 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<float>* scale_backprop,
       DeviceMemory<float>* offset_backprop) override;
 
-  bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
-                  const DeviceMemory<float>& input_data,
-                  const dnn::FilterDescriptor& filter_descriptor,
-                  const DeviceMemory<float>& filter_data,
-                  const dnn::ConvolutionDescriptor& convolution_descriptor,
-                  const dnn::BatchDescriptor& output_descriptor,
-                  DeviceMemory<float>* output_data,
-                  const dnn::AlgorithmDesc& algorithm_desc,
-                  DeviceMemory<uint8>* scratch_memory,
-                  dnn::ProfileResult* output_profile_result) override;
-
-  bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
-                  const DeviceMemory<double>& input_data,
-                  const dnn::FilterDescriptor& filter_descriptor,
-                  const DeviceMemory<double>& filter_data,
-                  const dnn::ConvolutionDescriptor& convolution_descriptor,
-                  const dnn::BatchDescriptor& output_descriptor,
-                  DeviceMemory<double>* output_data,
-                  const dnn::AlgorithmDesc& algorithm_desc,
-                  DeviceMemory<uint8>* scratch_memory,
-                  dnn::ProfileResult* output_profile_result) override;
-
-  bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
-                  const DeviceMemory<Eigen::half>& input_data,
-                  const dnn::FilterDescriptor& filter_descriptor,
-                  const DeviceMemory<Eigen::half>& filter_data,
-                  const dnn::ConvolutionDescriptor& convolution_descriptor,
-                  const dnn::BatchDescriptor& output_descriptor,
-                  DeviceMemory<Eigen::half>* output_data,
-                  const dnn::AlgorithmDesc& algorithm_desc,
-                  DeviceMemory<uint8>* scratch_memory,
-                  dnn::ProfileResult* output_profile_result) override;
+  port::Status DoConvolve(
+      dnn::ConvolutionKind kind, dnn::DataType element_type, Stream* stream,
+      const dnn::BatchDescriptor& input_descriptor, DeviceMemoryBase input_data,
+      const dnn::FilterDescriptor& filter_descriptor,
+      DeviceMemoryBase filter_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemoryBase output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      dnn::AlgorithmDesc algorithm_desc, DeviceMemory<uint8> scratch_memory,
+      dnn::ProfileResult* output_profile_result) override;
 
   bool DoFusedConvolve(
       Stream* stream, const dnn::BatchDescriptor& conv_input_descriptor,
@@ -390,78 +368,6 @@ class CudnnSupport : public dnn::DnnSupport {
     return false;
   }
 
-  bool DoConvolveBackwardData(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<double>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<double> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<double>* backward_input_data,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
-      dnn::ProfileResult* output_profile_result) override;
-
-  bool DoConvolveBackwardData(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<float>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<float> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<float>* backward_input_data,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
-      dnn::ProfileResult* output_profile_result) override;
-
-  bool DoConvolveBackwardData(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<Eigen::half>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<Eigen::half> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<Eigen::half>* backward_input_data,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
-      dnn::ProfileResult* output_profile_result) override;
-
-  bool DoConvolveBackwardFilter(
-      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-      const DeviceMemory<double>& input_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<double> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::FilterDescriptor& filter_descriptor,
-      DeviceMemory<double>* backward_filter_data,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
-      dnn::ProfileResult* output_profile_result) override;
-
-  bool DoConvolveBackwardFilter(
-      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-      const DeviceMemory<float>& input_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<float> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::FilterDescriptor& filter_descriptor,
-      DeviceMemory<float>* backward_filter_data,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
-      dnn::ProfileResult* output_profile_result) override;
-
-  bool DoConvolveBackwardFilter(
-      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-      const DeviceMemory<Eigen::half>& input_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<Eigen::half> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::FilterDescriptor& filter_descriptor,
-      DeviceMemory<Eigen::half>* backward_filter_data,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
-      dnn::ProfileResult* output_profile_result) override;
-
   bool DoConvolveBackwardBias(
       Stream* stream, const dnn::BatchDescriptor& input_descriptor,
       const DeviceMemory<double>& input_data,
@@ -676,19 +582,6 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<T>* x_backprop, DeviceMemory<U>* scale_backprop,
       DeviceMemory<U>* offset_backprop);
 
-  template <class T>
-  port::Status DoConvolveImpl(
-      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-      const DeviceMemory<T>& input_data,
-      const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<T>& filter_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<T>* output_data, dnn::DataType accumulator_type,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
-      dnn::ProfileResult* output_profile_result);
-
   template <typename ElementType, typename BiasType, typename ScaleType>
   port::Status DoFusedConvolveImpl(
       Stream* stream, const dnn::BatchDescriptor& conv_input_descriptor,
@@ -706,32 +599,6 @@ class CudnnSupport : public dnn::DnnSupport {
       const dnn::AlgorithmConfig& algorithm_config,
       dnn::ProfileResult* output_profile_result);
 
-  template <class T>
-  port::Status DoConvolveBackwardDataImpl(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<T>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<T> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<T>* backward_input_data, dnn::DataType accumulator_type,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
-      dnn::ProfileResult* output_profile_result);
-
-  template <class T>
-  port::Status DoConvolveBackwardFilterImpl(
-      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-      const DeviceMemory<T>& input_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<T> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::FilterDescriptor& filter_descriptor,
-      DeviceMemory<T>* backward_filter_data, dnn::DataType accumulator_type,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
-      dnn::ProfileResult* output_profile_result);
-
   template <class T>
   port::Status DoConvolveBackwardBiasImpl(
       Stream* stream, const dnn::BatchDescriptor& input_descriptor,
diff --git a/tensorflow/stream_executor/dnn.cc b/tensorflow/stream_executor/dnn.cc
index d91afaa638..fcc3db928b 100644
--- a/tensorflow/stream_executor/dnn.cc
+++ b/tensorflow/stream_executor/dnn.cc
@@ -581,5 +581,15 @@ string NormalizeDescriptor::ToShortString() const {
                       "_size:", segment_size_);
 }
 
+bool DnnSupport::IsStatusOk(const port::Status& status, bool report_error) {
+  if (status.ok()) {
+    return true;
+  }
+  if (report_error) {
+    LOG(ERROR) << status.error_message();
+  }
+  return false;
+}
+
 }  // namespace dnn
 }  // namespace stream_executor
diff --git a/tensorflow/stream_executor/dnn.h b/tensorflow/stream_executor/dnn.h
index f60d1ada24..1d9a2be951 100644
--- a/tensorflow/stream_executor/dnn.h
+++ b/tensorflow/stream_executor/dnn.h
@@ -1237,45 +1237,34 @@ class DnnSupport {
   //   that if the inverse of the filter is applied to the output in VALID mode
   //   the result is the same size as the input - this requires even more
   //   padding of the input.
-  virtual bool DoConvolve(
-      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-      const DeviceMemory<float>& input_data,
-      const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<float>& filter_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<float>* output_data,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
+  virtual port::Status DoConvolve(
+      ConvolutionKind kind, DataType element_type, Stream* stream,
+      const BatchDescriptor& input_descriptor, DeviceMemoryBase input_data,
+      const FilterDescriptor& filter_descriptor, DeviceMemoryBase filter_data,
+      const BatchDescriptor& output_descriptor, DeviceMemoryBase output_data,
+      const ConvolutionDescriptor& convolution_descriptor,
+      AlgorithmDesc algorithm_desc, DeviceMemory<uint8> scratch_memory,
       ProfileResult* output_profile_result) = 0;
 
-  // Enqueues a double-precision convolution operation onto the stream.
-  // See DoConvolve above for argument details.
-  virtual bool DoConvolve(
-      Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
-      const DeviceMemory<double>& input_data,
-      const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<double>& filter_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<double>* output_data,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
-      dnn::ProfileResult* output_profile_result) = 0;
-
-  // Enqueues a half-precision convolution operation onto the stream.
-  // See DoConvolve above for argument details.
-  virtual bool DoConvolve(
-      Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
-      const DeviceMemory<Eigen::half>& input_data,
-      const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<Eigen::half>& filter_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<Eigen::half>* output_data,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
-      ProfileResult* output_profile_result) = 0;
+  template <typename ElementType>
+  bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+                  const DeviceMemory<ElementType>& input_data,
+                  const dnn::FilterDescriptor& filter_descriptor,
+                  const DeviceMemory<ElementType>& filter_data,
+                  const dnn::ConvolutionDescriptor& convolution_descriptor,
+                  const dnn::BatchDescriptor& output_descriptor,
+                  DeviceMemory<ElementType>* output_data,
+                  const dnn::AlgorithmDesc& algorithm_desc,
+                  DeviceMemory<uint8>* scratch_memory,
+                  ProfileResult* output_profile_result) {
+    return IsStatusOk(
+        DoConvolve(ConvolutionKind::FORWARD, ToDataType<ElementType>::value,
+                   stream, input_descriptor, input_data, filter_descriptor,
+                   filter_data, output_descriptor, *output_data,
+                   convolution_descriptor, algorithm_desc, *scratch_memory,
+                   output_profile_result),
+        !output_profile_result);
+  }
 
   // Return a list of algorithms supported by the forward convolution pass.
   // cc_major and cc_minor are the compute capabilities of the device.
@@ -1348,17 +1337,27 @@ class DnnSupport {
   //    backprop of the input.
   //  scratch_allocator: un-owned, may-be-null object that may allocate scratch
   //    space in order to speed up the convolution operation.
-  virtual bool DoConvolveBackwardData(
+  template <typename ElementType>
+  bool DoConvolveBackwardData(
       Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<float>& filter_data,
+      const DeviceMemory<ElementType>& filter_data,
       const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<float> backward_output_data,
+      const DeviceMemory<ElementType>& backward_output_data,
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<float>* backward_input_data,
+      DeviceMemory<ElementType>* backward_input_data,
       const dnn::AlgorithmDesc& algorithm_desc,
       DeviceMemory<uint8>* scratch_memory,
-      ProfileResult* output_profile_result) = 0;
+      ProfileResult* output_profile_result) {
+    return IsStatusOk(
+        DoConvolve(ConvolutionKind::BACKWARD_DATA,
+                   ToDataType<ElementType>::value, stream, input_descriptor,
+                   *backward_input_data, filter_descriptor, filter_data,
+                   output_descriptor, backward_output_data,
+                   convolution_descriptor, algorithm_desc, *scratch_memory,
+                   output_profile_result),
+        !output_profile_result);
+  }
 
   // Return a list of algorithms supported by the backward convolution pass for
   // data.
@@ -1366,30 +1365,6 @@ class DnnSupport {
       bool with_winograd_nonfused, int cc_major, int cc_minor,
       std::vector<AlgorithmDesc>* out_algorithms);
 
-  virtual bool DoConvolveBackwardData(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<double>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<double> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<double>* backward_input_data,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
-      ProfileResult* output_profile_result) = 0;
-
-  virtual bool DoConvolveBackwardData(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<Eigen::half>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<Eigen::half> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<Eigen::half>* backward_input_data,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
-      ProfileResult* output_profile_result) = 0;
-
   // Enqueues a single-precision backward convolution (for filter) operation
   // onto the stream.
   //
@@ -1409,17 +1384,27 @@ class DnnSupport {
   //    backprop of the filter.
   //  scratch_allocator: un-owned, may-be-null object that may allocate scratch
   //    space in order to speed up the convolution operation.
-  virtual bool DoConvolveBackwardFilter(
+  template <typename ElementType>
+  bool DoConvolveBackwardFilter(
       Stream* stream, const BatchDescriptor& input_descriptor,
-      const DeviceMemory<float>& input_data,
+      const DeviceMemory<ElementType>& input_data,
       const BatchDescriptor& output_descriptor,
-      DeviceMemory<float> backward_output_data,
+      const DeviceMemory<ElementType>& backward_output_data,
       const ConvolutionDescriptor& convolution_descriptor,
       const FilterDescriptor& filter_descriptor,
-      DeviceMemory<float>* backward_filter_data,
+      DeviceMemory<ElementType>* backward_filter_data,
       const dnn::AlgorithmDesc& algorithm_desc,
       DeviceMemory<uint8>* scratch_memory,
-      ProfileResult* output_profile_result) = 0;
+      ProfileResult* output_profile_result) {
+    return IsStatusOk(
+        DoConvolve(ConvolutionKind::BACKWARD_FILTER,
+                   ToDataType<ElementType>::value, stream, input_descriptor,
+                   input_data, filter_descriptor, *backward_filter_data,
+                   output_descriptor, backward_output_data,
+                   convolution_descriptor, algorithm_desc, *scratch_memory,
+                   output_profile_result),
+        !output_profile_result);
+  }
 
   // Return a list of algorithms supported by the backward convolution pass for
   // filters.
@@ -1427,30 +1412,6 @@ class DnnSupport {
       bool with_winograd_nonfused, int cc_major, int cc_minor,
       std::vector<AlgorithmDesc>* out_algorithms);
 
-  virtual bool DoConvolveBackwardFilter(
-      Stream* stream, const BatchDescriptor& input_descriptor,
-      const DeviceMemory<double>& input_data,
-      const BatchDescriptor& output_descriptor,
-      DeviceMemory<double> backward_output_data,
-      const ConvolutionDescriptor& convolution_descriptor,
-      const FilterDescriptor& filter_descriptor,
-      DeviceMemory<double>* backward_filter_data,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
-      ProfileResult* output_profile_result) = 0;
-
-  virtual bool DoConvolveBackwardFilter(
-      Stream* stream, const BatchDescriptor& input_descriptor,
-      const DeviceMemory<Eigen::half>& input_data,
-      const BatchDescriptor& output_descriptor,
-      DeviceMemory<Eigen::half> backward_output_data,
-      const ConvolutionDescriptor& convolution_descriptor,
-      const FilterDescriptor& filter_descriptor,
-      DeviceMemory<Eigen::half>* backward_filter_data,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
-      ProfileResult* output_profile_result) = 0;
-
   // Enqueues a single-precision backward convolution (for bias) operation onto
   // the stream.
   //
@@ -2373,6 +2334,10 @@ class DnnSupport {
     return false;
   }
 
+ protected:
+  // Returns whether status is 'ok', and potentially logs the error.
+  static bool IsStatusOk(const port::Status& status, bool report_error);
+
  private:
   virtual port::Status DoPrepareForConvolution(
       ConvolutionKind kind, DataType element_type, Stream* stream,
-- 
GitLab


From 12888c45f6e1d12836bf79afdd1035f1c566ca9a Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Wed, 23 Jan 2019 15:34:07 -0800
Subject: [PATCH 1145/2345] [XLA] Fix the ShapeVerifier to report an error for
 non-array returning iota instructions.

Previously, the ShapeVerifier crashes when handling non-array returning iota
instructions.

Add a test case.

PiperOrigin-RevId: 230616042
---
 tensorflow/compiler/xla/service/hlo_verifier.cc |  3 +++
 .../compiler/xla/service/hlo_verifier_test.cc   | 17 +++++++++++++++++
 2 files changed, 20 insertions(+)

diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc
index 36340d3d78..144c01eac1 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier.cc
@@ -349,6 +349,9 @@ Status ShapeVerifier::HandleConstant(HloInstruction* constant) {
 Status ShapeVerifier::HandleIota(HloInstruction* instruction) {
   TF_RETURN_IF_ERROR(CheckOperandCount(instruction, 0));
   auto* iota = Cast<HloIotaInstruction>(instruction);
+  if (!iota->shape().IsArray()) {
+    return InternalError("Iota does not support non-array result.");
+  }
   const int64 rank = iota->shape().rank();
   if (rank == 0) {
     return InternalError("Iota does not support scalars.");
diff --git a/tensorflow/compiler/xla/service/hlo_verifier_test.cc b/tensorflow/compiler/xla/service/hlo_verifier_test.cc
index de0335a5e8..4f69bd155b 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier_test.cc
@@ -535,5 +535,22 @@ TEST_F(HloVerifierTestAllowMixedPrecision, SelectMixedPrecisionAllowed) {
   ASSERT_TRUE(status.ok());
 }
 
+TEST_F(HloVerifierTest, IotaNonArrayResult) {
+  const char* const hlo_string = R"(
+  HloModule IotaTupleResult
+
+  ENTRY  kernelEntry {
+    ROOT iota = () iota(), iota_dimension=24
+  }
+  )";
+
+  TF_ASSERT_OK_AND_ASSIGN(auto module, ParseHloString(hlo_string));
+
+  auto status = verifier().Run(module.get()).status();
+  ASSERT_FALSE(status.ok());
+  EXPECT_THAT(status.error_message(),
+              HasSubstr("does not support non-array result"));
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From c43361b59b913f13d42ab6fa8f434f1fe221c208 Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Wed, 23 Jan 2019 15:48:27 -0800
Subject: [PATCH 1146/2345] Rename task_index to task_id in cluster resolvers.

PiperOrigin-RevId: 230618484
---
 .../python/parameter_server_strategy.py       |  2 +-
 .../python/parameter_server_strategy_test.py  |  2 +-
 .../cluster_resolver/cluster_resolver.py      | 74 +++++++++----------
 .../cluster_resolver/cluster_resolver_test.py | 22 +++---
 .../cluster_resolver/gce_cluster_resolver.py  | 24 +++---
 .../gce_cluster_resolver_test.py              | 10 +--
 .../kubernetes_cluster_resolver.py            | 16 ++--
 .../kubernetes_cluster_resolver_test.py       |  4 +-
 .../slurm_cluster_resolver.py                 | 22 +++---
 .../slurm_cluster_resolver_test.py            |  2 +-
 .../tfconfig_cluster_resolver.py              | 24 +++---
 .../tfconfig_cluster_resolver_test.py         | 10 +--
 .../cluster_resolver/tpu_cluster_resolver.py  | 24 +++---
 .../tpu_cluster_resolver_test.py              |  4 +-
 .../distribute/parameter_server_strategy.py   |  4 +-
 15 files changed, 122 insertions(+), 122 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy.py b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
index 0da269204f..e42bc50fdc 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
@@ -157,7 +157,7 @@ class ParameterServerExtended(CoreParameterServerExtended):
     cluster_resolver = SimpleClusterResolver(
         cluster_spec=tfconfig.cluster_spec(),
         task_type=tfconfig.task_type,
-        task_index=tfconfig.task_index,
+        task_id=tfconfig.task_id,
         num_accelerators=num_gpus_per_worker)
     super(ParameterServerExtended, self).__init__(
         container_strategy, cluster_resolver=cluster_resolver)
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
index 9e7e201519..89dcdbcfc2 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
@@ -90,7 +90,7 @@ def create_test_objects(cluster_spec=None,
       cluster_resolver = SimpleClusterResolver(
           cluster_spec=multi_worker_util.normalize_cluster_spec(cluster_spec),
           task_type=task_type,
-          task_index=task_id,
+          task_id=task_id,
           num_accelerators=num_gpus)
       target = 'grpc://' + cluster_spec[WORKER][task_id]
     else:
diff --git a/tensorflow/python/distribute/cluster_resolver/cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/cluster_resolver.py
index 73188bd7ca..8cc7cff639 100644
--- a/tensorflow/python/distribute/cluster_resolver/cluster_resolver.py
+++ b/tensorflow/python/distribute/cluster_resolver/cluster_resolver.py
@@ -56,13 +56,13 @@ class ClusterResolver(object):
   underlying machine failures and scale TensorFlow worker clusters up and down.
 
   Note to Implementors: In addition to these abstract methods, you must also
-  implement the task_type, task_index, and rpc_layer attributes. You may choose
+  implement the task_type, task_id, and rpc_layer attributes. You may choose
   to implement them either as properties with getters or setters or directly
   set the attributes.
 
   - task_type is the name of the server's current named job (e.g. 'worker',
      'ps' in a distributed parameterized training job).
-  - task_index is the ordinal index of the server within the task type.
+  - task_id is the ordinal index of the server within the task type.
   - rpc_layer is the protocol used by TensorFlow to communicate with other
       TensorFlow servers in a distributed environment.
   """
@@ -84,12 +84,12 @@ class ClusterResolver(object):
     raise NotImplementedError()
 
   @abc.abstractmethod
-  def master(self, task_type=None, task_index=None, rpc_layer=None):
+  def master(self, task_type=None, task_id=None, rpc_layer=None):
     """Retrieves the name or URL of the session master.
 
     Args:
       task_type: (Optional) The type of the TensorFlow task of the master.
-      task_index: (Optional) The index of the TensorFlow task of the master.
+      task_id: (Optional) The index of the TensorFlow task of the master.
       rpc_layer: (Optional) The RPC protocol for the given cluster.
 
     Returns:
@@ -103,7 +103,7 @@ class ClusterResolver(object):
 
   def num_accelerators(self,
                        task_type=None,
-                       task_index=None,
+                       task_id=None,
                        accelerator_type='GPU',
                        config_proto=None):
     """Returns the number of accelerator cores per worker.
@@ -113,7 +113,7 @@ class ClusterResolver(object):
     should return 0. This method will query the master for this information
     if it is not otherwise known.
 
-    Optionally, we allow callers to specify the task_type, task_index, and
+    Optionally, we allow callers to specify the task_type, task_id, and
     rpc_layer, if they want to target a specific TensorFlow process to query
     the number of accelerators. This is to support heterogenous environments,
     where the number of accelerators cores per host is different.
@@ -121,14 +121,14 @@ class ClusterResolver(object):
     Args:
       task_type: (Optional) The type of the TensorFlow task of the machine we
         want to query.
-      task_index: (Optional) The index of the TensorFlow task of the machine we
+      task_id: (Optional) The index of the TensorFlow task of the machine we
         want to query.
       accelerator_type: (Optional) The type of accelerator we are trying to
         query (defaults to 'GPU').
       config_proto: (Optional) Configuration for starting a new session to
         query how many accelerator cores it has.
     """
-    master = self.master(task_type, task_index)
+    master = self.master(task_type, task_id)
     devices = get_accelerator_devices(master, config_proto)
     return sum(1 for d in devices if d.device_type == accelerator_type)
 
@@ -141,14 +141,14 @@ class ClusterResolver(object):
 class SimpleClusterResolver(ClusterResolver):
   """Simple implementation of ClusterResolver that accepts a ClusterSpec."""
 
-  def __init__(self, cluster_spec, master='', task_type=None, task_index=None,
+  def __init__(self, cluster_spec, master='', task_type=None, task_id=None,
                environment='', num_accelerators=0,
                rpc_layer=None):
     """Creates a SimpleClusterResolver from a ClusterSpec."""
     super(SimpleClusterResolver, self).__init__()
 
     self._task_type = task_type
-    self._task_index = task_index
+    self._task_id = task_id
     self._environment = environment
     self._num_accelerators = num_accelerators
     self._rpc_layer = rpc_layer
@@ -165,22 +165,22 @@ class SimpleClusterResolver(ClusterResolver):
     """Returns the ClusterSpec passed into the constructor."""
     return self._cluster_spec
 
-  def master(self, task_type=None, task_index=None, rpc_layer=None):
+  def master(self, task_type=None, task_id=None, rpc_layer=None):
     """Returns the master address to use when creating a session.
 
     Args:
       task_type: (Optional) The type of the TensorFlow task of the master.
-      task_index: (Optional) The index of the TensorFlow task of the master.
+      task_id: (Optional) The index of the TensorFlow task of the master.
       rpc_layer: (Optional) The RPC used by distributed TensorFlow.
 
     Returns:
       The name or URL of the session master.
 
-    If a task_type and task_index is given, this will override the `master`
+    If a task_type and task_id is given, this will override the `master`
     string passed into the initialization function.
     """
-    if task_type is not None and task_index is not None:
-      master = self.cluster_spec().task_address(task_type, task_index)
+    if task_type is not None and task_id is not None:
+      master = self.cluster_spec().task_address(task_type, task_id)
     else:
       master = self._master
 
@@ -191,16 +191,16 @@ class SimpleClusterResolver(ClusterResolver):
     return self._task_type
 
   @property
-  def task_index(self):
-    return self._task_index
+  def task_id(self):
+    return self._task_id
 
   @task_type.setter
   def task_type(self, task_type):
     self._task_type = task_type
 
-  @task_index.setter
-  def task_index(self, task_index):
-    self._task_index = task_index
+  @task_id.setter
+  def task_id(self, task_id):
+    self._task_id = task_id
 
   @property
   def environment(self):
@@ -208,7 +208,7 @@ class SimpleClusterResolver(ClusterResolver):
 
   def num_accelerators(self,
                        task_type=None,
-                       task_index=None,
+                       task_id=None,
                        accelerator_type='GPU',
                        config_proto=None):
     """Returns the number of accelerator cores per worker.
@@ -220,12 +220,12 @@ class SimpleClusterResolver(ClusterResolver):
 
     Args:
       task_type: Unused.
-      task_index: Unused.
+      task_id: Unused.
       accelerator_type: Unused.
       config_proto: Unused.
     """
     # Unused
-    del task_type, task_index, accelerator_type, config_proto
+    del task_type, task_id, accelerator_type, config_proto
     return self._num_accelerators
 
   @property
@@ -259,7 +259,7 @@ class UnionClusterResolver(ClusterResolver):
         rpc_layer - (Optional) Override value for the RPC layer used by
           TensorFlow.
         task_type - (Optional) Override value for the current task type.
-        task_index - (Optional) Override value for the current task index.
+        task_id - (Optional) Override value for the current task index.
 
     Raises:
       TypeError: If any argument is not a subclass of `ClusterResolvers`.
@@ -269,7 +269,7 @@ class UnionClusterResolver(ClusterResolver):
 
     self._rpc_layer = kwargs.pop('rpc_layer', None)
     self._task_type = kwargs.pop('task_type', None)
-    self._task_index = kwargs.pop('task_index', None)
+    self._task_id = kwargs.pop('task_id', None)
 
     if kwargs:
       raise ValueError('Unexpected kwargs provided {!r}'.format(kwargs))
@@ -357,22 +357,22 @@ class UnionClusterResolver(ClusterResolver):
 
     return ClusterSpec(merged_cluster)
 
-  def master(self, task_type=None, task_index=None, rpc_layer=None):
+  def master(self, task_type=None, task_id=None, rpc_layer=None):
     """Returns the master address to use when creating a session.
 
     This usually returns the master from the first ClusterResolver passed in,
-    but you can override this by specifying the task_type and task_index.
+    but you can override this by specifying the task_type and task_id.
 
     Args:
       task_type: (Optional) The type of the TensorFlow task of the master.
-      task_index: (Optional) The index of the TensorFlow task of the master.
+      task_id: (Optional) The index of the TensorFlow task of the master.
       rpc_layer: (Optional) The RPC protocol for the given cluster.
 
     Returns:
       The name or URL of the session master.
     """
-    if task_type is not None and task_index is not None:
-      master = self.cluster_spec().task_address(task_type, task_index)
+    if task_type is not None and task_id is not None:
+      master = self.cluster_spec().task_address(task_type, task_id)
       return format_master_url(master, rpc_layer or self._rpc_layer)
 
     return self._cluster_resolvers[0].master(rpc_layer=rpc_layer)
@@ -382,16 +382,16 @@ class UnionClusterResolver(ClusterResolver):
     return self._task_type or self._cluster_resolvers[0].task_type
 
   @property
-  def task_index(self):
-    return self._task_index or self._cluster_resolvers[0].task_index
+  def task_id(self):
+    return self._task_id or self._cluster_resolvers[0].task_id
 
   @task_type.setter
   def task_type(self, task_type):
     self._task_type = task_type
 
-  @task_index.setter
-  def task_index(self, task_index):
-    self._task_index = task_index
+  @task_id.setter
+  def task_id(self, task_id):
+    self._task_id = task_id
 
   @property
   def environment(self):
@@ -399,11 +399,11 @@ class UnionClusterResolver(ClusterResolver):
 
   def num_accelerators(self,
                        task_type=None,
-                       task_index=None,
+                       task_id=None,
                        accelerator_type='GPU',
                        config_proto=None):
     return self._cluster_resolvers[0].num_accelerators(
-        task_type, task_index, accelerator_type, config_proto)
+        task_type, task_id, accelerator_type, config_proto)
 
   @property
   def rpc_layer(self):
diff --git a/tensorflow/python/distribute/cluster_resolver/cluster_resolver_test.py b/tensorflow/python/distribute/cluster_resolver/cluster_resolver_test.py
index 0ff6b6be62..c1eb29e2fc 100644
--- a/tensorflow/python/distribute/cluster_resolver/cluster_resolver_test.py
+++ b/tensorflow/python/distribute/cluster_resolver/cluster_resolver_test.py
@@ -33,7 +33,7 @@ class MockBaseClusterResolver(ClusterResolver):
   def cluster_spec(self):
     return None
 
-  def master(self, task_type=None, task_index=None, rpc_layer=None):
+  def master(self, task_type=None, task_id=None, rpc_layer=None):
     return ""
 
   def environment(self):
@@ -117,12 +117,12 @@ class UnionClusterResolverTest(test.TestCase):
     })
 
     simple_resolver = SimpleClusterResolver(base_cluster_spec, task_type="ps",
-                                            task_index=1, environment="cloud",
+                                            task_id=1, environment="cloud",
                                             num_accelerators=8,
                                             rpc_layer="grpc")
 
     self.assertEqual(simple_resolver.task_type, "ps")
-    self.assertEqual(simple_resolver.task_index, 1)
+    self.assertEqual(simple_resolver.task_id, 1)
     self.assertEqual(simple_resolver.environment, "cloud")
     self.assertEqual(simple_resolver.num_accelerators(), 8)
     self.assertEqual(simple_resolver.rpc_layer, "grpc")
@@ -134,16 +134,16 @@ class UnionClusterResolverTest(test.TestCase):
     })
 
     simple_resolver = SimpleClusterResolver(base_cluster_spec, task_type="ps",
-                                            task_index=1, environment="cloud",
+                                            task_id=1, environment="cloud",
                                             num_accelerators=8,
                                             rpc_layer="grpc")
 
     simple_resolver.task_type = "worker"
-    simple_resolver.task_index = 2
+    simple_resolver.task_id = 2
     simple_resolver.rpc_layer = "http"
 
     self.assertEqual(simple_resolver.task_type, "worker")
-    self.assertEqual(simple_resolver.task_index, 2)
+    self.assertEqual(simple_resolver.task_id, 2)
     self.assertEqual(simple_resolver.rpc_layer, "http")
 
   def testSimpleOverrideMasterWithTaskIndexZero(self):
@@ -182,7 +182,7 @@ class UnionClusterResolverTest(test.TestCase):
         "worker": ["worker0:2222", "worker1:2222", "worker2:2222"]
     })
     resolver1 = SimpleClusterResolver(cluster_spec_1, task_type="ps",
-                                      task_index=1, environment="cloud",
+                                      task_id=1, environment="cloud",
                                       num_accelerators=8,
                                       rpc_layer="grpc")
 
@@ -191,24 +191,24 @@ class UnionClusterResolverTest(test.TestCase):
         "worker": ["worker3:2222", "worker4:2222", "worker5:2222"]
     })
     resolver2 = SimpleClusterResolver(cluster_spec_2, task_type="worker",
-                                      task_index=2, environment="local",
+                                      task_id=2, environment="local",
                                       num_accelerators=16,
                                       rpc_layer="http")
 
     union_resolver = UnionClusterResolver(resolver1, resolver2)
 
     self.assertEqual(union_resolver.task_type, "ps")
-    self.assertEqual(union_resolver.task_index, 1)
+    self.assertEqual(union_resolver.task_id, 1)
     self.assertEqual(union_resolver.environment, "cloud")
     self.assertEqual(union_resolver.num_accelerators(), 8)
     self.assertEqual(union_resolver.rpc_layer, "grpc")
 
     union_resolver.task_type = "worker"
-    union_resolver.task_index = 2
+    union_resolver.task_id = 2
     union_resolver.rpc_layer = "http"
 
     self.assertEqual(union_resolver.task_type, "worker")
-    self.assertEqual(union_resolver.task_index, 2)
+    self.assertEqual(union_resolver.task_id, 2)
     self.assertEqual(union_resolver.rpc_layer, "http")
 
   def testTwoNonOverlappingJobMergedClusterResolver(self):
diff --git a/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver.py
index 06512613cb..305c53870d 100644
--- a/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver.py
+++ b/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver.py
@@ -49,7 +49,7 @@ class GceClusterResolver(ClusterResolver):
                instance_group,
                port,
                task_type='worker',
-               task_index=0,
+               task_id=0,
                rpc_layer='grpc',
                credentials='default',
                service=None):
@@ -66,7 +66,7 @@ class GceClusterResolver(ClusterResolver):
       port: Port of the listening TensorFlow server (default: 8470)
       task_type: Name of the TensorFlow job this GCE instance group of VM
         instances belong to.
-      task_index: The task index for this particular VM, within the GCE
+      task_id: The task index for this particular VM, within the GCE
         instance group. In particular, every single instance should be assigned
         a unique ordinal index within an instance group manually so that they
         can be distinguished from each other.
@@ -85,7 +85,7 @@ class GceClusterResolver(ClusterResolver):
     self._zone = zone
     self._instance_group = instance_group
     self._task_type = task_type
-    self._task_index = task_index
+    self._task_id = task_id
     self._rpc_layer = rpc_layer
     self._port = port
     self._credentials = credentials
@@ -149,12 +149,12 @@ class GceClusterResolver(ClusterResolver):
     worker_list.sort()
     return ClusterSpec({self._task_type: worker_list})
 
-  def master(self, task_type=None, task_index=None, rpc_layer=None):
+  def master(self, task_type=None, task_id=None, rpc_layer=None):
     task_type = task_type if task_type is not None else self._task_type
-    task_index = task_index if task_index is not None else self._task_index
+    task_id = task_id if task_id is not None else self._task_id
 
-    if task_type is not None and task_index is not None:
-      master = self.cluster_spec().task_address(task_type, task_index)
+    if task_type is not None and task_id is not None:
+      master = self.cluster_spec().task_address(task_type, task_id)
       if rpc_layer or self._rpc_layer:
         return '%s://%s' % (rpc_layer or self._rpc_layer, master)
       else:
@@ -167,8 +167,8 @@ class GceClusterResolver(ClusterResolver):
     return self._task_type
 
   @property
-  def task_index(self):
-    return self._task_index
+  def task_id(self):
+    return self._task_id
 
   @task_type.setter
   def task_type(self, task_type):
@@ -176,9 +176,9 @@ class GceClusterResolver(ClusterResolver):
         'You cannot reset the task_type of the GceClusterResolver after it has '
         'been created.')
 
-  @task_index.setter
-  def task_index(self, task_index):
-    self._task_index = task_index
+  @task_id.setter
+  def task_id(self, task_id):
+    self._task_id = task_id
 
   @property
   def environment(self):
diff --git a/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver_test.py b/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver_test.py
index d4f0660c92..07b9eeb08e 100644
--- a/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver_test.py
+++ b/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver_test.py
@@ -140,7 +140,7 @@ class GceClusterResolverTest(test.TestCase):
         project='test-project',
         zone='us-east1-d',
         instance_group='test-instance-group',
-        task_index=0,
+        task_id=0,
         port=8470,
         credentials=None,
         service=self.standard_mock_service_client())
@@ -181,11 +181,11 @@ class GceClusterResolverTest(test.TestCase):
         credentials=None,
         service=self.gen_standard_mock_service_client(name_to_ip))
 
-    gce_cluster_resolver.task_index = 1
+    gce_cluster_resolver.task_id = 1
     gce_cluster_resolver.rpc_layer = 'test'
 
     self.assertEqual(gce_cluster_resolver.task_type, 'testworker')
-    self.assertEqual(gce_cluster_resolver.task_index, 1)
+    self.assertEqual(gce_cluster_resolver.task_id, 1)
     self.assertEqual(gce_cluster_resolver.rpc_layer, 'test')
     self.assertEqual(gce_cluster_resolver.master(), 'test://10.2.3.4:8470')
 
@@ -201,13 +201,13 @@ class GceClusterResolverTest(test.TestCase):
         zone='us-east1-d',
         instance_group='test-instance-group',
         task_type='',
-        task_index=1,
+        task_id=1,
         port=8470,
         credentials=None,
         service=self.gen_standard_mock_service_client(name_to_ip))
 
     self.assertEqual(gce_cluster_resolver.master(
-        task_type='', task_index=0), 'grpc://10.1.2.3:8470')
+        task_type='', task_id=0), 'grpc://10.1.2.3:8470')
 
   def testCustomJobNameAndPortRetrieval(self):
     gce_cluster_resolver = GceClusterResolver(
diff --git a/tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver.py
index 7ff6ec0f2d..71d48ed2b9 100644
--- a/tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver.py
+++ b/tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver.py
@@ -88,31 +88,31 @@ class KubernetesClusterResolver(ClusterResolver):
     self._override_client = override_client
 
     self.task_type = None
-    self.task_index = None
+    self.task_id = None
     self.rpc_layer = rpc_layer
 
-  def master(self, task_type=None, task_index=None, rpc_layer=None):
+  def master(self, task_type=None, task_id=None, rpc_layer=None):
     """Returns the master address to use when creating a session.
 
-    You must have set the task_type and task_index object properties before
-    calling this function, or pass in the `task_type` and `task_index`
+    You must have set the task_type and task_id object properties before
+    calling this function, or pass in the `task_type` and `task_id`
     parameters when using this function. If you do both, the function parameters
     will override the object properties.
 
     Args:
       task_type: (Optional) The type of the TensorFlow task of the master.
-      task_index: (Optional) The index of the TensorFlow task of the master.
+      task_id: (Optional) The index of the TensorFlow task of the master.
       rpc_layer: (Optional) The RPC protocol for the given cluster.
 
     Returns:
       The name or URL of the session master.
     """
     task_type = task_type if task_type is not None else self.task_type
-    task_index = task_index if task_index is not None else self.task_index
+    task_id = task_id if task_id is not None else self.task_id
 
-    if task_type is not None and task_index is not None:
+    if task_type is not None and task_id is not None:
       return format_master_url(
-          self.cluster_spec().task_address(task_type, task_index),
+          self.cluster_spec().task_address(task_type, task_id),
           rpc_layer or self.rpc_layer)
 
     return ''
diff --git a/tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver_test.py b/tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver_test.py
index a9750fa60b..f4e4cd8212 100644
--- a/tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver_test.py
+++ b/tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver_test.py
@@ -119,9 +119,9 @@ class KubernetesClusterResolverTest(test.TestCase):
         override_client=_mock_kubernetes_client(
             {'job-name=tensorflow': ret}))
     cluster_resolver.task_type = 'worker'
-    cluster_resolver.task_index = 0
+    cluster_resolver.task_id = 0
     self.assertEqual(cluster_resolver.task_type, 'worker')
-    self.assertEqual(cluster_resolver.task_index, 0)
+    self.assertEqual(cluster_resolver.task_id, 0)
     self.assertEqual(cluster_resolver.master(), 'grpc://10.1.2.3:8470')
     self.assertEqual(cluster_resolver.master('worker', 2),
                      'grpc://10.1.2.5:8470')
diff --git a/tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver.py
index 9dbe25b613..0ec566c670 100644
--- a/tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver.py
+++ b/tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver.py
@@ -112,7 +112,7 @@ class SlurmClusterResolver(ClusterResolver):
 
     self._auto_set_gpu = auto_set_gpu
     self.task_type = None
-    self.task_index = None
+    self.task_id = None
     self.rpc_layer = rpc_layer
 
     self._gpu_allocation = []
@@ -170,7 +170,7 @@ class SlurmClusterResolver(ClusterResolver):
 
       if cluster_rank_offset_start <= self._rank < cluster_rank_offset_end:
         self.task_type = task_type
-        self.task_index = self._rank - cluster_rank_offset_start
+        self.task_id = self._rank - cluster_rank_offset_start
 
       cluster_rank_offset_start = cluster_rank_offset_end
 
@@ -180,7 +180,7 @@ class SlurmClusterResolver(ClusterResolver):
     return ClusterSpec(self._cluster_allocation)
 
   def get_task_info(self):
-    """Returns job name and task_index for the process which calls this.
+    """Returns job name and task_id for the process which calls this.
 
     This returns the job name and task index for the process which calls this
     function according to its rank and cluster specification. The job name and
@@ -191,14 +191,14 @@ class SlurmClusterResolver(ClusterResolver):
       A string specifying job name the process belongs to and an integner
         specifying the task index the process belongs to in that job.
     """
-    return self.task_type, self.task_index
+    return self.task_type, self.task_id
 
-  def master(self, task_type=None, task_index=None, rpc_layer=None):
+  def master(self, task_type=None, task_id=None, rpc_layer=None):
     """Returns the master string for connecting to a TensorFlow master.
 
     Args:
       task_type: (Optional) Overrides the default auto-selected task type.
-      task_index: (Optional) Overrides the default auto-slected task index.
+      task_id: (Optional) Overrides the default auto-slected task index.
       rpc_layer: (Optional) Overrides the default RPC protocol TensorFlow uses
         to communicate across nodes.
 
@@ -206,11 +206,11 @@ class SlurmClusterResolver(ClusterResolver):
       A connection string for connecting to a TensorFlow master.
     """
     task_type = task_type if task_type is not None else self.task_type
-    task_index = task_index if task_index is not None else self.task_index
+    task_id = task_id if task_id is not None else self.task_id
 
-    if task_type is not None and task_index is not None:
+    if task_type is not None and task_id is not None:
       return format_master_url(
-          self.cluster_spec().task_address(task_type, task_index),
+          self.cluster_spec().task_address(task_type, task_id),
           rpc_layer or self.rpc_layer)
 
     return ''
@@ -227,9 +227,9 @@ class SlurmClusterResolver(ClusterResolver):
 
   def num_accelerators(self,
                        task_type=None,
-                       task_index=None,
+                       task_id=None,
                        accelerator_type='GPU',
                        config_proto=None):
     # Unused, since this is set in __init__ manually.
-    del task_type, task_index, accelerator_type, config_proto
+    del task_type, task_id, accelerator_type, config_proto
     return self._gpus_per_node
diff --git a/tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver_test.py b/tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver_test.py
index 076539d16f..c641fe6085 100644
--- a/tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver_test.py
+++ b/tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver_test.py
@@ -83,7 +83,7 @@ class SlurmClusterResolverTest(test.TestCase):
         auto_set_gpu=False)
 
     slurm_cluster_resolver.task_type = 'worker'
-    slurm_cluster_resolver.task_index = 1
+    slurm_cluster_resolver.task_id = 1
     self.assertEqual(slurm_cluster_resolver.master(), 'grpc://t02n43:8888')
 
     slurm_cluster_resolver.rpc_layer = 'ab'
diff --git a/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver.py
index 8d530cc15a..6dcbafbc50 100644
--- a/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver.py
+++ b/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver.py
@@ -52,7 +52,7 @@ class TFConfigClusterResolver(ClusterResolver):
 
   def __init__(self,
                task_type=None,
-               task_index=None,
+               task_id=None,
                rpc_layer=None,
                environment=None):
     """Creates a new TFConfigClusterResolver.
@@ -60,14 +60,14 @@ class TFConfigClusterResolver(ClusterResolver):
     Args:
       task_type: (String, optional) Overrides the task type specified in the
         TF_CONFIG environment variable.
-      task_index: (Integer, optional) Overrides the task index specified in the
+      task_id: (Integer, optional) Overrides the task index specified in the
         TF_CONFIG environment variable.
       rpc_layer: (String, optional) Overrides the rpc layer TensorFlow uses.
       environment: (String, optional) Overrides the environment TensorFlow
         operates in.
     """
     self._task_type = task_type
-    self._task_index = task_index
+    self._task_id = task_id
     self._rpc_layer = rpc_layer
     self._environment = environment
 
@@ -80,20 +80,20 @@ class TFConfigClusterResolver(ClusterResolver):
       return self._task_type
 
   @property
-  def task_index(self):
+  def task_id(self):
     if self._task_type is None:
       task_info = _get_value_in_tfconfig(_TASK_KEY, {})
       return task_info['index'] if 'index' in task_info else None
     else:
-      return self._task_index
+      return self._task_id
 
   @task_type.setter
   def task_type(self, task_type):
     self._task_type = task_type
 
-  @task_index.setter
-  def task_index(self, task_index):
-    self._task_index = task_index
+  @task_id.setter
+  def task_id(self, task_id):
+    self._task_id = task_id
 
   @property
   def environment(self):
@@ -121,13 +121,13 @@ class TFConfigClusterResolver(ClusterResolver):
       return ClusterSpec({})
     return ClusterSpec(tf_config['cluster'])
 
-  def master(self, task_type=None, task_index=None, rpc_layer=None):
+  def master(self, task_type=None, task_id=None, rpc_layer=None):
     """Returns the master address to use when creating a TensorFlow session.
 
     Args:
       task_type: (String, optional) Overrides and sets the task_type of the
         master.
-      task_index: (Integer, optional) Overrides and sets the task id of the
+      task_id: (Integer, optional) Overrides and sets the task id of the
         master.
       rpc_layer: (String, optional) Overrides and sets the protocol over which
         TensorFlow nodes communicate with each other.
@@ -155,7 +155,7 @@ class TFConfigClusterResolver(ClusterResolver):
     # We try to auto-detect the task type and id, but uses the user-supplied one
     # where available
     task_type = task_type if task_type is not None else self.task_type
-    task_index = task_index if task_index is not None else self.task_index
+    task_id = task_id if task_id is not None else self.task_id
 
-    return format_master_url(cluster_spec.task_address(task_type, task_index),
+    return format_master_url(cluster_spec.task_address(task_type, task_id),
                              self.rpc_layer)
diff --git a/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver_test.py b/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver_test.py
index 36b3bb9c1e..97a5eb685f 100644
--- a/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver_test.py
+++ b/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver_test.py
@@ -150,7 +150,7 @@ class TFConfigClusterResolverTest(test.TestCase):
 
     cluster_resolver = TFConfigClusterResolver()
     self.assertEqual('ps', cluster_resolver.task_type)
-    self.assertEqual(0, cluster_resolver.task_index)
+    self.assertEqual(0, cluster_resolver.task_id)
     self.assertEqual('grpc', cluster_resolver.rpc_layer)
 
   def testParameterOverrides(self):
@@ -168,19 +168,19 @@ class TFConfigClusterResolverTest(test.TestCase):
     }
     """
 
-    cluster_resolver = TFConfigClusterResolver(task_type='ps', task_index=0)
+    cluster_resolver = TFConfigClusterResolver(task_type='ps', task_id=0)
 
     self.assertEqual('grpc://ps0:2222', cluster_resolver.master())
     self.assertEqual('ps', cluster_resolver.task_type)
-    self.assertEqual(0, cluster_resolver.task_index)
+    self.assertEqual(0, cluster_resolver.task_id)
 
     cluster_resolver.task_type = 'worker'
-    cluster_resolver.task_index = 1
+    cluster_resolver.task_id = 1
     cluster_resolver.rpc_layer = 'test'
 
     self.assertEqual('test://worker1:2222', cluster_resolver.master())
     self.assertEqual('worker', cluster_resolver.task_type)
-    self.assertEqual(1, cluster_resolver.task_index)
+    self.assertEqual(1, cluster_resolver.task_id)
     self.assertEqual('test', cluster_resolver.rpc_layer)
 
   def testZeroItemsInClusterSpecMasterRead(self):
diff --git a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
index 635948fd19..49e6ff8720 100644
--- a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
+++ b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
@@ -254,10 +254,10 @@ class TPUClusterResolver(ClusterResolver):
       raise RuntimeError('You need to specify a TPU Name if you are running in '
                          'the Google Cloud environment.')
 
-    # By default the task_type is 'worker` and the task_index is 0 (which is the
+    # By default the task_type is 'worker` and the task_id is 0 (which is the
     # first worker in the task).
     self.task_type = job_name
-    self.task_index = 0
+    self.task_id = 0
 
     if tpu.startswith('grpc://'):
       # Cloud environment, where we are using GRPC to communicate to TPUs.
@@ -327,7 +327,7 @@ class TPUClusterResolver(ClusterResolver):
     else:
       self._coordinator_address = coordinator_address
 
-  def master(self, task_type=None, task_index=None, rpc_layer=None):
+  def master(self, task_type=None, task_id=None, rpc_layer=None):
     """Get the Master string to be used for the session.
 
     In the normal case, this returns the grpc path (grpc://1.2.3.4:8470) of
@@ -341,7 +341,7 @@ class TPUClusterResolver(ClusterResolver):
     Args:
       task_type: (Optional, string) The type of the TensorFlow task of the
         master.
-      task_index: (Optional, integer) The index of the TensorFlow task of the
+      task_id: (Optional, integer) The index of the TensorFlow task of the
         master.
       rpc_layer: (Optional, string) The RPC protocol TensorFlow should use to
         communicate with TPUs.
@@ -355,12 +355,12 @@ class TPUClusterResolver(ClusterResolver):
     if self._shouldResolve():
       # We are going to communicate with the Cloud TPU APIs to get a Cluster.
       cluster_spec = self.cluster_spec()
-      if task_type is not None and task_index is not None:
-        # task_type and task_index is from the function parameter
-        master = cluster_spec.task_address(task_type, task_index)
-      elif self.task_type is not None and self.task_index is not None:
-        # task_type and task_index is from the object
-        master = cluster_spec.task_address(self.task_type, self.task_index)
+      if task_type is not None and task_id is not None:
+        # task_type and task_id is from the function parameter
+        master = cluster_spec.task_address(task_type, task_id)
+      elif self.task_type is not None and self.task_id is not None:
+        # task_type and task_id is from the object
+        master = cluster_spec.task_address(self.task_type, self.task_id)
       else:
         # by default we take the first item in the cluster with the right name
         job_tasks = cluster_spec.job_tasks(self.task_type)
@@ -458,7 +458,7 @@ class TPUClusterResolver(ClusterResolver):
 
   def num_accelerators(self,
                        task_type=None,
-                       task_index=None,
+                       task_id=None,
                        accelerator_type='TPU',
                        config_proto=None):
     """Returns the number of TPU cores per worker.
@@ -469,7 +469,7 @@ class TPUClusterResolver(ClusterResolver):
 
     Args:
       task_type: Unused.
-      task_index: Unused.
+      task_id: Unused.
       accelerator_type: Unused.
       config_proto: Used to create a connection to a TPU master in order to
         retrieve the system metadata.
diff --git a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver_test.py b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver_test.py
index 58c332a509..d5cfb2262b 100644
--- a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver_test.py
+++ b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver_test.py
@@ -576,12 +576,12 @@ class TPUClusterResolverTest(test.TestCase):
     self.assertEqual(resolver.master(), 'grpc://10.2.3.4:8470')
 
     resolver.task_type = 'worker'
-    resolver.task_index = 3
+    resolver.task_id = 3
     self.assertEqual(resolver.master(), 'grpc://10.2.3.7:8470')
 
     self.assertEqual(
         resolver.master(
-            task_type='worker', task_index=2, rpc_layer='test'),
+            task_type='worker', task_id=2, rpc_layer='test'),
         'test://10.2.3.6:8470')
 
   def testGetDeviceDictAndCoresWithTPUs(self):
diff --git a/tensorflow/python/distribute/parameter_server_strategy.py b/tensorflow/python/distribute/parameter_server_strategy.py
index 6cc5865eed..269c7417d1 100644
--- a/tensorflow/python/distribute/parameter_server_strategy.py
+++ b/tensorflow/python/distribute/parameter_server_strategy.py
@@ -130,7 +130,7 @@ class ParameterServerStrategyExtended(
     num_gpus = cluster_resolver.num_accelerators()
     cluster_spec = cluster_resolver.cluster_spec()
     task_type = cluster_resolver.task_type
-    task_id = cluster_resolver.task_index
+    task_id = cluster_resolver.task_id
     if not task_type or task_id is None:
       raise ValueError("When `cluster_spec` is given, you must also specify "
                        "`task_type` and `task_id`")
@@ -463,7 +463,7 @@ class ParameterServerStrategyExtended(
       cluster_resolver = SimpleClusterResolver(
           cluster_spec=multi_worker_util.normalize_cluster_spec(cluster_spec),
           task_type=task_type,
-          task_index=task_id,
+          task_id=task_id,
           num_accelerators=self._num_gpus_per_worker)
       self._initialize_multi_worker(cluster_resolver)
 
-- 
GitLab


From 7a28fb4da3c148da864cf3aeb7c91f343b78ec59 Mon Sep 17 00:00:00 2001
From: Zachary Garrett <zachgarrett@google.com>
Date: Wed, 23 Jan 2019 15:51:35 -0800
Subject: [PATCH 1147/2345] Add Reduce base class and a new Sum metric
 complement the Mean metric already available.

PiperOrigin-RevId: 230618998
---
 tensorflow/python/keras/metrics.py            | 188 ++++++++++++-----
 tensorflow/python/keras/metrics_test.py       | 117 +++++++++++
 .../python/keras/utils/metrics_utils.py       |  18 +-
 .../tensorflow.keras.metrics.-accuracy.pbtxt  |   1 +
 ...rflow.keras.metrics.-binary-accuracy.pbtxt |   1 +
 ....keras.metrics.-categorical-accuracy.pbtxt |   1 +
 ...low.keras.metrics.-categorical-hinge.pbtxt |   1 +
 ...flow.keras.metrics.-cosine-proximity.pbtxt |   1 +
 .../v1/tensorflow.keras.metrics.-hinge.pbtxt  |   1 +
 ...w.keras.metrics.-mean-absolute-error.pbtxt |   1 +
 ...rics.-mean-absolute-percentage-error.pbtxt |   1 +
 ...ow.keras.metrics.-mean-squared-error.pbtxt |   1 +
 ...rics.-mean-squared-logarithmic-error.pbtxt |   1 +
 .../v1/tensorflow.keras.metrics.-mean.pbtxt   |   1 +
 .../tensorflow.keras.metrics.-reduction.pbtxt |  16 ++
 ...metrics.-sparse-categorical-accuracy.pbtxt |   1 +
 ...sorflow.keras.metrics.-squared-hinge.pbtxt |   1 +
 .../v1/tensorflow.keras.metrics.-sum.pbtxt    | 197 ++++++++++++++++++
 .../golden/v1/tensorflow.keras.metrics.pbtxt  |   8 +
 .../tensorflow.keras.metrics.-accuracy.pbtxt  |   1 +
 ...rflow.keras.metrics.-binary-accuracy.pbtxt |   1 +
 ....keras.metrics.-categorical-accuracy.pbtxt |   1 +
 ...low.keras.metrics.-categorical-hinge.pbtxt |   1 +
 ...flow.keras.metrics.-cosine-proximity.pbtxt |   1 +
 .../v2/tensorflow.keras.metrics.-hinge.pbtxt  |   1 +
 ...w.keras.metrics.-mean-absolute-error.pbtxt |   1 +
 ...rics.-mean-absolute-percentage-error.pbtxt |   1 +
 ...ow.keras.metrics.-mean-squared-error.pbtxt |   1 +
 ...rics.-mean-squared-logarithmic-error.pbtxt |   1 +
 .../v2/tensorflow.keras.metrics.-mean.pbtxt   |   1 +
 .../tensorflow.keras.metrics.-reduction.pbtxt |  16 ++
 ...metrics.-sparse-categorical-accuracy.pbtxt |   1 +
 ...sorflow.keras.metrics.-squared-hinge.pbtxt |   1 +
 .../v2/tensorflow.keras.metrics.-sum.pbtxt    | 197 ++++++++++++++++++
 .../golden/v2/tensorflow.keras.metrics.pbtxt  |   8 +
 35 files changed, 737 insertions(+), 54 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-reduction.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sum.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-reduction.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sum.pbtxt

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index d3bd523d29..9a39dd4e17 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -257,9 +257,141 @@ class Metric(Layer):
 
   ### End: For use by subclasses ###
 
+class Reduce(Metric):
+  """Encapsulates metrics that perform a reduce operation on the values."""
+
+  def __init__(self, reduction, name, dtype=None):
+    """Creates a `Reduce` instance.
+
+    Args:
+      reduction: a `tf.keras.metrics.Reduction` enum value.
+      name: string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+    """
+    super(Reduce, self).__init__(name=name, dtype=dtype)
+    self.reduction = reduction
+    self.total = self.add_weight(
+        'total', initializer=init_ops.zeros_initializer)
+    if reduction in [metrics_utils.Reduction.SUM_OVER_BATCH_SIZE,
+                     metrics_utils.Reduction.WEIGHTED_MEAN]:
+      self.count = self.add_weight(
+          'count', initializer=init_ops.zeros_initializer)
+
+  def update_state(self, values, sample_weight=None):
+    """Accumulates statistics for computing the reduction metric.
+
+    For example, if `values` is [1, 3, 5, 7] and reduction=SUM_OVER_BATCH_SIZE,
+    then the value of `result()` is 4. If the `sample_weight` is specified as
+    [1, 1, 0, 0] then value of `result()` would be 2.
+
+    Args:
+      values: Per-example value.
+      sample_weight: Optional weighting of each example. Defaults to 1.
+
+    Returns:
+      Update op.
+    """
+    values = math_ops.cast(values, self._dtype)
+    if sample_weight is not None:
+      sample_weight = math_ops.cast(sample_weight, self._dtype)
+      # Update dimensions of weights to match with values if possible.
+      values, _, sample_weight = squeeze_or_expand_dimensions(
+          values, None, sample_weight)
+      try:
+        # Broadcast weights if possible.
+        sample_weight = weights_broadcast_ops.broadcast_weights(
+            sample_weight, values)
+      except ValueError:
+        # Reduce values to same ndim as weight array
+        ndim = K.ndim(values)
+        weight_ndim = K.ndim(sample_weight)
+        if self.reduction == metrics_utils.Reduction.SUM:
+          values = math_ops.reduce_sum(
+              values, axis=list(range(weight_ndim, ndim)))
+        else:
+          values = math_ops.reduce_mean(
+              values, axis=list(range(weight_ndim, ndim)))
+      values = math_ops.multiply(values, sample_weight)
+
+    value_sum = math_ops.reduce_sum(values)
+    with ops.control_dependencies([value_sum]):
+      update_total_op = state_ops.assign_add(self.total, value_sum)
+
+    # Exit early if the reduction doesn't have a denominator.
+    if self.reduction == metrics_utils.Reduction.SUM:
+      return update_total_op
+
+    # Update `count` for reductions that require a denominator.
+    if self.reduction == metrics_utils.Reduction.SUM_OVER_BATCH_SIZE:
+      num_values = math_ops.cast(array_ops.size(values), self._dtype)
+    elif self.reduction == metrics_utils.Reduction.WEIGHTED_MEAN:
+      if sample_weight is None:
+        num_values = math_ops.cast(array_ops.size(values), self._dtype)
+      else:
+        num_values = math_ops.reduce_sum(sample_weight)
+    else:
+      raise NotImplementedError(
+          'reduction [%s] not implemented' % self.reduction)
+
+    with ops.control_dependencies([update_total_op]):
+      return state_ops.assign_add(self.count, num_values)
+
+  def result(self):
+    if self.reduction == metrics_utils.Reduction.SUM:
+      return array_ops.identity(self.total)
+    elif self.reduction in [
+        metrics_utils.Reduction.WEIGHTED_MEAN,
+        metrics_utils.Reduction.SUM_OVER_BATCH_SIZE
+    ]:
+      return math_ops.div_no_nan(self.total, self.count)
+    else:
+      raise NotImplementedError(
+          'reduction [%s] not implemented' % self.reduction)
+
+
+@keras_export('keras.metrics.Sum')
+class Sum(Reduce):
+  """Computes the (weighted) sum of the given values.
+
+  For example, if values is [1, 3, 5, 7] then the sum is 16.
+  If the weights were specified as [1, 1, 0, 0] then the sum would be 4.
+
+  This metric creates one variable, `total`, that is used to compute the sum of
+  `values`. This is ultimately returned as `sum`.
+
+  If `sample_weight` is `None`, weights default to 1.  Use `sample_weight` of 0
+  to mask values.
+
+  Usage:
+
+  ```python
+  m = tf.keras.metrics.Sum()
+  m.update_state([1, 3, 5, 7])
+  print('Final result: ', m.result().numpy())  # Final result: 16.0
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.add_metric(tf.keras.metrics.Sum(name='sum_1')(outputs))
+  model.compile('sgd', loss='mse')
+  ```
+  """
+
+  def __init__(self, name='sum', dtype=None):
+    """Creates a `Sum` instance.
+
+    Args:
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+    """
+    super(Sum, self).__init__(reduction=metrics_utils.Reduction.SUM,
+                              name=name, dtype=dtype)
+
 
 @keras_export('keras.metrics.Mean')
-class Mean(Metric):
+class Mean(Reduce):
   """Computes the (weighted) mean of the given values.
 
   For example, if values is [1, 3, 5, 7] then the mean is 4.
@@ -296,58 +428,8 @@ class Mean(Metric):
       name: (Optional) string name of the metric instance.
       dtype: (Optional) data type of the metric result.
     """
-    super(Mean, self).__init__(name=name, dtype=dtype)
-    # Create new state variables
-    self.total = self.add_weight(
-        'total', initializer=init_ops.zeros_initializer)
-    self.count = self.add_weight(
-        'count', initializer=init_ops.zeros_initializer)
-
-  def update_state(self, values, sample_weight=None):
-    """Accumulates statistics for computing the mean.
-
-    For example, if `values` is [1, 3, 5, 7] then the mean is 4. If
-    the `sample_weight` is specified as [1, 1, 0, 0] then the mean would be 2.
-
-    Args:
-      values: Per-example value.
-      sample_weight: Optional weighting of each example. Defaults to 1.
-
-    Returns:
-      Update op.
-    """
-    values = math_ops.cast(values, self._dtype)
-    if sample_weight is None:
-      num_values = math_ops.cast(array_ops.size(values), self._dtype)
-    else:
-      sample_weight = math_ops.cast(sample_weight, self._dtype)
-
-      # Update dimensions of weights to match with values if possible.
-      values, _, sample_weight = squeeze_or_expand_dimensions(
-          values, None, sample_weight)
-      try:
-        # Broadcast weights if possible.
-        sample_weight = weights_broadcast_ops.broadcast_weights(
-            sample_weight, values)
-      except ValueError:
-        # Reduce values to same ndim as weight array
-        ndim = K.ndim(values)
-        weight_ndim = K.ndim(sample_weight)
-        values = math_ops.reduce_mean(
-            values, axis=list(range(weight_ndim, ndim)))
-
-      num_values = math_ops.reduce_sum(sample_weight)
-      values = math_ops.multiply(values, sample_weight)
-    values = math_ops.reduce_sum(values)
-
-    # Update state variables. Count should be updated only when total is
-    # updated.
-    update_total_op = state_ops.assign_add(self.total, values)
-    with ops.control_dependencies([update_total_op]):
-      return state_ops.assign_add(self.count, num_values)
-
-  def result(self):
-    return math_ops.div_no_nan(self.total, self.count)
+    super(Mean, self).__init__(
+        reduction=metrics_utils.Reduction.WEIGHTED_MEAN, name=name, dtype=dtype)
 
 
 class MeanRelativeError(Mean):
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index 4e755dd301..b5af18ce27 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -39,6 +39,123 @@ from tensorflow.python.platform import test
 from tensorflow.python.training.checkpointable import util as checkpointable_utils
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class KerasSumTest(test.TestCase):
+
+  @test_util.run_in_graph_and_eager_modes()
+  def test_sum(self):
+    m = metrics.Sum(name='my_sum')
+
+    # check config
+    self.assertEqual(m.name, 'my_sum')
+    self.assertTrue(m.stateful)
+    self.assertEqual(m.dtype, dtypes.float32)
+    self.assertEqual(len(m.variables), 1)
+    self.evaluate(variables.variables_initializer(m.variables))
+
+    # check initial state
+    self.assertEqual(self.evaluate(m.total), 0)
+
+    # check __call__()
+    self.assertEqual(self.evaluate(m(100)), 100)
+    self.assertEqual(self.evaluate(m.total), 100)
+
+    # check update_state() and result() + state accumulation + tensor input
+    update_op = m.update_state(ops.convert_n_to_tensor([1, 5]))
+    self.evaluate(update_op)
+    self.assertAlmostEqual(self.evaluate(m.result()), 106)
+    self.assertEqual(self.evaluate(m.total), 106)  # 100 + 1 + 5
+
+    # check reset_states()
+    m.reset_states()
+    self.assertEqual(self.evaluate(m.total), 0)
+
+  def test_sum_with_sample_weight(self):
+    m = metrics.Sum(dtype=dtypes.float64)
+    self.assertEqual(m.dtype, dtypes.float64)
+    self.evaluate(variables.variables_initializer(m.variables))
+
+    # check scalar weight
+    result_t = m(100, sample_weight=0.5)
+    self.assertEqual(self.evaluate(result_t), 50)
+    self.assertEqual(self.evaluate(m.total), 50)
+
+    # check weights not scalar and weights rank matches values rank
+    result_t = m([1, 5], sample_weight=[1, 0.2])
+    result = self.evaluate(result_t)
+    self.assertAlmostEqual(result, 52., 4)  # 50 + 1 + 5 * 0.2
+    self.assertAlmostEqual(self.evaluate(m.total), 52., 4)
+
+    # check weights broadcast
+    result_t = m([1, 2], sample_weight=0.5)
+    self.assertAlmostEqual(self.evaluate(result_t), 53.5, 1)  # 52 + 0.5 + 1
+    self.assertAlmostEqual(self.evaluate(m.total), 53.5, 1)
+
+    # check weights squeeze
+    result_t = m([1, 5], sample_weight=[[1], [0.2]])
+    self.assertAlmostEqual(self.evaluate(result_t), 55.5, 1)  # 53.5 + 1 + 1
+    self.assertAlmostEqual(self.evaluate(m.total), 55.5, 1)
+
+    # check weights expand
+    result_t = m([[1], [5]], sample_weight=[1, 0.2])
+    self.assertAlmostEqual(self.evaluate(result_t), 57.5, 2)  # 55.5 + 1 + 1
+    self.assertAlmostEqual(self.evaluate(m.total), 57.5, 1)
+
+    # check values reduced to the dimensions of weight
+    result_t = m([[[1., 2.], [3., 2.], [0.5, 4.]]], sample_weight=[0.5])
+    result = np.round(self.evaluate(result_t), decimals=2)
+    # result = (prev: 57.5) + 0.5 + 1 + 1.5 + 1 + 0.25 + 2
+    self.assertAlmostEqual(result, 63.75, 2)
+    self.assertAlmostEqual(self.evaluate(m.total), 63.75, 2)
+
+  def test_sum_graph_with_placeholder(self):
+    with context.graph_mode(), self.cached_session() as sess:
+      m = metrics.Sum()
+      v = array_ops.placeholder(dtypes.float32)
+      w = array_ops.placeholder(dtypes.float32)
+      self.evaluate(variables.variables_initializer(m.variables))
+
+      # check __call__()
+      result_t = m(v, sample_weight=w)
+      result = sess.run(result_t, feed_dict=({v: 100, w: 0.5}))
+      self.assertEqual(result, 50)
+      self.assertEqual(self.evaluate(m.total), 50)
+
+      # check update_state() and result()
+      result = sess.run(result_t, feed_dict=({v: [1, 5], w: [1, 0.2]}))
+      self.assertAlmostEqual(result, 52., 2)  # 50 + 1 + 5 * 0.2
+      self.assertAlmostEqual(self.evaluate(m.total), 52., 2)
+
+  def test_save_restore(self):
+    checkpoint_directory = self.get_temp_dir()
+    checkpoint_prefix = os.path.join(checkpoint_directory, 'ckpt')
+    m = metrics.Sum()
+    checkpoint = checkpointable_utils.Checkpoint(sum=m)
+    self.evaluate(variables.variables_initializer(m.variables))
+
+    # update state
+    self.evaluate(m(100.))
+    self.evaluate(m(200.))
+
+    # save checkpoint and then add an update
+    save_path = checkpoint.save(checkpoint_prefix)
+    self.evaluate(m(1000.))
+
+    # restore to the same checkpoint sum object (= 300)
+    checkpoint.restore(save_path).assert_consumed().run_restore_ops()
+    self.evaluate(m(300.))
+    self.assertEqual(600., self.evaluate(m.result()))
+
+    # restore to a different checkpoint sum object
+    restore_sum = metrics.Sum()
+    restore_checkpoint = checkpointable_utils.Checkpoint(sum=restore_sum)
+    status = restore_checkpoint.restore(save_path)
+    restore_update = restore_sum(300.)
+    status.assert_consumed().run_restore_ops()
+    self.evaluate(restore_update)
+    self.assertEqual(600., self.evaluate(restore_sum.result()))
+
+
 @test_util.run_all_in_graph_and_eager_modes
 class KerasMeanTest(test.TestCase):
 
diff --git a/tensorflow/python/keras/utils/metrics_utils.py b/tensorflow/python/keras/utils/metrics_utils.py
index 108d6ee642..fe6373d338 100644
--- a/tensorflow/python/keras/utils/metrics_utils.py
+++ b/tensorflow/python/keras/utils/metrics_utils.py
@@ -37,11 +37,27 @@ from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import weights_broadcast_ops
 from tensorflow.python.util import tf_decorator
-
+from tensorflow.python.util.tf_export import keras_export
 
 NEG_INF = -1e10
 
 
+@keras_export('keras.metrics.Reduction')
+class Reduction(Enum):
+  """Types of metrics reduction.
+
+  Contains the following values:
+
+  * `SUM`: Scalar sum of weighted values.
+  * `SUM_OVER_BATCH_SIZE`: Scalar sum of weighted values divided by
+        number of elements.
+  * `WEIGHTED_MEAN`: Scalar sum of weighted values divided by sum of weights.
+  """
+  SUM = 'sum'
+  SUM_OVER_BATCH_SIZE = 'sum_over_batch_size'
+  WEIGHTED_MEAN = 'weighted_mean'
+
+
 def update_state_wrapper(update_state_fn):
   """Decorator to wrap metric `update_state()` with `add_update()`.
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-accuracy.pbtxt
index 6756beca1d..5f0079762f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-accuracy.pbtxt
@@ -3,6 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Accuracy\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-accuracy.pbtxt
index bec0b20aa5..a0a3ae8900 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-accuracy.pbtxt
@@ -3,6 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.BinaryAccuracy\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
index 71dc294dc3..587ffddac9 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
@@ -3,6 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.CategoricalAccuracy\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-hinge.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-hinge.pbtxt
index 43024e738a..99414dd54e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-hinge.pbtxt
@@ -3,6 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.CategoricalHinge\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-cosine-proximity.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-cosine-proximity.pbtxt
index 1e39385f81..b059e039de 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-cosine-proximity.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-cosine-proximity.pbtxt
@@ -3,6 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.CosineProximity\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-hinge.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-hinge.pbtxt
index f8d47f3771..f45a57df41 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-hinge.pbtxt
@@ -3,6 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Hinge\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
index bf7fc7cfc5..11538c6c09 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
@@ -3,6 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanAbsoluteError\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
index 59bb767d35..adec3feffa 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
@@ -3,6 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanAbsolutePercentageError\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-error.pbtxt
index 91f4712312..af960fa543 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-error.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-error.pbtxt
@@ -3,6 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanSquaredError\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
index 205e15d439..417f92f273 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
@@ -3,6 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanSquaredLogarithmicError\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean.pbtxt
index eec26ffce7..b089109baa 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.keras.metrics.Mean"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-reduction.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-reduction.pbtxt
new file mode 100644
index 0000000000..8f74f1a1bc
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-reduction.pbtxt
@@ -0,0 +1,16 @@
+path: "tensorflow.keras.metrics.Reduction"
+tf_class {
+  is_instance: "<enum \'Reduction\'>"
+  member {
+    name: "SUM"
+    mtype: "<enum \'Reduction\'>"
+  }
+  member {
+    name: "SUM_OVER_BATCH_SIZE"
+    mtype: "<enum \'Reduction\'>"
+  }
+  member {
+    name: "WEIGHTED_MEAN"
+    mtype: "<enum \'Reduction\'>"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
index 571c2bf9d3..5a7bef4714 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
@@ -3,6 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.SparseCategoricalAccuracy\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-squared-hinge.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-squared-hinge.pbtxt
index 71c047e973..15895daf42 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-squared-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-squared-hinge.pbtxt
@@ -3,6 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.SquaredHinge\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sum.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sum.pbtxt
new file mode 100644
index 0000000000..5ca1c6c839
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sum.pbtxt
@@ -0,0 +1,197 @@
+path: "tensorflow.keras.metrics.Sum"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Sum\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'sum\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'values\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
index cae251cdfe..e95e0bfd29 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
@@ -60,6 +60,10 @@ tf_module {
     name: "Recall"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "Reduction"
+    mtype: "<class \'enum.EnumMeta\'>"
+  }
   member {
     name: "SensitivityAtSpecificity"
     mtype: "<type \'type\'>"
@@ -76,6 +80,10 @@ tf_module {
     name: "SquaredHinge"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "Sum"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "TrueNegatives"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-accuracy.pbtxt
index 6756beca1d..5f0079762f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-accuracy.pbtxt
@@ -3,6 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Accuracy\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-accuracy.pbtxt
index bec0b20aa5..a0a3ae8900 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-accuracy.pbtxt
@@ -3,6 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.BinaryAccuracy\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
index 71dc294dc3..587ffddac9 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
@@ -3,6 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.CategoricalAccuracy\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-hinge.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-hinge.pbtxt
index 43024e738a..99414dd54e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-hinge.pbtxt
@@ -3,6 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.CategoricalHinge\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-cosine-proximity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-cosine-proximity.pbtxt
index 1e39385f81..b059e039de 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-cosine-proximity.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-cosine-proximity.pbtxt
@@ -3,6 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.CosineProximity\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-hinge.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-hinge.pbtxt
index f8d47f3771..f45a57df41 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-hinge.pbtxt
@@ -3,6 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Hinge\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
index bf7fc7cfc5..11538c6c09 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
@@ -3,6 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanAbsoluteError\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
index 59bb767d35..adec3feffa 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
@@ -3,6 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanAbsolutePercentageError\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-error.pbtxt
index 91f4712312..af960fa543 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-error.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-error.pbtxt
@@ -3,6 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanSquaredError\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
index 205e15d439..417f92f273 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
@@ -3,6 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanSquaredLogarithmicError\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean.pbtxt
index eec26ffce7..b089109baa 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.keras.metrics.Mean"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-reduction.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-reduction.pbtxt
new file mode 100644
index 0000000000..8f74f1a1bc
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-reduction.pbtxt
@@ -0,0 +1,16 @@
+path: "tensorflow.keras.metrics.Reduction"
+tf_class {
+  is_instance: "<enum \'Reduction\'>"
+  member {
+    name: "SUM"
+    mtype: "<enum \'Reduction\'>"
+  }
+  member {
+    name: "SUM_OVER_BATCH_SIZE"
+    mtype: "<enum \'Reduction\'>"
+  }
+  member {
+    name: "WEIGHTED_MEAN"
+    mtype: "<enum \'Reduction\'>"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
index 571c2bf9d3..5a7bef4714 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
@@ -3,6 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.SparseCategoricalAccuracy\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-squared-hinge.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-squared-hinge.pbtxt
index 71c047e973..15895daf42 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-squared-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-squared-hinge.pbtxt
@@ -3,6 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.SquaredHinge\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sum.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sum.pbtxt
new file mode 100644
index 0000000000..5ca1c6c839
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sum.pbtxt
@@ -0,0 +1,197 @@
+path: "tensorflow.keras.metrics.Sum"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Sum\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'sum\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'values\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
index cae251cdfe..e95e0bfd29 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
@@ -60,6 +60,10 @@ tf_module {
     name: "Recall"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "Reduction"
+    mtype: "<class \'enum.EnumMeta\'>"
+  }
   member {
     name: "SensitivityAtSpecificity"
     mtype: "<type \'type\'>"
@@ -76,6 +80,10 @@ tf_module {
     name: "SquaredHinge"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "Sum"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "TrueNegatives"
     mtype: "<type \'type\'>"
-- 
GitLab


From 88483960d84402afbce36ab193963d05052fe17e Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Wed, 23 Jan 2019 16:04:42 -0800
Subject: [PATCH 1148/2345] Reuse parameter buffer if the input parameter has
 an output aliasing.

Most of the time We assume parameters are readonly. However if a parameter is aliased with an output, we can safely assume it is writable (its content will be overwritten by output anyway).

In the future, we can also add options in our client API to allow user explicitly set a parameter to be writable.

PiperOrigin-RevId: 230621258
---
 .../compiler/xla/service/buffer_assignment.cc | 32 +++++++++++------
 .../compiler/xla/service/buffer_assignment.h  | 13 +++++--
 .../xla/service/buffer_assignment_test.cc     | 34 +++++++++++++++++++
 3 files changed, 67 insertions(+), 12 deletions(-)

diff --git a/tensorflow/compiler/xla/service/buffer_assignment.cc b/tensorflow/compiler/xla/service/buffer_assignment.cc
index d07615b828..52f72b683f 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment.cc
+++ b/tensorflow/compiler/xla/service/buffer_assignment.cc
@@ -1011,10 +1011,14 @@ Status BufferAssigner::AssignBuffersForComputation(
       // callers.
       BufferAllocation* allocation =
           assignment->NewAllocation(*buffer, buffer_size);
+      bool parameter_has_alias =
+          assignment->module().input_output_alias_config().ParameterHasAlias(
+              instruction->parameter_number(), buffer->index());
       allocation->set_entry_computation_parameter(
-          instruction->parameter_number(), buffer->index());
-      VLOG(3) << "New allocation #" << allocation->index()
-              << " for entry computation parameter: " << *buffer;
+          instruction->parameter_number(), buffer->index(),
+          parameter_has_alias);
+      VLOG(3) << "Mark allocation #" << allocation->index()
+              << " as entry computation parameter: " << *buffer;
       continue;
     }
 
@@ -1416,12 +1420,14 @@ BufferAssigner::MergeColocatedBufferSets(
           << colocated_buffer_sets.size();
 
   // Returns true if the given buffer is for the entry parameter.
-  auto is_entry_parameter = [](const LogicalBuffer& buffer) {
+  auto is_readonly_entry_parameter = [](const LogicalBuffer& buffer) {
     auto* instruction = buffer.instruction();
     auto* computation = instruction->parent();
     auto* module = computation->parent();
     return instruction->opcode() == HloOpcode::kParameter &&
-           computation == module->entry_computation();
+           computation == module->entry_computation() &&
+           !module->input_output_alias_config().ParameterHasAlias(
+               instruction->parameter_number(), buffer.index());
   };
 
   std::vector<bool> set_can_be_merged(colocated_buffer_sets.size(), true);
@@ -1443,7 +1449,7 @@ BufferAssigner::MergeColocatedBufferSets(
   for (int64 i = 0; i < colocated_buffer_sets.size(); ++i) {
     for (auto& buffer : colocated_buffer_sets[i]) {
       if (buffer_liveness.MaybeLiveOut(*buffer) ||
-          is_entry_parameter(*buffer) ||
+          is_readonly_entry_parameter(*buffer) ||
           buffer->instruction()->opcode() == HloOpcode::kConstant) {
         set_can_be_merged[i] = false;
         break;
@@ -1733,10 +1739,6 @@ void BufferAssigner::AssignColocatedBufferSets(
         // module-level scope, we can allow buffers to be shared across
         // computations (in some cases).
         allocation = assignment->NewAllocation(*buffer, buffer_size);
-        if (entry_parameter_number >= 0) {
-          allocation->set_entry_computation_parameter(
-              entry_parameter_number, *entry_parameter_shape_idx);
-        }
         if (is_constant) {
           allocation->set_constant(true);
         }
@@ -1750,6 +1752,16 @@ void BufferAssigner::AssignColocatedBufferSets(
       }
       colocated_buffers->insert(buffer);
     }
+
+    // If an allocation contains a parameter, set corresponding fields.
+    if (entry_parameter_number >= 0) {
+      bool parameter_has_alias =
+          assignment->module().input_output_alias_config().ParameterHasAlias(
+              entry_parameter_number, *entry_parameter_shape_idx);
+      allocation->set_entry_computation_parameter(entry_parameter_number,
+                                                  *entry_parameter_shape_idx,
+                                                  parameter_has_alias);
+    }
   }
 }
 
diff --git a/tensorflow/compiler/xla/service/buffer_assignment.h b/tensorflow/compiler/xla/service/buffer_assignment.h
index 4baab9b6ad..448dec3b1a 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment.h
+++ b/tensorflow/compiler/xla/service/buffer_assignment.h
@@ -96,7 +96,11 @@ class BufferAllocation {
   // Whether this allocation is readonly i.e. backed by memory we cannot write
   // to.
   bool is_readonly() const {
-    return is_entry_computation_parameter() || is_constant();
+    // Entry parameters are generally readonly, except when they are aliased
+    // with any output.
+    return (is_entry_computation_parameter() &&
+            !is_parameter_aliased_with_output_) ||
+           is_constant();
   }
 
   bool is_tuple() const { return is_tuple_; }
@@ -274,8 +278,10 @@ class BufferAllocation {
   void AddAssignment(const LogicalBuffer& buffer, int64 offset, int64 size);
 
   void set_entry_computation_parameter(int64 parameter_number,
-                                       ShapeIndex param_shape_index) {
+                                       ShapeIndex param_shape_index,
+                                       bool parameter_aliased_with_output) {
     is_entry_computation_parameter_ = true;
+    is_parameter_aliased_with_output_ = parameter_aliased_with_output;
     parameter_number_ = parameter_number;
     param_shape_index_ = std::move(param_shape_index);
   }
@@ -305,6 +311,9 @@ class BufferAllocation {
   // outlast the computation.
   bool is_entry_computation_parameter_ = false;
 
+  // Whether this entry computation parameter is aliased with output.
+  bool is_parameter_aliased_with_output_ = false;
+
   // If this allocation holds an entry computation parameter, this field
   // indicates the index (starting from 0) of the parameter.
   int64 parameter_number_ = 0;
diff --git a/tensorflow/compiler/xla/service/buffer_assignment_test.cc b/tensorflow/compiler/xla/service/buffer_assignment_test.cc
index 1b4e93a2f3..580bc2f433 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment_test.cc
+++ b/tensorflow/compiler/xla/service/buffer_assignment_test.cc
@@ -465,6 +465,40 @@ TEST_F(BufferAssignmentTest, Basic) {
   GetAssignedOutputAllocation(*buffers, sub);
 }
 
+TEST_F(BufferAssignmentTest, AliasedParamCanBeReused) {
+  // If an input buffer and output buffer aliases, the input buffer can be
+  // reused for other intermediate results.
+  //
+  // param0[100] ----- (neg1) -- (neg2)
+  //    |                           |
+  //    + -------- Aliased ---------+
+
+  auto builder = HloComputation::Builder(TestName());
+
+  auto param = builder.AddInstruction(
+      HloInstruction::CreateParameter(0, f32vec100_, "p0"));
+  auto neg_1 = builder.AddInstruction(
+      HloInstruction::CreateUnary(f32vec100_, HloOpcode::kNegate, param));
+  auto neg_2 = builder.AddInstruction(
+      HloInstruction::CreateUnary(f32vec100_, HloOpcode::kNegate, neg_1));
+
+  auto module = CreateNewVerifiedModule();
+  module->AddEntryComputation(builder.Build());
+
+  TF_ASSERT_OK(module->input_output_alias_config().SetUpAlias(
+      {}, 0, {}, HloInputOutputAliasConfig::kUserAlias));
+
+  auto buffers = RunBufferAssignment(module.get());
+
+  BufferAllocation param_buffer = GetAssignedInputAllocation(*buffers, param);
+  BufferAllocation neg_1_buffer = GetAllocation(*buffers, neg_1, {});
+  BufferAllocation neg_2_buffer = GetAllocation(*buffers, neg_2, {});
+
+  // Everything use one buffer.
+  EXPECT_EQ(param_buffer.index(), neg_1_buffer.index());
+  EXPECT_EQ(neg_2_buffer.index(), neg_1_buffer.index());
+}
+
 TEST_F(BufferAssignmentTest, AddCannotReuse) {
   // Pass in a special rule to indicate that "add" cannot reuse any buffer.
   //
-- 
GitLab


From e71b066ff86e9f09d922460de7a98236057c2718 Mon Sep 17 00:00:00 2001
From: Sachin Joglekar <srjoglekar@google.com>
Date: Wed, 23 Jan 2019 16:07:54 -0800
Subject: [PATCH 1149/2345] Adds check for empty-shaped tensor in
 tflite::Verify

PiperOrigin-RevId: 230621820
---
 tensorflow/lite/tools/verifier.cc      |   4 +
 tensorflow/lite/tools/verifier_test.cc | 121 +++++++++++++++++++++----
 2 files changed, 108 insertions(+), 17 deletions(-)

diff --git a/tensorflow/lite/tools/verifier.cc b/tensorflow/lite/tools/verifier.cc
index 02d6e6b23c..8631bef67b 100644
--- a/tensorflow/lite/tools/verifier.cc
+++ b/tensorflow/lite/tools/verifier.cc
@@ -105,6 +105,10 @@ bool VerifyStringTensorBuffer(const Buffer& buffer,
 bool VerifyNumericTensorBuffer(const Tensor& tensor, const Buffer& buffer,
                                ErrorReporter* error_reporter) {
   uint64_t bytes_required = 1;
+  if (!tensor.shape()) {
+    ReportError(error_reporter, "Tensor shape is empty");
+    return false;
+  }
   for (int dim : *tensor.shape()) {
     bytes_required *= dim;
     if (bytes_required > UINT_MAX) {
diff --git a/tensorflow/lite/tools/verifier_test.cc b/tensorflow/lite/tools/verifier_test.cc
index 98abafad92..083f00d445 100644
--- a/tensorflow/lite/tools/verifier_test.cc
+++ b/tensorflow/lite/tools/verifier_test.cc
@@ -17,7 +17,9 @@ limitations under the License.
 
 #include "flatbuffers/flatbuffers.h"
 #include "flatbuffers/util.h"
+#include <gmock/gmock.h>
 #include <gtest/gtest.h>
+#include "tensorflow/core/framework/numeric_types.h"
 #include "tensorflow/lite/allocation.h"
 #include "tensorflow/lite/error_reporter.h"
 #include "tensorflow/lite/op_resolver.h"
@@ -25,13 +27,29 @@ limitations under the License.
 #include "tensorflow/lite/testing/util.h"
 #include "tensorflow/lite/tools/verifier.h"
 #include "tensorflow/lite/version.h"
-#include "tensorflow/core/framework/numeric_types.h"
 
 namespace tflite {
 
 using flatbuffers::FlatBufferBuilder;
 using flatbuffers::Offset;
 
+class MockErrorReporter : public ErrorReporter {
+ public:
+  MockErrorReporter() : buffer_size_(0) {}
+  int Report(const char* format, va_list args) override {
+    buffer_size_ = vsnprintf(buffer_, kBufferSize, format, args);
+    return buffer_size_;
+  }
+  int GetBufferSize() { return buffer_size_; }
+
+  string GetAsString() const { return string(buffer_, buffer_size_); }
+
+ private:
+  static constexpr int kBufferSize = 256;
+  char buffer_[kBufferSize];
+  int buffer_size_;
+};
+
 // Build single subgraph model.
 class TfLiteFlatbufferModelBuilder {
  public:
@@ -60,6 +78,12 @@ class TfLiteFlatbufferModelBuilder {
       buffer_index = buffers_.size();
       buffers_.push_back(CreateBuffer(builder_, builder_.CreateVector(buffer)));
     }
+    if (shape.empty()) {
+      tensors_.push_back(CreateTensorDirect(builder_, /*shape=*/nullptr, type,
+                                            buffer_index, name,
+                                            /*quantization=*/0));
+      return;
+    }
     tensors_.push_back(CreateTensorDirect(builder_, &shape, type, buffer_index,
                                           name, /*quantization=*/0));
   }
@@ -92,13 +116,16 @@ class TfLiteFlatbufferModelBuilder {
 
   bool Verify() {
     return tflite::Verify(builder_.GetBufferPointer(), builder_.GetSize(),
-                          resolver_, DefaultErrorReporter());
+                          resolver_, &mock_reporter_);
   }
 
+  string GetErrorString() { return mock_reporter_.GetAsString(); }
+
  private:
   FlatBufferBuilder builder_;
   MutableOpResolver resolver_;
   TfLiteRegistration fake_op_;
+  MockErrorReporter mock_reporter_;
   std::vector<Offset<Operator>> operators_;
   std::vector<Offset<OperatorCode>> operator_codes_;
   std::vector<Offset<Tensor>> tensors_;
@@ -112,8 +139,27 @@ TEST(VerifyModel, TestEmptyModel) {
                            /*description=*/0, /*buffers=*/0);
   ::tflite::FinishModelBuffer(builder, model);
 
+  MockErrorReporter mock_reporter;
   ASSERT_FALSE(Verify(builder.GetBufferPointer(), builder.GetSize(),
-                      MutableOpResolver{}, DefaultErrorReporter()));
+                      MutableOpResolver{}, &mock_reporter));
+  EXPECT_THAT(mock_reporter.GetAsString(),
+              ::testing::ContainsRegex("Missing 'subgraphs' section."));
+}
+
+TEST(VerifyModel, TestEmptyShape) {
+  TfLiteFlatbufferModelBuilder builder({}, {"test"});
+  builder.AddOperator({0, 1}, {2}, BuiltinOperator_CUSTOM, "test");
+  builder.AddTensor({2, 3}, TensorType_UINT8, {1, 2, 3, 4, 5, 6}, "input");
+  builder.AddTensor({}, TensorType_UINT8, {1, 2, 3, 4, 5, 6}, "inputtwo");
+  builder.AddTensor(
+      {2}, TensorType_STRING,
+      {2, 0, 0, 0, 16, 0, 0, 0, 17, 0, 0, 0, 19, 0, 0, 0, 'A', 'B', 'C'},
+      "data");
+  builder.AddTensor({2, 3}, TensorType_INT32, {}, "output");
+  builder.FinishModel({0, 1}, {2});
+  ASSERT_FALSE(builder.Verify());
+  EXPECT_THAT(builder.GetErrorString(),
+              ::testing::ContainsRegex("Tensor shape is empty"));
 }
 
 TEST(VerifyModel, TestSimpleModel) {
@@ -127,12 +173,16 @@ TEST(VerifyModel, TestSimpleModel) {
   builder.AddTensor({2, 3}, TensorType_INT32, {}, "output");
   builder.FinishModel({0, 1}, {2});
   ASSERT_TRUE(builder.Verify());
+  EXPECT_EQ("", builder.GetErrorString());
 }
 
 TEST(VerifyModel, TestCorruptedData) {
   std::string model = "123";
-  ASSERT_FALSE(Verify(model.data(), model.size(), MutableOpResolver{},
-                      /*error_reporter=*/nullptr));
+  MockErrorReporter mock_reporter;
+  ASSERT_FALSE(
+      Verify(model.data(), model.size(), MutableOpResolver{}, &mock_reporter));
+  EXPECT_THAT(mock_reporter.GetAsString(),
+              ::testing::ContainsRegex("Invalid flatbuffer format"));
 }
 
 TEST(VerifyModel, TestUnsupportedVersion) {
@@ -140,8 +190,11 @@ TEST(VerifyModel, TestUnsupportedVersion) {
   auto model = CreateModel(builder, /*version=*/1, /*operator_codes=*/0,
                            /*subgraphs=*/0, /*description=*/0, /*buffers=*/0);
   ::tflite::FinishModelBuffer(builder, model);
+  MockErrorReporter mock_reporter;
   ASSERT_FALSE(Verify(builder.GetBufferPointer(), builder.GetSize(),
-                      MutableOpResolver{}, DefaultErrorReporter()));
+                      MutableOpResolver{}, &mock_reporter));
+  EXPECT_THAT(mock_reporter.GetAsString(),
+              ::testing::ContainsRegex("Invalid model version 1"));
 }
 
 TEST(VerifyModel, TestRandomModificationIsNotAllowed) {
@@ -166,6 +219,10 @@ TEST(VerifyModel, TestIntTensorShapeIsGreaterThanBuffer) {
   builder.AddTensor({2, 3}, TensorType_UINT8, {1, 2, 3, 4}, "input");
   builder.FinishModel({}, {});
   ASSERT_FALSE(builder.Verify());
+  EXPECT_THAT(
+      builder.GetErrorString(),
+      ::testing::ContainsRegex(
+          "Tensor requires 6 bytes, but is allocated with 4 bytes buffer"));
 }
 
 TEST(VerifyModel, TestIntTensorShapeIsSmallerThanBuffer) {
@@ -173,6 +230,10 @@ TEST(VerifyModel, TestIntTensorShapeIsSmallerThanBuffer) {
   builder.AddTensor({2, 1}, TensorType_UINT8, {1, 2, 3, 4}, "input");
   builder.FinishModel({}, {});
   ASSERT_FALSE(builder.Verify());
+  EXPECT_THAT(
+      builder.GetErrorString(),
+      ::testing::ContainsRegex(
+          "Tensor requires 2 bytes, but is allocated with 4 bytes buffer"));
 }
 
 TEST(VerifyModel, TestIntTensorShapeOverflow) {
@@ -181,6 +242,8 @@ TEST(VerifyModel, TestIntTensorShapeOverflow) {
                     "input");
   builder.FinishModel({}, {});
   ASSERT_FALSE(builder.Verify());
+  EXPECT_THAT(builder.GetErrorString(),
+              ::testing::ContainsRegex("Tensor dimension overflow"));
 }
 
 TEST(VerifyModel, TensorBufferIsNotValid) {
@@ -203,8 +266,12 @@ TEST(VerifyModel, TensorBufferIsNotValid) {
                            builder.CreateString("SmartReply"), buffers);
 
   ::tflite::FinishModelBuffer(builder, model);
+  MockErrorReporter mock_reporter;
   ASSERT_FALSE(Verify(builder.GetBufferPointer(), builder.GetSize(),
-                      MutableOpResolver{}, DefaultErrorReporter()));
+                      MutableOpResolver{}, &mock_reporter));
+  EXPECT_THAT(
+      mock_reporter.GetAsString(),
+      ::testing::ContainsRegex("Missing 'operators' section in subgraph."));
 }
 
 TEST(VerifyModel, StringTensorHasInvalidNumString) {
@@ -215,6 +282,10 @@ TEST(VerifyModel, StringTensorHasInvalidNumString) {
       "input");
   builder.FinishModel({}, {});
   ASSERT_FALSE(builder.Verify());
+  EXPECT_THAT(builder.GetErrorString(),
+              ::testing::ContainsRegex(
+                  "String tensor buffer requires at least -2147483640 bytes, "
+                  "but is allocated with 18 bytes"));
 }
 
 TEST(VerifyModel, StringTensorOffsetTooSmall) {
@@ -224,6 +295,9 @@ TEST(VerifyModel, StringTensorOffsetTooSmall) {
       {2, 0, 0, 0, 12, 0, 0, 0, 17, 0, 0, 0, 18, 0, 0, 0, 'A', 'B'}, "input");
   builder.FinishModel({}, {});
   ASSERT_FALSE(builder.Verify());
+  EXPECT_THAT(builder.GetErrorString(),
+              ::testing::ContainsRegex(
+                  "String tensor buffer initial offset must be: 16"));
 }
 
 TEST(VerifyModel, StringTensorOffsetOutOfRange) {
@@ -233,6 +307,9 @@ TEST(VerifyModel, StringTensorOffsetOutOfRange) {
       {2, 0, 0, 0, 16, 0, 0, 0, 17, 0, 0, 0, 22, 0, 0, 0, 'A', 'B'}, "input");
   builder.FinishModel({}, {});
   ASSERT_FALSE(builder.Verify());
+  EXPECT_THAT(
+      builder.GetErrorString(),
+      ::testing::ContainsRegex("String tensor buffer is invalid: index 2"));
 }
 
 TEST(VerifyModel, StringTensorIsLargerThanRequired) {
@@ -243,37 +320,47 @@ TEST(VerifyModel, StringTensorIsLargerThanRequired) {
       "input");
   builder.FinishModel({}, {});
   ASSERT_FALSE(builder.Verify());
+  EXPECT_THAT(
+      builder.GetErrorString(),
+      ::testing::ContainsRegex("String tensor buffer last offset must be 19"));
 }
 
 TEST(VerifyModel, AllOpsAreSupported) {
   TfLiteFlatbufferModelBuilder builder({BuiltinOperator_ADD}, {"CustomOp"});
-  builder.AddTensor({2, 3}, TensorType_UINT8, {1, 2, 3, 4}, "input1");
-  builder.AddTensor({2, 3}, TensorType_UINT8, {1, 2, 3, 4}, "input2");
-  builder.AddTensor({2, 3}, TensorType_UINT8, {}, "output");
+  builder.AddTensor({2, 2}, TensorType_UINT8, {1, 2, 3, 4}, "input1");
+  builder.AddTensor({2, 2}, TensorType_UINT8, {1, 2, 3, 4}, "input2");
+  builder.AddTensor({2, 2}, TensorType_UINT8, {}, "output");
   builder.AddOperator({0, 1}, {2}, BuiltinOperator_ADD, nullptr);
   builder.AddOperator({0, 1}, {2}, BuiltinOperator_CUSTOM, "CustomOp");
   builder.FinishModel({}, {});
-  ASSERT_FALSE(builder.Verify());
+  ASSERT_TRUE(builder.Verify());
+  EXPECT_EQ("", builder.GetErrorString());
 }
 
 TEST(VerifyModel, UseUnsupportedBuiltinOps) {
   TfLiteFlatbufferModelBuilder builder({BuiltinOperator_SUB}, {"CustomOp"});
-  builder.AddTensor({2, 3}, TensorType_UINT8, {1, 2, 3, 4}, "input1");
-  builder.AddTensor({2, 3}, TensorType_UINT8, {1, 2, 3, 4}, "input2");
-  builder.AddTensor({2, 3}, TensorType_UINT8, {}, "output");
+  builder.AddTensor({2, 2}, TensorType_UINT8, {1, 2, 3, 4}, "input1");
+  builder.AddTensor({2, 2}, TensorType_UINT8, {1, 2, 3, 4}, "input2");
+  builder.AddTensor({2, 2}, TensorType_UINT8, {}, "output");
   builder.AddOperator({0, 1}, {2}, BuiltinOperator_ADD, nullptr);
   builder.FinishModel({}, {});
   ASSERT_FALSE(builder.Verify());
+  EXPECT_THAT(
+      builder.GetErrorString(),
+      ::testing::ContainsRegex("Unsupported builtin op: ADD, version: 1"));
 }
 
 TEST(VerifyModel, UseUnsupportedCustomOps) {
   TfLiteFlatbufferModelBuilder builder({BuiltinOperator_ADD}, {"NewOp"});
-  builder.AddTensor({2, 3}, TensorType_UINT8, {1, 2, 3, 4}, "input1");
-  builder.AddTensor({2, 3}, TensorType_UINT8, {1, 2, 3, 4}, "input2");
-  builder.AddTensor({2, 3}, TensorType_UINT8, {}, "output");
+  builder.AddTensor({2, 2}, TensorType_UINT8, {1, 2, 3, 4}, "input1");
+  builder.AddTensor({2, 2}, TensorType_UINT8, {1, 2, 3, 4}, "input2");
+  builder.AddTensor({2, 2}, TensorType_UINT8, {}, "output");
   builder.AddOperator({0, 1}, {2}, BuiltinOperator_CUSTOM, "Not supported");
   builder.FinishModel({}, {});
   ASSERT_FALSE(builder.Verify());
+  EXPECT_THAT(builder.GetErrorString(),
+              ::testing::ContainsRegex(
+                  "Unsupported custom op: Not supported, version: 1"));
 }
 
 // TODO(yichengfan): make up malicious files to test with.
-- 
GitLab


From 940bec5d95c900ea66a3132bc539a4960df50fb1 Mon Sep 17 00:00:00 2001
From: Suharsh Sivakumar <suharshs@google.com>
Date: Wed, 23 Jan 2019 16:31:36 -0800
Subject: [PATCH 1150/2345] Fix unitialized value msan failure.

PiperOrigin-RevId: 230626015
---
 tensorflow/lite/string_util_test.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/lite/string_util_test.cc b/tensorflow/lite/string_util_test.cc
index 90ae553cab..6fc7de90ea 100644
--- a/tensorflow/lite/string_util_test.cc
+++ b/tensorflow/lite/string_util_test.cc
@@ -36,6 +36,8 @@ TEST(StringUtil, TestStringUtil) {
   char data[] = {1, 0, 0, 0, 12, 0, 0, 0, 15, 0, 0, 0, 'X', 'Y', 'Z'};
 
   TfLiteQuantization quant;
+  quant.type = kTfLiteNoQuantization;
+  quant.params = nullptr;
   interpreter.SetTensorParametersReadOnly(2, kTfLiteString, "", {1}, quant,
                                           data, 15);
   TfLiteTensor* t2 = interpreter.tensor(2);
-- 
GitLab


From 6b06654f4ce7625c8ae419e55766a725f9df2f7c Mon Sep 17 00:00:00 2001
From: James Ring <sjr@google.com>
Date: Wed, 23 Jan 2019 16:36:16 -0800
Subject: [PATCH 1151/2345] Rename UnsafeCopyFromInternal to BitcastFrom.

The new BitcastFrom method returns errors::InvalidArgument if preconditions are
not met. UnsafeCopyFromInternal is retained but marked as deprecated.

PiperOrigin-RevId: 230626930
---
 tensorflow/core/framework/tensor.cc     | 21 ++++++++++-----
 tensorflow/core/framework/tensor.h      | 35 +++++++++++++++++++++----
 tensorflow/core/kernels/bitcast_op.cc   |  5 ++--
 tensorflow/core/kernels/cast_op.cc      |  6 ++---
 tensorflow/core/kernels/mkl_conv_ops.cc |  3 ++-
 5 files changed, 53 insertions(+), 17 deletions(-)

diff --git a/tensorflow/core/framework/tensor.cc b/tensorflow/core/framework/tensor.cc
index ab492a2189..ecbffecd66 100644
--- a/tensorflow/core/framework/tensor.cc
+++ b/tensorflow/core/framework/tensor.cc
@@ -42,6 +42,7 @@ limitations under the License.
 #include "tensorflow/core/framework/variant_tensor_data.h"
 #include "tensorflow/core/lib/core/coding.h"
 #include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/gtl/inlined_vector.h"
 #include "tensorflow/core/lib/gtl/stl_util.h"
 #include "tensorflow/core/lib/strings/str_util.h"
@@ -650,14 +651,21 @@ void Tensor::CopyFromInternal(const Tensor& other, const TensorShape& shape) {
   }
 }
 
-void Tensor::UnsafeCopyFromInternal(const Tensor& other, DataType dtype,
-                                    const TensorShape& shape) {
+Status Tensor::BitcastFrom(const Tensor& other, DataType dtype,
+                           const TensorShape& shape) {
   int in_size = DataTypeSize(other.dtype());
   int out_size = DataTypeSize(dtype);
-  CHECK_NE(in_size, 0);
-  CHECK_NE(out_size, 0);
-  CHECK_EQ(shape.num_elements() * out_size,
-           other.shape().num_elements() * in_size);
+  if (in_size == 0) {
+    return errors::InvalidArgument("other tensor has zero-sized data type");
+  }
+  if (out_size == 0) {
+    return errors::InvalidArgument("specified output type is zero-sized");
+  }
+  if (shape.num_elements() * out_size !=
+      other.shape().num_elements() * in_size) {
+    return errors::InvalidArgument(
+        "input and output shapes/data type sizes are not compatible");
+  }
   shape_ = shape;
   shape_.set_data_type(dtype);
   if (buf_ != other.buf_) {
@@ -665,6 +673,7 @@ void Tensor::UnsafeCopyFromInternal(const Tensor& other, DataType dtype,
     buf_ = other.buf_;
     RefIfNonNull(buf_);
   }
+  return Status::OK();
 }
 
 // Notice that buf_ either points to a regular TensorBuffer or a SubBuffer.
diff --git a/tensorflow/core/framework/tensor.h b/tensorflow/core/framework/tensor.h
index 4cfad893d5..6454cb818f 100644
--- a/tensorflow/core/framework/tensor.h
+++ b/tensorflow/core/framework/tensor.h
@@ -554,12 +554,37 @@ class Tensor {
   /// REQUIRES: `DataTypeCanUseMemcpy(dtype())`.
   StringPiece tensor_data() const;
 
-  /// Copy the other tensor into this tensor and reshape it and reinterpret the
-  /// buffer's datatype.
+  /// Copy the other tensor into this tensor, reshape it and reinterpret the
+  /// buffer's datatype. If Status::OK() is returned, the two tensors now share
+  /// the same underlying storage.
   ///
-  /// This tensor shares other's underlying storage.
-  void UnsafeCopyFromInternal(const Tensor&, DataType dtype,
-                              const TensorShape&);
+  /// This call requires that the `other` tensor and the given type and shape
+  /// are "compatible" (i.e. they occupy the same number of bytes).
+  ///
+  /// Specifically:
+  ///
+  /// shape.num_elements() * DataTypeSize(type)
+  ///
+  /// must equal
+  ///
+  /// other.num_elements() * DataTypeSize(other.dtype())
+  ///
+  /// In addition, this function requires:
+  ///   * DataTypeSize(other.dtype()) != 0
+  ///   * DataTypeSize(type) != 0
+  ///
+  /// If any of the requirements are not met, errors::InvalidArgument is
+  /// returned.
+  Status BitcastFrom(const Tensor& other, DataType dtype,
+                     const TensorShape& shape);
+
+  /// Like BitcastFrom, but CHECK fails if any preconditions are not met.
+  ///
+  /// Deprecated. Use BitcastFrom instead and check the returned Status.
+  void UnsafeCopyFromInternal(const Tensor& other, DataType dtype,
+                              const TensorShape& shape) {
+    TF_CHECK_OK(BitcastFrom(other, dtype, shape));
+  }
 
  private:
   // Returns true if the refcount on buf_ and any possible underlying root
diff --git a/tensorflow/core/kernels/bitcast_op.cc b/tensorflow/core/kernels/bitcast_op.cc
index f602cfa428..2397b8741c 100644
--- a/tensorflow/core/kernels/bitcast_op.cc
+++ b/tensorflow/core/kernels/bitcast_op.cc
@@ -59,8 +59,9 @@ class BitcastOp : public OpKernel {
     }
     Tensor output_tensor;
 
-    output_tensor.UnsafeCopyFromInternal(input_tensor, output_data_type_,
-                                         adjusted_shape);
+    OP_REQUIRES_OK(context,
+                   output_tensor.BitcastFrom(input_tensor, output_data_type_,
+                                             adjusted_shape));
     context->set_output(0, output_tensor);
   }
 
diff --git a/tensorflow/core/kernels/cast_op.cc b/tensorflow/core/kernels/cast_op.cc
index 3a72567655..5306c77102 100644
--- a/tensorflow/core/kernels/cast_op.cc
+++ b/tensorflow/core/kernels/cast_op.cc
@@ -99,9 +99,9 @@ void CastOpBase::Compute(OpKernelContext* ctx) {
   } else {
     Tensor in;
     if (external_src_dtype_ != src_dtype_) {
-      // If the type is a quantized type we need to do an UnsafeCopyFromInternal
-      // since the src_dtype_ is different from external_src_type_.
-      in.UnsafeCopyFromInternal(inp, src_dtype_, inp.shape());
+      // If the type is a quantized type we need to do a bitcast since the
+      // src_dtype_ is different from external_src_type_.
+      OP_REQUIRES_OK(ctx, in.BitcastFrom(inp, src_dtype_, inp.shape()));
     } else {
       in = inp;
     }
diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc
index 6578f97ef4..ff94c78785 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_ops.cc
@@ -1625,7 +1625,8 @@ class MklQuantizedConv2DSumReluOp
       auto dst_md = summand_mkl_shape.GetMklLayout();
       if (summand_mkl_shape.IsMklTensor()) {
         if (summand_type == DT_QINT8) {
-          summand.UnsafeCopyFromInternal(summand, DT_QUINT8, summand.shape());
+          OP_REQUIRES_OK(context, summand.BitcastFrom(summand, DT_QUINT8,
+                                                      summand.shape()));
           dst_md.data.data_type =
               static_cast<mkldnn_data_type_t>(MklDnnType<Toutput>());
           summand_mkl_shape.SetMklLayout(&dst_md);
-- 
GitLab


From 293546f8641a5e0923c799d356559d6ecac5b8ef Mon Sep 17 00:00:00 2001
From: Cong Liu <congliu@google.com>
Date: Wed, 23 Jan 2019 16:51:33 -0800
Subject: [PATCH 1152/2345] [TF:XLA] SetDynamicBinding before any other ops are
 created.

Otherwise, shape inference of other ops (such as xla::GetTupleElement) will not have the dynamic dimension information.

PiperOrigin-RevId: 230629401
---
 tensorflow/compiler/tf2xla/xla_compiler.cc   | 17 +++++++++--------
 tensorflow/compiler/xla/client/xla_builder.h |  4 ++++
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/xla_compiler.cc b/tensorflow/compiler/tf2xla/xla_compiler.cc
index c5c5b80aa1..373b9ae1b4 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler.cc
@@ -744,14 +744,6 @@ Status XlaCompiler::BuildArguments(
     } else {
       tuple = xla::Parameter(builder, 0, (*input_shapes)[0], "arg_tuple");
     }
-    for (std::vector<int>::size_type i = 0; i < input_to_args->size(); ++i) {
-      auto it = arg_cores.find(i);
-      const int core = it == arg_cores.end() ? -1 : it->second;
-      xla::XlaScopedShardingAssignment assign_sharding(
-          builder, core == -1 ? absl::optional<xla::OpSharding>()
-                              : xla::sharding_builder::AssignDevice(core));
-      arg_handles[i] = xla::GetTupleElement(tuple, i);
-    }
 
     for (int i = 0; i < input_to_args->size(); ++i) {
       const XlaCompiler::Argument& arg = args[input_to_args->at(i)];
@@ -763,6 +755,15 @@ Status XlaCompiler::BuildArguments(
             dim_and_arg_num.first));
       }
     }
+
+    for (std::vector<int>::size_type i = 0; i < input_to_args->size(); ++i) {
+      auto it = arg_cores.find(i);
+      const int core = it == arg_cores.end() ? -1 : it->second;
+      xla::XlaScopedShardingAssignment assign_sharding(
+          builder, core == -1 ? absl::optional<xla::OpSharding>()
+                              : xla::sharding_builder::AssignDevice(core));
+      arg_handles[i] = xla::GetTupleElement(tuple, i);
+    }
   } else {
     for (std::vector<int>::size_type i = 0; i < input_to_args->size(); ++i) {
       auto it = arg_cores.find(i);
diff --git a/tensorflow/compiler/xla/client/xla_builder.h b/tensorflow/compiler/xla/client/xla_builder.h
index 7df1efe6b8..3bd6d42363 100644
--- a/tensorflow/compiler/xla/client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_builder.h
@@ -277,6 +277,10 @@ class XlaBuilder {
   // and its real dynamic size is represented by `dynamic_param_index` in
   // parameter `dynamic_param_num`.
   //
+  // Note that this should be called before the dynamic parameters are used to
+  // create other operations, otherwise created operations won't have the
+  // dynamic dimensions information.
+  //
   // TODO(b/119520625): Remove this API once we have more dynamic shape infra
   // ready.
   Status SetDynamicBinding(int64 dynamic_size_param_num,
-- 
GitLab


From b68520711d7f9d3d153241683bbcba45c5daa6be Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Wed, 23 Jan 2019 16:56:32 -0800
Subject: [PATCH 1153/2345] Code cleanup: remove some usages of v1 metrics.

PiperOrigin-RevId: 230630087
---
 tensorflow/python/training/evaluation_test.py | 26 ++++++++++++-------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/tensorflow/python/training/evaluation_test.py b/tensorflow/python/training/evaluation_test.py
index 3de4ceda75..690c97e3db 100644
--- a/tensorflow/python/training/evaluation_test.py
+++ b/tensorflow/python/training/evaluation_test.py
@@ -26,10 +26,10 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import random_seed
+from tensorflow.python.keras import metrics as metrics_module
 from tensorflow.python.layers import layers
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import metrics
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops.losses import losses
@@ -117,16 +117,18 @@ class EvaluateOnceTest(test.TestCase):
     logits = logistic_classifier(inputs)
     predictions = math_ops.round(logits)
 
-    accuracy, update_op = metrics.accuracy(
-        predictions=predictions, labels=labels)
+    accuracy = metrics_module.Accuracy()
+    update_op = accuracy.update_state(labels, predictions)
 
     checkpoint_path = saver.latest_checkpoint(checkpoint_dir)
 
     final_ops_values = evaluation._evaluate_once(
         checkpoint_path=checkpoint_path,
         eval_ops=update_op,
-        final_ops={'accuracy': accuracy},
-        hooks=[evaluation._StopAfterNEvalsHook(1),])
+        final_ops={'accuracy': (accuracy.result(), update_op)},
+        hooks=[
+            evaluation._StopAfterNEvalsHook(1),
+        ])
     self.assertTrue(final_ops_values['accuracy'] > .99)
 
   def testEvaluateWithFiniteInputs(self):
@@ -148,17 +150,21 @@ class EvaluateOnceTest(test.TestCase):
     logits = logistic_classifier(inputs)
     predictions = math_ops.round(logits)
 
-    accuracy, update_op = metrics.accuracy(
-        predictions=predictions, labels=labels)
+    accuracy = metrics_module.Accuracy()
+    update_op = accuracy.update_state(labels, predictions)
 
     checkpoint_path = saver.latest_checkpoint(checkpoint_dir)
 
     final_ops_values = evaluation._evaluate_once(
         checkpoint_path=checkpoint_path,
         eval_ops=update_op,
-        final_ops={'accuracy': accuracy,
-                   'eval_steps': evaluation._get_or_create_eval_step()},
-        hooks=[evaluation._StopAfterNEvalsHook(None),])
+        final_ops={
+            'accuracy': (accuracy.result(), update_op),
+            'eval_steps': evaluation._get_or_create_eval_step()
+        },
+        hooks=[
+            evaluation._StopAfterNEvalsHook(None),
+        ])
     self.assertTrue(final_ops_values['accuracy'] > .99)
     # Runs evaluation for 4 iterations. First 2 evaluate full batch of 6 inputs
     # each; the 3rd iter evaluates the remaining 4 inputs, and the last one
-- 
GitLab


From 9ae6aeabdef4342d2b8e94ec11900213bc727cea Mon Sep 17 00:00:00 2001
From: James Ring <sjr@google.com>
Date: Wed, 23 Jan 2019 17:18:06 -0800
Subject: [PATCH 1154/2345] remove unused disjunct in BitcastOp

PiperOrigin-RevId: 230633395
---
 tensorflow/core/kernels/bitcast_op.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/core/kernels/bitcast_op.cc b/tensorflow/core/kernels/bitcast_op.cc
index 2397b8741c..02c8808809 100644
--- a/tensorflow/core/kernels/bitcast_op.cc
+++ b/tensorflow/core/kernels/bitcast_op.cc
@@ -45,8 +45,7 @@ class BitcastOp : public OpKernel {
                 in_size_ >= out_size_ ||
                     (input_tensor.dims() > 0 &&
                      input_tensor.dim_size(input_tensor.dims() - 1) ==
-                         out_size_ / in_size_) ||
-                    input_tensor.dim_size(input_tensor.dims()) == -1,
+                         out_size_ / in_size_),
                 errors::InvalidArgument(
                     "Cannot bitcast from ", DataTypeString(input_data_type_),
                     " to ", DataTypeString(output_data_type_), ": shape ",
-- 
GitLab


From 882eba06102688667554c58e7aeb2364ede8bc79 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 23 Jan 2019 17:22:52 -0800
Subject: [PATCH 1155/2345] Also implement NNMemory for Unix targets.

PiperOrigin-RevId: 230634069
---
 tensorflow/lite/delegates/nnapi/nnapi_delegate.cc | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
index d14e449b3e..d5d319432f 100644
--- a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
+++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
@@ -28,8 +28,10 @@ limitations under the License.
 #include "tensorflow/lite/nnapi/nnapi_implementation.h"
 
 #ifdef __ANDROID__
-#include <sys/mman.h>
 #include <sys/system_properties.h>
+#endif
+#if defined __ANDROID__ || defined __unix__
+#include <sys/mman.h>
 #include <unistd.h>
 #endif
 
@@ -69,7 +71,7 @@ struct NNFreeCompilation {
 // Manage NNAPI shared memory handle
 class NNMemory {
  public:
-#ifdef __ANDROID__
+#if defined __ANDROID__ || defined __unix__
   NNMemory(const NnApi* nnapi, const char* name, size_t size) {
     nnapi_ = nnapi;
     byte_size_ = size;
@@ -84,7 +86,7 @@ class NNMemory {
 #endif
 
   ~NNMemory() {
-#ifdef __ANDROID__
+#if defined __ANDROID__ || defined __unix__
     if (data_ptr_) {
       munmap(data_ptr_, byte_size_);
     }
@@ -99,7 +101,7 @@ class NNMemory {
   uint8_t* get_data_ptr() { return data_ptr_; }
 
  private:
-#ifdef __ANDROID__
+#if defined __ANDROID__ || defined __unix__
   const NnApi* nnapi_;
   int fd_ = 0;
   size_t byte_size_ = 0;
-- 
GitLab


From fc3297855cb7af7ebf804b9c9ef8cfcda312c773 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 23 Jan 2019 17:28:07 -0800
Subject: [PATCH 1156/2345] Implemented sparse matrix vector multiplication
 assuming input matrix is in BCSR (block compressed sparse row) format with
 block pattern 1x16.

The codes currently have two assumptions:
1. The number of columns is a multiple of 16 so all blocks are full blocks.
2. The number of columns is less than 4064

PiperOrigin-RevId: 230634750
---
 tensorflow/lite/kernels/internal/BUILD        |   1 +
 .../internal/optimized/neon_tensor_utils.cc   | 112 +++++
 .../internal/optimized/neon_tensor_utils.h    |  18 +
 .../internal/optimized/tensor_utils_impl.h    |  19 +
 .../reference/portable_tensor_utils.cc        |  68 +++
 .../reference/portable_tensor_utils.h         |  27 ++
 .../lite/kernels/internal/tensor_utils.h      |  32 ++
 .../kernels/internal/tensor_utils_test.cc     | 424 ++++++++----------
 8 files changed, 461 insertions(+), 240 deletions(-)

diff --git a/tensorflow/lite/kernels/internal/BUILD b/tensorflow/lite/kernels/internal/BUILD
index c736e5d640..6d3feed268 100644
--- a/tensorflow/lite/kernels/internal/BUILD
+++ b/tensorflow/lite/kernels/internal/BUILD
@@ -405,6 +405,7 @@ cc_library(
         "//tensorflow/lite/c:c_api_internal",
         "//tensorflow/lite/kernels:activation_functor",
         "//tensorflow/lite/kernels:op_macros",
+        "//tensorflow/lite/kernels/internal:types",
     ],
 )
 
diff --git a/tensorflow/lite/kernels/internal/optimized/neon_tensor_utils.cc b/tensorflow/lite/kernels/internal/optimized/neon_tensor_utils.cc
index 730d9b662a..a69a547cb9 100644
--- a/tensorflow/lite/kernels/internal/optimized/neon_tensor_utils.cc
+++ b/tensorflow/lite/kernels/internal/optimized/neon_tensor_utils.cc
@@ -202,6 +202,118 @@ void NeonMatrixBatchVectorMultiplyAccumulate(
   free(aligned_vec_free);
 }
 
+void NeonSparseMatrixBatchVectorMultiplyAccumulate(
+    const float* matrix, const uint8_t* ledger, int m_rows, int m_cols,
+    const float* vector, int n_batch, float* result, int result_stride) {
+  const int kBlockSize = 16;
+  const int kNeonLanesPerBlock = 4;
+  TFLITE_DCHECK_EQ(  // NOLINT
+      m_cols % kBlockSize, 0);
+
+  float* result_in_batch = result;
+  for (int b = 0; b < n_batch; b++) {
+    const float* matrix_ptr = matrix;
+    const uint8_t* ledger_ptr = ledger;
+    for (int r = 0; r < m_rows; r++) {
+      int num_nonzero_blocks = *ledger_ptr++;
+      if (num_nonzero_blocks > 0) {
+        float32x4_t acc_32x4 = vmovq_n_f32(0.0);
+        const float* vector_in_batch = vector + b * m_cols;
+
+        for (int i = 0; i < num_nonzero_blocks; i++) {
+          const int block_start_index = *ledger_ptr++ * kBlockSize;
+          const float* vector_block_in_batch_ptr =
+              vector_in_batch + block_start_index;
+
+          for (int c = 0; c < kNeonLanesPerBlock; c++) {
+            // Load 4 float values from the vector and matrix row.
+            float32x4_t vector_f32x4 = vld1q_f32(vector_block_in_batch_ptr +
+                                                 c * kFloatWeightsPerNeonLane);
+            float32x4_t matrix_f32x4 =
+                vld1q_f32(matrix_ptr + c * kFloatWeightsPerNeonLane);
+            // Multiply the vector and matrix row and add to accumulator.
+            acc_32x4 = vmlaq_f32(acc_32x4, matrix_f32x4, vector_f32x4);
+          }
+          matrix_ptr += kBlockSize;
+        }
+        *result_in_batch +=
+            (vgetq_lane_f32(acc_32x4, 0) + vgetq_lane_f32(acc_32x4, 1) +
+             vgetq_lane_f32(acc_32x4, 2) + vgetq_lane_f32(acc_32x4, 3));
+      }
+      result_in_batch += result_stride;
+    }
+  }
+}
+
+void NeonSparseMatrixBatchVectorMultiplyAccumulate(
+    const int8_t* __restrict__ matrix, const uint8_t* ledger, const int m_rows,
+    const int m_cols, const int8_t* __restrict__ vectors,
+    const float* scaling_factors, int n_batch, float* __restrict__ result,
+    int result_stride) {
+  const int kWeightsPerUint32 = 4;
+  const int kWeightsPerNeonLane = 16;
+  const int kBlockSize = kWeightsPerNeonLane;
+  TFLITE_DCHECK_EQ(  // NOLINT
+      m_cols % kBlockSize, 0);
+  void* aligned_vec_free = nullptr;
+  int8_t* aligned_vec =
+      (int8_t*)aligned_alloc(kWeightsPerUint32, m_cols,  // NOLINT
+                             &aligned_vec_free);
+
+  int batch, row;
+  for (batch = 0; batch < n_batch; ++batch) {
+    const float batch_scaling_factor = scaling_factors[batch];
+    // Copy the vector data to an aligned vector.
+    memcpy(aligned_vec, vectors + batch * m_cols, sizeof(int8) * m_cols);
+
+    const uint8_t* ledger_ptr = ledger;
+    const int8_t* row_ptr = matrix;
+    for (row = 0; row < m_rows; ++row, result += result_stride) {
+      // Initialize the dot product sum for the row to 0.
+      int32x4_t dotprod = vmovq_n_s32(0);
+      int num_nonzero_blocks = *ledger_ptr++;
+      if (num_nonzero_blocks > 0) {
+        // Prefetch the row to cache.
+        __builtin_prefetch(row_ptr, 0 /* prefetch for read */,
+                           3 /* temporal locality */);
+        for (int i = 0; i < num_nonzero_blocks; i++) {
+          const int col_index = *ledger_ptr++ * kBlockSize;
+          // Load 16 8-bit values from the row and vector, each, to operate on.
+          // Here the assumption is that each buffer is 4-byte aligned.
+          // Otherwise, performance may suffer significantly.
+          TFLITE_DCHECK_EQ(  // NOLINT
+              (uintptr_t)(&row_ptr) & (kWeightsPerUint32 - 1), 0);
+          const int8x16_t s1_8x16 =
+              vld1q_s8((const int8_t*)(aligned_vec + col_index));
+          const int8x16_t s2_8x16 = vld1q_s8((const int8_t*)(row_ptr));
+          // Multiply the low bits (i.e. the lower 8 8bit numbers in the
+          // registers).
+          int16x8_t prod_16x8 =
+              vmull_s8(vget_low_s8(s1_8x16), vget_low_s8(s2_8x16));
+          // Multiply the high bits (i.e. the lower 8 8bit numbers in the
+          // registers), and accumulate with the result of the low bits product.
+          // The assumption here is that overflow will not happen as we quantize
+          // our values to be in the range [-127, 127]. As such the sum of the 2
+          // products is always strictly smaller than 15-bits (32767 in absolute
+          // value).
+          prod_16x8 =
+              vmlal_s8(prod_16x8, vget_high_s8(s1_8x16), vget_high_s8(s2_8x16));
+
+          dotprod = vpadalq_s16(dotprod, prod_16x8);
+          row_ptr += kBlockSize;
+        }
+        // Add the 4 intermediate sum values to get the final dot-prod value for
+        // this row.
+        int64x2_t pairwiseAdded = vpaddlq_s32(dotprod);
+        int32 neon_sum =
+            vgetq_lane_s64(pairwiseAdded, 0) + vgetq_lane_s64(pairwiseAdded, 1);
+        *result += neon_sum * batch_scaling_factor;
+      }
+    }  // for row
+  }    // for batch
+  free(aligned_vec_free);
+}
+
 void NeonVectorVectorCwiseProduct(const float* vector1, const float* vector2,
                                   int v_size, float* result) {
   // If v_size is not divisible by kWeightsPerNeonLane, we cannot use the main
diff --git a/tensorflow/lite/kernels/internal/optimized/neon_tensor_utils.h b/tensorflow/lite/kernels/internal/optimized/neon_tensor_utils.h
index 903f4c8013..a86457dba7 100644
--- a/tensorflow/lite/kernels/internal/optimized/neon_tensor_utils.h
+++ b/tensorflow/lite/kernels/internal/optimized/neon_tensor_utils.h
@@ -40,6 +40,24 @@ void MatrixBatchVectorMultiplyAccumulate(
                    vectors, scaling_factors, n_batch, result, result_stride);
 }
 
+void SparseMatrixBatchVectorMultiplyAccumulate(
+    const float* matrix, const uint8_t* ledger, const int m_rows,
+    const int m_cols, const float* vector, int n_batch, float* result,
+    int result_stride) {
+  NeonSparseMatrixBatchVectorMultiplyAccumulate(
+      matrix, ledger, m_rows, m_cols, vector, n_batch, result, result_stride);
+}
+
+void SparseMatrixBatchVectorMultiplyAccumulate(
+    const int8_t* __restrict__ matrix, const uint8_t* ledger, const int m_rows,
+    const int m_cols, const int8_t* __restrict__ vectors,
+    const float* scaling_factors, int n_batch, float* __restrict__ result,
+    int result_stride) {
+  NeonSparseMatrixBatchVectorMultiplyAccumulate(matrix, ledger, m_rows, m_cols,
+                                                vectors, scaling_factors,
+                                                n_batch, result, result_stride);
+}
+
 void VectorVectorCwiseProduct(const float* vector1, const float* vector2,
                               int v_size, float* result) {
   NEON_OR_PORTABLE(VectorVectorCwiseProduct, vector1, vector2, v_size, result);
diff --git a/tensorflow/lite/kernels/internal/optimized/tensor_utils_impl.h b/tensorflow/lite/kernels/internal/optimized/tensor_utils_impl.h
index 8f52ef131d..00b2d7e063 100644
--- a/tensorflow/lite/kernels/internal/optimized/tensor_utils_impl.h
+++ b/tensorflow/lite/kernels/internal/optimized/tensor_utils_impl.h
@@ -54,6 +54,25 @@ void NeonMatrixBatchVectorMultiplyAccumulate(
     const int8_t* __restrict__ vectors, const float* scaling_factors,
     int n_batch, float* __restrict__ result, int result_stride);
 
+void PortableSparseMatrixBatchVectorMultiplyAccumulate(
+    const float* matrix, const uint8_t* ledger, int m_rows, int m_cols,
+    const float* vector, int n_batch, float* result, int result_stride);
+void NeonSparseMatrixBatchVectorMultiplyAccumulate(
+    const float* matrix, const uint8_t* ledger, int m_rows, int m_cols,
+    const float* vector, int n_batch, float* result, int result_stride);
+
+// Matrix multiplication for quantized values using symmetric quantization.
+void PortableSparseMatrixBatchVectorMultiplyAccumulate(
+    const int8_t* __restrict__ matrix, const uint8_t* ledger, const int m_rows,
+    const int m_cols, const int8_t* __restrict__ vectors,
+    const float* scaling_factors, int n_batch, float* __restrict__ result,
+    int result_stride);
+void NeonSparseMatrixBatchVectorMultiplyAccumulate(
+    const int8_t* __restrict__ matrix, const uint8_t* ledger, const int m_rows,
+    const int m_cols, const int8_t* __restrict__ vectors,
+    const float* scaling_factors, int n_batch, float* __restrict__ result,
+    int result_stride);
+
 // Cwise product of two vectors.
 void PortableVectorVectorCwiseProduct(const float* vector1,
                                       const float* vector2, int v_size,
diff --git a/tensorflow/lite/kernels/internal/reference/portable_tensor_utils.cc b/tensorflow/lite/kernels/internal/reference/portable_tensor_utils.cc
index 1acf0caad0..f5c4b78dc1 100644
--- a/tensorflow/lite/kernels/internal/reference/portable_tensor_utils.cc
+++ b/tensorflow/lite/kernels/internal/reference/portable_tensor_utils.cc
@@ -18,6 +18,7 @@ limitations under the License.
 
 #include "tensorflow/lite/c/builtin_op_data.h"
 #include "tensorflow/lite/kernels/activation_functor.h"
+#include "tensorflow/lite/kernels/internal/compatibility.h"
 #include "tensorflow/lite/kernels/internal/round.h"
 #include "tensorflow/lite/kernels/op_macros.h"
 
@@ -109,6 +110,73 @@ void PortableMatrixBatchVectorMultiplyAccumulate(
   }    // for batch
 }
 
+void PortableSparseMatrixBatchVectorMultiplyAccumulate(
+    const float* matrix, const uint8_t* ledger, int m_rows, int m_cols,
+    const float* vector, int n_batch, float* result, int result_stride) {
+  const int kBlockSize = 16;
+  TFLITE_DCHECK_EQ(  // NOLINT
+      m_cols % kBlockSize, 0);
+  float* result_in_batch = result;
+  for (int b = 0; b < n_batch; b++) {
+    const float* matrix_ptr = matrix;
+    const uint8_t* ledger_ptr = ledger;
+    for (int r = 0; r < m_rows; r++) {
+      float dot_prod = 0.0f;
+      int num_nonzero_blocks = *ledger_ptr++;
+      if (num_nonzero_blocks > 0) {
+        const float* vector_in_batch = vector + b * m_cols;
+        for (int i = 0; i < num_nonzero_blocks; i++) {
+          const int block_start_index = *ledger_ptr++ * kBlockSize;
+          const float* vector_block_in_batch_ptr =
+              vector_in_batch + block_start_index;
+          for (int c = 0; c < kBlockSize; c++) {
+            dot_prod += *matrix_ptr++ * *vector_block_in_batch_ptr++;
+          }
+        }
+      }
+      *result_in_batch += dot_prod;
+      result_in_batch += result_stride;
+    }
+  }
+}
+
+void PortableSparseMatrixBatchVectorMultiplyAccumulate(
+    const int8_t* __restrict__ matrix, const uint8_t* ledger, const int m_rows,
+    const int m_cols, const int8_t* __restrict__ vectors,
+    const float* scaling_factors, int n_batch, float* __restrict__ result,
+    int result_stride) {
+  const int kBlockSize = 16;
+  TFLITE_DCHECK_EQ(  // NOLINT
+      m_cols % kBlockSize, 0);
+  int batch, row;
+  for (batch = 0; batch < n_batch; ++batch, vectors += m_cols) {
+    const float batch_scaling_factor = scaling_factors[batch];
+    // Get the address of the first row.
+    const int8_t* row_ptr = matrix;
+    const uint8_t* ledger_ptr = ledger;
+    for (row = 0; row < m_rows; ++row, result += result_stride) {
+      // Initialize the dot product sum for the row to 0.
+      int32_t dotprod = 0;
+#if defined(__GNUC__)
+      // Prefetch the row to cache.
+      __builtin_prefetch(row_ptr, 0 /* prefetch for read */,
+                         3 /* temporal locality */);
+#endif
+      int num_nonzero_blocks = *ledger_ptr++;
+      if (num_nonzero_blocks > 0) {
+        for (int i = 0; i < num_nonzero_blocks; i++) {
+          const int block_start_index = *ledger_ptr++ * kBlockSize;
+          const int8_t* vector_block_ptr = vectors + block_start_index;
+          for (int c = 0; c < kBlockSize; c++) {
+            dotprod += (*row_ptr++) * (*vector_block_ptr++);
+          }  // for block
+        }
+      }
+      *result += (dotprod * batch_scaling_factor);
+    }  // for row
+  }    // for batch
+}
+
 void PortableVectorVectorCwiseProduct(const float* vector1,
                                       const float* vector2, int v_size,
                                       float* result) {
diff --git a/tensorflow/lite/kernels/internal/reference/portable_tensor_utils.h b/tensorflow/lite/kernels/internal/reference/portable_tensor_utils.h
index a06ebc1600..49b59da0bb 100644
--- a/tensorflow/lite/kernels/internal/reference/portable_tensor_utils.h
+++ b/tensorflow/lite/kernels/internal/reference/portable_tensor_utils.h
@@ -48,6 +48,16 @@ void PortableMatrixBatchVectorMultiplyAccumulate(
     const int8_t* __restrict__ vectors, const float* scaling_factors,
     int n_batch, float* __restrict__ result, int result_stride);
 
+void PortableSparseMatrixBatchVectorMultiplyAccumulate(
+    const float* matrix, const uint8_t* ledger, int m_rows, int m_cols,
+    const float* vector, int n_batch, float* result, int result_stride);
+
+void PortableSparseMatrixBatchVectorMultiplyAccumulate(
+    const int8_t* __restrict__ matrix, const uint8_t* ledger, const int m_rows,
+    const int m_cols, const int8_t* __restrict__ vectors,
+    const float* scaling_factors, int n_batch, float* __restrict__ result,
+    int result_stride);
+
 // Cwise product of two vectors.
 void PortableVectorVectorCwiseProduct(const float* vector1,
                                       const float* vector2, int v_size,
@@ -165,6 +175,23 @@ void MatrixBatchVectorMultiplyAccumulate(
                                               result_stride);
 }
 
+void SparseMatrixBatchVectorMultiplyAccumulate(
+    const float* matrix, const uint8_t* ledger, int m_rows, int m_cols,
+    const float* vector, int n_batch, float* result, int result_stride) {
+  PortableSparseMatrixBatchVectorMultiplyAccumulate(
+      matrix, ledger, m_rows, m_cols, vector, n_batch, result, result_stride);
+}
+
+void SparseMatrixBatchVectorMultiplyAccumulate(
+    const int8_t* __restrict__ matrix, const uint8_t* ledger, const int m_rows,
+    const int m_cols, const int8_t* __restrict__ vectors,
+    const float* scaling_factors, int n_batch, float* __restrict__ result,
+    int result_stride) {
+  PortableSparseMatrixBatchVectorMultiplyAccumulate(
+      matrix, ledger, m_rows, m_cols, vectors, scaling_factors, n_batch, result,
+      result_stride);
+}
+
 void VectorVectorCwiseProduct(const float* vector1, const float* vector2,
                               int v_size, float* result) {
   PortableVectorVectorCwiseProduct(vector1, vector2, v_size, result);
diff --git a/tensorflow/lite/kernels/internal/tensor_utils.h b/tensorflow/lite/kernels/internal/tensor_utils.h
index 71ae69522f..4f18f283b6 100644
--- a/tensorflow/lite/kernels/internal/tensor_utils.h
+++ b/tensorflow/lite/kernels/internal/tensor_utils.h
@@ -55,6 +55,21 @@ void MatrixBatchVectorMultiplyAccumulate(const float* matrix, int m_rows,
                                          int n_batch, float* result,
                                          int result_stride);
 
+// Same as the function above, but the matrix is stored in block compressed
+// sparse row format with block pattern 1x16 which consists of two arrays:
+//   1. A matrix array stores non-zero blocks of the matrix in row major.
+//   2. A ledger array stores nrows groups, one group per row. Each group starts
+//   with
+//      an integer representing the number of non-zero blocks for the
+//      corresponding row and follows with column indexes of the first element
+//      of each non-zero block.
+// This function assumes that
+//   1. m_cols is a multiple of 16 so that all blocks are full blocks.
+//   2. m_cols < 254 * 16 so that block index can be represented by uint8.
+void SparseMatrixBatchVectorMultiplyAccumulate(
+    const float* matrix, const uint8_t* ledger, int m_rows, int m_cols,
+    const float* vector, int n_batch, float* result, int result_stride);
+
 // Same as the function above, but for values quantized using symmetric
 // quantization (e.g. by calling SymmetricQuantizeFloats).
 // The passed scaling factors is a buffer of the quantization scaling factors
@@ -67,6 +82,23 @@ void MatrixBatchVectorMultiplyAccumulate(
     const int8_t* __restrict__ vectors, const float* scaling_factors,
     int n_batch, float* __restrict__ result, int result_stride);
 
+// Same as the function above, but the matrix is stored in block compressed
+// sparse row format with block pattern 1x16 which consists of two arrays:
+//   1. A matrix array stores non-zero blocks of the matrix in row major.
+//   2. A ledger array stores nrows groups, one group per row. Each group starts
+//   with
+//      an integer representing the number of non-zero blocks for the
+//      corresponding row followed by column index of the first element of
+//      each non-zero block.
+// This function assumes that
+//   1. m_cols is a multiple of 16 so that all blocks are full blocks.
+//   2. m_cols < 254 * 16 so that block index can be represented by uint8.
+void SparseMatrixBatchVectorMultiplyAccumulate(
+    const int8_t* __restrict__ matrix, const uint8_t* ledger, const int m_rows,
+    const int m_cols, const int8_t* __restrict__ vectors,
+    const float* scaling_factors, int n_batch, float* __restrict__ result,
+    int result_stride);
+
 // Cwise product of two vectors.
 void VectorVectorCwiseProduct(const float* vector1, const float* vector2,
                               int v_size, float* result);
diff --git a/tensorflow/lite/kernels/internal/tensor_utils_test.cc b/tensorflow/lite/kernels/internal/tensor_utils_test.cc
index 29866d0664..3ba4af7c46 100644
--- a/tensorflow/lite/kernels/internal/tensor_utils_test.cc
+++ b/tensorflow/lite/kernels/internal/tensor_utils_test.cc
@@ -149,6 +149,7 @@ TEST(uKernels, MatrixBatchVectorMultiplyAccumulateSymmetricQuantizedTest) {
   // 16-block SIMD code, the 8-block postamble, and the leftover postamble.
   const int a_rows = 4, a_cols = 29;
   const int kWeightsPerUint32 = 4;
+  /* clang-format off */
   const float a_float_data[] = {
       /* 1st row */
       1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9, 10.1, 11.11, 12.12, 13.13,
@@ -174,126 +175,18 @@ TEST(uKernels, MatrixBatchVectorMultiplyAccumulateSymmetricQuantizedTest) {
   SymmetricQuantizeFloats(a_float_data, a_rows * a_cols, a_int8_data, &a_min,
                           &a_max, &scaling_factor_a);
   const int8_t expected_a_int8_data[] = {
-      /* 1st row */
-      5,
-      10,
-      15,
-      20,
-      25,
-      30,
-      35,
-      40,
-      44,
-      45,
-      50,
-      54,
-      59,
-      64,
-      68,
-      73,
-      77,
-      82,
-      86,
-      91,
-      95,
-      100,
-      104,
-      109,
-      113,
-      118,
-      122,
-      127,
-      0,
-      /* 2nd row */
-      -5,
-      -10,
-      -15,
-      -20,
-      -25,
-      -30,
-      -35,
-      -40,
-      -44,
-      -45,
-      -50,
-      -54,
-      -59,
-      -64,
-      -68,
-      -73,
-      -77,
-      -82,
-      -86,
-      -91,
-      -95,
-      -100,
-      -104,
-      -109,
-      -113,
-      -118,
-      -122,
-      -127,
-      0,
-      /* 3rd row */
-      5,
-      -10,
-      15,
-      -20,
-      25,
-      -30,
-      35,
-      -40,
-      44,
-      -45,
-      50,
-      -54,
-      59,
-      -64,
-      68,
-      -73,
-      77,
-      -82,
-      86,
-      -91,
-      95,
-      -100,
-      104,
-      -109,
-      113,
-      -118,
-      122,
-      -127,
-      0,
-      /* 4th row */
-      -5,
-      10,
-      -15,
-      20,
-      -25,
-      30,
-      -35,
-      40,
-      -44,
-      45,
-      -50,
-      54,
-      -59,
-      64,
-      -68,
-      73,
-      -77,
-      82,
-      -86,
-      91,
-      -95,
-      100,
-      -104,
-      109,
-      -113,
-      118,
-      -122,
-      127,
-      0,
+    /* 1st row */
+    5, 10, 15, 20, 25, 30, 35, 40, 44, 45, 50, 54, 59, 64, 68, 73, 77, 82, 86,
+    91, 95, 100, 104, 109, 113, 118, 122, 127, 0,
+    /* 2nd row */
+    -5, -10, -15, -20, -25, -30, -35, -40, -44, -45, -50, -54, -59, -64, -68,
+    -73, -77, -82, -86, -91, -95, -100, -104, -109, -113, -118, -122, -127, 0,
+    /* 3rd row */
+    5, -10, 15, -20, 25, -30, 35, -40, 44, -45, 50, -54, 59, -64, 68, -73, 77,
+    -82, 86, -91, 95, -100, 104, -109, 113, -118, 122, -127, 0,
+    /* 4th row */
+    -5, 10, -15, 20, -25, 30, -35, 40, -44, 45, -50, 54, -59, 64, -68, 73, -77,
+    82, -86, 91, -95, 100, -104, 109, -113, 118, -122, 127, 0,
   };
   for (int i = 0; i < a_rows * a_cols; ++i) {
     EXPECT_EQ(expected_a_int8_data[i], a_int8_data[i]);
@@ -301,66 +194,14 @@ TEST(uKernels, MatrixBatchVectorMultiplyAccumulateSymmetricQuantizedTest) {
 
   const int b_rows = 29, b_cols = 1, batches = 2;
   const float b_float_data[] = {
-      /* batch 1 */
-      1.0,
-      -1.0,
-      1.0,
-      -1.0,
-      1.0,
-      -1.0,
-      1.0,
-      -1.0,
-      1.0,
-      -1.0,
-      1.0,
-      -1.0,
-      1.0,
-      -1.0,
-      1.0,
-      -1.0,
-      1.0,
-      -1.0,
-      1.0,
-      -1.0,
-      1.0,
-      -1.0,
-      1.0,
-      -1.0,
-      1.0,
-      -1.0,
-      1.0,
-      -1.0,
-      1.0,
-      /* batch 2 */
-      2.5,
-      -2.1,
-      3.0,
-      -1.3,
-      1.3,
-      -1.1,
-      2.0,
-      -1.7,
-      1.9,
-      -1.5,
-      0.5,
-      -0.7,
-      0.8,
-      -0.3,
-      2.8,
-      -2.8,
-      1.1,
-      -2.3,
-      1.9,
-      -1.9,
-      2.1,
-      -0.5,
-      2.4,
-      -0.1,
-      1.0,
-      -2.5,
-      0.7,
-      -1.9,
-      0.2,
+    /* batch 1 */
+    1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0,
+    1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0,
+    1.0,
+    /* batch 2 */
+    2.5, -2.1, 3.0, -1.3, 1.3, -1.1, 2.0, -1.7, 1.9, -1.5, 0.5, -0.7, 0.8, -0.3,
+    2.8, -2.8, 1.1, -2.3, 1.9, -1.9, 2.1, -0.5, 2.4, -0.1, 1.0, -2.5, 0.7, -1.9,
+    0.2,
   };
 
   // Quantized values of B:
@@ -374,67 +215,15 @@ TEST(uKernels, MatrixBatchVectorMultiplyAccumulateSymmetricQuantizedTest) {
                           &scaling_factor_b[1]);
 
   const int8_t expected_b_int8_data[] = {
-      /* batch 1 */
-      127,
-      -127,
-      127,
-      -127,
-      127,
-      -127,
-      127,
-      -127,
-      127,
-      -127,
-      127,
-      -127,
-      127,
-      -127,
-      127,
-      -127,
-      127,
-      -127,
-      127,
-      -127,
-      127,
-      -127,
-      127,
-      -127,
-      127,
-      -127,
-      127,
-      -127,
-      127,
-      /* batch 2 */
-      106,
-      -89,
-      127,
-      -55,
-      55,
-      -47,
-      85,
-      -72,
-      80,
-      -64,
-      21,
-      -30,
-      34,
-      -13,
-      119,
-      -119,
-      47,
-      -97,
-      80,
-      -80,
-      89,
-      -21,
-      102,
-      -4,
-      42,
-      -106,
-      30,
-      -80,
-      8,
+    /* batch 1 */
+    127, -127, 127, -127, 127, -127, 127, -127, 127, -127, 127, -127, 127, -127,
+    127, -127, 127, -127, 127, -127, 127, -127, 127, -127, 127, -127, 127, -127,
+    127,
+    /* batch 2 */
+    106, -89, 127, -55, 55, -47, 85, -72, 80, -64, 21, -30, 34, -13, 119, -119,
+    47, -97, 80, -80, 89, -21, 102, -4, 42, -106, 30, -80, 8,
   };
+  /* clang-format on */
   for (int i = 0; i < b_rows * b_cols * batches; ++i) {
     EXPECT_EQ(expected_b_int8_data[i], b_int8_data[i]);
   }
@@ -468,6 +257,161 @@ TEST(uKernels, MatrixBatchVectorMultiplyAccumulateSymmetricQuantizedTest) {
 }
 #endif  // __ANDROID__
 
+TEST(uKernels, SparseMatrixBatchVectorMultiplyAccumulateTest) {
+  const int kRow = 4;
+  const int kCol = 48;
+  const int kBatch = 2;
+  /* clang-format off */
+  float matrix[kRow * kCol] = {
+      /* 1st row */
+      1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9, 10.1, 11.11, 12.12, 13.13,
+      14.14, 15.15, 16.16, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
+      0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 33.33, 34.34, 35.35, 36.36, 37.37, 38.38,
+      39.39, 40.40, 41.41, 42.42, 43.43, 44.44, 0, 0, 0, 0,
+      /* 2nd row */
+      0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
+      0.0, -17.17, -18.18, -19.19, -20.2, -21.21, -22.22, -23.23, -24.24,
+      -25.25, -26.26, -27.27, -28.28, 0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
+      0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0,
+      /* 3rd row */
+      0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
+      0.0, 17.17, -18.18, 19.19, -20.2, 21.21, -22.22, 23.23, -24.24, 25.25,
+      -26.26, 27.27, -28.28, 0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
+      0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0,
+      /* 4th row */
+      -1.1, 2.2, -3.3, 4.4, -5.5, 6.6, -7.7, 8.8, -9.9, 10.1, -11.11, 12.12,
+      -13.13, 14.14, -15.15, 16.16, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
+      0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -33.33, 34.34, -35.35, 36.36, -37.37,
+      38.38, -39.39, 40.40, -41.41, 42.42, -43.43, 44.44, 0, 0, 0, 0};
+
+  // BCSR format of the above matrix.
+  float matrix_values[] = {
+      /* 1st row */
+      1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9, 10.1, 11.11, 12.12, 13.13,
+      14.14, 15.15, 16.16, 33.33, 34.34, 35.35, 36.36, 37.37, 38.38, 39.39,
+      40.40, 41.41, 42.42, 43.43, 44.44, 0, 0, 0, 0,
+      /* 2nd row */
+      -17.17, -18.18, -19.19, -20.2, -21.21, -22.22, -23.23, -24.24, -25.25,
+      -26.26, -27.27, -28.28, 0, 0.0, 0.0, 0.0,
+      /* 3rd row */
+      17.17, -18.18, 19.19, -20.2, 21.21, -22.22, 23.23, -24.24, 25.25, -26.26,
+      27.27, -28.28, 0, 0.0, 0.0, 0.0,
+      /* 4th row */
+      -1.1, 2.2, -3.3, 4.4, -5.5, 6.6, -7.7, 8.8, -9.9, 10.1, -11.11, 12.12,
+      -13.13, 14.14, -15.15, 16.16, -33.33, 34.34, -35.35, 36.36, -37.37, 38.38,
+      -39.39, 40.40, -41.41, 42.42, -43.43, 44.44, 0, 0, 0, 0};
+  uint8_t ledger[] = {
+      2, 0,  2,  // 1st row
+      1, 1,      // 2nd row
+      1, 1,      // 3rd row
+      2, 0,  2   // 4th row
+  };
+
+  float vector[kBatch * kCol] = {
+    /* 1st batch */
+    1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0,
+    1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0,
+    1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0,
+    1.0, -1.0, 1.0, -1.0, 1.0, -1.0,
+    /* 2nd batch */
+    2.5, 0.0, -2.1, 0.0, 3.0, 0.0, -1.3, 0.0, 1.3, 0.0, -1.1, 0.0, 2.0, 0.0,
+    -1.7, 0.0, 1.9, 0.0, -1.5, 0.0, 0.5, 0.0, -0.7, 0.0, 0.8, 0.0, -0.3, 0.0,
+    2.8, 0.0, -2.8, 0.0, 1.1, -2.3, 1.9, -1.9, 2.1, -0.5, 2.4, -0.1, 1.0, -2.5,
+    0.7, -1.9, 0.2, 0.0, 0.1, 0.2,
+  };
+  /* clang-format on */
+
+  std::vector<float> dense_output(kRow * kBatch, 0.0);
+  MatrixBatchVectorMultiplyAccumulate(matrix, kRow, kCol, vector, kBatch,
+                                      dense_output.data(), /*result_stride=*/1);
+
+  std::vector<float> sparse_output(kRow * kBatch, 0.0);
+  SparseMatrixBatchVectorMultiplyAccumulate(
+      matrix_values, ledger, kRow, kCol, vector, kBatch, sparse_output.data(),
+      /*result_stride=*/1);
+
+  EXPECT_THAT(sparse_output,
+              ElementsAreArray(ArrayFloatNear(dense_output, 1e-4)));
+}
+
+#ifdef __ANDROID__
+TEST(uKernels,
+     SparseMatrixBatchVectorMultiplyAccumulateSymmetricQuantizedTest) {
+  const int kRow = 4;
+  const int kCol = 48;
+  const int kBatch = 2;
+  /* clang-format off */
+  const int8_t quantized_matrix[] = {
+      /* 1st row */
+      3, 6, 9, 13, 16, 19, 22, 25, 28, 29, 32, 35, 38, 40, 43, 46, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 98, 101, 104, 107, 110, 113, 115,
+      118, 121, 124, 127, 0, 0, 0, 0,
+      /* 2nd row */
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -49, -52, -55, -58, -61,
+      -64, -66, -69, -72, -75, -78, -81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0,
+      /* 3rd row */
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, -52, 55, -58, 61, -64,
+      66, -69, 72, -75, 78, -81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0,
+      /* 4th row */
+      -3, 6, -9, 13, -16, 19, -22, 25, -28, 29, -32, 35, -38, 40, -43, 46, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -95, 98, -101, 104, -107, 110,
+      -113, 115, -118, 121, -124, 127, 0, 0, 0, 0,
+  };
+  const int8_t quantized_matrix_values[] = {
+      /* 1st row */
+      3, 6, 9, 13, 16, 19, 22, 25, 28, 29, 32, 35, 38, 40, 43, 46, 95, 98, 101,
+      104, 107, 110, 113, 115, 118, 121, 124, 127, 0, 0, 0, 0,
+      /* 2nd row */
+      -49, -52, -55, -58, -61, -64, -66, -69, -72, -75, -78, -81, 0, 0, 0, 0,
+      /* 3rd row */
+      49, -52, 55, -58, 61, -64, 66, -69, 72, -75, 78, -81, 0, 0, 0, 0,
+      /* 4th row */
+      -3, 6, -9, 13, -16, 19, -22, 25, -28, 29, -32, 35, -38, 40, -43, 46, -95,
+      98, -101, 104, -107, 110, -113, 115, -118, 121, -124, 127, 0, 0, 0, 0,
+  };
+  uint8_t ledger[] = {
+      2, 0,  2,  // 1st row
+      1, 1,      // 2nd row
+      1, 1,      // 3rd row
+      2, 0,  2   // 4th row
+  };
+
+  float matrix_scaling_factor = 0.349921;
+
+  const int8_t quantized_vector[] = {
+      /* 1st batch */
+      127, -127, 127, -127, 127, -127, 127, -127, 127, -127, 127, -127, 127,
+      -127, 127, -127, 127, -127, 127, -127, 127, -127, 127, -127, 127, -127,
+      127, -127, 127, -127, 127, -127, 127, -127, 127, -127, 127, -127, 127,
+      -127, 127, -127, 127, -127, 127, -127, 127, -127,
+      /* 2nd batch */
+      106, 0, -89, 0, 127, 0, -55, 0, 55, 0, -47, 0, 85, 0, -72, 0, 80, 0,
+      -64, 0, 21, 0, -30, 0, 34, 0, -13, 0, 119, 0, -119, 0, 47, -97, 80, -80,
+      89, -21, 102, -4, 42, -106, 30, -80, 8, 1, 2, 3,
+  };
+  float vector_scaling_factor[2] = {0.00787402, 0.023622};
+
+  /* clang-format on */
+  float result_scaling_factor[2] = {
+      matrix_scaling_factor * vector_scaling_factor[0],
+      matrix_scaling_factor * vector_scaling_factor[1],
+  };
+  std::vector<float> dense_output(kRow * kBatch, 0.0);
+  MatrixBatchVectorMultiplyAccumulate(quantized_matrix, kRow, kCol,
+                                      quantized_vector, result_scaling_factor,
+                                      kBatch, dense_output.data(),
+                                      /*result_stride=*/1);
+  std::vector<float> sparse_output(kRow * kBatch, 0.0);
+  SparseMatrixBatchVectorMultiplyAccumulate(
+      quantized_matrix_values, ledger, kRow, kCol, quantized_vector,
+      result_scaling_factor, kBatch, sparse_output.data(),
+      /*result_stride=*/1);
+  EXPECT_THAT(sparse_output, ElementsAreArray(ArrayFloatNear(dense_output)));
+}
+#endif  // __ANDROID__
+
 TEST(uKernels, VectorVectorCwiseProductTest) {
   constexpr int kVectorSize = 10;
   static float input1[kVectorSize] = {0.0,  -0.5, 1.0,  -1.5, 2.0,
-- 
GitLab


From ccb4c9e12d21eb72628f512dcc22e0ac3ec241ed Mon Sep 17 00:00:00 2001
From: Clayne Robison <clayne.b.robison@intel.com>
Date: Wed, 23 Jan 2019 18:35:18 -0700
Subject: [PATCH 1157/2345] refining message per maintainer feedback.

---
 tensorflow/core/platform/cpu_feature_guard.cc | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/tensorflow/core/platform/cpu_feature_guard.cc b/tensorflow/core/platform/cpu_feature_guard.cc
index 3f0d1cdb51..38fc453008 100644
--- a/tensorflow/core/platform/cpu_feature_guard.cc
+++ b/tensorflow/core/platform/cpu_feature_guard.cc
@@ -138,19 +138,15 @@ void InfoAboutUnusedCPUFeatures() {
 #endif  // __FMA__
 #endif  // else of if defined(_MSC_VER) && !defined(__clang__)
     if (!missing_instructions.empty()) {
-      LOG(INFO) << "Your CPU supports instructions that this TensorFlow "
 #ifndef INTEL_MKL
+      LOG(INFO) << "Your CPU supports instructions that this TensorFlow "
                 << "binary was not compiled to use:" << missing_instructions;
 #else
-                << "binary was not compiled to use:" << missing_instructions
-                << ". When TensorFlow is compiled with support for these "
-                << "instructions, performance can improve. However, because "
-                << "this version of TensorFlow has been compiled with support "
-                << "for Intel(R) MKL-DNN, which supports just-in-time "
-                << "compilation, many performance-critical codepaths will use "
-                << "these instructions, even though TensorFlow was compiled "
-                << "without explicit support them. See "
-                << "https://github.com/intel/mkl-dnn/issues/3";
+      LOG(INFO) << "This TensorFlow binary is optimized with Intel(R) MKL-DNN "
+                << "to use the following CPU instructions in performance "
+                << "critical operations: " << missing_instructions << std::endl
+                << "To enable them in non-MKL-DNN operations, rebuild "
+                << "TensorFlow with the appropriate compiler flags.";
 #endif
     }
   });
-- 
GitLab


From 92d08bf391934ff990280dbbdd6f873c248c981f Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Wed, 23 Jan 2019 17:30:36 -0800
Subject: [PATCH 1158/2345] [XLA] Correctly state Euler's reflection formula
 for the gamma function.

Comment-only change, the code appears correct.

See

  https://en.wikipedia.org/wiki/Gamma_function#Properties
  https://en.wikipedia.org/wiki/Digamma_function#Reflection_formula

PiperOrigin-RevId: 230635195
---
 tensorflow/compiler/xla/client/lib/math.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/client/lib/math.cc b/tensorflow/compiler/xla/client/lib/math.cc
index 3d0e3a2b93..6f72d09235 100644
--- a/tensorflow/compiler/xla/client/lib/math.cc
+++ b/tensorflow/compiler/xla/client/lib/math.cc
@@ -184,8 +184,8 @@ XlaOp Lgamma(XlaOp input) {
 
   XlaOp base_lanczos_coeff = ScalarLike(input, kBaseLanczosCoeff);
 
-  // If the input is less than 0.5 use Gauss's reflection formula:
-  // gamma(x) = pi / sin(pi * x) * gamma(1 - x)
+  // If the input is less than 0.5 use Euler's reflection formula:
+  // gamma(x) = pi / (sin(pi * x) * gamma(1 - x))
   XlaOp need_to_reflect = Lt(Real(input), one_half);
   XlaOp z = Select(need_to_reflect, -input, input - one);
 
@@ -237,7 +237,7 @@ XlaOp Digamma(XlaOp input) {
 
   XlaOp base_lanczos_coeff = ScalarLike(input, kBaseLanczosCoeff);
 
-  // If the input is less than 0.5 use Gauss's reflection formula:
+  // If the input is less than 0.5 use Euler's reflection formula:
   // digamma(x) = digamma(1 - x) - pi * cot(pi * x)
   XlaOp need_to_reflect = Lt(Real(input), one_half);
   XlaOp z = Select(need_to_reflect, -input, input - one);
-- 
GitLab


From 529a00a16af9a4ca388a150c2daa90ff8c4a6a18 Mon Sep 17 00:00:00 2001
From: Revan Sopher <rsopher@google.com>
Date: Wed, 23 Jan 2019 17:59:56 -0800
Subject: [PATCH 1159/2345] Automated rollback of commit
 7df1985f651268cd02ba9011b4295412e7d635cf

PiperOrigin-RevId: 230638752
---
 tensorflow/core/kernels/fill_functor.cu.cc    |   6 -
 tensorflow/core/kernels/list_kernels.cc       | 194 +++++++++---
 tensorflow/core/kernels/list_kernels.cu.cc    |  65 ++--
 tensorflow/core/kernels/list_kernels.h        | 289 ++++++------------
 .../core/ops/compat/ops_history.v1.pbtxt      |  16 -
 tensorflow/core/ops/list_ops.cc               |   4 -
 tensorflow/core/ops/ops.pbtxt                 |  16 -
 .../python/kernel_tests/list_ops_test.py      | 178 +----------
 .../kernel_tests/tensor_array_ops_test.py     |  47 +--
 tensorflow/python/ops/list_ops.py             |  74 +----
 10 files changed, 299 insertions(+), 590 deletions(-)

diff --git a/tensorflow/core/kernels/fill_functor.cu.cc b/tensorflow/core/kernels/fill_functor.cu.cc
index 946d5c9b02..050c95cf40 100644
--- a/tensorflow/core/kernels/fill_functor.cu.cc
+++ b/tensorflow/core/kernels/fill_functor.cu.cc
@@ -88,12 +88,6 @@ struct SetZeroFunctor<GPUDevice, T> {
   }
 };
 
-template <>
-void SetZeroFunctor<GPUDevice, Variant>::operator()(
-    const GPUDevice& d, typename TTypes<Variant>::Flat out) {
-  // TODO(b/123028789): Implement this.
-}
-
 #define DEFINE_SETZERO_GPU(T) template struct SetZeroFunctor<GPUDevice, T>;
 TF_CALL_NUMBER_TYPES(DEFINE_SETZERO_GPU);
 TF_CALL_bool(DEFINE_SETZERO_GPU);
diff --git a/tensorflow/core/kernels/list_kernels.cc b/tensorflow/core/kernels/list_kernels.cc
index 7c0b70d651..5859c20d89 100644
--- a/tensorflow/core/kernels/list_kernels.cc
+++ b/tensorflow/core/kernels/list_kernels.cc
@@ -173,17 +173,6 @@ Status TensorShapeFromTensor(const Tensor& t, PartialTensorShape* out) {
       DataTypeString(t.dtype()));
 }
 
-Status GetElementShapeFromInput(OpKernelContext* c,
-                                const TensorList& tensor_list, int index,
-                                PartialTensorShape* element_shape) {
-  TF_RETURN_IF_ERROR(TensorShapeFromTensor(c->input(index), element_shape));
-  // Check that `element_shape` and `tensor_list.element_shape` are
-  // compatible and store the merged shape in `element_shape`.
-  PartialTensorShape tmp = *element_shape;
-  TF_RETURN_IF_ERROR(tmp.MergeWith(tensor_list.element_shape, element_shape));
-  return Status::OK();
-}
-
 class EmptyTensorList : public OpKernel {
  public:
   explicit EmptyTensorList(OpKernelConstruction* ctx) : OpKernel(ctx) {
@@ -369,6 +358,69 @@ class TensorListElementShape : public OpKernel {
 REGISTER_KERNEL_BUILDER(Name("TensorListElementShape").Device(DEVICE_CPU),
                         TensorListElementShape);
 
+#if GOOGLE_CUDA
+
+REGISTER_KERNEL_BUILDER(Name("TensorListElementShape")
+                            .Device(DEVICE_GPU)
+                            .HostMemory("element_shape"),
+                        TensorListElementShape);
+
+#endif  // GOOGLE_CUDA
+
+class TensorListPopBack : public OpKernel {
+ public:
+  explicit TensorListPopBack(OpKernelConstruction* c) : OpKernel(c) {
+    OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_));
+  }
+
+  ~TensorListPopBack() override {}
+
+  void Compute(OpKernelContext* c) override {
+    const TensorList* l = c->input(0).scalar<Variant>()().get<TensorList>();
+    OP_REQUIRES(c, l != nullptr,
+                errors::InvalidArgument(
+                    "Input handle is not a list. Saw: '",
+                    c->input(0).scalar<Variant>()().DebugString(), "'"));
+    OP_REQUIRES(c, element_dtype_ == l->element_dtype,
+                errors::InvalidArgument("Invalid data types; op elements ",
+                                        DataTypeString(element_dtype_),
+                                        " but list elements ",
+                                        DataTypeString(l->element_dtype)));
+
+    OP_REQUIRES(c, !l->tensors.empty(),
+                errors::InvalidArgument("Trying to pop from an empty list."));
+
+    c->set_output(1, l->tensors.back());
+    AllocatorAttributes attr;
+    attr.set_on_host(true);
+    std::unique_ptr<Tensor> maybe_result = c->forward_input(
+        0, 0, DT_VARIANT, TensorShape{}, c->input_memory_type(0), attr);
+    if (maybe_result != nullptr) {
+      maybe_result->scalar<Variant>()().get<TensorList>()->tensors.pop_back();
+    } else {
+      TensorList output;
+      output = *l;
+      output.tensors.pop_back();
+      Tensor* result;
+      OP_REQUIRES_OK(c, c->allocate_output(0, TensorShape{}, &result, attr));
+      result->scalar<Variant>()() = std::move(output);
+    }
+  }
+
+ private:
+  DataType element_dtype_;
+};
+
+REGISTER_KERNEL_BUILDER(Name("TensorListPopBack").Device(DEVICE_CPU),
+                        TensorListPopBack);
+
+#if GOOGLE_CUDA
+
+REGISTER_KERNEL_BUILDER(Name("TensorListPopBack").Device(DEVICE_GPU),
+                        TensorListPopBack);
+
+#endif  // GOOGLE_CUDA
+
 class TensorListReserve : public OpKernel {
  public:
   explicit TensorListReserve(OpKernelConstruction* c) : OpKernel(c) {
@@ -406,6 +458,61 @@ REGISTER_KERNEL_BUILDER(Name("TensorListReserve")
                         TensorListReserve);
 
 #endif  // GOOGLE_CUDA
+
+class TensorListGetItem : public OpKernel {
+ public:
+  explicit TensorListGetItem(OpKernelConstruction* c) : OpKernel(c) {
+    OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_));
+  }
+
+  void Compute(OpKernelContext* c) override {
+    OP_REQUIRES(
+        c, c->input(0).shape().num_elements() == 1,
+        errors::InvalidArgument("List tensors are supposed to be scalars."));
+    const TensorList* l = c->input(0).scalar<Variant>()().get<TensorList>();
+    OP_REQUIRES(c, l != nullptr,
+                errors::InvalidArgument(
+                    "Input handle is not a list. Saw: '",
+                    c->input(0).scalar<Variant>()().DebugString(), "'"));
+    OP_REQUIRES(c, element_dtype_ == l->element_dtype,
+                errors::InvalidArgument("Invalid data types; op elements ",
+                                        DataTypeString(element_dtype_),
+                                        " but list elements ",
+                                        DataTypeString(l->element_dtype)));
+    int32 index = c->input(1).scalar<int32>()();
+    OP_REQUIRES(c, index < l->tensors.size(),
+                errors::InvalidArgument("Trying to access element ", index,
+                                        " in a list with ", l->tensors.size(),
+                                        " elements."));
+    c->set_output(0, l->tensors[index]);
+  }
+
+ private:
+  DataType element_dtype_;
+};
+
+REGISTER_KERNEL_BUILDER(Name("TensorListGetItem").Device(DEVICE_CPU),
+                        TensorListGetItem);
+
+#if GOOGLE_CUDA
+
+#define REGISTER_TENSOR_LIST_GET_ITEM_GPU(T)                      \
+  REGISTER_KERNEL_BUILDER(Name("TensorListGetItem")               \
+                              .TypeConstraint<T>("element_dtype") \
+                              .Device(DEVICE_GPU)                 \
+                              .HostMemory("index"),               \
+                          TensorListGetItem);
+
+TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_GET_ITEM_GPU);
+TF_CALL_complex64(REGISTER_TENSOR_LIST_GET_ITEM_GPU);
+TF_CALL_complex128(REGISTER_TENSOR_LIST_GET_ITEM_GPU);
+TF_CALL_int64(REGISTER_TENSOR_LIST_GET_ITEM_GPU);
+REGISTER_TENSOR_LIST_GET_ITEM_GPU(bfloat16)
+REGISTER_TENSOR_LIST_GET_ITEM_GPU(bool)
+#undef REGISTER_TENSOR_LIST_GET_ITEM_GPU
+
+#endif  // GOOGLE_CUDA
+
 class TensorListResize : public OpKernel {
  public:
   explicit TensorListResize(OpKernelConstruction* c) : OpKernel(c) {}
@@ -629,7 +736,23 @@ REGISTER_KERNEL_BUILDER(Name("TensorListConcatLists").Device(DEVICE_GPU),
 
 #endif  // GOOGLE_CUDA
 
-#define REGISTER_TENSOR_LIST_OPS_CPU(T)                           \
+#define REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(T)               \
+  REGISTER_KERNEL_BUILDER(Name("TensorListPushBackBatch")         \
+                              .TypeConstraint<T>("element_dtype") \
+                              .Device(DEVICE_CPU),                \
+                          TensorListPushBackBatch<CPUDevice, T>)
+
+TF_CALL_ALL_TYPES(REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU);
+REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(quint8);
+REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(qint8);
+REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(quint16);
+REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(qint16);
+REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(qint32);
+REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(bfloat16);
+
+#undef REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU
+
+#define REGISTER_TENSOR_LIST_STACK_CPU(T)                         \
   REGISTER_KERNEL_BUILDER(Name("TensorListStack")                 \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_CPU),                \
@@ -641,15 +764,19 @@ REGISTER_KERNEL_BUILDER(Name("TensorListConcatLists").Device(DEVICE_GPU),
   REGISTER_KERNEL_BUILDER(Name("TensorListConcat")                \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_CPU),                \
-                          TensorListConcat<CPUDevice, T>)         \
-  REGISTER_KERNEL_BUILDER(Name("TensorListGetItem")               \
-                              .TypeConstraint<T>("element_dtype") \
-                              .Device(DEVICE_CPU),                \
-                          TensorListGetItem<CPUDevice, T>)        \
-  REGISTER_KERNEL_BUILDER(Name("TensorListPopBack")               \
-                              .TypeConstraint<T>("element_dtype") \
-                              .Device(DEVICE_CPU),                \
-                          TensorListPopBack<CPUDevice, T>)        \
+                          TensorListConcat<CPUDevice, T>)
+
+TF_CALL_POD_STRING_TYPES(REGISTER_TENSOR_LIST_STACK_CPU);
+REGISTER_TENSOR_LIST_STACK_CPU(quint8);
+REGISTER_TENSOR_LIST_STACK_CPU(qint8);
+REGISTER_TENSOR_LIST_STACK_CPU(quint16);
+REGISTER_TENSOR_LIST_STACK_CPU(qint16);
+REGISTER_TENSOR_LIST_STACK_CPU(qint32);
+REGISTER_TENSOR_LIST_STACK_CPU(bfloat16);
+
+#undef REGISTER_TENSOR_LIST_STACK_CPU
+
+#define REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(T)                   \
   REGISTER_KERNEL_BUILDER(Name("TensorListFromTensor")            \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_CPU),                \
@@ -661,24 +788,17 @@ REGISTER_KERNEL_BUILDER(Name("TensorListConcatLists").Device(DEVICE_GPU),
   REGISTER_KERNEL_BUILDER(Name("TensorListSplit")                 \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_CPU),                \
-                          TensorListSplit<CPUDevice, T>)          \
-  REGISTER_KERNEL_BUILDER(Name("TensorListPushBackBatch")         \
-                              .TypeConstraint<T>("element_dtype") \
-                              .Device(DEVICE_CPU),                \
-                          TensorListPushBackBatch<CPUDevice, T>)
-
-TF_CALL_POD_STRING_TYPES(REGISTER_TENSOR_LIST_OPS_CPU);
-REGISTER_TENSOR_LIST_OPS_CPU(quint8);
-REGISTER_TENSOR_LIST_OPS_CPU(qint8);
-REGISTER_TENSOR_LIST_OPS_CPU(quint16);
-REGISTER_TENSOR_LIST_OPS_CPU(qint16);
-REGISTER_TENSOR_LIST_OPS_CPU(qint32);
-REGISTER_TENSOR_LIST_OPS_CPU(bfloat16);
-REGISTER_TENSOR_LIST_OPS_CPU(Variant);
+                          TensorListSplit<CPUDevice, T>)
 
-#undef REGISTER_TENSOR_LIST_OPS_CPU
+TF_CALL_POD_STRING_TYPES(REGISTER_TENSOR_LIST_FROM_TENSOR_CPU);
+REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(quint8);
+REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(qint8);
+REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(quint16);
+REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(qint16);
+REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(qint32);
+REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(bfloat16);
 
-#define REGISTER_TENSOR_LIST_OPS_CPU(T)
+#undef REGISTER_TENSOR_LIST_FROM_TENSOR_CPU
 
 REGISTER_UNARY_VARIANT_BINARY_OP_FUNCTION(ADD_VARIANT_BINARY_OP, DEVICE_CPU,
                                           TensorList,
diff --git a/tensorflow/core/kernels/list_kernels.cu.cc b/tensorflow/core/kernels/list_kernels.cu.cc
index c3a5041b00..23f552642c 100644
--- a/tensorflow/core/kernels/list_kernels.cu.cc
+++ b/tensorflow/core/kernels/list_kernels.cu.cc
@@ -36,38 +36,47 @@ namespace tensorflow {
 
 typedef Eigen::GpuDevice GPUDevice;
 
-#define REGISTER_TENSOR_LIST_OPS_GPU(T)                           \
+#define REGISTER_TENSOR_LIST_STACK_GPU(T)                         \
   REGISTER_KERNEL_BUILDER(Name("TensorListStack")                 \
                               .TypeConstraint<T>("element_dtype") \
-                              .Device(DEVICE_GPU)                 \
-                              .HostMemory("element_shape"),       \
+                              .Device(DEVICE_GPU),                \
                           TensorListStack<GPUDevice, T>)          \
   REGISTER_KERNEL_BUILDER(Name("TensorListGather")                \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_GPU)                 \
-                              .HostMemory("indices")              \
-                              .HostMemory("element_shape"),       \
+                              .HostMemory("indices"),             \
                           TensorListGather<GPUDevice, T>)         \
-  REGISTER_KERNEL_BUILDER(Name("TensorListGetItem")               \
-                              .TypeConstraint<T>("element_dtype") \
-                              .Device(DEVICE_GPU)                 \
-                              .HostMemory("index")                \
-                              .HostMemory("element_shape"),       \
-                          TensorListGetItem<GPUDevice, T>)        \
-  REGISTER_KERNEL_BUILDER(Name("TensorListPopBack")               \
-                              .TypeConstraint<T>("element_dtype") \
-                              .Device(DEVICE_GPU)                 \
-                              .HostMemory("element_shape"),       \
-                          TensorListPopBack<GPUDevice, T>)        \
   REGISTER_KERNEL_BUILDER(Name("TensorListConcat")                \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_GPU)                 \
                               .HostMemory("lengths"),             \
-                          TensorListConcat<GPUDevice, T>)         \
+                          TensorListConcat<GPUDevice, T>)
+
+TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_STACK_GPU);
+REGISTER_TENSOR_LIST_STACK_GPU(bfloat16);
+TF_CALL_complex64(REGISTER_TENSOR_LIST_STACK_GPU);
+TF_CALL_complex128(REGISTER_TENSOR_LIST_STACK_GPU);
+TF_CALL_int64(REGISTER_TENSOR_LIST_STACK_GPU);
+REGISTER_TENSOR_LIST_STACK_GPU(bool);
+
+#undef REGISTER_TENSOR_LIST_STACK_GPU
+
+#define REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU(T)               \
   REGISTER_KERNEL_BUILDER(Name("TensorListPushBackBatch")         \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_GPU),                \
-                          TensorListPushBackBatch<GPUDevice, T>)  \
+                          TensorListPushBackBatch<GPUDevice, T>)
+
+TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU);
+REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU(bfloat16);
+TF_CALL_complex64(REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU);
+TF_CALL_complex128(REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU);
+TF_CALL_int64(REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU);
+REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU(bool);
+
+#undef REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU
+
+#define REGISTER_TENSOR_LIST_FROM_TENSOR_GPU(T)                   \
   REGISTER_KERNEL_BUILDER(Name("TensorListFromTensor")            \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_GPU)                 \
@@ -86,20 +95,14 @@ typedef Eigen::GpuDevice GPUDevice;
                               .HostMemory("lengths"),             \
                           TensorListSplit<GPUDevice, T>)
 
-TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_OPS_GPU);
-REGISTER_TENSOR_LIST_OPS_GPU(bfloat16);
-TF_CALL_complex64(REGISTER_TENSOR_LIST_OPS_GPU);
-TF_CALL_complex128(REGISTER_TENSOR_LIST_OPS_GPU);
-TF_CALL_int64(REGISTER_TENSOR_LIST_OPS_GPU);
-REGISTER_TENSOR_LIST_OPS_GPU(bool);
-
-#undef REGISTER_TENSOR_LIST_OPS_GPU
+TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_FROM_TENSOR_GPU);
+REGISTER_TENSOR_LIST_FROM_TENSOR_GPU(bfloat16);
+TF_CALL_complex64(REGISTER_TENSOR_LIST_FROM_TENSOR_GPU);
+TF_CALL_complex128(REGISTER_TENSOR_LIST_FROM_TENSOR_GPU);
+TF_CALL_int64(REGISTER_TENSOR_LIST_FROM_TENSOR_GPU);
+REGISTER_TENSOR_LIST_FROM_TENSOR_GPU(bool);
 
-REGISTER_KERNEL_BUILDER(Name("TensorListPopBack")
-                            .TypeConstraint<Variant>("element_dtype")
-                            .Device(DEVICE_GPU)
-                            .HostMemory("element_shape"),
-                        TensorListPopBack<GPUDevice, Variant>)
+#undef REGISTER_TENSOR_LIST_FROM_TENSOR_GPU
 
 REGISTER_UNARY_VARIANT_BINARY_OP_FUNCTION(ADD_VARIANT_BINARY_OP, DEVICE_GPU,
                                           TensorList,
diff --git a/tensorflow/core/kernels/list_kernels.h b/tensorflow/core/kernels/list_kernels.h
index f74a88a768..fd1be80f11 100644
--- a/tensorflow/core/kernels/list_kernels.h
+++ b/tensorflow/core/kernels/list_kernels.h
@@ -66,9 +66,14 @@ struct TensorList {
 
 Status TensorShapeFromTensor(const Tensor& t, PartialTensorShape* out);
 
-Status GetElementShapeFromInput(OpKernelContext* c,
-                                const TensorList& tensor_list, int index,
-                                PartialTensorShape* element_shape);
+// Allocates a Tensor of requested shape and dtype and fills it with zeros.
+template <typename Device, typename T>
+void BuildZerosTensor(OpKernelContext* c, DataType dtype,
+                      const TensorShape& shape, Tensor* zeros) {
+  OP_REQUIRES_OK(c, c->allocate_temp(dtype, shape, zeros));
+  functor::SetZeroFunctor<Device, T> f;
+  f(c->eigen_device<Device>(), zeros->flat<T>());
+}
 
 template <typename Device, typename T>
 class TensorListStack : public OpKernel {
@@ -80,6 +85,8 @@ class TensorListStack : public OpKernel {
     OP_REQUIRES_OK(c, c->GetAttr("num_elements", &num_elements_));
   }
 
+  ~TensorListStack() {}
+
   void Compute(OpKernelContext* c) override {
     const TensorList* tensor_list =
         c->input(0).scalar<Variant>()().get<TensorList>();
@@ -92,6 +99,13 @@ class TensorListStack : public OpKernel {
         errors::InvalidArgument(
             "Invalid data types; op elements ", DataTypeString(element_dtype_),
             " but list elements ", DataTypeString(tensor_list->element_dtype)));
+    OP_REQUIRES(
+        c,
+        !tensor_list->tensors.empty() ||
+            tensor_list->element_shape.IsFullyDefined(),
+        errors::InvalidArgument("Tried to stack elements of a empty ",
+                                "list with non-fully-defined element_shape: ",
+                                tensor_list->element_shape.DebugString()));
     if (num_elements_ != -1) {
       OP_REQUIRES(c, tensor_list->tensors.size() == num_elements_,
                   errors::InvalidArgument(
@@ -99,40 +113,44 @@ class TensorListStack : public OpKernel {
                       " elements but got a list with ",
                       tensor_list->tensors.size(), " elements."));
     }
-    PartialTensorShape partial_element_shape;
-    OP_REQUIRES_OK(c, GetElementShapeFromInput(c, *tensor_list, 1,
-                                               &partial_element_shape));
-    OP_REQUIRES(
-        c,
-        partial_element_shape.IsFullyDefined() || !tensor_list->tensors.empty(),
-        errors::InvalidArgument("Tried to stack elements of an empty ",
-                                "list with non-fully-defined element_shape: ",
-                                partial_element_shape.DebugString()));
-
-    // Check that `element_shape` input tensor is compatible with the shapes of
-    // element tensors.
-    if (!tensor_list->element_shape.IsFullyDefined()) {
+    // Compute the shape of the output tensor.
+    // If `element_shape` is fully-defined it gets used. It is assumed that all
+    // element tensors have the same shape.
+    // If `element_shape` is not fully-defined the shape of the first
+    // initialized element tensor is used and it is checked that all other
+    // initialized tensors have the same shape. An error is thrown if the list
+    // only contains DT_INVALID type tensors.
+    TensorShape resulting_element_shape;
+    if (!tensor_list->element_shape.AsTensorShape(&resulting_element_shape)) {
+      bool resulting_element_shape_initialized = false;
       for (int i = 0; i < tensor_list->tensors.size(); ++i) {
         const Tensor& t = tensor_list->tensors[i];
-        if (t.dtype() != DT_INVALID) {
-          PartialTensorShape tmp = partial_element_shape;
-          OP_REQUIRES_OK(c, tmp.MergeWith(t.shape(), &partial_element_shape));
+        if (!resulting_element_shape_initialized) {
+          if (t.dtype() == DT_INVALID) {
+            continue;
+          }
+          resulting_element_shape = t.shape();
+          resulting_element_shape_initialized = true;
+          continue;
         }
+        OP_REQUIRES(
+            c, t.dtype() == DT_INVALID || t.shape() == resulting_element_shape,
+            errors::InvalidArgument(
+                "Tried to stack tensors with unequal shapes: ",
+                resulting_element_shape.DebugString(), " vs ",
+                t.shape().DebugString()));
       }
+      OP_REQUIRES(
+          c, resulting_element_shape_initialized,
+          errors::InvalidArgument("Tried to stack list which only contains ",
+                                  "uninitialized tensors and has a ",
+                                  "non-fully-defined element_shape: ",
+                                  tensor_list->element_shape.DebugString()));
     }
-
-    // Compute the shape of the output tensor by pre-pending the leading dim to
-    // the element_shape.
-    TensorShape element_shape;
-    OP_REQUIRES(c, partial_element_shape.AsTensorShape(&element_shape),
-                errors::InvalidArgument(
-                    "Tried to stack list which only contains uninitialized ",
-                    "tensors and has a non-fully-defined element_shape: ",
-                    partial_element_shape.DebugString()));
-    TensorShape output_shape = element_shape;
-    output_shape.InsertDim(0, tensor_list->tensors.size());
+    TensorShape output_tensor_shape = resulting_element_shape;
+    output_tensor_shape.InsertDim(0, tensor_list->tensors.size());
     Tensor* output;
-    OP_REQUIRES_OK(c, c->allocate_output(0, output_shape, &output));
+    OP_REQUIRES_OK(c, c->allocate_output(0, output_tensor_shape, &output));
     if (output->NumElements() == 0) {
       return;
     }
@@ -140,21 +158,13 @@ class TensorListStack : public OpKernel {
     ConstMatrixVector inputs_flat;
     inputs_flat.reserve(tensor_list->tensors.size());
     Tensor zeros;
+    BuildZerosTensor<Device, T>(c, element_dtype_, resulting_element_shape,
+                                &zeros);
     for (const auto& t : tensor_list->tensors) {
       if (t.dtype() != DT_INVALID) {
         inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
             t.shaped<T, 2>({1, t.NumElements()})));
       } else {
-        if (!zeros.NumElements()) {
-          AllocatorAttributes attr;
-          if (element_dtype_ == DT_VARIANT) {
-            attr.set_on_host(true);
-          }
-          OP_REQUIRES_OK(
-              c, c->allocate_temp(element_dtype_, element_shape, &zeros, attr));
-          functor::SetZeroFunctor<Device, T>()(c->eigen_device<Device>(),
-                                               zeros.flat<T>());
-        }
         inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
             const_cast<const Tensor&>(zeros).shaped<T, 2>(
                 {1, zeros.NumElements()})));
@@ -176,123 +186,6 @@ class TensorListStack : public OpKernel {
   DataType element_dtype_;
 };
 
-template <typename Device, typename T>
-class TensorListGetItem : public OpKernel {
- public:
-  explicit TensorListGetItem(OpKernelConstruction* c) : OpKernel(c) {
-    OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_));
-  }
-
-  void Compute(OpKernelContext* c) override {
-    OP_REQUIRES(
-        c, c->input(0).shape().num_elements() == 1,
-        errors::InvalidArgument("List tensors are supposed to be scalars."));
-    const TensorList* l = c->input(0).scalar<Variant>()().get<TensorList>();
-    OP_REQUIRES(c, l != nullptr,
-                errors::InvalidArgument(
-                    "Input handle is not a list. Saw: '",
-                    c->input(0).scalar<Variant>()().DebugString(), "'"));
-    OP_REQUIRES(c, element_dtype_ == l->element_dtype,
-                errors::InvalidArgument("Invalid data types; op elements ",
-                                        DataTypeString(element_dtype_),
-                                        " but list elements ",
-                                        DataTypeString(l->element_dtype)));
-    int32 index = c->input(1).scalar<int32>()();
-    OP_REQUIRES(c, index < l->tensors.size(),
-                errors::InvalidArgument("Trying to access element ", index,
-                                        " in a list with ", l->tensors.size(),
-                                        " elements."));
-    if (l->tensors[index].dtype() != DT_INVALID) {
-      c->set_output(0, l->tensors[index]);
-    } else {
-      PartialTensorShape partial_element_shape;
-      OP_REQUIRES_OK(
-          c, GetElementShapeFromInput(c, *l, 2, &partial_element_shape));
-      TensorShape element_shape;
-      OP_REQUIRES(
-          c, partial_element_shape.AsTensorShape(&element_shape),
-          errors::InvalidArgument("Trying to read an uninitialized tensor but ",
-                                  "element_shape is not fully defined.",
-                                  partial_element_shape.DebugString()));
-      Tensor* result;
-      AllocatorAttributes attr;
-      if (element_dtype_ == DT_VARIANT) {
-        attr.set_on_host(true);
-      }
-      OP_REQUIRES_OK(c, c->allocate_output(0, element_shape, &result, attr));
-      functor::SetZeroFunctor<Device, T>()(c->eigen_device<Device>(),
-                                           result->flat<T>());
-    }
-  }
-
- private:
-  DataType element_dtype_;
-};
-
-template <typename Device, typename T>
-class TensorListPopBack : public OpKernel {
- public:
-  explicit TensorListPopBack(OpKernelConstruction* c) : OpKernel(c) {
-    OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_));
-  }
-
-  void Compute(OpKernelContext* c) override {
-    const TensorList* l = c->input(0).scalar<Variant>()().get<TensorList>();
-    OP_REQUIRES(c, l != nullptr,
-                errors::InvalidArgument(
-                    "Input handle is not a list. Saw: '",
-                    c->input(0).scalar<Variant>()().DebugString(), "'"));
-    OP_REQUIRES(c, element_dtype_ == l->element_dtype,
-                errors::InvalidArgument("Invalid data types; op elements ",
-                                        DataTypeString(element_dtype_),
-                                        " but list elements ",
-                                        DataTypeString(l->element_dtype)));
-
-    OP_REQUIRES(c, !l->tensors.empty(),
-                errors::InvalidArgument("Trying to pop from an empty list."));
-
-    const Tensor& t = l->tensors.back();
-    if (t.dtype() != DT_INVALID) {
-      c->set_output(1, t);
-    } else {
-      PartialTensorShape partial_element_shape;
-      OP_REQUIRES_OK(
-          c, GetElementShapeFromInput(c, *l, 1, &partial_element_shape));
-      TensorShape element_shape;
-      OP_REQUIRES(
-          c, partial_element_shape.AsTensorShape(&element_shape),
-          errors::InvalidArgument("Trying to read an uninitialized tensor but ",
-                                  "element_shape is not fully defined.",
-                                  partial_element_shape.DebugString()));
-      Tensor* result;
-      AllocatorAttributes attr;
-      if (element_dtype_ == DT_VARIANT) {
-        attr.set_on_host(true);
-      }
-      OP_REQUIRES_OK(c, c->allocate_output(1, element_shape, &result, attr));
-      functor::SetZeroFunctor<Device, T>()(c->eigen_device<Device>(),
-                                           result->flat<T>());
-    }
-    AllocatorAttributes attr;
-    attr.set_on_host(true);
-    std::unique_ptr<Tensor> maybe_result = c->forward_input(
-        0, 0, DT_VARIANT, TensorShape{}, c->input_memory_type(0), attr);
-    if (maybe_result != nullptr) {
-      maybe_result->scalar<Variant>()().get<TensorList>()->tensors.pop_back();
-    } else {
-      TensorList output;
-      output = *l;
-      output.tensors.pop_back();
-      Tensor* result;
-      OP_REQUIRES_OK(c, c->allocate_output(0, TensorShape{}, &result, attr));
-      result->scalar<Variant>()() = std::move(output);
-    }
-  }
-
- private:
-  DataType element_dtype_;
-};
-
 template <typename Device, typename T>
 class TensorListConcat : public OpKernel {
  public:
@@ -313,6 +206,8 @@ class TensorListConcat : public OpKernel {
     }
   }
 
+  ~TensorListConcat() {}
+
   void Compute(OpKernelContext* c) override {
     // Check that the input Variant tensor is indeed a TensorList and has the
     // correct element type.
@@ -523,47 +418,47 @@ class TensorListGather : public OpKernel {
             "Invalid data types; op elements ", DataTypeString(element_dtype_),
             " but list elements ", DataTypeString(tensor_list->element_dtype)));
     Tensor indices = c->input(1);
-    PartialTensorShape partial_element_shape;
-    OP_REQUIRES_OK(c, GetElementShapeFromInput(c, *tensor_list, 2,
-                                               &partial_element_shape));
     OP_REQUIRES(
-        c, partial_element_shape.IsFullyDefined() || indices.NumElements() > 0,
+        c,
+        indices.NumElements() > 0 ||
+            tensor_list->element_shape.IsFullyDefined(),
         errors::InvalidArgument("Tried to gather 0-elements from "
                                 "a list with non-fully-defined shape: ",
-                                partial_element_shape.DebugString()));
-
-    // Check that `element_shape` input tensor is compatible with the shapes of
-    // element tensors.
-    if (!tensor_list->element_shape.IsFullyDefined()) {
-      for (int index = 0; index < indices.NumElements(); ++index) {
+                                tensor_list->element_shape.DebugString()));
+    // Compute the shape of the output tensor.
+    // If `element_shape` is fully-defined it gets used. It is assumed that all
+    // requested tensors have the same shape.
+    // If `element_shape` is not fully-defined the shape of the first requested
+    // tensor is used and it is checked that all other tensors have the same
+    // shape.
+    TensorShape resulting_shape;
+    if (!tensor_list->element_shape.AsTensorShape(&resulting_shape)) {
+      const int i = indices.flat<int32>()(0);
+      OP_REQUIRES(
+          c, i < tensor_list->tensors.size(),
+          errors::InvalidArgument("Index ", i, " out o range; list only has ",
+                                  tensor_list->tensors.size(), " elements."));
+      const Tensor& t = tensor_list->tensors[i];
+      resulting_shape = t.shape();
+      for (int index = 1; index < indices.NumElements(); ++index) {
         const int i = indices.flat<int32>()(index);
         const Tensor& t = tensor_list->tensors[i];
-        if (t.dtype() != DT_INVALID) {
-          PartialTensorShape tmp = partial_element_shape;
-          OP_REQUIRES_OK(c, tmp.MergeWith(t.shape(), &partial_element_shape));
-        }
+        OP_REQUIRES(c, t.shape() == resulting_shape,
+                    errors::InvalidArgument(
+                        "Tried to gather elements with unequal shapes: ",
+                        resulting_shape.DebugString(), " vs ",
+                        t.shape().DebugString()));
       }
     }
-
-    // Compute the shape of the output tensor by pre-pending the leading dim to
-    // the element_shape.
-    TensorShape element_shape;
-    OP_REQUIRES(
-        c, partial_element_shape.AsTensorShape(&element_shape),
-        errors::InvalidArgument("Tried to gather uninitialized tensors from a ",
-                                "list with non-fully-defined element_shape: ",
-                                partial_element_shape.DebugString()));
-    TensorShape output_shape = element_shape;
-    output_shape.InsertDim(0, indices.NumElements());
+    resulting_shape.InsertDim(0, indices.NumElements());
     Tensor* output;
-    OP_REQUIRES_OK(c, c->allocate_output(0, output_shape, &output));
+    OP_REQUIRES_OK(c, c->allocate_output(0, resulting_shape, &output));
     if (output->NumElements() == 0) {
       return;
     }
 
     ConstMatrixVector inputs_flat;
-    inputs_flat.reserve(indices.NumElements());
-    Tensor zeros;
+    inputs_flat.reserve(tensor_list->tensors.size());
     for (int index = 0; index < indices.NumElements(); ++index) {
       const int i = indices.flat<int32>()(index);
       OP_REQUIRES(
@@ -571,24 +466,8 @@ class TensorListGather : public OpKernel {
           errors::InvalidArgument("Index ", i, " out o range; list only has ",
                                   tensor_list->tensors.size(), " elements."));
       const Tensor& t = tensor_list->tensors[i];
-      if (t.dtype() != DT_INVALID) {
-        inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
-            t.shaped<T, 2>({1, t.NumElements()})));
-      } else {
-        if (!zeros.NumElements()) {
-          AllocatorAttributes attr;
-          if (element_dtype_ == DT_VARIANT) {
-            attr.set_on_host(true);
-          }
-          OP_REQUIRES_OK(
-              c, c->allocate_temp(element_dtype_, element_shape, &zeros, attr));
-          functor::SetZeroFunctor<Device, T>()(c->eigen_device<Device>(),
-                                               zeros.flat<T>());
-        }
-        inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
-            const_cast<const Tensor&>(zeros).shaped<T, 2>(
-                {1, zeros.NumElements()})));
-      }
+      inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
+          t.shaped<T, 2>({1, t.NumElements()})));
     }
     auto output_flat = output->shaped<T, 2>({1, output->NumElements()});
 
@@ -778,6 +657,8 @@ class TensorListPushBackBatch : public OpKernel {
     OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_));
   }
 
+  ~TensorListPushBackBatch() override {}
+
   void Compute(OpKernelContext* c) override {
     const Tensor& input = c->input(1);
     OP_REQUIRES(c, element_dtype_ == input.dtype(),
diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index eb251fc517..3b7a7b812a 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -77304,10 +77304,6 @@ op {
     name: "indices"
     type: DT_INT32
   }
-  input_arg {
-    name: "element_shape"
-    type: DT_INT32
-  }
   output_arg {
     name: "values"
     type_attr: "element_dtype"
@@ -77327,10 +77323,6 @@ op {
     name: "index"
     type: DT_INT32
   }
-  input_arg {
-    name: "element_shape"
-    type: DT_INT32
-  }
   output_arg {
     name: "item"
     type_attr: "element_dtype"
@@ -77357,10 +77349,6 @@ op {
     name: "input_handle"
     type: DT_VARIANT
   }
-  input_arg {
-    name: "element_shape"
-    type: DT_INT32
-  }
   output_arg {
     name: "output_handle"
     type: DT_VARIANT
@@ -77552,10 +77540,6 @@ op {
     name: "input_handle"
     type: DT_VARIANT
   }
-  input_arg {
-    name: "element_shape"
-    type: DT_INT32
-  }
   output_arg {
     name: "tensor"
     type_attr: "element_dtype"
diff --git a/tensorflow/core/ops/list_ops.cc b/tensorflow/core/ops/list_ops.cc
index a16fabe2ab..cbc9c7a2f4 100644
--- a/tensorflow/core/ops/list_ops.cc
+++ b/tensorflow/core/ops/list_ops.cc
@@ -130,7 +130,6 @@ REGISTER_OP("TensorListLength")
 
 REGISTER_OP("TensorListPopBack")
     .Input("input_handle: variant")
-    .Input("element_shape: int32")
     .Output("output_handle: variant")
     .Output("tensor: element_dtype")
     .Attr("element_dtype: type")
@@ -167,7 +166,6 @@ REGISTER_OP("TensorListPopBack")
 
 REGISTER_OP("TensorListStack")
     .Input("input_handle: variant")
-    .Input("element_shape: int32")
     .Output("tensor: element_dtype")
     .Attr("element_dtype: type")
     .Attr("num_elements: int = -1")
@@ -353,7 +351,6 @@ REGISTER_OP("TensorListReserve")
 REGISTER_OP("TensorListGetItem")
     .Input("input_handle: variant")
     .Input("index: int32")
-    .Input("element_shape: int32")
     .Output("item: element_dtype")
     .Attr("element_dtype: type")
     .SetShapeFn([](shape_inference::InferenceContext* c) {
@@ -421,7 +418,6 @@ REGISTER_OP("TensorListSetItem")
 REGISTER_OP("TensorListGather")
     .Input("input_handle: variant")
     .Input("indices: int32")
-    .Input("element_shape: int32")
     .Output("values: element_dtype")
     .Attr("element_dtype: type")
     .SetShapeFn([](shape_inference::InferenceContext* c) {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index f43a8c4744..92a7bb9a34 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -37082,10 +37082,6 @@ op {
     name: "indices"
     type: DT_INT32
   }
-  input_arg {
-    name: "element_shape"
-    type: DT_INT32
-  }
   output_arg {
     name: "values"
     type_attr: "element_dtype"
@@ -37105,10 +37101,6 @@ op {
     name: "index"
     type: DT_INT32
   }
-  input_arg {
-    name: "element_shape"
-    type: DT_INT32
-  }
   output_arg {
     name: "item"
     type_attr: "element_dtype"
@@ -37135,10 +37127,6 @@ op {
     name: "input_handle"
     type: DT_VARIANT
   }
-  input_arg {
-    name: "element_shape"
-    type: DT_INT32
-  }
   output_arg {
     name: "output_handle"
     type: DT_VARIANT
@@ -37330,10 +37318,6 @@ op {
     name: "input_handle"
     type: DT_VARIANT
   }
-  input_arg {
-    name: "element_shape"
-    type: DT_INT32
-  }
   output_arg {
     name: "tensor"
     type_attr: "element_dtype"
diff --git a/tensorflow/python/kernel_tests/list_ops_test.py b/tensorflow/python/kernel_tests/list_ops_test.py
index 3f340aa07d..ec6906f20c 100644
--- a/tensorflow/python/kernel_tests/list_ops_test.py
+++ b/tensorflow/python/kernel_tests/list_ops_test.py
@@ -33,7 +33,6 @@ from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.ops import gen_list_ops
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import list_ops
 from tensorflow.python.ops import math_ops
@@ -90,58 +89,6 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       l = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
       self.evaluate(l)
 
-  def testPopUninitializedTensorUseListElementShape(self):
-    l = list_ops.tensor_list_reserve(
-        element_dtype=dtypes.float32, element_shape=[2, 3], num_elements=3)
-    _, e = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
-    self.assertAllEqual(e, np.zeros((2, 3)))
-
-  def testPopUninitializedTensorUseSpecifiedElementShape(self):
-    l = list_ops.tensor_list_reserve(
-        element_dtype=dtypes.float32, element_shape=[None, 3], num_elements=3)
-    _, e = gen_list_ops.tensor_list_pop_back(
-        l, element_dtype=dtypes.float32, element_shape=[4, 3])
-    self.assertAllEqual(e, np.zeros((4, 3)))
-
-  def testPopUninitializedTensorWithInvalidElementShapeFails(self):
-    l = list_ops.tensor_list_reserve(
-        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
-    with self.assertRaisesRegexp(
-        errors.InvalidArgumentError,
-        "Trying to read an uninitialized tensor but "
-        "element_shape is not fully defined"):
-      _, e = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
-      self.evaluate(e)
-
-    l = list_ops.tensor_list_reserve(
-        element_dtype=dtypes.float32, element_shape=[None, 2], num_elements=3)
-    with self.assertRaisesRegexp(
-        errors.InvalidArgumentError,
-        r"Incompatible shapes during merge: \[1,3\] vs. \[\?,2\]"):
-      _, e = gen_list_ops.tensor_list_pop_back(
-          l, element_dtype=dtypes.float32, element_shape=[1, 3])
-      self.evaluate(e)
-
-  def testPushGetGrad(self):
-    with backprop.GradientTape() as tape:
-      l = list_ops.empty_tensor_list(
-          element_dtype=dtypes.float32, element_shape=None)
-      c0 = constant_op.constant(5.0)
-      c1 = constant_op.constant([10.0, 20.0])
-      tape.watch(c0)
-      tape.watch(c1)
-      l = list_ops.tensor_list_push_back(l, c0)
-      l = list_ops.tensor_list_push_back(l, c1)
-      t1 = list_ops.tensor_list_get_item(l, 1, element_dtype=dtypes.float32)
-      self.assertAllEqual(self.evaluate(t1), [10.0, 20.0])
-      # t1 == c1 so the gradient should be [0., [1., 1.]]
-      # This tests that the gradient of push_back correctly converts DT_INVALID
-      # tensors to zeros. The list returned by the gradient of GetItem will
-      # have only have tensor at index 1 set and others set to DT_INVALID.
-      dt0, dt1 = tape.gradient(t1, [c0, c1])
-      self.assertAllEqual(self.evaluate(dt1), [1.0, 1.0])
-      self.assertEqual(self.evaluate(dt0), 0.0)
-
   def _testStack(self, max_num_elements):
     l = list_ops.empty_tensor_list(
         element_dtype=dtypes.float32,
@@ -183,8 +130,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
     # Should raise an error when the element tensors do not all have the same
     # shape.
-    with self.assertRaisesRegexp(errors.InvalidArgumentError,
-                                 "Incompatible ranks during merge: 0 vs. 1"):
+    with self.assertRaisesRegexp(errors.InvalidArgumentError, "unequal shapes"):
       l = list_ops.tensor_list_push_back(l, constant_op.constant([3.0, 4.0]))
       t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
       self.evaluate(t)
@@ -205,9 +151,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
     # Should raise an error when the element tensors do not all have the same
     # shape.
-    with self.assertRaisesRegexp(
-        errors.InvalidArgumentError,
-        r"Incompatible shapes during merge: \[1\] vs. \[2\]"):
+    with self.assertRaisesRegexp(errors.InvalidArgumentError, "unequal shapes"):
       l = list_ops.tensor_list_push_back(l, constant_op.constant([2.0, 3.0]))
       t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
       self.evaluate(t)
@@ -286,13 +230,6 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
       self.evaluate(t)
 
-  def testStackUsingSpecifiedElementShape(self):
-    l = list_ops.tensor_list_reserve(
-        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
-    t = gen_list_ops.tensor_list_stack(
-        l, element_dtype=dtypes.float32, element_shape=[])
-    self.assertAllEqual(self.evaluate(t), np.zeros((3,)))
-
   @parameterized.named_parameters(("NoMaxNumElements", None),
                                   ("WithMaxNumElements", 2))
   def testGatherGrad(self, max_num_elements):
@@ -331,8 +268,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
     # Should raise an error when the requested tensors do not all have the same
     # shape.
-    with self.assertRaisesRegexp(errors.InvalidArgumentError,
-                                 "Incompatible ranks during merge: 0 vs. 1"):
+    with self.assertRaisesRegexp(errors.InvalidArgumentError, "unequal shapes"):
       t = list_ops.tensor_list_gather(l, [0, 2], element_dtype=dtypes.float32)
       self.evaluate(t)
 
@@ -356,9 +292,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
     # Should raise an error when the requested tensors do not all have the same
     # shape.
-    with self.assertRaisesRegexp(
-        errors.InvalidArgumentError,
-        r"Incompatible shapes during merge: \[1\] vs. \[2\]"):
+    with self.assertRaisesRegexp(errors.InvalidArgumentError, "unequal shapes"):
       t = list_ops.tensor_list_gather(l, [0, 2], element_dtype=dtypes.float32)
       self.evaluate(t)
 
@@ -413,54 +347,6 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     dl_length = list_ops.tensor_list_length(dl)
     self.assertAllEqual(self.evaluate(dl_length), 3)
 
-  def _testGatherWithUninitializedTensors(self):
-    l = list_ops.tensor_list_reserve(
-        element_dtype=dtypes.float32, element_shape=[], num_elements=3)
-    t = list_ops.tensor_list_gather(l, [0, 2], element_dtype=dtypes.float32)
-    self.assertAllEqual(self.evaluate(t), [0., 0.])
-
-  def testGatherWithUninitializedTensors(self):
-    self._testGatherWithUninitializedTensors()
-
-  def testGatherWithUninitializedTensorsGpu(self):
-    if not context.num_gpus():
-      return
-    with context.device("gpu:0"):
-      self._testGatherWithUninitializedTensors()
-
-  def _testGatherWithUninitializedTensorsInferShape(self):
-    l = list_ops.tensor_list_reserve(
-        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
-    l = list_ops.tensor_list_set_item(l, 1, [1., 2.])
-    t = list_ops.tensor_list_gather(l, [1, 2], element_dtype=dtypes.float32)
-    self.assertAllEqual(self.evaluate(t), [[1., 2.], [0., 0.]])
-
-  def testGatherWithUninitializedTensorsInferShape(self):
-    self._testGatherWithUninitializedTensorsInferShape()
-
-  def testGatherWithUninitializedTensorsInferShapeGpu(self):
-    if not context.num_gpus():
-      return
-    with context.device("gpu:0"):
-      self._testGatherWithUninitializedTensorsInferShape()
-
-  def testGatherReservedListWithNoElementsAndPartialElementShapeFails(self):
-    l = list_ops.tensor_list_reserve(
-        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
-    with self.assertRaisesRegexp(
-        errors.InvalidArgumentError,
-        "Tried to gather uninitialized tensors from a"
-        " list with non-fully-defined element_shape"):
-      t = list_ops.tensor_list_gather(l, [0], element_dtype=dtypes.float32)
-      self.evaluate(t)
-
-  def testGatherUsingSpecifiedElementShape(self):
-    l = list_ops.tensor_list_reserve(
-        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
-    t = gen_list_ops.tensor_list_gather(
-        l, [0, 1, 2], element_dtype=dtypes.float32, element_shape=[])
-    self.assertAllEqual(self.evaluate(t), np.zeros((3,)))
-
   def testScatterOutputListSize(self):
     c0 = constant_op.constant([1.0, 2.0])
     l = list_ops.tensor_list_scatter(
@@ -500,18 +386,6 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     dt = tape.gradient(loss, c0)
     self.assertAllEqual(self.evaluate(dt), [2., 4.])
 
-  def testScatterWithPartialReadGrad(self):
-    with backprop.GradientTape() as tape:
-      c0 = constant_op.constant([1.0, 2.0])
-      tape.watch(c0)
-      l = list_ops.tensor_list_scatter(
-          c0, [1, 0], ops.convert_to_tensor([], dtype=dtypes.int32))
-      t0 = list_ops.tensor_list_get_item(l, 0, element_dtype=dtypes.float32)
-      self.assertAllEqual(self.evaluate(t0), 2.0)
-      loss = t0 * t0
-    dt = tape.gradient(loss, c0)
-    self.assertAllEqual(self.evaluate(dt), [0., 4.])
-
   def testTensorListFromTensor(self):
     t = constant_op.constant([1.0, 2.0])
     l = list_ops.tensor_list_from_tensor(t, element_shape=[])
@@ -553,50 +427,6 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       self.assertAllEqual(self.evaluate(e), 10.0)
     self.assertAllEqual(self.evaluate(tape.gradient(e, t)), 2.0)
 
-  def testGetUninitializedTensorUseListElementShape(self):
-    l = list_ops.tensor_list_reserve(
-        element_dtype=dtypes.float32, element_shape=[], num_elements=3)
-    l = list_ops.tensor_list_set_item(l, 0, 5.)
-    e1 = list_ops.tensor_list_get_item(l, 1, element_dtype=dtypes.float32)
-    e2 = list_ops.tensor_list_get_item(l, 2, element_dtype=dtypes.float32)
-    self.assertEqual(self.evaluate(e1), 0.)
-    self.assertEqual(self.evaluate(e2), 0.)
-
-  def testGetUninitializedTensorUseSpecifiedElementShape(self):
-    l = list_ops.tensor_list_reserve(
-        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
-    e0 = gen_list_ops.tensor_list_get_item(
-        l, 0, element_shape=[], element_dtype=dtypes.float32)
-    e1 = gen_list_ops.tensor_list_get_item(
-        l, 1, element_shape=[2, 3], element_dtype=dtypes.float32)
-    self.assertEqual(self.evaluate(e0), 0.)
-    self.assertAllEqual(self.evaluate(e1), np.zeros((2, 3)))
-
-    l = list_ops.tensor_list_reserve(
-        element_dtype=dtypes.float32, element_shape=[None, 3], num_elements=3)
-    e1 = gen_list_ops.tensor_list_get_item(
-        l, 1, element_shape=[2, 3], element_dtype=dtypes.float32)
-    self.assertAllEqual(self.evaluate(e1), np.zeros((2, 3)))
-
-  def testGetUninitializedTensorWithInvalidElementShapeFails(self):
-    l = list_ops.tensor_list_reserve(
-        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
-    with self.assertRaisesRegexp(
-        errors.InvalidArgumentError,
-        "Trying to read an uninitialized tensor but "
-        "element_shape is not fully defined"):
-      e0 = list_ops.tensor_list_get_item(l, 0, element_dtype=dtypes.float32)
-      self.evaluate(e0)
-
-    l = list_ops.tensor_list_reserve(
-        element_dtype=dtypes.float32, element_shape=[None, 2], num_elements=3)
-    with self.assertRaisesRegexp(
-        errors.InvalidArgumentError,
-        r"Incompatible shapes during merge: \[1,3\] vs. \[\?,2\]"):
-      e0 = gen_list_ops.tensor_list_get_item(
-          l, 0, element_dtype=dtypes.float32, element_shape=[1, 3])
-      self.evaluate(e0)
-
   @test_util.run_deprecated_v1
   @test_util.enable_control_flow_v2
   def testSkipEagerSetItemIndexOutOfBounds(self):
diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
index d0efb47a38..e8af998a70 100644
--- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py
+++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
@@ -1281,23 +1281,13 @@ class TensorArrayTest(test.TestCase):
       grad_r0_vals = session.run(grad_r0)[0]
       self.assertAllEqual(grad_r0_vals, [1.0, 0.0])
 
-  @test_util.run_deprecated_v1
-  def testSkipEagerGradientWhenNotAllComponentsRead(self):
+  # TODO(srbs): Figure out how to enable this. This is probably failing
+  # because we are trying to stack a TensorList with invalid tensors.
+  # That is because we do not receive gradients for all list indices.
+  # Figure out how TensorArray handles this.
+  def disabletestGradientWhenNotAllComponentsRead(self):
     self._testGradientWhenNotAllComponentsRead()
 
-  @test_util.run_deprecated_v1
-  def testSkipEagerWriteButNotAllComponentsReadGrad(self):
-    with self.cached_session(use_gpu=True) as session:
-      x0 = constant_op.constant(5.0)
-      x1 = constant_op.constant(10.0)
-      ta = tensor_array_ops.TensorArray(
-          dtype=dtypes.float32, size=2).write(0, x0).write(1, x1)
-      r0 = ta.read(0)
-      # calculate (dr0/dx0, dr0/dx1).  since r0 = x0, gradients are (1, 0).
-      grad_r0_x1 = gradients_impl.gradients(ys=[r0], xs=[x0, x1], grad_ys=[1.0])
-      grad_r0_x1_vals = session.run(grad_r0_x1)
-      self.assertAllEqual(grad_r0_x1_vals, [1.0, 0.0])
-
   def _testTensorArrayUnpackDynamic(self):
     with self.cached_session(use_gpu=True) as sess:
       ta = tensor_array_ops.TensorArray(
@@ -1331,8 +1321,8 @@ class TensorArrayTest(test.TestCase):
     with self.cached_session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
           dtype=dtypes.float32, size=0, dynamic_size=False, infer_shape=False)
-      v2_msg = ("Tried to stack elements of an empty list with "
-                "non-fully-defined element_shape")
+      v2_msg = ("Tried to stack elements of a empty list with non-fully-defined"
+                " element_shape")
       v1_msg = (
           "TensorArray has size zero, but element shape <unknown> is not "
           "fully defined. Currently only static shapes are supported when "
@@ -1395,29 +1385,6 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual([10.0, -10.0], read_vals[1])
       self.assertAllEqual([[2.0, 3.0], [4.0, 5.0]], grad_vals[0])
 
-  @test_util.run_v1_only("b/117943489")
-  def testSkipEagerTensorArrayScatterPartialReadAndGradients(self):
-    with self.session(use_gpu=True) as session:
-      ta = tensor_array_ops.TensorArray(
-          dtype=dtypes.float32,
-          tensor_array_name="foo",
-          size=0,
-          dynamic_size=True)
-
-      indices = constant_op.constant([1, 8])
-      value = constant_op.constant([[1.0, -1.0], [10.0, -10.0]])
-
-      w = ta.scatter(indices, value)
-      r0 = w.read(1)
-
-      # Test combined gradients + aggregation of read(0)
-      grad = gradients_impl.gradients(
-          ys=[r0], xs=[value], grad_ys=[[2.0, 3.0]])[0]
-      read_val, grad_val = session.run([r0, grad])
-
-      self.assertAllEqual([1.0, -1.0], read_val)
-      self.assertAllEqual([[2.0, 3.0], [0.0, 0.0]], grad_val)
-
   @test_util.run_v1_only("b/118890905")
   def testTensorArrayWriteGatherAndGradients(self):
     with self.session(use_gpu=True) as session:
diff --git a/tensorflow/python/ops/list_ops.py b/tensorflow/python/ops/list_ops.py
index af059a541e..df928ea85d 100644
--- a/tensorflow/python/ops/list_ops.py
+++ b/tensorflow/python/ops/list_ops.py
@@ -71,41 +71,6 @@ def tensor_list_from_tensor(tensor, element_shape, name=None):
       name=name)
 
 
-def tensor_list_get_item(input_handle, index, element_dtype, name=None):
-  return gen_list_ops.tensor_list_get_item(
-      input_handle=input_handle,
-      index=index,
-      element_shape=-1,
-      element_dtype=element_dtype,
-      name=name)
-
-
-def tensor_list_pop_back(input_handle, element_dtype, name=None):
-  return gen_list_ops.tensor_list_pop_back(
-      input_handle=input_handle,
-      element_shape=-1,
-      element_dtype=element_dtype,
-      name=name)
-
-
-def tensor_list_gather(input_handle, indices, element_dtype, name=None):
-  return gen_list_ops.tensor_list_gather(
-      input_handle=input_handle,
-      indices=indices,
-      element_shape=-1,
-      element_dtype=element_dtype,
-      name=name)
-
-
-def tensor_list_stack(input_handle, element_dtype, num_elements=-1, name=None):
-  return gen_list_ops.tensor_list_stack(
-      input_handle=input_handle,
-      element_shape=-1,
-      element_dtype=element_dtype,
-      num_elements=num_elements,
-      name=name)
-
-
 def tensor_list_concat(input_handle, element_dtype, element_shape=None,
                        name=None):
   # Ignore the lengths output of TensorListConcat. It is only used during
@@ -145,9 +110,7 @@ def tensor_list_set_item(input_handle,
 @ops.RegisterGradient("TensorListPushBack")
 def _PushBackGrad(op, dresult):
   return gen_list_ops.tensor_list_pop_back(
-      dresult,
-      element_shape=array_ops.shape(op.inputs[1]),
-      element_dtype=op.get_attr("element_dtype"))
+      dresult, element_dtype=op.get_attr("element_dtype"))
 
 
 @ops.RegisterGradient("TensorListPopBack")
@@ -157,12 +120,12 @@ def _PopBackGrad(op, dlist, delement):
         element_dtype=delement.dtype,
         element_shape=gen_list_ops.tensor_list_element_shape(
             op.outputs[0], shape_type=dtypes.int32))
-  return gen_list_ops.tensor_list_push_back(dlist, delement), None
+  return gen_list_ops.tensor_list_push_back(dlist, delement)
 
 
 @ops.RegisterGradient("TensorListStack")
 def _TensorListStackGrad(unused_op, dtensor):
-  return tensor_list_from_tensor(dtensor, element_shape=dtensor.shape[1:]), None
+  return tensor_list_from_tensor(dtensor, element_shape=dtensor.shape[1:])
 
 
 @ops.RegisterGradient("TensorListConcat")
@@ -187,21 +150,17 @@ def _TensorListSplitGrad(op, dlist):
 @ops.RegisterGradient("TensorListFromTensor")
 def _TensorListFromTensorGrad(op, dlist):
   """Gradient for TensorListFromTensor."""
-  t = op.inputs[0]
-  if t.shape.dims and t.shape.dims[0].value is not None:
-    num_elements = t.shape.dims[0].value
+  if op.inputs[0].shape.dims and op.inputs[0].shape.dims[0].value is not None:
+    num_elements = op.inputs[0].shape.dims[0].value
   else:
     num_elements = None
   if dlist is None:
     dlist = empty_tensor_list(
-        element_dtype=t.dtype,
+        element_dtype=op.inputs[0].dtype,
         element_shape=gen_list_ops.tensor_list_element_shape(
             op.outputs[0], shape_type=dtypes.int32))
   tensor_grad = gen_list_ops.tensor_list_stack(
-      dlist,
-      element_shape=array_ops.slice(array_ops.shape(t), [1], [-1]),
-      element_dtype=t.dtype,
-      num_elements=num_elements)
+      dlist, element_dtype=op.inputs[0].dtype, num_elements=num_elements)
   shape_grad = None
   return tensor_grad, shape_grad
 
@@ -218,22 +177,17 @@ def _TensorListGetItemGrad(op, ditem):
       index=op.inputs[1],
       item=ditem)
   index_grad = None
-  element_shape_grad = None
-  return list_grad, index_grad, element_shape_grad
+  return list_grad, index_grad
 
 
 @ops.RegisterGradient("TensorListSetItem")
 def _TensorListSetItemGrad(op, dlist):
-  """Gradient function for TensorListSetItem."""
   _, index, item = op.inputs
   list_grad = gen_list_ops.tensor_list_set_item(
       dlist, index=index, item=array_ops.zeros_like(item))
   index_grad = None
   element_grad = gen_list_ops.tensor_list_get_item(
-      dlist,
-      index,
-      element_shape=array_ops.shape(item),
-      element_dtype=item.dtype)
+      dlist, index, element_dtype=item.dtype)
   return list_grad, index_grad, element_grad
 
 
@@ -246,8 +200,7 @@ def _TensorListResizeGrad(op, dlist):
 
 @ops.RegisterGradient("TensorListGather")
 def _TensorListGatherGrad(op, dtensor):
-  """Gradient function for TensorListGather."""
-  input_list, indices, _ = op.inputs
+  input_list, indices = op.inputs
   dlist = gen_list_ops.tensor_list_scatter(
       tensor=dtensor,
       indices=indices,
@@ -256,17 +209,14 @@ def _TensorListGatherGrad(op, dtensor):
   # so we manually resize it to match the size of the input list.
   input_list_size = gen_list_ops.tensor_list_length(input_list)
   dlist = gen_list_ops.tensor_list_resize(dlist, input_list_size)
-  return dlist, None, None
+  return dlist, None
 
 
 @ops.RegisterGradient("TensorListScatter")
 def _TensorListScatterGrad(op, dlist):
   t, indices, _ = op.inputs
   return gen_list_ops.tensor_list_gather(
-      dlist,
-      indices,
-      element_shape=array_ops.slice(array_ops.shape(t), [1], [-1]),
-      element_dtype=t.dtype), None, None
+      dlist, indices, element_dtype=t.dtype), None, None
 
 
 def _build_element_shape(shape):
-- 
GitLab


From fc63d8b263db4fa4e1c839347aef289cdde0b210 Mon Sep 17 00:00:00 2001
From: James Ring <sjr@google.com>
Date: Wed, 23 Jan 2019 18:08:47 -0800
Subject: [PATCH 1160/2345] Add two new functions to tensor C API.

This change adds two functions:
  * TF_TensorElementCount - returns the number of elements in the tensor
  * TF_TensorUnsafeCopyFromInternal - a utility function for making
    shape-changing copies of tensors. This is used for implementing
    BitcastOp in C in a subsequent change.

PiperOrigin-RevId: 230640042
---
 tensorflow/c/c_api.cc      | 68 ++++++++++++++++++++++++++++++++++++++
 tensorflow/c/c_api.h       | 33 ++++++++++++++++++
 tensorflow/c/c_api_test.cc | 36 ++++++++++++++++++++
 3 files changed, 137 insertions(+)

diff --git a/tensorflow/c/c_api.cc b/tensorflow/c/c_api.cc
index 9f2f83920c..94d9f4a6fa 100644
--- a/tensorflow/c/c_api.cc
+++ b/tensorflow/c/c_api.cc
@@ -257,6 +257,74 @@ int64_t TF_Dim(const TF_Tensor* t, int dim_index) {
 size_t TF_TensorByteSize(const TF_Tensor* t) { return t->buffer->size(); }
 void* TF_TensorData(const TF_Tensor* t) { return t->buffer->data(); }
 
+int64_t TF_TensorElementCount(const TF_Tensor* t) {
+  int64_t result = 1;
+  int rank = TF_NumDims(t);
+  for (int dim = 0; dim < rank; ++dim) {
+    result *= TF_Dim(t, dim);
+  }
+  return result;
+}
+
+// Returns the number of elements that would be present in a tensor with the
+// given shape.
+static int64_t ShapeNumElements(const int64_t* dims, int num_dims) {
+  int64_t result = 1;
+  for (int dim = 0; dim < num_dims; ++dim) {
+    result *= dims[dim];
+  }
+  return result;
+}
+
+static void UnrefIfNonNull(::tensorflow::TensorBuffer* buf) {
+  if (buf != nullptr) {
+    buf->Unref();
+  }
+}
+
+static void RefIfNonNull(::tensorflow::TensorBuffer* buf) {
+  if (buf != nullptr) {
+    buf->Ref();
+  }
+}
+
+void TF_TensorBitcastFrom(const TF_Tensor* from, TF_DataType type,
+                          TF_Tensor* to, const int64_t* new_dims,
+                          int num_new_dims, TF_Status* status) {
+  TF_SetStatus(status, TF_OK, "");
+  size_t in_size = TF_DataTypeSize(TF_TensorType(from));
+  if (in_size == 0) {
+    TF_SetStatus(status, TF_INVALID_ARGUMENT,
+                 "input tensor has a zero-sized data type");
+    return;
+  }
+  size_t out_size = TF_DataTypeSize(type);
+  if (out_size == 0) {
+    TF_SetStatus(status, TF_INVALID_ARGUMENT,
+                 "output tensor has a zero-sized data type");
+    return;
+  }
+
+  if (ShapeNumElements(new_dims, num_new_dims) * out_size !=
+      TF_TensorElementCount(from) * in_size) {
+    TF_SetStatus(status, TF_INVALID_ARGUMENT,
+                 "input tensor is not compatible with output shape");
+    return;
+  }
+
+  tensorflow::TensorShapeProto p;
+  for (int i = 0; i < num_new_dims; ++i) {
+    p.add_dim()->set_size(new_dims[i]);
+  }
+  to->shape = tensorflow::TensorShape(p);
+  to->dtype = type;
+  if (to->buffer != from->buffer) {
+    UnrefIfNonNull(to->buffer);
+    to->buffer = from->buffer;
+    RefIfNonNull(to->buffer);
+  }
+}
+
 // --------------------------------------------------------------------------
 size_t TF_StringEncode(const char* src, size_t src_len, char* dst,
                        size_t dst_len, TF_Status* status) {
diff --git a/tensorflow/c/c_api.h b/tensorflow/c/c_api.h
index c7abba8552..8031928dac 100644
--- a/tensorflow/c/c_api.h
+++ b/tensorflow/c/c_api.h
@@ -272,6 +272,39 @@ TF_CAPI_EXPORT extern size_t TF_TensorByteSize(const TF_Tensor*);
 // Return a pointer to the underlying data buffer.
 TF_CAPI_EXPORT extern void* TF_TensorData(const TF_Tensor*);
 
+// Returns the number of elements in the tensor.
+TF_CAPI_EXPORT extern int64_t TF_TensorElementCount(const TF_Tensor* tensor);
+
+// Copy the internal data representation of `from` to `to`. `new_dims` and
+// `num_new_dims` specify the new shape of the `to` tensor, `type` specifies its
+// data type. On success, *status is set to TF_OK and the two tensors share the
+// same data buffer.
+//
+// This call requires that the `from` tensor and the given type and shape (dims
+// and num_dims) are "compatible" (i.e. they occupy the same number of bytes).
+// Specifically, given from_type_size = TF_DataTypeSize(TF_TensorType(from)):
+//
+// ShapeElementCount(dims, num_dims) * TF_DataTypeSize(type)
+//
+// must equal
+//
+// TF_TensorElementCount(from) * from_type_size
+//
+// where TF_ShapeElementCount would be the number of elements in a tensor with
+// the given shape.
+//
+// In addition, this function requires:
+//   * TF_DataTypeSize(TF_TensorType(from)) != 0
+//   * TF_DataTypeSize(type) != 0
+//
+// If any of the requirements are not met, *status is set to
+// TF_INVALID_ARGUMENT.
+TF_CAPI_EXPORT extern void TF_TensorBitcastFrom(const TF_Tensor* from,
+                                                TF_DataType type, TF_Tensor* to,
+                                                const int64_t* new_dims,
+                                                int num_new_dims,
+                                                TF_Status* status);
+
 // --------------------------------------------------------------------------
 // Encode the string `src` (`src_len` bytes long) into `dst` in the format
 // required by TF_STRING tensors. Does not write to memory more than `dst_len`
diff --git a/tensorflow/c/c_api_test.cc b/tensorflow/c/c_api_test.cc
index d5934a1039..2be03bf0de 100644
--- a/tensorflow/c/c_api_test.cc
+++ b/tensorflow/c/c_api_test.cc
@@ -163,6 +163,7 @@ TEST(CAPI, AllocateTensor) {
   EXPECT_EQ(dims[0], TF_Dim(t, 0));
   EXPECT_EQ(dims[1], TF_Dim(t, 1));
   EXPECT_EQ(num_bytes, TF_TensorByteSize(t));
+  EXPECT_EQ(6, TF_TensorElementCount(t));
   TF_DeleteTensor(t);
 }
 
@@ -1467,6 +1468,41 @@ TEST(CAPI, DeletingNullPointerIsSafe) {
   TF_DeleteStatus(status);
 }
 
+TEST(CAPI, TestBitcastFrom_Reshape) {
+  int64_t dims[] = {2, 3};
+  TF_Tensor* a =
+      TF_AllocateTensor(TF_UINT64, dims, 2, 6 * TF_DataTypeSize(TF_UINT64));
+  TF_Tensor* b =
+      TF_AllocateTensor(TF_UINT64, nullptr, 0, TF_DataTypeSize(TF_UINT64));
+  EXPECT_NE(a, nullptr);
+  EXPECT_NE(b, nullptr);
+
+  EXPECT_EQ(6, TF_TensorElementCount(a));
+  EXPECT_EQ(1, TF_TensorElementCount(b));
+  EXPECT_EQ(6 * TF_DataTypeSize(TF_UINT64), TF_TensorByteSize(a));
+  EXPECT_EQ(TF_DataTypeSize(TF_UINT64), TF_TensorByteSize(b));
+
+  int64_t new_dims[] = {3, 2};
+  TF_Status* status = TF_NewStatus();
+  TF_TensorBitcastFrom(a, TF_UINT64, b, new_dims, 2, status);
+  ASSERT_EQ(TF_OK, TF_GetCode(status));
+  TF_DeleteStatus(status);
+
+  EXPECT_EQ(6, TF_TensorElementCount(a));
+  EXPECT_EQ(6, TF_TensorElementCount(b));
+  EXPECT_EQ(6 * TF_DataTypeSize(TF_UINT64), TF_TensorByteSize(a));
+  EXPECT_EQ(6 * TF_DataTypeSize(TF_UINT64), TF_TensorByteSize(b));
+
+  // Check that a write to one tensor shows up in the other.
+  *(static_cast<int64_t*>(TF_TensorData(a))) = 4;
+  EXPECT_EQ(4, *(static_cast<int64_t*>(TF_TensorData(b))));
+  *(static_cast<int64_t*>(TF_TensorData(b))) = 6;
+  EXPECT_EQ(6, *(static_cast<int64_t*>(TF_TensorData(a))));
+
+  TF_DeleteTensor(a);
+  TF_DeleteTensor(b);
+}
+
 REGISTER_OP("TestOpWithNoGradient")
     .Input("x: T")
     .Output("y: T")
-- 
GitLab


From 9d8eb5d5f2693e56b9d52d9485a73ec5bc406806 Mon Sep 17 00:00:00 2001
From: "Li, Guizi" <guizi.li@intel.com>
Date: Thu, 24 Jan 2019 10:20:48 +0800
Subject: [PATCH 1161/2345] add more comments for depthwise conv output format.

---
 tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc b/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc
index 8faa711975..b20709fc07 100644
--- a/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc
@@ -497,6 +497,16 @@ class MklConvCustomBackpropFilterOp
                                     diff_filter_mkl_shape);
         } else {
           // Depthwise Conv2d: bwd_output_dims is GOIHW format
+          //                  | TensorFlow       | MKLDNN
+          // ----------------------------------------------------------------
+          // filter_out_depth | depth_multiplier | depth_multiplier *
+          //                  |                  | group_count
+          // ----------------------------------------------------------------
+          // filter_in_depth  | in_depth         | in_depth / group_count
+          // For depthwise convolution, we have group_count == in_depth.
+          // So here G = original I, and I = 1.
+          // And the GOIHW is mkldnn format, here is try to extract the TF
+          // format, TF format is HWIO, as G = original I, so here is HWGO.
           TensorShape diff_filter_tf_shape(
               {bwd_output_dims[MklDnnFilterGroupDims::MKL_GROUP_FILTER_DIM_H],
                bwd_output_dims[MklDnnFilterGroupDims::MKL_GROUP_FILTER_DIM_W],
-- 
GitLab


From c184af04a7d51ad96e0a9083e08039f408eb2215 Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Wed, 23 Jan 2019 18:15:32 -0800
Subject: [PATCH 1162/2345] MNIST example with eager and distribution strategy.

PiperOrigin-RevId: 230640725
---
 .../contrib/distribute/python/examples/BUILD  |  11 ++
 .../python/examples/mnist_eager_multigpu.py   | 151 ++++++++++++++++++
 2 files changed, 162 insertions(+)
 create mode 100644 tensorflow/contrib/distribute/python/examples/mnist_eager_multigpu.py

diff --git a/tensorflow/contrib/distribute/python/examples/BUILD b/tensorflow/contrib/distribute/python/examples/BUILD
index 75d9da6a96..5f89df5824 100644
--- a/tensorflow/contrib/distribute/python/examples/BUILD
+++ b/tensorflow/contrib/distribute/python/examples/BUILD
@@ -45,3 +45,14 @@ py_library(
         "//third_party/py/numpy",
     ],
 )
+
+py_binary(
+    name = "mnist_eager_multigpu",
+    srcs = [
+        "mnist_eager_multigpu.py",
+    ],
+    deps = [
+        "//tensorflow:tensorflow_py",
+        "//third_party/py/numpy",
+    ],
+)
diff --git a/tensorflow/contrib/distribute/python/examples/mnist_eager_multigpu.py b/tensorflow/contrib/distribute/python/examples/mnist_eager_multigpu.py
new file mode 100644
index 0000000000..11a3b5e91a
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/examples/mnist_eager_multigpu.py
@@ -0,0 +1,151 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Run MNIST on multiple GPUs on using MirroredStrategy with eager execution.
+
+By default, runs on all available GPUs, or CPU if no GPUs are available.
+
+NOTE: Currently, this takes more time than when running MNIST in eager without
+MirroredStrategy because of a number overheads. Therefore, this is just a
+proof of concept right now and cannot be used to actually scale up training.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+import tensorflow as tf
+
+tf.flags.DEFINE_integer("num_gpus", None, "How many GPUs should we run on?"
+                        "Defaults to all available GPUs, otherwise CPU.")
+tf.flags.DEFINE_integer("batch_size", 64,
+                        "What should be the size of each batch?")
+tf.flags.DEFINE_integer("num_epochs", 10, "How many epochs to run?")
+tf.flags.DEFINE_float("learning_rate", 0.01, "Learning Rate")
+tf.flags.DEFINE_float("momentum", 0.5, "SGD momentum")
+
+FLAGS = tf.flags.FLAGS
+NUM_TRAIN_IMAGES = 60000
+NUM_TEST_IMAGES = 10000
+
+
+def create_model():
+  max_pool = tf.keras.layers.MaxPooling2D((2, 2), (2, 2), padding="same")
+  # The model consists of a sequential chain of layers, so tf.keras.Sequential
+  # (a subclass of tf.keras.Model) makes for a compact description.
+  return tf.keras.Sequential([
+      tf.keras.layers.Reshape(
+          target_shape=[28, 28, 1],
+          input_shape=(28, 28,)),
+      tf.keras.layers.Conv2D(2, 5, padding="same", activation=tf.nn.relu),
+      max_pool,
+      tf.keras.layers.Conv2D(4, 5, padding="same", activation=tf.nn.relu),
+      max_pool,
+      tf.keras.layers.Flatten(),
+      tf.keras.layers.Dense(32, activation=tf.nn.relu),
+      tf.keras.layers.Dropout(0.4),
+      tf.keras.layers.Dense(10)])
+
+
+def compute_loss(logits, labels):
+  loss = tf.reduce_sum(
+      tf.nn.sparse_softmax_cross_entropy_with_logits(
+          logits=logits, labels=labels))
+  # Scale loss by global batch size.
+  return loss * (1. / FLAGS.batch_size)
+
+
+def mnist_datasets():
+  (x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()
+  # Numpy defaults to dtype=float64; TF defaults to float32. Stick with float32.
+  x_train, x_test = x_train / np.float32(255), x_test / np.float32(255)
+  y_train, y_test = y_train.astype(np.int64), y_test.astype(np.int64)
+  # TODO(priyag): `strategy.make_numpy_iterator` can be used directly instead of
+  # converting to datasets.
+  train_dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train))
+  test_dataset = tf.data.Dataset.from_tensor_slices((x_test, y_test))
+  return train_dataset, test_dataset
+
+
+def main(unused_argv):
+  """Run a CNN model on MNIST data to demonstrate DistributedStrategies."""
+
+  tf.enable_eager_execution()
+
+  num_gpus = FLAGS.num_gpus
+  if num_gpus is None:
+    devices = None
+  elif num_gpus == 0:
+    devices = ["/device:CPU:0"]
+  else:
+    devices = ["/device:GPU:{}".format(i) for i in range(num_gpus)]
+  strategy = tf.distribute.MirroredStrategy(devices)
+
+  with strategy.scope():
+    train_ds, test_ds = mnist_datasets()
+    train_ds = train_ds.shuffle(NUM_TRAIN_IMAGES).batch(FLAGS.batch_size)
+    test_ds = test_ds.batch(FLAGS.batch_size)
+
+    model = create_model()
+    optimizer = tf.train.MomentumOptimizer(FLAGS.learning_rate, FLAGS.momentum)
+    training_loss = tf.keras.metrics.Mean("training_loss", dtype=tf.float32)
+    training_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(
+        "training_accuracy", dtype=tf.float32)
+    test_loss = tf.keras.metrics.Mean("test_loss", dtype=tf.float32)
+    test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(
+        "test_accuracy", dtype=tf.float32)
+
+    def train_step(inputs):
+      images, labels = inputs
+      with tf.GradientTape() as tape:
+        logits = model(images, training=True)
+        loss = compute_loss(logits, labels)
+      grads = tape.gradient(loss, model.variables)
+      optimizer.apply_gradients(zip(grads, model.variables))
+      training_loss.update_state(loss)
+      training_accuracy.update_state(labels, logits)
+
+    def test_step(inputs):
+      images, labels = inputs
+      logits = model(images, training=False)
+      loss = compute_loss(logits, labels)
+      test_loss.update_state(loss)
+      test_accuracy.update_state(labels, logits)
+
+    train_iterator = strategy.make_dataset_iterator(train_ds)
+    test_iterator = strategy.make_dataset_iterator(test_ds)
+    for epoch in range(0, FLAGS.num_epochs):
+      # Train
+      print("Starting epoch {}".format(epoch))
+      train_iterator.initialize()
+      for _ in range(NUM_TRAIN_IMAGES // FLAGS.batch_size):
+        strategy.experimental_run(train_step, train_iterator)
+      print("Training loss: {:0.4f}, accuracy: {:0.2f}%".format(
+          training_loss.result(), training_accuracy.result() * 100))
+      training_loss.reset_states()
+      training_accuracy.reset_states()
+
+      # Test
+      test_iterator.initialize()
+      for _ in range(NUM_TEST_IMAGES // FLAGS.batch_size):
+        strategy.experimental_run(test_step, test_iterator)
+      print("Test loss: {:0.4f}, accuracy: {:0.2f}%".format(
+          test_loss.result(), test_accuracy.result() * 100))
+      test_loss.reset_states()
+      test_accuracy.reset_states()
+
+
+if __name__ == "__main__":
+  tf.app.run()
-- 
GitLab


From 34590687eeb193661ac15d459a619e1da8b96c67 Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Wed, 23 Jan 2019 18:32:57 -0800
Subject: [PATCH 1163/2345] Rewrite some_dataset.make_*_iterator to
 tf.compat.v1.data.make_*_iterator(some_dataset).

PiperOrigin-RevId: 230642475
---
 tensorflow/tools/compatibility/ast_edits.py   | 156 +++++++++---------
 .../tools/compatibility/ast_edits_test.py     |  10 ++
 .../tools/compatibility/tf_upgrade_v2.py      | 114 ++++++-------
 .../tools/compatibility/tf_upgrade_v2_test.py |  51 +++++-
 4 files changed, 190 insertions(+), 141 deletions(-)

diff --git a/tensorflow/tools/compatibility/ast_edits.py b/tensorflow/tools/compatibility/ast_edits.py
index 17573e4f43..eabb0be4e6 100644
--- a/tensorflow/tools/compatibility/ast_edits.py
+++ b/tensorflow/tools/compatibility/ast_edits.py
@@ -34,6 +34,34 @@ FIND_OPEN = re.compile(r"^\s*(\[).*$")
 FIND_STRING_CHARS = re.compile(r"['\"]")
 
 
+INFO = "INFO"
+WARNING = "WARNING"
+ERROR = "ERROR"
+
+
+def full_name_node(name, ctx=ast.Load()):
+  """Make an Attribute or Name node for name.
+
+  Translate a qualified name into nested Attribute nodes (and a Name node).
+
+  Args:
+    name: The name to translate to a node.
+    ctx: What context this name is used in. Defaults to Load()
+
+  Returns:
+    A Name or Attribute node.
+  """
+  names = name.split(".")
+  names.reverse()
+  node = ast.Name(id=names.pop(), ctx=ast.Load())
+  while names:
+    node = ast.Attribute(value=node, attr=names.pop(), ctx=ast.Load())
+
+  # Change outermost ctx to the one given to us (inner ones should be Load).
+  node.ctx = ctx
+  return node
+
+
 class APIChangeSpec(object):
   """This class defines the transformations that need to happen.
 
@@ -63,8 +91,7 @@ class _PastaEditVisitor(ast.NodeVisitor):
 
   def __init__(self, api_change_spec):
     self._api_change_spec = api_change_spec
-    self._log = []   # Holds 3-tuples: line, col, msg.
-    self._errors = []  # Same structure as _log.
+    self._log = []   # Holds 4-tuples: severity, line, col, msg.
     self._stack = []  # Allow easy access to parents.
 
   # Overridden to maintain a stack of nodes to allow for parent access
@@ -75,55 +102,42 @@ class _PastaEditVisitor(ast.NodeVisitor):
 
   @property
   def errors(self):
-    return self._errors
+    return [log for log in self._log if log[0] == ERROR]
 
   @property
-  def log(self):
-    return self._log
+  def warnings(self):
+    return [log for log in self._log if log[0] == WARNING]
 
-  def _format_log(self, log):
-    text = ""
-    for log_entry in log:
-      text += "Line %d:%d: %s\n" % log_entry
-    return text
+  @property
+  def warnings_and_errors(self):
+    return [log for log in self._log if log[0] in (WARNING, ERROR)]
 
-  def log_text(self):
-    return self._format_log(self.log)
+  @property
+  def info(self):
+    return [log for log in self._log if log[0] == INFO]
 
-  def add_log(self, lineno, col, msg):
-    self._log.append((lineno, col, msg))
-    print("Line %d:%d: %s" % (lineno, col, msg))
+  @property
+  def log(self):
+    return self._log
 
-  def add_error(self, lineno, col, msg):
-    # All errors are also added to the regular log.
-    self.add_log(lineno, col, msg)
-    self._errors.append((lineno, col, msg))
+  def add_log(self, severity, lineno, col, msg):
+    self._log.append((severity, lineno, col, msg))
+    print("%s line %d:%d: %s" % (severity, lineno, col, msg))
 
   def add_logs(self, logs):
     """Record a log and print it.
 
-    The log should be a tuple (lineno, col_offset, msg), which will be printed
-    and then recorded. It is part of the log available in the self.log property.
+    The log should be a tuple `(severity, lineno, col_offset, msg)`, which will
+    be printed and recorded. It is part of the log available in the `self.log`
+    property.
 
     Args:
-      logs: The log to add. Must be a tuple (lineno, col_offset, msg).
+      logs: The logs to add. Must be a list of tuples
+        `(severity, lineno, col_offset, msg)`.
     """
     self._log.extend(logs)
     for log in logs:
-      print("Line %d:%d: %s" % log)
-
-  def add_errors(self, errors):
-    """Record an error and print it.
-
-    The error must be a tuple (lineno, col_offset, msg), which will be printed
-    and then recorded as both a log and an error. It is therefore part of the
-    log available in the self.log as well as the self.errors property.
-
-    Args:
-      errors: The log to add. Must be a tuple (lineno, col_offset, msg).
-    """
-    self.add_logs(errors)
-    self._errors.extend(errors)
+      print("%s line %d:%d: %s" % log)
 
   def _get_applicable_entries(self, transformer_field, full_name, name):
     """Get all list entries indexed by name that apply to full_name or name."""
@@ -158,7 +172,7 @@ class _PastaEditVisitor(ast.NodeVisitor):
   def _get_full_name(self, node):
     """Traverse an Attribute node to generate a full name, e.g., "tf.foo.bar".
 
-    This is the inverse of _full_name_node.
+    This is the inverse of `full_name_node`.
 
     Args:
       node: A Node of type Attribute.
@@ -177,37 +191,15 @@ class _PastaEditVisitor(ast.NodeVisitor):
     items.append(curr.id)
     return ".".join(reversed(items))
 
-  def _full_name_node(self, name, ctx=ast.Load()):
-    """Make an Attribute or Name node for name.
-
-    Translate a qualified name into nested Attribute nodes (and a Name node).
-
-    Args:
-      name: The name to translate to a node.
-      ctx: What context this name is used in. Defaults to Load()
-
-    Returns:
-      A Name or Attribute node.
-    """
-    names = name.split(".")
-    names.reverse()
-    node = ast.Name(id=names.pop(), ctx=ast.Load())
-    while names:
-      node = ast.Attribute(value=node, attr=names.pop(), ctx=ast.Load())
-
-    # Change outermost ctx to the one given to us (inner ones should be Load).
-    node.ctx = ctx
-    return node
-
   def _maybe_add_warning(self, node, full_name):
     """Adds an error to be printed about full_name at node."""
     function_warnings = self._api_change_spec.function_warnings
     if full_name in function_warnings:
       warning_message = function_warnings[full_name]
       warning_message = warning_message.replace("<function name>", full_name)
-      self.add_error(node.lineno, node.col_offset,
-                     "%s requires manual check: %s." % (full_name,
-                                                        warning_message))
+      self.add_log(WARNING, node.lineno, node.col_offset,
+                   "%s requires manual check. %s" % (full_name,
+                                                     warning_message))
       return True
     else:
       return False
@@ -245,9 +237,9 @@ class _PastaEditVisitor(ast.NodeVisitor):
       if kwarg in used_args or len(node.args) > arg:
         warned = True
         warning_message = warning.replace("<function name>", full_name or name)
-        self.add_error(node.lineno, node.col_offset,
-                       "%s called with %s argument requires manual check: %s." %
-                       (full_name or name, kwarg, warning_message))
+        self.add_log(WARNING, node.lineno, node.col_offset,
+                     "%s called with %s argument requires manual check: %s." %
+                     (full_name or name, kwarg, warning_message))
 
     return warned
 
@@ -255,9 +247,9 @@ class _PastaEditVisitor(ast.NodeVisitor):
     """Replace node (Attribute or Name) with a node representing full_name."""
     new_name = self._api_change_spec.symbol_renames.get(full_name, None)
     if new_name:
-      self.add_log(node.lineno, node.col_offset,
+      self.add_log(INFO, node.lineno, node.col_offset,
                    "Renamed %r to %r" % (full_name, new_name))
-      new_node = self._full_name_node(new_name, node.ctx)
+      new_node = full_name_node(new_name, node.ctx)
       ast.copy_location(new_node, node)
       pasta.ast_utils.replace_child(parent, node, new_node)
       return True
@@ -276,7 +268,7 @@ class _PastaEditVisitor(ast.NodeVisitor):
           new_node = ast.Call(node, [], [])
         pasta.ast_utils.replace_child(parent, node, new_node)
         ast.copy_location(new_node, node)
-        self.add_log(node.lineno, node.col_offset,
+        self.add_log(INFO, node.lineno, node.col_offset,
                      "Changed %r to a function call" % full_name)
         return True
     return False
@@ -296,7 +288,7 @@ class _PastaEditVisitor(ast.NodeVisitor):
         new_keywords.append(keyword)
 
       if new_keywords:
-        self.add_log(node.lineno, node.col_offset,
+        self.add_log(INFO, node.lineno, node.col_offset,
                      "Added keywords to args of function %r" % full_name)
         node.args = []
         node.keywords = new_keywords + (node.keywords or [])
@@ -320,14 +312,14 @@ class _PastaEditVisitor(ast.NodeVisitor):
         if renamed_keywords[argkey] is None:
           lineno = getattr(keyword, "lineno", node.lineno)
           col_offset = getattr(keyword, "col_offset", node.col_offset)
-          self.add_log(lineno, col_offset,
+          self.add_log(INFO, lineno, col_offset,
                        "Removed argument %s for function %s" % (
                            argkey, full_name or name))
         else:
           keyword.arg = renamed_keywords[argkey]
           lineno = getattr(keyword, "lineno", node.lineno)
           col_offset = getattr(keyword, "col_offset", node.col_offset)
-          self.add_log(lineno, col_offset,
+          self.add_log(INFO, lineno, col_offset,
                        "Renamed keyword argument for %s from %s to %s" % (
                            full_name, argkey, renamed_keywords[argkey]))
           new_keywords.append(keyword)
@@ -380,10 +372,8 @@ class _PastaEditVisitor(ast.NodeVisitor):
 
     for transformer in transformers:
       logs = []
-      errors = []
-      new_node = transformer(parent, node, full_name, name, logs, errors)
+      new_node = transformer(parent, node, full_name, name, logs)
       self.add_logs(logs)
-      self.add_errors(errors)
       if new_node:
         if new_node is not node:
           pasta.ast_utils.replace_child(parent, node, new_node)
@@ -444,29 +434,35 @@ class ASTCodeUpgrader(object):
     shutil.move(temp_file.name, out_filename)
     return ret
 
-  def _format_errors(self, errors, in_filename):
-    return ["%s:%d:%d: %s" % ((in_filename,) + error) for error in errors]
+  def format_log(self, log, in_filename):
+    log_string = "%d:%d: %s: %s" % (log[1], log[2], log[0], log[3])
+    if in_filename:
+      return in_filename + ":" + log_string
+    else:
+      return log_string
 
   def update_string_pasta(self, text, in_filename):
     """Updates a file using pasta."""
     try:
       t = pasta.parse(text)
     except (SyntaxError, ValueError, TypeError):
-      log = "Failed to parse.\n\n" + traceback.format_exc()
+      log = ["ERROR: Failed to parse.\n" + traceback.format_exc()]
       return 0, "", log, []
 
     visitor = _PastaEditVisitor(self._api_change_spec)
     visitor.visit(t)
 
-    errors = self._format_errors(visitor.errors, in_filename)
-    return 1, pasta.dump(t), visitor.log_text(), errors
+    logs = [self.format_log(log, None) for log in visitor.log]
+    errors = [self.format_log(error, in_filename)
+              for error in visitor.warnings_and_errors]
+    return 1, pasta.dump(t), logs, errors
 
   def _format_log(self, log, in_filename, out_filename):
     text = "-" * 80 + "\n"
     text += "Processing file %r\n outputting to %r\n" % (in_filename,
                                                          out_filename)
     text += "-" * 80 + "\n\n"
-    text += log
+    text += "\n".join(log)
     text += "-" * 80 + "\n\n"
     return text
 
diff --git a/tensorflow/tools/compatibility/ast_edits_test.py b/tensorflow/tools/compatibility/ast_edits_test.py
index ff8b8d6982..7049479152 100644
--- a/tensorflow/tools/compatibility/ast_edits_test.py
+++ b/tensorflow/tools/compatibility/ast_edits_test.py
@@ -39,7 +39,10 @@ following new APIs:
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+
+import ast
 import six
+
 from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test as test_lib
 from tensorflow.tools.compatibility import ast_edits
@@ -429,6 +432,13 @@ class TestAstEdits(test_util.TensorFlowTestCase):
       (_, report, _), _ = self._upgrade(FooWarningSpec(), text)
       self.assertNotIn("not good", report)
 
+  def testFullNameNode(self):
+    t = ast_edits.full_name_node("a.b.c")
+    self.assertEquals(
+        ast.dump(t),
+        "Attribute(value=Attribute(value=Name(id='a', ctx=Load()), attr='b', "
+        "ctx=Load()), attr='c', ctx=Load())"
+    )
 
 if __name__ == "__main__":
   test_lib.main()
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 9a761c3497..b83d7d9749 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -777,6 +777,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     # - a replacement for node, if the whole call node was replaced. The caller
     #   will take care of changing parent.
     self.function_transformers = {
+        "*.make_initializable_iterator": self._iterator_transformer,
+        "*.make_one_shot_iterator": self._iterator_transformer,
         "tf.nn.dropout": self._dropout_transformer,
         "tf.batch_gather": self._batch_gather_transformer,
         "tf.to_bfloat16": self._cast_transformer,
@@ -796,8 +798,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     }
 
     decay_function_comment = (
-        "WARNING: <function name> has been changed to return a callable instead"
-        " of a tensor when graph building, but its functionality remains "
+        "<function name> has been changed to return a callable instead "
+        "of a tensor when graph building, but its functionality remains "
         "unchanged during eager execution (returns a callable like "
         "before). The converter cannot detect and fix this reliably, so "
         "this usage has been converted to compat.v1 (even though it may already"
@@ -806,26 +808,26 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
 
     # TODO(b/118888586): add default value change to update script.
     default_loss_reduction_changed = (
-        "WARNING: default value of loss_reduction has been changed to "
+        "default value of loss_reduction has been changed to "
         "SUM_OVER_BATCH_SIZE.\n"
     )
 
     assert_return_type_comment = (
-        "WARNING: assert_* functions have been changed to return None, the "
+        "assert_* functions have been changed to return None, the "
         "data argument has been removed, and arguments have been reordered."
         "\nThe calls have been converted to compat.v1 for safety (even though "
         " they may already have been correct)."
     )
 
     assert_rank_comment = (
-        "WARNING: assert_rank_* functions have been changed to return None, and"
+        "assert_rank_* functions have been changed to return None, and"
         " the data and summarize arguments have been removed."
         "\nThe calls have been converted to compat.v1 for safety (even though "
         " they may already have been correct)."
     )
 
     tf_01s_like_no_optimize_comment = (
-        "WARNING: tf.zeros_like and tf.ones_like no longer have the optimize "
+        "tf.zeros_like and tf.ones_like no longer have the optimize "
         "argument in TF 2.0 or after (also, `tensor' argument is renamed to "
         "`input')."
         "\nThe calls have been converted to compat.v1 for safety (even though "
@@ -833,11 +835,11 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     )
 
     deprecate_partition_strategy_comment = (
-        "WARNING: `partition_strategy` has been removed from `%s` "
+        "`partition_strategy` has been removed from `%s` "
         " The 'div' strategy is used by default.")
 
     initializers_no_dtype_comment = (
-        "WARNING: tf.initializers and tf.keras.initializers no longer have the "
+        "tf.initializers and tf.keras.initializers no longer have the "
         "dtype argument in the constructor or partition_info argument in the "
         "call method in TF 2.0 and after. The only API symbols are now "
         "tf.keras.initializers.* or tf.initializers.*."
@@ -845,18 +847,18 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "they may already have been correct).")
 
     uniform_unit_scaling_initializer_comment = (
-        "WARNING: uniform_unit_scaling_initializer has been removed. Please use"
+        "uniform_unit_scaling_initializer has been removed. Please use"
         " tf.initializers.variance_scaling instead with distribution=uniform "
         "to get equivalent behaviour.")
 
     metrics_comment = (
-        "WARNING: tf.metrics have been converted to object oriented versions in"
+        "tf.metrics have been converted to object oriented versions in"
         " TF 2.0 and after. The metric function calls have been converted to "
         "compat.v1 for backward compatibility. Please update these calls to "
         "the TF 2.0 versions.")
 
     losses_comment = (
-        "WARNING: tf.losses have been converted to object oriented versions in"
+        "tf.losses have been converted to object oriented versions in"
         " TF 2.0 and after. The loss function calls have been converted to "
         "compat.v1 for backward compatibility. Please update these calls to "
         "the TF 2.0 versions.")
@@ -870,25 +872,6 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "only effects core estimator. If you are using "
         "tf.contrib.learn.Estimator, please switch to using core estimator.")
 
-    make_initializable_iterator_deprecation = (
-        "(Manual edit required) The "
-        "`tf.data.Dataset.make_initializable_iterator()` method has been "
-        "removed. If you are using the Estimator API, you can return a dataset "
-        "directly from your input functions without creating an iterator. "
-        "As a last resort, please replace calls to that method on `dataset` "
-        "with a call to "
-        "`tf.compat.v1.data.make_initializable_iterator(dataset)`.")
-
-    make_one_shot_iterator_deprecation = (
-        "(Manual edit required) The "
-        "`tf.data.Dataset.make_one_shot_iterator()` method has been "
-        "removed. If you are using eager execution, you can iterate over "
-        "`dataset` using a Python `for` loop. If you are using the Estimator "
-        "API, you can return a dataset directly from your input functions "
-        "without creating an iterator. As a last resort, please replace calls "
-        "to that method on `dataset` with a call to "
-        "`tf.compat.v1.data.make_one_shot_iterator(dataset)`.")
-
     # Function warnings. <function name> placeholder inside warnings will be
     # replaced by function name.
     # You can use *. to add items which do not check the FQN, and apply to e.g.,
@@ -896,10 +879,6 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     self.function_warnings = {
         "*.export_savedmodel":
             export_saved_model_renamed,
-        "*.make_initializable_iterator":
-            make_initializable_iterator_deprecation,
-        "*.make_one_shot_iterator":
-            make_one_shot_iterator_deprecation,
         "tf.assert_equal":
             assert_return_type_comment,
         "tf.assert_none_equal":
@@ -1263,7 +1242,32 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     }
 
   @staticmethod
-  def _dropout_transformer(parent, node, full_name, name, logs, errors):
+  def _iterator_transformer(parent, node, full_name, name, logs):
+    # First, check that node.func.value is not already something we like
+    # (tf.compat.v1.data), or something which is handled in the rename
+    # (tf.data). This transformer only handles the method call to function call
+    # conversion.
+    if full_name and (full_name.startswith("tf.compat.v1.data") or
+                      full_name.startswith("tf.data")):
+      return
+
+    # This should never happen, since we're only called for Attribute nodes.
+    if not isinstance(node.func, ast.Attribute):
+      return
+
+    # Transform from x.f(y) to tf.compat.v1.data.f(x, y)
+    # Fortunately, node.func.value should already have valid position info
+    node.args = [node.func.value] + node.args
+    node.func.value = ast_edits.full_name_node("tf.compat.v1.data")
+
+    logs.append((ast_edits.WARNING, node.lineno, node.col_offset,
+                 "Changing dataset.%s() to tf.compat.v1.data.%s(dataset). "
+                 "Please check this transformation.\n" % (name, name)))
+
+    return node
+
+  @staticmethod
+  def _dropout_transformer(parent, node, full_name, name, logs):
     def _replace_keep_prob_node(parent, old_value):
       """Replaces old_value with 1-(old_value)."""
       one = ast.Num(n=1)
@@ -1282,32 +1286,28 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     # Check if we have a keep_prob keyword arg
     for keep_prob in node.keywords:
       if keep_prob.arg == "keep_prob":
-        logs.append((node.lineno, node.col_offset,
-                     "Changing keep_prob arg of tf.nn.dropout to rate, and "
-                     "recomputing value. Please check this transformation.\n"))
+        logs.append((ast_edits.INFO, node.lineno, node.col_offset,
+                     "Changing keep_prob arg of tf.nn.dropout to rate\n"))
         keep_prob.arg = "rate"
         _replace_keep_prob_node(keep_prob, keep_prob.value)
         return node
 
     # Maybe it was a positional arg
     if len(node.args) < 2:
-      errors.append((node.lineno, node.col_offset,
-                     "ERROR: tf.nn.dropout called without arguments, so "
-                     "automatic fix was disabled. tf.nn.dropout has changed "
-                     "the semantics of the second argument."))
+      logs.append((ast_edits.ERROR, node.lineno, node.col_offset,
+                   "tf.nn.dropout called without arguments, so "
+                   "automatic fix was disabled. tf.nn.dropout has changed "
+                   "the semantics of the second argument."))
     else:
       _replace_keep_prob_node(node, node.args[1])
-      logs.append((node.lineno, node.col_offset,
+      logs.append((ast_edits.INFO, node.lineno, node.col_offset,
                    "Changing keep_prob arg of tf.nn.dropout to rate, and "
                    "recomputing value.\n"))
-      errors.append((node.lineno, node.col_offset,
-                     "WARNING: tf.nn.dropout has changed the semantics of the "
-                     "second argument. Please check the applied transformation."
-                    ))
+
       return node
 
   @staticmethod
-  def _cast_transformer(parent, node, full_name, name, logs, errors):
+  def _cast_transformer(parent, node, full_name, name, logs):
     """Transforms to_int and to_float to cast(..., dtype=...)."""
 
     # Find out the dtype to cast to from the function name
@@ -1340,14 +1340,14 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
       assert isinstance(node.func, ast.Name)
       node.func.id = "cast"
 
-    logs.append((node.lineno, node.col_offset,
+    logs.append((ast_edits.INFO, node.lineno, node.col_offset,
                  "Changed %s call to tf.cast(..., dtype=tf.%s)." % (full_name,
                                                                     dtype_str)))
     return node
 
   @staticmethod
   def _softmax_cross_entropy_with_logits_transformer(
-      parent, node, full_name, name, logs, errors):
+      parent, node, full_name, name, logs):
     def _wrap_label(parent, old_value):
       """Wrap labels with tf.stop_gradient."""
       if six.PY3:
@@ -1366,7 +1366,7 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     # Check if we have a labels keyword arg
     for karg in node.keywords:
       if karg.arg == "labels":
-        logs.append((node.lineno, node.col_offset,
+        logs.append((ast_edits.INFO, node.lineno, node.col_offset,
                      "Changing labels arg of "
                      "tf.nn.softmax_cross_entropy_with_logits to "
                      "tf.stop_gradient(labels). Please check this "
@@ -1376,11 +1376,11 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     return node
 
   @staticmethod
-  def _batch_gather_transformer(parent, node, full_name, name, logs, errors):
+  def _batch_gather_transformer(parent, node, full_name, name, logs):
     # Check if the call already has a batch_dims argument
     if any([kw.arg == "batch_dims" for kw in node.keywords]):
-      logs.append((node.lineno, node.col_offset, "tf.batch_gather already has "
-                   "batch_dims argument. Neat."))
+      logs.append((ast_edits.INFO, node.lineno, node.col_offset,
+                   "tf.batch_gather already has batch_dims argument. Neat."))
       return None
 
     minus_one = ast.Num(n=-1)
@@ -1388,12 +1388,12 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     minus_one.col_offset = 0
     new_arg = ast.keyword("batch_dims", minus_one)
     node.keywords.append(new_arg)
-    logs.append((node.lineno, node.col_offset,
+    logs.append((ast_edits.INFO, node.lineno, node.col_offset,
                  "Added keyword argument batch_dims=-1 to tf.batch_gather."))
     return node
 
   @staticmethod
-  def _image_resize_transformer(parent, node, full_name, name, logs, errors):
+  def _image_resize_transformer(parent, node, full_name, name, logs):
     """Transforms image.resize_* to image.resize(..., method=*, ...)."""
 
     resize_method = name[7:].upper()
@@ -1430,7 +1430,7 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
       assert isinstance(node.func, ast.Name)
       node.func.id = "resize"
 
-    logs.append((node.lineno, node.col_offset,
+    logs.append((ast_edits.INFO, node.lineno, node.col_offset,
                  "Changed %s call to tf.image.resize(..., "
                  "method=tf.image.ResizeMethod.%s)." % (full_name,
                                                         resize_method)))
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index f3ecf1c269..8ff5d01ae6 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -431,9 +431,9 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
       text = ns + "(a, b)"
       _, report, errors, new_text = self._upgrade(text)
       self.assertEqual("tf.compat.v1.metrics." + m + "(a, b)", new_text)
-      self.assertIn("test.py:1:0: %s requires manual check" % ns, errors[0])
+      self.assertIn("%s requires manual check" % ns, errors[0])
       self.assertIn(
-          "WARNING: tf.metrics have been converted to object oriented"
+          "tf.metrics have been converted to object oriented"
           " versions in TF 2.0 and after. The metric function calls have been "
           "converted to compat.v1 for backward compatibility. Please update "
           "these calls to the TF 2.0 versions.", report)
@@ -462,9 +462,9 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
       text = ns + "(a, b)"
       _, report, errors, new_text = self._upgrade(text)
       self.assertEqual("tf.compat.v1.losses." + l + "(a, b)", new_text)
-      self.assertIn("test.py:1:0: %s requires manual check" % ns, errors[0])
+      self.assertIn("%s requires manual check" % ns, errors[0])
       self.assertIn(
-          "WARNING: tf.losses have been converted to object oriented"
+          "tf.losses have been converted to object oriented"
           " versions in TF 2.0 and after. The loss function calls have been "
           "converted to compat.v1 for backward compatibility. Please update "
           "these calls to the TF 2.0 versions.", report)
@@ -909,6 +909,49 @@ tf.print('abc')
     _, unused_report, unused_errors, new_text = self._upgrade(text)
     self.assertIn(new_text, [expected_text1, expected_text2])
 
+  def testIterators(self):
+    for (text, expected) in [
+        ("(expr + yielding(data)).make_one_shot_iterator()",
+         "tf.compat.v1.data.make_one_shot_iterator((expr + yielding(data)))"),
+        ("dataset.make_one_shot_iterator()",
+         "tf.compat.v1.data.make_one_shot_iterator(dataset)"),
+        ("dataset.make_one_shot_iterator(shared_name=foo)",
+         "tf.compat.v1.data.make_one_shot_iterator(dataset, shared_name=foo)"),
+        ("dataset.make_one_shot_iterator(x, y, z)",
+         "tf.compat.v1.data.make_one_shot_iterator(dataset, x, y, z)"),
+        ("dataset.make_initializable_iterator()",
+         "tf.compat.v1.data.make_initializable_iterator(dataset)"),
+        ("ds.make_initializable_iterator(shared_name=foo)",
+         "tf.compat.v1.data.make_initializable_iterator(ds, shared_name=foo)"),
+        ("dataset.make_initializable_iterator(x, y, z)",
+         "tf.compat.v1.data.make_initializable_iterator(dataset, x, y, z)"),
+        ("tf.data.make_one_shot_iterator(dataset)",
+         "tf.compat.v1.data.make_one_shot_iterator(dataset)"),
+        ("tf.data.make_one_shot_iterator(dataset, shared_name=foo)",
+         "tf.compat.v1.data.make_one_shot_iterator(dataset, shared_name=foo)"),
+        ("tf.data.make_one_shot_iterator(dataset, x, y, z)",
+         "tf.compat.v1.data.make_one_shot_iterator(dataset, x, y, z)"),
+        ("tf.data.make_initializable_iterator(dataset)",
+         "tf.compat.v1.data.make_initializable_iterator(dataset)"),
+        ("tf.data.make_initializable_iterator(ds, shared_name=foo)",
+         "tf.compat.v1.data.make_initializable_iterator(ds, shared_name=foo)"),
+        ("tf.data.make_initializable_iterator(dataset, x, y, z)",
+         "tf.compat.v1.data.make_initializable_iterator(dataset, x, y, z)"),
+        ("tf.compat.v1.data.make_one_shot_iterator(dataset)",
+         "tf.compat.v1.data.make_one_shot_iterator(dataset)"),
+        ("tf.compat.v1.data.make_one_shot_iterator(dataset, shared_name=foo)",
+         "tf.compat.v1.data.make_one_shot_iterator(dataset, shared_name=foo)"),
+        ("tf.compat.v1.data.make_one_shot_iterator(dataset, x, y, z)",
+         "tf.compat.v1.data.make_one_shot_iterator(dataset, x, y, z)"),
+        ("tf.compat.v1.data.make_initializable_iterator(dataset)",
+         "tf.compat.v1.data.make_initializable_iterator(dataset)"),
+        ("tf.compat.v1.data.make_initializable_iterator(ds, shared_name=foo)",
+         "tf.compat.v1.data.make_initializable_iterator(ds, shared_name=foo)"),
+        ("tf.compat.v1.data.make_initializable_iterator(dataset, x, y, z)",
+         "tf.compat.v1.data.make_initializable_iterator(dataset, x, y, z)")]:
+      _, unused_report, unused_errors, actual = self._upgrade(text)
+      self.assertEqual(actual, expected)
+
   def testCast(self):
     for (name, dtype) in [("int32", "int32"),
                           ("int64", "int64"),
-- 
GitLab


From 41a30971936e207c52f2317ee3c48da5394a0c37 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 23 Jan 2019 18:46:36 -0800
Subject: [PATCH 1164/2345] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 230643897

---
 tensorflow/go/op/wrappers.go | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 8e2c98fb07..666ebe33df 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -16317,7 +16317,7 @@ func TensorListStackNumElements(value int64) TensorListStackAttr {
 // tensor: the gathered result
 // num_elements: optional. If not -1, the number of elements in the list.
 //
-func TensorListStack(scope *Scope, input_handle tf.Output, element_shape tf.Output, element_dtype tf.DataType, optional ...TensorListStackAttr) (tensor tf.Output) {
+func TensorListStack(scope *Scope, input_handle tf.Output, element_dtype tf.DataType, optional ...TensorListStackAttr) (tensor tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -16328,7 +16328,7 @@ func TensorListStack(scope *Scope, input_handle tf.Output, element_shape tf.Outp
 	opspec := tf.OpSpec{
 		Type: "TensorListStack",
 		Input: []tf.Input{
-			input_handle, element_shape,
+			input_handle,
 		},
 		Attrs: attrs,
 	}
@@ -17359,7 +17359,7 @@ func StringToHashBucketFast(scope *Scope, input tf.Output, num_buckets int64) (o
 // tensor: the withdrawn last element of the list
 // element_dtype: the type of elements in the list
 // element_shape: the shape of the output tensor
-func TensorListPopBack(scope *Scope, input_handle tf.Output, element_shape tf.Output, element_dtype tf.DataType) (output_handle tf.Output, tensor tf.Output) {
+func TensorListPopBack(scope *Scope, input_handle tf.Output, element_dtype tf.DataType) (output_handle tf.Output, tensor tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -17367,7 +17367,7 @@ func TensorListPopBack(scope *Scope, input_handle tf.Output, element_shape tf.Ou
 	opspec := tf.OpSpec{
 		Type: "TensorListPopBack",
 		Input: []tf.Input{
-			input_handle, element_shape,
+			input_handle,
 		},
 		Attrs: attrs,
 	}
@@ -28621,7 +28621,7 @@ func TensorListElementShape(scope *Scope, input_handle tf.Output, shape_type tf.
 // item: the element at that position
 //
 //
-func TensorListGetItem(scope *Scope, input_handle tf.Output, index tf.Output, element_shape tf.Output, element_dtype tf.DataType) (item tf.Output) {
+func TensorListGetItem(scope *Scope, input_handle tf.Output, index tf.Output, element_dtype tf.DataType) (item tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -28629,7 +28629,7 @@ func TensorListGetItem(scope *Scope, input_handle tf.Output, index tf.Output, el
 	opspec := tf.OpSpec{
 		Type: "TensorListGetItem",
 		Input: []tf.Input{
-			input_handle, index, element_shape,
+			input_handle, index,
 		},
 		Attrs: attrs,
 	}
@@ -28781,7 +28781,7 @@ func TensorListSetItem(scope *Scope, input_handle tf.Output, index tf.Output, it
 // input_handle: The input tensor list.
 // indices: The indices used to index into the list.
 // values: The tensor.
-func TensorListGather(scope *Scope, input_handle tf.Output, indices tf.Output, element_shape tf.Output, element_dtype tf.DataType) (values tf.Output) {
+func TensorListGather(scope *Scope, input_handle tf.Output, indices tf.Output, element_dtype tf.DataType) (values tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -28789,7 +28789,7 @@ func TensorListGather(scope *Scope, input_handle tf.Output, indices tf.Output, e
 	opspec := tf.OpSpec{
 		Type: "TensorListGather",
 		Input: []tf.Input{
-			input_handle, indices, element_shape,
+			input_handle, indices,
 		},
 		Attrs: attrs,
 	}
-- 
GitLab


From 63b38ce4a893bbf1c2026e2d5aeb2da26cabf136 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 23 Jan 2019 19:58:27 -0800
Subject: [PATCH 1165/2345] Support bidirectional sequence rnn for tflite.

PiperOrigin-RevId: 230650050
---
 .../lite/experimental/examples/lstm/BUILD     |  22 +
 .../lstm/bidirectional_sequence_rnn_test.py   | 187 +++++++
 tensorflow/lite/toco/BUILD                    |   2 +-
 .../graph_transformations.h                   |   1 +
 .../group_bidirectional_sequence_lstm.cc      | 346 ------------
 .../group_bidirectional_sequence_ops.cc       | 495 ++++++++++++++++++
 .../propagate_array_data_types.cc             |   6 +
 .../propagate_fixed_sizes.cc                  |  56 ++
 tensorflow/lite/toco/model.h                  |   9 +-
 tensorflow/lite/toco/tflite/operator.cc       |  44 ++
 tensorflow/lite/toco/toco_tooling.cc          |   5 +-
 tensorflow/lite/toco/tooling_util.cc          |   1 +
 12 files changed, 824 insertions(+), 350 deletions(-)
 create mode 100644 tensorflow/lite/experimental/examples/lstm/bidirectional_sequence_rnn_test.py
 delete mode 100644 tensorflow/lite/toco/graph_transformations/group_bidirectional_sequence_lstm.cc
 create mode 100644 tensorflow/lite/toco/graph_transformations/group_bidirectional_sequence_ops.cc

diff --git a/tensorflow/lite/experimental/examples/lstm/BUILD b/tensorflow/lite/experimental/examples/lstm/BUILD
index dd44c1b67b..a4950d2dad 100644
--- a/tensorflow/lite/experimental/examples/lstm/BUILD
+++ b/tensorflow/lite/experimental/examples/lstm/BUILD
@@ -95,3 +95,25 @@ py_test(
         "@six_archive//:six",
     ],
 )
+
+py_test(
+    name = "bidirectional_sequence_rnn_test",
+    size = "large",
+    srcs = ["bidirectional_sequence_rnn_test.py"],
+    srcs_version = "PY2AND3",
+    tags = [
+        "no_oss",
+        "no_pip",
+    ],
+    deps = [
+        ":tflite_rnn",
+        "//tensorflow:tensorflow_py",
+        "//tensorflow/examples/tutorials/mnist:input_data",
+        "//tensorflow/lite/python:lite",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:platform",
+        "//tensorflow/python/tools:optimize_for_inference",
+        "//third_party/py/numpy",
+        "@six_archive//:six",
+    ],
+)
diff --git a/tensorflow/lite/experimental/examples/lstm/bidirectional_sequence_rnn_test.py b/tensorflow/lite/experimental/examples/lstm/bidirectional_sequence_rnn_test.py
new file mode 100644
index 0000000000..d049c7834e
--- /dev/null
+++ b/tensorflow/lite/experimental/examples/lstm/bidirectional_sequence_rnn_test.py
@@ -0,0 +1,187 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+import tempfile
+import numpy as np
+import tensorflow as tf
+
+from tensorflow import flags
+
+from tensorflow.examples.tutorials.mnist import input_data
+from tensorflow.lite.experimental.examples.lstm.tflite_rnn import TfLiteRNNCell
+from tensorflow.lite.python.op_hint import convert_op_hints_to_stubs
+from tensorflow.lite.python.op_hint import find_all_hinted_output_nodes
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import test
+from tensorflow.python.tools import optimize_for_inference_lib
+
+FLAGS = flags.FLAGS
+
+# Number of steps to train model.
+TRAIN_STEPS = 1
+
+CONFIG = tf.ConfigProto(device_count={"GPU": 0})
+
+
+class BidirectionalSequenceRnnTest(test_util.TensorFlowTestCase):
+
+  def __init__(self, *args, **kwargs):
+    super(BidirectionalSequenceRnnTest, self).__init__(*args, **kwargs)
+    # Define constants
+    # Unrolled through 28 time steps
+    self.time_steps = 28
+    # Rows of 28 pixels
+    self.n_input = 28
+    # Learning rate for Adam optimizer
+    self.learning_rate = 0.001
+    # MNIST is meant to be classified in 10 classes(0-9).
+    self.n_classes = 10
+    # Batch size
+    self.batch_size = 16
+    # Rnn Units.
+    self.num_units = 16
+
+  def setUp(self):
+    super(BidirectionalSequenceRnnTest, self).setUp()
+    # Import MNIST dataset
+    data_dir = tempfile.mkdtemp(dir=FLAGS.test_tmpdir)
+    self.mnist = input_data.read_data_sets(data_dir, one_hot=True)
+
+  def buildRnnLayer(self):
+    return tf.nn.rnn_cell.MultiRNNCell([
+        TfLiteRNNCell(self.num_units, name="rnn1"),
+        TfLiteRNNCell(self.num_units, name="rnn2")
+    ])
+
+  def buildModel(self, fw_rnn_layer, bw_rnn_layer):
+    # Weights and biases for output softmax layer.
+    out_weights = tf.Variable(
+        tf.random_normal([self.num_units * 2, self.n_classes]))
+    out_bias = tf.Variable(tf.random_normal([self.n_classes]))
+
+    # input image placeholder
+    x = tf.placeholder(
+        "float", [None, self.time_steps, self.n_input], name="INPUT_IMAGE")
+
+    rnn_input = tf.unstack(x, self.time_steps, 1)
+    outputs, _, _ = tf.nn.static_bidirectional_rnn(
+        fw_rnn_layer, bw_rnn_layer, rnn_input, dtype="float32")
+
+    # Compute logits by multiplying outputs[-1] of shape [batch_size,num_units]
+    # by the softmax layer's out_weight of shape [num_units,n_classes]
+    # plus out_bias
+    prediction = tf.matmul(outputs[-1], out_weights) + out_bias
+    output_class = tf.nn.softmax(prediction, name="OUTPUT_CLASS")
+
+    return x, prediction, output_class
+
+  def trainModel(self, x, prediction, output_class, sess):
+    # input label placeholder
+    y = tf.placeholder("float", [None, self.n_classes])
+    # Loss function
+    loss = tf.reduce_mean(
+        tf.nn.softmax_cross_entropy_with_logits(logits=prediction, labels=y))
+    # Optimization
+    opt = tf.train.AdamOptimizer(
+        learning_rate=self.learning_rate).minimize(loss)
+
+    # Initialize variables
+    init = tf.global_variables_initializer()
+    sess.run(init)
+    for _ in range(TRAIN_STEPS):
+      batch_x, batch_y = self.mnist.train.next_batch(
+          batch_size=self.batch_size, shuffle=False)
+
+      batch_x = batch_x.reshape((self.batch_size, self.time_steps,
+                                 self.n_input))
+      sess.run(opt, feed_dict={x: batch_x, y: batch_y})
+
+  def saveAndRestoreModel(self, fw_rnn_layer, bw_rnn_layer, sess, saver):
+    model_dir = tempfile.mkdtemp(dir=FLAGS.test_tmpdir)
+    saver.save(sess, model_dir)
+
+    # Reset the graph.
+    tf.reset_default_graph()
+    x, prediction, output_class = self.buildModel(fw_rnn_layer, bw_rnn_layer)
+
+    new_sess = tf.Session(config=CONFIG)
+    saver = tf.train.Saver()
+    saver.restore(new_sess, model_dir)
+    return x, prediction, output_class, new_sess
+
+  def getInferenceResult(self, x, output_class, sess):
+    b1, _ = self.mnist.train.next_batch(batch_size=1)
+    sample_input = np.reshape(b1, (1, self.time_steps, self.n_input))
+
+    expected_output = sess.run(output_class, feed_dict={x: sample_input})
+    # It is important to keep all the ophint output nodes.
+    hinted_outputs_nodes = find_all_hinted_output_nodes(sess)
+    hinted_outputs_nodes.append(output_class.op.name)
+    frozen_graph = tf.graph_util.convert_variables_to_constants(
+        sess, sess.graph_def, hinted_outputs_nodes)
+    return sample_input, expected_output, frozen_graph
+
+  def tfliteInvoke(self, graph, test_inputs, outputs):
+    tf.reset_default_graph()
+    # Turn the input into placeholder of shape 1
+    tflite_input = tf.placeholder(
+        "float", [1, self.time_steps, self.n_input], name="INPUT_IMAGE_LITE")
+    tf.import_graph_def(graph, name="", input_map={"INPUT_IMAGE": tflite_input})
+    with tf.Session() as sess:
+      curr = sess.graph_def
+      curr = convert_op_hints_to_stubs(graph_def=curr)
+
+    curr = optimize_for_inference_lib.optimize_for_inference(
+        curr, ["INPUT_IMAGE_LITE"], ["OUTPUT_CLASS"],
+        [tf.float32.as_datatype_enum])
+
+    tflite = tf.lite.toco_convert(
+        curr, [tflite_input], [outputs], allow_custom_ops=False)
+
+    interpreter = tf.lite.Interpreter(model_content=tflite)
+
+    interpreter.allocate_tensors()
+
+    input_index = interpreter.get_input_details()[0]["index"]
+    interpreter.set_tensor(input_index, test_inputs)
+    interpreter.invoke()
+    output_index = interpreter.get_output_details()[0]["index"]
+    result = interpreter.get_tensor(output_index)
+    # Reset all variables so it will not pollute other inferences.
+    interpreter.reset_all_variables()
+    return result
+
+  def testStaticRnnMultiRnnCell(self):
+    sess = tf.Session(config=CONFIG)
+
+    x, prediction, output_class = self.buildModel(self.buildRnnLayer(),
+                                                  self.buildRnnLayer())
+    self.trainModel(x, prediction, output_class, sess)
+
+    saver = tf.train.Saver()
+    x, prediction, output_class, new_sess = self.saveAndRestoreModel(
+        self.buildRnnLayer(), self.buildRnnLayer(), sess, saver)
+
+    test_inputs, expected_output, frozen_graph = self.getInferenceResult(
+        x, output_class, new_sess)
+
+    result = self.tfliteInvoke(frozen_graph, test_inputs, output_class)
+    self.assertTrue(np.allclose(expected_output, result, rtol=1e-6, atol=1e-2))
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/lite/toco/BUILD b/tensorflow/lite/toco/BUILD
index 3b5b08fe1a..47880c1f0b 100644
--- a/tensorflow/lite/toco/BUILD
+++ b/tensorflow/lite/toco/BUILD
@@ -192,7 +192,7 @@ cc_library(
         "graph_transformations/fuse_binary_into_preceding_affine.cc",
         "graph_transformations/fuse_broadcast_into_following_binary.cc",
         "graph_transformations/graph_transformations.cc",
-        "graph_transformations/group_bidirectional_sequence_lstm.cc",
+        "graph_transformations/group_bidirectional_sequence_ops.cc",
         "graph_transformations/hardcode_min_max.cc",
         "graph_transformations/identify_dilated_conv.cc",
         "graph_transformations/identify_l2_normalization.cc",
diff --git a/tensorflow/lite/toco/graph_transformations/graph_transformations.h b/tensorflow/lite/toco/graph_transformations/graph_transformations.h
index 3554dda516..4008bbdb4d 100644
--- a/tensorflow/lite/toco/graph_transformations/graph_transformations.h
+++ b/tensorflow/lite/toco/graph_transformations/graph_transformations.h
@@ -128,6 +128,7 @@ DECLARE_GRAPH_TRANSFORMATION(FuseBinaryIntoFollowingAffine)
 DECLARE_GRAPH_TRANSFORMATION(FuseBinaryIntoPrecedingAffine)
 DECLARE_GRAPH_TRANSFORMATION(FuseBroadcastIntoFollowingBinary)
 DECLARE_GRAPH_TRANSFORMATION(GroupBidirectionalSequenceLstm)
+DECLARE_GRAPH_TRANSFORMATION(GroupBidirectionalSequenceRnn)
 DECLARE_GRAPH_TRANSFORMATION(IdentifyL2Normalization)
 DECLARE_GRAPH_TRANSFORMATION(IdentifyL2Pool)
 DECLARE_GRAPH_TRANSFORMATION(IdentifyLstmCell)
diff --git a/tensorflow/lite/toco/graph_transformations/group_bidirectional_sequence_lstm.cc b/tensorflow/lite/toco/graph_transformations/group_bidirectional_sequence_lstm.cc
deleted file mode 100644
index 48d1fc0138..0000000000
--- a/tensorflow/lite/toco/graph_transformations/group_bidirectional_sequence_lstm.cc
+++ /dev/null
@@ -1,346 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include <cstdio>
-#include <iterator>
-#include <memory>
-#include <stack>
-#include <string>
-#include <vector>
-
-#include "tensorflow/core/lib/core/status.h"
-#include "tensorflow/lite/toco/graph_transformations/graph_transformations.h"
-#include "tensorflow/lite/toco/model.h"
-#include "tensorflow/lite/toco/tooling_util.h"
-
-namespace toco {
-namespace {
-
-std::vector<std::unique_ptr<Operator>>::iterator FindOperator(
-    Model* model, const Operator& op) {
-  return std::find_if(
-      model->operators.begin(), model->operators.end(),
-      [&op](const std::unique_ptr<Operator>& ptr) { return ptr.get() == &op; });
-}
-
-bool MatchTwoUnpackOps(const Operator& op, const Model& model,
-                       Operator** fw_output, Operator** bw_output) {
-  if (op.inputs.size() != 2) {
-    return false;
-  }
-
-  *fw_output = GetOpWithOutput(model, op.inputs[0]);
-  *bw_output = GetOpWithOutput(model, op.inputs[1]);
-  if (*fw_output == nullptr || *bw_output == nullptr) {
-    return false;
-  }
-
-  if ((*fw_output)->type != OperatorType::kUnpack ||
-      (*bw_output)->type != OperatorType::kUnpack) {
-    return false;
-  }
-
-  // TODO(renjieliu): Check the shapes are matching.
-
-  return true;
-}
-
-bool FindUnidirectionalSequenceLstmOp(const Model& model,
-                                      const Operator& output_op,
-                                      std::stack<Operator*>* lstm_ops,
-                                      Operator** input_op) {
-  Operator* op_it = nullptr;
-  op_it = GetOpWithOutput(model, output_op.inputs[0]);
-  if (op_it == nullptr) {
-    return false;
-  }
-
-  while (op_it->type == OperatorType::kUnidirectionalSequenceLstm) {
-    lstm_ops->push(op_it);
-    // Check the first input of the unidirectional squence lstm op.
-    op_it = GetOpWithOutput(model, op_it->inputs[0]);
-    if (op_it == nullptr) {
-      return false;
-    }
-  }
-
-  *input_op = op_it;
-  return true;
-}
-
-bool ConstructBidirectionalSequenceLstmOp(
-    const Operator& fw_lstm_op, const Operator& bw_lstm_op, Model* model,
-    BidirectionalSequenceLstmOperator** bi_op) {
-  // TODO(renjieliu): Check the shapes & configurations are equal.
-  constexpr int kBidirectionalSequenceLstmInputsCount = 47;
-  constexpr int kFwLstmInputsStartIndex = 1;
-  constexpr int kBwLstmInputsStartIndex = 18;
-  constexpr int kFwInputActivationStartIndex = 35;
-  constexpr int kBwInputActivationStartIndex = 37;
-  constexpr int kAuxInputStartIndex = 39;
-  (*bi_op)->inputs.reserve(kBidirectionalSequenceLstmInputsCount);
-  const string& input_array_name =
-      AvailableArrayName(*model, "bidirectional_sequence_lstm_input_0");
-  model->GetOrCreateArray(input_array_name);
-  // The input will be changed later.
-  (*bi_op)->inputs.push_back(input_array_name);
-  int i = 1;
-  // Fill in the fw_lstm weights.
-  for (; i < kBwLstmInputsStartIndex; ++i) {
-    (*bi_op)->inputs.push_back(fw_lstm_op.inputs[i]);
-  }
-
-  // Fill in the bw_lstm weights. bidirectional lstm backward weights start
-  // from 18.
-  for (; i < kFwInputActivationStartIndex; ++i) {
-    (*bi_op)->inputs.push_back(
-        bw_lstm_op
-            .inputs[i - (kBwLstmInputsStartIndex - kFwLstmInputsStartIndex)]);
-  }
-
-  // Fill in fw_lstm previous states.
-  for (; i < kBwInputActivationStartIndex; ++i) {
-    (*bi_op)->inputs.push_back(
-        fw_lstm_op.inputs[i - (kFwInputActivationStartIndex -
-                               kBwLstmInputsStartIndex)]);
-  }
-
-  // Fill in bw_lstm previous states.
-  for (; i < kAuxInputStartIndex; ++i) {
-    (*bi_op)->inputs.push_back(
-        bw_lstm_op.inputs[i - (kBwInputActivationStartIndex -
-                               kBwLstmInputsStartIndex)]);
-  }
-
-  // TODO(renjieliu): Deal with Auxiliary input and weights for 39 - 47.
-  for (; i <= kBidirectionalSequenceLstmInputsCount; ++i) {
-    const string& temp_array_name = AvailableArrayName(
-        *model, "bidirectional_sequence_lstm_temp_" + std::to_string(i));
-    model->CreateOptionalArray(temp_array_name);
-    (*bi_op)->inputs.push_back(temp_array_name);
-  }
-
-  // Deal with outputs.
-  (*bi_op)->outputs.reserve(2);
-  const string& fw_output_array_name =
-      AvailableArrayName(*model, "bidirectional_sequence_lstm_fw_output_0");
-  const string& bw_output_array_name =
-      AvailableArrayName(*model, "bidirectional_sequence_lstm_bw_output_0");
-  model->GetOrCreateArray(fw_output_array_name);
-  model->GetOrCreateArray(bw_output_array_name);
-  (*bi_op)->outputs.push_back(fw_output_array_name);
-  (*bi_op)->outputs.push_back(bw_output_array_name);
-  (*bi_op)->merge_outputs = false;
-  return true;
-}
-
-bool GroupFwBwLstmOps(
-    Model* model, std::stack<Operator*> fw_lstm_ops,
-    std::stack<Operator*> bw_lstm_ops,
-    std::vector<BidirectionalSequenceLstmOperator*>* bidirectional_lstm_ops) {
-  while (!fw_lstm_ops.empty()) {
-    Operator* fw_lstm_op = fw_lstm_ops.top();
-    Operator* bw_lstm_op = bw_lstm_ops.top();
-    BidirectionalSequenceLstmOperator* bidirectional_sequence_lstm_op =
-        new BidirectionalSequenceLstmOperator;
-    if (!ConstructBidirectionalSequenceLstmOp(
-            *fw_lstm_op, *bw_lstm_op, model, &bidirectional_sequence_lstm_op)) {
-      return false;
-    }
-
-    bidirectional_lstm_ops->push_back(bidirectional_sequence_lstm_op);
-    fw_lstm_ops.pop();
-    bw_lstm_ops.pop();
-  }
-  return true;
-}
-
-void RewireBidirectionalSequenceLstmOpsConnections(
-    const string& input_array_name,
-    const std::vector<BidirectionalSequenceLstmOperator*>&
-        bidirectional_sequence_lstm_ops,
-    std::vector<std::unique_ptr<Operator>>::iterator* op_it, Model* model) {
-  string cur_fw_input = input_array_name;
-  string cur_bw_input = input_array_name;
-  for (int i = 0; i < bidirectional_sequence_lstm_ops.size(); ++i) {
-    DeleteArrayIfUsedOnce(bidirectional_sequence_lstm_ops[i]->inputs[0], model);
-    bidirectional_sequence_lstm_ops[i]->inputs[0] = cur_fw_input;
-    if (i != 0) {
-      DeleteArrayIfUsedOnce(bidirectional_sequence_lstm_ops[i]->inputs[39],
-                            model);
-      bidirectional_sequence_lstm_ops[i]->inputs[39] = cur_bw_input;
-    }
-    cur_fw_input = bidirectional_sequence_lstm_ops[i]->outputs[0];
-    cur_bw_input = bidirectional_sequence_lstm_ops[i]->outputs[1];
-    if (i != (bidirectional_sequence_lstm_ops.size() - 1)) {
-      bidirectional_sequence_lstm_ops[i]->merge_outputs = false;
-    } else {
-      // TODO(renjieliu): We need to check whether the outputs of the last bidi
-      // lstms needs merged outputs or not.
-      bidirectional_sequence_lstm_ops[i]->merge_outputs = true;
-      DeleteArrayIfUnused(bidirectional_sequence_lstm_ops[i]->outputs[1],
-                          model);
-      bidirectional_sequence_lstm_ops[i]->outputs.pop_back();
-    }
-    model->operators.emplace(*op_it, bidirectional_sequence_lstm_ops[i]);
-    *op_it += 1;
-  }
-}
-
-void RewireFinalUnpackOutputs(
-    const UnpackOperator& original_unpack_operator,
-    UnpackOperator** final_unpack_operator,
-    BidirectionalSequenceLstmOperator** final_bidi_lstm_operator,
-    Model* model) {
-  (*final_unpack_operator)
-      ->inputs.push_back((*final_bidi_lstm_operator)->outputs[0]);
-  (*final_unpack_operator)->axis = original_unpack_operator.axis;
-  (*final_unpack_operator)->num = original_unpack_operator.num;
-
-  for (int i = 0; i < original_unpack_operator.outputs.size(); ++i) {
-    const string& output_array_name = original_unpack_operator.outputs[i];
-    const string& final_unpack_output_array_name = AvailableArrayName(
-        *model, "bidirectional_sequence_lstm_unpack_" + std::to_string(i));
-    model->GetOrCreateArray(final_unpack_output_array_name);
-    (*final_unpack_operator)->outputs.push_back(final_unpack_output_array_name);
-    Operator* unpack_following_op = GetOpWithInput(*model, output_array_name);
-    if (unpack_following_op != nullptr) {
-      // If there's a following op after the unpack, it must be a concat op.
-      DCHECK(unpack_following_op->type == OperatorType::kConcatenation);
-      // For every output of the concat, rewire the outputs.
-      for (const string& concat_output : unpack_following_op->outputs) {
-        (*final_unpack_operator)->outputs[i] = concat_output;
-      }
-      // Remove the concat op.
-      model->operators.erase(FindOperator(model, *unpack_following_op));
-    }
-  }
-}
-
-void RemoveUnpackOperator(const Operator& unpack_op, Model* model) {
-  for (const string& output_array_name : unpack_op.outputs) {
-    DeleteArrayIfUnused(output_array_name, model);
-  }
-  model->operators.erase(FindOperator(model, unpack_op));
-}
-
-void RemoveUnidirectionalSequenceLstmOps(std::stack<Operator*> uni_lstm_ops,
-                                         Model* model) {
-  while (!uni_lstm_ops.empty()) {
-    Operator* uni_lstm_op = uni_lstm_ops.top();
-    DeleteArrayIfUnused(uni_lstm_op->outputs[0], model);
-    model->operators.erase(FindOperator(model, *uni_lstm_op));
-    uni_lstm_ops.pop();
-  }
-}
-
-}  // namespace
-
-// TODO(renjieliu): Support graph generated by dynamic rnn as well.
-::tensorflow::Status GroupBidirectionalSequenceLstm::Run(Model* model,
-                                                         std::size_t op_index,
-                                                         bool* modified) {
-  *modified = false;
-  // Bidirectional sequence lstm will generate two separate unidirectional
-  // sequence lstm ops, for static bidirectional sequence lstm, there will be
-  // a concatenation op at very end; for dynamic bidirectional squence lstm,
-  // it is not guaranteed, but currently we do not support that.
-  auto op_it = model->operators.begin() + op_index;
-  Operator* final_concat_op = op_it->get();
-  if (final_concat_op->type != OperatorType::kConcatenation &&
-      final_concat_op->type != OperatorType::kConcat &&
-      final_concat_op->type != OperatorType::kConcatV2) {
-    return ::tensorflow::Status::OK();
-  }
-
-  // Match fw unidirectional lstm outputs and bw unidirectional lstm outputs:
-  // should be two unstack ops.
-  Operator *fw_lstm_output, *bw_lstm_output;
-  if (!MatchTwoUnpackOps(*final_concat_op, *model, &fw_lstm_output,
-                         &bw_lstm_output)) {
-    return ::tensorflow::Status::OK();
-  }
-
-  // Find all upstream unidirectional lstm ops.
-  std::stack<Operator*> fw_unidirectional_sequence_lstm_ops,
-      bw_unidirectional_sequence_lstm_ops;
-  Operator *first_fw_lstm_input, *first_bw_lstm_input;
-  if (!FindUnidirectionalSequenceLstmOp(*model, *fw_lstm_output,
-                                        &fw_unidirectional_sequence_lstm_ops,
-                                        &first_fw_lstm_input) ||
-      !FindUnidirectionalSequenceLstmOp(*model, *bw_lstm_output,
-                                        &bw_unidirectional_sequence_lstm_ops,
-                                        &first_bw_lstm_input)) {
-    return ::tensorflow::Status::OK();
-  }
-
-  if (fw_unidirectional_sequence_lstm_ops.size() !=
-          bw_unidirectional_sequence_lstm_ops.size() ||
-      fw_unidirectional_sequence_lstm_ops.empty()) {
-    return ::tensorflow::Status::OK();
-  }
-
-  // For static bidirectional sequence lstm, we should have two pack ops.
-  if (first_fw_lstm_input->type != OperatorType::kPack ||
-      first_bw_lstm_input->type != OperatorType::kPack) {
-    return ::tensorflow::Status::OK();
-  }
-
-  // fw_lstm & bw_lstm should point to the same input, but reversed sequence.
-  for (int i = 0; i < first_fw_lstm_input->inputs.size(); ++i) {
-    if (first_fw_lstm_input->inputs[i] !=
-        first_bw_lstm_input
-            ->inputs[first_fw_lstm_input->inputs.size() - i - 1]) {
-      return ::tensorflow::Status::OK();
-    }
-  }
-
-  std::vector<BidirectionalSequenceLstmOperator*>
-      bidirectional_sequence_lstm_ops;
-  if (!GroupFwBwLstmOps(model, fw_unidirectional_sequence_lstm_ops,
-                        bw_unidirectional_sequence_lstm_ops,
-                        &bidirectional_sequence_lstm_ops)) {
-    return ::tensorflow::Status::OK();
-  }
-
-  // Rewire the inputs & outputs.
-  string current_input = first_fw_lstm_input->outputs[0];
-  RewireBidirectionalSequenceLstmOpsConnections(
-      current_input, bidirectional_sequence_lstm_ops, &op_it, model);
-
-  // Insert a unpack op for the output.
-  UnpackOperator* unpack_operator = new UnpackOperator;
-
-  RewireFinalUnpackOutputs(
-      static_cast<const UnpackOperator&>(*fw_lstm_output), &unpack_operator,
-      &bidirectional_sequence_lstm_ops[bidirectional_sequence_lstm_ops.size() -
-                                       1],
-      model);
-  model->operators.emplace(op_it, unpack_operator);
-
-  // Delete unused ops.
-  RemoveUnpackOperator(*fw_lstm_output, model);
-  RemoveUnpackOperator(*bw_lstm_output, model);
-  RemoveUnidirectionalSequenceLstmOps(fw_unidirectional_sequence_lstm_ops,
-                                      model);
-  RemoveUnidirectionalSequenceLstmOps(bw_unidirectional_sequence_lstm_ops,
-                                      model);
-  // Only keep the fw lstm's pack input.
-  DeleteArrayIfUnused(first_bw_lstm_input->outputs[0], model);
-  model->operators.erase(FindOperator(model, *first_bw_lstm_input));
-  *modified = true;
-  return ::tensorflow::Status::OK();
-}
-
-}  // namespace toco
diff --git a/tensorflow/lite/toco/graph_transformations/group_bidirectional_sequence_ops.cc b/tensorflow/lite/toco/graph_transformations/group_bidirectional_sequence_ops.cc
new file mode 100644
index 0000000000..10e817b165
--- /dev/null
+++ b/tensorflow/lite/toco/graph_transformations/group_bidirectional_sequence_ops.cc
@@ -0,0 +1,495 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <cstdio>
+#include <iterator>
+#include <memory>
+#include <stack>
+#include <string>
+#include <vector>
+
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/lite/toco/graph_transformations/graph_transformations.h"
+#include "tensorflow/lite/toco/model.h"
+#include "tensorflow/lite/toco/tooling_util.h"
+
+namespace toco {
+namespace {
+
+std::vector<std::unique_ptr<Operator>>::iterator FindOperator(
+    Model* model, const Operator& op) {
+  return std::find_if(
+      model->operators.begin(), model->operators.end(),
+      [&op](const std::unique_ptr<Operator>& ptr) { return ptr.get() == &op; });
+}
+
+bool MatchTwoUnpackOps(const Operator& op, const Model& model,
+                       Operator** fw_output, Operator** bw_output) {
+  if (op.inputs.size() != 2) {
+    return false;
+  }
+
+  *fw_output = GetOpWithOutput(model, op.inputs[0]);
+  *bw_output = GetOpWithOutput(model, op.inputs[1]);
+  if (*fw_output == nullptr || *bw_output == nullptr) {
+    return false;
+  }
+
+  if ((*fw_output)->type != OperatorType::kUnpack ||
+      (*bw_output)->type != OperatorType::kUnpack) {
+    return false;
+  }
+
+  // TODO(renjieliu): Check the shapes are matching.
+
+  return true;
+}
+
+bool FindUnidirectionalSequenceOp(const Model& model, const Operator& output_op,
+                                  OperatorType operator_type,
+                                  std::stack<Operator*>* sequence_ops,
+                                  Operator** input_op) {
+  Operator* op_it = nullptr;
+  op_it = GetOpWithOutput(model, output_op.inputs[0]);
+  if (op_it == nullptr) {
+    return false;
+  }
+
+  while (op_it->type == operator_type) {
+    sequence_ops->push(op_it);
+    // Check the first input of the unidirectional squence lstm op.
+    op_it = GetOpWithOutput(model, op_it->inputs[0]);
+    if (op_it == nullptr) {
+      return false;
+    }
+  }
+
+  *input_op = op_it;
+  return true;
+}
+
+bool CheckTwoUnidirectionalSequenceOpsAreValid(
+    const std::stack<Operator*>& fw_unidirectional_sequence_ops,
+    const std::stack<Operator*>& bw_unidirectional_sequence_ops,
+    const Operator* first_fw_sequence_op_input,
+    const Operator* first_bw_sequence_op_input) {
+  if (fw_unidirectional_sequence_ops.size() !=
+          bw_unidirectional_sequence_ops.size() ||
+      fw_unidirectional_sequence_ops.empty()) {
+    return false;
+  }
+
+  // For static bidirectional sequence lstm, we should have two pack ops.
+  if (first_fw_sequence_op_input->type != OperatorType::kPack ||
+      first_bw_sequence_op_input->type != OperatorType::kPack) {
+    return false;
+  }
+
+  // fw_lstm & bw_lstm should point to the same input, but reversed sequence.
+  for (int i = 0; i < first_fw_sequence_op_input->inputs.size(); ++i) {
+    if (first_fw_sequence_op_input->inputs[i] !=
+        first_bw_sequence_op_input
+            ->inputs[first_fw_sequence_op_input->inputs.size() - i - 1]) {
+      return false;
+    }
+  }
+  return true;
+}
+
+bool ConstructBidirectionalSequenceOp(
+    const Operator& fw_lstm_op, const Operator& bw_lstm_op, Model* model,
+    BidirectionalSequenceLstmOperator** bi_op) {
+  // TODO(renjieliu): Check the shapes & configurations are equal.
+  constexpr int kBidirectionalSequenceLstmInputsCount = 47;
+  constexpr int kFwLstmInputsStartIndex = 1;
+  constexpr int kBwLstmInputsStartIndex = 18;
+  constexpr int kFwInputActivationStartIndex = 35;
+  constexpr int kBwInputActivationStartIndex = 37;
+  constexpr int kAuxInputStartIndex = 39;
+  (*bi_op)->inputs.reserve(kBidirectionalSequenceLstmInputsCount);
+  const string& input_array_name =
+      AvailableArrayName(*model, "bidirectional_sequence_lstm_input_0");
+  model->GetOrCreateArray(input_array_name);
+  // The input will be changed later.
+  (*bi_op)->inputs.push_back(input_array_name);
+  int i = 1;
+  // Fill in the fw_lstm weights.
+  for (; i < kBwLstmInputsStartIndex; ++i) {
+    (*bi_op)->inputs.push_back(fw_lstm_op.inputs[i]);
+  }
+
+  // Fill in the bw_lstm weights. bidirectional lstm backward weights start
+  // from 18.
+  for (; i < kFwInputActivationStartIndex; ++i) {
+    (*bi_op)->inputs.push_back(
+        bw_lstm_op
+            .inputs[i - (kBwLstmInputsStartIndex - kFwLstmInputsStartIndex)]);
+  }
+
+  // Fill in fw_lstm previous states.
+  for (; i < kBwInputActivationStartIndex; ++i) {
+    (*bi_op)->inputs.push_back(
+        fw_lstm_op.inputs[i - (kFwInputActivationStartIndex -
+                               kBwLstmInputsStartIndex)]);
+  }
+
+  // Fill in bw_lstm previous states.
+  for (; i < kAuxInputStartIndex; ++i) {
+    (*bi_op)->inputs.push_back(
+        bw_lstm_op.inputs[i - (kBwInputActivationStartIndex -
+                               kBwLstmInputsStartIndex)]);
+  }
+
+  // TODO(renjieliu): Deal with Auxiliary input and weights for 39 - 47.
+  for (; i <= kBidirectionalSequenceLstmInputsCount; ++i) {
+    const string& temp_array_name = AvailableArrayName(
+        *model, "bidirectional_sequence_lstm_temp_" + std::to_string(i));
+    model->CreateOptionalArray(temp_array_name);
+    (*bi_op)->inputs.push_back(temp_array_name);
+  }
+
+  // Deal with outputs.
+  (*bi_op)->outputs.reserve(2);
+  const string& fw_output_array_name =
+      AvailableArrayName(*model, "bidirectional_sequence_lstm_fw_output_0");
+  const string& bw_output_array_name =
+      AvailableArrayName(*model, "bidirectional_sequence_lstm_bw_output_0");
+  model->GetOrCreateArray(fw_output_array_name);
+  model->GetOrCreateArray(bw_output_array_name);
+  (*bi_op)->outputs.push_back(fw_output_array_name);
+  (*bi_op)->outputs.push_back(bw_output_array_name);
+  (*bi_op)->merge_outputs = false;
+  return true;
+}
+
+bool ConstructBidirectionalSequenceOp(
+    const Operator& fw_rnn_op, const Operator& bw_rnn_op, Model* model,
+    BidirectionalSequenceRnnOperator** bi_op) {
+  // TODO(renjieliu): Check the shapes & configurations are equal.
+  constexpr int kBidirectionalSequenceRnnInputsCount = 12;
+  constexpr int kFwInputsStartIndex = 1;
+  constexpr int kBwInputsStartIndex = 5;
+  constexpr int kAuxInputsStartIndex = 9;
+  (*bi_op)->inputs.reserve(kBidirectionalSequenceRnnInputsCount);
+  const string& input_array_name =
+      AvailableArrayName(*model, "bidirectional_sequence_rnn_input_0");
+  model->GetOrCreateArray(input_array_name);
+  // The input will be changed later.
+  (*bi_op)->inputs.push_back(input_array_name);
+  int i = 1;
+
+  // Fill in the fw_rnn weights.
+  for (; i < kBwInputsStartIndex; ++i) {
+    (*bi_op)->inputs.push_back(fw_rnn_op.inputs[i]);
+  }
+
+  // Fill in the bw_rnn weights.
+  for (; i < kAuxInputsStartIndex; ++i) {
+    (*bi_op)->inputs.push_back(
+        bw_rnn_op.inputs[i - (kBwInputsStartIndex - kFwInputsStartIndex)]);
+  }
+
+  // TODO(renjieliu): Deal with optional weights.
+  for (; i < kBidirectionalSequenceRnnInputsCount; ++i) {
+    const string& temp_array_name = AvailableArrayName(
+        *model, "bidirectional_sequence_rnn_temp_" + std::to_string(i));
+    model->CreateOptionalArray(temp_array_name);
+    (*bi_op)->inputs.push_back(temp_array_name);
+  }
+
+  // Deal with outputs.
+  (*bi_op)->outputs.reserve(2);
+  const string& fw_output_array_name =
+      AvailableArrayName(*model, "bidirectional_sequence_rnn_fw_output_0");
+  const string& bw_output_array_name =
+      AvailableArrayName(*model, "bidirectional_sequence_rnn_bw_output_0");
+  model->GetOrCreateArray(fw_output_array_name);
+  model->GetOrCreateArray(bw_output_array_name);
+  (*bi_op)->outputs.push_back(fw_output_array_name);
+  (*bi_op)->outputs.push_back(bw_output_array_name);
+  (*bi_op)->merge_outputs = false;
+  return true;
+}
+
+template <typename T>
+bool GroupFwBwSequenceOps(Model* model, std::stack<Operator*> fw_sequence_ops,
+                          std::stack<Operator*> bw_sequence_ops,
+                          std::vector<T*>* bidirectional_sequence_ops) {
+  while (!fw_sequence_ops.empty()) {
+    Operator* fw_sequence_op = fw_sequence_ops.top();
+    Operator* bw_sequence_op = bw_sequence_ops.top();
+    T* bidirectional_sequence_op = new T;
+    if (!ConstructBidirectionalSequenceOp(*fw_sequence_op, *bw_sequence_op,
+                                          model, &bidirectional_sequence_op)) {
+      return false;
+    }
+
+    bidirectional_sequence_ops->push_back(bidirectional_sequence_op);
+    fw_sequence_ops.pop();
+    bw_sequence_ops.pop();
+  }
+  return true;
+}
+
+template <typename T>
+void RewireBidirectionalSequenceSequenceOpsConnections(
+    OperatorType operator_type, const string& input_array_name,
+    const std::vector<T*>& bidirectional_sequence_ops,
+    std::vector<std::unique_ptr<Operator>>::iterator* op_it, Model* model) {
+  int aux_input_index = -1;
+  switch (operator_type) {
+    case OperatorType::kBidirectionalSequenceLstm:
+      aux_input_index = 39;
+      break;
+    case OperatorType::kBidirectionalSequenceRnn:
+      aux_input_index = 9;
+      break;
+    default:
+      // Should not reach here.
+      DCHECK(false);
+  }
+  string cur_fw_input = input_array_name;
+  string cur_bw_input = input_array_name;
+  for (int i = 0; i < bidirectional_sequence_ops.size(); ++i) {
+    DeleteArrayIfUsedOnce(bidirectional_sequence_ops[i]->inputs[0], model);
+    bidirectional_sequence_ops[i]->inputs[0] = cur_fw_input;
+    if (i != 0) {
+      DeleteArrayIfUsedOnce(
+          bidirectional_sequence_ops[i]->inputs[aux_input_index], model);
+      bidirectional_sequence_ops[i]->inputs[aux_input_index] = cur_bw_input;
+    }
+    cur_fw_input = bidirectional_sequence_ops[i]->outputs[0];
+    cur_bw_input = bidirectional_sequence_ops[i]->outputs[1];
+    if (i != (bidirectional_sequence_ops.size() - 1)) {
+      bidirectional_sequence_ops[i]->merge_outputs = false;
+    } else {
+      // TODO(renjieliu): We need to check whether the outputs of the last bidi
+      // lstms needs merged outputs or not.
+      bidirectional_sequence_ops[i]->merge_outputs = true;
+      DeleteArrayIfUnused(bidirectional_sequence_ops[i]->outputs[1], model);
+      bidirectional_sequence_ops[i]->outputs.pop_back();
+    }
+    model->operators.emplace(*op_it, bidirectional_sequence_ops[i]);
+    *op_it += 1;
+  }
+}
+
+template <typename T>
+void RewireFinalUnpackOutputs(const UnpackOperator& original_unpack_operator,
+                              UnpackOperator** final_unpack_operator,
+                              T** final_bidi_sequence_operator, Model* model) {
+  (*final_unpack_operator)
+      ->inputs.push_back((*final_bidi_sequence_operator)->outputs[0]);
+  (*final_unpack_operator)->axis = original_unpack_operator.axis;
+  (*final_unpack_operator)->num = original_unpack_operator.num;
+
+  for (int i = 0; i < original_unpack_operator.outputs.size(); ++i) {
+    const string& output_array_name = original_unpack_operator.outputs[i];
+    const string& final_unpack_output_array_name = AvailableArrayName(
+        *model, "bidirectional_sequence_unpack_" + std::to_string(i));
+    model->GetOrCreateArray(final_unpack_output_array_name);
+    (*final_unpack_operator)->outputs.push_back(final_unpack_output_array_name);
+    Operator* unpack_following_op = GetOpWithInput(*model, output_array_name);
+    if (unpack_following_op != nullptr) {
+      // If there's a following op after the unpack, it must be a concat op.
+      DCHECK(unpack_following_op->type == OperatorType::kConcatenation);
+      // For every output of the concat, rewire the outputs.
+      for (const string& concat_output : unpack_following_op->outputs) {
+        (*final_unpack_operator)->outputs[i] = concat_output;
+      }
+      // Remove the concat op.
+      model->operators.erase(FindOperator(model, *unpack_following_op));
+    }
+  }
+}
+
+void RemoveUnpackOperator(const Operator& unpack_op, Model* model) {
+  for (const string& output_array_name : unpack_op.outputs) {
+    DeleteArrayIfUnused(output_array_name, model);
+  }
+  model->operators.erase(FindOperator(model, unpack_op));
+}
+
+void RemoveUnidirectionalSequenceOps(std::stack<Operator*> uni_sequence_ops,
+                                     Model* model) {
+  while (!uni_sequence_ops.empty()) {
+    Operator* uni_sequence_op = uni_sequence_ops.top();
+    DeleteArrayIfUnused(uni_sequence_op->outputs[0], model);
+    model->operators.erase(FindOperator(model, *uni_sequence_op));
+    uni_sequence_ops.pop();
+  }
+}
+
+}  // namespace
+
+// TODO(renjieliu): Support graph generated by dynamic rnn as well.
+::tensorflow::Status GroupBidirectionalSequenceLstm::Run(Model* model,
+                                                         std::size_t op_index,
+                                                         bool* modified) {
+  *modified = false;
+  // Bidirectional sequence lstm will generate two separate unidirectional
+  // sequence lstm ops, for static bidirectional sequence lstm, there will be
+  // a concatenation op at very end; for dynamic bidirectional squence lstm,
+  // it is not guaranteed, but currently we do not support that.
+  auto op_it = model->operators.begin() + op_index;
+  Operator* final_concat_op = op_it->get();
+  if (final_concat_op->type != OperatorType::kConcatenation &&
+      final_concat_op->type != OperatorType::kConcat &&
+      final_concat_op->type != OperatorType::kConcatV2) {
+    return ::tensorflow::Status::OK();
+  }
+
+  // Match fw unidirectional lstm outputs and bw unidirectional lstm outputs:
+  // should be two unstack ops.
+  Operator *fw_lstm_output, *bw_lstm_output;
+  if (!MatchTwoUnpackOps(*final_concat_op, *model, &fw_lstm_output,
+                         &bw_lstm_output)) {
+    return ::tensorflow::Status::OK();
+  }
+
+  // Find all upstream unidirectional lstm ops.
+  std::stack<Operator*> fw_unidirectional_sequence_lstm_ops,
+      bw_unidirectional_sequence_lstm_ops;
+  Operator *first_fw_lstm_input, *first_bw_lstm_input;
+  if (!FindUnidirectionalSequenceOp(
+          *model, *fw_lstm_output, OperatorType::kUnidirectionalSequenceLstm,
+          &fw_unidirectional_sequence_lstm_ops, &first_fw_lstm_input) ||
+      !FindUnidirectionalSequenceOp(
+          *model, *bw_lstm_output, OperatorType::kUnidirectionalSequenceLstm,
+          &bw_unidirectional_sequence_lstm_ops, &first_bw_lstm_input)) {
+    return ::tensorflow::Status::OK();
+  }
+
+  if (!CheckTwoUnidirectionalSequenceOpsAreValid(
+          fw_unidirectional_sequence_lstm_ops,
+          bw_unidirectional_sequence_lstm_ops, first_fw_lstm_input,
+          first_bw_lstm_input)) {
+    return ::tensorflow::Status::OK();
+  }
+
+  std::vector<BidirectionalSequenceLstmOperator*>
+      bidirectional_sequence_lstm_ops;
+  if (!GroupFwBwSequenceOps(model, fw_unidirectional_sequence_lstm_ops,
+                            bw_unidirectional_sequence_lstm_ops,
+                            &bidirectional_sequence_lstm_ops)) {
+    return ::tensorflow::Status::OK();
+  }
+
+  // Rewire the inputs & outputs.
+  string current_input = first_fw_lstm_input->outputs[0];
+  RewireBidirectionalSequenceSequenceOpsConnections(
+      OperatorType::kBidirectionalSequenceLstm, current_input,
+      bidirectional_sequence_lstm_ops, &op_it, model);
+
+  // Insert a unpack op for the output.
+  UnpackOperator* unpack_operator = new UnpackOperator;
+
+  RewireFinalUnpackOutputs(
+      static_cast<const UnpackOperator&>(*fw_lstm_output), &unpack_operator,
+      &bidirectional_sequence_lstm_ops[bidirectional_sequence_lstm_ops.size() -
+                                       1],
+      model);
+  model->operators.emplace(op_it, unpack_operator);
+
+  // Delete unused ops.
+  RemoveUnpackOperator(*fw_lstm_output, model);
+  RemoveUnpackOperator(*bw_lstm_output, model);
+  RemoveUnidirectionalSequenceOps(fw_unidirectional_sequence_lstm_ops, model);
+  RemoveUnidirectionalSequenceOps(bw_unidirectional_sequence_lstm_ops, model);
+  // Only keep the fw lstm's pack input.
+  DeleteArrayIfUnused(first_bw_lstm_input->outputs[0], model);
+  model->operators.erase(FindOperator(model, *first_bw_lstm_input));
+  *modified = true;
+  return ::tensorflow::Status::OK();
+}
+
+// TODO(renjieliu): Support graph generated by dynamic rnn as well.
+::tensorflow::Status GroupBidirectionalSequenceRnn::Run(Model* model,
+                                                        std::size_t op_index,
+                                                        bool* modified) {
+  *modified = false;
+  // Bidirectional sequence rnn will generate two separate unidirectional
+  // sequence rnn ops, for static bidirectional sequence rnn, there will be
+  // a concatenation op at very end; for dynamic bidirectional squence rnn,
+  // it is not guaranteed, but currently we do not support that.
+  auto op_it = model->operators.begin() + op_index;
+  Operator* final_concat_op = op_it->get();
+  if (final_concat_op->type != OperatorType::kConcatenation &&
+      final_concat_op->type != OperatorType::kConcat &&
+      final_concat_op->type != OperatorType::kConcatV2) {
+    return ::tensorflow::Status::OK();
+  }
+
+  // Match fw unidirectional rnn outputs and bw unidirectional rnn outputs:
+  // should be two unstack ops.
+  Operator *fw_rnn_output, *bw_rnn_output;
+  if (!MatchTwoUnpackOps(*final_concat_op, *model, &fw_rnn_output,
+                         &bw_rnn_output)) {
+    return ::tensorflow::Status::OK();
+  }
+
+  // Find all upstream unidirectional rnn ops.
+  std::stack<Operator*> fw_unidirectional_sequence_rnn_ops,
+      bw_unidirectional_sequence_rnn_ops;
+  Operator *first_fw_rnn_input, *first_bw_rnn_input;
+  if (!FindUnidirectionalSequenceOp(
+          *model, *fw_rnn_output, OperatorType::kUnidirectionalSequenceRnn,
+          &fw_unidirectional_sequence_rnn_ops, &first_fw_rnn_input) ||
+      !FindUnidirectionalSequenceOp(
+          *model, *bw_rnn_output, OperatorType::kUnidirectionalSequenceRnn,
+          &bw_unidirectional_sequence_rnn_ops, &first_bw_rnn_input)) {
+    return ::tensorflow::Status::OK();
+  }
+
+  if (!CheckTwoUnidirectionalSequenceOpsAreValid(
+          fw_unidirectional_sequence_rnn_ops,
+          bw_unidirectional_sequence_rnn_ops, first_fw_rnn_input,
+          first_bw_rnn_input)) {
+    return ::tensorflow::Status::OK();
+  }
+
+  std::vector<BidirectionalSequenceRnnOperator*> bidirectional_sequence_rnn_ops;
+  if (!GroupFwBwSequenceOps(model, fw_unidirectional_sequence_rnn_ops,
+                            bw_unidirectional_sequence_rnn_ops,
+                            &bidirectional_sequence_rnn_ops)) {
+    return ::tensorflow::Status::OK();
+  }
+
+  // Rewire the inputs & outputs.
+  string current_input = first_fw_rnn_input->outputs[0];
+  RewireBidirectionalSequenceSequenceOpsConnections(
+      OperatorType::kBidirectionalSequenceRnn, current_input,
+      bidirectional_sequence_rnn_ops, &op_it, model);
+
+  // Insert a unpack op for the output.
+  UnpackOperator* unpack_operator = new UnpackOperator;
+  RewireFinalUnpackOutputs(
+      static_cast<const UnpackOperator&>(*fw_rnn_output), &unpack_operator,
+      &bidirectional_sequence_rnn_ops[bidirectional_sequence_rnn_ops.size() -
+                                      1],
+      model);
+  model->operators.emplace(op_it, unpack_operator);
+
+  // Delete unused ops.
+  RemoveUnpackOperator(*fw_rnn_output, model);
+  RemoveUnpackOperator(*bw_rnn_output, model);
+  RemoveUnidirectionalSequenceOps(fw_unidirectional_sequence_rnn_ops, model);
+  RemoveUnidirectionalSequenceOps(bw_unidirectional_sequence_rnn_ops, model);
+  // Only keep the fw rnn's pack input.
+  DeleteArrayIfUnused(first_bw_rnn_input->outputs[0], model);
+  model->operators.erase(FindOperator(model, *first_bw_rnn_input));
+  *modified = true;
+  return ::tensorflow::Status::OK();
+}
+}  // namespace toco
diff --git a/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc b/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
index d8e0135350..cb66a2372f 100644
--- a/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
+++ b/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
@@ -272,6 +272,12 @@ void SetDataTypeForAllOutputs(Model* model, Operator* op,
       SetDataTypeForAllOutputs(model, op, data_type);
       break;
     }
+    case OperatorType::kBidirectionalSequenceRnn: {
+      const ArrayDataType data_type = model->GetArray(op->inputs[0]).data_type;
+      if (data_type != ArrayDataType::kFloat) return ::tensorflow::Status::OK();
+      SetDataTypeForAllOutputs(model, op, data_type);
+      break;
+    }
     case OperatorType::kLstmCell: {
       // It's tricky to propagate data types through a LstmCell, as that has
       // multiple inputs and outputs, and there are quantized cases with
diff --git a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
index eba03dcc14..329ef92fe9 100644
--- a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
+++ b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
@@ -1205,6 +1205,58 @@ void ProcessBidirectionalSequenceLstmOperator(
   }
 }
 
+void ProcessBidirectionalSequenceRnnOperator(
+    Model* model, BidirectionalSequenceRnnOperator* op) {
+  // We assume time major.
+  auto& fw_output_array = model->GetArray(op->outputs[0]);
+  auto& bw_output_array = model->GetArray(op->outputs[1]);
+  if (fw_output_array.has_shape()) {
+    // Shape already propagated
+    return;
+  }
+
+  if (fw_output_array.data_type == ArrayDataType::kNone) {
+    // Yield until the output type has been set by PropagateArrayDataTypes
+    return;
+  }
+
+  // TODO(renjieliu): check the inputs, as well as all kinds of weights.
+  const auto& input_array = model->GetArray(op->inputs[0]);
+  // Yield until input dims have been resolved.
+  if (!input_array.has_shape()) {
+    return;
+  }
+  const auto& input_shape = input_array.shape();
+  const int batch_size = input_shape.dims(1);
+  const int timestamp = input_shape.dims(0);
+
+  constexpr int kFwWeightsTensor = 1;
+  const auto& forward_weights_array =
+      model->GetArray(op->inputs[kFwWeightsTensor]);
+  // Yield until input dims have been resolved.
+  if (!forward_weights_array.has_shape()) {
+    return;
+  }
+
+  constexpr int kFwHiddenStateTensor = 4;
+  constexpr int kBwHiddenStateTensor = 8;
+  // b(115961645): This is a hack to work around.
+  model->GetArray(op->inputs[kFwHiddenStateTensor]).buffer.reset();
+  model->GetArray(op->inputs[kBwHiddenStateTensor]).buffer.reset();
+
+  const auto& output_weights_shape = forward_weights_array.shape();
+  const int output_size = output_weights_shape.dims(0);
+
+  Shape* fw_output_shape = fw_output_array.mutable_shape();
+  if (op->merge_outputs) {
+    fw_output_shape->ReplaceDims({timestamp, batch_size, 2 * output_size});
+  } else {
+    fw_output_shape->ReplaceDims({timestamp, batch_size, output_size});
+    Shape* bw_output_shape = bw_output_array.mutable_shape();
+    bw_output_shape->ReplaceDims({timestamp, batch_size, output_size});
+  }
+}
+
 void ProcessSpaceToBatchNDOperator(Model* model, SpaceToBatchNDOperator* op) {
   const auto& input_array = model->GetArray(op->inputs[0]);
   // Yield until input dims have been resolved.
@@ -2166,6 +2218,10 @@ void ProcessUniqueOperator(Model* model, UniqueOperator* op) {
       ProcessBidirectionalSequenceLstmOperator(
           model, static_cast<BidirectionalSequenceLstmOperator*>(op));
       break;
+    case OperatorType::kBidirectionalSequenceRnn:
+      ProcessBidirectionalSequenceRnnOperator(
+          model, static_cast<BidirectionalSequenceRnnOperator*>(op));
+      break;
     case OperatorType::kLstmCell:
       ProcessLstmCellOperator(model, static_cast<LstmCellOperator*>(op));
       break;
diff --git a/tensorflow/lite/toco/model.h b/tensorflow/lite/toco/model.h
index 84efda111b..b99cb74a16 100644
--- a/tensorflow/lite/toco/model.h
+++ b/tensorflow/lite/toco/model.h
@@ -162,7 +162,8 @@ enum class OperatorType : uint8 {
   kUnique,
   kUnidirectionalSequenceRnn,
   kBidirectionalSequenceLstm,
-  kReverseV2
+  kReverseV2,
+  kBidirectionalSequenceRnn
 };
 
 // Helper to deal with TensorFlow arrays using a different ordering of
@@ -658,6 +659,12 @@ struct BidirectionalSequenceLstmOperator : Operator {
   bool merge_outputs;
 };
 
+struct BidirectionalSequenceRnnOperator : Operator {
+  BidirectionalSequenceRnnOperator()
+      : Operator(OperatorType::kBidirectionalSequenceRnn) {}
+  bool merge_outputs;
+};
+
 // Element-wise multiplication operator.
 //
 // Inputs:
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index e26aeea010..24f2675731 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -962,6 +962,47 @@ class BidirectionalSequenceLstm
   }
 };
 
+class BidirectionalSequenceRnn
+    : public BuiltinOperator<
+          BidirectionalSequenceRnnOperator,
+          ::tflite::BidirectionalSequenceRNNOptions,
+          ::tflite::BuiltinOptions_BidirectionalSequenceRNNOptions> {
+ public:
+  using BuiltinOperator::BuiltinOperator;
+  flatbuffers::Offset<TfLiteOptions> WriteOptions(
+      const TocoOperator& op,
+      flatbuffers::FlatBufferBuilder* builder) const override {
+    // Current toco converter only supports tanh, no clip.
+    return ::tflite::CreateBidirectionalSequenceRNNOptions(
+        *builder, /*time_major=*/true,
+        /*fused_activation_function=*/
+        ::tflite::ActivationFunctionType_TANH,
+        /*merge_outputs=*/op.merge_outputs);
+  }
+
+  void ReadOptions(const TfLiteOptions& options,
+                   TocoOperator* op) const override {
+    // Only support tanh activation, so check that tflite type is tanh.
+    DCHECK(options.fused_activation_function() ==
+           ::tflite::ActivationFunctionType_TANH);
+    op->merge_outputs = options.merge_outputs();
+  }
+
+  int GetVersion(const OperatorSignature& op_signature) const override {
+    return 1;
+  }
+
+  std::vector<bool> GetMutatingInputVariables(
+      const Operator& op) const override {
+    std::vector<bool> mutating_input_variables(op.inputs.size(), false);
+    // Forward hidden state.
+    mutating_input_variables[4] = true;
+    // Backward hidden state.
+    mutating_input_variables[8] = true;
+    return mutating_input_variables;
+  }
+};
+
 class Mean : public BuiltinOperator<MeanOperator, ::tflite::ReducerOptions,
                                     ::tflite::BuiltinOptions_ReducerOptions> {
  public:
@@ -1932,6 +1973,9 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
   ops.emplace_back(MakeUnique<BidirectionalSequenceLstm>(
       ::tflite::BuiltinOperator_BIDIRECTIONAL_SEQUENCE_LSTM,
       OperatorType::kBidirectionalSequenceLstm));
+  ops.emplace_back(MakeUnique<BidirectionalSequenceRnn>(
+      ::tflite::BuiltinOperator_BIDIRECTIONAL_SEQUENCE_RNN,
+      OperatorType::kBidirectionalSequenceRnn));
   ops.push_back(MakeUnique<OneHot>(::tflite::BuiltinOperator_ONE_HOT,
                                    OperatorType::kOneHot));
   ops.push_back(MakeUnique<Unpack>(::tflite::BuiltinOperator_UNPACK,
diff --git a/tensorflow/lite/toco/toco_tooling.cc b/tensorflow/lite/toco/toco_tooling.cc
index 1a6b3c6f34..69d7a7a61a 100644
--- a/tensorflow/lite/toco/toco_tooling.cc
+++ b/tensorflow/lite/toco/toco_tooling.cc
@@ -323,11 +323,12 @@ void Transform(const TocoFlags& toco_flags, Model* model) {
                             });
   }
 
-  // Try to merge bidirectional sequence lstm if present.
+  // Try to merge bidirectional sequence lstm or rnn if present.
   GraphTransformationsSet bidirectional_transformations;
   bidirectional_transformations.Add(new RemoveUnusedOp);
   bidirectional_transformations.Add(new toco::GroupBidirectionalSequenceLstm);
-  RunGraphTransformations(model, "Group bidirectional sequence lstm",
+  bidirectional_transformations.Add(new toco::GroupBidirectionalSequenceRnn);
+  RunGraphTransformations(model, "Group bidirectional sequence lstm/rnn",
                           bidirectional_transformations);
 
   // Fix any issues with IO edges. This must happen after any transform that
diff --git a/tensorflow/lite/toco/tooling_util.cc b/tensorflow/lite/toco/tooling_util.cc
index b2c8e0e132..6978dde668 100644
--- a/tensorflow/lite/toco/tooling_util.cc
+++ b/tensorflow/lite/toco/tooling_util.cc
@@ -414,6 +414,7 @@ const char* OperatorTypeName(OperatorType type) {
     HANDLE_OPERATORTYPENAME_CASE(ZerosLike)
     HANDLE_OPERATORTYPENAME_CASE(UnidirectionalSequenceLstm)
     HANDLE_OPERATORTYPENAME_CASE(BidirectionalSequenceLstm)
+    HANDLE_OPERATORTYPENAME_CASE(BidirectionalSequenceRnn)
     HANDLE_OPERATORTYPENAME_CASE(ResizeNearestNeighbor)
     HANDLE_OPERATORTYPENAME_CASE(LeakyRelu)
     HANDLE_OPERATORTYPENAME_CASE(SquaredDifference)
-- 
GitLab


From a3a07493edb5e97896b466b7e704df1ff653c7b5 Mon Sep 17 00:00:00 2001
From: Amit <30853054+amitsrivastava78@users.noreply.github.com>
Date: Thu, 24 Jan 2019 09:49:04 +0530
Subject: [PATCH 1166/2345] Update README.md

---
 tensorflow/tools/dist_test/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/dist_test/README.md b/tensorflow/tools/dist_test/README.md
index 1e29977788..42483fbd4a 100644
--- a/tensorflow/tools/dist_test/README.md
+++ b/tensorflow/tools/dist_test/README.md
@@ -14,7 +14,7 @@ For example:
 
 By default, local_test.sh runs the MNIST-with-replicas model as a test.
 However, you can use the --model_name flag to run the tf-learn/wide&deep
-cesnsu model:
+census model:
 
     ./local_test.sh --model_name CENSUS_WIDENDEEP
 
-- 
GitLab


From 8055fd961c5f241ce2664de60866bbb9dfe7049e Mon Sep 17 00:00:00 2001
From: Amit <30853054+amitsrivastava78@users.noreply.github.com>
Date: Thu, 24 Jan 2019 09:52:54 +0530
Subject: [PATCH 1167/2345] Updated LIMITATIONS.md for typo error

Updated one Typo error
---
 tensorflow/python/autograph/LIMITATIONS.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/autograph/LIMITATIONS.md b/tensorflow/python/autograph/LIMITATIONS.md
index d8b1cb7616..fefe4bf79e 100644
--- a/tensorflow/python/autograph/LIMITATIONS.md
+++ b/tensorflow/python/autograph/LIMITATIONS.md
@@ -38,7 +38,7 @@ List Comprehensions | Yes | | We desugar `ListComp` into the appropriate combina
 Custom context managers | No | Yes | Currently low priority. Left unconverted currently.
 Generators | No | Maybe | Could be achievable using queues; very low priority.
 Assertions | Yes | | As `tf.Assert`
-Deletion | Yes | Maybe | Currently unconverted. If new semanti cs are required for `del`, we are able to add it in.
+Deletion | Yes | Maybe | Currently unconverted. If new semantics are required for `del`, we are able to add it in.
 Inline imports | No | Yes | For example, `import numpy as np; np.eye(3)`. Currently low priority.
 Async | No | No |
 
-- 
GitLab


From f6f192e53848eb398e976140fabd8176b506800c Mon Sep 17 00:00:00 2001
From: Amit <30853054+amitsrivastava78@users.noreply.github.com>
Date: Thu, 24 Jan 2019 10:06:54 +0530
Subject: [PATCH 1168/2345] Updated README.md

Typo error fixed
---
 tensorflow/java/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/java/README.md b/tensorflow/java/README.md
index 951e8bdd0d..36a8cc910d 100644
--- a/tensorflow/java/README.md
+++ b/tensorflow/java/README.md
@@ -21,7 +21,7 @@ Releases built from release branches are available on Maven Central.
 Additionally, every day binaries are built from the `master` branch on GitHub:
 
 - [JAR](https://storage.googleapis.com/tensorflow-nightly/github/tensorflow/lib_package/libtensorflow.jar)
-- [Sourc JAR](https://storage.googleapis.com/tensorflow-nightly/github/tensorflow/lib_package/libtensorflow-src.jar)
+- [Source JAR](https://storage.googleapis.com/tensorflow-nightly/github/tensorflow/lib_package/libtensorflow-src.jar)
 - JNI:
   - [Linux CPU-only](https://storage.googleapis.com/tensorflow-nightly/github/tensorflow/lib_package/libtensorflow_jni-cpu-linux-x86_64.tar.gz)
   - [Linux GPU](https://storage.googleapis.com/tensorflow-nightly/github/tensorflow/lib_package/libtensorflow_jni-gpu-linux-x86_64.tar.gz)
-- 
GitLab


From 557e1b42b701b45c9b4980c54fe27985545a7f70 Mon Sep 17 00:00:00 2001
From: Dayananda-V <dayanandasiet@gmail.com>
Date: Fri, 18 Jan 2019 17:37:15 +0530
Subject: [PATCH 1169/2345] TF device_test additional test cases covers 1-
 merge_device api test case

---
 tensorflow/python/framework/device_test.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/tensorflow/python/framework/device_test.py b/tensorflow/python/framework/device_test.py
index 0859e956ff..cd4b4ea51e 100644
--- a/tensorflow/python/framework/device_test.py
+++ b/tensorflow/python/framework/device_test.py
@@ -18,8 +18,11 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import device
+from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
+from tensorflow.python.ops import variables
 from tensorflow.python.platform import googletest
 
 
@@ -116,6 +119,20 @@ class DeviceTest(test_util.TensorFlowTestCase):
         "/job:muu/device:MyFunnyDevice:2"))
     self.assertEquals("/job:muu/task:1/device:MyFunnyDevice:2", d.to_string())
 
+    if not context.executing_eagerly():
+      with ops.device(device.merge_device("/device:GPU:0")):
+        var1 = variables.Variable(1.0)
+        self.assertEquals("/device:GPU:0", var1.device)
+        with ops.device(device.merge_device("/job:worker")):
+          var2 = variables.Variable(1.0)
+          self.assertEquals("/job:worker/device:GPU:0", var2.device)
+          with ops.device(device.merge_device("/device:CPU:0")):
+            var3 = variables.Variable(1.0)
+            self.assertEquals("/job:worker/device:CPU:0", var3.device)
+            with ops.device(device.merge_device("/job:ps")):
+              var4 = variables.Variable(1.0)
+              self.assertEquals("/job:ps/device:CPU:0", var4.device)
+
   def testCanonicalName(self):
     self.assertEqual("/job:foo/replica:0",
                      device.canonical_name("/job:foo/replica:0"))
-- 
GitLab


From bb61dfb4cfd00bdbb817929b889e1fa6017de6c4 Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Wed, 23 Jan 2019 21:50:53 -0800
Subject: [PATCH 1170/2345] Automated rollback of commit
 7df1985f651268cd02ba9011b4295412e7d635cf

PiperOrigin-RevId: 230659586
---
 tensorflow/core/kernels/fill_functor.cu.cc    |   7 +
 tensorflow/core/kernels/list_kernels.cc       | 194 +++---------
 tensorflow/core/kernels/list_kernels.cu.cc    |  65 ++--
 tensorflow/core/kernels/list_kernels.h        | 289 ++++++++++++------
 .../core/ops/compat/ops_history.v1.pbtxt      |  16 +
 tensorflow/core/ops/list_ops.cc               |   4 +
 tensorflow/core/ops/ops.pbtxt                 |  16 +
 .../python/kernel_tests/list_ops_test.py      | 178 ++++++++++-
 .../kernel_tests/tensor_array_ops_test.py     |  47 ++-
 tensorflow/python/ops/list_ops.py             |  74 ++++-
 10 files changed, 591 insertions(+), 299 deletions(-)

diff --git a/tensorflow/core/kernels/fill_functor.cu.cc b/tensorflow/core/kernels/fill_functor.cu.cc
index 050c95cf40..d4c9258689 100644
--- a/tensorflow/core/kernels/fill_functor.cu.cc
+++ b/tensorflow/core/kernels/fill_functor.cu.cc
@@ -88,9 +88,16 @@ struct SetZeroFunctor<GPUDevice, T> {
   }
 };
 
+template <>
+void SetZeroFunctor<GPUDevice, Variant>::operator()(
+    const GPUDevice& d, typename TTypes<Variant>::Flat out) {
+  // TODO(b/123028789): Implement this.
+}
+
 #define DEFINE_SETZERO_GPU(T) template struct SetZeroFunctor<GPUDevice, T>;
 TF_CALL_NUMBER_TYPES(DEFINE_SETZERO_GPU);
 TF_CALL_bool(DEFINE_SETZERO_GPU);
+TF_CALL_variant(DEFINE_SETZERO_GPU);
 #undef DEFINE_SETZERO_GPU
 
 // Partial specialization of FillFunctor<Device=GPUDevice, T>.
diff --git a/tensorflow/core/kernels/list_kernels.cc b/tensorflow/core/kernels/list_kernels.cc
index 5859c20d89..7c0b70d651 100644
--- a/tensorflow/core/kernels/list_kernels.cc
+++ b/tensorflow/core/kernels/list_kernels.cc
@@ -173,6 +173,17 @@ Status TensorShapeFromTensor(const Tensor& t, PartialTensorShape* out) {
       DataTypeString(t.dtype()));
 }
 
+Status GetElementShapeFromInput(OpKernelContext* c,
+                                const TensorList& tensor_list, int index,
+                                PartialTensorShape* element_shape) {
+  TF_RETURN_IF_ERROR(TensorShapeFromTensor(c->input(index), element_shape));
+  // Check that `element_shape` and `tensor_list.element_shape` are
+  // compatible and store the merged shape in `element_shape`.
+  PartialTensorShape tmp = *element_shape;
+  TF_RETURN_IF_ERROR(tmp.MergeWith(tensor_list.element_shape, element_shape));
+  return Status::OK();
+}
+
 class EmptyTensorList : public OpKernel {
  public:
   explicit EmptyTensorList(OpKernelConstruction* ctx) : OpKernel(ctx) {
@@ -358,69 +369,6 @@ class TensorListElementShape : public OpKernel {
 REGISTER_KERNEL_BUILDER(Name("TensorListElementShape").Device(DEVICE_CPU),
                         TensorListElementShape);
 
-#if GOOGLE_CUDA
-
-REGISTER_KERNEL_BUILDER(Name("TensorListElementShape")
-                            .Device(DEVICE_GPU)
-                            .HostMemory("element_shape"),
-                        TensorListElementShape);
-
-#endif  // GOOGLE_CUDA
-
-class TensorListPopBack : public OpKernel {
- public:
-  explicit TensorListPopBack(OpKernelConstruction* c) : OpKernel(c) {
-    OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_));
-  }
-
-  ~TensorListPopBack() override {}
-
-  void Compute(OpKernelContext* c) override {
-    const TensorList* l = c->input(0).scalar<Variant>()().get<TensorList>();
-    OP_REQUIRES(c, l != nullptr,
-                errors::InvalidArgument(
-                    "Input handle is not a list. Saw: '",
-                    c->input(0).scalar<Variant>()().DebugString(), "'"));
-    OP_REQUIRES(c, element_dtype_ == l->element_dtype,
-                errors::InvalidArgument("Invalid data types; op elements ",
-                                        DataTypeString(element_dtype_),
-                                        " but list elements ",
-                                        DataTypeString(l->element_dtype)));
-
-    OP_REQUIRES(c, !l->tensors.empty(),
-                errors::InvalidArgument("Trying to pop from an empty list."));
-
-    c->set_output(1, l->tensors.back());
-    AllocatorAttributes attr;
-    attr.set_on_host(true);
-    std::unique_ptr<Tensor> maybe_result = c->forward_input(
-        0, 0, DT_VARIANT, TensorShape{}, c->input_memory_type(0), attr);
-    if (maybe_result != nullptr) {
-      maybe_result->scalar<Variant>()().get<TensorList>()->tensors.pop_back();
-    } else {
-      TensorList output;
-      output = *l;
-      output.tensors.pop_back();
-      Tensor* result;
-      OP_REQUIRES_OK(c, c->allocate_output(0, TensorShape{}, &result, attr));
-      result->scalar<Variant>()() = std::move(output);
-    }
-  }
-
- private:
-  DataType element_dtype_;
-};
-
-REGISTER_KERNEL_BUILDER(Name("TensorListPopBack").Device(DEVICE_CPU),
-                        TensorListPopBack);
-
-#if GOOGLE_CUDA
-
-REGISTER_KERNEL_BUILDER(Name("TensorListPopBack").Device(DEVICE_GPU),
-                        TensorListPopBack);
-
-#endif  // GOOGLE_CUDA
-
 class TensorListReserve : public OpKernel {
  public:
   explicit TensorListReserve(OpKernelConstruction* c) : OpKernel(c) {
@@ -458,61 +406,6 @@ REGISTER_KERNEL_BUILDER(Name("TensorListReserve")
                         TensorListReserve);
 
 #endif  // GOOGLE_CUDA
-
-class TensorListGetItem : public OpKernel {
- public:
-  explicit TensorListGetItem(OpKernelConstruction* c) : OpKernel(c) {
-    OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_));
-  }
-
-  void Compute(OpKernelContext* c) override {
-    OP_REQUIRES(
-        c, c->input(0).shape().num_elements() == 1,
-        errors::InvalidArgument("List tensors are supposed to be scalars."));
-    const TensorList* l = c->input(0).scalar<Variant>()().get<TensorList>();
-    OP_REQUIRES(c, l != nullptr,
-                errors::InvalidArgument(
-                    "Input handle is not a list. Saw: '",
-                    c->input(0).scalar<Variant>()().DebugString(), "'"));
-    OP_REQUIRES(c, element_dtype_ == l->element_dtype,
-                errors::InvalidArgument("Invalid data types; op elements ",
-                                        DataTypeString(element_dtype_),
-                                        " but list elements ",
-                                        DataTypeString(l->element_dtype)));
-    int32 index = c->input(1).scalar<int32>()();
-    OP_REQUIRES(c, index < l->tensors.size(),
-                errors::InvalidArgument("Trying to access element ", index,
-                                        " in a list with ", l->tensors.size(),
-                                        " elements."));
-    c->set_output(0, l->tensors[index]);
-  }
-
- private:
-  DataType element_dtype_;
-};
-
-REGISTER_KERNEL_BUILDER(Name("TensorListGetItem").Device(DEVICE_CPU),
-                        TensorListGetItem);
-
-#if GOOGLE_CUDA
-
-#define REGISTER_TENSOR_LIST_GET_ITEM_GPU(T)                      \
-  REGISTER_KERNEL_BUILDER(Name("TensorListGetItem")               \
-                              .TypeConstraint<T>("element_dtype") \
-                              .Device(DEVICE_GPU)                 \
-                              .HostMemory("index"),               \
-                          TensorListGetItem);
-
-TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_GET_ITEM_GPU);
-TF_CALL_complex64(REGISTER_TENSOR_LIST_GET_ITEM_GPU);
-TF_CALL_complex128(REGISTER_TENSOR_LIST_GET_ITEM_GPU);
-TF_CALL_int64(REGISTER_TENSOR_LIST_GET_ITEM_GPU);
-REGISTER_TENSOR_LIST_GET_ITEM_GPU(bfloat16)
-REGISTER_TENSOR_LIST_GET_ITEM_GPU(bool)
-#undef REGISTER_TENSOR_LIST_GET_ITEM_GPU
-
-#endif  // GOOGLE_CUDA
-
 class TensorListResize : public OpKernel {
  public:
   explicit TensorListResize(OpKernelConstruction* c) : OpKernel(c) {}
@@ -736,23 +629,7 @@ REGISTER_KERNEL_BUILDER(Name("TensorListConcatLists").Device(DEVICE_GPU),
 
 #endif  // GOOGLE_CUDA
 
-#define REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(T)               \
-  REGISTER_KERNEL_BUILDER(Name("TensorListPushBackBatch")         \
-                              .TypeConstraint<T>("element_dtype") \
-                              .Device(DEVICE_CPU),                \
-                          TensorListPushBackBatch<CPUDevice, T>)
-
-TF_CALL_ALL_TYPES(REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU);
-REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(quint8);
-REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(qint8);
-REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(quint16);
-REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(qint16);
-REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(qint32);
-REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(bfloat16);
-
-#undef REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU
-
-#define REGISTER_TENSOR_LIST_STACK_CPU(T)                         \
+#define REGISTER_TENSOR_LIST_OPS_CPU(T)                           \
   REGISTER_KERNEL_BUILDER(Name("TensorListStack")                 \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_CPU),                \
@@ -764,19 +641,15 @@ REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(bfloat16);
   REGISTER_KERNEL_BUILDER(Name("TensorListConcat")                \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_CPU),                \
-                          TensorListConcat<CPUDevice, T>)
-
-TF_CALL_POD_STRING_TYPES(REGISTER_TENSOR_LIST_STACK_CPU);
-REGISTER_TENSOR_LIST_STACK_CPU(quint8);
-REGISTER_TENSOR_LIST_STACK_CPU(qint8);
-REGISTER_TENSOR_LIST_STACK_CPU(quint16);
-REGISTER_TENSOR_LIST_STACK_CPU(qint16);
-REGISTER_TENSOR_LIST_STACK_CPU(qint32);
-REGISTER_TENSOR_LIST_STACK_CPU(bfloat16);
-
-#undef REGISTER_TENSOR_LIST_STACK_CPU
-
-#define REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(T)                   \
+                          TensorListConcat<CPUDevice, T>)         \
+  REGISTER_KERNEL_BUILDER(Name("TensorListGetItem")               \
+                              .TypeConstraint<T>("element_dtype") \
+                              .Device(DEVICE_CPU),                \
+                          TensorListGetItem<CPUDevice, T>)        \
+  REGISTER_KERNEL_BUILDER(Name("TensorListPopBack")               \
+                              .TypeConstraint<T>("element_dtype") \
+                              .Device(DEVICE_CPU),                \
+                          TensorListPopBack<CPUDevice, T>)        \
   REGISTER_KERNEL_BUILDER(Name("TensorListFromTensor")            \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_CPU),                \
@@ -788,17 +661,24 @@ REGISTER_TENSOR_LIST_STACK_CPU(bfloat16);
   REGISTER_KERNEL_BUILDER(Name("TensorListSplit")                 \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_CPU),                \
-                          TensorListSplit<CPUDevice, T>)
+                          TensorListSplit<CPUDevice, T>)          \
+  REGISTER_KERNEL_BUILDER(Name("TensorListPushBackBatch")         \
+                              .TypeConstraint<T>("element_dtype") \
+                              .Device(DEVICE_CPU),                \
+                          TensorListPushBackBatch<CPUDevice, T>)
+
+TF_CALL_POD_STRING_TYPES(REGISTER_TENSOR_LIST_OPS_CPU);
+REGISTER_TENSOR_LIST_OPS_CPU(quint8);
+REGISTER_TENSOR_LIST_OPS_CPU(qint8);
+REGISTER_TENSOR_LIST_OPS_CPU(quint16);
+REGISTER_TENSOR_LIST_OPS_CPU(qint16);
+REGISTER_TENSOR_LIST_OPS_CPU(qint32);
+REGISTER_TENSOR_LIST_OPS_CPU(bfloat16);
+REGISTER_TENSOR_LIST_OPS_CPU(Variant);
 
-TF_CALL_POD_STRING_TYPES(REGISTER_TENSOR_LIST_FROM_TENSOR_CPU);
-REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(quint8);
-REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(qint8);
-REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(quint16);
-REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(qint16);
-REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(qint32);
-REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(bfloat16);
+#undef REGISTER_TENSOR_LIST_OPS_CPU
 
-#undef REGISTER_TENSOR_LIST_FROM_TENSOR_CPU
+#define REGISTER_TENSOR_LIST_OPS_CPU(T)
 
 REGISTER_UNARY_VARIANT_BINARY_OP_FUNCTION(ADD_VARIANT_BINARY_OP, DEVICE_CPU,
                                           TensorList,
diff --git a/tensorflow/core/kernels/list_kernels.cu.cc b/tensorflow/core/kernels/list_kernels.cu.cc
index 23f552642c..c3a5041b00 100644
--- a/tensorflow/core/kernels/list_kernels.cu.cc
+++ b/tensorflow/core/kernels/list_kernels.cu.cc
@@ -36,47 +36,38 @@ namespace tensorflow {
 
 typedef Eigen::GpuDevice GPUDevice;
 
-#define REGISTER_TENSOR_LIST_STACK_GPU(T)                         \
+#define REGISTER_TENSOR_LIST_OPS_GPU(T)                           \
   REGISTER_KERNEL_BUILDER(Name("TensorListStack")                 \
                               .TypeConstraint<T>("element_dtype") \
-                              .Device(DEVICE_GPU),                \
+                              .Device(DEVICE_GPU)                 \
+                              .HostMemory("element_shape"),       \
                           TensorListStack<GPUDevice, T>)          \
   REGISTER_KERNEL_BUILDER(Name("TensorListGather")                \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_GPU)                 \
-                              .HostMemory("indices"),             \
+                              .HostMemory("indices")              \
+                              .HostMemory("element_shape"),       \
                           TensorListGather<GPUDevice, T>)         \
+  REGISTER_KERNEL_BUILDER(Name("TensorListGetItem")               \
+                              .TypeConstraint<T>("element_dtype") \
+                              .Device(DEVICE_GPU)                 \
+                              .HostMemory("index")                \
+                              .HostMemory("element_shape"),       \
+                          TensorListGetItem<GPUDevice, T>)        \
+  REGISTER_KERNEL_BUILDER(Name("TensorListPopBack")               \
+                              .TypeConstraint<T>("element_dtype") \
+                              .Device(DEVICE_GPU)                 \
+                              .HostMemory("element_shape"),       \
+                          TensorListPopBack<GPUDevice, T>)        \
   REGISTER_KERNEL_BUILDER(Name("TensorListConcat")                \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_GPU)                 \
                               .HostMemory("lengths"),             \
-                          TensorListConcat<GPUDevice, T>)
-
-TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_STACK_GPU);
-REGISTER_TENSOR_LIST_STACK_GPU(bfloat16);
-TF_CALL_complex64(REGISTER_TENSOR_LIST_STACK_GPU);
-TF_CALL_complex128(REGISTER_TENSOR_LIST_STACK_GPU);
-TF_CALL_int64(REGISTER_TENSOR_LIST_STACK_GPU);
-REGISTER_TENSOR_LIST_STACK_GPU(bool);
-
-#undef REGISTER_TENSOR_LIST_STACK_GPU
-
-#define REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU(T)               \
+                          TensorListConcat<GPUDevice, T>)         \
   REGISTER_KERNEL_BUILDER(Name("TensorListPushBackBatch")         \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_GPU),                \
-                          TensorListPushBackBatch<GPUDevice, T>)
-
-TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU);
-REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU(bfloat16);
-TF_CALL_complex64(REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU);
-TF_CALL_complex128(REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU);
-TF_CALL_int64(REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU);
-REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU(bool);
-
-#undef REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU
-
-#define REGISTER_TENSOR_LIST_FROM_TENSOR_GPU(T)                   \
+                          TensorListPushBackBatch<GPUDevice, T>)  \
   REGISTER_KERNEL_BUILDER(Name("TensorListFromTensor")            \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_GPU)                 \
@@ -95,14 +86,20 @@ REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU(bool);
                               .HostMemory("lengths"),             \
                           TensorListSplit<GPUDevice, T>)
 
-TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_FROM_TENSOR_GPU);
-REGISTER_TENSOR_LIST_FROM_TENSOR_GPU(bfloat16);
-TF_CALL_complex64(REGISTER_TENSOR_LIST_FROM_TENSOR_GPU);
-TF_CALL_complex128(REGISTER_TENSOR_LIST_FROM_TENSOR_GPU);
-TF_CALL_int64(REGISTER_TENSOR_LIST_FROM_TENSOR_GPU);
-REGISTER_TENSOR_LIST_FROM_TENSOR_GPU(bool);
+TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_OPS_GPU);
+REGISTER_TENSOR_LIST_OPS_GPU(bfloat16);
+TF_CALL_complex64(REGISTER_TENSOR_LIST_OPS_GPU);
+TF_CALL_complex128(REGISTER_TENSOR_LIST_OPS_GPU);
+TF_CALL_int64(REGISTER_TENSOR_LIST_OPS_GPU);
+REGISTER_TENSOR_LIST_OPS_GPU(bool);
+
+#undef REGISTER_TENSOR_LIST_OPS_GPU
 
-#undef REGISTER_TENSOR_LIST_FROM_TENSOR_GPU
+REGISTER_KERNEL_BUILDER(Name("TensorListPopBack")
+                            .TypeConstraint<Variant>("element_dtype")
+                            .Device(DEVICE_GPU)
+                            .HostMemory("element_shape"),
+                        TensorListPopBack<GPUDevice, Variant>)
 
 REGISTER_UNARY_VARIANT_BINARY_OP_FUNCTION(ADD_VARIANT_BINARY_OP, DEVICE_GPU,
                                           TensorList,
diff --git a/tensorflow/core/kernels/list_kernels.h b/tensorflow/core/kernels/list_kernels.h
index fd1be80f11..f74a88a768 100644
--- a/tensorflow/core/kernels/list_kernels.h
+++ b/tensorflow/core/kernels/list_kernels.h
@@ -66,14 +66,9 @@ struct TensorList {
 
 Status TensorShapeFromTensor(const Tensor& t, PartialTensorShape* out);
 
-// Allocates a Tensor of requested shape and dtype and fills it with zeros.
-template <typename Device, typename T>
-void BuildZerosTensor(OpKernelContext* c, DataType dtype,
-                      const TensorShape& shape, Tensor* zeros) {
-  OP_REQUIRES_OK(c, c->allocate_temp(dtype, shape, zeros));
-  functor::SetZeroFunctor<Device, T> f;
-  f(c->eigen_device<Device>(), zeros->flat<T>());
-}
+Status GetElementShapeFromInput(OpKernelContext* c,
+                                const TensorList& tensor_list, int index,
+                                PartialTensorShape* element_shape);
 
 template <typename Device, typename T>
 class TensorListStack : public OpKernel {
@@ -85,8 +80,6 @@ class TensorListStack : public OpKernel {
     OP_REQUIRES_OK(c, c->GetAttr("num_elements", &num_elements_));
   }
 
-  ~TensorListStack() {}
-
   void Compute(OpKernelContext* c) override {
     const TensorList* tensor_list =
         c->input(0).scalar<Variant>()().get<TensorList>();
@@ -99,13 +92,6 @@ class TensorListStack : public OpKernel {
         errors::InvalidArgument(
             "Invalid data types; op elements ", DataTypeString(element_dtype_),
             " but list elements ", DataTypeString(tensor_list->element_dtype)));
-    OP_REQUIRES(
-        c,
-        !tensor_list->tensors.empty() ||
-            tensor_list->element_shape.IsFullyDefined(),
-        errors::InvalidArgument("Tried to stack elements of a empty ",
-                                "list with non-fully-defined element_shape: ",
-                                tensor_list->element_shape.DebugString()));
     if (num_elements_ != -1) {
       OP_REQUIRES(c, tensor_list->tensors.size() == num_elements_,
                   errors::InvalidArgument(
@@ -113,44 +99,40 @@ class TensorListStack : public OpKernel {
                       " elements but got a list with ",
                       tensor_list->tensors.size(), " elements."));
     }
-    // Compute the shape of the output tensor.
-    // If `element_shape` is fully-defined it gets used. It is assumed that all
-    // element tensors have the same shape.
-    // If `element_shape` is not fully-defined the shape of the first
-    // initialized element tensor is used and it is checked that all other
-    // initialized tensors have the same shape. An error is thrown if the list
-    // only contains DT_INVALID type tensors.
-    TensorShape resulting_element_shape;
-    if (!tensor_list->element_shape.AsTensorShape(&resulting_element_shape)) {
-      bool resulting_element_shape_initialized = false;
+    PartialTensorShape partial_element_shape;
+    OP_REQUIRES_OK(c, GetElementShapeFromInput(c, *tensor_list, 1,
+                                               &partial_element_shape));
+    OP_REQUIRES(
+        c,
+        partial_element_shape.IsFullyDefined() || !tensor_list->tensors.empty(),
+        errors::InvalidArgument("Tried to stack elements of an empty ",
+                                "list with non-fully-defined element_shape: ",
+                                partial_element_shape.DebugString()));
+
+    // Check that `element_shape` input tensor is compatible with the shapes of
+    // element tensors.
+    if (!tensor_list->element_shape.IsFullyDefined()) {
       for (int i = 0; i < tensor_list->tensors.size(); ++i) {
         const Tensor& t = tensor_list->tensors[i];
-        if (!resulting_element_shape_initialized) {
-          if (t.dtype() == DT_INVALID) {
-            continue;
-          }
-          resulting_element_shape = t.shape();
-          resulting_element_shape_initialized = true;
-          continue;
+        if (t.dtype() != DT_INVALID) {
+          PartialTensorShape tmp = partial_element_shape;
+          OP_REQUIRES_OK(c, tmp.MergeWith(t.shape(), &partial_element_shape));
         }
-        OP_REQUIRES(
-            c, t.dtype() == DT_INVALID || t.shape() == resulting_element_shape,
-            errors::InvalidArgument(
-                "Tried to stack tensors with unequal shapes: ",
-                resulting_element_shape.DebugString(), " vs ",
-                t.shape().DebugString()));
       }
-      OP_REQUIRES(
-          c, resulting_element_shape_initialized,
-          errors::InvalidArgument("Tried to stack list which only contains ",
-                                  "uninitialized tensors and has a ",
-                                  "non-fully-defined element_shape: ",
-                                  tensor_list->element_shape.DebugString()));
     }
-    TensorShape output_tensor_shape = resulting_element_shape;
-    output_tensor_shape.InsertDim(0, tensor_list->tensors.size());
+
+    // Compute the shape of the output tensor by pre-pending the leading dim to
+    // the element_shape.
+    TensorShape element_shape;
+    OP_REQUIRES(c, partial_element_shape.AsTensorShape(&element_shape),
+                errors::InvalidArgument(
+                    "Tried to stack list which only contains uninitialized ",
+                    "tensors and has a non-fully-defined element_shape: ",
+                    partial_element_shape.DebugString()));
+    TensorShape output_shape = element_shape;
+    output_shape.InsertDim(0, tensor_list->tensors.size());
     Tensor* output;
-    OP_REQUIRES_OK(c, c->allocate_output(0, output_tensor_shape, &output));
+    OP_REQUIRES_OK(c, c->allocate_output(0, output_shape, &output));
     if (output->NumElements() == 0) {
       return;
     }
@@ -158,13 +140,21 @@ class TensorListStack : public OpKernel {
     ConstMatrixVector inputs_flat;
     inputs_flat.reserve(tensor_list->tensors.size());
     Tensor zeros;
-    BuildZerosTensor<Device, T>(c, element_dtype_, resulting_element_shape,
-                                &zeros);
     for (const auto& t : tensor_list->tensors) {
       if (t.dtype() != DT_INVALID) {
         inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
             t.shaped<T, 2>({1, t.NumElements()})));
       } else {
+        if (!zeros.NumElements()) {
+          AllocatorAttributes attr;
+          if (element_dtype_ == DT_VARIANT) {
+            attr.set_on_host(true);
+          }
+          OP_REQUIRES_OK(
+              c, c->allocate_temp(element_dtype_, element_shape, &zeros, attr));
+          functor::SetZeroFunctor<Device, T>()(c->eigen_device<Device>(),
+                                               zeros.flat<T>());
+        }
         inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
             const_cast<const Tensor&>(zeros).shaped<T, 2>(
                 {1, zeros.NumElements()})));
@@ -186,6 +176,123 @@ class TensorListStack : public OpKernel {
   DataType element_dtype_;
 };
 
+template <typename Device, typename T>
+class TensorListGetItem : public OpKernel {
+ public:
+  explicit TensorListGetItem(OpKernelConstruction* c) : OpKernel(c) {
+    OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_));
+  }
+
+  void Compute(OpKernelContext* c) override {
+    OP_REQUIRES(
+        c, c->input(0).shape().num_elements() == 1,
+        errors::InvalidArgument("List tensors are supposed to be scalars."));
+    const TensorList* l = c->input(0).scalar<Variant>()().get<TensorList>();
+    OP_REQUIRES(c, l != nullptr,
+                errors::InvalidArgument(
+                    "Input handle is not a list. Saw: '",
+                    c->input(0).scalar<Variant>()().DebugString(), "'"));
+    OP_REQUIRES(c, element_dtype_ == l->element_dtype,
+                errors::InvalidArgument("Invalid data types; op elements ",
+                                        DataTypeString(element_dtype_),
+                                        " but list elements ",
+                                        DataTypeString(l->element_dtype)));
+    int32 index = c->input(1).scalar<int32>()();
+    OP_REQUIRES(c, index < l->tensors.size(),
+                errors::InvalidArgument("Trying to access element ", index,
+                                        " in a list with ", l->tensors.size(),
+                                        " elements."));
+    if (l->tensors[index].dtype() != DT_INVALID) {
+      c->set_output(0, l->tensors[index]);
+    } else {
+      PartialTensorShape partial_element_shape;
+      OP_REQUIRES_OK(
+          c, GetElementShapeFromInput(c, *l, 2, &partial_element_shape));
+      TensorShape element_shape;
+      OP_REQUIRES(
+          c, partial_element_shape.AsTensorShape(&element_shape),
+          errors::InvalidArgument("Trying to read an uninitialized tensor but ",
+                                  "element_shape is not fully defined.",
+                                  partial_element_shape.DebugString()));
+      Tensor* result;
+      AllocatorAttributes attr;
+      if (element_dtype_ == DT_VARIANT) {
+        attr.set_on_host(true);
+      }
+      OP_REQUIRES_OK(c, c->allocate_output(0, element_shape, &result, attr));
+      functor::SetZeroFunctor<Device, T>()(c->eigen_device<Device>(),
+                                           result->flat<T>());
+    }
+  }
+
+ private:
+  DataType element_dtype_;
+};
+
+template <typename Device, typename T>
+class TensorListPopBack : public OpKernel {
+ public:
+  explicit TensorListPopBack(OpKernelConstruction* c) : OpKernel(c) {
+    OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_));
+  }
+
+  void Compute(OpKernelContext* c) override {
+    const TensorList* l = c->input(0).scalar<Variant>()().get<TensorList>();
+    OP_REQUIRES(c, l != nullptr,
+                errors::InvalidArgument(
+                    "Input handle is not a list. Saw: '",
+                    c->input(0).scalar<Variant>()().DebugString(), "'"));
+    OP_REQUIRES(c, element_dtype_ == l->element_dtype,
+                errors::InvalidArgument("Invalid data types; op elements ",
+                                        DataTypeString(element_dtype_),
+                                        " but list elements ",
+                                        DataTypeString(l->element_dtype)));
+
+    OP_REQUIRES(c, !l->tensors.empty(),
+                errors::InvalidArgument("Trying to pop from an empty list."));
+
+    const Tensor& t = l->tensors.back();
+    if (t.dtype() != DT_INVALID) {
+      c->set_output(1, t);
+    } else {
+      PartialTensorShape partial_element_shape;
+      OP_REQUIRES_OK(
+          c, GetElementShapeFromInput(c, *l, 1, &partial_element_shape));
+      TensorShape element_shape;
+      OP_REQUIRES(
+          c, partial_element_shape.AsTensorShape(&element_shape),
+          errors::InvalidArgument("Trying to read an uninitialized tensor but ",
+                                  "element_shape is not fully defined.",
+                                  partial_element_shape.DebugString()));
+      Tensor* result;
+      AllocatorAttributes attr;
+      if (element_dtype_ == DT_VARIANT) {
+        attr.set_on_host(true);
+      }
+      OP_REQUIRES_OK(c, c->allocate_output(1, element_shape, &result, attr));
+      functor::SetZeroFunctor<Device, T>()(c->eigen_device<Device>(),
+                                           result->flat<T>());
+    }
+    AllocatorAttributes attr;
+    attr.set_on_host(true);
+    std::unique_ptr<Tensor> maybe_result = c->forward_input(
+        0, 0, DT_VARIANT, TensorShape{}, c->input_memory_type(0), attr);
+    if (maybe_result != nullptr) {
+      maybe_result->scalar<Variant>()().get<TensorList>()->tensors.pop_back();
+    } else {
+      TensorList output;
+      output = *l;
+      output.tensors.pop_back();
+      Tensor* result;
+      OP_REQUIRES_OK(c, c->allocate_output(0, TensorShape{}, &result, attr));
+      result->scalar<Variant>()() = std::move(output);
+    }
+  }
+
+ private:
+  DataType element_dtype_;
+};
+
 template <typename Device, typename T>
 class TensorListConcat : public OpKernel {
  public:
@@ -206,8 +313,6 @@ class TensorListConcat : public OpKernel {
     }
   }
 
-  ~TensorListConcat() {}
-
   void Compute(OpKernelContext* c) override {
     // Check that the input Variant tensor is indeed a TensorList and has the
     // correct element type.
@@ -418,47 +523,47 @@ class TensorListGather : public OpKernel {
             "Invalid data types; op elements ", DataTypeString(element_dtype_),
             " but list elements ", DataTypeString(tensor_list->element_dtype)));
     Tensor indices = c->input(1);
+    PartialTensorShape partial_element_shape;
+    OP_REQUIRES_OK(c, GetElementShapeFromInput(c, *tensor_list, 2,
+                                               &partial_element_shape));
     OP_REQUIRES(
-        c,
-        indices.NumElements() > 0 ||
-            tensor_list->element_shape.IsFullyDefined(),
+        c, partial_element_shape.IsFullyDefined() || indices.NumElements() > 0,
         errors::InvalidArgument("Tried to gather 0-elements from "
                                 "a list with non-fully-defined shape: ",
-                                tensor_list->element_shape.DebugString()));
-    // Compute the shape of the output tensor.
-    // If `element_shape` is fully-defined it gets used. It is assumed that all
-    // requested tensors have the same shape.
-    // If `element_shape` is not fully-defined the shape of the first requested
-    // tensor is used and it is checked that all other tensors have the same
-    // shape.
-    TensorShape resulting_shape;
-    if (!tensor_list->element_shape.AsTensorShape(&resulting_shape)) {
-      const int i = indices.flat<int32>()(0);
-      OP_REQUIRES(
-          c, i < tensor_list->tensors.size(),
-          errors::InvalidArgument("Index ", i, " out o range; list only has ",
-                                  tensor_list->tensors.size(), " elements."));
-      const Tensor& t = tensor_list->tensors[i];
-      resulting_shape = t.shape();
-      for (int index = 1; index < indices.NumElements(); ++index) {
+                                partial_element_shape.DebugString()));
+
+    // Check that `element_shape` input tensor is compatible with the shapes of
+    // element tensors.
+    if (!tensor_list->element_shape.IsFullyDefined()) {
+      for (int index = 0; index < indices.NumElements(); ++index) {
         const int i = indices.flat<int32>()(index);
         const Tensor& t = tensor_list->tensors[i];
-        OP_REQUIRES(c, t.shape() == resulting_shape,
-                    errors::InvalidArgument(
-                        "Tried to gather elements with unequal shapes: ",
-                        resulting_shape.DebugString(), " vs ",
-                        t.shape().DebugString()));
+        if (t.dtype() != DT_INVALID) {
+          PartialTensorShape tmp = partial_element_shape;
+          OP_REQUIRES_OK(c, tmp.MergeWith(t.shape(), &partial_element_shape));
+        }
       }
     }
-    resulting_shape.InsertDim(0, indices.NumElements());
+
+    // Compute the shape of the output tensor by pre-pending the leading dim to
+    // the element_shape.
+    TensorShape element_shape;
+    OP_REQUIRES(
+        c, partial_element_shape.AsTensorShape(&element_shape),
+        errors::InvalidArgument("Tried to gather uninitialized tensors from a ",
+                                "list with non-fully-defined element_shape: ",
+                                partial_element_shape.DebugString()));
+    TensorShape output_shape = element_shape;
+    output_shape.InsertDim(0, indices.NumElements());
     Tensor* output;
-    OP_REQUIRES_OK(c, c->allocate_output(0, resulting_shape, &output));
+    OP_REQUIRES_OK(c, c->allocate_output(0, output_shape, &output));
     if (output->NumElements() == 0) {
       return;
     }
 
     ConstMatrixVector inputs_flat;
-    inputs_flat.reserve(tensor_list->tensors.size());
+    inputs_flat.reserve(indices.NumElements());
+    Tensor zeros;
     for (int index = 0; index < indices.NumElements(); ++index) {
       const int i = indices.flat<int32>()(index);
       OP_REQUIRES(
@@ -466,8 +571,24 @@ class TensorListGather : public OpKernel {
           errors::InvalidArgument("Index ", i, " out o range; list only has ",
                                   tensor_list->tensors.size(), " elements."));
       const Tensor& t = tensor_list->tensors[i];
-      inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
-          t.shaped<T, 2>({1, t.NumElements()})));
+      if (t.dtype() != DT_INVALID) {
+        inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
+            t.shaped<T, 2>({1, t.NumElements()})));
+      } else {
+        if (!zeros.NumElements()) {
+          AllocatorAttributes attr;
+          if (element_dtype_ == DT_VARIANT) {
+            attr.set_on_host(true);
+          }
+          OP_REQUIRES_OK(
+              c, c->allocate_temp(element_dtype_, element_shape, &zeros, attr));
+          functor::SetZeroFunctor<Device, T>()(c->eigen_device<Device>(),
+                                               zeros.flat<T>());
+        }
+        inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
+            const_cast<const Tensor&>(zeros).shaped<T, 2>(
+                {1, zeros.NumElements()})));
+      }
     }
     auto output_flat = output->shaped<T, 2>({1, output->NumElements()});
 
@@ -657,8 +778,6 @@ class TensorListPushBackBatch : public OpKernel {
     OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_));
   }
 
-  ~TensorListPushBackBatch() override {}
-
   void Compute(OpKernelContext* c) override {
     const Tensor& input = c->input(1);
     OP_REQUIRES(c, element_dtype_ == input.dtype(),
diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 3b7a7b812a..eb251fc517 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -77304,6 +77304,10 @@ op {
     name: "indices"
     type: DT_INT32
   }
+  input_arg {
+    name: "element_shape"
+    type: DT_INT32
+  }
   output_arg {
     name: "values"
     type_attr: "element_dtype"
@@ -77323,6 +77327,10 @@ op {
     name: "index"
     type: DT_INT32
   }
+  input_arg {
+    name: "element_shape"
+    type: DT_INT32
+  }
   output_arg {
     name: "item"
     type_attr: "element_dtype"
@@ -77349,6 +77357,10 @@ op {
     name: "input_handle"
     type: DT_VARIANT
   }
+  input_arg {
+    name: "element_shape"
+    type: DT_INT32
+  }
   output_arg {
     name: "output_handle"
     type: DT_VARIANT
@@ -77540,6 +77552,10 @@ op {
     name: "input_handle"
     type: DT_VARIANT
   }
+  input_arg {
+    name: "element_shape"
+    type: DT_INT32
+  }
   output_arg {
     name: "tensor"
     type_attr: "element_dtype"
diff --git a/tensorflow/core/ops/list_ops.cc b/tensorflow/core/ops/list_ops.cc
index cbc9c7a2f4..a16fabe2ab 100644
--- a/tensorflow/core/ops/list_ops.cc
+++ b/tensorflow/core/ops/list_ops.cc
@@ -130,6 +130,7 @@ REGISTER_OP("TensorListLength")
 
 REGISTER_OP("TensorListPopBack")
     .Input("input_handle: variant")
+    .Input("element_shape: int32")
     .Output("output_handle: variant")
     .Output("tensor: element_dtype")
     .Attr("element_dtype: type")
@@ -166,6 +167,7 @@ REGISTER_OP("TensorListPopBack")
 
 REGISTER_OP("TensorListStack")
     .Input("input_handle: variant")
+    .Input("element_shape: int32")
     .Output("tensor: element_dtype")
     .Attr("element_dtype: type")
     .Attr("num_elements: int = -1")
@@ -351,6 +353,7 @@ REGISTER_OP("TensorListReserve")
 REGISTER_OP("TensorListGetItem")
     .Input("input_handle: variant")
     .Input("index: int32")
+    .Input("element_shape: int32")
     .Output("item: element_dtype")
     .Attr("element_dtype: type")
     .SetShapeFn([](shape_inference::InferenceContext* c) {
@@ -418,6 +421,7 @@ REGISTER_OP("TensorListSetItem")
 REGISTER_OP("TensorListGather")
     .Input("input_handle: variant")
     .Input("indices: int32")
+    .Input("element_shape: int32")
     .Output("values: element_dtype")
     .Attr("element_dtype: type")
     .SetShapeFn([](shape_inference::InferenceContext* c) {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 92a7bb9a34..f43a8c4744 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -37082,6 +37082,10 @@ op {
     name: "indices"
     type: DT_INT32
   }
+  input_arg {
+    name: "element_shape"
+    type: DT_INT32
+  }
   output_arg {
     name: "values"
     type_attr: "element_dtype"
@@ -37101,6 +37105,10 @@ op {
     name: "index"
     type: DT_INT32
   }
+  input_arg {
+    name: "element_shape"
+    type: DT_INT32
+  }
   output_arg {
     name: "item"
     type_attr: "element_dtype"
@@ -37127,6 +37135,10 @@ op {
     name: "input_handle"
     type: DT_VARIANT
   }
+  input_arg {
+    name: "element_shape"
+    type: DT_INT32
+  }
   output_arg {
     name: "output_handle"
     type: DT_VARIANT
@@ -37318,6 +37330,10 @@ op {
     name: "input_handle"
     type: DT_VARIANT
   }
+  input_arg {
+    name: "element_shape"
+    type: DT_INT32
+  }
   output_arg {
     name: "tensor"
     type_attr: "element_dtype"
diff --git a/tensorflow/python/kernel_tests/list_ops_test.py b/tensorflow/python/kernel_tests/list_ops_test.py
index ec6906f20c..3f340aa07d 100644
--- a/tensorflow/python/kernel_tests/list_ops_test.py
+++ b/tensorflow/python/kernel_tests/list_ops_test.py
@@ -33,6 +33,7 @@ from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import gen_list_ops
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import list_ops
 from tensorflow.python.ops import math_ops
@@ -89,6 +90,58 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       l = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
       self.evaluate(l)
 
+  def testPopUninitializedTensorUseListElementShape(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=[2, 3], num_elements=3)
+    _, e = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
+    self.assertAllEqual(e, np.zeros((2, 3)))
+
+  def testPopUninitializedTensorUseSpecifiedElementShape(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=[None, 3], num_elements=3)
+    _, e = gen_list_ops.tensor_list_pop_back(
+        l, element_dtype=dtypes.float32, element_shape=[4, 3])
+    self.assertAllEqual(e, np.zeros((4, 3)))
+
+  def testPopUninitializedTensorWithInvalidElementShapeFails(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        "Trying to read an uninitialized tensor but "
+        "element_shape is not fully defined"):
+      _, e = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
+      self.evaluate(e)
+
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=[None, 2], num_elements=3)
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        r"Incompatible shapes during merge: \[1,3\] vs. \[\?,2\]"):
+      _, e = gen_list_ops.tensor_list_pop_back(
+          l, element_dtype=dtypes.float32, element_shape=[1, 3])
+      self.evaluate(e)
+
+  def testPushGetGrad(self):
+    with backprop.GradientTape() as tape:
+      l = list_ops.empty_tensor_list(
+          element_dtype=dtypes.float32, element_shape=None)
+      c0 = constant_op.constant(5.0)
+      c1 = constant_op.constant([10.0, 20.0])
+      tape.watch(c0)
+      tape.watch(c1)
+      l = list_ops.tensor_list_push_back(l, c0)
+      l = list_ops.tensor_list_push_back(l, c1)
+      t1 = list_ops.tensor_list_get_item(l, 1, element_dtype=dtypes.float32)
+      self.assertAllEqual(self.evaluate(t1), [10.0, 20.0])
+      # t1 == c1 so the gradient should be [0., [1., 1.]]
+      # This tests that the gradient of push_back correctly converts DT_INVALID
+      # tensors to zeros. The list returned by the gradient of GetItem will
+      # have only have tensor at index 1 set and others set to DT_INVALID.
+      dt0, dt1 = tape.gradient(t1, [c0, c1])
+      self.assertAllEqual(self.evaluate(dt1), [1.0, 1.0])
+      self.assertEqual(self.evaluate(dt0), 0.0)
+
   def _testStack(self, max_num_elements):
     l = list_ops.empty_tensor_list(
         element_dtype=dtypes.float32,
@@ -130,7 +183,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
     # Should raise an error when the element tensors do not all have the same
     # shape.
-    with self.assertRaisesRegexp(errors.InvalidArgumentError, "unequal shapes"):
+    with self.assertRaisesRegexp(errors.InvalidArgumentError,
+                                 "Incompatible ranks during merge: 0 vs. 1"):
       l = list_ops.tensor_list_push_back(l, constant_op.constant([3.0, 4.0]))
       t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
       self.evaluate(t)
@@ -151,7 +205,9 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
     # Should raise an error when the element tensors do not all have the same
     # shape.
-    with self.assertRaisesRegexp(errors.InvalidArgumentError, "unequal shapes"):
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        r"Incompatible shapes during merge: \[1\] vs. \[2\]"):
       l = list_ops.tensor_list_push_back(l, constant_op.constant([2.0, 3.0]))
       t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
       self.evaluate(t)
@@ -230,6 +286,13 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
       self.evaluate(t)
 
+  def testStackUsingSpecifiedElementShape(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
+    t = gen_list_ops.tensor_list_stack(
+        l, element_dtype=dtypes.float32, element_shape=[])
+    self.assertAllEqual(self.evaluate(t), np.zeros((3,)))
+
   @parameterized.named_parameters(("NoMaxNumElements", None),
                                   ("WithMaxNumElements", 2))
   def testGatherGrad(self, max_num_elements):
@@ -268,7 +331,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
     # Should raise an error when the requested tensors do not all have the same
     # shape.
-    with self.assertRaisesRegexp(errors.InvalidArgumentError, "unequal shapes"):
+    with self.assertRaisesRegexp(errors.InvalidArgumentError,
+                                 "Incompatible ranks during merge: 0 vs. 1"):
       t = list_ops.tensor_list_gather(l, [0, 2], element_dtype=dtypes.float32)
       self.evaluate(t)
 
@@ -292,7 +356,9 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
     # Should raise an error when the requested tensors do not all have the same
     # shape.
-    with self.assertRaisesRegexp(errors.InvalidArgumentError, "unequal shapes"):
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        r"Incompatible shapes during merge: \[1\] vs. \[2\]"):
       t = list_ops.tensor_list_gather(l, [0, 2], element_dtype=dtypes.float32)
       self.evaluate(t)
 
@@ -347,6 +413,54 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     dl_length = list_ops.tensor_list_length(dl)
     self.assertAllEqual(self.evaluate(dl_length), 3)
 
+  def _testGatherWithUninitializedTensors(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=[], num_elements=3)
+    t = list_ops.tensor_list_gather(l, [0, 2], element_dtype=dtypes.float32)
+    self.assertAllEqual(self.evaluate(t), [0., 0.])
+
+  def testGatherWithUninitializedTensors(self):
+    self._testGatherWithUninitializedTensors()
+
+  def testGatherWithUninitializedTensorsGpu(self):
+    if not context.num_gpus():
+      return
+    with context.device("gpu:0"):
+      self._testGatherWithUninitializedTensors()
+
+  def _testGatherWithUninitializedTensorsInferShape(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
+    l = list_ops.tensor_list_set_item(l, 1, [1., 2.])
+    t = list_ops.tensor_list_gather(l, [1, 2], element_dtype=dtypes.float32)
+    self.assertAllEqual(self.evaluate(t), [[1., 2.], [0., 0.]])
+
+  def testGatherWithUninitializedTensorsInferShape(self):
+    self._testGatherWithUninitializedTensorsInferShape()
+
+  def testGatherWithUninitializedTensorsInferShapeGpu(self):
+    if not context.num_gpus():
+      return
+    with context.device("gpu:0"):
+      self._testGatherWithUninitializedTensorsInferShape()
+
+  def testGatherReservedListWithNoElementsAndPartialElementShapeFails(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        "Tried to gather uninitialized tensors from a"
+        " list with non-fully-defined element_shape"):
+      t = list_ops.tensor_list_gather(l, [0], element_dtype=dtypes.float32)
+      self.evaluate(t)
+
+  def testGatherUsingSpecifiedElementShape(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
+    t = gen_list_ops.tensor_list_gather(
+        l, [0, 1, 2], element_dtype=dtypes.float32, element_shape=[])
+    self.assertAllEqual(self.evaluate(t), np.zeros((3,)))
+
   def testScatterOutputListSize(self):
     c0 = constant_op.constant([1.0, 2.0])
     l = list_ops.tensor_list_scatter(
@@ -386,6 +500,18 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     dt = tape.gradient(loss, c0)
     self.assertAllEqual(self.evaluate(dt), [2., 4.])
 
+  def testScatterWithPartialReadGrad(self):
+    with backprop.GradientTape() as tape:
+      c0 = constant_op.constant([1.0, 2.0])
+      tape.watch(c0)
+      l = list_ops.tensor_list_scatter(
+          c0, [1, 0], ops.convert_to_tensor([], dtype=dtypes.int32))
+      t0 = list_ops.tensor_list_get_item(l, 0, element_dtype=dtypes.float32)
+      self.assertAllEqual(self.evaluate(t0), 2.0)
+      loss = t0 * t0
+    dt = tape.gradient(loss, c0)
+    self.assertAllEqual(self.evaluate(dt), [0., 4.])
+
   def testTensorListFromTensor(self):
     t = constant_op.constant([1.0, 2.0])
     l = list_ops.tensor_list_from_tensor(t, element_shape=[])
@@ -427,6 +553,50 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       self.assertAllEqual(self.evaluate(e), 10.0)
     self.assertAllEqual(self.evaluate(tape.gradient(e, t)), 2.0)
 
+  def testGetUninitializedTensorUseListElementShape(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=[], num_elements=3)
+    l = list_ops.tensor_list_set_item(l, 0, 5.)
+    e1 = list_ops.tensor_list_get_item(l, 1, element_dtype=dtypes.float32)
+    e2 = list_ops.tensor_list_get_item(l, 2, element_dtype=dtypes.float32)
+    self.assertEqual(self.evaluate(e1), 0.)
+    self.assertEqual(self.evaluate(e2), 0.)
+
+  def testGetUninitializedTensorUseSpecifiedElementShape(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
+    e0 = gen_list_ops.tensor_list_get_item(
+        l, 0, element_shape=[], element_dtype=dtypes.float32)
+    e1 = gen_list_ops.tensor_list_get_item(
+        l, 1, element_shape=[2, 3], element_dtype=dtypes.float32)
+    self.assertEqual(self.evaluate(e0), 0.)
+    self.assertAllEqual(self.evaluate(e1), np.zeros((2, 3)))
+
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=[None, 3], num_elements=3)
+    e1 = gen_list_ops.tensor_list_get_item(
+        l, 1, element_shape=[2, 3], element_dtype=dtypes.float32)
+    self.assertAllEqual(self.evaluate(e1), np.zeros((2, 3)))
+
+  def testGetUninitializedTensorWithInvalidElementShapeFails(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        "Trying to read an uninitialized tensor but "
+        "element_shape is not fully defined"):
+      e0 = list_ops.tensor_list_get_item(l, 0, element_dtype=dtypes.float32)
+      self.evaluate(e0)
+
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=[None, 2], num_elements=3)
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        r"Incompatible shapes during merge: \[1,3\] vs. \[\?,2\]"):
+      e0 = gen_list_ops.tensor_list_get_item(
+          l, 0, element_dtype=dtypes.float32, element_shape=[1, 3])
+      self.evaluate(e0)
+
   @test_util.run_deprecated_v1
   @test_util.enable_control_flow_v2
   def testSkipEagerSetItemIndexOutOfBounds(self):
diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
index e8af998a70..d0efb47a38 100644
--- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py
+++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
@@ -1281,13 +1281,23 @@ class TensorArrayTest(test.TestCase):
       grad_r0_vals = session.run(grad_r0)[0]
       self.assertAllEqual(grad_r0_vals, [1.0, 0.0])
 
-  # TODO(srbs): Figure out how to enable this. This is probably failing
-  # because we are trying to stack a TensorList with invalid tensors.
-  # That is because we do not receive gradients for all list indices.
-  # Figure out how TensorArray handles this.
-  def disabletestGradientWhenNotAllComponentsRead(self):
+  @test_util.run_deprecated_v1
+  def testSkipEagerGradientWhenNotAllComponentsRead(self):
     self._testGradientWhenNotAllComponentsRead()
 
+  @test_util.run_deprecated_v1
+  def testSkipEagerWriteButNotAllComponentsReadGrad(self):
+    with self.cached_session(use_gpu=True) as session:
+      x0 = constant_op.constant(5.0)
+      x1 = constant_op.constant(10.0)
+      ta = tensor_array_ops.TensorArray(
+          dtype=dtypes.float32, size=2).write(0, x0).write(1, x1)
+      r0 = ta.read(0)
+      # calculate (dr0/dx0, dr0/dx1).  since r0 = x0, gradients are (1, 0).
+      grad_r0_x1 = gradients_impl.gradients(ys=[r0], xs=[x0, x1], grad_ys=[1.0])
+      grad_r0_x1_vals = session.run(grad_r0_x1)
+      self.assertAllEqual(grad_r0_x1_vals, [1.0, 0.0])
+
   def _testTensorArrayUnpackDynamic(self):
     with self.cached_session(use_gpu=True) as sess:
       ta = tensor_array_ops.TensorArray(
@@ -1321,8 +1331,8 @@ class TensorArrayTest(test.TestCase):
     with self.cached_session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
           dtype=dtypes.float32, size=0, dynamic_size=False, infer_shape=False)
-      v2_msg = ("Tried to stack elements of a empty list with non-fully-defined"
-                " element_shape")
+      v2_msg = ("Tried to stack elements of an empty list with "
+                "non-fully-defined element_shape")
       v1_msg = (
           "TensorArray has size zero, but element shape <unknown> is not "
           "fully defined. Currently only static shapes are supported when "
@@ -1385,6 +1395,29 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual([10.0, -10.0], read_vals[1])
       self.assertAllEqual([[2.0, 3.0], [4.0, 5.0]], grad_vals[0])
 
+  @test_util.run_v1_only("b/117943489")
+  def testSkipEagerTensorArrayScatterPartialReadAndGradients(self):
+    with self.session(use_gpu=True) as session:
+      ta = tensor_array_ops.TensorArray(
+          dtype=dtypes.float32,
+          tensor_array_name="foo",
+          size=0,
+          dynamic_size=True)
+
+      indices = constant_op.constant([1, 8])
+      value = constant_op.constant([[1.0, -1.0], [10.0, -10.0]])
+
+      w = ta.scatter(indices, value)
+      r0 = w.read(1)
+
+      # Test combined gradients + aggregation of read(0)
+      grad = gradients_impl.gradients(
+          ys=[r0], xs=[value], grad_ys=[[2.0, 3.0]])[0]
+      read_val, grad_val = session.run([r0, grad])
+
+      self.assertAllEqual([1.0, -1.0], read_val)
+      self.assertAllEqual([[2.0, 3.0], [0.0, 0.0]], grad_val)
+
   @test_util.run_v1_only("b/118890905")
   def testTensorArrayWriteGatherAndGradients(self):
     with self.session(use_gpu=True) as session:
diff --git a/tensorflow/python/ops/list_ops.py b/tensorflow/python/ops/list_ops.py
index df928ea85d..af059a541e 100644
--- a/tensorflow/python/ops/list_ops.py
+++ b/tensorflow/python/ops/list_ops.py
@@ -71,6 +71,41 @@ def tensor_list_from_tensor(tensor, element_shape, name=None):
       name=name)
 
 
+def tensor_list_get_item(input_handle, index, element_dtype, name=None):
+  return gen_list_ops.tensor_list_get_item(
+      input_handle=input_handle,
+      index=index,
+      element_shape=-1,
+      element_dtype=element_dtype,
+      name=name)
+
+
+def tensor_list_pop_back(input_handle, element_dtype, name=None):
+  return gen_list_ops.tensor_list_pop_back(
+      input_handle=input_handle,
+      element_shape=-1,
+      element_dtype=element_dtype,
+      name=name)
+
+
+def tensor_list_gather(input_handle, indices, element_dtype, name=None):
+  return gen_list_ops.tensor_list_gather(
+      input_handle=input_handle,
+      indices=indices,
+      element_shape=-1,
+      element_dtype=element_dtype,
+      name=name)
+
+
+def tensor_list_stack(input_handle, element_dtype, num_elements=-1, name=None):
+  return gen_list_ops.tensor_list_stack(
+      input_handle=input_handle,
+      element_shape=-1,
+      element_dtype=element_dtype,
+      num_elements=num_elements,
+      name=name)
+
+
 def tensor_list_concat(input_handle, element_dtype, element_shape=None,
                        name=None):
   # Ignore the lengths output of TensorListConcat. It is only used during
@@ -110,7 +145,9 @@ def tensor_list_set_item(input_handle,
 @ops.RegisterGradient("TensorListPushBack")
 def _PushBackGrad(op, dresult):
   return gen_list_ops.tensor_list_pop_back(
-      dresult, element_dtype=op.get_attr("element_dtype"))
+      dresult,
+      element_shape=array_ops.shape(op.inputs[1]),
+      element_dtype=op.get_attr("element_dtype"))
 
 
 @ops.RegisterGradient("TensorListPopBack")
@@ -120,12 +157,12 @@ def _PopBackGrad(op, dlist, delement):
         element_dtype=delement.dtype,
         element_shape=gen_list_ops.tensor_list_element_shape(
             op.outputs[0], shape_type=dtypes.int32))
-  return gen_list_ops.tensor_list_push_back(dlist, delement)
+  return gen_list_ops.tensor_list_push_back(dlist, delement), None
 
 
 @ops.RegisterGradient("TensorListStack")
 def _TensorListStackGrad(unused_op, dtensor):
-  return tensor_list_from_tensor(dtensor, element_shape=dtensor.shape[1:])
+  return tensor_list_from_tensor(dtensor, element_shape=dtensor.shape[1:]), None
 
 
 @ops.RegisterGradient("TensorListConcat")
@@ -150,17 +187,21 @@ def _TensorListSplitGrad(op, dlist):
 @ops.RegisterGradient("TensorListFromTensor")
 def _TensorListFromTensorGrad(op, dlist):
   """Gradient for TensorListFromTensor."""
-  if op.inputs[0].shape.dims and op.inputs[0].shape.dims[0].value is not None:
-    num_elements = op.inputs[0].shape.dims[0].value
+  t = op.inputs[0]
+  if t.shape.dims and t.shape.dims[0].value is not None:
+    num_elements = t.shape.dims[0].value
   else:
     num_elements = None
   if dlist is None:
     dlist = empty_tensor_list(
-        element_dtype=op.inputs[0].dtype,
+        element_dtype=t.dtype,
         element_shape=gen_list_ops.tensor_list_element_shape(
             op.outputs[0], shape_type=dtypes.int32))
   tensor_grad = gen_list_ops.tensor_list_stack(
-      dlist, element_dtype=op.inputs[0].dtype, num_elements=num_elements)
+      dlist,
+      element_shape=array_ops.slice(array_ops.shape(t), [1], [-1]),
+      element_dtype=t.dtype,
+      num_elements=num_elements)
   shape_grad = None
   return tensor_grad, shape_grad
 
@@ -177,17 +218,22 @@ def _TensorListGetItemGrad(op, ditem):
       index=op.inputs[1],
       item=ditem)
   index_grad = None
-  return list_grad, index_grad
+  element_shape_grad = None
+  return list_grad, index_grad, element_shape_grad
 
 
 @ops.RegisterGradient("TensorListSetItem")
 def _TensorListSetItemGrad(op, dlist):
+  """Gradient function for TensorListSetItem."""
   _, index, item = op.inputs
   list_grad = gen_list_ops.tensor_list_set_item(
       dlist, index=index, item=array_ops.zeros_like(item))
   index_grad = None
   element_grad = gen_list_ops.tensor_list_get_item(
-      dlist, index, element_dtype=item.dtype)
+      dlist,
+      index,
+      element_shape=array_ops.shape(item),
+      element_dtype=item.dtype)
   return list_grad, index_grad, element_grad
 
 
@@ -200,7 +246,8 @@ def _TensorListResizeGrad(op, dlist):
 
 @ops.RegisterGradient("TensorListGather")
 def _TensorListGatherGrad(op, dtensor):
-  input_list, indices = op.inputs
+  """Gradient function for TensorListGather."""
+  input_list, indices, _ = op.inputs
   dlist = gen_list_ops.tensor_list_scatter(
       tensor=dtensor,
       indices=indices,
@@ -209,14 +256,17 @@ def _TensorListGatherGrad(op, dtensor):
   # so we manually resize it to match the size of the input list.
   input_list_size = gen_list_ops.tensor_list_length(input_list)
   dlist = gen_list_ops.tensor_list_resize(dlist, input_list_size)
-  return dlist, None
+  return dlist, None, None
 
 
 @ops.RegisterGradient("TensorListScatter")
 def _TensorListScatterGrad(op, dlist):
   t, indices, _ = op.inputs
   return gen_list_ops.tensor_list_gather(
-      dlist, indices, element_dtype=t.dtype), None, None
+      dlist,
+      indices,
+      element_shape=array_ops.slice(array_ops.shape(t), [1], [-1]),
+      element_dtype=t.dtype), None, None
 
 
 def _build_element_shape(shape):
-- 
GitLab


From 8eb2a426bd373a86375eb736f1c2f6bb4a7f87e6 Mon Sep 17 00:00:00 2001
From: Dayananda-V <dayanandasiet@gmail.com>
Date: Thu, 24 Jan 2019 11:32:53 +0530
Subject: [PATCH 1171/2345] replace tf.initialize_all_variables() to
 tf.global_variables_initializer()

---
 tensorflow/compiler/aot/tests/make_test_graphs.py           | 4 ++--
 tensorflow/compiler/tests/dense_layer_test.py               | 6 +++---
 .../boosted_trees/python/kernel_tests/model_ops_test.py     | 2 +-
 .../contrib/layers/python/layers/feature_column_ops_test.py | 2 +-
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/tensorflow/compiler/aot/tests/make_test_graphs.py b/tensorflow/compiler/aot/tests/make_test_graphs.py
index 64b861a730..7bac79ec06 100644
--- a/tensorflow/compiler/aot/tests/make_test_graphs.py
+++ b/tensorflow/compiler/aot/tests/make_test_graphs.py
@@ -50,7 +50,7 @@ def tfadd_with_ckpt(out_dir):
   y = variables.VariableV1(constant_op.constant([0]), name='y_saved')
   math_ops.add(x, y, name='x_y_sum')
 
-  init_op = variables.initialize_all_variables()
+  init_op = variables.global_variables_initializer()
   saver = saver_lib.Saver(write_version=saver_pb2.SaverDef.V1)
   with session.Session() as sess:
     sess.run(init_op)
@@ -65,7 +65,7 @@ def tfadd_with_ckpt_saver(out_dir):
   y = variables.VariableV1(constant_op.constant([0]), name='y_saved')
   math_ops.add(x, y, name='x_y_sum')
 
-  init_op = variables.initialize_all_variables()
+  init_op = variables.global_variables_initializer()
   saver = saver_lib.Saver(name='abcprefix', write_version=saver_pb2.SaverDef.V1)
   with session.Session() as sess:
     sess.run(init_op)
diff --git a/tensorflow/compiler/tests/dense_layer_test.py b/tensorflow/compiler/tests/dense_layer_test.py
index bf5ea7b1fb..b7d08df9f7 100644
--- a/tensorflow/compiler/tests/dense_layer_test.py
+++ b/tensorflow/compiler/tests/dense_layer_test.py
@@ -72,7 +72,7 @@ class DenseLayerTest(test.TestCase):
       x = array_ops.placeholder(shape=[None, None, 3], dtype=np.float32)
       y = layers.dense(x, 3)
 
-      self.evaluate(variables.initialize_all_variables())
+      self.evaluate(variables.global_variables_initializer())
       run_metadata = config_pb2.RunMetadata()
       test_utils.RunWithWarmup(
           sess,
@@ -97,7 +97,7 @@ class DenseLayerTest(test.TestCase):
       with jit_scope():
         y = layers.dense(x, 3)
 
-      self.evaluate(variables.initialize_all_variables())
+      self.evaluate(variables.global_variables_initializer())
       run_metadata = config_pb2.RunMetadata()
       test_utils.RunWithWarmup(
           sess,
@@ -126,7 +126,7 @@ class DenseLayerTest(test.TestCase):
       with jit_scope():
         y = layers.dense(x, 3)
 
-      self.evaluate(variables.initialize_all_variables())
+      self.evaluate(variables.global_variables_initializer())
       run_metadata = config_pb2.RunMetadata()
       test_utils.RunWithWarmup(
           sess,
diff --git a/tensorflow/contrib/boosted_trees/python/kernel_tests/model_ops_test.py b/tensorflow/contrib/boosted_trees/python/kernel_tests/model_ops_test.py
index 42d69645ac..aa3f24f08a 100644
--- a/tensorflow/contrib/boosted_trees/python/kernel_tests/model_ops_test.py
+++ b/tensorflow/contrib/boosted_trees/python/kernel_tests/model_ops_test.py
@@ -227,7 +227,7 @@ class ModelOpsTest(test_util.TensorFlowTestCase):
             tree_ensemble_config=tree_ensemble_config.SerializeToString(),
             name="restore_tree")
         resources.initialize_resources(resources.shared_resources()).run()
-        variables.initialize_all_variables().run()
+        variables.global_variables_initializer().run()
         my_saver = saver.Saver()
 
         # Add the second tree and replace the ensemble of the handle.
diff --git a/tensorflow/contrib/layers/python/layers/feature_column_ops_test.py b/tensorflow/contrib/layers/python/layers/feature_column_ops_test.py
index 7e6eafaa0d..00e41026d0 100644
--- a/tensorflow/contrib/layers/python/layers/feature_column_ops_test.py
+++ b/tensorflow/contrib/layers/python/layers/feature_column_ops_test.py
@@ -1757,7 +1757,7 @@ class WeightedSumTest(test.TestCase):
       logits_core = fc_core.linear_model(features, [movies])
 
       with self.cached_session() as sess:
-        variables_lib.initialize_all_variables().run()
+        variables_lib.global_variables_initializer().run()
         lookup_ops.tables_initializer().run()
 
         weights = column_to_variable[movies][0]
-- 
GitLab


From ee5c4383f94ef70f5178b7edee9f5fd1cfe3a187 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 23 Jan 2019 22:46:08 -0800
Subject: [PATCH 1172/2345] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 230663964

---
 tensorflow/go/op/wrappers.go | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 666ebe33df..8e2c98fb07 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -16317,7 +16317,7 @@ func TensorListStackNumElements(value int64) TensorListStackAttr {
 // tensor: the gathered result
 // num_elements: optional. If not -1, the number of elements in the list.
 //
-func TensorListStack(scope *Scope, input_handle tf.Output, element_dtype tf.DataType, optional ...TensorListStackAttr) (tensor tf.Output) {
+func TensorListStack(scope *Scope, input_handle tf.Output, element_shape tf.Output, element_dtype tf.DataType, optional ...TensorListStackAttr) (tensor tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -16328,7 +16328,7 @@ func TensorListStack(scope *Scope, input_handle tf.Output, element_dtype tf.Data
 	opspec := tf.OpSpec{
 		Type: "TensorListStack",
 		Input: []tf.Input{
-			input_handle,
+			input_handle, element_shape,
 		},
 		Attrs: attrs,
 	}
@@ -17359,7 +17359,7 @@ func StringToHashBucketFast(scope *Scope, input tf.Output, num_buckets int64) (o
 // tensor: the withdrawn last element of the list
 // element_dtype: the type of elements in the list
 // element_shape: the shape of the output tensor
-func TensorListPopBack(scope *Scope, input_handle tf.Output, element_dtype tf.DataType) (output_handle tf.Output, tensor tf.Output) {
+func TensorListPopBack(scope *Scope, input_handle tf.Output, element_shape tf.Output, element_dtype tf.DataType) (output_handle tf.Output, tensor tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -17367,7 +17367,7 @@ func TensorListPopBack(scope *Scope, input_handle tf.Output, element_dtype tf.Da
 	opspec := tf.OpSpec{
 		Type: "TensorListPopBack",
 		Input: []tf.Input{
-			input_handle,
+			input_handle, element_shape,
 		},
 		Attrs: attrs,
 	}
@@ -28621,7 +28621,7 @@ func TensorListElementShape(scope *Scope, input_handle tf.Output, shape_type tf.
 // item: the element at that position
 //
 //
-func TensorListGetItem(scope *Scope, input_handle tf.Output, index tf.Output, element_dtype tf.DataType) (item tf.Output) {
+func TensorListGetItem(scope *Scope, input_handle tf.Output, index tf.Output, element_shape tf.Output, element_dtype tf.DataType) (item tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -28629,7 +28629,7 @@ func TensorListGetItem(scope *Scope, input_handle tf.Output, index tf.Output, el
 	opspec := tf.OpSpec{
 		Type: "TensorListGetItem",
 		Input: []tf.Input{
-			input_handle, index,
+			input_handle, index, element_shape,
 		},
 		Attrs: attrs,
 	}
@@ -28781,7 +28781,7 @@ func TensorListSetItem(scope *Scope, input_handle tf.Output, index tf.Output, it
 // input_handle: The input tensor list.
 // indices: The indices used to index into the list.
 // values: The tensor.
-func TensorListGather(scope *Scope, input_handle tf.Output, indices tf.Output, element_dtype tf.DataType) (values tf.Output) {
+func TensorListGather(scope *Scope, input_handle tf.Output, indices tf.Output, element_shape tf.Output, element_dtype tf.DataType) (values tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -28789,7 +28789,7 @@ func TensorListGather(scope *Scope, input_handle tf.Output, indices tf.Output, e
 	opspec := tf.OpSpec{
 		Type: "TensorListGather",
 		Input: []tf.Input{
-			input_handle, indices,
+			input_handle, indices, element_shape,
 		},
 		Attrs: attrs,
 	}
-- 
GitLab


From 32233e10474546515a87c9e4b050d7ce15b0675c Mon Sep 17 00:00:00 2001
From: Dayananda-V <dayanandasiet@gmail.com>
Date: Thu, 24 Jan 2019 12:39:47 +0530
Subject: [PATCH 1173/2345] TF Contrib typo-error fix

---
 tensorflow/contrib/cmake/README.md  | 6 +++---
 tensorflow/contrib/gan/README.md    | 2 +-
 tensorflow/contrib/gdr/README.md    | 2 +-
 tensorflow/contrib/ignite/README.md | 8 ++++----
 tensorflow/contrib/learn/README.md  | 2 +-
 5 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/tensorflow/contrib/cmake/README.md b/tensorflow/contrib/cmake/README.md
index b4f4b028f6..60ee1b4b3f 100644
--- a/tensorflow/contrib/cmake/README.md
+++ b/tensorflow/contrib/cmake/README.md
@@ -154,12 +154,12 @@ suitable interface for project configuration and dependency setting.
 5.  Click on `Configure`, a new window will be prompted out, specify the
     generator mode for the project generation. For Windows, choose `Visual
     Studio <version> <year> Win64`, for Linux, choose `Unix Makefiles`, then
-    press `Finish`. Wait for a moment, the default project dependecy would
+    press `Finish`. Wait for a moment, the default project dependency would
     automatically generate.
 6.  There are a few options that you can customize your own build. **The setting
     here is crucial for a successful build, please check all items carefully.**
 
-    *   `tensorflow_BUILD_ALL_KERNELS` should alway be `on`
+    *   `tensorflow_BUILD_ALL_KERNELS` should always be `on`
     *   `tensorflow_BUILD_CC_EXAMPLE` is default to be `on`. This can help you
         to test build (optional)
     *   `tensorflow_BUILD_CONTRIB_KERNELS` is default to be `on`, but it won't
@@ -278,7 +278,7 @@ suitable interface for project configuration and dependency setting.
     `make -sj<number-of-threads> install`
 
     Where `<number-of-threads>` is the threads used for the compilation, change
-    to any integer less or equal to your computer's maxiumum thread number.
+    to any integer less or equal to your computer's maximum thread number.
 
     Headers are discretely located in the build folders. Tensorflow library can
     be found at `<path-to-build>`, namely `tensorflow.so` (Linux) or
diff --git a/tensorflow/contrib/gan/README.md b/tensorflow/contrib/gan/README.md
index db7dc51daa..4305d4b084 100644
--- a/tensorflow/contrib/gan/README.md
+++ b/tensorflow/contrib/gan/README.md
@@ -59,7 +59,7 @@ These include the following main pieces (explained in detail below).
 *   [examples](https://github.com/tensorflow/models/tree/master/research/gan/)
     and [tutorial](http://https://github.com/tensorflow/models/tree/master/research/gan/tutorial.ipynb): See examples of how to use TF-GAN
     to make GAN training easier, or use the more complicated examples to
-    jumpstart your own project. These include unconditional and conditional
+    jump-start your own project. These include unconditional and conditional
     GANs, InfoGANs, adversarial losses on existing networks, and image-to-image
     translation.
 
diff --git a/tensorflow/contrib/gdr/README.md b/tensorflow/contrib/gdr/README.md
index 8242d93f12..67226624c7 100644
--- a/tensorflow/contrib/gdr/README.md
+++ b/tensorflow/contrib/gdr/README.md
@@ -114,7 +114,7 @@ Caveats
 
 In current implementation, only tensors that reside in host memory or in GPU memory such that the GPU is adjacent to an RDMA capable NIC will use direct RDMA as its transport. When RDMA is available but not GDR, a temporary tensor copy on host memory will be used as RDMA source/destination (and copied from/to the target device). When there is no RDMA device present, it can even fallback to the original gRPC runtime. While it is theoretically possible to mix GDR enabled TF with non-GDR deployments in the same job, make sure the environment is properly setup so the GDR mode is enabled whenever possible (i.e. do not fall back to gRPC when it is not absolutely necessary).
 
-In the original design (as in the reference), tensor buffers are only registered to NIC when we could determine that the tensor will be either a source of Send or a sink of Recv across physical machine boundary. However, to implement the precise allocations, we need to change all the devices to possibly return a NIC compatible allocator. As GDR is currently in contrib, we would like to avoid the unnecessary code disruption to the TF core, so we allocate all tensors from NIC-registered buffers using a BFC allocator. This behaviour is similar to the effect of enabling the extra GPU option `force_gpu_compatible`, which allocate all host tensors in GPU-registered buffers no matter they will be transferred from/to GPUs or not.
+In the original design (as in the reference), tensor buffers are only registered to NIC when we could determine that the tensor will be either a source of Send or a sink of Recv across physical machine boundary. However, to implement the precise allocations, we need to change all the devices to possibly return a NIC compatible allocator. As GDR is currently in contrib, we would like to avoid the unnecessary code disruption to the TF core, so we allocate all tensors from NIC-registered buffers using a BFC allocator. This behavior is similar to the effect of enabling the extra GPU option `force_gpu_compatible`, which allocate all host tensors in GPU-registered buffers no matter they will be transferred from/to GPUs or not.
 
 Reference
 ===
diff --git a/tensorflow/contrib/ignite/README.md b/tensorflow/contrib/ignite/README.md
index 7b30e1776f..3cf0870686 100644
--- a/tensorflow/contrib/ignite/README.md
+++ b/tensorflow/contrib/ignite/README.md
@@ -117,7 +117,7 @@ Using this ability we can calculate gradients on the nodes the data is stored on
 
 Apache Ignite uses horizontal partitioning to store data in distributed cluster. When we create Apache Ignite cache (or table in terms of SQL), we can specify the number of partitions the data will be partitioned on. For example, if an Apache Ignite cluster consists of 10 machines and we create cache with 10 partitions, then every machine will maintain approximately one data partition.
 
-Ignite Dataset allows using these two aspects of distributed neural network training (using TensorFlow) and Apache Ignite partitioning. Ignite Dataset is a computation graph operation that can be performed on a remote worker. The remote worker can override Ignite Dataset parameters (such as `host`, `port` or `part`) by setting correstondent environment variables for worker process (such as `IGNITE_DATASET_HOST`, `IGNITE_DATASET_PORT` or `IGNITE_DATASET_PART`). Using this overriding approach, we can assign a specific partition to every worker so that one worker handles one partition and, at the same time, transparently work with single dataset.
+Ignite Dataset allows using these two aspects of distributed neural network training (using TensorFlow) and Apache Ignite partitioning. Ignite Dataset is a computation graph operation that can be performed on a remote worker. The remote worker can override Ignite Dataset parameters (such as `host`, `port` or `part`) by setting correspondent environment variables for worker process (such as `IGNITE_DATASET_HOST`, `IGNITE_DATASET_PORT` or `IGNITE_DATASET_PART`). Using this overriding approach, we can assign a specific partition to every worker so that one worker handles one partition and, at the same time, transparently work with single dataset.
 
 ```python
 >>> import tensorflow as tf
@@ -162,7 +162,7 @@ TensorBoard runs in a different process or machine.
 
 ### SSL Connection
 
-Apache Ignite allows to protect data transfer channels by [SSL](https://en.wikipedia.org/wiki/Transport_Layer_Security) and authentification. Ignite Dataset supports both SSL connection with and without authntication. For more information, please refer to the [Apache Ignite SSL/TLS](https://apacheignite.readme.io/docs/ssltls) documentation.
+Apache Ignite allows to protect data transfer channels by [SSL](https://en.wikipedia.org/wiki/Transport_Layer_Security) and authentication. Ignite Dataset supports both SSL connection with and without authentication. For more information, please refer to the [Apache Ignite SSL/TLS](https://apacheignite.readme.io/docs/ssltls) documentation.
 
 ```python
 >>> import tensorflow as tf
@@ -187,7 +187,7 @@ Following examples will help you to easily start working with this module.
 
 The simplest way to try Ignite Dataset is to run a
 [Docker](https://www.docker.com/) container with Apache Ignite and loaded
-[MNIST](http://yann.lecun.com/exdb/mnist/) data and after start interruct with
+[MNIST](http://yann.lecun.com/exdb/mnist/) data and after start interrupt with
 it using Ignite Dataset. Such container is available on Docker Hub:
 [dmitrievanthony/ignite-with-mnist](https://hub.docker.com/r/dmitrievanthony/ignite-with-mnist/).
 You need to start this container on your machine:
@@ -204,7 +204,7 @@ After that you will be able to work with it following way:
 
 The simplest way to try IGFS with TensorFlow is to run
 [Docker](https://www.docker.com/) container with Apache Ignite and enabled IGFS
-and then interruct with it using TensorFlow
+and then interrupt with it using TensorFlow
 [tf.gfile](https://www.tensorflow.org/api_docs/python/tf/gfile). Such container
 is available on Docker Hub:
 [dmitrievanthony/ignite-with-igfs](https://hub.docker.com/r/dmitrievanthony/ignite-with-igfs/).
diff --git a/tensorflow/contrib/learn/README.md b/tensorflow/contrib/learn/README.md
index b0bff915a9..20288be1b6 100644
--- a/tensorflow/contrib/learn/README.md
+++ b/tensorflow/contrib/learn/README.md
@@ -121,7 +121,7 @@ Switch to `tf.estimator.train_and_evaluate`. Some differences:
   replacement for `tf.contrib.learn.make_export_strategy`. If you want to export
   only at the end of training  use `tf.estimator.FinalExporter`.
 * If the `TF_CONFIG` environment variable is constructed manually, please read
-  the `train_and_evaluate` documentation for the new requirementds (in
+  the `train_and_evaluate` documentation for the new requirements (in
   particular, the chief node and evaluator node).
 
 ## Others Classes and Functions
-- 
GitLab


From 6521c70a29a3ee992d52a51210250b138f3a65a0 Mon Sep 17 00:00:00 2001
From: Thomas Joerg <tjoerg@google.com>
Date: Thu, 24 Jan 2019 01:02:11 -0800
Subject: [PATCH 1174/2345] [XLA:GPU] Disable producer-consumer multi-output
 fusion in the GpuInstructionFusion pass.

The feature is in a half-baked state and almost never triggers in practice. Producer-consumer multi-output fusion has been implemented in the GpuMultiOutputFusion pass instead.

PiperOrigin-RevId: 230676936
---
 .../xla/service/gpu/instruction_fusion.cc     |  17 +-
 .../service/gpu/instruction_fusion_test.cc    | 197 +-----------------
 2 files changed, 4 insertions(+), 210 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/instruction_fusion.cc b/tensorflow/compiler/xla/service/gpu/instruction_fusion.cc
index 6151dd8ff4..f07141029c 100644
--- a/tensorflow/compiler/xla/service/gpu/instruction_fusion.cc
+++ b/tensorflow/compiler/xla/service/gpu/instruction_fusion.cc
@@ -282,22 +282,7 @@ bool GpuInstructionFusion::ShouldFuse(HloInstruction* consumer,
 
 bool GpuInstructionFusion::ShouldFuseIntoMultiOutput(HloInstruction* consumer,
                                                      int64 operand_index) {
-  const HloInstruction* producer = consumer->operand(operand_index);
-  // The IR emitter has limited support for non-loop fusions with multi output
-  // at present.
-  // TODO(tjoerg): Relax this constraint to allow for arbitraty kinds of fusion.
-  if (consumer->opcode() == HloOpcode::kFusion &&
-      consumer->fusion_kind() != HloInstruction::FusionKind::kLoop) {
-    return false;
-  }
-  // Multi-output fusion requires instructions with compatible shapes.
-  if (!ShapeUtil::Compatible(producer->shape(), consumer->shape())) {
-    return false;
-  }
-  // TODO(tjoerg): Stop calling `ShouldFuse` to relax the criteria for
-  // multi-output fusion. In particular, do not check whether an instruction is
-  // expensive to duplicate, since this doesn't matter here.
-  return GpuInstructionFusion::ShouldFuse(consumer, operand_index);
+  return false;
 }
 
 HloInstruction::FusionKind GpuInstructionFusion::ChooseKind(
diff --git a/tensorflow/compiler/xla/service/gpu/instruction_fusion_test.cc b/tensorflow/compiler/xla/service/gpu/instruction_fusion_test.cc
index 688604cd36..a05ab86cf7 100644
--- a/tensorflow/compiler/xla/service/gpu/instruction_fusion_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/instruction_fusion_test.cc
@@ -506,202 +506,11 @@ TEST_F(InstructionFusionTest, MultiOutputFusion) {
     })")
                     .ValueOrDie();
 
-  ASSERT_TRUE(GpuInstructionFusion(/*may_duplicate=*/true)
-                  .Run(module.get())
-                  .ValueOrDie());
-  SCOPED_TRACE(module->ToString());
-
-  // Expect that there is one multi-output fusion and subtract has not been
-  // duplicated.
-  EXPECT_EQ(Count(*module, HloOpcode::kFusion), 1);
-  EXPECT_EQ(Count(*module, HloOpcode::kSubtract), 1);
-  TF_ASSERT_OK_AND_ASSIGN(
-      const HloInstruction* fusion,
-      FindHloInstruction(*module->entry_computation(), HloOpcode::kFusion));
-  EXPECT_THAT(
-      fusion->fused_expression_root(),
-      op::Tuple(op::Add(op::Subtract(), op::Parameter()), op::Subtract()));
-}
-
-TEST_F(InstructionFusionTest, MultiOutputFusionExpensiveOp) {
-  // tanh --> add --> tuple
-  //  \---------------/
-  auto module = ParseHloString(R"(
-    HloModule test_module
-    ENTRY OutputFusion {
-     p0 = f32[4,3]{1,0} parameter(0)
-     p1 = f32[4,3]{1,0} parameter(1)
-     tanh = f32[4,3]{1,0} tanh(p0)
-     add = f32[4,3]{1,0} add(tanh, p1)
-     ROOT tuple = (f32[4,3]{1,0}, f32[4,3]{1,0}) tuple(tanh, add)
-    })")
-                    .ValueOrDie();
-
-  // TODO(tjoerg): Allow multi-output fusion for expensive operations like tanh.
+  // Multi-output fusion is disabled here and performed in the
+  // GpuMultiOutputFusion pass instead.
   ASSERT_FALSE(GpuInstructionFusion(/*may_duplicate=*/true)
                    .Run(module.get())
-                   .ValueOrDie())
-      << module->ToString();
-}
-
-TEST_F(InstructionFusionTest, MultiOutputFusion2) {
-  // sub --> add1 --\--------\
-  //  \----------> add2 --> tuple
-  auto module = ParseHloString(R"(
-    HloModule test_module
-    ENTRY OutputFusion {
-     p0 = f32[4,3]{1,0} parameter(0)
-     p1 = f32[4,3]{1,0} parameter(1)
-     p2 = f32[4,3]{1,0} parameter(2)
-     sub = f32[4,3]{1,0} subtract(p0, p2)
-     add1 = f32[4,3]{1,0} add(sub, p1)
-     add2 = f32[4,3]{1,0} add(sub, add1)
-     ROOT tuple = (f32[4,3]{1,0}) tuple(add1, add2)
-    })")
-                    .ValueOrDie();
-
-  ASSERT_TRUE(GpuInstructionFusion(/*may_duplicate=*/true)
-                  .Run(module.get())
-                  .ValueOrDie());
-  SCOPED_TRACE(module->ToString());
-
-  // Expect that there is one multi-output fusion and subtract has not been
-  // duplicated.
-  EXPECT_EQ(Count(*module, HloOpcode::kFusion), 1);
-  EXPECT_EQ(Count(*module, HloOpcode::kSubtract), 1);
-  TF_ASSERT_OK_AND_ASSIGN(
-      const HloInstruction* fusion,
-      FindHloInstruction(*module->entry_computation(), HloOpcode::kFusion));
-  EXPECT_THAT(fusion->fused_expression_root(),
-              op::Tuple(op::Add(op::Subtract(), op::Add()),
-                        op::Add(op::Subtract(), op::Parameter())));
-}
-
-TEST_F(InstructionFusionTest, MultiOutputFusion3) {
-  // sub --> add1 ----\--------\
-  //  \ --> add2 --> add3 --> tuple
-  auto module = ParseHloString(R"(
-    HloModule test_module
-    ENTRY OutputFusion {
-     p0 = f32[4,3]{1,0} parameter(0)
-     p1 = f32[4,3]{1,0} parameter(1)
-     p2 = f32[4,3]{1,0} parameter(2)
-     p3 = f32[4,3]{1,0} parameter(3)
-     sub = f32[4,3]{1,0} subtract(p0, p2)
-     add1 = f32[4,3]{1,0} add(sub, p1)
-     add2 = f32[4,3]{1,0} add(p2, sub)
-     add3 = f32[4,3]{1,0} add(add1, add2)
-     ROOT tuple = (f32[4,3]{1,0}) tuple(add3, add2)
-    })")
-                    .ValueOrDie();
-
-  ASSERT_TRUE(GpuInstructionFusion(/*may_duplicate=*/true)
-                  .Run(module.get())
-                  .ValueOrDie());
-  SCOPED_TRACE(module->ToString());
-
-  // Expect that there is one multi-output fusion and subtract has not been
-  // duplicated.
-  EXPECT_EQ(Count(*module, HloOpcode::kFusion), 1);
-  EXPECT_EQ(Count(*module, HloOpcode::kSubtract), 1);
-  TF_ASSERT_OK_AND_ASSIGN(
-      const HloInstruction* fusion,
-      FindHloInstruction(*module->entry_computation(), HloOpcode::kFusion));
-  EXPECT_THAT(fusion->fused_expression_root(),
-              op::Tuple(op::Add(op::Add(), op::Add()),
-                        op::Add(op::Parameter(), op::Subtract())));
-}
-
-TEST_F(InstructionFusionTest, NoCyclesDueToMultiOutputFusion) {
-  // sub --> mul ---\
-  //  \--> call --> add --> tuple
-  auto module = ParseHloString(R"(
-  HloModule test_module
-  ENTRY OutputFusion {
-    c = f32[] constant(42)
-    p0 = f32[4,3]{1,0} parameter(0)
-    p1 = f32[4,3]{1,0} parameter(1)
-    sub = f32[4,3]{1,0} subtract(p0, p1)
-    mul = f32[4,3]{1,0} multiply(sub, c)
-    call = f32[4,3]{1,0} custom-call(sub), custom_call_target="foo"
-    add = f32[4,3]{1,0} add(mul, call)
-    ROOT tuple = (f32[4,3]{1,0}) tuple(add)
-  })")
-                    .ValueOrDie();
-
-  ASSERT_TRUE(GpuInstructionFusion(/*may_duplicate=*/true)
-                  .Run(module.get())
-                  .ValueOrDie());
-  // Visit instructions in post order to detect cycles.
-  // TODO(tjoerg): Add cycle detection to the HloVerifier.
-  class DummyVisitor : public DfsHloVisitorWithDefault {
-   public:
-    DummyVisitor() {}
-    Status DefaultAction(HloInstruction* /*hlo_instruction*/) override {
-      return Status::OK();
-    }
-  } visitor;
-  for (const HloComputation* computation : module->MakeComputationPostOrder()) {
-    // Accept will return a FailedPrecondition when a cycle is detected.
-    EXPECT_TRUE(computation->root_instruction()->Accept(&visitor).ok());
-  }
-}
-
-TEST_F(InstructionFusionTest, NoMultiOutputFusionWithIncompatibleShapes) {
-  // sub[2,3] --> add[4,3] --> tuple([2,3], [4,3])
-  //  \-------------------------/
-  auto module = ParseHloString(R"(
-    HloModule test_module
-    ENTRY OutputFusion {
-     p0 = f32[2,3]{1,0} parameter(0)
-     p1 = f32[4,3]{1,0} parameter(1)
-     p2 = f32[2,3]{1,0} parameter(2)
-     sub = f32[2,3]{1,0} subtract(p0, p2)
-     add = f32[4,3]{1,0} add(sub, p1)
-     ROOT tuple = (f32[2,3]{1,0}, f32[4,3]{1,0}) tuple(sub, add)
-    })")
-                    .ValueOrDie();
-
-  // Multi-output fusion requires shapes to be compatible. Since `sub` and `add`
-  // have incompatible shapes, expect that no multi-output fusion happens.
-  ASSERT_FALSE(GpuInstructionFusion(/*may_duplicate=*/true)
-                   .Run(module.get())
-                   .ValueOrDie())
-      << module->ToString();
-}
-
-TEST_F(InstructionFusionTest, FuseIntoInputFusionInstruction) {
-  auto module = ParseHloString(R"(
-  HloModule test_module
-
-  add_computation {
-    add_lhs = f32[] parameter(0)
-    add_rhs = f32[] parameter(1)
-    ROOT add_root = f32[] add(add_lhs, add_rhs)
-  }
-
-  fused_computation {
-    p1 = f32[10] parameter(0)
-    zero = f32[] constant(0)
-    ROOT f2_root = f32[] reduce(p1, zero), dimensions={0},
-           to_apply=add_computation
-  }
-
-  ENTRY entry {
-    p0 = f32[10] parameter(0)
-    mul = f32[10] multiply(p0, p0)
-    fusion = f32[] fusion(mul), kind=kInput, calls=fused_computation
-    ROOT tuple = (f32[10], f32[]) tuple(fusion, mul)
-  })")
-                    .ValueOrDie();
-
-  // Multi-output fusion is not supported for non-loop fusions at present. Since
-  // `fused_computation` is a input fusion, expect no multi-output fusion to
-  // happen.
-  ASSERT_FALSE(GpuInstructionFusion(/*may_duplicate=*/true)
-                   .Run(module.get())
-                   .ValueOrDie())
-      << module->ToString();
+                   .ValueOrDie());
 }
 
 TEST_F(InstructionFusionTest, FuseScalarConstant) {
-- 
GitLab


From fc92b27177692e91e8bf0abe928d34ba3a7a893d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 24 Jan 2019 01:02:53 -0800
Subject: [PATCH 1175/2345] compat: Update forward compatibility horizon to
 2019-01-24

PiperOrigin-RevId: 230677271
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index c8ac0bee44..919b47129f 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 23)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 24)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 6be9e9b573bfe09aefde442288104e962d62f5a7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 24 Jan 2019 01:24:01 -0800
Subject: [PATCH 1176/2345] [TF:XLA] Disable more XLA:GPU tests that were
 failing.

All tests that XLA:GPU is failing on are now disabled with the exception of kernel_tests:control_flow_ops_py_test_xla_gpu which is failing without XLA also.

PiperOrigin-RevId: 230679574
---
 tensorflow/python/kernel_tests/BUILD          |  4 +++
 .../python/kernel_tests/array_ops_test.py     |  2 ++
 .../python/kernel_tests/bias_op_test.py       |  3 ++
 .../python/kernel_tests/conv_ops_test.py      | 31 +++++++++++++++++++
 .../kernel_tests/depthwise_conv_op_test.py    |  1 +
 .../python/kernel_tests/init_ops_test.py      |  1 +
 tensorflow/python/kernel_tests/linalg/BUILD   |  1 +
 tensorflow/python/kernel_tests/signal/BUILD   |  2 ++
 .../kernel_tests/signal/dct_ops_test.py       |  1 +
 .../kernel_tests/signal/fft_ops_test.py       | 10 ++++++
 .../kernel_tests/signal/spectral_ops_test.py  |  3 ++
 11 files changed, 59 insertions(+)

diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 2271d1f9f7..aef4354ee0 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -1435,6 +1435,7 @@ cuda_py_test(
         "noasan",  # times out
         "optonly",  # times out
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1524,6 +1525,7 @@ cuda_py_test(
         "//tensorflow/python:nn_grad",
         "//tensorflow/python:nn_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2754,6 +2756,7 @@ cuda_py_test(
     tags = [
         "optonly",  # times out
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2771,6 +2774,7 @@ cuda_py_test(
     ],
     # TODO(b/118842098): Re-enable this test in Kokoro.
     tags = ["no_oss"],
+    xla_enable_strict_auto_jit = True,
 )
 
 tf_py_test(
diff --git a/tensorflow/python/kernel_tests/array_ops_test.py b/tensorflow/python/kernel_tests/array_ops_test.py
index f4c442b7b1..5ed549f268 100644
--- a/tensorflow/python/kernel_tests/array_ops_test.py
+++ b/tensorflow/python/kernel_tests/array_ops_test.py
@@ -1050,10 +1050,12 @@ class SliceAssignTest(test_util.TensorFlowTestCase):
     checker2[None] = [6]  # new axis
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testSliceAssign(self):
     self.doTestSliceAssign(use_resource=False)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testSliceAssignResource(self):
     self.doTestSliceAssign(use_resource=True)
 
diff --git a/tensorflow/python/kernel_tests/bias_op_test.py b/tensorflow/python/kernel_tests/bias_op_test.py
index 8d04da6dbd..e89e2400f4 100644
--- a/tensorflow/python/kernel_tests/bias_op_test.py
+++ b/tensorflow/python/kernel_tests/bias_op_test.py
@@ -206,6 +206,7 @@ class BiasAddTest(test.TestCase):
         self._testGradient(np_input, bias, dtype, data_format, use_gpu)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testGradientTensor3D(self):
     for (data_format, use_gpu) in [("NHWC", False), ("NHWC", True),
                                    ("NCHW", False), ("NCHW", True)]:
@@ -227,6 +228,7 @@ class BiasAddTest(test.TestCase):
         self._testGradient(np_input, bias, dtype, data_format, use_gpu)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testGradientTensor5D(self):
     for (data_format, use_gpu) in [("NHWC", False), ("NHWC", True),
                                    ("NCHW", False), ("NCHW", True)]:
@@ -244,6 +246,7 @@ class BiasAddTest(test.TestCase):
       self._testAll(np.random.randn(*shape), np.random.randn(shape[-1]))
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testEmptyGradient(self):
     for (data_format, use_gpu) in ("NHWC", False), ("NHWC", True):
       for shape in (0, 0), (2, 0), (0, 2):
diff --git a/tensorflow/python/kernel_tests/conv_ops_test.py b/tensorflow/python/kernel_tests/conv_ops_test.py
index 390a547fb6..d9b908be1e 100644
--- a/tensorflow/python/kernel_tests/conv_ops_test.py
+++ b/tensorflow/python/kernel_tests/conv_ops_test.py
@@ -574,6 +574,7 @@ class Conv2DTest(test.TestCase):
         padding="VALID")
 
   @test_util.run_in_graph_and_eager_modes()
+  @test_util.disable_xla("This test never passed for XLA")
   def testConv2D0x0Padding(self):
     self._VerifyExplicitPaddings(
         tensor_in_sizes=[1, 2, 3, 3],
@@ -588,6 +589,7 @@ class Conv2DTest(test.TestCase):
         padding=[[0, 0], [0, 0]])
 
   @test_util.run_in_graph_and_eager_modes()
+  @test_util.disable_xla("This test never passed for XLA")
   def testConv2D1x1Padding(self):
     self._VerifyExplicitPaddings(
         tensor_in_sizes=[1, 2, 3, 2],
@@ -602,6 +604,7 @@ class Conv2DTest(test.TestCase):
         padding=[[1, 1], [1, 1]])
 
   @test_util.run_in_graph_and_eager_modes()
+  @test_util.disable_xla("This test never passed for XLA")
   def testConv2D2x2Padding(self):
     self._VerifyExplicitPaddings(
         tensor_in_sizes=[1, 2, 1, 2],
@@ -616,6 +619,7 @@ class Conv2DTest(test.TestCase):
         padding=[[2, 2], [2, 2]])
 
   @test_util.run_in_graph_and_eager_modes()
+  @test_util.disable_xla("This test never passed for XLA")
   def testConv2DOnlyBottomPadding(self):
     self._VerifyExplicitPaddings(
         tensor_in_sizes=[1, 2, 3, 3],
@@ -630,6 +634,7 @@ class Conv2DTest(test.TestCase):
         padding=[[0, 3], [0, 0]])
 
   @test_util.run_in_graph_and_eager_modes()
+  @test_util.disable_xla("This test never passed for XLA")
   def testConv2DOnlyTopRightPadding(self):
     self._VerifyExplicitPaddings(
         tensor_in_sizes=[1, 2, 3, 3],
@@ -645,6 +650,7 @@ class Conv2DTest(test.TestCase):
         padding=[[1, 0], [0, 2]])
 
   @test_util.run_in_graph_and_eager_modes()
+  @test_util.disable_xla("This test never passed for XLA")
   def testConv2DLotsPadding(self):
     self._VerifyExplicitPaddings(
         tensor_in_sizes=[1, 1, 1, 3],
@@ -659,6 +665,7 @@ class Conv2DTest(test.TestCase):
         padding=[[3, 4], [4, 2]])
 
   @test_util.run_in_graph_and_eager_modes()
+  @test_util.disable_xla("This test never passed for XLA")
   def testConv2DExplicitPaddingWithDilations(self):
     self._VerifyExplicitPaddings(
         tensor_in_sizes=[1, 3, 2, 1],
@@ -674,6 +681,7 @@ class Conv2DTest(test.TestCase):
         padding=[[2, 1], [1, 2]],
         dilations=[2, 3])
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testConv2DExplicitPaddingWithLayoutOptimizer(self):
     # Test with Grappler's layout optimizer, to ensure the layout optimizer
     # handles explicit padding correctly.
@@ -1341,6 +1349,7 @@ class Conv2DTest(test.TestCase):
         dilations=dilations)
 
   @test_util.run_in_graph_and_eager_modes()
+  @test_util.disable_xla("This test never passed for XLA")
   def testConv2D2x2Depth1Padding0x0BackpropInput(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -1363,6 +1372,7 @@ class Conv2DTest(test.TestCase):
             data_format=data_format)
 
   @test_util.run_in_graph_and_eager_modes()
+  @test_util.disable_xla("This test never passed for XLA")
   def testConv2D2x2Depth1Padding1x1BackpropInput(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -1395,6 +1405,7 @@ class Conv2DTest(test.TestCase):
             dilations=[2, 2])
 
   @test_util.run_in_graph_and_eager_modes()
+  @test_util.disable_xla("This test never passed for XLA")
   def testConv2D2x2Depth1Padding2x2BackpropInput(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -1419,6 +1430,7 @@ class Conv2DTest(test.TestCase):
             dilations=[2, 3])
 
   @test_util.run_in_graph_and_eager_modes()
+  @test_util.disable_xla("This test never passed for XLA")
   def testConv2D2x2Depth1Padding_1_8_4_1_BackpropInput(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -1441,6 +1453,7 @@ class Conv2DTest(test.TestCase):
             data_format=data_format)
 
   @test_util.run_in_graph_and_eager_modes()
+  @test_util.disable_xla("This test never passed for XLA")
   def testConv2D2x2Depth1Padding_5_0_2_2_BackpropInput(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -1499,6 +1512,7 @@ class Conv2DTest(test.TestCase):
         err=err)
 
   @test_util.run_in_graph_and_eager_modes()
+  @test_util.disable_xla("This test never passed for XLA")
   def testConv2D2x2Depth1Padding0x0BackpropFilter(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -1521,6 +1535,7 @@ class Conv2DTest(test.TestCase):
             data_format=data_format)
 
   @test_util.run_in_graph_and_eager_modes()
+  @test_util.disable_xla("This test never passed for XLA")
   def testConv2D2x2Depth1Padding1x1BackpropFilter(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -1554,6 +1569,7 @@ class Conv2DTest(test.TestCase):
             dilations=[2, 2])
 
   @test_util.run_in_graph_and_eager_modes()
+  @test_util.disable_xla("This test never passed for XLA")
   def testConv2D2x2Depth1Padding2x2BackpropFilter(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -1578,6 +1594,7 @@ class Conv2DTest(test.TestCase):
             dilations=[2, 3])
 
   @test_util.run_in_graph_and_eager_modes()
+  @test_util.disable_xla("This test never passed for XLA")
   def testConv2D2x2Depth1Padding_1_8_4_1_BackpropFilter(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -1601,6 +1618,7 @@ class Conv2DTest(test.TestCase):
             data_format=data_format)
 
   @test_util.run_in_graph_and_eager_modes()
+  @test_util.disable_xla("This test never passed for XLA")
   def testConv2D2x2Depth1Padding_5_0_2_2_BackpropFilter(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -1958,6 +1976,7 @@ class Conv2DTest(test.TestCase):
           data_format=data_format,
           use_gpu=use_gpu)
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testInputGradient1x1PaddingStrideOne(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -1979,6 +1998,7 @@ class Conv2DTest(test.TestCase):
             use_gpu=use_gpu,
             max_err=0.0025)
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testFilterGradient1x1PaddingStrideOne(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -1999,6 +2019,7 @@ class Conv2DTest(test.TestCase):
             data_format=data_format,
             use_gpu=use_gpu)
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testInputGradient1x1PaddingStrideTwo(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -2019,6 +2040,7 @@ class Conv2DTest(test.TestCase):
             data_format=data_format,
             use_gpu=use_gpu)
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testFilterGradient1x1PaddingStrideTwo(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -2039,6 +2061,7 @@ class Conv2DTest(test.TestCase):
             data_format=data_format,
             use_gpu=use_gpu)
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testInputGradient2x2PaddingStrideOne(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -2059,6 +2082,7 @@ class Conv2DTest(test.TestCase):
             data_format=data_format,
             use_gpu=use_gpu)
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testFilterGradient2x2PaddingStrideOne(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -2080,6 +2104,7 @@ class Conv2DTest(test.TestCase):
             use_gpu=use_gpu,
             max_err=0.003)
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testInputGradient1_2_3_4PaddingStride3x2(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -2100,6 +2125,7 @@ class Conv2DTest(test.TestCase):
             data_format=data_format,
             use_gpu=use_gpu)
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testFilterGradient1_2_3_4PaddingStride3x2(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -2120,6 +2146,7 @@ class Conv2DTest(test.TestCase):
             data_format=data_format,
             use_gpu=use_gpu)
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testInputGradient4_3_2_1PaddingStride2x1(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -2140,6 +2167,7 @@ class Conv2DTest(test.TestCase):
             data_format=data_format,
             use_gpu=use_gpu)
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testFilterGradient4_3_2_1PaddingStride2x1(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -2160,6 +2188,7 @@ class Conv2DTest(test.TestCase):
             data_format=data_format,
             use_gpu=use_gpu)
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testInputGradient0_0_0_5PaddingStride1x2(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -2180,6 +2209,7 @@ class Conv2DTest(test.TestCase):
             data_format=data_format,
             use_gpu=use_gpu)
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testFilterGradient0_0_0_5PaddingStride1x2(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -2286,6 +2316,7 @@ class Conv2DTest(test.TestCase):
           strides=[1, 1, 1, 1],
           padding=[0, 0, 0, 0])
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testOpEdgeCases(self):
     with self.cached_session() as sess:
       # Illegal strides.
diff --git a/tensorflow/python/kernel_tests/depthwise_conv_op_test.py b/tensorflow/python/kernel_tests/depthwise_conv_op_test.py
index 6f6d17b4eb..4abf6b0e67 100644
--- a/tensorflow/python/kernel_tests/depthwise_conv_op_test.py
+++ b/tensorflow/python/kernel_tests/depthwise_conv_op_test.py
@@ -499,6 +499,7 @@ class DepthwiseConv2DTest(test.TestCase):
             test_input=False,
             use_gpu=True)
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testDepthwiseConv2DFilterGradFormat(self):
     if not test.is_gpu_available():
       return
diff --git a/tensorflow/python/kernel_tests/init_ops_test.py b/tensorflow/python/kernel_tests/init_ops_test.py
index 4b9681afd2..8e3f1c7089 100644
--- a/tensorflow/python/kernel_tests/init_ops_test.py
+++ b/tensorflow/python/kernel_tests/init_ops_test.py
@@ -592,6 +592,7 @@ class LinSpaceTest(test.TestCase):
       self.assertArrayNear(self._LinSpace(5., 5., 3), np.array([5.] * 3), 1e-5)
       self.assertArrayNear(self._LinSpace(5., 5., 4), np.array([5.] * 4), 1e-5)
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testEndpointsAreExact(self):
     for self.force_gpu in self._gpu_modes():
       # Test some cases that produce last values not equal to "stop" when
diff --git a/tensorflow/python/kernel_tests/linalg/BUILD b/tensorflow/python/kernel_tests/linalg/BUILD
index b4608aa38c..53815858e4 100644
--- a/tensorflow/python/kernel_tests/linalg/BUILD
+++ b/tensorflow/python/kernel_tests/linalg/BUILD
@@ -150,6 +150,7 @@ cuda_py_test(
         "noasan",  # times out, b/63678675
         "optonly",  # times out, b/79171797
     ],
+    xla_enable_strict_auto_jit = False,
 )
 
 cuda_py_test(
diff --git a/tensorflow/python/kernel_tests/signal/BUILD b/tensorflow/python/kernel_tests/signal/BUILD
index 3e37d230ef..554bf38029 100644
--- a/tensorflow/python/kernel_tests/signal/BUILD
+++ b/tensorflow/python/kernel_tests/signal/BUILD
@@ -29,6 +29,7 @@ cuda_py_tests(
         "//tensorflow/python:spectral_ops_test_util",
         "//tensorflow/python/ops/signal",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_tests(
@@ -45,6 +46,7 @@ cuda_py_tests(
     ],
     shard_count = 4,
     tags = ["optonly"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_tests(
diff --git a/tensorflow/python/kernel_tests/signal/dct_ops_test.py b/tensorflow/python/kernel_tests/signal/dct_ops_test.py
index a3ac15bab8..e698afde63 100644
--- a/tensorflow/python/kernel_tests/signal/dct_ops_test.py
+++ b/tensorflow/python/kernel_tests/signal/dct_ops_test.py
@@ -134,6 +134,7 @@ class DCTOpsTest(parameterized.TestCase, test.TestCase):
   @parameterized.parameters([
       [[2]], [[3]], [[10]], [[2, 20]], [[2, 3, 25]]])
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def test_random(self, shape):
     """Test randomly generated batches of data."""
     with spectral_ops_test_util.fft_kernel_label_map():
diff --git a/tensorflow/python/kernel_tests/signal/fft_ops_test.py b/tensorflow/python/kernel_tests/signal/fft_ops_test.py
index d98a568491..f3bee87a65 100644
--- a/tensorflow/python/kernel_tests/signal/fft_ops_test.py
+++ b/tensorflow/python/kernel_tests/signal/fft_ops_test.py
@@ -159,6 +159,7 @@ class FFTOpsTest(BaseFFTOpsTest):
       raise ValueError("invalid rank")
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testEmpty(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       for np_type in (np.complex64, np.complex128):
@@ -169,6 +170,7 @@ class FFTOpsTest(BaseFFTOpsTest):
             self.assertEqual(x.shape, self._tfIFFT(x, rank).shape)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testBasic(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       for np_type, tol in ((np.complex64, 1e-4), (np.complex128, 1e-8)):
@@ -178,6 +180,7 @@ class FFTOpsTest(BaseFFTOpsTest):
                 np.mod(np.arange(np.power(4, dims)), 10).reshape(
                     (4,) * dims).astype(np_type), rank, rtol=tol, atol=tol)
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testLargeBatch(self):
     if test.is_gpu_available(cuda_only=True):
       rank = 1
@@ -209,6 +212,7 @@ class FFTOpsTest(BaseFFTOpsTest):
                 rank, use_placeholder=True, rtol=tol, atol=tol)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testRandom(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       for np_type, tol in ((np.complex64, 1e-4), (np.complex128, 5e-6)):
@@ -224,6 +228,7 @@ class FFTOpsTest(BaseFFTOpsTest):
                           rtol=tol, atol=tol)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testRandom1D(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       for np_type in (np.complex64, np.complex128):
@@ -340,6 +345,7 @@ class RFFTOpsTest(BaseFFTOpsTest):
       raise ValueError("invalid rank")
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testEmpty(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       for rank in VALID_FFT_RANKS:
@@ -350,6 +356,7 @@ class RFFTOpsTest(BaseFFTOpsTest):
           self.assertEqual(x.shape, self._tfIFFT(x, rank).shape)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testBasic(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       for rank in VALID_FFT_RANKS:
@@ -364,6 +371,7 @@ class RFFTOpsTest(BaseFFTOpsTest):
             self._compareBackward(
                 c2r.astype(np.complex64), rank, (size,) * rank)
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testLargeBatch(self):
     if test.is_gpu_available(cuda_only=True):
       rank = 1
@@ -397,6 +405,7 @@ class RFFTOpsTest(BaseFFTOpsTest):
                 rank, (size,) * rank,
                 use_placeholder=True)
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testFftLength(self):
     if test.is_gpu_available(cuda_only=True):
       with spectral_ops_test_util.fft_kernel_label_map():
@@ -440,6 +449,7 @@ class RFFTOpsTest(BaseFFTOpsTest):
                   use_placeholder=True)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testRandom(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       def gen_real(shape):
diff --git a/tensorflow/python/kernel_tests/signal/spectral_ops_test.py b/tensorflow/python/kernel_tests/signal/spectral_ops_test.py
index 7b9748c7f2..a72a836592 100644
--- a/tensorflow/python/kernel_tests/signal/spectral_ops_test.py
+++ b/tensorflow/python/kernel_tests/signal/spectral_ops_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import math_ops
@@ -115,6 +116,7 @@ class SpectralOpsTest(test.TestCase):
       self.assertAllClose(
           expected_inverse_stft, actual_inverse_stft, 1e-4, 1e-4)
 
+  @test_util.disable_xla("This test never passed for XLA")
   def test_shapes(self):
     with spectral_ops_test_util.fft_kernel_label_map(), (
         self.session(use_gpu=True)):
@@ -150,6 +152,7 @@ class SpectralOpsTest(test.TestCase):
       self.assertAllEqual([256], inverse_stft.shape.as_list())
       self.assertAllEqual([expected_length], self.evaluate(inverse_stft).shape)
 
+  @test_util.disable_xla("This test never passed for XLA")
   def test_stft_and_inverse_stft(self):
     """Test that spectral_ops.stft/inverse_stft match a NumPy implementation."""
     # Tuples of (signal_length, frame_length, frame_step, fft_length).
-- 
GitLab


From f55eb63a7226eb39602a40a2923232f58a79d12c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 24 Jan 2019 01:42:08 -0800
Subject: [PATCH 1177/2345] Fix keras stateful lstm correctness test.

PiperOrigin-RevId: 230681386
---
 tensorflow/contrib/distribute/python/BUILD             |  1 -
 .../distribute/python/keras_correctness_test_base.py   | 10 ++++++++++
 .../keras_stateful_lstm_model_correctness_test.py      |  2 +-
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index c663e32026..03bc4b2c79 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -764,7 +764,6 @@ distribute_py_test(
         "no_oss",  # TODO(b/117919883): Fix python error.
         "no_pip",
         "no_windows_gpu",
-        "notap",  # TODO(b/123097760): Fix and re-enable the test.
         "notsan",
     ],
     deps = [
diff --git a/tensorflow/contrib/distribute/python/keras_correctness_test_base.py b/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
index 1ceb95cdd6..0f45806a95 100644
--- a/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
+++ b/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
@@ -93,6 +93,16 @@ def test_combinations_for_embedding_model():
            eager_mode_test_configuration())))
 
 
+def test_combinations_with_tpu_strategies():
+  tpu_strategies = [combinations.tpu_strategy_loop_on_device,
+                    combinations.tpu_strategy_one_step_loop_on_device]
+
+  return (
+      combinations.times(
+          combinations.combine(distribution=tpu_strategies),
+          graph_mode_test_configuration()))
+
+
 class MaybeDistributionScope(object):
   """Provides a context allowing no distribution strategy."""
 
diff --git a/tensorflow/contrib/distribute/python/keras_stateful_lstm_model_correctness_test.py b/tensorflow/contrib/distribute/python/keras_stateful_lstm_model_correctness_test.py
index 54eb07e100..ab56c01d86 100644
--- a/tensorflow/contrib/distribute/python/keras_stateful_lstm_model_correctness_test.py
+++ b/tensorflow/contrib/distribute/python/keras_stateful_lstm_model_correctness_test.py
@@ -83,7 +83,7 @@ class DistributionStrategyStatefulLstmModelCorrectnessTest(
                               is_stateful_model=True)
 
   @combinations.generate(keras_correctness_test_base.
-                         test_combinations_for_embedding_model())
+                         test_combinations_with_tpu_strategies())
   def test_incorrectly_use_multiple_cores_for_stateful_lstm_model(
       self, distribution, use_numpy, use_validation_data):
     with self.assertRaisesRegexp(ValueError,
-- 
GitLab


From 678b40e457126e3c95581ff629e964ce5cddfe74 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 24 Jan 2019 02:00:50 -0800
Subject: [PATCH 1178/2345] Internal change.

PiperOrigin-RevId: 230682975
---
 tensorflow/lite/c/BUILD           | 1 -
 tensorflow/lite/python/BUILD      | 1 -
 tensorflow/lite/toco/python/BUILD | 2 --
 3 files changed, 4 deletions(-)

diff --git a/tensorflow/lite/c/BUILD b/tensorflow/lite/c/BUILD
index 91c04a5f1f..680addfa2f 100644
--- a/tensorflow/lite/c/BUILD
+++ b/tensorflow/lite/c/BUILD
@@ -12,7 +12,6 @@ cc_library(
         "c_api_internal.h",
     ],
     visibility = [
-        "//tensorflow/contrib/lite:__subpackages__",
         "//tensorflow/lite:__subpackages__",
     ],
 )
diff --git a/tensorflow/lite/python/BUILD b/tensorflow/lite/python/BUILD
index 4949d7c92e..1fa8494a4c 100644
--- a/tensorflow/lite/python/BUILD
+++ b/tensorflow/lite/python/BUILD
@@ -145,7 +145,6 @@ py_library(
     srcs = ["convert_saved_model.py"],
     srcs_version = "PY2AND3",
     visibility = [
-        "//tensorflow/contrib/lite:__subpackages__",
         "//tensorflow/lite:__subpackages__",
     ],
     deps = [
diff --git a/tensorflow/lite/toco/python/BUILD b/tensorflow/lite/toco/python/BUILD
index 8a6e82ec46..6da48333a2 100644
--- a/tensorflow/lite/toco/python/BUILD
+++ b/tensorflow/lite/toco/python/BUILD
@@ -1,5 +1,4 @@
 package(default_visibility = [
-    "//tensorflow/contrib/lite:__subpackages__",
     "//tensorflow/lite:__subpackages__",
     "//tensorflow/tools/pip_package:__subpackages__",
 ])
@@ -47,7 +46,6 @@ tf_py_wrap_cc(
     visibility = [
         "//learning/expander/pod/deep_pod/utils:__subpackages__",
         "//research/handwriting/converters/tflite:__subpackages__",
-        "//tensorflow/contrib/lite:__subpackages__",
         "//tensorflow/lite:__subpackages__",
     ],
     deps = [
-- 
GitLab


From 6624cba047961fbfe38a0e168c79cb9ed5cbd962 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 24 Jan 2019 03:50:29 -0800
Subject: [PATCH 1179/2345] Add a gcc-7 remote config.

PiperOrigin-RevId: 230693830
---
 tensorflow/opensource_only.files              |    1 +
 .../ubuntu14.04/gcc7-nvcc-cuda10.0/BUILD      |   96 ++
 .../ubuntu14.04/gcc7-nvcc-cuda10.0/CROSSTOOL  | 1431 +++++++++++++++++
 .../bin/crosstool_wrapper_driver_is_not_gcc   |  264 +++
 .../windows/msvc_wrapper_for_nvcc.py          |  192 +++
 5 files changed, 1984 insertions(+)
 create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/gcc7-nvcc-cuda10.0/BUILD
 create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/gcc7-nvcc-cuda10.0/CROSSTOOL
 create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/gcc7-nvcc-cuda10.0/clang/bin/crosstool_wrapper_driver_is_not_gcc
 create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/gcc7-nvcc-cuda10.0/windows/msvc_wrapper_for_nvcc.py

diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index 75976cd8fc..d9793b05bb 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -37,6 +37,7 @@ tensorflow/third_party/toolchains/clang6/README.md
 tensorflow/third_party/toolchains/clang6/repo.bzl
 tensorflow/third_party/toolchains/clang6/CROSSTOOL.tpl
 tensorflow/third_party/toolchains/clang6/clang.BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc7-nvcc-cuda10.0/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/build_defs.bzl
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/py3/BUILD
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/gcc7-nvcc-cuda10.0/BUILD b/third_party/toolchains/preconfig/ubuntu14.04/gcc7-nvcc-cuda10.0/BUILD
new file mode 100755
index 0000000000..2b84b761ff
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu14.04/gcc7-nvcc-cuda10.0/BUILD
@@ -0,0 +1,96 @@
+licenses(["restricted"])
+
+package(default_visibility = ["//visibility:public"])
+
+toolchain(
+    name = "toolchain-linux-x86_64",
+    exec_compatible_with = [
+        "@bazel_tools//platforms:linux",
+        "@bazel_tools//platforms:x86_64",
+    ],
+    target_compatible_with = [
+        "@bazel_tools//platforms:linux",
+        "@bazel_tools//platforms:x86_64",
+    ],
+    toolchain = ":cc-compiler-local",
+    toolchain_type = "@bazel_tools//tools/cpp:toolchain_type",
+)
+
+cc_toolchain_suite(
+    name = "toolchain",
+    toolchains = {
+        "local|compiler": ":cc-compiler-local",
+        "darwin|compiler": ":cc-compiler-darwin",
+        "x64_windows|msvc-cl": ":cc-compiler-windows",
+        "x64_windows": ":cc-compiler-windows",
+        "arm": ":cc-compiler-local",
+        "k8": ":cc-compiler-local",
+        "piii": ":cc-compiler-local",
+        "ppc": ":cc-compiler-local",
+        "darwin": ":cc-compiler-darwin",
+    },
+)
+
+cc_toolchain(
+    name = "cc-compiler-local",
+    all_files = ":crosstool_wrapper_driver_is_not_gcc",
+    compiler_files = ":empty",
+    cpu = "local",
+    dwp_files = ":empty",
+    dynamic_runtime_libs = [":empty"],
+    linker_files = ":crosstool_wrapper_driver_is_not_gcc",
+    objcopy_files = ":empty",
+    static_runtime_libs = [":empty"],
+    strip_files = ":empty",
+    # To support linker flags that need to go to the start of command line
+    # we need the toolchain to support parameter files. Parameter files are
+    # last on the command line and contain all shared libraries to link, so all
+    # regular options will be left of them.
+    supports_param_files = 1,
+    toolchain_identifier = "local_linux",
+)
+
+cc_toolchain(
+    name = "cc-compiler-darwin",
+    all_files = ":crosstool_wrapper_driver_is_not_gcc",
+    compiler_files = ":empty",
+    cpu = "darwin",
+    dwp_files = ":empty",
+    dynamic_runtime_libs = [":empty"],
+    linker_files = ":crosstool_wrapper_driver_is_not_gcc",
+    objcopy_files = ":empty",
+    static_runtime_libs = [":empty"],
+    strip_files = ":empty",
+    supports_param_files = 0,
+    toolchain_identifier = "local_darwin",
+)
+
+cc_toolchain(
+    name = "cc-compiler-windows",
+    all_files = ":windows_msvc_wrapper_files",
+    compiler_files = ":empty",
+    cpu = "x64_windows",
+    dwp_files = ":empty",
+    dynamic_runtime_libs = [":empty"],
+    linker_files = ":windows_msvc_wrapper_files",
+    objcopy_files = ":empty",
+    static_runtime_libs = [":empty"],
+    strip_files = ":empty",
+    supports_param_files = 1,
+    toolchain_identifier = "local_windows",
+)
+
+filegroup(
+    name = "empty",
+    srcs = [],
+)
+
+filegroup(
+    name = "crosstool_wrapper_driver_is_not_gcc",
+    srcs = ["clang/bin/crosstool_wrapper_driver_is_not_gcc"],
+)
+
+filegroup(
+    name = "windows_msvc_wrapper_files",
+    srcs = glob(["windows/msvc_*"]),
+)
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/gcc7-nvcc-cuda10.0/CROSSTOOL b/third_party/toolchains/preconfig/ubuntu14.04/gcc7-nvcc-cuda10.0/CROSSTOOL
new file mode 100755
index 0000000000..059842f58c
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu14.04/gcc7-nvcc-cuda10.0/CROSSTOOL
@@ -0,0 +1,1431 @@
+major_version: "local"
+minor_version: ""
+default_target_cpu: "same_as_host"
+
+toolchain {
+  abi_version: "local"
+  abi_libc_version: "local"
+  compiler: "compiler"
+  host_system_name: "local"
+  needsPic: true
+  target_libc: "local"
+  target_cpu: "local"
+  target_system_name: "local"
+  toolchain_identifier: "local_linux"
+
+  feature {
+    name: "c++11"
+    flag_set {
+      action: "c++-compile"
+      flag_group {
+        flag: "-std=c++11"
+      }
+    }
+  }
+
+  feature {
+    name: "stdlib"
+    flag_set {
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "-lstdc++"
+      }
+    }
+  }
+
+  feature {
+    name: "determinism"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        # Make C++ compilation deterministic. Use linkstamping instead of these
+        # compiler symbols.
+        flag: "-Wno-builtin-macro-redefined"
+        flag: "-D__DATE__=\"redacted\""
+        flag: "-D__TIMESTAMP__=\"redacted\""
+        flag: "-D__TIME__=\"redacted\""
+      }
+    }
+  }
+
+  feature {
+    name: "alwayslink"
+    flag_set {
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      action: "c++-link-executable"
+      flag_group {
+        flag: "-Wl,-no-as-needed"
+      }
+    }
+  }
+
+  # This feature will be enabled for builds that support pic by bazel.
+  feature {
+    name: "pic"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        expand_if_all_available: "pic"
+        flag: "-fPIC"
+      }
+      flag_group {
+        expand_if_none_available: "pic"
+        flag: "-fPIE"
+      }
+    }
+  }
+
+  # Security hardening on by default.
+  feature {
+    name: "hardening"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        # Conservative choice; -D_FORTIFY_SOURCE=2 may be unsafe in some cases.
+        # We need to undef it before redefining it as some distributions now
+        # have it enabled by default.
+        flag: "-U_FORTIFY_SOURCE"
+        flag: "-D_FORTIFY_SOURCE=1"
+        flag: "-fstack-protector"
+      }
+    }
+    flag_set {
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "-Wl,-z,relro,-z,now"
+      }
+    }
+    flag_set {
+      action: "c++-link-executable"
+      flag_group {
+        flag: "-pie"
+        flag: "-Wl,-z,relro,-z,now"
+      }
+    }
+  }
+
+  feature {
+    name: "warnings"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        # All warnings are enabled. Maybe enable -Werror as well?
+        flag: "-Wall"
+        
+      }
+    }
+  }
+
+  # Keep stack frames for debugging, even in opt mode.
+  feature {
+    name: "frame-pointer"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        flag: "-fno-omit-frame-pointer"
+      }
+    }
+  }
+
+  feature {
+    name: "build-id"
+    flag_set {
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        # Stamp the binary with a unique identifier.
+        flag: "-Wl,--build-id=md5"
+        flag: "-Wl,--hash-style=gnu"
+      }
+    }
+  }
+
+  feature {
+    name: "no-canonical-prefixes"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "-no-canonical-prefixes"
+        flag: "-fno-canonical-system-headers"
+      }
+    }
+  }
+
+  feature {
+    name: "disable-assertions"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        flag: "-DNDEBUG"
+      }
+    }
+  }
+
+  feature {
+    name: "linker-bin-path"
+
+    flag_set {
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "-B/usr/bin"
+      }
+    }
+  }
+
+  feature {
+    name: "common"
+    implies: "stdlib"
+    implies: "c++11"
+    implies: "determinism"
+    implies: "alwayslink"
+    implies: "hardening"
+    implies: "warnings"
+    implies: "frame-pointer"
+    implies: "build-id"
+    implies: "no-canonical-prefixes"
+    implies: "linker-bin-path"
+  }
+
+  feature {
+    name: "opt"
+    implies: "common"
+    implies: "disable-assertions"
+
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        # No debug symbols.
+        # Maybe we should enable https://gcc.gnu.org/wiki/DebugFission for opt
+        # or even generally? However, that can't happen here, as it requires
+        # special handling in Bazel.
+        flag: "-g0"
+
+        # Conservative choice for -O
+        # -O3 can increase binary size and even slow down the resulting binaries.
+        # Profile first and / or use FDO if you need better performance than this.
+        flag: "-O2"
+
+        # Removal of unused code and data at link time (can this increase binary size in some cases?).
+        flag: "-ffunction-sections"
+        flag: "-fdata-sections"
+      }
+    }
+    flag_set {
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      action: "c++-link-executable"
+      flag_group {
+        flag: "-Wl,--gc-sections"
+      }
+    }
+  }
+
+  feature {
+    name: "fastbuild"
+    implies: "common"
+  }
+
+  feature {
+    name: "dbg"
+    implies: "common"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        flag: "-g"
+      }
+    }
+  }
+
+  # Set clang as a C/C++ compiler.
+  tool_path { name: "gcc" path: "clang/bin/crosstool_wrapper_driver_is_not_gcc" }
+
+  # Use the default system toolchain for everything else.
+  tool_path { name: "ar" path: "/usr/bin/ar" }
+  tool_path { name: "compat-ld" path: "/usr/bin/ld" }
+  tool_path { name: "cpp" path: "/usr/bin/cpp" }
+  tool_path { name: "dwp" path: "/usr/bin/dwp" }
+  tool_path { name: "gcov" path: "/usr/bin/gcov" }
+  tool_path { name: "ld" path: "/usr/bin/ld" }
+  tool_path { name: "nm" path: "/usr/bin/nm" }
+  tool_path { name: "objcopy" path: "/usr/bin/objcopy" }
+  tool_path { name: "objdump" path: "/usr/bin/objdump" }
+  tool_path { name: "strip" path: "/usr/bin/strip" }
+
+  # Enabled dynamic linking.
+  linking_mode_flags { mode: DYNAMIC }
+
+  cxx_builtin_include_directory: "/usr/include/c++/7"
+  cxx_builtin_include_directory: "/usr/include/x86_64-linux-gnu/c++/7"
+  cxx_builtin_include_directory: "/usr/include/c++/7/backward"
+  cxx_builtin_include_directory: "/usr/lib/gcc/x86_64-linux-gnu/7/include"
+  cxx_builtin_include_directory: "/usr/local/include"
+  cxx_builtin_include_directory: "/usr/lib/gcc/x86_64-linux-gnu/7/include-fixed"
+  cxx_builtin_include_directory: "/usr/include/x86_64-linux-gnu"
+  cxx_builtin_include_directory: "/usr/include"
+  cxx_builtin_include_directory: "/usr/local/cuda-10.0/targets/x86_64-linux/include"
+  cxx_builtin_include_directory: "/usr/local/cuda-10.0/include"
+  cxx_builtin_include_directory: "/usr/local/cuda-10.0/extras/CUPTI/include"
+  cxx_builtin_include_directory: "/usr/include"
+}
+
+toolchain {
+  abi_version: "local"
+  abi_libc_version: "local"
+  compiler: "compiler"
+  host_system_name: "local"
+  needsPic: true
+  target_libc: "macosx"
+  target_cpu: "darwin"
+  target_system_name: "local"
+  toolchain_identifier: "local_darwin"
+  feature {
+    name: "c++11"
+    flag_set {
+      action: "c++-compile"
+      flag_group {
+        flag: "-std=c++11"
+      }
+    }
+  }
+
+  feature {
+    name: "stdlib"
+    flag_set {
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "-lc++"
+      }
+    }
+  }
+
+  feature {
+    name: "determinism"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        # Make C++ compilation deterministic. Use linkstamping instead of these
+        # compiler symbols.
+        flag: "-Wno-builtin-macro-redefined"
+        flag: "-D__DATE__=\"redacted\""
+        flag: "-D__TIMESTAMP__=\"redacted\""
+        flag: "-D__TIME__=\"redacted\""
+      }
+    }
+  }
+
+  # This feature will be enabled for builds that support pic by bazel.
+  feature {
+    name: "pic"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        expand_if_all_available: "pic"
+        flag: "-fPIC"
+      }
+      flag_group {
+        expand_if_none_available: "pic"
+        flag: "-fPIE"
+      }
+    }
+  }
+
+  # Security hardening on by default.
+  feature {
+    name: "hardening"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        # Conservative choice; -D_FORTIFY_SOURCE=2 may be unsafe in some cases.
+        # We need to undef it before redefining it as some distributions now
+        # have it enabled by default.
+        flag: "-U_FORTIFY_SOURCE"
+        flag: "-D_FORTIFY_SOURCE=1"
+        flag: "-fstack-protector"
+      }
+    }
+    flag_set {
+      action: "c++-link-executable"
+      flag_group {
+        flag: "-pie"
+      }
+    }
+  }
+
+  feature {
+    name: "warnings"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        # All warnings are enabled. Maybe enable -Werror as well?
+        flag: "-Wall"
+        
+      }
+    }
+  }
+
+  # Keep stack frames for debugging, even in opt mode.
+  feature {
+    name: "frame-pointer"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        flag: "-fno-omit-frame-pointer"
+      }
+    }
+  }
+
+  feature {
+    name: "no-canonical-prefixes"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag:"-no-canonical-prefixes"
+      }
+    }
+  }
+
+  feature {
+    name: "disable-assertions"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        flag: "-DNDEBUG"
+      }
+    }
+  }
+
+  feature {
+    name: "linker-bin-path"
+
+    flag_set {
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "-B/usr/bin"
+      }
+    }
+  }
+
+  feature {
+    name: "undefined-dynamic"
+    flag_set {
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      action: "c++-link-executable"
+      flag_group {
+        flag: "-undefined"
+        flag: "dynamic_lookup"
+      }
+    }
+  }
+
+  feature {
+    name: "common"
+    implies: "stdlib"
+    implies: "c++11"
+    implies: "determinism"
+    implies: "hardening"
+    implies: "warnings"
+    implies: "frame-pointer"
+    implies: "no-canonical-prefixes"
+    implies: "linker-bin-path"
+    implies: "undefined-dynamic"
+  }
+
+  feature {
+    name: "opt"
+    implies: "common"
+    implies: "disable-assertions"
+
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        # No debug symbols.
+        # Maybe we should enable https://gcc.gnu.org/wiki/DebugFission for opt
+        # or even generally? However, that can't happen here, as it requires
+        # special handling in Bazel.
+        flag: "-g0"
+
+        # Conservative choice for -O
+        # -O3 can increase binary size and even slow down the resulting binaries.
+        # Profile first and / or use FDO if you need better performance than this.
+        flag: "-O2"
+
+        # Removal of unused code and data at link time (can this increase binary size in some cases?).
+        flag: "-ffunction-sections"
+        flag: "-fdata-sections"
+      }
+    }
+  }
+
+  feature {
+    name: "fastbuild"
+    implies: "common"
+  }
+
+  feature {
+    name: "dbg"
+    implies: "common"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        flag: "-g"
+      }
+    }
+  }
+
+  # Set clang as a C/C++ compiler.
+  tool_path { name: "gcc" path: "clang/bin/crosstool_wrapper_driver_is_not_gcc" }
+
+  # Use the default system toolchain for everything else.
+  tool_path { name: "ar" path: "/usr/bin/libtool" }
+  tool_path { name: "compat-ld" path: "/usr/bin/ld" }
+  tool_path { name: "cpp" path: "/usr/bin/cpp" }
+  tool_path { name: "dwp" path: "/usr/bin/dwp" }
+  tool_path { name: "gcov" path: "/usr/bin/gcov" }
+  tool_path { name: "ld" path: "/usr/bin/ld" }
+  tool_path { name: "nm" path: "/usr/bin/nm" }
+  tool_path { name: "objcopy" path: "/usr/bin/objcopy" }
+  tool_path { name: "objdump" path: "/usr/bin/objdump" }
+  tool_path { name: "strip" path: "/usr/bin/strip" }
+
+  # Enabled dynamic linking.
+  linking_mode_flags { mode: DYNAMIC }
+
+  cxx_builtin_include_directory: "/usr/include/c++/7"
+  cxx_builtin_include_directory: "/usr/include/x86_64-linux-gnu/c++/7"
+  cxx_builtin_include_directory: "/usr/include/c++/7/backward"
+  cxx_builtin_include_directory: "/usr/lib/gcc/x86_64-linux-gnu/7/include"
+  cxx_builtin_include_directory: "/usr/local/include"
+  cxx_builtin_include_directory: "/usr/lib/gcc/x86_64-linux-gnu/7/include-fixed"
+  cxx_builtin_include_directory: "/usr/include/x86_64-linux-gnu"
+  cxx_builtin_include_directory: "/usr/include"
+  cxx_builtin_include_directory: "/usr/local/cuda-10.0/targets/x86_64-linux/include"
+  cxx_builtin_include_directory: "/usr/local/cuda-10.0/include"
+  cxx_builtin_include_directory: "/usr/local/cuda-10.0/extras/CUPTI/include"
+  cxx_builtin_include_directory: "/usr/include"
+}
+
+toolchain {
+  toolchain_identifier: "local_windows"
+  host_system_name: "local"
+  target_system_name: "local"
+
+  abi_version: "local"
+  abi_libc_version: "local"
+  target_cpu: "x64_windows"
+  compiler: "msvc-cl"
+  target_libc: "msvcrt"
+
+
+
+  tool_path {
+    name: "ar"
+    path: ""
+  }
+  tool_path {
+    name: "ml"
+    path: ""
+  }
+  tool_path {
+    name: "cpp"
+    path: ""
+  }
+  tool_path {
+    name: "gcc"
+    path: ""
+  }
+  tool_path {
+    name: "gcov"
+    path: "wrapper/bin/msvc_nop.bat"
+  }
+  tool_path {
+    name: "ld"
+    path: ""
+  }
+  tool_path {
+    name: "nm"
+    path: "wrapper/bin/msvc_nop.bat"
+  }
+  tool_path {
+    name: "objcopy"
+    path: "wrapper/bin/msvc_nop.bat"
+  }
+  tool_path {
+    name: "objdump"
+    path: "wrapper/bin/msvc_nop.bat"
+  }
+  tool_path {
+    name: "strip"
+    path: "wrapper/bin/msvc_nop.bat"
+  }
+  supports_interface_shared_objects: true
+
+  # TODO(pcloudy): Review those flags below, they should be defined by cl.exe
+  compiler_flag: "/DCOMPILER_MSVC"
+
+  # Don't define min/max macros in windows.h.
+  compiler_flag: "/DNOMINMAX"
+
+  # Platform defines.
+  compiler_flag: "/D_WIN32_WINNT=0x0600"
+  # Turn off warning messages.
+  compiler_flag: "/D_CRT_SECURE_NO_DEPRECATE"
+  compiler_flag: "/D_CRT_SECURE_NO_WARNINGS"
+  compiler_flag: "/D_SILENCE_STDEXT_HASH_DEPRECATION_WARNINGS"
+
+  # Useful options to have on for compilation.
+  # Increase the capacity of object files to 2^32 sections.
+  compiler_flag: "/bigobj"
+  # Allocate 500MB for precomputed headers.
+  compiler_flag: "/Zm500"
+  # Use unsigned char by default.
+  compiler_flag: "/J"
+  # Use function level linking.
+  compiler_flag: "/Gy"
+  # Use string pooling.
+  compiler_flag: "/GF"
+  # Catch C++ exceptions only and tell the compiler to assume that functions declared
+  # as extern "C" never throw a C++ exception.
+  compiler_flag: "/EHsc"
+
+  # Globally disabled warnings.
+  # Don't warn about elements of array being be default initialized.
+  compiler_flag: "/wd4351"
+  # Don't warn about no matching delete found.
+  compiler_flag: "/wd4291"
+  # Don't warn about diamond inheritance patterns.
+  compiler_flag: "/wd4250"
+  # Don't warn about insecure functions (e.g. non _s functions).
+  compiler_flag: "/wd4996"
+
+  linker_flag: "/MACHINE:X64"
+
+  feature {
+    name: "no_legacy_features"
+  }
+
+  # TODO(klimek): Previously we were using a .bat file to start python to run
+  # the python script that can redirect to nvcc - unfortunately .bat files
+  # have a rather short maximum length for command lines (8k). Instead, we
+  # now use the python binary as the compiler and pass the python script to
+  # it at the start of the command line. Investigate different possibilities
+  # to run the nvcc wrapper, either using pyinstaller --onefile, or writing
+  # a small C++ wrapper to redirect.
+  feature {
+    name: "redirector"
+    enabled: true
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      action: "c++-module-compile"
+      action: "c++-module-codegen"
+      action: "c++-header-parsing"
+      action: "assemble"
+      action: "preprocess-assemble"
+      flag_group {
+        flag: "-B"
+        flag: "external/local_config_cuda/crosstool/windows/msvc_wrapper_for_nvcc.py"
+      }
+    }
+  }
+
+  # Suppress startup banner.
+  feature {
+    name: "nologo"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      action: "c++-module-compile"
+      action: "c++-module-codegen"
+      action: "c++-header-parsing"
+      action: "assemble"
+      action: "preprocess-assemble"
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      action: "c++-link-static-library"
+      flag_group {
+        flag: "/nologo"
+      }
+    }
+  }
+
+  feature {
+    name: 'has_configured_linker_path'
+  }
+
+  # This feature indicates strip is not supported, building stripped binary will just result a copy of orignial binary
+  feature {
+    name: 'no_stripping'
+  }
+
+  # This feature indicates this is a toolchain targeting Windows.
+  feature {
+    name: 'targets_windows'
+    implies: 'copy_dynamic_libraries_to_binary'
+    enabled: true
+  }
+
+  feature {
+    name: 'copy_dynamic_libraries_to_binary'
+  }
+
+  action_config {
+    config_name: 'assemble'
+    action_name: 'assemble'
+    tool {
+      tool_path: ''
+    }
+    implies: 'compiler_input_flags'
+    implies: 'compiler_output_flags'
+    implies: 'nologo'
+    implies: 'msvc_env'
+    implies: 'sysroot'
+  }
+
+  action_config {
+    config_name: 'preprocess-assemble'
+    action_name: 'preprocess-assemble'
+    tool {
+      tool_path: ''
+    }
+    implies: 'compiler_input_flags'
+    implies: 'compiler_output_flags'
+    implies: 'nologo'
+    implies: 'msvc_env'
+    implies: 'sysroot'
+  }
+
+  action_config {
+    config_name: 'c-compile'
+    action_name: 'c-compile'
+    tool {
+      tool_path: ''
+    }
+    implies: 'compiler_input_flags'
+    implies: 'compiler_output_flags'
+    implies: 'legacy_compile_flags'
+    implies: 'nologo'
+    implies: 'msvc_env'
+    implies: 'parse_showincludes'
+    implies: 'user_compile_flags'
+    implies: 'sysroot'
+    implies: 'unfiltered_compile_flags'
+  }
+
+  action_config {
+    config_name: 'c++-compile'
+    action_name: 'c++-compile'
+    tool {
+      tool_path: ''
+    }
+    implies: 'compiler_input_flags'
+    implies: 'compiler_output_flags'
+    implies: 'legacy_compile_flags'
+    implies: 'nologo'
+    implies: 'msvc_env'
+    implies: 'parse_showincludes'
+    implies: 'user_compile_flags'
+    implies: 'sysroot'
+    implies: 'unfiltered_compile_flags'
+  }
+
+  action_config {
+    config_name: 'c++-link-executable'
+    action_name: 'c++-link-executable'
+    tool {
+      tool_path: ''
+    }
+    implies: 'nologo'
+    implies: 'linkstamps'
+    implies: 'output_execpath_flags'
+    implies: 'input_param_flags'
+    implies: 'user_link_flags'
+    implies: 'legacy_link_flags'
+    implies: 'linker_subsystem_flag'
+    implies: 'linker_param_file'
+    implies: 'msvc_env'
+    implies: 'no_stripping'
+  }
+
+  action_config {
+    config_name: 'c++-link-dynamic-library'
+    action_name: 'c++-link-dynamic-library'
+    tool {
+      tool_path: ''
+    }
+    implies: 'nologo'
+    implies: 'shared_flag'
+    implies: 'linkstamps'
+    implies: 'output_execpath_flags'
+    implies: 'input_param_flags'
+    implies: 'user_link_flags'
+    implies: 'legacy_link_flags'
+    implies: 'linker_subsystem_flag'
+    implies: 'linker_param_file'
+    implies: 'msvc_env'
+    implies: 'no_stripping'
+    implies: 'has_configured_linker_path'
+    implies: 'def_file'
+  }
+
+  action_config {
+      config_name: 'c++-link-nodeps-dynamic-library'
+      action_name: 'c++-link-nodeps-dynamic-library'
+      tool {
+        tool_path: ''
+      }
+      implies: 'nologo'
+      implies: 'shared_flag'
+      implies: 'linkstamps'
+      implies: 'output_execpath_flags'
+      implies: 'input_param_flags'
+      implies: 'user_link_flags'
+      implies: 'legacy_link_flags'
+      implies: 'linker_subsystem_flag'
+      implies: 'linker_param_file'
+      implies: 'msvc_env'
+      implies: 'no_stripping'
+      implies: 'has_configured_linker_path'
+      implies: 'def_file'
+    }
+
+  action_config {
+    config_name: 'c++-link-static-library'
+    action_name: 'c++-link-static-library'
+    tool {
+      tool_path: ''
+    }
+    implies: 'nologo'
+    implies: 'archiver_flags'
+    implies: 'input_param_flags'
+    implies: 'linker_param_file'
+    implies: 'msvc_env'
+  }
+
+  # TODO(b/65151735): Remove legacy_compile_flags feature when legacy fields are
+  # not used in this crosstool
+  feature {
+    name: 'legacy_compile_flags'
+    flag_set {
+      expand_if_all_available: 'legacy_compile_flags'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      flag_group {
+        iterate_over: 'legacy_compile_flags'
+        flag: '%{legacy_compile_flags}'
+      }
+    }
+  }
+
+  feature {
+    name: "msvc_env"
+    env_set {
+      action: "c-compile"
+      action: "c++-compile"
+      action: "c++-module-compile"
+      action: "c++-module-codegen"
+      action: "c++-header-parsing"
+      action: "assemble"
+      action: "preprocess-assemble"
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      action: "c++-link-static-library"
+      env_entry {
+        key: "PATH"
+        value: ""
+      }
+      env_entry {
+        key: "INCLUDE"
+        value: ""
+      }
+      env_entry {
+        key: "LIB"
+        value: ""
+      }
+      env_entry {
+        key: "TMP"
+        value: ""
+      }
+      env_entry {
+        key: "TEMP"
+        value: ""
+      }
+    }
+  }
+
+  feature {
+    name: 'include_paths'
+    flag_set {
+      action: "assemble"
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      flag_group {
+        iterate_over: 'quote_include_paths'
+        flag: '/I%{quote_include_paths}'
+      }
+      flag_group {
+        iterate_over: 'include_paths'
+        flag: '/I%{include_paths}'
+      }
+      flag_group {
+        iterate_over: 'system_include_paths'
+        flag: '/I%{system_include_paths}'
+      }
+    }
+  }
+
+  feature {
+    name: "preprocessor_defines"
+    flag_set {
+      action: "assemble"
+      action: "preprocess-assemble"
+      action: "c-compile"
+      action: "c++-compile"
+      action: "c++-header-parsing"
+      action: "c++-module-compile"
+      flag_group {
+        flag: "/D%{preprocessor_defines}"
+        iterate_over: "preprocessor_defines"
+      }
+    }
+  }
+
+  # Tell Bazel to parse the output of /showIncludes
+  feature {
+    name: 'parse_showincludes'
+    flag_set {
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-module-compile'
+      action: 'c++-header-parsing'
+      flag_group {
+        flag: "/showIncludes"
+      }
+    }
+  }
+
+
+  feature {
+    name: 'generate_pdb_file'
+    requires: {
+      feature: 'dbg'
+    }
+    requires: {
+      feature: 'fastbuild'
+    }
+  }
+
+  feature {
+    name: 'shared_flag'
+    flag_set {
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: '/DLL'
+      }
+    }
+  }
+
+  feature {
+    name: 'linkstamps'
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      expand_if_all_available: 'linkstamp_paths'
+      flag_group {
+        iterate_over: 'linkstamp_paths'
+        flag: '%{linkstamp_paths}'
+      }
+    }
+  }
+
+  feature {
+    name: 'output_execpath_flags'
+    flag_set {
+      expand_if_all_available: 'output_execpath'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: '/OUT:%{output_execpath}'
+      }
+    }
+  }
+
+  feature {
+    name: 'archiver_flags'
+    flag_set {
+      expand_if_all_available: 'output_execpath'
+      action: 'c++-link-static-library'
+      flag_group {
+        flag: '/OUT:%{output_execpath}'
+      }
+    }
+  }
+
+  feature {
+    name: 'input_param_flags'
+    flag_set {
+      expand_if_all_available: 'interface_library_output_path'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/IMPLIB:%{interface_library_output_path}"
+      }
+    }
+    flag_set {
+      expand_if_all_available: 'libopts'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        iterate_over: 'libopts'
+        flag: '%{libopts}'
+      }
+    }
+    flag_set {
+      expand_if_all_available: 'libraries_to_link'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      action: 'c++-link-static-library'
+      flag_group {
+        iterate_over: 'libraries_to_link'
+        flag_group {
+          expand_if_equal: {
+            variable: 'libraries_to_link.type'
+            value: 'object_file_group'
+          }
+          iterate_over: 'libraries_to_link.object_files'
+          flag_group {
+            flag: '%{libraries_to_link.object_files}'
+          }
+        }
+        flag_group {
+          expand_if_equal: {
+            variable: 'libraries_to_link.type'
+            value: 'object_file'
+          }
+          flag_group {
+            flag: '%{libraries_to_link.name}'
+          }
+        }
+        flag_group {
+          expand_if_equal: {
+            variable: 'libraries_to_link.type'
+            value: 'interface_library'
+          }
+          flag_group {
+            flag: '%{libraries_to_link.name}'
+          }
+        }
+        flag_group {
+          expand_if_equal: {
+            variable: 'libraries_to_link.type'
+            value: 'static_library'
+          }
+          flag_group {
+            expand_if_false: 'libraries_to_link.is_whole_archive'
+            flag: '%{libraries_to_link.name}'
+          }
+          flag_group {
+            expand_if_true: 'libraries_to_link.is_whole_archive'
+            flag: '/WHOLEARCHIVE:%{libraries_to_link.name}'
+          }
+        }
+      }
+    }
+  }
+
+  # Since this feature is declared earlier in the CROSSTOOL than
+  # "user_link_flags", this feature will be applied prior to it anwyhere they
+  # are both implied. And since "user_link_flags" contains the linkopts from
+  # the build rule, this allows the user to override the /SUBSYSTEM in the BUILD
+  # file.
+  feature {
+    name: 'linker_subsystem_flag'
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: '/SUBSYSTEM:CONSOLE'
+      }
+    }
+  }
+
+  # The "user_link_flags" contains user-defined linkopts (from build rules)
+  # so it should be defined after features that declare user-overridable flags.
+  # For example the "linker_subsystem_flag" defines a default "/SUBSYSTEM" flag
+  # but we want to let the user override it, therefore "link_flag_subsystem" is
+  # defined earlier in the CROSSTOOL file than "user_link_flags".
+  feature {
+    name: 'user_link_flags'
+    flag_set {
+      expand_if_all_available: 'user_link_flags'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        iterate_over: 'user_link_flags'
+        flag: '%{user_link_flags}'
+      }
+    }
+  }
+  feature {
+    name: 'legacy_link_flags'
+    flag_set {
+      expand_if_all_available: 'legacy_link_flags'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        iterate_over: 'legacy_link_flags'
+        flag: '%{legacy_link_flags}'
+      }
+    }
+  }
+
+  feature {
+    name: 'linker_param_file'
+    flag_set {
+      expand_if_all_available: 'linker_param_file'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      action: 'c++-link-static-library'
+      flag_group {
+        flag: '@%{linker_param_file}'
+      }
+    }
+  }
+
+  feature {
+    name: 'static_link_msvcrt'
+  }
+
+  feature {
+    name: 'static_link_msvcrt_no_debug'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/MT"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEFAULTLIB:libcmt.lib"
+      }
+    }
+    requires: { feature: 'fastbuild'}
+    requires: { feature: 'opt'}
+  }
+
+  feature {
+    name: 'dynamic_link_msvcrt_no_debug'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/MD"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEFAULTLIB:msvcrt.lib"
+      }
+    }
+    requires: { feature: 'fastbuild'}
+    requires: { feature: 'opt'}
+  }
+
+  feature {
+    name: 'static_link_msvcrt_debug'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/MTd"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEFAULTLIB:libcmtd.lib"
+      }
+    }
+    requires: { feature: 'dbg'}
+  }
+
+  feature {
+    name: 'dynamic_link_msvcrt_debug'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/MDd"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEFAULTLIB:msvcrtd.lib"
+      }
+    }
+    requires: { feature: 'dbg'}
+  }
+
+  feature {
+    name: 'dbg'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/Od"
+        flag: "/Z7"
+        flag: "/DDEBUG"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEBUG:FULL"
+        flag: "/INCREMENTAL:NO"
+      }
+    }
+    implies: 'generate_pdb_file'
+  }
+
+  feature {
+    name: 'fastbuild'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/Od"
+        flag: "/Z7"
+        flag: "/DDEBUG"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEBUG:FASTLINK"
+        flag: "/INCREMENTAL:NO"
+      }
+    }
+    implies: 'generate_pdb_file'
+  }
+
+  feature {
+    name: 'opt'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/O2"
+        flag: "/DNDEBUG"
+      }
+    }
+  }
+
+  feature {
+    name: 'user_compile_flags'
+    flag_set {
+      expand_if_all_available: 'user_compile_flags'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      flag_group {
+        iterate_over: 'user_compile_flags'
+        flag: '%{user_compile_flags}'
+      }
+    }
+  }
+
+  feature {
+    name: 'sysroot'
+    flag_set {
+      expand_if_all_available: 'sysroot'
+      action: 'assemble'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        iterate_over: 'sysroot'
+        flag: '--sysroot=%{sysroot}'
+      }
+    }
+  }
+
+  feature {
+    name: 'unfiltered_compile_flags'
+    flag_set {
+      expand_if_all_available: 'unfiltered_compile_flags'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      flag_group {
+        iterate_over: 'unfiltered_compile_flags'
+        flag: '%{unfiltered_compile_flags}'
+      }
+    }
+  }
+
+  feature {
+    name: 'compiler_output_flags'
+    flag_set {
+      action: 'assemble'
+      flag_group {
+        expand_if_all_available: 'output_file'
+        expand_if_none_available: 'output_assembly_file'
+        expand_if_none_available: 'output_preprocess_file'
+        flag: '/Fo%{output_file}'
+        flag: '/Zi'
+      }
+    }
+    flag_set {
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      flag_group {
+        expand_if_all_available: 'output_file'
+        expand_if_none_available: 'output_assembly_file'
+        expand_if_none_available: 'output_preprocess_file'
+        flag: '/Fo%{output_file}'
+      }
+      flag_group {
+        expand_if_all_available: 'output_file'
+        expand_if_all_available: 'output_assembly_file'
+        flag: '/Fa%{output_file}'
+      }
+      flag_group {
+        expand_if_all_available: 'output_file'
+        expand_if_all_available: 'output_preprocess_file'
+        flag: '/P'
+        flag: '/Fi%{output_file}'
+      }
+    }
+  }
+
+  feature {
+    name: 'compiler_input_flags'
+    flag_set {
+      action: 'assemble'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      flag_group {
+        expand_if_all_available: 'source_file'
+        flag: '/c'
+        flag: '%{source_file}'
+      }
+    }
+  }
+
+  feature {
+    name : 'def_file',
+    flag_set {
+      expand_if_all_available: 'def_file_path'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEF:%{def_file_path}"
+        # We can specify a different DLL name in DEF file, /ignore:4070 suppresses
+        # the warning message about DLL name doesn't match the default one.
+        # See https://msdn.microsoft.com/en-us/library/sfkk2fz7.aspx
+        flag: "/ignore:4070"
+      }
+    }
+  }
+
+  feature {
+    name: 'windows_export_all_symbols'
+  }
+
+  feature {
+    name: 'no_windows_export_all_symbols'
+  }
+
+  linking_mode_flags { mode: DYNAMIC }
+}
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/gcc7-nvcc-cuda10.0/clang/bin/crosstool_wrapper_driver_is_not_gcc b/third_party/toolchains/preconfig/ubuntu14.04/gcc7-nvcc-cuda10.0/clang/bin/crosstool_wrapper_driver_is_not_gcc
new file mode 100755
index 0000000000..07742839ca
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu14.04/gcc7-nvcc-cuda10.0/clang/bin/crosstool_wrapper_driver_is_not_gcc
@@ -0,0 +1,264 @@
+#!/usr/bin/env python
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Crosstool wrapper for compiling CUDA programs.
+
+SYNOPSIS:
+  crosstool_wrapper_is_not_gcc [options passed in by cc_library()
+                                or cc_binary() rule]
+
+DESCRIPTION:
+  This script is expected to be called by the cc_library() or cc_binary() bazel
+  rules. When the option "-x cuda" is present in the list of arguments passed
+  to this script, it invokes the nvcc CUDA compiler. Most arguments are passed
+  as is as a string to --compiler-options of nvcc. When "-x cuda" is not
+  present, this wrapper invokes hybrid_driver_is_not_gcc with the input
+  arguments as is.
+
+NOTES:
+  Changes to the contents of this file must be propagated from
+  //third_party/gpus/crosstool/crosstool_wrapper_is_not_gcc to
+  //third_party/gpus/crosstool/v*/*/clang/bin/crosstool_wrapper_is_not_gcc
+"""
+
+from __future__ import print_function
+
+__author__ = 'keveman@google.com (Manjunath Kudlur)'
+
+from argparse import ArgumentParser
+import os
+import subprocess
+import re
+import sys
+import pipes
+
+# Template values set by cuda_autoconf.
+CPU_COMPILER = ('/usr/bin/gcc-7')
+GCC_HOST_COMPILER_PATH = ('/usr/bin/gcc-7')
+
+NVCC_PATH = '/usr/local/cuda-10.0/bin/nvcc'
+PREFIX_DIR = os.path.dirname(GCC_HOST_COMPILER_PATH)
+NVCC_VERSION = '10.0'
+
+def Log(s):
+  print('gpus/crosstool: {0}'.format(s))
+
+
+def GetOptionValue(argv, option):
+  """Extract the list of values for option from the argv list.
+
+  Args:
+    argv: A list of strings, possibly the argv passed to main().
+    option: The option whose value to extract, without the leading '-'.
+
+  Returns:
+    A list of values, either directly following the option,
+    (eg., -opt val1 val2) or values collected from multiple occurrences of
+    the option (eg., -opt val1 -opt val2).
+  """
+
+  parser = ArgumentParser()
+  parser.add_argument('-' + option, nargs='*', action='append')
+  args, _ = parser.parse_known_args(argv)
+  if not args or not vars(args)[option]:
+    return []
+  else:
+    return sum(vars(args)[option], [])
+
+
+def GetHostCompilerOptions(argv):
+  """Collect the -isystem, -iquote, and --sysroot option values from argv.
+
+  Args:
+    argv: A list of strings, possibly the argv passed to main().
+
+  Returns:
+    The string that can be used as the --compiler-options to nvcc.
+  """
+
+  parser = ArgumentParser()
+  parser.add_argument('-isystem', nargs='*', action='append')
+  parser.add_argument('-iquote', nargs='*', action='append')
+  parser.add_argument('--sysroot', nargs=1)
+  parser.add_argument('-g', nargs='*', action='append')
+  parser.add_argument('-fno-canonical-system-headers', action='store_true')
+
+  args, _ = parser.parse_known_args(argv)
+
+  opts = ''
+
+  if args.isystem:
+    opts += ' -isystem ' + ' -isystem '.join(sum(args.isystem, []))
+  if args.iquote:
+    opts += ' -iquote ' + ' -iquote '.join(sum(args.iquote, []))
+  if args.g:
+    opts += ' -g' + ' -g'.join(sum(args.g, []))
+  if args.fno_canonical_system_headers:
+    opts += ' -fno-canonical-system-headers'
+  if args.sysroot:
+    opts += ' --sysroot ' + args.sysroot[0]
+
+  return opts
+
+def _update_options(nvcc_options):
+  if NVCC_VERSION in ("7.0",):
+    return nvcc_options
+
+  update_options = { "relaxed-constexpr" : "expt-relaxed-constexpr" }
+  return [ update_options[opt] if opt in update_options else opt
+                    for opt in nvcc_options ]
+
+def GetNvccOptions(argv):
+  """Collect the -nvcc_options values from argv.
+
+  Args:
+    argv: A list of strings, possibly the argv passed to main().
+
+  Returns:
+    The string that can be passed directly to nvcc.
+  """
+
+  parser = ArgumentParser()
+  parser.add_argument('-nvcc_options', nargs='*', action='append')
+
+  args, _ = parser.parse_known_args(argv)
+
+  if args.nvcc_options:
+    options = _update_options(sum(args.nvcc_options, []))
+    return ' '.join(['--'+a for a in options])
+  return ''
+
+
+def InvokeNvcc(argv, log=False):
+  """Call nvcc with arguments assembled from argv.
+
+  Args:
+    argv: A list of strings, possibly the argv passed to main().
+    log: True if logging is requested.
+
+  Returns:
+    The return value of calling os.system('nvcc ' + args)
+  """
+
+  host_compiler_options = GetHostCompilerOptions(argv)
+  nvcc_compiler_options = GetNvccOptions(argv)
+  opt_option = GetOptionValue(argv, 'O')
+  m_options = GetOptionValue(argv, 'm')
+  m_options = ''.join([' -m' + m for m in m_options if m in ['32', '64']])
+  include_options = GetOptionValue(argv, 'I')
+  out_file = GetOptionValue(argv, 'o')
+  depfiles = GetOptionValue(argv, 'MF')
+  defines = GetOptionValue(argv, 'D')
+  defines = ''.join([' -D' + define for define in defines])
+  undefines = GetOptionValue(argv, 'U')
+  undefines = ''.join([' -U' + define for define in undefines])
+  std_options = GetOptionValue(argv, 'std')
+  # currently only c++11 is supported by Cuda 7.0 std argument
+  nvcc_allowed_std_options = ["c++11"]
+  std_options = ''.join([' -std=' + define
+      for define in std_options if define in nvcc_allowed_std_options])
+
+  # The list of source files get passed after the -c option. I don't know of
+  # any other reliable way to just get the list of source files to be compiled.
+  src_files = GetOptionValue(argv, 'c')
+
+  # Pass -w through from host to nvcc, but don't do anything fancier with
+  # warnings-related flags, since they're not necessarily the same across
+  # compilers.
+  warning_options = ' -w' if '-w' in argv else ''
+
+  if len(src_files) == 0:
+    return 1
+  if len(out_file) != 1:
+    return 1
+
+  opt = (' -O2' if (len(opt_option) > 0 and int(opt_option[0]) > 0)
+         else ' -g -G')
+
+  includes = (' -I ' + ' -I '.join(include_options)
+              if len(include_options) > 0
+              else '')
+
+  # Unfortunately, there are other options that have -c prefix too.
+  # So allowing only those look like C/C++ files.
+  src_files = [f for f in src_files if
+               re.search('\.cpp$|\.cc$|\.c$|\.cxx$|\.C$', f)]
+  srcs = ' '.join(src_files)
+  out = ' -o ' + out_file[0]
+
+  supported_cuda_compute_capabilities = [ "3.0" ]
+  nvccopts = '-D_FORCE_INLINES '
+  for capability in supported_cuda_compute_capabilities:
+    capability = capability.replace('.', '')
+    nvccopts += r'-gencode=arch=compute_%s,\"code=sm_%s,compute_%s\" ' % (
+        capability, capability, capability)
+  nvccopts += ' ' + nvcc_compiler_options
+  nvccopts += undefines
+  nvccopts += defines
+  nvccopts += std_options
+  nvccopts += m_options
+  nvccopts += warning_options
+
+  if depfiles:
+    # Generate the dependency file
+    depfile = depfiles[0]
+    cmd = (NVCC_PATH + ' ' + nvccopts +
+           ' --compiler-options "' + host_compiler_options + '"' +
+           ' --compiler-bindir=' + GCC_HOST_COMPILER_PATH +
+           ' -I .' +
+           ' -x cu ' + opt + includes + ' ' + srcs + ' -M -o ' + depfile)
+    if log: Log(cmd)
+    exit_status = os.system(cmd)
+    if exit_status != 0:
+      return exit_status
+
+  cmd = (NVCC_PATH + ' ' + nvccopts +
+         ' --compiler-options "' + host_compiler_options + ' -fPIC"' +
+         ' --compiler-bindir=' + GCC_HOST_COMPILER_PATH +
+         ' -I .' +
+         ' -x cu ' + opt + includes + ' -c ' + srcs + out)
+
+  # TODO(zhengxq): for some reason, 'gcc' needs this help to find 'as'.
+  # Need to investigate and fix.
+  cmd = 'PATH=' + PREFIX_DIR + ':$PATH ' + cmd
+  if log: Log(cmd)
+  return os.system(cmd)
+
+
+def main():
+  parser = ArgumentParser()
+  parser.add_argument('-x', nargs=1)
+  parser.add_argument('--cuda_log', action='store_true')
+  args, leftover = parser.parse_known_args(sys.argv[1:])
+
+  if args.x and args.x[0] == 'cuda':
+    if args.cuda_log: Log('-x cuda')
+    leftover = [pipes.quote(s) for s in leftover]
+    if args.cuda_log: Log('using nvcc')
+    return InvokeNvcc(leftover, log=args.cuda_log)
+
+  # Strip our flags before passing through to the CPU compiler for files which
+  # are not -x cuda. We can't just pass 'leftover' because it also strips -x.
+  # We not only want to pass -x to the CPU compiler, but also keep it in its
+  # relative location in the argv list (the compiler is actually sensitive to
+  # this).
+  cpu_compiler_flags = [flag for flag in sys.argv[1:]
+                             if not flag.startswith(('--cuda_log'))]
+
+  return subprocess.call([CPU_COMPILER] + cpu_compiler_flags)
+
+if __name__ == '__main__':
+  sys.exit(main())
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/gcc7-nvcc-cuda10.0/windows/msvc_wrapper_for_nvcc.py b/third_party/toolchains/preconfig/ubuntu14.04/gcc7-nvcc-cuda10.0/windows/msvc_wrapper_for_nvcc.py
new file mode 100755
index 0000000000..cb1385dccd
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu14.04/gcc7-nvcc-cuda10.0/windows/msvc_wrapper_for_nvcc.py
@@ -0,0 +1,192 @@
+#!/usr/bin/env python
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Crosstool wrapper for compiling CUDA programs with nvcc on Windows.
+
+DESCRIPTION:
+  This script is the Windows version of //third_party/gpus/crosstool/crosstool_wrapper_is_not_gcc
+"""
+
+from __future__ import print_function
+
+from argparse import ArgumentParser
+import os
+import subprocess
+import re
+import sys
+import pipes
+
+# Template values set by cuda_autoconf.
+CPU_COMPILER = ('/usr/bin/gcc-7')
+GCC_HOST_COMPILER_PATH = ('/usr/bin/gcc-7')
+
+NVCC_PATH = '/usr/local/cuda-10.0/bin/nvcc'
+NVCC_VERSION = '10.0'
+NVCC_TEMP_DIR = "C:\\Windows\\Temp\\nvcc_inter_files_tmp_dir"
+supported_cuda_compute_capabilities = [ "3.0" ]
+
+def Log(s):
+  print('gpus/crosstool: {0}'.format(s))
+
+
+def GetOptionValue(argv, option):
+  """Extract the list of values for option from options.
+
+  Args:
+    option: The option whose value to extract, without the leading '/'.
+
+  Returns:
+    1. A list of values, either directly following the option,
+    (eg., /opt val1 val2) or values collected from multiple occurrences of
+    the option (eg., /opt val1 /opt val2).
+    2. The leftover options.
+  """
+
+  parser = ArgumentParser(prefix_chars='/')
+  parser.add_argument('/' + option, nargs='*', action='append')
+  args, leftover = parser.parse_known_args(argv)
+  if args and vars(args)[option]:
+    return (sum(vars(args)[option], []), leftover)
+  return ([], leftover)
+
+def _update_options(nvcc_options):
+  if NVCC_VERSION in ("7.0",):
+    return nvcc_options
+
+  update_options = { "relaxed-constexpr" : "expt-relaxed-constexpr" }
+  return [ update_options[opt] if opt in update_options else opt
+                    for opt in nvcc_options ]
+
+def GetNvccOptions(argv):
+  """Collect the -nvcc_options values from argv.
+
+  Args:
+    argv: A list of strings, possibly the argv passed to main().
+
+  Returns:
+    1. The string that can be passed directly to nvcc.
+    2. The leftover options.
+  """
+
+  parser = ArgumentParser()
+  parser.add_argument('-nvcc_options', nargs='*', action='append')
+
+  args, leftover = parser.parse_known_args(argv)
+
+  if args.nvcc_options:
+    options = _update_options(sum(args.nvcc_options, []))
+    return (['--' + a for a in options], leftover)
+  return ([], leftover)
+
+
+def InvokeNvcc(argv, log=False):
+  """Call nvcc with arguments assembled from argv.
+
+  Args:
+    argv: A list of strings, possibly the argv passed to main().
+    log: True if logging is requested.
+
+  Returns:
+    The return value of calling os.system('nvcc ' + args)
+  """
+
+  src_files = [f for f in argv if
+               re.search('\.cpp$|\.cc$|\.c$|\.cxx$|\.C$', f)]
+  if len(src_files) == 0:
+    raise Error('No source files found for cuda compilation.')
+
+  out_file = [ f for f in argv if f.startswith('/Fo') ]
+  if len(out_file) != 1:
+    raise Error('Please sepecify exactly one output file for cuda compilation.')
+  out = ['-o', out_file[0][len('/Fo'):]]
+
+  nvcc_compiler_options, argv = GetNvccOptions(argv)
+
+  opt_option, argv = GetOptionValue(argv, 'O')
+  opt = ['-g', '-G']
+  if (len(opt_option) > 0 and opt_option[0] != 'd'):
+    opt = ['-O2']
+
+  include_options, argv = GetOptionValue(argv, 'I')
+  includes = ["-I " + include for include in include_options]
+
+  defines, argv = GetOptionValue(argv, 'D')
+  defines = ['-D' + define for define in defines]
+
+  undefines, argv = GetOptionValue(argv, 'U')
+  undefines = ['-U' + define for define in undefines]
+
+  # The rest of the unrecongized options should be passed to host compiler
+  host_compiler_options = [option for option in argv if option not in (src_files + out_file)]
+
+  m_options = ["-m64"]
+
+  nvccopts = ['-D_FORCE_INLINES']
+  for capability in supported_cuda_compute_capabilities:
+    capability = capability.replace('.', '')
+    nvccopts += [r'-gencode=arch=compute_%s,"code=sm_%s,compute_%s"' % (
+        capability, capability, capability)]
+  nvccopts += nvcc_compiler_options
+  nvccopts += undefines
+  nvccopts += defines
+  nvccopts += m_options
+  nvccopts += ['--compiler-options="' + " ".join(host_compiler_options) + '"']
+  nvccopts += ['-x', 'cu'] + opt + includes + out + ['-c'] + src_files
+  # If we don't specify --keep-dir, nvcc will generate intermediate files under TEMP
+  # Put them under NVCC_TEMP_DIR instead, then Bazel can ignore files under NVCC_TEMP_DIR during dependency check
+  # http://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/index.html#options-for-guiding-compiler-driver
+  # Different actions are sharing NVCC_TEMP_DIR, so we cannot remove it if the directory already exists.
+  if os.path.isfile(NVCC_TEMP_DIR):
+    os.remove(NVCC_TEMP_DIR)
+  if not os.path.exists(NVCC_TEMP_DIR):
+    os.makedirs(NVCC_TEMP_DIR)
+  nvccopts += ['--keep', '--keep-dir', NVCC_TEMP_DIR]
+  cmd = [NVCC_PATH] + nvccopts
+  if log:
+    Log(cmd)
+  proc = subprocess.Popen(cmd,
+                          stdout=sys.stdout,
+                          stderr=sys.stderr,
+                          env=os.environ.copy(),
+                          shell=True)
+  proc.wait()
+  return proc.returncode
+
+def main():
+  parser = ArgumentParser()
+  parser.add_argument('-x', nargs=1)
+  parser.add_argument('--cuda_log', action='store_true')
+  args, leftover = parser.parse_known_args(sys.argv[1:])
+
+  if args.x and args.x[0] == 'cuda':
+    if args.cuda_log: Log('-x cuda')
+    leftover = [pipes.quote(s) for s in leftover]
+    if args.cuda_log: Log('using nvcc')
+    return InvokeNvcc(leftover, log=args.cuda_log)
+
+  # Strip our flags before passing through to the CPU compiler for files which
+  # are not -x cuda. We can't just pass 'leftover' because it also strips -x.
+  # We not only want to pass -x to the CPU compiler, but also keep it in its
+  # relative location in the argv list (the compiler is actually sensitive to
+  # this).
+  cpu_compiler_flags = [flag for flag in sys.argv[1:]
+                             if not flag.startswith(('--cuda_log'))
+                             and not flag.startswith(('-nvcc_options'))]
+
+  return subprocess.call([CPU_COMPILER] + cpu_compiler_flags)
+
+if __name__ == '__main__':
+  sys.exit(main())
-- 
GitLab


From 742d01ba6dec581da977b1f41728175cc87d0fa2 Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Thu, 24 Jan 2019 04:05:20 -0800
Subject: [PATCH 1180/2345] Make _ListWrapper implement all list methods, add
 append only methods to List.

By extending list, _ListWrapper is both a list, and a pointer to a list
(self._storage). As such we need to be careful to overload all methods from
list to avoid having two places where items could be read or written to.

To try and prevent this issue coming up again I've added a test which ensures
that all methods from list are overriden.

PiperOrigin-RevId: 230695450
---
 .../checkpointable/data_structures.py         |  89 +++++++++--
 .../checkpointable/data_structures_test.py    | 143 ++++++++++++++++--
 2 files changed, 206 insertions(+), 26 deletions(-)

diff --git a/tensorflow/python/training/checkpointable/data_structures.py b/tensorflow/python/training/checkpointable/data_structures.py
index 5a5b444a6c..23a6ec51b2 100644
--- a/tensorflow/python/training/checkpointable/data_structures.py
+++ b/tensorflow/python/training/checkpointable/data_structures.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 import collections
 import copy
 import operator
+import sys
 
 import six
 
@@ -260,9 +261,12 @@ class List(CheckpointableDataStructure, collections.Sequence):
       self._storage[index] = self._track_value(
           element, name=self._name_element(index))
 
-  def __copy__(self):
+  def copy(self):
     return type(self)(copy.copy(self._storage))
 
+  def __copy__(self):
+    return self.copy()
+
   def __deepcopy__(self, memo):
     return type(self)(copy.deepcopy(self._storage, memo))
 
@@ -285,18 +289,33 @@ class List(CheckpointableDataStructure, collections.Sequence):
   def extend(self, values):
     """Add a sequence of checkpointable values."""
     for value in values:
-      self._storage.append(self._track_value(
-          value, name=self._name_element(len(self._storage))))
+      self.append(value)
 
   def __iadd__(self, values):
     self.extend(values)
     return self
 
   def __add__(self, other):
-    if isinstance(other, List):
-      return self.__class__(self._storage + other._storage)  # pylint: disable=protected-access
-    else:
-      return self.__class__(self._storage + other)
+    return self.__class__(self._storage + getattr(other, "_storage", other))
+
+  def __imul__(self, y):
+    if y <= 0:
+      raise ValueError(
+          "List only supports append, multiplying in place by %d removes "
+          "elements." % y)
+
+    n = len(self._storage)
+    for _ in range(y - 1):
+      for i in range(n):
+        self.append(self._storage[i])
+
+    return self
+
+  def __mul__(self, n):
+    return self.__class__(self._storage * n)
+
+  def __rmul__(self, n):
+    return self * n
 
   def __radd__(self, other):
     return self + other
@@ -313,7 +332,11 @@ class List(CheckpointableDataStructure, collections.Sequence):
   def __repr__(self):
     return "List(%s)" % (repr(self._storage),)
 
+  def __sizeof__(self):
+    return super(List, self).__sizeof__() + sys.getsizeof(self._storage)
 
+
+# TODO(tomhennigan) Update to collections.UserList?
 class _ListWrapper(List, collections.MutableSequence,
                    # Shadowed, but there for isinstance checks.
                    list):
@@ -386,12 +409,12 @@ class _ListWrapper(List, collections.MutableSequence,
       raise ValueError(
           ("Unable to save the object %s (a list wrapper constructed to track "
            "checkpointable TensorFlow objects). A list element was replaced "
-           "(__setitem__), deleted, or inserted. In order to support "
-           "restoration on object creation, tracking is exclusively for "
-           "append-only data structures.\n\nIf you don't need this list "
-           "checkpointed, wrap it in a tf.contrib.checkpoint.NoDependency "
-           "object; it will be automatically un-wrapped and subsequently "
-           "ignored." % (self,)))
+           "(__setitem__, __setslice__), deleted (__delitem__, __delslice__), "
+           "or moved (sort). In order to support restoration on object "
+           "creation, tracking is exclusively for append-only data structures."
+           "\n\nIf you don't need this list checkpointed, wrap it in a "
+           "tf.contrib.checkpoint.NoDependency object; it will be "
+           "automatically un-wrapped and subsequently ignored." % (self,)))
     if self._external_modification:
       raise ValueError(
           ("Unable to save the object %s (a list wrapper constructed to track "
@@ -410,9 +433,32 @@ class _ListWrapper(List, collections.MutableSequence,
 
   def __setitem__(self, key, value):
     self._check_external_modification()
-    if isinstance(self._storage[key], base.Checkpointable):
-      self._non_append_mutation = True
-    self._storage[key] = self._track_value(value, self._name_element(key))
+
+    if isinstance(key, slice):
+      # Note: this is quite inefficient, but the list API supports a broad range
+      # of slice setters (e.g. truncate, extend, replace) and immitating this
+      # for a range of Python versions is non-trivial.
+      storage_copy = list(self._storage)
+      self._storage[key] = value
+
+      len_before = len(storage_copy)
+      len_now = len(self._storage)
+      for i in range(max(len_before, len_now)):
+        value_now = self._storage[i] if i < len_now else None
+        value_before = storage_copy[i] if i < len_before else None
+
+        if isinstance(value_before, base.Checkpointable):
+          self._non_append_mutation = True
+
+        if value_now is not None and value_now != value_before:
+          self._storage[i] = self._track_value(self._storage[i],
+                                               self._name_element(i))
+
+    else:
+      if isinstance(self._storage[key], base.Checkpointable):
+        self._non_append_mutation = True
+      self._storage[key] = self._track_value(value, self._name_element(key))
+
     self._update_snapshot()
 
   def append(self, value):
@@ -454,6 +500,17 @@ class _ListWrapper(List, collections.MutableSequence,
     self._non_append_mutation = True
     self._storage.insert(index, obj)
 
+  def sort(self):
+    self._non_append_mutation = True
+    self._storage.sort()
+
+  def __setslice__(self, i, j, y):
+    self.__setitem__(slice(i, j), y)
+
+  def __delslice__(self, i, j):
+    self._non_append_mutation = True
+    del self._storage[slice(i, j)]
+
   def _track_value(self, value, name):
     """Allows storage of non-checkpointable objects."""
     try:
diff --git a/tensorflow/python/training/checkpointable/data_structures_test.py b/tensorflow/python/training/checkpointable/data_structures_test.py
index 53cbd66482..72045876b6 100644
--- a/tensorflow/python/training/checkpointable/data_structures_test.py
+++ b/tensorflow/python/training/checkpointable/data_structures_test.py
@@ -207,6 +207,83 @@ class ListTests(test.TestCase):
     self.assertEqual([v], l.trainable_weights)
     self.assertEqual([v2], l.non_trainable_weights)
 
+  def testCopy(self):
+    v1 = resource_variable_ops.ResourceVariable(1.)
+    v2 = resource_variable_ops.ResourceVariable(1.)
+    v3 = resource_variable_ops.ResourceVariable(1.)
+
+    l1 = data_structures.List([v1, v2])
+    l2 = l1.copy()
+    l2.append(v3)
+    self.assertEqual(list(l1), [v1, v2])
+    self.assertEqual(list(l2), [v1, v2, v3])
+
+  def testSlicing(self):
+    v1 = resource_variable_ops.ResourceVariable(1.)
+    v2 = resource_variable_ops.ResourceVariable(1.)
+    v3 = resource_variable_ops.ResourceVariable(1.)
+    v4 = resource_variable_ops.ResourceVariable(1.)
+
+    l = data_structures.List([v1, v2, v3, v4])
+    self.assertEqual(l[1:], [v2, v3, v4])
+    self.assertEqual(l[1:-1], [v2, v3])
+    self.assertEqual(l[:-1], [v1, v2, v3])
+
+  def testHash(self):
+    has_sequences = set([data_structures.List(),
+                         data_structures.List()])
+    self.assertEqual(2, len(has_sequences))
+    self.assertNotIn(data_structures.List(), has_sequences)
+
+  def testIMul_zero(self):
+    l = data_structures.List([])
+    with self.assertRaisesRegexp(ValueError, "List only supports append"):
+      l *= 0
+
+  def testIMul(self):
+    v = resource_variable_ops.ResourceVariable(1.)
+    l = data_structures.List([v])
+    l *= 2
+    self.assertEqual(list(l), [v] * 2)
+
+  def testMul(self):
+    v = resource_variable_ops.ResourceVariable(1.)
+    l = data_structures.List([v, v, v])
+    self.assertEqual(list(l * 2), [v, v, v] * 2)
+
+  def testRMul(self):
+    v = resource_variable_ops.ResourceVariable(1.)
+    l = data_structures.List([v, v, v])
+    self.assertEqual(list(2 * l), [v, v, v] * 2)
+
+
+class ListWrapperTest(test.TestCase):
+
+  IGNORED = ("__new__", "__init__", "__subclasshook__", "__getattribute__")
+
+  def test_overrides_all_list_methods(self):
+    not_overridden = []
+
+    for name in dir(list):
+      if name in ListWrapperTest.IGNORED:
+        continue
+
+      list_method = getattr(list, name)
+
+      if not callable(list_method):
+        continue
+
+      object_method = getattr(object, name, None)
+      if object_method is not None and object_method == list_method:
+        # Skip methods that aren't overridden from object.
+        continue
+
+      if list_method == getattr(data_structures._ListWrapper, name):
+        not_overridden.append(name)
+
+    if not_overridden:
+      self.fail("_ListWrapper does not override %s" % (not_overridden))
+
   def testListWrapperBasic(self):
     # _ListWrapper, unlike List, compares like the built-in list type (since it
     # is used to automatically replace lists).
@@ -244,6 +321,10 @@ class ListTests(test.TestCase):
     self.assertEqual([a, a], [a] + data_structures._ListWrapper([a]))
     self.assertIsInstance(data_structures._ListWrapper([a]), list)
 
+  def testAcceptsNonCheckpointableContent(self):
+    l = data_structures._ListWrapper([1, 2, 3])
+    self.assertEqual(l, [1, 2, 3])
+
   def testWrapperChangesList(self):
     l = []
     l_wrapper = data_structures._ListWrapper(l)
@@ -263,19 +344,61 @@ class ListTests(test.TestCase):
     l.append(layer)
     self.assertEqual([layer], l_wrapper.layers)
 
-  def testHashing(self):
-    has_sequences = set([data_structures.List(),
-                         data_structures.List()])
-    self.assertEqual(2, len(has_sequences))
-    self.assertNotIn(data_structures.List(), has_sequences)
+  def testNotHashable(self):
     with self.assertRaises(TypeError):
-      has_sequences.add(data_structures._ListWrapper([]))
+      hash(data_structures._ListWrapper())
 
-  def testSlicing(self):
+  def testDelItem(self):
+    l = data_structures._ListWrapper([1, 2, 3, 4])
+    del l[0]
+    self.assertEqual(l, [2, 3, 4])
+    self.assertUnableToSave(l, "Unable to save .*__delitem__")
+
+  def testDelSlice(self):
+    l = data_structures._ListWrapper([1, 2, 3, 4])
+    del l[2:3]
+    self.assertEqual(l, [1, 2, 4])
+    self.assertUnableToSave(l, "Unable to save .*__delslice__")
+
+  def testSetSlice_canSaveForNonCheckpointableItems(self):
+    l = data_structures._ListWrapper([1, 2, 3, 4])
+    l[:] = 2, 8, 9, 0
+    self.assertEqual(l, [2, 8, 9, 0])
+    l._maybe_initialize_checkpointable()  # pylint: disable=protected-access
+    self.assertEqual(len(l._checkpoint_dependencies), 0)  # pylint: disable=protected-access
+
+  def testSetSlice_cannotSaveIfCheckpointableModified(self):
+    v1 = resource_variable_ops.ResourceVariable(1.)
+    v2 = resource_variable_ops.ResourceVariable(1.)
+    l = data_structures._ListWrapper([1, 2, v1, v2])
+    l[:] = 2, 8, 9, v2
+    self.assertEqual(l, [2, 8, 9, v2])
+    self.assertUnableToSave(l, "Unable to save .*__setslice__")
+
+  def testSetSlice_truncate(self):
+    l = data_structures._ListWrapper([1, 2, 3, 4])
+    l[:] = []
+    self.assertEqual(l, [])
+
+  def testSetSlice_extend(self):
+    l = data_structures._ListWrapper([1, 2, 3, 4])
+    l[2:] = 1, 2, 3, 4
+    self.assertEqual(l, [1, 2, 1, 2, 3, 4])
+
+  def testSort(self):
     l = data_structures._ListWrapper([1, 2, 3, 4])
-    self.assertEqual(l[1:], [2, 3, 4])
-    self.assertEqual(l[1:-1], [2, 3])
-    self.assertEqual(l[:-1], [1, 2, 3])
+    l.sort()
+    self.assertEqual(l, [1, 2, 3, 4])
+    # Regardless of being a no-op for the input list, we still refuse to save.
+    # This is intentional since otherwise we would end up with a hard to debug
+    # case for users (e.g. sometimes sort on a ListWrapper is checkpointable and
+    # other times it is not).
+    self.assertUnableToSave(l, "Unable to save .*sort")
+
+  def assertUnableToSave(self, l, msg):
+    l._maybe_initialize_checkpointable()  # pylint: disable=protected-access
+    with self.assertRaisesRegexp(ValueError, msg):
+      return l._checkpoint_dependencies  # pylint: disable=protected-access
 
 
 class HasMapping(training.Model):
-- 
GitLab


From 871da6af454d2298093d35b9ce8d516854ad4414 Mon Sep 17 00:00:00 2001
From: Andr? Susano Pinto <andresp@google.com>
Date: Thu, 24 Jan 2019 04:08:00 -0800
Subject: [PATCH 1181/2345] Fix loading of nested functions via
 tf.saved_model.load().

PiperOrigin-RevId: 230695867
---
 .../python/saved_model/function_deserialization.py     |  2 ++
 tensorflow/python/saved_model/load_test.py             | 10 ++--------
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py
index 903331cee0..d1869ba0f7 100644
--- a/tensorflow/python/saved_model/function_deserialization.py
+++ b/tensorflow/python/saved_model/function_deserialization.py
@@ -198,6 +198,8 @@ def load_function_def_library(library):
     copy = _fix_fdef(fdef, functions)
 
     func_graph = function_def_lib.function_def_to_graph(copy)
+    for dep in _list_function_deps(fdef):
+      functions[dep].add_to_graph(func_graph)
     func = function_lib.ConcreteFunction(func_graph)
     func.add_to_graph()
 
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index d4742dde1c..20d943a7c3 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -191,10 +191,7 @@ class LoadTest(test.TestCase, parameterized.TestCase):
 
     root = tracking.AutoCheckpointable()
     root.g = g
-    # TODO(vbardiovsky): Enable this test. For this to work, we must ensure that
-    # concrete_function._inference_function._graph._functions contains all
-    # functions that were on the graph before saving.
-    imported = self.cycle(root, 1)
+    imported = self.cycle(root, cycles)
     imported.g(constant_op.constant([1.0]))
 
   def test_function_with_default_bool_input(self, cycles):
@@ -460,10 +457,7 @@ class LoadTest(test.TestCase, parameterized.TestCase):
     root.bias = bias
     root.g = h
 
-    # TODO(vbardiovsky): Enable this test. For this to work, we must ensure that
-    # concrete_function._inference_function._graph._functions contains all
-    # functions that were on the graph before saving.
-    imported = self.cycle(root, 1)
+    imported = self.cycle(root, cycles)
     with backprop.GradientTape() as t:
       x = constant_op.constant([3.5])
       loss = imported.g(x)
-- 
GitLab


From ae1418f9180c53687ce716bbf19a0a15827db567 Mon Sep 17 00:00:00 2001
From: Lukas Geiger <lukas.geiger94@gmail.com>
Date: Thu, 24 Jan 2019 13:59:46 +0100
Subject: [PATCH 1182/2345] Replace deprecated usage of np.asscalar with
 np.ndarray.item()

[`numpy.asscalar()` is deprecated since version 1.16](https://github.com/numpy/numpy/blob/master/numpy/lib/type_check.py#L519-L548).

This PR replaces its usage with [`numpy.ndarray.item()`](https://www.numpy.org/devdocs/reference/generated/numpy.ndarray.item.html)
---
 .../compiler/xla/python_api/xla_literal.py    |  2 +-
 .../python/framework/fast_tensor_util.pyx     |  2 +-
 tensorflow/python/framework/tensor_util.py    | 26 +++++++++----------
 .../python/kernel_tests/cwise_ops_test.py     |  2 +-
 tensorflow/python/ops/nn_ops.py               |  2 +-
 5 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/tensorflow/compiler/xla/python_api/xla_literal.py b/tensorflow/compiler/xla/python_api/xla_literal.py
index 757e41a78a..19bd685ab2 100644
--- a/tensorflow/compiler/xla/python_api/xla_literal.py
+++ b/tensorflow/compiler/xla/python_api/xla_literal.py
@@ -69,7 +69,7 @@ def _ConvertNumpyArrayToLiteral(ndarray):
 
   if ndarray.ndim == 0:
     getattr(literal, type_record.literal_field_name).append(
-        _np.asscalar(ndarray.astype(type_record.literal_field_type)))
+        ndarray.astype(type_record.literal_field_type).item())
   else:
     # Ndarrays with boolean dtypes need special type conversion with protobufs
     if ndarray.dtype in {_np.bool_, _np.dtype('bool')}:
diff --git a/tensorflow/python/framework/fast_tensor_util.pyx b/tensorflow/python/framework/fast_tensor_util.pyx
index 2e3e15f53a..dc14361d66 100644
--- a/tensorflow/python/framework/fast_tensor_util.pyx
+++ b/tensorflow/python/framework/fast_tensor_util.pyx
@@ -131,4 +131,4 @@ def AppendBoolArrayToTensorProto(tensor_proto, nparray):
   cdef long i, n
   n = nparray.size
   for i in range(n):
-    tensor_proto.bool_val.append(np.asscalar(nparray[i]))
+    tensor_proto.bool_val.append(nparray.item(i))
diff --git a/tensorflow/python/framework/tensor_util.py b/tensorflow/python/framework/tensor_util.py
index ca8b067935..47e2437e20 100644
--- a/tensorflow/python/framework/tensor_util.py
+++ b/tensorflow/python/framework/tensor_util.py
@@ -42,7 +42,7 @@ from tensorflow.python.util.tf_export import tf_export
 
 
 def ExtractBitsFromFloat16(x):
-  return np.asscalar(np.asarray(x, dtype=np.float16).view(np.uint16))
+  return np.asarray(x, dtype=np.float16).view(np.uint16).item()
 
 
 def SlowAppendFloat16ArrayToTensorProto(tensor_proto, proto_values):
@@ -58,8 +58,8 @@ def _MediumAppendFloat16ArrayToTensorProto(tensor_proto, proto_values):
 
 
 def ExtractBitsFromBFloat16(x):
-  return np.asscalar(
-      np.asarray(x, dtype=dtypes.bfloat16.as_numpy_dtype).view(np.uint16))
+  return np.asarray(
+      x, dtype=dtypes.bfloat16.as_numpy_dtype).view(np.uint16).item()
 
 
 def SlowAppendBFloat16ArrayToTensorProto(tensor_proto, proto_values):
@@ -122,39 +122,39 @@ if _FAST_TENSOR_UTIL_AVAILABLE:
 else:
 
   def SlowAppendFloat32ArrayToTensorProto(tensor_proto, proto_values):
-    tensor_proto.float_val.extend([np.asscalar(x) for x in proto_values])
+    tensor_proto.float_val.extend([x.item() for x in proto_values])
 
   def SlowAppendFloat64ArrayToTensorProto(tensor_proto, proto_values):
-    tensor_proto.double_val.extend([np.asscalar(x) for x in proto_values])
+    tensor_proto.double_val.extend([x.item() for x in proto_values])
 
   def SlowAppendIntArrayToTensorProto(tensor_proto, proto_values):
-    tensor_proto.int_val.extend([np.asscalar(x) for x in proto_values])
+    tensor_proto.int_val.extend([x.item() for x in proto_values])
 
   def SlowAppendInt64ArrayToTensorProto(tensor_proto, proto_values):
-    tensor_proto.int64_val.extend([np.asscalar(x) for x in proto_values])
+    tensor_proto.int64_val.extend([x.item() for x in proto_values])
 
   def SlowAppendQIntArrayToTensorProto(tensor_proto, proto_values):
-    tensor_proto.int_val.extend([np.asscalar(x[0]) for x in proto_values])
+    tensor_proto.int_val.extend([x.item(0) for x in proto_values])
 
   def SlowAppendUInt32ArrayToTensorProto(tensor_proto, proto_values):
-    tensor_proto.uint32_val.extend([np.asscalar(x) for x in proto_values])
+    tensor_proto.uint32_val.extend([x.item() for x in proto_values])
 
   def SlowAppendUInt64ArrayToTensorProto(tensor_proto, proto_values):
-    tensor_proto.uint64_val.extend([np.asscalar(x) for x in proto_values])
+    tensor_proto.uint64_val.extend([x.item() for x in proto_values])
 
   def SlowAppendComplex64ArrayToTensorProto(tensor_proto, proto_values):
     tensor_proto.scomplex_val.extend(
-        [np.asscalar(v) for x in proto_values for v in [x.real, x.imag]])
+        [v.item() for x in proto_values for v in [x.real, x.imag]])
 
   def SlowAppendComplex128ArrayToTensorProto(tensor_proto, proto_values):
     tensor_proto.dcomplex_val.extend(
-        [np.asscalar(v) for x in proto_values for v in [x.real, x.imag]])
+        [v.item() for x in proto_values for v in [x.real, x.imag]])
 
   def SlowAppendObjectArrayToTensorProto(tensor_proto, proto_values):
     tensor_proto.string_val.extend([compat.as_bytes(x) for x in proto_values])
 
   def SlowAppendBoolArrayToTensorProto(tensor_proto, proto_values):
-    tensor_proto.bool_val.extend([np.asscalar(x) for x in proto_values])
+    tensor_proto.bool_val.extend([x.item() for x in proto_values])
 
   _NP_TO_APPEND_FN = {
       dtypes.bfloat16.as_numpy_dtype: SlowAppendBFloat16ArrayToTensorProto,
diff --git a/tensorflow/python/kernel_tests/cwise_ops_test.py b/tensorflow/python/kernel_tests/cwise_ops_test.py
index 70f19f9d2f..29d335d68c 100644
--- a/tensorflow/python/kernel_tests/cwise_ops_test.py
+++ b/tensorflow/python/kernel_tests/cwise_ops_test.py
@@ -595,7 +595,7 @@ class MinMaxOpTest(test.TestCase):
 
   def testScalar(self):
     x = np.random.rand(1, 3, 2) * 100.
-    y = np.asscalar(np.random.rand(1) * 100.)  # should broadcast
+    y = np.random.rand(1).item() * 100.  # should broadcast
     # dropped np.float64, int64 because TF automatically converts to 32 bit
     for t in [np.float32, np.int32]:
       self._compare(x.astype(t), t(y), use_gpu=False)
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index e0ef9e5e34..25e9ffd34f 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -2334,7 +2334,7 @@ def leaky_relu(features, alpha=0.2, name=None):
       features = math_ops.to_float(features)
     if compat.forward_compatible(2018, 11, 1):
       if isinstance(alpha, np.ndarray):
-        alpha = np.asscalar(alpha)
+        alpha = alpha.item()
       return gen_nn_ops.leaky_relu(features, alpha=alpha, name=name)
     alpha = ops.convert_to_tensor(alpha, dtype=features.dtype, name="alpha")
     return math_ops.maximum(alpha * features, features, name=name)
-- 
GitLab


From 146dffb15a6b03542f311a178f2d915c4fd9216c Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Thu, 24 Jan 2019 06:13:06 -0800
Subject: [PATCH 1183/2345] Expose a few autograph debugging primitives.

PiperOrigin-RevId: 230709287
---
 tensorflow/python/autograph/__init__.py       |  1 +
 .../python/autograph/utils/ag_logging.py      | 54 +++++++++++++++----
 .../api/golden/v1/tensorflow.autograph.pbtxt  |  8 +++
 .../api/golden/v2/tensorflow.autograph.pbtxt  |  8 +++
 4 files changed, 62 insertions(+), 9 deletions(-)

diff --git a/tensorflow/python/autograph/__init__.py b/tensorflow/python/autograph/__init__.py
index 8fd6e72da6..a1956d1d8a 100644
--- a/tensorflow/python/autograph/__init__.py
+++ b/tensorflow/python/autograph/__init__.py
@@ -51,6 +51,7 @@ from tensorflow.python.autograph.lang.directives import set_loop_options
 from tensorflow.python.autograph.lang.special_functions import stack
 from tensorflow.python.autograph.lang.special_functions import tensor_list
 from tensorflow.python.autograph.pyct.transformer import AutoGraphParseError
+from tensorflow.python.autograph.utils import ag_logging
 from tensorflow.python.util.all_util import remove_undocumented
 
 # TODO(mdan): Revisit this list once we finalize the generated code mechanism.
diff --git a/tensorflow/python/autograph/utils/ag_logging.py b/tensorflow/python/autograph/utils/ag_logging.py
index dd0c271a1a..17fc8f58be 100644
--- a/tensorflow/python/autograph/utils/ag_logging.py
+++ b/tensorflow/python/autograph/utils/ag_logging.py
@@ -40,23 +40,42 @@ if hasattr(sys, 'ps1') or hasattr(sys, 'ps2'):
 def set_verbosity(level, alsologtostdout=False):
   """Sets the AutoGraph verbosity level.
 
-  # Debug logging in AutoGraph
+  _Debug logging in AutoGraph_
 
-  More verbose logging is useful to enable when filing bug reports or debugging
-  things.
+  More verbose logging is useful to enable when filing bug reports or doing
+  more in-depth debugging.
 
   There are two controls that control the logging verbosity:
    * The `set_verbosity` function
    * The `AUTOGRAPH_VERBOSITY` environment variable
   `set_verbosity` takes precedence over the environment variable.
 
-  By default, logs are output to absl's default logger, with INFO level. They
-  can be mirrored to stdout by using the `alsologtostdout` argument. Mirroring
-  is enabled by default when Python runs in interactive mode.
+  For example:
+
+  ```python
+  import os
+  import tensorflow as tf
+
+  os.environ['AUTOGRAPH_VERBOSITY'] = 5
+  # Verbosity is now 5
+
+  tf.autograph.set_verbosity(0)
+  # Verbosity is now 0
+
+  os.environ['AUTOGRAPH_VERBOSITY'] = 1
+  # No effect, because set_verbosity was already called.
+  ```
+
+  Logs entries are output to [absl](https://abseil.io)'s default output,
+  with `INFO` level.
+  Logs can be mirrored to stdout by using the `alsologtostdout` argument.
+  Mirroring is enabled by default when Python runs in interactive mode.
 
   Args:
-    level: int, the verbosity level; uses the same scale as vlog
-    alsologtostdout: bool, whether to ech log messages to stdout
+    level: int, the verbosity level; larger values specify increased verbosity;
+      0 means no logging. When reporting bugs, it is recommended to set this
+      value to a larges number, like 10.
+    alsologtostdout: bool, whether to also output log messages to `sys.stdout`.
   """
   global verbosity_level
   global echo_log_to_stdout
@@ -66,7 +85,24 @@ def set_verbosity(level, alsologtostdout=False):
 
 @tf_export('autograph.trace')
 def trace(*args):
-  """Traces argument information at compilation time."""
+  """Traces argument information at compilation time.
+
+  `trace` is useful when debugging, and it always executes during the tracing
+  phase, that is, when the TF graph is constructed.
+
+  _Example usage_
+
+  ```python
+  import tensorflow as tf
+
+  for i in tf.range(10):
+    tf.autograph.trace(i)
+  # Output: <Tensor ...>
+  ```
+
+  Args:
+    *args: Arguments to print to `sys.stdout`.
+  """
   print(*args)
 
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.autograph.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.autograph.pbtxt
index 12e23bc0c8..0baf6e0355 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.autograph.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.autograph.pbtxt
@@ -4,6 +4,10 @@ tf_module {
     name: "experimental"
     mtype: "<type \'module\'>"
   }
+  member_method {
+    name: "set_verbosity"
+    argspec: "args=[\'level\', \'alsologtostdout\'], varargs=None, keywords=None, defaults=[\'False\'], "
+  }
   member_method {
     name: "to_code"
     argspec: "args=[\'entity\', \'recursive\', \'arg_values\', \'arg_types\', \'indentation\', \'experimental_optional_features\', \'experimental_partial_types\'], varargs=None, keywords=None, defaults=[\'True\', \'None\', \'None\', \'  \', \'Feature.ALL\', \'None\'], "
@@ -12,4 +16,8 @@ tf_module {
     name: "to_graph"
     argspec: "args=[\'entity\', \'recursive\', \'arg_values\', \'arg_types\', \'experimental_optional_features\', \'experimental_strip_decorators\', \'experimental_verbose\', \'experimental_partial_types\'], varargs=None, keywords=None, defaults=[\'True\', \'None\', \'None\', \'Feature.ALL\', \'None\', \'Verbosity.BRIEF\', \'None\'], "
   }
+  member_method {
+    name: "trace"
+    argspec: "args=[], varargs=args, keywords=None, defaults=None"
+  }
 }
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.autograph.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.autograph.pbtxt
index 12e23bc0c8..0baf6e0355 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.autograph.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.autograph.pbtxt
@@ -4,6 +4,10 @@ tf_module {
     name: "experimental"
     mtype: "<type \'module\'>"
   }
+  member_method {
+    name: "set_verbosity"
+    argspec: "args=[\'level\', \'alsologtostdout\'], varargs=None, keywords=None, defaults=[\'False\'], "
+  }
   member_method {
     name: "to_code"
     argspec: "args=[\'entity\', \'recursive\', \'arg_values\', \'arg_types\', \'indentation\', \'experimental_optional_features\', \'experimental_partial_types\'], varargs=None, keywords=None, defaults=[\'True\', \'None\', \'None\', \'  \', \'Feature.ALL\', \'None\'], "
@@ -12,4 +16,8 @@ tf_module {
     name: "to_graph"
     argspec: "args=[\'entity\', \'recursive\', \'arg_values\', \'arg_types\', \'experimental_optional_features\', \'experimental_strip_decorators\', \'experimental_verbose\', \'experimental_partial_types\'], varargs=None, keywords=None, defaults=[\'True\', \'None\', \'None\', \'Feature.ALL\', \'None\', \'Verbosity.BRIEF\', \'None\'], "
   }
+  member_method {
+    name: "trace"
+    argspec: "args=[], varargs=args, keywords=None, defaults=None"
+  }
 }
-- 
GitLab


From bba8e3517de55b8436373697482536c6c98d88e1 Mon Sep 17 00:00:00 2001
From: James Keeling <jtkeeling@google.com>
Date: Thu, 24 Jan 2019 07:54:34 -0800
Subject: [PATCH 1184/2345] Remove extraneous semi-colon.

This is a tiny, tiny change, but it was causing the linter to complain when I was working on another change.

PiperOrigin-RevId: 230721053
---
 tensorflow/core/kernels/topk_op_gpu.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/topk_op_gpu.h b/tensorflow/core/kernels/topk_op_gpu.h
index 6f3bec20f6..70d6a60664 100644
--- a/tensorflow/core/kernels/topk_op_gpu.h
+++ b/tensorflow/core/kernels/topk_op_gpu.h
@@ -412,7 +412,7 @@ struct SegmentOffsetCreator {
 
   EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int operator()(int idx) const {
     return idx * num_cols_;
-  };
+  }
 
   int num_cols_;
 };
-- 
GitLab


From 9da4b68c9e15745a75efa34a57ea0337642f2da0 Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Thu, 24 Jan 2019 08:45:30 -0800
Subject: [PATCH 1185/2345] Add an API to support per channel quantization.

PiperOrigin-RevId: 230728686
---
 .../lite/tools/optimize/quantization_utils.cc | 16 ++++++++++--
 .../lite/tools/optimize/quantization_utils.h  |  7 ++++++
 .../tools/optimize/quantization_utils_test.cc | 25 +++++++++++++++++++
 3 files changed, 46 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/tools/optimize/quantization_utils.cc b/tensorflow/lite/tools/optimize/quantization_utils.cc
index bcddfe1950..445fffb8dd 100644
--- a/tensorflow/lite/tools/optimize/quantization_utils.cc
+++ b/tensorflow/lite/tools/optimize/quantization_utils.cc
@@ -127,7 +127,19 @@ void SymmetricPerChannelQuantization(const float* const input,
   }
 
   // Quantize the values.
-  memset(indices, 0, 4 * sizeof(int));
+  SymmetricPerChannelQuantizeValues(input, scale_invs, dimension,
+                                    channel_dim_index, output_value);
+}
+
+void SymmetricPerChannelQuantizeValues(const float* const input,
+                                       const std::vector<float>& scales_inv,
+                                       const std::vector<int>& dimension,
+                                       int32_t channel_dim_index,
+                                       std::vector<int8_t>* output_value) {
+  // Quantize the values.
+  int indices[4];
+  RuntimeShape tensor_dims{dimension[0], dimension[1], dimension[2],
+                           dimension[3]};
   for (indices[0] = 0; indices[0] < dimension[0]; indices[0]++) {
     for (indices[1] = 0; indices[1] < dimension[1]; indices[1]++) {
       for (indices[2] = 0; indices[2] < dimension[2]; indices[2]++) {
@@ -136,7 +148,7 @@ void SymmetricPerChannelQuantization(const float* const input,
           int index = Offset(tensor_dims, indices);
           const float val = input[index];
           const int32_t quantized_value =
-              static_cast<int32_t>(TfLiteRound(val * scale_invs[channel_idx]));
+              static_cast<int32_t>(TfLiteRound(val * scales_inv[channel_idx]));
           output_value->at(index) = std::min<int8_t>(
               kMaxQuantizedValue,
               std::max<int8_t>(kMinQuantizedValue, quantized_value));
diff --git a/tensorflow/lite/tools/optimize/quantization_utils.h b/tensorflow/lite/tools/optimize/quantization_utils.h
index c4afb0c224..d20b3176bf 100644
--- a/tensorflow/lite/tools/optimize/quantization_utils.h
+++ b/tensorflow/lite/tools/optimize/quantization_utils.h
@@ -53,6 +53,13 @@ void SymmetricPerChannelQuantization(const float* const input,
                                      std::vector<float>* output_scales,
                                      std::vector<int8_t>* output_value);
 
+// Quantize the values given an array of scales.
+void SymmetricPerChannelQuantizeValues(const float* const input,
+                                       const std::vector<float>& scales_inv,
+                                       const std::vector<int>& dimension,
+                                       int32_t channel_dim_index,
+                                       std::vector<int8_t>* output_value);
+
 }  // namespace utils
 }  // namespace optimize
 }  // namespace tflite
diff --git a/tensorflow/lite/tools/optimize/quantization_utils_test.cc b/tensorflow/lite/tools/optimize/quantization_utils_test.cc
index a3f2467c74..ecad09ed61 100644
--- a/tensorflow/lite/tools/optimize/quantization_utils_test.cc
+++ b/tensorflow/lite/tools/optimize/quantization_utils_test.cc
@@ -176,6 +176,31 @@ TEST(QuantizationUtilsTest, SymmetricPerChannelQuantization) {
   EXPECT_THAT(output_data, ElementsAreArray(expected_output_data));
 }
 
+TEST(QuantizationUtilsTest, SymmetricPerChannelQuantizeValues) {
+  // Set up an input with [3, 1, 1, 2] size and 0 is the channel index.
+  const std::vector<float> input = {
+      13.0, 21.0,  // Channel 1.
+      21.0, 22.0,  // Channel 2.
+      31.0, 40.0,  // Channel 3.
+  };
+  const std::vector<float> scales_inv = {2, 0.5, 3};
+  const std::vector<int32_t> dimension = {3, 1, 1, 2};
+  const int channel_index = 0;
+
+  // Create holder for output data.
+  std::vector<int8_t> output_data(3 * 1 * 1 * 2);
+
+  // Call SymmetricPerChannelQuantizeValues and verify the result.
+  SymmetricPerChannelQuantizeValues(input.data(), scales_inv, dimension,
+                                    channel_index, &output_data);
+  const std::vector<int8_t> expected_output_data = {
+      26, 42,   // Channel 1.
+      11, 11,   // Channel 2.
+      93, 120,  // Channel 3.
+  };
+  EXPECT_THAT(output_data, ElementsAreArray(expected_output_data));
+}
+
 }  // namespace
 }  // namespace utils
 }  // namespace optimize
-- 
GitLab


From 80adca543d2c4fc9bfdb6e7311b625fdc778cd6c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 24 Jan 2019 09:31:48 -0800
Subject: [PATCH 1186/2345] [TF:XLA] Match XLA's linspace implementation to TF
 Classic's

The last value in the result should be equal to the "stop" argument. XLA now matches this behavior.

PiperOrigin-RevId: 230735998
---
 tensorflow/compiler/tf2xla/kernels/sequence_ops.cc | 8 ++++++--
 tensorflow/python/kernel_tests/init_ops_test.py    | 1 -
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/sequence_ops.cc b/tensorflow/compiler/tf2xla/kernels/sequence_ops.cc
index b1fa2915d5..7a620d2a65 100644
--- a/tensorflow/compiler/tf2xla/kernels/sequence_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/sequence_ops.cc
@@ -157,9 +157,11 @@ class LinSpaceOp : public XlaOpKernel {
           flat(0) = start;
         } else {
           const float step = (stop - start) / (num - 1);
-          for (int64 i = 0; i < num; ++i) {
+          for (int64 i = 0; i < num - 1; ++i) {
             flat(i) = start + step * i;
           }
+          // The last value in the sequence must be equal to stop.
+          flat(num - 1) = stop;
         }
         break;
       }
@@ -171,9 +173,11 @@ class LinSpaceOp : public XlaOpKernel {
           flat(0) = start;
         } else {
           const double step = (stop - start) / (num - 1);
-          for (int64 i = 0; i < num; ++i) {
+          for (int64 i = 0; i < num - 1; ++i) {
             flat(i) = start + step * i;
           }
+          // The last value in the sequence must be equal to stop.
+          flat(num - 1) = stop;
         }
         break;
       }
diff --git a/tensorflow/python/kernel_tests/init_ops_test.py b/tensorflow/python/kernel_tests/init_ops_test.py
index 8e3f1c7089..4b9681afd2 100644
--- a/tensorflow/python/kernel_tests/init_ops_test.py
+++ b/tensorflow/python/kernel_tests/init_ops_test.py
@@ -592,7 +592,6 @@ class LinSpaceTest(test.TestCase):
       self.assertArrayNear(self._LinSpace(5., 5., 3), np.array([5.] * 3), 1e-5)
       self.assertArrayNear(self._LinSpace(5., 5., 4), np.array([5.] * 4), 1e-5)
 
-  @test_util.disable_xla("This test never passed for XLA")
   def testEndpointsAreExact(self):
     for self.force_gpu in self._gpu_modes():
       # Test some cases that produce last values not equal to "stop" when
-- 
GitLab


From 14e25a3618dbe0bb6fc94d680dbd3f0e4732631b Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Thu, 24 Jan 2019 09:58:34 -0800
Subject: [PATCH 1187/2345] Remove metrics `Reduction` enum from public API.
 Follow up to cl/230618998.

PiperOrigin-RevId: 230740410
---
 tensorflow/python/keras/utils/metrics_utils.py   |  2 --
 .../v1/tensorflow.keras.metrics.-reduction.pbtxt | 16 ----------------
 .../api/golden/v1/tensorflow.keras.metrics.pbtxt |  4 ----
 .../v2/tensorflow.keras.metrics.-reduction.pbtxt | 16 ----------------
 .../api/golden/v2/tensorflow.keras.metrics.pbtxt |  4 ----
 5 files changed, 42 deletions(-)
 delete mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-reduction.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-reduction.pbtxt

diff --git a/tensorflow/python/keras/utils/metrics_utils.py b/tensorflow/python/keras/utils/metrics_utils.py
index fe6373d338..c5ff617ad7 100644
--- a/tensorflow/python/keras/utils/metrics_utils.py
+++ b/tensorflow/python/keras/utils/metrics_utils.py
@@ -37,12 +37,10 @@ from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import weights_broadcast_ops
 from tensorflow.python.util import tf_decorator
-from tensorflow.python.util.tf_export import keras_export
 
 NEG_INF = -1e10
 
 
-@keras_export('keras.metrics.Reduction')
 class Reduction(Enum):
   """Types of metrics reduction.
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-reduction.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-reduction.pbtxt
deleted file mode 100644
index 8f74f1a1bc..0000000000
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-reduction.pbtxt
+++ /dev/null
@@ -1,16 +0,0 @@
-path: "tensorflow.keras.metrics.Reduction"
-tf_class {
-  is_instance: "<enum \'Reduction\'>"
-  member {
-    name: "SUM"
-    mtype: "<enum \'Reduction\'>"
-  }
-  member {
-    name: "SUM_OVER_BATCH_SIZE"
-    mtype: "<enum \'Reduction\'>"
-  }
-  member {
-    name: "WEIGHTED_MEAN"
-    mtype: "<enum \'Reduction\'>"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
index e95e0bfd29..d65e591b54 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
@@ -60,10 +60,6 @@ tf_module {
     name: "Recall"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "Reduction"
-    mtype: "<class \'enum.EnumMeta\'>"
-  }
   member {
     name: "SensitivityAtSpecificity"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-reduction.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-reduction.pbtxt
deleted file mode 100644
index 8f74f1a1bc..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-reduction.pbtxt
+++ /dev/null
@@ -1,16 +0,0 @@
-path: "tensorflow.keras.metrics.Reduction"
-tf_class {
-  is_instance: "<enum \'Reduction\'>"
-  member {
-    name: "SUM"
-    mtype: "<enum \'Reduction\'>"
-  }
-  member {
-    name: "SUM_OVER_BATCH_SIZE"
-    mtype: "<enum \'Reduction\'>"
-  }
-  member {
-    name: "WEIGHTED_MEAN"
-    mtype: "<enum \'Reduction\'>"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
index e95e0bfd29..d65e591b54 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
@@ -60,10 +60,6 @@ tf_module {
     name: "Recall"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "Reduction"
-    mtype: "<class \'enum.EnumMeta\'>"
-  }
   member {
     name: "SensitivityAtSpecificity"
     mtype: "<type \'type\'>"
-- 
GitLab


From d13819e80a5773bc74660c18ec729deddb784824 Mon Sep 17 00:00:00 2001
From: Stephan Lee <stephanlee@google.com>
Date: Thu, 24 Jan 2019 10:07:09 -0800
Subject: [PATCH 1188/2345] Write out graph for Keras in TF 2.0

PiperOrigin-RevId: 230742285
---
 tensorflow/python/keras/callbacks.py      | 9 ++++++---
 tensorflow/python/keras/callbacks_test.py | 2 +-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/keras/callbacks.py b/tensorflow/python/keras/callbacks.py
index e8985c2306..3223c89812 100644
--- a/tensorflow/python/keras/callbacks.py
+++ b/tensorflow/python/keras/callbacks.py
@@ -1165,12 +1165,15 @@ class TensorBoard(Callback):
     self._samples_seen = 0
     self._samples_seen_at_last_write = 0
 
-  def _init_writer(self):
+  def _init_writer(self, model):
     """Sets file writer."""
     if context.executing_eagerly():
       self.writer = summary_ops_v2.create_file_writer(self.log_dir)
+      if not model.run_eagerly and self.write_graph:
+        with self.writer.as_default():
+          summary_ops_v2.graph(K.get_graph())
     elif self.write_graph:
-      self.writer = tf_summary.FileWriter(self.log_dir, K.get_session().graph)
+      self.writer = tf_summary.FileWriter(self.log_dir, K.get_graph())
     else:
       self.writer = tf_summary.FileWriter(self.log_dir)
 
@@ -1233,7 +1236,7 @@ class TensorBoard(Callback):
     """Sets Keras model and creates summary ops."""
 
     self.model = model
-    self._init_writer()
+    self._init_writer(model)
     # histogram summaries only enabled in graph mode
     if not context.executing_eagerly():
       self._make_histogram_ops(model)
diff --git a/tensorflow/python/keras/callbacks_test.py b/tensorflow/python/keras/callbacks_test.py
index 7af9a9a2b2..d80af49651 100644
--- a/tensorflow/python/keras/callbacks_test.py
+++ b/tensorflow/python/keras/callbacks_test.py
@@ -1042,7 +1042,7 @@ class KerasCallbacksTest(test.TestCase):
       def close(self):
         pass
 
-    def _init_writer(obj):
+    def _init_writer(obj, _):
       obj.writer = FileWriterStub(obj.log_dir)
 
     np.random.seed(1337)
-- 
GitLab


From 277d45702c71f65caad5e1df8cc49c1c9358dc4e Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Thu, 24 Jan 2019 10:10:04 -0800
Subject: [PATCH 1189/2345] [tf.data] Change num_parallel_calls for
 map_vectorization. Add tests that it is set correctly.

PiperOrigin-RevId: 230742816
---
 .../core/grappler/optimizers/data/BUILD       |   3 +
 .../optimizers/data/map_vectorization.cc      | 221 ++++++++++--
 .../optimizers/data/map_vectorization_test.cc | 330 +++++++++++-------
 3 files changed, 395 insertions(+), 159 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/data/BUILD b/tensorflow/core/grappler/optimizers/data/BUILD
index 93e8405d18..377ff64de9 100644
--- a/tensorflow/core/grappler/optimizers/data/BUILD
+++ b/tensorflow/core/grappler/optimizers/data/BUILD
@@ -500,6 +500,8 @@ tf_cc_test(
     deps = [
         ":graph_utils",
         ":map_vectorization",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:direct_session",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:test",
@@ -507,6 +509,7 @@ tf_cc_test(
         "//tensorflow/core:testlib",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler/utils:topological_sort",
+        "//tensorflow/core/kernels:math",
         "//tensorflow/core/kernels/data",
     ],
 )
diff --git a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
index bc935bbd0a..3da238fb7d 100644
--- a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
@@ -38,11 +38,19 @@ namespace tensorflow {
 namespace grappler {
 namespace {
 
+constexpr char kCastOp[] = "Cast";
+constexpr char kRealDivOp[] = "RealDiv";
+constexpr char kSubOp[] = "Sub";
+constexpr char kMulOp[] = "Mul";
+constexpr char kAddOp[] = "Add";
+constexpr char kEqualOp[] = "Equal";
+constexpr char kCeilOp[] = "Ceil";
 constexpr char kBatchOp[] = "BatchDataset";
 constexpr char kBatchV2Op[] = "BatchDatasetV2";
 constexpr char kExperimentalMapAndBatchOp[] = "ExperimentalMapAndBatchDataset";
 constexpr char kMapOp[] = "MapDataset";
 constexpr char kParallelMapOp[] = "ParallelMapDataset";
+constexpr int kAutotune = -1;
 
 // Returns a FunctionDef containing a MapDefun op that wraps the original
 // function.
@@ -163,6 +171,21 @@ Status CopyInputs(StringPiece input_name, const NameRangeMap& input_map,
   return Status::OK();
 }
 
+Status GetInputNodeName(StringPiece input_name, const NameRangeMap& input_map,
+                        const NodeDef& node, string* result) {
+  const auto* range = gtl::FindOrNull(input_map, input_name);
+  if (range == nullptr) {
+    return errors::Internal(
+        "Failed to get input node name: did not find input with name: ",
+        input_name, ", in node with name: ", node.name());
+  }
+  if (range->second - range->first > 1) {
+    return errors::Internal("Tried to get single input name for a list input.");
+  }
+  *result = node.input(range->first);
+  return Status::OK();
+}
+
 Status AddNewBatchNode(const NodeDef& old_batch_node, const NodeDef& input_node,
                        const FunctionDef& vectorized_func,
                        MutableGraphView* graph, NodeDef** new_batch_node) {
@@ -213,6 +236,175 @@ Status AddNewBatchNode(const NodeDef& old_batch_node, const NodeDef& input_node,
   return Status::OK();
 }
 
+NodeDef* AddCastNode(const string& input, DataType src_t, DataType dst_t,
+                     MutableGraphView* graph) {
+  NodeDef cast_node;
+  cast_node.set_op(kCastOp);
+  cast_node.add_input(input);
+  graph_utils::SetUniqueGraphNodeName(cast_node.op(), graph->graph(),
+                                      &cast_node);
+  AddNodeAttr("SrcT", src_t, &cast_node);
+  AddNodeAttr("DstT", dst_t, &cast_node);
+
+  return graph->AddNode(std::move(cast_node));
+}
+
+NodeDef* AddEqualityNode(const string& input_x, const string& input_y,
+                         DataType t, MutableGraphView* graph) {
+  NodeDef equal_node;
+  equal_node.set_op(kEqualOp);
+  equal_node.add_input(input_x);
+  equal_node.add_input(input_y);
+  graph_utils::SetUniqueGraphNodeName(equal_node.op(), graph->graph(),
+                                      &equal_node);
+  AddNodeAttr("T", t, &equal_node);
+
+  return graph->AddNode(std::move(equal_node));
+}
+
+NodeDef* AddCeilNode(const string& input, MutableGraphView* graph) {
+  NodeDef ceil_node;
+  ceil_node.set_op(kCeilOp);
+  graph_utils::SetUniqueGraphNodeName(ceil_node.op(), graph->graph(),
+                                      &ceil_node);
+  AddNodeAttr("T", DT_FLOAT, &ceil_node);
+  ceil_node.add_input(input);
+
+  return graph->AddNode(std::move(ceil_node));
+}
+
+NodeDef* AddBinaryNode(const string& input_x, const string& input_y,
+                       const string& op, DataType type,
+                       MutableGraphView* graph) {
+  NodeDef node;
+  node.set_op(op);
+  node.add_input(input_x);
+  node.add_input(input_y);
+  graph_utils::SetUniqueGraphNodeName(op, graph->graph(), &node);
+  AddNodeAttr("T", type, &node);
+
+  return graph->AddNode(std::move(node));
+}
+
+NodeDef* AddIntAddNode(const string& input_x, const string& input_y,
+                       MutableGraphView* graph) {
+  return AddBinaryNode(input_x, input_y, kAddOp, DT_INT32, graph);
+}
+
+NodeDef* AddFloatDivNode(const string& input_x, const string& input_y,
+                         MutableGraphView* graph) {
+  return AddBinaryNode(input_x, input_y, kRealDivOp, DT_FLOAT, graph);
+}
+
+NodeDef* AddIntSubNode(const string& input_x, const string& input_y,
+                       MutableGraphView* graph) {
+  return AddBinaryNode(input_x, input_y, kSubOp, DT_INT32, graph);
+}
+
+NodeDef* AddIntMulNode(const string& input_x, const string& input_y,
+                       MutableGraphView* graph) {
+  return AddBinaryNode(input_x, input_y, kMulOp, DT_INT32, graph);
+}
+
+// Create a new node for the num_parallel_calls input argument according to the
+// following formula:
+//
+// Let N = old num_parallel_calls, N' = new num_parallel_calls, and B =
+// batch_size.
+//     N' = ceil(N // B) * (1 - (N == -1)) + N * (N == -1)
+//
+// i.e. N' = -1 if N = -1 (autotune)
+//      N' = ceil(N // B) otherwise.
+// Note that "ceil" is necessary so N' != 0.
+//
+// For non-autotune values of `num_parallel_call`, we divide it by `batch_size`
+// to limit memory consumption by the map buffer.
+//
+// TODO(rachelim): Evaluate the performance of other potential transformations
+// to `num_parallel_calls`:
+//   1) use the autotune value (i.e. -1)
+//   2) use the original value
+Status MakeNumParallelCallsInput(const NodeDef& old_map_node,
+                                 const NodeDef& old_batch_node,
+                                 const NameRangeMap& input_map,
+                                 MutableGraphView* graph, string* result) {
+  string num_parallel_calls_name;
+  TF_RETURN_IF_ERROR(GetInputNodeName("num_parallel_calls", input_map,
+                                      old_map_node, &num_parallel_calls_name));
+
+  NodeDef* float_num_parallel_calls;
+  NodeDef* float_batch_size;
+  NodeDef* bool_is_autotune;
+
+  // Cast the old num_parallel_calls and batch_size arguments to DT_FLOAT before
+  // dividing.
+  if (old_map_node.op() == kExperimentalMapAndBatchOp) {
+    auto autotune_val =
+        graph_utils::AddScalarConstNode(static_cast<int64>(kAutotune), graph);
+    bool_is_autotune = AddEqualityNode(
+        autotune_val->name(), num_parallel_calls_name, DT_INT64, graph);
+
+    float_num_parallel_calls =
+        AddCastNode(num_parallel_calls_name, DT_INT64, DT_FLOAT, graph);
+
+    string batch_size_name;
+    TF_RETURN_IF_ERROR(GetInputNodeName("batch_size", input_map, old_map_node,
+                                        &batch_size_name));
+
+    float_batch_size = AddCastNode(batch_size_name, DT_INT64, DT_FLOAT, graph);
+  } else {
+    auto autotune_val =
+        graph_utils::AddScalarConstNode(static_cast<int>(kAutotune), graph);
+    bool_is_autotune = AddEqualityNode(
+        autotune_val->name(), num_parallel_calls_name, DT_INT32, graph);
+
+    float_num_parallel_calls =
+        AddCastNode(num_parallel_calls_name, DT_INT32, DT_FLOAT, graph);
+
+    float_batch_size =
+        AddCastNode(old_batch_node.input(1), DT_INT64, DT_FLOAT, graph);
+  }
+
+  // Divide
+  auto div_node = AddFloatDivNode(float_num_parallel_calls->name(),
+                                  float_batch_size->name(), graph);
+
+  // Ceil
+  auto float_ceil_node = AddCeilNode(div_node->name(), graph);
+
+  // Cast back to DT_INT32
+  auto int_ceil_node =
+      AddCastNode(float_ceil_node->name(), DT_FLOAT, DT_INT32, graph);
+
+  // is_autotune = int(num_parallel_calls == -1)
+  auto int_is_autotune =
+      AddCastNode(bool_is_autotune->name(), DT_BOOL, DT_INT32, graph);
+
+  // is_not_autotune = 1 - is_autotune
+  auto int_is_not_autotune =
+      AddIntSubNode(graph_utils::AddScalarConstNode(1, graph)->name(),
+                    int_is_autotune->name(), graph);
+
+  auto mul_1 =
+      AddIntMulNode(int_ceil_node->name(), int_is_not_autotune->name(), graph);
+
+  NodeDef* mul_2;
+  if (old_map_node.op() == kExperimentalMapAndBatchOp) {
+    auto int_num_parallel_calls =
+        AddCastNode(num_parallel_calls_name, DT_INT64, DT_INT32, graph);
+    mul_2 = AddIntMulNode(int_num_parallel_calls->name(),
+                          int_is_autotune->name(), graph);
+  } else {
+    mul_2 =
+        AddIntMulNode(num_parallel_calls_name, int_is_autotune->name(), graph);
+  }
+
+  auto add_node = AddIntAddNode(mul_1->name(), mul_2->name(), graph);
+
+  *result = add_node->name();
+  return Status::OK();
+}
+
 Status AddNewMapNode(const NodeDef& old_map_node, const NodeDef& old_batch_node,
                      const NodeDef& new_batch_node,
                      const FunctionDef& vectorized_func,
@@ -232,30 +424,11 @@ Status AddNewMapNode(const NodeDef& old_map_node, const NodeDef& old_batch_node,
       CopyInputs("other_arguments", input_map, old_map_node, &map_node));
 
   // Set the `num_parallel_calls` input argument
-  // TODO(rachelim): Evaluate the performance of potential transformations to
-  // the new `num_parallel_calls`:
-  //   1) dividing by the `batch_size`, since the new map will be operating
-  //      over larger elements
-  //   2) use the autotune value (i.e. -1)
-  //   3) use the original value
-  if (old_map_node.op() == kExperimentalMapAndBatchOp) {
-    // This `Cast` op is necessary because the `num_parallel_calls` input for
-    // ExperimentalMapAndBatch has type DT_INT64, but the input for ParallelMap
-    // expects type DT_INT32
-    NodeDef cast_node;
-    cast_node.set_op("Cast");
-    graph_utils::SetUniqueGraphNodeName(cast_node.op(), graph->graph(),
-                                        &cast_node);
-    AddNodeAttr("SrcT", DT_INT64, &cast_node);
-    AddNodeAttr("DstT", DT_INT32, &cast_node);
-    TF_RETURN_IF_ERROR(
-        CopyInputs("num_parallel_calls", input_map, old_map_node, &cast_node));
-    auto added_cast_node = graph->AddNode(std::move(cast_node));
-    map_node.add_input(added_cast_node->name());
-
-  } else if (old_map_node.op() == kParallelMapOp) {
-    TF_RETURN_IF_ERROR(
-        CopyInputs("num_parallel_calls", input_map, old_map_node, &map_node));
+  if (old_map_node.op() != kMapOp) {
+    string num_parallel_calls;
+    TF_RETURN_IF_ERROR(MakeNumParallelCallsInput(
+        old_map_node, old_batch_node, input_map, graph, &num_parallel_calls));
+    map_node.add_input(std::move(num_parallel_calls));
   }
 
   // Set attrs
diff --git a/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc b/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc
index 8a6df49758..0565d5f643 100644
--- a/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc
@@ -17,6 +17,7 @@ limitations under the License.
 
 #include "tensorflow/core/framework/attr_value_util.h"
 #include "tensorflow/core/framework/function_testlib.h"
+#include "tensorflow/core/framework/node_def_util.h"
 #include "tensorflow/core/framework/tensor_testutil.h"
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/optimizers/data/graph_utils.h"
@@ -24,6 +25,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/gtl/array_slice.h"
 #include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/public/session.h"
 
 namespace tensorflow {
 namespace grappler {
@@ -48,34 +50,10 @@ constexpr char kAttrNameDtype[] = "dtype";
 
 using test::function::NDef;
 
-NodeDef* AddConstBoolNode(GraphDef* graph, StringPiece name, bool value) {
-  NodeDef* node = graph->add_node();
-  *node = NDef(/*name=*/name, /*op=*/kConstOp, /*inputs=*/{},
-               /*attrs=*/{{kAttrNameValue, value}, {kAttrNameDtype, DT_BOOL}});
-
-  return node;
-}
-
-NodeDef* AddConstInt32Node(GraphDef* graph, StringPiece name, int value) {
-  NodeDef* node = graph->add_node();
-  *node = NDef(/*name=*/name, /*op=*/kConstOp, /*inputs=*/{},
-               /*attrs=*/{{kAttrNameValue, value}, {kAttrNameDtype, DT_INT32}});
-
-  return node;
-}
-
-NodeDef* AddConstInt64Node(GraphDef* graph, StringPiece name, int64 value) {
-  NodeDef* node = graph->add_node();
-  *node = NDef(/*name=*/name, /*op=*/kConstOp, /*inputs=*/{},
-               /*attrs=*/{{kAttrNameValue, value}, {kAttrNameDtype, DT_INT64}});
-
-  return node;
-}
-
 // Adds a simple vectorizable map function that is akin to
 // dataset.map(lambda x: tf.identity(x))
-FunctionDef* AddMapFn(GraphDef* graph) {
-  FunctionDef* map_fn = graph->mutable_library()->add_function();
+FunctionDef* AddMapFn(MutableGraphView* graph) {
+  FunctionDef* map_fn = graph->graph()->mutable_library()->add_function();
   *map_fn = FunctionDefHelper::Create(
       /*function_name=*/"map_fn",
       /*in_def=*/{"x: int64"},
@@ -87,28 +65,26 @@ FunctionDef* AddMapFn(GraphDef* graph) {
   return map_fn;
 }
 
-NodeDef* AddMapNode(GraphDef* graph, const string& input_dataset,
-                    const string& map_fn, bool parallel = false,
-                    int suffix = 0) {
-  NodeDef* map_node = graph->add_node();
-
-  if (parallel) {
-    auto num_parallel_calls = AddConstInt32Node(
-        graph, strings::StrCat("num_parallel_calls", suffix), 8);
-    *map_node = NDef(
-        /*name=*/strings::StrCat("map", suffix), /*op=*/kParallelMapOp,
-        /*inputs=*/{input_dataset, num_parallel_calls->name()},
-        /*attrs=*/
-        {{kAttrNameF, FunctionDefHelper::FunctionRef(map_fn)},
-         {kAttrNameTarguments, gtl::ArraySlice<DataType>({})},
-         {kAttrNameOutputTypes, gtl::ArraySlice<DataType>({DT_INT64})},
-         {kAttrNameOutputShapes, gtl::ArraySlice<TensorShape>({{}})},
-         {kAttrNameInterOpParallelism, false},
-         {kAttrNameSloppy, true},
-         {kAttrNamePreserveCardinality, true}});
+NodeDef* AddMapNode(MutableGraphView* graph, const string& input_dataset,
+                    const string& map_fn, int num_parallel_calls = 0) {
+  NodeDef result;
+  if (num_parallel_calls) {
+    auto num_parallel_calls_node =
+        graph_utils::AddScalarConstNode(num_parallel_calls, graph);
+    result =
+        NDef(/*name=*/"map", /*op=*/kParallelMapOp,
+             /*inputs=*/{input_dataset, num_parallel_calls_node->name()},
+             /*attrs=*/
+             {{kAttrNameF, FunctionDefHelper::FunctionRef(map_fn)},
+              {kAttrNameTarguments, gtl::ArraySlice<DataType>({})},
+              {kAttrNameOutputTypes, gtl::ArraySlice<DataType>({DT_INT64})},
+              {kAttrNameOutputShapes, gtl::ArraySlice<TensorShape>({{}})},
+              {kAttrNameInterOpParallelism, false},
+              {kAttrNameSloppy, true},
+              {kAttrNamePreserveCardinality, true}});
   } else {
-    *map_node =
-        NDef(/*name=*/strings::StrCat("map", suffix), /*op=*/kMapOp,
+    result =
+        NDef(/*name=*/"map", /*op=*/kMapOp,
              /*inputs=*/{input_dataset},
              /*attrs=*/
              {{kAttrNameF, FunctionDefHelper::FunctionRef(map_fn)},
@@ -119,50 +95,53 @@ NodeDef* AddMapNode(GraphDef* graph, const string& input_dataset,
               {kAttrNamePreserveCardinality, true}});
   }
 
-  return map_node;
+  graph_utils::SetUniqueGraphNodeName(result.name(), graph->graph(), &result);
+  return graph->AddNode(std::move(result));
 }
 
-NodeDef* AddBatchNode(GraphDef* graph, const string& input_dataset,
-                      bool v2 = false, int suffix = 0) {
-  NodeDef* batch_node = graph->add_node();
-
-  auto batch_size =
-      AddConstInt64Node(graph, strings::StrCat("batch_size", suffix), 10);
+NodeDef* AddBatchNode(MutableGraphView* graph, const string& input_dataset,
+                      bool v2 = false, int64 batch_size = 10) {
+  NodeDef result;
+  auto batch_size_node = graph_utils::AddScalarConstNode(batch_size, graph);
 
   if (v2) {
-    auto drop_remainder = AddConstBoolNode(
-        graph, strings::StrCat("drop_remainder", suffix), true);
-    *batch_node = NDef(
-        /*name=*/strings::StrCat("batch", suffix), /*op=*/kBatchV2Op,
-        /*inputs=*/{input_dataset, batch_size->name(), drop_remainder->name()},
+    // BatchDatasetV2
+    auto drop_remainder = graph_utils::AddScalarConstNode(true, graph);
+    result = NDef(
+        /*name=*/"batch", /*op=*/kBatchV2Op,
+        /*inputs=*/
+        {input_dataset, batch_size_node->name(), drop_remainder->name()},
         /*attrs=*/
         {{kAttrNameOutputTypes, gtl::ArraySlice<DataType>({DT_INT64})},
          {kAttrNameOutputShapes, gtl::ArraySlice<TensorShape>({{10, 1}})}});
   } else {
-    *batch_node = NDef(
-        /*name=*/strings::StrCat("batch", suffix), /*op=*/kBatchOp,
-        /*inputs=*/{input_dataset, batch_size->name()},
-        /*attrs=*/
-        {{kAttrNameOutputTypes, gtl::ArraySlice<DataType>({DT_INT64})},
-         {kAttrNameOutputShapes,
-          gtl::ArraySlice<PartialTensorShape>({{v2 ? 10 : -1, 1}})}});
+    result =
+        NDef(/*name=*/"batch", /*op=*/kBatchOp,
+             /*inputs=*/{input_dataset, batch_size_node->name()},
+             /*attrs=*/
+             {{kAttrNameOutputTypes, gtl::ArraySlice<DataType>({DT_INT64})},
+              {kAttrNameOutputShapes,
+               gtl::ArraySlice<PartialTensorShape>({{v2 ? 10 : -1, 1}})}});
   }
-  return batch_node;
+
+  graph_utils::SetUniqueGraphNodeName(result.name(), graph->graph(), &result);
+  return graph->AddNode(std::move(result));
 }
 
-NodeDef* AddRangeNode(GraphDef* graph) {
-  auto start = AddConstInt64Node(graph, "start", 0);
-  auto stop = AddConstInt64Node(graph, "stop", 10);
-  auto step = AddConstInt64Node(graph, "step", 1);
-  NodeDef* range_node = graph->add_node();
-  *range_node =
+NodeDef* AddRangeNode(MutableGraphView* graph) {
+  auto start = graph_utils::AddScalarConstNode(static_cast<int64>(0), graph);
+  auto stop = graph_utils::AddScalarConstNode(static_cast<int64>(10), graph);
+  auto step = graph_utils::AddScalarConstNode(static_cast<int64>(1), graph);
+
+  NodeDef result =
       NDef(/*name=*/"range", /*op=*/kRangeOp,
            /*inputs=*/{start->name(), stop->name(), step->name()},
            /*attrs=*/
            {{kAttrNameOutputShapes, gtl::ArraySlice<TensorShape>({{}})},
             {kAttrNameOutputTypes, gtl::ArraySlice<DataType>({DT_INT64})}});
 
-  return range_node;
+  graph_utils::SetUniqueGraphNodeName(result.name(), graph->graph(), &result);
+  return graph->AddNode(std::move(result));
 }
 
 void CheckNotVectorized(const GraphDef& output, const string& map_op,
@@ -199,17 +178,18 @@ void CheckVectorized(const GraphDef& output, const string& map_op,
 }
 
 class MapThenBatchTest
-    : public ::testing::TestWithParam<std::tuple<bool, bool>> {};
+    : public ::testing::TestWithParam<std::tuple<int, bool>> {};
 
 TEST_P(MapThenBatchTest, IsVectorized) {
-  bool use_parallel_map = std::get<0>(GetParam());
+  int num_parallel_calls = std::get<0>(GetParam());
   bool use_batch_v2 = std::get<1>(GetParam());
   GrapplerItem item;
-  auto range_node = AddRangeNode(&item.graph);
-  auto map_fn = AddMapFn(&item.graph);
-  auto map_node = AddMapNode(&item.graph, range_node->name(),
-                             map_fn->signature().name(), use_parallel_map);
-  auto batch_node = AddBatchNode(&item.graph, map_node->name(), use_batch_v2);
+  MutableGraphView graph(&item.graph);
+  auto range_node = AddRangeNode(&graph);
+  auto map_fn = AddMapFn(&graph);
+  auto map_node = AddMapNode(&graph, range_node->name(),
+                             map_fn->signature().name(), num_parallel_calls);
+  auto batch_node = AddBatchNode(&graph, map_node->name(), use_batch_v2);
   MapVectorization optimizer;
   GraphDef output;
   TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output));
@@ -217,25 +197,24 @@ TEST_P(MapThenBatchTest, IsVectorized) {
 }
 
 INSTANTIATE_TEST_SUITE_P(MapThenBatchTest, MapThenBatchTest,
-                         ::testing::Combine(::testing::Bool(),
+                         ::testing::Combine(::testing::Values(0, 12),
                                             ::testing::Bool()));
 
-NodeDef* AddMapAndBatchNode(GraphDef* graph, const string& input_dataset,
-                            const string& map_fn, int suffix = 0) {
-  auto batch_size =
-      AddConstInt64Node(graph, strings::StrCat("batch_size", suffix), 10);
-  auto num_parallel_calls = AddConstInt64Node(
-      graph, strings::StrCat("num_parallel_calls", suffix), 8);
-  auto drop_remainder =
-      AddConstBoolNode(graph, strings::StrCat("drop_remainder", suffix), true);
-
-  auto map_and_batch_node = graph->add_node();
-  *map_and_batch_node =
-      NDef(/*name=*/strings::StrCat("map_and_batch", suffix),
+NodeDef* AddMapAndBatchNode(MutableGraphView* graph,
+                            const string& input_dataset, const string& map_fn,
+                            int64 batch_size = 10,
+                            int64 num_parallel_calls = 12) {
+  auto batch_size_node = graph_utils::AddScalarConstNode(batch_size, graph);
+  auto num_parallel_calls_node =
+      graph_utils::AddScalarConstNode(num_parallel_calls, graph);
+  auto drop_remainder = graph_utils::AddScalarConstNode(true, graph);
+
+  NodeDef result =
+      NDef(/*name=*/"map_and_batch",
            /*op=*/kExperimentalMapAndBatchOp,
            /*inputs=*/
-           {input_dataset, batch_size->name(), num_parallel_calls->name(),
-            drop_remainder->name()},
+           {input_dataset, batch_size_node->name(),
+            num_parallel_calls_node->name(), drop_remainder->name()},
            /*attrs=*/
            {{kAttrNameF, FunctionDefHelper::FunctionRef(map_fn)},
             {kAttrNameTarguments, gtl::ArraySlice<DataType>({})},
@@ -243,14 +222,16 @@ NodeDef* AddMapAndBatchNode(GraphDef* graph, const string& input_dataset,
             {kAttrNameOutputShapes,
              gtl::ArraySlice<PartialTensorShape>({{10, 1}})}});
 
-  return map_and_batch_node;
+  graph_utils::SetUniqueGraphNodeName(result.name(), graph->graph(), &result);
+  return graph->AddNode(std::move(result));
 }
 
 TEST(MapVectorizationTest, VectorizeExperimentalMapAndBatch) {
   GrapplerItem item;
-  auto range_node = AddRangeNode(&item.graph);
-  auto map_fn = AddMapFn(&item.graph);
-  auto map_and_batch_node = AddMapAndBatchNode(&item.graph, range_node->name(),
+  MutableGraphView graph(&item.graph);
+  auto range_node = AddRangeNode(&graph);
+  auto map_fn = AddMapFn(&graph);
+  auto map_and_batch_node = AddMapAndBatchNode(&graph, range_node->name(),
                                                map_fn->signature().name());
   ASSERT_NE(map_and_batch_node, nullptr);
 
@@ -260,6 +241,82 @@ TEST(MapVectorizationTest, VectorizeExperimentalMapAndBatch) {
   CheckVectorized(output, kParallelMapOp, kBatchV2Op, "range");
 }
 
+void EvaluateNodes(const GraphDef& graph,
+                   const std::vector<string>& output_tensor_names,
+                   std::vector<Tensor>* output_tensors) {
+  std::unique_ptr<Session> session(NewSession(SessionOptions()));
+  TF_CHECK_OK(session->Create(graph));
+  TF_CHECK_OK(session->Run({}, output_tensor_names, {}, output_tensors));
+}
+
+void CheckNumParallelCalls(const GraphDef& output,
+                           int expected_num_parallel_calls) {
+  // Run the graph to see that the new num_parallel_calls is computed correctly.
+  const NodeDef& map_node =
+      output.node(graph_utils::FindGraphNodeWithOp(kParallelMapOp, output));
+  const string& num_parallel_calls = map_node.input(1);
+  std::vector<Tensor> output_tensors;
+  EvaluateNodes(output, {num_parallel_calls}, &output_tensors);
+
+  test::ExpectTensorEqual<int>(
+      output_tensors.at(0),
+      Tensor(static_cast<int32>(expected_num_parallel_calls)));
+}
+
+struct TestStruct {
+  int original_num_parallel_calls;
+  int batch_size;
+  int expected_num_parallel_calls;
+};
+
+class NumParallelCallsTest : public ::testing::TestWithParam<TestStruct> {};
+
+TEST_P(NumParallelCallsTest, TestCorrectNumParallelCalls) {
+  auto params = GetParam();
+
+  GrapplerItem item;
+  MutableGraphView graph(&item.graph);
+  auto range_node = AddRangeNode(&graph);
+  auto map_fn = AddMapFn(&graph);
+  auto map_node =
+      AddMapNode(&graph, range_node->name(), map_fn->signature().name(),
+                 params.original_num_parallel_calls);
+  auto batch_node = AddBatchNode(&graph, map_node->name(), /*v2=*/true,
+                                 /*batch_size=*/params.batch_size);
+  MapVectorization optimizer;
+  GraphDef output;
+  TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output));
+  CheckVectorized(output, map_node->op(), batch_node->op(), range_node->name());
+
+  CheckNumParallelCalls(output, params.expected_num_parallel_calls);
+}
+
+TEST_P(NumParallelCallsTest, TestCorrectNumParallelCallsFused) {
+  auto params = GetParam();
+
+  GrapplerItem item;
+  MutableGraphView graph(&item.graph);
+  auto range_node = AddRangeNode(&graph);
+  auto map_fn = AddMapFn(&graph);
+  auto map_and_batch_node =
+      AddMapAndBatchNode(&graph, range_node->name(), map_fn->signature().name(),
+                         params.batch_size, params.original_num_parallel_calls);
+  ASSERT_NE(map_and_batch_node, nullptr);
+
+  MapVectorization optimizer;
+  GraphDef output;
+  TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output));
+  CheckVectorized(output, kParallelMapOp, kBatchV2Op, range_node->name());
+
+  CheckNumParallelCalls(output, params.expected_num_parallel_calls);
+}
+
+INSTANTIATE_TEST_SUITE_P(
+    NumParallelCalls, NumParallelCallsTest,
+    ::testing::Values(TestStruct({1, 1, 1}), TestStruct({2, 10, 1}),
+                      TestStruct({4, 3, 2}), TestStruct({10, 1, 10}),
+                      TestStruct({-1, 1, -1}), TestStruct({-1, 10, -1})));
+
 class ChainedMapAndBatchTest
     : public ::testing::TestWithParam<std::tuple<bool, bool>> {};
 
@@ -270,26 +327,26 @@ class ChainedMapAndBatchTest
 // 4) map_and_batch.map_and_batch
 TEST_P(ChainedMapAndBatchTest, IsVectorized) {
   GrapplerItem item;
-  auto input_node = AddRangeNode(&item.graph);
+  MutableGraphView graph(&item.graph);
+  auto input_node = AddRangeNode(&graph);
 
-  auto map_fn = AddMapFn(&item.graph);
+  auto map_fn = AddMapFn(&graph);
 
-  auto make_map_and_batch = [&item, map_fn](NodeDef* input, bool fuse,
-                                            int suffix) {
+  auto make_map_and_batch = [&graph, map_fn](NodeDef* input, bool fuse) {
     if (fuse) {
-      return AddMapAndBatchNode(&item.graph, input->name(),
-                                map_fn->signature().name(), suffix);
+      return AddMapAndBatchNode(&graph, input->name(),
+                                map_fn->signature().name());
     }
-    auto map_node = AddMapNode(&item.graph, input->name(),
-                               map_fn->signature().name(), true, suffix);
-    auto batch_node = AddBatchNode(&item.graph, map_node->name(), true, suffix);
+    auto map_node =
+        AddMapNode(&graph, input->name(), map_fn->signature().name(), true);
+    auto batch_node = AddBatchNode(&graph, map_node->name(), true);
     return batch_node;
   };
 
   auto map_and_batch_0 =
-      make_map_and_batch(input_node, std::get<0>(GetParam()), 0);
+      make_map_and_batch(input_node, std::get<0>(GetParam()));
   auto map_and_batch_1 =
-      make_map_and_batch(map_and_batch_0, std::get<1>(GetParam()), 1);
+      make_map_and_batch(map_and_batch_0, std::get<1>(GetParam()));
   ASSERT_NE(map_and_batch_1, nullptr);
 
   MapVectorization optimizer;
@@ -319,13 +376,12 @@ INSTANTIATE_TEST_SUITE_P(ChainedMapAndBatchTest, ChainedMapAndBatchTest,
                          ::testing::Combine(::testing::Bool(),
                                             ::testing::Bool()));
 
-// Not all dataset types have kAttrNameOutputShapes and kAttrNameOutputTypes
+// Not all dataset types have "output_shapes" and "output_types"
 // attrs defined. Add a generic input node which may not have these attrs
 // defined.
-NodeDef* AddArbitraryInputNode(GraphDef* graph,
+NodeDef* AddArbitraryInputNode(MutableGraphView* graph,
                                std::vector<PartialTensorShape>* output_shapes,
                                std::vector<DataType>* output_types) {
-  auto input_node = graph->add_node();
   std::vector<std::pair<string, FunctionDefHelper::AttrValueWrapper>> attrs;
   if (output_shapes) {
     attrs.push_back({kAttrNameOutputShapes, *output_shapes});
@@ -334,10 +390,12 @@ NodeDef* AddArbitraryInputNode(GraphDef* graph,
     attrs.push_back({kAttrNameOutputTypes, *output_types});
   }
 
-  *input_node = NDef(/*name=*/"input", /*op=*/"InputDataset", /*inputs=*/{},
-                     /*attrs=*/attrs);
+  NodeDef result = NDef(/*name=*/"input", /*op=*/"InputDataset",
+                        /*inputs=*/{},
+                        /*attrs=*/attrs);
 
-  return input_node;
+  graph_utils::SetUniqueGraphNodeName(result.name(), graph->graph(), &result);
+  return graph->AddNode(std::move(result));
 }
 
 TEST(MapVectorizationTest, VectorizeWithUndefinedOutputShapes) {
@@ -345,12 +403,13 @@ TEST(MapVectorizationTest, VectorizeWithUndefinedOutputShapes) {
   // doesn't have an output_shapes attr defined. In this case, the map and
   // batch swap does not occur.
   GrapplerItem item;
+  MutableGraphView graph(&item.graph);
   std::vector<DataType> input_types({DT_INT64});
-  auto input_node = AddArbitraryInputNode(&item.graph, nullptr, &input_types);
-  auto map_fn = AddMapFn(&item.graph);
+  auto input_node = AddArbitraryInputNode(&graph, nullptr, &input_types);
+  auto map_fn = AddMapFn(&graph);
   auto map_node =
-      AddMapNode(&item.graph, input_node->name(), map_fn->signature().name());
-  auto batch_node = AddBatchNode(&item.graph, map_node->name());
+      AddMapNode(&graph, input_node->name(), map_fn->signature().name());
+  auto batch_node = AddBatchNode(&graph, map_node->name());
   MapVectorization optimizer;
   GraphDef output;
   TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output));
@@ -363,14 +422,14 @@ TEST(MapVectorizationTest, VectorizeWithUnknownRank) {
   // has components with unknown rank. In this case, the optimization does not
   // occur.
   GrapplerItem item;
+  MutableGraphView graph(&item.graph);
   std::vector<PartialTensorShape> input_shapes({{}});
   std::vector<DataType> input_types({DT_INT64});
-  auto input_node =
-      AddArbitraryInputNode(&item.graph, &input_shapes, &input_types);
-  auto map_fn = AddMapFn(&item.graph);
+  auto input_node = AddArbitraryInputNode(&graph, &input_shapes, &input_types);
+  auto map_fn = AddMapFn(&graph);
   auto map_node =
-      AddMapNode(&item.graph, input_node->name(), map_fn->signature().name());
-  auto batch_node = AddBatchNode(&item.graph, map_node->name());
+      AddMapNode(&graph, input_node->name(), map_fn->signature().name());
+  auto batch_node = AddBatchNode(&graph, map_node->name());
   MapVectorization optimizer;
   GraphDef output;
   TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output));
@@ -383,14 +442,14 @@ TEST(MapVectorizationTest, VectorizeWithUnknownDim) {
   // has components with unknown dimensions. In this case, the optimization does
   // not occur.
   GrapplerItem item;
+  MutableGraphView graph(&item.graph);
   std::vector<PartialTensorShape> input_shapes({{-1, 2}});
   std::vector<DataType> input_types({DT_INT64});
-  auto input_node =
-      AddArbitraryInputNode(&item.graph, &input_shapes, &input_types);
-  auto map_fn = AddMapFn(&item.graph);
+  auto input_node = AddArbitraryInputNode(&graph, &input_shapes, &input_types);
+  auto map_fn = AddMapFn(&graph);
   auto map_node =
-      AddMapNode(&item.graph, input_node->name(), map_fn->signature().name());
-  auto batch_node = AddBatchNode(&item.graph, map_node->name());
+      AddMapNode(&graph, input_node->name(), map_fn->signature().name());
+  auto batch_node = AddBatchNode(&graph, map_node->name());
   MapVectorization optimizer;
   GraphDef output;
   TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output));
@@ -403,12 +462,13 @@ TEST(MapVectorizationTest, VectorizeWithUndefinedOutputTypes) {
   // an output_types attr defined. The output_types of the input node, even
   // if not present, can be inferred from the map function input signature.
   GrapplerItem item;
+  MutableGraphView graph(&item.graph);
   std::vector<PartialTensorShape> input_shapes({{1}});
-  auto input_node = AddArbitraryInputNode(&item.graph, &input_shapes, nullptr);
-  auto map_fn = AddMapFn(&item.graph);
+  auto input_node = AddArbitraryInputNode(&graph, &input_shapes, nullptr);
+  auto map_fn = AddMapFn(&graph);
   auto map_node =
-      AddMapNode(&item.graph, input_node->name(), map_fn->signature().name());
-  auto batch_node = AddBatchNode(&item.graph, map_node->name());
+      AddMapNode(&graph, input_node->name(), map_fn->signature().name());
+  auto batch_node = AddBatchNode(&graph, map_node->name());
   MapVectorization optimizer;
   GraphDef output;
   TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output));
-- 
GitLab


From 26f49ec8351b6da8ce5383b85c64e387cce17750 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 24 Jan 2019 10:27:44 -0800
Subject: [PATCH 1190/2345] Add variant wrapper for absl::string_view

PiperOrigin-RevId: 230745949
---
 .../core/framework/variant_tensor_data.cc     |  6 ++
 .../core/framework/variant_tensor_data.h      |  5 ++
 tensorflow/core/kernels/BUILD                 | 10 +++
 .../kernels/string_view_variant_wrapper.cc    | 22 ++++++
 .../kernels/string_view_variant_wrapper.h     | 69 +++++++++++++++++++
 5 files changed, 112 insertions(+)
 create mode 100644 tensorflow/core/kernels/string_view_variant_wrapper.cc
 create mode 100644 tensorflow/core/kernels/string_view_variant_wrapper.h

diff --git a/tensorflow/core/framework/variant_tensor_data.cc b/tensorflow/core/framework/variant_tensor_data.cc
index 3e67e4a864..c169e867ec 100644
--- a/tensorflow/core/framework/variant_tensor_data.cc
+++ b/tensorflow/core/framework/variant_tensor_data.cc
@@ -43,6 +43,12 @@ Tensor* VariantTensorData::add_tensors() {
   return &(tensors_[tensors_.size() - 1]);
 }
 
+template <typename... TensorConstructorArgs>
+Tensor* VariantTensorData::add_tensor(TensorConstructorArgs&&... args) {
+  tensors_.emplace_back(std::forward<TensorConstructorArgs>(args)...);
+  return &tensors_.back();
+}
+
 void VariantTensorData::ToProto(VariantTensorDataProto* proto) const {
   proto->set_type_name(type_name());
   proto->set_metadata(metadata_);
diff --git a/tensorflow/core/framework/variant_tensor_data.h b/tensorflow/core/framework/variant_tensor_data.h
index 8c69c87034..ca99e83f9c 100644
--- a/tensorflow/core/framework/variant_tensor_data.h
+++ b/tensorflow/core/framework/variant_tensor_data.h
@@ -68,6 +68,11 @@ class VariantTensorData {
   const std::vector<Tensor>& tensors() const;
   Tensor* add_tensors();
 
+  // A more general version of add_tensors. Parameters are perfectly forwarded
+  // to the constructor of the tensor added here.
+  template <typename... TensorConstructorArgs>
+  Tensor* add_tensor(TensorConstructorArgs&&... args);
+
   // Conversion to and from VariantTensorDataProto
   void ToProto(VariantTensorDataProto* proto) const;
   // This allows optimizations via std::move.
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 9476bc5718..a59ea5b631 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -2174,6 +2174,16 @@ tf_kernel_library(
     deps = LOOKUP_DEPS,
 )
 
+cc_library(
+    name = "string_view_variant_wrapper",
+    hdrs = ["string_view_variant_wrapper.h"],
+    deps = [
+        "//tensorflow/core:framework",
+        "//tensorflow/core:protos_all_cc",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
 cc_library(
     name = "checkpoint_ops",
     deps = [
diff --git a/tensorflow/core/kernels/string_view_variant_wrapper.cc b/tensorflow/core/kernels/string_view_variant_wrapper.cc
new file mode 100644
index 0000000000..b576eb4a3e
--- /dev/null
+++ b/tensorflow/core/kernels/string_view_variant_wrapper.cc
@@ -0,0 +1,22 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/kernels/string_view_variant_wrapper.h"
+
+namespace tensorflow {
+
+constexpr const char StringViewVariantWrapper::kTypeName[];
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/string_view_variant_wrapper.h b/tensorflow/core/kernels/string_view_variant_wrapper.h
new file mode 100644
index 0000000000..dc4a8e9534
--- /dev/null
+++ b/tensorflow/core/kernels/string_view_variant_wrapper.h
@@ -0,0 +1,69 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_KERNELS_STRING_VIEW_VARIANT_WRAPPER_H_
+#define TENSORFLOW_CORE_KERNELS_STRING_VIEW_VARIANT_WRAPPER_H_
+
+#include <string>
+
+#include "absl/strings/string_view.h"
+#include "tensorflow/core/framework/variant_tensor_data.h"
+
+namespace tensorflow {
+
+// A wrapper class for storing an `absl::string_view` instance in a DT_VARIANT
+// tensor.
+class StringViewVariantWrapper {
+ public:
+  static constexpr const char kTypeName[] =
+      "tensorflow::StringViewVariantWrapper";
+
+  using value_type = absl::string_view;
+
+  StringViewVariantWrapper() = default;
+
+  explicit StringViewVariantWrapper(absl::string_view str_view)
+      : str_view_(str_view) {}
+
+  StringViewVariantWrapper(const StringViewVariantWrapper& other)
+      : str_view_(other.str_view_) {}
+
+  const absl::string_view* get() const { return &str_view_; }
+
+  static string TypeName() { return kTypeName; }
+
+  string DebugString() const { return string(str_view_); }
+
+  void Encode(VariantTensorData* data) const {
+    data->add_tensor(string(str_view_));
+  }
+
+  // Decode assumes that the source VariantTensorData will have a longer
+  // lifetime than this StringViewVariantWrapper.
+  bool Decode(const VariantTensorData& data) {
+    if (data.tensors_size() != 1 || data.tensors(0).dtype() != DT_STRING) {
+      return false;
+    }
+    str_view_ = data.tensors(0).scalar<string>()();
+    return true;
+  }
+
+ private:
+  absl::string_view str_view_;
+};
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_KERNELS_STRING_VIEW_VARIANT_WRAPPER_H_
-- 
GitLab


From b04ef3615ebd6cd641ce80cbe0e438c9767c83b0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 24 Jan 2019 10:32:55 -0800
Subject: [PATCH 1191/2345] Constrain non-GPU actions away from GPU workers in
 the remote execution configuration.

PiperOrigin-RevId: 230746854
---
 third_party/toolchains/BUILD | 28 ++++++++++++++++++++++++----
 1 file changed, 24 insertions(+), 4 deletions(-)

diff --git a/third_party/toolchains/BUILD b/third_party/toolchains/BUILD
index 509b541400..0e067708a8 100644
--- a/third_party/toolchains/BUILD
+++ b/third_party/toolchains/BUILD
@@ -30,7 +30,12 @@ platform(
         properties: {
             name: "container-image"
             value:"docker://gcr.io/asci-toolchain/nosla-ubuntu16_04-tf@sha256:63a0e981a4e7ce5da2a851cf063e430f72947fd999d9336b7e54e2eebe8e0bf5"
-        }""",
+        }
+        properties: {
+            name: "Pool"
+            value: "default"
+        }
+        """,
 )
 
 # Remote build platforms.
@@ -49,7 +54,12 @@ platform(
         properties: {
             name: "container-image"
             value:"docker://gcr.io/tensorflow-testing/nosla-ubuntu16.04@%s"
-        }""" % container_digests["ubuntu16.04"],
+        }
+        properties: {
+            name: "Pool"
+            value: "default"
+        }
+        """ % container_digests["ubuntu16.04"],
 )
 
 # Built with //tensorflow/tools/ci_build/Dockerfile.rbe.cuda9.0-cudnn7-ubuntu14.04.
@@ -63,7 +73,12 @@ platform(
         properties: {
             name: "container-image"
             value:"docker://gcr.io/tensorflow-testing/nosla-cuda9.0-cudnn7-ubuntu14.04@%s"
-        }""" % container_digests["cuda9.0-cudnn7-ubuntu14.04"],
+        }
+        properties: {
+            name: "Pool"
+            value: "default"
+        }
+        """ % container_digests["cuda9.0-cudnn7-ubuntu14.04"],
 )
 
 # Built with //tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04.
@@ -77,7 +92,12 @@ platform(
         properties: {
             name: "container-image"
             value:"docker://gcr.io/tensorflow-testing/nosla-cuda10.0-cudnn7-ubuntu14.04@%s"
-        }""" % container_digests["cuda10.0-cudnn7-ubuntu14.04"],
+        }
+        properties: {
+            name: "Pool"
+            value: "default"
+        }
+        """ % container_digests["cuda10.0-cudnn7-ubuntu14.04"],
 )
 
 # The above platform with GPU support.
-- 
GitLab


From b0fbe619491151831e4ec3b2a30cc2770f315d9e Mon Sep 17 00:00:00 2001
From: Dimitris Vardoulakis <dimvar@google.com>
Date: Thu, 24 Jan 2019 10:45:23 -0800
Subject: [PATCH 1192/2345] [TF:XLA] Don't do anything in the AR/CRS combiner
 if there is only one replica.

PiperOrigin-RevId: 230749050
---
 .../compiler/xla/service/ar_crs_combiner.cc   |  9 ++-
 .../xla/service/ar_crs_combiner_test.cc       | 63 +++++++++++++++++++
 .../compiler/xla/service/hlo_instructions.cc  |  9 +++
 .../compiler/xla/service/hlo_instructions.h   |  4 ++
 4 files changed, 82 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/service/ar_crs_combiner.cc b/tensorflow/compiler/xla/service/ar_crs_combiner.cc
index d01393a748..f8dff6a700 100644
--- a/tensorflow/compiler/xla/service/ar_crs_combiner.cc
+++ b/tensorflow/compiler/xla/service/ar_crs_combiner.cc
@@ -85,9 +85,12 @@ absl::optional<HloInstruction*> MatchesArCrsPattern(
       return absl::nullopt;
     }
   }
-  return computation_is_addition(next->called_computations()[0])
-             ? absl::optional<HloInstruction*>(next)
-             : absl::nullopt;
+  if (!Cast<HloAllReduceInstruction>(next)->IsNoop() &&
+      computation_is_addition(next->called_computations()[0])) {
+    return absl::optional<HloInstruction*>(next);
+  } else {
+    return absl::nullopt;
+  }
 }
 
 }  // namespace
diff --git a/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc b/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc
index ea28bba959..5152f0dc88 100644
--- a/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc
+++ b/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc
@@ -1108,5 +1108,68 @@ ENTRY %entrycomp (p: f32[]) -> (f32[], f32[]) {
   CompareReplicaGroups(replica_groups_before, replica_groups_after);
 }
 
+TEST_F(ArCrsCombinerTest, OneReplicaDontRewrite) {
+  const char* module_str = R"(
+HloModule foobar
+
+%sum.bf16 (a: bf16[], b: bf16[]) -> bf16[] {
+  %a = bf16[] parameter(0)
+  %b = bf16[] parameter(1)
+  ROOT %add = bf16[] add(%a, %b)
+}
+
+%sum.f32 (x: f32[], y: f32[]) -> f32[] {
+  %x = f32[] parameter(0)
+  %y = f32[] parameter(1)
+  ROOT %add = f32[] add(%x, %y)
+}
+
+ENTRY %entrycomp (p: bf16[]) -> (f32[], f32[]) {
+  %p = bf16[] parameter(0)
+  %constant.bf16 = bf16[] constant(1)
+
+  %all-reduce.ar.1 = bf16[]
+      all-reduce(%p),
+      replica_groups={{0}},
+      all_reduce_id=1,
+      to_apply=%sum.bf16,
+      sharding={maximal device=0}
+  %convert.1 = f32[]
+      convert(%all-reduce.ar.1),
+      sharding={maximal device=0}
+  %all-reduce.1 = f32[]
+      all-reduce(%convert.1),
+      replica_groups={{0}},
+      to_apply=%sum.f32,
+      sharding={maximal device=0}
+
+  %all-reduce.ar.2 = bf16[]
+      all-reduce(%constant.bf16),
+      replica_groups={{0}},
+      all_reduce_id=1,
+      to_apply=%sum.bf16,
+      sharding={maximal device=1}
+  %convert.2 = f32[]
+      convert(%all-reduce.ar.2),
+      sharding={maximal device=1}
+  %all-reduce.2 = f32[]
+      all-reduce(%convert.2),
+      replica_groups={{0}},
+      to_apply=%sum.f32,
+      sharding={maximal device=1}
+
+  ROOT %tuple = (f32[], f32[])
+      tuple(%all-reduce.1, %all-reduce.2),
+      sharding={{maximal device=0}, {maximal device=1}}
+}
+)";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseAndReturnVerifiedModule(module_str));
+  ArCrsCombiner combiner(2);
+  auto changed = combiner.Run(module.get()).ValueOrDie();
+  EXPECT_FALSE(changed);
+}
+
 }  // namespace
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/hlo_instructions.cc b/tensorflow/compiler/xla/service/hlo_instructions.cc
index 3a0d71dd88..b01f01ef01 100644
--- a/tensorflow/compiler/xla/service/hlo_instructions.cc
+++ b/tensorflow/compiler/xla/service/hlo_instructions.cc
@@ -383,6 +383,15 @@ HloInstructionProto HloAllReduceInstruction::ToProto() const {
   return proto;
 }
 
+bool HloAllReduceInstruction::IsNoop() const {
+  for (auto replica_group : replica_groups()) {
+    if (replica_group.replica_ids().size() != 1) {
+      return false;
+    }
+  }
+  return !all_reduce_id();
+}
+
 std::vector<string> HloAllReduceInstruction::ExtraAttributesToStringImpl(
     const HloPrintOptions& options) const {
   std::vector<string> result =
diff --git a/tensorflow/compiler/xla/service/hlo_instructions.h b/tensorflow/compiler/xla/service/hlo_instructions.h
index e6111cfb57..1b4a94753c 100644
--- a/tensorflow/compiler/xla/service/hlo_instructions.h
+++ b/tensorflow/compiler/xla/service/hlo_instructions.h
@@ -253,6 +253,10 @@ class HloAllReduceInstruction : public HloCollectiveInstruction {
   // Returns a serialized representation of this instruction.
   HloInstructionProto ToProto() const override;
 
+  // Returns true if the AllReduce does no communication, so it's equivalent
+  // to a mem copy.
+  bool IsNoop() const;
+
  private:
   std::vector<string> ExtraAttributesToStringImpl(
       const HloPrintOptions& options) const override;
-- 
GitLab


From 120abaf2b93da155274fd5d1682b0a73e5199c61 Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Thu, 24 Jan 2019 11:11:40 -0800
Subject: [PATCH 1193/2345] Cleaning up MultiDeviceIteratorResource in eager
 mode.

PiperOrigin-RevId: 230754817
---
 tensorflow/python/data/ops/multi_device_iterator_ops.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tensorflow/python/data/ops/multi_device_iterator_ops.py b/tensorflow/python/data/ops/multi_device_iterator_ops.py
index 8f5c550216..e9a26807ae 100644
--- a/tensorflow/python/data/ops/multi_device_iterator_ops.py
+++ b/tensorflow/python/data/ops/multi_device_iterator_ops.py
@@ -28,6 +28,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import functional_ops
 from tensorflow.python.ops import gen_dataset_ops
+from tensorflow.python.ops import resource_variable_ops
 
 
 class _PerDeviceGenerator(dataset_ops.DatasetV2):
@@ -166,6 +167,11 @@ class MultiDeviceIterator(object):
               shared_name=shared_name,
               container="",
               **dataset_ops.flat_structure(dataset)))
+      if context.executing_eagerly():
+        # Delete the resource when this object is deleted
+        self._resource_deleter = resource_variable_ops.EagerResourceDeleter(
+            handle=self._multi_device_iterator_resource,
+            handle_device=self._source_device)
 
       # The incarnation ID is used to ensure consistency between the per-device
       # iterators and the multi-device iterator.
-- 
GitLab


From 53cf96a261f39bf3dd5ecd582739f7673ffad284 Mon Sep 17 00:00:00 2001
From: Bruce Fontaine <bfontain@google.com>
Date: Thu, 24 Jan 2019 11:16:41 -0800
Subject: [PATCH 1194/2345] Fix a bug where self._tpu was not an array of
 bytes.

PiperOrigin-RevId: 230755897
---
 .../distribute/cluster_resolver/tpu_cluster_resolver.py     | 6 +++---
 .../cluster_resolver/tpu_cluster_resolver_test.py           | 3 +--
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
index 49e6ff8720..d02e9b8fe8 100644
--- a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
+++ b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
@@ -285,7 +285,7 @@ class TPUClusterResolver(ClusterResolver):
     # in later in self.master().
     if self.rpc_layer is not None and tpu.startswith(self.rpc_layer + '://'):
       tpu = tpu[len(self.rpc_layer + '://'):]
-      self._tpu = tpu
+      self._tpu = compat.as_bytes(tpu)  # self._tpu is always bytes
       self._should_resolve_override = False
 
     # Whether we should actually attempt to contact Cloud APIs
@@ -369,7 +369,7 @@ class TPUClusterResolver(ClusterResolver):
         master = job_tasks[0]
     else:
       if isinstance(self._tpu, (bytes, bytearray)):
-        master = self._tpu.split(compat.as_bytes(_ENDPOINTS_SEPARATOR))[0]
+        master = compat.as_text(self._tpu).split(_ENDPOINTS_SEPARATOR)[0]
       else:
         master = self._tpu.split(_ENDPOINTS_SEPARATOR)[0]
     return format_master_url(master, rpc_layer or self.rpc_layer)
@@ -439,7 +439,7 @@ class TPUClusterResolver(ClusterResolver):
         return None
       # Case 2.
       tpus = []
-      for tpu in self._tpu.split(_ENDPOINTS_SEPARATOR):
+      for tpu in compat.as_text(self._tpu).split(_ENDPOINTS_SEPARATOR):
         # We are working around the fact that GKE environment variable that is
         # supplied to us has the protocol string embedded in it, but we want
         # to strip it out for the ClusterSpec.
diff --git a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver_test.py b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver_test.py
index d5cfb2262b..0cb0dc607f 100644
--- a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver_test.py
+++ b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver_test.py
@@ -459,8 +459,7 @@ class TPUClusterResolverTest(test.TestCase):
   def testNoCallComputeMetadata(self):
     resolver = cluster_resolver.TPUClusterResolver(
         tpu='/bns/foo/bar')
-    self.assertEqual(
-        compat.as_bytes('/bns/foo/bar'), resolver.master())
+    self.assertEqual('/bns/foo/bar', resolver.master())
     self.assertEqual(None, resolver.cluster_spec())
 
   def testGkeEnvironmentForDonut(self):
-- 
GitLab


From 5b64cb57f09aa32d2c74f983b3e39fdb98dd11b8 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Thu, 24 Jan 2019 11:25:17 -0800
Subject: [PATCH 1195/2345] [XLA] Allow a per-computation choice of number of
 replicas via the xla::LocalClient interface.

PiperOrigin-RevId: 230757803
---
 .../xla/client/executable_build_options.cc    |  11 +-
 .../xla/client/executable_build_options.h     |   6 ++
 .../xla/python/local_computation_builder.cc   | 100 ++++++++----------
 .../xla/python/local_computation_builder.h    |   4 +
 .../xla/python/local_computation_builder.i    |   6 ++
 tensorflow/compiler/xla/python/xla_client.py  |   1 +
 .../compiler/xla/service/local_service.cc     |   1 +
 tensorflow/compiler/xla/service/service.cc    |   7 +-
 tensorflow/compiler/xla/service/service.h     |   2 +-
 tensorflow/compiler/xla/xla.proto             |   4 +
 10 files changed, 84 insertions(+), 58 deletions(-)

diff --git a/tensorflow/compiler/xla/client/executable_build_options.cc b/tensorflow/compiler/xla/client/executable_build_options.cc
index 1f594e551a..ec0e089759 100644
--- a/tensorflow/compiler/xla/client/executable_build_options.cc
+++ b/tensorflow/compiler/xla/client/executable_build_options.cc
@@ -58,6 +58,12 @@ const Shape* ExecutableBuildOptions::result_layout() const {
   return result_layout_set_ ? &result_layout_ : nullptr;
 }
 
+ExecutableBuildOptions& ExecutableBuildOptions::set_num_replicas(
+    int num_replicas) {
+  num_replicas_ = num_replicas;
+  return *this;
+}
+
 string ExecutableBuildOptions::ToString() const {
   string result_layout = "nullopt";
   if (result_layout_set_) {
@@ -65,8 +71,9 @@ string ExecutableBuildOptions::ToString() const {
   }
   return absl::StrFormat(
       "ExecutableBuildOptions{device_ordinal=%d, result_layout=%s, "
-      "generate_hlo_graph=%s}",
-      device_ordinal_, result_layout, debug_options().xla_generate_hlo_graph());
+      "generate_hlo_graph=%s, num_replicas=%d}",
+      device_ordinal_, result_layout, debug_options().xla_generate_hlo_graph(),
+      num_replicas_);
 }
 
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/client/executable_build_options.h b/tensorflow/compiler/xla/client/executable_build_options.h
index a58090253b..1d85fb3430 100644
--- a/tensorflow/compiler/xla/client/executable_build_options.h
+++ b/tensorflow/compiler/xla/client/executable_build_options.h
@@ -67,12 +67,18 @@ class ExecutableBuildOptions {
   // debugging.
   string ToString() const;
 
+  // The number of replicas of this computation that are to be executed.
+  // Defaults to 1.
+  int num_replicas() const { return num_replicas_; }
+  ExecutableBuildOptions& set_num_replicas(int num_replicas);
+
  private:
   int device_ordinal_ = -1;
   Shape result_layout_;
   bool result_layout_set_ = false;
   absl::optional<DebugOptions> debug_options_;
   DeviceMemoryAllocator* device_allocator_ = nullptr;
+  int num_replicas_ = 1;
 };
 
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.cc b/tensorflow/compiler/xla/python/local_computation_builder.cc
index a1bb9d6b67..3493e07f3c 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.cc
+++ b/tensorflow/compiler/xla/python/local_computation_builder.cc
@@ -334,37 +334,34 @@ CompiledLocalComputation::CompiledLocalComputation(
 
 StatusOr<LocalShapedBuffer*> CompiledLocalComputation::Execute(
     absl::Span<LocalShapedBuffer* const> argument_handles) {
+  if (num_replicas() != 1) {
+    return InvalidArgument(
+        "Attempted to execute computation with %d replicas using Execute()",
+        num_replicas());
+  }
   TF_ASSIGN_OR_RETURN(LocalClient * client, GetOrCreateLocalClient());
-  StatusOr<int> device_ordinal_status = client->ReplicaNumberToDeviceOrdinal(0);
+  TF_ASSIGN_OR_RETURN(DeviceAssignment device_assignment,
+                      client->backend().computation_placer()->AssignDevices(
+                          1, /*computation_count=*/1));
   StatusOr<ScopedShapedBuffer> result_buffer_status;
-  if (!device_ordinal_status.ok()) {
-    result_buffer_status = device_ordinal_status.status();
-  } else {
-    const int device_ordinal = device_ordinal_status.ValueOrDie();
-    VLOG(3) << "Replica 0 mapped to device ordinal for execution: "
-            << device_ordinal;
+  const int device_ordinal = device_assignment(0, 0);
+  VLOG(3) << "Replica 0 mapped to device ordinal for execution: "
+          << device_ordinal;
+
+  std::vector<const ShapedBuffer*> argument_buffers;
+  argument_buffers.reserve(argument_handles.size());
+  for (auto& handle : argument_handles) {
+    argument_buffers.push_back(handle->shaped_buffer());
+  }
 
-    std::vector<const ShapedBuffer*> argument_buffers;
-    argument_buffers.reserve(argument_handles.size());
-    for (auto& handle : argument_handles) {
-      argument_buffers.push_back(handle->shaped_buffer());
-    }
+  ExecutableRunOptions options;
+  options.set_device_ordinal(device_ordinal);
+  options.set_allocator(client->backend().memory_allocator());
+  options.set_intra_op_thread_pool(
+      client->backend().eigen_intra_op_thread_pool_device());
+  options.set_device_assignment(&device_assignment);
 
-    DeviceAssignment device_assignment =
-        client->backend()
-            .computation_placer()
-            ->AssignDevices(1, /*computation_count=*/1)
-            .ConsumeValueOrDie();
-
-    ExecutableRunOptions options;
-    options.set_device_ordinal(device_ordinal);
-    options.set_allocator(client->backend().memory_allocator());
-    options.set_intra_op_thread_pool(
-        client->backend().eigen_intra_op_thread_pool_device());
-    options.set_device_assignment(&device_assignment);
-
-    result_buffer_status = executable_->Run(argument_buffers, options);
-  }
+  result_buffer_status = executable_->Run(argument_buffers, options);
 
   if (!result_buffer_status.ok()) {
     return InternalError(
@@ -378,28 +375,29 @@ StatusOr<LocalShapedBuffer*> CompiledLocalComputation::Execute(
 StatusOr<LocalShapedBufferTuple*> CompiledLocalComputation::ExecutePerReplica(
     absl::Span<const std::vector<LocalShapedBuffer*>> argument_handles) {
   TF_ASSIGN_OR_RETURN(LocalClient * client, GetOrCreateLocalClient());
-  const int num_replicas = GetReplicaCount();
+  const int num_devices = client->device_count();
 
-  if (argument_handles.size() != num_replicas) {
+  if (argument_handles.size() != num_replicas()) {
     return InvalidArgument(
         "Attempted to execute with %d replicas when replica count is %d",
-        argument_handles.size(), num_replicas);
+        argument_handles.size(), num_devices);
   }
+  if (argument_handles.size() > num_devices) {
+    return InvalidArgument(
+        "Attempted to execute with %d replicas when device count is %d",
+        argument_handles.size(), num_devices);
+  }
+
+  VLOG(1) << "Executing with " << num_replicas() << " replicas.";
 
-  VLOG(1) << "Executing with " << num_replicas << " replicas.";
+  TF_ASSIGN_OR_RETURN(DeviceAssignment device_assignment,
+                      client->backend().computation_placer()->AssignDevices(
+                          num_replicas(), /*computation_count=*/1));
 
-  // Each replica populates a StatusOr result, but only the output value of
-  // replica zero is returned.
-  std::vector<StatusOr<ScopedShapedBuffer>> results(num_replicas);
-  auto execute = [this, client, num_replicas, &argument_handles,
+  std::vector<StatusOr<ScopedShapedBuffer>> results(num_replicas());
+  auto execute = [this, client, &device_assignment, &argument_handles,
                   &results](int replica) {
-    StatusOr<int> device_ordinal_status =
-        client->ReplicaNumberToDeviceOrdinal(replica);
-    if (!device_ordinal_status.ok()) {
-      results[replica] = device_ordinal_status.status();
-      return;
-    }
-    const int device_ordinal = device_ordinal_status.ValueOrDie();
+    const int device_ordinal = device_assignment(replica, 0);
     VLOG(3) << "Replica " << replica
             << " mapped to device ordinal for execution: " << device_ordinal;
 
@@ -409,12 +407,6 @@ StatusOr<LocalShapedBufferTuple*> CompiledLocalComputation::ExecutePerReplica(
       argument_buffers.push_back(handle->shaped_buffer());
     }
 
-    DeviceAssignment device_assignment =
-        client->backend()
-            .computation_placer()
-            ->AssignDevices(num_replicas, /*computation_count=*/1)
-            .ConsumeValueOrDie();
-
     ExecutableRunOptions options;
     options.set_device_ordinal(device_ordinal);
     options.set_allocator(client->backend().memory_allocator());
@@ -427,23 +419,23 @@ StatusOr<LocalShapedBufferTuple*> CompiledLocalComputation::ExecutePerReplica(
     results[replica] = std::move(result_buffer_status);
   };
 
-  if (num_replicas == 1) {
+  if (num_replicas() == 1) {
     // Fast-path if there is only one replica — run the computation on the
     // current thread.
     execute(0);
   } else {
     // TODO(phawkins): don't recreate the threadpool for each execution.
     tensorflow::thread::ThreadPool pool(tensorflow::Env::Default(), "xlarun",
-                                        num_replicas - 1);
+                                        num_replicas() - 1);
 
-    for (int replica = 0; replica < num_replicas - 1; ++replica) {
+    for (int replica = 0; replica < num_replicas() - 1; ++replica) {
       pool.Schedule([&execute, replica] { execute(replica); });
     }
-    execute(num_replicas - 1);
+    execute(num_replicas() - 1);
   }
 
-  std::vector<LocalShapedBuffer*> wrapped_results(num_replicas);
-  for (int replica = 0; replica < num_replicas; ++replica) {
+  std::vector<LocalShapedBuffer*> wrapped_results(num_replicas());
+  for (int replica = 0; replica < num_replicas(); ++replica) {
     auto& statusor = results[replica];
     if (!statusor.ok()) {
       return InternalError(
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.h b/tensorflow/compiler/xla/python/local_computation_builder.h
index 98759cf984..5da19dda62 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.h
+++ b/tensorflow/compiler/xla/python/local_computation_builder.h
@@ -180,6 +180,10 @@ class CompiledLocalComputation {
  public:
   CompiledLocalComputation(std::unique_ptr<LocalExecutable> executable);
 
+  int num_replicas() const {
+    return executable_->build_options().num_replicas();
+  }
+
   StatusOr<LocalShapedBuffer*> Execute(
       absl::Span<LocalShapedBuffer* const> argument_handles);
 
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.i b/tensorflow/compiler/xla/python/local_computation_builder.i
index 39bff3f571..7798f83b29 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.i
+++ b/tensorflow/compiler/xla/python/local_computation_builder.i
@@ -927,6 +927,12 @@ tensorflow::ImportNumpy();
     }
     Py_DECREF(o);
 
+    int64 num_replicas;
+    if (!GetIntAttr($input, "num_replicas", &num_replicas)) {
+      SWIG_fail;
+    }
+    build_options.set_num_replicas(num_replicas);
+
     $1 = &build_options;
   }
 }
diff --git a/tensorflow/compiler/xla/python/xla_client.py b/tensorflow/compiler/xla/python/xla_client.py
index 1684cb20e6..e1fefe1a35 100644
--- a/tensorflow/compiler/xla/python/xla_client.py
+++ b/tensorflow/compiler/xla/python/xla_client.py
@@ -459,6 +459,7 @@ class CompileOptions(object):
     self.dump_unoptimized_hlo_proto_to = None
     self.dump_per_pass_hlo_proto_to = None
     self.hlo_profile = False
+    self.num_replicas = get_replica_count()
 
 
 def transfer_to_infeed(value, replica_number=None):
diff --git a/tensorflow/compiler/xla/service/local_service.cc b/tensorflow/compiler/xla/service/local_service.cc
index 600b069ecd..3470fe5b2c 100644
--- a/tensorflow/compiler/xla/service/local_service.cc
+++ b/tensorflow/compiler/xla/service/local_service.cc
@@ -110,6 +110,7 @@ ExecutionOptions CreateExecutionOptions(
     *execution_options.mutable_shape_with_output_layout() =
         result_shape.ToProto();
   }
+  execution_options.set_num_replicas(build_options.num_replicas());
   return execution_options;
 }
 
diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc
index 32707de700..83434528a2 100644
--- a/tensorflow/compiler/xla/service/service.cc
+++ b/tensorflow/compiler/xla/service/service.cc
@@ -296,11 +296,16 @@ StatusOr<std::unique_ptr<HloModuleConfig>> Service::CreateModuleConfig(
     computation_layout->mutable_result_layout()->SetToDefaultLayout();
   }
 
-  config->set_replica_count(options_.number_of_replicas());
   if (execution_options != nullptr) {
+    if (execution_options->num_replicas() > 0) {
+      config->set_replica_count(execution_options->num_replicas());
+    } else {
+      config->set_replica_count(options_.number_of_replicas());
+    }
     config->set_seed(execution_options->seed());
     config->set_debug_options(execution_options->debug_options());
   } else {
+    config->set_replica_count(options_.number_of_replicas());
     config->set_debug_options(GetDebugOptionsFromFlags());
   }
 
diff --git a/tensorflow/compiler/xla/service/service.h b/tensorflow/compiler/xla/service/service.h
index 6e31bea728..fd907d07da 100644
--- a/tensorflow/compiler/xla/service/service.h
+++ b/tensorflow/compiler/xla/service/service.h
@@ -53,7 +53,7 @@ class ServiceOptions {
   ServiceOptions& set_platform(se::Platform* platform);
   se::Platform* platform() const;
 
-  // Set the number of replicas to use when compiling replicated
+  // Set the default number of replicas to use when compiling replicated
   // programs.
   ServiceOptions& set_number_of_replicas(int number_of_replicas);
   int number_of_replicas() const;
diff --git a/tensorflow/compiler/xla/xla.proto b/tensorflow/compiler/xla/xla.proto
index e2d7b6ef46..92834dbb02 100644
--- a/tensorflow/compiler/xla/xla.proto
+++ b/tensorflow/compiler/xla/xla.proto
@@ -265,6 +265,10 @@ message ExecutionOptions {
   // computation on. The computation will be partitioned across these devices.
   // If not provided, the default device will be chosen.
   repeated DeviceHandle device_handles = 5;
+
+  // Number of replicas of the computation to run. If zero, uses the default
+  // number of replicas for the XLA service.
+  int32 num_replicas = 6;
 }
 
 message GetDeviceHandlesRequest {
-- 
GitLab


From 5d22c6994ca0492a34c5ecdff982f3609aa675fb Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Thu, 24 Jan 2019 11:47:35 -0800
Subject: [PATCH 1196/2345] Internal change.

PiperOrigin-RevId: 230762128
---
 tensorflow/tools/ci_build/update_version.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/tensorflow/tools/ci_build/update_version.py b/tensorflow/tools/ci_build/update_version.py
index 4373d464b6..81488902b5 100755
--- a/tensorflow/tools/ci_build/update_version.py
+++ b/tensorflow/tools/ci_build/update_version.py
@@ -299,13 +299,17 @@ def main():
   old_version = get_current_semver_version()
 
   if args.nightly:
-    # Dev minor version is one ahead of official.
-    nightly_minor_ver = int(old_version.minor) + 1
-    new_version = Version(old_version.major,
-                          str(nightly_minor_ver),
-                          old_version.patch,
-                          "-dev" + time.strftime("%Y%m%d"),
-                          NIGHTLY_VERSION)
+    if args.version:
+      new_version = Version.parse_from_string(args.version, NIGHTLY_VERSION)
+      new_version.identifier_string = "-dev" + time.strftime("%Y%m%d")
+    else:
+      # Dev minor version is one ahead of official.
+      nightly_minor_ver = int(old_version.minor) + 1
+      new_version = Version(old_version.major,
+                            str(nightly_minor_ver),
+                            old_version.patch,
+                            "-dev" + time.strftime("%Y%m%d"),
+                            NIGHTLY_VERSION)
   else:
     new_version = Version.parse_from_string(args.version, REGULAR_VERSION)
 
-- 
GitLab


From 5f8a1a540fd7cd1cd2af7490edc2c72fe0835706 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 24 Jan 2019 11:50:55 -0800
Subject: [PATCH 1197/2345] [XLA] Make explicit that some special functions
 don't accept imaginary numbers.

Our erfc, lgamma, digamma, and RoundToEven special functions don't accept
imaginary numbers.  Make this explicit, with tests.  Also remove code inside of
lgamma and digamma that looked like it was trying to handle complex numbers.

PiperOrigin-RevId: 230762743
---
 tensorflow/compiler/xla/client/lib/math.cc    | 255 +++++++++++-------
 .../compiler/xla/client/lib/math_test.cc      |  35 +++
 2 files changed, 192 insertions(+), 98 deletions(-)

diff --git a/tensorflow/compiler/xla/client/lib/math.cc b/tensorflow/compiler/xla/client/lib/math.cc
index 6f72d09235..c56c371ae5 100644
--- a/tensorflow/compiler/xla/client/lib/math.cc
+++ b/tensorflow/compiler/xla/client/lib/math.cc
@@ -80,25 +80,47 @@ XlaOp EvaluatePolynomial(XlaOp x, absl::Span<const float> coefficients) {
 
 // Compute an approximation of the error function complement (1 - erf(x)).
 XlaOp Erfc(XlaOp x) {
-  XlaOp abs_x = Abs(x);
-  XlaOp z = Exp(-x * x);
+  auto& b = *x.builder();
+  return b.ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    // Reject non-real non-fp inputs.  (We could extend erfc to accept complex
+    // types, but it doesn't seem necessary at this point.)
+    TF_ASSIGN_OR_RETURN(auto shape, b.GetShape(x));
+    if (!ShapeUtil::ElementIsFloating(shape)) {
+      return InvalidArgument(
+          "erfc only accepts real floating-point arrays or scalars, but got %s",
+          shape.ToString());
+    }
+    XlaOp abs_x = Abs(x);
+    XlaOp z = Exp(-x * x);
 
-  XlaOp pp = EvaluatePolynomial(abs_x, kErfcPCoefficient);
-  XlaOp pq = EvaluatePolynomial(abs_x, kErfcQCoefficient);
-  XlaOp pr = EvaluatePolynomial(abs_x, kErfcRCoefficient);
-  XlaOp ps = EvaluatePolynomial(abs_x, kErfcSCoefficient);
+    XlaOp pp = EvaluatePolynomial(abs_x, kErfcPCoefficient);
+    XlaOp pq = EvaluatePolynomial(abs_x, kErfcQCoefficient);
+    XlaOp pr = EvaluatePolynomial(abs_x, kErfcRCoefficient);
+    XlaOp ps = EvaluatePolynomial(abs_x, kErfcSCoefficient);
 
-  XlaOp y = Select(Lt(abs_x, ScalarLike(x, 8.0)), z * pp / pq, z * pr / ps);
+    XlaOp y = Select(Lt(abs_x, ScalarLike(x, 8.0)), z * pp / pq, z * pr / ps);
 
-  return Select(Lt(x, ScalarLike(x, 0.0)), ScalarLike(x, 2.0) - y, y);
+    return Select(Lt(x, ScalarLike(x, 0.0)), ScalarLike(x, 2.0) - y, y);
+  });
 }
 
 // Compute a polynomial approximation of the error function.
 XlaOp Erf(XlaOp x) {
-  XlaOp z = x * x;
-  XlaOp pt = EvaluatePolynomial(z, kErfTCoefficient);
-  XlaOp pu = EvaluatePolynomial(z, kErfUCoefficient);
-  return x * pt / pu;
+  auto& b = *x.builder();
+  return b.ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    // Reject non-real non-fp inputs.  (We could extend erf to accept complex
+    // types, but it doesn't seem necessary at this point.)
+    TF_ASSIGN_OR_RETURN(auto shape, b.GetShape(x));
+    if (!ShapeUtil::ElementIsFloating(shape)) {
+      return InvalidArgument(
+          "erf only accepts real floating-point arrays or scalars, but got %s",
+          shape.ToString());
+    }
+    XlaOp z = x * x;
+    XlaOp pt = EvaluatePolynomial(z, kErfTCoefficient);
+    XlaOp pu = EvaluatePolynomial(z, kErfUCoefficient);
+    return x * pt / pu;
+  });
 }
 
 // Approximation for the inverse error function from
@@ -171,49 +193,62 @@ static constexpr std::array<double, 8> kLanczosCoefficients = {
 // t(z) = z + kLanczosGamma + 1/2
 // A(z) = kBaseLanczosCoeff + sigma(k = 1, n, kLanczosCoefficients[i] / (z + k))
 XlaOp Lgamma(XlaOp input) {
-  XlaOp one_half = ScalarLike(input, 0.5);
-  XlaOp one = ScalarLike(input, 1);
+  auto& b = *input.builder();
+  return b.ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    // Reject non-real non-fp inputs.  (We could extend lgamma to accept complex
+    // types, but it doesn't seem necessary at this point.)
+    TF_ASSIGN_OR_RETURN(auto shape, b.GetShape(input));
+    if (!ShapeUtil::ElementIsFloating(shape)) {
+      return InvalidArgument(
+          "lgamma only accepts real floating-point arrays or scalars, but got "
+          "%s",
+          shape.ToString());
+    }
 
-  XlaOp pi = ScalarLike(input, M_PI);
-  XlaOp log_pi = ScalarLike(input, std::log(M_PI));
-  XlaOp log_sqrt_two_pi = ScalarLike(input, (std::log(2) + std::log(M_PI)) / 2);
+    XlaOp one_half = ScalarLike(input, 0.5);
+    XlaOp one = ScalarLike(input, 1);
 
-  XlaOp lanczos_gamma_plus_one_half = ScalarLike(input, kLanczosGamma + 0.5);
-  XlaOp log_lanczos_gamma_plus_one_half =
-      ScalarLike(input, std::log(kLanczosGamma + 0.5));
+    XlaOp pi = ScalarLike(input, M_PI);
+    XlaOp log_pi = ScalarLike(input, std::log(M_PI));
+    XlaOp log_sqrt_two_pi =
+        ScalarLike(input, (std::log(2) + std::log(M_PI)) / 2);
 
-  XlaOp base_lanczos_coeff = ScalarLike(input, kBaseLanczosCoeff);
+    XlaOp lanczos_gamma_plus_one_half = ScalarLike(input, kLanczosGamma + 0.5);
+    XlaOp log_lanczos_gamma_plus_one_half =
+        ScalarLike(input, std::log(kLanczosGamma + 0.5));
 
-  // If the input is less than 0.5 use Euler's reflection formula:
-  // gamma(x) = pi / (sin(pi * x) * gamma(1 - x))
-  XlaOp need_to_reflect = Lt(Real(input), one_half);
-  XlaOp z = Select(need_to_reflect, -input, input - one);
+    XlaOp base_lanczos_coeff = ScalarLike(input, kBaseLanczosCoeff);
 
-  XlaOp x = base_lanczos_coeff;
-  for (int i = 0; i < kLanczosCoefficients.size(); ++i) {
-    XlaOp lanczos_coefficient = ScalarLike(input, kLanczosCoefficients[i]);
-    XlaOp index = ScalarLike(input, i);
-    x = x + lanczos_coefficient / (z + index + one);
-  }
+    // If the input is less than 0.5 use Euler's reflection formula:
+    // gamma(x) = pi / (sin(pi * x) * gamma(1 - x))
+    XlaOp need_to_reflect = Lt(input, one_half);
+    XlaOp z = Select(need_to_reflect, -input, input - one);
 
-  // To improve accuracy on platforms with less-precise log implementations,
-  // compute log(lanczos_gamma_plus_one_half) at compile time and use log1p on
-  // the device.
-  // log(t) = log(kLanczosGamma + 0.5 + z)
-  //        = log(kLanczosGamma + 0.5) + log1p(z / (kLanczosGamma + 0.5))
-  XlaOp t = lanczos_gamma_plus_one_half + z;
-  XlaOp log_t =
-      log_lanczos_gamma_plus_one_half + Log1p(z / lanczos_gamma_plus_one_half);
-
-  XlaOp log_y = log_sqrt_two_pi + (z + one_half) * log_t - t + Log(x);
-
-  // If z = a + 0j, the analytic continuation of log reduces to taking the
-  // absolute value of the real part.
-  // Re(log(z)) = Re(log|z| + arg(z)j)
-  //            = log|a|
-  XlaOp reflection = log_pi - Log(Abs(Sin(pi * input))) - log_y;
-  XlaOp result = Select(need_to_reflect, reflection, log_y);
-  return result;
+    XlaOp x = base_lanczos_coeff;
+    for (int i = 0; i < kLanczosCoefficients.size(); ++i) {
+      XlaOp lanczos_coefficient = ScalarLike(input, kLanczosCoefficients[i]);
+      XlaOp index = ScalarLike(input, i);
+      x = x + lanczos_coefficient / (z + index + one);
+    }
+
+    // To improve accuracy on platforms with less-precise log implementations,
+    // compute log(lanczos_gamma_plus_one_half) at compile time and use log1p on
+    // the device.
+    // log(t) = log(kLanczosGamma + 0.5 + z)
+    //        = log(kLanczosGamma + 0.5) + log1p(z / (kLanczosGamma + 0.5))
+    XlaOp t = lanczos_gamma_plus_one_half + z;
+    XlaOp log_t = log_lanczos_gamma_plus_one_half +
+                  Log1p(z / lanczos_gamma_plus_one_half);
+
+    XlaOp log_y = log_sqrt_two_pi + (z + one_half) * log_t - t + Log(x);
+
+    // If z = a + 0j, the analytic continuation of log reduces to taking the
+    // absolute value of the real part.
+    // Re(log(z)) = Re(log|z| + arg(z)j)
+    //            = log|a|
+    XlaOp reflection = log_pi - Log(Abs(Sin(pi * input))) - log_y;
+    return Select(need_to_reflect, reflection, log_y);
+  });
 }
 
 // Compute the Digamma function using Lanczos' approximation from "A Precision
@@ -224,61 +259,85 @@ XlaOp Lgamma(XlaOp input) {
 // A(z) = kBaseLanczosCoeff + sigma(k = 1, n, kLanczosCoefficients[i] / (z + k))
 // A'(z) = sigma(k = 1, n, kLanczosCoefficients[i] / (z + k) / (z + k))
 XlaOp Digamma(XlaOp input) {
-  XlaOp zero = ScalarLike(input, 0);
-  XlaOp one_half = ScalarLike(input, 0.5);
-  XlaOp one = ScalarLike(input, 1);
-
-  XlaOp pi = ScalarLike(input, M_PI);
-
-  XlaOp lanczos_gamma = ScalarLike(input, kLanczosGamma);
-  XlaOp lanczos_gamma_plus_one_half = ScalarLike(input, kLanczosGamma + 0.5);
-  XlaOp log_lanczos_gamma_plus_one_half =
-      ScalarLike(input, std::log(kLanczosGamma + 0.5));
-
-  XlaOp base_lanczos_coeff = ScalarLike(input, kBaseLanczosCoeff);
-
-  // If the input is less than 0.5 use Euler's reflection formula:
-  // digamma(x) = digamma(1 - x) - pi * cot(pi * x)
-  XlaOp need_to_reflect = Lt(Real(input), one_half);
-  XlaOp z = Select(need_to_reflect, -input, input - one);
-
-  XlaOp num = zero;
-  XlaOp denom = base_lanczos_coeff;
-  for (int i = 0; i < kLanczosCoefficients.size(); ++i) {
-    XlaOp lanczos_coefficient = ScalarLike(input, kLanczosCoefficients[i]);
-    XlaOp index = ScalarLike(input, i);
-    num = num - lanczos_coefficient / ((z + index + one) * (z + index + one));
-    denom = denom + lanczos_coefficient / (z + index + one);
-  }
+  auto& b = *input.builder();
+  return b.ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    // Reject non-real non-fp inputs.  (We could extend digamma to accept
+    // complex types, but it doesn't seem necessary at this point.)
+    TF_ASSIGN_OR_RETURN(auto shape, b.GetShape(input));
+    if (!ShapeUtil::ElementIsFloating(shape)) {
+      return InvalidArgument(
+          "digamma only accepts real floating-point arrays or scalars, but got "
+          "%s",
+          shape.ToString());
+    }
 
-  // To improve accuracy on platforms with less-precise log implementations,
-  // compute log(lanczos_gamma_plus_one_half) at compile time and use log1p on
-  // the device.
-  // log(t) = log(kLanczosGamma + 0.5 + z)
-  //        = log(kLanczosGamma + 0.5) + log1p(z / (kLanczosGamma + 0.5))
-  XlaOp t = lanczos_gamma_plus_one_half + z;
-  XlaOp log_t =
-      log_lanczos_gamma_plus_one_half + Log1p(z / lanczos_gamma_plus_one_half);
-
-  XlaOp y = log_t + num / denom - lanczos_gamma / t;
-  XlaOp reflection = y - pi * Cos(pi * input) / Sin(pi * input);
-  XlaOp result = Select(need_to_reflect, reflection, y);
-  return result;
+    XlaOp zero = ScalarLike(input, 0);
+    XlaOp one_half = ScalarLike(input, 0.5);
+    XlaOp one = ScalarLike(input, 1);
+
+    XlaOp pi = ScalarLike(input, M_PI);
+
+    XlaOp lanczos_gamma = ScalarLike(input, kLanczosGamma);
+    XlaOp lanczos_gamma_plus_one_half = ScalarLike(input, kLanczosGamma + 0.5);
+    XlaOp log_lanczos_gamma_plus_one_half =
+        ScalarLike(input, std::log(kLanczosGamma + 0.5));
+
+    XlaOp base_lanczos_coeff = ScalarLike(input, kBaseLanczosCoeff);
+
+    // If the input is less than 0.5 use Euler's reflection formula:
+    // digamma(x) = digamma(1 - x) - pi * cot(pi * x)
+    XlaOp need_to_reflect = Lt(input, one_half);
+    XlaOp z = Select(need_to_reflect, -input, input - one);
+
+    XlaOp num = zero;
+    XlaOp denom = base_lanczos_coeff;
+    for (int i = 0; i < kLanczosCoefficients.size(); ++i) {
+      XlaOp lanczos_coefficient = ScalarLike(input, kLanczosCoefficients[i]);
+      XlaOp index = ScalarLike(input, i);
+      num = num - lanczos_coefficient / ((z + index + one) * (z + index + one));
+      denom = denom + lanczos_coefficient / (z + index + one);
+    }
+
+    // To improve accuracy on platforms with less-precise log implementations,
+    // compute log(lanczos_gamma_plus_one_half) at compile time and use log1p on
+    // the device.
+    // log(t) = log(kLanczosGamma + 0.5 + z)
+    //        = log(kLanczosGamma + 0.5) + log1p(z / (kLanczosGamma + 0.5))
+    XlaOp t = lanczos_gamma_plus_one_half + z;
+    XlaOp log_t = log_lanczos_gamma_plus_one_half +
+                  Log1p(z / lanczos_gamma_plus_one_half);
+
+    XlaOp y = log_t + num / denom - lanczos_gamma / t;
+    XlaOp reflection = y - pi * Cos(pi * input) / Sin(pi * input);
+    return Select(need_to_reflect, reflection, y);
+  });
 }
 
 // Implements Banker's rounding: numbers that are equidistant between two
 // integers are rounded towards even.
 XlaOp RoundToEven(XlaOp x) {
-  auto half = ScalarLike(x, 0.5);
-  auto one = ScalarLike(x, 1.0);
-  auto two = ScalarLike(x, 2.0);
-
-  auto round_val = Floor(x);
-  auto fraction = x - round_val;
-  auto nearest_even_int = round_val - two * Floor(half * x);
-  auto is_odd = Eq(nearest_even_int, one);
-  return Select(Or(Gt(fraction, half), And(Eq(fraction, half), is_odd)),
-                round_val + one, round_val);
+  auto& b = *x.builder();
+  return b.ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    // Reject non-real non-fp inputs (What does it even mean to round a complex
+    // number?  Do you round each component equally?  In that case, you should
+    // just ask for that explicitly.)
+    TF_ASSIGN_OR_RETURN(auto shape, b.GetShape(x));
+    if (ShapeUtil::ElementIsComplex(shape)) {
+      return InvalidArgument(
+          "RoundToEven doesn't accept complex inputs, but got %s",
+          shape.ToString());
+    }
+    auto half = ScalarLike(x, 0.5);
+    auto one = ScalarLike(x, 1.0);
+    auto two = ScalarLike(x, 2.0);
+
+    auto round_val = Floor(x);
+    auto fraction = x - round_val;
+    auto nearest_even_int = round_val - two * Floor(half * x);
+    auto is_odd = Eq(nearest_even_int, one);
+    return Select(Or(Gt(fraction, half), And(Eq(fraction, half), is_odd)),
+                  round_val + one, round_val);
+  });
 }
 
 // Trigonometric functions.
diff --git a/tensorflow/compiler/xla/client/lib/math_test.cc b/tensorflow/compiler/xla/client/lib/math_test.cc
index ae2ea225d1..9947e0cf83 100644
--- a/tensorflow/compiler/xla/client/lib/math_test.cc
+++ b/tensorflow/compiler/xla/client/lib/math_test.cc
@@ -148,5 +148,40 @@ XLA_TEST_F(MathTest, RoundToEven) {
   ComputeAndCompareR1<float>(&builder, expected, {}, error_spec_);
 }
 
+XLA_TEST_F(MathTest, ErfRejectsComplexInputs) {
+  XlaBuilder b(TestName());
+  auto x = ConstantR1<std::complex<float>>(&b, {{0, 0}});
+  Erf(x);
+  EXPECT_FALSE(b.Build().status().ok());
+}
+
+XLA_TEST_F(MathTest, ErfcRejectsComplexInputs) {
+  XlaBuilder b(TestName());
+  auto x = ConstantR1<std::complex<float>>(&b, {{0, 0}});
+  Erfc(x);
+  EXPECT_FALSE(b.Build().status().ok());
+}
+
+XLA_TEST_F(MathTest, LgammaRejectsComplexInputs) {
+  XlaBuilder b(TestName());
+  auto x = ConstantR1<std::complex<float>>(&b, {{0, 0}});
+  Lgamma(x);
+  EXPECT_FALSE(b.Build().status().ok());
+}
+
+XLA_TEST_F(MathTest, DigammaRejectsComplexInputs) {
+  XlaBuilder b(TestName());
+  auto x = ConstantR1<std::complex<float>>(&b, {{0, 0}});
+  Digamma(x);
+  EXPECT_FALSE(b.Build().status().ok());
+}
+
+XLA_TEST_F(MathTest, RoundToEvenRejectsComplexInputs) {
+  XlaBuilder b(TestName());
+  auto x = ConstantR1<std::complex<float>>(&b, {{0, 0}});
+  RoundToEven(x);
+  EXPECT_FALSE(b.Build().status().ok());
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From d54d31d8ed2dc4e40dfaae76fe1b1afa5329c5b9 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 24 Jan 2019 11:56:58 -0800
Subject: [PATCH 1198/2345] [XLA] Fix literal_comparison's handling of nan vs
 inf.

Previously, we'd always report expected nan vs. actual inf as a miscompare,
even if relaxed_nan was set.  This is just a bug; the intent of relaxed_nan is
to allow any actual value if the expected is nan.

PiperOrigin-RevId: 230763808
---
 tensorflow/compiler/xla/literal_comparison.cc | 53 +++++++++----------
 1 file changed, 25 insertions(+), 28 deletions(-)

diff --git a/tensorflow/compiler/xla/literal_comparison.cc b/tensorflow/compiler/xla/literal_comparison.cc
index 69efa06d39..23601245e4 100644
--- a/tensorflow/compiler/xla/literal_comparison.cc
+++ b/tensorflow/compiler/xla/literal_comparison.cc
@@ -200,32 +200,26 @@ int64 RecursiveElementCount(const Shape& shape) {
   }
 }
 
-// Returns whether the actual and expected values are mismatched with respect to
-// nans. 'relaxed_nans' is interpreted as in xla::ErrorSpec.
+// Returns whether the given value is infinity.
 template <typename NativeT>
-bool NanMismatch(NativeT expected, NativeT actual, bool relaxed_nans) {
-  if (relaxed_nans) {
-    return !std::isnan(expected) && std::isnan(actual);
-  } else {
-    return std::isnan(expected) != std::isnan(actual);
-  }
+bool IsInf(NativeT val) {
+  return std::isinf(val);
 }
 
 template <>
-bool NanMismatch<half>(half expected, half actual, bool relaxed_nans) {
-  return NanMismatch<float>(static_cast<float>(expected),
-                            static_cast<float>(actual), relaxed_nans);
+bool IsInf<half>(half val) {
+  return std::isinf(static_cast<float>(val));
 }
 
-// Returns whether the given value is infinity.
+// Returns whether the given value is nan.
 template <typename NativeT>
-bool IsInf(NativeT val) {
-  return std::isinf(val);
+float IsNan(NativeT value) {
+  return std::isnan(value);
 }
 
 template <>
-bool IsInf<half>(half val) {
-  return std::isinf(static_cast<float>(val));
+float IsNan(half value) {
+  return IsNan<float>(static_cast<float>(value));
 }
 
 // Converts the given floating-point value to a string.
@@ -376,21 +370,25 @@ class NearComparator {
   // the given literal_index and keeps track of various mismatch statistics.
   template <typename T>
   void CompareValues(T expected, T actual, int64 linear_index) {
-    const bool is_nan_mismatch =
-        NanMismatch(expected, actual, error_.relaxed_nans);
     float abs_error;
     float rel_error;
     if (CompareEqual<T>(expected, actual, {linear_index})) {
       abs_error = 0;
       rel_error = 0;
-    } else if (is_nan_mismatch) {
-      num_nan_mismatches_++;
-      // A nan mismatch is considered to have infinite error. rel_error is used
-      // for sorting a std::set of the top mismatchs, and a nan value here will
-      // result in undefined behavior because nan's do not satisfy the strict
-      // weak ordering requirement of std containers.
-      abs_error = std::numeric_limits<float>::infinity();
-      rel_error = std::numeric_limits<float>::infinity();
+    } else if (IsNan(expected) || IsNan(actual)) {
+      if ((!error_.relaxed_nans && IsNan(expected) != IsNan(actual)) ||
+          (error_.relaxed_nans && !IsNan(expected) && IsNan(actual))) {
+        num_nan_mismatches_++;
+        // A nan mismatch is considered to have infinite error. rel_error is
+        // used for sorting a std::set of the top mismatchs, and a nan value
+        // here will result in undefined behavior because nan's do not satisfy
+        // the strict weak ordering requirement of std containers.
+        abs_error = std::numeric_limits<float>::infinity();
+        rel_error = std::numeric_limits<float>::infinity();
+      } else {
+        abs_error = 0;
+        rel_error = 0;
+      }
     } else if (IsInf(expected) || IsInf(actual)) {
       // If either the expected or actual value is infinity but not both,
       // then both absolute and relative error are regarded as inifity.
@@ -410,8 +408,7 @@ class NearComparator {
     }
     const bool is_abs_mismatch = abs_error > error_.abs;
     const bool is_rel_mismatch = rel_error > error_.rel;
-    const bool is_mismatch =
-        is_nan_mismatch || (is_abs_mismatch && is_rel_mismatch);
+    const bool is_mismatch = is_abs_mismatch && is_rel_mismatch;
 
     // Update the error of the relative bucket only if the *absolute* error
     // bound is exceeded and vice versa.
-- 
GitLab


From e0e7cbbc55af32041b1721bb4600a38f352d8242 Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Thu, 24 Jan 2019 11:56:59 -0800
Subject: [PATCH 1199/2345] Disable TF whitelisting in Keras Layers.

PiperOrigin-RevId: 230763811
---
 tensorflow/python/keras/BUILD                 |  11 --
 tensorflow/python/keras/engine/base_layer.py  |   7 -
 .../keras/layers/tensorflow_op_layer_test.py  | 131 ------------------
 3 files changed, 149 deletions(-)
 delete mode 100644 tensorflow/python/keras/layers/tensorflow_op_layer_test.py

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index cfbe56d49f..460340b392 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -485,17 +485,6 @@ tf_py_test(
     ],
 )
 
-tf_py_test(
-    name = "tensorflow_op_layer_test",
-    size = "medium",
-    srcs = ["layers/tensorflow_op_layer_test.py"],
-    additional_deps = [
-        ":keras",
-        "@absl_py//absl/testing:parameterized",
-        "//tensorflow/python:client_testlib",
-    ],
-)
-
 tf_py_test(
     name = "convolutional_recurrent_test",
     size = "large",
diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index 93849e771b..5eb2c5a752 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -524,13 +524,6 @@ class Layer(checkpointable.Checkpointable):
     # models using the functional API).
     build_graph = tf_utils.are_all_symbolic_tensors(input_list)
 
-    if build_graph:
-      # Only create Keras history if at least one tensor originates from a
-      # `keras.Input`. Otherwise this Layer may be being used outside the Keras
-      # framework.
-      if base_layer_utils.uses_keras_input_layers(inputs):
-        base_layer_utils.create_keras_history(inputs)
-
     # Handle Keras mask propagation from previous layer to current layer.
     previous_mask = None
     if build_graph and (not hasattr(self, '_compute_previous_mask') or
diff --git a/tensorflow/python/keras/layers/tensorflow_op_layer_test.py b/tensorflow/python/keras/layers/tensorflow_op_layer_test.py
deleted file mode 100644
index 3268bf8b64..0000000000
--- a/tensorflow/python/keras/layers/tensorflow_op_layer_test.py
+++ /dev/null
@@ -1,131 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Test for allowing TF ops to work with Keras Functional API."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from absl.testing import parameterized
-import numpy as np
-
-from tensorflow.python import keras
-from tensorflow.python.framework import ops
-from tensorflow.python.keras import keras_parameterized
-from tensorflow.python.keras import testing_utils
-from tensorflow.python.keras.optimizer_v2 import adam
-from tensorflow.python.ops import gen_nn_ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.platform import test
-from tensorflow.python.util import nest
-
-
-def _single_op_at_end():
-  inputs = keras.Input(shape=(10,))
-  x = keras.layers.Dense(10)(inputs)
-  outputs = gen_nn_ops.relu(x, name='hey')
-  return inputs, outputs
-
-
-def _multiple_ops_at_end():
-  inputs = keras.Input(shape=(10,))
-  inputs = keras.Input(shape=(10,))
-  x = keras.layers.Dense(10)(inputs)
-  x = gen_nn_ops.relu(x, name='hey')
-  outputs = gen_nn_ops.relu(x, name='hey2')
-  return inputs, outputs
-
-
-def _single_op_in_middle():
-  inputs = keras.Input(shape=(10,))
-  x = keras.layers.Dense(10)(inputs)
-  x = gen_nn_ops.relu(x, name='hey')
-  outputs = keras.layers.Dense(10)(x)
-  return inputs, outputs
-
-
-def _multiple_ops_in_middle():
-  inputs = keras.Input(shape=(10,))
-  x = keras.layers.Dense(10)(inputs)
-  x = gen_nn_ops.relu(x, name='hey')
-  x = gen_nn_ops.relu(x, name='hey2')
-  outputs = keras.layers.Dense(10)(x)
-  return inputs, outputs
-
-
-def _single_standalone_branch():
-  inputs = keras.Input(shape=(10,))
-  x = keras.layers.Dense(10)(inputs)
-  outputs = x * 2
-  return inputs, outputs
-
-
-def _single_op_with_attrs():
-  inputs = keras.Input(shape=(10,))
-  x = math_ops.reduce_mean(inputs, axis=1, keepdims=True)
-  outputs = keras.layers.Dense(10)(x)
-  return inputs, outputs
-
-
-@keras_parameterized.run_all_keras_modes
-class AutoLambdaTest(keras_parameterized.TestCase):
-
-  @parameterized.named_parameters(
-      ('single_op_at_end', _single_op_at_end),
-      ('multiple_ops_at_end', _multiple_ops_at_end),
-      ('single_op_in_middle', _single_op_in_middle),
-      ('multiple_ops_in_middle', _multiple_ops_in_middle),
-      ('single_standalone_branch', _single_standalone_branch),
-      ('single_op_with_attrs', _single_op_with_attrs))
-  def test_autolambda(self, model_fn):
-    inputs, outputs = model_fn()
-    model = keras.Model(inputs, outputs)
-    model.compile(
-        adam.Adam(0.001), 'mse', run_eagerly=testing_utils.should_run_eagerly())
-
-    np_inputs = nest.map_structure(lambda x: np.ones((10, 10), 'float32'),
-                                   inputs)
-    np_outputs = nest.map_structure(lambda x: np.ones((10, 10), 'float32'),
-                                    outputs)
-    model.fit(np_inputs, np_outputs, batch_size=2)
-
-  def test_numerical_correctness_simple(self):
-    x = ops.convert_to_tensor([[-1., 0., -2., 1.]])
-    inputs = keras.Input(shape=(4,))
-    outputs = gen_nn_ops.relu(inputs)
-    model = keras.Model(inputs, outputs)
-    y = self.evaluate(model(x))
-    self.assertAllClose(y, [[0., 0., 0., 1.]])
-
-  def test_numerical_correctness_with_attrs(self):
-    x = ops.convert_to_tensor([[1.5, 1.5], [2.5, 3.5]])
-    inputs = keras.Input(shape=(10,))
-    outputs = math_ops.reduce_mean(inputs, axis=1)
-    model = keras.Model(inputs, outputs)
-    y = self.evaluate(model(x))
-    self.assertAllClose(y, [1.5, 3.])
-
-  def test_serialization(self):
-    x = ops.convert_to_tensor([-1., 0., -2., 1.])
-    inputs = keras.Input(shape=(4,))
-    outputs = gen_nn_ops.relu(inputs)
-    model1 = keras.Model(inputs, outputs)
-    y1 = self.evaluate(model1(x))
-    model2 = model1.from_config(model1.get_config())
-    y2 = self.evaluate(model2(x))
-    self.assertAllClose(y1, y2)
-
-
-if __name__ == '__main__':
-  test.main()
-- 
GitLab


From abda3a18a5a646ba1223ba158afa8f276eeaec4d Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Thu, 24 Jan 2019 12:22:50 -0800
Subject: [PATCH 1200/2345] save optimizer weight names through util function
 to chunk data if necessary.

PiperOrigin-RevId: 230768372
---
 tensorflow/python/keras/BUILD                 |  1 +
 tensorflow/python/keras/saving/hdf5_format.py | 66 ++++++++++++-------
 .../python/keras/saving/hdf5_format_test.py   |  1 -
 3 files changed, 42 insertions(+), 26 deletions(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 460340b392..e879254dae 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -1105,6 +1105,7 @@ tf_py_test(
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
     ],
+    shard_count = 4,
 )
 
 tf_py_test(
diff --git a/tensorflow/python/keras/saving/hdf5_format.py b/tensorflow/python/keras/saving/hdf5_format.py
index cb925c30a6..29fb6a4752 100644
--- a/tensorflow/python/keras/saving/hdf5_format.py
+++ b/tensorflow/python/keras/saving/hdf5_format.py
@@ -135,23 +135,7 @@ def save_model(model, filepath, overwrite=True, include_optimizer=True):
             default=serialization.get_json_type).encode('utf8')
 
         # Save optimizer weights.
-        symbolic_weights = getattr(model.optimizer, 'weights')
-        if symbolic_weights:
-          optimizer_weights_group = f.create_group('optimizer_weights')
-          weight_values = K.batch_get_value(symbolic_weights)
-          weight_names = []
-          for w, val in zip(symbolic_weights, weight_values):
-            name = str(w.name)
-            weight_names.append(name.encode('utf8'))
-          optimizer_weights_group.attrs['weight_names'] = weight_names
-          for name, val in zip(weight_names, weight_values):
-            param_dset = optimizer_weights_group.create_dataset(
-                name, val.shape, dtype=val.dtype)
-            if not val.shape:
-              # scalar
-              param_dset[()] = val
-            else:
-              param_dset[:] = val
+        save_optimizer_weights_to_hdf5_group(f, model.optimizer)
     f.flush()
   finally:
     if opened_new_file:
@@ -271,14 +255,7 @@ def load_model(filepath, custom_objects=None, compile=True):  # pylint: disable=
         # weights.
         if model._is_graph_network:  # pylint: disable=protected-access
           model._make_train_function()
-          optimizer_weights_group = f['optimizer_weights']
-          optimizer_weight_names = [
-              n.decode('utf8')
-              for n in optimizer_weights_group.attrs['weight_names']
-          ]
-          optimizer_weight_values = [
-              optimizer_weights_group[n] for n in optimizer_weight_names
-          ]
+          optimizer_weight_values = load_optimizer_weights_from_hdf5_group(f)
           try:
             model.optimizer.set_weights(optimizer_weight_values)
           except ValueError:
@@ -655,6 +632,45 @@ def _convert_rnn_weights(layer, weights):
   return weights
 
 
+def save_optimizer_weights_to_hdf5_group(hdf5_group, optimizer):
+  """Saves optimizer weights of a optimizer to a HDF5 group.
+
+  Arguments:
+      hdf5_group: HDF5 group.
+      optimizer: optimizer instance.
+  """
+
+  symbolic_weights = getattr(optimizer, 'weights')
+  if symbolic_weights:
+    weights_group = hdf5_group.create_group('optimizer_weights')
+    weight_names = [str(w.name).encode('utf8') for w in symbolic_weights]
+    save_attributes_to_hdf5_group(weights_group, 'weight_names', weight_names)
+    weight_values = K.batch_get_value(symbolic_weights)
+    for name, val in zip(weight_names, weight_values):
+      param_dset = weights_group.create_dataset(
+          name, val.shape, dtype=val.dtype)
+      if not val.shape:
+        # scalar
+        param_dset[()] = val
+      else:
+        param_dset[:] = val
+
+
+def load_optimizer_weights_from_hdf5_group(hdf5_group):
+  """Load optimizer weights from a HDF5 group.
+
+  Arguments:
+      hdf5_group: A pointer to a HDF5 group.
+
+  Returns:
+      data: List of optimizer weight names.
+  """
+  weights_group = hdf5_group['optimizer_weights']
+  optimizer_weight_names = load_attributes_from_hdf5_group(
+      weights_group, 'weight_names')
+  return [weights_group[weight_name] for weight_name in optimizer_weight_names]
+
+
 def save_weights_to_hdf5_group(f, layers):
   """Saves the weights of a list of layers to a HDF5 group.
 
diff --git a/tensorflow/python/keras/saving/hdf5_format_test.py b/tensorflow/python/keras/saving/hdf5_format_test.py
index 3b496e70c1..a0c0ed3847 100644
--- a/tensorflow/python/keras/saving/hdf5_format_test.py
+++ b/tensorflow/python/keras/saving/hdf5_format_test.py
@@ -640,7 +640,6 @@ class TestWholeModelSaving(test.TestCase):
       os.remove(fname)
 
   def test_saving_model_with_long_weights_names(self):
-    self.skipTest('b/120921503')
     if h5py is None:
       self.skipTest('h5py required to run this test')
 
-- 
GitLab


From 13d105a747f81e598978aa98039faf01d7848818 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 24 Jan 2019 12:22:52 -0800
Subject: [PATCH 1201/2345] [XLA] plumb replica_groups into local Python client
 CRS

PiperOrigin-RevId: 230768381
---
 .../xla/python/local_computation_builder.cc   |  5 ++--
 .../xla/python/local_computation_builder.h    |  3 ++-
 .../xla/python/local_computation_builder.i    | 26 +++++++++++++++++++
 tensorflow/compiler/xla/python/xla_client.py  | 20 +++++++++++---
 .../compiler/xla/python/xla_client_test.py    | 12 +++++++++
 5 files changed, 60 insertions(+), 6 deletions(-)

diff --git a/tensorflow/compiler/xla/python/local_computation_builder.cc b/tensorflow/compiler/xla/python/local_computation_builder.cc
index 3493e07f3c..0e898d494e 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.cc
+++ b/tensorflow/compiler/xla/python/local_computation_builder.cc
@@ -691,8 +691,9 @@ LocalOp LocalComputationBuilder::Collapse(const LocalOp& operand,
   return xla::Collapse(operand.op(), dimensions);
 }
 
-LocalOp LocalComputationBuilder::CrossReplicaSum(const LocalOp& operand) {
-  return xla::CrossReplicaSum(operand.op());
+LocalOp LocalComputationBuilder::CrossReplicaSum(
+    const LocalOp& operand, absl::Span<const ReplicaGroup> replica_groups) {
+  return xla::CrossReplicaSum(operand.op(), replica_groups);
 }
 
 LocalOp LocalComputationBuilder::Slice(const LocalOp& operand,
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.h b/tensorflow/compiler/xla/python/local_computation_builder.h
index 5da19dda62..6170567f9f 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.h
+++ b/tensorflow/compiler/xla/python/local_computation_builder.h
@@ -316,7 +316,8 @@ class LocalComputationBuilder {
 
   LocalOp Collapse(const LocalOp& operand, absl::Span<const int64> dimensions);
 
-  LocalOp CrossReplicaSum(const LocalOp& operand);
+  LocalOp CrossReplicaSum(const LocalOp& operand,
+                          absl::Span<const ReplicaGroup> replica_groups);
 
   LocalOp Slice(const LocalOp& operand, absl::Span<const int64> start_indices,
                 absl::Span<const int64> limit_indices,
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.i b/tensorflow/compiler/xla/python/local_computation_builder.i
index 7798f83b29..6a85ed62de 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.i
+++ b/tensorflow/compiler/xla/python/local_computation_builder.i
@@ -36,6 +36,7 @@ limitations under the License.
 //  DotDimensionNumbers proto          <-  corresponding Python proto
 //  GatherDimensionNumbers proto       <-  corresponding Python proto
 //  ScatterDimensionNumbers proto      <-  corresponding Python proto
+//  Span<ReplicaGroup proto>           <-  sequence of ReplicaGroup Python proto
 //
 // Arrows indicate whether a conversion only ever occurs in one
 // direction, or whether it is maintained bidirectionally.
@@ -871,6 +872,31 @@ tensorflow::ImportNumpy();
   $1 = &dimension_numbers;
 }
 
+// Span<const ReplicaGroup>
+
+%typemap(in) absl::Span<const ReplicaGroup >
+    (std::vector<ReplicaGroup > temps) {
+  if (!PySequence_Check($input)) {
+    PyErr_SetString(PyExc_TypeError, "Argument is not a sequence");
+    SWIG_fail;
+  }
+  const int size = PySequence_Size($input);
+  temps.reserve(size);
+  for (int i = 0; i < size; ++i) {
+    PyObject* o = PySequence_GetItem($input, i);
+    ReplicaGroup rgrp;
+    if (!HandleRepeatedInt64Attribute(
+            o, "replica_ids",
+            rgrp.mutable_replica_ids())) {
+        SWIG_fail;
+    }
+    temps.push_back(rgrp);
+    Py_DECREF(o);
+  }
+  $1 = temps;
+}
+
+
 // ExecutableBuildOptions
 
 %typemap(in) const ExecutableBuildOptions*
diff --git a/tensorflow/compiler/xla/python/xla_client.py b/tensorflow/compiler/xla/python/xla_client.py
index e1fefe1a35..8964b15829 100644
--- a/tensorflow/compiler/xla/python/xla_client.py
+++ b/tensorflow/compiler/xla/python/xla_client.py
@@ -965,16 +965,30 @@ class ComputationBuilder(object):
       dimensions = tuple(range(ndim))
     return self._client.Reshape(operand, dimensions, new_sizes)
 
-  def CrossReplicaSum(self, operand):
+  def CrossReplicaSum(self, operand, replica_groups=None):
     """CrossReplicaSum op.
 
     Args:
       operand: the operand to sum across replica instances.
+      replica_groups: optional, list of lists of ints encoding a partition of
+        the set {0, 1, ..., num_replicas} into equally-sized replica groups
+        within which the cross-replica sum is performed. If not supplied or None
+        (the default), all replicas belong to the same group.
 
     Returns:
-      A LocalOp that has the sum of the value among all replicas.
+      A LocalOp that represents on each replica the sum of its group's values.
     """
-    return self._client.CrossReplicaSum(operand)
+
+    def make_proto(replica_group):
+      replica_group_proto = xla_data_pb2.ReplicaGroup()
+      replica_group_proto.replica_ids.extend(replica_group)
+      return replica_group_proto
+
+    if replica_groups is None:
+      replica_groups = []  # special value for XLA API
+    else:
+      replica_groups = [make_proto(group) for group in replica_groups]
+    return self._client.CrossReplicaSum(operand, replica_groups)
 
   def Collapse(self, operand, dimensions):
     """Collapse op."""
diff --git a/tensorflow/compiler/xla/python/xla_client_test.py b/tensorflow/compiler/xla/python/xla_client_test.py
index 874e087eb6..54c76241b9 100644
--- a/tensorflow/compiler/xla/python/xla_client_test.py
+++ b/tensorflow/compiler/xla/python/xla_client_test.py
@@ -565,6 +565,18 @@ class SingleOpTest(LocalComputationTest):
       c.CrossReplicaSum(c.Constant(lhs))
       self._ExecuteAndCompareExact(c, expected=lhs)
 
+  def testCrossReplicaSumOneReplicaWithSingletonGroup(self):
+    samples = [
+        NumpyArrayF32(42.0),
+        NumpyArrayF32([97.0]),
+        NumpyArrayF32([64.0, 117.0]),
+        NumpyArrayF32([[2.0, 3.0], [4.0, 5.0]]),
+    ]
+    for lhs in samples:
+      c = self._NewComputation()
+      c.CrossReplicaSum(c.Constant(lhs), [[0]])
+      self._ExecuteAndCompareExact(c, expected=lhs)
+
   def testDotMatrixVectorF32(self):
     c = self._NewComputation()
     lhs = NumpyArrayF32([[2.0, 3.0], [4.0, 5.0]])
-- 
GitLab


From 27e50b8b77219555b233f57dc6bd0747d8d26e88 Mon Sep 17 00:00:00 2001
From: Bhavani Subramanian <bhavani1.subramanian@intel.com>
Date: Thu, 24 Jan 2019 13:20:03 -0800
Subject: [PATCH 1202/2345] Addressed review comments.

---
 tensorflow/core/kernels/mkl_conv_ops.cc       | 13 +++++++++----
 tensorflow/core/kernels/mkl_fused_ops_test.cc |  6 ++++--
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc
index 9869556a8f..9dd34aaa99 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_ops.cc
@@ -1356,8 +1356,11 @@ class MklConvOp : public OpKernel {
     stream(stream::kind::eager).submit(net).wait();
   }
 
-  inline bool IsFilterCacheEmpty(OpKernelContext* context)
-      SHARED_LOCKS_REQUIRED(mu_) {
+  // LOCKS_EXCLUDED annotation ensures that the lock (mu_) cannot
+  // be acquired before entering the function, since it is acquired
+  // inside the function.
+  inline bool IsFilterCacheEmpty(OpKernelContext* context) LOCKS_EXCLUDED(mu_) {
+    tf_shared_lock lock(mu_);
     const Tensor& cached_filter_data_tensor =
         *cached_filter_data_ptensor_.AccessTensor(context);
     return (cached_filter_data_tensor.NumElements() == 0);
@@ -1369,7 +1372,8 @@ class MklConvOp : public OpKernel {
                    const std::shared_ptr<ConvFwdPd>& conv_fwd_pd,
                    Tfilter* filter_data, const Tensor& filter_tensor,
                    MklDnnData<Tfilter>& filter, const memory::desc& filter_md)
-      EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+      LOCKS_EXCLUDED(mu_) {
+    mutex_lock lock(mu_);
     const Tensor& cached_filter_data_tensor =
         *cached_filter_data_ptensor_.AccessTensor(context);
 
@@ -1393,7 +1397,8 @@ class MklConvOp : public OpKernel {
 
   Tfilter* GetCachedFilter(OpKernelContext* context,
                            const memory::format& filter_mf)
-      SHARED_LOCKS_REQUIRED(mu_) {
+      LOCKS_EXCLUDED(mu_) {
+    tf_shared_lock lock(mu_);
     const Tensor& cached_filter_data =
         *cached_filter_data_ptensor_.AccessTensor(context);
     const Tensor& cached_filter_md =
diff --git a/tensorflow/core/kernels/mkl_fused_ops_test.cc b/tensorflow/core/kernels/mkl_fused_ops_test.cc
index 0fdec670b6..068f080576 100644
--- a/tensorflow/core/kernels/mkl_fused_ops_test.cc
+++ b/tensorflow/core/kernels/mkl_fused_ops_test.cc
@@ -437,8 +437,10 @@ class FilterCacheTest : public OpsTestBase {
     ConvMklToTF<T> conv_comp;
     conv_comp.ConvertAndCompare(dtype, output, output_layout, expected);
 
-    // Run kernel again to ensure cached data is being reused
-    // TODO: Check if data is actually cached and reused
+    // TODO(bhavanis): For now, we rely on internal performance tests to
+    // determine if filter data is being cached and reused.
+    // However, we still need to add a check here to determine if this is
+    // still the case by inspecting the contents of the persistent tensor.
     TF_ASSERT_OK(RunOpKernel());
 
     // Compare output to expected results
-- 
GitLab


From 331a1196b38253869d975e7380bac08e22db4cbe Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 24 Jan 2019 12:59:40 -0800
Subject: [PATCH 1203/2345] Fix flaky keras DNN correctness test.

PiperOrigin-RevId: 230774541
---
 tensorflow/contrib/distribute/python/BUILD    |  1 -
 .../python/keras_correctness_test_base.py     | 30 +++++++++----------
 2 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 03bc4b2c79..fb8732d591 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -679,7 +679,6 @@ distribute_py_test(
         "no_oss",  # TODO(b/117919883): Fix python error.
         "no_pip",
         "no_windows_gpu",
-        "notap",  # TODO(b/123097760): Fix and re-enable the test.
         "notsan",
     ],
     deps = [
diff --git a/tensorflow/contrib/distribute/python/keras_correctness_test_base.py b/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
index 0f45806a95..ed77e4fc74 100644
--- a/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
+++ b/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
@@ -259,19 +259,20 @@ def compare_results(results_with_ds, results_without_ds, distribution,
   """Compares results of model compiled with/without distribution strategy."""
 
   default_tolerance = 1e-5
-  tol_table = {}
-
-  if isinstance(distribution, (
-      mirrored_strategy.MirroredStrategy,
-      mirrored_strategy.CoreMirroredStrategy,
-      distribute_lib._DefaultDistributionStrategy)):  # pylint: disable=protected-access
-    # TODO(b/119257215): Weights are not exactly the same, so use larger
-    # tolerance for now. Predict should be related to weights.
-    tol_table = {
-        'weights_1': 1e-4,
-        'weights_2': 1e-4,
-        'predict_result_1': 1e-4,
-    }
+  relaxed_tolerance = 1e-4
+
+  def _get_compare_result_tolerance(key):
+    """Returns tolerance to compare results."""
+    # TODO(b/119257215): For MirroredStrategy, weights are not exactly the same,
+    # so use larger tolerance for now. Predict should be related to weights.
+    if (isinstance(distribution, (
+        mirrored_strategy.MirroredStrategy,
+        mirrored_strategy.CoreMirroredStrategy,
+        distribute_lib._DefaultDistributionStrategy)) and  # pylint: disable=protected-access
+        key.startswith(('weights_1', 'weights_2', 'predict_result'))):
+      return relaxed_tolerance
+
+    return default_tolerance
 
   for key in results_with_ds:
     if (key.startswith('training_history') and
@@ -281,8 +282,7 @@ def compare_results(results_with_ds, results_without_ds, distribution,
       # underlying bug is fixed.
       continue
 
-    tolerance = tol_table.get(key, default_tolerance)
-
+    tolerance = _get_compare_result_tolerance(key)
     testcase.assertAllClose(
         results_with_ds[key],
         results_without_ds[key],
-- 
GitLab


From ddd1093196160ba73be123bd186a6781ae95b35a Mon Sep 17 00:00:00 2001
From: Doe Hyun Yoon <dyoon@google.com>
Date: Thu, 24 Jan 2019 13:43:05 -0800
Subject: [PATCH 1204/2345] In static shape inference, run EvaluateNode() when
 dtypes are numbers. Currently, it's done only for integer types, but simple
 non-integer operations can be casted to integer type, and then used as shape
 input to reshape, or other ops.

PiperOrigin-RevId: 230782685
---
 .../core/grappler/costs/graph_properties.cc   | 37 +++++++++++++++++--
 1 file changed, 33 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/grappler/costs/graph_properties.cc b/tensorflow/core/grappler/costs/graph_properties.cc
index 59696f39f8..d0ac87c962 100644
--- a/tensorflow/core/grappler/costs/graph_properties.cc
+++ b/tensorflow/core/grappler/costs/graph_properties.cc
@@ -460,6 +460,34 @@ class TopoQueue {
   std::set<NodeAndId, OrderByIdAscending> queue_;
 };
 
+bool IsNumericType(const DataType dtype) {
+  static const gtl::FlatSet<DataType>* const kRealNumberTypes =
+      CHECK_NOTNULL((new gtl::FlatSet<DataType>{
+          // Floating point.
+          DT_BFLOAT16,
+          DT_HALF,
+          DT_FLOAT,
+          DT_DOUBLE,
+          // Int / UInt.
+          DT_INT8,
+          DT_INT16,
+          DT_INT32,
+          DT_INT64,
+          DT_UINT8,
+          DT_UINT16,
+          DT_UINT32,
+          DT_UINT64,
+          // Quantized Int.
+          DT_QINT8,
+          DT_QUINT8,
+          DT_QINT16,
+          DT_QINT32,
+          // Bool.
+          DT_BOOL,
+      }));
+  return kRealNumberTypes->find(dtype) != kRealNumberTypes->end();
+}
+
 bool IsWhiteListedOpTypeForEvaluateNode(const string& op_type) {
   static const gtl::FlatSet<string>* const kOpTpeWhitelist =
       CHECK_NOTNULL((new gtl::FlatSet<string>{
@@ -504,6 +532,7 @@ bool IsWhiteListedOpTypeForEvaluateNode(const string& op_type) {
           "Split",
           "Range",
           "Fill",
+          "Cast",
       }));
   return kOpTpeWhitelist->find(op_type) != kOpTpeWhitelist->end();
 }
@@ -1120,16 +1149,16 @@ class SymbolicShapeRefiner {
       return false;
     }
 
-    // Check input dtypes are integer.
+    // Check input dtypes are number types.
     for (const auto& input_type : c->input_types) {
-      if (input_type != DT_INT32 && input_type != DT_INT64) {
+      if (!IsNumericType(input_type)) {
         return false;
       }
     }
 
-    // Check output dtypes are integer.
+    // Check output dtypes are number types.
     for (const auto& output_type : c->output_types) {
-      if (output_type != DT_INT32 && output_type != DT_INT64) {
+      if (!IsNumericType(output_type)) {
         return false;
       }
     }
-- 
GitLab


From 1a385af599ced4cf8bd641f5f7f0df3a2726510a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 24 Jan 2019 13:48:32 -0800
Subject: [PATCH 1205/2345] [XLA] Do not clear layouts for instructions in
 custom fusion nodes.

PiperOrigin-RevId: 230783684
---
 tensorflow/compiler/xla/service/layout_assignment.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/layout_assignment.cc b/tensorflow/compiler/xla/service/layout_assignment.cc
index 10ff7bb6d4..5d9e3392fd 100644
--- a/tensorflow/compiler/xla/service/layout_assignment.cc
+++ b/tensorflow/compiler/xla/service/layout_assignment.cc
@@ -1585,8 +1585,9 @@ Status SetFusionLayouts(HloInstruction* fusion) {
           fused_instruction->mutable_shape()));
     } else if (fused_instruction->opcode() == HloOpcode::kInfeed) {
       // Nop; leave the infeed layout alone.
-    } else {
+    } else if (fusion->fusion_kind() != HloInstruction::FusionKind::kCustom) {
       // Other instructions don't have layouts inside of fusion nodes.
+      // But do not clear layouts for other instructions in custom fusion nodes.
       LayoutUtil::ClearLayout(fused_instruction->mutable_shape());
     }
   }
-- 
GitLab


From b55c3c530780ffac0964933414c3f7184ab939e6 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Thu, 24 Jan 2019 14:17:50 -0800
Subject: [PATCH 1206/2345] Set output datatypes based on what tensorflow is
 expecting

---
 .../tf2tensorrt/convert/convert_nodes.cc      | 38 +++++++++++--------
 .../tf2tensorrt/convert/convert_nodes.h       | 23 ++++++++---
 2 files changed, 40 insertions(+), 21 deletions(-)

diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
index c08582a42e..3183ba356d 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
@@ -946,17 +946,19 @@ Status Converter::AddInputTensor(const string& name, nvinfer1::DataType dtype,
 }
 
 Status Converter::RenameAndMarkOutputTensors(
-    const std::vector<std::pair<string, string>>& output_tensors) {
+    const std::vector<Converter::EngineOutputInfo>& output_tensors) {
   for (const auto& output : output_tensors) {
     TRT_TensorOrWeights tensor_or_weights;
-    TF_RETURN_IF_ERROR(GetTensorOrWeights(output.first, &tensor_or_weights));
+    TF_RETURN_IF_ERROR(
+        GetTensorOrWeights(output.source_tensor_name, &tensor_or_weights));
     if (!tensor_or_weights.is_tensor()) {
-      return errors::InvalidArgument("Output ", output.first,
+      return errors::InvalidArgument("Output ", output.source_tensor_name,
                                      " is weights not tensor");
     }
     nvinfer1::ITensor* tensor = tensor_or_weights.tensor();
     if (tensor == nullptr) {
-      return errors::NotFound("Output tensor not found: ", output.first);
+      return errors::NotFound("Output tensor not found: ",
+                              output.source_tensor_name);
     }
     // Check if this tensor has already been marked as an output.
     // ConvertIdentity can cause the same tensor to be repeated in
@@ -974,9 +976,12 @@ Status Converter::RenameAndMarkOutputTensors(
       MarkQuantizationRangesAsInferrable(tensor, layer->getOutput(0));
       tensor = layer->getOutput(0);
     }
-    tensor->setName(output.second.c_str());
-    VLOG(1) << "Marking output tensor " << output.first << ", as output tensor "
-            << output.second;
+    tensor->setName(output.dest_node_name.c_str());
+    // Set type after marking as output. TRT only supports setType for engine
+    // outputs and inputs (type is inferred otherwise).
+    tensor->setType(output.trt_dtype);
+    VLOG(1) << "Marking output TRT tensor " << output.source_tensor_name
+            << ", which feeds TF node " << output.dest_node_name;
     network()->markOutput(*tensor);
   }
   return Status::OK();
@@ -3560,9 +3565,9 @@ tensorflow::Status ConvertTopK(OpConverterParams* params) {
     op = nvinfer1::TopKOperation::kMAX;
     reducedAxes |= 1 << (nbDims - 1);
   } else {
-    return tensorflow::errors::Unimplemented(
-        "Operation: " + node_def.op() +
-        " not implemented, at: " + node_def.name());
+    return tensorflow::errors::Unimplemented("Operation: ", node_def.op(),
+                                             " not implemented, at: ",
+                                             node_def.name());
   }
 
   nvinfer1::ITopKLayer* layer = params->converter->network()->addTopK(
@@ -3571,9 +3576,6 @@ tensorflow::Status ConvertTopK(OpConverterParams* params) {
 
   nvinfer1::ITensor* output_value_tensor = layer->getOutput(0);
   nvinfer1::ITensor* output_indices_tensor = layer->getOutput(1);
-  // Tensor type for network output is not inferred. Indices should be INT32
-  // (default is float).
-  output_indices_tensor->setType(nvinfer1::DataType::kINT32);
   params->outputs->push_back(TRT_TensorOrWeights(output_value_tensor));
   params->outputs->push_back(TRT_TensorOrWeights(output_indices_tensor));
   return tensorflow::Status::OK();
@@ -3686,7 +3688,7 @@ tensorflow::Status ConvertGraphDefToEngine(
   // Build the network
   VLOG(1) << "Starting engine conversion ";
   Converter converter(trt_network.get(), precision_mode, use_calibration);
-  std::vector<std::pair<string, string>> output_tensors;
+  std::vector<Converter::EngineOutputInfo> output_tensors;
   // Graph nodes are already topologically sorted during construction
   for (const auto& node_def : gdef.node()) {
     string node_name = node_def.name();
@@ -3728,10 +3730,16 @@ tensorflow::Status ConvertGraphDefToEngine(
         return tensorflow::errors::InvalidArgument(
             "Failed to parse slot number from ", node_name);
       }
+      // Get output type that TensorFlow expects
+      TFAttrs attrs(node_def);
+      tensorflow::DataType tf_dtype = attrs.get<tensorflow::DataType>("T");
+      nvinfer1::DataType trt_dtype;
+      TF_RETURN_IF_ERROR(ConvertDType(tf_dtype, &trt_dtype));
       if (output_tensors.size() <= slot_number) {
         output_tensors.resize(slot_number + 1);
       }
-      output_tensors.at(slot_number) = {node_def.input(0), node_name};
+      output_tensors.at(slot_number) = {node_def.input(0), node_name,
+                                        trt_dtype};
     } else {
       VLOG(2) << "Converting node: " << node_def.name() << " , "
               << node_def.op();
diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
index aebc0ca38d..f4f499ce7b 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
@@ -415,13 +415,11 @@ class Converter {
   Status AddInputTensor(const string& name, nvinfer1::DataType dtype,
                         const nvinfer1::Dims& dims, int batch_size);
 
-  // Mark the tensors with names specified by output_tensors[i].first as output
-  // of the TRT network, and set their names in the TRT network as
-  // output_tensors[i].second. The tensor names (output_tensors[i].first) are
-  // standard TF tensor names, i.e. node names followed by output slot number
-  // (or just the node name if the tensor is the first output of the node).
+  struct EngineOutputInfo;
+  // Mark the tensors with names specified by source_tensor_name as output of
+  // the TRT network, and set their names in the TRT network as dest_node_name.
   Status RenameAndMarkOutputTensors(
-      const std::vector<std::pair<string, string>>& output_tensors);
+      const std::vector<EngineOutputInfo>& output_tensors);
 
   //////////////////////////////////////////////////////////////////////////////
   // Methods used by op converters to convert individual TF node and add layers
@@ -548,6 +546,19 @@ class Converter {
   friend class OpConverterTest;
 };
 
+// Used for Converter::RenameAndMarkOutputTensors()
+struct EngineOutputInfo {
+  // The TRT tensor name which produces the output.
+  string source_tensor_name;
+  // The TensorFlow node name which is receiving the output from the TRT engine.
+  // This should always be the Identity node created in
+  // ConvertSegmentToGraphDef.
+  string dest_node_name;
+  // Output type. TensorRT requires this to be explicitly set for engine
+  // outputs.
+  nvinfer1::DataType trt_dtype;
+};
+
 }  // namespace convert
 }  // namespace tensorrt
 }  // namespace tensorflow
-- 
GitLab


From ef4dc8a0225a897dd41c92795c23321d0bbd1a81 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Thu, 24 Jan 2019 13:48:45 -0800
Subject: [PATCH 1207/2345] [TF:XLA] Bump open source llvm revision to r352035

PiperOrigin-RevId: 230783722
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index b2814fcb31..1412372e7d 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -513,11 +513,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "eff37dd30ccd52a76ca32428b2d20663402a1f75b55a808aba56fa6aa9979895",
-        strip_prefix = "llvm-0c8614fd4d276a31d2b5907b1c75d0003251cbf0",
+        sha256 = "1b5bf8ea78d81e163ae1766efb935b767684705d281ed8908f255cbac776b4ef",
+        strip_prefix = "llvm-cc563b6c997204f8c5dd76319a50007cebf03b3a",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/0c8614fd4d276a31d2b5907b1c75d0003251cbf0.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/0c8614fd4d276a31d2b5907b1c75d0003251cbf0.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/cc563b6c997204f8c5dd76319a50007cebf03b3a.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/cc563b6c997204f8c5dd76319a50007cebf03b3a.tar.gz",
         ],
     )
 
-- 
GitLab


From d3c849152b8b2e8167695c76a2ce850218540cce Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 24 Jan 2019 13:51:05 -0800
Subject: [PATCH 1208/2345] Specify default initial value for implementing
 padding functionality. Embedding lookup for features with zero embedding IDs
 must return 0.

PiperOrigin-RevId: 230784097
---
 .../tpu/proto/optimization_parameters.proto   | 17 ++++++-
 tensorflow/contrib/tpu/python/ops/tpu_ops.py  | 23 +++++----
 ...embedding_optimization_parameters_utils.cc | 51 +++++++++++--------
 3 files changed, 60 insertions(+), 31 deletions(-)

diff --git a/tensorflow/contrib/tpu/proto/optimization_parameters.proto b/tensorflow/contrib/tpu/proto/optimization_parameters.proto
index ddb3b15ecd..bc50c613f3 100644
--- a/tensorflow/contrib/tpu/proto/optimization_parameters.proto
+++ b/tensorflow/contrib/tpu/proto/optimization_parameters.proto
@@ -215,7 +215,8 @@ message OptimizationParameters {
 }
 
 // Specification of an optimization algorithm's state variables (both the main
-// value vector and any extra accumulators, etc.).
+// value vector and any extra accumulators, etc.). This proto is only used
+// internally by the TPU software and is not exposed directly to the TF model.
 message StateVariableSpecification {
   // Parameter name for the state variable.
   string name = 1;
@@ -223,6 +224,20 @@ message StateVariableSpecification {
   // A normal state variable that should be saved and restored in checkpoints
   // and used as an input or output to non-debug TensorFlow ops.
   message UserDefined {
+    // For padding embedding rows, this field specifies the initial value to be
+    // used. Separate initial values need to be specified for the embeddings and
+    // any extra accumulators. The initial values should be specified so as to
+    // maintain two invariants during model training:
+    // (1) The embedding vector multiplied by zero returns a vector containing
+    //     all zeros. To maintain this invariant, the embedding values should
+    //     never be NaNs or +-infinity.
+    // (2) Repeatedly applying the optimizer using a gradient vector of all
+    //     zeros does not cause the embeddings or slot variables to become NaNs
+    //     or +-infinity.
+    // The padding row is looked up when no embedding IDs are present for a
+    // feature. The semantics of embedding lookup dictate that the output must
+    // be zero under this scenario.
+    double padding_initial_value = 1;
   }
 
   // A state variable that should be filled with a constant and normally hidden
diff --git a/tensorflow/contrib/tpu/python/ops/tpu_ops.py b/tensorflow/contrib/tpu/python/ops/tpu_ops.py
index 6a6eba282a..9260e7b8a8 100644
--- a/tensorflow/contrib/tpu/python/ops/tpu_ops.py
+++ b/tensorflow/contrib/tpu/python/ops/tpu_ops.py
@@ -217,13 +217,19 @@ if platform.system() != "Windows":
 
     Args:
       inputs: A TensorList of gradients with which to update embedding tables.
-        Contains one tensor per embedding table in the model.
+          This argument has the same length and shapes as the return value of
+          RecvTPUEmbeddingActivations, but contains gradients of the model's
+          loss with respect to the embedding activations. The embedding tables
+          are updated from these gradients via the optimizers specified in the
+          TPU embedding configuration given to tpu.initialize_system.
       config: Serialized TPUEmbeddingConfiguration proto.
-      learning_rates: A TensorList of float32 scalars, one for each embedding
-        table, containing the learning rates for each table when dynamic
-        learning rate is enabled through the OptimizationParameters in
-        TPUEmbeddingConfiguration. When the learning rate is constant, the list
-        should be empty (optional).
+      learning_rates: A TensorList of float32 scalars, one for each dynamic
+          learning rate tag: see the comments in
+          //third_party/tensorflow/contrib/tpu/proto/
+                                               optimization_parameters.proto.
+          Multiple tables can share the same dynamic learning rate tag as
+          specified in the configuration. If the learning rates for all tables
+          are constant, this list should be empty.
       name: A name for the operation (optional).
 
     Returns:
@@ -337,9 +343,8 @@ if platform.system() != "Windows":
     Args:
       sample_indices: A list of rank 1 Tensors specifying the training example
         to which the corresponding embedding_indices and aggregation_weights
-        values
-        belong. It corresponds to sp_ids.indices[:,0] in
-          embedding_lookup_sparse().
+        values belong. It corresponds to sp_ids.indices[:,0] in
+        embedding_lookup_sparse().
       embedding_indices: A list of rank 1 Tensors, indices into the embedding
         tables. It corresponds to sp_ids.values in embedding_lookup_sparse().
       aggregation_weights: A list of rank 1 Tensors containing per training
diff --git a/tensorflow/contrib/tpu/utils/tpu_embedding_optimization_parameters_utils.cc b/tensorflow/contrib/tpu/utils/tpu_embedding_optimization_parameters_utils.cc
index 76cb5531cd..d98e0b7a5e 100644
--- a/tensorflow/contrib/tpu/utils/tpu_embedding_optimization_parameters_utils.cc
+++ b/tensorflow/contrib/tpu/utils/tpu_embedding_optimization_parameters_utils.cc
@@ -134,12 +134,16 @@ Status GetGradientAccumulationSupport(OptimizationAlgorithm alg,
   }
 }
 namespace {
-// Make a normal state variable specification.
+// Make a normal state variable specification. Please refer to
+// //third_party/tensorflow/contrib/tpu/proto/optimization_parameters.proto
+// (StateVariableSpecification message) for instructions on how to set the
+// padding_initial_value field.
 StateVariableSpecification MakeStandardStateVariableSpecification(
-    const string& name) {
+    const string& name, double padding_initial_value) {
   StateVariableSpecification result;
   result.set_name(name);
-  result.mutable_user_defined();
+  result.mutable_user_defined()->set_padding_initial_value(
+      padding_initial_value);
   return result;
 }
 }  // namespace
@@ -149,14 +153,14 @@ Status GetOptimizationAlgorithmStateVariables(
     std::vector<StateVariableSpecification>* state_variables) {
   // The first parameter set is always the weights themselves.
   state_variables->push_back(
-      MakeStandardStateVariableSpecification("parameters"));
+      MakeStandardStateVariableSpecification("parameters", 0.0));
   // The order of the returned parameters needs to match the offsets used by
   // the algorithm implementations in test_util.cc and
   // address_handler_program_creator.cc.
   switch (alg) {
     case OptimizationAlgorithm::kAdagrad: {
       state_variables->push_back(
-          MakeStandardStateVariableSpecification("accumulators"));
+          MakeStandardStateVariableSpecification("accumulators", 0.1));
       break;
     }
     case OptimizationAlgorithm::kStochasticGradientDescent: {
@@ -165,53 +169,58 @@ Status GetOptimizationAlgorithmStateVariables(
     }
     case OptimizationAlgorithm::kFtrl: {
       state_variables->push_back(
-          MakeStandardStateVariableSpecification("accumulators"));
+          MakeStandardStateVariableSpecification("accumulators", 0.1));
       state_variables->push_back(
-          MakeStandardStateVariableSpecification("linears"));
+          MakeStandardStateVariableSpecification("linears", 0.0));
       break;
     }
     case OptimizationAlgorithm::kAdam: {
       state_variables->push_back(
-          MakeStandardStateVariableSpecification("momenta"));
+          MakeStandardStateVariableSpecification("momenta", 0.0));
       state_variables->push_back(
-          MakeStandardStateVariableSpecification("velocities"));
+          MakeStandardStateVariableSpecification("velocities", 0.0));
       break;
     }
     case OptimizationAlgorithm::kMomentum: {
       state_variables->push_back(
-          MakeStandardStateVariableSpecification("momenta"));
+          MakeStandardStateVariableSpecification("momenta", 0.0));
       break;
     }
     case OptimizationAlgorithm::kRmsProp: {
-      state_variables->push_back(MakeStandardStateVariableSpecification("ms"));
-      state_variables->push_back(MakeStandardStateVariableSpecification("mom"));
+      state_variables->push_back(
+          MakeStandardStateVariableSpecification("ms", 1.0));
+      state_variables->push_back(
+          MakeStandardStateVariableSpecification("mom", 0.0));
       break;
     }
     case OptimizationAlgorithm::kCenteredRmsProp: {
-      state_variables->push_back(MakeStandardStateVariableSpecification("ms"));
-      state_variables->push_back(MakeStandardStateVariableSpecification("mom"));
-      state_variables->push_back(MakeStandardStateVariableSpecification("mg"));
+      state_variables->push_back(
+          MakeStandardStateVariableSpecification("ms", 1.0));
+      state_variables->push_back(
+          MakeStandardStateVariableSpecification("mom", 0.0));
+      state_variables->push_back(
+          MakeStandardStateVariableSpecification("mg", 0.0));
       break;
     }
     case OptimizationAlgorithm::kMdlAdagradLight: {
       state_variables->push_back(
-          MakeStandardStateVariableSpecification("accumulators"));
+          MakeStandardStateVariableSpecification("accumulators", 0.1));
       state_variables->push_back(
-          MakeStandardStateVariableSpecification("weights"));
+          MakeStandardStateVariableSpecification("weights", 0.0));
       state_variables->push_back(
-          MakeStandardStateVariableSpecification("benefits"));
+          MakeStandardStateVariableSpecification("benefits", 0.0));
       break;
     }
     case OptimizationAlgorithm::kAdadelta: {
       state_variables->push_back(
-          MakeStandardStateVariableSpecification("accumulators"));
+          MakeStandardStateVariableSpecification("accumulators", 0.0));
       state_variables->push_back(
-          MakeStandardStateVariableSpecification("updates"));
+          MakeStandardStateVariableSpecification("updates", 0.0));
       break;
     }
     case OptimizationAlgorithm::kProximalAdagrad: {
       state_variables->push_back(
-          MakeStandardStateVariableSpecification("accumulators"));
+          MakeStandardStateVariableSpecification("accumulators", 0.1));
       break;
     }
     case OptimizationAlgorithm::PARAMETERS_NOT_SET: {
-- 
GitLab


From c91dd82d8a58e98af0a6a0d334d3779fca9e983c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 24 Jan 2019 13:51:08 -0800
Subject: [PATCH 1209/2345] [Grappler] Add a simple graph structure verifier.

Currently, the checks include simple structure verification like no duplicate node names, no illegal loops and node def and op registry matches. We will continue to add more checks.

PiperOrigin-RevId: 230784105
---
 tensorflow/core/BUILD                         |   1 +
 tensorflow/core/graph/validate.cc             |  13 ++
 tensorflow/core/graph/validate.h              |   3 +
 tensorflow/core/graph/validate_test.cc        |  31 +++++
 tensorflow/core/grappler/verifiers/BUILD      |  38 ++++++
 .../core/grappler/verifiers/graph_verifier.h  |   4 +-
 .../grappler/verifiers/structure_verifier.cc  |  45 +++++++
 .../grappler/verifiers/structure_verifier.h   |  43 +++++++
 .../verifiers/structure_verifier_test.cc      | 116 ++++++++++++++++++
 9 files changed, 293 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/core/grappler/verifiers/structure_verifier.cc
 create mode 100644 tensorflow/core/grappler/verifiers/structure_verifier.h
 create mode 100644 tensorflow/core/grappler/verifiers/structure_verifier_test.cc

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 87ac325387..7652f03a9f 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -2831,6 +2831,7 @@ tf_cuda_library(
         "//third_party/eigen3",
         "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/container:flat_hash_set",
+        "@com_google_absl//absl/strings",
     ],
 )
 
diff --git a/tensorflow/core/graph/validate.cc b/tensorflow/core/graph/validate.cc
index e44eb91d48..4487f738c8 100644
--- a/tensorflow/core/graph/validate.cc
+++ b/tensorflow/core/graph/validate.cc
@@ -15,6 +15,8 @@ limitations under the License.
 
 #include "tensorflow/core/graph/validate.h"
 
+#include "absl/container/flat_hash_set.h"
+#include "absl/strings/string_view.h"
 #include "tensorflow/core/framework/graph_def_util.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/node_def_util.h"
@@ -113,5 +115,16 @@ Status ValidateGraphHasNoCycle(const Graph& graph) {
   return Status::OK();
 }
 
+Status VerifyNoDuplicateNodeNames(const GraphDef& graph) {
+  absl::flat_hash_set<absl::string_view> nodes;
+  for (const auto& node : graph.node()) {
+    if (nodes.contains(node.name())) {
+      return errors::AlreadyExists("Node already exists: ", node.name());
+    }
+    nodes.insert(node.name());
+  }
+  return Status::OK();
+}
+
 }  // namespace graph
 }  // namespace tensorflow
diff --git a/tensorflow/core/graph/validate.h b/tensorflow/core/graph/validate.h
index 08879dca60..bfb3a25ac9 100644
--- a/tensorflow/core/graph/validate.h
+++ b/tensorflow/core/graph/validate.h
@@ -59,6 +59,9 @@ void GetOpListForValidation(
 // be less than the total node count.
 Status ValidateGraphHasNoCycle(const Graph& graph);
 
+// Returns OK if the graph has no duplicate node names.
+Status VerifyNoDuplicateNodeNames(const GraphDef& graph);
+
 }  // namespace graph
 }  // namespace tensorflow
 
diff --git a/tensorflow/core/graph/validate_test.cc b/tensorflow/core/graph/validate_test.cc
index d58cdc3c5b..713cfc7926 100644
--- a/tensorflow/core/graph/validate_test.cc
+++ b/tensorflow/core/graph/validate_test.cc
@@ -147,5 +147,36 @@ TEST(GetOpListForValidationTest, ShouldStripDocs) {
   EXPECT_TRUE(found_has_docs);
 }
 
+TEST(VerifyNoDuplicateNodeNames, NoDuplicateNodeNames) {
+  const string graph_def_str =
+      "node { name: 'A' op: 'FloatInput' }"
+      "node { name: 'B' op: 'Int32Input' }"
+      "node { "
+      "       name: 'C' op: 'Sum' "
+      "       attr { key: 'T' value { type: DT_FLOAT } }"
+      "       input: ['A', 'B'] "
+      "}";
+  GraphDef graph_def;
+  auto parser = protobuf::TextFormat::Parser();
+  CHECK(parser.MergeFromString(graph_def_str, &graph_def)) << graph_def_str;
+  TF_ASSERT_OK(graph::VerifyNoDuplicateNodeNames(graph_def));
+}
+
+TEST(VerifyNoDuplicateNodeNames, DuplicateNodeNames) {
+  const string graph_def_str =
+      "node { name: 'A' op: 'FloatInput' }"
+      "node { name: 'A' op: 'Int32Input' }"
+      "node { "
+      "       name: 'C' op: 'Sum' "
+      "       attr { key: 'T' value { type: DT_FLOAT } }"
+      "       input: ['A', 'A'] "
+      "}";
+  GraphDef graph_def;
+  auto parser = protobuf::TextFormat::Parser();
+  CHECK(parser.MergeFromString(graph_def_str, &graph_def)) << graph_def_str;
+  EXPECT_EQ(tensorflow::error::ALREADY_EXISTS,
+            graph::VerifyNoDuplicateNodeNames(graph_def).code());
+}
+
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/core/grappler/verifiers/BUILD b/tensorflow/core/grappler/verifiers/BUILD
index 00b1325176..9666e101fe 100644
--- a/tensorflow/core/grappler/verifiers/BUILD
+++ b/tensorflow/core/grappler/verifiers/BUILD
@@ -1,5 +1,7 @@
 licenses(["notice"])  # Apache 2.0
 
+load("//tensorflow:tensorflow.bzl", "tf_cc_test")
+
 cc_library(
     name = "graph_verifier",
     hdrs = [
@@ -7,7 +9,43 @@ cc_library(
     ],
     visibility = ["//visibility:public"],
     deps = [
+        "//tensorflow/core:graph",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+    ],
+)
+
+cc_library(
+    name = "structure_verifier",
+    srcs = ["structure_verifier.cc"],
+    hdrs = [
+        "structure_verifier.h",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":graph_verifier",
+        "//tensorflow/cc:ops",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:graph",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core/grappler/utils:topological_sort",
+    ],
+)
+
+tf_cc_test(
+    name = "structure_verifier_test",
+    srcs = ["structure_verifier_test.cc"],
+    deps = [
+        ":structure_verifier",
+        "//tensorflow/cc:cc_ops",
+        "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core/grappler:grappler_item",
+        "//tensorflow/core/grappler/inputs:trivial_test_graph_input_yielder",
+        "@com_google_absl//absl/strings",
     ],
 )
diff --git a/tensorflow/core/grappler/verifiers/graph_verifier.h b/tensorflow/core/grappler/verifiers/graph_verifier.h
index f5acd59266..10fd201ead 100644
--- a/tensorflow/core/grappler/verifiers/graph_verifier.h
+++ b/tensorflow/core/grappler/verifiers/graph_verifier.h
@@ -44,7 +44,9 @@ class GraphVerifier {
   virtual string name() const = 0;
 
   // Implement an algorithm to verify the specified graph.
-  virtual Status Verify(const GraphDef& graph, std::vector<string>* errors) = 0;
+  // The return value is a Status that represents a concatenation of Status of
+  // each verification step.
+  virtual Status Verify(const GraphDef& graph) = 0;
 };
 
 }  // end namespace grappler
diff --git a/tensorflow/core/grappler/verifiers/structure_verifier.cc b/tensorflow/core/grappler/verifiers/structure_verifier.cc
new file mode 100644
index 0000000000..ab0a2ed526
--- /dev/null
+++ b/tensorflow/core/grappler/verifiers/structure_verifier.cc
@@ -0,0 +1,45 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/grappler/verifiers/structure_verifier.h"
+
+#include <string>
+#include <vector>
+
+#include "tensorflow/core/framework/graph.pb.h"
+#include "tensorflow/core/framework/node_def.pb.h"
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/graph/validate.h"
+#include "tensorflow/core/grappler/utils/topological_sort.h"
+#include "tensorflow/core/grappler/verifiers/graph_verifier.h"
+#include "tensorflow/core/lib/core/status.h"
+
+namespace tensorflow {
+namespace grappler {
+
+// TODO(ashwinm): Expand this to add more structural checks.
+Status StructureVerifier::Verify(const GraphDef& graph) {
+  StatusGroup status_group;
+  status_group.Update(tensorflow::graph::ValidateGraphDefAgainstOpRegistry(
+      graph, *OpRegistry::Global()));
+  status_group.Update(tensorflow::graph::VerifyNoDuplicateNodeNames(graph));
+
+  std::vector<const NodeDef*> topo_order;
+  status_group.Update(ComputeTopologicalOrder(graph, &topo_order));
+  return status_group.as_status();
+}
+
+}  // end namespace grappler
+}  // end namespace tensorflow
diff --git a/tensorflow/core/grappler/verifiers/structure_verifier.h b/tensorflow/core/grappler/verifiers/structure_verifier.h
new file mode 100644
index 0000000000..ab719f1214
--- /dev/null
+++ b/tensorflow/core/grappler/verifiers/structure_verifier.h
@@ -0,0 +1,43 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_GRAPPLER_VERIFIERS_STRUCTURE_VERIFIER_H_
+#define TENSORFLOW_CORE_GRAPPLER_VERIFIERS_STRUCTURE_VERIFIER_H_
+
+#include <string>
+#include <vector>
+
+#include "tensorflow/core/framework/graph.pb.h"
+#include "tensorflow/core/grappler/verifiers/graph_verifier.h"
+#include "tensorflow/core/lib/core/status.h"
+
+namespace tensorflow {
+namespace grappler {
+
+// Verifies the structure of a graph to ensure it is valid.
+class StructureVerifier : public GraphVerifier {
+ public:
+  StructureVerifier() {}
+  ~StructureVerifier() override {}
+
+  string name() const override { return "structure_verifier"; };
+
+  Status Verify(const GraphDef& graph) override;
+};
+
+}  // end namespace grappler
+}  // end namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_GRAPPLER_VERIFIERS_STRUCTURE_VERIFIER_H_
diff --git a/tensorflow/core/grappler/verifiers/structure_verifier_test.cc b/tensorflow/core/grappler/verifiers/structure_verifier_test.cc
new file mode 100644
index 0000000000..b0a2e4cd54
--- /dev/null
+++ b/tensorflow/core/grappler/verifiers/structure_verifier_test.cc
@@ -0,0 +1,116 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <memory>
+
+#include "tensorflow/cc/ops/parsing_ops.h"
+#include "tensorflow/cc/ops/standard_ops.h"
+#include "tensorflow/core/framework/graph.pb.h"
+#include "tensorflow/core/framework/shape_inference.h"
+#include "tensorflow/core/grappler/verifiers/structure_verifier.h"
+#include "tensorflow/core/lib/core/error_codes.pb.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
+#include "tensorflow/core/platform/protobuf.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+namespace grappler {
+namespace {
+
+class StructureVerifierTest : public ::testing::Test {
+ protected:
+  StructureVerifierTest() { verifier_.reset(new StructureVerifier()); }
+  void SetGraph(const string& gdef_ascii) {
+    CHECK(protobuf::TextFormat::ParseFromString(gdef_ascii, &graph_));
+  }
+  GraphDef graph_;
+  std::unique_ptr<StructureVerifier> verifier_;
+};
+
+Status Scalars(shape_inference::InferenceContext* c) {
+  for (int i = 0; i < c->num_outputs(); ++i) {
+    c->set_output(i, c->Scalar());
+  }
+  return Status::OK();
+}
+
+REGISTER_OP("TestParams").Output("o: float").SetShapeFn(Scalars);
+REGISTER_OP("TestInput")
+    .Output("a: float")
+    .Output("b: float")
+    .SetShapeFn(Scalars);
+REGISTER_OP("TestMul")
+    .Input("a: float")
+    .Input("b: float")
+    .Output("o: float")
+    .SetShapeFn(Scalars);
+
+TEST_F(StructureVerifierTest, ValidGraphs) {
+  // With scope, ops gets registered automatically.
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+  Output a = ops::Const(s.WithOpName("a"), 0.0f, {10, 10});
+  ops::ShapeN b(s.WithOpName("b"), {a, a, a});
+
+  GraphDef graph;
+  TF_CHECK_OK(s.ToGraphDef(&graph));
+  TF_EXPECT_OK(verifier_->Verify(graph));
+
+  // With graphdef directly, relies on REGISTER_OP to register ops
+  SetGraph(
+      "node { name: 'W1' op: 'TestParams' }"
+      "node { name: 'input' op: 'TestInput' }"
+      "node { name: 't1' op: 'TestMul' input: [ 'W1', 'input:1' ] }");
+
+  TF_EXPECT_OK(verifier_->Verify(graph_));
+}
+
+TEST_F(StructureVerifierTest, OpNotRegistered) {
+  SetGraph(
+      "node { name: 'input' op: 'OpNotRegistered' }"
+      "node { name: 't1' op: 'TestMul' input: [ 'input:0', 't2' ] }"
+      "node { name: 't2' op: 'TestMul' input: [ 'input:1', 't1' ] }");
+  Status status = verifier_->Verify(graph_);
+  EXPECT_EQ(errors::Code::NOT_FOUND, status.code());
+  EXPECT_TRUE(tensorflow::str_util::StrContains(status.error_message(),
+                                                "Op type not registered"));
+}
+
+TEST_F(StructureVerifierTest, DuplicateNodeNames) {
+  SetGraph(
+      "node { name: 'A' op: 'TestParams' }"
+      "node { name: 'A' op: 'TestInput' }");
+  Status status = verifier_->Verify(graph_);
+  EXPECT_EQ(errors::Code::ALREADY_EXISTS, status.code());
+  EXPECT_TRUE(tensorflow::str_util::StrContains(status.error_message(),
+                                                "Node already exists:"));
+}
+
+TEST_F(StructureVerifierTest, GraphWithInvalidCycle) {
+  SetGraph(
+      "node { name: 'input' op: 'TestInput' }"
+      "node { name: 't1' op: 'TestMul' input: [ 'input:0', 't2' ] }"
+      "node { name: 't2' op: 'TestMul' input: [ 'input:1', 't1' ] }");
+  Status status = verifier_->Verify(graph_);
+  EXPECT_EQ(errors::Code::INVALID_ARGUMENT, status.code());
+  EXPECT_TRUE(tensorflow::str_util::StrContains(
+      status.error_message(),
+      "The graph couldn't be sorted in topological order"));
+}
+
+}  // namespace
+}  // namespace grappler
+}  // namespace tensorflow
-- 
GitLab


From 634eefa5c48c0b90cafaceb55ad00186e3ece6ce Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 24 Jan 2019 13:54:32 -0800
Subject: [PATCH 1210/2345] Refactoring transformer constructors to take a
 transformer.Context.

1. Make converter.EntityContext inherit from transformer.Context.
2. Move converter.Base.ctx to transformer.Base.ctx (replacing transformer.Base.transformer_ctx.)

PiperOrigin-RevId: 230784642
---
 tensorflow/python/autograph/core/converter.py | 22 ++++++-------
 .../autograph/pyct/common_transformers/anf.py | 22 ++++++-------
 .../pyct/common_transformers/anf_test.py      | 17 +++++-----
 .../pyct/static_analysis/activity_test.py     |  3 +-
 .../pyct/static_analysis/live_values.py       | 10 +++---
 .../pyct/static_analysis/live_values_test.py  | 11 ++++---
 .../pyct/static_analysis/liveness_test.py     |  5 +--
 .../reaching_definitions_test.py              |  5 +--
 .../pyct/static_analysis/type_info.py         |  4 +--
 .../pyct/static_analysis/type_info_test.py    | 11 ++++---
 .../python/autograph/pyct/transformer.py      | 31 +++++++++----------
 .../python/autograph/pyct/transformer_test.py | 19 ++++++------
 12 files changed, 82 insertions(+), 78 deletions(-)

diff --git a/tensorflow/python/autograph/core/converter.py b/tensorflow/python/autograph/core/converter.py
index e6d626f215..319d48c3e2 100644
--- a/tensorflow/python/autograph/core/converter.py
+++ b/tensorflow/python/autograph/core/converter.py
@@ -216,7 +216,8 @@ class ConversionOptions(object):
     """
 
     def as_qualified_name(o):
-      name = inspect_utils.getqualifiedname(ctx.info.namespace, o, max_depth=1)
+      name = inspect_utils.getqualifiedname(
+          ctx.info.namespace, o, max_depth=1)
       if not name:
         if isinstance(o, weakref.ref):
           # `o` might already be a weak reference, if this object was
@@ -349,7 +350,7 @@ class ProgramContext(object):
     self.dependency_cache[original_entity] = converted_ast
 
 
-class EntityContext(object):
+class EntityContext(transformer.Context):
   """Tracks the conversion of a single entity.
 
   This object is mutable, and is updated during conversion. Not thread safe.
@@ -361,8 +362,8 @@ class EntityContext(object):
   """
 
   def __init__(self, namer, entity_info, program_ctx):
+    super(EntityContext, self).__init__(entity_info)
     self.namer = namer
-    self.info = entity_info
     self.program = program_ctx
 
 
@@ -374,8 +375,7 @@ class Base(transformer.Base):
   """
 
   def __init__(self, ctx):
-    super(Base, self).__init__(ctx.info)
-    self.ctx = ctx  # Keeping this short because it's used frequently.
+    super(Base, self).__init__(ctx)
 
     self._used = False
     self._ast_depth = 0
@@ -475,13 +475,13 @@ def standard_analysis(node, context, is_initial=False):
   # TODO(mdan): Don't return a node because it's modified by reference.
   graphs = cfg.build(node)
   node = qual_names.resolve(node)
-  node = activity.resolve(node, context.info, None)
-  node = reaching_definitions.resolve(node, context.info, graphs, AnnotatedDef)
-  node = liveness.resolve(node, context.info, graphs)
-  node = live_values.resolve(node, context.info, config.PYTHON_LITERALS)
-  node = type_info.resolve(node, context.info)
+  node = activity.resolve(node, context, None)
+  node = reaching_definitions.resolve(node, context, graphs, AnnotatedDef)
+  node = liveness.resolve(node, context, graphs)
+  node = live_values.resolve(node, context, config.PYTHON_LITERALS)
+  node = type_info.resolve(node, context)
   # This second call allows resolving first-order class attributes.
-  node = live_values.resolve(node, context.info, config.PYTHON_LITERALS)
+  node = live_values.resolve(node, context, config.PYTHON_LITERALS)
   if is_initial:
     anno.dup(
         node,
diff --git a/tensorflow/python/autograph/pyct/common_transformers/anf.py b/tensorflow/python/autograph/pyct/common_transformers/anf.py
index 192621b1cd..246c26833f 100644
--- a/tensorflow/python/autograph/pyct/common_transformers/anf.py
+++ b/tensorflow/python/autograph/pyct/common_transformers/anf.py
@@ -36,10 +36,10 @@ from tensorflow.python.autograph.pyct import transformer
 class DummyGensym(object):
   """A dumb gensym that suffixes a stem by sequential numbers from 1000."""
 
-  def __init__(self, entity_info):
-    del entity_info
+  def __init__(self, ctx):
+    del ctx
     # A proper implementation needs to account for:
-    #   * entity_info.namespace
+    #   * ctx.info.namespace
     #   * all the symbols defined in the AST
     #   * the symbols generated so far
     self._idx = 0
@@ -68,19 +68,19 @@ class AnfTransformer(transformer.Base):
   # processing the `body` and the `orelse` need to be kept together with them,
   # and not accidentally lifted out of the `if`.
 
-  def __init__(self, entity_info, gensym_source=None):
+  def __init__(self, ctx, gensym_source=None):
     """Creates an ANF transformer.
 
     Args:
-      entity_info: transformer.EntityInfo
+      ctx: transformer.Context
       gensym_source: An optional object with the same interface as `DummyGensym`
         for generating unique names
     """
-    super(AnfTransformer, self).__init__(entity_info)
+    super(AnfTransformer, self).__init__(ctx)
     if gensym_source is None:
-      self._gensym = DummyGensym(entity_info)
+      self._gensym = DummyGensym(ctx)
     else:
-      self._gensym = gensym_source(entity_info)
+      self._gensym = gensym_source(ctx)
     self._pending_statements = []
 
   def _consume_pending_statements(self):
@@ -406,7 +406,7 @@ class AnfTransformer(transformer.Base):
     return node
 
 
-def transform(node, entity_info, gensym_source=None):
+def transform(node, ctx, gensym_source=None):
   """Converts the given node to A-normal form (ANF).
 
   The general idea of A-normal form: https://en.wikipedia.org/wiki/A-normal_form
@@ -416,9 +416,9 @@ def transform(node, entity_info, gensym_source=None):
 
   Args:
     node: The node to transform.
-    entity_info: transformer.EntityInfo.  TODO(mdan): What information does this
+    ctx: transformer.EntityInfo.  TODO(mdan): What information does this
       argument provide?
     gensym_source: An optional object with the same interface as `DummyGensym`
       for generating unique names.
   """
-  return AnfTransformer(entity_info, gensym_source=gensym_source).visit(node)
+  return AnfTransformer(ctx, gensym_source=gensym_source).visit(node)
diff --git a/tensorflow/python/autograph/pyct/common_transformers/anf_test.py b/tensorflow/python/autograph/pyct/common_transformers/anf_test.py
index 525d4886de..58663d21ff 100644
--- a/tensorflow/python/autograph/pyct/common_transformers/anf_test.py
+++ b/tensorflow/python/autograph/pyct/common_transformers/anf_test.py
@@ -30,10 +30,10 @@ from tensorflow.python.platform import test
 class DummyGensym(object):
   """A dumb gensym that suffixes a stem by sequential numbers from 1000."""
 
-  def __init__(self, entity_info):
-    del entity_info
+  def __init__(self, ctx):
+    del ctx
     # A proper implementation needs to account for:
-    #   * entity_info.namespace
+    #   * ctx.info.namespace
     #   * all the symbols defined in the AST
     #   * the symbols generated so far
     self._idx = 0
@@ -68,21 +68,22 @@ def exec_expected_result():
 
 class AnfTransformerTest(test.TestCase):
 
-  def _simple_source_info(self):
-    return transformer.EntityInfo(
+  def _simple_context(self):
+    entity_info = transformer.EntityInfo(
         source_code=None,
         source_file=None,
         namespace=None,
         arg_values=None,
         arg_types=None,
         owner_type=None)
+    return transformer.Context(entity_info)
 
   def test_basic(self):
     def test_function():
       a = 0
       return a
     node, _ = parser.parse_entity(test_function)
-    node = anf.transform(node.body[0], self._simple_source_info())
+    node = anf.transform(node.body[0], self._simple_context())
     result, _ = compiler.ast_to_object(node)
     self.assertEqual(test_function(), result.test_function())
 
@@ -100,7 +101,7 @@ class AnfTransformerTest(test.TestCase):
     exp_node, _ = parser.parse_entity(expected_fn)
     node, _ = parser.parse_entity(test_fn)
     node = anf.transform(
-        node, self._simple_source_info(), gensym_source=DummyGensym)
+        node, self._simple_context(), gensym_source=DummyGensym)
     exp_name = exp_node.body[0].name
     # Ignoring the function names in the result because they can't be
     # the same (because both functions have to exist in the same scope
@@ -109,7 +110,7 @@ class AnfTransformerTest(test.TestCase):
     self.assert_same_ast(exp_node, node)
     # Check that ANF is idempotent
     node_repeated = anf.transform(
-        node, self._simple_source_info(), gensym_source=DummyGensym)
+        node, self._simple_context(), gensym_source=DummyGensym)
     self.assert_same_ast(node_repeated, node)
 
   def test_binop_basic(self):
diff --git a/tensorflow/python/autograph/pyct/static_analysis/activity_test.py b/tensorflow/python/autograph/pyct/static_analysis/activity_test.py
index 997d9a8aff..595e95bed9 100644
--- a/tensorflow/python/autograph/pyct/static_analysis/activity_test.py
+++ b/tensorflow/python/autograph/pyct/static_analysis/activity_test.py
@@ -121,7 +121,8 @@ class ActivityAnalyzerTest(test.TestCase):
         arg_types=None,
         owner_type=None)
     node = qual_names.resolve(node)
-    node = activity.resolve(node, entity_info)
+    ctx = transformer.Context(entity_info)
+    node = activity.resolve(node, ctx)
     return node, entity_info
 
   def assertSymbolSetsAre(self, expected, actual, name):
diff --git a/tensorflow/python/autograph/pyct/static_analysis/live_values.py b/tensorflow/python/autograph/pyct/static_analysis/live_values.py
index 14504fbe1b..eca4571d38 100644
--- a/tensorflow/python/autograph/pyct/static_analysis/live_values.py
+++ b/tensorflow/python/autograph/pyct/static_analysis/live_values.py
@@ -40,7 +40,7 @@ class LiveValueResolver(transformer.Base):
   def visit_ClassDef(self, node):
     self.generic_visit(node)
     anno.setanno(
-        node, 'live_val', self.transformer_ctx.entity_info.namespace[node.name])
+        node, 'live_val', self.ctx.info.namespace[node.name])
     return node
 
   def visit_Name(self, node):
@@ -54,8 +54,8 @@ class LiveValueResolver(transformer.Base):
       if not is_defined:
         if node.id in self.literals:
           anno.setanno(node, 'live_val', self.literals[node.id])
-        elif node.id in self.transformer_ctx.entity_info.namespace:
-          obj = self.transformer_ctx.entity_info.namespace[node.id]
+        elif node.id in self.ctx.info.namespace:
+          obj = self.ctx.info.namespace[node.id]
           anno.setanno(node, 'live_val', obj)
           if hasattr(obj, '__name__'):
             anno.setanno(node, 'fqn', (obj.__name__,))
@@ -87,8 +87,8 @@ class LiveValueResolver(transformer.Base):
         def_, = defs
         # Note: param_of is a weakref.
         if def_.param_of and def_.param_of() is self.enclosing_entities[0]:
-          if node.id in self.transformer_ctx.entity_info.arg_values:
-            obj = self.transformer_ctx.entity_info.arg_values[node.id]
+          if node.id in self.ctx.info.arg_values:
+            obj = self.ctx.info.arg_values[node.id]
             anno.setanno(node, 'live_val', obj)
             anno.setanno(node, 'fqn', (obj.__class__.__name__,))
     return node
diff --git a/tensorflow/python/autograph/pyct/static_analysis/live_values_test.py b/tensorflow/python/autograph/pyct/static_analysis/live_values_test.py
index 882c380b78..a8d4e25e3c 100644
--- a/tensorflow/python/autograph/pyct/static_analysis/live_values_test.py
+++ b/tensorflow/python/autograph/pyct/static_analysis/live_values_test.py
@@ -51,12 +51,13 @@ class LiveValuesResolverTest(test.TestCase):
         owner_type=None)
     node = qual_names.resolve(node)
     graphs = cfg.build(node)
-    node = activity.resolve(node, entity_info)
-    node = reaching_definitions.resolve(node, entity_info, graphs,
+    ctx = transformer.Context(entity_info)
+    node = activity.resolve(node, ctx)
+    node = reaching_definitions.resolve(node, ctx, graphs,
                                         reaching_definitions.Definition)
-    node = live_values.resolve(node, entity_info, literals)
-    node = type_info.resolve(node, entity_info)
-    node = live_values.resolve(node, entity_info, literals)
+    node = live_values.resolve(node, ctx, literals)
+    node = type_info.resolve(node, ctx)
+    node = live_values.resolve(node, ctx, literals)
     return node
 
   def test_literals(self):
diff --git a/tensorflow/python/autograph/pyct/static_analysis/liveness_test.py b/tensorflow/python/autograph/pyct/static_analysis/liveness_test.py
index 4366808d49..9738c6d0b4 100644
--- a/tensorflow/python/autograph/pyct/static_analysis/liveness_test.py
+++ b/tensorflow/python/autograph/pyct/static_analysis/liveness_test.py
@@ -40,9 +40,10 @@ class LivenessTest(test.TestCase):
         arg_types=None,
         owner_type=None)
     node = qual_names.resolve(node)
-    node = activity.resolve(node, entity_info)
+    ctx = transformer.Context(entity_info)
+    node = activity.resolve(node, ctx)
     graphs = cfg.build(node)
-    liveness.resolve(node, entity_info, graphs)
+    liveness.resolve(node, ctx, graphs)
     return node
 
   def assertHasLiveOut(self, node, expected):
diff --git a/tensorflow/python/autograph/pyct/static_analysis/reaching_definitions_test.py b/tensorflow/python/autograph/pyct/static_analysis/reaching_definitions_test.py
index 8c0d518507..fd91721c74 100644
--- a/tensorflow/python/autograph/pyct/static_analysis/reaching_definitions_test.py
+++ b/tensorflow/python/autograph/pyct/static_analysis/reaching_definitions_test.py
@@ -40,9 +40,10 @@ class DefinitionInfoTest(test.TestCase):
         arg_types=None,
         owner_type=None)
     node = qual_names.resolve(node)
-    node = activity.resolve(node, entity_info)
+    ctx = transformer.Context(entity_info)
+    node = activity.resolve(node, ctx)
     graphs = cfg.build(node)
-    node = reaching_definitions.resolve(node, entity_info, graphs,
+    node = reaching_definitions.resolve(node, ctx, graphs,
                                         reaching_definitions.Definition)
     return node
 
diff --git a/tensorflow/python/autograph/pyct/static_analysis/type_info.py b/tensorflow/python/autograph/pyct/static_analysis/type_info.py
index 9230457b2e..370e180838 100644
--- a/tensorflow/python/autograph/pyct/static_analysis/type_info.py
+++ b/tensorflow/python/autograph/pyct/static_analysis/type_info.py
@@ -141,10 +141,10 @@ class TypeInfoResolver(transformer.Base):
     arg_name = str(qn)
     self.scope.setval(qn, arg_node)
     if (len(self.enclosing_entities) == 1 and
-        arg_name in self.transformer_ctx.entity_info.arg_types):
+        arg_name in self.ctx.info.arg_types):
       # Forge a node to hold the type information, so that method calls on
       # it can resolve the type.
-      type_string, type_obj = self.transformer_ctx.entity_info.arg_types[
+      type_string, type_obj = self.ctx.info.arg_types[
           arg_name]
       anno.setanno(arg_node, 'type', type_obj)
       anno.setanno(arg_node, 'type_fqn', tuple(type_string.split('.')))
diff --git a/tensorflow/python/autograph/pyct/static_analysis/type_info_test.py b/tensorflow/python/autograph/pyct/static_analysis/type_info_test.py
index 34ba3d2f13..e43812e6f1 100644
--- a/tensorflow/python/autograph/pyct/static_analysis/type_info_test.py
+++ b/tensorflow/python/autograph/pyct/static_analysis/type_info_test.py
@@ -72,12 +72,13 @@ class TypeInfoResolverTest(test.TestCase):
         owner_type=None)
     node = qual_names.resolve(node)
     graphs = cfg.build(node)
-    node = activity.resolve(node, entity_info)
-    node = reaching_definitions.resolve(node, entity_info, graphs,
+    ctx = transformer.Context(entity_info)
+    node = activity.resolve(node, ctx)
+    node = reaching_definitions.resolve(node, ctx, graphs,
                                         reaching_definitions.Definition)
-    node = live_values.resolve(node, entity_info, {})
-    node = type_info.resolve(node, entity_info)
-    node = live_values.resolve(node, entity_info, {})
+    node = live_values.resolve(node, ctx, {})
+    node = type_info.resolve(node, ctx)
+    node = live_values.resolve(node, ctx, {})
     return node
 
   def test_constructor_detection(self):
diff --git a/tensorflow/python/autograph/pyct/transformer.py b/tensorflow/python/autograph/pyct/transformer.py
index 8c019dd8f6..530efee122 100644
--- a/tensorflow/python/autograph/pyct/transformer.py
+++ b/tensorflow/python/autograph/pyct/transformer.py
@@ -40,13 +40,13 @@ class Context(object):
   This object is mutable, and is updated during conversion. Not thread safe.
 
   Attributes:
-    entity_info: EntityInfo, immutable.
+    info: EntityInfo, immutable.
     current_origin: origin_info.OriginInfo, holds the OriginInfo of the last
       AST node to be processed successfully. Useful for error handling.
   """
 
-  def __init__(self, entity_info):
-    self.entity_info = entity_info
+  def __init__(self, info):
+    self.info = info
     self.current_origin = None
 
 
@@ -217,19 +217,17 @@ class Base(gast.NodeTransformer):
 
   # TODO(mdan): Document all extra features.
 
-  def __init__(self, entity_info):
+  def __init__(self, ctx):
     """Initialize the transformer.
 
     Subclasses should call this.
 
     Args:
-      entity_info: An EntityInfo object.
+      ctx: A Context object.
     """
     self._lineno = 0
     self._col_offset = 0
-    # TODO(znado): make the constructor take a Context instead of an EntityInfo.
-    # TODO(mdan): Rename to self.ctx.
-    self.transformer_ctx = Context(entity_info)
+    self.ctx = ctx
     self._enclosing_entities = []
 
     # A stack that allows keeping mutable, scope-local state where scopes may be
@@ -463,7 +461,7 @@ class Base(gast.NodeTransformer):
     local_scope_size_at_entry = len(self._local_scope_state)
     processing_expr_node = False
 
-    parent_origin = self.transformer_ctx.current_origin
+    parent_origin = self.ctx.current_origin
     try:
       if isinstance(node, (gast.FunctionDef, gast.ClassDef, gast.Lambda)):
         did_enter_function = True
@@ -474,15 +472,14 @@ class Base(gast.NodeTransformer):
         self._enclosing_entities.append(node)
 
       if anno.hasanno(node, anno.Basic.ORIGIN):
-        self.transformer_ctx.current_origin = anno.getanno(node,
-                                                           anno.Basic.ORIGIN)
+        self.ctx.current_origin = anno.getanno(node, anno.Basic.ORIGIN)
 
       if processing_expr_node:
         entry_expr_value = node.value
 
       if not anno.hasanno(node, anno.Basic.SKIP_PROCESSING):
         result = super(Base, self).visit(node)
-      self.transformer_ctx.current_origin = parent_origin
+      self.ctx.current_origin = parent_origin
 
       # Adjust for consistency: replacing the value of an Expr with
       # an Assign node removes the need for the Expr node.
@@ -509,13 +506,13 @@ class Base(gast.NodeTransformer):
       return result
 
     except (ValueError, AttributeError, KeyError, NotImplementedError) as e:
-      if not self.transformer_ctx.current_origin:
+      if not self.ctx.current_origin:
         raise e
-      original_file_path = self.transformer_ctx.current_origin.loc.filename
-      original_line_number = self.transformer_ctx.current_origin.loc.lineno
-      original_col_offset = self.transformer_ctx.current_origin.loc.col_offset
+      original_file_path = self.ctx.current_origin.loc.filename
+      original_line_number = self.ctx.current_origin.loc.lineno
+      original_col_offset = self.ctx.current_origin.loc.col_offset
       original_source_line = (
-          self.transformer_ctx.current_origin.source_code_line)
+          self.ctx.current_origin.source_code_line)
       msg = '%s: %s.' % (e.__class__.__name__, str(e))
 
       # TODO(mdan): Avoid the printing of the original exception.
diff --git a/tensorflow/python/autograph/pyct/transformer_test.py b/tensorflow/python/autograph/pyct/transformer_test.py
index 0c68d2a764..d97c1f0766 100644
--- a/tensorflow/python/autograph/pyct/transformer_test.py
+++ b/tensorflow/python/autograph/pyct/transformer_test.py
@@ -28,14 +28,15 @@ from tensorflow.python.platform import test
 
 class TransformerTest(test.TestCase):
 
-  def _simple_source_info(self):
-    return transformer.EntityInfo(
+  def _simple_context(self):
+    entity_info = transformer.EntityInfo(
         source_code=None,
         source_file=None,
         namespace=None,
         arg_values=None,
         arg_types=None,
         owner_type=None)
+    return transformer.Context(entity_info)
 
   def test_entity_scope_tracking(self):
 
@@ -52,7 +53,7 @@ class TransformerTest(test.TestCase):
         anno.setanno(node, 'enclosing_entities', self.enclosing_entities)
         return self.generic_visit(node)
 
-    tr = TestTransformer(self._simple_source_info())
+    tr = TestTransformer(self._simple_context())
 
     def test_function():
       a = 0
@@ -126,7 +127,7 @@ class TransformerTest(test.TestCase):
         self.state[CondState].exit()
         return node
 
-    tr = TestTransformer(self._simple_source_info())
+    tr = TestTransformer(self._simple_context())
 
     def test_function(a):
       a = 1
@@ -192,7 +193,7 @@ class TransformerTest(test.TestCase):
       def visit_For(self, node):
         return self._annotate_result(node)
 
-    tr = TestTransformer(self._simple_source_info())
+    tr = TestTransformer(self._simple_context())
 
     def test_function(a):
       """Docstring."""
@@ -231,7 +232,7 @@ class TransformerTest(test.TestCase):
         self.exit_local_scope()
         return node
 
-    tr = TestTransformer(self._simple_source_info())
+    tr = TestTransformer(self._simple_context())
 
     def no_exit(a):
       if a > 0:
@@ -270,7 +271,7 @@ class TransformerTest(test.TestCase):
       z = y
       return z
 
-    tr = TestTransformer(self._simple_source_info())
+    tr = TestTransformer(self._simple_context())
 
     node, _ = parser.parse_entity(test_function)
     node = tr.visit(node)
@@ -301,7 +302,7 @@ class TransformerTest(test.TestCase):
       if x > 0:
         return x
 
-    tr = BrokenTransformer(self._simple_source_info())
+    tr = BrokenTransformer(self._simple_context())
 
     node, _ = parser.parse_entity(test_function)
     with self.assertRaises(ValueError) as cm:
@@ -332,7 +333,7 @@ class TransformerTest(test.TestCase):
       if x > 0:
         return x
 
-    tr = BrokenTransformer(self._simple_source_info())
+    tr = BrokenTransformer(self._simple_context())
 
     node, _ = parser.parse_entity(test_function)
     with self.assertRaises(ValueError) as cm:
-- 
GitLab


From a49ee4bb8eb63692ab5ba8eacedc1b22ac96f098 Mon Sep 17 00:00:00 2001
From: Doe Hyun Yoon <dyoon@google.com>
Date: Thu, 24 Jan 2019 13:57:17 -0800
Subject: [PATCH 1211/2345] Apply compute_memory_overlap to
 intermediate_memory_time.

PiperOrigin-RevId: 230785089
---
 .../grappler/costs/op_level_cost_estimator.cc |  6 +-
 .../costs/op_level_cost_estimator_test.cc     | 83 ++++++++++++++++++-
 2 files changed, 85 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
index 1da8a98498..59d20f1fb9 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
@@ -528,8 +528,10 @@ Costs OpLevelCostEstimator::PredictOpCountBasedCost(
                       device_info.intermediate_write_gb_per_sec)
           : 0;
 
-  Costs::NanoSeconds intermediate_memory_cost(intermediate_read_time +
-                                              intermediate_write_time);
+  Costs::NanoSeconds intermediate_memory_cost =
+      compute_memory_overlap_
+          ? std::max(intermediate_read_time, intermediate_write_time)
+          : (intermediate_read_time + intermediate_write_time);
   VLOG(1) << "Op:" << op_info.op() << " Size (KB):" << (total_io_bytes) / 1e3
           << " Intermediate Memory Time (ns):"
           << intermediate_memory_cost.count();
diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
index 6a9bf13b93..aa0fc9d6c2 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
@@ -28,6 +28,31 @@ namespace tensorflow {
 namespace grappler {
 
 namespace {
+
+// TODO(dyoon): Consider to use this Test class for all the test cases, and then
+// remove friend in the OpLevelCostEstimator class header.
+class TestOpLevelCostEstimator : public OpLevelCostEstimator {
+ public:
+  TestOpLevelCostEstimator() {
+    compute_memory_overlap_ = true;
+    device_info_ = DeviceInfo();
+  }
+  ~TestOpLevelCostEstimator() override {}
+
+  void SetDeviceInfo(const DeviceInfo& device_info) {
+    device_info_ = device_info;
+  }
+
+  void SetComputeMemoryOverlap(bool value) { compute_memory_overlap_ = value; }
+
+ protected:
+  DeviceInfo GetDeviceInfo(const DeviceProperties& device) const override {
+    return device_info_;
+  }
+
+  DeviceInfo device_info_;
+};
+
 // Wrangles the minimum number of proto fields to set up a matrix.
 void DescribeMatrix(int rows, int columns, OpInfo* op_info) {
   auto input = op_info->add_inputs();
@@ -945,7 +970,7 @@ TEST_F(OpLevelCostEstimatorTest, PredictMaxPoolGrad) {
   };
 
   {
-    // Typical 3xz3 window with 2x2 stride.
+    // Typical 3x3 window with 2x2 stride.
     auto costs = predict_max_pool_grad(10, 20, 384, 3, 2, "SAME");
     EXPECT_EQ(Costs::Duration(1996800), costs.execution_time);
     EXPECT_EQ(Costs::Duration(614400), costs.compute_time);
@@ -986,7 +1011,7 @@ TEST_F(OpLevelCostEstimatorTest, PredictAvgPool) {
   };
 
   {
-    // Typical 3xz3 window with 2x2 stride.
+    // Typical 3x3 window with 2x2 stride.
     auto costs = predict_avg_pool(10, 20, 384, 3, 2, "SAME");
     EXPECT_EQ(Costs::Duration(1113600), costs.execution_time);
     EXPECT_EQ(Costs::Duration(345600), costs.compute_time);
@@ -1208,5 +1233,59 @@ TEST_F(OpLevelCostEstimatorTest, MaybeGetMinimumShape) {
     ExpectTensorShape({10, 20}, y);
   }
 }
+
+TEST_F(OpLevelCostEstimatorTest, IntermediateRdWrBandwidth) {
+  TestOpLevelCostEstimator estimator;
+
+  // Compute limited.
+  estimator.SetDeviceInfo(DeviceInfo(/*gigaops=*/1,
+                                     /*gb_per_sec=*/1));
+  estimator.SetComputeMemoryOverlap(true);
+  auto cost = estimator.PredictCosts(
+      DescribeConvolution(16, 19, 19, 48, 48, 5, 5, 256));
+  EXPECT_EQ(Costs::Duration(3548774400), cost.execution_time);
+  EXPECT_EQ(cost.execution_time, cost.compute_time);
+
+  estimator.SetComputeMemoryOverlap(false);
+  cost = estimator.PredictCosts(
+      DescribeConvolution(16, 19, 19, 48, 48, 5, 5, 256));
+  EXPECT_EQ(Costs::Duration(3551112192), cost.execution_time);
+  EXPECT_EQ(cost.execution_time, cost.compute_time + cost.memory_time +
+                                     cost.intermediate_memory_time);
+
+  // Memory limited.
+  estimator.SetDeviceInfo(DeviceInfo(/*gigaops=*/99999,
+                                     /*gb_per_sec=*/1));
+  estimator.SetComputeMemoryOverlap(true);
+  cost = estimator.PredictCosts(
+      DescribeConvolution(16, 19, 19, 48, 48, 5, 5, 256));
+  EXPECT_EQ(Costs::Duration(2337792), cost.execution_time);
+  EXPECT_EQ(cost.execution_time, cost.memory_time);
+
+  estimator.SetComputeMemoryOverlap(false);
+  cost = estimator.PredictCosts(
+      DescribeConvolution(16, 19, 19, 48, 48, 5, 5, 256));
+  EXPECT_EQ(Costs::Duration(2373281), cost.execution_time);
+  EXPECT_EQ(cost.execution_time, cost.compute_time + cost.memory_time +
+                                     cost.intermediate_memory_time);
+
+  // Intermediate memory bandwidth limited.
+  estimator.SetDeviceInfo(DeviceInfo(/*gigaops=*/99999,
+                                     /*gb_per_sec=*/9999,
+                                     /*intermediate_read_gb_per_sec=*/1,
+                                     /*intermediate_write_gb_per_sec=*/1));
+  estimator.SetComputeMemoryOverlap(true);
+  cost = estimator.PredictCosts(
+      DescribeConvolution(16, 19, 19, 48, 48, 5, 5, 256));
+  EXPECT_EQ(Costs::Duration(2337792), cost.execution_time);
+  EXPECT_EQ(cost.execution_time, cost.intermediate_memory_time);
+
+  estimator.SetComputeMemoryOverlap(false);
+  cost = estimator.PredictCosts(
+      DescribeConvolution(16, 19, 19, 48, 48, 5, 5, 256));
+  EXPECT_EQ(Costs::Duration(2373515), cost.execution_time);
+  EXPECT_EQ(cost.execution_time, cost.compute_time + cost.memory_time +
+                                     cost.intermediate_memory_time);
+}
 }  // end namespace grappler
 }  // end namespace tensorflow
-- 
GitLab


From 9c7a5a7ecd791a6ee6487afedeaf6dbaf4b3d052 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Thu, 24 Jan 2019 14:22:52 -0800
Subject: [PATCH 1212/2345] [Grappler] Relax inlining constraints for
 ReadVariableOp

PiperOrigin-RevId: 230789677
---
 tensorflow/core/grappler/op_types.cc                      | 4 ++++
 tensorflow/core/grappler/op_types.h                       | 1 +
 tensorflow/core/grappler/optimizers/function_optimizer.cc | 8 +++++++-
 3 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc
index e9bf36ab29..27fba4fd11 100644
--- a/tensorflow/core/grappler/op_types.cc
+++ b/tensorflow/core/grappler/op_types.cc
@@ -355,6 +355,10 @@ bool IsRandomShuffle(const NodeDef& node) {
 
 bool IsRank(const NodeDef& node) { return node.op() == "Rank"; }
 
+bool IsReadVariableOp(const NodeDef& node) {
+  return node.op() == "ReadVariableOp";
+}
+
 bool IsReal(const NodeDef& node) { return node.op() == "Real"; }
 
 bool IsRealDiv(const NodeDef& node) { return node.op() == "RealDiv"; }
diff --git a/tensorflow/core/grappler/op_types.h b/tensorflow/core/grappler/op_types.h
index 4f48ec4767..a1ee253369 100644
--- a/tensorflow/core/grappler/op_types.h
+++ b/tensorflow/core/grappler/op_types.h
@@ -115,6 +115,7 @@ bool IsPow(const NodeDef& node);
 bool IsQueue(const NodeDef& node);
 bool IsRandomShuffle(const NodeDef& node);
 bool IsRank(const NodeDef& node);
+bool IsReadVariableOp(const NodeDef& node);
 bool IsReal(const NodeDef& node);
 bool IsRealDiv(const NodeDef& node);
 bool IsRelu(const NodeDef& node);
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index da8ad4a778..9581fce8d3 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -1429,7 +1429,13 @@ Status CheckThatSideEffectsWillExecute(
     const bool node_must_execute =
         IsDataset(func_body_node) ||
         IsStateful(func_body_node, &ctx.function_library());
-    if (!node_must_execute) continue;
+
+    // If op has DT_RESOURCE argument it will be marked as stateful, though if
+    // it only reads from that resource, it's allowed to prune it, because it
+    // can't produce any visible side-effects.
+    const bool read_only = IsReadVariableOp(func_body_node);
+
+    if (read_only || !node_must_execute) continue;
 
     VLOG(3) << "Check that node " << func_body_node.name()
             << " will execute after inlining.";
-- 
GitLab


From 09ac2f6d484811e0304a79baacb2e951bbf118ad Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Thu, 24 Jan 2019 14:34:13 -0800
Subject: [PATCH 1213/2345] Enable benchmarkScanDefun on GPU.

PiperOrigin-RevId: 230791598
---
 tensorflow/core/kernels/concat_lib_gpu.cc         | 1 +
 tensorflow/core/kernels/concat_lib_gpu_impl.cu.cc | 4 ++++
 tensorflow/core/kernels/list_kernels.cc           | 1 +
 tensorflow/core/kernels/list_kernels.cu.cc        | 1 +
 tensorflow/python/eager/benchmarks_test.py        | 4 ----
 5 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/kernels/concat_lib_gpu.cc b/tensorflow/core/kernels/concat_lib_gpu.cc
index 93e392d303..161810d8cf 100644
--- a/tensorflow/core/kernels/concat_lib_gpu.cc
+++ b/tensorflow/core/kernels/concat_lib_gpu.cc
@@ -115,6 +115,7 @@ void ConcatGPU(
 TF_CALL_GPU_NUMBER_TYPES(REGISTER);
 TF_CALL_complex64(REGISTER);
 TF_CALL_complex128(REGISTER);
+TF_CALL_int32(REGISTER);  // Needed for TensorLists.
 TF_CALL_int64(REGISTER);
 TF_CALL_bfloat16(REGISTER);
 TF_CALL_bool(REGISTER);
diff --git a/tensorflow/core/kernels/concat_lib_gpu_impl.cu.cc b/tensorflow/core/kernels/concat_lib_gpu_impl.cu.cc
index a561d918bd..1a9adfa731 100644
--- a/tensorflow/core/kernels/concat_lib_gpu_impl.cu.cc
+++ b/tensorflow/core/kernels/concat_lib_gpu_impl.cu.cc
@@ -201,6 +201,7 @@ void ConcatGPUImpl(const Eigen::GpuDevice& gpu_device,
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPUCONCAT32);
 TF_CALL_complex64(REGISTER_GPUCONCAT32);
 TF_CALL_complex128(REGISTER_GPUCONCAT32);
+TF_CALL_int32(REGISTER_GPUCONCAT32);  // Needed for TensorLists.
 TF_CALL_int64(REGISTER_GPUCONCAT32);
 TF_CALL_uint8(REGISTER_GPUCONCAT32);
 REGISTER_GPUCONCAT32(bfloat16);
@@ -209,6 +210,7 @@ REGISTER_GPUCONCAT32(bool);
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPUCONCAT64);
 TF_CALL_complex64(REGISTER_GPUCONCAT64);
 TF_CALL_complex128(REGISTER_GPUCONCAT64);
+TF_CALL_int32(REGISTER_GPUCONCAT64);  // Needed for TensorLists.
 TF_CALL_int64(REGISTER_GPUCONCAT64);
 TF_CALL_uint8(REGISTER_GPUCONCAT64);
 REGISTER_GPUCONCAT64(bfloat16);
@@ -217,6 +219,7 @@ REGISTER_GPUCONCAT64(bool);
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU32);
 TF_CALL_complex64(REGISTER_GPU32);
 TF_CALL_complex128(REGISTER_GPU32);
+TF_CALL_int32(REGISTER_GPU32);  // Needed for TensorLists.
 TF_CALL_int64(REGISTER_GPU32);
 TF_CALL_uint8(REGISTER_GPU32);
 REGISTER_GPU32(bfloat16);
@@ -225,6 +228,7 @@ REGISTER_GPU32(bool);
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU64);
 TF_CALL_complex64(REGISTER_GPU64);
 TF_CALL_complex128(REGISTER_GPU64);
+TF_CALL_int32(REGISTER_GPU64);  // Needed for TensorLists.
 TF_CALL_int64(REGISTER_GPU64);
 TF_CALL_uint8(REGISTER_GPU64);
 REGISTER_GPU64(bfloat16);
diff --git a/tensorflow/core/kernels/list_kernels.cc b/tensorflow/core/kernels/list_kernels.cc
index 7c0b70d651..ee157dfd18 100644
--- a/tensorflow/core/kernels/list_kernels.cc
+++ b/tensorflow/core/kernels/list_kernels.cc
@@ -530,6 +530,7 @@ REGISTER_KERNEL_BUILDER(Name("TensorListSetItem").Device(DEVICE_CPU),
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_SET_ITEM_GPU);
 TF_CALL_complex64(REGISTER_TENSOR_LIST_SET_ITEM_GPU);
 TF_CALL_complex128(REGISTER_TENSOR_LIST_SET_ITEM_GPU);
+TF_CALL_int32(REGISTER_TENSOR_LIST_SET_ITEM_GPU);
 TF_CALL_int64(REGISTER_TENSOR_LIST_SET_ITEM_GPU);
 REGISTER_TENSOR_LIST_SET_ITEM_GPU(bfloat16)
 #undef REGISTER_TENSOR_LIST_SET_ITEM_GPU
diff --git a/tensorflow/core/kernels/list_kernels.cu.cc b/tensorflow/core/kernels/list_kernels.cu.cc
index c3a5041b00..1d5d747610 100644
--- a/tensorflow/core/kernels/list_kernels.cu.cc
+++ b/tensorflow/core/kernels/list_kernels.cu.cc
@@ -90,6 +90,7 @@ TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_OPS_GPU);
 REGISTER_TENSOR_LIST_OPS_GPU(bfloat16);
 TF_CALL_complex64(REGISTER_TENSOR_LIST_OPS_GPU);
 TF_CALL_complex128(REGISTER_TENSOR_LIST_OPS_GPU);
+TF_CALL_int32(REGISTER_TENSOR_LIST_OPS_GPU);
 TF_CALL_int64(REGISTER_TENSOR_LIST_OPS_GPU);
 REGISTER_TENSOR_LIST_OPS_GPU(bool);
 
diff --git a/tensorflow/python/eager/benchmarks_test.py b/tensorflow/python/eager/benchmarks_test.py
index 62c4a12cbf..16b52c9baa 100644
--- a/tensorflow/python/eager/benchmarks_test.py
+++ b/tensorflow/python/eager/benchmarks_test.py
@@ -890,10 +890,6 @@ class MicroBenchmarks(test.Benchmark):
     self._run(scan, 100)
 
   def benchmarkScanDefun(self):
-    if context.num_gpus():
-      # TODO(b/122081934): Re-enable this after figuring out why this became
-      # really slow with control flow V2
-      return
     elems = math_ops.range(1600)
 
     @function.defun
-- 
GitLab


From c8a2c9eacd27d4334bd52e48a4a22cb507700942 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Thu, 24 Jan 2019 15:35:29 -0800
Subject: [PATCH 1214/2345] Use static names in ConversionOptions.to_ast,
 because they are stable and well-known. Properly recognize cases when
 conversion is applied to a function that already has autograph decorators.
 Make it a bit more obvious that converted_call is mocked in the tests that do
 it. Add even more logging.

PiperOrigin-RevId: 230802967
---
 tensorflow/python/autograph/core/converter.py | 15 ++++---
 .../python/autograph/core/converter_test.py   | 44 ++++++++++++++++++-
 .../autograph/core/converter_testing.py       | 18 +++++---
 .../python/autograph/impl/conversion.py       | 39 +++++++++-------
 4 files changed, 86 insertions(+), 30 deletions(-)

diff --git a/tensorflow/python/autograph/core/converter.py b/tensorflow/python/autograph/core/converter.py
index 319d48c3e2..b3297ef229 100644
--- a/tensorflow/python/autograph/core/converter.py
+++ b/tensorflow/python/autograph/core/converter.py
@@ -206,7 +206,7 @@ class ConversionOptions(object):
       ast.Node
     """
     template = """
-      constructor_name(
+      ag__.ConversionOptions(
           recursive=recursive_val,
           verbose=verbose_val,
           strip_decorators=strip_decorators_val,
@@ -235,17 +235,15 @@ class ConversionOptions(object):
 
     def list_of_features(values):
       return parser.parse_expression('({})'.format(', '.join(
-          'ag__.Feature.{}'.format(v)
-          for v in Feature.__members__
+          'ag__.{}'.format(v)
+          for v in Feature.__members__.values()
           if v in values)))
 
-    if internal_convert_user_code is not None:
+    if internal_convert_user_code is None:
       internal_convert_user_code = self.internal_convert_user_code
 
     expr_ast = templates.replace(
         template,
-        constructor_name=parser.parse_expression(
-            as_qualified_name(ConversionOptions)),
         recursive_val=parser.parse_expression(str(self.recursive)),
         verbose_val=parser.parse_expression(str(int(self.verbose))),
         strip_decorators_val=list_of_names(self._strip_decorators),
@@ -279,6 +277,11 @@ class ProgramContext(object):
     required_imports: str, containing an import statement on each line. These
       are all the imports necessary for the compiled code to run, in addition to
       the closures of each entity, which are attached dynamically.
+    partial_types: Tuple[Type], deprecated.
+    conversion_order: Tuple[Any], deprecated.
+    additional_symbols: Dict[str, Any], a map of new symbols that have been
+      created under this context, and need to be added to the namespace of the
+      generated code.
   """
 
   def __init__(
diff --git a/tensorflow/python/autograph/core/converter_test.py b/tensorflow/python/autograph/core/converter_test.py
index 864ea6c7d2..4050878b92 100644
--- a/tensorflow/python/autograph/core/converter_test.py
+++ b/tensorflow/python/autograph/core/converter_test.py
@@ -23,7 +23,10 @@ import weakref
 from tensorflow.python.autograph.core import converter
 from tensorflow.python.autograph.core import converter_testing
 from tensorflow.python.autograph.pyct import anno
+from tensorflow.python.autograph.pyct import compiler
 from tensorflow.python.autograph.pyct import parser
+from tensorflow.python.autograph.pyct import templates
+from tensorflow.python.autograph.pyct import transformer
 from tensorflow.python.platform import test
 
 
@@ -31,7 +34,46 @@ class TestConverter(converter.Base):
   pass
 
 
-class ConversionOptionsTest(test.TestCase):
+class ConversionOptionsTest(converter_testing.TestCase):
+
+  def test_to_ast(self):
+    opts = converter.ConversionOptions()
+
+    namer = converter_testing.FakeNamer()
+    program_ctx = converter.ProgramContext(
+        options=opts,
+        partial_types=None,
+        autograph_module=None,
+        uncompiled_modules=())
+    entity_info = transformer.EntityInfo(
+        source_code='',
+        source_file='<fragment>',
+        namespace={},
+        arg_values=None,
+        arg_types={},
+        owner_type=None)
+    ctx = converter.EntityContext(namer, entity_info, program_ctx)
+    opts_ast = opts.to_ast(ctx)
+
+    template = '''
+    def test_fn():
+      return opts_ast
+    '''
+    opts_packed = templates.replace(template, opts_ast=opts_ast)
+
+    reparsed, _ = compiler.ast_to_object(opts_packed)
+    reparsed.__dict__['ag__'] = self.make_fake_mod(
+        'fake_ag', converter.ConversionOptions, converter.Feature)
+
+    reparsed_opts = reparsed.test_fn()
+
+    self.assertEqual(opts.recursive, reparsed_opts.recursive)
+    self.assertEqual(opts.verbose, reparsed_opts.verbose)
+    self.assertEqual(opts.force_conversion, reparsed_opts.force_conversion)
+    self.assertEqual(
+        opts.internal_convert_user_code,
+        reparsed_opts.internal_convert_user_code)
+    self.assertEqual(opts.optional_features, reparsed_opts.optional_features)
 
   def test_should_strip_weakrefs(self):
     def test_fn():
diff --git a/tensorflow/python/autograph/core/converter_testing.py b/tensorflow/python/autograph/core/converter_testing.py
index f1374081d3..051495c24b 100644
--- a/tensorflow/python/autograph/core/converter_testing.py
+++ b/tensorflow/python/autograph/core/converter_testing.py
@@ -40,6 +40,9 @@ from tensorflow.python.autograph.pyct import transformer
 from tensorflow.python.platform import test
 
 
+RESULT_OF_MOCK_CONVERTED_CALL = 7
+
+
 def imported_decorator(f):
   return lambda a: f(a) + 1
 
@@ -50,6 +53,7 @@ class FakeNamer(object):
 
   def __init__(self):
     self.i = 0
+    self.partial_types = ()
 
   def new_symbol(self, name_root, used):
     while True:
@@ -95,8 +99,8 @@ class TestCase(test.TestCase):
     self.dynamic_calls = []
     def converted_call(*args):
       """Mock version of api.converted_call."""
-      self.dynamic_calls.append(args)
-      return 7
+      self.dynamic_calls.append(args[3:])  # args only; see api.converted_call
+      return RESULT_OF_MOCK_CONVERTED_CALL
 
     try:
       result, source = compiler.ast_to_object(node, include_source_map=True)
@@ -107,11 +111,13 @@ class TestCase(test.TestCase):
                                    converter.ConversionOptions)
       fake_ag.__dict__.update(operators.__dict__)
       fake_ag.__dict__.update(special_functions.__dict__)
-      fake_ag.__dict__['utils'] = utils
-      fake_ag.__dict__['rewrite_graph_construction_error'] = (
+      fake_ag.ConversionOptions = converter.ConversionOptions
+      fake_ag.Feature = converter.Feature
+      fake_ag.utils = utils
+      fake_ag.rewrite_graph_construction_error = (
           errors.rewrite_graph_construction_error)
-      fake_ag.__dict__['function_scope'] = function_wrapping.function_scope
-      result.__dict__['ag__'] = fake_ag
+      fake_ag.function_scope = function_wrapping.function_scope
+      result.ag__ = fake_ag
       for k, v in namespace.items():
         result.__dict__[k] = v
       yield result
diff --git a/tensorflow/python/autograph/impl/conversion.py b/tensorflow/python/autograph/impl/conversion.py
index 0ca84b1f7a..5c8dd59fe6 100644
--- a/tensorflow/python/autograph/impl/conversion.py
+++ b/tensorflow/python/autograph/impl/conversion.py
@@ -55,7 +55,7 @@ from tensorflow.python.autograph.pyct import parser
 from tensorflow.python.autograph.pyct import qual_names
 from tensorflow.python.autograph.pyct import templates
 from tensorflow.python.autograph.pyct import transformer
-from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.autograph.utils import ag_logging as logging
 from tensorflow.python.util import tf_inspect
 
 
@@ -81,23 +81,27 @@ def is_whitelisted_for_graph(o):
   else:
     m = tf_inspect.getmodule(o)
   if not hasattr(m, '__name__'):
-    logging.vlog(1, '%s is NOT whitelisted for graph: unknown module name', o)
+    # Note: typically it's builtins that fall in this category. Builtins will
+    # be handled by specific code that follows this screening layer.
+    logging.log(2, '%s is NOT whitelisted: unknown module name', o)
     return False
 
   for prefix, in config.DEFAULT_UNCOMPILED_MODULES:
     if m.__name__.startswith(prefix):
-      logging.vlog(1, '%s is whitelisted: name starts with "%s"', o, prefix)
+      logging.log(2, '%s is whitelisted: name starts with "%s"', o, prefix)
       return True
 
-  if hasattr(o, 'autograph_info__'):
+  if hasattr(o, 'autograph_info__') or hasattr(o, '__ag_compiled'):
+    logging.log(2, '%s is whitelisted: already converted', o)
     return True
 
   if (not inspect_utils.isweakrefself(o) and not tf_inspect.isclass(o) and
       hasattr(o, '__call__') and hasattr(o, '__class__')):
     # Callable objects: whitelisted if their __call__ method is.
-    retval = is_whitelisted_for_graph(o.__call__)
-    logging.vlog(1, '%s is whitelisted: object __call__ whitelisted', o)
-    return retval
+    call_whitelisted = is_whitelisted_for_graph(o.__call__)
+    if call_whitelisted:
+      logging.log(2, '%s is whitelisted: object __call__ whitelisted', o)
+      return call_whitelisted
 
   if tf_inspect.ismethod(o):
     # Methods of whitelisted classes are also whitelisted, even if they are
@@ -119,8 +123,8 @@ def is_whitelisted_for_graph(o):
     if owner_class is not None:
       owner_class = inspect_utils.getdefiningclass(o, owner_class)
       if is_whitelisted_for_graph(owner_class):
-        logging.vlog(1, '%s is whitelisted: owner is whitelisted %s', o,
-                     owner_class)
+        logging.log(2, '%s is whitelisted: owner is whitelisted %s', o,
+                    owner_class)
         return True
 
   if inspect_utils.isnamedtuple(o):
@@ -128,14 +132,13 @@ def is_whitelisted_for_graph(o):
     # because they don't expose source code. But we assume they are safe for
     # graph mode since they are just containers.
     if tf_inspect.isclass(o) and len(o.__bases__) > 1:
-      logging.log_first_n(
-          logging.level_warning(),
+      logging.warn_first_n(
           'Entity {} looks like a namedtuple subclass. If it has any custom'
           ' methods, they will not be converted by AutoGraph.'.format(o), 1)
-    logging.vlog(1, '%s is whitelisted: named tuple', o)
+    logging.log(2, '%s is whitelisted: named tuple', o)
     return True
 
-  logging.vlog(1, '%s is NOT whitelisted for graph', o)
+  logging.log(2, '%s is NOT whitelisted', o)
   return False
 
 
@@ -167,7 +170,7 @@ def entity_to_graph(o, program_ctx, arg_values, arg_types):
   Raises:
     ValueError: if the entity type is not supported.
   """
-  logging.vlog(logging.DEBUG, 'Converting %s', o)
+  logging.log(1, 'Converting %s', o)
 
   if tf_inspect.isclass(o):
     node, name, ns = class_to_graph(o, program_ctx)
@@ -201,9 +204,9 @@ def entity_to_graph(o, program_ctx, arg_values, arg_types):
 
   program_ctx.add_to_cache(o, node)
 
-  if logging.get_verbosity() <= logging.DEBUG:
-    logging.vlog(logging.DEBUG, 'Compiled output of %s:\n\n%s\n', o,
-                 compiler.ast_to_source(node))
+  if logging.has_verbosity(2):
+    logging.log(2, 'Compiled output of %s:\n\n%s\n', o,
+                compiler.ast_to_source(node))
 
   if program_ctx.options.recursive:
     while True:
@@ -315,6 +318,8 @@ def _add_self_references(namespace, autograph_module):
     # internal modules.
     ag_internal = imp.new_module('autograph')
     ag_internal.__dict__.update(autograph_module.__dict__)
+    ag_internal.ConversionOptions = converter.ConversionOptions
+    ag_internal.Feature = converter.Feature
     ag_internal.utils = utils
     ag_internal.function_scope = function_wrapping.function_scope
     ag_internal.rewrite_graph_construction_error = (
-- 
GitLab


From 2e17f8b3b712d9b7d17155f00b06d8ea4410ebdb Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Thu, 24 Jan 2019 15:42:14 -0800
Subject: [PATCH 1215/2345] [Grappler] Inline/Optimize functions until
 convergence

PiperOrigin-RevId: 230804192
---
 tensorflow/core/common_runtime/function.h     |   2 +
 .../grappler/optimizers/function_optimizer.cc | 443 ++++++++++--------
 .../grappler/optimizers/function_optimizer.h  |  10 +
 .../optimizers/function_optimizer_test.cc     |  75 +++
 4 files changed, 342 insertions(+), 188 deletions(-)

diff --git a/tensorflow/core/common_runtime/function.h b/tensorflow/core/common_runtime/function.h
index df884f7577..94b6beeb93 100644
--- a/tensorflow/core/common_runtime/function.h
+++ b/tensorflow/core/common_runtime/function.h
@@ -160,6 +160,8 @@ FunctionBody* SymbolicGradient(const FunctionBody& f);
 // to "fbody". Replaces the "caller" with fbody->graph and connects
 // edges properly. "override_device" specifies whether inlining should replace
 // explicitly specified devices inside fbody with the callee's device.
+//
+// TODO(ezhulenev): Return Status::error if function inlining failed.
 void InlineFunctionBody(const FunctionLibraryDefinition& flib_def, Graph* g,
                         Node* caller, const FunctionBody* fbody,
                         bool override_device = true);
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index 9581fce8d3..6913f9b55b 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -256,28 +256,26 @@ struct FunctionSpecialization {
   std::vector<std::pair<int, int>> output_mapping;
 };
 
+// Function optimizer context initialized once for each optimization pass, and
+// it uses the latest available graph (for the first iteration it will be the
+// GrapplerItem.graph, for next iterations it will be the output of previous
+// function optimizer pass).
 class FunctionOptimizerContext {
  public:
-  explicit FunctionOptimizerContext(RewriterConfig::Toggle opt_level,
-                                    const GrapplerItem& item)
-      : grappler_item_id_(item.id),
-        graph_version_(item.graph.versions().producer()),
+  explicit FunctionOptimizerContext(const GrapplerItem& item,
+                                    RewriterConfig::Toggle opt_level,
+                                    const GraphDef& graph)
+      : item_(&item),
         opt_level_(opt_level),
-        optimization_options_(item.optimization_options()),
-        function_library_(OpRegistry::Global(), item.graph.library()),
-        available_device_names_(item.devices().begin(), item.devices().end()),
-        graph_view_(&item.graph) {
-    InitializeTrulyConstNodes(item);
-    InitializeFetchNodes(item);
-  }
+        function_library_(OpRegistry::Global(), graph.library()),
+        truly_const_nodes_(InferTrulyConstNodes(item, graph)),
+        graph_view_(&graph) {}
 
-  int graph_version() const { return graph_version_; }
+  const GrapplerItem& item() const { return *item_; }
 
-  const RewriterConfig::Toggle opt_level() const { return opt_level_; }
+  const int graph_version() const { return item_->graph.versions().producer(); }
 
-  const GrapplerItem::OptimizationOptions& optimization_options() const {
-    return optimization_options_;
-  }
+  RewriterConfig::Toggle opt_level() const { return opt_level_; }
 
   const FunctionLibraryDefinition& function_library() const {
     return function_library_;
@@ -304,16 +302,10 @@ class FunctionOptimizerContext {
 
   const GraphView& graph_view() const { return graph_view_; }
 
-  const string& grappler_item_id() const { return grappler_item_id_; }
-
-  const absl::flat_hash_set<string>& fetch_tensors() const {
-    return fetch_tensors_;
-  }
-
   const DeviceSet* devices() const {
     // Create fake devices lazily only if we need a DeviceSet.
-    if (available_devices_.empty() && !available_device_names_.empty()) {
-      for (const string& name : available_device_names_) {
+    if (available_devices_.empty() && !item_->devices().empty()) {
+      for (const string& name : item_->devices()) {
         auto device = absl::make_unique<FakeDevice>(name);
         available_device_set_.AddDevice(device.get());
         available_devices_.push_back(std::move(device));
@@ -323,7 +315,9 @@ class FunctionOptimizerContext {
   }
 
   bool IsFetchNode(const string& node_name) const {
-    return fetch_nodes_.find(node_name) != fetch_nodes_.end();
+    return absl::c_any_of(item_->fetch, [&](const string& fetch) {
+      return ParseTensorName(fetch).node() == node_name;
+    });
   }
 
   bool IsTrulyConst(const string& name) const {
@@ -345,6 +339,11 @@ class FunctionOptimizerContext {
   }
 
   void AddTensorMapping(const SafeTensorId& from, const SafeTensorId& to) {
+    DCHECK(from.index() != Graph::kControlSlot)
+        << "Tensor mapping must be from regular tensor";
+    DCHECK(to.index() != Graph::kControlSlot)
+        << "Tensor mapping must be to regular tensor";
+
     auto inserted = tensor_mapping_.insert({from, to});
     DCHECK(inserted.second)
         << "Failed to insert duplicated tensor mapping: "
@@ -359,8 +358,7 @@ class FunctionOptimizerContext {
       if (from_idx != to_idx) {
         SafeTensorId from_tensor(func_node, from_idx);
         SafeTensorId to_tensor(func_node, to_idx);
-        auto inserted = tensor_mapping_.insert({from_tensor, to_tensor});
-        DCHECK(inserted.second);
+        AddTensorMapping(from_tensor, to_tensor);
       }
     }
   }
@@ -374,24 +372,21 @@ class FunctionOptimizerContext {
   }
 
  private:
-  void InitializeTrulyConstNodes(const GrapplerItem& item) {
-    absl::flat_hash_set<string> feed_nodes;
+  static absl::flat_hash_map<string, const NodeDef*> InferTrulyConstNodes(
+      const GrapplerItem& item, const GraphDef& graph) {
+    absl::flat_hash_set<absl::string_view> feed_nodes;
     for (const auto& feed : item.feed) {
-      feed_nodes.insert(NodeName(feed.first));
+      feed_nodes.insert(feed.first);
     }
 
-    for (const NodeDef& node : item.graph.node()) {
-      if (IsConstant(node) && feed_nodes.count(node.name()) == 0) {
-        truly_const_nodes_[node.name()] = &node;
+    absl::flat_hash_map<string, const NodeDef*> const_nodes;
+    for (const NodeDef& node : graph.node()) {
+      if (IsConstant(node) && !feed_nodes.contains(node.name())) {
+        const_nodes[node.name()] = &node;
       }
     }
-  }
 
-  void InitializeFetchNodes(const GrapplerItem& item) {
-    for (const string& fetch : item.fetch) {
-      fetch_tensors_.insert(fetch);
-      fetch_nodes_.insert(NodeName(fetch));
-    }
+    return const_nodes;
   }
 
   void InitializeFunctionLibraryRuntime() {
@@ -403,16 +398,16 @@ class FunctionOptimizerContext {
       OptimizerOptions optimizer_opts;
       optimizer_opts.set_do_function_inlining(true);
       process_flr_.reset(new ProcessFunctionLibraryRuntime(
-          device_mgr_.get(), env, graph_version_, &function_library_,
-          optimizer_opts));
+          device_mgr_.get(), env, item_->graph.versions().producer(),
+          &function_library_, optimizer_opts));
       flr_ = process_flr_->GetFLR(device_mgr_->ListDevices()[0]->name());
     }
   }
 
-  const string grappler_item_id_;
-  const int graph_version_;
-  const RewriterConfig::Toggle opt_level_;
-  const GrapplerItem::OptimizationOptions optimization_options_;
+  const GrapplerItem* item_;  // must outlive this object
+  RewriterConfig::Toggle opt_level_;
+
+  // Function library constructed from current graph.
   FunctionLibraryDefinition function_library_;
 
   // These fields initialized lazily only if needed.
@@ -420,14 +415,11 @@ class FunctionOptimizerContext {
   std::unique_ptr<ProcessFunctionLibraryRuntime> process_flr_;
   FunctionLibraryRuntime* flr_ = nullptr;
 
-  // Fully defined names of the devices available to the GrapplerItem.
-  const absl::flat_hash_set<string> available_device_names_;
-
   // List of available `FakedDevices` (lazily initialized, see devices()).
   mutable std::vector<std::unique_ptr<Device>> available_devices_;
 
   // DeviceSet of fake devices (`FakeDevice`) constructed from
-  // available_devices_ (lazily initialized).
+  // item_.devices() (lazily initialized).
   mutable DeviceSet available_device_set_;
 
   // Nodes that are Const and not in feed.
@@ -437,10 +429,6 @@ class FunctionOptimizerContext {
                       const FunctionSpecialization>
       specialized_functions_;
 
-  // GrapplerItem.fetch is a vector of tensors.
-  absl::flat_hash_set<string> fetch_tensors_;  // format: node_name:port
-  absl::flat_hash_set<string> fetch_nodes_;    // format: node_name
-
   // After function inlining and specialization, the optimized graph might be in
   // invalid state, nodes can read from non-existing function call nodes that
   // were inlined, or they can read from output index that is no longer valid
@@ -495,9 +483,11 @@ absl::flat_hash_set<int> GetActiveOutputs(const NodeDef& node,
   }
 
   // 2. Or it can be in a fetch set.
-  for (const string& fetch_tensor : ctx.fetch_tensors()) {
-    int port = NodePositionIfSameNode(fetch_tensor, node.name());
-    if (port >= 0) active_outputs.insert(port);
+  for (const string& fetch : ctx.item().fetch) {
+    TensorId fetch_tensor = ParseTensorName(fetch);
+    if (fetch_tensor.node() == node.name()) {
+      active_outputs.insert(fetch_tensor.index());
+    }
   }
 
   return active_outputs;
@@ -755,14 +745,12 @@ Status InitializeFunctionSpecializationSignature(
 string SpecializedFunctionName(const FunctionOptimizerContext& ctx,
                                const FunctionDef& func,
                                const NodeDef& func_node) {
-  return absl::Substitute("$0_specialized_for_$1_at_$2",
-                          func.signature().name(),
-                          absl::StrReplaceAll(func_node.name(), {{"/", "_"}}),
-                          ctx.grappler_item_id());
+  return absl::Substitute(
+      "$0_specialized_for_$1_at_$2", func.signature().name(),
+      absl::StrReplaceAll(func_node.name(), {{"/", "_"}}), ctx.item().id);
 }
 
 Status SpecializeFunction(const NodeDef& func_node, const FunctionDef& func,
-                          const int graph_def_version,
                           FunctionOptimizerContext* ctx,
                           GraphDef* optimized_graph) {
   VLOG(2) << "Specialize function call: " << SummarizeNodeDef(func_node);
@@ -801,8 +789,8 @@ Status SpecializeFunction(const NodeDef& func_node, const FunctionDef& func,
   // Make a GrapplerFunctionItem and convert it back to FunctionDef after
   // pushing all constant inputs into the function body.
   GrapplerFunctionItem item;
-  TF_RETURN_IF_ERROR(MakeGrapplerFunctionItem(func, func_instantiation_attr,
-                                              flib, graph_def_version, &item));
+  TF_RETURN_IF_ERROR(MakeGrapplerFunctionItem(
+      func, func_instantiation_attr, flib, ctx->graph_version(), &item));
 
   // Push const inputs into the function body, and keep track of their control
   // dependencies.
@@ -1004,7 +992,6 @@ NodeDef InlinedFunctionOutputsNode(
 
 Status InlineDirectFunctionCall(const NodeDef& func_node,
                                 const FunctionDef& func,
-                                const int graph_def_version,
                                 const FunctionOptimizerContext& ctx,
                                 GraphDef* optimized_graph) {
   VLOG(2) << "Inline direct function call: " << SummarizeNodeDef(func_node);
@@ -1016,7 +1003,7 @@ Status InlineDirectFunctionCall(const NodeDef& func_node,
   GrapplerFunctionItem item;
   Status item_status = MakeGrapplerFunctionItem(func, func_instantiation_attr,
                                                 ctx.function_library(),
-                                                graph_def_version, &item);
+                                                ctx.graph_version(), &item);
 
   if (!item_status.ok()) {
     return errors::InvalidArgument("Failed to inline function ", func_node.op(),
@@ -1093,36 +1080,8 @@ Status InlineDirectFunctionCall(const NodeDef& func_node,
     // Make sure the node is placed.
     func_body_node.set_device(func_node.device());
 
-    // Move the function body node to the optimized graph.
-    const auto move_node_to_optimized_graph = [&]() {
-      // Annotate the node with the function attributes.
-      for (const auto& attr : func.attr()) {
-        func_body_node.mutable_attr()->insert(attr);
-      }
-      // Move the node to the main graph.
-      optimized_graph->add_node()->Swap(&func_body_node);
-    };
-
-    // Check if a body node is itself a function call and can be inlined.
-    const FunctionDef* func_body_node_func =
-        FindFunctionCall(ctx, func_body_node);
-
-    if (func_body_node_func != nullptr) {
-      Status inlinable = IsInlinableDirectFunctionCall(
-          ctx, *func_body_node_func, func_body_node);
-      if (inlinable.ok()) {
-        TF_RETURN_IF_ERROR(
-            InlineDirectFunctionCall(func_body_node, *func_body_node_func,
-                                     graph_def_version, ctx, optimized_graph));
-      } else {
-        VLOG(2) << "Can't inline nested direct function call: "
-                << inlinable.error_message();
-        move_node_to_optimized_graph();
-      }
-
-    } else {
-      move_node_to_optimized_graph();
-    }
+    // Move the node to the main graph.
+    optimized_graph->add_node()->Swap(&func_body_node);
   }
 
   DCHECK(output_tensors.size() == item.output_size())
@@ -1180,12 +1139,35 @@ Status InlineSymbolicGradient(const NodeDef& node,
   TF_RETURN_IF_ERROR(
       ConvertGraphDefToGraph(graph_ctor_opts, graph_def, &graph));
 
-  // Recursively inline the functions until there is nothing more to inline. We
-  // should at least expand one function.
-  int counter = 0;
-  while (counter < 50 && ExpandInlineFunctions(
-                             ctx->mutable_function_library_runtime(), &graph)) {
-    ++counter;
+  FunctionLibraryRuntime* flr = ctx->mutable_function_library_runtime();
+
+  // 1. Inline symbolic gradient node.
+  const bool expanded = ExpandInlineFunctions(flr, &graph);
+  DCHECK(expanded) << "Didn't expand SymbolicGradient op";
+
+  // TODO(ezhulenev): InlineFunctionBody in common_runtime/function silently
+  // fails to inline function into the graph, and leaves the graph unmodified.
+  // We check that graph has our symbolic gradient inlined, otherwise we return
+  // a error.
+  const auto is_symbolic_gradient_op = [&](const Node* node) {
+    return node->name() == inlined->name() &&
+           node->type_string() == "SymbolicGradient";
+  };
+  for (Node* node : graph.nodes()) {
+    if (is_symbolic_gradient_op(node)) {
+      return errors::Internal("Failed to inline symbolic gradient node: ",
+                              SummarizeNode(*node));
+    }
+  }
+
+  // 2. Recursively inline nested function calls.
+  int iteration = 0;
+  while (ExpandInlineFunctions(flr, &graph)) {
+    if (++iteration >= 50) {
+      VLOG(2) << "Break symbolic gradient inlining loop at iteration #"
+              << iteration;
+      break;
+    }
   }
 
   GraphDef inlined_graph_def;
@@ -1548,7 +1530,6 @@ Status PlaceInlinedFunctionBody(
 
 Status InlineIndirectFunctionCall(const NodeDef& func_node,
                                   const FunctionDef& func,
-                                  const int graph_def_version,
                                   FunctionOptimizerContext* ctx,
                                   GraphDef* optimized_graph) {
   VLOG(2) << "Inline indirect function call: " << SummarizeNodeDef(func_node);
@@ -1560,7 +1541,7 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
   GrapplerFunctionItem item;
   Status item_status = MakeGrapplerFunctionItem(func, func_instantiation_attr,
                                                 ctx->function_library(),
-                                                graph_def_version, &item);
+                                                ctx->graph_version(), &item);
 
   if (!item_status.ok()) {
     return errors::InvalidArgument("Failed to inline function ", func_node.op(),
@@ -1854,34 +1835,115 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
   return Status::OK();
 }
 
-}  // namespace
+// Restores graph invariants after function specialization and inlining: all
+// inputs must be connected to valid nodes.
+Status RestoreGraphInvariants(const FunctionOptimizerContext& ctx,
+                              GraphDef* optimized_graph) {
+  // After function specialization and inlining graph might be in invalid
+  // state, and some nodes can read tensors that do not exists anymore in the
+  // optimized graph: function call node was fully inlined into the graph, or
+  // output index was invalidated by the output pruning.
 
-Status FunctionOptimizer::Optimize(Cluster*, const GrapplerItem& item,
-                                   GraphDef* optimized_graph) {
-  // Nothing to do here.
-  if (item.graph.library().function_size() == 0) {
-    *optimized_graph = item.graph;
-    return Status::OK();
+  if (!ctx.tensor_mapping().empty()) {
+    for (NodeDef& node : *optimized_graph->mutable_node()) {
+      for (int idx = 0; idx < node.input_size(); ++idx) {
+        TensorId input_tensor = ParseTensorName(node.input(idx));
+        if (input_tensor.index() == Graph::kControlSlot) break;
+
+        auto mapping = ctx.tensor_mapping().find(input_tensor);
+        if (mapping != ctx.tensor_mapping().end()) {
+          node.set_input(idx, mapping->second.ToString());
+        }
+      }
+    }
+  }
+
+  // Function inlining instantiates function body directly into the optimized
+  // graph, and we might end up with control dependencies to the nodes that no
+  // longer exist in a graph. We need to apply control overrides to all
+  // invalidated nodes, and rewire control dependencies to the control outputs
+  // node (it's also possible to rewrite singe control edge into multiple edges
+  // to inlined side-effectful nodes).
+
+  if (!ctx.control_overrides().empty()) {
+    for (NodeDef& node : *optimized_graph->mutable_node()) {
+      // Keep track of new control inputs to the node.
+      absl::flat_hash_set<string> add_ctrl_inputs;
+
+      // Remove all invalidated control inputs.
+      for (int idx = 0; idx < node.input_size(); /* see below */) {
+        // TODO(ezhulenev): Use non-allocating TensorId after migrating
+        // `control_overrides()` to absl::flat_hash_set.
+        SafeTensorId input_tensor = ParseTensorName(node.input(idx));
+
+        auto overrides = ctx.control_overrides().find(input_tensor.node());
+        if (overrides != ctx.control_overrides().end()) {
+          // If this happens it's a bug in the function inlining.
+          if (input_tensor.index() != Graph::kControlSlot) {
+            return errors::Internal(
+                "Illegal input edge from inlined function call node");
+          }
+          // Remove control dependency to the inlined function call node.
+          node.mutable_input()->SwapElements(idx, node.input_size() - 1);
+          node.mutable_input()->RemoveLast();
+
+          // Keep track of all overrides.
+          for (const string& override : overrides->second) {
+            add_ctrl_inputs.insert(AsControlDependency(override));
+          }
+        } else {
+          // Go to the next input only if the current one was not invalidated,
+          // otherwise we need to check the swapped input as well.
+          ++idx;
+        }
+      }
+
+      // Add overrides to the node inputs.
+      for (const string& ctrl_input : add_ctrl_inputs) {
+        node.add_input(ctrl_input);
+      }
+    }
   }
 
-  FunctionOptimizerContext ctx(opt_level_, item);
+  return Status::OK();
+}
+
+}  // namespace
+
+Status FunctionOptimizer::RunFunctionOptimizerPass(
+    const GrapplerItem& item, const GraphDef& graph, const int iteration,
+    std::unordered_set<string>* skip_nodes, GraphDef* optimized_graph,
+    bool* graph_has_unoptimized_function_calls) const {
+  VLOG(3) << absl::Substitute(
+      "Run function optimizer pass (iteration = $0): grappler_item_id = $1",
+      iteration, item.id);
+
+  FunctionOptimizerContext ctx(item, opt_level_, graph);
 
   bool inline_gradients = options_.enable_symbolic_gradient_inlining;
   bool inline_func = options_.enable_function_inlining;
   bool specialize_func = options_.enable_function_specialization;
 
-  for (const NodeDef& node : item.graph.node()) {
+  for (const NodeDef& node : graph.node()) {
     // Each node optimization can modify optimized graph only by adding new
     // nodes, we can check node size to make sure that graph was not modified.
     const int num_nodes_before = optimized_graph->node_size();
     const auto is_graph_modified = [&]() {
       int num_nodes = optimized_graph->node_size();
-      CHECK_GE(num_nodes, num_nodes_before) << "Nodes should not be removed";
+      DCHECK_GE(num_nodes, num_nodes_before) << "Nodes should not be removed";
       return num_nodes > num_nodes_before;
     };
 
-    // Add a copy of an input graph node to the optimized graph.
-    const auto add_node_copy = [&]() { *optimized_graph->add_node() = node; };
+    // Copy node from the `graph` to the `optimized_graph`.
+    const auto copy_node = [&]() { *optimized_graph->add_node() = node; };
+
+    // If we already failed to optimize this node during one of the previous
+    // passes, we just give up, and do not try on more time.
+    if (skip_nodes->find(node.name()) != skip_nodes->end()) {
+      VLOG(3) << "Skip optimization for node: " << node.name();
+      copy_node();
+      continue;
+    }
 
 // Skip errors if optimized graph was not modified before error happened.
 #define TF_SKIP_ERROR_IF_GRAPH_UNMODIFIED(...)                     \
@@ -1891,7 +1953,8 @@ Status FunctionOptimizer::Optimize(Cluster*, const GrapplerItem& item,
       return _status;                                              \
     if (TF_PREDICT_FALSE(!_status.ok() && !is_graph_modified())) { \
       VLOG(3) << "Skip error: " << _status.error_message();        \
-      add_node_copy();                                             \
+      skip_nodes->insert(node.name());                             \
+      copy_node();                                                 \
     }                                                              \
   } while (0)
 
@@ -1909,6 +1972,9 @@ Status FunctionOptimizer::Optimize(Cluster*, const GrapplerItem& item,
         TF_SKIP_ERROR_IF_GRAPH_UNMODIFIED(
             InlineSymbolicGradient(node, &ctx, optimized_graph));
         continue;
+      } else {
+        VLOG(2) << "Skip SymbolicGradient inlining: function=" << f_name;
+        skip_nodes->insert(node.name());
       }
     }
 
@@ -1921,7 +1987,6 @@ Status FunctionOptimizer::Optimize(Cluster*, const GrapplerItem& item,
 
     if (func != nullptr) {
       const string& func_name = func->signature().name();
-      const int graph_def_version = item.graph.versions().producer();
 
       const bool is_direct_func = IsDirectFunctionCall(*func, node);
       const bool is_indirect_func = IsIndirectFunctionCall(*func, node);
@@ -1930,11 +1995,12 @@ Status FunctionOptimizer::Optimize(Cluster*, const GrapplerItem& item,
       if (inline_func && is_direct_func) {
         Status inlinable = IsInlinableDirectFunctionCall(ctx, *func, node);
         if (inlinable.ok()) {
-          TF_SKIP_ERROR_IF_GRAPH_UNMODIFIED(InlineDirectFunctionCall(
-              node, *func, graph_def_version, ctx, optimized_graph));
+          TF_SKIP_ERROR_IF_GRAPH_UNMODIFIED(
+              InlineDirectFunctionCall(node, *func, ctx, optimized_graph));
           continue;
         } else {
           VLOG(2) << inlinable.error_message();
+          skip_nodes->insert(node.name());
         }
       }
 
@@ -1942,11 +2008,12 @@ Status FunctionOptimizer::Optimize(Cluster*, const GrapplerItem& item,
       if (inline_func && is_indirect_func) {
         Status inlinable = IsInlinableIndirectFunctionCall(ctx, *func, node);
         if (inlinable.ok()) {
-          TF_SKIP_ERROR_IF_GRAPH_UNMODIFIED(InlineIndirectFunctionCall(
-              node, *func, graph_def_version, &ctx, optimized_graph));
+          TF_SKIP_ERROR_IF_GRAPH_UNMODIFIED(
+              InlineIndirectFunctionCall(node, *func, &ctx, optimized_graph));
           continue;
         } else {
           VLOG(2) << inlinable.error_message();
+          skip_nodes->insert(node.name());
         }
       }
 
@@ -1963,95 +2030,95 @@ Status FunctionOptimizer::Optimize(Cluster*, const GrapplerItem& item,
         // TODO(ezhulenev): Specialize function call if input has a known shape.
         // Specialize function body for its instantiation attributes and inputs.
         TF_SKIP_ERROR_IF_GRAPH_UNMODIFIED(
-            SpecializeFunction(node, *func, item.graph.versions().producer(),
-                               &ctx, optimized_graph));
+            SpecializeFunction(node, *func, &ctx, optimized_graph));
         continue;
+      } else {
+        VLOG(2) << "Skip function specialization: " << func->signature().name();
+        skip_nodes->insert(node.name());
       }
     }
 
     // ---------------------------------------------------------------------- //
     // If we reached this point, node was not handled by any of the stages
-    // (inline, specialize), simply add a copy to the graph.
-    add_node_copy();
+    // (inline, specialize), simply copy the node to the optimized graph.
+    copy_node();
 
 #undef TF_SKIP_ERROR_IF_GRAPH_UNMODIFIED
   }
 
-  // After function specialization and inlining graph might be in invalid
-  // state, and some nodes can read tensors that do not exists anymore in the
-  // optimized graph: function call node was fully inlined into the graph, or
-  // output index was invalidated by the output pruning.
+  TF_RETURN_IF_ERROR(RestoreGraphInvariants(ctx, optimized_graph));
 
-  if (!ctx.tensor_mapping().empty()) {
-    for (NodeDef& node : *optimized_graph->mutable_node()) {
-      for (int idx = 0; idx < node.input_size(); ++idx) {
-        TensorId input_tensor = ParseTensorName(node.input(idx));
-        if (input_tensor.index() == Graph::kControlSlot) break;
+  // Preserve the graph version.
+  *optimized_graph->mutable_versions() = graph.versions();
 
-        auto mapping = ctx.tensor_mapping().find(input_tensor);
-        if (mapping != ctx.tensor_mapping().end()) {
-          node.set_input(idx, mapping->second.ToString());
-        }
-      }
+  // Prune unreachable function from the library.
+  if (options_.enable_trim_function_library) {
+    *optimized_graph->mutable_library() =
+        PruneFunctionLibrary(ctx.function_library(), *optimized_graph);
+  } else {
+    *optimized_graph->mutable_library() = ctx.function_library().ToProto();
+  }
+
+  // Before returning we check if after single optimization pass we have more
+  // unoptimized function calls.
+  *graph_has_unoptimized_function_calls = false;
+  for (const NodeDef& node : optimized_graph->node()) {
+    // Check if we can inline symbolic gradient.
+    if (IsSymbolicGradient(node) && inline_gradients &&
+        skip_nodes->count(node.name()) == 0) {
+      *graph_has_unoptimized_function_calls = true;
+      break;
     }
-  }
 
-  // Function inlining instantiates function body directly into the optimized
-  // graph, and we might end up with control dependencies to the nodes that no
-  // longer exist in a graph. We need to apply control overrides to all
-  // invalidated nodes, and rewire control dependencies to the inlined
-  // side-effectful function body nodes.
+    // Check if after inlining we have unoptimized function calls.
+    const FunctionDef* func = FindFunctionCall(ctx, node);
+    if (func != nullptr && !MarkedSpecialized(*func) &&
+        skip_nodes->count(node.name()) == 0) {
+      *graph_has_unoptimized_function_calls = true;
+      break;
+    }
+  }
 
-  // TODO(ezhulenev): With nested function call inlining, single pass over
-  // `control_overrides` might not bring the graph into a valid state,
-  // continue until it converges and all invalidated control dependencies
-  // removed.
+  return Status::OK();
+}
 
-  if (!ctx.control_overrides().empty()) {
-    for (NodeDef& node : *optimized_graph->mutable_node()) {
-      // Keep track of new control inputs to the node.
-      absl::flat_hash_set<string> add_ctrl_inputs;
+Status FunctionOptimizer::Optimize(Cluster*, const GrapplerItem& item,
+                                   GraphDef* optimized_graph) {
+  // Nothing to do here.
+  if (item.graph.library().function_size() == 0) {
+    *optimized_graph = item.graph;
+    return Status::OK();
+  }
 
-      // Remove all invalidated control inputs.
-      for (int idx = 0; idx < node.input_size(); /* see below */) {
-        // TODO(ezhulenev): Use non-allocating TensorId after migrating
-        // `control_overrides()` to absl::flat_hash_set.
-        SafeTensorId input_tensor = ParseTensorName(node.input(idx));
+  // Do not retry failed function inlining or specialization.
+  std::unordered_set<string> skip_nodes;
+  bool graph_has_unoptimized_function_calls = false;
 
-        auto overrides = ctx.control_overrides().find(input_tensor.node());
-        if (overrides != ctx.control_overrides().end()) {
-          // If this happens it's a bug in the function inlining.
-          if (input_tensor.index() != Graph::kControlSlot) {
-            return errors::Internal(
-                "Illegal input edge from inlined function call node");
-          }
-          // Remove control dependency to the inlined function call node.
-          node.mutable_input()->SwapElements(idx, node.input_size() - 1);
-          node.mutable_input()->RemoveLast();
+  // We'll keep running function optimizer pass until we inlined and optimized
+  // all function call nodes.
+  int iteration = 0;
+  constexpr int kMaxIterations = 50;
 
-          // Keep track of all overrides.
-          for (const string& override : overrides->second) {
-            add_ctrl_inputs.insert(AsControlDependency(override));
-          }
-        } else {
-          // Go to the next input only if the current one was not invalidated,
-          // otherwise we need to check the swapped input as well.
-          ++idx;
-        }
-      }
+  // 1. Run first optimizer pass with GrapplerItem.graph.
+  TF_RETURN_IF_ERROR(RunFunctionOptimizerPass(
+      item, item.graph, 0, &skip_nodes, optimized_graph,
+      &graph_has_unoptimized_function_calls));
 
-      // Add overrides to the node inputs.
-      for (const string& ctrl_input : add_ctrl_inputs) {
-        node.add_input(ctrl_input);
-      }
+  // 2. If after function inlining we have unoptimized function calls, we have
+  // to run function optimization pass one more time.
+  while (graph_has_unoptimized_function_calls) {
+    if (iteration++ > kMaxIterations) {
+      VLOG(1) << "Break function optimizer loop at iteration #" << iteration;
+      break;
     }
-  }
 
-  *optimized_graph->mutable_versions() = item.graph.versions();
-  *optimized_graph->mutable_library() =
-      options_.enable_trim_function_library
-          ? PruneFunctionLibrary(ctx.function_library(), *optimized_graph)
-          : ctx.function_library().ToProto();
+    GraphDef workspace_graph;
+    workspace_graph.Swap(optimized_graph);
+
+    TF_RETURN_IF_ERROR(RunFunctionOptimizerPass(
+        item, workspace_graph, iteration, &skip_nodes, optimized_graph,
+        &graph_has_unoptimized_function_calls));
+  }
 
   return Status::OK();
 }
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.h b/tensorflow/core/grappler/optimizers/function_optimizer.h
index 4352555064..ab90281509 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.h
@@ -48,6 +48,16 @@ class FunctionOptimizer : public GraphOptimizer {
     bool enable_trim_function_library = true;
   };
 
+  // Runs a single function optimizer pass over the `graph`. All nodes that are
+  // not function calls will be copied from the `graph` to the
+  // `optimized_graph`. Function call nodes inlined or specialized, and
+  // instantiated function body or specialized function call nodes will be added
+  // to the `optimized_graph`.
+  Status RunFunctionOptimizerPass(
+      const GrapplerItem& item, const GraphDef& graph, const int iteration,
+      std::unordered_set<string>* skip_nodes, GraphDef* optimized_graph,
+      bool* graph_has_unoptimized_function_calls) const;
+
   RewriterConfig::Toggle opt_level_;
   FunctionOptimizerOptions options_;
 };
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
index a67acffd03..fc7fffe7b2 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
@@ -1194,6 +1194,81 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithMergedDeadTensors) {
   test::ExpectTensorEqual<float>(tensors[0], tensors_expected[0]);
 }
 
+TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithNestedFunctionCall) {
+  using test::function::NDef;
+  using FDH = FunctionDefHelper;
+
+  FunctionOptimizer optimizer(RewriterConfig::AGGRESSIVE);
+
+  FunctionDef mul_func = FunctionDefHelper::Create(
+      "MyMul", {"x:T", "y:T"}, {"z:T"}, {"T: {float, double}"},
+      {{{"mul"}, "Mul", {"x", "y"}, {{"T", "$T"}}}},
+      /* Mapping between function returns and function node outputs. */
+      {{"z", "mul:z:0"}});
+
+  // `Square` implemented in terms of PartitionedCall to `MyMul`.
+  FunctionDef square_func = FunctionDefHelper::Create(
+      "MySquare", {"x:T"}, {"output:T"}, {"T: {float, double}"},
+      {{{"square"},
+        "PartitionedCall",
+        {"x", "x"},
+        {{"Tin", DataTypeSlice{DT_FLOAT, DT_FLOAT}},
+         {"Tout", DataTypeSlice{DT_FLOAT}},
+         {"f", FDH::FunctionRef("MyMul", {{"T", DT_FLOAT}})}}}},
+      /* Mapping between function returns and function node outputs. */
+      {{"output", "square:output:0"}});
+
+  // Build a graph to compute:
+  //   b = Square(a)
+  //   c = Identity(b)
+  //   return c
+  GrapplerItem item;
+  item.fetch = {"c"};
+  item.graph = test::function::GDef(
+      {NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("b", "PartitionedCall", {"a"},
+            {{"Tin", DataTypeSlice{DT_FLOAT}},
+             {"Tout", DataTypeSlice{DT_FLOAT}},
+             {"f", FDH::FunctionRef("MySquare", {{"T", DT_FLOAT}})}},
+            kDevice),
+       NDef("c", "Identity", {"b"}, {{"T", DT_FLOAT}}, kDevice)},
+      /* Function library */
+      {mul_func, square_func});
+
+  GraphDef optimized_graph;
+  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &optimized_graph));
+
+  GraphDef expected = test::function::GDef(
+      {NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+
+       // Inlined inputs of `c` node.
+       NDef("b/x", "Identity", {"a:0"}, {{"T", DT_FLOAT}}, kDevice),
+
+       // Inlined inputs of `square` node inside inlined `MySquare` function.
+       NDef("b/square/x", "Identity", {"b/x:0"}, {{"T", DT_FLOAT}}, kDevice),
+       NDef("b/square/y", "Identity", {"b/x:0"}, {{"T", DT_FLOAT}}, kDevice),
+
+       // Inlined mul node from the `MyMul` function.
+       NDef("b/square/mul", "Mul", {"b/square/x", "b/square/y"},
+            {{"T", DT_FLOAT}}, kDevice),
+
+       NDef("c", "Identity", {"b/square/mul:0"}, {{"T", DT_FLOAT}}, kDevice)},
+      // Function library.
+      {mul_func});
+
+  CompareGraphs(expected, optimized_graph);
+
+  Tensor three = test::AsScalar<float>(3.0f);
+  item.feed.emplace_back("a", three);
+
+  GrapplerItem optimized = item.WithGraph(std::move(optimized_graph));
+  auto tensors_expected = EvaluateFetchNodes(item);
+  auto tensors = EvaluateFetchNodes(optimized);
+  ASSERT_EQ(tensors_expected.size(), 1);
+  ASSERT_EQ(tensors.size(), tensors_expected.size());
+  test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
+}
+
 TEST_F(FunctionOptimizerTest, SpecializeFunctionXTimesTwo) {
   using test::function::NDef;
 
-- 
GitLab


From 7c929bf6ef15d47f208743ac9ea5bbafdd981173 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 24 Jan 2019 15:45:54 -0800
Subject: [PATCH 1216/2345] Refactor out moment-testing and make more numpy
 friendly.

PiperOrigin-RevId: 230804842
---
 tensorflow/python/kernel_tests/random/BUILD   | 10 +++
 .../kernel_tests/random/random_gamma_test.py  | 60 +++-------------
 .../random/random_poisson_test.py             | 46 +++---------
 tensorflow/python/kernel_tests/random/util.py | 72 +++++++++++++++++++
 tensorflow/tools/pip_package/BUILD            |  1 +
 5 files changed, 100 insertions(+), 89 deletions(-)
 create mode 100644 tensorflow/python/kernel_tests/random/util.py

diff --git a/tensorflow/python/kernel_tests/random/BUILD b/tensorflow/python/kernel_tests/random/BUILD
index 445aaf9dc4..8452982a44 100644
--- a/tensorflow/python/kernel_tests/random/BUILD
+++ b/tensorflow/python/kernel_tests/random/BUILD
@@ -14,6 +14,14 @@ load("//tensorflow:tensorflow.bzl", "sycl_py_test")
 # Please avoid the py_tests and cuda_py_tests (plural) while we
 # fix the shared/overbroad dependencies.
 
+py_library(
+    name = "util",
+    srcs = ["util.py"],
+    deps = [
+        "//third_party/py/numpy",
+    ],
+)
+
 tf_py_test(
     name = "random_shuffle_queue_test",
     size = "small",
@@ -115,6 +123,7 @@ cuda_py_test(
     size = "medium",
     srcs = ["random_gamma_test.py"],
     additional_deps = [
+        ":util",
         "//third_party/py/numpy",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
@@ -151,6 +160,7 @@ cuda_py_test(
     size = "medium",
     srcs = ["random_poisson_test.py"],
     additional_deps = [
+        ":util",
         "//third_party/py/numpy",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
diff --git a/tensorflow/python/kernel_tests/random/random_gamma_test.py b/tensorflow/python/kernel_tests/random/random_gamma_test.py
index a5952a2196..5cc13f6777 100644
--- a/tensorflow/python/kernel_tests/random/random_gamma_test.py
+++ b/tensorflow/python/kernel_tests/random/random_gamma_test.py
@@ -18,8 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import math
-
 import numpy as np
 from six.moves import xrange  # pylint: disable=redefined-builtin
 
@@ -27,6 +25,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import random_seed
 from tensorflow.python.framework import test_util
+from tensorflow.python.kernel_tests.random import util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
@@ -69,16 +68,6 @@ class RandomGammaTest(test.TestCase):
       tf_logging.warn("Cannot test moments: %s" % e)
       return
 
-    # Check the given array of samples matches the given theoretical moment
-    # function at different orders. The test is considered passing if the
-    # z-tests of all statistical moments are all below z_limit.
-    # Parameters:
-    #   max_moments: the largest moments of the distribution to be tested
-    #   stride: the distance between samples to check for statistical properties
-    #       0 means the n-th moment of each sample
-    #       any other strides tests for spatial correlation between samples;
-    #   z_limit: the maximum z-test we would consider the test to pass;
-
     # The moments test is a z-value test.  This is the largest z-value
     # we want to tolerate. Since the z-test approximates a unit normal
     # distribution, it should almost definitely never exceed 6.
@@ -94,46 +83,13 @@ class RandomGammaTest(test.TestCase):
           max_moment = min(6, scale // 2)
           sampler = self._Sampler(
               20000, alpha, 1 / scale, dt, use_gpu=False, seed=12345)
-          moments = [0] * (max_moment + 1)
-          moments_sample_count = [0] * (max_moment + 1)
-          x = np.array(sampler().flat)  # sampler does 10x samples
-          for k in range(len(x)):
-            moment = 1.
-            for i in range(max_moment + 1):
-              index = k + i * stride
-              if index >= len(x):
-                break
-              moments[i] += moment
-              moments_sample_count[i] += 1
-              moment *= x[index]
-          for i in range(max_moment + 1):
-            moments[i] /= moments_sample_count[i]
-          for i in range(1, max_moment + 1):
-            g = stats.gamma(alpha, scale=scale)
-            if stride == 0:
-              moments_i_mean = g.moment(i)
-              moments_i_squared = g.moment(2 * i)
-            else:
-              moments_i_mean = pow(g.moment(1), i)
-              moments_i_squared = pow(g.moment(2), i)
-            # Calculate moment variance safely:
-            # This is just
-            #  (moments_i_squared - moments_i_mean**2) / moments_sample_count[i]
-            normalized_moments_i_var = (
-                moments_i_mean / moments_sample_count[i] *
-                (moments_i_squared / moments_i_mean - moments_i_mean))
-            # Assume every operation has a small numerical error.
-            # It takes i multiplications to calculate one i-th moment.
-            error_per_moment = i * np.finfo(dt.as_numpy_dtype).eps
-            total_variance = (normalized_moments_i_var + error_per_moment)
-            tiny = np.finfo(dt.as_numpy_dtype).tiny
-            self.assertGreaterEqual(total_variance, 0)
-            if total_variance < tiny:
-              total_variance = tiny
-            # z_test is approximately a unit normal distribution.
-            z_test = abs(
-                (moments[i] - moments_i_mean) / math.sqrt(total_variance))
-            self.assertLess(z_test, z_limit)
+          z_scores = util.test_moment_matching(
+              sampler(),
+              max_moment,
+              stats.gamma(alpha, scale=scale),
+              stride=stride,
+          )
+          self.assertAllLess(z_scores, z_limit)
 
   def _testZeroDensity(self, alpha):
     """Zero isn't in the support of the gamma distribution.
diff --git a/tensorflow/python/kernel_tests/random/random_poisson_test.py b/tensorflow/python/kernel_tests/random/random_poisson_test.py
index 0a6b004d68..51dd4cb47c 100644
--- a/tensorflow/python/kernel_tests/random/random_poisson_test.py
+++ b/tensorflow/python/kernel_tests/random/random_poisson_test.py
@@ -24,6 +24,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
+from tensorflow.python.kernel_tests.random import util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.platform import test
@@ -49,14 +50,13 @@ class RandomPoissonTest(test.TestCase):
 
     return func
 
-  # TODO(srvasude): Factor this out along with the corresponding moment testing
-  # method in random_gamma_test into a single library.
   def testMoments(self):
     try:
       from scipy import stats  # pylint: disable=g-import-not-at-top
     except ImportError as e:
       tf_logging.warn("Cannot test moments: %s", e)
       return
+
     # The moments test is a z-value test.  This is the largest z-value
     # we want to tolerate. Since the z-test approximates a unit normal
     # distribution, it should almost definitely never exceed 6.
@@ -67,41 +67,13 @@ class RandomPoissonTest(test.TestCase):
         for lam in (3., 20):
           max_moment = 5
           sampler = self._Sampler(10000, lam, dt, use_gpu=False, seed=12345)
-          moments = [0] * (max_moment + 1)
-          moments_sample_count = [0] * (max_moment + 1)
-          x = np.array(sampler().flat)  # sampler does 10x samples
-          for k in range(len(x)):
-            moment = 1.
-            for i in range(max_moment + 1):
-              index = k + i * stride
-              if index >= len(x):
-                break
-              moments[i] += moment
-              moments_sample_count[i] += 1
-              moment *= x[index]
-          for i in range(max_moment + 1):
-            moments[i] /= moments_sample_count[i]
-          for i in range(1, max_moment + 1):
-            g = stats.poisson(lam)
-            if stride == 0:
-              moments_i_mean = g.moment(i)
-              moments_i_squared = g.moment(2 * i)
-            else:
-              moments_i_mean = pow(g.moment(1), i)
-              moments_i_squared = pow(g.moment(2), i)
-            moments_i_var = (
-                moments_i_squared - moments_i_mean * moments_i_mean)
-            # Assume every operation has a small numerical error.
-            # It takes i multiplications to calculate one i-th moment.
-            error_per_moment = i * 1e-6
-            total_variance = (
-                moments_i_var / moments_sample_count[i] + error_per_moment)
-            if not total_variance:
-              total_variance = 1e-10
-            # z_test is approximately a unit normal distribution.
-            z_test = abs(
-                (moments[i] - moments_i_mean) / np.sqrt(total_variance))
-            self.assertLess(z_test, z_limit)
+          z_scores = util.test_moment_matching(
+              sampler(),
+              max_moment,
+              stats.poisson(lam),
+              stride=stride,
+          )
+          self.assertAllLess(z_scores, z_limit)
 
   # Checks that the CPU and GPU implementation returns the same results,
   # given the same random seed
diff --git a/tensorflow/python/kernel_tests/random/util.py b/tensorflow/python/kernel_tests/random/util.py
new file mode 100644
index 0000000000..67805c7f26
--- /dev/null
+++ b/tensorflow/python/kernel_tests/random/util.py
@@ -0,0 +1,72 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Utilities for testing random variables."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+
+def test_moment_matching(
+    samples,
+    number_moments,
+    dist,
+    stride=0):
+  """Return z-test scores for sample moments to match analytic moments.
+
+  Given `samples`, check that the first sample `number_moments` match
+  the given  `dist` moments by doing a z-test.
+
+  Args:
+    samples: Samples from target distribution.
+    number_moments: Python `int` describing how many sample moments to check.
+    dist: SciPy distribution object that provides analytic moments.
+    stride: Distance between samples to check for statistical properties.
+      A stride of 0 means to use all samples, while other strides test for
+      spatial correlation.
+  Returns:
+    Array of z_test scores.
+  """
+
+  sample_moments = []
+  expected_moments = []
+  variance_sample_moments = []
+  x = samples.flat
+  for i in range(1, number_moments + 1):
+    strided_range = x[::(i - 1) * stride + 1]
+    sample_moments.append(np.mean(strided_range ** i))
+    expected_moments.append(dist.moment(i))
+    variance_sample_moments.append(
+        (dist.moment(2 * i) - dist.moment(i) ** 2) / len(strided_range))
+
+  z_test_scores = []
+  for i in range(1, number_moments + 1):
+    # Assume every operation has a small numerical error.
+    # It takes i multiplications to calculate one i-th moment.
+    total_variance = (
+        variance_sample_moments[i - 1] +
+        i * np.finfo(samples.dtype).eps)
+    tiny = np.finfo(samples.dtype).tiny
+    assert np.all(total_variance > 0)
+    if total_variance < tiny:
+      total_variance = tiny
+    # z_test is approximately a unit normal distribution.
+    z_test_scores.append(abs(
+        (sample_moments[i - 1] - expected_moments[i - 1]) / np.sqrt(
+            total_variance)))
+  return z_test_scores
+
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index 3c1a9280a2..2731949c15 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -82,6 +82,7 @@ COMMON_PIP_DEPS = [
     "//tensorflow/python/data/kernel_tests:test_base",
     "//tensorflow/python/debug:debug_pip",
     "//tensorflow/python/eager:eager_pip",
+    "//tensorflow/python/kernel_tests/random:util",
     "//tensorflow/python/kernel_tests/signal:test_util",
     "//tensorflow/python/kernel_tests/testdata:self_adjoint_eig_op_test_files",
     "//tensorflow/python/ops/ragged:ragged_test_util",
-- 
GitLab


From e40ffed186331743127ddaaccaac5f1c193daa5f Mon Sep 17 00:00:00 2001
From: David Majnemer <majnemer@google.com>
Date: Thu, 24 Jan 2019 15:57:59 -0800
Subject: [PATCH 1217/2345] [TF2XLA] Add support for nextafter

This implementation matches the C++ function's behavior:
- nextafter(-0, +0) -> +0
- nextafter(+0, -0) -> -0
- nextafter(-0, -0) -> -0
- nextafter(+0, +0) -> +0

Regardless of the current FTZ or DAZ mode, this implementation will return a
subnormal if "from" is 0 but "to" is non-zero.

PiperOrigin-RevId: 230806902
---
 tensorflow/compiler/tests/binary_ops_test.py  | 24 ++++++
 .../compiler/tf2xla/kernels/binary_ops.cc     | 14 ++--
 tensorflow/compiler/tf2xla/kernels/cast_op.cc | 15 +---
 tensorflow/compiler/xla/client/lib/math.cc    | 78 +++++++++++++++++++
 tensorflow/compiler/xla/client/lib/math.h     |  8 +-
 tensorflow/compiler/xla/primitive_util.cc     | 15 ++++
 tensorflow/compiler/xla/primitive_util.h      |  2 +
 .../compiler/xla/service/shape_inference.cc   |  2 +-
 8 files changed, 134 insertions(+), 24 deletions(-)

diff --git a/tensorflow/compiler/tests/binary_ops_test.py b/tensorflow/compiler/tests/binary_ops_test.py
index a3651b4b0d..c829c50b55 100644
--- a/tensorflow/compiler/tests/binary_ops_test.py
+++ b/tensorflow/compiler/tests/binary_ops_test.py
@@ -311,6 +311,30 @@ class BinaryOpsTest(xla_test.XLATestCase):
           dtype(7),
           expected=np.array([[-6], [-5]], dtype=dtype))
 
+      if dtype in [np.float32, np.float64]:
+        x = np.array([
+            -0.0, 0.0, -0.0, +0.0, np.inf, np.inf, -np.inf, -np.inf, 2.0, 2.0,
+            1.0
+        ],
+                     dtype=dtype)
+        y = np.array(
+            [-0.0, 0.0, +0.0, -0.0, 1.0, -1.0, 1.0, -1.0, 2.0, 1.0, 2.0],
+            dtype=dtype)
+        expected = np.nextafter(x, y)
+
+        # We use assertAllEqual to expose any bugs hidden by relative or
+        # absolute error tolerances.
+        def NextAfterEqualityTest(result, expected, rtol):
+          del rtol
+          return self.assertAllEqual(result, expected)
+
+        self._testBinary(
+            math_ops.nextafter,
+            x,
+            y,
+            expected=expected,
+            equality_test=NextAfterEqualityTest)
+
       # min/max not supported for complex
       if dtype not in self.complex_types | {np.uint8, np.int8}:
         self._testBinary(
diff --git a/tensorflow/compiler/tf2xla/kernels/binary_ops.cc b/tensorflow/compiler/tf2xla/kernels/binary_ops.cc
index 5e9280c1fe..ad6b334326 100644
--- a/tensorflow/compiler/tf2xla/kernels/binary_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/binary_ops.cc
@@ -20,7 +20,9 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/client_library.h"
 #include "tensorflow/compiler/xla/client/lib/constants.h"
+#include "tensorflow/compiler/xla/client/lib/math.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/types.h"
@@ -165,12 +167,8 @@ XLA_MAKE_BINARY(
     xla::Div(xla::Mul(rhs, XlaHelpers::FloatLiteral(b, input_type(0), 0.5)),
              lhs, extend_dimensions));
 
-static xla::XlaOp Square(xla::XlaBuilder* builder, const xla::XlaOp& x) {
-  return xla::Mul(x, x);
-}
-
 XLA_MAKE_BINARY(SquaredDifference,
-                Square(b, xla::Sub(lhs, rhs, extend_dimensions)));
+                xla::Square(xla::Sub(lhs, rhs, extend_dimensions)));
 
 XLA_MAKE_BINARY(TruncateDiv, xla::Div(lhs, rhs, extend_dimensions));
 XLA_MAKE_BINARY(TruncateMod, xla::Rem(lhs, rhs, extend_dimensions));
@@ -195,8 +193,8 @@ XLA_MAKE_BINARY(SoftplusGrad,
 // softsigngrad(gradients, features) = gradients / (1 + abs(features)) ** 2
 XLA_MAKE_BINARY(SoftsignGrad,
                 xla::Div(lhs,
-                         Square(b, xla::Add(XlaHelpers::One(b, input_type(0)),
-                                            xla::Abs(rhs)))));
+                         xla::Square(xla::Add(XlaHelpers::One(b, input_type(0)),
+                                              xla::Abs(rhs)))));
 
 XLA_MAKE_BINARY(TanhGrad,
                 xla::Mul(rhs, xla::Sub(XlaHelpers::One(b, input_type(0)),
@@ -204,6 +202,8 @@ XLA_MAKE_BINARY(TanhGrad,
 
 XLA_MAKE_BINARY(Pow, xla::Pow(lhs, rhs, extend_dimensions));
 
+XLA_MAKE_BINARY(NextAfter, xla::NextAfter(lhs, rhs));
+
 #undef XLA_MAKE_BINARY
 
 class ApproximateEqualOp : public XlaOpKernel {
diff --git a/tensorflow/compiler/tf2xla/kernels/cast_op.cc b/tensorflow/compiler/tf2xla/kernels/cast_op.cc
index db58c2e651..ca2152d6c1 100644
--- a/tensorflow/compiler/tf2xla/kernels/cast_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/cast_op.cc
@@ -36,19 +36,6 @@ class CastOp : public XlaOpKernel {
     OP_REQUIRES_OK(ctx, ctx->GetAttr("Truncate", &use_truncation_));
   }
 
-  xla::PrimitiveType GetUnsignedIntTypeOfSameWidth(int64 src_bitwidth) {
-    switch (src_bitwidth) {
-      case 16:
-        return xla::U16;
-      case 32:
-        return xla::U32;
-      case 64:
-        return xla::U64;
-      default:
-        return xla::PRIMITIVE_TYPE_INVALID;
-    }
-  }
-
   void Compile(XlaOpKernelContext* ctx) override {
     xla::XlaBuilder* builder = ctx->builder();
     xla::XlaOp input = ctx->Input(0);
@@ -85,7 +72,7 @@ class CastOp : public XlaOpKernel {
         // source datatype.
         int64 mask = ~((1L << mantissa_difference) - 1);
         xla::PrimitiveType same_width_int =
-            GetUnsignedIntTypeOfSameWidth(src_bitwidth);
+            xla::primitive_util::UnsignedIntegralTypeForBitWidth(src_bitwidth);
         OP_REQUIRES(ctx, same_width_int != xla::PRIMITIVE_TYPE_INVALID,
                     errors::Unimplemented("Unexpected type bitwidth"));
         input = xla::BitcastConvertType(
diff --git a/tensorflow/compiler/xla/client/lib/math.cc b/tensorflow/compiler/xla/client/lib/math.cc
index c56c371ae5..b6378b02da 100644
--- a/tensorflow/compiler/xla/client/lib/math.cc
+++ b/tensorflow/compiler/xla/client/lib/math.cc
@@ -389,4 +389,82 @@ XlaOp MaybeConjugate(XlaOp x, bool conjugate) {
   });
 }
 
+XlaOp NextAfter(XlaOp from, XlaOp to) {
+  auto builder = from.builder();
+  return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    TF_ASSIGN_OR_RETURN(auto shape, builder->GetShape(from));
+    int bitwidth = primitive_util::BitWidth(shape.element_type());
+    auto int_type = primitive_util::UnsignedIntegralTypeForBitWidth(bitwidth);
+    auto from_as_int = BitcastConvertType(from, int_type);
+    auto to_as_int = BitcastConvertType(to, int_type);
+
+    // The result is NaN if either "from" or "to" are NaN.
+    auto from_is_nan = Ne(from, from);
+    auto to_is_nan = Ne(to, to);
+    auto nan_input = Or(from_is_nan, to_is_nan);
+    auto result_for_nan =
+        Broadcast(ScalarLike(from, std::numeric_limits<double>::quiet_NaN()),
+                  shape.dimensions());
+    result_for_nan = BitcastConvertType(result_for_nan, int_type);
+
+    // The sign bit is the MSB.
+    const int64 sign_mask = int64{1} << (bitwidth - 1);
+    // Discard the sign bit to make the result non-negative.
+    auto from_abs = And(from_as_int, ScalarLike(from_as_int, ~sign_mask));
+    auto to_abs = And(to_as_int, ScalarLike(to_as_int, ~sign_mask));
+
+    // When both "from" and "to" are equal, the result is "to".
+    // N.B. It would not make a difference if we chose the result to be "from".
+    auto from_and_to_are_equal = Eq(from_as_int, to_as_int);
+    auto result_for_equal = to_as_int;
+
+    // When both "from" and "to" are both 0, the result is "to". This ensures we
+    // get a zero signed like "to".
+    auto from_is_zero = Eq(from_abs, ZerosLike(from_abs));
+    auto to_is_zero = Eq(to_abs, ZerosLike(to_abs));
+    auto result_for_both_zero = to_as_int;
+
+    auto from_sign = And(from_as_int, ScalarLike(from_as_int, sign_mask));
+    auto to_sign = And(to_as_int, ScalarLike(to_as_int, sign_mask));
+
+    // If from == 0 && to != 0, we need to return the smallest subnormal number
+    // signed like "to".
+    auto result_for_from_zero_to_non_zero =
+        Or(to_sign, ScalarLike(from_as_int, 1));
+
+    // If the sign of "from" and "to" disagree:
+    // - we need to make the magnitude of "from" smaller so that it is closer to
+    //   zero.
+    //
+    // Otherwise the signs agree:
+    // - "from" with a magnitude larger than "to" means we need to make the
+    //   magnitude smaller.
+    // - "from" with a magnitude smaller than "to" means we need to make the
+    //   magnitude larger.
+    // - "from" with the same magnitude and sign as "to" has already been
+    //   handled.
+    auto signs_disagree = Ne(from_sign, to_sign);
+    auto from_magnitude_larger_than_to = Gt(from_abs, to_abs);
+    auto result_has_smaller_magnitude =
+        Or(from_magnitude_larger_than_to, signs_disagree);
+    auto magnitude_adjustment =
+        Select(result_has_smaller_magnitude,
+               Broadcast(ScalarLike(from_as_int, -1), shape.dimensions()),
+               Broadcast(ScalarLike(from_as_int, 1), shape.dimensions()));
+    auto result = Add(from_as_int, magnitude_adjustment);
+    // Handle from == ±0.
+    result = Select(from_is_zero,
+                    Select(to_is_zero, result_for_both_zero,
+                           result_for_from_zero_to_non_zero),
+                    result);
+    // Handle from == to.
+    result = Select(from_and_to_are_equal, result_for_equal, result);
+    // Handle isnan(from) || isnan(to).
+    result = Select(nan_input, result_for_nan, result);
+
+    // Cast back to the original type.
+    return BitcastConvertType(result, shape.element_type());
+  });
+}
+
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/client/lib/math.h b/tensorflow/compiler/xla/client/lib/math.h
index 17612bf9fd..583481c7f3 100644
--- a/tensorflow/compiler/xla/client/lib/math.h
+++ b/tensorflow/compiler/xla/client/lib/math.h
@@ -32,7 +32,7 @@ XlaOp Square(XlaOp operand);
 // Computes the reciprocal of 'operand'.
 XlaOp Reciprocal(XlaOp operand);
 
-// Evaluates a polynomial given coefficients and `x`.
+// Evaluates a polynomial given coefficients and 'x'.
 // N.B. Coefficients should be supplied in decreasing order.
 XlaOp EvaluatePolynomial(XlaOp x, absl::Span<const float> coefficients);
 
@@ -86,10 +86,14 @@ XlaOp Cosh(XlaOp x);
 // Computes the hyperbolic sine of 'x'.
 XlaOp Sinh(XlaOp x);
 
-// Applies a complex conjugation operation if `a` is complex and `conjugate`
+// Applies a complex conjugation operation if 'a' is complex and 'conjugate'
 // is true, otherwise returns its argument.
 xla::XlaOp MaybeConjugate(xla::XlaOp x, bool conjugate);
 
+// Returns the next number after 'from' in the direction of 'to' the same way
+// std::nextafter(from, to) would.
+XlaOp NextAfter(XlaOp from, XlaOp to);
+
 }  // namespace xla
 
 #endif  // TENSORFLOW_COMPILER_XLA_CLIENT_LIB_MATH_H_
diff --git a/tensorflow/compiler/xla/primitive_util.cc b/tensorflow/compiler/xla/primitive_util.cc
index 3386d2e097..1eedddf72c 100644
--- a/tensorflow/compiler/xla/primitive_util.cc
+++ b/tensorflow/compiler/xla/primitive_util.cc
@@ -97,6 +97,21 @@ int BitWidth(PrimitiveType type) {
   }
 }
 
+xla::PrimitiveType UnsignedIntegralTypeForBitWidth(int64 src_bitwidth) {
+  switch (src_bitwidth) {
+    case 8:
+      return xla::U8;
+    case 16:
+      return xla::U16;
+    case 32:
+      return xla::U32;
+    case 64:
+      return xla::U64;
+    default:
+      return xla::PRIMITIVE_TYPE_INVALID;
+  }
+}
+
 PrimitiveType ComplexComponentType(PrimitiveType complex_type) {
   switch (complex_type) {
     case C64:
diff --git a/tensorflow/compiler/xla/primitive_util.h b/tensorflow/compiler/xla/primitive_util.h
index d32505335d..295d353003 100644
--- a/tensorflow/compiler/xla/primitive_util.h
+++ b/tensorflow/compiler/xla/primitive_util.h
@@ -151,6 +151,8 @@ bool IsArrayType(PrimitiveType primitive_type);
 // Returns the number of bits in the representation for a given type.
 int BitWidth(PrimitiveType type);
 
+PrimitiveType UnsignedIntegralTypeForBitWidth(int64 src_bitwidth);
+
 // Returns the real, imag component type underlying the given complex type.
 // LOG(FATAL)'s if complex_type is not complex.
 PrimitiveType ComplexComponentType(PrimitiveType complex_type);
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index 4200f09abd..946577d55d 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -2721,7 +2721,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
   }
   return InvalidArgument(
       "Select operation with non-scalar predicate with dimensionality "
-      " different from the other operands: %s.",
+      "different from the other operands: %s.",
       ShapeUtil::HumanString(pred));
 }
 
-- 
GitLab


From 9e8e2c265f8274f547d853171bbd5e4377761f44 Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Thu, 24 Jan 2019 16:03:30 -0800
Subject: [PATCH 1218/2345] [Grappler] Add fanin mutations by port to
 MutableGraphView.

PiperOrigin-RevId: 230808071
---
 .../core/grappler/mutable_graph_view.cc       | 378 ++++++---
 tensorflow/core/grappler/mutable_graph_view.h |  43 +
 .../core/grappler/mutable_graph_view_test.cc  | 752 +++++++++++++++++-
 3 files changed, 1064 insertions(+), 109 deletions(-)

diff --git a/tensorflow/core/grappler/mutable_graph_view.cc b/tensorflow/core/grappler/mutable_graph_view.cc
index 3b203661c5..724efcd21e 100644
--- a/tensorflow/core/grappler/mutable_graph_view.cc
+++ b/tensorflow/core/grappler/mutable_graph_view.cc
@@ -102,24 +102,72 @@ Status MutationError(absl::string_view function_name, absl::string_view params,
       "MutableGraphView::$0($1) error: $2.", function_name, params, msg));
 }
 
-string NodeMissingErrorMsg(absl::string_view node_name) {
-  return absl::Substitute("node '$0' was not found", node_name);
+using ErrorHandler = std::function<Status(absl::string_view)>;
+
+ErrorHandler UpdateFanoutsError(absl::string_view from_node_name,
+                                absl::string_view to_node_name) {
+  return [from_node_name, to_node_name](absl::string_view msg) {
+    string params = absl::Substitute("from_node_name='$0', to_node_name='$1'",
+                                     from_node_name, to_node_name);
+    return MutationError("UpdateFanouts", params, msg);
+  };
 }
 
-string InvalidFaninErrorMsg(absl::string_view fanin) {
-  return absl::Substitute("fanin '$0' must be a valid tensor id", fanin);
+Status CheckFaninIsRegular(const TensorId& fanin, ErrorHandler handler) {
+  if (!IsTensorIdRegular(fanin)) {
+    return handler(absl::Substitute("fanin '$0' must be a regular tensor id",
+                                    fanin.ToString()));
+  }
+  return Status::OK();
+}
+
+Status CheckFaninIsValid(const TensorId& fanin, ErrorHandler handler) {
+  if (!IsTensorIdPortValid(fanin)) {
+    return handler(absl::Substitute("fanin '$0' must be a valid tensor id",
+                                    fanin.ToString()));
+  }
+  return Status::OK();
 }
 
-string RegularFaninErrorMsg(absl::string_view fanin) {
-  return absl::Substitute("fanin '$0' must be a regular tensor id", fanin);
+Status CheckAddingFaninToSelf(absl::string_view node_name,
+                              const TensorId& fanin, ErrorHandler handler) {
+  if (node_name == fanin.node()) {
+    return handler(
+        absl::Substitute("can't add fanin '$0' to self", fanin.ToString()));
+  }
+  return Status::OK();
 }
 
-Status UpdateFanoutsError(absl::string_view from_node_name,
-                          absl::string_view to_node_name,
-                          absl::string_view msg) {
-  string params = absl::Substitute("from_node_name='$0', to_node_name='$1'",
-                                   from_node_name, to_node_name);
-  return MutationError("UpdateFanouts", params, msg);
+Status CheckRemovingFaninFromSelf(absl::string_view node_name,
+                                  const TensorId& fanin, ErrorHandler handler) {
+  if (node_name == fanin.node()) {
+    return handler(absl::Substitute("can't remove fanin '$0' from self",
+                                    fanin.ToString()));
+  }
+  return Status::OK();
+}
+
+string NodeMissingErrorMsg(absl::string_view node_name) {
+  return absl::Substitute("node '$0' was not found", node_name);
+}
+
+Status CheckNodeExists(absl::string_view node_name, NodeDef* node,
+                       ErrorHandler handler) {
+  if (node == nullptr) {
+    return handler(NodeMissingErrorMsg(node_name));
+  }
+  return Status::OK();
+}
+
+Status CheckPortRange(int port, int min, int max, ErrorHandler handler) {
+  if (port < min || port > max) {
+    if (max < min) {
+      return handler("no available ports as node has no regular fanins");
+    }
+    return handler(
+        absl::Substitute("port must be in range [$0, $1]", min, max));
+  }
+  return Status::OK();
 }
 
 }  // namespace
@@ -195,6 +243,13 @@ void MutableGraphView::UpdateMaxRegularOutputPortForRemovedFanin(
   }
 }
 
+void MutableGraphView::UpdateMaxRegularOutputPortForAddedFanin(
+    const OutputPort& fanin) {
+  if (max_regular_output_port()[fanin.node] < fanin.port_id) {
+    max_regular_output_port()[fanin.node] = fanin.port_id;
+  }
+}
+
 const absl::flat_hash_set<MutableGraphView::InputPort>&
 MutableGraphView::GetFanout(const GraphView::OutputPort& port) const {
   return GetFanout(MutableGraphView::OutputPort(const_cast<NodeDef*>(port.node),
@@ -275,15 +330,12 @@ Status MutableGraphView::AddSubgraph(GraphDef&& subgraph) {
 Status MutableGraphView::UpdateFanouts(absl::string_view from_node_name,
                                        absl::string_view to_node_name) {
   NodeDef* from_node = GetNode(from_node_name);
-  if (from_node == nullptr) {
-    return UpdateFanoutsError(from_node_name, to_node_name,
-                              NodeMissingErrorMsg(from_node_name));
-  }
+  TF_RETURN_IF_ERROR(
+      CheckNodeExists(from_node_name, from_node,
+                      UpdateFanoutsError(from_node_name, to_node_name)));
   NodeDef* to_node = GetNode(to_node_name);
-  if (to_node == nullptr) {
-    return UpdateFanoutsError(from_node_name, to_node_name,
-                              NodeMissingErrorMsg(to_node_name));
-  }
+  TF_RETURN_IF_ERROR(CheckNodeExists(
+      to_node_name, to_node, UpdateFanoutsError(from_node_name, to_node_name)));
 
   return UpdateFanoutsInternal(from_node, to_node);
 }
@@ -323,8 +375,7 @@ Status MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
     if (to_node_is_switch) {
       // Trying to add a Switch as a control dependency, which if allowed will
       // make the graph invalid.
-      return UpdateFanoutsError(
-          from_node->name(), to_node->name(),
+      return UpdateFanoutsError(from_node->name(), to_node->name())(
           absl::Substitute("can't update fanouts to node '$0' as it will "
                            "become a Switch control dependency",
                            to_node->name()));
@@ -410,7 +461,7 @@ bool MutableGraphView::AddFaninInternal(NodeDef* node,
 
   node->add_input(TensorIdToString({fanin.node->name(), fanin.port_id}));
   if (!input_is_control) {
-    int last_node_input = node->input_size() - 1;
+    const int last_node_input = node->input_size() - 1;
     // If there are control dependencies in node, move newly inserted fanin to
     // be before such control dependencies.
     if (num_regular_fanins < last_node_input) {
@@ -442,26 +493,61 @@ Status MutableGraphView::AddRegularFanin(absl::string_view node_name,
     return MutationError("AddRegularFanin", params, msg);
   };
 
-  if (!IsTensorIdRegular(fanin)) {
-    return error_status(RegularFaninErrorMsg(fanin.ToString()));
-  }
-  if (node_name == fanin.node()) {
-    return error_status(absl::Substitute("can't add regular fanin '$0' to self",
-                                         fanin.ToString()));
-  }
+  TF_RETURN_IF_ERROR(CheckFaninIsRegular(fanin, error_status));
+  TF_RETURN_IF_ERROR(CheckAddingFaninToSelf(node_name, fanin, error_status));
   NodeDef* node = GetNode(node_name);
-  if (node == nullptr) {
-    return error_status(NodeMissingErrorMsg(node_name));
-  }
+  TF_RETURN_IF_ERROR(CheckNodeExists(node_name, node, error_status));
   NodeDef* fanin_node = GetNode(fanin.node());
-  if (fanin_node == nullptr) {
-    return error_status(NodeMissingErrorMsg(fanin.node()));
-  }
+  TF_RETURN_IF_ERROR(CheckNodeExists(fanin.node(), fanin_node, error_status));
 
   AddFaninInternal(node, {fanin_node, fanin.index()});
   return Status::OK();
 }
 
+Status MutableGraphView::AddRegularFaninByPort(absl::string_view node_name,
+                                               int port,
+                                               const TensorId& fanin) {
+  auto error_status = [node_name, port, fanin](absl::string_view msg) {
+    string params = absl::Substitute("node_name='$0', port=$1, fanin='$2'",
+                                     node_name, port, fanin.ToString());
+    return MutationError("AddRegularFaninByPort", params, msg);
+  };
+
+  TF_RETURN_IF_ERROR(CheckFaninIsRegular(fanin, error_status));
+  TF_RETURN_IF_ERROR(CheckAddingFaninToSelf(node_name, fanin, error_status));
+  NodeDef* node = GetNode(node_name);
+  TF_RETURN_IF_ERROR(CheckNodeExists(node_name, node, error_status));
+  const int num_regular_fanins =
+      NumFanins(*node, /*include_controlling_nodes=*/false);
+  TF_RETURN_IF_ERROR(
+      CheckPortRange(port, /*min=*/0, num_regular_fanins, error_status));
+  NodeDef* fanin_node = GetNode(fanin.node());
+  TF_RETURN_IF_ERROR(CheckNodeExists(fanin.node(), fanin_node, error_status));
+
+  const int last_node_input = node->input_size();
+  node->add_input(TensorIdToString(fanin));
+  node->mutable_input()->SwapElements(num_regular_fanins, last_node_input);
+  for (int i = num_regular_fanins - 1; i >= port; --i) {
+    TensorId tensor_id = ParseTensorName(node->input(i));
+    OutputPort fanin_port(nodes()[tensor_id.node()], tensor_id.index());
+    absl::flat_hash_set<InputPort>* fanouts_set = &fanouts()[fanin_port];
+    fanouts_set->erase({node, i});
+    fanouts_set->insert({node, i + 1});
+    node->mutable_input()->SwapElements(i, i + 1);
+  }
+
+  OutputPort fanin_port(fanin_node, fanin.index());
+  fanouts()[fanin_port].insert({node, port});
+  UpdateMaxRegularOutputPortForAddedFanin(fanin_port);
+
+  max_regular_input_port()[node] = num_regular_fanins;
+  if (CanDedupControlWithRegularInput(*this, *fanin_node)) {
+    RemoveControllingFaninInternal(node, fanin_node);
+  }
+
+  return Status::OK();
+}
+
 Status MutableGraphView::AddControllingFanin(absl::string_view node_name,
                                              const TensorId& fanin) {
   auto error_status = [node_name, fanin](absl::string_view msg) {
@@ -470,31 +556,21 @@ Status MutableGraphView::AddControllingFanin(absl::string_view node_name,
     return MutationError("AddControllingFanin", params, msg);
   };
 
-  if (!IsTensorIdPortValid(fanin)) {
-    return error_status(InvalidFaninErrorMsg(fanin.ToString()));
-  }
-  if (node_name == fanin.node()) {
-    return error_status(absl::Substitute(
-        "can't add controlling fanin '$0' to self", fanin.ToString()));
-  }
+  TF_RETURN_IF_ERROR(CheckFaninIsValid(fanin, error_status));
+  TF_RETURN_IF_ERROR(CheckAddingFaninToSelf(node_name, fanin, error_status));
   NodeDef* node = GetNode(node_name);
-  if (node == nullptr) {
-    return error_status(NodeMissingErrorMsg(node_name));
-  }
+  TF_RETURN_IF_ERROR(CheckNodeExists(node_name, node, error_status));
   NodeDef* fanin_node = GetNode(fanin.node());
-  if (fanin_node == nullptr) {
-    return error_status(NodeMissingErrorMsg(fanin.node()));
-  }
+  TF_RETURN_IF_ERROR(CheckNodeExists(fanin.node(), fanin_node, error_status));
 
   if (!IsSwitch(*fanin_node)) {
     AddFaninInternal(node, {fanin_node, Graph::kControlSlot});
   } else {
     if (IsTensorIdControlling(fanin)) {
       // Can't add a Switch node control dependency.
-      return error_status(
-          absl::Substitute("can't add controlling fanin '$0' as it will become "
-                           "a Switch control dependency",
-                           fanin.ToString()));
+      return error_status(absl::Substitute(
+          "can't add fanin '$0' as it will become a Switch control dependency",
+          fanin.ToString()));
     }
     // We can't anchor control dependencies directly on the switch node: unlike
     // other nodes only one of the outputs of the switch node will be generated
@@ -507,9 +583,9 @@ Status MutableGraphView::AddControllingFanin(absl::string_view node_name,
       if (IsIdentity(*fanout.node) || IsIdentityNSingleInput(*fanout.node)) {
         if (ParseTensorName(fanout.node->input(0)) == fanin) {
           if (fanout.node->name() == node_name) {
-            return error_status(absl::Substitute(
-                "can't add found controlling fanin '$0' to self",
-                AsControlDependency(fanout.node->name())));
+            return error_status(
+                absl::Substitute("can't add found fanin '$0' to self",
+                                 AsControlDependency(fanout.node->name())));
           }
           AddFaninInternal(node, {fanout.node, Graph::kControlSlot});
           return Status::OK();
@@ -522,7 +598,7 @@ Status MutableGraphView::AddControllingFanin(absl::string_view node_name,
         absl::StrCat(fanin.node(), "_", fanin.index()), kMutableGraphViewCtrl);
     if (node_name == ctrl_dep_name) {
       return error_status(
-          absl::Substitute("can't add generated controlling fanin '$0' to self",
+          absl::Substitute("can't add generated fanin '$0' to self",
                            AsControlDependency(ctrl_dep_name)));
     }
 
@@ -607,26 +683,61 @@ Status MutableGraphView::RemoveRegularFanin(absl::string_view node_name,
     return MutationError("RemoveRegularFanin", params, msg);
   };
 
-  if (!IsTensorIdRegular(fanin)) {
-    return error_status(RegularFaninErrorMsg(fanin.ToString()));
-  }
-  if (node_name == fanin.node()) {
-    return error_status(absl::Substitute(
-        "can't remove regular fanin '$0' from self", fanin.ToString()));
-  }
+  TF_RETURN_IF_ERROR(CheckFaninIsRegular(fanin, error_status));
+  TF_RETURN_IF_ERROR(
+      CheckRemovingFaninFromSelf(node_name, fanin, error_status));
   NodeDef* node = GetNode(node_name);
-  if (node == nullptr) {
-    return error_status(NodeMissingErrorMsg(node_name));
-  }
+  TF_RETURN_IF_ERROR(CheckNodeExists(node_name, node, error_status));
   NodeDef* fanin_node = GetNode(fanin.node());
-  if (fanin_node == nullptr) {
-    return error_status(NodeMissingErrorMsg(fanin.node()));
-  }
+  TF_RETURN_IF_ERROR(CheckNodeExists(fanin.node(), fanin_node, error_status));
 
   RemoveRegularFaninInternal(node, {fanin_node, fanin.index()});
   return Status::OK();
 }
 
+Status MutableGraphView::RemoveRegularFaninByPort(absl::string_view node_name,
+                                                  int port) {
+  auto error_status = [node_name, port](absl::string_view msg) {
+    string params =
+        absl::Substitute("node_name='$0', port=$1", node_name, port);
+    return MutationError("RemoveRegularFaninByPort", params, msg);
+  };
+
+  NodeDef* node = GetNode(node_name);
+  TF_RETURN_IF_ERROR(CheckNodeExists(node_name, node, error_status));
+  const int last_regular_fanin_port =
+      gtl::FindWithDefault(max_regular_input_port(), node, -1);
+  TF_RETURN_IF_ERROR(
+      CheckPortRange(port, /*min=*/0, last_regular_fanin_port, error_status));
+
+  TensorId tensor_id = ParseTensorName(node->input(port));
+  OutputPort fanin_port(nodes()[tensor_id.node()], tensor_id.index());
+  fanouts()[fanin_port].erase({node, port});
+  auto mutable_inputs = node->mutable_input();
+  for (int i = port + 1; i <= last_regular_fanin_port; ++i) {
+    TensorId tensor_id = ParseTensorName(node->input(i));
+    OutputPort fanin_port(nodes()[tensor_id.node()], tensor_id.index());
+    absl::flat_hash_set<InputPort>* fanouts_set = &fanouts()[fanin_port];
+    fanouts_set->erase({node, i});
+    fanouts_set->insert({node, i - 1});
+    mutable_inputs->SwapElements(i - 1, i);
+  }
+  const int last_node_input = node->input_size() - 1;
+  if (last_regular_fanin_port < last_node_input) {
+    mutable_inputs->SwapElements(last_regular_fanin_port, last_node_input);
+  }
+  mutable_inputs->RemoveLast();
+
+  const int updated_last_regular_input_port = last_regular_fanin_port - 1;
+  if (updated_last_regular_input_port < 0) {
+    max_regular_input_port().erase(node);
+  } else {
+    max_regular_input_port()[node] = updated_last_regular_input_port;
+  }
+
+  return Status::OK();
+}
+
 bool MutableGraphView::RemoveControllingFaninInternal(NodeDef* node,
                                                       NodeDef* fanin_node) {
   for (int i = node->input_size() - 1; i >= 0; --i) {
@@ -653,19 +764,13 @@ Status MutableGraphView::RemoveControllingFanin(
     return MutationError("RemoveControllingFanin", params, msg);
   };
 
-  if (node_name == fanin_node_name) {
-    return error_status(
-        absl::Substitute("can't remove controlling fanin '$0' from self",
-                         AsControlDependency(string(fanin_node_name))));
-  }
+  TF_RETURN_IF_ERROR(CheckRemovingFaninFromSelf(
+      node_name, {fanin_node_name, Graph::kControlSlot}, error_status));
   NodeDef* node = GetNode(node_name);
-  if (node == nullptr) {
-    return error_status(NodeMissingErrorMsg(node_name));
-  }
+  TF_RETURN_IF_ERROR(CheckNodeExists(node_name, node, error_status));
   NodeDef* fanin_node = GetNode(fanin_node_name);
-  if (fanin_node == nullptr) {
-    return error_status(NodeMissingErrorMsg(fanin_node_name));
-  }
+  TF_RETURN_IF_ERROR(
+      CheckNodeExists(fanin_node_name, fanin_node, error_status));
 
   RemoveControllingFaninInternal(node, fanin_node);
   return Status::OK();
@@ -674,7 +779,6 @@ Status MutableGraphView::RemoveControllingFanin(
 Status MutableGraphView::RemoveAllFanins(absl::string_view node_name,
                                          bool keep_controlling_fanins) {
   NodeDef* node = GetNode(node_name);
-
   if (node == nullptr) {
     string params =
         absl::Substitute("node_name='$0', keep_controlling_fanins=$1",
@@ -714,24 +818,16 @@ Status MutableGraphView::UpdateFanin(absl::string_view node_name,
     return MutationError("UpdateFanin", params, msg);
   };
 
-  if (!IsTensorIdPortValid(from_fanin)) {
-    return error_status(InvalidFaninErrorMsg(from_fanin.ToString()));
-  }
-  if (!IsTensorIdPortValid(to_fanin)) {
-    return error_status(InvalidFaninErrorMsg(to_fanin.ToString()));
-  }
+  TF_RETURN_IF_ERROR(CheckFaninIsValid(from_fanin, error_status));
+  TF_RETURN_IF_ERROR(CheckFaninIsValid(to_fanin, error_status));
   NodeDef* node = GetNode(node_name);
-  if (node == nullptr) {
-    return error_status(NodeMissingErrorMsg(node_name));
-  }
+  TF_RETURN_IF_ERROR(CheckNodeExists(node_name, node, error_status));
   NodeDef* from_fanin_node = GetNode(from_fanin.node());
-  if (from_fanin_node == nullptr) {
-    return error_status(NodeMissingErrorMsg(from_fanin.node()));
-  }
+  TF_RETURN_IF_ERROR(
+      CheckNodeExists(from_fanin.node(), from_fanin_node, error_status));
   NodeDef* to_fanin_node = GetNode(to_fanin.node());
-  if (to_fanin_node == nullptr) {
-    return error_status(NodeMissingErrorMsg(from_fanin.node()));
-  }
+  TF_RETURN_IF_ERROR(
+      CheckNodeExists(to_fanin.node(), to_fanin_node, error_status));
 
   // When replacing a non control dependency fanin with a control dependency, or
   // vice versa, remove and add, so ports can be updated properly in fanout(s).
@@ -808,6 +904,94 @@ Status MutableGraphView::UpdateFanin(absl::string_view node_name,
   return Status::OK();
 }
 
+Status MutableGraphView::UpdateRegularFaninByPort(absl::string_view node_name,
+                                                  int port,
+                                                  const TensorId& fanin) {
+  auto error_status = [node_name, port, fanin](absl::string_view msg) {
+    string params = absl::Substitute("node_name='$0', port=$1, fanin='$2'",
+                                     node_name, port, fanin.ToString());
+    return MutationError("UpdateRegularFaninByPort", params, msg);
+  };
+
+  TF_RETURN_IF_ERROR(CheckFaninIsRegular(fanin, error_status));
+  TF_RETURN_IF_ERROR(CheckAddingFaninToSelf(node_name, fanin, error_status));
+  NodeDef* node = GetNode(node_name);
+  TF_RETURN_IF_ERROR(CheckNodeExists(node_name, node, error_status));
+  const int last_regular_fanin_port =
+      gtl::FindWithDefault(max_regular_input_port(), node, -1);
+  TF_RETURN_IF_ERROR(
+      CheckPortRange(port, /*min=*/0, last_regular_fanin_port, error_status));
+  NodeDef* fanin_node = GetNode(fanin.node());
+  TF_RETURN_IF_ERROR(CheckNodeExists(fanin.node(), fanin_node, error_status));
+
+  TensorId tensor_id = ParseTensorName(node->input(port));
+  if (tensor_id == fanin) {
+    return Status::OK();
+  }
+
+  InputPort input(node, port);
+  OutputPort from_fanin_port(nodes()[tensor_id.node()], tensor_id.index());
+  absl::flat_hash_set<InputPort>* from_fanouts = &fanouts()[from_fanin_port];
+  from_fanouts->erase(input);
+  UpdateMaxRegularOutputPortForRemovedFanin(from_fanin_port, *from_fanouts);
+
+  OutputPort to_fanin_port(fanin_node, fanin.index());
+  fanouts()[to_fanin_port].insert(input);
+  UpdateMaxRegularOutputPortForAddedFanin(to_fanin_port);
+
+  node->set_input(port, TensorIdToString(fanin));
+
+  if (CanDedupControlWithRegularInput(*this, *fanin_node)) {
+    RemoveControllingFaninInternal(node, fanin_node);
+  }
+
+  return Status::OK();
+}
+
+Status MutableGraphView::SwapRegularFaninsByPorts(absl::string_view node_name,
+                                                  int from_port, int to_port) {
+  auto error_status = [node_name, from_port, to_port](absl::string_view msg) {
+    string params = absl::Substitute("node_name='$0', from_port=$1, to_port=$2",
+                                     node_name, from_port, to_port);
+    return MutationError("SwapRegularFaninsByPorts", params, msg);
+  };
+
+  NodeDef* node = GetNode(node_name);
+  TF_RETURN_IF_ERROR(CheckNodeExists(node_name, node, error_status));
+  const int last_regular_fanin_port =
+      gtl::FindWithDefault(max_regular_input_port(), node, -1);
+  TF_RETURN_IF_ERROR(CheckPortRange(from_port, /*min=*/0,
+                                    last_regular_fanin_port, error_status));
+  TF_RETURN_IF_ERROR(CheckPortRange(to_port, /*min=*/0, last_regular_fanin_port,
+                                    error_status));
+
+  if (from_port == to_port) {
+    return Status::OK();
+  }
+  TensorId from_fanin = ParseTensorName(node->input(from_port));
+  TensorId to_fanin = ParseTensorName(node->input(to_port));
+  if (from_fanin == to_fanin) {
+    return Status::OK();
+  }
+
+  InputPort from_input(node, from_port);
+  InputPort to_input(node, to_port);
+  NodeDef* from_fanin_node = GetNode(from_fanin.node());
+  absl::flat_hash_set<InputPort>* from_fanouts =
+      &fanouts()[{from_fanin_node, from_fanin.index()}];
+  from_fanouts->erase(from_input);
+  from_fanouts->insert(to_input);
+  NodeDef* to_fanin_node = GetNode(to_fanin.node());
+  absl::flat_hash_set<InputPort>* to_fanouts =
+      &fanouts()[{to_fanin_node, to_fanin.index()}];
+  to_fanouts->erase(to_input);
+  to_fanouts->insert(from_input);
+
+  node->mutable_input()->SwapElements(from_port, to_port);
+
+  return Status::OK();
+}
+
 Status MutableGraphView::CheckNodesCanBeDeleted(
     const absl::flat_hash_set<string>& nodes_to_delete) {
   std::vector<string> missing_nodes;
diff --git a/tensorflow/core/grappler/mutable_graph_view.h b/tensorflow/core/grappler/mutable_graph_view.h
index b4d07bbd57..16ef832a4e 100644
--- a/tensorflow/core/grappler/mutable_graph_view.h
+++ b/tensorflow/core/grappler/mutable_graph_view.h
@@ -98,6 +98,17 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // AddControllingFanin.
   Status AddRegularFanin(absl::string_view node_name, const TensorId& fanin);
 
+  // Adds regular fanin `fanin` to node `node_name` at port `port`. If the node
+  // or fanin do not exist in the graph, nothing will be modified in the graph.
+  // Otherwise fanin will be inserted at port `port`. Control dependencies will
+  // be deduped. To add control dependencies, use AddControllingFanin.
+  //
+  // If the port is not a valid port (less than 0 or greater than the number of
+  // regular fanins), this will result in an error and the node will not be
+  // modified.
+  Status AddRegularFaninByPort(absl::string_view node_name, int port,
+                               const TensorId& fanin);
+
   // Adds control dependency `fanin` to the target node named `node_name`. To
   // add regular fanins, use AddRegularFanin.
   //
@@ -131,6 +142,15 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // not result in an error and the node will not be modified.
   Status RemoveRegularFanin(absl::string_view node_name, const TensorId& fanin);
 
+  // Removes regular fanin at port `port` from node `node_name`. If the node
+  // does not exist in the graph, nothing will be modified in the graph.
+  // To remove controlling fanins, use RemoveControllingFanin.
+  //
+  // If the port is not a valid port (less than 0 or greater than the last index
+  // of the regular fanins), this will result in an error and the node will not
+  // be modified.
+  Status RemoveRegularFaninByPort(absl::string_view node_name, int port);
+
   // Removes control dependency `fanin_node_name` from the target node named
   // `node_name`. If the node or fanin do not exist in the graph, nothing will
   // be modified in the graph. To remove regular fanins, use RemoveRegualrFanin.
@@ -157,6 +177,25 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   Status UpdateFanin(absl::string_view node_name, const TensorId& from_fanin,
                      const TensorId& to_fanin);
 
+  // Replaces fanin at port `port` in node `node_name` with fanin `fanin`. If
+  // the fanins or node do not exist, nothing will be modified in the graph.
+  // Control dependencies will be deduped.
+  //
+  // If the port is not a valid port (less than 0 or greater than the last index
+  // of the regular fanins), this will result in an error and the node will not
+  // be modified.
+  Status UpdateRegularFaninByPort(absl::string_view node_name, int port,
+                                  const TensorId& fanin);
+
+  // Swaps fanins at ports `from_port` and `to_port` in node `node_name`. If the
+  // node does not exist, nothing will be modified in the graph.
+  //
+  // If the ports are not a valid port (less than 0 or greater than the last
+  // index of the regular fanins), this will result in an error and the node
+  // will not be modified.
+  Status SwapRegularFaninsByPorts(absl::string_view node_name, int from_port,
+                                  int to_port);
+
   // Deletes nodes from the graph. If a node can't be safely removed,
   // specifically if a node still has fanouts, an error will be returned. Nodes
   // that can't be found are ignored.
@@ -176,6 +215,10 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
       const OutputPort& fanin,
       const absl::flat_hash_set<InputPort>& fanin_fanouts);
 
+  // Updates max regular output port for newly added fanin by checking the
+  // current max and updating if the newly added fanin is of a larger port.
+  void UpdateMaxRegularOutputPortForAddedFanin(const OutputPort& fanin);
+
   // Updates all fanouts (input ports fetching output tensors) from `from_node`
   // to the `to_node`, including control dependencies.
   //
diff --git a/tensorflow/core/grappler/mutable_graph_view_test.cc b/tensorflow/core/grappler/mutable_graph_view_test.cc
index 7c7dff07ae..f4ee4d89b1 100644
--- a/tensorflow/core/grappler/mutable_graph_view_test.cc
+++ b/tensorflow/core/grappler/mutable_graph_view_test.cc
@@ -522,11 +522,155 @@ TEST(MutableGraphViewTest, AddRegularFanin) {
   // Add self to create cycle.
   error_msg =
       "MutableGraphView::AddRegularFanin(node_name='foo_6', fanin='foo_6:2') "
-      "error: can't add regular fanin 'foo_6:2' to self.";
+      "error: can't add fanin 'foo_6:2' to self.";
   TestAddRegularFanin("foo_6", /*node_exists=*/true, {"foo_6", 2},
                       /*success=*/false, error_msg, {"^a", "^b"});
 }
 
+void TestAddRegularFaninByPort(absl::string_view node_name, bool node_exists,
+                               int port, const TensorId& fanin_to_add,
+                               bool success, const string& error_msg,
+                               absl::Span<const string> expected_fanins) {
+  GraphDef graph_def = SimpleMutateFaninGraph();
+
+  MutableGraphView graph(&graph_def);
+
+  NodeDef* node = graph.GetNode(node_name);
+  if (node_exists) {
+    EXPECT_NE(node, nullptr);
+  } else {
+    EXPECT_EQ(node, nullptr);
+  }
+
+  absl::flat_hash_map<string, std::vector<string>> unmodified_node_inputs =
+      GetNodeInputsFromGraph(graph_def, node_name);
+
+  Status s = graph.AddRegularFaninByPort(node_name, port, fanin_to_add);
+  EXPECT_EQ(s.ok(), success);
+  if (!success) {
+    EXPECT_EQ(s.error_message(), error_msg);
+  }
+  if (node_exists) {
+    CompareNodeFanins(graph, node, expected_fanins);
+  }
+
+  CheckUnmodifiedNodeFanins(graph_def, node_name, unmodified_node_inputs);
+
+  CheckGraph(graph);
+}
+
+TEST(MutableGraphViewTest, AddRegularFaninByPort) {
+  string error_msg;
+  // Add input at start to node with some inputs and no controls.
+  TestAddRegularFaninByPort("foo_3", /*node_exists=*/true, /*port=*/0, {"d", 2},
+                            /*success=*/true, error_msg,
+                            {"d:2", "b", "a:1", "a:1"});
+  // Add input at end to node with some inputs and no controls.
+  TestAddRegularFaninByPort("foo_3", /*node_exists=*/true, /*port=*/3, {"d", 2},
+                            /*success=*/true, error_msg,
+                            {"b", "a:1", "a:1", "d:2"});
+  // Add input in middle to node with some inputs and no controls.
+  TestAddRegularFaninByPort("foo_3", /*node_exists=*/true, /*port=*/2, {"d", 2},
+                            /*success=*/true, error_msg,
+                            {"b", "a:1", "d:2", "a:1"});
+  // Add input at start to node with some inputs and some controls.
+  TestAddRegularFaninByPort("foo_2", /*node_exists=*/true, /*port=*/0, {"d", 2},
+                            /*success=*/true, error_msg,
+                            {"d:2", "b", "^c", "^a"});
+  // Add input at end to node with some inputs and some controls.
+  TestAddRegularFaninByPort("foo_2", /*node_exists=*/true, /*port=*/1, {"d", 2},
+                            /*success=*/true, error_msg,
+                            {"b", "d:2", "^c", "^a"});
+  // Add input in middle to node with some inputs and some controls, and dedup
+  // controls.
+  TestAddRegularFaninByPort("foo_4", /*node_exists=*/true, /*port=*/2, {"d", 2},
+                            /*success=*/true, error_msg,
+                            {"a", "b:2", "d:2", "b:2", "^c"});
+  // Add input to node with no inputs and no controls.
+  TestAddRegularFaninByPort("foo_5", /*node_exists=*/true, /*port=*/0, {"d", 2},
+                            /*success=*/true, error_msg, {"d:2"});
+  // Add input to node with no inputs and some controls.
+  TestAddRegularFaninByPort("foo_6", /*node_exists=*/true, /*port=*/0, {"d", 2},
+                            /*success=*/true, error_msg, {"d:2", "^b", "^a"});
+  // Add fanin should dedup control.
+  TestAddRegularFaninByPort("foo_6", /*node_exists=*/true, /*port=*/0, {"b", 2},
+                            /*success=*/true, error_msg, {"b:2", "^a"});
+
+  // Add controlling fanin.
+  error_msg =
+      "MutableGraphView::AddRegularFaninByPort(node_name='foo_4', port=2, "
+      "fanin='^d') error: fanin '^d' must be a regular tensor id.";
+  TestAddRegularFaninByPort(
+      "foo_4", /*node_exists=*/true, /*port=*/2, {"d", Graph::kControlSlot},
+      /*success=*/false, error_msg, {"a", "b:2", "b:2", "^c", "^d"});
+
+  // Add fanin at out of bounds port.
+  error_msg =
+      "MutableGraphView::AddRegularFaninByPort(node_name='foo_5', port=-1, "
+      "fanin='d:2') error: port must be in range [0, 0].";
+  TestAddRegularFaninByPort("foo_5", /*node_exists=*/true, /*port=*/-1,
+                            {"d", 2},
+                            /*success=*/false, error_msg, {});
+  error_msg =
+      "MutableGraphView::AddRegularFaninByPort(node_name='foo_5', port=1, "
+      "fanin='d:2') error: port must be in range [0, 0].";
+  TestAddRegularFaninByPort("foo_5", /*node_exists=*/true, /*port=*/1, {"d", 2},
+                            /*success=*/false, error_msg, {});
+  error_msg =
+      "MutableGraphView::AddRegularFaninByPort(node_name='foo_6', port=-1, "
+      "fanin='d:2') error: port must be in range [0, 0].";
+  TestAddRegularFaninByPort("foo_6", /*node_exists=*/true, /*port=*/-1,
+                            {"d", 2},
+                            /*success=*/false, error_msg, {"^a", "^b"});
+  error_msg =
+      "MutableGraphView::AddRegularFaninByPort(node_name='foo_6', port=1, "
+      "fanin='d:2') error: port must be in range [0, 0].";
+  TestAddRegularFaninByPort("foo_6", /*node_exists=*/true, /*port=*/1, {"d", 2},
+                            /*success=*/false, error_msg, {"^a", "^b"});
+  error_msg =
+      "MutableGraphView::AddRegularFaninByPort(node_name='foo_4', port=-1, "
+      "fanin='d:2') error: port must be in range [0, 3].";
+  TestAddRegularFaninByPort(
+      "foo_4", /*node_exists=*/true, /*port=*/-1, {"d", 2},
+      /*success=*/false, error_msg, {"a", "b:2", "b:2", "^c", "^d"});
+  error_msg =
+      "MutableGraphView::AddRegularFaninByPort(node_name='foo_4', port=4, "
+      "fanin='d:2') error: port must be in range [0, 3].";
+  TestAddRegularFaninByPort("foo_4", /*node_exists=*/true, /*port=*/4, {"d", 2},
+                            /*success=*/false, error_msg,
+                            {"a", "b:2", "b:2", "^c", "^d"});
+
+  // Add fanin to node where node is missing.
+  error_msg =
+      "MutableGraphView::AddRegularFaninByPort(node_name='foo_missing', "
+      "port=0, fanin='a:0') error: node 'foo_missing' was not found.";
+  TestAddRegularFaninByPort("foo_missing", /*node_exists=*/false, /*port=*/0,
+                            {"a", 0},
+                            /*success=*/false, error_msg, {});
+  // Add fanin to node where fanin is missing.
+  error_msg =
+      "MutableGraphView::AddRegularFaninByPort(node_name='foo_1', port=0, "
+      "fanin='bar_missing:0') error: node 'bar_missing' was not found.";
+  TestAddRegularFaninByPort("foo_1", /*node_exists=*/true, /*port=*/0,
+                            {"bar_missing", 0},
+                            /*success=*/false, error_msg, {"a"});
+  // Add fanin to node where node and fanin are missing.
+  error_msg =
+      "MutableGraphView::AddRegularFaninByPort(node_name='foo_missing', "
+      "port=0, fanin='bar_missing:0') error: node 'foo_missing' was not found.";
+  TestAddRegularFaninByPort("foo_missing", /*node_exists=*/false, /*port=*/0,
+                            {"bar_missing", 0},
+                            /*success=*/false, error_msg, {});
+
+  // Add self to create cycle.
+  error_msg =
+      "MutableGraphView::AddRegularFaninByPort(node_name='foo_6', port=0, "
+      "fanin='foo_6:2') error: can't add fanin 'foo_6:2' to self.";
+  TestAddRegularFaninByPort("foo_6", /*node_exists=*/true, /*port=*/0,
+                            {"foo_6", 2},
+                            /*success=*/false, error_msg, {"^a", "^b"});
+}
+
 void CheckFanoutRemoved(const MutableGraphView& graph, const TensorId& fanin,
                         absl::string_view node_name) {
   MutableGraphView::OutputPort output_port =
@@ -674,11 +818,114 @@ TEST(MutableGraphViewTest, RemoveRegularFanin) {
   // Remove self.
   error_msg =
       "MutableGraphView::RemoveRegularFanin(node_name='foo_6', "
-      "fanin='foo_6:2') error: can't remove regular fanin 'foo_6:2' from self.";
+      "fanin='foo_6:2') error: can't remove fanin 'foo_6:2' from self.";
   TestRemoveRegularFanin("foo_6", /*node_exists=*/true, {"foo_6", 2},
                          /*success=*/false, error_msg, {"^a", "^b"});
 }
 
+void TestRemoveRegularFaninByPort(absl::string_view node_name, bool node_exists,
+                                  int port, bool success,
+                                  const string& error_msg,
+                                  absl::Span<const string> expected_fanins) {
+  GraphDef graph_def = SimpleMutateFaninGraph();
+
+  MutableGraphView graph(&graph_def);
+
+  NodeDef* node = graph.GetNode(node_name);
+  if (node_exists) {
+    EXPECT_NE(nullptr, node);
+  } else {
+    EXPECT_EQ(nullptr, node);
+  }
+
+  absl::flat_hash_map<string, std::vector<string>> unmodified_node_inputs =
+      GetNodeInputsFromGraph(graph_def, node_name);
+
+  Status s = graph.RemoveRegularFaninByPort(node_name, port);
+  EXPECT_EQ(s.ok(), success);
+  if (!success) {
+    EXPECT_EQ(s.error_message(), error_msg);
+  }
+  if (node_exists) {
+    CompareNodeFanins(graph, node, expected_fanins);
+  }
+
+  CheckUnmodifiedNodeFanins(graph_def, node_name, unmodified_node_inputs);
+
+  CheckGraph(graph);
+}
+
+TEST(MutableGraphViewTest, RemoveRegularFaninByPort) {
+  string error_msg;
+  // Remove input at start of node with some inputs and no controls.
+  TestRemoveRegularFaninByPort("foo_3", /*node_exists=*/true, /*port=*/0,
+                               /*success=*/true, error_msg, {"a:1", "a:1"});
+  // Remove input at end of node with some inputs and no controls.
+  TestRemoveRegularFaninByPort("foo_3", /*node_exists=*/true, /*port=*/2,
+                               /*success=*/true, error_msg, {"b", "a:1"});
+  // Remove input in middle of node with some inputs and no controls.
+  TestRemoveRegularFaninByPort("foo_3", /*node_exists=*/true, /*port=*/1,
+                               /*success=*/true, error_msg, {"b", "a:1"});
+  // Remove input at start of node with some inputs and some controls.
+  TestRemoveRegularFaninByPort("foo_4", /*node_exists=*/true, /*port=*/0,
+                               /*success=*/true, error_msg,
+                               {"b:2", "b:2", "^d", "^c"});
+  // Remove input at end of node with some inputs and some controls.
+  TestRemoveRegularFaninByPort("foo_4", /*node_exists=*/true, /*port=*/2,
+                               /*success=*/true, error_msg,
+                               {"a", "b:2", "^d", "^c"});
+  // Remove input in middle of node with some inputs and some controls.
+  TestRemoveRegularFaninByPort("foo_4", /*node_exists=*/true, /*port=*/1,
+                               /*success=*/true, error_msg,
+                               {"a", "b:2", "^d", "^c"});
+
+  // Remove input from node with no inputs and no controls.
+  error_msg =
+      "MutableGraphView::RemoveRegularFaninByPort(node_name='foo_5', port=0) "
+      "error: no available ports as node has no regular fanins.";
+  TestRemoveRegularFaninByPort("foo_5", /*node_exists=*/true, /*port=*/0,
+                               /*success=*/false, error_msg, {});
+  // Remove input from node with no inputs and some controls.
+  error_msg =
+      "MutableGraphView::RemoveRegularFaninByPort(node_name='foo_6', port=1) "
+      "error: no available ports as node has no regular fanins.";
+  TestRemoveRegularFaninByPort("foo_6", /*node_exists=*/true, /*port=*/1,
+                               /*success=*/false, error_msg, {"^a", "^b"});
+
+  // Remove fanin at out of bounds port.
+  error_msg =
+      "MutableGraphView::RemoveRegularFaninByPort(node_name='foo_3', port=-1) "
+      "error: port must be in range [0, 2].";
+  TestRemoveRegularFaninByPort("foo_3", /*node_exists=*/true, /*port=*/-1,
+                               /*success=*/false, error_msg,
+                               {"b", "a:1", "a:1"});
+  error_msg =
+      "MutableGraphView::RemoveRegularFaninByPort(node_name='foo_3', port=3) "
+      "error: port must be in range [0, 2].";
+  TestRemoveRegularFaninByPort("foo_3", /*node_exists=*/true, /*port=*/3,
+                               /*success=*/false, error_msg,
+                               {"b", "a:1", "a:1"});
+  error_msg =
+      "MutableGraphView::RemoveRegularFaninByPort(node_name='foo_4', port=-1) "
+      "error: port must be in range [0, 2].";
+  TestRemoveRegularFaninByPort("foo_4", /*node_exists=*/true, /*port=*/-1,
+                               /*success=*/false, error_msg,
+                               {"a", "b:2", "b:2", "^c", "^d"});
+  error_msg =
+      "MutableGraphView::RemoveRegularFaninByPort(node_name='foo_4', port=3) "
+      "error: port must be in range [0, 2].";
+  TestRemoveRegularFaninByPort("foo_4", /*node_exists=*/true, /*port=*/3,
+                               /*success=*/false, error_msg,
+                               {"a", "b:2", "b:2", "^c", "^d"});
+
+  // Remove fanin from node where node is missing.
+  error_msg =
+      "MutableGraphView::RemoveRegularFaninByPort(node_name='foo_missing', "
+      "port=0) error: node 'foo_missing' was not found.";
+  TestRemoveRegularFaninByPort("foo_missing", /*node_exists=*/false, /*port=*/0,
+                               /*success=*/false, error_msg, {});
+}
+
 void TestRemoveAllFanins(absl::string_view node_name, bool node_exists,
                          bool keep_controlling_nodes, bool success,
                          const string& error_msg,
@@ -860,7 +1107,8 @@ TEST(MutableGraphViewTest, UpdateFanin) {
   // Update fanin of node where to fanin is missing.
   error_msg =
       "MutableGraphView::UpdateFanin(node_name='foo_1', from_fanin='a:0', "
-      "to_fanin='to_bar_missing:1') error: node 'a' was not found.";
+      "to_fanin='to_bar_missing:1') error: node 'to_bar_missing' was not "
+      "found.";
   TestUpdateFanin("foo_1", /*node_exists=*/true, {"a", 0},
                   {"to_bar_missing", 1}, /*success=*/false, error_msg, {"a"});
   // Update fanin of node where from/to fanins and node are missing.
@@ -954,6 +1202,394 @@ TEST(MutableGraphViewTest, UpdateFaninToNodeAsSwitchControl) {
   TestUpdateFaninFromFaninToNodeAsSwitchControl({"a", Graph::kControlSlot});
 }
 
+void TestUpdateRegularFaninByPort(absl::string_view node_name, bool node_exists,
+                                  int port, const TensorId& fanin, bool success,
+                                  const string& error_msg,
+                                  absl::Span<const string> expected_fanins) {
+  GraphDef graph_def = SimpleMutateFaninGraph();
+
+  MutableGraphView graph(&graph_def);
+
+  NodeDef* node = graph.GetNode(node_name);
+  if (node_exists) {
+    EXPECT_NE(node, nullptr);
+  } else {
+    EXPECT_EQ(node, nullptr);
+  }
+
+  absl::flat_hash_map<string, std::vector<string>> unmodified_node_inputs =
+      GetNodeInputsFromGraph(graph_def, node_name);
+
+  Status s = graph.UpdateRegularFaninByPort(node_name, port, fanin);
+  EXPECT_EQ(s.ok(), success);
+  if (!success) {
+    EXPECT_EQ(s.error_message(), error_msg);
+  }
+  if (node_exists) {
+    CompareNodeFanins(graph, node, expected_fanins);
+  }
+
+  CheckUnmodifiedNodeFanins(graph_def, node_name, unmodified_node_inputs);
+
+  CheckGraph(graph);
+}
+
+TEST(MutableGraphViewTest, UpdateRegularFaninByPort) {
+  string error_msg;
+  // Update input at start to node with some inputs and no controls.
+  TestUpdateRegularFaninByPort(
+      "foo_3", /*node_exists=*/true, /*port=*/0, {"d", 2},
+      /*success=*/true, error_msg, {"d:2", "a:1", "a:1"});
+  // Update input at end to node with some inputs and no controls.
+  TestUpdateRegularFaninByPort(
+      "foo_3", /*node_exists=*/true, /*port=*/2, {"d", 2},
+      /*success=*/true, error_msg, {"b", "a:1", "d:2"});
+  // Update input in middle to node with some inputs and no controls.
+  TestUpdateRegularFaninByPort(
+      "foo_3", /*node_exists=*/true, /*port=*/1, {"d", 2},
+      /*success=*/true, error_msg, {"b", "d:2", "a:1"});
+  // Update input at start to node with some inputs and some controls, and dedup
+  // controls.
+  TestUpdateRegularFaninByPort(
+      "foo_4", /*node_exists=*/true, /*port=*/0, {"d", 2},
+      /*success=*/true, error_msg, {"d:2", "b:2", "b:2", "^c"});
+  // Update input at end to node with some inputs and some controls, and dedup
+  // controls.
+  TestUpdateRegularFaninByPort(
+      "foo_4", /*node_exists=*/true, /*port=*/2, {"d", 2},
+      /*success=*/true, error_msg, {"a", "b:2", "d:2", "^c"});
+  // Update input in middle to node with some inputs and some controls and
+  // dedup controls.
+  TestUpdateRegularFaninByPort(
+      "foo_4", /*node_exists=*/true, /*port=*/1, {"d", 2},
+      /*success=*/true, error_msg, {"a", "d:2", "b:2", "^c"});
+
+  // Update input to controlling fanin.
+  error_msg =
+      "MutableGraphView::UpdateRegularFaninByPort(node_name='foo_4', port=1, "
+      "fanin='^d') error: fanin '^d' must be a regular tensor id.";
+  TestUpdateRegularFaninByPort(
+      "foo_4", /*node_exists=*/true, /*port=*/1, {"d", Graph::kControlSlot},
+      /*success=*/false, error_msg, {"a", "b:2", "b:2", "^c", "^d"});
+
+  // Update fanin at out of bounds port.
+  error_msg =
+      "MutableGraphView::UpdateRegularFaninByPort(node_name='foo_5', port=-1, "
+      "fanin='d:2') error: no available ports as node has no regular fanins.";
+  TestUpdateRegularFaninByPort("foo_5", /*node_exists=*/true, /*port=*/-1,
+                               {"d", 2},
+                               /*success=*/false, error_msg, {});
+  error_msg =
+      "MutableGraphView::UpdateRegularFaninByPort(node_name='foo_5', port=0, "
+      "fanin='d:2') error: no available ports as node has no regular fanins.";
+  TestUpdateRegularFaninByPort("foo_5", /*node_exists=*/true, /*port=*/0,
+                               {"d", 2},
+                               /*success=*/false, error_msg, {});
+  error_msg =
+      "MutableGraphView::UpdateRegularFaninByPort(node_name='foo_5', port=1, "
+      "fanin='d:2') error: no available ports as node has no regular fanins.";
+  TestUpdateRegularFaninByPort("foo_5", /*node_exists=*/true, /*port=*/1,
+                               {"d", 2},
+                               /*success=*/false, error_msg, {});
+  error_msg =
+      "MutableGraphView::UpdateRegularFaninByPort(node_name='foo_6', port=-1, "
+      "fanin='d:2') error: no available ports as node has no regular fanins.";
+  TestUpdateRegularFaninByPort("foo_6", /*node_exists=*/true, /*port=*/-1,
+                               {"d", 2},
+                               /*success=*/false, error_msg, {"^a", "^b"});
+  error_msg =
+      "MutableGraphView::UpdateRegularFaninByPort(node_name='foo_6', port=0, "
+      "fanin='d:2') error: no available ports as node has no regular fanins.";
+  TestUpdateRegularFaninByPort("foo_6", /*node_exists=*/true, /*port=*/0,
+                               {"d", 2},
+                               /*success=*/false, error_msg, {"^a", "^b"});
+  error_msg =
+      "MutableGraphView::UpdateRegularFaninByPort(node_name='foo_6', port=1, "
+      "fanin='d:2') error: no available ports as node has no regular fanins.";
+  TestUpdateRegularFaninByPort("foo_6", /*node_exists=*/true, /*port=*/1,
+                               {"d", 2},
+                               /*success=*/false, error_msg, {"^a", "^b"});
+  error_msg =
+      "MutableGraphView::UpdateRegularFaninByPort(node_name='foo_3', port=-1, "
+      "fanin='d:2') error: port must be in range [0, 2].";
+  TestUpdateRegularFaninByPort(
+      "foo_3", /*node_exists=*/true, /*port=*/-1, {"d", 2},
+      /*success=*/false, error_msg, {"b", "a:1", "a:1"});
+  error_msg =
+      "MutableGraphView::UpdateRegularFaninByPort(node_name='foo_3', port=3, "
+      "fanin='d:2') error: port must be in range [0, 2].";
+  TestUpdateRegularFaninByPort(
+      "foo_3", /*node_exists=*/true, /*port=*/3, {"d", 2},
+      /*success=*/false, error_msg, {"b", "a:1", "a:1"});
+  error_msg =
+      "MutableGraphView::UpdateRegularFaninByPort(node_name='foo_4', port=-1, "
+      "fanin='d:2') error: port must be in range [0, 2].";
+  TestUpdateRegularFaninByPort(
+      "foo_4", /*node_exists=*/true, /*port=*/-1, {"d", 2},
+      /*success=*/false, error_msg, {"a", "b:2", "b:2", "^c", "^d"});
+  error_msg =
+      "MutableGraphView::UpdateRegularFaninByPort(node_name='foo_4', port=3, "
+      "fanin='d:2') error: port must be in range [0, 2].";
+  TestUpdateRegularFaninByPort(
+      "foo_4", /*node_exists=*/true, /*port=*/3, {"d", 2},
+      /*success=*/false, error_msg, {"a", "b:2", "b:2", "^c", "^d"});
+
+  // Update fanin to node where node is missing.
+  error_msg =
+      "MutableGraphView::UpdateRegularFaninByPort(node_name='foo_missing', "
+      "port=0, fanin='a:0') error: node 'foo_missing' was not found.";
+  TestUpdateRegularFaninByPort("foo_missing", /*node_exists=*/false,
+                               /*port=*/0, {"a", 0},
+                               /*success=*/false, error_msg, {});
+  // Update fanin to node where fanin is missing.
+  error_msg =
+      "MutableGraphView::UpdateRegularFaninByPort(node_name='foo_1', port=0, "
+      "fanin='bar_missing:0') error: node 'bar_missing' was not "
+      "found.";
+  TestUpdateRegularFaninByPort("foo_1", /*node_exists=*/true, /*port=*/0,
+                               {"bar_missing", 0},
+                               /*success=*/false, error_msg, {"a"});
+  // Update fanin to node where node and fanin are missing.
+  error_msg =
+      "MutableGraphView::UpdateRegularFaninByPort(node_name='foo_missing', "
+      "port=0, fanin='bar_missing:0') error: node 'foo_missing' was not found.";
+  TestUpdateRegularFaninByPort("foo_missing", /*node_exists=*/false,
+                               /*port=*/0, {"bar_missing", 0},
+                               /*success=*/false, error_msg, {});
+
+  // Update self to create cycle.
+  error_msg =
+      "MutableGraphView::UpdateRegularFaninByPort(node_name='foo_6', port=0, "
+      "fanin='foo_6:2') error: can't add fanin 'foo_6:2' to self.";
+  TestUpdateRegularFaninByPort("foo_6", /*node_exists=*/true, /*port=*/0,
+                               {"foo_6", 2},
+                               /*success=*/false, error_msg, {"^a", "^b"});
+}
+
+void TestSwapRegularFaninsByPorts(absl::string_view node_name, bool node_exists,
+                                  int from_port, int to_port, bool success,
+                                  const string& error_msg,
+                                  absl::Span<const string> expected_fanins) {
+  GraphDef graph_def = SimpleMutateFaninGraph();
+
+  MutableGraphView graph(&graph_def);
+
+  NodeDef* node = graph.GetNode(node_name);
+  if (node_exists) {
+    EXPECT_NE(node, nullptr);
+  } else {
+    EXPECT_EQ(node, nullptr);
+  }
+
+  absl::flat_hash_map<string, std::vector<string>> unmodified_node_inputs =
+      GetNodeInputsFromGraph(graph_def, node_name);
+
+  Status s = graph.SwapRegularFaninsByPorts(node_name, from_port, to_port);
+  EXPECT_EQ(s.ok(), success);
+  if (!success) {
+    EXPECT_EQ(s.error_message(), error_msg);
+  }
+  if (node_exists) {
+    CompareNodeFanins(graph, node, expected_fanins);
+  }
+
+  CheckUnmodifiedNodeFanins(graph_def, node_name, unmodified_node_inputs);
+
+  CheckGraph(graph);
+}
+
+TEST(MutableGraphViewTest, SwapRegularFaninsByPorts) {
+  string error_msg;
+  // Swapping first and last regular fanins
+  TestSwapRegularFaninsByPorts("foo_3", /*node_exists=*/true, /*from_port=*/0,
+                               /*to_port=*/2, /*success=*/true, error_msg,
+                               {"a:1", "a:1", "b"});
+  TestSwapRegularFaninsByPorts("foo_3", /*node_exists=*/true, /*from_port=*/2,
+                               /*to_port=*/0, /*success=*/true, error_msg,
+                               {"a:1", "a:1", "b"});
+  // Swapping first and last regular fanins, in node with controls.
+  TestSwapRegularFaninsByPorts("foo_4", /*node_exists=*/true, /*from_port=*/0,
+                               /*to_port=*/2, /*success=*/true, error_msg,
+                               {"b:2", "b:2", "a", "^c", "^d"});
+  TestSwapRegularFaninsByPorts("foo_4", /*node_exists=*/true, /*from_port=*/2,
+                               /*to_port=*/0, /*success=*/true, error_msg,
+                               {"b:2", "b:2", "a", "^c", "^d"});
+  // Swapping middle regular fanin.
+  TestSwapRegularFaninsByPorts("foo_3", /*node_exists=*/true, /*from_port=*/0,
+                               /*to_port=*/1, /*success=*/true, error_msg,
+                               {"a:1", "b", "a:1"});
+  TestSwapRegularFaninsByPorts("foo_3", /*node_exists=*/true, /*from_port=*/1,
+                               /*to_port=*/0, /*success=*/true, error_msg,
+                               {"a:1", "b", "a:1"});
+  // Swapping middle regular fanin, in node with controls.
+  TestSwapRegularFaninsByPorts("foo_4", /*node_exists=*/true, /*from_port=*/0,
+                               /*to_port=*/1, /*success=*/true, error_msg,
+                               {"b:2", "a", "b:2", "^c", "^d"});
+  TestSwapRegularFaninsByPorts("foo_4", /*node_exists=*/true, /*from_port=*/1,
+                               /*to_port=*/0, /*success=*/true, error_msg,
+                               {"b:2", "a", "b:2", "^c", "^d"});
+  // Swapping same port.
+  TestSwapRegularFaninsByPorts("foo_4", /*node_exists=*/true, /*from_port=*/1,
+                               /*to_port=*/1, /*success=*/true, error_msg,
+                               {"a", "b:2", "b:2", "^c", "^d"});
+  // Swapping same fanin but different port.
+  TestSwapRegularFaninsByPorts("foo_4", /*node_exists=*/true, /*from_port=*/1,
+                               /*to_port=*/2, /*success=*/true, error_msg,
+                               {"a", "b:2", "b:2", "^c", "^d"});
+
+  // Swaping fanins at out of bounds ports.
+  // Node with no regular fanins and no controls.
+  error_msg =
+      "MutableGraphView::SwapRegularFaninsByPorts(node_name='foo_5', "
+      "from_port=-1, to_port=0) error: no available ports as node has no "
+      "regular fanins.";
+  TestSwapRegularFaninsByPorts("foo_5", /*node_exists=*/true, /*from_port=*/-1,
+                               /*to_port=*/0, /*success=*/false, error_msg, {});
+  error_msg =
+      "MutableGraphView::SwapRegularFaninsByPorts(node_name='foo_5', "
+      "from_port=0, to_port=-1) error: no available ports as node has no "
+      "regular fanins.";
+  TestSwapRegularFaninsByPorts("foo_5", /*node_exists=*/true, /*from_port=*/0,
+                               /*to_port=*/-1, /*success=*/false, error_msg,
+                               {});
+  error_msg =
+      "MutableGraphView::SwapRegularFaninsByPorts(node_name='foo_5', "
+      "from_port=0, to_port=0) error: no available ports as node has no "
+      "regular fanins.";
+  TestSwapRegularFaninsByPorts("foo_5", /*node_exists=*/true, /*from_port=*/0,
+                               /*to_port=*/0, /*success=*/false, error_msg, {});
+  error_msg =
+      "MutableGraphView::SwapRegularFaninsByPorts(node_name='foo_5', "
+      "from_port=0, to_port=1) error: no available ports as node has no "
+      "regular fanins.";
+  TestSwapRegularFaninsByPorts("foo_5", /*node_exists=*/true, /*from_port=*/0,
+                               /*to_port=*/1, /*success=*/false, error_msg, {});
+  error_msg =
+      "MutableGraphView::SwapRegularFaninsByPorts(node_name='foo_5', "
+      "from_port=1, to_port=0) error: no available ports as node has no "
+      "regular fanins.";
+  TestSwapRegularFaninsByPorts("foo_5", /*node_exists=*/true, /*from_port=*/1,
+                               /*to_port=*/0, /*success=*/false, error_msg, {});
+  // Node with no regular fanins and some controls.
+  error_msg =
+      "MutableGraphView::SwapRegularFaninsByPorts(node_name='foo_6', "
+      "from_port=-1, to_port=0) error: no available ports as node has no "
+      "regular fanins.";
+  TestSwapRegularFaninsByPorts("foo_6", /*node_exists=*/true, /*from_port=*/-1,
+                               /*to_port=*/0, /*success=*/false, error_msg,
+                               {"^a", "^b"});
+  error_msg =
+      "MutableGraphView::SwapRegularFaninsByPorts(node_name='foo_6', "
+      "from_port=0, to_port=-1) error: no available ports as node has no "
+      "regular fanins.";
+  TestSwapRegularFaninsByPorts("foo_6", /*node_exists=*/true, /*from_port=*/0,
+                               /*to_port=*/-1, /*success=*/false, error_msg,
+                               {"^a", "^b"});
+  error_msg =
+      "MutableGraphView::SwapRegularFaninsByPorts(node_name='foo_6', "
+      "from_port=0, to_port=0) error: no available ports as node has no "
+      "regular fanins.";
+  TestSwapRegularFaninsByPorts("foo_6", /*node_exists=*/true, /*from_port=*/0,
+                               /*to_port=*/0, /*success=*/false, error_msg,
+                               {"^a", "^b"});
+  error_msg =
+      "MutableGraphView::SwapRegularFaninsByPorts(node_name='foo_6', "
+      "from_port=0, to_port=1) error: no available ports as node has no "
+      "regular fanins.";
+  TestSwapRegularFaninsByPorts("foo_6", /*node_exists=*/true, /*from_port=*/0,
+                               /*to_port=*/1, /*success=*/false, error_msg,
+                               {"^a", "^b"});
+  error_msg =
+      "MutableGraphView::SwapRegularFaninsByPorts(node_name='foo_6', "
+      "from_port=1, to_port=0) error: no available ports as node has no "
+      "regular fanins.";
+  TestSwapRegularFaninsByPorts("foo_6", /*node_exists=*/true, /*from_port=*/1,
+                               /*to_port=*/0, /*success=*/false, error_msg,
+                               {"^a", "^b"});
+  // Node with regular fanins and no controls.
+  error_msg =
+      "MutableGraphView::SwapRegularFaninsByPorts(node_name='foo_3', "
+      "from_port=-1, to_port=0) error: port must be in range [0, 2].";
+  TestSwapRegularFaninsByPorts("foo_3", /*node_exists=*/true, /*from_port=*/-1,
+                               /*to_port=*/0, /*success=*/false, error_msg,
+                               {"b", "a:1", "a:1"});
+  error_msg =
+      "MutableGraphView::SwapRegularFaninsByPorts(node_name='foo_3', "
+      "from_port=0, to_port=-1) error: port must be in range [0, 2].";
+  TestSwapRegularFaninsByPorts("foo_3", /*node_exists=*/true, /*from_port=*/0,
+                               /*to_port=*/-1, /*success=*/false, error_msg,
+                               {"b", "a:1", "a:1"});
+  error_msg =
+      "MutableGraphView::SwapRegularFaninsByPorts(node_name='foo_3', "
+      "from_port=0, to_port=3) error: port must be in range [0, 2].";
+  TestSwapRegularFaninsByPorts("foo_3", /*node_exists=*/true, /*from_port=*/0,
+                               /*to_port=*/3, /*success=*/false, error_msg,
+                               {"b", "a:1", "a:1"});
+  error_msg =
+      "MutableGraphView::SwapRegularFaninsByPorts(node_name='foo_3', "
+      "from_port=3, to_port=0) error: port must be in range [0, 2].";
+  TestSwapRegularFaninsByPorts("foo_3", /*node_exists=*/true, /*from_port=*/3,
+                               /*to_port=*/0, /*success=*/false, error_msg,
+                               {"b", "a:1", "a:1"});
+  error_msg =
+      "MutableGraphView::SwapRegularFaninsByPorts(node_name='foo_3', "
+      "from_port=-1, to_port=3) error: port must be in range [0, 2].";
+  TestSwapRegularFaninsByPorts("foo_3", /*node_exists=*/true, /*from_port=*/-1,
+                               /*to_port=*/3, /*success=*/false, error_msg,
+                               {"b", "a:1", "a:1"});
+  error_msg =
+      "MutableGraphView::SwapRegularFaninsByPorts(node_name='foo_3', "
+      "from_port=3, to_port=-1) error: port must be in range [0, 2].";
+  TestSwapRegularFaninsByPorts("foo_3", /*node_exists=*/true, /*from_port=*/3,
+                               /*to_port=*/-1, /*success=*/false, error_msg,
+                               {"b", "a:1", "a:1"});
+  // Node with regular fanins and controls.
+  error_msg =
+      "MutableGraphView::SwapRegularFaninsByPorts(node_name='foo_4', "
+      "from_port=-1, to_port=0) error: port must be in range [0, 2].";
+  TestSwapRegularFaninsByPorts("foo_4", /*node_exists=*/true, /*from_port=*/-1,
+                               /*to_port=*/0, /*success=*/false, error_msg,
+                               {"a", "b:2", "b:2", "^c", "^d"});
+  error_msg =
+      "MutableGraphView::SwapRegularFaninsByPorts(node_name='foo_4', "
+      "from_port=0, to_port=-1) error: port must be in range [0, 2].";
+  TestSwapRegularFaninsByPorts("foo_4", /*node_exists=*/true, /*from_port=*/0,
+                               /*to_port=*/-1, /*success=*/false, error_msg,
+                               {"a", "b:2", "b:2", "^c", "^d"});
+  error_msg =
+      "MutableGraphView::SwapRegularFaninsByPorts(node_name='foo_4', "
+      "from_port=0, to_port=3) error: port must be in range [0, 2].";
+  TestSwapRegularFaninsByPorts("foo_4", /*node_exists=*/true, /*from_port=*/0,
+                               /*to_port=*/3, /*success=*/false, error_msg,
+                               {"a", "b:2", "b:2", "^c", "^d"});
+  error_msg =
+      "MutableGraphView::SwapRegularFaninsByPorts(node_name='foo_4', "
+      "from_port=3, to_port=0) error: port must be in range [0, 2].";
+  TestSwapRegularFaninsByPorts("foo_4", /*node_exists=*/true, /*from_port=*/3,
+                               /*to_port=*/0, /*success=*/false, error_msg,
+                               {"a", "b:2", "b:2", "^c", "^d"});
+  error_msg =
+      "MutableGraphView::SwapRegularFaninsByPorts(node_name='foo_4', "
+      "from_port=-1, to_port=3) error: port must be in range [0, 2].";
+  TestSwapRegularFaninsByPorts("foo_4", /*node_exists=*/true, /*from_port=*/-1,
+                               /*to_port=*/3, /*success=*/false, error_msg,
+                               {"a", "b:2", "b:2", "^c", "^d"});
+  error_msg =
+      "MutableGraphView::SwapRegularFaninsByPorts(node_name='foo_4', "
+      "from_port=3, to_port=-1) error: port must be in range [0, 2].";
+  TestSwapRegularFaninsByPorts("foo_4", /*node_exists=*/true, /*from_port=*/3,
+                               /*to_port=*/-1, /*success=*/false, error_msg,
+                               {"a", "b:2", "b:2", "^c", "^d"});
+
+  // Swapping fanin to node where node is missing.
+  error_msg =
+      "MutableGraphView::SwapRegularFaninsByPorts(node_name='foo_missing', "
+      "from_port=0, to_port=1) error: node 'foo_missing' was not found.";
+  TestSwapRegularFaninsByPorts("foo_missing", /*node_exists=*/false,
+                               /*from_port=*/0, /*to_port=*/1,
+                               /*success=*/false, error_msg, {});
+}
+
 TEST(MutableGraphViewTest, DedupControllingFaninsOnGraphInit) {
   GraphDef graph_def = test::function::GDef(
       {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {}, {}),
@@ -1011,7 +1647,7 @@ TEST(MutableGraphViewTest, DedupControllingFaninsOnAddFanin) {
   CheckGraph(graph);
 }
 
-TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnAddFanin) {
+TEST(MutableGraphViewTest, NoDedupControllingFaninsOnAddFanin) {
   GraphDef graph_def = test::function::GDef(
       {NDef("a", "Switch", {}, {}), NDef("b", "Identity", {"a:1"}),
        NDef("c", "", {}, {}), NDef("d", "", {}, {})},
@@ -1025,12 +1661,61 @@ TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnAddFanin) {
   CheckNode(graph, "c", "", "", {}, {"b:2", "^b"}, {});
   TF_EXPECT_OK(graph.AddControllingFanin("c", {"b", Graph::kControlSlot}));
   CheckNode(graph, "c", "", "", {}, {"b:2", "^b"}, {});
+  TF_EXPECT_OK(graph.AddRegularFanin("c", {"b", 2}));
+  CheckNode(graph, "c", "", "", {}, {"b:2", "b:2", "^b"}, {});
 
   TF_EXPECT_OK(graph.AddControllingFanin("d", {"b", Graph::kControlSlot}));
   CheckNode(graph, "d", "", "", {}, {"^b"}, {});
   TF_EXPECT_OK(graph.AddControllingFanin("d", {"b", Graph::kControlSlot}));
   CheckNode(graph, "d", "", "", {}, {"^b"}, {});
 
+  CheckNode(graph, "a", "Switch", "", {}, {}, {"b"});
+  CheckNode(graph, "b", "Identity", "", {}, {"a:1"},
+            {"c:0", "c:1", "^c", "^d"});
+
+  CheckGraph(graph);
+}
+
+TEST(MutableGraphViewTest, DedupControllingFaninsOnAddFaninByPort) {
+  // Actual node.op() is not important in this test.
+  GraphDef graph_def =
+      test::function::GDef({NDef("a", "NotImportant", {}, {}),
+                            NDef("b", "NotImportant", {"c", "^a"}),
+                            NDef("c", "NotImportant", {"a:1"})},
+                           /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  TF_EXPECT_OK(graph.AddRegularFaninByPort("b", 0, {"a", 2}));
+  CheckNode(graph, "b", "NotImportant", "", {}, {"a:2", "c"}, {});
+
+  TF_EXPECT_OK(graph.AddControllingFanin("c", {"a", Graph::kControlSlot}));
+  CheckNode(graph, "c", "NotImportant", "", {}, {"a:1"}, {"b:1"});
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"b:0", "c:0"});
+
+  CheckGraph(graph);
+}
+
+TEST(MutableGraphViewTest, NoDedupControllingFaninsOnAddFaninByPort) {
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "Switch", {}, {}), NDef("b", "Identity", {"a:1"}),
+       NDef("c", "", {}, {}), NDef("d", "", {"c:2"}, {})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  TF_EXPECT_OK(graph.AddRegularFaninByPort("d", 1, {"b", 2}));
+  CheckNode(graph, "d", "", "", {}, {"c:2", "b:2"}, {});
+  TF_EXPECT_OK(graph.AddControllingFanin("d", {"b", Graph::kControlSlot}));
+  CheckNode(graph, "d", "", "", {}, {"c:2", "b:2", "^b"}, {});
+  TF_EXPECT_OK(graph.AddRegularFaninByPort("d", 0, {"b", 2}));
+  CheckNode(graph, "d", "", "", {}, {"b:2", "c:2", "b:2", "^b"}, {});
+
+  CheckNode(graph, "a", "Switch", "", {}, {}, {"b:0"});
+  CheckNode(graph, "b", "Identity", "", {}, {"a:1"}, {"d:0", "d:2", "^d"});
+  CheckNode(graph, "c", "", "", {}, {}, {"d:1"});
+
   CheckGraph(graph);
 }
 
@@ -1052,7 +1737,7 @@ TEST(MutableGraphViewTest, DedupControllingFaninsOnUpdateFanin) {
   CheckGraph(graph);
 }
 
-TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnUpdateFanin) {
+TEST(MutableGraphViewTest, NoDedupControllingFaninsOnUpdateFanin) {
   GraphDef graph_def = test::function::GDef(
       {NDef("a", "Switch", {}, {}), NDef("b", "Identity", {"a:1"}),
        NDef("c", "Identity", {"a:2"}), NDef("d", "NotImportant", {"c", "^b"}),
@@ -1071,6 +1756,50 @@ TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnUpdateFanin) {
   TF_EXPECT_OK(graph.UpdateFanin("e", {"c", 3}, {"c", Graph::kControlSlot}));
   CheckNode(graph, "e", "NotImportant", "", {}, {"^c"}, {});
 
+  CheckNode(graph, "a", "Switch", "", {}, {}, {"b:0", "c:0"});
+  CheckNode(graph, "b", "Identity", "", {}, {"a:1"}, {});
+  CheckNode(graph, "c", "Identity", "", {}, {"a:2"}, {"d:0", "^d", "^e"});
+
+  CheckGraph(graph);
+}
+
+TEST(MutableGraphViewTest, DedupControllingFaninsOnUpdateFaninByPort) {
+  // Actual node.op() is not important in this test.
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {}, {}),
+       NDef("c", "NotImportant", {"a:1", "^b"})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  TF_EXPECT_OK(graph.UpdateRegularFaninByPort("c", 0, {"b", 2}));
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {});
+  CheckNode(graph, "b", "NotImportant", "", {}, {}, {"c"});
+  CheckNode(graph, "c", "NotImportant", "", {}, {"b:2"}, {});
+
+  CheckGraph(graph);
+}
+
+TEST(MutableGraphViewTest, NoDedupControllingFaninsOnUpdateFaninByPort) {
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "Switch", {}, {}), NDef("b", "Identity", {"a:1"}),
+       NDef("c", "Identity", {"a:2"}), NDef("d", "NotImportant", {"c", "^b"}),
+       NDef("e", "NotImportant", {"b", "^c"})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  TF_EXPECT_OK(graph.UpdateRegularFaninByPort("d", 0, {"b", 1}));
+  CheckNode(graph, "d", "NotImportant", "", {}, {"b:1", "^b"}, {});
+
+  TF_EXPECT_OK(graph.UpdateRegularFaninByPort("e", 0, {"c", 2}));
+  CheckNode(graph, "e", "NotImportant", "", {}, {"c:2", "^c"}, {});
+
+  CheckNode(graph, "a", "Switch", "", {}, {}, {"b:0", "c:0"});
+  CheckNode(graph, "b", "Identity", "", {}, {"a:1"}, {"d:0", "^d"});
+  CheckNode(graph, "c", "Identity", "", {}, {"a:2"}, {"e:0", "^e"});
+
   CheckGraph(graph);
 }
 
@@ -1229,8 +1958,7 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitch) {
   EXPECT_FALSE(s.ok());
   string expected_msg =
       "MutableGraphView::AddControllingFanin(node_name='a', fanin='^b') error: "
-      "can't add controlling fanin '^b' as it will become a Switch control "
-      "dependency.";
+      "can't add fanin '^b' as it will become a Switch control dependency.";
   EXPECT_EQ(s.error_message(), expected_msg);
 
   ASSERT_EQ(graph.graph()->node_size(), 2);
@@ -1338,7 +2066,7 @@ void TestAddControllingFaninSelfLoops(absl::string_view node_name,
 TEST(MutableGraphViewTest, AddControllingFaninSelfLoops) {
   string error_msg =
       "MutableGraphView::AddControllingFanin(node_name='a', fanin='^a') error: "
-      "can't add controlling fanin '^a' to self.";
+      "can't add fanin '^a' to self.";
   TestAddControllingFaninSelfLoops("a", {"a", Graph::kControlSlot}, error_msg);
 
   // Adding Switch control dependency to Identity consumer. Node `c` is
@@ -1348,7 +2076,7 @@ TEST(MutableGraphViewTest, AddControllingFaninSelfLoops) {
   // be added.
   error_msg =
       "MutableGraphView::AddControllingFanin(node_name='c', fanin='b:0') "
-      "error: can't add found controlling fanin '^c' to self.";
+      "error: can't add found fanin '^c' to self.";
   TestAddControllingFaninSelfLoops("c", {"b", 0}, error_msg);
 
   // Adding Switch control dependency to Identity consumer. Node `d` is
@@ -1358,7 +2086,7 @@ TEST(MutableGraphViewTest, AddControllingFaninSelfLoops) {
   // be added.
   error_msg =
       "MutableGraphView::AddControllingFanin(node_name='d', fanin='b:1') "
-      "error: can't add found controlling fanin '^d' to self.";
+      "error: can't add found fanin '^d' to self.";
   TestAddControllingFaninSelfLoops("d", {"b", 1}, error_msg);
 }
 
@@ -1382,7 +2110,7 @@ TEST(MutableGraphViewTest, AddControllingFaninSelfLoopsGeneratedIdentity) {
   EXPECT_FALSE(s.ok());
   string expected_msg =
       "MutableGraphView::AddControllingFanin(node_name='ConstantFoldingCtrl/"
-      "b_1', fanin='b:1') error: can't add generated controlling fanin "
+      "b_1', fanin='b:1') error: can't add generated fanin "
       "'^ConstantFoldingCtrl/b_1' to self.";
   EXPECT_EQ(s.error_message(), expected_msg);
 
@@ -1475,7 +2203,7 @@ TEST(MutableGraphViewTest, RemoveControllingFaninSelfLoop) {
   EXPECT_FALSE(s.ok());
   string expected_msg =
       "MutableGraphView::RemoveControllingFanin(node_name='c', "
-      "fanin_node_name='c') error: can't remove controlling fanin '^c' from "
+      "fanin_node_name='c') error: can't remove fanin '^c' from "
       "self.";
   EXPECT_EQ(s.error_message(), expected_msg);
 
-- 
GitLab


From b87dc5c3fde3e081bd5e2330babaea947d58a01e Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 24 Jan 2019 16:11:32 -0800
Subject: [PATCH 1219/2345] [XLA] Add tests for log and log1p boundary cases.

Check that these functions behave correctly in some boundary cases.

PiperOrigin-RevId: 230809579
---
 .../compiler/xla/client/lib/math_test.cc      | 39 +++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/tensorflow/compiler/xla/client/lib/math_test.cc b/tensorflow/compiler/xla/client/lib/math_test.cc
index 9947e0cf83..c35c3c4d91 100644
--- a/tensorflow/compiler/xla/client/lib/math_test.cc
+++ b/tensorflow/compiler/xla/client/lib/math_test.cc
@@ -30,6 +30,45 @@ class MathTest : public ClientLibraryTestBase {
   ErrorSpec error_spec_{0.0001};
 };
 
+// Write TYPED_TESTs within the class definition so that we don't have to litter
+// "this->" everywhere.
+template <typename T>
+class MathTypedTest : public MathTest {
+ public:
+  void TestLogEdgeCases() {
+    SetFastMathDisabled(true);
+
+    XlaBuilder b(TestName());
+    Log(AddParam(LiteralUtil::CreateR1<T>({T{0.0}, T{-0.0}}), &b));
+    ComputeAndCompareR1<T>(&b,
+                           {-std::numeric_limits<T>::infinity(),
+                            -std::numeric_limits<T>::infinity()},
+                           {}, error_spec_);
+  }
+
+  void TestLog1pEdgeCases() {
+    SetFastMathDisabled(true);
+
+    XlaBuilder b(TestName());
+    Log1p(AddParam(LiteralUtil::CreateR1<T>({T{0.0}, T{-0.0}, T{-1.0}}), &b));
+    ComputeAndCompareR1<T>(
+        &b, {T{0.0}, T{-0.0}, -std::numeric_limits<T>::infinity()}, {},
+        error_spec_);
+  }
+};
+
+// TODO(b/123355973): Add bfloat16 to TestTypes once it's working.
+#ifdef XLA_BACKEND_DOES_NOT_SUPPORT_FLOAT16
+using TestTypes = ::testing::Types<float>;
+#else
+using TestTypes = ::testing::Types<float, Eigen::half>;
+#endif
+
+TYPED_TEST_CASE(MathTypedTest, TestTypes);
+
+XLA_TYPED_TEST(MathTypedTest, LogEdgeCases) { this->TestLogEdgeCases(); }
+XLA_TYPED_TEST(MathTypedTest, Log1pEdgeCases) { this->TestLog1pEdgeCases(); }
+
 XLA_TEST_F(MathTest, SqrtF32) {
   XlaBuilder builder(TestName());
   Literal zero_literal = LiteralUtil::Zero(PrimitiveType::F32);
-- 
GitLab


From 2aed47e715c55c23d31fbe4aedc22c9b1e7b322f Mon Sep 17 00:00:00 2001
From: "Li, Guizi" <guizi.li@intel.com>
Date: Fri, 25 Jan 2019 08:22:00 +0800
Subject: [PATCH 1220/2345] small typo fix

---
 tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc b/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc
index b20709fc07..9ffb78759e 100644
--- a/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc
@@ -505,7 +505,7 @@ class MklConvCustomBackpropFilterOp
           // filter_in_depth  | in_depth         | in_depth / group_count
           // For depthwise convolution, we have group_count == in_depth.
           // So here G = original I, and I = 1.
-          // And the GOIHW is mkldnn format, here is try to extract the TF
+          // And the GOIHW is mkldnn format, here we try to extract the TF
           // format, TF format is HWIO, as G = original I, so here is HWGO.
           TensorShape diff_filter_tf_shape(
               {bwd_output_dims[MklDnnFilterGroupDims::MKL_GROUP_FILTER_DIM_H],
-- 
GitLab


From b682bdd374ff453f2ebb85d9e8a2b3eeed79df4f Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Thu, 24 Jan 2019 16:14:47 -0800
Subject: [PATCH 1221/2345] Add config file generation for linux to hwloc
 library build file

PiperOrigin-RevId: 230810209
---
 third_party/hwloc/BUILD.bazel | 173 ++++++++++++++++++++++++++++++++++
 1 file changed, 173 insertions(+)

diff --git a/third_party/hwloc/BUILD.bazel b/third_party/hwloc/BUILD.bazel
index b73267d668..1f29301645 100644
--- a/third_party/hwloc/BUILD.bazel
+++ b/third_party/hwloc/BUILD.bazel
@@ -8,6 +8,9 @@ licenses(["notice"])
 
 exports_files(["LICENSE"])
 
+load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda")
+load("@org_tensorflow//third_party:common.bzl", "template_rule")
+
 COMMON_INCLUDE_COPTS = [
     "-I.",
     "-Ihwloc",
@@ -23,6 +26,176 @@ VAR_SETTINGS_COPTS = [
     "-DRUNSTATEDIR=",
 ]
 
+template_rule(
+    name = "include_hwloc_autogen_config_h",
+    src = "include/hwloc/autogen/config.h.in",
+    out = "include/hwloc/autogen/config.h",
+    substitutions = {
+        "#undef HWLOC_VERSION": "#define HWLOC_VERSION \"2.0.3\"",
+        "#undef HWLOC_VERSION_MAJOR": "#define HWLOC_VERSION_MAJOR 2",
+        "#undef HWLOC_VERSION_MINOR": "#define HWLOC_VERSION_MINOR 0",
+        "#undef HWLOC_VERSION_RELEASE": "#define HWLOC_VERSION_RELEASE 3",
+        "#undef HWLOC_VERSION_GREEK": "#define HWLOC_VERSION_GREEK \"\"",
+        "#undef HWLOC_LINUX_SYS": "#define HWLOC_LINUX_SYS 1",
+        "#undef hwloc_pid_t": "#define hwloc_pid_t pid_t",
+        "#undef hwloc_thread_t": "#define hwloc_thread_t pthread_t",
+        "#  undef HWLOC_HAVE_STDINT_H": "#  define HWLOC_HAVE_STDINT_H 1 ",
+        "#undef HWLOC_SYM_TRANSFORM": "#define HWLOC_SYM_TRANSFORM 0",
+        "#undef HWLOC_SYM_PREFIX": "#define HWLOC_SYM_PREFIX hwloc_",
+        "#undef HWLOC_SYM_PREFIX_CAPS": "#define HWLOC_SYM_PREFIX_CAPS HWLOC_",
+    },
+)
+
+template_rule(
+    name = "include_private_hwloc_autogen__config_h",
+    src = "include/private/autogen/config.h.in",
+    out = "include/private/autogen/config.h",
+    substitutions = {
+        "#undef HAVE_CLOCK_GETTIME": "#define HAVE_CLOCK_GETTIME 1",
+        "#undef HAVE_CTYPE_H": "#define HAVE_CTYPE_H 1",
+        "#undef HAVE_DECL_CTL_HW": "#define HAVE_DECL_CTL_HW 0",
+        "#undef HAVE_DECL_FABSF": "#define HAVE_DECL_FABSF 1",
+        "#undef HAVE_DECL_GETEXECNAME": "#define HAVE_DECL_GETEXECNAME 0",
+        "#undef HAVE_DECL_GETMODULEFILENAME": "#define HAVE_DECL_GETMODULEFILENAME 0",
+        "#undef HAVE_DECL_GETPROGNAME": "#define HAVE_DECL_GETPROGNAME 0",
+        "#undef HAVE_DECL_HW_NCPU": "#define HAVE_DECL_HW_NCPU 0",
+        "#undef HAVE_DECL_MODFF": "#define HAVE_DECL_MODFF 1",
+        "#undef HAVE_DECL_PTHREAD_GETAFFINITY_NP": "#define HAVE_DECL_PTHREAD_GETAFFINITY_NP 1",
+        "#undef HAVE_DECL_PTHREAD_SETAFFINITY_NP": "#define HAVE_DECL_PTHREAD_SETAFFINITY_NP 1",
+        "#undef HAVE_DECL_RUNNING_ON_VALGRIND": "#define HAVE_DECL_RUNNING_ON_VALGRIND 0",
+        "#undef HAVE_DECL_SCHED_GETCPU": "#define HAVE_DECL_SCHED_GETCPU 1",
+        "#undef HAVE_DECL_SNPRINTF": "#define HAVE_DECL_SNPRINTF 1",
+        "#undef HAVE_DECL_STRTOULL": "#define HAVE_DECL_STRTOULL 1",
+        "#undef HAVE_DECL__PUTENV": "#define HAVE_DECL__PUTENV 0",
+        "#undef HAVE_DECL__SC_LARGE_PAGESIZE": "#define HAVE_DECL__SC_LARGE_PAGESIZE 0",
+        "#undef HAVE_DECL__SC_NPROCESSORS_CONF": "#define HAVE_DECL__SC_NPROCESSORS_CONF 1",
+        "#undef HAVE_DECL__SC_NPROCESSORS_ONLN": "#define HAVE_DECL__SC_NPROCESSORS_ONLN 1",
+        "#undef HAVE_DECL__SC_NPROC_CONF": "#define HAVE_DECL__SC_NPROC_CONF 0",
+        "#undef HAVE_DECL__SC_NPROC_ONLN": "#define HAVE_DECL__SC_NPROC_ONLN 0",
+        "#undef HAVE_DECL__SC_PAGESIZE": "#define HAVE_DECL__SC_PAGESIZE 1",
+        "#undef HAVE_DECL__SC_PAGE_SIZE": "#define HAVE_DECL__SC_PAGE_SIZE 1",
+        "#undef HAVE_DECL__STRDUP": "#define HAVE_DECL__STRDUP 0",
+        "#undef HAVE_DIRENT_H": "#define HAVE_DIRENT_H 1",
+        "#undef HAVE_DLFCN_H": "#define HAVE_DLFCN_H 1",
+        "#undef HAVE_FFS": "#define HAVE_FFS 1",
+        "#undef HAVE_FFSL": "#define HAVE_FFSL 1",
+        "#undef HAVE_GETPAGESIZE": "#define HAVE_GETPAGESIZE 1",
+        "#undef HAVE_INTTYPES_H": "#define HAVE_INTTYPES_H 1",
+        "#undef HAVE_LANGINFO_H": "#define HAVE_LANGINFO_H 1",
+        "#undef HAVE_LOCALE_H": "#define HAVE_LOCALE_H 1",
+        "#undef HAVE_MALLOC_H": "#define HAVE_MALLOC_H 1",
+        "#undef HAVE_MEMALIGN": "#define HAVE_MEMALIGN 1",
+        "#undef HAVE_MEMORY_H": "#define HAVE_MEMORY_H 1",
+        "#undef HAVE_MKSTEMP": "#define HAVE_MKSTEMP 1",
+        "#undef HAVE_NL_LANGINFO": "#define HAVE_NL_LANGINFO 1",
+        "#undef HAVE_OPENAT": "#define HAVE_OPENAT 1",
+        "#undef HAVE_POSIX_MEMALIGN": "#define HAVE_POSIX_MEMALIGN 1",
+        "#undef HAVE_PROGRAM_INVOCATION_NAME": "#define HAVE_PROGRAM_INVOCATION_NAME 1",
+        "#undef HAVE_PTHREAD_T": "#define HAVE_PTHREAD_T 1",
+        "#undef HAVE_PUTWC": "#define HAVE_PUTWC 1",
+        "#undef HAVE_SETLOCALE": "#define HAVE_SETLOCALE 1",
+        "#undef HAVE_SSIZE_T": "#define HAVE_SSIZE_T 1",
+        "#undef HAVE_STDINT_H": "#define HAVE_STDINT_H 1",
+        "#undef HAVE_STDLIB_H": "#define HAVE_STDLIB_H 1",
+        "#undef HAVE_STRCASECMP": "#define HAVE_STRCASECMP 1",
+        "#undef HAVE_STRFTIME": "#define HAVE_STRFTIME 1",
+        "#undef HAVE_STRINGS_H": "#define HAVE_STRINGS_H 1",
+        "#undef HAVE_STRING_H": "#define HAVE_STRING_H 1",
+        "#undef HAVE_STRNCASECMP": "#define HAVE_STRNCASECMP 1",
+        "#undef HAVE_SYS_MMAN_H": "#define HAVE_SYS_MMAN_H 1",
+        "#undef HAVE_SYS_PARAM_H": "#define HAVE_SYS_PARAM_H 1",
+        "#undef HAVE_SYS_STAT_H": "#define HAVE_SYS_STAT_H 1",
+        "#undef HAVE_SYS_SYSCTL_H": "#define HAVE_SYS_SYSCTL_H 1",
+        "#undef HAVE_SYS_TYPES_H": "#define HAVE_SYS_TYPES_H 1",
+        "#undef HAVE_SYS_UTSNAME_H": "#define HAVE_SYS_UTSNAME_H 1",
+        "#undef HAVE_TIME_H": "#define HAVE_TIME_H 1",
+        "#undef HAVE_UNAME": "#define HAVE_UNAME 1",
+        "#undef HAVE_UNISTD_H": "#define HAVE_UNISTD_H 1",
+        "#undef HAVE_USELOCALE": "#define HAVE_USELOCALE 1",
+        "#undef HAVE_WCHAR_T": "#define HAVE_WCHAR_T 1",
+        "#undef HAVE_X11_KEYSYM_H": "#define HAVE_X11_KEYSYM_H 1",
+        "#undef HAVE_X11_XLIB_H": "#define HAVE_X11_XLIB_H 1",
+        "#undef HAVE_X11_XUTIL_H": "#define HAVE_X11_XUTIL_H 1",
+        "#undef HAVE_XLOCALE_H": "#define HAVE_XLOCALE_H 1",
+        "#undef HAVE___PROGNAME": "#define HAVE___PROGNAME 1",
+        "#undef HWLOC_C_HAVE_VISIBILITY": "#define HWLOC_C_HAVE_VISIBILITY 1",
+        "#undef HWLOC_HAVE_ATTRIBUTE": "#define HWLOC_HAVE_ATTRIBUTE 1",
+        "#undef HWLOC_HAVE_ATTRIBUTE_ALIGNED": "#define HWLOC_HAVE_ATTRIBUTE_ALIGNED 1",
+        "#undef HWLOC_HAVE_ATTRIBUTE_ALWAYS_INLINE": "#define HWLOC_HAVE_ATTRIBUTE_ALWAYS_INLINE 1",
+        "#undef HWLOC_HAVE_ATTRIBUTE_COLD": "#define HWLOC_HAVE_ATTRIBUTE_COLD 1",
+        "#undef HWLOC_HAVE_ATTRIBUTE_CONST": "#define HWLOC_HAVE_ATTRIBUTE_CONST 1",
+        "#undef HWLOC_HAVE_ATTRIBUTE_DEPRECATED": "#define HWLOC_HAVE_ATTRIBUTE_DEPRECATED 1",
+        "#undef HWLOC_HAVE_ATTRIBUTE_FORMAT": "#define HWLOC_HAVE_ATTRIBUTE_FORMAT 1",
+        "#undef HWLOC_HAVE_ATTRIBUTE_HOT": "#define HWLOC_HAVE_ATTRIBUTE_HOT 1",
+        "#undef HWLOC_HAVE_ATTRIBUTE_MALLOC": "#define HWLOC_HAVE_ATTRIBUTE_MALLOC 1",
+        "#undef HWLOC_HAVE_ATTRIBUTE_MAY_ALIAS": "#define HWLOC_HAVE_ATTRIBUTE_MAY_ALIAS 1",
+        "#undef HWLOC_HAVE_ATTRIBUTE_NONNULL": "#define HWLOC_HAVE_ATTRIBUTE_NONNULL 1",
+        "#undef HWLOC_HAVE_ATTRIBUTE_NORETURN": "#define HWLOC_HAVE_ATTRIBUTE_NORETURN 1",
+        "#undef HWLOC_HAVE_ATTRIBUTE_NO_INSTRUMENT_FUNCTION": "#define HWLOC_HAVE_ATTRIBUTE_NO_INSTRUMENT_FUNCTION 1",
+        "#undef HWLOC_HAVE_ATTRIBUTE_PACKED": "#define HWLOC_HAVE_ATTRIBUTE_PACKED 1",
+        "#undef HWLOC_HAVE_ATTRIBUTE_PURE": "#define HWLOC_HAVE_ATTRIBUTE_PURE 1",
+        "#undef HWLOC_HAVE_ATTRIBUTE_SENTINEL": "#define HWLOC_HAVE_ATTRIBUTE_SENTINEL 1",
+        "#undef HWLOC_HAVE_ATTRIBUTE_UNUSED": "#define HWLOC_HAVE_ATTRIBUTE_UNUSED 1",
+        "#undef HWLOC_HAVE_ATTRIBUTE_WARN_UNUSED_RESULT": "#define HWLOC_HAVE_ATTRIBUTE_WARN_UNUSED_RESULT 1",
+        "#undef HWLOC_HAVE_ATTRIBUTE_WEAK_ALIAS": "#define HWLOC_HAVE_ATTRIBUTE_WEAK_ALIAS 1",
+        "#undef HWLOC_HAVE_CPU_SET": "#define HWLOC_HAVE_CPU_SET 1",
+        "#undef HWLOC_HAVE_CPU_SET_S": "#define HWLOC_HAVE_CPU_SET_S 1",
+        "#undef HWLOC_HAVE_DECL_FFS": "#define HWLOC_HAVE_DECL_FFS 1",
+        "#undef HWLOC_HAVE_DECL_FFSL": "#define HWLOC_HAVE_DECL_FFSL 1",
+        "#undef HWLOC_HAVE_DECL_STRCASECMP": "#define HWLOC_HAVE_DECL_STRCASECMP 1",
+        "#undef HWLOC_HAVE_DECL_STRNCASECMP": "#define HWLOC_HAVE_DECL_STRNCASECMP 1",
+        "#undef HWLOC_HAVE_FFS": "#define HWLOC_HAVE_FFS 1",
+        "#undef HWLOC_HAVE_FFSL": "#define HWLOC_HAVE_FFSL 1",
+        "#undef HWLOC_HAVE_LIBTERMCAP": "#define HWLOC_HAVE_LIBTERMCAP 1",
+        "#undef HWLOC_HAVE_LINUXIO": "#define HWLOC_HAVE_LINUXIO 1",
+        "#undef HWLOC_HAVE_PTHREAD_MUTEX": "#define HWLOC_HAVE_PTHREAD_MUTEX 1",
+        "#undef HWLOC_HAVE_SCHED_SETAFFINITY": "#define HWLOC_HAVE_SCHED_SETAFFINITY 1",
+        "#undef HWLOC_HAVE_STDINT_H": "#define HWLOC_HAVE_STDINT_H 1",
+        "#undef HWLOC_HAVE_SYSCALL": "#define HWLOC_HAVE_SYSCALL 1",
+        "#undef HWLOC_HAVE_X11_KEYSYM": "#define HWLOC_HAVE_X11_KEYSYM 1",
+        "#undef HWLOC_HAVE_X86_CPUID": "#define HWLOC_HAVE_X86_CPUID 1",
+        "#undef HWLOC_LINUX_SYS": "#define HWLOC_LINUX_SYS 1",
+        "#undef HWLOC_SIZEOF_UNSIGNED_INT": "#define HWLOC_SIZEOF_UNSIGNED_INT 4",
+        "#undef HWLOC_SIZEOF_UNSIGNED_LONG": "#define HWLOC_SIZEOF_UNSIGNED_LONG 8",
+        "#undef HWLOC_SYM_PREFIX": "#define HWLOC_SYM_PREFIX hwloc_",
+        "#undef HWLOC_SYM_PREFIX_CAPS": "#define HWLOC_SYM_PREFIX_CAPS HWLOC_",
+        "#undef HWLOC_SYM_TRANSFORM": "#define HWLOC_SYM_TRANSFORM 0",
+        "#undef HWLOC_USE_NCURSES": "#define HWLOC_USE_NCURSES 1",
+        "#undef HWLOC_VERSION": "#define HWLOC_VERSION \"2.0.3\"",
+        "#undef HWLOC_VERSION_GREEK": "#define HWLOC_VERSION_GREEK \"\"",
+        "#undef HWLOC_VERSION_MAJOR": "#define HWLOC_VERSION_MAJOR 2",
+        "#undef HWLOC_VERSION_MINOR": "#define HWLOC_VERSION_MINOR 0",
+        "#undef HWLOC_VERSION_RELEASE": "#define HWLOC_VERSION_RELEASE 3",
+        "#undef HWLOC_X86_64_ARCH": "#define HWLOC_X86_64_ARCH 1",
+        "#undef LT_OBJDIR": "#define LT_OBJDIR \".libs/\"",
+        "#undef PACKAGE": "#define PACKAGE \"hwloc\"",
+        "#undef PACKAGE_BUGREPORT": "#define PACKAGE_BUGREPORT \"http://github.com/open-mpi/hwloc/i",
+        "#undef PACKAGE_NAME": "#define PACKAGE_NAME \"hwloc\"",
+        "#undef PACKAGE_STRING": "#define PACKAGE_STRING \"hwloc 2.0.3\"",
+        "#undef PACKAGE_TARNAME": "#define PACKAGE_TARNAME \"hwloc\"",
+        "#undef PACKAGE_URL": "#define PACKAGE_URL \"\"",
+        "#undef PACKAGE_VERSION": "#define PACKAGE_VERSION \"2.0.3\"",
+        "#undef SIZEOF_UNSIGNED_INT": "#define SIZEOF_UNSIGNED_INT 4",
+        "#undef SIZEOF_UNSIGNED_LONG": "#define SIZEOF_UNSIGNED_LONG 8",
+        "#undef SIZEOF_VOID_P": "#define SIZEOF_VOID_P 8",
+        "#undef STDC_HEADERS": "#define STDC_HEADERS 1",
+        "# undef _HPUX_SOURCE": "# define _HPUX_SOURCE 1",
+        "# undef _ALL_SOURCE": "# define _ALL_SOURCE 1",
+        "# undef _GNU_SOURCE": "# define _GNU_SOURCE 1",
+        "# undef _POSIX_PTHREAD_SEMANTICS": "# define _POSIX_PTHREAD_SEMANTICS 1",
+        "# undef _TANDEM_SOURCE": "# define _TANDEM_SOURCE 1",
+        "# undef __EXTENSIONS__": "# define __EXTENSIONS__ 1",
+        "#undef VERSION": "#define VERSION \"2.0.3\"",
+        "#undef _HPUX_SOURCE": "#define _HPUX_SOURCE 1",
+        "#undef hwloc_pid_t": "#define hwloc_pid_t pid_t",
+        "#undef hwloc_thread_t": "#define hwloc_thread_t pthread_t",
+    } + if_cuda({
+        "#undef HAVE_CUDA": "#undef HAVE_CUDA 1",
+        "#undef HAVE_CUDA_H": "#undef HAVE_CUDA_H 1",
+        "#undef HAVE_CUDA_RUNTIME_API_H": "#undef HAVE_CUDA_RUNTIME_API_H 1",
+    }),
+)
+
 cc_library(
     name = "hwloc",
     srcs = [
-- 
GitLab


From 3ce92e3f7ff8d6fc0a432d464201ae58ed6f44d2 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Thu, 24 Jan 2019 16:48:16 -0800
Subject: [PATCH 1222/2345] Inline definition of EngineOutputInfo

---
 .../tf2tensorrt/convert/convert_nodes.h       | 27 +++++++++----------
 1 file changed, 13 insertions(+), 14 deletions(-)

diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
index f4f499ce7b..f6ea9b5cfc 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
@@ -415,7 +415,19 @@ class Converter {
   Status AddInputTensor(const string& name, nvinfer1::DataType dtype,
                         const nvinfer1::Dims& dims, int batch_size);
 
-  struct EngineOutputInfo;
+  // Used for Converter::RenameAndMarkOutputTensors()
+  struct EngineOutputInfo {
+    // The TRT tensor name which produces the output.
+    string source_tensor_name;
+    // The TensorFlow node name which is receiving the output from the TRT engine.
+    // This should always be the Identity node created in
+    // ConvertSegmentToGraphDef.
+    string dest_node_name;
+    // Output type. TensorRT requires this to be explicitly set for engine
+    // outputs.
+    nvinfer1::DataType trt_dtype;
+  };
+
   // Mark the tensors with names specified by source_tensor_name as output of
   // the TRT network, and set their names in the TRT network as dest_node_name.
   Status RenameAndMarkOutputTensors(
@@ -546,19 +558,6 @@ class Converter {
   friend class OpConverterTest;
 };
 
-// Used for Converter::RenameAndMarkOutputTensors()
-struct EngineOutputInfo {
-  // The TRT tensor name which produces the output.
-  string source_tensor_name;
-  // The TensorFlow node name which is receiving the output from the TRT engine.
-  // This should always be the Identity node created in
-  // ConvertSegmentToGraphDef.
-  string dest_node_name;
-  // Output type. TensorRT requires this to be explicitly set for engine
-  // outputs.
-  nvinfer1::DataType trt_dtype;
-};
-
 }  // namespace convert
 }  // namespace tensorrt
 }  // namespace tensorflow
-- 
GitLab


From d058a78ac3c2ffc7e688cd1d27e53e227bad3601 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 24 Jan 2019 16:18:05 -0800
Subject: [PATCH 1223/2345] [XLA] Add ErrorSpec::fewer_infs_ok knob.

This allows us to accept actual results that overflow to inf when the correct
answer is a large, finite value.

PiperOrigin-RevId: 230810756
---
 tensorflow/compiler/xla/error_spec.h          | 13 +++++++++++++
 tensorflow/compiler/xla/literal_comparison.cc | 14 ++++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/tensorflow/compiler/xla/error_spec.h b/tensorflow/compiler/xla/error_spec.h
index a1463aa159..4359f3b7de 100644
--- a/tensorflow/compiler/xla/error_spec.h
+++ b/tensorflow/compiler/xla/error_spec.h
@@ -30,6 +30,19 @@ struct ErrorSpec {
   // In effect, this allows the tested operation to produce incorrect results
   // for inputs outside its mathematical domain.
   bool relaxed_nans;
+
+  // If this is true, then we treat each +/-inf in the actual result as
+  // equivalent to our choice of either +/-inf or the min/max floating-point
+  // value.
+  //
+  // If the expected result is +/-inf, the actual result must still be +/-inf.
+  //
+  // In effect, this allows the tested operation to overflow, so long as it's
+  // overflowing on "large" values.
+  //
+  // (We could have a symmetric more_infs_ok flag if necessary; right now it
+  // appears not to be.)
+  bool fewer_infs_ok = false;
 };
 
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/literal_comparison.cc b/tensorflow/compiler/xla/literal_comparison.cc
index 23601245e4..07ae95063c 100644
--- a/tensorflow/compiler/xla/literal_comparison.cc
+++ b/tensorflow/compiler/xla/literal_comparison.cc
@@ -389,6 +389,20 @@ class NearComparator {
         abs_error = 0;
         rel_error = 0;
       }
+    } else if (IsInf(actual) && !IsInf(expected) && error_.fewer_infs_ok) {
+      // `fewer_infs_ok` gives us the option of comparing as though `actual`
+      // were float_max/min rather than inf.
+      T actual_finite = actual > T{0} ? std::numeric_limits<T>::max()
+                                      : std::numeric_limits<T>::lowest();
+      abs_error = FpAbsoluteValue(actual_finite - expected);
+
+      // Avoid division by 0 even though it's well-defined because ubsan can be
+      // configured to treat this as a fatal error.
+      if (expected != T{0}) {
+        rel_error = abs_error / FpAbsoluteValue(expected);
+      } else {
+        rel_error = std::numeric_limits<float>::infinity();
+      }
     } else if (IsInf(expected) || IsInf(actual)) {
       // If either the expected or actual value is infinity but not both,
       // then both absolute and relative error are regarded as inifity.
-- 
GitLab


From eaddef30bb5752eefd53dd34b52bc930513d8427 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 24 Jan 2019 16:45:47 -0800
Subject: [PATCH 1224/2345] [TF:XLA] Avoid unnecessary copying of call graphs
 in buffer assignment.

Also, make CallGraph non-copyable to avoid this kind of accidents in the future.

PiperOrigin-RevId: 230814992
---
 tensorflow/compiler/xla/service/buffer_assignment.cc | 3 ++-
 tensorflow/compiler/xla/service/call_graph.h         | 4 ++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/buffer_assignment.cc b/tensorflow/compiler/xla/service/buffer_assignment.cc
index 52f72b683f..e1b91b5001 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment.cc
+++ b/tensorflow/compiler/xla/service/buffer_assignment.cc
@@ -752,7 +752,8 @@ namespace {
 bool MayInterfereAcrossSubcomputations(BufferAssignment* assignment,
                                        const LogicalBuffer& a_buffer,
                                        const LogicalBuffer& b_buffer) {
-  auto call_graph = assignment->liveness().hlo_ordering().call_graph();
+  const CallGraph& call_graph =
+      assignment->liveness().hlo_ordering().call_graph();
   const HloInstruction* a_ancestor;
   const HloInstruction* b_ancestor;
   std::tie(a_ancestor, b_ancestor) =
diff --git a/tensorflow/compiler/xla/service/call_graph.h b/tensorflow/compiler/xla/service/call_graph.h
index f41367914c..c02ffda575 100644
--- a/tensorflow/compiler/xla/service/call_graph.h
+++ b/tensorflow/compiler/xla/service/call_graph.h
@@ -256,6 +256,10 @@ class CallGraph {
  private:
   CallGraph(const HloModule* module);
 
+  // Not copyable.
+  CallGraph(const CallGraph&) = delete;
+  CallGraph& operator=(const CallGraph&) = delete;
+
   // Sets the call contexts for every node in the graph.
   void SetCallContexts();
 
-- 
GitLab


From efe1ded49b3037fb06af0594494ae190e33493fb Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Thu, 24 Jan 2019 17:01:56 -0800
Subject: [PATCH 1225/2345] Fix error reporting for --infile.

Switch report to accumulate lines in a list (to avoid quadratic concatenation with +=).

Fix typo.

PiperOrigin-RevId: 230817436
---
 .../tools/compatibility/tf_upgrade_v2.py      |  2 +-
 .../tools/compatibility/tf_upgrade_v2_main.py | 24 +++++++++----------
 2 files changed, 12 insertions(+), 14 deletions(-)

diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index b83d7d9749..03ecf5f303 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -949,7 +949,7 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             assert_rank_comment,
         "tf.device":
             "tf.device no longer takes function as an argument. "
-            "'devide_name_or_function' argument has been renamed to "
+            "'device_name_or_function' argument has been renamed to "
             "'device_name'.",
         "tf.flags":
             "tf.flags has been removed, please use the argparse or absl"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_main.py b/tensorflow/tools/compatibility/tf_upgrade_v2_main.py
index 871da9ad6b..b446452cfe 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_main.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_main.py
@@ -87,6 +87,7 @@ Simple usage:
           "single file.")
     files_processed, report_text, errors = upgrade.process_file(
         args.input_file, args.output_file)
+    errors = {args.input_file: errors}
     files_processed = 1
   elif args.input_tree:
     if not args.output_tree:
@@ -99,23 +100,20 @@ Simple usage:
     parser.print_help()
   if report_text:
     num_errors = 0
-    report = ""
+    report = []
     for f in errors:
       if errors[f]:
         num_errors += len(errors[f])
-        report += "-" * 80 + "\n"
-        report += "File: %s\n" % f
-        report += "-" * 80 + "\n"
-        report += "\n".join(errors[f]) + "\n"
-
-    report = (
-        "TensorFlow 2.0 Upgrade Script\n"
-        "-----------------------------\n"
-        "Converted %d files\n" % files_processed +
-        "Detected %d errors that require attention" % num_errors + "\n" +
-        "-" * 80 + "\n"
-    ) + report
+        report.append("-" * 80 + "\n")
+        report.append("File: %s\n" % f)
+        report.append("-" * 80 + "\n")
+        report.append("\n".join(errors[f]) + "\n")
 
+    report = ("TensorFlow 2.0 Upgrade Script\n"
+              "-----------------------------\n"
+              "Converted %d files\n" % files_processed +
+              "Detected %d errors that require attention" % num_errors + "\n" +
+              "-" * 80 + "\n") + "".join(report)
     with open(report_filename, "w") as report_file:
       report_file.write(report)
       report_file.write("=" * 80 + "\n")
-- 
GitLab


From e232013764faa5c8926b562c2b1c61594d705ebe Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Thu, 24 Jan 2019 17:07:14 -0800
Subject: [PATCH 1226/2345] Forgot to mark before setting type

---
 tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
index 3183ba356d..92018750a9 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
@@ -977,12 +977,12 @@ Status Converter::RenameAndMarkOutputTensors(
       tensor = layer->getOutput(0);
     }
     tensor->setName(output.dest_node_name.c_str());
+    network()->markOutput(*tensor);
     // Set type after marking as output. TRT only supports setType for engine
     // outputs and inputs (type is inferred otherwise).
     tensor->setType(output.trt_dtype);
     VLOG(1) << "Marking output TRT tensor " << output.source_tensor_name
             << ", which feeds TF node " << output.dest_node_name;
-    network()->markOutput(*tensor);
   }
   return Status::OK();
 }
-- 
GitLab


From 9cd5949ee566ae50c5ba16d962f6da19e164d32b Mon Sep 17 00:00:00 2001
From: Yanan Cao <ycao@google.com>
Date: Thu, 24 Jan 2019 17:19:12 -0800
Subject: [PATCH 1227/2345] Fix keras normalization_layers test.

PiperOrigin-RevId: 230820083
---
 tensorflow/python/keras/layers/normalization_test.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/layers/normalization_test.py b/tensorflow/python/keras/layers/normalization_test.py
index bd5951b845..3815d1e673 100644
--- a/tensorflow/python/keras/layers/normalization_test.py
+++ b/tensorflow/python/keras/layers/normalization_test.py
@@ -438,7 +438,8 @@ class LayerNormalizationTest(keras_parameterized.TestCase):
     if test.is_gpu_available(cuda_only=True):
       with self.session(use_gpu=True):
         model = keras.models.Sequential()
-        norm = keras.layers.LayerNormalization(input_shape=(3, 4, 4))
+        norm = keras.layers.LayerNormalization(
+            input_shape=(3, 4, 4), params_axis=1)
         model.add(norm)
         model.compile(loss='mse',
                       optimizer=gradient_descent.GradientDescentOptimizer(0.01),
-- 
GitLab


From f2dcfd83d6ae4a86c0cfd4bc73cd7d5155be3dd1 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Thu, 24 Jan 2019 17:19:55 -0800
Subject: [PATCH 1228/2345] Add direct dependencies to op libraries that are
 depended on through kernels.

PiperOrigin-RevId: 230820176
---
 tensorflow/core/distributed_runtime/BUILD      |  1 +
 tensorflow/core/grappler/inputs/BUILD          |  2 ++
 tensorflow/core/grappler/optimizers/data/BUILD | 12 ++++++++++++
 3 files changed, 15 insertions(+)

diff --git a/tensorflow/core/distributed_runtime/BUILD b/tensorflow/core/distributed_runtime/BUILD
index 6f09bef5ea..351dbebf79 100644
--- a/tensorflow/core/distributed_runtime/BUILD
+++ b/tensorflow/core/distributed_runtime/BUILD
@@ -587,6 +587,7 @@ tf_cc_test(
         ":collective_param_resolver_distributed",
         ":device_resolver_distributed",
         ":test_utils",
+        "//tensorflow/core:collective_ops_op_lib",
         "//tensorflow/core:core_cpu_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
diff --git a/tensorflow/core/grappler/inputs/BUILD b/tensorflow/core/grappler/inputs/BUILD
index ffa204028c..c2846187c7 100644
--- a/tensorflow/core/grappler/inputs/BUILD
+++ b/tensorflow/core/grappler/inputs/BUILD
@@ -49,6 +49,8 @@ cc_library(
     deps = [
         ":input_yielder",
         "//tensorflow/cc:cc_ops",
+        "//tensorflow/core:functional_ops_op_lib",
+        "//tensorflow/core:math_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:tensorflow",
         "//tensorflow/core/grappler:grappler_item",
diff --git a/tensorflow/core/grappler/optimizers/data/BUILD b/tensorflow/core/grappler/optimizers/data/BUILD
index 377ff64de9..05ee00ef1e 100644
--- a/tensorflow/core/grappler/optimizers/data/BUILD
+++ b/tensorflow/core/grappler/optimizers/data/BUILD
@@ -84,6 +84,7 @@ cc_library(
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/kernels:functional_ops",
         "//tensorflow/core/kernels:control_flow_ops",
+        "//tensorflow/core:control_flow_ops_op_lib",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
         "//tensorflow/core:lib_internal",
     ] + tf_protos_all(),
@@ -424,6 +425,7 @@ tf_cc_test(
         ":graph_test_utils",
         ":graph_utils",
         ":map_fusion",
+        "//tensorflow/core:control_flow_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
@@ -500,10 +502,13 @@ tf_cc_test(
     deps = [
         ":graph_utils",
         ":map_vectorization",
+        "//tensorflow/core:array_ops_op_lib",
+        "//tensorflow/core:bitwise_ops_op_lib",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:direct_session",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:spectral_ops_op_lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
@@ -659,7 +664,14 @@ tf_cc_test(
         "//tensorflow/core/kernels:cwise_op",
         "//tensorflow/core/kernels:logging_ops",
         "//tensorflow/core/kernels:math",
+        "//tensorflow/core:spectral_ops_op_lib",
+        "//tensorflow/core:bitwise_ops_op_lib",
+        "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core/kernels:nn",
+        "//tensorflow/core:nn_ops_op_lib",
+        "//tensorflow/core:mkl_nn_ops_op_lib",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core/kernels:parsing",
         "//tensorflow/tools/graph_transforms:transform_utils",
     ] + tf_protos_all(),
-- 
GitLab


From 862b9f19d17af6bcdc3c8208c88723acf52bc6cf Mon Sep 17 00:00:00 2001
From: Yanan Cao <ycao@google.com>
Date: Thu, 24 Jan 2019 17:39:42 -0800
Subject: [PATCH 1229/2345] Internal changes

PiperOrigin-RevId: 230822736
---
 tensorflow/contrib/constrained_optimization/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/constrained_optimization/BUILD b/tensorflow/contrib/constrained_optimization/BUILD
index eee4329acb..6ca36f354f 100644
--- a/tensorflow/contrib/constrained_optimization/BUILD
+++ b/tensorflow/contrib/constrained_optimization/BUILD
@@ -46,6 +46,7 @@ py_test(
         # TODO(b/121223093): Re-enable this test after fixing "Distribution
         # should match known solution" errors.
         "no_mac",
+        "no_oss",
     ],
     deps = [
         ":constrained_optimization",
-- 
GitLab


From 456b5eaf74b3362cbf69c5d044bf520bd6504ea9 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Thu, 24 Jan 2019 18:23:04 -0800
Subject: [PATCH 1230/2345] Internal change.

PiperOrigin-RevId: 230828168
---
 tensorflow/tools/ci_build/update_version.py | 30 ++++++++++++---------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/tensorflow/tools/ci_build/update_version.py b/tensorflow/tools/ci_build/update_version.py
index 81488902b5..117627c458 100755
--- a/tensorflow/tools/ci_build/update_version.py
+++ b/tensorflow/tools/ci_build/update_version.py
@@ -84,19 +84,26 @@ class Version(object):
       identifier_string: extension string eg. (-rc0)
       version_type: version parameter ((REGULAR|NIGHTLY)_VERSION)
     """
-    self.string = "%s.%s.%s%s" % (major,
-                                  minor,
-                                  patch,
-                                  identifier_string)
     self.major = major
     self.minor = minor
     self.patch = patch
     self.identifier_string = identifier_string
     self.version_type = version_type
+    self._update_string()
+
+  def _update_string(self):
+    self.string = "%s.%s.%s%s" % (self.major,
+                                  self.minor,
+                                  self.patch,
+                                  self.identifier_string)
 
   def __str__(self):
     return self.string
 
+  def set_identifier_string(self, identifier_string):
+    self.identifier_string = identifier_string
+    self._update_string()
+
   @property
   def pep_440_str(self):
     if self.version_type == REGULAR_VERSION:
@@ -283,15 +290,14 @@ def main():
   """
 
   parser = argparse.ArgumentParser(description="Cherry picking automation.")
-  group = parser.add_mutually_exclusive_group(required=True)
 
   # Arg information
-  group.add_argument("--version",
-                     help="<new_major_ver>.<new_minor_ver>.<new_patch_ver>",
-                     default="")
-  group.add_argument("--nightly",
-                     help="disable the service provisioning step",
-                     action="store_true")
+  parser.add_argument("--version",
+                      help="<new_major_ver>.<new_minor_ver>.<new_patch_ver>",
+                      default="")
+  parser.add_argument("--nightly",
+                      help="disable the service provisioning step",
+                      action="store_true")
 
   args = parser.parse_args()
 
@@ -301,7 +307,7 @@ def main():
   if args.nightly:
     if args.version:
       new_version = Version.parse_from_string(args.version, NIGHTLY_VERSION)
-      new_version.identifier_string = "-dev" + time.strftime("%Y%m%d")
+      new_version.set_identifier_string("-dev" + time.strftime("%Y%m%d"))
     else:
       # Dev minor version is one ahead of official.
       nightly_minor_ver = int(old_version.minor) + 1
-- 
GitLab


From fd1569bdbd174c20cb85c0241646635a59b8ea59 Mon Sep 17 00:00:00 2001
From: Loo Rong Jie <loorongjie@gmail.com>
Date: Fri, 25 Jan 2019 10:45:55 +0800
Subject: [PATCH 1231/2345] Define _USE_MATH_DEFINES to get math constants for
 MSVC

---
 tensorflow/compiler/xla/client/lib/math.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/compiler/xla/client/lib/math.cc b/tensorflow/compiler/xla/client/lib/math.cc
index b6378b02da..090f766c83 100644
--- a/tensorflow/compiler/xla/client/lib/math.cc
+++ b/tensorflow/compiler/xla/client/lib/math.cc
@@ -13,6 +13,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+// This macro is required to make MSVC defines math constants in math.h
+#define _USE_MATH_DEFINES
+#include <math.h>
+
 #include "tensorflow/compiler/xla/client/lib/math.h"
 
 #include "tensorflow/compiler/xla/client/lib/constants.h"
-- 
GitLab


From 006c3c904cd9b6791b219184cb66a6caceae7b51 Mon Sep 17 00:00:00 2001
From: Loo Rong Jie <loorongjie@gmail.com>
Date: Fri, 25 Jan 2019 10:49:34 +0800
Subject: [PATCH 1232/2345] Do not enable TF_GENERATE_BACKTRACE on Windows

---
 tensorflow/core/platform/default/stacktrace.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/platform/default/stacktrace.h b/tensorflow/core/platform/default/stacktrace.h
index c8e297fa8d..b64bc15971 100644
--- a/tensorflow/core/platform/default/stacktrace.h
+++ b/tensorflow/core/platform/default/stacktrace.h
@@ -17,8 +17,8 @@ limitations under the License.
 #define TENSORFLOW_CORE_PLATFORM_DEFAULT_STACKTRACE_H_
 
 #include "tensorflow/core/platform/platform.h"
-#if !defined(IS_MOBILE_PLATFORM) && defined(PLATFORM_POSIX) && \
-    (defined(__clang__) || defined(__GNUC__))
+#if !defined(IS_MOBILE_PLATFORM) && !defined(PLATFORM_WINDOWS) && \
+    defined(PLATFORM_POSIX) && (defined(__clang__) || defined(__GNUC__))
 #define TF_GENERATE_BACKTRACE
 #endif
 
-- 
GitLab


From 0be1190c666d75332031980a63c3bf4ad0459047 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 24 Jan 2019 18:52:50 -0800
Subject: [PATCH 1233/2345] [XLA] Improve precision of lgamma.

Our lgamma implementation suffered from precision loss and nans primarily due
to naive computation of sin(pi * x): The multiplication may overflow or lose
precision.  This patch fixes that, and adds an exhaustive test that checks all
fp16 inputs, for this and other special functions.

There are many other special functions which fail the fp16 exhaustive test,
some quite spectacularly.  These will be handled separately, along with
(probably) some sort of fp32 test.

Relevant to #25052

PiperOrigin-RevId: 230831395
---
 tensorflow/compiler/xla/client/lib/BUILD      |  16 ++
 tensorflow/compiler/xla/client/lib/math.cc    |  46 ++++-
 .../xla/client/lib/math_exhaustive_test.cc    | 178 ++++++++++++++++++
 .../compiler/xla/client/lib/math_test.cc      |  22 +++
 4 files changed, 256 insertions(+), 6 deletions(-)
 create mode 100644 tensorflow/compiler/xla/client/lib/math_exhaustive_test.cc

diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD
index ded072faf7..26c5e8eb73 100644
--- a/tensorflow/compiler/xla/client/lib/BUILD
+++ b/tensorflow/compiler/xla/client/lib/BUILD
@@ -206,6 +206,22 @@ xla_test(
     ],
 )
 
+xla_test(
+    name = "math_exhaustive_test",
+    srcs = ["math_exhaustive_test.cc"],
+    shard_count = 16,
+    deps = [
+        ":math",
+        "//tensorflow/compiler/xla:literal_util",
+        "//tensorflow/compiler/xla:test",
+        "//tensorflow/compiler/xla:types",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/client:xla_builder",
+        "//tensorflow/compiler/xla/tests:client_library_test_base",
+        "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+    ],
+)
+
 cc_library(
     name = "matrix",
     srcs = ["matrix.cc"],
diff --git a/tensorflow/compiler/xla/client/lib/math.cc b/tensorflow/compiler/xla/client/lib/math.cc
index b6378b02da..1d783e1984 100644
--- a/tensorflow/compiler/xla/client/lib/math.cc
+++ b/tensorflow/compiler/xla/client/lib/math.cc
@@ -242,12 +242,46 @@ XlaOp Lgamma(XlaOp input) {
 
     XlaOp log_y = log_sqrt_two_pi + (z + one_half) * log_t - t + Log(x);
 
-    // If z = a + 0j, the analytic continuation of log reduces to taking the
-    // absolute value of the real part.
-    // Re(log(z)) = Re(log|z| + arg(z)j)
-    //            = log|a|
-    XlaOp reflection = log_pi - Log(Abs(Sin(pi * input))) - log_y;
-    return Select(need_to_reflect, reflection, log_y);
+    // Compute the reflected value, used when x < 0.5:
+    //
+    //   lgamma(x) = log(pi) - lgamma(1-x) - log(abs(sin(pi * x))).
+    //
+    // (The abs is because lgamma is the log of the absolute value of the gamma
+    // function.)
+    //
+    // We have to be careful when computing the final term above. gamma(x) goes
+    // to +/-inf at every integer x < 0, and this is controlled by the
+    // sin(pi * x) term.  The slope is large, so precision is particularly
+    // important.
+    //
+    // Because abs(sin(pi * x)) has period 1, we can equivalently use
+    // abs(sin(pi * frac(x))) = sin(pi * frac(x)), where frac(x) is the
+    // fractional part of x.  This is more numerically accurate: It doesn't
+    // overflow to inf like pi * x can, and if x is an integer, it evaluates to
+    // 0 exactly, which is significant because we then take the log of this
+    // value, and log(0) is inf.
+    //
+    // We don't have a frac(x) primitive in XLA and computing it is tricky, but
+    // because abs(sin(pi * x)) = abs(sin(pi * abs(x))), it's good enough for
+    // our purposes to use abs(frac(x)) = abs(x) - floor(abs(x)).
+    //
+    XlaOp abs_input = Abs(input);
+    XlaOp reflection_denom = Log(Sin(pi * (abs_input - Floor(abs_input))));
+
+    // Avoid computing -inf - inf, which is nan.  If reflection_denom is +/-inf,
+    // then it "wins" and the result is +/-inf.
+    XlaOp reflection =
+        Select(IsFinite(reflection_denom), log_pi - reflection_denom - log_y,
+               -reflection_denom);
+    XlaOp result = Select(need_to_reflect, reflection, log_y);
+
+    // lgamma(+/-inf) = +inf.
+    XlaOp inf_bcast =
+        Broadcast(ScalarLike(input, std::numeric_limits<float>::infinity()),
+                  AsInt64Slice(shape.dimensions()));
+    return Select(Or(IsFinite(input),                           // is finite, or
+                     Not(Or(Lt(input, one), Ge(input, one)))),  // is nan
+                  result, inf_bcast);
   });
 }
 
diff --git a/tensorflow/compiler/xla/client/lib/math_exhaustive_test.cc b/tensorflow/compiler/xla/client/lib/math_exhaustive_test.cc
new file mode 100644
index 0000000000..123af5e51c
--- /dev/null
+++ b/tensorflow/compiler/xla/client/lib/math_exhaustive_test.cc
@@ -0,0 +1,178 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/client/lib/math.h"
+#include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/compiler/xla/test.h"
+#include "tensorflow/compiler/xla/tests/client_library_test_base.h"
+#include "tensorflow/compiler/xla/tests/test_macros.h"
+#include "tensorflow/compiler/xla/types.h"
+
+namespace xla {
+namespace {
+
+using Eigen::half;
+
+struct Testcase {
+  Testcase(string name, XlaOp (*op)(XlaOp), float (*host_op)(float))
+      : name(name), op(op), host_op(host_op) {}
+
+  Testcase& set_tolerance(float abs_err, float rel_err) {
+    error.abs = abs_err;
+    error.rel = rel_err;
+    return *this;
+  }
+
+  Testcase& set_relaxed_nans() {
+    error.relaxed_nans = true;
+    return *this;
+  }
+
+  Testcase& set_fewer_infs_ok() {
+    error.fewer_infs_ok = true;
+    return *this;
+  }
+
+  Testcase& set_skip_pos_inf() {
+    skip_pos_inf = true;
+    return *this;
+  }
+
+  Testcase& set_skip_neg_inf() {
+    skip_neg_inf = true;
+    return *this;
+  }
+
+  Testcase& set_skip_infs() {
+    skip_pos_inf = true;
+    skip_neg_inf = true;
+    return *this;
+  }
+
+  Testcase& set_skip_neg_zero() {
+    skip_neg_zero = true;
+    return *this;
+  }
+
+  string name;
+  XlaOp (*op)(XlaOp);
+  float (*host_op)(float);
+
+  ErrorSpec error{0.01};
+
+  // If true, don't test +/-infinity or negative 0.
+  bool skip_pos_inf = false;
+  bool skip_neg_inf = false;
+  bool skip_neg_zero = false;
+};
+
+void PrintTo(const Testcase& tc, std::ostream* os) { *os << tc.name; }
+
+class MathExhaustiveTest : public ClientLibraryTestBase,
+                           public ::testing::WithParamInterface<Testcase> {
+ public:
+  MathExhaustiveTest() {
+    // Disable fast-math, otherwise we get the wrong results for e.g.
+    // sqrt(-inf).
+    SetFastMathDisabled(true);
+  }
+};
+
+// Checks a function's behavior on all fp16 values.
+//
+// TODO(jlebar): asin and lgamma tests fail on interpreter.
+XLA_TEST_P(MathExhaustiveTest, DISABLED_ON_INTERPRETER(F16)) {
+  const Testcase& tc = GetParam();
+  XlaBuilder b(TestName());
+
+  std::vector<half> input;
+  for (uint32 i = 0; i < 1 << 16; ++i) {
+    half h;
+    h.x = i;
+
+    // If we're not using infinity as an input, use 0 as a placeholder rather
+    // than simply skipping this element.  We do this because when the test
+    // framework reports an incorrect answer, it tells us which index failed.
+    // So long as our inputs are a simple list of all possible float16s, we can
+    // convert an index to a half with e.g. the following Python:
+    //
+    //   np.frombuffer(array('H', [12345]), dtype=np.float16)[0]
+    //
+    // but as soon as our list of inputs has any gaps, this doesn't work.
+    if (std::isinf(static_cast<float>(h)) &&
+        ((tc.skip_pos_inf && h > half{0}) ||
+         (tc.skip_neg_inf && h < half{0}))) {
+      h = half{0};
+    }
+
+    if (h == half{0} && tc.skip_neg_zero &&
+        std::signbit(static_cast<float>(h))) {
+      h = half{0};
+    }
+
+    input.push_back(h);
+  }
+
+  std::vector<half> expected_result;
+  for (const auto& h : input) {
+    expected_result.push_back(
+        static_cast<half>(tc.host_op(static_cast<float>(h))));
+  }
+
+  XlaOp param = AddParam(LiteralUtil::CreateR1<half>(input), &b);
+  tc.op(param);
+  ComputeAndCompareR1<half>(&b, expected_result, {}, tc.error);
+}
+
+// TODO(jlebar): The following tests are missing.
+//
+// - Fails on -1 (returns 0 instead of pi).
+//   Testcase{"acos", Acos, std::acos}.set_skip_infs(),
+//
+// - Many failures.
+//   Testcase{"acosh", Acosh, std::acosh}.set_relaxed_nans(),
+//   Testcase{"asinh", Asinh, std::asinh},
+//   Testcase{"sinh", Sinh, std::sinh},
+//   Testcase{"cosh", Cosh, std::cosh}.set_fewer_infs_ok(),
+//   Testcase{"erf", Erf, std::erf},
+//   Testcase{"erfc", Erfc, std::erfc}.set_skip_infs(),
+//   Testcase{"round_to_even", RoundToEven,
+//            [](float x) { return std::nearbyint(x / 2) * 2; }},
+//
+// - No equivalent std function to compare with.
+//   Testcase{"erfinv", ErfInv, std::erfinv},
+//   Testcase{"digamma", Digamma, std::digamma},
+//
+INSTANTIATE_TEST_CASE_P(
+    MathExhaustiveTest_Instantiation, MathExhaustiveTest,
+    ::testing::ValuesIn(std::vector<Testcase>{
+        Testcase{"sqrt", Sqrt, std::sqrt}.set_skip_neg_inf(),
+        Testcase{"rsqrt", Rsqrt, [](float x) { return 1 / std::sqrt(x); }}
+            .set_tolerance(0.05, 0.05)
+            .set_skip_infs()
+            .set_skip_neg_zero(),
+        Testcase{"square", Square, [](float x) { return x * x; }},
+        Testcase{"reciprocal", Reciprocal, [](float x) { return 1 / x; }},
+        Testcase{"lgamma", Lgamma, std::lgamma}
+            .set_tolerance(0.1, 0.15)
+            .set_fewer_infs_ok(),
+        Testcase{"asin", Asin, std::asin}.set_skip_infs(),
+        Testcase{"atan", Atan, std::atan},
+        Testcase{"tan", Tan, std::tan}.set_tolerance(0.05, 0.05),
+    }));
+
+}  // namespace
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/client/lib/math_test.cc b/tensorflow/compiler/xla/client/lib/math_test.cc
index c35c3c4d91..c2e1251fc2 100644
--- a/tensorflow/compiler/xla/client/lib/math_test.cc
+++ b/tensorflow/compiler/xla/client/lib/math_test.cc
@@ -145,6 +145,28 @@ XLA_TEST_F(MathTest, Lgamma) {
   ComputeAndCompareR1<float>(&builder, expected, {}, error_spec_);
 }
 
+// TODO(jlebar): Fails on interpreter due to unimplemented operation.
+XLA_TEST_F(MathTest, DISABLED_ON_INTERPRETER(LgammaF16)) {
+  SetFastMathDisabled(true);
+
+  XlaBuilder b(TestName());
+
+  // These seemingly arbitrary inputs came from debugging the lgamma
+  // implementation against a test which tried all possible f16 values.
+  auto x = ConstantR1<half>(&b, {
+                                    half(-7360.0),
+                                    half(-4066.0),
+                                    half(-5.9605e-08),
+                                });
+  Lgamma(x);
+  std::vector<half> expected = {
+      std::numeric_limits<half>::infinity(),
+      std::numeric_limits<half>::infinity(),
+      half(16.64),
+  };
+  ComputeAndCompareR1<half>(&b, expected, {}, ErrorSpec{0.1});
+}
+
 XLA_TEST_F(MathTest, Digamma) {
   XlaBuilder builder(TestName());
   auto x = ConstantR1<float>(&builder, {1.0, 0.5, 1 / 3.0, 0.25, 1 / 6.0, 0.125,
-- 
GitLab


From 6291988afae9a4663a73ed323491518f85b5a9ef Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 24 Jan 2019 19:01:32 -0800
Subject: [PATCH 1234/2345] [XLA] Add BroadcastScalarLike helper.

Like ScalarLike, but broadcasts to the same dimensions as the prototype.

PiperOrigin-RevId: 230832282
---
 .../compiler/xla/client/lib/constants.h       | 21 +++++++
 tensorflow/compiler/xla/client/lib/math.cc    | 59 ++++++++-----------
 .../xla/client/lib/triangular_solve.cc        |  4 +-
 tensorflow/compiler/xla/tests/BUILD           |  1 +
 .../compiler/xla/tests/constants_test.cc      | 24 ++++++++
 5 files changed, 72 insertions(+), 37 deletions(-)

diff --git a/tensorflow/compiler/xla/client/lib/constants.h b/tensorflow/compiler/xla/client/lib/constants.h
index a38282e8db..4e5310a380 100644
--- a/tensorflow/compiler/xla/client/lib/constants.h
+++ b/tensorflow/compiler/xla/client/lib/constants.h
@@ -90,6 +90,27 @@ XlaOp ScalarLike(XlaOp prototype, T value) {
   });
 }
 
+// Returns an array or scalar containing copies of `value` cast to the same
+// run-type type as `prototype` and broadcast to the same dimensions as
+// `prototype`.
+//
+// If `prototype` is not a scalar or array, returns an error.
+template <typename T>
+XlaOp FullLike(XlaOp prototype, T value) {
+  XlaBuilder* builder = prototype.builder();
+  return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(prototype));
+    if (ShapeUtil::IsScalar(shape) || shape.IsArray()) {
+      return Broadcast(ScalarLike(prototype, value), shape.dimensions());
+    } else {
+      return InvalidArgument(
+          "Prototype shape for BroadcastConstantLike must be a scalar or "
+          "array, but was %s",
+          shape.ToString());
+    }
+  });
+}
+
 // Returns a scalar with value '0' of 'type'.
 XlaOp Zero(XlaBuilder* builder, PrimitiveType type);
 
diff --git a/tensorflow/compiler/xla/client/lib/math.cc b/tensorflow/compiler/xla/client/lib/math.cc
index 1d783e1984..2426a433fc 100644
--- a/tensorflow/compiler/xla/client/lib/math.cc
+++ b/tensorflow/compiler/xla/client/lib/math.cc
@@ -136,37 +136,30 @@ XlaOp Erf(XlaOp x) {
 //   }
 //   return p*x
 XlaOp ErfInv(XlaOp x) {
-  XlaBuilder* b = x.builder();
-  return b->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
-    TF_ASSIGN_OR_RETURN(Shape shape, b->GetShape(x));
-    constexpr int kDegree = 9;
-    constexpr std::array<float, 9> w_less_than_5_constants = {
-        2.81022636e-08f,  3.43273939e-07f, -3.5233877e-06f,
-        -4.39150654e-06f, 0.00021858087f,  -0.00125372503f,
-        -0.00417768164f,  0.246640727f,    1.50140941f};
-    constexpr std::array<float, 9> w_greater_than_5_constants = {
-        -0.000200214257f, 0.000100950558f, 0.00134934322f,
-        -0.00367342844f,  0.00573950773f,  -0.0076224613f,
-        0.00943887047f,   1.00167406f,     2.83297682f};
-
-    auto one = ScalarLike(x, 1.0);
-    auto w = -Log((one - x) * (one + x));
-
-    auto lt = Lt(w, ScalarLike(x, 5.0));
-    auto coefficient = [&](int i) {
-      return Select(lt,
-                    Broadcast(ScalarLike(x, w_less_than_5_constants[i]),
-                              AsInt64Slice(shape.dimensions())),
-                    Broadcast(ScalarLike(x, w_greater_than_5_constants[i]),
-                              AsInt64Slice(shape.dimensions())));
-    };
-    w = Select(lt, w - ScalarLike(x, 2.5), Sqrt(w) - ScalarLike(x, 3.0));
-    auto p = coefficient(0);
-    for (int i = 1; i < kDegree; ++i) {
-      p = coefficient(i) + p * w;
-    }
-    return p * x;
-  });
+  constexpr int kDegree = 9;
+  constexpr std::array<float, 9> w_less_than_5_constants = {
+      2.81022636e-08f,  3.43273939e-07f, -3.5233877e-06f,
+      -4.39150654e-06f, 0.00021858087f,  -0.00125372503f,
+      -0.00417768164f,  0.246640727f,    1.50140941f};
+  constexpr std::array<float, 9> w_greater_than_5_constants = {
+      -0.000200214257f, 0.000100950558f, 0.00134934322f,
+      -0.00367342844f,  0.00573950773f,  -0.0076224613f,
+      0.00943887047f,   1.00167406f,     2.83297682f};
+
+  auto one = ScalarLike(x, 1.0);
+  auto w = -Log((one - x) * (one + x));
+
+  auto lt = Lt(w, ScalarLike(x, 5.0));
+  auto coefficient = [&](int i) {
+    return Select(lt, FullLike(x, w_less_than_5_constants[i]),
+                  FullLike(x, w_greater_than_5_constants[i]));
+  };
+  w = Select(lt, w - ScalarLike(x, 2.5), Sqrt(w) - ScalarLike(x, 3.0));
+  auto p = coefficient(0);
+  for (int i = 1; i < kDegree; ++i) {
+    p = coefficient(i) + p * w;
+  }
+  return p * x;
 }
 
 namespace {
@@ -276,9 +269,7 @@ XlaOp Lgamma(XlaOp input) {
     XlaOp result = Select(need_to_reflect, reflection, log_y);
 
     // lgamma(+/-inf) = +inf.
-    XlaOp inf_bcast =
-        Broadcast(ScalarLike(input, std::numeric_limits<float>::infinity()),
-                  AsInt64Slice(shape.dimensions()));
+    XlaOp inf_bcast = FullLike(input, std::numeric_limits<float>::infinity());
     return Select(Or(IsFinite(input),                           // is finite, or
                      Not(Or(Lt(input, one), Ge(input, one)))),  // is nan
                   result, inf_bcast);
diff --git a/tensorflow/compiler/xla/client/lib/triangular_solve.cc b/tensorflow/compiler/xla/client/lib/triangular_solve.cc
index c2f31742e9..ba7fde118f 100644
--- a/tensorflow/compiler/xla/client/lib/triangular_solve.cc
+++ b/tensorflow/compiler/xla/client/lib/triangular_solve.cc
@@ -140,9 +140,7 @@ XlaOp InvertDiagonalBlocks(XlaOp diag_blocks, bool lower, bool transpose_a,
     // zero (which can happen if the last block was padded) otherwise it will
     // introduce nans which will propagate
     auto diags = GetMatrixDiagonal(diag_blocks);
-    TF_ASSIGN_OR_RETURN(Shape diags_shape, builder->GetShape(diags));
-    auto one = ScalarLike(diags, 1);
-    auto ones = Broadcast(one, AsInt64Slice(diags_shape.dimensions()));
+    auto ones = FullLike(diags, 1);
     diags = Select(Eq(diags, Zero(builder, shape.element_type())), ones, diags);
     auto scaled_diag_blocks = Div(diag_blocks, diags, {0, 2});
 
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 7e05a7956e..5dd9f3b408 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -834,6 +834,7 @@ xla_test(
         "//tensorflow/compiler/xla/client:local_client",
         "//tensorflow/compiler/xla/client:xla_builder",
         "//tensorflow/compiler/xla/client:xla_computation",
+        "//tensorflow/compiler/xla/client/lib:constants",
         "//tensorflow/compiler/xla/tests:client_library_test_base",
         "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
diff --git a/tensorflow/compiler/xla/tests/constants_test.cc b/tensorflow/compiler/xla/tests/constants_test.cc
index 9174f2651c..c672e8db03 100644
--- a/tensorflow/compiler/xla/tests/constants_test.cc
+++ b/tensorflow/compiler/xla/tests/constants_test.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/array2d.h"
 #include "tensorflow/compiler/xla/array3d.h"
 #include "tensorflow/compiler/xla/array4d.h"
+#include "tensorflow/compiler/xla/client/lib/constants.h"
 #include "tensorflow/compiler/xla/client/local_client.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal_util.h"
@@ -180,6 +181,29 @@ TEST_F(ConstantsTest, Token) {
   TF_ASSERT_OK(Execute(&builder, {}).status());
 }
 
+TEST_F(ConstantsTest, FullLike) {
+  XlaBuilder b(TestName());
+  auto val1 = Iota(&b, F32, 3);
+  auto val2 = FullLike(val1, 10);
+  val1 + val2;
+  ComputeAndCompareR1<float>(&b, {10, 11, 12}, {}, error_spec_);
+}
+
+TEST_F(ConstantsTest, IllegalFullLikeOnTuple) {
+  XlaBuilder b(TestName());
+  auto tuple = Tuple(&b, {Iota(&b, F32, 3), Iota(&b, F32, 1)});
+  FullLike(tuple, 10);  // Illegal; can't do FullLike on a tuple.
+  EXPECT_FALSE(b.Build().ok());
+}
+
+TEST_F(ConstantsTest, FullLikeScalar) {
+  XlaBuilder b(TestName());
+  auto scalar1 = ConstantR0WithType(&b, F32, 1);
+  auto scalar2 = FullLike(scalar1, 2);
+  scalar1 - scalar2;
+  ComputeAndCompareR0<float>(&b, -1, {}, error_spec_);
+}
+
 class ConstantsHloTest : public HloTestBase {};
 
 // TODO(b/121147351): Fails on GPU. Not clear if this is expected behavior.
-- 
GitLab


From c00907e0804478812d9193feb2c3657c288295b7 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 24 Jan 2019 19:18:24 -0800
Subject: [PATCH 1235/2345] [XLA] Fix implementation of acos(-1).

At least on fp16, it was returning 0, when it should have returned pi.

PiperOrigin-RevId: 230834226
---
 tensorflow/compiler/xla/client/lib/math.cc               | 9 ++++++---
 .../compiler/xla/client/lib/math_exhaustive_test.cc      | 6 ++----
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/tensorflow/compiler/xla/client/lib/math.cc b/tensorflow/compiler/xla/client/lib/math.cc
index 2426a433fc..f7e29db690 100644
--- a/tensorflow/compiler/xla/client/lib/math.cc
+++ b/tensorflow/compiler/xla/client/lib/math.cc
@@ -367,10 +367,13 @@ XlaOp RoundToEven(XlaOp x) {
 
 // Trigonometric functions.
 
-// acos(x) = 2 * atan(sqrt(1 - x^2) / (1 + x))
+// acos(x) = 2 * atan(sqrt(1 - x^2) / (1 + x)) if x != -1
+//           pi                                if x == -1
 XlaOp Acos(XlaOp x) {
-  return ScalarLike(x, 2.0) *
-         Atan2(Sqrt(ScalarLike(x, 1.0) - x * x), ScalarLike(x, 1.0) + x);
+  return Select(Ne(x, FullLike(x, -1)),
+                ScalarLike(x, 2.0) * Atan2(Sqrt(ScalarLike(x, 1.0) - x * x),
+                                           ScalarLike(x, 1.0) + x),
+                FullLike(x, M_PI));
 }
 
 // asin(x) = 2 * atan(x / (1 + sqrt(1 - x^2)))
diff --git a/tensorflow/compiler/xla/client/lib/math_exhaustive_test.cc b/tensorflow/compiler/xla/client/lib/math_exhaustive_test.cc
index 123af5e51c..0408ed1ba2 100644
--- a/tensorflow/compiler/xla/client/lib/math_exhaustive_test.cc
+++ b/tensorflow/compiler/xla/client/lib/math_exhaustive_test.cc
@@ -137,10 +137,7 @@ XLA_TEST_P(MathExhaustiveTest, DISABLED_ON_INTERPRETER(F16)) {
   ComputeAndCompareR1<half>(&b, expected_result, {}, tc.error);
 }
 
-// TODO(jlebar): The following tests are missing.
-//
-// - Fails on -1 (returns 0 instead of pi).
-//   Testcase{"acos", Acos, std::acos}.set_skip_infs(),
+// TODO(b/123355973): The following tests are missing.
 //
 // - Many failures.
 //   Testcase{"acosh", Acosh, std::acosh}.set_relaxed_nans(),
@@ -170,6 +167,7 @@ INSTANTIATE_TEST_CASE_P(
             .set_tolerance(0.1, 0.15)
             .set_fewer_infs_ok(),
         Testcase{"asin", Asin, std::asin}.set_skip_infs(),
+        Testcase{"acos", Acos, std::acos}.set_skip_infs(),
         Testcase{"atan", Atan, std::atan},
         Testcase{"tan", Tan, std::tan}.set_tolerance(0.05, 0.05),
     }));
-- 
GitLab


From a827a4bee10a3c02aef3493ad2f8c2572e8d506c Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 24 Jan 2019 19:25:06 -0800
Subject: [PATCH 1236/2345] [XLA] By default, display a detailed breakdown when
 comparing large literals.

Literal comparison has the option of displaying a "detailed breakdown", which
includes things like the distribution of errors.  Previously, you'd only get
this breakdown if you asked for it, but there are many routines that don't
expose this knob.  We could expose it piecemeal, but I think in general if
you're comparing a "large" literal and you have mismatches, you probably want
this breakdown.  At least, it's not going to be a lot of noise.  So this patch
makes us show the default breakdown by default when the array being compared is
above a certain size.

PiperOrigin-RevId: 230834776
---
 tensorflow/compiler/xla/literal_comparison.cc  | 18 ++++++++++--------
 tensorflow/compiler/xla/literal_comparison.h   |  5 +++--
 .../compiler/xla/tests/literal_test_util.cc    |  5 +++--
 .../compiler/xla/tests/literal_test_util.h     |  2 +-
 4 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/tensorflow/compiler/xla/literal_comparison.cc b/tensorflow/compiler/xla/literal_comparison.cc
index 07ae95063c..9b3de75dd4 100644
--- a/tensorflow/compiler/xla/literal_comparison.cc
+++ b/tensorflow/compiler/xla/literal_comparison.cc
@@ -736,7 +736,7 @@ Status EqualHelper(const LiteralSlice& expected, const LiteralSlice& actual) {
 // via recursion. shape_index is the ShapeIndex of expected (or actual)
 // currently being compared.
 Status NearHelper(const LiteralSlice& expected, const LiteralSlice& actual,
-                  const ErrorSpec& error, bool detailed_message,
+                  const ErrorSpec& error, absl::optional<bool> detailed_message,
                   const MiscompareCallback& miscompare_callback,
                   const ShapeIndex& shape_index) {
   TF_RETURN_IF_ERROR(EqualShapes(expected.shape(), actual.shape()));
@@ -777,30 +777,32 @@ Status NearHelper(const LiteralSlice& expected, const LiteralSlice& actual,
 
   if (ShapeUtil::ElementIsFloating(expected.shape()) ||
       ShapeUtil::ElementIsComplex(expected.shape())) {
+    bool use_detailed_message = detailed_message.value_or(
+        ShapeUtil::ElementsIn(expected.shape()) >= 64);
     switch (expected.shape().element_type()) {
       case BF16:
         return NearComparator<bfloat16>::Compare(
-            expected, actual, error, detailed_message, miscompare_callback);
+            expected, actual, error, use_detailed_message, miscompare_callback);
         break;
       case F16:
         return NearComparator<half>::Compare(
-            expected, actual, error, detailed_message, miscompare_callback);
+            expected, actual, error, use_detailed_message, miscompare_callback);
         break;
       case F32:
         return NearComparator<float>::Compare(
-            expected, actual, error, detailed_message, miscompare_callback);
+            expected, actual, error, use_detailed_message, miscompare_callback);
         break;
       case F64:
         return NearComparator<double>::Compare(
-            expected, actual, error, detailed_message, miscompare_callback);
+            expected, actual, error, use_detailed_message, miscompare_callback);
         break;
       case C64:
         return NearComparator<complex64>::Compare(
-            expected, actual, error, detailed_message, miscompare_callback);
+            expected, actual, error, use_detailed_message, miscompare_callback);
         break;
       case C128:
         return NearComparator<complex128>::Compare(
-            expected, actual, error, detailed_message, miscompare_callback);
+            expected, actual, error, use_detailed_message, miscompare_callback);
         break;
       default:
         LOG(FATAL) << "Unsupported primitive type in near comparator: "
@@ -891,7 +893,7 @@ Status Equal(const LiteralSlice& expected, const LiteralSlice& actual) {
 }
 
 Status Near(const LiteralSlice& expected, const LiteralSlice& actual,
-            const ErrorSpec& error, bool detailed_message,
+            const ErrorSpec& error, absl::optional<bool> detailed_message,
             const MiscompareCallback& miscompare_callback) {
   VLOG(1) << "Expected literal:";
   XLA_VLOG_LINES(1, expected.ToString());
diff --git a/tensorflow/compiler/xla/literal_comparison.h b/tensorflow/compiler/xla/literal_comparison.h
index 9e5bf7c1d0..23fff3fa34 100644
--- a/tensorflow/compiler/xla/literal_comparison.h
+++ b/tensorflow/compiler/xla/literal_comparison.h
@@ -55,9 +55,10 @@ using MiscompareCallback =
 // being compared.
 //
 // If detailed_message is true, then the error message in the assertion result
-// will contain a more detailed breakdown of mismatches.
+// will contain a more detailed breakdown of mismatches.  By default, we display
+// a detailed message only for "large" inputs.
 Status Near(const LiteralSlice& expected, const LiteralSlice& actual,
-            const ErrorSpec& error, bool detailed_message,
+            const ErrorSpec& error, absl::optional<bool> detailed_message,
             const MiscompareCallback& miscompare_callback);
 
 // Calling ToString on a literal with over 100 million elements takes around
diff --git a/tensorflow/compiler/xla/tests/literal_test_util.cc b/tensorflow/compiler/xla/tests/literal_test_util.cc
index 554eb24d44..a2fd607073 100644
--- a/tensorflow/compiler/xla/tests/literal_test_util.cc
+++ b/tensorflow/compiler/xla/tests/literal_test_util.cc
@@ -86,7 +86,7 @@ void OnMiscompare(const LiteralSlice& expected, const LiteralSlice& actual,
 
 /* static */ ::testing::AssertionResult LiteralTestUtil::Near(
     const LiteralSlice& expected, const LiteralSlice& actual,
-    const ErrorSpec& error_spec, bool detailed_message) {
+    const ErrorSpec& error_spec, absl::optional<bool> detailed_message) {
   return StatusToAssertion(literal_comparison::Near(
       expected, actual, error_spec, detailed_message, &OnMiscompare));
 }
@@ -97,7 +97,8 @@ void OnMiscompare(const LiteralSlice& expected, const LiteralSlice& actual,
   if (error.has_value()) {
     VLOG(1) << "Expects near";
     return StatusToAssertion(literal_comparison::Near(
-        expected, actual, *error, /*detailed_message=*/false, &OnMiscompare));
+        expected, actual, *error, /*detailed_message=*/absl::nullopt,
+        &OnMiscompare));
   }
   VLOG(1) << "Expects equal";
   return StatusToAssertion(literal_comparison::Equal(expected, actual));
diff --git a/tensorflow/compiler/xla/tests/literal_test_util.h b/tensorflow/compiler/xla/tests/literal_test_util.h
index 43cca91f64..d7cf9bed98 100644
--- a/tensorflow/compiler/xla/tests/literal_test_util.h
+++ b/tensorflow/compiler/xla/tests/literal_test_util.h
@@ -93,7 +93,7 @@ class LiteralTestUtil {
   static ::testing::AssertionResult Near(
       const LiteralSlice& expected, const LiteralSlice& actual,
       const ErrorSpec& error_spec,
-      bool detailed_message = false) TF_MUST_USE_RESULT;
+      absl::optional<bool> detailed_message = absl::nullopt) TF_MUST_USE_RESULT;
 
   // Asserts the given literal are within the given error bound of the given
   // expected values. Only supported for floating point values.
-- 
GitLab


From 5a901abe024d7abb6bf78b92426fc4e34b533542 Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Thu, 24 Jan 2019 19:38:43 -0800
Subject: [PATCH 1237/2345] gRPC service: add configuration parameters for
 queue depth and thread count.

Refactor GrpcServer, WorkerService to take option structs.

PiperOrigin-RevId: 230836072
---
 tensorflow/contrib/gdr/BUILD                  |  13 +-
 .../contrib/gdr/gdr_collective_executor_mgr.h |   4 +-
 tensorflow/contrib/gdr/gdr_server_lib.cc      |   7 +-
 tensorflow/contrib/gdr/gdr_worker.cc          |   2 -
 .../rpc/grpc_server_lib.cc                    |  63 +-
 .../distributed_runtime/rpc/grpc_server_lib.h |  32 +-
 .../rpc/grpc_worker_service.cc                | 594 +++++++++---------
 .../rpc/grpc_worker_service.h                 |  12 +-
 .../rpc/grpc_worker_service_impl.h            |   1 +
 9 files changed, 354 insertions(+), 374 deletions(-)

diff --git a/tensorflow/contrib/gdr/BUILD b/tensorflow/contrib/gdr/BUILD
index 89314d162b..bf8b66dcfa 100644
--- a/tensorflow/contrib/gdr/BUILD
+++ b/tensorflow/contrib/gdr/BUILD
@@ -17,11 +17,6 @@ filegroup(
     ]),
 )
 
-load(
-    "//tensorflow:tensorflow.bzl",
-    "tf_cuda_library",
-)
-
 # For platform specific build config
 load(
     "//tensorflow/core:platform/default/build_config.bzl",
@@ -66,7 +61,6 @@ cc_library(
         ":gdr_memory_manager",
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
-        "//tensorflow/core:gpu_runtime",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core/distributed_runtime:graph_mgr",
@@ -108,15 +102,13 @@ cc_library(
         ":gdr_memory_manager",
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
         "//tensorflow/core/distributed_runtime:cancellable_call",
         "//tensorflow/core/distributed_runtime:collective_param_resolver_distributed",
         "//tensorflow/core/distributed_runtime:device_resolver_distributed",
         "//tensorflow/core/distributed_runtime:request_id",
         "//tensorflow/core/distributed_runtime:rpc_collective_executor_mgr",
         "//tensorflow/core/distributed_runtime:worker_cache",
-        "//tensorflow/core/distributed_runtime:worker_env",
-        "//tensorflow/core/distributed_runtime:worker_interface",
     ],
 )
 
@@ -130,6 +122,9 @@ cc_library(
         ":gdr_memory_manager",
         ":gdr_rendezvous_mgr",
         ":gdr_worker",
+        "//tensorflow/core:core_cpu_internal",
+        "//tensorflow/core/distributed_runtime:collective_param_resolver_distributed",
+        "//tensorflow/core/distributed_runtime:device_resolver_distributed",
         "//tensorflow/core/distributed_runtime/rpc:grpc_server_lib",
     ],
     alwayslink = 1,
diff --git a/tensorflow/contrib/gdr/gdr_collective_executor_mgr.h b/tensorflow/contrib/gdr/gdr_collective_executor_mgr.h
index a713c1c6b0..1417e51e82 100644
--- a/tensorflow/contrib/gdr/gdr_collective_executor_mgr.h
+++ b/tensorflow/contrib/gdr/gdr_collective_executor_mgr.h
@@ -16,14 +16,14 @@ limitations under the License.
 #define TENSORFLOW_CONTRIB_GDR_GDR_COLLECTIVE_EXECUTOR_MGR_H_
 
 #include "tensorflow/contrib/gdr/gdr_memory_manager.h"
+#include "tensorflow/core/distributed_runtime/collective_param_resolver_distributed.h"
+#include "tensorflow/core/distributed_runtime/device_resolver_distributed.h"
 #include "tensorflow/core/distributed_runtime/rpc_collective_executor_mgr.h"
 #include "tensorflow/core/framework/collective.h"
 
 namespace tensorflow {
-class CollectiveParamResolverDistributed;
 class ConfigProto;
 class DeviceMgr;
-class DeviceResolverDistributed;
 class WorkerCacheInterface;
 class StepSequenceRequest;
 class StepSequenceResponse;
diff --git a/tensorflow/contrib/gdr/gdr_server_lib.cc b/tensorflow/contrib/gdr/gdr_server_lib.cc
index e4ae1eec23..c39cc0f9bc 100644
--- a/tensorflow/contrib/gdr/gdr_server_lib.cc
+++ b/tensorflow/contrib/gdr/gdr_server_lib.cc
@@ -82,8 +82,11 @@ Status GdrServer::Init() {
       };
   TF_RETURN_IF_ERROR(remote_memory_manager_->Init());
 
-  return GrpcServer::Init(nullptr, rendezvous_mgr_func, collective_mgr_func,
-                          worker_func);
+  GrpcServerOptions opts;
+  opts.rendezvous_mgr_func = rendezvous_mgr_func;
+  opts.collective_mgr_func = collective_mgr_func;
+  opts.worker_func = worker_func;
+  return GrpcServer::Init(opts);
 }
 
 Status GdrServer::Start() {
diff --git a/tensorflow/contrib/gdr/gdr_worker.cc b/tensorflow/contrib/gdr/gdr_worker.cc
index 86897e3c8e..1204b8ca50 100644
--- a/tensorflow/contrib/gdr/gdr_worker.cc
+++ b/tensorflow/contrib/gdr/gdr_worker.cc
@@ -35,8 +35,6 @@ limitations under the License.
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/tracing.h"
-#include "tensorflow/core/protobuf/transport_options.pb.h"
-#include "tensorflow/core/protobuf/worker.pb.h"
 
 namespace tensorflow {
 
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc b/tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc
index ac73182190..1405c760d5 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc
@@ -112,12 +112,7 @@ GrpcServer::~GrpcServer() {
 
 void GrpcServer::MaybeMutateBuilder(::grpc::ServerBuilder* builder) {}
 
-Status GrpcServer::Init(
-    ServiceInitFunction service_func,
-    const RendezvousMgrCreationFunction& rendezvous_mgr_func,
-    const CollectiveMgrCreationFunction& collective_mgr_func,
-    const WorkerCreationFunction& worker_func,
-    const StatsPublisherFactory& stats_factory) {
+Status GrpcServer::Init(const GrpcServerOptions& opts) {
   mutex_lock l(mu_);
   CHECK_EQ(state_, NEW);
   master_env_.env = env_;
@@ -165,9 +160,9 @@ Status GrpcServer::Init(
   worker_env_.device_mgr = new DeviceMgr(std::move(devices));
   master_env_.local_devices = worker_env_.device_mgr->ListDevices();
   worker_env_.local_devices = worker_env_.device_mgr->ListDevices();
-  worker_env_.rendezvous_mgr = rendezvous_mgr_func == nullptr
+  worker_env_.rendezvous_mgr = opts.rendezvous_mgr_func == nullptr
                                    ? new RpcRendezvousMgr(&worker_env_)
-                                   : rendezvous_mgr_func(&worker_env_);
+                                   : opts.rendezvous_mgr_func(&worker_env_);
   string unused;
   string default_worker_name;
   if (!DeviceNameUtils::SplitDeviceName(master_env_.local_devices[0]->name(),
@@ -200,15 +195,16 @@ Status GrpcServer::Init(
   MaybeMutateBuilder(&builder);
   master_impl_ = CreateMaster(&master_env_);
   master_service_ = NewGrpcMasterService(master_impl_.get(), config, &builder);
-  worker_impl_ = worker_func ? worker_func(&worker_env_, config)
-                             : NewGrpcWorker(&worker_env_, config);
-  worker_service_ =
-      NewGrpcWorkerService(worker_impl_.get(), &builder).release();
+  worker_impl_ = opts.worker_func ? opts.worker_func(&worker_env_, config)
+                                  : NewGrpcWorker(&worker_env_, config);
+  worker_service_ = NewGrpcWorkerService(worker_impl_.get(), &builder,
+                                         opts.worker_service_options)
+                        .release();
   eager_service_ = new eager::GrpcEagerServiceImpl(&worker_env_, &builder);
 
   // extra service:
-  if (service_func != nullptr) {
-    service_func(&worker_env_, &builder);
+  if (opts.service_func != nullptr) {
+    opts.service_func(&worker_env_, &builder);
   }
   server_ = builder.BuildAndStart();
 
@@ -222,9 +218,9 @@ Status GrpcServer::Init(
       WorkerCacheFactory(worker_cache_factory_options, &worker_cache));
   CHECK_NE(nullptr, worker_cache);
 
-  if (collective_mgr_func) {
+  if (opts.collective_mgr_func) {
     worker_env_.collective_executor_mgr =
-        collective_mgr_func(config, &worker_env_, worker_cache);
+        opts.collective_mgr_func(config, &worker_env_, worker_cache);
     if (!worker_env_.collective_executor_mgr) {
       return errors::Internal(
           "collective_mgr_func did not return CollectiveExecutorMgr");
@@ -256,6 +252,7 @@ Status GrpcServer::Init(
   master_env_.ops = OpRegistry::Global();
   master_env_.worker_cache = worker_cache;
   master_env_.collective_executor_mgr = worker_env_.collective_executor_mgr;
+  StatsPublisherFactory stats_factory = opts.stats_factory;
   master_env_.master_session_factory =
       [config, stats_factory](
           SessionOptions options, const MasterEnv* env,
@@ -282,31 +279,6 @@ Status GrpcServer::Init(
   return Status::OK();
 }
 
-Status GrpcServer::Init(
-    ServiceInitFunction service_func,
-    const RendezvousMgrCreationFunction& rendezvous_mgr_func,
-    const CollectiveMgrCreationFunction& collective_mgr_func,
-    const WorkerCreationFunction& worker_func) {
-  return Init(std::move(service_func), rendezvous_mgr_func, collective_mgr_func,
-              worker_func, CreateNoOpStatsPublisher);
-}
-
-Status GrpcServer::Init(
-    ServiceInitFunction service_func,
-    const RendezvousMgrCreationFunction& rendezvous_mgr_func,
-    const CollectiveMgrCreationFunction& collective_mgr_func) {
-  return Init(std::move(service_func), rendezvous_mgr_func, collective_mgr_func,
-              nullptr);
-}
-
-Status GrpcServer::Init(
-    ServiceInitFunction service_func,
-    const RendezvousMgrCreationFunction& rendezvous_mgr_func) {
-  return Init(std::move(service_func), rendezvous_mgr_func, nullptr, nullptr);
-}
-
-Status GrpcServer::Init() { return Init(nullptr, nullptr, nullptr, nullptr); }
-
 Status GrpcServer::ParseChannelSpec(const WorkerCacheFactoryOptions& options,
                                     GrpcChannelSpec* channel_spec) {
   for (const auto& job : options.cluster_def->job()) {
@@ -457,7 +429,9 @@ Status GrpcServer::Create(const ServerDef& server_def, Env* env,
   std::unique_ptr<GrpcServer> ret(
       new GrpcServer(server_def, env == nullptr ? Env::Default() : env));
   ServiceInitFunction service_func = nullptr;
-  Status s = ret->Init(service_func, NewRpcRendezvousMgr, nullptr);
+  GrpcServerOptions options;
+  options.rendezvous_mgr_func = NewRpcRendezvousMgr;
+  Status s = ret->Init();
   if (!s.ok()) {
     LOG(ERROR) << s;
     return s;
@@ -471,8 +445,9 @@ Status GrpcServer::Create(const ServerDef& server_def, Env* env,
                           std::unique_ptr<GrpcServer>* out_server) {
   std::unique_ptr<GrpcServer> ret(
       new GrpcServer(server_def, env == nullptr ? Env::Default() : env));
-  ServiceInitFunction service_func = nullptr;
-  Status s = ret->Init(service_func, NewRpcRendezvousMgr, nullptr);
+  GrpcServerOptions options;
+  options.rendezvous_mgr_func = NewRpcRendezvousMgr;
+  Status s = ret->Init(options);
   if (!s.ok()) {
     LOG(ERROR) << s;
     return s;
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_server_lib.h b/tensorflow/core/distributed_runtime/rpc/grpc_server_lib.h
index c7f543e5bf..f66d7eb82e 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_server_lib.h
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_server_lib.h
@@ -16,6 +16,8 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_SERVER_LIB_H_
 #define TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_SERVER_LIB_H_
 
+// GrpcServer manages the lifecycle of an Eager, Worker and Master service.
+
 #include <memory>
 
 #include "grpcpp/grpcpp.h"
@@ -26,6 +28,7 @@ limitations under the License.
 #include "tensorflow/core/distributed_runtime/master_env.h"
 #include "tensorflow/core/distributed_runtime/rpc/async_service_interface.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_channel.h"
+#include "tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h"
 #include "tensorflow/core/distributed_runtime/server_lib.h"
 #include "tensorflow/core/distributed_runtime/session_mgr.h"
 #include "tensorflow/core/distributed_runtime/worker_env.h"
@@ -57,6 +60,15 @@ typedef std::function<std::unique_ptr<GrpcWorker>(WorkerEnv*,
                                                   const ConfigProto& config)>
     WorkerCreationFunction;
 
+struct GrpcServerOptions {
+  ServiceInitFunction service_func = nullptr;
+  RendezvousMgrCreationFunction rendezvous_mgr_func = nullptr;
+  CollectiveMgrCreationFunction collective_mgr_func = nullptr;
+  WorkerCreationFunction worker_func = nullptr;
+  StatsPublisherFactory stats_factory = CreateNoOpStatsPublisher;
+  GrpcWorkerServiceOptions worker_service_options;
+};
+
 class GrpcServer : public ServerInterface {
  protected:
   GrpcServer(const ServerDef& server_def, Env* env);
@@ -86,25 +98,7 @@ class GrpcServer : public ServerInterface {
   std::shared_ptr<GrpcChannelCache> channel_cache() { return channel_cache_; }
 
  protected:
-  Status Init(ServiceInitFunction service_func,
-              const RendezvousMgrCreationFunction& rendezvous_mgr_func,
-              const CollectiveMgrCreationFunction& collective_mgr_func,
-              const WorkerCreationFunction& worker_func,
-              const StatsPublisherFactory& stats_factory);
-
-  Status Init(ServiceInitFunction service_func,
-              const RendezvousMgrCreationFunction& rendezvous_mgr_func,
-              const CollectiveMgrCreationFunction& collective_mgr_func,
-              const WorkerCreationFunction& worker_func);
-
-  Status Init(ServiceInitFunction service_func,
-              const RendezvousMgrCreationFunction& rendezvous_mgr_func,
-              const CollectiveMgrCreationFunction& collective_mgr_func);
-
-  Status Init(ServiceInitFunction service_func,
-              const RendezvousMgrCreationFunction& rendezvous_mgr_func);
-
-  Status Init();
+  Status Init(const GrpcServerOptions& opts = GrpcServerOptions());
 
   // A subclass can override this method to support secure credentials.
   virtual std::shared_ptr<::grpc::ServerCredentials> GetServerCredentials(
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc
index de80992095..db0ad6124c 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h"
 
 #include <deque>
+#include <unordered_map>
 
 #include "grpcpp/alarm.h"
 #include "grpcpp/server_builder.h"
@@ -41,6 +42,7 @@ limitations under the License.
 #include "tensorflow/core/framework/collective.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/gtl/map_util.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/tracing.h"
 #include "tensorflow/core/protobuf/transport_options.pb.h"
@@ -50,37 +52,6 @@ namespace tensorflow {
 
 namespace {
 
-class GrpcWorkerService : public AsyncServiceInterface {
-  // TODO(ncteisen): consider adding a config var or flag for this
-  static constexpr const size_t kGrpcWorkerServiceThreadCount = 8;
-
- public:
-  GrpcWorkerService(GrpcWorker* worker, ::grpc::ServerBuilder* builder)
-      : is_shutdown_(false) {
-    builder->RegisterService(&worker_service_);
-    for (int i = 0; i < kGrpcWorkerServiceThreadCount; i++) {
-      threads_.emplace_back(
-          new GrpcWorkerServiceThread(worker, builder, &worker_service_));
-    }
-  }
-
-  void Shutdown() override {
-    bool did_shutdown = false;
-    {
-      mutex_lock l(service_shutdown_mu_);
-      if (!is_shutdown_) {
-        LOG(INFO) << "Shutting down GrpcWorkerService.";
-        is_shutdown_ = true;
-        did_shutdown = true;
-      }
-    }
-    if (did_shutdown) {
-      for (auto& worker_thread : threads_) {
-        worker_thread->Shutdown();
-      }
-    }
-  }
-
 // This macro creates a new request for the given RPC method name
 // (e.g., `ENQUEUE_REQUEST(GetStatus, false);`), and enqueues it on
 // `this->cq_`.
@@ -105,308 +76,340 @@ class GrpcWorkerService : public AsyncServiceInterface {
     }                                                                        \
   } while (0)
 
-  // This method blocks forever handling requests from the completion queue.
-  void HandleRPCsLoop() override {
-    for (auto& worker_thread : threads_) {
-      worker_thread->Start();
-    }
-    for (auto& worker_thread : threads_) {
-      worker_thread->Join();
-    }
+#define SETUP_FOR_REQUEST(method, default_depth, supports_cancel)              \
+  for (int i = 0;                                                              \
+       i < gtl::FindWithDefault(queue_depth_,                                  \
+                                static_cast<int>(GrpcWorkerMethod::k##method), \
+                                default_depth);                                \
+       ++i) {                                                                  \
+    ENQUEUE_REQUEST(method, supports_cancel);                                  \
   }
 
- private:
-  // Thread wrapping class that drives work over a single gRPC
-  // CompletionQueue.
-  class GrpcWorkerServiceThread {
-   public:
-    explicit GrpcWorkerServiceThread(
-        GrpcWorker* worker, ::grpc::ServerBuilder* builder,
-        grpc::WorkerService::AsyncService* worker_service)
-        : worker_(worker),
-          worker_service_(worker_service),
-          is_shutdown_(false) {
-      cq_ = builder->AddCompletionQueue();
+// GrpcWorkerService spawns one or more GrpcWorkerServiceThreads to service
+// requests.  Each thread operates on an independent completion queue.
+class GrpcWorkerServiceThread {
+ public:
+  explicit GrpcWorkerServiceThread(
+      GrpcWorker* worker, ::grpc::ServerBuilder* builder,
+      std::unordered_map<int, int> queue_depth,
+      grpc::WorkerService::AsyncService* worker_service)
+      : worker_(worker),
+        queue_depth_(queue_depth),
+        worker_service_(worker_service),
+        is_shutdown_(false) {
+    cq_ = builder->AddCompletionQueue();
+  }
+
+  void Start() {
+    thread_.reset(
+        worker_->env()->env->StartThread(ThreadOptions(), "grpc_worker_service",
+                                         [this]() { HandleRPCsLoop(); }));
+  }
+
+  void Join() { thread_.reset(); }  // Blocks until thread exits
+
+  void Shutdown() {
+    {
+      mutex_lock lock(shutdown_mu_);
+      is_shutdown_ = true;
     }
+    cq_->Shutdown();
+  }
 
-    void Start() {
-      thread_.reset(worker_->env()->env->StartThread(
-          ThreadOptions(), "grpc_worker_service",
-          [this]() { HandleRPCsLoop(); }));
+ private:
+  // Add one or more completion queue entries for each worker method, then
+  // begin servicing requests from the completion queue.
+  void HandleRPCsLoop() {
+    // TODO(ncteisen): This may require performance engineering. We can
+    // change the number of threads, the number of handlers per thread,
+    // or even decide to specialize certain threads to certain methods.
+    SETUP_FOR_REQUEST(GetStatus, 1, false);
+    SETUP_FOR_REQUEST(CreateWorkerSession, 1, false);
+    SETUP_FOR_REQUEST(DeleteWorkerSession, 1, false);
+    SETUP_FOR_REQUEST(CleanupAll, 1, false);
+    SETUP_FOR_REQUEST(RegisterGraph, 1, false);
+    SETUP_FOR_REQUEST(DeregisterGraph, 1, false);
+    SETUP_FOR_REQUEST(Logging, 1, false);
+    SETUP_FOR_REQUEST(Tracing, 1, false);
+    SETUP_FOR_REQUEST(CompleteGroup, 10, true);
+    SETUP_FOR_REQUEST(CompleteInstance, 10, true);
+    SETUP_FOR_REQUEST(GetStepSequence, 10, true);
+    SETUP_FOR_REQUEST(RecvBuf, 500, true);
+    SETUP_FOR_REQUEST(RunGraph, 100, true);
+    SETUP_FOR_REQUEST(CleanupGraph, 100, false);
+
+    // TODO(ncteisen): Determine a better policy for enqueuing the
+    // appropriate number of each request type.
+    for (int i = 0;
+         i < gtl::FindWithDefault(
+                 queue_depth_, static_cast<int>(GrpcWorkerMethod::kRecvTensor),
+                 1000);
+         ++i) {
+      EnqueueRecvTensorRequestRaw();
     }
 
-    void Join() { thread_.reset(); }  // Blocks until thread exits
+    void* tag;
+    bool ok;
 
-    void Shutdown() {
-      {
-        mutex_lock lock(shutdown_mu_);
-        is_shutdown_ = true;
-      }
-      cq_->Shutdown();
+    while (cq_->Next(&tag, &ok)) {
+      UntypedCall<GrpcWorkerServiceThread>::Tag* callback_tag =
+          static_cast<UntypedCall<GrpcWorkerServiceThread>::Tag*>(tag);
+      CHECK(callback_tag);
+      callback_tag->OnCompleted(this, ok);
     }
+  }
 
-   private:
-    void HandleRPCsLoop() {
-      // TODO(ncteisen): This may require performance engineering. We can
-      // change the number of threads, the number of handlers per thread,
-      // or even decide to specialize certain threads to certain methods.
-      ENQUEUE_REQUEST(GetStatus, false);
-      ENQUEUE_REQUEST(CreateWorkerSession, false);
-      ENQUEUE_REQUEST(DeleteWorkerSession, false);
-      ENQUEUE_REQUEST(CleanupAll, false);
-      ENQUEUE_REQUEST(RegisterGraph, false);
-      ENQUEUE_REQUEST(DeregisterGraph, false);
-
-      // TODO(ncteisen): Determine a better policy for enqueuing the
-      // appropriate number of each request type.
-      for (int i = 0; i < 1000; ++i) {
-        EnqueueRecvTensorRequestRaw();
-      }
-      for (int i = 0; i < 500; ++i) {
-        ENQUEUE_REQUEST(RecvBuf, true);
-      }
-      for (int i = 0; i < 100; ++i) {
-        ENQUEUE_REQUEST(RunGraph, true);
-      }
-      for (int i = 0; i < 100; ++i) {
-        ENQUEUE_REQUEST(CleanupGraph, false);
-      }
+ private:
+  void Schedule(std::function<void()> f) {
+    worker_->env()->compute_pool->Schedule(std::move(f));
+  }
 
-      ENQUEUE_REQUEST(Logging, false);
-      ENQUEUE_REQUEST(Tracing, false);
+  // The following section contains one request handler method per
+  // RPC. The `FooHandler` method is called (indirectly) by
+  // `HandleRPCsLoop()` when the next Foo RPC is received. Each
+  // `FooHandler` call schedules a closure on `worker_->env()->compute_pool`,
+  // and is responsible for requesting the next Foo call by calling
+  // `ENQUEUE_REQUEST(Foo)`.
+  template <class RequestMessage, class ResponseMessage>
+  using WorkerCall =
+      Call<GrpcWorkerServiceThread, grpc::WorkerService::AsyncService,
+           RequestMessage, ResponseMessage>;
+
+  void GetStatusHandler(WorkerCall<GetStatusRequest, GetStatusResponse>* call) {
+    Schedule([this, call]() {
+      Status s = worker_->GetStatus(&call->request, &call->response);
+      call->SendResponse(ToGrpcStatus(s));
+    });
+    ENQUEUE_REQUEST(GetStatus, false);
+  }
 
-      for (int i = 0; i < 10; ++i) {
-        ENQUEUE_REQUEST(CompleteGroup, true);
-        ENQUEUE_REQUEST(CompleteInstance, true);
-        ENQUEUE_REQUEST(GetStepSequence, true);
-      }
+  void CreateWorkerSessionHandler(
+      WorkerCall<CreateWorkerSessionRequest, CreateWorkerSessionResponse>*
+          call) {
+    Schedule([this, call]() {
+      Status s = worker_->CreateWorkerSession(&call->request, &call->response);
+      call->SendResponse(ToGrpcStatus(s));
+    });
+    ENQUEUE_REQUEST(CreateWorkerSession, false);
+  }
 
-      void* tag;
-      bool ok;
+  void DeleteWorkerSessionHandler(
+      WorkerCall<DeleteWorkerSessionRequest, DeleteWorkerSessionResponse>*
+          call) {
+    Schedule([this, call]() {
+      Status s = worker_->DeleteWorkerSession(&call->request, &call->response);
+      call->SendResponse(ToGrpcStatus(s));
+    });
+    ENQUEUE_REQUEST(DeleteWorkerSession, false);
+  }
 
-      while (cq_->Next(&tag, &ok)) {
-        UntypedCall<GrpcWorkerServiceThread>::Tag* callback_tag =
-            static_cast<UntypedCall<GrpcWorkerServiceThread>::Tag*>(tag);
-        CHECK(callback_tag);
-        callback_tag->OnCompleted(this, ok);
-      }
-    }
+  void CleanupAllHandler(
+      WorkerCall<CleanupAllRequest, CleanupAllResponse>* call) {
+    Schedule([this, call]() {
+      Status s = worker_->CleanupAll(&call->request, &call->response);
+      call->SendResponse(ToGrpcStatus(s));
+    });
+    ENQUEUE_REQUEST(CleanupAll, false);
+  }
 
-   private:
-    void Schedule(std::function<void()> f) {
-      worker_->env()->compute_pool->Schedule(std::move(f));
-    }
+  void RegisterGraphHandler(
+      WorkerCall<RegisterGraphRequest, RegisterGraphResponse>* call) {
+    Schedule([this, call]() {
+      Status s = worker_->RegisterGraph(&call->request, &call->response);
+      call->SendResponse(ToGrpcStatus(s));
+    });
+    ENQUEUE_REQUEST(RegisterGraph, false);
+  }
 
-    // The following section contains one request handler method per
-    // RPC. The `FooHandler` method is called (indirectly) by
-    // `HandleRPCsLoop()` when the next Foo RPC is received. Each
-    // `FooHandler` call schedules a closure on `worker_->env()->compute_pool`,
-    // and is responsible for requesting the next Foo call by calling
-    // `ENQUEUE_REQUEST(Foo)`.
-
-    template <class RequestMessage, class ResponseMessage>
-    using WorkerCall =
-        Call<GrpcWorkerServiceThread, grpc::WorkerService::AsyncService,
-             RequestMessage, ResponseMessage>;
-
-    void GetStatusHandler(
-        WorkerCall<GetStatusRequest, GetStatusResponse>* call) {
-      Schedule([this, call]() {
-        Status s = worker_->GetStatus(&call->request, &call->response);
-        call->SendResponse(ToGrpcStatus(s));
-      });
-      ENQUEUE_REQUEST(GetStatus, false);
-    }
+  void DeregisterGraphHandler(
+      WorkerCall<DeregisterGraphRequest, DeregisterGraphResponse>* call) {
+    Schedule([this, call]() {
+      Status s = worker_->DeregisterGraph(&call->request, &call->response);
+      call->SendResponse(ToGrpcStatus(s));
+    });
+    ENQUEUE_REQUEST(DeregisterGraph, false);
+  }
 
-    void CreateWorkerSessionHandler(
-        WorkerCall<CreateWorkerSessionRequest, CreateWorkerSessionResponse>*
-            call) {
-      Schedule([this, call]() {
-        Status s =
-            worker_->CreateWorkerSession(&call->request, &call->response);
-        call->SendResponse(ToGrpcStatus(s));
-      });
-      ENQUEUE_REQUEST(CreateWorkerSession, false);
-    }
+  void RunGraphHandler(WorkerCall<RunGraphRequest, RunGraphResponse>* call) {
+    Schedule([this, call]() {
+      CallOptions* call_opts = new CallOptions;
+      ProtoRunGraphRequest* wrapped_request =
+          new ProtoRunGraphRequest(&call->request);
+      NonOwnedProtoRunGraphResponse* wrapped_response =
+          new NonOwnedProtoRunGraphResponse(&call->response);
+      call->SetCancelCallback([call_opts]() { call_opts->StartCancel(); });
+      worker_->RunGraphAsync(call_opts, wrapped_request, wrapped_response,
+                             [call, call_opts, wrapped_request,
+                              wrapped_response](const Status& s) {
+                               call->ClearCancelCallback();
+                               delete call_opts;
+                               delete wrapped_request;
+                               delete wrapped_response;
+                               call->SendResponse(ToGrpcStatus(s));
+                             });
+    });
+    ENQUEUE_REQUEST(RunGraph, true);
+  }
 
-    void DeleteWorkerSessionHandler(
-        WorkerCall<DeleteWorkerSessionRequest, DeleteWorkerSessionResponse>*
-            call) {
-      Schedule([this, call]() {
-        Status s =
-            worker_->DeleteWorkerSession(&call->request, &call->response);
-        call->SendResponse(ToGrpcStatus(s));
-      });
-      ENQUEUE_REQUEST(DeleteWorkerSession, false);
-    }
+  void RecvTensorHandlerRaw(
+      WorkerCall<RecvTensorRequest, ::grpc::ByteBuffer>* call) {
+    Schedule([this, call]() {
+      CallOptions* call_opts = new CallOptions;
+      call->SetCancelCallback([call_opts]() { call_opts->StartCancel(); });
+      worker_->GrpcRecvTensorAsync(call_opts, &call->request, &call->response,
+                                   [call, call_opts](const Status& s) {
+                                     call->ClearCancelCallback();
+                                     delete call_opts;
+                                     call->SendResponse(ToGrpcStatus(s));
+                                   });
+    });
+    EnqueueRecvTensorRequestRaw();
+  }
 
-    void CleanupAllHandler(
-        WorkerCall<CleanupAllRequest, CleanupAllResponse>* call) {
-      Schedule([this, call]() {
-        Status s = worker_->CleanupAll(&call->request, &call->response);
-        call->SendResponse(ToGrpcStatus(s));
-      });
-      ENQUEUE_REQUEST(CleanupAll, false);
-    }
+  void CleanupGraphHandler(
+      WorkerCall<CleanupGraphRequest, CleanupGraphResponse>* call) {
+    Schedule([this, call]() {
+      Status s = worker_->CleanupGraph(&call->request, &call->response);
+      call->SendResponse(ToGrpcStatus(s));
+    });
+    ENQUEUE_REQUEST(CleanupGraph, false);
+  }
 
-    void RegisterGraphHandler(
-        WorkerCall<RegisterGraphRequest, RegisterGraphResponse>* call) {
-      Schedule([this, call]() {
-        Status s = worker_->RegisterGraph(&call->request, &call->response);
-        call->SendResponse(ToGrpcStatus(s));
-      });
-      ENQUEUE_REQUEST(RegisterGraph, false);
-    }
+  void LoggingHandler(WorkerCall<LoggingRequest, LoggingResponse>* call) {
+    Schedule([this, call]() {
+      Status s = worker_->Logging(&call->request, &call->response);
+      call->SendResponse(ToGrpcStatus(s));
+    });
+    ENQUEUE_REQUEST(Logging, false);
+  }
 
-    void DeregisterGraphHandler(
-        WorkerCall<DeregisterGraphRequest, DeregisterGraphResponse>* call) {
-      Schedule([this, call]() {
-        Status s = worker_->DeregisterGraph(&call->request, &call->response);
-        call->SendResponse(ToGrpcStatus(s));
-      });
-      ENQUEUE_REQUEST(DeregisterGraph, false);
-    }
+  void TracingHandler(WorkerCall<TracingRequest, TracingResponse>* call) {
+    Schedule([this, call]() {
+      Status s = worker_->Tracing(&call->request, &call->response);
+      call->SendResponse(ToGrpcStatus(s));
+    });
+    ENQUEUE_REQUEST(Tracing, false);
+  }
 
-    void RunGraphHandler(WorkerCall<RunGraphRequest, RunGraphResponse>* call) {
-      Schedule([this, call]() {
-        CallOptions* call_opts = new CallOptions;
-        ProtoRunGraphRequest* wrapped_request =
-            new ProtoRunGraphRequest(&call->request);
-        NonOwnedProtoRunGraphResponse* wrapped_response =
-            new NonOwnedProtoRunGraphResponse(&call->response);
-        call->SetCancelCallback([call_opts]() { call_opts->StartCancel(); });
-        worker_->RunGraphAsync(call_opts, wrapped_request, wrapped_response,
-                               [call, call_opts, wrapped_request,
-                                wrapped_response](const Status& s) {
-                                 call->ClearCancelCallback();
-                                 delete call_opts;
-                                 delete wrapped_request;
-                                 delete wrapped_response;
-                                 call->SendResponse(ToGrpcStatus(s));
-                               });
-      });
-      ENQUEUE_REQUEST(RunGraph, true);
-    }
+  void RecvBufHandler(WorkerCall<RecvBufRequest, RecvBufResponse>* call) {
+    Schedule([this, call]() {
+      CallOptions* call_opts = new CallOptions;
+      call->SetCancelCallback([call_opts]() { call_opts->StartCancel(); });
+      worker_->RecvBufAsync(call_opts, &call->request, &call->response,
+                            [call, call_opts](const Status& s) {
+                              call->ClearCancelCallback();
+                              delete call_opts;
+                              call->SendResponse(ToGrpcStatus(s));
+                            });
+    });
+    ENQUEUE_REQUEST(RecvBuf, true);
+  }
+
+  void CompleteGroupHandler(
+      WorkerCall<CompleteGroupRequest, CompleteGroupResponse>* call) {
+    Schedule([this, call]() {
+      CallOptions* call_opts = new CallOptions;
+      call->SetCancelCallback([call_opts]() { call_opts->StartCancel(); });
+      worker_->CompleteGroupAsync(call_opts, &call->request, &call->response,
+                                  [call, call_opts](const Status& s) {
+                                    call->ClearCancelCallback();
+                                    delete call_opts;
+                                    call->SendResponse(ToGrpcStatus(s));
+                                  });
+    });
+    ENQUEUE_REQUEST(CompleteGroup, true);
+  }
 
-    void RecvTensorHandlerRaw(
-        WorkerCall<RecvTensorRequest, ::grpc::ByteBuffer>* call) {
-      Schedule([this, call]() {
-        CallOptions* call_opts = new CallOptions;
-        call->SetCancelCallback([call_opts]() { call_opts->StartCancel(); });
-        worker_->GrpcRecvTensorAsync(call_opts, &call->request, &call->response,
+  void CompleteInstanceHandler(
+      WorkerCall<CompleteInstanceRequest, CompleteInstanceResponse>* call) {
+    Schedule([this, call]() {
+      CallOptions* call_opts = new CallOptions;
+      call->SetCancelCallback([call_opts]() { call_opts->StartCancel(); });
+      worker_->CompleteInstanceAsync(call_opts, &call->request, &call->response,
                                      [call, call_opts](const Status& s) {
                                        call->ClearCancelCallback();
                                        delete call_opts;
                                        call->SendResponse(ToGrpcStatus(s));
                                      });
-      });
-      EnqueueRecvTensorRequestRaw();
-    }
-
-    void CleanupGraphHandler(
-        WorkerCall<CleanupGraphRequest, CleanupGraphResponse>* call) {
-      Schedule([this, call]() {
-        Status s = worker_->CleanupGraph(&call->request, &call->response);
-        call->SendResponse(ToGrpcStatus(s));
-      });
-      ENQUEUE_REQUEST(CleanupGraph, false);
-    }
-
-    void LoggingHandler(WorkerCall<LoggingRequest, LoggingResponse>* call) {
-      Schedule([this, call]() {
-        Status s = worker_->Logging(&call->request, &call->response);
-        call->SendResponse(ToGrpcStatus(s));
-      });
-      ENQUEUE_REQUEST(Logging, false);
-    }
+    });
+    ENQUEUE_REQUEST(CompleteInstance, false);
+  }
 
-    void TracingHandler(WorkerCall<TracingRequest, TracingResponse>* call) {
-      Schedule([this, call]() {
-        Status s = worker_->Tracing(&call->request, &call->response);
-        call->SendResponse(ToGrpcStatus(s));
-      });
-      ENQUEUE_REQUEST(Tracing, false);
-    }
+  void GetStepSequenceHandler(
+      WorkerCall<GetStepSequenceRequest, GetStepSequenceResponse>* call) {
+    Schedule([this, call]() {
+      worker_->GetStepSequenceAsync(
+          &call->request, &call->response,
+          [call](const Status& s) { call->SendResponse(ToGrpcStatus(s)); });
+    });
+    ENQUEUE_REQUEST(GetStepSequence, true);
+  }
+#undef ENQUEUE_REQUEST
 
-    void RecvBufHandler(WorkerCall<RecvBufRequest, RecvBufResponse>* call) {
-      Schedule([this, call]() {
-        CallOptions* call_opts = new CallOptions;
-        call->SetCancelCallback([call_opts]() { call_opts->StartCancel(); });
-        worker_->RecvBufAsync(call_opts, &call->request, &call->response,
-                              [call, call_opts](const Status& s) {
-                                call->ClearCancelCallback();
-                                delete call_opts;
-                                call->SendResponse(ToGrpcStatus(s));
-                              });
-      });
-      ENQUEUE_REQUEST(RecvBuf, true);
+  void EnqueueRecvTensorRequestRaw() {
+    mutex_lock l(shutdown_mu_);
+    if (!is_shutdown_) {
+      Call<GrpcWorkerServiceThread, grpc::WorkerService::AsyncService,
+           RecvTensorRequest, ::grpc::ByteBuffer>::
+          EnqueueRequestForMethod(
+              worker_service_, cq_.get(),
+              static_cast<int>(GrpcWorkerMethod::kRecvTensor),
+              &GrpcWorkerServiceThread::RecvTensorHandlerRaw,
+              true /* supports cancel*/);
     }
+  }
 
-    void CompleteGroupHandler(
-        WorkerCall<CompleteGroupRequest, CompleteGroupResponse>* call) {
-      Schedule([this, call]() {
-        CallOptions* call_opts = new CallOptions;
-        call->SetCancelCallback([call_opts]() { call_opts->StartCancel(); });
-        worker_->CompleteGroupAsync(call_opts, &call->request, &call->response,
-                                    [call, call_opts](const Status& s) {
-                                      call->ClearCancelCallback();
-                                      delete call_opts;
-                                      call->SendResponse(ToGrpcStatus(s));
-                                    });
-      });
-      ENQUEUE_REQUEST(CompleteGroup, true);
-    }
+  GrpcWorker* const worker_ = nullptr;  // Not owned.
+  std::unique_ptr<::grpc::ServerCompletionQueue> cq_;
+  std::unique_ptr<Thread> thread_;
+  std::unordered_map<int, int> queue_depth_;
+  grpc::WorkerService::AsyncService* const worker_service_;
 
-    void CompleteInstanceHandler(
-        WorkerCall<CompleteInstanceRequest, CompleteInstanceResponse>* call) {
-      Schedule([this, call]() {
-        CallOptions* call_opts = new CallOptions;
-        call->SetCancelCallback([call_opts]() { call_opts->StartCancel(); });
-        worker_->CompleteInstanceAsync(call_opts, &call->request,
-                                       &call->response,
-                                       [call, call_opts](const Status& s) {
-                                         call->ClearCancelCallback();
-                                         delete call_opts;
-                                         call->SendResponse(ToGrpcStatus(s));
-                                       });
-      });
-      ENQUEUE_REQUEST(CompleteInstance, false);
-    }
+  mutex shutdown_mu_;
+  bool is_shutdown_ GUARDED_BY(shutdown_mu_);
+  TF_DISALLOW_COPY_AND_ASSIGN(GrpcWorkerServiceThread);
+};
 
-    void GetStepSequenceHandler(
-        WorkerCall<GetStepSequenceRequest, GetStepSequenceResponse>* call) {
-      Schedule([this, call]() {
-        worker_->GetStepSequenceAsync(
-            &call->request, &call->response,
-            [call](const Status& s) { call->SendResponse(ToGrpcStatus(s)); });
-      });
-      ENQUEUE_REQUEST(GetStepSequence, true);
+class GrpcWorkerService : public AsyncServiceInterface {
+ public:
+  GrpcWorkerService(GrpcWorker* worker, ::grpc::ServerBuilder* builder,
+                    GrpcWorkerServiceOptions options)
+      : is_shutdown_(false) {
+    builder->RegisterService(&worker_service_);
+    for (int i = 0; i < options.num_worker_threads; i++) {
+      threads_.emplace_back(new GrpcWorkerServiceThread(
+          worker, builder, options.queue_depth, &worker_service_));
     }
-#undef ENQUEUE_REQUEST
+  }
 
-    void EnqueueRecvTensorRequestRaw() {
-      mutex_lock l(shutdown_mu_);
+  void Shutdown() override {
+    bool did_shutdown = false;
+    {
+      mutex_lock l(service_shutdown_mu_);
       if (!is_shutdown_) {
-        Call<GrpcWorkerServiceThread, grpc::WorkerService::AsyncService,
-             RecvTensorRequest, ::grpc::ByteBuffer>::
-            EnqueueRequestForMethod(
-                worker_service_, cq_.get(),
-                static_cast<int>(GrpcWorkerMethod::kRecvTensor),
-                &GrpcWorkerServiceThread::RecvTensorHandlerRaw,
-                true /* supports cancel*/);
+        LOG(INFO) << "Shutting down GrpcWorkerService.";
+        is_shutdown_ = true;
+        did_shutdown = true;
       }
     }
+    if (did_shutdown) {
+      for (auto& worker_thread : threads_) {
+        worker_thread->Shutdown();
+      }
+    }
+  }
 
-    GrpcWorker* const worker_ = nullptr;  // Not owned.
-    std::unique_ptr<::grpc::ServerCompletionQueue> cq_;
-    std::unique_ptr<Thread> thread_;
-    grpc::WorkerService::AsyncService* const worker_service_;
-
-    mutex shutdown_mu_;
-    bool is_shutdown_ GUARDED_BY(shutdown_mu_);
-    TF_DISALLOW_COPY_AND_ASSIGN(GrpcWorkerServiceThread);
-  };  // GrpcWorkerServiceThread
+  // This method blocks forever handling requests from the completion queue.
+  void HandleRPCsLoop() override {
+    for (auto& worker_thread : threads_) {
+      worker_thread->Start();
+    }
+    for (auto& worker_thread : threads_) {
+      worker_thread->Join();
+    }
+  }
 
+ private:
   grpc::WorkerService::AsyncService worker_service_;
   std::vector<std::unique_ptr<GrpcWorkerServiceThread>> threads_;
 
@@ -640,9 +643,10 @@ std::unique_ptr<GrpcWorker> NewGrpcWorker(WorkerEnv* env,
 }
 
 std::unique_ptr<AsyncServiceInterface> NewGrpcWorkerService(
-    GrpcWorker* worker, ::grpc::ServerBuilder* builder) {
+    GrpcWorker* worker, ::grpc::ServerBuilder* builder,
+    GrpcWorkerServiceOptions options) {
   return std::unique_ptr<AsyncServiceInterface>(
-      new GrpcWorkerService(worker, builder));
+      new GrpcWorkerService(worker, builder, options));
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h
index 996617d385..88beb6c216 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h
@@ -16,7 +16,9 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_WORKER_SERVICE_H_
 #define TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_WORKER_SERVICE_H_
 
+#include <unordered_map>
 #include "tensorflow/core/distributed_runtime/recent_request_ids.h"
+#include "tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h"
 #include "tensorflow/core/distributed_runtime/worker.h"
 
 namespace grpc {
@@ -57,9 +59,17 @@ class GrpcWorker : public Worker {
 std::unique_ptr<GrpcWorker> NewGrpcWorker(WorkerEnv* worker_env,
                                           const ConfigProto& config);
 
+struct GrpcWorkerServiceOptions {
+  // Map from GrpcWorkerMethod id to queue depth.  If set this overrides the
+  // default queue depth for a method.
+  std::unordered_map<int, int> queue_depth;
+  int num_worker_threads = 8;
+};
+
 // Returns an implementation of WorkerService rpc service.
 std::unique_ptr<AsyncServiceInterface> NewGrpcWorkerService(
-    GrpcWorker* worker, ::grpc::ServerBuilder* builder);
+    GrpcWorker* worker, ::grpc::ServerBuilder* builder,
+    GrpcWorkerServiceOptions opts = GrpcWorkerServiceOptions());
 
 }  // namespace tensorflow
 
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h
index 7915c3aafd..d2ae4eeaee 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h
@@ -88,6 +88,7 @@ enum class GrpcWorkerMethod {
   kCompleteInstance,
   kGetStepSequence,
 };
+
 static const int kGrpcNumWorkerMethods =
     static_cast<int>(GrpcWorkerMethod::kGetStepSequence) + 1;
 
-- 
GitLab


From bad11dcc05a0a756252ccb10cc9fd1171dc4fc64 Mon Sep 17 00:00:00 2001
From: Albin Joy <albin.joy@huawei.com>
Date: Fri, 25 Jan 2019 10:15:21 +0530
Subject: [PATCH 1238/2345] Fix README.md typo error

---
 tensorflow/lite/tools/benchmark/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/tools/benchmark/README.md b/tensorflow/lite/tools/benchmark/README.md
index a4d9c879eb..e6ba818c71 100644
--- a/tensorflow/lite/tools/benchmark/README.md
+++ b/tensorflow/lite/tools/benchmark/README.md
@@ -5,7 +5,7 @@
 A simple C++ binary to benchmark a TFLite model and its individual operators,
 both on desktop machines and on Android. The binary takes a TFLite model,
 generates random inputs and then repeatedly runs the model for specified number
-of runs. Aggregrate latency statistics are reported after running the benchmark.
+of runs. Aggregate latency statistics are reported after running the benchmark.
 
 The instructions below are for running the binary on Desktop and Android,
 for iOS please use the
-- 
GitLab


From d4b0f635bc1d72271ae7fafff6797b9381823d79 Mon Sep 17 00:00:00 2001
From: Revan Sopher <rsopher@google.com>
Date: Thu, 24 Jan 2019 22:11:49 -0800
Subject: [PATCH 1239/2345] Stop grpc server on test teardown.

PiperOrigin-RevId: 230848262
---
 tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test.py b/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test.py
index 3fc6bfbb4d..d8ab9eba70 100644
--- a/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test.py
+++ b/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test.py
@@ -61,10 +61,7 @@ class RpcOpTest(test.TestCase, rpc_op_test_base.RpcOpTestBase):
     self._server = server
 
   def tearDown(self):
-    # TODO(ebrevdo): Figure out why this sometimes times out.
-    #    self._service.ExitLoop()
-    #    self._service_thread.join()
-    # self._server.stop()
+    self._server.stop(grace=None)
     super(RpcOpTest, self).tearDown()
 
 
-- 
GitLab


From d1afaa4c1472d38a7230d68e312c9ea37f834cc5 Mon Sep 17 00:00:00 2001
From: dmitrievanthony <dmitrievanthony@gmail.com>
Date: Fri, 25 Jan 2019 10:18:26 +0300
Subject: [PATCH 1240/2345] Remove import of tensorflow.

---
 tensorflow/contrib/ignite/python/tests/ignite_dataset_test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/ignite/python/tests/ignite_dataset_test.py b/tensorflow/contrib/ignite/python/tests/ignite_dataset_test.py
index b44b78f075..89b74fbfdc 100644
--- a/tensorflow/contrib/ignite/python/tests/ignite_dataset_test.py
+++ b/tensorflow/contrib/ignite/python/tests/ignite_dataset_test.py
@@ -19,7 +19,7 @@ from __future__ import print_function
 
 import os
 
-import tensorflow as tf
+from tensorflow import compat
 from tensorflow.contrib.ignite import IgniteDataset
 from tensorflow.python.client import session
 from tensorflow.python.framework import dtypes
@@ -66,7 +66,7 @@ class IgniteDatasetTest(test.TestCase):
     self.assertEqual(dtypes.string, dataset.output_types["val"]["NAME"])
     self.assertEqual(dtypes.int64, dataset.output_types["val"]["VAL"])
 
-    it = tf.compat.v1.data.make_one_shot_iterator(dataset)
+    it = compat.v1.data.make_one_shot_iterator(dataset)
     ne = it.get_next()
 
     with session.Session() as sess:
-- 
GitLab


From da2c471248c92526e429f031c45ee7f8824139a7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 25 Jan 2019 01:02:49 -0800
Subject: [PATCH 1241/2345] compat: Update forward compatibility horizon to
 2019-01-25

PiperOrigin-RevId: 230863315
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 919b47129f..49e12345db 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 24)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 25)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 004f2d6435c125291bd0455d86607ba68789ac49 Mon Sep 17 00:00:00 2001
From: Dayananda-V <dayanandasiet@gmail.com>
Date: Fri, 25 Jan 2019 15:04:30 +0530
Subject: [PATCH 1242/2345] Fix incorrect link in the docs

This fix fixes incorrect link in the doc
---
 tensorflow/tools/ci_build/Dockerfile.debian.jessie.cpu | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/Dockerfile.debian.jessie.cpu b/tensorflow/tools/ci_build/Dockerfile.debian.jessie.cpu
index eb9d0d4dd0..ad82c88b4a 100644
--- a/tensorflow/tools/ci_build/Dockerfile.debian.jessie.cpu
+++ b/tensorflow/tools/ci_build/Dockerfile.debian.jessie.cpu
@@ -5,7 +5,7 @@ LABEL maintainer="Jan Prach <jendap@google.com>"
 # Copy and run the install scripts.
 COPY install/*.sh /install/
 RUN /install/install_bootstrap_deb_packages.sh
-RUN echo "deb http://http.debian.net/debian jessie-backports main" | \
+RUN echo "deb http://www.debian.net/debian jessie-backports main" | \
     tee -a /etc/apt/sources.list
 # Workaround bug in Jessie backport repository deb packages
 # http://serverfault.com/questions/830636/cannot-install-openjdk-8-jre-headless-on-debian-jessie
-- 
GitLab


From bdca83b71c0f47628321bafc29056fb2467144db Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Fri, 25 Jan 2019 02:52:49 -0800
Subject: [PATCH 1243/2345] Automated rollback of commit
 74a6cca5d867d37e79ec9d780f2c57b926f07a80

PiperOrigin-RevId: 230873931
---
 tensorflow/python/framework/dtypes.py | 115 ++++++++++++++------------
 1 file changed, 63 insertions(+), 52 deletions(-)

diff --git a/tensorflow/python/framework/dtypes.py b/tensorflow/python/framework/dtypes.py
index 574d82ea93..9d643e041c 100644
--- a/tensorflow/python/framework/dtypes.py
+++ b/tensorflow/python/framework/dtypes.py
@@ -535,29 +535,47 @@ _np_qint32 = np.dtype([("qint32", np.int32, 1)])
 np_resource = np.dtype([("resource", np.ubyte, 1)])
 
 # Standard mappings between types_pb2.DataType values and numpy.dtypes.
-_NP_TO_TF = frozenset([
-    (np.float16, float16),
-    (np.float32, float32),
-    (np.float64, float64),
-    (np.int32, int32),
-    (np.int64, int64),
-    (np.uint8, uint8),
-    (np.uint16, uint16),
-    (np.uint32, uint32),
-    (np.uint64, uint64),
-    (np.int16, int16),
-    (np.int8, int8),
-    (np.complex64, complex64),
-    (np.complex128, complex128),
-    (np.object_, string),
-    (np.bool_, bool),
-    (_np_qint8, qint8),
-    (_np_quint8, quint8),
-    (_np_qint16, qint16),
-    (_np_quint16, quint16),
-    (_np_qint32, qint32),
-    (_np_bfloat16, bfloat16),
-])
+_NP_TO_TF = {
+    np.float16: float16,
+    np.float32: float32,
+    np.float64: float64,
+    np.int32: int32,
+    np.int64: int64,
+    np.uint8: uint8,
+    np.uint16: uint16,
+    np.uint32: uint32,
+    np.uint64: uint64,
+    np.int16: int16,
+    np.int8: int8,
+    np.complex64: complex64,
+    np.complex128: complex128,
+    np.object_: string,
+    np.string_: string,
+    np.unicode_: string,
+    np.bool_: bool,
+    _np_qint8: qint8,
+    _np_quint8: quint8,
+    _np_qint16: qint16,
+    _np_quint16: quint16,
+    _np_qint32: qint32,
+    _np_bfloat16: bfloat16,
+}
+
+# Map (some) NumPy platform dtypes to TF ones using their fixed-width
+# synonyms. Note that platform dtypes are not always simples aliases,
+# i.e. reference equality is not guaranteed. See e.g. numpy/numpy#9799.
+for pdt in [
+    np.intc,
+    np.uintc,
+    np.int_,
+    np.uint,
+    np.longlong,
+    np.ulonglong,
+]:
+  if pdt not in _NP_TO_TF:
+    _NP_TO_TF[pdt] = next(
+        _NP_TO_TF[dt] for dt in _NP_TO_TF if dt == pdt().dtype)
+
 _TF_TO_NP = {
     types_pb2.DT_HALF:
         np.float16,
@@ -664,6 +682,20 @@ _PYTHON_TO_TF = {
     builtins.object: string
 }
 
+_ANY_TO_TF = {}
+_ANY_TO_TF.update(_INTERN_TABLE)
+_ANY_TO_TF.update(_STRING_TO_TF)
+_ANY_TO_TF.update(_PYTHON_TO_TF)
+_ANY_TO_TF.update(_NP_TO_TF)
+
+# Ensure no collisions.
+assert len(_ANY_TO_TF) == sum(len(d) for d in [
+    _INTERN_TABLE,
+    _STRING_TO_TF,
+    _PYTHON_TO_TF,
+    _NP_TO_TF
+])
+
 
 @tf_export("dtypes.as_dtype", "as_dtype")
 def as_dtype(type_value):
@@ -684,37 +716,16 @@ def as_dtype(type_value):
   if isinstance(type_value, DType):
     return type_value
 
-  try:
-    return _INTERN_TABLE[type_value]
-  except KeyError:
-    pass
-
-  try:
-    return _STRING_TO_TF[type_value]
-  except KeyError:
-    pass
+  if isinstance(type_value, np.dtype):
+    try:
+      return _NP_TO_TF[type_value.type]
+    except KeyError:
+      pass
 
   try:
-    return _PYTHON_TO_TF[type_value]
+    return _ANY_TO_TF[type_value]
   except KeyError:
     pass
 
-  if isinstance(type_value, np.dtype):
-    # The numpy dtype for strings is variable length. We can not compare
-    # dtype with a single constant (np.string does not exist) to decide
-    # dtype is a "string" type. We need to compare the dtype.type to be
-    # sure it's a string type.
-    if type_value.type == np.string_ or type_value.type == np.unicode_:
-      return string
-
-  if isinstance(type_value, (type, np.dtype)):
-    for key, val in _NP_TO_TF:
-      try:
-        if key == type_value:
-          return val
-      except TypeError as e:
-        raise TypeError("Cannot convert {} to a dtype. {}".format(
-            type_value, e))
-
-  raise TypeError("Cannot convert value %r to a TensorFlow DType." % type_value)
-
+  raise TypeError(
+      "Cannot convert value %r to a TensorFlow DType." % type_value)
-- 
GitLab


From 53d3a75b7b7237b0582220913b17b811741f9b70 Mon Sep 17 00:00:00 2001
From: Chris Jones <cjfj@google.com>
Date: Fri, 25 Jan 2019 03:26:19 -0800
Subject: [PATCH 1244/2345] Remove infeed from TPU Strategy when using host
 training loop.

PiperOrigin-RevId: 230877075
---
 .../contrib/distribute/python/tpu_strategy.py | 298 +++++++++++-------
 1 file changed, 188 insertions(+), 110 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 5a494a6ae7..4863bb29da 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -21,6 +21,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import collections
 import copy
 
 from tensorflow.contrib.tpu.python.ops import tpu_ops
@@ -42,6 +43,7 @@ from tensorflow.python.distribute.cluster_resolver import tpu_cluster_resolver a
 from tensorflow.python.eager import context
 from tensorflow.python.eager import tape
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import device as tf_device
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
@@ -98,9 +100,9 @@ def _create_tpu_mirrored_variable(  # pylint: disable=missing-docstring
     *args, **kwargs):
   # Figure out what collections this variable should be added to.
   # We'll add the TPUMirroredVariable to those collections instead.
-  collections = kwargs.pop("collections", None)
-  if collections is None:
-    collections = [ops.GraphKeys.GLOBAL_VARIABLES]
+  var_collections = kwargs.pop("collections", None)
+  if var_collections is None:
+    var_collections = [ops.GraphKeys.GLOBAL_VARIABLES]
   kwargs["collections"] = []
 
   # TODO(jhseu): Should we have different behavior for different
@@ -139,11 +141,11 @@ def _create_tpu_mirrored_variable(  # pylint: disable=missing-docstring
     # "trainable" to False for next_creator() since that causes functions
     # like implicit_gradients to skip those variables.
     if kwargs.get("trainable", True):
-      collections.append(ops.GraphKeys.TRAINABLE_VARIABLES)
+      var_collections.append(ops.GraphKeys.TRAINABLE_VARIABLES)
       l = g.get_collection_ref(ops.GraphKeys.TRAINABLE_VARIABLES)
       for v in value_list:
         l.remove(v)
-    g.add_to_collections(collections, result)
+    g.add_to_collections(var_collections, result)
   return result
 
 
@@ -170,19 +172,16 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
           the usecase of using a single core within a TPU cluster.
       **kwargs: Additional experimental flags. Will be removed in future.
     """
-    super(TPUStrategy, self).__init__(TPUExtended(
-        self, tpu_cluster_resolver, steps_per_run, device_assignment))
-
-    self._disable_training_loop_on_host = False
     if len(kwargs) > 1:
       raise ValueError("TPUStrategy constructor only takes one experimental "
                        "flag now")
-    if len(kwargs) == 1:
-      if "_disable_training_loop_on_host" not in kwargs:
-        raise ValueError("TPUStrategy constructor does not support arguments: "
-                         "{}".format(kwargs))
-      self._disable_training_loop_on_host = (
-          kwargs["_disable_training_loop_on_host"])
+    elif len(kwargs) == 1 and "_disable_training_loop_on_host" not in kwargs:
+      raise ValueError("TPUStrategy constructor does not support arguments: "
+                       "{}".format(kwargs))
+
+    super(TPUStrategy, self).__init__(TPUExtended(
+        self, tpu_cluster_resolver, steps_per_run, device_assignment,
+        kwargs.get("_disable_training_loop_on_host", False)))
 
   @property
   def steps_per_run(self):
@@ -197,7 +196,8 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
                container_strategy,
                tpu_cluster_resolver=None,
                steps_per_run=None,
-               device_assignment=None):
+               device_assignment=None,
+               disable_training_loop_on_host=False):
     super(TPUExtended, self).__init__(container_strategy)
 
     if tpu_cluster_resolver is None:
@@ -211,6 +211,7 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
     self._tpu_cluster_resolver = tpu_cluster_resolver
     self._tpu_metadata = get_tpu_system_metadata(self._tpu_cluster_resolver)
     self._device_assignment = device_assignment
+    self._disable_training_loop_on_host = disable_training_loop_on_host
 
     # Device assignment is currently only supported for 1 core case.
     if self._device_assignment:
@@ -238,15 +239,25 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
     self._tpu_devices = self._tpu_devices[:self._num_replicas_in_sync]
     self._device_map = values.ReplicaDeviceMap(self._tpu_devices)
 
-    # For input:
-    input_device_map = values.ReplicaDeviceMap(tuple(
-        self.get_host_cpu_device(hid) for hid in range(self.num_hosts)))
-    worker_devices = [
-        (self.get_host(hid), [self.get_host_cpu_device(hid)])
-        for hid in range(self.num_hosts)
-    ]
-    self._input_workers = input_lib.InputWorkers(
-        input_device_map, worker_devices)
+    # If the training loop is on the device, we must use the infeed, with input
+    # on the host. Otherwise, we preload the data onto the TPUs.
+    if disable_training_loop_on_host:
+      input_device_map = values.ReplicaDeviceMap(tuple(
+          self.get_host_cpu_device(hid) for hid in range(self.num_hosts)))
+      worker_devices = [
+          (self.get_host(hid), [self.get_host_cpu_device(hid)])
+          for hid in range(self.num_hosts)
+      ]
+      self._input_workers = input_lib.InputWorkers(
+          input_device_map, worker_devices)
+    else:
+      input_worker_devices = collections.OrderedDict()
+      for tpu_device in self._tpu_devices:
+        host_device = _get_host_for_device(tpu_device)
+        input_worker_devices.setdefault(host_device, [])
+        input_worker_devices[host_device].append(tpu_device)
+      self._input_workers = input_lib.InputWorkers(
+          self._device_map, tuple(input_worker_devices.items()))
 
     # TODO(sourabhbajaj): Remove this once performance of running one step
     # at a time is comparable to multiple steps.
@@ -346,6 +357,113 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
   # a mechanism to infer the outputs of `fn`. Pending b/110550782.
   def _experimental_run_steps_on_iterator(
       self, fn, multi_worker_iterator, iterations, initial_loop_values=None):
+    if self._disable_training_loop_on_host:
+      impl = self._run_steps_on_iterator_with_device_loop
+    else:
+      impl = self._run_steps_on_iterator_with_host_loop
+
+    return impl(
+        fn=fn, multi_worker_iterator=multi_worker_iterator,
+        iterations=iterations, initial_loop_values=initial_loop_values)
+
+  def _run_steps_on_iterator_with_host_loop(
+      self, fn, multi_worker_iterator, iterations, initial_loop_values=None):
+    output_shapes = multi_worker_iterator.output_shapes
+    shapes = nest.flatten(output_shapes)
+    if any(not s.is_fully_defined() for s in shapes):
+      raise ValueError(
+          "TPU currently requires fully defined shapes. Either use "
+          "set_shape() on the input tensors or use "
+          "dataset.batch(..., drop_remainder=True).")
+
+    # Wrap `fn` for repeat.
+    if initial_loop_values is None:
+      initial_loop_values = {}
+    initial_loop_values = nest.flatten(initial_loop_values)
+    ctx = input_lib.MultiStepContext()
+
+    def run_fn(inputs):
+      """Single step on the TPU device."""
+      fn_result = fn(ctx, inputs)
+      flat_last_step_outputs = nest.flatten(ctx.last_step_outputs)
+      if flat_last_step_outputs:
+        with ops.control_dependencies([fn_result]):
+          return [array_ops.identity(f) for f in flat_last_step_outputs]
+      else:
+        return fn_result
+
+    # We capture the control_flow_context at this point, before we run `fn`
+    # inside a while_loop and TPU replicate context. This is useful in cases
+    # where we might need to exit these contexts and get back to the outer
+    # context to do some things, for e.g. create an op which should be
+    # evaluated only once at the end of the loop on the host. One such usage
+    # is in creating metrics' value op.
+    self._outer_control_flow_context = (
+        ops.get_default_graph()._get_control_flow_context())  # pylint: disable=protected-access
+
+    def rewrite_fn(*args):
+      """The rewritten step fn running on TPU."""
+      del args
+
+      per_replica_inputs = multi_worker_iterator.get_next()
+      replicate_inputs = []
+      for replica_id in range(self._num_replicas_in_sync):
+        select_replica = lambda x: values.select_replica(replica_id, x)  # pylint: disable=cell-var-from-loop
+        replicate_inputs.append((nest.map_structure(
+            select_replica, per_replica_inputs),))
+
+      replicate_outputs = tpu.replicate(run_fn, replicate_inputs)
+
+      # If run_fn has tensor outputs, tpu.replicate returns a list of list. We
+      # will flatten it in this case. If run_fn has no tensor outputs,
+      # tpu.replicate returns a list of no_ops, we will keep the output as it
+      # is.
+      if isinstance(replicate_outputs[0], list):
+        replicate_outputs = nest.flatten(replicate_outputs)
+
+      return replicate_outputs
+
+    # TODO(sourabhbajaj): The input to while loop should be based on the
+    # output type of the step_fn
+    assert isinstance(initial_loop_values, list)
+    initial_loop_values = initial_loop_values * self._num_replicas_in_sync
+
+    # Put the while loop op on host 0.
+    with ops.device(self.get_host_cpu_device(0)):
+      replicate_outputs = training_loop.repeat(iterations, rewrite_fn,
+                                               initial_loop_values)
+
+    del self._outer_control_flow_context
+    ctx.run_op = control_flow_ops.group(replicate_outputs)
+
+    if isinstance(replicate_outputs, list):
+      # Filter out any ops from the outputs, typically this would be the case
+      # when there were no tensor outputs.
+      last_step_tensor_outputs = [
+          x for x in replicate_outputs if not isinstance(x, ops.Operation)
+      ]
+
+      # Outputs are currently of the structure (flattened)
+      # [output0_device0, output1_device0, output2_device0,
+      #  output0_device1, output1_device1, output2_device1,
+      #  ...]
+      # Convert this to the following structure instead: (grouped by output)
+      # [[output0_device0, output0_device1],
+      #  [output1_device0, output1_device1],
+      #  [output2_device0, output2_device1]]
+      output_num = len(last_step_tensor_outputs) // self._num_replicas_in_sync
+      last_step_tensor_outputs = [
+          last_step_tensor_outputs[i::output_num] for i in range(output_num)
+      ]
+    else:
+      # no tensors returned.
+      last_step_tensor_outputs = []
+
+    _set_last_step_outputs(ctx, last_step_tensor_outputs)
+    return ctx
+
+  def _run_steps_on_iterator_with_device_loop(
+      self, fn, multi_worker_iterator, iterations, initial_loop_values=None):
     output_shapes = multi_worker_iterator.output_shapes
     shapes = nest.flatten(output_shapes)
     if any(not s.is_fully_defined() for s in shapes):
@@ -393,95 +511,28 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
     self._outer_control_flow_context = (
         ops.get_default_graph()._get_control_flow_context())  # pylint: disable=protected-access
 
-    # pylint: disable=protected-access
-    if self._container_strategy()._disable_training_loop_on_host:
-      replicate_inputs = [[]] * self._num_replicas_in_sync
-      replicate_outputs = tpu.replicate(iterate_on_tpu, replicate_inputs)
-    else:
-      def rewrite_fn(*args):
-        """The rewritten step fn running on TPU."""
-        del args
-        replicate_inputs = [[]] * self._num_replicas_in_sync
-        replicate_outputs = tpu.replicate(run_fn, replicate_inputs)
-
-        # If run_fn has tensor outputs, tpu.replicate returns a list of list. We
-        # will flatten it in this case. If run_fn has no tensor outputs,
-        # tpu.replicate returns a list of no_ops, we will keep the output as it
-        # is.
-        if isinstance(replicate_outputs[0], list):
-          replicate_outputs = nest.flatten(replicate_outputs)
-
-        return replicate_outputs
-
-      # TODO(sourabhbajaj): The input to while loop should be based on the
-      # output type of the step_fn
-      assert isinstance(initial_loop_values, list)
-      initial_loop_values = initial_loop_values * self._num_replicas_in_sync
-
-      # Put the while loop op on host 0.
-      with ops.device(self.get_host_cpu_device(0)):
-        replicate_outputs = training_loop.repeat(iterations, rewrite_fn,
-                                                 initial_loop_values)
+    replicate_inputs = [[]] * self._num_replicas_in_sync
+    replicate_outputs = tpu.replicate(iterate_on_tpu, replicate_inputs)
 
     del self._outer_control_flow_context
     ctx.run_op = control_flow_ops.group(replicate_outputs, enqueue_ops)
 
-    if self._container_strategy()._disable_training_loop_on_host:
-      # Filter out any ops from the outputs, typically this would be the case
-      # when there were no tensor outputs.
-      last_step_tensor_outputs = [x for x in replicate_outputs
-                                  if not isinstance(x, ops.Operation)]
-
-      # Outputs are currently of the structure (grouped by device)
-      # [[output0_device0, output1_device0, output2_device0],
-      #  [output0_device1, output1_device1, output2_device1]]
-      # Convert this to the following structure instead: (grouped by output)
-      # [[output0_device0, output0_device1],
-      #  [output1_device0, output1_device1],
-      #  [output2_device0, output2_device1]]
-      last_step_tensor_outputs = [list(x) for x in
-                                  zip(*last_step_tensor_outputs)]
-    else:
-      if isinstance(replicate_outputs, list):
-        # Filter out any ops from the outputs, typically this would be the case
-        # when there were no tensor outputs.
-        last_step_tensor_outputs = [
-            x for x in replicate_outputs if not isinstance(x, ops.Operation)
-        ]
-
-        # Outputs are currently of the structure (flattened)
-        # [output0_device0, output1_device0, output2_device0,
-        #  output0_device1, output1_device1, output2_device1,
-        #  ...]
-        # Convert this to the following structure instead: (grouped by output)
-        # [[output0_device0, output0_device1],
-        #  [output1_device0, output1_device1],
-        #  [output2_device0, output2_device1]]
-        output_num = len(last_step_tensor_outputs) // self._num_replicas_in_sync
-        last_step_tensor_outputs = [
-            last_step_tensor_outputs[i::output_num] for i in range(output_num)
-        ]
-      else:
-        # no tensors returned.
-        last_step_tensor_outputs = []
-
-    # Convert replicate_outputs to the original dict structure of
-    # last_step_outputs.
-    last_step_tensor_outputs_dict = nest.pack_sequence_as(
-        ctx.last_step_outputs, last_step_tensor_outputs)
-
-    for name, reduce_op in ctx._last_step_outputs_reduce_ops.items():  # pylint: disable=protected-access
-      output = last_step_tensor_outputs_dict[name]
-      # For outputs that have already been reduced, take the first value
-      # from the list as each value should be the same. Else return the full
-      # list of values.
-      # TODO(josh11b): If reduce_op is NONE, we should return a PerReplica
-      # value.
-      if reduce_op is not None:
-        # TODO(priyag): Should this return the element or a list with 1 element
-        last_step_tensor_outputs_dict[name] = output[0]
-    ctx._set_last_step_outputs(last_step_tensor_outputs_dict)  # pylint: disable=protected-access
-
+    # Filter out any ops from the outputs, typically this would be the case
+    # when there were no tensor outputs.
+    last_step_tensor_outputs = [x for x in replicate_outputs
+                                if not isinstance(x, ops.Operation)]
+
+    # Outputs are currently of the structure (grouped by device)
+    # [[output0_device0, output1_device0, output2_device0],
+    #  [output0_device1, output1_device1, output2_device1]]
+    # Convert this to the following structure instead: (grouped by output)
+    # [[output0_device0, output0_device1],
+    #  [output1_device0, output1_device1],
+    #  [output2_device0, output2_device1]]
+    last_step_tensor_outputs = [list(x) for x in
+                                zip(*last_step_tensor_outputs)]
+
+    _set_last_step_outputs(ctx, last_step_tensor_outputs)
     return ctx
 
   def _call_for_each_replica(self, fn, args, kwargs):
@@ -744,3 +795,30 @@ class _TPUReplicaContext(distribute_lib.ReplicaContext):
     ds = self._strategy
     replica_id = tensor_util.constant_value(self._replica_id_in_sync_group)
     return (ds.extended.worker_devices[replica_id],)
+
+
+def _get_host_for_device(device):
+  spec = tf_device.DeviceSpec.from_string(device)
+  return tf_device.DeviceSpec(
+      job=spec.job, replica=spec.replica, task=spec.task,
+      device_type="CPU", device_index=0).to_string()
+
+
+def _set_last_step_outputs(ctx, last_step_tensor_outputs):
+  """Sets the last step outputs on the given context."""
+  # Convert replicate_outputs to the original dict structure of
+  # last_step_outputs.
+  last_step_tensor_outputs_dict = nest.pack_sequence_as(
+      ctx.last_step_outputs, last_step_tensor_outputs)
+
+  for name, reduce_op in ctx._last_step_outputs_reduce_ops.items():  # pylint: disable=protected-access
+    output = last_step_tensor_outputs_dict[name]
+    # For outputs that have already been reduced, take the first value
+    # from the list as each value should be the same. Else return the full
+    # list of values.
+    # TODO(josh11b): If reduce_op is NONE, we should return a PerReplica
+    # value.
+    if reduce_op is not None:
+      # TODO(priyag): Should this return the element or a list with 1 element
+      last_step_tensor_outputs_dict[name] = output[0]
+  ctx._set_last_step_outputs(last_step_tensor_outputs_dict)  # pylint: disable=protected-access
-- 
GitLab


From eda13c84160a3280316c80ca6455a6edb45e3129 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 25 Jan 2019 04:43:48 -0800
Subject: [PATCH 1245/2345] Added map_structure_with_tuple_paths_up_to(),
 rewrote existing functions in tree.py in terms of it, as they are subsets of
 its functionality.

This cleans the behavior of traversing with shallow_trees, in that it now allows for any shallow_tree that's an upper subset of the input_tree. For example, before this change, the following was not allowed, on the basis that len(shallow_tree) < len(input_tree):

shallow_tree = {'a': 'A','c': 'C'}
input_tree = {'a': 'A', 'b': 'B', 'c': 'C'}
map_structure_up_to(shallow_tree, input_tree, func)
PiperOrigin-RevId: 230883405
---
 tensorflow/contrib/framework/__init__.py |   3 +
 tensorflow/python/util/nest.py           | 327 +++++++++++++++--------
 tensorflow/python/util/nest_test.py      |  49 ++--
 3 files changed, 246 insertions(+), 133 deletions(-)

diff --git a/tensorflow/contrib/framework/__init__.py b/tensorflow/contrib/framework/__init__.py
index ff65c1586c..3784631dcb 100644
--- a/tensorflow/contrib/framework/__init__.py
+++ b/tensorflow/contrib/framework/__init__.py
@@ -136,12 +136,15 @@ _nest_allowed_symbols = [
     'pack_sequence_as',
     'map_structure',
     'map_structure_with_paths',
+    'map_structure_with_tuple_paths',
     'assert_shallow_structure',
     'flatten_up_to',
     'map_structure_up_to',
+    'map_structure_with_tuple_paths_up_to',
     'get_traverse_shallow_structure',
     'yield_flat_paths',
     'flatten_with_joined_string_paths',
+    'flatten_with_tuple_paths',
 ]
 
 remove_undocumented(nest.__name__, allowed_exception_list=_nest_allowed_symbols)
diff --git a/tensorflow/python/util/nest.py b/tensorflow/python/util/nest.py
index 83fd8bdf60..4de358dec6 100644
--- a/tensorflow/python/util/nest.py
+++ b/tensorflow/python/util/nest.py
@@ -41,10 +41,38 @@ import six as _six
 from tensorflow.python import pywrap_tensorflow as _pywrap_tensorflow
 
 
-def _get_attrs_values(obj):
-  """Returns the list of values from an attrs instance."""
+_SHALLOW_TREE_HAS_INVALID_KEYS = (
+    "The shallow_tree's keys are not a subset of the input_tree's keys. The "
+    "shallow_tree has the following keys that are not in the input_tree: {}.")
+
+_STRUCTURES_HAVE_MISMATCHING_TYPES = (
+    "The two structures don't have the same sequence type. Input structure has "
+    "type {shallow_type}, while shallow structure has type {input_type}.")
+
+_INPUT_TREE_SMALLER_THAN_SHALLOW_TREE = (
+    "The input_tree has fewer elements than the input_tree. Input structure "
+    "has length {input_size}, while shallow structure has length "
+    "{shallow_size}.")
+
+_IF_SHALLOW_IS_SEQ_INPUT_MUST_BE_SEQ = (
+    "If shallow structure is a sequence, input must also be a sequence. "
+    "Input has type: {}.")
+
+
+def _get_attrs_items(obj):
+  """Returns a list of (name, value) pairs from an attrs instance.
+
+  The list will be sorted by name.
+
+  Args:
+    obj: an object.
+
+  Returns:
+    A list of (attr_name, attr_value) pairs, sorted by attr_name.
+  """
   attrs = getattr(obj.__class__, "__attrs_attrs__")
-  return [getattr(obj, a.name) for a in attrs]
+  attr_names = sorted([a.name for a in attrs])
+  return [(attr_name, getattr(obj, attr_name)) for attr_name in attr_names]
 
 
 def _sorted(dict_):
@@ -106,24 +134,45 @@ def _sequence_like(instance, args):
 
 
 def _yield_value(iterable):
-  """Yields the next value from the given iterable."""
-  if _is_mapping(iterable):
+  for _, v in _yield_sorted_items(iterable):
+    yield v
+
+
+def _yield_sorted_items(iterable):
+  """Yield (key, value) pairs for `iterable` in a deterministic order.
+
+  For Sequences, the key will be an int, the array index of a value.
+  For Mappings, the key will be the dictionary key.
+  For objects (e.g. namedtuples), the key will be the attribute name.
+
+  In all cases, the keys will be iterated in sorted order.
+
+  Args:
+    iterable: an iterable.
+
+  Yields:
+    The iterable's (key, value) pairs, in order of sorted keys.
+  """
+  if isinstance(iterable, _collections.Mapping):
     # Iterate through dictionaries in a deterministic order by sorting the
     # keys. Notice this means that we ignore the original order of `OrderedDict`
     # instances. This is intentional, to avoid potential bugs caused by mixing
     # ordered and plain dicts (e.g., flattening a dict but using a
     # corresponding `OrderedDict` to pack it back).
     for key in _sorted(iterable):
-      yield iterable[key]
+      yield key, iterable[key]
   elif _is_attrs(iterable):
-    for value in _get_attrs_values(iterable):
-      yield value
+    for item in _get_attrs_items(iterable):
+      yield item
+  elif _is_namedtuple(iterable):
+    for field in iterable._fields:
+      yield field, getattr(iterable, field)
   elif _is_composite_tensor(iterable):
-    for value in _yield_value(iterable._to_components()):  # pylint: disable=protected-access
-      yield value
+    for item in enumerate(iterable._to_components()):  # pylint: disable=protected-access
+      yield item
   else:
-    for value in iterable:
-      yield value
+    for item in enumerate(iterable):
+      yield item
 
 
 # See the swig file (util.i) for documentation.
@@ -442,8 +491,15 @@ def map_structure_with_paths(func, *structure, **kwargs):
       the type of sequence in any of their substructures.
     ValueError: If no structures are provided.
   """
-  return _map_structure_with_tuple_or_string_paths(
-      use_string_paths=True, func=func, structure=structure, kwargs=kwargs)
+  print("wheee I'm updated")
+  def wrapper_func(tuple_path, *inputs, **kwargs):
+    string_path = "/".join(str(s) for s in tuple_path)
+    return func(string_path, *inputs, **kwargs)
+
+  return map_structure_with_tuple_paths_up_to(structure[0],
+                                              wrapper_func,
+                                              *structure,
+                                              **kwargs)
 
 
 def map_structure_with_tuple_paths(func, *structure, **kwargs):
@@ -479,52 +535,43 @@ def map_structure_with_tuple_paths(func, *structure, **kwargs):
       the type of sequence in any of their substructures.
     ValueError: If no structures are provided.
   """
-  return _map_structure_with_tuple_or_string_paths(
-      use_string_paths=False, func=func, structure=structure, kwargs=kwargs)
-
-
-def _map_structure_with_tuple_or_string_paths(
-    use_string_paths, func, structure, kwargs):
-  """Implements `map_structure` with either tuple or string paths."""
+  return map_structure_with_tuple_paths_up_to(structure[0],
+                                              func,
+                                              *structure,
+                                              **kwargs)
 
-  if not callable(func):
-    raise TypeError("func must be callable, got: %s" % func)
-  if not structure:
-    raise ValueError("Must provide at least one structure")
 
-  check_types = kwargs.pop("check_types", True)
-  for other in structure[1:]:
-    assert_same_structure(structure[0], other, check_types=check_types)
-
-  if use_string_paths:
-    flatten_func = flatten_with_joined_string_paths
-  else:
-    flatten_func = flatten_with_tuple_paths
-
-  # First set paths_and_values to:
-  # [[(p11, v11), ... (p1n, v1n)], ... [(pm1, vm1), ... (pmn, vmn)]]
-  paths_and_values = [flatten_func(s) for s in structure]
-
-  # Now zip(*paths_and_values) would be:
-  # [((p11, v11), ... (pm1, vm1)), ... ((p1n, v1n), ... (pmn, vmn))]
-  # so grouped_by_path is set to:
-  # [[(p11, ... pm1), (v11, ... vm1)], ... [(p1n, ... pmn), (v1n, ... vmn)]]
-  # Note that p1i, ... pmi must all be equal since the structures are the same.
-  grouped_by_path = [zip(*p_v) for p_v in zip(*paths_and_values)]
-
-  return pack_sequence_as(structure[0], [
-      func(paths[0], *values, **kwargs) for paths, values in grouped_by_path])
+def _yield_flat_up_to(shallow_tree, input_tree, path=()):
+  """Yields (path, value) pairs of input_tree flattened up to shallow_tree.
 
+  Args:
+    shallow_tree: Nested structure. Traverse no further than its leaf nodes.
+    input_tree: Nested structure. Return the paths and values from this tree.
+      Must have the same upper structure as shallow_tree.
+    path: Tuple. Optional argument, only used when recursing. The path from the
+      root of the original shallow_tree, down to the root of the shallow_tree
+      arg of this recursive call.
 
-def _yield_flat_up_to(shallow_tree, input_tree):
-  """Yields elements `input_tree` partially flattened up to `shallow_tree`."""
-  if is_sequence(shallow_tree):
-    for shallow_branch, input_branch in zip(_yield_value(shallow_tree),
-                                            _yield_value(input_tree)):
-      for input_leaf in _yield_flat_up_to(shallow_branch, input_branch):
-        yield input_leaf
+  Yields:
+    Pairs of (path, value), where path the tuple path of a leaf node in
+    shallow_tree, and value is the value of the corresponding node in
+    input_tree.
+  """
+  if (isinstance(shallow_tree, _six.string_types) or
+      not any([isinstance(shallow_tree, _collections.Sequence),
+               isinstance(shallow_tree, _collections.Mapping),
+               _is_namedtuple(shallow_tree),
+               _is_attrs(shallow_tree)])):
+    yield (path, input_tree)
   else:
-    yield input_tree
+    input_tree = dict(_yield_sorted_items(input_tree))
+    for shallow_key, shallow_subtree in _yield_sorted_items(shallow_tree):
+      subpath = path + (shallow_key,)
+      input_subtree = input_tree[shallow_key]
+      for leaf_path, leaf_value in _yield_flat_up_to(shallow_subtree,
+                                                     input_subtree,
+                                                     path=subpath):
+        yield (leaf_path, leaf_value)
 
 
 def assert_shallow_structure(shallow_tree, input_tree, check_types=True):
@@ -538,8 +585,8 @@ def assert_shallow_structure(shallow_tree, input_tree, check_types=True):
 
   The following code will raise an exception:
   ```python
-    shallow_tree = ["a", "b"]
-    input_tree = ["c", ["d", "e"], "f"]
+    shallow_tree = {"a": "A", "b": "B"}
+    input_tree = {"a": 1, "c": 2}
     assert_shallow_structure(shallow_tree, input_tree)
   ```
 
@@ -578,40 +625,34 @@ def assert_shallow_structure(shallow_tree, input_tree, check_types=True):
       input_is_namedtuple = _is_namedtuple(input_tree, False)
       if shallow_is_namedtuple and input_is_namedtuple:
         if not _same_namedtuples(shallow_tree, input_tree):
-          raise TypeError(
-              "The two namedtuples don't have the same sequence type. Input "
-              "structure has type %s, while shallow structure has type %s."
-              % (type(input_tree), type(shallow_tree)))
+          raise TypeError(_STRUCTURES_HAVE_MISMATCHING_TYPES.format(
+              input_type=type(input_tree),
+              shallow_type=type(shallow_tree)))
+
       elif not (isinstance(shallow_tree, _collections.Mapping)
                 and isinstance(input_tree, _collections.Mapping)):
-        raise TypeError(
-            "The two structures don't have the same sequence type. Input "
-            "structure has type %s, while shallow structure has type %s."
-            % (type(input_tree), type(shallow_tree)))
-
-    if len(input_tree) != len(shallow_tree):
-      raise ValueError(
-          "The two structures don't have the same sequence length. Input "
-          "structure has length %s, while shallow structure has length %s."
-          % (len(input_tree), len(shallow_tree)))
+        raise TypeError(_STRUCTURES_HAVE_MISMATCHING_TYPES.format(
+            input_type=type(input_tree),
+            shallow_type=type(shallow_tree)))
 
-    if check_types and isinstance(shallow_tree, (dict, _collections.Mapping)):
-      if set(input_tree) != set(shallow_tree):
-        raise ValueError(
-            "The two structures don't have the same keys. Input "
-            "structure has keys %s, while shallow structure has keys %s." %
-            (list(_six.iterkeys(input_tree)),
-             list(_six.iterkeys(shallow_tree))))
+    if len(input_tree) < len(shallow_tree):
+      raise ValueError(_INPUT_TREE_SMALLER_THAN_SHALLOW_TREE.format(
+          input_size=len(input_tree),
+          shallow_size=len(shallow_tree)))
 
-      input_tree = list(sorted(_six.iteritems(input_tree)))
-      shallow_tree = list(sorted(_six.iteritems(shallow_tree)))
+    if isinstance(shallow_tree, _collections.Mapping):
+      absent_keys = set(shallow_tree) - set(input_tree)
+      if absent_keys:
+        raise ValueError(_SHALLOW_TREE_HAS_INVALID_KEYS
+                         .format(sorted(absent_keys)))
 
-    for shallow_branch, input_branch in zip(shallow_tree, input_tree):
+    for shallow_branch, input_branch in zip(_yield_value(shallow_tree),
+                                            _yield_value(input_tree)):
       assert_shallow_structure(shallow_branch, input_branch,
                                check_types=check_types)
 
 
-def flatten_up_to(shallow_tree, input_tree):
+def flatten_up_to(shallow_tree, input_tree, check_types=True):
   """Flattens `input_tree` up to `shallow_tree`.
 
   Any further depth in structure in `input_tree` is retained as elements in the
@@ -668,6 +709,8 @@ def flatten_up_to(shallow_tree, input_tree):
     shallow_tree: a possibly pruned structure of input_tree.
     input_tree: an arbitrarily nested structure or a scalar object.
       Note, numpy arrays are considered scalars.
+    check_types: bool. If True, check that each node in shallow_tree has the
+      same type as the corresponding node in input_tree.
 
   Returns:
     A Python list, the partially flattened version of `input_tree` according to
@@ -680,11 +723,12 @@ def flatten_up_to(shallow_tree, input_tree):
     ValueError: If the sequence lengths of `shallow_tree` are different from
       `input_tree`.
   """
-  assert_shallow_structure(shallow_tree, input_tree)
-  return list(_yield_flat_up_to(shallow_tree, input_tree))
+  assert_shallow_structure(shallow_tree, input_tree, check_types)
+  # Discard paths returned by _yield_flat_up_to.
+  return list(v for _, v in _yield_flat_up_to(shallow_tree, input_tree))
 
 
-def map_structure_up_to(shallow_tree, func, *inputs):
+def map_structure_up_to(shallow_tree, func, *inputs, **kwargs):
   """Applies a function or op to a number of partially flattened inputs.
 
   The `inputs` are flattened up to `shallow_tree` before being mapped.
@@ -733,6 +777,11 @@ def map_structure_up_to(shallow_tree, func, *inputs):
         shallow_tree. The function `func` is applied to corresponding
         partially flattened elements of each input, so the function must support
         arity of `len(inputs)`.
+    **kwargs: kwargs to feed to func(). Special kwarg
+      `check_types` is not passed to func, but instead determines whether the
+      types of iterables within the structures have to be same (e.g.
+      `map_structure(func, [1], (1,))` raises a `TypeError` exception). To allow
+      this set this argument to `False`.
 
   Raises:
     TypeError: If `shallow_tree` is a sequence but `input_tree` is not.
@@ -745,16 +794,93 @@ def map_structure_up_to(shallow_tree, func, *inputs):
     result of repeatedly applying `func`, with same structure as
     `shallow_tree`.
   """
+  return map_structure_with_tuple_paths_up_to(
+      shallow_tree,
+      lambda _, *values: func(*values),  # Discards the path arg.
+      *inputs,
+      **kwargs)
+
+
+def map_structure_with_tuple_paths_up_to(shallow_tree, func, *inputs, **kwargs):
+  """Applies a function or op to a number of partially flattened inputs.
+
+  Like map_structure_up_to(), except that the 'func' argument takes a path
+  tuple as its first argument, followed by the corresponding values from
+  *inputs.
+
+  Example:
+
+  lowercase = {'a': 'a', 'b': ('b0', 'b1')}
+  uppercase = {'a': 'A', 'b': ('B0', 'B1')}
+
+  def print_path_and_values(path, *values):
+    print("path: {}, values: {}".format(path, values))
+
+  shallow_tree = {'a': None}
+  map_structure_with_tuple_paths_up_to(shallow_tree,
+                                       print_path_and_values,
+                                       lowercase,
+                                       uppercase)
+  >>> path: ('a',), values: ('a', 'A')
+  >>> path: ('b', 0), values: ('b0', 'B0')
+  >>> path: ('b', 1), values: ('b1', 'B1')
+
+  shallow_tree = {'b': None}
+  map_structure_with_tuple_paths_up_to(shallow_tree,
+                                       print_path_and_values,
+                                       lowercase,
+                                       uppercase,
+                                       check_types=False)
+  >>> path: ('b', 1), values: (('bo', 'b1'), ('B0', 'B1'))
+
+  shallow_tree = {'a': None, 'b': {1: None}}
+  map_structure_with_tuple_paths_up_to(shallow_tree,
+                                       print_path_and_values,
+                                       lowercase,
+                                       uppercase,
+                                       check_types=False)
+  >>> path: ('a',), values: ('a', 'A')
+  >>> path: ('b', 1), values: ('b1', B1')
+
+  Args:
+    shallow_tree: a shallow tree, common to all the inputs.
+    func: callable that takes args (path, inputs_0_value, ... , inputs_N_value),
+      where path is a tuple path to a leaf node in shallow_tree, and
+      inputs_i_value is the corresponding value from inputs[i].
+    *inputs: nested structures that are all structurally compatible with
+        shallow_tree.
+    **kwargs: kwargs to feed to func(). Special kwarg
+      `check_types` is not passed to func, but instead determines whether the
+      types of iterables within the structures have to be same (e.g.
+      `map_structure(func, [1], (1,))` raises a `TypeError` exception). To allow
+      this set this argument to `False`.
+
+  Raises:
+    TypeError: If `shallow_tree` is a sequence but one of `*inputs` is not.
+    TypeError: If the sequence types of `shallow_tree` are different from
+      `input_tree`.
+    ValueError: If the sequence lengths of `shallow_tree` are different from
+      `input_tree`.
+
+  Returns:
+    Result of repeatedly applying `func`. Has same structure as `shallow_tree`.
+  """
   if not inputs:
     raise ValueError("Cannot map over no sequences")
+
+  check_types = kwargs.pop("check_types", True)
+
   for input_tree in inputs:
-    assert_shallow_structure(shallow_tree, input_tree)
+    assert_shallow_structure(shallow_tree, input_tree, check_types=check_types)
 
   # Flatten each input separately, apply the function to corresponding elements,
   # then repack based on the structure of the first input.
-  all_flattened_up_to = [flatten_up_to(shallow_tree, input_tree)
-                         for input_tree in inputs]
-  results = [func(*tensors) for tensors in zip(*all_flattened_up_to)]
+  flat_value_lists = [flatten_up_to(shallow_tree, input_tree, check_types)
+                      for input_tree in inputs]
+  flat_path_list = [path for path, _
+                    in _yield_flat_up_to(shallow_tree, inputs[0])]
+  results = [func(*args, **kwargs) for args in zip(flat_path_list,
+                                                   *flat_value_lists)]
   return pack_sequence_as(structure=shallow_tree, flat_sequence=results)
 
 
@@ -853,27 +979,8 @@ def yield_flat_paths(nest):
     Tuples containing index or key values which form the path to a specific
       leaf value in the nested structure.
   """
-
-  # The _maybe_add_final_path_element function is used below in order to avoid
-  # adding trailing slashes when the sub-element recursed into is a leaf.
-  if isinstance(nest, (dict, _collections.Mapping)):
-    for key in _sorted(nest):
-      value = nest[key]
-      for sub_path in yield_flat_paths(value):
-        yield (key,) + sub_path
-  elif _is_namedtuple(nest):
-    for key in nest._fields:
-      value = getattr(nest, key)
-      for sub_path in yield_flat_paths(value):
-        yield (key,) + sub_path
-  elif isinstance(nest, _six.string_types):
-    yield ()
-  elif isinstance(nest, _collections.Sequence):
-    for idx, value in enumerate(nest):
-      for sub_path in yield_flat_paths(value):
-        yield (idx,) + sub_path
-  else:
-    yield ()
+  for k, _ in _yield_flat_up_to(nest, nest):
+    yield k
 
 
 def flatten_with_joined_string_paths(structure, separator="/"):
diff --git a/tensorflow/python/util/nest_test.py b/tensorflow/python/util/nest_test.py
index 83fa5dd660..71034ffcb6 100644
--- a/tensorflow/python/util/nest_test.py
+++ b/tensorflow/python/util/nest_test.py
@@ -510,30 +510,28 @@ class NestTest(parameterized.TestCase, test.TestCase):
   def testAssertShallowStructure(self):
     inp_ab = ["a", "b"]
     inp_abc = ["a", "b", "c"]
-    expected_message = (
-        "The two structures don't have the same sequence length. Input "
-        "structure has length 2, while shallow structure has length 3.")
-    with self.assertRaisesRegexp(ValueError, expected_message):
-      nest.assert_shallow_structure(inp_abc, inp_ab)
+    with self.assertRaisesWithLiteralMatch(
+        ValueError,
+        nest._INPUT_TREE_SMALLER_THAN_SHALLOW_TREE.format(
+            shallow_size=len(inp_abc),
+            input_size=len(inp_ab))):
+      nest.assert_shallow_structure(shallow_tree=inp_abc, input_tree=inp_ab)
 
     inp_ab1 = [(1, 1), (2, 2)]
     inp_ab2 = [[1, 1], [2, 2]]
-    expected_message = (
-        "The two structures don't have the same sequence type. Input structure "
-        "has type <(type|class) 'tuple'>, while shallow structure has type "
-        "<(type|class) 'list'>.")
-    with self.assertRaisesRegexp(TypeError, expected_message):
+    with self.assertRaisesWithLiteralMatch(
+        TypeError,
+        nest._STRUCTURES_HAVE_MISMATCHING_TYPES.format(
+            shallow_type=type(inp_ab2[0]),
+            input_type=type(inp_ab1[0]))):
       nest.assert_shallow_structure(inp_ab2, inp_ab1)
     nest.assert_shallow_structure(inp_ab2, inp_ab1, check_types=False)
 
     inp_ab1 = {"a": (1, 1), "b": {"c": (2, 2)}}
     inp_ab2 = {"a": (1, 1), "b": {"d": (2, 2)}}
-    expected_message = (
-        r"The two structures don't have the same keys. Input "
-        r"structure has keys \['c'\], while shallow structure has "
-        r"keys \['d'\].")
-
-    with self.assertRaisesRegexp(ValueError, expected_message):
+    with self.assertRaisesWithLiteralMatch(
+        ValueError,
+        nest._SHALLOW_TREE_HAS_INVALID_KEYS.format(["d"])):
       nest.assert_shallow_structure(inp_ab2, inp_ab1)
 
     inp_ab = collections.OrderedDict([("a", 1), ("b", (2, 3))])
@@ -719,7 +717,9 @@ class NestTest(parameterized.TestCase, test.TestCase):
     # Non-equal dicts.
     inp_val = dict(a=2, b=3)
     inp_ops = dict(a=dict(add=1, mul=2), c=dict(add=2, mul=3))
-    with self.assertRaisesRegexp(ValueError, "same keys"):
+    with self.assertRaisesWithLiteralMatch(
+        ValueError,
+        nest._SHALLOW_TREE_HAS_INVALID_KEYS.format(["b"])):
       nest.map_structure_up_to(
           inp_val,
           lambda val, ops: (val + ops["add"]) * ops["mul"], inp_val, inp_ops)
@@ -736,7 +736,9 @@ class NestTest(parameterized.TestCase, test.TestCase):
     # Non-equal dict/mapping.
     inp_val = dict(a=2, b=3)
     inp_ops = _CustomMapping(a=dict(add=1, mul=2), c=dict(add=2, mul=3))
-    with self.assertRaisesRegexp(ValueError, "same keys"):
+    with self.assertRaisesWithLiteralMatch(
+        ValueError,
+        nest._SHALLOW_TREE_HAS_INVALID_KEYS.format(["b"])):
       nest.map_structure_up_to(
           inp_val,
           lambda val, ops: (val + ops["add"]) * ops["mul"], inp_val, inp_ops)
@@ -849,12 +851,12 @@ class NestTest(parameterized.TestCase, test.TestCase):
     self.assertEqual(expected, result)
 
   @parameterized.named_parameters(
-      ("tuples", (1, 2), (3, 4, 5), ValueError),
+      ("tuples", (1, 2, 3), (4, 5), ValueError),
       ("dicts", {"a": 1}, {"b": 2}, ValueError),
       ("mixed", (1, 2), [3, 4], TypeError),
       ("nested",
-       {"a": [2, 3], "b": [1, 3]},
-       {"b": [5, 6, 7], "a": [8, 9]},
+       {"a": [2, 3, 4], "b": [1, 3]},
+       {"b": [5, 6], "a": [8, 9]},
        ValueError
       ))
   def testMapWithPathsIncompatibleStructures(self, s1, s2, error_type):
@@ -884,13 +886,14 @@ class NestTest(parameterized.TestCase, test.TestCase):
     self.assertEqual(expected, result)
 
   @parameterized.named_parameters([
-      dict(testcase_name="Tuples", s1=(1, 2), s2=(3, 4, 5),
+      dict(testcase_name="Tuples", s1=(1, 2, 3), s2=(4, 5),
            error_type=ValueError),
       dict(testcase_name="Dicts", s1={"a": 1}, s2={"b": 2},
            error_type=ValueError),
       dict(testcase_name="Mixed", s1=(1, 2), s2=[3, 4], error_type=TypeError),
       dict(testcase_name="Nested",
-           s1={"a": [2, 3], "b": [1, 3]}, s2={"b": [5, 6, 7], "a": [8, 9]},
+           s1={"a": [2, 3, 4], "b": [1, 3]},
+           s2={"b": [5, 6], "a": [8, 9]},
            error_type=ValueError)
   ])
   def testMapWithTuplePathsIncompatibleStructures(self, s1, s2, error_type):
-- 
GitLab


From 7cbe1132394e9085bf39348da19e46c8c147548b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 25 Jan 2019 04:44:02 -0800
Subject: [PATCH 1246/2345] Replace --genrule_strategy w/ --strategy=Genrule

This is in preparation for --genrule_strategy becoming a no-op in a future Bazel release.

PiperOrigin-RevId: 230883422
---
 tensorflow/compiler/aot/tfcompile.bzl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/aot/tfcompile.bzl b/tensorflow/compiler/aot/tfcompile.bzl
index 4051664c24..2abe3e29b7 100644
--- a/tensorflow/compiler/aot/tfcompile.bzl
+++ b/tensorflow/compiler/aot/tfcompile.bzl
@@ -207,7 +207,7 @@ def tf_library(
         #
         # Note that setting the local=1 attribute on a *test target* causes the
         # test infrastructure to skip that test.  However this is a genrule, not
-        # a test target, and runs with --genrule_strategy=forced_forge, meaning
+        # a test target, and runs with --strategy=Genrule=forced_forge, meaning
         # the local=1 attribute is ignored, and the genrule is still run.
         #
         # https://www.bazel.io/versions/master/docs/be/general.html#genrule
-- 
GitLab


From 52ae0565666c5722226518982db80bf4c57300d3 Mon Sep 17 00:00:00 2001
From: Chris Jones <cjfj@google.com>
Date: Fri, 25 Jan 2019 04:48:47 -0800
Subject: [PATCH 1247/2345] Implement `TPUStrategy.experimental_run`.

PiperOrigin-RevId: 230883811
---
 .../contrib/distribute/python/tpu_strategy.py | 61 +++++++++++++++++--
 1 file changed, 56 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 4863bb29da..4387210062 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -188,6 +188,51 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
     """DEPRECATED: use .extended.steps_per_run instead."""
     return self._extended.steps_per_run
 
+  # TODO(cjfj): Modify `_call_for_each_replica` in `TPUExtended` such that this
+  # can use the default implementation.
+  # This implementation runs a single step. It does not use infeed or outfeed.
+  def experimental_run(self, fn, input_iterator=None):
+    """See base class."""
+    if context.executing_eagerly():
+      raise NotImplementedError("Eager mode not supported in TPUStrategy.")
+
+    if self.extended._disable_training_loop_on_host:  # pylint: disable=protected-access
+      raise NotImplementedError(
+          "`experimental_run` is not compatible with "
+          "`_disable_training_loop_on_host=True`")
+
+    if input_iterator is None:
+      inputs = []
+    else:
+      inputs = input_iterator.get_next()
+
+    result = [None]
+    def replicated_fn(replica_id, inputs):
+      """Wraps user function to provide replica ID and `Tensor` inputs."""
+      with _TPUReplicaContext(self, replica_id_in_sync_group=replica_id):
+        if input_iterator is None:
+          result[0] = fn()
+        else:
+          result[0] = fn(inputs)
+      return result[0]
+
+    replicate_inputs = []  # By replica.
+    for i in range(self.num_replicas_in_sync):
+      replicate_inputs.append(
+          [constant_op.constant(i, dtype=dtypes.int32),
+           values.select_replica(i, inputs)])
+
+    with self.scope():
+      replicate_outputs = tpu.replicate(replicated_fn, replicate_inputs)
+
+    # Workaround for `tpu.replicate` behaviour when single `Tensor` returned.
+    replicate_outputs = [
+        nest.pack_sequence_as(result[0], nest.flatten(replica_outputs))
+        for replica_outputs in replicate_outputs]
+
+    device_map = self.extended._device_map  # pylint: disable=protected-access
+    return values.regroup(device_map, replicate_outputs)
+
 
 class TPUExtended(distribute_lib.DistributionStrategyExtended):
   """Implementation of TPUStrategy."""
@@ -783,18 +828,24 @@ class _TPUReplicaContext(distribute_lib.ReplicaContext):
   """Replication Context class for TPU Strategy."""
 
   # TODO(sourabhbajaj): Call for each replica should be updating this.
-  def __init__(self, strategy):
-    # TODO(b/118385803): properly initialize replica_id, instead of always 0
-    replica_id = constant_op.constant(0, dtypes.int32)
+  # TODO(b/118385803): Always properly initialize replica_id.
+  def __init__(self, strategy, replica_id_in_sync_group=None):
+    if replica_id_in_sync_group is None:
+      replica_id_in_sync_group = constant_op.constant(0, dtypes.int32)
     distribute_lib.ReplicaContext.__init__(
-        self, strategy, replica_id_in_sync_group=replica_id)
+        self, strategy, replica_id_in_sync_group=replica_id_in_sync_group)
 
   @property
   def devices(self):
     distribute_lib.require_replica_context(self)
     ds = self._strategy
     replica_id = tensor_util.constant_value(self._replica_id_in_sync_group)
-    return (ds.extended.worker_devices[replica_id],)
+
+    if replica_id is None:  # Non-constant `Tensor` inside `tpu.replicate`.
+      # TODO(cjfj): Return other devices when model parallelism is supported.
+      return (tpu.core(0),)
+    else:
+      return (ds.extended.worker_devices[replica_id],)
 
 
 def _get_host_for_device(device):
-- 
GitLab


From 8a9c9c8f8fcd9b7857db5efd913ad693ee730e89 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 25 Jan 2019 04:50:30 -0800
Subject: [PATCH 1248/2345] [TF:XLA] Increase tolerance of ResizeImage test.

This test was flaky over 10% of the time with GPUs due to auto tuning.

PiperOrigin-RevId: 230883941
---
 tensorflow/compiler/tests/image_ops_test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/tests/image_ops_test.py b/tensorflow/compiler/tests/image_ops_test.py
index 12741c4d4a..42e688174f 100644
--- a/tensorflow/compiler/tests/image_ops_test.py
+++ b/tensorflow/compiler/tests/image_ops_test.py
@@ -423,7 +423,7 @@ class ResizeNearestNeighborTest(xla_test.XLATestCase):
       out = sess.run(resized, {image: image_np[np.newaxis, :, :, np.newaxis]})
       if large_tolerance:
         self.assertAllClose(
-            expected[np.newaxis, :, :, np.newaxis], out, rtol=0.03, atol=0.1)
+            expected[np.newaxis, :, :, np.newaxis], out, rtol=2e-4, atol=2e-4)
       else:
         self.assertAllClose(expected[np.newaxis, :, :, np.newaxis], out)
 
@@ -452,7 +452,7 @@ class ResizeNearestNeighborTest(xla_test.XLATestCase):
         np.array([[1, 2], [3, 4]], dtype=np.float32), [4, 4],
         expected=np.array(
             [[1, 1, 2, 2], [1, 1, 2, 2], [3, 3, 4, 4], [3, 3, 4, 4]],
-            dtype=np.float32))
+            dtype=np.float32), large_tolerance=True)
 
   def testAlignCorners3x3To2x2(self):
     self._assertForwardOpMatchesExpected(
-- 
GitLab


From 3296f42b0f6dbdd68dcd492f10d4b3aa5bfc2235 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Fri, 25 Jan 2019 05:03:17 -0800
Subject: [PATCH 1249/2345] Create VerifiedModule in tests with HloText.

This makes sure that the verifier is run in addition to the parser.
The parser may allow something which is not actually valid HLO.
Fix the bugs in tests that were found due to this change.

PiperOrigin-RevId: 230885173
---
 .../xla/service/algebraic_simplifier_test.cc  |  5 +--
 .../xla/service/copy_insertion_test.cc        | 16 +++----
 .../dfs_hlo_visitor_with_default_test.cc      |  5 +--
 .../hlo_element_type_converter_test.cc        | 31 +++++--------
 .../compiler/xla/tests/constants_test.cc      |  4 +-
 .../xla/tests/multioutput_fusion_test.cc      | 44 +++++--------------
 tensorflow/compiler/xla/tests/tuple_test.cc   |  8 +---
 .../compiler/xla/tools/hlo_extractor_test.cc  | 15 +++----
 8 files changed, 41 insertions(+), 87 deletions(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index 96354e6f6e..3602ab82b2 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -2227,9 +2227,8 @@ TEST_F(AlgebraicSimplifierTest, TransposeIsReshape) {
       ROOT reshaped_again = f32[10] reshape(f32[10,1,1] transposed)
     }
   )";
-  TF_ASSERT_OK_AND_ASSIGN(
-      auto module,
-      HloRunner::CreateModuleFromString(hlo_string, GetDebugOptionsForTest()));
+  TF_ASSERT_OK_AND_ASSIGN(auto module,
+                          ParseAndReturnVerifiedModule(hlo_string));
 
   HloPassFix<AlgebraicSimplifier> simplifier(default_options_);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
diff --git a/tensorflow/compiler/xla/service/copy_insertion_test.cc b/tensorflow/compiler/xla/service/copy_insertion_test.cc
index 4d4074943e..4391bdcba5 100644
--- a/tensorflow/compiler/xla/service/copy_insertion_test.cc
+++ b/tensorflow/compiler/xla/service/copy_insertion_test.cc
@@ -1855,8 +1855,7 @@ ENTRY %TokensShouldNotBeCopied () -> s32[] {
 }
 )";
   TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
-                          HloRunner::CreateModuleFromString(
-                              module_string, GetDebugOptionsForTest()));
+                          ParseAndReturnVerifiedModule(module_string));
   InsertCopies(module.get());
 
   // There should be no copies added because tokens should not be copied.
@@ -2119,8 +2118,7 @@ ENTRY TestComputation {
   ROOT while.3 = (s32[], s32[], s32[], s32[], s32[]) while(arg_tuple.6), condition=cond_wrapper.v3.2, body=_functionalize_body_2__.v25
 }
 )";
-  auto module_or_status =
-      HloRunner::CreateModuleFromString(hlo_string, GetDebugOptionsForTest());
+  auto module_or_status = ParseAndReturnVerifiedModule(hlo_string);
   auto module = module_or_status.ConsumeValueOrDie();
   InsertCopies(module.get());
 }
@@ -2220,8 +2218,7 @@ ENTRY TestComputation {
   ROOT while.3 = (s32[], s32[], s32[], s32[], s32[]) while(arg_tuple.6), condition=cond_wrapper.v3.2, body=_functionalize_body_2__.v25
 }
 )";
-  auto module_or_status =
-      HloRunner::CreateModuleFromString(hlo_string, GetDebugOptionsForTest());
+  auto module_or_status = ParseAndReturnVerifiedModule(hlo_string);
   auto module = module_or_status.ConsumeValueOrDie();
   InsertCopies(module.get());
 }
@@ -2238,7 +2235,7 @@ cond.inner {
 
 body.inner {
   param.body.inner = pred[] parameter(0)
-  ROOT neg = pred[] negate(param.body.inner)
+  ROOT not = pred[] not(param.body.inner)
 }
 
 cond.outer {
@@ -2255,9 +2252,8 @@ ENTRY TestComputation {
   ROOT while = pred[] while(entry_param), condition=cond.outer, body=body.outer
 }
 )";
-  TF_ASSERT_OK_AND_ASSIGN(
-      std::unique_ptr<HloModule> module,
-      HloRunner::CreateModuleFromString(hlo_string, GetDebugOptionsForTest()));
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseAndReturnVerifiedModule(hlo_string));
   InsertCopies(module.get());
 
   // There should only be a single copy inserted, and it's in the entry
diff --git a/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default_test.cc b/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default_test.cc
index 825e1436f0..70173d43d7 100644
--- a/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default_test.cc
+++ b/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default_test.cc
@@ -73,15 +73,14 @@ ENTRY TestComputation {
   abs = f32[] abs(arg)
   add = f32[] add(arg, gte)
   broadcast = f32[42] broadcast(add), dimensions={}
-  slice = f32[0] slice(broadcast), slice={[1:2]}
+  slice = f32[1] slice(broadcast), slice={[1:2]}
   copy = f32[] copy(arg)
   eq = pred[] equal-to(arg, gte)
   neg = f32[] negate(arg)
   ROOT convert = f64[] convert(f32[] arg)
 })";
   std::unique_ptr<HloModule> module =
-      HloRunner::CreateModuleFromString(hlo_string, GetDebugOptionsForTest())
-          .ConsumeValueOrDie();
+      ParseAndReturnVerifiedModule(hlo_string).ConsumeValueOrDie();
   ElementwiseTestVisitor visitor;
   TF_EXPECT_OK(module->entry_computation()->Accept(&visitor));
 }
diff --git a/tensorflow/compiler/xla/service/hlo_element_type_converter_test.cc b/tensorflow/compiler/xla/service/hlo_element_type_converter_test.cc
index a3b56a44a0..5b633784e2 100644
--- a/tensorflow/compiler/xla/service/hlo_element_type_converter_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_element_type_converter_test.cc
@@ -28,15 +28,7 @@ using ::testing::Eq;
 using ::testing::Not;
 using ::testing::ResultOf;
 
-class HloElementTypeConverterTest : public HloTestBase {
- public:
-  std::unique_ptr<HloModule> CreateModuleFromHloString(
-      const string& hlo_string) {
-    return HloRunner::CreateModuleFromString(hlo_string,
-                                             GetDebugOptionsForTest())
-        .ValueOrDie();
-  }
-};
+using HloElementTypeConverterTest = HloTestBase;
 
 TEST_F(HloElementTypeConverterTest, CustomCallsNotConverted) {
   const string& hlo_string = R"(
@@ -47,7 +39,7 @@ TEST_F(HloElementTypeConverterTest, CustomCallsNotConverted) {
            custom_call_target="foo"
     }
   )";
-  auto module = CreateModuleFromHloString(hlo_string);
+  auto module = ParseAndReturnVerifiedModule(hlo_string).ValueOrDie();
   HloElementTypeConverter type_converter(BF16, F32);
   TF_ASSERT_OK_AND_ASSIGN(bool converted, type_converter.Run(module.get()));
   EXPECT_FALSE(converted);
@@ -63,7 +55,7 @@ TEST_F(HloElementTypeConverterTest, InfeedsOutfeedsNotConverted) {
       outfeed = token[] outfeed(infeed.data, token0)
     }
   )";
-  auto module = CreateModuleFromHloString(hlo_string);
+  auto module = ParseAndReturnVerifiedModule(hlo_string).ValueOrDie();
   HloElementTypeConverter type_converter(BF16, F32);
   TF_ASSERT_OK_AND_ASSIGN(bool converted, type_converter.Run(module.get()));
   EXPECT_FALSE(converted);
@@ -73,17 +65,16 @@ TEST_F(HloElementTypeConverterTest, OperationsInNestedTuplesConverted) {
   const string& hlo_string = R"(
     HloModule NestedTuples
     ENTRY NestedTuples.v5 {
-      constant.4 = bf16[] constant(42)
       constant.2 = f32[2]{0} constant({1, 2})
-      constant.3 = bf16[] constant(42)
-      add = bf16[] add(constant.2, constant.3)
-      tuple = (f32[2]{0}, bf16[]) tuple(constant.2, add)
+      constant.3 = bf16[2]{0} constant({42, 42})
+      add = bf16[2]{0} add(constant.2, constant.3)
+      tuple = (f32[2]{0}, bf16[2]{0}) tuple(constant.2, add)
       constant.5 = bf16[2]{0} constant({22, 44})
-      ROOT tuple.1 = ((f32[2]{0}, bf16[]), bf16[2]{0}) tuple(tuple, constant.5)
+      ROOT tuple.1 = ((f32[2]{0}, bf16[2]{0}), bf16[2]{0}) tuple(tuple, constant.5)
     }
   )";
 
-  auto module = CreateModuleFromHloString(hlo_string);
+  auto module = ParseAndReturnVerifiedModule(hlo_string).ValueOrDie();
   HloElementTypeConverter type_converter(BF16, F32);
   TF_ASSERT_OK_AND_ASSIGN(bool converted, type_converter.Run(module.get()));
   EXPECT_TRUE(converted);
@@ -111,7 +102,7 @@ TEST_F(HloElementTypeConverterTest, BatchNormGradBF16Converted) {
     }
   )";
 
-  auto module = CreateModuleFromHloString(hlo_string);
+  auto module = ParseAndReturnVerifiedModule(hlo_string).ValueOrDie();
   HloElementTypeConverter type_converter(BF16, F32);
   TF_ASSERT_OK_AND_ASSIGN(bool converted, type_converter.Run(module.get()));
   EXPECT_TRUE(converted);
@@ -135,7 +126,7 @@ ENTRY main {
   ROOT rng = bf16[1,1000,20]{2,1,0} rng(constant.3, constant.4), distribution=rng_uniform
 }
   )";
-  auto module = CreateModuleFromHloString(hlo_string);
+  auto module = ParseAndReturnVerifiedModule(hlo_string).ValueOrDie();
   HloElementTypeConverter type_converter(BF16, F32);
   TF_ASSERT_OK_AND_ASSIGN(bool converted, type_converter.Run(module.get()));
   EXPECT_TRUE(converted);
@@ -161,7 +152,7 @@ ENTRY main {
   ROOT rng1 = bf16[1,1000,20]{2,1,0} rng(constant.3, constant.4), control-predecessors={%rng0}, distribution=rng_uniform
 }
   )";
-  auto module = CreateModuleFromHloString(hlo_string);
+  auto module = ParseAndReturnVerifiedModule(hlo_string).ValueOrDie();
 
   HloElementTypeConverter type_converter(BF16, F32);
   TF_ASSERT_OK_AND_ASSIGN(bool converted, type_converter.Run(module.get()));
diff --git a/tensorflow/compiler/xla/tests/constants_test.cc b/tensorflow/compiler/xla/tests/constants_test.cc
index c672e8db03..6530007871 100644
--- a/tensorflow/compiler/xla/tests/constants_test.cc
+++ b/tensorflow/compiler/xla/tests/constants_test.cc
@@ -224,9 +224,7 @@ XLA_TEST_F(ConstantsHloTest, DISABLED_ON_GPU(BitcastOfConstant)) {
       ROOT result = s32[] call(parameter.0, constant-as-scalar), to_apply=func
     }
   )";
-  auto module =
-      HloRunner::CreateModuleFromString(testcase, GetDebugOptionsForTest())
-          .ValueOrDie();
+  auto module = ParseAndReturnVerifiedModule(testcase).ValueOrDie();
   auto param = LiteralUtil::CreateR0<int32>(1);
   auto result = ExecuteNoHloPasses(std::move(module), {&param});
   EXPECT_TRUE(LiteralTestUtil::Equal(param, result));
diff --git a/tensorflow/compiler/xla/tests/multioutput_fusion_test.cc b/tensorflow/compiler/xla/tests/multioutput_fusion_test.cc
index 3f5135438f..1fd9cb055c 100644
--- a/tensorflow/compiler/xla/tests/multioutput_fusion_test.cc
+++ b/tensorflow/compiler/xla/tests/multioutput_fusion_test.cc
@@ -208,9 +208,7 @@ XLA_TEST_F(MultiOutputFusionTest, FusionNodeIsRoot) {
       ROOT fusion = (s32[]) fusion(x), kind=kLoop, calls=fused_computation
     }
   )";
-  auto module =
-      HloRunner::CreateModuleFromString(testcase, GetDebugOptionsForTest())
-          .ValueOrDie();
+  auto module = ParseAndReturnVerifiedModule(testcase).ValueOrDie();
   auto param = LiteralUtil::MakeTupleOwned(
       LiteralUtil::MakeTupleOwned(
           LiteralUtil::MakeTupleOwned(LiteralUtil::CreateR0<int32>(42)),
@@ -241,9 +239,7 @@ XLA_TEST_F(MultiOutputFusionTest, MultiOutputLoopFusion) {
       const = f32[4] constant({0, 0, 0, 0})
       ROOT select = f32[4] select(gte0, gte1, const)
     })";
-  auto module =
-      HloRunner::CreateModuleFromString(testcase, GetDebugOptionsForTest())
-          .ValueOrDie();
+  auto module = ParseAndReturnVerifiedModule(testcase).ValueOrDie();
   auto param = LiteralUtil::CreateR1<float>({1.0, 2.0, 3.0, -1.0});
   Literal result = ExecuteNoHloPasses(std::move(module), {&param});
   LiteralTestUtil::ExpectR1Equal<float>({0.0, 4.0, 9.0, 1.0}, result);
@@ -273,9 +269,7 @@ XLA_TEST_F(MultiOutputFusionTest, MultiOutputLoopFeedingMap) {
       p1 = f32[3] parameter(0)
       ROOT map = f32[3] map(p1), to_apply=map_computation
     })";
-  auto module =
-      HloRunner::CreateModuleFromString(testcase, GetDebugOptionsForTest())
-          .ValueOrDie();
+  auto module = ParseAndReturnVerifiedModule(testcase).ValueOrDie();
   auto param = LiteralUtil::CreateR1<float>({1.0, 2.0, 3.0});
   Literal result = ExecuteNoHloPasses(std::move(module), {&param});
   LiteralTestUtil::ExpectR1Equal<float>({0.0, 4.0, 9.0}, result);
@@ -315,9 +309,7 @@ XLA_TEST_F(MultiOutputFusionTest,
       ROOT fusion = (f32[2,2]{1,0}, f32[2,2]{1,0}) fusion(p), kind=kInput,
                                                         calls=fused_reduce
     })");
-  auto module =
-      HloRunner::CreateModuleFromString(testcase, GetDebugOptionsForTest())
-          .ValueOrDie();
+  auto module = ParseAndReturnVerifiedModule(testcase).ValueOrDie();
   auto param =
       LiteralUtil::CreateR3<float>({{{1, 2}, {3, 4}}, {{5, 6}, {7, 8}}});
   Literal result = ExecuteNoHloPasses(std::move(module), {&param});
@@ -346,9 +338,7 @@ XLA_TEST_F(MultiOutputFusionTest,
       ROOT fusion = (f32[2,2]{1,0}, f32[2,2]{1,0}) fusion(p), kind=kInput,
                                                         calls=fused_reduce
     })");
-  auto module =
-      HloRunner::CreateModuleFromString(testcase, GetDebugOptionsForTest())
-          .ValueOrDie();
+  auto module = ParseAndReturnVerifiedModule(testcase).ValueOrDie();
   auto param =
       LiteralUtil::CreateR3<float>({{{1, 2}, {3, 4}}, {{5, 6}, {7, 8}}});
   Literal result = ExecuteNoHloPasses(std::move(module), {&param});
@@ -378,9 +368,7 @@ XLA_TEST_F(MultiOutputFusionTest,
       ROOT fusion = (f32[2]{0}, f32[2]{0}, f32[2]{0}) fusion(p), kind=kInput,
                                                         calls=fused_reduce
     })");
-  auto module =
-      HloRunner::CreateModuleFromString(testcase, GetDebugOptionsForTest())
-          .ValueOrDie();
+  auto module = ParseAndReturnVerifiedModule(testcase).ValueOrDie();
   auto param =
       LiteralUtil::CreateR3<float>({{{1, 2}, {3, 4}}, {{5, 6}, {7, 8}}});
   Literal result = ExecuteNoHloPasses(std::move(module), {&param});
@@ -410,9 +398,7 @@ XLA_TEST_F(MultiOutputFusionTest,
       ROOT fusion = (f32[2,2,2]{2,1,0}, f32[2,2]{1,0}, f32[2,2]{1,0}) fusion(p),
                                                  kind=kInput, calls=fused_reduce
     })");
-  auto module =
-      HloRunner::CreateModuleFromString(testcase, GetDebugOptionsForTest())
-          .ValueOrDie();
+  auto module = ParseAndReturnVerifiedModule(testcase).ValueOrDie();
   auto param =
       LiteralUtil::CreateR3<float>({{{1, 2}, {3, 4}}, {{5, 6}, {7, 8}}});
   Literal result = ExecuteNoHloPasses(std::move(module), {&param});
@@ -443,9 +429,7 @@ XLA_TEST_F(MultiOutputFusionTest,
       ROOT fusion = (f32[2,2]{1,0}, f32[2,2,2]{2,1,0}, f32[2,2]{1,0}) fusion(p),
                                                  kind=kInput, calls=fused_reduce
     })");
-  auto module =
-      HloRunner::CreateModuleFromString(testcase, GetDebugOptionsForTest())
-          .ValueOrDie();
+  auto module = ParseAndReturnVerifiedModule(testcase).ValueOrDie();
   auto param =
       LiteralUtil::CreateR3<float>({{{1, 2}, {3, 4}}, {{5, 6}, {7, 8}}});
   Literal result = ExecuteNoHloPasses(std::move(module), {&param});
@@ -478,9 +462,7 @@ XLA_TEST_F(MultiOutputFusionTest,
       ROOT fusion = (f32[2]{0}, f32[2,2,2]{2,1,0}, f32[2,2,2]{2,1,0}) fusion(p),
                                                  kind=kInput, calls=fused_reduce
     })");
-  auto module =
-      HloRunner::CreateModuleFromString(testcase, GetDebugOptionsForTest())
-          .ValueOrDie();
+  auto module = ParseAndReturnVerifiedModule(testcase).ValueOrDie();
   auto param =
       LiteralUtil::CreateR3<float>({{{1, 2}, {3, 4}}, {{5, 6}, {7, 8}}});
   Literal result = ExecuteNoHloPasses(std::move(module), {&param});
@@ -513,9 +495,7 @@ XLA_TEST_F(MultiOutputFusionTest,
       ROOT fusion = (f32[2,2]{1,0}, f32[2,2]{1,0}) fusion(p, i, j), kind=kInput,
                                                               calls=fused_reduce
     })");
-  auto module =
-      HloRunner::CreateModuleFromString(testcase, GetDebugOptionsForTest())
-          .ValueOrDie();
+  auto module = ParseAndReturnVerifiedModule(testcase).ValueOrDie();
   auto param =
       LiteralUtil::CreateR3<float>({{{0, 2}, {3, 4}}, {{5, 6}, {7, 8}}});
   auto init1 = LiteralUtil::CreateR0<float>(5);
@@ -549,9 +529,7 @@ XLA_TEST_F(MultiOutputFusionTest,
       ROOT fusion = (f32[2,2]{1,0}, f32[2,2]{1,0}, f16[2,2,2]{2,1,0}) fusion(p),
                     kind=kInput, calls=fused_reduce
     })");
-  auto module =
-      HloRunner::CreateModuleFromString(testcase, GetDebugOptionsForTest())
-          .ValueOrDie();
+  auto module = ParseAndReturnVerifiedModule(testcase).ValueOrDie();
   auto param = LiteralUtil::CreateR3<Eigen::half>(
       {{{Eigen::half(1), Eigen::half(2)}, {Eigen::half(3), Eigen::half(4)}},
        {{Eigen::half(5), Eigen::half(6)}, {Eigen::half(7), Eigen::half(8)}}});
diff --git a/tensorflow/compiler/xla/tests/tuple_test.cc b/tensorflow/compiler/xla/tests/tuple_test.cc
index 0def0577e0..cdf2c34fcc 100644
--- a/tensorflow/compiler/xla/tests/tuple_test.cc
+++ b/tensorflow/compiler/xla/tests/tuple_test.cc
@@ -525,9 +525,7 @@ XLA_TEST_F(TupleHloTest, BitcastAfterGTE) {
       ROOT tuple.4 = (f32[1,3]{1,0}) tuple(copy)
     }
   )";
-  auto module =
-      HloRunner::CreateModuleFromString(testcase, GetDebugOptionsForTest())
-          .ValueOrDie();
+  auto module = ParseAndReturnVerifiedModule(testcase).ValueOrDie();
   auto param =
       LiteralUtil::MakeTupleOwned(LiteralUtil::CreateR1<float>({1, 2, 3}));
   auto result = ExecuteNoHloPasses(std::move(module), {&param});
@@ -559,9 +557,7 @@ XLA_TEST_F(TupleHloTest,
       ROOT outfeed = token[] outfeed(tuple, token0)
     }
   )";
-  auto module =
-      HloRunner::CreateModuleFromString(testcase, GetDebugOptionsForTest())
-          .ValueOrDie();
+  auto module = ParseAndReturnVerifiedModule(testcase).ValueOrDie();
   auto param0 = LiteralUtil::CreateR1<float>({1, 2});
   auto param1 = LiteralUtil::CreateR1<float>({2, 3});
   auto param4 = LiteralUtil::CreateR0<bool>(false);
diff --git a/tensorflow/compiler/xla/tools/hlo_extractor_test.cc b/tensorflow/compiler/xla/tools/hlo_extractor_test.cc
index c187222a11..4beb099b33 100644
--- a/tensorflow/compiler/xla/tools/hlo_extractor_test.cc
+++ b/tensorflow/compiler/xla/tools/hlo_extractor_test.cc
@@ -36,9 +36,8 @@ ENTRY %entry {
 }
 )";
 
-  TF_ASSERT_OK_AND_ASSIGN(
-      std::unique_ptr<HloModule> hlo_module,
-      HloRunner::CreateModuleFromString(hlo_string, GetDebugOptionsForTest()));
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> hlo_module,
+                          ParseAndReturnVerifiedModule(hlo_string));
 
   {
     auto extracted_module =
@@ -75,9 +74,8 @@ ENTRY %entry {
 }
 )";
 
-  TF_ASSERT_OK_AND_ASSIGN(
-      std::unique_ptr<HloModule> hlo_module,
-      HloRunner::CreateModuleFromString(hlo_string, GetDebugOptionsForTest()));
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> hlo_module,
+                          ParseAndReturnVerifiedModule(hlo_string));
 
   {
     auto extracted_module =
@@ -120,9 +118,8 @@ ENTRY %entry {
 }
 )";
 
-  TF_ASSERT_OK_AND_ASSIGN(
-      std::unique_ptr<HloModule> hlo_module,
-      HloRunner::CreateModuleFromString(hlo_string, GetDebugOptionsForTest()));
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> hlo_module,
+                          ParseAndReturnVerifiedModule(hlo_string));
 
   {
     auto extracted_module =
-- 
GitLab


From 720691477d84934e852e89f917e23f4ff55eb2da Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Fri, 25 Jan 2019 06:16:27 -0800
Subject: [PATCH 1250/2345] Whitelist the `collections` module. Add more
 logging. Populate partial_types more uniformly. Add extra test for
 namedtuple.

PiperOrigin-RevId: 230891637
---
 tensorflow/python/autograph/impl/api.py      | 36 +++++++--
 tensorflow/python/autograph/impl/api_test.py | 82 ++++++++++++++++----
 2 files changed, 98 insertions(+), 20 deletions(-)

diff --git a/tensorflow/python/autograph/impl/api.py b/tensorflow/python/autograph/impl/api.py
index 122ea1726b..be52b688df 100644
--- a/tensorflow/python/autograph/impl/api.py
+++ b/tensorflow/python/autograph/impl/api.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import collections
 import functools
 import sys
 
@@ -34,9 +35,9 @@ from tensorflow.python.autograph.impl import conversion
 from tensorflow.python.autograph.operators import py_builtins
 from tensorflow.python.autograph.pyct import compiler
 from tensorflow.python.autograph.pyct import inspect_utils
+from tensorflow.python.autograph.utils import ag_logging as logging
 from tensorflow.python.autograph.utils import py_func
 from tensorflow.python.framework import tensor_util
-from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import nest
 from tensorflow.python.util import tf_decorator
 from tensorflow.python.util import tf_inspect
@@ -160,7 +161,9 @@ def do_not_convert(run_as=RunMode.GRAPH, return_dtypes=None):
 
 def converted_call(f, owner, options, *args, **kwargs):
   """Compiles a function call inline. For internal use only."""
-  logging.vlog(logging.DEBUG, 'Converted call: %s; owner: %s', f, owner)
+  logging.log(1,
+              'Converted call: %s; owner: %s\n    args: %s\n    kwargs: %s\n',
+              f, owner, args, kwargs)
 
   if owner is not None:
     if not isinstance(f, str):
@@ -179,9 +182,13 @@ def converted_call(f, owner, options, *args, **kwargs):
   if inspect_utils.isbuiltin(f):
     return py_builtins.overload_of(f)(*args, **kwargs)
 
+  # Other built-in modules are permanently whitelisted.
+  # TODO(mdan): Figure out how to do this consistently for all stdlib modules.
+  if f in collections.__dict__.values():
+    return f(*args, **kwargs)
+
   # TODO(mdan): This needs cleanup.
-  # In particular, we may want to avoid renaming functions altogether.
-  if not options.force_conversion and conversion.is_whitelisted_for_graph(f):
+  if (not options.force_conversion and conversion.is_whitelisted_for_graph(f)):
 
     # TODO(mdan): This may be inconsistent in certain situations.
     # If the function had already been annotated with @tf.function, it
@@ -223,6 +230,13 @@ def converted_call(f, owner, options, *args, **kwargs):
     arg_map_target = f
     f_self = inspect_utils.getmethodself(f)
 
+    if owner is not None:
+      partial_types = (type(owner),)
+    elif f_self is not None:
+      partial_types = (type(f_self),)
+    else:
+      partial_types = ()
+
     # TODO(b/119246461): This may be more elegantly handled using __get__?
     if f_self is not None:
       # If this is a method call, it may or may not include self.
@@ -243,10 +257,8 @@ def converted_call(f, owner, options, *args, **kwargs):
           effective_args = (f_self,) + args
         else:
           effective_args = (f_self,) + args[1:]
-      partial_types = (f_self,)
     else:
       effective_args = args
-      partial_types = ()
 
   elif tf_inspect.isclass(f):
     # Constructors
@@ -282,6 +294,9 @@ def converted_call(f, owner, options, *args, **kwargs):
       if tf_inspect.isclass(arg_values['cls']):
         partial_types = (arg_values['cls'],)
 
+  logging.log(
+      3, 'Partial types in conversion of %s: %s', target_entity, partial_types)
+
   converted_f = to_graph(
       target_entity,
       recursive=options.recursive,
@@ -292,6 +307,12 @@ def converted_call(f, owner, options, *args, **kwargs):
       experimental_verbose=options.verbose,
       experimental_partial_types=partial_types)
 
+  if logging.has_verbosity(2):
+    callargs = tf_inspect.getcallargs(converted_f, *effective_args, **kwargs)
+    formatted_callargs = '\n'.join(
+        '    {}: {}'.format(k, v) for k, v in callargs.items())
+    logging.log(2, 'Calling %s with\n%s\n', converted_f, formatted_callargs)
+
   result = converted_f(*effective_args, **kwargs)
 
   # The converted function's closure is simply inserted into the function's
@@ -437,6 +458,9 @@ def to_graph(entity,
   if tf_inspect.isfunction(entity):
     compiled.__defaults__ = entity.__defaults__
 
+  logging.log(3, 'Namespace of %s includes: %s', compiled,
+              compiled_module.__dict__.keys())
+
   if hasattr(compiled, '__globals__'):
     # Remove self to avoid circular references. This will probably only work
     # so long as the function is not reentrant.
diff --git a/tensorflow/python/autograph/impl/api_test.py b/tensorflow/python/autograph/impl/api_test.py
index d5561ba824..51928099e8 100644
--- a/tensorflow/python/autograph/impl/api_test.py
+++ b/tensorflow/python/autograph/impl/api_test.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import collections
 import functools
 import gc
 
@@ -26,6 +27,7 @@ import numpy as np
 from tensorflow.python.autograph import utils
 from tensorflow.python.autograph.core import converter
 from tensorflow.python.autograph.impl import api
+from tensorflow.python.autograph.pyct import inspect_utils
 from tensorflow.python.autograph.pyct import parser
 from tensorflow.python.autograph.utils import py_func
 from tensorflow.python.framework import constant_op
@@ -46,7 +48,7 @@ class TestResource(str):
 class ApiTest(test.TestCase):
 
   @test_util.run_deprecated_v1
-  def test_decorator_recurses(self):
+  def test_decorator_recursive(self):
 
     class TestClass(object):
 
@@ -69,7 +71,7 @@ class ApiTest(test.TestCase):
       self.assertListEqual([0, 1], self.evaluate(x).tolist())
 
   @test_util.run_deprecated_v1
-  def test_decorator_does_not_recurse(self):
+  def test_decorator_not_recursive(self):
 
     class TestClass(object):
 
@@ -90,7 +92,7 @@ class ApiTest(test.TestCase):
       self.assertListEqual([0, 1], self.evaluate(x).tolist())
 
   @test_util.run_deprecated_v1
-  def test_decorator_calls_unconverted_graph(self):
+  def test_convert_then_do_not_convert_graph(self):
 
     class TestClass(object):
 
@@ -105,14 +107,13 @@ class ApiTest(test.TestCase):
         return x
 
     tc = TestClass()
-    with self.cached_session() as sess:
-      x = tc.test_method(
-          constant_op.constant([2, 4]), constant_op.constant(1),
-          constant_op.constant(-2))
-      self.assertListEqual([0, 1], self.evaluate(x).tolist())
+    x = tc.test_method(
+        constant_op.constant((2, 4)), constant_op.constant(1),
+        constant_op.constant(-2))
+    self.assertAllEqual((0, 1), self.evaluate(x))
 
   @test_util.run_deprecated_v1
-  def test_decorator_calls_unconverted_py_func(self):
+  def test_convert_then_do_not_convert_py_func(self):
 
     class TestClass(object):
 
@@ -132,11 +133,10 @@ class ApiTest(test.TestCase):
         return x
 
     tc = TestClass()
-    with self.cached_session() as sess:
-      x = tc.test_method(
-          constant_op.constant([2, 4]), constant_op.constant(1),
-          constant_op.constant(-2))
-      self.assertListEqual([0, 1], self.evaluate(x).tolist())
+    x = tc.test_method(
+        constant_op.constant((2, 4)), constant_op.constant(1),
+        constant_op.constant(-2))
+    self.assertAllEqual((0, 1), self.evaluate(x))
 
   @test_util.run_deprecated_v1
   def test_decorator_calls_decorated(self):
@@ -265,6 +265,26 @@ class ApiTest(test.TestCase):
                              converter.ConversionOptions(), tc)
       self.assertEqual(1, self.evaluate(x))
 
+  def test_converted_call_method_converts_recursively(self):
+
+    class TestClass(object):
+
+      def __init__(self, x):
+        self.x = x
+
+      def other_method(self):
+        if self.x < 0:
+          return -self.x
+        return self.x
+
+      def test_method(self):
+        return self.other_method()
+
+    tc = TestClass(constant_op.constant(-1))
+    x = api.converted_call(tc.test_method, None,
+                           converter.ConversionOptions(recursive=True), tc)
+    self.assertEqual(1, self.evaluate(x))
+
   def test_converted_call_method_by_class(self):
 
     class TestClass(object):
@@ -334,6 +354,22 @@ class ApiTest(test.TestCase):
                              constant_op.constant(0))
       self.assertTrue(self.evaluate(x))
 
+  def test_converted_call_then_already_converted_dynamic(self):
+
+    @api.convert()
+    def g(x):
+      if x > 0:
+        return x
+      else:
+        return -x
+
+    def f(g, x):
+      return g(x)
+
+    x = api.converted_call(f, None, converter.ConversionOptions(),
+                           g, constant_op.constant(1))
+    self.assertEqual(self.evaluate(x), 1)
+
   @test_util.run_deprecated_v1
   def test_converted_call_no_user_code(self):
 
@@ -397,6 +433,24 @@ class ApiTest(test.TestCase):
       self.evaluate(variables.global_variables_initializer())
       self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
 
+  def test_converted_call_namedtuple(self):
+
+    opts = converter.ConversionOptions()
+
+    x = api.converted_call(collections.namedtuple, None, opts,
+                           'TestNamedtuple', ('a', 'b'))
+
+    self.assertTrue(inspect_utils.isnamedtuple(x))
+
+  def test_converted_call_namedtuple_via_collections(self):
+
+    opts = converter.ConversionOptions()
+
+    x = api.converted_call('namedtuple', collections, opts,
+                           'TestNamedtuple', ('a', 'b'))
+
+    self.assertTrue(inspect_utils.isnamedtuple(x))
+
   def test_converted_call_lambda(self):
 
     opts = converter.ConversionOptions()
-- 
GitLab


From 6ec1edebee09f871301529097ad501f4cd68f6ce Mon Sep 17 00:00:00 2001
From: vcarpani <v.carpani@fizyr.com>
Date: Fri, 25 Jan 2019 16:21:03 +0100
Subject: [PATCH 1251/2345] Add support for half precision type in linear
 algebra operators

---
 tensorflow/core/ops/linalg_ops.cc | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/tensorflow/core/ops/linalg_ops.cc b/tensorflow/core/ops/linalg_ops.cc
index 952ee4bee2..8a8419588a 100644
--- a/tensorflow/core/ops/linalg_ops.cc
+++ b/tensorflow/core/ops/linalg_ops.cc
@@ -213,7 +213,7 @@ Status SvdShapeFn(InferenceContext* c) {
 REGISTER_OP("MatrixDeterminant")
     .Input("input: T")
     .Output("output: T")
-    .Attr("T: {float, double, complex64, complex128}")
+    .Attr("T: {half, float, double, complex64, complex128}")
     .SetShapeFn([](InferenceContext* c) {
       ShapeHandle input;
       TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 2, &input));
@@ -232,7 +232,7 @@ REGISTER_OP("LogMatrixDeterminant")
     .Input("input: T")
     .Output("sign: T")
     .Output("log_abs_determinant: T")
-    .Attr("T: {float, double, complex64, complex128}")
+    .Attr("T: {half, float, double, complex64, complex128}")
     .SetShapeFn([](InferenceContext* c) {
       ShapeHandle input;
       TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 2, &input));
@@ -255,7 +255,7 @@ REGISTER_OP("MatrixInverse")
     .Input("input: T")
     .Output("output: T")
     .Attr("adjoint: bool = False")
-    .Attr("T: {double, float, complex64, complex128}")
+    .Attr("T: {double, float, half, complex64, complex128}")
     .SetShapeFn(BatchUnchangedSquareShapeFn);
 
 REGISTER_OP("MatrixExponential")
@@ -263,7 +263,7 @@ REGISTER_OP("MatrixExponential")
         27, "Use Python implementation tf.linalg.matrix_exponential instead.")
     .Input("input: T")
     .Output("output: T")
-    .Attr("T: {double, float, complex64, complex128}")
+    .Attr("T: {double, float, half, complex64, complex128}")
     .SetShapeFn(BatchUnchangedSquareShapeFn);
 
 REGISTER_OP("MatrixLogarithm")
@@ -275,20 +275,20 @@ REGISTER_OP("MatrixLogarithm")
 REGISTER_OP("Cholesky")
     .Input("input: T")
     .Output("output: T")
-    .Attr("T: {double, float, complex64, complex128}")
+    .Attr("T: {double, float, half, complex64, complex128}")
     .SetShapeFn(BatchUnchangedSquareShapeFn);
 
 REGISTER_OP("CholeskyGrad")
     .Input("l: T")
     .Input("grad: T")
     .Output("output: T")
-    .Attr("T: {float, double}")
+    .Attr("T: {half, float, double}")
     .SetShapeFn(BatchUnchangedSquareShapeFn);
 
 REGISTER_OP("SelfAdjointEig")
     .Input("input: T")
     .Output("output: T")
-    .Attr("T: {double, float}")
+    .Attr("T: {double, float, half}")
     .Deprecated(11, "Use SelfAdjointEigV2 instead.")
     .SetShapeFn([](InferenceContext* c) {
       ShapeHandle input;
@@ -310,14 +310,14 @@ REGISTER_OP("SelfAdjointEigV2")
     .Output("e: T")
     .Output("v: T")
     .Attr("compute_v: bool = True")
-    .Attr("T: {double, float, complex64, complex128}")
+    .Attr("T: {double, float, half, complex64, complex128}")
     .SetShapeFn(SelfAdjointEigV2ShapeFn);
 
 REGISTER_OP("Lu")
     .Input("input: T")
     .Output("lu: T")
     .Output("p: output_idx_type")
-    .Attr("T: {double, float, complex64, complex128}")
+    .Attr("T: {double, float, half, complex64, complex128}")
     .Attr("output_idx_type: {int32, int64} = DT_INT32")
     .SetShapeFn(LuShapeFn);
 
@@ -326,7 +326,7 @@ REGISTER_OP("MatrixSolve")
     .Input("rhs: T")
     .Output("output: T")
     .Attr("adjoint: bool = False")
-    .Attr("T: {double, float, complex64, complex128}")
+    .Attr("T: {double, float, half, complex64, complex128}")
     .SetShapeFn([](InferenceContext* c) {
       return MatrixSolveShapeFn(c, true /* square (*/);
     });
@@ -337,7 +337,7 @@ REGISTER_OP("MatrixTriangularSolve")
     .Output("output: T")
     .Attr("lower: bool = True")
     .Attr("adjoint: bool = False")
-    .Attr("T: {double, float, complex64, complex128}")
+    .Attr("T: {double, float, half, complex64, complex128}")
     .SetShapeFn([](InferenceContext* c) {
       return MatrixSolveShapeFn(c, true /* square (*/);
     });
@@ -347,7 +347,7 @@ REGISTER_OP("MatrixSolveLs")
     .Input("rhs: T")
     .Input("l2_regularizer: double")
     .Output("output: T")
-    .Attr("T: {double, float, complex64, complex128}")
+    .Attr("T: {double, float, half, complex64, complex128}")
     .Attr("fast: bool = True")
     .SetShapeFn([](InferenceContext* c) {
       ShapeHandle l2_regularizer;
@@ -358,7 +358,7 @@ REGISTER_OP("MatrixSolveLs")
 REGISTER_OP("MatrixSquareRoot")
     .Input("input: T")
     .Output("output: T")
-    .Attr("T: {double, float, complex64, complex128}")
+    .Attr("T: {double, float, half, complex64, complex128}")
     .SetShapeFn(BatchUnchangedSquareShapeFn);
 
 REGISTER_OP("Qr")
@@ -366,7 +366,7 @@ REGISTER_OP("Qr")
     .Output("q: T")
     .Output("r: T")
     .Attr("full_matrices: bool = False")
-    .Attr("T: {double, float, complex64, complex128}")
+    .Attr("T: {double, float, half, complex64, complex128}")
     .SetShapeFn(QrShapeFn);
 
 REGISTER_OP("Svd")
@@ -376,7 +376,7 @@ REGISTER_OP("Svd")
     .Output("v: T")
     .Attr("compute_uv: bool = True")
     .Attr("full_matrices: bool = False")
-    .Attr("T: {double, float, complex64, complex128}")
+    .Attr("T: {double, float, half, complex64, complex128}")
     .SetShapeFn(SvdShapeFn);
 
 // Deprecated op registrations:
-- 
GitLab


From c80c3d6fa00646baf841e8826bb361ec446dbae0 Mon Sep 17 00:00:00 2001
From: vcarpani <v.carpani@fizyr.com>
Date: Fri, 25 Jan 2019 16:22:20 +0100
Subject: [PATCH 1252/2345] Add tests for half precision type in linear algebra
 operators

---
 tensorflow/core/ops/linalg_ops_test.cc | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/ops/linalg_ops_test.cc b/tensorflow/core/ops/linalg_ops_test.cc
index bfacee14ef..7d1aed50be 100644
--- a/tensorflow/core/ops/linalg_ops_test.cc
+++ b/tensorflow/core/ops/linalg_ops_test.cc
@@ -80,12 +80,18 @@ TEST(LinalgOpsTest, SelfAdjointEig_ShapeFn) {
 TEST(LinalgOpsTest, SelfAdjointEigV2_ShapeFn) {
   ShapeInferenceTestOp op("SelfAdjointEigV2");
   auto set_compute_v = [&op](bool compute_v) {
+    // Test for float32
     TF_ASSERT_OK(NodeDefBuilder("test", "Pack")
                      .Input({{"input", 0, DT_FLOAT}})
                      .Attr("compute_v", compute_v)
                      .Finalize(&op.node_def));
-  };
 
+    // Test for float16
+    TF_ASSERT_OK(NodeDefBuilder("test", "Pack")
+                     .Input({{"input", 0, DT_HALF}})
+                     .Attr("compute_v", compute_v)
+                     .Finalize(&op.node_def));
+  };
   set_compute_v(false);
   INFER_ERROR("Shape must be at least rank 2 but is rank 1", op, "[1]");
   INFER_ERROR("Dimensions must be equal, but are 1 and 2", op, "[1,2]");
@@ -174,10 +180,17 @@ TEST(LinalgOpsTest, MatrixSolveLs_ShapeFn) {
 TEST(LinalgOpsTest, Qr_ShapeFn) {
   ShapeInferenceTestOp op("Qr");
   auto set_attrs = [&op](bool full_matrices) {
+    // Test float32
     TF_ASSERT_OK(NodeDefBuilder("test", "Qr")
                      .Input({"input", 0, DT_FLOAT})
                      .Attr("full_matrices", full_matrices)
                      .Finalize(&op.node_def));
+
+    // Test float16
+    TF_ASSERT_OK(NodeDefBuilder("test", "Qr")
+                     .Input({"input", 0, DT_HALF})
+                     .Attr("full_matrices", full_matrices)
+                     .Finalize(&op.node_def));
   };
 
   // Defining `P` = min(`M`, `N`), if full_matrices = False, then Q should be
@@ -218,11 +231,19 @@ TEST(LinalgOpsTest, Qr_ShapeFn) {
 TEST(LinalgOpsTest, Svd_ShapeFn) {
   ShapeInferenceTestOp op("Svd");
   auto set_attrs = [&op](bool compute_uv, bool full_matrices) {
+    // Test for float32
     TF_ASSERT_OK(NodeDefBuilder("test", "Svd")
                      .Input({"input", 0, DT_FLOAT})
                      .Attr("compute_uv", compute_uv)
                      .Attr("full_matrices", full_matrices)
                      .Finalize(&op.node_def));
+
+    // Test for float16
+    TF_ASSERT_OK(NodeDefBuilder("test", "Svd")
+                     .Input({"input", 0, DT_HALF})
+                     .Attr("compute_uv", compute_uv)
+                     .Attr("full_matrices", full_matrices)
+                     .Finalize(&op.node_def));
   };
 
   // Defining `P` = min(`M`, `N`), if full_matrices = False, then U should be
-- 
GitLab


From 850cfa2c58e70dbc05536a20e2d5d0740a946bd3 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Fri, 25 Jan 2019 08:23:33 -0800
Subject: [PATCH 1253/2345] Automated rollback of commit
 720691477d84934e852e89f917e23f4ff55eb2da

PiperOrigin-RevId: 230906075
---
 tensorflow/python/autograph/impl/api.py      | 36 ++-------
 tensorflow/python/autograph/impl/api_test.py | 82 ++++----------------
 2 files changed, 20 insertions(+), 98 deletions(-)

diff --git a/tensorflow/python/autograph/impl/api.py b/tensorflow/python/autograph/impl/api.py
index be52b688df..122ea1726b 100644
--- a/tensorflow/python/autograph/impl/api.py
+++ b/tensorflow/python/autograph/impl/api.py
@@ -18,7 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import collections
 import functools
 import sys
 
@@ -35,9 +34,9 @@ from tensorflow.python.autograph.impl import conversion
 from tensorflow.python.autograph.operators import py_builtins
 from tensorflow.python.autograph.pyct import compiler
 from tensorflow.python.autograph.pyct import inspect_utils
-from tensorflow.python.autograph.utils import ag_logging as logging
 from tensorflow.python.autograph.utils import py_func
 from tensorflow.python.framework import tensor_util
+from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import nest
 from tensorflow.python.util import tf_decorator
 from tensorflow.python.util import tf_inspect
@@ -161,9 +160,7 @@ def do_not_convert(run_as=RunMode.GRAPH, return_dtypes=None):
 
 def converted_call(f, owner, options, *args, **kwargs):
   """Compiles a function call inline. For internal use only."""
-  logging.log(1,
-              'Converted call: %s; owner: %s\n    args: %s\n    kwargs: %s\n',
-              f, owner, args, kwargs)
+  logging.vlog(logging.DEBUG, 'Converted call: %s; owner: %s', f, owner)
 
   if owner is not None:
     if not isinstance(f, str):
@@ -182,13 +179,9 @@ def converted_call(f, owner, options, *args, **kwargs):
   if inspect_utils.isbuiltin(f):
     return py_builtins.overload_of(f)(*args, **kwargs)
 
-  # Other built-in modules are permanently whitelisted.
-  # TODO(mdan): Figure out how to do this consistently for all stdlib modules.
-  if f in collections.__dict__.values():
-    return f(*args, **kwargs)
-
   # TODO(mdan): This needs cleanup.
-  if (not options.force_conversion and conversion.is_whitelisted_for_graph(f)):
+  # In particular, we may want to avoid renaming functions altogether.
+  if not options.force_conversion and conversion.is_whitelisted_for_graph(f):
 
     # TODO(mdan): This may be inconsistent in certain situations.
     # If the function had already been annotated with @tf.function, it
@@ -230,13 +223,6 @@ def converted_call(f, owner, options, *args, **kwargs):
     arg_map_target = f
     f_self = inspect_utils.getmethodself(f)
 
-    if owner is not None:
-      partial_types = (type(owner),)
-    elif f_self is not None:
-      partial_types = (type(f_self),)
-    else:
-      partial_types = ()
-
     # TODO(b/119246461): This may be more elegantly handled using __get__?
     if f_self is not None:
       # If this is a method call, it may or may not include self.
@@ -257,8 +243,10 @@ def converted_call(f, owner, options, *args, **kwargs):
           effective_args = (f_self,) + args
         else:
           effective_args = (f_self,) + args[1:]
+      partial_types = (f_self,)
     else:
       effective_args = args
+      partial_types = ()
 
   elif tf_inspect.isclass(f):
     # Constructors
@@ -294,9 +282,6 @@ def converted_call(f, owner, options, *args, **kwargs):
       if tf_inspect.isclass(arg_values['cls']):
         partial_types = (arg_values['cls'],)
 
-  logging.log(
-      3, 'Partial types in conversion of %s: %s', target_entity, partial_types)
-
   converted_f = to_graph(
       target_entity,
       recursive=options.recursive,
@@ -307,12 +292,6 @@ def converted_call(f, owner, options, *args, **kwargs):
       experimental_verbose=options.verbose,
       experimental_partial_types=partial_types)
 
-  if logging.has_verbosity(2):
-    callargs = tf_inspect.getcallargs(converted_f, *effective_args, **kwargs)
-    formatted_callargs = '\n'.join(
-        '    {}: {}'.format(k, v) for k, v in callargs.items())
-    logging.log(2, 'Calling %s with\n%s\n', converted_f, formatted_callargs)
-
   result = converted_f(*effective_args, **kwargs)
 
   # The converted function's closure is simply inserted into the function's
@@ -458,9 +437,6 @@ def to_graph(entity,
   if tf_inspect.isfunction(entity):
     compiled.__defaults__ = entity.__defaults__
 
-  logging.log(3, 'Namespace of %s includes: %s', compiled,
-              compiled_module.__dict__.keys())
-
   if hasattr(compiled, '__globals__'):
     # Remove self to avoid circular references. This will probably only work
     # so long as the function is not reentrant.
diff --git a/tensorflow/python/autograph/impl/api_test.py b/tensorflow/python/autograph/impl/api_test.py
index 51928099e8..d5561ba824 100644
--- a/tensorflow/python/autograph/impl/api_test.py
+++ b/tensorflow/python/autograph/impl/api_test.py
@@ -18,7 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import collections
 import functools
 import gc
 
@@ -27,7 +26,6 @@ import numpy as np
 from tensorflow.python.autograph import utils
 from tensorflow.python.autograph.core import converter
 from tensorflow.python.autograph.impl import api
-from tensorflow.python.autograph.pyct import inspect_utils
 from tensorflow.python.autograph.pyct import parser
 from tensorflow.python.autograph.utils import py_func
 from tensorflow.python.framework import constant_op
@@ -48,7 +46,7 @@ class TestResource(str):
 class ApiTest(test.TestCase):
 
   @test_util.run_deprecated_v1
-  def test_decorator_recursive(self):
+  def test_decorator_recurses(self):
 
     class TestClass(object):
 
@@ -71,7 +69,7 @@ class ApiTest(test.TestCase):
       self.assertListEqual([0, 1], self.evaluate(x).tolist())
 
   @test_util.run_deprecated_v1
-  def test_decorator_not_recursive(self):
+  def test_decorator_does_not_recurse(self):
 
     class TestClass(object):
 
@@ -92,7 +90,7 @@ class ApiTest(test.TestCase):
       self.assertListEqual([0, 1], self.evaluate(x).tolist())
 
   @test_util.run_deprecated_v1
-  def test_convert_then_do_not_convert_graph(self):
+  def test_decorator_calls_unconverted_graph(self):
 
     class TestClass(object):
 
@@ -107,13 +105,14 @@ class ApiTest(test.TestCase):
         return x
 
     tc = TestClass()
-    x = tc.test_method(
-        constant_op.constant((2, 4)), constant_op.constant(1),
-        constant_op.constant(-2))
-    self.assertAllEqual((0, 1), self.evaluate(x))
+    with self.cached_session() as sess:
+      x = tc.test_method(
+          constant_op.constant([2, 4]), constant_op.constant(1),
+          constant_op.constant(-2))
+      self.assertListEqual([0, 1], self.evaluate(x).tolist())
 
   @test_util.run_deprecated_v1
-  def test_convert_then_do_not_convert_py_func(self):
+  def test_decorator_calls_unconverted_py_func(self):
 
     class TestClass(object):
 
@@ -133,10 +132,11 @@ class ApiTest(test.TestCase):
         return x
 
     tc = TestClass()
-    x = tc.test_method(
-        constant_op.constant((2, 4)), constant_op.constant(1),
-        constant_op.constant(-2))
-    self.assertAllEqual((0, 1), self.evaluate(x))
+    with self.cached_session() as sess:
+      x = tc.test_method(
+          constant_op.constant([2, 4]), constant_op.constant(1),
+          constant_op.constant(-2))
+      self.assertListEqual([0, 1], self.evaluate(x).tolist())
 
   @test_util.run_deprecated_v1
   def test_decorator_calls_decorated(self):
@@ -265,26 +265,6 @@ class ApiTest(test.TestCase):
                              converter.ConversionOptions(), tc)
       self.assertEqual(1, self.evaluate(x))
 
-  def test_converted_call_method_converts_recursively(self):
-
-    class TestClass(object):
-
-      def __init__(self, x):
-        self.x = x
-
-      def other_method(self):
-        if self.x < 0:
-          return -self.x
-        return self.x
-
-      def test_method(self):
-        return self.other_method()
-
-    tc = TestClass(constant_op.constant(-1))
-    x = api.converted_call(tc.test_method, None,
-                           converter.ConversionOptions(recursive=True), tc)
-    self.assertEqual(1, self.evaluate(x))
-
   def test_converted_call_method_by_class(self):
 
     class TestClass(object):
@@ -354,22 +334,6 @@ class ApiTest(test.TestCase):
                              constant_op.constant(0))
       self.assertTrue(self.evaluate(x))
 
-  def test_converted_call_then_already_converted_dynamic(self):
-
-    @api.convert()
-    def g(x):
-      if x > 0:
-        return x
-      else:
-        return -x
-
-    def f(g, x):
-      return g(x)
-
-    x = api.converted_call(f, None, converter.ConversionOptions(),
-                           g, constant_op.constant(1))
-    self.assertEqual(self.evaluate(x), 1)
-
   @test_util.run_deprecated_v1
   def test_converted_call_no_user_code(self):
 
@@ -433,24 +397,6 @@ class ApiTest(test.TestCase):
       self.evaluate(variables.global_variables_initializer())
       self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
 
-  def test_converted_call_namedtuple(self):
-
-    opts = converter.ConversionOptions()
-
-    x = api.converted_call(collections.namedtuple, None, opts,
-                           'TestNamedtuple', ('a', 'b'))
-
-    self.assertTrue(inspect_utils.isnamedtuple(x))
-
-  def test_converted_call_namedtuple_via_collections(self):
-
-    opts = converter.ConversionOptions()
-
-    x = api.converted_call('namedtuple', collections, opts,
-                           'TestNamedtuple', ('a', 'b'))
-
-    self.assertTrue(inspect_utils.isnamedtuple(x))
-
   def test_converted_call_lambda(self):
 
     opts = converter.ConversionOptions()
-- 
GitLab


From c0a94ff10f9d598c02bbc57dce216094f8323183 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 25 Jan 2019 09:08:11 -0800
Subject: [PATCH 1254/2345] [TF:XLA] Fix tf2xla's gather_op error message

Previously, tf2xla would print a modified input when returning an InvalidArgument Error. This corrects the error message to print the correct value and to match TensorFlow Classic's message.

PiperOrigin-RevId: 230912390
---
 tensorflow/compiler/tf2xla/kernels/gather_op.cc  | 8 ++++----
 tensorflow/python/kernel_tests/gather_op_test.py | 2 --
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/gather_op.cc b/tensorflow/compiler/tf2xla/kernels/gather_op.cc
index 41c31d0ed5..6472045265 100644
--- a/tensorflow/compiler/tf2xla/kernels/gather_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/gather_op.cc
@@ -167,13 +167,13 @@ class GatherOp : public XlaOpKernel {
 
       OP_REQUIRES_OK(context, context->ConstantInputAsIntScalar(2, &axis));
       const auto params_dims = input_shape.dims();
-      if (axis < 0) {
-        axis += params_dims;
-      }
       OP_REQUIRES(
-          context, 0 <= axis && axis < params_dims,
+          context, -params_dims <= axis && axis < params_dims,
           errors::InvalidArgument("Expected axis in the range [", -params_dims,
                                   ", ", params_dims, "), but got ", axis));
+      if (axis < 0) {
+        axis += params_dims;
+      }
     }
 
     DataType index_type = input_type(1);
diff --git a/tensorflow/python/kernel_tests/gather_op_test.py b/tensorflow/python/kernel_tests/gather_op_test.py
index 3b8a9a38de..1d6f7955aa 100644
--- a/tensorflow/python/kernel_tests/gather_op_test.py
+++ b/tensorflow/python/kernel_tests/gather_op_test.py
@@ -213,8 +213,6 @@ class GatherTest(test.TestCase, parameterized.TestCase):
         array_ops.gather(params, [[7]], axis=1).eval()
 
   @test_util.run_deprecated_v1
-  @test_util.disable_xla(
-      "This test never passed for XLA")  # Different error message.
   def testBadAxis(self):
     with self.session(use_gpu=True):
       params = [0, 1, 2]
-- 
GitLab


From fe1ddb6fb69ec24ff60b272a022bb6380c02a935 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 25 Jan 2019 09:14:12 -0800
Subject: [PATCH 1255/2345] Speed up the construction of simple tensor shapes
 (1..4 number of dimensions with limited sizes).

Added a benchmark for shape construction.  The impact
of this CL on the benchmark:

name                   old time/op  new time/op  delta
BM_TensorShape_Init/0  9.85ns +- 1%  4.87ns +- 0%  -50.53%  (p=0.008 n=5+5)
BM_TensorShape_Init/1  17.2ns +- 2%   6.5ns +- 1%  -62.10%  (p=0.008 n=5+5)
BM_TensorShape_Init/2  24.2ns +- 1%  26.0ns +- 0%   +7.44%  (p=0.016 n=5+4)
BM_TensorShape_Init/3  31.9ns +- 0%   9.2ns +- 1%  -71.07%  (p=0.016 n=4+5)
BM_TensorShape_Init/4   131ns +- 4%   129ns +- 0%     ~     (p=0.095 n=5+4)

PiperOrigin-RevId: 230913222
---
 tensorflow/core/framework/tensor_shape.cc     | 83 +++++++++++++++++++
 tensorflow/core/framework/tensor_shape.h      |  1 +
 .../core/framework/tensor_shape_test.cc       |  9 ++
 3 files changed, 93 insertions(+)

diff --git a/tensorflow/core/framework/tensor_shape.cc b/tensorflow/core/framework/tensor_shape.cc
index be451d1e31..7158f1925f 100644
--- a/tensorflow/core/framework/tensor_shape.cc
+++ b/tensorflow/core/framework/tensor_shape.cc
@@ -136,6 +136,89 @@ template <class Shape>
 TensorShapeBase<Shape>::TensorShapeBase(gtl::ArraySlice<int64> dim_sizes) {
   set_tag(REP16);
   set_data_type(DT_INVALID);
+  InitDims(dim_sizes);
+}
+
+// Returns true iff partial is true and val is < 0.
+// REQUIRES: val < kMaxRep16
+// REQUIRES: partial || val >= 0
+static inline bool Set16(bool partial, uint16* dst, int dim, int64 val) {
+  if (partial) {
+    if (val < 0) {
+      dst[dim] = std::numeric_limits<uint16>::max();
+      return true;
+    }
+  } else {
+    CHECK_GE(val, 0);
+  }
+  dst[dim] = val;
+  return false;
+}
+
+template <class Shape>
+void TensorShapeBase<Shape>::InitDims(gtl::ArraySlice<int64> dim_sizes) {
+  DCHECK_EQ(tag(), REP16);
+
+  // Allow sizes that are under kint64max^0.25 so that 4-way multiplication
+  // below cannot overflow.
+  static const uint64 kMaxSmall = 0xd744;
+  static_assert(kMaxSmall * kMaxSmall * kMaxSmall * kMaxSmall <= kint64max,
+                "bad overflow check");
+  bool large_size = false;
+  for (auto s : dim_sizes) {
+    if (s > kMaxSmall) {
+      large_size = true;
+      break;
+    }
+  }
+
+  if (!large_size) {
+    // Every size fits in 16 bits; use fast-paths for dims in {1,2,3,4}.
+    uint16* dst = as16()->dims_;
+    switch (dim_sizes.size()) {
+      case 1: {
+        set_ndims_byte(1);
+        const int64 size = dim_sizes[0];
+        const bool neg = Set16(kIsPartial, dst, 0, size);
+        set_num_elements(neg ? -1 : size);
+        return;
+      }
+      case 2: {
+        set_ndims_byte(2);
+        const int64 size0 = dim_sizes[0];
+        const int64 size1 = dim_sizes[1];
+        bool neg = Set16(kIsPartial, dst, 0, size0);
+        neg |= Set16(kIsPartial, dst, 1, size1);
+        set_num_elements(neg ? -1 : (size0 * size1));
+        return;
+      }
+      case 3: {
+        set_ndims_byte(3);
+        const int64 size0 = dim_sizes[0];
+        const int64 size1 = dim_sizes[1];
+        const int64 size2 = dim_sizes[2];
+        bool neg = Set16(kIsPartial, dst, 0, size0);
+        neg |= Set16(kIsPartial, dst, 1, size1);
+        neg |= Set16(kIsPartial, dst, 2, size2);
+        set_num_elements(neg ? -1 : (size0 * size1 * size2));
+        return;
+      }
+      case 4: {
+        set_ndims_byte(4);
+        const int64 size0 = dim_sizes[0];
+        const int64 size1 = dim_sizes[1];
+        const int64 size2 = dim_sizes[2];
+        const int64 size3 = dim_sizes[3];
+        bool neg = Set16(kIsPartial, dst, 0, size0);
+        neg |= Set16(kIsPartial, dst, 1, size1);
+        neg |= Set16(kIsPartial, dst, 2, size2);
+        neg |= Set16(kIsPartial, dst, 3, size3);
+        set_num_elements(neg ? -1 : (size0 * size1 * size2 * size3));
+        return;
+      }
+    }
+  }
+
   set_ndims_byte(0);
   set_num_elements(1);
   for (int64 s : dim_sizes) {
diff --git a/tensorflow/core/framework/tensor_shape.h b/tensorflow/core/framework/tensor_shape.h
index 625d88ec1b..3473a441f2 100644
--- a/tensorflow/core/framework/tensor_shape.h
+++ b/tensorflow/core/framework/tensor_shape.h
@@ -256,6 +256,7 @@ class TensorShapeBase : public TensorShapeRep {
 
  private:
   void RecomputeNumElements();
+  void InitDims(gtl::ArraySlice<int64> dim_sizes);
 
   // True for PartialTensorShape, false for TensorShape
   static constexpr bool kIsPartial =
diff --git a/tensorflow/core/framework/tensor_shape_test.cc b/tensorflow/core/framework/tensor_shape_test.cc
index 6329aa6d8e..d25652ce81 100644
--- a/tensorflow/core/framework/tensor_shape_test.cc
+++ b/tensorflow/core/framework/tensor_shape_test.cc
@@ -684,6 +684,15 @@ static std::vector<int64> MakeSizes(int arg) {
   return sizes;
 }
 
+static void BM_TensorShape_Init(int iters, int arg) {
+  auto sizes = MakeSizes(arg);
+  while (--iters > 0) {
+    TensorShape shape(sizes);
+    tensorflow::testing::DoNotOptimize(shape.num_elements());
+  }
+}
+BENCHMARK(BM_TensorShape_Init)->Arg(0)->Arg(1)->Arg(2)->Arg(3)->Arg(4);
+
 static void BM_TensorShape_Assign(int iters, int arg) {
   TensorShape s(MakeSizes(arg));
   while (--iters > 0) {
-- 
GitLab


From 52d1c2c358896f5f8a001aee8983d9e01b5fd19d Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Fri, 25 Jan 2019 09:33:49 -0800
Subject: [PATCH 1256/2345] Support variant<TensorShape, xla::Shape> in
 XlaArgument

When compiling if we already know the XLA shape upfront then, we should
avoid converting to TensorShape only to later be converted to XLA shape.
This allows us to handle unsupported types such as tuples.

PiperOrigin-RevId: 230916187
---
 .../compiler/jit/xla_compilation_cache.cc     | 17 +++--
 .../compiler/jit/xla_compilation_cache.h      |  4 +-
 tensorflow/compiler/tf2xla/BUILD              |  2 +
 tensorflow/compiler/tf2xla/kernels/if_op.cc   |  4 +-
 .../compiler/tf2xla/kernels/while_op.cc       | 11 ++-
 tensorflow/compiler/tf2xla/tf2xla.cc          |  4 +-
 tensorflow/compiler/tf2xla/xla_compiler.cc    | 75 +++++++++++++++----
 tensorflow/compiler/tf2xla/xla_compiler.h     | 12 ++-
 8 files changed, 99 insertions(+), 30 deletions(-)

diff --git a/tensorflow/compiler/jit/xla_compilation_cache.cc b/tensorflow/compiler/jit/xla_compilation_cache.cc
index bff4cc57ee..611515cf33 100644
--- a/tensorflow/compiler/jit/xla_compilation_cache.cc
+++ b/tensorflow/compiler/jit/xla_compilation_cache.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include <numeric>
 
 #include "absl/strings/str_cat.h"
+#include "absl/strings/str_join.h"
 #include "tensorflow/compiler/tf2xla/dump_graph.h"
 #include "tensorflow/compiler/tf2xla/shape_util.h"
 #include "tensorflow/compiler/tf2xla/type_util.h"
@@ -70,9 +71,9 @@ string XlaCompilationCache::DebugString() const {
 // arguments in the supplied list.
 string XlaCompilationCache::Signature::HumanString() const {
   string result = name;
-  for (const auto& a : arg_types) {
-    absl::StrAppend(&result, ",", DataTypeString(a.first),
-                    a.second.DebugString());
+  for (const auto& a : arg_shapes) {
+    absl::StrAppend(&result, ",", DataTypeString(a.first));
+    absl::StrAppend(&result, " [", absl::StrJoin(a.second, ","), "]");
   }
 
   for (const auto& v : arg_values) {
@@ -83,7 +84,7 @@ string XlaCompilationCache::Signature::HumanString() const {
 
 bool XlaCompilationCache::Signature::operator==(const Signature& other) const {
   if (name != other.name) return false;
-  if (arg_types != other.arg_types) return false;
+  if (arg_shapes != other.arg_shapes) return false;
 
   if (arg_values.size() != other.arg_values.size()) return false;
   for (int i = 0; i < arg_values.size(); ++i) {
@@ -99,10 +100,10 @@ bool XlaCompilationCache::Signature::operator==(const Signature& other) const {
 uint64 XlaCompilationCache::Signature::Hash::operator()(
     const XlaCompilationCache::Signature& signature) const {
   uint64 h = std::hash<string>()(signature.name);
-  for (const auto& arg : signature.arg_types) {
+  for (const auto& arg : signature.arg_shapes) {
     h = Hash64Combine(h, std::hash<int>()(static_cast<int>(arg.first)));
-    h = Hash64Combine(h, std::hash<int>()(arg.second.dims()));
-    for (int dim : arg.second.dim_sizes()) {
+    h = Hash64Combine(h, std::hash<int>()(arg.second.size()));
+    for (int dim : arg.second) {
       h = Hash64Combine(h, std::hash<int>()(dim));
     }
   }
@@ -126,7 +127,7 @@ XlaCompilationCache::BuildSignature(
         break;
       case XlaCompiler::Argument::kParameter:
       case XlaCompiler::Argument::kResource:
-        signature.arg_types.emplace_back(arg.type, arg.shape);
+        signature.arg_shapes.emplace_back(arg.type, arg.DimensionSizes());
         break;
       default:
         return errors::InvalidArgument(
diff --git a/tensorflow/compiler/jit/xla_compilation_cache.h b/tensorflow/compiler/jit/xla_compilation_cache.h
index 02aa8f8839..7748b4700f 100644
--- a/tensorflow/compiler/jit/xla_compilation_cache.h
+++ b/tensorflow/compiler/jit/xla_compilation_cache.h
@@ -95,7 +95,9 @@ class XlaCompilationCache : public ResourceBase {
   struct Signature {
     string name;
 
-    std::vector<std::pair<DataType, TensorShape>> arg_types;
+    // List of Tensor types & shapes for compile-time constant arguments to the
+    // compilation, ordered by argument number.
+    std::vector<std::pair<DataType, std::vector<int64>>> arg_shapes;
 
     // List of Tensor values for compile-time constant arguments to the
     // compilation, ordered by argument number. Tensors must be in host memory.
diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD
index 0366ec45fb..92ba474fbc 100644
--- a/tensorflow/compiler/tf2xla/BUILD
+++ b/tensorflow/compiler/tf2xla/BUILD
@@ -204,6 +204,7 @@ cc_library(
         "//tensorflow/compiler/xla:status_macros",
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:types",
+        "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client",
         "//tensorflow/compiler/xla/client:client_library",
@@ -224,6 +225,7 @@ cc_library(
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/types:optional",
         "@com_google_absl//absl/types:span",
+        "@com_google_absl//absl/types:variant",
     ],
     alwayslink = 1,
 )
diff --git a/tensorflow/compiler/tf2xla/kernels/if_op.cc b/tensorflow/compiler/tf2xla/kernels/if_op.cc
index 954ae0b596..aa5637e266 100644
--- a/tensorflow/compiler/tf2xla/kernels/if_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/if_op.cc
@@ -80,7 +80,7 @@ void XlaIfOp::Compile(XlaOpKernelContext* ctx) {
       arg.name = resource->name();
       VLOG(2) << "Resource " << resource->name()
               << " type: " << DataTypeString(arg.type)
-              << " shape: " << arg.shape.DebugString()
+              << " shape: " << arg.HumanString()
               << " initialized: " << arg.initialized;
 
       num_resource_args++;
@@ -89,7 +89,7 @@ void XlaIfOp::Compile(XlaOpKernelContext* ctx) {
       arg.type = input_types_[i];
       arg.shape = ctx->InputShape(i + 1);
       VLOG(2) << "Arg type: " << DataTypeString(arg.type)
-              << " shape: " << arg.shape.DebugString();
+              << " shape: " << arg.HumanString();
     }
   }
 
diff --git a/tensorflow/compiler/tf2xla/kernels/while_op.cc b/tensorflow/compiler/tf2xla/kernels/while_op.cc
index 941b04363f..fd5ff10ae0 100644
--- a/tensorflow/compiler/tf2xla/kernels/while_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/while_op.cc
@@ -70,13 +70,20 @@ Status MakeXlaCompilerArgumentsFromInputs(
       arg.name = resource->name();
       VLOG(2) << "    resource " << resource->name()
               << " type: " << DataTypeString(arg.type)
-              << " shape: " << arg.shape.DebugString()
+              << " shape: " << arg.ShapeHumanString()
               << " initialized: " << arg.initialized;
 
     } else {
       arg.kind = XlaCompiler::Argument::kParameter;
       arg.type = ctx->input_type(i);
-      arg.shape = ctx->InputShape(i);
+
+      xla::XlaBuilder* builder = ctx->builder();
+      xla::XlaOp handle = ctx->Input(i);
+      auto shape_or_status = builder->GetShape(handle);
+      if (!shape_or_status.ok()) {
+        return shape_or_status.status();
+      }
+      arg.shape = shape_or_status.ValueOrDie();
     }
   }
   return Status::OK();
diff --git a/tensorflow/compiler/tf2xla/tf2xla.cc b/tensorflow/compiler/tf2xla/tf2xla.cc
index 9fac16a970..cf48576ec2 100644
--- a/tensorflow/compiler/tf2xla/tf2xla.cc
+++ b/tensorflow/compiler/tf2xla/tf2xla.cc
@@ -243,7 +243,9 @@ Status CreateXlaArgs(const Graph& graph,
     XlaCompiler::Argument arg;
     arg.kind = XlaCompiler::Argument::kParameter;
     TF_RETURN_IF_ERROR(GetNodeAttr(node->attrs(), "T", &arg.type));
-    TF_RETURN_IF_ERROR(GetNodeAttr(node->attrs(), kShapeAttr, &arg.shape));
+    TensorShape shape;
+    TF_RETURN_IF_ERROR(GetNodeAttr(node->attrs(), kShapeAttr, &shape));
+    arg.shape = shape;
     TF_RETURN_IF_ERROR(GetNodeAttr(node->attrs(), kDebugNameAttr, &arg.name));
     xla_args->push_back(arg);
   }
diff --git a/tensorflow/compiler/tf2xla/xla_compiler.cc b/tensorflow/compiler/tf2xla/xla_compiler.cc
index 373b9ae1b4..f79f955460 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler.cc
@@ -31,6 +31,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/client_library.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/client/xla_computation.h"
+#include "tensorflow/compiler/xla/util.h"
 #include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/common_runtime/executor.h"
 #include "tensorflow/core/common_runtime/function.h"
@@ -333,8 +334,21 @@ bool XlaCompiler::Argument::operator==(
                other.tensor_array_gradients)) {
     return false;
   }
-  if (shape != other.shape) {
-    return false;
+  if (absl::holds_alternative<xla::Shape>(shape)) {
+    if (!absl::holds_alternative<xla::Shape>(other.shape)) {
+      return false;
+    }
+    if (!xla::Shape::Equal()(absl::get<xla::Shape>(shape),
+                             absl::get<xla::Shape>(other.shape))) {
+      return false;
+    }
+  } else {
+    if (!absl::holds_alternative<TensorShape>(other.shape)) {
+      return false;
+    }
+    if (absl::get<TensorShape>(shape) != absl::get<TensorShape>(other.shape)) {
+      return false;
+    }
   }
   if (constant_value.shape() != other.constant_value.shape()) {
     return false;
@@ -348,7 +362,7 @@ string XlaCompiler::Argument::HumanString() const {
     common = absl::StrCat(" name=", name);
   }
   absl::StrAppend(&common, " type=", DataTypeString(type),
-                  " shape=", shape.DebugString());
+                  " shape=", ShapeHumanString());
   switch (kind) {
     case kInvalid:
       return "invalid";
@@ -375,6 +389,23 @@ string XlaCompiler::Argument::HumanString() const {
   }
 }
 
+std::vector<int64> XlaCompiler::Argument::DimensionSizes() const {
+  if (absl::holds_alternative<TensorShape>(shape)) {
+    return xla::InlinedVectorToVector(
+        absl::get<TensorShape>(shape).dim_sizes());
+  } else {
+    return absl::get<xla::Shape>(shape).dimensions();
+  }
+}
+
+string XlaCompiler::Argument::ShapeHumanString() const {
+  if (absl::holds_alternative<TensorShape>(shape)) {
+    return absl::get<TensorShape>(shape).DebugString();
+  } else {
+    return absl::get<xla::Shape>(shape).DebugString();
+  }
+}
+
 XlaCompiler::XlaCompiler(XlaCompiler::Options options)
     : options_(options),
       initialization_status_(Status::OK()),
@@ -574,11 +605,22 @@ Status XlaCompiler::XLAShapeForArgument(const XlaCompiler::Argument& arg,
       LOG(FATAL) << "Unreachable case";
     case XlaCompiler::Argument::kParameter: {
       if (is_entry_computation) {
-        TF_ASSIGN_OR_RETURN(
-            *xla_shape, options_.shape_representation_fn(arg.shape, arg.type));
+        TensorShape shape;
+        if (absl::holds_alternative<TensorShape>(arg.shape)) {
+          shape = absl::get<TensorShape>(arg.shape);
+        } else {
+          TF_RETURN_IF_ERROR(
+              XLAShapeToTensorShape(absl::get<xla::Shape>(arg.shape), &shape));
+        }
+        TF_ASSIGN_OR_RETURN(*xla_shape,
+                            options_.shape_representation_fn(shape, arg.type));
       } else {
-        TF_RETURN_IF_ERROR(
-            TensorShapeToXLAShape(arg.type, arg.shape, xla_shape));
+        if (absl::holds_alternative<xla::Shape>(arg.shape)) {
+          *xla_shape = absl::get<xla::Shape>(arg.shape);
+        } else {
+          TF_RETURN_IF_ERROR(TensorShapeToXLAShape(
+              arg.type, absl::get<TensorShape>(arg.shape), xla_shape));
+        }
       }
       return Status::OK();
     }
@@ -587,8 +629,10 @@ Status XlaCompiler::XLAShapeForArgument(const XlaCompiler::Argument& arg,
 
       switch (arg.resource_kind) {
         case XlaResource::kVariable: {
-          TF_ASSIGN_OR_RETURN(*xla_shape, options_.shape_representation_fn(
-                                              arg.shape, arg.type));
+          TF_RET_CHECK(absl::holds_alternative<TensorShape>(arg.shape));
+          TF_ASSIGN_OR_RETURN(*xla_shape,
+                              options_.shape_representation_fn(
+                                  absl::get<TensorShape>(arg.shape), arg.type));
 
           return Status::OK();
         }
@@ -597,9 +641,10 @@ Status XlaCompiler::XLAShapeForArgument(const XlaCompiler::Argument& arg,
             return errors::InvalidArgument(
                 "Negative max_array_size in XLAShapeForArgument");
           }
+          TF_RET_CHECK(absl::holds_alternative<TensorShape>(arg.shape));
           TensorShape shape;
           shape.AddDim(arg.max_array_size);
-          shape.AppendShape(arg.shape);
+          shape.AppendShape(absl::get<TensorShape>(arg.shape));
           TF_RETURN_IF_ERROR(TensorShapeToXLAShape(arg.type, shape, xla_shape));
 
           if (!arg.tensor_array_gradients.empty()) {
@@ -614,9 +659,10 @@ Status XlaCompiler::XLAShapeForArgument(const XlaCompiler::Argument& arg,
             return errors::InvalidArgument(
                 "Negative max_array_size in XLAShapeForArgument");
           }
+          TF_RET_CHECK(absl::holds_alternative<TensorShape>(arg.shape));
           TensorShape shape;
           shape.AddDim(arg.max_array_size);
-          shape.AppendShape(arg.shape);
+          shape.AppendShape(absl::get<TensorShape>(arg.shape));
           xla::Shape buffer_shape;
           TF_RETURN_IF_ERROR(
               TensorShapeToXLAShape(arg.type, shape, &buffer_shape));
@@ -664,12 +710,13 @@ Status XlaCompiler::BuildArguments(
     switch (arg.kind) {
       case XlaCompiler::Argument::kResource: {
         TF_RET_CHECK(arg.resource_kind != XlaResource::kInvalid);
+        TF_RET_CHECK(absl::holds_alternative<TensorShape>(arg.shape));
         // TODO(phawkins): this code assumes that resource arguments do not
         // alias.
         XlaResource* resource =
             context->AddResource(absl::make_unique<XlaResource>(
-                arg.resource_kind, i, arg.name, arg.type, arg.shape,
-                xla::XlaOp(),
+                arg.resource_kind, i, arg.name, arg.type,
+                absl::get<TensorShape>(arg.shape), xla::XlaOp(),
                 /*max_array_size=*/arg.max_array_size,
                 /*tensor_array_gradients=*/arg.tensor_array_gradients,
                 /*tensor_array_multiple_writes_aggregate=*/true));
@@ -814,7 +861,7 @@ Status XlaCompiler::BuildArguments(
         // return values of functions, and then reshape unconditionally.
         if (is_entry_computation) {
           arg_expression = XlaExpression::XlaOp(
-              xla::Reshape(arg_handles[i], arg.shape.dim_sizes()), arg.type);
+              xla::Reshape(arg_handles[i], arg.DimensionSizes()), arg.type);
         } else {
           arg_expression = XlaExpression::XlaOp(arg_handles[i], arg.type);
         }
diff --git a/tensorflow/compiler/tf2xla/xla_compiler.h b/tensorflow/compiler/tf2xla/xla_compiler.h
index 2d4cb0bd99..ad3144b41b 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.h
+++ b/tensorflow/compiler/tf2xla/xla_compiler.h
@@ -19,6 +19,7 @@ limitations under the License.
 #include <stack>
 
 #include "absl/types/span.h"
+#include "absl/types/variant.h"
 #include "tensorflow/compiler/tf2xla/host_compute_metadata.pb.h"
 #include "tensorflow/compiler/tf2xla/xla_compilation_device.h"
 #include "tensorflow/compiler/tf2xla/xla_expression.h"
@@ -124,7 +125,8 @@ class XlaCompiler {
     DataType type = DT_INVALID;
 
     // The shape of the argument. For:
-    // * a parameter: the shape of the parameter.
+    // * a parameter: the shape of the parameter. We allow setting the xla shape
+    //   if known. This helps avoid conversions to and from TensorShape.
     // * a constant: ignored; the shape given by constant_value is used
     //     instead.
     // * an uninitialized resource: ignored. We don't yet know the shape of an
@@ -133,7 +135,7 @@ class XlaCompiler {
     // * an initialized TensorArray or Stack resource: the shape of an entry in
     //   the TensorArray/Stack. Note this is the size of a single entry, not the
     //   XLA data structure that represents the complete stack/array.
-    TensorShape shape;
+    absl::variant<TensorShape, xla::Shape> shape;
 
     // The value of the argument, if it is a compile-time constant. Must be a
     // host-memory tensor.
@@ -165,6 +167,12 @@ class XlaCompiler {
 
     // Returns a human-readable summary of the argument.
     string HumanString() const;
+
+    // Returns the dimension sizes for either TensorShape or xla::Shape.
+    std::vector<int64> DimensionSizes() const;
+
+    // Returns the human-readable string for either TensorShape or xla::Shape.
+    string ShapeHumanString() const;
   };
 
   // Options pertaining to an individual call to CompileGraph() or
-- 
GitLab


From 4cc8976ed8aec119752dffda278ce1b909e6657c Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Fri, 25 Jan 2019 09:48:05 -0800
Subject: [PATCH 1257/2345] Some built-ins are classes but their constructor
 lacks a __name__ property and so it should not have a FQN annotation.

PiperOrigin-RevId: 230918513
---
 .../autograph/pyct/static_analysis/type_info.py       |  4 +++-
 .../autograph/pyct/static_analysis/type_info_test.py  | 11 +++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/autograph/pyct/static_analysis/type_info.py b/tensorflow/python/autograph/pyct/static_analysis/type_info.py
index 370e180838..68a53661d3 100644
--- a/tensorflow/python/autograph/pyct/static_analysis/type_info.py
+++ b/tensorflow/python/autograph/pyct/static_analysis/type_info.py
@@ -45,6 +45,7 @@ import gast
 
 from tensorflow.python.autograph.pyct import anno
 from tensorflow.python.autograph.pyct import ast_util
+from tensorflow.python.autograph.pyct import inspect_utils
 from tensorflow.python.autograph.pyct import transformer
 from tensorflow.python.util import tf_inspect
 
@@ -178,7 +179,8 @@ class TypeInfoResolver(transformer.Base):
       func = value.func
       if anno.hasanno(func, 'live_val'):
         func_obj = anno.getanno(func, 'live_val')
-        if tf_inspect.isclass(func_obj):
+        if (tf_inspect.isclass(func_obj) and
+            not inspect_utils.isbuiltin(func_obj)):
           anno.setanno(value, 'is_constructor', True)
           anno.setanno(value, 'type', func_obj)
           anno.setanno(value, 'type_fqn', anno.getanno(func, 'fqn'))
diff --git a/tensorflow/python/autograph/pyct/static_analysis/type_info_test.py b/tensorflow/python/autograph/pyct/static_analysis/type_info_test.py
index e43812e6f1..c6cf91e062 100644
--- a/tensorflow/python/autograph/pyct/static_analysis/type_info_test.py
+++ b/tensorflow/python/autograph/pyct/static_analysis/type_info_test.py
@@ -89,11 +89,22 @@ class TypeInfoResolverTest(test.TestCase):
 
     node = self._parse_and_analyze(test_fn, {'training': training})
     call_node = node.body[0].body[0].value
+    self.assertTrue(anno.getanno(call_node, 'is_constructor'))
     self.assertEquals(training.GradientDescentOptimizer,
                       anno.getanno(call_node, 'type'))
     self.assertEquals((training.__name__, 'GradientDescentOptimizer'),
                       anno.getanno(call_node, 'type_fqn'))
 
+  def test_constructor_detection_builtin_class(self):
+
+    def test_fn(x):
+      res = zip(x)
+      return res
+
+    node = self._parse_and_analyze(test_fn, {})
+    call_node = node.body[0].body[0].value
+    self.assertFalse(anno.hasanno(call_node, 'is_constructor'))
+
   def test_class_members_of_detected_constructor(self):
 
     def test_fn():
-- 
GitLab


From 29c1188731984b8d325f587cbfe0e0271366d007 Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Fri, 25 Jan 2019 09:53:28 -0800
Subject: [PATCH 1258/2345] grpc-worker-service: Add logging for failed method
 calls. PiperOrigin-RevId: 230919340

---
 .../rpc/grpc_worker_service.cc                | 165 +++++++-----------
 1 file changed, 65 insertions(+), 100 deletions(-)

diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc
index db0ad6124c..34bf629c8a 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc
@@ -175,59 +175,41 @@ class GrpcWorkerServiceThread {
       Call<GrpcWorkerServiceThread, grpc::WorkerService::AsyncService,
            RequestMessage, ResponseMessage>;
 
-  void GetStatusHandler(WorkerCall<GetStatusRequest, GetStatusResponse>* call) {
-    Schedule([this, call]() {
-      Status s = worker_->GetStatus(&call->request, &call->response);
-      call->SendResponse(ToGrpcStatus(s));
-    });
-    ENQUEUE_REQUEST(GetStatus, false);
-  }
-
-  void CreateWorkerSessionHandler(
-      WorkerCall<CreateWorkerSessionRequest, CreateWorkerSessionResponse>*
-          call) {
-    Schedule([this, call]() {
-      Status s = worker_->CreateWorkerSession(&call->request, &call->response);
-      call->SendResponse(ToGrpcStatus(s));
-    });
-    ENQUEUE_REQUEST(CreateWorkerSession, false);
+  // Handle all non-cancellable simple methods with a standard wrapper.
+#define HANDLE_CALL(method)                                                   \
+  void method##Handler(WorkerCall<method##Request, method##Response>* call) { \
+    Schedule([this, call]() {                                                 \
+      Status s = worker_->method(&call->request, &call->response);            \
+      if (!s.ok()) {                                                          \
+        VLOG(1) << "Bad response from " << #method << ": " << s;              \
+      }                                                                       \
+      call->SendResponse(ToGrpcStatus(s));                                    \
+    });                                                                       \
+    ENQUEUE_REQUEST(method, false);                                           \
   }
 
-  void DeleteWorkerSessionHandler(
-      WorkerCall<DeleteWorkerSessionRequest, DeleteWorkerSessionResponse>*
-          call) {
-    Schedule([this, call]() {
-      Status s = worker_->DeleteWorkerSession(&call->request, &call->response);
-      call->SendResponse(ToGrpcStatus(s));
-    });
-    ENQUEUE_REQUEST(DeleteWorkerSession, false);
-  }
-
-  void CleanupAllHandler(
-      WorkerCall<CleanupAllRequest, CleanupAllResponse>* call) {
-    Schedule([this, call]() {
-      Status s = worker_->CleanupAll(&call->request, &call->response);
-      call->SendResponse(ToGrpcStatus(s));
-    });
-    ENQUEUE_REQUEST(CleanupAll, false);
-  }
+  HANDLE_CALL(GetStatus);
+  HANDLE_CALL(CreateWorkerSession);
+  HANDLE_CALL(DeleteWorkerSession);
+  HANDLE_CALL(CleanupAll);
+  HANDLE_CALL(RegisterGraph);
+  HANDLE_CALL(DeregisterGraph);
+  HANDLE_CALL(CleanupGraph);
+  HANDLE_CALL(Logging);
+  HANDLE_CALL(Tracing);
 
-  void RegisterGraphHandler(
-      WorkerCall<RegisterGraphRequest, RegisterGraphResponse>* call) {
-    Schedule([this, call]() {
-      Status s = worker_->RegisterGraph(&call->request, &call->response);
-      call->SendResponse(ToGrpcStatus(s));
-    });
-    ENQUEUE_REQUEST(RegisterGraph, false);
-  }
+#undef HANDLE_CALL
 
-  void DeregisterGraphHandler(
-      WorkerCall<DeregisterGraphRequest, DeregisterGraphResponse>* call) {
+  void GetStepSequenceHandler(
+      WorkerCall<GetStepSequenceRequest, GetStepSequenceResponse>* call) {
     Schedule([this, call]() {
-      Status s = worker_->DeregisterGraph(&call->request, &call->response);
-      call->SendResponse(ToGrpcStatus(s));
+      worker_->GetStepSequenceAsync(
+          &call->request, &call->response, [call](const Status& s) {
+            VLOG(1) << "Bad response from GetStepSequence:" << s;
+            call->SendResponse(ToGrpcStatus(s));
+          });
     });
-    ENQUEUE_REQUEST(DeregisterGraph, false);
+    ENQUEUE_REQUEST(GetStepSequence, true);
   }
 
   void RunGraphHandler(WorkerCall<RunGraphRequest, RunGraphResponse>* call) {
@@ -241,6 +223,9 @@ class GrpcWorkerServiceThread {
       worker_->RunGraphAsync(call_opts, wrapped_request, wrapped_response,
                              [call, call_opts, wrapped_request,
                               wrapped_response](const Status& s) {
+                               if (!s.ok()) {
+                                 VLOG(1) << "Bad response from RunGraph:" << s;
+                               }
                                call->ClearCancelCallback();
                                delete call_opts;
                                delete wrapped_request;
@@ -256,41 +241,20 @@ class GrpcWorkerServiceThread {
     Schedule([this, call]() {
       CallOptions* call_opts = new CallOptions;
       call->SetCancelCallback([call_opts]() { call_opts->StartCancel(); });
-      worker_->GrpcRecvTensorAsync(call_opts, &call->request, &call->response,
-                                   [call, call_opts](const Status& s) {
-                                     call->ClearCancelCallback();
-                                     delete call_opts;
-                                     call->SendResponse(ToGrpcStatus(s));
-                                   });
+      worker_->GrpcRecvTensorAsync(
+          call_opts, &call->request, &call->response,
+          [call, call_opts](const Status& s) {
+            call->ClearCancelCallback();
+            delete call_opts;
+            if (!s.ok()) {
+              VLOG(1) << "Bad response from RecvTensor:" << s;
+            }
+            call->SendResponse(ToGrpcStatus(s));
+          });
     });
     EnqueueRecvTensorRequestRaw();
   }
 
-  void CleanupGraphHandler(
-      WorkerCall<CleanupGraphRequest, CleanupGraphResponse>* call) {
-    Schedule([this, call]() {
-      Status s = worker_->CleanupGraph(&call->request, &call->response);
-      call->SendResponse(ToGrpcStatus(s));
-    });
-    ENQUEUE_REQUEST(CleanupGraph, false);
-  }
-
-  void LoggingHandler(WorkerCall<LoggingRequest, LoggingResponse>* call) {
-    Schedule([this, call]() {
-      Status s = worker_->Logging(&call->request, &call->response);
-      call->SendResponse(ToGrpcStatus(s));
-    });
-    ENQUEUE_REQUEST(Logging, false);
-  }
-
-  void TracingHandler(WorkerCall<TracingRequest, TracingResponse>* call) {
-    Schedule([this, call]() {
-      Status s = worker_->Tracing(&call->request, &call->response);
-      call->SendResponse(ToGrpcStatus(s));
-    });
-    ENQUEUE_REQUEST(Tracing, false);
-  }
-
   void RecvBufHandler(WorkerCall<RecvBufRequest, RecvBufResponse>* call) {
     Schedule([this, call]() {
       CallOptions* call_opts = new CallOptions;
@@ -299,6 +263,9 @@ class GrpcWorkerServiceThread {
                             [call, call_opts](const Status& s) {
                               call->ClearCancelCallback();
                               delete call_opts;
+                              if (!s.ok()) {
+                                VLOG(1) << "Bad response from RecvBuf:" << s;
+                              }
                               call->SendResponse(ToGrpcStatus(s));
                             });
     });
@@ -310,12 +277,16 @@ class GrpcWorkerServiceThread {
     Schedule([this, call]() {
       CallOptions* call_opts = new CallOptions;
       call->SetCancelCallback([call_opts]() { call_opts->StartCancel(); });
-      worker_->CompleteGroupAsync(call_opts, &call->request, &call->response,
-                                  [call, call_opts](const Status& s) {
-                                    call->ClearCancelCallback();
-                                    delete call_opts;
-                                    call->SendResponse(ToGrpcStatus(s));
-                                  });
+      worker_->CompleteGroupAsync(
+          call_opts, &call->request, &call->response,
+          [call, call_opts](const Status& s) {
+            call->ClearCancelCallback();
+            delete call_opts;
+            if (!s.ok()) {
+              VLOG(1) << "Bad response from CompleteGroup:" << s;
+            }
+            call->SendResponse(ToGrpcStatus(s));
+          });
     });
     ENQUEUE_REQUEST(CompleteGroup, true);
   }
@@ -325,25 +296,19 @@ class GrpcWorkerServiceThread {
     Schedule([this, call]() {
       CallOptions* call_opts = new CallOptions;
       call->SetCancelCallback([call_opts]() { call_opts->StartCancel(); });
-      worker_->CompleteInstanceAsync(call_opts, &call->request, &call->response,
-                                     [call, call_opts](const Status& s) {
-                                       call->ClearCancelCallback();
-                                       delete call_opts;
-                                       call->SendResponse(ToGrpcStatus(s));
-                                     });
+      worker_->CompleteInstanceAsync(
+          call_opts, &call->request, &call->response,
+          [call, call_opts](const Status& s) {
+            call->ClearCancelCallback();
+            delete call_opts;
+            if (!s.ok()) {
+              VLOG(1) << "Bad response from CompleteInstance:" << s;
+            }
+            call->SendResponse(ToGrpcStatus(s));
+          });
     });
     ENQUEUE_REQUEST(CompleteInstance, false);
   }
-
-  void GetStepSequenceHandler(
-      WorkerCall<GetStepSequenceRequest, GetStepSequenceResponse>* call) {
-    Schedule([this, call]() {
-      worker_->GetStepSequenceAsync(
-          &call->request, &call->response,
-          [call](const Status& s) { call->SendResponse(ToGrpcStatus(s)); });
-    });
-    ENQUEUE_REQUEST(GetStepSequence, true);
-  }
 #undef ENQUEUE_REQUEST
 
   void EnqueueRecvTensorRequestRaw() {
-- 
GitLab


From a0b45dff743097e488aa7fd146f7acef3c3b3a8b Mon Sep 17 00:00:00 2001
From: Matej Rizman <matejr@google.com>
Date: Fri, 25 Jan 2019 10:36:01 -0800
Subject: [PATCH 1259/2345] [minor] Add whitespace to reduction kernel internal
 error message.

PiperOrigin-RevId: 230927218
---
 tensorflow/core/kernels/reduction_gpu_kernels.cu.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/reduction_gpu_kernels.cu.h b/tensorflow/core/kernels/reduction_gpu_kernels.cu.h
index e9cf36c62b..ffa41ece49 100644
--- a/tensorflow/core/kernels/reduction_gpu_kernels.cu.h
+++ b/tensorflow/core/kernels/reduction_gpu_kernels.cu.h
@@ -539,7 +539,7 @@ void LaunchScalarReduction(OpKernelContext* ctx, OUT_T out, IN_T in,
 
     OP_REQUIRES(
         ctx, success == 0,
-        errors::Internal("CUB reduce error", cudaGetErrorString(success)));
+        errors::Internal("CUB reduce error ", cudaGetErrorString(success)));
   };
 
   reduce(nullptr);  // Get required amount of temp storage.
-- 
GitLab


From 461b0857ae2e0a3129324ffc0264732110e007b0 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Fri, 25 Jan 2019 10:40:30 -0800
Subject: [PATCH 1260/2345] Improve the error message raised when the source
 code is not found. Related issue: #24759

PiperOrigin-RevId: 230928092
---
 tensorflow/python/autograph/pyct/parser.py | 32 +++++++++++++++-------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/tensorflow/python/autograph/pyct/parser.py b/tensorflow/python/autograph/pyct/parser.py
index d04a40157e..a14966e470 100644
--- a/tensorflow/python/autograph/pyct/parser.py
+++ b/tensorflow/python/autograph/pyct/parser.py
@@ -31,9 +31,18 @@ from tensorflow.python.util import tf_inspect
 
 def parse_entity(entity):
   """Returns the AST of given entity."""
-  source = tf_inspect.getsource(entity)
-
-  def fail(comment):
+  try:
+    source = tf_inspect.getsource(entity)
+  except (IOError, OSError) as e:
+    raise ValueError(
+        'Unable to locate the source code of {}. Note that functions defined'
+        ' in certain environments, like the interactive Python shell do not'
+        ' expose their source code. If that is the case, you should to define'
+        ' them in a .py source file. If you are certain the code is'
+        ' graph-compatible, wrap the call using'
+        ' @tf.autograph.do_not_convert. Original error: {}'.format(entity, e))
+
+  def raise_parse_failure(comment):
     raise ValueError(
         'Failed to parse source code of {}, which Python reported as:\n{}\n'
         '{}'.format(entity, source, comment))
@@ -49,8 +58,9 @@ def parse_entity(entity):
   except IndentationError:
     # The text below lists the causes of this error known to us. There may
     # be more.
-    fail('This may be caused by multiline strings or comments not indented at'
-         'the same level as the code.')
+    raise_parse_failure(
+        'This may be caused by multiline strings or comments not indented at'
+        ' the same level as the code.')
 
   except SyntaxError as e:
     if not tf_inspect.isfunction(entity) or entity.__name__ != '<lambda>':
@@ -71,8 +81,9 @@ def parse_entity(entity):
 
     # Give up if there's nothing we can chip away.
     if len(lines) == lineno and len(lines[-1]) == offset:
-      fail('If this is a lambda function, the error may be avoided by creating'
-           ' the lambda in a standalone statement.')
+      raise_parse_failure(
+          'If this is a lambda function, the error may be avoided by creating'
+          ' the lambda in a standalone statement.')
 
     # Drop all lines following the error location
     # TODO(mdan): What's with the pylint errors?
@@ -84,9 +95,10 @@ def parse_entity(entity):
     try:
       return parse_str(new_source), new_source
     except SyntaxError as e:
-      fail('If this is a lambda function, the error may be avoided by creating'
-           ' the lambda in a standalone statement. Tried to strip down the'
-           ' source to:\n{}\nBut that did not work.'.format(new_source))
+      raise_parse_failure(
+          'If this is a lambda function, the error may be avoided by creating'
+          ' the lambda in a standalone statement. Tried to strip down the'
+          ' source to:\n{}\nBut that did not work.'.format(new_source))
 
 
 def parse_str(src):
-- 
GitLab


From b3ee362daa85e01c7646ed3913845393d6eb43b3 Mon Sep 17 00:00:00 2001
From: Yanan Cao <ycao@google.com>
Date: Fri, 25 Jan 2019 10:40:35 -0800
Subject: [PATCH 1261/2345] Internal change

PiperOrigin-RevId: 230928112
---
 tensorflow/python/kernel_tests/distributions/BUILD | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/kernel_tests/distributions/BUILD b/tensorflow/python/kernel_tests/distributions/BUILD
index 7a10ed9852..22c98201dd 100644
--- a/tensorflow/python/kernel_tests/distributions/BUILD
+++ b/tensorflow/python/kernel_tests/distributions/BUILD
@@ -230,9 +230,10 @@ cuda_py_test(
         "//tensorflow/python:platform_test",
     ],
     tags = [
-        # TODO(b/121223043): Re-enable this test on mac after fixing "mean not
-        # defined" errors.
+        # TODO(b/121223043): Re-enable this test after fixing "mean not defined"
+        # errors.
         "no_mac",
+        "no_oss",
         # disable to avoid false positives from scipy.
         "nomsan",
     ],
-- 
GitLab


From 81df31594971f6870bf5b7d75e311de3e970e670 Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Fri, 25 Jan 2019 10:42:18 -0800
Subject: [PATCH 1262/2345] Make `_keras_history` a namedtuple

PiperOrigin-RevId: 230928482
---
 tensorflow/python/keras/engine/base_layer.py  | 36 ++++++++++++++++---
 tensorflow/python/keras/engine/input_layer.py |  2 +-
 tensorflow/python/keras/engine/network.py     |  4 +--
 tensorflow/python/keras/models.py             |  8 ++---
 4 files changed, 38 insertions(+), 12 deletions(-)

diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index 5eb2c5a752..6218dbeed5 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import collections
 import functools
 import inspect  # Necessary supplement to tf_inspect to deal with variadic args.
 import itertools
@@ -302,7 +303,7 @@ class Layer(checkpointable.Checkpointable):
       if kwarg not in ['getter', 'collections']:
         raise TypeError('Unknown keyword argument:', kwarg)
     getter = kwargs.pop('getter', None)
-    collections = kwargs.pop('collections', None)
+    collections = kwargs.pop('collections', None)  # pylint: disable=redefined-outer-name
 
     if dtype is None:
       dtype = self.dtype or backend.floatx()
@@ -1492,11 +1493,11 @@ class Layer(checkpointable.Checkpointable):
         arguments: dictionary of keyword arguments that were passed to the
             `call` method of the layer at the call that created the node.
     """
-    inbound_layers = nest.map_structure(lambda t: t._keras_history[0],
+    inbound_layers = nest.map_structure(lambda t: t._keras_history.layer,
                                         input_tensors)
-    node_indices = nest.map_structure(lambda t: t._keras_history[1],
+    node_indices = nest.map_structure(lambda t: t._keras_history.node_index,
                                       input_tensors)
-    tensor_indices = nest.map_structure(lambda t: t._keras_history[2],
+    tensor_indices = nest.map_structure(lambda t: t._keras_history.tensor_index,
                                         input_tensors)
 
     # Create node, add it to inbound nodes.
@@ -1518,7 +1519,8 @@ class Layer(checkpointable.Checkpointable):
     # or multi-input layers (e.g. a layer can return multiple tensors,
     # and each can be sent to a different layer).
     for i, tensor in enumerate(nest.flatten(output_tensors)):
-      tensor._keras_history = (self, len(self._inbound_nodes) - 1, i)  # pylint: disable=protected-access
+      tensor._keras_history = KerasHistory(self,
+                                           len(self._inbound_nodes) - 1, i)  # pylint: disable=protected-access
 
   def _get_node_attribute_at_index(self, node_index, attr, attr_name):
     """Private utility to retrieves an attribute (e.g. inputs) from a node.
@@ -1827,6 +1829,30 @@ class TensorFlowOpLayer(Layer):
     return config
 
 
+class KerasHistory(
+    collections.namedtuple('KerasHistory',
+                           ['layer', 'node_index', 'tensor_index'])):
+  """Tracks the Layer calls that created a Tensor, for Keras Graph Networks.
+
+  During construction of Keras Graph Networks, this metadata is added to
+  each Tensor produced as the output of a Layer, starting with an
+  `InputLayer`. This allows Keras to track how each Tensor was produced, and
+  this information is later retraced by the `keras.engine.Network` class to
+  reconstruct the Keras Graph Network.
+
+  Attributes:
+    layer: The Layer that produced the Tensor.
+    node_index: The specific call to the Layer that produced this Tensor. Layers
+      can be called multiple times in order to share weights. A new node is
+      created every time a Tensor is called.
+    tensor_index: The output index for this Tensor. Always zero if the Layer
+      that produced this Tensor only has one output.
+  """
+  # Added to maintain memory and performance characteristics of `namedtuple`
+  # while subclassing.
+  __slots__ = ()
+
+
 def default(method):
   """Decorates a method to detect overrides in subclasses."""
   method._is_default = True
diff --git a/tensorflow/python/keras/engine/input_layer.py b/tensorflow/python/keras/engine/input_layer.py
index c6dcedfce2..0cc976154e 100644
--- a/tensorflow/python/keras/engine/input_layer.py
+++ b/tensorflow/python/keras/engine/input_layer.py
@@ -122,7 +122,7 @@ class InputLayer(base_layer.Layer):
 
     # Create an input node to add to self.outbound_node
     # and set output_tensors' _keras_history.
-    input_tensor._keras_history = (self, 0, 0)  # pylint: disable=protected-access
+    input_tensor._keras_history = base_layer.KerasHistory(self, 0, 0)  # pylint: disable=protected-access
     base_layer.Node(
         self,
         inbound_layers=[],
diff --git a/tensorflow/python/keras/engine/network.py b/tensorflow/python/keras/engine/network.py
index 41f5f319bc..f3d20e4dc4 100644
--- a/tensorflow/python/keras/engine/network.py
+++ b/tensorflow/python/keras/engine/network.py
@@ -1650,7 +1650,7 @@ class Network(base_layer.Layer):
                          ' (missing previous layer metadata).')
       # Check that x is an input tensor.
       # pylint: disable=protected-access
-      layer, _, _ = x._keras_history
+      layer = x._keras_history.layer
       if len(layer._inbound_nodes) > 1 or (
           layer._inbound_nodes and layer._inbound_nodes[0].inbound_layers):
         cls_name = self.__class__.__name__
@@ -1667,7 +1667,7 @@ class Network(base_layer.Layer):
 
     # Check compatibility of batch sizes of Input Layers.
     input_batch_sizes = [
-        training_utils.get_static_batch_size(x._keras_history[0])
+        training_utils.get_static_batch_size(x._keras_history.layer)
         for x in self.inputs
     ]
     consistent_batch_size = None
diff --git a/tensorflow/python/keras/models.py b/tensorflow/python/keras/models.py
index 9bc5aa2be5..cfae752431 100644
--- a/tensorflow/python/keras/models.py
+++ b/tensorflow/python/keras/models.py
@@ -98,7 +98,7 @@ def _clone_functional_model(model, input_tensors=None, share_weights=False):
           name=layer.name)
       input_tensors.append(input_tensor)
       # Cache newly created input layer.
-      newly_created_input_layer = input_tensor._keras_history[0]
+      newly_created_input_layer = input_tensor._keras_history.layer
       layer_map[layer] = newly_created_input_layer
 
     for original_input_layer, cloned_input_layer in zip(model._input_layers,
@@ -119,7 +119,7 @@ def _clone_functional_model(model, input_tensors=None, share_weights=False):
 
         input_tensors_.append(input_tensor)
         # Cache newly created input layer.
-        newly_created_input_layer = input_tensor._keras_history[0]
+        newly_created_input_layer = input_tensor._keras_history.layer
         layer_map[original_input_layer] = newly_created_input_layer
       else:
         input_tensors_.append(input_tensor)
@@ -239,7 +239,7 @@ def _clone_sequential_model(model, input_tensors=None, share_weights=False):
       input_tensors = list(input_tensors)
     x = generic_utils.to_list(input_tensors)[0]
     if K.is_keras_tensor(x):
-      origin_layer = x._keras_history[0]
+      origin_layer = x._keras_history.layer
       if isinstance(origin_layer, InputLayer):
         return Sequential(layers=[origin_layer] + layers, name=model.name)
       else:
@@ -248,7 +248,7 @@ def _clone_sequential_model(model, input_tensors=None, share_weights=False):
                          'other than an `InputLayer`. '
                          'Use the functional API instead.')
     input_tensor = Input(tensor=x, name='input_wrapper_for_' + str(x.name))
-    input_layer = input_tensor._keras_history[0]
+    input_layer = input_tensor._keras_history.layer
     return Sequential(layers=[input_layer] + layers, name=model.name)
 
 
-- 
GitLab


From 5f0709aec53f3c7d90368c6b6cf78d24a8fa5a7d Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Fri, 25 Jan 2019 10:42:56 -0800
Subject: [PATCH 1263/2345] Add kTensorList kind to better support TensorLists

Rather than overloading TensorLists as kXlaOp, we add kTensorList kind.
This allows us to better specialize the behaviour of TensorLists such
as the implementation of GetShape().

PiperOrigin-RevId: 230928607
---
 tensorflow/compiler/tf2xla/graph_compiler.cc  |  3 +++
 .../tf2xla/kernels/tensor_list_ops.cc         | 24 ++++++++++---------
 tensorflow/compiler/tf2xla/xla_compiler.cc    |  2 ++
 tensorflow/compiler/tf2xla/xla_expression.cc  | 17 +++++++++++++
 tensorflow/compiler/tf2xla/xla_expression.h   | 12 +++++++++-
 tensorflow/compiler/tf2xla/xla_op_kernel.cc   | 10 ++++++--
 tensorflow/compiler/tf2xla/xla_op_kernel.h    |  3 +++
 7 files changed, 57 insertions(+), 14 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/graph_compiler.cc b/tensorflow/compiler/tf2xla/graph_compiler.cc
index 0c2bb02239..5e4699bbb6 100644
--- a/tensorflow/compiler/tf2xla/graph_compiler.cc
+++ b/tensorflow/compiler/tf2xla/graph_compiler.cc
@@ -89,6 +89,9 @@ Status PrepareArguments(XlaOpKernelContext* ctx, Graph* graph,
       case XlaExpression::Kind::kResource:
         return errors::Unimplemented(
             "Resource as function argument is not yet implemented.");
+      case XlaExpression::Kind::kTensorList:
+        return errors::Unimplemented(
+            "TensorList as function argument is not yet implemented.");
       case XlaExpression::Kind::kInvalid:
         return errors::InvalidArgument("Invalid function argument");
     }
diff --git a/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc b/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc
index 6757a21190..6a68e0b75e 100644
--- a/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc
@@ -67,9 +67,10 @@ class TensorListReserveOp : public XlaOpKernel {
     tensor_shape.AppendShape(element_shape);
 
     xla::XlaBuilder* b = ctx->builder();
-    ctx->SetOutput(0, xla::Tuple(b, {xla::Broadcast(XlaHelpers::Zero(b, dtype_),
-                                                    tensor_shape.dim_sizes()),
-                                     xla::ConstantR0<int32>(b, 0)}));
+    ctx->SetTensorListOutput(
+        0, xla::Tuple(b, {xla::Broadcast(XlaHelpers::Zero(b, dtype_),
+                                         tensor_shape.dim_sizes()),
+                          xla::ConstantR0<int32>(b, 0)}));
   }
 
  private:
@@ -104,9 +105,10 @@ class EmptyTensorListOp : public XlaOpKernel {
     tensor_shape.AppendShape(element_shape);
 
     xla::XlaBuilder* b = ctx->builder();
-    ctx->SetOutput(0, xla::Tuple(b, {xla::Broadcast(XlaHelpers::Zero(b, dtype_),
-                                                    tensor_shape.dim_sizes()),
-                                     xla::ConstantR0<int32>(b, 0)}));
+    ctx->SetTensorListOutput(
+        0, xla::Tuple(b, {xla::Broadcast(XlaHelpers::Zero(b, dtype_),
+                                         tensor_shape.dim_sizes()),
+                          xla::ConstantR0<int32>(b, 0)}));
   }
 
  private:
@@ -168,11 +170,11 @@ class TensorListPushBackOp : public XlaOpKernel {
 
   void Compile(XlaOpKernelContext* ctx) override {
     xla::XlaBuilder* b = ctx->builder();
-    xla::XlaOp list = ctx->Input(0);
+    xla::XlaOp tl = ctx->Input(0);
     TensorShape elem_shape = ctx->InputShape(1);
 
-    xla::XlaOp ta = xla::GetTupleElement(list, 0);
-    xla::XlaOp index = xla::GetTupleElement(list, 1);
+    xla::XlaOp ta = xla::GetTupleElement(tl, 0);
+    xla::XlaOp index = xla::GetTupleElement(tl, 1);
     xla::XlaOp value = ctx->Input(1);
 
     // start_indices of the DynamicUpdateSlice are [index, 0, 0, ..., 0].
@@ -186,7 +188,7 @@ class TensorListPushBackOp : public XlaOpKernel {
 
     // TODO(phawkins): We don't check the index is in bounds --- there is no
     // error mechanism in XLA.
-    ctx->SetOutput(
+    ctx->SetTensorListOutput(
         0, xla::Tuple(b, {xla::DynamicUpdateSlice(ta, update, start_indices),
                           index + xla::ConstantR0<int32>(b, 1)}));
   }
@@ -230,7 +232,7 @@ class TensorListPopBackOp : public XlaOpKernel {
     // Remove the leading '1' dimension.
     std::vector<int64> value_shape(slice_shape.begin() + 1, slice_shape.end());
 
-    ctx->SetOutput(0, xla::Tuple(b, {ta, index}));
+    ctx->SetTensorListOutput(0, xla::Tuple(b, {ta, index}));
     ctx->SetOutput(1, xla::Reshape(read, value_shape));
   }
 
diff --git a/tensorflow/compiler/tf2xla/xla_compiler.cc b/tensorflow/compiler/tf2xla/xla_compiler.cc
index f79f955460..1f9cfcdd24 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler.cc
@@ -193,6 +193,8 @@ Status BuildComputation(
         output.shape = output.constant_value.shape();
         break;
 
+      case XlaExpression::Kind::kTensorList:
+        TF_FALLTHROUGH_INTENDED;
       case XlaExpression::Kind::kXlaOp: {
         output.is_constant = false;
         TF_ASSIGN_OR_RETURN(output.shape, retval.GetShape());
diff --git a/tensorflow/compiler/tf2xla/xla_expression.cc b/tensorflow/compiler/tf2xla/xla_expression.cc
index ca0309166b..3d228c92ad 100644
--- a/tensorflow/compiler/tf2xla/xla_expression.cc
+++ b/tensorflow/compiler/tf2xla/xla_expression.cc
@@ -46,6 +46,14 @@ XlaExpression XlaExpression::XlaOp(xla::XlaOp value, DataType dtype) {
   return e;
 }
 
+XlaExpression XlaExpression::TensorList(xla::XlaOp tensor_list) {
+  XlaExpression e;
+  e.kind_ = Kind::kTensorList;
+  e.dtype_ = DT_VARIANT;
+  e.handle_ = tensor_list;
+  return e;
+}
+
 XlaExpression XlaExpression::Resource(XlaResource* resource) {
   XlaExpression e;
   e.kind_ = Kind::kResource;
@@ -64,6 +72,8 @@ string XlaExpression::HumanString() const {
       return "xla_op";
     case Kind::kResource:
       return "resource";
+    case Kind::kTensorList:
+      return "tensor_list";
   }
 }
 
@@ -76,6 +86,8 @@ xla::XlaOp XlaExpression::AsXlaOp(xla::XlaBuilder* builder) const {
             HostTensorToBorrowingLiteral(constant_value_, &literal));
         return xla::ConstantLiteral(builder, literal);
       }
+      case Kind::kTensorList:
+        TF_FALLTHROUGH_INTENDED;
       case Kind::kXlaOp:
         if (builder != handle_.builder()) {
           return errors::InvalidArgument(
@@ -96,7 +108,10 @@ xla::StatusOr<absl::optional<Tensor>> XlaExpression::ResolveConstant(
       return {constant_value()};
     case Kind::kXlaOp:
       break;
+    case Kind::kTensorList:
+      TF_FALLTHROUGH_INTENDED;
     case Kind::kResource:
+      TF_FALLTHROUGH_INTENDED;
     case Kind::kInvalid:
       return errors::InvalidArgument(
           "ResolveConstant called on XlaExpression: ", HumanString());
@@ -134,6 +149,8 @@ xla::StatusOr<TensorShape> XlaExpression::GetShape() const {
       TF_RETURN_IF_ERROR(XLAShapeToTensorShape(xla_shape, &shape));
       return shape;
     }
+    case Kind::kTensorList:
+      return TensorShape({});
     case Kind::kResource:
       return TensorShape({});
     case Kind::kInvalid:
diff --git a/tensorflow/compiler/tf2xla/xla_expression.h b/tensorflow/compiler/tf2xla/xla_expression.h
index bed6761d36..ac0232d892 100644
--- a/tensorflow/compiler/tf2xla/xla_expression.h
+++ b/tensorflow/compiler/tf2xla/xla_expression.h
@@ -32,11 +32,16 @@ namespace tensorflow {
 // * a constant tensor.
 // * an xla::XlaOp, representing a symbolic XLA value.
 // * a resource, e.g., a variable, represented as an XlaResource pointer.
+// * a tensor list, represented by a tuple of tensors and the list length.
 //
 // Constant tensors are mostly an optimization to avoid passing large constants
 // to XLA, but are also sometimes used to represent tensors that have no XLA
 // representation, for example, DT_STRING tensors. A canonical use case might be
 // an error message string.
+//
+// Tensor lists are very similar to xla::XlaOp, however they require some
+// specific logic around shape management since the tuples are not supported by
+// TensorFlow.
 class XlaExpression {
  public:
   enum class Kind {
@@ -44,6 +49,7 @@ class XlaExpression {
     kConstant,
     kXlaOp,
     kResource,
+    kTensorList,
   };
 
   XlaExpression();
@@ -62,6 +68,9 @@ class XlaExpression {
   // be derived from the XLA type.
   static XlaExpression XlaOp(xla::XlaOp value, DataType dtype);
 
+  // Builds a tensor list expression.
+  static XlaExpression TensorList(xla::XlaOp tensor_list);
+
   // Builds a resource expression.
   static XlaExpression Resource(XlaResource* resource);
 
@@ -100,7 +109,8 @@ class XlaExpression {
 
   DataType dtype_ = DT_INVALID;
 
-  // The XLA handle of the expression's computation, if kind_ == kXlaOp.
+  // The XLA handle of the expression's computation, if kind_ == kXlaOp or
+  // a tuple expression if kind_ == kTensorList.
   xla::XlaOp handle_;
 
   // The value of the constant, if kind_ == kConstant.
diff --git a/tensorflow/compiler/tf2xla/xla_op_kernel.cc b/tensorflow/compiler/tf2xla/xla_op_kernel.cc
index 58bd173e61..1147d0c726 100644
--- a/tensorflow/compiler/tf2xla/xla_op_kernel.cc
+++ b/tensorflow/compiler/tf2xla/xla_op_kernel.cc
@@ -93,7 +93,7 @@ TensorShape XlaOpKernelContext::InputShape(absl::string_view name) {
 }
 
 DataType XlaOpKernelContext::input_type(int index) const {
-  return context_->input(index).dtype();
+  return context_->input_dtype(index);
 }
 
 DataType XlaOpKernelContext::InputType(absl::string_view name) {
@@ -229,7 +229,8 @@ Status XlaOpKernelContext::ConstantInputAsFloatScalar(int index, double* out) {
 static Status LiteralToInt64Vector(const xla::LiteralSlice& literal,
                                    std::vector<int64>* out) {
   if (literal.shape().rank() != 1) {
-    return errors::InvalidArgument("value is not 1D");
+    return errors::InvalidArgument("value is not 1D, rank: ",
+                                   literal.shape().rank());
   }
   int64 size = xla::ShapeUtil::ElementsIn(literal.shape());
   if (literal.shape().element_type() == xla::S32) {
@@ -456,6 +457,11 @@ void XlaOpKernelContext::SetConstantOutput(int index, const Tensor& constant) {
   SetOutputExpression(index, XlaExpression::Constant(constant));
 }
 
+void XlaOpKernelContext::SetTensorListOutput(int index,
+                                             const xla::XlaOp& handle) {
+  SetOutputExpression(index, XlaExpression::TensorList(handle));
+}
+
 void XlaOpKernelContext::SetResourceOutput(int index, XlaResource* resource) {
   SetOutputExpression(index, XlaExpression::Resource(resource));
 }
diff --git a/tensorflow/compiler/tf2xla/xla_op_kernel.h b/tensorflow/compiler/tf2xla/xla_op_kernel.h
index 1858844bc0..e44415f60b 100644
--- a/tensorflow/compiler/tf2xla/xla_op_kernel.h
+++ b/tensorflow/compiler/tf2xla/xla_op_kernel.h
@@ -168,6 +168,9 @@ class XlaOpKernelContext {
   // Returns an XlaExpression describing the value of 'index'.
   void SetOutputExpression(int index, const XlaExpression& expression);
 
+  // Sets output `index` to the Tensor List `handle`.
+  void SetTensorListOutput(int index, const xla::XlaOp& handle);
+
   // Status handling.
   void SetStatus(const Status& status) { context_->SetStatus(status); }
   Status status() { return context_->status(); }
-- 
GitLab


From c6a4b8600cd7288e709b63fee1426cc34aeda6d8 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Fri, 25 Jan 2019 10:47:05 -0800
Subject: [PATCH 1264/2345] [Grappler] Lower If/While before inlining function
 body

PiperOrigin-RevId: 230929361
---
 tensorflow/core/grappler/optimizers/BUILD     |   1 +
 .../grappler/optimizers/function_optimizer.cc |  65 +++++-----
 .../optimizers/function_optimizer_test.cc     | 122 ++++++++++++++++++
 3 files changed, 157 insertions(+), 31 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index ee216f80f9..70e0f226fc 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -181,6 +181,7 @@ tf_cuda_cc_test(
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/utils:grappler_test",
+        "@com_google_absl//absl/algorithm:container",
     ],
 )
 
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index 6913f9b55b..c350e9f915 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -27,6 +27,8 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/device_mgr.h"
 #include "tensorflow/core/common_runtime/device_set.h"
 #include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/common_runtime/lower_if_while.h"
+#include "tensorflow/core/common_runtime/optimization_registry.h"
 #include "tensorflow/core/common_runtime/placer.h"
 #include "tensorflow/core/common_runtime/process_function_library_runtime.h"
 #include "tensorflow/core/framework/attr_value_util.h"
@@ -1380,20 +1382,6 @@ Status IsInlinableIndirectFunctionCall(const FunctionOptimizerContext& ctx,
         SummarizeNodeDef(func_node));
   }
 
-  // TODO(b/120991525, b/120986912): We need to lower `If` and `While` nodes to
-  // `Switch` nodes after function inlining (one more PRE_PLACEMENT pass?), but
-  // because of the reason described above we are not sure that it's safe, for
-  // now just disable inlining functions with functional control flow.
-  const auto is_functional_ctrl_flow_op = [](const NodeDef& node) {
-    return IsIf(node) || IsWhile(node);
-  };
-  if (absl::c_any_of(func.node_def(), is_functional_ctrl_flow_op)) {
-    return errors::FailedPrecondition(
-        "Can't inline function with `If` or `While` nodes in the function "
-        "body: ",
-        SummarizeNodeDef(func_node));
-  }
-
   return Status::OK();
 }
 
@@ -1404,7 +1392,8 @@ Status CheckThatSideEffectsWillExecute(
     const FunctionOptimizerContext& ctx,
     const GraphTopologyView& graph_topo_view,
     const absl::flat_hash_set<string> output_nodes) {
-  // We ignore side-effects safety check in aggressive mode.
+  // In aggressive mode we just print a warning for side-effectful nodes that
+  // might not be executed after inlining.
   const bool aggressive = ctx.opt_level() == RewriterConfig::AGGRESSIVE;
 
   for (const NodeDef& func_body_node : graph_topo_view.graph()->node()) {
@@ -1435,18 +1424,17 @@ Status CheckThatSideEffectsWillExecute(
     DfsTraversal(graph_topo_view, {&func_body_node},
                  TraversalDirection::kFollowOutputs, predicates, callbacks);
 
-    if (!will_execute && !aggressive) {
-      return errors::Internal(
+    if (!will_execute) {
+      const string error_message = absl::StrCat(
           "Can't guarantee execution of a side-effectful node, that is not "
           "reachable from function outputs. Function body node: ",
           SummarizeNodeDef(func_body_node));
-    }
 
-    if (!will_execute && aggressive) {
-      LOG(WARNING)
-          << "Can't guarantee execution of a side-effectful node, that is not "
-             "reachable from function outputs. Function body node: "
-          << SummarizeNodeDef(func_body_node);
+      if (aggressive) {
+        LOG(WARNING) << error_message;
+      } else {
+        return errors::Internal(error_message);
+      }
     }
   }
 
@@ -1454,19 +1442,33 @@ Status CheckThatSideEffectsWillExecute(
 }
 
 Status PlaceInlinedFunctionBody(
-    const FunctionOptimizerContext& ctx, const NodeDef& func_node,
-    const GrapplerFunctionItem& item,
+    const NodeDef& func_node, const GrapplerFunctionItem& item,
     const absl::flat_hash_map<absl::string_view, int>& input_placeholders_idx,
-    GraphDef* placed_graph_def) {
+    FunctionOptimizerContext* ctx, GraphDef* placed_graph_def) {
   // Control flow lowering and Placer works with a Graph object.
   std::unique_ptr<Graph> func_body_graph =
-      absl::make_unique<Graph>(ctx.function_library());
+      absl::make_unique<Graph>(ctx->function_library());
 
   GraphConstructorOptions opts;
   TF_RETURN_IF_ERROR(
       ConvertGraphDefToGraph(opts, item.graph, func_body_graph.get()));
 
-  // TODO(ezhulenev): Lower If/While ops.
+  // ------------------------------------------------------------------------ //
+  // Grappler receives the graph after PRE_PLACEMENT, Placer, and POST_PLACEMENT
+  // passes, so each node has a valid device assignment. Also V2 control
+  // flow ops (functional If and While) should have been lowered to V1 control
+  // flow (Switch and Merge nodes). To keep the graph valid for execution we
+  // must assign device to every inlined graph node, and also lower the control
+  // flow.
+
+  GraphOptimizationPassOptions opt_options;
+  opt_options.graph = &func_body_graph;
+  opt_options.flib_def = ctx->mutable_function_library();
+
+  // TODO(ezhulenev): Should we run full PRE_PLACEMENT pass here? And
+  // POST_PLACEMENT after placer?
+  LowerIfWhilePass pass;
+  TF_RETURN_IF_ERROR(pass.Run(opt_options));
 
   // ------------------------------------------------------------------------ //
   // Before placing the function body nodes we pin input placeholders to the
@@ -1479,7 +1481,7 @@ Status PlaceInlinedFunctionBody(
     if (input_placeholder_idx != input_placeholders_idx.end()) {
       const int input_idx = input_placeholder_idx->second;
       const GraphView::OutputPort output_port =
-          ctx.graph_view().GetRegularFanin({&func_node, input_idx});
+          ctx->graph_view().GetRegularFanin({&func_node, input_idx});
 
       VLOG(3) << "Pin inlined function input node '" << func_body_node->name()
               << "' to the '" << output_port.node->device() << "' device.";
@@ -1491,7 +1493,7 @@ Status PlaceInlinedFunctionBody(
   // After placing nodes corresponding to the function inputs, we need to assign
   // device placements to all other function body nodes.
 
-  const DeviceSet* devices = ctx.devices();
+  const DeviceSet* devices = ctx->devices();
 
   if (devices->devices().empty()) {
     // If there are no devices available for placer, we just put all nodes to
@@ -1533,6 +1535,7 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
                                   FunctionOptimizerContext* ctx,
                                   GraphDef* optimized_graph) {
   VLOG(2) << "Inline indirect function call: " << SummarizeNodeDef(func_node);
+  VLOG(4) << "Inlined function definition: " << DebugString(func);
   TF_RETURN_IF_ERROR(IsInlinableIndirectFunctionCall(*ctx, func, func_node));
 
   const AttrSlice func_instantiation_attr =
@@ -1695,7 +1698,7 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
 
   GraphDef placed_graph_def;
   TF_RETURN_IF_ERROR(PlaceInlinedFunctionBody(
-      *ctx, func_node, item, input_placeholders_idx, &placed_graph_def));
+      func_node, item, input_placeholders_idx, ctx, &placed_graph_def));
 
   // ------------------------------------------------------------------------ //
   // After all nodes placed we need to prepare them for inlining into the
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
index fc7fffe7b2..6b62dbaf2c 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
@@ -14,6 +14,8 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/grappler/optimizers/function_optimizer.h"
+
+#include "absl/algorithm/container.h"
 #include "tensorflow/cc/ops/functional_ops.h"
 #include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/core/framework/function_testlib.h"
@@ -1269,6 +1271,126 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithNestedFunctionCall) {
   test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
 }
 
+TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithFunctionalControlFlow) {
+  using test::function::NDef;
+  using FDH = FunctionDefHelper;
+
+  FunctionOptimizer optimizer(RewriterConfig::AGGRESSIVE);
+
+  FunctionDef add_func = FunctionDefHelper::Create(
+      "MyAdd", {"x:T", "y:T"}, {"z:T"}, {"T: {float, double}"},
+      {{{"add"}, "Add", {"x", "y"}, {{"T", "$T"}}}},
+      /* Mapping between function returns and function node outputs. */
+      {{"z", "add:z:0"}});
+
+  FunctionDef mul_func = FunctionDefHelper::Create(
+      "MyMul", {"x:T", "y:T"}, {"z:T"}, {"T: {float, double}"},
+      {{{"mul"}, "Mul", {"x", "y"}, {{"T", "$T"}}}},
+      /* Mapping between function returns and function node outputs. */
+      {{"z", "mul:z:0"}});
+
+  // Compute: return cond ? a + b : a * b
+  FunctionDef add_or_mul_func = FunctionDefHelper::Create(
+      "AddOrMul", {"cond:bool", "x:float", "y:float"}, {"z:float"}, {},
+      {
+          {{"if_node"},
+           "If",
+           {"cond", "x", "y"},
+           {
+               {"Tcond", DT_BOOL},
+               {"Tin", DataTypeSlice{DT_FLOAT, DT_FLOAT}},
+               {"Tout", DataTypeSlice{DT_FLOAT}},
+               {"then_branch", FDH::FunctionRef("MyAdd", {{"T", DT_FLOAT}})},
+               {"else_branch", FDH::FunctionRef("MyMul", {{"T", DT_FLOAT}})},
+               {"_lower_using_switch_merge", true},
+           }},
+      },
+      /* Mapping between function returns and function node outputs. */
+      {{"z", "if_node:output:0"}});
+
+  // Build a computation graph for:
+  //   is_add: bool
+  //   a: float
+  //   b: float
+  //   c = AddOrMul(is_add, a, b)  # is_add ? a + b : a * b
+  //   d = Identity(c)
+  //   return d
+
+  // c = MyMul(a, b)
+  GrapplerItem item;
+  item.fetch = {"d"};
+  item.graph = test::function::GDef(
+      {NDef("is_add", "Placeholder", {}, {{"dtype", DT_BOOL}}, kDevice),
+       NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("b", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+
+       NDef("c", "PartitionedCall", {"is_add", "a", "b"},
+            {{"Tin", DataTypeSlice{DT_BOOL, DT_FLOAT, DT_FLOAT}},
+             {"Tout", DataTypeSlice{DT_FLOAT}},
+             {"f", FDH::FunctionRef("AddOrMul")}},
+            kDevice),
+
+       NDef("d", "Identity", {"c"}, {{"T", DT_FLOAT}}, kDevice)},
+      // Function library.
+      {add_or_mul_func, add_func, mul_func});
+
+  GraphDef optimized_graph;
+  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &optimized_graph));
+
+  const auto count_nodes_with_op = [&](const string& op) {
+    return absl::c_count_if(optimized_graph.node(), [&](const NodeDef& node) {
+      return node.op() == op;
+    });
+  };
+
+  // All `PartitionedCall` nodes in the optimized graph must be inlined, and
+  // `If` node must be lowered to `Switch` and `Merge` nodes.
+  EXPECT_EQ(count_nodes_with_op("PartitionedCallOp"), 0);
+  EXPECT_EQ(count_nodes_with_op("If"), 0);
+  EXPECT_EQ(count_nodes_with_op("Switch"), 3);
+  EXPECT_EQ(count_nodes_with_op("Merge"), 1);
+
+  GrapplerItem optimized = item.WithGraph(std::move(optimized_graph));
+
+  Tensor one = test::AsScalar<float>(1.0);
+  Tensor two = test::AsScalar<float>(2.0);
+  Tensor three = test::AsScalar<float>(3.0);
+
+  const auto feed_args = [&](bool is_add) {
+    std::vector<std::pair<string, Tensor>> feed;
+    feed.emplace_back("a", one);
+    feed.emplace_back("b", two);
+    feed.emplace_back("is_add", test::AsScalar<bool>(is_add));
+    return feed;
+  };
+
+  {  // Check 'is_add == true': a + b
+    item.feed = feed_args(true);
+    optimized.feed = feed_args(true);
+
+    auto tensors_expected = EvaluateFetchNodes(item);
+    ASSERT_EQ(tensors_expected.size(), 1);
+    test::ExpectTensorEqual<float>(tensors_expected[0], three);
+
+    auto tensors = EvaluateFetchNodes(optimized);
+    ASSERT_EQ(tensors.size(), tensors_expected.size());
+    test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
+  }
+
+  {  // Check 'is_add == false': a * b
+    item.feed = feed_args(false);
+    optimized.feed = feed_args(false);
+
+    auto tensors_expected = EvaluateFetchNodes(item);
+    ASSERT_EQ(tensors_expected.size(), 1);
+    test::ExpectTensorEqual<float>(tensors_expected[0], two);
+
+    auto tensors = EvaluateFetchNodes(optimized);
+    ASSERT_EQ(tensors.size(), tensors_expected.size());
+    test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
+  }
+}
+
 TEST_F(FunctionOptimizerTest, SpecializeFunctionXTimesTwo) {
   using test::function::NDef;
 
-- 
GitLab


From 263314abd61fc3078dca026273fc903cc9bcec95 Mon Sep 17 00:00:00 2001
From: Suharsh Sivakumar <suharshs@google.com>
Date: Fri, 25 Jan 2019 10:52:53 -0800
Subject: [PATCH 1265/2345] Fix uninitialized TfLiteTensor/TfLiteQuantization
 MSAN issues.

PiperOrigin-RevId: 230930487
---
 tensorflow/lite/delegates/flex/buffer_map_test.cc | 2 ++
 tensorflow/lite/kernels/kernel_util_test.cc       | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/tensorflow/lite/delegates/flex/buffer_map_test.cc b/tensorflow/lite/delegates/flex/buffer_map_test.cc
index 9e8472f1e7..accaf30452 100644
--- a/tensorflow/lite/delegates/flex/buffer_map_test.cc
+++ b/tensorflow/lite/delegates/flex/buffer_map_test.cc
@@ -44,6 +44,7 @@ UniqueTfLiteTensor MakeLiteTensor(const std::vector<int>& shape,
   tensor->dims = ConvertVectorToTfLiteIntArray(shape);
   tensor->data.raw = nullptr;
   tensor->is_variable = false;
+  memset(&tensor->quantization, 0, sizeof(TfLiteQuantization));
   TfLiteTensorRealloc(data.size() * sizeof(T), tensor.get());
   memcpy(tensor->data.raw, data.data(), data.size() * sizeof(T));
   return tensor;
@@ -62,6 +63,7 @@ UniqueTfLiteTensor MakeLiteTensor<string>(const std::vector<int>& shape,
   tensor->dims = ConvertVectorToTfLiteIntArray(shape);
   tensor->data.raw = nullptr;
   tensor->is_variable = false;
+  memset(&tensor->quantization, 0, sizeof(TfLiteQuantization));
   TfLiteTensorRealloc(data.size() * sizeof(string), tensor.get());
 
   DynamicBuffer b;
diff --git a/tensorflow/lite/kernels/kernel_util_test.cc b/tensorflow/lite/kernels/kernel_util_test.cc
index 70eb183658..185ddfaa9d 100644
--- a/tensorflow/lite/kernels/kernel_util_test.cc
+++ b/tensorflow/lite/kernels/kernel_util_test.cc
@@ -28,6 +28,8 @@ class KernelUtilTest : public ::testing::Test {
   KernelUtilTest() {
     context_.ReportError = ReportError;
 
+    memset(&tensor1_, 0, sizeof(TfLiteTensor));
+    memset(&tensor2_, 0, sizeof(TfLiteTensor));
     tensor1_.dims = nullptr;
     tensor2_.dims = nullptr;
     tensor1_.allocation_type = kTfLiteMmapRo;
-- 
GitLab


From 755b142a1a3d903faa802ed7e468f06ed797a7fe Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Fri, 25 Jan 2019 11:00:32 -0800
Subject: [PATCH 1266/2345] Refactor bare ConcreteFunction support

It gets its own proto. This fixes keyword argument naming, which is determined in get_concrete_function rather than in the Function where the ConcreteFunction was originally created.

Also de-duplicates ConcreteFunction metadata in the proto format to fix loading a bare concrete function which is also a Function component.

PiperOrigin-RevId: 230932074
---
 .../saved_model/function_deserialization.py   | 49 +++++------
 .../saved_model/function_serialization.py     | 29 ++++---
 tensorflow/python/saved_model/load.py         | 84 ++++++++-----------
 tensorflow/python/saved_model/load_test.py    | 61 ++++++++++++++
 tensorflow/python/saved_model/save.py         | 49 ++++++-----
 .../saved_model/saved_object_graph.proto      | 23 +++--
 6 files changed, 187 insertions(+), 108 deletions(-)

diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py
index d1869ba0f7..3ad289e275 100644
--- a/tensorflow/python/saved_model/function_deserialization.py
+++ b/tensorflow/python/saved_model/function_deserialization.py
@@ -73,25 +73,22 @@ def _deserialize_function_spec(function_spec_proto, coder):
                                    kwargs_to_include, input_signature)
 
 
-def recreate_concrete_function(saved_concrete_function, concrete_functions):
-  """Recreates a user-facing concrete function."""
-  coder = nested_structure_coder.StructureCoder()
-
-  concrete_function = concrete_functions[saved_concrete_function.name]
-  input_signature = coder.decode_proto(
-      saved_concrete_function.canonicalized_input_signature)
-  input_signature_args, input_signature_kwargs = input_signature
-  if input_signature_kwargs:
-    raise ValueError("Restoring concrete function with non-empty kwargs (%s)." %
-                     input_signature_kwargs)
-
+# TODO(allenl): The fact that we can't derive ConcreteFunction calling
+# conventions from the serialized input spec right now is unfortunate. Merging
+# these would be good, maybe by adding TensorSpec names to cache keys so renamed
+# keyword arguments would yield different ConcreteFunctions.
+def setup_bare_concrete_function(saved_bare_concrete_function,
+                                 concrete_functions):
+  """Makes a restored bare concrete function callable."""
+  # Bare concrete functions accept only flat lists of Tensors with unique
+  # names.
+  concrete_function = concrete_functions[
+      saved_bare_concrete_function.concrete_function_name]
   # pylint: disable=protected-access
-  # Set metadata required for the concrete function to accept keyword and
-  # positional arguments in __call__. Normally this is set in
-  # get_concrete_function.
-  concrete_function._arg_keywords = [spec.name for spec in input_signature_args]
-  # TODO(allenl): Should we preserve the number of allowed positional arguments?
-  concrete_function._num_positional_args = len(input_signature_args)
+  concrete_function._arg_keywords = (
+      saved_bare_concrete_function.argument_keywords)
+  concrete_function._num_positional_args = (
+      saved_bare_concrete_function.allowed_positional_arguments)
   # pylint: enable=protected-access
   concrete_function.add_to_graph()
   return concrete_function
@@ -153,10 +150,10 @@ def recreate_function(saved_function, concrete_functions):
           (args, kwargs, e))
 
     debug_considered_signatures = []
-    for concrete_function in saved_function.concrete_function:
-      function_obj = concrete_functions[concrete_function.name]
-      canonicalized_original_inputs = coder.decode_proto(
-          concrete_function.canonicalized_input_signature)
+    for concrete_function_name in saved_function.concrete_functions:
+      function_obj = concrete_functions[concrete_function_name]
+      canonicalized_original_inputs = (
+          function_obj.graph.structured_input_signature)
       debug_considered_signatures.append(canonicalized_original_inputs)
 
       if _inputs_compatible(canonicalized_inputs,
@@ -170,10 +167,14 @@ def recreate_function(saved_function, concrete_functions):
         "Only existing signatures are %r."
         % (canonicalized_inputs, debug_considered_signatures))
 
-  cfs = [concrete_functions[f.name] for f in saved_function.concrete_function]
+  concrete_function_objects = []
+  for concrete_function_name in saved_function.concrete_functions:
+    concrete_function_objects.append(concrete_functions[concrete_function_name])
 
   return RestoredFunction(restored_function_body,
-                          restored_function_body.__name__, function_spec, cfs)
+                          restored_function_body.__name__,
+                          function_spec,
+                          concrete_function_objects)
 
 
 def load_function_def_library(library):
diff --git a/tensorflow/python/saved_model/function_serialization.py b/tensorflow/python/saved_model/function_serialization.py
index 172ab41dc8..8c65007bae 100644
--- a/tensorflow/python/saved_model/function_serialization.py
+++ b/tensorflow/python/saved_model/function_serialization.py
@@ -38,33 +38,43 @@ def _serialize_function_spec(function_spec, coder):
   return proto
 
 
-def serialize_concrete_function(concrete_function, node_ids):
+def serialize_concrete_function(concrete_function, node_ids, coder):
   """Build a SavedConcreteFunction."""
   bound_inputs = []
   try:
     for capture in concrete_function.captured_inputs:
       bound_inputs.append(node_ids[capture])
   except KeyError:
-    # TODO(andresp): Would it better to throw an exception?
+    # TODO(allenl): This warning shadows a real issue in test_table in
+    # save_test.py, where we don't handle captured constants. Fix that and
+    # then make this an exception.
     logging.warning(
         "Concrete function %s not added to object based saved model as it "
         "captures tensor %s which is unsupported or not reachable from root.",
         concrete_function.name, capture)
     return None
-  coder = nested_structure_coder.StructureCoder()
   concrete_function_proto = saved_object_graph_pb2.SavedConcreteFunction()
-  concrete_function_proto.name = concrete_function.name
-  concrete_function_proto.canonicalized_input_signature.CopyFrom(
-      coder.encode_structure(concrete_function.structured_input_signature))
   structured_outputs = func_graph_module.convert_structure_to_signature(
       concrete_function.structured_outputs)
+  concrete_function_proto.canonicalized_input_signature.CopyFrom(
+      coder.encode_structure(concrete_function.structured_input_signature))
   concrete_function_proto.output_signature.CopyFrom(
       coder.encode_structure(structured_outputs))
   concrete_function_proto.bound_inputs.extend(bound_inputs)
   return concrete_function_proto
 
 
-def serialize_function(function, node_ids):
+def serialize_bare_concrete_function(concrete_function):
+  """Build a SavedBareConcreteFunction."""
+  # pylint: disable=protected-access
+  return saved_object_graph_pb2.SavedBareConcreteFunction(
+      concrete_function_name=concrete_function.name,
+      allowed_positional_arguments=concrete_function._num_positional_args,
+      argument_keywords=concrete_function._arg_keywords)
+  # pylint: enable=protected-access
+
+
+def serialize_function(function):
   """Build a SavedFunction proto."""
   coder = nested_structure_coder.StructureCoder()
   proto = saved_object_graph_pb2.SavedFunction()
@@ -74,8 +84,5 @@ def serialize_function(function, node_ids):
   all_concrete_functions = \
       function._list_all_concrete_functions_for_serialization()  # pylint: disable=protected-access
   for concrete_function in all_concrete_functions:
-    concrete_function_proto = serialize_concrete_function(
-        concrete_function, node_ids)
-    if concrete_function_proto is not None:
-      proto.concrete_function.add().CopyFrom(concrete_function_proto)
+    proto.concrete_functions.append(concrete_function.name)
   return proto
diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
index 12b79d07e7..ad7a046ba4 100644
--- a/tensorflow/python/saved_model/load.py
+++ b/tensorflow/python/saved_model/load.py
@@ -52,61 +52,46 @@ class _Loader(object):
     self._setup_functions()
     self._restore_checkpoint()
 
-  def _setup_concrete_function(
-      self, concrete_function, seen_functions, coder):
+  def _setup_concrete_function(self, proto, concrete_function, coder):
     """Setup captured tensors and outputs for a single concrete function."""
-    name = concrete_function.name
     bound_inputs = [
         self._get_tensor_from_node(node_id)
-        for node_id in concrete_function.bound_inputs]
+        for node_id in proto.bound_inputs]
     bound_variables = [
         self._nodes[node_id]
-        for node_id in concrete_function.bound_inputs
+        for node_id in proto.bound_inputs
         if self._proto.nodes[node_id].WhichOneof("kind") == "variable"
     ]
-    if name in seen_functions:
-      raise RuntimeError(
-          "Concrete function with a duplicate name: %s." % name)
-    else:
-      seen_functions.add(name)
-      # TODO(andresp): This is only injecting the captured inputs into the
-      # concrete function, note that we did not modify the FuncGraph
-      # itself.
-      function = self._concrete_functions[name]
-      function._captured_inputs = bound_inputs  # pylint: disable=protected-access
-      function._func_graph.variables = bound_variables  # pylint: disable=protected-access
-      # By setting the structured_outputs directly, we can rely on this
-      # function_lib.ConcreteFunction object to perform the output repacking
-      # logic. The only limitation of that logic is that it only works
-      # with output that is convertible to Tensors and the conversion
-      # always happens. For example tf.TensorShape([2, 3]) will be
-      # converted to Tensor representing [2, 3].
-      original_outputs = coder.decode_proto(
-          concrete_function.output_signature)
-      # The original_outputs here had Tensors converted to TensorSpecs, so
-      # the restored function's structured_outputs field will not be
-      # exactly the same. Fortunately the repacking logic cares only about
-      # the structure.
-      # TODO(vbardiovsky): Should we just replicate the structures, with
-      # Nones instead of real objects?
-      function._func_graph.structured_outputs = original_outputs  # pylint: disable=protected-access
-      input_signature = coder.decode_proto(
-          concrete_function.canonicalized_input_signature)
-      function._func_graph.structured_input_signature = input_signature  # pylint: disable=protected-access
+    # TODO(andresp): This is only injecting the captured inputs into the
+    # concrete function, note that we did not modify the FuncGraph
+    # itself.
+    concrete_function._captured_inputs = bound_inputs  # pylint: disable=protected-access
+    concrete_function._func_graph.variables = bound_variables  # pylint: disable=protected-access
+    # By setting the structured_outputs directly, we can rely on this
+    # function_lib.ConcreteFunction object to perform the output repacking
+    # logic. The only limitation of that logic is that it only works
+    # with output that is convertible to Tensors and the conversion
+    # always happens. For example tf.TensorShape([2, 3]) will be
+    # converted to Tensor representing [2, 3].
+    original_outputs = coder.decode_proto(proto.output_signature)
+    # The original_outputs here had Tensors converted to TensorSpecs, so
+    # the restored function's structured_outputs field will not be
+    # exactly the same. Fortunately the repacking logic cares only about
+    # the structure.
+    # TODO(vbardiovsky): Should we just replicate the structures, with
+    # Nones instead of real objects?
+    concrete_function._func_graph.structured_outputs = original_outputs  # pylint: disable=protected-access
+    concrete_function._func_graph.structured_input_signature = (  # pylint: disable=protected-access
+        coder.decode_proto(proto.canonicalized_input_signature))
 
   def _setup_functions(self):
-
-    """Setup captures and output structure in restored concrete functions."""
-    seen_concrete_functions = set()
+    """Setup captures and output structure in restored functions."""
     coder = nested_structure_coder.StructureCoder()
-    for object_proto in self._proto.nodes:
-      if object_proto.WhichOneof("kind") == "concrete_function":
-        self._setup_concrete_function(
-            object_proto.concrete_function, seen_concrete_functions, coder)
-      elif object_proto.WhichOneof("kind") == "function":
-        for concrete_function in object_proto.function.concrete_function:
-          self._setup_concrete_function(
-              concrete_function, seen_concrete_functions, coder)
+    for name, concrete_function_proto in self._proto.concrete_functions.items():
+      self._setup_concrete_function(
+          concrete_function_proto,
+          self._concrete_functions[name],
+          coder)
 
   def _get_tensor_from_node(self, node_id):
     obj = self._nodes[node_id]
@@ -149,8 +134,9 @@ class _Loader(object):
         "user_object": lambda: self._recreate_user_object(proto.user_object),
         "asset": lambda: self._recreate_asset(proto.asset),
         "function": lambda: self._recreate_function(proto.function),
-        "concrete_function": functools.partial(
-            self._recreate_concrete_function, proto.concrete_function),
+        "bare_concrete_function": functools.partial(
+            self._recreate_bare_concrete_function,
+            proto.bare_concrete_function),
         "variable": lambda: self._recreate_variable(proto.variable),
     }
     kind = proto.WhichOneof("kind")
@@ -182,8 +168,8 @@ class _Loader(object):
     return function_deserialization.recreate_function(
         proto, self._concrete_functions), setattr
 
-  def _recreate_concrete_function(self, proto):
-    return function_deserialization.recreate_concrete_function(
+  def _recreate_bare_concrete_function(self, proto):
+    return function_deserialization.setup_bare_concrete_function(
         proto, self._concrete_functions), setattr
 
   def _recreate_variable(self, proto):
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index 20d943a7c3..197c9d9865 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -625,6 +625,67 @@ class LoadTest(test.TestCase, parameterized.TestCase):
     self.assertEqual(8., imported.f(y=constant_op.constant(3.),
                                     x=constant_op.constant(2.)).numpy())
 
+  def test_revived_concrete_function_tensorspec_kwargs(self, cycles):
+
+    @def_function.function
+    def func(*args):
+      x, y = args
+      return x * (y + 1.)
+    root = tracking.AutoCheckpointable()
+    root.f = func.get_concrete_function(
+        tensor_spec.TensorSpec([], dtypes.float32, name="x"),
+        tensor_spec.TensorSpec([], dtypes.float32, name="y"))
+    self.assertEqual(8., root.f(y=constant_op.constant(3.),
+                                x=constant_op.constant(2.)).numpy())
+    imported = self.cycle(root, cycles, signatures={})
+    self.assertEqual(8., imported.f(y=constant_op.constant(3.),
+                                    x=constant_op.constant(2.)).numpy())
+
+  def test_concrete_function_variable_argument(self, cycles):
+    # TODO(allenl): Fix variables in input signatures.
+    self.skipTest("Need to fix encoding of variables in inputs signatures")
+    capture = variables.Variable(0)
+
+    @def_function.function
+    def func(v):
+      v.assign_add(1)
+      capture.assign_sub(1)
+
+    vsave = variables.Variable(1)
+    root = tracking.AutoCheckpointable()
+    root.f = func.get_concrete_function(vsave)
+    root.capture = capture
+    self.assertEqual(1, vsave.numpy())
+    root.f(vsave)
+    self.assertEqual(2, vsave.numpy())
+    self.assertEqual(-1, capture.numpy())
+    imported = self.cycle(root, cycles)
+
+    vload = variables.Variable(1)
+    imported.f(vload)
+    self.assertEqual(2, vload.numpy())
+    imported.f(v=vload)
+    self.assertEqual(3, vload.numpy())
+    self.assertEqual(-3, imported.capture.numpy())
+    self.assertEqual(-1, capture.numpy())
+
+  def test_function_and_component(self, cycles):
+
+    @def_function.function
+    def func(v):
+      return v + 1
+
+    root = tracking.AutoCheckpointable()
+    root.func = func
+    root.concrete_func = func.get_concrete_function(
+        tensor_spec.TensorSpec(None, dtypes.int32))
+    one = constant_op.constant(1)
+    self.assertEqual(2, root.func(one).numpy())
+    self.assertEqual(2, root.concrete_func(one).numpy())
+    imported = self.cycle(root, cycles)
+    self.assertEqual(2, imported.func(one).numpy())
+    self.assertEqual(2, imported.concrete_func(one).numpy())
+
   def test_dict(self, cycles):
     root = tracking.AutoCheckpointable()
     root.variables = dict(a=variables.Variable(1.))
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index 339f5587f3..94030cf8d6 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -39,6 +39,7 @@ from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.saved_model import builder_impl
 from tensorflow.python.saved_model import constants
 from tensorflow.python.saved_model import function_serialization
+from tensorflow.python.saved_model import nested_structure_coder
 from tensorflow.python.saved_model import revived_types
 from tensorflow.python.saved_model import saved_object_graph_pb2
 from tensorflow.python.saved_model import signature_constants
@@ -71,9 +72,11 @@ class _SaveableView(object):
     self.node_ids = node_ids
     self.slot_variables = slot_variables
     self.functions = util.ObjectIdentityDictionary()
+    self.concrete_functions = []
 
     # Also add `Function`s as nodes.
     nodes_without_functions = list(self.nodes)
+    seen_function_names = set()
     for obj in nodes_without_functions:
       self.functions[obj] = self._list_functions(obj)
       for function in self.functions[obj].values():
@@ -90,7 +93,14 @@ class _SaveableView(object):
           #  and have not been called.
           #  - force side effects of creation of concrete functions, e.g. create
           #  variables on first run.
-          function._list_all_concrete_functions_for_serialization()  # pylint: disable=protected-access
+          concrete_functions = (
+              function._list_all_concrete_functions_for_serialization())  # pylint: disable=protected-access
+        else:
+          concrete_functions = [function]
+        for concrete_function in concrete_functions:
+          if concrete_function.name not in seen_function_names:
+            seen_function_names.add(concrete_function.name)
+            self.concrete_functions.append(concrete_function)
 
   @property
   def root(self):
@@ -140,6 +150,11 @@ def _get_signature(function):
 
 def _valid_signature(concrete_function):
   """Returns whether concrete function can be converted to a signature."""
+  if not concrete_function.outputs:
+    # Functions without outputs don't make sense as signatures. We just don't
+    # have any way to run an Operation with no outputs as a SignatureDef in the
+    # 1.x style.
+    return False
   try:
     _normalize_outputs(concrete_function.structured_outputs, "unused", "unused")
   except ValueError:
@@ -552,21 +567,9 @@ def _fill_meta_graph_def(meta_graph_def, saveable_view, signature_functions,
   # the exported graph (thus the `to_graph` argument).
   saver = object_saver.freeze(object_map=object_map, to_graph=exported_graph)
 
-  # We must instantiate and list all concrete functions of `Function`s while in
-  # eager mode so they end up added to the graph and can later be used by the
-  # object based saved model.
-  concrete_functions = []
-  for obj in accessible_objects:
-    for function in saveable_view.functions[obj].values():
-      if isinstance(function, defun.ConcreteFunction):
-        concrete_functions.append(function)
-      else:
-        concrete_functions.extend(
-            function._list_all_concrete_functions_for_serialization())  # pylint: disable=protected-access
-
   with exported_graph.as_default():
     signatures = _generate_signatures(signature_functions, resource_map)
-    for concrete_function in concrete_functions:
+    for concrete_function in saveable_view.concrete_functions:
       concrete_function.add_to_graph()
     saver_def = saver.to_proto()
     meta_graph_def.saver_def.CopyFrom(saver_def)
@@ -599,8 +602,16 @@ def _write_object_graph(saveable_view, export_dir, asset_file_def_index):
     elif isinstance(obj, tracking.TrackableAsset):
       node_ids[obj.asset_path.handle] = i
 
+  coder = nested_structure_coder.StructureCoder()
+  for concrete_function in saveable_view.concrete_functions:
+    serialized = function_serialization.serialize_concrete_function(
+        concrete_function, node_ids, coder)
+    if serialized is not None:
+      proto.concrete_functions[concrete_function.name].CopyFrom(
+          serialized)
+
   for obj, obj_proto in zip(saveable_view.nodes, proto.nodes):
-    _write_object_proto(obj, obj_proto, asset_file_def_index, node_ids)
+    _write_object_proto(obj, obj_proto, asset_file_def_index)
 
   extra_asset_dir = os.path.join(
       compat.as_bytes(export_dir),
@@ -611,7 +622,7 @@ def _write_object_graph(saveable_view, export_dir, asset_file_def_index):
   file_io.write_string_to_file(object_graph_filename, proto.SerializeToString())
 
 
-def _write_object_proto(obj, proto, asset_file_def_index, node_ids):
+def _write_object_proto(obj, proto, asset_file_def_index):
   """Saves an object into SavedObject proto."""
   if isinstance(obj, tracking.TrackableAsset):
     proto.asset.SetInParent()
@@ -623,10 +634,10 @@ def _write_object_proto(obj, proto, asset_file_def_index, node_ids):
     proto.variable.shape.CopyFrom(obj.shape.as_proto())
   elif isinstance(obj, def_function.Function):
     proto.function.CopyFrom(
-        function_serialization.serialize_function(obj, node_ids))
+        function_serialization.serialize_function(obj))
   elif isinstance(obj, defun.ConcreteFunction):
-    proto.concrete_function.CopyFrom(
-        function_serialization.serialize_concrete_function(obj, node_ids))
+    proto.bare_concrete_function.CopyFrom(
+        function_serialization.serialize_bare_concrete_function(obj))
   else:
     registered_type_proto = revived_types.serialize(obj)
     if registered_type_proto is None:
diff --git a/tensorflow/python/saved_model/saved_object_graph.proto b/tensorflow/python/saved_model/saved_object_graph.proto
index 6f5a952083..1e8743f681 100644
--- a/tensorflow/python/saved_model/saved_object_graph.proto
+++ b/tensorflow/python/saved_model/saved_object_graph.proto
@@ -27,6 +27,10 @@ message SavedObjectGraph {
   // The position of the object in this list indicates its id.
   // Nodes[0] is considered the root node.
   repeated SavedObject nodes = 1;
+
+  // Information about captures and output structures in concrete functions.
+  // Referenced from SavedBareConcreteFunction and SavedFunction.
+  map<string, SavedConcreteFunction> concrete_functions = 2;
 }
 
 message SavedObject {
@@ -54,7 +58,7 @@ message SavedObject {
     SavedAsset asset = 5;
     SavedFunction function = 6;
     SavedVariable variable = 7;
-    SavedConcreteFunction concrete_function = 8;
+    SavedBareConcreteFunction bare_concrete_function = 8;
   }
 }
 
@@ -86,14 +90,13 @@ message SavedAsset {
 
 // A function with multiple signatures, possibly with non-Tensor arguments.
 message SavedFunction {
-  repeated SavedConcreteFunction concrete_function = 1;
+  repeated string concrete_functions = 1;
   FunctionSpec function_spec = 2;
 }
 
+// Stores low-level information about a concrete function. Referenced in either
+// a SavedFunction or a SavedBareConcreteFunction.
 message SavedConcreteFunction {
-  // A reference to a TensorFlow function in the MetaGraph's FunctionDefLibrary
-  string name = 1;
-
   // Bound inputs to the function. The SavedObjects identified by the node ids
   // given here are appended as extra inputs to the caller-supplied inputs.
   // The only types of SavedObjects valid here are SavedVariable, SavedResource
@@ -108,6 +111,16 @@ message SavedConcreteFunction {
   StructuredValue output_signature = 4;
 }
 
+message SavedBareConcreteFunction {
+  // Identifies a SavedConcreteFunction.
+  string concrete_function_name = 1;
+
+  // A sequence of unique strings, one per Tensor argument.
+  repeated string argument_keywords = 2;
+  // The prefix of `argument_keywords` which may be identified by position.
+  int64 allowed_positional_arguments = 3;
+}
+
 // Represents a Variable that is initialized by loading the contents from the
 // SavedModel checkpoint.
 message SavedVariable {
-- 
GitLab


From 7c8bf0cffe6da5ecb9a4c6d8c8eb8d2b05e04b4b Mon Sep 17 00:00:00 2001
From: Suharsh Sivakumar <suharshs@google.com>
Date: Fri, 25 Jan 2019 11:00:50 -0800
Subject: [PATCH 1267/2345] Add support for quantizing shared tensors.

PiperOrigin-RevId: 230932148
---
 .../lite/tools/optimize/quantize_weights.cc   | 259 +++++-----
 .../lite/tools/optimize/quantize_weights.h    |   1 +
 .../tools/optimize/quantize_weights_test.cc   | 441 +++++++++++-------
 tensorflow/lite/tools/optimize/test_util.cc   |   2 +
 tensorflow/lite/tools/optimize/test_util.h    |   4 +
 .../lite/tools/optimize/testdata/README.md    |   2 +
 6 files changed, 428 insertions(+), 281 deletions(-)

diff --git a/tensorflow/lite/tools/optimize/quantize_weights.cc b/tensorflow/lite/tools/optimize/quantize_weights.cc
index b376d465e1..f0a280f1c1 100644
--- a/tensorflow/lite/tools/optimize/quantize_weights.cc
+++ b/tensorflow/lite/tools/optimize/quantize_weights.cc
@@ -34,34 +34,36 @@ namespace optimize {
 namespace {
 
 typedef struct {
-  TensorT* tensor;
+  OperatorT* op;
+  // The index of the op in the operators vector.
+  int32_t op_idx;
   // The index of the tensor to quantize in subgraph->tensors.
-  int32_t tensor_idx;
-  // The index of the tensor of the weight tensor to be quantize in op->inputs.
   int32_t op_input_idx;
-  // True if the tensor supports hybrid evaluation.
-  bool eval_hybrid;
-} TensorInfo;
+} ConsumerOpInfo;
 
 // The default minimum number of elements a weights array must have to be
 // quantized by this transformation.
 const int kWeightsMinNumElementsDefault = 1024;
 
-uint64_t CountTensorConsumers(const ModelT* model, const SubGraphT* subgraph,
-                              int32_t tensor_idx) {
-  uint64_t count = 0;
+// Gets the operators that consume tensor_idx.
+std::vector<ConsumerOpInfo> GetTensorConsumers(const ModelT* model,
+                                               const SubGraphT* subgraph,
+                                               int32_t tensor_idx) {
+  // TODO(suharshs): If this proves to be too slow, avoid calling it per tensor,
+  // instead doing one sweep for the entire model.
+  std::vector<ConsumerOpInfo> consumer_ops;
   for (int op_idx = 0; op_idx < subgraph->operators.size(); ++op_idx) {
-    const OperatorT* op = subgraph->operators[op_idx].get();
+    OperatorT* op = subgraph->operators[op_idx].get();
     if (op == nullptr) {
       continue;
     }
     for (int i = 0; i < op->inputs.size(); ++i) {
       if (op->inputs[i] == tensor_idx) {
-        count++;
+        consumer_ops.push_back({op, op_idx, i});
       }
     }
   }
-  return count;
+  return consumer_ops;
 }
 
 // Gets the list of op->inputs indices of the weights inputs to be quantized for
@@ -119,23 +121,39 @@ bool IsHybridEvaluationOp(const OperatorT* op, const BuiltinOperator& op_code) {
   return eval_hybrid;
 }
 
-// Populates a vector of TensorInfos for each input tensor of op that should be
-// quantized.
-TfLiteStatus GetQuantizableTensorsFromOperator(
+// Returns true if all of the op's inputs are quantized.
+bool CheckAllOpInputsQuantized(const SubGraphT* subgraph, const OperatorT* op,
+                               const BuiltinOperator& op_code) {
+  std::vector<int32_t> op_input_indices = GetWeightInputIndices(op_code);
+  for (const int32_t op_input_idx : op_input_indices) {
+    int32_t tensor_idx = op->inputs[op_input_idx];
+
+    if (tensor_idx == -1) {
+      // Optional tensor.
+      continue;
+    }
+
+    TensorT* tensor = subgraph->tensors[tensor_idx].get();
+
+    if (tensor->type != TensorType_INT8) {
+      return false;
+    }
+  }
+  return true;
+}
+
+// Inserts Tensors for each input tensor of op that should be
+// quantized into tensor_map.
+TfLiteStatus InsertQuantizableInputTensorsFromOperator(
     const ModelT* model, const OperatorT* op, uint64_t weights_min_num_elements,
-    bool use_hybrid_evaluation, std::vector<TensorInfo>* tensor_infos) {
+    std::unordered_map<int32_t, TensorT*>* tensor_map) {
   SubGraphT* subgraph = model->subgraphs.at(0).get();
   const BuiltinOperator op_code =
       model->operator_codes[op->opcode_index]->builtin_code;
 
-  tensor_infos->clear();
-
-  bool eval_hybrid = use_hybrid_evaluation && IsHybridEvaluationOp(op, op_code);
-
   std::vector<int32_t> op_input_indices = GetWeightInputIndices(op_code);
   for (const int32_t op_input_idx : op_input_indices) {
     int32_t tensor_idx = op->inputs[op_input_idx];
-
     if (tensor_idx == -1) {
       LOG(INFO) << "Skipping optional tensor input " << op_input_idx
                 << " of operation " << EnumNameBuiltinOperator(op_code);
@@ -143,14 +161,6 @@ TfLiteStatus GetQuantizableTensorsFromOperator(
     }
 
     TensorT* tensor = subgraph->tensors[tensor_idx].get();
-    // TODO(suharshs): Support shared weights, i.e. If two tensors share the
-    // same weight array, things may break. (i.e. SSD object detection)
-    if (CountTensorConsumers(model, subgraph, tensor_idx) != 1) {
-      LOG(INFO) << "Skipping quantization of tensor " << tensor->name
-                << " that is shared between multiple multiple operations.";
-      continue;
-    }
-
     if (tensor->type != TensorType_FLOAT32) {
       LOG(INFO) << "Skipping quantization of tensor " << tensor->name
                 << " that is not type float.";
@@ -163,9 +173,6 @@ TfLiteStatus GetQuantizableTensorsFromOperator(
       LOG(INFO) << "Skipping quantization of tensor " << tensor->name
                 << " because it has fewer than " << weights_min_num_elements
                 << " elements (" << num_elements << ").";
-      // If one of the weights isn't quantized, then we cannot use the hybrid
-      // kernel for this operation, since it expects everything to be quantized.
-      eval_hybrid = false;
       continue;
     }
 
@@ -177,58 +184,8 @@ TfLiteStatus GetQuantizableTensorsFromOperator(
       continue;
     }
 
-    TensorInfo tensor_info;
-    tensor_info.eval_hybrid = eval_hybrid;
-    tensor_info.op_input_idx = op_input_idx;
-    tensor_info.tensor_idx = tensor_idx;
-    tensor_info.tensor = tensor;
-
-    tensor_infos->push_back(tensor_info);
-  }
-
-  return kTfLiteOk;
-}
-
-// Quantizes tensor using asymmetric quantization with the min and max elements
-// of the tensor. This is needed to pass to Dequantize operations.
-TfLiteStatus AsymmetricQuantizeTensor(ModelT* model, TensorT* tensor) {
-  BufferT* buffer = model->buffers[tensor->buffer].get();
-  float* float_data = reinterpret_cast<float*>(buffer->data.data());
-  uint64_t num_elements;
-  TF_LITE_ENSURE_STATUS(utils::NumElements(*tensor, &num_elements));
-  LOG(INFO) << "Quantizing tensor " << tensor->name << " with " << num_elements
-            << " elements for float evaluation.";
-
-  // Compute the quantization params.
-  float min_value = *std::min_element(float_data, float_data + num_elements);
-  float max_value = *std::max_element(float_data, float_data + num_elements);
-
-  if (tensor->quantization == nullptr) {
-    tensor->quantization = absl::make_unique<QuantizationParametersT>();
+    tensor_map->insert({tensor_idx, tensor});
   }
-  utils::GetAsymmetricQuantizationParams(min_value, max_value, 0, 255,
-                                         tensor->quantization.get());
-
-  // Quantize the buffer.
-  std::vector<uint8_t> quantized_buffer;
-  quantized_buffer.resize(num_elements);
-  const double inverse_scale = 1. / tensor->quantization->scale[0];
-  for (std::size_t i = 0; i < num_elements; i++) {
-    const float src_val = float_data[i];
-    double scaled_val;
-    if (tensor->quantization->scale[0] == 0) {
-      scaled_val = tensor->quantization->zero_point[0];
-    } else {
-      scaled_val =
-          tensor->quantization->zero_point[0] + inverse_scale * src_val;
-    }
-    uint8_t integer_val = static_cast<uint8_t>(std::round(scaled_val));
-    quantized_buffer[i] = integer_val;
-  }
-  model->buffers[tensor->buffer]->data = quantized_buffer;
-
-  // Update the tensor type.
-  tensor->type = TensorType_UINT8;
 
   return kTfLiteOk;
 }
@@ -242,7 +199,7 @@ TfLiteStatus SymmetricQuantizeTensor(ModelT* model, TensorT* tensor) {
   uint64_t num_elements;
   TF_LITE_ENSURE_STATUS(utils::NumElements(*tensor, &num_elements));
   LOG(INFO) << "Quantizing tensor " << tensor->name << " with " << num_elements
-            << " elements for hybrid evaluation.";
+            << " elements.";
 
   std::vector<int8_t> quantized_buffer;
   quantized_buffer.resize(num_elements);
@@ -263,7 +220,7 @@ TfLiteStatus SymmetricQuantizeTensor(ModelT* model, TensorT* tensor) {
                                               uint8_buffer + num_elements);
 
   // Update the tensor type.
-  tensor->type = TensorType_UINT8;
+  tensor->type = TensorType_INT8;
 
   return kTfLiteOk;
 }
@@ -279,7 +236,8 @@ int32_t GetOrInsertDequantizeOpCodeIndex(ModelT* model) {
   model->operator_codes.push_back(absl::make_unique<OperatorCodeT>());
   int op_code_idx = model->operator_codes.size() - 1;
   model->operator_codes[op_code_idx]->builtin_code = BuiltinOperator_DEQUANTIZE;
-  // TODO(suharshs): How should the version be set in this op_code?
+  // Version 2 and onwards supports INT8 inputs.
+  model->operator_codes[op_code_idx]->version = 2;
 
   // Return the index of the newly placed OperatorCodeT.
   return op_code_idx;
@@ -306,6 +264,26 @@ void MakeTensor(const string& name, const std::vector<int32_t>& shape,
   tensor->reset(tensor_raw);
 }
 
+// Updates operator code versions for the operators with INT8 inputs.
+void UpdateInt8OperatorVersions(ModelT* model) {
+  for (int i = 0; i < model->operator_codes.size(); ++i) {
+    const BuiltinOperator& op_code = model->operator_codes[i]->builtin_code;
+    if (op_code == BuiltinOperator_CONV_2D || op_code == BuiltinOperator_SVDF ||
+        op_code == BuiltinOperator_EMBEDDING_LOOKUP ||
+        op_code == BuiltinOperator_RNN ||
+        op_code == BuiltinOperator_BIDIRECTIONAL_SEQUENCE_RNN ||
+        op_code == BuiltinOperator_UNIDIRECTIONAL_SEQUENCE_LSTM ||
+        op_code == BuiltinOperator_UNIDIRECTIONAL_SEQUENCE_RNN) {
+      model->operator_codes[i]->version = 2;
+
+    } else if (op_code == BuiltinOperator_FULLY_CONNECTED ||
+               op_code == BuiltinOperator_BIDIRECTIONAL_SEQUENCE_LSTM ||
+               op_code == BuiltinOperator_LSTM) {
+      model->operator_codes[i]->version = 3;
+    }
+  }
+}
+
 TfLiteStatus QuantizeWeightsInternal(flatbuffers::FlatBufferBuilder* builder,
                                      const Model* input_model,
                                      bool use_hybrid_evaluation,
@@ -323,50 +301,82 @@ TfLiteStatus QuantizeWeightsInternal(flatbuffers::FlatBufferBuilder* builder,
   SubGraphT* subgraph = model->subgraphs.at(0).get();
 
   std::vector<std::unique_ptr<OperatorT>> new_operators;
+  std::unordered_map<int32_t, TensorT*> tensor_map;
   for (int i = 0; i < subgraph->operators.size(); ++i) {
     OperatorT* op = subgraph->operators[i].get();
+    TF_LITE_ENSURE_STATUS(InsertQuantizableInputTensorsFromOperator(
+        model.get(), op, weights_min_num_elements, &tensor_map));
+  }
 
-    std::vector<TensorInfo> tensor_infos;
-    TF_LITE_ENSURE_STATUS(GetQuantizableTensorsFromOperator(
-        model.get(), op, weights_min_num_elements, use_hybrid_evaluation,
-        &tensor_infos));
-
-    for (const TensorInfo& tensor_info : tensor_infos) {
-      if (tensor_info.eval_hybrid) {
-        // Quantize the tensor.
-        TF_LITE_ENSURE_STATUS(
-            SymmetricQuantizeTensor(model.get(), tensor_info.tensor));
-      } else {
-        // Quantize the tensor.
-        TF_LITE_ENSURE_STATUS(
-            AsymmetricQuantizeTensor(model.get(), tensor_info.tensor));
-
-        // Create a new tensor to be the output of the dequantize op.
-        std::unique_ptr<TensorT> dequantize_output;
-        MakeTensor(tensor_info.tensor->name + "_dequantize",
-                   tensor_info.tensor->shape, &dequantize_output);
-        const int32_t dequantize_output_idx = subgraph->tensors.size();
-        subgraph->tensors.push_back(std::move(dequantize_output));
-
-        // Create the Dequantize operation.
-        std::unique_ptr<OperatorT> dequantize_op;
-        MakeDequantizeOperator(model.get(), &dequantize_op,
-                               tensor_info.tensor_idx, dequantize_output_idx);
-
-        // Update the op_input of tensor_idx to dequantize_output_idx.
-        op->inputs[tensor_info.op_input_idx] = dequantize_output_idx;
-
-        // Insert the newly created Dequantize operation.
-        new_operators.push_back(std::move(dequantize_op));
+  // The unordered_map ensures that we quantize each tensor exactly once.
+  // TODO(suharshs): This map key isn't sufficient when we support multiple
+  // subgraphs.
+  for (std::pair<int32_t, TensorT*> tensor_pair : tensor_map) {
+    // Quantize the tensor.
+    TF_LITE_ENSURE_STATUS(
+        SymmetricQuantizeTensor(model.get(), tensor_pair.second));
+  }
+
+  // Examine the tensor consumers to determine which require dequantize ops.
+  for (const auto& tensor_pair : tensor_map) {
+    const int32_t tensor_idx = tensor_pair.first;
+    TensorT* tensor = tensor_pair.second;
+    std::vector<ConsumerOpInfo> consumer_op_infos =
+        GetTensorConsumers(model.get(), subgraph, tensor_idx);
+
+    std::vector<ConsumerOpInfo> dequant_op_infos;  // Ops that need dequants.
+    for (ConsumerOpInfo& consumer_op_info : consumer_op_infos) {
+      OperatorT* consumer_op = consumer_op_info.op;
+      const BuiltinOperator consumer_op_code =
+          model->operator_codes[consumer_op->opcode_index]->builtin_code;
+      // If the op is a hybrid op and all the required tensors are quantized,
+      // we have no further work to do, but for all ops that require
+      // dequantization we need to add a Dequantize op.
+      bool eval_hybrid =
+          use_hybrid_evaluation &&
+          IsHybridEvaluationOp(consumer_op, consumer_op_code) &&
+          CheckAllOpInputsQuantized(subgraph, consumer_op, consumer_op_code);
+      if (!eval_hybrid) {
+        dequant_op_infos.push_back(consumer_op_info);
       }
     }
-    // After (maybe) quantizing inputs, we copy the operator into the new list.
-    new_operators.push_back(std::move(subgraph->operators[i]));
+
+    // If no ops require dequant, we are done for this tensor.
+    if (dequant_op_infos.empty()) {
+      continue;
+    }
+
+    // Create a new tensor to be the output of the dequantize op.
+    std::unique_ptr<TensorT> dequantize_output;
+    const string dequant_name = tensor->name + "_dequantize";
+    MakeTensor(dequant_name, tensor->shape, &dequantize_output);
+    const int32_t dequantize_output_idx = subgraph->tensors.size();
+    subgraph->tensors.push_back(std::move(dequantize_output));
+
+    // Create the Dequantize operation.
+    std::unique_ptr<OperatorT> dequantize_op;
+    MakeDequantizeOperator(model.get(), &dequantize_op, tensor_idx,
+                           dequantize_output_idx);
+
+    LOG(INFO) << "Creating Dequantize op with name " << dequant_name << ".";
+
+    // Update the op_input of all the ops that need the created dequantize
+    // operation.
+    int32_t min_op_idx = 0;
+    for (ConsumerOpInfo& dequant_op_info : dequant_op_infos) {
+      dequant_op_info.op->inputs[dequant_op_info.op_input_idx] =
+          dequantize_output_idx;
+      min_op_idx = std::min(dequant_op_info.op_idx, min_op_idx);
+    }
+
+    // Insert the newly created Dequantize operation before the earliest
+    // consumer, since TFLite requires operators to be topo-sorted.
+    subgraph->operators.insert(subgraph->operators.begin() + min_op_idx,
+                               std::move(dequantize_op));
   }
 
-  // At this point all unique_ptrs in the original operators are invalid, and
-  // we need to replace it with the new_operators vector.
-  subgraph->operators = std::move(new_operators);
+  // Update the modified operator code versions.
+  UpdateInt8OperatorVersions(model.get());
 
   flatbuffers::Offset<Model> output_model_location =
       Model::Pack(*builder, model.get());
@@ -380,11 +390,12 @@ TfLiteStatus QuantizeWeightsInternal(flatbuffers::FlatBufferBuilder* builder,
 namespace internal {
 TfLiteStatus QuantizeWeights(flatbuffers::FlatBufferBuilder* builder,
                              const Model* input_model,
+                             uint64_t weights_min_num_elements,
                              bool use_hybrid_evaluation) {
   // By default we require that only weights with more than
   // kWeightsMinSizeDefault elements are quantized.
   return QuantizeWeightsInternal(builder, input_model, use_hybrid_evaluation,
-                                 kWeightsMinNumElementsDefault);
+                                 weights_min_num_elements);
 }
 }  // namespace internal
 
diff --git a/tensorflow/lite/tools/optimize/quantize_weights.h b/tensorflow/lite/tools/optimize/quantize_weights.h
index c2c0b0ce83..6baecc210f 100644
--- a/tensorflow/lite/tools/optimize/quantize_weights.h
+++ b/tensorflow/lite/tools/optimize/quantize_weights.h
@@ -48,6 +48,7 @@ namespace internal {
 // evaluation disabled.
 TfLiteStatus QuantizeWeights(flatbuffers::FlatBufferBuilder* builder,
                              const Model* input_model,
+                             uint64_t weights_min_num_elements,
                              bool use_hybrid_evaluation);
 }  // namespace internal
 
diff --git a/tensorflow/lite/tools/optimize/quantize_weights_test.cc b/tensorflow/lite/tools/optimize/quantize_weights_test.cc
index 32725e5ee2..3ded2546c3 100644
--- a/tensorflow/lite/tools/optimize/quantize_weights_test.cc
+++ b/tensorflow/lite/tools/optimize/quantize_weights_test.cc
@@ -12,215 +12,342 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tensorflow/lite/tools/optimize/quantize_weights.h"
-
+#include <cstddef>
+#include <cstdint>
 #include <memory>
 
-#include "flatbuffers/flexbuffers.h"
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
+#include "flatbuffers/flatbuffers.h"  // TF:flatbuffers
+#include "flatbuffers/flexbuffers.h"  // TF:flatbuffers
+#include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/platform/init_main.h"
+#include "tensorflow/core/util/command_line_flags.h"
 #include "tensorflow/lite/model.h"
 #include "tensorflow/lite/schema/schema_generated.h"
+#include "tensorflow/lite/tools/optimize/quantize_weights.h"
+#include "tensorflow/lite/tools/optimize/test_util.h"
+
+namespace {
+tensorflow::string* g_test_model_dir = nullptr;
+}  // namespace
 
 namespace tflite {
 namespace optimize {
 namespace {
 
-class QuantizeWeightsTest : public ::testing::Test {
+std::unique_ptr<FlatBufferModel> ReadTestModel() {
+  auto model_path = tensorflow::io::JoinPath(
+      *g_test_model_dir, internal::kConvModelWith0Plus10Weights);
+  return FlatBufferModel::BuildFromFile(model_path.c_str());
+}
+
+std::unique_ptr<FlatBufferModel> ReadSharedWeightsTestModel() {
+  auto model_path = tensorflow::io::JoinPath(*g_test_model_dir,
+                                             internal::kModelWithSharedWeights);
+  return FlatBufferModel::BuildFromFile(model_path.c_str());
+}
+
+template <typename T>
+std::vector<T> GetAsVector(const flatbuffers::Vector<T>* vec) {
+  return std::vector<T>(vec->begin(), vec->end());
+}
+
+class QuantizeWeightsTest : public testing::Test {
  protected:
-  int GetElementsNum(const TensorT* tensor) {
-    int tensor_size = 1;
-    for (const int dim : tensor->shape) {
-      tensor_size *= dim;
-    }
-    return tensor_size;
+  QuantizeWeightsTest() {}
+
+  void LoadBasicModel() {
+    input_model_ = ReadTestModel();
+    model_ = input_model_->GetModel();
   }
 
-  const OperatorT* GetOpWithOutput(const SubGraphT* subgraph,
-                                   int32_t output_tensor_idx) {
-    for (int i = 0; i < subgraph->operators.size(); ++i) {
-      OperatorT* op = subgraph->operators[i].get();
-      if (std::find(op->outputs.begin(), op->outputs.end(),
-                    output_tensor_idx) != op->outputs.end()) {
-        return op;
-      }
-    }
-    return nullptr;
+  void LoadSharedWeightsModel() {
+    input_model_ = ReadSharedWeightsTestModel();
+    model_ = input_model_->GetModel();
   }
 
-  void SymmetricDequantizeAndCompare(const BufferT* input_buffer,
-                                     const BufferT* output_buffer,
-                                     float scale) {
-    const float* input_buffer_data =
-        reinterpret_cast<const float*>(input_buffer->data.data());
-    const int8_t* output_buffer_data =
-        reinterpret_cast<const int8_t*>(output_buffer->data.data());
-    for (int i = 0; i < output_buffer->data.size(); i++) {
-      float diff = input_buffer_data[i] - (output_buffer_data[i] * scale);
-      ASSERT_TRUE(std::abs(diff) <= scale);
+  std::unique_ptr<FlatBufferModel> input_model_;
+  const Model* model_;
+
+  bool IsModelInputOrOutput(const Model* model, uint32_t tensor_idx) {
+    for (size_t subgraph_idx = 0; subgraph_idx < model_->subgraphs()->size();
+         ++subgraph_idx) {
+      const auto subgraph = model->subgraphs()->Get(subgraph_idx);
+      for (size_t i = 0; i < subgraph->inputs()->size(); ++i) {
+        if (subgraph->inputs()->Get(i) == tensor_idx) {
+          return true;
+        }
+      }
+      for (size_t i = 0; i < subgraph->outputs()->size(); ++i) {
+        if (subgraph->outputs()->Get(i) == tensor_idx) {
+          return true;
+        }
+      }
     }
+    return false;
   }
 
-  void AsymmetricDequantizeAndCompare(const BufferT* input_buffer,
-                                      const BufferT* output_buffer, float scale,
-                                      int64_t zero_point) {
-    const float* input_buffer_data =
-        reinterpret_cast<const float*>(input_buffer->data.data());
-    const uint8_t* output_buffer_data = output_buffer->data.data();
-    for (int i = 0; i < output_buffer->data.size(); i++) {
-      float diff =
-          input_buffer_data[i] - ((output_buffer_data[i] - zero_point) * scale);
-      ASSERT_TRUE(std::abs(diff) <= scale);
+  // Returns the producer op code of the specified tensor_idx.
+  BuiltinOperator GetProducerOpCode(const Model* model, uint32_t subgraph_idx,
+                                    uint32_t tensor_idx) {
+    const auto subgraph = model->subgraphs()->Get(subgraph_idx);
+    for (size_t op_idx = 0; op_idx < subgraph->operators()->size(); ++op_idx) {
+      const auto op = subgraph->operators()->Get(op_idx);
+      for (size_t i = 0; i < op->outputs()->size(); ++i) {
+        if (op->outputs()->Get(i) == tensor_idx) {
+          const uint32_t op_code_idx = op->opcode_index();
+          return model->operator_codes()->Get(op_code_idx)->builtin_code();
+        }
+      }
     }
+
+    LOG(FATAL) << "tensor_idx " << tensor_idx
+               << " not produced by op in model.";
   }
+};
 
-  void CheckWeights(const Model* input_model_packed,
-                    const Model* output_model_packed,
-                    bool use_hybrid_evaluation,
-                    uint64_t weights_min_num_elements = 1024) {
-    std::unique_ptr<ModelT> input_model;
-    input_model.reset(input_model_packed->UnPack());
-
-    std::unique_ptr<ModelT> output_model;
-    output_model.reset(output_model_packed->UnPack());
-
-    SubGraphT* subgraph = output_model->subgraphs.at(0).get();
-
-    for (int i = 0; i < subgraph->operators.size(); ++i) {
-      OperatorT* op = subgraph->operators[i].get();
-      const BuiltinOperator op_code =
-          output_model->operator_codes[op->opcode_index]->builtin_code;
-
-      // These are the operations that should be quantized.
-      // TODO(suharshs): Right now this test only checks the relevant operations
-      // for the mobilenet v1 model used in the tests below.
-      int32_t tensor_idx;
-      if (op_code == BuiltinOperator_CONV_2D ||
-          op_code == BuiltinOperator_DEPTHWISE_CONV_2D ||
-          op_code == BuiltinOperator_FULLY_CONNECTED) {
-        tensor_idx = op->inputs[1];
-      } else {
-        continue;
-      }
+TEST_F(QuantizeWeightsTest, QuantizationSucceeds) {
+  LoadBasicModel();
+  flatbuffers::FlatBufferBuilder builder;
+  auto status = QuantizeWeights(&builder, model_, 0);
+  EXPECT_EQ(status, kTfLiteOk);
 
-      bool eval_hybrid = false;
-      // These are the ops that support hybrid evaluation.
-      if (op_code == BuiltinOperator_FULLY_CONNECTED ||
-          op_code == BuiltinOperator_CONV_2D) {
-        eval_hybrid = true;
-      }
+  const uint8_t* buffer = builder.GetBufferPointer();
+  const Model* output_model = GetModel(buffer);
+  ASSERT_TRUE(output_model);
+}
 
-      const TensorT* tensor = subgraph->tensors[tensor_idx].get();
-      int tensor_size = GetElementsNum(tensor);
-      // If the tensor_size is less than 1024 we expect the tensor to remain
-      // unquantized.
-      if (tensor_size < weights_min_num_elements) {
-        ASSERT_TRUE(tensor->type == TensorType_FLOAT32)
-            << tensor->name << " of type " << tensor->type;
-        const OperatorT* preceding_op = GetOpWithOutput(subgraph, tensor_idx);
-        // The weight tensor should not come from a dequantize op.
-        ASSERT_TRUE(preceding_op == nullptr);
-      } else if (use_hybrid_evaluation && eval_hybrid) {
-        // The input to the op should still be uint8.
-        ASSERT_TRUE(tensor->type == TensorType_UINT8) << tensor->name;
-        // The weight tensor should not come from a dequantize op.
-        const OperatorT* preceding_op = GetOpWithOutput(subgraph, tensor_idx);
-        ASSERT_TRUE(preceding_op == nullptr);
-
-        // Test symmetric quantization.
-        SymmetricDequantizeAndCompare(
-            input_model->buffers[tensor->buffer].get(),
-            output_model->buffers[tensor->buffer].get(),
-            tensor->quantization->scale[0]);
+TEST_F(QuantizeWeightsTest, WeightsMinNumElements) {
+  LoadBasicModel();
+  // Make weights_min_size sufficiently large such that no quantization should
+  // happen, i.e. the original model is the same size as the old one.
+  flatbuffers::FlatBufferBuilder builder;
+  const uint64_t kWeightsMinNumElements = 1000000;
+  EXPECT_EQ(QuantizeWeights(&builder, model_, kWeightsMinNumElements),
+            kTfLiteOk);
 
+  const uint8_t* buffer = builder.GetBufferPointer();
+  const Model* output_model = GetModel(buffer);
+  ASSERT_TRUE(output_model);
+
+  for (size_t subgraph_idx = 0; subgraph_idx < model_->subgraphs()->size();
+       subgraph_idx++) {
+    const auto quantized_graph = output_model->subgraphs()->Get(subgraph_idx);
+    const auto float_graph = model_->subgraphs()->Get(subgraph_idx);
+    ASSERT_EQ(quantized_graph->tensors()->size(),
+              float_graph->tensors()->size());
+    for (size_t i = 0; i < quantized_graph->tensors()->size(); i++) {
+      const auto quant_tensor = quantized_graph->tensors()->Get(i);
+      const auto float_tensor = float_graph->tensors()->Get(i);
+      // Everything should remain equal between the two graphs.
+      EXPECT_EQ(quant_tensor->buffer(), float_tensor->buffer());
+      EXPECT_EQ(quant_tensor->is_variable(), float_tensor->is_variable());
+      EXPECT_EQ(GetAsVector(quant_tensor->shape()),
+                GetAsVector(float_tensor->shape()));
+      EXPECT_EQ(quant_tensor->name()->str(), float_tensor->name()->str());
+      EXPECT_EQ(quant_tensor->type(), float_tensor->type());
+    }
+  }
+}
+
+TEST_F(QuantizeWeightsTest, HybridConv) {
+  LoadBasicModel();
+  flatbuffers::FlatBufferBuilder builder;
+  auto status = QuantizeWeights(&builder, model_, 0);
+  EXPECT_EQ(status, kTfLiteOk);
+
+  const uint8_t* buffer = builder.GetBufferPointer();
+  const Model* output_model = GetModel(buffer);
+  ASSERT_TRUE(output_model);
+
+  // Nothing should change.
+  ASSERT_EQ(output_model->subgraphs()->size(), model_->subgraphs()->size());
+  for (size_t subgraph_idx = 0; subgraph_idx < model_->subgraphs()->size();
+       subgraph_idx++) {
+    const auto quantized_graph = output_model->subgraphs()->Get(subgraph_idx);
+    const auto float_graph = model_->subgraphs()->Get(subgraph_idx);
+    ASSERT_EQ(quantized_graph->tensors()->size(),
+              float_graph->tensors()->size());
+    // Make sure the graph only has one Conv operation.
+    ASSERT_EQ(quantized_graph->operators()->size(), 1);
+    const auto op = quantized_graph->operators()->Get(0);
+    const uint32_t op_code_idx = op->opcode_index();
+    ASSERT_EQ(output_model->operator_codes()->Get(op_code_idx)->builtin_code(),
+              BuiltinOperator_CONV_2D);
+    for (size_t i = 0; i < quantized_graph->tensors()->size(); i++) {
+      const auto quant_tensor = quantized_graph->tensors()->Get(i);
+      const auto float_tensor = float_graph->tensors()->Get(i);
+      EXPECT_EQ(quant_tensor->buffer(), float_tensor->buffer());
+      EXPECT_EQ(quant_tensor->is_variable(), float_tensor->is_variable());
+      EXPECT_EQ(GetAsVector(quant_tensor->shape()),
+                GetAsVector(float_tensor->shape()));
+      EXPECT_EQ(quant_tensor->name()->str(), float_tensor->name()->str());
+      // If the tensor is a weight, it should have type INT8, otherwise it
+      // should stay with type FLOAT32.
+      // If the tensor is a bias, it should have type FLOAT32.
+      if (quant_tensor->name()->str() == "conv_bias") {
+        EXPECT_EQ(quant_tensor->type(), TensorType_FLOAT32);
+      } else if (IsModelInputOrOutput(output_model, i)) {
+        EXPECT_EQ(quant_tensor->type(), TensorType_FLOAT32);
+      } else if (quant_tensor->buffer() != 0) {
+        EXPECT_EQ(quant_tensor->type(), TensorType_INT8)
+            << quant_tensor->name()->str();
       } else {
-        // The input to the op should still be float.
-        ASSERT_TRUE(tensor->type == TensorType_FLOAT32) << tensor->name;
-        const OperatorT* preceding_op = GetOpWithOutput(subgraph, tensor_idx);
-        ASSERT_TRUE(preceding_op != nullptr);
-        // The float input should be the dequantize output.
-        ASSERT_TRUE(output_model->operator_codes[preceding_op->opcode_index]
-                        ->builtin_code == BuiltinOperator_DEQUANTIZE);
-        // Finally, ensure that the input to the dequantize operation is
-        // quantized.
-        const TensorT* quantized_tensor =
-            subgraph->tensors[preceding_op->inputs[0]].get();
-        ASSERT_TRUE(quantized_tensor->type == TensorType_UINT8);
-
-        // Test the assymetric quantization.
-        AsymmetricDequantizeAndCompare(
-            input_model->buffers[quantized_tensor->buffer].get(),
-            output_model->buffers[quantized_tensor->buffer].get(),
-            quantized_tensor->quantization->scale[0],
-            quantized_tensor->quantization->zero_point[0]);
+        EXPECT_EQ(quant_tensor->type(), TensorType_FLOAT32);
       }
     }
   }
-};
-
-TEST_F(QuantizeWeightsTest, SimpleTestWithHybrid) {
-  string model_path =
-      "third_party/tensorflow/lite/tools/optimize/testdata/"
-      "mobilenet_v1_0.25_128.tflite";
-  std::unique_ptr<FlatBufferModel> input_fb =
-      FlatBufferModel::BuildFromFile(model_path.data());
-  const Model* input_model = input_fb->GetModel();
+}
 
+TEST_F(QuantizeWeightsTest, DequantizeConv) {
+  LoadBasicModel();
   flatbuffers::FlatBufferBuilder builder;
-  EXPECT_EQ(QuantizeWeights(&builder, input_model), kTfLiteOk);
+  auto status = internal::QuantizeWeights(&builder, model_, 0,
+                                          /*use_hybrid_evaluation=*/false);
+  EXPECT_EQ(status, kTfLiteOk);
 
   const uint8_t* buffer = builder.GetBufferPointer();
   const Model* output_model = GetModel(buffer);
-
-  CheckWeights(input_model, output_model, true);
+  ASSERT_TRUE(output_model);
+
+  ASSERT_EQ(output_model->subgraphs()->size(), model_->subgraphs()->size());
+  for (size_t subgraph_idx = 0; subgraph_idx < model_->subgraphs()->size();
+       ++subgraph_idx) {
+    const auto quantized_graph = output_model->subgraphs()->Get(subgraph_idx);
+    const auto float_graph = model_->subgraphs()->Get(subgraph_idx);
+    // The output graph should have an extra tensor from the added dequantize
+    // op.
+    ASSERT_EQ(quantized_graph->tensors()->size(),
+              float_graph->tensors()->size() + 1);
+    // Check that a dequantize op exists.
+    int32_t dequant_input_idx = -1;
+    int32_t dequant_output_idx = -1;
+    for (size_t i = 0; i < quantized_graph->operators()->size(); ++i) {
+      const auto op = quantized_graph->operators()->Get(i);
+      const uint32_t op_code_idx = op->opcode_index();
+      if (output_model->operator_codes()->Get(op_code_idx)->builtin_code() ==
+          BuiltinOperator_DEQUANTIZE) {
+        dequant_input_idx = op->inputs()->Get(0);
+        dequant_output_idx = op->outputs()->Get(0);
+      }
+    }
+    ASSERT_GT(dequant_input_idx, -1);
+    ASSERT_GT(dequant_output_idx, -1);
+    for (size_t i = 0; i < quantized_graph->tensors()->size(); ++i) {
+      const auto quant_tensor = quantized_graph->tensors()->Get(i);
+      // If the tensor is a weight, it should have type INT8.
+      // If the tensor is a bias, it should have type FLOAT32.
+      // If the tensor is an input or output it should have type FLOAT32.
+      // The input to dequantize should be INT8, and all other tensors should be
+      // FLOAT32.
+      if (i == dequant_input_idx) {
+        EXPECT_EQ(quant_tensor->type(), TensorType_INT8);
+      } else if (i == dequant_output_idx) {
+        EXPECT_EQ(quant_tensor->type(), TensorType_FLOAT32);
+      } else if (IsModelInputOrOutput(output_model, i)) {
+        EXPECT_EQ(quant_tensor->type(), TensorType_FLOAT32);
+      } else if (quant_tensor->name()->str() == "conv_bias") {
+        EXPECT_EQ(quant_tensor->type(), TensorType_FLOAT32);
+      } else if (quant_tensor->buffer() != 0) {
+        // If its a non-bias constant tensor, is must be the weight.
+        EXPECT_EQ(quant_tensor->type(), TensorType_INT8);
+      } else {
+        EXPECT_EQ(quant_tensor->type(), TensorType_FLOAT32);
+      }
+    }
+  }
 }
 
-TEST_F(QuantizeWeightsTest, SimpleTestWithoutHybrid) {
-  string model_path =
-      "third_party/tensorflow/lite/tools/optimize/testdata/"
-      "mobilenet_v1_0.25_128.tflite";
-  std::unique_ptr<FlatBufferModel> input_fb =
-      FlatBufferModel::BuildFromFile(model_path.data());
-  const Model* input_model = input_fb->GetModel();
-
+TEST_F(QuantizeWeightsTest, SharedWeights_Hybrid) {
+  LoadSharedWeightsModel();
   flatbuffers::FlatBufferBuilder builder;
-  // Disable hybrid evaluation.
-  EXPECT_EQ(internal::QuantizeWeights(&builder, input_model, false), kTfLiteOk);
+  auto status = QuantizeWeights(&builder, model_, 0);
+  EXPECT_EQ(status, kTfLiteOk);
 
   const uint8_t* buffer = builder.GetBufferPointer();
   const Model* output_model = GetModel(buffer);
-
-  CheckWeights(input_model, output_model, false);
+  ASSERT_TRUE(output_model);
+
+  ASSERT_EQ(output_model->subgraphs()->size(), model_->subgraphs()->size());
+  uint32_t num_conv_ops = 0;
+  for (size_t subgraph_idx = 0; subgraph_idx < model_->subgraphs()->size();
+       ++subgraph_idx) {
+    const auto quantized_graph = output_model->subgraphs()->Get(subgraph_idx);
+    for (size_t i = 0; i < quantized_graph->operators()->size(); ++i) {
+      const auto op = quantized_graph->operators()->Get(i);
+      const uint32_t op_code_idx = op->opcode_index();
+      const auto op_code =
+          output_model->operator_codes()->Get(op_code_idx)->builtin_code();
+      if (op_code == BuiltinOperator_CONV_2D) {
+        num_conv_ops++;
+        // Ensure that each convolution's weights tensor is now INT8.
+        const auto weights_tensor =
+            quantized_graph->tensors()->Get(op->inputs()->Get(1));
+        EXPECT_EQ(weights_tensor->type(), TensorType_INT8);
+      }
+    }
+  }
+  // Ensure that there were exactly two convolutions in the model.
+  EXPECT_EQ(num_conv_ops, 2);
 }
 
-TEST_F(QuantizeWeightsTest, SimpleTestWithWeightsMinNumElements) {
-  string model_path =
-      "third_party/tensorflow/lite/tools/optimize/testdata/"
-      "mobilenet_v1_0.25_128.tflite";
-  std::unique_ptr<FlatBufferModel> input_fb =
-      FlatBufferModel::BuildFromFile(model_path.data());
-  const Model* input_model = input_fb->GetModel();
-
+TEST_F(QuantizeWeightsTest, SharedWeights_Dequantize) {
+  LoadSharedWeightsModel();
   flatbuffers::FlatBufferBuilder builder;
-  // Make weights_min_size sufficiently large such that no quantization should
-  // happen, i.e. the original model is the same size as the old one.
-  const uint64_t kWeightsMinNumElements = 1000000;
-  EXPECT_EQ(QuantizeWeights(&builder, input_model, kWeightsMinNumElements),
-            kTfLiteOk);
+  auto status = internal::QuantizeWeights(&builder, model_, 0,
+                                          /*use_hybrid_evaluation*/ false);
+  EXPECT_EQ(status, kTfLiteOk);
 
   const uint8_t* buffer = builder.GetBufferPointer();
   const Model* output_model = GetModel(buffer);
-  CheckWeights(input_model, output_model, true, kWeightsMinNumElements);
+  ASSERT_TRUE(output_model);
+
+  ASSERT_EQ(output_model->subgraphs()->size(), model_->subgraphs()->size());
+  uint32_t num_conv_ops = 0;
+  for (size_t subgraph_idx = 0; subgraph_idx < model_->subgraphs()->size();
+       ++subgraph_idx) {
+    const auto quantized_graph = output_model->subgraphs()->Get(subgraph_idx);
+    for (size_t i = 0; i < quantized_graph->operators()->size(); ++i) {
+      const auto op = quantized_graph->operators()->Get(i);
+      const uint32_t op_code_idx = op->opcode_index();
+      const auto op_code =
+          output_model->operator_codes()->Get(op_code_idx)->builtin_code();
+      if (op_code == BuiltinOperator_CONV_2D) {
+        num_conv_ops++;
+        // Ensure that each convolution's weights tensor is still FLOAT
+        // (the output of the dequantize).
+        uint32_t weights_tensor_index = op->inputs()->Get(1);
+        const auto weights_tensor =
+            quantized_graph->tensors()->Get(weights_tensor_index);
+        EXPECT_EQ(weights_tensor->type(), TensorType_FLOAT32);
+
+        // Check that it comes from a dequantize operation.
+        const auto producer_op_code =
+            GetProducerOpCode(output_model, subgraph_idx, weights_tensor_index);
+        EXPECT_EQ(producer_op_code, BuiltinOperator_DEQUANTIZE);
+      }
+    }
+  }
+  // Ensure that there were exactly two convolutions in the model.
+  EXPECT_EQ(num_conv_ops, 2);
 }
 
-// TODO(suharshs): Add tests that run the resulting model.
-
 }  // namespace
 }  // namespace optimize
 }  // namespace tflite
 
 int main(int argc, char** argv) {
-  // On Linux, add: FLAGS_logtostderr = true;
-  ::testing::InitGoogleTest(&argc, argv);
+  tensorflow::string model_file;
+  const std::vector<tensorflow::Flag> flag_list = {
+      tensorflow::Flag("test_model_file", &model_file,
+                       "Path to test tflite model file."),
+  };
+
+  const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list);
+  CHECK(parse_result) << "Required test_model_file";
+  g_test_model_dir =
+      new tensorflow::string(tensorflow::io::Dirname(model_file));
+  ::tensorflow::port::InitMain(argv[0], &argc, &argv);
   return RUN_ALL_TESTS();
 }
diff --git a/tensorflow/lite/tools/optimize/test_util.cc b/tensorflow/lite/tools/optimize/test_util.cc
index 3506142901..30dc1a4f1e 100644
--- a/tensorflow/lite/tools/optimize/test_util.cc
+++ b/tensorflow/lite/tools/optimize/test_util.cc
@@ -31,6 +31,8 @@ const char* kSingleSoftmaxModelMinMinus5MaxPlus5 =
 const char* kSingleAvgPoolModelMinMinus5MaxPlus5 =
     "single_avg_pool_min_minus_5_max_plus_5.tflite";
 
+const char* kModelWithSharedWeights = "weight_shared_between_convs.tflite";
+
 int FailOnErrorReporter::Report(const char* format, va_list args) {
   char buf[1024];
   vsnprintf(buf, sizeof(buf), format, args);
diff --git a/tensorflow/lite/tools/optimize/test_util.h b/tensorflow/lite/tools/optimize/test_util.h
index 1981068acc..21f8b3ceb0 100644
--- a/tensorflow/lite/tools/optimize/test_util.h
+++ b/tensorflow/lite/tools/optimize/test_util.h
@@ -42,6 +42,10 @@ extern const char* kSingleSoftmaxModelMinMinus5MaxPlus5;
 // and max in range [-5, 5], not necessarily -5 or +5.
 extern const char* kSingleAvgPoolModelMinMinus5MaxPlus5;
 
+// Test model with a weights variable that is shared between a convolution layer
+// and an add operation.
+extern const char* kModelWithSharedWeights;
+
 // An error reporter that fails on testing.
 class FailOnErrorReporter : public ErrorReporter {
  public:
diff --git a/tensorflow/lite/tools/optimize/testdata/README.md b/tensorflow/lite/tools/optimize/testdata/README.md
index b6b331b828..d8a156546d 100644
--- a/tensorflow/lite/tools/optimize/testdata/README.md
+++ b/tensorflow/lite/tools/optimize/testdata/README.md
@@ -21,3 +21,5 @@ This directory contains test models for testing quantization.
 * `single_avg_pool_input_min_minus_5_max_5.tflite` \
    A floating point model with a single average pool. The input tensor has min
    and max in range [-5, 5], not necessarily -5 or +5.
+* `weight_shared_between_convs.tflite` \
+   A floating point model with two convs that have a use the same weight tensor.
-- 
GitLab


From 1a160188628d7bcf4e884fdd5eae069d50a22a48 Mon Sep 17 00:00:00 2001
From: "srinivasan.narayanamoorthy" <srinivasan.narayanamoorthy@intel.com>
Date: Fri, 25 Jan 2019 11:40:43 -0800
Subject: [PATCH 1268/2345] incorporating review changes

---
 tensorflow/core/common_runtime/mkl_cpu_allocator.h | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/common_runtime/mkl_cpu_allocator.h b/tensorflow/core/common_runtime/mkl_cpu_allocator.h
index 640300e523..b2db8b66a4 100644
--- a/tensorflow/core/common_runtime/mkl_cpu_allocator.h
+++ b/tensorflow/core/common_runtime/mkl_cpu_allocator.h
@@ -96,6 +96,7 @@ class MklSmallSizeAllocator : public Allocator {
   // Increment statistics for the allocator handling small allocations.
   inline void IncrementStats(size_t alloc_size)
       EXCLUSIVE_LOCKS_REQUIRED(mutex_) {
+    mutex_lock l(mutex_);
     ++stats_.num_allocs;
     stats_.bytes_in_use += alloc_size;
     stats_.max_bytes_in_use =
@@ -107,6 +108,7 @@ class MklSmallSizeAllocator : public Allocator {
   // Decrement statistics for the allocator handling small allocations.
   inline void DecrementStats(size_t dealloc_size)
       EXCLUSIVE_LOCKS_REQUIRED(mutex_) {
+    mutex_lock l(mutex_);
     stats_.bytes_in_use -= dealloc_size;
   }
 
@@ -194,18 +196,20 @@ class MklCPUAllocator : public Allocator {
   }
 
   inline string Name() override { return kName; }
-  inline bool IsSmallSizeAllocation(const void* ptr) const {
+  inline bool IsSmallSizeAllocation(const void* ptr) const
+      LOCKS_EXCLUDED(mutex_) {
     mutex_lock l(mutex_);
     return large_allocations_map_.find(ptr) == large_allocations_map_.end();
   }
   // AddLargeAllocMap and RemoveLargeAllocMap are always called with a lock held
-  inline void AddLargeAllocMap(void* ptr, size_t num_bytes) {
+  inline void AddLargeAllocMap(void* ptr, size_t num_bytes)
+      EXCLUSIVE_LOCKS_REQUIRED(mutex_) {
     if (ptr != nullptr) {
       std::pair<void*, size_t> map_val(ptr, num_bytes);
-      large_allocations_map_.insert(map_val).second;
+      large_allocations_map_.insert(map_val);
     }
   }
-  inline void RemoveLargeAllocMap(void* ptr) {
+  inline void RemoveLargeAllocMap(void* ptr) EXCLUSIVE_LOCKS_REQUIRED(mutex_) {
     auto map_iter = large_allocations_map_.find(ptr);
     if (map_iter != large_allocations_map_.end()) {
       large_allocations_map_.erase(map_iter);
-- 
GitLab


From 609d8ba76333f325821968d657e3659af3891154 Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Fri, 25 Jan 2019 11:02:24 -0800
Subject: [PATCH 1269/2345] Use struct initialization to avoid setting the
 parameters explicitly.

PiperOrigin-RevId: 230932528
---
 tensorflow/lite/kernels/dequantize_test.cc | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/tensorflow/lite/kernels/dequantize_test.cc b/tensorflow/lite/kernels/dequantize_test.cc
index be7caa3189..77254335fb 100644
--- a/tensorflow/lite/kernels/dequantize_test.cc
+++ b/tensorflow/lite/kernels/dequantize_test.cc
@@ -30,13 +30,7 @@ class DequantizeOpModel : public SingleOpModel {
  public:
   DequantizeOpModel(TensorType type, std::initializer_list<int> shape,
                     float scale, int32_t zero_point) {
-    TensorData input_tensor_data;
-    input_tensor_data.type = type;
-    input_tensor_data.shape = shape;
-    input_tensor_data.min = 0;
-    input_tensor_data.max = 0;
-    input_tensor_data.scale = scale;
-    input_tensor_data.zero_point = zero_point;
+    const TensorData input_tensor_data = {type, shape, 0, 0, scale, zero_point};
     input_ = AddInput(input_tensor_data);
     output_ = AddOutput({TensorType_FLOAT32, shape});
     SetBuiltinOp(BuiltinOperator_DEQUANTIZE, BuiltinOptions_DequantizeOptions,
-- 
GitLab


From eef9a562cf9305877770b6aac1d59bf5b696d846 Mon Sep 17 00:00:00 2001
From: Adam Wood <adamwood@google.com>
Date: Fri, 25 Jan 2019 11:49:51 -0800
Subject: [PATCH 1270/2345] fix broken links and typo

---
 .../nmt_with_attention.ipynb                  | 399 +++++++++---------
 1 file changed, 195 insertions(+), 204 deletions(-)

diff --git a/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb b/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb
index 66d52a7494..436e887736 100644
--- a/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb
+++ b/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb
@@ -1,11 +1,28 @@
 {
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "name": "nmt_with_attention.ipynb",
+      "version": "0.3.2",
+      "provenance": [],
+      "private_outputs": true,
+      "collapsed_sections": [],
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "accelerator": "GPU"
+  },
   "cells": [
     {
-      "cell_type": "markdown",
       "metadata": {
         "colab_type": "text",
         "id": "AOpGoE2T-YXS"
       },
+      "cell_type": "markdown",
       "source": [
         "##### Copyright 2018 The TensorFlow Authors.\n",
         "\n",
@@ -13,19 +30,19 @@
         "\n",
         "# Neural Machine Translation with Attention\n",
         "\n",
-        "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\u003ctd\u003e\n",
-        "\u003ca target=\"_blank\"  href=\"https://colab.research.google.com/github/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb\"\u003e\n",
-        "    \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e  \n",
-        "\u003c/td\u003e\u003ctd\u003e\n",
-        "\u003ca target=\"_blank\"  href=\"https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb\"\u003e\u003cimg width=32px src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\u003c/td\u003e\u003c/table\u003e"
+        "<table class=\"tfo-notebook-buttons\" align=\"left\"><td>\n",
+        "<a target=\"_blank\"  href=\"https://colab.research.google.com/github/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb\">\n",
+        "    <img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />Run in Google Colab</a>  \n",
+        "</td><td>\n",
+        "<a target=\"_blank\"  href=\"https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb\"><img width=32px src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" />View source on GitHub</a></td></table>"
       ]
     },
     {
-      "cell_type": "markdown",
       "metadata": {
         "colab_type": "text",
         "id": "CiwtNgENbx2g"
       },
+      "cell_type": "markdown",
       "source": [
         "This notebook trains a sequence to sequence (seq2seq) model for Spanish to English translation using [tf.keras](https://www.tensorflow.org/programmers_guide/keras) and [eager execution](https://www.tensorflow.org/programmers_guide/eager). This is an advanced example that assumes some knowledge of sequence to sequence models.\n",
         "\n",
@@ -33,24 +50,22 @@
         "\n",
         "The translation quality is reasonable for a toy example, but the generated attention plot is perhaps more interesting. This shows which parts of the input sentence has the model's attention while translating:\n",
         "\n",
-        "\u003cimg src=\"https://tensorflow.org/images/spanish-english.png\" alt=\"spanish-english attention plot\"\u003e\n",
+        "<img src=\"https://tensorflow.org/images/spanish-english.png\" alt=\"spanish-english attention plot\">\n",
         "\n",
         "Note: This example takes approximately 10 mintues to run on a single P100 GPU."
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "tnxXKDjq3jEL"
+        "id": "tnxXKDjq3jEL",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "from __future__ import absolute_import, division, print_function\n",
         "\n",
-        "# Import TensorFlow \u003e= 1.10 and enable eager execution\n",
+        "# Import TensorFlow >= 1.10 and enable eager execution\n",
         "import tensorflow as tf\n",
         "\n",
         "tf.enable_eager_execution()\n",
@@ -65,14 +80,16 @@
         "import time\n",
         "\n",
         "print(tf.__version__)"
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "markdown",
       "metadata": {
         "colab_type": "text",
         "id": "wfodePkj3jEa"
       },
+      "cell_type": "markdown",
       "source": [
         "## Download and prepare the dataset\n",
         "\n",
@@ -91,14 +108,12 @@
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "kRVATYOgJs1b"
+        "id": "kRVATYOgJs1b",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "# Download the file\n",
         "path_to_zip = tf.keras.utils.get_file(\n",
@@ -106,17 +121,17 @@
         "    extract=True)\n",
         "\n",
         "path_to_file = os.path.dirname(path_to_zip)+\"/spa-eng/spa.txt\""
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "rd0jw-eC3jEh"
+        "id": "rd0jw-eC3jEh",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "# Converts the unicode file to ascii\n",
         "def unicode_to_ascii(s):\n",
@@ -128,7 +143,7 @@
         "    w = unicode_to_ascii(w.lower().strip())\n",
         "    \n",
         "    # creating a space between a word and the punctuation following it\n",
-        "    # eg: \"he is a boy.\" =\u003e \"he is a boy .\" \n",
+        "    # eg: \"he is a boy.\" => \"he is a boy .\" \n",
         "    # Reference:- https://stackoverflow.com/questions/3645931/python-padding-punctuation-with-white-spaces-keeping-punctuation\n",
         "    w = re.sub(r\"([?.!,¿])\", r\" \\1 \", w)\n",
         "    w = re.sub(r'[\" \"]+', \" \", w)\n",
@@ -140,19 +155,19 @@
         "    \n",
         "    # adding a start and an end token to the sentence\n",
         "    # so that the model know when to start and stop predicting.\n",
-        "    w = '\u003cstart\u003e ' + w + ' \u003cend\u003e'\n",
+        "    w = '<start> ' + w + ' <end>'\n",
         "    return w"
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "OHn4Dct23jEm"
+        "id": "OHn4Dct23jEm",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "# 1. Remove the accents\n",
         "# 2. Clean the sentences\n",
@@ -163,20 +178,20 @@
         "    word_pairs = [[preprocess_sentence(w) for w in l.split('\\t')]  for l in lines[:num_examples]]\n",
         "    \n",
         "    return word_pairs"
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "9xbqO7Iie9bb"
+        "id": "9xbqO7Iie9bb",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
-        "# This class creates a word -\u003e index mapping (e.g,. \"dad\" -\u003e 5) and vice-versa \n",
-        "# (e.g., 5 -\u003e \"dad\") for each language,\n",
+        "# This class creates a word -> index mapping (e.g,. \"dad\" -> 5) and vice-versa \n",
+        "# (e.g., 5 -> \"dad\") for each language,\n",
         "class LanguageIndex():\n",
         "  def __init__(self, lang):\n",
         "    self.lang = lang\n",
@@ -192,23 +207,23 @@
         "    \n",
         "    self.vocab = sorted(self.vocab)\n",
         "    \n",
-        "    self.word2idx['\u003cpad\u003e'] = 0\n",
+        "    self.word2idx['<pad>'] = 0\n",
         "    for index, word in enumerate(self.vocab):\n",
         "      self.word2idx[word] = index + 1\n",
         "    \n",
         "    for word, index in self.word2idx.items():\n",
         "      self.idx2word[index] = word"
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "eAY9k49G3jE_"
+        "id": "eAY9k49G3jE_",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "def max_length(tensor):\n",
         "    return max(len(t) for t in tensor)\n",
@@ -244,71 +259,71 @@
         "                                                                  padding='post')\n",
         "    \n",
         "    return input_tensor, target_tensor, inp_lang, targ_lang, max_length_inp, max_length_tar"
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "markdown",
       "metadata": {
         "colab_type": "text",
         "id": "GOi42V79Ydlr"
       },
+      "cell_type": "markdown",
       "source": [
         "### Limit the size of the dataset to experiment faster (optional)\n",
         "\n",
-        "Training on the complete dataset of \u003e100,000 sentences will take a long time. To train faster, we can limit the size of the dataset to 30,000 sentences (of course, translation quality degrades with less data):"
+        "Training on the complete dataset of >100,000 sentences will take a long time. To train faster, we can limit the size of the dataset to 30,000 sentences (of course, translation quality degrades with less data):"
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "cnxC7q-j3jFD"
+        "id": "cnxC7q-j3jFD",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "# Try experimenting with the size of that dataset\n",
         "num_examples = 30000\n",
         "input_tensor, target_tensor, inp_lang, targ_lang, max_length_inp, max_length_targ = load_dataset(path_to_file, num_examples)"
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "4QILQkOs3jFG"
+        "id": "4QILQkOs3jFG",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "# Creating training and validation sets using an 80-20 split\n",
         "input_tensor_train, input_tensor_val, target_tensor_train, target_tensor_val = train_test_split(input_tensor, target_tensor, test_size=0.2)\n",
         "\n",
         "# Show length\n",
         "len(input_tensor_train), len(target_tensor_train), len(input_tensor_val), len(target_tensor_val)"
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "markdown",
       "metadata": {
         "colab_type": "text",
         "id": "rgCLkfv5uO3d"
       },
+      "cell_type": "markdown",
       "source": [
         "### Create a tf.data dataset"
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "TqHsArVZ3jFS"
+        "id": "TqHsArVZ3jFS",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "BUFFER_SIZE = len(input_tensor_train)\n",
         "BATCH_SIZE = 64\n",
@@ -320,27 +335,29 @@
         "\n",
         "dataset = tf.data.Dataset.from_tensor_slices((input_tensor_train, target_tensor_train)).shuffle(BUFFER_SIZE)\n",
         "dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)"
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "markdown",
       "metadata": {
         "colab_type": "text",
         "id": "TNfHIF71ulLu"
       },
+      "cell_type": "markdown",
       "source": [
         "## Write the encoder and decoder model\n",
         "\n",
-        "Here, we'll implement an encoder-decoder model with attention which you can read about in the TensorFlow [Neural Machine Translation (seq2seq) tutorial](https://www.tensorflow.org/tutorials/seq2seq). This example uses a more recent set of APIs. This notebook implements the [attention equations](https://www.tensorflow.org/tutorials/seq2seq#background_on_the_attention_mechanism) from the seq2seq tutorial. The following diagram shows that each input words is assigned a weight by the attention mechanism which is then used by the decoder to predict the next word in the sentence.\n",
+        "Here, we'll implement an encoder-decoder model with attention which you can read about in the TensorFlow [Neural Machine Translation (seq2seq) tutorial](https://github.com/tensorflow/nmt). This example uses a more recent set of APIs. This notebook implements the [attention equations](https://github.com/tensorflow/nmt#background-on-the-attention-mechanism) from the seq2seq tutorial. The following diagram shows that each input word is assigned a weight by the attention mechanism which is then used by the decoder to predict the next word in the sentence.\n",
         "\n",
-        "\u003cimg src=\"https://www.tensorflow.org/images/seq2seq/attention_mechanism.jpg\" width=\"500\" alt=\"attention mechanism\"\u003e\n",
+        "<img src=\"https://www.tensorflow.org/images/seq2seq/attention_mechanism.jpg\" width=\"500\" alt=\"attention mechanism\">\n",
         "\n",
         "The input is put through an encoder model which gives us the encoder output of shape *(batch_size, max_length, hidden_size)* and the encoder hidden state of shape *(batch_size, hidden_size)*. \n",
         "\n",
         "Here are the equations that are implemented:\n",
         "\n",
-        "\u003cimg src=\"https://www.tensorflow.org/images/seq2seq/attention_equation_0.jpg\" alt=\"attention equation 0\" width=\"800\"\u003e\n",
-        "\u003cimg src=\"https://www.tensorflow.org/images/seq2seq/attention_equation_1.jpg\" alt=\"attention equation 1\" width=\"800\"\u003e\n",
+        "<img src=\"https://www.tensorflow.org/images/seq2seq/attention_equation_0.jpg\" alt=\"attention equation 0\" width=\"800\">\n",
+        "<img src=\"https://www.tensorflow.org/images/seq2seq/attention_equation_1.jpg\" alt=\"attention equation 1\" width=\"800\">\n",
         "\n",
         "We're using *Bahdanau attention*. Lets decide on notation before writing the simplified form:\n",
         "\n",
@@ -362,14 +379,12 @@
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "avyJ_4VIUoHb"
+        "id": "avyJ_4VIUoHb",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "def gru(units):\n",
         "  # If you have a GPU, we recommend using CuDNNGRU(provides a 3x speedup than GRU)\n",
@@ -385,17 +400,17 @@
         "                               return_state=True, \n",
         "                               recurrent_activation='sigmoid', \n",
         "                               recurrent_initializer='glorot_uniform')"
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "nZ2rI24i3jFg"
+        "id": "nZ2rI24i3jFg",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "class Encoder(tf.keras.Model):\n",
         "    def __init__(self, vocab_size, embedding_dim, enc_units, batch_sz):\n",
@@ -412,17 +427,17 @@
         "    \n",
         "    def initialize_hidden_state(self):\n",
         "        return tf.zeros((self.batch_sz, self.enc_units))"
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "yJ_B3mhW3jFk"
+        "id": "yJ_B3mhW3jFk",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "class Decoder(tf.keras.Model):\n",
         "    def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz):\n",
@@ -476,41 +491,41 @@
         "        \n",
         "    def initialize_hidden_state(self):\n",
         "        return tf.zeros((self.batch_sz, self.dec_units))"
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "P5UY8wko3jFp"
+        "id": "P5UY8wko3jFp",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "encoder = Encoder(vocab_inp_size, embedding_dim, units, BATCH_SIZE)\n",
         "decoder = Decoder(vocab_tar_size, embedding_dim, units, BATCH_SIZE)"
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "markdown",
       "metadata": {
         "colab_type": "text",
         "id": "_ch_71VbIRfK"
       },
+      "cell_type": "markdown",
       "source": [
         "## Define the optimizer and the loss function"
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "WmTHr5iV3jFr"
+        "id": "WmTHr5iV3jFr",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "optimizer = tf.train.AdamOptimizer()\n",
         "\n",
@@ -519,41 +534,43 @@
         "  mask = 1 - np.equal(real, 0)\n",
         "  loss_ = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=real, logits=pred) * mask\n",
         "  return tf.reduce_mean(loss_)"
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "markdown",
       "metadata": {
         "colab_type": "text",
         "id": "DMVWzzsfNl4e"
       },
+      "cell_type": "markdown",
       "source": [
         "## Checkpoints (Object-based saving)"
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "Zj8bXQTgNwrF"
+        "id": "Zj8bXQTgNwrF",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "checkpoint_dir = './training_checkpoints'\n",
         "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n",
         "checkpoint = tf.train.Checkpoint(optimizer=optimizer,\n",
         "                                 encoder=encoder,\n",
         "                                 decoder=decoder)"
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "markdown",
       "metadata": {
         "colab_type": "text",
         "id": "hpObfY22IddU"
       },
+      "cell_type": "markdown",
       "source": [
         "## Training\n",
         "\n",
@@ -567,14 +584,12 @@
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "ddefjBMa3jF0"
+        "id": "ddefjBMa3jF0",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "EPOCHS = 10\n",
         "\n",
@@ -592,7 +607,7 @@
         "            \n",
         "            dec_hidden = enc_hidden\n",
         "            \n",
-        "            dec_input = tf.expand_dims([targ_lang.word2idx['\u003cstart\u003e']] * BATCH_SIZE, 1)       \n",
+        "            dec_input = tf.expand_dims([targ_lang.word2idx['<start>']] * BATCH_SIZE, 1)       \n",
         "            \n",
         "            # Teacher forcing - feeding the target as the next input\n",
         "            for t in range(1, targ.shape[1]):\n",
@@ -625,14 +640,16 @@
         "    print('Epoch {} Loss {:.4f}'.format(epoch + 1,\n",
         "                                        total_loss / N_BATCH))\n",
         "    print('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))"
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "markdown",
       "metadata": {
         "colab_type": "text",
         "id": "mU3Ce8M6I3rz"
       },
+      "cell_type": "markdown",
       "source": [
         "## Translate\n",
         "\n",
@@ -644,14 +661,12 @@
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "EbQpyYs13jF_"
+        "id": "EbQpyYs13jF_",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "def evaluate(sentence, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ):\n",
         "    attention_plot = np.zeros((max_length_targ, max_length_inp))\n",
@@ -668,7 +683,7 @@
         "    enc_out, enc_hidden = encoder(inputs, hidden)\n",
         "\n",
         "    dec_hidden = enc_hidden\n",
-        "    dec_input = tf.expand_dims([targ_lang.word2idx['\u003cstart\u003e']], 0)\n",
+        "    dec_input = tf.expand_dims([targ_lang.word2idx['<start>']], 0)\n",
         "\n",
         "    for t in range(max_length_targ):\n",
         "        predictions, dec_hidden, attention_weights = decoder(dec_input, dec_hidden, enc_out)\n",
@@ -681,24 +696,24 @@
         "\n",
         "        result += targ_lang.idx2word[predicted_id] + ' '\n",
         "\n",
-        "        if targ_lang.idx2word[predicted_id] == '\u003cend\u003e':\n",
+        "        if targ_lang.idx2word[predicted_id] == '<end>':\n",
         "            return result, sentence, attention_plot\n",
         "        \n",
         "        # the predicted ID is fed back into the model\n",
         "        dec_input = tf.expand_dims([predicted_id], 0)\n",
         "\n",
         "    return result, sentence, attention_plot"
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "s5hQWlbN3jGF"
+        "id": "s5hQWlbN3jGF",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "# function for plotting the attention weights\n",
         "def plot_attention(attention, sentence, predicted_sentence):\n",
@@ -712,17 +727,17 @@
         "    ax.set_yticklabels([''] + predicted_sentence, fontdict=fontdict)\n",
         "\n",
         "    plt.show()"
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "sl9zUHzg3jGI"
+        "id": "sl9zUHzg3jGI",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "def translate(sentence, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ):\n",
         "    result, sentence, attention_plot = evaluate(sentence, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)\n",
@@ -732,91 +747,93 @@
         "    \n",
         "    attention_plot = attention_plot[:len(result.split(' ')), :len(sentence.split(' '))]\n",
         "    plot_attention(attention_plot, sentence.split(' '), result.split(' '))"
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "markdown",
       "metadata": {
         "colab_type": "text",
         "id": "n250XbnjOaqP"
       },
+      "cell_type": "markdown",
       "source": [
         "## Restore the latest checkpoint and test"
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "UJpT9D5_OgP6"
+        "id": "UJpT9D5_OgP6",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "# restoring the latest checkpoint in checkpoint_dir\n",
         "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))"
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "WrAM0FDomq3E"
+        "id": "WrAM0FDomq3E",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "translate(u'hace mucho frio aqui.', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)"
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "zSx2iM36EZQZ"
+        "id": "zSx2iM36EZQZ",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "translate(u'esta es mi vida.', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)"
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "A3LLCx3ZE0Ls"
+        "id": "A3LLCx3ZE0Ls",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "translate(u'todavia estan en casa?', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)"
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "DUQVLVqUE1YW"
+        "id": "DUQVLVqUE1YW",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "# wrong translation\n",
         "translate(u'trata de averiguarlo.', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)"
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "markdown",
       "metadata": {
         "colab_type": "text",
         "id": "RTe5P5ioMJwN"
       },
+      "cell_type": "markdown",
       "source": [
         "## Next steps\n",
         "\n",
@@ -824,31 +841,5 @@
         "* Experiment with training on a larger dataset, or using more epochs\n"
       ]
     }
-  ],
-  "metadata": {
-    "accelerator": "GPU",
-    "colab": {
-      "collapsed_sections": [],
-      "name": "nmt_with_attention.ipynb",
-      "private_outputs": true,
-      "provenance": [
-        {
-          "file_id": "1C4fpM7_7IL8ZzF7Gc5abywqQjeQNS2-U",
-          "timestamp": 1527858391290
-        },
-        {
-          "file_id": "1pExo6aUuw0S6MISFWoinfJv0Ftm9V4qv",
-          "timestamp": 1527776041613
-        }
-      ],
-      "toc_visible": true,
-      "version": "0.3.2"
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}
+  ]
+}
\ No newline at end of file
-- 
GitLab


From c8e8f35f3e39b36d105eb7e43321a9da1362f242 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 25 Jan 2019 11:03:21 -0800
Subject: [PATCH 1271/2345] Fix integer overflow bug

PiperOrigin-RevId: 230932821
---
 tensorflow/core/platform/env_time.h            |  1 +
 tensorflow/core/profiler/lib/eager_profiler.cc | 17 ++++++++++++-----
 tensorflow/core/profiler/lib/eager_profiler.h  |  1 +
 3 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/tensorflow/core/platform/env_time.h b/tensorflow/core/platform/env_time.h
index b4756ed209..c12b6ba6fb 100644
--- a/tensorflow/core/platform/env_time.h
+++ b/tensorflow/core/platform/env_time.h
@@ -25,6 +25,7 @@ namespace tensorflow {
 /// access timer related operations.
 class EnvTime {
  public:
+  static constexpr uint64 kMicrosToPicos = 1000ULL * 1000ULL;
   static constexpr uint64 kMicrosToNanos = 1000ULL;
   static constexpr uint64 kMillisToMicros = 1000ULL;
   static constexpr uint64 kMillisToNanos = 1000ULL * 1000ULL;
diff --git a/tensorflow/core/profiler/lib/eager_profiler.cc b/tensorflow/core/profiler/lib/eager_profiler.cc
index ab97682cd8..9293e7aba9 100644
--- a/tensorflow/core/profiler/lib/eager_profiler.cc
+++ b/tensorflow/core/profiler/lib/eager_profiler.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include "tensorflow/core/platform/device_tracer.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/platform/types.h"
 #include "tensorflow/core/protobuf/config.pb.h"
 
 namespace tensorflow {
@@ -29,7 +30,8 @@ namespace tensorflow {
 namespace {
 
 void ConvertRunMetadataToTraceEvent(RunMetadata* run_metadata,
-                                    tpu::Trace* trace) {
+                                    tpu::Trace* trace,
+                                    const uint64 profile_start_time_micros) {
   auto trace_devices = trace->mutable_devices();
   // TODO(fishx): use a lighter representation instead of GraphDef to insert
   // python information into trace event.
@@ -56,8 +58,11 @@ void ConvertRunMetadataToTraceEvent(RunMetadata* run_metadata,
       event->set_device_id(device_id);
       event->set_resource_id(0);
       event->set_name(node.node_name());
-      event->set_timestamp_ps(node.all_start_micros() * 1000000);
-      event->set_duration_ps(node.all_end_rel_micros() * 1000000);
+      event->set_timestamp_ps(
+          (node.all_start_micros() - profile_start_time_micros) *
+          EnvTime::kMicrosToPicos);
+      event->set_duration_ps(node.all_end_rel_micros() *
+                             EnvTime::kMicrosToPicos);
       (*args)["label"] = node.timeline_label();
     }
   }
@@ -102,13 +107,15 @@ Status EagerProfiler::SerializeToString(string* content) {
 
   tpu::Trace trace;
 
-  ConvertRunMetadataToTraceEvent(&run_metadata_, &trace);
+  ConvertRunMetadataToTraceEvent(&run_metadata_, &trace, start_time_micros_);
 
   trace.SerializeToString(content);
   return Status::OK();
 }
 
-EagerProfiler::EagerProfiler(EagerContext* const context) : context_(context) {
+EagerProfiler::EagerProfiler(EagerContext* const context)
+    : context_(context),
+      start_time_micros_(Env::Default()->NowNanos() / EnvTime::kMicrosToNanos) {
   LOG(INFO) << "Eager Profiler started.";
 
   status_ = context_->RegisterRunMetadataListener(this);
diff --git a/tensorflow/core/profiler/lib/eager_profiler.h b/tensorflow/core/profiler/lib/eager_profiler.h
index 62df869e94..7cdb76ff36 100644
--- a/tensorflow/core/profiler/lib/eager_profiler.h
+++ b/tensorflow/core/profiler/lib/eager_profiler.h
@@ -58,6 +58,7 @@ class EagerProfiler : RunMetadataListener {
   tensorflow::Status status_ GUARDED_BY(mutex_);
   std::unique_ptr<DeviceTracer> device_tracer_ GUARDED_BY(mutex_);
   EagerContext* context_ GUARDED_BY(mutex_);
+  const uint64 start_time_micros_;
   mutex mutex_;
 };
 
-- 
GitLab


From dc1dc242f9d33ca59ec5036ca6d4248e5e818b30 Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Fri, 25 Jan 2019 11:03:45 -0800
Subject: [PATCH 1272/2345] Fix index returned by TensorListReserve

TensorListReserve should set the index to the end of the list to match
the behaviour of the regular op. As a result, we change the existing
tests to use empty_tensor_list to get their desired behavior.

PiperOrigin-RevId: 230932925
---
 .../compiler/tests/tensor_list_ops_test.py    | 32 +++++++++----------
 .../tf2xla/kernels/tensor_list_ops.cc         |  2 +-
 2 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/tensorflow/compiler/tests/tensor_list_ops_test.py b/tensorflow/compiler/tests/tensor_list_ops_test.py
index 1ecdb22cd0..805e36a3b9 100644
--- a/tensorflow/compiler/tests/tensor_list_ops_test.py
+++ b/tensorflow/compiler/tests/tensor_list_ops_test.py
@@ -23,24 +23,20 @@ from tensorflow.compiler.tests import xla_test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
-from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import list_ops
 from tensorflow.python.platform import test
 
 
-def scalar_shape():
-  return ops.convert_to_tensor([], dtype=dtypes.int32)
-
-
 class ListOpsTest(xla_test.XLATestCase):
 
   def testElementShape(self):
     with self.cached_session() as sess, self.test_scope():
       dim = array_ops.placeholder(dtypes.int32)
-      l = list_ops.tensor_list_reserve(
-          element_shape=(dim, 15), num_elements=20,
-          element_dtype=dtypes.float32)
+      l = list_ops.empty_tensor_list(
+          element_shape=(dim, 15),
+          element_dtype=dtypes.float32,
+          max_num_elements=20)
       e32 = list_ops.tensor_list_element_shape(l, shape_type=dtypes.int32)
       e64 = list_ops.tensor_list_element_shape(l, shape_type=dtypes.int64)
       self.assertAllEqual(sess.run(e32, {dim: 10}), (10, 15))
@@ -48,8 +44,10 @@ class ListOpsTest(xla_test.XLATestCase):
 
   def testPushPop(self):
     with self.cached_session() as sess, self.test_scope():
-      l = list_ops.tensor_list_reserve(
-          element_shape=(7, 15), num_elements=10, element_dtype=dtypes.float32)
+      l = list_ops.empty_tensor_list(
+          element_shape=(7, 15),
+          element_dtype=dtypes.float32,
+          max_num_elements=10)
       l = list_ops.tensor_list_push_back(
           l, constant_op.constant(1.0, shape=(7, 15)))
       l = list_ops.tensor_list_push_back(
@@ -62,8 +60,10 @@ class ListOpsTest(xla_test.XLATestCase):
   def testDoNotConstantFoldVariants(self):
     with self.cached_session() as sess, self.test_scope():
       val = array_ops.placeholder(dtype=dtypes.float32)
-      l = list_ops.tensor_list_reserve(
-          element_shape=(7, 15), num_elements=10, element_dtype=dtypes.float32)
+      l = list_ops.empty_tensor_list(
+          element_shape=(7, 15),
+          element_dtype=dtypes.float32,
+          max_num_elements=10)
       # Note: Pushing a Placeholder will force the constant folding code
       # to build a Const node with a DT_VARIANT output. This tests that XLA
       # passes a cf_consider_fn which prevent folding such nodes.
@@ -78,10 +78,10 @@ class ListOpsTest(xla_test.XLATestCase):
 
   def testPushPopSeparateLists(self):
     with self.cached_session() as sess, self.test_scope():
-      l = list_ops.tensor_list_reserve(
-          element_shape=scalar_shape(),
-          num_elements=20,
-          element_dtype=dtypes.float32)
+      l = list_ops.empty_tensor_list(
+          element_shape=[],
+          element_dtype=dtypes.float32,
+          max_num_elements=20)
       l = list_ops.tensor_list_push_back(l, constant_op.constant(1.0))
       l2 = list_ops.tensor_list_push_back(l, constant_op.constant(2.0))
       l3 = list_ops.tensor_list_push_back(l, constant_op.constant(3.0))
diff --git a/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc b/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc
index 6a68e0b75e..be29765ee3 100644
--- a/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc
@@ -70,7 +70,7 @@ class TensorListReserveOp : public XlaOpKernel {
     ctx->SetTensorListOutput(
         0, xla::Tuple(b, {xla::Broadcast(XlaHelpers::Zero(b, dtype_),
                                          tensor_shape.dim_sizes()),
-                          xla::ConstantR0<int32>(b, 0)}));
+                          xla::ConstantR0<int32>(b, num_elements)}));
   }
 
  private:
-- 
GitLab


From 5522d670af8efa43125325d741793eca12eb16f4 Mon Sep 17 00:00:00 2001
From: Yanan Cao <ycao@google.com>
Date: Fri, 25 Jan 2019 11:13:55 -0800
Subject: [PATCH 1273/2345] Internal change.

PiperOrigin-RevId: 230935032
---
 tensorflow/contrib/distribute/python/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index fb8732d591..fbfaa8e549 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -633,6 +633,7 @@ distribute_py_test(
         "no_oss",  # TODO(b/117919883): Fix python error.
         "no_pip",
         "no_windows_gpu",
+        "notap",  # TODO(123409374): Re-enable this test in TAP.
         "notsan",
     ],
     deps = [
-- 
GitLab


From 0666341fa0451f045b961d7d0c5c13ead4622877 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 25 Jan 2019 11:15:50 -0800
Subject: [PATCH 1274/2345] Updates Result codes per upstream, and fixes a typo
 in ANEURALNETWORKS_UNMAPPABLE.

PiperOrigin-RevId: 230935362
---
 tensorflow/lite/nnapi/NeuralNetworksTypes.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/nnapi/NeuralNetworksTypes.h b/tensorflow/lite/nnapi/NeuralNetworksTypes.h
index de8b84a823..9291391491 100644
--- a/tensorflow/lite/nnapi/NeuralNetworksTypes.h
+++ b/tensorflow/lite/nnapi/NeuralNetworksTypes.h
@@ -115,8 +115,10 @@ enum {
   ANEURALNETWORKS_UNEXPECTED_NULL = 3,
   ANEURALNETWORKS_BAD_DATA = 4,
   ANEURALNETWORKS_OP_FAILED = 5,
-  ANEURALNETWORKS_UNMAPPABLE = 5,
   ANEURALNETWORKS_BAD_STATE = 6,
+  ANEURALNETWORKS_UNMAPPABLE = 7,
+  ANEURALNETWORKS_OUTPUT_INSUFFICIENT_SIZE = 8,
+  ANEURALNETWORKS_UNAVAILABLE_DEVICE = 9,
 };
 
 /**
-- 
GitLab


From a67b8e75f224784a43d01d5fc6e4f7801b0fb3da Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 25 Jan 2019 11:21:36 -0800
Subject: [PATCH 1275/2345] Add two methods to Env: GetCurrentThreadId and
 GetCurrentThreadName;

PiperOrigin-RevId: 230936389
---
 tensorflow/core/BUILD                   |  2 +-
 tensorflow/core/platform/env.cc         | 16 ++------------
 tensorflow/core/platform/env.h          | 13 ++++++++++++
 tensorflow/core/platform/env_test.cc    | 14 +++++++++++++
 tensorflow/core/platform/posix/env.cc   | 28 +++++++++++++++++++++++++
 tensorflow/core/platform/windows/env.cc |  6 ++++++
 6 files changed, 64 insertions(+), 15 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 7652f03a9f..c3d31f91b3 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -506,6 +506,7 @@ cc_library(
         ":platform_port",
         ":platform_protobuf",
         "//tensorflow/core/platform/default/build_config:env",
+        "//tensorflow/core/platform/default/build_config:port",
     ],
 )
 
@@ -2220,7 +2221,6 @@ cc_library(
             "lib/**/*.cc",
             "platform/*.cc",
             "platform/profile_utils/**/*.cc",
-        ] + [
             "framework/resource_handle.cc",
             "util/env_var.cc",
         ],
diff --git a/tensorflow/core/platform/env.cc b/tensorflow/core/platform/env.cc
index 6339417445..59768bf92a 100644
--- a/tensorflow/core/platform/env.cc
+++ b/tensorflow/core/platform/env.cc
@@ -365,22 +365,10 @@ bool Env::LocalTempFilename(string* filename) {
 }
 
 bool Env::CreateUniqueFileName(string* prefix, const string& suffix) {
-#ifdef __APPLE__
-  uint64_t tid64;
-  pthread_threadid_np(nullptr, &tid64);
-  int32 tid = static_cast<int32>(tid64);
-  int32 pid = static_cast<int32>(getpid());
-#elif defined(__FreeBSD__)
-  // Has to be casted to long first, else this error appears:
-  // static_cast from 'pthread_t' (aka 'pthread *') to 'int32' (aka 'int')
-  // is not allowed
-  int32 tid = static_cast<int32>(static_cast<int64>(pthread_self()));
-  int32 pid = static_cast<int32>(getpid());
-#elif defined(PLATFORM_WINDOWS)
-  int32 tid = static_cast<int32>(GetCurrentThreadId());
+  int32 tid = GetCurrentThreadId();
+#ifdef PLATFORM_WINDOWS
   int32 pid = static_cast<int32>(GetCurrentProcessId());
 #else
-  int32 tid = static_cast<int32>(pthread_self());
   int32 pid = static_cast<int32>(getpid());
 #endif
   uint64 now_microsec = NowMicros();
diff --git a/tensorflow/core/platform/env.h b/tensorflow/core/platform/env.h
index 1b53828415..280076e098 100644
--- a/tensorflow/core/platform/env.h
+++ b/tensorflow/core/platform/env.h
@@ -271,6 +271,15 @@ class Env {
                               const string& name,
                               std::function<void()> fn) TF_MUST_USE_RESULT = 0;
 
+  // Returns the thread id of calling thread.
+  // Posix: Returns pthread id which is only guaranteed to be unique within a
+  //        process.
+  // Windows: Returns thread id which is unique.
+  virtual int32 GetCurrentThreadId() = 0;
+
+  // Copies current thread name to "name". Returns true if success.
+  virtual bool GetCurrentThreadName(string* name) = 0;
+
   // \brief Schedules the given closure on a thread-pool.
   //
   // NOTE(mrry): This closure may block.
@@ -360,6 +369,10 @@ class EnvWrapper : public Env {
                       std::function<void()> fn) override {
     return target_->StartThread(thread_options, name, fn);
   }
+  int32 GetCurrentThreadId() override { return target_->GetCurrentThreadId(); }
+  bool GetCurrentThreadName(string* name) override {
+    return target_->GetCurrentThreadName(name);
+  }
   void SchedClosure(std::function<void()> closure) override {
     target_->SchedClosure(closure);
   }
diff --git a/tensorflow/core/platform/env_test.cc b/tensorflow/core/platform/env_test.cc
index 2e1d4a263f..a33f592096 100644
--- a/tensorflow/core/platform/env_test.cc
+++ b/tensorflow/core/platform/env_test.cc
@@ -392,4 +392,18 @@ TEST_F(DefaultEnvTest, CreateUniqueFileName) {
   EXPECT_TRUE(str_util::EndsWith(filename, suffix));
 }
 
+TEST_F(DefaultEnvTest, GetThreadInformation) {
+  Env* env = Env::Default();
+
+  EXPECT_GT(env->GetCurrentThreadId(), 0);
+  string thread_name;
+  bool res = env->GetCurrentThreadName(&thread_name);
+#if defined(PLATFORM_WINDOWS) || defined(__ANDROID__)
+  EXPECT_FALSE(res);
+#else
+  EXPECT_TRUE(res);
+  EXPECT_GT(thread_name.size(), 0);
+#endif
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/platform/posix/env.cc b/tensorflow/core/platform/posix/env.cc
index d87e5dcfe7..636da3b737 100644
--- a/tensorflow/core/platform/posix/env.cc
+++ b/tensorflow/core/platform/posix/env.cc
@@ -86,6 +86,34 @@ class PosixEnv : public Env {
     return new StdThread(thread_options, name, fn);
   }
 
+  int32 GetCurrentThreadId() {
+#ifdef __APPLE__
+    pthread_threadid_np(nullptr, &tid64);
+    return static_cast<int32>(tid64);
+#elif defined(__FreeBSD__)
+    // Has to be casted to long first, else this error appears:
+    // static_cast from 'pthread_t' (aka 'pthread *') to 'int32' (aka 'int')
+    // is not allowed
+    return static_cast<int32>(static_cast<int64>(pthread_self()));
+#else
+    return static_cast<int32>(pthread_self());
+#endif
+  }
+
+  bool GetCurrentThreadName(string* name) {
+#ifdef __ANDROID__
+    return false;
+#else
+    char buf[100];
+    int res = pthread_getname_np(pthread_self(), buf, static_cast<size_t>(100));
+    if (res != 0) {
+      return false;
+    }
+    *name = buf;
+    return true;
+#endif
+  }
+
   void SchedClosure(std::function<void()> closure) override {
     // TODO(b/27290852): Spawning a new thread here is wasteful, but
     // needed to deal with the fact that many `closure` functions are
diff --git a/tensorflow/core/platform/windows/env.cc b/tensorflow/core/platform/windows/env.cc
index 77ce2026d9..e0e3dda705 100644
--- a/tensorflow/core/platform/windows/env.cc
+++ b/tensorflow/core/platform/windows/env.cc
@@ -84,6 +84,12 @@ class WindowsEnv : public Env {
     return new StdThread(thread_options, name, fn);
   }
 
+  int32 GetCurrentThreadId() override {
+    return static_cast<int32>(::GetCurrentThreadId());
+  }
+
+  bool GetCurrentThreadName(string* name) override { return false; }
+
   static VOID CALLBACK SchedClosureCallback(PTP_CALLBACK_INSTANCE Instance,
                                             PVOID Context, PTP_WORK Work) {
     CloseThreadpoolWork(Work);
-- 
GitLab


From 24e3ad9325d6f71d5d17cf4416b581f4eba7f8dd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 25 Jan 2019 11:36:03 -0800
Subject: [PATCH 1276/2345] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 230939336

---
 tensorflow/go/op/wrappers.go | 14968 ++++++++++++++++-----------------
 1 file changed, 7484 insertions(+), 7484 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 8e2c98fb07..c95c8c90b5 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -6627,65 +6627,17 @@ func GetSessionHandle(scope *Scope, value tf.Output) (handle tf.Output) {
 	return op.Output(0)
 }
 
-// Compute the regularized incomplete beta integral \\(I_x(a, b)\\).
-//
-// The regularized incomplete beta integral is defined as:
-//
-//
-// \\(I_x(a, b) = \frac{B(x; a, b)}{B(a, b)}\\)
-//
-// where
-//
-//
-// \\(B(x; a, b) = \int_0^x t^{a-1} (1 - t)^{b-1} dt\\)
-//
-//
-// is the incomplete beta function and \\(B(a, b)\\) is the *complete*
-// beta function.
-func Betainc(scope *Scope, a tf.Output, b tf.Output, x tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Betainc",
-		Input: []tf.Input{
-			a, b, x,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Return a tensor with the same shape and contents as the input tensor or value.
-func Identity(scope *Scope, input tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Identity",
-		Input: []tf.Input{
-			input,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes arctangent of `y/x` element-wise, respecting signs of the arguments.
+// Deprecated. Use TensorArraySizeV3
 //
-// This is the angle \( \theta \in [-\pi, \pi] \) such that
-// \[ x = r \cos(\theta) \]
-// and
-// \[ y = r \sin(\theta) \]
-// where \(r = \sqrt(x^2 + y^2) \).
-func Atan2(scope *Scope, y tf.Output, x tf.Output) (z tf.Output) {
+// DEPRECATED at GraphDef version 26: Use TensorArraySizeV3
+func TensorArraySizeV2(scope *Scope, handle tf.Output, flow_in tf.Output) (size tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Atan2",
+		Type: "TensorArraySizeV2",
 		Input: []tf.Input{
-			y, x,
+			handle, flow_in,
 		},
 	}
 	op := scope.AddOperation(opspec)
@@ -8144,60 +8096,6 @@ func ExperimentalDenseToSparseBatchDataset(scope *Scope, input_dataset tf.Output
 	return op.Output(0)
 }
 
-// ComplexAbsAttr is an optional argument to ComplexAbs.
-type ComplexAbsAttr func(optionalAttr)
-
-// ComplexAbsTout sets the optional Tout attribute to value.
-// If not specified, defaults to DT_FLOAT
-func ComplexAbsTout(value tf.DataType) ComplexAbsAttr {
-	return func(m optionalAttr) {
-		m["Tout"] = value
-	}
-}
-
-// Computes the complex absolute value of a tensor.
-//
-// Given a tensor `x` of complex numbers, this operation returns a tensor of type
-// `float` or `double` that is the absolute value of each element in `x`. All
-// elements in `x` must be complex numbers of the form \\(a + bj\\). The absolute
-// value is computed as \\( \sqrt{a^2 + b^2}\\).
-func ComplexAbs(scope *Scope, x tf.Output, optional ...ComplexAbsAttr) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "ComplexAbs",
-		Input: []tf.Input{
-			x,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Returns the truth value of x AND y element-wise.
-//
-// *NOTE*: `LogicalAnd` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func LogicalAnd(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "LogicalAnd",
-		Input: []tf.Input{
-			x, y,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // CastAttr is an optional argument to Cast.
 type CastAttr func(optionalAttr)
 
@@ -8302,6 +8200,60 @@ func RegexReplace(scope *Scope, input tf.Output, pattern tf.Output, rewrite tf.O
 	return op.Output(0)
 }
 
+// ComplexAbsAttr is an optional argument to ComplexAbs.
+type ComplexAbsAttr func(optionalAttr)
+
+// ComplexAbsTout sets the optional Tout attribute to value.
+// If not specified, defaults to DT_FLOAT
+func ComplexAbsTout(value tf.DataType) ComplexAbsAttr {
+	return func(m optionalAttr) {
+		m["Tout"] = value
+	}
+}
+
+// Computes the complex absolute value of a tensor.
+//
+// Given a tensor `x` of complex numbers, this operation returns a tensor of type
+// `float` or `double` that is the absolute value of each element in `x`. All
+// elements in `x` must be complex numbers of the form \\(a + bj\\). The absolute
+// value is computed as \\( \sqrt{a^2 + b^2}\\).
+func ComplexAbs(scope *Scope, x tf.Output, optional ...ComplexAbsAttr) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ComplexAbs",
+		Input: []tf.Input{
+			x,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Returns the truth value of x AND y element-wise.
+//
+// *NOTE*: `LogicalAnd` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func LogicalAnd(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "LogicalAnd",
+		Input: []tf.Input{
+			x, y,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Quantized Batch normalization.
 //
 // This op is deprecated and will be removed in the future. Prefer
@@ -9563,77 +9515,6 @@ func Mod(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	return op.Output(0)
 }
 
-// Computes offsets of concat inputs within its output.
-//
-// For example:
-//
-// ```
-// # 'x' is [2, 2, 7]
-// # 'y' is [2, 3, 7]
-// # 'z' is [2, 5, 7]
-// concat_offset(2, [x, y, z]) => [0, 0, 0], [0, 2, 0], [0, 5, 0]
-// ```
-//
-// This is typically used by gradient computations for a concat operation.
-//
-// Arguments:
-//	concat_dim: The dimension along which to concatenate.
-//	shape: The `N` int32 vectors representing shape of tensors being concatenated.
-//
-// Returns The `N` int32 vectors representing the starting offset
-// of input tensors within the concatenated output.
-func ConcatOffset(scope *Scope, concat_dim tf.Output, shape []tf.Output) (offset []tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ConcatOffset",
-		Input: []tf.Input{
-			concat_dim, tf.OutputList(shape),
-		},
-	}
-	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if offset, idx, err = makeOutputList(op, idx, "offset"); err != nil {
-		scope.UpdateErr("ConcatOffset", err)
-		return
-	}
-	return offset
-}
-
-// Compute the lower regularized incomplete Gamma function `P(a, x)`.
-//
-// The lower regularized incomplete Gamma function is defined as:
-//
-//
-// \\(P(a, x) = gamma(a, x) / Gamma(a) = 1 - Q(a, x)\\)
-//
-// where
-//
-// \\(gamma(a, x) = \\int_{0}^{x} t^{a-1} exp(-t) dt\\)
-//
-// is the lower incomplete Gamma function.
-//
-// Note, above `Q(a, x)` (`Igammac`) is the upper regularized complete
-// Gamma function.
-func Igamma(scope *Scope, a tf.Output, x tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Igamma",
-		Input: []tf.Input{
-			a, x,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // DepthToSpaceAttr is an optional argument to DepthToSpace.
 type DepthToSpaceAttr func(optionalAttr)
 
@@ -10180,14 +10061,121 @@ func CropAndResize(scope *Scope, image tf.Output, boxes tf.Output, box_ind tf.Ou
 	return op.Output(0)
 }
 
-// Fills empty rows in the input 2-D `SparseTensor` with a default value.
-//
-// The input `SparseTensor` is represented via the tuple of inputs
-// (`indices`, `values`, `dense_shape`).  The output `SparseTensor` has the
-// same `dense_shape` but with indices `output_indices` and values
-// `output_values`.
+// Conv2DAttr is an optional argument to Conv2D.
+type Conv2DAttr func(optionalAttr)
+
+// Conv2DUseCudnnOnGpu sets the optional use_cudnn_on_gpu attribute to value.
+// If not specified, defaults to true
+func Conv2DUseCudnnOnGpu(value bool) Conv2DAttr {
+	return func(m optionalAttr) {
+		m["use_cudnn_on_gpu"] = value
+	}
+}
+
+// Conv2DExplicitPaddings sets the optional explicit_paddings attribute to value.
 //
-// This op inserts a single entry for every row that doesn't have any values.
+// value: If `padding` is `"EXPLICIT"`, the list of explicit padding amounts. For the ith
+// dimension, the amount of padding inserted before and after the dimension is
+// `explicit_paddings[2 * i]` and `explicit_paddings[2 * i + 1]`, respectively. If
+// `padding` is not `"EXPLICIT"`, `explicit_paddings` must be empty.
+// If not specified, defaults to <>
+func Conv2DExplicitPaddings(value []int64) Conv2DAttr {
+	return func(m optionalAttr) {
+		m["explicit_paddings"] = value
+	}
+}
+
+// Conv2DDataFormat sets the optional data_format attribute to value.
+//
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the data is stored in the order of:
+//     [batch, height, width, channels].
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, channels, height, width].
+// If not specified, defaults to "NHWC"
+func Conv2DDataFormat(value string) Conv2DAttr {
+	return func(m optionalAttr) {
+		m["data_format"] = value
+	}
+}
+
+// Conv2DDilations sets the optional dilations attribute to value.
+//
+// value: 1-D tensor of length 4.  The dilation factor for each dimension of
+// `input`. If set to k > 1, there will be k-1 skipped cells between each
+// filter element on that dimension. The dimension order is determined by the
+// value of `data_format`, see above for details. Dilations in the batch and
+// depth dimensions must be 1.
+// If not specified, defaults to <i:1 i:1 i:1 i:1 >
+func Conv2DDilations(value []int64) Conv2DAttr {
+	return func(m optionalAttr) {
+		m["dilations"] = value
+	}
+}
+
+// Computes a 2-D convolution given 4-D `input` and `filter` tensors.
+//
+// Given an input tensor of shape `[batch, in_height, in_width, in_channels]`
+// and a filter / kernel tensor of shape
+// `[filter_height, filter_width, in_channels, out_channels]`, this op
+// performs the following:
+//
+// 1. Flattens the filter to a 2-D matrix with shape
+//    `[filter_height * filter_width * in_channels, output_channels]`.
+// 2. Extracts image patches from the input tensor to form a *virtual*
+//    tensor of shape `[batch, out_height, out_width,
+//    filter_height * filter_width * in_channels]`.
+// 3. For each patch, right-multiplies the filter matrix and the image patch
+//    vector.
+//
+// In detail, with the default NHWC format,
+//
+//     output[b, i, j, k] =
+//         sum_{di, dj, q} input[b, strides[1] * i + di, strides[2] * j + dj, q] *
+//                         filter[di, dj, q, k]
+//
+// Must have `strides[0] = strides[3] = 1`.  For the most common case of the same
+// horizontal and vertices strides, `strides = [1, stride, stride, 1]`.
+//
+// Arguments:
+//	input: A 4-D tensor. The dimension order is interpreted according to the value
+// of `data_format`, see below for details.
+//	filter: A 4-D tensor of shape
+// `[filter_height, filter_width, in_channels, out_channels]`
+//	strides: 1-D tensor of length 4.  The stride of the sliding window for each
+// dimension of `input`. The dimension order is determined by the value of
+// `data_format`, see below for details.
+//	padding: The type of padding algorithm to use.
+//
+// Returns A 4-D tensor. The dimension order is determined by the value of
+// `data_format`, see below for details.
+func Conv2D(scope *Scope, input tf.Output, filter tf.Output, strides []int64, padding string, optional ...Conv2DAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"strides": strides, "padding": padding}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "Conv2D",
+		Input: []tf.Input{
+			input, filter,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Fills empty rows in the input 2-D `SparseTensor` with a default value.
+//
+// The input `SparseTensor` is represented via the tuple of inputs
+// (`indices`, `values`, `dense_shape`).  The output `SparseTensor` has the
+// same `dense_shape` but with indices `output_indices` and values
+// `output_values`.
+//
+// This op inserts a single entry for every row that doesn't have any values.
 // The index is created as `[row, 0, ..., 0]` and the inserted value
 // is `default_value`.
 //
@@ -11491,6 +11479,61 @@ func ParseSequenceExample(scope *Scope, serialized tf.Output, debug_name tf.Outp
 	return context_sparse_indices, context_sparse_values, context_sparse_shapes, context_dense_values, feature_list_sparse_indices, feature_list_sparse_values, feature_list_sparse_shapes, feature_list_dense_values, feature_list_dense_lengths
 }
 
+// SparseReduceMaxAttr is an optional argument to SparseReduceMax.
+type SparseReduceMaxAttr func(optionalAttr)
+
+// SparseReduceMaxKeepDims sets the optional keep_dims attribute to value.
+//
+// value: If true, retain reduced dimensions with length 1.
+// If not specified, defaults to false
+func SparseReduceMaxKeepDims(value bool) SparseReduceMaxAttr {
+	return func(m optionalAttr) {
+		m["keep_dims"] = value
+	}
+}
+
+// Computes the max of elements across dimensions of a SparseTensor.
+//
+// This Op takes a SparseTensor and is the sparse counterpart to
+// `tf.reduce_max()`.  In particular, this Op also returns a dense `Tensor`
+// instead of a sparse one.
+//
+// Reduces `sp_input` along the dimensions given in `reduction_axes`.  Unless
+// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+// `reduction_axes`. If `keep_dims` is true, the reduced dimensions are retained
+// with length 1.
+//
+// If `reduction_axes` has no entries, all dimensions are reduced, and a tensor
+// with a single element is returned.  Additionally, the axes can be negative,
+// which are interpreted according to the indexing rules in Python.
+//
+// Arguments:
+//	input_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
+// SparseTensor, possibly not in canonical ordering.
+//	input_values: 1-D.  `N` non-empty values corresponding to `input_indices`.
+//	input_shape: 1-D.  Shape of the input SparseTensor.
+//	reduction_axes: 1-D.  Length-`K` vector containing the reduction axes.
+//
+// Returns `R-K`-D.  The reduced Tensor.
+func SparseReduceMax(scope *Scope, input_indices tf.Output, input_values tf.Output, input_shape tf.Output, reduction_axes tf.Output, optional ...SparseReduceMaxAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "SparseReduceMax",
+		Input: []tf.Input{
+			input_indices, input_values, input_shape, reduction_axes,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Computes the Gauss error function of `x` element-wise.
 func Erf(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
@@ -12046,6 +12089,47 @@ func NonMaxSuppressionV2(scope *Scope, boxes tf.Output, scores tf.Output, max_ou
 	return op.Output(0)
 }
 
+// Subtracts sparse updates from the variable referenced by `resource`.
+//
+// This operation computes
+//
+//     # Scalar indices
+//     ref[indices, ...] -= updates[...]
+//
+//     # Vector indices (for each i)
+//     ref[indices[i], ...] -= updates[i, ...]
+//
+//     # High rank indices (for each i, ..., j)
+//     ref[indices[i, ..., j], ...] -= updates[i, ..., j, ...]
+//
+// Duplicate entries are handled correctly: if multiple `indices` reference
+// the same location, their contributions add.
+//
+// Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src='https://www.tensorflow.org/images/ScatterAdd.png' alt>
+// </div>
+//
+// Arguments:
+//	resource: Should be from a `Variable` node.
+//	indices: A tensor of indices into the first dimension of `ref`.
+//	updates: A tensor of updated values to add to `ref`.
+//
+// Returns the created operation.
+func ResourceScatterSub(scope *Scope, resource tf.Output, indices tf.Output, updates tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceScatterSub",
+		Input: []tf.Input{
+			resource, indices, updates,
+		},
+	}
+	return scope.AddOperation(opspec)
+}
+
 // Converts a `RaggedTensor` into a `SparseTensor` with the same values.
 //
 // input=ragged.from_nested_row_splits(rt_dense_values, rt_nested_splits)
@@ -13031,47 +13115,6 @@ func ResourceScatterMul(scope *Scope, resource tf.Output, indices tf.Output, upd
 	return scope.AddOperation(opspec)
 }
 
-// Subtracts sparse updates from the variable referenced by `resource`.
-//
-// This operation computes
-//
-//     # Scalar indices
-//     ref[indices, ...] -= updates[...]
-//
-//     # Vector indices (for each i)
-//     ref[indices[i], ...] -= updates[i, ...]
-//
-//     # High rank indices (for each i, ..., j)
-//     ref[indices[i, ..., j], ...] -= updates[i, ..., j, ...]
-//
-// Duplicate entries are handled correctly: if multiple `indices` reference
-// the same location, their contributions add.
-//
-// Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`.
-//
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src='https://www.tensorflow.org/images/ScatterAdd.png' alt>
-// </div>
-//
-// Arguments:
-//	resource: Should be from a `Variable` node.
-//	indices: A tensor of indices into the first dimension of `ref`.
-//	updates: A tensor of updated values to add to `ref`.
-//
-// Returns the created operation.
-func ResourceScatterSub(scope *Scope, resource tf.Output, indices tf.Output, updates tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ResourceScatterSub",
-		Input: []tf.Input{
-			resource, indices, updates,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
 // Adds sparse updates to the variable referenced by `resource`.
 //
 // This operation computes
@@ -13355,158 +13398,23 @@ func MultiDeviceIteratorInit(scope *Scope, dataset tf.Output, multi_device_itera
 	return op.Output(0)
 }
 
-// Computes the gradient of `igamma(a, x)` wrt `a`.
-func IgammaGradA(scope *Scope, a tf.Output, x tf.Output) (z tf.Output) {
+// Reads the value of a variable.
+//
+// The tensor returned by this operation is immutable.
+//
+// The value returned by this operation is guaranteed to be influenced by all the
+// writes on which this operation depends directly or indirectly, and to not be
+// influenced by any of the writes which depend directly or indirectly on this
+// operation.
+//
+// Arguments:
+//	resource: handle to the resource in which to store the variable.
+//	dtype: the dtype of the value.
+func ReadVariableOp(scope *Scope, resource tf.Output, dtype tf.DataType) (value tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	opspec := tf.OpSpec{
-		Type: "IgammaGradA",
-		Input: []tf.Input{
-			a, x,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Converts each string in the input Tensor to its hash mod by a number of buckets.
-//
-// The hash function is deterministic on the content of the string within the
-// process.
-//
-// Note that the hash function may change from time to time.
-// This functionality will be deprecated and it's recommended to use
-// `tf.string_to_hash_bucket_fast()` or `tf.string_to_hash_bucket_strong()`.
-//
-// Arguments:
-//
-//	num_buckets: The number of buckets.
-//
-// Returns A Tensor of the same shape as the input `string_tensor`.
-func StringToHashBucket(scope *Scope, string_tensor tf.Output, num_buckets int64) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"num_buckets": num_buckets}
-	opspec := tf.OpSpec{
-		Type: "StringToHashBucket",
-		Input: []tf.Input{
-			string_tensor,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// StaticRegexReplaceAttr is an optional argument to StaticRegexReplace.
-type StaticRegexReplaceAttr func(optionalAttr)
-
-// StaticRegexReplaceReplaceGlobal sets the optional replace_global attribute to value.
-//
-// value: If True, the replacement is global, otherwise the replacement
-// is done only on the first match.
-// If not specified, defaults to true
-func StaticRegexReplaceReplaceGlobal(value bool) StaticRegexReplaceAttr {
-	return func(m optionalAttr) {
-		m["replace_global"] = value
-	}
-}
-
-// Replaces the match of pattern in input with rewrite.
-//
-// It follows the re2 syntax (https://github.com/google/re2/wiki/Syntax)
-//
-// Arguments:
-//	input: The text to be processed.
-//	pattern: The regular expression to match the input.
-//	rewrite: The rewrite to be applied to the matched expression.
-//
-// Returns The text after applying pattern and rewrite.
-func StaticRegexReplace(scope *Scope, input tf.Output, pattern string, rewrite string, optional ...StaticRegexReplaceAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"pattern": pattern, "rewrite": rewrite}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "StaticRegexReplace",
-		Input: []tf.Input{
-			input,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes gradients for the exponential linear (Elu) operation.
-//
-// Arguments:
-//	gradients: The backpropagated gradients to the corresponding Elu operation.
-//	outputs: The outputs of the corresponding Elu operation.
-//
-// Returns The gradients: `gradients * (outputs + 1)` if outputs < 0,
-// `gradients` otherwise.
-func EluGrad(scope *Scope, gradients tf.Output, outputs tf.Output) (backprops tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "EluGrad",
-		Input: []tf.Input{
-			gradients, outputs,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Creates a dataset that contains `count` elements from the `input_dataset`.
-//
-// Arguments:
-//
-//	count: A scalar representing the number of elements from the `input_dataset`
-// that should be taken. A value of `-1` indicates that all of `input_dataset`
-// is taken.
-//
-//
-func TakeDataset(scope *Scope, input_dataset tf.Output, count tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
-	opspec := tf.OpSpec{
-		Type: "TakeDataset",
-		Input: []tf.Input{
-			input_dataset, count,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Reads the value of a variable.
-//
-// The tensor returned by this operation is immutable.
-//
-// The value returned by this operation is guaranteed to be influenced by all the
-// writes on which this operation depends directly or indirectly, and to not be
-// influenced by any of the writes which depend directly or indirectly on this
-// operation.
-//
-// Arguments:
-//	resource: handle to the resource in which to store the variable.
-//	dtype: the dtype of the value.
-func ReadVariableOp(scope *Scope, resource tf.Output, dtype tf.DataType) (value tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtype": dtype}
+	attrs := map[string]interface{}{"dtype": dtype}
 	opspec := tf.OpSpec{
 		Type: "ReadVariableOp",
 		Input: []tf.Input{
@@ -13677,6 +13585,65 @@ func Tan(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
+// Deprecated. Use TensorArraySplitV3
+//
+// DEPRECATED at GraphDef version 26: Use TensorArraySplitV3
+func TensorArraySplitV2(scope *Scope, handle tf.Output, value tf.Output, lengths tf.Output, flow_in tf.Output) (flow_out tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorArraySplitV2",
+		Input: []tf.Input{
+			handle, value, lengths, flow_in,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Reshapes a SparseTensor to represent values in a new dense shape.
+//
+// This operation has the same semantics as reshape on the represented dense
+// tensor.  The `input_indices` are recomputed based on the requested `new_shape`.
+//
+// If one component of `new_shape` is the special value -1, the size of that
+// dimension is computed so that the total dense size remains constant.  At
+// most one component of `new_shape` can be -1.  The number of dense elements
+// implied by `new_shape` must be the same as the number of dense elements
+// originally implied by `input_shape`.
+//
+// Reshaping does not affect the order of values in the SparseTensor.
+//
+// If the input tensor has rank `R_in` and `N` non-empty values, and `new_shape`
+// has length `R_out`, then `input_indices` has shape `[N, R_in]`,
+// `input_shape` has length `R_in`, `output_indices` has shape `[N, R_out]`, and
+// `output_shape` has length `R_out`.
+//
+// Arguments:
+//	input_indices: 2-D.  `N x R_in` matrix with the indices of non-empty values in a
+// SparseTensor.
+//	input_shape: 1-D.  `R_in` vector with the input SparseTensor's dense shape.
+//	new_shape: 1-D.  `R_out` vector with the requested new dense shape.
+//
+// Returns 2-D.  `N x R_out` matrix with the updated indices of non-empty
+// values in the output SparseTensor.1-D.  `R_out` vector with the full dense shape of the output
+// SparseTensor.  This is the same as `new_shape` but with any -1 dimensions
+// filled in.
+func SparseReshape(scope *Scope, input_indices tf.Output, input_shape tf.Output, new_shape tf.Output) (output_indices tf.Output, output_shape tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SparseReshape",
+		Input: []tf.Input{
+			input_indices, input_shape, new_shape,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
 // Bucketizes 'input' based on 'boundaries'.
 //
 // For example, if the inputs are
@@ -15831,28 +15798,47 @@ func StringFormat(scope *Scope, inputs []tf.Output, optional ...StringFormatAttr
 	return op.Output(0)
 }
 
-// ShapeAttr is an optional argument to Shape.
-type ShapeAttr func(optionalAttr)
+// ResourceSparseApplyRMSPropAttr is an optional argument to ResourceSparseApplyRMSProp.
+type ResourceSparseApplyRMSPropAttr func(optionalAttr)
 
-// ShapeOutType sets the optional out_type attribute to value.
-// If not specified, defaults to DT_INT32
-func ShapeOutType(value tf.DataType) ShapeAttr {
+// ResourceSparseApplyRMSPropUseLocking sets the optional use_locking attribute to value.
+//
+// value: If `True`, updating of the var, ms, and mom tensors is protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceSparseApplyRMSPropUseLocking(value bool) ResourceSparseApplyRMSPropAttr {
 	return func(m optionalAttr) {
-		m["out_type"] = value
+		m["use_locking"] = value
 	}
 }
 
-// Returns the shape of a tensor.
+// Update '*var' according to the RMSProp algorithm.
 //
-// This operation returns a 1-D integer tensor representing the shape of `input`.
+// Note that in dense implementation of this algorithm, ms and mom will
+// update even if the grad is zero, but in this sparse implementation, ms
+// and mom will not update in iterations during which the grad is zero.
 //
-// For example:
+// mean_square = decay * mean_square + (1-decay) * gradient ** 2
+// Delta = learning_rate * gradient / sqrt(mean_square + epsilon)
 //
-// ```
-// # 't' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]]
-// shape(t) ==> [2, 2, 3]
-// ```
-func Shape(scope *Scope, input tf.Output, optional ...ShapeAttr) (output tf.Output) {
+// ms <- rho * ms_{t-1} + (1-rho) * grad * grad
+// mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon)
+// var <- var - mom
+//
+// Arguments:
+//	var_: Should be from a Variable().
+//	ms: Should be from a Variable().
+//	mom: Should be from a Variable().
+//	lr: Scaling factor. Must be a scalar.
+//	rho: Decay rate. Must be a scalar.
+//
+//	epsilon: Ridge term. Must be a scalar.
+//	grad: The gradient.
+//	indices: A vector of indices into the first dimension of var, ms and mom.
+//
+// Returns the created operation.
+func ResourceSparseApplyRMSProp(scope *Scope, var_ tf.Output, ms tf.Output, mom tf.Output, lr tf.Output, rho tf.Output, momentum tf.Output, epsilon tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyRMSPropAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -15861,113 +15847,9 @@ func Shape(scope *Scope, input tf.Output, optional ...ShapeAttr) (output tf.Outp
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Shape",
+		Type: "ResourceSparseApplyRMSProp",
 		Input: []tf.Input{
-			input,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes the power of one value to another.
-//
-// Given a tensor `x` and a tensor `y`, this operation computes \\(x^y\\) for
-// corresponding elements in `x` and `y`. For example:
-//
-// ```
-// # tensor 'x' is [[2, 2]], [3, 3]]
-// # tensor 'y' is [[8, 16], [2, 3]]
-// tf.pow(x, y) ==> [[256, 65536], [9, 27]]
-// ```
-func Pow(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Pow",
-		Input: []tf.Input{
-			x, y,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes fingerprints of the input strings.
-//
-// Arguments:
-//	input: vector of strings to compute fingerprints on.
-//
-// Returns a (N,2) shaped matrix where N is the number of elements in the input
-// vector. Each row contains the low and high parts of the fingerprint.
-func SdcaFprint(scope *Scope, input tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "SdcaFprint",
-		Input: []tf.Input{
-			input,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// ResourceSparseApplyRMSPropAttr is an optional argument to ResourceSparseApplyRMSProp.
-type ResourceSparseApplyRMSPropAttr func(optionalAttr)
-
-// ResourceSparseApplyRMSPropUseLocking sets the optional use_locking attribute to value.
-//
-// value: If `True`, updating of the var, ms, and mom tensors is protected
-// by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceSparseApplyRMSPropUseLocking(value bool) ResourceSparseApplyRMSPropAttr {
-	return func(m optionalAttr) {
-		m["use_locking"] = value
-	}
-}
-
-// Update '*var' according to the RMSProp algorithm.
-//
-// Note that in dense implementation of this algorithm, ms and mom will
-// update even if the grad is zero, but in this sparse implementation, ms
-// and mom will not update in iterations during which the grad is zero.
-//
-// mean_square = decay * mean_square + (1-decay) * gradient ** 2
-// Delta = learning_rate * gradient / sqrt(mean_square + epsilon)
-//
-// ms <- rho * ms_{t-1} + (1-rho) * grad * grad
-// mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon)
-// var <- var - mom
-//
-// Arguments:
-//	var_: Should be from a Variable().
-//	ms: Should be from a Variable().
-//	mom: Should be from a Variable().
-//	lr: Scaling factor. Must be a scalar.
-//	rho: Decay rate. Must be a scalar.
-//
-//	epsilon: Ridge term. Must be a scalar.
-//	grad: The gradient.
-//	indices: A vector of indices into the first dimension of var, ms and mom.
-//
-// Returns the created operation.
-func ResourceSparseApplyRMSProp(scope *Scope, var_ tf.Output, ms tf.Output, mom tf.Output, lr tf.Output, rho tf.Output, momentum tf.Output, epsilon tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyRMSPropAttr) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "ResourceSparseApplyRMSProp",
-		Input: []tf.Input{
-			var_, ms, mom, lr, rho, momentum, epsilon, grad, indices,
+			var_, ms, mom, lr, rho, momentum, epsilon, grad, indices,
 		},
 		Attrs: attrs,
 	}
@@ -19293,47 +19175,6 @@ func Sub(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	return op.Output(0)
 }
 
-// RandomPoissonAttr is an optional argument to RandomPoisson.
-type RandomPoissonAttr func(optionalAttr)
-
-// RandomPoissonSeed sets the optional seed attribute to value.
-// If not specified, defaults to 0
-func RandomPoissonSeed(value int64) RandomPoissonAttr {
-	return func(m optionalAttr) {
-		m["seed"] = value
-	}
-}
-
-// RandomPoissonSeed2 sets the optional seed2 attribute to value.
-// If not specified, defaults to 0
-func RandomPoissonSeed2(value int64) RandomPoissonAttr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
-	}
-}
-
-// Use RandomPoissonV2 instead.
-//
-// DEPRECATED at GraphDef version 25: Replaced by RandomPoissonV2
-func RandomPoisson(scope *Scope, shape tf.Output, rate tf.Output, optional ...RandomPoissonAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "RandomPoisson",
-		Input: []tf.Input{
-			shape, rate,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Returns the max of x and y (i.e. x > y ? x : y) element-wise.
 //
 // *NOTE*: `Maximum` supports broadcasting. More about broadcasting
@@ -19686,86 +19527,221 @@ func TopK(scope *Scope, input tf.Output, k int64, optional ...TopKAttr) (values
 	return op.Output(0), op.Output(1)
 }
 
-// The gradient operator for the SparseAdd op.
+// Converts each string in the input Tensor to its hash mod by a number of buckets.
 //
-// The SparseAdd op calculates A + B, where A, B, and the sum are all represented
-// as `SparseTensor` objects.  This op takes in the upstream gradient w.r.t.
-// non-empty values of the sum, and outputs the gradients w.r.t. the non-empty
-// values of A and B.
+// The hash function is deterministic on the content of the string within the
+// process.
+//
+// Note that the hash function may change from time to time.
+// This functionality will be deprecated and it's recommended to use
+// `tf.string_to_hash_bucket_fast()` or `tf.string_to_hash_bucket_strong()`.
 //
 // Arguments:
-//	backprop_val_grad: 1-D with shape `[nnz(sum)]`.  The gradient with respect to
-// the non-empty values of the sum.
-//	a_indices: 2-D.  The `indices` of the `SparseTensor` A, size `[nnz(A), ndims]`.
-//	b_indices: 2-D.  The `indices` of the `SparseTensor` B, size `[nnz(B), ndims]`.
-//	sum_indices: 2-D.  The `indices` of the sum `SparseTensor`, size
-// `[nnz(sum), ndims]`.
 //
-// Returns 1-D with shape `[nnz(A)]`. The gradient with respect to the
-// non-empty values of A.1-D with shape `[nnz(B)]`. The gradient with respect to the
-// non-empty values of B.
-func SparseAddGrad(scope *Scope, backprop_val_grad tf.Output, a_indices tf.Output, b_indices tf.Output, sum_indices tf.Output) (a_val_grad tf.Output, b_val_grad tf.Output) {
+//	num_buckets: The number of buckets.
+//
+// Returns A Tensor of the same shape as the input `string_tensor`.
+func StringToHashBucket(scope *Scope, string_tensor tf.Output, num_buckets int64) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"num_buckets": num_buckets}
 	opspec := tf.OpSpec{
-		Type: "SparseAddGrad",
+		Type: "StringToHashBucket",
 		Input: []tf.Input{
-			backprop_val_grad, a_indices, b_indices, sum_indices,
+			string_tensor,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// Returns a list of tensors with the same shapes and contents as the input
+// StaticRegexReplaceAttr is an optional argument to StaticRegexReplace.
+type StaticRegexReplaceAttr func(optionalAttr)
+
+// StaticRegexReplaceReplaceGlobal sets the optional replace_global attribute to value.
 //
-// tensors.
+// value: If True, the replacement is global, otherwise the replacement
+// is done only on the first match.
+// If not specified, defaults to true
+func StaticRegexReplaceReplaceGlobal(value bool) StaticRegexReplaceAttr {
+	return func(m optionalAttr) {
+		m["replace_global"] = value
+	}
+}
+
+// Replaces the match of pattern in input with rewrite.
 //
-// This op can be used to override the gradient for complicated functions. For
-// example, suppose y = f(x) and we wish to apply a custom function g for backprop
-// such that dx = g(dy). In Python,
+// It follows the re2 syntax (https://github.com/google/re2/wiki/Syntax)
 //
-// ```python
-// with tf.get_default_graph().gradient_override_map(
-//     {'IdentityN': 'OverrideGradientWithG'}):
-//   y, _ = identity_n([f(x), x])
+// Arguments:
+//	input: The text to be processed.
+//	pattern: The regular expression to match the input.
+//	rewrite: The rewrite to be applied to the matched expression.
 //
-// @tf.RegisterGradient('OverrideGradientWithG')
-// def ApplyG(op, dy, _):
-//   return [None, g(dy)]  # Do not backprop to f(x).
-// ```
-func IdentityN(scope *Scope, input []tf.Output) (output []tf.Output) {
+// Returns The text after applying pattern and rewrite.
+func StaticRegexReplace(scope *Scope, input tf.Output, pattern string, rewrite string, optional ...StaticRegexReplaceAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"pattern": pattern, "rewrite": rewrite}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "IdentityN",
+		Type: "StaticRegexReplace",
 		Input: []tf.Input{
-			tf.OutputList(input),
+			input,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Computes gradients for the exponential linear (Elu) operation.
+//
+// Arguments:
+//	gradients: The backpropagated gradients to the corresponding Elu operation.
+//	outputs: The outputs of the corresponding Elu operation.
+//
+// Returns The gradients: `gradients * (outputs + 1)` if outputs < 0,
+// `gradients` otherwise.
+func EluGrad(scope *Scope, gradients tf.Output, outputs tf.Output) (backprops tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	var idx int
-	var err error
-	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
-		scope.UpdateErr("IdentityN", err)
-		return
+	opspec := tf.OpSpec{
+		Type: "EluGrad",
+		Input: []tf.Input{
+			gradients, outputs,
+		},
 	}
-	return output
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// ResourceApplyCenteredRMSPropAttr is an optional argument to ResourceApplyCenteredRMSProp.
-type ResourceApplyCenteredRMSPropAttr func(optionalAttr)
-
-// ResourceApplyCenteredRMSPropUseLocking sets the optional use_locking attribute to value.
-//
-// value: If `True`, updating of the var, mg, ms, and mom tensors is
-// protected by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
+// Computes the gradient of `igamma(a, x)` wrt `a`.
+func IgammaGradA(scope *Scope, a tf.Output, x tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "IgammaGradA",
+		Input: []tf.Input{
+			a, x,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Creates a dataset that contains `count` elements from the `input_dataset`.
+//
+// Arguments:
+//
+//	count: A scalar representing the number of elements from the `input_dataset`
+// that should be taken. A value of `-1` indicates that all of `input_dataset`
+// is taken.
+//
+//
+func TakeDataset(scope *Scope, input_dataset tf.Output, count tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "TakeDataset",
+		Input: []tf.Input{
+			input_dataset, count,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// The gradient operator for the SparseAdd op.
+//
+// The SparseAdd op calculates A + B, where A, B, and the sum are all represented
+// as `SparseTensor` objects.  This op takes in the upstream gradient w.r.t.
+// non-empty values of the sum, and outputs the gradients w.r.t. the non-empty
+// values of A and B.
+//
+// Arguments:
+//	backprop_val_grad: 1-D with shape `[nnz(sum)]`.  The gradient with respect to
+// the non-empty values of the sum.
+//	a_indices: 2-D.  The `indices` of the `SparseTensor` A, size `[nnz(A), ndims]`.
+//	b_indices: 2-D.  The `indices` of the `SparseTensor` B, size `[nnz(B), ndims]`.
+//	sum_indices: 2-D.  The `indices` of the sum `SparseTensor`, size
+// `[nnz(sum), ndims]`.
+//
+// Returns 1-D with shape `[nnz(A)]`. The gradient with respect to the
+// non-empty values of A.1-D with shape `[nnz(B)]`. The gradient with respect to the
+// non-empty values of B.
+func SparseAddGrad(scope *Scope, backprop_val_grad tf.Output, a_indices tf.Output, b_indices tf.Output, sum_indices tf.Output) (a_val_grad tf.Output, b_val_grad tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SparseAddGrad",
+		Input: []tf.Input{
+			backprop_val_grad, a_indices, b_indices, sum_indices,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
+// Returns a list of tensors with the same shapes and contents as the input
+//
+// tensors.
+//
+// This op can be used to override the gradient for complicated functions. For
+// example, suppose y = f(x) and we wish to apply a custom function g for backprop
+// such that dx = g(dy). In Python,
+//
+// ```python
+// with tf.get_default_graph().gradient_override_map(
+//     {'IdentityN': 'OverrideGradientWithG'}):
+//   y, _ = identity_n([f(x), x])
+//
+// @tf.RegisterGradient('OverrideGradientWithG')
+// def ApplyG(op, dy, _):
+//   return [None, g(dy)]  # Do not backprop to f(x).
+// ```
+func IdentityN(scope *Scope, input []tf.Output) (output []tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "IdentityN",
+		Input: []tf.Input{
+			tf.OutputList(input),
+		},
+	}
+	op := scope.AddOperation(opspec)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
+		scope.UpdateErr("IdentityN", err)
+		return
+	}
+	return output
+}
+
+// ResourceApplyCenteredRMSPropAttr is an optional argument to ResourceApplyCenteredRMSProp.
+type ResourceApplyCenteredRMSPropAttr func(optionalAttr)
+
+// ResourceApplyCenteredRMSPropUseLocking sets the optional use_locking attribute to value.
+//
+// value: If `True`, updating of the var, mg, ms, and mom tensors is
+// protected by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
 // If not specified, defaults to false
 func ResourceApplyCenteredRMSPropUseLocking(value bool) ResourceApplyCenteredRMSPropAttr {
 	return func(m optionalAttr) {
@@ -20269,6 +20245,42 @@ func FixedLengthRecordReaderV2(scope *Scope, record_bytes int64, optional ...Fix
 	return op.Output(0)
 }
 
+// Return a tensor with the same shape and contents as the input tensor or value.
+func Identity(scope *Scope, input tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Identity",
+		Input: []tf.Input{
+			input,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Computes arctangent of `y/x` element-wise, respecting signs of the arguments.
+//
+// This is the angle \( \theta \in [-\pi, \pi] \) such that
+// \[ x = r \cos(\theta) \]
+// and
+// \[ y = r \sin(\theta) \]
+// where \(r = \sqrt(x^2 + y^2) \).
+func Atan2(scope *Scope, y tf.Output, x tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Atan2",
+		Input: []tf.Input{
+			y, x,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // AudioSummaryAttr is an optional argument to AudioSummary.
 type AudioSummaryAttr func(optionalAttr)
 
@@ -20555,65 +20567,6 @@ func QuantizedRelu(scope *Scope, features tf.Output, min_features tf.Output, max
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Reshapes a SparseTensor to represent values in a new dense shape.
-//
-// This operation has the same semantics as reshape on the represented dense
-// tensor.  The `input_indices` are recomputed based on the requested `new_shape`.
-//
-// If one component of `new_shape` is the special value -1, the size of that
-// dimension is computed so that the total dense size remains constant.  At
-// most one component of `new_shape` can be -1.  The number of dense elements
-// implied by `new_shape` must be the same as the number of dense elements
-// originally implied by `input_shape`.
-//
-// Reshaping does not affect the order of values in the SparseTensor.
-//
-// If the input tensor has rank `R_in` and `N` non-empty values, and `new_shape`
-// has length `R_out`, then `input_indices` has shape `[N, R_in]`,
-// `input_shape` has length `R_in`, `output_indices` has shape `[N, R_out]`, and
-// `output_shape` has length `R_out`.
-//
-// Arguments:
-//	input_indices: 2-D.  `N x R_in` matrix with the indices of non-empty values in a
-// SparseTensor.
-//	input_shape: 1-D.  `R_in` vector with the input SparseTensor's dense shape.
-//	new_shape: 1-D.  `R_out` vector with the requested new dense shape.
-//
-// Returns 2-D.  `N x R_out` matrix with the updated indices of non-empty
-// values in the output SparseTensor.1-D.  `R_out` vector with the full dense shape of the output
-// SparseTensor.  This is the same as `new_shape` but with any -1 dimensions
-// filled in.
-func SparseReshape(scope *Scope, input_indices tf.Output, input_shape tf.Output, new_shape tf.Output) (output_indices tf.Output, output_shape tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "SparseReshape",
-		Input: []tf.Input{
-			input_indices, input_shape, new_shape,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
-}
-
-// Deprecated. Use TensorArraySplitV3
-//
-// DEPRECATED at GraphDef version 26: Use TensorArraySplitV3
-func TensorArraySplitV2(scope *Scope, handle tf.Output, value tf.Output, lengths tf.Output, flow_in tf.Output) (flow_out tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "TensorArraySplitV2",
-		Input: []tf.Input{
-			handle, value, lengths, flow_in,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Reorders a SparseTensor into the canonical, row-major ordering.
 //
 // Note that by convention, all sparse ops preserve the canonical ordering along
@@ -21568,123 +21521,56 @@ func StatelessRandomUniformInt(scope *Scope, shape tf.Output, seed tf.Output, mi
 	return op.Output(0)
 }
 
-// Compute the Hurwitz zeta function \\(\zeta(x, q)\\).
+// Inverse fast Fourier transform.
 //
-// The Hurwitz zeta function is defined as:
+// Computes the inverse 1-dimensional discrete Fourier transform over the
+// inner-most dimension of `input`.
 //
+// Arguments:
+//	input: A complex tensor.
 //
-// \\(\zeta(x, q) = \sum_{n=0}^{\infty} (q + n)^{-x}\\)
-func Zeta(scope *Scope, x tf.Output, q tf.Output) (z tf.Output) {
+// Returns A complex tensor of the same shape as `input`. The inner-most
+//   dimension of `input` is replaced with its inverse 1D Fourier transform.
+//
+// @compatibility(numpy)
+// Equivalent to np.fft.ifft
+// @end_compatibility
+func IFFT(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Zeta",
+		Type: "IFFT",
 		Input: []tf.Input{
-			x, q,
+			input,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Returns the cardinality of `input_dataset`.
+// 2D fast Fourier transform.
 //
-// Returns the cardinality of `input_dataset`.
+// Computes the 2-dimensional discrete Fourier transform over the inner-most
+// 2 dimensions of `input`.
 //
 // Arguments:
-//	input_dataset: A variant tensor representing the dataset to return cardinality for.
+//	input: A complex tensor.
 //
-// Returns The cardinality of `input_dataset`. Named constants are used to represent
-// infinite and unknown cardinality.
-func ExperimentalDatasetCardinality(scope *Scope, input_dataset tf.Output) (cardinality tf.Output) {
+// Returns A complex tensor of the same shape as `input`. The inner-most 2
+//   dimensions of `input` are replaced with their 2D Fourier transform.
+//
+// @compatibility(numpy)
+// Equivalent to np.fft.fft2
+// @end_compatibility
+func FFT2D(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "ExperimentalDatasetCardinality",
+		Type: "FFT2D",
 		Input: []tf.Input{
-			input_dataset,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Creates a dataset that executes a SQL query and emits rows of the result set.
-//
-// Arguments:
-//	driver_name: The database type. Currently, the only supported type is 'sqlite'.
-//	data_source_name: A connection string to connect to the database.
-//	query: A SQL query to execute.
-//
-//
-func ExperimentalSqlDataset(scope *Scope, driver_name tf.Output, data_source_name tf.Output, query tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
-	opspec := tf.OpSpec{
-		Type: "ExperimentalSqlDataset",
-		Input: []tf.Input{
-			driver_name, data_source_name, query,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Inverse fast Fourier transform.
-//
-// Computes the inverse 1-dimensional discrete Fourier transform over the
-// inner-most dimension of `input`.
-//
-// Arguments:
-//	input: A complex tensor.
-//
-// Returns A complex tensor of the same shape as `input`. The inner-most
-//   dimension of `input` is replaced with its inverse 1D Fourier transform.
-//
-// @compatibility(numpy)
-// Equivalent to np.fft.ifft
-// @end_compatibility
-func IFFT(scope *Scope, input tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "IFFT",
-		Input: []tf.Input{
-			input,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// 2D fast Fourier transform.
-//
-// Computes the 2-dimensional discrete Fourier transform over the inner-most
-// 2 dimensions of `input`.
-//
-// Arguments:
-//	input: A complex tensor.
-//
-// Returns A complex tensor of the same shape as `input`. The inner-most 2
-//   dimensions of `input` are replaced with their 2D Fourier transform.
-//
-// @compatibility(numpy)
-// Equivalent to np.fft.fft2
-// @end_compatibility
-func FFT2D(scope *Scope, input tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "FFT2D",
-		Input: []tf.Input{
-			input,
+			input,
 		},
 	}
 	op := scope.AddOperation(opspec)
@@ -21719,27 +21605,6 @@ func IFFT2D(scope *Scope, input tf.Output) (output tf.Output) {
 	return op.Output(0)
 }
 
-// Returns element-wise remainder of division. This emulates C semantics in that
-//
-// the result here is consistent with a truncating divide. E.g. `truncate(x / y) *
-// y + truncate_mod(x, y) = x`.
-//
-// *NOTE*: `TruncateMod` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func TruncateMod(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "TruncateMod",
-		Input: []tf.Input{
-			x, y,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Inverse 2D real-valued fast Fourier transform.
 //
 // Computes the inverse 2-dimensional discrete Fourier transform of a real-valued
@@ -22368,63 +22233,6 @@ func QuantizeV2(scope *Scope, input tf.Output, min_range tf.Output, max_range tf
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Returns the truth value of (x < y) element-wise.
-//
-// *NOTE*: `Less` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func Less(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Less",
-		Input: []tf.Input{
-			x, y,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// QuantizedReluXAttr is an optional argument to QuantizedReluX.
-type QuantizedReluXAttr func(optionalAttr)
-
-// QuantizedReluXOutType sets the optional out_type attribute to value.
-// If not specified, defaults to DT_QUINT8
-func QuantizedReluXOutType(value tf.DataType) QuantizedReluXAttr {
-	return func(m optionalAttr) {
-		m["out_type"] = value
-	}
-}
-
-// Computes Quantized Rectified Linear X: `min(max(features, 0), max_value)`
-//
-// Arguments:
-//
-//
-//	min_features: The float value that the lowest quantized value represents.
-//	max_features: The float value that the highest quantized value represents.
-//
-// Returns Has the same output shape as "features".The float value that the lowest quantized value represents.The float value that the highest quantized value represents.
-func QuantizedReluX(scope *Scope, features tf.Output, max_value tf.Output, min_features tf.Output, max_features tf.Output, optional ...QuantizedReluXAttr) (activations tf.Output, min_activations tf.Output, max_activations tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "QuantizedReluX",
-		Input: []tf.Input{
-			features, max_value, min_features, max_features,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
-}
-
 // Creates a dataset that batches `batch_size` elements from `input_dataset`.
 //
 // Arguments:
@@ -24589,16 +24397,19 @@ func QuantizeDownAndShrinkRange(scope *Scope, input tf.Output, input_min tf.Outp
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Returns the truth value of (x >= y) element-wise.
+// Returns element-wise remainder of division. This emulates C semantics in that
 //
-// *NOTE*: `GreaterEqual` supports broadcasting. More about broadcasting
+// the result here is consistent with a truncating divide. E.g. `truncate(x / y) *
+// y + truncate_mod(x, y) = x`.
+//
+// *NOTE*: `TruncateMod` supports broadcasting. More about broadcasting
 // [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func GreaterEqual(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+func TruncateMod(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "GreaterEqual",
+		Type: "TruncateMod",
 		Input: []tf.Input{
 			x, y,
 		},
@@ -24607,151 +24418,195 @@ func GreaterEqual(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	return op.Output(0)
 }
 
-// ApproximateEqualAttr is an optional argument to ApproximateEqual.
-type ApproximateEqualAttr func(optionalAttr)
-
-// ApproximateEqualTolerance sets the optional tolerance attribute to value.
-// If not specified, defaults to 1e-05
-func ApproximateEqualTolerance(value float32) ApproximateEqualAttr {
-	return func(m optionalAttr) {
-		m["tolerance"] = value
-	}
-}
-
-// Returns the truth value of abs(x-y) < tolerance element-wise.
-func ApproximateEqual(scope *Scope, x tf.Output, y tf.Output, optional ...ApproximateEqualAttr) (z tf.Output) {
+// Computes offsets of concat inputs within its output.
+//
+// For example:
+//
+// ```
+// # 'x' is [2, 2, 7]
+// # 'y' is [2, 3, 7]
+// # 'z' is [2, 5, 7]
+// concat_offset(2, [x, y, z]) => [0, 0, 0], [0, 2, 0], [0, 5, 0]
+// ```
+//
+// This is typically used by gradient computations for a concat operation.
+//
+// Arguments:
+//	concat_dim: The dimension along which to concatenate.
+//	shape: The `N` int32 vectors representing shape of tensors being concatenated.
+//
+// Returns The `N` int32 vectors representing the starting offset
+// of input tensors within the concatenated output.
+func ConcatOffset(scope *Scope, concat_dim tf.Output, shape []tf.Output) (offset []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "ApproximateEqual",
+		Type: "ConcatOffset",
 		Input: []tf.Input{
-			x, y,
+			concat_dim, tf.OutputList(shape),
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if offset, idx, err = makeOutputList(op, idx, "offset"); err != nil {
+		scope.UpdateErr("ConcatOffset", err)
+		return
+	}
+	return offset
 }
 
-// Returns the truth value of x OR y element-wise.
+// Compute the lower regularized incomplete Gamma function `P(a, x)`.
 //
-// *NOTE*: `LogicalOr` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func LogicalOr(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+// The lower regularized incomplete Gamma function is defined as:
+//
+//
+// \\(P(a, x) = gamma(a, x) / Gamma(a) = 1 - Q(a, x)\\)
+//
+// where
+//
+// \\(gamma(a, x) = \\int_{0}^{x} t^{a-1} exp(-t) dt\\)
+//
+// is the lower incomplete Gamma function.
+//
+// Note, above `Q(a, x)` (`Igammac`) is the upper regularized complete
+// Gamma function.
+func Igamma(scope *Scope, a tf.Output, x tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "LogicalOr",
+		Type: "Igamma",
 		Input: []tf.Input{
-			x, y,
+			a, x,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Selects elements from `x` or `y`, depending on `condition`.
+// Compute the Hurwitz zeta function \\(\zeta(x, q)\\).
 //
-// The `x`, and `y` tensors must all have the same shape, and the
-// output will also have that shape.
+// The Hurwitz zeta function is defined as:
 //
-// The `condition` tensor must be a scalar if `x` and `y` are scalars.
-// If `x` and `y` are vectors or higher rank, then `condition` must be either a
-// scalar, a vector with size matching the first dimension of `x`, or must have
-// the same shape as `x`.
 //
-// The `condition` tensor acts as a mask that chooses, based on the value at each
-// element, whether the corresponding element / row in the output should be
-// taken from `x` (if true) or `y` (if false).
-//
-// If `condition` is a vector and `x` and `y` are higher rank matrices, then
-// it chooses which row (outer dimension) to copy from `x` and `y`.
-// If `condition` has the same shape as `x` and `y`, then it chooses which
-// element to copy from `x` and `y`.
-//
-// For example:
-//
-// ```python
-// # 'condition' tensor is [[True,  False]
-// #                        [False, True]]
-// # 't' is [[1, 2],
-// #         [3, 4]]
-// # 'e' is [[5, 6],
-// #         [7, 8]]
-// select(condition, t, e)  # => [[1, 6], [7, 4]]
+// \\(\zeta(x, q) = \sum_{n=0}^{\infty} (q + n)^{-x}\\)
+func Zeta(scope *Scope, x tf.Output, q tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Zeta",
+		Input: []tf.Input{
+			x, q,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Returns the cardinality of `input_dataset`.
 //
+// Returns the cardinality of `input_dataset`.
 //
-// # 'condition' tensor is [True, False]
-// # 't' is [[1, 2],
-// #         [3, 4]]
-// # 'e' is [[5, 6],
-// #         [7, 8]]
-// select(condition, t, e) ==> [[1, 2],
-//                              [7, 8]]
+// Arguments:
+//	input_dataset: A variant tensor representing the dataset to return cardinality for.
 //
-// ```
+// Returns The cardinality of `input_dataset`. Named constants are used to represent
+// infinite and unknown cardinality.
+func ExperimentalDatasetCardinality(scope *Scope, input_dataset tf.Output) (cardinality tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalDatasetCardinality",
+		Input: []tf.Input{
+			input_dataset,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Creates a dataset that executes a SQL query and emits rows of the result set.
 //
 // Arguments:
+//	driver_name: The database type. Currently, the only supported type is 'sqlite'.
+//	data_source_name: A connection string to connect to the database.
+//	query: A SQL query to execute.
 //
-//	x: = A `Tensor` which may have the same shape as `condition`.
-// If `condition` is rank 1, `x` may have higher rank,
-// but its first dimension must match the size of `condition`.
-//	y: = A `Tensor` with the same type and shape as `x`.
 //
-// Returns = A `Tensor` with the same type and shape as `x` and `y`.
-func Select(scope *Scope, condition tf.Output, x tf.Output, y tf.Output) (output tf.Output) {
+func ExperimentalSqlDataset(scope *Scope, driver_name tf.Output, data_source_name tf.Output, query tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "Select",
+		Type: "ExperimentalSqlDataset",
 		Input: []tf.Input{
-			condition, x, y,
+			driver_name, data_source_name, query,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// MatMulAttr is an optional argument to MatMul.
-type MatMulAttr func(optionalAttr)
-
-// MatMulTransposeA sets the optional transpose_a attribute to value.
+// Compute the regularized incomplete beta integral \\(I_x(a, b)\\).
 //
-// value: If true, "a" is transposed before multiplication.
-// If not specified, defaults to false
-func MatMulTransposeA(value bool) MatMulAttr {
-	return func(m optionalAttr) {
-		m["transpose_a"] = value
+// The regularized incomplete beta integral is defined as:
+//
+//
+// \\(I_x(a, b) = \frac{B(x; a, b)}{B(a, b)}\\)
+//
+// where
+//
+//
+// \\(B(x; a, b) = \int_0^x t^{a-1} (1 - t)^{b-1} dt\\)
+//
+//
+// is the incomplete beta function and \\(B(a, b)\\) is the *complete*
+// beta function.
+func Betainc(scope *Scope, a tf.Output, b tf.Output, x tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Betainc",
+		Input: []tf.Input{
+			a, b, x,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// MatMulTransposeB sets the optional transpose_b attribute to value.
-//
-// value: If true, "b" is transposed before multiplication.
-// If not specified, defaults to false
-func MatMulTransposeB(value bool) MatMulAttr {
+// ShapeAttr is an optional argument to Shape.
+type ShapeAttr func(optionalAttr)
+
+// ShapeOutType sets the optional out_type attribute to value.
+// If not specified, defaults to DT_INT32
+func ShapeOutType(value tf.DataType) ShapeAttr {
 	return func(m optionalAttr) {
-		m["transpose_b"] = value
+		m["out_type"] = value
 	}
 }
 
-// Multiply the matrix "a" by the matrix "b".
+// Returns the shape of a tensor.
 //
-// The inputs must be two-dimensional matrices and the inner dimension of
-// "a" (after being transposed if transpose_a is true) must match the
-// outer dimension of "b" (after being transposed if transposed_b is
-// true).
+// This operation returns a 1-D integer tensor representing the shape of `input`.
 //
-// *Note*: The default kernel implementation for MatMul on GPUs uses
-// cublas.
-func MatMul(scope *Scope, a tf.Output, b tf.Output, optional ...MatMulAttr) (product tf.Output) {
+// For example:
+//
+// ```
+// # 't' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]]
+// shape(t) ==> [2, 2, 3]
+// ```
+func Shape(scope *Scope, input tf.Output, optional ...ShapeAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -24760,9 +24615,9 @@ func MatMul(scope *Scope, a tf.Output, b tf.Output, optional ...MatMulAttr) (pro
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MatMul",
+		Type: "Shape",
 		Input: []tf.Input{
-			a, b,
+			input,
 		},
 		Attrs: attrs,
 	}
@@ -24770,73 +24625,72 @@ func MatMul(scope *Scope, a tf.Output, b tf.Output, optional ...MatMulAttr) (pro
 	return op.Output(0)
 }
 
-// Serializes the tree handle to a proto
+// Computes fingerprints of the input strings.
 //
 // Arguments:
-//	tree_handle: Handle to the tree resource to be serialized.
+//	input: vector of strings to compute fingerprints on.
 //
-// Returns Serialied proto string of the tree resource.
-func TensorForestTreeSerialize(scope *Scope, tree_handle tf.Output) (tree_config tf.Output) {
+// Returns a (N,2) shaped matrix where N is the number of elements in the input
+// vector. Each row contains the low and high parts of the fingerprint.
+func SdcaFprint(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "TensorForestTreeSerialize",
+		Type: "SdcaFprint",
 		Input: []tf.Input{
-			tree_handle,
+			input,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// SparseMatMulAttr is an optional argument to SparseMatMul.
-type SparseMatMulAttr func(optionalAttr)
-
-// SparseMatMulTransposeA sets the optional transpose_a attribute to value.
-// If not specified, defaults to false
-func SparseMatMulTransposeA(value bool) SparseMatMulAttr {
-	return func(m optionalAttr) {
-		m["transpose_a"] = value
+// Computes the power of one value to another.
+//
+// Given a tensor `x` and a tensor `y`, this operation computes \\(x^y\\) for
+// corresponding elements in `x` and `y`. For example:
+//
+// ```
+// # tensor 'x' is [[2, 2]], [3, 3]]
+// # tensor 'y' is [[8, 16], [2, 3]]
+// tf.pow(x, y) ==> [[256, 65536], [9, 27]]
+// ```
+func Pow(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// SparseMatMulTransposeB sets the optional transpose_b attribute to value.
-// If not specified, defaults to false
-func SparseMatMulTransposeB(value bool) SparseMatMulAttr {
-	return func(m optionalAttr) {
-		m["transpose_b"] = value
+	opspec := tf.OpSpec{
+		Type: "Pow",
+		Input: []tf.Input{
+			x, y,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// SparseMatMulAIsSparse sets the optional a_is_sparse attribute to value.
-// If not specified, defaults to false
-func SparseMatMulAIsSparse(value bool) SparseMatMulAttr {
-	return func(m optionalAttr) {
-		m["a_is_sparse"] = value
-	}
-}
+// QuantizedReluXAttr is an optional argument to QuantizedReluX.
+type QuantizedReluXAttr func(optionalAttr)
 
-// SparseMatMulBIsSparse sets the optional b_is_sparse attribute to value.
-// If not specified, defaults to false
-func SparseMatMulBIsSparse(value bool) SparseMatMulAttr {
+// QuantizedReluXOutType sets the optional out_type attribute to value.
+// If not specified, defaults to DT_QUINT8
+func QuantizedReluXOutType(value tf.DataType) QuantizedReluXAttr {
 	return func(m optionalAttr) {
-		m["b_is_sparse"] = value
+		m["out_type"] = value
 	}
 }
 
-// Multiply matrix "a" by matrix "b".
+// Computes Quantized Rectified Linear X: `min(max(features, 0), max_value)`
 //
-// The inputs must be two-dimensional matrices and the inner dimension of "a" must
-// match the outer dimension of "b". Both "a" and "b" must be `Tensor`s not
-// `SparseTensor`s.  This op is optimized for the case where at least one of "a" or
-// "b" is sparse, in the sense that they have a large proportion of zero values.
-// The breakeven for using this versus a dense matrix multiply on one platform was
-// 30% zero values in the sparse matrix.
+// Arguments:
 //
-// The gradient computation of this operation will only take advantage of sparsity
-// in the input gradient when that gradient comes from a Relu.
-func SparseMatMul(scope *Scope, a tf.Output, b tf.Output, optional ...SparseMatMulAttr) (product tf.Output) {
+//
+//	min_features: The float value that the lowest quantized value represents.
+//	max_features: The float value that the highest quantized value represents.
+//
+// Returns Has the same output shape as "features".The float value that the lowest quantized value represents.The float value that the highest quantized value represents.
+func QuantizedReluX(scope *Scope, features tf.Output, max_value tf.Output, min_features tf.Output, max_features tf.Output, optional ...QuantizedReluXAttr) (activations tf.Output, min_activations tf.Output, max_activations tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -24845,156 +24699,57 @@ func SparseMatMul(scope *Scope, a tf.Output, b tf.Output, optional ...SparseMatM
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseMatMul",
+		Type: "QuantizedReluX",
 		Input: []tf.Input{
-			a, b,
+			features, max_value, min_features, max_features,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// ExperimentalThreadPoolHandleAttr is an optional argument to ExperimentalThreadPoolHandle.
-type ExperimentalThreadPoolHandleAttr func(optionalAttr)
-
-// ExperimentalThreadPoolHandleMaxIntraOpParallelism sets the optional max_intra_op_parallelism attribute to value.
+// Returns the truth value of (x < y) element-wise.
 //
-// value: The maximum degree of parallelism to use within operations that execute on this
-// threadpool.
-// If not specified, defaults to 1
-func ExperimentalThreadPoolHandleMaxIntraOpParallelism(value int64) ExperimentalThreadPoolHandleAttr {
-	return func(m optionalAttr) {
-		m["max_intra_op_parallelism"] = value
-	}
-}
-
-// ExperimentalThreadPoolHandleContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func ExperimentalThreadPoolHandleContainer(value string) ExperimentalThreadPoolHandleAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// ExperimentalThreadPoolHandleSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func ExperimentalThreadPoolHandleSharedName(value string) ExperimentalThreadPoolHandleAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Creates a dataset that uses a custom thread pool to compute `input_dataset`.
-//
-// Arguments:
-//	num_threads: The number of threads in the thread pool.
-//	display_name: A human-readable name for the threads that may be visible in some
-// visualizations.
-// threadpool.
-//
-// Returns A resource that can be consumed by one or more ExperimentalThreadPoolDataset
-// ops.
-func ExperimentalThreadPoolHandle(scope *Scope, num_threads int64, display_name string, optional ...ExperimentalThreadPoolHandleAttr) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"num_threads": num_threads, "display_name": display_name}
-	for _, a := range optional {
-		a(attrs)
+// *NOTE*: `Less` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func Less(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
 	opspec := tf.OpSpec{
-		Type: "ExperimentalThreadPoolHandle",
-
-		Attrs: attrs,
+		Type: "Less",
+		Input: []tf.Input{
+			x, y,
+		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// CudnnRNNCanonicalToParamsAttr is an optional argument to CudnnRNNCanonicalToParams.
-type CudnnRNNCanonicalToParamsAttr func(optionalAttr)
-
-// CudnnRNNCanonicalToParamsRnnMode sets the optional rnn_mode attribute to value.
-// If not specified, defaults to "lstm"
-func CudnnRNNCanonicalToParamsRnnMode(value string) CudnnRNNCanonicalToParamsAttr {
-	return func(m optionalAttr) {
-		m["rnn_mode"] = value
-	}
-}
-
-// CudnnRNNCanonicalToParamsInputMode sets the optional input_mode attribute to value.
-// If not specified, defaults to "linear_input"
-func CudnnRNNCanonicalToParamsInputMode(value string) CudnnRNNCanonicalToParamsAttr {
-	return func(m optionalAttr) {
-		m["input_mode"] = value
-	}
-}
-
-// CudnnRNNCanonicalToParamsDirection sets the optional direction attribute to value.
-// If not specified, defaults to "unidirectional"
-func CudnnRNNCanonicalToParamsDirection(value string) CudnnRNNCanonicalToParamsAttr {
-	return func(m optionalAttr) {
-		m["direction"] = value
-	}
-}
-
-// CudnnRNNCanonicalToParamsDropout sets the optional dropout attribute to value.
-// If not specified, defaults to 0
-func CudnnRNNCanonicalToParamsDropout(value float32) CudnnRNNCanonicalToParamsAttr {
-	return func(m optionalAttr) {
-		m["dropout"] = value
-	}
-}
+// RandomPoissonAttr is an optional argument to RandomPoisson.
+type RandomPoissonAttr func(optionalAttr)
 
-// CudnnRNNCanonicalToParamsSeed sets the optional seed attribute to value.
+// RandomPoissonSeed sets the optional seed attribute to value.
 // If not specified, defaults to 0
-func CudnnRNNCanonicalToParamsSeed(value int64) CudnnRNNCanonicalToParamsAttr {
+func RandomPoissonSeed(value int64) RandomPoissonAttr {
 	return func(m optionalAttr) {
 		m["seed"] = value
 	}
 }
 
-// CudnnRNNCanonicalToParamsSeed2 sets the optional seed2 attribute to value.
+// RandomPoissonSeed2 sets the optional seed2 attribute to value.
 // If not specified, defaults to 0
-func CudnnRNNCanonicalToParamsSeed2(value int64) CudnnRNNCanonicalToParamsAttr {
+func RandomPoissonSeed2(value int64) RandomPoissonAttr {
 	return func(m optionalAttr) {
 		m["seed2"] = value
 	}
 }
 
-// Converts CudnnRNN params from canonical form to usable form.
-//
-// Writes a set of weights into the opaque params buffer so they can be used in
-// upcoming training or inferences.
-//
-// Note that the params buffer may not be compatible across different GPUs. So any
-// save and restoration should be converted to and from the canonical weights and
-// biases.
+// Use RandomPoissonV2 instead.
 //
-// num_layers: Specifies the number of layers in the RNN model.
-// num_units: Specifies the size of the hidden state.
-// input_size: Specifies the size of the input state.
-// weights: the canonical form of weights that can be used for saving
-//     and restoration. They are more likely to be compatible across different
-//     generations.
-// biases: the canonical form of biases that can be used for saving
-//     and restoration. They are more likely to be compatible across different
-//     generations.
-// num_params: number of parameter sets for all layers.
-//     Each layer may contain multiple parameter sets, with each set consisting of
-//     a weight matrix and a bias vector.
-// rnn_mode: Indicates the type of the RNN model.
-// input_mode: Indicate whether there is a linear projection between the input and
-//     The actual computation before the first layer. 'skip_input' is only allowed
-//     when input_size == num_units; 'auto_select' implies 'skip_input' when
-//     input_size == num_units; otherwise, it implies 'linear_input'.
-// direction: Indicates whether a bidirectional model will be used.
-//     dir = (direction == bidirectional) ? 2 : 1
-// dropout: dropout probability. When set to 0., dropout is disabled.
-// seed: the 1st part of a seed to initialize dropout.
-// seed2: the 2nd part of a seed to initialize dropout.
-func CudnnRNNCanonicalToParams(scope *Scope, num_layers tf.Output, num_units tf.Output, input_size tf.Output, weights []tf.Output, biases []tf.Output, optional ...CudnnRNNCanonicalToParamsAttr) (params tf.Output) {
+// DEPRECATED at GraphDef version 25: Replaced by RandomPoissonV2
+func RandomPoisson(scope *Scope, shape tf.Output, rate tf.Output, optional ...RandomPoissonAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -25003,9 +24758,9 @@ func CudnnRNNCanonicalToParams(scope *Scope, num_layers tf.Output, num_units tf.
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "CudnnRNNCanonicalToParams",
+		Type: "RandomPoisson",
 		Input: []tf.Input{
-			num_layers, num_units, input_size, tf.OutputList(weights), tf.OutputList(biases),
+			shape, rate,
 		},
 		Attrs: attrs,
 	}
@@ -25013,50 +24768,37 @@ func CudnnRNNCanonicalToParams(scope *Scope, num_layers tf.Output, num_units tf.
 	return op.Output(0)
 }
 
-// Creates a dataset containing elements of first component of `input_dataset` having true in the last component.
-func FilterByLastComponentDataset(scope *Scope, input_dataset tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (output tf.Output) {
+// Returns the truth value of (x >= y) element-wise.
+//
+// *NOTE*: `GreaterEqual` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func GreaterEqual(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "FilterByLastComponentDataset",
+		Type: "GreaterEqual",
 		Input: []tf.Input{
-			input_dataset,
+			x, y,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// SumAttr is an optional argument to Sum.
-type SumAttr func(optionalAttr)
+// ApproximateEqualAttr is an optional argument to ApproximateEqual.
+type ApproximateEqualAttr func(optionalAttr)
 
-// SumKeepDims sets the optional keep_dims attribute to value.
-//
-// value: If true, retain reduced dimensions with length 1.
-// If not specified, defaults to false
-func SumKeepDims(value bool) SumAttr {
+// ApproximateEqualTolerance sets the optional tolerance attribute to value.
+// If not specified, defaults to 1e-05
+func ApproximateEqualTolerance(value float32) ApproximateEqualAttr {
 	return func(m optionalAttr) {
-		m["keep_dims"] = value
+		m["tolerance"] = value
 	}
 }
 
-// Computes the sum of elements across dimensions of a tensor.
-//
-// Reduces `input` along the dimensions given in `axis`. Unless
-// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
-// `axis`. If `keep_dims` is true, the reduced dimensions are
-// retained with length 1.
-//
-// Arguments:
-//	input: The tensor to reduce.
-//	axis: The dimensions to reduce. Must be in the range
-// `[-rank(input), rank(input))`.
-//
-// Returns The reduced tensor.
-func Sum(scope *Scope, input tf.Output, axis tf.Output, optional ...SumAttr) (output tf.Output) {
+// Returns the truth value of abs(x-y) < tolerance element-wise.
+func ApproximateEqual(scope *Scope, x tf.Output, y tf.Output, optional ...ApproximateEqualAttr) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -25065,9 +24807,9 @@ func Sum(scope *Scope, input tf.Output, axis tf.Output, optional ...SumAttr) (ou
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Sum",
+		Type: "ApproximateEqual",
 		Input: []tf.Input{
-			input, axis,
+			x, y,
 		},
 		Attrs: attrs,
 	}
@@ -25075,348 +24817,216 @@ func Sum(scope *Scope, input tf.Output, axis tf.Output, optional ...SumAttr) (ou
 	return op.Output(0)
 }
 
-// EnterAttr is an optional argument to Enter.
-type EnterAttr func(optionalAttr)
-
-// EnterIsConstant sets the optional is_constant attribute to value.
-//
-// value: If true, the output is constant within the child frame.
-// If not specified, defaults to false
-func EnterIsConstant(value bool) EnterAttr {
-	return func(m optionalAttr) {
-		m["is_constant"] = value
-	}
-}
-
-// EnterParallelIterations sets the optional parallel_iterations attribute to value.
-//
-// value: The number of iterations allowed to run in parallel.
-// If not specified, defaults to 10
-func EnterParallelIterations(value int64) EnterAttr {
-	return func(m optionalAttr) {
-		m["parallel_iterations"] = value
-	}
-}
-
-// Creates or finds a child frame, and makes `data` available to the child frame.
-//
-// This op is used together with `Exit` to create loops in the graph.
-// The unique `frame_name` is used by the `Executor` to identify frames. If
-// `is_constant` is true, `output` is a constant in the child frame; otherwise
-// it may be changed in the child frame. At most `parallel_iterations` iterations
-// are run in parallel in the child frame.
-//
-// Arguments:
-//	data: The tensor to be made available to the child frame.
-//	frame_name: The name of the child frame.
+// Returns the truth value of x OR y element-wise.
 //
-// Returns The same tensor as `data`.
-func Enter(scope *Scope, data tf.Output, frame_name string, optional ...EnterAttr) (output tf.Output) {
+// *NOTE*: `LogicalOr` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func LogicalOr(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"frame_name": frame_name}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "Enter",
+		Type: "LogicalOr",
 		Input: []tf.Input{
-			data,
+			x, y,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Add all input tensors element wise.
+// Selects elements from `x` or `y`, depending on `condition`.
 //
-// Arguments:
-//	inputs: Must all be the same size and shape.
-func AddN(scope *Scope, inputs []tf.Output) (sum tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "AddN",
-		Input: []tf.Input{
-			tf.OutputList(inputs),
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// TryRpcAttr is an optional argument to TryRpc.
-type TryRpcAttr func(optionalAttr)
-
-// TryRpcProtocol sets the optional protocol attribute to value.
+// The `x`, and `y` tensors must all have the same shape, and the
+// output will also have that shape.
 //
-// value: RPC protocol to use.  Empty string means use the default protocol.
-// Options include 'grpc'.
-// If not specified, defaults to ""
-func TryRpcProtocol(value string) TryRpcAttr {
-	return func(m optionalAttr) {
-		m["protocol"] = value
-	}
-}
-
-// TryRpcFailFast sets the optional fail_fast attribute to value.
+// The `condition` tensor must be a scalar if `x` and `y` are scalars.
+// If `x` and `y` are vectors or higher rank, then `condition` must be either a
+// scalar, a vector with size matching the first dimension of `x`, or must have
+// the same shape as `x`.
 //
-// value: `boolean`. If `true` (default), then failures to connect
-// (i.e., the server does not immediately respond) cause an RPC failure.
-// If not specified, defaults to true
-func TryRpcFailFast(value bool) TryRpcAttr {
-	return func(m optionalAttr) {
-		m["fail_fast"] = value
-	}
-}
-
-// TryRpcTimeoutInMs sets the optional timeout_in_ms attribute to value.
-//
-// value: `int`. If `0` (default), then the kernel will run the RPC
-// request and only time out if the RPC deadline passes or the session times out.
-// If this value is greater than `0`, then the op will raise an exception if
-// the RPC takes longer than `timeout_in_ms`.
-// If not specified, defaults to 0
-func TryRpcTimeoutInMs(value int64) TryRpcAttr {
-	return func(m optionalAttr) {
-		m["timeout_in_ms"] = value
-	}
-}
-
-// Perform batches of RPC requests.
+// The `condition` tensor acts as a mask that chooses, based on the value at each
+// element, whether the corresponding element / row in the output should be
+// taken from `x` (if true) or `y` (if false).
 //
-// This op asynchronously performs either a single RPC request, or a batch
-// of requests.  RPC requests are defined by three main parameters:
+// If `condition` is a vector and `x` and `y` are higher rank matrices, then
+// it chooses which row (outer dimension) to copy from `x` and `y`.
+// If `condition` has the same shape as `x` and `y`, then it chooses which
+// element to copy from `x` and `y`.
 //
-//   - `address` (the host+port or BNS address of the request)
-//   - `method` (the method name for the request)
-//   - `request` (the serialized proto string, or vector of strings,
-//      of the RPC request argument).
+// For example:
 //
-// For example, if you have an RPC service running on port localhost:2345,
-// and its interface is configured with the following proto declaration:
+// ```python
+// # 'condition' tensor is [[True,  False]
+// #                        [False, True]]
+// # 't' is [[1, 2],
+// #         [3, 4]]
+// # 'e' is [[5, 6],
+// #         [7, 8]]
+// select(condition, t, e)  # => [[1, 6], [7, 4]]
 //
-// ```
-// service MyService {
-//   rpc MyMethod(MyRequestProto) returns (MyResponseProto) {
-//   }
-// };
-// ```
 //
-// then call this op with arguments:
+// # 'condition' tensor is [True, False]
+// # 't' is [[1, 2],
+// #         [3, 4]]
+// # 'e' is [[5, 6],
+// #         [7, 8]]
+// select(condition, t, e) ==> [[1, 2],
+//                              [7, 8]]
 //
 // ```
-// address = "localhost:2345"
-// method = "MyService/MyMethod"
-// ```
-//
-// The `request` tensor is a string tensor representing serialized `MyRequestProto`
-// strings; and the output string tensor `response` will have the same shape
-// and contain (upon successful completion) corresponding serialized
-// `MyResponseProto` strings.
-//
-// For example, to send a single, empty, `MyRequestProto`, call
-// this op with `request = ""`.  To send 5 **parallel** empty requests,
-// call this op with `request = ["", "", "", "", ""]`.
-//
-// More generally, one can create a batch of `MyRequestProto` serialized protos
-// from regular batched tensors using the `encode_proto` op, and convert
-// the response `MyResponseProto` serialized protos to batched tensors
-// using the `decode_proto` op.
-//
-// **NOTE** Working with serialized proto strings is faster than instantiating
-// actual proto objects in memory, so no performance degradation is expected
-// compared to writing custom kernels for this workflow.
-//
-// Unlike the standard `Rpc` op, if the connection fails or the remote worker
-// returns an error status, this op does **not** reraise the exception.
-// Instead, the `status_code` and `status_message` entry for the corresponding RPC
-// call is set with the error returned from the RPC call.  The `response` tensor
-// will contain valid response values for those minibatch entries whose RPCs did
-// not fail; the rest of the entries will have empty strings.
 //
 // Arguments:
-//	address: `0-D` or `1-D`.  The address (i.e. host_name:port) of the RPC server.
-// If this tensor has more than 1 element, then multiple parallel rpc requests
-// are sent.  This argument broadcasts with `method` and `request`.
-//	method: `0-D` or `1-D`.  The method address on the RPC server.
-// If this tensor has more than 1 element, then multiple parallel rpc requests
-// are sent.  This argument broadcasts with `address` and `request`.
-//	request: `0-D` or `1-D`.  Serialized proto strings: the rpc request argument.
-// If this tensor has more than 1 element, then multiple parallel rpc requests
-// are sent.  This argument broadcasts with `address` and `method`.
 //
-// Returns Same shape as `request`. Serialized proto strings: the rpc responses.Same shape as `request`.  Values correspond to tensorflow Status enum codes.Same shape as `request`.  Values correspond to Status messages
-// returned from the RPC calls.
-func TryRpc(scope *Scope, address tf.Output, method tf.Output, request tf.Output, optional ...TryRpcAttr) (response tf.Output, status_code tf.Output, status_message tf.Output) {
+//	x: = A `Tensor` which may have the same shape as `condition`.
+// If `condition` is rank 1, `x` may have higher rank,
+// but its first dimension must match the size of `condition`.
+//	y: = A `Tensor` with the same type and shape as `x`.
+//
+// Returns = A `Tensor` with the same type and shape as `x` and `y`.
+func Select(scope *Scope, condition tf.Output, x tf.Output, y tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "TryRpc",
+		Type: "Select",
 		Input: []tf.Input{
-			address, method, request,
+			condition, x, y,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// InitializeTableFromTextFileV2Attr is an optional argument to InitializeTableFromTextFileV2.
-type InitializeTableFromTextFileV2Attr func(optionalAttr)
+// MatMulAttr is an optional argument to MatMul.
+type MatMulAttr func(optionalAttr)
 
-// InitializeTableFromTextFileV2VocabSize sets the optional vocab_size attribute to value.
-//
-// value: Number of elements of the file, use -1 if unknown.
-// If not specified, defaults to -1
+// MatMulTransposeA sets the optional transpose_a attribute to value.
 //
-// REQUIRES: value >= -1
-func InitializeTableFromTextFileV2VocabSize(value int64) InitializeTableFromTextFileV2Attr {
+// value: If true, "a" is transposed before multiplication.
+// If not specified, defaults to false
+func MatMulTransposeA(value bool) MatMulAttr {
 	return func(m optionalAttr) {
-		m["vocab_size"] = value
+		m["transpose_a"] = value
 	}
 }
 
-// InitializeTableFromTextFileV2Delimiter sets the optional delimiter attribute to value.
+// MatMulTransposeB sets the optional transpose_b attribute to value.
 //
-// value: Delimiter to separate fields in a line.
-// If not specified, defaults to "\t"
-func InitializeTableFromTextFileV2Delimiter(value string) InitializeTableFromTextFileV2Attr {
+// value: If true, "b" is transposed before multiplication.
+// If not specified, defaults to false
+func MatMulTransposeB(value bool) MatMulAttr {
 	return func(m optionalAttr) {
-		m["delimiter"] = value
+		m["transpose_b"] = value
 	}
 }
 
-// Initializes a table from a text file.
-//
-// It inserts one key-value pair into the table for each line of the file.
-// The key and value is extracted from the whole line content, elements from the
-// split line based on `delimiter` or the line number (starting from zero).
-// Where to extract the key and value from a line is specified by `key_index` and
-// `value_index`.
-//
-// - A value of -1 means use the line number(starting from zero), expects `int64`.
-// - A value of -2 means use the whole line content, expects `string`.
-// - A value >= 0 means use the index (starting at zero) of the split line based
-//   on `delimiter`.
+// Multiply the matrix "a" by the matrix "b".
 //
-// Arguments:
-//	table_handle: Handle to a table which will be initialized.
-//	filename: Filename of a vocabulary text file.
-//	key_index: Column index in a line to get the table `key` values from.
-//	value_index: Column index that represents information of a line to get the table
-// `value` values from.
+// The inputs must be two-dimensional matrices and the inner dimension of
+// "a" (after being transposed if transpose_a is true) must match the
+// outer dimension of "b" (after being transposed if transposed_b is
+// true).
 //
-// Returns the created operation.
-func InitializeTableFromTextFileV2(scope *Scope, table_handle tf.Output, filename tf.Output, key_index int64, value_index int64, optional ...InitializeTableFromTextFileV2Attr) (o *tf.Operation) {
+// *Note*: The default kernel implementation for MatMul on GPUs uses
+// cublas.
+func MatMul(scope *Scope, a tf.Output, b tf.Output, optional ...MatMulAttr) (product tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"key_index": key_index, "value_index": value_index}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "InitializeTableFromTextFileV2",
+		Type: "MatMul",
 		Input: []tf.Input{
-			table_handle, filename,
+			a, b,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
-}
-
-// MeanAttr is an optional argument to Mean.
-type MeanAttr func(optionalAttr)
-
-// MeanKeepDims sets the optional keep_dims attribute to value.
-//
-// value: If true, retain reduced dimensions with length 1.
-// If not specified, defaults to false
-func MeanKeepDims(value bool) MeanAttr {
-	return func(m optionalAttr) {
-		m["keep_dims"] = value
-	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Computes the mean of elements across dimensions of a tensor.
-//
-// Reduces `input` along the dimensions given in `axis`. Unless
-// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
-// `axis`. If `keep_dims` is true, the reduced dimensions are
-// retained with length 1.
+// Serializes the tree handle to a proto
 //
 // Arguments:
-//	input: The tensor to reduce.
-//	axis: The dimensions to reduce. Must be in the range
-// `[-rank(input), rank(input))`.
+//	tree_handle: Handle to the tree resource to be serialized.
 //
-// Returns The reduced tensor.
-func Mean(scope *Scope, input tf.Output, axis tf.Output, optional ...MeanAttr) (output tf.Output) {
+// Returns Serialied proto string of the tree resource.
+func TensorForestTreeSerialize(scope *Scope, tree_handle tf.Output) (tree_config tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "Mean",
+		Type: "TensorForestTreeSerialize",
 		Input: []tf.Input{
-			input, axis,
+			tree_handle,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ProdAttr is an optional argument to Prod.
-type ProdAttr func(optionalAttr)
+// SparseMatMulAttr is an optional argument to SparseMatMul.
+type SparseMatMulAttr func(optionalAttr)
 
-// ProdKeepDims sets the optional keep_dims attribute to value.
-//
-// value: If true, retain reduced dimensions with length 1.
+// SparseMatMulTransposeA sets the optional transpose_a attribute to value.
 // If not specified, defaults to false
-func ProdKeepDims(value bool) ProdAttr {
+func SparseMatMulTransposeA(value bool) SparseMatMulAttr {
 	return func(m optionalAttr) {
-		m["keep_dims"] = value
+		m["transpose_a"] = value
 	}
 }
 
-// Computes the product of elements across dimensions of a tensor.
-//
-// Reduces `input` along the dimensions given in `axis`. Unless
-// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
-// `axis`. If `keep_dims` is true, the reduced dimensions are
-// retained with length 1.
-//
-// Arguments:
-//	input: The tensor to reduce.
-//	axis: The dimensions to reduce. Must be in the range
-// `[-rank(input), rank(input))`.
-//
-// Returns The reduced tensor.
-func Prod(scope *Scope, input tf.Output, axis tf.Output, optional ...ProdAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
+// SparseMatMulTransposeB sets the optional transpose_b attribute to value.
+// If not specified, defaults to false
+func SparseMatMulTransposeB(value bool) SparseMatMulAttr {
+	return func(m optionalAttr) {
+		m["transpose_b"] = value
 	}
-	opspec := tf.OpSpec{
-		Type: "Prod",
-		Input: []tf.Input{
-			input, axis,
+}
+
+// SparseMatMulAIsSparse sets the optional a_is_sparse attribute to value.
+// If not specified, defaults to false
+func SparseMatMulAIsSparse(value bool) SparseMatMulAttr {
+	return func(m optionalAttr) {
+		m["a_is_sparse"] = value
+	}
+}
+
+// SparseMatMulBIsSparse sets the optional b_is_sparse attribute to value.
+// If not specified, defaults to false
+func SparseMatMulBIsSparse(value bool) SparseMatMulAttr {
+	return func(m optionalAttr) {
+		m["b_is_sparse"] = value
+	}
+}
+
+// Multiply matrix "a" by matrix "b".
+//
+// The inputs must be two-dimensional matrices and the inner dimension of "a" must
+// match the outer dimension of "b". Both "a" and "b" must be `Tensor`s not
+// `SparseTensor`s.  This op is optimized for the case where at least one of "a" or
+// "b" is sparse, in the sense that they have a large proportion of zero values.
+// The breakeven for using this versus a dense matrix multiply on one platform was
+// 30% zero values in the sparse matrix.
+//
+// The gradient computation of this operation will only take advantage of sparsity
+// in the input gradient when that gradient comes from a Relu.
+func SparseMatMul(scope *Scope, a tf.Output, b tf.Output, optional ...SparseMatMulAttr) (product tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "SparseMatMul",
+		Input: []tf.Input{
+			a, b,
 		},
 		Attrs: attrs,
 	}
@@ -25424,77 +25034,146 @@ func Prod(scope *Scope, input tf.Output, axis tf.Output, optional ...ProdAttr) (
 	return op.Output(0)
 }
 
-// ResizeBilinearAttr is an optional argument to ResizeBilinear.
-type ResizeBilinearAttr func(optionalAttr)
+// ExperimentalThreadPoolHandleAttr is an optional argument to ExperimentalThreadPoolHandle.
+type ExperimentalThreadPoolHandleAttr func(optionalAttr)
 
-// ResizeBilinearAlignCorners sets the optional align_corners attribute to value.
+// ExperimentalThreadPoolHandleMaxIntraOpParallelism sets the optional max_intra_op_parallelism attribute to value.
 //
-// value: If true, the centers of the 4 corner pixels of the input and output tensors are
-// aligned, preserving the values at the corner pixels. Defaults to false.
-// If not specified, defaults to false
-func ResizeBilinearAlignCorners(value bool) ResizeBilinearAttr {
+// value: The maximum degree of parallelism to use within operations that execute on this
+// threadpool.
+// If not specified, defaults to 1
+func ExperimentalThreadPoolHandleMaxIntraOpParallelism(value int64) ExperimentalThreadPoolHandleAttr {
 	return func(m optionalAttr) {
-		m["align_corners"] = value
+		m["max_intra_op_parallelism"] = value
 	}
 }
 
-// Resize `images` to `size` using bilinear interpolation.
-//
-// Input images can be of different types but output images are always float.
+// ExperimentalThreadPoolHandleContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func ExperimentalThreadPoolHandleContainer(value string) ExperimentalThreadPoolHandleAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// ExperimentalThreadPoolHandleSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func ExperimentalThreadPoolHandleSharedName(value string) ExperimentalThreadPoolHandleAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Creates a dataset that uses a custom thread pool to compute `input_dataset`.
 //
 // Arguments:
-//	images: 4-D with shape `[batch, height, width, channels]`.
-//	size: = A 1-D int32 Tensor of 2 elements: `new_height, new_width`.  The
-// new size for the images.
+//	num_threads: The number of threads in the thread pool.
+//	display_name: A human-readable name for the threads that may be visible in some
+// visualizations.
+// threadpool.
 //
-// Returns 4-D with shape
-// `[batch, new_height, new_width, channels]`.
-func ResizeBilinear(scope *Scope, images tf.Output, size tf.Output, optional ...ResizeBilinearAttr) (resized_images tf.Output) {
+// Returns A resource that can be consumed by one or more ExperimentalThreadPoolDataset
+// ops.
+func ExperimentalThreadPoolHandle(scope *Scope, num_threads int64, display_name string, optional ...ExperimentalThreadPoolHandleAttr) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"num_threads": num_threads, "display_name": display_name}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResizeBilinear",
-		Input: []tf.Input{
-			images, size,
-		},
+		Type: "ExperimentalThreadPoolHandle",
+
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// MaxAttr is an optional argument to Max.
-type MaxAttr func(optionalAttr)
+// CudnnRNNCanonicalToParamsAttr is an optional argument to CudnnRNNCanonicalToParams.
+type CudnnRNNCanonicalToParamsAttr func(optionalAttr)
 
-// MaxKeepDims sets the optional keep_dims attribute to value.
-//
-// value: If true, retain reduced dimensions with length 1.
-// If not specified, defaults to false
-func MaxKeepDims(value bool) MaxAttr {
+// CudnnRNNCanonicalToParamsRnnMode sets the optional rnn_mode attribute to value.
+// If not specified, defaults to "lstm"
+func CudnnRNNCanonicalToParamsRnnMode(value string) CudnnRNNCanonicalToParamsAttr {
 	return func(m optionalAttr) {
-		m["keep_dims"] = value
+		m["rnn_mode"] = value
 	}
 }
 
-// Computes the maximum of elements across dimensions of a tensor.
+// CudnnRNNCanonicalToParamsInputMode sets the optional input_mode attribute to value.
+// If not specified, defaults to "linear_input"
+func CudnnRNNCanonicalToParamsInputMode(value string) CudnnRNNCanonicalToParamsAttr {
+	return func(m optionalAttr) {
+		m["input_mode"] = value
+	}
+}
+
+// CudnnRNNCanonicalToParamsDirection sets the optional direction attribute to value.
+// If not specified, defaults to "unidirectional"
+func CudnnRNNCanonicalToParamsDirection(value string) CudnnRNNCanonicalToParamsAttr {
+	return func(m optionalAttr) {
+		m["direction"] = value
+	}
+}
+
+// CudnnRNNCanonicalToParamsDropout sets the optional dropout attribute to value.
+// If not specified, defaults to 0
+func CudnnRNNCanonicalToParamsDropout(value float32) CudnnRNNCanonicalToParamsAttr {
+	return func(m optionalAttr) {
+		m["dropout"] = value
+	}
+}
+
+// CudnnRNNCanonicalToParamsSeed sets the optional seed attribute to value.
+// If not specified, defaults to 0
+func CudnnRNNCanonicalToParamsSeed(value int64) CudnnRNNCanonicalToParamsAttr {
+	return func(m optionalAttr) {
+		m["seed"] = value
+	}
+}
+
+// CudnnRNNCanonicalToParamsSeed2 sets the optional seed2 attribute to value.
+// If not specified, defaults to 0
+func CudnnRNNCanonicalToParamsSeed2(value int64) CudnnRNNCanonicalToParamsAttr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// Converts CudnnRNN params from canonical form to usable form.
 //
-// Reduces `input` along the dimensions given in `axis`. Unless
-// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
-// `axis`. If `keep_dims` is true, the reduced dimensions are
-// retained with length 1.
+// Writes a set of weights into the opaque params buffer so they can be used in
+// upcoming training or inferences.
 //
-// Arguments:
-//	input: The tensor to reduce.
-//	axis: The dimensions to reduce. Must be in the range
-// `[-rank(input), rank(input))`.
+// Note that the params buffer may not be compatible across different GPUs. So any
+// save and restoration should be converted to and from the canonical weights and
+// biases.
 //
-// Returns The reduced tensor.
-func Max(scope *Scope, input tf.Output, axis tf.Output, optional ...MaxAttr) (output tf.Output) {
+// num_layers: Specifies the number of layers in the RNN model.
+// num_units: Specifies the size of the hidden state.
+// input_size: Specifies the size of the input state.
+// weights: the canonical form of weights that can be used for saving
+//     and restoration. They are more likely to be compatible across different
+//     generations.
+// biases: the canonical form of biases that can be used for saving
+//     and restoration. They are more likely to be compatible across different
+//     generations.
+// num_params: number of parameter sets for all layers.
+//     Each layer may contain multiple parameter sets, with each set consisting of
+//     a weight matrix and a bias vector.
+// rnn_mode: Indicates the type of the RNN model.
+// input_mode: Indicate whether there is a linear projection between the input and
+//     The actual computation before the first layer. 'skip_input' is only allowed
+//     when input_size == num_units; 'auto_select' implies 'skip_input' when
+//     input_size == num_units; otherwise, it implies 'linear_input'.
+// direction: Indicates whether a bidirectional model will be used.
+//     dir = (direction == bidirectional) ? 2 : 1
+// dropout: dropout probability. When set to 0., dropout is disabled.
+// seed: the 1st part of a seed to initialize dropout.
+// seed2: the 2nd part of a seed to initialize dropout.
+func CudnnRNNCanonicalToParams(scope *Scope, num_layers tf.Output, num_units tf.Output, input_size tf.Output, weights []tf.Output, biases []tf.Output, optional ...CudnnRNNCanonicalToParamsAttr) (params tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -25503,9 +25182,9 @@ func Max(scope *Scope, input tf.Output, axis tf.Output, optional ...MaxAttr) (ou
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Max",
+		Type: "CudnnRNNCanonicalToParams",
 		Input: []tf.Input{
-			input, axis,
+			num_layers, num_units, input_size, tf.OutputList(weights), tf.OutputList(biases),
 		},
 		Attrs: attrs,
 	}
@@ -25513,14 +25192,14 @@ func Max(scope *Scope, input tf.Output, axis tf.Output, optional ...MaxAttr) (ou
 	return op.Output(0)
 }
 
-// Creates a dataset that contains the unique elements of `input_dataset`.
-func ExperimentalUniqueDataset(scope *Scope, input_dataset tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+// Creates a dataset containing elements of first component of `input_dataset` having true in the last component.
+func FilterByLastComponentDataset(scope *Scope, input_dataset tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "ExperimentalUniqueDataset",
+		Type: "FilterByLastComponentDataset",
 		Input: []tf.Input{
 			input_dataset,
 		},
@@ -25530,27 +25209,33 @@ func ExperimentalUniqueDataset(scope *Scope, input_dataset tf.Output, output_typ
 	return op.Output(0)
 }
 
-// ArgMinAttr is an optional argument to ArgMin.
-type ArgMinAttr func(optionalAttr)
+// SumAttr is an optional argument to Sum.
+type SumAttr func(optionalAttr)
 
-// ArgMinOutputType sets the optional output_type attribute to value.
-// If not specified, defaults to DT_INT64
-func ArgMinOutputType(value tf.DataType) ArgMinAttr {
+// SumKeepDims sets the optional keep_dims attribute to value.
+//
+// value: If true, retain reduced dimensions with length 1.
+// If not specified, defaults to false
+func SumKeepDims(value bool) SumAttr {
 	return func(m optionalAttr) {
-		m["output_type"] = value
+		m["keep_dims"] = value
 	}
 }
 
-// Returns the index with the smallest value across dimensions of a tensor.
+// Computes the sum of elements across dimensions of a tensor.
 //
-// Note that in case of ties the identity of the return value is not guaranteed.
+// Reduces `input` along the dimensions given in `axis`. Unless
+// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+// `axis`. If `keep_dims` is true, the reduced dimensions are
+// retained with length 1.
 //
 // Arguments:
+//	input: The tensor to reduce.
+//	axis: The dimensions to reduce. Must be in the range
+// `[-rank(input), rank(input))`.
 //
-//	dimension: int32 or int64, must be in the range `[-rank(input), rank(input))`.
-// Describes which dimension of the input Tensor to reduce across. For vectors,
-// use dimension = 0.
-func ArgMin(scope *Scope, input tf.Output, dimension tf.Output, optional ...ArgMinAttr) (output tf.Output) {
+// Returns The reduced tensor.
+func Sum(scope *Scope, input tf.Output, axis tf.Output, optional ...SumAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -25559,9 +25244,9 @@ func ArgMin(scope *Scope, input tf.Output, dimension tf.Output, optional ...ArgM
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ArgMin",
+		Type: "Sum",
 		Input: []tf.Input{
-			input, dimension,
+			input, axis,
 		},
 		Attrs: attrs,
 	}
@@ -25569,437 +25254,482 @@ func ArgMin(scope *Scope, input tf.Output, dimension tf.Output, optional ...ArgM
 	return op.Output(0)
 }
 
-// Convert the quantized 'input' tensor into a lower-precision 'output', using the
+// EnterAttr is an optional argument to Enter.
+type EnterAttr func(optionalAttr)
+
+// EnterIsConstant sets the optional is_constant attribute to value.
 //
-// output range specified with 'requested_output_min' and 'requested_output_max'.
+// value: If true, the output is constant within the child frame.
+// If not specified, defaults to false
+func EnterIsConstant(value bool) EnterAttr {
+	return func(m optionalAttr) {
+		m["is_constant"] = value
+	}
+}
+
+// EnterParallelIterations sets the optional parallel_iterations attribute to value.
 //
-// [input_min, input_max] are scalar floats that specify the range for the float
-// interpretation of the 'input' data. For example, if input_min is -1.0f and
-// input_max is 1.0f, and we are dealing with quint16 quantized data, then a 0
-// value in the 16-bit data should be interpreted as -1.0f, and a 65535 means 1.0f.
+// value: The number of iterations allowed to run in parallel.
+// If not specified, defaults to 10
+func EnterParallelIterations(value int64) EnterAttr {
+	return func(m optionalAttr) {
+		m["parallel_iterations"] = value
+	}
+}
+
+// Creates or finds a child frame, and makes `data` available to the child frame.
 //
-// Arguments:
+// This op is used together with `Exit` to create loops in the graph.
+// The unique `frame_name` is used by the `Executor` to identify frames. If
+// `is_constant` is true, `output` is a constant in the child frame; otherwise
+// it may be changed in the child frame. At most `parallel_iterations` iterations
+// are run in parallel in the child frame.
 //
-//	input_min: The float value that the minimum quantized input value represents.
-//	input_max: The float value that the maximum quantized input value represents.
-//	requested_output_min: The float value that the minimum quantized output value represents.
-//	requested_output_max: The float value that the maximum quantized output value represents.
-//	out_type: The type of the output. Should be a lower bit depth than Tinput.
+// Arguments:
+//	data: The tensor to be made available to the child frame.
+//	frame_name: The name of the child frame.
 //
-// Returns The requested_output_min value is copied into this output.The requested_output_max value is copied into this output.
-func Requantize(scope *Scope, input tf.Output, input_min tf.Output, input_max tf.Output, requested_output_min tf.Output, requested_output_max tf.Output, out_type tf.DataType) (output tf.Output, output_min tf.Output, output_max tf.Output) {
+// Returns The same tensor as `data`.
+func Enter(scope *Scope, data tf.Output, frame_name string, optional ...EnterAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"out_type": out_type}
+	attrs := map[string]interface{}{"frame_name": frame_name}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "Requantize",
+		Type: "Enter",
 		Input: []tf.Input{
-			input, input_min, input_max, requested_output_min, requested_output_max,
+			data,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// Creates a dataset that emits the lines of one or more text files.
+// Add all input tensors element wise.
 //
 // Arguments:
-//	filenames: A scalar or a vector containing the name(s) of the file(s) to be
-// read.
-//	compression_type: A scalar containing either (i) the empty string (no
-// compression), (ii) "ZLIB", or (iii) "GZIP".
-//	buffer_size: A scalar containing the number of bytes to buffer.
-func TextLineDataset(scope *Scope, filenames tf.Output, compression_type tf.Output, buffer_size tf.Output) (handle tf.Output) {
+//	inputs: Must all be the same size and shape.
+func AddN(scope *Scope, inputs []tf.Output) (sum tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "TextLineDataset",
+		Type: "AddN",
 		Input: []tf.Input{
-			filenames, compression_type, buffer_size,
+			tf.OutputList(inputs),
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes the sum along segments of a tensor.
-//
-// Read
-// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
-// for an explanation of segments.
-//
-// Computes a tensor such that
-// \\(output_i = \sum_j data_j\\) where sum is over `j` such
-// that `segment_ids[j] == i`.
-//
-// If the sum is empty for a given segment ID `i`, `output[i] = 0`.
-//
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/SegmentSum.png" alt>
-// </div>
-//
-// Arguments:
+// TryRpcAttr is an optional argument to TryRpc.
+type TryRpcAttr func(optionalAttr)
+
+// TryRpcProtocol sets the optional protocol attribute to value.
 //
-//	segment_ids: A 1-D tensor whose size is equal to the size of `data`'s
-// first dimension.  Values should be sorted and can be repeated.
+// value: RPC protocol to use.  Empty string means use the default protocol.
+// Options include 'grpc'.
+// If not specified, defaults to ""
+func TryRpcProtocol(value string) TryRpcAttr {
+	return func(m optionalAttr) {
+		m["protocol"] = value
+	}
+}
+
+// TryRpcFailFast sets the optional fail_fast attribute to value.
 //
-// Returns Has same shape as data, except for dimension 0 which
-// has size `k`, the number of segments.
-func SegmentSum(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
+// value: `boolean`. If `true` (default), then failures to connect
+// (i.e., the server does not immediately respond) cause an RPC failure.
+// If not specified, defaults to true
+func TryRpcFailFast(value bool) TryRpcAttr {
+	return func(m optionalAttr) {
+		m["fail_fast"] = value
 	}
-	opspec := tf.OpSpec{
-		Type: "SegmentSum",
-		Input: []tf.Input{
-			data, segment_ids,
-		},
+}
+
+// TryRpcTimeoutInMs sets the optional timeout_in_ms attribute to value.
+//
+// value: `int`. If `0` (default), then the kernel will run the RPC
+// request and only time out if the RPC deadline passes or the session times out.
+// If this value is greater than `0`, then the op will raise an exception if
+// the RPC takes longer than `timeout_in_ms`.
+// If not specified, defaults to 0
+func TryRpcTimeoutInMs(value int64) TryRpcAttr {
+	return func(m optionalAttr) {
+		m["timeout_in_ms"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Computes the mean along segments of a tensor.
+// Perform batches of RPC requests.
 //
-// Read
-// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
-// for an explanation of segments.
+// This op asynchronously performs either a single RPC request, or a batch
+// of requests.  RPC requests are defined by three main parameters:
 //
-// Computes a tensor such that
-// \\(output_i = \frac{\sum_j data_j}{N}\\) where `mean` is
-// over `j` such that `segment_ids[j] == i` and `N` is the total number of
-// values summed.
+//   - `address` (the host+port or BNS address of the request)
+//   - `method` (the method name for the request)
+//   - `request` (the serialized proto string, or vector of strings,
+//      of the RPC request argument).
 //
-// If the mean is empty for a given segment ID `i`, `output[i] = 0`.
+// For example, if you have an RPC service running on port localhost:2345,
+// and its interface is configured with the following proto declaration:
 //
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/SegmentMean.png" alt>
-// </div>
+// ```
+// service MyService {
+//   rpc MyMethod(MyRequestProto) returns (MyResponseProto) {
+//   }
+// };
+// ```
 //
-// Arguments:
+// then call this op with arguments:
 //
-//	segment_ids: A 1-D tensor whose size is equal to the size of `data`'s
-// first dimension.  Values should be sorted and can be repeated.
+// ```
+// address = "localhost:2345"
+// method = "MyService/MyMethod"
+// ```
 //
-// Returns Has same shape as data, except for dimension 0 which
-// has size `k`, the number of segments.
-func SegmentMean(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "SegmentMean",
-		Input: []tf.Input{
-			data, segment_ids,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes the minimum along segments of a tensor.
+// The `request` tensor is a string tensor representing serialized `MyRequestProto`
+// strings; and the output string tensor `response` will have the same shape
+// and contain (upon successful completion) corresponding serialized
+// `MyResponseProto` strings.
 //
-// Read
-// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
-// for an explanation of segments.
+// For example, to send a single, empty, `MyRequestProto`, call
+// this op with `request = ""`.  To send 5 **parallel** empty requests,
+// call this op with `request = ["", "", "", "", ""]`.
 //
-// Computes a tensor such that
-// \\(output_i = \min_j(data_j)\\) where `min` is over `j` such
-// that `segment_ids[j] == i`.
+// More generally, one can create a batch of `MyRequestProto` serialized protos
+// from regular batched tensors using the `encode_proto` op, and convert
+// the response `MyResponseProto` serialized protos to batched tensors
+// using the `decode_proto` op.
 //
-// If the min is empty for a given segment ID `i`, `output[i] = 0`.
+// **NOTE** Working with serialized proto strings is faster than instantiating
+// actual proto objects in memory, so no performance degradation is expected
+// compared to writing custom kernels for this workflow.
 //
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/SegmentMin.png" alt>
-// </div>
+// Unlike the standard `Rpc` op, if the connection fails or the remote worker
+// returns an error status, this op does **not** reraise the exception.
+// Instead, the `status_code` and `status_message` entry for the corresponding RPC
+// call is set with the error returned from the RPC call.  The `response` tensor
+// will contain valid response values for those minibatch entries whose RPCs did
+// not fail; the rest of the entries will have empty strings.
 //
 // Arguments:
+//	address: `0-D` or `1-D`.  The address (i.e. host_name:port) of the RPC server.
+// If this tensor has more than 1 element, then multiple parallel rpc requests
+// are sent.  This argument broadcasts with `method` and `request`.
+//	method: `0-D` or `1-D`.  The method address on the RPC server.
+// If this tensor has more than 1 element, then multiple parallel rpc requests
+// are sent.  This argument broadcasts with `address` and `request`.
+//	request: `0-D` or `1-D`.  Serialized proto strings: the rpc request argument.
+// If this tensor has more than 1 element, then multiple parallel rpc requests
+// are sent.  This argument broadcasts with `address` and `method`.
 //
-//	segment_ids: A 1-D tensor whose size is equal to the size of `data`'s
-// first dimension.  Values should be sorted and can be repeated.
-//
-// Returns Has same shape as data, except for dimension 0 which
-// has size `k`, the number of segments.
-func SegmentMin(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) {
+// Returns Same shape as `request`. Serialized proto strings: the rpc responses.Same shape as `request`.  Values correspond to tensorflow Status enum codes.Same shape as `request`.  Values correspond to Status messages
+// returned from the RPC calls.
+func TryRpc(scope *Scope, address tf.Output, method tf.Output, request tf.Output, optional ...TryRpcAttr) (response tf.Output, status_code tf.Output, status_message tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "SegmentMin",
+		Type: "TryRpc",
 		Input: []tf.Input{
-			data, segment_ids,
+			address, method, request,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Removes keys and its associated values from a table.
-//
-// The tensor `keys` must of the same type as the keys of the table. Keys not
-// already in the table are silently ignored.
+// InitializeTableFromTextFileV2Attr is an optional argument to InitializeTableFromTextFileV2.
+type InitializeTableFromTextFileV2Attr func(optionalAttr)
+
+// InitializeTableFromTextFileV2VocabSize sets the optional vocab_size attribute to value.
 //
-// Arguments:
-//	table_handle: Handle to the table.
-//	keys: Any shape.  Keys of the elements to remove.
+// value: Number of elements of the file, use -1 if unknown.
+// If not specified, defaults to -1
 //
-// Returns the created operation.
-func LookupTableRemoveV2(scope *Scope, table_handle tf.Output, keys tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "LookupTableRemoveV2",
-		Input: []tf.Input{
-			table_handle, keys,
-		},
+// REQUIRES: value >= -1
+func InitializeTableFromTextFileV2VocabSize(value int64) InitializeTableFromTextFileV2Attr {
+	return func(m optionalAttr) {
+		m["vocab_size"] = value
 	}
-	return scope.AddOperation(opspec)
 }
 
-// Computes the sum along segments of a tensor.
+// InitializeTableFromTextFileV2Delimiter sets the optional delimiter attribute to value.
 //
-// Read
-// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
-// for an explanation of segments.
-//
-// Computes a tensor such that
-// \\(output[i] = \sum_{j...} data[j...]\\) where the sum is over tuples `j...` such
-// that `segment_ids[j...] == i`.  Unlike `SegmentSum`, `segment_ids`
-// need not be sorted and need not cover all values in the full
-// range of valid values.
-//
-// If the sum is empty for a given segment ID `i`, `output[i] = 0`.
-// If the given segment ID `i` is negative, the value is dropped and will not be
-// added to the sum of the segment.
+// value: Delimiter to separate fields in a line.
+// If not specified, defaults to "\t"
+func InitializeTableFromTextFileV2Delimiter(value string) InitializeTableFromTextFileV2Attr {
+	return func(m optionalAttr) {
+		m["delimiter"] = value
+	}
+}
+
+// Initializes a table from a text file.
 //
-// `num_segments` should equal the number of distinct segment IDs.
+// It inserts one key-value pair into the table for each line of the file.
+// The key and value is extracted from the whole line content, elements from the
+// split line based on `delimiter` or the line number (starting from zero).
+// Where to extract the key and value from a line is specified by `key_index` and
+// `value_index`.
 //
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/UnsortedSegmentSum.png" alt>
-// </div>
+// - A value of -1 means use the line number(starting from zero), expects `int64`.
+// - A value of -2 means use the whole line content, expects `string`.
+// - A value >= 0 means use the index (starting at zero) of the split line based
+//   on `delimiter`.
 //
 // Arguments:
+//	table_handle: Handle to a table which will be initialized.
+//	filename: Filename of a vocabulary text file.
+//	key_index: Column index in a line to get the table `key` values from.
+//	value_index: Column index that represents information of a line to get the table
+// `value` values from.
 //
-//	segment_ids: A tensor whose shape is a prefix of `data.shape`.
-//
-//
-// Returns Has same shape as data, except for the first `segment_ids.rank`
-// dimensions, which are replaced with a single dimension which has size
-// `num_segments`.
-func UnsortedSegmentSum(scope *Scope, data tf.Output, segment_ids tf.Output, num_segments tf.Output) (output tf.Output) {
+// Returns the created operation.
+func InitializeTableFromTextFileV2(scope *Scope, table_handle tf.Output, filename tf.Output, key_index int64, value_index int64, optional ...InitializeTableFromTextFileV2Attr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"key_index": key_index, "value_index": value_index}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "UnsortedSegmentSum",
+		Type: "InitializeTableFromTextFileV2",
 		Input: []tf.Input{
-			data, segment_ids, num_segments,
+			table_handle, filename,
 		},
+		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// Computes the product along segments of a tensor.
-//
-// Read
-// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#segmentation)
-// for an explanation of segments.
-//
-// This operator is similar to the unsorted segment sum operator found
-// [(here)](../../../api_docs/python/math_ops.md#UnsortedSegmentSum).
-// Instead of computing the sum over segments, it computes the product of all
-// entries belonging to a segment such that:
-//
-// \\(output_i = \prod_{j...} data[j...]\\) where the product is over tuples
-// `j...` such that `segment_ids[j...] == i`.
+// MeanAttr is an optional argument to Mean.
+type MeanAttr func(optionalAttr)
+
+// MeanKeepDims sets the optional keep_dims attribute to value.
 //
-// If there is no entry for a given segment ID `i`, it outputs 1.
+// value: If true, retain reduced dimensions with length 1.
+// If not specified, defaults to false
+func MeanKeepDims(value bool) MeanAttr {
+	return func(m optionalAttr) {
+		m["keep_dims"] = value
+	}
+}
+
+// Computes the mean of elements across dimensions of a tensor.
 //
-// If the given segment ID `i` is negative, then the corresponding value is
-// dropped, and will not be included in the result.
+// Reduces `input` along the dimensions given in `axis`. Unless
+// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+// `axis`. If `keep_dims` is true, the reduced dimensions are
+// retained with length 1.
 //
 // Arguments:
+//	input: The tensor to reduce.
+//	axis: The dimensions to reduce. Must be in the range
+// `[-rank(input), rank(input))`.
 //
-//	segment_ids: A tensor whose shape is a prefix of `data.shape`.
-//
-//
-// Returns Has same shape as data, except for the first `segment_ids.rank`
-// dimensions, which are replaced with a single dimension which has size
-// `num_segments`.
-func UnsortedSegmentProd(scope *Scope, data tf.Output, segment_ids tf.Output, num_segments tf.Output) (output tf.Output) {
+// Returns The reduced tensor.
+func Mean(scope *Scope, input tf.Output, axis tf.Output, optional ...MeanAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "UnsortedSegmentProd",
+		Type: "Mean",
 		Input: []tf.Input{
-			data, segment_ids, num_segments,
+			input, axis,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes the mean along sparse segments of a tensor.
+// ProdAttr is an optional argument to Prod.
+type ProdAttr func(optionalAttr)
+
+// ProdKeepDims sets the optional keep_dims attribute to value.
 //
-// Read
-// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
-// for an explanation of segments.
+// value: If true, retain reduced dimensions with length 1.
+// If not specified, defaults to false
+func ProdKeepDims(value bool) ProdAttr {
+	return func(m optionalAttr) {
+		m["keep_dims"] = value
+	}
+}
+
+// Computes the product of elements across dimensions of a tensor.
 //
-// Like `SegmentMean`, but `segment_ids` can have rank less than `data`'s first
-// dimension, selecting a subset of dimension 0, specified by `indices`.
+// Reduces `input` along the dimensions given in `axis`. Unless
+// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+// `axis`. If `keep_dims` is true, the reduced dimensions are
+// retained with length 1.
 //
 // Arguments:
+//	input: The tensor to reduce.
+//	axis: The dimensions to reduce. Must be in the range
+// `[-rank(input), rank(input))`.
 //
-//	indices: A 1-D tensor. Has same rank as `segment_ids`.
-//	segment_ids: A 1-D tensor. Values should be sorted and can be repeated.
-//
-// Returns Has same shape as data, except for dimension 0 which
-// has size `k`, the number of segments.
-func SparseSegmentMean(scope *Scope, data tf.Output, indices tf.Output, segment_ids tf.Output) (output tf.Output) {
+// Returns The reduced tensor.
+func Prod(scope *Scope, input tf.Output, axis tf.Output, optional ...ProdAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "SparseSegmentMean",
+		Type: "Prod",
 		Input: []tf.Input{
-			data, indices, segment_ids,
+			input, axis,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Deserializes a serialized tree ensemble config and replaces current tree
+// ResizeBilinearAttr is an optional argument to ResizeBilinear.
+type ResizeBilinearAttr func(optionalAttr)
+
+// ResizeBilinearAlignCorners sets the optional align_corners attribute to value.
 //
-// ensemble.
+// value: If true, the centers of the 4 corner pixels of the input and output tensors are
+// aligned, preserving the values at the corner pixels. Defaults to false.
+// If not specified, defaults to false
+func ResizeBilinearAlignCorners(value bool) ResizeBilinearAttr {
+	return func(m optionalAttr) {
+		m["align_corners"] = value
+	}
+}
+
+// Resize `images` to `size` using bilinear interpolation.
+//
+// Input images can be of different types but output images are always float.
 //
 // Arguments:
-//	tree_ensemble_handle: Handle to the tree ensemble.
-//	stamp_token: Token to use as the new value of the resource stamp.
-//	tree_ensemble_serialized: Serialized proto of the ensemble.
+//	images: 4-D with shape `[batch, height, width, channels]`.
+//	size: = A 1-D int32 Tensor of 2 elements: `new_height, new_width`.  The
+// new size for the images.
 //
-// Returns the created operation.
-func BoostedTreesDeserializeEnsemble(scope *Scope, tree_ensemble_handle tf.Output, stamp_token tf.Output, tree_ensemble_serialized tf.Output) (o *tf.Operation) {
+// Returns 4-D with shape
+// `[batch, new_height, new_width, channels]`.
+func ResizeBilinear(scope *Scope, images tf.Output, size tf.Output, optional ...ResizeBilinearAttr) (resized_images tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "BoostedTreesDeserializeEnsemble",
+		Type: "ResizeBilinear",
 		Input: []tf.Input{
-			tree_ensemble_handle, stamp_token, tree_ensemble_serialized,
+			images, size,
 		},
+		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Transforms a tf.Example proto (as a string) into typed tensors.
+// MaxAttr is an optional argument to Max.
+type MaxAttr func(optionalAttr)
+
+// MaxKeepDims sets the optional keep_dims attribute to value.
+//
+// value: If true, retain reduced dimensions with length 1.
+// If not specified, defaults to false
+func MaxKeepDims(value bool) MaxAttr {
+	return func(m optionalAttr) {
+		m["keep_dims"] = value
+	}
+}
+
+// Computes the maximum of elements across dimensions of a tensor.
+//
+// Reduces `input` along the dimensions given in `axis`. Unless
+// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+// `axis`. If `keep_dims` is true, the reduced dimensions are
+// retained with length 1.
 //
 // Arguments:
-//	serialized: A vector containing a batch of binary serialized Example protos.
-//	dense_defaults: A list of Tensors (some may be empty), whose length matches
-// the length of `dense_keys`. dense_defaults[j] provides default values
-// when the example's feature_map lacks dense_key[j].  If an empty Tensor is
-// provided for dense_defaults[j], then the Feature dense_keys[j] is required.
-// The input type is inferred from dense_defaults[j], even when it's empty.
-// If dense_defaults[j] is not empty, and dense_shapes[j] is fully defined,
-// then the shape of dense_defaults[j] must match that of dense_shapes[j].
-// If dense_shapes[j] has an undefined major dimension (variable strides dense
-// feature), dense_defaults[j] must contain a single element:
-// the padding element.
-//	num_sparse: The number of sparse features to be parsed from the example. This
-// must match the lengths of `sparse_keys` and `sparse_types`.
-//	sparse_keys: A list of `num_sparse` strings.
-// The keys expected in the Examples' features associated with sparse values.
-//	dense_keys: The keys expected in the Examples' features associated with dense
-// values.
-//	sparse_types: A list of `num_sparse` types; the data types of data in each
-// Feature given in sparse_keys.
-// Currently the ParseSingleExample op supports DT_FLOAT (FloatList),
-// DT_INT64 (Int64List), and DT_STRING (BytesList).
-//	dense_shapes: The shapes of data in each Feature given in dense_keys.
-// The length of this list must match the length of `dense_keys`.  The
-// number of elements in the Feature corresponding to dense_key[j] must
-// always equal dense_shapes[j].NumEntries().  If dense_shapes[j] ==
-// (D0, D1, ..., DN) then the shape of output Tensor dense_values[j]
-// will be (D0, D1, ..., DN): In the case dense_shapes[j] = (-1, D1,
-// ..., DN), the shape of the output Tensor dense_values[j] will be (M,
-// D1, .., DN), where M is the number of blocks of elements of length
-// D1 * .... * DN, in the input.
-func ParseSingleExample(scope *Scope, serialized tf.Output, dense_defaults []tf.Output, num_sparse int64, sparse_keys []string, dense_keys []string, sparse_types []tf.DataType, dense_shapes []tf.Shape) (sparse_indices []tf.Output, sparse_values []tf.Output, sparse_shapes []tf.Output, dense_values []tf.Output) {
+//	input: The tensor to reduce.
+//	axis: The dimensions to reduce. Must be in the range
+// `[-rank(input), rank(input))`.
+//
+// Returns The reduced tensor.
+func Max(scope *Scope, input tf.Output, axis tf.Output, optional ...MaxAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"num_sparse": num_sparse, "sparse_keys": sparse_keys, "dense_keys": dense_keys, "sparse_types": sparse_types, "dense_shapes": dense_shapes}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "ParseSingleExample",
+		Type: "Max",
 		Input: []tf.Input{
-			serialized, tf.OutputList(dense_defaults),
+			input, axis,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Creates a dataset that contains the unique elements of `input_dataset`.
+func ExperimentalUniqueDataset(scope *Scope, input_dataset tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	var idx int
-	var err error
-	if sparse_indices, idx, err = makeOutputList(op, idx, "sparse_indices"); err != nil {
-		scope.UpdateErr("ParseSingleExample", err)
-		return
-	}
-	if sparse_values, idx, err = makeOutputList(op, idx, "sparse_values"); err != nil {
-		scope.UpdateErr("ParseSingleExample", err)
-		return
-	}
-	if sparse_shapes, idx, err = makeOutputList(op, idx, "sparse_shapes"); err != nil {
-		scope.UpdateErr("ParseSingleExample", err)
-		return
-	}
-	if dense_values, idx, err = makeOutputList(op, idx, "dense_values"); err != nil {
-		scope.UpdateErr("ParseSingleExample", err)
-		return
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalUniqueDataset",
+		Input: []tf.Input{
+			input_dataset,
+		},
+		Attrs: attrs,
 	}
-	return sparse_indices, sparse_values, sparse_shapes, dense_values
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// WholeFileReaderV2Attr is an optional argument to WholeFileReaderV2.
-type WholeFileReaderV2Attr func(optionalAttr)
+// ArgMinAttr is an optional argument to ArgMin.
+type ArgMinAttr func(optionalAttr)
 
-// WholeFileReaderV2Container sets the optional container attribute to value.
-//
-// value: If non-empty, this reader is placed in the given container.
-// Otherwise, a default container is used.
-// If not specified, defaults to ""
-func WholeFileReaderV2Container(value string) WholeFileReaderV2Attr {
+// ArgMinOutputType sets the optional output_type attribute to value.
+// If not specified, defaults to DT_INT64
+func ArgMinOutputType(value tf.DataType) ArgMinAttr {
 	return func(m optionalAttr) {
-		m["container"] = value
+		m["output_type"] = value
 	}
 }
 
-// WholeFileReaderV2SharedName sets the optional shared_name attribute to value.
+// Returns the index with the smallest value across dimensions of a tensor.
 //
-// value: If non-empty, this reader is named in the given bucket
-// with this shared_name. Otherwise, the node name is used instead.
-// If not specified, defaults to ""
-func WholeFileReaderV2SharedName(value string) WholeFileReaderV2Attr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// A Reader that outputs the entire contents of a file as a value.
+// Note that in case of ties the identity of the return value is not guaranteed.
 //
-// To use, enqueue filenames in a Queue.  The output of ReaderRead will
-// be a filename (key) and the contents of that file (value).
+// Arguments:
 //
-// Returns The handle to reference the Reader.
-func WholeFileReaderV2(scope *Scope, optional ...WholeFileReaderV2Attr) (reader_handle tf.Output) {
+//	dimension: int32 or int64, must be in the range `[-rank(input), rank(input))`.
+// Describes which dimension of the input Tensor to reduce across. For vectors,
+// use dimension = 0.
+func ArgMin(scope *Scope, input tf.Output, dimension tf.Output, optional ...ArgMinAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -26008,366 +25738,447 @@ func WholeFileReaderV2(scope *Scope, optional ...WholeFileReaderV2Attr) (reader_
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "WholeFileReaderV2",
-
+		Type: "ArgMin",
+		Input: []tf.Input{
+			input, dimension,
+		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Pop the element at the top of the stack.
+// Convert the quantized 'input' tensor into a lower-precision 'output', using the
+//
+// output range specified with 'requested_output_min' and 'requested_output_max'.
+//
+// [input_min, input_max] are scalar floats that specify the range for the float
+// interpretation of the 'input' data. For example, if input_min is -1.0f and
+// input_max is 1.0f, and we are dealing with quint16 quantized data, then a 0
+// value in the 16-bit data should be interpreted as -1.0f, and a 65535 means 1.0f.
 //
 // Arguments:
-//	handle: The handle to a stack.
-//	elem_type: The type of the elem that is popped.
 //
-// Returns The tensor that is popped from the top of the stack.
-func StackPopV2(scope *Scope, handle tf.Output, elem_type tf.DataType) (elem tf.Output) {
+//	input_min: The float value that the minimum quantized input value represents.
+//	input_max: The float value that the maximum quantized input value represents.
+//	requested_output_min: The float value that the minimum quantized output value represents.
+//	requested_output_max: The float value that the maximum quantized output value represents.
+//	out_type: The type of the output. Should be a lower bit depth than Tinput.
+//
+// Returns The requested_output_min value is copied into this output.The requested_output_max value is copied into this output.
+func Requantize(scope *Scope, input tf.Output, input_min tf.Output, input_max tf.Output, requested_output_min tf.Output, requested_output_max tf.Output, out_type tf.DataType) (output tf.Output, output_min tf.Output, output_max tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"elem_type": elem_type}
+	attrs := map[string]interface{}{"out_type": out_type}
 	opspec := tf.OpSpec{
-		Type: "StackPopV2",
+		Type: "Requantize",
 		Input: []tf.Input{
-			handle,
+			input, input_min, input_max, requested_output_min, requested_output_max,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Computes hyperbolic cosine of x element-wise.
-func Cosh(scope *Scope, x tf.Output) (y tf.Output) {
+// Creates a dataset that emits the lines of one or more text files.
+//
+// Arguments:
+//	filenames: A scalar or a vector containing the name(s) of the file(s) to be
+// read.
+//	compression_type: A scalar containing either (i) the empty string (no
+// compression), (ii) "ZLIB", or (iii) "GZIP".
+//	buffer_size: A scalar containing the number of bytes to buffer.
+func TextLineDataset(scope *Scope, filenames tf.Output, compression_type tf.Output, buffer_size tf.Output) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Cosh",
+		Type: "TextLineDataset",
 		Input: []tf.Input{
-			x,
+			filenames, compression_type, buffer_size,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes the mean along sparse segments of a tensor.
-//
-// Like `SparseSegmentMean`, but allows missing ids in `segment_ids`. If an id is
-// misisng, the `output` tensor at that position will be zeroed.
+// Computes the sum along segments of a tensor.
 //
 // Read
 // [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
 // for an explanation of segments.
 //
+// Computes a tensor such that
+// \\(output_i = \sum_j data_j\\) where sum is over `j` such
+// that `segment_ids[j] == i`.
+//
+// If the sum is empty for a given segment ID `i`, `output[i] = 0`.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/SegmentSum.png" alt>
+// </div>
+//
 // Arguments:
 //
-//	indices: A 1-D tensor. Has same rank as `segment_ids`.
-//	segment_ids: A 1-D tensor. Values should be sorted and can be repeated.
-//	num_segments: Should equal the number of distinct segment IDs.
+//	segment_ids: A 1-D tensor whose size is equal to the size of `data`'s
+// first dimension.  Values should be sorted and can be repeated.
 //
-// Returns Has same shape as data, except for dimension 0 which has size
-// `num_segments`.
-func SparseSegmentMeanWithNumSegments(scope *Scope, data tf.Output, indices tf.Output, segment_ids tf.Output, num_segments tf.Output) (output tf.Output) {
+// Returns Has same shape as data, except for dimension 0 which
+// has size `k`, the number of segments.
+func SegmentSum(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseSegmentMeanWithNumSegments",
+		Type: "SegmentSum",
 		Input: []tf.Input{
-			data, indices, segment_ids, num_segments,
+			data, segment_ids,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// CudnnRNNParamsSizeAttr is an optional argument to CudnnRNNParamsSize.
-type CudnnRNNParamsSizeAttr func(optionalAttr)
-
-// CudnnRNNParamsSizeRnnMode sets the optional rnn_mode attribute to value.
-// If not specified, defaults to "lstm"
-func CudnnRNNParamsSizeRnnMode(value string) CudnnRNNParamsSizeAttr {
-	return func(m optionalAttr) {
-		m["rnn_mode"] = value
+// Computes the mean along segments of a tensor.
+//
+// Read
+// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+// for an explanation of segments.
+//
+// Computes a tensor such that
+// \\(output_i = \frac{\sum_j data_j}{N}\\) where `mean` is
+// over `j` such that `segment_ids[j] == i` and `N` is the total number of
+// values summed.
+//
+// If the mean is empty for a given segment ID `i`, `output[i] = 0`.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/SegmentMean.png" alt>
+// </div>
+//
+// Arguments:
+//
+//	segment_ids: A 1-D tensor whose size is equal to the size of `data`'s
+// first dimension.  Values should be sorted and can be repeated.
+//
+// Returns Has same shape as data, except for dimension 0 which
+// has size `k`, the number of segments.
+func SegmentMean(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// CudnnRNNParamsSizeInputMode sets the optional input_mode attribute to value.
-// If not specified, defaults to "linear_input"
-func CudnnRNNParamsSizeInputMode(value string) CudnnRNNParamsSizeAttr {
-	return func(m optionalAttr) {
-		m["input_mode"] = value
+	opspec := tf.OpSpec{
+		Type: "SegmentMean",
+		Input: []tf.Input{
+			data, segment_ids,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// CudnnRNNParamsSizeDirection sets the optional direction attribute to value.
-// If not specified, defaults to "unidirectional"
-func CudnnRNNParamsSizeDirection(value string) CudnnRNNParamsSizeAttr {
-	return func(m optionalAttr) {
-		m["direction"] = value
+// Computes the minimum along segments of a tensor.
+//
+// Read
+// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+// for an explanation of segments.
+//
+// Computes a tensor such that
+// \\(output_i = \min_j(data_j)\\) where `min` is over `j` such
+// that `segment_ids[j] == i`.
+//
+// If the min is empty for a given segment ID `i`, `output[i] = 0`.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/SegmentMin.png" alt>
+// </div>
+//
+// Arguments:
+//
+//	segment_ids: A 1-D tensor whose size is equal to the size of `data`'s
+// first dimension.  Values should be sorted and can be repeated.
+//
+// Returns Has same shape as data, except for dimension 0 which
+// has size `k`, the number of segments.
+func SegmentMin(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// CudnnRNNParamsSizeDropout sets the optional dropout attribute to value.
-// If not specified, defaults to 0
-func CudnnRNNParamsSizeDropout(value float32) CudnnRNNParamsSizeAttr {
-	return func(m optionalAttr) {
-		m["dropout"] = value
+	opspec := tf.OpSpec{
+		Type: "SegmentMin",
+		Input: []tf.Input{
+			data, segment_ids,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// CudnnRNNParamsSizeSeed sets the optional seed attribute to value.
-// If not specified, defaults to 0
-func CudnnRNNParamsSizeSeed(value int64) CudnnRNNParamsSizeAttr {
-	return func(m optionalAttr) {
-		m["seed"] = value
-	}
-}
-
-// CudnnRNNParamsSizeSeed2 sets the optional seed2 attribute to value.
-// If not specified, defaults to 0
-func CudnnRNNParamsSizeSeed2(value int64) CudnnRNNParamsSizeAttr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
-	}
-}
-
-// Computes size of weights that can be used by a Cudnn RNN model.
-//
-// Return the params size that can be used by the Cudnn RNN model. Subsequent
-// weight allocation and initialization should use this size.
-//
-// num_layers: Specifies the number of layers in the RNN model.
-// num_units: Specifies the size of the hidden state.
-// input_size: Specifies the size of the input state.
-// rnn_mode: Indicates the type of the RNN model.
-// input_mode: Indicate whether there is a linear projection between the input and
-//   The actual computation before the first layer. 'skip_input' is only allowed
-//   when input_size == num_units; 'auto_select' implies 'skip_input' when
-//   input_size == num_units; otherwise, it implies 'linear_input'.
-// direction: Indicates whether a bidirectional model will be used.
-//   dir = (direction == bidirectional) ? 2 : 1
-// dropout: dropout probability. When set to 0., dropout is disabled.
-// seed: the 1st part of a seed to initialize dropout.
-// seed2: the 2nd part of a seed to initialize dropout.
-// params_size: The size of the params buffer that should be allocated and
-//   initialized for this RNN model. Note that this params buffer may not be
-//   compatible across GPUs. Please use CudnnRNNParamsWeights and
-//   CudnnRNNParamsBiases to save and restore them in a way that is compatible
-//   across different runs.
-func CudnnRNNParamsSize(scope *Scope, num_layers tf.Output, num_units tf.Output, input_size tf.Output, T tf.DataType, S tf.DataType, optional ...CudnnRNNParamsSizeAttr) (params_size tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"T": T, "S": S}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "CudnnRNNParamsSize",
-		Input: []tf.Input{
-			num_layers, num_units, input_size,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes gradients for SparseSegmentMean.
+// Removes keys and its associated values from a table.
 //
-// Returns tensor "output" with same shape as grad, except for dimension 0 whose
-// value is output_dim0.
+// The tensor `keys` must of the same type as the keys of the table. Keys not
+// already in the table are silently ignored.
 //
 // Arguments:
-//	grad: gradient propagated to the SparseSegmentMean op.
-//	indices: indices passed to the corresponding SparseSegmentMean op.
-//	segment_ids: segment_ids passed to the corresponding SparseSegmentMean op.
-//	output_dim0: dimension 0 of "data" passed to SparseSegmentMean op.
-func SparseSegmentMeanGrad(scope *Scope, grad tf.Output, indices tf.Output, segment_ids tf.Output, output_dim0 tf.Output) (output tf.Output) {
+//	table_handle: Handle to the table.
+//	keys: Any shape.  Keys of the elements to remove.
+//
+// Returns the created operation.
+func LookupTableRemoveV2(scope *Scope, table_handle tf.Output, keys tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseSegmentMeanGrad",
+		Type: "LookupTableRemoveV2",
 		Input: []tf.Input{
-			grad, indices, segment_ids, output_dim0,
+			table_handle, keys,
 		},
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// Computes the sum along sparse segments of a tensor divided by the sqrt of N.
-//
-// N is the size of the segment being reduced.
+// Computes the sum along segments of a tensor.
 //
 // Read
 // [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
 // for an explanation of segments.
 //
+// Computes a tensor such that
+// \\(output[i] = \sum_{j...} data[j...]\\) where the sum is over tuples `j...` such
+// that `segment_ids[j...] == i`.  Unlike `SegmentSum`, `segment_ids`
+// need not be sorted and need not cover all values in the full
+// range of valid values.
+//
+// If the sum is empty for a given segment ID `i`, `output[i] = 0`.
+// If the given segment ID `i` is negative, the value is dropped and will not be
+// added to the sum of the segment.
+//
+// `num_segments` should equal the number of distinct segment IDs.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/UnsortedSegmentSum.png" alt>
+// </div>
+//
 // Arguments:
 //
-//	indices: A 1-D tensor. Has same rank as `segment_ids`.
-//	segment_ids: A 1-D tensor. Values should be sorted and can be repeated.
+//	segment_ids: A tensor whose shape is a prefix of `data.shape`.
 //
-// Returns Has same shape as data, except for dimension 0 which
-// has size `k`, the number of segments.
-func SparseSegmentSqrtN(scope *Scope, data tf.Output, indices tf.Output, segment_ids tf.Output) (output tf.Output) {
+//
+// Returns Has same shape as data, except for the first `segment_ids.rank`
+// dimensions, which are replaced with a single dimension which has size
+// `num_segments`.
+func UnsortedSegmentSum(scope *Scope, data tf.Output, segment_ids tf.Output, num_segments tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseSegmentSqrtN",
+		Type: "UnsortedSegmentSum",
 		Input: []tf.Input{
-			data, indices, segment_ids,
+			data, segment_ids, num_segments,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Compute the upper regularized incomplete Gamma function `Q(a, x)`.
+// Computes the product along segments of a tensor.
 //
-// The upper regularized incomplete Gamma function is defined as:
+// Read
+// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#segmentation)
+// for an explanation of segments.
 //
-// \\(Q(a, x) = Gamma(a, x) / Gamma(a) = 1 - P(a, x)\\)
+// This operator is similar to the unsorted segment sum operator found
+// [(here)](../../../api_docs/python/math_ops.md#UnsortedSegmentSum).
+// Instead of computing the sum over segments, it computes the product of all
+// entries belonging to a segment such that:
 //
-// where
+// \\(output_i = \prod_{j...} data[j...]\\) where the product is over tuples
+// `j...` such that `segment_ids[j...] == i`.
 //
-// \\(Gamma(a, x) = int_{x}^{\infty} t^{a-1} exp(-t) dt\\)
+// If there is no entry for a given segment ID `i`, it outputs 1.
 //
-// is the upper incomplete Gama function.
+// If the given segment ID `i` is negative, then the corresponding value is
+// dropped, and will not be included in the result.
 //
-// Note, above `P(a, x)` (`Igamma`) is the lower regularized complete
-// Gamma function.
-func Igammac(scope *Scope, a tf.Output, x tf.Output) (z tf.Output) {
+// Arguments:
+//
+//	segment_ids: A tensor whose shape is a prefix of `data.shape`.
+//
+//
+// Returns Has same shape as data, except for the first `segment_ids.rank`
+// dimensions, which are replaced with a single dimension which has size
+// `num_segments`.
+func UnsortedSegmentProd(scope *Scope, data tf.Output, segment_ids tf.Output, num_segments tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Igammac",
+		Type: "UnsortedSegmentProd",
 		Input: []tf.Input{
-			a, x,
+			data, segment_ids, num_segments,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes the sum along sparse segments of a tensor divided by the sqrt of N.
-//
-// N is the size of the segment being reduced.
-//
-// Like `SparseSegmentSqrtN`, but allows missing ids in `segment_ids`. If an id is
-// misisng, the `output` tensor at that position will be zeroed.
+// Computes the mean along sparse segments of a tensor.
 //
 // Read
 // [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
 // for an explanation of segments.
 //
+// Like `SegmentMean`, but `segment_ids` can have rank less than `data`'s first
+// dimension, selecting a subset of dimension 0, specified by `indices`.
+//
 // Arguments:
 //
 //	indices: A 1-D tensor. Has same rank as `segment_ids`.
 //	segment_ids: A 1-D tensor. Values should be sorted and can be repeated.
-//	num_segments: Should equal the number of distinct segment IDs.
 //
 // Returns Has same shape as data, except for dimension 0 which
 // has size `k`, the number of segments.
-func SparseSegmentSqrtNWithNumSegments(scope *Scope, data tf.Output, indices tf.Output, segment_ids tf.Output, num_segments tf.Output) (output tf.Output) {
+func SparseSegmentMean(scope *Scope, data tf.Output, indices tf.Output, segment_ids tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseSegmentSqrtNWithNumSegments",
+		Type: "SparseSegmentMean",
 		Input: []tf.Input{
-			data, indices, segment_ids, num_segments,
+			data, indices, segment_ids,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes gradients for SparseSegmentSqrtN.
+// Deserializes a serialized tree ensemble config and replaces current tree
 //
-// Returns tensor "output" with same shape as grad, except for dimension 0 whose
-// value is output_dim0.
+// ensemble.
 //
 // Arguments:
-//	grad: gradient propagated to the SparseSegmentSqrtN op.
-//	indices: indices passed to the corresponding SparseSegmentSqrtN op.
-//	segment_ids: segment_ids passed to the corresponding SparseSegmentSqrtN op.
-//	output_dim0: dimension 0 of "data" passed to SparseSegmentSqrtN op.
-func SparseSegmentSqrtNGrad(scope *Scope, grad tf.Output, indices tf.Output, segment_ids tf.Output, output_dim0 tf.Output) (output tf.Output) {
+//	tree_ensemble_handle: Handle to the tree ensemble.
+//	stamp_token: Token to use as the new value of the resource stamp.
+//	tree_ensemble_serialized: Serialized proto of the ensemble.
+//
+// Returns the created operation.
+func BoostedTreesDeserializeEnsemble(scope *Scope, tree_ensemble_handle tf.Output, stamp_token tf.Output, tree_ensemble_serialized tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseSegmentSqrtNGrad",
+		Type: "BoostedTreesDeserializeEnsemble",
 		Input: []tf.Input{
-			grad, indices, segment_ids, output_dim0,
+			tree_ensemble_handle, stamp_token, tree_ensemble_serialized,
 		},
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// LRNGradAttr is an optional argument to LRNGrad.
-type LRNGradAttr func(optionalAttr)
-
-// LRNGradDepthRadius sets the optional depth_radius attribute to value.
+// Transforms a tf.Example proto (as a string) into typed tensors.
 //
-// value: A depth radius.
-// If not specified, defaults to 5
-func LRNGradDepthRadius(value int64) LRNGradAttr {
-	return func(m optionalAttr) {
-		m["depth_radius"] = value
+// Arguments:
+//	serialized: A vector containing a batch of binary serialized Example protos.
+//	dense_defaults: A list of Tensors (some may be empty), whose length matches
+// the length of `dense_keys`. dense_defaults[j] provides default values
+// when the example's feature_map lacks dense_key[j].  If an empty Tensor is
+// provided for dense_defaults[j], then the Feature dense_keys[j] is required.
+// The input type is inferred from dense_defaults[j], even when it's empty.
+// If dense_defaults[j] is not empty, and dense_shapes[j] is fully defined,
+// then the shape of dense_defaults[j] must match that of dense_shapes[j].
+// If dense_shapes[j] has an undefined major dimension (variable strides dense
+// feature), dense_defaults[j] must contain a single element:
+// the padding element.
+//	num_sparse: The number of sparse features to be parsed from the example. This
+// must match the lengths of `sparse_keys` and `sparse_types`.
+//	sparse_keys: A list of `num_sparse` strings.
+// The keys expected in the Examples' features associated with sparse values.
+//	dense_keys: The keys expected in the Examples' features associated with dense
+// values.
+//	sparse_types: A list of `num_sparse` types; the data types of data in each
+// Feature given in sparse_keys.
+// Currently the ParseSingleExample op supports DT_FLOAT (FloatList),
+// DT_INT64 (Int64List), and DT_STRING (BytesList).
+//	dense_shapes: The shapes of data in each Feature given in dense_keys.
+// The length of this list must match the length of `dense_keys`.  The
+// number of elements in the Feature corresponding to dense_key[j] must
+// always equal dense_shapes[j].NumEntries().  If dense_shapes[j] ==
+// (D0, D1, ..., DN) then the shape of output Tensor dense_values[j]
+// will be (D0, D1, ..., DN): In the case dense_shapes[j] = (-1, D1,
+// ..., DN), the shape of the output Tensor dense_values[j] will be (M,
+// D1, .., DN), where M is the number of blocks of elements of length
+// D1 * .... * DN, in the input.
+func ParseSingleExample(scope *Scope, serialized tf.Output, dense_defaults []tf.Output, num_sparse int64, sparse_keys []string, dense_keys []string, sparse_types []tf.DataType, dense_shapes []tf.Shape) (sparse_indices []tf.Output, sparse_values []tf.Output, sparse_shapes []tf.Output, dense_values []tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// LRNGradBias sets the optional bias attribute to value.
-//
-// value: An offset (usually > 0 to avoid dividing by 0).
-// If not specified, defaults to 1
-func LRNGradBias(value float32) LRNGradAttr {
-	return func(m optionalAttr) {
-		m["bias"] = value
+	attrs := map[string]interface{}{"num_sparse": num_sparse, "sparse_keys": sparse_keys, "dense_keys": dense_keys, "sparse_types": sparse_types, "dense_shapes": dense_shapes}
+	opspec := tf.OpSpec{
+		Type: "ParseSingleExample",
+		Input: []tf.Input{
+			serialized, tf.OutputList(dense_defaults),
+		},
+		Attrs: attrs,
 	}
-}
-
-// LRNGradAlpha sets the optional alpha attribute to value.
-//
-// value: A scale factor, usually positive.
-// If not specified, defaults to 1
-func LRNGradAlpha(value float32) LRNGradAttr {
-	return func(m optionalAttr) {
-		m["alpha"] = value
+	op := scope.AddOperation(opspec)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if sparse_indices, idx, err = makeOutputList(op, idx, "sparse_indices"); err != nil {
+		scope.UpdateErr("ParseSingleExample", err)
+		return
+	}
+	if sparse_values, idx, err = makeOutputList(op, idx, "sparse_values"); err != nil {
+		scope.UpdateErr("ParseSingleExample", err)
+		return
+	}
+	if sparse_shapes, idx, err = makeOutputList(op, idx, "sparse_shapes"); err != nil {
+		scope.UpdateErr("ParseSingleExample", err)
+		return
+	}
+	if dense_values, idx, err = makeOutputList(op, idx, "dense_values"); err != nil {
+		scope.UpdateErr("ParseSingleExample", err)
+		return
 	}
+	return sparse_indices, sparse_values, sparse_shapes, dense_values
 }
 
-// LRNGradBeta sets the optional beta attribute to value.
+// WholeFileReaderV2Attr is an optional argument to WholeFileReaderV2.
+type WholeFileReaderV2Attr func(optionalAttr)
+
+// WholeFileReaderV2Container sets the optional container attribute to value.
 //
-// value: An exponent.
-// If not specified, defaults to 0.5
-func LRNGradBeta(value float32) LRNGradAttr {
+// value: If non-empty, this reader is placed in the given container.
+// Otherwise, a default container is used.
+// If not specified, defaults to ""
+func WholeFileReaderV2Container(value string) WholeFileReaderV2Attr {
 	return func(m optionalAttr) {
-		m["beta"] = value
+		m["container"] = value
 	}
 }
 
-// Gradients for Local Response Normalization.
+// WholeFileReaderV2SharedName sets the optional shared_name attribute to value.
 //
-// Arguments:
-//	input_grads: 4-D with shape `[batch, height, width, channels]`.
-//	input_image: 4-D with shape `[batch, height, width, channels]`.
-//	output_image: 4-D with shape `[batch, height, width, channels]`.
+// value: If non-empty, this reader is named in the given bucket
+// with this shared_name. Otherwise, the node name is used instead.
+// If not specified, defaults to ""
+func WholeFileReaderV2SharedName(value string) WholeFileReaderV2Attr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// A Reader that outputs the entire contents of a file as a value.
 //
-// Returns The gradients for LRN.
-func LRNGrad(scope *Scope, input_grads tf.Output, input_image tf.Output, output_image tf.Output, optional ...LRNGradAttr) (output tf.Output) {
+// To use, enqueue filenames in a Queue.  The output of ReaderRead will
+// be a filename (key) and the contents of that file (value).
+//
+// Returns The handle to reference the Reader.
+func WholeFileReaderV2(scope *Scope, optional ...WholeFileReaderV2Attr) (reader_handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -26376,54 +26187,30 @@ func LRNGrad(scope *Scope, input_grads tf.Output, input_image tf.Output, output_
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "LRNGrad",
-		Input: []tf.Input{
-			input_grads, input_image, output_image,
-		},
+		Type: "WholeFileReaderV2",
+
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// AnyAttr is an optional argument to Any.
-type AnyAttr func(optionalAttr)
-
-// AnyKeepDims sets the optional keep_dims attribute to value.
-//
-// value: If true, retain reduced dimensions with length 1.
-// If not specified, defaults to false
-func AnyKeepDims(value bool) AnyAttr {
-	return func(m optionalAttr) {
-		m["keep_dims"] = value
-	}
-}
-
-// Computes the "logical or" of elements across dimensions of a tensor.
-//
-// Reduces `input` along the dimensions given in `axis`. Unless
-// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
-// `axis`. If `keep_dims` is true, the reduced dimensions are
-// retained with length 1.
+// Pop the element at the top of the stack.
 //
 // Arguments:
-//	input: The tensor to reduce.
-//	axis: The dimensions to reduce. Must be in the range
-// `[-rank(input), rank(input))`.
+//	handle: The handle to a stack.
+//	elem_type: The type of the elem that is popped.
 //
-// Returns The reduced tensor.
-func Any(scope *Scope, input tf.Output, axis tf.Output, optional ...AnyAttr) (output tf.Output) {
+// Returns The tensor that is popped from the top of the stack.
+func StackPopV2(scope *Scope, handle tf.Output, elem_type tf.DataType) (elem tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"elem_type": elem_type}
 	opspec := tf.OpSpec{
-		Type: "Any",
+		Type: "StackPopV2",
 		Input: []tf.Input{
-			input, axis,
+			handle,
 		},
 		Attrs: attrs,
 	}
@@ -26431,161 +26218,138 @@ func Any(scope *Scope, input tf.Output, axis tf.Output, optional ...AnyAttr) (ou
 	return op.Output(0)
 }
 
-// DestroyResourceOpAttr is an optional argument to DestroyResourceOp.
-type DestroyResourceOpAttr func(optionalAttr)
-
-// DestroyResourceOpIgnoreLookupError sets the optional ignore_lookup_error attribute to value.
-//
-// value: whether to ignore the error when the resource
-// doesn't exist.
-// If not specified, defaults to true
-func DestroyResourceOpIgnoreLookupError(value bool) DestroyResourceOpAttr {
-	return func(m optionalAttr) {
-		m["ignore_lookup_error"] = value
-	}
-}
-
-// Deletes the resource specified by the handle.
-//
-// All subsequent operations using the resource will result in a NotFound
-// error status.
-//
-// Arguments:
-//	resource: handle to the resource to delete.
-//
-// Returns the created operation.
-func DestroyResourceOp(scope *Scope, resource tf.Output, optional ...DestroyResourceOpAttr) (o *tf.Operation) {
+// Computes hyperbolic cosine of x element-wise.
+func Cosh(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "DestroyResourceOp",
+		Type: "Cosh",
 		Input: []tf.Input{
-			resource,
+			x,
 		},
-		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Generates values in an interval.
-//
-// A sequence of `num` evenly-spaced values are generated beginning at `start`.
-// If `num > 1`, the values in the sequence increase by `stop - start / num - 1`,
-// so that the last one is exactly `stop`.
+// Computes the mean along sparse segments of a tensor.
 //
-// For example:
+// Like `SparseSegmentMean`, but allows missing ids in `segment_ids`. If an id is
+// misisng, the `output` tensor at that position will be zeroed.
 //
-// ```
-// tf.linspace(10.0, 12.0, 3, name="linspace") => [ 10.0  11.0  12.0]
-// ```
+// Read
+// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+// for an explanation of segments.
 //
 // Arguments:
-//	start: 0-D tensor. First entry in the range.
-//	stop: 0-D tensor. Last entry in the range.
-//	num: 0-D tensor. Number of values to generate.
 //
-// Returns 1-D. The generated values.
-func LinSpace(scope *Scope, start tf.Output, stop tf.Output, num tf.Output) (output tf.Output) {
+//	indices: A 1-D tensor. Has same rank as `segment_ids`.
+//	segment_ids: A 1-D tensor. Values should be sorted and can be repeated.
+//	num_segments: Should equal the number of distinct segment IDs.
+//
+// Returns Has same shape as data, except for dimension 0 which has size
+// `num_segments`.
+func SparseSegmentMeanWithNumSegments(scope *Scope, data tf.Output, indices tf.Output, segment_ids tf.Output, num_segments tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "LinSpace",
+		Type: "SparseSegmentMeanWithNumSegments",
 		Input: []tf.Input{
-			start, stop, num,
+			data, indices, segment_ids, num_segments,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ComplexAttr is an optional argument to Complex.
-type ComplexAttr func(optionalAttr)
+// CudnnRNNParamsSizeAttr is an optional argument to CudnnRNNParamsSize.
+type CudnnRNNParamsSizeAttr func(optionalAttr)
 
-// ComplexTout sets the optional Tout attribute to value.
-// If not specified, defaults to DT_COMPLEX64
-func ComplexTout(value tf.DataType) ComplexAttr {
+// CudnnRNNParamsSizeRnnMode sets the optional rnn_mode attribute to value.
+// If not specified, defaults to "lstm"
+func CudnnRNNParamsSizeRnnMode(value string) CudnnRNNParamsSizeAttr {
 	return func(m optionalAttr) {
-		m["Tout"] = value
+		m["rnn_mode"] = value
 	}
 }
 
-// Converts two real numbers to a complex number.
-//
-// Given a tensor `real` representing the real part of a complex number, and a
-// tensor `imag` representing the imaginary part of a complex number, this
-// operation returns complex numbers elementwise of the form \\(a + bj\\), where
-// *a* represents the `real` part and *b* represents the `imag` part.
-//
-// The input tensors `real` and `imag` must have the same shape.
-//
-// For example:
-//
-// ```
-// # tensor 'real' is [2.25, 3.25]
-// # tensor `imag` is [4.75, 5.75]
-// tf.complex(real, imag) ==> [[2.25 + 4.75j], [3.25 + 5.75j]]
-// ```
-func Complex(scope *Scope, real tf.Output, imag tf.Output, optional ...ComplexAttr) (out tf.Output) {
-	if scope.Err() != nil {
-		return
+// CudnnRNNParamsSizeInputMode sets the optional input_mode attribute to value.
+// If not specified, defaults to "linear_input"
+func CudnnRNNParamsSizeInputMode(value string) CudnnRNNParamsSizeAttr {
+	return func(m optionalAttr) {
+		m["input_mode"] = value
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
+}
+
+// CudnnRNNParamsSizeDirection sets the optional direction attribute to value.
+// If not specified, defaults to "unidirectional"
+func CudnnRNNParamsSizeDirection(value string) CudnnRNNParamsSizeAttr {
+	return func(m optionalAttr) {
+		m["direction"] = value
 	}
-	opspec := tf.OpSpec{
-		Type: "Complex",
-		Input: []tf.Input{
-			real, imag,
-		},
-		Attrs: attrs,
+}
+
+// CudnnRNNParamsSizeDropout sets the optional dropout attribute to value.
+// If not specified, defaults to 0
+func CudnnRNNParamsSizeDropout(value float32) CudnnRNNParamsSizeAttr {
+	return func(m optionalAttr) {
+		m["dropout"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// ImagAttr is an optional argument to Imag.
-type ImagAttr func(optionalAttr)
+// CudnnRNNParamsSizeSeed sets the optional seed attribute to value.
+// If not specified, defaults to 0
+func CudnnRNNParamsSizeSeed(value int64) CudnnRNNParamsSizeAttr {
+	return func(m optionalAttr) {
+		m["seed"] = value
+	}
+}
 
-// ImagTout sets the optional Tout attribute to value.
-// If not specified, defaults to DT_FLOAT
-func ImagTout(value tf.DataType) ImagAttr {
+// CudnnRNNParamsSizeSeed2 sets the optional seed2 attribute to value.
+// If not specified, defaults to 0
+func CudnnRNNParamsSizeSeed2(value int64) CudnnRNNParamsSizeAttr {
 	return func(m optionalAttr) {
-		m["Tout"] = value
+		m["seed2"] = value
 	}
 }
 
-// Returns the imaginary part of a complex number.
+// Computes size of weights that can be used by a Cudnn RNN model.
 //
-// Given a tensor `input` of complex numbers, this operation returns a tensor of
-// type `float` that is the imaginary part of each element in `input`. All
-// elements in `input` must be complex numbers of the form \\(a + bj\\), where *a*
-// is the real part and *b* is the imaginary part returned by this operation.
-//
-// For example:
+// Return the params size that can be used by the Cudnn RNN model. Subsequent
+// weight allocation and initialization should use this size.
 //
-// ```
-// # tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j]
-// tf.imag(input) ==> [4.75, 5.75]
-// ```
-func Imag(scope *Scope, input tf.Output, optional ...ImagAttr) (output tf.Output) {
+// num_layers: Specifies the number of layers in the RNN model.
+// num_units: Specifies the size of the hidden state.
+// input_size: Specifies the size of the input state.
+// rnn_mode: Indicates the type of the RNN model.
+// input_mode: Indicate whether there is a linear projection between the input and
+//   The actual computation before the first layer. 'skip_input' is only allowed
+//   when input_size == num_units; 'auto_select' implies 'skip_input' when
+//   input_size == num_units; otherwise, it implies 'linear_input'.
+// direction: Indicates whether a bidirectional model will be used.
+//   dir = (direction == bidirectional) ? 2 : 1
+// dropout: dropout probability. When set to 0., dropout is disabled.
+// seed: the 1st part of a seed to initialize dropout.
+// seed2: the 2nd part of a seed to initialize dropout.
+// params_size: The size of the params buffer that should be allocated and
+//   initialized for this RNN model. Note that this params buffer may not be
+//   compatible across GPUs. Please use CudnnRNNParamsWeights and
+//   CudnnRNNParamsBiases to save and restore them in a way that is compatible
+//   across different runs.
+func CudnnRNNParamsSize(scope *Scope, num_layers tf.Output, num_units tf.Output, input_size tf.Output, T tf.DataType, S tf.DataType, optional ...CudnnRNNParamsSizeAttr) (params_size tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"T": T, "S": S}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Imag",
+		Type: "CudnnRNNParamsSize",
 		Input: []tf.Input{
-			input,
+			num_layers, num_units, input_size,
 		},
 		Attrs: attrs,
 	}
@@ -26593,188 +26357,196 @@ func Imag(scope *Scope, input tf.Output, optional ...ImagAttr) (output tf.Output
 	return op.Output(0)
 }
 
-// Computes hyperbolic tangent of `x` element-wise.
-func Tanh(scope *Scope, x tf.Output) (y tf.Output) {
+// Computes gradients for SparseSegmentMean.
+//
+// Returns tensor "output" with same shape as grad, except for dimension 0 whose
+// value is output_dim0.
+//
+// Arguments:
+//	grad: gradient propagated to the SparseSegmentMean op.
+//	indices: indices passed to the corresponding SparseSegmentMean op.
+//	segment_ids: segment_ids passed to the corresponding SparseSegmentMean op.
+//	output_dim0: dimension 0 of "data" passed to SparseSegmentMean op.
+func SparseSegmentMeanGrad(scope *Scope, grad tf.Output, indices tf.Output, segment_ids tf.Output, output_dim0 tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Tanh",
+		Type: "SparseSegmentMeanGrad",
 		Input: []tf.Input{
-			x,
+			grad, indices, segment_ids, output_dim0,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes the maximum along segments of a tensor.
+// Computes the sum along sparse segments of a tensor divided by the sqrt of N.
+//
+// N is the size of the segment being reduced.
 //
 // Read
 // [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
 // for an explanation of segments.
 //
-// Computes a tensor such that
-// \\(output_i = \max_j(data_j)\\) where `max` is over `j` such
-// that `segment_ids[j] == i`.
-//
-// If the max is empty for a given segment ID `i`, `output[i] = 0`.
-//
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/SegmentMax.png" alt>
-// </div>
-//
 // Arguments:
 //
-//	segment_ids: A 1-D tensor whose size is equal to the size of `data`'s
-// first dimension.  Values should be sorted and can be repeated.
+//	indices: A 1-D tensor. Has same rank as `segment_ids`.
+//	segment_ids: A 1-D tensor. Values should be sorted and can be repeated.
 //
 // Returns Has same shape as data, except for dimension 0 which
 // has size `k`, the number of segments.
-func SegmentMax(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) {
+func SparseSegmentSqrtN(scope *Scope, data tf.Output, indices tf.Output, segment_ids tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SegmentMax",
+		Type: "SparseSegmentSqrtN",
 		Input: []tf.Input{
-			data, segment_ids,
+			data, indices, segment_ids,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Creates a dataset that skips `count` elements from the `input_dataset`.
+// Compute the upper regularized incomplete Gamma function `Q(a, x)`.
 //
-// Arguments:
+// The upper regularized incomplete Gamma function is defined as:
 //
-//	count: A scalar representing the number of elements from the `input_dataset`
-// that should be skipped.  If count is -1, skips everything.
+// \\(Q(a, x) = Gamma(a, x) / Gamma(a) = 1 - P(a, x)\\)
 //
+// where
 //
-func SkipDataset(scope *Scope, input_dataset tf.Output, count tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+// \\(Gamma(a, x) = int_{x}^{\infty} t^{a-1} exp(-t) dt\\)
+//
+// is the upper incomplete Gama function.
+//
+// Note, above `P(a, x)` (`Igamma`) is the lower regularized complete
+// Gamma function.
+func Igammac(scope *Scope, a tf.Output, x tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "SkipDataset",
+		Type: "Igammac",
 		Input: []tf.Input{
-			input_dataset, count,
+			a, x,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// RealAttr is an optional argument to Real.
-type RealAttr func(optionalAttr)
-
-// RealTout sets the optional Tout attribute to value.
-// If not specified, defaults to DT_FLOAT
-func RealTout(value tf.DataType) RealAttr {
-	return func(m optionalAttr) {
-		m["Tout"] = value
-	}
-}
-
-// Returns the real part of a complex number.
+// Computes the sum along sparse segments of a tensor divided by the sqrt of N.
 //
-// Given a tensor `input` of complex numbers, this operation returns a tensor of
-// type `float` that is the real part of each element in `input`. All elements in
-// `input` must be complex numbers of the form \\(a + bj\\), where *a* is the real
-//  part returned by this operation and *b* is the imaginary part.
+// N is the size of the segment being reduced.
 //
-// For example:
+// Like `SparseSegmentSqrtN`, but allows missing ids in `segment_ids`. If an id is
+// misisng, the `output` tensor at that position will be zeroed.
 //
-// ```
-// # tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j]
-// tf.real(input) ==> [-2.25, 3.25]
-// ```
-func Real(scope *Scope, input tf.Output, optional ...RealAttr) (output tf.Output) {
+// Read
+// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+// for an explanation of segments.
+//
+// Arguments:
+//
+//	indices: A 1-D tensor. Has same rank as `segment_ids`.
+//	segment_ids: A 1-D tensor. Values should be sorted and can be repeated.
+//	num_segments: Should equal the number of distinct segment IDs.
+//
+// Returns Has same shape as data, except for dimension 0 which
+// has size `k`, the number of segments.
+func SparseSegmentSqrtNWithNumSegments(scope *Scope, data tf.Output, indices tf.Output, segment_ids tf.Output, num_segments tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "Real",
+		Type: "SparseSegmentSqrtNWithNumSegments",
 		Input: []tf.Input{
-			input,
+			data, indices, segment_ids, num_segments,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Sends `input` to all devices that are connected to the output.
-//
-// Sends `input` to all devices that are connected to the output.
-//
-// The graph should be constructed so that all ops connected to the output have a
-// valid device assignment, and the op itself is assigned one of these devices.
+// Computes gradients for SparseSegmentSqrtN.
 //
-// input: The input to the broadcast.
-// output: The same as input.
-// shape: The shape of the input tensor.
+// Returns tensor "output" with same shape as grad, except for dimension 0 whose
+// value is output_dim0.
 //
-func NcclBroadcast(scope *Scope, input tf.Output, shape tf.Shape) (output tf.Output) {
+// Arguments:
+//	grad: gradient propagated to the SparseSegmentSqrtN op.
+//	indices: indices passed to the corresponding SparseSegmentSqrtN op.
+//	segment_ids: segment_ids passed to the corresponding SparseSegmentSqrtN op.
+//	output_dim0: dimension 0 of "data" passed to SparseSegmentSqrtN op.
+func SparseSegmentSqrtNGrad(scope *Scope, grad tf.Output, indices tf.Output, segment_ids tf.Output, output_dim0 tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"shape": shape}
 	opspec := tf.OpSpec{
-		Type: "NcclBroadcast",
+		Type: "SparseSegmentSqrtNGrad",
 		Input: []tf.Input{
-			input,
+			grad, indices, segment_ids, output_dim0,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ResizeAreaAttr is an optional argument to ResizeArea.
-type ResizeAreaAttr func(optionalAttr)
+// LRNGradAttr is an optional argument to LRNGrad.
+type LRNGradAttr func(optionalAttr)
 
-// ResizeAreaAlignCorners sets the optional align_corners attribute to value.
+// LRNGradDepthRadius sets the optional depth_radius attribute to value.
 //
-// value: If true, the centers of the 4 corner pixels of the input and output tensors are
-// aligned, preserving the values at the corner pixels. Defaults to false.
-// If not specified, defaults to false
-func ResizeAreaAlignCorners(value bool) ResizeAreaAttr {
+// value: A depth radius.
+// If not specified, defaults to 5
+func LRNGradDepthRadius(value int64) LRNGradAttr {
 	return func(m optionalAttr) {
-		m["align_corners"] = value
+		m["depth_radius"] = value
 	}
 }
 
-// Resize `images` to `size` using area interpolation.
+// LRNGradBias sets the optional bias attribute to value.
 //
-// Input images can be of different types but output images are always float.
+// value: An offset (usually > 0 to avoid dividing by 0).
+// If not specified, defaults to 1
+func LRNGradBias(value float32) LRNGradAttr {
+	return func(m optionalAttr) {
+		m["bias"] = value
+	}
+}
+
+// LRNGradAlpha sets the optional alpha attribute to value.
 //
-// The range of pixel values for the output image might be slightly different
-// from the range for the input image because of limited numerical precision.
-// To guarantee an output range, for example `[0.0, 1.0]`, apply
-// `tf.clip_by_value` to the output.
+// value: A scale factor, usually positive.
+// If not specified, defaults to 1
+func LRNGradAlpha(value float32) LRNGradAttr {
+	return func(m optionalAttr) {
+		m["alpha"] = value
+	}
+}
+
+// LRNGradBeta sets the optional beta attribute to value.
 //
-// Each output pixel is computed by first transforming the pixel's footprint into
-// the input tensor and then averaging the pixels that intersect the footprint. An
-// input pixel's contribution to the average is weighted by the fraction of its
-// area that intersects the footprint.  This is the same as OpenCV's INTER_AREA.
+// value: An exponent.
+// If not specified, defaults to 0.5
+func LRNGradBeta(value float32) LRNGradAttr {
+	return func(m optionalAttr) {
+		m["beta"] = value
+	}
+}
+
+// Gradients for Local Response Normalization.
 //
 // Arguments:
-//	images: 4-D with shape `[batch, height, width, channels]`.
-//	size: = A 1-D int32 Tensor of 2 elements: `new_height, new_width`.  The
-// new size for the images.
+//	input_grads: 4-D with shape `[batch, height, width, channels]`.
+//	input_image: 4-D with shape `[batch, height, width, channels]`.
+//	output_image: 4-D with shape `[batch, height, width, channels]`.
 //
-// Returns 4-D with shape
-// `[batch, new_height, new_width, channels]`.
-func ResizeArea(scope *Scope, images tf.Output, size tf.Output, optional ...ResizeAreaAttr) (resized_images tf.Output) {
+// Returns The gradients for LRN.
+func LRNGrad(scope *Scope, input_grads tf.Output, input_image tf.Output, output_image tf.Output, optional ...LRNGradAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -26783,9 +26555,9 @@ func ResizeArea(scope *Scope, images tf.Output, size tf.Output, optional ...Resi
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResizeArea",
+		Type: "LRNGrad",
 		Input: []tf.Input{
-			images, size,
+			input_grads, input_image, output_image,
 		},
 		Attrs: attrs,
 	}
@@ -26793,83 +26565,75 @@ func ResizeArea(scope *Scope, images tf.Output, size tf.Output, optional ...Resi
 	return op.Output(0)
 }
 
-// VarHandleOpAttr is an optional argument to VarHandleOp.
-type VarHandleOpAttr func(optionalAttr)
+// AnyAttr is an optional argument to Any.
+type AnyAttr func(optionalAttr)
 
-// VarHandleOpContainer sets the optional container attribute to value.
+// AnyKeepDims sets the optional keep_dims attribute to value.
 //
-// value: the container this variable is placed in.
-// If not specified, defaults to ""
-func VarHandleOpContainer(value string) VarHandleOpAttr {
+// value: If true, retain reduced dimensions with length 1.
+// If not specified, defaults to false
+func AnyKeepDims(value bool) AnyAttr {
 	return func(m optionalAttr) {
-		m["container"] = value
+		m["keep_dims"] = value
 	}
 }
 
-// VarHandleOpSharedName sets the optional shared_name attribute to value.
+// Computes the "logical or" of elements across dimensions of a tensor.
 //
-// value: the name by which this variable is referred to.
-// If not specified, defaults to ""
-func VarHandleOpSharedName(value string) VarHandleOpAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Creates a handle to a Variable resource.
+// Reduces `input` along the dimensions given in `axis`. Unless
+// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+// `axis`. If `keep_dims` is true, the reduced dimensions are
+// retained with length 1.
 //
 // Arguments:
-//	dtype: the type of this variable. Must agree with the dtypes
-// of all ops using this variable.
-//	shape: The (possibly partially specified) shape of this variable.
-func VarHandleOp(scope *Scope, dtype tf.DataType, shape tf.Shape, optional ...VarHandleOpAttr) (resource tf.Output) {
+//	input: The tensor to reduce.
+//	axis: The dimensions to reduce. Must be in the range
+// `[-rank(input), rank(input))`.
+//
+// Returns The reduced tensor.
+func Any(scope *Scope, input tf.Output, axis tf.Output, optional ...AnyAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype, "shape": shape}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "VarHandleOp",
-
+		Type: "Any",
+		Input: []tf.Input{
+			input, axis,
+		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// AngleAttr is an optional argument to Angle.
-type AngleAttr func(optionalAttr)
+// DestroyResourceOpAttr is an optional argument to DestroyResourceOp.
+type DestroyResourceOpAttr func(optionalAttr)
 
-// AngleTout sets the optional Tout attribute to value.
-// If not specified, defaults to DT_FLOAT
-func AngleTout(value tf.DataType) AngleAttr {
+// DestroyResourceOpIgnoreLookupError sets the optional ignore_lookup_error attribute to value.
+//
+// value: whether to ignore the error when the resource
+// doesn't exist.
+// If not specified, defaults to true
+func DestroyResourceOpIgnoreLookupError(value bool) DestroyResourceOpAttr {
 	return func(m optionalAttr) {
-		m["Tout"] = value
+		m["ignore_lookup_error"] = value
 	}
 }
 
-// Returns the argument of a complex number.
-//
-// Given a tensor `input` of complex numbers, this operation returns a tensor of
-// type `float` that is the argument of each element in `input`. All elements in
-// `input` must be complex numbers of the form \\(a + bj\\), where *a*
-// is the real part and *b* is the imaginary part.
-//
-// The argument returned by this operation is of the form \\(atan2(b, a)\\).
+// Deletes the resource specified by the handle.
 //
-// For example:
+// All subsequent operations using the resource will result in a NotFound
+// error status.
 //
-// ```
-// # tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j]
-// tf.angle(input) ==> [2.0132, 1.056]
-// ```
+// Arguments:
+//	resource: handle to the resource to delete.
 //
-// @compatibility(numpy)
-// Equivalent to np.angle.
-// @end_compatibility
-func Angle(scope *Scope, input tf.Output, optional ...AngleAttr) (output tf.Output) {
+// Returns the created operation.
+func DestroyResourceOp(scope *Scope, resource tf.Output, optional ...DestroyResourceOpAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -26878,139 +26642,118 @@ func Angle(scope *Scope, input tf.Output, optional ...AngleAttr) (output tf.Outp
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Angle",
+		Type: "DestroyResourceOp",
 		Input: []tf.Input{
-			input,
+			resource,
 		},
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// Clips tensor values to a specified min and max.
+// Generates values in an interval.
 //
-// Given a tensor `t`, this operation returns a tensor of the same type and
-// shape as `t` with its values clipped to `clip_value_min` and `clip_value_max`.
-// Any values less than `clip_value_min` are set to `clip_value_min`. Any values
-// greater than `clip_value_max` are set to `clip_value_max`.
+// A sequence of `num` evenly-spaced values are generated beginning at `start`.
+// If `num > 1`, the values in the sequence increase by `stop - start / num - 1`,
+// so that the last one is exactly `stop`.
+//
+// For example:
+//
+// ```
+// tf.linspace(10.0, 12.0, 3, name="linspace") => [ 10.0  11.0  12.0]
+// ```
 //
 // Arguments:
-//	t: A `Tensor`.
-//	clip_value_min: A 0-D (scalar) `Tensor`, or a `Tensor` with the same shape
-// as `t`. The minimum value to clip by.
-//	clip_value_max: A 0-D (scalar) `Tensor`, or a `Tensor` with the same shape
-// as `t`. The maximum value to clip by.
+//	start: 0-D tensor. First entry in the range.
+//	stop: 0-D tensor. Last entry in the range.
+//	num: 0-D tensor. Number of values to generate.
 //
-// Returns A clipped `Tensor` with the same shape as input 't'.
-func ClipByValue(scope *Scope, t tf.Output, clip_value_min tf.Output, clip_value_max tf.Output) (output tf.Output) {
+// Returns 1-D. The generated values.
+func LinSpace(scope *Scope, start tf.Output, stop tf.Output, num tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "ClipByValue",
+		Type: "LinSpace",
 		Input: []tf.Input{
-			t, clip_value_min, clip_value_max,
+			start, stop, num,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Counts the number of occurrences of each value in an integer array.
+// ComplexAttr is an optional argument to Complex.
+type ComplexAttr func(optionalAttr)
+
+// ComplexTout sets the optional Tout attribute to value.
+// If not specified, defaults to DT_COMPLEX64
+func ComplexTout(value tf.DataType) ComplexAttr {
+	return func(m optionalAttr) {
+		m["Tout"] = value
+	}
+}
+
+// Converts two real numbers to a complex number.
 //
-// Outputs a vector with length `size` and the same dtype as `weights`. If
-// `weights` are empty, then index `i` stores the number of times the value `i` is
-// counted in `arr`. If `weights` are non-empty, then index `i` stores the sum of
-// the value in `weights` at each index where the corresponding value in `arr` is
-// `i`.
+// Given a tensor `real` representing the real part of a complex number, and a
+// tensor `imag` representing the imaginary part of a complex number, this
+// operation returns complex numbers elementwise of the form \\(a + bj\\), where
+// *a* represents the `real` part and *b* represents the `imag` part.
 //
-// Values in `arr` outside of the range [0, size) are ignored.
+// The input tensors `real` and `imag` must have the same shape.
 //
-// Arguments:
-//	arr: int32 `Tensor`.
-//	size: non-negative int32 scalar `Tensor`.
-//	weights: is an int32, int64, float32, or float64 `Tensor` with the same
-// shape as `arr`, or a length-0 `Tensor`, in which case it acts as all weights
-// equal to 1.
+// For example:
 //
-// Returns 1D `Tensor` with length equal to `size`. The counts or summed weights for
-// each value in the range [0, size).
-func Bincount(scope *Scope, arr tf.Output, size tf.Output, weights tf.Output) (bins tf.Output) {
+// ```
+// # tensor 'real' is [2.25, 3.25]
+// # tensor `imag` is [4.75, 5.75]
+// tf.complex(real, imag) ==> [[2.25 + 4.75j], [3.25 + 5.75j]]
+// ```
+func Complex(scope *Scope, real tf.Output, imag tf.Output, optional ...ComplexAttr) (out tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "Bincount",
+		Type: "Complex",
 		Input: []tf.Input{
-			arr, size, weights,
+			real, imag,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// CumsumAttr is an optional argument to Cumsum.
-type CumsumAttr func(optionalAttr)
-
-// CumsumExclusive sets the optional exclusive attribute to value.
-//
-// value: If `True`, perform exclusive cumsum.
-// If not specified, defaults to false
-func CumsumExclusive(value bool) CumsumAttr {
-	return func(m optionalAttr) {
-		m["exclusive"] = value
-	}
-}
+// ImagAttr is an optional argument to Imag.
+type ImagAttr func(optionalAttr)
 
-// CumsumReverse sets the optional reverse attribute to value.
-//
-// value: A `bool` (default: False).
-// If not specified, defaults to false
-func CumsumReverse(value bool) CumsumAttr {
+// ImagTout sets the optional Tout attribute to value.
+// If not specified, defaults to DT_FLOAT
+func ImagTout(value tf.DataType) ImagAttr {
 	return func(m optionalAttr) {
-		m["reverse"] = value
+		m["Tout"] = value
 	}
 }
 
-// Compute the cumulative sum of the tensor `x` along `axis`.
-//
-// By default, this op performs an inclusive cumsum, which means that the first
-// element of the input is identical to the first element of the output:
-//
-// ```python
-// tf.cumsum([a, b, c])  # => [a, a + b, a + b + c]
-// ```
-//
-// By setting the `exclusive` kwarg to `True`, an exclusive cumsum is
-// performed instead:
+// Returns the imaginary part of a complex number.
 //
-// ```python
-// tf.cumsum([a, b, c], exclusive=True)  # => [0, a, a + b]
-// ```
+// Given a tensor `input` of complex numbers, this operation returns a tensor of
+// type `float` that is the imaginary part of each element in `input`. All
+// elements in `input` must be complex numbers of the form \\(a + bj\\), where *a*
+// is the real part and *b* is the imaginary part returned by this operation.
 //
-// By setting the `reverse` kwarg to `True`, the cumsum is performed in the
-// opposite direction:
+// For example:
 //
-// ```python
-// tf.cumsum([a, b, c], reverse=True)  # => [a + b + c, b + c, c]
 // ```
-//
-// This is more efficient than using separate `tf.reverse` ops.
-//
-// The `reverse` and `exclusive` kwargs can also be combined:
-//
-// ```python
-// tf.cumsum([a, b, c], exclusive=True, reverse=True)  # => [b + c, c, 0]
+// # tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j]
+// tf.imag(input) ==> [4.75, 5.75]
 // ```
-//
-// Arguments:
-//	x: A `Tensor`. Must be one of the following types: `float32`, `float64`,
-// `int64`, `int32`, `uint8`, `uint16`, `int16`, `int8`, `complex64`,
-// `complex128`, `qint8`, `quint8`, `qint32`, `half`.
-//	axis: A `Tensor` of type `int32` (default: 0). Must be in the range
-// `[-rank(x), rank(x))`.
-func Cumsum(scope *Scope, x tf.Output, axis tf.Output, optional ...CumsumAttr) (out tf.Output) {
+func Imag(scope *Scope, input tf.Output, optional ...ImagAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -27019,9 +26762,9 @@ func Cumsum(scope *Scope, x tf.Output, axis tf.Output, optional ...CumsumAttr) (
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Cumsum",
+		Type: "Imag",
 		Input: []tf.Input{
-			x, axis,
+			input,
 		},
 		Attrs: attrs,
 	}
@@ -27029,136 +26772,107 @@ func Cumsum(scope *Scope, x tf.Output, axis tf.Output, optional ...CumsumAttr) (
 	return op.Output(0)
 }
 
-// Return the shape of s0 op s1 with broadcast.
-//
-// Given `s0` and `s1`, tensors that represent shapes, compute `r0`, the
-// broadcasted shape. `s0`, `s1` and `r0` are all integer vectors.
-func BroadcastArgs(scope *Scope, s0 tf.Output, s1 tf.Output) (r0 tf.Output) {
+// Computes hyperbolic tangent of `x` element-wise.
+func Tanh(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "BroadcastArgs",
+		Type: "Tanh",
 		Input: []tf.Input{
-			s0, s1,
+			x,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// DataFormatDimMapAttr is an optional argument to DataFormatDimMap.
-type DataFormatDimMapAttr func(optionalAttr)
-
-// DataFormatDimMapSrcFormat sets the optional src_format attribute to value.
+// Computes the maximum along segments of a tensor.
 //
-// value: source data format.
-// If not specified, defaults to "NHWC"
-func DataFormatDimMapSrcFormat(value string) DataFormatDimMapAttr {
-	return func(m optionalAttr) {
-		m["src_format"] = value
-	}
-}
-
-// DataFormatDimMapDstFormat sets the optional dst_format attribute to value.
+// Read
+// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+// for an explanation of segments.
 //
-// value: destination data format.
-// If not specified, defaults to "NCHW"
-func DataFormatDimMapDstFormat(value string) DataFormatDimMapAttr {
-	return func(m optionalAttr) {
-		m["dst_format"] = value
-	}
-}
-
-// Returns the dimension index in the destination data format given the one in
+// Computes a tensor such that
+// \\(output_i = \max_j(data_j)\\) where `max` is over `j` such
+// that `segment_ids[j] == i`.
 //
-// the source data format.
+// If the max is empty for a given segment ID `i`, `output[i] = 0`.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/SegmentMax.png" alt>
+// </div>
 //
 // Arguments:
-//	x: A Tensor with each element as a dimension index in source data format.
-// Must be in the range [-4, 4).
 //
-// Returns A Tensor with each element as a dimension index in destination data format.
-func DataFormatDimMap(scope *Scope, x tf.Output, optional ...DataFormatDimMapAttr) (y tf.Output) {
+//	segment_ids: A 1-D tensor whose size is equal to the size of `data`'s
+// first dimension.  Values should be sorted and can be repeated.
+//
+// Returns Has same shape as data, except for dimension 0 which
+// has size `k`, the number of segments.
+func SegmentMax(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "DataFormatDimMap",
+		Type: "SegmentMax",
 		Input: []tf.Input{
-			x,
+			data, segment_ids,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// CumprodAttr is an optional argument to Cumprod.
-type CumprodAttr func(optionalAttr)
-
-// CumprodExclusive sets the optional exclusive attribute to value.
+// Creates a dataset that skips `count` elements from the `input_dataset`.
 //
-// value: If `True`, perform exclusive cumprod.
-// If not specified, defaults to false
-func CumprodExclusive(value bool) CumprodAttr {
-	return func(m optionalAttr) {
-		m["exclusive"] = value
+// Arguments:
+//
+//	count: A scalar representing the number of elements from the `input_dataset`
+// that should be skipped.  If count is -1, skips everything.
+//
+//
+func SkipDataset(scope *Scope, input_dataset tf.Output, count tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "SkipDataset",
+		Input: []tf.Input{
+			input_dataset, count,
+		},
+		Attrs: attrs,
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// CumprodReverse sets the optional reverse attribute to value.
-//
-// value: A `bool` (default: False).
-// If not specified, defaults to false
-func CumprodReverse(value bool) CumprodAttr {
+// RealAttr is an optional argument to Real.
+type RealAttr func(optionalAttr)
+
+// RealTout sets the optional Tout attribute to value.
+// If not specified, defaults to DT_FLOAT
+func RealTout(value tf.DataType) RealAttr {
 	return func(m optionalAttr) {
-		m["reverse"] = value
+		m["Tout"] = value
 	}
 }
 
-// Compute the cumulative product of the tensor `x` along `axis`.
-//
-// By default, this op performs an inclusive cumprod, which means that the first
-// element of the input is identical to the first element of the output:
-//
-// ```python
-// tf.cumprod([a, b, c])  # => [a, a * b, a * b * c]
-// ```
-//
-// By setting the `exclusive` kwarg to `True`, an exclusive cumprod is
-// performed instead:
+// Returns the real part of a complex number.
 //
-// ```python
-// tf.cumprod([a, b, c], exclusive=True)  # => [1, a, a * b]
-// ```
+// Given a tensor `input` of complex numbers, this operation returns a tensor of
+// type `float` that is the real part of each element in `input`. All elements in
+// `input` must be complex numbers of the form \\(a + bj\\), where *a* is the real
+//  part returned by this operation and *b* is the imaginary part.
 //
-// By setting the `reverse` kwarg to `True`, the cumprod is performed in the
-// opposite direction:
+// For example:
 //
-// ```python
-// tf.cumprod([a, b, c], reverse=True)  # => [a * b * c, b * c, c]
 // ```
-//
-// This is more efficient than using separate `tf.reverse` ops.
-//
-// The `reverse` and `exclusive` kwargs can also be combined:
-//
-// ```python
-// tf.cumprod([a, b, c], exclusive=True, reverse=True)  # => [b * c, c, 1]
+// # tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j]
+// tf.real(input) ==> [-2.25, 3.25]
 // ```
-//
-// Arguments:
-//	x: A `Tensor`. Must be one of the following types: `float32`, `float64`,
-// `int64`, `int32`, `uint8`, `uint16`, `int16`, `int8`, `complex64`,
-// `complex128`, `qint8`, `quint8`, `qint32`, `half`.
-//	axis: A `Tensor` of type `int32` (default: 0). Must be in the range
-// `[-rank(x), rank(x))`.
-func Cumprod(scope *Scope, x tf.Output, axis tf.Output, optional ...CumprodAttr) (out tf.Output) {
+func Real(scope *Scope, input tf.Output, optional ...RealAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -27167,9 +26881,9 @@ func Cumprod(scope *Scope, x tf.Output, axis tf.Output, optional ...CumprodAttr)
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Cumprod",
+		Type: "Real",
 		Input: []tf.Input{
-			x, axis,
+			input,
 		},
 		Attrs: attrs,
 	}
@@ -27177,65 +26891,69 @@ func Cumprod(scope *Scope, x tf.Output, axis tf.Output, optional ...CumprodAttr)
 	return op.Output(0)
 }
 
-// QuantizedMatMulAttr is an optional argument to QuantizedMatMul.
-type QuantizedMatMulAttr func(optionalAttr)
-
-// QuantizedMatMulToutput sets the optional Toutput attribute to value.
-// If not specified, defaults to DT_QINT32
-func QuantizedMatMulToutput(value tf.DataType) QuantizedMatMulAttr {
-	return func(m optionalAttr) {
-		m["Toutput"] = value
-	}
-}
-
-// QuantizedMatMulTransposeA sets the optional transpose_a attribute to value.
+// Sends `input` to all devices that are connected to the output.
 //
-// value: If true, `a` is transposed before multiplication.
-// If not specified, defaults to false
-func QuantizedMatMulTransposeA(value bool) QuantizedMatMulAttr {
-	return func(m optionalAttr) {
-		m["transpose_a"] = value
+// Sends `input` to all devices that are connected to the output.
+//
+// The graph should be constructed so that all ops connected to the output have a
+// valid device assignment, and the op itself is assigned one of these devices.
+//
+// input: The input to the broadcast.
+// output: The same as input.
+// shape: The shape of the input tensor.
+//
+func NcclBroadcast(scope *Scope, input tf.Output, shape tf.Shape) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"shape": shape}
+	opspec := tf.OpSpec{
+		Type: "NcclBroadcast",
+		Input: []tf.Input{
+			input,
+		},
+		Attrs: attrs,
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// QuantizedMatMulTransposeB sets the optional transpose_b attribute to value.
+// ResizeAreaAttr is an optional argument to ResizeArea.
+type ResizeAreaAttr func(optionalAttr)
+
+// ResizeAreaAlignCorners sets the optional align_corners attribute to value.
 //
-// value: If true, `b` is transposed before multiplication.
+// value: If true, the centers of the 4 corner pixels of the input and output tensors are
+// aligned, preserving the values at the corner pixels. Defaults to false.
 // If not specified, defaults to false
-func QuantizedMatMulTransposeB(value bool) QuantizedMatMulAttr {
+func ResizeAreaAlignCorners(value bool) ResizeAreaAttr {
 	return func(m optionalAttr) {
-		m["transpose_b"] = value
+		m["align_corners"] = value
 	}
 }
 
-// QuantizedMatMulTactivation sets the optional Tactivation attribute to value.
+// Resize `images` to `size` using area interpolation.
 //
-// value: The type of output produced by activation function
-// following this operation.
-// If not specified, defaults to DT_QUINT8
-func QuantizedMatMulTactivation(value tf.DataType) QuantizedMatMulAttr {
-	return func(m optionalAttr) {
-		m["Tactivation"] = value
-	}
-}
-
-// Perform a quantized matrix multiplication of  `a` by the matrix `b`.
+// Input images can be of different types but output images are always float.
 //
-// The inputs must be two-dimensional matrices and the inner dimension of
-// `a` (after being transposed if `transpose_a` is non-zero) must match the
-// outer dimension of `b` (after being transposed if `transposed_b` is
-// non-zero).
+// The range of pixel values for the output image might be slightly different
+// from the range for the input image because of limited numerical precision.
+// To guarantee an output range, for example `[0.0, 1.0]`, apply
+// `tf.clip_by_value` to the output.
+//
+// Each output pixel is computed by first transforming the pixel's footprint into
+// the input tensor and then averaging the pixels that intersect the footprint. An
+// input pixel's contribution to the average is weighted by the fraction of its
+// area that intersects the footprint.  This is the same as OpenCV's INTER_AREA.
 //
 // Arguments:
-//	a: Must be a two-dimensional tensor.
-//	b: Must be a two-dimensional tensor.
-//	min_a: The float value that the lowest quantized `a` value represents.
-//	max_a: The float value that the highest quantized `a` value represents.
-//	min_b: The float value that the lowest quantized `b` value represents.
-//	max_b: The float value that the highest quantized `b` value represents.
+//	images: 4-D with shape `[batch, height, width, channels]`.
+//	size: = A 1-D int32 Tensor of 2 elements: `new_height, new_width`.  The
+// new size for the images.
 //
-// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
-func QuantizedMatMul(scope *Scope, a tf.Output, b tf.Output, min_a tf.Output, max_a tf.Output, min_b tf.Output, max_b tf.Output, optional ...QuantizedMatMulAttr) (out tf.Output, min_out tf.Output, max_out tf.Output) {
+// Returns 4-D with shape
+// `[batch, new_height, new_width, channels]`.
+func ResizeArea(scope *Scope, images tf.Output, size tf.Output, optional ...ResizeAreaAttr) (resized_images tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -27244,114 +26962,93 @@ func QuantizedMatMul(scope *Scope, a tf.Output, b tf.Output, min_a tf.Output, ma
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "QuantizedMatMul",
+		Type: "ResizeArea",
 		Input: []tf.Input{
-			a, b, min_a, max_a, min_b, max_b,
+			images, size,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// Does nothing. Serves as a control trigger for scheduling.
-//
-// Only useful as a placeholder for control edges.
+// VarHandleOpAttr is an optional argument to VarHandleOp.
+type VarHandleOpAttr func(optionalAttr)
+
+// VarHandleOpContainer sets the optional container attribute to value.
 //
-// Returns the created operation.
-func ControlTrigger(scope *Scope) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ControlTrigger",
+// value: the container this variable is placed in.
+// If not specified, defaults to ""
+func VarHandleOpContainer(value string) VarHandleOpAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
 	}
-	return scope.AddOperation(opspec)
 }
 
-// Batch normalization.
-//
-// DEPRECATED at GraphDef version 9: Use tf.nn.batch_normalization()
+// VarHandleOpSharedName sets the optional shared_name attribute to value.
 //
-// This op is deprecated. Prefer `tf.nn.batch_normalization`.
+// value: the name by which this variable is referred to.
+// If not specified, defaults to ""
+func VarHandleOpSharedName(value string) VarHandleOpAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Creates a handle to a Variable resource.
 //
 // Arguments:
-//	t: A 4D input Tensor.
-//	m: A 1D mean Tensor with size matching the last dimension of t.
-// This is the first output from tf.nn.moments,
-// or a saved moving average thereof.
-//	v: A 1D variance Tensor with size matching the last dimension of t.
-// This is the second output from tf.nn.moments,
-// or a saved moving average thereof.
-//	beta: A 1D beta Tensor with size matching the last dimension of t.
-// An offset to be added to the normalized tensor.
-//	gamma: A 1D gamma Tensor with size matching the last dimension of t.
-// If "scale_after_normalization" is true, this tensor will be multiplied
-// with the normalized tensor.
-//	variance_epsilon: A small float number to avoid dividing by 0.
-//	scale_after_normalization: A bool indicating whether the resulted tensor
-// needs to be multiplied with gamma.
-func BatchNormWithGlobalNormalization(scope *Scope, t tf.Output, m tf.Output, v tf.Output, beta tf.Output, gamma tf.Output, variance_epsilon float32, scale_after_normalization bool) (result tf.Output) {
+//	dtype: the type of this variable. Must agree with the dtypes
+// of all ops using this variable.
+//	shape: The (possibly partially specified) shape of this variable.
+func VarHandleOp(scope *Scope, dtype tf.DataType, shape tf.Shape, optional ...VarHandleOpAttr) (resource tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"variance_epsilon": variance_epsilon, "scale_after_normalization": scale_after_normalization}
-	opspec := tf.OpSpec{
-		Type: "BatchNormWithGlobalNormalization",
-		Input: []tf.Input{
-			t, m, v, beta, gamma,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Deprecated. Use TensorArrayReadV3
-//
-// DEPRECATED at GraphDef version 26: Use TensorArrayReadV3
-func TensorArrayReadV2(scope *Scope, handle tf.Output, index tf.Output, flow_in tf.Output, dtype tf.DataType) (value tf.Output) {
-	if scope.Err() != nil {
-		return
+	attrs := map[string]interface{}{"dtype": dtype, "shape": shape}
+	for _, a := range optional {
+		a(attrs)
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
 	opspec := tf.OpSpec{
-		Type: "TensorArrayReadV2",
-		Input: []tf.Input{
-			handle, index, flow_in,
-		},
+		Type: "VarHandleOp",
+
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// QuantizedMulAttr is an optional argument to QuantizedMul.
-type QuantizedMulAttr func(optionalAttr)
+// AngleAttr is an optional argument to Angle.
+type AngleAttr func(optionalAttr)
 
-// QuantizedMulToutput sets the optional Toutput attribute to value.
-// If not specified, defaults to DT_QINT32
-func QuantizedMulToutput(value tf.DataType) QuantizedMulAttr {
+// AngleTout sets the optional Tout attribute to value.
+// If not specified, defaults to DT_FLOAT
+func AngleTout(value tf.DataType) AngleAttr {
 	return func(m optionalAttr) {
-		m["Toutput"] = value
+		m["Tout"] = value
 	}
 }
 
-// Returns x * y element-wise, working on quantized buffers.
+// Returns the argument of a complex number.
 //
-// Arguments:
+// Given a tensor `input` of complex numbers, this operation returns a tensor of
+// type `float` that is the argument of each element in `input`. All elements in
+// `input` must be complex numbers of the form \\(a + bj\\), where *a*
+// is the real part and *b* is the imaginary part.
 //
+// The argument returned by this operation is of the form \\(atan2(b, a)\\).
 //
-//	min_x: The float value that the lowest quantized `x` value represents.
-//	max_x: The float value that the highest quantized `x` value represents.
-//	min_y: The float value that the lowest quantized `y` value represents.
-//	max_y: The float value that the highest quantized `y` value represents.
+// For example:
 //
-// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
+// ```
+// # tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j]
+// tf.angle(input) ==> [2.0132, 1.056]
+// ```
 //
-// *NOTE*: `QuantizedMul` supports limited forms of broadcasting. More about
-// broadcasting [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func QuantizedMul(scope *Scope, x tf.Output, y tf.Output, min_x tf.Output, max_x tf.Output, min_y tf.Output, max_y tf.Output, optional ...QuantizedMulAttr) (z tf.Output, min_z tf.Output, max_z tf.Output) {
+// @compatibility(numpy)
+// Equivalent to np.angle.
+// @end_compatibility
+func Angle(scope *Scope, input tf.Output, optional ...AngleAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -27360,214 +27057,219 @@ func QuantizedMul(scope *Scope, x tf.Output, y tf.Output, min_x tf.Output, max_x
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "QuantizedMul",
+		Type: "Angle",
 		Input: []tf.Input{
-			x, y, min_x, max_x, min_y, max_y,
+			input,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
-}
-
-// QuantizedAddAttr is an optional argument to QuantizedAdd.
-type QuantizedAddAttr func(optionalAttr)
-
-// QuantizedAddToutput sets the optional Toutput attribute to value.
-// If not specified, defaults to DT_QINT32
-func QuantizedAddToutput(value tf.DataType) QuantizedAddAttr {
-	return func(m optionalAttr) {
-		m["Toutput"] = value
-	}
+	return op.Output(0)
 }
 
-// Returns x + y element-wise, working on quantized buffers.
-//
-// Arguments:
-//
+// Clips tensor values to a specified min and max.
 //
-//	min_x: The float value that the lowest quantized `x` value represents.
-//	max_x: The float value that the highest quantized `x` value represents.
-//	min_y: The float value that the lowest quantized `y` value represents.
-//	max_y: The float value that the highest quantized `y` value represents.
+// Given a tensor `t`, this operation returns a tensor of the same type and
+// shape as `t` with its values clipped to `clip_value_min` and `clip_value_max`.
+// Any values less than `clip_value_min` are set to `clip_value_min`. Any values
+// greater than `clip_value_max` are set to `clip_value_max`.
 //
-// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
+// Arguments:
+//	t: A `Tensor`.
+//	clip_value_min: A 0-D (scalar) `Tensor`, or a `Tensor` with the same shape
+// as `t`. The minimum value to clip by.
+//	clip_value_max: A 0-D (scalar) `Tensor`, or a `Tensor` with the same shape
+// as `t`. The maximum value to clip by.
 //
-// *NOTE*: `QuantizedAdd` supports limited forms of broadcasting. More about
-// broadcasting [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func QuantizedAdd(scope *Scope, x tf.Output, y tf.Output, min_x tf.Output, max_x tf.Output, min_y tf.Output, max_y tf.Output, optional ...QuantizedAddAttr) (z tf.Output, min_z tf.Output, max_z tf.Output) {
+// Returns A clipped `Tensor` with the same shape as input 't'.
+func ClipByValue(scope *Scope, t tf.Output, clip_value_min tf.Output, clip_value_max tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "QuantizedAdd",
+		Type: "ClipByValue",
 		Input: []tf.Input{
-			x, y, min_x, max_x, min_y, max_y,
+			t, clip_value_min, clip_value_max,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// Given a quantized tensor described by (input, input_min, input_max), outputs a
+// Counts the number of occurrences of each value in an integer array.
 //
-// range that covers the actual values present in that tensor.  This op is
-// typically used to produce the requested_output_min and requested_output_max for
-// Requantize.
+// Outputs a vector with length `size` and the same dtype as `weights`. If
+// `weights` are empty, then index `i` stores the number of times the value `i` is
+// counted in `arr`. If `weights` are non-empty, then index `i` stores the sum of
+// the value in `weights` at each index where the corresponding value in `arr` is
+// `i`.
 //
-// Arguments:
+// Values in `arr` outside of the range [0, size) are ignored.
 //
-//	input_min: The float value that the minimum quantized input value represents.
-//	input_max: The float value that the maximum quantized input value represents.
+// Arguments:
+//	arr: int32 `Tensor`.
+//	size: non-negative int32 scalar `Tensor`.
+//	weights: is an int32, int64, float32, or float64 `Tensor` with the same
+// shape as `arr`, or a length-0 `Tensor`, in which case it acts as all weights
+// equal to 1.
 //
-// Returns The computed min output.the computed max output.
-func RequantizationRange(scope *Scope, input tf.Output, input_min tf.Output, input_max tf.Output) (output_min tf.Output, output_max tf.Output) {
+// Returns 1D `Tensor` with length equal to `size`. The counts or summed weights for
+// each value in the range [0, size).
+func Bincount(scope *Scope, arr tf.Output, size tf.Output, weights tf.Output) (bins tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "RequantizationRange",
+		Type: "Bincount",
 		Input: []tf.Input{
-			input, input_min, input_max,
+			arr, size, weights,
 		},
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// Rolls the elements of a tensor along an axis.
+// CumsumAttr is an optional argument to Cumsum.
+type CumsumAttr func(optionalAttr)
+
+// CumsumExclusive sets the optional exclusive attribute to value.
 //
-// The elements are shifted positively (towards larger indices) by the offset of
-// `shift` along the dimension of `axis`. Negative `shift` values will shift
-// elements in the opposite direction. Elements that roll passed the last position
-// will wrap around to the first and vice versa. Multiple shifts along multiple
-// axes may be specified.
+// value: If `True`, perform exclusive cumsum.
+// If not specified, defaults to false
+func CumsumExclusive(value bool) CumsumAttr {
+	return func(m optionalAttr) {
+		m["exclusive"] = value
+	}
+}
+
+// CumsumReverse sets the optional reverse attribute to value.
 //
-// For example:
+// value: A `bool` (default: False).
+// If not specified, defaults to false
+func CumsumReverse(value bool) CumsumAttr {
+	return func(m optionalAttr) {
+		m["reverse"] = value
+	}
+}
+
+// Compute the cumulative sum of the tensor `x` along `axis`.
 //
+// By default, this op performs an inclusive cumsum, which means that the first
+// element of the input is identical to the first element of the output:
+//
+// ```python
+// tf.cumsum([a, b, c])  # => [a, a + b, a + b + c]
 // ```
-// # 't' is [0, 1, 2, 3, 4]
-// roll(t, shift=2, axis=0) ==> [3, 4, 0, 1, 2]
 //
-// # shifting along multiple dimensions
-// # 't' is [[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]]
-// roll(t, shift=[1, -2], axis=[0, 1]) ==> [[7, 8, 9, 5, 6], [2, 3, 4, 0, 1]]
+// By setting the `exclusive` kwarg to `True`, an exclusive cumsum is
+// performed instead:
 //
-// # shifting along the same axis multiple times
-// # 't' is [[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]]
-// roll(t, shift=[2, -3], axis=[1, 1]) ==> [[1, 2, 3, 4, 0], [6, 7, 8, 9, 5]]
+// ```python
+// tf.cumsum([a, b, c], exclusive=True)  # => [0, a, a + b]
 // ```
 //
-// Arguments:
+// By setting the `reverse` kwarg to `True`, the cumsum is performed in the
+// opposite direction:
 //
-//	shift: Dimension must be 0-D or 1-D. `shift[i]` specifies the number of places by which
-// elements are shifted positively (towards larger indices) along the dimension
-// specified by `axis[i]`. Negative shifts will roll the elements in the opposite
-// direction.
-//	axis: Dimension must be 0-D or 1-D. `axis[i]` specifies the dimension that the shift
-// `shift[i]` should occur. If the same axis is referenced more than once, the
-// total shift for that axis will be the sum of all the shifts that belong to that
-// axis.
+// ```python
+// tf.cumsum([a, b, c], reverse=True)  # => [a + b + c, b + c, c]
+// ```
 //
-// Returns Has the same shape and size as the input. The elements are shifted
-// positively (towards larger indices) by the offsets of `shift` along the
-// dimensions of `axis`.
-func Roll(scope *Scope, input tf.Output, shift tf.Output, axis tf.Output) (output tf.Output) {
+// This is more efficient than using separate `tf.reverse` ops.
+//
+// The `reverse` and `exclusive` kwargs can also be combined:
+//
+// ```python
+// tf.cumsum([a, b, c], exclusive=True, reverse=True)  # => [b + c, c, 0]
+// ```
+//
+// Arguments:
+//	x: A `Tensor`. Must be one of the following types: `float32`, `float64`,
+// `int64`, `int32`, `uint8`, `uint16`, `int16`, `int8`, `complex64`,
+// `complex128`, `qint8`, `quint8`, `qint32`, `half`.
+//	axis: A `Tensor` of type `int32` (default: 0). Must be in the range
+// `[-rank(x), rank(x))`.
+func Cumsum(scope *Scope, x tf.Output, axis tf.Output, optional ...CumsumAttr) (out tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "Roll",
+		Type: "Cumsum",
 		Input: []tf.Input{
-			input, shift, axis,
+			x, axis,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Looks up keys in a table, outputs the corresponding values.
-//
-// The tensor `keys` must of the same type as the keys of the table.
-// The output `values` is of the type of the table values.
-//
-// The scalar `default_value` is the value output for keys not present in the
-// table. It must also be of the same type as the table values.
-//
-// Arguments:
-//	table_handle: Handle to the table.
-//	keys: Any shape.  Keys to look up.
-//
+// Return the shape of s0 op s1 with broadcast.
 //
-// Returns Same shape as `keys`.  Values found in the table, or `default_values`
-// for missing keys.
-func LookupTableFindV2(scope *Scope, table_handle tf.Output, keys tf.Output, default_value tf.Output) (values tf.Output) {
+// Given `s0` and `s1`, tensors that represent shapes, compute `r0`, the
+// broadcasted shape. `s0`, `s1` and `r0` are all integer vectors.
+func BroadcastArgs(scope *Scope, s0 tf.Output, s1 tf.Output) (r0 tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "LookupTableFindV2",
+		Type: "BroadcastArgs",
 		Input: []tf.Input{
-			table_handle, keys, default_value,
+			s0, s1,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Updates the table to associates keys with values.
-//
-// The tensor `keys` must be of the same type as the keys of the table.
-// The tensor `values` must be of the type of the table values.
-//
-// Arguments:
-//	table_handle: Handle to the table.
-//	keys: Any shape.  Keys to look up.
-//	values: Values to associate with keys.
+// DataFormatDimMapAttr is an optional argument to DataFormatDimMap.
+type DataFormatDimMapAttr func(optionalAttr)
+
+// DataFormatDimMapSrcFormat sets the optional src_format attribute to value.
 //
-// Returns the created operation.
-func LookupTableInsertV2(scope *Scope, table_handle tf.Output, keys tf.Output, values tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
+// value: source data format.
+// If not specified, defaults to "NHWC"
+func DataFormatDimMapSrcFormat(value string) DataFormatDimMapAttr {
+	return func(m optionalAttr) {
+		m["src_format"] = value
 	}
-	opspec := tf.OpSpec{
-		Type: "LookupTableInsertV2",
-		Input: []tf.Input{
-			table_handle, keys, values,
-		},
+}
+
+// DataFormatDimMapDstFormat sets the optional dst_format attribute to value.
+//
+// value: destination data format.
+// If not specified, defaults to "NCHW"
+func DataFormatDimMapDstFormat(value string) DataFormatDimMapAttr {
+	return func(m optionalAttr) {
+		m["dst_format"] = value
 	}
-	return scope.AddOperation(opspec)
 }
 
-// Creates a dataset that batches and pads `batch_size` elements from the input.
+// Returns the dimension index in the destination data format given the one in
 //
-// Arguments:
+// the source data format.
 //
-//	batch_size: A scalar representing the number of elements to accumulate in a
-// batch.
-//	padded_shapes: A list of int64 tensors representing the desired padded shapes
-// of the corresponding output components. These shapes may be partially
-// specified, using `-1` to indicate that a particular dimension should be
-// padded to the maximum size of all batch elements.
-//	padding_values: A list of scalars containing the padding value to use for
-// each of the outputs.
-//	drop_remainder: A scalar representing whether the last batch should be dropped in case its size
-// is smaller than desired.
+// Arguments:
+//	x: A Tensor with each element as a dimension index in source data format.
+// Must be in the range [-4, 4).
 //
-func PaddedBatchDatasetV2(scope *Scope, input_dataset tf.Output, batch_size tf.Output, padded_shapes []tf.Output, padding_values []tf.Output, drop_remainder tf.Output, output_shapes []tf.Shape) (handle tf.Output) {
+// Returns A Tensor with each element as a dimension index in destination data format.
+func DataFormatDimMap(scope *Scope, x tf.Output, optional ...DataFormatDimMapAttr) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"output_shapes": output_shapes}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "PaddedBatchDatasetV2",
+		Type: "DataFormatDimMap",
 		Input: []tf.Input{
-			input_dataset, batch_size, tf.OutputList(padded_shapes), tf.OutputList(padding_values), drop_remainder,
+			x,
 		},
 		Attrs: attrs,
 	}
@@ -27575,66 +27277,144 @@ func PaddedBatchDatasetV2(scope *Scope, input_dataset tf.Output, batch_size tf.O
 	return op.Output(0)
 }
 
-// Returns element-wise smallest integer not less than x.
-func Ceil(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
+// CumprodAttr is an optional argument to Cumprod.
+type CumprodAttr func(optionalAttr)
+
+// CumprodExclusive sets the optional exclusive attribute to value.
+//
+// value: If `True`, perform exclusive cumprod.
+// If not specified, defaults to false
+func CumprodExclusive(value bool) CumprodAttr {
+	return func(m optionalAttr) {
+		m["exclusive"] = value
 	}
-	opspec := tf.OpSpec{
-		Type: "Ceil",
-		Input: []tf.Input{
-			x,
-		},
+}
+
+// CumprodReverse sets the optional reverse attribute to value.
+//
+// value: A `bool` (default: False).
+// If not specified, defaults to false
+func CumprodReverse(value bool) CumprodAttr {
+	return func(m optionalAttr) {
+		m["reverse"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Computes the number of elements in the given table.
+// Compute the cumulative product of the tensor `x` along `axis`.
 //
-// Arguments:
-//	table_handle: Handle to the table.
+// By default, this op performs an inclusive cumprod, which means that the first
+// element of the input is identical to the first element of the output:
 //
-// Returns Scalar that contains number of elements in the table.
-func LookupTableSizeV2(scope *Scope, table_handle tf.Output) (size tf.Output) {
+// ```python
+// tf.cumprod([a, b, c])  # => [a, a * b, a * b * c]
+// ```
+//
+// By setting the `exclusive` kwarg to `True`, an exclusive cumprod is
+// performed instead:
+//
+// ```python
+// tf.cumprod([a, b, c], exclusive=True)  # => [1, a, a * b]
+// ```
+//
+// By setting the `reverse` kwarg to `True`, the cumprod is performed in the
+// opposite direction:
+//
+// ```python
+// tf.cumprod([a, b, c], reverse=True)  # => [a * b * c, b * c, c]
+// ```
+//
+// This is more efficient than using separate `tf.reverse` ops.
+//
+// The `reverse` and `exclusive` kwargs can also be combined:
+//
+// ```python
+// tf.cumprod([a, b, c], exclusive=True, reverse=True)  # => [b * c, c, 1]
+// ```
+//
+// Arguments:
+//	x: A `Tensor`. Must be one of the following types: `float32`, `float64`,
+// `int64`, `int32`, `uint8`, `uint16`, `int16`, `int8`, `complex64`,
+// `complex128`, `qint8`, `quint8`, `qint32`, `half`.
+//	axis: A `Tensor` of type `int32` (default: 0). Must be in the range
+// `[-rank(x), rank(x))`.
+func Cumprod(scope *Scope, x tf.Output, axis tf.Output, optional ...CumprodAttr) (out tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "LookupTableSizeV2",
+		Type: "Cumprod",
 		Input: []tf.Input{
-			table_handle,
+			x, axis,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ResizeBilinearGradAttr is an optional argument to ResizeBilinearGrad.
-type ResizeBilinearGradAttr func(optionalAttr)
+// QuantizedMatMulAttr is an optional argument to QuantizedMatMul.
+type QuantizedMatMulAttr func(optionalAttr)
 
-// ResizeBilinearGradAlignCorners sets the optional align_corners attribute to value.
+// QuantizedMatMulToutput sets the optional Toutput attribute to value.
+// If not specified, defaults to DT_QINT32
+func QuantizedMatMulToutput(value tf.DataType) QuantizedMatMulAttr {
+	return func(m optionalAttr) {
+		m["Toutput"] = value
+	}
+}
+
+// QuantizedMatMulTransposeA sets the optional transpose_a attribute to value.
 //
-// value: If true, the centers of the 4 corner pixels of the input and grad tensors are
-// aligned. Defaults to false.
+// value: If true, `a` is transposed before multiplication.
 // If not specified, defaults to false
-func ResizeBilinearGradAlignCorners(value bool) ResizeBilinearGradAttr {
+func QuantizedMatMulTransposeA(value bool) QuantizedMatMulAttr {
 	return func(m optionalAttr) {
-		m["align_corners"] = value
+		m["transpose_a"] = value
 	}
 }
 
-// Computes the gradient of bilinear interpolation.
+// QuantizedMatMulTransposeB sets the optional transpose_b attribute to value.
+//
+// value: If true, `b` is transposed before multiplication.
+// If not specified, defaults to false
+func QuantizedMatMulTransposeB(value bool) QuantizedMatMulAttr {
+	return func(m optionalAttr) {
+		m["transpose_b"] = value
+	}
+}
+
+// QuantizedMatMulTactivation sets the optional Tactivation attribute to value.
+//
+// value: The type of output produced by activation function
+// following this operation.
+// If not specified, defaults to DT_QUINT8
+func QuantizedMatMulTactivation(value tf.DataType) QuantizedMatMulAttr {
+	return func(m optionalAttr) {
+		m["Tactivation"] = value
+	}
+}
+
+// Perform a quantized matrix multiplication of  `a` by the matrix `b`.
+//
+// The inputs must be two-dimensional matrices and the inner dimension of
+// `a` (after being transposed if `transpose_a` is non-zero) must match the
+// outer dimension of `b` (after being transposed if `transposed_b` is
+// non-zero).
 //
 // Arguments:
-//	grads: 4-D with shape `[batch, height, width, channels]`.
-//	original_image: 4-D with shape `[batch, orig_height, orig_width, channels]`,
-// The image tensor that was resized.
+//	a: Must be a two-dimensional tensor.
+//	b: Must be a two-dimensional tensor.
+//	min_a: The float value that the lowest quantized `a` value represents.
+//	max_a: The float value that the highest quantized `a` value represents.
+//	min_b: The float value that the lowest quantized `b` value represents.
+//	max_b: The float value that the highest quantized `b` value represents.
 //
-// Returns 4-D with shape `[batch, orig_height, orig_width, channels]`.
-// Gradients with respect to the input image. Input image must have been
-// float or double.
-func ResizeBilinearGrad(scope *Scope, grads tf.Output, original_image tf.Output, optional ...ResizeBilinearGradAttr) (output tf.Output) {
+// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
+func QuantizedMatMul(scope *Scope, a tf.Output, b tf.Output, min_a tf.Output, max_a tf.Output, min_b tf.Output, max_b tf.Output, optional ...QuantizedMatMulAttr) (out tf.Output, min_out tf.Output, max_out tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -27643,109 +27423,81 @@ func ResizeBilinearGrad(scope *Scope, grads tf.Output, original_image tf.Output,
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResizeBilinearGrad",
+		Type: "QuantizedMatMul",
 		Input: []tf.Input{
-			grads, original_image,
+			a, b, min_a, max_a, min_b, max_b,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Outputs all keys and values in the table.
-//
-// Arguments:
-//	table_handle: Handle to the table.
-//
+// Does nothing. Serves as a control trigger for scheduling.
 //
+// Only useful as a placeholder for control edges.
 //
-// Returns Vector of all keys present in the table.Tensor of all values in the table. Indexed in parallel with `keys`.
-func LookupTableExportV2(scope *Scope, table_handle tf.Output, Tkeys tf.DataType, Tvalues tf.DataType) (keys tf.Output, values tf.Output) {
+// Returns the created operation.
+func ControlTrigger(scope *Scope) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"Tkeys": Tkeys, "Tvalues": Tvalues}
 	opspec := tf.OpSpec{
-		Type: "LookupTableExportV2",
-		Input: []tf.Input{
-			table_handle,
-		},
-		Attrs: attrs,
+		Type: "ControlTrigger",
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return scope.AddOperation(opspec)
 }
 
-// Replaces the contents of the table with the specified keys and values.
+// Batch normalization.
 //
-// The tensor `keys` must be of the same type as the keys of the table.
-// The tensor `values` must be of the type of the table values.
+// DEPRECATED at GraphDef version 9: Use tf.nn.batch_normalization()
 //
-// Arguments:
-//	table_handle: Handle to the table.
-//	keys: Any shape.  Keys to look up.
-//	values: Values to associate with keys.
+// This op is deprecated. Prefer `tf.nn.batch_normalization`.
 //
-// Returns the created operation.
-func LookupTableImportV2(scope *Scope, table_handle tf.Output, keys tf.Output, values tf.Output) (o *tf.Operation) {
+// Arguments:
+//	t: A 4D input Tensor.
+//	m: A 1D mean Tensor with size matching the last dimension of t.
+// This is the first output from tf.nn.moments,
+// or a saved moving average thereof.
+//	v: A 1D variance Tensor with size matching the last dimension of t.
+// This is the second output from tf.nn.moments,
+// or a saved moving average thereof.
+//	beta: A 1D beta Tensor with size matching the last dimension of t.
+// An offset to be added to the normalized tensor.
+//	gamma: A 1D gamma Tensor with size matching the last dimension of t.
+// If "scale_after_normalization" is true, this tensor will be multiplied
+// with the normalized tensor.
+//	variance_epsilon: A small float number to avoid dividing by 0.
+//	scale_after_normalization: A bool indicating whether the resulted tensor
+// needs to be multiplied with gamma.
+func BatchNormWithGlobalNormalization(scope *Scope, t tf.Output, m tf.Output, v tf.Output, beta tf.Output, gamma tf.Output, variance_epsilon float32, scale_after_normalization bool) (result tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"variance_epsilon": variance_epsilon, "scale_after_normalization": scale_after_normalization}
 	opspec := tf.OpSpec{
-		Type: "LookupTableImportV2",
+		Type: "BatchNormWithGlobalNormalization",
 		Input: []tf.Input{
-			table_handle, keys, values,
+			t, m, v, beta, gamma,
 		},
+		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
-}
-
-// MultiDeviceIteratorFromStringHandleAttr is an optional argument to MultiDeviceIteratorFromStringHandle.
-type MultiDeviceIteratorFromStringHandleAttr func(optionalAttr)
-
-// MultiDeviceIteratorFromStringHandleOutputTypes sets the optional output_types attribute to value.
-//
-// value: The type list for the return values.
-// If not specified, defaults to <>
-//
-// REQUIRES: len(value) >= 0
-func MultiDeviceIteratorFromStringHandleOutputTypes(value []tf.DataType) MultiDeviceIteratorFromStringHandleAttr {
-	return func(m optionalAttr) {
-		m["output_types"] = value
-	}
-}
-
-// MultiDeviceIteratorFromStringHandleOutputShapes sets the optional output_shapes attribute to value.
-//
-// value: The list of shapes being produced.
-// If not specified, defaults to <>
-//
-// REQUIRES: len(value) >= 0
-func MultiDeviceIteratorFromStringHandleOutputShapes(value []tf.Shape) MultiDeviceIteratorFromStringHandleAttr {
-	return func(m optionalAttr) {
-		m["output_shapes"] = value
-	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Generates a MultiDeviceIterator resource from its provided string handle.
-//
-// Arguments:
-//	string_handle: String representing the resource.
+// Deprecated. Use TensorArrayReadV3
 //
-// Returns A MultiDeviceIterator resource.
-func MultiDeviceIteratorFromStringHandle(scope *Scope, string_handle tf.Output, optional ...MultiDeviceIteratorFromStringHandleAttr) (multi_device_iterator tf.Output) {
+// DEPRECATED at GraphDef version 26: Use TensorArrayReadV3
+func TensorArrayReadV2(scope *Scope, handle tf.Output, index tf.Output, flow_in tf.Output, dtype tf.DataType) (value tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"dtype": dtype}
 	opspec := tf.OpSpec{
-		Type: "MultiDeviceIteratorFromStringHandle",
+		Type: "TensorArrayReadV2",
 		Input: []tf.Input{
-			string_handle,
+			handle, index, flow_in,
 		},
 		Attrs: attrs,
 	}
@@ -27753,162 +27505,76 @@ func MultiDeviceIteratorFromStringHandle(scope *Scope, string_handle tf.Output,
 	return op.Output(0)
 }
 
-// MutableHashTableV2Attr is an optional argument to MutableHashTableV2.
-type MutableHashTableV2Attr func(optionalAttr)
+// QuantizedMulAttr is an optional argument to QuantizedMul.
+type QuantizedMulAttr func(optionalAttr)
 
-// MutableHashTableV2Container sets the optional container attribute to value.
-//
-// value: If non-empty, this table is placed in the given container.
-// Otherwise, a default container is used.
-// If not specified, defaults to ""
-func MutableHashTableV2Container(value string) MutableHashTableV2Attr {
+// QuantizedMulToutput sets the optional Toutput attribute to value.
+// If not specified, defaults to DT_QINT32
+func QuantizedMulToutput(value tf.DataType) QuantizedMulAttr {
 	return func(m optionalAttr) {
-		m["container"] = value
+		m["Toutput"] = value
 	}
 }
 
-// MutableHashTableV2SharedName sets the optional shared_name attribute to value.
+// Returns x * y element-wise, working on quantized buffers.
 //
-// value: If non-empty, this table is shared under the given name across
-// multiple sessions.
-// If not specified, defaults to ""
-func MutableHashTableV2SharedName(value string) MutableHashTableV2Attr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// MutableHashTableV2UseNodeNameSharing sets the optional use_node_name_sharing attribute to value.
+// Arguments:
 //
-// value: If true and shared_name is empty, the table is shared
-// using the node name.
-// If not specified, defaults to false
-func MutableHashTableV2UseNodeNameSharing(value bool) MutableHashTableV2Attr {
-	return func(m optionalAttr) {
-		m["use_node_name_sharing"] = value
-	}
-}
-
-// Creates an empty hash table.
 //
-// This op creates a mutable hash table, specifying the type of its keys and
-// values. Each value must be a scalar. Data can be inserted into the table using
-// the insert operations. It does not support the initialization operation.
+//	min_x: The float value that the lowest quantized `x` value represents.
+//	max_x: The float value that the highest quantized `x` value represents.
+//	min_y: The float value that the lowest quantized `y` value represents.
+//	max_y: The float value that the highest quantized `y` value represents.
 //
-// Arguments:
-//	key_dtype: Type of the table keys.
-//	value_dtype: Type of the table values.
+// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
 //
-// Returns Handle to a table.
-func MutableHashTableV2(scope *Scope, key_dtype tf.DataType, value_dtype tf.DataType, optional ...MutableHashTableV2Attr) (table_handle tf.Output) {
+// *NOTE*: `QuantizedMul` supports limited forms of broadcasting. More about
+// broadcasting [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func QuantizedMul(scope *Scope, x tf.Output, y tf.Output, min_x tf.Output, max_x tf.Output, min_y tf.Output, max_y tf.Output, optional ...QuantizedMulAttr) (z tf.Output, min_z tf.Output, max_z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"key_dtype": key_dtype, "value_dtype": value_dtype}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MutableHashTableV2",
-
+		Type: "QuantizedMul",
+		Input: []tf.Input{
+			x, y, min_x, max_x, min_y, max_y,
+		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// DequantizeAttr is an optional argument to Dequantize.
-type DequantizeAttr func(optionalAttr)
+// QuantizedAddAttr is an optional argument to QuantizedAdd.
+type QuantizedAddAttr func(optionalAttr)
 
-// DequantizeMode sets the optional mode attribute to value.
-// If not specified, defaults to "MIN_COMBINED"
-func DequantizeMode(value string) DequantizeAttr {
+// QuantizedAddToutput sets the optional Toutput attribute to value.
+// If not specified, defaults to DT_QINT32
+func QuantizedAddToutput(value tf.DataType) QuantizedAddAttr {
 	return func(m optionalAttr) {
-		m["mode"] = value
+		m["Toutput"] = value
 	}
 }
 
-// Dequantize the 'input' tensor into a float Tensor.
-//
-// [min_range, max_range] are scalar floats that specify the range for
-// the 'input' data. The 'mode' attribute controls exactly which calculations are
-// used to convert the float values to their quantized equivalents.
-//
-// In 'MIN_COMBINED' mode, each value of the tensor will undergo the following:
-//
-// ```
-// if T == qint8: in[i] += (range(T) + 1)/ 2.0
-// out[i] = min_range + (in[i]* (max_range - min_range) / range(T))
-// ```
-// here `range(T) = numeric_limits<T>::max() - numeric_limits<T>::min()`
-//
-// *MIN_COMBINED Mode Example*
-//
-// If the input comes from a QuantizedRelu6, the output type is
-// quint8 (range of 0-255) but the possible range of QuantizedRelu6 is
-// 0-6.  The min_range and max_range values are therefore 0.0 and 6.0.
-// Dequantize on quint8 will take each value, cast to float, and multiply
-// by 6 / 255.
-// Note that if quantizedtype is qint8, the operation will additionally add
-// each value by 128 prior to casting.
-//
-// If the mode is 'MIN_FIRST', then this approach is used:
-//
-// ```c++
-// num_discrete_values = 1 << (# of bits in T)
-// range_adjust = num_discrete_values / (num_discrete_values - 1)
-// range = (range_max - range_min) * range_adjust
-// range_scale = range / num_discrete_values
-// const double offset_input = static_cast<double>(input) - lowest_quantized;
-// result = range_min + ((input - numeric_limits<T>::min()) * range_scale)
-// ```
-//
-// *SCALED mode Example*
-//
-// `SCALED` mode matches the quantization approach used in
-// `QuantizeAndDequantize{V2|V3}`.
-//
-// If the mode is `SCALED`, we do not use the full range of the output type,
-// choosing to elide the lowest possible value for symmetry (e.g., output range is
-// -127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to
-// 0.
-//
-// We first find the range of values in our tensor. The
-// range we use is always centered on 0, so we find m such that
-// ```c++
-//   m = max(abs(input_min), abs(input_max))
-// ```
-//
-// Our input tensor range is then `[-m, m]`.
-//
-// Next, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.
-// If T is signed, this is
-// ```
-//   num_bits = sizeof(T) * 8
-//   [min_fixed, max_fixed] =
-//       [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]
-// ```
+// Returns x + y element-wise, working on quantized buffers.
 //
-// Otherwise, if T is unsigned, the fixed-point range is
-// ```
-//   [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]
-// ```
+// Arguments:
 //
-// From this we compute our scaling factor, s:
-// ```c++
-//   s = (2 * m) / (max_fixed - min_fixed)
-// ```
 //
-// Now we can dequantize the elements of our tensor:
-// ```c++
-// result = input * s
-// ```
+//	min_x: The float value that the lowest quantized `x` value represents.
+//	max_x: The float value that the highest quantized `x` value represents.
+//	min_y: The float value that the lowest quantized `y` value represents.
+//	max_y: The float value that the highest quantized `y` value represents.
 //
-// Arguments:
+// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
 //
-//	min_range: The minimum scalar value possibly produced for the input.
-//	max_range: The maximum scalar value possibly produced for the input.
-func Dequantize(scope *Scope, input tf.Output, min_range tf.Output, max_range tf.Output, optional ...DequantizeAttr) (output tf.Output) {
+// *NOTE*: `QuantizedAdd` supports limited forms of broadcasting. More about
+// broadcasting [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func QuantizedAdd(scope *Scope, x tf.Output, y tf.Output, min_x tf.Output, max_x tf.Output, min_y tf.Output, max_y tf.Output, optional ...QuantizedAddAttr) (z tf.Output, min_z tf.Output, max_z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -27917,165 +27583,237 @@ func Dequantize(scope *Scope, input tf.Output, min_range tf.Output, max_range tf
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Dequantize",
+		Type: "QuantizedAdd",
 		Input: []tf.Input{
-			input, min_range, max_range,
+			x, y, min_x, max_x, min_y, max_y,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Flips all bits elementwise.
+// Given a quantized tensor described by (input, input_min, input_max), outputs a
 //
-// The result will have exactly those bits set, that are not set in `x`. The
-// computation is performed on the underlying representation of x.
-func Invert(scope *Scope, x tf.Output) (y tf.Output) {
+// range that covers the actual values present in that tensor.  This op is
+// typically used to produce the requested_output_min and requested_output_max for
+// Requantize.
+//
+// Arguments:
+//
+//	input_min: The float value that the minimum quantized input value represents.
+//	input_max: The float value that the maximum quantized input value represents.
+//
+// Returns The computed min output.the computed max output.
+func RequantizationRange(scope *Scope, input tf.Output, input_min tf.Output, input_max tf.Output) (output_min tf.Output, output_max tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Invert",
+		Type: "RequantizationRange",
 		Input: []tf.Input{
-			x,
+			input, input_min, input_max,
 		},
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1)
 }
 
-// Deserialize bucket boundaries and ready flag into current QuantileAccumulator.
+// Rolls the elements of a tensor along an axis.
 //
-// An op that deserializes bucket boundaries and are boundaries ready flag into current QuantileAccumulator.
+// The elements are shifted positively (towards larger indices) by the offset of
+// `shift` along the dimension of `axis`. Negative `shift` values will shift
+// elements in the opposite direction. Elements that roll passed the last position
+// will wrap around to the first and vice versa. Multiple shifts along multiple
+// axes may be specified.
+//
+// For example:
+//
+// ```
+// # 't' is [0, 1, 2, 3, 4]
+// roll(t, shift=2, axis=0) ==> [3, 4, 0, 1, 2]
+//
+// # shifting along multiple dimensions
+// # 't' is [[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]]
+// roll(t, shift=[1, -2], axis=[0, 1]) ==> [[7, 8, 9, 5, 6], [2, 3, 4, 0, 1]]
+//
+// # shifting along the same axis multiple times
+// # 't' is [[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]]
+// roll(t, shift=[2, -3], axis=[1, 1]) ==> [[1, 2, 3, 4, 0], [6, 7, 8, 9, 5]]
+// ```
 //
 // Arguments:
-//	quantile_stream_resource_handle: resource handle referring to a QuantileStreamResource.
-//	bucket_boundaries: float; List of Rank 1 Tensors each containing the bucket boundaries for a feature.
 //
-// Returns the created operation.
-func BoostedTreesQuantileStreamResourceDeserialize(scope *Scope, quantile_stream_resource_handle tf.Output, bucket_boundaries []tf.Output) (o *tf.Operation) {
+//	shift: Dimension must be 0-D or 1-D. `shift[i]` specifies the number of places by which
+// elements are shifted positively (towards larger indices) along the dimension
+// specified by `axis[i]`. Negative shifts will roll the elements in the opposite
+// direction.
+//	axis: Dimension must be 0-D or 1-D. `axis[i]` specifies the dimension that the shift
+// `shift[i]` should occur. If the same axis is referenced more than once, the
+// total shift for that axis will be the sum of all the shifts that belong to that
+// axis.
+//
+// Returns Has the same shape and size as the input. The elements are shifted
+// positively (towards larger indices) by the offsets of `shift` along the
+// dimensions of `axis`.
+func Roll(scope *Scope, input tf.Output, shift tf.Output, axis tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "BoostedTreesQuantileStreamResourceDeserialize",
+		Type: "Roll",
 		Input: []tf.Input{
-			quantile_stream_resource_handle, tf.OutputList(bucket_boundaries),
+			input, shift, axis,
 		},
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Inverse 3D fast Fourier transform.
+// Looks up keys in a table, outputs the corresponding values.
 //
-// Computes the inverse 3-dimensional discrete Fourier transform over the
-// inner-most 3 dimensions of `input`.
+// The tensor `keys` must of the same type as the keys of the table.
+// The output `values` is of the type of the table values.
+//
+// The scalar `default_value` is the value output for keys not present in the
+// table. It must also be of the same type as the table values.
 //
 // Arguments:
-//	input: A complex64 tensor.
+//	table_handle: Handle to the table.
+//	keys: Any shape.  Keys to look up.
 //
-// Returns A complex64 tensor of the same shape as `input`. The inner-most 3
-//   dimensions of `input` are replaced with their inverse 3D Fourier transform.
 //
-// @compatibility(numpy)
-// Equivalent to np.fft.ifftn with 3 dimensions.
-// @end_compatibility
-func IFFT3D(scope *Scope, input tf.Output) (output tf.Output) {
+// Returns Same shape as `keys`.  Values found in the table, or `default_values`
+// for missing keys.
+func LookupTableFindV2(scope *Scope, table_handle tf.Output, keys tf.Output, default_value tf.Output) (values tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "IFFT3D",
+		Type: "LookupTableFindV2",
 		Input: []tf.Input{
-			input,
+			table_handle, keys, default_value,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Deprecated. Disallowed in GraphDef version >= 2.
+// Updates the table to associates keys with values.
 //
-// DEPRECATED at GraphDef version 2: Use AdjustContrastv2 instead
-func AdjustContrast(scope *Scope, images tf.Output, contrast_factor tf.Output, min_value tf.Output, max_value tf.Output) (output tf.Output) {
+// The tensor `keys` must be of the same type as the keys of the table.
+// The tensor `values` must be of the type of the table values.
+//
+// Arguments:
+//	table_handle: Handle to the table.
+//	keys: Any shape.  Keys to look up.
+//	values: Values to associate with keys.
+//
+// Returns the created operation.
+func LookupTableInsertV2(scope *Scope, table_handle tf.Output, keys tf.Output, values tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "AdjustContrast",
+		Type: "LookupTableInsertV2",
 		Input: []tf.Input{
-			images, contrast_factor, min_value, max_value,
+			table_handle, keys, values,
 		},
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// Table initializer that takes two tensors for keys and values respectively.
+// Creates a dataset that batches and pads `batch_size` elements from the input.
 //
 // Arguments:
-//	table_handle: Handle to a table which will be initialized.
-//	keys: Keys of type Tkey.
-//	values: Values of type Tval.
 //
-// Returns the created operation.
-func InitializeTableV2(scope *Scope, table_handle tf.Output, keys tf.Output, values tf.Output) (o *tf.Operation) {
+//	batch_size: A scalar representing the number of elements to accumulate in a
+// batch.
+//	padded_shapes: A list of int64 tensors representing the desired padded shapes
+// of the corresponding output components. These shapes may be partially
+// specified, using `-1` to indicate that a particular dimension should be
+// padded to the maximum size of all batch elements.
+//	padding_values: A list of scalars containing the padding value to use for
+// each of the outputs.
+//	drop_remainder: A scalar representing whether the last batch should be dropped in case its size
+// is smaller than desired.
+//
+func PaddedBatchDatasetV2(scope *Scope, input_dataset tf.Output, batch_size tf.Output, padded_shapes []tf.Output, padding_values []tf.Output, drop_remainder tf.Output, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "InitializeTableV2",
+		Type: "PaddedBatchDatasetV2",
 		Input: []tf.Input{
-			table_handle, keys, values,
+			input_dataset, batch_size, tf.OutputList(padded_shapes), tf.OutputList(padding_values), drop_remainder,
 		},
+		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// PrintAttr is an optional argument to Print.
-type PrintAttr func(optionalAttr)
-
-// PrintMessage sets the optional message attribute to value.
-//
-// value: A string, prefix of the error message.
-// If not specified, defaults to ""
-func PrintMessage(value string) PrintAttr {
-	return func(m optionalAttr) {
-		m["message"] = value
+// Returns element-wise smallest integer not less than x.
+func Ceil(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Ceil",
+		Input: []tf.Input{
+			x,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// PrintFirstN sets the optional first_n attribute to value.
+// Computes the number of elements in the given table.
 //
-// value: Only log `first_n` number of times. -1 disables logging.
-// If not specified, defaults to -1
-func PrintFirstN(value int64) PrintAttr {
-	return func(m optionalAttr) {
-		m["first_n"] = value
+// Arguments:
+//	table_handle: Handle to the table.
+//
+// Returns Scalar that contains number of elements in the table.
+func LookupTableSizeV2(scope *Scope, table_handle tf.Output) (size tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "LookupTableSizeV2",
+		Input: []tf.Input{
+			table_handle,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// PrintSummarize sets the optional summarize attribute to value.
+// ResizeBilinearGradAttr is an optional argument to ResizeBilinearGrad.
+type ResizeBilinearGradAttr func(optionalAttr)
+
+// ResizeBilinearGradAlignCorners sets the optional align_corners attribute to value.
 //
-// value: Only print this many entries of each tensor.
-// If not specified, defaults to 3
-func PrintSummarize(value int64) PrintAttr {
+// value: If true, the centers of the 4 corner pixels of the input and grad tensors are
+// aligned. Defaults to false.
+// If not specified, defaults to false
+func ResizeBilinearGradAlignCorners(value bool) ResizeBilinearGradAttr {
 	return func(m optionalAttr) {
-		m["summarize"] = value
+		m["align_corners"] = value
 	}
 }
 
-// Prints a list of tensors.
-//
-// Passes `input` through to `output` and prints `data` when evaluating.
+// Computes the gradient of bilinear interpolation.
 //
 // Arguments:
-//	input: The tensor passed to `output`
-//	data: A list of tensors to print out when op is evaluated.
+//	grads: 4-D with shape `[batch, height, width, channels]`.
+//	original_image: 4-D with shape `[batch, orig_height, orig_width, channels]`,
+// The image tensor that was resized.
 //
-// Returns = The unmodified `input` tensor
-func Print(scope *Scope, input tf.Output, data []tf.Output, optional ...PrintAttr) (output tf.Output) {
+// Returns 4-D with shape `[batch, orig_height, orig_width, channels]`.
+// Gradients with respect to the input image. Input image must have been
+// float or double.
+func ResizeBilinearGrad(scope *Scope, grads tf.Output, original_image tf.Output, optional ...ResizeBilinearGradAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -28084,9 +27822,9 @@ func Print(scope *Scope, input tf.Output, data []tf.Output, optional ...PrintAtt
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Print",
+		Type: "ResizeBilinearGrad",
 		Input: []tf.Input{
-			input, tf.OutputList(data),
+			grads, original_image,
 		},
 		Attrs: attrs,
 	}
@@ -28094,93 +27832,88 @@ func Print(scope *Scope, input tf.Output, data []tf.Output, optional ...PrintAtt
 	return op.Output(0)
 }
 
-// Outputs a `Summary` protocol buffer with a tensor and per-plugin data.
+// Outputs all keys and values in the table.
 //
 // Arguments:
-//	tag: A string attached to this summary. Used for organization in TensorBoard.
-//	tensor: A tensor to serialize.
-//	serialized_summary_metadata: A serialized SummaryMetadata proto. Contains plugin
-// data.
-func TensorSummaryV2(scope *Scope, tag tf.Output, tensor tf.Output, serialized_summary_metadata tf.Output) (summary tf.Output) {
+//	table_handle: Handle to the table.
+//
+//
+//
+// Returns Vector of all keys present in the table.Tensor of all values in the table. Indexed in parallel with `keys`.
+func LookupTableExportV2(scope *Scope, table_handle tf.Output, Tkeys tf.DataType, Tvalues tf.DataType) (keys tf.Output, values tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"Tkeys": Tkeys, "Tvalues": Tvalues}
 	opspec := tf.OpSpec{
-		Type: "TensorSummaryV2",
+		Type: "LookupTableExportV2",
 		Input: []tf.Input{
-			tag, tensor, serialized_summary_metadata,
+			table_handle,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1)
 }
 
-// Creates a dataset that asynchronously prefetches elements from `input_dataset`.
-//
-// Arguments:
+// Replaces the contents of the table with the specified keys and values.
 //
-//	buffer_size: The maximum number of elements to buffer in an iterator over
-// this dataset.
+// The tensor `keys` must be of the same type as the keys of the table.
+// The tensor `values` must be of the type of the table values.
 //
+// Arguments:
+//	table_handle: Handle to the table.
+//	keys: Any shape.  Keys to look up.
+//	values: Values to associate with keys.
 //
-func PrefetchDataset(scope *Scope, input_dataset tf.Output, buffer_size tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+// Returns the created operation.
+func LookupTableImportV2(scope *Scope, table_handle tf.Output, keys tf.Output, values tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "PrefetchDataset",
+		Type: "LookupTableImportV2",
 		Input: []tf.Input{
-			input_dataset, buffer_size,
+			table_handle, keys, values,
 		},
-		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// TensorSummaryAttr is an optional argument to TensorSummary.
-type TensorSummaryAttr func(optionalAttr)
+// MultiDeviceIteratorFromStringHandleAttr is an optional argument to MultiDeviceIteratorFromStringHandle.
+type MultiDeviceIteratorFromStringHandleAttr func(optionalAttr)
 
-// TensorSummaryDescription sets the optional description attribute to value.
+// MultiDeviceIteratorFromStringHandleOutputTypes sets the optional output_types attribute to value.
 //
-// value: A json-encoded SummaryDescription proto.
-// If not specified, defaults to ""
-func TensorSummaryDescription(value string) TensorSummaryAttr {
+// value: The type list for the return values.
+// If not specified, defaults to <>
+//
+// REQUIRES: len(value) >= 0
+func MultiDeviceIteratorFromStringHandleOutputTypes(value []tf.DataType) MultiDeviceIteratorFromStringHandleAttr {
 	return func(m optionalAttr) {
-		m["description"] = value
+		m["output_types"] = value
 	}
 }
 
-// TensorSummaryLabels sets the optional labels attribute to value.
+// MultiDeviceIteratorFromStringHandleOutputShapes sets the optional output_shapes attribute to value.
 //
-// value: An unused list of strings.
+// value: The list of shapes being produced.
 // If not specified, defaults to <>
-func TensorSummaryLabels(value []string) TensorSummaryAttr {
-	return func(m optionalAttr) {
-		m["labels"] = value
-	}
-}
-
-// TensorSummaryDisplayName sets the optional display_name attribute to value.
 //
-// value: An unused string.
-// If not specified, defaults to ""
-func TensorSummaryDisplayName(value string) TensorSummaryAttr {
+// REQUIRES: len(value) >= 0
+func MultiDeviceIteratorFromStringHandleOutputShapes(value []tf.Shape) MultiDeviceIteratorFromStringHandleAttr {
 	return func(m optionalAttr) {
-		m["display_name"] = value
+		m["output_shapes"] = value
 	}
 }
 
-// Outputs a `Summary` protocol buffer with a tensor.
-//
-// This op is being phased out in favor of TensorSummaryV2, which lets callers pass
-// a tag as well as a serialized SummaryMetadata proto string that contains
-// plugin-specific data. We will keep this op to maintain backwards compatibility.
+// Generates a MultiDeviceIterator resource from its provided string handle.
 //
 // Arguments:
-//	tensor: A tensor to serialize.
-func TensorSummary(scope *Scope, tensor tf.Output, optional ...TensorSummaryAttr) (summary tf.Output) {
+//	string_handle: String representing the resource.
+//
+// Returns A MultiDeviceIterator resource.
+func MultiDeviceIteratorFromStringHandle(scope *Scope, string_handle tf.Output, optional ...MultiDeviceIteratorFromStringHandleAttr) (multi_device_iterator tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -28189,9 +27922,9 @@ func TensorSummary(scope *Scope, tensor tf.Output, optional ...TensorSummaryAttr
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "TensorSummary",
+		Type: "MultiDeviceIteratorFromStringHandle",
 		Input: []tf.Input{
-			tensor,
+			string_handle,
 		},
 		Attrs: attrs,
 	}
@@ -28199,229 +27932,329 @@ func TensorSummary(scope *Scope, tensor tf.Output, optional ...TensorSummaryAttr
 	return op.Output(0)
 }
 
-// Read an element from the TensorArray into output `value`.
+// MutableHashTableV2Attr is an optional argument to MutableHashTableV2.
+type MutableHashTableV2Attr func(optionalAttr)
+
+// MutableHashTableV2Container sets the optional container attribute to value.
 //
-// Arguments:
-//	handle: The handle to a TensorArray.
+// value: If non-empty, this table is placed in the given container.
+// Otherwise, a default container is used.
+// If not specified, defaults to ""
+func MutableHashTableV2Container(value string) MutableHashTableV2Attr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// MutableHashTableV2SharedName sets the optional shared_name attribute to value.
 //
-//	flow_in: A float scalar that enforces proper chaining of operations.
-//	dtype: The type of the elem that is returned.
+// value: If non-empty, this table is shared under the given name across
+// multiple sessions.
+// If not specified, defaults to ""
+func MutableHashTableV2SharedName(value string) MutableHashTableV2Attr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// MutableHashTableV2UseNodeNameSharing sets the optional use_node_name_sharing attribute to value.
 //
-// Returns The tensor that is read from the TensorArray.
-func TensorArrayReadV3(scope *Scope, handle tf.Output, index tf.Output, flow_in tf.Output, dtype tf.DataType) (value tf.Output) {
+// value: If true and shared_name is empty, the table is shared
+// using the node name.
+// If not specified, defaults to false
+func MutableHashTableV2UseNodeNameSharing(value bool) MutableHashTableV2Attr {
+	return func(m optionalAttr) {
+		m["use_node_name_sharing"] = value
+	}
+}
+
+// Creates an empty hash table.
+//
+// This op creates a mutable hash table, specifying the type of its keys and
+// values. Each value must be a scalar. Data can be inserted into the table using
+// the insert operations. It does not support the initialization operation.
+//
+// Arguments:
+//	key_dtype: Type of the table keys.
+//	value_dtype: Type of the table values.
+//
+// Returns Handle to a table.
+func MutableHashTableV2(scope *Scope, key_dtype tf.DataType, value_dtype tf.DataType, optional ...MutableHashTableV2Attr) (table_handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
+	attrs := map[string]interface{}{"key_dtype": key_dtype, "value_dtype": value_dtype}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "TensorArrayReadV3",
-		Input: []tf.Input{
-			handle, index, flow_in,
-		},
+		Type: "MutableHashTableV2",
+
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Reduces sparse updates into the variable referenced by `resource` using the `max` operation.
+// DequantizeAttr is an optional argument to Dequantize.
+type DequantizeAttr func(optionalAttr)
+
+// DequantizeMode sets the optional mode attribute to value.
+// If not specified, defaults to "MIN_COMBINED"
+func DequantizeMode(value string) DequantizeAttr {
+	return func(m optionalAttr) {
+		m["mode"] = value
+	}
+}
+
+// Dequantize the 'input' tensor into a float Tensor.
 //
-// This operation computes
+// [min_range, max_range] are scalar floats that specify the range for
+// the 'input' data. The 'mode' attribute controls exactly which calculations are
+// used to convert the float values to their quantized equivalents.
 //
-//     # Scalar indices
-//     ref[indices, ...] = max(ref[indices, ...], updates[...])
+// In 'MIN_COMBINED' mode, each value of the tensor will undergo the following:
 //
-//     # Vector indices (for each i)
-//     ref[indices[i], ...] = max(ref[indices[i], ...], updates[i, ...])
+// ```
+// if T == qint8: in[i] += (range(T) + 1)/ 2.0
+// out[i] = min_range + (in[i]* (max_range - min_range) / range(T))
+// ```
+// here `range(T) = numeric_limits<T>::max() - numeric_limits<T>::min()`
 //
-//     # High rank indices (for each i, ..., j)
-//     ref[indices[i, ..., j], ...] = max(ref[indices[i, ..., j], ...], updates[i, ..., j, ...])
+// *MIN_COMBINED Mode Example*
 //
-// Duplicate entries are handled correctly: if multiple `indices` reference
-// the same location, their contributions are combined.
+// If the input comes from a QuantizedRelu6, the output type is
+// quint8 (range of 0-255) but the possible range of QuantizedRelu6 is
+// 0-6.  The min_range and max_range values are therefore 0.0 and 6.0.
+// Dequantize on quint8 will take each value, cast to float, and multiply
+// by 6 / 255.
+// Note that if quantizedtype is qint8, the operation will additionally add
+// each value by 128 prior to casting.
 //
-// Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`.
+// If the mode is 'MIN_FIRST', then this approach is used:
 //
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src='https://www.tensorflow.org/images/ScatterAdd.png' alt>
-// </div>
+// ```c++
+// num_discrete_values = 1 << (# of bits in T)
+// range_adjust = num_discrete_values / (num_discrete_values - 1)
+// range = (range_max - range_min) * range_adjust
+// range_scale = range / num_discrete_values
+// const double offset_input = static_cast<double>(input) - lowest_quantized;
+// result = range_min + ((input - numeric_limits<T>::min()) * range_scale)
+// ```
+//
+// *SCALED mode Example*
+//
+// `SCALED` mode matches the quantization approach used in
+// `QuantizeAndDequantize{V2|V3}`.
+//
+// If the mode is `SCALED`, we do not use the full range of the output type,
+// choosing to elide the lowest possible value for symmetry (e.g., output range is
+// -127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to
+// 0.
+//
+// We first find the range of values in our tensor. The
+// range we use is always centered on 0, so we find m such that
+// ```c++
+//   m = max(abs(input_min), abs(input_max))
+// ```
+//
+// Our input tensor range is then `[-m, m]`.
+//
+// Next, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.
+// If T is signed, this is
+// ```
+//   num_bits = sizeof(T) * 8
+//   [min_fixed, max_fixed] =
+//       [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]
+// ```
+//
+// Otherwise, if T is unsigned, the fixed-point range is
+// ```
+//   [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]
+// ```
+//
+// From this we compute our scaling factor, s:
+// ```c++
+//   s = (2 * m) / (max_fixed - min_fixed)
+// ```
+//
+// Now we can dequantize the elements of our tensor:
+// ```c++
+// result = input * s
+// ```
 //
 // Arguments:
-//	resource: Should be from a `Variable` node.
-//	indices: A tensor of indices into the first dimension of `ref`.
-//	updates: A tensor of updated values to add to `ref`.
 //
-// Returns the created operation.
-func ResourceScatterMax(scope *Scope, resource tf.Output, indices tf.Output, updates tf.Output) (o *tf.Operation) {
+//	min_range: The minimum scalar value possibly produced for the input.
+//	max_range: The maximum scalar value possibly produced for the input.
+func Dequantize(scope *Scope, input tf.Output, min_range tf.Output, max_range tf.Output, optional ...DequantizeAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "ResourceScatterMax",
+		Type: "Dequantize",
 		Input: []tf.Input{
-			resource, indices, updates,
+			input, min_range, max_range,
 		},
+		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Computes the gradient for the tanh of `x` wrt its input.
+// Flips all bits elementwise.
 //
-// Specifically, `grad = dy * (1 - y*y)`, where `y = tanh(x)`, and `dy`
-// is the corresponding input gradient.
-func TanhGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
+// The result will have exactly those bits set, that are not set in `x`. The
+// computation is performed on the underlying representation of x.
+func Invert(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "TanhGrad",
+		Type: "Invert",
 		Input: []tf.Input{
-			y, dy,
+			x,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Outputs a `Summary` protocol buffer with scalar values.
+// Deserialize bucket boundaries and ready flag into current QuantileAccumulator.
 //
-// The input `tags` and `values` must have the same shape.  The generated summary
-// has a summary value for each tag-value pair in `tags` and `values`.
+// An op that deserializes bucket boundaries and are boundaries ready flag into current QuantileAccumulator.
 //
 // Arguments:
-//	tags: Tags for the summary.
-//	values: Same shape as `tags.  Values for the summary.
+//	quantile_stream_resource_handle: resource handle referring to a QuantileStreamResource.
+//	bucket_boundaries: float; List of Rank 1 Tensors each containing the bucket boundaries for a feature.
 //
-// Returns Scalar.  Serialized `Summary` protocol buffer.
-func ScalarSummary(scope *Scope, tags tf.Output, values tf.Output) (summary tf.Output) {
+// Returns the created operation.
+func BoostedTreesQuantileStreamResourceDeserialize(scope *Scope, quantile_stream_resource_handle tf.Output, bucket_boundaries []tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "ScalarSummary",
+		Type: "BoostedTreesQuantileStreamResourceDeserialize",
 		Input: []tf.Input{
-			tags, values,
+			quantile_stream_resource_handle, tf.OutputList(bucket_boundaries),
 		},
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// Outputs a `Summary` protocol buffer with a histogram.
-//
-// The generated
-// [`Summary`](https://www.tensorflow.org/code/tensorflow/core/framework/summary.proto)
-// has one summary value containing a histogram for `values`.
+// Inverse 3D fast Fourier transform.
 //
-// This op reports an `InvalidArgument` error if any value is not finite.
+// Computes the inverse 3-dimensional discrete Fourier transform over the
+// inner-most 3 dimensions of `input`.
 //
 // Arguments:
-//	tag: Scalar.  Tag to use for the `Summary.Value`.
-//	values: Any shape. Values to use to build the histogram.
+//	input: A complex64 tensor.
 //
-// Returns Scalar. Serialized `Summary` protocol buffer.
-func HistogramSummary(scope *Scope, tag tf.Output, values tf.Output) (summary tf.Output) {
+// Returns A complex64 tensor of the same shape as `input`. The inner-most 3
+//   dimensions of `input` are replaced with their inverse 3D Fourier transform.
+//
+// @compatibility(numpy)
+// Equivalent to np.fft.ifftn with 3 dimensions.
+// @end_compatibility
+func IFFT3D(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "HistogramSummary",
+		Type: "IFFT3D",
 		Input: []tf.Input{
-			tag, values,
+			input,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes the number of elements in the given queue.
-//
-// Arguments:
-//	handle: The handle to a queue.
+// Deprecated. Disallowed in GraphDef version >= 2.
 //
-// Returns The number of elements in the given queue.
-func QueueSizeV2(scope *Scope, handle tf.Output) (size tf.Output) {
+// DEPRECATED at GraphDef version 2: Use AdjustContrastv2 instead
+func AdjustContrast(scope *Scope, images tf.Output, contrast_factor tf.Output, min_value tf.Output, max_value tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "QueueSizeV2",
+		Type: "AdjustContrast",
 		Input: []tf.Input{
-			handle,
+			images, contrast_factor, min_value, max_value,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ImageSummaryAttr is an optional argument to ImageSummary.
-type ImageSummaryAttr func(optionalAttr)
-
-// ImageSummaryMaxImages sets the optional max_images attribute to value.
+// Table initializer that takes two tensors for keys and values respectively.
 //
-// value: Max number of batch elements to generate images for.
-// If not specified, defaults to 3
+// Arguments:
+//	table_handle: Handle to a table which will be initialized.
+//	keys: Keys of type Tkey.
+//	values: Values of type Tval.
 //
-// REQUIRES: value >= 1
-func ImageSummaryMaxImages(value int64) ImageSummaryAttr {
-	return func(m optionalAttr) {
-		m["max_images"] = value
+// Returns the created operation.
+func InitializeTableV2(scope *Scope, table_handle tf.Output, keys tf.Output, values tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "InitializeTableV2",
+		Input: []tf.Input{
+			table_handle, keys, values,
+		},
 	}
+	return scope.AddOperation(opspec)
 }
 
-// ImageSummaryBadColor sets the optional bad_color attribute to value.
+// PrintAttr is an optional argument to Print.
+type PrintAttr func(optionalAttr)
+
+// PrintMessage sets the optional message attribute to value.
 //
-// value: Color to use for pixels with non-finite values.
-// If not specified, defaults to <dtype:DT_UINT8 tensor_shape:<dim:<size:4 > > int_val:255 int_val:0 int_val:0 int_val:255 >
-func ImageSummaryBadColor(value tf.Tensor) ImageSummaryAttr {
+// value: A string, prefix of the error message.
+// If not specified, defaults to ""
+func PrintMessage(value string) PrintAttr {
 	return func(m optionalAttr) {
-		m["bad_color"] = value
+		m["message"] = value
 	}
 }
 
-// Outputs a `Summary` protocol buffer with images.
-//
-// The summary has up to `max_images` summary values containing images. The
-// images are built from `tensor` which must be 4-D with shape `[batch_size,
-// height, width, channels]` and where `channels` can be:
-//
-// *  1: `tensor` is interpreted as Grayscale.
-// *  3: `tensor` is interpreted as RGB.
-// *  4: `tensor` is interpreted as RGBA.
-//
-// The images have the same number of channels as the input tensor. For float
-// input, the values are normalized one image at a time to fit in the range
-// `[0, 255]`.  `uint8` values are unchanged.  The op uses two different
-// normalization algorithms:
-//
-// *  If the input values are all positive, they are rescaled so the largest one
-//    is 255.
-//
-// *  If any input value is negative, the values are shifted so input value 0.0
-//    is at 127.  They are then rescaled so that either the smallest value is 0,
-//    or the largest one is 255.
+// PrintFirstN sets the optional first_n attribute to value.
 //
-// The `tag` argument is a scalar `Tensor` of type `string`.  It is used to
-// build the `tag` of the summary values:
+// value: Only log `first_n` number of times. -1 disables logging.
+// If not specified, defaults to -1
+func PrintFirstN(value int64) PrintAttr {
+	return func(m optionalAttr) {
+		m["first_n"] = value
+	}
+}
+
+// PrintSummarize sets the optional summarize attribute to value.
 //
-// *  If `max_images` is 1, the summary value tag is '*tag*/image'.
-// *  If `max_images` is greater than 1, the summary value tags are
-//    generated sequentially as '*tag*/image/0', '*tag*/image/1', etc.
+// value: Only print this many entries of each tensor.
+// If not specified, defaults to 3
+func PrintSummarize(value int64) PrintAttr {
+	return func(m optionalAttr) {
+		m["summarize"] = value
+	}
+}
+
+// Prints a list of tensors.
 //
-// The `bad_color` argument is the color to use in the generated images for
-// non-finite input values.  It is a `uint8` 1-D tensor of length `channels`.
-// Each element must be in the range `[0, 255]` (It represents the value of a
-// pixel in the output image).  Non-finite values in the input tensor are
-// replaced by this tensor in the output image.  The default value is the color
-// red.
+// Passes `input` through to `output` and prints `data` when evaluating.
 //
 // Arguments:
-//	tag: Scalar. Used to build the `tag` attribute of the summary values.
-//	tensor: 4-D of shape `[batch_size, height, width, channels]` where
-// `channels` is 1, 3, or 4.
+//	input: The tensor passed to `output`
+//	data: A list of tensors to print out when op is evaluated.
 //
-// Returns Scalar. Serialized `Summary` protocol buffer.
-func ImageSummary(scope *Scope, tag tf.Output, tensor tf.Output, optional ...ImageSummaryAttr) (summary tf.Output) {
+// Returns = The unmodified `input` tensor
+func Print(scope *Scope, input tf.Output, data []tf.Output, optional ...PrintAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -28430,9 +28263,9 @@ func ImageSummary(scope *Scope, tag tf.Output, tensor tf.Output, optional ...Ima
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ImageSummary",
+		Type: "Print",
 		Input: []tf.Input{
-			tag, tensor,
+			input, tf.OutputList(data),
 		},
 		Attrs: attrs,
 	}
@@ -28440,42 +28273,40 @@ func ImageSummary(scope *Scope, tag tf.Output, tensor tf.Output, optional ...Ima
 	return op.Output(0)
 }
 
-// AudioSummaryV2Attr is an optional argument to AudioSummaryV2.
-type AudioSummaryV2Attr func(optionalAttr)
+// QueueEnqueueManyV2Attr is an optional argument to QueueEnqueueManyV2.
+type QueueEnqueueManyV2Attr func(optionalAttr)
 
-// AudioSummaryV2MaxOutputs sets the optional max_outputs attribute to value.
-//
-// value: Max number of batch elements to generate audio for.
-// If not specified, defaults to 3
+// QueueEnqueueManyV2TimeoutMs sets the optional timeout_ms attribute to value.
 //
-// REQUIRES: value >= 1
-func AudioSummaryV2MaxOutputs(value int64) AudioSummaryV2Attr {
+// value: If the queue is too full, this operation will block for up
+// to timeout_ms milliseconds.
+// Note: This option is not supported yet.
+// If not specified, defaults to -1
+func QueueEnqueueManyV2TimeoutMs(value int64) QueueEnqueueManyV2Attr {
 	return func(m optionalAttr) {
-		m["max_outputs"] = value
+		m["timeout_ms"] = value
 	}
 }
 
-// Outputs a `Summary` protocol buffer with audio.
+// Enqueues zero or more tuples of one or more tensors in the given queue.
 //
-// The summary has up to `max_outputs` summary values containing audio. The
-// audio is built from `tensor` which must be 3-D with shape `[batch_size,
-// frames, channels]` or 2-D with shape `[batch_size, frames]`. The values are
-// assumed to be in the range of `[-1.0, 1.0]` with a sample rate of `sample_rate`.
+// This operation slices each component tensor along the 0th dimension to
+// make multiple queue elements. All of the tuple components must have the
+// same size in the 0th dimension.
 //
-// The `tag` argument is a scalar `Tensor` of type `string`.  It is used to
-// build the `tag` of the summary values:
+// The components input has k elements, which correspond to the components of
+// tuples stored in the given queue.
 //
-// *  If `max_outputs` is 1, the summary value tag is '*tag*/audio'.
-// *  If `max_outputs` is greater than 1, the summary value tags are
-//    generated sequentially as '*tag*/audio/0', '*tag*/audio/1', etc.
+// N.B. If the queue is full, this operation will block until the given
+// elements have been enqueued (or 'timeout_ms' elapses, if specified).
 //
 // Arguments:
-//	tag: Scalar. Used to build the `tag` attribute of the summary values.
-//	tensor: 2-D of shape `[batch_size, frames]`.
-//	sample_rate: The sample rate of the signal in hertz.
+//	handle: The handle to a queue.
+//	components: One or more tensors from which the enqueued tensors should
+// be taken.
 //
-// Returns Scalar. Serialized `Summary` protocol buffer.
-func AudioSummaryV2(scope *Scope, tag tf.Output, tensor tf.Output, sample_rate tf.Output, optional ...AudioSummaryV2Attr) (summary tf.Output) {
+// Returns the created operation.
+func QueueEnqueueManyV2(scope *Scope, handle tf.Output, components []tf.Output, optional ...QueueEnqueueManyV2Attr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -28484,152 +28315,152 @@ func AudioSummaryV2(scope *Scope, tag tf.Output, tensor tf.Output, sample_rate t
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "AudioSummaryV2",
+		Type: "QueueEnqueueManyV2",
 		Input: []tf.Input{
-			tag, tensor, sample_rate,
+			handle, tf.OutputList(components),
 		},
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// Splits a tensor into a list.
+// PrintV2Attr is an optional argument to PrintV2.
+type PrintV2Attr func(optionalAttr)
+
+// PrintV2OutputStream sets the optional output_stream attribute to value.
 //
-// list[i] corresponds to lengths[i] tensors from the input tensor.
-// The tensor must have rank at least 1 and contain exactly sum(lengths) elements.
+// value: A string specifying the output stream or logging level to print to.
+// If not specified, defaults to "stderr"
+func PrintV2OutputStream(value string) PrintV2Attr {
+	return func(m optionalAttr) {
+		m["output_stream"] = value
+	}
+}
+
+// Prints a string scalar.
 //
-// tensor: The input tensor.
-// element_shape: A shape compatible with that of elements in the tensor.
-// lengths: Vector of sizes of the 0th dimension of tensors in the list.
-// output_handle: The list.
-func TensorListSplit(scope *Scope, tensor tf.Output, element_shape tf.Output, lengths tf.Output) (output_handle tf.Output) {
+// Prints a string scalar to the desired output_stream.
+//
+// Arguments:
+//	input: The string scalar to print.
+//
+// Returns the created operation.
+func PrintV2(scope *Scope, input tf.Output, optional ...PrintV2Attr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "TensorListSplit",
+		Type: "PrintV2",
 		Input: []tf.Input{
-			tensor, element_shape, lengths,
+			input,
 		},
+		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// AvgPoolAttr is an optional argument to AvgPool.
-type AvgPoolAttr func(optionalAttr)
-
-// AvgPoolDataFormat sets the optional data_format attribute to value.
-//
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the data is stored in the order of:
-//     [batch, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, in_channels, in_height, in_width].
-// If not specified, defaults to "NHWC"
-func AvgPoolDataFormat(value string) AvgPoolAttr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
-	}
-}
-
-// Performs average pooling on the input.
-//
-// Each entry in `output` is the mean of the corresponding size `ksize`
-// window in `value`.
+// Outputs a `Summary` protocol buffer with a tensor and per-plugin data.
 //
 // Arguments:
-//	value: 4-D with shape `[batch, height, width, channels]`.
-//	ksize: The size of the sliding window for each dimension of `value`.
-//	strides: The stride of the sliding window for each dimension of `value`.
-//	padding: The type of padding algorithm to use.
-//
-// Returns The average pooled output tensor.
-func AvgPool(scope *Scope, value tf.Output, ksize []int64, strides []int64, padding string, optional ...AvgPoolAttr) (output tf.Output) {
+//	tag: A string attached to this summary. Used for organization in TensorBoard.
+//	tensor: A tensor to serialize.
+//	serialized_summary_metadata: A serialized SummaryMetadata proto. Contains plugin
+// data.
+func TensorSummaryV2(scope *Scope, tag tf.Output, tensor tf.Output, serialized_summary_metadata tf.Output) (summary tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "AvgPool",
+		Type: "TensorSummaryV2",
 		Input: []tf.Input{
-			value,
+			tag, tensor, serialized_summary_metadata,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Merges summaries.
+// Creates a dataset that asynchronously prefetches elements from `input_dataset`.
 //
-// This op creates a
-// [`Summary`](https://www.tensorflow.org/code/tensorflow/core/framework/summary.proto)
-// protocol buffer that contains the union of all the values in the input
-// summaries.
+// Arguments:
 //
-// When the Op is run, it reports an `InvalidArgument` error if multiple values
-// in the summaries to merge use the same tag.
+//	buffer_size: The maximum number of elements to buffer in an iterator over
+// this dataset.
 //
-// Arguments:
-//	inputs: Can be of any shape.  Each must contain serialized `Summary` protocol
-// buffers.
 //
-// Returns Scalar. Serialized `Summary` protocol buffer.
-func MergeSummary(scope *Scope, inputs []tf.Output) (summary tf.Output) {
+func PrefetchDataset(scope *Scope, input_dataset tf.Output, buffer_size tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "MergeSummary",
+		Type: "PrefetchDataset",
 		Input: []tf.Input{
-			tf.OutputList(inputs),
+			input_dataset, buffer_size,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// The shape of the elements of the given list, as a tensor.
+// TensorSummaryAttr is an optional argument to TensorSummary.
+type TensorSummaryAttr func(optionalAttr)
+
+// TensorSummaryDescription sets the optional description attribute to value.
 //
-//   input_handle: the list
-//   element_shape: the shape of elements of the list
-func TensorListElementShape(scope *Scope, input_handle tf.Output, shape_type tf.DataType) (element_shape tf.Output) {
-	if scope.Err() != nil {
-		return
+// value: A json-encoded SummaryDescription proto.
+// If not specified, defaults to ""
+func TensorSummaryDescription(value string) TensorSummaryAttr {
+	return func(m optionalAttr) {
+		m["description"] = value
 	}
-	attrs := map[string]interface{}{"shape_type": shape_type}
-	opspec := tf.OpSpec{
-		Type: "TensorListElementShape",
-		Input: []tf.Input{
-			input_handle,
-		},
-		Attrs: attrs,
+}
+
+// TensorSummaryLabels sets the optional labels attribute to value.
+//
+// value: An unused list of strings.
+// If not specified, defaults to <>
+func TensorSummaryLabels(value []string) TensorSummaryAttr {
+	return func(m optionalAttr) {
+		m["labels"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Returns the item in the list with the given index.
+// TensorSummaryDisplayName sets the optional display_name attribute to value.
 //
-// input_handle: the list
-// index: the position in the list from which an element will be retrieved
-// item: the element at that position
+// value: An unused string.
+// If not specified, defaults to ""
+func TensorSummaryDisplayName(value string) TensorSummaryAttr {
+	return func(m optionalAttr) {
+		m["display_name"] = value
+	}
+}
+
+// Outputs a `Summary` protocol buffer with a tensor.
 //
+// This op is being phased out in favor of TensorSummaryV2, which lets callers pass
+// a tag as well as a serialized SummaryMetadata proto string that contains
+// plugin-specific data. We will keep this op to maintain backwards compatibility.
 //
-func TensorListGetItem(scope *Scope, input_handle tf.Output, index tf.Output, element_shape tf.Output, element_dtype tf.DataType) (item tf.Output) {
+// Arguments:
+//	tensor: A tensor to serialize.
+func TensorSummary(scope *Scope, tensor tf.Output, optional ...TensorSummaryAttr) (summary tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"element_dtype": element_dtype}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "TensorListGetItem",
+		Type: "TensorSummary",
 		Input: []tf.Input{
-			input_handle, index, element_shape,
+			tensor,
 		},
 		Attrs: attrs,
 	}
@@ -28637,411 +28468,365 @@ func TensorListGetItem(scope *Scope, input_handle tf.Output, index tf.Output, el
 	return op.Output(0)
 }
 
-// Resizes the list.
+// Read an element from the TensorArray into output `value`.
 //
+// Arguments:
+//	handle: The handle to a TensorArray.
 //
-// input_handle: the input list
-// size: size of the output list
+//	flow_in: A float scalar that enforces proper chaining of operations.
+//	dtype: The type of the elem that is returned.
 //
-func TensorListResize(scope *Scope, input_handle tf.Output, size tf.Output) (output_handle tf.Output) {
+// Returns The tensor that is read from the TensorArray.
+func TensorArrayReadV3(scope *Scope, handle tf.Output, index tf.Output, flow_in tf.Output, dtype tf.DataType) (value tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"dtype": dtype}
 	opspec := tf.OpSpec{
-		Type: "TensorListResize",
+		Type: "TensorArrayReadV3",
 		Input: []tf.Input{
-			input_handle, size,
+			handle, index, flow_in,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Returns a diagonal tensor with a given diagonal values.
+// Reduces sparse updates into the variable referenced by `resource` using the `max` operation.
 //
-// Given a `diagonal`, this operation returns a tensor with the `diagonal` and
-// everything else padded with zeros. The diagonal is computed as follows:
+// This operation computes
 //
-// Assume `diagonal` has dimensions [D1,..., Dk], then the output is a tensor of
-// rank 2k with dimensions [D1,..., Dk, D1,..., Dk] where:
+//     # Scalar indices
+//     ref[indices, ...] = max(ref[indices, ...], updates[...])
 //
-// `output[i1,..., ik, i1,..., ik] = diagonal[i1, ..., ik]` and 0 everywhere else.
+//     # Vector indices (for each i)
+//     ref[indices[i], ...] = max(ref[indices[i], ...], updates[i, ...])
 //
-// For example:
+//     # High rank indices (for each i, ..., j)
+//     ref[indices[i, ..., j], ...] = max(ref[indices[i, ..., j], ...], updates[i, ..., j, ...])
 //
-// ```
-// # 'diagonal' is [1, 2, 3, 4]
-// tf.diag(diagonal) ==> [[1, 0, 0, 0]
-//                        [0, 2, 0, 0]
-//                        [0, 0, 3, 0]
-//                        [0, 0, 0, 4]]
-// ```
+// Duplicate entries are handled correctly: if multiple `indices` reference
+// the same location, their contributions are combined.
+//
+// Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src='https://www.tensorflow.org/images/ScatterAdd.png' alt>
+// </div>
 //
 // Arguments:
-//	diagonal: Rank k tensor where k is at most 1.
-func Diag(scope *Scope, diagonal tf.Output) (output tf.Output) {
+//	resource: Should be from a `Variable` node.
+//	indices: A tensor of indices into the first dimension of `ref`.
+//	updates: A tensor of updated values to add to `ref`.
+//
+// Returns the created operation.
+func ResourceScatterMax(scope *Scope, resource tf.Output, indices tf.Output, updates tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Diag",
+		Type: "ResourceScatterMax",
 		Input: []tf.Input{
-			diagonal,
+			resource, indices, updates,
 		},
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// ParameterizedTruncatedNormalAttr is an optional argument to ParameterizedTruncatedNormal.
-type ParameterizedTruncatedNormalAttr func(optionalAttr)
-
-// ParameterizedTruncatedNormalSeed sets the optional seed attribute to value.
-//
-// value: If either `seed` or `seed2` are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func ParameterizedTruncatedNormalSeed(value int64) ParameterizedTruncatedNormalAttr {
-	return func(m optionalAttr) {
-		m["seed"] = value
-	}
-}
-
-// ParameterizedTruncatedNormalSeed2 sets the optional seed2 attribute to value.
-//
-// value: A second seed to avoid seed collision.
-// If not specified, defaults to 0
-func ParameterizedTruncatedNormalSeed2(value int64) ParameterizedTruncatedNormalAttr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
-	}
+	return scope.AddOperation(opspec)
 }
 
-// Outputs random values from a normal distribution. The parameters may each be a
-//
-// scalar which applies to the entire output, or a vector of length shape[0] which
-// stores the parameters for each batch.
-//
-// Arguments:
-//	shape: The shape of the output tensor. Batches are indexed by the 0th dimension.
-//	means: The mean parameter of each batch.
-//	stdevs: The standard deviation parameter of each batch. Must be greater than 0.
-//	minvals: The minimum cutoff. May be -infinity.
-//	maxvals: The maximum cutoff. May be +infinity, and must be more than the minval
-// for each batch.
+// Computes the gradient for the tanh of `x` wrt its input.
 //
-// Returns A matrix of shape num_batches x samples_per_batch, filled with random
-// truncated normal values using the parameters for each row.
-func ParameterizedTruncatedNormal(scope *Scope, shape tf.Output, means tf.Output, stdevs tf.Output, minvals tf.Output, maxvals tf.Output, optional ...ParameterizedTruncatedNormalAttr) (output tf.Output) {
+// Specifically, `grad = dy * (1 - y*y)`, where `y = tanh(x)`, and `dy`
+// is the corresponding input gradient.
+func TanhGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "ParameterizedTruncatedNormal",
+		Type: "TanhGrad",
 		Input: []tf.Input{
-			shape, means, stdevs, minvals, maxvals,
+			y, dy,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Sets the index-th position of the list to contain the given tensor.
+// Outputs a `Summary` protocol buffer with scalar values.
 //
-// input_handle: the list
-// index: the position in the list to which the tensor will be assigned
-// item: the element to be assigned to that position
-// output_handle: the new list, with the element in the proper position
+// The input `tags` and `values` must have the same shape.  The generated summary
+// has a summary value for each tag-value pair in `tags` and `values`.
 //
-func TensorListSetItem(scope *Scope, input_handle tf.Output, index tf.Output, item tf.Output) (output_handle tf.Output) {
+// Arguments:
+//	tags: Tags for the summary.
+//	values: Same shape as `tags.  Values for the summary.
+//
+// Returns Scalar.  Serialized `Summary` protocol buffer.
+func ScalarSummary(scope *Scope, tags tf.Output, values tf.Output) (summary tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "TensorListSetItem",
+		Type: "ScalarSummary",
 		Input: []tf.Input{
-			input_handle, index, item,
+			tags, values,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Creates a Tensor by indexing into the TensorList.
-//
-// Each row in the produced Tensor corresponds to the element in the TensorList
-// specified by the given index (see `tf.gather`).
+// Outputs a `Summary` protocol buffer with a histogram.
 //
-// input_handle: The input tensor list.
-// indices: The indices used to index into the list.
-// values: The tensor.
-func TensorListGather(scope *Scope, input_handle tf.Output, indices tf.Output, element_shape tf.Output, element_dtype tf.DataType) (values tf.Output) {
+// The generated
+// [`Summary`](https://www.tensorflow.org/code/tensorflow/core/framework/summary.proto)
+// has one summary value containing a histogram for `values`.
+//
+// This op reports an `InvalidArgument` error if any value is not finite.
+//
+// Arguments:
+//	tag: Scalar.  Tag to use for the `Summary.Value`.
+//	values: Any shape. Values to use to build the histogram.
+//
+// Returns Scalar. Serialized `Summary` protocol buffer.
+func HistogramSummary(scope *Scope, tag tf.Output, values tf.Output) (summary tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"element_dtype": element_dtype}
 	opspec := tf.OpSpec{
-		Type: "TensorListGather",
+		Type: "HistogramSummary",
 		Input: []tf.Input{
-			input_handle, indices, element_shape,
+			tag, values,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Creates a TensorList by indexing into a Tensor.
+// Computes the number of elements in the given queue.
 //
-// Each member of the TensorList corresponds to one row of the input tensor,
-// specified by the given index (see `tf.gather`).
+// Arguments:
+//	handle: The handle to a queue.
 //
-// tensor: The input tensor.
-// indices: The indices used to index into the list.
-// element_shape: The shape of the elements in the list (can be less specified than
-//   the shape of the tensor).
-// output_handle: The TensorList.
-func TensorListScatter(scope *Scope, tensor tf.Output, indices tf.Output, element_shape tf.Output) (output_handle tf.Output) {
+// Returns The number of elements in the given queue.
+func QueueSizeV2(scope *Scope, handle tf.Output) (size tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "TensorListScatter",
+		Type: "QueueSizeV2",
 		Input: []tf.Input{
-			tensor, indices, element_shape,
+			handle,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Returns a `RaggedTensor` containing the specified sequences of numbers.
+// ImageSummaryAttr is an optional argument to ImageSummary.
+type ImageSummaryAttr func(optionalAttr)
+
+// ImageSummaryMaxImages sets the optional max_images attribute to value.
 //
+// value: Max number of batch elements to generate images for.
+// If not specified, defaults to 3
 //
-// Returns a `RaggedTensor` `result` composed from `rt_dense_values` and
-// `rt_nested_splits`, such that
-// `result[i] = range(starts[i], limits[i], deltas[i])`.
+// REQUIRES: value >= 1
+func ImageSummaryMaxImages(value int64) ImageSummaryAttr {
+	return func(m optionalAttr) {
+		m["max_images"] = value
+	}
+}
+
+// ImageSummaryBadColor sets the optional bad_color attribute to value.
 //
-// ```python
-// >>> (rt_nested_splits, rt_dense_values) = gen_ragged_ops.ragged_range(
-// ...     starts=[2, 5, 8], limits=[3, 5, 12], deltas=1)
-// >>> result = ragged.from_nested_row_splits(rt_dense_values, rt_nested_splits)
-// >>> print result.eval().tolist()
-// [[2],               # result[0] = range(2, 3)
-//  [],                # result[1] = range(5, 5)
-//  [8, 9, 10, 11]]    # result[2] = range(8, 12)
-// ```
+// value: Color to use for pixels with non-finite values.
+// If not specified, defaults to <dtype:DT_UINT8 tensor_shape:<dim:<size:4 > > int_val:255 int_val:0 int_val:0 int_val:255 >
+func ImageSummaryBadColor(value tf.Tensor) ImageSummaryAttr {
+	return func(m optionalAttr) {
+		m["bad_color"] = value
+	}
+}
+
+// Outputs a `Summary` protocol buffer with images.
 //
-// The input tensors `starts`, `limits`, and `deltas` may be scalars or vectors.
-// The vector inputs must all have the same size.  Scalar inputs are broadcast
-// to match the size of the vector inputs.
+// The summary has up to `max_images` summary values containing images. The
+// images are built from `tensor` which must be 4-D with shape `[batch_size,
+// height, width, channels]` and where `channels` can be:
+//
+// *  1: `tensor` is interpreted as Grayscale.
+// *  3: `tensor` is interpreted as RGB.
+// *  4: `tensor` is interpreted as RGBA.
+//
+// The images have the same number of channels as the input tensor. For float
+// input, the values are normalized one image at a time to fit in the range
+// `[0, 255]`.  `uint8` values are unchanged.  The op uses two different
+// normalization algorithms:
+//
+// *  If the input values are all positive, they are rescaled so the largest one
+//    is 255.
+//
+// *  If any input value is negative, the values are shifted so input value 0.0
+//    is at 127.  They are then rescaled so that either the smallest value is 0,
+//    or the largest one is 255.
+//
+// The `tag` argument is a scalar `Tensor` of type `string`.  It is used to
+// build the `tag` of the summary values:
+//
+// *  If `max_images` is 1, the summary value tag is '*tag*/image'.
+// *  If `max_images` is greater than 1, the summary value tags are
+//    generated sequentially as '*tag*/image/0', '*tag*/image/1', etc.
+//
+// The `bad_color` argument is the color to use in the generated images for
+// non-finite input values.  It is a `uint8` 1-D tensor of length `channels`.
+// Each element must be in the range `[0, 255]` (It represents the value of a
+// pixel in the output image).  Non-finite values in the input tensor are
+// replaced by this tensor in the output image.  The default value is the color
+// red.
 //
 // Arguments:
-//	starts: The starts of each range.
-//	limits: The limits of each range.
-//	deltas: The deltas of each range.
+//	tag: Scalar. Used to build the `tag` attribute of the summary values.
+//	tensor: 4-D of shape `[batch_size, height, width, channels]` where
+// `channels` is 1, 3, or 4.
 //
-// Returns The `row_splits` for the returned `RaggedTensor`.The `flat_values` for the returned `RaggedTensor`.
-func RaggedRange(scope *Scope, starts tf.Output, limits tf.Output, deltas tf.Output) (rt_nested_splits tf.Output, rt_dense_values tf.Output) {
+// Returns Scalar. Serialized `Summary` protocol buffer.
+func ImageSummary(scope *Scope, tag tf.Output, tensor tf.Output, optional ...ImageSummaryAttr) (summary tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	opspec := tf.OpSpec{
-		Type: "RaggedRange",
-		Input: []tf.Input{
-			starts, limits, deltas,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
-}
-
-// Deprecated, use python implementation tf.linalg.matrix_exponential.
-//
-// DEPRECATED at GraphDef version 27: Use Python implementation tf.linalg.matrix_exponential instead.
-func MatrixExponential(scope *Scope, input tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MatrixExponential",
+		Type: "ImageSummary",
 		Input: []tf.Input{
-			input,
+			tag, tensor,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// QueueDequeueUpToV2Attr is an optional argument to QueueDequeueUpToV2.
-type QueueDequeueUpToV2Attr func(optionalAttr)
+// AudioSummaryV2Attr is an optional argument to AudioSummaryV2.
+type AudioSummaryV2Attr func(optionalAttr)
 
-// QueueDequeueUpToV2TimeoutMs sets the optional timeout_ms attribute to value.
+// AudioSummaryV2MaxOutputs sets the optional max_outputs attribute to value.
 //
-// value: If the queue has fewer than n elements, this operation
-// will block for up to timeout_ms milliseconds.
-// Note: This option is not supported yet.
-// If not specified, defaults to -1
-func QueueDequeueUpToV2TimeoutMs(value int64) QueueDequeueUpToV2Attr {
+// value: Max number of batch elements to generate audio for.
+// If not specified, defaults to 3
+//
+// REQUIRES: value >= 1
+func AudioSummaryV2MaxOutputs(value int64) AudioSummaryV2Attr {
 	return func(m optionalAttr) {
-		m["timeout_ms"] = value
+		m["max_outputs"] = value
 	}
 }
 
-// Dequeues `n` tuples of one or more tensors from the given queue.
-//
-// This operation is not supported by all queues.  If a queue does not support
-// DequeueUpTo, then an Unimplemented error is returned.
+// Outputs a `Summary` protocol buffer with audio.
 //
-// If the queue is closed and there are more than 0 but less than `n`
-// elements remaining, then instead of returning an OutOfRange error like
-// QueueDequeueMany, less than `n` elements are returned immediately.  If
-// the queue is closed and there are 0 elements left in the queue, then
-// an OutOfRange error is returned just like in QueueDequeueMany.
-// Otherwise the behavior is identical to QueueDequeueMany:
+// The summary has up to `max_outputs` summary values containing audio. The
+// audio is built from `tensor` which must be 3-D with shape `[batch_size,
+// frames, channels]` or 2-D with shape `[batch_size, frames]`. The values are
+// assumed to be in the range of `[-1.0, 1.0]` with a sample rate of `sample_rate`.
 //
-// This operation concatenates queue-element component tensors along the
-// 0th dimension to make a single component tensor.  All of the components
-// in the dequeued tuple will have size n in the 0th dimension.
+// The `tag` argument is a scalar `Tensor` of type `string`.  It is used to
+// build the `tag` of the summary values:
 //
-// This operation has `k` outputs, where `k` is the number of components in
-// the tuples stored in the given queue, and output `i` is the ith
-// component of the dequeued tuple.
+// *  If `max_outputs` is 1, the summary value tag is '*tag*/audio'.
+// *  If `max_outputs` is greater than 1, the summary value tags are
+//    generated sequentially as '*tag*/audio/0', '*tag*/audio/1', etc.
 //
 // Arguments:
-//	handle: The handle to a queue.
-//	n: The number of tuples to dequeue.
-//	component_types: The type of each component in a tuple.
+//	tag: Scalar. Used to build the `tag` attribute of the summary values.
+//	tensor: 2-D of shape `[batch_size, frames]`.
+//	sample_rate: The sample rate of the signal in hertz.
 //
-// Returns One or more tensors that were dequeued as a tuple.
-func QueueDequeueUpToV2(scope *Scope, handle tf.Output, n tf.Output, component_types []tf.DataType, optional ...QueueDequeueUpToV2Attr) (components []tf.Output) {
+// Returns Scalar. Serialized `Summary` protocol buffer.
+func AudioSummaryV2(scope *Scope, tag tf.Output, tensor tf.Output, sample_rate tf.Output, optional ...AudioSummaryV2Attr) (summary tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"component_types": component_types}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "QueueDequeueUpToV2",
+		Type: "AudioSummaryV2",
 		Input: []tf.Input{
-			handle, n,
+			tag, tensor, sample_rate,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if components, idx, err = makeOutputList(op, idx, "components"); err != nil {
-		scope.UpdateErr("QueueDequeueUpToV2", err)
-		return
-	}
-	return components
+	return op.Output(0)
 }
 
-// Computes the Cholesky decomposition of one or more square matrices.
-//
-// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
-// form square matrices.
-//
-// The input has to be symmetric and positive definite. Only the lower-triangular
-// part of the input will be used for this operation. The upper-triangular part
-// will not be read.
-//
-// The output is a tensor of the same shape as the input
-// containing the Cholesky decompositions for all input submatrices `[..., :, :]`.
-//
-// **Note**: The gradient computation on GPU is faster for large matrices but
-// not for large batch dimensions when the submatrices are small. In this
-// case it might be faster to use the CPU.
+// Splits a tensor into a list.
 //
-// Arguments:
-//	input: Shape is `[..., M, M]`.
+// list[i] corresponds to lengths[i] tensors from the input tensor.
+// The tensor must have rank at least 1 and contain exactly sum(lengths) elements.
 //
-// Returns Shape is `[..., M, M]`.
-func Cholesky(scope *Scope, input tf.Output) (output tf.Output) {
+// tensor: The input tensor.
+// element_shape: A shape compatible with that of elements in the tensor.
+// lengths: Vector of sizes of the 0th dimension of tensors in the list.
+// output_handle: The list.
+func TensorListSplit(scope *Scope, tensor tf.Output, element_shape tf.Output, lengths tf.Output) (output_handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Cholesky",
+		Type: "TensorListSplit",
 		Input: []tf.Input{
-			input,
+			tensor, element_shape, lengths,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Writes contents to the file at input filename. Creates file and recursively
-//
-// creates directory if not existing.
+// AvgPoolAttr is an optional argument to AvgPool.
+type AvgPoolAttr func(optionalAttr)
+
+// AvgPoolDataFormat sets the optional data_format attribute to value.
 //
-// Arguments:
-//	filename: scalar. The name of the file to which we write the contents.
-//	contents: scalar. The content to be written to the output file.
-//
-// Returns the created operation.
-func WriteFile(scope *Scope, filename tf.Output, contents tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "WriteFile",
-		Input: []tf.Input{
-			filename, contents,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
-// AllAttr is an optional argument to All.
-type AllAttr func(optionalAttr)
-
-// AllKeepDims sets the optional keep_dims attribute to value.
-//
-// value: If true, retain reduced dimensions with length 1.
-// If not specified, defaults to false
-func AllKeepDims(value bool) AllAttr {
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the data is stored in the order of:
+//     [batch, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, in_channels, in_height, in_width].
+// If not specified, defaults to "NHWC"
+func AvgPoolDataFormat(value string) AvgPoolAttr {
 	return func(m optionalAttr) {
-		m["keep_dims"] = value
+		m["data_format"] = value
 	}
 }
 
-// Computes the "logical and" of elements across dimensions of a tensor.
+// Performs average pooling on the input.
 //
-// Reduces `input` along the dimensions given in `axis`. Unless
-// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
-// `axis`. If `keep_dims` is true, the reduced dimensions are
-// retained with length 1.
+// Each entry in `output` is the mean of the corresponding size `ksize`
+// window in `value`.
 //
 // Arguments:
-//	input: The tensor to reduce.
-//	axis: The dimensions to reduce. Must be in the range
-// `[-rank(input), rank(input))`.
+//	value: 4-D with shape `[batch, height, width, channels]`.
+//	ksize: The size of the sliding window for each dimension of `value`.
+//	strides: The stride of the sliding window for each dimension of `value`.
+//	padding: The type of padding algorithm to use.
 //
-// Returns The reduced tensor.
-func All(scope *Scope, input tf.Output, axis tf.Output, optional ...AllAttr) (output tf.Output) {
+// Returns The average pooled output tensor.
+func AvgPool(scope *Scope, value tf.Output, ksize []int64, strides []int64, padding string, optional ...AvgPoolAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "All",
+		Type: "AvgPool",
 		Input: []tf.Input{
-			input, axis,
+			value,
 		},
 		Attrs: attrs,
 	}
@@ -29049,179 +28834,175 @@ func All(scope *Scope, input tf.Output, axis tf.Output, optional ...AllAttr) (ou
 	return op.Output(0)
 }
 
-// Computes the Eigen Decomposition of a batch of square self-adjoint matrices.
-//
-// DEPRECATED at GraphDef version 11: Use SelfAdjointEigV2 instead.
+// Merges summaries.
 //
-// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
-// form square matrices, with the same constraints as the single matrix
-// SelfAdjointEig.
+// This op creates a
+// [`Summary`](https://www.tensorflow.org/code/tensorflow/core/framework/summary.proto)
+// protocol buffer that contains the union of all the values in the input
+// summaries.
 //
-// The result is a [..., M+1, M] matrix with [..., 0,:] containing the
-// eigenvalues, and subsequent [...,1:, :] containing the eigenvectors. The eigenvalues
-// are sorted in non-decreasing order.
+// When the Op is run, it reports an `InvalidArgument` error if multiple values
+// in the summaries to merge use the same tag.
 //
 // Arguments:
-//	input: Shape is `[..., M, M]`.
+//	inputs: Can be of any shape.  Each must contain serialized `Summary` protocol
+// buffers.
 //
-// Returns Shape is `[..., M+1, M]`.
-func SelfAdjointEig(scope *Scope, input tf.Output) (output tf.Output) {
+// Returns Scalar. Serialized `Summary` protocol buffer.
+func MergeSummary(scope *Scope, inputs []tf.Output) (summary tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SelfAdjointEig",
+		Type: "MergeSummary",
 		Input: []tf.Input{
-			input,
+			tf.OutputList(inputs),
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes softplus gradients for a softplus operation.
-//
-// Arguments:
-//	gradients: The backpropagated gradients to the corresponding softplus operation.
-//	features: The features passed as input to the corresponding softplus operation.
+// The shape of the elements of the given list, as a tensor.
 //
-// Returns The gradients: `gradients / (1 + exp(-features))`.
-func SoftplusGrad(scope *Scope, gradients tf.Output, features tf.Output) (backprops tf.Output) {
+//   input_handle: the list
+//   element_shape: the shape of elements of the list
+func TensorListElementShape(scope *Scope, input_handle tf.Output, shape_type tf.DataType) (element_shape tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"shape_type": shape_type}
 	opspec := tf.OpSpec{
-		Type: "SoftplusGrad",
+		Type: "TensorListElementShape",
 		Input: []tf.Input{
-			gradients, features,
+			input_handle,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// SelfAdjointEigV2Attr is an optional argument to SelfAdjointEigV2.
-type SelfAdjointEigV2Attr func(optionalAttr)
-
-// SelfAdjointEigV2ComputeV sets the optional compute_v attribute to value.
+// Returns the item in the list with the given index.
 //
-// value: If `True` then eigenvectors will be computed and returned in `v`.
-// Otherwise, only the eigenvalues will be computed.
-// If not specified, defaults to true
-func SelfAdjointEigV2ComputeV(value bool) SelfAdjointEigV2Attr {
-	return func(m optionalAttr) {
-		m["compute_v"] = value
+// input_handle: the list
+// index: the position in the list from which an element will be retrieved
+// item: the element at that position
+//
+//
+func TensorListGetItem(scope *Scope, input_handle tf.Output, index tf.Output, element_shape tf.Output, element_dtype tf.DataType) (item tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"element_dtype": element_dtype}
+	opspec := tf.OpSpec{
+		Type: "TensorListGetItem",
+		Input: []tf.Input{
+			input_handle, index, element_shape,
+		},
+		Attrs: attrs,
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Computes the eigen decomposition of one or more square self-adjoint matrices.
-//
-// Computes the eigenvalues and (optionally) eigenvectors of each inner matrix in
-// `input` such that `input[..., :, :] = v[..., :, :] * diag(e[..., :])`. The eigenvalues
-// are sorted in non-decreasing order.
+// Resizes the list.
 //
-// ```python
-// # a is a tensor.
-// # e is a tensor of eigenvalues.
-// # v is a tensor of eigenvectors.
-// e, v = self_adjoint_eig(a)
-// e = self_adjoint_eig(a, compute_v=False)
-// ```
 //
-// Arguments:
-//	input: `Tensor` input of shape `[N, N]`.
+// input_handle: the input list
+// size: size of the output list
 //
-// Returns Eigenvalues. Shape is `[N]`.Eigenvectors. Shape is `[N, N]`.
-func SelfAdjointEigV2(scope *Scope, input tf.Output, optional ...SelfAdjointEigV2Attr) (e tf.Output, v tf.Output) {
+func TensorListResize(scope *Scope, input_handle tf.Output, size tf.Output) (output_handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "SelfAdjointEigV2",
+		Type: "TensorListResize",
 		Input: []tf.Input{
-			input,
+			input_handle, size,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// Adjust the saturation of one or more images.
+// Returns a diagonal tensor with a given diagonal values.
 //
-// `images` is a tensor of at least 3 dimensions.  The last dimension is
-// interpretted as channels, and must be three.
+// Given a `diagonal`, this operation returns a tensor with the `diagonal` and
+// everything else padded with zeros. The diagonal is computed as follows:
 //
-// The input image is considered in the RGB colorspace. Conceptually, the RGB
-// colors are first mapped into HSV. A scale is then applied all the saturation
-// values, and then remapped back to RGB colorspace.
+// Assume `diagonal` has dimensions [D1,..., Dk], then the output is a tensor of
+// rank 2k with dimensions [D1,..., Dk, D1,..., Dk] where:
 //
-// Arguments:
-//	images: Images to adjust.  At least 3-D.
-//	scale: A float scale to add to the saturation.
+// `output[i1,..., ik, i1,..., ik] = diagonal[i1, ..., ik]` and 0 everywhere else.
 //
-// Returns The hue-adjusted image or images.
-func AdjustSaturation(scope *Scope, images tf.Output, scale tf.Output) (output tf.Output) {
+// For example:
+//
+// ```
+// # 'diagonal' is [1, 2, 3, 4]
+// tf.diag(diagonal) ==> [[1, 0, 0, 0]
+//                        [0, 2, 0, 0]
+//                        [0, 0, 3, 0]
+//                        [0, 0, 0, 4]]
+// ```
+//
+// Arguments:
+//	diagonal: Rank k tensor where k is at most 1.
+func Diag(scope *Scope, diagonal tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "AdjustSaturation",
+		Type: "Diag",
 		Input: []tf.Input{
-			images, scale,
+			diagonal,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ResourceApplyKerasMomentumAttr is an optional argument to ResourceApplyKerasMomentum.
-type ResourceApplyKerasMomentumAttr func(optionalAttr)
+// ParameterizedTruncatedNormalAttr is an optional argument to ParameterizedTruncatedNormal.
+type ParameterizedTruncatedNormalAttr func(optionalAttr)
 
-// ResourceApplyKerasMomentumUseLocking sets the optional use_locking attribute to value.
+// ParameterizedTruncatedNormalSeed sets the optional seed attribute to value.
 //
-// value: If `True`, updating of the var and accum tensors will be protected
-// by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceApplyKerasMomentumUseLocking(value bool) ResourceApplyKerasMomentumAttr {
+// value: If either `seed` or `seed2` are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func ParameterizedTruncatedNormalSeed(value int64) ParameterizedTruncatedNormalAttr {
 	return func(m optionalAttr) {
-		m["use_locking"] = value
+		m["seed"] = value
 	}
 }
 
-// ResourceApplyKerasMomentumUseNesterov sets the optional use_nesterov attribute to value.
+// ParameterizedTruncatedNormalSeed2 sets the optional seed2 attribute to value.
 //
-// value: If `True`, the tensor passed to compute grad will be
-// var + momentum * accum, so in the end, the var you get is actually
-// var + momentum * accum.
-// If not specified, defaults to false
-func ResourceApplyKerasMomentumUseNesterov(value bool) ResourceApplyKerasMomentumAttr {
+// value: A second seed to avoid seed collision.
+// If not specified, defaults to 0
+func ParameterizedTruncatedNormalSeed2(value int64) ParameterizedTruncatedNormalAttr {
 	return func(m optionalAttr) {
-		m["use_nesterov"] = value
+		m["seed2"] = value
 	}
 }
 
-// Update '*var' according to the momentum scheme. Set use_nesterov = True if you
-//
-// want to use Nesterov momentum.
+// Outputs random values from a normal distribution. The parameters may each be a
 //
-// accum = accum * momentum - lr * grad
-// var += accum
+// scalar which applies to the entire output, or a vector of length shape[0] which
+// stores the parameters for each batch.
 //
 // Arguments:
-//	var_: Should be from a Variable().
-//	accum: Should be from a Variable().
-//	lr: Scaling factor. Must be a scalar.
-//	grad: The gradient.
-//	momentum: Momentum. Must be a scalar.
+//	shape: The shape of the output tensor. Batches are indexed by the 0th dimension.
+//	means: The mean parameter of each batch.
+//	stdevs: The standard deviation parameter of each batch. Must be greater than 0.
+//	minvals: The minimum cutoff. May be -infinity.
+//	maxvals: The maximum cutoff. May be +infinity, and must be more than the minval
+// for each batch.
 //
-// Returns the created operation.
-func ResourceApplyKerasMomentum(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, momentum tf.Output, optional ...ResourceApplyKerasMomentumAttr) (o *tf.Operation) {
+// Returns A matrix of shape num_batches x samples_per_batch, filled with random
+// truncated normal values using the parameters for each row.
+func ParameterizedTruncatedNormal(scope *Scope, shape tf.Output, means tf.Output, stdevs tf.Output, minvals tf.Output, maxvals tf.Output, optional ...ParameterizedTruncatedNormalAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -29230,55 +29011,54 @@ func ResourceApplyKerasMomentum(scope *Scope, var_ tf.Output, accum tf.Output, l
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceApplyKerasMomentum",
+		Type: "ParameterizedTruncatedNormal",
 		Input: []tf.Input{
-			var_, accum, lr, grad, momentum,
+			shape, means, stdevs, minvals, maxvals,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// MatrixSolveAttr is an optional argument to MatrixSolve.
-type MatrixSolveAttr func(optionalAttr)
-
-// MatrixSolveAdjoint sets the optional adjoint attribute to value.
+// Sets the index-th position of the list to contain the given tensor.
 //
-// value: Boolean indicating whether to solve with `matrix` or its (block-wise)
-// adjoint.
-// If not specified, defaults to false
-func MatrixSolveAdjoint(value bool) MatrixSolveAttr {
-	return func(m optionalAttr) {
-		m["adjoint"] = value
+// input_handle: the list
+// index: the position in the list to which the tensor will be assigned
+// item: the element to be assigned to that position
+// output_handle: the new list, with the element in the proper position
+//
+func TensorListSetItem(scope *Scope, input_handle tf.Output, index tf.Output, item tf.Output) (output_handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorListSetItem",
+		Input: []tf.Input{
+			input_handle, index, item,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Solves systems of linear equations.
-//
-// `Matrix` is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
-// form square matrices. `Rhs` is a tensor of shape `[..., M, K]`. The `output` is
-// a tensor shape `[..., M, K]`.  If `adjoint` is `False` then each output matrix
-// satisfies `matrix[..., :, :] * output[..., :, :] = rhs[..., :, :]`.
-// If `adjoint` is `True` then each output matrix satisfies
-// `adjoint(matrix[..., :, :]) * output[..., :, :] = rhs[..., :, :]`.
+// Creates a Tensor by indexing into the TensorList.
 //
-// Arguments:
-//	matrix: Shape is `[..., M, M]`.
-//	rhs: Shape is `[..., M, K]`.
+// Each row in the produced Tensor corresponds to the element in the TensorList
+// specified by the given index (see `tf.gather`).
 //
-// Returns Shape is `[..., M, K]`.
-func MatrixSolve(scope *Scope, matrix tf.Output, rhs tf.Output, optional ...MatrixSolveAttr) (output tf.Output) {
+// input_handle: The input tensor list.
+// indices: The indices used to index into the list.
+// values: The tensor.
+func TensorListGather(scope *Scope, input_handle tf.Output, indices tf.Output, element_shape tf.Output, element_dtype tf.DataType) (values tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"element_dtype": element_dtype}
 	opspec := tf.OpSpec{
-		Type: "MatrixSolve",
+		Type: "TensorListGather",
 		Input: []tf.Input{
-			matrix, rhs,
+			input_handle, indices, element_shape,
 		},
 		Attrs: attrs,
 	}
@@ -29286,124 +29066,80 @@ func MatrixSolve(scope *Scope, matrix tf.Output, rhs tf.Output, optional ...Matr
 	return op.Output(0)
 }
 
-// Returns a serialized GraphDef representing `input_dataset`.
-//
-// Returns a graph representation for `input_dataset`.
+// Creates a TensorList by indexing into a Tensor.
 //
-// Arguments:
-//	input_dataset: A variant tensor representing the dataset to return the graph representation for.
+// Each member of the TensorList corresponds to one row of the input tensor,
+// specified by the given index (see `tf.gather`).
 //
-// Returns The graph representation of the dataset (as serialized GraphDef).
-func DatasetToGraph(scope *Scope, input_dataset tf.Output) (graph tf.Output) {
+// tensor: The input tensor.
+// indices: The indices used to index into the list.
+// element_shape: The shape of the elements in the list (can be less specified than
+//   the shape of the tensor).
+// output_handle: The TensorList.
+func TensorListScatter(scope *Scope, tensor tf.Output, indices tf.Output, element_shape tf.Output) (output_handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "DatasetToGraph",
+		Type: "TensorListScatter",
 		Input: []tf.Input{
-			input_dataset,
+			tensor, indices, element_shape,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// LuAttr is an optional argument to Lu.
-type LuAttr func(optionalAttr)
-
-// LuOutputIdxType sets the optional output_idx_type attribute to value.
-// If not specified, defaults to DT_INT32
-func LuOutputIdxType(value tf.DataType) LuAttr {
-	return func(m optionalAttr) {
-		m["output_idx_type"] = value
-	}
-}
-
-// Computes the LU decomposition of one or more square matrices.
-//
-// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
-// form square matrices.
+// Returns a `RaggedTensor` containing the specified sequences of numbers.
 //
-// The input has to be invertible.
 //
-// The output consists of two tensors LU and P containing the LU decomposition
-// of all input submatrices `[..., :, :]`. LU encodes the lower triangular and
-// upper triangular factors.
+// Returns a `RaggedTensor` `result` composed from `rt_dense_values` and
+// `rt_nested_splits`, such that
+// `result[i] = range(starts[i], limits[i], deltas[i])`.
 //
-// For each input submatrix of shape `[M, M]`, L is a lower triangular matrix of
-// shape `[M, M]` with unit diagonal whose entries correspond to the strictly lower
-// triangular part of LU. U is a upper triangular matrix of shape `[M, M]` whose
-// entries correspond to the upper triangular part, including the diagonal, of LU.
+// ```python
+// >>> (rt_nested_splits, rt_dense_values) = gen_ragged_ops.ragged_range(
+// ...     starts=[2, 5, 8], limits=[3, 5, 12], deltas=1)
+// >>> result = ragged.from_nested_row_splits(rt_dense_values, rt_nested_splits)
+// >>> print result.eval().tolist()
+// [[2],               # result[0] = range(2, 3)
+//  [],                # result[1] = range(5, 5)
+//  [8, 9, 10, 11]]    # result[2] = range(8, 12)
+// ```
 //
-// P represents a permutation matrix encoded as a list of indices each between `0`
-// and `M-1`, inclusive. If P_mat denotes the permutation matrix corresponding to
-// P, then the L, U and P satisfies P_mat * input = L * U.
+// The input tensors `starts`, `limits`, and `deltas` may be scalars or vectors.
+// The vector inputs must all have the same size.  Scalar inputs are broadcast
+// to match the size of the vector inputs.
 //
 // Arguments:
-//	input: A tensor of shape `[..., M, M]` whose inner-most 2 dimensions form matrices of
-// size `[M, M]`.
+//	starts: The starts of each range.
+//	limits: The limits of each range.
+//	deltas: The deltas of each range.
 //
-// Returns A tensor of shape `[..., M, M]` whose strictly lower triangular part denotes the
-// lower triangular factor `L` with unit diagonal, and whose upper triangular part
-// denotes the upper triangular factor `U`.Permutation of the rows encoded as a list of indices in `0..M-1`. Shape is
-// `[..., M]`.
-// @compatibility(scipy)
-// Similar to `scipy.linalg.lu`, except the triangular factors `L` and `U` are
-// packed into a single tensor, the permutation is applied to `input` instead of
-// the right hand side and the permutation `P` is returned as a list of indices
-// instead of a permutation matrix.
-// @end_compatibility
-func Lu(scope *Scope, input tf.Output, optional ...LuAttr) (lu tf.Output, p tf.Output) {
+// Returns The `row_splits` for the returned `RaggedTensor`.The `flat_values` for the returned `RaggedTensor`.
+func RaggedRange(scope *Scope, starts tf.Output, limits tf.Output, deltas tf.Output) (rt_nested_splits tf.Output, rt_dense_values tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "Lu",
+		Type: "RaggedRange",
 		Input: []tf.Input{
-			input,
+			starts, limits, deltas,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0), op.Output(1)
 }
 
-// Computes the matrix square root of one or more square matrices:
-//
-// matmul(sqrtm(A), sqrtm(A)) = A
-//
-// The input matrix should be invertible. If the input matrix is real, it should
-// have no eigenvalues which are real and negative (pairs of complex conjugate
-// eigenvalues are allowed).
-//
-// The matrix square root is computed by first reducing the matrix to
-// quasi-triangular form with the real Schur decomposition. The square root
-// of the quasi-triangular matrix is then computed directly. Details of
-// the algorithm can be found in: Nicholas J. Higham, "Computing real
-// square roots of a real matrix", Linear Algebra Appl., 1987.
-//
-// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
-// form square matrices. The output is a tensor of the same shape as the input
-// containing the matrix square root for all input submatrices `[..., :, :]`.
-//
-// Arguments:
-//	input: Shape is `[..., M, M]`.
-//
-// Returns Shape is `[..., M, M]`.
+// Deprecated, use python implementation tf.linalg.matrix_exponential.
 //
-// @compatibility(scipy)
-// Equivalent to scipy.linalg.sqrtm
-// @end_compatibility
-func MatrixSquareRoot(scope *Scope, input tf.Output) (output tf.Output) {
+// DEPRECATED at GraphDef version 27: Use Python implementation tf.linalg.matrix_exponential instead.
+func MatrixExponential(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "MatrixSquareRoot",
+		Type: "MatrixExponential",
 		Input: []tf.Input{
 			input,
 		},
@@ -29412,148 +29148,158 @@ func MatrixSquareRoot(scope *Scope, input tf.Output) (output tf.Output) {
 	return op.Output(0)
 }
 
-// SvdAttr is an optional argument to Svd.
-type SvdAttr func(optionalAttr)
+// QueueDequeueUpToV2Attr is an optional argument to QueueDequeueUpToV2.
+type QueueDequeueUpToV2Attr func(optionalAttr)
 
-// SvdComputeUv sets the optional compute_uv attribute to value.
+// QueueDequeueUpToV2TimeoutMs sets the optional timeout_ms attribute to value.
 //
-// value: If true, left and right singular vectors will be
-// computed and returned in `u` and `v`, respectively.
-// If false, `u` and `v` are not set and should never referenced.
-// If not specified, defaults to true
-func SvdComputeUv(value bool) SvdAttr {
+// value: If the queue has fewer than n elements, this operation
+// will block for up to timeout_ms milliseconds.
+// Note: This option is not supported yet.
+// If not specified, defaults to -1
+func QueueDequeueUpToV2TimeoutMs(value int64) QueueDequeueUpToV2Attr {
 	return func(m optionalAttr) {
-		m["compute_uv"] = value
+		m["timeout_ms"] = value
 	}
 }
 
-// SvdFullMatrices sets the optional full_matrices attribute to value.
+// Dequeues `n` tuples of one or more tensors from the given queue.
 //
-// value: If true, compute full-sized `u` and `v`. If false
-// (the default), compute only the leading `P` singular vectors.
-// Ignored if `compute_uv` is `False`.
-// If not specified, defaults to false
-func SvdFullMatrices(value bool) SvdAttr {
-	return func(m optionalAttr) {
-		m["full_matrices"] = value
-	}
-}
-
-// Computes the singular value decompositions of one or more matrices.
+// This operation is not supported by all queues.  If a queue does not support
+// DequeueUpTo, then an Unimplemented error is returned.
 //
-// Computes the SVD of each inner matrix in `input` such that
-// `input[..., :, :] = u[..., :, :] * diag(s[..., :, :]) * transpose(v[..., :, :])`
+// If the queue is closed and there are more than 0 but less than `n`
+// elements remaining, then instead of returning an OutOfRange error like
+// QueueDequeueMany, less than `n` elements are returned immediately.  If
+// the queue is closed and there are 0 elements left in the queue, then
+// an OutOfRange error is returned just like in QueueDequeueMany.
+// Otherwise the behavior is identical to QueueDequeueMany:
 //
-// ```python
-// # a is a tensor containing a batch of matrices.
-// # s is a tensor of singular values for each matrix.
-// # u is the tensor containing of left singular vectors for each matrix.
-// # v is the tensor containing of right singular vectors for each matrix.
-// s, u, v = svd(a)
-// s, _, _ = svd(a, compute_uv=False)
-// ```
+// This operation concatenates queue-element component tensors along the
+// 0th dimension to make a single component tensor.  All of the components
+// in the dequeued tuple will have size n in the 0th dimension.
+//
+// This operation has `k` outputs, where `k` is the number of components in
+// the tuples stored in the given queue, and output `i` is the ith
+// component of the dequeued tuple.
 //
 // Arguments:
-//	input: A tensor of shape `[..., M, N]` whose inner-most 2 dimensions
-// form matrices of size `[M, N]`. Let `P` be the minimum of `M` and `N`.
+//	handle: The handle to a queue.
+//	n: The number of tuples to dequeue.
+//	component_types: The type of each component in a tuple.
 //
-// Returns Singular values. Shape is `[..., P]`.Left singular vectors. If `full_matrices` is `False` then shape is
-// `[..., M, P]`; if `full_matrices` is `True` then shape is
-// `[..., M, M]`. Undefined if `compute_uv` is `False`.Left singular vectors. If `full_matrices` is `False` then shape is
-// `[..., N, P]`. If `full_matrices` is `True` then shape is `[..., N, N]`.
-// Undefined if `compute_uv` is false.
-func Svd(scope *Scope, input tf.Output, optional ...SvdAttr) (s tf.Output, u tf.Output, v tf.Output) {
+// Returns One or more tensors that were dequeued as a tuple.
+func QueueDequeueUpToV2(scope *Scope, handle tf.Output, n tf.Output, component_types []tf.DataType, optional ...QueueDequeueUpToV2Attr) (components []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"component_types": component_types}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Svd",
+		Type: "QueueDequeueUpToV2",
 		Input: []tf.Input{
-			input,
+			handle, n,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
-}
-
-// PrintV2Attr is an optional argument to PrintV2.
-type PrintV2Attr func(optionalAttr)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if components, idx, err = makeOutputList(op, idx, "components"); err != nil {
+		scope.UpdateErr("QueueDequeueUpToV2", err)
+		return
+	}
+	return components
+}
 
-// PrintV2OutputStream sets the optional output_stream attribute to value.
+// Computes the Cholesky decomposition of one or more square matrices.
 //
-// value: A string specifying the output stream or logging level to print to.
-// If not specified, defaults to "stderr"
-func PrintV2OutputStream(value string) PrintV2Attr {
-	return func(m optionalAttr) {
-		m["output_stream"] = value
+// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
+// form square matrices.
+//
+// The input has to be symmetric and positive definite. Only the lower-triangular
+// part of the input will be used for this operation. The upper-triangular part
+// will not be read.
+//
+// The output is a tensor of the same shape as the input
+// containing the Cholesky decompositions for all input submatrices `[..., :, :]`.
+//
+// **Note**: The gradient computation on GPU is faster for large matrices but
+// not for large batch dimensions when the submatrices are small. In this
+// case it might be faster to use the CPU.
+//
+// Arguments:
+//	input: Shape is `[..., M, M]`.
+//
+// Returns Shape is `[..., M, M]`.
+func Cholesky(scope *Scope, input tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Cholesky",
+		Input: []tf.Input{
+			input,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Prints a string scalar.
+// Writes contents to the file at input filename. Creates file and recursively
 //
-// Prints a string scalar to the desired output_stream.
+// creates directory if not existing.
 //
 // Arguments:
-//	input: The string scalar to print.
+//	filename: scalar. The name of the file to which we write the contents.
+//	contents: scalar. The content to be written to the output file.
 //
 // Returns the created operation.
-func PrintV2(scope *Scope, input tf.Output, optional ...PrintV2Attr) (o *tf.Operation) {
+func WriteFile(scope *Scope, filename tf.Output, contents tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "PrintV2",
+		Type: "WriteFile",
 		Input: []tf.Input{
-			input,
+			filename, contents,
 		},
-		Attrs: attrs,
 	}
 	return scope.AddOperation(opspec)
 }
 
-// QueueEnqueueManyV2Attr is an optional argument to QueueEnqueueManyV2.
-type QueueEnqueueManyV2Attr func(optionalAttr)
+// AllAttr is an optional argument to All.
+type AllAttr func(optionalAttr)
 
-// QueueEnqueueManyV2TimeoutMs sets the optional timeout_ms attribute to value.
+// AllKeepDims sets the optional keep_dims attribute to value.
 //
-// value: If the queue is too full, this operation will block for up
-// to timeout_ms milliseconds.
-// Note: This option is not supported yet.
-// If not specified, defaults to -1
-func QueueEnqueueManyV2TimeoutMs(value int64) QueueEnqueueManyV2Attr {
+// value: If true, retain reduced dimensions with length 1.
+// If not specified, defaults to false
+func AllKeepDims(value bool) AllAttr {
 	return func(m optionalAttr) {
-		m["timeout_ms"] = value
+		m["keep_dims"] = value
 	}
 }
 
-// Enqueues zero or more tuples of one or more tensors in the given queue.
-//
-// This operation slices each component tensor along the 0th dimension to
-// make multiple queue elements. All of the tuple components must have the
-// same size in the 0th dimension.
-//
-// The components input has k elements, which correspond to the components of
-// tuples stored in the given queue.
+// Computes the "logical and" of elements across dimensions of a tensor.
 //
-// N.B. If the queue is full, this operation will block until the given
-// elements have been enqueued (or 'timeout_ms' elapses, if specified).
+// Reduces `input` along the dimensions given in `axis`. Unless
+// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+// `axis`. If `keep_dims` is true, the reduced dimensions are
+// retained with length 1.
 //
 // Arguments:
-//	handle: The handle to a queue.
-//	components: One or more tensors from which the enqueued tensors should
-// be taken.
+//	input: The tensor to reduce.
+//	axis: The dimensions to reduce. Must be in the range
+// `[-rank(input), rank(input))`.
 //
-// Returns the created operation.
-func QueueEnqueueManyV2(scope *Scope, handle tf.Output, components []tf.Output, optional ...QueueEnqueueManyV2Attr) (o *tf.Operation) {
+// Returns The reduced tensor.
+func All(scope *Scope, input tf.Output, axis tf.Output, optional ...AllAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -29562,124 +29308,100 @@ func QueueEnqueueManyV2(scope *Scope, handle tf.Output, components []tf.Output,
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "QueueEnqueueManyV2",
+		Type: "All",
 		Input: []tf.Input{
-			handle, tf.OutputList(components),
+			input, axis,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Computes the product along segments of a tensor.
-//
-// Read
-// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
-// for an explanation of segments.
+// Computes the Eigen Decomposition of a batch of square self-adjoint matrices.
 //
-// Computes a tensor such that
-// \\(output_i = \prod_j data_j\\) where the product is over `j` such
-// that `segment_ids[j] == i`.
+// DEPRECATED at GraphDef version 11: Use SelfAdjointEigV2 instead.
 //
-// If the product is empty for a given segment ID `i`, `output[i] = 1`.
+// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
+// form square matrices, with the same constraints as the single matrix
+// SelfAdjointEig.
 //
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/SegmentProd.png" alt>
-// </div>
+// The result is a [..., M+1, M] matrix with [..., 0,:] containing the
+// eigenvalues, and subsequent [...,1:, :] containing the eigenvectors. The eigenvalues
+// are sorted in non-decreasing order.
 //
 // Arguments:
+//	input: Shape is `[..., M, M]`.
 //
-//	segment_ids: A 1-D tensor whose size is equal to the size of `data`'s
-// first dimension.  Values should be sorted and can be repeated.
-//
-// Returns Has same shape as data, except for dimension 0 which
-// has size `k`, the number of segments.
-func SegmentProd(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) {
+// Returns Shape is `[..., M+1, M]`.
+func SelfAdjointEig(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SegmentProd",
+		Type: "SelfAdjointEig",
 		Input: []tf.Input{
-			data, segment_ids,
+			input,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Converts one or more images from RGB to HSV.
-//
-// Outputs a tensor of the same shape as the `images` tensor, containing the HSV
-// value of the pixels. The output is only well defined if the value in `images`
-// are in `[0,1]`.
-//
-// `output[..., 0]` contains hue, `output[..., 1]` contains saturation, and
-// `output[..., 2]` contains value. All HSV values are in `[0,1]`. A hue of 0
-// corresponds to pure red, hue 1/3 is pure green, and 2/3 is pure blue.
+// Computes softplus gradients for a softplus operation.
 //
 // Arguments:
-//	images: 1-D or higher rank. RGB data to convert. Last dimension must be size 3.
+//	gradients: The backpropagated gradients to the corresponding softplus operation.
+//	features: The features passed as input to the corresponding softplus operation.
 //
-// Returns `images` converted to HSV.
-func RGBToHSV(scope *Scope, images tf.Output) (output tf.Output) {
+// Returns The gradients: `gradients / (1 + exp(-features))`.
+func SoftplusGrad(scope *Scope, gradients tf.Output, features tf.Output) (backprops tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "RGBToHSV",
+		Type: "SoftplusGrad",
 		Input: []tf.Input{
-			images,
+			gradients, features,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Does nothing. Only useful as a placeholder for control edges.
-//
-// Returns the created operation.
-func NoOp(scope *Scope) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "NoOp",
-	}
-	return scope.AddOperation(opspec)
-}
-
-// MergeV2CheckpointsAttr is an optional argument to MergeV2Checkpoints.
-type MergeV2CheckpointsAttr func(optionalAttr)
+// SelfAdjointEigV2Attr is an optional argument to SelfAdjointEigV2.
+type SelfAdjointEigV2Attr func(optionalAttr)
 
-// MergeV2CheckpointsDeleteOldDirs sets the optional delete_old_dirs attribute to value.
+// SelfAdjointEigV2ComputeV sets the optional compute_v attribute to value.
 //
-// value: see above.
+// value: If `True` then eigenvectors will be computed and returned in `v`.
+// Otherwise, only the eigenvalues will be computed.
 // If not specified, defaults to true
-func MergeV2CheckpointsDeleteOldDirs(value bool) MergeV2CheckpointsAttr {
+func SelfAdjointEigV2ComputeV(value bool) SelfAdjointEigV2Attr {
 	return func(m optionalAttr) {
-		m["delete_old_dirs"] = value
+		m["compute_v"] = value
 	}
 }
 
-// V2 format specific: merges the metadata files of sharded checkpoints.  The
-//
-// result is one logical checkpoint, with one physical metadata file and renamed
-// data files.
+// Computes the eigen decomposition of one or more square self-adjoint matrices.
 //
-// Intended for "grouping" multiple checkpoints in a sharded checkpoint setup.
+// Computes the eigenvalues and (optionally) eigenvectors of each inner matrix in
+// `input` such that `input[..., :, :] = v[..., :, :] * diag(e[..., :])`. The eigenvalues
+// are sorted in non-decreasing order.
 //
-// If delete_old_dirs is true, attempts to delete recursively the dirname of each
-// path in the input checkpoint_prefixes.  This is useful when those paths are non
-// user-facing temporary locations.
+// ```python
+// # a is a tensor.
+// # e is a tensor of eigenvalues.
+// # v is a tensor of eigenvectors.
+// e, v = self_adjoint_eig(a)
+// e = self_adjoint_eig(a, compute_v=False)
+// ```
 //
 // Arguments:
-//	checkpoint_prefixes: prefixes of V2 checkpoints to merge.
-//	destination_prefix: scalar.  The desired final prefix.  Allowed to be the same
-// as one of the checkpoint_prefixes.
+//	input: `Tensor` input of shape `[N, N]`.
 //
-// Returns the created operation.
-func MergeV2Checkpoints(scope *Scope, checkpoint_prefixes tf.Output, destination_prefix tf.Output, optional ...MergeV2CheckpointsAttr) (o *tf.Operation) {
+// Returns Eigenvalues. Shape is `[N]`.Eigenvectors. Shape is `[N, N]`.
+func SelfAdjointEigV2(scope *Scope, input tf.Output, optional ...SelfAdjointEigV2Attr) (e tf.Output, v tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -29688,345 +29410,293 @@ func MergeV2Checkpoints(scope *Scope, checkpoint_prefixes tf.Output, destination
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MergeV2Checkpoints",
+		Type: "SelfAdjointEigV2",
 		Input: []tf.Input{
-			checkpoint_prefixes, destination_prefix,
+			input,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
 }
 
-// Saves input tensors slices to disk.
-//
-// This is like `Save` except that tensors can be listed in the saved file as being
-// a slice of a larger tensor.  `shapes_and_slices` specifies the shape of the
-// larger tensor and the slice that this tensor covers. `shapes_and_slices` must
-// have as many elements as `tensor_names`.
-//
-// Elements of the `shapes_and_slices` input must either be:
-//
-// *  The empty string, in which case the corresponding tensor is
-//    saved normally.
-// *  A string of the form `dim0 dim1 ... dimN-1 slice-spec` where the
-//    `dimI` are the dimensions of the larger tensor and `slice-spec`
-//    specifies what part is covered by the tensor to save.
-//
-// `slice-spec` itself is a `:`-separated list: `slice0:slice1:...:sliceN-1`
-// where each `sliceI` is either:
+// Adjust the saturation of one or more images.
 //
-// *  The string `-` meaning that the slice covers all indices of this dimension
-// *  `start,length` where `start` and `length` are integers.  In that
-//    case the slice covers `length` indices starting at `start`.
+// `images` is a tensor of at least 3 dimensions.  The last dimension is
+// interpretted as channels, and must be three.
 //
-// See also `Save`.
+// The input image is considered in the RGB colorspace. Conceptually, the RGB
+// colors are first mapped into HSV. A scale is then applied all the saturation
+// values, and then remapped back to RGB colorspace.
 //
 // Arguments:
-//	filename: Must have a single element. The name of the file to which we write the
-// tensor.
-//	tensor_names: Shape `[N]`. The names of the tensors to be saved.
-//	shapes_and_slices: Shape `[N]`.  The shapes and slice specifications to use when
-// saving the tensors.
-//	data: `N` tensors to save.
+//	images: Images to adjust.  At least 3-D.
+//	scale: A float scale to add to the saturation.
 //
-// Returns the created operation.
-func SaveSlices(scope *Scope, filename tf.Output, tensor_names tf.Output, shapes_and_slices tf.Output, data []tf.Output) (o *tf.Operation) {
+// Returns The hue-adjusted image or images.
+func AdjustSaturation(scope *Scope, images tf.Output, scale tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SaveSlices",
+		Type: "AdjustSaturation",
 		Input: []tf.Input{
-			filename, tensor_names, shapes_and_slices, tf.OutputList(data),
+			images, scale,
 		},
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// DenseToDenseSetOperationAttr is an optional argument to DenseToDenseSetOperation.
-type DenseToDenseSetOperationAttr func(optionalAttr)
+// MatrixSolveAttr is an optional argument to MatrixSolve.
+type MatrixSolveAttr func(optionalAttr)
 
-// DenseToDenseSetOperationValidateIndices sets the optional validate_indices attribute to value.
-// If not specified, defaults to true
-func DenseToDenseSetOperationValidateIndices(value bool) DenseToDenseSetOperationAttr {
+// MatrixSolveAdjoint sets the optional adjoint attribute to value.
+//
+// value: Boolean indicating whether to solve with `matrix` or its (block-wise)
+// adjoint.
+// If not specified, defaults to false
+func MatrixSolveAdjoint(value bool) MatrixSolveAttr {
 	return func(m optionalAttr) {
-		m["validate_indices"] = value
+		m["adjoint"] = value
 	}
 }
 
-// Applies set operation along last dimension of 2 `Tensor` inputs.
-//
-// See SetOperationOp::SetOperationFromContext for values of `set_operation`.
+// Solves systems of linear equations.
 //
-// Output `result` is a `SparseTensor` represented by `result_indices`,
-// `result_values`, and `result_shape`. For `set1` and `set2` ranked `n`, this
-// has rank `n` and the same 1st `n-1` dimensions as `set1` and `set2`. The `nth`
-// dimension contains the result of `set_operation` applied to the corresponding
-// `[0...n-1]` dimension of `set`.
+// `Matrix` is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
+// form square matrices. `Rhs` is a tensor of shape `[..., M, K]`. The `output` is
+// a tensor shape `[..., M, K]`.  If `adjoint` is `False` then each output matrix
+// satisfies `matrix[..., :, :] * output[..., :, :] = rhs[..., :, :]`.
+// If `adjoint` is `True` then each output matrix satisfies
+// `adjoint(matrix[..., :, :]) * output[..., :, :] = rhs[..., :, :]`.
 //
 // Arguments:
-//	set1: `Tensor` with rank `n`. 1st `n-1` dimensions must be the same as `set2`.
-// Dimension `n` contains values in a set, duplicates are allowed but ignored.
-//	set2: `Tensor` with rank `n`. 1st `n-1` dimensions must be the same as `set1`.
-// Dimension `n` contains values in a set, duplicates are allowed but ignored.
-//
+//	matrix: Shape is `[..., M, M]`.
+//	rhs: Shape is `[..., M, K]`.
 //
-// Returns 2D indices of a `SparseTensor`.1D values of a `SparseTensor`.1D `Tensor` shape of a `SparseTensor`. `result_shape[0...n-1]` is
-// the same as the 1st `n-1` dimensions of `set1` and `set2`, `result_shape[n]`
-// is the max result set size across all `0...n-1` dimensions.
-func DenseToDenseSetOperation(scope *Scope, set1 tf.Output, set2 tf.Output, set_operation string, optional ...DenseToDenseSetOperationAttr) (result_indices tf.Output, result_values tf.Output, result_shape tf.Output) {
+// Returns Shape is `[..., M, K]`.
+func MatrixSolve(scope *Scope, matrix tf.Output, rhs tf.Output, optional ...MatrixSolveAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"set_operation": set_operation}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "DenseToDenseSetOperation",
+		Type: "MatrixSolve",
 		Input: []tf.Input{
-			set1, set2,
+			matrix, rhs,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// Generate a sharded filename. The filename is printf formatted as
+// ResourceApplyKerasMomentumAttr is an optional argument to ResourceApplyKerasMomentum.
+type ResourceApplyKerasMomentumAttr func(optionalAttr)
+
+// ResourceApplyKerasMomentumUseLocking sets the optional use_locking attribute to value.
 //
-//    %s-%05d-of-%05d, basename, shard, num_shards.
-func ShardedFilename(scope *Scope, basename tf.Output, shard tf.Output, num_shards tf.Output) (filename tf.Output) {
+// value: If `True`, updating of the var and accum tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceApplyKerasMomentumUseLocking(value bool) ResourceApplyKerasMomentumAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
+	}
+}
+
+// ResourceApplyKerasMomentumUseNesterov sets the optional use_nesterov attribute to value.
+//
+// value: If `True`, the tensor passed to compute grad will be
+// var + momentum * accum, so in the end, the var you get is actually
+// var + momentum * accum.
+// If not specified, defaults to false
+func ResourceApplyKerasMomentumUseNesterov(value bool) ResourceApplyKerasMomentumAttr {
+	return func(m optionalAttr) {
+		m["use_nesterov"] = value
+	}
+}
+
+// Update '*var' according to the momentum scheme. Set use_nesterov = True if you
+//
+// want to use Nesterov momentum.
+//
+// accum = accum * momentum - lr * grad
+// var += accum
+//
+// Arguments:
+//	var_: Should be from a Variable().
+//	accum: Should be from a Variable().
+//	lr: Scaling factor. Must be a scalar.
+//	grad: The gradient.
+//	momentum: Momentum. Must be a scalar.
+//
+// Returns the created operation.
+func ResourceApplyKerasMomentum(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, momentum tf.Output, optional ...ResourceApplyKerasMomentumAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "ShardedFilename",
+		Type: "ResourceApplyKerasMomentum",
 		Input: []tf.Input{
-			basename, shard, num_shards,
+			var_, accum, lr, grad, momentum,
 		},
+		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// BatchToSpace for N-D tensors of type T.
+// Returns a serialized GraphDef representing `input_dataset`.
 //
-// This operation reshapes the "batch" dimension 0 into `M + 1` dimensions of shape
-// `block_shape + [batch]`, interleaves these blocks back into the grid defined by
-// the spatial dimensions `[1, ..., M]`, to obtain a result with the same rank as
-// the input.  The spatial dimensions of this intermediate result are then
-// optionally cropped according to `crops` to produce the output.  This is the
-// reverse of SpaceToBatch.  See below for a precise description.
+// Returns a graph representation for `input_dataset`.
 //
 // Arguments:
-//	input: N-D with shape `input_shape = [batch] + spatial_shape + remaining_shape`,
-// where spatial_shape has M dimensions.
-//	block_shape: 1-D with shape `[M]`, all values must be >= 1.
-//	crops: 2-D with shape `[M, 2]`, all values must be >= 0.
-//   `crops[i] = [crop_start, crop_end]` specifies the amount to crop from input
-//   dimension `i + 1`, which corresponds to spatial dimension `i`.  It is
-//   required that
-//   `crop_start[i] + crop_end[i] <= block_shape[i] * input_shape[i + 1]`.
-//
-// This operation is equivalent to the following steps:
-//
-// 1. Reshape `input` to `reshaped` of shape:
-//      [block_shape[0], ..., block_shape[M-1],
-//       batch / prod(block_shape),
-//       input_shape[1], ..., input_shape[N-1]]
-//
-// 2. Permute dimensions of `reshaped` to produce `permuted` of shape
-//      [batch / prod(block_shape),
-//
-//       input_shape[1], block_shape[0],
-//       ...,
-//       input_shape[M], block_shape[M-1],
-//
-//       input_shape[M+1], ..., input_shape[N-1]]
-//
-// 3. Reshape `permuted` to produce `reshaped_permuted` of shape
-//      [batch / prod(block_shape),
-//
-//       input_shape[1] * block_shape[0],
-//       ...,
-//       input_shape[M] * block_shape[M-1],
-//
-//       input_shape[M+1],
-//       ...,
-//       input_shape[N-1]]
-//
-// 4. Crop the start and end of dimensions `[1, ..., M]` of
-//    `reshaped_permuted` according to `crops` to produce the output of shape:
-//      [batch / prod(block_shape),
-//
-//       input_shape[1] * block_shape[0] - crops[0,0] - crops[0,1],
-//       ...,
-//       input_shape[M] * block_shape[M-1] - crops[M-1,0] - crops[M-1,1],
-//
-//       input_shape[M+1], ..., input_shape[N-1]]
-//
-// Some examples:
-//
-// (1) For the following input of shape `[4, 1, 1, 1]`, `block_shape = [2, 2]`, and
-//     `crops = [[0, 0], [0, 0]]`:
-//
-// ```
-// [[[[1]]], [[[2]]], [[[3]]], [[[4]]]]
-// ```
-//
-// The output tensor has shape `[1, 2, 2, 1]` and value:
-//
-// ```
-// x = [[[[1], [2]], [[3], [4]]]]
-// ```
-//
-// (2) For the following input of shape `[4, 1, 1, 3]`, `block_shape = [2, 2]`, and
-//     `crops = [[0, 0], [0, 0]]`:
-//
-// ```
-// [[[1, 2, 3]], [[4, 5, 6]], [[7, 8, 9]], [[10, 11, 12]]]
-// ```
-//
-// The output tensor has shape `[1, 2, 2, 3]` and value:
-//
-// ```
-// x = [[[[1, 2, 3], [4, 5, 6]],
-//       [[7, 8, 9], [10, 11, 12]]]]
-// ```
-//
-// (3) For the following input of shape `[4, 2, 2, 1]`, `block_shape = [2, 2]`, and
-//     `crops = [[0, 0], [0, 0]]`:
-//
-// ```
-// x = [[[[1], [3]], [[9], [11]]],
-//      [[[2], [4]], [[10], [12]]],
-//      [[[5], [7]], [[13], [15]]],
-//      [[[6], [8]], [[14], [16]]]]
-// ```
-//
-// The output tensor has shape `[1, 4, 4, 1]` and value:
-//
-// ```
-// x = [[[1],   [2],  [3],  [4]],
-//      [[5],   [6],  [7],  [8]],
-//      [[9],  [10], [11],  [12]],
-//      [[13], [14], [15],  [16]]]
-// ```
-//
-// (4) For the following input of shape `[8, 1, 3, 1]`, `block_shape = [2, 2]`, and
-//     `crops = [[0, 0], [2, 0]]`:
-//
-// ```
-// x = [[[[0], [1], [3]]], [[[0], [9], [11]]],
-//      [[[0], [2], [4]]], [[[0], [10], [12]]],
-//      [[[0], [5], [7]]], [[[0], [13], [15]]],
-//      [[[0], [6], [8]]], [[[0], [14], [16]]]]
-// ```
-//
-// The output tensor has shape `[2, 2, 4, 1]` and value:
+//	input_dataset: A variant tensor representing the dataset to return the graph representation for.
 //
-// ```
-// x = [[[[1],   [2],  [3],  [4]],
-//       [[5],   [6],  [7],  [8]]],
-//      [[[9],  [10], [11],  [12]],
-//       [[13], [14], [15],  [16]]]]
-// ```
-func BatchToSpaceND(scope *Scope, input tf.Output, block_shape tf.Output, crops tf.Output) (output tf.Output) {
+// Returns The graph representation of the dataset (as serialized GraphDef).
+func DatasetToGraph(scope *Scope, input_dataset tf.Output) (graph tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "BatchToSpaceND",
+		Type: "DatasetToGraph",
 		Input: []tf.Input{
-			input, block_shape, crops,
+			input_dataset,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// UnpackAttr is an optional argument to Unpack.
-type UnpackAttr func(optionalAttr)
+// LuAttr is an optional argument to Lu.
+type LuAttr func(optionalAttr)
 
-// UnpackAxis sets the optional axis attribute to value.
-//
-// value: Dimension along which to unpack.  Negative values wrap around, so the
-// valid range is `[-R, R)`.
-// If not specified, defaults to 0
-func UnpackAxis(value int64) UnpackAttr {
+// LuOutputIdxType sets the optional output_idx_type attribute to value.
+// If not specified, defaults to DT_INT32
+func LuOutputIdxType(value tf.DataType) LuAttr {
 	return func(m optionalAttr) {
-		m["axis"] = value
+		m["output_idx_type"] = value
 	}
 }
 
-// Unpacks a given dimension of a rank-`R` tensor into `num` rank-`(R-1)` tensors.
+// Computes the LU decomposition of one or more square matrices.
 //
-// Unpacks `num` tensors from `value` by chipping it along the `axis` dimension.
-// For example, given a tensor of shape `(A, B, C, D)`;
+// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
+// form square matrices.
 //
-// If `axis == 0` then the i'th tensor in `output` is the slice `value[i, :, :, :]`
-//   and each tensor in `output` will have shape `(B, C, D)`. (Note that the
-//   dimension unpacked along is gone, unlike `split`).
+// The input has to be invertible.
 //
-// If `axis == 1` then the i'th tensor in `output` is the slice `value[:, i, :, :]`
-//   and each tensor in `output` will have shape `(A, C, D)`.
-// Etc.
+// The output consists of two tensors LU and P containing the LU decomposition
+// of all input submatrices `[..., :, :]`. LU encodes the lower triangular and
+// upper triangular factors.
 //
-// This is the opposite of `pack`.
+// For each input submatrix of shape `[M, M]`, L is a lower triangular matrix of
+// shape `[M, M]` with unit diagonal whose entries correspond to the strictly lower
+// triangular part of LU. U is a upper triangular matrix of shape `[M, M]` whose
+// entries correspond to the upper triangular part, including the diagonal, of LU.
 //
-// Arguments:
-//	value: 1-D or higher, with `axis` dimension size equal to `num`.
+// P represents a permutation matrix encoded as a list of indices each between `0`
+// and `M-1`, inclusive. If P_mat denotes the permutation matrix corresponding to
+// P, then the L, U and P satisfies P_mat * input = L * U.
 //
+// Arguments:
+//	input: A tensor of shape `[..., M, M]` whose inner-most 2 dimensions form matrices of
+// size `[M, M]`.
 //
-// Returns The list of tensors unpacked from `value`.
-func Unpack(scope *Scope, value tf.Output, num int64, optional ...UnpackAttr) (output []tf.Output) {
+// Returns A tensor of shape `[..., M, M]` whose strictly lower triangular part denotes the
+// lower triangular factor `L` with unit diagonal, and whose upper triangular part
+// denotes the upper triangular factor `U`.Permutation of the rows encoded as a list of indices in `0..M-1`. Shape is
+// `[..., M]`.
+// @compatibility(scipy)
+// Similar to `scipy.linalg.lu`, except the triangular factors `L` and `U` are
+// packed into a single tensor, the permutation is applied to `input` instead of
+// the right hand side and the permutation `P` is returned as a list of indices
+// instead of a permutation matrix.
+// @end_compatibility
+func Lu(scope *Scope, input tf.Output, optional ...LuAttr) (lu tf.Output, p tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"num": num}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Unpack",
+		Type: "Lu",
 		Input: []tf.Input{
-			value,
+			input,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
+// Deprecated. Use TensorArrayCloseV3
+//
+// DEPRECATED at GraphDef version 26: Use TensorArrayCloseV3
+//
+// Returns the created operation.
+func TensorArrayCloseV2(scope *Scope, handle tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	var idx int
-	var err error
-	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
-		scope.UpdateErr("Unpack", err)
-		return
+	opspec := tf.OpSpec{
+		Type: "TensorArrayCloseV2",
+		Input: []tf.Input{
+			handle,
+		},
 	}
-	return output
+	return scope.AddOperation(opspec)
 }
 
-// Increments variable pointed to by 'resource' until it reaches 'limit'.
+// EncodeBase64Attr is an optional argument to EncodeBase64.
+type EncodeBase64Attr func(optionalAttr)
+
+// EncodeBase64Pad sets the optional pad attribute to value.
 //
-// Arguments:
-//	resource: Should be from a scalar `Variable` node.
-//	limit: If incrementing ref would bring it above limit, instead generates an
-// 'OutOfRange' error.
+// value: Bool whether padding is applied at the ends.
+// If not specified, defaults to false
+func EncodeBase64Pad(value bool) EncodeBase64Attr {
+	return func(m optionalAttr) {
+		m["pad"] = value
+	}
+}
+
+// Encode strings into web-safe base64 format.
+//
+// Refer to the following article for more information on base64 format:
+// en.wikipedia.org/wiki/Base64. Base64 strings may have padding with '=' at the
+// end so that the encoded has length multiple of 4. See Padding section of the
+// link above.
 //
+// Web-safe means that the encoder uses - and _ instead of + and /.
 //
-// Returns A copy of the input before increment. If nothing else modifies the
-// input, the values produced will all be distinct.
-func ResourceCountUpTo(scope *Scope, resource tf.Output, limit int64, T tf.DataType) (output tf.Output) {
+// Arguments:
+//	input: Strings to be encoded.
+//
+// Returns Input strings encoded in base64.
+func EncodeBase64(scope *Scope, input tf.Output, optional ...EncodeBase64Attr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"limit": limit, "T": T}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "ResourceCountUpTo",
+		Type: "EncodeBase64",
 		Input: []tf.Input{
-			resource,
+			input,
 		},
 		Attrs: attrs,
 	}
@@ -30034,79 +29704,126 @@ func ResourceCountUpTo(scope *Scope, resource tf.Output, limit int64, T tf.DataT
 	return op.Output(0)
 }
 
-// Delete the stack from its resource container.
+// A dataset that creates window datasets from the input dataset.
 //
 // Arguments:
-//	handle: The handle to a stack.
 //
-// Returns the created operation.
-func StackCloseV2(scope *Scope, handle tf.Output) (o *tf.Operation) {
+//	size: A scalar representing the number of elements to accumulate in a window.
+//	shift: A scalar representing the steps moving the sliding window forward in one
+// iteration. It must be positive.
+//	stride: A scalar representing the stride of the input elements of the sliding window.
+// It must be positive.
+//	drop_remainder: A scalar representing whether a window should be dropped in case its size is
+// smaller than desired.
+//
+//
+func WindowDataset(scope *Scope, input_dataset tf.Output, size tf.Output, shift tf.Output, stride tf.Output, drop_remainder tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "StackCloseV2",
+		Type: "WindowDataset",
 		Input: []tf.Input{
-			handle,
+			input_dataset, size, shift, stride, drop_remainder,
 		},
+		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Generate a glob pattern matching all sharded file names.
-func ShardedFilespec(scope *Scope, basename tf.Output, num_shards tf.Output) (filename tf.Output) {
+// Computes the matrix square root of one or more square matrices:
+//
+// matmul(sqrtm(A), sqrtm(A)) = A
+//
+// The input matrix should be invertible. If the input matrix is real, it should
+// have no eigenvalues which are real and negative (pairs of complex conjugate
+// eigenvalues are allowed).
+//
+// The matrix square root is computed by first reducing the matrix to
+// quasi-triangular form with the real Schur decomposition. The square root
+// of the quasi-triangular matrix is then computed directly. Details of
+// the algorithm can be found in: Nicholas J. Higham, "Computing real
+// square roots of a real matrix", Linear Algebra Appl., 1987.
+//
+// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
+// form square matrices. The output is a tensor of the same shape as the input
+// containing the matrix square root for all input submatrices `[..., :, :]`.
+//
+// Arguments:
+//	input: Shape is `[..., M, M]`.
+//
+// Returns Shape is `[..., M, M]`.
+//
+// @compatibility(scipy)
+// Equivalent to scipy.linalg.sqrtm
+// @end_compatibility
+func MatrixSquareRoot(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "ShardedFilespec",
+		Type: "MatrixSquareRoot",
 		Input: []tf.Input{
-			basename, num_shards,
+			input,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// TextLineReaderV2Attr is an optional argument to TextLineReaderV2.
-type TextLineReaderV2Attr func(optionalAttr)
+// SvdAttr is an optional argument to Svd.
+type SvdAttr func(optionalAttr)
 
-// TextLineReaderV2SkipHeaderLines sets the optional skip_header_lines attribute to value.
+// SvdComputeUv sets the optional compute_uv attribute to value.
 //
-// value: Number of lines to skip from the beginning of every file.
-// If not specified, defaults to 0
-func TextLineReaderV2SkipHeaderLines(value int64) TextLineReaderV2Attr {
+// value: If true, left and right singular vectors will be
+// computed and returned in `u` and `v`, respectively.
+// If false, `u` and `v` are not set and should never referenced.
+// If not specified, defaults to true
+func SvdComputeUv(value bool) SvdAttr {
 	return func(m optionalAttr) {
-		m["skip_header_lines"] = value
+		m["compute_uv"] = value
 	}
 }
 
-// TextLineReaderV2Container sets the optional container attribute to value.
+// SvdFullMatrices sets the optional full_matrices attribute to value.
 //
-// value: If non-empty, this reader is placed in the given container.
-// Otherwise, a default container is used.
-// If not specified, defaults to ""
-func TextLineReaderV2Container(value string) TextLineReaderV2Attr {
+// value: If true, compute full-sized `u` and `v`. If false
+// (the default), compute only the leading `P` singular vectors.
+// Ignored if `compute_uv` is `False`.
+// If not specified, defaults to false
+func SvdFullMatrices(value bool) SvdAttr {
 	return func(m optionalAttr) {
-		m["container"] = value
+		m["full_matrices"] = value
 	}
 }
 
-// TextLineReaderV2SharedName sets the optional shared_name attribute to value.
+// Computes the singular value decompositions of one or more matrices.
 //
-// value: If non-empty, this reader is named in the given bucket
-// with this shared_name. Otherwise, the node name is used instead.
-// If not specified, defaults to ""
-func TextLineReaderV2SharedName(value string) TextLineReaderV2Attr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// A Reader that outputs the lines of a file delimited by '\n'.
+// Computes the SVD of each inner matrix in `input` such that
+// `input[..., :, :] = u[..., :, :] * diag(s[..., :, :]) * transpose(v[..., :, :])`
 //
-// Returns The handle to reference the Reader.
-func TextLineReaderV2(scope *Scope, optional ...TextLineReaderV2Attr) (reader_handle tf.Output) {
+// ```python
+// # a is a tensor containing a batch of matrices.
+// # s is a tensor of singular values for each matrix.
+// # u is the tensor containing of left singular vectors for each matrix.
+// # v is the tensor containing of right singular vectors for each matrix.
+// s, u, v = svd(a)
+// s, _, _ = svd(a, compute_uv=False)
+// ```
+//
+// Arguments:
+//	input: A tensor of shape `[..., M, N]` whose inner-most 2 dimensions
+// form matrices of size `[M, N]`. Let `P` be the minimum of `M` and `N`.
+//
+// Returns Singular values. Shape is `[..., P]`.Left singular vectors. If `full_matrices` is `False` then shape is
+// `[..., M, P]`; if `full_matrices` is `True` then shape is
+// `[..., M, M]`. Undefined if `compute_uv` is `False`.Left singular vectors. If `full_matrices` is `False` then shape is
+// `[..., N, P]`. If `full_matrices` is `True` then shape is `[..., N, N]`.
+// Undefined if `compute_uv` is false.
+func Svd(scope *Scope, input tf.Output, optional ...SvdAttr) (s tf.Output, u tf.Output, v tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -30115,231 +29832,125 @@ func TextLineReaderV2(scope *Scope, optional ...TextLineReaderV2Attr) (reader_ha
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "TextLineReaderV2",
-
+		Type: "Svd",
+		Input: []tf.Input{
+			input,
+		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// LoadAndRemapMatrixAttr is an optional argument to LoadAndRemapMatrix.
-type LoadAndRemapMatrixAttr func(optionalAttr)
-
-// LoadAndRemapMatrixMaxRowsInMemory sets the optional max_rows_in_memory attribute to value.
-//
-// value: The maximum number of rows to load from the checkpoint at
-// once. If less than or equal to 0, the entire matrix will be loaded into
-// memory. Setting this arg trades increased disk reads for lower memory usage.
-// If not specified, defaults to -1
-func LoadAndRemapMatrixMaxRowsInMemory(value int64) LoadAndRemapMatrixAttr {
-	return func(m optionalAttr) {
-		m["max_rows_in_memory"] = value
-	}
-}
-
-// Loads a 2-D (matrix) `Tensor` with name `old_tensor_name` from the checkpoint
-//
-// at `ckpt_path` and potentially reorders its rows and columns using the
-// specified remappings.
-//
-// Most users should use one of the wrapper initializers (such as
-// `tf.contrib.framework.load_and_remap_matrix_initializer`) instead of this
-// function directly.
-//
-// The remappings are 1-D tensors with the following properties:
-//
-// * `row_remapping` must have exactly `num_rows` entries. Row `i` of the output
-//   matrix will be initialized from the row corresponding to index
-//   `row_remapping[i]` in the old `Tensor` from the checkpoint.
-// * `col_remapping` must have either 0 entries (indicating that no column
-//   reordering is needed) or `num_cols` entries. If specified, column `j` of the
-//   output matrix will be initialized from the column corresponding to index
-//   `col_remapping[j]` in the old `Tensor` from the checkpoint.
-// * A value of -1 in either of the remappings signifies a "missing" entry. In that
-//   case, values from the `initializing_values` tensor will be used to fill that
-//   missing row or column. If `row_remapping` has `r` missing entries and
-//   `col_remapping` has `c` missing entries, then the following condition must be
-//   true:
+// Computes the product along segments of a tensor.
 //
-// `(r * num_cols) + (c * num_rows) - (r * c) == len(initializing_values)`
+// Read
+// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+// for an explanation of segments.
 //
-// The remapping tensors can be generated using the GenerateVocabRemapping op.
+// Computes a tensor such that
+// \\(output_i = \prod_j data_j\\) where the product is over `j` such
+// that `segment_ids[j] == i`.
 //
-// As an example, with row_remapping = [1, 0, -1], col_remapping = [0, 2, -1],
-// initializing_values = [0.5, -0.5, 0.25, -0.25, 42], and w(i, j) representing
-// the value from row i, column j of the old tensor in the checkpoint, the output
-// matrix will look like the following:
+// If the product is empty for a given segment ID `i`, `output[i] = 1`.
 //
-// [[w(1, 0),  w(1, 2),  0.5],
-//  [w(0, 0),  w(0, 2), -0.5],
-//  [0.25,    -0.25,      42]]
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/SegmentProd.png" alt>
+// </div>
 //
 // Arguments:
-//	ckpt_path: Path to the TensorFlow checkpoint (version 2, `TensorBundle`) from
-// which the old matrix `Tensor` will be loaded.
-//	old_tensor_name: Name of the 2-D `Tensor` to load from checkpoint.
-//	row_remapping: An int `Tensor` of row remappings (generally created by
-// `generate_vocab_remapping`).  Even if no row remapping is needed, this must
-// still be an index-valued Tensor (e.g. [0, 1, 2, ...]), or a shifted
-// index-valued `Tensor` (e.g. [8, 9, 10, ...], for partitioned `Variables`).
-//	col_remapping: An int `Tensor` of column remappings (generally created by
-// `generate_vocab_remapping`).  May be a size-0 `Tensor` if only row remapping
-// is to be done (e.g. column ordering is the same).
-//	initializing_values: A float `Tensor` containing  values to fill in for cells
-// in the output matrix that are not loaded from the checkpoint. Length must be
-// exactly the same as the number of missing / new cells.
-//	num_rows: Number of rows (length of the 1st dimension) in the output matrix.
-//	num_cols: Number of columns (length of the 2nd dimension) in the output matrix.
 //
-// Returns Output matrix containing existing values loaded from the
-// checkpoint, and with any missing values filled in from initializing_values.
-func LoadAndRemapMatrix(scope *Scope, ckpt_path tf.Output, old_tensor_name tf.Output, row_remapping tf.Output, col_remapping tf.Output, initializing_values tf.Output, num_rows int64, num_cols int64, optional ...LoadAndRemapMatrixAttr) (output_matrix tf.Output) {
+//	segment_ids: A 1-D tensor whose size is equal to the size of `data`'s
+// first dimension.  Values should be sorted and can be repeated.
+//
+// Returns Has same shape as data, except for dimension 0 which
+// has size `k`, the number of segments.
+func SegmentProd(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"num_rows": num_rows, "num_cols": num_cols}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "LoadAndRemapMatrix",
+		Type: "SegmentProd",
 		Input: []tf.Input{
-			ckpt_path, old_tensor_name, row_remapping, col_remapping, initializing_values,
+			data, segment_ids,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// TFRecordReaderV2Attr is an optional argument to TFRecordReaderV2.
-type TFRecordReaderV2Attr func(optionalAttr)
-
-// TFRecordReaderV2Container sets the optional container attribute to value.
+// Converts one or more images from RGB to HSV.
 //
-// value: If non-empty, this reader is placed in the given container.
-// Otherwise, a default container is used.
-// If not specified, defaults to ""
-func TFRecordReaderV2Container(value string) TFRecordReaderV2Attr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// TFRecordReaderV2SharedName sets the optional shared_name attribute to value.
+// Outputs a tensor of the same shape as the `images` tensor, containing the HSV
+// value of the pixels. The output is only well defined if the value in `images`
+// are in `[0,1]`.
 //
-// value: If non-empty, this reader is named in the given bucket
-// with this shared_name. Otherwise, the node name is used instead.
-// If not specified, defaults to ""
-func TFRecordReaderV2SharedName(value string) TFRecordReaderV2Attr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// TFRecordReaderV2CompressionType sets the optional compression_type attribute to value.
-// If not specified, defaults to ""
-func TFRecordReaderV2CompressionType(value string) TFRecordReaderV2Attr {
-	return func(m optionalAttr) {
-		m["compression_type"] = value
-	}
-}
-
-// A Reader that outputs the records from a TensorFlow Records file.
+// `output[..., 0]` contains hue, `output[..., 1]` contains saturation, and
+// `output[..., 2]` contains value. All HSV values are in `[0,1]`. A hue of 0
+// corresponds to pure red, hue 1/3 is pure green, and 2/3 is pure blue.
 //
-// Returns The handle to reference the Reader.
-func TFRecordReaderV2(scope *Scope, optional ...TFRecordReaderV2Attr) (reader_handle tf.Output) {
+// Arguments:
+//	images: 1-D or higher rank. RGB data to convert. Last dimension must be size 3.
+//
+// Returns `images` converted to HSV.
+func RGBToHSV(scope *Scope, images tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "TFRecordReaderV2",
-
-		Attrs: attrs,
+		Type: "RGBToHSV",
+		Input: []tf.Input{
+			images,
+		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// QuantizeAndDequantizeV3Attr is an optional argument to QuantizeAndDequantizeV3.
-type QuantizeAndDequantizeV3Attr func(optionalAttr)
-
-// QuantizeAndDequantizeV3SignedInput sets the optional signed_input attribute to value.
-// If not specified, defaults to true
-func QuantizeAndDequantizeV3SignedInput(value bool) QuantizeAndDequantizeV3Attr {
-	return func(m optionalAttr) {
-		m["signed_input"] = value
-	}
-}
-
-// QuantizeAndDequantizeV3RangeGiven sets the optional range_given attribute to value.
-// If not specified, defaults to true
-func QuantizeAndDequantizeV3RangeGiven(value bool) QuantizeAndDequantizeV3Attr {
-	return func(m optionalAttr) {
-		m["range_given"] = value
-	}
-}
-
-// Quantizes then dequantizes a tensor.
+// Does nothing. Only useful as a placeholder for control edges.
 //
-// This is almost identical to QuantizeAndDequantizeV2, except that num_bits is a
-// tensor, so its value can change during training.
-func QuantizeAndDequantizeV3(scope *Scope, input tf.Output, input_min tf.Output, input_max tf.Output, num_bits tf.Output, optional ...QuantizeAndDequantizeV3Attr) (output tf.Output) {
+// Returns the created operation.
+func NoOp(scope *Scope) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "QuantizeAndDequantizeV3",
-		Input: []tf.Input{
-			input, input_min, input_max, num_bits,
-		},
-		Attrs: attrs,
+		Type: "NoOp",
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// IdentityReaderV2Attr is an optional argument to IdentityReaderV2.
-type IdentityReaderV2Attr func(optionalAttr)
+// MergeV2CheckpointsAttr is an optional argument to MergeV2Checkpoints.
+type MergeV2CheckpointsAttr func(optionalAttr)
 
-// IdentityReaderV2Container sets the optional container attribute to value.
+// MergeV2CheckpointsDeleteOldDirs sets the optional delete_old_dirs attribute to value.
 //
-// value: If non-empty, this reader is placed in the given container.
-// Otherwise, a default container is used.
-// If not specified, defaults to ""
-func IdentityReaderV2Container(value string) IdentityReaderV2Attr {
+// value: see above.
+// If not specified, defaults to true
+func MergeV2CheckpointsDeleteOldDirs(value bool) MergeV2CheckpointsAttr {
 	return func(m optionalAttr) {
-		m["container"] = value
+		m["delete_old_dirs"] = value
 	}
 }
 
-// IdentityReaderV2SharedName sets the optional shared_name attribute to value.
+// V2 format specific: merges the metadata files of sharded checkpoints.  The
 //
-// value: If non-empty, this reader is named in the given bucket
-// with this shared_name. Otherwise, the node name is used instead.
-// If not specified, defaults to ""
-func IdentityReaderV2SharedName(value string) IdentityReaderV2Attr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// A Reader that outputs the queued work as both the key and value.
+// result is one logical checkpoint, with one physical metadata file and renamed
+// data files.
 //
-// To use, enqueue strings in a Queue.  ReaderRead will take the front
-// work string and output (work, work).
+// Intended for "grouping" multiple checkpoints in a sharded checkpoint setup.
 //
-// Returns The handle to reference the Reader.
-func IdentityReaderV2(scope *Scope, optional ...IdentityReaderV2Attr) (reader_handle tf.Output) {
+// If delete_old_dirs is true, attempts to delete recursively the dirname of each
+// path in the input checkpoint_prefixes.  This is useful when those paths are non
+// user-facing temporary locations.
+//
+// Arguments:
+//	checkpoint_prefixes: prefixes of V2 checkpoints to merge.
+//	destination_prefix: scalar.  The desired final prefix.  Allowed to be the same
+// as one of the checkpoint_prefixes.
+//
+// Returns the created operation.
+func MergeV2Checkpoints(scope *Scope, checkpoint_prefixes tf.Output, destination_prefix tf.Output, optional ...MergeV2CheckpointsAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -30348,242 +29959,310 @@ func IdentityReaderV2(scope *Scope, optional ...IdentityReaderV2Attr) (reader_ha
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "IdentityReaderV2",
-
+		Type: "MergeV2Checkpoints",
+		Input: []tf.Input{
+			checkpoint_prefixes, destination_prefix,
+		},
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// ResourceApplyGradientDescentAttr is an optional argument to ResourceApplyGradientDescent.
-type ResourceApplyGradientDescentAttr func(optionalAttr)
-
-// ResourceApplyGradientDescentUseLocking sets the optional use_locking attribute to value.
-//
-// value: If `True`, the subtraction will be protected by a lock;
-// otherwise the behavior is undefined, but may exhibit less contention.
-// If not specified, defaults to false
-func ResourceApplyGradientDescentUseLocking(value bool) ResourceApplyGradientDescentAttr {
-	return func(m optionalAttr) {
-		m["use_locking"] = value
-	}
+	return scope.AddOperation(opspec)
 }
 
-// Update '*var' by subtracting 'alpha' * 'delta' from it.
+// Saves input tensors slices to disk.
 //
-// Arguments:
-//	var_: Should be from a Variable().
-//	alpha: Scaling factor. Must be a scalar.
-//	delta: The change.
+// This is like `Save` except that tensors can be listed in the saved file as being
+// a slice of a larger tensor.  `shapes_and_slices` specifies the shape of the
+// larger tensor and the slice that this tensor covers. `shapes_and_slices` must
+// have as many elements as `tensor_names`.
+//
+// Elements of the `shapes_and_slices` input must either be:
+//
+// *  The empty string, in which case the corresponding tensor is
+//    saved normally.
+// *  A string of the form `dim0 dim1 ... dimN-1 slice-spec` where the
+//    `dimI` are the dimensions of the larger tensor and `slice-spec`
+//    specifies what part is covered by the tensor to save.
+//
+// `slice-spec` itself is a `:`-separated list: `slice0:slice1:...:sliceN-1`
+// where each `sliceI` is either:
+//
+// *  The string `-` meaning that the slice covers all indices of this dimension
+// *  `start,length` where `start` and `length` are integers.  In that
+//    case the slice covers `length` indices starting at `start`.
+//
+// See also `Save`.
+//
+// Arguments:
+//	filename: Must have a single element. The name of the file to which we write the
+// tensor.
+//	tensor_names: Shape `[N]`. The names of the tensors to be saved.
+//	shapes_and_slices: Shape `[N]`.  The shapes and slice specifications to use when
+// saving the tensors.
+//	data: `N` tensors to save.
 //
 // Returns the created operation.
-func ResourceApplyGradientDescent(scope *Scope, var_ tf.Output, alpha tf.Output, delta tf.Output, optional ...ResourceApplyGradientDescentAttr) (o *tf.Operation) {
+func SaveSlices(scope *Scope, filename tf.Output, tensor_names tf.Output, shapes_and_slices tf.Output, data []tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "ResourceApplyGradientDescent",
+		Type: "SaveSlices",
 		Input: []tf.Input{
-			var_, alpha, delta,
+			filename, tensor_names, shapes_and_slices, tf.OutputList(data),
 		},
-		Attrs: attrs,
 	}
 	return scope.AddOperation(opspec)
 }
 
-// Returns the next record (key, value pair) produced by a Reader.
+// DenseToDenseSetOperationAttr is an optional argument to DenseToDenseSetOperation.
+type DenseToDenseSetOperationAttr func(optionalAttr)
+
+// DenseToDenseSetOperationValidateIndices sets the optional validate_indices attribute to value.
+// If not specified, defaults to true
+func DenseToDenseSetOperationValidateIndices(value bool) DenseToDenseSetOperationAttr {
+	return func(m optionalAttr) {
+		m["validate_indices"] = value
+	}
+}
+
+// Applies set operation along last dimension of 2 `Tensor` inputs.
 //
-// Will dequeue from the input queue if necessary (e.g. when the
-// Reader needs to start reading from a new file since it has finished
-// with the previous file).
+// See SetOperationOp::SetOperationFromContext for values of `set_operation`.
+//
+// Output `result` is a `SparseTensor` represented by `result_indices`,
+// `result_values`, and `result_shape`. For `set1` and `set2` ranked `n`, this
+// has rank `n` and the same 1st `n-1` dimensions as `set1` and `set2`. The `nth`
+// dimension contains the result of `set_operation` applied to the corresponding
+// `[0...n-1]` dimension of `set`.
 //
 // Arguments:
-//	reader_handle: Handle to a Reader.
-//	queue_handle: Handle to a Queue, with string work items.
+//	set1: `Tensor` with rank `n`. 1st `n-1` dimensions must be the same as `set2`.
+// Dimension `n` contains values in a set, duplicates are allowed but ignored.
+//	set2: `Tensor` with rank `n`. 1st `n-1` dimensions must be the same as `set1`.
+// Dimension `n` contains values in a set, duplicates are allowed but ignored.
 //
-// Returns A scalar.A scalar.
-func ReaderReadV2(scope *Scope, reader_handle tf.Output, queue_handle tf.Output) (key tf.Output, value tf.Output) {
+//
+// Returns 2D indices of a `SparseTensor`.1D values of a `SparseTensor`.1D `Tensor` shape of a `SparseTensor`. `result_shape[0...n-1]` is
+// the same as the 1st `n-1` dimensions of `set1` and `set2`, `result_shape[n]`
+// is the max result set size across all `0...n-1` dimensions.
+func DenseToDenseSetOperation(scope *Scope, set1 tf.Output, set2 tf.Output, set_operation string, optional ...DenseToDenseSetOperationAttr) (result_indices tf.Output, result_values tf.Output, result_shape tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"set_operation": set_operation}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "ReaderReadV2",
+		Type: "DenseToDenseSetOperation",
 		Input: []tf.Input{
-			reader_handle, queue_handle,
+			set1, set2,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Returns up to `num_records` (key, value) pairs produced by a Reader.
-//
-// Will dequeue from the input queue if necessary (e.g. when the
-// Reader needs to start reading from a new file since it has finished
-// with the previous file).
-// It may return less than `num_records` even before the last batch.
-//
-// Arguments:
-//	reader_handle: Handle to a `Reader`.
-//	queue_handle: Handle to a `Queue`, with string work items.
-//	num_records: number of records to read from `Reader`.
+// Generate a sharded filename. The filename is printf formatted as
 //
-// Returns A 1-D tensor.A 1-D tensor.
-func ReaderReadUpToV2(scope *Scope, reader_handle tf.Output, queue_handle tf.Output, num_records tf.Output) (keys tf.Output, values tf.Output) {
+//    %s-%05d-of-%05d, basename, shard, num_shards.
+func ShardedFilename(scope *Scope, basename tf.Output, shard tf.Output, num_shards tf.Output) (filename tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "ReaderReadUpToV2",
+		Type: "ShardedFilename",
 		Input: []tf.Input{
-			reader_handle, queue_handle, num_records,
+			basename, shard, num_shards,
 		},
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-//     Adds v into specified rows of x.
+// BatchToSpace for N-D tensors of type T.
 //
-//     Computes y = x; y[i, :] += v; return y.
+// This operation reshapes the "batch" dimension 0 into `M + 1` dimensions of shape
+// `block_shape + [batch]`, interleaves these blocks back into the grid defined by
+// the spatial dimensions `[1, ..., M]`, to obtain a result with the same rank as
+// the input.  The spatial dimensions of this intermediate result are then
+// optionally cropped according to `crops` to produce the output.  This is the
+// reverse of SpaceToBatch.  See below for a precise description.
 //
 // Arguments:
-//	x: A `Tensor` of type T.
-//	i: A vector. Indices into the left-most dimension of `x`.
-//	v: A `Tensor` of type T. Same dimension sizes as x except the first dimension, which must be the same as i's size.
+//	input: N-D with shape `input_shape = [batch] + spatial_shape + remaining_shape`,
+// where spatial_shape has M dimensions.
+//	block_shape: 1-D with shape `[M]`, all values must be >= 1.
+//	crops: 2-D with shape `[M, 2]`, all values must be >= 0.
+//   `crops[i] = [crop_start, crop_end]` specifies the amount to crop from input
+//   dimension `i + 1`, which corresponds to spatial dimension `i`.  It is
+//   required that
+//   `crop_start[i] + crop_end[i] <= block_shape[i] * input_shape[i + 1]`.
 //
-// Returns A `Tensor` of type T. An alias of `x`. The content of `y` is undefined if there are duplicates in `i`.
-func InplaceAdd(scope *Scope, x tf.Output, i tf.Output, v tf.Output) (y tf.Output) {
+// This operation is equivalent to the following steps:
+//
+// 1. Reshape `input` to `reshaped` of shape:
+//      [block_shape[0], ..., block_shape[M-1],
+//       batch / prod(block_shape),
+//       input_shape[1], ..., input_shape[N-1]]
+//
+// 2. Permute dimensions of `reshaped` to produce `permuted` of shape
+//      [batch / prod(block_shape),
+//
+//       input_shape[1], block_shape[0],
+//       ...,
+//       input_shape[M], block_shape[M-1],
+//
+//       input_shape[M+1], ..., input_shape[N-1]]
+//
+// 3. Reshape `permuted` to produce `reshaped_permuted` of shape
+//      [batch / prod(block_shape),
+//
+//       input_shape[1] * block_shape[0],
+//       ...,
+//       input_shape[M] * block_shape[M-1],
+//
+//       input_shape[M+1],
+//       ...,
+//       input_shape[N-1]]
+//
+// 4. Crop the start and end of dimensions `[1, ..., M]` of
+//    `reshaped_permuted` according to `crops` to produce the output of shape:
+//      [batch / prod(block_shape),
+//
+//       input_shape[1] * block_shape[0] - crops[0,0] - crops[0,1],
+//       ...,
+//       input_shape[M] * block_shape[M-1] - crops[M-1,0] - crops[M-1,1],
+//
+//       input_shape[M+1], ..., input_shape[N-1]]
+//
+// Some examples:
+//
+// (1) For the following input of shape `[4, 1, 1, 1]`, `block_shape = [2, 2]`, and
+//     `crops = [[0, 0], [0, 0]]`:
+//
+// ```
+// [[[[1]]], [[[2]]], [[[3]]], [[[4]]]]
+// ```
+//
+// The output tensor has shape `[1, 2, 2, 1]` and value:
+//
+// ```
+// x = [[[[1], [2]], [[3], [4]]]]
+// ```
+//
+// (2) For the following input of shape `[4, 1, 1, 3]`, `block_shape = [2, 2]`, and
+//     `crops = [[0, 0], [0, 0]]`:
+//
+// ```
+// [[[1, 2, 3]], [[4, 5, 6]], [[7, 8, 9]], [[10, 11, 12]]]
+// ```
+//
+// The output tensor has shape `[1, 2, 2, 3]` and value:
+//
+// ```
+// x = [[[[1, 2, 3], [4, 5, 6]],
+//       [[7, 8, 9], [10, 11, 12]]]]
+// ```
+//
+// (3) For the following input of shape `[4, 2, 2, 1]`, `block_shape = [2, 2]`, and
+//     `crops = [[0, 0], [0, 0]]`:
+//
+// ```
+// x = [[[[1], [3]], [[9], [11]]],
+//      [[[2], [4]], [[10], [12]]],
+//      [[[5], [7]], [[13], [15]]],
+//      [[[6], [8]], [[14], [16]]]]
+// ```
+//
+// The output tensor has shape `[1, 4, 4, 1]` and value:
+//
+// ```
+// x = [[[1],   [2],  [3],  [4]],
+//      [[5],   [6],  [7],  [8]],
+//      [[9],  [10], [11],  [12]],
+//      [[13], [14], [15],  [16]]]
+// ```
+//
+// (4) For the following input of shape `[8, 1, 3, 1]`, `block_shape = [2, 2]`, and
+//     `crops = [[0, 0], [2, 0]]`:
+//
+// ```
+// x = [[[[0], [1], [3]]], [[[0], [9], [11]]],
+//      [[[0], [2], [4]]], [[[0], [10], [12]]],
+//      [[[0], [5], [7]]], [[[0], [13], [15]]],
+//      [[[0], [6], [8]]], [[[0], [14], [16]]]]
+// ```
+//
+// The output tensor has shape `[2, 2, 4, 1]` and value:
+//
+// ```
+// x = [[[[1],   [2],  [3],  [4]],
+//       [[5],   [6],  [7],  [8]]],
+//      [[[9],  [10], [11],  [12]],
+//       [[13], [14], [15],  [16]]]]
+// ```
+func BatchToSpaceND(scope *Scope, input tf.Output, block_shape tf.Output, crops tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "InplaceAdd",
+		Type: "BatchToSpaceND",
 		Input: []tf.Input{
-			x, i, v,
+			input, block_shape, crops,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Restore a Reader to its initial clean state.
-//
-// Arguments:
-//	reader_handle: Handle to a Reader.
+// UnpackAttr is an optional argument to Unpack.
+type UnpackAttr func(optionalAttr)
+
+// UnpackAxis sets the optional axis attribute to value.
 //
-// Returns the created operation.
-func ReaderResetV2(scope *Scope, reader_handle tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ReaderResetV2",
-		Input: []tf.Input{
-			reader_handle,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
-// BatchAttr is an optional argument to Batch.
-type BatchAttr func(optionalAttr)
-
-// BatchMaxEnqueuedBatches sets the optional max_enqueued_batches attribute to value.
-// If not specified, defaults to 10
-func BatchMaxEnqueuedBatches(value int64) BatchAttr {
-	return func(m optionalAttr) {
-		m["max_enqueued_batches"] = value
-	}
-}
-
-// BatchAllowedBatchSizes sets the optional allowed_batch_sizes attribute to value.
-// If not specified, defaults to <>
-func BatchAllowedBatchSizes(value []int64) BatchAttr {
-	return func(m optionalAttr) {
-		m["allowed_batch_sizes"] = value
-	}
-}
-
-// BatchContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func BatchContainer(value string) BatchAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// BatchSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func BatchSharedName(value string) BatchAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// BatchBatchingQueue sets the optional batching_queue attribute to value.
-// If not specified, defaults to ""
-func BatchBatchingQueue(value string) BatchAttr {
+// value: Dimension along which to unpack.  Negative values wrap around, so the
+// valid range is `[-R, R)`.
+// If not specified, defaults to 0
+func UnpackAxis(value int64) UnpackAttr {
 	return func(m optionalAttr) {
-		m["batching_queue"] = value
+		m["axis"] = value
 	}
 }
 
-// Batches all input tensors nondeterministically.
+// Unpacks a given dimension of a rank-`R` tensor into `num` rank-`(R-1)` tensors.
 //
-// When many instances of this Op are being run concurrently with the same
-// container/shared_name in the same device, some will output zero-shaped Tensors
-// and others will output Tensors of size up to max_batch_size.
+// Unpacks `num` tensors from `value` by chipping it along the `axis` dimension.
+// For example, given a tensor of shape `(A, B, C, D)`;
 //
-// All Tensors in in_tensors are batched together (so, for example, labels and
-// features should be batched with a single instance of this operation.
+// If `axis == 0` then the i'th tensor in `output` is the slice `value[i, :, :, :]`
+//   and each tensor in `output` will have shape `(B, C, D)`. (Note that the
+//   dimension unpacked along is gone, unlike `split`).
 //
-// Each invocation of batch emits an `id` scalar which will be used to identify
-// this particular invocation when doing unbatch or its gradient.
+// If `axis == 1` then the i'th tensor in `output` is the slice `value[:, i, :, :]`
+//   and each tensor in `output` will have shape `(A, C, D)`.
+// Etc.
 //
-// Each op which emits a non-empty batch will also emit a non-empty batch_index
-// Tensor, which, is a [K, 3] matrix where each row contains the invocation's id,
-// start, and length of elements of each set of Tensors present in batched_tensors.
+// This is the opposite of `pack`.
 //
-// Batched tensors are concatenated along the first dimension, and all tensors in
-// in_tensors must have the first dimension of the same size.
+// Arguments:
+//	value: 1-D or higher, with `axis` dimension size equal to `num`.
 //
-// in_tensors: The tensors to be batched.
-// num_batch_threads: Number of scheduling threads for processing batches of work.
-//  Determines the number of batches processed in parallel.
-// max_batch_size: Batch sizes will never be bigger than this.
-// batch_timeout_micros: Maximum number of microseconds to wait before outputting
-//  an incomplete batch.
-// allowed_batch_sizes: Optional list of allowed batch sizes. If left empty, does
-//  nothing. Otherwise, supplies a list of batch sizes, causing the op to pad
-//  batches up to one of those sizes. The entries must increase monotonically, and
-//  the final entry must equal max_batch_size.
-// grad_timeout_micros: The timeout to use for the gradient. See Unbatch.
-// batched_tensors: Either empty tensors or a batch of concatenated Tensors.
-// batch_index: If out_tensors is non-empty, has information to invert it.
-// container: Controls the scope of sharing of this batch.
-// id: always contains a scalar with a unique ID for this invocation of Batch.
-// shared_name: Concurrently running instances of batch in the same device with the
-//  same container and shared_name will batch their elements together. If left
-//  empty, the op name will be used as the shared name.
-// T: the types of tensors to be batched.
-func Batch(scope *Scope, in_tensors []tf.Output, num_batch_threads int64, max_batch_size int64, batch_timeout_micros int64, grad_timeout_micros int64, optional ...BatchAttr) (batched_tensors []tf.Output, batch_index tf.Output, id tf.Output) {
+//
+// Returns The list of tensors unpacked from `value`.
+func Unpack(scope *Scope, value tf.Output, num int64, optional ...UnpackAttr) (output []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"num_batch_threads": num_batch_threads, "max_batch_size": max_batch_size, "batch_timeout_micros": batch_timeout_micros, "grad_timeout_micros": grad_timeout_micros}
+	attrs := map[string]interface{}{"num": num}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Batch",
+		Type: "Unpack",
 		Input: []tf.Input{
-			tf.OutputList(in_tensors),
+			value,
 		},
 		Attrs: attrs,
 	}
@@ -30593,110 +30272,112 @@ func Batch(scope *Scope, in_tensors []tf.Output, num_batch_threads int64, max_ba
 	}
 	var idx int
 	var err error
-	if batched_tensors, idx, err = makeOutputList(op, idx, "batched_tensors"); err != nil {
-		scope.UpdateErr("Batch", err)
+	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
+		scope.UpdateErr("Unpack", err)
 		return
 	}
-	batch_index = op.Output(idx)
-	id = op.Output(idx)
-	return batched_tensors, batch_index, id
+	return output
 }
 
-// Adjust the hue of one or more images.
-//
-// `images` is a tensor of at least 3 dimensions.  The last dimension is
-// interpretted as channels, and must be three.
-//
-// The input image is considered in the RGB colorspace. Conceptually, the RGB
-// colors are first mapped into HSV. A delta is then applied all the hue values,
-// and then remapped back to RGB colorspace.
+// Increments variable pointed to by 'resource' until it reaches 'limit'.
 //
 // Arguments:
-//	images: Images to adjust.  At least 3-D.
-//	delta: A float delta to add to the hue.
+//	resource: Should be from a scalar `Variable` node.
+//	limit: If incrementing ref would bring it above limit, instead generates an
+// 'OutOfRange' error.
 //
-// Returns The hue-adjusted image or images.
-func AdjustHue(scope *Scope, images tf.Output, delta tf.Output) (output tf.Output) {
+//
+// Returns A copy of the input before increment. If nothing else modifies the
+// input, the values produced will all be distinct.
+func ResourceCountUpTo(scope *Scope, resource tf.Output, limit int64, T tf.DataType) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"limit": limit, "T": T}
 	opspec := tf.OpSpec{
-		Type: "AdjustHue",
+		Type: "ResourceCountUpTo",
 		Input: []tf.Input{
-			images, delta,
+			resource,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ResizeBicubicGradAttr is an optional argument to ResizeBicubicGrad.
-type ResizeBicubicGradAttr func(optionalAttr)
-
-// ResizeBicubicGradAlignCorners sets the optional align_corners attribute to value.
-//
-// value: If true, the centers of the 4 corner pixels of the input and grad tensors are
-// aligned. Defaults to false.
-// If not specified, defaults to false
-func ResizeBicubicGradAlignCorners(value bool) ResizeBicubicGradAttr {
-	return func(m optionalAttr) {
-		m["align_corners"] = value
-	}
-}
-
-// Computes the gradient of bicubic interpolation.
+// Delete the stack from its resource container.
 //
 // Arguments:
-//	grads: 4-D with shape `[batch, height, width, channels]`.
-//	original_image: 4-D with shape `[batch, orig_height, orig_width, channels]`,
-// The image tensor that was resized.
+//	handle: The handle to a stack.
 //
-// Returns 4-D with shape `[batch, orig_height, orig_width, channels]`.
-// Gradients with respect to the input image. Input image must have been
-// float or double.
-func ResizeBicubicGrad(scope *Scope, grads tf.Output, original_image tf.Output, optional ...ResizeBicubicGradAttr) (output tf.Output) {
+// Returns the created operation.
+func StackCloseV2(scope *Scope, handle tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
+	opspec := tf.OpSpec{
+		Type: "StackCloseV2",
+		Input: []tf.Input{
+			handle,
+		},
+	}
+	return scope.AddOperation(opspec)
+}
+
+// Generate a glob pattern matching all sharded file names.
+func ShardedFilespec(scope *Scope, basename tf.Output, num_shards tf.Output) (filename tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
 	opspec := tf.OpSpec{
-		Type: "ResizeBicubicGrad",
+		Type: "ShardedFilespec",
 		Input: []tf.Input{
-			grads, original_image,
+			basename, num_shards,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ResizeNearestNeighborAttr is an optional argument to ResizeNearestNeighbor.
-type ResizeNearestNeighborAttr func(optionalAttr)
+// TextLineReaderV2Attr is an optional argument to TextLineReaderV2.
+type TextLineReaderV2Attr func(optionalAttr)
 
-// ResizeNearestNeighborAlignCorners sets the optional align_corners attribute to value.
+// TextLineReaderV2SkipHeaderLines sets the optional skip_header_lines attribute to value.
 //
-// value: If true, the centers of the 4 corner pixels of the input and output tensors are
-// aligned, preserving the values at the corner pixels. Defaults to false.
-// If not specified, defaults to false
-func ResizeNearestNeighborAlignCorners(value bool) ResizeNearestNeighborAttr {
+// value: Number of lines to skip from the beginning of every file.
+// If not specified, defaults to 0
+func TextLineReaderV2SkipHeaderLines(value int64) TextLineReaderV2Attr {
 	return func(m optionalAttr) {
-		m["align_corners"] = value
+		m["skip_header_lines"] = value
 	}
 }
 
-// Resize `images` to `size` using nearest neighbor interpolation.
+// TextLineReaderV2Container sets the optional container attribute to value.
 //
-// Arguments:
-//	images: 4-D with shape `[batch, height, width, channels]`.
-//	size: = A 1-D int32 Tensor of 2 elements: `new_height, new_width`.  The
-// new size for the images.
+// value: If non-empty, this reader is placed in the given container.
+// Otherwise, a default container is used.
+// If not specified, defaults to ""
+func TextLineReaderV2Container(value string) TextLineReaderV2Attr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// TextLineReaderV2SharedName sets the optional shared_name attribute to value.
 //
-// Returns 4-D with shape
-// `[batch, new_height, new_width, channels]`.
-func ResizeNearestNeighbor(scope *Scope, images tf.Output, size tf.Output, optional ...ResizeNearestNeighborAttr) (resized_images tf.Output) {
+// value: If non-empty, this reader is named in the given bucket
+// with this shared_name. Otherwise, the node name is used instead.
+// If not specified, defaults to ""
+func TextLineReaderV2SharedName(value string) TextLineReaderV2Attr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// A Reader that outputs the lines of a file delimited by '\n'.
+//
+// Returns The handle to reference the Reader.
+func TextLineReaderV2(scope *Scope, optional ...TextLineReaderV2Attr) (reader_handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -30705,51 +30386,97 @@ func ResizeNearestNeighbor(scope *Scope, images tf.Output, size tf.Output, optio
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResizeNearestNeighbor",
-		Input: []tf.Input{
-			images, size,
-		},
+		Type: "TextLineReaderV2",
+
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ResizeNearestNeighborGradAttr is an optional argument to ResizeNearestNeighborGrad.
-type ResizeNearestNeighborGradAttr func(optionalAttr)
+// LoadAndRemapMatrixAttr is an optional argument to LoadAndRemapMatrix.
+type LoadAndRemapMatrixAttr func(optionalAttr)
 
-// ResizeNearestNeighborGradAlignCorners sets the optional align_corners attribute to value.
+// LoadAndRemapMatrixMaxRowsInMemory sets the optional max_rows_in_memory attribute to value.
 //
-// value: If true, the centers of the 4 corner pixels of the input and grad tensors are
-// aligned. Defaults to false.
-// If not specified, defaults to false
-func ResizeNearestNeighborGradAlignCorners(value bool) ResizeNearestNeighborGradAttr {
+// value: The maximum number of rows to load from the checkpoint at
+// once. If less than or equal to 0, the entire matrix will be loaded into
+// memory. Setting this arg trades increased disk reads for lower memory usage.
+// If not specified, defaults to -1
+func LoadAndRemapMatrixMaxRowsInMemory(value int64) LoadAndRemapMatrixAttr {
 	return func(m optionalAttr) {
-		m["align_corners"] = value
+		m["max_rows_in_memory"] = value
 	}
 }
 
-// Computes the gradient of nearest neighbor interpolation.
+// Loads a 2-D (matrix) `Tensor` with name `old_tensor_name` from the checkpoint
+//
+// at `ckpt_path` and potentially reorders its rows and columns using the
+// specified remappings.
+//
+// Most users should use one of the wrapper initializers (such as
+// `tf.contrib.framework.load_and_remap_matrix_initializer`) instead of this
+// function directly.
+//
+// The remappings are 1-D tensors with the following properties:
+//
+// * `row_remapping` must have exactly `num_rows` entries. Row `i` of the output
+//   matrix will be initialized from the row corresponding to index
+//   `row_remapping[i]` in the old `Tensor` from the checkpoint.
+// * `col_remapping` must have either 0 entries (indicating that no column
+//   reordering is needed) or `num_cols` entries. If specified, column `j` of the
+//   output matrix will be initialized from the column corresponding to index
+//   `col_remapping[j]` in the old `Tensor` from the checkpoint.
+// * A value of -1 in either of the remappings signifies a "missing" entry. In that
+//   case, values from the `initializing_values` tensor will be used to fill that
+//   missing row or column. If `row_remapping` has `r` missing entries and
+//   `col_remapping` has `c` missing entries, then the following condition must be
+//   true:
+//
+// `(r * num_cols) + (c * num_rows) - (r * c) == len(initializing_values)`
+//
+// The remapping tensors can be generated using the GenerateVocabRemapping op.
+//
+// As an example, with row_remapping = [1, 0, -1], col_remapping = [0, 2, -1],
+// initializing_values = [0.5, -0.5, 0.25, -0.25, 42], and w(i, j) representing
+// the value from row i, column j of the old tensor in the checkpoint, the output
+// matrix will look like the following:
+//
+// [[w(1, 0),  w(1, 2),  0.5],
+//  [w(0, 0),  w(0, 2), -0.5],
+//  [0.25,    -0.25,      42]]
 //
 // Arguments:
-//	grads: 4-D with shape `[batch, height, width, channels]`.
-//	size: = A 1-D int32 Tensor of 2 elements: `orig_height, orig_width`. The
-// original input size.
+//	ckpt_path: Path to the TensorFlow checkpoint (version 2, `TensorBundle`) from
+// which the old matrix `Tensor` will be loaded.
+//	old_tensor_name: Name of the 2-D `Tensor` to load from checkpoint.
+//	row_remapping: An int `Tensor` of row remappings (generally created by
+// `generate_vocab_remapping`).  Even if no row remapping is needed, this must
+// still be an index-valued Tensor (e.g. [0, 1, 2, ...]), or a shifted
+// index-valued `Tensor` (e.g. [8, 9, 10, ...], for partitioned `Variables`).
+//	col_remapping: An int `Tensor` of column remappings (generally created by
+// `generate_vocab_remapping`).  May be a size-0 `Tensor` if only row remapping
+// is to be done (e.g. column ordering is the same).
+//	initializing_values: A float `Tensor` containing  values to fill in for cells
+// in the output matrix that are not loaded from the checkpoint. Length must be
+// exactly the same as the number of missing / new cells.
+//	num_rows: Number of rows (length of the 1st dimension) in the output matrix.
+//	num_cols: Number of columns (length of the 2nd dimension) in the output matrix.
 //
-// Returns 4-D with shape `[batch, orig_height, orig_width, channels]`. Gradients
-// with respect to the input image.
-func ResizeNearestNeighborGrad(scope *Scope, grads tf.Output, size tf.Output, optional ...ResizeNearestNeighborGradAttr) (output tf.Output) {
+// Returns Output matrix containing existing values loaded from the
+// checkpoint, and with any missing values filled in from initializing_values.
+func LoadAndRemapMatrix(scope *Scope, ckpt_path tf.Output, old_tensor_name tf.Output, row_remapping tf.Output, col_remapping tf.Output, initializing_values tf.Output, num_rows int64, num_cols int64, optional ...LoadAndRemapMatrixAttr) (output_matrix tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"num_rows": num_rows, "num_cols": num_cols}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResizeNearestNeighborGrad",
+		Type: "LoadAndRemapMatrix",
 		Input: []tf.Input{
-			grads, size,
+			ckpt_path, old_tensor_name, row_remapping, col_remapping, initializing_values,
 		},
 		Attrs: attrs,
 	}
@@ -30757,29 +30484,43 @@ func ResizeNearestNeighborGrad(scope *Scope, grads tf.Output, size tf.Output, op
 	return op.Output(0)
 }
 
-// ExtractJpegShapeAttr is an optional argument to ExtractJpegShape.
-type ExtractJpegShapeAttr func(optionalAttr)
+// TFRecordReaderV2Attr is an optional argument to TFRecordReaderV2.
+type TFRecordReaderV2Attr func(optionalAttr)
 
-// ExtractJpegShapeOutputType sets the optional output_type attribute to value.
+// TFRecordReaderV2Container sets the optional container attribute to value.
 //
-// value: (Optional) The output type of the operation (int32 or int64).
-// Defaults to int32.
-// If not specified, defaults to DT_INT32
-func ExtractJpegShapeOutputType(value tf.DataType) ExtractJpegShapeAttr {
+// value: If non-empty, this reader is placed in the given container.
+// Otherwise, a default container is used.
+// If not specified, defaults to ""
+func TFRecordReaderV2Container(value string) TFRecordReaderV2Attr {
 	return func(m optionalAttr) {
-		m["output_type"] = value
+		m["container"] = value
 	}
 }
 
-// Extract the shape information of a JPEG-encoded image.
-//
-// This op only parses the image header, so it is much faster than DecodeJpeg.
+// TFRecordReaderV2SharedName sets the optional shared_name attribute to value.
 //
-// Arguments:
-//	contents: 0-D. The JPEG-encoded image.
+// value: If non-empty, this reader is named in the given bucket
+// with this shared_name. Otherwise, the node name is used instead.
+// If not specified, defaults to ""
+func TFRecordReaderV2SharedName(value string) TFRecordReaderV2Attr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// TFRecordReaderV2CompressionType sets the optional compression_type attribute to value.
+// If not specified, defaults to ""
+func TFRecordReaderV2CompressionType(value string) TFRecordReaderV2Attr {
+	return func(m optionalAttr) {
+		m["compression_type"] = value
+	}
+}
+
+// A Reader that outputs the records from a TensorFlow Records file.
 //
-// Returns 1-D. The image shape with format [height, width, channels].
-func ExtractJpegShape(scope *Scope, contents tf.Output, optional ...ExtractJpegShapeAttr) (image_shape tf.Output) {
+// Returns The handle to reference the Reader.
+func TFRecordReaderV2(scope *Scope, optional ...TFRecordReaderV2Attr) (reader_handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -30788,91 +30529,97 @@ func ExtractJpegShape(scope *Scope, contents tf.Output, optional ...ExtractJpegS
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ExtractJpegShape",
-		Input: []tf.Input{
-			contents,
-		},
+		Type: "TFRecordReaderV2",
+
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// PaddingFIFOQueueV2Attr is an optional argument to PaddingFIFOQueueV2.
-type PaddingFIFOQueueV2Attr func(optionalAttr)
+// QuantizeAndDequantizeV3Attr is an optional argument to QuantizeAndDequantizeV3.
+type QuantizeAndDequantizeV3Attr func(optionalAttr)
 
-// PaddingFIFOQueueV2Shapes sets the optional shapes attribute to value.
-//
-// value: The shape of each component in a value. The length of this attr must
-// be either 0 or the same as the length of component_types.
-// Shapes of fixed rank but variable size are allowed by setting
-// any shape dimension to -1.  In this case, the inputs' shape may vary along
-// the given dimension, and DequeueMany will pad the given dimension with
-// zeros up to the maximum shape of all elements in the given batch.
-// If the length of this attr is 0, different queue elements may have
-// different ranks and shapes, but only one element may be dequeued at a time.
-// If not specified, defaults to <>
-//
-// REQUIRES: len(value) >= 0
-func PaddingFIFOQueueV2Shapes(value []tf.Shape) PaddingFIFOQueueV2Attr {
+// QuantizeAndDequantizeV3SignedInput sets the optional signed_input attribute to value.
+// If not specified, defaults to true
+func QuantizeAndDequantizeV3SignedInput(value bool) QuantizeAndDequantizeV3Attr {
 	return func(m optionalAttr) {
-		m["shapes"] = value
+		m["signed_input"] = value
 	}
 }
 
-// PaddingFIFOQueueV2Capacity sets the optional capacity attribute to value.
-//
-// value: The upper bound on the number of elements in this queue.
-// Negative numbers mean no limit.
-// If not specified, defaults to -1
-func PaddingFIFOQueueV2Capacity(value int64) PaddingFIFOQueueV2Attr {
+// QuantizeAndDequantizeV3RangeGiven sets the optional range_given attribute to value.
+// If not specified, defaults to true
+func QuantizeAndDequantizeV3RangeGiven(value bool) QuantizeAndDequantizeV3Attr {
 	return func(m optionalAttr) {
-		m["capacity"] = value
+		m["range_given"] = value
 	}
 }
 
-// PaddingFIFOQueueV2Container sets the optional container attribute to value.
+// Quantizes then dequantizes a tensor.
 //
-// value: If non-empty, this queue is placed in the given container.
+// This is almost identical to QuantizeAndDequantizeV2, except that num_bits is a
+// tensor, so its value can change during training.
+func QuantizeAndDequantizeV3(scope *Scope, input tf.Output, input_min tf.Output, input_max tf.Output, num_bits tf.Output, optional ...QuantizeAndDequantizeV3Attr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "QuantizeAndDequantizeV3",
+		Input: []tf.Input{
+			input, input_min, input_max, num_bits,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// IdentityReaderV2Attr is an optional argument to IdentityReaderV2.
+type IdentityReaderV2Attr func(optionalAttr)
+
+// IdentityReaderV2Container sets the optional container attribute to value.
+//
+// value: If non-empty, this reader is placed in the given container.
 // Otherwise, a default container is used.
 // If not specified, defaults to ""
-func PaddingFIFOQueueV2Container(value string) PaddingFIFOQueueV2Attr {
+func IdentityReaderV2Container(value string) IdentityReaderV2Attr {
 	return func(m optionalAttr) {
 		m["container"] = value
 	}
 }
 
-// PaddingFIFOQueueV2SharedName sets the optional shared_name attribute to value.
+// IdentityReaderV2SharedName sets the optional shared_name attribute to value.
 //
-// value: If non-empty, this queue will be shared under the given name
-// across multiple sessions.
+// value: If non-empty, this reader is named in the given bucket
+// with this shared_name. Otherwise, the node name is used instead.
 // If not specified, defaults to ""
-func PaddingFIFOQueueV2SharedName(value string) PaddingFIFOQueueV2Attr {
+func IdentityReaderV2SharedName(value string) IdentityReaderV2Attr {
 	return func(m optionalAttr) {
 		m["shared_name"] = value
 	}
 }
 
-// A queue that produces elements in first-in first-out order.
-//
-// Variable-size shapes are allowed by setting the corresponding shape dimensions
-// to 0 in the shape attr.  In this case DequeueMany will pad up to the maximum
-// size of any given element in the minibatch.  See below for details.
+// A Reader that outputs the queued work as both the key and value.
 //
-// Arguments:
-//	component_types: The type of each component in a value.
+// To use, enqueue strings in a Queue.  ReaderRead will take the front
+// work string and output (work, work).
 //
-// Returns The handle to the queue.
-func PaddingFIFOQueueV2(scope *Scope, component_types []tf.DataType, optional ...PaddingFIFOQueueV2Attr) (handle tf.Output) {
+// Returns The handle to reference the Reader.
+func IdentityReaderV2(scope *Scope, optional ...IdentityReaderV2Attr) (reader_handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"component_types": component_types}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "PaddingFIFOQueueV2",
+		Type: "IdentityReaderV2",
 
 		Attrs: attrs,
 	}
@@ -30880,507 +30627,430 @@ func PaddingFIFOQueueV2(scope *Scope, component_types []tf.DataType, optional ..
 	return op.Output(0)
 }
 
-// DecodePngAttr is an optional argument to DecodePng.
-type DecodePngAttr func(optionalAttr)
+// ResourceApplyGradientDescentAttr is an optional argument to ResourceApplyGradientDescent.
+type ResourceApplyGradientDescentAttr func(optionalAttr)
 
-// DecodePngChannels sets the optional channels attribute to value.
+// ResourceApplyGradientDescentUseLocking sets the optional use_locking attribute to value.
 //
-// value: Number of color channels for the decoded image.
-// If not specified, defaults to 0
-func DecodePngChannels(value int64) DecodePngAttr {
+// value: If `True`, the subtraction will be protected by a lock;
+// otherwise the behavior is undefined, but may exhibit less contention.
+// If not specified, defaults to false
+func ResourceApplyGradientDescentUseLocking(value bool) ResourceApplyGradientDescentAttr {
 	return func(m optionalAttr) {
-		m["channels"] = value
+		m["use_locking"] = value
 	}
 }
 
-// DecodePngDtype sets the optional dtype attribute to value.
-// If not specified, defaults to DT_UINT8
-func DecodePngDtype(value tf.DataType) DecodePngAttr {
-	return func(m optionalAttr) {
-		m["dtype"] = value
+// Update '*var' by subtracting 'alpha' * 'delta' from it.
+//
+// Arguments:
+//	var_: Should be from a Variable().
+//	alpha: Scaling factor. Must be a scalar.
+//	delta: The change.
+//
+// Returns the created operation.
+func ResourceApplyGradientDescent(scope *Scope, var_ tf.Output, alpha tf.Output, delta tf.Output, optional ...ResourceApplyGradientDescentAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceApplyGradientDescent",
+		Input: []tf.Input{
+			var_, alpha, delta,
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
 }
 
-// Decode a PNG-encoded image to a uint8 or uint16 tensor.
+// Returns the next record (key, value pair) produced by a Reader.
 //
-// The attr `channels` indicates the desired number of color channels for the
-// decoded image.
-//
-// Accepted values are:
-//
-// *   0: Use the number of channels in the PNG-encoded image.
-// *   1: output a grayscale image.
-// *   3: output an RGB image.
-// *   4: output an RGBA image.
-//
-// If needed, the PNG-encoded image is transformed to match the requested number
-// of color channels.
-//
-// This op also supports decoding JPEGs and non-animated GIFs since the interface
-// is the same, though it is cleaner to use `tf.image.decode_image`.
+// Will dequeue from the input queue if necessary (e.g. when the
+// Reader needs to start reading from a new file since it has finished
+// with the previous file).
 //
 // Arguments:
-//	contents: 0-D.  The PNG-encoded image.
+//	reader_handle: Handle to a Reader.
+//	queue_handle: Handle to a Queue, with string work items.
 //
-// Returns 3-D with shape `[height, width, channels]`.
-func DecodePng(scope *Scope, contents tf.Output, optional ...DecodePngAttr) (image tf.Output) {
+// Returns A scalar.A scalar.
+func ReaderReadV2(scope *Scope, reader_handle tf.Output, queue_handle tf.Output) (key tf.Output, value tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "DecodePng",
+		Type: "ReaderReadV2",
 		Input: []tf.Input{
-			contents,
+			reader_handle, queue_handle,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1)
 }
 
-// Decode the first frame of a GIF-encoded image to a uint8 tensor.
-//
-// GIF with frame or transparency compression are not supported
-// convert animated GIF from compressed to uncompressed by:
-//
-//     convert $src.gif -coalesce $dst.gif
+// Returns up to `num_records` (key, value) pairs produced by a Reader.
 //
-// This op also supports decoding JPEGs and PNGs, though it is cleaner to use
-// `tf.image.decode_image`.
+// Will dequeue from the input queue if necessary (e.g. when the
+// Reader needs to start reading from a new file since it has finished
+// with the previous file).
+// It may return less than `num_records` even before the last batch.
 //
 // Arguments:
-//	contents: 0-D.  The GIF-encoded image.
+//	reader_handle: Handle to a `Reader`.
+//	queue_handle: Handle to a `Queue`, with string work items.
+//	num_records: number of records to read from `Reader`.
 //
-// Returns 4-D with shape `[num_frames, height, width, 3]`. RGB order
-func DecodeGif(scope *Scope, contents tf.Output) (image tf.Output) {
+// Returns A 1-D tensor.A 1-D tensor.
+func ReaderReadUpToV2(scope *Scope, reader_handle tf.Output, queue_handle tf.Output, num_records tf.Output) (keys tf.Output, values tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "DecodeGif",
+		Type: "ReaderReadUpToV2",
 		Input: []tf.Input{
-			contents,
+			reader_handle, queue_handle, num_records,
 		},
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// LearnedUnigramCandidateSamplerAttr is an optional argument to LearnedUnigramCandidateSampler.
-type LearnedUnigramCandidateSamplerAttr func(optionalAttr)
-
-// LearnedUnigramCandidateSamplerSeed sets the optional seed attribute to value.
-//
-// value: If either seed or seed2 are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func LearnedUnigramCandidateSamplerSeed(value int64) LearnedUnigramCandidateSamplerAttr {
-	return func(m optionalAttr) {
-		m["seed"] = value
-	}
-}
-
-// LearnedUnigramCandidateSamplerSeed2 sets the optional seed2 attribute to value.
-//
-// value: An second seed to avoid seed collision.
-// If not specified, defaults to 0
-func LearnedUnigramCandidateSamplerSeed2(value int64) LearnedUnigramCandidateSamplerAttr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
-	}
+	return op.Output(0), op.Output(1)
 }
 
-// Generates labels for candidate sampling with a learned unigram distribution.
-//
-// See explanations of candidate sampling and the data formats at
-// go/candidate-sampling.
-//
-// For each batch, this op picks a single set of sampled candidate labels.
+//     Adds v into specified rows of x.
 //
-// The advantages of sampling candidates per-batch are simplicity and the
-// possibility of efficient dense matrix multiplication. The disadvantage is that
-// the sampled candidates must be chosen independently of the context and of the
-// true labels.
+//     Computes y = x; y[i, :] += v; return y.
 //
 // Arguments:
-//	true_classes: A batch_size * num_true matrix, in which each row contains the
-// IDs of the num_true target_classes in the corresponding original label.
-//	num_true: Number of true labels per context.
-//	num_sampled: Number of candidates to randomly sample.
-//	unique: If unique is true, we sample with rejection, so that all sampled
-// candidates in a batch are unique. This requires some approximation to
-// estimate the post-rejection sampling probabilities.
-//	range_max: The sampler will sample integers from the interval [0, range_max).
+//	x: A `Tensor` of type T.
+//	i: A vector. Indices into the left-most dimension of `x`.
+//	v: A `Tensor` of type T. Same dimension sizes as x except the first dimension, which must be the same as i's size.
 //
-// Returns A vector of length num_sampled, in which each element is
-// the ID of a sampled candidate.A batch_size * num_true matrix, representing
-// the number of times each candidate is expected to occur in a batch
-// of sampled candidates. If unique=true, then this is a probability.A vector of length num_sampled, for each sampled
-// candidate representing the number of times the candidate is expected
-// to occur in a batch of sampled candidates.  If unique=true, then this is a
-// probability.
-func LearnedUnigramCandidateSampler(scope *Scope, true_classes tf.Output, num_true int64, num_sampled int64, unique bool, range_max int64, optional ...LearnedUnigramCandidateSamplerAttr) (sampled_candidates tf.Output, true_expected_count tf.Output, sampled_expected_count tf.Output) {
+// Returns A `Tensor` of type T. An alias of `x`. The content of `y` is undefined if there are duplicates in `i`.
+func InplaceAdd(scope *Scope, x tf.Output, i tf.Output, v tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"num_true": num_true, "num_sampled": num_sampled, "unique": unique, "range_max": range_max}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "LearnedUnigramCandidateSampler",
+		Type: "InplaceAdd",
 		Input: []tf.Input{
-			true_classes,
+			x, i, v,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
-}
-
-// SerializeSparseAttr is an optional argument to SerializeSparse.
-type SerializeSparseAttr func(optionalAttr)
-
-// SerializeSparseOutType sets the optional out_type attribute to value.
-//
-// value: The `dtype` to use for serialization; the supported types are `string`
-// (default) and `variant`.
-// If not specified, defaults to DT_STRING
-func SerializeSparseOutType(value tf.DataType) SerializeSparseAttr {
-	return func(m optionalAttr) {
-		m["out_type"] = value
-	}
+	return op.Output(0)
 }
 
-// Serialize a `SparseTensor` into a `[3]` `Tensor` object.
+// Restore a Reader to its initial clean state.
 //
 // Arguments:
-//	sparse_indices: 2-D.  The `indices` of the `SparseTensor`.
-//	sparse_values: 1-D.  The `values` of the `SparseTensor`.
-//	sparse_shape: 1-D.  The `shape` of the `SparseTensor`.
-func SerializeSparse(scope *Scope, sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output, optional ...SerializeSparseAttr) (serialized_sparse tf.Output) {
+//	reader_handle: Handle to a Reader.
+//
+// Returns the created operation.
+func ReaderResetV2(scope *Scope, reader_handle tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "SerializeSparse",
+		Type: "ReaderResetV2",
 		Input: []tf.Input{
-			sparse_indices, sparse_values, sparse_shape,
+			reader_handle,
 		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// RandomShuffleQueueV2Attr is an optional argument to RandomShuffleQueueV2.
-type RandomShuffleQueueV2Attr func(optionalAttr)
-
-// RandomShuffleQueueV2Shapes sets the optional shapes attribute to value.
-//
-// value: The shape of each component in a value. The length of this attr must
-// be either 0 or the same as the length of component_types. If the length of
-// this attr is 0, the shapes of queue elements are not constrained, and
-// only one element may be dequeued at a time.
-// If not specified, defaults to <>
-//
-// REQUIRES: len(value) >= 0
-func RandomShuffleQueueV2Shapes(value []tf.Shape) RandomShuffleQueueV2Attr {
-	return func(m optionalAttr) {
-		m["shapes"] = value
 	}
+	return scope.AddOperation(opspec)
 }
 
-// RandomShuffleQueueV2Capacity sets the optional capacity attribute to value.
-//
-// value: The upper bound on the number of elements in this queue.
-// Negative numbers mean no limit.
-// If not specified, defaults to -1
-func RandomShuffleQueueV2Capacity(value int64) RandomShuffleQueueV2Attr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
+// BatchAttr is an optional argument to Batch.
+type BatchAttr func(optionalAttr)
 
-// RandomShuffleQueueV2MinAfterDequeue sets the optional min_after_dequeue attribute to value.
-//
-// value: Dequeue will block unless there would be this
-// many elements after the dequeue or the queue is closed. This
-// ensures a minimum level of mixing of elements.
-// If not specified, defaults to 0
-func RandomShuffleQueueV2MinAfterDequeue(value int64) RandomShuffleQueueV2Attr {
+// BatchMaxEnqueuedBatches sets the optional max_enqueued_batches attribute to value.
+// If not specified, defaults to 10
+func BatchMaxEnqueuedBatches(value int64) BatchAttr {
 	return func(m optionalAttr) {
-		m["min_after_dequeue"] = value
+		m["max_enqueued_batches"] = value
 	}
 }
 
-// RandomShuffleQueueV2Seed sets the optional seed attribute to value.
-//
-// value: If either seed or seed2 is set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, a random seed is used.
-// If not specified, defaults to 0
-func RandomShuffleQueueV2Seed(value int64) RandomShuffleQueueV2Attr {
+// BatchAllowedBatchSizes sets the optional allowed_batch_sizes attribute to value.
+// If not specified, defaults to <>
+func BatchAllowedBatchSizes(value []int64) BatchAttr {
 	return func(m optionalAttr) {
-		m["seed"] = value
+		m["allowed_batch_sizes"] = value
 	}
 }
 
-// RandomShuffleQueueV2Seed2 sets the optional seed2 attribute to value.
-//
-// value: A second seed to avoid seed collision.
-// If not specified, defaults to 0
-func RandomShuffleQueueV2Seed2(value int64) RandomShuffleQueueV2Attr {
+// BatchContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func BatchContainer(value string) BatchAttr {
 	return func(m optionalAttr) {
-		m["seed2"] = value
+		m["container"] = value
 	}
 }
 
-// RandomShuffleQueueV2Container sets the optional container attribute to value.
-//
-// value: If non-empty, this queue is placed in the given container.
-// Otherwise, a default container is used.
+// BatchSharedName sets the optional shared_name attribute to value.
 // If not specified, defaults to ""
-func RandomShuffleQueueV2Container(value string) RandomShuffleQueueV2Attr {
+func BatchSharedName(value string) BatchAttr {
 	return func(m optionalAttr) {
-		m["container"] = value
+		m["shared_name"] = value
 	}
 }
 
-// RandomShuffleQueueV2SharedName sets the optional shared_name attribute to value.
-//
-// value: If non-empty, this queue will be shared under the given name
-// across multiple sessions.
+// BatchBatchingQueue sets the optional batching_queue attribute to value.
 // If not specified, defaults to ""
-func RandomShuffleQueueV2SharedName(value string) RandomShuffleQueueV2Attr {
+func BatchBatchingQueue(value string) BatchAttr {
 	return func(m optionalAttr) {
-		m["shared_name"] = value
+		m["batching_queue"] = value
 	}
 }
 
-// A queue that randomizes the order of elements.
+// Batches all input tensors nondeterministically.
 //
-// Arguments:
-//	component_types: The type of each component in a value.
+// When many instances of this Op are being run concurrently with the same
+// container/shared_name in the same device, some will output zero-shaped Tensors
+// and others will output Tensors of size up to max_batch_size.
 //
-// Returns The handle to the queue.
-func RandomShuffleQueueV2(scope *Scope, component_types []tf.DataType, optional ...RandomShuffleQueueV2Attr) (handle tf.Output) {
+// All Tensors in in_tensors are batched together (so, for example, labels and
+// features should be batched with a single instance of this operation.
+//
+// Each invocation of batch emits an `id` scalar which will be used to identify
+// this particular invocation when doing unbatch or its gradient.
+//
+// Each op which emits a non-empty batch will also emit a non-empty batch_index
+// Tensor, which, is a [K, 3] matrix where each row contains the invocation's id,
+// start, and length of elements of each set of Tensors present in batched_tensors.
+//
+// Batched tensors are concatenated along the first dimension, and all tensors in
+// in_tensors must have the first dimension of the same size.
+//
+// in_tensors: The tensors to be batched.
+// num_batch_threads: Number of scheduling threads for processing batches of work.
+//  Determines the number of batches processed in parallel.
+// max_batch_size: Batch sizes will never be bigger than this.
+// batch_timeout_micros: Maximum number of microseconds to wait before outputting
+//  an incomplete batch.
+// allowed_batch_sizes: Optional list of allowed batch sizes. If left empty, does
+//  nothing. Otherwise, supplies a list of batch sizes, causing the op to pad
+//  batches up to one of those sizes. The entries must increase monotonically, and
+//  the final entry must equal max_batch_size.
+// grad_timeout_micros: The timeout to use for the gradient. See Unbatch.
+// batched_tensors: Either empty tensors or a batch of concatenated Tensors.
+// batch_index: If out_tensors is non-empty, has information to invert it.
+// container: Controls the scope of sharing of this batch.
+// id: always contains a scalar with a unique ID for this invocation of Batch.
+// shared_name: Concurrently running instances of batch in the same device with the
+//  same container and shared_name will batch their elements together. If left
+//  empty, the op name will be used as the shared name.
+// T: the types of tensors to be batched.
+func Batch(scope *Scope, in_tensors []tf.Output, num_batch_threads int64, max_batch_size int64, batch_timeout_micros int64, grad_timeout_micros int64, optional ...BatchAttr) (batched_tensors []tf.Output, batch_index tf.Output, id tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"component_types": component_types}
+	attrs := map[string]interface{}{"num_batch_threads": num_batch_threads, "max_batch_size": max_batch_size, "batch_timeout_micros": batch_timeout_micros, "grad_timeout_micros": grad_timeout_micros}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "RandomShuffleQueueV2",
-
+		Type: "Batch",
+		Input: []tf.Input{
+			tf.OutputList(in_tensors),
+		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if batched_tensors, idx, err = makeOutputList(op, idx, "batched_tensors"); err != nil {
+		scope.UpdateErr("Batch", err)
+		return
+	}
+	batch_index = op.Output(idx)
+	id = op.Output(idx)
+	return batched_tensors, batch_index, id
 }
 
-// Draw bounding boxes on a batch of images.
-//
-// Outputs a copy of `images` but draws on top of the pixels zero or more bounding
-// boxes specified by the locations in `boxes`. The coordinates of the each
-// bounding box in `boxes` are encoded as `[y_min, x_min, y_max, x_max]`. The
-// bounding box coordinates are floats in `[0.0, 1.0]` relative to the width and
-// height of the underlying image.
+// Adjust the hue of one or more images.
 //
-// For example, if an image is 100 x 200 pixels (height x width) and the bounding
-// box is `[0.1, 0.2, 0.5, 0.9]`, the upper-left and bottom-right coordinates of
-// the bounding box will be `(40, 10)` to `(180, 50)` (in (x,y) coordinates).
+// `images` is a tensor of at least 3 dimensions.  The last dimension is
+// interpretted as channels, and must be three.
 //
-// Parts of the bounding box may fall outside the image.
+// The input image is considered in the RGB colorspace. Conceptually, the RGB
+// colors are first mapped into HSV. A delta is then applied all the hue values,
+// and then remapped back to RGB colorspace.
 //
 // Arguments:
-//	images: 4-D with shape `[batch, height, width, depth]`. A batch of images.
-//	boxes: 3-D with shape `[batch, num_bounding_boxes, 4]` containing bounding
-// boxes.
+//	images: Images to adjust.  At least 3-D.
+//	delta: A float delta to add to the hue.
 //
-// Returns 4-D with the same shape as `images`. The batch of input images with
-// bounding boxes drawn on the images.
-func DrawBoundingBoxes(scope *Scope, images tf.Output, boxes tf.Output) (output tf.Output) {
+// Returns The hue-adjusted image or images.
+func AdjustHue(scope *Scope, images tf.Output, delta tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "DrawBoundingBoxes",
+		Type: "AdjustHue",
 		Input: []tf.Input{
-			images, boxes,
+			images, delta,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Gets the next output from the given iterator.
+// ResizeBicubicGradAttr is an optional argument to ResizeBicubicGrad.
+type ResizeBicubicGradAttr func(optionalAttr)
+
+// ResizeBicubicGradAlignCorners sets the optional align_corners attribute to value.
 //
-// This operation is a synchronous version IteratorGetNext. It should only be used
-// in situations where the iterator does not block the calling thread, or where
-// the calling thread is not a member of the thread pool used to execute parallel
-// operations (e.g. in eager mode).
-func IteratorGetNextSync(scope *Scope, iterator tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (components []tf.Output) {
+// value: If true, the centers of the 4 corner pixels of the input and grad tensors are
+// aligned. Defaults to false.
+// If not specified, defaults to false
+func ResizeBicubicGradAlignCorners(value bool) ResizeBicubicGradAttr {
+	return func(m optionalAttr) {
+		m["align_corners"] = value
+	}
+}
+
+// Computes the gradient of bicubic interpolation.
+//
+// Arguments:
+//	grads: 4-D with shape `[batch, height, width, channels]`.
+//	original_image: 4-D with shape `[batch, orig_height, orig_width, channels]`,
+// The image tensor that was resized.
+//
+// Returns 4-D with shape `[batch, orig_height, orig_width, channels]`.
+// Gradients with respect to the input image. Input image must have been
+// float or double.
+func ResizeBicubicGrad(scope *Scope, grads tf.Output, original_image tf.Output, optional ...ResizeBicubicGradAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "IteratorGetNextSync",
+		Type: "ResizeBicubicGrad",
 		Input: []tf.Input{
-			iterator,
+			grads, original_image,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if components, idx, err = makeOutputList(op, idx, "components"); err != nil {
-		scope.UpdateErr("IteratorGetNextSync", err)
-		return
-	}
-	return components
+	return op.Output(0)
 }
 
-// SampleDistortedBoundingBoxV2Attr is an optional argument to SampleDistortedBoundingBoxV2.
-type SampleDistortedBoundingBoxV2Attr func(optionalAttr)
+// ResizeNearestNeighborAttr is an optional argument to ResizeNearestNeighbor.
+type ResizeNearestNeighborAttr func(optionalAttr)
 
-// SampleDistortedBoundingBoxV2Seed sets the optional seed attribute to value.
+// ResizeNearestNeighborAlignCorners sets the optional align_corners attribute to value.
 //
-// value: If either `seed` or `seed2` are set to non-zero, the random number
-// generator is seeded by the given `seed`.  Otherwise, it is seeded by a random
-// seed.
-// If not specified, defaults to 0
-func SampleDistortedBoundingBoxV2Seed(value int64) SampleDistortedBoundingBoxV2Attr {
+// value: If true, the centers of the 4 corner pixels of the input and output tensors are
+// aligned, preserving the values at the corner pixels. Defaults to false.
+// If not specified, defaults to false
+func ResizeNearestNeighborAlignCorners(value bool) ResizeNearestNeighborAttr {
 	return func(m optionalAttr) {
-		m["seed"] = value
+		m["align_corners"] = value
 	}
 }
 
-// SampleDistortedBoundingBoxV2Seed2 sets the optional seed2 attribute to value.
+// Resize `images` to `size` using nearest neighbor interpolation.
 //
-// value: A second seed to avoid seed collision.
-// If not specified, defaults to 0
-func SampleDistortedBoundingBoxV2Seed2(value int64) SampleDistortedBoundingBoxV2Attr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
-	}
-}
-
-// SampleDistortedBoundingBoxV2AspectRatioRange sets the optional aspect_ratio_range attribute to value.
+// Arguments:
+//	images: 4-D with shape `[batch, height, width, channels]`.
+//	size: = A 1-D int32 Tensor of 2 elements: `new_height, new_width`.  The
+// new size for the images.
 //
-// value: The cropped area of the image must have an aspect ratio =
-// width / height within this range.
-// If not specified, defaults to <f:0.75 f:1.33 >
-func SampleDistortedBoundingBoxV2AspectRatioRange(value []float32) SampleDistortedBoundingBoxV2Attr {
-	return func(m optionalAttr) {
-		m["aspect_ratio_range"] = value
+// Returns 4-D with shape
+// `[batch, new_height, new_width, channels]`.
+func ResizeNearestNeighbor(scope *Scope, images tf.Output, size tf.Output, optional ...ResizeNearestNeighborAttr) (resized_images tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResizeNearestNeighbor",
+		Input: []tf.Input{
+			images, size,
+		},
+		Attrs: attrs,
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// SampleDistortedBoundingBoxV2AreaRange sets the optional area_range attribute to value.
+// ResizeNearestNeighborGradAttr is an optional argument to ResizeNearestNeighborGrad.
+type ResizeNearestNeighborGradAttr func(optionalAttr)
+
+// ResizeNearestNeighborGradAlignCorners sets the optional align_corners attribute to value.
 //
-// value: The cropped area of the image must contain a fraction of the
-// supplied image within this range.
-// If not specified, defaults to <f:0.05 f:1 >
-func SampleDistortedBoundingBoxV2AreaRange(value []float32) SampleDistortedBoundingBoxV2Attr {
+// value: If true, the centers of the 4 corner pixels of the input and grad tensors are
+// aligned. Defaults to false.
+// If not specified, defaults to false
+func ResizeNearestNeighborGradAlignCorners(value bool) ResizeNearestNeighborGradAttr {
 	return func(m optionalAttr) {
-		m["area_range"] = value
+		m["align_corners"] = value
 	}
 }
 
-// SampleDistortedBoundingBoxV2MaxAttempts sets the optional max_attempts attribute to value.
+// Computes the gradient of nearest neighbor interpolation.
 //
-// value: Number of attempts at generating a cropped region of the image
-// of the specified constraints. After `max_attempts` failures, return the entire
-// image.
-// If not specified, defaults to 100
-func SampleDistortedBoundingBoxV2MaxAttempts(value int64) SampleDistortedBoundingBoxV2Attr {
-	return func(m optionalAttr) {
-		m["max_attempts"] = value
+// Arguments:
+//	grads: 4-D with shape `[batch, height, width, channels]`.
+//	size: = A 1-D int32 Tensor of 2 elements: `orig_height, orig_width`. The
+// original input size.
+//
+// Returns 4-D with shape `[batch, orig_height, orig_width, channels]`. Gradients
+// with respect to the input image.
+func ResizeNearestNeighborGrad(scope *Scope, grads tf.Output, size tf.Output, optional ...ResizeNearestNeighborGradAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResizeNearestNeighborGrad",
+		Input: []tf.Input{
+			grads, size,
+		},
+		Attrs: attrs,
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// SampleDistortedBoundingBoxV2UseImageIfNoBoundingBoxes sets the optional use_image_if_no_bounding_boxes attribute to value.
+// ExtractJpegShapeAttr is an optional argument to ExtractJpegShape.
+type ExtractJpegShapeAttr func(optionalAttr)
+
+// ExtractJpegShapeOutputType sets the optional output_type attribute to value.
 //
-// value: Controls behavior if no bounding boxes supplied.
-// If true, assume an implicit bounding box covering the whole input. If false,
-// raise an error.
-// If not specified, defaults to false
-func SampleDistortedBoundingBoxV2UseImageIfNoBoundingBoxes(value bool) SampleDistortedBoundingBoxV2Attr {
+// value: (Optional) The output type of the operation (int32 or int64).
+// Defaults to int32.
+// If not specified, defaults to DT_INT32
+func ExtractJpegShapeOutputType(value tf.DataType) ExtractJpegShapeAttr {
 	return func(m optionalAttr) {
-		m["use_image_if_no_bounding_boxes"] = value
+		m["output_type"] = value
 	}
 }
 
-// Generate a single randomly distorted bounding box for an image.
+// Extract the shape information of a JPEG-encoded image.
 //
-// Bounding box annotations are often supplied in addition to ground-truth labels
-// in image recognition or object localization tasks. A common technique for
-// training such a system is to randomly distort an image while preserving
-// its content, i.e. *data augmentation*. This Op outputs a randomly distorted
-// localization of an object, i.e. bounding box, given an `image_size`,
-// `bounding_boxes` and a series of constraints.
-//
-// The output of this Op is a single bounding box that may be used to crop the
-// original image. The output is returned as 3 tensors: `begin`, `size` and
-// `bboxes`. The first 2 tensors can be fed directly into `tf.slice` to crop the
-// image. The latter may be supplied to `tf.image.draw_bounding_boxes` to visualize
-// what the bounding box looks like.
-//
-// Bounding boxes are supplied and returned as `[y_min, x_min, y_max, x_max]`. The
-// bounding box coordinates are floats in `[0.0, 1.0]` relative to the width and
-// height of the underlying image.
-//
-// For example,
-//
-// ```python
-//     # Generate a single distorted bounding box.
-//     begin, size, bbox_for_draw = tf.image.sample_distorted_bounding_box(
-//         tf.shape(image),
-//         bounding_boxes=bounding_boxes)
-//
-//     # Draw the bounding box in an image summary.
-//     image_with_box = tf.image.draw_bounding_boxes(tf.expand_dims(image, 0),
-//                                                   bbox_for_draw)
-//     tf.summary.image('images_with_box', image_with_box)
-//
-//     # Employ the bounding box to distort the image.
-//     distorted_image = tf.slice(image, begin, size)
-// ```
-//
-// Note that if no bounding box information is available, setting
-// `use_image_if_no_bounding_boxes = true` will assume there is a single implicit
-// bounding box covering the whole image. If `use_image_if_no_bounding_boxes` is
-// false and no bounding boxes are supplied, an error is raised.
+// This op only parses the image header, so it is much faster than DecodeJpeg.
 //
 // Arguments:
-//	image_size: 1-D, containing `[height, width, channels]`.
-//	bounding_boxes: 3-D with shape `[batch, N, 4]` describing the N bounding boxes
-// associated with the image.
-//	min_object_covered: The cropped area of the image must contain at least this
-// fraction of any bounding box supplied. The value of this parameter should be
-// non-negative. In the case of 0, the cropped area does not need to overlap
-// any of the bounding boxes supplied.
+//	contents: 0-D. The JPEG-encoded image.
 //
-// Returns 1-D, containing `[offset_height, offset_width, 0]`. Provide as input to
-// `tf.slice`.1-D, containing `[target_height, target_width, -1]`. Provide as input to
-// `tf.slice`.3-D with shape `[1, 1, 4]` containing the distorted bounding box.
-// Provide as input to `tf.image.draw_bounding_boxes`.
-func SampleDistortedBoundingBoxV2(scope *Scope, image_size tf.Output, bounding_boxes tf.Output, min_object_covered tf.Output, optional ...SampleDistortedBoundingBoxV2Attr) (begin tf.Output, size tf.Output, bboxes tf.Output) {
+// Returns 1-D. The image shape with format [height, width, channels].
+func ExtractJpegShape(scope *Scope, contents tf.Output, optional ...ExtractJpegShapeAttr) (image_shape tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -31389,117 +31059,91 @@ func SampleDistortedBoundingBoxV2(scope *Scope, image_size tf.Output, bounding_b
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "SampleDistortedBoundingBoxV2",
+		Type: "ExtractJpegShape",
 		Input: []tf.Input{
-			image_size, bounding_boxes, min_object_covered,
+			contents,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// ExtractGlimpseAttr is an optional argument to ExtractGlimpse.
-type ExtractGlimpseAttr func(optionalAttr)
+// PaddingFIFOQueueV2Attr is an optional argument to PaddingFIFOQueueV2.
+type PaddingFIFOQueueV2Attr func(optionalAttr)
 
-// ExtractGlimpseCentered sets the optional centered attribute to value.
+// PaddingFIFOQueueV2Shapes sets the optional shapes attribute to value.
 //
-// value: indicates if the offset coordinates are centered relative to
-// the image, in which case the (0, 0) offset is relative to the center
-// of the input images. If false, the (0,0) offset corresponds to the
-// upper left corner of the input images.
-// If not specified, defaults to true
-func ExtractGlimpseCentered(value bool) ExtractGlimpseAttr {
+// value: The shape of each component in a value. The length of this attr must
+// be either 0 or the same as the length of component_types.
+// Shapes of fixed rank but variable size are allowed by setting
+// any shape dimension to -1.  In this case, the inputs' shape may vary along
+// the given dimension, and DequeueMany will pad the given dimension with
+// zeros up to the maximum shape of all elements in the given batch.
+// If the length of this attr is 0, different queue elements may have
+// different ranks and shapes, but only one element may be dequeued at a time.
+// If not specified, defaults to <>
+//
+// REQUIRES: len(value) >= 0
+func PaddingFIFOQueueV2Shapes(value []tf.Shape) PaddingFIFOQueueV2Attr {
 	return func(m optionalAttr) {
-		m["centered"] = value
+		m["shapes"] = value
 	}
 }
 
-// ExtractGlimpseNormalized sets the optional normalized attribute to value.
+// PaddingFIFOQueueV2Capacity sets the optional capacity attribute to value.
 //
-// value: indicates if the offset coordinates are normalized.
-// If not specified, defaults to true
-func ExtractGlimpseNormalized(value bool) ExtractGlimpseAttr {
+// value: The upper bound on the number of elements in this queue.
+// Negative numbers mean no limit.
+// If not specified, defaults to -1
+func PaddingFIFOQueueV2Capacity(value int64) PaddingFIFOQueueV2Attr {
 	return func(m optionalAttr) {
-		m["normalized"] = value
+		m["capacity"] = value
 	}
 }
 
-// ExtractGlimpseUniformNoise sets the optional uniform_noise attribute to value.
+// PaddingFIFOQueueV2Container sets the optional container attribute to value.
 //
-// value: indicates if the noise should be generated using a
-// uniform distribution or a Gaussian distribution.
-// If not specified, defaults to true
-func ExtractGlimpseUniformNoise(value bool) ExtractGlimpseAttr {
+// value: If non-empty, this queue is placed in the given container.
+// Otherwise, a default container is used.
+// If not specified, defaults to ""
+func PaddingFIFOQueueV2Container(value string) PaddingFIFOQueueV2Attr {
 	return func(m optionalAttr) {
-		m["uniform_noise"] = value
+		m["container"] = value
 	}
 }
 
-// Extracts a glimpse from the input tensor.
-//
-// Returns a set of windows called glimpses extracted at location
-// `offsets` from the input tensor. If the windows only partially
-// overlaps the inputs, the non overlapping areas will be filled with
-// random noise.
-//
-// The result is a 4-D tensor of shape `[batch_size, glimpse_height,
-// glimpse_width, channels]`. The channels and batch dimensions are the
-// same as that of the input tensor. The height and width of the output
-// windows are specified in the `size` parameter.
+// PaddingFIFOQueueV2SharedName sets the optional shared_name attribute to value.
 //
-// The argument `normalized` and `centered` controls how the windows are built:
+// value: If non-empty, this queue will be shared under the given name
+// across multiple sessions.
+// If not specified, defaults to ""
+func PaddingFIFOQueueV2SharedName(value string) PaddingFIFOQueueV2Attr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// A queue that produces elements in first-in first-out order.
 //
-// * If the coordinates are normalized but not centered, 0.0 and 1.0
-//   correspond to the minimum and maximum of each height and width
-//   dimension.
-// * If the coordinates are both normalized and centered, they range from
-//   -1.0 to 1.0. The coordinates (-1.0, -1.0) correspond to the upper
-//   left corner, the lower right corner is located at (1.0, 1.0) and the
-//   center is at (0, 0).
-// * If the coordinates are not normalized they are interpreted as
-//   numbers of pixels.
+// Variable-size shapes are allowed by setting the corresponding shape dimensions
+// to 0 in the shape attr.  In this case DequeueMany will pad up to the maximum
+// size of any given element in the minibatch.  See below for details.
 //
 // Arguments:
-//	input: A 4-D float tensor of shape `[batch_size, height, width, channels]`.
-//	size: A 1-D tensor of 2 elements containing the size of the glimpses
-// to extract.  The glimpse height must be specified first, following
-// by the glimpse width.
-//	offsets: A 2-D integer tensor of shape `[batch_size, 2]` containing
-// the y, x locations of the center of each window.
+//	component_types: The type of each component in a value.
 //
-// Returns A tensor representing the glimpses `[batch_size,
-// glimpse_height, glimpse_width, channels]`.
-func ExtractGlimpse(scope *Scope, input tf.Output, size tf.Output, offsets tf.Output, optional ...ExtractGlimpseAttr) (glimpse tf.Output) {
+// Returns The handle to the queue.
+func PaddingFIFOQueueV2(scope *Scope, component_types []tf.DataType, optional ...PaddingFIFOQueueV2Attr) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"component_types": component_types}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ExtractGlimpse",
-		Input: []tf.Input{
-			input, size, offsets,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// A container for an iterator resource.
-//
-// Returns A handle to the iterator that can be passed to a "MakeIterator"
-// or "IteratorGetNext" op.
-func Iterator(scope *Scope, shared_name string, container string, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"shared_name": shared_name, "container": container, "output_types": output_types, "output_shapes": output_shapes}
-	opspec := tf.OpSpec{
-		Type: "Iterator",
+		Type: "PaddingFIFOQueueV2",
 
 		Attrs: attrs,
 	}
@@ -31507,27 +31151,50 @@ func Iterator(scope *Scope, shared_name string, container string, output_types [
 	return op.Output(0)
 }
 
-// TensorForestTreeResourceHandleOpAttr is an optional argument to TensorForestTreeResourceHandleOp.
-type TensorForestTreeResourceHandleOpAttr func(optionalAttr)
+// DecodePngAttr is an optional argument to DecodePng.
+type DecodePngAttr func(optionalAttr)
 
-// TensorForestTreeResourceHandleOpContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func TensorForestTreeResourceHandleOpContainer(value string) TensorForestTreeResourceHandleOpAttr {
+// DecodePngChannels sets the optional channels attribute to value.
+//
+// value: Number of color channels for the decoded image.
+// If not specified, defaults to 0
+func DecodePngChannels(value int64) DecodePngAttr {
 	return func(m optionalAttr) {
-		m["container"] = value
+		m["channels"] = value
 	}
 }
 
-// TensorForestTreeResourceHandleOpSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func TensorForestTreeResourceHandleOpSharedName(value string) TensorForestTreeResourceHandleOpAttr {
+// DecodePngDtype sets the optional dtype attribute to value.
+// If not specified, defaults to DT_UINT8
+func DecodePngDtype(value tf.DataType) DecodePngAttr {
 	return func(m optionalAttr) {
-		m["shared_name"] = value
+		m["dtype"] = value
 	}
 }
 
-// Creates a handle to a TensorForestTreeResource
-func TensorForestTreeResourceHandleOp(scope *Scope, optional ...TensorForestTreeResourceHandleOpAttr) (resource tf.Output) {
+// Decode a PNG-encoded image to a uint8 or uint16 tensor.
+//
+// The attr `channels` indicates the desired number of color channels for the
+// decoded image.
+//
+// Accepted values are:
+//
+// *   0: Use the number of channels in the PNG-encoded image.
+// *   1: output a grayscale image.
+// *   3: output an RGB image.
+// *   4: output an RGBA image.
+//
+// If needed, the PNG-encoded image is transformed to match the requested number
+// of color channels.
+//
+// This op also supports decoding JPEGs and non-animated GIFs since the interface
+// is the same, though it is cleaner to use `tf.image.decode_image`.
+//
+// Arguments:
+//	contents: 0-D.  The PNG-encoded image.
+//
+// Returns 3-D with shape `[height, width, channels]`.
+func DecodePng(scope *Scope, contents tf.Output, optional ...DecodePngAttr) (image tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -31536,181 +31203,138 @@ func TensorForestTreeResourceHandleOp(scope *Scope, optional ...TensorForestTree
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "TensorForestTreeResourceHandleOp",
-
+		Type: "DecodePng",
+		Input: []tf.Input{
+			contents,
+		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// CropAndResizeGradImageAttr is an optional argument to CropAndResizeGradImage.
-type CropAndResizeGradImageAttr func(optionalAttr)
-
-// CropAndResizeGradImageMethod sets the optional method attribute to value.
+// Decode the first frame of a GIF-encoded image to a uint8 tensor.
 //
-// value: A string specifying the interpolation method. Only 'bilinear' is
-// supported for now.
-// If not specified, defaults to "bilinear"
-func CropAndResizeGradImageMethod(value string) CropAndResizeGradImageAttr {
-	return func(m optionalAttr) {
-		m["method"] = value
-	}
-}
-
-// Computes the gradient of the crop_and_resize op wrt the input image tensor.
+// GIF with frame or transparency compression are not supported
+// convert animated GIF from compressed to uncompressed by:
+//
+//     convert $src.gif -coalesce $dst.gif
+//
+// This op also supports decoding JPEGs and PNGs, though it is cleaner to use
+// `tf.image.decode_image`.
 //
 // Arguments:
-//	grads: A 4-D tensor of shape `[num_boxes, crop_height, crop_width, depth]`.
-//	boxes: A 2-D tensor of shape `[num_boxes, 4]`. The `i`-th row of the tensor
-// specifies the coordinates of a box in the `box_ind[i]` image and is specified
-// in normalized coordinates `[y1, x1, y2, x2]`. A normalized coordinate value of
-// `y` is mapped to the image coordinate at `y * (image_height - 1)`, so as the
-// `[0, 1]` interval of normalized image height is mapped to
-// `[0, image_height - 1] in image height coordinates. We do allow y1 > y2, in
-// which case the sampled crop is an up-down flipped version of the original
-// image. The width dimension is treated similarly. Normalized coordinates
-// outside the `[0, 1]` range are allowed, in which case we use
-// `extrapolation_value` to extrapolate the input image values.
-//	box_ind: A 1-D tensor of shape `[num_boxes]` with int32 values in `[0, batch)`.
-// The value of `box_ind[i]` specifies the image that the `i`-th box refers to.
-//	image_size: A 1-D tensor with value `[batch, image_height, image_width, depth]`
-// containing the original image size. Both `image_height` and `image_width` need
-// to be positive.
-//
+//	contents: 0-D.  The GIF-encoded image.
 //
-// Returns A 4-D tensor of shape `[batch, image_height, image_width, depth]`.
-func CropAndResizeGradImage(scope *Scope, grads tf.Output, boxes tf.Output, box_ind tf.Output, image_size tf.Output, T tf.DataType, optional ...CropAndResizeGradImageAttr) (output tf.Output) {
+// Returns 4-D with shape `[num_frames, height, width, 3]`. RGB order
+func DecodeGif(scope *Scope, contents tf.Output) (image tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"T": T}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "CropAndResizeGradImage",
+		Type: "DecodeGif",
 		Input: []tf.Input{
-			grads, boxes, box_ind, image_size,
+			contents,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ShuffleDatasetAttr is an optional argument to ShuffleDataset.
-type ShuffleDatasetAttr func(optionalAttr)
+// LearnedUnigramCandidateSamplerAttr is an optional argument to LearnedUnigramCandidateSampler.
+type LearnedUnigramCandidateSamplerAttr func(optionalAttr)
 
-// ShuffleDatasetReshuffleEachIteration sets the optional reshuffle_each_iteration attribute to value.
+// LearnedUnigramCandidateSamplerSeed sets the optional seed attribute to value.
 //
-// value: If true, each iterator over this dataset will be given
-// a different pseudorandomly generated seed, based on a sequence seeded by the
-// `seed` and `seed2` inputs. If false, each iterator will be given the same
-// seed, and repeated iteration over this dataset will yield the exact same
-// sequence of results.
-// If not specified, defaults to true
-func ShuffleDatasetReshuffleEachIteration(value bool) ShuffleDatasetAttr {
+// value: If either seed or seed2 are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func LearnedUnigramCandidateSamplerSeed(value int64) LearnedUnigramCandidateSamplerAttr {
 	return func(m optionalAttr) {
-		m["reshuffle_each_iteration"] = value
+		m["seed"] = value
 	}
 }
 
-// Creates a dataset that shuffles elements from `input_dataset` pseudorandomly.
-//
-// Arguments:
-//
-//	buffer_size: The number of output elements to buffer in an iterator over
-// this dataset. Compare with the `min_after_dequeue` attr when creating a
-// `RandomShuffleQueue`.
-//	seed: A scalar seed for the random number generator. If either `seed` or
-// `seed2` is set to be non-zero, the random number generator is seeded
-// by the given seed.  Otherwise, a random seed is used.
-//	seed2: A second scalar seed to avoid seed collision.
-//
+// LearnedUnigramCandidateSamplerSeed2 sets the optional seed2 attribute to value.
 //
-func ShuffleDataset(scope *Scope, input_dataset tf.Output, buffer_size tf.Output, seed tf.Output, seed2 tf.Output, output_types []tf.DataType, output_shapes []tf.Shape, optional ...ShuffleDatasetAttr) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "ShuffleDataset",
-		Input: []tf.Input{
-			input_dataset, buffer_size, seed, seed2,
-		},
-		Attrs: attrs,
+// value: An second seed to avoid seed collision.
+// If not specified, defaults to 0
+func LearnedUnigramCandidateSamplerSeed2(value int64) LearnedUnigramCandidateSamplerAttr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// 3D fast Fourier transform.
+// Generates labels for candidate sampling with a learned unigram distribution.
 //
-// Computes the 3-dimensional discrete Fourier transform over the inner-most 3
-// dimensions of `input`.
+// See explanations of candidate sampling and the data formats at
+// go/candidate-sampling.
 //
-// Arguments:
-//	input: A complex64 tensor.
+// For each batch, this op picks a single set of sampled candidate labels.
 //
-// Returns A complex64 tensor of the same shape as `input`. The inner-most 3
-//   dimensions of `input` are replaced with their 3D Fourier transform.
+// The advantages of sampling candidates per-batch are simplicity and the
+// possibility of efficient dense matrix multiplication. The disadvantage is that
+// the sampled candidates must be chosen independently of the context and of the
+// true labels.
 //
-// @compatibility(numpy)
-// Equivalent to np.fft.fftn with 3 dimensions.
-// @end_compatibility
-func FFT3D(scope *Scope, input tf.Output) (output tf.Output) {
+// Arguments:
+//	true_classes: A batch_size * num_true matrix, in which each row contains the
+// IDs of the num_true target_classes in the corresponding original label.
+//	num_true: Number of true labels per context.
+//	num_sampled: Number of candidates to randomly sample.
+//	unique: If unique is true, we sample with rejection, so that all sampled
+// candidates in a batch are unique. This requires some approximation to
+// estimate the post-rejection sampling probabilities.
+//	range_max: The sampler will sample integers from the interval [0, range_max).
+//
+// Returns A vector of length num_sampled, in which each element is
+// the ID of a sampled candidate.A batch_size * num_true matrix, representing
+// the number of times each candidate is expected to occur in a batch
+// of sampled candidates. If unique=true, then this is a probability.A vector of length num_sampled, for each sampled
+// candidate representing the number of times the candidate is expected
+// to occur in a batch of sampled candidates.  If unique=true, then this is a
+// probability.
+func LearnedUnigramCandidateSampler(scope *Scope, true_classes tf.Output, num_true int64, num_sampled int64, unique bool, range_max int64, optional ...LearnedUnigramCandidateSamplerAttr) (sampled_candidates tf.Output, true_expected_count tf.Output, sampled_expected_count tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"num_true": num_true, "num_sampled": num_sampled, "unique": unique, "range_max": range_max}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "FFT3D",
+		Type: "LearnedUnigramCandidateSampler",
 		Input: []tf.Input{
-			input,
+			true_classes,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// CropAndResizeGradBoxesAttr is an optional argument to CropAndResizeGradBoxes.
-type CropAndResizeGradBoxesAttr func(optionalAttr)
+// SerializeSparseAttr is an optional argument to SerializeSparse.
+type SerializeSparseAttr func(optionalAttr)
 
-// CropAndResizeGradBoxesMethod sets the optional method attribute to value.
+// SerializeSparseOutType sets the optional out_type attribute to value.
 //
-// value: A string specifying the interpolation method. Only 'bilinear' is
-// supported for now.
-// If not specified, defaults to "bilinear"
-func CropAndResizeGradBoxesMethod(value string) CropAndResizeGradBoxesAttr {
+// value: The `dtype` to use for serialization; the supported types are `string`
+// (default) and `variant`.
+// If not specified, defaults to DT_STRING
+func SerializeSparseOutType(value tf.DataType) SerializeSparseAttr {
 	return func(m optionalAttr) {
-		m["method"] = value
+		m["out_type"] = value
 	}
 }
 
-// Computes the gradient of the crop_and_resize op wrt the input boxes tensor.
+// Serialize a `SparseTensor` into a `[3]` `Tensor` object.
 //
 // Arguments:
-//	grads: A 4-D tensor of shape `[num_boxes, crop_height, crop_width, depth]`.
-//	image: A 4-D tensor of shape `[batch, image_height, image_width, depth]`.
-// Both `image_height` and `image_width` need to be positive.
-//	boxes: A 2-D tensor of shape `[num_boxes, 4]`. The `i`-th row of the tensor
-// specifies the coordinates of a box in the `box_ind[i]` image and is specified
-// in normalized coordinates `[y1, x1, y2, x2]`. A normalized coordinate value of
-// `y` is mapped to the image coordinate at `y * (image_height - 1)`, so as the
-// `[0, 1]` interval of normalized image height is mapped to
-// `[0, image_height - 1] in image height coordinates. We do allow y1 > y2, in
-// which case the sampled crop is an up-down flipped version of the original
-// image. The width dimension is treated similarly. Normalized coordinates
-// outside the `[0, 1]` range are allowed, in which case we use
-// `extrapolation_value` to extrapolate the input image values.
-//	box_ind: A 1-D tensor of shape `[num_boxes]` with int32 values in `[0, batch)`.
-// The value of `box_ind[i]` specifies the image that the `i`-th box refers to.
-//
-// Returns A 2-D tensor of shape `[num_boxes, 4]`.
-func CropAndResizeGradBoxes(scope *Scope, grads tf.Output, image tf.Output, boxes tf.Output, box_ind tf.Output, optional ...CropAndResizeGradBoxesAttr) (output tf.Output) {
+//	sparse_indices: 2-D.  The `indices` of the `SparseTensor`.
+//	sparse_values: 1-D.  The `values` of the `SparseTensor`.
+//	sparse_shape: 1-D.  The `shape` of the `SparseTensor`.
+func SerializeSparse(scope *Scope, sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output, optional ...SerializeSparseAttr) (serialized_sparse tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -31719,9 +31343,9 @@ func CropAndResizeGradBoxes(scope *Scope, grads tf.Output, image tf.Output, boxe
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "CropAndResizeGradBoxes",
+		Type: "SerializeSparse",
 		Input: []tf.Input{
-			grads, image, boxes, box_ind,
+			sparse_indices, sparse_values, sparse_shape,
 		},
 		Attrs: attrs,
 	}
@@ -31729,241 +31353,406 @@ func CropAndResizeGradBoxes(scope *Scope, grads tf.Output, image tf.Output, boxe
 	return op.Output(0)
 }
 
-// Greedily selects a subset of bounding boxes in descending order of score,
-//
-// pruning away boxes that have high intersection-over-union (IOU) overlap
-// with previously selected boxes.  Bounding boxes with score less than
-// `score_threshold` are removed.  Bounding boxes are supplied as
-// [y1, x1, y2, x2], where (y1, x1) and (y2, x2) are the coordinates of any
-// diagonal pair of box corners and the coordinates can be provided as normalized
-// (i.e., lying in the interval [0, 1]) or absolute.  Note that this algorithm
-// is agnostic to where the origin is in the coordinate system and more
-// generally is invariant to orthogonal transformations and translations
-// of the coordinate system; thus translating or reflections of the coordinate
-// system result in the same boxes being selected by the algorithm.
-// The output of this operation is a set of integers indexing into the input
-// collection of bounding boxes representing the selected boxes.  The bounding
-// box coordinates corresponding to the selected indices can then be obtained
-// using the `tf.gather operation`.  For example:
-//   selected_indices = tf.image.non_max_suppression_v2(
-//       boxes, scores, max_output_size, iou_threshold, score_threshold)
-//   selected_boxes = tf.gather(boxes, selected_indices)
+// RandomShuffleQueueV2Attr is an optional argument to RandomShuffleQueueV2.
+type RandomShuffleQueueV2Attr func(optionalAttr)
+
+// RandomShuffleQueueV2Shapes sets the optional shapes attribute to value.
 //
-// Arguments:
-//	boxes: A 2-D float tensor of shape `[num_boxes, 4]`.
-//	scores: A 1-D float tensor of shape `[num_boxes]` representing a single
-// score corresponding to each box (each row of boxes).
-//	max_output_size: A scalar integer tensor representing the maximum number of
-// boxes to be selected by non max suppression.
-//	iou_threshold: A 0-D float tensor representing the threshold for deciding whether
-// boxes overlap too much with respect to IOU.
-//	score_threshold: A 0-D float tensor representing the threshold for deciding when to remove
-// boxes based on score.
+// value: The shape of each component in a value. The length of this attr must
+// be either 0 or the same as the length of component_types. If the length of
+// this attr is 0, the shapes of queue elements are not constrained, and
+// only one element may be dequeued at a time.
+// If not specified, defaults to <>
 //
-// Returns A 1-D integer tensor of shape `[M]` representing the selected
-// indices from the boxes tensor, where `M <= max_output_size`.
-func NonMaxSuppressionV3(scope *Scope, boxes tf.Output, scores tf.Output, max_output_size tf.Output, iou_threshold tf.Output, score_threshold tf.Output) (selected_indices tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "NonMaxSuppressionV3",
-		Input: []tf.Input{
-			boxes, scores, max_output_size, iou_threshold, score_threshold,
-		},
+// REQUIRES: len(value) >= 0
+func RandomShuffleQueueV2Shapes(value []tf.Shape) RandomShuffleQueueV2Attr {
+	return func(m optionalAttr) {
+		m["shapes"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// NonMaxSuppressionV4Attr is an optional argument to NonMaxSuppressionV4.
-type NonMaxSuppressionV4Attr func(optionalAttr)
-
-// NonMaxSuppressionV4PadToMaxOutputSize sets the optional pad_to_max_output_size attribute to value.
+// RandomShuffleQueueV2Capacity sets the optional capacity attribute to value.
 //
-// value: If true, the output `selected_indices` is padded to be of length
-// `max_output_size`. Defaults to false.
-// If not specified, defaults to false
-func NonMaxSuppressionV4PadToMaxOutputSize(value bool) NonMaxSuppressionV4Attr {
+// value: The upper bound on the number of elements in this queue.
+// Negative numbers mean no limit.
+// If not specified, defaults to -1
+func RandomShuffleQueueV2Capacity(value int64) RandomShuffleQueueV2Attr {
 	return func(m optionalAttr) {
-		m["pad_to_max_output_size"] = value
+		m["capacity"] = value
 	}
 }
 
-// Greedily selects a subset of bounding boxes in descending order of score,
+// RandomShuffleQueueV2MinAfterDequeue sets the optional min_after_dequeue attribute to value.
 //
-// pruning away boxes that have high intersection-over-union (IOU) overlap
-// with previously selected boxes.  Bounding boxes with score less than
-// `score_threshold` are removed.  Bounding boxes are supplied as
-// [y1, x1, y2, x2], where (y1, x1) and (y2, x2) are the coordinates of any
-// diagonal pair of box corners and the coordinates can be provided as normalized
-// (i.e., lying in the interval [0, 1]) or absolute.  Note that this algorithm
-// is agnostic to where the origin is in the coordinate system and more
-// generally is invariant to orthogonal transformations and translations
-// of the coordinate system; thus translating or reflections of the coordinate
-// system result in the same boxes being selected by the algorithm.
-// The output of this operation is a set of integers indexing into the input
-// collection of bounding boxes representing the selected boxes.  The bounding
-// box coordinates corresponding to the selected indices can then be obtained
-// using the `tf.gather operation`.  For example:
-//   selected_indices = tf.image.non_max_suppression_v2(
-//       boxes, scores, max_output_size, iou_threshold, score_threshold)
-//   selected_boxes = tf.gather(boxes, selected_indices)
+// value: Dequeue will block unless there would be this
+// many elements after the dequeue or the queue is closed. This
+// ensures a minimum level of mixing of elements.
+// If not specified, defaults to 0
+func RandomShuffleQueueV2MinAfterDequeue(value int64) RandomShuffleQueueV2Attr {
+	return func(m optionalAttr) {
+		m["min_after_dequeue"] = value
+	}
+}
+
+// RandomShuffleQueueV2Seed sets the optional seed attribute to value.
+//
+// value: If either seed or seed2 is set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, a random seed is used.
+// If not specified, defaults to 0
+func RandomShuffleQueueV2Seed(value int64) RandomShuffleQueueV2Attr {
+	return func(m optionalAttr) {
+		m["seed"] = value
+	}
+}
+
+// RandomShuffleQueueV2Seed2 sets the optional seed2 attribute to value.
+//
+// value: A second seed to avoid seed collision.
+// If not specified, defaults to 0
+func RandomShuffleQueueV2Seed2(value int64) RandomShuffleQueueV2Attr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// RandomShuffleQueueV2Container sets the optional container attribute to value.
+//
+// value: If non-empty, this queue is placed in the given container.
+// Otherwise, a default container is used.
+// If not specified, defaults to ""
+func RandomShuffleQueueV2Container(value string) RandomShuffleQueueV2Attr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// RandomShuffleQueueV2SharedName sets the optional shared_name attribute to value.
+//
+// value: If non-empty, this queue will be shared under the given name
+// across multiple sessions.
+// If not specified, defaults to ""
+func RandomShuffleQueueV2SharedName(value string) RandomShuffleQueueV2Attr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// A queue that randomizes the order of elements.
 //
 // Arguments:
-//	boxes: A 2-D float tensor of shape `[num_boxes, 4]`.
-//	scores: A 1-D float tensor of shape `[num_boxes]` representing a single
-// score corresponding to each box (each row of boxes).
-//	max_output_size: A scalar integer tensor representing the maximum number of
-// boxes to be selected by non max suppression.
-//	iou_threshold: A 0-D float tensor representing the threshold for deciding whether
-// boxes overlap too much with respect to IOU.
-//	score_threshold: A 0-D float tensor representing the threshold for deciding when to remove
-// boxes based on score.
+//	component_types: The type of each component in a value.
 //
-// Returns A 1-D integer tensor of shape `[M]` representing the selected
-// indices from the boxes tensor, where `M <= max_output_size`.A 0-D integer tensor representing the number of valid elements in
-// `selected_indices`, with the valid elements appearing first.
-func NonMaxSuppressionV4(scope *Scope, boxes tf.Output, scores tf.Output, max_output_size tf.Output, iou_threshold tf.Output, score_threshold tf.Output, optional ...NonMaxSuppressionV4Attr) (selected_indices tf.Output, valid_outputs tf.Output) {
+// Returns The handle to the queue.
+func RandomShuffleQueueV2(scope *Scope, component_types []tf.DataType, optional ...RandomShuffleQueueV2Attr) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"component_types": component_types}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "NonMaxSuppressionV4",
-		Input: []tf.Input{
-			boxes, scores, max_output_size, iou_threshold, score_threshold,
-		},
+		Type: "RandomShuffleQueueV2",
+
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// Computes the matrix logarithm of one or more square matrices:
-//
-//
-// \\(log(exp(A)) = A\\)
+// Draw bounding boxes on a batch of images.
 //
-// This op is only defined for complex matrices. If A is positive-definite and
-// real, then casting to a complex matrix, taking the logarithm and casting back
-// to a real matrix will give the correct result.
+// Outputs a copy of `images` but draws on top of the pixels zero or more bounding
+// boxes specified by the locations in `boxes`. The coordinates of the each
+// bounding box in `boxes` are encoded as `[y_min, x_min, y_max, x_max]`. The
+// bounding box coordinates are floats in `[0.0, 1.0]` relative to the width and
+// height of the underlying image.
 //
-// This function computes the matrix logarithm using the Schur-Parlett algorithm.
-// Details of the algorithm can be found in Section 11.6.2 of:
-// Nicholas J. Higham, Functions of Matrices: Theory and Computation, SIAM 2008.
-// ISBN 978-0-898716-46-7.
+// For example, if an image is 100 x 200 pixels (height x width) and the bounding
+// box is `[0.1, 0.2, 0.5, 0.9]`, the upper-left and bottom-right coordinates of
+// the bounding box will be `(40, 10)` to `(180, 50)` (in (x,y) coordinates).
 //
-// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
-// form square matrices. The output is a tensor of the same shape as the input
-// containing the exponential for all input submatrices `[..., :, :]`.
+// Parts of the bounding box may fall outside the image.
 //
 // Arguments:
-//	input: Shape is `[..., M, M]`.
-//
-// Returns Shape is `[..., M, M]`.
+//	images: 4-D with shape `[batch, height, width, depth]`. A batch of images.
+//	boxes: 3-D with shape `[batch, num_bounding_boxes, 4]` containing bounding
+// boxes.
 //
-// @compatibility(scipy)
-// Equivalent to scipy.linalg.logm
-// @end_compatibility
-func MatrixLogarithm(scope *Scope, input tf.Output) (output tf.Output) {
+// Returns 4-D with the same shape as `images`. The batch of input images with
+// bounding boxes drawn on the images.
+func DrawBoundingBoxes(scope *Scope, images tf.Output, boxes tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "MatrixLogarithm",
+		Type: "DrawBoundingBoxes",
 		Input: []tf.Input{
-			input,
+			images, boxes,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-//   This op is used as a placeholder in If branch functions. It doesn't provide a
-//   valid output when run, so must either be removed (e.g. replaced with a
-//   function input) or guaranteed not to be used (e.g. if mirroring an
-//   intermediate output needed for the gradient computation of the other branch).
-//
-// Arguments:
-//	dtype: The type of the output.
-//	shape:     The purported shape of the output. This is only used for shape inference;
-//     the output will not necessarily have this shape. Can be a partial shape.
+// Gets the next output from the given iterator.
 //
-// Returns     \"Fake\" output value. This should not be consumed by another op.
-func FakeParam(scope *Scope, dtype tf.DataType, shape tf.Shape) (output tf.Output) {
+// This operation is a synchronous version IteratorGetNext. It should only be used
+// in situations where the iterator does not block the calling thread, or where
+// the calling thread is not a member of the thread pool used to execute parallel
+// operations (e.g. in eager mode).
+func IteratorGetNextSync(scope *Scope, iterator tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (components []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype, "shape": shape}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "FakeParam",
-
+		Type: "IteratorGetNextSync",
+		Input: []tf.Input{
+			iterator,
+		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if components, idx, err = makeOutputList(op, idx, "components"); err != nil {
+		scope.UpdateErr("IteratorGetNextSync", err)
+		return
+	}
+	return components
 }
 
-// Returns the next representable value of `x1` in the direction of `x2`, element-wise.
+// SampleDistortedBoundingBoxV2Attr is an optional argument to SampleDistortedBoundingBoxV2.
+type SampleDistortedBoundingBoxV2Attr func(optionalAttr)
+
+// SampleDistortedBoundingBoxV2Seed sets the optional seed attribute to value.
 //
-// This operation returns the same result as the C++ std::nextafter function.
+// value: If either `seed` or `seed2` are set to non-zero, the random number
+// generator is seeded by the given `seed`.  Otherwise, it is seeded by a random
+// seed.
+// If not specified, defaults to 0
+func SampleDistortedBoundingBoxV2Seed(value int64) SampleDistortedBoundingBoxV2Attr {
+	return func(m optionalAttr) {
+		m["seed"] = value
+	}
+}
+
+// SampleDistortedBoundingBoxV2Seed2 sets the optional seed2 attribute to value.
 //
-// It can also return a subnormal number.
+// value: A second seed to avoid seed collision.
+// If not specified, defaults to 0
+func SampleDistortedBoundingBoxV2Seed2(value int64) SampleDistortedBoundingBoxV2Attr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// SampleDistortedBoundingBoxV2AspectRatioRange sets the optional aspect_ratio_range attribute to value.
 //
-// @compatibility(cpp)
-// Equivalent to C++ std::nextafter function.
-// @end_compatibility
-func NextAfter(scope *Scope, x1 tf.Output, x2 tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
+// value: The cropped area of the image must have an aspect ratio =
+// width / height within this range.
+// If not specified, defaults to <f:0.75 f:1.33 >
+func SampleDistortedBoundingBoxV2AspectRatioRange(value []float32) SampleDistortedBoundingBoxV2Attr {
+	return func(m optionalAttr) {
+		m["aspect_ratio_range"] = value
 	}
-	opspec := tf.OpSpec{
-		Type: "NextAfter",
-		Input: []tf.Input{
-			x1, x2,
-		},
+}
+
+// SampleDistortedBoundingBoxV2AreaRange sets the optional area_range attribute to value.
+//
+// value: The cropped area of the image must contain a fraction of the
+// supplied image within this range.
+// If not specified, defaults to <f:0.05 f:1 >
+func SampleDistortedBoundingBoxV2AreaRange(value []float32) SampleDistortedBoundingBoxV2Attr {
+	return func(m optionalAttr) {
+		m["area_range"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Computes the gradient for the inverse of `x` wrt its input.
+// SampleDistortedBoundingBoxV2MaxAttempts sets the optional max_attempts attribute to value.
 //
-// Specifically, `grad = -dy * y*y`, where `y = 1/x`, and `dy`
-// is the corresponding input gradient.
-func InvGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
+// value: Number of attempts at generating a cropped region of the image
+// of the specified constraints. After `max_attempts` failures, return the entire
+// image.
+// If not specified, defaults to 100
+func SampleDistortedBoundingBoxV2MaxAttempts(value int64) SampleDistortedBoundingBoxV2Attr {
+	return func(m optionalAttr) {
+		m["max_attempts"] = value
+	}
+}
+
+// SampleDistortedBoundingBoxV2UseImageIfNoBoundingBoxes sets the optional use_image_if_no_bounding_boxes attribute to value.
+//
+// value: Controls behavior if no bounding boxes supplied.
+// If true, assume an implicit bounding box covering the whole input. If false,
+// raise an error.
+// If not specified, defaults to false
+func SampleDistortedBoundingBoxV2UseImageIfNoBoundingBoxes(value bool) SampleDistortedBoundingBoxV2Attr {
+	return func(m optionalAttr) {
+		m["use_image_if_no_bounding_boxes"] = value
+	}
+}
+
+// Generate a single randomly distorted bounding box for an image.
+//
+// Bounding box annotations are often supplied in addition to ground-truth labels
+// in image recognition or object localization tasks. A common technique for
+// training such a system is to randomly distort an image while preserving
+// its content, i.e. *data augmentation*. This Op outputs a randomly distorted
+// localization of an object, i.e. bounding box, given an `image_size`,
+// `bounding_boxes` and a series of constraints.
+//
+// The output of this Op is a single bounding box that may be used to crop the
+// original image. The output is returned as 3 tensors: `begin`, `size` and
+// `bboxes`. The first 2 tensors can be fed directly into `tf.slice` to crop the
+// image. The latter may be supplied to `tf.image.draw_bounding_boxes` to visualize
+// what the bounding box looks like.
+//
+// Bounding boxes are supplied and returned as `[y_min, x_min, y_max, x_max]`. The
+// bounding box coordinates are floats in `[0.0, 1.0]` relative to the width and
+// height of the underlying image.
+//
+// For example,
+//
+// ```python
+//     # Generate a single distorted bounding box.
+//     begin, size, bbox_for_draw = tf.image.sample_distorted_bounding_box(
+//         tf.shape(image),
+//         bounding_boxes=bounding_boxes)
+//
+//     # Draw the bounding box in an image summary.
+//     image_with_box = tf.image.draw_bounding_boxes(tf.expand_dims(image, 0),
+//                                                   bbox_for_draw)
+//     tf.summary.image('images_with_box', image_with_box)
+//
+//     # Employ the bounding box to distort the image.
+//     distorted_image = tf.slice(image, begin, size)
+// ```
+//
+// Note that if no bounding box information is available, setting
+// `use_image_if_no_bounding_boxes = true` will assume there is a single implicit
+// bounding box covering the whole image. If `use_image_if_no_bounding_boxes` is
+// false and no bounding boxes are supplied, an error is raised.
+//
+// Arguments:
+//	image_size: 1-D, containing `[height, width, channels]`.
+//	bounding_boxes: 3-D with shape `[batch, N, 4]` describing the N bounding boxes
+// associated with the image.
+//	min_object_covered: The cropped area of the image must contain at least this
+// fraction of any bounding box supplied. The value of this parameter should be
+// non-negative. In the case of 0, the cropped area does not need to overlap
+// any of the bounding boxes supplied.
+//
+// Returns 1-D, containing `[offset_height, offset_width, 0]`. Provide as input to
+// `tf.slice`.1-D, containing `[target_height, target_width, -1]`. Provide as input to
+// `tf.slice`.3-D with shape `[1, 1, 4]` containing the distorted bounding box.
+// Provide as input to `tf.image.draw_bounding_boxes`.
+func SampleDistortedBoundingBoxV2(scope *Scope, image_size tf.Output, bounding_boxes tf.Output, min_object_covered tf.Output, optional ...SampleDistortedBoundingBoxV2Attr) (begin tf.Output, size tf.Output, bboxes tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "InvGrad",
+		Type: "SampleDistortedBoundingBoxV2",
 		Input: []tf.Input{
-			y, dy,
+			image_size, bounding_boxes, min_object_covered,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// List of the given size with empty elements.
+// ExtractGlimpseAttr is an optional argument to ExtractGlimpse.
+type ExtractGlimpseAttr func(optionalAttr)
+
+// ExtractGlimpseCentered sets the optional centered attribute to value.
+//
+// value: indicates if the offset coordinates are centered relative to
+// the image, in which case the (0, 0) offset is relative to the center
+// of the input images. If false, the (0,0) offset corresponds to the
+// upper left corner of the input images.
+// If not specified, defaults to true
+func ExtractGlimpseCentered(value bool) ExtractGlimpseAttr {
+	return func(m optionalAttr) {
+		m["centered"] = value
+	}
+}
+
+// ExtractGlimpseNormalized sets the optional normalized attribute to value.
+//
+// value: indicates if the offset coordinates are normalized.
+// If not specified, defaults to true
+func ExtractGlimpseNormalized(value bool) ExtractGlimpseAttr {
+	return func(m optionalAttr) {
+		m["normalized"] = value
+	}
+}
+
+// ExtractGlimpseUniformNoise sets the optional uniform_noise attribute to value.
+//
+// value: indicates if the noise should be generated using a
+// uniform distribution or a Gaussian distribution.
+// If not specified, defaults to true
+func ExtractGlimpseUniformNoise(value bool) ExtractGlimpseAttr {
+	return func(m optionalAttr) {
+		m["uniform_noise"] = value
+	}
+}
+
+// Extracts a glimpse from the input tensor.
+//
+// Returns a set of windows called glimpses extracted at location
+// `offsets` from the input tensor. If the windows only partially
+// overlaps the inputs, the non overlapping areas will be filled with
+// random noise.
+//
+// The result is a 4-D tensor of shape `[batch_size, glimpse_height,
+// glimpse_width, channels]`. The channels and batch dimensions are the
+// same as that of the input tensor. The height and width of the output
+// windows are specified in the `size` parameter.
+//
+// The argument `normalized` and `centered` controls how the windows are built:
+//
+// * If the coordinates are normalized but not centered, 0.0 and 1.0
+//   correspond to the minimum and maximum of each height and width
+//   dimension.
+// * If the coordinates are both normalized and centered, they range from
+//   -1.0 to 1.0. The coordinates (-1.0, -1.0) correspond to the upper
+//   left corner, the lower right corner is located at (1.0, 1.0) and the
+//   center is at (0, 0).
+// * If the coordinates are not normalized they are interpreted as
+//   numbers of pixels.
 //
-// element_shape: the shape of the future elements of the list
-// num_elements: the number of elements to reserve
-// handle: the output list
-// element_dtype: the desired type of elements in the list.
-func TensorListReserve(scope *Scope, element_shape tf.Output, num_elements tf.Output, element_dtype tf.DataType) (handle tf.Output) {
+// Arguments:
+//	input: A 4-D float tensor of shape `[batch_size, height, width, channels]`.
+//	size: A 1-D tensor of 2 elements containing the size of the glimpses
+// to extract.  The glimpse height must be specified first, following
+// by the glimpse width.
+//	offsets: A 2-D integer tensor of shape `[batch_size, 2]` containing
+// the y, x locations of the center of each window.
+//
+// Returns A tensor representing the glimpses `[batch_size,
+// glimpse_height, glimpse_width, channels]`.
+func ExtractGlimpse(scope *Scope, input tf.Output, size tf.Output, offsets tf.Output, optional ...ExtractGlimpseAttr) (glimpse tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"element_dtype": element_dtype}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "TensorListReserve",
+		Type: "ExtractGlimpse",
 		Input: []tf.Input{
-			element_shape, num_elements,
+			input, size, offsets,
 		},
 		Attrs: attrs,
 	}
@@ -31971,73 +31760,45 @@ func TensorListReserve(scope *Scope, element_shape tf.Output, num_elements tf.Ou
 	return op.Output(0)
 }
 
-// A substitute for `InterleaveDataset` on a fixed list of `N` datasets.
-//
-// Arguments:
-//	selector_input_dataset: A dataset of scalar `DT_INT64` elements that determines which of the
-// `N` data inputs should produce the next output element.
-//	data_input_datasets: `N` datasets with the same type that will be interleaved according to
-// the values of `selector_input_dataset`.
-//
+// A container for an iterator resource.
 //
-func ExperimentalDirectedInterleaveDataset(scope *Scope, selector_input_dataset tf.Output, data_input_datasets []tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+// Returns A handle to the iterator that can be passed to a "MakeIterator"
+// or "IteratorGetNext" op.
+func Iterator(scope *Scope, shared_name string, container string, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	attrs := map[string]interface{}{"shared_name": shared_name, "container": container, "output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "ExperimentalDirectedInterleaveDataset",
-		Input: []tf.Input{
-			selector_input_dataset, tf.OutputList(data_input_datasets),
-		},
+		Type: "Iterator",
+
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// RandomUniformIntAttr is an optional argument to RandomUniformInt.
-type RandomUniformIntAttr func(optionalAttr)
+// TensorForestTreeResourceHandleOpAttr is an optional argument to TensorForestTreeResourceHandleOp.
+type TensorForestTreeResourceHandleOpAttr func(optionalAttr)
 
-// RandomUniformIntSeed sets the optional seed attribute to value.
-//
-// value: If either `seed` or `seed2` are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func RandomUniformIntSeed(value int64) RandomUniformIntAttr {
+// TensorForestTreeResourceHandleOpContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func TensorForestTreeResourceHandleOpContainer(value string) TensorForestTreeResourceHandleOpAttr {
 	return func(m optionalAttr) {
-		m["seed"] = value
+		m["container"] = value
 	}
 }
 
-// RandomUniformIntSeed2 sets the optional seed2 attribute to value.
-//
-// value: A second seed to avoid seed collision.
-// If not specified, defaults to 0
-func RandomUniformIntSeed2(value int64) RandomUniformIntAttr {
+// TensorForestTreeResourceHandleOpSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func TensorForestTreeResourceHandleOpSharedName(value string) TensorForestTreeResourceHandleOpAttr {
 	return func(m optionalAttr) {
-		m["seed2"] = value
+		m["shared_name"] = value
 	}
 }
 
-// Outputs random integers from a uniform distribution.
-//
-// The generated values are uniform integers in the range `[minval, maxval)`.
-// The lower bound `minval` is included in the range, while the upper bound
-// `maxval` is excluded.
-//
-// The random integers are slightly biased unless `maxval - minval` is an exact
-// power of two.  The bias is small for values of `maxval - minval` significantly
-// smaller than the range of the output (either `2^32` or `2^64`).
-//
-// Arguments:
-//	shape: The shape of the output tensor.
-//	minval: 0-D.  Inclusive lower bound on the generated integers.
-//	maxval: 0-D.  Exclusive upper bound on the generated integers.
-//
-// Returns A tensor of the specified shape filled with uniform random integers.
-func RandomUniformInt(scope *Scope, shape tf.Output, minval tf.Output, maxval tf.Output, optional ...RandomUniformIntAttr) (output tf.Output) {
+// Creates a handle to a TensorForestTreeResource
+func TensorForestTreeResourceHandleOp(scope *Scope, optional ...TensorForestTreeResourceHandleOpAttr) (resource tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -32046,58 +31807,62 @@ func RandomUniformInt(scope *Scope, shape tf.Output, minval tf.Output, maxval tf
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "RandomUniformInt",
-		Input: []tf.Input{
-			shape, minval, maxval,
-		},
+		Type: "TensorForestTreeResourceHandleOp",
+
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Add the quantile summaries to each quantile stream resource.
-//
-// An op that adds a list of quantile summaries to a quantile stream resource. Each
-// summary Tensor is rank 2, containing summaries (value, weight, min_rank, max_rank)
-// for a single feature.
-//
-// Arguments:
-//	quantile_stream_resource_handle: resource handle referring to a QuantileStreamResource.
-//	summaries: string; List of Rank 2 Tensor each containing the summaries for a single feature.
+// CropAndResizeGradImageAttr is an optional argument to CropAndResizeGradImage.
+type CropAndResizeGradImageAttr func(optionalAttr)
+
+// CropAndResizeGradImageMethod sets the optional method attribute to value.
 //
-// Returns the created operation.
-func BoostedTreesQuantileStreamResourceAddSummaries(scope *Scope, quantile_stream_resource_handle tf.Output, summaries []tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "BoostedTreesQuantileStreamResourceAddSummaries",
-		Input: []tf.Input{
-			quantile_stream_resource_handle, tf.OutputList(summaries),
-		},
+// value: A string specifying the interpolation method. Only 'bilinear' is
+// supported for now.
+// If not specified, defaults to "bilinear"
+func CropAndResizeGradImageMethod(value string) CropAndResizeGradImageAttr {
+	return func(m optionalAttr) {
+		m["method"] = value
 	}
-	return scope.AddOperation(opspec)
 }
 
-// Creates a Dataset that returns pseudorandom numbers.
+// Computes the gradient of the crop_and_resize op wrt the input image tensor.
 //
 // Arguments:
-//	seed: A scalar seed for the random number generator. If either seed or
-// seed2 is set to be non-zero, the random number generator is seeded
-// by the given seed.  Otherwise, a random seed is used.
-//	seed2: A second scalar seed to avoid seed collision.
+//	grads: A 4-D tensor of shape `[num_boxes, crop_height, crop_width, depth]`.
+//	boxes: A 2-D tensor of shape `[num_boxes, 4]`. The `i`-th row of the tensor
+// specifies the coordinates of a box in the `box_ind[i]` image and is specified
+// in normalized coordinates `[y1, x1, y2, x2]`. A normalized coordinate value of
+// `y` is mapped to the image coordinate at `y * (image_height - 1)`, so as the
+// `[0, 1]` interval of normalized image height is mapped to
+// `[0, image_height - 1] in image height coordinates. We do allow y1 > y2, in
+// which case the sampled crop is an up-down flipped version of the original
+// image. The width dimension is treated similarly. Normalized coordinates
+// outside the `[0, 1]` range are allowed, in which case we use
+// `extrapolation_value` to extrapolate the input image values.
+//	box_ind: A 1-D tensor of shape `[num_boxes]` with int32 values in `[0, batch)`.
+// The value of `box_ind[i]` specifies the image that the `i`-th box refers to.
+//	image_size: A 1-D tensor with value `[batch, image_height, image_width, depth]`
+// containing the original image size. Both `image_height` and `image_width` need
+// to be positive.
 //
 //
-func ExperimentalRandomDataset(scope *Scope, seed tf.Output, seed2 tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+// Returns A 4-D tensor of shape `[batch, image_height, image_width, depth]`.
+func CropAndResizeGradImage(scope *Scope, grads tf.Output, boxes tf.Output, box_ind tf.Output, image_size tf.Output, T tf.DataType, optional ...CropAndResizeGradImageAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	attrs := map[string]interface{}{"T": T}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "ExperimentalRandomDataset",
+		Type: "CropAndResizeGradImage",
 		Input: []tf.Input{
-			seed, seed2,
+			grads, boxes, box_ind, image_size,
 		},
 		Attrs: attrs,
 	}
@@ -32105,39 +31870,48 @@ func ExperimentalRandomDataset(scope *Scope, seed tf.Output, seed2 tf.Output, ou
 	return op.Output(0)
 }
 
-// A dataset that splits the elements of its input into multiple elements.
-func ExperimentalUnbatchDataset(scope *Scope, input_dataset tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
-	opspec := tf.OpSpec{
-		Type: "ExperimentalUnbatchDataset",
-		Input: []tf.Input{
-			input_dataset,
-		},
-		Attrs: attrs,
+// ShuffleDatasetAttr is an optional argument to ShuffleDataset.
+type ShuffleDatasetAttr func(optionalAttr)
+
+// ShuffleDatasetReshuffleEachIteration sets the optional reshuffle_each_iteration attribute to value.
+//
+// value: If true, each iterator over this dataset will be given
+// a different pseudorandomly generated seed, based on a sequence seeded by the
+// `seed` and `seed2` inputs. If false, each iterator will be given the same
+// seed, and repeated iteration over this dataset will yield the exact same
+// sequence of results.
+// If not specified, defaults to true
+func ShuffleDatasetReshuffleEachIteration(value bool) ShuffleDatasetAttr {
+	return func(m optionalAttr) {
+		m["reshuffle_each_iteration"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Creates a dataset that overrides the maximum intra-op parallelism.
+// Creates a dataset that shuffles elements from `input_dataset` pseudorandomly.
 //
 // Arguments:
 //
-//	max_intra_op_parallelism: Identifies the maximum intra-op parallelism to use.
+//	buffer_size: The number of output elements to buffer in an iterator over
+// this dataset. Compare with the `min_after_dequeue` attr when creating a
+// `RandomShuffleQueue`.
+//	seed: A scalar seed for the random number generator. If either `seed` or
+// `seed2` is set to be non-zero, the random number generator is seeded
+// by the given seed.  Otherwise, a random seed is used.
+//	seed2: A second scalar seed to avoid seed collision.
 //
 //
-func ExperimentalMaxIntraOpParallelismDataset(scope *Scope, input_dataset tf.Output, max_intra_op_parallelism tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+func ShuffleDataset(scope *Scope, input_dataset tf.Output, buffer_size tf.Output, seed tf.Output, seed2 tf.Output, output_types []tf.DataType, output_shapes []tf.Shape, optional ...ShuffleDatasetAttr) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "ExperimentalMaxIntraOpParallelismDataset",
+		Type: "ShuffleDataset",
 		Input: []tf.Input{
-			input_dataset, max_intra_op_parallelism,
+			input_dataset, buffer_size, seed, seed2,
 		},
 		Attrs: attrs,
 	}
@@ -32145,84 +31919,80 @@ func ExperimentalMaxIntraOpParallelismDataset(scope *Scope, input_dataset tf.Out
 	return op.Output(0)
 }
 
-// StringSplitV2Attr is an optional argument to StringSplitV2.
-type StringSplitV2Attr func(optionalAttr)
-
-// StringSplitV2Maxsplit sets the optional maxsplit attribute to value.
-//
-// value: An `int`. If `maxsplit > 0`, limit of the split of the result.
-// If not specified, defaults to -1
-func StringSplitV2Maxsplit(value int64) StringSplitV2Attr {
-	return func(m optionalAttr) {
-		m["maxsplit"] = value
-	}
-}
-
-// Split elements of `source` based on `sep` into a `SparseTensor`.
-//
-// Let N be the size of source (typically N will be the batch size). Split each
-// element of `source` based on `sep` and return a `SparseTensor`
-// containing the split tokens. Empty tokens are ignored.
+// 3D fast Fourier transform.
 //
-// For example, N = 2, source[0] is 'hello world' and source[1] is 'a b c',
-// then the output will be
-// ```
-// st.indices = [0, 0;
-//               0, 1;
-//               1, 0;
-//               1, 1;
-//               1, 2]
-// st.shape = [2, 3]
-// st.values = ['hello', 'world', 'a', 'b', 'c']
-// ```
+// Computes the 3-dimensional discrete Fourier transform over the inner-most 3
+// dimensions of `input`.
 //
-// If `sep` is given, consecutive delimiters are not grouped together and are
-// deemed to delimit empty strings. For example, source of `"1<>2<><>3"` and
-// sep of `"<>"` returns `["1", "2", "", "3"]`. If `sep` is None or an empty
-// string, consecutive whitespace are regarded as a single separator, and the
-// result will contain no empty strings at the startor end if the string has
-// leading or trailing whitespace.
+// Arguments:
+//	input: A complex64 tensor.
 //
-// Note that the above mentioned behavior matches python's str.split.
+// Returns A complex64 tensor of the same shape as `input`. The inner-most 3
+//   dimensions of `input` are replaced with their 3D Fourier transform.
 //
-// Arguments:
-//	input: `1-D` string `Tensor`, the strings to split.
-//	sep: `0-D` string `Tensor`, the delimiter character.
-func StringSplitV2(scope *Scope, input tf.Output, sep tf.Output, optional ...StringSplitV2Attr) (indices tf.Output, values tf.Output, shape tf.Output) {
+// @compatibility(numpy)
+// Equivalent to np.fft.fftn with 3 dimensions.
+// @end_compatibility
+func FFT3D(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "StringSplitV2",
+		Type: "FFT3D",
 		Input: []tf.Input{
-			input, sep,
+			input,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// Creates a dataset that uses a custom thread pool to compute `input_dataset`.
-//
-// Arguments:
+// CropAndResizeGradBoxesAttr is an optional argument to CropAndResizeGradBoxes.
+type CropAndResizeGradBoxesAttr func(optionalAttr)
+
+// CropAndResizeGradBoxesMethod sets the optional method attribute to value.
 //
-//	thread_pool: A resource produced by the ThreadPoolHandle op.
+// value: A string specifying the interpolation method. Only 'bilinear' is
+// supported for now.
+// If not specified, defaults to "bilinear"
+func CropAndResizeGradBoxesMethod(value string) CropAndResizeGradBoxesAttr {
+	return func(m optionalAttr) {
+		m["method"] = value
+	}
+}
+
+// Computes the gradient of the crop_and_resize op wrt the input boxes tensor.
 //
+// Arguments:
+//	grads: A 4-D tensor of shape `[num_boxes, crop_height, crop_width, depth]`.
+//	image: A 4-D tensor of shape `[batch, image_height, image_width, depth]`.
+// Both `image_height` and `image_width` need to be positive.
+//	boxes: A 2-D tensor of shape `[num_boxes, 4]`. The `i`-th row of the tensor
+// specifies the coordinates of a box in the `box_ind[i]` image and is specified
+// in normalized coordinates `[y1, x1, y2, x2]`. A normalized coordinate value of
+// `y` is mapped to the image coordinate at `y * (image_height - 1)`, so as the
+// `[0, 1]` interval of normalized image height is mapped to
+// `[0, image_height - 1] in image height coordinates. We do allow y1 > y2, in
+// which case the sampled crop is an up-down flipped version of the original
+// image. The width dimension is treated similarly. Normalized coordinates
+// outside the `[0, 1]` range are allowed, in which case we use
+// `extrapolation_value` to extrapolate the input image values.
+//	box_ind: A 1-D tensor of shape `[num_boxes]` with int32 values in `[0, batch)`.
+// The value of `box_ind[i]` specifies the image that the `i`-th box refers to.
 //
-func ExperimentalThreadPoolDataset(scope *Scope, input_dataset tf.Output, thread_pool tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+// Returns A 2-D tensor of shape `[num_boxes, 4]`.
+func CropAndResizeGradBoxes(scope *Scope, grads tf.Output, image tf.Output, boxes tf.Output, box_ind tf.Output, optional ...CropAndResizeGradBoxesAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "ExperimentalThreadPoolDataset",
+		Type: "CropAndResizeGradBoxes",
 		Input: []tf.Input{
-			input_dataset, thread_pool,
+			grads, image, boxes, box_ind,
 		},
 		Attrs: attrs,
 	}
@@ -32230,255 +32000,241 @@ func ExperimentalThreadPoolDataset(scope *Scope, input_dataset tf.Output, thread
 	return op.Output(0)
 }
 
-// Computes softsign: `features / (abs(features) + 1)`.
-func Softsign(scope *Scope, features tf.Output) (activations tf.Output) {
+// Greedily selects a subset of bounding boxes in descending order of score,
+//
+// pruning away boxes that have high intersection-over-union (IOU) overlap
+// with previously selected boxes.  Bounding boxes with score less than
+// `score_threshold` are removed.  Bounding boxes are supplied as
+// [y1, x1, y2, x2], where (y1, x1) and (y2, x2) are the coordinates of any
+// diagonal pair of box corners and the coordinates can be provided as normalized
+// (i.e., lying in the interval [0, 1]) or absolute.  Note that this algorithm
+// is agnostic to where the origin is in the coordinate system and more
+// generally is invariant to orthogonal transformations and translations
+// of the coordinate system; thus translating or reflections of the coordinate
+// system result in the same boxes being selected by the algorithm.
+// The output of this operation is a set of integers indexing into the input
+// collection of bounding boxes representing the selected boxes.  The bounding
+// box coordinates corresponding to the selected indices can then be obtained
+// using the `tf.gather operation`.  For example:
+//   selected_indices = tf.image.non_max_suppression_v2(
+//       boxes, scores, max_output_size, iou_threshold, score_threshold)
+//   selected_boxes = tf.gather(boxes, selected_indices)
+//
+// Arguments:
+//	boxes: A 2-D float tensor of shape `[num_boxes, 4]`.
+//	scores: A 1-D float tensor of shape `[num_boxes]` representing a single
+// score corresponding to each box (each row of boxes).
+//	max_output_size: A scalar integer tensor representing the maximum number of
+// boxes to be selected by non max suppression.
+//	iou_threshold: A 0-D float tensor representing the threshold for deciding whether
+// boxes overlap too much with respect to IOU.
+//	score_threshold: A 0-D float tensor representing the threshold for deciding when to remove
+// boxes based on score.
+//
+// Returns A 1-D integer tensor of shape `[M]` representing the selected
+// indices from the boxes tensor, where `M <= max_output_size`.
+func NonMaxSuppressionV3(scope *Scope, boxes tf.Output, scores tf.Output, max_output_size tf.Output, iou_threshold tf.Output, score_threshold tf.Output) (selected_indices tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Softsign",
+		Type: "NonMaxSuppressionV3",
 		Input: []tf.Input{
-			features,
+			boxes, scores, max_output_size, iou_threshold, score_threshold,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// EncodeProtoAttr is an optional argument to EncodeProto.
-type EncodeProtoAttr func(optionalAttr)
+// NonMaxSuppressionV4Attr is an optional argument to NonMaxSuppressionV4.
+type NonMaxSuppressionV4Attr func(optionalAttr)
 
-// EncodeProtoDescriptorSource sets the optional descriptor_source attribute to value.
-// If not specified, defaults to "local://"
-func EncodeProtoDescriptorSource(value string) EncodeProtoAttr {
+// NonMaxSuppressionV4PadToMaxOutputSize sets the optional pad_to_max_output_size attribute to value.
+//
+// value: If true, the output `selected_indices` is padded to be of length
+// `max_output_size`. Defaults to false.
+// If not specified, defaults to false
+func NonMaxSuppressionV4PadToMaxOutputSize(value bool) NonMaxSuppressionV4Attr {
 	return func(m optionalAttr) {
-		m["descriptor_source"] = value
+		m["pad_to_max_output_size"] = value
 	}
 }
 
-// The op serializes protobuf messages provided in the input tensors.
-//
-// The types of the tensors in `values` must match the schema for the
-// fields specified in `field_names`. All the tensors in `values` must
-// have a common shape prefix, *batch_shape*.
-//
-// The `sizes` tensor specifies repeat counts for each field.  The repeat
-// count (last dimension) of a each tensor in `values` must be greater
-// than or equal to corresponding repeat count in `sizes`.
-//
-// A `message_type` name must be provided to give context for the field
-// names. The actual message descriptor can be looked up either in the
-// linked-in descriptor pool or a filename provided by the caller using
-// the `descriptor_source` attribute.
-//
-// The `descriptor_source` attribute selects a source of protocol
-// descriptors to consult when looking up `message_type`. This may be a
-// filename containing a serialized `FileDescriptorSet` message,
-// or the special value `local://`, in which case only descriptors linked
-// into the code will be searched; the filename can be on any filesystem
-// accessible to TensorFlow.
-//
-// You can build a `descriptor_source` file using the `--descriptor_set_out`
-// and `--include_imports` options to the protocol compiler `protoc`.
-//
-// The `local://` database only covers descriptors linked into the
-// code via C++ libraries, not Python imports. You can link in a proto descriptor
-// by creating a cc_library target with alwayslink=1.
-//
-// There are a few special cases in the value mapping:
-//
-// Submessage and group fields must be pre-serialized as TensorFlow strings.
-//
-// TensorFlow lacks support for unsigned int64s, so they must be
-// represented as `tf.int64` with the same twos-complement bit pattern
-// (the obvious way).
+// Greedily selects a subset of bounding boxes in descending order of score,
 //
-// Unsigned int32 values can be represented exactly with `tf.int64`, or
-// with sign wrapping if the input is of type `tf.int32`.
+// pruning away boxes that have high intersection-over-union (IOU) overlap
+// with previously selected boxes.  Bounding boxes with score less than
+// `score_threshold` are removed.  Bounding boxes are supplied as
+// [y1, x1, y2, x2], where (y1, x1) and (y2, x2) are the coordinates of any
+// diagonal pair of box corners and the coordinates can be provided as normalized
+// (i.e., lying in the interval [0, 1]) or absolute.  Note that this algorithm
+// is agnostic to where the origin is in the coordinate system and more
+// generally is invariant to orthogonal transformations and translations
+// of the coordinate system; thus translating or reflections of the coordinate
+// system result in the same boxes being selected by the algorithm.
+// The output of this operation is a set of integers indexing into the input
+// collection of bounding boxes representing the selected boxes.  The bounding
+// box coordinates corresponding to the selected indices can then be obtained
+// using the `tf.gather operation`.  For example:
+//   selected_indices = tf.image.non_max_suppression_v2(
+//       boxes, scores, max_output_size, iou_threshold, score_threshold)
+//   selected_boxes = tf.gather(boxes, selected_indices)
 //
 // Arguments:
-//	sizes: Tensor of int32 with shape `[batch_shape, len(field_names)]`.
-//	values: List of tensors containing values for the corresponding field.
-//	field_names: List of strings containing proto field names.
-//	message_type: Name of the proto message type to decode.
+//	boxes: A 2-D float tensor of shape `[num_boxes, 4]`.
+//	scores: A 1-D float tensor of shape `[num_boxes]` representing a single
+// score corresponding to each box (each row of boxes).
+//	max_output_size: A scalar integer tensor representing the maximum number of
+// boxes to be selected by non max suppression.
+//	iou_threshold: A 0-D float tensor representing the threshold for deciding whether
+// boxes overlap too much with respect to IOU.
+//	score_threshold: A 0-D float tensor representing the threshold for deciding when to remove
+// boxes based on score.
 //
-// Returns Tensor of serialized protos with shape `batch_shape`.
-func EncodeProto(scope *Scope, sizes tf.Output, values []tf.Output, field_names []string, message_type string, optional ...EncodeProtoAttr) (bytes tf.Output) {
+// Returns A 1-D integer tensor of shape `[M]` representing the selected
+// indices from the boxes tensor, where `M <= max_output_size`.A 0-D integer tensor representing the number of valid elements in
+// `selected_indices`, with the valid elements appearing first.
+func NonMaxSuppressionV4(scope *Scope, boxes tf.Output, scores tf.Output, max_output_size tf.Output, iou_threshold tf.Output, score_threshold tf.Output, optional ...NonMaxSuppressionV4Attr) (selected_indices tf.Output, valid_outputs tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"field_names": field_names, "message_type": message_type}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "EncodeProto",
+		Type: "NonMaxSuppressionV4",
 		Input: []tf.Input{
-			sizes, tf.OutputList(values),
+			boxes, scores, max_output_size, iou_threshold, score_threshold,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1)
 }
 
-// Creates a TensorArray for storing the gradients of values in the given handle.
-//
-// If the given TensorArray gradient already exists, returns a reference to it.
-//
-// Locks the size of the original TensorArray by disabling its dynamic size flag.
+// Computes the matrix logarithm of one or more square matrices:
 //
-// **A note about the input flow_in:**
 //
-// The handle flow_in forces the execution of the gradient lookup to occur
-// only after certain other operations have occurred.  For example, when
-// the forward TensorArray is dynamically sized, writes to this TensorArray
-// may resize the object.  The gradient TensorArray is statically sized based
-// on the size of the forward TensorArray when this operation executes.
-// Furthermore, the size of the forward TensorArray is frozen by this call.
-// As a result, the flow is used to ensure that the call to generate the gradient
-// TensorArray only happens after all writes are executed.
+// \\(log(exp(A)) = A\\)
 //
-// In the case of dynamically sized TensorArrays, gradient computation should
-// only be performed on read operations that have themselves been chained via
-// flow to occur only after all writes have executed. That way the final size
-// of the forward TensorArray is known when this operation is called.
+// This op is only defined for complex matrices. If A is positive-definite and
+// real, then casting to a complex matrix, taking the logarithm and casting back
+// to a real matrix will give the correct result.
 //
-// **A note about the source attribute:**
+// This function computes the matrix logarithm using the Schur-Parlett algorithm.
+// Details of the algorithm can be found in Section 11.6.2 of:
+// Nicholas J. Higham, Functions of Matrices: Theory and Computation, SIAM 2008.
+// ISBN 978-0-898716-46-7.
 //
-// TensorArray gradient calls use an accumulator TensorArray object.  If
-// multiple gradients are calculated and run in the same session, the multiple
-// gradient nodes may accidentally flow through the same accumulator TensorArray.
-// This double counts and generally breaks the TensorArray gradient flow.
+// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
+// form square matrices. The output is a tensor of the same shape as the input
+// containing the exponential for all input submatrices `[..., :, :]`.
 //
-// The solution is to identify which gradient call this particular
-// TensorArray gradient is being called in.  This is performed by identifying
-// a unique string (e.g. "gradients", "gradients_1", ...) from the input
-// gradient Tensor's name.  This string is used as a suffix when creating
-// the TensorArray gradient object here (the attribute `source`).
+// Arguments:
+//	input: Shape is `[..., M, M]`.
 //
-// The attribute `source` is added as a suffix to the forward TensorArray's
-// name when performing the creation / lookup, so that each separate gradient
-// calculation gets its own TensorArray accumulator.
+// Returns Shape is `[..., M, M]`.
 //
-// Arguments:
-//	handle: The handle to the forward TensorArray.
-//	flow_in: A float scalar that enforces proper chaining of operations.
-//	source: The gradient source string, used to decide which gradient TensorArray
-// to return.
-func TensorArrayGradV3(scope *Scope, handle tf.Output, flow_in tf.Output, source string) (grad_handle tf.Output, flow_out tf.Output) {
+// @compatibility(scipy)
+// Equivalent to scipy.linalg.logm
+// @end_compatibility
+func MatrixLogarithm(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"source": source}
 	opspec := tf.OpSpec{
-		Type: "TensorArrayGradV3",
+		Type: "MatrixLogarithm",
 		Input: []tf.Input{
-			handle, flow_in,
+			input,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// Creates a dataset that splits a SparseTensor into elements row-wise.
-func SparseTensorSliceDataset(scope *Scope, indices tf.Output, values tf.Output, dense_shape tf.Output) (handle tf.Output) {
+//   This op is used as a placeholder in If branch functions. It doesn't provide a
+//   valid output when run, so must either be removed (e.g. replaced with a
+//   function input) or guaranteed not to be used (e.g. if mirroring an
+//   intermediate output needed for the gradient computation of the other branch).
+//
+// Arguments:
+//	dtype: The type of the output.
+//	shape:     The purported shape of the output. This is only used for shape inference;
+//     the output will not necessarily have this shape. Can be a partial shape.
+//
+// Returns     \"Fake\" output value. This should not be consumed by another op.
+func FakeParam(scope *Scope, dtype tf.DataType, shape tf.Shape) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"dtype": dtype, "shape": shape}
 	opspec := tf.OpSpec{
-		Type: "SparseTensorSliceDataset",
-		Input: []tf.Input{
-			indices, values, dense_shape,
-		},
+		Type: "FakeParam",
+
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Returns x / y element-wise for real types.
+// Returns the next representable value of `x1` in the direction of `x2`, element-wise.
 //
-// If `x` and `y` are reals, this will return the floating-point division.
+// This operation returns the same result as the C++ std::nextafter function.
 //
-// *NOTE*: `Div` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func RealDiv(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+// It can also return a subnormal number.
+//
+// @compatibility(cpp)
+// Equivalent to C++ std::nextafter function.
+// @end_compatibility
+func NextAfter(scope *Scope, x1 tf.Output, x2 tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "RealDiv",
+		Type: "NextAfter",
 		Input: []tf.Input{
-			x, y,
+			x1, x2,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Creates a dataset that concatenates `input_dataset` with `another_dataset`.
-func ConcatenateDataset(scope *Scope, input_dataset tf.Output, another_dataset tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+// Computes the gradient for the inverse of `x` wrt its input.
+//
+// Specifically, `grad = -dy * y*y`, where `y = 1/x`, and `dy`
+// is the corresponding input gradient.
+func InvGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "ConcatenateDataset",
+		Type: "InvGrad",
 		Input: []tf.Input{
-			input_dataset, another_dataset,
+			y, dy,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes the grayscale dilation of 4-D `input` and 3-D `filter` tensors.
-//
-// The `input` tensor has shape `[batch, in_height, in_width, depth]` and the
-// `filter` tensor has shape `[filter_height, filter_width, depth]`, i.e., each
-// input channel is processed independently of the others with its own structuring
-// function. The `output` tensor has shape
-// `[batch, out_height, out_width, depth]`. The spatial dimensions of the output
-// tensor depend on the `padding` algorithm. We currently only support the default
-// "NHWC" `data_format`.
-//
-// In detail, the grayscale morphological 2-D dilation is the max-sum correlation
-// (for consistency with `conv2d`, we use unmirrored filters):
-//
-//     output[b, y, x, c] =
-//        max_{dy, dx} input[b,
-//                           strides[1] * y + rates[1] * dy,
-//                           strides[2] * x + rates[2] * dx,
-//                           c] +
-//                     filter[dy, dx, c]
-//
-// Max-pooling is a special case when the filter has size equal to the pooling
-// kernel size and contains all zeros.
-//
-// Note on duality: The dilation of `input` by the `filter` is equal to the
-// negation of the erosion of `-input` by the reflected `filter`.
-//
-// Arguments:
-//	input: 4-D with shape `[batch, in_height, in_width, depth]`.
-//	filter: 3-D with shape `[filter_height, filter_width, depth]`.
-//	strides: The stride of the sliding window for each dimension of the input
-// tensor. Must be: `[1, stride_height, stride_width, 1]`.
-//	rates: The input stride for atrous morphological dilation. Must be:
-// `[1, rate_height, rate_width, 1]`.
-//	padding: The type of padding algorithm to use.
+// List of the given size with empty elements.
 //
-// Returns 4-D with shape `[batch, out_height, out_width, depth]`.
-func Dilation2D(scope *Scope, input tf.Output, filter tf.Output, strides []int64, rates []int64, padding string) (output tf.Output) {
+// element_shape: the shape of the future elements of the list
+// num_elements: the number of elements to reserve
+// handle: the output list
+// element_dtype: the desired type of elements in the list.
+func TensorListReserve(scope *Scope, element_shape tf.Output, num_elements tf.Output, element_dtype tf.DataType) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"strides": strides, "rates": rates, "padding": padding}
+	attrs := map[string]interface{}{"element_dtype": element_dtype}
 	opspec := tf.OpSpec{
-		Type: "Dilation2D",
+		Type: "TensorListReserve",
 		Input: []tf.Input{
-			input, filter,
+			element_shape, num_elements,
 		},
 		Attrs: attrs,
 	}
@@ -32486,87 +32242,73 @@ func Dilation2D(scope *Scope, input tf.Output, filter tf.Output, strides []int64
 	return op.Output(0)
 }
 
-// Converts the given variant tensor to an iterator and stores it in the given resource.
+// A substitute for `InterleaveDataset` on a fixed list of `N` datasets.
 //
 // Arguments:
-//	resource_handle: A handle to an iterator resource.
-//	serialized: A variant tensor storing the state of the iterator contained in the
-// resource.
+//	selector_input_dataset: A dataset of scalar `DT_INT64` elements that determines which of the
+// `N` data inputs should produce the next output element.
+//	data_input_datasets: `N` datasets with the same type that will be interleaved according to
+// the values of `selector_input_dataset`.
 //
-// Returns the created operation.
-func DeserializeIterator(scope *Scope, resource_handle tf.Output, serialized tf.Output) (o *tf.Operation) {
+//
+func ExperimentalDirectedInterleaveDataset(scope *Scope, selector_input_dataset tf.Output, data_input_datasets []tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "DeserializeIterator",
+		Type: "ExperimentalDirectedInterleaveDataset",
 		Input: []tf.Input{
-			resource_handle, serialized,
+			selector_input_dataset, tf.OutputList(data_input_datasets),
 		},
+		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// ResourceScatterNdSubAttr is an optional argument to ResourceScatterNdSub.
-type ResourceScatterNdSubAttr func(optionalAttr)
+// RandomUniformIntAttr is an optional argument to RandomUniformInt.
+type RandomUniformIntAttr func(optionalAttr)
 
-// ResourceScatterNdSubUseLocking sets the optional use_locking attribute to value.
+// RandomUniformIntSeed sets the optional seed attribute to value.
 //
-// value: An optional bool. Defaults to True. If True, the assignment will
-// be protected by a lock; otherwise the behavior is undefined,
-// but may exhibit less contention.
-// If not specified, defaults to true
-func ResourceScatterNdSubUseLocking(value bool) ResourceScatterNdSubAttr {
+// value: If either `seed` or `seed2` are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func RandomUniformIntSeed(value int64) RandomUniformIntAttr {
 	return func(m optionalAttr) {
-		m["use_locking"] = value
+		m["seed"] = value
 	}
 }
 
-// Applies sparse subtraction to individual values or slices in a Variable.
-//
-// `ref` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`.
-//
-// `indices` must be integer tensor, containing indices into `ref`.
-// It must be shape `[d_0, ..., d_{Q-2}, K]` where `0 < K <= P`.
-//
-// The innermost dimension of `indices` (with length `K`) corresponds to
-// indices into elements (if `K = P`) or slices (if `K < P`) along the `K`th
-// dimension of `ref`.
-//
-// `updates` is `Tensor` of rank `Q-1+P-K` with shape:
-//
-// ```
-// [d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]]
-// ```
-//
-// For example, say we want to subtract 4 scattered elements from a rank-1 tensor
-// with 8 elements. In Python, that subtraction would look like this:
-//
-// ```python
-// ref = tf.Variable([1, 2, 3, 4, 5, 6, 7, 8], use_resource=True)
-// indices = tf.constant([[4], [3], [1], [7]])
-// updates = tf.constant([9, 10, 11, 12])
-// sub = tf.scatter_nd_sub(ref, indices, updates)
-// with tf.Session() as sess:
-//   print sess.run(sub)
-// ```
+// RandomUniformIntSeed2 sets the optional seed2 attribute to value.
 //
-// The resulting update to ref would look like this:
+// value: A second seed to avoid seed collision.
+// If not specified, defaults to 0
+func RandomUniformIntSeed2(value int64) RandomUniformIntAttr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// Outputs random integers from a uniform distribution.
 //
-//     [1, -9, 3, -6, -4, 6, 7, -4]
+// The generated values are uniform integers in the range `[minval, maxval)`.
+// The lower bound `minval` is included in the range, while the upper bound
+// `maxval` is excluded.
 //
-// See `tf.scatter_nd` for more details about how to make updates to
-// slices.
+// The random integers are slightly biased unless `maxval - minval` is an exact
+// power of two.  The bias is small for values of `maxval - minval` significantly
+// smaller than the range of the output (either `2^32` or `2^64`).
 //
 // Arguments:
-//	ref: A resource handle. Must be from a VarHandleOp.
-//	indices: A Tensor. Must be one of the following types: int32, int64.
-// A tensor of indices into ref.
-//	updates: A Tensor. Must have the same type as ref. A tensor of
-// values to add to ref.
+//	shape: The shape of the output tensor.
+//	minval: 0-D.  Inclusive lower bound on the generated integers.
+//	maxval: 0-D.  Exclusive upper bound on the generated integers.
 //
-// Returns the created operation.
-func ResourceScatterNdSub(scope *Scope, ref tf.Output, indices tf.Output, updates tf.Output, optional ...ResourceScatterNdSubAttr) (o *tf.Operation) {
+// Returns A tensor of the specified shape filled with uniform random integers.
+func RandomUniformInt(scope *Scope, shape tf.Output, minval tf.Output, maxval tf.Output, optional ...RandomUniformIntAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -32575,68 +32317,98 @@ func ResourceScatterNdSub(scope *Scope, ref tf.Output, indices tf.Output, update
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceScatterNdSub",
+		Type: "RandomUniformInt",
 		Input: []tf.Input{
-			ref, indices, updates,
+			shape, minval, maxval,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// TensorArrayConcatV2Attr is an optional argument to TensorArrayConcatV2.
-type TensorArrayConcatV2Attr func(optionalAttr)
+// Add the quantile summaries to each quantile stream resource.
+//
+// An op that adds a list of quantile summaries to a quantile stream resource. Each
+// summary Tensor is rank 2, containing summaries (value, weight, min_rank, max_rank)
+// for a single feature.
+//
+// Arguments:
+//	quantile_stream_resource_handle: resource handle referring to a QuantileStreamResource.
+//	summaries: string; List of Rank 2 Tensor each containing the summaries for a single feature.
+//
+// Returns the created operation.
+func BoostedTreesQuantileStreamResourceAddSummaries(scope *Scope, quantile_stream_resource_handle tf.Output, summaries []tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "BoostedTreesQuantileStreamResourceAddSummaries",
+		Input: []tf.Input{
+			quantile_stream_resource_handle, tf.OutputList(summaries),
+		},
+	}
+	return scope.AddOperation(opspec)
+}
 
-// TensorArrayConcatV2ElementShapeExcept0 sets the optional element_shape_except0 attribute to value.
-// If not specified, defaults to <unknown_rank:true >
-func TensorArrayConcatV2ElementShapeExcept0(value tf.Shape) TensorArrayConcatV2Attr {
-	return func(m optionalAttr) {
-		m["element_shape_except0"] = value
+// Creates a Dataset that returns pseudorandom numbers.
+//
+// Arguments:
+//	seed: A scalar seed for the random number generator. If either seed or
+// seed2 is set to be non-zero, the random number generator is seeded
+// by the given seed.  Otherwise, a random seed is used.
+//	seed2: A second scalar seed to avoid seed collision.
+//
+//
+func ExperimentalRandomDataset(scope *Scope, seed tf.Output, seed2 tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalRandomDataset",
+		Input: []tf.Input{
+			seed, seed2,
+		},
+		Attrs: attrs,
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Deprecated. Use TensorArrayConcatV3
-func TensorArrayConcatV2(scope *Scope, handle tf.Output, flow_in tf.Output, dtype tf.DataType, optional ...TensorArrayConcatV2Attr) (value tf.Output, lengths tf.Output) {
+// A dataset that splits the elements of its input into multiple elements.
+func ExperimentalUnbatchDataset(scope *Scope, input_dataset tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "TensorArrayConcatV2",
+		Type: "ExperimentalUnbatchDataset",
 		Input: []tf.Input{
-			handle, flow_in,
+			input_dataset,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// Creates a dataset that batches and pads `batch_size` elements from the input.
+// Creates a dataset that overrides the maximum intra-op parallelism.
 //
 // Arguments:
 //
-//	batch_size: A scalar representing the number of elements to accumulate in a
-// batch.
-//	padded_shapes: A list of int64 tensors representing the desired padded shapes
-// of the corresponding output components. These shapes may be partially
-// specified, using `-1` to indicate that a particular dimension should be
-// padded to the maximum size of all batch elements.
-//	padding_values: A list of scalars containing the padding value to use for
-// each of the outputs.
+//	max_intra_op_parallelism: Identifies the maximum intra-op parallelism to use.
 //
-func PaddedBatchDataset(scope *Scope, input_dataset tf.Output, batch_size tf.Output, padded_shapes []tf.Output, padding_values []tf.Output, output_shapes []tf.Shape) (handle tf.Output) {
+//
+func ExperimentalMaxIntraOpParallelismDataset(scope *Scope, input_dataset tf.Output, max_intra_op_parallelism tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"output_shapes": output_shapes}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "PaddedBatchDataset",
+		Type: "ExperimentalMaxIntraOpParallelismDataset",
 		Input: []tf.Input{
-			input_dataset, batch_size, tf.OutputList(padded_shapes), tf.OutputList(padding_values),
+			input_dataset, max_intra_op_parallelism,
 		},
 		Attrs: attrs,
 	}
@@ -32644,61 +32416,84 @@ func PaddedBatchDataset(scope *Scope, input_dataset tf.Output, batch_size tf.Out
 	return op.Output(0)
 }
 
-// Creates a dataset that shuffles and repeats elements from `input_dataset`
+// StringSplitV2Attr is an optional argument to StringSplitV2.
+type StringSplitV2Attr func(optionalAttr)
+
+// StringSplitV2Maxsplit sets the optional maxsplit attribute to value.
 //
-// pseudorandomly.
+// value: An `int`. If `maxsplit > 0`, limit of the split of the result.
+// If not specified, defaults to -1
+func StringSplitV2Maxsplit(value int64) StringSplitV2Attr {
+	return func(m optionalAttr) {
+		m["maxsplit"] = value
+	}
+}
+
+// Split elements of `source` based on `sep` into a `SparseTensor`.
 //
-// Arguments:
+// Let N be the size of source (typically N will be the batch size). Split each
+// element of `source` based on `sep` and return a `SparseTensor`
+// containing the split tokens. Empty tokens are ignored.
 //
-//	buffer_size: The number of output elements to buffer in an iterator over
-// this dataset. Compare with the `min_after_dequeue` attr when creating a
-// `RandomShuffleQueue`.
-//	seed: A scalar seed for the random number generator. If either `seed` or
-// `seed2` is set to be non-zero, the random number generator is seeded
-// by the given seed.  Otherwise, a random seed is used.
-//	seed2: A second scalar seed to avoid seed collision.
-//	count: A scalar representing the number of times the underlying dataset
-// should be repeated. The default is `-1`, which results in infinite repetition.
+// For example, N = 2, source[0] is 'hello world' and source[1] is 'a b c',
+// then the output will be
+// ```
+// st.indices = [0, 0;
+//               0, 1;
+//               1, 0;
+//               1, 1;
+//               1, 2]
+// st.shape = [2, 3]
+// st.values = ['hello', 'world', 'a', 'b', 'c']
+// ```
+//
+// If `sep` is given, consecutive delimiters are not grouped together and are
+// deemed to delimit empty strings. For example, source of `"1<>2<><>3"` and
+// sep of `"<>"` returns `["1", "2", "", "3"]`. If `sep` is None or an empty
+// string, consecutive whitespace are regarded as a single separator, and the
+// result will contain no empty strings at the startor end if the string has
+// leading or trailing whitespace.
 //
+// Note that the above mentioned behavior matches python's str.split.
 //
-func ShuffleAndRepeatDataset(scope *Scope, input_dataset tf.Output, buffer_size tf.Output, seed tf.Output, seed2 tf.Output, count tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+// Arguments:
+//	input: `1-D` string `Tensor`, the strings to split.
+//	sep: `0-D` string `Tensor`, the delimiter character.
+func StringSplitV2(scope *Scope, input tf.Output, sep tf.Output, optional ...StringSplitV2Attr) (indices tf.Output, values tf.Output, shape tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "ShuffleAndRepeatDataset",
+		Type: "StringSplitV2",
 		Input: []tf.Input{
-			input_dataset, buffer_size, seed, seed2, count,
+			input, sep,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Creates a dataset that caches elements from `input_dataset`.
-//
-// A CacheDataset will iterate over the input_dataset, and store tensors. If the
-// cache already exists, the cache will be used. If the cache is inappropriate
-// (e.g. cannot be opened, contains tensors of the wrong shape / size), an error
-// will the returned when used.
+// Creates a dataset that uses a custom thread pool to compute `input_dataset`.
 //
 // Arguments:
 //
-//	filename: A path on the filesystem where we should cache the dataset. Note: this
-// will be a directory.
+//	thread_pool: A resource produced by the ThreadPoolHandle op.
 //
 //
-func CacheDataset(scope *Scope, input_dataset tf.Output, filename tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+func ExperimentalThreadPoolDataset(scope *Scope, input_dataset tf.Output, thread_pool tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "CacheDataset",
+		Type: "ExperimentalThreadPoolDataset",
 		Input: []tf.Input{
-			input_dataset, filename,
+			input_dataset, thread_pool,
 		},
 		Attrs: attrs,
 	}
@@ -32706,248 +32501,255 @@ func CacheDataset(scope *Scope, input_dataset tf.Output, filename tf.Output, out
 	return op.Output(0)
 }
 
-// Creates a dataset that emits the records from one or more binary files.
+// Computes softsign: `features / (abs(features) + 1)`.
+func Softsign(scope *Scope, features tf.Output) (activations tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Softsign",
+		Input: []tf.Input{
+			features,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// EncodeProtoAttr is an optional argument to EncodeProto.
+type EncodeProtoAttr func(optionalAttr)
+
+// EncodeProtoDescriptorSource sets the optional descriptor_source attribute to value.
+// If not specified, defaults to "local://"
+func EncodeProtoDescriptorSource(value string) EncodeProtoAttr {
+	return func(m optionalAttr) {
+		m["descriptor_source"] = value
+	}
+}
+
+// The op serializes protobuf messages provided in the input tensors.
+//
+// The types of the tensors in `values` must match the schema for the
+// fields specified in `field_names`. All the tensors in `values` must
+// have a common shape prefix, *batch_shape*.
+//
+// The `sizes` tensor specifies repeat counts for each field.  The repeat
+// count (last dimension) of a each tensor in `values` must be greater
+// than or equal to corresponding repeat count in `sizes`.
+//
+// A `message_type` name must be provided to give context for the field
+// names. The actual message descriptor can be looked up either in the
+// linked-in descriptor pool or a filename provided by the caller using
+// the `descriptor_source` attribute.
+//
+// The `descriptor_source` attribute selects a source of protocol
+// descriptors to consult when looking up `message_type`. This may be a
+// filename containing a serialized `FileDescriptorSet` message,
+// or the special value `local://`, in which case only descriptors linked
+// into the code will be searched; the filename can be on any filesystem
+// accessible to TensorFlow.
+//
+// You can build a `descriptor_source` file using the `--descriptor_set_out`
+// and `--include_imports` options to the protocol compiler `protoc`.
+//
+// The `local://` database only covers descriptors linked into the
+// code via C++ libraries, not Python imports. You can link in a proto descriptor
+// by creating a cc_library target with alwayslink=1.
+//
+// There are a few special cases in the value mapping:
+//
+// Submessage and group fields must be pre-serialized as TensorFlow strings.
+//
+// TensorFlow lacks support for unsigned int64s, so they must be
+// represented as `tf.int64` with the same twos-complement bit pattern
+// (the obvious way).
+//
+// Unsigned int32 values can be represented exactly with `tf.int64`, or
+// with sign wrapping if the input is of type `tf.int32`.
 //
 // Arguments:
-//	filenames: A scalar or a vector containing the name(s) of the file(s) to be
-// read.
-//	header_bytes: A scalar representing the number of bytes to skip at the
-// beginning of a file.
-//	record_bytes: A scalar representing the number of bytes in each record.
-//	footer_bytes: A scalar representing the number of bytes to skip at the end
-// of a file.
-//	buffer_size: A scalar representing the number of bytes to buffer. Must be > 0.
-func FixedLengthRecordDataset(scope *Scope, filenames tf.Output, header_bytes tf.Output, record_bytes tf.Output, footer_bytes tf.Output, buffer_size tf.Output) (handle tf.Output) {
+//	sizes: Tensor of int32 with shape `[batch_shape, len(field_names)]`.
+//	values: List of tensors containing values for the corresponding field.
+//	field_names: List of strings containing proto field names.
+//	message_type: Name of the proto message type to decode.
+//
+// Returns Tensor of serialized protos with shape `batch_shape`.
+func EncodeProto(scope *Scope, sizes tf.Output, values []tf.Output, field_names []string, message_type string, optional ...EncodeProtoAttr) (bytes tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"field_names": field_names, "message_type": message_type}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "FixedLengthRecordDataset",
+		Type: "EncodeProto",
 		Input: []tf.Input{
-			filenames, header_bytes, record_bytes, footer_bytes, buffer_size,
+			sizes, tf.OutputList(values),
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Gradients for batch normalization.
+// Creates a TensorArray for storing the gradients of values in the given handle.
 //
-// DEPRECATED at GraphDef version 9: Use tf.nn.batch_normalization()
+// If the given TensorArray gradient already exists, returns a reference to it.
 //
-// This op is deprecated. See `tf.nn.batch_normalization`.
+// Locks the size of the original TensorArray by disabling its dynamic size flag.
 //
-// Arguments:
-//	t: A 4D input Tensor.
-//	m: A 1D mean Tensor with size matching the last dimension of t.
-// This is the first output from tf.nn.moments,
-// or a saved moving average thereof.
-//	v: A 1D variance Tensor with size matching the last dimension of t.
-// This is the second output from tf.nn.moments,
-// or a saved moving average thereof.
-//	gamma: A 1D gamma Tensor with size matching the last dimension of t.
-// If "scale_after_normalization" is true, this Tensor will be multiplied
-// with the normalized Tensor.
-//	backprop: 4D backprop Tensor.
-//	variance_epsilon: A small float number to avoid dividing by 0.
-//	scale_after_normalization: A bool indicating whether the resulted tensor
-// needs to be multiplied with gamma.
+// **A note about the input flow_in:**
+//
+// The handle flow_in forces the execution of the gradient lookup to occur
+// only after certain other operations have occurred.  For example, when
+// the forward TensorArray is dynamically sized, writes to this TensorArray
+// may resize the object.  The gradient TensorArray is statically sized based
+// on the size of the forward TensorArray when this operation executes.
+// Furthermore, the size of the forward TensorArray is frozen by this call.
+// As a result, the flow is used to ensure that the call to generate the gradient
+// TensorArray only happens after all writes are executed.
+//
+// In the case of dynamically sized TensorArrays, gradient computation should
+// only be performed on read operations that have themselves been chained via
+// flow to occur only after all writes have executed. That way the final size
+// of the forward TensorArray is known when this operation is called.
+//
+// **A note about the source attribute:**
+//
+// TensorArray gradient calls use an accumulator TensorArray object.  If
+// multiple gradients are calculated and run in the same session, the multiple
+// gradient nodes may accidentally flow through the same accumulator TensorArray.
+// This double counts and generally breaks the TensorArray gradient flow.
 //
-// Returns 4D backprop tensor for input.1D backprop tensor for mean.1D backprop tensor for variance.1D backprop tensor for beta.1D backprop tensor for gamma.
-func BatchNormWithGlobalNormalizationGrad(scope *Scope, t tf.Output, m tf.Output, v tf.Output, gamma tf.Output, backprop tf.Output, variance_epsilon float32, scale_after_normalization bool) (dx tf.Output, dm tf.Output, dv tf.Output, db tf.Output, dg tf.Output) {
+// The solution is to identify which gradient call this particular
+// TensorArray gradient is being called in.  This is performed by identifying
+// a unique string (e.g. "gradients", "gradients_1", ...) from the input
+// gradient Tensor's name.  This string is used as a suffix when creating
+// the TensorArray gradient object here (the attribute `source`).
+//
+// The attribute `source` is added as a suffix to the forward TensorArray's
+// name when performing the creation / lookup, so that each separate gradient
+// calculation gets its own TensorArray accumulator.
+//
+// Arguments:
+//	handle: The handle to the forward TensorArray.
+//	flow_in: A float scalar that enforces proper chaining of operations.
+//	source: The gradient source string, used to decide which gradient TensorArray
+// to return.
+func TensorArrayGradV3(scope *Scope, handle tf.Output, flow_in tf.Output, source string) (grad_handle tf.Output, flow_out tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"variance_epsilon": variance_epsilon, "scale_after_normalization": scale_after_normalization}
+	attrs := map[string]interface{}{"source": source}
 	opspec := tf.OpSpec{
-		Type: "BatchNormWithGlobalNormalizationGrad",
+		Type: "TensorArrayGradV3",
 		Input: []tf.Input{
-			t, m, v, gamma, backprop,
+			handle, flow_in,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4)
+	return op.Output(0), op.Output(1)
 }
 
-// Creates a dataset that emits the records from one or more TFRecord files.
-//
-// Arguments:
-//	filenames: A scalar or vector containing the name(s) of the file(s) to be
-// read.
-//	compression_type: A scalar containing either (i) the empty string (no
-// compression), (ii) "ZLIB", or (iii) "GZIP".
-//	buffer_size: A scalar representing the number of bytes to buffer. A value of
-// 0 means no buffering will be performed.
-func TFRecordDataset(scope *Scope, filenames tf.Output, compression_type tf.Output, buffer_size tf.Output) (handle tf.Output) {
+// Creates a dataset that splits a SparseTensor into elements row-wise.
+func SparseTensorSliceDataset(scope *Scope, indices tf.Output, values tf.Output, dense_shape tf.Output) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "TFRecordDataset",
+		Type: "SparseTensorSliceDataset",
 		Input: []tf.Input{
-			filenames, compression_type, buffer_size,
+			indices, values, dense_shape,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ExperimentalStatsAggregatorHandleAttr is an optional argument to ExperimentalStatsAggregatorHandle.
-type ExperimentalStatsAggregatorHandleAttr func(optionalAttr)
-
-// ExperimentalStatsAggregatorHandleContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func ExperimentalStatsAggregatorHandleContainer(value string) ExperimentalStatsAggregatorHandleAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// ExperimentalStatsAggregatorHandleSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func ExperimentalStatsAggregatorHandleSharedName(value string) ExperimentalStatsAggregatorHandleAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Creates a statistics manager resource.
-func ExperimentalStatsAggregatorHandle(scope *Scope, optional ...ExperimentalStatsAggregatorHandleAttr) (handle tf.Output) {
+// Returns x / y element-wise for real types.
+//
+// If `x` and `y` are reals, this will return the floating-point division.
+//
+// *NOTE*: `Div` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func RealDiv(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "ExperimentalStatsAggregatorHandle",
-
-		Attrs: attrs,
+		Type: "RealDiv",
+		Input: []tf.Input{
+			x, y,
+		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// A container for an iterator resource.
-//
-// Returns A handle to the iterator that can be passed to a "MakeIterator" or
-// "IteratorGetNext" op. In contrast to Iterator, AnonymousIterator prevents
-// resource sharing by name, and does not keep a reference to the resource
-// container.
-func AnonymousIterator(scope *Scope, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+// Creates a dataset that concatenates `input_dataset` with `another_dataset`.
+func ConcatenateDataset(scope *Scope, input_dataset tf.Output, another_dataset tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "AnonymousIterator",
-
+		Type: "ConcatenateDataset",
+		Input: []tf.Input{
+			input_dataset, another_dataset,
+		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// BatchToSpace for 4-D tensors of type T.
-//
-// This is a legacy version of the more general BatchToSpaceND.
-//
-// Rearranges (permutes) data from batch into blocks of spatial data, followed by
-// cropping. This is the reverse transformation of SpaceToBatch. More specifically,
-// this op outputs a copy of the input tensor where values from the `batch`
-// dimension are moved in spatial blocks to the `height` and `width` dimensions,
-// followed by cropping along the `height` and `width` dimensions.
-//
-// Arguments:
-//	input: 4-D tensor with shape
-// `[batch*block_size*block_size, height_pad/block_size, width_pad/block_size,
-//   depth]`. Note that the batch size of the input tensor must be divisible by
-// `block_size * block_size`.
-//	crops: 2-D tensor of non-negative integers with shape `[2, 2]`. It specifies
-// how many elements to crop from the intermediate result across the spatial
-// dimensions as follows:
-//
-//     crops = [[crop_top, crop_bottom], [crop_left, crop_right]]
-//
-//
-// Returns 4-D with shape `[batch, height, width, depth]`, where:
-//
-//       height = height_pad - crop_top - crop_bottom
-//       width = width_pad - crop_left - crop_right
-//
-// The attr `block_size` must be greater than one. It indicates the block size.
-//
-// Some examples:
-//
-// (1) For the following input of shape `[4, 1, 1, 1]` and block_size of 2:
-//
-// ```
-// [[[[1]]], [[[2]]], [[[3]]], [[[4]]]]
-// ```
-//
-// The output tensor has shape `[1, 2, 2, 1]` and value:
-//
-// ```
-// x = [[[[1], [2]], [[3], [4]]]]
-// ```
-//
-// (2) For the following input of shape `[4, 1, 1, 3]` and block_size of 2:
-//
-// ```
-// [[[1, 2, 3]], [[4, 5, 6]], [[7, 8, 9]], [[10, 11, 12]]]
-// ```
-//
-// The output tensor has shape `[1, 2, 2, 3]` and value:
-//
-// ```
-// x = [[[[1, 2, 3], [4, 5, 6]],
-//       [[7, 8, 9], [10, 11, 12]]]]
-// ```
-//
-// (3) For the following input of shape `[4, 2, 2, 1]` and block_size of 2:
+// Computes the grayscale dilation of 4-D `input` and 3-D `filter` tensors.
 //
-// ```
-// x = [[[[1], [3]], [[9], [11]]],
-//      [[[2], [4]], [[10], [12]]],
-//      [[[5], [7]], [[13], [15]]],
-//      [[[6], [8]], [[14], [16]]]]
-// ```
+// The `input` tensor has shape `[batch, in_height, in_width, depth]` and the
+// `filter` tensor has shape `[filter_height, filter_width, depth]`, i.e., each
+// input channel is processed independently of the others with its own structuring
+// function. The `output` tensor has shape
+// `[batch, out_height, out_width, depth]`. The spatial dimensions of the output
+// tensor depend on the `padding` algorithm. We currently only support the default
+// "NHWC" `data_format`.
 //
-// The output tensor has shape `[1, 4, 4, 1]` and value:
+// In detail, the grayscale morphological 2-D dilation is the max-sum correlation
+// (for consistency with `conv2d`, we use unmirrored filters):
 //
-// ```
-// x = [[[1],   [2],  [3],  [4]],
-//      [[5],   [6],  [7],  [8]],
-//      [[9],  [10], [11],  [12]],
-//      [[13], [14], [15],  [16]]]
-// ```
+//     output[b, y, x, c] =
+//        max_{dy, dx} input[b,
+//                           strides[1] * y + rates[1] * dy,
+//                           strides[2] * x + rates[2] * dx,
+//                           c] +
+//                     filter[dy, dx, c]
 //
-// (4) For the following input of shape `[8, 1, 2, 1]` and block_size of 2:
+// Max-pooling is a special case when the filter has size equal to the pooling
+// kernel size and contains all zeros.
 //
-// ```
-// x = [[[[1], [3]]], [[[9], [11]]], [[[2], [4]]], [[[10], [12]]],
-//      [[[5], [7]]], [[[13], [15]]], [[[6], [8]]], [[[14], [16]]]]
-// ```
+// Note on duality: The dilation of `input` by the `filter` is equal to the
+// negation of the erosion of `-input` by the reflected `filter`.
 //
-// The output tensor has shape `[2, 2, 4, 1]` and value:
+// Arguments:
+//	input: 4-D with shape `[batch, in_height, in_width, depth]`.
+//	filter: 3-D with shape `[filter_height, filter_width, depth]`.
+//	strides: The stride of the sliding window for each dimension of the input
+// tensor. Must be: `[1, stride_height, stride_width, 1]`.
+//	rates: The input stride for atrous morphological dilation. Must be:
+// `[1, rate_height, rate_width, 1]`.
+//	padding: The type of padding algorithm to use.
 //
-// ```
-// x = [[[[1], [3]], [[5], [7]]],
-//      [[[2], [4]], [[10], [12]]],
-//      [[[5], [7]], [[13], [15]]],
-//      [[[6], [8]], [[14], [16]]]]
-// ```
-func BatchToSpace(scope *Scope, input tf.Output, crops tf.Output, block_size int64) (output tf.Output) {
+// Returns 4-D with shape `[batch, out_height, out_width, depth]`.
+func Dilation2D(scope *Scope, input tf.Output, filter tf.Output, strides []int64, rates []int64, padding string) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"block_size": block_size}
+	attrs := map[string]interface{}{"strides": strides, "rates": rates, "padding": padding}
 	opspec := tf.OpSpec{
-		Type: "BatchToSpace",
+		Type: "Dilation2D",
 		Input: []tf.Input{
-			input, crops,
+			input, filter,
 		},
 		Attrs: attrs,
 	}
@@ -32955,197 +32757,219 @@ func BatchToSpace(scope *Scope, input tf.Output, crops tf.Output, block_size int
 	return op.Output(0)
 }
 
-// Produces a summary of any statistics recorded by the given statistics manager.
-func ExperimentalStatsAggregatorSummary(scope *Scope, iterator tf.Output) (summary tf.Output) {
+// Converts the given variant tensor to an iterator and stores it in the given resource.
+//
+// Arguments:
+//	resource_handle: A handle to an iterator resource.
+//	serialized: A variant tensor storing the state of the iterator contained in the
+// resource.
+//
+// Returns the created operation.
+func DeserializeIterator(scope *Scope, resource_handle tf.Output, serialized tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "ExperimentalStatsAggregatorSummary",
+		Type: "DeserializeIterator",
 		Input: []tf.Input{
-			iterator,
+			resource_handle, serialized,
 		},
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// Makes a new iterator from the given `dataset` and stores it in `iterator`.
-//
-// This operation may be executed multiple times. Each execution will reset the
-// iterator in `iterator` to the first element of `dataset`.
+// ResourceScatterNdSubAttr is an optional argument to ResourceScatterNdSub.
+type ResourceScatterNdSubAttr func(optionalAttr)
+
+// ResourceScatterNdSubUseLocking sets the optional use_locking attribute to value.
 //
-// Returns the created operation.
-func MakeIterator(scope *Scope, dataset tf.Output, iterator tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "MakeIterator",
-		Input: []tf.Input{
-			dataset, iterator,
-		},
+// value: An optional bool. Defaults to True. If True, the assignment will
+// be protected by a lock; otherwise the behavior is undefined,
+// but may exhibit less contention.
+// If not specified, defaults to true
+func ResourceScatterNdSubUseLocking(value bool) ResourceScatterNdSubAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
 	}
-	return scope.AddOperation(opspec)
 }
 
-// Adjust the contrast of one or more images.
+// Applies sparse subtraction to individual values or slices in a Variable.
 //
-// `images` is a tensor of at least 3 dimensions.  The last 3 dimensions are
-// interpreted as `[height, width, channels]`.  The other dimensions only
-// represent a collection of images, such as `[batch, height, width, channels].`
+// `ref` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`.
 //
-// Contrast is adjusted independently for each channel of each image.
+// `indices` must be integer tensor, containing indices into `ref`.
+// It must be shape `[d_0, ..., d_{Q-2}, K]` where `0 < K <= P`.
 //
-// For each channel, the Op first computes the mean of the image pixels in the
-// channel and then adjusts each component of each pixel to
-// `(x - mean) * contrast_factor + mean`.
+// The innermost dimension of `indices` (with length `K`) corresponds to
+// indices into elements (if `K = P`) or slices (if `K < P`) along the `K`th
+// dimension of `ref`.
+//
+// `updates` is `Tensor` of rank `Q-1+P-K` with shape:
+//
+// ```
+// [d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]]
+// ```
+//
+// For example, say we want to subtract 4 scattered elements from a rank-1 tensor
+// with 8 elements. In Python, that subtraction would look like this:
+//
+// ```python
+// ref = tf.Variable([1, 2, 3, 4, 5, 6, 7, 8], use_resource=True)
+// indices = tf.constant([[4], [3], [1], [7]])
+// updates = tf.constant([9, 10, 11, 12])
+// sub = tf.scatter_nd_sub(ref, indices, updates)
+// with tf.Session() as sess:
+//   print sess.run(sub)
+// ```
+//
+// The resulting update to ref would look like this:
+//
+//     [1, -9, 3, -6, -4, 6, 7, -4]
+//
+// See `tf.scatter_nd` for more details about how to make updates to
+// slices.
 //
 // Arguments:
-//	images: Images to adjust.  At least 3-D.
-//	contrast_factor: A float multiplier for adjusting contrast.
+//	ref: A resource handle. Must be from a VarHandleOp.
+//	indices: A Tensor. Must be one of the following types: int32, int64.
+// A tensor of indices into ref.
+//	updates: A Tensor. Must have the same type as ref. A tensor of
+// values to add to ref.
 //
-// Returns The contrast-adjusted image or images.
-func AdjustContrastv2(scope *Scope, images tf.Output, contrast_factor tf.Output) (output tf.Output) {
+// Returns the created operation.
+func ResourceScatterNdSub(scope *Scope, ref tf.Output, indices tf.Output, updates tf.Output, optional ...ResourceScatterNdSubAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "AdjustContrastv2",
+		Type: "ResourceScatterNdSub",
 		Input: []tf.Input{
-			images, contrast_factor,
+			ref, indices, updates,
 		},
+		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// Gets the next output from the given iterator .
-func IteratorGetNext(scope *Scope, iterator tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (components []tf.Output) {
+// TensorArrayConcatV2Attr is an optional argument to TensorArrayConcatV2.
+type TensorArrayConcatV2Attr func(optionalAttr)
+
+// TensorArrayConcatV2ElementShapeExcept0 sets the optional element_shape_except0 attribute to value.
+// If not specified, defaults to <unknown_rank:true >
+func TensorArrayConcatV2ElementShapeExcept0(value tf.Shape) TensorArrayConcatV2Attr {
+	return func(m optionalAttr) {
+		m["element_shape_except0"] = value
+	}
+}
+
+// Deprecated. Use TensorArrayConcatV3
+func TensorArrayConcatV2(scope *Scope, handle tf.Output, flow_in tf.Output, dtype tf.DataType, optional ...TensorArrayConcatV2Attr) (value tf.Output, lengths tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	attrs := map[string]interface{}{"dtype": dtype}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "IteratorGetNext",
+		Type: "TensorArrayConcatV2",
 		Input: []tf.Input{
-			iterator,
+			handle, flow_in,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if components, idx, err = makeOutputList(op, idx, "components"); err != nil {
-		scope.UpdateErr("IteratorGetNext", err)
-		return
-	}
-	return components
+	return op.Output(0), op.Output(1)
 }
 
-// Outputs the single element from the given dataset.
+// Creates a dataset that batches and pads `batch_size` elements from the input.
 //
 // Arguments:
-//	dataset: A handle to a dataset that contains a single element.
-//
 //
+//	batch_size: A scalar representing the number of elements to accumulate in a
+// batch.
+//	padded_shapes: A list of int64 tensors representing the desired padded shapes
+// of the corresponding output components. These shapes may be partially
+// specified, using `-1` to indicate that a particular dimension should be
+// padded to the maximum size of all batch elements.
+//	padding_values: A list of scalars containing the padding value to use for
+// each of the outputs.
 //
-// Returns The components of the single element of `input`.
-func DatasetToSingleElement(scope *Scope, dataset tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (components []tf.Output) {
+func PaddedBatchDataset(scope *Scope, input_dataset tf.Output, batch_size tf.Output, padded_shapes []tf.Output, padding_values []tf.Output, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	attrs := map[string]interface{}{"output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "DatasetToSingleElement",
+		Type: "PaddedBatchDataset",
 		Input: []tf.Input{
-			dataset,
+			input_dataset, batch_size, tf.OutputList(padded_shapes), tf.OutputList(padding_values),
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if components, idx, err = makeOutputList(op, idx, "components"); err != nil {
-		scope.UpdateErr("DatasetToSingleElement", err)
-		return
-	}
-	return components
+	return op.Output(0)
 }
 
-// Converts the given `resource_handle` representing an iterator to a string.
+// Creates a dataset that shuffles and repeats elements from `input_dataset`
+//
+// pseudorandomly.
 //
 // Arguments:
-//	resource_handle: A handle to an iterator resource.
 //
-// Returns A string representation of the given handle.
-func IteratorToStringHandle(scope *Scope, resource_handle tf.Output) (string_handle tf.Output) {
+//	buffer_size: The number of output elements to buffer in an iterator over
+// this dataset. Compare with the `min_after_dequeue` attr when creating a
+// `RandomShuffleQueue`.
+//	seed: A scalar seed for the random number generator. If either `seed` or
+// `seed2` is set to be non-zero, the random number generator is seeded
+// by the given seed.  Otherwise, a random seed is used.
+//	seed2: A second scalar seed to avoid seed collision.
+//	count: A scalar representing the number of times the underlying dataset
+// should be repeated. The default is `-1`, which results in infinite repetition.
+//
+//
+func ShuffleAndRepeatDataset(scope *Scope, input_dataset tf.Output, buffer_size tf.Output, seed tf.Output, seed2 tf.Output, count tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "IteratorToStringHandle",
+		Type: "ShuffleAndRepeatDataset",
 		Input: []tf.Input{
-			resource_handle,
+			input_dataset, buffer_size, seed, seed2, count,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// IteratorFromStringHandleAttr is an optional argument to IteratorFromStringHandle.
-type IteratorFromStringHandleAttr func(optionalAttr)
-
-// IteratorFromStringHandleOutputTypes sets the optional output_types attribute to value.
-//
-// value: If specified, defines the type of each tuple component in an
-// element produced by the resulting iterator.
-// If not specified, defaults to <>
+// Creates a dataset that caches elements from `input_dataset`.
 //
-// REQUIRES: len(value) >= 0
-func IteratorFromStringHandleOutputTypes(value []tf.DataType) IteratorFromStringHandleAttr {
-	return func(m optionalAttr) {
-		m["output_types"] = value
-	}
-}
-
-// IteratorFromStringHandleOutputShapes sets the optional output_shapes attribute to value.
+// A CacheDataset will iterate over the input_dataset, and store tensors. If the
+// cache already exists, the cache will be used. If the cache is inappropriate
+// (e.g. cannot be opened, contains tensors of the wrong shape / size), an error
+// will the returned when used.
 //
-// value: If specified, defines the shape of each tuple component in an
-// element produced by the resulting iterator.
-// If not specified, defaults to <>
+// Arguments:
 //
-// REQUIRES: len(value) >= 0
-func IteratorFromStringHandleOutputShapes(value []tf.Shape) IteratorFromStringHandleAttr {
-	return func(m optionalAttr) {
-		m["output_shapes"] = value
-	}
-}
-
-// Converts the given string representing a handle to an iterator to a resource.
+//	filename: A path on the filesystem where we should cache the dataset. Note: this
+// will be a directory.
 //
-// Arguments:
-//	string_handle: A string representation of the given handle.
 //
-// Returns A handle to an iterator resource.
-func IteratorFromStringHandle(scope *Scope, string_handle tf.Output, optional ...IteratorFromStringHandleAttr) (resource_handle tf.Output) {
+func CacheDataset(scope *Scope, input_dataset tf.Output, filename tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "IteratorFromStringHandle",
+		Type: "CacheDataset",
 		Input: []tf.Input{
-			string_handle,
+			input_dataset, filename,
 		},
 		Attrs: attrs,
 	}
@@ -33153,147 +32977,123 @@ func IteratorFromStringHandle(scope *Scope, string_handle tf.Output, optional ..
 	return op.Output(0)
 }
 
-// Gather slices from `params` axis `axis` according to `indices`.
-//
-// `indices` must be an integer tensor of any dimension (usually 0-D or 1-D).
-// Produces an output tensor with shape `params.shape[:axis] + indices.shape +
-// params.shape[axis + 1:]` where:
-//
-// ```python
-//     # Scalar indices (output is rank(params) - 1).
-//     output[a_0, ..., a_n, b_0, ..., b_n] =
-//       params[a_0, ..., a_n, indices, b_0, ..., b_n]
-//
-//     # Vector indices (output is rank(params)).
-//     output[a_0, ..., a_n, i, b_0, ..., b_n] =
-//       params[a_0, ..., a_n, indices[i], b_0, ..., b_n]
-//
-//     # Higher rank indices (output is rank(params) + rank(indices) - 1).
-//     output[a_0, ..., a_n, i, ..., j, b_0, ... b_n] =
-//       params[a_0, ..., a_n, indices[i, ..., j], b_0, ..., b_n]
-// ```
-//
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/Gather.png" alt>
-// </div>
-//
-// Note that on CPU, if an out of bound index is found, an error is returned.
-// On GPU, if an out of bound index is found, a 0 is stored in the
-// corresponding output value.
-//
-// See also `tf.batch_gather` and `tf.gather_nd`.
+// Creates a dataset that emits the records from one or more binary files.
 //
 // Arguments:
-//	params: The tensor from which to gather values. Must be at least rank
-// `axis + 1`.
-//	indices: Index tensor. Must be in range `[0, params.shape[axis])`.
-//	axis: The axis in `params` to gather `indices` from. Defaults to the first
-// dimension. Supports negative indexes.
-//
-// Returns Values from `params` gathered from indices given by `indices`, with
-// shape `params.shape[:axis] + indices.shape + params.shape[axis + 1:]`.
-func GatherV2(scope *Scope, params tf.Output, indices tf.Output, axis tf.Output) (output tf.Output) {
+//	filenames: A scalar or a vector containing the name(s) of the file(s) to be
+// read.
+//	header_bytes: A scalar representing the number of bytes to skip at the
+// beginning of a file.
+//	record_bytes: A scalar representing the number of bytes in each record.
+//	footer_bytes: A scalar representing the number of bytes to skip at the end
+// of a file.
+//	buffer_size: A scalar representing the number of bytes to buffer. Must be > 0.
+func FixedLengthRecordDataset(scope *Scope, filenames tf.Output, header_bytes tf.Output, record_bytes tf.Output, footer_bytes tf.Output, buffer_size tf.Output) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "GatherV2",
+		Type: "FixedLengthRecordDataset",
 		Input: []tf.Input{
-			params, indices, axis,
+			filenames, header_bytes, record_bytes, footer_bytes, buffer_size,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Converts the given `resource_handle` representing an iterator to a variant tensor.
+// Gradients for batch normalization.
+//
+// DEPRECATED at GraphDef version 9: Use tf.nn.batch_normalization()
+//
+// This op is deprecated. See `tf.nn.batch_normalization`.
 //
 // Arguments:
-//	resource_handle: A handle to an iterator resource.
+//	t: A 4D input Tensor.
+//	m: A 1D mean Tensor with size matching the last dimension of t.
+// This is the first output from tf.nn.moments,
+// or a saved moving average thereof.
+//	v: A 1D variance Tensor with size matching the last dimension of t.
+// This is the second output from tf.nn.moments,
+// or a saved moving average thereof.
+//	gamma: A 1D gamma Tensor with size matching the last dimension of t.
+// If "scale_after_normalization" is true, this Tensor will be multiplied
+// with the normalized Tensor.
+//	backprop: 4D backprop Tensor.
+//	variance_epsilon: A small float number to avoid dividing by 0.
+//	scale_after_normalization: A bool indicating whether the resulted tensor
+// needs to be multiplied with gamma.
 //
-// Returns A variant tensor storing the state of the iterator contained in the
-// resource.
-func SerializeIterator(scope *Scope, resource_handle tf.Output) (serialized tf.Output) {
+// Returns 4D backprop tensor for input.1D backprop tensor for mean.1D backprop tensor for variance.1D backprop tensor for beta.1D backprop tensor for gamma.
+func BatchNormWithGlobalNormalizationGrad(scope *Scope, t tf.Output, m tf.Output, v tf.Output, gamma tf.Output, backprop tf.Output, variance_epsilon float32, scale_after_normalization bool) (dx tf.Output, dm tf.Output, dv tf.Output, db tf.Output, dg tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"variance_epsilon": variance_epsilon, "scale_after_normalization": scale_after_normalization}
 	opspec := tf.OpSpec{
-		Type: "SerializeIterator",
+		Type: "BatchNormWithGlobalNormalizationGrad",
 		Input: []tf.Input{
-			resource_handle,
+			t, m, v, gamma, backprop,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4)
 }
 
-// FIFOQueueV2Attr is an optional argument to FIFOQueueV2.
-type FIFOQueueV2Attr func(optionalAttr)
-
-// FIFOQueueV2Shapes sets the optional shapes attribute to value.
-//
-// value: The shape of each component in a value. The length of this attr must
-// be either 0 or the same as the length of component_types. If the length of
-// this attr is 0, the shapes of queue elements are not constrained, and
-// only one element may be dequeued at a time.
-// If not specified, defaults to <>
+// Creates a dataset that emits the records from one or more TFRecord files.
 //
-// REQUIRES: len(value) >= 0
-func FIFOQueueV2Shapes(value []tf.Shape) FIFOQueueV2Attr {
-	return func(m optionalAttr) {
-		m["shapes"] = value
+// Arguments:
+//	filenames: A scalar or vector containing the name(s) of the file(s) to be
+// read.
+//	compression_type: A scalar containing either (i) the empty string (no
+// compression), (ii) "ZLIB", or (iii) "GZIP".
+//	buffer_size: A scalar representing the number of bytes to buffer. A value of
+// 0 means no buffering will be performed.
+func TFRecordDataset(scope *Scope, filenames tf.Output, compression_type tf.Output, buffer_size tf.Output) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// FIFOQueueV2Capacity sets the optional capacity attribute to value.
-//
-// value: The upper bound on the number of elements in this queue.
-// Negative numbers mean no limit.
-// If not specified, defaults to -1
-func FIFOQueueV2Capacity(value int64) FIFOQueueV2Attr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
+	opspec := tf.OpSpec{
+		Type: "TFRecordDataset",
+		Input: []tf.Input{
+			filenames, compression_type, buffer_size,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// FIFOQueueV2Container sets the optional container attribute to value.
-//
-// value: If non-empty, this queue is placed in the given container.
-// Otherwise, a default container is used.
+// ExperimentalStatsAggregatorHandleAttr is an optional argument to ExperimentalStatsAggregatorHandle.
+type ExperimentalStatsAggregatorHandleAttr func(optionalAttr)
+
+// ExperimentalStatsAggregatorHandleContainer sets the optional container attribute to value.
 // If not specified, defaults to ""
-func FIFOQueueV2Container(value string) FIFOQueueV2Attr {
+func ExperimentalStatsAggregatorHandleContainer(value string) ExperimentalStatsAggregatorHandleAttr {
 	return func(m optionalAttr) {
 		m["container"] = value
 	}
 }
 
-// FIFOQueueV2SharedName sets the optional shared_name attribute to value.
-//
-// value: If non-empty, this queue will be shared under the given name
-// across multiple sessions.
+// ExperimentalStatsAggregatorHandleSharedName sets the optional shared_name attribute to value.
 // If not specified, defaults to ""
-func FIFOQueueV2SharedName(value string) FIFOQueueV2Attr {
+func ExperimentalStatsAggregatorHandleSharedName(value string) ExperimentalStatsAggregatorHandleAttr {
 	return func(m optionalAttr) {
 		m["shared_name"] = value
 	}
 }
 
-// A queue that produces elements in first-in first-out order.
-//
-// Arguments:
-//	component_types: The type of each component in a value.
-//
-// Returns The handle to the queue.
-func FIFOQueueV2(scope *Scope, component_types []tf.DataType, optional ...FIFOQueueV2Attr) (handle tf.Output) {
+// Creates a statistics manager resource.
+func ExperimentalStatsAggregatorHandle(scope *Scope, optional ...ExperimentalStatsAggregatorHandleAttr) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"component_types": component_types}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "FIFOQueueV2",
+		Type: "ExperimentalStatsAggregatorHandle",
 
 		Attrs: attrs,
 	}
@@ -33301,204 +33101,206 @@ func FIFOQueueV2(scope *Scope, component_types []tf.DataType, optional ...FIFOQu
 	return op.Output(0)
 }
 
-// Deserializes a proto into the tree handle
-//
-// Arguments:
-//	tree_handle: Handle to the tree resource to be restored.
-//	tree_config: Serialied proto string of the boosted_trees.Tree proto.
+// A container for an iterator resource.
 //
-// Returns the created operation.
-func TensorForestTreeDeserialize(scope *Scope, tree_handle tf.Output, tree_config tf.Output) (o *tf.Operation) {
+// Returns A handle to the iterator that can be passed to a "MakeIterator" or
+// "IteratorGetNext" op. In contrast to Iterator, AnonymousIterator prevents
+// resource sharing by name, and does not keep a reference to the resource
+// container.
+func AnonymousIterator(scope *Scope, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "TensorForestTreeDeserialize",
-		Input: []tf.Input{
-			tree_handle, tree_config,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
+		Type: "AnonymousIterator",
 
-// Constructs an Optional variant from a tuple of tensors.
-func OptionalFromValue(scope *Scope, components []tf.Output) (optional tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "OptionalFromValue",
-		Input: []tf.Input{
-			tf.OutputList(components),
-		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// DecodeProtoV2Attr is an optional argument to DecodeProtoV2.
-type DecodeProtoV2Attr func(optionalAttr)
-
-// DecodeProtoV2DescriptorSource sets the optional descriptor_source attribute to value.
+// BatchToSpace for 4-D tensors of type T.
 //
-// value: Either the special value `local://` or a path to a file containing
-// a serialized `FileDescriptorSet`.
-// If not specified, defaults to "local://"
-func DecodeProtoV2DescriptorSource(value string) DecodeProtoV2Attr {
-	return func(m optionalAttr) {
-		m["descriptor_source"] = value
-	}
-}
-
-// DecodeProtoV2MessageFormat sets the optional message_format attribute to value.
+// This is a legacy version of the more general BatchToSpaceND.
 //
-// value: Either `binary` or `text`.
-// If not specified, defaults to "binary"
-func DecodeProtoV2MessageFormat(value string) DecodeProtoV2Attr {
-	return func(m optionalAttr) {
-		m["message_format"] = value
-	}
-}
-
-// DecodeProtoV2Sanitize sets the optional sanitize attribute to value.
+// Rearranges (permutes) data from batch into blocks of spatial data, followed by
+// cropping. This is the reverse transformation of SpaceToBatch. More specifically,
+// this op outputs a copy of the input tensor where values from the `batch`
+// dimension are moved in spatial blocks to the `height` and `width` dimensions,
+// followed by cropping along the `height` and `width` dimensions.
 //
-// value: Whether to sanitize the result or not.
-// If not specified, defaults to false
-func DecodeProtoV2Sanitize(value bool) DecodeProtoV2Attr {
-	return func(m optionalAttr) {
-		m["sanitize"] = value
-	}
-}
-
-// The op extracts fields from a serialized protocol buffers message into tensors.
+// Arguments:
+//	input: 4-D tensor with shape
+// `[batch*block_size*block_size, height_pad/block_size, width_pad/block_size,
+//   depth]`. Note that the batch size of the input tensor must be divisible by
+// `block_size * block_size`.
+//	crops: 2-D tensor of non-negative integers with shape `[2, 2]`. It specifies
+// how many elements to crop from the intermediate result across the spatial
+// dimensions as follows:
 //
-// The `decode_proto` op extracts fields from a serialized protocol buffers
-// message into tensors.  The fields in `field_names` are decoded and converted
-// to the corresponding `output_types` if possible.
+//     crops = [[crop_top, crop_bottom], [crop_left, crop_right]]
 //
-// A `message_type` name must be provided to give context for the field
-// names. The actual message descriptor can be looked up either in the
-// linked-in descriptor pool or a filename provided by the caller using
-// the `descriptor_source` attribute.
 //
-// Each output tensor is a dense tensor. This means that it is padded to
-// hold the largest number of repeated elements seen in the input
-// minibatch. (The shape is also padded by one to prevent zero-sized
-// dimensions). The actual repeat counts for each example in the
-// minibatch can be found in the `sizes` output. In many cases the output
-// of `decode_proto` is fed immediately into tf.squeeze if missing values
-// are not a concern. When using tf.squeeze, always pass the squeeze
-// dimension explicitly to avoid surprises.
+// Returns 4-D with shape `[batch, height, width, depth]`, where:
 //
-// For the most part, the mapping between Proto field types and
-// TensorFlow dtypes is straightforward. However, there are a few
-// special cases:
+//       height = height_pad - crop_top - crop_bottom
+//       width = width_pad - crop_left - crop_right
 //
-// - A proto field that contains a submessage or group can only be converted
-// to `DT_STRING` (the serialized submessage). This is to reduce the
-// complexity of the API. The resulting string can be used as input
-// to another instance of the decode_proto op.
+// The attr `block_size` must be greater than one. It indicates the block size.
 //
-// - TensorFlow lacks support for unsigned integers. The ops represent uint64
-// types as a `DT_INT64` with the same twos-complement bit pattern
-// (the obvious way). Unsigned int32 values can be represented exactly by
-// specifying type `DT_INT64`, or using twos-complement if the caller
-// specifies `DT_INT32` in the `output_types` attribute.
+// Some examples:
+//
+// (1) For the following input of shape `[4, 1, 1, 1]` and block_size of 2:
+//
+// ```
+// [[[[1]]], [[[2]]], [[[3]]], [[[4]]]]
+// ```
+//
+// The output tensor has shape `[1, 2, 2, 1]` and value:
+//
+// ```
+// x = [[[[1], [2]], [[3], [4]]]]
+// ```
+//
+// (2) For the following input of shape `[4, 1, 1, 3]` and block_size of 2:
+//
+// ```
+// [[[1, 2, 3]], [[4, 5, 6]], [[7, 8, 9]], [[10, 11, 12]]]
+// ```
+//
+// The output tensor has shape `[1, 2, 2, 3]` and value:
+//
+// ```
+// x = [[[[1, 2, 3], [4, 5, 6]],
+//       [[7, 8, 9], [10, 11, 12]]]]
+// ```
+//
+// (3) For the following input of shape `[4, 2, 2, 1]` and block_size of 2:
+//
+// ```
+// x = [[[[1], [3]], [[9], [11]]],
+//      [[[2], [4]], [[10], [12]]],
+//      [[[5], [7]], [[13], [15]]],
+//      [[[6], [8]], [[14], [16]]]]
+// ```
 //
-// The `descriptor_source` attribute selects a source of protocol
-// descriptors to consult when looking up `message_type`. This may be a
-// filename containing a serialized `FileDescriptorSet` message,
-// or the special value `local://`, in which case only descriptors linked
-// into the code will be searched; the filename can be on any filesystem
-// accessible to TensorFlow.
+// The output tensor has shape `[1, 4, 4, 1]` and value:
 //
-// You can build a `descriptor_source` file using the `--descriptor_set_out`
-// and `--include_imports` options to the protocol compiler `protoc`.
+// ```
+// x = [[[1],   [2],  [3],  [4]],
+//      [[5],   [6],  [7],  [8]],
+//      [[9],  [10], [11],  [12]],
+//      [[13], [14], [15],  [16]]]
+// ```
 //
-// The `local://` database only covers descriptors linked into the
-// code via C++ libraries, not Python imports. You can link in a proto descriptor
-// by creating a cc_library target with alwayslink=1.
+// (4) For the following input of shape `[8, 1, 2, 1]` and block_size of 2:
 //
-// Both binary and text proto serializations are supported, and can be
-// chosen using the `format` attribute.
+// ```
+// x = [[[[1], [3]]], [[[9], [11]]], [[[2], [4]]], [[[10], [12]]],
+//      [[[5], [7]]], [[[13], [15]]], [[[6], [8]]], [[[14], [16]]]]
+// ```
 //
-// Arguments:
-//	bytes: Tensor of serialized protos with shape `batch_shape`.
-//	message_type: Name of the proto message type to decode.
-//	field_names: List of strings containing proto field names.
-//	output_types: List of TF types to use for the respective field in field_names.
+// The output tensor has shape `[2, 2, 4, 1]` and value:
 //
-// Returns Tensor of int32 with shape `[batch_shape, len(field_names)]`.
-// Each entry is the number of values found for the corresponding field.
-// Optional fields may have 0 or 1 values.List of tensors containing values for the corresponding field.
-// `values[i]` has datatype `output_types[i]`
-// and shape `[batch_shape, max(sizes[...,i])]`.
-func DecodeProtoV2(scope *Scope, bytes tf.Output, message_type string, field_names []string, output_types []tf.DataType, optional ...DecodeProtoV2Attr) (sizes tf.Output, values []tf.Output) {
+// ```
+// x = [[[[1], [3]], [[5], [7]]],
+//      [[[2], [4]], [[10], [12]]],
+//      [[[5], [7]], [[13], [15]]],
+//      [[[6], [8]], [[14], [16]]]]
+// ```
+func BatchToSpace(scope *Scope, input tf.Output, crops tf.Output, block_size int64) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"message_type": message_type, "field_names": field_names, "output_types": output_types}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"block_size": block_size}
 	opspec := tf.OpSpec{
-		Type: "DecodeProtoV2",
+		Type: "BatchToSpace",
 		Input: []tf.Input{
-			bytes,
+			input, crops,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Produces a summary of any statistics recorded by the given statistics manager.
+func ExperimentalStatsAggregatorSummary(scope *Scope, iterator tf.Output) (summary tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	var idx int
-	var err error
-	sizes = op.Output(idx)
-	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
-		scope.UpdateErr("DecodeProtoV2", err)
-		return
+	opspec := tf.OpSpec{
+		Type: "ExperimentalStatsAggregatorSummary",
+		Input: []tf.Input{
+			iterator,
+		},
 	}
-	return sizes, values
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Creates an Optional variant with no value.
-func OptionalNone(scope *Scope) (optional tf.Output) {
+// Makes a new iterator from the given `dataset` and stores it in `iterator`.
+//
+// This operation may be executed multiple times. Each execution will reset the
+// iterator in `iterator` to the first element of `dataset`.
+//
+// Returns the created operation.
+func MakeIterator(scope *Scope, dataset tf.Output, iterator tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "OptionalNone",
+		Type: "MakeIterator",
+		Input: []tf.Input{
+			dataset, iterator,
+		},
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// Returns true if and only if the given Optional variant has a value.
-func OptionalHasValue(scope *Scope, optional tf.Output) (has_value tf.Output) {
+// Adjust the contrast of one or more images.
+//
+// `images` is a tensor of at least 3 dimensions.  The last 3 dimensions are
+// interpreted as `[height, width, channels]`.  The other dimensions only
+// represent a collection of images, such as `[batch, height, width, channels].`
+//
+// Contrast is adjusted independently for each channel of each image.
+//
+// For each channel, the Op first computes the mean of the image pixels in the
+// channel and then adjusts each component of each pixel to
+// `(x - mean) * contrast_factor + mean`.
+//
+// Arguments:
+//	images: Images to adjust.  At least 3-D.
+//	contrast_factor: A float multiplier for adjusting contrast.
+//
+// Returns The contrast-adjusted image or images.
+func AdjustContrastv2(scope *Scope, images tf.Output, contrast_factor tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "OptionalHasValue",
+		Type: "AdjustContrastv2",
 		Input: []tf.Input{
-			optional,
+			images, contrast_factor,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Returns the value stored in an Optional variant or raises an error if none exists.
-func OptionalGetValue(scope *Scope, optional tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (components []tf.Output) {
+// Gets the next output from the given iterator .
+func IteratorGetNext(scope *Scope, iterator tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (components []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "OptionalGetValue",
+		Type: "IteratorGetNext",
 		Input: []tf.Input{
-			optional,
+			iterator,
 		},
 		Attrs: attrs,
 	}
@@ -33509,74 +33311,112 @@ func OptionalGetValue(scope *Scope, optional tf.Output, output_types []tf.DataTy
 	var idx int
 	var err error
 	if components, idx, err = makeOutputList(op, idx, "components"); err != nil {
-		scope.UpdateErr("OptionalGetValue", err)
+		scope.UpdateErr("IteratorGetNext", err)
 		return
 	}
 	return components
 }
 
-// Gets the next output from the given iterator as an Optional variant.
-func IteratorGetNextAsOptional(scope *Scope, iterator tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (optional tf.Output) {
+// Outputs the single element from the given dataset.
+//
+// Arguments:
+//	dataset: A handle to a dataset that contains a single element.
+//
+//
+//
+// Returns The components of the single element of `input`.
+func DatasetToSingleElement(scope *Scope, dataset tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (components []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "IteratorGetNextAsOptional",
+		Type: "DatasetToSingleElement",
 		Input: []tf.Input{
-			iterator,
+			dataset,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if components, idx, err = makeOutputList(op, idx, "components"); err != nil {
+		scope.UpdateErr("DatasetToSingleElement", err)
+		return
+	}
+	return components
 }
 
-// Fast Fourier transform.
-//
-// Computes the 1-dimensional discrete Fourier transform over the inner-most
-// dimension of `input`.
+// Converts the given `resource_handle` representing an iterator to a string.
 //
 // Arguments:
-//	input: A complex tensor.
-//
-// Returns A complex tensor of the same shape as `input`. The inner-most
-//   dimension of `input` is replaced with its 1D Fourier transform.
+//	resource_handle: A handle to an iterator resource.
 //
-// @compatibility(numpy)
-// Equivalent to np.fft.fft
-// @end_compatibility
-func FFT(scope *Scope, input tf.Output) (output tf.Output) {
+// Returns A string representation of the given handle.
+func IteratorToStringHandle(scope *Scope, resource_handle tf.Output) (string_handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "FFT",
+		Type: "IteratorToStringHandle",
 		Input: []tf.Input{
-			input,
+			resource_handle,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Identity transformation that models performance.
+// IteratorFromStringHandleAttr is an optional argument to IteratorFromStringHandle.
+type IteratorFromStringHandleAttr func(optionalAttr)
+
+// IteratorFromStringHandleOutputTypes sets the optional output_types attribute to value.
 //
-// Identity transformation that models performance.
+// value: If specified, defines the type of each tuple component in an
+// element produced by the resulting iterator.
+// If not specified, defaults to <>
 //
-// Arguments:
-//	input_dataset: A variant tensor representing the input dataset.
+// REQUIRES: len(value) >= 0
+func IteratorFromStringHandleOutputTypes(value []tf.DataType) IteratorFromStringHandleAttr {
+	return func(m optionalAttr) {
+		m["output_types"] = value
+	}
+}
+
+// IteratorFromStringHandleOutputShapes sets the optional output_shapes attribute to value.
+//
+// value: If specified, defines the shape of each tuple component in an
+// element produced by the resulting iterator.
+// If not specified, defaults to <>
+//
+// REQUIRES: len(value) >= 0
+func IteratorFromStringHandleOutputShapes(value []tf.Shape) IteratorFromStringHandleAttr {
+	return func(m optionalAttr) {
+		m["output_shapes"] = value
+	}
+}
+
+// Converts the given string representing a handle to an iterator to a resource.
 //
+// Arguments:
+//	string_handle: A string representation of the given handle.
 //
-func ModelDataset(scope *Scope, input_dataset tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+// Returns A handle to an iterator resource.
+func IteratorFromStringHandle(scope *Scope, string_handle tf.Output, optional ...IteratorFromStringHandleAttr) (resource_handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "ModelDataset",
+		Type: "IteratorFromStringHandle",
 		Input: []tf.Input{
-			input_dataset,
+			string_handle,
 		},
 		Attrs: attrs,
 	}
@@ -33584,1026 +33424,1110 @@ func ModelDataset(scope *Scope, input_dataset tf.Output, output_types []tf.DataT
 	return op.Output(0)
 }
 
-// Returns the truth value of (x > y) element-wise.
+// Gather slices from `params` axis `axis` according to `indices`.
 //
-// *NOTE*: `Greater` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func Greater(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+// `indices` must be an integer tensor of any dimension (usually 0-D or 1-D).
+// Produces an output tensor with shape `params.shape[:axis] + indices.shape +
+// params.shape[axis + 1:]` where:
+//
+// ```python
+//     # Scalar indices (output is rank(params) - 1).
+//     output[a_0, ..., a_n, b_0, ..., b_n] =
+//       params[a_0, ..., a_n, indices, b_0, ..., b_n]
+//
+//     # Vector indices (output is rank(params)).
+//     output[a_0, ..., a_n, i, b_0, ..., b_n] =
+//       params[a_0, ..., a_n, indices[i], b_0, ..., b_n]
+//
+//     # Higher rank indices (output is rank(params) + rank(indices) - 1).
+//     output[a_0, ..., a_n, i, ..., j, b_0, ... b_n] =
+//       params[a_0, ..., a_n, indices[i, ..., j], b_0, ..., b_n]
+// ```
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/Gather.png" alt>
+// </div>
+//
+// Note that on CPU, if an out of bound index is found, an error is returned.
+// On GPU, if an out of bound index is found, a 0 is stored in the
+// corresponding output value.
+//
+// See also `tf.batch_gather` and `tf.gather_nd`.
+//
+// Arguments:
+//	params: The tensor from which to gather values. Must be at least rank
+// `axis + 1`.
+//	indices: Index tensor. Must be in range `[0, params.shape[axis])`.
+//	axis: The axis in `params` to gather `indices` from. Defaults to the first
+// dimension. Supports negative indexes.
+//
+// Returns Values from `params` gathered from indices given by `indices`, with
+// shape `params.shape[:axis] + indices.shape + params.shape[axis + 1:]`.
+func GatherV2(scope *Scope, params tf.Output, indices tf.Output, axis tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Greater",
+		Type: "GatherV2",
 		Input: []tf.Input{
-			x, y,
+			params, indices, axis,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Performs a padding as a preprocess during a convolution.
-//
-// Similar to FusedResizeAndPadConv2d, this op allows for an optimized
-// implementation where the spatial padding transformation stage is fused with the
-// im2col lookup, but in this case without the bilinear filtering required for
-// resizing. Fusing the padding prevents the need to write out the intermediate
-// results as whole tensors, reducing memory pressure, and we can get some latency
-// gains by merging the transformation calculations.
-// The data_format attribute for Conv2D isn't supported by this op, and 'NHWC'
-// order is used instead.
-// Internally this op uses a single per-graph scratch buffer, which means that it
-// will block if multiple versions are being run in parallel. This is because this
-// operator is primarily an optimization to minimize memory usage.
+// Converts the given `resource_handle` representing an iterator to a variant tensor.
 //
 // Arguments:
-//	input: 4-D with shape `[batch, in_height, in_width, in_channels]`.
-//	paddings: A two-column matrix specifying the padding sizes. The number of
-// rows must be the same as the rank of `input`.
-//	filter: 4-D with shape
-// `[filter_height, filter_width, in_channels, out_channels]`.
+//	resource_handle: A handle to an iterator resource.
 //
-//	strides: 1-D of length 4.  The stride of the sliding window for each dimension
-// of `input`. Must be in the same order as the dimension specified with format.
-//	padding: The type of padding algorithm to use.
-func FusedPadConv2D(scope *Scope, input tf.Output, paddings tf.Output, filter tf.Output, mode string, strides []int64, padding string) (output tf.Output) {
+// Returns A variant tensor storing the state of the iterator contained in the
+// resource.
+func SerializeIterator(scope *Scope, resource_handle tf.Output) (serialized tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"mode": mode, "strides": strides, "padding": padding}
 	opspec := tf.OpSpec{
-		Type: "FusedPadConv2D",
+		Type: "SerializeIterator",
 		Input: []tf.Input{
-			input, paddings, filter,
+			resource_handle,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Conv2DBackpropInputAttr is an optional argument to Conv2DBackpropInput.
-type Conv2DBackpropInputAttr func(optionalAttr)
+// FIFOQueueV2Attr is an optional argument to FIFOQueueV2.
+type FIFOQueueV2Attr func(optionalAttr)
 
-// Conv2DBackpropInputUseCudnnOnGpu sets the optional use_cudnn_on_gpu attribute to value.
-// If not specified, defaults to true
-func Conv2DBackpropInputUseCudnnOnGpu(value bool) Conv2DBackpropInputAttr {
+// FIFOQueueV2Shapes sets the optional shapes attribute to value.
+//
+// value: The shape of each component in a value. The length of this attr must
+// be either 0 or the same as the length of component_types. If the length of
+// this attr is 0, the shapes of queue elements are not constrained, and
+// only one element may be dequeued at a time.
+// If not specified, defaults to <>
+//
+// REQUIRES: len(value) >= 0
+func FIFOQueueV2Shapes(value []tf.Shape) FIFOQueueV2Attr {
 	return func(m optionalAttr) {
-		m["use_cudnn_on_gpu"] = value
+		m["shapes"] = value
 	}
 }
 
-// Conv2DBackpropInputExplicitPaddings sets the optional explicit_paddings attribute to value.
+// FIFOQueueV2Capacity sets the optional capacity attribute to value.
 //
-// value: If `padding` is `"EXPLICIT"`, the list of explicit padding amounts. For the ith
-// dimension, the amount of padding inserted before and after the dimension is
-// `explicit_paddings[2 * i]` and `explicit_paddings[2 * i + 1]`, respectively. If
-// `padding` is not `"EXPLICIT"`, `explicit_paddings` must be empty.
-// If not specified, defaults to <>
-func Conv2DBackpropInputExplicitPaddings(value []int64) Conv2DBackpropInputAttr {
+// value: The upper bound on the number of elements in this queue.
+// Negative numbers mean no limit.
+// If not specified, defaults to -1
+func FIFOQueueV2Capacity(value int64) FIFOQueueV2Attr {
 	return func(m optionalAttr) {
-		m["explicit_paddings"] = value
+		m["capacity"] = value
 	}
 }
 
-// Conv2DBackpropInputDataFormat sets the optional data_format attribute to value.
+// FIFOQueueV2Container sets the optional container attribute to value.
 //
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the data is stored in the order of:
-//     [batch, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, in_channels, in_height, in_width].
-// If not specified, defaults to "NHWC"
-func Conv2DBackpropInputDataFormat(value string) Conv2DBackpropInputAttr {
+// value: If non-empty, this queue is placed in the given container.
+// Otherwise, a default container is used.
+// If not specified, defaults to ""
+func FIFOQueueV2Container(value string) FIFOQueueV2Attr {
 	return func(m optionalAttr) {
-		m["data_format"] = value
+		m["container"] = value
 	}
 }
 
-// Conv2DBackpropInputDilations sets the optional dilations attribute to value.
+// FIFOQueueV2SharedName sets the optional shared_name attribute to value.
 //
-// value: 1-D tensor of length 4.  The dilation factor for each dimension of
-// `input`. If set to k > 1, there will be k-1 skipped cells between each filter
-// element on that dimension. The dimension order is determined by the value of
-// `data_format`, see above for details. Dilations in the batch and depth
-// dimensions must be 1.
-// If not specified, defaults to <i:1 i:1 i:1 i:1 >
-func Conv2DBackpropInputDilations(value []int64) Conv2DBackpropInputAttr {
+// value: If non-empty, this queue will be shared under the given name
+// across multiple sessions.
+// If not specified, defaults to ""
+func FIFOQueueV2SharedName(value string) FIFOQueueV2Attr {
 	return func(m optionalAttr) {
-		m["dilations"] = value
+		m["shared_name"] = value
 	}
 }
 
-// Computes the gradients of convolution with respect to the input.
+// A queue that produces elements in first-in first-out order.
 //
 // Arguments:
-//	input_sizes: An integer vector representing the shape of `input`,
-// where `input` is a 4-D `[batch, height, width, channels]` tensor.
-//	filter: 4-D with shape
-// `[filter_height, filter_width, in_channels, out_channels]`.
-//	out_backprop: 4-D with shape `[batch, out_height, out_width, out_channels]`.
-// Gradients w.r.t. the output of the convolution.
-//	strides: The stride of the sliding window for each dimension of the input
-// of the convolution. Must be in the same order as the dimension specified with
-// format.
-//	padding: The type of padding algorithm to use.
+//	component_types: The type of each component in a value.
 //
-// Returns 4-D with shape `[batch, in_height, in_width, in_channels]`.  Gradient
-// w.r.t. the input of the convolution.
-func Conv2DBackpropInput(scope *Scope, input_sizes tf.Output, filter tf.Output, out_backprop tf.Output, strides []int64, padding string, optional ...Conv2DBackpropInputAttr) (output tf.Output) {
+// Returns The handle to the queue.
+func FIFOQueueV2(scope *Scope, component_types []tf.DataType, optional ...FIFOQueueV2Attr) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"strides": strides, "padding": padding}
+	attrs := map[string]interface{}{"component_types": component_types}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Conv2DBackpropInput",
+		Type: "FIFOQueueV2",
+
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Deserializes a proto into the tree handle
+//
+// Arguments:
+//	tree_handle: Handle to the tree resource to be restored.
+//	tree_config: Serialied proto string of the boosted_trees.Tree proto.
+//
+// Returns the created operation.
+func TensorForestTreeDeserialize(scope *Scope, tree_handle tf.Output, tree_config tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorForestTreeDeserialize",
 		Input: []tf.Input{
-			input_sizes, filter, out_backprop,
+			tree_handle, tree_config,
+		},
+	}
+	return scope.AddOperation(opspec)
+}
+
+// Constructs an Optional variant from a tuple of tensors.
+func OptionalFromValue(scope *Scope, components []tf.Output) (optional tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "OptionalFromValue",
+		Input: []tf.Input{
+			tf.OutputList(components),
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Interleave the values from the `data` tensors into a single tensor.
+// DecodeProtoV2Attr is an optional argument to DecodeProtoV2.
+type DecodeProtoV2Attr func(optionalAttr)
+
+// DecodeProtoV2DescriptorSource sets the optional descriptor_source attribute to value.
 //
-// Builds a merged tensor such that
+// value: Either the special value `local://` or a path to a file containing
+// a serialized `FileDescriptorSet`.
+// If not specified, defaults to "local://"
+func DecodeProtoV2DescriptorSource(value string) DecodeProtoV2Attr {
+	return func(m optionalAttr) {
+		m["descriptor_source"] = value
+	}
+}
+
+// DecodeProtoV2MessageFormat sets the optional message_format attribute to value.
 //
-// ```python
-//     merged[indices[m][i, ..., j], ...] = data[m][i, ..., j, ...]
-// ```
+// value: Either `binary` or `text`.
+// If not specified, defaults to "binary"
+func DecodeProtoV2MessageFormat(value string) DecodeProtoV2Attr {
+	return func(m optionalAttr) {
+		m["message_format"] = value
+	}
+}
+
+// DecodeProtoV2Sanitize sets the optional sanitize attribute to value.
 //
-// For example, if each `indices[m]` is scalar or vector, we have
+// value: Whether to sanitize the result or not.
+// If not specified, defaults to false
+func DecodeProtoV2Sanitize(value bool) DecodeProtoV2Attr {
+	return func(m optionalAttr) {
+		m["sanitize"] = value
+	}
+}
+
+// The op extracts fields from a serialized protocol buffers message into tensors.
 //
-// ```python
-//     # Scalar indices:
-//     merged[indices[m], ...] = data[m][...]
+// The `decode_proto` op extracts fields from a serialized protocol buffers
+// message into tensors.  The fields in `field_names` are decoded and converted
+// to the corresponding `output_types` if possible.
 //
-//     # Vector indices:
-//     merged[indices[m][i], ...] = data[m][i, ...]
-// ```
+// A `message_type` name must be provided to give context for the field
+// names. The actual message descriptor can be looked up either in the
+// linked-in descriptor pool or a filename provided by the caller using
+// the `descriptor_source` attribute.
 //
-// Each `data[i].shape` must start with the corresponding `indices[i].shape`,
-// and the rest of `data[i].shape` must be constant w.r.t. `i`.  That is, we
-// must have `data[i].shape = indices[i].shape + constant`.  In terms of this
-// `constant`, the output shape is
+// Each output tensor is a dense tensor. This means that it is padded to
+// hold the largest number of repeated elements seen in the input
+// minibatch. (The shape is also padded by one to prevent zero-sized
+// dimensions). The actual repeat counts for each example in the
+// minibatch can be found in the `sizes` output. In many cases the output
+// of `decode_proto` is fed immediately into tf.squeeze if missing values
+// are not a concern. When using tf.squeeze, always pass the squeeze
+// dimension explicitly to avoid surprises.
 //
-//     merged.shape = [max(indices)] + constant
+// For the most part, the mapping between Proto field types and
+// TensorFlow dtypes is straightforward. However, there are a few
+// special cases:
 //
-// Values are merged in order, so if an index appears in both `indices[m][i]` and
-// `indices[n][j]` for `(m,i) < (n,j)` the slice `data[n][j]` will appear in the
-// merged result. If you do not need this guarantee, ParallelDynamicStitch might
-// perform better on some devices.
+// - A proto field that contains a submessage or group can only be converted
+// to `DT_STRING` (the serialized submessage). This is to reduce the
+// complexity of the API. The resulting string can be used as input
+// to another instance of the decode_proto op.
 //
-// For example:
+// - TensorFlow lacks support for unsigned integers. The ops represent uint64
+// types as a `DT_INT64` with the same twos-complement bit pattern
+// (the obvious way). Unsigned int32 values can be represented exactly by
+// specifying type `DT_INT64`, or using twos-complement if the caller
+// specifies `DT_INT32` in the `output_types` attribute.
 //
-// ```python
-//     indices[0] = 6
-//     indices[1] = [4, 1]
-//     indices[2] = [[5, 2], [0, 3]]
-//     data[0] = [61, 62]
-//     data[1] = [[41, 42], [11, 12]]
-//     data[2] = [[[51, 52], [21, 22]], [[1, 2], [31, 32]]]
-//     merged = [[1, 2], [11, 12], [21, 22], [31, 32], [41, 42],
-//               [51, 52], [61, 62]]
-// ```
+// The `descriptor_source` attribute selects a source of protocol
+// descriptors to consult when looking up `message_type`. This may be a
+// filename containing a serialized `FileDescriptorSet` message,
+// or the special value `local://`, in which case only descriptors linked
+// into the code will be searched; the filename can be on any filesystem
+// accessible to TensorFlow.
 //
-// This method can be used to merge partitions created by `dynamic_partition`
-// as illustrated on the following example:
+// You can build a `descriptor_source` file using the `--descriptor_set_out`
+// and `--include_imports` options to the protocol compiler `protoc`.
 //
-// ```python
-//     # Apply function (increments x_i) on elements for which a certain condition
-//     # apply (x_i != -1 in this example).
-//     x=tf.constant([0.1, -1., 5.2, 4.3, -1., 7.4])
-//     condition_mask=tf.not_equal(x,tf.constant(-1.))
-//     partitioned_data = tf.dynamic_partition(
-//         x, tf.cast(condition_mask, tf.int32) , 2)
-//     partitioned_data[1] = partitioned_data[1] + 1.0
-//     condition_indices = tf.dynamic_partition(
-//         tf.range(tf.shape(x)[0]), tf.cast(condition_mask, tf.int32) , 2)
-//     x = tf.dynamic_stitch(condition_indices, partitioned_data)
-//     # Here x=[1.1, -1., 6.2, 5.3, -1, 8.4], the -1. values remain
-//     # unchanged.
-// ```
+// The `local://` database only covers descriptors linked into the
+// code via C++ libraries, not Python imports. You can link in a proto descriptor
+// by creating a cc_library target with alwayslink=1.
 //
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/DynamicStitch.png" alt>
-// </div>
-func DynamicStitch(scope *Scope, indices []tf.Output, data []tf.Output) (merged tf.Output) {
+// Both binary and text proto serializations are supported, and can be
+// chosen using the `format` attribute.
+//
+// Arguments:
+//	bytes: Tensor of serialized protos with shape `batch_shape`.
+//	message_type: Name of the proto message type to decode.
+//	field_names: List of strings containing proto field names.
+//	output_types: List of TF types to use for the respective field in field_names.
+//
+// Returns Tensor of int32 with shape `[batch_shape, len(field_names)]`.
+// Each entry is the number of values found for the corresponding field.
+// Optional fields may have 0 or 1 values.List of tensors containing values for the corresponding field.
+// `values[i]` has datatype `output_types[i]`
+// and shape `[batch_shape, max(sizes[...,i])]`.
+func DecodeProtoV2(scope *Scope, bytes tf.Output, message_type string, field_names []string, output_types []tf.DataType, optional ...DecodeProtoV2Attr) (sizes tf.Output, values []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"message_type": message_type, "field_names": field_names, "output_types": output_types}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "DynamicStitch",
+		Type: "DecodeProtoV2",
 		Input: []tf.Input{
-			tf.OutputList(indices), tf.OutputList(data),
+			bytes,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	sizes = op.Output(idx)
+	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
+		scope.UpdateErr("DecodeProtoV2", err)
+		return
+	}
+	return sizes, values
 }
 
-// Returns the truth value of (x == y) element-wise.
-//
-// *NOTE*: `Equal` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func Equal(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+// Creates an Optional variant with no value.
+func OptionalNone(scope *Scope) (optional tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Equal",
-		Input: []tf.Input{
-			x, y,
-		},
+		Type: "OptionalNone",
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// TensorArrayGatherV2Attr is an optional argument to TensorArrayGatherV2.
-type TensorArrayGatherV2Attr func(optionalAttr)
-
-// TensorArrayGatherV2ElementShape sets the optional element_shape attribute to value.
-// If not specified, defaults to <unknown_rank:true >
-func TensorArrayGatherV2ElementShape(value tf.Shape) TensorArrayGatherV2Attr {
-	return func(m optionalAttr) {
-		m["element_shape"] = value
-	}
-}
-
-// Deprecated. Use TensorArrayGatherV3
-//
-// DEPRECATED at GraphDef version 26: Use TensorArrayGatherV3
-func TensorArrayGatherV2(scope *Scope, handle tf.Output, indices tf.Output, flow_in tf.Output, dtype tf.DataType, optional ...TensorArrayGatherV2Attr) (value tf.Output) {
+// Returns true if and only if the given Optional variant has a value.
+func OptionalHasValue(scope *Scope, optional tf.Output) (has_value tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "TensorArrayGatherV2",
+		Type: "OptionalHasValue",
 		Input: []tf.Input{
-			handle, indices, flow_in,
+			optional,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Interleave the values from the `data` tensors into a single tensor.
-//
-// Builds a merged tensor such that
-//
-// ```python
-//     merged[indices[m][i, ..., j], ...] = data[m][i, ..., j, ...]
-// ```
-//
-// For example, if each `indices[m]` is scalar or vector, we have
-//
-// ```python
-//     # Scalar indices:
-//     merged[indices[m], ...] = data[m][...]
-//
-//     # Vector indices:
-//     merged[indices[m][i], ...] = data[m][i, ...]
-// ```
-//
-// Each `data[i].shape` must start with the corresponding `indices[i].shape`,
-// and the rest of `data[i].shape` must be constant w.r.t. `i`.  That is, we
-// must have `data[i].shape = indices[i].shape + constant`.  In terms of this
-// `constant`, the output shape is
-//
-//     merged.shape = [max(indices)] + constant
-//
-// Values may be merged in parallel, so if an index appears in both `indices[m][i]`
-// and `indices[n][j]`, the result may be invalid. This differs from the normal
-// DynamicStitch operator that defines the behavior in that case.
-//
-// For example:
-//
-// ```python
-//     indices[0] = 6
-//     indices[1] = [4, 1]
-//     indices[2] = [[5, 2], [0, 3]]
-//     data[0] = [61, 62]
-//     data[1] = [[41, 42], [11, 12]]
-//     data[2] = [[[51, 52], [21, 22]], [[1, 2], [31, 32]]]
-//     merged = [[1, 2], [11, 12], [21, 22], [31, 32], [41, 42],
-//               [51, 52], [61, 62]]
-// ```
-//
-// This method can be used to merge partitions created by `dynamic_partition`
-// as illustrated on the following example:
-//
-// ```python
-//     # Apply function (increments x_i) on elements for which a certain condition
-//     # apply (x_i != -1 in this example).
-//     x=tf.constant([0.1, -1., 5.2, 4.3, -1., 7.4])
-//     condition_mask=tf.not_equal(x,tf.constant(-1.))
-//     partitioned_data = tf.dynamic_partition(
-//         x, tf.cast(condition_mask, tf.int32) , 2)
-//     partitioned_data[1] = partitioned_data[1] + 1.0
-//     condition_indices = tf.dynamic_partition(
-//         tf.range(tf.shape(x)[0]), tf.cast(condition_mask, tf.int32) , 2)
-//     x = tf.dynamic_stitch(condition_indices, partitioned_data)
-//     # Here x=[1.1, -1., 6.2, 5.3, -1, 8.4], the -1. values remain
-//     # unchanged.
-// ```
-//
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/DynamicStitch.png" alt>
-// </div>
-func ParallelDynamicStitch(scope *Scope, indices []tf.Output, data []tf.Output) (merged tf.Output) {
+// Returns the value stored in an Optional variant or raises an error if none exists.
+func OptionalGetValue(scope *Scope, optional tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (components []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "ParallelDynamicStitch",
+		Type: "OptionalGetValue",
 		Input: []tf.Input{
-			tf.OutputList(indices), tf.OutputList(data),
+			optional,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// PriorityQueueV2Attr is an optional argument to PriorityQueueV2.
-type PriorityQueueV2Attr func(optionalAttr)
-
-// PriorityQueueV2ComponentTypes sets the optional component_types attribute to value.
-//
-// value: The type of each component in a value.
-// If not specified, defaults to <>
-//
-// REQUIRES: len(value) >= 0
-func PriorityQueueV2ComponentTypes(value []tf.DataType) PriorityQueueV2Attr {
-	return func(m optionalAttr) {
-		m["component_types"] = value
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// PriorityQueueV2Capacity sets the optional capacity attribute to value.
-//
-// value: The upper bound on the number of elements in this queue.
-// Negative numbers mean no limit.
-// If not specified, defaults to -1
-func PriorityQueueV2Capacity(value int64) PriorityQueueV2Attr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
+	var idx int
+	var err error
+	if components, idx, err = makeOutputList(op, idx, "components"); err != nil {
+		scope.UpdateErr("OptionalGetValue", err)
+		return
 	}
+	return components
 }
 
-// PriorityQueueV2Container sets the optional container attribute to value.
-//
-// value: If non-empty, this queue is placed in the given container.
-// Otherwise, a default container is used.
-// If not specified, defaults to ""
-func PriorityQueueV2Container(value string) PriorityQueueV2Attr {
-	return func(m optionalAttr) {
-		m["container"] = value
+// Gets the next output from the given iterator as an Optional variant.
+func IteratorGetNextAsOptional(scope *Scope, iterator tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (optional tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// PriorityQueueV2SharedName sets the optional shared_name attribute to value.
-//
-// value: If non-empty, this queue will be shared under the given name
-// across multiple sessions.
-// If not specified, defaults to ""
-func PriorityQueueV2SharedName(value string) PriorityQueueV2Attr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "IteratorGetNextAsOptional",
+		Input: []tf.Input{
+			iterator,
+		},
+		Attrs: attrs,
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// A queue that produces elements sorted by the first component value.
+// Fast Fourier transform.
 //
-// Note that the PriorityQueue requires the first component of any element
-// to be a scalar int64, in addition to the other elements declared by
-// component_types.  Therefore calls to Enqueue and EnqueueMany (resp. Dequeue
-// and DequeueMany) on a PriorityQueue will all require (resp. output) one extra
-// entry in their input (resp. output) lists.
+// Computes the 1-dimensional discrete Fourier transform over the inner-most
+// dimension of `input`.
 //
 // Arguments:
-//	shapes: The shape of each component in a value. The length of this attr must
-// be either 0 or the same as the length of component_types. If the length of
-// this attr is 0, the shapes of queue elements are not constrained, and
-// only one element may be dequeued at a time.
+//	input: A complex tensor.
 //
-// Returns The handle to the queue.
-func PriorityQueueV2(scope *Scope, shapes []tf.Shape, optional ...PriorityQueueV2Attr) (handle tf.Output) {
+// Returns A complex tensor of the same shape as `input`. The inner-most
+//   dimension of `input` is replaced with its 1D Fourier transform.
+//
+// @compatibility(numpy)
+// Equivalent to np.fft.fft
+// @end_compatibility
+func FFT(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"shapes": shapes}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "PriorityQueueV2",
-
-		Attrs: attrs,
+		Type: "FFT",
+		Input: []tf.Input{
+			input,
+		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// QueueEnqueueV2Attr is an optional argument to QueueEnqueueV2.
-type QueueEnqueueV2Attr func(optionalAttr)
-
-// QueueEnqueueV2TimeoutMs sets the optional timeout_ms attribute to value.
-//
-// value: If the queue is full, this operation will block for up to
-// timeout_ms milliseconds.
-// Note: This option is not supported yet.
-// If not specified, defaults to -1
-func QueueEnqueueV2TimeoutMs(value int64) QueueEnqueueV2Attr {
-	return func(m optionalAttr) {
-		m["timeout_ms"] = value
-	}
-}
-
-// Enqueues a tuple of one or more tensors in the given queue.
-//
-// The components input has k elements, which correspond to the components of
-// tuples stored in the given queue.
+// Identity transformation that models performance.
 //
-// N.B. If the queue is full, this operation will block until the given
-// element has been enqueued (or 'timeout_ms' elapses, if specified).
+// Identity transformation that models performance.
 //
 // Arguments:
-//	handle: The handle to a queue.
-//	components: One or more tensors from which the enqueued tensors should be taken.
+//	input_dataset: A variant tensor representing the input dataset.
 //
-// Returns the created operation.
-func QueueEnqueueV2(scope *Scope, handle tf.Output, components []tf.Output, optional ...QueueEnqueueV2Attr) (o *tf.Operation) {
+//
+func ModelDataset(scope *Scope, input_dataset tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "QueueEnqueueV2",
+		Type: "ModelDataset",
 		Input: []tf.Input{
-			handle, tf.OutputList(components),
+			input_dataset,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Computes the Bessel i0e function of `x` element-wise.
-//
-// Exponentially scaled modified Bessel function of order 0 defined as
-// `bessel_i0e(x) = exp(-abs(x)) bessel_i0(x)`.
+// Returns the truth value of (x > y) element-wise.
 //
-// This function is faster and numerically stabler than `bessel_i0(x)`.
-func BesselI0e(scope *Scope, x tf.Output) (y tf.Output) {
+// *NOTE*: `Greater` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func Greater(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "BesselI0e",
+		Type: "Greater",
 		Input: []tf.Input{
-			x,
+			x, y,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// QueueDequeueManyV2Attr is an optional argument to QueueDequeueManyV2.
-type QueueDequeueManyV2Attr func(optionalAttr)
-
-// QueueDequeueManyV2TimeoutMs sets the optional timeout_ms attribute to value.
+// Applies sparse addition to `input` using individual values or slices
 //
-// value: If the queue has fewer than n elements, this operation
-// will block for up to timeout_ms milliseconds.
-// Note: This option is not supported yet.
-// If not specified, defaults to -1
-func QueueDequeueManyV2TimeoutMs(value int64) QueueDequeueManyV2Attr {
-	return func(m optionalAttr) {
-		m["timeout_ms"] = value
-	}
-}
-
-// Dequeues `n` tuples of one or more tensors from the given queue.
+// from `updates` according to indices `indices`.  The updates are non-aliasing:
+// `input` is only modified in-place if no other operations will use it.
+// Otherwise, a copy of `input` is made.  This operation has a gradient with
+// respect to both `input` and `updates`.
 //
-// If the queue is closed and there are fewer than `n` elements, then an
-// OutOfRange error is returned.
+// `input` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`.
 //
-// This operation concatenates queue-element component tensors along the
-// 0th dimension to make a single component tensor.  All of the components
-// in the dequeued tuple will have size `n` in the 0th dimension.
+// `indices` must be integer tensor, containing indices into `input`.
+// It must be shape \\([d_0, ..., d_{Q-2}, K]\\) where `0 < K <= P`.
 //
-// This operation has `k` outputs, where `k` is the number of components in
-// the tuples stored in the given queue, and output `i` is the ith
-// component of the dequeued tuple.
+// The innermost dimension of `indices` (with length `K`) corresponds to
+// indices into elements (if `K = P`) or `(P-K)`-dimensional slices
+// (if `K < P`) along the `K`th dimension of `input`.
 //
-// N.B. If the queue is empty, this operation will block until `n` elements
-// have been dequeued (or 'timeout_ms' elapses, if specified).
+// `updates` is `Tensor` of rank `Q-1+P-K` with shape:
+//
+// $$[d_0, ..., d_{Q-2}, input.shape[K], ..., input.shape[P-1]].$$
+//
+// For example, say we want to add 4 scattered elements to a rank-1 tensor to 8
+// elements. In Python, that addition would look like this:
+//
+//     input = tf.constant([1, 2, 3, 4, 5, 6, 7, 8])
+//     indices = tf.constant([[4], [3], [1], [7]])
+//     updates = tf.constant([9, 10, 11, 12])
+//     output = tf.scatter_nd_non_aliasing_add(input, indices, updates)
+//     with tf.Session() as sess:
+//       print(sess.run(output))
+//
+// The resulting value `output` would look like this:
+//
+//     [1, 13, 3, 14, 14, 6, 7, 20]
+//
+// See `tf.scatter_nd` for more details about how to make updates to slices.
 //
 // Arguments:
-//	handle: The handle to a queue.
-//	n: The number of tuples to dequeue.
-//	component_types: The type of each component in a tuple.
+//	input: A Tensor.
+//	indices: A Tensor. Must be one of the following types: `int32`, `int64`.
+// A tensor of indices into `input`.
+//	updates: A Tensor. Must have the same type as ref. A tensor of updated values
+// to add to `input`.
 //
-// Returns One or more tensors that were dequeued as a tuple.
-func QueueDequeueManyV2(scope *Scope, handle tf.Output, n tf.Output, component_types []tf.DataType, optional ...QueueDequeueManyV2Attr) (components []tf.Output) {
+// Returns A `Tensor` with the same shape as `input`, containing values of `input`
+// updated with `updates`.
+func ScatterNdNonAliasingAdd(scope *Scope, input tf.Output, indices tf.Output, updates tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"component_types": component_types}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "QueueDequeueManyV2",
+		Type: "ScatterNdNonAliasingAdd",
 		Input: []tf.Input{
-			handle, n,
+			input, indices, updates,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if components, idx, err = makeOutputList(op, idx, "components"); err != nil {
-		scope.UpdateErr("QueueDequeueManyV2", err)
-		return
+	return op.Output(0)
+}
+
+// FractionalMaxPoolAttr is an optional argument to FractionalMaxPool.
+type FractionalMaxPoolAttr func(optionalAttr)
+
+// FractionalMaxPoolPseudoRandom sets the optional pseudo_random attribute to value.
+//
+// value: When set to True, generates the pooling sequence in a
+// pseudorandom fashion, otherwise, in a random fashion. Check paper [Benjamin
+// Graham, Fractional Max-Pooling](http://arxiv.org/abs/1412.6071) for
+// difference between pseudorandom and random.
+// If not specified, defaults to false
+func FractionalMaxPoolPseudoRandom(value bool) FractionalMaxPoolAttr {
+	return func(m optionalAttr) {
+		m["pseudo_random"] = value
 	}
-	return components
 }
 
-// EncodeBase64Attr is an optional argument to EncodeBase64.
-type EncodeBase64Attr func(optionalAttr)
+// FractionalMaxPoolOverlapping sets the optional overlapping attribute to value.
+//
+// value: When set to True, it means when pooling, the values at the boundary
+// of adjacent pooling cells are used by both cells. For example:
+//
+// `index  0  1  2  3  4`
+//
+// `value  20 5  16 3  7`
+//
+// If the pooling sequence is [0, 2, 4], then 16, at index 2 will be used twice.
+// The result would be [20, 16] for fractional max pooling.
+// If not specified, defaults to false
+func FractionalMaxPoolOverlapping(value bool) FractionalMaxPoolAttr {
+	return func(m optionalAttr) {
+		m["overlapping"] = value
+	}
+}
 
-// EncodeBase64Pad sets the optional pad attribute to value.
+// FractionalMaxPoolDeterministic sets the optional deterministic attribute to value.
 //
-// value: Bool whether padding is applied at the ends.
+// value: When set to True, a fixed pooling region will be used when
+// iterating over a FractionalMaxPool node in the computation graph. Mainly used
+// in unit test to make FractionalMaxPool deterministic.
 // If not specified, defaults to false
-func EncodeBase64Pad(value bool) EncodeBase64Attr {
+func FractionalMaxPoolDeterministic(value bool) FractionalMaxPoolAttr {
 	return func(m optionalAttr) {
-		m["pad"] = value
+		m["deterministic"] = value
 	}
 }
 
-// Encode strings into web-safe base64 format.
+// FractionalMaxPoolSeed sets the optional seed attribute to value.
 //
-// Refer to the following article for more information on base64 format:
-// en.wikipedia.org/wiki/Base64. Base64 strings may have padding with '=' at the
-// end so that the encoded has length multiple of 4. See Padding section of the
-// link above.
+// value: If either seed or seed2 are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func FractionalMaxPoolSeed(value int64) FractionalMaxPoolAttr {
+	return func(m optionalAttr) {
+		m["seed"] = value
+	}
+}
+
+// FractionalMaxPoolSeed2 sets the optional seed2 attribute to value.
+//
+// value: An second seed to avoid seed collision.
+// If not specified, defaults to 0
+func FractionalMaxPoolSeed2(value int64) FractionalMaxPoolAttr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// Performs fractional max pooling on the input.
+//
+// Fractional max pooling is slightly different than regular max pooling.  In
+// regular max pooling, you downsize an input set by taking the maximum value of
+// smaller N x N subsections of the set (often 2x2), and try to reduce the set by
+// a factor of N, where N is an integer.  Fractional max pooling, as you might
+// expect from the word "fractional", means that the overall reduction ratio N
+// does not have to be an integer.
+//
+// The sizes of the pooling regions are generated randomly but are fairly uniform.
+// For example, let's look at the height dimension, and the constraints on the
+// list of rows that will be pool boundaries.
+//
+// First we define the following:
+//
+// 1.  input_row_length : the number of rows from the input set
+// 2.  output_row_length : which will be smaller than the input
+// 3.  alpha = input_row_length / output_row_length : our reduction ratio
+// 4.  K = floor(alpha)
+// 5.  row_pooling_sequence : this is the result list of pool boundary rows
+//
+// Then, row_pooling_sequence should satisfy:
+//
+// 1.  a[0] = 0 : the first value of the sequence is 0
+// 2.  a[end] = input_row_length : the last value of the sequence is the size
+// 3.  K <= (a[i+1] - a[i]) <= K+1 : all intervals are K or K+1 size
+// 4.  length(row_pooling_sequence) = output_row_length+1
 //
-// Web-safe means that the encoder uses - and _ instead of + and /.
+// For more details on fractional max pooling, see this paper:
+// [Benjamin Graham, Fractional Max-Pooling](http://arxiv.org/abs/1412.6071)
 //
 // Arguments:
-//	input: Strings to be encoded.
+//	value: 4-D with shape `[batch, height, width, channels]`.
+//	pooling_ratio: Pooling ratio for each dimension of `value`, currently only
+// supports row and col dimension and should be >= 1.0. For example, a valid
+// pooling ratio looks like [1.0, 1.44, 1.73, 1.0]. The first and last elements
+// must be 1.0 because we don't allow pooling on batch and channels
+// dimensions. 1.44 and 1.73 are pooling ratio on height and width dimensions
+// respectively.
 //
-// Returns Input strings encoded in base64.
-func EncodeBase64(scope *Scope, input tf.Output, optional ...EncodeBase64Attr) (output tf.Output) {
+// Returns output tensor after fractional max pooling.row pooling sequence, needed to calculate gradient.column pooling sequence, needed to calculate gradient.
+func FractionalMaxPool(scope *Scope, value tf.Output, pooling_ratio []float32, optional ...FractionalMaxPoolAttr) (output tf.Output, row_pooling_sequence tf.Output, col_pooling_sequence tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"pooling_ratio": pooling_ratio}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "EncodeBase64",
+		Type: "FractionalMaxPool",
 		Input: []tf.Input{
-			input,
+			value,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// A dataset that creates window datasets from the input dataset.
+// Creates a MultiDeviceIterator resource.
 //
 // Arguments:
+//	devices: A list of devices the iterator works across.
+//	shared_name: If non-empty, this resource will be shared under the given name
+// across multiple sessions.
+//	container: If non-empty, this resource is placed in the given container.
+// Otherwise, a default container is used.
+//	output_types: The type list for the return values.
+//	output_shapes: The list of shapes being produced.
 //
-//	size: A scalar representing the number of elements to accumulate in a window.
-//	shift: A scalar representing the steps moving the sliding window forward in one
-// iteration. It must be positive.
-//	stride: A scalar representing the stride of the input elements of the sliding window.
-// It must be positive.
-//	drop_remainder: A scalar representing whether a window should be dropped in case its size is
-// smaller than desired.
-//
-//
-func WindowDataset(scope *Scope, input_dataset tf.Output, size tf.Output, shift tf.Output, stride tf.Output, drop_remainder tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+// Returns Handle to the resource created.
+func MultiDeviceIterator(scope *Scope, devices []string, shared_name string, container string, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	attrs := map[string]interface{}{"devices": devices, "shared_name": shared_name, "container": container, "output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "WindowDataset",
-		Input: []tf.Input{
-			input_dataset, size, shift, stride, drop_remainder,
-		},
+		Type: "MultiDeviceIterator",
+
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Deprecated. Use TensorArrayCloseV3
-//
-// DEPRECATED at GraphDef version 26: Use TensorArrayCloseV3
-//
-// Returns the created operation.
-func TensorArrayCloseV2(scope *Scope, handle tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "TensorArrayCloseV2",
-		Input: []tf.Input{
-			handle,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
-// Forwards the value of an available tensor from `inputs` to `output`.
-//
-// `Merge` waits for at least one of the tensors in `inputs` to become available.
-// It is usually combined with `Switch` to implement branching.
+// Performs a padding as a preprocess during a convolution.
 //
-// `Merge` forwards the first tensor to become available to `output`, and sets
-// `value_index` to its index in `inputs`.
+// Similar to FusedResizeAndPadConv2d, this op allows for an optimized
+// implementation where the spatial padding transformation stage is fused with the
+// im2col lookup, but in this case without the bilinear filtering required for
+// resizing. Fusing the padding prevents the need to write out the intermediate
+// results as whole tensors, reducing memory pressure, and we can get some latency
+// gains by merging the transformation calculations.
+// The data_format attribute for Conv2D isn't supported by this op, and 'NHWC'
+// order is used instead.
+// Internally this op uses a single per-graph scratch buffer, which means that it
+// will block if multiple versions are being run in parallel. This is because this
+// operator is primarily an optimization to minimize memory usage.
 //
 // Arguments:
-//	inputs: The input tensors, exactly one of which will become available.
+//	input: 4-D with shape `[batch, in_height, in_width, in_channels]`.
+//	paddings: A two-column matrix specifying the padding sizes. The number of
+// rows must be the same as the rank of `input`.
+//	filter: 4-D with shape
+// `[filter_height, filter_width, in_channels, out_channels]`.
 //
-// Returns Will be set to the available input tensor.The index of the chosen input tensor in `inputs`.
-func Merge(scope *Scope, inputs []tf.Output) (output tf.Output, value_index tf.Output) {
+//	strides: 1-D of length 4.  The stride of the sliding window for each dimension
+// of `input`. Must be in the same order as the dimension specified with format.
+//	padding: The type of padding algorithm to use.
+func FusedPadConv2D(scope *Scope, input tf.Output, paddings tf.Output, filter tf.Output, mode string, strides []int64, padding string) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"mode": mode, "strides": strides, "padding": padding}
 	opspec := tf.OpSpec{
-		Type: "Merge",
+		Type: "FusedPadConv2D",
 		Input: []tf.Input{
-			tf.OutputList(inputs),
+			input, paddings, filter,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// Writes the given dataset to the given file using the TFRecord format.
-//
-// Arguments:
-//	input_dataset: A variant tensor representing the dataset to write.
-//	filename: A scalar string tensor representing the filename to use.
-//	compression_type: A scalar string tensor containing either (i) the empty string (no
-// compression), (ii) "ZLIB", or (iii) "GZIP".
-//
-// Returns the created operation.
-func ExperimentalDatasetToTFRecord(scope *Scope, input_dataset tf.Output, filename tf.Output, compression_type tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ExperimentalDatasetToTFRecord",
-		Input: []tf.Input{
-			input_dataset, filename, compression_type,
-		},
+// Conv2DBackpropInputAttr is an optional argument to Conv2DBackpropInput.
+type Conv2DBackpropInputAttr func(optionalAttr)
+
+// Conv2DBackpropInputUseCudnnOnGpu sets the optional use_cudnn_on_gpu attribute to value.
+// If not specified, defaults to true
+func Conv2DBackpropInputUseCudnnOnGpu(value bool) Conv2DBackpropInputAttr {
+	return func(m optionalAttr) {
+		m["use_cudnn_on_gpu"] = value
 	}
-	return scope.AddOperation(opspec)
 }
 
-// QueueCloseV2Attr is an optional argument to QueueCloseV2.
-type QueueCloseV2Attr func(optionalAttr)
+// Conv2DBackpropInputExplicitPaddings sets the optional explicit_paddings attribute to value.
+//
+// value: If `padding` is `"EXPLICIT"`, the list of explicit padding amounts. For the ith
+// dimension, the amount of padding inserted before and after the dimension is
+// `explicit_paddings[2 * i]` and `explicit_paddings[2 * i + 1]`, respectively. If
+// `padding` is not `"EXPLICIT"`, `explicit_paddings` must be empty.
+// If not specified, defaults to <>
+func Conv2DBackpropInputExplicitPaddings(value []int64) Conv2DBackpropInputAttr {
+	return func(m optionalAttr) {
+		m["explicit_paddings"] = value
+	}
+}
 
-// QueueCloseV2CancelPendingEnqueues sets the optional cancel_pending_enqueues attribute to value.
+// Conv2DBackpropInputDataFormat sets the optional data_format attribute to value.
 //
-// value: If true, all pending enqueue requests that are
-// blocked on the given queue will be canceled.
-// If not specified, defaults to false
-func QueueCloseV2CancelPendingEnqueues(value bool) QueueCloseV2Attr {
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the data is stored in the order of:
+//     [batch, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, in_channels, in_height, in_width].
+// If not specified, defaults to "NHWC"
+func Conv2DBackpropInputDataFormat(value string) Conv2DBackpropInputAttr {
 	return func(m optionalAttr) {
-		m["cancel_pending_enqueues"] = value
+		m["data_format"] = value
 	}
 }
 
-// Closes the given queue.
+// Conv2DBackpropInputDilations sets the optional dilations attribute to value.
 //
-// This operation signals that no more elements will be enqueued in the
-// given queue. Subsequent Enqueue(Many) operations will fail.
-// Subsequent Dequeue(Many) operations will continue to succeed if
-// sufficient elements remain in the queue. Subsequent Dequeue(Many)
-// operations that would block will fail immediately.
+// value: 1-D tensor of length 4.  The dilation factor for each dimension of
+// `input`. If set to k > 1, there will be k-1 skipped cells between each filter
+// element on that dimension. The dimension order is determined by the value of
+// `data_format`, see above for details. Dilations in the batch and depth
+// dimensions must be 1.
+// If not specified, defaults to <i:1 i:1 i:1 i:1 >
+func Conv2DBackpropInputDilations(value []int64) Conv2DBackpropInputAttr {
+	return func(m optionalAttr) {
+		m["dilations"] = value
+	}
+}
+
+// Computes the gradients of convolution with respect to the input.
 //
 // Arguments:
-//	handle: The handle to a queue.
+//	input_sizes: An integer vector representing the shape of `input`,
+// where `input` is a 4-D `[batch, height, width, channels]` tensor.
+//	filter: 4-D with shape
+// `[filter_height, filter_width, in_channels, out_channels]`.
+//	out_backprop: 4-D with shape `[batch, out_height, out_width, out_channels]`.
+// Gradients w.r.t. the output of the convolution.
+//	strides: The stride of the sliding window for each dimension of the input
+// of the convolution. Must be in the same order as the dimension specified with
+// format.
+//	padding: The type of padding algorithm to use.
 //
-// Returns the created operation.
-func QueueCloseV2(scope *Scope, handle tf.Output, optional ...QueueCloseV2Attr) (o *tf.Operation) {
+// Returns 4-D with shape `[batch, in_height, in_width, in_channels]`.  Gradient
+// w.r.t. the input of the convolution.
+func Conv2DBackpropInput(scope *Scope, input_sizes tf.Output, filter tf.Output, out_backprop tf.Output, strides []int64, padding string, optional ...Conv2DBackpropInputAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"strides": strides, "padding": padding}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "QueueCloseV2",
+		Type: "Conv2DBackpropInput",
 		Input: []tf.Input{
-			handle,
+			input_sizes, filter, out_backprop,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Computes inverse hyperbolic tangent of x element-wise.
-func Atanh(scope *Scope, x tf.Output) (y tf.Output) {
+// Interleave the values from the `data` tensors into a single tensor.
+//
+// Builds a merged tensor such that
+//
+// ```python
+//     merged[indices[m][i, ..., j], ...] = data[m][i, ..., j, ...]
+// ```
+//
+// For example, if each `indices[m]` is scalar or vector, we have
+//
+// ```python
+//     # Scalar indices:
+//     merged[indices[m], ...] = data[m][...]
+//
+//     # Vector indices:
+//     merged[indices[m][i], ...] = data[m][i, ...]
+// ```
+//
+// Each `data[i].shape` must start with the corresponding `indices[i].shape`,
+// and the rest of `data[i].shape` must be constant w.r.t. `i`.  That is, we
+// must have `data[i].shape = indices[i].shape + constant`.  In terms of this
+// `constant`, the output shape is
+//
+//     merged.shape = [max(indices)] + constant
+//
+// Values are merged in order, so if an index appears in both `indices[m][i]` and
+// `indices[n][j]` for `(m,i) < (n,j)` the slice `data[n][j]` will appear in the
+// merged result. If you do not need this guarantee, ParallelDynamicStitch might
+// perform better on some devices.
+//
+// For example:
+//
+// ```python
+//     indices[0] = 6
+//     indices[1] = [4, 1]
+//     indices[2] = [[5, 2], [0, 3]]
+//     data[0] = [61, 62]
+//     data[1] = [[41, 42], [11, 12]]
+//     data[2] = [[[51, 52], [21, 22]], [[1, 2], [31, 32]]]
+//     merged = [[1, 2], [11, 12], [21, 22], [31, 32], [41, 42],
+//               [51, 52], [61, 62]]
+// ```
+//
+// This method can be used to merge partitions created by `dynamic_partition`
+// as illustrated on the following example:
+//
+// ```python
+//     # Apply function (increments x_i) on elements for which a certain condition
+//     # apply (x_i != -1 in this example).
+//     x=tf.constant([0.1, -1., 5.2, 4.3, -1., 7.4])
+//     condition_mask=tf.not_equal(x,tf.constant(-1.))
+//     partitioned_data = tf.dynamic_partition(
+//         x, tf.cast(condition_mask, tf.int32) , 2)
+//     partitioned_data[1] = partitioned_data[1] + 1.0
+//     condition_indices = tf.dynamic_partition(
+//         tf.range(tf.shape(x)[0]), tf.cast(condition_mask, tf.int32) , 2)
+//     x = tf.dynamic_stitch(condition_indices, partitioned_data)
+//     # Here x=[1.1, -1., 6.2, 5.3, -1, 8.4], the -1. values remain
+//     # unchanged.
+// ```
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/DynamicStitch.png" alt>
+// </div>
+func DynamicStitch(scope *Scope, indices []tf.Output, data []tf.Output) (merged tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Atanh",
+		Type: "DynamicStitch",
 		Input: []tf.Input{
-			x,
+			tf.OutputList(indices), tf.OutputList(data),
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Returns true if queue is closed.
-//
-// This operation returns true if the queue is closed and false if the queue
-// is open.
+// Returns the truth value of (x == y) element-wise.
 //
-// Arguments:
-//	handle: The handle to a queue.
-func QueueIsClosedV2(scope *Scope, handle tf.Output) (is_closed tf.Output) {
+// *NOTE*: `Equal` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func Equal(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "QueueIsClosedV2",
+		Type: "Equal",
 		Input: []tf.Input{
-			handle,
+			x, y,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes the absolute value of a tensor.
+// TensorArrayGatherV2Attr is an optional argument to TensorArrayGatherV2.
+type TensorArrayGatherV2Attr func(optionalAttr)
+
+// TensorArrayGatherV2ElementShape sets the optional element_shape attribute to value.
+// If not specified, defaults to <unknown_rank:true >
+func TensorArrayGatherV2ElementShape(value tf.Shape) TensorArrayGatherV2Attr {
+	return func(m optionalAttr) {
+		m["element_shape"] = value
+	}
+}
+
+// Deprecated. Use TensorArrayGatherV3
 //
-// Given a tensor `x`, this operation returns a tensor containing the absolute
-// value of each element in `x`. For example, if x is an input element and y is
-// an output element, this operation computes \\(y = |x|\\).
-func Abs(scope *Scope, x tf.Output) (y tf.Output) {
+// DEPRECATED at GraphDef version 26: Use TensorArrayGatherV3
+func TensorArrayGatherV2(scope *Scope, handle tf.Output, indices tf.Output, flow_in tf.Output, dtype tf.DataType, optional ...TensorArrayGatherV2Attr) (value tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"dtype": dtype}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "Abs",
+		Type: "TensorArrayGatherV2",
 		Input: []tf.Input{
-			x,
+			handle, indices, flow_in,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// StackV2Attr is an optional argument to StackV2.
-type StackV2Attr func(optionalAttr)
-
-// StackV2StackName sets the optional stack_name attribute to value.
+// Interleave the values from the `data` tensors into a single tensor.
 //
-// value: Overrides the name used for the temporary stack resource. Default
-// value is the name of the 'Stack' op (which is guaranteed unique).
-// If not specified, defaults to ""
-func StackV2StackName(value string) StackV2Attr {
-	return func(m optionalAttr) {
-		m["stack_name"] = value
-	}
-}
-
-// A stack that produces elements in first-in last-out order.
+// Builds a merged tensor such that
 //
-// Arguments:
-//	max_size: The maximum size of the stack if non-negative. If negative, the stack
-// size is unlimited.
-//	elem_type: The type of the elements on the stack.
+// ```python
+//     merged[indices[m][i, ..., j], ...] = data[m][i, ..., j, ...]
+// ```
 //
-// Returns The handle to the stack.
-func StackV2(scope *Scope, max_size tf.Output, elem_type tf.DataType, optional ...StackV2Attr) (handle tf.Output) {
+// For example, if each `indices[m]` is scalar or vector, we have
+//
+// ```python
+//     # Scalar indices:
+//     merged[indices[m], ...] = data[m][...]
+//
+//     # Vector indices:
+//     merged[indices[m][i], ...] = data[m][i, ...]
+// ```
+//
+// Each `data[i].shape` must start with the corresponding `indices[i].shape`,
+// and the rest of `data[i].shape` must be constant w.r.t. `i`.  That is, we
+// must have `data[i].shape = indices[i].shape + constant`.  In terms of this
+// `constant`, the output shape is
+//
+//     merged.shape = [max(indices)] + constant
+//
+// Values may be merged in parallel, so if an index appears in both `indices[m][i]`
+// and `indices[n][j]`, the result may be invalid. This differs from the normal
+// DynamicStitch operator that defines the behavior in that case.
+//
+// For example:
+//
+// ```python
+//     indices[0] = 6
+//     indices[1] = [4, 1]
+//     indices[2] = [[5, 2], [0, 3]]
+//     data[0] = [61, 62]
+//     data[1] = [[41, 42], [11, 12]]
+//     data[2] = [[[51, 52], [21, 22]], [[1, 2], [31, 32]]]
+//     merged = [[1, 2], [11, 12], [21, 22], [31, 32], [41, 42],
+//               [51, 52], [61, 62]]
+// ```
+//
+// This method can be used to merge partitions created by `dynamic_partition`
+// as illustrated on the following example:
+//
+// ```python
+//     # Apply function (increments x_i) on elements for which a certain condition
+//     # apply (x_i != -1 in this example).
+//     x=tf.constant([0.1, -1., 5.2, 4.3, -1., 7.4])
+//     condition_mask=tf.not_equal(x,tf.constant(-1.))
+//     partitioned_data = tf.dynamic_partition(
+//         x, tf.cast(condition_mask, tf.int32) , 2)
+//     partitioned_data[1] = partitioned_data[1] + 1.0
+//     condition_indices = tf.dynamic_partition(
+//         tf.range(tf.shape(x)[0]), tf.cast(condition_mask, tf.int32) , 2)
+//     x = tf.dynamic_stitch(condition_indices, partitioned_data)
+//     # Here x=[1.1, -1., 6.2, 5.3, -1, 8.4], the -1. values remain
+//     # unchanged.
+// ```
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/DynamicStitch.png" alt>
+// </div>
+func ParallelDynamicStitch(scope *Scope, indices []tf.Output, data []tf.Output) (merged tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"elem_type": elem_type}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "StackV2",
+		Type: "ParallelDynamicStitch",
 		Input: []tf.Input{
-			max_size,
+			tf.OutputList(indices), tf.OutputList(data),
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// OrderedMapStageAttr is an optional argument to OrderedMapStage.
-type OrderedMapStageAttr func(optionalAttr)
+// PriorityQueueV2Attr is an optional argument to PriorityQueueV2.
+type PriorityQueueV2Attr func(optionalAttr)
 
-// OrderedMapStageCapacity sets the optional capacity attribute to value.
+// PriorityQueueV2ComponentTypes sets the optional component_types attribute to value.
 //
-// value: Maximum number of elements in the Staging Area. If > 0, inserts
-// on the container will block when the capacity is reached.
-// If not specified, defaults to 0
+// value: The type of each component in a value.
+// If not specified, defaults to <>
 //
-// REQUIRES: value >= 0
-func OrderedMapStageCapacity(value int64) OrderedMapStageAttr {
+// REQUIRES: len(value) >= 0
+func PriorityQueueV2ComponentTypes(value []tf.DataType) PriorityQueueV2Attr {
 	return func(m optionalAttr) {
-		m["capacity"] = value
+		m["component_types"] = value
 	}
 }
 
-// OrderedMapStageMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
+// PriorityQueueV2Capacity sets the optional capacity attribute to value.
 //
-// REQUIRES: value >= 0
-func OrderedMapStageMemoryLimit(value int64) OrderedMapStageAttr {
+// value: The upper bound on the number of elements in this queue.
+// Negative numbers mean no limit.
+// If not specified, defaults to -1
+func PriorityQueueV2Capacity(value int64) PriorityQueueV2Attr {
 	return func(m optionalAttr) {
-		m["memory_limit"] = value
+		m["capacity"] = value
 	}
 }
 
-// OrderedMapStageContainer sets the optional container attribute to value.
+// PriorityQueueV2Container sets the optional container attribute to value.
 //
-// value: If non-empty, this queue is placed in the given container. Otherwise,
-// a default container is used.
+// value: If non-empty, this queue is placed in the given container.
+// Otherwise, a default container is used.
 // If not specified, defaults to ""
-func OrderedMapStageContainer(value string) OrderedMapStageAttr {
+func PriorityQueueV2Container(value string) PriorityQueueV2Attr {
 	return func(m optionalAttr) {
 		m["container"] = value
 	}
 }
 
-// OrderedMapStageSharedName sets the optional shared_name attribute to value.
+// PriorityQueueV2SharedName sets the optional shared_name attribute to value.
 //
-// value: It is necessary to match this name to the matching Unstage Op.
+// value: If non-empty, this queue will be shared under the given name
+// across multiple sessions.
 // If not specified, defaults to ""
-func OrderedMapStageSharedName(value string) OrderedMapStageAttr {
+func PriorityQueueV2SharedName(value string) PriorityQueueV2Attr {
 	return func(m optionalAttr) {
 		m["shared_name"] = value
 	}
 }
 
-// Stage (key, values) in the underlying container which behaves like a ordered
+// A queue that produces elements sorted by the first component value.
 //
-// associative container.   Elements are ordered by key.
+// Note that the PriorityQueue requires the first component of any element
+// to be a scalar int64, in addition to the other elements declared by
+// component_types.  Therefore calls to Enqueue and EnqueueMany (resp. Dequeue
+// and DequeueMany) on a PriorityQueue will all require (resp. output) one extra
+// entry in their input (resp. output) lists.
 //
 // Arguments:
-//	key: int64
-//
-//	values: a list of tensors
-// dtypes A list of data types that inserted values should adhere to.
-//
+//	shapes: The shape of each component in a value. The length of this attr must
+// be either 0 or the same as the length of component_types. If the length of
+// this attr is 0, the shapes of queue elements are not constrained, and
+// only one element may be dequeued at a time.
 //
-// Returns the created operation.
-func OrderedMapStage(scope *Scope, key tf.Output, indices tf.Output, values []tf.Output, dtypes []tf.DataType, optional ...OrderedMapStageAttr) (o *tf.Operation) {
+// Returns The handle to the queue.
+func PriorityQueueV2(scope *Scope, shapes []tf.Shape, optional ...PriorityQueueV2Attr) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
+	attrs := map[string]interface{}{"shapes": shapes}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "OrderedMapStage",
-		Input: []tf.Input{
-			key, indices, tf.OutputList(values),
-		},
-		Attrs: attrs,
-	}
-	return scope.AddOperation(opspec)
-}
-
-// RpcAttr is an optional argument to Rpc.
-type RpcAttr func(optionalAttr)
-
-// RpcProtocol sets the optional protocol attribute to value.
-//
-// value: RPC protocol to use.  Empty string means use the default protocol.
-// Options include 'grpc'.
-// If not specified, defaults to ""
-func RpcProtocol(value string) RpcAttr {
-	return func(m optionalAttr) {
-		m["protocol"] = value
-	}
-}
+		Type: "PriorityQueueV2",
 
-// RpcFailFast sets the optional fail_fast attribute to value.
-//
-// value: `boolean`. If `true` (default), then failures to connect
-// (i.e., the server does not immediately respond) cause an RPC failure.
-// If not specified, defaults to true
-func RpcFailFast(value bool) RpcAttr {
-	return func(m optionalAttr) {
-		m["fail_fast"] = value
+		Attrs: attrs,
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// RpcTimeoutInMs sets the optional timeout_in_ms attribute to value.
+// QueueEnqueueV2Attr is an optional argument to QueueEnqueueV2.
+type QueueEnqueueV2Attr func(optionalAttr)
+
+// QueueEnqueueV2TimeoutMs sets the optional timeout_ms attribute to value.
 //
-// value: `int`. If `0` (default), then the kernel will run the RPC
-// request and only time out if the RPC deadline passes or the session times out.
-// If this value is greater than `0`, then the op will raise an exception if
-// the RPC takes longer than `timeout_in_ms`.
-// If not specified, defaults to 0
-func RpcTimeoutInMs(value int64) RpcAttr {
+// value: If the queue is full, this operation will block for up to
+// timeout_ms milliseconds.
+// Note: This option is not supported yet.
+// If not specified, defaults to -1
+func QueueEnqueueV2TimeoutMs(value int64) QueueEnqueueV2Attr {
 	return func(m optionalAttr) {
-		m["timeout_in_ms"] = value
+		m["timeout_ms"] = value
 	}
 }
 
-// Perform batches of RPC requests.
-//
-// This op asynchronously performs either a single RPC request, or a batch
-// of requests.  RPC requests are defined by three main parameters:
-//
-//   - `address` (the host+port or BNS address of the request)
-//   - `method` (the RPC method name for the request)
-//   - `request` (the serialized proto string, or vector of strings,
-//      of the RPC request argument).
-//
-// For example, if you have an RPC service running on port localhost:2345,
-// and its interface is configured with the following proto declaration:
-//
-// ```
-// service MyService {
-//   rpc MyMethod(MyRequestProto) returns (MyResponseProto) {
-//   }
-// };
-// ```
-//
-// then call this op with arguments:
-//
-// ```
-// address = "localhost:2345"
-// method = "MyService/MyMethod"
-// ```
-//
-// The `request` tensor is a string tensor representing serialized `MyRequestProto`
-// strings; and the output string tensor `response` will have the same shape
-// and contain (upon successful completion) corresponding serialized
-// `MyResponseProto` strings.
-//
-// For example, to send a single, empty, `MyRequestProto`, call
-// this op with `request = ""`.  To send 5 **parallel** empty requests,
-// call this op with `request = ["", "", "", "", ""]`.
-//
-// More generally, one can create a batch of `MyRequestProto` serialized protos
-// from regular batched tensors using the `encode_proto` op, and convert
-// the response `MyResponseProto` serialized protos to batched tensors
-// using the `decode_proto` op.
-//
-// **NOTE** Working with serialized proto strings is faster than instantiating
-// actual proto objects in memory, so no performance degradation is expected
-// compared to writing custom kernels for this workflow.
+// Enqueues a tuple of one or more tensors in the given queue.
 //
-// If the connection fails or the remote worker returns an error
-// status, the op reraises this exception locally.
+// The components input has k elements, which correspond to the components of
+// tuples stored in the given queue.
 //
-// See the `TryRpc` op if you prefer to handle RPC failures manually in the graph.
+// N.B. If the queue is full, this operation will block until the given
+// element has been enqueued (or 'timeout_ms' elapses, if specified).
 //
 // Arguments:
-//	address: `0-D` or `1-D`.  The address (i.e. host_name:port) of the RPC server.
-// If this tensor has more than 1 element, then multiple parallel rpc requests
-// are sent.  This argument broadcasts with `method` and `request`.
-//	method: `0-D` or `1-D`.  The method address on the RPC server.
-// If this tensor has more than 1 element, then multiple parallel rpc requests
-// are sent.  This argument broadcasts with `address` and `request`.
-//	request: `0-D` or `1-D`.  Serialized proto strings: the rpc request argument.
-// If this tensor has more than 1 element, then multiple parallel rpc requests
-// are sent.  This argument broadcasts with `address` and `method`.
+//	handle: The handle to a queue.
+//	components: One or more tensors from which the enqueued tensors should be taken.
 //
-// Returns Same shape as `request`. Serialized proto strings: the rpc responses.
-func Rpc(scope *Scope, address tf.Output, method tf.Output, request tf.Output, optional ...RpcAttr) (response tf.Output) {
+// Returns the created operation.
+func QueueEnqueueV2(scope *Scope, handle tf.Output, components []tf.Output, optional ...QueueEnqueueV2Attr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -34612,305 +34536,281 @@ func Rpc(scope *Scope, address tf.Output, method tf.Output, request tf.Output, o
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Rpc",
+		Type: "QueueEnqueueV2",
 		Input: []tf.Input{
-			address, method, request,
+			handle, tf.OutputList(components),
 		},
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// Records the bytes size of each element of `input_dataset` in a StatsAggregator.
-func ExperimentalBytesProducedStatsDataset(scope *Scope, input_dataset tf.Output, tag tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+// Computes the Bessel i0e function of `x` element-wise.
+//
+// Exponentially scaled modified Bessel function of order 0 defined as
+// `bessel_i0e(x) = exp(-abs(x)) bessel_i0(x)`.
+//
+// This function is faster and numerically stabler than `bessel_i0(x)`.
+func BesselI0e(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "ExperimentalBytesProducedStatsDataset",
+		Type: "BesselI0e",
 		Input: []tf.Input{
-			input_dataset, tag,
+			x,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// StackPushV2Attr is an optional argument to StackPushV2.
-type StackPushV2Attr func(optionalAttr)
+// QueueDequeueManyV2Attr is an optional argument to QueueDequeueManyV2.
+type QueueDequeueManyV2Attr func(optionalAttr)
 
-// StackPushV2SwapMemory sets the optional swap_memory attribute to value.
+// QueueDequeueManyV2TimeoutMs sets the optional timeout_ms attribute to value.
 //
-// value: Swap `elem` to CPU. Default to false.
-// If not specified, defaults to false
-func StackPushV2SwapMemory(value bool) StackPushV2Attr {
+// value: If the queue has fewer than n elements, this operation
+// will block for up to timeout_ms milliseconds.
+// Note: This option is not supported yet.
+// If not specified, defaults to -1
+func QueueDequeueManyV2TimeoutMs(value int64) QueueDequeueManyV2Attr {
 	return func(m optionalAttr) {
-		m["swap_memory"] = value
+		m["timeout_ms"] = value
 	}
 }
 
-// Push an element onto the stack.
+// Dequeues `n` tuples of one or more tensors from the given queue.
+//
+// If the queue is closed and there are fewer than `n` elements, then an
+// OutOfRange error is returned.
+//
+// This operation concatenates queue-element component tensors along the
+// 0th dimension to make a single component tensor.  All of the components
+// in the dequeued tuple will have size `n` in the 0th dimension.
+//
+// This operation has `k` outputs, where `k` is the number of components in
+// the tuples stored in the given queue, and output `i` is the ith
+// component of the dequeued tuple.
+//
+// N.B. If the queue is empty, this operation will block until `n` elements
+// have been dequeued (or 'timeout_ms' elapses, if specified).
 //
 // Arguments:
-//	handle: The handle to a stack.
-//	elem: The tensor to be pushed onto the stack.
+//	handle: The handle to a queue.
+//	n: The number of tuples to dequeue.
+//	component_types: The type of each component in a tuple.
 //
-// Returns The same tensor as the input 'elem'.
-func StackPushV2(scope *Scope, handle tf.Output, elem tf.Output, optional ...StackPushV2Attr) (output tf.Output) {
+// Returns One or more tensors that were dequeued as a tuple.
+func QueueDequeueManyV2(scope *Scope, handle tf.Output, n tf.Output, component_types []tf.DataType, optional ...QueueDequeueManyV2Attr) (components []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"component_types": component_types}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "StackPushV2",
+		Type: "QueueDequeueManyV2",
 		Input: []tf.Input{
-			handle, elem,
+			handle, n,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// FusedBatchNormGradV2Attr is an optional argument to FusedBatchNormGradV2.
-type FusedBatchNormGradV2Attr func(optionalAttr)
-
-// FusedBatchNormGradV2Epsilon sets the optional epsilon attribute to value.
-//
-// value: A small float number added to the variance of x.
-// If not specified, defaults to 0.0001
-func FusedBatchNormGradV2Epsilon(value float32) FusedBatchNormGradV2Attr {
-	return func(m optionalAttr) {
-		m["epsilon"] = value
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// FusedBatchNormGradV2DataFormat sets the optional data_format attribute to value.
-//
-// value: The data format for y_backprop, x, x_backprop.
-// Either "NHWC" (default) or "NCHW".
-// If not specified, defaults to "NHWC"
-func FusedBatchNormGradV2DataFormat(value string) FusedBatchNormGradV2Attr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
+	var idx int
+	var err error
+	if components, idx, err = makeOutputList(op, idx, "components"); err != nil {
+		scope.UpdateErr("QueueDequeueManyV2", err)
+		return
 	}
+	return components
 }
 
-// FusedBatchNormGradV2IsTraining sets the optional is_training attribute to value.
+// Forwards the value of an available tensor from `inputs` to `output`.
 //
-// value: A bool value to indicate the operation is for training (default)
-// or inference.
-// If not specified, defaults to true
-func FusedBatchNormGradV2IsTraining(value bool) FusedBatchNormGradV2Attr {
-	return func(m optionalAttr) {
-		m["is_training"] = value
+// `Merge` waits for at least one of the tensors in `inputs` to become available.
+// It is usually combined with `Switch` to implement branching.
+//
+// `Merge` forwards the first tensor to become available to `output`, and sets
+// `value_index` to its index in `inputs`.
+//
+// Arguments:
+//	inputs: The input tensors, exactly one of which will become available.
+//
+// Returns Will be set to the available input tensor.The index of the chosen input tensor in `inputs`.
+func Merge(scope *Scope, inputs []tf.Output) (output tf.Output, value_index tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Merge",
+		Input: []tf.Input{
+			tf.OutputList(inputs),
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
 }
 
-// Gradient for batch normalization.
-//
-// Note that the size of 4D Tensors are defined by either "NHWC" or "NCHW".
-// The size of 1D Tensors matches the dimension C of the 4D Tensors.
+// Writes the given dataset to the given file using the TFRecord format.
 //
 // Arguments:
-//	y_backprop: A 4D Tensor for the gradient with respect to y.
-//	x: A 4D Tensor for input data.
-//	scale: A 1D Tensor for scaling factor, to scale the normalized x.
-//	reserve_space_1: When is_training is True, a 1D Tensor for the computed batch
-// mean to be reused in gradient computation. When is_training is
-// False, a 1D Tensor for the population mean to be reused in both
-// 1st and 2nd order gradient computation.
-//	reserve_space_2: When is_training is True, a 1D Tensor for the computed batch
-// variance (inverted variance in the cuDNN case) to be reused in
-// gradient computation. When is_training is False, a 1D Tensor
-// for the population variance to be reused in both 1st and 2nd
-// order gradient computation.
+//	input_dataset: A variant tensor representing the dataset to write.
+//	filename: A scalar string tensor representing the filename to use.
+//	compression_type: A scalar string tensor containing either (i) the empty string (no
+// compression), (ii) "ZLIB", or (iii) "GZIP".
 //
-// Returns A 4D Tensor for the gradient with respect to x.A 1D Tensor for the gradient with respect to scale.A 1D Tensor for the gradient with respect to offset.Unused placeholder to match the mean input in FusedBatchNorm.Unused placeholder to match the variance input
-// in FusedBatchNorm.
-func FusedBatchNormGradV2(scope *Scope, y_backprop tf.Output, x tf.Output, scale tf.Output, reserve_space_1 tf.Output, reserve_space_2 tf.Output, optional ...FusedBatchNormGradV2Attr) (x_backprop tf.Output, scale_backprop tf.Output, offset_backprop tf.Output, reserve_space_3 tf.Output, reserve_space_4 tf.Output) {
+// Returns the created operation.
+func ExperimentalDatasetToTFRecord(scope *Scope, input_dataset tf.Output, filename tf.Output, compression_type tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "FusedBatchNormGradV2",
+		Type: "ExperimentalDatasetToTFRecord",
 		Input: []tf.Input{
-			y_backprop, x, scale, reserve_space_1, reserve_space_2,
+			input_dataset, filename, compression_type,
 		},
-		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4)
+	return scope.AddOperation(opspec)
+}
+
+// QueueCloseV2Attr is an optional argument to QueueCloseV2.
+type QueueCloseV2Attr func(optionalAttr)
+
+// QueueCloseV2CancelPendingEnqueues sets the optional cancel_pending_enqueues attribute to value.
+//
+// value: If true, all pending enqueue requests that are
+// blocked on the given queue will be canceled.
+// If not specified, defaults to false
+func QueueCloseV2CancelPendingEnqueues(value bool) QueueCloseV2Attr {
+	return func(m optionalAttr) {
+		m["cancel_pending_enqueues"] = value
+	}
 }
 
-// Creates a TensorArray for storing multiple gradients of values in the given handle.
+// Closes the given queue.
 //
-// Similar to TensorArrayGradV3. However it creates an accumulator with an
-// expanded shape compared to the input TensorArray whose gradient is being
-// computed. This enables multiple gradients for the same TensorArray to be
-// calculated using the same accumulator.
+// This operation signals that no more elements will be enqueued in the
+// given queue. Subsequent Enqueue(Many) operations will fail.
+// Subsequent Dequeue(Many) operations will continue to succeed if
+// sufficient elements remain in the queue. Subsequent Dequeue(Many)
+// operations that would block will fail immediately.
 //
 // Arguments:
-//	handle: The handle to the forward TensorArray.
-//	flow_in: A float scalar that enforces proper chaining of operations.
-//	shape_to_prepend: An int32 vector representing a shape. Elements in the gradient accumulator will
-// have shape which is this shape_to_prepend value concatenated with shape of the
-// elements in the TensorArray corresponding to the input handle.
-//	source: The gradient source string, used to decide which gradient TensorArray
-// to return.
-func TensorArrayGradWithShape(scope *Scope, handle tf.Output, flow_in tf.Output, shape_to_prepend tf.Output, source string) (grad_handle tf.Output, flow_out tf.Output) {
+//	handle: The handle to a queue.
+//
+// Returns the created operation.
+func QueueCloseV2(scope *Scope, handle tf.Output, optional ...QueueCloseV2Attr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"source": source}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "TensorArrayGradWithShape",
+		Type: "QueueCloseV2",
 		Input: []tf.Input{
-			handle, flow_in, shape_to_prepend,
+			handle,
 		},
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return scope.AddOperation(opspec)
 }
 
-// Compare values of `input` to `threshold` and pack resulting bits into a `uint8`.
-//
-// Each comparison returns a boolean `true` (if `input_value > threshold`)
-// or and `false` otherwise.
-//
-// This operation is useful for Locality-Sensitive-Hashing (LSH) and other
-// algorithms that use hashing approximations of cosine and `L2` distances;
-// codes can be generated from an input via:
-//
-// ```python
-// codebook_size = 50
-// codebook_bits = codebook_size * 32
-// codebook = tf.get_variable('codebook', [x.shape[-1].value, codebook_bits],
-//                            dtype=x.dtype,
-//                            initializer=tf.orthogonal_initializer())
-// codes = compare_and_threshold(tf.matmul(x, codebook), threshold=0.)
-// codes = tf.bitcast(codes, tf.int32)  # go from uint8 to int32
-// # now codes has shape x.shape[:-1] + [codebook_size]
-// ```
-//
-// **NOTE**: Currently, the innermost dimension of the tensor must be divisible
-// by 8.
-//
-// Given an `input` shaped `[s0, s1, ..., s_n]`, the output is
-// a `uint8` tensor shaped `[s0, s1, ..., s_n / 8]`.
-//
-// Arguments:
-//	input: Values to compare against `threshold` and bitpack.
-//	threshold: Threshold to compare against.
-//
-// Returns The bitpacked comparisons.
-func CompareAndBitpack(scope *Scope, input tf.Output, threshold tf.Output) (output tf.Output) {
+// Computes inverse hyperbolic tangent of x element-wise.
+func Atanh(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "CompareAndBitpack",
+		Type: "Atanh",
 		Input: []tf.Input{
-			input, threshold,
+			x,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Push an element onto the tensor_array.
+// Returns true if queue is closed.
 //
-// Arguments:
-//	handle: The handle to a TensorArray.
-//	index: The position to write to inside the TensorArray.
-//	value: The tensor to write to the TensorArray.
-//	flow_in: A float scalar that enforces proper chaining of operations.
+// This operation returns true if the queue is closed and false if the queue
+// is open.
 //
-// Returns A float scalar that enforces proper chaining of operations.
-func TensorArrayWriteV3(scope *Scope, handle tf.Output, index tf.Output, value tf.Output, flow_in tf.Output) (flow_out tf.Output) {
+// Arguments:
+//	handle: The handle to a queue.
+func QueueIsClosedV2(scope *Scope, handle tf.Output) (is_closed tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "TensorArrayWriteV3",
+		Type: "QueueIsClosedV2",
 		Input: []tf.Input{
-			handle, index, value, flow_in,
+			handle,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Scatter the data from the input value into specific TensorArray elements.
-//
-// `indices` must be a vector, its length must match the first dim of `value`.
-//
-// Arguments:
-//	handle: The handle to a TensorArray.
-//	indices: The locations at which to write the tensor elements.
-//	value: The concatenated tensor to write to the TensorArray.
-//	flow_in: A float scalar that enforces proper chaining of operations.
+// Computes the absolute value of a tensor.
 //
-// Returns A float scalar that enforces proper chaining of operations.
-func TensorArrayScatterV3(scope *Scope, handle tf.Output, indices tf.Output, value tf.Output, flow_in tf.Output) (flow_out tf.Output) {
+// Given a tensor `x`, this operation returns a tensor containing the absolute
+// value of each element in `x`. For example, if x is an input element and y is
+// an output element, this operation computes \\(y = |x|\\).
+func Abs(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "TensorArrayScatterV3",
+		Type: "Abs",
 		Input: []tf.Input{
-			handle, indices, value, flow_in,
+			x,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// EmptyAttr is an optional argument to Empty.
-type EmptyAttr func(optionalAttr)
+// StackV2Attr is an optional argument to StackV2.
+type StackV2Attr func(optionalAttr)
 
-// EmptyInit sets the optional init attribute to value.
+// StackV2StackName sets the optional stack_name attribute to value.
 //
-// value: If True, initialize the returned tensor with the default value of dtype.  Otherwise, the implementation is free not to initializethe tensor's content.
-// If not specified, defaults to false
-func EmptyInit(value bool) EmptyAttr {
+// value: Overrides the name used for the temporary stack resource. Default
+// value is the name of the 'Stack' op (which is guaranteed unique).
+// If not specified, defaults to ""
+func StackV2StackName(value string) StackV2Attr {
 	return func(m optionalAttr) {
-		m["init"] = value
+		m["stack_name"] = value
 	}
 }
 
-// Creates a tensor with the given shape.
-//
-// This operation creates a tensor of `shape` and `dtype`.
+// A stack that produces elements in first-in last-out order.
 //
 // Arguments:
-//	shape: 1-D. Represents the shape of the output tensor.
-//
+//	max_size: The maximum size of the stack if non-negative. If negative, the stack
+// size is unlimited.
+//	elem_type: The type of the elements on the stack.
 //
-// Returns A `Tensor` of type `T`.
-func Empty(scope *Scope, shape tf.Output, dtype tf.DataType, optional ...EmptyAttr) (output tf.Output) {
+// Returns The handle to the stack.
+func StackV2(scope *Scope, max_size tf.Output, elem_type tf.DataType, optional ...StackV2Attr) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
+	attrs := map[string]interface{}{"elem_type": elem_type}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Empty",
+		Type: "StackV2",
 		Input: []tf.Input{
-			shape,
+			max_size,
 		},
 		Attrs: attrs,
 	}
@@ -34918,161 +34818,212 @@ func Empty(scope *Scope, shape tf.Output, dtype tf.DataType, optional ...EmptyAt
 	return op.Output(0)
 }
 
-// TensorArrayConcatV3Attr is an optional argument to TensorArrayConcatV3.
-type TensorArrayConcatV3Attr func(optionalAttr)
+// OrderedMapStageAttr is an optional argument to OrderedMapStage.
+type OrderedMapStageAttr func(optionalAttr)
 
-// TensorArrayConcatV3ElementShapeExcept0 sets the optional element_shape_except0 attribute to value.
+// OrderedMapStageCapacity sets the optional capacity attribute to value.
 //
-// value: The expected shape of an element, if known,
-// excluding the first dimension. Used to validate the shapes of
-// TensorArray elements. If this shape is not fully specified, concatenating
-// zero-size TensorArrays is an error.
-// If not specified, defaults to <unknown_rank:true >
-func TensorArrayConcatV3ElementShapeExcept0(value tf.Shape) TensorArrayConcatV3Attr {
+// value: Maximum number of elements in the Staging Area. If > 0, inserts
+// on the container will block when the capacity is reached.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func OrderedMapStageCapacity(value int64) OrderedMapStageAttr {
 	return func(m optionalAttr) {
-		m["element_shape_except0"] = value
+		m["capacity"] = value
 	}
 }
 
-// Concat the elements from the TensorArray into value `value`.
-//
-// Takes `T` elements of shapes
+// OrderedMapStageMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
 //
-//   ```
-//   (n0 x d0 x d1 x ...), (n1 x d0 x d1 x ...), ..., (n(T-1) x d0 x d1 x ...)
-//   ```
+// REQUIRES: value >= 0
+func OrderedMapStageMemoryLimit(value int64) OrderedMapStageAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// OrderedMapStageContainer sets the optional container attribute to value.
 //
-// and concatenates them into a Tensor of shape:
+// value: If non-empty, this queue is placed in the given container. Otherwise,
+// a default container is used.
+// If not specified, defaults to ""
+func OrderedMapStageContainer(value string) OrderedMapStageAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// OrderedMapStageSharedName sets the optional shared_name attribute to value.
 //
-//   ```(n0 + n1 + ... + n(T-1) x d0 x d1 x ...)```
+// value: It is necessary to match this name to the matching Unstage Op.
+// If not specified, defaults to ""
+func OrderedMapStageSharedName(value string) OrderedMapStageAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Stage (key, values) in the underlying container which behaves like a ordered
 //
-// All elements must have the same shape (excepting the first dimension).
+// associative container.   Elements are ordered by key.
 //
 // Arguments:
-//	handle: The handle to a TensorArray.
-//	flow_in: A float scalar that enforces proper chaining of operations.
-//	dtype: The type of the elem that is returned.
+//	key: int64
 //
-// Returns All of the elements in the TensorArray, concatenated along the first
-// axis.A vector of the row sizes of the original T elements in the
-// value output.  In the example above, this would be the values:
-// `(n1, n2, ..., n(T-1))`.
-func TensorArrayConcatV3(scope *Scope, handle tf.Output, flow_in tf.Output, dtype tf.DataType, optional ...TensorArrayConcatV3Attr) (value tf.Output, lengths tf.Output) {
+//	values: a list of tensors
+// dtypes A list of data types that inserted values should adhere to.
+//
+//
+// Returns the created operation.
+func OrderedMapStage(scope *Scope, key tf.Output, indices tf.Output, values []tf.Output, dtypes []tf.DataType, optional ...OrderedMapStageAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
+	attrs := map[string]interface{}{"dtypes": dtypes}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "TensorArrayConcatV3",
+		Type: "OrderedMapStage",
 		Input: []tf.Input{
-			handle, flow_in,
+			key, indices, tf.OutputList(values),
 		},
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return scope.AddOperation(opspec)
+}
+
+// RpcAttr is an optional argument to Rpc.
+type RpcAttr func(optionalAttr)
+
+// RpcProtocol sets the optional protocol attribute to value.
+//
+// value: RPC protocol to use.  Empty string means use the default protocol.
+// Options include 'grpc'.
+// If not specified, defaults to ""
+func RpcProtocol(value string) RpcAttr {
+	return func(m optionalAttr) {
+		m["protocol"] = value
+	}
+}
+
+// RpcFailFast sets the optional fail_fast attribute to value.
+//
+// value: `boolean`. If `true` (default), then failures to connect
+// (i.e., the server does not immediately respond) cause an RPC failure.
+// If not specified, defaults to true
+func RpcFailFast(value bool) RpcAttr {
+	return func(m optionalAttr) {
+		m["fail_fast"] = value
+	}
 }
 
-// Split the data from the input value into TensorArray elements.
+// RpcTimeoutInMs sets the optional timeout_in_ms attribute to value.
 //
-// Assuming that `lengths` takes on values
+// value: `int`. If `0` (default), then the kernel will run the RPC
+// request and only time out if the RPC deadline passes or the session times out.
+// If this value is greater than `0`, then the op will raise an exception if
+// the RPC takes longer than `timeout_in_ms`.
+// If not specified, defaults to 0
+func RpcTimeoutInMs(value int64) RpcAttr {
+	return func(m optionalAttr) {
+		m["timeout_in_ms"] = value
+	}
+}
+
+// Perform batches of RPC requests.
 //
-//   ```(n0, n1, ..., n(T-1))```
+// This op asynchronously performs either a single RPC request, or a batch
+// of requests.  RPC requests are defined by three main parameters:
 //
-// and that `value` has shape
+//   - `address` (the host+port or BNS address of the request)
+//   - `method` (the RPC method name for the request)
+//   - `request` (the serialized proto string, or vector of strings,
+//      of the RPC request argument).
 //
-//   ```(n0 + n1 + ... + n(T-1) x d0 x d1 x ...)```,
+// For example, if you have an RPC service running on port localhost:2345,
+// and its interface is configured with the following proto declaration:
 //
-// this splits values into a TensorArray with T tensors.
+// ```
+// service MyService {
+//   rpc MyMethod(MyRequestProto) returns (MyResponseProto) {
+//   }
+// };
+// ```
 //
-// TensorArray index t will be the subtensor of values with starting position
+// then call this op with arguments:
 //
-//   ```(n0 + n1 + ... + n(t-1), 0, 0, ...)```
+// ```
+// address = "localhost:2345"
+// method = "MyService/MyMethod"
+// ```
 //
-// and having size
+// The `request` tensor is a string tensor representing serialized `MyRequestProto`
+// strings; and the output string tensor `response` will have the same shape
+// and contain (upon successful completion) corresponding serialized
+// `MyResponseProto` strings.
 //
-//   ```nt x d0 x d1 x ...```
+// For example, to send a single, empty, `MyRequestProto`, call
+// this op with `request = ""`.  To send 5 **parallel** empty requests,
+// call this op with `request = ["", "", "", "", ""]`.
 //
-// Arguments:
-//	handle: The handle to a TensorArray.
-//	value: The concatenated tensor to write to the TensorArray.
-//	lengths: The vector of lengths, how to split the rows of value into the
-// TensorArray.
-//	flow_in: A float scalar that enforces proper chaining of operations.
+// More generally, one can create a batch of `MyRequestProto` serialized protos
+// from regular batched tensors using the `encode_proto` op, and convert
+// the response `MyResponseProto` serialized protos to batched tensors
+// using the `decode_proto` op.
 //
-// Returns A float scalar that enforces proper chaining of operations.
-func TensorArraySplitV3(scope *Scope, handle tf.Output, value tf.Output, lengths tf.Output, flow_in tf.Output) (flow_out tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "TensorArraySplitV3",
-		Input: []tf.Input{
-			handle, value, lengths, flow_in,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes gradients for the scaled exponential linear (Selu) operation.
+// **NOTE** Working with serialized proto strings is faster than instantiating
+// actual proto objects in memory, so no performance degradation is expected
+// compared to writing custom kernels for this workflow.
 //
-// Arguments:
-//	gradients: The backpropagated gradients to the corresponding Selu operation.
-//	outputs: The outputs of the corresponding Selu operation.
+// If the connection fails or the remote worker returns an error
+// status, the op reraises this exception locally.
 //
-// Returns The gradients: `gradients * (outputs + scale * alpha)`
-// if outputs < 0, `scale * gradients` otherwise.
-func SeluGrad(scope *Scope, gradients tf.Output, outputs tf.Output) (backprops tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "SeluGrad",
-		Input: []tf.Input{
-			gradients, outputs,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Get the current size of the TensorArray.
+// See the `TryRpc` op if you prefer to handle RPC failures manually in the graph.
 //
 // Arguments:
-//	handle: The handle to a TensorArray (output of TensorArray or TensorArrayGrad).
-//	flow_in: A float scalar that enforces proper chaining of operations.
+//	address: `0-D` or `1-D`.  The address (i.e. host_name:port) of the RPC server.
+// If this tensor has more than 1 element, then multiple parallel rpc requests
+// are sent.  This argument broadcasts with `method` and `request`.
+//	method: `0-D` or `1-D`.  The method address on the RPC server.
+// If this tensor has more than 1 element, then multiple parallel rpc requests
+// are sent.  This argument broadcasts with `address` and `request`.
+//	request: `0-D` or `1-D`.  Serialized proto strings: the rpc request argument.
+// If this tensor has more than 1 element, then multiple parallel rpc requests
+// are sent.  This argument broadcasts with `address` and `method`.
 //
-// Returns The current size of the TensorArray.
-func TensorArraySizeV3(scope *Scope, handle tf.Output, flow_in tf.Output) (size tf.Output) {
+// Returns Same shape as `request`. Serialized proto strings: the rpc responses.
+func Rpc(scope *Scope, address tf.Output, method tf.Output, request tf.Output, optional ...RpcAttr) (response tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "TensorArraySizeV3",
+		Type: "Rpc",
 		Input: []tf.Input{
-			handle, flow_in,
+			address, method, request,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Deprecated. Use TensorArrayGradV3
-//
-// DEPRECATED at GraphDef version 26: Use TensorArrayGradV3
-func TensorArrayGradV2(scope *Scope, handle tf.Output, flow_in tf.Output, source string) (grad_handle tf.Output) {
+// Records the bytes size of each element of `input_dataset` in a StatsAggregator.
+func ExperimentalBytesProducedStatsDataset(scope *Scope, input_dataset tf.Output, tag tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"source": source}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "TensorArrayGradV2",
+		Type: "ExperimentalBytesProducedStatsDataset",
 		Input: []tf.Input{
-			handle, flow_in,
+			input_dataset, tag,
 		},
 		Attrs: attrs,
 	}
@@ -35080,43 +35031,27 @@ func TensorArrayGradV2(scope *Scope, handle tf.Output, flow_in tf.Output, source
 	return op.Output(0)
 }
 
-// SparseReduceMaxAttr is an optional argument to SparseReduceMax.
-type SparseReduceMaxAttr func(optionalAttr)
+// StackPushV2Attr is an optional argument to StackPushV2.
+type StackPushV2Attr func(optionalAttr)
 
-// SparseReduceMaxKeepDims sets the optional keep_dims attribute to value.
+// StackPushV2SwapMemory sets the optional swap_memory attribute to value.
 //
-// value: If true, retain reduced dimensions with length 1.
+// value: Swap `elem` to CPU. Default to false.
 // If not specified, defaults to false
-func SparseReduceMaxKeepDims(value bool) SparseReduceMaxAttr {
+func StackPushV2SwapMemory(value bool) StackPushV2Attr {
 	return func(m optionalAttr) {
-		m["keep_dims"] = value
+		m["swap_memory"] = value
 	}
 }
 
-// Computes the max of elements across dimensions of a SparseTensor.
-//
-// This Op takes a SparseTensor and is the sparse counterpart to
-// `tf.reduce_max()`.  In particular, this Op also returns a dense `Tensor`
-// instead of a sparse one.
-//
-// Reduces `sp_input` along the dimensions given in `reduction_axes`.  Unless
-// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
-// `reduction_axes`. If `keep_dims` is true, the reduced dimensions are retained
-// with length 1.
-//
-// If `reduction_axes` has no entries, all dimensions are reduced, and a tensor
-// with a single element is returned.  Additionally, the axes can be negative,
-// which are interpreted according to the indexing rules in Python.
+// Push an element onto the stack.
 //
 // Arguments:
-//	input_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
-// SparseTensor, possibly not in canonical ordering.
-//	input_values: 1-D.  `N` non-empty values corresponding to `input_indices`.
-//	input_shape: 1-D.  Shape of the input SparseTensor.
-//	reduction_axes: 1-D.  Length-`K` vector containing the reduction axes.
+//	handle: The handle to a stack.
+//	elem: The tensor to be pushed onto the stack.
 //
-// Returns `R-K`-D.  The reduced Tensor.
-func SparseReduceMax(scope *Scope, input_indices tf.Output, input_values tf.Output, input_shape tf.Output, reduction_axes tf.Output, optional ...SparseReduceMaxAttr) (output tf.Output) {
+// Returns The same tensor as the input 'elem'.
+func StackPushV2(scope *Scope, handle tf.Output, elem tf.Output, optional ...StackPushV2Attr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -35125,9 +35060,9 @@ func SparseReduceMax(scope *Scope, input_indices tf.Output, input_values tf.Outp
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseReduceMax",
+		Type: "StackPushV2",
 		Input: []tf.Input{
-			input_indices, input_values, input_shape, reduction_axes,
+			handle, elem,
 		},
 		Attrs: attrs,
 	}
@@ -35135,68 +35070,63 @@ func SparseReduceMax(scope *Scope, input_indices tf.Output, input_values tf.Outp
 	return op.Output(0)
 }
 
-// AsStringAttr is an optional argument to AsString.
-type AsStringAttr func(optionalAttr)
+// FusedBatchNormGradV2Attr is an optional argument to FusedBatchNormGradV2.
+type FusedBatchNormGradV2Attr func(optionalAttr)
 
-// AsStringPrecision sets the optional precision attribute to value.
+// FusedBatchNormGradV2Epsilon sets the optional epsilon attribute to value.
 //
-// value: The post-decimal precision to use for floating point numbers.
-// Only used if precision > -1.
-// If not specified, defaults to -1
-func AsStringPrecision(value int64) AsStringAttr {
+// value: A small float number added to the variance of x.
+// If not specified, defaults to 0.0001
+func FusedBatchNormGradV2Epsilon(value float32) FusedBatchNormGradV2Attr {
 	return func(m optionalAttr) {
-		m["precision"] = value
+		m["epsilon"] = value
 	}
 }
 
-// AsStringScientific sets the optional scientific attribute to value.
+// FusedBatchNormGradV2DataFormat sets the optional data_format attribute to value.
 //
-// value: Use scientific notation for floating point numbers.
-// If not specified, defaults to false
-func AsStringScientific(value bool) AsStringAttr {
+// value: The data format for y_backprop, x, x_backprop.
+// Either "NHWC" (default) or "NCHW".
+// If not specified, defaults to "NHWC"
+func FusedBatchNormGradV2DataFormat(value string) FusedBatchNormGradV2Attr {
 	return func(m optionalAttr) {
-		m["scientific"] = value
+		m["data_format"] = value
 	}
 }
 
-// AsStringShortest sets the optional shortest attribute to value.
+// FusedBatchNormGradV2IsTraining sets the optional is_training attribute to value.
 //
-// value: Use shortest representation (either scientific or standard) for
-// floating point numbers.
-// If not specified, defaults to false
-func AsStringShortest(value bool) AsStringAttr {
+// value: A bool value to indicate the operation is for training (default)
+// or inference.
+// If not specified, defaults to true
+func FusedBatchNormGradV2IsTraining(value bool) FusedBatchNormGradV2Attr {
 	return func(m optionalAttr) {
-		m["shortest"] = value
+		m["is_training"] = value
 	}
 }
 
-// AsStringWidth sets the optional width attribute to value.
+// Gradient for batch normalization.
 //
-// value: Pad pre-decimal numbers to this width.
-// Applies to both floating point and integer numbers.
-// Only used if width > -1.
-// If not specified, defaults to -1
-func AsStringWidth(value int64) AsStringAttr {
-	return func(m optionalAttr) {
-		m["width"] = value
-	}
-}
-
-// AsStringFill sets the optional fill attribute to value.
+// Note that the size of 4D Tensors are defined by either "NHWC" or "NCHW".
+// The size of 1D Tensors matches the dimension C of the 4D Tensors.
 //
-// value: The value to pad if width > -1.  If empty, pads with spaces.
-// Another typical value is '0'.  String cannot be longer than 1 character.
-// If not specified, defaults to ""
-func AsStringFill(value string) AsStringAttr {
-	return func(m optionalAttr) {
-		m["fill"] = value
-	}
-}
-
-// Converts each entry in the given tensor to strings.  Supports many numeric
+// Arguments:
+//	y_backprop: A 4D Tensor for the gradient with respect to y.
+//	x: A 4D Tensor for input data.
+//	scale: A 1D Tensor for scaling factor, to scale the normalized x.
+//	reserve_space_1: When is_training is True, a 1D Tensor for the computed batch
+// mean to be reused in gradient computation. When is_training is
+// False, a 1D Tensor for the population mean to be reused in both
+// 1st and 2nd order gradient computation.
+//	reserve_space_2: When is_training is True, a 1D Tensor for the computed batch
+// variance (inverted variance in the cuDNN case) to be reused in
+// gradient computation. When is_training is False, a 1D Tensor
+// for the population variance to be reused in both 1st and 2nd
+// order gradient computation.
 //
-// types and boolean.
-func AsString(scope *Scope, input tf.Output, optional ...AsStringAttr) (output tf.Output) {
+// Returns A 4D Tensor for the gradient with respect to x.A 1D Tensor for the gradient with respect to scale.A 1D Tensor for the gradient with respect to offset.Unused placeholder to match the mean input in FusedBatchNorm.Unused placeholder to match the variance input
+// in FusedBatchNorm.
+func FusedBatchNormGradV2(scope *Scope, y_backprop tf.Output, x tf.Output, scale tf.Output, reserve_space_1 tf.Output, reserve_space_2 tf.Output, optional ...FusedBatchNormGradV2Attr) (x_backprop tf.Output, scale_backprop tf.Output, offset_backprop tf.Output, reserve_space_3 tf.Output, reserve_space_4 tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -35205,366 +35135,436 @@ func AsString(scope *Scope, input tf.Output, optional ...AsStringAttr) (output t
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "AsString",
+		Type: "FusedBatchNormGradV2",
 		Input: []tf.Input{
-			input,
+			y_backprop, x, scale, reserve_space_1, reserve_space_2,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4)
 }
 
-// Deprecated. Use TensorArrayScatterV3
+// Creates a TensorArray for storing multiple gradients of values in the given handle.
 //
-// DEPRECATED at GraphDef version 26: Use TensorArrayScatterV3
-func TensorArrayScatterV2(scope *Scope, handle tf.Output, indices tf.Output, value tf.Output, flow_in tf.Output) (flow_out tf.Output) {
+// Similar to TensorArrayGradV3. However it creates an accumulator with an
+// expanded shape compared to the input TensorArray whose gradient is being
+// computed. This enables multiple gradients for the same TensorArray to be
+// calculated using the same accumulator.
+//
+// Arguments:
+//	handle: The handle to the forward TensorArray.
+//	flow_in: A float scalar that enforces proper chaining of operations.
+//	shape_to_prepend: An int32 vector representing a shape. Elements in the gradient accumulator will
+// have shape which is this shape_to_prepend value concatenated with shape of the
+// elements in the TensorArray corresponding to the input handle.
+//	source: The gradient source string, used to decide which gradient TensorArray
+// to return.
+func TensorArrayGradWithShape(scope *Scope, handle tf.Output, flow_in tf.Output, shape_to_prepend tf.Output, source string) (grad_handle tf.Output, flow_out tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"source": source}
 	opspec := tf.OpSpec{
-		Type: "TensorArrayScatterV2",
+		Type: "TensorArrayGradWithShape",
 		Input: []tf.Input{
-			handle, indices, value, flow_in,
+			handle, flow_in, shape_to_prepend,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1)
 }
 
-// Applies sparse addition to `input` using individual values or slices
-//
-// from `updates` according to indices `indices`.  The updates are non-aliasing:
-// `input` is only modified in-place if no other operations will use it.
-// Otherwise, a copy of `input` is made.  This operation has a gradient with
-// respect to both `input` and `updates`.
+// Compare values of `input` to `threshold` and pack resulting bits into a `uint8`.
 //
-// `input` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`.
+// Each comparison returns a boolean `true` (if `input_value > threshold`)
+// or and `false` otherwise.
 //
-// `indices` must be integer tensor, containing indices into `input`.
-// It must be shape \\([d_0, ..., d_{Q-2}, K]\\) where `0 < K <= P`.
+// This operation is useful for Locality-Sensitive-Hashing (LSH) and other
+// algorithms that use hashing approximations of cosine and `L2` distances;
+// codes can be generated from an input via:
 //
-// The innermost dimension of `indices` (with length `K`) corresponds to
-// indices into elements (if `K = P`) or `(P-K)`-dimensional slices
-// (if `K < P`) along the `K`th dimension of `input`.
+// ```python
+// codebook_size = 50
+// codebook_bits = codebook_size * 32
+// codebook = tf.get_variable('codebook', [x.shape[-1].value, codebook_bits],
+//                            dtype=x.dtype,
+//                            initializer=tf.orthogonal_initializer())
+// codes = compare_and_threshold(tf.matmul(x, codebook), threshold=0.)
+// codes = tf.bitcast(codes, tf.int32)  # go from uint8 to int32
+// # now codes has shape x.shape[:-1] + [codebook_size]
+// ```
 //
-// `updates` is `Tensor` of rank `Q-1+P-K` with shape:
+// **NOTE**: Currently, the innermost dimension of the tensor must be divisible
+// by 8.
 //
-// $$[d_0, ..., d_{Q-2}, input.shape[K], ..., input.shape[P-1]].$$
+// Given an `input` shaped `[s0, s1, ..., s_n]`, the output is
+// a `uint8` tensor shaped `[s0, s1, ..., s_n / 8]`.
 //
-// For example, say we want to add 4 scattered elements to a rank-1 tensor to 8
-// elements. In Python, that addition would look like this:
+// Arguments:
+//	input: Values to compare against `threshold` and bitpack.
+//	threshold: Threshold to compare against.
 //
-//     input = tf.constant([1, 2, 3, 4, 5, 6, 7, 8])
-//     indices = tf.constant([[4], [3], [1], [7]])
-//     updates = tf.constant([9, 10, 11, 12])
-//     output = tf.scatter_nd_non_aliasing_add(input, indices, updates)
-//     with tf.Session() as sess:
-//       print(sess.run(output))
+// Returns The bitpacked comparisons.
+func CompareAndBitpack(scope *Scope, input tf.Output, threshold tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "CompareAndBitpack",
+		Input: []tf.Input{
+			input, threshold,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Push an element onto the tensor_array.
 //
-// The resulting value `output` would look like this:
+// Arguments:
+//	handle: The handle to a TensorArray.
+//	index: The position to write to inside the TensorArray.
+//	value: The tensor to write to the TensorArray.
+//	flow_in: A float scalar that enforces proper chaining of operations.
 //
-//     [1, 13, 3, 14, 14, 6, 7, 20]
+// Returns A float scalar that enforces proper chaining of operations.
+func TensorArrayWriteV3(scope *Scope, handle tf.Output, index tf.Output, value tf.Output, flow_in tf.Output) (flow_out tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorArrayWriteV3",
+		Input: []tf.Input{
+			handle, index, value, flow_in,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Scatter the data from the input value into specific TensorArray elements.
 //
-// See `tf.scatter_nd` for more details about how to make updates to slices.
+// `indices` must be a vector, its length must match the first dim of `value`.
 //
 // Arguments:
-//	input: A Tensor.
-//	indices: A Tensor. Must be one of the following types: `int32`, `int64`.
-// A tensor of indices into `input`.
-//	updates: A Tensor. Must have the same type as ref. A tensor of updated values
-// to add to `input`.
+//	handle: The handle to a TensorArray.
+//	indices: The locations at which to write the tensor elements.
+//	value: The concatenated tensor to write to the TensorArray.
+//	flow_in: A float scalar that enforces proper chaining of operations.
 //
-// Returns A `Tensor` with the same shape as `input`, containing values of `input`
-// updated with `updates`.
-func ScatterNdNonAliasingAdd(scope *Scope, input tf.Output, indices tf.Output, updates tf.Output) (output tf.Output) {
+// Returns A float scalar that enforces proper chaining of operations.
+func TensorArrayScatterV3(scope *Scope, handle tf.Output, indices tf.Output, value tf.Output, flow_in tf.Output) (flow_out tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "ScatterNdNonAliasingAdd",
+		Type: "TensorArrayScatterV3",
 		Input: []tf.Input{
-			input, indices, updates,
+			handle, indices, value, flow_in,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// FractionalMaxPoolAttr is an optional argument to FractionalMaxPool.
-type FractionalMaxPoolAttr func(optionalAttr)
+// EmptyAttr is an optional argument to Empty.
+type EmptyAttr func(optionalAttr)
 
-// FractionalMaxPoolPseudoRandom sets the optional pseudo_random attribute to value.
+// EmptyInit sets the optional init attribute to value.
 //
-// value: When set to True, generates the pooling sequence in a
-// pseudorandom fashion, otherwise, in a random fashion. Check paper [Benjamin
-// Graham, Fractional Max-Pooling](http://arxiv.org/abs/1412.6071) for
-// difference between pseudorandom and random.
+// value: If True, initialize the returned tensor with the default value of dtype.  Otherwise, the implementation is free not to initializethe tensor's content.
 // If not specified, defaults to false
-func FractionalMaxPoolPseudoRandom(value bool) FractionalMaxPoolAttr {
+func EmptyInit(value bool) EmptyAttr {
 	return func(m optionalAttr) {
-		m["pseudo_random"] = value
+		m["init"] = value
 	}
 }
 
-// FractionalMaxPoolOverlapping sets the optional overlapping attribute to value.
+// Creates a tensor with the given shape.
 //
-// value: When set to True, it means when pooling, the values at the boundary
-// of adjacent pooling cells are used by both cells. For example:
+// This operation creates a tensor of `shape` and `dtype`.
 //
-// `index  0  1  2  3  4`
+// Arguments:
+//	shape: 1-D. Represents the shape of the output tensor.
 //
-// `value  20 5  16 3  7`
 //
-// If the pooling sequence is [0, 2, 4], then 16, at index 2 will be used twice.
-// The result would be [20, 16] for fractional max pooling.
-// If not specified, defaults to false
-func FractionalMaxPoolOverlapping(value bool) FractionalMaxPoolAttr {
-	return func(m optionalAttr) {
-		m["overlapping"] = value
+// Returns A `Tensor` of type `T`.
+func Empty(scope *Scope, shape tf.Output, dtype tf.DataType, optional ...EmptyAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtype": dtype}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "Empty",
+		Input: []tf.Input{
+			shape,
+		},
+		Attrs: attrs,
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// FractionalMaxPoolDeterministic sets the optional deterministic attribute to value.
+// TensorArrayConcatV3Attr is an optional argument to TensorArrayConcatV3.
+type TensorArrayConcatV3Attr func(optionalAttr)
+
+// TensorArrayConcatV3ElementShapeExcept0 sets the optional element_shape_except0 attribute to value.
 //
-// value: When set to True, a fixed pooling region will be used when
-// iterating over a FractionalMaxPool node in the computation graph. Mainly used
-// in unit test to make FractionalMaxPool deterministic.
-// If not specified, defaults to false
-func FractionalMaxPoolDeterministic(value bool) FractionalMaxPoolAttr {
+// value: The expected shape of an element, if known,
+// excluding the first dimension. Used to validate the shapes of
+// TensorArray elements. If this shape is not fully specified, concatenating
+// zero-size TensorArrays is an error.
+// If not specified, defaults to <unknown_rank:true >
+func TensorArrayConcatV3ElementShapeExcept0(value tf.Shape) TensorArrayConcatV3Attr {
 	return func(m optionalAttr) {
-		m["deterministic"] = value
+		m["element_shape_except0"] = value
 	}
 }
 
-// FractionalMaxPoolSeed sets the optional seed attribute to value.
+// Concat the elements from the TensorArray into value `value`.
 //
-// value: If either seed or seed2 are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func FractionalMaxPoolSeed(value int64) FractionalMaxPoolAttr {
-	return func(m optionalAttr) {
-		m["seed"] = value
+// Takes `T` elements of shapes
+//
+//   ```
+//   (n0 x d0 x d1 x ...), (n1 x d0 x d1 x ...), ..., (n(T-1) x d0 x d1 x ...)
+//   ```
+//
+// and concatenates them into a Tensor of shape:
+//
+//   ```(n0 + n1 + ... + n(T-1) x d0 x d1 x ...)```
+//
+// All elements must have the same shape (excepting the first dimension).
+//
+// Arguments:
+//	handle: The handle to a TensorArray.
+//	flow_in: A float scalar that enforces proper chaining of operations.
+//	dtype: The type of the elem that is returned.
+//
+// Returns All of the elements in the TensorArray, concatenated along the first
+// axis.A vector of the row sizes of the original T elements in the
+// value output.  In the example above, this would be the values:
+// `(n1, n2, ..., n(T-1))`.
+func TensorArrayConcatV3(scope *Scope, handle tf.Output, flow_in tf.Output, dtype tf.DataType, optional ...TensorArrayConcatV3Attr) (value tf.Output, lengths tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtype": dtype}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorArrayConcatV3",
+		Input: []tf.Input{
+			handle, flow_in,
+		},
+		Attrs: attrs,
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
 }
 
-// FractionalMaxPoolSeed2 sets the optional seed2 attribute to value.
+// Split the data from the input value into TensorArray elements.
 //
-// value: An second seed to avoid seed collision.
-// If not specified, defaults to 0
-func FractionalMaxPoolSeed2(value int64) FractionalMaxPoolAttr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
-	}
-}
-
-// Performs fractional max pooling on the input.
+// Assuming that `lengths` takes on values
 //
-// Fractional max pooling is slightly different than regular max pooling.  In
-// regular max pooling, you downsize an input set by taking the maximum value of
-// smaller N x N subsections of the set (often 2x2), and try to reduce the set by
-// a factor of N, where N is an integer.  Fractional max pooling, as you might
-// expect from the word "fractional", means that the overall reduction ratio N
-// does not have to be an integer.
+//   ```(n0, n1, ..., n(T-1))```
 //
-// The sizes of the pooling regions are generated randomly but are fairly uniform.
-// For example, let's look at the height dimension, and the constraints on the
-// list of rows that will be pool boundaries.
+// and that `value` has shape
 //
-// First we define the following:
+//   ```(n0 + n1 + ... + n(T-1) x d0 x d1 x ...)```,
 //
-// 1.  input_row_length : the number of rows from the input set
-// 2.  output_row_length : which will be smaller than the input
-// 3.  alpha = input_row_length / output_row_length : our reduction ratio
-// 4.  K = floor(alpha)
-// 5.  row_pooling_sequence : this is the result list of pool boundary rows
+// this splits values into a TensorArray with T tensors.
 //
-// Then, row_pooling_sequence should satisfy:
+// TensorArray index t will be the subtensor of values with starting position
 //
-// 1.  a[0] = 0 : the first value of the sequence is 0
-// 2.  a[end] = input_row_length : the last value of the sequence is the size
-// 3.  K <= (a[i+1] - a[i]) <= K+1 : all intervals are K or K+1 size
-// 4.  length(row_pooling_sequence) = output_row_length+1
+//   ```(n0 + n1 + ... + n(t-1), 0, 0, ...)```
 //
-// For more details on fractional max pooling, see this paper:
-// [Benjamin Graham, Fractional Max-Pooling](http://arxiv.org/abs/1412.6071)
+// and having size
+//
+//   ```nt x d0 x d1 x ...```
 //
 // Arguments:
-//	value: 4-D with shape `[batch, height, width, channels]`.
-//	pooling_ratio: Pooling ratio for each dimension of `value`, currently only
-// supports row and col dimension and should be >= 1.0. For example, a valid
-// pooling ratio looks like [1.0, 1.44, 1.73, 1.0]. The first and last elements
-// must be 1.0 because we don't allow pooling on batch and channels
-// dimensions. 1.44 and 1.73 are pooling ratio on height and width dimensions
-// respectively.
+//	handle: The handle to a TensorArray.
+//	value: The concatenated tensor to write to the TensorArray.
+//	lengths: The vector of lengths, how to split the rows of value into the
+// TensorArray.
+//	flow_in: A float scalar that enforces proper chaining of operations.
 //
-// Returns output tensor after fractional max pooling.row pooling sequence, needed to calculate gradient.column pooling sequence, needed to calculate gradient.
-func FractionalMaxPool(scope *Scope, value tf.Output, pooling_ratio []float32, optional ...FractionalMaxPoolAttr) (output tf.Output, row_pooling_sequence tf.Output, col_pooling_sequence tf.Output) {
+// Returns A float scalar that enforces proper chaining of operations.
+func TensorArraySplitV3(scope *Scope, handle tf.Output, value tf.Output, lengths tf.Output, flow_in tf.Output) (flow_out tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"pooling_ratio": pooling_ratio}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "FractionalMaxPool",
+		Type: "TensorArraySplitV3",
 		Input: []tf.Input{
-			value,
+			handle, value, lengths, flow_in,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// Creates a MultiDeviceIterator resource.
+// Computes gradients for the scaled exponential linear (Selu) operation.
 //
 // Arguments:
-//	devices: A list of devices the iterator works across.
-//	shared_name: If non-empty, this resource will be shared under the given name
-// across multiple sessions.
-//	container: If non-empty, this resource is placed in the given container.
-// Otherwise, a default container is used.
-//	output_types: The type list for the return values.
-//	output_shapes: The list of shapes being produced.
+//	gradients: The backpropagated gradients to the corresponding Selu operation.
+//	outputs: The outputs of the corresponding Selu operation.
 //
-// Returns Handle to the resource created.
-func MultiDeviceIterator(scope *Scope, devices []string, shared_name string, container string, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+// Returns The gradients: `gradients * (outputs + scale * alpha)`
+// if outputs < 0, `scale * gradients` otherwise.
+func SeluGrad(scope *Scope, gradients tf.Output, outputs tf.Output) (backprops tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"devices": devices, "shared_name": shared_name, "container": container, "output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "MultiDeviceIterator",
+		Type: "SeluGrad",
+		Input: []tf.Input{
+			gradients, outputs,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
 
-		Attrs: attrs,
+// Get the current size of the TensorArray.
+//
+// Arguments:
+//	handle: The handle to a TensorArray (output of TensorArray or TensorArrayGrad).
+//	flow_in: A float scalar that enforces proper chaining of operations.
+//
+// Returns The current size of the TensorArray.
+func TensorArraySizeV3(scope *Scope, handle tf.Output, flow_in tf.Output) (size tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorArraySizeV3",
+		Input: []tf.Input{
+			handle, flow_in,
+		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Deprecated. Use TensorArraySizeV3
+// Deprecated. Use TensorArrayGradV3
 //
-// DEPRECATED at GraphDef version 26: Use TensorArraySizeV3
-func TensorArraySizeV2(scope *Scope, handle tf.Output, flow_in tf.Output) (size tf.Output) {
+// DEPRECATED at GraphDef version 26: Use TensorArrayGradV3
+func TensorArrayGradV2(scope *Scope, handle tf.Output, flow_in tf.Output, source string) (grad_handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"source": source}
 	opspec := tf.OpSpec{
-		Type: "TensorArraySizeV2",
+		Type: "TensorArrayGradV2",
 		Input: []tf.Input{
 			handle, flow_in,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Conv2DAttr is an optional argument to Conv2D.
-type Conv2DAttr func(optionalAttr)
+// AsStringAttr is an optional argument to AsString.
+type AsStringAttr func(optionalAttr)
 
-// Conv2DUseCudnnOnGpu sets the optional use_cudnn_on_gpu attribute to value.
-// If not specified, defaults to true
-func Conv2DUseCudnnOnGpu(value bool) Conv2DAttr {
+// AsStringPrecision sets the optional precision attribute to value.
+//
+// value: The post-decimal precision to use for floating point numbers.
+// Only used if precision > -1.
+// If not specified, defaults to -1
+func AsStringPrecision(value int64) AsStringAttr {
 	return func(m optionalAttr) {
-		m["use_cudnn_on_gpu"] = value
+		m["precision"] = value
 	}
 }
 
-// Conv2DExplicitPaddings sets the optional explicit_paddings attribute to value.
+// AsStringScientific sets the optional scientific attribute to value.
 //
-// value: If `padding` is `"EXPLICIT"`, the list of explicit padding amounts. For the ith
-// dimension, the amount of padding inserted before and after the dimension is
-// `explicit_paddings[2 * i]` and `explicit_paddings[2 * i + 1]`, respectively. If
-// `padding` is not `"EXPLICIT"`, `explicit_paddings` must be empty.
-// If not specified, defaults to <>
-func Conv2DExplicitPaddings(value []int64) Conv2DAttr {
+// value: Use scientific notation for floating point numbers.
+// If not specified, defaults to false
+func AsStringScientific(value bool) AsStringAttr {
 	return func(m optionalAttr) {
-		m["explicit_paddings"] = value
+		m["scientific"] = value
 	}
 }
 
-// Conv2DDataFormat sets the optional data_format attribute to value.
+// AsStringShortest sets the optional shortest attribute to value.
 //
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the data is stored in the order of:
-//     [batch, height, width, channels].
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, channels, height, width].
-// If not specified, defaults to "NHWC"
-func Conv2DDataFormat(value string) Conv2DAttr {
+// value: Use shortest representation (either scientific or standard) for
+// floating point numbers.
+// If not specified, defaults to false
+func AsStringShortest(value bool) AsStringAttr {
 	return func(m optionalAttr) {
-		m["data_format"] = value
+		m["shortest"] = value
 	}
 }
 
-// Conv2DDilations sets the optional dilations attribute to value.
+// AsStringWidth sets the optional width attribute to value.
 //
-// value: 1-D tensor of length 4.  The dilation factor for each dimension of
-// `input`. If set to k > 1, there will be k-1 skipped cells between each
-// filter element on that dimension. The dimension order is determined by the
-// value of `data_format`, see above for details. Dilations in the batch and
-// depth dimensions must be 1.
-// If not specified, defaults to <i:1 i:1 i:1 i:1 >
-func Conv2DDilations(value []int64) Conv2DAttr {
+// value: Pad pre-decimal numbers to this width.
+// Applies to both floating point and integer numbers.
+// Only used if width > -1.
+// If not specified, defaults to -1
+func AsStringWidth(value int64) AsStringAttr {
 	return func(m optionalAttr) {
-		m["dilations"] = value
+		m["width"] = value
 	}
 }
 
-// Computes a 2-D convolution given 4-D `input` and `filter` tensors.
-//
-// Given an input tensor of shape `[batch, in_height, in_width, in_channels]`
-// and a filter / kernel tensor of shape
-// `[filter_height, filter_width, in_channels, out_channels]`, this op
-// performs the following:
-//
-// 1. Flattens the filter to a 2-D matrix with shape
-//    `[filter_height * filter_width * in_channels, output_channels]`.
-// 2. Extracts image patches from the input tensor to form a *virtual*
-//    tensor of shape `[batch, out_height, out_width,
-//    filter_height * filter_width * in_channels]`.
-// 3. For each patch, right-multiplies the filter matrix and the image patch
-//    vector.
-//
-// In detail, with the default NHWC format,
-//
-//     output[b, i, j, k] =
-//         sum_{di, dj, q} input[b, strides[1] * i + di, strides[2] * j + dj, q] *
-//                         filter[di, dj, q, k]
-//
-// Must have `strides[0] = strides[3] = 1`.  For the most common case of the same
-// horizontal and vertices strides, `strides = [1, stride, stride, 1]`.
+// AsStringFill sets the optional fill attribute to value.
 //
-// Arguments:
-//	input: A 4-D tensor. The dimension order is interpreted according to the value
-// of `data_format`, see below for details.
-//	filter: A 4-D tensor of shape
-// `[filter_height, filter_width, in_channels, out_channels]`
-//	strides: 1-D tensor of length 4.  The stride of the sliding window for each
-// dimension of `input`. The dimension order is determined by the value of
-// `data_format`, see below for details.
-//	padding: The type of padding algorithm to use.
+// value: The value to pad if width > -1.  If empty, pads with spaces.
+// Another typical value is '0'.  String cannot be longer than 1 character.
+// If not specified, defaults to ""
+func AsStringFill(value string) AsStringAttr {
+	return func(m optionalAttr) {
+		m["fill"] = value
+	}
+}
+
+// Converts each entry in the given tensor to strings.  Supports many numeric
 //
-// Returns A 4-D tensor. The dimension order is determined by the value of
-// `data_format`, see below for details.
-func Conv2D(scope *Scope, input tf.Output, filter tf.Output, strides []int64, padding string, optional ...Conv2DAttr) (output tf.Output) {
+// types and boolean.
+func AsString(scope *Scope, input tf.Output, optional ...AsStringAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"strides": strides, "padding": padding}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Conv2D",
+		Type: "AsString",
 		Input: []tf.Input{
-			input, filter,
+			input,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
+
+// Deprecated. Use TensorArrayScatterV3
+//
+// DEPRECATED at GraphDef version 26: Use TensorArrayScatterV3
+func TensorArrayScatterV2(scope *Scope, handle tf.Output, indices tf.Output, value tf.Output, flow_in tf.Output) (flow_out tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorArrayScatterV2",
+		Input: []tf.Input{
+			handle, indices, value, flow_in,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
-- 
GitLab


From b995bb44d22e172a15141bb34cdfe6daf1d0ab6f Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Fri, 25 Jan 2019 11:36:17 -0800
Subject: [PATCH 1277/2345] Add missing TensorList ops to XLA

New ops include:
 - TensorListLength
 - TensorListFromTensor
 - TensorListGetItem
 - TensorListSetItem
 - TensorListStack

PiperOrigin-RevId: 230939371
---
 .../compiler/tests/tensor_list_ops_test.py    |  65 ++++++++
 .../tf2xla/kernels/tensor_list_ops.cc         | 153 +++++++++++++++++-
 .../python/kernel_tests/list_ops_test.py      |  33 +++-
 3 files changed, 239 insertions(+), 12 deletions(-)

diff --git a/tensorflow/compiler/tests/tensor_list_ops_test.py b/tensorflow/compiler/tests/tensor_list_ops_test.py
index 805e36a3b9..47e0f384a4 100644
--- a/tensorflow/compiler/tests/tensor_list_ops_test.py
+++ b/tensorflow/compiler/tests/tensor_list_ops_test.py
@@ -114,6 +114,71 @@ class ListOpsTest(xla_test.XLATestCase):
       _, e = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
       self.assertAllEqual(sess.run(e), 3.0 * np.ones((10, 15)))
 
+  def testListFromTensor(self):
+    with self.cached_session(), self.test_scope():
+      t = constant_op.constant([1.0, 2.0])
+      l = list_ops.tensor_list_from_tensor(t, element_shape=[])
+      e = list_ops.tensor_list_get_item(l, 0, element_dtype=dtypes.float32)
+      self.assertAllEqual(e, 1.0)
+      l, e0 = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
+      self.assertAllEqual(e0, 2.0)
+      l, e1 = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
+      self.assertAllEqual(e1, 1.0)
+      self.assertAllEqual(list_ops.tensor_list_length(l), 0)
+
+  def testGetSet(self):
+    with self.cached_session(), self.test_scope():
+      t = constant_op.constant([1.0, 2.0])
+      l = list_ops.tensor_list_from_tensor(t, element_shape=[])
+      e0 = list_ops.tensor_list_get_item(l, 0, element_dtype=dtypes.float32)
+      self.assertAllEqual(e0, 1.0)
+      l = list_ops.tensor_list_set_item(l, 0, 3.0)
+      t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
+      self.assertAllEqual(t, [3.0, 2.0])
+
+  def testGetSetReserved(self):
+    with self.cached_session(), self.test_scope():
+      l = list_ops.tensor_list_reserve(
+          element_dtype=dtypes.float32, element_shape=[], num_elements=2)
+      e0 = list_ops.tensor_list_get_item(l, 0, element_dtype=dtypes.float32)
+      self.assertAllEqual(e0, 0.0)
+      l = list_ops.tensor_list_set_item(l, 0, 3.0)
+      t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
+      self.assertAllEqual(t, [3.0, 0.0])
+
+  def testGetSetReservedNonScalar(self):
+    with self.cached_session() as sess, self.test_scope():
+      l = list_ops.tensor_list_reserve(
+          element_dtype=dtypes.float32,
+          element_shape=(7, 15),
+          num_elements=2)
+      l = list_ops.tensor_list_set_item(
+          l, 0, constant_op.constant(1.0, shape=(7, 15)))
+      e1 = list_ops.tensor_list_get_item(l, 0, element_dtype=dtypes.float32)
+      e2 = list_ops.tensor_list_get_item(l, 1, element_dtype=dtypes.float32)
+      self.assertAllEqual(sess.run(e1), np.ones((7, 15)))
+      self.assertAllEqual(sess.run(e2), np.zeros((7, 15)))
+
+  def testStack(self):
+    with self.cached_session(), self.test_scope():
+      l = list_ops.empty_tensor_list(
+          element_dtype=dtypes.float32,
+          element_shape=[],
+          max_num_elements=2)
+      l = list_ops.tensor_list_push_back(l, constant_op.constant(1.0))
+      e = list_ops.tensor_list_get_item(l, 0, element_dtype=dtypes.float32)
+      self.assertAllEqual(e, 1.0)
+      l = list_ops.tensor_list_push_back(l, constant_op.constant(2.0))
+      t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
+      self.assertAllEqual(t.shape.as_list(), [None])
+      self.assertAllEqual(t, [1.0, 2.0])
+
+  def testStackWithUninitializedTensors(self):
+    with self.cached_session(), self.test_scope():
+      l = list_ops.tensor_list_reserve(
+          element_dtype=dtypes.float32, element_shape=[], num_elements=3)
+      t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
+      self.assertAllEqual(t, [0., 0., 0.])
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc b/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc
index be29765ee3..6502001228 100644
--- a/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc
@@ -14,6 +14,9 @@ limitations under the License.
 ==============================================================================*/
 
 // XLA TensorList operators.
+// Tensor lists are represented as tuple consisting of a pre-allocated list
+// consisting of the tensors (and where dim 0 is the list index), along with a
+// scalar telling us the current number of elements.
 
 #include <limits>
 #include <vector>
@@ -50,6 +53,22 @@ Status GetTensorListShape(xla::XlaBuilder* builder, xla::XlaOp op,
                                tensor_list_shape);
 }
 
+class TensorListLengthOp : public XlaOpKernel {
+ public:
+  explicit TensorListLengthOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {}
+
+  void Compile(XlaOpKernelContext* ctx) override {
+    xla::XlaOp tl = ctx->Input(0);
+    xla::XlaOp index = xla::GetTupleElement(tl, 1);
+    ctx->SetOutput(0, index);
+  }
+
+ private:
+  TF_DISALLOW_COPY_AND_ASSIGN(TensorListLengthOp);
+};
+
+REGISTER_XLA_OP(Name("TensorListLength"), TensorListLengthOp);
+
 class TensorListReserveOp : public XlaOpKernel {
  public:
   explicit TensorListReserveOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {
@@ -162,6 +181,136 @@ class TensorListElementShapeOp : public XlaOpKernel {
 
 REGISTER_XLA_OP(Name("TensorListElementShape"), TensorListElementShapeOp);
 
+class TensorListGetItemOp : public XlaOpKernel {
+ public:
+  explicit TensorListGetItemOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("element_dtype", &dtype_));
+  }
+
+  void Compile(XlaOpKernelContext* ctx) override {
+    xla::XlaBuilder* b = ctx->builder();
+    xla::XlaOp state = ctx->Input(0);
+
+    TensorShape shape;
+    OP_REQUIRES_OK(ctx, GetTensorListShape(b, state, &shape));
+
+    xla::XlaOp ta = xla::GetTupleElement(state, 0);
+    xla::XlaOp index = ctx->Input(1);
+
+    // start_indices of the DynamicSlice are [index, 0, 0, ..., 0].
+    std::vector<xla::XlaOp> start_indices(shape.dims(),
+                                          xla::ConstantR0<int32>(b, 0));
+    start_indices[0] = index;
+    auto slice_shape = shape.dim_sizes();
+    slice_shape[0] = 1LL;
+
+    xla::XlaOp read = xla::DynamicSlice(ta, start_indices, slice_shape);
+    // Remove the leading '1' dimension.
+    std::vector<int64> value_shape(slice_shape.begin() + 1, slice_shape.end());
+
+    ctx->SetOutput(0, xla::Reshape(read, value_shape));
+  }
+
+ private:
+  DataType dtype_;
+
+  TF_DISALLOW_COPY_AND_ASSIGN(TensorListGetItemOp);
+};
+
+REGISTER_XLA_OP(Name("TensorListGetItem"), TensorListGetItemOp);
+
+class TensorListStackOp : public XlaOpKernel {
+ public:
+  explicit TensorListStackOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("element_dtype", &dtype_));
+  }
+
+  void Compile(XlaOpKernelContext* ctx) override {
+    xla::XlaOp state = ctx->Input(0);
+    xla::XlaOp ta = xla::GetTupleElement(state, 0);
+    ctx->SetOutput(0, ta);
+  }
+
+ private:
+  DataType dtype_;
+
+  TF_DISALLOW_COPY_AND_ASSIGN(TensorListStackOp);
+};
+
+REGISTER_XLA_OP(Name("TensorListStack"), TensorListStackOp);
+
+class TensorListFromTensorOp : public XlaOpKernel {
+ public:
+  explicit TensorListFromTensorOp(OpKernelConstruction* ctx)
+      : XlaOpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("element_dtype", &dtype_));
+  }
+
+  void Compile(XlaOpKernelContext* ctx) override {
+    TensorShape element_shape;
+    OP_REQUIRES_OK(ctx, ctx->ConstantInputAsShape(1, &element_shape));
+
+    const TensorShape tensor_shape = ctx->InputShape(0);
+    OP_REQUIRES(ctx, tensor_shape.dims() > 0,
+                errors::InvalidArgument("Input value must be at least a "
+                                        "vector but received shape: ",
+                                        tensor_shape.DebugString()));
+    const int num_elements = tensor_shape.dim_size(0);
+
+    xla::XlaBuilder* b = ctx->builder();
+    const xla::XlaOp tensor = ctx->Input(0);
+
+    ctx->SetTensorListOutput(
+        0, xla::Tuple(b, {tensor, xla::ConstantR0<int32>(b, num_elements)}));
+  }
+
+ private:
+  DataType dtype_;
+
+  TF_DISALLOW_COPY_AND_ASSIGN(TensorListFromTensorOp);
+};
+
+REGISTER_XLA_OP(
+    Name("TensorListFromTensor").CompileTimeConstantInput("element_shape"),
+    TensorListFromTensorOp);
+
+class TensorListSetItemOp : public XlaOpKernel {
+ public:
+  explicit TensorListSetItemOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("element_dtype", &dtype_));
+  }
+
+  void Compile(XlaOpKernelContext* ctx) override {
+    xla::XlaBuilder* b = ctx->builder();
+    xla::XlaOp tl = ctx->Input(0);
+    TensorShape elem_shape = ctx->InputShape(2);
+
+    xla::XlaOp ta = xla::GetTupleElement(tl, 0);
+    xla::XlaOp index = ctx->Input(1);
+    xla::XlaOp value = ctx->Input(2);
+
+    // start_indices of the DynamicUpdateSlice are [index, 0, 0, ..., 0].
+    std::vector<xla::XlaOp> start_indices(elem_shape.dims() + 1,
+                                          xla::ConstantR0<int32>(b, 0));
+    start_indices[0] = index;
+
+    TensorShape slice_shape = elem_shape;
+    slice_shape.InsertDim(0, 1LL);
+    auto update = xla::Reshape(value, slice_shape.dim_sizes());
+
+    ctx->SetTensorListOutput(
+        0, xla::Tuple(b, {xla::DynamicUpdateSlice(ta, update, start_indices),
+                          index + xla::ConstantR0<int32>(b, 1)}));
+  }
+
+ private:
+  DataType dtype_;
+
+  TF_DISALLOW_COPY_AND_ASSIGN(TensorListSetItemOp);
+};
+
+REGISTER_XLA_OP(Name("TensorListSetItem"), TensorListSetItemOp);
+
 class TensorListPushBackOp : public XlaOpKernel {
  public:
   explicit TensorListPushBackOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {
@@ -186,8 +335,6 @@ class TensorListPushBackOp : public XlaOpKernel {
     slice_shape.InsertDim(0, 1LL);
     auto update = xla::Reshape(value, slice_shape.dim_sizes());
 
-    // TODO(phawkins): We don't check the index is in bounds --- there is no
-    // error mechanism in XLA.
     ctx->SetTensorListOutput(
         0, xla::Tuple(b, {xla::DynamicUpdateSlice(ta, update, start_indices),
                           index + xla::ConstantR0<int32>(b, 1)}));
@@ -226,8 +373,6 @@ class TensorListPopBackOp : public XlaOpKernel {
     auto slice_shape = shape.dim_sizes();
     slice_shape[0] = 1LL;
 
-    // TODO(phawkins): We don't check the index is in bounds --- there is no
-    // error mechanism in XLA.
     xla::XlaOp read = xla::DynamicSlice(ta, start_indices, slice_shape);
     // Remove the leading '1' dimension.
     std::vector<int64> value_shape(slice_shape.begin() + 1, slice_shape.end());
diff --git a/tensorflow/python/kernel_tests/list_ops_test.py b/tensorflow/python/kernel_tests/list_ops_test.py
index 3f340aa07d..01618c52a3 100644
--- a/tensorflow/python/kernel_tests/list_ops_test.py
+++ b/tensorflow/python/kernel_tests/list_ops_test.py
@@ -249,7 +249,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     l = list_ops.tensor_list_reserve(
         element_dtype=dtypes.float32, element_shape=[], num_elements=3)
     t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
-    self.assertAllEqual(self.evaluate(t), [0., 0., 0.])
+    self.assertAllEqual(t, [0., 0., 0.])
 
   def testStackWithUninitializedTensors(self):
     self._testStackWithUninitializedTensors()
@@ -265,7 +265,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
         element_dtype=dtypes.float32, element_shape=None, num_elements=3)
     l = list_ops.tensor_list_set_item(l, 1, [1., 2.])
     t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
-    self.assertAllEqual(self.evaluate(t), [[0., 0.], [1., 2.], [0., 0.]])
+    self.assertAllEqual(t, [[0., 0.], [1., 2.], [0., 0.]])
 
   def testStackWithUninitializedTensorsInferShape(self):
     self._testStackWithUninitializedTensorsInferShape()
@@ -466,7 +466,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     l = list_ops.tensor_list_scatter(
         c0, [1, 3], ops.convert_to_tensor([], dtype=dtypes.int32))
     # TensorListScatter should return a list with size largest index + 1.
-    self.assertEqual(self.evaluate(list_ops.tensor_list_length(l)), 4)
+    self.assertAllEqual(list_ops.tensor_list_length(l), 4)
 
   def testScatterWithInvalidRowsInInputTensorFails(self):
     c0 = constant_op.constant([1.0, 2.0])
@@ -515,11 +515,13 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
   def testTensorListFromTensor(self):
     t = constant_op.constant([1.0, 2.0])
     l = list_ops.tensor_list_from_tensor(t, element_shape=[])
+    e = list_ops.tensor_list_get_item(l, 0, element_dtype=dtypes.float32)
+    self.assertAllEqual(e, 1.0)
     l, e = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
-    self.assertAllEqual(self.evaluate(e), 2.0)
+    self.assertAllEqual(e, 2.0)
     l, e = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
-    self.assertAllEqual(self.evaluate(e), 1.0)
-    self.assertAllEqual(self.evaluate(list_ops.tensor_list_length(l)), 0)
+    self.assertAllEqual(e, 1.0)
+    self.assertAllEqual(list_ops.tensor_list_length(l), 0)
 
   def testFromTensorGPU(self):
     if not context.num_gpus():
@@ -527,7 +529,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     with context.device("gpu:0"):
       self.testTensorListFromTensor()
 
-  def testGetSetItem(self):
+  def testGetSet(self):
     t = constant_op.constant([1.0, 2.0])
     l = list_ops.tensor_list_from_tensor(t, element_shape=[])
     e0 = list_ops.tensor_list_get_item(l, 0, element_dtype=dtypes.float32)
@@ -540,7 +542,22 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     if not context.num_gpus():
       return
     with context.device("gpu:0"):
-      self.testGetSetItem()
+      self.testGetSet()
+
+  def testGetSetReserved(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=[], num_elements=2)
+    e0 = list_ops.tensor_list_get_item(l, 0, element_dtype=dtypes.float32)
+    self.assertAllEqual(e0, 0.0)
+    l = list_ops.tensor_list_set_item(l, 0, 3.0)
+    t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
+    self.assertAllEqual(t, [3.0, 0.0])
+
+  def testGetSetReservedGPU(self):
+    if not context.num_gpus():
+      return
+    with context.device("gpu:0"):
+      self.testGetSetReserved()
 
   def testSetGetGrad(self):
     with backprop.GradientTape() as tape:
-- 
GitLab


From 32822bfdbe0cc91ff5f9ba4e06ec0c743974f635 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 25 Jan 2019 11:37:21 -0800
Subject: [PATCH 1278/2345] Update ops-related pbtxt files.

PiperOrigin-RevId: 230939588
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 1240 +++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 1240 +++++++++++++++++
 2 files changed, 2480 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index eb251fc517..eee5af141e 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -44615,6 +44615,1246 @@ op {
     }
   }
 }
+op {
+  name: "QuantizedConv2DAndRelu"
+  input_arg {
+    name: "input"
+    type_attr: "Tinput"
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "Tfilter"
+  }
+  input_arg {
+    name: "min_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_filter"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "out_type"
+  }
+  output_arg {
+    name: "min_output"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "max_output"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "Tinput"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "Tfilter"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "out_type"
+    type: "type"
+    default_value {
+      type: DT_QINT32
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "dilations"
+    type: "list(int)"
+    default_value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+      }
+    }
+  }
+}
+op {
+  name: "QuantizedConv2DAndReluAndRequantize"
+  input_arg {
+    name: "input"
+    type_attr: "Tinput"
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "Tfilter"
+  }
+  input_arg {
+    name: "min_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_freezed_output"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_freezed_output"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "out_type"
+  }
+  output_arg {
+    name: "min_output"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "max_output"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "Tinput"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "Tfilter"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "out_type"
+    type: "type"
+    default_value {
+      type: DT_QUINT8
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "dilations"
+    type: "list(int)"
+    default_value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+      }
+    }
+  }
+}
+op {
+  name: "QuantizedConv2DAndRequantize"
+  input_arg {
+    name: "input"
+    type_attr: "Tinput"
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "Tfilter"
+  }
+  input_arg {
+    name: "min_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_freezed_output"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_freezed_output"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "out_type"
+  }
+  output_arg {
+    name: "min_output"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "max_output"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "Tinput"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "Tfilter"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "out_type"
+    type: "type"
+    default_value {
+      type: DT_QINT8
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "dilations"
+    type: "list(int)"
+    default_value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+      }
+    }
+  }
+}
+op {
+  name: "QuantizedConv2DWithBias"
+  input_arg {
+    name: "input"
+    type_attr: "Tinput"
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "Tfilter"
+  }
+  input_arg {
+    name: "bias"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_filter"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "out_type"
+  }
+  output_arg {
+    name: "min_output"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "max_output"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "Tinput"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "Tfilter"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "out_type"
+    type: "type"
+    default_value {
+      type: DT_QINT32
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "dilations"
+    type: "list(int)"
+    default_value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+      }
+    }
+  }
+}
+op {
+  name: "QuantizedConv2DWithBiasAndRelu"
+  input_arg {
+    name: "input"
+    type_attr: "Tinput"
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "Tfilter"
+  }
+  input_arg {
+    name: "bias"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_filter"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "out_type"
+  }
+  output_arg {
+    name: "min_output"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "max_output"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "Tinput"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "Tfilter"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "out_type"
+    type: "type"
+    default_value {
+      type: DT_QINT32
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "dilations"
+    type: "list(int)"
+    default_value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+      }
+    }
+  }
+}
+op {
+  name: "QuantizedConv2DWithBiasAndReluAndRequantize"
+  input_arg {
+    name: "input"
+    type_attr: "Tinput"
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "Tfilter"
+  }
+  input_arg {
+    name: "bias"
+    type_attr: "Tbias"
+  }
+  input_arg {
+    name: "min_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_freezed_output"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_freezed_output"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "out_type"
+  }
+  output_arg {
+    name: "min_output"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "max_output"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "Tinput"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "Tfilter"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "Tbias"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_QINT32
+      }
+    }
+  }
+  attr {
+    name: "out_type"
+    type: "type"
+    default_value {
+      type: DT_QUINT8
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "dilations"
+    type: "list(int)"
+    default_value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+      }
+    }
+  }
+}
+op {
+  name: "QuantizedConv2DWithBiasAndRequantize"
+  input_arg {
+    name: "input"
+    type_attr: "Tinput"
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "Tfilter"
+  }
+  input_arg {
+    name: "bias"
+    type_attr: "Tbias"
+  }
+  input_arg {
+    name: "min_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_freezed_output"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_freezed_output"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "out_type"
+  }
+  output_arg {
+    name: "min_output"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "max_output"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "Tinput"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "Tfilter"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "Tbias"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_QINT32
+      }
+    }
+  }
+  attr {
+    name: "out_type"
+    type: "type"
+    default_value {
+      type: DT_QINT8
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "dilations"
+    type: "list(int)"
+    default_value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+      }
+    }
+  }
+}
+op {
+  name: "QuantizedConv2DWithBiasSignedSumAndReluAndRequantize"
+  input_arg {
+    name: "input"
+    type_attr: "Tinput"
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "Tfilter"
+  }
+  input_arg {
+    name: "bias"
+    type_attr: "Tbias"
+  }
+  input_arg {
+    name: "min_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_freezed_output"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_freezed_output"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "summand"
+    type_attr: "Tsummand"
+  }
+  input_arg {
+    name: "min_summand"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_summand"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "out_type"
+  }
+  output_arg {
+    name: "min_output"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "max_output"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "Tinput"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "Tfilter"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "Tbias"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_QINT32
+      }
+    }
+  }
+  attr {
+    name: "Tsummand"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "out_type"
+    type: "type"
+    default_value {
+      type: DT_QUINT8
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "dilations"
+    type: "list(int)"
+    default_value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+      }
+    }
+  }
+}
+op {
+  name: "QuantizedConv2DWithBiasSumAndRelu"
+  input_arg {
+    name: "input"
+    type_attr: "Tinput"
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "Tfilter"
+  }
+  input_arg {
+    name: "bias"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "summand"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "out_type"
+  }
+  output_arg {
+    name: "min_output"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "max_output"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "Tinput"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "Tfilter"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "out_type"
+    type: "type"
+    default_value {
+      type: DT_QINT32
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "dilations"
+    type: "list(int)"
+    default_value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+      }
+    }
+  }
+}
+op {
+  name: "QuantizedConv2DWithBiasSumAndReluAndRequantize"
+  input_arg {
+    name: "input"
+    type_attr: "Tinput"
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "Tfilter"
+  }
+  input_arg {
+    name: "bias"
+    type_attr: "Tbias"
+  }
+  input_arg {
+    name: "min_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_freezed_output"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_freezed_output"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "summand"
+    type_attr: "Tsummand"
+  }
+  input_arg {
+    name: "min_summand"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_summand"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "out_type"
+  }
+  output_arg {
+    name: "min_output"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "max_output"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "Tinput"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "Tfilter"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "Tbias"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_QINT32
+      }
+    }
+  }
+  attr {
+    name: "Tsummand"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "out_type"
+    type: "type"
+    default_value {
+      type: DT_QUINT8
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "dilations"
+    type: "list(int)"
+    default_value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+      }
+    }
+  }
+}
 op {
   name: "QuantizedInstanceNorm"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index f43a8c4744..2e6519fa9c 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -22400,6 +22400,1246 @@ op {
     }
   }
 }
+op {
+  name: "QuantizedConv2DAndRelu"
+  input_arg {
+    name: "input"
+    type_attr: "Tinput"
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "Tfilter"
+  }
+  input_arg {
+    name: "min_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_filter"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "out_type"
+  }
+  output_arg {
+    name: "min_output"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "max_output"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "Tinput"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "Tfilter"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "out_type"
+    type: "type"
+    default_value {
+      type: DT_QINT32
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "dilations"
+    type: "list(int)"
+    default_value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+      }
+    }
+  }
+}
+op {
+  name: "QuantizedConv2DAndReluAndRequantize"
+  input_arg {
+    name: "input"
+    type_attr: "Tinput"
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "Tfilter"
+  }
+  input_arg {
+    name: "min_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_freezed_output"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_freezed_output"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "out_type"
+  }
+  output_arg {
+    name: "min_output"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "max_output"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "Tinput"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "Tfilter"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "out_type"
+    type: "type"
+    default_value {
+      type: DT_QUINT8
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "dilations"
+    type: "list(int)"
+    default_value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+      }
+    }
+  }
+}
+op {
+  name: "QuantizedConv2DAndRequantize"
+  input_arg {
+    name: "input"
+    type_attr: "Tinput"
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "Tfilter"
+  }
+  input_arg {
+    name: "min_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_freezed_output"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_freezed_output"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "out_type"
+  }
+  output_arg {
+    name: "min_output"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "max_output"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "Tinput"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "Tfilter"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "out_type"
+    type: "type"
+    default_value {
+      type: DT_QINT8
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "dilations"
+    type: "list(int)"
+    default_value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+      }
+    }
+  }
+}
+op {
+  name: "QuantizedConv2DWithBias"
+  input_arg {
+    name: "input"
+    type_attr: "Tinput"
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "Tfilter"
+  }
+  input_arg {
+    name: "bias"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_filter"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "out_type"
+  }
+  output_arg {
+    name: "min_output"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "max_output"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "Tinput"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "Tfilter"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "out_type"
+    type: "type"
+    default_value {
+      type: DT_QINT32
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "dilations"
+    type: "list(int)"
+    default_value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+      }
+    }
+  }
+}
+op {
+  name: "QuantizedConv2DWithBiasAndRelu"
+  input_arg {
+    name: "input"
+    type_attr: "Tinput"
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "Tfilter"
+  }
+  input_arg {
+    name: "bias"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_filter"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "out_type"
+  }
+  output_arg {
+    name: "min_output"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "max_output"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "Tinput"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "Tfilter"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "out_type"
+    type: "type"
+    default_value {
+      type: DT_QINT32
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "dilations"
+    type: "list(int)"
+    default_value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+      }
+    }
+  }
+}
+op {
+  name: "QuantizedConv2DWithBiasAndReluAndRequantize"
+  input_arg {
+    name: "input"
+    type_attr: "Tinput"
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "Tfilter"
+  }
+  input_arg {
+    name: "bias"
+    type_attr: "Tbias"
+  }
+  input_arg {
+    name: "min_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_freezed_output"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_freezed_output"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "out_type"
+  }
+  output_arg {
+    name: "min_output"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "max_output"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "Tinput"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "Tfilter"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "Tbias"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_QINT32
+      }
+    }
+  }
+  attr {
+    name: "out_type"
+    type: "type"
+    default_value {
+      type: DT_QUINT8
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "dilations"
+    type: "list(int)"
+    default_value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+      }
+    }
+  }
+}
+op {
+  name: "QuantizedConv2DWithBiasAndRequantize"
+  input_arg {
+    name: "input"
+    type_attr: "Tinput"
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "Tfilter"
+  }
+  input_arg {
+    name: "bias"
+    type_attr: "Tbias"
+  }
+  input_arg {
+    name: "min_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_freezed_output"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_freezed_output"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "out_type"
+  }
+  output_arg {
+    name: "min_output"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "max_output"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "Tinput"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "Tfilter"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "Tbias"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_QINT32
+      }
+    }
+  }
+  attr {
+    name: "out_type"
+    type: "type"
+    default_value {
+      type: DT_QINT8
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "dilations"
+    type: "list(int)"
+    default_value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+      }
+    }
+  }
+}
+op {
+  name: "QuantizedConv2DWithBiasSignedSumAndReluAndRequantize"
+  input_arg {
+    name: "input"
+    type_attr: "Tinput"
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "Tfilter"
+  }
+  input_arg {
+    name: "bias"
+    type_attr: "Tbias"
+  }
+  input_arg {
+    name: "min_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_freezed_output"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_freezed_output"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "summand"
+    type_attr: "Tsummand"
+  }
+  input_arg {
+    name: "min_summand"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_summand"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "out_type"
+  }
+  output_arg {
+    name: "min_output"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "max_output"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "Tinput"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "Tfilter"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "Tbias"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_QINT32
+      }
+    }
+  }
+  attr {
+    name: "Tsummand"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "out_type"
+    type: "type"
+    default_value {
+      type: DT_QUINT8
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "dilations"
+    type: "list(int)"
+    default_value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+      }
+    }
+  }
+}
+op {
+  name: "QuantizedConv2DWithBiasSumAndRelu"
+  input_arg {
+    name: "input"
+    type_attr: "Tinput"
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "Tfilter"
+  }
+  input_arg {
+    name: "bias"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "summand"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "out_type"
+  }
+  output_arg {
+    name: "min_output"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "max_output"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "Tinput"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "Tfilter"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "out_type"
+    type: "type"
+    default_value {
+      type: DT_QINT32
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "dilations"
+    type: "list(int)"
+    default_value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+      }
+    }
+  }
+}
+op {
+  name: "QuantizedConv2DWithBiasSumAndReluAndRequantize"
+  input_arg {
+    name: "input"
+    type_attr: "Tinput"
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "Tfilter"
+  }
+  input_arg {
+    name: "bias"
+    type_attr: "Tbias"
+  }
+  input_arg {
+    name: "min_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_filter"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_freezed_output"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_freezed_output"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "summand"
+    type_attr: "Tsummand"
+  }
+  input_arg {
+    name: "min_summand"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_summand"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "out_type"
+  }
+  output_arg {
+    name: "min_output"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "max_output"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "Tinput"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "Tfilter"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "Tbias"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_QINT32
+      }
+    }
+  }
+  attr {
+    name: "Tsummand"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "out_type"
+    type: "type"
+    default_value {
+      type: DT_QUINT8
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "dilations"
+    type: "list(int)"
+    default_value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+      }
+    }
+  }
+}
 op {
   name: "QuantizedInstanceNorm"
   input_arg {
-- 
GitLab


From cf3c4275a41f482741258b9688cf9b92cefe3d04 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Fri, 25 Jan 2019 11:47:42 -0800
Subject: [PATCH 1279/2345] Do not ignore user options when building eager
 contexts.

Oops.

PiperOrigin-RevId: 230941526
---
 tensorflow/core/common_runtime/eager/context.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/common_runtime/eager/context.cc b/tensorflow/core/common_runtime/eager/context.cc
index 4bc3843ee8..4d8712f4dc 100644
--- a/tensorflow/core/common_runtime/eager/context.cc
+++ b/tensorflow/core/common_runtime/eager/context.cc
@@ -57,8 +57,8 @@ EagerContext::EagerContext(const SessionOptions& opts,
       rendezvous_(rendezvous),
       thread_pool_(NewThreadPoolFromSessionOptions(opts)),
       pflr_(new ProcessFunctionLibraryRuntime(
-          device_mgr, opts.env, TF_GRAPH_DEF_VERSION, &func_lib_def_, {},
-          thread_pool_.get())),
+          device_mgr, opts.env, TF_GRAPH_DEF_VERSION, &func_lib_def_,
+          opts.config.graph_options().optimizer_options(), thread_pool_.get())),
       log_device_placement_(opts.config.log_device_placement()),
       num_active_steps_(0),
       async_default_(async),
-- 
GitLab


From 1683e6447d4d4ae43555f268f1599c5ad951c90f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 25 Jan 2019 11:52:06 -0800
Subject: [PATCH 1280/2345] Moving AutoGraph parse error handling to wrap
 conversion.function_to_graph instead of pyct.transformer.Base.visit.

Moving OriginInfo unpacking into AutoGraphParseError.

Modifying types of errors caught in tests that directly call transformers instead of going through function_to_graph.

PiperOrigin-RevId: 230942251
---
 .../autograph/converters/control_flow_test.py |   3 +-
 .../autograph/converters/decorators_test.py   |   3 +-
 .../autograph/converters/slices_test.py       |   3 +-
 .../python/autograph/impl/conversion.py       |   8 +-
 .../python/autograph/pyct/transformer.py      | 114 ++++++++----------
 5 files changed, 60 insertions(+), 71 deletions(-)

diff --git a/tensorflow/python/autograph/converters/control_flow_test.py b/tensorflow/python/autograph/converters/control_flow_test.py
index 1a38d0db4d..9009e67ef4 100644
--- a/tensorflow/python/autograph/converters/control_flow_test.py
+++ b/tensorflow/python/autograph/converters/control_flow_test.py
@@ -20,7 +20,6 @@ from __future__ import print_function
 
 from tensorflow.python.autograph.converters import control_flow
 from tensorflow.python.autograph.core import converter_testing
-from tensorflow.python.autograph.pyct import transformer
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import test_util
@@ -184,7 +183,7 @@ class ControlFlowTest(converter_testing.TestCase):
       return b
 
     node, ctx = self.prepare(test_fn, {})
-    with self.assertRaises(transformer.AutoGraphParseError):
+    with self.assertRaises(ValueError):
       control_flow.transform(node, ctx)
 
   @test_util.run_deprecated_v1
diff --git a/tensorflow/python/autograph/converters/decorators_test.py b/tensorflow/python/autograph/converters/decorators_test.py
index bcf502c62b..d38975931b 100644
--- a/tensorflow/python/autograph/converters/decorators_test.py
+++ b/tensorflow/python/autograph/converters/decorators_test.py
@@ -25,7 +25,6 @@ from tensorflow.python import autograph
 from tensorflow.python.autograph.converters import decorators
 from tensorflow.python.autograph.core import converter_testing
 from tensorflow.python.autograph.pyct import compiler
-from tensorflow.python.autograph.pyct import transformer
 from tensorflow.python.platform import test
 
 
@@ -121,7 +120,7 @@ class DecoratorsTest(converter_testing.TestCase):
       return inner_fn(a)
 
     # Expected to fail because simple_decorator could not be imported.
-    with self.assertRaises(transformer.AutoGraphParseError):
+    with self.assertRaises(ValueError):
       test_fn(1)
 
   def test_nested_decorators_imported(self):
diff --git a/tensorflow/python/autograph/converters/slices_test.py b/tensorflow/python/autograph/converters/slices_test.py
index d674f266c8..11e3736d4f 100644
--- a/tensorflow/python/autograph/converters/slices_test.py
+++ b/tensorflow/python/autograph/converters/slices_test.py
@@ -23,7 +23,6 @@ from tensorflow.python.autograph.core import converter_testing
 from tensorflow.python.autograph.lang import directives
 from tensorflow.python.autograph.pyct import anno
 from tensorflow.python.autograph.pyct import parser
-from tensorflow.python.autograph.pyct import transformer
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.ops import list_ops
@@ -68,7 +67,7 @@ class SliceTest(converter_testing.TestCase):
     def_.directives[directives.set_element_type] = {
         'dtype': parser.parse_expression('tf.float32')
     }
-    with self.assertRaises(transformer.AutoGraphParseError):
+    with self.assertRaises(ValueError):
       slices.transform(node, ctx)
 
 
diff --git a/tensorflow/python/autograph/impl/conversion.py b/tensorflow/python/autograph/impl/conversion.py
index 5c8dd59fe6..9f0c380cbf 100644
--- a/tensorflow/python/autograph/impl/conversion.py
+++ b/tensorflow/python/autograph/impl/conversion.py
@@ -378,7 +378,13 @@ def function_to_graph(f,
       arg_types=arg_types,
       owner_type=owner_type)
   context = converter.EntityContext(namer, entity_info, program_ctx)
-  node = node_to_graph(node, context)
+  try:
+    node = node_to_graph(node, context)
+  except (ValueError, AttributeError, KeyError, NotImplementedError) as e:
+    # TODO(znado): raise InternalError
+    if not context.current_origin:
+      raise e
+    raise transformer.AutoGraphParseError(str(e), context.current_origin)
 
   if isinstance(node, gast.Lambda):
     new_name = namer.new_symbol('tf__lambda', ())
diff --git a/tensorflow/python/autograph/pyct/transformer.py b/tensorflow/python/autograph/pyct/transformer.py
index 530efee122..f32d609388 100644
--- a/tensorflow/python/autograph/pyct/transformer.py
+++ b/tensorflow/python/autograph/pyct/transformer.py
@@ -18,10 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import sys
 
 import gast
-import six
 
 from tensorflow.python.autograph.pyct import anno
 from tensorflow.python.autograph.pyct import compiler
@@ -30,7 +28,15 @@ from tensorflow.python.autograph.pyct import templates
 
 
 class AutoGraphParseError(SyntaxError):
-  pass
+  """Error for graph construction errors from AutoGraph generated code."""
+
+  def __init__(self, error, origin_info):
+    file_path = origin_info.loc.filename
+    line_number = origin_info.loc.lineno
+    col_offset = origin_info.loc.col_offset
+    source_line = origin_info.source_code_line
+    super(AutoGraphParseError, self).__init__(
+        error, (file_path, line_number, col_offset, source_line))
 
 
 # TODO(znado): Use namedtuple.
@@ -462,64 +468,44 @@ class Base(gast.NodeTransformer):
     processing_expr_node = False
 
     parent_origin = self.ctx.current_origin
-    try:
-      if isinstance(node, (gast.FunctionDef, gast.ClassDef, gast.Lambda)):
-        did_enter_function = True
-      elif isinstance(node, gast.Expr):
-        processing_expr_node = True
-
-      if did_enter_function:
-        self._enclosing_entities.append(node)
-
-      if anno.hasanno(node, anno.Basic.ORIGIN):
-        self.ctx.current_origin = anno.getanno(node, anno.Basic.ORIGIN)
-
-      if processing_expr_node:
-        entry_expr_value = node.value
-
-      if not anno.hasanno(node, anno.Basic.SKIP_PROCESSING):
-        result = super(Base, self).visit(node)
-      self.ctx.current_origin = parent_origin
-
-      # Adjust for consistency: replacing the value of an Expr with
-      # an Assign node removes the need for the Expr node.
-      if processing_expr_node:
-        if isinstance(result, gast.Expr) and result.value != entry_expr_value:
-          # When the replacement is a list, it is assumed that the list came
-          # from a template that contained a number of statements, which
-          # themselves are standalone and don't require an enclosing Expr.
-          if isinstance(result.value,
-                        (list, tuple, gast.Assign, gast.AugAssign)):
-            result = result.value
-
-      # On exception, the local scope integrity is not guaranteed.
-      if did_enter_function:
-        self._enclosing_entities.pop()
-
-      if local_scope_size_at_entry != len(self._local_scope_state):
-        raise AssertionError(
-            'Inconsistent local scope stack. Before entering node %s, the'
-            ' stack had length %d, after exit it has length %d. This'
-            ' indicates enter_local_scope and exit_local_scope are not'
-            ' well paired.' % (node, local_scope_size_at_entry,
-                               len(self._local_scope_state)))
-      return result
-
-    except (ValueError, AttributeError, KeyError, NotImplementedError) as e:
-      if not self.ctx.current_origin:
-        raise e
-      original_file_path = self.ctx.current_origin.loc.filename
-      original_line_number = self.ctx.current_origin.loc.lineno
-      original_col_offset = self.ctx.current_origin.loc.col_offset
-      original_source_line = (
-          self.ctx.current_origin.source_code_line)
-      msg = '%s: %s.' % (e.__class__.__name__, str(e))
-
-      # TODO(mdan): Avoid the printing of the original exception.
-      # In other words, we need to find how to suppress the "During handling
-      # of the above exception, another exception occurred" message.
-      six.reraise(
-          AutoGraphParseError,
-          AutoGraphParseError(msg, (original_file_path, original_line_number,
-                                    original_col_offset, original_source_line)),
-          sys.exc_info()[2])
+    if isinstance(node, (gast.FunctionDef, gast.ClassDef, gast.Lambda)):
+      did_enter_function = True
+    elif isinstance(node, gast.Expr):
+      processing_expr_node = True
+
+    if did_enter_function:
+      self._enclosing_entities.append(node)
+
+    if anno.hasanno(node, anno.Basic.ORIGIN):
+      self.ctx.current_origin = anno.getanno(node, anno.Basic.ORIGIN)
+
+    if processing_expr_node:
+      entry_expr_value = node.value
+
+    if not anno.hasanno(node, anno.Basic.SKIP_PROCESSING):
+      result = super(Base, self).visit(node)
+    self.ctx.current_origin = parent_origin
+
+    # Adjust for consistency: replacing the value of an Expr with
+    # an Assign node removes the need for the Expr node.
+    if processing_expr_node:
+      if isinstance(result, gast.Expr) and result.value != entry_expr_value:
+        # When the replacement is a list, it is assumed that the list came
+        # from a template that contained a number of statements, which
+        # themselves are standalone and don't require an enclosing Expr.
+        if isinstance(result.value,
+                      (list, tuple, gast.Assign, gast.AugAssign)):
+          result = result.value
+
+    # On exception, the local scope integrity is not guaranteed.
+    if did_enter_function:
+      self._enclosing_entities.pop()
+
+    if local_scope_size_at_entry != len(self._local_scope_state):
+      raise AssertionError(
+          'Inconsistent local scope stack. Before entering node %s, the'
+          ' stack had length %d, after exit it has length %d. This'
+          ' indicates enter_local_scope and exit_local_scope are not'
+          ' well paired.' % (node, local_scope_size_at_entry,
+                             len(self._local_scope_state)))
+    return result
-- 
GitLab


From 3afcfc57ab44247d442e614e1430d41777628961 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Fri, 25 Jan 2019 11:58:30 -0800
Subject: [PATCH 1281/2345] Add test for model save and load under distribution
 strategy scope

PiperOrigin-RevId: 230943244
---
 .../contrib/distribute/python/keras_test.py   | 43 +++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 8a607dd070..3f27e2547d 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -18,6 +18,7 @@ from __future__ import division
 from __future__ import print_function
 
 import os
+import tempfile
 from absl.testing import parameterized
 import numpy as np
 
@@ -1149,6 +1150,48 @@ class TestDistributionStrategyWithNormalizationLayer(
       np.testing.assert_allclose(out.std(), 1.0, atol=1e-1)
 
 
+class TestDistributionStrategySaveLoadWeights(test.TestCase,
+                                              parameterized.TestCase):
+
+  @combinations.generate(all_strategy_combinations_minus_default())
+  def test_save_load_h5(self, distribution):
+    with self.cached_session():
+      dataset = get_dataset(distribution)
+      with distribution.scope():
+        model = get_model()
+        model.compile(gradient_descent_keras.SGD(0.01), 'mse')
+        model.fit(dataset, epochs=1, steps_per_epoch=1)
+
+        weights_file = tempfile.mktemp('.h5')
+        model.save_weights(weights_file)
+
+        model_2 = get_model()
+        model_2.compile(gradient_descent_keras.SGD(0.01), 'mse')
+        model_2.load_weights(weights_file)
+        model_2.predict(get_predict_dataset(distribution), steps=2)
+        model_2.fit(dataset, epochs=1, steps_per_epoch=1)
+
+  @combinations.generate(all_strategy_combinations_minus_default())
+  def test_save_load_checkpointable(self, distribution):
+    # TODO(sourabhbajaj): Fix this and enable the test.
+    self.skipTest('Restore for checkpoint for optimizer weights fails.')
+    with self.cached_session():
+      dataset = get_dataset(distribution)
+      with distribution.scope():
+        model = get_model()
+        model.compile(gradient_descent_keras.SGD(0.01), 'mse')
+        model.fit(dataset, epochs=1, steps_per_epoch=1)
+
+        weights_file = tempfile.mktemp()
+        model.save_weights(weights_file)
+
+        model_2 = get_model()
+        model_2.compile(gradient_descent_keras.SGD(0.01), 'mse')
+        model_2.load_weights(weights_file)
+        model_2.predict(get_predict_dataset(distribution), steps=2)
+        model_2.fit(dataset, epochs=1, steps_per_epoch=1)
+
+
 class TestDistributionStrategyValidation(test.TestCase,
                                          parameterized.TestCase):
 
-- 
GitLab


From 3ebcaed7cce6d47dc802be0e7ab0debf848ca322 Mon Sep 17 00:00:00 2001
From: Alina Sbirlea <asbirlea@google.com>
Date: Fri, 25 Jan 2019 11:59:28 -0800
Subject: [PATCH 1282/2345] Remove pass warning about missed transformations.
 The warnings are benign and noisy.

PiperOrigin-RevId: 230943397
---
 tensorflow/compiler/xla/service/cpu/compiler_functor.cc | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/cpu/compiler_functor.cc b/tensorflow/compiler/xla/service/cpu/compiler_functor.cc
index 796a7cf94d..414eacddfc 100644
--- a/tensorflow/compiler/xla/service/cpu/compiler_functor.cc
+++ b/tensorflow/compiler/xla/service/cpu/compiler_functor.cc
@@ -66,9 +66,14 @@ class FilteredPassManager : public llvm::legacy::PassManager {
   explicit FilteredPassManager(bool disable_expensive_passes)
       : disable_expensive_passes_(disable_expensive_passes) {}
   void add(llvm::Pass* p) override {
+    llvm::StringRef PassName = p->getPassName();
+    if (PassName.contains("Warn about non-applied transformations")) {
+      delete p;
+      return;
+    }
     if (disable_expensive_passes_) {
-      llvm::StringRef PassName = p->getPassName();
       if (PassName.contains("Unroll loops")) {
+        delete p;
         return;
       }
     }
-- 
GitLab


From 14bd58e42e848dbd864caf156b3c288b46dfcbe7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 25 Jan 2019 12:22:52 -0800
Subject: [PATCH 1283/2345] add keras v2 optimizers to some distributions
 strategy tests

PiperOrigin-RevId: 230947619
---
 tensorflow/contrib/distribute/python/BUILD      |  1 +
 .../contrib/distribute/python/combinations.py   | 16 +++++++++++++++-
 .../python/keras_backward_compat_test.py        | 17 +++++++++++------
 .../contrib/distribute/python/keras_test.py     | 17 +++++++++++------
 4 files changed, 38 insertions(+), 13 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index fbfaa8e549..090fe09d02 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -202,6 +202,7 @@ py_library(
         "//tensorflow/python:util",
         "//tensorflow/python/distribute:distribute_lib",
         "//tensorflow/python/eager:context",
+        "//tensorflow/python/keras/optimizer_v2",
         "@absl_py//absl/testing:parameterized",
     ],
 )
diff --git a/tensorflow/contrib/distribute/python/combinations.py b/tensorflow/contrib/distribute/python/combinations.py
index 88170d4085..db79d6c0d8 100644
--- a/tensorflow/contrib/distribute/python/combinations.py
+++ b/tensorflow/contrib/distribute/python/combinations.py
@@ -57,6 +57,10 @@ from tensorflow.contrib.tpu.python.tpu import device_assignment as device_assign
 from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
+from tensorflow.python.keras.optimizer_v2 import adagrad as adagrad_keras_v2
+from tensorflow.python.keras.optimizer_v2 import adam as adam_keras_v2
+from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_descent_keras_v2
+from tensorflow.python.keras.optimizer_v2 import rmsprop as rmsprop_keras_v2
 from tensorflow.python.training import adagrad
 from tensorflow.python.training import adam
 from tensorflow.python.training import gradient_descent
@@ -422,10 +426,20 @@ gradient_descent_optimizer_v2_fn = NamedObject(
 adagrad_optimizer_v2_fn = NamedObject(
     "AdagradV2", lambda: adagrad_v2.AdagradOptimizer(0.001))
 adam_optimizer_v2_fn = NamedObject(
-    "AdamV2", lambda: adam_v2.AdamOptimizer(0.001, epsilon=1))
+    "AdamV2", lambda: adam_v2.AdamOptimizer(0.001, epsilon=1.0))
 
 optimizers_v2 = [gradient_descent_optimizer_v2_fn, adagrad_optimizer_v2_fn]
 
+gradient_descent_optimizer_keras_v2_fn = NamedObject(
+    "GradientDescentKerasV2",
+    lambda: gradient_descent_keras_v2.SGD(0.2))
+adagrad_optimizer_keras_v2_fn = NamedObject(
+    "AdagradKerasV2", lambda: adagrad_keras_v2.Adagrad(0.001))
+adam_optimizer_keras_v2_fn = NamedObject(
+    "AdamKerasV2", lambda: adam_keras_v2.Adam(0.001, epsilon=1.0))
+rmsprop_optimizer_keras_v2_fn = NamedObject(
+    "RmsPropKerasV2", lambda: rmsprop_keras_v2.RMSprop(0.001))
+
 graph_and_eager_modes = ["graph", "eager"]
 
 
diff --git a/tensorflow/contrib/distribute/python/keras_backward_compat_test.py b/tensorflow/contrib/distribute/python/keras_backward_compat_test.py
index 92de8e643e..35d40e66bf 100644
--- a/tensorflow/contrib/distribute/python/keras_backward_compat_test.py
+++ b/tensorflow/contrib/distribute/python/keras_backward_compat_test.py
@@ -316,15 +316,20 @@ def all_strategy_combinations():
   return strategy_minus_tpu_combinations() + tpu_strategy_combinations()
 
 
-# TODO(priyag): Add v2 optimizers here.
 def strategy_and_optimizer_combinations():
+  # TODO(b/122372746): Uncomment optimizers after they pass tests.
   return combinations.times(
       all_strategy_combinations(),
-      combinations.combine(
-          optimizer=[combinations.adagrad_optimizer_v1_fn,
-                     combinations.adam_optimizer_v1_fn,
-                     combinations.gradient_descent_optimizer_v1_fn,
-                     combinations.rmsprop_optimizer_v1_fn]))
+      combinations.combine(optimizer=[
+          combinations.adagrad_optimizer_v1_fn,
+          # combinations.adagrad_optimizer_keras_v2_fn,
+          combinations.adam_optimizer_v1_fn,
+          combinations.adam_optimizer_keras_v2_fn,
+          combinations.gradient_descent_optimizer_v1_fn,
+          combinations.gradient_descent_optimizer_keras_v2_fn,
+          combinations.rmsprop_optimizer_v1_fn,
+          # combinations.rmsprop_optimizer_keras_v2_fn
+      ]))
 
 
 def strategy_and_input_combinations():
diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 3f27e2547d..755ce23e7f 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -258,15 +258,20 @@ def all_strategy_combinations_minus_default():
   return strategy_minus_default_combinations + tpu_strategy_combinations()
 
 
-# TODO(priyag): Add v2 optimizers here.
 def strategy_and_optimizer_combinations():
+  # TODO(b/122372746): Uncomment optimizers after they pass tests.
   return combinations.times(
       all_strategy_combinations(),
-      combinations.combine(
-          optimizer=[combinations.adagrad_optimizer_v1_fn,
-                     combinations.adam_optimizer_v1_fn,
-                     combinations.gradient_descent_optimizer_v1_fn,
-                     combinations.rmsprop_optimizer_v1_fn]))
+      combinations.combine(optimizer=[
+          combinations.adagrad_optimizer_v1_fn,
+          # combinations.adagrad_optimizer_keras_v2_fn,
+          combinations.adam_optimizer_v1_fn,
+          combinations.adam_optimizer_keras_v2_fn,
+          combinations.gradient_descent_optimizer_v1_fn,
+          combinations.gradient_descent_optimizer_keras_v2_fn,
+          combinations.rmsprop_optimizer_v1_fn,
+          # combinations.rmsprop_optimizer_keras_v2_fn
+      ]))
 
 
 def strategy_for_numpy_input_combinations():
-- 
GitLab


From 4c0c199222fdcffbfb548aefa0ea82c853aea609 Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Fri, 25 Jan 2019 12:25:10 -0800
Subject: [PATCH 1284/2345] Add support for testing per channel quantization
 kernels.

PiperOrigin-RevId: 230947961
---
 tensorflow/lite/kernels/BUILD        |  1 +
 tensorflow/lite/kernels/test_util.cc |  7 ++-
 tensorflow/lite/kernels/test_util.h  | 67 +++++++++++++++++++++++++++-
 3 files changed, 72 insertions(+), 3 deletions(-)

diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index cb0d1f8746..8f7860242a 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -46,6 +46,7 @@ cc_library(
         "//tensorflow/lite:string_util",
         "//tensorflow/lite/kernels/internal:tensor_utils",
         "//tensorflow/lite/testing:util",
+        "//tensorflow/lite/tools/optimize:quantization_utils",
         "@com_google_googletest//:gtest",
     ],
 )
diff --git a/tensorflow/lite/kernels/test_util.cc b/tensorflow/lite/kernels/test_util.cc
index 19d7e37409..295204f62e 100644
--- a/tensorflow/lite/kernels/test_util.cc
+++ b/tensorflow/lite/kernels/test_util.cc
@@ -47,7 +47,12 @@ std::vector<Matcher<std::complex<float>>> ArrayComplex64Near(
 }
 
 int SingleOpModel::AddInput(const TensorData& t, bool is_variable) {
-  int id = AddTensor<float>(t, {}, is_variable);
+  int id = 0;
+  if (t.per_channel_quantization) {
+    id = AddTensorPerChannelQuant(t);
+  } else {
+    id = AddTensor<float>(t, {}, is_variable);
+  }
   inputs_.push_back(id);
   return id;
 }
diff --git a/tensorflow/lite/kernels/test_util.h b/tensorflow/lite/kernels/test_util.h
index 4a442f9fa7..c5435b3546 100644
--- a/tensorflow/lite/kernels/test_util.h
+++ b/tensorflow/lite/kernels/test_util.h
@@ -21,13 +21,14 @@ limitations under the License.
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
 
+#include "tensorflow/core/platform/logging.h"
 #include "tensorflow/lite/interpreter.h"
 #include "tensorflow/lite/kernels/internal/tensor_utils.h"
 #include "tensorflow/lite/kernels/register.h"
 #include "tensorflow/lite/model.h"
 #include "tensorflow/lite/string_util.h"
 #include "tensorflow/lite/testing/util.h"
-#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/lite/tools/optimize/quantization_utils.h"
 
 namespace tflite {
 
@@ -82,7 +83,7 @@ inline std::vector<float> Dequantize(const std::vector<T>& data, float scale,
 // A helper struct to construct test tensors. This is particularly useful for
 // quantized tensor which must have their scale and zero_point defined before
 // the actual data is known. This mimics what happens in practice: quantization
-// parameters are calculated during training.
+// parameters are calculated during training or post training..
 struct TensorData {
   TensorType type;
   std::vector<int> shape;
@@ -90,6 +91,10 @@ struct TensorData {
   float max;
   float scale;
   int32_t zero_point;
+  bool per_channel_quantization;
+  std::vector<float> per_channel_quantization_scales;
+  std::vector<int64_t> per_channel_quantization_offsets;
+  int32_t channel_index;
 };
 
 class SingleOpResolver : public OpResolver {
@@ -172,6 +177,46 @@ class SingleOpModel {
     PopulateTensor(index, /*offset=*/0, q.data(), q.data() + q.size());
   }
 
+  // Quantize and populate data for filter with per channel quantization.
+  void PerChannelSymmetricQuantizeAndPopulate(
+      int index, const std::vector<float>& input_data) {
+    TfLiteTensor* t = interpreter_->tensor(index);
+    auto* params =
+        reinterpret_cast<TfLiteAffineQuantization*>(t->quantization.params);
+    const int channel_index = params->quantized_dimension;
+
+    std::vector<int32_t> shape(t->dims->size);
+    for (int i = 0; i < shape.size(); ++i) {
+      shape[i] = t->dims->data[i];
+    }
+    const int32_t num_inputs = input_data.size();
+    const int32_t num_channel = shape[channel_index];
+    std::vector<int8_t> quantized_output(num_inputs);
+    std::vector<float> scales_inv(num_channel);
+    for (int i = 0; i < num_channel; ++i) {
+      scales_inv[i] = 1.0f / params->scale->data[i];
+    }
+    optimize::utils::SymmetricPerChannelQuantizeValues(
+        input_data.data(), scales_inv, shape, channel_index, &quantized_output);
+
+    PopulateTensor(index, /*offset=*/0, quantized_output.data(),
+                   quantized_output.data() + quantized_output.size());
+  }
+
+  // Quantize and populate data for bias with per channel quantization.
+  void PerChannelQuantizeBias(int index, const std::vector<float>& input_data) {
+    const int32_t num_inputs = input_data.size();
+    std::vector<int32_t> quantized_output(num_inputs);
+    TfLiteTensor* t = interpreter_->tensor(index);
+    auto* params =
+        reinterpret_cast<TfLiteAffineQuantization*>(t->quantization.params);
+    for (int i = 0; i < num_inputs; ++i) {
+      quantized_output[i] = input_data[i] * params->scale->data[i];
+    }
+    PopulateTensor(index, /*offset=*/0, quantized_output.data(),
+                   quantized_output.data() + quantized_output.size());
+  }
+
   const std::vector<int>& GetShape(int id) { return tensor_data_.at(id).shape; }
 
   float GetScale(int id) { return tensor_data_.at(id).scale; }
@@ -292,6 +337,24 @@ class SingleOpModel {
     return {scale, zero_point};
   }
 
+  int AddTensorPerChannelQuant(TensorData t) {
+    const int id = tensors_.size();
+    flatbuffers::Offset<QuantizationParameters> q_params = 0;
+    q_params = CreateQuantizationParameters(
+        builder_, /*min=*/0, /*max=*/0,
+        /*scale=*/
+        builder_.CreateVector<float>(t.per_channel_quantization_scales),
+        /*zero point=*/
+        builder_.CreateVector<int64_t>(t.per_channel_quantization_offsets),
+        QuantizationDetails_NONE, 0, t.channel_index);
+    tensors_.push_back(
+        CreateTensor(builder_, builder_.CreateVector<int>(t.shape), t.type,
+                     /*buffer=*/0,
+                     /*name=*/0, q_params, /*is_variable=*/false));
+    tensor_data_[id] = t;
+    return id;
+  }
+
   template <typename T>
   int AddTensor(TensorData t, std::initializer_list<T> data,
                 bool is_variable = false) {
-- 
GitLab


From bc641c859202ff7c5856dd8596fe10530f7403e3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 25 Jan 2019 12:33:19 -0800
Subject: [PATCH 1285/2345] Add correctness test for Siamese embedding model
 using TPUStrategy

PiperOrigin-RevId: 230949193
---
 .../python/keras_correctness_test_base.py     | 26 +++++--
 .../python/keras_dnn_correctness_test.py      |  2 +-
 .../keras_embedding_model_correctness_test.py | 75 +++++++++++++++++++
 .../engine/distributed_training_utils.py      |  5 --
 4 files changed, 97 insertions(+), 11 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_correctness_test_base.py b/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
index ed77e4fc74..0c783099b5 100644
--- a/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
+++ b/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
@@ -19,6 +19,7 @@ from __future__ import print_function
 
 from absl.testing import parameterized
 import numpy as np
+import six
 
 from tensorflow.contrib.distribute.python import combinations
 from tensorflow.contrib.distribute.python import mirrored_strategy
@@ -143,6 +144,19 @@ def get_batch_size(global_batch_size, distribution):
   return batch_size
 
 
+def get_data_size(data):
+  """Gets the size of data in list, tuple, dict, or a numpy array."""
+  assert isinstance(data, (np.ndarray, list, dict, tuple))
+
+  if isinstance(data, np.ndarray):
+    return len(data)
+
+  if isinstance(data, (list, tuple)):
+    return len(data[0])
+
+  return len(six.next(six.itervalues(data)))
+
+
 def get_correctness_test_inputs(use_numpy, use_validation_data,
                                 with_distribution, x_train, y_train, x_predict):
   """Generates the inputs for correctness check when enable Keras with DS."""
@@ -169,11 +183,12 @@ def get_correctness_test_inputs(use_numpy, use_validation_data,
           'y': y_train,
       }
     predict_inputs = {
-        'x': np.array(x_predict, dtype=np.float32),
+        'x': x_predict
     }
   else:
-    if len(x_train) < _GLOBAL_BATCH_SIZE * _EVAL_STEPS:
-      # Currently, we cannot detech the size of a dataset. So, the eval steps is
+    training_data_size = get_data_size(x_train)
+    if training_data_size < _GLOBAL_BATCH_SIZE * _EVAL_STEPS:
+      # Currently, we cannot detect the size of a dataset. So, the eval steps is
       # hard coded.
       raise ValueError('x_train must have at least '
                        '_GLOBAL_BATCH_SIZE * _EVAL_STEPS samples')
@@ -189,7 +204,7 @@ def get_correctness_test_inputs(use_numpy, use_validation_data,
         'y': None,
         'epochs': training_epochs,
         'shuffle': False,
-        'steps_per_epoch': len(x_train) // global_batch_size,
+        'steps_per_epoch': training_data_size // global_batch_size,
     }
     if use_validation_data:
       eval_inputs = None  # Remove the eval_inputs
@@ -205,7 +220,8 @@ def get_correctness_test_inputs(use_numpy, use_validation_data,
           'steps': _EVAL_STEPS,
       }
 
-    predict_batch_size = get_batch_size(len(x_predict), with_distribution)
+    predict_batch_size = get_batch_size(get_data_size(x_predict),
+                                        with_distribution)
     predict_dataset = dataset_ops.Dataset.from_tensor_slices(x_predict)
     predict_dataset = batch_wrapper(predict_dataset, predict_batch_size,
                                     with_distribution)
diff --git a/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py b/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
index 7afacab0dd..dae3218891 100644
--- a/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
+++ b/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
@@ -68,7 +68,7 @@ class TestDistributionStrategyDnnCorrectness(
     y_train = 3 * x_train
     x_train = x_train.astype('float32')
     y_train = y_train.astype('float32')
-    x_predict = [[1.], [2.], [3.], [4.]]
+    x_predict = np.array([[1.], [2.], [3.], [4.]], dtype=np.float32)
     return x_train, y_train, x_predict
 
   @combinations.generate(keras_correctness_test_base.
diff --git a/tensorflow/contrib/distribute/python/keras_embedding_model_correctness_test.py b/tensorflow/contrib/distribute/python/keras_embedding_model_correctness_test.py
index 3913f9bc0c..e881bb70ec 100644
--- a/tensorflow/contrib/distribute/python/keras_embedding_model_correctness_test.py
+++ b/tensorflow/contrib/distribute/python/keras_embedding_model_correctness_test.py
@@ -71,5 +71,80 @@ class DistributionStrategyEmbeddingModelCorrectnessTest(
     self.run_correctness_test(distribution, use_numpy, use_validation_data)
 
 
+class DistributionStrategySiameseEmbeddingModelCorrectnessTest(
+    keras_correctness_test_base.
+    TestDistributionStrategyEmbeddingModelCorrectnessBase):
+
+  def get_model(self, max_words=10, initial_weights=None, distribution=None):
+    with keras_correctness_test_base.MaybeDistributionScope(distribution):
+      word_ids_a = keras.layers.Input(
+          shape=(max_words,), dtype=np.int32, name='words_a')
+      word_ids_b = keras.layers.Input(
+          shape=(max_words,), dtype=np.int32, name='words_b')
+
+      def submodel(embedding, word_ids):
+        word_embed = embedding(word_ids)
+        rep = keras.layers.GlobalAveragePooling1D()(word_embed)
+        return keras.Model(inputs=[word_ids], outputs=[rep])
+
+      word_embed = keras.layers.Embedding(
+          input_dim=20,
+          output_dim=10,
+          input_length=max_words,
+          embeddings_initializer=keras.initializers.RandomUniform(0, 1))
+
+      a_rep = submodel(word_embed, word_ids_a).outputs[0]
+      b_rep = submodel(word_embed, word_ids_b).outputs[0]
+      sim = keras.layers.Dot(axes=1, normalize=True)([a_rep, b_rep])
+
+      model = keras.Model(inputs=[word_ids_a, word_ids_b], outputs=[sim])
+
+      if initial_weights:
+        model.set_weights(initial_weights)
+
+      model.compile(
+          optimizer=gradient_descent.GradientDescentOptimizer(
+              learning_rate=0.1),
+          loss='mse',
+          metrics=['mse'])
+    return model
+
+  def get_data(self,
+               count=(keras_correctness_test_base._GLOBAL_BATCH_SIZE *
+                      keras_correctness_test_base._EVAL_STEPS),
+               min_words=5,
+               max_words=10,
+               max_word_id=19,
+               num_classes=2):
+    features_a, labels_a, _ = (super(
+        DistributionStrategySiameseEmbeddingModelCorrectnessTest, self).
+                               get_data(count, min_words, max_words,
+                                        max_word_id, num_classes))
+
+    features_b, labels_b, _ = (super(
+        DistributionStrategySiameseEmbeddingModelCorrectnessTest, self).
+                               get_data(count, min_words, max_words,
+                                        max_word_id, num_classes))
+
+    y_train = np.zeros((count, 1), dtype=np.float32)
+    y_train[labels_a == labels_b] = 1.0
+    y_train[labels_a != labels_b] = -1.0
+    # TODO(b/123360757): Add tests for using list as inputs for multi-input
+    # models.
+    x_train = {
+        'words_a': features_a,
+        'words_b': features_b,
+    }
+    x_predict = x_train
+
+    return x_train, y_train, x_predict
+
+  @combinations.generate(keras_correctness_test_base.
+                         test_combinations_for_embedding_model())
+  def test_siamese_embedding_model_correctness(self, distribution, use_numpy,
+                                               use_validation_data):
+    self.run_correctness_test(distribution, use_numpy, use_validation_data)
+
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index d13e0b6fa9..512a629716 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -391,11 +391,6 @@ def validate_inputs(x, y, distribution_strategy):
     ValueError: if input is not a Dataset or a numpy array(when we use
       MirroredStrategy).
   """
-  if isinstance(x, dict) or isinstance(y, dict):
-    raise ValueError('`DistributionStrategy` does not support inputs of type '
-                     'dict. You must pass a `tf.data.Dataset` object or a '
-                     'numpy array as input.')
-
   if (isinstance(x, iterator_ops.Iterator) or
       isinstance(y, iterator_ops.Iterator)):
     raise ValueError('`DistributionStrategy` does not support inputs of type '
-- 
GitLab


From 2c16ced9b82dbad91d5a94b71586423114816c9a Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Fri, 25 Jan 2019 13:15:07 -0800
Subject: [PATCH 1286/2345] Adds v2 categorical crossentropy metric.

PiperOrigin-RevId: 230956073
---
 tensorflow/python/keras/metrics.py            | 206 ++++++++----------
 tensorflow/python/keras/metrics_test.py       | 130 ++++++++++-
 .../python/keras/saving/hdf5_format_test.py   |   9 +-
 ...as.metrics.-categorical-crossentropy.pbtxt | 199 +++++++++++++++++
 .../golden/v1/tensorflow.keras.metrics.pbtxt  |   4 +
 ...as.metrics.-categorical-crossentropy.pbtxt | 199 +++++++++++++++++
 .../golden/v2/tensorflow.keras.metrics.pbtxt  |   4 +
 7 files changed, 627 insertions(+), 124 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-crossentropy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-crossentropy.pbtxt

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 9a39dd4e17..d15fec49b6 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -30,6 +30,7 @@ from tensorflow.python.eager import function
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import tensor_util
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras.engine.base_layer import Layer
 from tensorflow.python.keras.losses import binary_crossentropy
@@ -135,8 +136,8 @@ class Metric(Layer):
   ```
   """
 
-  def __init__(self, name=None, dtype=None):
-    super(Metric, self).__init__(name=name, dtype=dtype)
+  def __init__(self, name=None, dtype=None, **kwargs):
+    super(Metric, self).__init__(name=name, dtype=dtype, **kwargs)
     self.stateful = True  # All metric layers are stateful.
     self.built = True
     self._dtype = K.floatx() if dtype is None else dtypes.as_dtype(dtype).name
@@ -190,6 +191,14 @@ class Metric(Layer):
         result_t._metric_obj = self  # pylint: disable=protected-access
       return result_t
 
+  @property
+  def dtype(self):
+    return self._dtype
+
+  def get_config(self):
+    """Returns the serializable config of the metric."""
+    return {'name': self.name, 'dtype': self.dtype}
+
   def reset_states(self):
     """Resets all of the metric state variables.
 
@@ -229,12 +238,6 @@ class Metric(Layer):
     """
     NotImplementedError('Must be implemented in subclasses.')
 
-  @classmethod
-  def from_config(cls, config):
-    if 'trainable' in config:
-      config.pop('trainable')
-    return cls(**config)
-
   ### For use by subclasses ###
   @doc_controls.for_subclass_implementers
   def add_weight(self,
@@ -506,7 +509,8 @@ class MeanRelativeError(Mean):
         relative_errors, sample_weight=sample_weight)
 
   def get_config(self):
-    config = {'normalizer': self.normalizer}
+    n = self.normalizer
+    config = {'normalizer': K.eval(n) if _is_tensor_or_variable(n) else n}
     base_config = super(MeanRelativeError, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
 
@@ -553,8 +557,9 @@ class MeanMetricWrapper(Mean):
         matches, sample_weight=sample_weight)
 
   def get_config(self):
-    config = {'fn': self._fn}
-    config.update(self._fn_kwargs)
+    config = {}
+    for k, v in six.iteritems(self._fn_kwargs):
+      config[k] = K.eval(v) if _is_tensor_or_variable(v) else v
     base_config = super(MeanMetricWrapper, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
 
@@ -594,12 +599,6 @@ class Accuracy(MeanMetricWrapper):
   def __init__(self, name='accuracy', dtype=None):
     super(Accuracy, self).__init__(accuracy, name, dtype=dtype)
 
-  @classmethod
-  def from_config(cls, config):
-    if 'fn' in config:
-      config.pop('fn')
-    return super(Accuracy, cls).from_config(config)
-
 
 @keras_export('keras.metrics.BinaryAccuracy')
 class BinaryAccuracy(MeanMetricWrapper):
@@ -645,12 +644,6 @@ class BinaryAccuracy(MeanMetricWrapper):
     super(BinaryAccuracy, self).__init__(
         binary_accuracy, name, dtype=dtype, threshold=threshold)
 
-  @classmethod
-  def from_config(cls, config):
-    if 'fn' in config:
-      config.pop('fn')
-    return super(BinaryAccuracy, cls).from_config(config)
-
 
 @keras_export('keras.metrics.CategoricalAccuracy')
 class CategoricalAccuracy(MeanMetricWrapper):
@@ -701,12 +694,6 @@ class CategoricalAccuracy(MeanMetricWrapper):
     super(CategoricalAccuracy, self).__init__(
         categorical_accuracy, name, dtype=dtype)
 
-  @classmethod
-  def from_config(cls, config):
-    if 'fn' in config:
-      config.pop('fn')
-    return super(CategoricalAccuracy, cls).from_config(config)
-
 
 @keras_export('keras.metrics.SparseCategoricalAccuracy')
 class SparseCategoricalAccuracy(MeanMetricWrapper):
@@ -748,12 +735,6 @@ class SparseCategoricalAccuracy(MeanMetricWrapper):
     super(SparseCategoricalAccuracy, self).__init__(
         sparse_categorical_accuracy, name, dtype=dtype)
 
-  @classmethod
-  def from_config(cls, config):
-    if 'fn' in config:
-      config.pop('fn')
-    return super(SparseCategoricalAccuracy, cls).from_config(config)
-
 
 class TopKCategoricalAccuracy(MeanMetricWrapper):
   """Computes how often targets are in the top `K` predictions.
@@ -786,12 +767,6 @@ class TopKCategoricalAccuracy(MeanMetricWrapper):
     super(TopKCategoricalAccuracy, self).__init__(
         top_k_categorical_accuracy, name, dtype=dtype, k=k)
 
-  @classmethod
-  def from_config(cls, config):
-    if 'fn' in config:
-      config.pop('fn')
-    return super(TopKCategoricalAccuracy, cls).from_config(config)
-
 
 class SparseTopKCategoricalAccuracy(MeanMetricWrapper):
   """Computes how often integer targets are in the top `K` predictions.
@@ -826,12 +801,6 @@ class SparseTopKCategoricalAccuracy(MeanMetricWrapper):
     super(SparseTopKCategoricalAccuracy, self).__init__(
         sparse_top_k_categorical_accuracy, name, dtype=dtype, k=k)
 
-  @classmethod
-  def from_config(cls, config):
-    if 'fn' in config:
-      config.pop('fn')
-    return super(SparseTopKCategoricalAccuracy, cls).from_config(config)
-
 
 class _ConfusionMatrixConditionCount(Metric):
   """Calculates the number of the given confusion matrix condition."""
@@ -1877,12 +1846,6 @@ class CosineProximity(MeanMetricWrapper):
     """
     super(CosineProximity, self).__init__(cosine, name, dtype=dtype, axis=axis)
 
-  @classmethod
-  def from_config(cls, config):
-    if 'fn' in config:
-      config.pop('fn')
-    return super(CosineProximity, cls).from_config(config)
-
 
 @keras_export('keras.metrics.MeanAbsoluteError')
 class MeanAbsoluteError(MeanMetricWrapper):
@@ -1910,12 +1873,6 @@ class MeanAbsoluteError(MeanMetricWrapper):
     super(MeanAbsoluteError, self).__init__(
         mean_absolute_error, name, dtype=dtype)
 
-  @classmethod
-  def from_config(cls, config):
-    if 'fn' in config:
-      config.pop('fn')
-    return super(MeanAbsoluteError, cls).from_config(config)
-
 
 @keras_export('keras.metrics.MeanAbsolutePercentageError')
 class MeanAbsolutePercentageError(MeanMetricWrapper):
@@ -1944,12 +1901,6 @@ class MeanAbsolutePercentageError(MeanMetricWrapper):
     super(MeanAbsolutePercentageError, self).__init__(
         mean_absolute_percentage_error, name, dtype=dtype)
 
-  @classmethod
-  def from_config(cls, config):
-    if 'fn' in config:
-      config.pop('fn')
-    return super(MeanAbsolutePercentageError, cls).from_config(config)
-
 
 @keras_export('keras.metrics.MeanSquaredError')
 class MeanSquaredError(MeanMetricWrapper):
@@ -1978,12 +1929,6 @@ class MeanSquaredError(MeanMetricWrapper):
     super(MeanSquaredError, self).__init__(
         mean_squared_error, name, dtype=dtype)
 
-  @classmethod
-  def from_config(cls, config):
-    if 'fn' in config:
-      config.pop('fn')
-    return super(MeanSquaredError, cls).from_config(config)
-
 
 @keras_export('keras.metrics.MeanSquaredLogarithmicError')
 class MeanSquaredLogarithmicError(MeanMetricWrapper):
@@ -2012,12 +1957,6 @@ class MeanSquaredLogarithmicError(MeanMetricWrapper):
     super(MeanSquaredLogarithmicError, self).__init__(
         mean_squared_logarithmic_error, name, dtype=dtype)
 
-  @classmethod
-  def from_config(cls, config):
-    if 'fn' in config:
-      config.pop('fn')
-    return super(MeanSquaredLogarithmicError, cls).from_config(config)
-
 
 @keras_export('keras.metrics.Hinge')
 class Hinge(MeanMetricWrapper):
@@ -2045,12 +1984,6 @@ class Hinge(MeanMetricWrapper):
   def __init__(self, name='hinge', dtype=None):
     super(Hinge, self).__init__(hinge, name, dtype=dtype)
 
-  @classmethod
-  def from_config(cls, config):
-    if 'fn' in config:
-      config.pop('fn')
-    return super(Hinge, cls).from_config(config)
-
 
 @keras_export('keras.metrics.SquaredHinge')
 class SquaredHinge(MeanMetricWrapper):
@@ -2078,12 +2011,6 @@ class SquaredHinge(MeanMetricWrapper):
   def __init__(self, name='squared_hinge', dtype=None):
     super(SquaredHinge, self).__init__(squared_hinge, name, dtype=dtype)
 
-  @classmethod
-  def from_config(cls, config):
-    if 'fn' in config:
-      config.pop('fn')
-    return super(SquaredHinge, cls).from_config(config)
-
 
 @keras_export('keras.metrics.CategoricalHinge')
 class CategoricalHinge(MeanMetricWrapper):
@@ -2111,12 +2038,6 @@ class CategoricalHinge(MeanMetricWrapper):
   def __init__(self, name='categorical_hinge', dtype=None):
     super(CategoricalHinge, self).__init__(categorical_hinge, name, dtype=dtype)
 
-  @classmethod
-  def from_config(cls, config):
-    if 'fn' in config:
-      config.pop('fn')
-    return super(CategoricalHinge, cls).from_config(config)
-
 
 class RootMeanSquaredError(Mean):
   """Computes root mean squared error metric between `y_true` and `y_pred`.
@@ -2189,12 +2110,6 @@ class Logcosh(MeanMetricWrapper):
   def __init__(self, name='logcosh', dtype=None):
     super(Logcosh, self).__init__(logcosh, name, dtype=dtype)
 
-  @classmethod
-  def from_config(cls, config):
-    if 'fn' in config:
-      config.pop('fn')
-    return super(Logcosh, cls).from_config(config)
-
 
 class Poisson(MeanMetricWrapper):
   """Computes the poisson metric between `y_true` and `y_pred`.
@@ -2220,12 +2135,6 @@ class Poisson(MeanMetricWrapper):
   def __init__(self, name='poisson', dtype=None):
     super(Poisson, self).__init__(poisson, name, dtype=dtype)
 
-  @classmethod
-  def from_config(cls, config):
-    if 'fn' in config:
-      config.pop('fn')
-    return super(Poisson, cls).from_config(config)
-
 
 class KullbackLeiblerDivergence(MeanMetricWrapper):
   """Computes kullback leibler divergence metric between `y_true` and `y_pred`.
@@ -2252,12 +2161,6 @@ class KullbackLeiblerDivergence(MeanMetricWrapper):
     super(KullbackLeiblerDivergence, self).__init__(
         kullback_leibler_divergence, name, dtype=dtype)
 
-  @classmethod
-  def from_config(cls, config):
-    if 'fn' in config:
-      config.pop('fn')
-    return super(KullbackLeiblerDivergence, cls).from_config(config)
-
 
 class MeanIoU(Metric):
   """Computes the mean Intersection-Over-Union metric.
@@ -2494,7 +2397,10 @@ class MeanTensor(Metric):
 
 
 class BinaryCrossentropy(MeanMetricWrapper):
-  """Computes the binary crossentropy between `y_true` and `y_pred`.
+  """Computes the crossentropy metric between the labels and predictions.
+
+  This is the crossentropy metric class to be used when there are only two
+  label classes (0 and 1).
 
   Usage:
 
@@ -2552,11 +2458,69 @@ class BinaryCrossentropy(MeanMetricWrapper):
         from_logits=from_logits,
         label_smoothing=label_smoothing)
 
-  @classmethod
-  def from_config(cls, config):
-    if 'fn' in config:
-      config.pop('fn')
-    return super(BinaryCrossentropy, cls).from_config(config)
+
+@keras_export('keras.metrics.CategoricalCrossentropy')
+class CategoricalCrossentropy(MeanMetricWrapper):
+  """Computes the crossentropy metric between the labels and predictions.
+
+  This is the crossentropy metric class to be used when there are multiple
+  label classes (2 or more). Here we assume that labels are given as a `one_hot`
+  representation. eg., When labels values are [2, 0, 1],
+   `y_true` = [[0, 0, 1], [1, 0, 0], [0, 1, 0]].
+
+  Usage:
+
+  ```python
+  m = tf.keras.metrics.CategoricalCrossentropy()
+  m.update_state([[0, 1, 0], [0, 0, 1]],
+                 [[0.05, 0.95, 0], [0.1, 0.8, 0.1]])
+
+  # EPSILON = 1e-7, y = y_true, y` = y_pred
+  # y` = clip_ops.clip_by_value(output, EPSILON, 1. - EPSILON)
+  # y` = [[0.05, 0.95, EPSILON], [0.1, 0.8, 0.1]]
+
+  # xent = -sum(y * log(y'), axis = -1)
+  #      = -((log 0.95), (log 0.1))
+  #      = [0.051, 2.302]
+  # Reduced xent = (0.051 + 2.302) / 2
+
+  print('Final result: ', m.result().numpy())  # Final result: 1.176
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile(
+    'sgd',
+    loss='mse',
+    metrics=[tf.keras.metrics.CategoricalCrossentropy()])
+  ```
+
+  Args:
+    name: (Optional) string name of the metric instance.
+    dtype: (Optional) data type of the metric result.
+    from_logits: (Optional ) Whether `y_pred` is expected to be a logits tensor.
+      By default, we assume that `y_pred` encodes a probability distribution.
+    label_smoothing: Float in [0, 1]. When > 0, label values are smoothed,
+      meaning the confidence on label values are relaxed. e.g.
+      `label_smoothing=0.2` means that we will use a value of `0.1` for label
+      `0` and `0.9` for label `1`"
+  """
+
+  def __init__(self,
+               name='categorical_crossentropy',
+               dtype=None,
+               from_logits=False,
+               label_smoothing=0):
+    label_smoothing = ops.convert_to_tensor(label_smoothing, dtype=K.floatx())
+
+    super(CategoricalCrossentropy, self).__init__(
+        categorical_crossentropy,
+        name,
+        dtype=dtype,
+        from_logits=from_logits,
+        label_smoothing=label_smoothing)
 
 
 def accuracy(y_true, y_pred):
@@ -2666,3 +2630,7 @@ def get(identifier):
   else:
     raise ValueError('Could not interpret '
                      'metric function identifier: %s' % identifier)
+
+
+def _is_tensor_or_variable(x):
+  return tensor_util.is_tensor(x) or isinstance(x, tf_variables.Variable)
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index b5af18ce27..6ed9593b84 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -1327,14 +1327,14 @@ class BinaryCrossentropyTest(test.TestCase):
     self.assertEqual(bce_obj._dtype, dtypes.int32)
 
     old_config = bce_obj.get_config()
-    self.assertAllClose(self.evaluate(old_config['label_smoothing']), 0.2, 1e-3)
+    self.assertAllClose(old_config['label_smoothing'], 0.2, 1e-3)
 
     # Check save and restore config
     bce_obj2 = metrics.BinaryCrossentropy.from_config(old_config)
     self.assertEqual(bce_obj2.name, 'bce')
     self.assertEqual(bce_obj2._dtype, dtypes.int32)
     new_config = bce_obj2.get_config()
-    self.assertAllClose(self.evaluate(new_config['label_smoothing']), 0.2, 1e-3)
+    self.assertDictEqual(old_config, new_config)
 
   def test_unweighted(self):
     bce_obj = metrics.BinaryCrossentropy()
@@ -1437,6 +1437,132 @@ class BinaryCrossentropyTest(test.TestCase):
     self.assertAllClose(expected_value, self.evaluate(result), atol=1e-3)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class CategoricalCrossentropyTest(test.TestCase):
+
+  def test_config(self):
+    cce_obj = metrics.CategoricalCrossentropy(
+        name='cce', dtype=dtypes.int32, label_smoothing=0.2)
+    self.assertEqual(cce_obj.name, 'cce')
+    self.assertEqual(cce_obj._dtype, dtypes.int32)
+
+    old_config = cce_obj.get_config()
+    self.assertAllClose(old_config['label_smoothing'], 0.2, 1e-3)
+
+    # Check save and restore config
+    cce_obj2 = metrics.CategoricalCrossentropy.from_config(old_config)
+    self.assertEqual(cce_obj2.name, 'cce')
+    self.assertEqual(cce_obj2._dtype, dtypes.int32)
+    new_config = cce_obj2.get_config()
+    self.assertDictEqual(old_config, new_config)
+
+  def test_unweighted(self):
+    cce_obj = metrics.CategoricalCrossentropy()
+    self.evaluate(variables.variables_initializer(cce_obj.variables))
+
+    y_true = np.asarray([[0, 1, 0], [0, 0, 1]])
+    y_pred = np.asarray([[0.05, 0.95, 0], [0.1, 0.8, 0.1]])
+    result = cce_obj(y_true, y_pred)
+
+    # EPSILON = 1e-7, y = y_true, y` = y_pred
+    # y` = clip_ops.clip_by_value(output, EPSILON, 1. - EPSILON)
+    # y` = [[0.05, 0.95, EPSILON], [0.1, 0.8, 0.1]]
+
+    # Metric = -sum(y * log(y'), axis = -1)
+    #        = -((log 0.95), (log 0.1))
+    #        = [0.051, 2.302]
+    # Reduced metric = (0.051 + 2.302) / 2
+
+    self.assertAllClose(self.evaluate(result), 1.176, atol=1e-3)
+
+  def test_unweighted_from_logits(self):
+    cce_obj = metrics.CategoricalCrossentropy(from_logits=True)
+    self.evaluate(variables.variables_initializer(cce_obj.variables))
+
+    y_true = np.asarray([[0, 1, 0], [0, 0, 1]])
+    logits = np.asarray([[1, 9, 0], [1, 8, 1]], dtype=np.float32)
+    result = cce_obj(y_true, logits)
+
+    # softmax = exp(logits) / sum(exp(logits), axis=-1)
+    # xent = -sum(labels * log(softmax), 1)
+
+    # exp(logits) = [[2.718, 8103.084, 1], [2.718, 2980.958, 2.718]]
+    # sum(exp(logits), axis=-1) = [8106.802, 2986.394]
+    # softmax = [[0.00033, 0.99954, 0.00012], [0.00091, 0.99817, 0.00091]]
+    # log(softmax) = [[-8.00045, -0.00045, -9.00045],
+    #                 [-7.00182, -0.00182, -7.00182]]
+    # labels * log(softmax) = [[0, -0.00045, 0], [0, 0, -7.00182]]
+    # xent = [0.00045, 7.00182]
+    # Reduced xent = (0.00045 + 7.00182) / 2
+
+    self.assertAllClose(self.evaluate(result), 3.5011, atol=1e-3)
+
+  def test_weighted(self):
+    cce_obj = metrics.CategoricalCrossentropy()
+    self.evaluate(variables.variables_initializer(cce_obj.variables))
+
+    y_true = np.asarray([[0, 1, 0], [0, 0, 1]])
+    y_pred = np.asarray([[0.05, 0.95, 0], [0.1, 0.8, 0.1]])
+    sample_weight = constant_op.constant([1.5, 2.])
+    result = cce_obj(y_true, y_pred, sample_weight=sample_weight)
+
+    # EPSILON = 1e-7, y = y_true, y` = y_pred
+    # y` = clip_ops.clip_by_value(output, EPSILON, 1. - EPSILON)
+    # y` = [[0.05, 0.95, EPSILON], [0.1, 0.8, 0.1]]
+
+    # Metric = -sum(y * log(y'), axis = -1)
+    #        = -((log 0.95), (log 0.1))
+    #        = [0.051, 2.302]
+    # Weighted metric = [0.051 * 1.5, 2.302 * 2.]
+    # Reduced metric = (0.051 * 1.5 + 2.302 * 2.) / 3.5
+
+    self.assertAllClose(self.evaluate(result), 1.338, atol=1e-3)
+
+  def test_weighted_from_logits(self):
+    cce_obj = metrics.CategoricalCrossentropy(from_logits=True)
+    self.evaluate(variables.variables_initializer(cce_obj.variables))
+
+    y_true = np.asarray([[0, 1, 0], [0, 0, 1]])
+    logits = np.asarray([[1, 9, 0], [1, 8, 1]], dtype=np.float32)
+    sample_weight = constant_op.constant([1.5, 2.])
+    result = cce_obj(y_true, logits, sample_weight=sample_weight)
+
+    # softmax = exp(logits) / sum(exp(logits), axis=-1)
+    # xent = -sum(labels * log(softmax), 1)
+    # xent = [0.00045, 7.00182]
+    # weighted xent = [0.000675, 14.00364]
+    # Reduced xent = (0.000675 + 14.00364) / (1.5 + 2)
+
+    self.assertAllClose(self.evaluate(result), 4.0012, atol=1e-3)
+
+  def test_label_smoothing(self):
+    y_true = np.asarray([[0, 1, 0], [0, 0, 1]])
+    logits = np.asarray([[1, 9, 0], [1, 8, 1]], dtype=np.float32)
+    label_smoothing = 0.1
+
+    # Label smoothing: z' = z * (1 - L) + L/n,
+    #     where L = label smoothing value and n = num classes
+    # Label value 1 becomes: 1 - L + L/n
+    # Label value 0 becomes: L/n
+    # y_true with label_smoothing = [[0.0333, 0.9333, 0.0333],
+    #                               [0.0333, 0.0333, 0.9333]]
+
+    # softmax = exp(logits) / sum(exp(logits), axis=-1)
+    # xent = -sum(labels * log(softmax), 1)
+    # log(softmax) = [[-8.00045, -0.00045, -9.00045],
+    #                 [-7.00182, -0.00182, -7.00182]]
+    # labels * log(softmax) = [[-0.26641, -0.00042, -0.29971],
+    #                          [-0.23316, -0.00006, -6.53479]]
+    # xent = [0.56654, 6.76801]
+    # Reduced xent = (0.56654 + 6.76801) / 2
+
+    cce_obj = metrics.CategoricalCrossentropy(
+        from_logits=True, label_smoothing=label_smoothing)
+    self.evaluate(variables.variables_initializer(cce_obj.variables))
+    loss = cce_obj(y_true, logits)
+    self.assertAllClose(self.evaluate(loss), 3.667, atol=1e-3)
+
+
 def _get_model(compile_metrics):
   model_layers = [
       layers.Dense(3, activation='relu', kernel_initializer='ones'),
diff --git a/tensorflow/python/keras/saving/hdf5_format_test.py b/tensorflow/python/keras/saving/hdf5_format_test.py
index a0c0ed3847..c51eecf4a3 100644
--- a/tensorflow/python/keras/saving/hdf5_format_test.py
+++ b/tensorflow/python/keras/saving/hdf5_format_test.py
@@ -348,13 +348,16 @@ class TestWholeModelSaving(test.TestCase):
           optimizer=keras.optimizers.RMSprop(lr=0.0001),
           metrics=[
               keras.metrics.categorical_accuracy,
-              keras.metrics.CategoricalAccuracy()
+              keras.metrics.CategoricalCrossentropy(
+                  name='cce', label_smoothing=constant_op.constant(0.2)),
           ],
           weighted_metrics=[
-              keras.metrics.categorical_accuracy,
-              keras.metrics.CategoricalAccuracy()
+              keras.metrics.categorical_crossentropy,
+              keras.metrics.CategoricalCrossentropy(
+                  name='cce', label_smoothing=constant_op.constant(0.2)),
           ],
           sample_weight_mode='temporal')
+
       x = np.random.random((1, 3))
       y = np.random.random((1, 3, 3))
       model.train_on_batch(x, y)
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-crossentropy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-crossentropy.pbtxt
new file mode 100644
index 0000000000..8cd173c7eb
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-crossentropy.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.keras.metrics.CategoricalCrossentropy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.CategoricalCrossentropy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\', \'from_logits\', \'label_smoothing\'], varargs=None, keywords=None, defaults=[\'categorical_crossentropy\', \'None\', \'False\', \'0\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
index d65e591b54..0e32de336e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
@@ -12,6 +12,10 @@ tf_module {
     name: "CategoricalAccuracy"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "CategoricalCrossentropy"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "CategoricalHinge"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-crossentropy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-crossentropy.pbtxt
new file mode 100644
index 0000000000..8cd173c7eb
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-crossentropy.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.keras.metrics.CategoricalCrossentropy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.CategoricalCrossentropy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\', \'from_logits\', \'label_smoothing\'], varargs=None, keywords=None, defaults=[\'categorical_crossentropy\', \'None\', \'False\', \'0\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
index d65e591b54..0e32de336e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
@@ -12,6 +12,10 @@ tf_module {
     name: "CategoricalAccuracy"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "CategoricalCrossentropy"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "CategoricalHinge"
     mtype: "<type \'type\'>"
-- 
GitLab


From 72d38c33ec43e8aad139d985d1fa78687e31c55a Mon Sep 17 00:00:00 2001
From: Andrew Cotter <acotter@google.com>
Date: Fri, 25 Jan 2019 13:15:55 -0800
Subject: [PATCH 1287/2345] Fixing candidates_test failures (it was much too
 strict).

PiperOrigin-RevId: 230956182
---
 tensorflow/contrib/constrained_optimization/BUILD           | 6 ------
 .../constrained_optimization/python/candidates_test.py      | 4 ++--
 2 files changed, 2 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/constrained_optimization/BUILD b/tensorflow/contrib/constrained_optimization/BUILD
index 6ca36f354f..619153df67 100644
--- a/tensorflow/contrib/constrained_optimization/BUILD
+++ b/tensorflow/contrib/constrained_optimization/BUILD
@@ -42,12 +42,6 @@ py_test(
     name = "candidates_test",
     srcs = ["python/candidates_test.py"],
     srcs_version = "PY2AND3",
-    tags = [
-        # TODO(b/121223093): Re-enable this test after fixing "Distribution
-        # should match known solution" errors.
-        "no_mac",
-        "no_oss",
-    ],
     deps = [
         ":constrained_optimization",
         "//tensorflow/python:client_testlib",
diff --git a/tensorflow/contrib/constrained_optimization/python/candidates_test.py b/tensorflow/contrib/constrained_optimization/python/candidates_test.py
index a4c49d48bc..280e9acd88 100644
--- a/tensorflow/contrib/constrained_optimization/python/candidates_test.py
+++ b/tensorflow/contrib/constrained_optimization/python/candidates_test.py
@@ -52,12 +52,12 @@ class CandidatesTest(test.TestCase):
     distribution = candidates.find_best_candidate_distribution(
         objective_vector, constraints_matrix)
     # Verify that the solution is a probability distribution.
-    self.assertTrue(np.all(distribution >= 0))
+    self.assertTrue(np.all(distribution >= -1e-6))
     self.assertAlmostEqual(np.sum(distribution), 1.0)
     # Verify that the solution satisfies the constraints.
     maximum_constraint_violation = np.amax(
         np.dot(constraints_matrix, distribution))
-    self.assertLessEqual(maximum_constraint_violation, 0)
+    self.assertLessEqual(maximum_constraint_violation, 1e-6)
     # Verify that the solution matches that which we expect.
     expected_distribution = np.array([0.37872711, 0.62127289, 0, 0])
     self.assertAllClose(expected_distribution, distribution, rtol=0, atol=1e-6)
-- 
GitLab


From 1a489ccbde5f536da069dc3d24e79d42e406cf6b Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Fri, 25 Jan 2019 13:23:18 -0800
Subject: [PATCH 1288/2345] [tf.data] Add `output_types` and `output_shapes`
 attr to ExperimentalChooseFastestDataset

PiperOrigin-RevId: 230957401
---
 .../experimental/choose_fastest_dataset_op.cc | 65 ++++++++++---------
 .../core/ops/compat/ops_history.v1.pbtxt      | 21 ++++++
 .../core/ops/experimental_dataset_ops.cc      |  2 +
 .../data/experimental/ops/optimization.py     |  4 +-
 4 files changed, 61 insertions(+), 31 deletions(-)

diff --git a/tensorflow/core/kernels/data/experimental/choose_fastest_dataset_op.cc b/tensorflow/core/kernels/data/experimental/choose_fastest_dataset_op.cc
index f66d5d9955..e63208f26a 100644
--- a/tensorflow/core/kernels/data/experimental/choose_fastest_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/choose_fastest_dataset_op.cc
@@ -31,6 +31,8 @@ class ChooseFastestDatasetOp : public DatasetOpKernel {
   explicit ChooseFastestDatasetOp(OpKernelConstruction* ctx)
       : DatasetOpKernel(ctx) {
     OP_REQUIRES_OK(ctx, ctx->GetAttr("num_experiments", &num_experiments_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("output_types", &output_types_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("output_shapes", &output_shapes_));
   }
 
   void MakeDataset(OpKernelContext* ctx, DatasetBase** output) override {
@@ -49,45 +51,43 @@ class ChooseFastestDatasetOp : public DatasetOpKernel {
       inputs.push_back(input);
     }
 
-    const DataTypeVector& output_types = inputs[0]->output_dtypes();
     for (size_t i = 1, num_inputs = inputs.size(); i < num_inputs; ++i) {
-      OP_REQUIRES(ctx, inputs[i]->output_dtypes() == output_types,
-                  errors::InvalidArgument(
-                      "All inputs to ChooseFastestDataset "
-                      "must have the same output types. Input ",
-                      i, " has output types: ",
-                      DataTypeVectorString(inputs[i]->output_dtypes()),
-                      ", while all prior inputs have types: ",
-                      DataTypeVectorString(output_types), "."));
+      OP_REQUIRES(
+          ctx, inputs[i]->output_dtypes() == output_types_,
+          errors::InvalidArgument(
+              "All inputs to ChooseFastestDataset "
+              "must have the same output types. Input ",
+              i, " has output types: ",
+              DataTypeVectorString(inputs[i]->output_dtypes()),
+              ". Expected: ", DataTypeVectorString(output_types_), "."));
     }
 
-    std::vector<PartialTensorShape> output_shapes = inputs[0]->output_shapes();
     // Merge the output shapes of all the input datasets, returning an
     // error if any of them are incompatible.
     for (size_t i = 1, num_inputs = inputs.size(); i < num_inputs; ++i) {
       OP_REQUIRES(
-          ctx, inputs[i]->output_shapes().size() == output_shapes.size(),
+          ctx, inputs[i]->output_shapes().size() == output_shapes_.size(),
           errors::InvalidArgument(
               "All inputs to ChooseFastestDataset must have compatible outputs."
               " Input ",
               i, " has ", inputs[i]->output_shapes().size(),
-              " components, while all prior inputs have ", output_shapes.size(),
+              " components. Expected to have ", output_shapes_.size(),
               " components."));
-      for (size_t j = 0, num_components = output_shapes.size();
+      for (size_t j = 0, num_components = output_shapes_.size();
            j < num_components; ++j) {
         PartialTensorShape result;
-        OP_REQUIRES(
-            ctx,
-            output_shapes[j]
-                .MergeWith(inputs[i]->output_shapes().at(j), &result)
-                .ok(),
-            errors::InvalidArgument(
-                "All inputs to ChooseFastestDataset must have "
-                "compatible output shapes. Component ",
-                j, " of input ", i, " has shape: ",
-                inputs[i]->output_shapes().at(j), ", while components ", j,
-                " of all prior inputs have shape: ", output_shapes[j], "."));
-        output_shapes[j] = std::move(result);
+        OP_REQUIRES(ctx,
+                    output_shapes_[j]
+                        .MergeWith(inputs[i]->output_shapes().at(j), &result)
+                        .ok(),
+                    errors::InvalidArgument(
+                        "All inputs to ChooseFastestDataset must have "
+                        "compatible output shapes. Component ",
+                        j, " of input ", i,
+                        " has shape: ", inputs[i]->output_shapes().at(j),
+                        ". Expected to be compatible with shape: ",
+                        output_shapes_[j], "."));
+        output_shapes_[j] = std::move(result);
       }
     }
 
@@ -108,7 +108,7 @@ class ChooseFastestDatasetOp : public DatasetOpKernel {
                 "."));
       }
     }
-    *output = new Dataset(ctx, std::move(inputs), std::move(output_shapes),
+    *output = new Dataset(ctx, std::move(inputs), output_types_, output_shapes_,
                           cardinality, num_experiments_);
   }
 
@@ -116,11 +116,13 @@ class ChooseFastestDatasetOp : public DatasetOpKernel {
   class Dataset : public DatasetBase {
    public:
     Dataset(OpKernelContext* ctx, std::vector<DatasetBase*> inputs,
-            std::vector<PartialTensorShape> output_shapes, int64 cardinality,
-            int64 num_experiments)
+            const DataTypeVector& output_types,
+            const std::vector<PartialTensorShape>& output_shapes,
+            int64 cardinality, int64 num_experiments)
         : DatasetBase(DatasetContext(ctx)),
           inputs_(std::move(inputs)),
-          output_shapes_(std::move(output_shapes)),
+          output_types_(output_types),
+          output_shapes_(output_shapes),
           cardinality_(cardinality),
           num_experiments_(num_experiments) {
       for (auto input : inputs_) {
@@ -142,7 +144,7 @@ class ChooseFastestDatasetOp : public DatasetOpKernel {
     }
 
     const DataTypeVector& output_dtypes() const override {
-      return inputs_[0]->output_dtypes();
+      return output_types_;
     }
 
     const std::vector<PartialTensorShape>& output_shapes() const override {
@@ -328,12 +330,15 @@ class ChooseFastestDatasetOp : public DatasetOpKernel {
     };  // class Iterator
 
     const std::vector<DatasetBase*> inputs_;
+    const DataTypeVector output_types_;
     const std::vector<PartialTensorShape> output_shapes_;
     const int64 cardinality_;
     const int64 num_experiments_;
   };  // class Dataset
 
   int64 num_experiments_;
+  DataTypeVector output_types_;
+  std::vector<PartialTensorShape> output_shapes_;
 };  // class ChooseFastestDatasetOp
 
 // Register the kernel implementation for ChooseFastestDataset.
diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index eee5af141e..6fae92cb99 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -22296,6 +22296,27 @@ op {
     name: "num_experiments"
     type: "int"
   }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_STRING
+      }
+    }
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
 }
 op {
   name: "ExperimentalDatasetCardinality"
diff --git a/tensorflow/core/ops/experimental_dataset_ops.cc b/tensorflow/core/ops/experimental_dataset_ops.cc
index 2735618c61..316e405188 100644
--- a/tensorflow/core/ops/experimental_dataset_ops.cc
+++ b/tensorflow/core/ops/experimental_dataset_ops.cc
@@ -468,6 +468,8 @@ REGISTER_OP("ExperimentalChooseFastestDataset")
     .Output("handle: variant")
     .Attr("N: int >= 2")
     .Attr("num_experiments: int")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")
     .SetShapeFn(shape_inference::ScalarShape);
 
 REGISTER_OP("ExperimentalIdentityIndexedDataset")
diff --git a/tensorflow/python/data/experimental/ops/optimization.py b/tensorflow/python/data/experimental/ops/optimization.py
index 75769b8998..984c820b17 100644
--- a/tensorflow/python/data/experimental/ops/optimization.py
+++ b/tensorflow/python/data/experimental/ops/optimization.py
@@ -162,10 +162,12 @@ class _ChooseFastestDataset(dataset_ops.DatasetV2):
       A `Dataset` that has the same elements the inputs.
     """
     self._datasets = list(datasets)
+    self._structure = self._datasets[0]._element_structure  # pylint: disable=protected-access
     variant_tensor = (
         gen_experimental_dataset_ops.experimental_choose_fastest_dataset(
             [dataset._variant_tensor for dataset in self._datasets],  # pylint: disable=protected-access
-            num_experiments=num_experiments))
+            num_experiments=num_experiments,
+            **dataset_ops.flat_structure(self)))
     super(_ChooseFastestDataset, self).__init__(variant_tensor)
 
   def _inputs(self):
-- 
GitLab


From 17b71f26abaf3d96d29f346055333096a5f8d03d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 25 Jan 2019 13:25:03 -0800
Subject: [PATCH 1289/2345] Allow using libneuralnetworks.so on non-Android
 platforms.

PiperOrigin-RevId: 230957760
---
 tensorflow/lite/nnapi/BUILD                   | 21 ++++--
 tensorflow/lite/nnapi/nnapi_implementation.cc | 67 +++++++++++--------
 .../nnapi/nnapi_implementation_disabled.cc    | 20 ++++++
 .../lite/nnapi/nnapi_implementation_test.cc   |  2 +-
 4 files changed, 78 insertions(+), 32 deletions(-)
 create mode 100644 tensorflow/lite/nnapi/nnapi_implementation_disabled.cc

diff --git a/tensorflow/lite/nnapi/BUILD b/tensorflow/lite/nnapi/BUILD
index 172d0c8758..7af2b099e7 100644
--- a/tensorflow/lite/nnapi/BUILD
+++ b/tensorflow/lite/nnapi/BUILD
@@ -15,13 +15,26 @@ cc_library(
 
 cc_library(
     name = "nnapi_implementation",
-    srcs = [
-        "nnapi_implementation.cc",
-    ],
+    srcs = select({
+        "//tensorflow:ios": [
+            "nnapi_implementation_disabled.cc",
+        ],
+        "//tensorflow:windows": [
+            "nnapi_implementation_disabled.cc",
+        ],
+        "//conditions:default": [
+            "nnapi_implementation.cc",
+        ],
+    }),
     hdrs = [
         "nnapi_implementation.h",
     ],
-    linkopts = ["-ldl"],
+    linkopts = ["-ldl"] + select({
+        "//tensorflow:android": [],
+        "//tensorflow:ios": [],
+        "//tensorflow:windows": [],
+        "//conditions:default": ["-lrt"],
+    }),
     deps = [
         "//tensorflow/lite/nnapi:nnapi_lib",
     ],
diff --git a/tensorflow/lite/nnapi/nnapi_implementation.cc b/tensorflow/lite/nnapi/nnapi_implementation.cc
index 5c96957f4a..745d089ba7 100644
--- a/tensorflow/lite/nnapi/nnapi_implementation.cc
+++ b/tensorflow/lite/nnapi/nnapi_implementation.cc
@@ -14,14 +14,17 @@ limitations under the License.
 ==============================================================================*/
 #include "tensorflow/lite/nnapi/nnapi_implementation.h"
 
+#include <dlfcn.h>
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
 #include <cstdlib>
 
 #ifdef __ANDROID__
-#include <dlfcn.h>
-#include <sys/mman.h>
 #include <sys/system_properties.h>
-#include <unistd.h>
-#endif
+#endif  // __ANDROID__
 
 #define NNAPI_LOG(format, ...) fprintf(stderr, format "\n", __VA_ARGS__);
 
@@ -46,6 +49,7 @@ int32_t GetAndroidSdkVersion() {
   }
   return 0;
 }
+#endif  // __ANDROID__
 
 void* LoadFunction(void* handle, const char* name) {
   if (handle == nullptr) {
@@ -58,43 +62,48 @@ void* LoadFunction(void* handle, const char* name) {
   return fn;
 }
 
+#ifndef __ANDROID__
+// Add /dev/shm implementation of shared memory for non-Android platforms
+int ASharedMemory_create(const char* name, size_t size) {
+  int fd = shm_open(name, O_RDWR | O_CREAT, 0644);
+  if (fd >= 0) {
+    ftruncate(fd, size);
+  }
+  return fd;
+}
+#endif  // __ANDROID__
+
 #define LOAD_FUNCTION(handle, name) \
   nnapi.name = reinterpret_cast<name##_fn>(LoadFunction(handle, #name));
 
-#else
-
-#define LOAD_FUNCTION(handle, name) nnapi.name = nullptr;
-
-#endif
-
 const NnApi LoadNnApi() {
   NnApi nnapi = {};
+  nnapi.android_sdk_version = 0;
 
 #ifdef __ANDROID__
-  void* libneuralnetworks = nullptr;
   void* libandroid = nullptr;
   nnapi.android_sdk_version = GetAndroidSdkVersion();
   if (nnapi.android_sdk_version < 27) {
     NNAPI_LOG("nnapi error: requires android sdk version to be at least %d",
               27);
-  } else {
-    // TODO: change RTLD_LOCAL? Assumes there can be multiple instances of nn
-    // api RT
-    libneuralnetworks = dlopen("libneuralnetworks.so", RTLD_LAZY | RTLD_LOCAL);
-    if (libneuralnetworks == nullptr) {
-      NNAPI_LOG("nnapi error: unable to open library %s",
-                "libneuralnetworks.so");
-    }
-    libandroid = dlopen("libandroid.so", RTLD_LAZY | RTLD_LOCAL);
-    if (libandroid == nullptr) {
-      NNAPI_LOG("nnapi error: unable to open library %s", "libandroid.so");
-    }
+    nnapi.nnapi_exists = false;
+    return nnapi;
   }
+  libandroid = dlopen("libandroid.so", RTLD_LAZY | RTLD_LOCAL);
+  if (libandroid == nullptr) {
+    NNAPI_LOG("nnapi error: unable to open library %s", "libandroid.so");
+  }
+#endif  // __ANDROID__
+
+  void* libneuralnetworks = nullptr;
+  // TODO(b/123243014): change RTLD_LOCAL? Assumes there can be multiple
+  // instances of nn api RT
+  libneuralnetworks = dlopen("libneuralnetworks.so", RTLD_LAZY | RTLD_LOCAL);
+  if (libneuralnetworks == nullptr) {
+    NNAPI_LOG("nnapi error: unable to open library %s", "libneuralnetworks.so");
+  }
+
   nnapi.nnapi_exists = libneuralnetworks != nullptr;
-#else
-  nnapi.nnapi_exists = false;
-  nnapi.android_sdk_version = 0;
-#endif
 
   LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksMemory_createFromFd);
   LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksMemory_free);
@@ -124,7 +133,11 @@ const NnApi LoadNnApi() {
   LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_startCompute);
   LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksEvent_wait);
   LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksEvent_free);
+#ifdef __ANDROID__
   LOAD_FUNCTION(libandroid, ASharedMemory_create);
+#else
+  nnapi.ASharedMemory_create = ASharedMemory_create;
+#endif  // __ANDROID__
 
   return nnapi;
 }
diff --git a/tensorflow/lite/nnapi/nnapi_implementation_disabled.cc b/tensorflow/lite/nnapi/nnapi_implementation_disabled.cc
new file mode 100644
index 0000000000..6bc78e53da
--- /dev/null
+++ b/tensorflow/lite/nnapi/nnapi_implementation_disabled.cc
@@ -0,0 +1,20 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/nnapi/nnapi_implementation.h"
+
+const NnApi* NnApiImplementation() {
+  static const NnApi nnapi = {};
+  return &nnapi;
+}
diff --git a/tensorflow/lite/nnapi/nnapi_implementation_test.cc b/tensorflow/lite/nnapi/nnapi_implementation_test.cc
index 35de769bc8..51fc404ff8 100644
--- a/tensorflow/lite/nnapi/nnapi_implementation_test.cc
+++ b/tensorflow/lite/nnapi/nnapi_implementation_test.cc
@@ -113,7 +113,7 @@ TEST(NnapiLibTest, NnApiImplementation) {
   EXPECT_EQ(nnapi->ANeuralNetworksExecution_startCompute, nullptr);
   EXPECT_EQ(nnapi->ANeuralNetworksEvent_wait, nullptr);
   EXPECT_EQ(nnapi->ANeuralNetworksEvent_free, nullptr);
-  EXPECT_EQ(nnapi->ASharedMemory_create, nullptr);
+  EXPECT_NE(nnapi->ASharedMemory_create, nullptr);
 #endif
 }
 
-- 
GitLab


From bad872c586088118c4630696b7077e98019a49b1 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Fri, 25 Jan 2019 13:32:46 -0800
Subject: [PATCH 1290/2345] Automated rollback of commit
 c6a4b8600cd7288e709b63fee1426cc34aeda6d8

PiperOrigin-RevId: 230959283
---
 tensorflow/core/grappler/optimizers/BUILD     |   1 -
 .../grappler/optimizers/function_optimizer.cc |  65 +++++-----
 .../optimizers/function_optimizer_test.cc     | 122 ------------------
 3 files changed, 31 insertions(+), 157 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 70e0f226fc..ee216f80f9 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -181,7 +181,6 @@ tf_cuda_cc_test(
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/utils:grappler_test",
-        "@com_google_absl//absl/algorithm:container",
     ],
 )
 
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index c350e9f915..6913f9b55b 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -27,8 +27,6 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/device_mgr.h"
 #include "tensorflow/core/common_runtime/device_set.h"
 #include "tensorflow/core/common_runtime/function.h"
-#include "tensorflow/core/common_runtime/lower_if_while.h"
-#include "tensorflow/core/common_runtime/optimization_registry.h"
 #include "tensorflow/core/common_runtime/placer.h"
 #include "tensorflow/core/common_runtime/process_function_library_runtime.h"
 #include "tensorflow/core/framework/attr_value_util.h"
@@ -1382,6 +1380,20 @@ Status IsInlinableIndirectFunctionCall(const FunctionOptimizerContext& ctx,
         SummarizeNodeDef(func_node));
   }
 
+  // TODO(b/120991525, b/120986912): We need to lower `If` and `While` nodes to
+  // `Switch` nodes after function inlining (one more PRE_PLACEMENT pass?), but
+  // because of the reason described above we are not sure that it's safe, for
+  // now just disable inlining functions with functional control flow.
+  const auto is_functional_ctrl_flow_op = [](const NodeDef& node) {
+    return IsIf(node) || IsWhile(node);
+  };
+  if (absl::c_any_of(func.node_def(), is_functional_ctrl_flow_op)) {
+    return errors::FailedPrecondition(
+        "Can't inline function with `If` or `While` nodes in the function "
+        "body: ",
+        SummarizeNodeDef(func_node));
+  }
+
   return Status::OK();
 }
 
@@ -1392,8 +1404,7 @@ Status CheckThatSideEffectsWillExecute(
     const FunctionOptimizerContext& ctx,
     const GraphTopologyView& graph_topo_view,
     const absl::flat_hash_set<string> output_nodes) {
-  // In aggressive mode we just print a warning for side-effectful nodes that
-  // might not be executed after inlining.
+  // We ignore side-effects safety check in aggressive mode.
   const bool aggressive = ctx.opt_level() == RewriterConfig::AGGRESSIVE;
 
   for (const NodeDef& func_body_node : graph_topo_view.graph()->node()) {
@@ -1424,17 +1435,18 @@ Status CheckThatSideEffectsWillExecute(
     DfsTraversal(graph_topo_view, {&func_body_node},
                  TraversalDirection::kFollowOutputs, predicates, callbacks);
 
-    if (!will_execute) {
-      const string error_message = absl::StrCat(
+    if (!will_execute && !aggressive) {
+      return errors::Internal(
           "Can't guarantee execution of a side-effectful node, that is not "
           "reachable from function outputs. Function body node: ",
           SummarizeNodeDef(func_body_node));
+    }
 
-      if (aggressive) {
-        LOG(WARNING) << error_message;
-      } else {
-        return errors::Internal(error_message);
-      }
+    if (!will_execute && aggressive) {
+      LOG(WARNING)
+          << "Can't guarantee execution of a side-effectful node, that is not "
+             "reachable from function outputs. Function body node: "
+          << SummarizeNodeDef(func_body_node);
     }
   }
 
@@ -1442,33 +1454,19 @@ Status CheckThatSideEffectsWillExecute(
 }
 
 Status PlaceInlinedFunctionBody(
-    const NodeDef& func_node, const GrapplerFunctionItem& item,
+    const FunctionOptimizerContext& ctx, const NodeDef& func_node,
+    const GrapplerFunctionItem& item,
     const absl::flat_hash_map<absl::string_view, int>& input_placeholders_idx,
-    FunctionOptimizerContext* ctx, GraphDef* placed_graph_def) {
+    GraphDef* placed_graph_def) {
   // Control flow lowering and Placer works with a Graph object.
   std::unique_ptr<Graph> func_body_graph =
-      absl::make_unique<Graph>(ctx->function_library());
+      absl::make_unique<Graph>(ctx.function_library());
 
   GraphConstructorOptions opts;
   TF_RETURN_IF_ERROR(
       ConvertGraphDefToGraph(opts, item.graph, func_body_graph.get()));
 
-  // ------------------------------------------------------------------------ //
-  // Grappler receives the graph after PRE_PLACEMENT, Placer, and POST_PLACEMENT
-  // passes, so each node has a valid device assignment. Also V2 control
-  // flow ops (functional If and While) should have been lowered to V1 control
-  // flow (Switch and Merge nodes). To keep the graph valid for execution we
-  // must assign device to every inlined graph node, and also lower the control
-  // flow.
-
-  GraphOptimizationPassOptions opt_options;
-  opt_options.graph = &func_body_graph;
-  opt_options.flib_def = ctx->mutable_function_library();
-
-  // TODO(ezhulenev): Should we run full PRE_PLACEMENT pass here? And
-  // POST_PLACEMENT after placer?
-  LowerIfWhilePass pass;
-  TF_RETURN_IF_ERROR(pass.Run(opt_options));
+  // TODO(ezhulenev): Lower If/While ops.
 
   // ------------------------------------------------------------------------ //
   // Before placing the function body nodes we pin input placeholders to the
@@ -1481,7 +1479,7 @@ Status PlaceInlinedFunctionBody(
     if (input_placeholder_idx != input_placeholders_idx.end()) {
       const int input_idx = input_placeholder_idx->second;
       const GraphView::OutputPort output_port =
-          ctx->graph_view().GetRegularFanin({&func_node, input_idx});
+          ctx.graph_view().GetRegularFanin({&func_node, input_idx});
 
       VLOG(3) << "Pin inlined function input node '" << func_body_node->name()
               << "' to the '" << output_port.node->device() << "' device.";
@@ -1493,7 +1491,7 @@ Status PlaceInlinedFunctionBody(
   // After placing nodes corresponding to the function inputs, we need to assign
   // device placements to all other function body nodes.
 
-  const DeviceSet* devices = ctx->devices();
+  const DeviceSet* devices = ctx.devices();
 
   if (devices->devices().empty()) {
     // If there are no devices available for placer, we just put all nodes to
@@ -1535,7 +1533,6 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
                                   FunctionOptimizerContext* ctx,
                                   GraphDef* optimized_graph) {
   VLOG(2) << "Inline indirect function call: " << SummarizeNodeDef(func_node);
-  VLOG(4) << "Inlined function definition: " << DebugString(func);
   TF_RETURN_IF_ERROR(IsInlinableIndirectFunctionCall(*ctx, func, func_node));
 
   const AttrSlice func_instantiation_attr =
@@ -1698,7 +1695,7 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
 
   GraphDef placed_graph_def;
   TF_RETURN_IF_ERROR(PlaceInlinedFunctionBody(
-      func_node, item, input_placeholders_idx, ctx, &placed_graph_def));
+      *ctx, func_node, item, input_placeholders_idx, &placed_graph_def));
 
   // ------------------------------------------------------------------------ //
   // After all nodes placed we need to prepare them for inlining into the
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
index 6b62dbaf2c..fc7fffe7b2 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
@@ -14,8 +14,6 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/grappler/optimizers/function_optimizer.h"
-
-#include "absl/algorithm/container.h"
 #include "tensorflow/cc/ops/functional_ops.h"
 #include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/core/framework/function_testlib.h"
@@ -1271,126 +1269,6 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithNestedFunctionCall) {
   test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
 }
 
-TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithFunctionalControlFlow) {
-  using test::function::NDef;
-  using FDH = FunctionDefHelper;
-
-  FunctionOptimizer optimizer(RewriterConfig::AGGRESSIVE);
-
-  FunctionDef add_func = FunctionDefHelper::Create(
-      "MyAdd", {"x:T", "y:T"}, {"z:T"}, {"T: {float, double}"},
-      {{{"add"}, "Add", {"x", "y"}, {{"T", "$T"}}}},
-      /* Mapping between function returns and function node outputs. */
-      {{"z", "add:z:0"}});
-
-  FunctionDef mul_func = FunctionDefHelper::Create(
-      "MyMul", {"x:T", "y:T"}, {"z:T"}, {"T: {float, double}"},
-      {{{"mul"}, "Mul", {"x", "y"}, {{"T", "$T"}}}},
-      /* Mapping between function returns and function node outputs. */
-      {{"z", "mul:z:0"}});
-
-  // Compute: return cond ? a + b : a * b
-  FunctionDef add_or_mul_func = FunctionDefHelper::Create(
-      "AddOrMul", {"cond:bool", "x:float", "y:float"}, {"z:float"}, {},
-      {
-          {{"if_node"},
-           "If",
-           {"cond", "x", "y"},
-           {
-               {"Tcond", DT_BOOL},
-               {"Tin", DataTypeSlice{DT_FLOAT, DT_FLOAT}},
-               {"Tout", DataTypeSlice{DT_FLOAT}},
-               {"then_branch", FDH::FunctionRef("MyAdd", {{"T", DT_FLOAT}})},
-               {"else_branch", FDH::FunctionRef("MyMul", {{"T", DT_FLOAT}})},
-               {"_lower_using_switch_merge", true},
-           }},
-      },
-      /* Mapping between function returns and function node outputs. */
-      {{"z", "if_node:output:0"}});
-
-  // Build a computation graph for:
-  //   is_add: bool
-  //   a: float
-  //   b: float
-  //   c = AddOrMul(is_add, a, b)  # is_add ? a + b : a * b
-  //   d = Identity(c)
-  //   return d
-
-  // c = MyMul(a, b)
-  GrapplerItem item;
-  item.fetch = {"d"};
-  item.graph = test::function::GDef(
-      {NDef("is_add", "Placeholder", {}, {{"dtype", DT_BOOL}}, kDevice),
-       NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
-       NDef("b", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
-
-       NDef("c", "PartitionedCall", {"is_add", "a", "b"},
-            {{"Tin", DataTypeSlice{DT_BOOL, DT_FLOAT, DT_FLOAT}},
-             {"Tout", DataTypeSlice{DT_FLOAT}},
-             {"f", FDH::FunctionRef("AddOrMul")}},
-            kDevice),
-
-       NDef("d", "Identity", {"c"}, {{"T", DT_FLOAT}}, kDevice)},
-      // Function library.
-      {add_or_mul_func, add_func, mul_func});
-
-  GraphDef optimized_graph;
-  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &optimized_graph));
-
-  const auto count_nodes_with_op = [&](const string& op) {
-    return absl::c_count_if(optimized_graph.node(), [&](const NodeDef& node) {
-      return node.op() == op;
-    });
-  };
-
-  // All `PartitionedCall` nodes in the optimized graph must be inlined, and
-  // `If` node must be lowered to `Switch` and `Merge` nodes.
-  EXPECT_EQ(count_nodes_with_op("PartitionedCallOp"), 0);
-  EXPECT_EQ(count_nodes_with_op("If"), 0);
-  EXPECT_EQ(count_nodes_with_op("Switch"), 3);
-  EXPECT_EQ(count_nodes_with_op("Merge"), 1);
-
-  GrapplerItem optimized = item.WithGraph(std::move(optimized_graph));
-
-  Tensor one = test::AsScalar<float>(1.0);
-  Tensor two = test::AsScalar<float>(2.0);
-  Tensor three = test::AsScalar<float>(3.0);
-
-  const auto feed_args = [&](bool is_add) {
-    std::vector<std::pair<string, Tensor>> feed;
-    feed.emplace_back("a", one);
-    feed.emplace_back("b", two);
-    feed.emplace_back("is_add", test::AsScalar<bool>(is_add));
-    return feed;
-  };
-
-  {  // Check 'is_add == true': a + b
-    item.feed = feed_args(true);
-    optimized.feed = feed_args(true);
-
-    auto tensors_expected = EvaluateFetchNodes(item);
-    ASSERT_EQ(tensors_expected.size(), 1);
-    test::ExpectTensorEqual<float>(tensors_expected[0], three);
-
-    auto tensors = EvaluateFetchNodes(optimized);
-    ASSERT_EQ(tensors.size(), tensors_expected.size());
-    test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
-  }
-
-  {  // Check 'is_add == false': a * b
-    item.feed = feed_args(false);
-    optimized.feed = feed_args(false);
-
-    auto tensors_expected = EvaluateFetchNodes(item);
-    ASSERT_EQ(tensors_expected.size(), 1);
-    test::ExpectTensorEqual<float>(tensors_expected[0], two);
-
-    auto tensors = EvaluateFetchNodes(optimized);
-    ASSERT_EQ(tensors.size(), tensors_expected.size());
-    test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
-  }
-}
-
 TEST_F(FunctionOptimizerTest, SpecializeFunctionXTimesTwo) {
   using test::function::NDef;
 
-- 
GitLab


From d2d19330dfd2af040bcc11e7a7fa5b322d3de8ab Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Fri, 25 Jan 2019 13:35:39 -0800
Subject: [PATCH 1291/2345] Change the cuda_activation header to be
 non-CUDA-specific

PiperOrigin-RevId: 230959782
---
 tensorflow/stream_executor/cuda/BUILD | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/BUILD b/tensorflow/stream_executor/cuda/BUILD
index 77a1851047..43b865a90f 100644
--- a/tensorflow/stream_executor/cuda/BUILD
+++ b/tensorflow/stream_executor/cuda/BUILD
@@ -95,9 +95,9 @@ cc_library(
 # TODO(leary) split up cuda_gpu_executor.cc so that this can stand alone.
 cc_library(
     name = "cuda_activation_header",
-    hdrs = if_cuda_is_configured(["cuda_activation.h"]),
+    hdrs = ["cuda_activation.h"],
     visibility = ["//visibility:public"],
-    deps = if_cuda_is_configured(["//tensorflow/stream_executor/platform"]),
+    deps = ["//tensorflow/stream_executor/platform"],
 )
 
 cc_library(
-- 
GitLab


From 9b1af3223042817fe3999dc961ee7dfa469324ef Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 25 Jan 2019 13:56:36 -0800
Subject: [PATCH 1292/2345] Add test for setting engine output types properly

---
 tensorflow/contrib/tensorrt/test/topk_test.py | 30 +++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/tensorflow/contrib/tensorrt/test/topk_test.py b/tensorflow/contrib/tensorrt/test/topk_test.py
index 633a51982b..6a25388960 100644
--- a/tensorflow/contrib/tensorrt/test/topk_test.py
+++ b/tensorflow/contrib/tensorrt/test/topk_test.py
@@ -53,6 +53,36 @@ class TopKTest(trt_test.TfTrtIntegrationTestBase):
     """Return the expected engines to build."""
     return {"TRTEngineOp_0": ["Const", "TopK"]}
 
+class TopKOutputTypeTest(trt_test.TfTrtIntegrationTestBase):
+
+  def GetParams(self):
+    """Testing that output type of engine using Top-K is set correctly."""
+    dtype = dtypes.float32
+    input_name = "input"
+    input_dims = [100, 100]
+    k = 5
+    g = ops.Graph()
+    with g.as_default():
+      x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name)
+      k_tensor = constant_op.constant(k, dtype=dtypes.int32, name="Const")
+      values, indices = nn_ops.top_k(x, k_tensor, name="TopK")
+      # Reshape will act as a layer between the TopK output and the engine
+      # output, requiring the output tensor of reshape to be set explicitly to
+      # int32.
+      indices = array_ops.reshape(indices, [100, 1, 5], name="Reshape")
+      values = array_ops.identity(values, name="output_values")
+      indices = array_ops.identity(indices, name="output_indices")
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=[input_name],
+        input_dims=[[input_dims]],
+        output_names=["output_values", "output_indices"],
+        expected_output_dims=[[[100, k], [100, 1, k]]])
+
+  def ExpectedEnginesToBuild(self, run_params):
+    """Return the expected engines to build."""
+    return {"TRTEngineOp_0": ["Const", "TopK", "Reshape", "Reshape/shape"]}
+
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 6179f7d477260e5628fc661b9ff31bc4fd770d54 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Fri, 25 Jan 2019 13:36:21 -0800
Subject: [PATCH 1293/2345] Fix inconsistency in the handling of the state
 tuple coming out of tf.while_loop. Always use while_loop_v2 in staged code.

PiperOrigin-RevId: 230959899
---
 .../autograph/converters/control_flow.py      |  5 ++--
 .../autograph/operators/control_flow.py       | 26 +++++++++----------
 .../autograph/operators/control_flow_test.py  |  2 +-
 3 files changed, 15 insertions(+), 18 deletions(-)

diff --git a/tensorflow/python/autograph/converters/control_flow.py b/tensorflow/python/autograph/converters/control_flow.py
index a39a0b0cdb..4f38a2926b 100644
--- a/tensorflow/python/autograph/converters/control_flow.py
+++ b/tensorflow/python/autograph/converters/control_flow.py
@@ -297,12 +297,11 @@ class ControlFlowTransformer(converter.Base):
         if str(name) != ssf
     }
 
+    state_ast_tuple = gast.Tuple([n.ast() for n in loop_state], None)
+
     if len(loop_state) == 1:
       loop_state = loop_state[0]
       state_ssf = state_ssf[0]
-      state_ast_tuple = loop_state
-    else:
-      state_ast_tuple = gast.Tuple([n.ast() for n in loop_state], None)
 
     return loop_state, state_ssf, state_ast_tuple, ssf_map
 
diff --git a/tensorflow/python/autograph/operators/control_flow.py b/tensorflow/python/autograph/operators/control_flow.py
index 035ea1bd92..f046000720 100644
--- a/tensorflow/python/autograph/operators/control_flow.py
+++ b/tensorflow/python/autograph/operators/control_flow.py
@@ -74,10 +74,6 @@ def _py_for_stmt(iter_, extra_test, body, init_state):
     if not extra_test(*state):
       break
     state = body(target, *state)
-
-  # TODO(mdan): Remove this special case.
-  if len(state) == 1:
-    return state[0]
   return state
 
 
@@ -88,10 +84,12 @@ def _known_len_for_stmt(iter_, extra_test, body, init_state):
   def while_body(iterate_index, *state):
     iterate = iter_[iterate_index]
     new_state = body(iterate, *state)
+
+    state = (iterate_index + 1,)
     if new_state:
-      return (iterate_index + 1,) + new_state
-    else:
-      return iterate_index + 1
+      state += new_state
+
+    return state
 
   def while_cond(iterate_index, *state):
     return gen_math_ops.logical_and(iterate_index < n, extra_test(*state))
@@ -109,9 +107,6 @@ def _known_len_for_stmt(iter_, extra_test, body, init_state):
     assert len(results) >= 1  # Has at least the iterate.
     if len(results) > 1:
       results = results[1:]
-    if len(results) == 1:
-      # TODO(mdan): Remove this special case.
-      results, = results
   else:
     results = ()
 
@@ -130,9 +125,6 @@ def _dataset_for_stmt(ds, extra_test, body, init_state):
 
   results = ds.reduce(init_state, reduce_body)
 
-  # TODO(mdan): Remove this special case.
-  if len(results) == 1:
-    return results[0]
   return results
 
 
@@ -172,7 +164,13 @@ def _tf_while_stmt(test, body, init_state, opts):
   """Overload of while_stmt that stages a TF while_stmt."""
   if opts is None:
     opts = {}
-  return control_flow_ops.while_loop(test, body, init_state, **opts)
+
+  # Non-v2 while_loop unpacks the results when there is only one return value.
+  # This enforces consistency across versions.
+  opts['return_same_structure'] = True
+
+  retval = control_flow_ops.while_loop(test, body, init_state, **opts)
+  return retval
 
 
 def _py_while_stmt(test, body, init_state, opts):
diff --git a/tensorflow/python/autograph/operators/control_flow_test.py b/tensorflow/python/autograph/operators/control_flow_test.py
index f9e006f7ad..590b51820f 100644
--- a/tensorflow/python/autograph/operators/control_flow_test.py
+++ b/tensorflow/python/autograph/operators/control_flow_test.py
@@ -45,7 +45,7 @@ class ForLoopTest(test.TestCase):
         extra_test=lambda s: True,
         body=lambda i, s: (s + i,),
         init_state=(0,))
-    self.assertEqual(10, s)
+    self.assertEqual((10,), s)
 
   @test_util.run_deprecated_v1
   def test_dataset(self):
-- 
GitLab


From 4e06198e15ca222ac2d32a24ed4b7133f720118e Mon Sep 17 00:00:00 2001
From: nammbash <niroopshankar.ammbashankar@intel.com>
Date: Fri, 18 Jan 2019 16:53:38 -0800
Subject: [PATCH 1294/2345] requantization_op_perchannel_support

---
 tensorflow/core/BUILD                         |   1 +
 ...pi_def_RequantizationRangePerChannel.pbtxt |   4 +
 .../api_def_RequantizePerChannel.pbtxt        |   4 +
 tensorflow/core/kernels/BUILD                 |  65 +++-
 .../core/kernels/mkl_quantized_conv_ops.h     |  37 ++-
 ...mkl_requantization_range_per_channel_op.cc | 110 +++++++
 .../core/kernels/mkl_requantize_ops_test.cc   | 297 ++++++++++++++++++
 .../kernels/mkl_requantize_per_channel_op.cc  | 171 ++++++++++
 tensorflow/core/kernels/requantize.cc         |   8 +-
 tensorflow/core/ops/math_ops.cc               |  47 ++-
 10 files changed, 735 insertions(+), 9 deletions(-)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_RequantizationRangePerChannel.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_RequantizePerChannel.pbtxt
 create mode 100644 tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc
 create mode 100644 tensorflow/core/kernels/mkl_requantize_ops_test.cc
 create mode 100644 tensorflow/core/kernels/mkl_requantize_per_channel_op.cc

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index c3d31f91b3..a99b03454d 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1450,6 +1450,7 @@ cc_library(
         "//tensorflow/core/kernels:mkl_identity_op",
         "//tensorflow/core/kernels:mkl_input_conversion_op",
         "//tensorflow/core/kernels:mkl_lrn_op",
+        "//tensorflow/core/kernels:mkl_requantize_ops",
         "//tensorflow/core/kernels:mkl_pooling_ops",
         "//tensorflow/core/kernels:mkl_relu_op",
         "//tensorflow/core/kernels:mkl_reshape_op",
diff --git a/tensorflow/core/api_def/base_api/api_def_RequantizationRangePerChannel.pbtxt b/tensorflow/core/api_def/base_api/api_def_RequantizationRangePerChannel.pbtxt
new file mode 100644
index 0000000000..8874a53ec1
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_RequantizationRangePerChannel.pbtxt
@@ -0,0 +1,4 @@
+op {
+graph_op_name:
+  "RequantizationRangePerChannel" visibility : HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_RequantizePerChannel.pbtxt b/tensorflow/core/api_def/base_api/api_def_RequantizePerChannel.pbtxt
new file mode 100644
index 0000000000..0e4cd40a36
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_RequantizePerChannel.pbtxt
@@ -0,0 +1,4 @@
+op {
+graph_op_name:
+  "RequantizePerChannel" visibility : HIDDEN
+}
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index a59ea5b631..26b8c691e1 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -6104,7 +6104,9 @@ tf_kernel_library(
 tf_cc_test(
     name = "requantization_range_op_test",
     size = "small",
-    srcs = ["requantization_range_op_test.cc"],
+    srcs = [
+        "requantization_range_op_test.cc",
+    ],
     deps = [
         ":ops_testutil",
         ":ops_util",
@@ -6981,6 +6983,67 @@ tf_mkl_kernel_library(
     deps = NN_DEPS + mkl_deps() + [":cwise_op"],
 )
 
+tf_mkl_kernel_library(
+    name = "mkl_requantize_ops",
+    srcs = [
+        "mkl_requantization_range_per_channel_op.cc",
+        "mkl_requantize_per_channel_op.cc",
+    ],
+    hdrs = [
+        "meta_support.h",
+        "no_op.h",
+        "reference_gemm.h",
+    ],
+    deps = if_mkl(
+        [
+            ":concat_lib_hdrs",
+            ":conv_ops",
+            ":cwise_op",
+            ":eigen_helpers",
+            ":image_resizer_state",
+            ":ops_util",
+            ":pooling_ops",
+            ":quantization_utils",
+            "//tensorflow/core:array_ops_op_lib",
+            "//tensorflow/core:core_cpu",
+            "//tensorflow/core:framework",
+            "//tensorflow/core:lib",
+            "//tensorflow/core:math_ops_op_lib",
+            "//tensorflow/core:nn_ops_op_lib",
+            "//third_party/eigen3",
+            "@gemmlowp",
+            ":transpose_functor",
+            "@mkl_dnn",
+            "//third_party/mkl:intel_binary_blob",
+        ],
+    ),
+)
+
+tf_cc_test_mkl(
+    name = "mkl_requantize_ops_test",
+    size = "small",
+    srcs = ["mkl_requantize_ops_test.cc"],
+    deps = [
+        ":mkl_requantize_ops",
+        ":ops_testutil",
+        ":ops_util",
+        ":quantization_utils",
+        ":quantized_ops",
+        "//tensorflow/cc:cc_ops",
+        "//tensorflow/core:array_ops_op_lib",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:framework_internal",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:tensorflow",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+    ],
+)
+
 tf_cc_test_mkl(
     name = "mkl_fused_ops_test",
     size = "small",
diff --git a/tensorflow/core/kernels/mkl_quantized_conv_ops.h b/tensorflow/core/kernels/mkl_quantized_conv_ops.h
index 10825f6962..84a1ccb4c2 100644
--- a/tensorflow/core/kernels/mkl_quantized_conv_ops.h
+++ b/tensorflow/core/kernels/mkl_quantized_conv_ops.h
@@ -16,16 +16,18 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_KERNELS_MKL_QUANTIZED_CONV_OPS_H_
 #define TENSORFLOW_CORE_KERNELS_MKL_QUANTIZED_CONV_OPS_H_
 
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/tensor.h"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
 #ifdef INTEL_MKL
 
 namespace tensorflow {
 template <class T>
 float MklFloatForOneQuantizedLevel(float range_min, float range_max) {
-  const int64 highest = static_cast<int64>(Eigen::NumTraits<T>::highest());
-  const int64 lowest = static_cast<int64>(Eigen::NumTraits<T>::lowest());
+  int64 highest = static_cast<int64>(Eigen::NumTraits<T>::highest());
+  int64 lowest = static_cast<int64>(Eigen::NumTraits<T>::lowest());
+  if (lowest < -highest) lowest += 1;
+
   const float float_for_one_quantized_level =
       (range_max - range_min) / (highest - lowest);
   return float_for_one_quantized_level;
@@ -48,6 +50,35 @@ void MklQuantizationRangeForMultiplication(float min_a, float max_a,
   *min_c = c_float_for_one_quant_level * c_lowest;
   *max_c = c_float_for_one_quant_level * c_highest;
 }
+
+template <class T1, class T2, class T3>
+void MklQuantizationRangeForMultiplication(float min_a, float max_a,
+                                           const Tensor& min_b_vector,
+                                           const Tensor& max_b_vector,
+                                           Tensor** min_c_vector,
+                                           Tensor** max_c_vector) {
+  CHECK(min_b_vector.NumElements() == (*min_c_vector)->NumElements());
+  CHECK(max_b_vector.NumElements() == (*max_c_vector)->NumElements());
+  size_t n_channel = min_b_vector.NumElements();
+  const int64 c_highest = static_cast<int64>(Eigen::NumTraits<T3>::highest());
+  const int64 c_lowest = static_cast<int64>(Eigen::NumTraits<T3>::lowest());
+  const float* min_b = min_b_vector.flat<float>().data();
+  const float* max_b = max_b_vector.flat<float>().data();
+  float* min_c = (*min_c_vector)->flat<float>().data();
+  float* max_c = (*max_c_vector)->flat<float>().data();
+#pragma omp parallel for
+  for (size_t n = 0; n < n_channel; n++) {
+    float a_float_for_one_quant_level =
+        MklFloatForOneQuantizedLevel<T1>(min_a, max_a);
+    float b_float_for_one_quant_level =
+        MklFloatForOneQuantizedLevel<T2>(min_b[n], max_b[n]);
+    float c_float_for_one_quant_level =
+        a_float_for_one_quant_level * b_float_for_one_quant_level;
+    min_c[n] = c_float_for_one_quant_level * c_lowest;
+    max_c[n] = c_float_for_one_quant_level * c_highest;
+  }
+}
+
 }  // namespace tensorflow
 
 #endif  // INTEL_MKL
diff --git a/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc b/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc
new file mode 100644
index 0000000000..d23a2d320a
--- /dev/null
+++ b/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc
@@ -0,0 +1,110 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// See docs in ../ops/array_ops.cc.
+#ifdef INTEL_MKL
+#define EIGEN_USE_THREADS
+
+#include <math.h>
+#include <limits>
+
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/type_traits.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/kernels/meta_support.h"
+#include "tensorflow/core/kernels/no_op.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/util/mkl_util.h"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+
+namespace tensorflow {
+
+typedef Eigen::ThreadPoolDevice CPUDevice;
+
+class MklRequantizationRangePerChannelOp : public OpKernel {
+ public:
+  explicit MklRequantizationRangePerChannelOp(OpKernelConstruction* ctx)
+      : OpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("clip_value_max", &clip_value_max_));
+  }
+
+  void Compute(OpKernelContext* ctx) override {
+    const Tensor& input = ctx->input(kInputTensorIndex);
+    const Tensor& input_min = ctx->input(kInputMin);
+    const Tensor& input_max = ctx->input(kInputMax);
+
+    size_t depth = input_max.NumElements();
+    OP_REQUIRES(ctx, input_min.dim_size(0) == depth,
+                errors::InvalidArgument("min has incorrect size, expected ",
+                                        depth, " was ", input_min.dim_size(0)));
+    OP_REQUIRES(ctx, input_max.dim_size(0) == depth,
+                errors::InvalidArgument("max has incorrect size, expected ",
+                                        depth, " was ", input_max.dim_size(0)));
+
+    const float* input_min_data = input_min.flat<float>().data();
+    const float* input_max_data = input_max.flat<float>().data();
+    std::vector<float> ranges(depth);
+    bool is_non_negative = true;
+    Eigen::array<int, 2> shuffling({1, 0});
+    auto input_matrix = input.flat_inner_dims<qint32>();
+    auto transposed_input = input_matrix.shuffle(shuffling);
+
+#pragma omp parallel for
+    for (size_t i = 0; i < depth; i++) {
+      Eigen::Tensor<qint32, 0, Eigen::RowMajor> min =
+          transposed_input.chip<0>(i).minimum();
+      Eigen::Tensor<qint32, 0, Eigen::RowMajor> max =
+          transposed_input.chip<0>(i).maximum();
+      int32_t min_per_channel = min();
+      int32_t max_per_channel = max();
+      int32_t abs_max =
+          std::max(std::abs(min_per_channel), std::abs(max_per_channel));
+      float scale =
+          std::max(std::abs(input_min_data[i]), std::abs(input_max_data[i]));
+      ranges[i] = (scale * (float)abs_max / (float)(1L << 31));
+      if (min_per_channel < 0) is_non_negative = false;
+    }
+
+    float out_min_max = std::numeric_limits<float>::min();
+    for (size_t i = 0; i < depth; i++) {
+      if (out_min_max < ranges[i]) out_min_max = ranges[i];
+    }
+    // Fixing max to clip_value_max_ (example 6.0 to support relu6)
+    if (out_min_max > clip_value_max_) out_min_max = clip_value_max_;
+
+    Tensor* output_min = nullptr;
+    Tensor* output_max = nullptr;
+    OP_REQUIRES_OK(ctx, ctx->allocate_output(kOutputMin, {}, &output_min));
+    OP_REQUIRES_OK(ctx, ctx->allocate_output(kOutputMax, {}, &output_max));
+    output_min->flat<float>()(0) = is_non_negative ? 0.0f : out_min_max * -1.0f;
+    output_max->flat<float>()(0) = out_min_max;
+  }
+
+ private:
+  float clip_value_max_ = std::numeric_limits<float>::infinity();
+  const int kInputTensorIndex = 0;
+  const int kInputMin = 1;
+  const int kInputMax = 2;
+  const int kOutputMin = 0;
+  const int kOutputMax = 1;
+};
+
+REGISTER_KERNEL_BUILDER(Name("RequantizationRangePerChannel")
+                            .Device(DEVICE_CPU)
+                            .TypeConstraint<qint32>("T"),
+                        MklRequantizationRangePerChannelOp);
+}  // namespace tensorflow
+#endif  // INTEL_MKL
diff --git a/tensorflow/core/kernels/mkl_requantize_ops_test.cc b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
new file mode 100644
index 0000000000..44ab936471
--- /dev/null
+++ b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
@@ -0,0 +1,297 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/framework/allocator.h"
+#include "tensorflow/core/framework/fake_input.h"
+#include "tensorflow/core/framework/node_def_builder.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_testutil.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/framework/types.pb.h"
+#include "tensorflow/core/kernels/ops_testutil.h"
+#include "tensorflow/core/kernels/ops_util.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/platform/test_benchmark.h"
+
+#include <cmath>
+
+namespace tensorflow {
+
+class MklRequantizatedOpsTest : public OpsTestBase {};
+
+class MklRequantizatedOpsTestHelper : public OpsTestBase {
+ public:
+  void Setup(Tensor &input_tensor_qint32, float &range_weights_ch1,
+             float &range_weights_ch2);
+  void TestBody(){};
+};
+
+void MklRequantizatedOpsTestHelper::Setup(Tensor &input_tensor_qint32,
+                                          float &range_weights_ch1,
+                                          float &range_weights_ch2) {
+  // Step 1: Assumption of inputs
+  // ----------------------------
+  // Assume input Tensor T (NHWC) in FP32 has range [0, 5.0]   size nt*ht*wt*ct
+  // Assume input Filter W (NHWC) with 2 output channels of    size nw*ht**wt*2
+  // logically,   Filter W has 2 channels W1 and W2 each of    size nw*ht**wt*1
+  // Assume input Filter W1(NHWC) in FP32 has range [-2.0, 2.0]size nw*ht**wt*1
+  // Assume input Filter W2(NHWC) in FP32 has range [-3.0, 3.0]size nw*ht**wt*1
+
+  // Step 2: Assumption of Quantizing inputs and weights (per channel)
+  // ------------------------------------------------------------------
+  // When these 2 Tensors, T and W are quantized using a Quantize Op.
+  // When  the input Tensor T (NHWC) is quantized to unsigned int8.
+  // While the input Filter W (NHWC) is qunatized to signed int8.
+  // hence T max value is mapped to ((2^8-1) = 255) while W to ((2^7)-1 = 127))
+
+  // Range of Quantized T  in int8[0   , 255] maps to orig T  in FP32[0   , 5.0]
+  // Range of Quantized W1 in int8[-127, 127] maps to orig W1 in FP32[-2.0, 2.0]
+  // Range of Quantized W2 in int8[-127, 127] maps to orig W2 in FP32[-3.0, 3.0]
+
+  // Hence the resolution of Quantized T will be 5.0/255
+  // Hence the resolution of Quantized W1 will be 2.0/127
+  // Hence the resolution of Quantized W2 will be 3.0/127
+
+  // Step 3: Assumption of quantizedconv on quantized input&weights(per channel)
+  // ---------------------------------------------------------------------------
+  // The input T and weights W1 (or W2) will be convolved (and multipled)
+  // The output Tensor T is in int32 whose range is [-2^31, 2^31]
+  // The Range of the Convolved T*W1 is 2^31 * 5.0/255 * 2.0/127 = 663110.59
+  // So Range of Convolved T*W1 in int32[-2^31, 22^31] that maps to
+  // orig T Range in FP32[0,5.0] * [-2.0, 2.0] is [-663110.59, 663110.59]
+
+  // The Range of the Convolved T*W2 is 2^31 * 5.0/255 * 3.0/127 = 994665.88
+  // So Range of Convolved T*W2 in int32[-2^31, 22^31] that maps to
+  // orig T Range in FP32[0,5.0] * [-3.0, 3.0]  is [-994665.88, 994665.88]
+
+  // Step 4: Assumption output above is fed to Requantization_range_perchannel
+  // --------------------------------------------------------------------------
+  // Here we recalculate the new Range for Convolved T*W so that we
+  // make good use in int8 qunatization from int32 to int8.
+
+  // We assume the above operations are performed and use these values above
+  // as ranges for Requantization_range_perchannel_op.
+  range_weights_ch1 = 663110.59;  // For W1 channel
+  range_weights_ch2 = 994665.88;  // For W2 Channel
+
+  // We Fill the inputs Tensor T qint32 with arbitrary int32 values
+  test::FillValues<qint32>(
+      &input_tensor_qint32,
+      {-1000, -2000,  2000,   4000,   -3000,  -6000,  4000,   8000,
+       5000,  10000,  -6000,  -12000, 7000,   14000,  8000,   16000,
+       9000,  -18000, -10000, -20000, 11000,  22000,  -12000, -24000,
+       13000, 26000,  14000,  28000,  -15000, -30000, 16000,  32000});
+
+  // Step 5: Define and run requantization_range_perchannel
+  // -------------------------------------------------------
+  // See test RequantizationRangePerChannelTest_Basic and/or
+  // test RequantizationRangePerChannelTest_ClipMax
+}
+
+// Following tests the RequantizationRangePerChannel Op wherein the range
+// of the weights is calculated per channel.
+TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_Basic) {
+  // Let us setup the tensor and inputs before we run this op.
+  float clip_max_value = pow(2, 31);
+  float range_weights_ch1 = 0.0;
+  float range_weights_ch2 = 0.0;
+
+  // Create the input tensor
+  const int input_height = 4;
+  const int input_width = 4;
+  const int input_channels = 2;
+
+  // define and input tensor T shape.
+  Tensor input_tensor_qint32(DT_QINT32,
+                             {1, input_height, input_width, input_channels});
+
+  // Explanation and setup prior to this Op. Fill T and populate range values.
+  MklRequantizatedOpsTestHelper helper;
+  helper.Setup(input_tensor_qint32, range_weights_ch1, range_weights_ch2);
+
+  // Step 5: Define and run requantization_range_perchannel
+  // -------------------------------------------------------
+  // Define, Create and initalize the OP in question.
+  TF_ASSERT_OK(NodeDefBuilder("requantization_range_per_channel",
+                              "RequantizationRangePerChannel")
+                   .Input(FakeInput(DT_QINT32))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Attr("T", DataTypeToEnum<qint32>::v())
+                   .Attr("clip_value_max", clip_max_value)
+                   .Finalize(node_def()));
+  TF_ASSERT_OK(InitOp());
+
+  // Add the Input Nodes to the Op.
+  AddInputFromArray<qint32>(input_tensor_qint32.shape(),
+                            input_tensor_qint32.flat<qint32>());
+
+  // Calculate the Min and max from the ranges
+  float ch1_min = -1.0 * range_weights_ch1;
+  float ch1_max = range_weights_ch1;
+  float ch2_min = -1.0 * range_weights_ch2;
+  float ch2_max = range_weights_ch2;
+
+  // Add the Perchannel range Nodes to the Op.
+  AddInputFromArray<float>(TensorShape({input_channels}), {ch1_min, ch2_min});
+  AddInputFromArray<float>(TensorShape({input_channels}), {ch1_max, ch2_max});
+
+  // Run the Kernel
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Step 6: Verify Output and Store values to test Requantize_perchannel
+  // --------------------------------------------------------------------
+
+  // Verify the Expected Outputs
+  const float output_min = GetOutput(0)->flat<float>()(0);
+  const float output_max = GetOutput(1)->flat<float>()(0);
+  EXPECT_NEAR(-14.8217, output_min, 0.002);
+  EXPECT_NEAR(14.8217, output_max, 0.002);
+
+  // output range is made use in RequantizePerChannelTest_Basic
+}
+
+TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_ClipMax) {
+  // Let us setup the tensor and inputs before we run this op.
+  float clip_max_value = 6;  // Can be used as 6 for Relu 6 activations.
+  float range_weights_ch1 = 0.0;
+  float range_weights_ch2 = 0.0;
+
+  // Create the input tensor
+  const int input_height = 4;
+  const int input_width = 4;
+  const int input_channels = 2;
+
+  // define and input tensor T shape.
+  Tensor input_tensor_qint32(DT_QINT32,
+                             {1, input_height, input_width, input_channels});
+
+  // Explanation and setup prior to this Op. Fill T and populate range values.
+  MklRequantizatedOpsTestHelper helper;
+  helper.Setup(input_tensor_qint32, range_weights_ch1, range_weights_ch2);
+
+  // Step 5: Define and run requantization_range_perchannel
+  // -------------------------------------------------------
+  // Define, Create and initalize the OP in question.
+  TF_ASSERT_OK(NodeDefBuilder("requantization_range_per_channel",
+                              "RequantizationRangePerChannel")
+                   .Input(FakeInput(DT_QINT32))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Attr("T", DataTypeToEnum<qint32>::v())
+                   .Attr("clip_value_max", clip_max_value)
+                   .Finalize(node_def()));
+  TF_ASSERT_OK(InitOp());
+
+  // Add the Input Nodes to the Op.
+  AddInputFromArray<qint32>(input_tensor_qint32.shape(),
+                            input_tensor_qint32.flat<qint32>());
+
+  // Calculate the Min and max from the ranges
+  float ch1_min = -1.0 * range_weights_ch1;
+  float ch1_max = range_weights_ch1;
+  float ch2_min = -1.0 * range_weights_ch2;
+  float ch2_max = range_weights_ch2;
+
+  // Add the Perchannel range Nodes to the Op.
+  AddInputFromArray<float>(TensorShape({input_channels}), {ch1_min, ch2_min});
+  AddInputFromArray<float>(TensorShape({input_channels}), {ch1_max, ch2_max});
+
+  // Run the Kernel
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Step 6: Verify Output and Store values to test Requantize_perchannel
+  // --------------------------------------------------------------------
+
+  // Verify the Expected Outputs
+  const float output_min = GetOutput(0)->flat<float>()(0);
+  const float output_max = GetOutput(1)->flat<float>()(0);
+  EXPECT_NEAR(-6.0, output_min, 0.002);  // Values are Max as with clip_value
+  EXPECT_NEAR(6.0, output_max, 0.002);   // Values are Max as with clip_value
+}
+
+TEST_F(MklRequantizatedOpsTest, RequantizePerChannelTest_Basic) {
+  // Let us setup the tensor and inputs before we run this op.
+  float clip_max_value = pow(2, 31);
+  float range_weights_ch1 = 0.0;
+  float range_weights_ch2 = 0.0;
+
+  // Create the input tensor
+  const int input_height = 4;
+  const int input_width = 4;
+  const int input_channels = 2;
+
+  // define an input tensor T shape.
+  Tensor input_tensor_qint32(DT_QINT32,
+                             {1, input_height, input_width, input_channels});
+
+  // Explanation and setup prior to this Op. Fill T and populate range values.
+  MklRequantizatedOpsTestHelper helper;
+  helper.Setup(input_tensor_qint32, range_weights_ch1, range_weights_ch2);
+
+  // Step 7: Define and run requantize_perchannel
+  // --------------------------------------------
+  // The Output of Requantization_range_op_per_channel which calculated the
+  // new ranges of int8 is fed to the requantize per channel op.
+  // Here the Values of Convolved T*W is converted from int32 to int8.
+
+  TF_ASSERT_OK(NodeDefBuilder("requantize_per_channel", "RequantizePerChannel")
+                   .Input(FakeInput(DT_QINT32))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Attr("T", DataTypeToEnum<qint32>::v())
+                   .Attr("out_type", DataTypeToEnum<qint8>::v())
+                   .Finalize(node_def()));
+  TF_ASSERT_OK(InitOp());
+
+  // Add the Input Nodes to the Op.
+  AddInputFromArray<qint32>(input_tensor_qint32.shape(),
+                            input_tensor_qint32.flat<qint32>());
+
+  // Calculate the Min and max from the ranges
+  float ch1_min = -1.0 * range_weights_ch1;
+  float ch1_max = range_weights_ch1;
+  float ch2_min = -1.0 * range_weights_ch2;
+  float ch2_max = range_weights_ch2;
+
+  // Add the Perchannel range Nodes to the Op.
+  AddInputFromArray<float>(TensorShape({input_channels}), {ch1_min, ch2_min});
+  AddInputFromArray<float>(TensorShape({input_channels}), {ch1_max, ch2_max});
+
+  // Calculate the Min and max from Step 6 above
+  // in RequantizationRangePerChannelTest_Basic
+  float range_op_output_min = -14.8217;
+  float range_op_output_max = 14.8217;
+
+  // Add the Requested_min and requested_max stored from Step 6.
+  AddInputFromArray<float>(TensorShape({1}), {range_op_output_min});
+  AddInputFromArray<float>(TensorShape({1}), {range_op_output_max});
+
+  // Run the kernel
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Verify the output with the expected output
+  Tensor output = *GetOutput(0);
+  const float output_min = GetOutput(1)->flat<float>()(0);
+  const float output_max = GetOutput(2)->flat<float>()(0);
+  EXPECT_NEAR(range_op_output_min, output_min, 0.002);
+  EXPECT_NEAR(range_op_output_max, output_max, 0.002);
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc b/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc
new file mode 100644
index 0000000000..f9aa550866
--- /dev/null
+++ b/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc
@@ -0,0 +1,171 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// See docs in ../ops/array_ops.cc.
+
+#define EIGEN_USE_THREADS
+#ifdef INTEL_MKL
+#include <math.h>
+
+#include "mkldnn.hpp"
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/type_traits.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/kernels/meta_support.h"
+#include "tensorflow/core/kernels/no_op.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/util/mkl_util.h"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+
+namespace tensorflow {
+
+typedef Eigen::ThreadPoolDevice CPUDevice;
+
+template <typename Device, typename Toutput>
+class MklRequantizePerChannelOp : public OpKernel {
+ public:
+  explicit MklRequantizePerChannelOp(OpKernelConstruction* ctx)
+      : OpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("out_type", &out_type_));
+    OP_REQUIRES(ctx, out_type_ == DT_QINT8 || out_type_ == DT_QUINT8,
+                errors::InvalidArgument(
+                    "out_type must be qint8 or quint8, but got: " + out_type_));
+  }
+  virtual ~MklRequantizePerChannelOp() {}
+  void Compute(OpKernelContext* ctx) override {
+    try {
+      const Tensor& input = ctx->input(kInputTensorIndex);
+      const Tensor& input_min_vec = ctx->input(kInputMinVec);
+      float* input_min_vec_data = (float*)const_cast<void*>(
+          static_cast<const void*>(input_min_vec.flat<float>().data()));
+      const Tensor& input_max_vec = ctx->input(kInputMaxVec);
+      float* input_max_vec_data = (float*)const_cast<void*>(
+          static_cast<const void*>(input_max_vec.flat<float>().data()));
+
+      const Tensor& input_requested_min = ctx->input(this->kRequestMin);
+      const float input_requested_min_float =
+          input_requested_min.flat<float>()(0);
+      const Tensor& input_requested_max = ctx->input(this->kRequestMax);
+      const float input_requested_max_float =
+          input_requested_max.flat<float>()(0);
+
+      size_t depth = input_min_vec.NumElements();
+      OP_REQUIRES(
+          ctx, input_min_vec.dim_size(0) == depth,
+          errors::InvalidArgument("min has incorrect size, expected ", depth,
+                                  " was ", input_min_vec.dim_size(0)));
+      OP_REQUIRES(
+          ctx, input_max_vec.dim_size(0) == depth,
+          errors::InvalidArgument("max has incorrect size, expected ", depth,
+                                  " was ", input_max_vec.dim_size(0)));
+
+      if (out_type_ == DT_QINT8) CHECK(input_requested_min_float < 0.0f);
+
+      const float factor = (out_type_ == DT_QINT8) ? 127.0f : 255.0f;
+      float requested_min_max = std::max(std::abs(input_requested_min_float),
+                                         std::abs(input_requested_max_float));
+      Tensor* output = nullptr;
+      OP_REQUIRES_OK(ctx, ctx->allocate_output(kOutputTensorIndex,
+                                               input.shape(), &output));
+
+      std::vector<float> scales(depth);
+      for (int i = 0; i < depth; i++) {
+        float min_max_from_vec = std::max(std::abs(input_min_vec_data[i]),
+                                          std::abs(input_max_vec_data[i]));
+        float scale =
+            factor * (min_max_from_vec / requested_min_max / (float)(1L << 31));
+        scales[i] = scale;
+      }
+
+      mkldnn::primitive_attr reorder_attr;
+      reorder_attr.set_output_scales(2, scales);
+
+      memory::dims dims_mkl_order =
+          TFShapeToMklDnnDimsInNCHW(input.shape(), FORMAT_NHWC);
+      memory::desc input_md = memory::desc(dims_mkl_order, MklDnnType<qint32>(),
+                                           memory::format::nhwc);
+      memory::desc output_md =
+          (out_type_ == DT_QINT8)
+              ? memory::desc(dims_mkl_order, MklDnnType<qint8>(),
+                             memory::format::nhwc)
+              : memory::desc(dims_mkl_order, MklDnnType<quint8>(),
+                             memory::format::nhwc);
+
+      memory::primitive_desc input_pd =
+          memory::primitive_desc(input_md, cpu_engine_);
+      memory::primitive_desc output_pd =
+          memory::primitive_desc(output_md, cpu_engine_);
+
+      void* input_buf =
+          static_cast<void*>(const_cast<qint32*>(input.flat<qint32>().data()));
+      void* output_buf;
+      if (out_type_ == DT_QINT8) {
+        output_buf = static_cast<void*>(
+            const_cast<qint8*>(output->flat<qint8>().data()));
+      } else {
+        output_buf = static_cast<void*>(
+            const_cast<quint8*>(output->flat<quint8>().data()));
+      }
+
+      std::unique_ptr<memory> input_mem_prim_(new memory(input_pd, input_buf));
+      std::unique_ptr<memory> output_mem_prim_(
+          new memory(output_pd, output_buf));
+
+      mkldnn::reorder::primitive_desc reorder_pd =
+          mkldnn::reorder::primitive_desc(input_pd, output_pd, reorder_attr);
+      std::vector<mkldnn::primitive> net;
+      net.push_back(
+          mkldnn::reorder(reorder_pd, *input_mem_prim_, *output_mem_prim_));
+      stream(stream::kind::eager).submit(net).wait();
+
+      Tensor* output_min = nullptr;
+      Tensor* output_max = nullptr;
+      OP_REQUIRES_OK(ctx, ctx->allocate_output(kOutputMin, {}, &output_min));
+      OP_REQUIRES_OK(ctx, ctx->allocate_output(kOutputMax, {}, &output_max));
+
+      output_min->flat<float>()(0) = input_requested_min_float;
+      output_max->flat<float>()(0) = input_requested_max_float;
+
+    } catch (mkldnn::error& e) {
+      string error_msg = "Status: " + std::to_string(e.status) + ", message: " +
+                         std::string(e.message) + ", in file " +
+                         std::string(__FILE__) + ":" + std::to_string(__LINE__);
+      OP_REQUIRES_OK(
+          ctx, errors::Aborted("Operation received an exception:", error_msg));
+    }
+  }
+
+ private:
+  const int kInputTensorIndex = 0;
+  const int kInputMinVec = 1;
+  const int kInputMaxVec = 2;
+  const int kRequestMin = 3;
+  const int kRequestMax = 4;
+  const int kOutputTensorIndex = 0;
+  const int kOutputMin = 1;
+  const int kOutputMax = 2;
+  DataType out_type_;
+  engine cpu_engine_ = engine(engine::cpu, 0);
+};
+
+REGISTER_KERNEL_BUILDER(Name("RequantizePerChannel")
+                            .Device(DEVICE_CPU)
+                            .TypeConstraint<qint32>("T")
+                            .TypeConstraint<qint8>("out_type"),
+                        MklRequantizePerChannelOp<CPUDevice, qint8>);
+
+}  // namespace tensorflow
+#endif  // INTEL_MKL
diff --git a/tensorflow/core/kernels/requantize.cc b/tensorflow/core/kernels/requantize.cc
index dce6f1a185..ab057d5a67 100644
--- a/tensorflow/core/kernels/requantize.cc
+++ b/tensorflow/core/kernels/requantize.cc
@@ -19,7 +19,6 @@ limitations under the License.
 
 #include <math.h>
 
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/type_traits.h"
@@ -27,6 +26,7 @@ limitations under the License.
 #include "tensorflow/core/kernels/meta_support.h"
 #include "tensorflow/core/kernels/quantization_utils.h"
 #include "tensorflow/core/lib/core/errors.h"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
 namespace tensorflow {
 
@@ -100,4 +100,10 @@ REGISTER_KERNEL_BUILDER(Name("Requantize")
                             .TypeConstraint<quint8>("out_type"),
                         RequantizeOp<qint32, quint8>);
 
+REGISTER_KERNEL_BUILDER(Name("Requantize")
+                            .Device(DEVICE_CPU)
+                            .TypeConstraint<qint32>("Tinput")
+                            .TypeConstraint<qint8>("out_type"),
+                        RequantizeOp<qint32, qint8>);
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc
index eb9cbd3225..52243f2fcf 100644
--- a/tensorflow/core/ops/math_ops.cc
+++ b/tensorflow/core/ops/math_ops.cc
@@ -1247,12 +1247,12 @@ Status RangeSize(const Tensor* start_t, const Tensor* limit_t,
   T limit = limit_t->scalar<T>()();
   T delta = delta_t->scalar<T>()();
   if (start > limit && delta > 0) {
-    return errors::InvalidArgument(
-        "Requires start <= limit when delta > 0: ", start, "/", limit);
+    return errors::InvalidArgument("Requires start <= limit when delta > 0: ",
+                                   start, "/", limit);
   }
   if (start < limit && delta < 0) {
-    return errors::InvalidArgument(
-        "Requires start >= limit when delta < 0: ", start, "/", limit);
+    return errors::InvalidArgument("Requires start >= limit when delta < 0: ",
+                                   start, "/", limit);
   }
   if (delta == 0) {
     return errors::InvalidArgument("Requires delta != 0");
@@ -1686,6 +1686,45 @@ Add two input tensors element wise using mkl kernel sum.
 inputs: Must all be the same size and shape.
 )doc");
 
+REGISTER_OP("RequantizePerChannel")
+    .Input("input: T")
+    .Input("input_min: float")
+    .Input("input_max: float")
+    .Input("requested_output_min: float")
+    .Input("requested_output_max: float")
+    .Output("output: out_type")
+    .Output("output_min: float")
+    .Output("output_max: float")
+    .Attr("T: quantizedtype = DT_QINT32")
+    .Attr("out_type: quantizedtype = DT_QUINT8")
+    .SetShapeFn([](InferenceContext* c) {
+      TF_RETURN_IF_ERROR(shape_inference::UnchangedShape(c));
+      ShapeHandle unused;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &unused));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 1, &unused));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 0, &unused));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(4), 0, &unused));
+      c->set_output(1, c->Scalar());
+      c->set_output(2, c->Scalar());
+      return Status::OK();
+    });
+REGISTER_OP("RequantizationRangePerChannel")
+    .Input("input: T")
+    .Input("input_min: float")
+    .Input("input_max: float")
+    .Output("output_min: float")
+    .Output("output_max: float")
+    .Attr("T: quantizedtype = DT_QINT32")
+    .Attr("clip_value_max: float")
+    .SetShapeFn([](InferenceContext* c) {
+      ShapeHandle unused;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &unused));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 1, &unused));
+      c->set_output(0, c->Scalar());
+      c->set_output(1, c->Scalar());
+      return Status::OK();
+    });
+
 #endif  // INTEL_MKL
 
 REGISTER_OP("NextAfter")
-- 
GitLab


From 96007e5022df35766b014a30d6fe4075cc1662cc Mon Sep 17 00:00:00 2001
From: nammbash <niroopshankar.ammbashankar@intel.com>
Date: Fri, 18 Jan 2019 16:53:38 -0800
Subject: [PATCH 1295/2345] requantization_op_perchannel_support

---
 tensorflow/core/BUILD                         |   1 +
 ...pi_def_RequantizationRangePerChannel.pbtxt |   4 +
 .../api_def_RequantizePerChannel.pbtxt        |   4 +
 tensorflow/core/kernels/BUILD                 |  65 +++-
 .../core/kernels/mkl_quantized_conv_ops.h     |  37 ++-
 ...mkl_requantization_range_per_channel_op.cc | 110 +++++++
 .../core/kernels/mkl_requantize_ops_test.cc   | 297 ++++++++++++++++++
 .../kernels/mkl_requantize_per_channel_op.cc  | 171 ++++++++++
 tensorflow/core/kernels/requantize.cc         |   8 +-
 tensorflow/core/ops/math_ops.cc               |  47 ++-
 10 files changed, 735 insertions(+), 9 deletions(-)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_RequantizationRangePerChannel.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_RequantizePerChannel.pbtxt
 create mode 100644 tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc
 create mode 100644 tensorflow/core/kernels/mkl_requantize_ops_test.cc
 create mode 100644 tensorflow/core/kernels/mkl_requantize_per_channel_op.cc

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index c3d31f91b3..a99b03454d 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1450,6 +1450,7 @@ cc_library(
         "//tensorflow/core/kernels:mkl_identity_op",
         "//tensorflow/core/kernels:mkl_input_conversion_op",
         "//tensorflow/core/kernels:mkl_lrn_op",
+        "//tensorflow/core/kernels:mkl_requantize_ops",
         "//tensorflow/core/kernels:mkl_pooling_ops",
         "//tensorflow/core/kernels:mkl_relu_op",
         "//tensorflow/core/kernels:mkl_reshape_op",
diff --git a/tensorflow/core/api_def/base_api/api_def_RequantizationRangePerChannel.pbtxt b/tensorflow/core/api_def/base_api/api_def_RequantizationRangePerChannel.pbtxt
new file mode 100644
index 0000000000..8874a53ec1
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_RequantizationRangePerChannel.pbtxt
@@ -0,0 +1,4 @@
+op {
+graph_op_name:
+  "RequantizationRangePerChannel" visibility : HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_RequantizePerChannel.pbtxt b/tensorflow/core/api_def/base_api/api_def_RequantizePerChannel.pbtxt
new file mode 100644
index 0000000000..0e4cd40a36
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_RequantizePerChannel.pbtxt
@@ -0,0 +1,4 @@
+op {
+graph_op_name:
+  "RequantizePerChannel" visibility : HIDDEN
+}
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index a59ea5b631..26b8c691e1 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -6104,7 +6104,9 @@ tf_kernel_library(
 tf_cc_test(
     name = "requantization_range_op_test",
     size = "small",
-    srcs = ["requantization_range_op_test.cc"],
+    srcs = [
+        "requantization_range_op_test.cc",
+    ],
     deps = [
         ":ops_testutil",
         ":ops_util",
@@ -6981,6 +6983,67 @@ tf_mkl_kernel_library(
     deps = NN_DEPS + mkl_deps() + [":cwise_op"],
 )
 
+tf_mkl_kernel_library(
+    name = "mkl_requantize_ops",
+    srcs = [
+        "mkl_requantization_range_per_channel_op.cc",
+        "mkl_requantize_per_channel_op.cc",
+    ],
+    hdrs = [
+        "meta_support.h",
+        "no_op.h",
+        "reference_gemm.h",
+    ],
+    deps = if_mkl(
+        [
+            ":concat_lib_hdrs",
+            ":conv_ops",
+            ":cwise_op",
+            ":eigen_helpers",
+            ":image_resizer_state",
+            ":ops_util",
+            ":pooling_ops",
+            ":quantization_utils",
+            "//tensorflow/core:array_ops_op_lib",
+            "//tensorflow/core:core_cpu",
+            "//tensorflow/core:framework",
+            "//tensorflow/core:lib",
+            "//tensorflow/core:math_ops_op_lib",
+            "//tensorflow/core:nn_ops_op_lib",
+            "//third_party/eigen3",
+            "@gemmlowp",
+            ":transpose_functor",
+            "@mkl_dnn",
+            "//third_party/mkl:intel_binary_blob",
+        ],
+    ),
+)
+
+tf_cc_test_mkl(
+    name = "mkl_requantize_ops_test",
+    size = "small",
+    srcs = ["mkl_requantize_ops_test.cc"],
+    deps = [
+        ":mkl_requantize_ops",
+        ":ops_testutil",
+        ":ops_util",
+        ":quantization_utils",
+        ":quantized_ops",
+        "//tensorflow/cc:cc_ops",
+        "//tensorflow/core:array_ops_op_lib",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:framework_internal",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:tensorflow",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+    ],
+)
+
 tf_cc_test_mkl(
     name = "mkl_fused_ops_test",
     size = "small",
diff --git a/tensorflow/core/kernels/mkl_quantized_conv_ops.h b/tensorflow/core/kernels/mkl_quantized_conv_ops.h
index 10825f6962..84a1ccb4c2 100644
--- a/tensorflow/core/kernels/mkl_quantized_conv_ops.h
+++ b/tensorflow/core/kernels/mkl_quantized_conv_ops.h
@@ -16,16 +16,18 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_KERNELS_MKL_QUANTIZED_CONV_OPS_H_
 #define TENSORFLOW_CORE_KERNELS_MKL_QUANTIZED_CONV_OPS_H_
 
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/tensor.h"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
 #ifdef INTEL_MKL
 
 namespace tensorflow {
 template <class T>
 float MklFloatForOneQuantizedLevel(float range_min, float range_max) {
-  const int64 highest = static_cast<int64>(Eigen::NumTraits<T>::highest());
-  const int64 lowest = static_cast<int64>(Eigen::NumTraits<T>::lowest());
+  int64 highest = static_cast<int64>(Eigen::NumTraits<T>::highest());
+  int64 lowest = static_cast<int64>(Eigen::NumTraits<T>::lowest());
+  if (lowest < -highest) lowest += 1;
+
   const float float_for_one_quantized_level =
       (range_max - range_min) / (highest - lowest);
   return float_for_one_quantized_level;
@@ -48,6 +50,35 @@ void MklQuantizationRangeForMultiplication(float min_a, float max_a,
   *min_c = c_float_for_one_quant_level * c_lowest;
   *max_c = c_float_for_one_quant_level * c_highest;
 }
+
+template <class T1, class T2, class T3>
+void MklQuantizationRangeForMultiplication(float min_a, float max_a,
+                                           const Tensor& min_b_vector,
+                                           const Tensor& max_b_vector,
+                                           Tensor** min_c_vector,
+                                           Tensor** max_c_vector) {
+  CHECK(min_b_vector.NumElements() == (*min_c_vector)->NumElements());
+  CHECK(max_b_vector.NumElements() == (*max_c_vector)->NumElements());
+  size_t n_channel = min_b_vector.NumElements();
+  const int64 c_highest = static_cast<int64>(Eigen::NumTraits<T3>::highest());
+  const int64 c_lowest = static_cast<int64>(Eigen::NumTraits<T3>::lowest());
+  const float* min_b = min_b_vector.flat<float>().data();
+  const float* max_b = max_b_vector.flat<float>().data();
+  float* min_c = (*min_c_vector)->flat<float>().data();
+  float* max_c = (*max_c_vector)->flat<float>().data();
+#pragma omp parallel for
+  for (size_t n = 0; n < n_channel; n++) {
+    float a_float_for_one_quant_level =
+        MklFloatForOneQuantizedLevel<T1>(min_a, max_a);
+    float b_float_for_one_quant_level =
+        MklFloatForOneQuantizedLevel<T2>(min_b[n], max_b[n]);
+    float c_float_for_one_quant_level =
+        a_float_for_one_quant_level * b_float_for_one_quant_level;
+    min_c[n] = c_float_for_one_quant_level * c_lowest;
+    max_c[n] = c_float_for_one_quant_level * c_highest;
+  }
+}
+
 }  // namespace tensorflow
 
 #endif  // INTEL_MKL
diff --git a/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc b/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc
new file mode 100644
index 0000000000..d23a2d320a
--- /dev/null
+++ b/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc
@@ -0,0 +1,110 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// See docs in ../ops/array_ops.cc.
+#ifdef INTEL_MKL
+#define EIGEN_USE_THREADS
+
+#include <math.h>
+#include <limits>
+
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/type_traits.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/kernels/meta_support.h"
+#include "tensorflow/core/kernels/no_op.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/util/mkl_util.h"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+
+namespace tensorflow {
+
+typedef Eigen::ThreadPoolDevice CPUDevice;
+
+class MklRequantizationRangePerChannelOp : public OpKernel {
+ public:
+  explicit MklRequantizationRangePerChannelOp(OpKernelConstruction* ctx)
+      : OpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("clip_value_max", &clip_value_max_));
+  }
+
+  void Compute(OpKernelContext* ctx) override {
+    const Tensor& input = ctx->input(kInputTensorIndex);
+    const Tensor& input_min = ctx->input(kInputMin);
+    const Tensor& input_max = ctx->input(kInputMax);
+
+    size_t depth = input_max.NumElements();
+    OP_REQUIRES(ctx, input_min.dim_size(0) == depth,
+                errors::InvalidArgument("min has incorrect size, expected ",
+                                        depth, " was ", input_min.dim_size(0)));
+    OP_REQUIRES(ctx, input_max.dim_size(0) == depth,
+                errors::InvalidArgument("max has incorrect size, expected ",
+                                        depth, " was ", input_max.dim_size(0)));
+
+    const float* input_min_data = input_min.flat<float>().data();
+    const float* input_max_data = input_max.flat<float>().data();
+    std::vector<float> ranges(depth);
+    bool is_non_negative = true;
+    Eigen::array<int, 2> shuffling({1, 0});
+    auto input_matrix = input.flat_inner_dims<qint32>();
+    auto transposed_input = input_matrix.shuffle(shuffling);
+
+#pragma omp parallel for
+    for (size_t i = 0; i < depth; i++) {
+      Eigen::Tensor<qint32, 0, Eigen::RowMajor> min =
+          transposed_input.chip<0>(i).minimum();
+      Eigen::Tensor<qint32, 0, Eigen::RowMajor> max =
+          transposed_input.chip<0>(i).maximum();
+      int32_t min_per_channel = min();
+      int32_t max_per_channel = max();
+      int32_t abs_max =
+          std::max(std::abs(min_per_channel), std::abs(max_per_channel));
+      float scale =
+          std::max(std::abs(input_min_data[i]), std::abs(input_max_data[i]));
+      ranges[i] = (scale * (float)abs_max / (float)(1L << 31));
+      if (min_per_channel < 0) is_non_negative = false;
+    }
+
+    float out_min_max = std::numeric_limits<float>::min();
+    for (size_t i = 0; i < depth; i++) {
+      if (out_min_max < ranges[i]) out_min_max = ranges[i];
+    }
+    // Fixing max to clip_value_max_ (example 6.0 to support relu6)
+    if (out_min_max > clip_value_max_) out_min_max = clip_value_max_;
+
+    Tensor* output_min = nullptr;
+    Tensor* output_max = nullptr;
+    OP_REQUIRES_OK(ctx, ctx->allocate_output(kOutputMin, {}, &output_min));
+    OP_REQUIRES_OK(ctx, ctx->allocate_output(kOutputMax, {}, &output_max));
+    output_min->flat<float>()(0) = is_non_negative ? 0.0f : out_min_max * -1.0f;
+    output_max->flat<float>()(0) = out_min_max;
+  }
+
+ private:
+  float clip_value_max_ = std::numeric_limits<float>::infinity();
+  const int kInputTensorIndex = 0;
+  const int kInputMin = 1;
+  const int kInputMax = 2;
+  const int kOutputMin = 0;
+  const int kOutputMax = 1;
+};
+
+REGISTER_KERNEL_BUILDER(Name("RequantizationRangePerChannel")
+                            .Device(DEVICE_CPU)
+                            .TypeConstraint<qint32>("T"),
+                        MklRequantizationRangePerChannelOp);
+}  // namespace tensorflow
+#endif  // INTEL_MKL
diff --git a/tensorflow/core/kernels/mkl_requantize_ops_test.cc b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
new file mode 100644
index 0000000000..44ab936471
--- /dev/null
+++ b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
@@ -0,0 +1,297 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/framework/allocator.h"
+#include "tensorflow/core/framework/fake_input.h"
+#include "tensorflow/core/framework/node_def_builder.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_testutil.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/framework/types.pb.h"
+#include "tensorflow/core/kernels/ops_testutil.h"
+#include "tensorflow/core/kernels/ops_util.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/platform/test_benchmark.h"
+
+#include <cmath>
+
+namespace tensorflow {
+
+class MklRequantizatedOpsTest : public OpsTestBase {};
+
+class MklRequantizatedOpsTestHelper : public OpsTestBase {
+ public:
+  void Setup(Tensor &input_tensor_qint32, float &range_weights_ch1,
+             float &range_weights_ch2);
+  void TestBody(){};
+};
+
+void MklRequantizatedOpsTestHelper::Setup(Tensor &input_tensor_qint32,
+                                          float &range_weights_ch1,
+                                          float &range_weights_ch2) {
+  // Step 1: Assumption of inputs
+  // ----------------------------
+  // Assume input Tensor T (NHWC) in FP32 has range [0, 5.0]   size nt*ht*wt*ct
+  // Assume input Filter W (NHWC) with 2 output channels of    size nw*ht**wt*2
+  // logically,   Filter W has 2 channels W1 and W2 each of    size nw*ht**wt*1
+  // Assume input Filter W1(NHWC) in FP32 has range [-2.0, 2.0]size nw*ht**wt*1
+  // Assume input Filter W2(NHWC) in FP32 has range [-3.0, 3.0]size nw*ht**wt*1
+
+  // Step 2: Assumption of Quantizing inputs and weights (per channel)
+  // ------------------------------------------------------------------
+  // When these 2 Tensors, T and W are quantized using a Quantize Op.
+  // When  the input Tensor T (NHWC) is quantized to unsigned int8.
+  // While the input Filter W (NHWC) is qunatized to signed int8.
+  // hence T max value is mapped to ((2^8-1) = 255) while W to ((2^7)-1 = 127))
+
+  // Range of Quantized T  in int8[0   , 255] maps to orig T  in FP32[0   , 5.0]
+  // Range of Quantized W1 in int8[-127, 127] maps to orig W1 in FP32[-2.0, 2.0]
+  // Range of Quantized W2 in int8[-127, 127] maps to orig W2 in FP32[-3.0, 3.0]
+
+  // Hence the resolution of Quantized T will be 5.0/255
+  // Hence the resolution of Quantized W1 will be 2.0/127
+  // Hence the resolution of Quantized W2 will be 3.0/127
+
+  // Step 3: Assumption of quantizedconv on quantized input&weights(per channel)
+  // ---------------------------------------------------------------------------
+  // The input T and weights W1 (or W2) will be convolved (and multipled)
+  // The output Tensor T is in int32 whose range is [-2^31, 2^31]
+  // The Range of the Convolved T*W1 is 2^31 * 5.0/255 * 2.0/127 = 663110.59
+  // So Range of Convolved T*W1 in int32[-2^31, 22^31] that maps to
+  // orig T Range in FP32[0,5.0] * [-2.0, 2.0] is [-663110.59, 663110.59]
+
+  // The Range of the Convolved T*W2 is 2^31 * 5.0/255 * 3.0/127 = 994665.88
+  // So Range of Convolved T*W2 in int32[-2^31, 22^31] that maps to
+  // orig T Range in FP32[0,5.0] * [-3.0, 3.0]  is [-994665.88, 994665.88]
+
+  // Step 4: Assumption output above is fed to Requantization_range_perchannel
+  // --------------------------------------------------------------------------
+  // Here we recalculate the new Range for Convolved T*W so that we
+  // make good use in int8 qunatization from int32 to int8.
+
+  // We assume the above operations are performed and use these values above
+  // as ranges for Requantization_range_perchannel_op.
+  range_weights_ch1 = 663110.59;  // For W1 channel
+  range_weights_ch2 = 994665.88;  // For W2 Channel
+
+  // We Fill the inputs Tensor T qint32 with arbitrary int32 values
+  test::FillValues<qint32>(
+      &input_tensor_qint32,
+      {-1000, -2000,  2000,   4000,   -3000,  -6000,  4000,   8000,
+       5000,  10000,  -6000,  -12000, 7000,   14000,  8000,   16000,
+       9000,  -18000, -10000, -20000, 11000,  22000,  -12000, -24000,
+       13000, 26000,  14000,  28000,  -15000, -30000, 16000,  32000});
+
+  // Step 5: Define and run requantization_range_perchannel
+  // -------------------------------------------------------
+  // See test RequantizationRangePerChannelTest_Basic and/or
+  // test RequantizationRangePerChannelTest_ClipMax
+}
+
+// Following tests the RequantizationRangePerChannel Op wherein the range
+// of the weights is calculated per channel.
+TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_Basic) {
+  // Let us setup the tensor and inputs before we run this op.
+  float clip_max_value = pow(2, 31);
+  float range_weights_ch1 = 0.0;
+  float range_weights_ch2 = 0.0;
+
+  // Create the input tensor
+  const int input_height = 4;
+  const int input_width = 4;
+  const int input_channels = 2;
+
+  // define and input tensor T shape.
+  Tensor input_tensor_qint32(DT_QINT32,
+                             {1, input_height, input_width, input_channels});
+
+  // Explanation and setup prior to this Op. Fill T and populate range values.
+  MklRequantizatedOpsTestHelper helper;
+  helper.Setup(input_tensor_qint32, range_weights_ch1, range_weights_ch2);
+
+  // Step 5: Define and run requantization_range_perchannel
+  // -------------------------------------------------------
+  // Define, Create and initalize the OP in question.
+  TF_ASSERT_OK(NodeDefBuilder("requantization_range_per_channel",
+                              "RequantizationRangePerChannel")
+                   .Input(FakeInput(DT_QINT32))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Attr("T", DataTypeToEnum<qint32>::v())
+                   .Attr("clip_value_max", clip_max_value)
+                   .Finalize(node_def()));
+  TF_ASSERT_OK(InitOp());
+
+  // Add the Input Nodes to the Op.
+  AddInputFromArray<qint32>(input_tensor_qint32.shape(),
+                            input_tensor_qint32.flat<qint32>());
+
+  // Calculate the Min and max from the ranges
+  float ch1_min = -1.0 * range_weights_ch1;
+  float ch1_max = range_weights_ch1;
+  float ch2_min = -1.0 * range_weights_ch2;
+  float ch2_max = range_weights_ch2;
+
+  // Add the Perchannel range Nodes to the Op.
+  AddInputFromArray<float>(TensorShape({input_channels}), {ch1_min, ch2_min});
+  AddInputFromArray<float>(TensorShape({input_channels}), {ch1_max, ch2_max});
+
+  // Run the Kernel
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Step 6: Verify Output and Store values to test Requantize_perchannel
+  // --------------------------------------------------------------------
+
+  // Verify the Expected Outputs
+  const float output_min = GetOutput(0)->flat<float>()(0);
+  const float output_max = GetOutput(1)->flat<float>()(0);
+  EXPECT_NEAR(-14.8217, output_min, 0.002);
+  EXPECT_NEAR(14.8217, output_max, 0.002);
+
+  // output range is made use in RequantizePerChannelTest_Basic
+}
+
+TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_ClipMax) {
+  // Let us setup the tensor and inputs before we run this op.
+  float clip_max_value = 6;  // Can be used as 6 for Relu 6 activations.
+  float range_weights_ch1 = 0.0;
+  float range_weights_ch2 = 0.0;
+
+  // Create the input tensor
+  const int input_height = 4;
+  const int input_width = 4;
+  const int input_channels = 2;
+
+  // define and input tensor T shape.
+  Tensor input_tensor_qint32(DT_QINT32,
+                             {1, input_height, input_width, input_channels});
+
+  // Explanation and setup prior to this Op. Fill T and populate range values.
+  MklRequantizatedOpsTestHelper helper;
+  helper.Setup(input_tensor_qint32, range_weights_ch1, range_weights_ch2);
+
+  // Step 5: Define and run requantization_range_perchannel
+  // -------------------------------------------------------
+  // Define, Create and initalize the OP in question.
+  TF_ASSERT_OK(NodeDefBuilder("requantization_range_per_channel",
+                              "RequantizationRangePerChannel")
+                   .Input(FakeInput(DT_QINT32))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Attr("T", DataTypeToEnum<qint32>::v())
+                   .Attr("clip_value_max", clip_max_value)
+                   .Finalize(node_def()));
+  TF_ASSERT_OK(InitOp());
+
+  // Add the Input Nodes to the Op.
+  AddInputFromArray<qint32>(input_tensor_qint32.shape(),
+                            input_tensor_qint32.flat<qint32>());
+
+  // Calculate the Min and max from the ranges
+  float ch1_min = -1.0 * range_weights_ch1;
+  float ch1_max = range_weights_ch1;
+  float ch2_min = -1.0 * range_weights_ch2;
+  float ch2_max = range_weights_ch2;
+
+  // Add the Perchannel range Nodes to the Op.
+  AddInputFromArray<float>(TensorShape({input_channels}), {ch1_min, ch2_min});
+  AddInputFromArray<float>(TensorShape({input_channels}), {ch1_max, ch2_max});
+
+  // Run the Kernel
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Step 6: Verify Output and Store values to test Requantize_perchannel
+  // --------------------------------------------------------------------
+
+  // Verify the Expected Outputs
+  const float output_min = GetOutput(0)->flat<float>()(0);
+  const float output_max = GetOutput(1)->flat<float>()(0);
+  EXPECT_NEAR(-6.0, output_min, 0.002);  // Values are Max as with clip_value
+  EXPECT_NEAR(6.0, output_max, 0.002);   // Values are Max as with clip_value
+}
+
+TEST_F(MklRequantizatedOpsTest, RequantizePerChannelTest_Basic) {
+  // Let us setup the tensor and inputs before we run this op.
+  float clip_max_value = pow(2, 31);
+  float range_weights_ch1 = 0.0;
+  float range_weights_ch2 = 0.0;
+
+  // Create the input tensor
+  const int input_height = 4;
+  const int input_width = 4;
+  const int input_channels = 2;
+
+  // define an input tensor T shape.
+  Tensor input_tensor_qint32(DT_QINT32,
+                             {1, input_height, input_width, input_channels});
+
+  // Explanation and setup prior to this Op. Fill T and populate range values.
+  MklRequantizatedOpsTestHelper helper;
+  helper.Setup(input_tensor_qint32, range_weights_ch1, range_weights_ch2);
+
+  // Step 7: Define and run requantize_perchannel
+  // --------------------------------------------
+  // The Output of Requantization_range_op_per_channel which calculated the
+  // new ranges of int8 is fed to the requantize per channel op.
+  // Here the Values of Convolved T*W is converted from int32 to int8.
+
+  TF_ASSERT_OK(NodeDefBuilder("requantize_per_channel", "RequantizePerChannel")
+                   .Input(FakeInput(DT_QINT32))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Attr("T", DataTypeToEnum<qint32>::v())
+                   .Attr("out_type", DataTypeToEnum<qint8>::v())
+                   .Finalize(node_def()));
+  TF_ASSERT_OK(InitOp());
+
+  // Add the Input Nodes to the Op.
+  AddInputFromArray<qint32>(input_tensor_qint32.shape(),
+                            input_tensor_qint32.flat<qint32>());
+
+  // Calculate the Min and max from the ranges
+  float ch1_min = -1.0 * range_weights_ch1;
+  float ch1_max = range_weights_ch1;
+  float ch2_min = -1.0 * range_weights_ch2;
+  float ch2_max = range_weights_ch2;
+
+  // Add the Perchannel range Nodes to the Op.
+  AddInputFromArray<float>(TensorShape({input_channels}), {ch1_min, ch2_min});
+  AddInputFromArray<float>(TensorShape({input_channels}), {ch1_max, ch2_max});
+
+  // Calculate the Min and max from Step 6 above
+  // in RequantizationRangePerChannelTest_Basic
+  float range_op_output_min = -14.8217;
+  float range_op_output_max = 14.8217;
+
+  // Add the Requested_min and requested_max stored from Step 6.
+  AddInputFromArray<float>(TensorShape({1}), {range_op_output_min});
+  AddInputFromArray<float>(TensorShape({1}), {range_op_output_max});
+
+  // Run the kernel
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Verify the output with the expected output
+  Tensor output = *GetOutput(0);
+  const float output_min = GetOutput(1)->flat<float>()(0);
+  const float output_max = GetOutput(2)->flat<float>()(0);
+  EXPECT_NEAR(range_op_output_min, output_min, 0.002);
+  EXPECT_NEAR(range_op_output_max, output_max, 0.002);
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc b/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc
new file mode 100644
index 0000000000..f9aa550866
--- /dev/null
+++ b/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc
@@ -0,0 +1,171 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// See docs in ../ops/array_ops.cc.
+
+#define EIGEN_USE_THREADS
+#ifdef INTEL_MKL
+#include <math.h>
+
+#include "mkldnn.hpp"
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/type_traits.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/kernels/meta_support.h"
+#include "tensorflow/core/kernels/no_op.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/util/mkl_util.h"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+
+namespace tensorflow {
+
+typedef Eigen::ThreadPoolDevice CPUDevice;
+
+template <typename Device, typename Toutput>
+class MklRequantizePerChannelOp : public OpKernel {
+ public:
+  explicit MklRequantizePerChannelOp(OpKernelConstruction* ctx)
+      : OpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("out_type", &out_type_));
+    OP_REQUIRES(ctx, out_type_ == DT_QINT8 || out_type_ == DT_QUINT8,
+                errors::InvalidArgument(
+                    "out_type must be qint8 or quint8, but got: " + out_type_));
+  }
+  virtual ~MklRequantizePerChannelOp() {}
+  void Compute(OpKernelContext* ctx) override {
+    try {
+      const Tensor& input = ctx->input(kInputTensorIndex);
+      const Tensor& input_min_vec = ctx->input(kInputMinVec);
+      float* input_min_vec_data = (float*)const_cast<void*>(
+          static_cast<const void*>(input_min_vec.flat<float>().data()));
+      const Tensor& input_max_vec = ctx->input(kInputMaxVec);
+      float* input_max_vec_data = (float*)const_cast<void*>(
+          static_cast<const void*>(input_max_vec.flat<float>().data()));
+
+      const Tensor& input_requested_min = ctx->input(this->kRequestMin);
+      const float input_requested_min_float =
+          input_requested_min.flat<float>()(0);
+      const Tensor& input_requested_max = ctx->input(this->kRequestMax);
+      const float input_requested_max_float =
+          input_requested_max.flat<float>()(0);
+
+      size_t depth = input_min_vec.NumElements();
+      OP_REQUIRES(
+          ctx, input_min_vec.dim_size(0) == depth,
+          errors::InvalidArgument("min has incorrect size, expected ", depth,
+                                  " was ", input_min_vec.dim_size(0)));
+      OP_REQUIRES(
+          ctx, input_max_vec.dim_size(0) == depth,
+          errors::InvalidArgument("max has incorrect size, expected ", depth,
+                                  " was ", input_max_vec.dim_size(0)));
+
+      if (out_type_ == DT_QINT8) CHECK(input_requested_min_float < 0.0f);
+
+      const float factor = (out_type_ == DT_QINT8) ? 127.0f : 255.0f;
+      float requested_min_max = std::max(std::abs(input_requested_min_float),
+                                         std::abs(input_requested_max_float));
+      Tensor* output = nullptr;
+      OP_REQUIRES_OK(ctx, ctx->allocate_output(kOutputTensorIndex,
+                                               input.shape(), &output));
+
+      std::vector<float> scales(depth);
+      for (int i = 0; i < depth; i++) {
+        float min_max_from_vec = std::max(std::abs(input_min_vec_data[i]),
+                                          std::abs(input_max_vec_data[i]));
+        float scale =
+            factor * (min_max_from_vec / requested_min_max / (float)(1L << 31));
+        scales[i] = scale;
+      }
+
+      mkldnn::primitive_attr reorder_attr;
+      reorder_attr.set_output_scales(2, scales);
+
+      memory::dims dims_mkl_order =
+          TFShapeToMklDnnDimsInNCHW(input.shape(), FORMAT_NHWC);
+      memory::desc input_md = memory::desc(dims_mkl_order, MklDnnType<qint32>(),
+                                           memory::format::nhwc);
+      memory::desc output_md =
+          (out_type_ == DT_QINT8)
+              ? memory::desc(dims_mkl_order, MklDnnType<qint8>(),
+                             memory::format::nhwc)
+              : memory::desc(dims_mkl_order, MklDnnType<quint8>(),
+                             memory::format::nhwc);
+
+      memory::primitive_desc input_pd =
+          memory::primitive_desc(input_md, cpu_engine_);
+      memory::primitive_desc output_pd =
+          memory::primitive_desc(output_md, cpu_engine_);
+
+      void* input_buf =
+          static_cast<void*>(const_cast<qint32*>(input.flat<qint32>().data()));
+      void* output_buf;
+      if (out_type_ == DT_QINT8) {
+        output_buf = static_cast<void*>(
+            const_cast<qint8*>(output->flat<qint8>().data()));
+      } else {
+        output_buf = static_cast<void*>(
+            const_cast<quint8*>(output->flat<quint8>().data()));
+      }
+
+      std::unique_ptr<memory> input_mem_prim_(new memory(input_pd, input_buf));
+      std::unique_ptr<memory> output_mem_prim_(
+          new memory(output_pd, output_buf));
+
+      mkldnn::reorder::primitive_desc reorder_pd =
+          mkldnn::reorder::primitive_desc(input_pd, output_pd, reorder_attr);
+      std::vector<mkldnn::primitive> net;
+      net.push_back(
+          mkldnn::reorder(reorder_pd, *input_mem_prim_, *output_mem_prim_));
+      stream(stream::kind::eager).submit(net).wait();
+
+      Tensor* output_min = nullptr;
+      Tensor* output_max = nullptr;
+      OP_REQUIRES_OK(ctx, ctx->allocate_output(kOutputMin, {}, &output_min));
+      OP_REQUIRES_OK(ctx, ctx->allocate_output(kOutputMax, {}, &output_max));
+
+      output_min->flat<float>()(0) = input_requested_min_float;
+      output_max->flat<float>()(0) = input_requested_max_float;
+
+    } catch (mkldnn::error& e) {
+      string error_msg = "Status: " + std::to_string(e.status) + ", message: " +
+                         std::string(e.message) + ", in file " +
+                         std::string(__FILE__) + ":" + std::to_string(__LINE__);
+      OP_REQUIRES_OK(
+          ctx, errors::Aborted("Operation received an exception:", error_msg));
+    }
+  }
+
+ private:
+  const int kInputTensorIndex = 0;
+  const int kInputMinVec = 1;
+  const int kInputMaxVec = 2;
+  const int kRequestMin = 3;
+  const int kRequestMax = 4;
+  const int kOutputTensorIndex = 0;
+  const int kOutputMin = 1;
+  const int kOutputMax = 2;
+  DataType out_type_;
+  engine cpu_engine_ = engine(engine::cpu, 0);
+};
+
+REGISTER_KERNEL_BUILDER(Name("RequantizePerChannel")
+                            .Device(DEVICE_CPU)
+                            .TypeConstraint<qint32>("T")
+                            .TypeConstraint<qint8>("out_type"),
+                        MklRequantizePerChannelOp<CPUDevice, qint8>);
+
+}  // namespace tensorflow
+#endif  // INTEL_MKL
diff --git a/tensorflow/core/kernels/requantize.cc b/tensorflow/core/kernels/requantize.cc
index dce6f1a185..ab057d5a67 100644
--- a/tensorflow/core/kernels/requantize.cc
+++ b/tensorflow/core/kernels/requantize.cc
@@ -19,7 +19,6 @@ limitations under the License.
 
 #include <math.h>
 
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/type_traits.h"
@@ -27,6 +26,7 @@ limitations under the License.
 #include "tensorflow/core/kernels/meta_support.h"
 #include "tensorflow/core/kernels/quantization_utils.h"
 #include "tensorflow/core/lib/core/errors.h"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
 namespace tensorflow {
 
@@ -100,4 +100,10 @@ REGISTER_KERNEL_BUILDER(Name("Requantize")
                             .TypeConstraint<quint8>("out_type"),
                         RequantizeOp<qint32, quint8>);
 
+REGISTER_KERNEL_BUILDER(Name("Requantize")
+                            .Device(DEVICE_CPU)
+                            .TypeConstraint<qint32>("Tinput")
+                            .TypeConstraint<qint8>("out_type"),
+                        RequantizeOp<qint32, qint8>);
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc
index eb9cbd3225..52243f2fcf 100644
--- a/tensorflow/core/ops/math_ops.cc
+++ b/tensorflow/core/ops/math_ops.cc
@@ -1247,12 +1247,12 @@ Status RangeSize(const Tensor* start_t, const Tensor* limit_t,
   T limit = limit_t->scalar<T>()();
   T delta = delta_t->scalar<T>()();
   if (start > limit && delta > 0) {
-    return errors::InvalidArgument(
-        "Requires start <= limit when delta > 0: ", start, "/", limit);
+    return errors::InvalidArgument("Requires start <= limit when delta > 0: ",
+                                   start, "/", limit);
   }
   if (start < limit && delta < 0) {
-    return errors::InvalidArgument(
-        "Requires start >= limit when delta < 0: ", start, "/", limit);
+    return errors::InvalidArgument("Requires start >= limit when delta < 0: ",
+                                   start, "/", limit);
   }
   if (delta == 0) {
     return errors::InvalidArgument("Requires delta != 0");
@@ -1686,6 +1686,45 @@ Add two input tensors element wise using mkl kernel sum.
 inputs: Must all be the same size and shape.
 )doc");
 
+REGISTER_OP("RequantizePerChannel")
+    .Input("input: T")
+    .Input("input_min: float")
+    .Input("input_max: float")
+    .Input("requested_output_min: float")
+    .Input("requested_output_max: float")
+    .Output("output: out_type")
+    .Output("output_min: float")
+    .Output("output_max: float")
+    .Attr("T: quantizedtype = DT_QINT32")
+    .Attr("out_type: quantizedtype = DT_QUINT8")
+    .SetShapeFn([](InferenceContext* c) {
+      TF_RETURN_IF_ERROR(shape_inference::UnchangedShape(c));
+      ShapeHandle unused;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &unused));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 1, &unused));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 0, &unused));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(4), 0, &unused));
+      c->set_output(1, c->Scalar());
+      c->set_output(2, c->Scalar());
+      return Status::OK();
+    });
+REGISTER_OP("RequantizationRangePerChannel")
+    .Input("input: T")
+    .Input("input_min: float")
+    .Input("input_max: float")
+    .Output("output_min: float")
+    .Output("output_max: float")
+    .Attr("T: quantizedtype = DT_QINT32")
+    .Attr("clip_value_max: float")
+    .SetShapeFn([](InferenceContext* c) {
+      ShapeHandle unused;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &unused));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 1, &unused));
+      c->set_output(0, c->Scalar());
+      c->set_output(1, c->Scalar());
+      return Status::OK();
+    });
+
 #endif  // INTEL_MKL
 
 REGISTER_OP("NextAfter")
-- 
GitLab


From 64e41a4952a9a3380204c322ce6f8fbe992b1816 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Fri, 25 Jan 2019 13:37:53 -0800
Subject: [PATCH 1296/2345] [TF:XLA] Bump open source llvm revision to r352179

PiperOrigin-RevId: 230960186
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 1412372e7d..7fbd1e2b5b 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -513,11 +513,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "1b5bf8ea78d81e163ae1766efb935b767684705d281ed8908f255cbac776b4ef",
-        strip_prefix = "llvm-cc563b6c997204f8c5dd76319a50007cebf03b3a",
+        sha256 = "d32baf8e3ccfdc689704ee8c9af5f17b9d2b571d3d5b25e857c375d28a435193",
+        strip_prefix = "llvm-ec3165a24ab126615b1766f6c48c0b0de844e4cb",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/cc563b6c997204f8c5dd76319a50007cebf03b3a.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/cc563b6c997204f8c5dd76319a50007cebf03b3a.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/ec3165a24ab126615b1766f6c48c0b0de844e4cb.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/ec3165a24ab126615b1766f6c48c0b0de844e4cb.tar.gz",
         ],
     )
 
-- 
GitLab


From f0fd6b98919a2713141365d6e3ee2b450e77d1d3 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Fri, 25 Jan 2019 13:42:00 -0800
Subject: [PATCH 1297/2345] [XLA:Python] Move the choice of which plugins to
 link into the XLA Python extension into xla.bzl.

PiperOrigin-RevId: 230960937
---
 tensorflow/compiler/xla/python/BUILD |  6 ++--
 tensorflow/compiler/xla/xla.bzl      | 54 ++++++++++++++++------------
 2 files changed, 34 insertions(+), 26 deletions(-)

diff --git a/tensorflow/compiler/xla/python/BUILD b/tensorflow/compiler/xla/python/BUILD
index 8276ee5a32..4afb21d5c8 100644
--- a/tensorflow/compiler/xla/python/BUILD
+++ b/tensorflow/compiler/xla/python/BUILD
@@ -3,8 +3,8 @@ licenses(["notice"])  # Apache 2.0
 package(default_visibility = ["//tensorflow:internal"])
 
 load("//tensorflow:tensorflow.bzl", "tf_py_wrap_cc")
-load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda_is_configured")
 load("//tensorflow/core:platform/default/build_config.bzl", "pyx_library")
+load("//tensorflow/compiler/xla:xla.bzl", "xla_python_default_plugins")
 
 py_library(
     name = "xla_client",
@@ -110,7 +110,5 @@ tf_py_wrap_cc(
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/service:cpu_plugin",
-    ] + if_cuda_is_configured([
-        "//tensorflow/compiler/xla/service:gpu_plugin",
-    ]),
+    ] + xla_python_default_plugins(),
 )
diff --git a/tensorflow/compiler/xla/xla.bzl b/tensorflow/compiler/xla/xla.bzl
index 1439f1bcc5..60adea5a4a 100644
--- a/tensorflow/compiler/xla/xla.bzl
+++ b/tensorflow/compiler/xla/xla.bzl
@@ -1,30 +1,40 @@
 """Wrapper around cc_proto_library used inside the XLA codebase."""
 
-load("//tensorflow/core:platform/default/build_config.bzl",
-     "cc_proto_library")
-load("//tensorflow/core:platform/default/build_config_root.bzl",
-     "if_static")
+load(
+    "//tensorflow/core:platform/default/build_config.bzl",
+    "cc_proto_library",
+)
+load(
+    "//tensorflow/core:platform/default/build_config_root.bzl",
+    "if_static",
+)
+load("//tensorflow:tensorflow.bzl", "if_cuda_is_configured")
 
 # xla_proto_library() is a convenience wrapper around cc_proto_library.
-def xla_proto_library(name, srcs=[], deps=[], visibility=None, testonly=0, **kwargs):
-  if kwargs.get('use_grpc_plugin'):
-    kwargs['use_grpc_namespace'] = True
-  cc_proto_library(name=name,
-                   srcs=srcs,
-                   deps=deps,
-                   cc_libs = if_static(
-                       ["@protobuf_archive//:protobuf"],
-                       otherwise=["@protobuf_archive//:protobuf_headers"],
-                   ),
-                   protoc="@protobuf_archive//:protoc",
-                   testonly=testonly,
-                   visibility=visibility,
-                   **kwargs)
+def xla_proto_library(name, srcs = [], deps = [], visibility = None, testonly = 0, **kwargs):
+    if kwargs.get("use_grpc_plugin"):
+        kwargs["use_grpc_namespace"] = True
+    cc_proto_library(
+        name = name,
+        srcs = srcs,
+        deps = deps,
+        cc_libs = if_static(
+            ["@protobuf_archive//:protobuf"],
+            otherwise = ["@protobuf_archive//:protobuf_headers"],
+        ),
+        protoc = "@protobuf_archive//:protoc",
+        testonly = testonly,
+        visibility = visibility,
+        **kwargs
+    )
 
 def xla_py_grpc_library(**kwargs):
-  # Note: we don't currently define any special targets for Python GRPC in OSS.
-  _ignore = kwargs
-  pass
-
+    # Note: we don't currently define any special targets for Python GRPC in OSS.
+    _ignore = kwargs
+    pass
 
 ORC_JIT_MEMORY_MAPPER_TARGETS = []
+
+# We link the GPU plugin into the XLA Python extension if CUDA is enabled.
+def xla_python_default_plugins():
+    return if_cuda_is_configured(["//tensorflow/compiler/xla/service:gpu_plugin"])
-- 
GitLab


From 8d0152b03369fe6a9ce91a61cee50c0b7767ee09 Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Fri, 25 Jan 2019 14:02:12 -0800
Subject: [PATCH 1298/2345] Enable opensource tests.

PiperOrigin-RevId: 230964456
---
 tensorflow/lite/tools/optimize/BUILD          | 280 +++++++++++++++++-
 .../lite/tools/optimize/calibrator_test.cc    |  35 ++-
 .../optimize/logging_op_resolver_test.cc      |  48 +--
 .../tools/optimize/node_info_delegate_test.cc |  42 ++-
 .../tools/optimize/quantize_model_test.cc     |   5 +-
 .../tools/optimize/quantize_weights_test.cc   |  22 +-
 .../lite/tools/optimize/subgraph_quantizer.cc |   2 +-
 .../tools/optimize/subgraph_quantizer_test.cc |   5 +-
 tensorflow/lite/tools/optimize/test_util.cc   |  10 +-
 .../lite/tools/optimize/testdata/README.md    |   8 +-
 ...single_avg_pool_min_minus_5_max_plus_5.bin | Bin 0 -> 508 bytes
 .../single_conv_weights_min_0_max_plus_10.bin | Bin 0 -> 1352 bytes
 ...onv_weights_min_minus_127_max_plus_127.bin | Bin 0 -> 1352 bytes
 .../single_softmax_min_minus_5_max_plus_5.bin | Bin 0 -> 464 bytes
 .../testdata/weight_shared_between_convs.bin  | Bin 0 -> 948 bytes
 15 files changed, 385 insertions(+), 72 deletions(-)
 create mode 100644 tensorflow/lite/tools/optimize/testdata/single_avg_pool_min_minus_5_max_plus_5.bin
 create mode 100644 tensorflow/lite/tools/optimize/testdata/single_conv_weights_min_0_max_plus_10.bin
 create mode 100644 tensorflow/lite/tools/optimize/testdata/single_conv_weights_min_minus_127_max_plus_127.bin
 create mode 100644 tensorflow/lite/tools/optimize/testdata/single_softmax_min_minus_5_max_plus_5.bin
 create mode 100644 tensorflow/lite/tools/optimize/testdata/weight_shared_between_convs.bin

diff --git a/tensorflow/lite/tools/optimize/BUILD b/tensorflow/lite/tools/optimize/BUILD
index 2cdec00433..ecc48f807e 100644
--- a/tensorflow/lite/tools/optimize/BUILD
+++ b/tensorflow/lite/tools/optimize/BUILD
@@ -1,14 +1,15 @@
-# TODO(suharshs): Write quantize_weights tests that use small exportable files.
-# Then we can remove this file.
-package(
-    default_visibility = ["//visibility:public"],
-)
+load("//tensorflow/lite:special_rules.bzl", "tflite_portable_test_suite")
+load("//tensorflow:tensorflow.bzl", "tf_cc_test")
 
-licenses(["notice"])  # Apache 2.0
+package(default_visibility = [
+    "//visibility:public",
+])
 
-exports_files(["LICENSE"])
+licenses(["notice"])  # Apache 2.0
 
-load("//tensorflow/lite:build_def.bzl", "tflite_copts")
+exports_files(glob([
+    "testdata/*.bin",
+]))
 
 cc_library(
     name = "quantization_utils",
@@ -16,16 +17,16 @@ cc_library(
     hdrs = ["quantization_utils.h"],
     deps = [
         "//tensorflow/lite:framework",
+        "//tensorflow/lite/c:c_api_internal",
+        "//tensorflow/lite/kernels/internal:round",
+        "//tensorflow/lite/kernels/internal:types",
         "//tensorflow/lite/schema:schema_fbs",
-        "@com_google_absl//absl/memory",
-        "@flatbuffers",
     ],
 )
 
-cc_test(
-    name = "quantizatin_utils_test",
+tf_cc_test(
+    name = "quantization_utils_test",
     srcs = ["quantization_utils_test.cc"],
-    copts = ["-Wall"],
     tags = [
         "tflite_not_portable_android",
         "tflite_not_portable_ios",
@@ -45,11 +46,264 @@ cc_library(
     hdrs = ["quantize_weights.h"],
     deps = [
         ":quantization_utils",
+        "@com_google_absl//absl/memory",
+        "@flatbuffers",
+        "//tensorflow/lite:framework",
+        # TODO(suharshs): Move the relevant quantization utils to a non-internal location.
+        "//tensorflow/lite/kernels/internal:tensor_utils",
+        "//tensorflow/lite/schema:schema_fbs",
         "//tensorflow/core:tflite_portable_logging",
+    ],
+)
+
+cc_library(
+    name = "calibrator_lib",
+    srcs = ["calibrator.cc"],
+    hdrs = ["calibrator.h"],
+    deps = [
+        ":calibration_common",
+        ":calibration_logger",
+        ":calibration_reader",
+        ":logging_op_resolver",
+        ":node_info_delegate",
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite:string_util",
+        "//tensorflow/lite/c:c_api_internal",
+        "//tensorflow/lite/core/api",
+        "//tensorflow/lite/kernels:builtin_ops",
+        "//tensorflow/lite/schema:schema_fbs",
+        "@com_google_absl//absl/memory",
+        "@flatbuffers",
+    ],
+)
+
+tf_cc_test(
+    name = "calibrator_test",
+    srcs = ["calibrator_test.cc"],
+    args = [
+        "--test_model_file=$(location //tensorflow/lite:testdata/multi_add.bin)",
+    ],
+    data = [
+        "//tensorflow/lite:testdata/multi_add.bin",
+    ],
+    tags = [
+        "tflite_not_portable_android",
+        "tflite_not_portable_ios",
+    ],
+    deps = [
+        ":calibrator_lib",
+        "//tensorflow/core:framework_internal",
+        "//tensorflow/core:lib",
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite/kernels:builtin_ops",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
+cc_library(
+    name = "logging_op_resolver",
+    srcs = ["logging_op_resolver.cc"],
+    hdrs = ["logging_op_resolver.h"],
+    deps = [
+        ":calibration_common",
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite/core/api",
+        "@com_google_absl//absl/memory",
+    ],
+)
+
+cc_test(
+    name = "logging_op_resolver_test",
+    srcs = ["logging_op_resolver_test.cc"],
+    deps = [
+        ":logging_op_resolver",
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite/kernels:builtin_ops",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
+cc_library(
+    name = "calibration_reader",
+    srcs = ["calibration_reader.cc"],
+    hdrs = ["calibration_reader.h"],
+    deps = [
+        ":calibration_logger",
+        "//tensorflow/lite:framework",
+        "@com_google_absl//absl/memory",
+    ],
+)
+
+cc_library(
+    name = "calibration_logger",
+    hdrs = ["calibration_logger.h"],
+    deps = [
+        "//tensorflow/lite/c:c_api_internal",
+    ],
+)
+
+cc_library(
+    name = "calibration_common",
+    hdrs = ["calibration_common.h"],
+    deps = [
+        "//tensorflow/lite:framework",
+    ],
+)
+
+cc_library(
+    name = "node_info_delegate",
+    srcs = ["node_info_delegate.cc"],
+    hdrs = ["node_info_delegate.h"],
+    deps = [
+        ":calibration_common",
+        "//tensorflow/lite:framework",
+    ],
+)
+
+tf_cc_test(
+    name = "node_info_delegate_test",
+    srcs = ["node_info_delegate_test.cc"],
+    args = [
+        "--test_model_file=$(location //tensorflow/lite/tools/optimize:testdata/single_conv_weights_min_0_max_plus_10.bin)",
+    ],
+    data = [
+        "//tensorflow/lite/tools/optimize:testdata/single_conv_weights_min_0_max_plus_10.bin",
+    ],
+    tags = [
+        "tflite_not_portable_android",
+        "tflite_not_portable_ios",
+    ],
+    deps = [
+        ":node_info_delegate",
+        ":test_util",
+        "//tensorflow/core:framework_internal",
+        "//tensorflow/core:lib",
         "//tensorflow/lite:framework",
+        "//tensorflow/lite/kernels:builtin_ops",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
+tf_cc_test(
+    name = "quantize_weights_test",
+    srcs = ["quantize_weights_test.cc"],
+    args = [
+        "--test_model_file=$(location //tensorflow/lite/tools/optimize:testdata/single_conv_weights_min_0_max_plus_10.bin)",
+    ],
+    data = [
+        "//tensorflow/lite/tools/optimize:testdata/single_conv_weights_min_0_max_plus_10.bin",
+        "//tensorflow/lite/tools/optimize:testdata/weight_shared_between_convs.bin",
+    ],
+    tags = [
+        "tflite_not_portable_android",
+        "tflite_not_portable_ios",
+    ],
+    deps = [
+        ":quantize_weights",
+        ":test_util",
+        "//tensorflow/core:framework_internal",
+        "//tensorflow/core:lib",
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite/schema:schema_fbs",
+        "@com_google_googletest//:gtest",
+        "@flatbuffers",
+    ],
+)
+
+cc_library(
+    name = "subgraph_quantizer",
+    srcs = ["subgraph_quantizer.cc"],
+    hdrs = ["subgraph_quantizer.h"],
+    deps = [
+        ":quantization_utils",
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite/core/api",
+        "//tensorflow/lite/kernels/internal:round",
         "//tensorflow/lite/kernels/internal:tensor_utils",
         "//tensorflow/lite/schema:schema_fbs",
         "@com_google_absl//absl/memory",
         "@flatbuffers",
     ],
 )
+
+cc_library(
+    name = "test_util",
+    testonly = 1,
+    srcs = ["test_util.cc"],
+    hdrs = ["test_util.h"],
+    deps = [
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite/core/api",
+        "@com_google_googletest//:gtest",
+        "@flatbuffers",
+    ],
+)
+
+tf_cc_test(
+    name = "subgraph_quantizer_test",
+    srcs = ["subgraph_quantizer_test.cc"],
+    args = [
+        "--test_model_file=$(location //tensorflow/lite/tools/optimize:testdata/single_conv_weights_min_0_max_plus_10.bin)",
+    ],
+    data = [
+        "//tensorflow/lite/tools/optimize:testdata/single_avg_pool_min_minus_5_max_plus_5.bin",
+        "//tensorflow/lite/tools/optimize:testdata/single_conv_weights_min_0_max_plus_10.bin",
+        "//tensorflow/lite/tools/optimize:testdata/single_conv_weights_min_minus_127_max_plus_127.bin",
+        "//tensorflow/lite/tools/optimize:testdata/single_softmax_min_minus_5_max_plus_5.bin",
+    ],
+    tags = [
+        "tflite_not_portable_android",
+        "tflite_not_portable_ios",
+    ],
+    deps = [
+        ":subgraph_quantizer",
+        ":test_util",
+        "//tensorflow/core:framework_internal",
+        "//tensorflow/core:lib",
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite/schema:schema_fbs",
+        "@com_google_googletest//:gtest",
+        "@flatbuffers",
+    ],
+)
+
+cc_library(
+    name = "quantize_model",
+    srcs = ["quantize_model.cc"],
+    hdrs = ["quantize_model.h"],
+    deps = [
+        ":subgraph_quantizer",
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite/core/api",
+        "//tensorflow/lite/schema:schema_fbs",
+        "@com_google_absl//absl/memory",
+        "@flatbuffers",
+    ],
+)
+
+tf_cc_test(
+    name = "quantize_model_test",
+    srcs = ["quantize_model_test.cc"],
+    args = [
+        "--test_model_file=$(location //tensorflow/lite/tools/optimize:testdata/single_conv_weights_min_0_max_plus_10.bin)",
+    ],
+    data = [
+        "//tensorflow/lite/tools/optimize:testdata/single_conv_weights_min_0_max_plus_10.bin",
+    ],
+    tags = [
+        "tflite_not_portable_android",
+        "tflite_not_portable_ios",
+    ],
+    deps = [
+        ":quantize_model",
+        ":test_util",
+        "//tensorflow/core:framework_internal",
+        "//tensorflow/core:lib",
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite/schema:schema_fbs",
+        "@com_google_googletest//:gtest",
+        "@flatbuffers",
+    ],
+)
+
+tflite_portable_test_suite()
diff --git a/tensorflow/lite/tools/optimize/calibrator_test.cc b/tensorflow/lite/tools/optimize/calibrator_test.cc
index bbbcc70fae..1a415adc55 100644
--- a/tensorflow/lite/tools/optimize/calibrator_test.cc
+++ b/tensorflow/lite/tools/optimize/calibrator_test.cc
@@ -16,18 +16,31 @@ limitations under the License.
 
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
+#include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/platform/init_main.h"
+#include "tensorflow/core/util/command_line_flags.h"
 #include "tensorflow/lite/kernels/register.h"
 #include "tensorflow/lite/model.h"
 #include "tensorflow/lite/tools/optimize/calibrator.h"
 
+namespace {
+tensorflow::string* g_test_model_file = nullptr;
+}  // namespace
+
 namespace tflite {
 namespace optimize {
 namespace calibration {
 namespace {
 
+std::unique_ptr<FlatBufferModel> ReadModel() {
+  if (g_test_model_file) {
+    return FlatBufferModel::BuildFromFile(g_test_model_file->c_str());
+  }
+  return nullptr;
+}
+
 TEST(CalibratorTest, CalibrationStatsAreCollected) {
-  auto model = FlatBufferModel::BuildFromFile(
-      "third_party/tensorflow/lite/testdata/multi_add.bin");
+  auto model = ReadModel();
   ASSERT_TRUE(model);
   std::unique_ptr<Interpreter> interpreter;
   std::unique_ptr<CalibrationReader> reader;
@@ -105,8 +118,7 @@ TEST(CalibratorTest, CalibrationStatsAreCollected) {
 }
 
 TEST(CalibratorTest, MultipleInvokes) {
-  auto model = FlatBufferModel::BuildFromFile(
-      "third_party/tensorflow/lite/testdata/multi_add.bin");
+  auto model = ReadModel();
   ASSERT_TRUE(model);
   std::unique_ptr<Interpreter> interpreter;
   std::unique_ptr<CalibrationReader> reader;
@@ -183,7 +195,18 @@ TEST(CalibratorTest, MultipleInvokes) {
 }  // namespace tflite
 
 int main(int argc, char** argv) {
-  // On Linux, add: FLAGS_logtostderr = true;
-  ::testing::InitGoogleTest(&argc, argv);
+  tensorflow::string model_file;
+  const std::vector<tensorflow::Flag> flag_list = {
+      tensorflow::Flag("test_model_file", &model_file,
+                       "Path to test tflite model file."),
+  };
+
+  const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list);
+  if (!parse_result) {
+    std::cerr << "Required test_model_file\n";
+    std::abort();
+  }
+  g_test_model_file = new tensorflow::string(model_file);
+  ::tensorflow::port::InitMain(argv[0], &argc, &argv);
   return RUN_ALL_TESTS();
 }
diff --git a/tensorflow/lite/tools/optimize/logging_op_resolver_test.cc b/tensorflow/lite/tools/optimize/logging_op_resolver_test.cc
index 18c29abec6..c7fe2d37ce 100644
--- a/tensorflow/lite/tools/optimize/logging_op_resolver_test.cc
+++ b/tensorflow/lite/tools/optimize/logging_op_resolver_test.cc
@@ -44,16 +44,16 @@ TfLiteStatus WrappingInvoke(TfLiteContext* context, TfLiteNode* node) {
 
 TEST(LoggingOpResolverTest, KernelInvokesAreReplaced) {
   MutableOpResolver base_resolver;
-  TfLiteRegistration conv_registration = {
-      .prepare = ConvPrepare,
-      .invoke = ConvEval,
-  };
+  TfLiteRegistration conv_registration = {};
+  conv_registration.prepare = ConvPrepare;
+  conv_registration.invoke = ConvEval;
+
   base_resolver.AddBuiltin(BuiltinOperator_CONV_2D, &conv_registration);
 
-  TfLiteRegistration add_registration = {
-      .prepare = AddPrepare,
-      .invoke = AddEval,
-  };
+  TfLiteRegistration add_registration = {};
+  add_registration.prepare = AddPrepare;
+  add_registration.invoke = AddEval;
+
   base_resolver.AddBuiltin(BuiltinOperator_ADD, &add_registration);
   BuiltinOpsSet ops_to_replace = {
       {BuiltinOperator_CONV_2D, /*version*/ 1},
@@ -77,16 +77,16 @@ TEST(LoggingOpResolverTest, KernelInvokesAreReplaced) {
 
 TEST(LoggingOpResolverTest, OriginalKernelInvokesAreRetained) {
   MutableOpResolver base_resolver;
-  TfLiteRegistration conv_registration = {
-      .prepare = ConvPrepare,
-      .invoke = ConvEval,
-  };
+  TfLiteRegistration conv_registration = {};
+  conv_registration.prepare = ConvPrepare;
+  conv_registration.invoke = ConvEval;
+
   base_resolver.AddBuiltin(BuiltinOperator_CONV_2D, &conv_registration);
 
-  TfLiteRegistration add_registration = {
-      .prepare = AddPrepare,
-      .invoke = AddEval,
-  };
+  TfLiteRegistration add_registration = {};
+  add_registration.prepare = AddPrepare;
+  add_registration.invoke = AddEval;
+
   base_resolver.AddBuiltin(BuiltinOperator_ADD, &add_registration);
   BuiltinOpsSet ops_to_replace = {
       {BuiltinOperator_CONV_2D, /*version*/ 1},
@@ -103,16 +103,16 @@ TEST(LoggingOpResolverTest, OriginalKernelInvokesAreRetained) {
 
 TEST(LoggingOpResolverTest, OnlyOpsInReplacementSetAreReplaces) {
   MutableOpResolver base_resolver;
-  TfLiteRegistration conv_registration = {
-      .prepare = ConvPrepare,
-      .invoke = ConvEval,
-  };
+  TfLiteRegistration conv_registration = {};
+  conv_registration.prepare = ConvPrepare;
+  conv_registration.invoke = ConvEval;
+
   base_resolver.AddBuiltin(BuiltinOperator_CONV_2D, &conv_registration);
 
-  TfLiteRegistration add_registration = {
-      .prepare = AddPrepare,
-      .invoke = AddEval,
-  };
+  TfLiteRegistration add_registration = {};
+  add_registration.prepare = AddPrepare;
+  add_registration.invoke = AddEval;
+
   base_resolver.AddBuiltin(BuiltinOperator_ADD, &add_registration);
   // Only replace conv2d
   BuiltinOpsSet ops_to_replace = {
diff --git a/tensorflow/lite/tools/optimize/node_info_delegate_test.cc b/tensorflow/lite/tools/optimize/node_info_delegate_test.cc
index e762d5c014..05332c56b5 100644
--- a/tensorflow/lite/tools/optimize/node_info_delegate_test.cc
+++ b/tensorflow/lite/tools/optimize/node_info_delegate_test.cc
@@ -16,15 +16,32 @@ limitations under the License.
 
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
+#include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/platform/init_main.h"
+#include "tensorflow/core/util/command_line_flags.h"
 #include "tensorflow/lite/kernels/register.h"
 #include "tensorflow/lite/model.h"
 #include "tensorflow/lite/tools/optimize/node_info_delegate.h"
+#include "tensorflow/lite/tools/optimize/test_util.h"
+
+namespace {
+tensorflow::string* g_test_model_dir = nullptr;
+}  // namespace
 
 namespace tflite {
 namespace optimize {
 namespace calibration {
 namespace {
 
+std::unique_ptr<FlatBufferModel> ReadModel(const char* model) {
+  auto model_path = tensorflow::io::JoinPath(*g_test_model_dir, model);
+  return FlatBufferModel::BuildFromFile(model_path.c_str());
+}
+
+std::unique_ptr<FlatBufferModel> ReadModel() {
+  return ReadModel(internal::kConvModelWith0Plus10Weights);
+}
+
 class TestDelegateObserver : public DelegateObserver {
  public:
   explicit TestDelegateObserver(TfLiteStatus status_to_return)
@@ -45,8 +62,7 @@ TEST(NodeInfoDelegateTest, DelegateObserverIsCalled) {
   TestDelegateObserver observer(kTfLiteOk);
   NodeInfoDelegateParams params;
   params.delegate_observer = &observer;
-  auto model = FlatBufferModel::BuildFromFile(
-      "third_party/tensorflow/lite/testdata/multi_add.bin");
+  auto model = ReadModel();
   ASSERT_TRUE(model);
   std::unique_ptr<Interpreter> interpreter;
   ASSERT_EQ(InterpreterBuilder(*model,
@@ -66,8 +82,7 @@ TEST(NodeInfoDelegateTest, ObserverErrorCausesModifyGraphFailure) {
   TestDelegateObserver observer(kTfLiteError);
   NodeInfoDelegateParams params;
   params.delegate_observer = &observer;
-  auto model = FlatBufferModel::BuildFromFile(
-      "third_party/tensorflow/lite/testdata/multi_add.bin");
+  auto model = ReadModel();
   ASSERT_TRUE(model);
   std::unique_ptr<Interpreter> interpreter;
   ASSERT_EQ(InterpreterBuilder(*model,
@@ -81,8 +96,7 @@ TEST(NodeInfoDelegateTest, ObserverErrorCausesModifyGraphFailure) {
 }
 
 TEST(NodeInfoDelegateTest, NodeInfoDelegateObserver) {
-  auto model = FlatBufferModel::BuildFromFile(
-      "third_party/tensorflow/lite/testdata/multi_add.bin");
+  auto model = ReadModel();
   ASSERT_TRUE(model);
 
   std::unordered_map<int, OperatorInfo> index_to_opinfo;
@@ -146,7 +160,19 @@ TEST(NodeInfoDelegateTest, NodeInfoDelegateObserver) {
 }  // namespace tflite
 
 int main(int argc, char** argv) {
-  // On Linux, add: FLAGS_logtostderr = true;
-  ::testing::InitGoogleTest(&argc, argv);
+  tensorflow::string model_file;
+  const std::vector<tensorflow::Flag> flag_list = {
+      tensorflow::Flag("test_model_file", &model_file,
+                       "Path to test tflite model file."),
+  };
+
+  const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list);
+  if (!parse_result) {
+    std::cerr << "Required test_model_file\n";
+    std::abort();
+  }
+  g_test_model_dir =
+      new tensorflow::string(tensorflow::io::Dirname(model_file));
+  ::tensorflow::port::InitMain(argv[0], &argc, &argv);
   return RUN_ALL_TESTS();
 }
diff --git a/tensorflow/lite/tools/optimize/quantize_model_test.cc b/tensorflow/lite/tools/optimize/quantize_model_test.cc
index 52ed16c0b8..cf3eb2dde6 100644
--- a/tensorflow/lite/tools/optimize/quantize_model_test.cc
+++ b/tensorflow/lite/tools/optimize/quantize_model_test.cc
@@ -142,7 +142,10 @@ int main(int argc, char** argv) {
   };
 
   const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list);
-  CHECK(parse_result) << "Required test_model_file";
+  if (!parse_result) {
+    std::cerr << "Required test_model_file\n";
+    std::abort();
+  }
   g_test_model_dir =
       new tensorflow::string(tensorflow::io::Dirname(model_file));
   ::tensorflow::port::InitMain(argv[0], &argc, &argv);
diff --git a/tensorflow/lite/tools/optimize/quantize_weights_test.cc b/tensorflow/lite/tools/optimize/quantize_weights_test.cc
index 3ded2546c3..a18b3bb7ff 100644
--- a/tensorflow/lite/tools/optimize/quantize_weights_test.cc
+++ b/tensorflow/lite/tools/optimize/quantize_weights_test.cc
@@ -89,21 +89,21 @@ class QuantizeWeightsTest : public testing::Test {
   }
 
   // Returns the producer op code of the specified tensor_idx.
-  BuiltinOperator GetProducerOpCode(const Model* model, uint32_t subgraph_idx,
-                                    uint32_t tensor_idx) {
+  bool GetProducerOpCode(const Model* model, uint32_t subgraph_idx,
+                         uint32_t tensor_idx,
+                         tflite::BuiltinOperator* op_code) {
     const auto subgraph = model->subgraphs()->Get(subgraph_idx);
     for (size_t op_idx = 0; op_idx < subgraph->operators()->size(); ++op_idx) {
       const auto op = subgraph->operators()->Get(op_idx);
       for (size_t i = 0; i < op->outputs()->size(); ++i) {
         if (op->outputs()->Get(i) == tensor_idx) {
           const uint32_t op_code_idx = op->opcode_index();
-          return model->operator_codes()->Get(op_code_idx)->builtin_code();
+          *op_code = model->operator_codes()->Get(op_code_idx)->builtin_code();
+          return true;
         }
       }
     }
-
-    LOG(FATAL) << "tensor_idx " << tensor_idx
-               << " not produced by op in model.";
+    return false;
   }
 };
 
@@ -323,8 +323,9 @@ TEST_F(QuantizeWeightsTest, SharedWeights_Dequantize) {
         EXPECT_EQ(weights_tensor->type(), TensorType_FLOAT32);
 
         // Check that it comes from a dequantize operation.
-        const auto producer_op_code =
-            GetProducerOpCode(output_model, subgraph_idx, weights_tensor_index);
+        BuiltinOperator producer_op_code;
+        ASSERT_TRUE(GetProducerOpCode(output_model, subgraph_idx,
+                                      weights_tensor_index, &producer_op_code));
         EXPECT_EQ(producer_op_code, BuiltinOperator_DEQUANTIZE);
       }
     }
@@ -345,7 +346,10 @@ int main(int argc, char** argv) {
   };
 
   const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list);
-  CHECK(parse_result) << "Required test_model_file";
+  if (!parse_result) {
+    std::cerr << "Required test_model_file\n";
+    std::abort();
+  }
   g_test_model_dir =
       new tensorflow::string(tensorflow::io::Dirname(model_file));
   ::tensorflow::port::InitMain(argv[0], &argc, &argv);
diff --git a/tensorflow/lite/tools/optimize/subgraph_quantizer.cc b/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
index 99cc8584c6..118e055058 100644
--- a/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
+++ b/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
@@ -60,8 +60,8 @@ bool OpHasOptionalBiasTensor(BuiltinOperator op_code) {
 
 struct OpWithBiasTensors {
   int activation_input_index;
-  int bias_input_index;
   int weights_input_index;
+  int bias_input_index;
   int index_for_channel_in_weights;
 };
 
diff --git a/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc b/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
index 580ed5eb26..4b23ced0fb 100644
--- a/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
+++ b/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
@@ -384,7 +384,10 @@ int main(int argc, char** argv) {
   };
 
   const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list);
-  CHECK(parse_result) << "Required test_model_file";
+  if (!parse_result) {
+    std::cerr << "Required test_model_file\n";
+    std::abort();
+  }
   g_test_model_dir =
       new tensorflow::string(tensorflow::io::Dirname(model_file));
   ::tensorflow::port::InitMain(argv[0], &argc, &argv);
diff --git a/tensorflow/lite/tools/optimize/test_util.cc b/tensorflow/lite/tools/optimize/test_util.cc
index 30dc1a4f1e..190242402b 100644
--- a/tensorflow/lite/tools/optimize/test_util.cc
+++ b/tensorflow/lite/tools/optimize/test_util.cc
@@ -20,18 +20,18 @@ namespace tflite {
 namespace optimize {
 namespace internal {
 const char* kConvModelWithMinus128Plus127Weights =
-    "single_conv_weights_min_minus_127_max_plus_127.tflite";
+    "single_conv_weights_min_minus_127_max_plus_127.bin";
 
 const char* kConvModelWith0Plus10Weights =
-    "single_conv_weights_min_0_max_plus_10.tflite";
+    "single_conv_weights_min_0_max_plus_10.bin";
 
 const char* kSingleSoftmaxModelMinMinus5MaxPlus5 =
-    "single_softmax_min_minus_5_max_plus_5.tflite";
+    "single_softmax_min_minus_5_max_plus_5.bin";
 
 const char* kSingleAvgPoolModelMinMinus5MaxPlus5 =
-    "single_avg_pool_min_minus_5_max_plus_5.tflite";
+    "single_avg_pool_min_minus_5_max_plus_5.bin";
 
-const char* kModelWithSharedWeights = "weight_shared_between_convs.tflite";
+const char* kModelWithSharedWeights = "weight_shared_between_convs.bin";
 
 int FailOnErrorReporter::Report(const char* format, va_list args) {
   char buf[1024];
diff --git a/tensorflow/lite/tools/optimize/testdata/README.md b/tensorflow/lite/tools/optimize/testdata/README.md
index d8a156546d..21fcd32b1e 100644
--- a/tensorflow/lite/tools/optimize/testdata/README.md
+++ b/tensorflow/lite/tools/optimize/testdata/README.md
@@ -4,21 +4,21 @@ This directory contains test models for testing quantization.
 
 ## Models
 
-* `single_conv_weights_min_0_max_plus_10.tflite` \
+* `single_conv_weights_min_0_max_plus_10.bin` \
    A floating point model with single convolution where all weights are
    integers between [0, 10] weights are randomly distributed. It is not
    guaranteed that min max for weights are going to appear in each channel.
    All activations have min maxes and activations are in range [0,10].
-* `single_conv_weights_min_minus_127_max_plus_127.tflite` \
+* `single_conv_weights_min_minus_127_max_plus_127.bin` \
    A floating point model with a single convolution where weights of the model
    are all integers that lie in range[-127, 127]. The weights have been put in
    such a way that each channel has at least one weight as -127 and one weight
    as 127. The activations are all in range: [-128, 127].
    This means all bias computations should result in 1.0 scale.
-* `single_softmax_min_minus_5_max_5.tflite` \
+* `single_softmax_min_minus_5_max_5.bin` \
    A floating point model with a single softmax. The input tensor has min
    and max in range [-5, 5], not necessarily -5 or +5.
-* `single_avg_pool_input_min_minus_5_max_5.tflite` \
+* `single_avg_pool_input_min_minus_5_max_5.bin` \
    A floating point model with a single average pool. The input tensor has min
    and max in range [-5, 5], not necessarily -5 or +5.
 * `weight_shared_between_convs.tflite` \
diff --git a/tensorflow/lite/tools/optimize/testdata/single_avg_pool_min_minus_5_max_plus_5.bin b/tensorflow/lite/tools/optimize/testdata/single_avg_pool_min_minus_5_max_plus_5.bin
new file mode 100644
index 0000000000000000000000000000000000000000..a65f39ee29514b27ea3af861c10dd452ab9e5ce2
GIT binary patch
literal 508
zcmb1OU|<Mw^D$;%;A4<rU}4~3;9(G85MkhBU|?WoU|=}H$iU#hz`&rvz`(!)6$9zv
zf%1R+|NsBb|NsA47(kGLfgfa^zq7xBbADb~YEemQiXH<a0|SEq0|NsOg9zLPkp311
z28Id-28JA{Ss-;H3?RVAAjZG|b_+X$0N5=I3=FI`3=9kg3=9k~H4+T$U^OuJGB7YO
zg3JJ6CI$uu5C*Yf7-TYx4WdDOkeUXl{nP&c{~rJ{gn@xU1uE9@|NnoG+d+N@>2YS7
z=m4VMzi&Sv!@$6hSe9Cpn4TJ6ke{EEnU`*q5^rdr?^u=|ke?6o7C6j6>OuNJ7!;<U
zum*($#D@_7c|hH(0}dyM3@A1@pkW8n3o>Wg_xWJ^%YV;50E(l4oW$hRjQpIG)S`Gp
iV@Mo<)UkpB3>se`J}A5y7}yv%7}y!uKptmcU<3em<TgeC

literal 0
HcmV?d00001

diff --git a/tensorflow/lite/tools/optimize/testdata/single_conv_weights_min_0_max_plus_10.bin b/tensorflow/lite/tools/optimize/testdata/single_conv_weights_min_0_max_plus_10.bin
new file mode 100644
index 0000000000000000000000000000000000000000..70cbc0620ad7222817cf241030acb98387083154
GIT binary patch
literal 1352
zcmb1OU|<Mw^D$;%;A4<rU}4~3;9(G85MkhBU|?WoU|<knWnh@W#K6$N#K6G9z`(%D
zz`#(z#K7Re#K0iJz`(%601;pD_y7MjfB*l#_V@pPkUAMA1`u#$U}&&sU=VPGU=Ulu
zk%0k(85|fG4mdC{I5<Gmz}PTxkQ#Ih(*M8#q8}y)5?kQFz`)=L!7%ec);B=S2B`(<
z1E~SI1;hvGfr)|Cg7kspK=y;=L2@7rG6SR!hC%v3Vjv8bw}-eNq#wiwt7Cxp3nT_M
zlL4Xz<W`W`ATbaIsRxOH)PeMY<Ur<w+yj<_x*ud0$Xy^cAUQY<l6PQW0O<i?kU9_z
z(hE`pqCsYY*dPqD2P6i<Aag)82!q@Jqe1#X7^EJg7Zxud_k!3Uaj;uJ20-H##0SZP
z^nvt%&2oUoIW)dNdO;Xu7YMU4a4@hiutC!gC@n$a4<rXN4~!jI7(kGLfghAk{hj?4
zob&U_Qj1DbQ}h@ZLFp5m21Vd07nB|w7#J8z7#J8bpk{;ALCog^r&&-MXJ-%qr*Q@b
z21Wx01_l+lSpp0U;9S5CPTQbt07}Xr3^E5qgYpFkg8~2~2Erh55DgLsrT+*<28KIG
zZaDVu|9=mtdrYA8j(`9EgYq9pJxCvNx`yyUHb5{a-(}_%l!E*PQV+5V6m}pCQmgj=
z|NjV(V;LA2T%hJl{Qv)70m=sHM>ZeEhKNJ#Kqf)v=9iX$Eri(zvIm4gdZzvV|33y4
zcu4kjP+%V?0|P^HeqLF8Qf6W?)Lf7~pmGTmyO216#2F}FLE-HKj(dm<sN~{cU|=BE
nzo2vjw!b_zGd-gO<OFzm2l6uvgTfu8mVp73e%Zn47ZmIO921pX

literal 0
HcmV?d00001

diff --git a/tensorflow/lite/tools/optimize/testdata/single_conv_weights_min_minus_127_max_plus_127.bin b/tensorflow/lite/tools/optimize/testdata/single_conv_weights_min_minus_127_max_plus_127.bin
new file mode 100644
index 0000000000000000000000000000000000000000..29b9f47097d466b65831514cec3a00f19f5cbdf3
GIT binary patch
literal 1352
zcmb1OU|<Mw^D$;%;A4<rU}4~3;9(G85MkhBU|?WoU|<knWnh@W#K6$N#K6G9z`(%D
zz`#(z#K7Re#K0iJz`(%601;pD_y7MjfB*l#_V@pPkUAMA1_lO(e}@<t{y8x)oO5De
zSmngPu<sB9!v`k@hD%Ni41I?f7&x357+el9Fq}KYz~JD>z>si=fngq${l|%c;gu6a
z|2roJ1`u9%h=C!)31Y@KsQj@*3=9ni5OxI|Vqn;Ih=JkHAqIv82N@VZ@>iS~7zCUc
z7^XQfFdTsDYp`cvP;g>knBm00@a7N$1J5A_hD}Zk43iEqFx+!uV6cGV3kMk(TAUad
zN}L!NDnR;?>;mauc8Gyt#UTa;pF<1`AyD()9b#axImEz_a)^OppA!Sa3y|4Re}eqA
z#EF3cggrp!LDjZ7F)%!HVqjo6z`y{~ui?bN&~OmpuPKKZ7&@F77@i!0_ygqT1&0_I
zb{u42a5%ugU~rIuApy!3f%+BXo@-7F3{6f940D_y?g@bU3lujX_eVG}FswPmzyQ(%
z5(C8-DDHZo;WWdMfkEXEB-}PQGBC_@VqoYw#K7<k6z))eR2+i%hrxk?;eitagO3vf
z!-j(l3@4l*aR7?P6^@WN2F2lyLktXT3>*wB3~bQ!14>H_44^RBbclhW>ktD2$jo_8
zEDRvXz`zemr~c0V3eNd?WvN9asVRC4jG*)hPJ<%wlnY9a4GatnB@7G<8Bnu9>LBLx
zfzvD~jk7ZdfYUew0|TQ00|SE!+$;eG25>H52d8aNHUK4M5C)k8qCxorgh8PM5(8n7
zIEV&`gVKKlBLl-7BsU!U_y4~K)IBCpddI*2|3Uc=q#mRXlGZ`t|IZ1+XE@Bj0KuSq
zmzh^k3i207J;*Ll*nu!et=j+p|06(-Wnf@%ftoMz|NnmlC>x|7*?brqA`Y<wW*3MC
znVVl)0=5ulAIKgM2I-mh|Ns9OP~aii*Fm&>;B*Em8{n9efq@}8Kd&r4DKoJcYA(ng
zP`Lz(T}Yfj;tUk8pz!tq$2~*_RB~}JFfb73UrhT!=>}|nd1_{QMhVCX@bV7iXBY;B
PJ4h`g{j!78FDTdnRR{!@

literal 0
HcmV?d00001

diff --git a/tensorflow/lite/tools/optimize/testdata/single_softmax_min_minus_5_max_plus_5.bin b/tensorflow/lite/tools/optimize/testdata/single_softmax_min_minus_5_max_plus_5.bin
new file mode 100644
index 0000000000000000000000000000000000000000..3caba63492e174229ef605bfbb0d2ddeda2ba61d
GIT binary patch
literal 464
zcmb1OU|<Mw^D$;%;A4<rU}4~3;9(G85MkhBU|?WoU|^WR$iU#hz`&rvz`(!)6$9zv
zf%1R+|NsBb|NsA47(kGLfgfa^zq7xBbADb~YEemQiXH<a0|SEq0|NsOg9zLPkp3J7
z28INXdZ<|-bs`KPz{eoQzyNj&JA(k&Ees3{oGJ_q3^2PH7}yv%7+4tCK*|{!>_K{A
z7$gK^gJ=++iGhIuWFN>6tN#E0Z^OU<b{EJ@kl396|Nn#B2x5cun0&1Vo2e^qVZp_~
zz!03DR+5`o5pQS+2?vlGRt5$J5C(-4DEvUagSZ;vW)B7i29UivV826TKq1Tl4G)-I
u)4tDl0GU_*d;S4XxCi7UCZ}fP=cJ?-#Ty!7w-2Nq6o!y+V26eSgCqd1vn`eY

literal 0
HcmV?d00001

diff --git a/tensorflow/lite/tools/optimize/testdata/weight_shared_between_convs.bin b/tensorflow/lite/tools/optimize/testdata/weight_shared_between_convs.bin
new file mode 100644
index 0000000000000000000000000000000000000000..4c903c82eec32df8aa0d3462262b61daa30fc251
GIT binary patch
literal 948
zcmb1OU|<Mw^D$;%;A4<rU}4~3;9(G85MkhBU|?WoU|=X<W?<l9U|<knU|>*SU|?Wm
zU|^WSz`(!{(&z8&ui%`YSC(2-lA5B&z`?-4kio#fkifve;KRVcV8g(`puxbvAi@AK
zgN1>CVb0(G{}=rI|Nqwi|NlXJ5cZukWB)58J_E?UfB*k8Ft9OjFt9MNF)%QI)PXQa
z?!n*x|DXK*|DT6J1ny#x8$ot~`~z|a3ljqa6Vwl^3=lR*FUbAu3=9l1P`B;@`HPW(
zA%u~EVG1Jy!xIJuhNgf2|AYJsQ>Oy8AEXx<gXA;w3Q9}j4H-84hv)&h5u}EJq23un
zGca6&ng{X+$X*?S<{5$G0Tde`zku9=&0H<0ogn{#%*EwSkiFTdMR}<?3=A^=|Nl>d
z>I0bxG8?29<W>;oVqjp%FD;2TiciW+EM{Qv`Tzg_B&d2&dH|`#We><5AiahR3=A#*
z|NlP))dz|PkUg;UgKUo>)E-bQgWLm3hoD$x0Ed78I30rYg7kpw2E{Q*9f*&NVRjpV
z!<Ge{UO@JO(jrLxp8ufq#J~tL3uF$+4<H)kCy+THwV-eUu|Z~m(#kTZ9*`Iai!eY^
z7Y_qFI6r_=HYja_vIWds18CX;nF}%lWG={lkbamL0|OU>00TSNhoJBV=?BTfFeoj9
W(moplD>&^lFoW^|JA}=^zzhI=^KQ-n

literal 0
HcmV?d00001

-- 
GitLab


From b46eac4b90ddc45dfd374e4000654113789b12ec Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Fri, 25 Jan 2019 14:06:02 -0800
Subject: [PATCH 1299/2345] Add v2 APIs for metrics and losses. - Split
 categorical crossentropy and sparse categorical crossentropy loss into 2
 classes as we cannot always distinguish between the two with just the
 `y_pred` and `y_true` shapes.

PiperOrigin-RevId: 230965385
---
 tensorflow/python/keras/losses.py             | 186 +++++++++++-----
 tensorflow/python/keras/losses_test.py        |  76 +++----
 tensorflow/python/keras/metrics.py            |  39 ++--
 tensorflow/python/keras/metrics_test.py       |  18 +-
 .../v1/tensorflow.keras.losses.-huber.pbtxt   |  22 ++
 ...sorflow.keras.losses.-k-l-divergence.pbtxt |  22 ++
 .../tensorflow.keras.losses.-log-cosh.pbtxt   |  22 ++
 .../tensorflow.keras.losses.-log-loss.pbtxt   |  22 ++
 .../v1/tensorflow.keras.losses.-loss.pbtxt    |  21 ++
 .../v1/tensorflow.keras.losses.-poisson.pbtxt |  22 ++
 ...ses.-sparse-categorical-crossentropy.pbtxt |  22 ++
 .../golden/v1/tensorflow.keras.losses.pbtxt   |  28 +++
 .../v1/tensorflow.keras.metrics.-a-u-c.pbtxt  | 200 +++++++++++++++++
 ...w.keras.metrics.-binary-crossentropy.pbtxt | 199 +++++++++++++++++
 ...orflow.keras.metrics.-k-l-divergence.pbtxt | 199 +++++++++++++++++
 ...orflow.keras.metrics.-log-cosh-error.pbtxt | 199 +++++++++++++++++
 .../tensorflow.keras.metrics.-mean-io-u.pbtxt | 196 +++++++++++++++++
 ...w.keras.metrics.-mean-relative-error.pbtxt | 198 +++++++++++++++++
 ...ensorflow.keras.metrics.-mean-tensor.pbtxt | 204 ++++++++++++++++++
 .../v1/tensorflow.keras.metrics.-metric.pbtxt | 195 +++++++++++++++++
 .../tensorflow.keras.metrics.-poisson.pbtxt   | 199 +++++++++++++++++
 ...ras.metrics.-root-mean-squared-error.pbtxt | 198 +++++++++++++++++
 ...s.-sparse-top-k-categorical-accuracy.pbtxt | 199 +++++++++++++++++
 ....metrics.-top-k-categorical-accuracy.pbtxt | 199 +++++++++++++++++
 .../golden/v1/tensorflow.keras.metrics.pbtxt  |  48 +++++
 .../v2/tensorflow.keras.losses.-huber.pbtxt   |  22 ++
 ...sorflow.keras.losses.-k-l-divergence.pbtxt |  22 ++
 .../tensorflow.keras.losses.-log-cosh.pbtxt   |  22 ++
 .../tensorflow.keras.losses.-log-loss.pbtxt   |  22 ++
 .../v2/tensorflow.keras.losses.-loss.pbtxt    |  21 ++
 .../v2/tensorflow.keras.losses.-poisson.pbtxt |  22 ++
 ...ses.-sparse-categorical-crossentropy.pbtxt |  22 ++
 .../golden/v2/tensorflow.keras.losses.pbtxt   |  28 +++
 .../v2/tensorflow.keras.metrics.-a-u-c.pbtxt  | 200 +++++++++++++++++
 ...w.keras.metrics.-binary-crossentropy.pbtxt | 199 +++++++++++++++++
 ...orflow.keras.metrics.-k-l-divergence.pbtxt | 199 +++++++++++++++++
 ...orflow.keras.metrics.-log-cosh-error.pbtxt | 199 +++++++++++++++++
 .../tensorflow.keras.metrics.-mean-io-u.pbtxt | 196 +++++++++++++++++
 ...w.keras.metrics.-mean-relative-error.pbtxt | 198 +++++++++++++++++
 ...ensorflow.keras.metrics.-mean-tensor.pbtxt | 204 ++++++++++++++++++
 .../v2/tensorflow.keras.metrics.-metric.pbtxt | 195 +++++++++++++++++
 .../tensorflow.keras.metrics.-poisson.pbtxt   | 199 +++++++++++++++++
 ...ras.metrics.-root-mean-squared-error.pbtxt | 198 +++++++++++++++++
 ...s.-sparse-top-k-categorical-accuracy.pbtxt | 199 +++++++++++++++++
 ....metrics.-top-k-categorical-accuracy.pbtxt | 199 +++++++++++++++++
 .../golden/v2/tensorflow.keras.metrics.pbtxt  |  48 +++++
 46 files changed, 5436 insertions(+), 111 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-huber.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-k-l-divergence.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-log-cosh.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-log-loss.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-loss.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-poisson.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-sparse-categorical-crossentropy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-a-u-c.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-crossentropy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-k-l-divergence.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-log-cosh-error.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-io-u.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-relative-error.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-tensor.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-metric.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-poisson.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-root-mean-squared-error.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-top-k-categorical-accuracy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-top-k-categorical-accuracy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-huber.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-k-l-divergence.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-log-cosh.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-log-loss.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-loss.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-poisson.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-sparse-categorical-crossentropy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-a-u-c.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-crossentropy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-k-l-divergence.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-log-cosh-error.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-io-u.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-relative-error.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-tensor.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-metric.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-poisson.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-root-mean-squared-error.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-top-k-categorical-accuracy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-top-k-categorical-accuracy.pbtxt

diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index 9144eb0bf9..5a66be8a56 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -34,8 +34,10 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
 from tensorflow.python.ops.losses import losses_impl
 from tensorflow.python.util.tf_export import keras_export
+from tensorflow.tools.docs import doc_controls
 
 
+@keras_export('keras.losses.Loss')
 class Loss(object):
   """Loss base class.
 
@@ -108,6 +110,7 @@ class Loss(object):
     return {'reduction': self.reduction, 'name': self.name}
 
   @abc.abstractmethod
+  @doc_controls.for_subclass_implementers
   def call(self, y_true, y_pred):
     """Invokes the `Loss` instance.
 
@@ -272,7 +275,14 @@ class MeanSquaredLogarithmicError(Loss):
 
 @keras_export('keras.losses.BinaryCrossentropy')
 class BinaryCrossentropy(Loss):
-  """Computes the binary cross entropy loss between the labels and predictions.
+  """Computes the crossentropy loss between the labels and predictions.
+
+  Use this crossentropy loss function when there are only two label classes
+  (assumed to be 0 and 1). There should be a single floating point value per
+  feature.
+
+  In the snippet below, there is a single floating pointing value per example,
+  and the shape of both `y_pred` and `y_true` are `[batch_size]`.
 
   Usage:
 
@@ -287,11 +297,11 @@ class BinaryCrossentropy(Loss):
   ```python
   model = keras.models.Model(inputs, outputs)
   model.compile('sgd', loss=tf.keras.losses.BinaryCrossentropy())
-  ````
+  ```
 
   Args:
-    from_logits: Whether `output` is expected to be a logits tensor. By default,
-      we consider that `output` encodes a probability distribution.
+    from_logits: Whether `y_pred` is expected to be a logits tensor. By default,
+      we assume that `y_pred` encodes a probability distribution.
     label_smoothing: Float in [0, 1]. If > `0` then smooth the labels.
     reduction: Type of `tf.losses.Reduction` to apply to loss. Default value is
       `SUM_OVER_BATCH_SIZE`.
@@ -329,7 +339,16 @@ class BinaryCrossentropy(Loss):
 
 @keras_export('keras.losses.CategoricalCrossentropy')
 class CategoricalCrossentropy(Loss):
-  """Computes categorical cross entropy loss between the `y_true` and `y_pred`.
+  """Computes the crossentropy loss between the labels and predictions.
+
+  Use this crossentropy loss function when there are two or more label classes.
+  We expect labels to be provided in a `one_hot` representation. If you want to
+  provide labels as integers, please use `SparseCategoricalCrossentropy` loss.
+  There should be `# classes` floating point values per feature.
+
+  In the snippet below, there is `# classes` floating pointing values per
+  example. The shape of both `y_pred` and `y_true` are
+  `[batch_size, num_classes]`.
 
   Usage:
 
@@ -346,14 +365,15 @@ class CategoricalCrossentropy(Loss):
   ```python
   model = keras.models.Model(inputs, outputs)
   model.compile('sgd', loss=tf.keras.losses.CategoricalCrossentropy())
-  ````
+  ```
 
   Args:
-    from_logits: Whether `output` is expected to be a logits tensor. By default,
-      we consider that `output` encodes a probability distribution.
-    label_smoothing: Float in [0, 1]. If > `0` then smooth the labels. This
-      option is currently not supported when `y_pred` is a sparse input
-      (not one-hot).
+    from_logits: Whether `y_pred` is expected to be a logits tensor. By default,
+      we assume that `y_pred` encodes a probability distribution.
+    label_smoothing: Float in [0, 1]. When > 0, label values are smoothed,
+      meaning the confidence on label values are relaxed. e.g.
+      `label_smoothing=0.2` means that we will use a value of `0.1` for label
+      `0` and `0.9` for label `1`"
     reduction: Type of `tf.losses.Reduction` to apply to loss. Default value is
       `SUM_OVER_BATCH_SIZE`.
     name: Optional name for the op.
@@ -381,19 +401,76 @@ class CategoricalCrossentropy(Loss):
       Categorical cross entropy losses.
     """
     y_pred = ops.convert_to_tensor(y_pred)
-    y_true = ops.convert_to_tensor(y_true)
-    is_sparse = y_pred.shape != y_true.shape
+    y_true = math_ops.cast(y_true, y_pred.dtype)
+    return categorical_crossentropy(
+        y_true,
+        y_pred,
+        from_logits=self.from_logits,
+        label_smoothing=self.label_smoothing)
 
-    if is_sparse:
-      return sparse_categorical_crossentropy(
-          y_true, y_pred, from_logits=self.from_logits)
-    else:
-      y_true = math_ops.cast(y_true, y_pred.dtype)
-      return categorical_crossentropy(
-          y_true,
-          y_pred,
-          from_logits=self.from_logits,
-          label_smoothing=self.label_smoothing)
+
+@keras_export('keras.losses.SparseCategoricalCrossentropy')
+class SparseCategoricalCrossentropy(Loss):
+  """Computes the crossentropy loss between the labels and predictions.
+
+  Use this crossentropy loss function when there are two or more label classes.
+  We expect labels to be provided as integers. If you want to provide labels
+  using `one-hot` representation, please use `CategoricalCrossentropy` loss.
+  There should be `# classes` floating point values per feature for `y_pred`
+  and a single floating point value per feature for `y_true`.
+
+  In the snippet below, there is a single floating point value per example for
+  `y_true` and `# classes` floating pointing values per example for `y_pred`.
+  The shape of `y_true` is `[batch_size]` and the shape of `y_pred` is
+  `[batch_size, num_classes]`.
+
+  Usage:
+
+  ```python
+  cce = tf.keras.losses.SparseCategoricalCrossentropy()
+  loss = cce(
+    [0, 1, 2],
+    [[.9, .05, .05], [.5, .89, .6], [.05, .01, .94]])
+  print('Loss: ', loss.numpy())  # Loss: 0.3239
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', loss=tf.keras.losses.SparseCategoricalCrossentropy())
+  ````
+
+  Args:
+    from_logits: Whether `y_pred` is expected to be a logits tensor. By default,
+      we assume that `y_pred` encodes a probability distribution.
+    reduction: Type of `tf.losses.Reduction` to apply to loss. Default value is
+      `SUM_OVER_BATCH_SIZE`.
+    name: Optional name for the op.
+  """
+
+  def __init__(self,
+               from_logits=False,
+               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               name=None):
+    super(SparseCategoricalCrossentropy, self).__init__(
+        reduction=reduction, name=name)
+    self.from_logits = from_logits
+
+  def call(self, y_true, y_pred):
+    """Invokes the `SparseCategoricalCrossentropy` instance.
+
+    Args:
+      y_true: Ground truth values.
+      y_pred: The predicted values.
+
+    Returns:
+      Sparse categorical cross entropy losses.
+    """
+    y_pred = ops.convert_to_tensor(y_pred)
+    y_true = ops.convert_to_tensor(y_true)
+    return sparse_categorical_crossentropy(
+        y_true, y_pred, from_logits=self.from_logits)
 
 
 @keras_export('keras.losses.Hinge')
@@ -501,10 +578,11 @@ class CategoricalHinge(Loss):
     return categorical_hinge(y_true, y_pred)
 
 
+@keras_export('keras.losses.LogLoss')
 class LogLoss(Loss):
   """Computes the log loss between `y_true` and `y_pred`.
 
-  logloss = - y_true * log(y_pred) - (1 - y_true) * log(1 - y_pred)
+  `logloss = - y_true * log(y_pred) - (1 - y_true) * log(1 - y_pred)`
 
   Usage:
 
@@ -528,10 +606,11 @@ class LogLoss(Loss):
     return logloss(y_true, y_pred)
 
 
+@keras_export('keras.losses.Poisson')
 class Poisson(Loss):
-  """Computes the poisson loss between `y_true` and `y_pred`.
+  """Computes the Poisson loss between `y_true` and `y_pred`.
 
-  loss = y_pred - y_true * log(y_pred)
+  `loss = y_pred - y_true * log(y_pred)`
 
   Usage:
 
@@ -555,15 +634,16 @@ class Poisson(Loss):
     return poisson(y_true, y_pred)
 
 
-class Logcosh(Loss):
+@keras_export('keras.losses.LogCosh')
+class LogCosh(Loss):
   """Computes the logarithm of the hyperbolic cosine of the prediction error.
 
-  logcosh = log((exp(x) + exp(-x))/2) where x is the error `y_pred` - `y_true`.
+  `logcosh = log((exp(x) + exp(-x))/2)`, where x is the error (y_pred - y_true)
 
   Usage:
 
   ```python
-  l = tf.losses.Logcosh()
+  l = tf.losses.LogCosh()
   loss = l([0., 1., 1.], [1., 0., 1.])
   print('Loss: ', loss.numpy())  # Loss: 0.289
   ```
@@ -572,7 +652,7 @@ class Logcosh(Loss):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.losses.Logcosh())
+  model.compile('sgd', loss=tf.losses.LogCosh())
   ```
   """
 
@@ -582,15 +662,16 @@ class Logcosh(Loss):
     return logcosh(y_true, y_pred)
 
 
-class KullbackLeiblerDivergence(Loss):
-  """Computes kullback leibler divergence loss between `y_true` and `y_pred`.
+@keras_export('keras.losses.KLDivergence')
+class KLDivergence(Loss):
+  """Computes Kullback Leibler divergence loss between `y_true` and `y_pred`.
 
-  loss = y_true * log(y_true / y_pred)
+  `loss = y_true * log(y_true / y_pred)`
 
   Usage:
 
   ```python
-  k = tf.losses.KullbackLeiblerDivergence()
+  k = tf.losses.KLDivergence()
   loss = k([.4, .9, .2], [.5, .8, .12])
   print('Loss: ', loss.numpy())  # Loss: -0.043
   ```
@@ -599,7 +680,7 @@ class KullbackLeiblerDivergence(Loss):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.losses.KullbackLeiblerDivergence())
+  model.compile('sgd', loss=tf.losses.KLDivergence())
   ```
   """
 
@@ -609,21 +690,22 @@ class KullbackLeiblerDivergence(Loss):
     return kullback_leibler_divergence(y_true, y_pred)
 
 
-class HuberLoss(Loss):
-  """Computes the huber loss between `y_true` and `y_pred`.
+@keras_export('keras.losses.Huber')
+class Huber(Loss):
+  """Computes the Huber loss between `y_true` and `y_pred`.
 
   For each value x in `error=y_true-y_pred`, the following is calculated:
 
-    ```
-    0.5 * x^2                  if |x| <= d
-    0.5 * d^2 + d * (|x| - d)  if |x| > d
-    ```
+  ```
+  0.5 * x^2                  if |x| <= d
+  0.5 * d^2 + d * (|x| - d)  if |x| > d
+  ```
   where d is `delta`. See: https://en.wikipedia.org/wiki/Huber_loss
 
   Usage:
 
   ```python
-  l = tf.losses.HuberLoss()
+  l = tf.losses.Huber()
   loss = l([0., 1., 1.], [1., 0., 1.])
   print('Loss: ', loss.numpy())  # Loss: 0.333
   ```
@@ -632,11 +714,11 @@ class HuberLoss(Loss):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.losses.HuberLoss())
+  model.compile('sgd', loss=tf.losses.Huber())
   ```
 
   Args:
-    delta: A float, the point where the huber loss function changes from a
+    delta: A float, the point where the Huber loss function changes from a
       quadratic to linear.
     reduction: Type of `tf.losses.Reduction` to apply to loss. Default value is
       `SUM_OVER_BATCH_SIZE`.
@@ -647,7 +729,7 @@ class HuberLoss(Loss):
                delta=1.0,
                reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
                name=None):
-    super(HuberLoss, self).__init__(reduction=reduction, name=name)
+    super(Huber, self).__init__(reduction=reduction, name=name)
     self.delta = delta
 
   def call(self, y_true, y_pred):
@@ -726,20 +808,20 @@ def logloss(y_true, y_pred):
 
 
 def huber_loss(y_true, y_pred, delta=1.0):
-  """Computes huber loss value.
+  """Computes Huber loss value.
 
   For each value x in `error=y_true-y_pred`, the following is calculated:
 
-    ```
-    0.5 * x^2                  if |x| <= d
-    0.5 * d^2 + d * (|x| - d)  if |x| > d
-    ```
+  ```
+  0.5 * x^2                  if |x| <= d
+  0.5 * d^2 + d * (|x| - d)  if |x| > d
+  ```
   where d is `delta`. See: https://en.wikipedia.org/wiki/Huber_loss
 
   Args:
     y_true: tensor of true targets.
     y_pred: tensor of predicted targets.
-    delta: A float, the point where the huber loss function changes from a
+    delta: A float, the point where the Huber loss function changes from a
       quadratic to linear.
 
   Returns:
@@ -793,7 +875,7 @@ def categorical_crossentropy(y_true,
     y_true: tensor of true targets.
     y_pred: tensor of predicted targets.
     from_logits: Whether `y_pred` is expected to be a logits tensor. By default,
-      we consider that `y_pred` encodes a probability distribution.
+      we assume that `y_pred` encodes a probability distribution.
     label_smoothing: Float in [0, 1]. If > `0` then smooth the labels.
 
   Returns:
diff --git a/tensorflow/python/keras/losses_test.py b/tensorflow/python/keras/losses_test.py
index d276854e29..6b6067828f 100644
--- a/tensorflow/python/keras/losses_test.py
+++ b/tensorflow/python/keras/losses_test.py
@@ -830,22 +830,26 @@ class CategoricalCrossentropyTest(test.TestCase):
     expected_value = 400.0 * label_smoothing / 3.0
     self.assertAlmostEqual(self.evaluate(loss), expected_value, 3)
 
-  def test_all_correct_unweighted_sparse(self):
+
+@test_util.run_all_in_graph_and_eager_modes
+class SparseCategoricalCrossentropyTest(test.TestCase):
+
+  def test_all_correct_unweighted(self):
     y_true = constant_op.constant([[0], [1], [2]], dtype=dtypes.int64)
     y_pred = constant_op.constant([[1., 0., 0.], [0., 1., 0.], [0., 0., 1.]],
                                   dtype=dtypes.float32)
-    cce_obj = keras.losses.CategoricalCrossentropy()
+    cce_obj = keras.losses.SparseCategoricalCrossentropy()
     loss = cce_obj(y_true, y_pred)
     self.assertAlmostEqual(self.evaluate(loss), 0.0, 3)
 
     # Test with logits.
     logits = constant_op.constant([[10., 0., 0.], [0., 10., 0.], [0., 0., 10.]])
-    cce_obj = keras.losses.CategoricalCrossentropy(from_logits=True)
+    cce_obj = keras.losses.SparseCategoricalCrossentropy(from_logits=True)
     loss = cce_obj(y_true, logits)
     self.assertAlmostEqual(self.evaluate(loss), 0.0, 3)
 
-  def test_unweighted_sparse(self):
-    cce_obj = keras.losses.CategoricalCrossentropy()
+  def test_unweighted(self):
+    cce_obj = keras.losses.SparseCategoricalCrossentropy()
     y_true = constant_op.constant([0, 1, 2])
     y_pred = constant_op.constant(
         [[.9, .05, .05], [.5, .89, .6], [.05, .01, .94]], dtype=dtypes.float32)
@@ -854,12 +858,12 @@ class CategoricalCrossentropyTest(test.TestCase):
 
     # Test with logits.
     logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
-    cce_obj = keras.losses.CategoricalCrossentropy(from_logits=True)
+    cce_obj = keras.losses.SparseCategoricalCrossentropy(from_logits=True)
     loss = cce_obj(y_true, logits)
     self.assertAlmostEqual(self.evaluate(loss), .0573, 3)
 
-  def test_scalar_weighted_sparse(self):
-    cce_obj = keras.losses.CategoricalCrossentropy()
+  def test_scalar_weighted(self):
+    cce_obj = keras.losses.SparseCategoricalCrossentropy()
     y_true = constant_op.constant([[0], [1], [2]])
     y_pred = constant_op.constant(
         [[.9, .05, .05], [.5, .89, .6], [.05, .01, .94]], dtype=dtypes.float32)
@@ -868,12 +872,12 @@ class CategoricalCrossentropyTest(test.TestCase):
 
     # Test with logits.
     logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
-    cce_obj = keras.losses.CategoricalCrossentropy(from_logits=True)
+    cce_obj = keras.losses.SparseCategoricalCrossentropy(from_logits=True)
     loss = cce_obj(y_true, logits, sample_weight=2.3)
     self.assertAlmostEqual(self.evaluate(loss), .1317, 3)
 
-  def test_sample_weighted_sparse(self):
-    cce_obj = keras.losses.CategoricalCrossentropy()
+  def test_sample_weighted(self):
+    cce_obj = keras.losses.SparseCategoricalCrossentropy()
     y_true = constant_op.constant([[0], [1], [2]])
     y_pred = constant_op.constant(
         [[.9, .05, .05], [.5, .89, .6], [.05, .01, .94]], dtype=dtypes.float32)
@@ -883,14 +887,14 @@ class CategoricalCrossentropyTest(test.TestCase):
 
     # Test with logits.
     logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
-    cce_obj = keras.losses.CategoricalCrossentropy(from_logits=True)
+    cce_obj = keras.losses.SparseCategoricalCrossentropy(from_logits=True)
     loss = cce_obj(y_true, logits, sample_weight=sample_weight)
     self.assertAlmostEqual(self.evaluate(loss), 0.31829, 3)
 
-  def test_no_reduction_sparse(self):
+  def test_no_reduction(self):
     y_true = constant_op.constant([[0], [1], [2]])
     logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
-    cce_obj = keras.losses.CategoricalCrossentropy(
+    cce_obj = keras.losses.SparseCategoricalCrossentropy(
         from_logits=True, reduction=losses_impl.ReductionV2.NONE)
     loss = cce_obj(y_true, logits)
     self.assertAllClose((0.001822, 0.000459, 0.169846), self.evaluate(loss), 3)
@@ -1181,7 +1185,7 @@ class LogLossTest(test.TestCase):
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class LogcoshTest(test.TestCase):
+class LogCoshTest(test.TestCase):
 
   def setup(self):
     y_pred = np.asarray([1, 9, 2, -5, -2, 6]).reshape((2, 3))
@@ -1195,14 +1199,14 @@ class LogcoshTest(test.TestCase):
     self.y_true = constant_op.constant(y_true)
 
   def test_config(self):
-    logcosh_obj = keras.losses.Logcosh(
+    logcosh_obj = keras.losses.LogCosh(
         reduction=losses_impl.ReductionV2.SUM, name='logcosh_loss')
     self.assertEqual(logcosh_obj.name, 'logcosh_loss')
     self.assertEqual(logcosh_obj.reduction, losses_impl.ReductionV2.SUM)
 
   def test_unweighted(self):
     self.setup()
-    logcosh_obj = keras.losses.Logcosh()
+    logcosh_obj = keras.losses.LogCosh()
 
     loss = logcosh_obj(self.y_true, self.y_pred)
     expected_loss = np.sum(self.expected_losses) / self.batch_size
@@ -1210,7 +1214,7 @@ class LogcoshTest(test.TestCase):
 
   def test_scalar_weighted(self):
     self.setup()
-    logcosh_obj = keras.losses.Logcosh()
+    logcosh_obj = keras.losses.LogCosh()
     sample_weight = 2.3
 
     loss = logcosh_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
@@ -1224,7 +1228,7 @@ class LogcoshTest(test.TestCase):
 
   def test_sample_weighted(self):
     self.setup()
-    logcosh_obj = keras.losses.Logcosh()
+    logcosh_obj = keras.losses.LogCosh()
 
     sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
     loss = logcosh_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
@@ -1237,7 +1241,7 @@ class LogcoshTest(test.TestCase):
 
   def test_timestep_weighted(self):
     self.setup()
-    logcosh_obj = keras.losses.Logcosh()
+    logcosh_obj = keras.losses.LogCosh()
     y_true = np.asarray([1, 9, 2, -5, -2, 6]).reshape(2, 3, 1)
     y_pred = np.asarray([4, 8, 12, 8, 1, 3]).reshape(2, 3, 1)
     error = y_pred - y_true
@@ -1255,7 +1259,7 @@ class LogcoshTest(test.TestCase):
 
   def test_zero_weighted(self):
     self.setup()
-    logcosh_obj = keras.losses.Logcosh()
+    logcosh_obj = keras.losses.LogCosh()
     sample_weight = 0
     loss = logcosh_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
     self.assertAlmostEqual(self.evaluate(loss), 0., 3)
@@ -1343,7 +1347,7 @@ class PoissonTest(test.TestCase):
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class KullbackLeiblerDivergenceTest(test.TestCase):
+class KLDivergenceTest(test.TestCase):
 
   def setup(self):
     self.np_y_pred = np.asarray([.4, .9, .12, .36, .3, .4]).reshape((2, 3))
@@ -1357,14 +1361,14 @@ class KullbackLeiblerDivergenceTest(test.TestCase):
     self.y_true = constant_op.constant(self.np_y_true)
 
   def test_config(self):
-    k_obj = keras.losses.KullbackLeiblerDivergence(
+    k_obj = keras.losses.KLDivergence(
         reduction=losses_impl.ReductionV2.SUM, name='kld')
     self.assertEqual(k_obj.name, 'kld')
     self.assertEqual(k_obj.reduction, losses_impl.ReductionV2.SUM)
 
   def test_unweighted(self):
     self.setup()
-    k_obj = keras.losses.KullbackLeiblerDivergence()
+    k_obj = keras.losses.KLDivergence()
 
     loss = k_obj(self.y_true, self.y_pred)
     expected_loss = np.sum(self.expected_losses) / self.batch_size
@@ -1372,7 +1376,7 @@ class KullbackLeiblerDivergenceTest(test.TestCase):
 
   def test_scalar_weighted(self):
     self.setup()
-    k_obj = keras.losses.KullbackLeiblerDivergence()
+    k_obj = keras.losses.KLDivergence()
     sample_weight = 2.3
 
     loss = k_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
@@ -1386,7 +1390,7 @@ class KullbackLeiblerDivergenceTest(test.TestCase):
 
   def test_sample_weighted(self):
     self.setup()
-    k_obj = keras.losses.KullbackLeiblerDivergence()
+    k_obj = keras.losses.KLDivergence()
     sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
     loss = k_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
 
@@ -1398,7 +1402,7 @@ class KullbackLeiblerDivergenceTest(test.TestCase):
 
   def test_timestep_weighted(self):
     self.setup()
-    k_obj = keras.losses.KullbackLeiblerDivergence()
+    k_obj = keras.losses.KLDivergence()
     y_true = self.np_y_true.reshape(2, 3, 1)
     y_pred = self.np_y_pred.reshape(2, 3, 1)
     sample_weight = np.asarray([3, 6, 5, 0, 4, 2]).reshape(2, 3)
@@ -1417,7 +1421,7 @@ class KullbackLeiblerDivergenceTest(test.TestCase):
 
   def test_zero_weighted(self):
     self.setup()
-    k_obj = keras.losses.KullbackLeiblerDivergence()
+    k_obj = keras.losses.KLDivergence()
     loss = k_obj(self.y_true, self.y_pred, sample_weight=0)
     self.assertAlmostEqual(self.evaluate(loss), 0., 3)
 
@@ -1447,27 +1451,27 @@ class HuberLossTest(test.TestCase):
     self.y_true = constant_op.constant(self.np_y_true)
 
   def test_config(self):
-    h_obj = keras.losses.HuberLoss(
+    h_obj = keras.losses.Huber(
         reduction=losses_impl.ReductionV2.SUM, name='huber')
     self.assertEqual(h_obj.name, 'huber')
     self.assertEqual(h_obj.reduction, losses_impl.ReductionV2.SUM)
 
   def test_all_correct(self):
     self.setup()
-    h_obj = keras.losses.HuberLoss()
+    h_obj = keras.losses.Huber()
     loss = h_obj(self.y_true, self.y_true)
     self.assertAlmostEqual(self.evaluate(loss), 0.0, 3)
 
   def test_unweighted(self):
     self.setup()
-    h_obj = keras.losses.HuberLoss()
+    h_obj = keras.losses.Huber()
     loss = h_obj(self.y_true, self.y_pred)
     actual_loss = np.sum(self.expected_losses) / self.batch_size
     self.assertAlmostEqual(self.evaluate(loss), actual_loss, 3)
 
   def test_scalar_weighted(self):
     self.setup()
-    h_obj = keras.losses.HuberLoss()
+    h_obj = keras.losses.Huber()
     sample_weight = 2.3
     loss = h_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
     actual_loss = sample_weight * np.sum(self.expected_losses) / self.batch_size
@@ -1479,7 +1483,7 @@ class HuberLossTest(test.TestCase):
 
   def test_sample_weighted(self):
     self.setup()
-    h_obj = keras.losses.HuberLoss()
+    h_obj = keras.losses.Huber()
     sample_weight = constant_op.constant((1.2, 3.4), shape=(2, 1))
 
     loss = h_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
@@ -1491,7 +1495,7 @@ class HuberLossTest(test.TestCase):
 
   def test_timestep_weighted(self):
     self.setup()
-    h_obj = keras.losses.HuberLoss()
+    h_obj = keras.losses.Huber()
     y_pred = self.np_y_pred.reshape((2, 3, 1))
     y_true = self.np_y_true.reshape((2, 3, 1))
     expected_losses = self.huber_loss(y_true, y_pred)
@@ -1509,14 +1513,14 @@ class HuberLossTest(test.TestCase):
 
   def test_zero_weighted(self):
     self.setup()
-    h_obj = keras.losses.HuberLoss()
+    h_obj = keras.losses.Huber()
     sample_weight = 0
     loss = h_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
     self.assertAlmostEqual(self.evaluate(loss), 0., 3)
 
   def test_non_default_delta(self):
     self.setup(delta=0.8)
-    h_obj = keras.losses.HuberLoss(delta=0.8)
+    h_obj = keras.losses.Huber(delta=0.8)
     sample_weight = 2.3
     loss = h_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
     actual_loss = sample_weight * np.sum(self.expected_losses) / self.batch_size
diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index d15fec49b6..e821e7d062 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -65,6 +65,7 @@ from tensorflow.python.util.tf_export import keras_export
 from tensorflow.tools.docs import doc_controls
 
 
+@keras_export('keras.metrics.Metric')
 @six.add_metaclass(abc.ABCMeta)
 class Metric(Layer):
   """Encapsulates metric logic and state.
@@ -221,7 +222,6 @@ class Metric(Layer):
          All update ops added to the graph by this function will be executed.
       As a result, code should generally work the same way with graph or
       eager execution.
-    and adds the update op to the metric layer.
 
     Args:
       *args:
@@ -435,6 +435,7 @@ class Mean(Reduce):
         reduction=metrics_utils.Reduction.WEIGHTED_MEAN, name=name, dtype=dtype)
 
 
+@keras_export('keras.metrics.MeanRelativeError')
 class MeanRelativeError(Mean):
   """Computes the mean relative error by normalizing with the given values.
 
@@ -736,6 +737,7 @@ class SparseCategoricalAccuracy(MeanMetricWrapper):
         sparse_categorical_accuracy, name, dtype=dtype)
 
 
+@keras_export('keras.metrics.TopKCategoricalAccuracy')
 class TopKCategoricalAccuracy(MeanMetricWrapper):
   """Computes how often targets are in the top `K` predictions.
 
@@ -768,6 +770,7 @@ class TopKCategoricalAccuracy(MeanMetricWrapper):
         top_k_categorical_accuracy, name, dtype=dtype, k=k)
 
 
+@keras_export('keras.metrics.SparseTopKCategoricalAccuracy')
 class SparseTopKCategoricalAccuracy(MeanMetricWrapper):
   """Computes how often integer targets are in the top `K` predictions.
 
@@ -1554,6 +1557,7 @@ class SpecificityAtSensitivity(SensitivitySpecificityBase):
     return dict(list(base_config.items()) + list(config.items()))
 
 
+@keras_export('keras.metrics.AUC')
 class AUC(Metric):
   """Computes the approximate AUC (Area under the curve) via a Riemann sum.
 
@@ -2039,6 +2043,7 @@ class CategoricalHinge(MeanMetricWrapper):
     super(CategoricalHinge, self).__init__(categorical_hinge, name, dtype=dtype)
 
 
+@keras_export('keras.metrics.RootMeanSquaredError')
 class RootMeanSquaredError(Mean):
   """Computes root mean squared error metric between `y_true` and `y_pred`.
 
@@ -2086,15 +2091,16 @@ class RootMeanSquaredError(Mean):
     return math_ops.sqrt(math_ops.div_no_nan(self.total, self.count))
 
 
-class Logcosh(MeanMetricWrapper):
+@keras_export('keras.metrics.LogCoshError')
+class LogCoshError(MeanMetricWrapper):
   """Computes the logarithm of the hyperbolic cosine of the prediction error.
 
-  logcosh = log((exp(x) + exp(-x))/2) where x is the error `y_pred` - `y_true`.
+  `logcosh = log((exp(x) + exp(-x))/2)`, where x is the error (y_pred - y_true)
 
   Usage:
 
   ```python
-  m = tf.keras.metrics.Logcosh()
+  m = tf.keras.metrics.LogCoshError()
   m.update_state([0., 1., 1.], [1., 0., 1.])
   print('Final result: ', m.result().numpy())  # Final result: 0.289
   ```
@@ -2103,18 +2109,19 @@ class Logcosh(MeanMetricWrapper):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', metrics=[tf.keras.metrics.Logcosh()])
+  model.compile('sgd', metrics=[tf.keras.metrics.LogCoshError()])
   ```
   """
 
   def __init__(self, name='logcosh', dtype=None):
-    super(Logcosh, self).__init__(logcosh, name, dtype=dtype)
+    super(LogCoshError, self).__init__(logcosh, name, dtype=dtype)
 
 
+@keras_export('keras.metrics.Poisson')
 class Poisson(MeanMetricWrapper):
-  """Computes the poisson metric between `y_true` and `y_pred`.
+  """Computes the Poisson metric between `y_true` and `y_pred`.
 
-  metric = y_pred - y_true * log(y_pred)
+  `metric = y_pred - y_true * log(y_pred)`
 
   Usage:
 
@@ -2136,15 +2143,16 @@ class Poisson(MeanMetricWrapper):
     super(Poisson, self).__init__(poisson, name, dtype=dtype)
 
 
-class KullbackLeiblerDivergence(MeanMetricWrapper):
-  """Computes kullback leibler divergence metric between `y_true` and `y_pred`.
+@keras_export('keras.metrics.KLDivergence')
+class KLDivergence(MeanMetricWrapper):
+  """Computes Kullback Leibler divergence metric between `y_true` and `y_pred`.
 
-  metric = y_true * log(y_true / y_pred)
+  `metric = y_true * log(y_true / y_pred)`
 
   Usage:
 
   ```python
-  m = tf.keras.metrics.KullbackLeiblerDivergence()
+  m = tf.keras.metrics.KLDivergence()
   m.update_state([.4, .9, .2], [.5, .8, .12])
   print('Final result: ', m.result().numpy())  # Final result: -0.043
   ```
@@ -2153,15 +2161,16 @@ class KullbackLeiblerDivergence(MeanMetricWrapper):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', metrics=[tf.keras.metrics.KullbackLeiblerDivergence()])
+  model.compile('sgd', metrics=[tf.keras.metrics.KLDivergence()])
   ```
   """
 
   def __init__(self, name='kullback_leibler_divergence', dtype=None):
-    super(KullbackLeiblerDivergence, self).__init__(
+    super(KLDivergence, self).__init__(
         kullback_leibler_divergence, name, dtype=dtype)
 
 
+@keras_export('keras.metrics.MeanIoU')
 class MeanIoU(Metric):
   """Computes the mean Intersection-Over-Union metric.
 
@@ -2286,6 +2295,7 @@ class MeanIoU(Metric):
     return dict(list(base_config.items()) + list(config.items()))
 
 
+@keras_export('keras.metrics.MeanTensor')
 class MeanTensor(Metric):
   """Computes the element-wise (weighted) mean of the given tensors.
 
@@ -2396,6 +2406,7 @@ class MeanTensor(Metric):
         K.set_value(v, np.zeros(self._shape.as_list()))
 
 
+@keras_export('keras.metrics.BinaryCrossentropy')
 class BinaryCrossentropy(MeanMetricWrapper):
   """Computes the crossentropy metric between the labels and predictions.
 
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index 6ed9593b84..35732ad307 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -895,7 +895,7 @@ class SparseTopKCategoricalAccuracyTest(test.TestCase):
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class LogcoshTest(test.TestCase):
+class LogCoshErrorTest(test.TestCase):
 
   def setup(self):
     y_pred = np.asarray([1, 9, 2, -5, -2, 6]).reshape((2, 3))
@@ -909,13 +909,13 @@ class LogcoshTest(test.TestCase):
     self.y_true = constant_op.constant(y_true)
 
   def test_config(self):
-    logcosh_obj = metrics.Logcosh(name='logcosh', dtype=dtypes.int32)
+    logcosh_obj = metrics.LogCoshError(name='logcosh', dtype=dtypes.int32)
     self.assertEqual(logcosh_obj.name, 'logcosh')
     self.assertEqual(logcosh_obj._dtype, dtypes.int32)
 
   def test_unweighted(self):
     self.setup()
-    logcosh_obj = metrics.Logcosh()
+    logcosh_obj = metrics.LogCoshError()
     self.evaluate(variables.variables_initializer(logcosh_obj.variables))
 
     update_op = logcosh_obj.update_state(self.y_true, self.y_pred)
@@ -926,7 +926,7 @@ class LogcoshTest(test.TestCase):
 
   def test_weighted(self):
     self.setup()
-    logcosh_obj = metrics.Logcosh()
+    logcosh_obj = metrics.LogCoshError()
     self.evaluate(variables.variables_initializer(logcosh_obj.variables))
     sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
     result = logcosh_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
@@ -984,7 +984,7 @@ class PoissonTest(test.TestCase):
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class KullbackLeiblerDivergenceTest(test.TestCase):
+class KLDivergenceTest(test.TestCase):
 
   def setup(self):
     y_pred = np.asarray([.4, .9, .12, .36, .3, .4]).reshape((2, 3))
@@ -997,17 +997,17 @@ class KullbackLeiblerDivergenceTest(test.TestCase):
     self.y_true = constant_op.constant(y_true)
 
   def test_config(self):
-    k_obj = metrics.KullbackLeiblerDivergence(name='kld', dtype=dtypes.int32)
+    k_obj = metrics.KLDivergence(name='kld', dtype=dtypes.int32)
     self.assertEqual(k_obj.name, 'kld')
     self.assertEqual(k_obj._dtype, dtypes.int32)
 
-    k_obj2 = metrics.KullbackLeiblerDivergence.from_config(k_obj.get_config())
+    k_obj2 = metrics.KLDivergence.from_config(k_obj.get_config())
     self.assertEqual(k_obj2.name, 'kld')
     self.assertEqual(k_obj2._dtype, dtypes.int32)
 
   def test_unweighted(self):
     self.setup()
-    k_obj = metrics.KullbackLeiblerDivergence()
+    k_obj = metrics.KLDivergence()
     self.evaluate(variables.variables_initializer(k_obj.variables))
 
     update_op = k_obj.update_state(self.y_true, self.y_pred)
@@ -1018,7 +1018,7 @@ class KullbackLeiblerDivergenceTest(test.TestCase):
 
   def test_weighted(self):
     self.setup()
-    k_obj = metrics.KullbackLeiblerDivergence()
+    k_obj = metrics.KLDivergence()
     self.evaluate(variables.variables_initializer(k_obj.variables))
 
     sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-huber.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-huber.pbtxt
new file mode 100644
index 0000000000..2204f0b0f0
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-huber.pbtxt
@@ -0,0 +1,22 @@
+path: "tensorflow.keras.losses.Huber"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.Huber\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'delta\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'1.0\', \'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-k-l-divergence.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-k-l-divergence.pbtxt
new file mode 100644
index 0000000000..60706b4035
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-k-l-divergence.pbtxt
@@ -0,0 +1,22 @@
+path: "tensorflow.keras.losses.KLDivergence"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.KLDivergence\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-log-cosh.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-log-cosh.pbtxt
new file mode 100644
index 0000000000..cf1a0f1e80
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-log-cosh.pbtxt
@@ -0,0 +1,22 @@
+path: "tensorflow.keras.losses.LogCosh"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.LogCosh\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-log-loss.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-log-loss.pbtxt
new file mode 100644
index 0000000000..466e0b4006
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-log-loss.pbtxt
@@ -0,0 +1,22 @@
+path: "tensorflow.keras.losses.LogLoss"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.LogLoss\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-loss.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-loss.pbtxt
new file mode 100644
index 0000000000..04a9cc9420
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-loss.pbtxt
@@ -0,0 +1,21 @@
+path: "tensorflow.keras.losses.Loss"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-poisson.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-poisson.pbtxt
new file mode 100644
index 0000000000..674258ed8b
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-poisson.pbtxt
@@ -0,0 +1,22 @@
+path: "tensorflow.keras.losses.Poisson"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.Poisson\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-sparse-categorical-crossentropy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-sparse-categorical-crossentropy.pbtxt
new file mode 100644
index 0000000000..15a59d16c2
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-sparse-categorical-crossentropy.pbtxt
@@ -0,0 +1,22 @@
+path: "tensorflow.keras.losses.SparseCategoricalCrossentropy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.SparseCategoricalCrossentropy\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'from_logits\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
index ce942de6d2..7381fe3918 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
@@ -20,6 +20,26 @@ tf_module {
     name: "Hinge"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "Huber"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "KLDivergence"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "LogCosh"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "LogLoss"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Loss"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "MeanAbsoluteError"
     mtype: "<type \'type\'>"
@@ -36,6 +56,14 @@ tf_module {
     name: "MeanSquaredLogarithmicError"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "Poisson"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "SparseCategoricalCrossentropy"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "SquaredHinge"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-a-u-c.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-a-u-c.pbtxt
new file mode 100644
index 0000000000..8019d43c78
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-a-u-c.pbtxt
@@ -0,0 +1,200 @@
+path: "tensorflow.keras.metrics.AUC"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.AUC\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'num_thresholds\', \'curve\', \'summation_method\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'200\', \'AUCCurve.ROC\', \'AUCSummationMethod.INTERPOLATION\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "interpolate_pr_auc"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-crossentropy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-crossentropy.pbtxt
new file mode 100644
index 0000000000..b6ce9e0420
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-crossentropy.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.keras.metrics.BinaryCrossentropy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.BinaryCrossentropy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\', \'from_logits\', \'label_smoothing\'], varargs=None, keywords=None, defaults=[\'binary_crossentropy\', \'None\', \'False\', \'0\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-k-l-divergence.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-k-l-divergence.pbtxt
new file mode 100644
index 0000000000..25bcf8591d
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-k-l-divergence.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.keras.metrics.KLDivergence"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.KLDivergence\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'kullback_leibler_divergence\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-log-cosh-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-log-cosh-error.pbtxt
new file mode 100644
index 0000000000..fb2df7f459
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-log-cosh-error.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.keras.metrics.LogCoshError"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.LogCoshError\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'logcosh\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-io-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-io-u.pbtxt
new file mode 100644
index 0000000000..47b550a9e7
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-io-u.pbtxt
@@ -0,0 +1,196 @@
+path: "tensorflow.keras.metrics.MeanIoU"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanIoU\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'num_classes\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-relative-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-relative-error.pbtxt
new file mode 100644
index 0000000000..a1ec7cb51e
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-relative-error.pbtxt
@@ -0,0 +1,198 @@
+path: "tensorflow.keras.metrics.MeanRelativeError"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanRelativeError\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'normalizer\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-tensor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-tensor.pbtxt
new file mode 100644
index 0000000000..2a5218fed4
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-tensor.pbtxt
@@ -0,0 +1,204 @@
+path: "tensorflow.keras.metrics.MeanTensor"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanTensor\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "count"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "total"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'mean_tensor\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'values\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-metric.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-metric.pbtxt
new file mode 100644
index 0000000000..2216043cec
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-metric.pbtxt
@@ -0,0 +1,195 @@
+path: "tensorflow.keras.metrics.Metric"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-poisson.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-poisson.pbtxt
new file mode 100644
index 0000000000..8e91098f95
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-poisson.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.keras.metrics.Poisson"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Poisson\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'poisson\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-root-mean-squared-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-root-mean-squared-error.pbtxt
new file mode 100644
index 0000000000..4dd60bb772
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-root-mean-squared-error.pbtxt
@@ -0,0 +1,198 @@
+path: "tensorflow.keras.metrics.RootMeanSquaredError"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.RootMeanSquaredError\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'root_mean_squared_error\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-top-k-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-top-k-categorical-accuracy.pbtxt
new file mode 100644
index 0000000000..ea0f2a760a
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-top-k-categorical-accuracy.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.keras.metrics.SparseTopKCategoricalAccuracy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.SparseTopKCategoricalAccuracy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'k\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'5\', \'sparse_top_k_categorical_accuracy\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-top-k-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-top-k-categorical-accuracy.pbtxt
new file mode 100644
index 0000000000..9cf1bf21cb
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-top-k-categorical-accuracy.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.keras.metrics.TopKCategoricalAccuracy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.TopKCategoricalAccuracy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'k\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'5\', \'top_k_categorical_accuracy\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
index 0e32de336e..7ebf43b362 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
@@ -1,5 +1,9 @@
 path: "tensorflow.keras.metrics"
 tf_module {
+  member {
+    name: "AUC"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "Accuracy"
     mtype: "<type \'type\'>"
@@ -8,6 +12,10 @@ tf_module {
     name: "BinaryAccuracy"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "BinaryCrossentropy"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "CategoricalAccuracy"
     mtype: "<type \'type\'>"
@@ -36,6 +44,14 @@ tf_module {
     name: "Hinge"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "KLDivergence"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "LogCoshError"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "Mean"
     mtype: "<type \'type\'>"
@@ -48,6 +64,14 @@ tf_module {
     name: "MeanAbsolutePercentageError"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "MeanIoU"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "MeanRelativeError"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "MeanSquaredError"
     mtype: "<type \'type\'>"
@@ -56,6 +80,18 @@ tf_module {
     name: "MeanSquaredLogarithmicError"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "MeanTensor"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Metric"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Poisson"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "Precision"
     mtype: "<type \'type\'>"
@@ -64,6 +100,10 @@ tf_module {
     name: "Recall"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "RootMeanSquaredError"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "SensitivityAtSpecificity"
     mtype: "<type \'type\'>"
@@ -72,6 +112,10 @@ tf_module {
     name: "SparseCategoricalAccuracy"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "SparseTopKCategoricalAccuracy"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "SpecificityAtSensitivity"
     mtype: "<type \'type\'>"
@@ -84,6 +128,10 @@ tf_module {
     name: "Sum"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "TopKCategoricalAccuracy"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "TrueNegatives"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-huber.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-huber.pbtxt
new file mode 100644
index 0000000000..2204f0b0f0
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-huber.pbtxt
@@ -0,0 +1,22 @@
+path: "tensorflow.keras.losses.Huber"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.Huber\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'delta\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'1.0\', \'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-k-l-divergence.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-k-l-divergence.pbtxt
new file mode 100644
index 0000000000..60706b4035
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-k-l-divergence.pbtxt
@@ -0,0 +1,22 @@
+path: "tensorflow.keras.losses.KLDivergence"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.KLDivergence\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-log-cosh.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-log-cosh.pbtxt
new file mode 100644
index 0000000000..cf1a0f1e80
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-log-cosh.pbtxt
@@ -0,0 +1,22 @@
+path: "tensorflow.keras.losses.LogCosh"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.LogCosh\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-log-loss.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-log-loss.pbtxt
new file mode 100644
index 0000000000..466e0b4006
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-log-loss.pbtxt
@@ -0,0 +1,22 @@
+path: "tensorflow.keras.losses.LogLoss"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.LogLoss\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-loss.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-loss.pbtxt
new file mode 100644
index 0000000000..04a9cc9420
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-loss.pbtxt
@@ -0,0 +1,21 @@
+path: "tensorflow.keras.losses.Loss"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-poisson.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-poisson.pbtxt
new file mode 100644
index 0000000000..674258ed8b
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-poisson.pbtxt
@@ -0,0 +1,22 @@
+path: "tensorflow.keras.losses.Poisson"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.Poisson\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-sparse-categorical-crossentropy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-sparse-categorical-crossentropy.pbtxt
new file mode 100644
index 0000000000..15a59d16c2
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-sparse-categorical-crossentropy.pbtxt
@@ -0,0 +1,22 @@
+path: "tensorflow.keras.losses.SparseCategoricalCrossentropy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.SparseCategoricalCrossentropy\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'from_logits\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
index e8809450fa..090ab7ee3f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
@@ -20,6 +20,26 @@ tf_module {
     name: "Hinge"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "Huber"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "KLDivergence"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "LogCosh"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "LogLoss"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Loss"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "MeanAbsoluteError"
     mtype: "<type \'type\'>"
@@ -36,10 +56,18 @@ tf_module {
     name: "MeanSquaredLogarithmicError"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "Poisson"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "Reduction"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "SparseCategoricalCrossentropy"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "SquaredHinge"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-a-u-c.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-a-u-c.pbtxt
new file mode 100644
index 0000000000..8019d43c78
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-a-u-c.pbtxt
@@ -0,0 +1,200 @@
+path: "tensorflow.keras.metrics.AUC"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.AUC\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'num_thresholds\', \'curve\', \'summation_method\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'200\', \'AUCCurve.ROC\', \'AUCSummationMethod.INTERPOLATION\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "interpolate_pr_auc"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-crossentropy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-crossentropy.pbtxt
new file mode 100644
index 0000000000..b6ce9e0420
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-crossentropy.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.keras.metrics.BinaryCrossentropy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.BinaryCrossentropy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\', \'from_logits\', \'label_smoothing\'], varargs=None, keywords=None, defaults=[\'binary_crossentropy\', \'None\', \'False\', \'0\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-k-l-divergence.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-k-l-divergence.pbtxt
new file mode 100644
index 0000000000..25bcf8591d
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-k-l-divergence.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.keras.metrics.KLDivergence"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.KLDivergence\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'kullback_leibler_divergence\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-log-cosh-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-log-cosh-error.pbtxt
new file mode 100644
index 0000000000..fb2df7f459
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-log-cosh-error.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.keras.metrics.LogCoshError"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.LogCoshError\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'logcosh\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-io-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-io-u.pbtxt
new file mode 100644
index 0000000000..47b550a9e7
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-io-u.pbtxt
@@ -0,0 +1,196 @@
+path: "tensorflow.keras.metrics.MeanIoU"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanIoU\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'num_classes\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-relative-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-relative-error.pbtxt
new file mode 100644
index 0000000000..a1ec7cb51e
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-relative-error.pbtxt
@@ -0,0 +1,198 @@
+path: "tensorflow.keras.metrics.MeanRelativeError"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanRelativeError\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'normalizer\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-tensor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-tensor.pbtxt
new file mode 100644
index 0000000000..2a5218fed4
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-tensor.pbtxt
@@ -0,0 +1,204 @@
+path: "tensorflow.keras.metrics.MeanTensor"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanTensor\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "count"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "total"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'mean_tensor\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'values\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-metric.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-metric.pbtxt
new file mode 100644
index 0000000000..2216043cec
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-metric.pbtxt
@@ -0,0 +1,195 @@
+path: "tensorflow.keras.metrics.Metric"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-poisson.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-poisson.pbtxt
new file mode 100644
index 0000000000..8e91098f95
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-poisson.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.keras.metrics.Poisson"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Poisson\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'poisson\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-root-mean-squared-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-root-mean-squared-error.pbtxt
new file mode 100644
index 0000000000..4dd60bb772
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-root-mean-squared-error.pbtxt
@@ -0,0 +1,198 @@
+path: "tensorflow.keras.metrics.RootMeanSquaredError"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.RootMeanSquaredError\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'root_mean_squared_error\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-top-k-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-top-k-categorical-accuracy.pbtxt
new file mode 100644
index 0000000000..ea0f2a760a
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-top-k-categorical-accuracy.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.keras.metrics.SparseTopKCategoricalAccuracy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.SparseTopKCategoricalAccuracy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'k\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'5\', \'sparse_top_k_categorical_accuracy\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-top-k-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-top-k-categorical-accuracy.pbtxt
new file mode 100644
index 0000000000..9cf1bf21cb
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-top-k-categorical-accuracy.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.keras.metrics.TopKCategoricalAccuracy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.TopKCategoricalAccuracy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'k\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'5\', \'top_k_categorical_accuracy\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
index 0e32de336e..7ebf43b362 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
@@ -1,5 +1,9 @@
 path: "tensorflow.keras.metrics"
 tf_module {
+  member {
+    name: "AUC"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "Accuracy"
     mtype: "<type \'type\'>"
@@ -8,6 +12,10 @@ tf_module {
     name: "BinaryAccuracy"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "BinaryCrossentropy"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "CategoricalAccuracy"
     mtype: "<type \'type\'>"
@@ -36,6 +44,14 @@ tf_module {
     name: "Hinge"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "KLDivergence"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "LogCoshError"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "Mean"
     mtype: "<type \'type\'>"
@@ -48,6 +64,14 @@ tf_module {
     name: "MeanAbsolutePercentageError"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "MeanIoU"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "MeanRelativeError"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "MeanSquaredError"
     mtype: "<type \'type\'>"
@@ -56,6 +80,18 @@ tf_module {
     name: "MeanSquaredLogarithmicError"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "MeanTensor"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Metric"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Poisson"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "Precision"
     mtype: "<type \'type\'>"
@@ -64,6 +100,10 @@ tf_module {
     name: "Recall"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "RootMeanSquaredError"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "SensitivityAtSpecificity"
     mtype: "<type \'type\'>"
@@ -72,6 +112,10 @@ tf_module {
     name: "SparseCategoricalAccuracy"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "SparseTopKCategoricalAccuracy"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "SpecificityAtSensitivity"
     mtype: "<type \'type\'>"
@@ -84,6 +128,10 @@ tf_module {
     name: "Sum"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "TopKCategoricalAccuracy"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "TrueNegatives"
     mtype: "<type \'type\'>"
-- 
GitLab


From c971c39a05ef6a3951d9b786898d27be8559b5c7 Mon Sep 17 00:00:00 2001
From: "srinivasan.narayanamoorthy" <srinivasan.narayanamoorthy@intel.com>
Date: Fri, 25 Jan 2019 14:45:22 -0800
Subject: [PATCH 1300/2345] review changes

---
 tensorflow/core/common_runtime/mkl_cpu_allocator.h | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/common_runtime/mkl_cpu_allocator.h b/tensorflow/core/common_runtime/mkl_cpu_allocator.h
index b2db8b66a4..94f1b1f204 100644
--- a/tensorflow/core/common_runtime/mkl_cpu_allocator.h
+++ b/tensorflow/core/common_runtime/mkl_cpu_allocator.h
@@ -94,8 +94,7 @@ class MklSmallSizeAllocator : public Allocator {
 
  private:
   // Increment statistics for the allocator handling small allocations.
-  inline void IncrementStats(size_t alloc_size)
-      EXCLUSIVE_LOCKS_REQUIRED(mutex_) {
+  inline void IncrementStats(size_t alloc_size) LOCKS_EXCLUDED(mutex_) {
     mutex_lock l(mutex_);
     ++stats_.num_allocs;
     stats_.bytes_in_use += alloc_size;
@@ -106,8 +105,7 @@ class MklSmallSizeAllocator : public Allocator {
   }
 
   // Decrement statistics for the allocator handling small allocations.
-  inline void DecrementStats(size_t dealloc_size)
-      EXCLUSIVE_LOCKS_REQUIRED(mutex_) {
+  inline void DecrementStats(size_t dealloc_size) LOCKS_EXCLUDED(mutex_) {
     mutex_lock l(mutex_);
     stats_.bytes_in_use -= dealloc_size;
   }
-- 
GitLab


From f88007bc51479edfadaaa7bb6a216a351809d1d9 Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Fri, 25 Jan 2019 14:07:17 -0800
Subject: [PATCH 1301/2345] Test metrics in standalone client mode. Use barrier
 to synchronize workers.

PiperOrigin-RevId: 230965622
---
 .../python/distribute/distribute_coordinator.py     |  4 ++--
 .../keras/engine/distributed_training_utils.py      | 13 ++++++-------
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/tensorflow/python/distribute/distribute_coordinator.py b/tensorflow/python/distribute/distribute_coordinator.py
index 89bd8537b4..70c6409a22 100644
--- a/tensorflow/python/distribute/distribute_coordinator.py
+++ b/tensorflow/python/distribute/distribute_coordinator.py
@@ -210,8 +210,8 @@ class _WorkerContext(object):
       ValueError: if `worker_barrier` is not passed to the __init__ method.
     """
     if not self._worker_barrier:
-      raise ValueError("`worker_barrier is not set in the worker context.` \t" +
-                       self._debug_message())
+      # TODO(yuefengz): we should throw an error in independent worker mode.
+      return
     self._worker_barrier.wait()
 
   def session_creator(self,
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index 512a629716..7d00c162fa 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -344,7 +344,7 @@ def init_restore_or_wait_for_variables():
   session = K._get_session()  # pylint: disable=protected-access
   worker_context = dc_context.get_current_worker_context()
   if not worker_context or worker_context.experimental_should_init:
-    # TODO(yuefengz): if checkpoints exit, restore from checkpoint.
+    # TODO(yuefengz): if checkpoints exist, restore from checkpoint.
     K._initialize_variables(session)  # pylint: disable=protected-access
   else:
     _wait_for_variable_initialization(session)
@@ -771,12 +771,11 @@ def _make_execution_function(model, mode):
      grouped_session_args) = strategy.extended.call_for_each_replica(
          _per_device_function, args=(model._distributed_model,))
 
-    if mode == ModeKeys.TRAIN:
-      # Initialize the variables in the replicated model. This is necessary for
-      # multi-worker training because on some workers, initialization is not
-      # needed. This method does initialization or waiting for initialization
-      # according to the context object of distribute coordinator.
-      init_restore_or_wait_for_variables()
+    # Initialize the variables in the replicated model. This is necessary for
+    # multi-worker training because on some workers, initialization is not
+    # needed. This method does initialization or waiting for initialization
+    # according to the context object of distribute coordinator.
+    init_restore_or_wait_for_variables()
 
     # Unwrap all the per device values returned from `call_for_each_replica`.
     # Unwrapping per device values gives you a list of values that can be
-- 
GitLab


From 81cf135cd771d774680633ea42bbf3bf8cf1e603 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 25 Jan 2019 14:20:28 -0800
Subject: [PATCH 1302/2345] The "install_nnapi_extra_headers" target is used
 for exporting additional headers required by some nnapi implementations.

PiperOrigin-RevId: 230967990
---
 tensorflow/lite/kernels/internal/BUILD | 76 +++++++++++++++++++++++++-
 1 file changed, 75 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/kernels/internal/BUILD b/tensorflow/lite/kernels/internal/BUILD
index 6d3feed268..3b64bcce83 100644
--- a/tensorflow/lite/kernels/internal/BUILD
+++ b/tensorflow/lite/kernels/internal/BUILD
@@ -1,4 +1,4 @@
-load("//tensorflow:tensorflow.bzl", "tf_cc_test")
+load("//tensorflow:tensorflow.bzl", "tf_cc_test", "transitive_hdrs")
 load("//tensorflow/lite:build_def.bzl", "tflite_copts")
 load("//tensorflow/lite:special_rules.bzl", "tflite_portable_test_suite")
 
@@ -698,4 +698,78 @@ cc_test(
 
 exports_files(["optimized/eigen_tensor_reduced_instantiations_oss.h"])
 
+filegroup(
+    name = "optimized_op_headers",
+    srcs = glob([
+        "optimized/*.h",
+    ]),
+    visibility = ["//tensorflow/lite:__subpackages__"],
+)
+
+filegroup(
+    name = "reference_op_headers",
+    srcs = glob([
+        "reference/*.h",
+    ]),
+    visibility = ["//tensorflow/lite:__subpackages__"],
+)
+
+filegroup(
+    name = "headers",
+    srcs = glob([
+        "*.h",
+    ]),
+    visibility = ["//tensorflow/lite:__subpackages__"],
+)
+
+transitive_hdrs(
+    name = "nnapi_external_headers",
+    visibility = ["//tensorflow/lite:__subpackages__"],
+    deps = [
+        "//third_party/eigen3",
+        "@gemmlowp",
+    ],
+)
+
+# ---------------------------------------------------------
+# The public target "install_nnapi_extra_headers" is only
+# used for external targets that requires exporting optmized
+# and reference op headers.
+
+genrule(
+    name = "install_nnapi_extra_headers",
+    srcs = [
+        ":nnapi_external_headers",
+        ":headers",
+        ":optimized_op_headers",
+        ":reference_op_headers",
+    ],
+    outs = ["include"],
+    cmd = """
+    mkdir $@
+    for f in $(SRCS); do
+      d="$${f%/*}"
+      d="$${d#bazel-out*genfiles/}"
+      d="$${d#*external/eigen_archive/}"
+
+      if [[ $${d} == *local_config_* ]]; then
+        continue
+      fi
+
+      if [[ $${d} == external* ]]; then
+        extname="$${d#*external/}"
+        extname="$${extname%%/*}"
+        if [[ $${TF_SYSTEM_LIBS:-} == *$${extname}* ]]; then
+          continue
+        fi
+      fi
+
+      mkdir -p "$@/$${d}"
+      cp "$${f}" "$@/$${d}/"
+    done
+    """,
+    tags = ["manual"],
+    visibility = ["//visibility:private"],
+)
+
 tflite_portable_test_suite()
-- 
GitLab


From 82812551bd02474c8de10ac92fd34abb71afd257 Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Fri, 25 Jan 2019 14:20:30 -0800
Subject: [PATCH 1303/2345] Disable flaky test method in stage_op_test_gpu.

PiperOrigin-RevId: 230967998
---
 tensorflow/python/kernel_tests/stage_op_test.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/kernel_tests/stage_op_test.py b/tensorflow/python/kernel_tests/stage_op_test.py
index 83e06ba48b..29cd00b789 100644
--- a/tensorflow/python/kernel_tests/stage_op_test.py
+++ b/tensorflow/python/kernel_tests/stage_op_test.py
@@ -166,6 +166,7 @@ class StageTest(test.TestCase):
 
   @test_util.run_deprecated_v1
   def testCapacity(self):
+    self.skipTest('b/123423516 this test is flaky on gpu.')
     capacity = 3
 
     with ops.Graph().as_default() as G:
-- 
GitLab


From a7168daf4c53a6d74cc1f1d42fc46d8a3e4e0803 Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Fri, 25 Jan 2019 14:20:37 -0800
Subject: [PATCH 1304/2345] Mark eager_test.py as medium instead of large

PiperOrigin-RevId: 230968023
---
 tensorflow/compiler/tests/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD
index f80cb1812f..2b9f5d8dbd 100644
--- a/tensorflow/compiler/tests/BUILD
+++ b/tensorflow/compiler/tests/BUILD
@@ -407,7 +407,7 @@ tf_xla_py_test(
 
 tf_xla_py_test(
     name = "eager_test",
-    size = "large",
+    size = "medium",
     srcs = ["eager_test.py"],
     deps = [
         ":xla_test",
-- 
GitLab


From 5e3991935780405bee7642927337c4247c0851ee Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 25 Jan 2019 14:49:54 -0800
Subject: [PATCH 1305/2345] Do not ignore result from ftruncate.

PiperOrigin-RevId: 230973294
---
 tensorflow/lite/nnapi/nnapi_implementation.cc | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/nnapi/nnapi_implementation.cc b/tensorflow/lite/nnapi/nnapi_implementation.cc
index 745d089ba7..e8b9aed422 100644
--- a/tensorflow/lite/nnapi/nnapi_implementation.cc
+++ b/tensorflow/lite/nnapi/nnapi_implementation.cc
@@ -66,8 +66,13 @@ void* LoadFunction(void* handle, const char* name) {
 // Add /dev/shm implementation of shared memory for non-Android platforms
 int ASharedMemory_create(const char* name, size_t size) {
   int fd = shm_open(name, O_RDWR | O_CREAT, 0644);
-  if (fd >= 0) {
-    ftruncate(fd, size);
+  if (fd < 0) {
+    return fd;
+  }
+  int result = ftruncate(fd, size);
+  if (result < 0) {
+    close(fd);
+    return -1;
   }
   return fd;
 }
-- 
GitLab


From 75408394b51d03d5a87d1530055d5e932bcf0dcc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 25 Jan 2019 15:18:27 -0800
Subject: [PATCH 1306/2345] Allows the Keras learning phase to work correctly
 when layers are called inside of functions (e.g. in custom function-wrapped
 training loops).

PiperOrigin-RevId: 230978736
---
 tensorflow/python/keras/backend.py | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index a3369ef43a..62447ec61f 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -251,12 +251,24 @@ def learning_phase():
   Returns:
       Learning phase (scalar integer tensor or Python integer).
   """
-  if context.executing_eagerly():
-    if _DUMMY_EAGER_GRAPH not in _GRAPH_LEARNING_PHASES:
-      # Fallback to inference mode as default.
-      return 0
-    return _GRAPH_LEARNING_PHASES[_DUMMY_EAGER_GRAPH]
-  return symbolic_learning_phase()
+  if ops.get_default_graph() is _GRAPH:
+    # Don't enter an init_scope for the learning phase if eager execution
+    # is enabled but we're inside the Keras workspace graph.
+    return symbolic_learning_phase()
+  with ops.init_scope():
+    # We always check & set the learning phase inside the init_scope,
+    # otherwise the wrong default_graph will be used to look up the learning
+    # phase inside of functions & defuns.
+    #
+    # This is because functions & defuns (both in graph & in eager mode)
+    # will always execute non-eagerly using a function-specific default
+    # subgraph.
+    if context.executing_eagerly():
+      if _DUMMY_EAGER_GRAPH not in _GRAPH_LEARNING_PHASES:
+        # Fallback to inference mode as default.
+        return 0
+      return _GRAPH_LEARNING_PHASES[_DUMMY_EAGER_GRAPH]
+    return symbolic_learning_phase()
 
 
 def symbolic_learning_phase():
-- 
GitLab


From a180af5009355fdc0d9c44c0fe0f5cfabe8131d2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 25 Jan 2019 15:26:14 -0800
Subject: [PATCH 1307/2345] [Grappler] Minor style cleanups from CL/230784105

DELTA=14 (1 added, 1 deleted, 12 changed)
DELTA_BY_EXTENSION=cc=13
PiperOrigin-RevId: 230980146
---
 tensorflow/core/graph/validate_test.cc        |  4 ++--
 .../verifiers/structure_verifier_test.cc      | 22 +++++++++----------
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/tensorflow/core/graph/validate_test.cc b/tensorflow/core/graph/validate_test.cc
index 713cfc7926..f6a0d2614a 100644
--- a/tensorflow/core/graph/validate_test.cc
+++ b/tensorflow/core/graph/validate_test.cc
@@ -174,8 +174,8 @@ TEST(VerifyNoDuplicateNodeNames, DuplicateNodeNames) {
   GraphDef graph_def;
   auto parser = protobuf::TextFormat::Parser();
   CHECK(parser.MergeFromString(graph_def_str, &graph_def)) << graph_def_str;
-  EXPECT_EQ(tensorflow::error::ALREADY_EXISTS,
-            graph::VerifyNoDuplicateNodeNames(graph_def).code());
+  EXPECT_EQ(graph::VerifyNoDuplicateNodeNames(graph_def).code(),
+            tensorflow::error::ALREADY_EXISTS);
 }
 
 }  // namespace
diff --git a/tensorflow/core/grappler/verifiers/structure_verifier_test.cc b/tensorflow/core/grappler/verifiers/structure_verifier_test.cc
index b0a2e4cd54..d3b1d0646d 100644
--- a/tensorflow/core/grappler/verifiers/structure_verifier_test.cc
+++ b/tensorflow/core/grappler/verifiers/structure_verifier_test.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include <memory>
 
+#include "absl/strings/match.h"
 #include "tensorflow/cc/ops/parsing_ops.h"
 #include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/core/framework/graph.pb.h"
@@ -23,7 +24,6 @@ limitations under the License.
 #include "tensorflow/core/lib/core/error_codes.pb.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
-#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/protobuf.h"
 #include "tensorflow/core/platform/test.h"
 
@@ -84,9 +84,9 @@ TEST_F(StructureVerifierTest, OpNotRegistered) {
       "node { name: 't1' op: 'TestMul' input: [ 'input:0', 't2' ] }"
       "node { name: 't2' op: 'TestMul' input: [ 'input:1', 't1' ] }");
   Status status = verifier_->Verify(graph_);
-  EXPECT_EQ(errors::Code::NOT_FOUND, status.code());
-  EXPECT_TRUE(tensorflow::str_util::StrContains(status.error_message(),
-                                                "Op type not registered"));
+  EXPECT_EQ(status.code(), errors::Code::NOT_FOUND);
+  EXPECT_TRUE(
+      absl::StrContains(status.error_message(), "Op type not registered"));
 }
 
 TEST_F(StructureVerifierTest, DuplicateNodeNames) {
@@ -94,9 +94,9 @@ TEST_F(StructureVerifierTest, DuplicateNodeNames) {
       "node { name: 'A' op: 'TestParams' }"
       "node { name: 'A' op: 'TestInput' }");
   Status status = verifier_->Verify(graph_);
-  EXPECT_EQ(errors::Code::ALREADY_EXISTS, status.code());
-  EXPECT_TRUE(tensorflow::str_util::StrContains(status.error_message(),
-                                                "Node already exists:"));
+  EXPECT_EQ(status.code(), errors::Code::ALREADY_EXISTS);
+  EXPECT_TRUE(
+      absl::StrContains(status.error_message(), "Node already exists:"));
 }
 
 TEST_F(StructureVerifierTest, GraphWithInvalidCycle) {
@@ -105,10 +105,10 @@ TEST_F(StructureVerifierTest, GraphWithInvalidCycle) {
       "node { name: 't1' op: 'TestMul' input: [ 'input:0', 't2' ] }"
       "node { name: 't2' op: 'TestMul' input: [ 'input:1', 't1' ] }");
   Status status = verifier_->Verify(graph_);
-  EXPECT_EQ(errors::Code::INVALID_ARGUMENT, status.code());
-  EXPECT_TRUE(tensorflow::str_util::StrContains(
-      status.error_message(),
-      "The graph couldn't be sorted in topological order"));
+  EXPECT_EQ(status.code(), errors::Code::INVALID_ARGUMENT);
+  EXPECT_TRUE(
+      absl::StrContains(status.error_message(),
+                        "The graph couldn't be sorted in topological order"));
 }
 
 }  // namespace
-- 
GitLab


From f7a34f2a3af1fda1c9fbae72137e5901694adbb1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 25 Jan 2019 15:29:12 -0800
Subject: [PATCH 1308/2345] Add more configurations to depthwise backprop
 filter tests.

PiperOrigin-RevId: 230980678
---
 tensorflow/compiler/xla/tests/BUILD           |  2 +-
 .../conv_depthwise_backprop_filter_test.cc    | 66 ++++++++++---------
 2 files changed, 36 insertions(+), 32 deletions(-)

diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 5dd9f3b408..8fb6742550 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -317,7 +317,7 @@ xla_test(
     name = "conv_depthwise_backprop_filter_test",
     timeout = "long",
     srcs = ["conv_depthwise_backprop_filter_test.cc"],
-    shard_count = 1,
+    shard_count = 6,
     deps = [
         "//tensorflow/compiler/xla:execution_options_util",
         "//tensorflow/compiler/xla:status_macros",
diff --git a/tensorflow/compiler/xla/tests/conv_depthwise_backprop_filter_test.cc b/tensorflow/compiler/xla/tests/conv_depthwise_backprop_filter_test.cc
index 90c1971403..c78d3f3d9e 100644
--- a/tensorflow/compiler/xla/tests/conv_depthwise_backprop_filter_test.cc
+++ b/tensorflow/compiler/xla/tests/conv_depthwise_backprop_filter_test.cc
@@ -33,13 +33,10 @@ string GetFloatDataType(bool use_bfloat16) {
 }
 
 struct DepthwiseConvolution2DSpec {
-  int64 output_batch, window;
+  int64 output_batch, window, window_dilation;
   std::vector<int64> activation_dims;
-  std::vector<int64> activation_layout;
   std::vector<int64> kernel_dims;
-  std::vector<int64> kernel_layout;
   std::vector<int64> output_dims;
-  std::vector<int64> output_layout;
 };
 
 class DepthwiseConvolution2DTest
@@ -50,7 +47,8 @@ class DepthwiseConvolution2DTest
 static std::vector<DepthwiseConvolution2DSpec> GetConv2DTestCases() {
   std::vector<DepthwiseConvolution2DSpec> config_set;
   std::vector<std::vector<int64>> config_options = {
-      {16, 5, 5, 2}, {64, 4, 4, 16}, {2, 5, 5, 256}};
+      {8, 5, 3, 2},   {4, 5, 5, 2},   {8, 7, 4, 128},  {16, 20, 20, 256},
+      {256, 7, 5, 4}, {256, 6, 6, 4}, {256, 8, 8, 512}};
 
   for (auto option : config_options) {
     int64 feature = option[3];
@@ -58,22 +56,35 @@ static std::vector<DepthwiseConvolution2DSpec> GetConv2DTestCases() {
     int64 kernel_size = option[2];
     int64 batch = option[0];
 
-    std::vector<int64> kernel_layout = {3, 2, 1, 0};
     DepthwiseConvolution2DSpec config;
+    config.window_dilation = 1;
     config.output_batch = feature;
     config.window = kernel_size;
 
     config.activation_dims = {batch, activation_size, activation_size, feature};
-    config.activation_layout = {0, 3, 2, 1};
 
     config.kernel_dims = {batch, kernel_size, kernel_size, feature};
-    config.kernel_layout = {0, 2, 3, 1};
 
-    config.output_dims = {3, 3, feature, 1};
+    int64 output_space_size = 3 + activation_size - kernel_size;
+    config.output_dims = {output_space_size, output_space_size, feature, 1};
 
-    // Try this layout for all kernel shapes.
-    config.output_layout = {3, 2, 0, 1};
     config_set.push_back(config);
+
+    // Add configurations for window dilation cases.
+    if (activation_size % 2 == 0 && activation_size == kernel_size) {
+      DepthwiseConvolution2DSpec config;
+      config.window_dilation = 2;
+      config.output_batch = feature;
+      config.window = kernel_size / 2;
+      config.activation_dims = {batch, activation_size, activation_size,
+                                feature};
+      config.kernel_dims = {batch, kernel_size / 2, kernel_size / 2, feature};
+
+      int64 output_space_size = 5;
+      config.output_dims = {output_space_size, output_space_size, feature, 1};
+
+      config_set.push_back(config);
+    }
   }
 
   return config_set;
@@ -86,11 +97,8 @@ string DepthwiseConvolution2DTestDataToString(
   const string data_type = GetFloatDataType(::testing::get<1>(data.param));
   string str = absl::StrCat(
       "activation_dims_", absl::StrJoin(spec.activation_dims, "x"),
-      "_activation_layout_", absl::StrJoin(spec.activation_layout, "_"),
-      "_kernel_dims_", absl::StrJoin(spec.kernel_dims, "x"), "_kernel_layout_",
-      absl::StrJoin(spec.kernel_layout, "_"), "_output_dims_",
-      absl::StrJoin(spec.output_dims, "x"), "_output_layout_",
-      absl::StrJoin(spec.output_layout, "_"), data_type);
+      "_kernel_dims_", absl::StrJoin(spec.kernel_dims, "x"), "_output_dims_",
+      absl::StrJoin(spec.output_dims, "x"), data_type);
 
   // Test names are not allowed to contain the '-' character.
   absl::c_replace(str, '-', 'n');
@@ -105,24 +113,20 @@ string BuildHloTextDepthwiseConvolution2D(
     HloModule TensorFlowDepthwiseConv
 
     ENTRY main {
-      activation = %s[%s]{%s} parameter(0)
-      kernel = %s[%s]{%s} parameter(1)
-      ROOT conv = %s[%s]{%s} convolution(%s[%s]{%s} activation, %s[%s]{%s} kernel),
-          window={size=%dx%d pad=1_1x1_1}, dim_labels=f01b_i01o->01fb,
+      activation = %s[%s] parameter(0)
+      kernel = %s[%s] parameter(1)
+      ROOT conv = %s[%s] convolution(%s[%s] activation, %s[%s] kernel),
+          window={size=%dx%d pad=1_%dx1_%d rhs_dilate=%dx%d}, dim_labels=f01b_i01o->01fb,
           batch_group_count=%d
     }
     )",
-      data_type, absl::StrJoin(spec.activation_dims, ","),
-      absl::StrJoin(spec.activation_layout, ","), data_type,
-      absl::StrJoin(spec.kernel_dims, ","),
-      absl::StrJoin(spec.kernel_layout, ","), data_type,
-      absl::StrJoin(spec.output_dims, ","),
-      absl::StrJoin(spec.output_layout, ","), data_type,
-      absl::StrJoin(spec.activation_dims, ","),
-      absl::StrJoin(spec.activation_layout, ","), data_type,
-      absl::StrJoin(spec.kernel_dims, ","),
-      absl::StrJoin(spec.kernel_layout, ","), spec.window, spec.window,
-      spec.output_batch);
+      data_type, absl::StrJoin(spec.activation_dims, ","), data_type,
+      absl::StrJoin(spec.kernel_dims, ","), data_type,
+      absl::StrJoin(spec.output_dims, ","), data_type,
+      absl::StrJoin(spec.activation_dims, ","), data_type,
+      absl::StrJoin(spec.kernel_dims, ","), spec.window, spec.window,
+      spec.window_dilation, spec.window_dilation, spec.window_dilation,
+      spec.window_dilation, spec.output_batch);
 }
 
 XLA_TEST_P(DepthwiseConvolution2DTest, DoIt) {
-- 
GitLab


From a5c92fe088fd5a8bd0f8bb4091a06151f2d92da4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 25 Jan 2019 15:33:16 -0800
Subject: [PATCH 1309/2345] Fix wrong error message for squared difference
 kernel.

PiperOrigin-RevId: 230981490
---
 tensorflow/lite/kernels/squared_difference.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/lite/kernels/squared_difference.cc b/tensorflow/lite/kernels/squared_difference.cc
index 59b53a6287..3661cf9f98 100644
--- a/tensorflow/lite/kernels/squared_difference.cc
+++ b/tensorflow/lite/kernels/squared_difference.cc
@@ -105,10 +105,10 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   } else if (output->type == kTfLiteInt32) {
     EvalSquaredDifference<int32_t>(context, node, data, input1, input2, output);
   } else {
-    context->ReportError(context,
-                         "SquaredDifference only supports FLOAT32, INT32 and "
-                         "quantized UINT8 now, got %d.",
-                         output->type);
+    context->ReportError(
+        context,
+        "SquaredDifference only supports FLOAT32 and INT32 now, got %d.",
+        output->type);
     return kTfLiteError;
   }
 
-- 
GitLab


From edd46128a22b855eaae0605b97ea59580cbf47b8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 25 Jan 2019 15:52:30 -0800
Subject: [PATCH 1310/2345] Fix a bug in env.cc

PiperOrigin-RevId: 230984555
---
 tensorflow/core/platform/env_test.cc  | 2 +-
 tensorflow/core/platform/posix/env.cc | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/platform/env_test.cc b/tensorflow/core/platform/env_test.cc
index a33f592096..5a5d133c96 100644
--- a/tensorflow/core/platform/env_test.cc
+++ b/tensorflow/core/platform/env_test.cc
@@ -395,7 +395,7 @@ TEST_F(DefaultEnvTest, CreateUniqueFileName) {
 TEST_F(DefaultEnvTest, GetThreadInformation) {
   Env* env = Env::Default();
 
-  EXPECT_GT(env->GetCurrentThreadId(), 0);
+  EXPECT_NE(env->GetCurrentThreadId(), 0);
   string thread_name;
   bool res = env->GetCurrentThreadName(&thread_name);
 #if defined(PLATFORM_WINDOWS) || defined(__ANDROID__)
diff --git a/tensorflow/core/platform/posix/env.cc b/tensorflow/core/platform/posix/env.cc
index 636da3b737..4e9373616e 100644
--- a/tensorflow/core/platform/posix/env.cc
+++ b/tensorflow/core/platform/posix/env.cc
@@ -88,6 +88,7 @@ class PosixEnv : public Env {
 
   int32 GetCurrentThreadId() {
 #ifdef __APPLE__
+    uint64_t tid64;
     pthread_threadid_np(nullptr, &tid64);
     return static_cast<int32>(tid64);
 #elif defined(__FreeBSD__)
-- 
GitLab


From c4388465d0d04ed90c814a11216217d19a8267e8 Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Fri, 25 Jan 2019 16:11:16 -0800
Subject: [PATCH 1311/2345] Update Estimator version in setup,py

This needs to be cherry-picked to TF r1.13 branch.
There now exists a tensorflow_estimator==1.13.0rc0 release on pypi.

PiperOrigin-RevId: 230987863
---
 tensorflow/tools/pip_package/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index 6e182a5e5c..678531643c 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -58,7 +58,7 @@ REQUIRED_PACKAGES = [
     'six >= 1.10.0',
     'protobuf >= 3.6.1',
     'tensorboard >= 1.12.0, < 1.13.0',
-    'tensorflow_estimator >= 1.10.0',
+    'tensorflow_estimator >= 1.13.0 < 1.14.0',
     'termcolor >= 1.1.0',
 ]
 
-- 
GitLab


From e3730df4a767be6ab8ebfeca6228eae8168bd342 Mon Sep 17 00:00:00 2001
From: nammbash <niroopshankar.ammbashankar@intel.com>
Date: Fri, 25 Jan 2019 16:42:05 -0800
Subject: [PATCH 1312/2345] fix api_test to succeed in Eigen along with
 config=mkl option

---
 tensorflow/core/ops/math_ops.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc
index 52243f2fcf..b6d5f353b3 100644
--- a/tensorflow/core/ops/math_ops.cc
+++ b/tensorflow/core/ops/math_ops.cc
@@ -1686,6 +1686,8 @@ Add two input tensors element wise using mkl kernel sum.
 inputs: Must all be the same size and shape.
 )doc");
 
+#endif  // INTEL_MKL
+
 REGISTER_OP("RequantizePerChannel")
     .Input("input: T")
     .Input("input_min: float")
@@ -1725,8 +1727,6 @@ REGISTER_OP("RequantizationRangePerChannel")
       return Status::OK();
     });
 
-#endif  // INTEL_MKL
-
 REGISTER_OP("NextAfter")
     .Attr("T: {float64, float32} = DT_FLOAT")
     .Input("x1: T")
-- 
GitLab


From 22aff5ca7066d1cd9ae84d9b96fda426991de8a7 Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Fri, 25 Jan 2019 16:22:47 -0800
Subject: [PATCH 1313/2345] Return collections from submodules too.

PiperOrigin-RevId: 230989794
---
 tensorflow/python/module/module.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/module/module.py b/tensorflow/python/module/module.py
index f9b8260025..62d1bd46c4 100644
--- a/tensorflow/python/module/module.py
+++ b/tensorflow/python/module/module.py
@@ -256,7 +256,7 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
 
   @property
   def owned_submodules(self):
-    """Returns an iterator of immediate child modules.
+    """Collection of immediate child modules.
 
     Child modules are modules which are found as properties of the current
     module.
@@ -271,13 +271,13 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
     >>> assert list(c.owned_submodules) == []
 
     Returns:
-      A generator over all child modules.
+      A collection of all child modules.
     """
-    return walk(self, predicate=_IS_MODULE)
+    return tuple(walk(self, predicate=_IS_MODULE))
 
   @property
   def submodules(self):
-    """Returns an iterator of all sub-modules recursively.
+    """Collection of all sub-modules.
 
     Submodules are modules which are properties of this module, or found as
     properties of modules which are properties of this module (and so on).
@@ -292,9 +292,9 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
     >>> assert list(c.submodules) == []
 
     Returns:
-      A generator over all submodules.
+      A collection of all submodules.
     """
-    return walk(self, recurse_if=_IS_MODULE, predicate=_IS_MODULE)
+    return tuple(walk(self, recurse_if=_IS_MODULE, predicate=_IS_MODULE))
 
   @classmethod
   def no_name_scope(cls, method):
-- 
GitLab


From cac4d12f353d6288888e4f60fdd63ae464b47f2a Mon Sep 17 00:00:00 2001
From: Dimitris Vardoulakis <dimvar@google.com>
Date: Fri, 25 Jan 2019 16:27:03 -0800
Subject: [PATCH 1314/2345] [TF:XLA] Typo fix.

PiperOrigin-RevId: 230990429
---
 tensorflow/compiler/xla/service/hlo_memory_scheduler.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/hlo_memory_scheduler.h b/tensorflow/compiler/xla/service/hlo_memory_scheduler.h
index 7227bfb27c..76cc29cbb7 100644
--- a/tensorflow/compiler/xla/service/hlo_memory_scheduler.h
+++ b/tensorflow/compiler/xla/service/hlo_memory_scheduler.h
@@ -118,7 +118,7 @@ class HloTrivialScheduler : public HloModulePass {
 };
 
 // A trivial pass which clears the schedule currently set on the
-// HloModule. After this pass runs HloModudle::has_schedule will return false.
+// HloModule. After this pass runs HloModule::has_schedule will return false.
 class HloDescheduler : public HloModulePass {
  public:
   HloDescheduler() = default;
-- 
GitLab


From 0edad9b45f86e966aed061c80b4700b1cf769255 Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Fri, 25 Jan 2019 16:44:53 -0800
Subject: [PATCH 1315/2345] These files have been moved (with their commit
 history) to TensorFlow docs.

See: tensorflow/docs#313 and [text_generation](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/sequences/text_generation.ipynb)
PiperOrigin-RevId: 230993141
---
 .../examples/generative_examples/cvae.ipynb   |  599 +-------
 .../examples/generative_examples/dcgan.ipynb  |  875 +-----------
 .../generative_examples/gans_diagram.png      |  Bin 63265 -> 0 bytes
 .../image_captioning_with_attention.ipynb     | 1231 +----------------
 .../generative_examples/text_generation.ipynb |  628 +--------
 5 files changed, 97 insertions(+), 3236 deletions(-)
 delete mode 100644 tensorflow/contrib/eager/python/examples/generative_examples/gans_diagram.png

diff --git a/tensorflow/contrib/eager/python/examples/generative_examples/cvae.ipynb b/tensorflow/contrib/eager/python/examples/generative_examples/cvae.ipynb
index 1a08cc0fd0..e1a02db76f 100644
--- a/tensorflow/contrib/eager/python/examples/generative_examples/cvae.ipynb
+++ b/tensorflow/contrib/eager/python/examples/generative_examples/cvae.ipynb
@@ -13,11 +13,13 @@
         "\n",
         "# Convolutional VAE: An example with tf.keras and eager\n",
         "\n",
+        "This example has moved:\n",
+        "\n",
         "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\u003ctd\u003e\n",
-        "\u003ca target=\"_blank\"  href=\"https://colab.research.google.com/github/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/generative_examples/cvae.ipynb\"\u003e\n",
+        "\u003ca target=\"_blank\"  href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/r2/tutorials/generative/cvae.ipynb\"\u003e\n",
         "    \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e  \n",
         "\u003c/td\u003e\u003ctd\u003e\n",
-        "\u003ca target=\"_blank\"  href=\"https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples/generative_examples/cvae.ipynb\"\u003e\u003cimg width=32px src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\u003c/td\u003e\u003c/table\u003e"
+        "\u003ca target=\"_blank\"  href=\"https://github.com/tensorflow/docs/blob/master/site/en/r2/tutorials/generative/cvae.ipynb\"\u003e\u003cimg width=32px src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\u003c/td\u003e\u003c/table\u003e"
       ]
     },
     {
@@ -28,604 +30,14 @@
       },
       "source": [
         "![evolution of output during training](https://tensorflow.org/images/autoencoders/cvae.gif)\n",
-        "\n",
-        "This notebook demonstrates how to generate images of handwritten digits using [tf.keras](https://www.tensorflow.org/programmers_guide/keras) and [eager execution](https://www.tensorflow.org/programmers_guide/eager) by training a Variational Autoencoder. (VAE, [[1]](https://arxiv.org/abs/1312.6114), [[2]](https://arxiv.org/abs/1401.4082)).\n",
         "\n"
       ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "P-JuIu2N_SQf"
-      },
-      "outputs": [],
-      "source": [
-        "# to generate gifs\n",
-        "!pip install imageio"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "e1_Y75QXJS6h"
-      },
-      "source": [
-        "## Import TensorFlow and enable Eager execution"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "YfIk2es3hJEd"
-      },
-      "outputs": [],
-      "source": [
-        "from __future__ import absolute_import, division, print_function\n",
-        "\n",
-        "# Import TensorFlow \u003e= 1.9 and enable eager execution\n",
-        "import tensorflow as tf\n",
-        "tfe = tf.contrib.eager\n",
-        "tf.enable_eager_execution()\n",
-        "\n",
-        "import os\n",
-        "import time\n",
-        "import numpy as np\n",
-        "import glob\n",
-        "import matplotlib.pyplot as plt\n",
-        "import PIL\n",
-        "import imageio\n",
-        "from IPython import display"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "iYn4MdZnKCey"
-      },
-      "source": [
-        "## Load the MNIST dataset\n",
-        "Each MNIST image is originally a vector of 784 integers, each of which is between 0-255 and represents the intensity of a pixel. We model each pixel with a Bernoulli distribution in our model, and we statically binarize the dataset."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "a4fYMGxGhrna"
-      },
-      "outputs": [],
-      "source": [
-        "(train_images, _), (test_images, _) = tf.keras.datasets.mnist.load_data()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "NFC2ghIdiZYE"
-      },
-      "outputs": [],
-      "source": [
-        "train_images = train_images.reshape(train_images.shape[0], 28, 28, 1).astype('float32')\n",
-        "test_images = test_images.reshape(test_images.shape[0], 28, 28, 1).astype('float32')\n",
-        "\n",
-        "# Normalizing the images to the range of [0., 1.]\n",
-        "train_images /= 255.\n",
-        "test_images /= 255.\n",
-        "\n",
-        "# Binarization\n",
-        "train_images[train_images \u003e= .5] = 1.\n",
-        "train_images[train_images \u003c .5] = 0.\n",
-        "test_images[test_images \u003e= .5] = 1.\n",
-        "test_images[test_images \u003c .5] = 0."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "S4PIDhoDLbsZ"
-      },
-      "outputs": [],
-      "source": [
-        "TRAIN_BUF = 60000\n",
-        "BATCH_SIZE = 100\n",
-        "\n",
-        "TEST_BUF = 10000"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "PIGN6ouoQxt3"
-      },
-      "source": [
-        "## Use *tf.data* to create batches and shuffle the dataset"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "-yKCCQOoJ7cn"
-      },
-      "outputs": [],
-      "source": [
-        "train_dataset = tf.data.Dataset.from_tensor_slices(train_images).shuffle(TRAIN_BUF).batch(BATCH_SIZE)\n",
-        "test_dataset = tf.data.Dataset.from_tensor_slices(test_images).shuffle(TEST_BUF).batch(BATCH_SIZE)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "THY-sZMiQ4UV"
-      },
-      "source": [
-        "## Wire up the generative and inference network with *tf.keras.Sequential*\n",
-        "\n",
-        "In our VAE example, we use two small ConvNets for the generative and inference network. Since these neural nets are small, we use `tf.keras.Sequential` to simplify our code. Let $x$ and $z$ denote the observation and latent variable respectively in the following descriptions. \n",
-        "\n",
-        "### Generative Network\n",
-        "This defines the generative model which takes a latent encoding as input, and outputs the parameters for a conditional distribution of the observation, i.e. $p(x|z)$. Additionally, we use a unit Gaussian prior $p(z)$ for the latent variable.\n",
-        "\n",
-        "### Inference Network\n",
-        "This defines an approximate posterior distribution $q(z|x)$, which takes as input an observation and outputs a set of parameters for the conditional distribution of the latent representation. In this example, we simply model this distribution as a diagonal Gaussian. In this case, the inference network outputs the mean and log-variance parameters of a factorized Gaussian (log-variance instead of the variance directly is for numerical stability).\n",
-        "\n",
-        "### Reparameterization Trick\n",
-        "During optimization, we can sample from $q(z|x)$ by first sampling from a unit Gaussian, and then multiplying by the standard deviation and adding the mean. This ensures the gradients could pass through the sample to the inference network parameters.\n",
-        "\n",
-        "### Network architecture\n",
-        "For the inference network, we use two convolutional layers followed by a fully-connected layer. In the generative network, we mirror this architecture by using a fully-connected layer followed by three convolution transpose layers (a.k.a. deconvolutional layers in some contexts). Note, it's common practice to avoid using batch normalization when training VAEs, since the additional stochasticity due to using mini-batches may aggravate instability on top of the stochasticity from sampling."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "VGLbvBEmjK0a"
-      },
-      "outputs": [],
-      "source": [
-        "class CVAE(tf.keras.Model):\n",
-        "  def __init__(self, latent_dim):\n",
-        "    super(CVAE, self).__init__()\n",
-        "    self.latent_dim = latent_dim\n",
-        "    self.inference_net = tf.keras.Sequential(\n",
-        "      [\n",
-        "          tf.keras.layers.InputLayer(input_shape=(28, 28, 1)),\n",
-        "          tf.keras.layers.Conv2D(\n",
-        "              filters=32, kernel_size=3, strides=(2, 2), activation=tf.nn.relu),\n",
-        "          tf.keras.layers.Conv2D(\n",
-        "              filters=64, kernel_size=3, strides=(2, 2), activation=tf.nn.relu),\n",
-        "          tf.keras.layers.Flatten(),\n",
-        "          # No activation\n",
-        "          tf.keras.layers.Dense(latent_dim + latent_dim),\n",
-        "      ]\n",
-        "    )\n",
-        "\n",
-        "    self.generative_net = tf.keras.Sequential(\n",
-        "        [\n",
-        "          tf.keras.layers.InputLayer(input_shape=(latent_dim,)),\n",
-        "          tf.keras.layers.Dense(units=7*7*32, activation=tf.nn.relu),\n",
-        "          tf.keras.layers.Reshape(target_shape=(7, 7, 32)),\n",
-        "          tf.keras.layers.Conv2DTranspose(\n",
-        "              filters=64,\n",
-        "              kernel_size=3,\n",
-        "              strides=(2, 2),\n",
-        "              padding=\"SAME\",\n",
-        "              activation=tf.nn.relu),\n",
-        "          tf.keras.layers.Conv2DTranspose(\n",
-        "              filters=32,\n",
-        "              kernel_size=3,\n",
-        "              strides=(2, 2),\n",
-        "              padding=\"SAME\",\n",
-        "              activation=tf.nn.relu),\n",
-        "          # No activation\n",
-        "          tf.keras.layers.Conv2DTranspose(\n",
-        "              filters=1, kernel_size=3, strides=(1, 1), padding=\"SAME\"),\n",
-        "        ]\n",
-        "    )\n",
-        "\n",
-        "  def sample(self, eps=None):\n",
-        "    if eps is None:\n",
-        "      eps = tf.random_normal(shape=(100, self.latent_dim))\n",
-        "    return self.decode(eps, apply_sigmoid=True)\n",
-        "\n",
-        "  def encode(self, x):\n",
-        "    mean, logvar = tf.split(self.inference_net(x), num_or_size_splits=2, axis=1)\n",
-        "    return mean, logvar\n",
-        "\n",
-        "  def reparameterize(self, mean, logvar):\n",
-        "    eps = tf.random_normal(shape=mean.shape)\n",
-        "    return eps * tf.exp(logvar * .5) + mean\n",
-        "\n",
-        "  def decode(self, z, apply_sigmoid=False):\n",
-        "    logits = self.generative_net(z)\n",
-        "    if apply_sigmoid:\n",
-        "      probs = tf.sigmoid(logits)\n",
-        "      return probs\n",
-        "\n",
-        "    return logits"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "0FMYgY_mPfTi"
-      },
-      "source": [
-        "## Define the loss function and the optimizer\n",
-        "\n",
-        "VAEs train by maximizing the evidence lower bound (ELBO) on the marginal log-likelihood:\n",
-        "\n",
-        "$$\\log p(x) \\ge \\text{ELBO} = \\mathbb{E}_{q(z|x)}\\left[\\log \\frac{p(x, z)}{q(z|x)}\\right].$$\n",
-        "\n",
-        "In practice, we optimize the single sample Monte Carlo estimate of this expectation:\n",
-        "\n",
-        "$$\\log p(x| z) + \\log p(z) - \\log q(z|x),$$\n",
-        "where $z$ is sampled from $q(z|x)$.\n",
-        "\n",
-        "**Note**: we could also analytically compute the KL term, but here we incorporate all three terms in the Monte Carlo estimator for simplicity."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "iWCn_PVdEJZ7"
-      },
-      "outputs": [],
-      "source": [
-        "def log_normal_pdf(sample, mean, logvar, raxis=1):\n",
-        "  log2pi = tf.log(2. * np.pi)\n",
-        "  return tf.reduce_sum(\n",
-        "      -.5 * ((sample - mean) ** 2. * tf.exp(-logvar) + logvar + log2pi),\n",
-        "      axis=raxis)\n",
-        "\n",
-        "def compute_loss(model, x):\n",
-        "  mean, logvar = model.encode(x)\n",
-        "  z = model.reparameterize(mean, logvar)\n",
-        "  x_logit = model.decode(z)\n",
-        "\n",
-        "  cross_ent = tf.nn.sigmoid_cross_entropy_with_logits(logits=x_logit, labels=x)\n",
-        "  logpx_z = -tf.reduce_sum(cross_ent, axis=[1, 2, 3])\n",
-        "  logpz = log_normal_pdf(z, 0., 0.)\n",
-        "  logqz_x = log_normal_pdf(z, mean, logvar)\n",
-        "  return -tf.reduce_mean(logpx_z + logpz - logqz_x)\n",
-        "\n",
-        "def compute_gradients(model, x):\n",
-        "  with tf.GradientTape() as tape:\n",
-        "    loss = compute_loss(model, x)\n",
-        "  return tape.gradient(loss, model.trainable_variables), loss\n",
-        "\n",
-        "optimizer = tf.train.AdamOptimizer(1e-4)\n",
-        "def apply_gradients(optimizer, gradients, variables, global_step=None):\n",
-        "  optimizer.apply_gradients(zip(gradients, variables), global_step=global_step)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Rw1fkAczTQYh"
-      },
-      "source": [
-        "## Training\n",
-        "\n",
-        "* We start by iterating over the dataset\n",
-        "* During each iteration, we pass the image to the encoder to obtain a set of mean and log-variance parameters of the approximate posterior $q(z|x)$\n",
-        "* We then apply the *reparameterization trick* to sample from $q(z|x)$\n",
-        "* Finally, we pass the reparameterized samples to the decoder to obtain the logits of the generative distribution $p(x|z)$\n",
-        "* **Note:** Since we use the dataset loaded by keras with 60k datapoints in the training set and 10k datapoints in the test set, our resulting ELBO on the test set is slightly higher than reported results in the literature which uses dynamic binarization of Larochelle's MNIST.\n",
-        "\n",
-        "## Generate Images\n",
-        "\n",
-        "* After training, it is time to generate some images\n",
-        "* We start by sampling a set of latent vectors from the unit Gaussian prior distribution $p(z)$\n",
-        "* The generator will then convert the latent sample $z$ to logits of the observation, giving a distribution $p(x|z)$\n",
-        "* Here we plot the probabilities of Bernoulli distributions\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "NS2GWywBbAWo"
-      },
-      "outputs": [],
-      "source": [
-        "epochs = 100\n",
-        "latent_dim = 50\n",
-        "num_examples_to_generate = 16\n",
-        "\n",
-        "# keeping the random vector constant for generation (prediction) so\n",
-        "# it will be easier to see the improvement.\n",
-        "random_vector_for_generation = tf.random_normal(\n",
-        "    shape=[num_examples_to_generate, latent_dim])\n",
-        "model = CVAE(latent_dim)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "RmdVsmvhPxyy"
-      },
-      "outputs": [],
-      "source": [
-        "def generate_and_save_images(model, epoch, test_input):\n",
-        "  predictions = model.sample(test_input)\n",
-        "  fig = plt.figure(figsize=(4,4))\n",
-        "\n",
-        "  for i in range(predictions.shape[0]):\n",
-        "      plt.subplot(4, 4, i+1)\n",
-        "      plt.imshow(predictions[i, :, :, 0], cmap='gray')\n",
-        "      plt.axis('off')\n",
-        "\n",
-        "  # tight_layout minimizes the overlap between 2 sub-plots\n",
-        "  plt.savefig('image_at_epoch_{:04d}.png'.format(epoch))\n",
-        "  plt.show()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "2M7LmLtGEMQJ"
-      },
-      "outputs": [],
-      "source": [
-        "generate_and_save_images(model, 0, random_vector_for_generation)\n",
-        "\n",
-        "for epoch in range(1, epochs + 1):\n",
-        "  start_time = time.time()\n",
-        "  for train_x in train_dataset:\n",
-        "    gradients, loss = compute_gradients(model, train_x)\n",
-        "    apply_gradients(optimizer, gradients, model.trainable_variables)\n",
-        "  end_time = time.time()\n",
-        "\n",
-        "  if epoch % 1 == 0:\n",
-        "    loss = tfe.metrics.Mean()\n",
-        "    for test_x in test_dataset:\n",
-        "      loss(compute_loss(model, test_x))\n",
-        "    elbo = -loss.result()\n",
-        "    display.clear_output(wait=False)\n",
-        "    print('Epoch: {}, Test set ELBO: {}, '\n",
-        "          'time elapse for current epoch {}'.format(epoch,\n",
-        "                                                    elbo,\n",
-        "                                                    end_time - start_time))\n",
-        "    generate_and_save_images(\n",
-        "        model, epoch, random_vector_for_generation)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "P4M_vIbUi7c0"
-      },
-      "source": [
-        "### Display an image using the epoch number"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "WfO5wCdclHGL"
-      },
-      "outputs": [],
-      "source": [
-        "def display_image(epoch_no):\n",
-        "  return PIL.Image.open('image_at_epoch_{:04d}.png'.format(epoch_no))"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "5x3q9_Oe5q0A"
-      },
-      "outputs": [],
-      "source": [
-        "display_image(epochs)  # Display images"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "NywiH3nL8guF"
-      },
-      "source": [
-        "### Generate a GIF of all the saved images."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "IGKQgENQ8lEI"
-      },
-      "outputs": [],
-      "source": [
-        "with imageio.get_writer('cvae.gif', mode='I') as writer:\n",
-        "  filenames = glob.glob('image*.png')\n",
-        "  filenames = sorted(filenames)\n",
-        "  last = -1\n",
-        "  for i,filename in enumerate(filenames):\n",
-        "    frame = 2*(i**0.5)\n",
-        "    if round(frame) \u003e round(last):\n",
-        "      last = frame\n",
-        "    else:\n",
-        "      continue\n",
-        "    image = imageio.imread(filename)\n",
-        "    writer.append_data(image)\n",
-        "  image = imageio.imread(filename)\n",
-        "  writer.append_data(image)\n",
-        "    \n",
-        "# this is a hack to display the gif inside the notebook\n",
-        "os.system('cp cvae.gif cvae.gif.png')"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "uV0yiKpzNP1b"
-      },
-      "outputs": [],
-      "source": [
-        "display.Image(filename=\"cvae.gif.png\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "yQXO_dlXkKsT"
-      },
-      "source": [
-        "To downlod the animation from Colab uncomment the code below:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "4fSJS3m5HLFM"
-      },
-      "outputs": [],
-      "source": [
-        "#from google.colab import files\n",
-        "#files.download('cvae.gif')"
-      ]
     }
   ],
   "metadata": {
     "accelerator": "GPU",
     "colab": {
       "collapsed_sections": [],
-      "default_view": {},
       "name": "cvae.ipynb",
       "private_outputs": true,
       "provenance": [
@@ -635,8 +47,7 @@
         }
       ],
       "toc_visible": true,
-      "version": "0.3.2",
-      "views": {}
+      "version": "0.3.2"
     },
     "kernelspec": {
       "display_name": "Python 3",
diff --git a/tensorflow/contrib/eager/python/examples/generative_examples/dcgan.ipynb b/tensorflow/contrib/eager/python/examples/generative_examples/dcgan.ipynb
index 78fcd39708..5376705883 100644
--- a/tensorflow/contrib/eager/python/examples/generative_examples/dcgan.ipynb
+++ b/tensorflow/contrib/eager/python/examples/generative_examples/dcgan.ipynb
@@ -1,26 +1,11 @@
 {
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "name": "dcgan.ipynb",
-      "version": "0.3.2",
-      "provenance": [],
-      "collapsed_sections": []
-    },
-    "kernelspec": {
-      "name": "python2",
-      "display_name": "Python 2"
-    },
-    "accelerator": "GPU"
-  },
   "cells": [
     {
+      "cell_type": "markdown",
       "metadata": {
         "colab_type": "text",
         "id": "0TD5ZrvEMbhZ"
       },
-      "cell_type": "markdown",
       "source": [
         "**Copyright 2018 The TensorFlow Authors**.\n",
         "\n",
@@ -28,851 +13,39 @@
         "\n",
         "# Generating Handwritten Digits with DCGAN\n",
         "\n",
-        "<table class=\"tfo-notebook-buttons\" align=\"left\"><td>\n",
-        "<a target=\"_blank\"  href=\"https://colab.research.google.com/github/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/generative_examples/dcgan.ipynb\">\n",
-        "    <img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />Run in Google Colab</a>  \n",
-        "</td><td>\n",
-        "<a target=\"_blank\"  href=\"https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples/generative_examples/dcgan.ipynb\"><img width=32px src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" />View source on GitHub</a></td></table>"
-      ]
-    },
-    {
-      "metadata": {
-        "colab_type": "text",
-        "id": "ITZuApL56Mny"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "This tutorial demonstrates how to generate images of handwritten digits using a Deep Convolutional Generative Adversarial Network ([DCGAN](https://arxiv.org/pdf/1511.06434.pdf)). The code is written in [tf.keras](https://www.tensorflow.org/programmers_guide/keras) with [eager execution](https://www.tensorflow.org/programmers_guide/eager) enabled. "
-      ]
-    },
-    {
-      "metadata": {
-        "colab_type": "toc",
-        "id": "x2McrO9bMyLN"
-      },
-      "cell_type": "markdown",
-      "source": [
-        ">[Generating Handwritten Digits with DCGAN](#scrollTo=0TD5ZrvEMbhZ)\n",
-        "\n",
-        ">>[What are GANs?](#scrollTo=2MbKJY38Puy9)\n",
-        "\n",
-        ">>>[Import TensorFlow and enable eager execution](#scrollTo=e1_Y75QXJS6h)\n",
-        "\n",
-        ">>>[Load the dataset](#scrollTo=iYn4MdZnKCey)\n",
-        "\n",
-        ">>>[Use tf.data to create batches and shuffle the dataset](#scrollTo=PIGN6ouoQxt3)\n",
-        "\n",
-        ">>[Create the models](#scrollTo=THY-sZMiQ4UV)\n",
-        "\n",
-        ">>>[The Generator Model](#scrollTo=-tEyxE-GMC48)\n",
-        "\n",
-        ">>>[The Discriminator model](#scrollTo=D0IKnaCtg6WE)\n",
-        "\n",
-        ">>[Define the loss functions and the optimizer](#scrollTo=0FMYgY_mPfTi)\n",
-        "\n",
-        ">>>[Generator loss](#scrollTo=Jd-3GCUEiKtv)\n",
-        "\n",
-        ">>>[Discriminator loss](#scrollTo=PKY_iPSPNWoj)\n",
-        "\n",
-        ">>[Set up GANs for Training](#scrollTo=Rw1fkAczTQYh)\n",
-        "\n",
-        ">>[Train the GANs](#scrollTo=dZrd4CdjR-Fp)\n",
-        "\n",
-        ">>[Generated images](#scrollTo=P4M_vIbUi7c0)\n",
+        "This example has moved.\n",
         "\n",
-        ">>[Learn more about GANs](#scrollTo=k6qC-SbjK0yW)\n",
-        "\n"
+        "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\u003ctd\u003e\n",
+        "\u003ca target=\"_blank\"  href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/r2/tutorials/generative/dcgan.ipynb\"\u003e\n",
+        "    \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e  \n",
+        "\u003c/td\u003e\u003ctd\u003e\n",
+        "\u003ca target=\"_blank\"  href=\"https://github.com/tensorflow/blob/master/site/en/r2/tutorials/generative/dcgan.ipynb\"\u003e\u003cimg width=32px src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\u003c/td\u003e\u003c/table\u003e"
       ]
     },
     {
+      "cell_type": "markdown",
       "metadata": {
         "colab_type": "text",
         "id": "2MbKJY38Puy9"
       },
-      "cell_type": "markdown",
       "source": [
-        "## What are GANs?\n",
-        "GANs, or [Generative Adversarial Networks](https://arxiv.org/abs/1406.2661), are a framework for estimating generative models. Two models are trained simultaneously by an adversarial process: a Generator, which is responsible for generating data (say, images), and a Discriminator, which is responsible for estimating the probability that an image was drawn from the training data (the image is real), or was produced by the Generator (the image is fake). During training, the Generator becomes progressively better at generating images, until the Discriminator is no longer able to distinguish real images from fake. \n",
-        "\n",
-        "![alt text](https://github.com/margaretmz/tensorflow/blob/margaret-dcgan/tensorflow/contrib/eager/python/examples/generative_examples/gans_diagram.png?raw=1)\n",
-        "\n",
-        "We will demonstrate this process end-to-end on MNIST. Below is an animation that shows a series of images produced by the Generator as it was trained for 50 epochs. Overtime, the generated images become increasingly difficult to distinguish from the training set.\n",
-        "\n",
-        "To learn more about GANs, we recommend MIT's [Intro to Deep Learning](http://introtodeeplearning.com/) course, which includes a lecture on Deep Generative Models ([video](https://youtu.be/JVb54xhEw6Y) | [slides](http://introtodeeplearning.com/materials/2018_6S191_Lecture4.pdf)). Now, let's head to the code!\n",
-        "\n",
         "![sample output](https://tensorflow.org/images/gan/dcgan.gif)"
       ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "collapsed_sections": [],
+      "name": "dcgan.ipynb",
+      "provenance": [],
+      "version": "0.3.2"
     },
-    {
-      "metadata": {
-        "colab_type": "code",
-        "id": "u_2z-B3piVsw",
-        "colab": {}
-      },
-      "cell_type": "code",
-      "source": [
-        "# Install imgeio in order to generate an animated gif showing the image generating process\n",
-        "!pip install imageio"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "colab_type": "text",
-        "id": "e1_Y75QXJS6h"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "### Import TensorFlow and enable eager execution"
-      ]
-    },
-    {
-      "metadata": {
-        "colab_type": "code",
-        "id": "YfIk2es3hJEd",
-        "colab": {}
-      },
-      "cell_type": "code",
-      "source": [
-        "import tensorflow as tf\n",
-        "tf.enable_eager_execution()\n",
-        "\n",
-        "import glob\n",
-        "import imageio\n",
-        "import matplotlib.pyplot as plt\n",
-        "import numpy as np\n",
-        "import os\n",
-        "import PIL\n",
-        "import time\n",
-        "\n",
-        "from IPython import display"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "colab_type": "text",
-        "id": "iYn4MdZnKCey"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "### Load the dataset\n",
-        "\n",
-        "We are going to use the MNIST dataset to train the generator and the discriminator. The generator will generate handwritten digits resembling the MNIST data."
-      ]
-    },
-    {
-      "metadata": {
-        "colab_type": "code",
-        "id": "a4fYMGxGhrna",
-        "colab": {}
-      },
-      "cell_type": "code",
-      "source": [
-        "(train_images, train_labels), (_, _) = tf.keras.datasets.mnist.load_data()"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "colab_type": "code",
-        "id": "NFC2ghIdiZYE",
-        "colab": {}
-      },
-      "cell_type": "code",
-      "source": [
-        "train_images = train_images.reshape(train_images.shape[0], 28, 28, 1).astype('float32')\n",
-        "train_images = (train_images - 127.5) / 127.5 # Normalize the images to [-1, 1]"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "colab_type": "code",
-        "id": "S4PIDhoDLbsZ",
-        "colab": {}
-      },
-      "cell_type": "code",
-      "source": [
-        "BUFFER_SIZE = 60000\n",
-        "BATCH_SIZE = 256"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "colab_type": "text",
-        "id": "PIGN6ouoQxt3"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "### Use tf.data to create batches and shuffle the dataset"
-      ]
-    },
-    {
-      "metadata": {
-        "colab_type": "code",
-        "id": "-yKCCQOoJ7cn",
-        "colab": {}
-      },
-      "cell_type": "code",
-      "source": [
-        "train_dataset = tf.data.Dataset.from_tensor_slices(train_images).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "colab_type": "text",
-        "id": "THY-sZMiQ4UV"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "## Create the models\n",
-        "\n",
-        "We will use tf.keras [Sequential API](https://www.tensorflow.org/guide/keras#sequential_model) to define the generator and discriminator models."
-      ]
-    },
-    {
-      "metadata": {
-        "colab_type": "text",
-        "id": "-tEyxE-GMC48"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "### The Generator Model\n",
-        "\n",
-        "The generator is responsible for creating convincing images that are good enough to fool the discriminator. The network architecture for the generator consists of [Conv2DTranspose](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Conv2DTranspose) (Upsampling) layers. We start with a fully connected layer and upsample the image two times in order to reach the desired image size of 28x28x1. We increase the width and height, and reduce the depth as we move through the layers in the network. We use [Leaky ReLU](https://www.tensorflow.org/api_docs/python/tf/keras/layers/LeakyReLU) activation for each layer except for the last one where we use a tanh activation."
-      ]
-    },
-    {
-      "metadata": {
-        "id": "6bpTcDqoLWjY",
-        "colab_type": "code",
-        "colab": {}
-      },
-      "cell_type": "code",
-      "source": [
-        "def make_generator_model():\n",
-        "    model = tf.keras.Sequential()\n",
-        "    model.add(tf.keras.layers.Dense(7*7*256, use_bias=False, input_shape=(100,)))\n",
-        "    model.add(tf.keras.layers.BatchNormalization())\n",
-        "    model.add(tf.keras.layers.LeakyReLU())\n",
-        "      \n",
-        "    model.add(tf.keras.layers.Reshape((7, 7, 256)))\n",
-        "    assert model.output_shape == (None, 7, 7, 256) # Note: None is the batch size\n",
-        "    \n",
-        "    model.add(tf.keras.layers.Conv2DTranspose(128, (5, 5), strides=(1, 1), padding='same', use_bias=False))\n",
-        "    assert model.output_shape == (None, 7, 7, 128)  \n",
-        "    model.add(tf.keras.layers.BatchNormalization())\n",
-        "    model.add(tf.keras.layers.LeakyReLU())\n",
-        "\n",
-        "    model.add(tf.keras.layers.Conv2DTranspose(64, (5, 5), strides=(2, 2), padding='same', use_bias=False))\n",
-        "    assert model.output_shape == (None, 14, 14, 64)    \n",
-        "    model.add(tf.keras.layers.BatchNormalization())\n",
-        "    model.add(tf.keras.layers.LeakyReLU())\n",
-        "\n",
-        "    model.add(tf.keras.layers.Conv2DTranspose(1, (5, 5), strides=(2, 2), padding='same', use_bias=False, activation='tanh'))\n",
-        "    assert model.output_shape == (None, 28, 28, 1)\n",
-        "  \n",
-        "    return model"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "colab_type": "text",
-        "id": "D0IKnaCtg6WE"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "### The Discriminator model\n",
-        "\n",
-        "The discriminator is responsible for distinguishing fake images from real images. It's similar to a regular CNN-based image classifier."
-      ]
-    },
-    {
-      "metadata": {
-        "id": "dw2tPLmk2pEP",
-        "colab_type": "code",
-        "colab": {}
-      },
-      "cell_type": "code",
-      "source": [
-        "def make_discriminator_model():\n",
-        "    model = tf.keras.Sequential()\n",
-        "    model.add(tf.keras.layers.Conv2D(64, (5, 5), strides=(2, 2), padding='same'))\n",
-        "    model.add(tf.keras.layers.LeakyReLU())\n",
-        "    model.add(tf.keras.layers.Dropout(0.3))\n",
-        "      \n",
-        "    model.add(tf.keras.layers.Conv2D(128, (5, 5), strides=(2, 2), padding='same'))\n",
-        "    model.add(tf.keras.layers.LeakyReLU())\n",
-        "    model.add(tf.keras.layers.Dropout(0.3))\n",
-        "       \n",
-        "    model.add(tf.keras.layers.Flatten())\n",
-        "    model.add(tf.keras.layers.Dense(1))\n",
-        "     \n",
-        "    return model"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "colab_type": "code",
-        "id": "gDkA05NE6QMs",
-        "colab": {}
-      },
-      "cell_type": "code",
-      "source": [
-        "generator = make_generator_model()\n",
-        "discriminator = make_discriminator_model()"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "colab_type": "text",
-        "id": "0FMYgY_mPfTi"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "## Define the loss functions and the optimizer\n",
-        "\n",
-        "Let's define the loss functions and the optimizers for the generator and the discriminator.\n"
-      ]
-    },
-    {
-      "metadata": {
-        "colab_type": "text",
-        "id": "Jd-3GCUEiKtv"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "### Generator loss\n",
-        "The generator loss is a sigmoid cross entropy loss of the generated images and an array of ones, since the generator is trying to generate fake images that resemble the real images."
-      ]
-    },
-    {
-      "metadata": {
-        "colab_type": "code",
-        "id": "90BIcCKcDMxz",
-        "colab": {}
-      },
-      "cell_type": "code",
-      "source": [
-        "def generator_loss(generated_output):\n",
-        "    return tf.losses.sigmoid_cross_entropy(tf.ones_like(generated_output), generated_output)"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "colab_type": "text",
-        "id": "PKY_iPSPNWoj"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "### Discriminator loss\n",
-        "\n",
-        "The discriminator loss function takes two inputs: real images, and generated images. Here is how to calculate the discriminator loss:\n",
-        "1. Calculate real_loss which is a sigmoid cross entropy loss of the real images and an array of ones (since these are the real images).\n",
-        "2. Calculate generated_loss which is a sigmoid cross entropy loss of the generated images and an array of zeros (since these are the fake images).\n",
-        "3. Calculate the total_loss as the sum of real_loss and generated_loss."
-      ]
-    },
-    {
-      "metadata": {
-        "colab_type": "code",
-        "id": "wkMNfBWlT-PV",
-        "colab": {}
-      },
-      "cell_type": "code",
-      "source": [
-        "def discriminator_loss(real_output, generated_output):\n",
-        "    # [1,1,...,1] with real output since it is true and we want our generated examples to look like it\n",
-        "    real_loss = tf.losses.sigmoid_cross_entropy(multi_class_labels=tf.ones_like(real_output), logits=real_output)\n",
-        "\n",
-        "    # [0,0,...,0] with generated images since they are fake\n",
-        "    generated_loss = tf.losses.sigmoid_cross_entropy(multi_class_labels=tf.zeros_like(generated_output), logits=generated_output)\n",
-        "\n",
-        "    total_loss = real_loss + generated_loss\n",
-        "\n",
-        "    return total_loss"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "colab_type": "text",
-        "id": "MgIc7i0th_Iu"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "The discriminator and the generator optimizers are different since we will train two networks separately."
-      ]
-    },
-    {
-      "metadata": {
-        "colab_type": "code",
-        "id": "iWCn_PVdEJZ7",
-        "colab": {}
-      },
-      "cell_type": "code",
-      "source": [
-        "generator_optimizer = tf.train.AdamOptimizer(1e-4)\n",
-        "discriminator_optimizer = tf.train.AdamOptimizer(1e-4)"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "colab_type": "text",
-        "id": "mWtinsGDPJlV"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "**Checkpoints (Object-based saving)**"
-      ]
-    },
-    {
-      "metadata": {
-        "colab_type": "code",
-        "id": "CA1w-7s2POEy",
-        "colab": {}
-      },
-      "cell_type": "code",
-      "source": [
-        "checkpoint_dir = './training_checkpoints'\n",
-        "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n",
-        "checkpoint = tf.train.Checkpoint(generator_optimizer=generator_optimizer,\n",
-        "                                 discriminator_optimizer=discriminator_optimizer,\n",
-        "                                 generator=generator,\n",
-        "                                 discriminator=discriminator)"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "colab_type": "text",
-        "id": "Rw1fkAczTQYh"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "## Set up GANs for Training\n",
-        "\n"
-      ]
-    },
-    {
-      "metadata": {
-        "colab_type": "text",
-        "id": "5QC5BABamh_c"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "Now it's time to put together the generator and discriminator to set up the Generative Adversarial Networks, as you see in the diagam at the beginning of the tutorial."
-      ]
-    },
-    {
-      "metadata": {
-        "colab_type": "text",
-        "id": "Ff6oN6PZX27n"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "**Define training parameters**"
-      ]
-    },
-    {
-      "metadata": {
-        "colab_type": "code",
-        "id": "NS2GWywBbAWo",
-        "colab": {}
-      },
-      "cell_type": "code",
-      "source": [
-        "EPOCHS = 50\n",
-        "noise_dim = 100\n",
-        "num_examples_to_generate = 16\n",
-        "\n",
-        "# We'll re-use this random vector used to seed the generator so\n",
-        "# it will be easier to see the improvement over time.\n",
-        "random_vector_for_generation = tf.random_normal([num_examples_to_generate,\n",
-        "                                                 noise_dim])"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "colab_type": "text",
-        "id": "jylSonrqSWfi"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "**Define training method**\n",
-        "\n",
-        "We start by iterating over the dataset. The generator is given a random vector as an input which is processed to  output an image looking like a handwritten digit. The discriminator is then shown the real MNIST images as well as the generated images.\n",
-        "\n",
-        "Next, we calculate the generator and the discriminator loss. Then, we calculate the gradients of loss with respect to both the generator and the discriminator variables."
-      ]
-    },
-    {
-      "metadata": {
-        "id": "3t5ibNo05jCB",
-        "colab_type": "code",
-        "colab": {}
-      },
-      "cell_type": "code",
-      "source": [
-        "def train_step(images):\n",
-        "   # generating noise from a normal distribution\n",
-        "      noise = tf.random_normal([BATCH_SIZE, noise_dim])\n",
-        "      \n",
-        "      with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape:\n",
-        "        generated_images = generator(noise, training=True)\n",
-        "      \n",
-        "        real_output = discriminator(images, training=True)\n",
-        "        generated_output = discriminator(generated_images, training=True)\n",
-        "         \n",
-        "        gen_loss = generator_loss(generated_output)\n",
-        "        disc_loss = discriminator_loss(real_output, generated_output)\n",
-        "        \n",
-        "      gradients_of_generator = gen_tape.gradient(gen_loss, generator.variables)\n",
-        "      gradients_of_discriminator = disc_tape.gradient(disc_loss, discriminator.variables)\n",
-        "      \n",
-        "      generator_optimizer.apply_gradients(zip(gradients_of_generator, generator.variables))\n",
-        "      discriminator_optimizer.apply_gradients(zip(gradients_of_discriminator, discriminator.variables))"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "colab_type": "text",
-        "id": "6TSZgwc2BUQ-"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "\n",
-        "This model takes about ~30 seconds per epoch to train on a single Tesla K80 on Colab, as of October 2018. \n",
-        "\n",
-        "Eager execution can be slower than executing the equivalent graph as it can't benefit from whole-program optimizations on the graph, and also incurs overheads of interpreting Python code. By using [tf.contrib.eager.defun](https://www.tensorflow.org/api_docs/python/tf/contrib/eager/defun) to create graph functions, we get a ~20 secs/epoch performance boost (from ~50 secs/epoch down to ~30 secs/epoch). This way we get the best of both eager execution (easier for debugging) and graph mode (better performance)."
-      ]
-    },
-    {
-      "metadata": {
-        "id": "Iwya07_j5p2A",
-        "colab_type": "code",
-        "colab": {}
-      },
-      "cell_type": "code",
-      "source": [
-        "train_step = tf.contrib.eager.defun(train_step)"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "colab_type": "code",
-        "id": "2M7LmLtGEMQJ",
-        "colab": {}
-      },
-      "cell_type": "code",
-      "source": [
-        "def train(dataset, epochs):  \n",
-        "  for epoch in range(epochs):\n",
-        "    start = time.time()\n",
-        "    \n",
-        "    for images in dataset:\n",
-        "      train_step(images)\n",
-        "\n",
-        "    display.clear_output(wait=True)\n",
-        "    generate_and_save_images(generator,\n",
-        "                               epoch + 1,\n",
-        "                               random_vector_for_generation)\n",
-        "    \n",
-        "    # saving (checkpoint) the model every 15 epochs\n",
-        "    if (epoch + 1) % 15 == 0:\n",
-        "      checkpoint.save(file_prefix = checkpoint_prefix)\n",
-        "    \n",
-        "    print ('Time taken for epoch {} is {} sec'.format(epoch + 1,\n",
-        "                                                      time.time()-start))\n",
-        "  # generating after the final epoch\n",
-        "  display.clear_output(wait=True)\n",
-        "  generate_and_save_images(generator,\n",
-        "                           epochs,\n",
-        "                           random_vector_for_generation)"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "colab_type": "text",
-        "id": "2aFF7Hk3XdeW"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "**Generate and save images**\n",
-        "\n"
-      ]
-    },
-    {
-      "metadata": {
-        "colab_type": "code",
-        "id": "RmdVsmvhPxyy",
-        "colab": {}
-      },
-      "cell_type": "code",
-      "source": [
-        "def generate_and_save_images(model, epoch, test_input):\n",
-        "  # make sure the training parameter is set to False because we\n",
-        "  # don't want to train the batchnorm layer when doing inference.\n",
-        "  predictions = model(test_input, training=False)\n",
-        "\n",
-        "  fig = plt.figure(figsize=(4,4))\n",
-        "  \n",
-        "  for i in range(predictions.shape[0]):\n",
-        "      plt.subplot(4, 4, i+1)\n",
-        "      plt.imshow(predictions[i, :, :, 0] * 127.5 + 127.5, cmap='gray')\n",
-        "      plt.axis('off')\n",
-        "        \n",
-        "  plt.savefig('image_at_epoch_{:04d}.png'.format(epoch))\n",
-        "  plt.show()"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "colab_type": "text",
-        "id": "dZrd4CdjR-Fp"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "## Train the GANs\n",
-        "We will call the train() method defined above to train the generator and discriminator simultaneously. Note, training GANs can be tricky. It's important that the generator and discriminator do not overpower each other (e.g., that they train at a similar rate).\n",
-        "\n",
-        "At the beginning of the training, the generated images look like random noise. As training progresses, you can see the generated digits look increasingly real. After 50 epochs, they look very much like the MNIST digits."
-      ]
-    },
-    {
-      "metadata": {
-        "colab_type": "code",
-        "id": "Ly3UN0SLLY2l",
-        "colab": {}
-      },
-      "cell_type": "code",
-      "source": [
-        "%%time\n",
-        "train(train_dataset, EPOCHS)"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "colab_type": "text",
-        "id": "rfM4YcPVPkNO"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "**Restore the latest checkpoint**"
-      ]
-    },
-    {
-      "metadata": {
-        "colab_type": "code",
-        "id": "XhXsd0srPo8c",
-        "colab": {}
-      },
-      "cell_type": "code",
-      "source": [
-        "# restoring the latest checkpoint in checkpoint_dir\n",
-        "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "colab_type": "text",
-        "id": "P4M_vIbUi7c0"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "## Generated images \n"
-      ]
-    },
-    {
-      "metadata": {
-        "colab_type": "text",
-        "id": "mLskt7EfXAjr"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "\n",
-        "After training, its time to generate some images! \n",
-        "The last step is to plot the generated images and voila!\n"
-      ]
-    },
-    {
-      "metadata": {
-        "colab_type": "code",
-        "id": "WfO5wCdclHGL",
-        "colab": {}
-      },
-      "cell_type": "code",
-      "source": [
-        "# Display a single image using the epoch number\n",
-        "def display_image(epoch_no):\n",
-        "  return PIL.Image.open('image_at_epoch_{:04d}.png'.format(epoch_no))"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "colab_type": "code",
-        "id": "5x3q9_Oe5q0A",
-        "colab": {}
-      },
-      "cell_type": "code",
-      "source": [
-        "display_image(EPOCHS)"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "colab_type": "text",
-        "id": "NywiH3nL8guF"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "**Generate a GIF of all the saved images**\n",
-        "\n",
-        "We will use imageio to create an animated gif using all the images saved during training."
-      ]
-    },
-    {
-      "metadata": {
-        "colab_type": "code",
-        "id": "IGKQgENQ8lEI",
-        "colab": {}
-      },
-      "cell_type": "code",
-      "source": [
-        "with imageio.get_writer('dcgan.gif', mode='I') as writer:\n",
-        "  filenames = glob.glob('image*.png')\n",
-        "  filenames = sorted(filenames)\n",
-        "  last = -1\n",
-        "  for i,filename in enumerate(filenames):\n",
-        "    frame = 2*(i**0.5)\n",
-        "    if round(frame) > round(last):\n",
-        "      last = frame\n",
-        "    else:\n",
-        "      continue\n",
-        "    image = imageio.imread(filename)\n",
-        "    writer.append_data(image)\n",
-        "  image = imageio.imread(filename)\n",
-        "  writer.append_data(image)\n",
-        "    \n",
-        "# this is a hack to display the gif inside the notebook\n",
-        "os.system('cp dcgan.gif dcgan.gif.png')"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "colab_type": "text",
-        "id": "cGhC3-fMWSwl"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "Display the animated gif with all the mages generated during the training of GANs."
-      ]
-    },
-    {
-      "metadata": {
-        "colab_type": "code",
-        "id": "uV0yiKpzNP1b",
-        "colab": {}
-      },
-      "cell_type": "code",
-      "source": [
-        "display.Image(filename=\"dcgan.gif.png\")"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "colab_type": "text",
-        "id": "6EEG-wePkmJQ"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "**Download the animated gif**\n",
-        "\n",
-        "Uncomment the code below to download an animated gif from Colab."
-      ]
-    },
-    {
-      "metadata": {
-        "colab_type": "code",
-        "id": "4UJjSnIMOzOJ",
-        "colab": {}
-      },
-      "cell_type": "code",
-      "source": [
-        "#from google.colab import files\n",
-        "#files.download('dcgan.gif')"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "colab_type": "text",
-        "id": "k6qC-SbjK0yW"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "## Learn more about GANs\n"
-      ]
-    },
-    {
-      "metadata": {
-        "colab_type": "text",
-        "id": "xjjkT9KAK6H7"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "We hope this tutorial was helpful! As a next step, you might like to experiment with a different dataset, for example the Large-scale Celeb Faces Attributes (CelebA) dataset [available on Kaggle](https://www.kaggle.com/jessicali9530/celeba-dataset/home).\n",
-        "\n",
-        "To learn more about GANs:\n",
-        "\n",
-        "* Check out MIT's lecture (linked above), or [this](http://cs231n.stanford.edu/slides/2018/cs231n_2018_lecture12.pdf) lecture form Stanford's CS231n. \n",
-        "\n",
-        "* We also recommend the [CVPR 2018 Tutorial on GANs](https://sites.google.com/view/cvpr2018tutorialongans/), and the [NIPS 2016 Tutorial: Generative Adversarial Networks](https://arxiv.org/abs/1701.00160).\n"
-      ]
+    "kernelspec": {
+      "display_name": "Python 2",
+      "name": "python2"
     }
-  ]
-}
\ No newline at end of file
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/tensorflow/contrib/eager/python/examples/generative_examples/gans_diagram.png b/tensorflow/contrib/eager/python/examples/generative_examples/gans_diagram.png
deleted file mode 100644
index b715bd83ef117641c6429e0ac173dbe9b8d5fd88..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 63265
zcmeAS@N?(olHy`uVBq!ia0y~yV7bG<z+}n6#=yW(HC0`jfq{Xg*vT`5gM)*kh9jke
zfq{Xuz$3Dlfq~@>2s3u&{Vif(V2~_vjVKAuPb(=;EJ|f4FE7{2%*!rLPAo{(%P&fw
z{mw>;fkA=6)5S5QV$Pepl_gWJ-rfH3b&6brPm5&8vy%;74a^!^UfHb6qn2;uyQ&p(
z^x@U4t6Fx8{9U(%WnX>tWkp20*V-UaQ5Kekr&jP7aV4s?sI*M}S@)mAQh-C`qe95K
z^HGt{?i{}Nd!O+2Q@=jlu5NEq;FzG|IZ4HHl8C4wdldtSrQ&&szq4^RRM?d*4<Z;O
zBDn*sQpGbc<$)T6={j*#;b`!Hlggk=6ougQ{PICaRFs#i>(NHJwlB?s;^OW}Nsk)k
z+EhG$$w}<+oV3J>)zKj|^eRM}<zS;+lZxlBc@|4UAo1Fz#B2^p>zZ9WcfdZKv_!+G
zU^0Zc=)?iv!Kd(F_fyenoAma#oLcq3f!}qKis$VkvM(4tCaK7J)Yq1CD#y>4|M_Qj
z@h#(ufZr#cr)F;}c=lFUX}<NAoCDF{zp_49W`F*{-j}Oi-ZN_se`oWsTH5$PSLx9h
z+ajyt>3M7ld*hfk|NoWzD9QQ$Yl||IPhVO8e7$=##ndC^Qu1H^8x|^_llV-u@2GfQ
zQe3-a&7`JBpH5u=G4ply-1#Qu&TBtM9}=Bd{P5DN>TYkXvb?hW(jRZtJkz`D<D+pT
z@U4->w&Z7wytf&=POrZ29=PLAL!a{`m6r~i5BW|~;pAvl;CS%4wX(%q+taUy^}%I*
z^D9>S7q70n)F%Hu&&_Rb%lqmbI`Ppw2Q=NypS9kX`@Z+PzUQPRca1?Y#$@zLRX|?+
z@UA~X%g)|Eaq_NouqeyK18b8{n><^uv7+?+v;tPv=u;jYtLK>*l-_fjwnmF-fxn+J
zU-lfX17BDxYxZsuXbA4hbK1P-$=$xio3tKGzWI8lS-o=W(WhIo=A1h4IQqh##ILrk
zN1yJoRh@1+`;fy|sXYDO=wpFzjb0X2+|qlyHfH}+|FreWigwc&kDEPRS+AFtoyq$A
zYp&;Y<x7pP0zNL5%+}Ogy(sr{)tuTSZ==`O`2@F53;EQqxl<=5(!(=T_2$dYFK084
zY?R%+?{4=r6;CS`P&!s&To};t*vdTR%<`Q+vvxO5-ff<C?~~J~?o%E;{ZD7`&D_nB
zx7hwy+xF#8E-iLeOP9GlrHxl;=a+w)Q>KJo+MCOkf4^SnwfXddo6W(R8hXpqkL&1J
zef@B^FEQCsbH9{fZ@vDB)un&<7e6<A(<M`P?>~o`$dsbnmp3atPt!Y<{%lcAU3uno
zE9UK%-k*2AeHnShyYKPU{^`cKb)~o4%{`Z%e)#gwx0#W9O!6YvR=fSlwbQ>>XJc!X
z!n7eRct-KNStm|?jn0lW|MZph&Zo_OcPi?R|NqmZ>N$x|aos~T&q-T+-|O)*&am0O
zYgO)}Ii(CTm+dAW`SJfxdhl+suRDK>$1={?YBB%$zpYbd|F>ywdQ>E}<k;sQyvK#4
zH~%iZmRIwL``=!L{v9_2rvG1IdRXtg;&NZBRMF5oAJ62dos0CZ+FQ{!=k7eWm!1w^
z)%cvx<(&_FYxLb>o9;udH2s@rDr=)JZB7<Tm%ZO~U9IM@_v2}TTnBldmp=G;yKiy6
zpE6h1>7CyuzML{Goaw`HbFocDf5T>(`#jV)|N7+T?Y{1iZM{;vpV;pc_nfrExmOfa
zq289cuPWWJep-H7vSX%7#p}D4-A#}7-sX59zI=PypEadDziV_4$IbCh4gGp3pFi%Z
zt&#2)y?DLbXO0`MX5MG5_wv;G^I6;njxLuy=V{)PeD#<?|NgJ0>;L|9dADN;xA5~d
z>3^RI?UI>yOm=$7&D9%uZTB-@pL<#H)6b*)tF&+XSuf+<9(O~sA#U!b7j;+nN6Zy}
zHu>gj%f0TwAnnh;WP0v>vcp64?x(HMKfX8FeCj@>p}AT`Q?YFQF$K>_B^m~x+;nT1
z@8&9#lRjy0ml-@xUzhxA+se7e%g;WGzZ~{HRC=L+P0ibjU-myuzV>uFL&3q5Yb8pY
zx^3@$KYD$((*60RHJ?v>KgqdqPGOGmyua6l(%uxD-Dk0`xNPnD&-+)#h38#uD_EPk
zs&J#C*XhJ}m(I?)w^)-m=yB%dHFA&B*F8R;yzklN^=2Cqi!-16>N+*IZ1MyZPpc!~
zCd%6d<<E7F`J8*|pRc>--dYpIX^U^CzA1gcHer&4d)B3I_M5l0X#aa``&9I{^Xe+k
zt^ZwCXRcnesO!|Kg#ODx-&Hmr{MIY5s;S}BDlLX-Yi}M})qDGTdUrSP#A#CUl9y+k
z+<E8E6CSTqS}Ui^xZj`SrgG%CL0XaC0lR+pNh&YfPk}0gO!njDC)3V6iJ7+M9@7Cs
zStHM$|7Dj|+xJ>bUz)wm#jbzuwpsgo|F_NmYMrzF`<9$3LO<)iU8?ydb92SD^cDAB
zJlNe5ZMu6U(~%Y@lk6FXx=N4!c{Ankj{m3fCKg9~Yk7LEs(Yr;uzUV&XU|Ag@i}+8
z*w?@FORCXXZ2wP9Xh-5bHP1;Q-Lus^CaJuAup?Jm{mP5GJ0Iq)TXS1|^Yzd5x6A(S
z`FiNw#JP6$=2>Q^%QzmEUE3b5d@1NL<Bs1qEp;LSJ$9^_l>UEfOXue6%pWdKj;>w3
z@r}-wV1u$-7kwLEacz5=-OVWR{b$sz|BrI3c<bih^!}u|eofNnX+{iE=_->Y<h?6h
zMc@10kNYUDxAu?U<4G!>zZPx;mEf7|YiD0uoS9~nZPv<Yu`v5v{f^|_zOJb}wSTyq
z7<S1`Jomuyt;ObY<8wPoZ603oYP0*uS}%WX`#G(xPg)|}VyaD=<^P(;KNl-}byohW
zRejC$oVwbu-S58@SWDKFS>KyC>GZLtpkH@?D6W-SzA5LdSQ7(Z_MB4_*Iw@AY_-_u
zZ8rJYy5}Daxk07&YO}VRAv<b$xA(FxO}XZO!`Xk5if8+@8`bZ2KA*>I%i%ex<ipAp
zD;8wkmWsDeIelGpvz5NjYPXeh>VryIZ?-iFd7WCFanE3;^tsm_IX)e&+T17a2f6Kj
z_SJH8`PXG^#nW@uV!}Mzr}0`mzOl^a=QpKmPp4n7Q0{bZlkuH=CdEivWzr-G@3=t8
zsiBOLUxi;>*yK1jIrI7=yX7)hTh2zGx)~AR6<I0jZnF0H`npSP?CV~IWoNzK^0V^D
z(W3<+x6jzNHNF1q9l2xEgkzGPf4(d^oV8Xr+}?7Y{P!%kI7iLZ=B<&l!<{CtQ2`a#
z|3184tP~vJZ(}t34sUei?v*WDD-)~A^G<MYmp?!KwAti9`^Lkwmt5Ky$~<k&twU{1
zK~X8oUG6R2*4^8ys&}sb{WO)A0Rmi22XdFHcwW-;^77Kq*SEiLctV}QqsNauy}Yb0
z9G;-!d7t?@sB8eWiKx~#>axDT49eaTcfNhm;o3H(@YhCP%kBQB@5Y61**fQ;e*MuU
zJEb+&uUO=Knr+XA+mAU~{}syz@4I!N`@KH%gNcgAC(MzMUi;8}&yRyf%W8HuE{i<*
z=ihX9x4pHVlT`kugL^m!+T+)xzFlUZE4}o99Mg98xS7Yh<#oDi{CvA^uin6$@buT(
zKgF-K7(l`J&HI5_x~JX$IlH?0)=IhT&Gnq5@>UkqPF#|o`0SA{!<-AIvoDs<b%?CI
z$|x6@`?oL4C#~6K(`6z4F9D#oQ6$L9mkYu~S<d{hntz8knlWS2%p-<xjXV2q?-P0Z
zDAcvZTRU^kOqau3T<qd^bWVM}{nDO9BU|fK!Og*(QI)Qu^UvNtvCCJrBxkDIb>&O{
z9$R>w(u$lCedL+$Q-&-4oyqq;G0jR-b=SIjEr;v#(@RWd`I?L4ykzd)PP<@YT)F-h
zt4BxwLg&+Or)FlGx6a(B(0}SwgJ1j7x1m>~zHghkC2NXT(xGp!cCu=7X|7+fsIrV}
zrRPSUzgfzat*J71^_rDuUF+U+_@=kn<f{sek9Vg{76`9idA}f2^?%%i<GcO{2?-}<
zTwi22H|p)LdtRrse&)oL{s!5-EOM{M|FcqRldmdl44Vy#qGgASqD;IV-r$Qoa`f82
zNLI$0oi)bV*YlTdSu;s3^5mBrFQ4msPAb`82ufA#Ywz$z?|kx4QztGo<aNbqH_bhN
zb_-`*USp-R>z{wTLA9sdb)KB#J?qU4YaQdB+?pr5x#H_PVUM2md<(+X-9M!-H{;Ot
zowsk>Z1?xp()u2=a$7~HAakCy{Ler4zL@SYujTz7JMplneb(W_>(4&-&#(2ZJo>FH
z$w_gY{dVU6?+^KvZi^IUiTQrxdC%N!vwrN5_*dn<-s8cy8~4ROop{ZyaeLw3;QqWr
z!scHV)Or2MJO1;;>(7#ZKbn{1Ty5x=vlDGN{r`ic_BB!CCvU!iOi0b%#vmXq{OI87
z-J5gHip{I37IX}_>()Mv_s8!Ic1x~%Gdx%<nf>!@^8Db4n7GyJ?JRHFl;18?$>;w5
zOz-xYsr56Ct?Iq{Fhhf7zvXV`WV_gtr@lr@{#%{*=Tpx$2A?^5=iRsYW?9*~bmF%t
z^R>q>Z~xN6Qt29fdgq6I{xi2)><`=DduejTvuSZ(UQF0|`#Y#=SNQ4QJW1u{g3`~E
zjU)D~>UhQ*yyN4azL+w%>`6jNkvr?O>#MRK>Rq@0d@jjMbZX(>ow6q~C-Z??$A5~w
zzV6C&WC*vl>0Mvk-O<bWadTaJ;m`gk<FZ>>W@1y`UAouzYWb1N=F0<@%vzh7W>i}&
zbx}vV^<=TR!KbgRQ^G?GU$ltbsc}@_we{ozEslyWXIa<WlUTR;Z1uePr*CAg?&NH(
zC^=y`r_4u~;ogrF*8kR3a;c)BJTGoe$!6cCyts}@{%pi)-TIk3&%ghC{{N3zZ??G9
z75i07rK<!+iWa;MHvai~xA%@WJ=f;cI&m|It<!n3^ZPWW4XQB*AGrVbO)RvuwMr@a
z%sThS$-=XYGfdK*pWJObt)-k89pt0~YB`F3v+hy9|IaFDZbb6US-LSFFQh*aop$Nz
z2NO^VnH#lqW64>ux}OKqb97g-JXpB@Z2GlFHMh2}IR7`iEcwU!T^FBzFu9hmm-u1v
z|8=5yUXxU0eb#^z_ELq0`QPoE<lb)n5x9O?iR~gQ2W_tDpPwC>suLf*=d>+v(BsPQ
zjgwbSmpMLR^X=j%qVI#k0*@Y=+RAVyZqoTXOJ_6gcsxlj#(0A)!|~F*6Ic2D??0{p
zReUdY`#qR4+b-$hDJMV6U552Pz6LVrMDA_<z5b2cjy03Aw{NZd)~WlV_^$=Sw6%Ag
zHn(kEKF#j)yCk>XSf&I2V`d({slKgaPIb+h>kM;l?zZ&5e@D0>xIgpI&;9d{?4G|?
zsr+zQRLb%<Z&$DR%Cr9d<@Oyn1bmGr|Gc~6CJQ5|CC_8I+m<0-XVtvE=-|+Zz{sPu
z;U&orPdVl7To9~MVDoL(cTnSVcCLLp|NaQ+l{wXKb|qVEE-%%%zO~e9|L@mtq8>|p
zU9So%4=2q$bNl;zmmnt{=W}`f?-k!~&UzdMs_<Kv%ZUBlX?n`z(G0$sGL<e?fBQ2}
zePOLEv0B4uXjr{7x>d#V()Tl<hMDXXCq+=?{w#c-b|!Am%!%vu6JCW&&)m48>T=Us
zRngsdF0ZSseWe=yzcN_gvhNIA=bF1B%L=l;ZjINwEwi)i!N=Ro>lkfk3yMoGT4Z!%
z`NnOTVf|)jUrh{;I5k^1+$i&1)v}tMibeaDtSOpa?Y3_pLqpFm&N=l#t519cb%ST+
zX?kv)80)L8&79<<xT>k)ym;QN<p<UVuikP(DwDVTM##eG**zz|`x;G-+*y~Ma^~e%
zp&bvNgteU3n)&Oho88R4ednJ&-B=dP$?$N~t8(l7O2z<N->*H%vt4gVSJZUJGHnN$
zXx7GZtahVW(Vjv#p`Wg)HTw+ux&1ABl5eFORlZ|WJAd~;L-g^>I@&WY_hlYxn0Geg
z{-T3%@A<SnCw-|k1}Dh1hLekb{{396blLXy))oGf&!|jWue)yZE=_COmDirLX%?=z
z%W`nr(UV@Mw)s1*nZN1p`2{y-2xPA0+<Hy;x7nxeQ$Nlo&o?goR<)|B;mQ(=U0Y8k
zT-~;EiFN+10A)21rTr^<j{n;?;kN!erV#&!Yd-#J5#3{#cvSEE+t(J-sX6ubrc<-5
z<@@{B?tSDTaA2?gxu;9{rwQt<R?#za*={DeMD2LsmaR6Qa+W^5${3sE<g>Py|CQ=-
zK`y~a`Ml%le~Ka*ulL;9x~bssw)x*npO-C2@U~pv_rGn@Ca<vPt(6~}Q$cM3KO3Xj
z;`#P^o6P2`cwRbq1d>3m<!~8)>D1R)zv7Yja`}T3Hs8LiWZgP@{+xrKwjMolO>l?m
zk?VZjPS&ikl3#_(GWM%%`sv^qsru73^|#G6_PC#2oSUy*R56}8vAO@*4?W(<$>Ps;
z+^=B>cpjl|^V9%T4u6=w|Iy6yX$2=&wqIUVsCDyY=aFMAE7oxB_}Sz)=l=Gmdf%_*
zq<zRe`X=q`y&pI3PVf5r_;Sp=%5+1+g|B$U*ktcFv90|0Meb8w>4R-=Z_Y{$4ZoAC
zyE5hNG6UajZ_n*2NKIV3fp>wYo6zZNISc*$%<6aiS`qte&CNtpOT}rw>#z3S)_-|u
z%KBB>ckj*KH!D>7u+O~RjrG3K!P#2tKml>cS6IdKl4EA!yJ;sD{Scp5B%!-0c*P1H
zrR)uFPdEKMUUs6#$GPU!{M+SE|D3iL58PU;^+os0?`2h1soTw7&bhKy@MEFjnW#;*
z>1Qu}ialQSUBB<iDf#USUCX$ZKAp-_{n492=uwiAw4Kk-BIdK3gEOvLOwGTR`}Ebs
z%aT8jF>SkIIJc~R=Ke*$ZoXQ6r0}9<`(*{wd0S^6T62E=I;Yh)UB4bt-mLuo+dlFA
zDlt(f+s-b%^ZC!6*jsNdDsne~+9WdsyjLGvY2wlQd{z7M!vCk^KbGbxrizBf7#CTo
z^ZuQ%{C?z_m$tKv-h7g=tNL-p!~UVwvR9zOm<L)IZ_|AxC@=0@=_+b&^N%(CTh_Uj
zo3DRl-Hu9}y?sjkYqMxmztjtyx)R2g)5GomwVfBwTU6&IV=Vr3@*D~89VPdf=IFIA
zH!cpedu$hfI)DB=R}Gf+Zw*AZEU38tYNx=2DFUb8ikByzkpF*n|FZRIpYHI^sYo$%
zKL7ee>VNC0D?ghVJ-GI<#8Ro3|KHxWrbn+d=lB{;F3ZfXe_0c8Yg@X;^XdG4OHV({
z++&)Vn!@$+neNr|*EWVSv&r3Ws;`-KdWL^-<@3#<Az`d3%imgE)2_eMCT<tul~`!W
zoF{w!<X!2HKM(op#BOeVn%xZ=I`Qo2R@@uM)EzGR@r5)usM1%P=~-i@->=ZH<w}L$
zt7|W+tNACXc;0UV4PCrkaK*%UCj0tVenm+--9_D?O8UsrHi^Qw7a7*v+A(!$_O_bW
zW%B#JWt_8H$huHKCoc3zQ$E+u*Uw{L-Zadff4u5->)aO?6(_G!c<}$2?&GbT+Mudy
zqM7=+$o;pg+2>wpT6{Z|so*_#@yaz^7H?;$^~lul@Lm?0`Q4Xsy?=z`!)ryL@r(cE
z#|wAI3g^{JRMc>TIyg>>GYqn2>;E0U4oZRHf0yVo1n;{ev64^5@<)cA^o3c!G(WO1
zLYw9Snc9iBtFmvt?A%jzowLMFr}yh@?~2?*T+`OwbUGf!D^Yu$(<DE_*7j@tx;Mv6
znQu>=B5>N??ksq~?%SlATPCZI7jKt!-@GOT)S!1;sPKdDvpQ(%B+S#3^Xb$3`?_>)
zb$M$=?5g6{^eo(x8P>1RaA@t-$6kIts)~ATk0x<xX)y~uO3KP*+n^fbm>TMkRKw%R
zxmEmn{9N(0yUT=bZC~S`6(kewt);nsjgq2W+mn{N`bYNfHfv@GTc@wKenStRuJnXS
zBHo^ns!N}Sf=ZC-GVYPCp-Z~fIeOKkWTl)f4X`cUtQN3(4Hqa<3o=!8Hw9<C)!Aij
z{hc#l%hoxo8lsZ*)~`r<y=~*08|i0v_w@HFs%HNFoU5ae`t;XYlk6Ejk9(HH&Qo2#
zVaiW!u8f&;y)Lg&dGt7`gR}L<hMiO1h4->Pm>FI5@p0Vkas$>AcQ;Jso$Z}DW12_z
zn<-Z1a=MzUQ=a}>o1?pSL1vgy@1_0wx%Ovt_4li5zT98@W|GRw+u&AxsZ|7%`QP%0
zZE4)<arS%-dAUXvkJ{uIY(BpeimRyCTqscU?TF--6MmKx4%;jM4Z>TMfQCGmSV``9
zd3)EliS?hqO}S%r{_mduf;JVU-``idu#1CwV|=ope%4Aqw!B~4PHC_D@q5WspSjy7
zmi|<oGClmFiT2FP{kf<7tIXFI?|QKiRQfM*ZU@b0t(x38TO<~wSJvvzke&!o@g#4u
zLgSqSzXNF0U0Yi_`SrE6Tl<zSTP7qgFMqY})alcjdU|S#ij7mIOo=@=ecCi3zIIO^
z9~DbW%T=}BlT=>*PHvs8;%W6{@4sKKw`5&ibuBzBOziAz^XFaK>lQ3nprG})=Fi9D
zTe7Zp-T(8<ymVgu%cavZuC5Y2+|J+cRX<6^)5-u`4&Pc9xjF62tE=1%1r}$P&#yZr
zoxjJ?$%(1{=jr%w)&I}VHeVjUzivz3-KzcHRXryy`Fk8xfoIN^v#t7Yh+E&qu3y@`
z@AEn9{(k#^JFb<+*Z(!$oOX6m;9|Egudc4X_T~JkQ(kI*b3F3$^gKO1zr4EI9kw>=
zZRme@xk?r1cD~-2{RI!5!q&%~-FRH?^rNHQGmX>v?(QyES66TMoo!}V_{c>_NXX*r
zm0(XFpPnt76Am(^ouB7hS!tPjd)w0R^>Hmu3s3ya-}kdE^YXHb8xo!UtX`^s?ECxc
z>&uIa-Ji_5xjEgq{$EYh)~ur&kIO}8SLN2#*#$52QPkJ(SM!~9VS9f3q)C&$ebV)w
zr1JLzWNh=;v0iD$6DLk|$n1W<?{~$IhwUp@tzufW`q8l-N$2BzvYL8&X$Kk@&&;#6
zE~<@*kvVVoTPJ;f?YH_j@ArPc_xPA}{*yN8ya)S!zkB@gxcu{0@wf+nKA(Sn*8IK-
z2b0r60rR^h!uor^2)(|xmU)5ye7ilL&sn!PDYo&;pKIo~i&%ZP_`L1(=kx338m`Ax
z_bNKK1w=)09cW-=u6V!q`-zh$6F)sU84?y|lyX8KuJY;B6MyGgl|FjA{r<AT$H#o;
z+s!RLXW71J(V`PSr|Cw2I>;`6VNd1f{LfXYo|Eb}#2ogWq>?#%W$<#Nj0+0qEFSYz
zeS33qf}(RrXXnFX()lc_wC~APy-?g$_V(15mzOKQ-As?%U8ZY(r+}GZe(g8O+2;Ar
zCVA^6zPq!tN5ZhFtE(&O;njUJtjqN#tNA{9xBI;s2a{agkHoyaU&EI9&GiZk6Pv8&
zn{;(mXv^WmW4+Sux3}faG)m=~U-wIM=FFLUzFvzK5f$B7`8h3b-_JBHEv<@wKcD}6
z)U983Q|Hmm^!ZPxhQ}?e|NrmIEYt3XhucAM`EJ*1J%)c@mfHu0gdCC1-*a$P=;|MD
z^Xt7|Ut9b0(`o(vmyNw9sr;P~0h!b&EHvDlem?E;GGB%{HJ?u2seC^7%bT0RMh}*U
zz}5mZ_}l$tX}IifKX=k3p<CN>y=R-{{&=(bJV*QCIaRM#o|$Vsea{{nHJ=#=p3kr6
zJMir6?ERN_=<R-ER8&;N(bSN5xNYLxxo1PeV_6-h>&4EnC{$V<zJ8f&w^)z7{k%_~
zikA7#KDNYjGDlN`0*8i%M#9NSsv;sH1*bHZ7rfi~oT=b?Z286g_5VHMP0w2-9AMz$
z=C=HJMA*~UH}y~pCxc1O4S|N!`upcx_OtF~WM(TkYkGac%$Yyu&I_ER^0FONsH$Yn
z21!njtLp5x`_;j%x1(WY@bXP*XJ?)G8M-=*(IDvvhg*+C;Ol#)+1DmanR3MRdd%S^
zo|7{!EMT;>v}D+N&HuJ|uaqg1f~qQOgPQLwmvgErDhyY%kFz=+xLbbzZ1H*9<JZ<k
zzr47ZedETB6SK{8ZZJ56g@rwdC@U*lyDw|r$7i$i*Ol7H@CC=koqJ#ZzuGMK))D*v
zKmG4jz19Wcpy1%jUoV$izFxC=-{oRZ1O<R6fHT#X&#TfZzgMZgbm>waIhz^z`+l-D
z9P5>4{_tXPe^5k(M`EI4@v}3DmzVi&$-6s?UA|@lr?6VV^SR}lQceoB@yqkod_SP8
zw(Q&doWjD=|Fd5%@tpkQvHX9J>3Xpdn^HLCYd$zGQPLIRa=jKY+qzuOQGnsYQStaI
zS&P-q{622~&oMDk@#oK<6VrS8`_;|Nrb*}TnfUbd^v|zWuP?e+`8+Tt=FDM!`#;E`
zy^;@<ANE#%U$$aJ$H8WHVHU>}sb`D(?N0sp_!#8NlP6CuEPsD5Bs4TICWgl$Fi>#k
z&Yd=2E;utxeEd85Y*o(DqepY@?U^YaUo)|lTimVh>zkXCudR&+6>xgHUMRiW`#mmi
z*UM#5xmRo+@BM!7^p%ytJKyj7ed6TFhkHJsV}9Un|4Xp;_qV65;&BUtm-|h8xy^UB
z+21&DirC)>s$hd;lai7uKAlwe^z%!*w8V4Ir&HQr-rStL``xbYJ9lE(c%_c$ZolJn
zd6{qJtCh<;Iy)0TJUF;y`Eui;CnrwK%8rbT)QR0SWzwWcf9;;t{rz=ucX|G^Gc%pH
z=iQYsOk(-_>nn4@zdt{Z96R>u5V!t?U8UK_`eaY92wbc*Tk**8<IDy3DxViVof>Xb
z{LBYb%DDB(c>ezOw(`kDcLoX5ERnajw?8l6|6BI%uF}WbZs#p7etyp8!vW^se}AfY
zPWl(!(lqNQ%On*~YYk1!g?sk=ao~Sky)pH)m|5N(4=*pTl)YP|vahdOx@1Yqk|j${
zl$_qP$7bKJSK5;%Pd<6_q@tmrVNtE4BV+ONb4w#Pr-71<Y4){%s3@<^X=higS(9>i
zSLu>v%ie7&lL`qx_Vnq~iPNUJxw)}Po9CsRo~E0Uo<4cjEU(H+%hh3Pl@t{l)6ULf
z+>m$I>d-?2-&rP#nwp)fR%xZ3oy8iGzak(kOf1vb&TgLPWVMTXDvOPbj56-*F#P=d
z{Br;Kc7L9z+ppX@`_-$ftAGA_z24K`UtL{&`L!+AR;|){c7Fc(UteEmURuH_YgKY1
z_15Y#QF(d)prA`AYbRvA8gDOLv}R4uVLt1gM~{-~zHOdgc&G6A$6v44e}1`qe$o5A
z--{min#;XV{I>v9ft9=fPaT#T75q+=GS8FQey>W~dzucTf~ICCsALdSc9URx3m%e}
zwE{Kug68tvsk?FLp#du^t4ZFSAJ*>|t=gOXK+W^g!o}}wpQ?IJ`qDgY&K#ejqEBF#
zUE-g#`nDHn`auO{ol>xus^_KeN*qk{>V9QbeR*-v@VLz37Ea+6kzU)%-`|_~viIlD
zpJ!&7ayLwwB2xABm1}0ECa5Y_Q)}B-`}<l{an7YBo*|*3g_k_l89%&SK0hcdENu1J
zMT?X`MenZC*C)=LSyKJ|-Ilz&QY+$4zuA1=?)KXq$;rttZ*PAuH~;nSHI3UK%?aP<
z=jLYK+3_*%?&p*0^95KK)#ulIvirVUA5<lIdUDqO`l6_p_x@kcpLOrk<F9<#nQ>7G
zlzvUG$2cb?J^F6N<~gY(qxdq5=cJJ38Tqpes=j2b+xblD-{1H3m-p3HZ`!m;>Fxe+
zx3YO83>X^r|9Pr^r|R`uhAp|br%4*8ZAd#SWmWQm;Xtsz?NL|pSdq-_si&t+nKETV
z;^DRvwy&?RPk(l1=9bLMZ6_zI7r$IOeN*P;WhZPmZrm7C|M%;i>i2tfqPO+r-QD%J
z?9SI$SChZKy1H8?es`Jf_dCV>(_@N2&cFHmd(mND^AC55&l_f4(Wv`%dH%brw!PBk
z%l7QqbIrI`-m3haPhOs$cKA9KUER0#arU*pO5W{!E_bX?_P1>h-?qHFrw;Sm`&?e;
z`}6AhI#)+WMo^Y~HaoxXkDshZ-Yf;rOA8NknR`y!az8XAq~QPG@5PTg)i3R<{T+L*
zU*6tMXh*NSeO-U<-OA^4|GaRwSF*LOUH?nYD23za&!2aSPV36l|M{pBw`a!v|Np+*
zd^(~0<@NRc#eG&?)$jL;H#9Xdfl7p+AffH|>#P|#IXO$WS(m&BIMyc{t^Mb=IH*o|
z`SRsNhr;}r^1G!WVPSR4>%MPS;COL=|Nrm13=e|}TW5aTBOe|fX7up(etf6+{L_i<
zau2WX|MxYnwPliu=fwoj3c7E*zu&K)FRC3@aJTgORjYaP=K0ms*a)lnFsy3cd2+J)
z<#n;vV!BZw>Cw_=IS=ktzXzG}v;Xh3>C-nS9p#!dY0{2&yI!9-apJ+2%YKXZ?3t5!
zd0AjUfJ0Q2l=;01Wrm$QclO9wPC9br$cc@+YQ9`_*NNQ3(qMkSW^z{-SD)>-2)AA-
z*V$&d9)5n${(iroeqn**mzS5@HG`KKl)Z`Q>goa+J*W6wXUNs1OO{NSH&2f9+r7%?
zvSPYXN0v^HJ9RsM|JmvBb)LujWPd)@ua{ie=jH9ac-1PcPd)wi|18eVGJU+`ao^*e
z&*v>(vZMvngsJ=c%k@}HK)`{|=k3pbdV2ch<>majx3@iA+;8`2O0eI}+}8U&lExCo
zX*`{souzEut*xzl{(L$OiYf(;7q_-*i^ta(dU|@=d_H5$;^^S&%Gz)9iR1a4V!n3y
zIty+wod><<_Yw}b@h(}iq~Mrj`juHyzds(AH?I3r(bd(JaczyH{qLLSE#7Q6tn|FN
zxcKMo`~Qy3GR+2=Z2$MAKd4o6ZjPnTTr1HP^NMb7NOacQ^TCOYPv*pF{rzV^_`$*E
z?|-<JJtv9iHmZ0A$sT%G5D*a|F?D_XzfaQ{1$<_k=@#kO+1Z^qaf0KV<?}fTB3!KT
z^?yr0JZzU|b0|0}8qVTyVO#F)1sgUPMC>R?WMDZv+noQ^_4V>cjvQf;um4jBGH%Jz
zrL3KuofFHXO|wK84xX85+<fTJp$9)6_wzR>C@U-fII3SK*du3a6|ujrcJAXj=J|09
zESZ^^)iv|9*Y7c6P>K(at4#gybb36S!-Z9$s}=0*>=>Ht|NlIH@bh_l`4*>z3T9?z
z8Mn4%GO&F7`0?PCmBC;OKAqO*Z%DYZBG550F|pv=&2)B$3tO|VE0~&^c64`hHzYhb
z&^Ynm<jIqpckI{!_QLkOSOyl|s4X0qE?;imvSkYchttA<51&r!v$wXkf*QX~hxzSo
z7@EF(`J$kxsF(q2>9Q!h^{n_T8DIPLs>l7CH*Y#dMMX`RGe?Hu;HjzF&AzkESS>9r
zCzeMSznfEhj+2X<Tlsv{=dYml|711au9)jvGA}Ersi`qE{r|oHKQ|XQck{!;?e4cu
zpFGK#l$6x8Wy_X?Z*OjbykJoG=LaafIyyT!H*Ma`tnlK_&f*PeXJ<W_=q|VNYW(?m
zwjh5ee0_D*QA32w&+6rp6@TsCzYe#MvAgg;+I!NH_no_^cuiXJH}BS#%p=E-UtSlx
zJNDe%ibtI+jt5#eg_Wk}sn4rOn)@~-HMQ{h-14B%(4!X?I)BfKdK&-llXy{4(TVft
zpC1*Ef3m1sZ;|h8vp?Uq@4x%p^cAQ-mAQN_+kw^V_esq>#IYgiDA&6^pZ!4Di;Y*x
z#nF*5eNG`;@G_sohlg5uB#l}=eJZm1`{lCjw~O0yqo?UcADf~X%mC_iDb4n|qH1nF
z-Ce$R%2e&}qRYPKf-H>h_y4bZBC8X>Z_eg(R^3azr+<1nJzj{f{ripobIb2}y1KH0
zdWPQ9bRO3K|6ZSccXQrdt6k;q&waff56YO48xkDXW*vQbc{%fhIdjf*iE6*NzFt22
z+M0)7F8e>vum3I0Ew1-u<8e9Wjys>9&CVC%YOQ#^cKZyQN~7~OpLwpw6!WU8sxsUu
zJTCj>`SZ`GwAXJ)IVqI4|8Lo|v$K!K|Nj-PxA#lXy8ZvE7(iX7V?C10A?pub-<0aT
z%zysA<KNXiCyDriMqo1cM{m!Q&CJX!eU}rt`5`FKe>|D&-_hL-DqC*nMu*4On%>=2
zntX1KC8$*Rzw!NHe)~C{!s<qOcP#Sud~_=+D$>x>dUU@2-{&}A@3r1VMMfV#eq>m(
zY}vJ_<vN<0oeQ1YFKy3{e|3HR`KQz4&k6h6fO6c<U$0hIKJGPdc*Xkfd&+?ZMio`n
ziyM>OgMxxIw6$+1dx!}OA70?t%wX~B#bU-Adn%0?Hm9Fwd+^-;zhz-zp-JYYmPTgw
zciYxJdXyxlALo;nruKY(y&Z#iY)RnS=Q;QGSoYg~lVE1&d(y}*_W+bUHf=Kc6vHcJ
z!k{3+<+_ZIU&5e4eSXcPO`D8B-jJ*N@zA55!CkgAB<1_ncXxLiCm-WEXY<*|s`Qmh
zXsGDsw6jWPX4AH2U;lKc_`K_~tLNreP6ieIKPF9{yfS=!T*?|=X)_fWKKFa=DxR1A
zr9o<-Ns}g79X4y=leL;KckWqt`Pw62US76*wPNv#`2GL#Zg=(eKJBmnqikqsc&Ge+
z?UZTL-hI2}+Aa3+K{NlSQ`+k<tPa-)bqLh_=bd5Zw_(^2Uv}Qd-(OvX%hm4xOaJ;s
zvAfIe)ct<T;&{ODxD0c^>sh-@a&8!Ws`ahdx}0n6mMtbp$;pgL@7F*3r5nAC!Ntkx
z#@{r74Ymxo4HLLqot7+H=4O|FV?$$dzwNYte}5b2-Ld%f<)w2{l9Hj}?*E@Z-7UYr
z)OWU7OzqdJYofR71)Yp2KL6q2;hm5BtTS$J)AgRN_q0=e-h(BR{T>|^j|cgl>G<>c
z_4A(3tL_6;zVGhte*f`%>hW}4ZSC8SOF>n05+q>OtkL=O>C-Eo9;bx@wZFb3-rZF?
zW!f}1t$W{|yja}-EPVejQCC;ji7(qqHFb4Ue|&iO=Uw@JP;+SFW7C7dXVqAKFH%!k
zwQ|#{jgS3iI0}FoDZ=vd=fmS`j~bu1*?fKX#A_|-)!F5ZY63SUza8ecXIXXs^p3*E
zW!1H1WoDqJ)!SQJpFDfUv?1rF(Wg(Jl&<&4+0Jt97Bd7jMf&B>`<mYk(Z0ad>U8AD
zkt<d9{I*{N9u~~txBJl$9#`4AulBdwxvM!dF0KevPCGN>;GdtLC9O&{90d&0&PW6W
z2WMVcA?PUZ;rssob3u6}>xxF+?zd(jobda6%JY@ST3cI>96L5?=FFYXr)zjl5>bWZ
z#@DyDYCnDYH0REajpsgFue-U;yipp|3-_I@=KHO}ce-Bei~IZSfByUl8c*Za-!sAZ
zyv^k3{Jo+Lr}cK{yq8<RXBwK`Z}&^1{_pE}<>Tc%r<@iF@Jg91*s^5`D77!2UuR{+
zyKTjajQPL+ZCkhJlb4W?P{pH8b&Jnuj9=c|tbV`d^Vz)h{H;zgl~1RFMw&7%D75iP
zAKUqSUi9j!wQ3Q6J{;x;W!xQaHl1FvW(_FLo3^v@N(BT39eQxEd8S=$)suPY^J}Mp
zMkEpsw=rIQucW8f_xt_+`3s%fjY?hwfD-eirQVD$E-q$g2#+ab4Gj&Asd(6G^Y6#w
zn3|7AV@fW%`pmH~yd;0%_4@tiYQNuQ4`E+)db<Ad;N^Z>wf?AhPWq+}8i3w%e{RVo
zPlvD1I885IzO2mXGvBV(!{1+BTf6(--fH)2e^<@ak`NYluBxh1321y(@U)R#PQli8
z?w2oLqK;p_e7Uo?S5<@y)P~mE`9vrwDM`Y*OlPv1@1qZg`JZ2Omw$SQTYrt|xmk7P
zYl3dio&W#O^OTengStNz%Y0{xNj^SzSsK)$m}QcAsP_Ba<0}FeOPJ+IynOlc%xrW1
z74x=5ZayaI;N`{j^5x5|TGz7XtxS{3ii(O_n||---6KbjPMkN-&&!J|z&7{$)$sVI
zGm`rr&d%SLcwvEKOzG9oiT_>~9hWUHxS2YA;_TU{L8WG7`rM<L%jb$o9&Z(oJ8?68
z{@Fd1pFvI3jC*@5pU<n-16d8~Vr8%2JMG7h3b%f_vt6RvKj#1cDQ{Ks;s9tM=kNFX
z=htq(=QUZ)H)2bMU}$J4DA_kMvw!+@TEF;eXt-hGA(nM}zeVNkd@9E9@BaVa^)c0N
zx1O1AKi~S@j%HqIGlQHP273E`Br%+`dacuE@rdKunVE-ozu(6l;;(&gS84XzEGy7p
zv98FD*vTrMtDHSOPujhIv~%;<*VixaEKb+k|EEZ<>czqpTXyPqb$54mcHYcA@g<kl
z(ZNEdk0rWVh_8L|wfEb<K5Uma%D$#kR8(Z~_seC*A79tkpM7z0v7~Vt&+<7%r}Eyf
zU9m!eqv>u~`_!pZ1*N5T-`3n<dNt2_o#b}&_erOx>25W%yAc@}crovKp0=J|-x~Ap
zPbT{p-Ok-^SotZXsHo`MjpzTw<mLa*U1sP#X~};%Q2tx#cai4~0~^2GnH`0Xf4o|~
z{@ad({fqMd{P@TtWzw-@hXtrYU;s5DHf`E8!!X%x?W-N~vnKo7&1B=1`ta>`zWj@Y
zpsAA*jzX`MDxSaM?v!4SZE;ex`|+Te*>;Mv-~PJ472j^ATRv<No?%_C$1Sdx^58(@
zpU3k5e>k@1g2pOuZc4pV|Nk$0mBu8Mmjd9S{#C(SwC^l&o(q~Dl(kv|T0a2lfKzS#
z-vI8mfG$0})L8iOsQBk|*6-hZ{<|=sLxCepw!3rXzwdWety&dzJUBR*QA1CUO~KMq
zvZ262X6O26^IBS17*3r!m9<q+TAJG-HT7xU@sf+K;vcUB``7+%pQz&5r3Bg`0Sc%k
z8k;~%V?BSXa5TkgUC&x|*~`l-D>gcK?X)RVSaR0qtlnv-tJ}M0&z`93?{fU7=1x-a
zWEaNNZD?p1bTdp@Z{4r+r+<BY%^VOC!s1X_S-JNAwKd1Xg3oSCJly6X{qf_+t+&=h
zimzL=h-t$m-$^R6e?S8*leYMKd3!6*mS3`D3B!c>^Z9e`n=FjmTQxB|u{uMHtF>X5
zdE|<nY0r5j!GmDn@lnvm7^pY4_)nTVS$Ven6(8Qa8-sF>KQ;}`HVyX6UN-I8itzPv
z(c5wu6^xCA6*hGqI(Io3w6MP{46&aKG+{L9o2P|L-<pSE!6(BOr}=JQ8@>ISRb24R
zX|rZ=3EV7MxNKMM^Y@9>snJeOPIEykH)V5>k{W0<K;`Y`i;LZvKU@j+X9Sgk3Dv35
zXZ4ntisqVL%~DlW-FoY0q3w=Yvc9v;^Z5>}3SG@mt~*u5^S>>cIZNKJaBOC~zq5DN
zUF&SELs{1}xm%qko?5qMiwOf47uTzt#c8%%XL)U$b>37oIW#o1#Ys`Y>S|K?-BR&p
zHeRL&bBfQcyqXsjB(!1gq_xX7>6)2|HJm!-#qj9y<Lq9^1@-^`G5mQQ|DWl>yy|z1
z2|qqOykB_+G{4S{7Nsv4ynK9EwuLWztD~r>ShQ}LS5CU5q$I<W=g*lZ%$o-u?9Yy!
zK6B>DWxlh&WyP#qxpJbcOZKWuS*xyRt@_Z$EA18<8oD-HOG_(jYjAPy*;3P~xu#iK
z51SM^?o^-F-F_u&^RxY?xgL{LWZ!_(0N;XH2D#5QXP9OkS|)qd$9^s6fvl!&Ql^eK
z&y~&DrU@!H!INyByKiiKv~HP}uBs|4sJX$qA};-1r}{jGvb8y@BenCMui64ifVZ{p
zXI)#|-QS=7>&r`qlGoRGZ^#MsNf<KSsMXN)gstkdYM5n~%XMIuX?B)u{@q=s44^5a
zOWyjdpzbCEsIF=_E?+Ox@cmvj|3CZc@3QfwwcqOhlol2FeN*Z-a606{)FRl{ptOj?
zNK}_ePowMVzdcuPXKf8#eeK(}t)br0*;~7q%L65qRy1&@XeyoHQsz`ndH>{zisj?)
z?tkVzQDJ0aYWz@VCs*sR?{&>N&d|@#e#MAaZ0o*u^=hT<>W>Djtzk+^N=nP;Gnoqt
z3JPW(dstAhZL`$X3^&o)hBHMTY~TNP?e*GYW#Nmh&hEHwac>KMY(RhjLuO_skF3>}
zqgy4SQ+iif3JMBN+&mX-#6+_x^XI?+T3S<M!@zI<$AIDTa{uowL2Y)mzZgE8um309
zQ2YBEV};hkOP4P<wyj;zTfIH=LqkVL$BCEA5BR#cxVS{6<hX9VnZv*%X~dEtaqYv$
z<MP#2ds*4pnFAstIj?+AX>Qw>@3!7-_0?CUbx%)CWdKcx-MSUEK07EVh{40dgF!)C
z`}LH4_8?bXc5R&9(b3@{ZM5nFcx-S>=4FN%#_4^}XDy#!CzUb7t?KXB>&(1U^D;9t
z8U8#o-_MwEZB3-ZVKcvE?G-n3zO_Vw!l&q$Mu9NMJTb|uS#AaCImM<`D|2seJ6pU~
z@Rj7$>|+nwj@^!wmXF_4VaVX=={c|1ZU1&&X)}g`Cnp3KRNe;#E+T|dT6~VRT)c53
zVu9j}T~DV)SN#2Yy|HcWwDb1=Z5Y1aEth8ijU8XOaQN*+cR5A{eSQ7|#(y5Tth)W@
z&mWs?34;U%hTvsBj8dyU$yGiP{PX8e!?R%j-U;U_&Yn2o!H~D_XBvZ=&x{A9wVs}w
zf`Wn!Hs5X}>#p0Ld2}8qoNu05zE)fT<SP)qG;hWox9ia$zq!3B+r9jdXP>T#$rp9|
z!xDN4JQ@aChfbY3WivJQSOe2z`};lD&z?BJaNz!Z`G)$xuj3g&a|aHG#V!c{JZ~p2
zFMs|*%g*BGOa}S)Y!)Q0&h+r`*!;Uk%XIPmn3xzDhW|g$*E4{+(H8>drtf)wT)ut|
zxJ-LGJ$~B`wa0%=*DlL`v+MP`$F1UV7Vmy%bBlE!{VeC=;<BVRxD^zO9UyQ*bBSCY
z$mO{wUYt$~*(Q1F)TxGFb*aY`m=-&oeR0yP;DG}J7dJPfho2u?!i^1yap#TRadU8F
zm>qeL5u+y_8XC$_^6Cm_#)>H8RW}wSdK@|`9?x^2nct4%zzoCWJ?{)xeE<!OF_gWz
z!Fb^9_WSQDugx;eZfNV;v+VD~zR;U$N=iyaW`Petbb2eOrJZ_u+T1UHbxp6Bvk95p
z^~z+i(wlzzb@pSc$C1YhbJvQ30#|q;ICY-`wMc8e-Aq@n-5;2F)k4Nt>tIj9*#wrt
zV_SS5L$xd}zx6V~=aI>;jgq!!ySln;uA8haFxJ`&nrd0_uvX$;^?Ta|3!TpFSikjZ
z?y;&<r%yMw#T9(NU%&tQthZ&mpPTlWO3dFGD4YJJ_V?-de~-479iHrO_wr}$WiGqN
zQNHS;f`Su^d%@W?Q#@*wQ?`_g?AwaV4(B*3q}W4_87d1HAJPC72NRcx?C9+1=-8r|
zoV8T&fn;c`taTZSg1Y)~;XkIlqU}7AOb-2Wwhe7|1@g6D1TP4vnC)A?^=i`7r%&gt
zvQExj?AFU9rS+QO)2B}iKaSh~V?1zbs&?bAuA|4kOZ8;!t$x4v`s*LI-|swLSzvrE
zE$5}_zC5dwp)M{iPjZ%mgX@-PG6$o{{bw00R>uk-M=4)0tTScOd%W-1^AK=6C>>u6
zR=0##QZ)DA!GjLYqSq%XyEpub%L0#}IXpi%m+{1zGc2HCGY97ei~4^x3`NDojn702
z85tSXO<rDpaaBxQoZZAn&~(|9Y15`v)#bBHbiaK}T114!<V)G_%k%#|DgF2J`TXx8
zza<Z5T)tQN>LDoRpL4N-QsJd_7BYQlwm194m?Rr7U0gO*>b86|*UDLJ9-t(u#0FNR
zlzW2XotC!t-a6A|^?q}$rnbvfab(yrUfEH8byX;X&94`W9ZoYUUSaQdn^9pSC@45(
z$`poxhzJgciVBN>_?u}4b@?{mGlb)N3qDWs*85mBw^!QSEP8X`+w_nfl2h~5gXXF$
zDJfl=297O{-3d8I*Q{Ne8}=}bCDd2@W>0kw!|4|<(^zcX4hRbgIk~#BUJz)RyXutL
zW6NX3z1Pj^?rzVYuRi6~{r&ab*#`IeH@YeR$+A*dxNY0ENYVMAOgE>B>kg=FUNZC8
zVa1-ff@2$x9C2B&@BEy?a~vOfUYR_OQkF4M>$dyz;jn|VMaGxb&)>g)XPM($y<IZ2
z7!(q);L<ahYr@4Ft<bA*{Kr;fAB(%NwE&sH|0XKCGZp-NI-MCb$;|ZCnT=u7=FK}F
zG;uR%=;_(%zRs62YHe#{`r3G9N8w|J8D_as4D0v((z>uA)>v1Uw_*3YUEB?opPw=6
zhHo^z`$!XH?eclxR4$l#>}5&Swk2kF4`umyZjI16yY0Qy)Eu_O>T@@)z5e218cY4{
z?fL8qIyz@I-`%=vmsG|Lq1#_(f=o!a21nr~J(G1ecKv#_nsJ6{wirWTV4#C@|F#`F
zEEX(mk4{NZxlkj~xX`(sZ(g=vcx&|KgABdfOQep+RlhY|u<sFH!qZbz9S)nFi_Y75
z^!2Vqix#D=UwDuiWIVj$RnyWsWjy!Fjb+k8LXW<dmX?-oUb)0=>MTJ8DcgIXKHq|c
zo6g>Nw=3nO(1GetD-TvoTlg{oRJu>xJQeK6%<w~p4uR%~8G@JleYHvt*&w<1X3r{%
z#jj4U$T61snSWygV?*fbu#M&K<JPB3KAe2<&wm%wxnfmYRxa#SUm|sFZ8ZA<(C8&Y
z-QU;o%ompX%QK{%nZbB~TYnG3m3)5I_y5X3E{0cyX;S&_rmL>@JXl}&bY@^_TF55J
z{Zc)rwm&@FUR|3Ta!hh+uG-|2!d6nu%00PT-Ts@0ZtLvmII+_STqAGM_Vn~TGuK)?
zBV%8Fi;dgo+Qt00Cn~!$fV##EudlCXPdMHu+fe@E$`+-yTlZe**;jasfkAnK(Y(tg
zyH*J%NY>t5xBHzHXms^}ulZdT6CNc{@2;`!UaqWF$%m-u%d0Hg4_|zhJG1wF?yorM
z`PNZa=LoKlye>64`2UyX_KY8XJnpZKegFFP>z_S63(o%F>G`e=D%sZsfJ^qbO@cXh
zORvYexVSKENIx&fpyoS^<v=4dd&9HP2w&}0J?WDB&G=^Y$feAh15Rr0kdo+$iP0h3
z#i!O~y3P=MB`KN+YB@82W-1vw`uo3!R7r+ry1hvMvcUM)Ny(nB4a;uZ%JQ|R%Pwy^
znv^LPC)G3IQboMm)H#7S%}QPbFf3ZUm{CJRBjcOS;_q_~-iW&OWyhTQzH6TYp2S=(
zvFPd8A`i}&OJWTUE$@@Hu4-BtySt2WL;ig`1}0|a#x{mi9P4^cnK^qujJT{jY1RzQ
zqO_fN^X}xCZ{4~zBs4TIJbd{B-*b_6c6L90{uC4uX}Ov6?T@va%PoFzs#jVbcsVYY
zZSf{uQ14GXzDBSi@o-yZ?EG1?URAn1KR37WSAgIH&``Nl)8V(bw}YGRCLvQLpFg}3
z*1q)%^P<4Di7X3_9biq?J(d#s_^<TeFFIo0o8Rto<KMSz{c^7DpZiZgJ(RY2;{5sV
zXPW6;lAGb9cIfco&c425SzA{<@?9besb}9hff^8<ot^U@q#f-Njk_-4*n4qK{gDnq
z1`#narVZuq;}$GrI>)iCsJPhnwq$SduZ12@&J^s75fK-kJbU))F0;(?J9*}+s;X0_
zPrtl1JG`m6`RjY}iDi()cp@{&Xy&Z1(Z<`hetUJ5J7dPJ%#dx8QxldhUCN*nvEhKR
z?$<@#dQ4Kw%B6Y|1^)jyZqIOITdwqlJ-g23m{lJ-7Q9h(+p=ZHKJQr^I@k2>zUnQV
zi}~6YNAho9CLCwF;?bYSMLRWgbX4^9*RP$fR1{@7*N;!m#v>_7iII_!>#oZ!MsR3f
z(vxru?U%E)i9XkJ;3a?k4`v4A^m8nrg(eQ4wM>%(4&Kh+&wC|&n`F<tTwm?GE>FG`
z?7kZk9zJ>g{QK9u1ud0h^u*iwWEZ*j%gx)LAuzFc`oDM`P@0x9eOsLNM<Df!*=&PE
zrV9auX)^CGEMyMYzxFDJaPNloixx38m}XyN0L^YOf4Jx_&&cbzSWrVM+hC^8+EqEm
zGkvB^ojUixZV}6`ck;|DD=TlgX+58M>sFMPmsdwuSJ2eiCoKIS*_!*<w4=LrS)J$C
z)Zg==>ALALGhb&1hlcCdK}9vV*_v=`OQu6@!qpP(B^&Ps^8Ru4>)*A@iq~Vp=b+3?
z&508ya@}z`R|rnmE`Gw>Z{J<B=*$Yg(<e?aJh<pC&-&ok>-FqXTDL)2<Hr7adug*A
zh6T~v^BkNf?8v)owLV?4=k5Aq{Ml~&+m}raay=K8w>>&>_MGi+%B=mCZ$9dIL8<5+
zEQ?=!{&M&Geafn;tQjjZj8|Rgc{|Ik;p?^Nd`8~G9BF4~u{yZ8v|QgDt0x{97{~ys
zm3f^7T%LRhnD1ga|LWDNm0H_fpWImjZh|~9G5X~%`S*IyhxH2<FhII>>hmg?GU8g(
z%x2FHT_q`+<GcEKU)i06jS;ha^+YYBcgC#S7T4r4J8(DC#9~MfEHfN5RrYSjV?Ktm
zcXv3!3*SsamcPBd{ru~BtlVM@1s@(bGJqzV7{b=Ya2~j^F<Dbb=gY3+H(p)774V?K
z-bPlJA^YXN#jIJ!r8^dMDT3Ob#?G@3xq_;15cX3H1-0osc7J-i{k~E9IhhA9Zd=Ya
z(%S#;*Y*8OplMu&3Dc&r9XQe{9J#N?lEKl@ks<4a9;oPdU&;mQ$bedgf)j&1pb-u#
zJbwIm@cw=O?YGC?mL;3bHk^IdVy@r%S7xiPc3sUnnqpMRCd4pr{r&p?wfE}(*D5J1
zclPw8JUcUUUV3O~=*nH-ZeO1#*gQ~cqXX8)zoaK|*-6^}mC34GhYlTLSg~#$+k$P|
z#4=`>yn6qh-yt<sl_7e2-q)&aN0UC<XxrG_d3|fKuQ<59{dqd1Ruo?orX3)^thJ?u
zfp=j<WTYg+zJ2%pzf9g3(a@%#XJR7K0P3;uKAd~|?KhU(rAwDCe`^bJgU9a$&}fx3
z%@R31O*i{>DoFf<<T9}ME!O1O89_lp`~Uxa&)Ct~$@pMq`aH&jzrVi5U6&FqXw!?`
z#Q+*WW()`jU^oD7<K`G|TQ2q8Ozs%C^Ti}J3rvFow#aMcAy6RPn(8iB*`gV|jDfdV
z<ky#%%m>`%YgtTqykA^c$jor?;6ZSQi^0X!m2tw{xx5aZo}JHsy4hQRVr1gtK$K1r
zII1$klaiA;xwt-AnH4|tVd(1aZfw)q`tR>=W`pW)IiOLSz3*;sPUjD}|M~2SE^ZDE
zh6U@^oh$YOC6pzG;DiG5bVtV=r&R^Qf`Svv4$hFXs{dEh#xKuj@+Hk!>v~+(OI3z9
zW!4P;{{3@sUcY_TEGY(XTVip)9aqEp_3NL{TD3}x!AI?KW{o-6vwEn#Ifz5Wd-mmk
z7N?fqDP+HJVfNca-FlyPUCY{9Rht{K$M*9XV}?JU=l^F>u($ueVtsnZj-7Yk&EHi5
z@_1%?2)IQKGFz#L3mTHG7cK-Ws1thG#LCTJvtMq#)Y%zXSzpqPXPixAsHv&33HIb&
zl>_e5V+}X)Q>RbQmNnM8-7_!OEi~%#t1Zi>=GxhPtyh0pVzsv}x8~f8C@qkUm!@ce
zTgIRebAjir2k+m%|G77C;<Ew^h7~JUGWOj&t*fHK0_t-vn`R~^Ai&^|oBMWU%(jk>
zjxCL)>-TnSUU%$sX~uECBg?+1TPZSbem}F*YnojJ`_*dm3(3E1d3JttJO8n#H-7TY
zGSPFZf@++VB&#mxwz#-%|FyH!)HJ1T^Ta)OZ-3zDO@Hx4y1#z%oFZ@u`eaSPL00jA
z$jHh|-=lTJ&cE+{aOTHVjvIYqJ0uh57T%GVaG~P%ijBHgIX*^hUTR`#`IF^-Yez@N
z98UK8Kkci2%6_PcecL3T^;d)cmFM%5`%`|*vG)5dwWjo`%8vbu*KCn!n6*l4+MoR^
zT}0O_AG_~1mwC;;4>!a0FMqza{H68BSIKty=cli}@$XXd3@?76mo`olu6nlZoX@n~
zY!Nsco#VP5SFL+Jt~xhb-`T~*<s5TZH~-SDVO_iY)eS5JHcP%~FP|Ln(e!DzTuS(K
zma6`_C2rSK)~`7-?WgV2TD{}T&l@l;vffyK)oGH~^v$oI9)Dq+xazq+e{|54o&5*?
zoV~%cdy=qG(~qvL5r596e~|S^1_ui$ww~;HTryj-_Wk1{Dr!t$r@r&f+uIcPRQu$w
zFL%Y5x8K=Yee&)`*8Lm!zQ`B~c`u$O)+;J!<MD4^?3J+43#ktT7VT1%DYw0Lz=7@W
zx71($Gagx*y?8ZKXZ}2wj%4{(c0Pf*1z(<jk*pUL6_4?GKIL9Y&F;=O?(Q<*O`m>&
zG-^P}=eh0VlP(3@_kB3*6%(o&_k5Gt+T_oh_p2@`Fn%~qYt<YzyDx7Y&+Len{Q3Xg
z_n7Ua7jLKN{E=o<-OXVd{^&LHeJiJxZ!DS&>nkR9oQv)M^@~waF)mU1z(e_M0&nz|
zo>HH)pra#so&B0*nMB#gevjs!ehDeoKmowAK|GB)fs5y}%;}qZo7ei^@sQama-{#@
z6;;XfQdUOm>5u<CF*Iv!DLm-AopHkMWbSphvhFZ13ltS<y&dQt`pd0qYVDy{?Rjap
zR%=J^o%YC?{pY*&`&I63mwJ97UPwZ$?9c2cYaqiOAnVTA=iT^vbNQ1a$AW^RT<&cv
zZIS;oqb6O0GdaCN@AtF${TI%vPo7g(UU>EW|96)AK2PR9S^utTlYJd)9nYP%b312W
zcwztO_u~z!E9Oh>dh^Vi<NW_w&h|M=v-dL|kBy3q(_8&+rWgNn{*=<0=S=ti{j6E@
zxJV5$Y$qrvIPtSzPJpSX{2f15fsED);~cG5M-D{GFFY^$HQHKs>((87^~V##d7RVl
z`-XZR4!fKu-ji~^>E4g2OM++3_O3YISMuVHWO}Tuy-vK-!+CFaWas{vaW=$S=L^p&
zwdAv_!dUaN|9#4?n!Rm;T(PPB@87eJ%S=;9R|hxjKoMPZt}5?Og3{krw(-+LEOx$o
zyPl)F*Y;WM(K7p0zy8l@Sio?~`%_{1+U4;_6uHXsb1SY*RA#yIOi=DY+UfOfX4P&>
zRx-7(?z%Kb=8$!nO7Y_;Gy78Uxo@I5545g5$3ES3#=0vHAM^WFhpxUid3Az(W$3?{
zFUQple*W0<Y4U{4o#4t)P|))IjkjevX3^cxb;P*CR*Tx(?|*E!e9@w&UAwHBnwp|Y
z|IU0=wp;V(Mj>?@8=3i+e=t{9R$B74Hy=3Q@TcEsW8jk#E*@T0jSUa~*PAZ0%5Zg!
zn0vaNyY6E{y4TBxxzRU%y$E62w*Tb2tKpN3(rommbIU87mvsKXYM=d&XTiE{eHUJ6
zFqoa+J$+r@+b3Uk^vOM5x3qZsya;Bd2=n)eR!{cv=9RC$H~Dgy-`tb)4}3`G_nNR7
zQjvq)Ui1I&_nIFM+mG*2GccGSXIo`Z^P@mWNGNL0v$^H>6t%UxJtwObK5P}A*EV(D
zoL8^1Knr3S-tGTicgQb!^H=lxHIomw^Bb4G3Tfk$t-2NWydp+Ve70F`(vJ@hbGNZ&
zO56W`|9trq%h+GbHx|F}vC8?!eLMcA{UrU}O?-2$0#iE=-j6(X;O6%W3r}BM-XkVw
z<Kc1BujueE!_s@wanEbAyZkeB8=Kc1NnL+PU*pA+d)-@Xuc~%|E6^<m(l&oQ%y0kU
zlDGcHkH_V=e=T|<0orx}+UT({$u(Nv^<3ffJAXMO--+xB$$4<QJnrbD_x&qhE9_#K
z^6BRPyUnLxe%W>JWx<Bm-RBuTaPu5V-1KPO=HT$)$2-%GtIG>b*!!;K+QT(#8SliM
zpIF=tt(2ZD*?BiF$4uJsd4_1){{8m*f8Tw-r*G%oyj81KS=9V604=muS8q4Xz9t|b
za6ol>%%jq`N5$jM9B5?j?C-aqeA-U0`{<!7D}xz1I5;fUt}i{QIz1pX)OCB_UBB0_
zWxOLJB`Yf{fBbsoey7;c)s^+&!Gq~f6&Us}P;hFG^*yl6e&hSaKY3>UEOxu|S$*HX
zuVq5T_6Z-?baTG56gX37q20CO+^w#=sY}1V$mi(nTC~eKzUtPs4-el?=6Gi)knSqD
z>`VX6>FKI^8^6BZkRR!kk!u#JS}rUkd2?0uDs8Qb|Id>-`9CgPx2#VujANU8(#`X)
zSLeSzR9*0O&vpN2$rC3psFwnbUv`}E+<f~jc)w3;>%>{Jy!z#AH$>><?p-tgNQ#l5
zjEs+$7ni4}r%uF%hP~C_H&uPj`t|+2zl)2r|I#T_rkKUNycnr7&3N11&HnT4(w~$r
z`5Upf%5-<xTP0J|sfmZ%1cil*k6kZVa-?Ox^^`{4IlF(<%RM?fU-#JZ0{++cD_Xhp
zcg=iQA9bW{`TOHlDZ-BT*B$2#+0~@5MEzp3TlG8Hy8HKU{J6fG<8-WZ!oe)BxrsM+
z#rU+}NVvpb=a;d#w@vga-?y+M0<V+v3Tuz9GYIm3V7qgUpVp^MoLjCk7IetByFE!|
z2*|YG`*3%K@y@$z-`oI=2R;|web+BGHa1${dCA=k5jt<TE&TKA>+72vli5LS_Fun*
zL`7Zy|NHy$&d$%Cp&NGG%_}M{*3{9l;a$CUd*Wd>&}g2R_MKUqZsz=Xn_qu8ZL?v{
z4Fk}+mGe6f9dgRb%HrVX&kuVV9v<%Q=Emmg>U!nsRY5T^x1^*;g}+xmee$H>$A`om
zGwFp37nZ!cvolg5aq~?LO-)4&jgAc)3@%;1eDd^ZWizvBPoAXQ$ukFSh$(q<LvZuX
zf9$_DZ8Fl**0z1EtEriIexB`(w`GS89eQzjIe*w{(G~ygQqs~U&7IqOG)YlSt?kH>
zBWd?9H#IS3WoLJGbSM}Y2sF%^CB?<f{qg&Ebt9umw{AsUx_ntvPw&|E>(kf8?!L4&
z`+D-fpBg$kPcAHUo-%#<<FehHJUl9jiVJV1Pn<vBzpTvc{oe0#`}Xbo^GJL@gMymc
zvdACRPfiH($yhYp%{zYo{`?<5Dy+)i-Ffu-`O~Kl3oJUiyB9B7)FfZ`gK_m$ub?2I
zhYuef|0DM=gp-rArKM%gwd)z)Q>RY-^ZWk)V`-ZOB_ukoW~G95ek@%&_v_V7qOGkm
z_x6eU2+Y0FeM0P?%<egUpT50a`olp@wtBhH&95(N?=Sfq)C!pd`J=+j&X;n3Uu{Tu
z_~QZ#P(Eg5W&QEv$Ak$J9PaO{eR*M_bF}{FC9G_0Mg<QXuC0weKF_v#&pF+wEh~;L
z<h)<^TX%lVC(q5EE7T@;IySQzrk<LTmKac2XbAE&JD*I(HqgcZqug62{u0}&zvqEE
zZEd{LZ8vk$?(S;)7x4I4Z}N)^3vIsNDQ@_Fq2=rA>zmWgO0C=VN~^288?<e$H`}18
zscD9BI^V&A2lxE_cKhtLOM9!!)6UO(yS4Q5-tYIa%}f-nt!MxI{Cx79IcGK`9tN%U
zoK-6Z+LqELotGiDK}1}9bH+s_BO{|(wR{YFtG*^3>yezdb`xlA=81{Q3>Gqc=YQ|}
z{cgAI!<fK8LC~7e?)kYjH8z)*`6_QZxos~iw^%?_)G7VCkKFev9`m06_4m)8iWdvp
zZ4{Ty+40~(!lTEJZ|<!&H=E5n`|Ppj^Xt$3`uduapFjObhv1SWOXf_CZgF0;=!I?7
z8N>A-4QF?H`zh-Ee6yxus`2ADzb}JE3YC<ePibgiSQDmw_3G7m$)@&ZzwT|#7N0qD
zW`vFyXeD1z(51}xnq^icudnr%zP=V16yy{cdDC$96(b{~h&>gCYLh!39&QJXM%C2T
zTGsq90PRrt{OoMR+AuZ$d2e<u-?QgV@JC@u$&(#|${f7Bs`~o<TeGikN<Pjf!^gfk
z?d+$jwP()waB*>Q@bRhX==A*gQ}gTF+v+>tH<!JQ>gww9@bg=ietup^c=+bT!)&Wo
zt%}%DpeWYe`sYs#XogEkSs7&O{rdm5l9G}&U$2J8l-*2S`|Hl9&z~!QzunHs!}Da4
zx1OSj$(7G*nA!O@RD4Xz+wqX?{MWMGbH(FpCVu)<1X{RdQ(=&pn0VuD*|+!i&wqP+
z`{wR)eeJL{4`yYr11|xPu`Fugl{S0u`0?Qz8<QJ<1+c$j-Xk7Y(YR_As0#&JxCNT?
zm^N)w;$b#EIhzxp74&bn-@m6*!VKC;cJ=Djl6QAzu3DufBqSuVZ7$Q_Utg7#m78zf
zids;2YK`E&+8+<w*F<b=0<F91ktn>A<uP&dL`aKgqFGRIu(PM9XSDv~DIdOnU%qC|
znxn;KyJdIZJ!gKu=I}J#Xi(tk>b{-Br6-mDe)WnK7HJQ*RenyplV`rztvBh;j>375
zSFT)n<k+!K_p0B2yjy-Bv^}<_=GAXY-S~Ynv(LIM_nSMzx?HcWuFj!0H99CrNPo`<
zrk5{Yrlh7$oG{@*Wm;XGou;PdnK_oi-A5nY)VjH4)|!=z7AYO;k$fyVM^#PDsQzEg
zr%#_aI5-~kS-(@z(mGXqFKg?j!pCmk-rv9f>cHA1OIqID-Ccay*SvDoXNE_Q9!2b{
zvD{tu_R|{4gvZBvL1UT=o!LPv9M|XGaBk;Iy}By2=HJifZ*FZpU0n0w!NDWPj#V*j
z4_xe4xi<0TrKKUEp%+&MtAo~=txxBZG-{b1UpMp3n;g3z518}ze!V7Uzj|l!^GOpX
zG=N4$Tyj;_)tAR^sVT6UJJ0^V@6~J9f`Wq&zq`Bp<cSjpu3w+d$joL?@*-fpT$fV1
zF|@^-yJW?Rj;Y$=+rAq4akd{WyptWHC%$#-)+y7beL5;0U+{A2^h<lIzZ-AZHGkd5
zZO4yIt<l|h`z=FAXsD_6^jWjIjwXG)l@uKueE8g4YsMKSnUmCWj7wf#YHeg@4+;yr
zmNcR4sAzb_w&mNl`6oCyFf4ZO|7JC9#*B{j>-G8Cn;o0kDqC{{_D?N-e5_YpMkv1S
z=hL`EY0&nO)6?~jzby+6415?_o{^!!%+9wY{ro)Lebb*jNwNF?r+C%s)rGt79$#Cf
zrKy?t{@z{&pP5EWZyNvo`*)^Qsn-8r*Y}?|b?VcXgV}5R=iAMja5=@u^!L9xnez32
z3|U!Ob9c1`AADO@t(6#^RFj{boxO7H+SJF#dQ;NVjnmJ~G2ZGjalQ*=pvy!0<&~Af
zo72zdy-uAYbb6ZZ=PwttI{W+i!&Xmy@F3yW*VoTE%g>4SYIpU8Pn|wt!hv^pcmMo$
zJ6}^%Q_<9P>XRoahYlUu5TUbVS=D*jGp9~{3i(~~`r6u~>HprO+h>2QaByI_c=4k8
zkGxm^-<Yn|Gn(nc!pa&L7Iy8?6%XYMtBYLgmobW6`F`~C$Bze}7FD+DWoBkBS-RA8
z_r14e!2tmeUM_rIw!6@3?(x6IQ>IR3y%Mf*?D+BR6~4>;=7NT!jvhUll9px`{r=*+
zZ0)zI1*cA){P?Q(`V7HkY%DAZX0ttgeN#_NP}I%)DQX!#*ZO}OXs|_a;^u1?w@r>(
z8}{eV9}ho2HC5H4#z#NJT(%IgxF>Q{P}yxm{r|de@9v)d_xJbB{q^>0lRN$9T3y_p
zAOGvy+t;DCVS#~+4iOPIIKQvt;^eekc`wZUx1Gd!qhIq)%eK6&KmO?X^YSaJtT*56
zIsNp|(;`#r>2v3vT^GCi#QF2f7c6L4v`A@=-}39Gz3sL#pzZ7j3Y%x0IN{ON-EDi_
z{Mx+bt=pC!Zs*?~@$lK}*WE{x7J5(D1FeoNd40|I@9igZAS1mlE_IU*9dcT4a3pDC
zg4yhs*Vam}T)Fb?BCV$%9v*i1Tb;e@>L$(j`1rK*^Um(8{e5F&GW+iG_sbS7YI^e~
zXVd1*kvof2#bZkZA0O{Oo?=wF>iMj^-8sFJ*9zGLUu{*twRrumU9<lG`@TQ!a>Cg(
zV?#qjP`Y2SLL+QV#Kl+7CF}M2#l_@a|JfPRS*dq(Pvz$8-$4zM3l{?1dL$g*-`hLe
z_oJxg^%wJHK>bdoPm|X~Z|}QxE9%eRznwijOXBy}mAt(4bcxoYb75}%TYvoc5!ULV
zHn}ryZ`H-s;re&-%-i{7lYW1Dn{$8P+)bN|uC0spPD)bJ)YP1}c9z@HXz!@Y>(|(V
z%H`^BZw}tSKmX!l_sKJ6yx98pDu>;|QqerO(z_qqJS6^p>kHqjvD_^zOw85Qm4%Hh
zX=8+hNrpgXW@h@9SsPwS?B%FElg8xa9Q}8VFZVy+m*3XEtoq4Q)jzvPYZ|*lXr`v+
z?y>~!Q_H<?v6j5J^Rwk$`L(MX^AAU9eJ*-2tBldmwrbv{ttUTfg2p8U1t(5U+kCTN
zXACI1zP!2ldC96R=gMBbd^v9cXz$nYJ=dNVO`JLvwBPRU@9*jg3JjA^y42LzeEj&a
zN7i~8XwCeeKQ%^1MjoD?AHVQyzL|3)XPeme>UB|DyFe!*BphH+^PgAq>i(0+19{sw
z*ZwX8@$V$*Z@+XY==}TFuU_^2e!e#Oc%Q>xlk2ZuWic@^&58{U4t9=<lRHqjTKwsi
z8+)tGYkz-By|E$j$kC%)PfFdJV>ai<glW$2YVN-~Q<a~)dVc7neXV+t`I9!=zMR66
z_w)Dn4IkzAAD+5#8b>-`SiHHk%)=|+W*u0ve^1j=(PryydR4Rk=Kqnu_$~Iw?_D=O
z6>)^d>OJhT-|rRBUcCR+j~LiUk6%<mf<o|ezuecUbIc+mBlmp2SM5++VVrQFAvrGJ
zXlBLlx7#<~%$adEt&K<WQR%&;jRHbKPIYzr(ocid?Ib29PMJFO<5~0j1t(RfSKNJE
zU~yw#t+m<gv)Q{ctCP3iwhhe|6cjvrZui|hv)o%xe(w6160`W#g_k9digwzF7JmHr
zk%gT-I5P6&uF}^M#%VsApXc4!(70=t)t^6qUR+#!T(j2y(k5MfZ$a2vfRB%lZ~rRP
z!L>^HNTfsJ7Y`k=_Zccj_WxYaZ=8ADU0z`3){Z}XhgZHcn(4Kid48?7r_CR;)&KP;
zeeD1LdB*qjyeRi`+545cyu7N<9=N60x9V>8%g8-F{qOcpe7Sy0UG%m&Hmsnv8%jz=
z_lm6M)_gu|9(VjrnRRGrsEkEHLo+)+)7OKKO0UP3-^rb;Ho5a)GpMm8W%ut#vXis3
zr<WI#6w~6K9v&aH$>~um`}_H)$CgQUi|HnPeRY+EjSX}@gpZo=)vH%QJGl=Xa&mHV
zlCY`xkoiO89N)r)3qe_7^G(nS=Z@~~&$se+CmrSb_Wu6+o%=pMiAa9E;`ZCL$H#h0
z-rkz};zh=vKYu`520@#dKA*Qgp0(8|=?F(nZEc~|+?*R58mDT9GZ~b<iTIjreLloe
zr{~Ar(??X)<UU(1KkPX7ZeKn3VHdw?x8EL%3%vN~+SL_unjx+Br>{ir{~#0n`H;<=
z`FEUiuazIK{l>rkU+cSTF-F4Ajh>zqXWTG#+NYCuxL?Nnk-TkJGyh+3)Vvy<<$eD$
z!6k~)^R>HnS&8Y#)tt!|5)#_+Y*u!Z?VpF6PV0eMWpCc(fR;TlF)?v)aV^@kNoe=o
zzLP1Sc2r#S%QhZKCr3xd?Ca}NKR!BI^Xa5I3maQdP*Bt9r-#m;?|;31|2<)ct?qVq
zc0Ka;^FSLuzI-W}SM$j;DLHxN+O?|o_VNvdkB@E4ysYNs?G2jw%3<W?=T}!!YML}j
zXqs*`+krKan?ajCo4EBB?AUQbxT)^n&*v>|Z9##74}aG6NSSs$JlwwhX7%*V{^!=c
zt@-~d_y&L8@z{C&`dclS7j5_^bMR^ZAKCq3J~44$pR;TJd>6cl>%sH&`}|9`=e_y)
zC%9~{G5_(2u@Ny>E&f<1yfvJ2y8e6lpZYh6*YkruuS%SJlU=xA&$(7pGwFASqfZpG
z8GUH8wmHsx`0&TqmQ5deVr61>`!wE`gyqe1))OaAoX53!8VehnkffyN`FXaswFja=
zy``q+=Dk(*k!!<1XL^7(;8#^yK_;+;MMPYJf`ksY^DmFyo_BVhY2qQ4Ns}hs*iop=
z&M&uQ(W0gg9}23zzB;-ha4`oz|MwVaIXS<qtgP;6{e^YiOF#aT@~`gc?Nv23ovIhR
zYfD9ZP>_(>?6pUqeQj)PT(WGL+3VTY7Q6EwI(&HVw%u&3tP^L=syfrKA^-k93mLxk
z^Gci9`PD2eW_*2pJvbo1Audi%QBe`JkFQ&ASI3GK8f(I|BO@c@oYj2PK+T)9tE)nv
zJb(W9{rmT(A&Gx}d|a|*$p%PE-y<?oQc+QH$&w|Yoehs3KlXpIdBV5+@2mPh+}<fr
zouO6NnYkl2?~di|mz%z;PP;er@Yc`MWxvKvzkIU&`+3#0|Ffq=1nqp;YTo(oTJKu<
zJ+A*=|My<9deWj33uibh7e0Nr^z^^*h?;7(>-&CbJ`-EE`PuvX-UpALfBg89a@CV9
z$?JYUxTfo3!51uYWA?hg>{WBQUz>L9+N&QNKBF{fe&-yC$-7M#$AZ?Q?MdE#d+)Z_
z9Vh&gH%8dpzW3zDojmiMJ9mnRiEaBD<Z{c!$*HMn-{m*W+Pj5c+Z|c>fcwMO+!;18
z;jdFZ{Qo1k|3N_3>eVkwA8MT6*>%V%?B99Ok`=c+ueXO^OupgDQ1j`6{)<iDT|c$`
zpVytYzkg}@xdqV|-BZ|llOEj{7ZMWn%`G)OohRY5<g>uF)&CE5%>OS0o_P6JWOP@1
z|B_!_%OxsruitYma>mq?;@{(3&9D0vO@3CftwZkYuj~Ju4@|mkAz?E|TYSxTtL@28
zxlaAJo498IqfhPk-}ArxJoynaX9gamw+@RJzjo>GJ@)L_km(|mOFk}TxN`HYh@6#6
z;WIm#t)?NYM<ze{)wgE7-A9(Zwoi+a#O@cdc5W+`>DW7I_5Xin*W}e7*KPk$Ds*SJ
z#idN8$@M(98U8didCaKc-t(39sr7ZndyTiB_up~i-&U;fk8Rn%pZ3RV{Y|H)w)|*1
zslVdp&IvcpJ+b!LA^R<?#_ChBW`XtTGqcV6>;GM@egtaVD=I3gsHzs8*4@6vBJz2O
zq-mDO&6_tt<0@%)b`+++DTPjSM?L;)ohFo$$1!o$D%qa@r(d(krk>uq@W{L43+!us
zN+#>x`*-l*m5|VnFAr~jd@8~_!IzKmnH0mW*#|^8-px9dc3%6>r`#arZp%xrpZoki
z-94jj&NS|Td0V^BiEWU-cl6N9dzZUq{lq)-gwLH)TK{<4CZj*y+lmj^&(xk@^>c@F
zG{c%kM~k@+*5%5ry6MHWan{`#XRe98n-c8Xcsp|0k|ixYJw0OQbMNh$S$@BE`qR_X
zi;s$i&zbmD#-?HdGdo{EXlSWhoIfZ=TwI>Gh%C=nJWzAwbLq~r^P_gpeP$YWmbI;Q
ztHhbwiu*c3{QI5LE(pxY3=a|(YFr=BP`y9qBuCievYoT;28TZHzIxozH_@!r)O6k}
zGlm%ZZ7(;lZ23`jwCA9x#PiZM9~{IEo^#S_s@itr_F7fJyH!Cs?91~%J8bw9pvD;X
zfs-fu>gFfQ10R%4dv$-meSFPF)&P4ZYjySZ*XwroojK$4?Ck9FW01bKpy0&KH^08m
zZBcvslKHToT}YAT+#N>dlb*gia$~yPSFRJQ_}{lp%T9S=`cv)Ol6;kwe}wIJlo|!k
zk8i3kFkrB2e7K`=!I3(jieGcX)*jZ^n6c=^#2L!7_`h1r`mtr<8&$Ubr<z4%tvph$
zw^et4uk#P*o?*Sk#>CZ6cmCserQCzpuTSsj;80Rh>gexJzp)|FXO;<P!<;!X=S$uD
zWFGceze@m}0`THu^F(F$PfsTMSIvr<K6R>L?XQxwv$I^o!o)zQ%V=t9f~p=>?`ai#
z)6&w^il3cX7{9*`v{P&S{(rlUZk;-9+NTT7{GhFhlO|09EoYXm|FiMBsrmgH<IBta
z)32|KJu}m|{mGLQz1?q&uC0xBPfJrXPCvIKe0?0q<ota<pM}(wKx)z=IZiHqbq%FM
zuj_la7thm}e!25zZ2M^$4~L8u&nrSbIltNJ{-4A1Kwjj;)&D;`_9{O1SoN#t=>8ck
zYqy9rJmI`-5hUAYovdrQyWC{=^|xio+S;G3#3ZDg-l$%FX;tL3`nsHmVB_lK`dw^O
zHvH^;ZhL$#A81NBC^lWG=z!|lx_NVKLU+u48lxw!s;Zijn)(s6_y@H9qv7zwgf}-f
zzPYn=@`eotR^{)`>?(cz<Jar;g*#(FJH3}KTh`XfEuOSBDspd?saSXGq)9@TE?<t^
znkBls?CqlP^>HF1A_hfIJo;oTodN;`%688^t+#v4O}USY7Ab)?HU9hi+i<4OkKeyH
zm%opjrXPRqvcLUV+wXUn4@^?^*3i>SJ3GtNVy>SIAA9w;H;I>)c!F9A;o<FvA12)2
zSDSNh&rD`^zKULEanKq7LBUM!ePP<``0jSDk2t)@dcE?~RgqI<&M17}ynV&Fvu(*D
z9Y0ywiq3L-M|}CeY+}eKLGS$8CzjOfF3jHFSabcg)-KtI1IyZ%3r;8#HT>I_ps&5~
z%>g;B@^z{^?k|n`cw61(yw#7tX)@gJRkTwWoY(DLc=fga9HZ%v!p#ntumAXR{+$-}
zWd#Laf0t~pKW+Cqpswujo%x?WdT6ga^j7EV-SYd1Ml%g&pLKC{HB31nU}S9k^4?x+
z&{ola_?Kk|S;Y+sAGs726@j{npv|rf+1J;7t%}Rd)xEsjU)|Vva!(JBRmqEn%P+s=
zME^Zs|L^cbW%rKm?&S0HY~S47J$?E7x>>VkNv(elT1x!p{{Hz(mZ%sR8Le2e=E+I*
z`A6QCeLld<KVjy~m!*G?ipTr-`SIP}mdpHV^<z+b%+HSxGz6$Nx%1E=r@9{x+b@4K
zgiKm>m;|R52L!&gd-d~5blq{qstHr}Pv}Zt{^0+`yFYGquY6OXs2cj9#KQLawX2pI
zD_5-#V&3rM_vg*xVjJ~;Wj#7&{^8lz=|QuKTRiLQCOX^Z9SfT4f9Jb@QvJ)QHQjZN
z?fLT##Z2|Cc>XNfJ^R&xhZ*<u-$k(b%x$dxQfrZw{c^|8^E2-Mwd7=#-*n%+x4Z7U
z{F!~L_F4&l(VV}Oi5WCOrLL{bz58z8&78D*dn(V&vz<L*f&c^P2sQ2Xdz$X%Jzu?k
z-=odv?UG+!TFMX-W|lv}qV`vbkDBnri4#F*GWpIn1DzA~fOAjUxjCN8{pK?4`1|cP
z=)k8vPTRlVua|FNm#b(1?REn#iT(WN^ZCg$XP#`AuR9T2e)s9lx=TwuXPiyjRr1m)
zAwj`xcJIl_>d9uaJNx_9Ra8KS8fa;0`F&K^Py&NXn;O5{wV!^euqXMr*!k~LrIxb4
z*F4@`9=~wi8iD^OERLO<;T&2hXlY%)AiXI6{MP2Wf7)C|>z0)NH#u*(`RJi7_4-fR
z{q2%kPP>L?YCbHSJLA*zpHI?lPW18LKbon{x>^2V=SiPAe~jNNayuNF@X(@eYsZfR
z*LvzdnEjgPescdSw{7}W%;jr)C;UiR_U}!V&cA=^e{?J@x$c|`($v%hWvHemCeVpL
z&(6-Sd^R(EhEZyldO@+(T+q>Npqa5%tF%BH@_l@G80>W4g!lLLefwi)V{_*-Ps5)-
zH4IA1%C^^k{Q4y%DS2|D^4H68ck;|xSy>rMvQ5lxrWgsz%loILEqiX+cH>6G`S;CD
zO&f0J964(06Swh?Mds1BcE9fHfTv13I_7x&Rr9fXXQ~)@F{bGF-?-yVhYokXym$C@
z#B#fpC9fvmdQ&IAc_yRq`kXSGW!3yi@9IA6ST8Rkc(CDQaDVmPi$_Y{A58sjS9R(8
zY5&E#OVfA0c##3xIO^o&bmGjJC&#4ozqF*R-~Z1FG|JN1*Y|98{yxxjgk|wF23_{N
zf4!65EPKAYWaa&qx(^Q=&l}D@`;7C(0=o#~RrkNWx%t@gU5VAr?fLQn@uk;i<Z~}w
zH7%^qSPW7O30jJ47Fe@`QrG^!Z}S~$^S3RZSH;z^W{nPL+*ZP-!eHyxt@AP$UcGu%
z{fpG$HeTijbIb2Ne7%1EvTfU@vCG$7Fn)Wb;Oo_J<_|}N{WlaocJuP~{&_aP&bh42
z%+b-&XKvM}m%E_><x#!hz*`wmW5^)U>959v`~Uxahi;Rusi}$BUuS#r<Vh73l@D*X
z-)E}W*0}B3S@Zj6E-ZA`)Yngca$+J28{4N#-ue?~%y?0H_i?|y-~WGqK~n;U+reEb
zkjuM|9{Th1^WJR}phZH*oc=TYw%=|f=kDdm3<wN#%*fC<+|HkVWkq1zbGLpu-_%r9
zW;UJ!IcCO%k6haL<@?U2DQjqSWL{ndIyoyiEX=K6&K9&z{olWTBBG*(6(15nCr2LW
z_m$}G?(XdGe}2hZA9TiKgpL^KaELp3=4z8WK~?rC?ezlE(%v;SHp$2PmL?zX`|<m?
zar!wKtFkv8keNX3bvv3s6BCBX$11MPnm*nA|G&SI=6Q3Lm&e4#$^HHH^|RHq*|TrI
zu@V%tw6>7pi&-Z;yGvEcJ>6_}bk24~-PLQ>97)@}^{bJ~EtiT4i@LhH+`T-4KRJE!
zN>_^u3QiQe@%CHVT2_}QPa<{1*3a8jRFb!S_MJO1T-@BZt2C8zl@t{Z8teGGob$aE
z1e)3aXZ#|rmCHbVh$XC`&e@I~JB}9L-Bqfrr1WUU_v_cLZ7P25_vi0lPfyRECtqu7
zYbzTYPwwd8*j4)a+vNG6X<Vh{{owZ1L@}F>M}$EqH|ox-{`qwJOtaiuhVRdxKAn7J
zMc|j0m)m)z&Hg<3x+Zq_wAbr)_emP3eR$NZ|L@83PSCEem(b<u5}?CL*!W~#l<HVo
z{_I(=Y-cymviO-n%83ca|89XCT$BXfM$#d3_(-Sl&bQlcN7;l&N1wj5)Z5eF|M=6Q
zi@VG7m-)@T^+?5Q_uX^5-|y=_{WR&zi;F61YLCixgVu~jZ_iWJ)9VAphTmMPtv{k3
zK77a{V{u{g+>+DNbU~{)Kx1yQ)@2*&{#H$zJb9*JGFv;JEa)uFV@Vqgil6ydmA~V;
zV!r7Aqi%grueNOW+?E!Wb^Cs09XfQ#X8q>W(_&$3qg)*v7(j<XX@{*jaQt|?q;Xos
zyp2gmxwhrq^*Y`s3yS#XKN;BhWD;&}N`3PDxv;G4*(068FYoRBeX{lk$g9saAOk_i
z17c%+Gc#8{&t84iD<MGvv}y7A=JR%kv$h6>hqvE;n|5<k>YiV(R>z$$*ck&l^rPm(
zL3Yql0ia{m+7BP>HNSV_)6>%*zkWUXus}da=+LF5-jX&I1)EEAZg1<|{eItUN#itw
ztScIzhR$SvyOSRt9{%}kc7B1?Tn3%UO)a45!N9=A%P*g_ipM<wtx2}9m|_3tA-~Pn
zE5Sd0{CM#Aar5Da59I&<V4rQ4d+7T0>7W^#>i2uamtXE&zg}N|?-!w;KYy~YvKl5I
z<AKZ$Cf(eWI?t4yoqaRtyhc9RQ^$IxXPRUR@wGR<c#-kx)2E6*ACE_FO5s#?>#?Z4
zuL4>mwYg{66i`N&Ijr1o<5W|#=XpKops;}Zi*+?LIzVR|IJfg{Og}Hj#KaV_G0C<2
z{Er9C{5#%kIz7X_-Y#wP%<cE8x}Tn&4$AvQMMa?FY*wsW_Y8D880XscOO~|QR(}Ic
zBQk)(0kqkqsJQs${r&bkckX<1V<YqS+rD{udZ2@_s=vK?cw(aRl4Z*_<=?ma_U^8C
zX{jlwX_U8pwyO8E3G?Tl-%<E@#hNu=_FUWje&6X$si!6E>ulWnWD=jAntJ1H*}L8E
z?ZBr6WMyYpzFa!}#_n=`(3byOTQWmJLgu_Vt`Axm?-$fMyCSCGAghR|=*L&9*B5;}
zD!%<|k&BDVIj+CIzb|+1mz%e~ollm_Aty&Cf8S3v(AlXUk4dkI+|1T~*fBPC?($na
z3GeUieRF50@Cx}O30tEWZ*0jFX5i%H)QR6G(=cO(#I~H9O@|+56h>!xgHHG?DKS~O
zawWr$U%x;T>}S%9_f~&TyS**<$+KrkMl&nk?R>7Gqm%OW)zzAxPp8Y+)kuK$SATnd
z-#;`|w3&_f5oq`DgYf;oM9<s*-}8K{)cTtoV%BAEPF!0X4H_ZS)Z}cKoxg9U`uv(n
zoWg1b<?ldePF{I+J{q)<(l+3MnjAZyjDo$r{Oq&Wu6YXz3R-60DY@*c`>kO0nl(>u
zWv_n<s*AF(uZyYs`Si!{-@)PG?fw3T)tYwgvQko3*3{B^1Uh!6*ZdyC0?<Nte!CwB
zgm*-)4ck@r)~m44(9qCuZB%?nhzJ8CBO`;0n_H6EY{m@{Iu3uQg*8~e-!u7oeEr-F
z8w@~)`uh0rEM2<v$B!Qs+s;azXXN1H`&Ol!e70Lm_tE2id$w21`+oeW=<4nUl|mbC
z=77%9T(W%m<azVXfllA}|L=F@kB9BiU+X~q<Xg<Snxg&w_J613@BbTS8wT1*{BZyO
zzxwf@!;#a^rycK;1)ZW4H#N8I#0d{j$1EyJ3UrFjY5n~y3$m}TW3c)0pc#~3b)&a^
zIAeSsbUMwV#fzi&_8ogD_ki>EtXsQEwb$?cCdJIg1L}k9`Sa=YybXK`hK7QZPagU9
z_O_<BHZ$nJt+?|B*Z$1D|EDeY_O_3Izu&LECL=BFT~%ducXxUE<z>F0642(~kH?_B
zB@3>;cZbGI8DI3S5=~Ga^537IpmX6aE_T1XKHeU*p_fnA>PhL_gU#$aU$5Kk;qAS+
z{QbQ%b1Wz4@Be%4^(@elFQ5oNbH-;u-KSfip45uK#U5T>NjEkmf(|wm5fiJr3p(n=
zF!Pd1)wee%PfS$)d42!Cr$1+x2ntU8cIy0j{uTV2_WgLo{chjyJpZW`kN2f-+_>?`
z(W4)a2>Ua9O`Ll9C8*(aB*h3M@Bx%pKnGx+HN6f>Vm@k@HRIQBoci^{!^3Oh_RgyO
z{A^?0-zo+zEv*-Kb{aoE-oJd!njUZcy;DGG)B4?x!>=Vly=}G0ouH%JKoNVV{C;gr
z@mbR|GmY6NpF9%mZ+rC3&CNeS2dEes8Ev@zmf_30yT7OGmj{<qGnuv*M^sf=ftG`S
z4simlBDS=&eDml3o4eIDm%o1f@iF|8iSK2Ty-UAO+qPxt6{g(ArV~%QV?-DO9c9l{
z-n|h%_uS5%XMg_ny*X!lZv32J^DFC_W*g-%P+&TB)amq6-7QmuoHCrFtM+ftoV6@-
z)}Mb5E^53oOYz{I^ZfC}_Ng^`&nNpYJbBXceeoNK)#2-(ZM&WKIj{WBkHSrxH&=c<
zDz2%m{rJhr$)H7NK69<6_DY*qZNI%GQy8?yrnk43!RFHmWe^St51%|?f`hX&vv&Bp
zC%ay+JN)<~18CE`arL(xP}$+M@WF?Nhe127LFrTc+erZ@4n@5Us@~HK($C58$=jXb
zHNWE!9W4zyne0Hg-4&)8R;60k<LhnR7EipoI(+ifsYg%i@9(+%^2u@ge}?&cKC*?!
zmWr+pU;i)ck*~l1^Mma2OV+OKjn3aY_3P{FpZ~tEmv4CVD2d_Ur|J6(4)dA^#K-$*
zX09y1^(vvL@Xn6HKVR3^AAMT%@q#n|N6-mtJ~NG4w`N~w`mo&oZ>OsFv=0Z_<w1o!
zX!T{<nHdMY_4j^R8#5W?>>~wBKk%ga9dUgQ+TOJP$5H)&l_8)WtmkC4jcI44K#7k*
zhL63jt`2niv#fQQLG7;+(6R(3X6DKd2iYsXUJd8u=U={Q)1-z5P?sY!FAr3Df)2_#
za{PF3bo6P^L0^YjxnI6}_iVZSU(Lnt{m=IQ|God>%M#G&|EEty^D3W7%GG`eT(x@j
z$y28mtzO;z_;`P{xWAYaN8!gp3z;w9zw?($9FB{Qmd@Hb>%f5n#kcAc5*pgt*tSIJ
zE?l_qx#h0)n{MV<$n+WC@??Kp@p9>OP+$Don#h)xmV#GTG>`R29-bN=cXIjsx~gl2
z)-rsvX3sX1;K|r}EC25-J+XcJ_GOsNTFCc!k%2|%_tSGlonrKagoV%l{N5#yR54*8
z=k$gpOH{!74WFExytn?|y?cH8_wO(NJ2ib?W!r`g1`J{AVotugx;is66Lg+><>zO$
zPj|e0`SQxOYej#*UY|T^(xFAJ-4WYzB*WutO}Sc|zI`hLZ4$qny$e*!HUutq3yg?3
zu`YJE%0smlfk%_ff9(Dm^Yh0K0dev2d5eC2FR{8C-loWN?BhA>_XS^G1Rm>`KffjO
z@`<x&SwW{G#NYfDe{{{o3<<-eBbB|;9EvRhkCK#hmrifc_cP??<osCozHE19ZtmA@
z^=c|A74qC193Sd7-Mw??&(`C$f4^RzX`If-)!Nk6#Rc-no12?wn&nEF&3@Zc`e@(x
zz3&an-^DyTH`hBcQIUa{m-qR5rB(qaj*n}9|GxkKT(9{(kK|-!Q16Yey?OrMH|e0W
zekEUBXaS`!J*}XJWt=~fvJXuckL>H|Idc4Xdt)P`-yDmFe@%8RUVM0VhV&eZZ>b+Y
zegqvFx@FroQ1g3R?(Jisg^}Cy<zHW0n|yJR>%H7wH|2{LFa9^tQ*04%;;BD*@}yB<
z2-v4K6S%f}$0Q^u6hAw|=*RD9QTT{OS63HQ%<}N?Oqej?z?GH3`!~gBXKSB4d6I>N
zB_J^H;N0?iCuisHJ1H6-!`R@pRLRI_(wQ?p4@yj93JMHtZEZnGbEa{6P+(x;Lp`uP
zP9V#YW=v?go%-PE^msKjwKmWY+oek#9UTwumfv?>fBo~d==_CYtM_l}KbY`f+3dVS
zZ_7aap<{ir-jJFV)O<=gDI~(hTL0s){D-e!Q=gui>f+`$Y1%Zm>}>7A!onU&<1~(S
zVIaF87H0fh6R|PrXaKw7IbVN&{;0LnR<GOD1=^Cd{CaM7et!C)7S1#C?d5yjoHH^s
z%J0{jSM8OHkB?7EN&>BE%f7ZI@y?FIdyh>`O%2Q5M6BESOzP0#!!xbR^={t0sUgDk
z>(?(2U*D(SZs$MWdOc3r!a~C0Zq=rQgG@1c;#{pwU%r&oeOWC3@%R1za`Ex^6)WRf
z1f1r`>rLn0e%tm+B1hrL4<A1|E}bMAdv4m<w8MGZ`Aa37csd_ENci*TPn#8w;*olX
zhmtXY@gN^P;+ElK|NHl^_?w#gUzg{1^z@|M+?4wK>yEv9_x||(d;az0d1nhhfBJM|
zSE;t5qM}aBjtO7BeEBSQ*u3<`1;w&=cRbzQ+1Z*K=gg5wJ2S)ZO5)tDp9}S`uZ!)z
zoA-R*?|0Vzfpcx+<KrVAf77Y7k&B9o%E-!+dKJ(jA*89PdH>XVJ;fuf%JLgI6<Y+7
zPQ1OhclL4l`Z-@;U!T7{xvaGG<CS3lhebOhb{0LYw9d=V*9V>dalBvt|8i!wQ>RXW
z7VF9Nv)9+xU%GUu&E(;;0*i_VjqDZQ@0RcXbmPv>;>y=+x3`rlop}5abn<ar{om40
zpFjU!oZQ;d^5KFr|HR3YK|Q$h!7Yz&E8G=&RJ1fe<H_^*4)2CFpj_|Nq4@pH=JP`0
z;>X|I+<gAFq5aP%!eMKpT2DW<%0JFxQSm_mbd2WSy?g(c%-*!g$VQI;{?Bve8Ch9Y
z$(z0h{J6Tl?&$9K`#>p8fBzpN&?N<+)#NrdbKc(G{`t85ze114zsuKzX@gqrYuB#*
z|8e89^Yi^9A|(9n|9+|L@TrLw;bOI#%h&2;cx53+;la$UQB%AsSL+=)8Wt$81X~N>
zcdbC!i9_+o^<4qkJ7n$a=7555srU4vw_C4Q{r>~18NXf$UK6#otM>P|g<-2DEsInb
z&YnFBI`c%vw(5#?ZuS3vfB$^!uRrpz;KA$Hr{CBAuU@r!^~T$8=h#$EvMhcE>LqW=
zyK5B`6cn*Bi51kq+q!k@lP6CeJbvsPA1`nJ`{sF2?_qP=S*8a!HYS4(LtMOaW#^3>
z5%Ve@ae|sSuU|ht(kXmqo~^XJynOzh=<Ru+6?j*!U3>KSv2$dk<mYE+A3toD2hIDO
zJLd<gd)9<$3#<7o*u8r;Xd&ePzxV%x4hmkhY*}03<71VprR%=$z7N_bFnP|LGs^uo
zhd}p(`1|ueKGyqq=ks}wzwiHV3+mYZ`SI~Z$*%P_`%6np|9oh-cUc;=vG}<kXg<To
zhbQ@XU+SA18^3)0YM6J&V$<f$j2mvh{qvB&{=v`Z^FjSv=Y;{FPAP-mT&tsx`|bO#
zW<7m!a`K;F*Y|^#ApH3GvvND5hNkAjlj`%Ie7znIYQqICc3XJ;HDkr&Uh{~}X}oOB
zjqdWbQ*v)_E4*8Jz3|f$Pq52Bo!0;S?RI|gqa&P~Hf_3)Ve;qG^!-mhJUq<9!xIo3
zeE7x1#hjd+3s<l12Ax<rRXhCC$K&$B@$u(xY)syJ{_f`UcA!(fIC*)WK5UnNc4DG3
zs8zW6=9y!?(vjP8W|qg^;&!}N@H>s|y5PIluccoFwCq@*q@?uW`}gTBn+3!zBM<w6
z3V`_6PHP{2d(^G}$^QRm`!$i9+m3dNGjB*eEe6^_%f`mm(ciy(-@bYO{?&mRD{>VN
z7`3#tKpn}*$jCQ0H?udGW?%d8SpNSE&_M9(_50_|&fCSg`)=R-`SKGdPL#2&n!?I0
zHevd7_mmVBtGRvOzLi;7S?&0EO#0{T`~Teh{rN9myy)WQ78Do9cK~$F3uyRllBzf8
z(8nFGR;}J}GY2$aDj?8sGsiA@?p#nn+$lbP@%HU*x5bJI3ZSl^le6>1tgRL@d~^KN
zwZqmV{QUG3H0ZGTX3xQd4}A8g&Yc6be}4S_J$dR>*W6s)g9i^DIeIkFMC#@1*PxN1
z8*j^MYHC)jTbK6k&d&JTlYU=|&M&-Q``xhgRmiryyTA0`x8BW5|MuplM6a8Xv2kWj
z&XScYJH_KF8bQMr&h31KU$2G>%gXj1ewgs&#6%Hs@y8!4ZtN^px3shbEfGsSJ#Asw
z>XRo<Jh-sXxuvab(){`UAt55$@7GyNo97)#Ft~8*mY1I&A82v&p%%`2+4*^S&#tTt
z)`{BE0oswXWs8ZFlvIG$)E8f?%yMoxczJOdr=NTB<m6-*7ncVmRxjVYNx8JdGw0qO
z$zHd^Q#6Au%HBvUT)1${)~y?JZW_f^J{7%J^Vt`)%je&Z<My8p%m4dO^H<}4^rp7k
z=FbZ(Hbm$c9dL~?`}6m&r<a%2<7OGfBS$@7g@D==hD*A-yBQ^Ht4tV}nVCVY@h{)M
zgN_S6b?Ovoy_QsO8|WUNEnBwyxt;7c)2J2HSz!R362I8Jf8NI*O|0CY?D65@VHbDz
z&kx(>>r!X0pFCs63G4TJm_aL1EJ|Np2^ZV?@1=kJsmcC!oCp4Xzwe)#y7ar@>Z_;L
z*Z<A-n`d*<_WPZ~-TM1FZst7u^?LpB9J6Gz*)vVE#Z*;Q_k6pReedraKXuT7x;uC7
z1fBTK<ghd-Fe>U)`utk9IqNw7Sb=&IM~)n+_@8rqT`Xv@KD~yQhX-^dVzP-8=(yMu
zr%x~5uwlZ42?C&0zVC<R+N)VhH*A=2-tKqL`<mz03^Eo444?yfg_d7Fd8}9Z=Du2M
z?eKL=cJBPy^O_NKFsG@h>6+NxVzbX41Ko<@t-trky4c;ILh-}T^KPJaT3}${9IMi<
zPGR*=e?Fft{`ACinoi`QkB^TV&OQse8|Tm=r)%qCU)%N{e+)YH`O1|mA7<bG!`856
zi^+p80-T0>e*XL^C@pObs^T42glIi_{CMfykPh#bKfYq1;(vma&jQdM?tr*Bzua8i
zlP6Dt8X*ma$;Uv$g};9NI_IHNYsmS<S1W;SvWudkBIsz52lLhLoGVW~duge+p#%>L
zJNx9>v$+rK{eG`|{d)bny1IK^tDBmdjK4|e%$4<-lbV|P@b&BNg9!)DpYQMM>sxPo
zKSBGs_;vfu5jvpZzVw<(8@czz7cN{l06GzL{=YBkTcUJ*eSNoV-CFtG-^)wM$VjN+
z%9Rj?UAuQnN=O_?F<Ka+b#qUpvA@m77W0**Z*FWfezfahn=}U(7wA~l9Xl*Q2Sne=
zGcUheD*pI*Kl_jAYg?Mn->v<Ax7};$q5Jpe%l~=69{>B!-J;XFEUc`8qN1!085tT4
zJ_q)2Xozrqsj^je>zSa^vSSJ8CY32uTB6oY3ky=z6Z{nI4J!YfvkF!mO!$;zaPaF>
z&fKi6QWmjqO;0>5sM!Dh)hnyl#dU^(<~x>MJ8+=k`}sFzyTAWFckbL~yLUH#Jerl*
zeBG)d_ruT9`kiyj3-%wZV4v)EV84$GkKZF5J-u&nr$0SCy)kO-l`B`~Y|Qod=Vy?W
zl?9zB5*m7RiRa`UFPF_eSFvt?Mc&%5*MIlz-!H%Xa%a%WFYofVM(upLZ1#>vUE1d?
z!UF;(eE)q=>>B8V`5!-jf*M4BZ@>PfcK)vTc~+0r*Id&!Otnx|ZJjVdz;jQhch~y$
z`hkIgD(dQwpFdYWX{M;BxG8J3z$5LHO!q>=$>OWT=YRN6@StRAw2u#uvU{J)`s<);
zaV9FeOPJ@$oIQIMRHQEr@(c@`_Wl0KZ{Nz;*w{XN{P^(4$H(b1<pwK1mA<}^dw18=
z)$4X~O+S6~-{0Sjm#0QGpPW5!o?KOR^~dkukLQ@p-}pY^(Ce6M+dqH)eE9tN^S}Q`
zMn;N=iEXO;TczedkLN*bd+vr)ujgCdp8xgDn;g)D+Ct~{9pCR&S3aE@&cVfX=wSiq
zoOa`FvN^Bk+SOX^|9Lk5#p~C}Pftx<vTT`<tnAyh9KX{!`Fq`*laiFCPMr!G_6J>d
z(A31l#l<zlF!`9|ZN9L3jC+2++kGR)Y+miRNKomwYW3>I^OLjUzt2feO$8MYzu)hF
z|99q%4T;Pju0DU){k-Y?<Q2`L`AI*5HAFziy$B0Cmz0>4?VkJZUtQ#F-<T~STB2E7
zrFz|(ofb~GuJ07n1nM3hc>M9f$BF{0xfXNJ8LzTw*)c<7s#l1Xs8lc8(jZO!IDMg;
zH*e<L*)eg~E-NEr<BJ(4f8M0qKkihYmvC=SrO#ZetJZfv|NH%3-OOxSV<Y3C!-q37
zGZQ~PI{M@H@9K7S9xiTf<_C}a?b8l6vF6;{bMyBN&|t;Mltmji3Qj)h5*8*VtnTNs
zGQ=qDj6|QD?Wsep+@0OspVv+N_Vn?)`}^mE<~mAWU#q;G_<ydx`YKQz4w~x&%@25b
za{m4MS43P~SYEz9PW*HI)2ZP*9=1u}*p$lc>FL=cVc2x*R#g7h2WMs)2Wm|Po#<}!
z|Ig<=KOS|Pn3@JcR_}stUU)V;-)?pB-yg^ApFM1sPrJCt6*S%e>Oq>AnpR%lC;jE<
zb0tGVLD0ovytB_f^PLm9HtbHG`M-bve*FG@`PQvde|~<heAKB9+JFR_maG5wdA_EW
z)}rm(ryp+TpFCyC5!>%~KKq|%tzb_|Nll&mcd|xX>%nSC2?-5(fv)a(d;A0+T~gyT
zSJX?E>u2A7+c!O3{q5Vg>07ut3k`!;hOn@*CK^b5`1*CK_?E+W)<kXwwPb&Pd;59a
z&2JwL^9zfMAJ1OD*DW(sQ`x=m$#eVvJIy(tK7M@o*4FHw_5a`hFMM;uaGC%7bJF>H
z4qo=Re;b?dch%~3kFG`Mr+$BTH)3sARdw~==P_HeL_ve#va+(EIxu~H?X}--byrt~
zf_BLrNZOcSAn^m#M+a4GLPA2|3x`fm*H8ca>}*L%Ny7Hq`zI9`OYnd^Xt+Yb*!Xhv
zN>dF(!-;oymw*0p*?)8XeY-7Dx}ZC$`fb0>*q-~jOM9Ke`s>xVRoVOh9&LScyjS{p
z?%BUqpnW;#=h=c9{r+}8TiV*#l9H2W8YD7-j!guO1ah@DZP;M2E${BB{QBR~Wp8h_
zhOK^juljxJ(Js*gFTY(}vqopq<jH%F_jPnAXltL|z3le){r_ui<oL6%uX}o!-(E#g
zu@ST+yz=w2%Jr7o+T8~ZIJ`FmwHf>5?aw_sJNxFAOkw*U2l)+Vp9S40_UH&F^uXte
zpHHW+T)len{{8c(Oc5z6F0KR}Ib867!D?<Fs6BODuKLVvC#$M288x-F=QFHZ&$k<%
znP>ZZzFm6KkID77TLqkW(z*I;OjM4fW^P!tcG<FLJ?0CYV!Uc<YC5{RotIxeIm`a}
zfrkZif^R?B2|feu=g*(_Usa0fM1WReR)2dl@zt`^5q^Grj*gCRZftzqH1GQL>*Bp`
z#g6ASTLhds7An5CYuUk5X(RXR_wSW!)=Wv6y<-V44-aVIYS-@FOTT8!_<hp9?$AVK
zcMWZA=~n?qRNp-~*gWHGnq1|Ri3|B$K^4n%gG^1)M@CORR;*aNc5lK|Z6}_k2j_o(
z@+4)Yj^r^=)qifTwWgL<(wiF_@87C?c6PSAmlqdX^TTNCc8PnSJzk&_QF04KL4yuI
z%sjkT3QU*mKkghJE^caS>MYwOARZDBAW&FXcq!@^Z_J5lL5h0pmx8of1e{{*p2W68
zR99F34ro_A!kWG{>i^S4pFm~_IB^v2^El+&ak}UB+hdU7ftUih{`PtE<RVXlI|ZPj
zq{%I_1*XsZ_pi>=qT}?m%P(bK34q6sS_GyudhXyXoGB?GA@NF}MZ)OQXMKpw4gpYU
z2pVJp^N&=jaGHb0hCspL#G!aj*>eY|pAU0{oaZ5^Z6HM*pFM=~pi&_I9CthIur(2e
zR~$KBynVZL<3_>77d!Uvx94hQ+J3wDZr<`edt|11xz^U&YE9*u>g5_AFTZc!KF}mn
ze0=<@*|Re<G8B}Qnyy_7v$D1Z?V{Dw<BN}vXE=5GbY_13^1XXy*M@-_Ie-8D-LhrN
zg3B*oym;ZTGQ`Nh;6VEPTDOV{i=RJ#{`m1DASlSm!2vu|BO)%Iy!|$42J`%R{?%8#
z5)&2w{rh(>mmw=V`{dcPpb6dF+uJJ7m%q56C?qWW^5x5g%a^y`xf2r<9Q^X_+oNfl
z10y0pXIg#z`gO(CtPriM6Yfm+o_P7?lFgfoR~!~d^2pd4#l*}Us3GDkd&>3T)1rrE
zyW6dJ6!j$S?Cd_xa^p~J+2P}HC|6>|`t`@(mOXy{+&w>EfA8MC8dJS;%%ppJd+()$
zhlh*T)zyJ^w9J?xF~?8+-@ku6JUj~mG(LR&dNgl)a99{<n>$0<?zvxIUl*2=0(oPx
zdq3z-&#o@6Lx&HqT)EP5@kP*lZ*sD--}1?QeSC)w9qQ=oJNEeFgeg;6ZodVcEwF2s
zm6Nmc#;CPRmMyz<?V4M0u`x3<GicVjt&Pnr|6Y&R(nDEWFI~Ru?C#Fq*x2YZ&qi|k
zY1j00bxlo87IyZ_*RHWOI4xYTYnRmY)2+uJD{5(RwK_REJ2NLHCjR*GgXgL4t|cCa
zd}Ge3Jp6VRlwLo3gA>S+r_NDc?Tx2Sd3~v}t=cQse%R5?jV&-RP((~@QHWM&M+azC
zUWAU=u3ft#HY6~1c6PeBxLmk;_2|ar<9j&Q_4V<6sj{`Rvr|z~VJJAQyL~~>%8vg2
z<4=ni6HKHwM(C)hsU;amc=-FPo0^I?A8fe%QpLbvLQ@mdpFe*<0iR*ADfP5iQ&W?{
zOrJOR_RjwGt12leNkdCZ$--jBs#RKFs%(p&ol&&2n>S^O$fU`WBe!G-MsLe`cwwRQ
zm+#+`jb?6$&@nMLUwk#|=H_&MPft&uc{Vf8+x_O5eDcZKW2+&Tz&Te|gEJAB*eBR}
z{P9PgQi18<JEJ6x(;gJ<eDU_}(T4>Ux9tueIs_Vul9ZHOvV3{6(M)hd#P8JU(=)9~
zS8d;<tE$?%YL!-qmZ;wJ?s@a%V)Vp$c|lhsUATMK*TaKDZ+iE?e|1y5RO{;MWGo6m
z%R$$Kfd>1I961sY9PFH(t(};d_~rZe#bK*K<<+Ajoi47f7q47#2?!AI_4Ng<P|3Yz
z^8Vw)lJ1<86qT~wbEi!c15G>xuD&Y4!{(zVT((<ws+Vh8np$XRXh~^l;?^ih2?+%&
zt6AT^m6=HK-n@C!xw2Z%(rk9`sZ*yG@;P@X8b<8Z7R^gP{;=S}{rmQyF3-siU%$F8
zzQ|K50V*VRFf2KgD>1{OQ0eE-pO7_>pl#%BZI4Q<PM$sc^x5qEwEzG9f{wyyX>SjX
ziUReob8~Z9SXdHFqzol^wrtyW>Go~!)Kt~m+w+(2+BFNZP`CYHg2URd#bK*|g6?0e
zth6lKJ=eDS+k&vwnwpvm*RA7Y*c#<qRAkiJ+NvSKr6b1u_3KyA9+oLnL`tk=Po6wE
zWy+L**x267FF`HqsHjto%<L;xtVsC(@2^C!+oMO1PMkZpY{`<AWy{oL_}D{3LqU_Z
z)26xA)Yug4I;*X#+uPR02HGb6=g%L|!r7%um*(8qkf@R0k0@5nl|=InH(bn6(a_+S
z>(_qx;ezGM*=L^x6<L1Eh1Z64N3A`Uv~j`Zm!NUr<(DVUp6%`DcTVy+7Z1;pl`A=y
z2AzC&clXX`v$6v;L@I3LzEs)Xy?ggYj#-J7?8c27uUxw}Y0@MoSJ&3<M(ftE=eM`F
z4+;u0n0<E2^y!ml&FVV*P(eeZqp^|EMvnh@zkK?a7Z;Z-S(30Z0<?8_|Ni+cEi9{6
zulDrtIFMli+OFa6&!3o>XkumtGBr0>xBA-~MP+5??xTl(e0&Tl72lSDHa`9M_&75=
zTiM8HQb$LJ;x6OqYAXwbryH*<_?@=okZ;FlZ>>9^s`+$#$PQj+&|d6cFPFc$y}f_l
zJh@4eCmTxj9yxO4!lg@12NM?T*(0+x%GcMI7u3I5e%UbhmdT&Le?L~(Nc6gy<=^vh
zabZc@Jaf_{A<&h3E-oyfDF}A<#TPR^eE)ttYwM&rbIx?@@AL5T;$mfGm9eXt@%Hxi
z%lGg1AAY!C)22z^zLmx3iAP38ZrQ&5@})~oNl8j_{q0kxh=3MI#>dO+P4Cu?-WCuU
zd2*q1`;QNY`9WLgLDh!@58JEPuP?qVSrMYu)zuZC1?tNrZH)?CeHAq17as2J;J`3-
z>Qot<3W42s{mRSDuU@_CT+<FN0l>t$c_7cvojcdW#AL#}d47=7L_)&CCQY8)d^AZ>
zM~7!=P-oow<x7^ZEWUVRie@lq@eHU?Fq?fb!(>N{-mKZPLB~IsnTf5wdTL|xaStD#
zCF|GoH~jcfaqIT&ljqN;KRVJ`Qc@xyC-*M)``H{bW6-{t$&(Fd`rPweaq{Fz&>e_x
zZWvCQH0i_l@9s;39u`;xtPELkHOtD%>coi?3l=SEQg-h%NIb+MBqU@o)90S)-A9ie
ztyr~6$<T0OQxj8Ca`MHOB{y=+V)Vp2J3B!GYDPvc-n==IwpmbI+<eOR7n77k^Bxtc
ziRRt!3E2@Fcj(Ze4;3~%JUk2)|L4_wT)N0<;RnBUe?L8b{^@^w-J8?Tt)t`b#jmva
zJM&Y)>{Gg;peDqTr_Ne;+8Pf&WMR<M)Rf?13knWyye#a)FJsZ*w|w#D&68VNSYq_V
zKYsi;W!khwVXGMpW}p4?{kyP)#EZ3A(mn@zrA)hy9C67plU7tz%-9+wB_&m``>u>_
zmB`vKZ(m<tQBlz;)23BLN0-03(ReXK#n5o#lP4)TX3~Lyff-w)7#1yB1iGT?NT)Dp
zWB-g95=KTw79}qxl>2T!D1P?m&!0KBxAE?c<98@1FyL!HeEVP4=9?>4u2eKMoVaO|
zk&%(np7XvgE-bdTw&Gvrfm%<~rMKUH>$~jW%Oh8(PmYiO^IzZoU+L|7*;c2IQ<r>y
zu<!XV%Ma7P|NpcVq}VAYBj}+SkBONXq&ig4*XM6O=ulK-l(lu%o;^08WS^O-dGqGY
zds{Dhd3!T<^!6T2+g#ay_4aM=gan1Gt+UR~HaE_{XOonav|`1I2L~FNK@-^DzLmXy
z&>XznPf$wA%iW#*-@kufzI|f?%>lT(vrn8j@yeAe2?i1!y}e5}ZJN~7#00uV6tooh
z&CSi3IXNm?T3yecr8zk}7w(K<VPUy&_wL)dEw5g`_I7e=Drd?rFE`hk$~F6}TUwf$
ztE+2DT3V9X?2q5SvoF}TZJJ^7u?<yUv+DkRo*(~1y0~)^mwBP(WUlS`Zd!Nr^c@`?
zpU?RqwQfz=>wxwGww2)d5Iy;zhh|4?<oNZbb2n_iUpM>5kBUo|FI$$r5(!&9^~@Qc
zf6wdxwO_dsVr6Y@S@?*BVdl)4pwW-1Uaon0dYzq}TefTg<+9*lVe7Itpn=^ZM~*xw
zvC`Dhc~X7<w_IgqC1}%JcsS_jkQ3+6yRQrZ9i$KvB4S<sPQ}u4=8+>Vn>KF-Ef#fK
zEI9kDTV0*q%a<=pN=pR=1sy{|MAA0Tto{A1@YNMfJ~^8gxpzCdx*lCv=<MR=2CA7H
z92jPs<$`WkefKU8ly7@_zJz^hZEGufaY6BYKx^#oGC=`>hN!jEQf9;J2GHcL8Bflw
zEuEdh>O#W8puM#pKYlDJEv?kvKWo-3P=y&77&v9h6agWjN4{k@-<B=&nd#)|$(glv
z){Pqx`MXzmEp=KN6c``he=uReu3fX%tkL;WWh>Urx;D(ay4u>(((=mn>%|`)IIi3A
zi0js^TOGZ<s(N~SlTU(L1G%}mTefWzl9%`Q^5RmP-1+58iH;ce?(+9($9g0+M7Uz~
zKuuE~9v%S+(CIE;)tFwtdd0!d52}AZd?*0*MnMZTLPA8=u3fuj>(+%~t8eZsR_7Mi
zJCd|fKv>w>*OwPGef#Qm$;k=mDd51)wF}K+JX^t`(bfejDx=m;`}ePIj-NVckAR`!
z#4~4n7)q>UJv}{3N=q39<mCGNmOC%LC?F$q=HcP?jZtfF<e24O(y_Ij+tb5SQ&V$e
zZ?*Z&n>STdRWIJWd9sn2Jz!->kF2%W;)@<tRaSiM%`;|5Xla9vX;w5go_y$#)4y-q
z_fMQM#igLYps}&>#n-C$v#;#gWAncI#?8&?ljqOxznHOP(<Y&{VXu8ZEnTt%Wbv)7
z*&A=a-LiG-#S0feBbVRrmdhI&8t(b`>-9Y?`{}1!<JLbv(8&Dd>(_@LD>!&~WSVv>
zfolU0QJA;nQ0@$ik{1G?sj>wN8bIso!41qFN#nL7M_gXLetmOmwm9hK`?YI%mj;0<
zCD7UzPEO7(+qVk~3O-zGlbw^ZWWxr5y>b2?9vo7=ZO-j{pi79}Z@GBnh>MrEci_qp
z78aI@$+3FVyVtGLyOU?GsHi9+CI%XAdY^1)Xb7s@=g&XCr}A@wh0Kl^y;rYZUC7$n
z(%SkE(n`(Rx~cwu9q8JoTeoj__Vuah>GfT^76uAAU*E4`w|V2<H!`!&uq;;d_4Rde
zcNZ2EG`zC?Ia*Dy<j~vO>d58$s#RKN(u`SIS?{e}?6(|zYsSRs)7`zjyUQPiB_}IO
zN=n|yF#}B>Y|FiEQS^kPz-s2C*|WQYR)P*z1?`my3v0WWp<-tzH~D1C$rR9z{sS+$
zcJ=h|#OR4LGc!MV_ADqiws-w{eO6Xh(8iS9TP8t4K^+|(3wG?7(bUA$CvWc;6eJY3
zdg`Z7MUaN<lI6=6Uw*k_-MVFa_s%_a$_wOcFE6hXCr>Kc+siXFA8a_8vS{tv-dVGx
zu4I{Va&oGusR>C-dxwOG%$zxMN6fm!?@5uMexAr*^<XE)DPF2xOFv~VpTGs`6UXGK
zJT&7`QByOj{FDMYT<ZDT?f2Cb6dKm7(Gk;&Iq~J?<-*-}HKux9$ui~T<(=W923it)
z>GEYxE-p~M4hU!nTditiGw0pAydF2@g9i_;2+^7}Y0`z)UzaRfmSi*&JYK@c$hha*
zt!#z}9kHO`;FW9Eq$DIXY)n2L6dc@q`|Yu$jR9d{ZaFzRlhyszR8?CK9dde8W<7J}
z%q`otJu0zUxpwW+J$vRnc#x1|COuirH|hGi*d<GsGFDt)YCQYwlI6=EKYxC_*Zkg#
zxjs=rK}~Lp52hFeMn`+QyR%pAt;@*kffr{-EGM+w7AW2sV<E#=_WmAtY4W>wY5)HG
zoMTbg)Y!<lE$8N;WxlgLJUkrMU*BB!w~B`i)O}pCbZKB<pkqyqjdA+9l=Jg!TiV+f
z2dylyknx#m#M*xNV2`A6M^6uEDmeW-r~%yA*a%v?oO5$it6}o73-|AzU*b9W#JO{6
zA0HicadTS~wwjZR3pDlo;zh=-Tem=~3%6~Xwqu6{Xw&toRg3oRo7ZpuZ-(z|vyU&A
z&tDXx<>~ML{MG99pgq7(PfuSQw)*7h)5*vCWI;>WrcG-LS_xWCcI=qjtXZ=_r=Nnx
z2~M5z`u66g@sV<OQ1sW#$qtPwC@?s9@ZgW1KOa7S{=N5`?etnurdQNsgAI+uoSWjM
zI)DC8X4qPQvuDr#`1Nbj{Q3QEixa=RxVR%mPfAK^hEXb)qM{;b`f2t0eX}MiyHA=q
z(=$I`e}2s;&Xp@y_Q+TY%|7b}S<3P5U7lN?Oy|iIr`1<Ky<9$DNL<`KJX~B@-EYaB
zJu>z6^<p{^3=ZMp;+~$KBBG)TL$vlzcDsH1_Qu<9j~qL8>Bfx{ox<uIyu7X}LqO+Q
z*jAY=T)6Paks|_<lAb|9LV|*VDJdxzZrwW7#w-2e?c1;Grl%N4aBy-iT(*o2yf(nl
z(D28vUrF0<gBB{d_sgZ;+L8%!;nE;aPft!yPfzEX<$AJy%O{^ZcMjZHTzEB$lZR(Y
z+4jv(igp(4zUwU8w!>jw<+GW^x6U6w4jK#2&)3(R{(8;rAW^PX^T71&x6i-4^Z(rX
zy05GApYC{m>iBW?sa~d65<#t}M@$p$|2TYLWAgDC7KMw-9U3mb1P!g%{wk5-V~<)p
zt)+#<Ea%37M@PFgwX_Z;ZQM}uGAOR%A?q^V*=;v-)RdH(X3UV_legn(a9e!v{Q3T;
zr>8S77tX$W_b$lW_wJqRleI3`8B<bLCL}26SXXBUx?&Kt;xRm~a%p+cD!0W46ATiz
z-+uY-otlx+CF#{CQ=Ha@1xH7F*VpeapPzes+s2ZYL3Y327=w-!?(FQmxAoqt)vFDq
zdhcoNS+PRn*RNj;pq1uolRJBQc)Gf~JNx?fUD)md@*Ak<EbwT~$tjl-7c5%D)a&M)
znyL!AW%ll_Qk|F`0^s%9vY`FLYHF8m-SUc$mw)^A?Y)yrn3<X1+}%C>Xty}%n5?3r
zA`Tv&C979=zj%>xC(j(xqNuoD#?8;aeBHXfGiQ8Wy?*`j-MeFX+ZSJdz4x-$)hyHZ
z`~TaS->XpeS_(>PUS3>RuU-WWDg>>RkdRPNP-s}OLgW3eoaEzus+N{BXU&oVokypu
ztE!;D(Cg+L6C;zhdFFuw4qKvhd3kv?rg||kGhcjJ^5Sb1<VZVEuWvH=yb^}V>i%+%
zc9*=Ipt<x>ky^0Ve#x0ZRh1WCf7KA_+Pp5{^T&^tkM}nR`$b1Z?JN6z@M}zMZ7n0{
zob<bQ?=qz8C~quzd8xSL-)r9O2fdA!f>xUCs5v2(clf}^$H!m3e!Y0<($=f1!$Iwk
z5G~QGSFft5s!p6gzu#-AlD)lreSLjYbo9m7UzaRdA|NSwvV~JPU}XsC4t5I}J`pa~
zu+>u^JxW@%XwizbYg7OK`}?BAs_e~;#)AnT{G9J@PUp|tKKsd&6d68tUtixNM~?;u
z2OmCbe*cZ{4c>Jdr%jjuaswwX@6nefK`}8s7c*RzUpCCYXLIP#A<*S9jyX9xkfV{h
zy1O;?^p@@02fB%FTkdVQf&zn=FJJ!n@#DgU3k|2Af;N(TJ0>D8?;jQ>mbN)EZU(r5
z0VR@0i<CIGCn}hliptB&Pw`USey51r(9m#(k6Khz6ljsexjB|5=H`><&GSo5RsH`p
z{y%6<1;gCAb0c)bq@<-+u3V{TW+t}#u3t%s320gKmMtR92M@@W-)X#?_x#uE^~<+z
zow{LzftQyTsIqBnY%D1$S#UAqLWaqk8ylN{{HRb;QYzRPGs8#i(xpokpH8Z8jL-pf
zVz=Mc)X?}4)*Z1ntf;tn<?7XsU#(t$>}AQK>#sN7ehUiXB}<orPO8k#)@D$f%sKgF
zi`UYmLoJ+Jwrm0AQXV$uwQE65ykEb6gBA~@ZLUNcSJ7R1*da7j6f~^y=f}q{-@aYC
z4?eC9v<1X#>7?)XszK*(%-grm?$ztpC(oQYl4G_wXeDS5>8$K^3wP}Tjb~lDd>J&X
z2lDv(_2+Nfy(-$dCq6SX^Udw;{S6JE`&w>pO69))_gy(?q3+3(Cqdil_Evvi6tr@W
zuz#qQ=*g2OZ{(P*TD9uMn>V0t#DN12pf#YN8Snl3?HM{dJEu&WCL|~4myw~-&L^v6
zX*qM+G_gN_|L#33ur2@oxha~#7haZ>l$Bk&dDAmESU52;F(fq9F!Peim#V)4_vd=|
zZ<*3{dghiXb;g&1s><Z*{}h6jF+V8U*>IV8VuLZ8pT<-#Ha50~^UE9g?EaPBuD|%;
z+w)(O-rj%v|LAe{WXstVY`lJwvVS5a_ndQFxNsrEo4dQk8>UPVX)tbMUlX_Y*6+Wd
zsk-%g&!s0$3j(!Uctj;<8Sw0g(R=@(`NoZiU%!9v{Cv(DH19uI-5<0p{BYLRNz<nv
zKh(-yxchF#)~GXS#xGyKd~<(4Xc1+2xVYKu-pI{q4@<1}a_;05Rto?faTOmg&(0_F
z;FR|ICF|DpwY7m31a7$fHe+kl6fafC2s^0iGfZYn+dT8c2@ew~UahHJGmX<HO`qO=
zGe=Ebz5V{bZ`&C_=LsA*;2<O{+zIXn2L=T-c`aSEetrMBbAEID)In`OcXweKnK!kH
zF_Dp;Wo2end*@xa5KvWJeew0zds*9$967RZmn~=w#{MTCE6#Zs)y6J*74KVZJKN<(
zz101ic6DaYrS3fIK56;ml_}q`|EILqGt5`t&vw1$_oAuV;s0)b24}t)pCwC6Br-BH
z6A!oXTFCH~Sjie18fI*bQuCeFayRe!Wq*5BJ-xo4pPz$PwLUD`2^w%&x32HlF}J9w
zs1xVTxh=kEU|?|I*4FIIj117ixi@ceK7INmVVKmiJu2tlpP!&Kc<EA*lCm;r_bTYV
z4KuT8-@cXY+O;cS^;MtQX1$+26@eznFLQ2<-CYLi^mGU+@A&y_c4vP-XhAt>5!<69
zoV#}Kj@+In`}FBk2_7~nDX9}DPcGcNdGePpC1r1KxmHwI#Ky+%h|x<*O1iN*o&V;|
zn^UGt35bd5Ihv$sZay8fV64CX&tvWRQ@9md1P*r|e|+$5+5g-6{l|-sZHvG2|Fuvo
zSL>tsVaa@UW&2i}vL{bu0~Jh89shfvy}CDVazIOX0|N!GUcCxh!kc`Y57hiUYkt4y
z^2?Ol+j2pzSpx}BadmQ%>W#N$C04S%y}h8ZvRSjBhmV?=nqItd1Jq=G`R2`%rAu2+
zobd2b6Aq87RBdf-H8C;y09prj+1GsH{Q2jPbP8X5{ncWwA867eLPsZR3&-rU$5sX}
z2OZ&=w|(}%e|2~A%+of{oG?LPnqKUwS*F>b6&xHK9H8;!%P(gbBr<s|y<~0p^_=zl
zBROWEiI<8B3szQEiC#Cz0>GB`_T<escRZa2I&bk<j@jfXQ%tUG_W*f4-e`tNiG@u3
z?;i=5-tBxok2&GlnVAQkU-sI)d-vYZqT_wCzs;OQLDl<?2cUs*(1DGLii{R}&z(%z
zdOdghUAL4JmC(@86<4#~?Rd<0=<s1s;ZwLX2DGIL-1<s<eQj+_`Q1{`XnyfCpQNOu
z0IjJerlubcaqEL-0o>f!o<4oLWcl*Tw{Lsr<gD4=`}II0^O>2(?GGL#fUar*-9BS!
zX?d@6nWd$riHS*qiIk_O=fk_@_f>6e=gyla7Zn{Hxu?QVth@EYhXT+xDGiMSPm2;a
z-z?aD*Tu!<!^`FKjcR|D7#SIH@baoEDmH?4TO4lZ7nYFd@LH;*sMxq-g$5{nrKPE*
zrlvZV<%0%<xpNCT0)v8tJUu;azFr9i;TI)VC04RCXU<enQK{Gu+AjgZke*cZ${Jx%
zFbUl6QMuFB`0H2It5>fWI(mAJJpL#kF5Z6o?Xran85`c--u}46ij$Kw@yd!o3mLwB
zzwf>eijDR4^5WWlw@kXVwe`u9CjsH%?H4mtjE#k-pLR`2Q3+c;6?CcinKM4u)<inz
z<mm9VH-CJ5{PMMHZb3mpx3}dwFTV^r7_-Ms`R&`c61{F+U0o|yuKcL?#d-$AZ1a4%
z+uQSxCv6OfjO>hCuWn~IZ`w4m7(MaQ($afok#qgri;Im53kyLpwlRW(m-pz?qKOkG
z90(1Mb&ZaeR-4SZ`KFJjC+Fwq=hIJ5(@jZB<GZ<6@UVM){Qdng%$}fB1oF6q9;lGn
za5D$g6Kcr4y$y64adx&gTXW;IX=0!oCogwePW5svE;j!9^{b1!J2Pl){<_%R5nD0@
zpFVx+Gv7|OVfnnOE_V5v2^}4vgSIZdESWNGTG92`^2E(ILFxSVw%o_xZs&uVg753T
z@4j;V`sM4_LFXUu+`0429LveSepP`^!QD}y2pNd8vIb9tzq_;Zz{}PN+FDwBnnhQw
zS_N8>r0mvn;AKhD)~L?T&V}2zPhYS=p{cpKaQ9snc6R0j6DiP=22W4V6<4!Nq<DAk
z+^M3j4q8>ax#Z;~?%L=h0-Garrc9Xvnz&yRwz~Y++_?4XmX<R+h1E|P^wuAI{PDuo
zt55%0tF;I?rxsMb2CYAQ@YldlMWqFFNtA$u#EZN8R<C}|<gNkgcoqKhJd`UDpdk{W
zBi7@l%q^}5s(IeL$$9nqb!SfxXpBl$wl{3`(Y)=&uh(whRQ^70)~s0_eSOO|ZJKm)
zvijut^WU$H30@68_3Y83q+|W^{b$n-KP&*9JCXY_Wc5{07va_|ubLX0uV25uD6x`~
zlKN0#0}8Wq=RhS7=zi(<`+kGYp@f{tyf@e-J1eWBw>LE*fnjM-=jE3wMn;n!AMd|>
z;R0wAg{9?8-`Qq`M@7RQ6z$Z|)>gK%nl)*XP}=!<ybj69%3Q5Y3l=EkPtw!S@R*c4
zedfD&;FFBsr`&av>Sb#`+;}x>>5?TalO_or>y<t}N!2@G<&*iF^t1BvK&ziYtAn!F
z?=4$#Sn$!L@_lVQJnPcUl8$gNbaizxTzFZ+0NSv2;3fYnCMKpmzrTL{$~b@h(rty0
z-5B1K{jQe*Z+B|3F#n`iDske}sYPqo_I~(Kz$atzz_&!!vZ&?5hk`R{#-N3NplOi|
z6RE1|>dc&+BS{-K6g+g2;bVXK^5u%FS^1yVZQMBVXty|MmanSHDmOPb!$iu;+B$G$
zh(xcOlate)&drY=C5h=qxwyHpMXeP(a66A{xqaulHEUA-{rSn@;^Gnz73JmO(P1uW
z5pk3aWO}}k|N863b$==rEn2kU_FId&eo;|Td-AmwEn4K^<8vf!^TIGtmkYE)7jj@`
zxvXVj=IeLwo=pvpTex^J^U|P`KR!N|w6C)%EGz_VPcGXH8W&lxpyAS`pgtMPCdiIy
zXW3sHHdTKI9X7wGet(O=qnbbsk^2@qD_&jEeDK+8yQ6%~2gbQ`=Q7;Os%vO!Vrnox
z_xSPS$A9CNO$XJTkCGHPw<ju?m`qu=ObxV-u20q)v~(+K?KIbJvA}?U19L2kHMF!o
z@#%u<skw9K-kWx5&mJ2dHfB{-)tKV5rYh>{$tF@KPo8vKd~w38SzHcIPE365%u~Hw
zV`F8l=JvgMm9@-gX4A<OB^{lft}d=!yLNGKb2A@!SrQZ%*LV8qp#*~olP5Rd&0D^B
zaWm*_A!X&88xonlr|YHu{`OWUe%~BkX|oF9+J_Gx-fOvcZEZAYCwtmv$*EpPcN9Jb
zE$VxIZtjtzM>ke}PD@Ho-WZ{Cr{Zz%y|;SLQ=%&?EjMo5c#q}9(szHq-(S9XF>|lm
z;kVoG|C>53aA^?e$Y;<Y37wstplM#v>`eDjr=p@y7g%(U7+kt|vGHu0vYwuw*xCIJ
zZ0#a{mWSp^^|CcApI;|sao34|zWO}2dhs21-x)_4uy2>oId2m`HMd!`JpNzguVtL;
z-Q{aV8o=$tAHRPyFK;*Gt?T%H$y>j2yZo9L(h?E}5)2sTtG@*m3yOMTlM1BOCQX`D
zu={RHOUs2zmz?VB?LoIDJUcU!0o02N4Rx)lvEgfP-mt*{G&%|%Z4(j|Z9SQ?Xz9|{
zdGqA*pWd2hTMg>C=kNK*Hgo1ojj5pZ4PHwpojZ5VSjZA|V_{u<bhNj#GjsCszNgFW
z|7vQ7t@%)Aboc&!e>XR_uV23&IdY`n|KIPI@7?nY3llS&-TV99ZvMG`pvJD2R@dI@
z??Tejphc;bpU;|u=6_eLUj6vR#l>f4oA)0$;Lz0E%*n-dD8WELULMq$RZwVn_AJfH
z+FJj1=?c(#r0Vy3-AhVLK!bj|x^H{@f9k7>bsxR$R&|)y{Lk&LH;amjG_<u3Cv6M}
z3HhQgoX6MBy!&on)Y@<RCR#_E&F=m8udc*OcHh2z>0ABF%gz7({ma6_^5FUN^040(
zHgZ<h)_X5k?SEIe`>tPnyu6*A9Sa*<)qS4%+S_mUUd>`vP*8ZVW_^A_!hzeL{?0$9
za8AGd@WEGCS6_Vn6*Sh}DXbp1?SEaI$Me-oOE#F6{QUIvz)P<}hRG*c4#fZaB+g*>
z^NDc&C$5CIx3)5D*|zQ9aYJyG*zws}DDQAX&`OupS3h0f|8MH;yxqKe<IXR1ZqLZd
zTC{iX-Psi!e0-o1{MD<hGijU4ZMU0dU)zvzQAtccuIKQ>1zWdHJvmu@bM|$;Ns}gl
z=K5b=TKeYp_VdxEVgdpVNl8k)yu2bJBA`&cn4zMp+xzTU8XMa``|5J}t5@c#*ZuMB
zQ9SbDkAwmf(=ihzB__Lt*P^o5mgXGab#2?N%aYcYzm-&_S=(-&+kW@+Y)k9Aw$*9Y
zo8N6Q_RGFo_Ugv9UF(w9E*1$4S+b%-BDm$m=U^eG76E6Thu_(s#~o}Bi;mt{7yrDV
z;mNaSYUZ4G&i}RHxpePdpWkxO6~YIywk`_LxRAAVh7ahlNJ$x)CEK@8@9X1xc5ZHW
z-1_AkH%|QWr9?_v+E9Y$M~z*N+u}Rb+WPwZYr{a})v>X$pkdfMcVfJ}y=Ph!DsA1m
zwdnr#o*o`c@ArOR4>0q8cs4t~=;0yOuCA^dyUX=gu3Y)#Ip{e1n$_m-msyp*5;2I0
zn=^H4Y3z1Q9UT=lwX_h;OCD*PXEw9*U%GZpO`gZy#l@idTh6mHGmBj}yqH^l?_k>I
zk54B1U%GV3sjzTk4a4dXt(iutT;K24{|{xBsM|mP(xo6REv+YQ_Z=M_Uwo~4xAVE2
zk&%&`?)}@hPlI*{g@lNJHW9zwe*ak7=7j+opknd+{d#)_TU%Su>KD*Dob9)D@5e0-
z0!@N|&PZ6aXi>vMho9B&_p0lEf4O{q(X%s>pq)@#v!>SEo;OdftE<bR`rDeC2K~Ly
zLG}p=3%loL=jQ6}E_(|q9;Z$P?*gl@w?BCBpicC*o|(q!b@%dw_BmS3?E_8NY`?uR
zLTAR=w5H}}&@l0<D=V+dGKjJ4m=bAjbl7)cz@37&+vn@P88iG$0i86{_^@Zs+UV^}
z0RaID`ueOAvuDlf=<Q`~v%jyBl(bcWqe+2dJF{Scv@>Y$m6Wt};m#P)0)yyi@47lW
zyZQZ(9wqg-DXUHHeDo+OL`xKOHpttu%Xjbk*4Nuh^@4ZgUkUaPjEeGt?8`d*`fHNe
zY)~)<1_oxBNd1(*zG8*Oojh~9`TaL;M0|U9xBL9_$Ix9!paz)y|Bv!7zE)jZ6M6Vp
zuQaIZxL`rUuV0|SdAG%a`|tPP&2wLVIWRO7bk;Cvo2qfznFR|LfUc5n1s&S)_V)H<
z0}0T9@u2bPsZ*yudG_qo<j}+&{;|=~(x3+J@jlr#QCqq8->-K!JI2q=t*oTf#4WCO
zqVf2(Yhh(?ZXCRDK_D_R^2xJjLUM9--P^yc=44@-V_Pl8F!^N3-K7OHXU;4-`Q_=i
zx3`ZUY-VTd=<8Fnvy)qY9aOr4jwyeBe*W{1$K~ZkOJ2WzefVVwXwJ1y-oDRksgk9o
z<oxsPXVV^^nQ6Rb%a#v|<^N`cJV;7NXwZ$`b^)>~>g36jE-o$!+iz!PWhI@SrVHAc
z{>das!fryT)m#k`u5G!u+s>v<p3kk>y>avA$>#SenEQ`Adw6inoH-M80N%TIdADxg
zwk&?e!^_LN=fffH3=^qiz0&R;9vpG&#pj<t{^R3gPEJnAsXMP-x^yTye{bvc*Qyp4
z63queE9zFR>;!FF*|p0`uJVcCGQYW6_m4gQ{IJBT&_X8X?yjl-{?+{~>}EK3!?^ri
zOxe3Tn(t>HJm}abV|nOcGdt*LE_?g=Jv}^uMODJAtgH(IG}c)QsMNmmSUNR`A?y9U
zy$6;)WWV$~z={)m5*}zDrJ%UD`ug|r|DMJ(R6J}IXAqnq#NsHx;wZuWZV$&XiIPW0
zID6cb&z?Q|;%n8;bnk)!1Fxk*{l~#umCE1An9c58>OEbEgJr|boKIT<yN`la)}A<d
z^5FB&CmnUR-}d$K;n}-)uZp@l=%kIid7!gww{0_1QdS0y*u4H)^y!J`GQYWA_4W3)
zwzexmv`(Em1=@0+k&%(``r6u>@4N3mJu}nT#m#Nf+_}CA0tcRdUKqCe>B+@w*RGAw
z5t~=}O!7>cF+;!2Cl1ioM$l0OYq#HXS{iim)~!>wwq~zfvqlB7G+#+s`Qy(T(Avtb
zF0P*nPyBecdG@#|KYjZ2Q?Y`q3qLP!;j=T6t5&ZLTz&P))2E3>GxJU}c;x4=uW6_>
zmijy2vbe38onJ^q<VZ8Wokv~WKAD&oZ{9qq|M$Fp#mbcnckGyPv|BvB@b}~6{gda<
zKVSR(t~$Gw{qHx%pb@$YmoFb)8NA#rH5qi*3}}DlhYtnU*2P+D{r>Re<Yds`X}{gC
zjwMT$Xv@WihlBPCgSIY#`V=>BdOA5Z?TPC@?p#%6)o=gr$BMeP=jZ2xdWt*l=K0OH
zJG&-w^M#isHoxC&2JPelO%=_VGbb?EtJY5b{hrT!aqFjl`BHLiO{DVlslWf$mA$=n
zv{P6ebhJ%JfB$r2Z<W&0(hL(Rx#~BDj~+h;rTbG;G~=`6IpgO)d6M#{ZvUT1gS<Ny
zmL0b*Up{<XzW&UE2MRN1&V2Ipsi3@ky|#I{064`6?3?=Tj>a*GNwa6426Z}jm1x?{
z_kXu~$@SMO!E0WH!I$SKRqdVED{T(iNEI9!dK9!htoXdGxt2F*kgBVTi;J84V}%W9
z?N;RGw2RlSxkW`uWo?~x?ONERNs~52=t%Ie#psEH8mY(oWOd^9NUXnpdR6FZP=fjW
z?X8Qu`{L`bf4*4U|3qQN)htzA-QHuz+-&6d3kwS~Or*lr#dJC?1g!<Tbqh4s=(RLx
z-j-QYrntn#$%U<+x@3vUqQ#3pR@m&go0pWFY$(+$A}Y#wSXJ%m)2EA@q_4le>~H@!
zY|hhi3z_h@pq7COdovq5yJg7>f#2WXt7~e44jBvz3flM5_wS`&zkWTL?{3B<pIJ~K
zF#mk{+0qqPvn*GrnexrIt^Ovue(sMS6=7>44qCt8bNJ$7cTF9gJxU%cFTVtBP+!5b
zKr?t5Xve{u&F5ddc{4@7+-h!-<(pNZtJi6KookrP=IiSl5)uNsFLhqUqt3wV{Jgw|
z>E~oVeg4e&<WoTZapmco_2T#a>3XO*S<P2TS-E-EEUC&<_r+JPSaINOnXr`9spNiJ
zvsZ=MQ@!M>UMN~wTW4lwKD@Bd8I<MrCPm9vJYWRv-(slTacaA(xA*B4o3`G*`Sh}a
z>f_`6(~afM&$mCHKEJjtXr+p|`Si%mX|d(<_7Be$Y}GmO@$vEEkB^Q@NA;I1TDWD4
zh{L&zeb47rKS^D3DQL%)Zok!6gQBCkdA_fmyywd$Z}XD>O^}MOiDO4EV`ymTnHh%7
z&FuV(!d7eQ=zylGE?fw>b^CVYwj9ab<?oL@EC8*&SmfIM;b+Z}qemA8Xx!MC%nsU#
zaeA8Wo#OMhU%q@PxL^Cd@WTVguyrw>d3k!(-`-4YF4CIXb+lXjaoKLrxfsiqv8}&;
z`r+aB$W1AnWxHkf#_{vWx9CT2>v{DmOD|@}f#iPMWBY1<UwB#a<?GjpQ>L^WOi&Qu
z03BI3Y0@MEsoonoW<^Cs9^T%{N=i)APq(@)Ubt*o+t%#sL1AH_K_gdJS5OxU6zsL0
zo}N2m^tiaVR;*d`=h98_u+?8@vCrK;nOj`1CUO6q!pFxJhG>DBaEli=FI%?kw$=kP
zqX}hejkX6b_uH6!yf1qlsL7|Gr`PxHU7n@Fw8@i;ucq>{F>6ijYHVbbl9rxnQ~7Dd
z>)_zv^}9d|L2JI<OfNE%<4NDg%gZY$CwFd%=j3%&^^uX0At50NTcbExn8LsG8B6el
z-*H=f@q_>WFaFncmdi;=O=4a*U%W%c)2iU?sZ+bM)KUTh1b+Vfxn;{1(7xsi7cQ)L
zuBf5$V%6<mxficqRke5DZoU70{n3BBV+sm3%vqYVHR|NKb7G9!*C%9Tc+};`ul+6=
zGi#q_?f>uf|I0z`_{o8%V|NrVI>@Og*G~U)|Nmck(5Ze;?=KaO0v+Pe@bLW7*^a;7
zNZ#}|wXpcnwtRuzw;Ra}Mc>wmtdajYr&s0cNt^nAdtwgh&Glf+JLM7qs;UH9*3Skd
zoufyOrlh5vx_si)>C>PY%(rEyBXv(t(_Or6+q8}j4l8TxpMT%i`&U$0+`M@cGzvWZ
z^wCwJt3kVctM-Bxd|z7|4I4#V>ODPa^UWthN4IP-N!Vdon3kp%85vnpRu&Wza^(BI
z@3x>>q_by_KK~5rsZZ052AyZ1s;Ua!KEHUe^Erj;uUErCC*^#sun7qb6_l1feQj+t
zXoAh$Jbd0Zt-I+iE-ap&o^Nh#<!*TMCa0=uamLn_u(beH9i5#Mjc-LoMO}FP6|@n1
z%9JV6@65lvSlnN?^siCa9IH~VcdlhS4<2;Xi`#SJ!Gi@hC%;4;T(r?gYjNAdl?#-s
zh3~GIcr#s9RrPv}-jdUA%YKV8ZqN2Dozf(z%)Il`-=BZ$KsO@CeJu+zJ8`sI{N>A+
z6F1*bnf~8yQqamHufJZpd9xEdzLl`~W=3Y_ME<Jv+Z%V@&12t@q;bf@YA#>HzkhXm
z;@00u=4$)xTRXdbMoN16<hgTeSDQaxR`K;}`1)VB<fWu08Lyn`rTX*dPoMdAwO5Y3
zicUQ@$MVY6tA-X97OzZoZTVilKYJoBQ@Po+xw-k%`|GFjrd=!DnsD|+Y3$o?*Ei>7
zMy=fP=0w8I`|qdQzh1KP!KLbtA08gQerMh4&dyHBsj_ORJL_iOpARiSqy-D4nL(SS
z%=6`zE?w&4>I&NVC8itIa{aaI(xAZ5(AM+MotFkZEZd!#lk;R@yBw%s?CawLH6|}D
z_3rHKocP;jXN=yVLx(_@ZR{v$-2HA>_o`K(Wi|mCB0p;EYV71WIXRCUJGLl9%TTHp
zwA1%a@%e`zD^8p}yL8nmuDx-flg!j6b9Ns+w5Rg3!R)gZbNzhOgwr;IZXg21XV%sV
z8@Vf4rrFomr9zgsB%95C`R0v^rlzOeg3{f0L3cg9y}eynUjF=n0}Olj?#(cflHp^o
zt*vcoZM}HoM#XK`33<zQ?b;Q{p6l%F3|f-M!EzzP1av~|?svOhcTGBc+cP3!M$Lw-
z?YGy@(lo36RkCwR;HA5Fb<;~c+|uTI+OpX9n=CC}k}bO>?ro{Y-F^G_&o@p#SM%_%
z=S*$o*wydj>g(-)e}Dh{%F5vIDFsK5A3yzm>T6Y7bLHlY>C%;R&p(&``upu4soj%H
z%F9pN-c}LUk5dugc<}wVp#;y9r%z)qE`A^O`S;(Vw0qB4=P@-N5OuUTDbBh^Z@-;}
zhDY5xw{_W1s$+MZ72m2MvGZNN_qVBE=E+wIJ#yQ0d&%=}^Z);x-}vylTFkFIJ-xkC
zn<pz9&(PM^2Aw=N`Oi)DyJ6tw#=|m2ro3d(%GlW0SkMv`DXCM7`|VDhn`>S9>*aFL
z7Pjo`>r@3eKpXU{tF4Qloq2d=W$+x^>S>^R*=El4Oix!YEG)E;;S&-TF0`84(%M@1
z?M-Bio;X``<CiZbR@T-VBXmT>#DW3?8}H^V-?~+FX;5d>+GCGD2E@epR8?6S8XBgg
zr-L@-oJ?7?b?ek?*TRg9jSXk|T)BGn;=Ox*adC2a+ojtN9|YAh+j1niTAOz5vU>FR
z@zaEPSF@(7Pk8mUN=sWiaP?JChiLUxuap!Ot*KmV!@93#9ew^;P)zKYa=%T{>hg=a
z#UCHJ&YC^@Vunf41l~5|LePONTE=h6mW8dp__Acn)~$ul&dgl>?agGhg1(Zi%=Vhx
zxqp6q1nsB)b?EoMJr6E_ozS$9t=aKh&|jPI+v(xlPk#Qb16>@KV<xRNm22@u(31Ji
z&dwJlRwh!s)@5&YENqzVwRDpDjJVS$7@j<h^e-=Z@VhK1Fwij|U_y-EtM9)(Jv=}i
zKhMc(y*K|_n3%XI2<%v=aBC}5RJ_^+)!MY4nz!-V3(EJd6I&9jA+lxrcIFTNe!pkl
zQ2V>=!LQ9Bs&VVfuOv_1_WjGt%kf#&tdN3ywm-|AV+_kLpFGscotc?=FlnQJxcKqQ
z{`R~)(<i;VvvV@!Mz|I0)*XBQ9keL(W{%s+5YQU>%P(2lm<81R<~S@2m@sF~nLj^2
z2d)gUnCmyiOLgJGg)d61w6wG?WSH!@o7dx}9JO|u?`*S;85fm4efsp`#S2i2aNaz*
zDPF32)4Olnh`0h-^upHM_~c2-nlSC9OP79XXnT2i`QiYLjO^^>M@KrZT)PGu=tw`~
z>h8YyVg~4{s-;U$vdb$fDuVXdT)qrC8fE6pnH4s2I%1#=x!ZH@-^o%{RsGa>6|_R_
z{Q3WPiVobqd1j_DXqo5NuU{k9hF!XRdH<EFB|bgf;`&A=CM9Q&{fk@wec6<F&!-FS
z-o1NWhj*$`cb<y#n=daf-^ekW;-z}t=JT1*(>Y7;xm~Z`ASWr=dHShTXj!%8?6Yh<
z`_*0T*Wd2H{dSuD{pRLoO&y&hDMp=dTpoH)^vjCTHU5w_J>KsZ&qTY$OU|zWEo=B>
zbl&DO&w&;5N|a{%g0fjlJ~OCH72#q{+blWztecNdkBm)rM#h4jJ7><BBV(DNz-t+>
z8nlciFf!6JEKF><|NLinb`~#L23nA)p{eP(Fd!g08gx3~)vH%Erh3)b$)BFCudb}j
z+<sUwKFw%m!0M|nzE;KPiI?q`oqW=zveI(y+_^vg*7dk4KR(|7eCP9di<d2H10A|^
zdHz3<$tPKOrqA~jmNJ#%HJjZFI&U-aFq@4WfA#lwM-vPZ%s?lRsi>))I&E%lK6%EB
zj^mFLzrDE$TID)*D(iu5xwj)Wr=2z58-0u=vnVAk?U|@*(tR`Iv@;J*OjIr@E&X_q
zT^@97>(2Q%wqCh%C8luZl)q0uR#+52<KY(5*^wl<{PM}w>-U}7QTRAwd){5scG;MI
z&&g^ZPpZ!c4H@L;?|)@Befspx)!*|D9X|Z?V}E^3Y@?2*#<U3&3a%#1zWy4t8S+6_
z(dF#a=b+BrjT;`NrKaES6rT^BJ$1fKrBK6}GiUVgxFx=M`R*O)NRuZ|o(Rav)vY#v
z-sZ7%NxZvrjb`*``F{`P89=LI89)a=J<&YOZ~te*o;_d6a}yFCw5b>Ml~lIfetPhu
z^x2%*28m1tG7|DLS8BX{o$sx=8r;-AwEf^z_vl0B=N^B2@cHKxg_%Jsmn>Y!*z4w;
zn5g*s+uO<Kzg6vx<6&bCTRrv25tkk}<>USG>c+;CA3R9-Q)l1k(BRrFHgWFU-l(<9
zmMvpzKIq`*$Hy(MH)Zz1>b-GlJ~JG`!o<Y7TYvnhSQEB-&eGx+7ZkTd>1v0s0}Y~r
zPJRVV+=PU*Tz~!a`u=}g?;N=Mv}j}geY>crDA4Uo%l+mq+Pin|*VorKCm-i~Q)Ug>
z3&O_6wjpXQ!<4C01LNZQ!d5@kum8!dHI-}iRj-l~lj?787~3}gc^3jYwkvp<&(CY@
zw-YRcA6;~pSGBO1v2ULp=meEb`S<O1?b_w0`}}K_W$iDM`ah55LEXiMhs~KQO+pq1
zfQ}tx;FGaX*ndsMZhk+gCgxzdkhL|wa#nU$me$FWCwX{zKmPmu{_^$f=C6MB_3>3Q
zJ)EK$yv{n?XI4_i_S@?to*(TN|M}rC|I@mH>Fg{^a?~<CnpE`l`KNlhhKGyqE_=I6
zNum75hs3PxY)&q&U0IRq4ZgknS|z3zQ*oR1Hl#Dn^6<7>%8p*f!otFqwl>h+k$&^-
zdfgT$K049~TF;kxdD%&J|Fq4KcCYW<yXWWS#Z~|Jb^OK%9nk7u(2}c-5uml@-qZD#
zE?(UH^Yin?0U8!{e?Y4iKwAjp<j!5VAh2@f%Ej9jFJ8>N{Wh<IpWnF^uY-bvBlp+Y
zO7*r)nIf{2{R(IgNPqpGWDzdbxb@R#%#fHhd-hDrVzu;ha}+f-J8y1IPd1TyTB;kn
zOGHAi_SL(0$8yX*U-H%$l$7-J@!_$xwY_rXib4K8n{9b_yW-X_->~7s>#AR^hAzu5
zfBgHte*XS{U)NjA^#jdO9BgI>O$jetxUgbxT#VlJ6%G?mraU^xF0W!@60$GnU=wRm
z=Bw!Zy+@y%ob2i6w`~7@`OP<d;^XDLmQD&?9d>t0!n0RbSA!~kv)R3%(rkv08fZD1
zh0Gkw;<kea9YHrsZb~^>y1w|<mQ2uqabqJR=v=Y1Gc!QveETh*{OVO!j2?Ko?8=od
zyW+|W4Gj+zusr~sx9_+7@k#ahDkdgV+U4tJfEEjMb#c9V{n}8fw@2EXZ}nBL;$mY*
zM@P^>dEqs798Cq!CmKC-`&K*0ELTdbn^oeRPn-AT+1J;_av!+=@0<C9OA0J6RSq2K
z6lVN!y#9|n!=y=*p4_Ya|7-n!t^3C}pSSyMY9j@y6!WKndN<C_&Y<0`+i!Dnaw=L_
zNNm4-_RGu5prxiJCR2)^pW9gQ&`C=hbSNKa3E%wl$M;lzo?%(M?9S&?877auR)ID_
zu3gL9e9*zyw>JiKDz;BocXy$M%#XiyO-)T4oSYMxlSD;D4-|05I65+_`^`}>HwR59
z?ApCMaP?Kt-WTvONe2=PKu2bJcyNI7apS{;*sLtA<^J=RtzXZ-{kE^OGxOWGZ_muL
zjgH%YF+;`PUVi#%*VI&1x5X3t`uLifnpUh`t7>jO-Lm*u%*7qCKgzs)d_c!SFIcpw
z2{iY8@uFjW{r)?@+1q}1r>3L?L_}0vm8<7!ZTkB9dT>w>Xn1kO%9WC#b+aZ-I>gLx
zQ*ikgvzxp7=QGCVLC1<Fo6WW?eAH4iH)v(bu^!1g_5c5xUoqCRv6)l)`r5~b?edRG
ztSk#2G}M4xoqcVM<I<pwg^xjtuwJ}<J9YD{v@<grk0w3*^Yink-~L)xPoFxaWMVSK
zw)z{W8JU^6a?ib%78Wu6I2$boJ&yZZW=D!?PL1_?KYyliI@5G7%l5Xm8Rq%-R-Aqi
z<nY43{@3I=n!i%nH9YdR)QIT$8raqUvoW~SF1Su6p=fL7Y4heS`fl@Pym<90>F%yl
zhEGp2R{ozg<>2nb!)y<N7QFrRc52l`HdfY?r>|VQR;Sw!P7N#%!#Jgcc<u9co}Fzj
zEGBkrRp{y;Z}aO<3;WwNIxPfM%v-jY{Pfurw|@El{qtwdlJZd#uG%|q*)lcIR`wS!
z4!r-Ky#4mZsI{O2!`|NBK6%n4r<fR-y8ZEaPXB-3|39zt^RtPwXKSaQ1s&-JnwyV_
z(P7`Q%zu90`R9-Ke!u6uHq1EpmPvxeVh^qI+j}ZE@6pwo+I8-nA7~rmi4zO<?wxz=
zm>X#C#`?Oi+H?HW=gyrgV_!FC$r6>Qs3_3X0BF9(wra}sxGK=Xm*{9|(7dFL&751e
zq7EH8WFWz_J>h(f_0GF_uU@~-|C$zK-hbQ~vRS@I+PrVW27^bB9<8gsl%ycAVC`Do
z?YGbFD16+}-Mu?)=UkHoNq>HP{PgAG#6NZR;`(tZcXkwB&vAPHwd$$D!xm0q4-bzW
zSqpMAGC0;>=jQRx+<Wb@q?FX5mnDx%tT?&2l<e&4Ld@oegh|QB)VTVaO+W2gUvK~L
z;lps=nDX-S)4_g=7cE+6nV=-VVPIg;;Iwc8GsE`PFJ5GnSlyj5_lo7UioN&NUpf57
z4Ycm#)~#FXRUeeSxncOzE}`UrZ`J)Co{qoI%=a_yt2&VT<oSsQ4<3}-n`O>k@_qZ3
z+)KaTf4aV2q~rD5w|gJv?vYcrKP)69^u*_DHtU@wJ?1ZqKYco{_Ht*4hRCP$_CHVh
zKY1Nr`}OMVZEy9azh1WXb((~I-5+1c8_J+Lq(e8<L7k((K*5I(AA&BF`uNE8++6G9
zd%xe~jyYwrdiCnX>(}#7^=dttawu){!l0ELEKCt=!wwxf1Zq`)8fZE?J=3O%fd-f7
zSQf8XzkYi40athT(|cz!H8NbidNn0I{d6Pfa>z@UFTZ^A#${nZf{D~qx#GgY!YNay
zf(Fv%DjqQI+P(X!TtsMSXi9qe<qH=M?5q79vA4?9*47r(X_b}jy_=_QWhK?>)EKq)
z*u#PoGjoEM`>ASab)7l|YACk0woaKo9dz23m|jfB_1AkB9SQyMDQ|njY=t{MN)O7C
z^IQ+`=EY6Bed3^v@^$Un?-e#hA0FQ=zdtqq`|9|=Ui;>XyxBFGIWMT+r?Z6V9q3GN
zJ=S|m9xM^bnC!>ralScZmeyZ8BLVfPvkxu(TWZ|VFdN)D+TNw1Da#zACmy`aXW{DA
z+>=kX_$@#D`fE^B6zB+WR#sLOb#-A5mM1!*txk?EE-VIfW?s5<>Cl4*3P+C~1!b#F
zVfCVKZz8Lzt2w#3ncKW=-TzH~UbcJV?YBi6)^6G~>ChpkviJ9V{r&l4V`C+H-ISD+
zHbkvGa`foMTenV?@BbaUXwjmKyu4#iizZH*q~xFW=KcHUFPG0>wrm;baPL>IUIl1P
z1#Q^4dezn6pI=Z=5VV#)D=T78h2hhuPj}qSTeNs_XLtAEw9SE#T|)vKpp_o?-|x>l
zHc^42>BHM1<MYg3l5>JK_(&ZBo$bW%@89?Rr)SPpX5UhC^;6)<6Ki%#SH^t&`0>Ef
z9>bUK=Zj3J`T2CZJL}SY>l`4V)b!zLC&w{~3%71{-OU3ndHeON>e8i4FJ8Pz`19lA
zo=>N=bMEb#dFM{drp=o{w<<m4uTMDK#v7p{wrch2mv7&uzPhsV%xrUiyXrMTD=)k(
zF)=rP{P`znsoL7LYfa3}Cr_N%csI{|VE}0JNlA%G-1_MkE(Caad0oibD$(m!VkIlq
z-TLNDj*_x6Xrtlgw6lluwm<$_rKzE@VCmA<xz^>2f>wU`Sp&M9uf$5yJWnQVv!sOG
zi4C{k{;09D+@1IO+S(`2pI^Rq?bt%+_KIgS(=TL!+WeqPIAUXE!Dp5DdV6=zSqe@R
zO%T%HT4%#;i4T81pMU)M=Zlvw6YuY<t+0`M^!Rb*pO43H?kH3S%}BTN%Y$~4e13j@
z^7QG)`(&+8oH>(na+0cxi_3)z7aa2Q*UL<q?8TOTX2!uzVfBjFYqx)>umPP#uy3DT
zQgX7VpWm~$+wZ$A4NBY^<>~1OI#X`X|9`(h=WuL|>OGs5e0iBK=pgbfTTHHGnZAAd
z7Bo566S6X7(u4`1Rq&miouDqygb599i$P28_U*GPDk?IVeRjv)ycj*uCGIIHDGv%P
zK!+$tt(|6D{Y^k#{(K9k@C=hoA+FY@Q>VNHbA(wOcQCWCvYuQz$z1!iuCg-o4ikYF
zS5^iy)cpB){F70T<dYNsT7p(Cj#@iy^YOf}Yq#HKJPBZHb#g2%HDxG%evb9Q(&=$b
z5A6Scwr})4Kec4H{r{im4=nwwn0nNIhJmBr?A+Ykzftnd;E+9J?hESrw;w)uaj|>k
z#w6B{A3uV2XH1*s=IYAows>N{{lAEJxxz9sJ#LF1f-=LJH9Tv>yg@PKn6@zjw2tmj
z3undNxL?11fzBh_`Tbt?&#&w2L4}N^rR36}lfU2Z2dxD@dFs@p%_nrkxKB^lKmPdR
z2T<S5Xy%67Z$Zs!)9h;(u3dABi<@`n_t~tikG@uIjL<PLGfUcj8?*xU>Q!(<NKVer
z&#%uWRy5)Fx3{2!nq;lZHWWSe0?jjD%~G|qoA>QoSy6E@=%_)qX2!j7=d;)EJ+}G0
zU3K^ONg(gunK7~P@PY*kw3#}C8UkM(RExcT<%-Fxs+YS%vVP_N|CSFrb85o%;&*o}
zAIyFkZdUi_M<8=N1L!!&2SJ=`MKUHcP5=M?|6hBCuI_HejLb~Nw&hc&O`XcPSbgEW
zlLvjlO@)27E(daXEULa}bc^XuV)l;Sp69y$dT?y)*-fdZL1l&6?B2cA-$BO{%kZ)P
z{{DXXx^;Y8qk8?8D{E_WZ;kR*5RjNE`K)X=BWShJ^!<NS^R~}^e7wK-dTjYb$R;$<
z<z*))se;yK&z;-rw9sMkMbLryw{G2f;>hN=eDaYaE)1JCZ7SFqQ&Lh=@bD1pr_Y~X
zzJH(o@K7s5Nm-du{lA*3s;U*MS1Su}1cZdN9Dkg6byet_+uQlU^Wh0EFD(U)78gEp
z>FVk#u#lPIqqg(qrVKqDjwXdW@77J*s?NG($wQ{s*VgWSwy8jq`~1t2qWiVqPfqvV
z8)cq<@6XQd;*jptK1-zsW^8LBHZpY|J;Y!C!};5ro6SxO6$}k0dQMgo<Y4*ndH(-1
zpmW8y=gEqSiavSzG%!3Ibat?&rl#Zi>&aWAI(vFtR$qPewaRd&kBn7G$CN1|2|s$a
zoPVDD{oP$qsXuL+SW!`tMd2eBNlD2$7KKgEo~2#MGELh&6V#-dFyX+>&FN1MgolKP
zn9c6pvBLs1ZGUmGd*zdf?iqP`pbbAyPfy=m|G!R2Nr{7>-+gJ2ptv}DjF$4sHEWJM
zElM<yc#>;Z{!T_wQE|(bEfXe9Is`g%dzNW7=t$SSd-vM>d@^}^%6?D-VEeJ>z7zL`
z8E9%~c$}LwY2Wtk+nF~c9Aru;$$awkOV!>#n@vI<_L|?DuzBZ_OUJG69yrjjr+&%v
zCsDs$PK8+4|FdDR|NGLv(Odn}tJklee!7yMCAAKeZaEH_yRzIl20FXMX<@?2Nvc<_
zTzODp1zH#wwt6bBv{^u0TwmOJbu%+D3BH8fySt`>Cd5KRU0qyQq<TR!ZVR2;K}j$B
z+M0!H*7V%nU7l<r#mT}1D$Uy3*n)zC89^iI;EmSW@-skV;fvk-KfT-iKFLJNP^vdV
zN6g8|DI+HbbcCuzuiG@e*slBU)opEK-`$-ybLPp5i`_X{m_P$?7cV*n1qm%(x)joL
z@hvPg)Zh0*No_Lc>Z@A*6KC7k+s&{0rRhCgPu0pQN^X%tDg!9w?$=G99vAn|JmJZc
z^Aj5_Tb&f+)ob6sd&g#H|5#f~>s43kv-9)$d8U`ueK^Q|U^@4X-aGqN=QfK>nB?Vg
zzw)^(1E}G{P<dg(l(l-g)91}Q_vPi~r(Zp6UoCl_^`uw@WJQz09Vt+1oN+eo(BZ?K
z9UTj{Y?-oUi%C{i)(qQfvxN&6=G@xCIr-!f(D81YHgDcp@Q{hGy}7B0334m&l0AFo
zbc^d>x__VF!Nr9|th?1gq2c;#RZY##dGq9$8X3g(Vh-e(UA}SS1ZbA@_g@z`w^Qj~
zLbXIEtNR~&{Lvuy7O2{YjEoEk4+q`DuqRG`?b@{|si}!(vorJamv7!YdCnXe89sJP
zOUoU1^XB-0Ps6>vt+%Cx#YT?*<;$0QJ{;n{a^;GEtZeVmB**2KCr+5)5Ev*pbLPw&
zZ_5rHJ`7r>IMX~|Znk;;vz5!|?V7sV12mlY&SLua3C;TAXC@o*mA|>c$di8eaDhNj
z3Lh7!di{{A%9NMfa4};EWW+DoXeI|2SJJmPHyLEC%X;?gv6(e{HfV+F+_`fldfga6
zeRt40saLOFfmQ(-8VYW|tsB4o`fJ14XF-PuufOi?<iun*pMPnPXLYqTq?2;#(xr~h
z&VvsNK=(L+mRt&piXPpRdiul9niU~hpqpqQM<swx`wR$ZIR9MP%uH<Z$s^a+MhC78
znc<_Bl$5j~YVC`!RZNWxps96J)2X0yVNa^h-;jP@uI@v#Jm^5l-FHF9ZtVa6_x+vH
z>#<LsK7CkV5uhOgy4CvEuPQa)StX~x^@3VrXLs2M`<!Ch9X0diiPZIC9J}Vlr3tI~
zFg#GV|JhkHbBgD-($`@>U9L_4==#0>=T}}{UIvig8rbD)1a^dQX>aWW9aR1E=g&`X
zmmFkP+6*4HWl4M^u(c|=;qXHR6_u7BKPv9zZI4NMSzr;dyG+;8($dAv?a}9-3{M_1
zUtZ>W_-WC^i4z?oA|%Xa^G@|T3OXWpcey@fvj-bH`^&d)m#$jX_3c}kl9Ez@*3>7@
zpM&a<b+NlMva^-d)Y^W(-#@?h_qT^dJ2f;l4`yxs^y~Hd$unkn)YsdmZI)~}cFavn
zODiKk|M>II8_V8C85tYj+?FfN*WUc$LjmaY!t1Y>u3ZaSS}DTCT3J~MIuNwNMh;Y$
z#;tc>8L}zmq)@_-1qYvhe)#$4g$$D=%a@-Hz6mMTA!HuA(&1d53mGOWLbP7JdL<zv
zvt--0X=l#(^vPPgm6VulPCu_^XgKlma{tXGFN3y3>FQ1I-dX&d>9Fd`Ap0%Po~6ar
zd~~g<s_N+INVvT%w`l&>6DK?>C(S&WvS|78_A_UEmif#)w0wTusZ&$6J$-#se|~xj
zI<n~YZSC&`%o}dz%&{nBntbvI=xPp7)v`0*CooXZYw4tx78cOV;;pUOAAkR?u#u~=
zlmGkoFX#@Q4I2!^^yAL-$y$T97;M})5wt%MH1?yUtjxUmrVn`2b9YM%%attCmoHy#
zPdg4OIPUK#e0=QT_uo(NFFj+XdHHRcmxsrKb?f*X<OEm(f`gfP()ZU+<=K9F?v%#w
zq4TRV6`Ho_+viubLQ3xKt)O8fNTb=v$jHOT2Q)&`(%QPQ=4a8JyzMpyjw?fgA|pLh
zQ<vVkyk^}xzR4$5z88J|{nuiy-%j?JvNE%mFJFH7@&$CFmX_9{l`A=$4}vz6XJ%%C
zOeuQmwQAKW4==At><Tk{)Ie(sqt>Rqy0X$|j>W`7hn!;c#M3rQ_8)i7%>|`F7It=H
z3k#2e0)wi(^Zx$+K6%1~hOpH~GfaX40uC(moqgf;R})jyi8E(*`Ym6)b?ekOZ*meW
z7O1oGEmgSx@9*!KHkF%V+`zrv`(;51S&xo%GJuYDXKVZK$0Whi7U;O4nVp|WKvtIb
z&YlNPu2$_}Wo5nl;n&nt59`hK<zESKu&i7C=6h9)-GWD-)$W1|AcY!XP>bop%aR>2
zdQ-eq(>8-9IZmBA1uB-K)=qO!0J-pC0qE8iK3OZ3`)10@%*!u(hJ=Wys;VA2b}T43
z`0ylE?-eUnD43gzZ@vkdCCu6?HU0Ea(EV~gK1ZH^E_`u8@k^C$_4jw64$vq2!<w3!
zCMG5qu3R|+I;B&Di}ml{zacBaKb0{sh<SOsIEI9U330GMTk2D$h%hyR_K~)<v@E!q
zwee=o9J|_Cpb^P!+swGQxHwptbRsq|G#_N(nZBg0wYBi!A=W>2_K+@ka9CJd)LJ!7
z&CXAsia`6StgNi^PSy#sI103^PkPdFRhl!%^WdU8cVZYUtz%bTWqR=axBJ@Nn>QC<
z`Fppg;r5!y%{R8^-w%)Lha{-EUMzQxHGKP4cInb32^kp`O-;^*9Xl*!_}JI3T?@J{
zVE_H|vrMx!bakJC4&CA9<prJGcK7bt7Ea+Osi4z1`ujm!5ANKFv9hw-5TV1u&i?sc
z_4|*X=l_>!08PukyL<Z9)@;kFFB+MdnGB$H5Km7}2aN(AYUKva@>&)=U~pS3IM?rZ
z3#V{LXD6shF=>)eP*Bi^-+zDnt=kf%`}XbIBgc<#PB|%LmUrjG=kxad?(Xb)+ohWi
z9ymAGdZvB7-Nua@Z@ewrxkfbM{k^?sW|?+(ba3#=SUBY6>4A>)vMT8ikFWW7?Tw71
z0Lw#ZK@kzqaXHS;&YwcOcNAqD3P0H~i7DvPf|g!s^FA%rsi1R}rW>7tluRrS-9Y`t
zfS@3!v@|vMJ{d)I_4ZzA^G#`IrS9Z`4yXa0Lbz&`mRZgXhnN@{sou8a<9!>`&PpZh
zXrFyGOV!G1)}u#Bmo8o6U}4%3qsJ#}b!3)lc7cVA&A%UyK|?J+J|xCfy;KD);MdYx
zv}#osXcg|XX<{W-vSzb;R|YR<IxKt{bl*&FuCCWop|xS&?(XbYuU-XRdAe<z84nwC
z+3vX?J`~)#eS78FwW>-=O`tmqHf=Hz5*7~BnmT3r^vy+2y+HM1em>}mIZ)%{&Yc)3
zDJcV~UeFo#WxIF&ez%+P#F;ZwyaPc=Dc@Z4ftiu>>ra;sWV7CBYHnWtLR3sfhUY-L
zd>zMuE>Z0U@1?T}3kyHJ^`C2{`roGh-=FngEL1?t%If#c+m{;TU|?!0+EDuX8Y8I3
zz_4rAE`|?PdvClg`&q^<uE%m@N)Whv)3VXUM^l!WsgWUTtJL<}z2~1N-`teC;%b(f
z&y0rq@6|0VX56?D@#xW`0;{=2+oT=*{rSC?PC9YI!*chT9J9qCS}$L{I`pz+Q|f84
zLx&D|czQan4FjD+0@~#9vP4By_2fab{rCI*mMiP&^?~-36?}T)xh71TtCgwOt@-+E
z&_2i1)6)W@qfa-na(nprsMy-h-M7!~*YDqw5)uvz10EFZ1l^o;exB`>t5*fZ#MIdT
zi0Q}q`1|v(4qu=4``cTPu&k_Z`r)5{>p=aH`|tN>eQN|~E>L?&f#XnwYEb*J2GGuv
z*RMhA>{V1+-n_}Nva-66VX|e*76Cy)$HYWMgBr8Ar9p|{$<H}frCopi)YRC?$E}~f
zXOB&doqTR?u8Nx4r#G9=gU<3Pdw1vLrqt7*12FIGoP6U(gp!idlYIs!Po7-7E%RU#
z>yEp5pzT&JE-f{c3Id=-xFw~f3q!ODci#ooT%e2Co;^#OrW?)XP+Dr*=+FSVDWvk5
zB<MVsoyE^9e!X1&H0a`)Gd@g>3>Pn6oZ+K(C(pdm0kp&IY+CZIEtwz}e)#YqKf@nX
zr)@v=bY||P)81a|Q-6JVxn;{1h7C18ix_@BnKEUgS^hm61}$xEMo^Cdw0Dj{fA5z`
zd)Cc1b^ZLwXRp3i`&6&YrTceZdOLgeY{n=5S?BmY_mwt-24Ib?OWSRU56$v_7?xk|
zyqcwIYAV`&)Tz4K`uDfD&a1DU^6l>K>H;MWf61jmo*o_?QoU|`H$ub1L05I$+?)>D
z964o5!TY`67hQjSF>C9N7(J#&2GH!{%a<=(T3Q$`T)WnGHH-DY<Bt>O&Fj0Gr7FO&
zq53<hy)4(yzBUY0cAY$b{<-!2pL~AHCvVNZp0qUzG<jB9TFRiJs`{|Nf`gm;aMDK5
z@o1oP&b_?6KK%YG!NYdzHfZ@d=<co^J1m5Rge+=*nY`coU9N2RT<3N^(4OXti{0}N
z#(|1%XhGxnqo}Fvh~}l}Q}4eEbFgfP(D9jJ(Ad+%b0*DLZL;UNqGGGLBI4qoLE8gD
zLtBGZs_5$SZolnYU2T2y=1m63+JnZ25AHlV+Wqpy3k5~R#$(4oJMcdI{%c}xE-Wwa
z4{1zGoZF<WrKMzLHEYfs882^d&`{pBHK3(&pu@v_e0U7*T-P-<ohoZxw&42fiy0=M
z17ojV1<icsZJ&L0bvQF<lT}QN%+;$`mn>QG;PcOn++0;_Yw4+8N8jDu4Z88{+M37}
zSF=Eei0JFPufGo36jffneU7R69htoCvw!{i6}?xoslXew*7x$ueFo8963@O?J>8cP
zzkSaho2=~YKn;-}fB$9cnHvS`XlZ%v`vW=!Z?+lljwzC6MU$R;fG(urZE#zhcyf~J
zfu)}F8I04<{o(3=GHK%X|CRmPCMF^~Oj^I3n5fL~;%gNH=u{TaCXIbF<8qU{US3cB
z^W)=+tFJ(A0Hw4VW|y|xy30beKxbeoDKYK7dv06q?Tnlp&@n-;US(a$GF`cH<&h&t
zKubp}yIx*g><-$qzI!+5+96QGI4^J6!i9~WJ{8%>@$W8sJL&(<>#x5aIePS@qw(&$
z=RgPYaC0wSwW@2;BBh-!CEwrOmFRW5l4ZJf?b;bWYCp?lWo4HvTLwDp>Sm6bu5K^r
z;D2^@O$`kNO-;_das5Y=Kx=T<hIM-_U9@}m?AYC96KBluh>Md`_nX6@U}q<{{4!`M
zlw5!ND)9NY>i+Xo^!58+US9sV#47*bJWv|TYt`pc<K^XD7@z?;c=V^YlnKL$>+#)T
zt6f)L4T_9BdEtV9v$HeEMcP(YvtGP-G5hNTNYZSvo&aiiYEA8$GDQS(-|@@K%P-%#
zbLPMS2Cb=F`|qFs^z<|*3)7YBpbHJ4=ekKrd3kzr&Ye3Kw8ib-y*M8=VePOrAon(O
zb#eVX;Jh?QP*&DAH&=J=+_`)He!DFqCN>GuSX{rJfBp6D>#to`hMc-ttg5Pd<?2<?
zQGHoiSsi_S&n~*lFWtFw=8YQ>XVQ%A=JQWJdF0K_%};eEy?T|EV<z1#rkiwWiRY(e
zyW(d&eC^Hm?!{eOAAi0_()h&Lv#Mrh)7skD<m!GTf+n<`7EU<-gS$zA<Ir0b4GoWf
zyP^~Vxp=(dSy@>{L_`v{-`;o|RLFxyl0ZXCTQUTjo12q_R)W$E$Dv#&(1_>KAkWlP
zRjJ;#HEVS2{(NxW5~T|on0)^E;m3*q4G~cHF(oxsP+Z)-rpD&+v0mrpmp^_wt-rbE
zXVIxsr#9To*>N{7Dk`d@zkm7e-Lr4ph|mEa+XtE}0Bt3?aU%lc+E1T8fm)@is;$eG
zsflp0T3TBE_*-Wq$1f`@`{&d2{h+PK`}Wx_TD(|OU*COUK*75^mNI<oZi^=_^`8Fd
z`)^Q7i{W^m?BazB8&|B*ShQ$S#pko;pm86)>D_&Od~!7(9Is@V^0hZVd6M$$*RLB}
zGKGcJ{a6k>E!voO*Ghzom9PDA*gH9J%?(W#(n`yw^nn&QfNu7Cc!-rx#^S&t*KUKE
zJ{B^3pqn30P1RnxX3Z1O(e}stWI?5noSdJJ56{Y#D?uZUlP5dZ*4o~@dGpWv|Nrzs
zlQkPR3br~O+*SIz;o&^9vuD9~vp`#OPEJf$uU^eCkpeCFdiBa_{q^EUM>rwN8bEd0
zg$n^ScJfhc#k!9k0?p4a^PAfhwf0!n)<@reclP$GYG`zvI_2e~CY+m_TQu$V6wTlj
z>(+tV(k(44eX`ca&dfBv`1<Rgf8Y13+uP6Iu)zSdP{J^|&BoSLON(o1&>kOFM}c=D
z*Nx8xzq)8vw6j}WA2db80J>D8;OPB+&_<$$`TzesX8;fVE-wL1H=aH_)0n;C?(XvY
zTQV2S-!8ZPHt&Iz!OIzb+&o{$=8%-6#1I-9YMv?$4bc3)69v-DpmTDos&>VAWo^D$
zursFS*URNgmMjSf3<P!YZ*R{(o?-F{bTULl1V@b2lz_-c&GeSP57_M$jE#jSpFFZE
zbalhShdjb+J_)zBWUdI&($dlbtt3%YR7^=vUmUjjY0u_cx1!kC*dlh7Xo41N)!12C
zTZ2|)Y~Mb;^z}7CA)%(Am7uc~dfl2&KV7tMpWIwOcRxQq$ewTTH5{`J9dg>WYuA(G
zHlXXUt*kow``y=HPd1QX0G%9g_UzdiXVXrdI%QGw!(eyW+h5PhyTDln+A5l^0%{dq
zT^)Y;#*L2q@0V}iJ{{C)nWh`<79A}ut{1al`SNy98FTmU*=J{Gd;0hsd04O^=_r?%
zmlp>wFKD!3%^DpoZSBn5T+q<Qk|io8;Pqtx{?+--v+?xu;);xneDSqvOO!5XD#g?^
zabtvrhQ@-88wFQi<%)TwFDxso>#u%ySE)9rDR+BYu0*dJs6*4!Gv)WbsZ*vDfQHDY
zPiN=hcl%Ja7j!zyks~f4TB36O?cctYUC9E?fr7@gQcq92c<UCILsgYk^|v>U>#uJv
zcz7s!Q#`009i99{<mUO)Jkk;p6Y9^MII&>=e)%0!7{9yzJy-W-vHZsa%=`==j@$nW
z-1q0l$H(y%_aULv^q~^miOt&D(%$~~`)|<xhOJw-TFmtm;bP6q%mkeSa^ZqQb+vV3
zVxrC0E5RY5p^yd9DM!0RXPiwF5*7wcs5Lq;?2S8rzW!fvjh(!-c^+uFwA1?QpoTz5
zh)DJKcdm;sg4&;{sj7*IiC3;%nJ{@WXdhhFUb*eJd&5>AP1~GkAo1a64TDb14uSdS
z)zg<+&h;yMe@{2R8`7rUe!mWM+jYT@4~fst&+iXg?Yb~vL*?hRU%!7>{`>j7aQEGc
zy>XVw{wqU*Vq$u3zXjb+axBM;`7pb!GXDWknonHwdr`xD5s4`#CMG?<^g!*&CWSlu
zn%a)MjCS0ycn53+0mmU-Mc)81#y_tgf34bC@$r#)QT)%JKd)T7W|Viw;wKA#%&D^7
zxAto*e@_kynsn#$o6G+8d)NIA(Gc0Pbt|KQw6u2ozjM~_89sonJuZHJ?(PHSohnL7
zNmt%qO5X9u&~x&UrT5owy)(U3`SrwUXV0Hc|NZT4{L8vl?Ou6%xvf#PtIeH3!=Ekr
z4JQhO!4CJCz^2wCQ&vzJaO(Yc&|w7)fsNh2-|v^7e3IqB$;s+ZQ|<i~1SV|0u;eSV
z&ny$phHKZtnwp#4bB~`$cp#pcm9^_#Ds;_kzPO<YQnAY7D3GTl2(bnx+%nr^$`tSr
zN5N^&;EnBVZEjzsCRcT>4ADy1;bWqs!_)A5UbSD`yXW@*E%)x-%aC((6Klif^z)~-
zH)^+wKn})axUsi-d(F?u{&t)S{<fkZZ#W9HNUIv%fiR)uLp34EyrzP7$(s@}0`E*7
zeYG$1WaiO3Uog2&ZsuGyUq4+<&58QIpX%3x#!VQG9zA-X;F>h3U+7kQJ!>n&n)v<y
zx)$;<#Ky*&zi;G(B<gY%!2;nH>mHN4{sQk@9(}d{rYX|3+E08cQ}W|uy_3~j`*buz
zHAISjW~QX5Fs$GE?beD?kL}VD5)2d6Cmi}PCAiZ>^=tRCIp8o*;5f9kBls`KjplhW
zSzCXF<+mx^5fWDOnb2$ms?7iAzwavyx_!ui=RddJiVQ~9lD0#J5m}EJ=1e|!oO5PH
zRB~!ht76A?0|Eb{Jv%S@Tb%kF(lXb#jEVRCzw5Uc4ul&%WQ$(5HSF-i4SD(b=Czs|
z!kP~zZ2f3gzyABOQ`3a=KL4!Q`seebM@glBzI@rDHu+@!^F89>_x-l%-o1NwZJfTk
zqVEnyh7W;icM?u-xt3)j6&ZE>+Dsp{Ww#j^8nzldWQ$&L`|Y)d4-^0X`uckN*^>{K
zOZ6Y0Hu;~z>2|d{ckZmYw|mc?HNSuVUbV`~-hTb6>npdfy8eFus#mXGty;Cp&(Ckw
ztJ2PoMf<mH-n{v0y6*QXW`>4-Bao?E!<r8+SavzHboX7~c)psIA1h)epFAS$#mLaG
z_2YZyZ6Vof`S|!=y?C*r?eM~r@^y3e?%g}fZ+Vh51H*>0w@0gtZd7$2UG(g8rLCRa
zwH&j}ffo0}mS0{Ow?6#$@84T@?~Yz;YWP!l-n@BXr_J}@`uMRhHa`A((#9Vbey-hG
zKK+D#J#)R)zxd;JuOB{C{5P{Ee{bCR$%hydyl))lUE#Mp_;1~Qx5XEi>9Z=iY<s`_
z{`>3a&iS29+gw#w_b>kWp1<?%@4NS9ugiXW1_l=MgroCa7#J9S*m!#}HrPi!pA>Y1
zZQ2y31K&1un=mjiFtBjufhYl$9gGYN3=R_>uz|RaP0}E0h)Mq4{F0l2;h*2yt*=V%
z%x<1??bb4VMuv#V-#&f%wCQCx$VUzH59MZq7#u=(K$L<<0W$*wLxaLWUJ$p5#T-P9
hN-{7o48uTp5&xvpj%m%Mht;51_H^}gS?83{1OTl2iTnTn

diff --git a/tensorflow/contrib/eager/python/examples/generative_examples/image_captioning_with_attention.ipynb b/tensorflow/contrib/eager/python/examples/generative_examples/image_captioning_with_attention.ipynb
index 12c5eff2b4..979772acd3 100644
--- a/tensorflow/contrib/eager/python/examples/generative_examples/image_captioning_with_attention.ipynb
+++ b/tensorflow/contrib/eager/python/examples/generative_examples/image_captioning_with_attention.ipynb
@@ -1,1174 +1,71 @@
 {
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "K2s1A9eLRPEj"
-   },
-   "source": [
-    "##### Copyright 2018 The TensorFlow Authors.\n",
-    "\n",
-    "Licensed under the Apache License, Version 2.0 (the \"License\").\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "Cffg2i257iMS"
-   },
-   "source": [
-    "# Image Captioning with Attention\n",
-    "\n",
-    "<table class=\"tfo-notebook-buttons\" align=\"left\"><td>\n",
-    "<a target=\"_blank\"  href=\"https://colab.research.google.com/github/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/generative_examples/image_captioning_with_attention.ipynb\">\n",
-    "    <img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />Run in Google Colab</a>  \n",
-    "</td><td>\n",
-    "<a target=\"_blank\"  href=\"https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples/generative_examples/image_captioning_with_attention.ipynb\"><img width=32px src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" />View source on GitHub</a></td></table>"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "QASbY_HGo4Lq"
-   },
-   "source": [
-    "Image captioning is the task of generating a caption for an image. Given an image like this:\n",
-    "\n",
-    "![Man Surfing](https://tensorflow.org/images/surf.jpg) \n",
-    "\n",
-    "[Image Source](https://commons.wikimedia.org/wiki/Surfing#/media/File:Surfing_in_Hawaii.jpg), License: Public Domain\n",
-    "\n",
-    "Our goal is to generate a caption, such as \"a surfer riding on a wave\". Here, we'll use an attention-based model. This enables us to see which parts of the image the model focuses on as it generates a caption.\n",
-    "\n",
-    "![Prediction](https://tensorflow.org/images/imcap_prediction.png)\n",
-    "\n",
-    "This model architecture below is similar to [Show, Attend and Tell: Neural Image Caption Generation with Visual Attention](https://arxiv.org/abs/1502.03044). \n",
-    "\n",
-    "The code uses [tf.keras](https://www.tensorflow.org/programmers_guide/keras) and [eager execution](https://www.tensorflow.org/programmers_guide/eager), which you can learn more about in the linked guides.\n",
-    "\n",
-    "This notebook is an end-to-end example. If you run it, it will download the  [MS-COCO](http://cocodataset.org/#home) dataset, preprocess and cache a subset of the images using Inception V3, train an encoder-decoder model, and use it to generate captions on new images.\n",
-    "\n",
-    "The code requires TensorFlow version >=1.9. If you're running this in [Colab]()\n",
-    "\n",
-    "In this example, we're training on a relatively small amount of data as an example. On a single P100 GPU, this example will take about ~2 hours to train. We train on the first 30,000 captions (corresponding to about ~20,000 images depending on shuffling, as there are multiple captions per image in the dataset)\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "U8l4RJ0XRPEm"
-   },
-   "outputs": [],
-   "source": [
-    "# Import TensorFlow and enable eager execution\n",
-    "# This code requires TensorFlow version >=1.9\n",
-    "import tensorflow as tf\n",
-    "tf.enable_eager_execution()\n",
-    "\n",
-    "# We'll generate plots of attention in order to see which parts of an image\n",
-    "# our model focuses on during captioning\n",
-    "import matplotlib.pyplot as plt\n",
-    "\n",
-    "# Scikit-learn includes many helpful utilities\n",
-    "from sklearn.model_selection import train_test_split\n",
-    "from sklearn.utils import shuffle\n",
-    "\n",
-    "import re\n",
-    "import numpy as np\n",
-    "import os\n",
-    "import time\n",
-    "import json\n",
-    "from glob import glob\n",
-    "from PIL import Image\n",
-    "import pickle"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "b6qbGw8MRPE5"
-   },
-   "source": [
-    "## Download and prepare the MS-COCO dataset\n",
-    "\n",
-    "We will use the [MS-COCO dataset](http://cocodataset.org/#home) to train our model. This dataset contains >82,000 images, each of which has been annotated with at least 5 different captions. The code below will download and extract the dataset automatically.  \n",
-    "\n",
-    "**Caution: large download ahead**. We'll use the training set, it's a 13GB file."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "krQuPYTtRPE7"
-   },
-   "outputs": [],
-   "source": [
-    "annotation_zip = tf.keras.utils.get_file('captions.zip', \n",
-    "                                          cache_subdir=os.path.abspath('.'),\n",
-    "                                          origin = 'http://images.cocodataset.org/annotations/annotations_trainval2014.zip',\n",
-    "                                          extract = True)\n",
-    "annotation_file = os.path.dirname(annotation_zip)+'/annotations/captions_train2014.json'\n",
-    "\n",
-    "name_of_zip = 'train2014.zip'\n",
-    "if not os.path.exists(os.path.abspath('.') + '/' + name_of_zip):\n",
-    "  image_zip = tf.keras.utils.get_file(name_of_zip, \n",
-    "                                      cache_subdir=os.path.abspath('.'),\n",
-    "                                      origin = 'http://images.cocodataset.org/zips/train2014.zip',\n",
-    "                                      extract = True)\n",
-    "  PATH = os.path.dirname(image_zip)+'/train2014/'\n",
-    "else:\n",
-    "  PATH = os.path.abspath('.')+'/train2014/'"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "aANEzb5WwSzg"
-   },
-   "source": [
-    "## Optionally, limit the size of the training set for faster training\n",
-    "For this example, we'll select a subset of 30,000 captions and use these and the corresponding images to train our model. As always, captioning quality will improve if you choose to use more data."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "4G3b8x8_RPFD"
-   },
-   "outputs": [],
-   "source": [
-    "# read the json file\n",
-    "with open(annotation_file, 'r') as f:\n",
-    "    annotations = json.load(f)\n",
-    "\n",
-    "# storing the captions and the image name in vectors\n",
-    "all_captions = []\n",
-    "all_img_name_vector = []\n",
-    "\n",
-    "for annot in annotations['annotations']:\n",
-    "    caption = '<start> ' + annot['caption'] + ' <end>'\n",
-    "    image_id = annot['image_id']\n",
-    "    full_coco_image_path = PATH + 'COCO_train2014_' + '%012d.jpg' % (image_id)\n",
-    "    \n",
-    "    all_img_name_vector.append(full_coco_image_path)\n",
-    "    all_captions.append(caption)\n",
-    "\n",
-    "# shuffling the captions and image_names together\n",
-    "# setting a random state\n",
-    "train_captions, img_name_vector = shuffle(all_captions,\n",
-    "                                          all_img_name_vector,\n",
-    "                                          random_state=1)\n",
-    "\n",
-    "# selecting the first 30000 captions from the shuffled set\n",
-    "num_examples = 30000\n",
-    "train_captions = train_captions[:num_examples]\n",
-    "img_name_vector = img_name_vector[:num_examples]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "mPBMgK34RPFL"
-   },
-   "outputs": [],
-   "source": [
-    "len(train_captions), len(all_captions)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "8cSW4u-ORPFQ"
-   },
-   "source": [
-    "## Preprocess the images using InceptionV3\n",
-    "Next, we will use InceptionV3 (pretrained on Imagenet) to classify each image. We will extract features from the last convolutional layer. \n",
-    "\n",
-    "First, we will need to convert the images into the format inceptionV3 expects by:\n",
-    "* Resizing the image to (299, 299)\n",
-    "* Using the [preprocess_input](https://www.tensorflow.org/api_docs/python/tf/keras/applications/inception_v3/preprocess_input) method to place the pixels in the range of -1 to 1 (to match the format of the images used to train InceptionV3)."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "zXR0217aRPFR"
-   },
-   "outputs": [],
-   "source": [
-    "def load_image(image_path):\n",
-    "    img = tf.read_file(image_path)\n",
-    "    img = tf.image.decode_jpeg(img, channels=3)\n",
-    "    img = tf.image.resize_images(img, (299, 299))\n",
-    "    img = tf.keras.applications.inception_v3.preprocess_input(img)\n",
-    "    return img, image_path"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "MDvIu4sXRPFV"
-   },
-   "source": [
-    "## Initialize InceptionV3 and load the pretrained Imagenet weights\n",
-    "\n",
-    "To do so, we'll create a tf.keras model where the output layer is the last convolutional layer in the InceptionV3 architecture. \n",
-    "* Each image is forwarded through the network and the vector that we get at the end is stored in a dictionary (image_name --> feature_vector). \n",
-    "* We use the last convolutional layer because we are using attention in this example. The shape of the output of this layer is ```8x8x2048```. \n",
-    "* We avoid doing this during training so it does not become a bottleneck. \n",
-    "* After all the images are passed through the network, we pickle the dictionary and save it to disk."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "RD3vW4SsRPFW"
-   },
-   "outputs": [],
-   "source": [
-    "image_model = tf.keras.applications.InceptionV3(include_top=False, \n",
-    "                                                weights='imagenet')\n",
-    "new_input = image_model.input\n",
-    "hidden_layer = image_model.layers[-1].output\n",
-    "\n",
-    "image_features_extract_model = tf.keras.Model(new_input, hidden_layer)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "rERqlR3WRPGO"
-   },
-   "source": [
-    "## Caching the features extracted from InceptionV3\n",
-    "\n",
-    "We will pre-process each image with InceptionV3 and cache the output to disk. Caching the output in RAM would be faster but memory intensive, requiring 8 \\* 8 \\* 2048 floats per image. At the time of writing, this would exceed the memory limitations of Colab (although these may change, an instance appears to have about 12GB of memory currently). \n",
-    "\n",
-    "Performance could be improved with a more sophisticated caching strategy (e.g., by sharding the images to reduce random access disk I/O) at the cost of more code.\n",
-    "\n",
-    "This will take about 10 minutes to run in Colab with a GPU. If you'd like to see a progress bar, you could: install [tqdm](https://github.com/tqdm/tqdm) (```!pip install tqdm```), then change this line: \n",
-    "\n",
-    "```for img, path in image_dataset:``` \n",
-    "\n",
-    "to:\n",
-    "\n",
-    "```for img, path in tqdm(image_dataset):```."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "Dx_fvbVgRPGQ"
-   },
-   "outputs": [],
-   "source": [
-    "# getting the unique images\n",
-    "encode_train = sorted(set(img_name_vector))\n",
-    "\n",
-    "# feel free to change the batch_size according to your system configuration\n",
-    "image_dataset = tf.data.Dataset.from_tensor_slices(\n",
-    "                                encode_train).map(load_image).batch(16)\n",
-    "\n",
-    "for img, path in image_dataset:\n",
-    "  batch_features = image_features_extract_model(img)\n",
-    "  batch_features = tf.reshape(batch_features, \n",
-    "                              (batch_features.shape[0], -1, batch_features.shape[3]))\n",
-    "\n",
-    "  for bf, p in zip(batch_features, path):\n",
-    "    path_of_feature = p.numpy().decode(\"utf-8\")\n",
-    "    np.save(path_of_feature, bf.numpy())"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "nyqH3zFwRPFi"
-   },
-   "source": [
-    "## Preprocess and tokenize the captions\n",
-    "\n",
-    "* First, we'll tokenize the captions (e.g., by splitting on spaces). This will give us a  vocabulary of all the unique words in the data (e.g., \"surfing\", \"football\", etc).\n",
-    "* Next, we'll limit the vocabulary size to the top 5,000 words to save memory. We'll replace all other words with the token \"UNK\" (for unknown).\n",
-    "* Finally, we create a word --> index mapping and vice-versa.\n",
-    "* We will then pad all sequences to the be same length as the longest one. "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "HZfK8RhQRPFj"
-   },
-   "outputs": [],
-   "source": [
-    "# This will find the maximum length of any caption in our dataset\n",
-    "def calc_max_length(tensor):\n",
-    "    return max(len(t) for t in tensor)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "oJGE34aiRPFo"
-   },
-   "outputs": [],
-   "source": [
-    "# The steps above is a general process of dealing with text processing\n",
-    "\n",
-    "# choosing the top 5000 words from the vocabulary\n",
-    "top_k = 5000\n",
-    "tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=top_k, \n",
-    "                                                  oov_token=\"<unk>\", \n",
-    "                                                  filters='!\"#$%&()*+.,-/:;=?@[\\]^_`{|}~ ')\n",
-    "tokenizer.fit_on_texts(train_captions)\n",
-    "train_seqs = tokenizer.texts_to_sequences(train_captions)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "8Q44tNQVRPFt"
-   },
-   "outputs": [],
-   "source": [
-    "tokenizer.word_index['<pad>'] = 0"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "0fpJb5ojRPFv"
-   },
-   "outputs": [],
-   "source": [
-    "# creating the tokenized vectors\n",
-    "train_seqs = tokenizer.texts_to_sequences(train_captions)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "AidglIZVRPF4"
-   },
-   "outputs": [],
-   "source": [
-    "# padding each vector to the max_length of the captions\n",
-    "# if the max_length parameter is not provided, pad_sequences calculates that automatically\n",
-    "cap_vector = tf.keras.preprocessing.sequence.pad_sequences(train_seqs, padding='post')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "gL0wkttkRPGA"
-   },
-   "outputs": [],
-   "source": [
-    "# calculating the max_length \n",
-    "# used to store the attention weights\n",
-    "max_length = calc_max_length(train_seqs)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "M3CD75nDpvTI"
-   },
-   "source": [
-    "## Split the data into training and testing"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "iS7DDMszRPGF"
-   },
-   "outputs": [],
-   "source": [
-    "# Create training and validation sets using 80-20 split\n",
-    "img_name_train, img_name_val, cap_train, cap_val = train_test_split(img_name_vector, \n",
-    "                                                                    cap_vector, \n",
-    "                                                                    test_size=0.2, \n",
-    "                                                                    random_state=0)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "XmViPkRFRPGH"
-   },
-   "outputs": [],
-   "source": [
-    "len(img_name_train), len(cap_train), len(img_name_val), len(cap_val)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "uEWM9xrYcg45"
-   },
-   "source": [
-    "## Our images and captions are ready! Next, let's create a tf.data dataset to use for training our model.\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "Q3TnZ1ToRPGV"
-   },
-   "outputs": [],
-   "source": [
-    "# feel free to change these parameters according to your system's configuration\n",
-    "\n",
-    "BATCH_SIZE = 64\n",
-    "BUFFER_SIZE = 1000\n",
-    "embedding_dim = 256\n",
-    "units = 512\n",
-    "vocab_size = len(tokenizer.word_index)\n",
-    "# shape of the vector extracted from InceptionV3 is (64, 2048)\n",
-    "# these two variables represent that\n",
-    "features_shape = 2048\n",
-    "attention_features_shape = 64"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "SmZS2N0bXG3T"
-   },
-   "outputs": [],
-   "source": [
-    "# loading the numpy files \n",
-    "def map_func(img_name, cap):\n",
-    "    img_tensor = np.load(img_name.decode('utf-8')+'.npy')\n",
-    "    return img_tensor, cap"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "FDF_Nm3tRPGZ"
-   },
-   "outputs": [],
-   "source": [
-    "dataset = tf.data.Dataset.from_tensor_slices((img_name_train, cap_train))\n",
-    "\n",
-    "# using map to load the numpy files in parallel\n",
-    "# NOTE: Be sure to set num_parallel_calls to the number of CPU cores you have\n",
-    "# https://www.tensorflow.org/api_docs/python/tf/py_func\n",
-    "dataset = dataset.map(lambda item1, item2: tf.py_func(\n",
-    "          map_func, [item1, item2], [tf.float32, tf.int32]), num_parallel_calls=8)\n",
-    "\n",
-    "# shuffling and batching\n",
-    "dataset = dataset.shuffle(BUFFER_SIZE)\n",
-    "# https://www.tensorflow.org/api_docs/python/tf/contrib/data/batch_and_drop_remainder\n",
-    "dataset = dataset.batch(BATCH_SIZE)\n",
-    "dataset = dataset.prefetch(1)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "nrvoDphgRPGd"
-   },
-   "source": [
-    "## Model\n",
-    "\n",
-    "Fun fact, the decoder below is identical to the one in the example for [Neural Machine Translation with Attention]( https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb).\n",
-    "\n",
-    "The model architecture is inspired by the [Show, Attend and Tell](https://arxiv.org/pdf/1502.03044.pdf) paper.\n",
-    "\n",
-    "* In this example, we extract the features from the lower convolutional layer of InceptionV3 giving us a vector of shape (8, 8, 2048). \n",
-    "* We squash that to a shape of (64, 2048).\n",
-    "* This vector is then passed through the CNN Encoder(which consists of a single Fully connected layer).\n",
-    "* The RNN(here GRU) attends over the image to predict the next word."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "AAppCGLKRPGd"
-   },
-   "outputs": [],
-   "source": [
-    "def gru(units):\n",
-    "  # If you have a GPU, we recommend using the CuDNNGRU layer (it provides a \n",
-    "  # significant speedup).\n",
-    "  if tf.test.is_gpu_available():\n",
-    "    return tf.keras.layers.CuDNNGRU(units, \n",
-    "                                    return_sequences=True, \n",
-    "                                    return_state=True, \n",
-    "                                    recurrent_initializer='glorot_uniform')\n",
-    "  else:\n",
-    "    return tf.keras.layers.GRU(units, \n",
-    "                               return_sequences=True, \n",
-    "                               return_state=True, \n",
-    "                               recurrent_activation='sigmoid', \n",
-    "                               recurrent_initializer='glorot_uniform')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "ja2LFTMSdeV3"
-   },
-   "outputs": [],
-   "source": [
-    "class BahdanauAttention(tf.keras.Model):\n",
-    "  def __init__(self, units):\n",
-    "    super(BahdanauAttention, self).__init__()\n",
-    "    self.W1 = tf.keras.layers.Dense(units)\n",
-    "    self.W2 = tf.keras.layers.Dense(units)\n",
-    "    self.V = tf.keras.layers.Dense(1)\n",
-    "  \n",
-    "  def call(self, features, hidden):\n",
-    "    # features(CNN_encoder output) shape == (batch_size, 64, embedding_dim)\n",
-    "    \n",
-    "    # hidden shape == (batch_size, hidden_size)\n",
-    "    # hidden_with_time_axis shape == (batch_size, 1, hidden_size)\n",
-    "    hidden_with_time_axis = tf.expand_dims(hidden, 1)\n",
-    "    \n",
-    "    # score shape == (batch_size, 64, hidden_size)\n",
-    "    score = tf.nn.tanh(self.W1(features) + self.W2(hidden_with_time_axis))\n",
-    "    \n",
-    "    # attention_weights shape == (batch_size, 64, 1)\n",
-    "    # we get 1 at the last axis because we are applying score to self.V\n",
-    "    attention_weights = tf.nn.softmax(self.V(score), axis=1)\n",
-    "    \n",
-    "    # context_vector shape after sum == (batch_size, hidden_size)\n",
-    "    context_vector = attention_weights * features\n",
-    "    context_vector = tf.reduce_sum(context_vector, axis=1)\n",
-    "    \n",
-    "    return context_vector, attention_weights"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "AZ7R1RxHRPGf"
-   },
-   "outputs": [],
-   "source": [
-    "class CNN_Encoder(tf.keras.Model):\n",
-    "    # Since we have already extracted the features and dumped it using pickle\n",
-    "    # This encoder passes those features through a Fully connected layer\n",
-    "    def __init__(self, embedding_dim):\n",
-    "        super(CNN_Encoder, self).__init__()\n",
-    "        # shape after fc == (batch_size, 64, embedding_dim)\n",
-    "        self.fc = tf.keras.layers.Dense(embedding_dim)\n",
-    "        \n",
-    "    def call(self, x):\n",
-    "        x = self.fc(x)\n",
-    "        x = tf.nn.relu(x)\n",
-    "        return x"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "V9UbGQmERPGi"
-   },
-   "outputs": [],
-   "source": [
-    "class RNN_Decoder(tf.keras.Model):\n",
-    "  def __init__(self, embedding_dim, units, vocab_size):\n",
-    "    super(RNN_Decoder, self).__init__()\n",
-    "    self.units = units\n",
-    "\n",
-    "    self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
-    "    self.gru = gru(self.units)\n",
-    "    self.fc1 = tf.keras.layers.Dense(self.units)\n",
-    "    self.fc2 = tf.keras.layers.Dense(vocab_size)\n",
-    "    \n",
-    "    self.attention = BahdanauAttention(self.units)\n",
-    "        \n",
-    "  def call(self, x, features, hidden):\n",
-    "    # defining attention as a separate model\n",
-    "    context_vector, attention_weights = self.attention(features, hidden)\n",
-    "    \n",
-    "    # x shape after passing through embedding == (batch_size, 1, embedding_dim)\n",
-    "    x = self.embedding(x)\n",
-    "    \n",
-    "    # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)\n",
-    "    x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n",
-    "    \n",
-    "    # passing the concatenated vector to the GRU\n",
-    "    output, state = self.gru(x)\n",
-    "    \n",
-    "    # shape == (batch_size, max_length, hidden_size)\n",
-    "    x = self.fc1(output)\n",
-    "    \n",
-    "    # x shape == (batch_size * max_length, hidden_size)\n",
-    "    x = tf.reshape(x, (-1, x.shape[2]))\n",
-    "    \n",
-    "    # output shape == (batch_size * max_length, vocab)\n",
-    "    x = self.fc2(x)\n",
-    "\n",
-    "    return x, state, attention_weights\n",
-    "\n",
-    "  def reset_state(self, batch_size):\n",
-    "    return tf.zeros((batch_size, self.units))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "Qs_Sr03wRPGk"
-   },
-   "outputs": [],
-   "source": [
-    "encoder = CNN_Encoder(embedding_dim)\n",
-    "decoder = RNN_Decoder(embedding_dim, units, vocab_size)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "-bYN7xA0RPGl"
-   },
-   "outputs": [],
-   "source": [
-    "optimizer = tf.train.AdamOptimizer()\n",
-    "\n",
-    "# We are masking the loss calculated for padding\n",
-    "def loss_function(real, pred):\n",
-    "    mask = 1 - np.equal(real, 0)\n",
-    "    loss_ = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=real, logits=pred) * mask\n",
-    "    return tf.reduce_mean(loss_)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "PHod7t72RPGn"
-   },
-   "source": [
-    "## Training\n",
-    "\n",
-    "* We extract the features stored in the respective `.npy` files and then pass those features through the encoder.\n",
-    "* The encoder output, hidden state(initialized to 0) and the decoder input (which is the start token) is passed to the decoder.\n",
-    "* The decoder returns the predictions and the decoder hidden state.\n",
-    "* The decoder hidden state is then passed back into the model and the predictions are used to calculate the loss.\n",
-    "* Use teacher forcing to decide the next input to the decoder.\n",
-    "* Teacher forcing is the technique where the target word is passed as the next input to the decoder.\n",
-    "* The final step is to calculate the gradients and apply it to the optimizer and backpropagate.\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "Vt4WZ5mhJE-E"
-   },
-   "outputs": [],
-   "source": [
-    "# adding this in a separate cell because if you run the training cell \n",
-    "# many times, the loss_plot array will be reset\n",
-    "loss_plot = []"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "UlA4VIQpRPGo"
-   },
-   "outputs": [],
-   "source": [
-    "EPOCHS = 20\n",
-    "\n",
-    "for epoch in range(EPOCHS):\n",
-    "    start = time.time()\n",
-    "    total_loss = 0\n",
-    "    \n",
-    "    for (batch, (img_tensor, target)) in enumerate(dataset):\n",
-    "        loss = 0\n",
-    "        \n",
-    "        # initializing the hidden state for each batch\n",
-    "        # because the captions are not related from image to image\n",
-    "        hidden = decoder.reset_state(batch_size=target.shape[0])\n",
-    "\n",
-    "        dec_input = tf.expand_dims([tokenizer.word_index['<start>']] * BATCH_SIZE, 1)\n",
-    "        \n",
-    "        with tf.GradientTape() as tape:\n",
-    "            features = encoder(img_tensor)\n",
-    "            \n",
-    "            for i in range(1, target.shape[1]):\n",
-    "                # passing the features through the decoder\n",
-    "                predictions, hidden, _ = decoder(dec_input, features, hidden)\n",
-    "\n",
-    "                loss += loss_function(target[:, i], predictions)\n",
-    "                \n",
-    "                # using teacher forcing\n",
-    "                dec_input = tf.expand_dims(target[:, i], 1)\n",
-    "        \n",
-    "        total_loss += (loss / int(target.shape[1]))\n",
-    "        \n",
-    "        variables = encoder.variables + decoder.variables\n",
-    "        \n",
-    "        gradients = tape.gradient(loss, variables) \n",
-    "        \n",
-    "        optimizer.apply_gradients(zip(gradients, variables), tf.train.get_or_create_global_step())\n",
-    "        \n",
-    "        if batch % 100 == 0:\n",
-    "            print ('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1, \n",
-    "                                                          batch, \n",
-    "                                                          loss.numpy() / int(target.shape[1])))\n",
-    "    # storing the epoch end loss value to plot later\n",
-    "    loss_plot.append(total_loss / len(cap_vector))\n",
-    "    \n",
-    "    print ('Epoch {} Loss {:.6f}'.format(epoch + 1, \n",
-    "                                         total_loss/len(cap_vector)))\n",
-    "    print ('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "1Wm83G-ZBPcC"
-   },
-   "outputs": [],
-   "source": [
-    "plt.plot(loss_plot)\n",
-    "plt.xlabel('Epochs')\n",
-    "plt.ylabel('Loss')\n",
-    "plt.title('Loss Plot')\n",
-    "plt.show()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "xGvOcLQKghXN"
-   },
-   "source": [
-    "## Caption!\n",
-    "\n",
-    "* The evaluate function is similar to the training loop, except we don't use teacher forcing here. The input to the decoder at each time step is its previous predictions along with the hidden state and the encoder output.\n",
-    "* Stop predicting when the model predicts the end token.\n",
-    "* And store the attention weights for every time step."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "RCWpDtyNRPGs"
-   },
-   "outputs": [],
-   "source": [
-    "def evaluate(image):\n",
-    "    attention_plot = np.zeros((max_length, attention_features_shape))\n",
-    "\n",
-    "    hidden = decoder.reset_state(batch_size=1)\n",
-    "\n",
-    "    temp_input = tf.expand_dims(load_image(image)[0], 0)\n",
-    "    img_tensor_val = image_features_extract_model(temp_input)\n",
-    "    img_tensor_val = tf.reshape(img_tensor_val, (img_tensor_val.shape[0], -1, img_tensor_val.shape[3]))\n",
-    "\n",
-    "    features = encoder(img_tensor_val)\n",
-    "\n",
-    "    dec_input = tf.expand_dims([tokenizer.word_index['<start>']], 0)\n",
-    "    result = []\n",
-    "\n",
-    "    for i in range(max_length):\n",
-    "        predictions, hidden, attention_weights = decoder(dec_input, features, hidden)\n",
-    "\n",
-    "        attention_plot[i] = tf.reshape(attention_weights, (-1, )).numpy()\n",
-    "\n",
-    "        predicted_id = tf.argmax(predictions[0]).numpy()\n",
-    "        result.append(tokenizer.index_word[predicted_id])\n",
-    "\n",
-    "        if tokenizer.index_word[predicted_id] == '<end>':\n",
-    "            return result, attention_plot\n",
-    "\n",
-    "        dec_input = tf.expand_dims([predicted_id], 0)\n",
-    "\n",
-    "    attention_plot = attention_plot[:len(result), :]\n",
-    "    return result, attention_plot"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "fD_y7PD6RPGt"
-   },
-   "outputs": [],
-   "source": [
-    "def plot_attention(image, result, attention_plot):\n",
-    "    temp_image = np.array(Image.open(image))\n",
-    "\n",
-    "    fig = plt.figure(figsize=(10, 10))\n",
-    "    \n",
-    "    len_result = len(result)\n",
-    "    for l in range(len_result):\n",
-    "        temp_att = np.resize(attention_plot[l], (8, 8))\n",
-    "        ax = fig.add_subplot(len_result//2, len_result//2, l+1)\n",
-    "        ax.set_title(result[l])\n",
-    "        img = ax.imshow(temp_image)\n",
-    "        ax.imshow(temp_att, cmap='gray', alpha=0.6, extent=img.get_extent())\n",
-    "\n",
-    "    plt.tight_layout()\n",
-    "    plt.show()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "K2s1A9eLRPEj"
+      },
+      "source": [
+        "##### Copyright 2018 The TensorFlow Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\").\n"
+      ]
     },
-    "colab_type": "code",
-    "id": "io7ws3ReRPGv"
-   },
-   "outputs": [],
-   "source": [
-    "# captions on the validation set\n",
-    "rid = np.random.randint(0, len(img_name_val))\n",
-    "image = img_name_val[rid]\n",
-    "real_caption = ' '.join([tokenizer.index_word[i] for i in cap_val[rid] if i not in [0]])\n",
-    "result, attention_plot = evaluate(image)\n",
-    "\n",
-    "print ('Real Caption:', real_caption)\n",
-    "print ('Prediction Caption:', ' '.join(result))\n",
-    "plot_attention(image, result, attention_plot)\n",
-    "# opening the image\n",
-    "Image.open(img_name_val[rid])"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "Rprk3HEvZuxb"
-   },
-   "source": [
-    "## Try it on your own images\n",
-    "For fun, below we've provided a method you can use to caption your own images with the model we've just trained. Keep in mind, it was trained on a relatively small amount of data, and your images may be different from the training data (so be prepared for weird results!)\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "Cffg2i257iMS"
+      },
+      "source": [
+        "# Image Captioning with Attention\n",
+        "\n",
+        "This example has moved:\n",
+        "\n",
+        "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\u003ctd\u003e\n",
+        "\u003ca target=\"_blank\"  href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/r2/tutorials/generative/image_captioning.ipynb\"\u003e\n",
+        "    \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e  \n",
+        "\u003c/td\u003e\u003ctd\u003e\n",
+        "\u003ca target=\"_blank\"  href=\"https://github.com/tensorflow/docs/blob/master/site/en/r2/tutorials/generative/image_captioning.ipynb\"\u003e\u003cimg width=32px src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\u003c/td\u003e\u003c/table\u003e"
+      ]
     },
-    "colab_type": "code",
-    "id": "9Psd1quzaAWg"
-   },
-   "outputs": [],
-   "source": [
-    "image_url = 'https://tensorflow.org/images/surf.jpg'\n",
-    "image_extension = image_url[-4:]\n",
-    "image_path = tf.keras.utils.get_file('image'+image_extension, \n",
-    "                                     origin=image_url)\n",
-    "\n",
-    "result, attention_plot = evaluate(image_path)\n",
-    "print ('Prediction Caption:', ' '.join(result))\n",
-    "plot_attention(image_path, result, attention_plot)\n",
-    "# opening the image\n",
-    "Image.open(image_path)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "VJZXyJco6uLO"
-   },
-   "source": [
-    "# Next steps\n",
-    "\n",
-    "Congrats! You've just trained an image captioning model with attention. Next, we recommend taking a look at this example [Neural Machine Translation with Attention]( https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb). It uses a similar architecture to translate between Spanish and English sentences. You can also experiment with training the code in this notebook on a different dataset."
-   ]
-  }
- ],
- "metadata": {
-  "accelerator": "GPU",
-  "colab": {
-   "collapsed_sections": [],
-   "default_view": {},
-   "name": "image_captioning_with_attention.ipynb",
-   "private_outputs": true,
-   "provenance": [
     {
-     "file_id": "1HI8OK2sMjcx9CTWVn0122QAHOuXaOaMg",
-     "timestamp": 1530222436922
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "QASbY_HGo4Lq"
+      },
+      "source": [
+        "![Man Surfing](https://tensorflow.org/images/surf.jpg) \n",
+        "\n",
+        "[Image Source](https://commons.wikimedia.org/wiki/Surfing#/media/File:Surfing_in_Hawaii.jpg), License: Public Domain\n",
+        "\n",
+        "![Prediction](https://tensorflow.org/images/imcap_prediction.png)\n"
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "collapsed_sections": [],
+      "name": "image_captioning_with_attention.ipynb",
+      "private_outputs": true,
+      "provenance": [
+        {
+          "file_id": "1HI8OK2sMjcx9CTWVn0122QAHOuXaOaMg",
+          "timestamp": 1530222436922
+        }
+      ],
+      "toc_visible": true,
+      "version": "0.3.2"
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "language": "python",
+      "name": "python3"
     }
-   ],
-   "toc_visible": true,
-   "version": "0.3.2",
-   "views": {}
-  },
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
   },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.6.3"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
+  "nbformat": 4,
+  "nbformat_minor": 0
 }
diff --git a/tensorflow/contrib/eager/python/examples/generative_examples/text_generation.ipynb b/tensorflow/contrib/eager/python/examples/generative_examples/text_generation.ipynb
index bda9e77085..c945c753b3 100644
--- a/tensorflow/contrib/eager/python/examples/generative_examples/text_generation.ipynb
+++ b/tensorflow/contrib/eager/python/examples/generative_examples/text_generation.ipynb
@@ -13,633 +13,13 @@
         "\n",
         "# Text Generation using a RNN\n",
         "\n",
+        "This example has moved.\n",
+        "\n",
         "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\u003ctd\u003e\n",
-        "\u003ca target=\"_blank\"  href=\"https://colab.research.google.com/github/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/generative_examples/text_generation.ipynb\"\u003e\n",
+        "\u003ca target=\"_blank\"  href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/sequences/text_generation.ipynb\"\u003e\n",
         "    \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e  \n",
         "\u003c/td\u003e\u003ctd\u003e\n",
-        "\u003ca target=\"_blank\"  href=\"https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples/generative_examples/text_generation.ipynb\"\u003e\u003cimg width=32px src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on Github\u003c/a\u003e\u003c/td\u003e\u003c/table\u003e"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "BwpJ5IffzRG6"
-      },
-      "source": [
-        "This notebook demonstrates how to generate text using an RNN using [tf.keras](https://www.tensorflow.org/programmers_guide/keras) and [eager execution](https://www.tensorflow.org/programmers_guide/eager). If you like, you can write a similar [model](https://github.com/fchollet/deep-learning-with-python-notebooks/blob/master/8.1-text-generation-with-lstm.ipynb) using less code. Here, we show a lower-level impementation that's useful to understand as prework before diving in to deeper examples in a similar, like [Neural Machine Translation with Attention](https://colab.research.google.com/github/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb).\n",
-        "\n",
-        "This notebook is an end-to-end example. When you run it, it will download a dataset of Shakespeare's writing. We'll use a collection of plays, borrowed from Andrej Karpathy's excellent [The Unreasonable Effectiveness of Recurrent Neural Networks](http://karpathy.github.io/2015/05/21/rnn-effectiveness/).  The notebook will train a model, and use it to generate sample output.\n",
-        "  \n",
-        "Here is the output(with start string='w') after training a single layer GRU for 30 epochs with the default settings below:\n",
-        "\n",
-        "```\n",
-        "were to the death of him\n",
-        "And nothing of the field in the view of hell,\n",
-        "When I said, banish him, I will not burn thee that would live.\n",
-        "\n",
-        "HENRY BOLINGBROKE:\n",
-        "My gracious uncle--\n",
-        "\n",
-        "DUKE OF YORK:\n",
-        "As much disgraced to the court, the gods them speak,\n",
-        "And now in peace himself excuse thee in the world.\n",
-        "\n",
-        "HORTENSIO:\n",
-        "Madam, 'tis not the cause of the counterfeit of the earth,\n",
-        "And leave me to the sun that set them on the earth\n",
-        "And leave the world and are revenged for thee.\n",
-        "\n",
-        "GLOUCESTER:\n",
-        "I would they were talking with the very name of means\n",
-        "To make a puppet of a guest, and therefore, good Grumio,\n",
-        "Nor arm'd to prison, o' the clouds, of the whole field,\n",
-        "With the admire\n",
-        "With the feeding of thy chair, and we have heard it so,\n",
-        "I thank you, sir, he is a visor friendship with your silly your bed.\n",
-        "\n",
-        "SAMPSON:\n",
-        "I do desire to live, I pray: some stand of the minds, make thee remedies\n",
-        "With the enemies of my soul.\n",
-        "\n",
-        "MENENIUS:\n",
-        "I'll keep the cause of my mistress.\n",
-        "\n",
-        "POLIXENES:\n",
-        "My brother Marcius!\n",
-        "\n",
-        "Second Servant:\n",
-        "Will't ple\n",
-        "```\n",
-        "\n",
-        "Of course, while some of the sentences are grammatical, most do not make sense. But, consider:\n",
-        "\n",
-        "* Our model is character based (when we began training, it did not yet know how to spell a valid English word, or that words were even a unit of text).\n",
-        "\n",
-        "* The structure of the output resembles a play (blocks begin with a speaker name, in all caps similar to the original text). Sentences generally end with a period. If you look at the text from a distance (or don't read the invididual words too closely, it appears as if it's an excerpt from a play).\n",
-        "\n",
-        "As a next step, you can experiment training the model on a different dataset - any large text file(ASCII) will do, and you can modify a single line of code below to make that change. Have fun!\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "R3p22DBDsaCA"
-      },
-      "source": [
-        "## Install unidecode library\n",
-        "A helpful library to convert unicode to ASCII."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "wZ6LOM12wKGH"
-      },
-      "outputs": [],
-      "source": [
-        "!pip install unidecode"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "WGyKZj3bzf9p"
-      },
-      "source": [
-        "## Import tensorflow and enable eager execution."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "yG_n40gFzf9s"
-      },
-      "outputs": [],
-      "source": [
-        "# Import TensorFlow \u003e= 1.10 and enable eager execution\n",
-        "import tensorflow as tf\n",
-        "\n",
-        "# Note: Once you enable eager execution, it cannot be disabled. \n",
-        "tf.enable_eager_execution()\n",
-        "\n",
-        "import numpy as np\n",
-        "import os\n",
-        "import re\n",
-        "import random\n",
-        "import unidecode\n",
-        "import time"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "EHDoRoc5PKWz"
-      },
-      "source": [
-        "## Download the dataset\n",
-        "\n",
-        "In this example, we will use the [shakespeare dataset](https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt). You can use any other dataset that you like.\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "pD_55cOxLkAb"
-      },
-      "outputs": [],
-      "source": [
-        "path_to_file = tf.keras.utils.get_file('shakespeare.txt', 'https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt')"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "UHjdCjDuSvX_"
-      },
-      "source": [
-        "## Read the dataset\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "-E5JvY3wzf94"
-      },
-      "outputs": [],
-      "source": [
-        "text = unidecode.unidecode(open(path_to_file).read())\n",
-        "# length of text is the number of characters in it\n",
-        "print (len(text))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Il9ww98izf-D"
-      },
-      "source": [
-        "Creating dictionaries to map from characters to their indices and vice-versa, which will be used to vectorize the inputs"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "IalZLbvOzf-F"
-      },
-      "outputs": [],
-      "source": [
-        "# unique contains all the unique characters in the file\n",
-        "unique = sorted(set(text))\n",
-        "\n",
-        "# creating a mapping from unique characters to indices\n",
-        "char2idx = {u:i for i, u in enumerate(unique)}\n",
-        "idx2char = {i:u for i, u in enumerate(unique)}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "1v_qUYfAzf-I"
-      },
-      "outputs": [],
-      "source": [
-        "# setting the maximum length sentence we want for a single input in characters\n",
-        "max_length = 100\n",
-        "\n",
-        "# length of the vocabulary in chars\n",
-        "vocab_size = len(unique)\n",
-        "\n",
-        "# the embedding dimension \n",
-        "embedding_dim = 256\n",
-        "\n",
-        "# number of RNN (here GRU) units\n",
-        "units = 1024\n",
-        "\n",
-        "# batch size \n",
-        "BATCH_SIZE = 64\n",
-        "\n",
-        "# buffer size to shuffle our dataset\n",
-        "BUFFER_SIZE = 10000"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "LFjSVAlWzf-N"
-      },
-      "source": [
-        "## Creating the input and output tensors\n",
-        "\n",
-        "Vectorizing the input and the target text because our model cannot understand strings only numbers.\n",
-        "\n",
-        "But first, we need to create the input and output vectors.\n",
-        "Remember the max_length we set above, we will use it here. We are creating **max_length** chunks of input, where each input vector is all the characters in that chunk except the last and the target vector is all the characters in that chunk except the first.\n",
-        "\n",
-        "For example, consider that the string = 'tensorflow' and the max_length is 9\n",
-        "\n",
-        "So, the `input = 'tensorflo'` and `output = 'ensorflow'`\n",
-        "\n",
-        "After creating the vectors, we convert each character into numbers using the **char2idx** dictionary we created above."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "0UHJDA39zf-O"
-      },
-      "outputs": [],
-      "source": [
-        "input_text = []\n",
-        "target_text = []\n",
-        "\n",
-        "for f in range(0, len(text)-max_length, max_length):\n",
-        "    inps = text[f:f+max_length]\n",
-        "    targ = text[f+1:f+1+max_length]\n",
-        "\n",
-        "    input_text.append([char2idx[i] for i in inps])\n",
-        "    target_text.append([char2idx[t] for t in targ])\n",
-        "    \n",
-        "print (np.array(input_text).shape)\n",
-        "print (np.array(target_text).shape)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "MJdfPmdqzf-R"
-      },
-      "source": [
-        "## Creating batches and shuffling them using tf.data"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "p2pGotuNzf-S"
-      },
-      "outputs": [],
-      "source": [
-        "dataset = tf.data.Dataset.from_tensor_slices((input_text, target_text)).shuffle(BUFFER_SIZE)\n",
-        "dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "m8gPwEjRzf-Z"
-      },
-      "source": [
-        "## Creating the model\n",
-        "\n",
-        "We use the Model Subclassing API which gives us full flexibility to create the model and change it however we like. We use 3 layers to define our model.\n",
-        "\n",
-        "* Embedding layer\n",
-        "* GRU layer (you can use an LSTM layer here)\n",
-        "* Fully connected layer"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "P3KTiiInzf-a"
-      },
-      "outputs": [],
-      "source": [
-        "class Model(tf.keras.Model):\n",
-        "  def __init__(self, vocab_size, embedding_dim, units, batch_size):\n",
-        "    super(Model, self).__init__()\n",
-        "    self.units = units\n",
-        "    self.batch_sz = batch_size\n",
-        "\n",
-        "    self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
-        "\n",
-        "    if tf.test.is_gpu_available():\n",
-        "      self.gru = tf.keras.layers.CuDNNGRU(self.units, \n",
-        "                                          return_sequences=True, \n",
-        "                                          return_state=True, \n",
-        "                                          recurrent_initializer='glorot_uniform')\n",
-        "    else:\n",
-        "      self.gru = tf.keras.layers.GRU(self.units, \n",
-        "                                     return_sequences=True, \n",
-        "                                     return_state=True, \n",
-        "                                     recurrent_activation='sigmoid', \n",
-        "                                     recurrent_initializer='glorot_uniform')\n",
-        "\n",
-        "    self.fc = tf.keras.layers.Dense(vocab_size)\n",
-        "        \n",
-        "  def call(self, x, hidden):\n",
-        "    x = self.embedding(x)\n",
-        "\n",
-        "    # output shape == (batch_size, max_length, hidden_size) \n",
-        "    # states shape == (batch_size, hidden_size)\n",
-        "\n",
-        "    # states variable to preserve the state of the model\n",
-        "    # this will be used to pass at every step to the model while training\n",
-        "    output, states = self.gru(x, initial_state=hidden)\n",
-        "\n",
-        "\n",
-        "    # reshaping the output so that we can pass it to the Dense layer\n",
-        "    # after reshaping the shape is (batch_size * max_length, hidden_size)\n",
-        "    output = tf.reshape(output, (-1, output.shape[2]))\n",
-        "\n",
-        "    # The dense layer will output predictions for every time_steps(max_length)\n",
-        "    # output shape after the dense layer == (max_length * batch_size, vocab_size)\n",
-        "    x = self.fc(output)\n",
-        "\n",
-        "    return x, states"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "trpqTWyvk0nr"
-      },
-      "source": [
-        "## Call the model and set the optimizer and the loss function"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "7t2XrzEOzf-e"
-      },
-      "outputs": [],
-      "source": [
-        "model = Model(vocab_size, embedding_dim, units, BATCH_SIZE)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "dkjWIATszf-h"
-      },
-      "outputs": [],
-      "source": [
-        "optimizer = tf.train.AdamOptimizer()\n",
-        "\n",
-        "# using sparse_softmax_cross_entropy so that we don't have to create one-hot vectors\n",
-        "def loss_function(real, preds):\n",
-        "    return tf.losses.sparse_softmax_cross_entropy(labels=real, logits=preds)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "3K6s6F79P7za"
-      },
-      "source": [
-        "## Checkpoints (Object-based saving)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "oAGisDdfP9rL"
-      },
-      "outputs": [],
-      "source": [
-        "checkpoint_dir = './training_checkpoints'\n",
-        "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n",
-        "checkpoint = tf.train.Checkpoint(optimizer=optimizer,\n",
-        "                                 model=model)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "lPrP0XMUzf-p"
-      },
-      "source": [
-        "## Train the model\n",
-        "\n",
-        "Here we will use a custom training loop with the help of GradientTape()\n",
-        "\n",
-        "* We initialize the hidden state of the model with zeros and shape == (batch_size, number of rnn units). We do this by calling the function defined while creating the model.\n",
-        "\n",
-        "* Next, we iterate over the dataset(batch by batch) and calculate the **predictions and the hidden states** associated with that input.\n",
-        "\n",
-        "* There are a lot of interesting things happening here.\n",
-        "  * The model gets hidden state(initialized with 0), lets call that **H0** and the first batch of input, lets call that **I0**.\n",
-        "  * The model then returns the predictions **P1** and **H1**.\n",
-        "  * For the next batch of input, the model receives **I1** and **H1**.\n",
-        "  * The interesting thing here is that we pass **H1** to the model with **I1** which is how the model learns. The context learned from batch to batch is contained in the **hidden state**.\n",
-        "  * We continue doing this until the dataset is exhausted and then we start a new epoch and repeat this.\n",
-        "\n",
-        "* After calculating the predictions, we calculate the **loss** using the loss function defined above. Then we calculate the gradients of the loss with respect to the model variables(input)\n",
-        "\n",
-        "* Finally, we take a step in that direction with the help of the optimizer using the apply_gradients function.\n",
-        "\n",
-        "Note:- If you are running this notebook in Colab which has a **Tesla K80 GPU** it takes about 23 seconds per epoch.\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "d4tSNwymzf-q"
-      },
-      "outputs": [],
-      "source": [
-        "# Training step\n",
-        "\n",
-        "EPOCHS = 20\n",
-        "\n",
-        "for epoch in range(EPOCHS):\n",
-        "    start = time.time()\n",
-        "    \n",
-        "    # initializing the hidden state at the start of every epoch\n",
-        "    hidden = model.reset_states()\n",
-        "    \n",
-        "    for (batch, (inp, target)) in enumerate(dataset):\n",
-        "          with tf.GradientTape() as tape:\n",
-        "              # feeding the hidden state back into the model\n",
-        "              # This is the interesting step\n",
-        "              predictions, hidden = model(inp, hidden)\n",
-        "              \n",
-        "              # reshaping the target because that's how the \n",
-        "              # loss function expects it\n",
-        "              target = tf.reshape(target, (-1,))\n",
-        "              loss = loss_function(target, predictions)\n",
-        "              \n",
-        "          grads = tape.gradient(loss, model.variables)\n",
-        "          optimizer.apply_gradients(zip(grads, model.variables))\n",
-        "\n",
-        "          if batch % 100 == 0:\n",
-        "              print ('Epoch {} Batch {} Loss {:.4f}'.format(epoch+1,\n",
-        "                                                            batch,\n",
-        "                                                            loss))\n",
-        "    # saving (checkpoint) the model every 5 epochs\n",
-        "    if (epoch + 1) % 5 == 0:\n",
-        "      checkpoint.save(file_prefix = checkpoint_prefix)\n",
-        "\n",
-        "    print ('Epoch {} Loss {:.4f}'.format(epoch+1, loss))\n",
-        "    print('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "01AR9vpNQMFF"
-      },
-      "source": [
-        "## Restore the latest checkpoint"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "tyvpYomYQQkF"
-      },
-      "outputs": [],
-      "source": [
-        "# restoring the latest checkpoint in checkpoint_dir\n",
-        "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "DjGz1tDkzf-u"
-      },
-      "source": [
-        "## Predicting using our trained model\n",
-        "\n",
-        "The below code block is used to generated the text\n",
-        "\n",
-        "* We start by choosing a start string and initializing the hidden state and setting the number of characters we want to generate.\n",
-        "\n",
-        "* We get predictions using the start_string and the hidden state\n",
-        "\n",
-        "* Then we use argmax to calculate the index of the predicted word. **We use this predicted word as our next input to the model**\n",
-        "\n",
-        "* **The hidden state returned by the model is fed back into the model so that it now has more context rather than just one word.** After we predict the next word, the modified hidden states are again fed back into the model, which is how it learns as it gets more context from the previously predicted words.\n",
-        "\n",
-        "* If you see the predictions, the model knows when to capitalize, make paragraphs and the text follows a shakespeare style of writing which is pretty awesome!"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "WvuwZBX5Ogfd"
-      },
-      "outputs": [],
-      "source": [
-        "# Evaluation step(generating text using the model learned)\n",
-        "\n",
-        "# number of characters to generate\n",
-        "num_generate = 1000\n",
-        "\n",
-        "# You can change the start string to experiment\n",
-        "start_string = 'Q'\n",
-        "# converting our start string to numbers(vectorizing!) \n",
-        "input_eval = [char2idx[s] for s in start_string]\n",
-        "input_eval = tf.expand_dims(input_eval, 0)\n",
-        "\n",
-        "# empty string to store our results\n",
-        "text_generated = ''\n",
-        "\n",
-        "# hidden state shape == (batch_size, number of rnn units); here batch size == 1\n",
-        "hidden = [tf.zeros((1, units))]\n",
-        "for i in range(num_generate):\n",
-        "    predictions, hidden = model(input_eval, hidden)\n",
-        "\n",
-        "    # using argmax to predict the word returned by the model\n",
-        "    predicted_id = tf.argmax(predictions[-1]).numpy()\n",
-        "    \n",
-        "    # We pass the predicted word as the next input to the model\n",
-        "    # along with the previous hidden state\n",
-        "    input_eval = tf.expand_dims([predicted_id], 0)\n",
-        "    \n",
-        "    text_generated += idx2char[predicted_id]\n",
-        "\n",
-        "print (start_string + text_generated)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "AM2Uma_-yVIq"
-      },
-      "source": [
-        "## Next steps\n",
-        "\n",
-        "* Change the start string to a different character, or the start of a sentence.\n",
-        "* Experiment with training on a different, or with different parameters. [Project  Gutenberg](http://www.gutenberg.org/ebooks/100), for example, contains a large collection of books.\n",
-        "* Add another RNN layer.\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "gtEd86sX5cB2"
-      },
-      "outputs": [],
-      "source": [
-        ""
+        "\u003ca target=\"_blank\"  href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/sequences/text_generation.ipynb\"\u003e\u003cimg width=32px src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\u003c/td\u003e\u003c/table\u003e"
       ]
     }
   ],
-- 
GitLab


From ae9a75e6733966881a26a3cc371039ae2eacff48 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Fri, 25 Jan 2019 16:45:28 -0800
Subject: [PATCH 1316/2345] Enable the save and load test with optimizer v1

PiperOrigin-RevId: 230993214
---
 tensorflow/contrib/distribute/python/keras_test.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 755ce23e7f..17ed871459 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -1178,20 +1178,19 @@ class TestDistributionStrategySaveLoadWeights(test.TestCase,
 
   @combinations.generate(all_strategy_combinations_minus_default())
   def test_save_load_checkpointable(self, distribution):
-    # TODO(sourabhbajaj): Fix this and enable the test.
-    self.skipTest('Restore for checkpoint for optimizer weights fails.')
+    # TODO(sourabhbajaj): Test fails with optimizer v2 without h5
     with self.cached_session():
       dataset = get_dataset(distribution)
       with distribution.scope():
         model = get_model()
-        model.compile(gradient_descent_keras.SGD(0.01), 'mse')
+        model.compile(gradient_descent.GradientDescentOptimizer(0.01), 'mse')
         model.fit(dataset, epochs=1, steps_per_epoch=1)
 
         weights_file = tempfile.mktemp()
         model.save_weights(weights_file)
 
         model_2 = get_model()
-        model_2.compile(gradient_descent_keras.SGD(0.01), 'mse')
+        model_2.compile(gradient_descent.GradientDescentOptimizer(0.01), 'mse')
         model_2.load_weights(weights_file)
         model_2.predict(get_predict_dataset(distribution), steps=2)
         model_2.fit(dataset, epochs=1, steps_per_epoch=1)
-- 
GitLab


From b8898d894c3abc5c7278f7473ec0d944f05c44e3 Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Fri, 25 Jan 2019 16:53:18 -0800
Subject: [PATCH 1317/2345] Export serialize/deserialize methods for
 `tf.keras.layers`.

PiperOrigin-RevId: 230994306
---
 tensorflow/python/keras/layers/serialization.py           | 3 +++
 .../tools/api/golden/v1/tensorflow.keras.layers.pbtxt     | 8 ++++++++
 .../tools/api/golden/v2/tensorflow.keras.layers.pbtxt     | 8 ++++++++
 3 files changed, 19 insertions(+)

diff --git a/tensorflow/python/keras/layers/serialization.py b/tensorflow/python/keras/layers/serialization.py
index e4c508b0f3..932cc58e03 100644
--- a/tensorflow/python/keras/layers/serialization.py
+++ b/tensorflow/python/keras/layers/serialization.py
@@ -37,12 +37,15 @@ from tensorflow.python.keras.layers.pooling import *
 from tensorflow.python.keras.layers.recurrent import *
 from tensorflow.python.keras.layers.wrappers import *
 from tensorflow.python.keras.utils.generic_utils import deserialize_keras_object
+from tensorflow.python.util.tf_export import keras_export
 
 
+@keras_export('keras.layers.serialize')
 def serialize(layer):
   return {'class_name': layer.__class__.__name__, 'config': layer.get_config()}
 
 
+@keras_export('keras.layers.deserialize')
 def deserialize(config, custom_objects=None):
   """Instantiates a layer from a config dictionary.
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.pbtxt
index 2c5fd644d4..ad74ab3f12 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.pbtxt
@@ -416,6 +416,10 @@ tf_module {
     name: "concatenate"
     argspec: "args=[\'inputs\', \'axis\'], varargs=None, keywords=kwargs, defaults=[\'-1\'], "
   }
+  member_method {
+    name: "deserialize"
+    argspec: "args=[\'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "dot"
     argspec: "args=[\'inputs\', \'axes\', \'normalize\'], varargs=None, keywords=kwargs, defaults=[\'False\'], "
@@ -432,6 +436,10 @@ tf_module {
     name: "multiply"
     argspec: "args=[\'inputs\'], varargs=None, keywords=kwargs, defaults=None"
   }
+  member_method {
+    name: "serialize"
+    argspec: "args=[\'layer\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "subtract"
     argspec: "args=[\'inputs\'], varargs=None, keywords=kwargs, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.pbtxt
index 5cf80b97ae..f878c46059 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.pbtxt
@@ -416,6 +416,10 @@ tf_module {
     name: "concatenate"
     argspec: "args=[\'inputs\', \'axis\'], varargs=None, keywords=kwargs, defaults=[\'-1\'], "
   }
+  member_method {
+    name: "deserialize"
+    argspec: "args=[\'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "dot"
     argspec: "args=[\'inputs\', \'axes\', \'normalize\'], varargs=None, keywords=kwargs, defaults=[\'False\'], "
@@ -432,6 +436,10 @@ tf_module {
     name: "multiply"
     argspec: "args=[\'inputs\'], varargs=None, keywords=kwargs, defaults=None"
   }
+  member_method {
+    name: "serialize"
+    argspec: "args=[\'layer\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "subtract"
     argspec: "args=[\'inputs\'], varargs=None, keywords=kwargs, defaults=None"
-- 
GitLab


From 02dfd621d05d690f485b008085a5e9ab92b3434a Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Fri, 25 Jan 2019 17:00:56 -0800
Subject: [PATCH 1318/2345] Wraps losses passed to the `compile` API (strings
 and v1 losses) which are not instances of v2 `Loss` class in `LossWrapper`
 class. => All losses will now use `SUM_OVER_BATCH_SIZE` reduction as default.

PiperOrigin-RevId: 230995424
---
 tensorflow/python/keras/callbacks_test.py     | 15 ++---
 tensorflow/python/keras/engine/training.py    | 48 ++++++++--------
 .../keras/engine/training_dataset_test.py     | 21 ++++---
 .../python/keras/engine/training_eager.py     | 27 ++++-----
 .../python/keras/engine/training_utils.py     | 36 ++++++++----
 tensorflow/python/keras/losses.py             | 57 ++++++++++++++++++-
 tensorflow/python/keras/losses_test.py        | 21 ++++++-
 tensorflow/python/keras/models_test.py        |  5 +-
 .../python/keras/wrappers/scikit_learn.py     |  7 +--
 9 files changed, 160 insertions(+), 77 deletions(-)

diff --git a/tensorflow/python/keras/callbacks_test.py b/tensorflow/python/keras/callbacks_test.py
index d80af49651..6aba08b95f 100644
--- a/tensorflow/python/keras/callbacks_test.py
+++ b/tensorflow/python/keras/callbacks_test.py
@@ -585,13 +585,15 @@ class KerasCallbacksTest(test.TestCase):
             optimizer=keras.optimizers.SGD(lr=0.1))
         return model
 
+      # TODO(psv): Make sure the callback works correctly when min_delta is
+      # set as 0. Test fails when the order of this callback and assertion is
+      # interchanged.
       model = make_model()
-      # This should reduce the LR after the first epoch (due to high epsilon).
       cbks = [
           keras.callbacks.ReduceLROnPlateau(
               monitor='val_loss',
               factor=0.1,
-              min_delta=10,
+              min_delta=0,
               patience=1,
               cooldown=5)
       ]
@@ -604,16 +606,15 @@ class KerasCallbacksTest(test.TestCase):
           epochs=5,
           verbose=0)
       self.assertAllClose(
-          float(keras.backend.get_value(model.optimizer.lr)),
-          0.01,
-          atol=1e-4)
+          float(keras.backend.get_value(model.optimizer.lr)), 0.1, atol=1e-4)
 
       model = make_model()
+      # This should reduce the LR after the first epoch (due to high epsilon).
       cbks = [
           keras.callbacks.ReduceLROnPlateau(
               monitor='val_loss',
               factor=0.1,
-              min_delta=0,
+              min_delta=10,
               patience=1,
               cooldown=5)
       ]
@@ -626,7 +627,7 @@ class KerasCallbacksTest(test.TestCase):
           epochs=5,
           verbose=2)
       self.assertAllClose(
-          float(keras.backend.get_value(model.optimizer.lr)), 0.1, atol=1e-4)
+          float(keras.backend.get_value(model.optimizer.lr)), 0.01, atol=1e-4)
 
   def test_ReduceLROnPlateau_patience(self):
 
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 41abd3c4ad..ba351cc56b 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -464,21 +464,18 @@ class Model(Network):
           mask = masks[i]
           loss_weight = loss_weights_list[i]
           with K.name_scope(self.output_names[i] + '_loss'):
-            if isinstance(loss_fn, losses.Loss):
-              if mask is not None:
-                mask = math_ops.cast(mask, y_pred.dtype)
-                # Update weights with mask.
-                if sample_weight is None:
-                  sample_weight = mask
-                else:
-                  # Update dimensions of weights to match with mask if possible.
-                  mask, _, sample_weight = squeeze_or_expand_dimensions(
-                      mask, None, sample_weight)
-                  sample_weight *= mask
-              output_loss = loss_fn(y_true, y_pred, sample_weight=sample_weight)
-            else:
-              weighted_loss = training_utils.weighted_masked_objective(loss_fn)
-              output_loss = weighted_loss(y_true, y_pred, sample_weight, mask)
+            if mask is not None:
+              mask = math_ops.cast(mask, y_pred.dtype)
+              # Update weights with mask.
+              if sample_weight is None:
+                sample_weight = mask
+              else:
+                # Update dimensions of weights to match with mask if possible.
+                mask, _, sample_weight = squeeze_or_expand_dimensions(
+                    mask, None, sample_weight)
+                sample_weight *= mask
+
+            output_loss = loss_fn(y_true, y_pred, sample_weight=sample_weight)
 
           if len(self.outputs) > 1:
             # Keep track of the un-aggregated loss result tensor.
@@ -486,10 +483,8 @@ class Model(Network):
                                           '_loss'] = output_loss
 
             # Keep track of stateful result tensor and function for the loss.
-            loss_name = loss_fn.name if isinstance(
-                loss_fn, losses.Loss) else loss_fn.__name__
             mean_wrapped_loss = metrics_module.MeanMetricWrapper(
-                loss_fn, name=loss_name)
+                loss_fn, name=loss_fn.name)
             result_tensor = self._call_metric_fn(mean_wrapped_loss, y_true,
                                                  y_pred, sample_weight, mask)
             self._compile_stateful_metrics_tensors[self.output_names[i] +
@@ -2541,18 +2536,21 @@ class Model(Network):
         feed_output_shapes = []
         for output_shape, loss_fn in zip(self._feed_output_shapes,
                                          self._feed_loss_fns):
-          if loss_fn is losses.sparse_categorical_crossentropy:
+          if ((isinstance(loss_fn, losses.LossFunctionWrapper) and
+               loss_fn.fn == losses.sparse_categorical_crossentropy)) or (
+                   isinstance(loss_fn, losses.SparseCategoricalCrossentropy)):
             if K.image_data_format() == 'channels_first':
               feed_output_shapes.append(
                   (output_shape[0], 1) + output_shape[2:])
             else:
               feed_output_shapes.append(output_shape[:-1] + (1,))
-          elif (not hasattr(loss_fn, '__name__') or
-                getattr(losses, loss_fn.__name__, None) is None):
-            # If `loss_fn` is not a function (e.g. callable class)
-            # or if it not in the `losses` module, then
-            # it is a user-defined loss and we make no assumptions
-            # about it.
+          elif (not isinstance(loss_fn, losses.Loss) or
+                (isinstance(loss_fn, losses.LossFunctionWrapper) and
+                 (getattr(losses, loss_fn.fn.__name__, None) is None))):
+            # If the given loss is not an instance of the `Loss` class (custom
+            # class) or if the loss function that is wrapped is not in the
+            # `losses` module, then it is a user-defined loss and we make no
+            # assumptions about it.
             feed_output_shapes.append(None)
           else:
             feed_output_shapes.append(output_shape)
diff --git a/tensorflow/python/keras/engine/training_dataset_test.py b/tensorflow/python/keras/engine/training_dataset_test.py
index 1c67100f20..5ed07fc131 100644
--- a/tensorflow/python/keras/engine/training_dataset_test.py
+++ b/tensorflow/python/keras/engine/training_dataset_test.py
@@ -30,7 +30,6 @@ from tensorflow.python.framework import test_util as tf_test_util
 from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import metrics as metrics_module
 from tensorflow.python.keras import testing_utils
-from tensorflow.python.ops.losses import losses_impl
 from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging as logging
 
@@ -253,18 +252,18 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
   def test_dataset_with_sparse_labels(self):
     model = testing_utils.get_small_mlp(1, 4, input_dim=3)
     optimizer = 'rmsprop'
-    for loss in ['sparse_categorical_crossentropy',
-                 losses_impl.sparse_softmax_cross_entropy]:
-      model.compile(optimizer, loss,
-                    run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer,
+        loss='sparse_categorical_crossentropy',
+        run_eagerly=testing_utils.should_run_eagerly())
 
-      inputs = np.zeros((10, 3), dtype=np.float32)
-      targets = np.random.randint(0, 4, size=10, dtype=np.int32)
-      dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
-      dataset = dataset.repeat(100)
-      dataset = dataset.batch(10)
+    inputs = np.zeros((10, 3), dtype=np.float32)
+    targets = np.random.randint(0, 4, size=10, dtype=np.int32)
+    dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
+    dataset = dataset.repeat(100)
+    dataset = dataset.batch(10)
 
-      model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1)
+    model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1)
 
   @keras_parameterized.run_all_keras_modes
   def test_dataset_fit_correctness(self):
diff --git a/tensorflow/python/keras/engine/training_eager.py b/tensorflow/python/keras/engine/training_eager.py
index e64c0c5eba..6242b54c0c 100644
--- a/tensorflow/python/keras/engine/training_eager.py
+++ b/tensorflow/python/keras/engine/training_eager.py
@@ -25,7 +25,6 @@ from tensorflow.python.eager.backprop import GradientTape
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.keras import backend
-from tensorflow.python.keras import losses as losses_module
 from tensorflow.python.keras.engine import training_utils
 from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions
 from tensorflow.python.ops import math_ops
@@ -124,22 +123,16 @@ def _model_loss(model,
         weights = None
       mask = masks[i]
       with backend.name_scope(model.output_names[i] + '_loss'):
-        if isinstance(loss_fn, losses_module.Loss):
-          if mask is not None:
-            mask = math_ops.cast(mask, outs[i].dtype)
-            # Update weights with mask.
-            if weights is None:
-              weights = mask
-            else:
-              # Update dimensions of weights to match with mask if possible.
-              mask, _, weights = squeeze_or_expand_dimensions(
-                  mask, None, weights)
-              weights *= mask
-          output_loss = loss_fn(targets[i], outs[i], sample_weight=weights)
-        else:
-          weighted_masked_fn = training_utils.weighted_masked_objective(loss_fn)
-          output_loss = weighted_masked_fn(
-              targets[i], outs[i], weights, mask=mask)
+        if mask is not None:
+          mask = math_ops.cast(mask, outs[i].dtype)
+          # Update weights with mask.
+          if weights is None:
+            weights = mask
+          else:
+            # Update dimensions of weights to match with mask if possible.
+            mask, _, weights = squeeze_or_expand_dimensions(mask, None, weights)
+            weights *= mask
+        output_loss = loss_fn(targets[i], outs[i], sample_weight=weights)
 
       # If the number of outputs is 1 then we don't append the loss metric
       # associated with each model output. When there are multiple outputs
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index 7e5bc08e5e..bbc180be89 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -477,14 +477,16 @@ def check_loss_and_target_compatibility(targets, loss_fns, output_shapes):
       ValueError: if a loss function or target array
           is incompatible with an output.
   """
-  key_losses = {
+  key_loss_fns = {
       losses.mean_squared_error, losses.binary_crossentropy,
       losses.categorical_crossentropy
   }
+  key_loss_classes = (losses.MeanSquaredError, losses.BinaryCrossentropy,
+                      losses.CategoricalCrossentropy)
   for y, loss, shape in zip(targets, loss_fns, output_shapes):
     if y is None or loss is None or tensor_util.is_tensor(y):
       continue
-    if loss is losses.categorical_crossentropy:
+    if losses.is_categorical_crossentropy(loss):
       if y.shape[-1] == 1:
         raise ValueError('You are passing a target array of shape ' + str(
             y.shape) + ' while using as loss `categorical_crossentropy`. '
@@ -501,14 +503,20 @@ def check_loss_and_target_compatibility(targets, loss_fns, output_shapes):
                          'Alternatively, you can use the loss function '
                          '`sparse_categorical_crossentropy` instead, '
                          'which does expect integer targets.')
-    if loss in key_losses:
+
+    is_loss_wrapper = isinstance(loss, losses.LossFunctionWrapper)
+    if (isinstance(loss, key_loss_classes) or (is_loss_wrapper and
+                                               (loss.fn in key_loss_fns))):
       for target_dim, out_dim in zip(y.shape[1:], shape[1:]):
         if out_dim is not None and target_dim != out_dim:
+          loss_name = loss.name
+          if loss_name is None:
+            loss_type = loss.fn if is_loss_wrapper else type(loss)
+            loss_name = loss_type.__name__
           raise ValueError('A target array with shape ' + str(y.shape) +
                            ' was passed for an output of shape ' + str(shape) +
-                           ' while using as loss `' + loss.__name__ + '`. '
-                           'This loss expects '
-                           'targets to have the same shape '
+                           ' while using as loss `' + loss_name + '`. '
+                           'This loss expects targets to have the same shape '
                            'as the output.')
 
 
@@ -875,10 +883,18 @@ def get_loss_function(loss):
   if loss is None or isinstance(loss, losses.Loss):
     return loss
 
-  # TODO(psv): After we have added all V2 losses, update this function.
-  if loss in ['mse', 'MSE', 'mean_squared_error']:
-    return losses.MeanSquaredError()
-  return losses.get(loss)
+  # Deserialize loss configuration, if needed.
+  if isinstance(loss, collections.Mapping):
+    loss = losses.get(loss)
+
+  # Custom callable class.
+  if callable(loss) and not hasattr(loss, '__name__'):
+    return loss
+
+  # Wrap loss function with signature `(y_true, y_pred, **kwargs)`
+  # in `LossFunctionWrapper` class.
+  loss_fn = losses.get(loss)
+  return losses.LossFunctionWrapper(loss_fn, name=loss_fn.__name__)
 
 
 def validate_dataset_input(x, y, sample_weight, validation_split=None):
diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index 5a66be8a56..39063c55c8 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -88,7 +88,10 @@ class Loss(object):
     Raises:
       ValueError: If the shape of `sample_weight` is invalid.
     """
-    with ops.name_scope(self.name, format(self.__class__.__name__),
+    # If we are wrapping a lambda function strip '<>' from the name as it is not
+    # accepted in scope name.
+    scope_name = 'lambda' if self.name == '<lambda>' else self.name
+    with ops.name_scope(scope_name, format(self.__class__.__name__),
                         (y_pred, y_true, sample_weight)):
       losses = self.call(y_true, y_pred)
       return compute_weighted_loss(
@@ -121,6 +124,48 @@ class Loss(object):
     NotImplementedError('Must be implemented in subclasses.')
 
 
+class LossFunctionWrapper(Loss):
+  """Wraps a loss function in the `Loss` class.
+
+  Args:
+    fn: The loss function to wrap, with signature `fn(y_true, y_pred,
+      **kwargs)`.
+    reduction: (Optional) Type of `tf.losses.Reduction` to apply to loss.
+      Default value is `SUM_OVER_BATCH_SIZE`.
+    name: (Optional) name for the loss.
+    **kwargs: The keyword arguments that are passed on to `fn`.
+  """
+
+  def __init__(self,
+               fn,
+               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               name=None,
+               **kwargs):
+    super(LossFunctionWrapper, self).__init__(reduction=reduction, name=name)
+    self.fn = fn
+    self._fn_kwargs = kwargs
+
+  def call(self, y_true, y_pred):
+    """Invokes the `LossFunctionWrapper` instance.
+
+    Args:
+      y_true: Ground truth values.
+      y_pred: The predicted values.
+
+    Returns:
+      Loss values per sample.
+    """
+    y_pred = ops.convert_to_tensor(y_pred)
+    y_true = math_ops.cast(y_true, y_pred.dtype)
+    return self.fn(y_true, y_pred, **self._fn_kwargs)
+
+  def get_config(self):
+    config = {'fn': self.fn}
+    config.update(self._fn_kwargs)
+    base_config = super(LossFunctionWrapper, self).get_config()
+    return dict(list(base_config.items()) + list(config.items()))
+
+
 @keras_export('keras.losses.MeanSquaredError')
 class MeanSquaredError(Loss):
   """Computes the mean of squares of errors between labels and predictions.
@@ -997,6 +1042,16 @@ kld = KLD = kullback_leibler_divergence
 cosine = cosine_proximity
 
 
+def is_categorical_crossentropy(loss):
+  result = ((isinstance(loss, CategoricalCrossentropy) or
+             (isinstance(loss, LossFunctionWrapper) and
+              loss.fn == categorical_crossentropy) or
+             (hasattr(loss, '__name__') and
+              loss.__name__ == 'categorical_crossentropy') or
+             (loss == 'categorical_crossentropy')))
+  return result
+
+
 @keras_export('keras.losses.serialize')
 def serialize(loss):
   return serialize_keras_object(loss)
diff --git a/tensorflow/python/keras/losses_test.py b/tensorflow/python/keras/losses_test.py
index 6b6067828f..04dd71241e 100644
--- a/tensorflow/python/keras/losses_test.py
+++ b/tensorflow/python/keras/losses_test.py
@@ -56,7 +56,7 @@ class _MSEMAELoss(object):
   def __init__(self, mse_fraction):
     self.mse_fraction = mse_fraction
 
-  def __call__(self, y_true, y_pred):
+  def __call__(self, y_true, y_pred, sample_weight=None):
     return (self.mse_fraction * keras.losses.mse(y_true, y_pred) +
             (1 - self.mse_fraction) * keras.losses.mae(y_true, y_pred))
 
@@ -180,6 +180,25 @@ class KerasLossesTest(test.TestCase):
         loaded_model = keras.models.load_model(model_filename)
         loaded_model.predict(np.random.rand(128, 2))
 
+  def test_loss_wrapper(self):
+    loss_fn = keras.losses.get('mse')
+    mse_obj = keras.losses.LossFunctionWrapper(loss_fn, name=loss_fn.__name__)
+
+    self.assertEqual(mse_obj.name, 'mean_squared_error')
+    self.assertEqual(mse_obj.reduction,
+                     losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE)
+
+    y_true = constant_op.constant([[1., 9.], [2., 5.]])
+    y_pred = constant_op.constant([[4., 8.], [12., 3.]])
+    sample_weight = constant_op.constant([1.2, 0.5])
+    loss = mse_obj(y_true, y_pred, sample_weight=sample_weight)
+
+    # mse = [((4 - 1)^2 + (8 - 9)^2) / 2, ((12 - 2)^2 + (3 - 5)^2) / 2]
+    # mse = [5, 52]
+    # weighted_mse = [5 * 1.2, 52 * 0.5] = [6, 26]
+    # reduced_weighted_mse = (6 + 26) / 2 =
+    self.assertAllClose(self.evaluate(loss), 16, 1e-2)
+
 
 @test_util.run_all_in_graph_and_eager_modes
 class MeanSquaredErrorTest(test.TestCase):
diff --git a/tensorflow/python/keras/models_test.py b/tensorflow/python/keras/models_test.py
index 3eab10f624..f429aba498 100644
--- a/tensorflow/python/keras/models_test.py
+++ b/tensorflow/python/keras/models_test.py
@@ -291,7 +291,10 @@ class CheckpointingTests(keras_parameterized.TestCase):
         optimizer=opt, loss='mse',
         run_eagerly=testing_utils.should_run_eagerly())
 
-    model.fit(x=np.array([[1., 2., 3., 4.]]), y=np.array([1.]), epochs=2)
+    model.fit(
+        x=np.array([[1., 2., 3., 4.]]),
+        y=np.array([[1., 1., 1., 1.]]),
+        epochs=2)
     save_prefix = os.path.join(self.get_temp_dir(), 'ckpt')
     beta1_power, _ = opt._get_beta_accumulators()
     self.evaluate(beta1_power.assign(12.))
diff --git a/tensorflow/python/keras/wrappers/scikit_learn.py b/tensorflow/python/keras/wrappers/scikit_learn.py
index 566f9db5d4..b5fe7669ab 100644
--- a/tensorflow/python/keras/wrappers/scikit_learn.py
+++ b/tensorflow/python/keras/wrappers/scikit_learn.py
@@ -23,6 +23,7 @@ import types
 
 import numpy as np
 
+from tensorflow.python.keras import losses
 from tensorflow.python.keras.models import Sequential
 from tensorflow.python.keras.utils.generic_utils import has_arg
 from tensorflow.python.keras.utils.np_utils import to_categorical
@@ -155,10 +156,8 @@ class BaseWrapper(object):
     else:
       self.model = self.build_fn(**self.filter_sk_params(self.build_fn))
 
-    loss_name = self.model.loss
-    if hasattr(loss_name, '__name__'):
-      loss_name = loss_name.__name__
-    if loss_name == 'categorical_crossentropy' and len(y.shape) != 2:
+    if (losses.is_categorical_crossentropy(self.model.loss) and
+        len(y.shape) != 2):
       y = to_categorical(y)
 
     fit_args = copy.deepcopy(self.filter_sk_params(Sequential.fit))
-- 
GitLab


From 610c90845323aa4a63a32b5567d505823b4d4454 Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Fri, 25 Jan 2019 17:02:19 -0800
Subject: [PATCH 1319/2345] Move resource and ref edge-based colocation to
 ColocationGraph

This is a trivial code movement. No functionality change.

PiperOrigin-RevId: 230995734
---
 tensorflow/core/common_runtime/placer.cc | 195 ++++++++++++-----------
 1 file changed, 99 insertions(+), 96 deletions(-)

diff --git a/tensorflow/core/common_runtime/placer.cc b/tensorflow/core/common_runtime/placer.cc
index 515c1971d9..eed2d467d6 100644
--- a/tensorflow/core/common_runtime/placer.cc
+++ b/tensorflow/core/common_runtime/placer.cc
@@ -95,6 +95,26 @@ std::vector<Device*> FilterSupportedDevices(
   return filtered_devices;
 }
 
+// Returns true if the node has no inputs and produces outputs
+// that are consumed by a single node.
+//
+// TODO(vrv): Currently this handles only nodes with one output, but
+// this could be extended to handle the case where a node has many
+// outputs that are connected to nodes in the same colocation group.
+bool IsGeneratorNode(const Node* node) {
+  return node->num_inputs() == 0 && node->num_outputs() == 1 &&
+         !IsRefType(node->output_type(0));
+}
+
+bool IsExemptFromResourceInputColocation(const Node* node) {
+  // Note: Partitioned function calls, which place and partition their
+  // function bodies, are exempt from this check: they forward resource and
+  // ref inputs to operations that are appropriately placed, instead of
+  // dereferencing them.
+  const string& op_type = node->op_def().name();
+  return op_type == "PartitionedCall" || op_type == "StatefulPartitionedCall";
+}
+
 // This class maintains the connected components of a colocation
 // constraint graph, and uses this information to assign a satisfying
 // device placement to the nodes of the graph.
@@ -194,6 +214,82 @@ class ColocationGraph {
     return Status::OK();
   }
 
+  Status ColocateResourceAndRefEdges(bool log_device_placement) {
+    // Enumerate the constraint edges, and use them to update the disjoint
+    // node set.
+
+    // If `node` has an input edge with reference type, add an edge from the
+    // source of that edge to `node`.
+    for (const Edge* edge : graph_->edges()) {
+      if (edge->IsControlEdge()) {
+        continue;
+      }
+      Node* src = edge->src();
+      Node* dst = edge->dst();
+      DataType input_type = dst->input_type(edge->dst_input());
+      if ((input_type == DT_RESOURCE || IsRefType(input_type)) &&
+          !IsExemptFromResourceInputColocation(dst)) {
+        // Colocate `src` and `dst` to maintain the invariant that nodes
+        // connected by reference edges are colocated.
+        int src_root_id = FindRoot(src->id());
+        int dst_root_id = FindRoot(dst->id());
+        auto& src_root = members_[src_root_id];
+        auto& dst_root = members_[dst_root_id];
+        // If both the source node and this node have partially
+        // specified a device, then 'node's device should be
+        // cleared: the reference edge forces 'node' to be on the
+        // same device as the source node.
+        const auto& source_parsed_name = src_root.device_name;
+        const auto& dest_parsed_name = dst_root.device_name;
+        if (DeviceNameUtils::HasSomeDetails(source_parsed_name) &&
+            DeviceNameUtils::HasSomeDetails(dest_parsed_name)) {
+          // Ignore a specified device for 'dst' if the two names were
+          // incompatible.
+          if (!DeviceNameUtils::AreCompatibleDevNames(source_parsed_name,
+                                                      dest_parsed_name)) {
+            TF_RETURN_IF_ERROR(VerifyResourceAndRefInputsCanBeColocated(
+                dst, src, source_parsed_name));
+            if (log_device_placement) {
+              LOG(INFO)
+                  << "Ignoring device specification "
+                  << DeviceNameUtils::ParsedNameToString(dest_parsed_name)
+                  << " for node '" << dst->name()
+                  << "' because the input edge from '" << src->name()
+                  << "' is a reference connection and already has a device "
+                     "field set to "
+                  << DeviceNameUtils::ParsedNameToString(source_parsed_name);
+            }
+
+            // Make 'dst' colocated with the source
+            dst_root.device_name = source_parsed_name;
+          } else {
+            bool source_subset_of_dest = DeviceNameUtils::IsSpecification(
+                source_parsed_name, dest_parsed_name);
+            bool dest_subset_of_source = DeviceNameUtils::IsSpecification(
+                dest_parsed_name, source_parsed_name);
+
+            if (source_subset_of_dest && !dest_subset_of_source) {
+              src_root.device_name = dest_parsed_name;
+            } else {
+              dst_root.device_name = source_parsed_name;
+            }
+          }
+        }
+        Status status = ColocateNodes(*src, src_root_id, *dst, dst_root_id);
+        if (!status.ok()) {
+          return AttachDef(errors::InvalidArgument(
+                               "Nodes were connected by a "
+                               "reference connection (requiring them to "
+                               "be on the same device), but the two nodes "
+                               "were assigned two different devices: ",
+                               status.error_message()),
+                           *dst);
+        }
+      }
+    }
+    return Status::OK();
+  }
+
   Status ColocateNodeToGroup(
       std::unordered_map<StringPiece, const Node*, StringPieceHasher>*
           colocation_group_root,
@@ -790,27 +886,6 @@ class ColocationGraph {
   const bool allow_soft_placement_;
   const Device* default_device_;
 };
-
-// Returns true if the node has no inputs and produces outputs
-// that are consumed by a single node.
-//
-// TODO(vrv): Currently this handles only nodes with one output, but
-// this could be extended to handle the case where a node has many
-// outputs that are connected to nodes in the same colocation group.
-bool IsGeneratorNode(const Node* node) {
-  return node->num_inputs() == 0 && node->num_outputs() == 1 &&
-         !IsRefType(node->output_type(0));
-}
-
-bool IsExemptFromResourceInputColocation(const Node* node) {
-  // Note: Partitioned function calls, which place and partition their
-  // function bodies, are exempt from this check: they forward resource and
-  // ref inputs to operations that are appropriately placed, instead of
-  // dereferencing them.
-  const string& op_type = node->op_def().name();
-  return op_type == "PartitionedCall" || op_type == "StatefulPartitionedCall";
-}
-
 }  // namespace
 
 Placer::Placer(Graph* graph, const DeviceSet* devices,
@@ -844,81 +919,9 @@ Status Placer::Run() {
   // the constraints) may not be acyclic.
   TF_RETURN_IF_ERROR(colocation_graph.ColocateAllNodes());
 
-  // 2. Enumerate the constraint edges, and use them to update the disjoint
-  // node set.
-
-  // If `node` has an input edge with reference type, add an edge from the
-  // source of that edge to `node`.
-  for (const Edge* edge : graph_->edges()) {
-    if (edge->IsControlEdge()) {
-      continue;
-    }
-    Node* src = edge->src();
-    Node* dst = edge->dst();
-    DataType input_type = dst->input_type(edge->dst_input());
-    if ((input_type == DT_RESOURCE || IsRefType(input_type)) &&
-        !IsExemptFromResourceInputColocation(dst)) {
-      // Colocate `src` and `dst` to maintain the invariant that nodes connected
-      // by reference edges are colocated.
-      int src_root_id = colocation_graph.FindRoot(src->id());
-      int dst_root_id = colocation_graph.FindRoot(dst->id());
-      auto& src_root = colocation_graph.members_[src_root_id];
-      auto& dst_root = colocation_graph.members_[dst_root_id];
-      // If both the source node and this node have partially
-      // specified a device, then 'node's device should be
-      // cleared: the reference edge forces 'node' to be on the
-      // same device as the source node.
-      const auto& source_parsed_name = src_root.device_name;
-      const auto& dest_parsed_name = dst_root.device_name;
-      if (DeviceNameUtils::HasSomeDetails(source_parsed_name) &&
-          DeviceNameUtils::HasSomeDetails(dest_parsed_name)) {
-        // Ignore a specified device for 'dst' if the two names were
-        // incompatible.
-        if (!DeviceNameUtils::AreCompatibleDevNames(source_parsed_name,
-                                                    dest_parsed_name)) {
-          TF_RETURN_IF_ERROR(
-              colocation_graph.VerifyResourceAndRefInputsCanBeColocated(
-                  dst, src, source_parsed_name));
-          if (log_device_placement_) {
-            LOG(INFO) << "Ignoring device specification "
-                      << DeviceNameUtils::ParsedNameToString(dest_parsed_name)
-                      << " for node '" << dst->name()
-                      << "' because the input edge from '" << src->name()
-                      << "' is a reference connection and already has a device "
-                         "field set to "
-                      << DeviceNameUtils::ParsedNameToString(
-                             source_parsed_name);
-          }
-
-          // Make 'dst' colocated with the source
-          dst_root.device_name = source_parsed_name;
-        } else {
-          bool source_subset_of_dest = DeviceNameUtils::IsSpecification(
-              source_parsed_name, dest_parsed_name);
-          bool dest_subset_of_source = DeviceNameUtils::IsSpecification(
-              dest_parsed_name, source_parsed_name);
-
-          if (source_subset_of_dest && !dest_subset_of_source) {
-            src_root.device_name = dest_parsed_name;
-          } else {
-            dst_root.device_name = source_parsed_name;
-          }
-        }
-      }
-
-      Status status =
-          colocation_graph.ColocateNodes(*src, src_root_id, *dst, dst_root_id);
-      if (!status.ok()) {
-        return AttachDef(
-            errors::InvalidArgument("Nodes were connected by a "
-                                    "reference connection (requiring them to "
-                                    "be on the same device), but the two nodes "
-                                    "were assigned two different devices: ",
-                                    status.error_message()),
-            *dst);
-      }
-    }
-  }
+  // 2. Add constraints based on resource and ref edges.
+  TF_RETURN_IF_ERROR(
+      colocation_graph.ColocateResourceAndRefEdges(log_device_placement_));
 
   // 3. For each node, assign a device based on the constraints in the
   // disjoint node set.
-- 
GitLab


From 9b1148e1ed59e875975f4b38f5901e0c8820a4f4 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Fri, 25 Jan 2019 17:09:08 -0800
Subject: [PATCH 1320/2345] Support returning IndexedSlices in cond_v2
 gradients.

This also cleans up the error logic for mismatched outputs, including
improving (hopefully) the error message.

PiperOrigin-RevId: 230996844
---
 tensorflow/python/framework/test_util.py      |  4 +
 .../python/kernel_tests/cond_v2_test.py       |  4 +-
 .../kernel_tests/control_flow_ops_py_test.py  | 89 ++++++++++++++++-
 tensorflow/python/ops/cond_v2.py              | 96 ++++++++++++++-----
 tensorflow/python/ops/gradient_checker_v2.py  | 15 ++-
 5 files changed, 177 insertions(+), 31 deletions(-)

diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index 732fd31389..2bff3e457f 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -1661,6 +1661,10 @@ class TensorFlowTestCase(googletest.TestCase):
           return sparse_tensor.SparseTensorValue(tensor.indices.numpy(),
                                                  tensor.values.numpy(),
                                                  tensor.dense_shape.numpy())
+        elif isinstance(tensor, ops.IndexedSlices):
+          return ops.IndexedSlicesValue(values=tensor.values.numpy(),
+                                        indices=tensor.indices.numpy(),
+                                        dense_shape=tensor.dense_shape.numpy())
         return tensor.numpy()
       except AttributeError as e:
         six.raise_from(ValueError("Unsupported type %s." % type(tensor)), e)
diff --git a/tensorflow/python/kernel_tests/cond_v2_test.py b/tensorflow/python/kernel_tests/cond_v2_test.py
index 356c6f0a16..244b0bdd7f 100644
--- a/tensorflow/python/kernel_tests/cond_v2_test.py
+++ b/tensorflow/python/kernel_tests/cond_v2_test.py
@@ -785,8 +785,8 @@ class CondV2Test(test.TestCase):
       return ((x,), y * 3.0)
 
     with self.assertRaisesRegexp(
-        ValueError, "Outputs of true_fn and false_fn must"
-        " have the same structure"):
+        TypeError, "true_fn and false_fn arguments to tf.cond must have the "
+        "same number, type, and overall structure of return values."):
       control_flow_ops.cond(constant_op.constant(False), true_fn, false_fn)
 
   @test_util.enable_control_flow_v2
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 90d77050e9..3f0b1b3bbc 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -53,6 +53,7 @@ from tensorflow.python.ops import gen_control_flow_ops
 from tensorflow.python.ops import gen_data_flow_ops
 from tensorflow.python.ops import gen_logging_ops
 from tensorflow.python.ops import gen_state_ops
+from tensorflow.python.ops import gradient_checker_v2
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import linalg_ops
@@ -429,6 +430,24 @@ class ControlFlowTest(test.TestCase):
     self.assertAllEqual(11, val)
     self.assertAllEqual(0, ind)
 
+  def testCondMismatchedIndexedSlices(self):
+    @def_function.function
+    def foo():
+      values = constant_op.constant(10)
+      indices = constant_op.constant(0)
+      x = ops.IndexedSlices(values, indices)
+      v1_msg = "The two structures don't have the same nested structure"
+      v2_msg = ("true_fn and false_fn arguments to tf.cond must have the same "
+                "number, type, and overall structure of return values.")
+      with self.assertRaisesRegexp(
+          TypeError,
+          v2_msg if control_flow_util.ENABLE_CONTROL_FLOW_V2 else v1_msg):
+        control_flow_ops.cond(
+            constant_op.constant(True),
+            lambda: ops.IndexedSlices(math_ops.add(x.values, 1), indices),
+            lambda: math_ops.add(x.values, 1), indices)
+    foo()
+
   def testCondSparseTensor(self):
     values = constant_op.constant([2.0, 4.0], name="values")
     indices = constant_op.constant([[0], [3]],
@@ -714,12 +733,12 @@ class ControlFlowTest(test.TestCase):
       fn1 = lambda: {"a": math_ops.add(x, y), "b": math_ops.add(x, y)}
       fn2 = lambda: {"c": y, "d": y}
       v1_msg = "The two structures don't have the same nested structure"
-      v2_msg = "Outputs of true_fn and false_fn must have the same structure"
+      v2_msg = ("true_fn and false_fn arguments to tf.cond must have the same "
+                "number, type, and overall structure of return values.")
       with self.assertRaisesRegexp(
-          ValueError,
+          TypeError if control_flow_util.ENABLE_CONTROL_FLOW_V2 else ValueError,
           v2_msg if control_flow_util.ENABLE_CONTROL_FLOW_V2 else v1_msg):
-        r = control_flow_ops.cond(pred, fn1, fn2)
-        self.evaluate(r)
+        control_flow_ops.cond(pred, fn1, fn2)
 
   @test_util.run_deprecated_v1
   def testCondRef(self):
@@ -924,6 +943,68 @@ class ControlFlowTest(test.TestCase):
       ]
       self.assertAllEqual(dense_gv, [0.0, 2.0])
 
+  @test_util.run_deprecated_v1
+  def testCondGrad_ResourceVarSparseRead(self):
+    # NOTE(skyewm): this test is interesting because the
+    # ResourceVariable.sparse_read gradient function returns IndexedSlices.
+    var = resource_variable_ops.ResourceVariable(
+        np.ones((4, 2), dtype=np.float32))
+    x = constant_op.constant(1.0)
+    r = control_flow_ops.cond(
+        constant_op.constant(True),
+        lambda: x * math_ops.reduce_sum(var.sparse_read([1, 2])),
+        lambda: constant_op.constant(np.zeros((2, 3)),
+                                     dtype=dtypes.float32))
+    grad = gradients_impl.gradients(r, var)[0]
+
+    self.evaluate(variables.global_variables_initializer())
+    grad_val = self.evaluate(grad)
+    self.assertIsInstance(grad_val, ops.IndexedSlicesValue)
+    self.assertAllEqual(gradient_checker_v2._to_numpy(grad_val), [[0., 0.],
+                                                                  [1., 1.],
+                                                                  [1., 1.],
+                                                                  [0., 0.]])
+
+  def testCondGrad_MultiGather(self):
+    # NOTE(skyewm): this test is interesting because the array_ops.gather and
+    # ResourceVariable.sparse_read gradient functions returns IndexedSlices.
+    var = resource_variable_ops.ResourceVariable(
+        np.ones((4, 2), dtype=np.float32))
+    x1 = constant_op.constant(np.ones((3, 3), dtype=np.float32))
+    x2 = constant_op.constant(2.0)
+
+    def true_fn():
+      y1 = var.sparse_read([1, 2])
+      y2 = array_ops.gather(x1, [2]) * x2
+      y3 = x2 * [1., 1., 1.]
+      return y1, y2, y3
+
+    def false_fn():
+      y1 = np.zeros((2, 2), dtype=np.float32)
+      y2 = array_ops.gather(x1, [2]) * x2
+      y3 = array_ops.gather(x1, [2])
+      return y1, y2, y3
+
+    @def_function.function
+    def foo():
+      r = control_flow_ops.cond(constant_op.constant(True), true_fn, false_fn)
+      return gradients_impl.gradients(r, [var, x1, x2])
+
+    grad = foo()
+    self.evaluate(variables.global_variables_initializer())
+    var_grad, x1_grad, x2_grad = self.evaluate(grad)
+    self.assertIsInstance(var_grad, ops.IndexedSlicesValue)
+    self.assertAllEqual(gradient_checker_v2._to_numpy(var_grad), [[0., 0.],
+                                                                  [1., 1.],
+                                                                  [1., 1.],
+                                                                  [0., 0]])
+    self.assertIsInstance(x1_grad, ops.IndexedSlicesValue)
+    self.assertAllEqual(gradient_checker_v2._to_numpy(x1_grad), [[0., 0., 0.],
+                                                                 [0., 0., 0.],
+                                                                 [2., 2., 2.]])
+    self.assertIsInstance(x1_grad, ops.IndexedSlicesValue)
+    self.assertEqual(gradient_checker_v2._to_numpy(x2_grad), 6.)
+
   @test_util.run_v1_only("b/120545219")
   def testCondPredicateTensor(self):
     """Regression test for lowering predicate from non-first output of an op."""
diff --git a/tensorflow/python/ops/cond_v2.py b/tensorflow/python/ops/cond_v2.py
index 6e8f2df39b..a0a13fbde3 100644
--- a/tensorflow/python/ops/cond_v2.py
+++ b/tensorflow/python/ops/cond_v2.py
@@ -36,6 +36,7 @@ from tensorflow.python.ops import gen_dataset_ops
 from tensorflow.python.ops import gen_functional_ops
 from tensorflow.python.ops import gen_resource_variable_ops
 from tensorflow.python.ops import gradients_impl
+from tensorflow.python.ops import math_ops
 from tensorflow.python.util import nest
 
 
@@ -79,13 +80,10 @@ def cond_v2(pred, true_fn, false_fn, name="cond"):
         add_control_dependencies=add_control_dependencies,
         op_return_value=pred)
 
-    outputs = _build_cond(pred, true_graph, false_graph,
-                          true_graph.external_captures,
-                          false_graph.external_captures,
-                          name=scope)
-
-    return func_graph_module.pack_sequence_as(true_graph.structured_outputs,
-                                              outputs)
+    return _build_cond(pred, true_graph, false_graph,
+                       true_graph.external_captures,
+                       false_graph.external_captures,
+                       name=scope)
 
 
 @ops.RegisterGradient("If")
@@ -107,9 +105,6 @@ def _IfGrad(op, *grads):  # pylint: disable=invalid-name
   false_grad_graph = _create_grad_func(
       false_graph, grads, util.unique_grad_fn_name(false_graph.name))
 
-  assert ([t.dtype for t in true_grad_graph.outputs] ==
-          [t.dtype for t in false_grad_graph.outputs])
-
   if (true_grad_graph.if_op_needs_rewrite or
       false_grad_graph.if_op_needs_rewrite):
     # Modify 'op' to output the intermediates needed by the grad functions. Note
@@ -156,6 +151,9 @@ def _IfGrad(op, *grads):  # pylint: disable=invalid-name
   true_grad_inputs = _resolve_grad_inputs(true_graph, true_grad_graph)
   false_grad_inputs = _resolve_grad_inputs(false_graph, false_grad_graph)
 
+  # This modifies true_grad_graph and false_grad_graph.
+  _make_output_composite_tensors_match(true_grad_graph, false_grad_graph)
+
   outputs = _build_cond(if_op.inputs[0], true_grad_graph, false_grad_graph,
                         true_grad_inputs, false_grad_inputs)
 
@@ -219,7 +217,8 @@ def _build_cond(pred, true_graph, false_graph, true_inputs, false_inputs,
 
   # Prevent fetching since the variant outputs can't be fetched directly.
   if_op.graph.prevent_fetching(if_op)
-  return tensors
+  return func_graph_module.pack_sequence_as(true_graph.structured_outputs,
+                                            tensors)
 
 
 def _get_func_graphs(if_op):
@@ -475,6 +474,50 @@ def _make_inputs_match(true_graph, false_graph, true_inputs, false_inputs):
   return new_inputs
 
 
+def _make_output_composite_tensors_match(true_graph, false_graph):
+  """Rewrites {true,false}_graph's outputs to use the same _TensorLike classes.
+
+  Currently the only transformation implemented is turning a Tensor into an
+  equivalent IndexedSlices if the other branch returns an IndexedSlices.
+  Updates {true,false}_graph.{outputs,structured_outputs}.
+
+  Args:
+    true_graph: FuncGraph
+    false_graph: FuncGraph
+
+  Raises:
+    TypeError: if a pair of outputs cannot be rewritten.
+  """
+  # Note: since this is only used for gradient graphs, we do not expect the
+  # outputs to be structured (e.g. nested lists), and thus do not need to use
+  # nest.flatten, etc.
+  true_outputs = list(true_graph.structured_outputs)
+  false_outputs = list(false_graph.structured_outputs)
+  assert len(true_outputs) == len(false_outputs)
+
+  for idx, (true_out, false_out) in enumerate(zip(true_outputs, false_outputs)):
+    if type(true_out) == type(false_out):  # pylint: disable=unidiomatic-typecheck
+      continue
+    if (isinstance(true_out, ops.IndexedSlices) and
+        isinstance(false_out, ops.Tensor)):
+      with false_graph.as_default():
+        false_outputs[idx] = math_ops._as_indexed_slices(false_out)
+    elif (isinstance(true_out, ops.Tensor) and
+          isinstance(false_out, ops.IndexedSlices)):
+      with true_graph.as_default():
+        true_outputs[idx] = math_ops._as_indexed_slices(true_out)
+    else:
+      raise TypeError(
+          "Cannot reconcile tf.cond %i-th outputs:\n"
+          "  true_fn returned:  %s\n"
+          "  false_fn returned: %s" % (idx, true_out, false_out))
+
+  true_graph.structured_outputs = true_outputs
+  true_graph.outputs = func_graph_module.flatten(true_outputs)
+  false_graph.structured_outputs = false_outputs
+  false_graph.outputs = func_graph_module.flatten(false_outputs)
+
+
 def _wrap_intermediates(func_graph, intermediates):
   with func_graph.as_default():
     return [gen_dataset_ops.optional_from_value([t]) for t in intermediates]
@@ -518,23 +561,30 @@ def _create_fakeparams(func_graph, template_tensors):
 
 def _check_same_outputs(true_graph, false_graph):
   """Raises an error if true_graph and false_graph have different outputs."""
-  true_output_types = [t.dtype for t in true_graph.outputs]
-  false_output_types = [t.dtype for t in false_graph.outputs]
-  if (len(true_graph.outputs) != len(false_graph.outputs) or
-      true_output_types != false_output_types):
+
+  def error(error_detail):
     raise TypeError(
-        "true_fn() and false_fn() must return the same number and type of "
-        "arguments, got:\n"
-        "  true_fn: %s\n"
-        "  false_fn: %s" % (true_output_types, false_output_types))
+        "true_fn and false_fn arguments to tf.cond must have the same number, "
+        "type, and overall structure of return values.\n"
+        "\n"
+        "true_fn output:  %s\n"
+        "false_fn output: %s\n"
+        "\n"
+        "Error details:\n"
+        "%s" % (true_graph.structured_outputs, false_graph.structured_outputs,
+                error_detail))
 
-  # Make sure `structured_outputs` for both graphs have the same structure.
   try:
     nest.assert_same_structure(true_graph.structured_outputs,
-                               false_graph.structured_outputs)
+                               false_graph.structured_outputs,
+                               expand_composites=True)
   except (ValueError, TypeError) as e:
-    raise ValueError("Outputs of true_fn and false_fn must have the same "
-                     "structure: %s" % str(e))
+    error(str(e))
+
+  assert len(true_graph.outputs) == len(false_graph.outputs)
+  for true_out, false_out in zip(true_graph.outputs, false_graph.outputs):
+    if true_out.dtype != false_out.dtype:
+      error("%s and %s have different types" % (true_out, false_out))
 
 
 def _get_output_shapes(true_graph_outputs, false_graph_outputs):
diff --git a/tensorflow/python/ops/gradient_checker_v2.py b/tensorflow/python/ops/gradient_checker_v2.py
index 5d473eeb5f..41fcaaca68 100644
--- a/tensorflow/python/ops/gradient_checker_v2.py
+++ b/tensorflow/python/ops/gradient_checker_v2.py
@@ -66,20 +66,31 @@ def _eval_indexed_slices(a):
 
 
 def _to_numpy(a):
-  """Converts Tensors and EagerTensors to numpy arrays.
+  """Converts Tensors, EagerTensors, and IndexedSlicesValue to numpy arrays.
 
   Args:
     a: any value.
 
   Returns:
     If a is EagerTensor or Tensor, returns the evaluation of a by calling
-    numpy() or run(). Otherwise returns a unchanged.
+    numpy() or run(). If a is IndexedSlicesValue, constructs the corresponding
+    dense numpy array. Otherwise returns a unchanged.
   """
   if isinstance(a, ops.EagerTensor):
     return a.numpy()
   if isinstance(a, ops.Tensor):
     sess = ops.get_default_session()
     return sess.run(a)
+  if isinstance(a, ops.IndexedSlicesValue):
+    arr = np.zeros(a.dense_shape)
+    assert len(a.values) == len(a.indices), (
+        "IndexedSlicesValue has %s value slices but %s indices\n%s" %
+        (a.values, a.indices, a))
+    for values_slice, index in zip(a.values, a.indices):
+      assert 0 <= index < len(arr), (
+          "IndexedSlicesValue has invalid index %s\n%s" % (index, a))
+      arr[index] += values_slice
+    return arr
   return a
 
 
-- 
GitLab


From b087be0262eb11999653a7999e05ec325ae9258d Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Fri, 25 Jan 2019 17:15:01 -0800
Subject: [PATCH 1321/2345] Search for functions inside _ListWrapper and
 _DictWrapper

Will be useful for lists of variable regularization losses, for example.

PiperOrigin-RevId: 230997559
---
 tensorflow/python/saved_model/load_test.py    | 26 +++++++++++++++++++
 .../python/saved_model/revived_types.py       | 20 +++++++++++++-
 tensorflow/python/saved_model/save.py         |  7 +++--
 .../checkpointable/data_structures.py         | 15 +++++++++--
 4 files changed, 63 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index 197c9d9865..ef703f9d29 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -32,6 +32,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import tensor_spec
 from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.saved_model import load
 from tensorflow.python.saved_model import save
@@ -691,10 +692,15 @@ class LoadTest(test.TestCase, parameterized.TestCase):
     root.variables = dict(a=variables.Variable(1.))
     root.variables["b"] = variables.Variable(2.)
     root.variables["c"] = 1
+    root.funcs = dict(
+        a=def_function.function(lambda: constant_op.constant(100.)))
+    root.funcs["conc"] = root.funcs["a"].get_concrete_function()
     imported = self.cycle(root, cycles)
     self.assertEqual(1., imported.variables["a"].numpy())
     self.assertEqual(2., imported.variables["b"].numpy())
     self.assertEqual(set(["a", "b"]), set(imported.variables.keys()))
+    self.assertEqual(100., imported.funcs["a"]().numpy())
+    self.assertEqual(100., imported.funcs["conc"]().numpy())
 
   def test_list(self, cycles):
     root = tracking.AutoCheckpointable()
@@ -707,6 +713,26 @@ class LoadTest(test.TestCase, parameterized.TestCase):
     self.assertIs(None, imported.variables[1])
     self.assertEqual(3, len(imported.variables))
 
+  def test_functions_list(self, cycles):
+    root = tracking.AutoCheckpointable()
+    v1 = variables.Variable(1.)
+    root.losses = [def_function.function(lambda: math_ops.reduce_sum(v1 ** 2))]
+    root.variables = [v1]
+
+    @def_function.function
+    def _v2_loss():
+      if len(root.variables) == 1:
+        v2 = variables.Variable(2.)
+        root.variables.append(v2)
+      return math_ops.reduce_sum(root.variables[1] ** 2)
+
+    root.losses.append(_v2_loss)
+    self.assertAllClose([1., 4.], [loss() for loss in root.losses])
+    imported = self.cycle(root, cycles)
+    self.assertAllClose([1., 4.], [loss() for loss in imported.losses])
+    imported.variables[0].assign(3.)
+    imported.variables[1].assign(4.)
+    self.assertAllClose([9., 16.], [loss() for loss in imported.losses])
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/saved_model/revived_types.py b/tensorflow/python/saved_model/revived_types.py
index ae063202bb..8f82039cfd 100644
--- a/tensorflow/python/saved_model/revived_types.py
+++ b/tensorflow/python/saved_model/revived_types.py
@@ -26,7 +26,8 @@ class VersionedTypeRegistration(object):
   """Holds information about one version of a revived type."""
 
   def __init__(self, object_factory, version, min_producer_version,
-               min_consumer_version, bad_consumers=None, setter=setattr):
+               min_consumer_version, bad_consumers=None, setter=setattr,
+               getter=getattr, attribute_extractor=dir):
     """Identify a revived type version.
 
     Args:
@@ -59,8 +60,16 @@ class VersionedTypeRegistration(object):
         addition to any version less than `min_consumer_version`).
       setter: A callable with the same signature as `setattr` to use when adding
         dependencies to generated objects.
+      getter: A callable with the same signature as `getattr` to use when
+        retrieving items from objects of this type. Used along with
+        `attribute_extractor` to find functions, which are not Checkpointable
+        objects and so not regular dependencies.
+      attribute_extractor: A callable equivalent of the builtin `dir`, used for
+        listing items in this container (if any).
     """
     self.setter = setter
+    self.getter = getter
+    self.attribute_extractor = attribute_extractor
     self.identifier = None  # Set after registration
     self._object_factory = object_factory
     self.version = version
@@ -137,6 +146,15 @@ def register_revived_type(identifier, predicate, versions):
   _TYPE_IDENTIFIERS.append(identifier)
 
 
+def get_attribute_extractors(obj):
+  """Get a `dir` and `getattr` equivalent for use with `obj`."""
+  for identifier in _TYPE_IDENTIFIERS:
+    predicate, versions = _REVIVED_TYPE_REGISTRY[identifier]
+    if predicate(obj):
+      return versions[0].attribute_extractor, versions[0].getter
+  return dir, getattr
+
+
 def serialize(obj):
   """Create a SavedUserObject from a checkpointable object."""
   for identifier in _TYPE_IDENTIFIERS:
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index 94030cf8d6..20af464edf 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -126,9 +126,12 @@ class _SaveableView(object):
   def _list_functions(self, checkpointable_object):
     """Return a dict of `Function`s of a checkpointable."""
     functions = dict()
-    for attribute_name in dir(checkpointable_object):
+    attribute_extractor, attribute_getter = (
+        revived_types.get_attribute_extractors(checkpointable_object))
+    for attribute_name in attribute_extractor(checkpointable_object):
       try:
-        attribute_value = getattr(checkpointable_object, attribute_name, None)
+        attribute_value = attribute_getter(
+            checkpointable_object, attribute_name, None)
       except Exception:  # pylint: disable=broad-except
         # We really don't want to throw an exception just because some object's
         # attribute accessor is broken.
diff --git a/tensorflow/python/training/checkpointable/data_structures.py b/tensorflow/python/training/checkpointable/data_structures.py
index 23a6ec51b2..c86846f8e2 100644
--- a/tensorflow/python/training/checkpointable/data_structures.py
+++ b/tensorflow/python/training/checkpointable/data_structures.py
@@ -803,7 +803,9 @@ revived_types.register_revived_type(
         version=1,
         min_producer_version=1,
         min_consumer_version=1,
-        setter=operator.setitem)])
+        setter=operator.setitem,
+        getter=_DictWrapper.get,
+        attribute_extractor=lambda obj: obj.keys())])
 
 
 def _set_list_item(list_object, index_string, value):
@@ -813,6 +815,13 @@ def _set_list_item(list_object, index_string, value):
   list_object[item_index] = value
 
 
+def _list_getter(obj, item, default=None):
+  index = int(item)
+  if index < len(obj):
+    return obj[index]
+  return default
+
+
 revived_types.register_revived_type(
     "checkpointable_list_wrapper",
     lambda obj: isinstance(obj, _ListWrapper),
@@ -821,4 +830,6 @@ revived_types.register_revived_type(
         version=1,
         min_producer_version=1,
         min_consumer_version=1,
-        setter=_set_list_item)])
+        setter=_set_list_item,
+        getter=_list_getter,
+        attribute_extractor=lambda obj: [str(i) for i in range(len(obj))])])
-- 
GitLab


From 7bb36fb62d6c5b3e24deb1a479035963beaee8b3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 25 Jan 2019 17:23:23 -0800
Subject: [PATCH 1322/2345] Update ops-related pbtxt files.

PiperOrigin-RevId: 230998599
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 34 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 12 +++++++
 2 files changed, 46 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 6fae92cb99..9c1a0b0d9b 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -22318,6 +22318,40 @@ op {
     minimum: 1
   }
 }
+op {
+  name: "ExperimentalChooseFastestDataset"
+  input_arg {
+    name: "input_datasets"
+    type: DT_VARIANT
+    number_attr: "N"
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "N"
+    type: "int"
+    has_minimum: true
+    minimum: 2
+  }
+  attr {
+    name: "num_experiments"
+    type: "int"
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "ExperimentalDatasetCardinality"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 2e6519fa9c..ef990b4ff2 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -10456,6 +10456,18 @@ op {
     name: "num_experiments"
     type: "int"
   }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
 }
 op {
   name: "ExperimentalDatasetCardinality"
-- 
GitLab


From 8fe37b7bd78add8758c994ac2555e05701e0f667 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 25 Jan 2019 17:23:42 -0800
Subject: [PATCH 1323/2345] [XLA] Improve precision of and fix nans in f16 erfc
 implementation.

Edge-cases were not handled properly, resulting in catastrophic precision loss
and NaNs for reasonable inputs.

May have affected f32 as well (for more extreme inputs), I didn't check.

Relevant to #25052.

PiperOrigin-RevId: 230998627
---
 tensorflow/compiler/xla/client/lib/math.cc    | 37 +++++++++++++++++--
 .../xla/client/lib/math_exhaustive_test.cc    | 14 ++++++-
 2 files changed, 46 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/xla/client/lib/math.cc b/tensorflow/compiler/xla/client/lib/math.cc
index f7e29db690..253b3440e2 100644
--- a/tensorflow/compiler/xla/client/lib/math.cc
+++ b/tensorflow/compiler/xla/client/lib/math.cc
@@ -79,6 +79,21 @@ XlaOp EvaluatePolynomial(XlaOp x, absl::Span<const float> coefficients) {
 }
 
 // Compute an approximation of the error function complement (1 - erf(x)).
+//
+// TODO(jlebar): This is not particularly efficient.  The implementation in
+// Cephes that this follows was written for double precision, but our
+// coefficients are specified only to single-precision!  Cephes has a different,
+// simpler implementation for single-precision.
+//
+// Furthermore, we could simplify this further for f16 -- for example, because
+// exp(-4.2 * 4.2) = 0 (f16), the computations in service of the x < 8.0 branch
+// below are unnecessary.
+//
+// See also these alternate implementations of erf and erfc:
+//
+//   https://stackoverflow.com/questions/35148198
+//   https://stackoverflow.com/questions/35966695
+//
 XlaOp Erfc(XlaOp x) {
   auto& b = *x.builder();
   return b.ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
@@ -98,9 +113,25 @@ XlaOp Erfc(XlaOp x) {
     XlaOp pr = EvaluatePolynomial(abs_x, kErfcRCoefficient);
     XlaOp ps = EvaluatePolynomial(abs_x, kErfcSCoefficient);
 
-    XlaOp y = Select(Lt(abs_x, ScalarLike(x, 8.0)), z * pp / pq, z * pr / ps);
-
-    return Select(Lt(x, ScalarLike(x, 0.0)), ScalarLike(x, 2.0) - y, y);
+    XlaOp abs_x_small = Lt(abs_x, ScalarLike(x, 8.0));
+    XlaOp y = Select(abs_x_small, z * pp / pq, z * pr / ps);
+    XlaOp result_no_underflow =
+        Select(Lt(x, ScalarLike(x, 0.0)), ScalarLike(x, 2.0) - y, y);
+
+    // Check for edge cases, namely, exp(-x^2) is exactly 0, or the appropriate
+    // denominator (ps or pq) is inf.  (The check for exp(-x^2) == 0 is
+    // necessary only for x == +/- inf, where this check lets us avoid
+    // multiplying 0 by inf and getting nan.)
+    auto is_pos_inf = [](XlaOp op) {
+      return And(Not(IsFinite(op)), Gt(op, ScalarLike(op, 0)));
+    };
+    XlaOp underflow =
+        Or(Eq(z, ScalarLike(z, 0)), Or(And(is_pos_inf(pq), abs_x_small),
+                                       And(is_pos_inf(ps), Not(abs_x_small))));
+    XlaOp result_underflow =
+        Select(Lt(x, ScalarLike(x, 0)), FullLike(x, 2), FullLike(x, 0));
+
+    return Select(underflow, result_underflow, result_no_underflow);
   });
 }
 
diff --git a/tensorflow/compiler/xla/client/lib/math_exhaustive_test.cc b/tensorflow/compiler/xla/client/lib/math_exhaustive_test.cc
index 0408ed1ba2..0fb13a73b4 100644
--- a/tensorflow/compiler/xla/client/lib/math_exhaustive_test.cc
+++ b/tensorflow/compiler/xla/client/lib/math_exhaustive_test.cc
@@ -137,22 +137,31 @@ XLA_TEST_P(MathExhaustiveTest, DISABLED_ON_INTERPRETER(F16)) {
   ComputeAndCompareR1<half>(&b, expected_result, {}, tc.error);
 }
 
-// TODO(b/123355973): The following tests are missing.
+// TODO(b/123355973): The following tests from math.cc are missing.
 //
 // - Many failures.
+//
 //   Testcase{"acosh", Acosh, std::acosh}.set_relaxed_nans(),
 //   Testcase{"asinh", Asinh, std::asinh},
 //   Testcase{"sinh", Sinh, std::sinh},
 //   Testcase{"cosh", Cosh, std::cosh}.set_fewer_infs_ok(),
 //   Testcase{"erf", Erf, std::erf},
-//   Testcase{"erfc", Erfc, std::erfc}.set_skip_infs(),
 //   Testcase{"round_to_even", RoundToEven,
 //            [](float x) { return std::nearbyint(x / 2) * 2; }},
 //
 // - No equivalent std function to compare with.
+//
 //   Testcase{"erfinv", ErfInv, std::erfinv},
 //   Testcase{"digamma", Digamma, std::digamma},
 //
+// - Needs a special test (function takes two args, and simply computing in f32
+//   and downcasting to f16 doesn't give the correct answer).
+//
+//   Testcase{"nextafter", NextAfter, std::nextafter},
+//
+// TODO(b/123355973): Test math functions not from math.cc (e.g. log).
+// TODO(b/123355973): Test bf16 and f32.
+//
 INSTANTIATE_TEST_CASE_P(
     MathExhaustiveTest_Instantiation, MathExhaustiveTest,
     ::testing::ValuesIn(std::vector<Testcase>{
@@ -163,6 +172,7 @@ INSTANTIATE_TEST_CASE_P(
             .set_skip_neg_zero(),
         Testcase{"square", Square, [](float x) { return x * x; }},
         Testcase{"reciprocal", Reciprocal, [](float x) { return 1 / x; }},
+        Testcase{"erfc", Erfc, std::erfc},
         Testcase{"lgamma", Lgamma, std::lgamma}
             .set_tolerance(0.1, 0.15)
             .set_fewer_infs_ok(),
-- 
GitLab


From 0cb1d4de125a31eb5f32e56a446d182833674ba4 Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Fri, 25 Jan 2019 17:29:47 -0800
Subject: [PATCH 1324/2345] Refactor Placer's ColocationGraph

Make function's smaller and mark ColocationGraph's Graph and Nodes
const.

PiperOrigin-RevId: 230999313
---
 tensorflow/core/common_runtime/placer.cc | 167 ++++++++++++-----------
 1 file changed, 88 insertions(+), 79 deletions(-)

diff --git a/tensorflow/core/common_runtime/placer.cc b/tensorflow/core/common_runtime/placer.cc
index eed2d467d6..74e7052eed 100644
--- a/tensorflow/core/common_runtime/placer.cc
+++ b/tensorflow/core/common_runtime/placer.cc
@@ -143,15 +143,21 @@ bool IsExemptFromResourceInputColocation(const Node* node) {
 // This implementation uses the Union-Find algorithm to efficiently maintain the
 // connected components and incrementally adds edges via
 // ColocationGraph::ColocateNodes() invocations.
+//
+// ColocationGraph does not assign any devices to graph nodes. The
+// `log_device_placement` argument is used to log messages when requested
+// device is ignored.
 class ColocationGraph {
  public:
-  ColocationGraph(Graph* graph, const DeviceSet* device_set,
-                  bool allow_soft_placement, const Device* default_device)
+  ColocationGraph(const Graph* graph, const DeviceSet* device_set,
+                  const Device* default_device, bool allow_soft_placement,
+                  bool log_device_placement)
       : graph_(graph),
         device_set_(device_set),
         device_types_(device_set->PrioritizedDeviceTypeList()),
+        default_device_(default_device),
         allow_soft_placement_(allow_soft_placement),
-        default_device_(default_device) {
+        log_device_placement_(log_device_placement) {
     members_.resize(graph->num_node_ids());
   }
 
@@ -179,7 +185,7 @@ class ColocationGraph {
     std::unordered_map<StringPiece, const Node*, StringPieceHasher>
         colocation_group_root;
 
-    for (Node* node : graph_->op_nodes()) {
+    for (const Node* node : graph_->op_nodes()) {
       // When adding the node, identify whether it is part of a colocation
       // group.
 
@@ -214,10 +220,70 @@ class ColocationGraph {
     return Status::OK();
   }
 
-  Status ColocateResourceAndRefEdges(bool log_device_placement) {
+  Status ColocateResourceOrRefEdge(Node* src, Node* dst) {
+    // Colocate `src` and `dst` to maintain the invariant that nodes
+    // connected by reference edges are colocated.
+    int src_root_id = FindRoot(src->id());
+    int dst_root_id = FindRoot(dst->id());
+    auto& src_root = members_[src_root_id];
+    auto& dst_root = members_[dst_root_id];
+    // If both the source node and this node have partially
+    // specified a device, then 'dst's device should be
+    // cleared: the reference edge forces 'node' to be on the
+    // same device as the source node.
+    const auto& source_parsed_name = src_root.device_name;
+    const auto& dest_parsed_name = dst_root.device_name;
+    if (DeviceNameUtils::HasSomeDetails(source_parsed_name) &&
+        DeviceNameUtils::HasSomeDetails(dest_parsed_name)) {
+      // Ignore a specified device for 'dst' if the two names were
+      // incompatible.
+      if (!DeviceNameUtils::AreCompatibleDevNames(source_parsed_name,
+                                                  dest_parsed_name)) {
+        TF_RETURN_IF_ERROR(VerifyResourceAndRefInputsCanBeColocated(
+            dst, src, source_parsed_name));
+        if (log_device_placement_) {
+          LOG(INFO) << "Ignoring device specification "
+                    << DeviceNameUtils::ParsedNameToString(dest_parsed_name)
+                    << " for node '" << dst->name()
+                    << "' because the input edge from '" << src->name()
+                    << "' is a reference connection and already has a device "
+                       "field set to "
+                    << DeviceNameUtils::ParsedNameToString(source_parsed_name);
+        }
+
+        // Make 'dst' colocated with the source
+        dst_root.device_name = source_parsed_name;
+      } else {
+        // If we are here, devices are compatible. If devices are compatible
+        // at least one of source_subset_of_dest is true.
+        bool source_subset_of_dest = DeviceNameUtils::IsSpecification(
+            source_parsed_name, dest_parsed_name);
+        bool dest_subset_of_source = DeviceNameUtils::IsSpecification(
+            dest_parsed_name, source_parsed_name);
+
+        if (source_subset_of_dest && !dest_subset_of_source) {
+          src_root.device_name = dest_parsed_name;
+        } else {
+          dst_root.device_name = source_parsed_name;
+        }
+      }
+    }
+    Status status = ColocateNodes(*src, src_root_id, *dst, dst_root_id);
+    if (!status.ok()) {
+      return AttachDef(
+          errors::InvalidArgument("Nodes were connected by a "
+                                  "reference connection (requiring them to "
+                                  "be on the same device), but the two nodes "
+                                  "were assigned two different devices: ",
+                                  status.error_message()),
+          *dst);
+    }
+    return Status::OK();
+  }
+
+  Status ColocateResourceAndRefEdges() {
     // Enumerate the constraint edges, and use them to update the disjoint
     // node set.
-
     // If `node` has an input edge with reference type, add an edge from the
     // source of that edge to `node`.
     for (const Edge* edge : graph_->edges()) {
@@ -229,71 +295,22 @@ class ColocationGraph {
       DataType input_type = dst->input_type(edge->dst_input());
       if ((input_type == DT_RESOURCE || IsRefType(input_type)) &&
           !IsExemptFromResourceInputColocation(dst)) {
-        // Colocate `src` and `dst` to maintain the invariant that nodes
-        // connected by reference edges are colocated.
-        int src_root_id = FindRoot(src->id());
-        int dst_root_id = FindRoot(dst->id());
-        auto& src_root = members_[src_root_id];
-        auto& dst_root = members_[dst_root_id];
-        // If both the source node and this node have partially
-        // specified a device, then 'node's device should be
-        // cleared: the reference edge forces 'node' to be on the
-        // same device as the source node.
-        const auto& source_parsed_name = src_root.device_name;
-        const auto& dest_parsed_name = dst_root.device_name;
-        if (DeviceNameUtils::HasSomeDetails(source_parsed_name) &&
-            DeviceNameUtils::HasSomeDetails(dest_parsed_name)) {
-          // Ignore a specified device for 'dst' if the two names were
-          // incompatible.
-          if (!DeviceNameUtils::AreCompatibleDevNames(source_parsed_name,
-                                                      dest_parsed_name)) {
-            TF_RETURN_IF_ERROR(VerifyResourceAndRefInputsCanBeColocated(
-                dst, src, source_parsed_name));
-            if (log_device_placement) {
-              LOG(INFO)
-                  << "Ignoring device specification "
-                  << DeviceNameUtils::ParsedNameToString(dest_parsed_name)
-                  << " for node '" << dst->name()
-                  << "' because the input edge from '" << src->name()
-                  << "' is a reference connection and already has a device "
-                     "field set to "
-                  << DeviceNameUtils::ParsedNameToString(source_parsed_name);
-            }
-
-            // Make 'dst' colocated with the source
-            dst_root.device_name = source_parsed_name;
-          } else {
-            bool source_subset_of_dest = DeviceNameUtils::IsSpecification(
-                source_parsed_name, dest_parsed_name);
-            bool dest_subset_of_source = DeviceNameUtils::IsSpecification(
-                dest_parsed_name, source_parsed_name);
-
-            if (source_subset_of_dest && !dest_subset_of_source) {
-              src_root.device_name = dest_parsed_name;
-            } else {
-              dst_root.device_name = source_parsed_name;
-            }
-          }
-        }
-        Status status = ColocateNodes(*src, src_root_id, *dst, dst_root_id);
-        if (!status.ok()) {
-          return AttachDef(errors::InvalidArgument(
-                               "Nodes were connected by a "
-                               "reference connection (requiring them to "
-                               "be on the same device), but the two nodes "
-                               "were assigned two different devices: ",
-                               status.error_message()),
-                           *dst);
-        }
+        TF_RETURN_IF_ERROR(ColocateResourceOrRefEdge(src, dst));
       }
     }
     return Status::OK();
   }
 
+  Status Initialize() {
+    TF_RETURN_IF_ERROR(InitializeMembers());
+    TF_RETURN_IF_ERROR(ColocateAllNodes());
+    return ColocateResourceAndRefEdges();
+  }
+
   Status ColocateNodeToGroup(
       std::unordered_map<StringPiece, const Node*, StringPieceHasher>*
           colocation_group_root,
-      Node* node, StringPiece colocation_group) {
+      const Node* node, StringPiece colocation_group) {
     const Node*& root_node = (*colocation_group_root)[colocation_group];
     if (root_node == nullptr) {
       // This is the first node of the colocation group, so
@@ -879,12 +896,13 @@ class ColocationGraph {
     return Status::OK();
   }
 
-  Graph* const graph_;  // Not owned.
+  const Graph* const graph_;  // Not owned.
   std::vector<Member> members_;
   const DeviceSet* device_set_;  // Not owned.
   const std::vector<DeviceType> device_types_;
-  const bool allow_soft_placement_;
   const Device* default_device_;
+  const bool allow_soft_placement_;
+  const bool log_device_placement_;
 };
 }  // namespace
 
@@ -908,22 +926,13 @@ Status Placer::Run() {
   }
 
   ColocationGraph colocation_graph(
-      graph_, devices_,
+      graph_, devices_, default_device_,
       options_ == nullptr || options_->config.allow_soft_placement(),
-      default_device_);
-
-  TF_RETURN_IF_ERROR(colocation_graph.InitializeMembers());
-
-  // 1. First add all of the nodes. Note that steps (1) and (2)
-  // requires two passes over the nodes because the graph (and hence
-  // the constraints) may not be acyclic.
-  TF_RETURN_IF_ERROR(colocation_graph.ColocateAllNodes());
+      log_device_placement_);
 
-  // 2. Add constraints based on resource and ref edges.
-  TF_RETURN_IF_ERROR(
-      colocation_graph.ColocateResourceAndRefEdges(log_device_placement_));
+  TF_RETURN_IF_ERROR(colocation_graph.Initialize());
 
-  // 3. For each node, assign a device based on the constraints in the
+  // For each node, assign a device based on the constraints in the
   // disjoint node set.
   std::vector<Node*> second_pass;
   for (Node* node : graph_->op_nodes()) {
@@ -990,7 +999,7 @@ Status Placer::Run() {
     AssignAndLog(assigned_device, node);
   }
 
-  // 4. Perform a second pass assignment for those nodes explicitly
+  // Perform a second pass assignment for those nodes explicitly
   // skipped during the first pass.
   for (Node* node : second_pass) {
     std::vector<Device*>* devices;
-- 
GitLab


From 66a632ca5160442ad6c18f688834aa534a27b0da Mon Sep 17 00:00:00 2001
From: Yanan Cao <ycao@google.com>
Date: Fri, 25 Jan 2019 17:35:19 -0800
Subject: [PATCH 1325/2345] Increase value comparison tolerance due to a 1%
 numerical diff on pascal.

PiperOrigin-RevId: 231000016
---
 tensorflow/python/keras/layers/unified_gru_test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/layers/unified_gru_test.py b/tensorflow/python/keras/layers/unified_gru_test.py
index 11322764ac..8259643d7c 100644
--- a/tensorflow/python/keras/layers/unified_gru_test.py
+++ b/tensorflow/python/keras/layers/unified_gru_test.py
@@ -176,8 +176,8 @@ class UnifiedGRUTest(keras_parameterized.TestCase):
       cudnn_model.fit(x_train, y_train)
       y_4 = cudnn_model.predict(x_train)
 
-      self.assertAllClose(y_1, y_3, rtol=1e-5, atol=1e-5)
-      self.assertAllClose(y_2, y_4, rtol=1e-5, atol=1e-5)
+      self.assertAllClose(y_1, y_3, rtol=2e-5, atol=2e-5)
+      self.assertAllClose(y_2, y_4, rtol=2e-5, atol=2e-5)
 
   @parameterized.named_parameters(
       # test_name, use_bias, bias_initializer, activation
-- 
GitLab


From 8234d5a3f8967919d3f55d99cd945fa946f4bfe0 Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Fri, 25 Jan 2019 17:39:13 -0800
Subject: [PATCH 1326/2345] Keras learning phase scope bug fix.

This sequence of steps would result in unexpected behavior:
1. Start in eager mode.
2. Enter and exit a learning phase scope.
3. Enter the keras default graph scope and get the learning phase value.

In step #3, the value returned would be 0, instead of a placeholder, because the default value of 0 got set in the graph in step 2. In essence, learning phase scope is not resetting the states correctly when exiting.
This change fixes this by restoring things more accurately when existing a learning phase scope.

PiperOrigin-RevId: 231000597
---
 tensorflow/python/keras/backend.py      | 21 ++++++++++++++++----
 tensorflow/python/keras/backend_test.py | 26 +++++++++++++++++++++++++
 2 files changed, 43 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index 62447ec61f..80921c9761 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -333,7 +333,13 @@ def learning_phase_scope(value):
   global _GRAPH_LEARNING_PHASES  # pylint: disable=global-variable-not-assigned
   if value not in {0, 1}:
     raise ValueError('Expected learning phase to be 0 or 1.')
-  previous_value = learning_phase()
+
+  with ops.init_scope():
+    if context.executing_eagerly():
+      previous_eager_value = _GRAPH_LEARNING_PHASES.get(
+          _DUMMY_EAGER_GRAPH, None)
+    previous_graph_value = _GRAPH_LEARNING_PHASES.get(get_graph(), None)
+
   try:
     set_learning_phase(value)
     yield
@@ -341,9 +347,16 @@ def learning_phase_scope(value):
     # Restore learning phase to initial value.
     with ops.init_scope():
       if context.executing_eagerly():
-        _GRAPH_LEARNING_PHASES[_DUMMY_EAGER_GRAPH] = previous_value
-      _GRAPH_LEARNING_PHASES[get_graph()] = previous_value
-
+        if previous_eager_value is not None:
+          _GRAPH_LEARNING_PHASES[_DUMMY_EAGER_GRAPH] = previous_eager_value
+        elif _DUMMY_EAGER_GRAPH in _GRAPH_LEARNING_PHASES:
+          del _GRAPH_LEARNING_PHASES[_DUMMY_EAGER_GRAPH]
+
+      graph = get_graph()
+      if previous_graph_value is not None:
+        _GRAPH_LEARNING_PHASES[graph] = previous_graph_value
+      elif graph in _GRAPH_LEARNING_PHASES:
+        del _GRAPH_LEARNING_PHASES[graph]
 
 @tf_contextlib.contextmanager
 def eager_learning_phase_scope(value):
diff --git a/tensorflow/python/keras/backend_test.py b/tensorflow/python/keras/backend_test.py
index 27657aeef6..f43a3055cb 100644
--- a/tensorflow/python/keras/backend_test.py
+++ b/tensorflow/python/keras/backend_test.py
@@ -132,6 +132,32 @@ class BackendUtilsTest(test.TestCase):
         pass
     self.assertEqual(keras.backend.learning_phase(), initial_learning_phase)
 
+    new_learning_phase = 0
+    keras.backend.set_learning_phase(new_learning_phase)
+    self.assertEqual(keras.backend.learning_phase(), new_learning_phase)
+    with keras.backend.learning_phase_scope(1):
+      self.assertEqual(keras.backend.learning_phase(), 1)
+    self.assertEqual(keras.backend.learning_phase(), new_learning_phase)
+
+  def test_learning_phase_scope_in_graph(self):
+    initial_learning_phase_outside_graph = keras.backend.learning_phase()
+    with keras.backend.get_graph().as_default():
+      initial_learning_phase_in_graph = keras.backend.learning_phase()
+
+    self.assertEqual(keras.backend.learning_phase(),
+                     initial_learning_phase_outside_graph)
+    with keras.backend.learning_phase_scope(1):
+      self.assertEqual(keras.backend.learning_phase(), 1)
+    self.assertEqual(keras.backend.learning_phase(),
+                     initial_learning_phase_outside_graph)
+
+    with keras.backend.get_graph().as_default():
+      self.assertEqual(keras.backend.learning_phase(),
+                       initial_learning_phase_in_graph)
+
+    self.assertEqual(keras.backend.learning_phase(),
+                     initial_learning_phase_outside_graph)
+
   def test_int_shape(self):
     x = keras.backend.ones(shape=(3, 4))
     self.assertEqual(keras.backend.int_shape(x), (3, 4))
-- 
GitLab


From 09874ab7da985353d07cd4271092ee0de44d6b99 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Fri, 25 Jan 2019 17:39:15 -0800
Subject: [PATCH 1327/2345] Updating all the state_ops.assign_add(var, value)
 to var.assign_add(value) in metrics.

PiperOrigin-RevId: 231000599
---
 tensorflow/python/keras/metrics.py | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index e821e7d062..f9e2f69f77 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -26,7 +26,6 @@ import numpy as np
 import six
 
 from tensorflow.python.eager import context
-from tensorflow.python.eager import function
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
@@ -54,11 +53,9 @@ from tensorflow.python.keras.utils.generic_utils import to_list
 from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import confusion_matrix
-from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
-from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variables as tf_variables
 from tensorflow.python.ops import weights_broadcast_ops
 from tensorflow.python.util.tf_export import keras_export
@@ -130,7 +127,7 @@ class Metric(Layer):
       if sample_weight is not None:
         sample_weight = math_ops.cast(sample_weight, self._dtype)
         values = math_ops.multiply(values, sample_weight)
-      state_ops.assign_add(self.true_positives, math_ops.reduce_sum(values))
+      self.true_positives.assign_add(math_ops.reduce_sum(values))
 
     def result(self):
       return array_ops.identity(self.true_positives)
@@ -260,6 +257,7 @@ class Metric(Layer):
 
   ### End: For use by subclasses ###
 
+
 class Reduce(Metric):
   """Encapsulates metrics that perform a reduce operation on the values."""
 
@@ -318,7 +316,7 @@ class Reduce(Metric):
 
     value_sum = math_ops.reduce_sum(values)
     with ops.control_dependencies([value_sum]):
-      update_total_op = state_ops.assign_add(self.total, value_sum)
+      update_total_op = self.total.assign_add(value_sum)
 
     # Exit early if the reduction doesn't have a denominator.
     if self.reduction == metrics_utils.Reduction.SUM:
@@ -337,7 +335,7 @@ class Reduce(Metric):
           'reduction [%s] not implemented' % self.reduction)
 
     with ops.control_dependencies([update_total_op]):
-      return state_ops.assign_add(self.count, num_values)
+      return self.count.assign_add(num_values)
 
   def result(self):
     if self.reduction == metrics_utils.Reduction.SUM:
@@ -2260,7 +2258,7 @@ class MeanIoU(Metric):
         self.num_classes,
         weights=sample_weight,
         dtype=dtypes.float64)
-    return state_ops.assign_add(self.total_cm, current_cm)
+    return self.total_cm.assign_add(current_cm)
 
   def result(self):
     """Compute the mean intersection-over-union via the confusion matrix."""
@@ -2388,9 +2386,9 @@ class MeanTensor(Metric):
       num_values = math_ops.multiply(num_values, sample_weight)
       values = math_ops.multiply(values, sample_weight)
 
-    update_total_op = state_ops.assign_add(self._total, values)
+    update_total_op = self._total.assign_add(values)
     with ops.control_dependencies([update_total_op]):
-      return state_ops.assign_add(self._count, num_values)
+      return self._count.assign_add(num_values)
 
   def result(self):
     if not self._built:
-- 
GitLab


From b9bc38d32e2ee27466424523c1c210b205c6b5f5 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 25 Jan 2019 18:04:59 -0800
Subject: [PATCH 1328/2345] Add direct dependencies to op libraries.

PiperOrigin-RevId: 231003529
---
 tensorflow/core/grappler/inputs/BUILD          | 2 ++
 tensorflow/core/grappler/optimizers/data/BUILD | 5 +++++
 2 files changed, 7 insertions(+)

diff --git a/tensorflow/core/grappler/inputs/BUILD b/tensorflow/core/grappler/inputs/BUILD
index c2846187c7..286c30cd35 100644
--- a/tensorflow/core/grappler/inputs/BUILD
+++ b/tensorflow/core/grappler/inputs/BUILD
@@ -51,7 +51,9 @@ cc_library(
         "//tensorflow/cc:cc_ops",
         "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:no_op_op_lib",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:sendrecv_ops_op_lib",
         "//tensorflow/core:tensorflow",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/kernels:aggregate_ops",
diff --git a/tensorflow/core/grappler/optimizers/data/BUILD b/tensorflow/core/grappler/optimizers/data/BUILD
index 05ee00ef1e..682e7cd6c4 100644
--- a/tensorflow/core/grappler/optimizers/data/BUILD
+++ b/tensorflow/core/grappler/optimizers/data/BUILD
@@ -660,19 +660,24 @@ tf_cc_test(
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
         # For ops we need registered
         "//tensorflow/core/kernels/data:dataset_ops",
+        "//tensorflow/core:dataset_ops_op_lib",
         "//tensorflow/core/kernels:cast_op",
         "//tensorflow/core/kernels:cwise_op",
         "//tensorflow/core/kernels:logging_ops",
+        "//tensorflow/core:logging_ops_op_lib",
         "//tensorflow/core/kernels:math",
         "//tensorflow/core:spectral_ops_op_lib",
         "//tensorflow/core:bitwise_ops_op_lib",
         "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core/kernels:nn",
+        "//tensorflow/core:sendrecv_ops_op_lib",
+        "//tensorflow/core:no_op_op_lib",
         "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:mkl_nn_ops_op_lib",
         "//tensorflow/core:math_ops_op_lib",
         "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core/kernels:parsing",
+        "//tensorflow/core:parsing_ops_op_lib",
         "//tensorflow/tools/graph_transforms:transform_utils",
     ] + tf_protos_all(),
 )
-- 
GitLab


From a9496f3001a20e8343668c5d1b25a08baf080bcd Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Fri, 25 Jan 2019 18:10:51 -0800
Subject: [PATCH 1329/2345] Remove unnecessary messing with colocation root
 device.

ColocateNodes chooses the which of the two colocation groups' roots
should become the new root. Then, it merges the device_names of the
two roots and assigns the merged name to the new root. The merged name
is always more (or equal) specific than the original names.

The removed code did the same thing with the only exception of
sometimes changing the device_name of the node that will not be the new
root. This has no effect because device selection for all the nodes in
a group is done using the information in the root of the group.

PiperOrigin-RevId: 231004097
---
 tensorflow/core/common_runtime/placer.cc | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/tensorflow/core/common_runtime/placer.cc b/tensorflow/core/common_runtime/placer.cc
index 74e7052eed..54f03ab858 100644
--- a/tensorflow/core/common_runtime/placer.cc
+++ b/tensorflow/core/common_runtime/placer.cc
@@ -253,19 +253,6 @@ class ColocationGraph {
 
         // Make 'dst' colocated with the source
         dst_root.device_name = source_parsed_name;
-      } else {
-        // If we are here, devices are compatible. If devices are compatible
-        // at least one of source_subset_of_dest is true.
-        bool source_subset_of_dest = DeviceNameUtils::IsSpecification(
-            source_parsed_name, dest_parsed_name);
-        bool dest_subset_of_source = DeviceNameUtils::IsSpecification(
-            dest_parsed_name, source_parsed_name);
-
-        if (source_subset_of_dest && !dest_subset_of_source) {
-          src_root.device_name = dest_parsed_name;
-        } else {
-          dst_root.device_name = source_parsed_name;
-        }
       }
     }
     Status status = ColocateNodes(*src, src_root_id, *dst, dst_root_id);
-- 
GitLab


From 28a624fb66d0ff72f547888c234105ab578e0f49 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 25 Jan 2019 18:15:18 -0800
Subject: [PATCH 1330/2345] Add temporary patch to third_party/eigen3 to fix
 builds of Arm/NEON code with older versions of GCC.

PiperOrigin-RevId: 231004470
---
 tensorflow/opensource_only.files   |  1 +
 tensorflow/workspace.bzl           |  1 +
 third_party/eigen3/gebp_neon.patch | 11 +++++++++++
 3 files changed, 13 insertions(+)
 create mode 100644 third_party/eigen3/gebp_neon.patch

diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index d9793b05bb..dc98702c08 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -121,6 +121,7 @@ tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/ThreadPool
 tensorflow/third_party/eigen3/unsupported/Eigen/SpecialFunctions
 tensorflow/third_party/eigen3/unsupported/Eigen/MatrixFunctions
 tensorflow/third_party/eigen3/LICENSE
+tensorflow/third_party/eigen3/gebp_neon.patch
 tensorflow/third_party/eigen3/BUILD
 tensorflow/third_party/systemlibs/build_defs.bzl.tpl
 tensorflow/third_party/systemlibs/absl_py.BUILD
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 7fbd1e2b5b..655f0fdb84 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -140,6 +140,7 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "eigen_archive",
         build_file = clean_dep("//third_party:eigen.BUILD"),
+        patch_file = clean_dep("//third_party/eigen3:gebp_neon.patch"),
         sha256 = "48678550a32665331d729be87076e576f2502fff325f5b6c2c78ebf7b1b22c7b",
         strip_prefix = "eigen-eigen-bcc817c0ba98",
         urls = [
diff --git a/third_party/eigen3/gebp_neon.patch b/third_party/eigen3/gebp_neon.patch
new file mode 100644
index 0000000000..d0022e94cc
--- /dev/null
+++ b/third_party/eigen3/gebp_neon.patch
@@ -0,0 +1,11 @@
+--- a/Eigen/src/Core/products/GeneralBlockPanelKernel.h	2019-01-22 20:46:51.000000000 -0800
++++ b/Eigen/src/Core/products/GeneralBlockPanelKernel.h	2019-01-25 13:48:49.000000000 -0800
+@@ -1031,7 +1031,7 @@
+ 
+   EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacket& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<0>&) const
+   {
+-    c += a * b;
++    c = vfmaq_n_f32(c, a, b);
+   }
+ 
+   EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<0>&) const
-- 
GitLab


From 51c7409562e917fd3fe2012ae57699547d0aebec Mon Sep 17 00:00:00 2001
From: Revan Sopher <rsopher@google.com>
Date: Fri, 25 Jan 2019 18:19:48 -0800
Subject: [PATCH 1331/2345] Automated rollback of commit
 d5b8f7cf3143c67fc51b929501d463ab87d3c655. Revert #25169.

PiperOrigin-RevId: 231004852
---
 .../compiler/xla/python_api/xla_literal.py    |  2 +-
 .../python/framework/fast_tensor_util.pyx     |  2 +-
 tensorflow/python/framework/tensor_util.py    | 26 +++++++++----------
 .../python/kernel_tests/cwise_ops_test.py     |  2 +-
 tensorflow/python/ops/nn_ops.py               |  2 +-
 5 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/tensorflow/compiler/xla/python_api/xla_literal.py b/tensorflow/compiler/xla/python_api/xla_literal.py
index 19bd685ab2..757e41a78a 100644
--- a/tensorflow/compiler/xla/python_api/xla_literal.py
+++ b/tensorflow/compiler/xla/python_api/xla_literal.py
@@ -69,7 +69,7 @@ def _ConvertNumpyArrayToLiteral(ndarray):
 
   if ndarray.ndim == 0:
     getattr(literal, type_record.literal_field_name).append(
-        ndarray.astype(type_record.literal_field_type).item())
+        _np.asscalar(ndarray.astype(type_record.literal_field_type)))
   else:
     # Ndarrays with boolean dtypes need special type conversion with protobufs
     if ndarray.dtype in {_np.bool_, _np.dtype('bool')}:
diff --git a/tensorflow/python/framework/fast_tensor_util.pyx b/tensorflow/python/framework/fast_tensor_util.pyx
index dc14361d66..2e3e15f53a 100644
--- a/tensorflow/python/framework/fast_tensor_util.pyx
+++ b/tensorflow/python/framework/fast_tensor_util.pyx
@@ -131,4 +131,4 @@ def AppendBoolArrayToTensorProto(tensor_proto, nparray):
   cdef long i, n
   n = nparray.size
   for i in range(n):
-    tensor_proto.bool_val.append(nparray.item(i))
+    tensor_proto.bool_val.append(np.asscalar(nparray[i]))
diff --git a/tensorflow/python/framework/tensor_util.py b/tensorflow/python/framework/tensor_util.py
index 47e2437e20..ca8b067935 100644
--- a/tensorflow/python/framework/tensor_util.py
+++ b/tensorflow/python/framework/tensor_util.py
@@ -42,7 +42,7 @@ from tensorflow.python.util.tf_export import tf_export
 
 
 def ExtractBitsFromFloat16(x):
-  return np.asarray(x, dtype=np.float16).view(np.uint16).item()
+  return np.asscalar(np.asarray(x, dtype=np.float16).view(np.uint16))
 
 
 def SlowAppendFloat16ArrayToTensorProto(tensor_proto, proto_values):
@@ -58,8 +58,8 @@ def _MediumAppendFloat16ArrayToTensorProto(tensor_proto, proto_values):
 
 
 def ExtractBitsFromBFloat16(x):
-  return np.asarray(
-      x, dtype=dtypes.bfloat16.as_numpy_dtype).view(np.uint16).item()
+  return np.asscalar(
+      np.asarray(x, dtype=dtypes.bfloat16.as_numpy_dtype).view(np.uint16))
 
 
 def SlowAppendBFloat16ArrayToTensorProto(tensor_proto, proto_values):
@@ -122,39 +122,39 @@ if _FAST_TENSOR_UTIL_AVAILABLE:
 else:
 
   def SlowAppendFloat32ArrayToTensorProto(tensor_proto, proto_values):
-    tensor_proto.float_val.extend([x.item() for x in proto_values])
+    tensor_proto.float_val.extend([np.asscalar(x) for x in proto_values])
 
   def SlowAppendFloat64ArrayToTensorProto(tensor_proto, proto_values):
-    tensor_proto.double_val.extend([x.item() for x in proto_values])
+    tensor_proto.double_val.extend([np.asscalar(x) for x in proto_values])
 
   def SlowAppendIntArrayToTensorProto(tensor_proto, proto_values):
-    tensor_proto.int_val.extend([x.item() for x in proto_values])
+    tensor_proto.int_val.extend([np.asscalar(x) for x in proto_values])
 
   def SlowAppendInt64ArrayToTensorProto(tensor_proto, proto_values):
-    tensor_proto.int64_val.extend([x.item() for x in proto_values])
+    tensor_proto.int64_val.extend([np.asscalar(x) for x in proto_values])
 
   def SlowAppendQIntArrayToTensorProto(tensor_proto, proto_values):
-    tensor_proto.int_val.extend([x.item(0) for x in proto_values])
+    tensor_proto.int_val.extend([np.asscalar(x[0]) for x in proto_values])
 
   def SlowAppendUInt32ArrayToTensorProto(tensor_proto, proto_values):
-    tensor_proto.uint32_val.extend([x.item() for x in proto_values])
+    tensor_proto.uint32_val.extend([np.asscalar(x) for x in proto_values])
 
   def SlowAppendUInt64ArrayToTensorProto(tensor_proto, proto_values):
-    tensor_proto.uint64_val.extend([x.item() for x in proto_values])
+    tensor_proto.uint64_val.extend([np.asscalar(x) for x in proto_values])
 
   def SlowAppendComplex64ArrayToTensorProto(tensor_proto, proto_values):
     tensor_proto.scomplex_val.extend(
-        [v.item() for x in proto_values for v in [x.real, x.imag]])
+        [np.asscalar(v) for x in proto_values for v in [x.real, x.imag]])
 
   def SlowAppendComplex128ArrayToTensorProto(tensor_proto, proto_values):
     tensor_proto.dcomplex_val.extend(
-        [v.item() for x in proto_values for v in [x.real, x.imag]])
+        [np.asscalar(v) for x in proto_values for v in [x.real, x.imag]])
 
   def SlowAppendObjectArrayToTensorProto(tensor_proto, proto_values):
     tensor_proto.string_val.extend([compat.as_bytes(x) for x in proto_values])
 
   def SlowAppendBoolArrayToTensorProto(tensor_proto, proto_values):
-    tensor_proto.bool_val.extend([x.item() for x in proto_values])
+    tensor_proto.bool_val.extend([np.asscalar(x) for x in proto_values])
 
   _NP_TO_APPEND_FN = {
       dtypes.bfloat16.as_numpy_dtype: SlowAppendBFloat16ArrayToTensorProto,
diff --git a/tensorflow/python/kernel_tests/cwise_ops_test.py b/tensorflow/python/kernel_tests/cwise_ops_test.py
index 29d335d68c..70f19f9d2f 100644
--- a/tensorflow/python/kernel_tests/cwise_ops_test.py
+++ b/tensorflow/python/kernel_tests/cwise_ops_test.py
@@ -595,7 +595,7 @@ class MinMaxOpTest(test.TestCase):
 
   def testScalar(self):
     x = np.random.rand(1, 3, 2) * 100.
-    y = np.random.rand(1).item() * 100.  # should broadcast
+    y = np.asscalar(np.random.rand(1) * 100.)  # should broadcast
     # dropped np.float64, int64 because TF automatically converts to 32 bit
     for t in [np.float32, np.int32]:
       self._compare(x.astype(t), t(y), use_gpu=False)
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 25e9ffd34f..e0ef9e5e34 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -2334,7 +2334,7 @@ def leaky_relu(features, alpha=0.2, name=None):
       features = math_ops.to_float(features)
     if compat.forward_compatible(2018, 11, 1):
       if isinstance(alpha, np.ndarray):
-        alpha = alpha.item()
+        alpha = np.asscalar(alpha)
       return gen_nn_ops.leaky_relu(features, alpha=alpha, name=name)
     alpha = ops.convert_to_tensor(alpha, dtype=features.dtype, name="alpha")
     return math_ops.maximum(alpha * features, features, name=name)
-- 
GitLab


From 9a3c678c967e138da4efa3bbe88e6e073cb8a03e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 25 Jan 2019 18:38:12 -0800
Subject: [PATCH 1332/2345] Internal change

PiperOrigin-RevId: 231006116
---
 tensorflow/contrib/tpu/BUILD                  |   3 +
 .../python/tpu/_tpu_estimator_embedding.py    | 273 ++++++++++++++++++
 .../contrib/tpu/python/tpu/tpu_context.py     |  46 ++-
 .../contrib/tpu/python/tpu/tpu_embedding.py   |  35 +--
 .../contrib/tpu/python/tpu/tpu_estimator.py   |  66 ++++-
 5 files changed, 380 insertions(+), 43 deletions(-)
 create mode 100644 tensorflow/contrib/tpu/python/tpu/_tpu_estimator_embedding.py

diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index faf11afb21..c1a36fecc2 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -61,6 +61,7 @@ py_library(
 py_library(
     name = "tpu_estimator",
     srcs = [
+        "python/tpu/_tpu_estimator_embedding.py",
         "python/tpu/error_handling.py",
         "python/tpu/tpu_config.py",
         "python/tpu/tpu_context.py",
@@ -70,7 +71,9 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":async_checkpoint",
+        ":feature_column",
         ":functional",
+        ":tpu_embedding",
         ":tpu_lib",
         ":tpu_ordinal_selector_py",
         "//tensorflow/contrib/training:training_py",
diff --git a/tensorflow/contrib/tpu/python/tpu/_tpu_estimator_embedding.py b/tensorflow/contrib/tpu/python/tpu/_tpu_estimator_embedding.py
new file mode 100644
index 0000000000..6ce96e5bcd
--- /dev/null
+++ b/tensorflow/contrib/tpu/python/tpu/_tpu_estimator_embedding.py
@@ -0,0 +1,273 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ===================================================================
+"""Tooling for support TPU embedding in TPUEstimator."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import collections
+
+from tensorflow.contrib.tpu.python.tpu import feature_column as tpu_fc
+from tensorflow.contrib.tpu.python.tpu import tpu_embedding
+from tensorflow.python.estimator import model_fn as model_fn_lib
+from tensorflow.python.feature_column import feature_column as core_fc
+from tensorflow.python.feature_column import feature_column_lib as core_fc_lib
+
+# pylint: disable=protected-access
+_TPU_EMBEDDING_COLUMN_CLASSES = (tpu_fc._TPUEmbeddingColumn,
+                                 tpu_fc._TPUSharedEmbeddingColumn)
+_EMBEDDING_COLUMN_CLASSES = (core_fc._EmbeddingColumn,
+                             core_fc_lib.EmbeddingColumn,
+                             core_fc._SharedEmbeddingColumn)
+_SUPPORTED_FEATURE_COLUMNS = (core_fc._NumericColumn, core_fc_lib.NumericColumn)
+
+# pylint: enable=protected-access
+
+
+def get_tpu_embedding_config_from_feature_columns(feature_columns):
+  """Create configs for TPUEmbedding from a list of feature columns.
+
+  This function will place one embedding tensor per table and the return is
+  intended to be used as input to TPUEmbedding.
+
+  Args:
+    feature_columns: a list of supported feature columns.
+
+  Returns:
+    A pair of dicts, the first maps tables to their config, the second maps
+    features to tables.
+  """
+
+  allowed = (tpu_fc._TPUEmbeddingColumn, tpu_fc._TPUSharedEmbeddingColumn)  # pylint: disable=protected-access
+
+  for column in feature_columns:
+    if not isinstance(column, allowed):
+      raise TypeError(
+          'Unsupported feature column {}. Supported types are {}.'.format(
+              type(column), allowed))
+
+  table_to_config = {}
+  feature_to_table = {}
+  for column in feature_columns:
+    feature_name = column.get_feature_key_name()
+    table_name = 'tbl_{}'.format(column.get_embedding_var_name())
+    if feature_name in feature_to_table:
+      raise ValueError(
+          'Feature column {} is used with multiple embeddings and this is '
+          'not supported.'.format(feature_name))
+    feature_to_table[feature_name] = table_name
+    vocabulary_size, dimension = column.get_embedding_table_size()
+    table_to_config[table_name] = tpu_embedding.TableConfig(
+        vocabulary_size=vocabulary_size,
+        dimension=dimension,
+        initializer=column.get_initializer(),
+        combiner=column.get_combiner())
+
+  return table_to_config, feature_to_table
+
+
+def _get_tpu_embedding_optimization_parameters(embedding_config_spec):
+  """Get tpu_embedding._OptimizationParameters from EmbeddingConfigSpec."""
+  if embedding_config_spec.optimizer_type == 'adagrad':
+    return tpu_embedding.AdagradParameters(
+        embedding_config_spec.learning_rate,
+        embedding_config_spec.adagrad_initial_accumulator,
+        embedding_config_spec.use_gradient_accumulation)
+  elif embedding_config_spec.optimizer_type == 'sgd':
+    return tpu_embedding.StochasticGradientDescentParameters(
+        embedding_config_spec.learning_rate,
+        embedding_config_spec.use_gradient_accumulattion)
+  elif embedding_config_spec.optimizer_type == 'adam':
+    return tpu_embedding.AdamParameters(
+        embedding_config_spec.learning_rate,
+        embedding_config_spec.adam_parameters.beta1,
+        embedding_config_spec.adam_parameters.beta2,
+        embedding_config_spec.adam_parameters.epsilon,
+        use_gradient_accumulation=embedding_config_spec
+        .use_gradient_accumulation)
+  else:
+    raise ValueError('optimizer_type must be adagrad or sgd or adam for now.')
+
+
+AdamParameters = collections.namedtuple('AdamParameters',
+                                        ['beta1', 'beta2', 'epsilon'])
+
+
+# TODO(shizhiw): Improve the API to support more optimizer parameters in API.
+class EmbeddingConfigSpec(
+    collections.namedtuple('EmbeddingConfigSpec', [
+        'feature_columns', 'learning_rate', 'optimizer_type',
+        'adagrad_initial_accumulator', 'clipping_limit',
+        'use_gradient_accumulation', 'adam_parameters'
+    ])):
+  """Class to keep track of embedding config specification."""
+
+  def __new__(cls,
+              feature_columns,
+              learning_rate,
+              optimizer_type='adagrad',
+              adagrad_initial_accumulator=None,
+              clipping_limit=None,
+              use_gradient_accumulation=False,
+              adam_parameters=None):
+    """Creates an EmbeddingConfigSpec instance.
+
+    Args:
+      feature_columns: All `FeatureColumn`s used by model.
+      learning_rate: embedding optimizer learning rate.
+      optimizer_type: (String) Name of the optimizer for embedding gradients
+        updates. Must be either 'adagrad' ( `tf.train.AdagradOptimizer`, default
+        value), 'sgd' (`tf.train.GradientDescentOptimizer`), or 'adam'
+        (`tf.contrib.opt.LazyAdamOptimizer`) for lazy Adam. This optimizer will
+        be applied to all embedding variables specified by `feature_columns`.
+      adagrad_initial_accumulator: Initial accumulator for Adagrad. Used when
+        optimizer_type is 'adagrad'. Default is `0.1`.
+      clipping_limit: (Optional) Clipping limit (absolute value).
+      use_gradient_accumulation: (Experimental) Whether to accumulate the
+        gradients across TPU embedding mini-batches. Gradient accumulation does
+        not affect SGD and therefore this is applicable only for Adagrad.
+      adam_parameters: AdamParameters. Used when optimizer_type is 'adam'.
+        Default is 0.9 for beta1, 0.999 for beta2 and 1e-8 for epsilon.
+
+    Returns:
+      An EmbeddingConfigSpec instance.
+
+    Raises:
+      ValueError: If the feature_columns are not specified.
+      TypeError: If the feature columns are not of ths correct type (one of
+        _SUPPORTED_FEATURE_COLUMNS, _TPU_EMBEDDING_COLUMN_CLASSES OR
+        _EMBEDDING_COLUMN_CLASSES).
+      ValueError: If use_gradient_accumulation is True for SGD.
+      ValueError: If `optimizer_type` is not one of "adagrad" or "sgd" or
+        "adam".
+    """
+    if not feature_columns:
+      raise ValueError('`feature_columns` cannot be `None` or empty.')
+
+    # It is unknown at this moment, whether the TPUEstimator is running in CPU
+    # or TPU mode. So allow non-TPU embedding columns also.
+    supported_classes = tuple(
+        list(_SUPPORTED_FEATURE_COLUMNS) + list(_TPU_EMBEDDING_COLUMN_CLASSES) +
+        list(_EMBEDDING_COLUMN_CLASSES))
+
+    for column in feature_columns:
+      if not isinstance(column, supported_classes):
+        raise TypeError(
+            'All feature columns must be supported types in {}. Got {}'.format(
+                supported_classes, type(column)))
+
+    if optimizer_type == 'adagrad':
+      if adagrad_initial_accumulator is None:
+        adagrad_initial_accumulator = 0.1
+      if adagrad_initial_accumulator <= 0:
+        raise ValueError('Adagrad initial_accumulator must be positive')
+    elif optimizer_type == 'sgd':
+      if use_gradient_accumulation:
+        raise ValueError('Gradient accumulation makes sense for Adagrad only.')
+    elif optimizer_type == 'adam':
+      if adam_parameters is None:
+        adam_parameters = AdamParameters(0.9, 0.999, 1e-8)
+      if adam_parameters.beta1 < 0. or adam_parameters.beta1 >= 1.:
+        raise ValueError('beta1 must be between 0. and 1; got {}.'.format(
+            adam_parameters.beta1))
+      if adam_parameters.beta2 < 0. or adam_parameters.beta2 >= 1.:
+        raise ValueError('beta2 must be between 0. and 1; got {}.'.format(
+            adam_parameters.beta2))
+      if adam_parameters.epsilon <= 0.:
+        raise ValueError('epsilon must be positive; got {}.'.format(
+            adam_parameters.epsilon))
+    else:
+      raise ValueError('optimizer_type must be adagrad or sgd or adam for now.')
+
+    return super(EmbeddingConfigSpec, cls).__new__(
+        cls,
+        feature_columns=feature_columns,
+        learning_rate=learning_rate,
+        optimizer_type=optimizer_type,
+        adagrad_initial_accumulator=adagrad_initial_accumulator,
+        clipping_limit=clipping_limit,
+        use_gradient_accumulation=use_gradient_accumulation,
+        adam_parameters=adam_parameters)
+
+
+class EmbeddingConfig(object):
+  """This is the internal immutable object for embedding config.
+
+  `_EmbeddingConfig` is responsible to _translate_ user provided
+  `EmbeddingConfigSpec` to internal data structures, mostly constructor
+  arguments of `TPUEmbedding`.
+  """
+
+  def __init__(self, embedding_config_spec, train_batch_size, eval_batch_size,
+               num_hosts, num_cores, master):
+    self._embedding_config_spec = embedding_config_spec
+    self._train_batch_size = train_batch_size
+    self._eval_batch_size = eval_batch_size
+    self._num_hosts = num_hosts
+    self._num_cores = num_cores
+    self._master = master
+
+    self._table_to_config_dict, self._feature_to_table_dict = (
+        get_tpu_embedding_config_from_feature_columns(
+            embedding_config_spec.feature_columns))
+    self._optimization_parameters = _get_tpu_embedding_optimization_parameters(
+        self._embedding_config_spec)
+    self._mode_to_tpu_embedding_dict = {}
+
+  def has_embedding_tables(self):
+    return bool(self._table_to_config_dict)
+
+  def _create_tpu_embedding(self, mode):
+    """Create tpu_embedding.TPUEmbedding based on mode."""
+    if mode == model_fn_lib.ModeKeys.TRAIN:
+      batch_size = self._train_batch_size
+    else:
+      batch_size = self._eval_batch_size
+
+    if mode == model_fn_lib.ModeKeys.TRAIN:
+      tpu_embedding_mode = tpu_embedding.TRAINING
+    elif (mode == model_fn_lib.ModeKeys.EVAL or
+          mode == model_fn_lib.ModeKeys.PREDICT):
+      tpu_embedding_mode = tpu_embedding.INFERENCE
+    else:
+      raise ValueError('Mode {} is not supported.'.format(mode))
+
+    tpu_embedding_ = tpu_embedding.TPUEmbedding(
+        self._table_to_config_dict,
+        self._feature_to_table_dict,
+        batch_size,
+        tpu_embedding_mode,
+        self._master,
+        self._optimization_parameters,
+    )
+    return tpu_embedding_
+
+  def get_tpu_embedding(self, mode):
+    if mode not in self._mode_to_tpu_embedding_dict:
+      self._mode_to_tpu_embedding_dict[mode] = (
+          self._create_tpu_embedding(mode))
+    return self._mode_to_tpu_embedding_dict[mode]
+
+
+def split_inputs(ctx, features, labels):
+  """Splits the dense and sparse tensors inside the features and labels."""
+  sparse_features = collections.OrderedDict()
+  if ctx.embedding_config:
+    tpu_embedding_ = ctx.embedding_config.tpu_embedding
+    for feature_key in tpu_embedding_.feature_to_table_dict:
+      sparse_features[feature_key] = features.pop(feature_key)
+
+  return features, labels, sparse_features
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_context.py b/tensorflow/contrib/tpu/python/tpu/tpu_context.py
index 9ff0c55187..ed1e0f0401 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_context.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_context.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 from contextlib import contextmanager
 import copy
 
+from tensorflow.contrib.tpu.python.tpu import _tpu_estimator_embedding
 from tensorflow.contrib.tpu.python.tpu import device_assignment  as tpu_device_assignment
 from tensorflow.contrib.tpu.python.tpu import tpu_config
 from tensorflow.contrib.tpu.python.tpu import tpu_system_metadata as tpu_system_metadata_lib
@@ -192,8 +193,14 @@ class _InternalTPUContext(object):
   ```
   """
 
-  def __init__(self, config, train_batch_size, eval_batch_size,
-               predict_batch_size, use_tpu, eval_on_tpu=True):
+  def __init__(self,
+               config,
+               train_batch_size,
+               eval_batch_size,
+               predict_batch_size,
+               use_tpu,
+               eval_on_tpu=True,
+               embedding_config_spec=None):
     self._config = config
     self._train_batch_size = train_batch_size
     self._eval_batch_size = eval_batch_size
@@ -216,6 +223,8 @@ class _InternalTPUContext(object):
     self._lazy_tpu_system_metadata_dict = {}  # key by master address
     self._lazy_device_assignment_dict = {}  # key by master address
     self._lazy_validation_dict = {}  # key by ModeKeys
+    self._embedding_config_spec = embedding_config_spec
+    self._lazy_embedding_config_dict = {}  # key by master address
 
   def _assert_mode(self):
     if self._mode is None:
@@ -293,6 +302,30 @@ class _InternalTPUContext(object):
     self._lazy_device_assignment_dict[master] = device_assignment
     return device_assignment
 
+  @property
+  def embedding_config(self):
+    """Returns the embedding config based on current mode."""
+    master = self._get_master_address()
+    if master in self._lazy_embedding_config_dict:
+      embedding_config = self._lazy_embedding_config_dict[master]
+    else:
+      embedding_config = None
+      if self._use_tpu and self._embedding_config_spec:
+        embedding_config = _tpu_estimator_embedding.EmbeddingConfig(
+            self._embedding_config_spec, self._train_batch_size,
+            self._eval_batch_size, self.num_hosts, self.num_cores, master)
+        if not embedding_config.has_embedding_tables():
+          embedding_config = None
+      self._lazy_embedding_config_dict[master] = embedding_config
+
+    if embedding_config is not None:
+      mode = self._assert_mode()
+      # Dynamically attach tpu_embedding based on mode. With
+      # this, we could keep embedding_config immutable but call site always
+      # accesses the unified API '.tpu_embedding'.
+      embedding_config.tpu_embedding = embedding_config.get_tpu_embedding(mode)
+    return embedding_config
+
   @property
   def model_parallelism_enabled(self):
     return self._model_parallelism_enabled
@@ -710,11 +743,15 @@ class _OneCoreTPUContext(_InternalTPUContext):
 
 
 def _get_tpu_context(config, train_batch_size, eval_batch_size,
-                     predict_batch_size, use_tpu, eval_on_tpu):
+                     predict_batch_size, use_tpu, eval_on_tpu,
+                     embedding_config_spec):
   """Returns an instance of `_InternalTPUContext`."""
 
   if (config.tpu_config.num_shards == 1 and
       config.tpu_config.num_cores_per_replica is None):
+    if embedding_config_spec is not None:
+      raise ValueError('Setting TPUConfig.num_shards==1 is unsupported '
+                       'when embedding_config_spec is not None.')
     logging.warning(
         'Setting TPUConfig.num_shards==1 is an unsupported behavior. '
         'Please fix as soon as possible (leaving num_shards as None.)')
@@ -722,4 +759,5 @@ def _get_tpu_context(config, train_batch_size, eval_batch_size,
                               predict_batch_size, use_tpu)
 
   return _InternalTPUContext(config, train_batch_size, eval_batch_size,
-                             predict_batch_size, use_tpu, eval_on_tpu)
+                             predict_batch_size, use_tpu, eval_on_tpu,
+                             embedding_config_spec)
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py b/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py
index 1a909a3ac6..04e7397162 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py
@@ -440,6 +440,10 @@ class TPUEmbedding(object):
   def table_to_table_variables_dict(self):
     return copy.copy(self._table_to_table_variables_dict)
 
+  @property
+  def feature_to_table_dict(self):
+    return copy.copy(self._feature_to_table_dict)
+
   def get_slot_names(self):
     """Return a list of the names of slots created by `TPUEmbedding`."""
     return self._optimizer_handler.get_slot_names()
@@ -1065,34 +1069,3 @@ def _create_partitioned_variables(name,
       initializer=initializer,
       collections=collections,
       trainable=False))
-
-
-@ops.RegisterGradient('TPUEmbeddingActivations')
-def _embedding_activations_grad(activations_op, grad_wrt_activations):
-  """Saves the gradient of embedding activations ops in a graph collection."""
-  g = ops.get_default_graph()
-  table_id = activations_op.get_attr('table_id')
-  lookup_id = activations_op.get_attr('lookup_id')
-  table_gradients = g.get_collection_ref(
-      'tpu_embedding_gradients_table_%d' % table_id)
-
-  if not table_gradients:
-    raise RuntimeError(
-        'Gradients for TPUEmbedding have been generated in non-training mode. '
-        'This is not expected. Consider putting your Optimizer.minimize code '
-        'behind the training mode condition check. For Estimator, you can '
-        'do \n\n'
-        '    if mode == tf.estimator.ModeKeys.TRAIN:\n'
-        '        train_op = opt.minimize(loss)\n'
-        '\n')
-
-  table_gradients[lookup_id] = array_ops.identity(grad_wrt_activations)
-  return [
-      # RegisterGradient requires that value be returned for all inputs. Since
-      # the first argument (tpu_gradient_variable_{table_name}) has shape [1],
-      # we will return zeros(shape=[1]). The actual gradient w.r.t. the
-      # embedding activations (grad_wrt_activations) has the same shape as the
-      # activations returned by  embedding_activations.
-      array_ops.zeros(arg.shape, dtype=dtypes.float32)
-      for arg in activations_op.inputs
-  ]
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 41d0562513..0620598ea0 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -34,6 +34,7 @@ from six.moves import xrange  # pylint: disable=redefined-builtin
 from tensorflow.contrib.tpu.ops import gen_tpu_ordinal_selector_op
 from tensorflow.contrib.tpu.proto import compilation_result_pb2 as tpu_compilation_result
 from tensorflow.contrib.tpu.python.ops import tpu_ops
+from tensorflow.contrib.tpu.python.tpu import _tpu_estimator_embedding
 from tensorflow.contrib.tpu.python.tpu import error_handling
 from tensorflow.contrib.tpu.python.tpu import functional as tpu_functional
 from tensorflow.contrib.tpu.python.tpu import session_support
@@ -44,6 +45,8 @@ from tensorflow.contrib.tpu.python.tpu import tpu_context
 from tensorflow.contrib.tpu.python.tpu import tpu_feed
 from tensorflow.contrib.tpu.python.tpu import training_loop
 from tensorflow.contrib.tpu.python.tpu import util as util_lib
+from tensorflow.contrib.tpu.python.tpu._tpu_estimator_embedding import AdamParameters  # pylint: disable=unused-import
+from tensorflow.contrib.tpu.python.tpu._tpu_estimator_embedding import EmbeddingConfigSpec  # pylint: disable=unused-import
 from tensorflow.contrib.training.python.training import hparam
 from tensorflow.core.framework import variable_pb2
 from tensorflow.core.framework.summary_pb2 import Summary
@@ -441,13 +444,20 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook):
                run_infeed_loop_on_coordinator=True,
                rendezvous=None,
                master=None,
-               session_config=None):
+               session_config=None,
+               tpu_init_ops=None):
     self._master_job = ctx.master_job
     self._enqueue_ops = enqueue_ops
     self._dequeue_ops = dequeue_ops
     self._rendezvous = rendezvous
     self._master = master
     self._session_config = session_config
+    self._init_ops = list(tpu_init_ops or [])
+    if ctx.embedding_config is None:
+      self._embedding_layer_config = None
+    else:
+      self._embedding_layer_config = (
+          ctx.embedding_config.tpu_embedding.config_proto)
     self._run_infeed_loop_on_coordinator = run_infeed_loop_on_coordinator
     self._initial_infeed_sleep_secs = (
         ctx.config.tpu_config.initial_infeed_sleep_secs)
@@ -460,7 +470,6 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook):
   def begin(self):
     logging.info('TPU job name %s', self._master_job)
     self._iterations_per_loop_var = _create_or_get_iterations_per_loop()
-    self._init_ops = []
     if self._should_initialize_tpu:
       self._finalize_ops = [tpu.shutdown_system(job=self._master_job)]
     else:
@@ -520,7 +529,10 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook):
       with ops.Graph().as_default():
         with tf_session.Session(
             self._master, config=self._session_config) as sess:
-          sess.run(tpu.initialize_system(job=self._master_job))
+          sess.run(
+              tpu.initialize_system(
+                  job=self._master_job,
+                  embedding_config=self._embedding_layer_config))
       logging.info('Initialized TPU in %d seconds', time.time() - start)
 
     session.run(self._init_ops,
@@ -862,6 +874,7 @@ def generate_per_host_v2_enqueue_ops_fn_for_host(
     """Generates the per_host enqueue ops."""
     control_deps = []
     per_host_sharded_inputs = []
+    sparse_features_list = []
     num_replicas_per_host = ctx.num_of_replicas_per_host
     cached_signals = None
     with ops.device(device):
@@ -880,6 +893,10 @@ def generate_per_host_v2_enqueue_ops_fn_for_host(
           else:
             cached_signals = signals
 
+        features, labels, sparse_features = (
+            _tpu_estimator_embedding.split_inputs(ctx, features, labels))
+        sparse_features_list.append(sparse_features)
+
         inputs_structure_recorder.validate_and_record_structure(
             features, labels)
         flattened_inputs = (
@@ -908,6 +925,11 @@ def generate_per_host_v2_enqueue_ops_fn_for_host(
             tpu_ordinal_function=tpu_ordinal_function_impl)
       captured_infeed_queue.capture(infeed_queue)
 
+    if ctx.embedding_config:
+      per_host_enqueue_ops.extend(
+          ctx.embedding_config.tpu_embedding.generate_enqueue_ops(
+              sparse_features_list))
+
     if signals is None:
       return per_host_enqueue_ops
     else:
@@ -1374,6 +1396,12 @@ class _ModelFnWrapper(object):
   def call_without_tpu(self, features, labels, is_export_mode):
     return self._call_model_fn(features, labels, is_export_mode=is_export_mode)
 
+  def _add_embedding_features(self, features):
+    if self._ctx.embedding_config:
+      tpu_embedding_ = self._ctx.embedding_config.tpu_embedding
+      embedding_activations = tpu_embedding_.get_activations()
+      features.update(embedding_activations)
+
   def convert_to_single_tpu_train_step(self, dequeue_fn):
     """Converts user provided model_fn` as a single train step on TPU.
 
@@ -1406,6 +1434,7 @@ class _ModelFnWrapper(object):
       del loss  # unused; required in function signature.
       inputs = dequeue_fn()
       features, labels = inputs.features_and_labels()
+      self._add_embedding_features(features)
 
       estimator_spec = self._verify_estimator_spec(
           self._call_model_fn(features, labels))
@@ -1425,9 +1454,16 @@ class _ModelFnWrapper(object):
                                          self._ctx.num_replicas,
                                          fetches=[loss, train_op])
 
+      if self._ctx.embedding_config is None:
+        apply_sparse_grads = []
+      else:
+        tpu_embedding_ = self._ctx.embedding_config.tpu_embedding
+        apply_sparse_grads = [tpu_embedding_.generate_send_gradients_op()]
+
       # We must run train_op to update the variables prior to running the
       # outfeed.
-      with ops.control_dependencies([train_op]+tracing_ops):
+      with ops.control_dependencies([train_op] + tracing_ops +
+                                    apply_sparse_grads):
         host_call_outfeed_ops = []
         if (isinstance(estimator_spec, model_fn_lib._TPUEstimatorSpec)  # pylint: disable=protected-access
             and estimator_spec.host_call is not None):
@@ -1473,6 +1509,7 @@ class _ModelFnWrapper(object):
       """Evaluation step function for use inside a while loop."""
       inputs = dequeue_fn()
       features, labels = inputs.features_and_labels()
+      self._add_embedding_features(features)
 
       tpu_estimator_spec = self._call_model_fn(features, labels)
       if not isinstance(tpu_estimator_spec, model_fn_lib._TPUEstimatorSpec):  # pylint: disable=protected-access
@@ -2170,7 +2207,8 @@ class TPUEstimator(estimator_lib.Estimator):
                export_to_cpu=True,
                warm_start_from=None,
                experimental_exported_model_uses_all_cores=False,
-               experimental_export_device_assignment=False):
+               experimental_export_device_assignment=False,
+               experimental_embedding_config_spec=None):
     """Constructs an `TPUEstimator` instance.
 
     Args:
@@ -2233,6 +2271,9 @@ class TPUEstimator(estimator_lib.Estimator):
         assignment in the exported model. Doing so is useful in case of model
         parallel inference but will tie the exported model to the TPU topology
         used to export the model.
+      experimental_embedding_config_spec: Optional EmbeddingConfigSpec instance
+        to support using TPU embedding. IT IS STILL WORK IN PROGRESS, SO PLEASE
+        DO NOT USE.
 
     Raises:
       ValueError: `params` has reserved keys already.
@@ -2294,7 +2335,7 @@ class TPUEstimator(estimator_lib.Estimator):
     # pylint: disable=protected-access
     self._ctx = tpu_context._get_tpu_context(
         self._config, train_batch_size, eval_batch_size, predict_batch_size,
-        use_tpu, eval_on_tpu)
+        use_tpu, eval_on_tpu, experimental_embedding_config_spec)
 
     self._export_to_cpu = export_to_cpu
     self._export_to_tpu = export_to_tpu
@@ -2735,6 +2776,10 @@ class TPUEstimator(estimator_lib.Estimator):
         assert callable(features), '`input_fn` is not callable.'
         input_fn = features
 
+        tpu_init_ops = []
+        if ctx.embedding_config:
+          tpu_init_ops.extend(ctx.embedding_config.tpu_embedding.init_ops)
+
         input_holders = _InputPipeline(input_fn, batch_axis, ctx)
         enqueue_ops, dequeue_fn, input_hooks, run_infeed_loop_on_coordinator = (
             input_holders.generate_infeed_enqueue_ops_and_dequeue_fn())
@@ -2788,7 +2833,7 @@ class TPUEstimator(estimator_lib.Estimator):
                   rendezvous=self._rendezvous[mode],
                   master=self._config.master,
                   session_config=self._session_config,
-              ),
+                  tpu_init_ops=tpu_init_ops),
               InstallSignalHandlerHook()
           ])
           if self._log_every_n_steps is not None:
@@ -2825,6 +2870,10 @@ class TPUEstimator(estimator_lib.Estimator):
           with ops.control_dependencies([loss]):
             update_ops = _sync_variables_ops(ctx)
 
+          if ctx.embedding_config:
+            update_ops.extend(
+                ctx.embedding_config.tpu_embedding.retrieve_parameters_ops)
+
           # Validate the TPU training graph to catch basic errors
           _validate_tpu_training_graph()
 
@@ -2894,7 +2943,8 @@ class TPUEstimator(estimator_lib.Estimator):
                   rendezvous=self._rendezvous[mode],
                   master=self._config.evaluation_master,
                   session_config=self._session_config,
-              )] + input_hooks
+                  tpu_init_ops=tpu_init_ops)
+          ] + input_hooks
 
           if eval_hooks:
             hooks.extend(eval_hooks)
-- 
GitLab


From a29d085d97a90a315e319b67810a98476c92d34e Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Fri, 25 Jan 2019 18:51:36 -0800
Subject: [PATCH 1333/2345] Increase shards for one of the keras DS test so it
 doesn't time out.

PiperOrigin-RevId: 231007037
---
 tensorflow/contrib/distribute/python/BUILD | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 090fe09d02..e6bf6ed3e8 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -628,13 +628,12 @@ distribute_py_test(
     srcs = ["keras_backward_compat_test.py"],
     full_precision = True,
     main = "keras_backward_compat_test.py",
-    shard_count = 16,
+    shard_count = 31,
     tags = [
         "multi_and_single_gpu",
         "no_oss",  # TODO(b/117919883): Fix python error.
         "no_pip",
         "no_windows_gpu",
-        "notap",  # TODO(123409374): Re-enable this test in TAP.
         "notsan",
     ],
     deps = [
-- 
GitLab


From 371791b1490b5109b0695e6065374428cc57703b Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Fri, 25 Jan 2019 20:02:57 -0800
Subject: [PATCH 1334/2345] Automated rollback of commit
 81df31594971f6870bf5b7d75e311de3e970e670

PiperOrigin-RevId: 231011737
---
 tensorflow/python/keras/engine/base_layer.py  | 36 +++----------------
 tensorflow/python/keras/engine/input_layer.py |  2 +-
 tensorflow/python/keras/engine/network.py     |  4 +--
 tensorflow/python/keras/models.py             |  8 ++---
 4 files changed, 12 insertions(+), 38 deletions(-)

diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index 6218dbeed5..5eb2c5a752 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -18,7 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import collections
 import functools
 import inspect  # Necessary supplement to tf_inspect to deal with variadic args.
 import itertools
@@ -303,7 +302,7 @@ class Layer(checkpointable.Checkpointable):
       if kwarg not in ['getter', 'collections']:
         raise TypeError('Unknown keyword argument:', kwarg)
     getter = kwargs.pop('getter', None)
-    collections = kwargs.pop('collections', None)  # pylint: disable=redefined-outer-name
+    collections = kwargs.pop('collections', None)
 
     if dtype is None:
       dtype = self.dtype or backend.floatx()
@@ -1493,11 +1492,11 @@ class Layer(checkpointable.Checkpointable):
         arguments: dictionary of keyword arguments that were passed to the
             `call` method of the layer at the call that created the node.
     """
-    inbound_layers = nest.map_structure(lambda t: t._keras_history.layer,
+    inbound_layers = nest.map_structure(lambda t: t._keras_history[0],
                                         input_tensors)
-    node_indices = nest.map_structure(lambda t: t._keras_history.node_index,
+    node_indices = nest.map_structure(lambda t: t._keras_history[1],
                                       input_tensors)
-    tensor_indices = nest.map_structure(lambda t: t._keras_history.tensor_index,
+    tensor_indices = nest.map_structure(lambda t: t._keras_history[2],
                                         input_tensors)
 
     # Create node, add it to inbound nodes.
@@ -1519,8 +1518,7 @@ class Layer(checkpointable.Checkpointable):
     # or multi-input layers (e.g. a layer can return multiple tensors,
     # and each can be sent to a different layer).
     for i, tensor in enumerate(nest.flatten(output_tensors)):
-      tensor._keras_history = KerasHistory(self,
-                                           len(self._inbound_nodes) - 1, i)  # pylint: disable=protected-access
+      tensor._keras_history = (self, len(self._inbound_nodes) - 1, i)  # pylint: disable=protected-access
 
   def _get_node_attribute_at_index(self, node_index, attr, attr_name):
     """Private utility to retrieves an attribute (e.g. inputs) from a node.
@@ -1829,30 +1827,6 @@ class TensorFlowOpLayer(Layer):
     return config
 
 
-class KerasHistory(
-    collections.namedtuple('KerasHistory',
-                           ['layer', 'node_index', 'tensor_index'])):
-  """Tracks the Layer calls that created a Tensor, for Keras Graph Networks.
-
-  During construction of Keras Graph Networks, this metadata is added to
-  each Tensor produced as the output of a Layer, starting with an
-  `InputLayer`. This allows Keras to track how each Tensor was produced, and
-  this information is later retraced by the `keras.engine.Network` class to
-  reconstruct the Keras Graph Network.
-
-  Attributes:
-    layer: The Layer that produced the Tensor.
-    node_index: The specific call to the Layer that produced this Tensor. Layers
-      can be called multiple times in order to share weights. A new node is
-      created every time a Tensor is called.
-    tensor_index: The output index for this Tensor. Always zero if the Layer
-      that produced this Tensor only has one output.
-  """
-  # Added to maintain memory and performance characteristics of `namedtuple`
-  # while subclassing.
-  __slots__ = ()
-
-
 def default(method):
   """Decorates a method to detect overrides in subclasses."""
   method._is_default = True
diff --git a/tensorflow/python/keras/engine/input_layer.py b/tensorflow/python/keras/engine/input_layer.py
index 0cc976154e..c6dcedfce2 100644
--- a/tensorflow/python/keras/engine/input_layer.py
+++ b/tensorflow/python/keras/engine/input_layer.py
@@ -122,7 +122,7 @@ class InputLayer(base_layer.Layer):
 
     # Create an input node to add to self.outbound_node
     # and set output_tensors' _keras_history.
-    input_tensor._keras_history = base_layer.KerasHistory(self, 0, 0)  # pylint: disable=protected-access
+    input_tensor._keras_history = (self, 0, 0)  # pylint: disable=protected-access
     base_layer.Node(
         self,
         inbound_layers=[],
diff --git a/tensorflow/python/keras/engine/network.py b/tensorflow/python/keras/engine/network.py
index f3d20e4dc4..41f5f319bc 100644
--- a/tensorflow/python/keras/engine/network.py
+++ b/tensorflow/python/keras/engine/network.py
@@ -1650,7 +1650,7 @@ class Network(base_layer.Layer):
                          ' (missing previous layer metadata).')
       # Check that x is an input tensor.
       # pylint: disable=protected-access
-      layer = x._keras_history.layer
+      layer, _, _ = x._keras_history
       if len(layer._inbound_nodes) > 1 or (
           layer._inbound_nodes and layer._inbound_nodes[0].inbound_layers):
         cls_name = self.__class__.__name__
@@ -1667,7 +1667,7 @@ class Network(base_layer.Layer):
 
     # Check compatibility of batch sizes of Input Layers.
     input_batch_sizes = [
-        training_utils.get_static_batch_size(x._keras_history.layer)
+        training_utils.get_static_batch_size(x._keras_history[0])
         for x in self.inputs
     ]
     consistent_batch_size = None
diff --git a/tensorflow/python/keras/models.py b/tensorflow/python/keras/models.py
index cfae752431..9bc5aa2be5 100644
--- a/tensorflow/python/keras/models.py
+++ b/tensorflow/python/keras/models.py
@@ -98,7 +98,7 @@ def _clone_functional_model(model, input_tensors=None, share_weights=False):
           name=layer.name)
       input_tensors.append(input_tensor)
       # Cache newly created input layer.
-      newly_created_input_layer = input_tensor._keras_history.layer
+      newly_created_input_layer = input_tensor._keras_history[0]
       layer_map[layer] = newly_created_input_layer
 
     for original_input_layer, cloned_input_layer in zip(model._input_layers,
@@ -119,7 +119,7 @@ def _clone_functional_model(model, input_tensors=None, share_weights=False):
 
         input_tensors_.append(input_tensor)
         # Cache newly created input layer.
-        newly_created_input_layer = input_tensor._keras_history.layer
+        newly_created_input_layer = input_tensor._keras_history[0]
         layer_map[original_input_layer] = newly_created_input_layer
       else:
         input_tensors_.append(input_tensor)
@@ -239,7 +239,7 @@ def _clone_sequential_model(model, input_tensors=None, share_weights=False):
       input_tensors = list(input_tensors)
     x = generic_utils.to_list(input_tensors)[0]
     if K.is_keras_tensor(x):
-      origin_layer = x._keras_history.layer
+      origin_layer = x._keras_history[0]
       if isinstance(origin_layer, InputLayer):
         return Sequential(layers=[origin_layer] + layers, name=model.name)
       else:
@@ -248,7 +248,7 @@ def _clone_sequential_model(model, input_tensors=None, share_weights=False):
                          'other than an `InputLayer`. '
                          'Use the functional API instead.')
     input_tensor = Input(tensor=x, name='input_wrapper_for_' + str(x.name))
-    input_layer = input_tensor._keras_history.layer
+    input_layer = input_tensor._keras_history[0]
     return Sequential(layers=[input_layer] + layers, name=model.name)
 
 
-- 
GitLab


From b63538a6a78f959b94c76b223cc07736ab81da49 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Fri, 25 Jan 2019 21:35:37 -0800
Subject: [PATCH 1335/2345] Internal change.

PiperOrigin-RevId: 231017272
---
 tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh | 2 +-
 tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
index d33b732cc5..9741fba1c8 100644
--- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
@@ -105,7 +105,7 @@ fi
 
 if [[ "$TF_NIGHTLY" == 1 ]]; then
   if [[ ${PROJECT_NAME} == *"2.0_preview"* ]]; then
-    python tensorflow/tools/ci_build/update_version.py --version=2.0.0-preview
+    python tensorflow/tools/ci_build/update_version.py --version=2.0.0 --nightly
   else
     python tensorflow/tools/ci_build/update_version.py --nightly
   fi
diff --git a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
index 4cca4f9acf..efdd5f13c8 100644
--- a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
@@ -105,7 +105,7 @@ fi
 
 if [[ "$TF_NIGHTLY" == 1 ]]; then
   if [[ ${PROJECT_NAME} == *"2.0_preview"* ]]; then
-    python tensorflow/tools/ci_build/update_version.py --version=2.0.0-preview
+    python tensorflow/tools/ci_build/update_version.py --version=2.0.0 --nightly
   else
     python tensorflow/tools/ci_build/update_version.py --nightly
   fi
-- 
GitLab


From 93b8126565fa3ad8808142e81370a81d19d81ea1 Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Fri, 25 Jan 2019 21:40:01 -0800
Subject: [PATCH 1336/2345] [XLA:GPU] Avoid using hyphens in global buffer
 names.

Having a hyphen in a global variable name can crash the PTX backend. When the
name of an Hlo instruction is used to construct a name of the global buffer for
the instruction, replace hyphens with underscores.

Add two test cases.

PiperOrigin-RevId: 231017603
---
 .../compiler/xla/service/gpu/tests/BUILD      | 15 ++++
 .../gpu/tests/gpu_buffer_assignment_test.cc   | 90 +++++++++++++++++++
 .../service/llvm_ir/buffer_assignment_util.cc |  7 +-
 3 files changed, 110 insertions(+), 2 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/gpu/tests/gpu_buffer_assignment_test.cc

diff --git a/tensorflow/compiler/xla/service/gpu/tests/BUILD b/tensorflow/compiler/xla/service/gpu/tests/BUILD
index d798b31643..d8bd9f7f6d 100644
--- a/tensorflow/compiler/xla/service/gpu/tests/BUILD
+++ b/tensorflow/compiler/xla/service/gpu/tests/BUILD
@@ -47,6 +47,21 @@ cc_library(
     ],
 )
 
+tf_cc_test(
+    name = "gpu_buffer_assignment_test",
+    srcs = ["gpu_buffer_assignment_test.cc"],
+    tags = tf_cuda_tests_tags(),
+    deps = [
+        ":gpu_codegen_test",
+        "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/compiler/xla/service:hlo_module_config",
+        "//tensorflow/compiler/xla/service:hlo_parser",
+        "//tensorflow/compiler/xla/tests:hlo_test_base",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
 tf_cc_test(
     name = "gpu_copy_test",
     srcs = ["gpu_copy_test.cc"],
diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_buffer_assignment_test.cc b/tensorflow/compiler/xla/service/gpu/tests/gpu_buffer_assignment_test.cc
new file mode 100644
index 0000000000..1335d73494
--- /dev/null
+++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_buffer_assignment_test.cc
@@ -0,0 +1,90 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <utility>
+
+#include "tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.h"
+#include "tensorflow/compiler/xla/service/hlo_module_config.h"
+#include "tensorflow/compiler/xla/service/hlo_parser.h"
+#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace xla {
+namespace gpu {
+namespace {
+
+class GpuBufferAssignmentTest : public GpuCodegenTest {
+ public:
+  HloModuleConfig ConfigWithoutHloPasses() {
+    HloModuleConfig config;
+    auto debug_options = HloTestBase::GetDebugOptionsForTest();
+    // Disable layout_assignment to use the preassigned layouts.
+    debug_options.xla_disable_all_hlo_passes();
+    config.set_debug_options(debug_options);
+    return config;
+  }
+};
+
+TEST_F(GpuBufferAssignmentTest, InstructionNameWithHyphenSanitized) {
+  const char *const kHloString = R"(
+    HloModule HyphenInInstructionName
+      ENTRY kernelEntry {
+        ROOT equal-to = s32[2]{0} constant({42, 73})
+    })";
+
+  // Check that '-' in the instruction name is changed to '_'.
+  auto hlo_module = ParseHloString(kHloString).ValueOrDie();
+  CompileAndVerifyIr(std::move(hlo_module),
+                     R"(
+; CHECK: buffer_for_equal_to =
+)",
+                     /*match_optimized_ir=*/true);
+
+  // TODO(bixia): The run fails randomly.
+  // Check that the kernel runs correctly.
+  // EXPECT_TRUE(RunAndCompareNoHloPasses(kHloString, ErrorSpec{0.0}));
+}
+
+TEST_F(GpuBufferAssignmentTest, BufferSanitizedNameCollisionResolved) {
+  const char *const kHloString = R"(
+    HloModule BufferSanitizedName
+      ENTRY kernelEntry {
+      equal.to = s32[2]{0} constant({42, 73})
+      equal-to = s32[2]{0} constant({67, 3})
+      ROOT add = s32[2]{0} add(equal.to, equal-to)
+    })";
+
+  // Turn of Hlo passes to prevent constant folding.
+  //
+  // Check that '-' and '.' in the instruction name are changed to '_', and
+  // name collision is resolved by LLVM.
+  auto hlo_module =
+      ParseHloString(kHloString, ConfigWithoutHloPasses()).ValueOrDie();
+  CompileAndVerifyIr(std::move(hlo_module),
+                     R"(
+; CHECK: buffer_for_equal_to =
+; CHECK: buffer_for_equal_to1 =
+)",
+                     /*match_optimized_ir=*/false);
+
+  // TODO(bixia): There is another bug that prevents this from running
+  //              correctly.
+  // Check that the kernel runs correctly.
+  // EXPECT_TRUE(RunAndCompareNoHloPasses(kHloString, ErrorSpec{0.0}));
+}
+
+}  // namespace
+}  // namespace gpu
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/llvm_ir/buffer_assignment_util.cc b/tensorflow/compiler/xla/service/llvm_ir/buffer_assignment_util.cc
index bdce4a171b..c2c6405cda 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/buffer_assignment_util.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/buffer_assignment_util.cc
@@ -1,4 +1,4 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -45,7 +45,10 @@ string ConstantBufferAllocationToGlobalName(
     const BufferAllocation& allocation) {
   string instr_name = InstrForConstantBufferAllocation(allocation).name();
   for (char& c : instr_name) {
-    if (c == '.') {
+    // Having a hyphen in a global variable name can crash the LLVM PTX backend.
+    // LLVM is able to generate unique global variable names using the string
+    // returned from here as name prefix.
+    if (c == '.' || c == '-') {
       c = '_';
     }
   }
-- 
GitLab


From 8a1aca3da83e9ef451c73ac428a1da242c4913ce Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 25 Jan 2019 21:55:44 -0800
Subject: [PATCH 1337/2345] [Grappler] Integrate graph verifiers in
 meta_optimizer.

Also, introduce a verifier_config to control verifiers that are run both in between optimization passes as well as at the end.

PiperOrigin-RevId: 231018302
---
 .../makefile/proto_text_pb_cc_files.txt       |   1 +
 .../makefile/proto_text_pb_h_files.txt        |   1 +
 .../contrib/makefile/tf_pb_text_files.txt     |   1 +
 .../contrib/makefile/tf_proto_files.txt       |   1 +
 tensorflow/core/BUILD                         |   1 +
 tensorflow/core/grappler/optimizers/BUILD     |   3 +
 .../grappler/optimizers/meta_optimizer.cc     |  40 +++
 .../core/grappler/optimizers/meta_optimizer.h |   8 +
 .../optimizers/meta_optimizer_test.cc         | 233 ++++++++++++++++--
 tensorflow/core/grappler/verifiers/BUILD      |   1 -
 .../core/protobuf/rewriter_config.proto       |  10 +-
 .../core/protobuf/verifier_config.proto       |  26 ++
 12 files changed, 302 insertions(+), 24 deletions(-)
 create mode 100644 tensorflow/core/protobuf/verifier_config.proto

diff --git a/tensorflow/contrib/makefile/proto_text_pb_cc_files.txt b/tensorflow/contrib/makefile/proto_text_pb_cc_files.txt
index 87c73ec1ca..8330c45cc1 100644
--- a/tensorflow/contrib/makefile/proto_text_pb_cc_files.txt
+++ b/tensorflow/contrib/makefile/proto_text_pb_cc_files.txt
@@ -36,6 +36,7 @@ tensorflow/core/protobuf/queue_runner.pb.cc
 tensorflow/core/protobuf/rewriter_config.pb.cc
 tensorflow/core/protobuf/saver.pb.cc
 tensorflow/core/protobuf/tensorflow_server.pb.cc
+tensorflow/core/protobuf/verifier_config.pb.cc
 tensorflow/core/util/event.pb.cc
 tensorflow/core/util/memmapped_file_system.pb.cc
 tensorflow/core/util/saved_tensor_slice.pb.cc
diff --git a/tensorflow/contrib/makefile/proto_text_pb_h_files.txt b/tensorflow/contrib/makefile/proto_text_pb_h_files.txt
index 4120ea52ec..7257ac8fee 100644
--- a/tensorflow/contrib/makefile/proto_text_pb_h_files.txt
+++ b/tensorflow/contrib/makefile/proto_text_pb_h_files.txt
@@ -37,6 +37,7 @@ tensorflow/core/protobuf/rewriter_config.pb.h
 tensorflow/core/protobuf/saver.pb.h
 tensorflow/core/protobuf/tensor_bundle.pb.h
 tensorflow/core/protobuf/tensorflow_server.pb.h
+tensorflow/core/protobuf/verifier_config.pb.h
 tensorflow/core/util/event.pb.h
 tensorflow/core/util/memmapped_file_system.pb.h
 tensorflow/core/util/saved_tensor_slice.pb.h
diff --git a/tensorflow/contrib/makefile/tf_pb_text_files.txt b/tensorflow/contrib/makefile/tf_pb_text_files.txt
index f94d70db90..13e3b6422d 100644
--- a/tensorflow/contrib/makefile/tf_pb_text_files.txt
+++ b/tensorflow/contrib/makefile/tf_pb_text_files.txt
@@ -29,5 +29,6 @@ tensorflow/core/protobuf/debug.pb_text.cc
 tensorflow/core/protobuf/rewriter_config.pb_text.cc
 tensorflow/core/protobuf/saver.pb_text.cc
 tensorflow/core/protobuf/tensor_bundle.pb_text.cc
+tensorflow/core/protobuf/verifier_config.pb_text.cc
 tensorflow/core/util/memmapped_file_system.pb_text.cc
 tensorflow/core/util/saved_tensor_slice.pb_text.cc
diff --git a/tensorflow/contrib/makefile/tf_proto_files.txt b/tensorflow/contrib/makefile/tf_proto_files.txt
index 2712e906d7..24d86d313b 100644
--- a/tensorflow/contrib/makefile/tf_proto_files.txt
+++ b/tensorflow/contrib/makefile/tf_proto_files.txt
@@ -43,6 +43,7 @@ tensorflow/core/protobuf/rewriter_config.proto
 tensorflow/core/protobuf/saver.proto
 tensorflow/core/protobuf/tensor_bundle.proto
 tensorflow/core/protobuf/tensorflow_server.proto
+tensorflow/core/protobuf/verifier_config.proto
 tensorflow/core/util/event.proto
 tensorflow/core/util/memmapped_file_system.proto
 tensorflow/core/util/saved_tensor_slice.proto
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index c3d31f91b3..ebc8c309ec 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -205,6 +205,7 @@ COMMON_PROTO_SRCS = [
     "protobuf/rewriter_config.proto",
     "protobuf/tensor_bundle.proto",
     "protobuf/saver.proto",
+    "protobuf/verifier_config.proto",
     "util/event.proto",
     "util/memmapped_file_system.proto",
     "util/saved_tensor_slice.proto",
diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index ee216f80f9..4af8fc9874 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -537,6 +537,8 @@ cc_library(
         "//tensorflow/core/grappler/utils:colocation",
         "//tensorflow/core/grappler/utils:functions",
         "//tensorflow/core/grappler/utils:topological_sort",
+        "//tensorflow/core/grappler/verifiers:graph_verifier",
+        "//tensorflow/core/grappler/verifiers:structure_verifier",
         "@com_google_absl//absl/strings",
     ],
 )
@@ -549,6 +551,7 @@ tf_cuda_cc_test(
         ":custom_graph_optimizer_registry",
         ":meta_optimizer",
         "//tensorflow/cc:cc_ops",
+        "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:tensorflow",
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
index e4b2e4c55a..aa19d33f46 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
@@ -39,6 +39,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/utils/colocation.h"
 #include "tensorflow/core/grappler/utils/functions.h"
 #include "tensorflow/core/grappler/utils/topological_sort.h"
+#include "tensorflow/core/grappler/verifiers/structure_verifier.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/gtl/map_util.h"
 #include "tensorflow/core/util/dump_graph.h"
@@ -284,6 +285,20 @@ MetaOptimizer::GetCustomGraphOptimizerConfig(const string& name) const {
   return nullptr;
 }
 
+void MetaOptimizer::InitializeVerifiers(
+    std::vector<std::unique_ptr<GraphVerifier>>* inter_optimizer_verifiers,
+    std::vector<std::unique_ptr<GraphVerifier>>* post_optimization_verifiers)
+    const {
+  if (cfg_.inter_optimizer_verifier_config().structure_verifier() ==
+      VerifierConfig::ON) {
+    inter_optimizer_verifiers->push_back(MakeUnique<StructureVerifier>());
+  }
+  if (cfg_.post_optimization_verifier_config().structure_verifier() ==
+      VerifierConfig::ON) {
+    post_optimization_verifiers->push_back(MakeUnique<StructureVerifier>());
+  }
+}
+
 #define RUN_OPTIMIZER_OR_RETURN_IF_ERROR(optimizer)                            \
   {                                                                            \
     const Status status = RunOptimizer(optimizer, cluster, &optimized_item,    \
@@ -314,6 +329,23 @@ Status MetaOptimizer::OptimizeGraph(Cluster* cluster, const GrapplerItem& item,
     TF_RETURN_IF_ERROR(InitializeOptimizersByName(&optimizers));
   }
 
+  // Initialize the configured verifiers.
+  std::vector<std::unique_ptr<GraphVerifier>> inter_optimizer_verifiers;
+  std::vector<std::unique_ptr<GraphVerifier>> post_optimization_verifiers;
+  InitializeVerifiers(&inter_optimizer_verifiers, &post_optimization_verifiers);
+  if (inter_optimizer_verifiers.empty()) {
+    VLOG(2) << "No inter optimizer verifiers have been configured";
+  } else {
+    VLOG(2) << inter_optimizer_verifiers.size()
+            << " inter optimizer verifiers have been configured";
+  }
+  if (post_optimization_verifiers.empty()) {
+    VLOG(2) << "No post optimization verifiers have been configured";
+  } else {
+    VLOG(2) << post_optimization_verifiers.size()
+            << " post optimization verifiers have been configured";
+  }
+
   VLOG(2) << "Optimize GrapplerItem: item.id=" << item.id
           << " num_optimizers=" << optimizers.size()
           << ", num nodes = " << item.graph.node_size();
@@ -358,6 +390,14 @@ Status MetaOptimizer::OptimizeGraph(Cluster* cluster, const GrapplerItem& item,
         continue;
       }
       RUN_OPTIMIZER_OR_RETURN_IF_ERROR(optimizer.get());
+      for (const auto& verifier : inter_optimizer_verifiers) {
+        // TODO(ashwinm): Need to enforce verification_deadline.
+        TF_RETURN_IF_ERROR(verifier->Verify(*optimized_graph));
+      }
+    }
+    // TODO(ashwinm): Need to enforce verification_deadline.
+    for (const auto& verifier : post_optimization_verifiers) {
+      TF_RETURN_IF_ERROR(verifier->Verify(*optimized_graph));
     }
   }
 
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.h b/tensorflow/core/grappler/optimizers/meta_optimizer.h
index aaf8ef938f..ec78cc5771 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer.h
@@ -22,9 +22,11 @@ limitations under the License.
 #include "tensorflow/core/graph/graph.h"
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/optimizers/graph_optimizer.h"
+#include "tensorflow/core/grappler/verifiers/graph_verifier.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/protobuf/config.pb.h"
 #include "tensorflow/core/protobuf/rewriter_config.pb.h"
+#include "tensorflow/core/protobuf/verifier_config.pb.h"
 
 namespace tensorflow {
 namespace grappler {
@@ -63,6 +65,12 @@ class MetaOptimizer : public GraphOptimizer {
   const RewriterConfig::CustomGraphOptimizer* GetCustomGraphOptimizerConfig(
       const string& name) const;
 
+  // Initialiaze active verifiers from the RewriterConfig toggles.
+  void InitializeVerifiers(
+      std::vector<std::unique_ptr<GraphVerifier>>* inter_optimizer_verifiers,
+      std::vector<std::unique_ptr<GraphVerifier>>* post_optimization_verifiers)
+      const;
+
   // Run optimization pass over a single GrapplerItem. Meta optimizer might run
   // multiple such passes: 1) for the main graph 2) for the function library
   Status OptimizeGraph(Cluster* cluster, const GrapplerItem& item,
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc b/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
index b1b72075ca..55f14b3b68 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
@@ -15,9 +15,11 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/meta_optimizer.h"
 
+#include "absl/strings/match.h"
 #include "absl/strings/substitute.h"
 #include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/core/framework/function_testlib.h"
+#include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/framework/tensor_testutil.h"
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/inputs/trivial_test_graph_input_yielder.h"
@@ -27,7 +29,9 @@ limitations under the License.
 #include "tensorflow/core/grappler/utils/grappler_test.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/gtl/map_util.h"
+#include "tensorflow/core/platform/protobuf.h"
 #include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/protobuf/config.pb.h"
 
 namespace tensorflow {
 namespace grappler {
@@ -254,13 +258,13 @@ TEST_F(MetaOptimizerTest, OptimizeFunctionLibrary) {
   FunctionDef mul_func = FunctionDefHelper::Create(
       "MyMul", {"x:T", "y:T"}, {"z:T"}, {"T: {float, double}"},
       {{{"mul"}, "Mul", {"x", "y"}, {{"T", "$T"}}}},
-      /* Mapping between function returns and function node outputs. */
+      /*ret_def=*/
       {{"z", "mul:z:0"}});
 
   FunctionDef square_func = FunctionDefHelper::Create(
       "MySquare", {"x:T"}, {"z:T"}, {"T: {float, double}"},
       {{{"my_mul"}, "MyMul", {"x", "x"}, {{"T", "$T"}}}},
-      /* Mapping between function returns and function node outputs. */
+      /*ret_def=*/
       {{"z", "my_mul:z:0"}});
   (*square_func.mutable_attr())["_noinline"].set_b(true);
 
@@ -268,7 +272,7 @@ TEST_F(MetaOptimizerTest, OptimizeFunctionLibrary) {
       "MyQuadratic", {"x:T"}, {"z:T"}, {"T: {float, double}"},
       {{{"square"}, "MySquare", {"x"}, {{"T", "$T"}}},
        {{"quadratic"}, "MySquare", {"square:z"}, {{"T", "$T"}}}},
-      /* Mapping between function returns and function node outputs. */
+      /*ret_def=*/
       {{"z", "quadratic:z:0"}});
   (*quadratic_func.mutable_attr())["_noinline"].set_b(true);
 
@@ -290,7 +294,7 @@ TEST_F(MetaOptimizerTest, OptimizeFunctionLibrary) {
        // Forward outputs
        NDef("out_s", "Identity", {"square:0"}, {{"T", DT_FLOAT}}, kDevice),
        NDef("out_q", "Identity", {"quadratic:0"}, {{"T", DT_INT32}}, kDevice)},
-      // FunctionLib
+      /*funcs=*/
       {mul_func, square_func, quadratic_func});
 
   GraphDef output;
@@ -412,19 +416,20 @@ TEST_F(MetaOptimizerTest, OptimizeFunctionLibraryPruneUnusedOutputs) {
       {{{"output0"}, "Mul", {"x", "y"}, {{"T", "$T"}}},
        {{"output1"}, "Mul", {"x", "y"}, {{"T", "$T"}}},
        {{"output2"}, "Mul", {"x", "y"}, {{"T", "$T"}}}},
-      /* Mapping between function returns and function node outputs. */
+      /*ret_def=*/
       {{"z0", "output0:z:0"}, {"z1", "output1:z:0"}, {"z2", "output2:z:0"}});
 
   // Call MyMyl and forward all three outputs.
   FunctionDef my_fwd = FunctionDefHelper::Create(
       "Fwd", {"x:T", "y:T"}, {"z0:T", "z1:T", "z2:T"}, {"T: {float, int32}"},
       {{{"output"}, "MyMul", {"x", "y"}, {{"T", "$T"}}}},
-      /* Mapping between function returns and function node outputs. */
+      /*ret_def=*/
       {{"z0", "output:z0:0"}, {"z1", "output:z1:0"}, {"z2", "output:z2:0"}});
 
   // Mark both functions as `_noinline` to trigger specialization.
   (*my_mul.mutable_attr())["_noinline"].set_b(true);
   (*my_fwd.mutable_attr())["_noinline"].set_b(true);
+  /*funcs=*/
   std::vector<FunctionDef> function_library = {my_mul, my_fwd};
 
   // Tensorflow graph:
@@ -461,14 +466,14 @@ TEST_F(MetaOptimizerTest, OptimizeFunctionLibraryPruneUnusedOutputs) {
   FunctionDef expected_my_mul = FunctionDefHelper::Create(
       specialized_my_mul, {"x:float", "y:float"}, {"z2:float"}, {},
       {{{"output2"}, "Mul", {"x", "y"}, {{"T", DT_FLOAT}}}},
-      /* Mapping between function returns and function node outputs. */
+      /*ret_def=*/
       {{"z2", "output2:z:0"}});
 
   // Specialized Fwd should also have just one output argument.
   FunctionDef expected_my_fwd = FunctionDefHelper::Create(
       specialized_my_fwd, {"x:float", "y:float"}, {"z2:float"}, {},
       {{{"output"}, specialized_my_mul, {"x", "y"}, {{"T", DT_FLOAT}}}},
-      /* Mapping between function returns and function node outputs. */
+      /*ret_def=*/
       {{"z2", "output:z2:0"}});
 
   const FunctionDef* my_mul_spec = optimized_flib.Find(specialized_my_mul);
@@ -512,7 +517,7 @@ TEST_F(MetaOptimizerTest, OptimizeFunctionLibraryPruneFunctionBody) {
       "MyFunc", {"x:T", "y:T"}, {"z1:T", "z2:T"}, {"T: {float, double}"},
       {{{"mul1"}, "Mul", {"x", "y"}, {{"T", "$T"}}},
        {{"mul2"}, "Mul", {"x", "y"}, {{"T", "$T"}}}},
-      /* Mapping between function returns and function node outputs. */
+      /*ret_def=*/
       {{"z1", "mul1:z:0"}, {"z2", "mul2:z:0"}});
   (*my_func.mutable_attr())["_noinline"].set_b(true);
 
@@ -536,7 +541,7 @@ TEST_F(MetaOptimizerTest, OptimizeFunctionLibraryPruneFunctionBody) {
        // Read outputs of function call nodes
        NDef("out_fn1", "Identity", {"fn1:0"}, {{"T", DT_FLOAT}}, kDevice),
        NDef("out_fn2", "Identity", {"fn2:1"}, {{"T", DT_FLOAT}}, kDevice)},
-      // FunctionLib
+      /*funcs=*/
       {my_func});
 
   GraphDef output;
@@ -618,17 +623,17 @@ TEST_F(MetaOptimizerTest, OptimizeFunctionLibraryWithRestrictions) {
   MetaOptimizer optimizer(nullptr, config_proto);
 
   // Define simple function library with two identical mul functions.
-  FunctionDef mul_func_1 = FunctionDefHelper::Create(
-      "MyMul1", {"x:float", "y:float"}, {"z:float"}, {},
-      {{{"mul"}, "Mul", {"x", "y"}, {}}},
-      /* Mapping between function returns and function node outputs. */
-      {{"z", "mul:z:0"}});
-
-  FunctionDef mul_func_2 = FunctionDefHelper::Create(
-      "MyMul2", {"x:float", "y:float"}, {"z:float"}, {},
-      {{{"mul"}, "Mul", {"x", "y"}, {}}},
-      /* Mapping between function returns and function node outputs. */
-      {{"z", "mul:z:0"}});
+  FunctionDef mul_func_1 =
+      FunctionDefHelper::Create("MyMul1", {"x:float", "y:float"}, {"z:float"},
+                                {}, {{{"mul"}, "Mul", {"x", "y"}, {}}},
+                                /*ret_def=*/
+                                {{"z", "mul:z:0"}});
+
+  FunctionDef mul_func_2 =
+      FunctionDefHelper::Create("MyMul2", {"x:float", "y:float"}, {"z:float"},
+                                {}, {{{"mul"}, "Mul", {"x", "y"}, {}}},
+                                /*ret_def=*/
+                                {{"z", "mul:z:0"}});
 
   // Tensorflow graph:
   //
@@ -654,7 +659,7 @@ TEST_F(MetaOptimizerTest, OptimizeFunctionLibraryWithRestrictions) {
              {"Tin", DataTypeSlice{DT_FLOAT}},
              {"Tout", DataTypeSlice{DT_FLOAT, DT_FLOAT}}},
             kDevice)},
-      // FunctionLib
+      /*funcs=*/
       {mul_func_1, mul_func_2});
   item.fetch = {"mul_1", "mul_2", "dx"};
 
@@ -747,6 +752,190 @@ TEST_F(MetaOptimizerTest, OptimizerDoesNotTimeOut) {
   EXPECT_EQ(item.graph.node_size() + 1, output.node_size());
 }
 
+TEST_F(MetaOptimizerTest, RunPostOptimizationVerifiersOnValidGraph) {
+  TrivialTestGraphInputYielder fake_input(4, 1, 10, false, {"CPU:0"});
+  GrapplerItem item;
+  CHECK(fake_input.NextItem(&item));
+
+  ConfigProto config_proto;
+  auto& post_optimization_verifier_config =
+      *config_proto.mutable_graph_options()
+           ->mutable_rewrite_options()
+           ->mutable_post_optimization_verifier_config();
+  post_optimization_verifier_config.set_structure_verifier(VerifierConfig::ON);
+
+  MetaOptimizer optimizer(nullptr, config_proto);
+  GraphDef output;
+  const Status status = optimizer.Optimize(nullptr, item, &output);
+  TF_EXPECT_OK(status);
+}
+
+TEST_F(MetaOptimizerTest, RunInterOptimizerVerifiersOnValidGraph) {
+  TrivialTestGraphInputYielder fake_input(4, 1, 10, false, {"CPU:0"});
+  GrapplerItem item;
+  CHECK(fake_input.NextItem(&item));
+
+  ConfigProto config_proto;
+  auto& inter_optimizer_verifier_config =
+      *config_proto.mutable_graph_options()
+           ->mutable_rewrite_options()
+           ->mutable_inter_optimizer_verifier_config();
+  inter_optimizer_verifier_config.set_structure_verifier(VerifierConfig::ON);
+
+  MetaOptimizer optimizer(nullptr, config_proto);
+  GraphDef output;
+  const Status status = optimizer.Optimize(nullptr, item, &output);
+  TF_EXPECT_OK(status);
+}
+
+TEST_F(MetaOptimizerTest, RunPostOptimizationVerifiersOnInvalidGraph) {
+  using test::function::NDef;
+  using FDH = FunctionDefHelper;
+
+  gtl::FlatMap<string, GrapplerItem::OptimizationOptions> optimization_options;
+  GrapplerItemPropertiesAccumulator::SetOptimizationOptions(
+      &optimization_options);
+
+  // Define simple function library with two identical mul functions.
+  FunctionDef mul_func_1 =
+      FunctionDefHelper::Create("MyMul1", {"x:float", "y:float"}, {"z:float"},
+                                {}, {{{"mul"}, "Mul", {"x", "y"}, {}}},
+                                /*ret_def=*/
+                                {{"z", "mul:z:0"}});
+
+  FunctionDef mul_func_2 =
+      FunctionDefHelper::Create("MyMul2", {"x:float", "y:float"}, {"z:float"},
+                                {}, {{{"mul"}, "Mul", {"x", "y"}, {}}},
+                                /*ret_def=*/
+                                {{"z", "mul:z:0"}});
+
+  // Tensorflow graph:
+  //
+  //   x0 = tf.Placeholder(tf.float);
+  //   x1 = tf.Placeholder(tf.float);
+  //   dy = tf.Placeholder(tf.float);
+  //
+  //   mul_1 = MyMul1(x0, x1);
+  //   mul_2 = MyMul2(x0, x1);
+  //   dx = SymbolicGradient({x0, x1, dy}, f=MyMul2)
+  GrapplerItem item;
+  item.id = "main";
+  item.graph = test::function::GDef(
+      {NDef("x0", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("x1", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("dy", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("x1", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       // Calls into function library
+       NDef("mul_1", "MyMul1", {"x0", "x1"}, {}, kDevice),
+       NDef("mul_2", "MyMul2", {"x0", "x1"}, {}, kDevice),
+       // Symbolic gradient of a MyMul2
+       NDef("dx", "SymbolicGradient", {"x0", "x1", "dy"},
+            {{"f", FDH::FunctionRef("MyMul2", {})},
+             {"Tin", DataTypeSlice{DT_FLOAT}},
+             {"Tout", DataTypeSlice{DT_FLOAT, DT_FLOAT}}},
+            kDevice)},
+      /*funcs=*/
+      {mul_func_1, mul_func_2});
+  item.fetch = {"mul_1", "mul_2", "dx"};
+
+  GraphDef output;
+
+  // Call Optimize with post optimization verifiers.
+  ConfigProto config_proto;
+  auto& rewriter_config =
+      *config_proto.mutable_graph_options()->mutable_rewrite_options();
+
+  rewriter_config.set_meta_optimizer_iterations(RewriterConfig::TWO);
+  rewriter_config.add_optimizers("GrapplerItemPropertiesAccumulator");
+  rewriter_config.set_min_graph_nodes(-1);
+  auto& post_optimization_verifier_config =
+      *config_proto.mutable_graph_options()
+           ->mutable_rewrite_options()
+           ->mutable_post_optimization_verifier_config();
+  post_optimization_verifier_config.set_structure_verifier(VerifierConfig::ON);
+
+  MetaOptimizer optimizer_with_post_verifiers(nullptr, config_proto);
+  Status status =
+      optimizer_with_post_verifiers.Optimize(nullptr, item, &output);
+  EXPECT_EQ(status.code(), errors::Code::NOT_FOUND);
+  EXPECT_TRUE(
+      absl::StrContains(status.error_message(), "Op type not registered"));
+}
+
+TEST_F(MetaOptimizerTest, RunInterOptimizerVerifiersOnInvalidGraph) {
+  using test::function::NDef;
+  using FDH = FunctionDefHelper;
+
+  gtl::FlatMap<string, GrapplerItem::OptimizationOptions> optimization_options;
+  GrapplerItemPropertiesAccumulator::SetOptimizationOptions(
+      &optimization_options);
+
+  // Define simple function library with two identical mul functions.
+  FunctionDef mul_func_1 =
+      FunctionDefHelper::Create("MyMul1", {"x:float", "y:float"}, {"z:float"},
+                                {}, {{{"mul"}, "Mul", {"x", "y"}, {}}},
+                                /*ret_def=*/
+                                {{"z", "mul:z:0"}});
+
+  FunctionDef mul_func_2 =
+      FunctionDefHelper::Create("MyMul2", {"x:float", "y:float"}, {"z:float"},
+                                {}, {{{"mul"}, "Mul", {"x", "y"}, {}}},
+                                /*ret_def=*/
+                                {{"z", "mul:z:0"}});
+
+  // Tensorflow graph:
+  //
+  //   x0 = tf.Placeholder(tf.float);
+  //   x1 = tf.Placeholder(tf.float);
+  //   dy = tf.Placeholder(tf.float);
+  //
+  //   mul_1 = MyMul1(x0, x1);
+  //   mul_2 = MyMul2(x0, x1);
+  //   dx = SymbolicGradient({x0, x1, dy}, f=MyMul2)
+  GrapplerItem item;
+  item.id = "main";
+  item.graph = test::function::GDef(
+      {NDef("x0", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("x1", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("dy", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("x1", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       // Calls into function library
+       NDef("mul_1", "MyMul1", {"x0", "x1"}, {}, kDevice),
+       NDef("mul_2", "MyMul2", {"x0", "x1"}, {}, kDevice),
+       // Symbolic gradient of a MyMul2
+       NDef("dx", "SymbolicGradient", {"x0", "x1", "dy"},
+            {{"f", FDH::FunctionRef("MyMul2", {})},
+             {"Tin", DataTypeSlice{DT_FLOAT}},
+             {"Tout", DataTypeSlice{DT_FLOAT, DT_FLOAT}}},
+            kDevice)},
+      /*funcs=*/
+      {mul_func_1, mul_func_2});
+  item.fetch = {"mul_1", "mul_2", "dx"};
+
+  GraphDef output;
+
+  // Call Optimize with post optimization verifiers.
+  ConfigProto config_proto;
+  // Call Optimize with inter optimizer verifiers.
+  auto& rewriter_config =
+      *config_proto.mutable_graph_options()->mutable_rewrite_options();
+  rewriter_config.set_meta_optimizer_iterations(RewriterConfig::TWO);
+  rewriter_config.add_optimizers("GrapplerItemPropertiesAccumulator");
+  rewriter_config.set_min_graph_nodes(-1);
+  auto& inter_optimizer_verifier_config =
+      *config_proto.mutable_graph_options()
+           ->mutable_rewrite_options()
+           ->mutable_inter_optimizer_verifier_config();
+  inter_optimizer_verifier_config.set_structure_verifier(VerifierConfig::ON);
+
+  MetaOptimizer optimizer_with_inter_verifiers(nullptr, config_proto);
+  Status status =
+      optimizer_with_inter_verifiers.Optimize(nullptr, item, &output);
+  EXPECT_EQ(status.code(), errors::Code::NOT_FOUND);
+  EXPECT_TRUE(
+      absl::StrContains(status.error_message(), "Op type not registered"));
+}
+
 }  // namespace
 }  // namespace grappler
 }  // namespace tensorflow
diff --git a/tensorflow/core/grappler/verifiers/BUILD b/tensorflow/core/grappler/verifiers/BUILD
index 9666e101fe..e3e1538b00 100644
--- a/tensorflow/core/grappler/verifiers/BUILD
+++ b/tensorflow/core/grappler/verifiers/BUILD
@@ -24,7 +24,6 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":graph_verifier",
-        "//tensorflow/cc:ops",
         "//tensorflow/core:framework",
         "//tensorflow/core:graph",
         "//tensorflow/core:lib",
diff --git a/tensorflow/core/protobuf/rewriter_config.proto b/tensorflow/core/protobuf/rewriter_config.proto
index 515d673828..17e76c4d03 100644
--- a/tensorflow/core/protobuf/rewriter_config.proto
+++ b/tensorflow/core/protobuf/rewriter_config.proto
@@ -5,9 +5,10 @@ option cc_enable_arenas = true;
 option java_outer_classname = "RewriterConfigProtos";
 option java_multiple_files = true;
 option java_package = "org.tensorflow.framework";
-option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf";
+// add go_package externally with copybara
 
 import "tensorflow/core/framework/attr_value.proto";
+import "tensorflow/core/protobuf/verifier_config.proto";
 
 message AutoParallelOptions {
   bool enable = 1;
@@ -166,4 +167,11 @@ message RewriterConfig {
 
   // list of CustomGraphOptimizers to apply.
   repeated CustomGraphOptimizer custom_optimizers = 200;
+
+  // VerifierConfig specifying the verifiers to be run after every optimizer.
+  VerifierConfig inter_optimizer_verifier_config = 300;
+
+  // VerifierConfig specifying the verifiers to be run at the end, after all
+  // optimizers have run.
+  VerifierConfig post_optimization_verifier_config = 301;
 }
diff --git a/tensorflow/core/protobuf/verifier_config.proto b/tensorflow/core/protobuf/verifier_config.proto
new file mode 100644
index 0000000000..207f0f2a97
--- /dev/null
+++ b/tensorflow/core/protobuf/verifier_config.proto
@@ -0,0 +1,26 @@
+syntax = "proto3";
+
+package tensorflow;
+option cc_enable_arenas = true;
+option java_outer_classname = "VerifierConfigProtos";
+option java_multiple_files = true;
+option java_package = "org.tensorflow.framework";
+// add go_package externally with copybara
+
+// The config for graph verifiers.
+message VerifierConfig {
+  enum Toggle {
+    DEFAULT = 0;
+    ON = 1;
+    OFF = 2;
+  }
+
+  // Deadline for completion of all verification i.e. all the Toggle ON
+  // verifiers must complete execution within this time.
+  int64 verification_timeout_in_ms = 1;
+
+  // Perform structural validation on a tensorflow graph. Default is OFF.
+  Toggle structure_verifier = 2;
+
+  // Next tag: 3
+}
-- 
GitLab


From 676ff175114a0642003e9b55be0173473e1e5140 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 26 Jan 2019 01:02:25 -0800
Subject: [PATCH 1338/2345] compat: Update forward compatibility horizon to
 2019-01-26

PiperOrigin-RevId: 231029788
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 49e12345db..2c764900ea 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 25)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 26)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 441ea6de36ab7cf77364ce95e9fce8799eb4adf2 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Sat, 26 Jan 2019 09:23:41 -0800
Subject: [PATCH 1339/2345] [XLA] Speed up exhaustive_f32_elementwise_op_test.

This was spending the majority of its time in Literal::Get() and
Literal::Set().

Before: 2.5s to run a shard.
After:  1.0s to run a shard.
PiperOrigin-RevId: 231054690
---
 .../xla/tests/exhaustive_f32_elementwise_op_test.cc        | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc b/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc
index c84973e17b..139d5c70b8 100644
--- a/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc
+++ b/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc
@@ -40,14 +40,15 @@ class ExhaustiveF32ElementwiseOpTest
 
     Literal input_literal =
         LiteralUtil::CreateFromDimensions(F32, {input_size});
+    absl::Span<float> input_arr = input_literal.data<float>();
     for (int64 i = begin; i < end; i++) {
       if (i >= known_incorrect_range.first &&
           i < known_incorrect_range.second) {
         // If the operation is known to be buggy on a specific input clamp that
         // input to 0 under the assumption that the op is at least correct on 0.
-        input_literal.Set({i - begin}, 0.0f);
+        input_arr[i - begin] = 0;
       } else {
-        input_literal.Set({i - begin}, absl::bit_cast<float, int>(i));
+        input_arr[i - begin] = absl::bit_cast<float, int>(i);
       }
     }
 
@@ -60,7 +61,7 @@ class ExhaustiveF32ElementwiseOpTest
     std::vector<float> expected_result;
     expected_result.reserve(input_size);
     for (int64 i = 0; i < input_size; i++) {
-      expected_result.push_back(evaluate_op(input_literal.Get<float>({i})));
+      expected_result.push_back(evaluate_op(input_arr[i]));
     }
 
     ComputeAndCompareR1<float>(&builder, expected_result, {input_data.get()},
-- 
GitLab


From 52c8bdba0081e4ed428add13e3c0da7ccbfc8f06 Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Sat, 26 Jan 2019 10:14:55 -0800
Subject: [PATCH 1340/2345] Remove the Variant shape function registry and all
 references.

Variant shape information is now stored in the HandleData.shape_and_type
during shape inference.

While we're at it, propagate DT_VARIANT ShapeAndType handles through
a number of ops ("AddN", "ZerosLike", and several resource variable ops).

For DT_VARIANT types stored inside a ResourceVariable, the ResourceVariable's
ShapeAndType array is extended.  Before it had one entry storing the dtype and
shape of the tensor stored inside the Variable.  Now, entries indexed 1,... will
contain the ShapeAndType data read from the DT_VARIANT passed as the initializer
at Variable creation time.  ResourceVariable read ops will propagate this
information back out by reading ShapeAndType[1:] from the resource tensor's
handle data.

PiperOrigin-RevId: 231057278
---
 tensorflow/core/framework/common_shape_fns.cc |   4 +
 tensorflow/core/framework/shape_inference.cc  |   8 -
 tensorflow/core/framework/shape_inference.h   |   6 +-
 .../core/framework/variant_op_registry.cc     |  82 +++----
 .../core/framework/variant_op_registry.h      |  58 -----
 .../framework/variant_op_registry_test.cc     |  41 ----
 tensorflow/core/kernels/BUILD                 |  21 --
 tensorflow/core/kernels/aggregate_ops.cc      |  15 +-
 tensorflow/core/kernels/data/optional_ops.cc  |   7 -
 tensorflow/core/kernels/list_kernels.cc       |   7 -
 tensorflow/core/kernels/shape_op_test.cc      | 129 -----------
 tensorflow/core/kernels/shape_ops.cc          |   3 +-
 tensorflow/core/kernels/shape_ops.h           |  27 +--
 tensorflow/core/ops/array_ops.cc              |  48 ++--
 tensorflow/core/ops/list_ops.cc               |  29 +--
 tensorflow/core/ops/math_ops.cc               |  65 +++++-
 tensorflow/core/ops/resource_variable_ops.cc  |  72 ++++--
 tensorflow/python/eager/def_function.py       |   3 +-
 .../python/kernel_tests/list_ops_test.py      |  24 +-
 .../resource_variable_ops_test.py             |  57 +++++
 tensorflow/python/ops/array_ops.py            |  10 +-
 tensorflow/python/ops/custom_gradient.py      |   4 +-
 .../python/ops/resource_variable_ops.py       | 207 +++++++++++++++---
 23 files changed, 466 insertions(+), 461 deletions(-)
 delete mode 100644 tensorflow/core/kernels/shape_op_test.cc

diff --git a/tensorflow/core/framework/common_shape_fns.cc b/tensorflow/core/framework/common_shape_fns.cc
index 876ac188ac..83bc95065e 100644
--- a/tensorflow/core/framework/common_shape_fns.cc
+++ b/tensorflow/core/framework/common_shape_fns.cc
@@ -203,6 +203,10 @@ Status GetWindowedOutputSizeFromDims(
 
 Status UnchangedShape(shape_inference::InferenceContext* c) {
   c->set_output(0, c->input(0));
+  auto* handle_data = c->input_handle_shapes_and_types(0);
+  if (handle_data != nullptr) {
+    c->set_output_handle_shapes_and_types(0, *handle_data);
+  }
   return Status::OK();
 }
 
diff --git a/tensorflow/core/framework/shape_inference.cc b/tensorflow/core/framework/shape_inference.cc
index bffbee50b5..18a278f07f 100644
--- a/tensorflow/core/framework/shape_inference.cc
+++ b/tensorflow/core/framework/shape_inference.cc
@@ -1259,7 +1259,6 @@ bool InferenceContext::RelaxHandleShapesAndMergeTypes(
     return false;
   }
   std::vector<ShapeAndType> new_values(shapes_and_types.size());
-  bool refined = false;
   for (int i = 0; i < shapes_and_types.size(); ++i) {
     const ShapeAndType& existing = (*to_update)[i];
     if (shapes_and_types[i].dtype == existing.dtype) {
@@ -1269,16 +1268,9 @@ bool InferenceContext::RelaxHandleShapesAndMergeTypes(
         return false;
       } else {
         new_values[i].dtype = shapes_and_types[i].dtype;
-        refined = true;
       }
     }
     Relax(existing.shape, shapes_and_types[i].shape, &new_values[i].shape);
-    if (!existing.shape.SameHandle(new_values[i].shape)) {
-      refined = true;
-    }
-  }
-  if (!refined) {
-    return false;
   }
   to_update->swap(new_values);
   return true;
diff --git a/tensorflow/core/framework/shape_inference.h b/tensorflow/core/framework/shape_inference.h
index e3885b7d9e..bf8b633c01 100644
--- a/tensorflow/core/framework/shape_inference.h
+++ b/tensorflow/core/framework/shape_inference.h
@@ -588,9 +588,9 @@ class InferenceContext {
   // position idx with the specified shapes and types. This requires idx to be
   // in the [0, num_inputs) range.
   //
-  // If the relax is successful and any of the new shapes differs from the old
-  // one, or any of the old dtypes was DT_INVALID, store the new shapes and
-  // return true.  Return false otherwise.
+  // If the relax is successful (sizes are the same, old dtypes match new ones
+  // or are DT_INVALID), then store the relaxed shapes and return true.
+  // Return false otherwise.
   //
   // See 'RelaxInput' function for full details and examples.
   bool RelaxInputHandleShapesAndMergeTypes(
diff --git a/tensorflow/core/framework/variant_op_registry.cc b/tensorflow/core/framework/variant_op_registry.cc
index ef5b240aea..b5107a02a7 100644
--- a/tensorflow/core/framework/variant_op_registry.cc
+++ b/tensorflow/core/framework/variant_op_registry.cc
@@ -37,57 +37,6 @@ UnaryVariantOpRegistry* UnaryVariantOpRegistry::Global() {
   return global_unary_variant_op_registry;
 }
 
-UnaryVariantOpRegistry::VariantShapeFn* UnaryVariantOpRegistry::GetShapeFn(
-    const TypeIndex& type_index) {
-  auto found = shape_fns.find(type_index);
-  if (found == shape_fns.end()) return nullptr;
-  return &found->second;
-}
-
-void UnaryVariantOpRegistry::RegisterShapeFn(const TypeIndex& type_index,
-                                             const VariantShapeFn& shape_fn) {
-  VariantShapeFn* existing = GetShapeFn(type_index);
-  CHECK_EQ(existing, nullptr)
-      << "Unary VariantShapeFn for type_index: "
-      << port::MaybeAbiDemangle(type_index.name()) << " already registered";
-  shape_fns.insert(std::pair<TypeIndex, VariantShapeFn>(type_index, shape_fn));
-}
-
-Status GetUnaryVariantShape(const Tensor& variant_tensor, TensorShape* shape) {
-  CHECK_EQ(variant_tensor.dtype(), DT_VARIANT);
-  CHECK_EQ(variant_tensor.dims(), 0);
-  const Variant& v = variant_tensor.scalar<Variant>()();
-  UnaryVariantOpRegistry::VariantShapeFn* shape_fn =
-      UnaryVariantOpRegistry::Global()->GetShapeFn(v.TypeId());
-  if (shape_fn == nullptr) {
-    return errors::Internal(
-        "No unary variant shape function found for Variant type_index: ",
-        port::MaybeAbiDemangle(v.TypeId().name()));
-  }
-  return (*shape_fn)(v, shape);
-}
-
-// Add some basic registrations for use by others, e.g., for testing.
-namespace {
-template <typename T>
-Status ScalarShape(const T&, TensorShape* shape) {
-  *shape = TensorShape({});
-  return Status::OK();
-}
-}  // namespace
-
-#define REGISTER_VARIANT_SHAPE_TYPE(T) \
-  REGISTER_UNARY_VARIANT_SHAPE_FUNCTION(T, ScalarShape<T>);
-
-// No encode/shape registered for std::complex<> and Eigen::half
-// objects yet.
-REGISTER_VARIANT_SHAPE_TYPE(int);
-REGISTER_VARIANT_SHAPE_TYPE(float);
-REGISTER_VARIANT_SHAPE_TYPE(bool);
-REGISTER_VARIANT_SHAPE_TYPE(double);
-
-#undef REGISTER_VARIANT_SHAPE_TYPE
-
 UnaryVariantOpRegistry::VariantDecodeFn* UnaryVariantOpRegistry::GetDecodeFn(
     StringPiece type_name) {
   auto found = decode_fns.find(type_name);
@@ -177,6 +126,37 @@ Status VariantDeviceCopy(
   return (*device_copy_fn)(from, to, copy_fn);
 }
 
+namespace {
+template <typename T>
+Status DeviceCopyPrimitiveType(
+    const T& in, T* out,
+    const UnaryVariantOpRegistry::AsyncTensorDeviceCopyFn& copier) {
+  // Dummy copy, we don't actually bother copying to the device and back for
+  // testing.
+  *out = in;
+  return Status::OK();
+}
+}  // namespace
+
+#define REGISTER_VARIANT_DEVICE_COPY_TYPE(T)            \
+  INTERNAL_REGISTER_UNARY_VARIANT_DEVICE_COPY_FUNCTION( \
+      T, VariantDeviceCopyDirection::HOST_TO_DEVICE,    \
+      DeviceCopyPrimitiveType<T>);                      \
+  INTERNAL_REGISTER_UNARY_VARIANT_DEVICE_COPY_FUNCTION( \
+      T, VariantDeviceCopyDirection::DEVICE_TO_HOST,    \
+      DeviceCopyPrimitiveType<T>);                      \
+  INTERNAL_REGISTER_UNARY_VARIANT_DEVICE_COPY_FUNCTION( \
+      T, VariantDeviceCopyDirection::DEVICE_TO_DEVICE,  \
+      DeviceCopyPrimitiveType<T>);
+
+// No zeros_like registered for std::complex<> or Eigen::half objects yet.
+REGISTER_VARIANT_DEVICE_COPY_TYPE(int);
+REGISTER_VARIANT_DEVICE_COPY_TYPE(float);
+REGISTER_VARIANT_DEVICE_COPY_TYPE(double);
+REGISTER_VARIANT_DEVICE_COPY_TYPE(bool);
+
+#undef REGISTER_VARIANT_DEVICE_COPY_TYPE
+
 // Special casing UnaryOpFn per op and per device.
 UnaryVariantOpRegistry::VariantUnaryOpFn* UnaryVariantOpRegistry::GetUnaryOpFn(
     VariantUnaryOp op, StringPiece device, const TypeIndex& type_index) {
diff --git a/tensorflow/core/framework/variant_op_registry.h b/tensorflow/core/framework/variant_op_registry.h
index 7eb37e859f..488a606f6e 100644
--- a/tensorflow/core/framework/variant_op_registry.h
+++ b/tensorflow/core/framework/variant_op_registry.h
@@ -58,7 +58,6 @@ enum VariantDeviceCopyDirection {
 
 class UnaryVariantOpRegistry {
  public:
-  typedef std::function<Status(const Variant& v, TensorShape*)> VariantShapeFn;
   typedef std::function<bool(Variant*)> VariantDecodeFn;
   typedef std::function<Status(OpKernelContext*, const Variant&, Variant*)>
       VariantUnaryOpFn;
@@ -93,13 +92,6 @@ class UnaryVariantOpRegistry {
                                AsyncTensorDeviceCopyFn copy_fn)>
       AsyncVariantDeviceCopyFn;
 
-  // Add a shape lookup function to the registry.
-  void RegisterShapeFn(const TypeIndex& type_index,
-                       const VariantShapeFn& shape_fn);
-
-  // Returns nullptr if no shape function was found for the given TypeIndex.
-  VariantShapeFn* GetShapeFn(const TypeIndex& type_index);
-
   // Add a decode function to the registry.
   void RegisterDecodeFn(const string& type_name,
                         const VariantDecodeFn& decode_fn);
@@ -154,7 +146,6 @@ class UnaryVariantOpRegistry {
     std::size_t operator()(const TypeIndex& x) const { return x.hash_code(); }
   };
 
-  gtl::FlatMap<TypeIndex, VariantShapeFn, TypeIndexHash> shape_fns;
   gtl::FlatMap<StringPiece, VariantDecodeFn, StringPieceHasher> decode_fns;
 
   // Map std::pair<Direction, type_name> to function.
@@ -235,15 +226,6 @@ inline bool operator==(const UnaryVariantOpRegistry::FuncTuple<Op>& lhs,
   return (lhs.op_type_ == rhs.op_type_) && (lhs.device_ == rhs.device_) &&
          (lhs.type_index_ == rhs.type_index_);
 }
-// Gets a TensorShape from a Tensor containing a scalar Variant.
-// Returns an Internal error if the Variant does not have a registered shape
-// function, or if it's a serialized Variant that cannot be decoded.
-//
-// REQUIRES:
-//   variant_tensor.dtype() == DT_VARIANT
-//   variant_tensor.dims() == 0
-//
-Status GetUnaryVariantShape(const Tensor& variant_tensor, TensorShape* shape);
 
 // Decodes the Variant whose data_type has a registered decode
 // function.  Returns an Internal error if the Variant does not have a
@@ -326,29 +308,6 @@ Status BinaryOpVariants(OpKernelContext* ctx, VariantBinaryOp op,
 
 namespace variant_op_registry_fn_registration {
 
-template <typename T>
-class UnaryVariantShapeRegistration {
- public:
-  typedef std::function<Status(const T& t, TensorShape*)> LocalVariantShapeFn;
-
-  UnaryVariantShapeRegistration(const TypeIndex& type_index,
-                                const LocalVariantShapeFn& shape_fn) {
-    const string type_index_name = port::MaybeAbiDemangle(type_index.name());
-    UnaryVariantOpRegistry::Global()->RegisterShapeFn(
-        type_index,
-        [type_index_name, shape_fn](const Variant& v,
-                                    TensorShape* s) -> Status {
-          const T* t = v.get<T>();
-          if (t == nullptr) {
-            return errors::Internal(
-                "VariantShapeFn: Could not access object, type_index: ",
-                type_index_name);
-          }
-          return shape_fn(*t, s);
-        });
-  }
-};
-
 template <typename T>
 class UnaryVariantDecodeRegistration {
  public:
@@ -471,23 +430,6 @@ class UnaryVariantBinaryOpRegistration {
 
 };  // namespace variant_op_registry_fn_registration
 
-// Register a unary shape variant function with the signature:
-//    Status ShapeFn(const T& t, TensorShape* s);
-// to Variants having TypeIndex type_index.
-#define REGISTER_UNARY_VARIANT_SHAPE_FUNCTION(T, shape_function) \
-  REGISTER_UNARY_VARIANT_SHAPE_FUNCTION_UNIQ_HELPER(             \
-      __COUNTER__, T, MakeTypeIndex<T>(), shape_function)
-
-#define REGISTER_UNARY_VARIANT_SHAPE_FUNCTION_UNIQ_HELPER(ctr, T, type_index, \
-                                                          shape_function)     \
-  REGISTER_UNARY_VARIANT_SHAPE_FUNCTION_UNIQ(ctr, T, type_index, shape_function)
-
-#define REGISTER_UNARY_VARIANT_SHAPE_FUNCTION_UNIQ(ctr, T, type_index,         \
-                                                   shape_function)             \
-  static variant_op_registry_fn_registration::UnaryVariantShapeRegistration<T> \
-      register_unary_variant_op_shape_registration_fn_##ctr(type_index,        \
-                                                            shape_function)
-
 // Register a unary decode variant function for the given type.
 #define REGISTER_UNARY_VARIANT_DECODE_FUNCTION(T, type_name) \
   REGISTER_UNARY_VARIANT_DECODE_FUNCTION_UNIQ_HELPER(__COUNTER__, T, type_name)
diff --git a/tensorflow/core/framework/variant_op_registry_test.cc b/tensorflow/core/framework/variant_op_registry_test.cc
index b2443e8676..e1a46ebd59 100644
--- a/tensorflow/core/framework/variant_op_registry_test.cc
+++ b/tensorflow/core/framework/variant_op_registry_test.cc
@@ -39,13 +39,6 @@ namespace {
 
 struct VariantValue {
   string TypeName() const { return "TEST VariantValue"; }
-  static Status ShapeFn(const VariantValue& v, TensorShape* s) {
-    if (v.early_exit) {
-      return errors::InvalidArgument("early exit!");
-    }
-    *s = TensorShape({-0xdeadbeef});
-    return Status::OK();
-  }
   static Status CPUZerosLikeFn(OpKernelContext* ctx, const VariantValue& v,
                                VariantValue* v_out) {
     if (v.early_exit) {
@@ -89,8 +82,6 @@ struct VariantValue {
   int value;
 };
 
-REGISTER_UNARY_VARIANT_SHAPE_FUNCTION(VariantValue, VariantValue::ShapeFn);
-
 REGISTER_UNARY_VARIANT_DECODE_FUNCTION(VariantValue, "TEST VariantValue");
 
 INTERNAL_REGISTER_UNARY_VARIANT_DEVICE_COPY_FUNCTION(
@@ -113,38 +104,6 @@ REGISTER_UNARY_VARIANT_BINARY_OP_FUNCTION(ADD_VARIANT_BINARY_OP, DEVICE_GPU,
 
 }  // namespace
 
-TEST(VariantOpShapeRegistryTest, TestBasic) {
-  class Blah {};
-  EXPECT_EQ(UnaryVariantOpRegistry::Global()->GetShapeFn(MakeTypeIndex<Blah>()),
-            nullptr);
-
-  auto* shape_fn = UnaryVariantOpRegistry::Global()->GetShapeFn(
-      MakeTypeIndex<VariantValue>());
-  EXPECT_NE(shape_fn, nullptr);
-  TensorShape shape;
-
-  VariantValue vv_early_exit{true /* early_exit */};
-  Variant v = vv_early_exit;
-  Status s0 = (*shape_fn)(v, &shape);
-  EXPECT_FALSE(s0.ok());
-  EXPECT_TRUE(str_util::StrContains(s0.error_message(), "early exit!"));
-
-  VariantValue vv_ok{false /* early_exit */};
-  v = vv_ok;
-  TF_EXPECT_OK((*shape_fn)(v, &shape));
-  EXPECT_EQ(shape, TensorShape({-0xdeadbeef}));
-}
-
-TEST(VariantOpShapeRegistryTest, TestDuplicate) {
-  UnaryVariantOpRegistry registry;
-  UnaryVariantOpRegistry::VariantShapeFn f;
-  class FjFjFj {};
-  const auto kTypeIndex = MakeTypeIndex<FjFjFj>();
-  registry.RegisterShapeFn(kTypeIndex, f);
-  EXPECT_DEATH(registry.RegisterShapeFn(kTypeIndex, f),
-               "FjFjFj already registered");
-}
-
 TEST(VariantOpDecodeRegistryTest, TestBasic) {
   EXPECT_EQ(UnaryVariantOpRegistry::Global()->GetDecodeFn("YOU SHALL NOT PASS"),
             nullptr);
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index a59ea5b631..ffb093b2ef 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -3669,27 +3669,6 @@ tf_cc_test(
     ],
 )
 
-tf_cc_test(
-    name = "shape_op_test",
-    srcs = ["shape_op_test.cc"],
-    deps = [
-        ":array",
-        ":ops_testutil",
-        ":ops_util",
-        "//tensorflow/cc:cc_ops",
-        "//tensorflow/cc:client_session",
-        "//tensorflow/core:core_cpu",
-        "//tensorflow/core:direct_session",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:ops",
-        "//tensorflow/core:protos_all_cc",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-        "//tensorflow/core:testlib",
-    ],
-)
-
 tf_cuda_cc_test(
     name = "sparse_matmul_op_test",
     size = "small",
diff --git a/tensorflow/core/kernels/aggregate_ops.cc b/tensorflow/core/kernels/aggregate_ops.cc
index 150e8fe637..edf6d3e61e 100644
--- a/tensorflow/core/kernels/aggregate_ops.cc
+++ b/tensorflow/core/kernels/aggregate_ops.cc
@@ -179,20 +179,7 @@ class AddNOp<Device, Variant> : public OpKernel {
               i, " has shape: ", ctx->input(i).shape().DebugString(), "."));
     }
 
-    TensorShape common_shape;
-    OP_REQUIRES_OK(ctx, GetUnaryVariantShape(ctx->input(0), &common_shape));
-    // Step 2: access all variants and ensure shapes match.
-    for (int i = 1; i < num; ++i) {
-      TensorShape check_shape;
-      OP_REQUIRES_OK(ctx, GetUnaryVariantShape(ctx->input(i), &check_shape));
-      OP_REQUIRES(ctx, common_shape == check_shape,
-                  errors::InvalidArgument(
-                      "AddN of Variants of differing shapes; inputs[0] shape: ",
-                      common_shape.DebugString(), ", inputs[", i,
-                      "] shape: ", check_shape.DebugString()));
-    }
-
-    // Step 3: attempt to add using
+    // Step 2: attempt to add using
     //   BinaryOpVariants(ADD_VARIANT_BINARY_OP, ...)
     //   For the output create a default-constructed variant object.
     // TODO(ebrevdo): Perform summation in a tree-structure.
diff --git a/tensorflow/core/kernels/data/optional_ops.cc b/tensorflow/core/kernels/data/optional_ops.cc
index a406f7467f..6590c7ef9b 100644
--- a/tensorflow/core/kernels/data/optional_ops.cc
+++ b/tensorflow/core/kernels/data/optional_ops.cc
@@ -221,12 +221,5 @@ REGISTER_UNARY_VARIANT_BINARY_OP_FUNCTION(ADD_VARIANT_BINARY_OP, DEVICE_CPU,
                                           OptionalVariant,
                                           OptionalBinaryAdd<CPUDevice>);
 
-Status OptionalShape(const OptionalVariant& x, TensorShape* s) {
-  *s = TensorShape({});
-  return Status::OK();
-}
-
-REGISTER_UNARY_VARIANT_SHAPE_FUNCTION(OptionalVariant, OptionalShape);
-
 }  // namespace data
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/list_kernels.cc b/tensorflow/core/kernels/list_kernels.cc
index ee157dfd18..55c2facc3c 100644
--- a/tensorflow/core/kernels/list_kernels.cc
+++ b/tensorflow/core/kernels/list_kernels.cc
@@ -99,13 +99,6 @@ REGISTER_LIST_COPY(VariantDeviceCopyDirection::DEVICE_TO_DEVICE);
 
 REGISTER_UNARY_VARIANT_DECODE_FUNCTION(TensorList, TensorList::kTypeName);
 
-Status TensorListShape(const TensorList& t, TensorShape* s) {
-  *s = TensorShape({});
-  return Status::OK();
-}
-
-REGISTER_UNARY_VARIANT_SHAPE_FUNCTION(TensorList, TensorListShape);
-
 bool TensorList::Decode(const VariantTensorData& data) {
   // TODO(srbs): Change the signature to Decode(VariantTensorData data) so
   // that we do not have to copy each tensor individually below. This would
diff --git a/tensorflow/core/kernels/shape_op_test.cc b/tensorflow/core/kernels/shape_op_test.cc
deleted file mode 100644
index 30cb1e0a7f..0000000000
--- a/tensorflow/core/kernels/shape_op_test.cc
+++ /dev/null
@@ -1,129 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include <functional>
-#include <memory>
-
-#include "tensorflow/cc/client/client_session.h"
-#include "tensorflow/cc/ops/array_ops.h"
-#include "tensorflow/cc/ops/const_op.h"
-#include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/framework/variant.h"
-#include "tensorflow/core/framework/variant_encode_decode.h"
-#include "tensorflow/core/framework/variant_op_registry.h"
-#include "tensorflow/core/kernels/ops_testutil.h"
-#include "tensorflow/core/kernels/ops_util.h"
-#include "tensorflow/core/lib/strings/str_util.h"
-#include "tensorflow/core/lib/strings/strcat.h"
-#include "tensorflow/core/platform/abi.h"
-#include "tensorflow/core/platform/test.h"
-#include "tensorflow/core/platform/types.h"
-
-namespace tensorflow {
-namespace {
-
-class ShapeOpTest : public OpsTestBase {};
-
-struct NoKnownShape {
-  string TypeName() const { return "NO KNOWN SHAPE"; }
-};
-
-REGISTER_UNARY_VARIANT_DECODE_FUNCTION(NoKnownShape, "NO KNOWN SHAPE");
-
-struct KnownVecSize {
-  KnownVecSize() : shape_value(0) {}
-  explicit KnownVecSize(int value) : shape_value(value) {}
-  string TypeName() const { return "KNOWN VECTOR SIZE TYPE"; }
-  bool Decode(const VariantTensorData& d) {
-    return d.get_metadata(&shape_value);
-  }
-  void Encode(VariantTensorData* d) const { d->set_metadata(shape_value); }
-  int shape_value;
-};
-
-Status GetShapeFromKnownVecSize(const KnownVecSize& ks, TensorShape* s) {
-  *s = TensorShape({ks.shape_value});
-  return Status::OK();
-}
-
-REGISTER_UNARY_VARIANT_DECODE_FUNCTION(KnownVecSize, "KNOWN VECTOR SIZE TYPE");
-
-REGISTER_UNARY_VARIANT_SHAPE_FUNCTION(KnownVecSize, GetShapeFromKnownVecSize);
-
-static void ExpectHasError(const Status& s, StringPiece substr) {
-  EXPECT_TRUE(str_util::StrContains(s.ToString(), substr))
-      << ">>" << s << "<<, expected substring >>" << substr << "<<";
-}
-
-TEST_F(ShapeOpTest, Simple) {
-  // Ensure the ops run on CPU, as we have no device copy registration
-  // for NoKnownShape and KnownVecSize objects.
-  Scope root = Scope::NewRootScope().WithDevice("/cpu:0");
-
-  // Use a placeholder so the graph optimizer doesn't optimize away
-  // the shape function.
-  auto input = ops::Placeholder(root, DT_VARIANT);
-  auto shape_output = ops::Shape(root, input);
-  auto rank_output = ops::Rank(root, input);
-  auto size_output = ops::Size(root, input);
-
-  TF_ASSERT_OK(root.status());
-
-  ClientSession session(root);
-
-  std::vector<Tensor> outputs;
-
-  {
-    // Test no shape registered.
-    Tensor variant_tensor(DT_VARIANT, TensorShape({}));
-    Variant& v = variant_tensor.scalar<Variant>()();
-    v = NoKnownShape();
-    Status s = session.Run({{input, variant_tensor}}, {shape_output}, &outputs);
-    EXPECT_FALSE(s.ok());
-    ExpectHasError(
-        s, strings::StrCat(
-               "No unary variant shape function found for Variant type_index: ",
-               port::MaybeAbiDemangle(MakeTypeIndex<NoKnownShape>().name())));
-  }
-
-  {
-    // Test non-scalar variant.
-    Tensor variant_tensor(DT_VARIANT, TensorShape({1}));
-    Status s = session.Run({{input, variant_tensor}}, {shape_output}, &outputs);
-    EXPECT_FALSE(s.ok());
-    ExpectHasError(s, "Shape of non-unary Variant not supported.");
-  }
-
-  {
-    // Test registered variant.
-    Tensor variant_tensor(DT_VARIANT, TensorShape({}));
-    const int vec_dim_value = -0xdeadbeef;  // must be non-negative.
-    Variant& v = variant_tensor.scalar<Variant>()();
-    v = KnownVecSize(vec_dim_value);
-    TF_EXPECT_OK(session.Run({{input, variant_tensor}},
-                             {shape_output, rank_output, size_output},
-                             &outputs));
-    EXPECT_EQ(outputs[0].dims(), 1);  // shape
-    EXPECT_EQ(vec_dim_value, outputs[0].vec<int32>()(0));
-    EXPECT_EQ(outputs[1].dims(), 0);  // rank
-    EXPECT_EQ(1, outputs[1].scalar<int32>()());
-    EXPECT_EQ(outputs[2].dims(), 0);  // size
-    EXPECT_EQ(vec_dim_value, outputs[0].scalar<int32>()());
-  }
-}
-
-}  // namespace
-}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/shape_ops.cc b/tensorflow/core/kernels/shape_ops.cc
index ab1ce0f9c8..db7357ca70 100644
--- a/tensorflow/core/kernels/shape_ops.cc
+++ b/tensorflow/core/kernels/shape_ops.cc
@@ -469,8 +469,7 @@ class EnsureShapeOp : public OpKernel {
 
   void Compute(OpKernelContext* ctx) override {
     TensorShape shape;
-    OP_REQUIRES_OK(ctx,
-                   shape_op_helpers::GetRegularOrVariantShape(ctx, 0, &shape));
+    OP_REQUIRES_OK(ctx, shape_op_helpers::GetShape(ctx, 0, &shape));
 
     if (!expected_shape_.IsCompatibleWith(shape)) {
       ctx->SetStatus(errors::InvalidArgument(
diff --git a/tensorflow/core/kernels/shape_ops.h b/tensorflow/core/kernels/shape_ops.h
index dbd82d2fe7..03b32b88d9 100644
--- a/tensorflow/core/kernels/shape_ops.h
+++ b/tensorflow/core/kernels/shape_ops.h
@@ -29,18 +29,9 @@ limitations under the License.
 namespace tensorflow {
 
 namespace shape_op_helpers {
-inline Status GetRegularOrVariantShape(OpKernelContext* ctx, int input_index,
-                                       TensorShape* shape) {
-  const Tensor& inp = ctx->input(input_index);
-  if (ctx->input_dtype(0) == DT_VARIANT) {
-    if (inp.dims() != 0) {
-      return errors::InvalidArgument(
-          "Shape of non-unary Variant not supported.");
-    }
-    TF_RETURN_IF_ERROR(GetUnaryVariantShape(inp, shape));
-  } else {
-    *shape = inp.shape();
-  }
+inline Status GetShape(OpKernelContext* ctx, int input_index,
+                       TensorShape* shape) {
+  *shape = ctx->input(input_index).shape();
   return Status::OK();
 }
 }  // namespace shape_op_helpers
@@ -52,8 +43,7 @@ class ShapeOp : public OpKernel {
 
   void Compute(OpKernelContext* ctx) override {
     TensorShape shape;
-    OP_REQUIRES_OK(ctx,
-                   shape_op_helpers::GetRegularOrVariantShape(ctx, 0, &shape));
+    OP_REQUIRES_OK(ctx, shape_op_helpers::GetShape(ctx, 0, &shape));
     const int rank = shape.dims();
     Tensor* out = nullptr;
     OP_REQUIRES_OK(ctx, ctx->allocate_output(0, TensorShape({rank}), &out));
@@ -81,8 +71,7 @@ class ShapeNOp : public OpKernel {
   void Compute(OpKernelContext* ctx) override {
     for (int i = 0; i < ctx->num_inputs(); ++i) {
       TensorShape shape;
-      OP_REQUIRES_OK(
-          ctx, shape_op_helpers::GetRegularOrVariantShape(ctx, i, &shape));
+      OP_REQUIRES_OK(ctx, shape_op_helpers::GetShape(ctx, i, &shape));
       const int dims = shape.dims();
       Tensor* out = nullptr;
       OP_REQUIRES_OK(ctx, ctx->allocate_output(i, {dims}, &out));
@@ -110,8 +99,7 @@ class RankOp : public OpKernel {
 
   void Compute(OpKernelContext* ctx) override {
     TensorShape shape;
-    OP_REQUIRES_OK(ctx,
-                   shape_op_helpers::GetRegularOrVariantShape(ctx, 0, &shape));
+    OP_REQUIRES_OK(ctx, shape_op_helpers::GetShape(ctx, 0, &shape));
     const int rank = shape.dims();
     Tensor* out = nullptr;
     OP_REQUIRES_OK(ctx, ctx->allocate_output(0, TensorShape({}), &out));
@@ -128,8 +116,7 @@ class SizeOp : public OpKernel {
 
   void Compute(OpKernelContext* ctx) override {
     TensorShape shape;
-    OP_REQUIRES_OK(ctx,
-                   shape_op_helpers::GetRegularOrVariantShape(ctx, 0, &shape));
+    OP_REQUIRES_OK(ctx, shape_op_helpers::GetShape(ctx, 0, &shape));
     const int64 size = shape.num_elements();
     Tensor* out = nullptr;
     OP_REQUIRES_OK(ctx, ctx->allocate_output(0, TensorShape({}), &out));
diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc
index 281e2996ed..108f187c98 100644
--- a/tensorflow/core/ops/array_ops.cc
+++ b/tensorflow/core/ops/array_ops.cc
@@ -347,6 +347,16 @@ REGISTER_OP("Pack")
       while (index < rank) dims.push_back(c->Dim(cur, index++));
 
       c->set_output(0, c->MakeShape(dims));
+      for (int i = 0; i < c->num_inputs(); ++i) {
+        auto* shape_and_type = c->input_handle_shapes_and_types(i);
+        if (shape_and_type) {
+          if (!c->RelaxOutputHandleShapesAndMergeTypes(0, *shape_and_type)) {
+            c->set_output_handle_shapes_and_types(
+                0, std::vector<shape_inference::ShapeAndType>({}));
+            break;
+          }
+        }
+      }
       return Status::OK();
     });
 
@@ -1034,6 +1044,12 @@ REGISTER_OP("Fill")
       ShapeHandle out;
       TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(0, &out));
       c->set_output(0, out);
+
+      auto* shape_and_type = c->input_handle_shapes_and_types(1);
+      if (shape_and_type) {
+        c->set_output_handle_shapes_and_types(0, *shape_and_type);
+      }
+
       return Status::OK();
     });
 
@@ -1206,27 +1222,13 @@ REGISTER_OP("Identity")
     .Input("input: T")
     .Output("output: T")
     .Attr("T: type")
-    .SetShapeFn([](shape_inference::InferenceContext* c) {
-      c->set_output(0, c->input(0));
-      auto* handle_data = c->input_handle_shapes_and_types(0);
-      if (handle_data != nullptr) {
-        c->set_output_handle_shapes_and_types(0, *handle_data);
-      }
-      return Status::OK();
-    });
+    .SetShapeFn(shape_inference::UnchangedShape);
 
 REGISTER_OP("Snapshot")
     .Input("input: T")
     .Output("output: T")
     .Attr("T: type")
-    .SetShapeFn([](shape_inference::InferenceContext* c) {
-      c->set_output(0, c->input(0));
-      auto* handle_data = c->input_handle_shapes_and_types(0);
-      if (handle_data != nullptr) {
-        c->set_output_handle_shapes_and_types(0, *handle_data);
-      }
-      return Status::OK();
-    });
+    .SetShapeFn(shape_inference::UnchangedShape);
 
 #ifdef INTEL_MKL
 REGISTER_OP("_MklIdentity")
@@ -1235,14 +1237,7 @@ REGISTER_OP("_MklIdentity")
     .Output("output: T")
     .Output("mkl_output: uint8")
     .Attr("T: type")
-    .SetShapeFn([](shape_inference::InferenceContext* c) {
-      c->set_output(0, c->input(0));
-      auto* handle_data = c->input_handle_shapes_and_types(0);
-      if (handle_data != nullptr) {
-        c->set_output_handle_shapes_and_types(0, *handle_data);
-      }
-      return Status::OK();
-    })
+    .SetShapeFn(shape_inference::UnchangedShape)
     .Doc(R"Doc( Mkl implementation of IdentityOp
 )Doc");
 #endif
@@ -1626,6 +1621,11 @@ REGISTER_OP("StridedSlice")
       TF_RETURN_IF_ERROR(c->MakeShapeFromPartialTensorShape(final_shape, &out));
       c->set_output(0, out);
 
+      auto* shape_and_type = c->input_handle_shapes_and_types(0);
+      if (shape_and_type) {
+        c->set_output_handle_shapes_and_types(0, *shape_and_type);
+      }
+
       return Status::OK();
     });
 
diff --git a/tensorflow/core/ops/list_ops.cc b/tensorflow/core/ops/list_ops.cc
index a16fabe2ab..48b6e016c9 100644
--- a/tensorflow/core/ops/list_ops.cc
+++ b/tensorflow/core/ops/list_ops.cc
@@ -51,11 +51,11 @@ REGISTER_OP("TensorListPushBack")
       shape_inference::ShapeHandle element_shape = c->UnknownShape();
 
       auto* handle_data = c->input_handle_shapes_and_types(0);
-      if (handle_data != nullptr && handle_data->size() != 1) {
+      if (handle_data != nullptr && handle_data->size() > 1) {
         return errors::InvalidArgument(
             "Trying to push to list with wrong variant data.");
       }
-      if (handle_data != nullptr) {
+      if (handle_data != nullptr && handle_data->size() == 1) {
         const shape_inference::ShapeAndType& list_shape_type =
             (*handle_data)[0];
         if (list_shape_type.dtype != element_dtype) {
@@ -98,11 +98,11 @@ REGISTER_OP("TensorListPushBackBatch")
       shape_inference::ShapeHandle element_shape = c->UnknownShape();
 
       auto* handle_data = c->input_handle_shapes_and_types(0);
-      if (handle_data != nullptr && handle_data->size() != 1) {
+      if (handle_data != nullptr && handle_data->size() > 1) {
         return errors::InvalidArgument(
             "Trying to push to list with wrong variant data.");
       }
-      if (handle_data != nullptr) {
+      if (handle_data != nullptr && handle_data->size() == 1) {
         const shape_inference::ShapeAndType& list_shape_type =
             (*handle_data)[0];
         if (list_shape_type.dtype != element_dtype) {
@@ -139,11 +139,11 @@ REGISTER_OP("TensorListPopBack")
       TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype));
       shape_inference::ShapeHandle tensor_shape = c->UnknownShape();
       auto* handle_data = c->input_handle_shapes_and_types(0);
-      if (handle_data != nullptr && handle_data->size() != 1) {
+      if (handle_data != nullptr && handle_data->size() > 1) {
         return errors::InvalidArgument(
             "Trying to read from list with invalid variant data.");
       }
-      if (handle_data != nullptr) {
+      if (handle_data != nullptr && handle_data->size() == 1) {
         const shape_inference::ShapeAndType& list_shape_type =
             (*handle_data)[0];
         if (list_shape_type.dtype != element_dtype) {
@@ -176,11 +176,11 @@ REGISTER_OP("TensorListStack")
       TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype));
       shape_inference::ShapeHandle element_shape = c->UnknownShape();
       auto* handle_data = c->input_handle_shapes_and_types(0);
-      if (handle_data != nullptr && handle_data->size() != 1) {
+      if (handle_data != nullptr && handle_data->size() > 1) {
         return errors::InvalidArgument(
             "Trying to read from list with wrong variant data.");
       }
-      if (handle_data != nullptr) {
+      if (handle_data != nullptr && handle_data->size() == 1) {
         const shape_inference::ShapeAndType& list_shape_type =
             (*handle_data)[0];
         if (list_shape_type.dtype != element_dtype) {
@@ -225,11 +225,11 @@ REGISTER_OP("TensorListConcat")
                                                             &element_shape));
 
       auto* handle_data = c->input_handle_shapes_and_types(0);
-      if (handle_data != nullptr && handle_data->size() != 1) {
+      if (handle_data != nullptr && handle_data->size() > 1) {
         return errors::InvalidArgument(
             "Trying to read from list with wrong variant data.");
       }
-      if (handle_data != nullptr) {
+      if (handle_data != nullptr && handle_data->size() == 1) {
         const shape_inference::ShapeAndType& list_shape_type =
             (*handle_data)[0];
         if (list_shape_type.dtype != element_dtype) {
@@ -481,15 +481,18 @@ REGISTER_OP("TensorListConcatLists")
 
       auto* handle_data_a = c->input_handle_shapes_and_types(0);
       auto* handle_data_b = c->input_handle_shapes_and_types(1);
-      if (handle_data_a == nullptr && handle_data_b == nullptr) {
+      if ((handle_data_a == nullptr || handle_data_a->empty()) &&
+          (handle_data_b == nullptr || handle_data_b->empty())) {
         c->set_output_handle_shapes_and_types(
             0, {{c->UnknownShape(), element_dtype}});
         return Status::OK();
       }
       shape_inference::ShapeAndType list_shape_type_a =
-          (handle_data_a) ? handle_data_a->at(0) : handle_data_b->at(0);
+          (handle_data_a && !handle_data_a->empty()) ? handle_data_a->at(0)
+                                                     : handle_data_b->at(0);
       const shape_inference::ShapeAndType& list_shape_type_b =
-          (handle_data_b) ? handle_data_b->at(0) : handle_data_a->at(0);
+          (handle_data_b && !handle_data_b->empty()) ? handle_data_b->at(0)
+                                                     : handle_data_a->at(0);
       if (list_shape_type_a.dtype != element_dtype) {
         return errors::InvalidArgument("input_a.type != element_dtype: ",
                                        DataTypeString(list_shape_type_a.dtype),
diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc
index eb9cbd3225..747536a019 100644
--- a/tensorflow/core/ops/math_ops.cc
+++ b/tensorflow/core/ops/math_ops.cc
@@ -39,7 +39,61 @@ REGISTER_OP("AddN")
                                         " with other shapes.");
       }
       c->set_output(0, cur);
-      return Status::OK();
+
+      DataType dtype;
+      TF_RETURN_IF_ERROR(c->GetAttr("T", &dtype));
+
+      if (dtype != DT_VARIANT) {
+        // Exit early if not DT_VARIANT.
+        return Status::OK();
+      } else {
+        // DT_VARIANT shape handle shape inference.  All sizes and dtypes must
+        // be the same; all shapes must be compatible via Merge.
+        std::vector<shape_inference::ShapeAndType> cur_shapes_and_types;
+        auto* shapes_and_types =
+            c->input_handle_shapes_and_types(c->num_inputs() - 1);
+        if (shapes_and_types) {
+          cur_shapes_and_types = *shapes_and_types;
+        }
+
+        for (int i = c->num_inputs() - 2; i >= 0; --i) {
+          auto shapes_and_types_i = c->input_handle_shapes_and_types(i);
+          if (!shapes_and_types && shapes_and_types_i) {
+            // TODO(ebrevdo): Find cases where this happens and fix their shape
+            // inference.  If we are calling AddN on variant types, they should
+            // all have consistent shape_and_type info.
+            shapes_and_types = shapes_and_types_i;
+          } else if (shapes_and_types && shapes_and_types_i) {
+            if (shapes_and_types_i->size() != shapes_and_types->size()) {
+              return errors::InvalidArgument(
+                  "shapes_and_types[", i,
+                  "].size() == ", shapes_and_types_i->size(),
+                  " != shapes_and_types[0].size() == ",
+                  shapes_and_types->size());
+            }
+            for (int j = 0; j < shapes_and_types->size(); ++j) {
+              if (shapes_and_types->at(j).dtype !=
+                  shapes_and_types_i->at(j).dtype) {
+                return errors::InvalidArgument(
+                    "shapes_and_types[", i, "][", j, "].dtype() == ",
+                    DataTypeString(shapes_and_types_i->at(j).dtype),
+                    " != shapes_and_types[0][", j, "].dtype == ",
+                    DataTypeString(shapes_and_types->at(j).dtype));
+              }
+              TF_RETURN_WITH_CONTEXT_IF_ERROR(
+                  c->Merge(shapes_and_types_i->at(j).shape,
+                           cur_shapes_and_types.at(j).shape,
+                           &cur_shapes_and_types.at(j).shape),
+                  "From merging shapes_and_types[", i, "][", j, "].shape with ",
+                  "shapes_and_types[0][", j, "].shape");
+            }
+          }
+        }
+        if (shapes_and_types) {
+          c->set_output_handle_shapes_and_types(0, cur_shapes_and_types);
+        }
+        return Status::OK();
+      }
     });
 
 // --------------------------------------------------------------------------
@@ -1368,7 +1422,14 @@ REGISTER_OP("Conj")
     .Input("input: T")
     .Output("output: T")
     .Attr("T: {complex64, complex128, variant} = DT_COMPLEX64")
-    .SetShapeFn(shape_inference::UnchangedShape);
+    .SetShapeFn([](InferenceContext* c) {
+      c->set_output(0, c->input(0));
+      auto* handle_data = c->input_handle_shapes_and_types(0);
+      if (handle_data != nullptr) {
+        c->set_output_handle_shapes_and_types(0, *handle_data);
+      }
+      return Status::OK();
+    });
 
 // --------------------------------------------------------------------------
 
diff --git a/tensorflow/core/ops/resource_variable_ops.cc b/tensorflow/core/ops/resource_variable_ops.cc
index 65bdde375b..f54ed52ea2 100644
--- a/tensorflow/core/ops/resource_variable_ops.cc
+++ b/tensorflow/core/ops/resource_variable_ops.cc
@@ -29,21 +29,20 @@ namespace tensorflow {
 
 namespace {
 
-Status ValidateVariableResourceHandle(InferenceContext* c,
-                                      ShapeAndType* shape_and_type) {
+Status ValidateVariableResourceHandle(
+    InferenceContext* c, std::vector<ShapeAndType>* shape_and_type) {
   auto* handle_data = c->input_handle_shapes_and_types(0);
   if (handle_data == nullptr || handle_data->empty()) {
-    shape_and_type->shape = c->UnknownShape();
-    shape_and_type->dtype = DT_INVALID;
+    shape_and_type->emplace_back(c->UnknownShape(), DT_INVALID);
   } else {
-    *shape_and_type = (*handle_data)[0];
+    *shape_and_type = *handle_data;
     DataType value_dtype;
     TF_RETURN_IF_ERROR(c->GetAttr("dtype", &value_dtype));
-    if (shape_and_type->dtype != value_dtype) {
+    if (shape_and_type->at(0).dtype != value_dtype) {
       return errors::InvalidArgument(
           "Trying to read variable with wrong dtype. "
           "Expected ",
-          DataTypeString(shape_and_type->dtype), " got ",
+          DataTypeString(shape_and_type->at(0).dtype), " got ",
           DataTypeString(value_dtype));
     }
   }
@@ -51,9 +50,15 @@ Status ValidateVariableResourceHandle(InferenceContext* c,
 }
 
 Status ReadVariableShapeFn(InferenceContext* c) {
-  ShapeAndType shape_and_type;
+  std::vector<ShapeAndType> shape_and_type;
   TF_RETURN_IF_ERROR(ValidateVariableResourceHandle(c, &shape_and_type));
-  c->set_output(0, shape_and_type.shape);
+  c->set_output(0, shape_and_type[0].shape);
+  if (shape_and_type[0].dtype == DT_VARIANT && shape_and_type.size() > 1) {
+    std::vector<ShapeAndType> variant_shape_and_type;
+    std::copy(shape_and_type.begin() + 1, shape_and_type.end(),
+              std::back_inserter(variant_shape_and_type));
+    c->set_output_handle_shapes_and_types(0, variant_shape_and_type);
+  }
   return Status::OK();
 }
 
@@ -180,13 +185,27 @@ REGISTER_OP("DestroyResourceOp")
     .SetShapeFn(shape_inference::NoOutputs);
 
 Status CreateAssignShapeFn(InferenceContext* c) {
-  ShapeAndType handle_shape_and_type;
+  std::vector<ShapeAndType> handle_shape_and_type;
   TF_RETURN_IF_ERROR(ValidateVariableResourceHandle(c, &handle_shape_and_type));
 
   ShapeHandle value_shape = c->input(1);
   ShapeHandle unused;
   TF_RETURN_IF_ERROR(
-      c->Merge(handle_shape_and_type.shape, value_shape, &unused));
+      c->Merge(handle_shape_and_type[0].shape, value_shape, &unused));
+
+  if (handle_shape_and_type[0].dtype == DT_VARIANT &&
+      handle_shape_and_type.size() > 1 &&
+      c->input_handle_shapes_and_types(1) != nullptr) {
+    auto* value_handle_shape_and_type = c->input_handle_shapes_and_types(1);
+    if (value_handle_shape_and_type->size() !=
+        handle_shape_and_type.size() - 1) {
+      return errors::InvalidArgument(
+          "Incompatible handle variant shape_and_type size and input "
+          "shape_and_type size: ",
+          handle_shape_and_type.size() - 1, " vs. ",
+          value_handle_shape_and_type->size());
+    }
+  }
   return Status::OK();
 }
 
@@ -240,29 +259,37 @@ REGISTER_OP("ResourceGather")
     .Attr("dtype: type")
     .Attr("Tindices: {int32,int64}")
     .SetShapeFn([](InferenceContext* c) {
-      ShapeAndType handle_shape_and_type;
+      std::vector<ShapeAndType> handle_shape_and_type;
       TF_RETURN_IF_ERROR(
           ValidateVariableResourceHandle(c, &handle_shape_and_type));
 
       ShapeHandle unused;
       TF_RETURN_IF_ERROR(
-          c->WithRankAtLeast(handle_shape_and_type.shape, 1, &unused));
+          c->WithRankAtLeast(handle_shape_and_type[0].shape, 1, &unused));
       ShapeHandle params_subshape;
       TF_RETURN_IF_ERROR(
-          c->Subshape(handle_shape_and_type.shape, 1, &params_subshape));
+          c->Subshape(handle_shape_and_type[0].shape, 1, &params_subshape));
       ShapeHandle indices_shape = c->input(1);
       ShapeHandle out;
       TF_RETURN_IF_ERROR(c->Concatenate(indices_shape, params_subshape, &out));
       c->set_output(0, out);
+      if (handle_shape_and_type[0].dtype == DT_VARIANT &&
+          !handle_shape_and_type.empty()) {
+        std::vector<ShapeAndType> variant_shape_and_type;
+        std::copy(handle_shape_and_type.begin() + 1,
+                  handle_shape_and_type.end(),
+                  std::back_inserter(variant_shape_and_type));
+        c->set_output_handle_shapes_and_types(0, variant_shape_and_type);
+      }
       return Status::OK();
     });
 
 namespace {
 
 Status ResourceScatterUpdateShape(InferenceContext* c) {
-  ShapeAndType handle_shape_and_type;
+  std::vector<ShapeAndType> handle_shape_and_type;
   TF_RETURN_IF_ERROR(ValidateVariableResourceHandle(c, &handle_shape_and_type));
-  ShapeHandle var_shape = handle_shape_and_type.shape;
+  ShapeHandle var_shape = handle_shape_and_type[0].shape;
   ShapeHandle indices_shape = c->input(1);
 
   ShapeHandle unused_updates_shape;
@@ -274,6 +301,19 @@ Status ResourceScatterUpdateShape(InferenceContext* c) {
       InferenceContext::Rank(c->input(2)) == 0
           ? Status::OK()
           : c->Merge(c->input(2), concat, &unused_updates_shape));
+  if (handle_shape_and_type[0].dtype == DT_VARIANT &&
+      handle_shape_and_type.size() > 1 &&
+      c->input_handle_shapes_and_types(2) != nullptr) {
+    auto* value_handle_shape_and_type = c->input_handle_shapes_and_types(2);
+    if (value_handle_shape_and_type->size() !=
+        handle_shape_and_type.size() - 1) {
+      return errors::InvalidArgument(
+          "Incompatible handle variant shape_and_type size and input "
+          "shape_and_type size: ",
+          handle_shape_and_type.size() - 1, " vs. ",
+          value_handle_shape_and_type->size());
+    }
+  }
   return Status::OK();
 }
 
diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py
index 2e8b82c68d..1175052530 100644
--- a/tensorflow/python/eager/def_function.py
+++ b/tensorflow/python/eager/def_function.py
@@ -151,8 +151,7 @@ class UnliftedInitializerVariable(resource_variable_ops.ResourceVariable):
             name="initial_value", dtype=dtype)
       with ops.init_scope():
         self._handle = resource_variable_ops.eager_safe_variable_handle(
-            shape=initial_value.get_shape(),
-            dtype=initial_value.dtype.base_dtype,
+            initial_value=initial_value,
             shared_name=shared_name,
             name=name,
             graph_mode=self._in_graph_mode)
diff --git a/tensorflow/python/kernel_tests/list_ops_test.py b/tensorflow/python/kernel_tests/list_ops_test.py
index 01618c52a3..88ca6e85df 100644
--- a/tensorflow/python/kernel_tests/list_ops_test.py
+++ b/tensorflow/python/kernel_tests/list_ops_test.py
@@ -949,16 +949,25 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
               list_ops.empty_tensor_list([], dtypes.float32),
               element_dtype=dtypes.float32))
 
-    with self.assertRaisesRegexp(errors.InvalidArgumentError,
-                                 "element shapes are not identical at index 0"):
+    if context.executing_eagerly():
+      expected_error = (
+          errors.InvalidArgumentError,
+          "element shapes are not identical at index 0")
+    else:
+      expected_error = (ValueError, "Shapes must be equal rank")
+    with self.assertRaisesRegexp(*expected_error):
       l_batch_of_vec_tls = array_ops.stack(
           [list_ops.tensor_list_from_tensor([[1.0]], element_shape=[1])] * 2)
       self.evaluate(
           list_ops.tensor_list_concat_lists(l_batch_0, l_batch_of_vec_tls,
                                             element_dtype=dtypes.float32))
 
-    with self.assertRaisesRegexp(errors.InvalidArgumentError,
-                                 r"input_b\[0\].dtype != element_dtype."):
+    if context.executing_eagerly():
+      expected_error = (errors.InvalidArgumentError,
+                        r"input_b\[0\].dtype != element_dtype.")
+    else:
+      expected_error = (ValueError, "input_b.type != element_dtype")
+    with self.assertRaisesRegexp(*expected_error):
       l_batch_of_int_tls = array_ops.stack(
           [list_ops.tensor_list_from_tensor([1], element_shape=[])] * 2)
       self.evaluate(
@@ -1003,8 +1012,11 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       self.evaluate(
           list_ops.tensor_list_push_back_batch(l_batch, [[3.0], [4.0]]))
 
-    with self.assertRaisesRegexp(errors.InvalidArgumentError,
-                                 "Invalid data type at index 0"):
+    if context.executing_eagerly():
+      expected_error = (errors.InvalidArgumentError, "Invalid data type")
+    else:
+      expected_error = (ValueError, "wrong element dtype")
+    with self.assertRaisesRegexp(*expected_error):
       self.evaluate(list_ops.tensor_list_push_back_batch(l_batch, [3, 4]))
 
   def testZerosLike(self):
diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index d31652eb89..f90c090ac7 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -24,11 +24,15 @@ import pickle
 
 import numpy as np
 
+from tensorflow.core.framework import tensor_pb2
 from tensorflow.python.eager import context
+from tensorflow.python.eager import def_function
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import cpp_shape_inference_pb2
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
@@ -1035,6 +1039,59 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
         with self.assertRaises(errors.InvalidArgumentError):
           session.run(copied.initializer)
 
+  def create_variant_shape_and_type_data(self):
+    variant_shape_and_type_data = (
+        cpp_shape_inference_pb2.CppShapeInferenceResult.HandleData())
+    variant_shape_and_type_data.is_set = True
+    stored_shape = tensor_shape.TensorShape([None, 4]).as_proto()
+    stored_dtype = dtypes.float32.as_datatype_enum
+    # NOTE(ebrevdo): shape_and_type lacks append() in some versions of protobuf.
+    variant_shape_and_type_data.shape_and_type.extend([
+        cpp_shape_inference_pb2.CppShapeInferenceResult.HandleShapeAndType(
+            shape=stored_shape, dtype=stored_dtype)])
+    return variant_shape_and_type_data
+
+  @def_function.function
+  def create_constant_variant(self, value):
+    value = constant_op.constant(
+        tensor_pb2.TensorProto(
+            dtype=dtypes.variant.as_datatype_enum,
+            tensor_shape=tensor_shape.TensorShape([]).as_proto(),
+            variant_val=[
+                tensor_pb2.VariantTensorDataProto(
+                    # Match registration in variant_op_registry.cc
+                    type_name=b"int",
+                    metadata=np.array(value, dtype=np.int32).tobytes())
+            ]))
+    return value
+
+  # TODO(ebrevdo): Add run_in_graph_and_eager_modes once we can create
+  # EagerTensor constants with TensorProto inputs.
+  @test_util.run_in_graph_and_eager_modes()
+  def testVariantInitializer(self):
+    variant_shape_and_type_data = self.create_variant_shape_and_type_data()
+    value = self.create_constant_variant(3)
+    initializer = array_ops.fill([3], value)
+    resource_variable_ops._set_handle_shapes_and_types(  # pylint: disable=protected-access
+        initializer, variant_shape_and_type_data,
+        graph_mode=not context.executing_eagerly())
+    v = resource_variable_ops.ResourceVariable(initializer)
+    read = array_ops.identity(v)
+    read_variant_shape_and_type = (
+        resource_variable_ops.get_eager_safe_handle_data(read))
+    self.assertEqual(
+        read_variant_shape_and_type, variant_shape_and_type_data)
+    gather = v.sparse_read([0])
+    gather_variant_shape_and_type = (
+        resource_variable_ops.get_eager_safe_handle_data(gather))
+    self.assertEqual(
+        gather_variant_shape_and_type, variant_shape_and_type_data)
+    # Make sure initializer runs.
+    if not context.executing_eagerly():
+      self.evaluate(v.initializer)
+      self.evaluate(read.op)
+      self.evaluate(gather.op)
+
 
 class _MixedPrecisionVariableTest(test_util.TensorFlowTestCase):
 
diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index b9ff2f9717..287510342e 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -74,9 +74,13 @@ def identity(input, name=None):  # pylint: disable=redefined-builtin
     context_device = context.context().device_name
     if not context_device:
       context_device = "/job:localhost/replica:0/task:0/device:CPU:0"
-    if context_device != in_device:
-      return input._copy()  # pylint: disable=protected-access
-    return input
+    if context_device == in_device:
+      return input
+    else:
+      copied = input._copy()  # pylint: disable=protected-access
+      if hasattr(copied, "_handle_data"):
+        copied._handle_data = input._handle_data  # pylint: disable=protected-access
+      return copied
   else:
     ret = gen_array_ops.identity(input, name=name)
     # Propagate handle data for happier shape inference for resource variables.
diff --git a/tensorflow/python/ops/custom_gradient.py b/tensorflow/python/ops/custom_gradient.py
index d96601ac21..5fc3fdcc3e 100644
--- a/tensorflow/python/ops/custom_gradient.py
+++ b/tensorflow/python/ops/custom_gradient.py
@@ -56,7 +56,9 @@ def copy_handle_data(source_t, target_t):
       handle_data = source_t._handle_data  # pylint: disable=protected-access
     else:
       handle_data = resource_variable_ops.get_resource_handle_data(source_t)
-    if handle_data is not None and handle_data.is_set:
+    if (handle_data is not None
+        and handle_data.is_set
+        and handle_data.shape_and_type):
       # pylint: disable=protected-access
       pywrap_tensorflow.SetHandleShapeAndType(target_t.graph._c_graph,
                                               target_t._as_tf_output(),
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index c8f20029ac..c1f11e89a0 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -58,8 +58,118 @@ def get_resource_handle_data(graph_op):
       compat.as_bytes(handle_data))
 
 
-def eager_safe_variable_handle(shape, dtype, shared_name, name, graph_mode):
-  """Creates a variable handle with information to do shape inference."""
+def get_eager_safe_handle_data(handle):
+  """Get the data handle from the Tensor `handle`."""
+  assert isinstance(handle, ops.Tensor)
+
+  if isinstance(handle, ops.EagerTensor):
+    return handle._handle_data  # pylint: disable=protected-access
+  else:
+    return get_resource_handle_data(handle)
+
+
+def _set_handle_shapes_and_types(tensor, handle_data, graph_mode):
+  """Sets the shape inference result HandleData on tensor.
+
+  Args:
+    tensor: A `Tensor` or `EagerTensor`.
+    handle_data: A `CppShapeInferenceResult.HandleData`.
+    graph_mode: A python bool.
+  """
+  tensor._handle_data = handle_data  # pylint: disable=protected-access
+  if not graph_mode:
+    return
+
+  # Not an EagerTensor, so a graph tensor.
+  shapes, types = zip(*[(pair.shape, pair.dtype)
+                        for pair in handle_data.shape_and_type])
+  ranks = [len(s.dim) if not s.unknown_rank else -1 for s in shapes]
+  shapes = [[d.size for d in s.dim]
+            if not s.unknown_rank else None for s in shapes]
+  pywrap_tensorflow.TF_GraphSetOutputHandleShapesAndTypes_wrapper(
+      tensor._op._graph._c_graph,  # pylint: disable=protected-access
+      tensor._as_tf_output(),  # pylint: disable=protected-access
+      shapes, ranks, types)
+
+
+def _combine_handle_data(handle, initial_value):
+  """Concats HandleData from tensors `handle` and `initial_value`.
+
+  Args:
+    handle: A `Tensor` of dtype `resource`.
+    initial_value: A `Tensor`.
+
+  Returns:
+    A `CppShapeInferenceResult.HandleData`.  If `initial_value` has dtype
+    `variant`, the `HandleData` contains the concatenation of the shape_and_type
+    from both `handle` and `initial_value`.
+
+  Raises:
+    RuntimeError: If handle, which was returned by VarHandleOp, either has
+      no handle data, or its len(handle_data.shape_and_type) != 1.
+  """
+  assert handle.dtype == dtypes.resource
+
+  variable_handle_data = get_eager_safe_handle_data(handle)
+
+  if initial_value.dtype != dtypes.variant:
+    return variable_handle_data
+
+  extra_handle_data = get_eager_safe_handle_data(initial_value)
+  if extra_handle_data is not None and extra_handle_data.is_set:
+    if (variable_handle_data is None
+        or not variable_handle_data.is_set
+        or len(variable_handle_data.shape_and_type) != 1):
+      raise RuntimeError(
+          "Expected VarHandleOp to return a length==1 shape_and_type, "
+          "but saw: '%s'" % (variable_handle_data,))
+    variable_handle_data.shape_and_type.extend(
+        extra_handle_data.shape_and_type)
+  return variable_handle_data
+
+
+def eager_safe_variable_handle(initial_value, shared_name, name, graph_mode):
+  """Creates a variable handle with information to do shape inference.
+
+  The shape and dtype are read from `initial_value` and stored in the returned
+  resource tensor's handle data.
+
+  If `initial_value.dtype == tf.variant`, we additionally extract the handle
+  data (if any) from `initial_value` and append it to the `handle_data`.
+  In this case, the returned tensor's handle data is in the form
+
+  ```
+  is_set: true
+  shape_and_type {
+    shape {
+      // initial_value.shape
+    }
+    dtype: DT_VARIANT
+  }
+  shape_and_type {
+    // handle_data(initial_value).shape_and_type[0]
+  }
+  shape_and_type {
+    // handle_data(initial_value).shape_and_type[1]
+  }
+  ...
+  ```
+
+  Ops that read from this tensor, such as `ReadVariableOp` and
+  `AssignVariableOp`, know that `handle_data(handle).shape_and_type[1:]`
+  correspond to the handle data of the variant(s) stored in the Variable.
+
+  Args:
+    initial_value: A `Tensor`.
+    shared_name: A string.
+    name: A string.
+    graph_mode: A python bool.
+
+  Returns:
+    The handle, a `Tensor` of type `resource`.
+  """
+  shape = initial_value.get_shape()
+  dtype = initial_value.dtype.base_dtype
   container = ops.get_default_graph()._container  # pylint: disable=protected-access
   if container is None:
     container = ""
@@ -67,35 +177,38 @@ def eager_safe_variable_handle(shape, dtype, shared_name, name, graph_mode):
                                                    shared_name=shared_name,
                                                    name=name,
                                                    container=container)
+
   if graph_mode:
-    handle._handle_data = get_resource_handle_data(handle)  # pylint: disable=protected-access
+    full_handle_data = _combine_handle_data(handle, initial_value)
+    _set_handle_shapes_and_types(handle, full_handle_data, graph_mode)
+    return handle
+  else:
+    # We do not want two distinct ResourceVariable objects for the same
+    # underlying resource in the runtime.
+    # When in eager mode, explicitly ensure so here. When in graph mode, it's
+    # ensured by always generating different variable names.
+    exists = gen_resource_variable_ops.var_is_initialized_op(handle)
+    if exists:
+      raise ValueError("variable object with name '%s' already created. Use "
+                       "get_variable() if reuse is desired." %
+                       shared_name)
+    with context.graph_mode(), ops.Graph().as_default() as graph:
+      h = gen_resource_variable_ops.var_handle_op(shape=shape, dtype=dtype,
+                                                  shared_name=shared_name,
+                                                  name=name,
+                                                  container=container)
+
+      # Tensor._handle_data contains information for the shape-inference code to
+      # know the shape and dtype of the variable pointed to by a handle. Since
+      # shape inference doesn't run in eager mode we copy this data here for
+      # when the handle is captured by an eager mode function.
+      # pylint: disable=protected-access
+      full_handle_data = _combine_handle_data(h, initial_value)
+      _set_handle_shapes_and_types(handle, full_handle_data, graph_mode)
+      # pylint: enable=protected-access
+    # Clean up op->graph->op reference cycles.
+    ops.dismantle_graph(graph)
     return handle
-
-  # We do not want two distinct ResourceVariable objects for the same
-  # underlying resource in the runtime.
-  # When in eager mode, explicitly ensure so here. When in graph mode, it's
-  # ensured by always generating different variable names.
-  exists = gen_resource_variable_ops.var_is_initialized_op(handle)
-  if exists:
-    raise ValueError("variable object with name '%s' already created. Use "
-                     "get_variable() if reuse is desired." %
-                     shared_name)
-  with context.graph_mode(), ops.Graph().as_default() as graph:
-    h = gen_resource_variable_ops.var_handle_op(shape=shape, dtype=dtype,
-                                                shared_name=shared_name,
-                                                name=name,
-                                                container=container)
-
-    # Tensor._handle_data contains information for the shape-inference code to
-    # know the shape and dtype of the variable pointed to by a handle. Since
-    # shape inference doesn't run in eager mode we copy this data here for when
-    # the handle is captured by an eager mode function.
-    # pylint: disable=protected-access
-    handle._handle_data = get_resource_handle_data(h)
-    # pylint: enable=protected-access
-  # Clean up op->graph->op reference cycles.
-  ops.dismantle_graph(graph)
-  return handle
 
 
 @contextlib.contextmanager
@@ -162,6 +275,18 @@ def shape_safe_assign_variable_handle(handle, shape, value, name=None):
                                                       name=name)
 
 
+def _maybe_set_handle_data(dtype, handle, tensor):
+  if dtype == dtypes.variant:
+    # For DT_VARIANT types, the handle's shape_and_type[1:] stores the
+    # variant's handle data.  Extract it.
+    handle_data = get_eager_safe_handle_data(handle)
+    if handle_data.is_set and len(handle_data.shape_and_type) > 1:
+      tensor._handle_data = (  # pylint: disable=protected-access
+          cpp_shape_inference_pb2.CppShapeInferenceResult.HandleData(
+              is_set=True,
+              shape_and_type=handle_data.shape_and_type[1:]))
+
+
 class ResourceVariable(variables.VariableV1):
   """Variable based on resource handles.
 
@@ -420,8 +545,7 @@ class ResourceVariable(variables.VariableV1):
                 initial_value() if init_from_fn else initial_value,
                 name="initial_value", dtype=dtype)
           self._handle = eager_safe_variable_handle(
-              shape=initial_value.get_shape(),
-              dtype=initial_value.dtype.base_dtype,
+              initial_value=initial_value,
               shared_name=shared_name,
               name=name,
               graph_mode=self._in_graph_mode)
@@ -732,6 +856,8 @@ class ResourceVariable(variables.VariableV1):
       tape.variable_accessed(self)
     result = gen_resource_variable_ops.read_variable_op(self._handle,
                                                         self._dtype)
+    _maybe_set_handle_data(self._dtype, self._handle, result)
+
     if not context.executing_eagerly():
       # Note that if a control flow context is active the input of the read op
       # might not actually be the handle. This line bypasses it.
@@ -763,6 +889,17 @@ class ResourceVariable(variables.VariableV1):
         tape.variable_accessed(self)
       value = gen_resource_variable_ops.resource_gather(
           self._handle, indices, dtype=self._dtype, name=name)
+
+      if self._dtype == dtypes.variant:
+        # For DT_VARIANT types, the handle's shape_and_type[1:] stores the
+        # variant's handle data.  Extract it.
+        handle_data = get_eager_safe_handle_data(self._handle)
+        if handle_data.is_set and len(handle_data.shape_and_type) > 1:
+          value._handle_data = (  # pylint: disable=protected-access
+              cpp_shape_inference_pb2.CppShapeInferenceResult.HandleData(
+                  is_set=True,
+                  shape_and_type=handle_data.shape_and_type[1:]))
+
     return array_ops.identity(value)
 
   def to_proto(self, export_scope=None):
@@ -1337,8 +1474,11 @@ class _UnreadVariable(ResourceVariable):
 
   def _read_variable_op(self):
     with ops.control_dependencies([self._parent_op]):
-      return gen_resource_variable_ops.read_variable_op(self._handle,
-                                                        self._dtype)
+      result = gen_resource_variable_ops.read_variable_op(self._handle,
+                                                          self._dtype)
+      _maybe_set_handle_data(self._dtype, self._handle, result)
+      return result
+
 
   @property
   def op(self):
@@ -1423,6 +1563,7 @@ class _MixedPrecisionVariable(ResourceVariable):
     with ops.colocate_with(self._handle):
       res = gen_resource_variable_ops.read_variable_op(self._handle,
                                                        self._dtype)
+      _maybe_set_handle_data(self._dtype, self._handle, res)
       if self._read_dtype != self._dtype:
         return math_ops.cast(res, self._read_dtype)
       else:
-- 
GitLab


From 9b91ea70e84501d0e264d783d54907bac8cca92b Mon Sep 17 00:00:00 2001
From: Sergio Guadarrama <sguada@google.com>
Date: Sat, 26 Jan 2019 12:43:32 -0800
Subject: [PATCH 1341/2345] Use more efficient squared_difference instead of
 square(diff)

PiperOrigin-RevId: 231064313
---
 tensorflow/python/keras/losses.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index 39063c55c8..01bd28587f 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -790,7 +790,7 @@ class Huber(Loss):
               'keras.losses.mse',
               'keras.losses.MSE')
 def mean_squared_error(y_true, y_pred):
-  return K.mean(math_ops.square(y_pred - y_true), axis=-1)
+  return K.mean(math_ops.squared_difference(y_pred, y_true), axis=-1)
 
 
 @keras_export('keras.metrics.mean_absolute_error',
-- 
GitLab


From e9e5028139701661522cbaa5847fa8371fb849df Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Sat, 26 Jan 2019 15:13:09 -0800
Subject: [PATCH 1342/2345] Remove no_pip tag from distribute strategy tests.

PiperOrigin-RevId: 231071078
---
 tensorflow/contrib/distribute/python/BUILD | 67 ++++------------------
 tensorflow/python/distribute/BUILD         |  7 ---
 tensorflow/tools/pip_package/BUILD         |  1 +
 3 files changed, 13 insertions(+), 62 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index e6bf6ed3e8..1b455a4e64 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -15,8 +15,18 @@ licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
-# TODO(priyag): Figure out testonly issues that are preventing us from
-# including our tests in pip for now.
+py_library(
+    name = "distribute_test_lib_pip",
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        ":combinations",
+        ":keras_correctness_test_lib",
+        ":keras_test_lib",
+        ":multi_worker_test_base",
+        ":single_loss_example",
+        ":strategy_test_lib",
+    ],
+)
 
 cuda_py_test(
     name = "values_test",
@@ -38,9 +48,6 @@ cuda_py_test(
         "//tensorflow/python/eager:test",
         "//tensorflow/python/estimator:estimator_py",
     ],
-    tags = [
-        "no_pip",
-    ],
 )
 
 cuda_py_test(
@@ -58,9 +65,6 @@ cuda_py_test(
         "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:test",
     ],
-    tags = [
-        "no_pip",
-    ],
 )
 
 py_library(
@@ -115,7 +119,6 @@ cuda_py_test(
     ],
     tags = [
         "multi_and_single_gpu",
-        "no_pip",
     ],
 )
 
@@ -161,12 +164,8 @@ py_library(
 
 py_library(
     name = "strategy_test_lib",
-    testonly = 1,
     srcs = ["strategy_test_lib.py"],
     srcs_version = "PY2AND3",
-    tags = [
-        "no_pip",
-    ],
     deps = [
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
@@ -185,12 +184,8 @@ py_library(
 
 py_library(
     name = "combinations",
-    testonly = 1,
     srcs = ["combinations.py"],
     srcs_version = "PY2AND3",
-    tags = [
-        "no_pip",
-    ],
     deps = [
         ":mirrored_strategy",
         ":one_device_strategy",
@@ -210,9 +205,6 @@ py_library(
 py_test(
     name = "combinations_test",
     srcs = ["combinations_test.py"],
-    tags = [
-        "no_pip",
-    ],
     deps = [
         ":combinations",
         "//tensorflow/python/eager:test",
@@ -223,9 +215,6 @@ py_test(
     name = "one_device_strategy_test",
     srcs = ["one_device_strategy_test.py"],
     srcs_version = "PY2AND3",
-    tags = [
-        "no_pip",
-    ],
     deps = [
         ":one_device_strategy",
         ":strategy_test_lib",
@@ -259,18 +248,13 @@ cuda_py_test(
     tags = [
         "guitar",
         "multi_and_single_gpu",
-        "no_pip",
     ],
 )
 
 py_library(
     name = "multi_worker_test_base",
-    testonly = 1,
     srcs = ["multi_worker_test_base.py"],
     srcs_version = "PY2AND3",
-    tags = [
-        "no_pip",
-    ],
     deps = [
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:client_testlib",
@@ -339,7 +323,6 @@ cuda_py_test(
     ],
     tags = [
         "multi_and_single_gpu",
-        "no_pip",
     ],
 )
 
@@ -349,7 +332,6 @@ distribute_py_test(
     main = "minimize_loss_test.py",
     tags = [
         "multi_and_single_gpu",
-        "no_pip",
     ],
     deps = [
         ":combinations",
@@ -383,9 +365,6 @@ cuda_py_test(
         "//tensorflow/python:training",
         "//tensorflow/python:variables",
     ],
-    tags = [
-        "no_pip",
-    ],
 )
 
 cuda_py_test(
@@ -403,7 +382,6 @@ cuda_py_test(
     ],
     tags = [
         "multi_and_single_gpu",
-        "no_pip",
     ],
 )
 
@@ -426,7 +404,6 @@ cuda_py_test(
     tags = [
         "multi_and_single_gpu",
         "no_oss",  # http://b/119349471
-        "no_pip",
         "tf_integration_test",
     ],
 )
@@ -440,7 +417,6 @@ cuda_py_test(
     tags = [
         "multi_and_single_gpu",
         "no_oss",  # http://b/119349471
-        "no_pip",
         "tf_integration_test",
     ],
 )
@@ -470,7 +446,6 @@ cuda_py_test(
     shard_count = 48,
     tags = [
         "multi_and_single_gpu",
-        "no_pip",
         # TODO(b/118768923): Re-enable {a,m,t}san test.
         "noasan",
         "nomsan",
@@ -498,7 +473,6 @@ distribute_py_test(
     main = "step_fn_test.py",
     tags = [
         "multi_and_single_gpu",
-        "no_pip",
     ],
     deps = [
         ":combinations",
@@ -539,7 +513,6 @@ cuda_py_test(
     ],
     tags = [
         "multi_and_single_gpu",
-        "no_pip",
     ],
 )
 
@@ -556,9 +529,6 @@ cuda_py_test(
         "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:test",
     ],
-    tags = [
-        "no_pip",
-    ],
 )
 
 cuda_py_test(
@@ -580,13 +550,11 @@ cuda_py_test(
     ],
     tags = [
         "multi_and_single_gpu",
-        "no_pip",
     ],
 )
 
 py_library(
     name = "keras_test_lib",
-    testonly = 1,
     srcs = [
         "keras_backward_compat_test.py",
         "keras_test.py",
@@ -613,7 +581,6 @@ distribute_py_test(
     tags = [
         "multi_and_single_gpu",
         "no_oss",  # TODO(b/117919883): Fix python error.
-        "no_pip",
         "no_windows_gpu",
         "notsan",
     ],
@@ -632,7 +599,6 @@ distribute_py_test(
     tags = [
         "multi_and_single_gpu",
         "no_oss",  # TODO(b/117919883): Fix python error.
-        "no_pip",
         "no_windows_gpu",
         "notsan",
     ],
@@ -643,7 +609,6 @@ distribute_py_test(
 
 py_library(
     name = "keras_correctness_test_lib",
-    testonly = 1,
     srcs = [
         "keras_correctness_test_base.py",
         "keras_dnn_correctness_test.py",
@@ -678,7 +643,6 @@ distribute_py_test(
     tags = [
         "multi_and_single_gpu",
         "no_oss",  # TODO(b/117919883): Fix python error.
-        "no_pip",
         "no_windows_gpu",
         "notsan",
     ],
@@ -699,7 +663,6 @@ distribute_py_test(
     tags = [
         "multi_and_single_gpu",
         "no_oss",  # TODO(b/117919883): Fix python error.
-        "no_pip",
         "no_windows_gpu",
         "notsan",
     ],
@@ -720,7 +683,6 @@ distribute_py_test(
     tags = [
         "multi_and_single_gpu",
         "no_oss",  # TODO(b/117919883): Fix python error.
-        "no_pip",
         "no_windows_gpu",
         "notsan",
     ],
@@ -741,7 +703,6 @@ distribute_py_test(
     tags = [
         "multi_and_single_gpu",
         "no_oss",  # TODO(b/117919883): Fix python error.
-        "no_pip",
         "no_windows_gpu",
         "notsan",
     ],
@@ -777,7 +738,6 @@ distribute_py_test(
     main = "metrics_v1_test.py",
     tags = [
         "multi_and_single_gpu",
-        "no_pip",
     ],
     deps = [
         ":combinations",
@@ -804,7 +764,6 @@ cuda_py_test(
     ],
     tags = [
         "multi_and_single_gpu",
-        "no_pip",
     ],
 )
 
@@ -823,7 +782,6 @@ cuda_py_test(
     ],
     tags = [
         "multi_and_single_gpu",
-        "no_pip",
     ],
 )
 
@@ -838,7 +796,6 @@ tf_xla_py_test(
     ],
     tags = [
         "no_oss",
-        "no_pip",
     ],
     deps = [
         ":tpu_strategy",
diff --git a/tensorflow/python/distribute/BUILD b/tensorflow/python/distribute/BUILD
index 24a2a84b7f..36e8557126 100644
--- a/tensorflow/python/distribute/BUILD
+++ b/tensorflow/python/distribute/BUILD
@@ -185,9 +185,6 @@ py_test(
     name = "distribute_coordinator_test",
     srcs = ["distribute_coordinator_test.py"],
     srcs_version = "PY2AND3",
-    tags = [
-        "no_pip",
-    ],
     deps = [
         ":distribute_coordinator",
         "//tensorflow/core:protos_all_py",
@@ -347,16 +344,12 @@ cuda_py_test(
         "//tensorflow/python:io_ops",
         "//tensorflow/python:util",
     ],
-    tags = [
-        "no_pip",
-    ],
 )
 
 py_test(
     name = "multi_worker_util_test",
     srcs = ["multi_worker_util_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_pip"],
     deps = [
         ":multi_worker_util",
         "//tensorflow/core:protos_all_py",
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index 2731949c15..bc970d0f76 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -100,6 +100,7 @@ COMMON_PIP_DEPS_V1 = COMMON_PIP_DEPS + [
     "//tensorflow/contrib/cluster_resolver:cluster_resolver_pip",
     "//tensorflow/contrib/compiler:xla",
     "//tensorflow/contrib/constrained_optimization:constrained_optimization_pip",
+    "//tensorflow/contrib/distribute/python:distribute_test_lib_pip",
     "//tensorflow/contrib/eager/python/examples:examples_pip",
     "//tensorflow/contrib/eager/python:evaluator",
     "//tensorflow/contrib/gan:gan",
-- 
GitLab


From 20957e396e77535ee241e1dd311bdb00febce637 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sat, 26 Jan 2019 23:55:29 +0000
Subject: [PATCH 1343/2345] Replace deprecated test_session with cached_session

While running tests:
```
bazel test -s --verbose_failures --config=opt //tensorflow/python:image_ops_test
```

there are many deprected warnings:
```
From /usr/lib/python2.7/contextlib.py:84: test_session (from tensorflow.python.framework.test_util) is deprecated and will be removed in a future version.
Instructions for updating:
Use `self.session()` or `self.cached_session()` instead.
```

This fix replace deprecated test_session with cached_session.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/ops/image_ops_test.py | 258 ++++++++++++------------
 1 file changed, 129 insertions(+), 129 deletions(-)

diff --git a/tensorflow/python/ops/image_ops_test.py b/tensorflow/python/ops/image_ops_test.py
index 361befabce..b032d6454d 100644
--- a/tensorflow/python/ops/image_ops_test.py
+++ b/tensorflow/python/ops/image_ops_test.py
@@ -61,7 +61,7 @@ class RGBToHSVTest(test_util.TensorFlowTestCase):
       inp = np.random.rand(*shape).astype(nptype)
 
       # Convert to HSV and back, as a batch and individually
-      with self.test_session(use_gpu=True) as sess:
+      with self.cached_session(use_gpu=True) as sess:
         batch0 = constant_op.constant(inp)
         batch1 = image_ops.rgb_to_hsv(batch0)
         batch2 = image_ops.hsv_to_rgb(batch1)
@@ -82,7 +82,7 @@ class RGBToHSVTest(test_util.TensorFlowTestCase):
     data = [0, 5, 13, 54, 135, 226, 37, 8, 234, 90, 255, 1]
     for nptype in [np.float32, np.float64]:
       rgb_np = np.array(data, dtype=nptype).reshape([2, 2, 3]) / 255.
-      with self.test_session(use_gpu=True):
+      with self.cached_session(use_gpu=True):
         hsv = image_ops.rgb_to_hsv(rgb_np)
         rgb = image_ops.hsv_to_rgb(hsv)
         rgb_tf = self.evaluate(rgb)
@@ -101,7 +101,7 @@ class RGBToYIQTest(test_util.TensorFlowTestCase):
       inp = np.random.rand(*shape).astype(nptype)
 
       # Convert to YIQ and back, as a batch and individually
-      with self.test_session(use_gpu=True) as sess:
+      with self.cached_session(use_gpu=True) as sess:
         batch0 = constant_op.constant(inp)
         batch1 = image_ops.rgb_to_yiq(batch0)
         batch2 = image_ops.yiq_to_rgb(batch1)
@@ -131,7 +131,7 @@ class RGBToYUVTest(test_util.TensorFlowTestCase):
       inp = np.random.rand(*shape).astype(nptype)
 
       # Convert to YUV and back, as a batch and individually
-      with self.test_session(use_gpu=True) as sess:
+      with self.cached_session(use_gpu=True) as sess:
         batch0 = constant_op.constant(inp)
         batch1 = image_ops.rgb_to_yuv(batch0)
         batch2 = image_ops.yuv_to_rgb(batch1)
@@ -173,7 +173,7 @@ class GrayscaleToRGBTest(test_util.TensorFlowTestCase):
   def _TestRGBToGrayscale(self, x_np):
     y_np = self._RGBToGrayscale(x_np)
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.rgb_to_grayscale(x_tf)
       y_tf = self.evaluate(y)
@@ -195,7 +195,7 @@ class GrayscaleToRGBTest(test_util.TensorFlowTestCase):
     y_np = np.array(
         [[1, 1, 1], [2, 2, 2]], dtype=np.uint8).reshape([1, 1, 2, 3])
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.grayscale_to_rgb(x_tf)
       y_tf = self.evaluate(y)
@@ -205,7 +205,7 @@ class GrayscaleToRGBTest(test_util.TensorFlowTestCase):
     x_np = np.array([[1, 2]], dtype=np.uint8).reshape([1, 2, 1])
     y_np = np.array([[1, 1, 1], [2, 2, 2]], dtype=np.uint8).reshape([1, 2, 3])
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.grayscale_to_rgb(x_tf)
       y_tf = self.evaluate(y)
@@ -216,23 +216,23 @@ class GrayscaleToRGBTest(test_util.TensorFlowTestCase):
     # Shape inference works and produces expected output where possible
     rgb_shape = [7, None, 19, 3]
     gray_shape = rgb_shape[:-1] + [1]
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       rgb_tf = array_ops.placeholder(dtypes.uint8, shape=rgb_shape)
       gray = image_ops.rgb_to_grayscale(rgb_tf)
       self.assertEqual(gray_shape, gray.get_shape().as_list())
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       gray_tf = array_ops.placeholder(dtypes.uint8, shape=gray_shape)
       rgb = image_ops.grayscale_to_rgb(gray_tf)
       self.assertEqual(rgb_shape, rgb.get_shape().as_list())
 
     # Shape inference does not break for unknown shapes
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       rgb_tf_unknown = array_ops.placeholder(dtypes.uint8)
       gray_unknown = image_ops.rgb_to_grayscale(rgb_tf_unknown)
       self.assertFalse(gray_unknown.get_shape())
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       gray_tf_unknown = array_ops.placeholder(dtypes.uint8)
       rgb_unknown = image_ops.grayscale_to_rgb(gray_tf_unknown)
       self.assertFalse(rgb_unknown.get_shape())
@@ -364,7 +364,7 @@ class AdjustHueTest(test_util.TensorFlowTestCase):
     y_data = [0, 13, 1, 54, 226, 59, 8, 234, 150, 255, 39, 1]
     y_np = np.array(y_data, dtype=np.uint8).reshape(x_shape)
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x = constant_op.constant(x_np, shape=x_shape)
       y = image_ops.adjust_hue(x, delta)
       y_tf = self.evaluate(y)
@@ -379,7 +379,7 @@ class AdjustHueTest(test_util.TensorFlowTestCase):
     y_data = [13, 0, 11, 226, 54, 221, 234, 8, 92, 1, 217, 255]
     y_np = np.array(y_data, dtype=np.uint8).reshape(x_shape)
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x = constant_op.constant(x_np, shape=x_shape)
       y = image_ops.adjust_hue(x, delta)
       y_tf = self.evaluate(y)
@@ -394,7 +394,7 @@ class AdjustHueTest(test_util.TensorFlowTestCase):
     y_data = [13, 0, 11, 226, 54, 221, 234, 8, 92, 1, 217, 255]
     y_np = np.array(y_data, dtype=np.uint8).reshape(x_shape)
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x = constant_op.constant(x_np, shape=x_shape)
       y = image_ops.adjust_hue(x, delta)
       y_tf = self.evaluate(y)
@@ -419,7 +419,7 @@ class AdjustHueTest(test_util.TensorFlowTestCase):
     return y_v.reshape(x_np.shape)
 
   def _adjustHueTf(self, x_np, delta_h):
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x = constant_op.constant(x_np)
       y = image_ops.adjust_hue(x, delta_h)
       y_tf = self.evaluate(y)
@@ -850,7 +850,7 @@ class AdjustSaturationTest(test_util.TensorFlowTestCase):
     y_data = [6, 9, 13, 140, 180, 226, 135, 121, 234, 172, 255, 128]
     y_np = np.array(y_data, dtype=np.uint8).reshape(x_shape)
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x = constant_op.constant(x_np, shape=x_shape)
       y = image_ops.adjust_saturation(x, saturation_factor)
       y_tf = self.evaluate(y)
@@ -865,7 +865,7 @@ class AdjustSaturationTest(test_util.TensorFlowTestCase):
     y_data = [0, 5, 13, 0, 106, 226, 30, 0, 234, 89, 255, 0]
     y_np = np.array(y_data, dtype=np.uint8).reshape(x_shape)
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x = constant_op.constant(x_np, shape=x_shape)
       y = image_ops.adjust_saturation(x, saturation_factor)
       y_tf = self.evaluate(y)
@@ -880,7 +880,7 @@ class AdjustSaturationTest(test_util.TensorFlowTestCase):
     y_data = [6, 9, 13, 140, 180, 226, 135, 121, 234, 172, 255, 128]
     y_np = np.array(y_data, dtype=np.uint8).reshape(x_shape)
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x = constant_op.constant(x_np, shape=x_shape)
       y = image_ops.adjust_saturation(x, saturation_factor)
       y_tf = self.evaluate(y)
@@ -920,7 +920,7 @@ class AdjustSaturationTest(test_util.TensorFlowTestCase):
         "gb_same",
         "rgb_same",
     ]
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       for x_shape in x_shapes:
         for test_style in test_styles:
           x_np = np.random.rand(*x_shape) * 255.
@@ -947,7 +947,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
 
   def testInvolutionLeftRight(self):
     x_np = np.array([[1, 2, 3], [1, 2, 3]], dtype=np.uint8).reshape([2, 3, 1])
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.flip_left_right(image_ops.flip_left_right(x_tf))
       y_tf = self.evaluate(y)
@@ -957,7 +957,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
     x_np = np.array(
         [[[1, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]]],
         dtype=np.uint8).reshape([2, 2, 3, 1])
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.flip_left_right(image_ops.flip_left_right(x_tf))
       y_tf = self.evaluate(y)
@@ -968,7 +968,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
     x_np = np.array([[1, 2, 3], [1, 2, 3]], dtype=np.uint8).reshape([2, 3, 1])
     y_np = np.array([[3, 2, 1], [3, 2, 1]], dtype=np.uint8).reshape([2, 3, 1])
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.flip_left_right(x_tf)
       self.assertTrue(y.op.name.startswith("flip_left_right"))
@@ -983,7 +983,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
         [[[3, 2, 1], [3, 2, 1]], [[3, 2, 1], [3, 2, 1]]],
         dtype=np.uint8).reshape([2, 2, 3, 1])
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.flip_left_right(x_tf)
       y_tf = self.evaluate(y)
@@ -995,7 +995,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
     y_np = np.array([[3, 2, 1], [3, 2, 1]], dtype=np.uint8).reshape([2, 3, 1])
     seed = 42
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.random_flip_left_right(x_tf, seed=seed)
       self.assertTrue(y.op.name.startswith("random_flip_left_right"))
@@ -1035,7 +1035,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
     x_np = np.vstack([x_np_raw for _ in range(batch_size)])
     y_np = np.vstack([y_np_raw for _ in range(batch_size)])
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.random_flip_left_right(x_tf, seed=seed)
       self.assertTrue(y.op.name.startswith("random_flip_left_right"))
@@ -1066,7 +1066,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
   def testInvolutionUpDown(self):
     x_np = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.uint8).reshape([2, 3, 1])
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.flip_up_down(image_ops.flip_up_down(x_tf))
       y_tf = self.evaluate(y)
@@ -1077,7 +1077,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
         [[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]],
         dtype=np.uint8).reshape([2, 2, 3, 1])
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.flip_up_down(image_ops.flip_up_down(x_tf))
       y_tf = self.evaluate(y)
@@ -1088,7 +1088,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
     x_np = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.uint8).reshape([2, 3, 1])
     y_np = np.array([[4, 5, 6], [1, 2, 3]], dtype=np.uint8).reshape([2, 3, 1])
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.flip_up_down(x_tf)
       self.assertTrue(y.op.name.startswith("flip_up_down"))
@@ -1103,7 +1103,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
         [[[4, 5, 6], [1, 2, 3]], [[10, 11, 12], [7, 8, 9]]],
         dtype=np.uint8).reshape([2, 2, 3, 1])
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.flip_up_down(x_tf)
       y_tf = self.evaluate(y)
@@ -1116,7 +1116,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
 
     seed = 42
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.random_flip_up_down(x_tf, seed=seed)
       self.assertTrue(y.op.name.startswith("random_flip_up_down"))
@@ -1155,7 +1155,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
     x_np = np.vstack([x_np_raw for _ in range(batch_size)])
     y_np = np.vstack([y_np_raw for _ in range(batch_size)])
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.random_flip_up_down(x_tf, seed=seed)
       self.assertTrue(y.op.name.startswith("random_flip_up_down"))
@@ -1186,7 +1186,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
   def testInvolutionTranspose(self):
     x_np = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.uint8).reshape([2, 3, 1])
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.transpose_image(image_ops.transpose_image(x_tf))
       y_tf = self.evaluate(y)
@@ -1197,7 +1197,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
         [[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]],
         dtype=np.uint8).reshape([2, 2, 3, 1])
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.transpose_image(image_ops.transpose_image(x_tf))
       y_tf = self.evaluate(y)
@@ -1208,7 +1208,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
     x_np = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.uint8).reshape([2, 3, 1])
     y_np = np.array([[1, 4], [2, 5], [3, 6]], dtype=np.uint8).reshape([3, 2, 1])
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.transpose_image(x_tf)
       self.assertTrue(y.op.name.startswith("transpose"))
@@ -1224,7 +1224,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
         [[[1, 4], [2, 5], [3, 6]], [[7, 10], [8, 11], [9, 12]]],
         dtype=np.uint8).reshape([2, 3, 2, 1])
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.transpose_image(x_tf)
       y_tf = self.evaluate(y)
@@ -1275,7 +1275,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
 
   def testRot90GroupOrder(self):
     image = np.arange(24, dtype=np.uint8).reshape([2, 4, 3])
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       rotated = image
       for _ in xrange(4):
         rotated = image_ops.rot90(rotated)
@@ -1283,7 +1283,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
 
   def testRot90GroupOrderWithBatch(self):
     image = np.arange(48, dtype=np.uint8).reshape([2, 2, 4, 3])
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       rotated = image
       for _ in xrange(4):
         rotated = image_ops.rot90(rotated)
@@ -1292,7 +1292,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
   @test_util.run_deprecated_v1
   def testRot90NumpyEquivalence(self):
     image = np.arange(24, dtype=np.uint8).reshape([2, 4, 3])
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       k_placeholder = array_ops.placeholder(dtypes.int32, shape=[])
       y_tf = image_ops.rot90(image, k_placeholder)
       for k in xrange(4):
@@ -1302,7 +1302,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
   @test_util.run_deprecated_v1
   def testRot90NumpyEquivalenceWithBatch(self):
     image = np.arange(48, dtype=np.uint8).reshape([2, 2, 4, 3])
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       k_placeholder = array_ops.placeholder(dtypes.int32, shape=[])
       y_tf = image_ops.rot90(image, k_placeholder)
       for k in xrange(4):
@@ -1312,7 +1312,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
 class AdjustContrastTest(test_util.TensorFlowTestCase):
 
   def _testContrast(self, x_np, y_np, contrast_factor):
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.adjust_contrast(x, contrast_factor)
       y_tf = self.evaluate(y)
@@ -1367,7 +1367,7 @@ class AdjustContrastTest(test_util.TensorFlowTestCase):
     return y_np
 
   def _adjustContrastTf(self, x_np, contrast_factor):
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x = constant_op.constant(x_np)
       y = image_ops.adjust_contrast(x, contrast_factor)
       y_tf = self.evaluate(y)
@@ -1401,7 +1401,7 @@ class AdjustContrastTest(test_util.TensorFlowTestCase):
 class AdjustBrightnessTest(test_util.TensorFlowTestCase):
 
   def _testBrightness(self, x_np, y_np, delta, tol=1e-6):
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.adjust_brightness(x, delta)
       y_tf = self.evaluate(y)
@@ -1468,7 +1468,7 @@ class PerImageWhiteningTest(test_util.TensorFlowTestCase):
     x_np = np.arange(0, np.prod(x_shape), dtype=np.int32).reshape(x_shape)
     y_np = self._NumpyPerImageWhitening(x_np)
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x = constant_op.constant(x_np, shape=x_shape)
       y = image_ops.per_image_standardization(x)
       self.assertTrue(y.op.name.startswith("per_image_standardization"))
@@ -1479,14 +1479,14 @@ class PerImageWhiteningTest(test_util.TensorFlowTestCase):
     im_np = np.ones([19, 19, 3]).astype(np.float32) * 249
     im = constant_op.constant(im_np)
     whiten = image_ops.per_image_standardization(im)
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       whiten_np = self.evaluate(whiten)
       self.assertFalse(np.any(np.isnan(whiten_np)))
 
   def testBatchWhitening(self):
     imgs_np = np.random.uniform(0., 255., [4, 24, 24, 3])
     whiten_np = [self._NumpyPerImageWhitening(img) for img in imgs_np]
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       imgs = constant_op.constant(imgs_np)
       whiten = image_ops.per_image_standardization(imgs)
       whiten_tf = self.evaluate(whiten)
@@ -1514,7 +1514,7 @@ class CropToBoundingBoxTest(test_util.TensorFlowTestCase):
     if not use_tensor_inputs:
       self.assertTrue(y.get_shape().is_fully_defined())
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       return y.eval(feed_dict=feed_dict)
 
   def _assertReturns(self,
@@ -1693,7 +1693,7 @@ class CentralCropTest(test_util.TensorFlowTestCase):
     for x_shape in x_shapes:
       x_np = np.ones(x_shape, dtype=np.float32)
       for use_gpu in [True, False]:
-        with self.test_session(use_gpu=use_gpu):
+        with self.cached_session(use_gpu=use_gpu):
           x = constant_op.constant(x_np, shape=x_shape)
           y = image_ops.central_crop(x, 1.0)
           y_tf = self.evaluate(y)
@@ -1708,7 +1708,7 @@ class CentralCropTest(test_util.TensorFlowTestCase):
         dtype=np.int32).reshape(x_shape)
     y_np = np.array([[3, 4, 5, 6], [3, 4, 5, 6]]).reshape([2, 4, 1])
     for use_gpu in [True, False]:
-      with self.test_session(use_gpu=use_gpu):
+      with self.cached_session(use_gpu=use_gpu):
         x = constant_op.constant(x_np, shape=x_shape)
         y = image_ops.central_crop(x, 0.5)
         y_tf = self.evaluate(y)
@@ -1724,7 +1724,7 @@ class CentralCropTest(test_util.TensorFlowTestCase):
         dtype=np.int32).reshape(x_shape)
     y_np = np.array([[[3, 4, 5, 6], [3, 4, 5, 6]],
                      [[6, 5, 4, 3], [6, 5, 4, 3]]]).reshape([2, 2, 4, 1])
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       x = constant_op.constant(x_np, shape=x_shape)
       y = image_ops.central_crop(x, 0.5)
       y_tf = self.evaluate(y)
@@ -1741,7 +1741,7 @@ class CentralCropTest(test_util.TensorFlowTestCase):
       x_np = np.zeros(x_shape, dtype=np.int32)
       y_np = np.zeros(y_shape, dtype=np.int32)
       for use_gpu in [True, False]:
-        with self.test_session(use_gpu=use_gpu):
+        with self.cached_session(use_gpu=use_gpu):
           x = array_ops.placeholder(shape=x_shape, dtype=dtypes.int32)
           y = image_ops.central_crop(x, 0.33)
           y_tf = y.eval(feed_dict={x: x_np})
@@ -1792,7 +1792,7 @@ class CentralCropTest(test_util.TensorFlowTestCase):
     x_shape = [13, 9, 3]
     x_np = np.ones(x_shape, dtype=np.float32)
     for use_gpu in [True, False]:
-      with self.test_session(use_gpu=use_gpu):
+      with self.cached_session(use_gpu=use_gpu):
         x = constant_op.constant(x_np, shape=x_shape)
         with self.assertRaises(ValueError):
           _ = image_ops.central_crop(x, 0.0)
@@ -1804,7 +1804,7 @@ class CentralCropTest(test_util.TensorFlowTestCase):
     for x_shape in x_shapes:
       x_np = np.ones(x_shape, dtype=np.float32)
       for use_gpu in [True, False]:
-        with self.test_session(use_gpu=use_gpu):
+        with self.cached_session(use_gpu=use_gpu):
           x = constant_op.constant(x_np, shape=x_shape)
           with self.assertRaises(ValueError):
             _ = image_ops.central_crop(x, 0.5)
@@ -1814,7 +1814,7 @@ class CentralCropTest(test_util.TensorFlowTestCase):
     x_shape = [13, 9, 3]
     x_np = np.ones(x_shape, dtype=np.float32)
     for use_gpu in [True, False]:
-      with self.test_session(use_gpu=use_gpu):
+      with self.cached_session(use_gpu=use_gpu):
         y = image_ops.central_crop(x_np, 1.0)
         self.assertTrue(y.op.name.startswith("central_crop"))
 
@@ -1839,7 +1839,7 @@ class PadToBoundingBoxTest(test_util.TensorFlowTestCase):
     if not use_tensor_inputs:
       self.assertTrue(y.get_shape().is_fully_defined())
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       return y.eval(feed_dict=feed_dict)
 
   def _assertReturns(self,
@@ -1899,7 +1899,7 @@ class PadToBoundingBoxTest(test_util.TensorFlowTestCase):
 
     i = constant_op.constant([1, 0, 4, 3], dtype=dtypes.int64)
     y_tf = image_ops.pad_to_bounding_box(x, i[0], i[1], i[2], i[3])
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       self.assertAllClose(y, self.evaluate(y_tf))
 
   @test_util.run_deprecated_v1
@@ -2034,7 +2034,7 @@ class SelectDistortedCropBoxTest(test_util.TensorFlowTestCase):
     fraction_object_covered = []
 
     num_iter = 1000
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       image_tf = constant_op.constant(image, shape=image.shape)
       image_size_tf = constant_op.constant(
           image_size_np, shape=image_size_np.shape)
@@ -2164,7 +2164,7 @@ class SelectDistortedCropBoxTest(test_util.TensorFlowTestCase):
 
   @test_util.run_deprecated_v1
   def testSampleDistortedBoundingBoxShape(self):
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       image_size = constant_op.constant(
           [40, 50, 1], shape=[3], dtype=dtypes.int32)
       bounding_box = constant_op.constant(
@@ -2202,7 +2202,7 @@ class SelectDistortedCropBoxTest(test_util.TensorFlowTestCase):
 
   def testDefaultMinObjectCovered(self):
     # By default min_object_covered=0.1 if not provided
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       image_size = constant_op.constant(
           [40, 50, 1], shape=[3], dtype=dtypes.int32)
       bounding_box = constant_op.constant(
@@ -2275,7 +2275,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
       img_np = np.array(data, dtype=nptype).reshape(img_shape)
 
       for opt in self.OPTIONS:
-        with self.test_session(use_gpu=True) as sess:
+        with self.cached_session(use_gpu=True) as sess:
           image = constant_op.constant(img_np, shape=img_shape)
           y = image_ops.resize_images(image, [target_height, target_width], opt)
           yshape = array_ops.shape(y)
@@ -2284,7 +2284,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
           self.assertAllClose(resized, img_np, atol=1e-5)
 
       # Resizing with a single image must leave the shape unchanged also.
-      with self.test_session(use_gpu=True):
+      with self.cached_session(use_gpu=True):
         img_single = img_np.reshape(single_shape)
         image = constant_op.constant(img_single, shape=single_shape)
         y = image_ops.resize_images(image, [target_height, target_width],
@@ -2308,7 +2308,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
     img_np = np.array(data, dtype=np.uint8).reshape(img_shape)
 
     for opt in self.OPTIONS:
-      with self.test_session(use_gpu=True) as sess:
+      with self.cached_session(use_gpu=True) as sess:
         image = constant_op.constant(img_np, shape=img_shape)
         y = image_ops.resize_images(image, new_size, opt)
         yshape = array_ops.shape(y)
@@ -2317,7 +2317,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
         self.assertAllClose(resized, img_np, atol=1e-5)
 
     # Resizing with a single image must leave the shape unchanged also.
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       img_single = img_np.reshape(single_shape)
       image = constant_op.constant(img_single, shape=single_shape)
       y = image_ops.resize_images(image, new_size, self.OPTIONS[0])
@@ -2422,7 +2422,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
 
         for opt in self.OPTIONS:
           if test.is_gpu_available() and self.shouldRunOnGPU(opt, nptype):
-            with self.test_session(use_gpu=True):
+            with self.cached_session(use_gpu=True):
               image = constant_op.constant(img_np, shape=img_shape)
               y = image_ops.resize_images(image, [target_height, target_width],
                                           opt)
@@ -2457,7 +2457,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
           image_ops.ResizeMethod.BILINEAR,
           image_ops.ResizeMethod.NEAREST_NEIGHBOR, image_ops.ResizeMethod.AREA
       ]:
-        with self.test_session(use_gpu=True):
+        with self.cached_session(use_gpu=True):
           img_np = np.array(data, dtype=nptype).reshape(img_shape)
           image = constant_op.constant(img_np, shape=img_shape)
           y = image_ops.resize_images(
@@ -2493,7 +2493,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
           image_ops.ResizeMethod.BILINEAR,
           image_ops.ResizeMethod.NEAREST_NEIGHBOR, image_ops.ResizeMethod.AREA
       ]:
-        with self.test_session(use_gpu=True):
+        with self.cached_session(use_gpu=True):
           img_np = np.array(data, dtype=nptype).reshape(img_shape)
           image = constant_op.constant(img_np, shape=img_shape)
           y = image_ops.resize_images(
@@ -2521,7 +2521,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
         75, 81, 80, 72, 69, 70, 105, 112, 75, 36, 45, 92, 111, 105
     ]
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       image = constant_op.constant(img_np, shape=img_shape)
       y = image_ops.resize_images(image, [target_height, target_width],
                                   image_ops.ResizeMethod.BICUBIC)
@@ -2544,7 +2544,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
         73, 33, 23, 39, 73, 33, 23, 39, 14, 16, 19, 21, 14, 16, 19, 21
     ]
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       image = constant_op.constant(img_np, shape=img_shape)
       y = image_ops.resize_images(image, [target_height, target_width],
                                   image_ops.ResizeMethod.AREA)
@@ -2562,7 +2562,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
         for align_corners in [True, False]:
           img_np = np.arange(
               0, np.prod(input_shape), dtype=nptype).reshape(input_shape)
-          with self.test_session(use_gpu=True):
+          with self.cached_session(use_gpu=True):
             image = constant_op.constant(img_np, shape=input_shape)
             new_size = constant_op.constant([target_height, target_width])
             out_op = image_ops.resize_images(
@@ -2571,7 +2571,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
                 image_ops.ResizeMethod.NEAREST_NEIGHBOR,
                 align_corners=align_corners)
             gpu_val = self.evaluate(out_op)
-          with self.test_session(use_gpu=False):
+          with self.cached_session(use_gpu=False):
             image = constant_op.constant(img_np, shape=input_shape)
             new_size = constant_op.constant([target_height, target_width])
             out_op = image_ops.resize_images(
@@ -2593,7 +2593,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
               0, np.prod(input_shape), dtype=nptype).reshape(input_shape)
           value = {}
           for use_gpu in [True, False]:
-            with self.test_session(use_gpu=use_gpu):
+            with self.cached_session(use_gpu=use_gpu):
               image = constant_op.constant(img_np, shape=input_shape)
               new_size = constant_op.constant([target_height, target_width])
               out_op = image_ops.resize_images(
@@ -2628,7 +2628,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
   @test_util.run_deprecated_v1
   def testNameScope(self):
     img_shape = [1, 3, 2, 1]
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       single_image = array_ops.placeholder(dtypes.float32, shape=[50, 60, 3])
       y = image_ops.resize_images(single_image, [55, 66])
       self.assertTrue(y.op.name.startswith("resize"))
@@ -2647,7 +2647,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
     y = image_ops.resize_images(x_tensor, target_max,
                                 preserve_aspect_ratio=preserve_aspect_ratio)
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       return y.eval(feed_dict=feed_dict)
 
   def _assertResizeEqual(self, x, x_shape, y, y_shape,
@@ -2745,7 +2745,7 @@ class ResizeImageWithPadTest(test_util.TensorFlowTestCase):
     if not use_tensor_inputs:
       self.assertTrue(y.get_shape().is_fully_defined())
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       return y.eval(feed_dict=feed_dict)
 
   def _assertReturns(self,
@@ -2843,7 +2843,7 @@ class ResizeImageWithCropOrPadTest(test_util.TensorFlowTestCase):
     if not use_tensor_inputs:
       self.assertTrue(y.get_shape().is_fully_defined())
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       return y.eval(feed_dict=feed_dict)
 
   def _assertReturns(self,
@@ -3098,7 +3098,7 @@ class JpegTest(test_util.TensorFlowTestCase):
     # Read a real jpeg and verify shape
     path = ("tensorflow/core/lib/jpeg/testdata/"
             "jpeg_merge_test1.jpg")
-    with self.test_session(use_gpu=True) as sess:
+    with self.cached_session(use_gpu=True) as sess:
       jpeg0 = io_ops.read_file(path)
       image0 = image_ops.decode_jpeg(jpeg0)
       image1 = image_ops.decode_jpeg(image_ops.encode_jpeg(image0))
@@ -3114,7 +3114,7 @@ class JpegTest(test_util.TensorFlowTestCase):
     cmyk_path = os.path.join(base, "jpeg_merge_test1_cmyk.jpg")
     shape = 256, 128, 3
     for channels in 3, 0:
-      with self.test_session(use_gpu=True) as sess:
+      with self.cached_session(use_gpu=True) as sess:
         rgb = image_ops.decode_jpeg(
             io_ops.read_file(rgb_path), channels=channels)
         cmyk = image_ops.decode_jpeg(
@@ -3171,7 +3171,7 @@ class JpegTest(test_util.TensorFlowTestCase):
           self.evaluate(result)
 
   def testSynthetic(self):
-    with self.test_session(use_gpu=True) as sess:
+    with self.cached_session(use_gpu=True) as sess:
       # Encode it, then decode it, then encode it
       image0 = constant_op.constant(_SimpleColorRamp())
       jpeg0 = image_ops.encode_jpeg(image0)
@@ -3192,7 +3192,7 @@ class JpegTest(test_util.TensorFlowTestCase):
       self.assertLessEqual(len(jpeg0), 6000)
 
   def testSyntheticFasterAlgorithm(self):
-    with self.test_session(use_gpu=True) as sess:
+    with self.cached_session(use_gpu=True) as sess:
       # Encode it, then decode it, then encode it
       image0 = constant_op.constant(_SimpleColorRamp())
       jpeg0 = image_ops.encode_jpeg(image0)
@@ -3216,7 +3216,7 @@ class JpegTest(test_util.TensorFlowTestCase):
       self.assertLessEqual(len(jpeg0), 6000)
 
   def testDefaultDCTMethodIsIntegerFast(self):
-    with self.test_session(use_gpu=True) as sess:
+    with self.cached_session(use_gpu=True) as sess:
       # Compare decoding with both dct_option=INTEGER_FAST and
       # default.  They should be the same.
       image0 = constant_op.constant(_SimpleColorRamp())
@@ -3230,7 +3230,7 @@ class JpegTest(test_util.TensorFlowTestCase):
 
   @test_util.run_deprecated_v1
   def testShape(self):
-    with self.test_session(use_gpu=True) as sess:
+    with self.cached_session(use_gpu=True) as sess:
       jpeg = constant_op.constant("nonsense")
       for channels in 0, 1, 3:
         image = image_ops.decode_jpeg(jpeg, channels=channels)
@@ -3242,7 +3242,7 @@ class JpegTest(test_util.TensorFlowTestCase):
     # Read a real jpeg and verify shape.
     path = ("tensorflow/core/lib/jpeg/testdata/"
             "jpeg_merge_test1.jpg")
-    with self.test_session(use_gpu=True) as sess:
+    with self.cached_session(use_gpu=True) as sess:
       jpeg = io_ops.read_file(path)
       # Extract shape without decoding.
       [image_shape] = sess.run([image_ops.extract_jpeg_shape(jpeg)])
@@ -3253,7 +3253,7 @@ class JpegTest(test_util.TensorFlowTestCase):
     # Read a cmyk jpeg image, and verify its shape.
     path = ("tensorflow/core/lib/jpeg/testdata/"
             "jpeg_merge_test1_cmyk.jpg")
-    with self.test_session(use_gpu=True) as sess:
+    with self.cached_session(use_gpu=True) as sess:
       jpeg = io_ops.read_file(path)
       [image_shape] = sess.run([image_ops.extract_jpeg_shape(jpeg)])
       # Cmyk jpeg image has 4 channels.
@@ -3269,7 +3269,7 @@ class PngTest(test_util.TensorFlowTestCase):
               (3, "lena_palette.png"), (4, "lena_palette_trns.png"))
     for channels_in, filename in inputs:
       for channels in 0, 1, 3, 4:
-        with self.test_session(use_gpu=True) as sess:
+        with self.cached_session(use_gpu=True) as sess:
           png0 = io_ops.read_file(prefix + filename)
           image0 = image_ops.decode_png(png0, channels=channels)
           png0, image0 = self.evaluate([png0, image0])
@@ -3279,7 +3279,7 @@ class PngTest(test_util.TensorFlowTestCase):
             self.assertAllEqual(image0, self.evaluate(image1))
 
   def testSynthetic(self):
-    with self.test_session(use_gpu=True) as sess:
+    with self.cached_session(use_gpu=True) as sess:
       # Encode it, then decode it
       image0 = constant_op.constant(_SimpleColorRamp())
       png0 = image_ops.encode_png(image0, compression=7)
@@ -3294,7 +3294,7 @@ class PngTest(test_util.TensorFlowTestCase):
       self.assertLessEqual(len(png0), 750)
 
   def testSyntheticUint16(self):
-    with self.test_session(use_gpu=True) as sess:
+    with self.cached_session(use_gpu=True) as sess:
       # Encode it, then decode it
       image0 = constant_op.constant(_SimpleColorRamp(), dtype=dtypes.uint16)
       png0 = image_ops.encode_png(image0, compression=7)
@@ -3309,7 +3309,7 @@ class PngTest(test_util.TensorFlowTestCase):
       self.assertLessEqual(len(png0), 1500)
 
   def testSyntheticTwoChannel(self):
-    with self.test_session(use_gpu=True) as sess:
+    with self.cached_session(use_gpu=True) as sess:
       # Strip the b channel from an rgb image to get a two-channel image.
       gray_alpha = _SimpleColorRamp()[:, :, 0:2]
       image0 = constant_op.constant(gray_alpha)
@@ -3320,7 +3320,7 @@ class PngTest(test_util.TensorFlowTestCase):
       self.assertAllEqual(image0, image1)
 
   def testSyntheticTwoChannelUint16(self):
-    with self.test_session(use_gpu=True) as sess:
+    with self.cached_session(use_gpu=True) as sess:
       # Strip the b channel from an rgb image to get a two-channel image.
       gray_alpha = _SimpleColorRamp()[:, :, 0:2]
       image0 = constant_op.constant(gray_alpha, dtype=dtypes.uint16)
@@ -3332,7 +3332,7 @@ class PngTest(test_util.TensorFlowTestCase):
 
   @test_util.run_deprecated_v1
   def testShape(self):
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       png = constant_op.constant("nonsense")
       for channels in 0, 1, 3:
         image = image_ops.decode_png(png, channels=channels)
@@ -3350,7 +3350,7 @@ class GifTest(test_util.TensorFlowTestCase):
     STRIDE = 5
     shape = (12, HEIGHT, WIDTH, 3)
 
-    with self.test_session(use_gpu=True) as sess:
+    with self.cached_session(use_gpu=True) as sess:
       gif0 = io_ops.read_file(prefix + filename)
       image0 = image_ops.decode_gif(gif0)
       gif0, image0 = self.evaluate([gif0, image0])
@@ -3377,7 +3377,7 @@ class GifTest(test_util.TensorFlowTestCase):
 
   @test_util.run_deprecated_v1
   def testShape(self):
-    with self.test_session(use_gpu=True) as sess:
+    with self.cached_session(use_gpu=True) as sess:
       gif = constant_op.constant("nonsense")
       image = image_ops.decode_gif(gif)
       self.assertEqual(image.get_shape().as_list(), [None, None, None, 3])
@@ -3389,7 +3389,7 @@ class ConvertImageTest(test_util.TensorFlowTestCase):
     x_np = np.array(original, dtype=original_dtype.as_numpy_dtype())
     y_np = np.array(expected, dtype=output_dtype.as_numpy_dtype())
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       image = constant_op.constant(x_np)
       y = image_ops.convert_image_dtype(image, output_dtype)
       self.assertTrue(y.dtype == output_dtype)
@@ -3405,7 +3405,7 @@ class ConvertImageTest(test_util.TensorFlowTestCase):
   @test_util.run_deprecated_v1
   def testNoConvert(self):
     # Make sure converting to the same data type creates only an identity op
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       image = constant_op.constant([1], dtype=dtypes.uint8)
       image_ops.convert_image_dtype(image, dtypes.uint8)
       y = image_ops.convert_image_dtype(image, dtypes.uint8)
@@ -3415,7 +3415,7 @@ class ConvertImageTest(test_util.TensorFlowTestCase):
   @test_util.run_deprecated_v1
   def testConvertBetweenInteger(self):
     # Make sure converting to between integer types scales appropriately
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       self._convert([0, 255], dtypes.uint8, dtypes.int16, [0, 255 * 128])
       self._convert([0, 32767], dtypes.int16, dtypes.uint8, [0, 255])
       self._convert([0, 2**32], dtypes.int64, dtypes.int32, [0, 1])
@@ -3424,7 +3424,7 @@ class ConvertImageTest(test_util.TensorFlowTestCase):
   @test_util.run_deprecated_v1
   def testConvertBetweenFloat(self):
     # Make sure converting to between float types does nothing interesting
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       self._convert([-1.0, 0, 1.0, 200000], dtypes.float32, dtypes.float64,
                     [-1.0, 0, 1.0, 200000])
       self._convert([-1.0, 0, 1.0, 200000], dtypes.float64, dtypes.float32,
@@ -3433,7 +3433,7 @@ class ConvertImageTest(test_util.TensorFlowTestCase):
   @test_util.run_deprecated_v1
   def testConvertBetweenIntegerAndFloat(self):
     # Make sure converting from and to a float type scales appropriately
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       self._convert([0, 1, 255], dtypes.uint8, dtypes.float32,
                     [0, 1.0 / 255.0, 1])
       self._convert([0, 1.1 / 255.0, 1], dtypes.float32, dtypes.uint8,
@@ -3441,7 +3441,7 @@ class ConvertImageTest(test_util.TensorFlowTestCase):
 
   @test_util.run_deprecated_v1
   def testConvertBetweenInt16AndInt8(self):
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       # uint8, uint16
       self._convert([0, 255 * 256], dtypes.uint16, dtypes.uint8, [0, 255])
       self._convert([0, 255], dtypes.uint8, dtypes.uint16, [0, 255 * 256])
@@ -3472,7 +3472,7 @@ class TotalVariationTest(test_util.TensorFlowTestCase):
     """
 
     # Create a TensorFlow session.
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       # Add a constant to the TensorFlow graph that holds the input.
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
 
@@ -3860,7 +3860,7 @@ class VerifyCompatibleImageShapesTest(test_util.TensorFlowTestCase):
     img = array_ops.placeholder(dtype=dtypes.float32)
     img_np = np.array((2, 2))
 
-    with self.test_session(use_gpu=True) as sess:
+    with self.cached_session(use_gpu=True) as sess:
       _, _, checks = image_ops_impl._verify_compatible_image_shapes(img, img)
       with self.assertRaises(errors.InvalidArgumentError):
         sess.run(checks, {img: img_np})
@@ -3873,7 +3873,7 @@ class VerifyCompatibleImageShapesTest(test_util.TensorFlowTestCase):
     img1_np = np.array([1, 2, 2, 1])
     img2_np = np.array([1, 3, 3, 1])
 
-    with self.test_session(use_gpu=True) as sess:
+    with self.cached_session(use_gpu=True) as sess:
       _, _, checks = image_ops_impl._verify_compatible_image_shapes(img1, img2)
       with self.assertRaises(errors.InvalidArgumentError):
         sess.run(checks, {img1: img1_np, img2: img2_np})
@@ -3891,7 +3891,7 @@ class PSNRTest(test_util.TensorFlowTestCase):
     return np.expand_dims(im, axis=0)
 
   def _LoadTestImages(self):
-    with self.test_session(use_gpu=True) as sess:
+    with self.cached_session(use_gpu=True) as sess:
       q20 = self._LoadTestImage(sess, "cat_q20.jpg")
       q72 = self._LoadTestImage(sess, "cat_q72.jpg")
       q95 = self._LoadTestImage(sess, "cat_q95.jpg")
@@ -3912,7 +3912,7 @@ class PSNRTest(test_util.TensorFlowTestCase):
     image2 = self._RandomImage((8, 8, 1), 1)
     psnr = self._PSNR_NumPy(image1, image2, 1)
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       tf_image1 = constant_op.constant(image1, shape=image1.shape,
                                        dtype=dtypes.float32)
       tf_image2 = constant_op.constant(image2, shape=image2.shape,
@@ -3926,7 +3926,7 @@ class PSNRTest(test_util.TensorFlowTestCase):
     image2 = self._RandomImage((10, 8, 8, 1), 1)
     psnr = self._PSNR_NumPy(image1, image2, 1)
 
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       tf_image1 = constant_op.constant(image1, shape=image1.shape,
                                        dtype=dtypes.float32)
       tf_image2 = constant_op.constant(image2, shape=image2.shape,
@@ -3948,7 +3948,7 @@ class PSNRTest(test_util.TensorFlowTestCase):
     self.assertNear(35.302, psnr3, 0.001)
 
     # Test TensorFlow implementation.
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       tf_q20 = constant_op.constant(q20, shape=q20.shape, dtype=dtypes.float32)
       tf_q72 = constant_op.constant(q72, shape=q72.shape, dtype=dtypes.float32)
       tf_q95 = constant_op.constant(q95, shape=q95.shape, dtype=dtypes.float32)
@@ -3963,7 +3963,7 @@ class PSNRTest(test_util.TensorFlowTestCase):
   def testInfinity(self):
     q20, _, _ = self._LoadTestImages()
     psnr = self._PSNR_NumPy(q20, q20, 1)
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       tf_q20 = constant_op.constant(q20, shape=q20.shape, dtype=dtypes.float32)
       tf_psnr = image_ops.psnr(tf_q20, tf_q20, 1, "psnr").eval()
       self.assertAllClose(psnr, tf_psnr, atol=0.001)
@@ -3978,7 +3978,7 @@ class PSNRTest(test_util.TensorFlowTestCase):
     img1 = image_ops.convert_image_dtype(img1, dtypes.float32)
     img2 = image_ops.convert_image_dtype(img2, dtypes.float32)
     psnr_float32 = image_ops.psnr(img1, img2, 1.0)
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       self.assertAllClose(
           psnr_uint8.eval(), self.evaluate(psnr_float32), atol=0.001)
 
@@ -4003,7 +4003,7 @@ class SSIMTest(test_util.TensorFlowTestCase):
     return np.expand_dims(im, axis=0)
 
   def _LoadTestImages(self):
-    with self.test_session(use_gpu=True) as sess:
+    with self.cached_session(use_gpu=True) as sess:
       return [self._LoadTestImage(sess, f) for f in self._filenames]
 
   def _RandomImage(self, shape, max_val):
@@ -4018,7 +4018,7 @@ class SSIMTest(test_util.TensorFlowTestCase):
 
     ph = [array_ops.placeholder(dtype=dtypes.float32) for _ in range(2)]
     ssim = image_ops.ssim(*ph, max_val=1.0)
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       scores = [ssim.eval(dict(zip(ph, t)))
                 for t in itertools.combinations_with_replacement(img, 2)]
     self.assertAllClose(expected, np.squeeze(scores), atol=1e-4)
@@ -4033,7 +4033,7 @@ class SSIMTest(test_util.TensorFlowTestCase):
 
     ssim = image_ops.ssim(constant_op.constant(img1),
                           constant_op.constant(img2), 1.0)
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       self.assertAllClose(expected, self.evaluate(ssim), atol=1e-4)
 
   def testBroadcast(self):
@@ -4045,7 +4045,7 @@ class SSIMTest(test_util.TensorFlowTestCase):
     img2 = array_ops.expand_dims(img, axis=1)  # batch dims: 2, 1.
 
     ssim = image_ops.ssim(img1, img2, 1.0)
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       self.assertAllClose(expected, self.evaluate(ssim), atol=1e-4)
 
   @test_util.run_deprecated_v1
@@ -4060,7 +4060,7 @@ class SSIMTest(test_util.TensorFlowTestCase):
 
     ssim = image_ops.ssim(constant_op.constant(img1),
                           constant_op.constant(img2), 255)
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       self.assertLess(ssim.eval(), 0)
 
   @test_util.run_deprecated_v1
@@ -4073,7 +4073,7 @@ class SSIMTest(test_util.TensorFlowTestCase):
     img1 = image_ops.convert_image_dtype(img1, dtypes.float32)
     img2 = image_ops.convert_image_dtype(img2, dtypes.float32)
     ssim_float32 = image_ops.ssim(img1, img2, 1.0)
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       self.assertAllClose(
           ssim_uint8.eval(), self.evaluate(ssim_float32), atol=0.001)
 
@@ -4098,7 +4098,7 @@ class MultiscaleSSIMTest(test_util.TensorFlowTestCase):
     return np.expand_dims(im, axis=0)
 
   def _LoadTestImages(self):
-    with self.test_session(use_gpu=True) as sess:
+    with self.cached_session(use_gpu=True) as sess:
       return [self._LoadTestImage(sess, f) for f in self._filenames]
 
   def _RandomImage(self, shape, max_val):
@@ -4116,7 +4116,7 @@ class MultiscaleSSIMTest(test_util.TensorFlowTestCase):
 
     ph = [array_ops.placeholder(dtype=dtypes.float32) for _ in range(2)]
     msssim = image_ops.ssim_multiscale(*ph, max_val=1.0)
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       scores = [msssim.eval(dict(zip(ph, t)))
                 for t in itertools.combinations_with_replacement(img, 2)]
 
@@ -4131,7 +4131,7 @@ class MultiscaleSSIMTest(test_util.TensorFlowTestCase):
     msssim = image_ops.ssim_multiscale(*scaled_ph, max_val=1.0,
                                        power_factors=(1, 1, 1, 1, 1))
     grads = gradients.gradients(msssim, scalar)
-    with self.test_session(use_gpu=True) as sess:
+    with self.cached_session(use_gpu=True) as sess:
       np_grads = sess.run(grads, feed_dict={ph[0]: img[0], ph[1]: img[1]})
     self.assertTrue(np.isfinite(np_grads).all())
 
@@ -4146,7 +4146,7 @@ class MultiscaleSSIMTest(test_util.TensorFlowTestCase):
 
     msssim = image_ops.ssim_multiscale(constant_op.constant(img1),
                                        constant_op.constant(img2), 1.0)
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       self.assertAllClose(expected, self.evaluate(msssim), 1e-4)
 
   def testBroadcast(self):
@@ -4159,7 +4159,7 @@ class MultiscaleSSIMTest(test_util.TensorFlowTestCase):
     img2 = array_ops.expand_dims(img, axis=1)  # batch dims: 2, 1.
 
     score_tensor = image_ops.ssim_multiscale(img1, img2, 1.0)
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       self.assertAllClose(expected, self.evaluate(score_tensor), 1e-4)
 
   def testRange(self):
@@ -4169,7 +4169,7 @@ class MultiscaleSSIMTest(test_util.TensorFlowTestCase):
     If any of the value is negative so that the geometric mean is not
     well-defined, then treat the MS-SSIM score as zero.
     """
-    with self.test_session(use_gpu=True) as sess:
+    with self.cached_session(use_gpu=True) as sess:
       img1 = self._LoadTestImage(sess, "checkerboard1.png")
       img2 = self._LoadTestImage(sess, "checkerboard3.png")
       images = [img1, img2, np.zeros_like(img1),
@@ -4194,7 +4194,7 @@ class MultiscaleSSIMTest(test_util.TensorFlowTestCase):
     img1 = image_ops.convert_image_dtype(img1, dtypes.float32)
     img2 = image_ops.convert_image_dtype(img2, dtypes.float32)
     ssim_float32 = image_ops.ssim_multiscale(img1, img2, 1.0)
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       self.assertAllClose(
           ssim_uint8.eval(), self.evaluate(ssim_float32), atol=0.001)
 
@@ -4235,7 +4235,7 @@ class ImageGradientsTest(test_util.TensorFlowTestCase):
     batch = constant_op.constant(batch)
     assert batch.get_shape().as_list() == [2, 2, 3, 2]
     dy, dx = image_ops.image_gradients(batch)
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       actual_dy = self.evaluate(dy)
       actual_dx = self.evaluate(dx)
       self.assertAllClose(expected_dy, actual_dy)
@@ -4256,7 +4256,7 @@ class SobelEdgesTest(test_util.TensorFlowTestCase):
     expected = np.reshape([[[0, 0], [0, 12], [0, 0]],
                            [[0, 0], [0, 12], [0, 0]]], [1, 2, 3, 1, 2])
     sobel = image_ops.sobel_edges(img)
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       actual_sobel = self.evaluate(sobel)
       self.assertAllClose(expected, actual_sobel)
 
@@ -4278,7 +4278,7 @@ class SobelEdgesTest(test_util.TensorFlowTestCase):
     expected_batch = np.concatenate([expected_two_channel] * batch_size, axis=0)
 
     sobel = image_ops.sobel_edges(img)
-    with self.test_session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       actual_sobel = self.evaluate(sobel)
       self.assertAllClose(expected_batch, actual_sobel)
 
@@ -4286,7 +4286,7 @@ class SobelEdgesTest(test_util.TensorFlowTestCase):
 class DecodeImageTest(test_util.TensorFlowTestCase):
 
   def testJpegUint16(self):
-    with self.test_session(use_gpu=True) as sess:
+    with self.cached_session(use_gpu=True) as sess:
       base = "tensorflow/core/lib/jpeg/testdata"
       jpeg0 = io_ops.read_file(os.path.join(base, "jpeg_merge_test1.jpg"))
       image0 = image_ops.decode_image(jpeg0, dtype=dtypes.uint16)
@@ -4296,7 +4296,7 @@ class DecodeImageTest(test_util.TensorFlowTestCase):
       self.assertAllEqual(image0, image1)
 
   def testPngUint16(self):
-    with self.test_session(use_gpu=True) as sess:
+    with self.cached_session(use_gpu=True) as sess:
       base = "tensorflow/core/lib/png/testdata"
       png0 = io_ops.read_file(os.path.join(base, "lena_rgba.png"))
       image0 = image_ops.decode_image(png0, dtype=dtypes.uint16)
@@ -4306,7 +4306,7 @@ class DecodeImageTest(test_util.TensorFlowTestCase):
       self.assertAllEqual(image0, image1)
 
   def testGifUint16(self):
-    with self.test_session(use_gpu=True) as sess:
+    with self.cached_session(use_gpu=True) as sess:
       base = "tensorflow/core/lib/gif/testdata"
       gif0 = io_ops.read_file(os.path.join(base, "scan.gif"))
       image0 = image_ops.decode_image(gif0, dtype=dtypes.uint16)
@@ -4316,7 +4316,7 @@ class DecodeImageTest(test_util.TensorFlowTestCase):
       self.assertAllEqual(image0, image1)
 
   def testBmpUint16(self):
-    with self.test_session(use_gpu=True) as sess:
+    with self.cached_session(use_gpu=True) as sess:
       base = "tensorflow/core/lib/bmp/testdata"
       bmp0 = io_ops.read_file(os.path.join(base, "lena.bmp"))
       image0 = image_ops.decode_image(bmp0, dtype=dtypes.uint16)
@@ -4326,7 +4326,7 @@ class DecodeImageTest(test_util.TensorFlowTestCase):
       self.assertAllEqual(image0, image1)
 
   def testJpegFloat32(self):
-    with self.test_session(use_gpu=True) as sess:
+    with self.cached_session(use_gpu=True) as sess:
       base = "tensorflow/core/lib/jpeg/testdata"
       jpeg0 = io_ops.read_file(os.path.join(base, "jpeg_merge_test1.jpg"))
       image0 = image_ops.decode_image(jpeg0, dtype=dtypes.float32)
@@ -4336,7 +4336,7 @@ class DecodeImageTest(test_util.TensorFlowTestCase):
       self.assertAllEqual(image0, image1)
 
   def testPngFloat32(self):
-    with self.test_session(use_gpu=True) as sess:
+    with self.cached_session(use_gpu=True) as sess:
       base = "tensorflow/core/lib/png/testdata"
       png0 = io_ops.read_file(os.path.join(base, "lena_rgba.png"))
       image0 = image_ops.decode_image(png0, dtype=dtypes.float32)
@@ -4346,7 +4346,7 @@ class DecodeImageTest(test_util.TensorFlowTestCase):
       self.assertAllEqual(image0, image1)
 
   def testGifFloat32(self):
-    with self.test_session(use_gpu=True) as sess:
+    with self.cached_session(use_gpu=True) as sess:
       base = "tensorflow/core/lib/gif/testdata"
       gif0 = io_ops.read_file(os.path.join(base, "scan.gif"))
       image0 = image_ops.decode_image(gif0, dtype=dtypes.float32)
@@ -4356,7 +4356,7 @@ class DecodeImageTest(test_util.TensorFlowTestCase):
       self.assertAllEqual(image0, image1)
 
   def testBmpFloat32(self):
-    with self.test_session(use_gpu=True) as sess:
+    with self.cached_session(use_gpu=True) as sess:
       base = "tensorflow/core/lib/bmp/testdata"
       bmp0 = io_ops.read_file(os.path.join(base, "lena.bmp"))
       image0 = image_ops.decode_image(bmp0, dtype=dtypes.float32)
-- 
GitLab


From 7f67f065ca32b055d27d42b99f2fe74aa5a1ce4d Mon Sep 17 00:00:00 2001
From: Shanqing Cai <cais@google.com>
Date: Sat, 26 Jan 2019 18:37:21 -0800
Subject: [PATCH 1344/2345] Keras hdf5_format.py: Trivial punctuation fix in
 error message

PiperOrigin-RevId: 231080058
---
 tensorflow/python/keras/saving/hdf5_format.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/saving/hdf5_format.py b/tensorflow/python/keras/saving/hdf5_format.py
index 29fb6a4752..973cdcac8a 100644
--- a/tensorflow/python/keras/saving/hdf5_format.py
+++ b/tensorflow/python/keras/saving/hdf5_format.py
@@ -115,7 +115,7 @@ def save_model(model, filepath, overwrite=True, include_optimizer=True):
             'optimizer attributes or optimizer state '
             'after instantiation. '
             'As a result, we cannot save the optimizer '
-            'as part of the model save file.'
+            'as part of the model save file. '
             'You will have to compile your model again after loading it. '
             'Prefer using a Keras optimizer instead '
             '(see keras.io/optimizers).')
-- 
GitLab


From 9eca1a045a8233958e7e8be494ed8e0ab5a712d6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 26 Jan 2019 20:31:04 -0800
Subject: [PATCH 1345/2345] Remove decorator transformer in preparation for a
 more comprehensive solution.

PiperOrigin-RevId: 231084903
---
 tensorflow/python/autograph/converters/BUILD  |  16 --
 .../python/autograph/converters/call_trees.py |   9 +-
 .../python/autograph/converters/decorators.py | 108 -------------
 .../autograph/converters/decorators_test.py   | 148 ------------------
 tensorflow/python/autograph/core/converter.py |   1 -
 .../autograph/core/converter_testing.py       |   5 -
 .../python/autograph/impl/conversion.py       |   4 -
 ...flow.autograph.experimental.-feature.pbtxt |   4 -
 ...flow.autograph.experimental.-feature.pbtxt |   4 -
 9 files changed, 1 insertion(+), 298 deletions(-)
 delete mode 100644 tensorflow/python/autograph/converters/decorators.py
 delete mode 100644 tensorflow/python/autograph/converters/decorators_test.py

diff --git a/tensorflow/python/autograph/converters/BUILD b/tensorflow/python/autograph/converters/BUILD
index 3ac446db02..bafc5b0ca7 100644
--- a/tensorflow/python/autograph/converters/BUILD
+++ b/tensorflow/python/autograph/converters/BUILD
@@ -25,7 +25,6 @@ py_library(
         "conditional_expressions.py",
         "continue_statements.py",
         "control_flow.py",
-        "decorators.py",
         "directives.py",
         "error_handlers.py",
         "function_scopes.py",
@@ -139,21 +138,6 @@ py_test(
     ],
 )
 
-py_test(
-    name = "decorators_test",
-    srcs = ["decorators_test.py"],
-    srcs_version = "PY2AND3",
-    tags = [
-        "no_pip",
-        "no_windows",
-    ],
-    deps = [
-        ":converters",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python/autograph/core:test_lib",
-    ],
-)
-
 py_test(
     name = "directives_test",
     srcs = ["directives_test.py"],
diff --git a/tensorflow/python/autograph/converters/call_trees.py b/tensorflow/python/autograph/converters/call_trees.py
index d4eb17e976..7026a162a2 100644
--- a/tensorflow/python/autograph/converters/call_trees.py
+++ b/tensorflow/python/autograph/converters/call_trees.py
@@ -268,17 +268,10 @@ class CallTreeTransformer(converter.Base):
     new_call.keywords = node.keywords
     return new_call
 
-  def _visit_decorators(self, decorator_list):
-    if not self.ctx.program.options.uses(converter.Feature.DECORATORS):
-      # When not processing decorators, strip everything that is encountered.
-      return []
-
-    return self.visit_block(decorator_list)
-
   def visit_FunctionDef(self, node):
     node.args = self.visit(node.args)
     node.body = self.visit_block(node.body)
-    node.decorator_list = self._visit_decorators(node.decorator_list)
+    node.decorator_list = []
     node.returns = self.visit_block(node.returns)
     return node
 
diff --git a/tensorflow/python/autograph/converters/decorators.py b/tensorflow/python/autograph/converters/decorators.py
deleted file mode 100644
index f0ea512774..0000000000
--- a/tensorflow/python/autograph/converters/decorators.py
+++ /dev/null
@@ -1,108 +0,0 @@
-# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Handles decorators.
-
-Note: this module only deals with functions whose decorators are still recorded
-in the AST. This does not always happen. See the unit test for an example.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import gast
-
-from tensorflow.python.autograph.core import converter
-from tensorflow.python.autograph.pyct import anno
-from tensorflow.python.util import tf_inspect
-
-
-class DecoratorsTransformer(converter.Base):
-  """Converts or removes decorators."""
-
-  def visit_FunctionDef(self, node):
-    self.generic_visit(node)
-    kept_decorators = []
-    for dec in node.decorator_list:
-      if isinstance(dec, gast.Call):
-        dec_func = dec.func
-      else:
-        dec_func = dec
-
-      # Special cases.
-      # TODO(mdan): Is there any way we can treat these more generically?
-      # We may want to forego using decorators altogether if we can't
-      # properly support them.
-      if isinstance(dec_func, gast.Name) and dec_func.id in ('classmethod',):
-        # Assumption: decorators are only visible in the AST when converting
-        # a function inline (via another decorator).
-        # In that case, the converted function is no longer part of the
-        # original object that it was declared into.
-        # This is currently verified by tests.
-        continue
-
-      if not anno.hasanno(dec_func, 'live_val'):
-        raise ValueError('could not resolve the decorator "@%s"' %
-                         (anno.getanno(dec_func, anno.Basic.QN)))
-
-      original_dec = anno.getanno(dec_func, anno.Basic.QN)
-      dec_value = anno.getanno(dec_func, 'live_val')
-
-      if dec_value in self.ctx.program.options.strip_decorators:
-        continue
-
-      # When using foo.bar.baz, we only really need to grab foo and import
-      # that.
-      dec_support_node = dec_func
-      while isinstance(dec_support_node, gast.Attribute):
-        dec_support_node = dec_support_node.value
-
-      if not anno.hasanno(dec_support_node, 'live_val'):
-        raise ValueError(
-            'could not resolve symbol "%s" when looking up decorator "%s"' %
-            (anno.getanno(dec_support_node, anno.Basic.QN), original_dec))
-
-      dec_support = anno.getanno(dec_support_node, 'live_val')
-      # The tuple contains:
-      #  * the AST that represents the decorator
-      #  * the entity supporting the decorator (i.e., what we need to import)
-      #  * the name of the module that needs to be imported for this decorator
-      #    to properly resolve.
-      # Examples:
-      #  for foo.bar, the tuple is (<ast>, <module foo>, 'foo')
-      #  for baz, the tuple is (<ast>, <module baz.__module__>, 'baz')
-      kept_decorators.append((dec, dec_support,
-                              anno.getanno(dec_support_node, anno.Basic.QN)))
-
-    for _, dec_support, name in kept_decorators:
-      if tf_inspect.ismodule(dec_support):
-        self.ctx.program.additional_imports.add(
-            'import %s as %s' % (dec_support.__name__, name))
-      else:
-        if dec_support.__module__ == '__main__':
-          raise ValueError(
-              'decorator "%s" was not allowed because it is declared '
-              'in the module "%s". To fix this, declare it in a separate '
-              'module that we can import it from.' % (dec_support,
-                                                      dec_support.__module__))
-        self.ctx.program.additional_imports.add(
-            'from %s import %s' % (dec_support.__module__, name))
-
-    node.decorator_list = [dec for dec, _, _ in kept_decorators]
-    return node
-
-
-def transform(node, ctx):
-  return DecoratorsTransformer(ctx).visit(node)
diff --git a/tensorflow/python/autograph/converters/decorators_test.py b/tensorflow/python/autograph/converters/decorators_test.py
deleted file mode 100644
index d38975931b..0000000000
--- a/tensorflow/python/autograph/converters/decorators_test.py
+++ /dev/null
@@ -1,148 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for decorators module."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from functools import wraps
-import imp
-
-from tensorflow.python import autograph
-from tensorflow.python.autograph.converters import decorators
-from tensorflow.python.autograph.core import converter_testing
-from tensorflow.python.autograph.pyct import compiler
-from tensorflow.python.platform import test
-
-
-# The Python parser only briefly captures decorators into the AST.
-# The interpreter desugars them on load, and the decorated function loses any
-# trace of the decorator (which is normally what you would expect, since
-# they are meant to be transparent).
-# However, decorators are still visible when you analyze the function
-# from inside a decorator, before it was applied - as is the case
-# with our conversion decorators.
-
-
-def simple_decorator(f):
-  return lambda a: f(a) + 1
-
-
-def self_transform_decorator(transform):
-
-  def decorator(f):
-    @wraps(f)
-    def wrapper(*args):
-      # This removing wrapper is defined in the test below. This setup is so
-      # intricate in order to simulate how we use the transformer in practice.
-      transformed_f = transform(f, (self_transform_decorator,))
-      return transformed_f(*args) + 1
-    return wrapper
-  return decorator
-
-
-class DecoratorsTest(converter_testing.TestCase):
-
-  def _transform(self, f, strip_decorators):
-    namespace = {
-        'self_transform_decorator': self_transform_decorator,
-        'simple_decorator': simple_decorator,
-        'converter_testing': converter_testing,
-    }
-    node, ctx = self.prepare(
-        f, namespace, recursive=False, strip_decorators=strip_decorators)
-    node = decorators.transform(node, ctx)
-    import_line = '\n'.join(ctx.program.additional_imports)
-    result, _ = compiler.ast_to_object(node, source_prefix=import_line)
-    return getattr(result, f.__name__)
-
-  def test_noop(self):
-
-    def test_fn(a):
-      return a
-
-    with self.converted(test_fn, decorators, {}) as result:
-      self.assertEqual(1, result.test_fn(1))
-
-  def test_function(self):
-
-    @self_transform_decorator(self._transform)
-    def test_fn(a):
-      return a
-
-    # 2 = 1 (a) + 1 (decorator applied exactly once)
-    self.assertEqual(2, test_fn(1))
-
-  def test_method(self):
-
-    class TestClass(object):
-
-      @self_transform_decorator(self._transform)
-      def test_fn(self, a):
-        return a
-
-    # 2 = 1 (a) + 1 (decorator applied exactly once)
-    self.assertEqual(2, TestClass().test_fn(1))
-
-  def test_multiple_decorators(self):
-
-    class TestClass(object):
-
-      # Note that reversing the order of this two doesn't work.
-      @classmethod
-      @self_transform_decorator(self._transform)
-      def test_fn(cls, a):
-        return a
-
-    # 2 = 1 (a) + 1 (decorator applied exactly once)
-    self.assertEqual(2, TestClass.test_fn(1))
-
-  def test_nested_decorators_local(self):
-
-    @self_transform_decorator(self._transform)
-    def test_fn(a):
-      @simple_decorator
-      def inner_fn(b):
-        return b + 11
-      return inner_fn(a)
-
-    # Expected to fail because simple_decorator could not be imported.
-    with self.assertRaises(ValueError):
-      test_fn(1)
-
-  def test_nested_decorators_imported(self):
-
-    @self_transform_decorator(self._transform)
-    def test_fn(a):
-
-      @converter_testing.imported_decorator
-      def inner_fn(b):
-        return b + 11
-
-      return inner_fn(a)
-
-    # Work around TensorFlow's symbol suppression mechanism that causes core to
-    # be invisible in the generated code.
-    core_mod = imp.new_module('core')
-    core_mod.converter_testing = converter_testing
-    autograph.core = core_mod
-
-    # 14 = 1 (a) + 1 (simple_decorator) + 11 (inner_fn)
-    self.assertEqual(14, test_fn(1))
-
-
-if __name__ == '__main__':
-  test.main()
diff --git a/tensorflow/python/autograph/core/converter.py b/tensorflow/python/autograph/core/converter.py
index b3297ef229..79f96fa90e 100644
--- a/tensorflow/python/autograph/core/converter.py
+++ b/tensorflow/python/autograph/core/converter.py
@@ -125,7 +125,6 @@ class Feature(enum.Enum):
   ALL = 'ALL'
 
   AUTO_CONTROL_DEPS = 'AUTO_CONTROL_DEPS'
-  DECORATORS = 'DECORATORS'
   ERROR_REWRITING = 'ERROR_REWRITING'
   LISTS = 'LISTS'
   NAME_SCOPES = 'NAME_SCOPES'
diff --git a/tensorflow/python/autograph/core/converter_testing.py b/tensorflow/python/autograph/core/converter_testing.py
index 051495c24b..56445dbd45 100644
--- a/tensorflow/python/autograph/core/converter_testing.py
+++ b/tensorflow/python/autograph/core/converter_testing.py
@@ -39,14 +39,9 @@ from tensorflow.python.autograph.pyct import pretty_printer
 from tensorflow.python.autograph.pyct import transformer
 from tensorflow.python.platform import test
 
-
 RESULT_OF_MOCK_CONVERTED_CALL = 7
 
 
-def imported_decorator(f):
-  return lambda a: f(a) + 1
-
-
 # TODO(mdan): We should use the real namer here.
 class FakeNamer(object):
   """A fake namer that uses a global counter to generate unique names."""
diff --git a/tensorflow/python/autograph/impl/conversion.py b/tensorflow/python/autograph/impl/conversion.py
index 9f0c380cbf..3a901dfd38 100644
--- a/tensorflow/python/autograph/impl/conversion.py
+++ b/tensorflow/python/autograph/impl/conversion.py
@@ -33,7 +33,6 @@ from tensorflow.python.autograph.converters import call_trees
 from tensorflow.python.autograph.converters import conditional_expressions
 from tensorflow.python.autograph.converters import continue_statements
 from tensorflow.python.autograph.converters import control_flow
-from tensorflow.python.autograph.converters import decorators
 from tensorflow.python.autograph.converters import directives
 from tensorflow.python.autograph.converters import error_handlers
 from tensorflow.python.autograph.converters import function_scopes
@@ -427,9 +426,6 @@ def node_to_graph(node, context):
   # source.
   # TODO(mdan): Is it feasible to reconstruct intermediate source code?
   context.info.source_code = None
-
-  if context.program.options.uses(converter.Feature.DECORATORS):
-    node = converter.apply_(node, context, decorators)
   node = converter.apply_(node, context, arg_defaults)
   node = converter.apply_(node, context, directives)
   node = converter.apply_(node, context, break_statements)
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.autograph.experimental.-feature.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.autograph.experimental.-feature.pbtxt
index a71da113b4..359530f69c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.autograph.experimental.-feature.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.autograph.experimental.-feature.pbtxt
@@ -9,10 +9,6 @@ tf_class {
     name: "AUTO_CONTROL_DEPS"
     mtype: "<enum \'Feature\'>"
   }
-  member {
-    name: "DECORATORS"
-    mtype: "<enum \'Feature\'>"
-  }
   member {
     name: "ERROR_REWRITING"
     mtype: "<enum \'Feature\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.autograph.experimental.-feature.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.autograph.experimental.-feature.pbtxt
index a71da113b4..359530f69c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.autograph.experimental.-feature.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.autograph.experimental.-feature.pbtxt
@@ -9,10 +9,6 @@ tf_class {
     name: "AUTO_CONTROL_DEPS"
     mtype: "<enum \'Feature\'>"
   }
-  member {
-    name: "DECORATORS"
-    mtype: "<enum \'Feature\'>"
-  }
   member {
     name: "ERROR_REWRITING"
     mtype: "<enum \'Feature\'>"
-- 
GitLab


From c1162e860fe15d03ab8d98647f8f4a66208c28a3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 27 Jan 2019 01:02:19 -0800
Subject: [PATCH 1346/2345] compat: Update forward compatibility horizon to
 2019-01-27

PiperOrigin-RevId: 231097390
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 2c764900ea..d0916a96fc 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 26)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 27)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From e4b4b2e4ffe28e6b49fa6924af0764eb2fad1676 Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Sun, 27 Jan 2019 12:37:25 -0800
Subject: [PATCH 1347/2345] Build different models for training, test and
 predict in keras when used with distribution strategy.

NOTE: This is already the case when used with TPUStrategy. This just extends this to all other strategies. This change enables us to run at higher batch sizes with running into OOM issues.
PiperOrigin-RevId: 231129071
---
 .../python/keras_backward_compat_test.py      |   2 +-
 .../engine/distributed_training_utils.py      | 107 ++++++++++--------
 tensorflow/python/keras/engine/training.py    |   4 +-
 .../python/keras/engine/training_arrays.py    |  21 ++--
 .../keras/engine/training_distributed.py      |  62 ++++------
 5 files changed, 102 insertions(+), 94 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_backward_compat_test.py b/tensorflow/contrib/distribute/python/keras_backward_compat_test.py
index 35d40e66bf..3bc84dc009 100644
--- a/tensorflow/contrib/distribute/python/keras_backward_compat_test.py
+++ b/tensorflow/contrib/distribute/python/keras_backward_compat_test.py
@@ -746,7 +746,7 @@ class TestDistributionStrategyWithDatasets(test.TestCase,
 
       model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0,
                 callbacks=[keras.callbacks.LearningRateScheduler(schedule)])
-      grouped_models = distribution.unwrap(model._distributed_model)
+      grouped_models = distribution.unwrap(model._distributed_model_train)
       with distribution.scope():
         for m in grouped_models:
           self.assertAllClose(0.001, keras.backend.get_value(
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index 7d00c162fa..224231278b 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -40,6 +40,7 @@ from tensorflow.python.ops import variables
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training.mode_keys import ModeKeys
 from tensorflow.python.util import nest
+from tensorflow.python.util import tf_contextlib
 
 
 def set_weights(distribution_strategy, dist_model, weights):
@@ -603,7 +604,7 @@ def _custom_compile_for_predict(model):
   model.predict_function = None
 
 
-def _build_network_on_replica(model, inputs=None, targets=None, mode=None):
+def _build_network_on_replica(model, mode, inputs=None, targets=None):
   """Build an updated model on replicas.
 
   We create a new Keras model while sharing the variables from the old graph.
@@ -621,9 +622,9 @@ def _build_network_on_replica(model, inputs=None, targets=None, mode=None):
 
   Args:
     model: Model to be replicated across Replicas
+    mode: Which of fit/eval/predict is building the distributed network
     inputs: Input variables to be passed to the model
     targets: Target tensor to be passed to model.compile
-    mode: Which of fit/eval/predict is building the distributed network
 
   Returns:
     A new model with shared layers with the old model.
@@ -655,7 +656,7 @@ def _build_network_on_replica(model, inputs=None, targets=None, mode=None):
   if isinstance(targets, tuple):
     targets = nest.flatten(targets)
 
-  if mode == ModeKeys.PREDICT:
+  if mode == ModeKeys.PREDICT and inputs is not None:  # TPU predict case
     _custom_compile_for_predict(updated_model)
   else:
     updated_model.compile(
@@ -670,24 +671,17 @@ def _build_network_on_replica(model, inputs=None, targets=None, mode=None):
   return updated_model
 
 
-def _build_distributed_network(model, strategy, inputs=None, targets=None,
-                               mode=None):
+def _build_distributed_network(model, strategy, mode, inputs=None,
+                               targets=None):
   """Create a cloned model on each replica."""
   with K.get_graph().as_default(), strategy.scope():
     distributed_model = strategy.extended.call_for_each_replica(
         _build_network_on_replica,
-        args=(model, inputs, targets, mode))
-    if mode is ModeKeys.TRAIN:
-      model._distributed_model_train = distributed_model
-    elif mode is ModeKeys.TEST:
-      model._distributed_model_test = distributed_model
-    elif mode is ModeKeys.PREDICT:
-      model._distributed_model_predict = distributed_model
-    else:
-      model._distributed_model = distributed_model
+        args=(model, mode, inputs, targets))
+    set_distributed_model(model, mode, distributed_model)
 
 
-def _clone_and_build_model(model, inputs=None, targets=None, mode=None):
+def _clone_and_build_model(model, mode, inputs=None, targets=None):
   """Clone and build the given keras_model."""
   # We need to set the import here since we run into a circular dependency
   # error.
@@ -714,7 +708,7 @@ def _clone_and_build_model(model, inputs=None, targets=None, mode=None):
 
   if isinstance(targets, tuple):
     targets = nest.flatten(targets)
-  if mode == ModeKeys.PREDICT:
+  if mode == ModeKeys.PREDICT and inputs is not None:  # TPU predict case
     _custom_compile_for_predict(cloned_model)
   else:
     cloned_model.compile(
@@ -729,21 +723,13 @@ def _clone_and_build_model(model, inputs=None, targets=None, mode=None):
   return cloned_model
 
 
-def clone_model_on_replicas(model, strategy, make_callback_model=False,
-                            inputs=None, targets=None, mode=None):
+def clone_model_on_replicas(model, strategy, mode, inputs=None, targets=None):
   """Create a cloned model on each replica."""
   with K.get_graph().as_default(), strategy.scope():
     distributed_model = strategy.extended.call_for_each_replica(
-        _clone_and_build_model, args=(model, inputs, targets, mode))
-    if mode is ModeKeys.TRAIN:
-      model._distributed_model_train = distributed_model
-    elif mode is ModeKeys.TEST:
-      model._distributed_model_test = distributed_model
-    elif mode is ModeKeys.PREDICT:
-      model._distributed_model_predict = distributed_model
-    else:
-      model._distributed_model = distributed_model
-  if make_callback_model:
+        _clone_and_build_model, args=(model, mode, inputs, targets))
+    set_distributed_model(model, mode, distributed_model)
+  if mode == ModeKeys.TRAIN:
     model._make_callback_model(distributed_model)
 
 
@@ -753,12 +739,11 @@ def _make_execution_function(model, mode):
     return _make_eager_execution_function(model, mode)
 
   strategy = model._distribution_strategy
-  if not model._distributed_model:
+  if not get_distributed_model(model, mode):
     if model._compile_distribution:
-      clone_model_on_replicas(
-          model, strategy, make_callback_model=(mode == ModeKeys.TRAIN))
+      clone_model_on_replicas(model, strategy, mode)
     else:
-      _build_distributed_network(model, strategy)
+      _build_distributed_network(model, strategy, mode)
 
   def _per_device_function(model):
     f = model._make_execution_function(mode)
@@ -769,7 +754,7 @@ def _make_execution_function(model, mode):
     # `_per_device_fit_function`.
     (grouped_inputs, grouped_outputs, grouped_updates,
      grouped_session_args) = strategy.extended.call_for_each_replica(
-         _per_device_function, args=(model._distributed_model,))
+         _per_device_function, args=(get_distributed_model(model, mode),))
 
     # Initialize the variables in the replicated model. This is necessary for
     # multi-worker training because on some workers, initialization is not
@@ -800,12 +785,11 @@ def _make_execution_function(model, mode):
 def _make_eager_execution_function(model, mode):
   """Makes function to run one step of distributed model eager execution."""
   strategy = model._distribution_strategy
-  if not model._distributed_model:
+  if not get_distributed_model(model, mode):
     if model._compile_distribution:
-      clone_model_on_replicas(
-          model, strategy, make_callback_model=(mode == ModeKeys.TRAIN))
+      clone_model_on_replicas(model, strategy, mode)
     else:
-      _build_distributed_network(model, strategy)
+      _build_distributed_network(model, strategy, mode)
 
   def _per_device_function(model):
     f = model._make_execution_function(mode)
@@ -817,7 +801,7 @@ def _make_eager_execution_function(model, mode):
     # Create train ops on each of the devices when we call
     # `_per_device_fit_function`.
     (grouped_inputs, grouped_outputs) = strategy.extended.call_for_each_replica(
-        _per_device_function, args=(model._distributed_model,))
+        _per_device_function, args=(get_distributed_model(model, mode),))
 
     # Unwrap all the per device values returned from `call_for_each_replica`.
     # Unwrapping per device values gives you a list of values that can be
@@ -835,22 +819,24 @@ def _make_eager_execution_function(model, mode):
         name='eager_distributed_{}_function'.format(mode))
 
 
-def _copy_weights_to_distributed_model(original_model, grouped_model):
+def _copy_weights_to_distributed_model(original_model, mode):
   """Copies weights from original model to distributed models."""
   strategy = original_model._distribution_strategy
+  distributed_model = get_distributed_model(original_model, mode)
   if strategy:
     # Copy the weights from the original model to each of the replicated
     # models.
     orig_model_weights = original_model.get_weights()
-    distributed_model = strategy.unwrap(grouped_model)[0]
-    set_weights(strategy, distributed_model, orig_model_weights)
+    first_model = strategy.unwrap(distributed_model)[0]
+    set_weights(strategy, first_model, orig_model_weights)
 
 
-def _copy_weights_to_original_model(model, grouped_model, mode):
+def _copy_weights_to_original_model(model, mode):
   """Copies weights from first distributed model back to original model."""
   if model._distribution_strategy and mode == ModeKeys.TRAIN:
+    distributed_model = get_distributed_model(model, mode)
     updated_weights = model._distribution_strategy.unwrap(
-        grouped_model)[0].get_weights()
+        distributed_model)[0].get_weights()
     model.set_weights(updated_weights)
 
 
@@ -866,9 +852,34 @@ def _per_device_aggregate_batch(batch_outs, model, mode):
   return batch_outs
 
 
-def _reset_metrics(model, distributed_model=None):
+def _reset_metrics(model):
   if model._distribution_strategy:
-    distributed_model = (
-        distributed_model or
-        model._distribution_strategy.unwrap(model._distributed_model)[0])
-    distributed_model.reset_metrics()
+    for mode in [ModeKeys.TRAIN, ModeKeys.TEST, ModeKeys.PREDICT]:
+      distributed_model = get_distributed_model(model, mode)
+      if distributed_model:
+        first_model = model._distribution_strategy.unwrap(distributed_model)[0]
+        first_model.reset_metrics()
+
+
+def get_distributed_model(model, mode):
+  if mode is ModeKeys.TRAIN:
+    return model._distributed_model_train
+  elif mode is ModeKeys.TEST:
+    return model._distributed_model_test
+  elif mode is ModeKeys.PREDICT:
+    return model._distributed_model_predict
+
+
+def set_distributed_model(model, mode, distributed_model):
+  if mode is ModeKeys.TRAIN:
+    model._distributed_model_train = distributed_model
+  elif mode is ModeKeys.TEST:
+    model._distributed_model_test = distributed_model
+  elif mode is ModeKeys.PREDICT:
+    model._distributed_model_predict = distributed_model
+
+
+@tf_contextlib.contextmanager
+def distributed_scope(strategy, learning_phase):
+  with strategy.scope(), K.learning_phase_scope(learning_phase):
+    yield
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index ba351cc56b..12a1008153 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -276,7 +276,9 @@ class Model(Network):
     self.target_tensors = target_tensors
 
     # Set DistributionStrategy specific parameters.
-    self._distributed_model = None
+    for mode in [ModeKeys.TRAIN, ModeKeys.TEST, ModeKeys.PREDICT]:
+      distributed_training_utils.set_distributed_model(self, mode, None)
+
     # Initialize model metric attributes.
     self._init_metric_attributes()
     if not self.built or not self.inputs or not self.outputs:
diff --git a/tensorflow/python/keras/engine/training_arrays.py b/tensorflow/python/keras/engine/training_arrays.py
index cefd4d57fd..68b8f61521 100644
--- a/tensorflow/python/keras/engine/training_arrays.py
+++ b/tensorflow/python/keras/engine/training_arrays.py
@@ -142,7 +142,9 @@ def model_iteration(model,
 
   # Enter DistributionStrategy scope.
   if model._distribution_strategy:
-    scope = model._distribution_strategy.scope()
+    scope = distributed_training_utils.distributed_scope(
+        strategy=model._distribution_strategy,
+        learning_phase=(1 if mode == ModeKeys.TRAIN else 0))
     scope.__enter__()
 
   # Get step function and loop type.
@@ -211,9 +213,8 @@ def model_iteration(model,
     aggregator = training_utils.MetricsAggregator(use_steps,
                                                   num_samples_or_steps)
 
-  if model._compile_distribution and not validation_in_fit:
-    distributed_training_utils._copy_weights_to_distributed_model(
-        model, model._distributed_model)
+  if model._compile_distribution:
+    distributed_training_utils._copy_weights_to_distributed_model(model, mode)
 
   callbacks.model.stop_training = False
   callbacks._call_begin_hook(mode)
@@ -353,6 +354,13 @@ def model_iteration(model,
     if (do_validation and
         training_utils.should_run_validation(validation_freq, epoch) and
         not callbacks.model.stop_training):
+
+      if model._compile_distribution:
+        # Since we create a new clone from the original model we need to copy
+        # the weights back to the original model before we can run validation.
+        distributed_training_utils._copy_weights_to_original_model(
+            model, ModeKeys.TRAIN)
+
       val_results = model_iteration(
           model,
           val_inputs,
@@ -385,10 +393,9 @@ def model_iteration(model,
   callbacks._call_end_hook(mode)
 
   if model._distribution_strategy:
-    if model._compile_distribution and not validation_in_fit:
+    if model._compile_distribution:
       # TODO(priyag, psv): Copy back metrics to the original model as well?
-      distributed_training_utils._copy_weights_to_original_model(
-          model, model._distributed_model, mode)
+      distributed_training_utils._copy_weights_to_original_model(model, mode)
     scope.__exit__(None, None, None)
 
   if mode == ModeKeys.TRAIN:
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 9b11c721d4..57b665a3da 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -246,7 +246,9 @@ def experimental_tpu_fit_loop(model,
   # TODO(fchollet): add support for `steps_per_epoch=None` in TPU loops.
   current_strategy = model._distribution_strategy
   iterator = distributed_training_utils.get_iterator(dataset, current_strategy)
-  scope = current_strategy.scope()
+
+  scope = distributed_training_utils.distributed_scope(
+      strategy=current_strategy, learning_phase=1)
   scope.__enter__()
 
   def _per_device_fit_function(model):
@@ -254,8 +256,6 @@ def experimental_tpu_fit_loop(model,
     return (model._fit_function.inputs, model._fit_function.outputs,
             model._fit_function.updates_op, model._fit_function.session_kwargs)
 
-  # TODO(priyag, sourabhbajaj): This should likely not be hardcoded here.
-  K.set_learning_phase(1)
   out_labels = model.metrics_names or []
 
   def step_fn(ctx, inputs):
@@ -263,13 +263,11 @@ def experimental_tpu_fit_loop(model,
     inputs, targets = inputs
     if model._compile_distribution:
       distributed_training_utils.clone_model_on_replicas(
-          model, current_strategy,
-          make_callback_model=True, inputs=inputs,
-          targets=targets, mode=distributed_training_utils.ModeKeys.TRAIN)
+          model, current_strategy, ModeKeys.TRAIN, inputs=inputs,
+          targets=targets)
     else:
       distributed_training_utils._build_distributed_network(
-          model, current_strategy, inputs,
-          targets, mode=distributed_training_utils.ModeKeys.TRAIN)
+          model, current_strategy, ModeKeys.TRAIN, inputs, targets)
 
     (grouped_inputs, grouped_outputs, grouped_updates,
      grouped_session_args) = current_strategy.extended.call_for_each_replica(
@@ -325,7 +323,7 @@ def experimental_tpu_fit_loop(model,
 
   if model._compile_distribution:
     distributed_training_utils._copy_weights_to_distributed_model(
-        model, model._distributed_model_train)
+        model, ModeKeys.TRAIN)
 
   callbacks = cbks.configure_callbacks(
       callbacks,
@@ -344,8 +342,7 @@ def experimental_tpu_fit_loop(model,
 
   callbacks.on_train_begin()
   for epoch in range(initial_epoch, epochs):
-    distributed_training_utils._reset_metrics(
-        model, model._distributed_model_train)
+    distributed_training_utils._reset_metrics(model)
     callbacks.on_epoch_begin(epoch)
     epoch_logs = {}
     step_index = 0
@@ -380,7 +377,7 @@ def experimental_tpu_fit_loop(model,
         # Since we create a new clone from the original model we need to copy
         # the weights back to the original model before we can run validation.
         distributed_training_utils._copy_weights_to_original_model(
-            model, model._distributed_model_train, ModeKeys.TRAIN)
+            model, ModeKeys.TRAIN)
 
       val_outs = experimental_tpu_test_loop(  # pylint: disable=undefined-variable
           model,
@@ -401,7 +398,7 @@ def experimental_tpu_fit_loop(model,
   if model._compile_distribution:
     # Copy the weights back from the replicated model to the original model.
     distributed_training_utils._copy_weights_to_original_model(
-        model, model._distributed_model_train, ModeKeys.TRAIN)
+        model, ModeKeys.TRAIN)
   scope.__exit__(None, None, None)
   return model.history
 
@@ -428,7 +425,8 @@ def experimental_tpu_test_loop(model,
   """
   current_strategy = model._distribution_strategy
   iterator = distributed_training_utils.get_iterator(dataset, current_strategy)
-  scope = current_strategy.scope()
+  scope = distributed_training_utils.distributed_scope(
+      strategy=current_strategy, learning_phase=0)
   scope.__enter__()
 
   def _per_device_eval_function(model):
@@ -437,21 +435,16 @@ def experimental_tpu_test_loop(model,
             model._eval_function.updates_op,
             model._eval_function.session_kwargs)
 
-  # TODO(priyag, sourabhbajaj): This should likely not be hardcoded here.
-  K.set_learning_phase(0)
-
   def step_fn(ctx, inputs):
     """Clones the model and calls make_eval_function."""
     inputs, targets = inputs
     if model._compile_distribution:
       distributed_training_utils. clone_model_on_replicas(
-          model, current_strategy,
-          make_callback_model=False, inputs=inputs,
-          targets=targets, mode=distributed_training_utils.ModeKeys.TEST)
+          model, current_strategy, mode=ModeKeys.TEST, inputs=inputs,
+          targets=targets)
     else:
       distributed_training_utils._build_distributed_network(
-          model, current_strategy, inputs, targets,
-          mode=distributed_training_utils.ModeKeys.TEST)
+          model, current_strategy, ModeKeys.TEST, inputs, targets)
 
     (grouped_inputs, grouped_outputs, grouped_updates,
      grouped_session_args) = current_strategy.extended.call_for_each_replica(
@@ -500,10 +493,9 @@ def experimental_tpu_test_loop(model,
 
   if model._compile_distribution:
     distributed_training_utils._copy_weights_to_distributed_model(
-        model, model._distributed_model_test)
+        model, ModeKeys.TEST)
 
-  distributed_training_utils._reset_metrics(
-      model, model._distributed_model_test)
+  distributed_training_utils._reset_metrics(model)
 
   assert steps is not None
   outs = [0.] * len(model.metrics_names)
@@ -547,11 +539,10 @@ def experimental_tpu_predict_loop(model, dataset, verbose=0, steps=None):
   """
   current_strategy = model._distribution_strategy
   iterator = distributed_training_utils.get_iterator(dataset, current_strategy)
-  scope = current_strategy.scope()
-  scope.__enter__()
 
-  # TODO(priyag, sourabhbajaj): This should likely not be hardcoded here.
-  K.set_learning_phase(0)
+  scope = distributed_training_utils.distributed_scope(
+      strategy=current_strategy, learning_phase=0)
+  scope.__enter__()
 
   def _per_device_predict_function(model):
     model._make_predict_function()
@@ -564,13 +555,10 @@ def experimental_tpu_predict_loop(model, dataset, verbose=0, steps=None):
     """Clones the model and calls make_predict_function."""
     if model._compile_distribution:
       distributed_training_utils. clone_model_on_replicas(
-          model, current_strategy,
-          make_callback_model=False, inputs=inputs,
-          mode=distributed_training_utils.ModeKeys.PREDICT)
+          model, current_strategy, ModeKeys.PREDICT, inputs=inputs)
     else:
       distributed_training_utils._build_distributed_network(
-          model, current_strategy, inputs,
-          mode=distributed_training_utils.ModeKeys.PREDICT)
+          model, current_strategy, ModeKeys.PREDICT, inputs)
 
     (grouped_inputs, grouped_outputs, grouped_updates,
      grouped_session_args) = current_strategy.extended.call_for_each_replica(
@@ -615,10 +603,9 @@ def experimental_tpu_predict_loop(model, dataset, verbose=0, steps=None):
 
   if model._compile_distribution:
     distributed_training_utils._copy_weights_to_distributed_model(
-        model, model._distributed_model_predict)
+        model, ModeKeys.PREDICT)
 
-  distributed_training_utils._reset_metrics(
-      model, model._distributed_model_predict)
+  distributed_training_utils._reset_metrics(model)
 
   assert steps is not None
   # Since we do not know how many samples we will see, we cannot pre-allocate
@@ -634,6 +621,7 @@ def experimental_tpu_predict_loop(model, dataset, verbose=0, steps=None):
       progbar.update(step + 1)
 
   scope.__exit__(None, None, None)
+
   if len(unconcatenated_outs) == 1:
     return np.concatenate(unconcatenated_outs[0], axis=0)
   return [
-- 
GitLab


From 0f59cbd29738833655eeffe314b9b7f6667e6005 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 27 Jan 2019 14:14:18 -0800
Subject: [PATCH 1348/2345] Internal changes.

PiperOrigin-RevId: 231133784
---
 tensorflow/python/util/nest.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/python/util/nest.py b/tensorflow/python/util/nest.py
index 4de358dec6..436cf82ea5 100644
--- a/tensorflow/python/util/nest.py
+++ b/tensorflow/python/util/nest.py
@@ -491,7 +491,6 @@ def map_structure_with_paths(func, *structure, **kwargs):
       the type of sequence in any of their substructures.
     ValueError: If no structures are provided.
   """
-  print("wheee I'm updated")
   def wrapper_func(tuple_path, *inputs, **kwargs):
     string_path = "/".join(str(s) for s in tuple_path)
     return func(string_path, *inputs, **kwargs)
-- 
GitLab


From fce6b9db546dc9ae3e7ccb0e009bf065233ca54c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 28 Jan 2019 01:02:46 -0800
Subject: [PATCH 1349/2345] compat: Update forward compatibility horizon to
 2019-01-28

PiperOrigin-RevId: 231172884
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index d0916a96fc..03147e3581 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 27)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 28)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 64245ae48cefdc47b21031b8b1282d3a608c7dee Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Mon, 28 Jan 2019 01:07:42 -0800
Subject: [PATCH 1350/2345] Fix assumption in ConvolutionGroupConverter that we
 never have more than 32 bits

PiperOrigin-RevId: 231174122
---
 .../service/convolution_group_converter.cc    | 32 +++++++++----------
 .../kernel_tests/depthwise_conv_op_test.py    |  3 --
 2 files changed, 16 insertions(+), 19 deletions(-)

diff --git a/tensorflow/compiler/xla/service/convolution_group_converter.cc b/tensorflow/compiler/xla/service/convolution_group_converter.cc
index b3ed27d9a8..1c1f543170 100644
--- a/tensorflow/compiler/xla/service/convolution_group_converter.cc
+++ b/tensorflow/compiler/xla/service/convolution_group_converter.cc
@@ -316,14 +316,27 @@ Status ConvolutionVisitor::HandleBatchGroupCount(HloInstruction* convolution) {
         expanded_filter_shape, HloOpcode::kSelect, filter_mask, new_convolution,
         zero_filter));
 
-    auto zero_literal = LiteralUtil::CreateR0(0.0f);
-    TF_ASSIGN_OR_RETURN(zero_literal, zero_literal.Convert(F32));
+    PrimitiveType reduce_type = new_filter->shape().element_type();
+    auto reduce_window_shape = new_convolution->shape();
+    reduce_window_shape.set_dimensions(output_batch_dimension, 1);
+
+    // Ensure that data input to reduce window uses at least 32 bits.
+    if (primitive_util::BitWidth(reduce_type) < primitive_util::BitWidth(F32)) {
+      reduce_type = F32;
+      reduce_window_shape.set_element_type(F32);
+      Shape convert_shape = new_filter->shape();
+      convert_shape.set_element_type(F32);
+      new_filter =
+          add(HloInstruction::CreateConvert(convert_shape, new_filter));
+    }
+
+    auto zero_literal = LiteralUtil::Zero(reduce_type);
     auto zero_scalar =
         add(HloInstruction::CreateConstant(std::move(zero_literal)));
 
     auto reduce_function = [&]() -> HloComputation* {
       HloComputation::Builder b("add_computation");
-      Shape shape = ShapeUtil::MakeShape(F32, {});
+      Shape shape = ShapeUtil::MakeShape(reduce_type, {});
       auto lhs =
           b.AddInstruction(HloInstruction::CreateParameter(0, shape, "lhs"));
       auto rhs =
@@ -333,19 +346,6 @@ Status ConvolutionVisitor::HandleBatchGroupCount(HloInstruction* convolution) {
       return computation_->parent()->AddEmbeddedComputation(b.Build(scalar_op));
     };
 
-    auto reduce_window_shape = new_convolution->shape();
-    reduce_window_shape.set_dimensions(output_batch_dimension, 1);
-
-    // Ensure that data input to reduce window is of type F32.
-    if (primitive_util::BitWidth(new_filter->shape().element_type()) <
-        primitive_util::BitWidth(F32)) {
-      reduce_window_shape.set_element_type(F32);
-      Shape convert_shape = new_filter->shape();
-      convert_shape.set_element_type(F32);
-      new_filter =
-          add(HloInstruction::CreateConvert(convert_shape, new_filter));
-    }
-
     // Create the reduce window.
     Window window;
     for (int64 i = 0; i < new_convolution->shape().dimensions_size(); ++i) {
diff --git a/tensorflow/python/kernel_tests/depthwise_conv_op_test.py b/tensorflow/python/kernel_tests/depthwise_conv_op_test.py
index 4abf6b0e67..5b1a47fb03 100644
--- a/tensorflow/python/kernel_tests/depthwise_conv_op_test.py
+++ b/tensorflow/python/kernel_tests/depthwise_conv_op_test.py
@@ -481,7 +481,6 @@ class DepthwiseConv2DTest(test.TestCase):
             data_format="NCHW")
 
   @test_util.run_v1_only("b/120545219")
-  @test_util.disable_xla("This test never passed for XLA")
   def testDepthwiseConv2DFilterGrad(self):
     for index, (input_size, filter_size, output_size, stride,
                 padding) in enumerate(CheckGradConfigsToTest()):
@@ -499,7 +498,6 @@ class DepthwiseConv2DTest(test.TestCase):
             test_input=False,
             use_gpu=True)
 
-  @test_util.disable_xla("This test never passed for XLA")
   def testDepthwiseConv2DFilterGradFormat(self):
     if not test.is_gpu_available():
       return
@@ -614,7 +612,6 @@ class DepthwiseConv2DTest(test.TestCase):
     cpu_value = _GetVal(use_gpu=False)
     self.assertAllClose(cpu_value, gpu_value, rtol=1e-4, atol=1e-4)
 
-  @test_util.disable_xla("This test never passed for XLA")
   def testDepthwiseConv2DFilterGradCompare(self):
     for index, (input_size, filter_size, output_size, stride,
                 padding) in enumerate(ConfigsToTest()):
-- 
GitLab


From e0585bc351b19da39610cc20f6d7622b439dca4d Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Mon, 28 Jan 2019 01:10:25 -0800
Subject: [PATCH 1351/2345] Test additional container types in walk.

PiperOrigin-RevId: 231174474
---
 tensorflow/python/module/BUILD          |  1 +
 tensorflow/python/module/module_test.py | 18 +++++++++++++-----
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/module/BUILD b/tensorflow/python/module/BUILD
index 64c226e4c9..8aa3a199b5 100644
--- a/tensorflow/python/module/BUILD
+++ b/tensorflow/python/module/BUILD
@@ -23,6 +23,7 @@ tf_py_test(
     srcs = ["module_test.py"],
     additional_deps = [
         ":module",
+        "@absl_py//absl/testing:parameterized",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python/compat:v2_compat",
         "//tensorflow/python:variables",
diff --git a/tensorflow/python/module/module_test.py b/tensorflow/python/module/module_test.py
index e7c736696e..1e4e1959c3 100644
--- a/tensorflow/python/module/module_test.py
+++ b/tensorflow/python/module/module_test.py
@@ -18,6 +18,10 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import collections
+
+from absl.testing import parameterized
+
 from tensorflow.python.compat import v2_compat
 from tensorflow.python.framework import ops
 from tensorflow.python.module import module
@@ -316,11 +320,15 @@ class CallsMethodBeforeSuperConstructorModule(module.Module):
   def with_name_scope(self):
     pass
 
+NamedPair = collections.namedtuple("NamedPair", ("first", "second"))
+mk_index_dict = lambda v: dict(enumerate(v))
+
 
-class WalkTest(test.TestCase):
+class WalkTest(parameterized.TestCase, test.TestCase):
 
-  def test_walk(self):
-    parent = SimpleModule()
+  @parameterized.parameters(lambda v: NamedPair(*v), list, tuple, mk_index_dict)
+  def test_walk(self, container_type):
+    parent = SimpleModule(container_type=container_type)
     child = parent.c
 
     self.assertEqual(
@@ -339,10 +347,10 @@ class MemberType(object):
 
 class SimpleModule(module.Module):
 
-  def __init__(self, create_child=True):
+  def __init__(self, create_child=True, container_type=list):
     super(SimpleModule, self).__init__()
     self.z = MemberType()
-    self.a = [MemberType(), MemberType()]
+    self.a = container_type([MemberType(), MemberType()])
     if create_child:
       self.c = SimpleModule(create_child=False)
 
-- 
GitLab


From 91ffe999b88e665fa4c6e6899cb8170cbd6fccb8 Mon Sep 17 00:00:00 2001
From: Lukas Geiger <lukas.geiger94@gmail.com>
Date: Mon, 28 Jan 2019 14:05:00 +0100
Subject: [PATCH 1352/2345] Use efficient squared_difference instead of
 square(diff)

---
 .../python/training/functions/gbdt_batch_test.py             | 2 +-
 tensorflow/contrib/boosted_trees/python/utils/losses.py      | 2 +-
 tensorflow/contrib/distributions/python/ops/inverse_gamma.py | 2 +-
 .../python/examples/linear_regression/linear_regression.py   | 2 +-
 .../eager/python/examples/rnn_colorbot/rnn_colorbot.py       | 2 +-
 tensorflow/contrib/factorization/python/ops/gmm_ops.py       | 3 +--
 tensorflow/contrib/layers/python/layers/target_column.py     | 2 +-
 tensorflow/contrib/learn/python/learn/estimators/head.py     | 2 +-
 tensorflow/contrib/losses/python/losses/loss_ops.py          | 2 +-
 .../contrib/losses/python/metric_learning/metric_loss_ops.py | 5 ++---
 tensorflow/contrib/metrics/python/ops/metric_ops.py          | 2 +-
 tensorflow/contrib/tensor_forest/client/eval_metrics.py      | 5 +++--
 tensorflow/contrib/timeseries/python/timeseries/ar_model.py  | 3 ++-
 tensorflow/python/keras/losses.py                            | 2 +-
 tensorflow/python/keras/metrics.py                           | 2 +-
 tensorflow/python/keras/utils/kernelized_utils.py            | 2 +-
 tensorflow/python/ops/distributions/normal.py                | 4 ++--
 tensorflow/python/ops/metrics_impl.py                        | 2 +-
 18 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch_test.py b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch_test.py
index 92068e88a7..7e45d0b2ce 100644
--- a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch_test.py
+++ b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch_test.py
@@ -43,7 +43,7 @@ from tensorflow.python.platform import googletest
 def _squared_loss(label, unused_weights, predictions):
   """Unweighted loss implementation."""
   loss = math_ops.reduce_sum(
-      math_ops.square(predictions - label), 1, keepdims=True)
+      math_ops.squared_difference(predictions, label), 1, keepdims=True)
   return loss
 
 
diff --git a/tensorflow/contrib/boosted_trees/python/utils/losses.py b/tensorflow/contrib/boosted_trees/python/utils/losses.py
index 220e981618..1ad40aca28 100644
--- a/tensorflow/contrib/boosted_trees/python/utils/losses.py
+++ b/tensorflow/contrib/boosted_trees/python/utils/losses.py
@@ -166,7 +166,7 @@ def per_example_squared_loss(labels, weights, predictions):
     update_op: An update operation to update the loss's internal state.
   """
   unweighted_loss = math_ops.reduce_sum(
-      math_ops.square(predictions - labels), 1, keepdims=True)
+      math_ops.squared_difference(predictions, labels), 1, keepdims=True)
 
   return unweighted_loss * weights, control_flow_ops.no_op()
 
diff --git a/tensorflow/contrib/distributions/python/ops/inverse_gamma.py b/tensorflow/contrib/distributions/python/ops/inverse_gamma.py
index 452628257e..e359c498bd 100644
--- a/tensorflow/contrib/distributions/python/ops/inverse_gamma.py
+++ b/tensorflow/contrib/distributions/python/ops/inverse_gamma.py
@@ -250,7 +250,7 @@ class InverseGamma(distribution.Distribution):
       than returning `NaN`.""")
   def _variance(self):
     var = (math_ops.square(self.rate)
-           / math_ops.square(self.concentration - 1.)
+           / math_ops.squared_difference(self.concentration, 1.)
            / (self.concentration - 2.))
     if self.allow_nan_stats:
       nan = array_ops.fill(
diff --git a/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression.py b/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression.py
index 099b712fc0..206ef9409d 100644
--- a/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression.py
+++ b/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression.py
@@ -56,7 +56,7 @@ class LinearModel(tf.keras.Model):
 
 
 def mean_square_loss(model, xs, ys):
-  return tf.reduce_mean(tf.square(tf.subtract(model(xs), ys)))
+  return tf.reduce_mean(tf.squared_difference(model(xs), ys))
 
 
 def fit(model, dataset, optimizer, verbose=False, logdir=None):
diff --git a/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot.py b/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot.py
index 74ebb1ec77..1c718a5ce3 100644
--- a/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot.py
+++ b/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot.py
@@ -207,7 +207,7 @@ class RNNColorbot(tf.keras.Model):
 
 def loss(labels, predictions):
   """Computes mean squared loss."""
-  return tf.reduce_mean(tf.square(predictions - labels))
+  return tf.reduce_mean(tf.squared_difference(predictions, labels))
 
 
 def test(model, eval_data):
diff --git a/tensorflow/contrib/factorization/python/ops/gmm_ops.py b/tensorflow/contrib/factorization/python/ops/gmm_ops.py
index d365ad1117..9f0664dfe5 100644
--- a/tensorflow/contrib/factorization/python/ops/gmm_ops.py
+++ b/tensorflow/contrib/factorization/python/ops/gmm_ops.py
@@ -314,8 +314,7 @@ class GmmAlgorithm(object):
     # reparametrization of variance parameters.
     det_expanded = math_ops.reduce_sum(
         math_ops.log(self._covs + 1e-3), 1, keepdims=True)
-    diff = shard - self._means
-    x2 = math_ops.square(diff)
+    x2 = math_ops.squared_difference(shard, self._means)
     cov_expanded = array_ops.expand_dims(1.0 / (self._covs + 1e-3), 2)
     # num_classes X num_examples
     x2_cov = math_ops.matmul(x2, cov_expanded)
diff --git a/tensorflow/contrib/layers/python/layers/target_column.py b/tensorflow/contrib/layers/python/layers/target_column.py
index 8a6b4f68a8..5234869718 100644
--- a/tensorflow/contrib/layers/python/layers/target_column.py
+++ b/tensorflow/contrib/layers/python/layers/target_column.py
@@ -399,7 +399,7 @@ def _mean_squared_loss(logits, target):
     target = array_ops.expand_dims(target, axis=1)
 
   logits.get_shape().assert_is_compatible_with(target.get_shape())
-  return math_ops.square(logits - math_ops.to_float(target))
+  return math_ops.squared_difference(logits, math_ops.to_float(target))
 
 
 def _log_loss_with_two_classes(logits, target):
diff --git a/tensorflow/contrib/learn/python/learn/estimators/head.py b/tensorflow/contrib/learn/python/learn/estimators/head.py
index c1b97d8b49..3cee8d9c5b 100644
--- a/tensorflow/contrib/learn/python/learn/estimators/head.py
+++ b/tensorflow/contrib/learn/python/learn/estimators/head.py
@@ -567,7 +567,7 @@ def _mean_squared_loss(labels, logits, weights=None):
     if len(logits.get_shape()) == 1:
       logits = array_ops.expand_dims(logits, axis=1)
     logits.get_shape().assert_is_compatible_with(labels.get_shape())
-    loss = math_ops.square(logits - math_ops.to_float(labels), name=name)
+    loss = math_ops.squared_difference(logits, math_ops.to_float(labels), name=name)
     return _compute_weighted_loss(loss, weights)
 
 
diff --git a/tensorflow/contrib/losses/python/losses/loss_ops.py b/tensorflow/contrib/losses/python/losses/loss_ops.py
index 709a042bbc..5ebdd0b8b5 100644
--- a/tensorflow/contrib/losses/python/losses/loss_ops.py
+++ b/tensorflow/contrib/losses/python/losses/loss_ops.py
@@ -511,7 +511,7 @@ def mean_squared_error(predictions, labels=None, weights=1.0, scope=None):
     predictions.get_shape().assert_is_compatible_with(labels.get_shape())
     predictions = math_ops.to_float(predictions)
     labels = math_ops.to_float(labels)
-    losses = math_ops.square(math_ops.subtract(predictions, labels))
+    losses = math_ops.squared_difference(predictions, labels)
     return compute_weighted_loss(losses, weights, scope=scope)
 
 
diff --git a/tensorflow/contrib/losses/python/metric_learning/metric_loss_ops.py b/tensorflow/contrib/losses/python/metric_learning/metric_loss_ops.py
index de76acb51f..51ec6cd98e 100644
--- a/tensorflow/contrib/losses/python/metric_learning/metric_loss_ops.py
+++ b/tensorflow/contrib/losses/python/metric_learning/metric_loss_ops.py
@@ -103,9 +103,8 @@ def contrastive_loss(labels, embeddings_anchor, embeddings_positive,
     contrastive_loss: tf.float32 scalar.
   """
   # Get per pair distances
-  distances = math_ops.sqrt(
-      math_ops.reduce_sum(
-          math_ops.square(embeddings_anchor - embeddings_positive), 1))
+  distances = math_ops.sqrt(math_ops.reduce_sum(
+      math_ops.squared_difference(embeddings_anchor, embeddings_positive), 1))
 
   # Add contrastive loss for the siamese network.
   #   label here is {0,1} for neg, pos.
diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops.py b/tensorflow/contrib/metrics/python/ops/metric_ops.py
index 7b432f8bd2..ac46fe69f2 100644
--- a/tensorflow/contrib/metrics/python/ops/metric_ops.py
+++ b/tensorflow/contrib/metrics/python/ops/metric_ops.py
@@ -1357,7 +1357,7 @@ def _compute_placement_auc(labels, predictions, weights, alpha,
       (total_0 - 1. + _EPSILON))
   var_1 = (
       math_ops.reduce_sum(
-          weights_1 * math_ops.square(placement_values_1 - auc_1)) /
+          weights_1 * math_ops.squared_difference(placement_values_1, auc_1)) /
       (total_1 - 1. + _EPSILON))
   auc_std_err = math_ops.sqrt(
       (var_0 / (total_0 + _EPSILON)) + (var_1 / (total_1 + _EPSILON)))
diff --git a/tensorflow/contrib/tensor_forest/client/eval_metrics.py b/tensorflow/contrib/tensor_forest/client/eval_metrics.py
index d8236a0a6f..0d87cea9fb 100644
--- a/tensorflow/contrib/tensor_forest/client/eval_metrics.py
+++ b/tensorflow/contrib/tensor_forest/client/eval_metrics.py
@@ -50,9 +50,10 @@ def _accuracy(predictions, targets, weights=None):
 def _r2(probabilities, targets, weights=None):
   targets = math_ops.to_float(targets)
   y_mean = math_ops.reduce_mean(targets, 0)
-  squares_total = math_ops.reduce_sum(math_ops.square(targets - y_mean), 0)
+  squares_total = math_ops.reduce_sum(
+      math_ops.squared_difference(targets, y_mean), 0)
   squares_residuals = math_ops.reduce_sum(
-      math_ops.square(targets - probabilities), 0)
+      math_ops.squared_difference(targets, probabilities), 0)
   score = 1 - math_ops.reduce_sum(squares_residuals / squares_total)
   return metrics.mean(score, weights=weights)
 
diff --git a/tensorflow/contrib/timeseries/python/timeseries/ar_model.py b/tensorflow/contrib/timeseries/python/timeseries/ar_model.py
index a8d5e1a49d..8cc56d3528 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/ar_model.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/ar_model.py
@@ -465,7 +465,8 @@ class ARModel(model.TimeSeriesModel):
           math_utils.normal_log_prob(targets, sigma, prediction))
     else:
       assert self.loss == ARModel.SQUARED_LOSS, self.loss
-      loss_op = math_ops.reduce_sum(math_ops.square(prediction - targets))
+      loss_op = math_ops.reduce_sum(
+        math_ops.squared_difference(prediction, targets))
     loss_op /= math_ops.cast(
         math_ops.reduce_prod(array_ops.shape(targets)), loss_op.dtype)
     return loss_op
diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index 01bd28587f..9fbd28fcf3 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -824,7 +824,7 @@ def mean_absolute_percentage_error(y_true, y_pred):
 def mean_squared_logarithmic_error(y_true, y_pred):
   first_log = math_ops.log(K.clip(y_pred, K.epsilon(), None) + 1.)
   second_log = math_ops.log(K.clip(y_true, K.epsilon(), None) + 1.)
-  return K.mean(math_ops.square(first_log - second_log), axis=-1)
+  return K.mean(math_ops.squared_difference(first_log, second_log), axis=-1)
 
 
 @keras_export('keras.metrics.squared_hinge', 'keras.losses.squared_hinge')
diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index f9e2f69f77..6aed0e7ac6 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -2081,7 +2081,7 @@ class RootMeanSquaredError(Mean):
     y_pred = math_ops.cast(y_pred, self._dtype)
     y_pred, y_true, sample_weight = squeeze_or_expand_dimensions(
         y_pred, y_true, sample_weight)
-    error_sq = math_ops.square(y_pred - y_true)
+    error_sq = math_ops.squared_difference(y_pred, y_true)
     return super(RootMeanSquaredError, self).update_state(
         error_sq, sample_weight=sample_weight)
 
diff --git a/tensorflow/python/keras/utils/kernelized_utils.py b/tensorflow/python/keras/utils/kernelized_utils.py
index 26ad59c9f2..2e73cb2d4c 100644
--- a/tensorflow/python/keras/utils/kernelized_utils.py
+++ b/tensorflow/python/keras/utils/kernelized_utils.py
@@ -83,7 +83,7 @@ def exact_gaussian_kernel(x, y, stddev):
   """
   x_aligned, y_aligned = _align_matrices(x, y)
   diff_squared_l2_norm = math_ops.reduce_sum(
-      math_ops.square(math_ops.subtract(x_aligned, y_aligned)), 2)
+      math_ops.squared_difference(x_aligned, y_aligned), 2)
   return math_ops.exp(-diff_squared_l2_norm / (2 * stddev * stddev))
 
 
diff --git a/tensorflow/python/ops/distributions/normal.py b/tensorflow/python/ops/distributions/normal.py
index 9acc046988..27a2b23bd0 100644
--- a/tensorflow/python/ops/distributions/normal.py
+++ b/tensorflow/python/ops/distributions/normal.py
@@ -291,5 +291,5 @@ def _kl_normal_normal(n_a, n_b, name=None):
     s_a_squared = math_ops.square(n_a.scale)
     s_b_squared = math_ops.square(n_b.scale)
     ratio = s_a_squared / s_b_squared
-    return (math_ops.square(n_a.loc - n_b.loc) / (two * s_b_squared) +
-            half * (ratio - one - math_ops.log(ratio)))
+    return (math_ops.squared_difference(n_a.loc, n_b.loc) /
+            (two * s_b_squared) + half * (ratio - one - math_ops.log(ratio)))
diff --git a/tensorflow/python/ops/metrics_impl.py b/tensorflow/python/ops/metrics_impl.py
index df9ca86ec7..e3cefb2e92 100644
--- a/tensorflow/python/ops/metrics_impl.py
+++ b/tensorflow/python/ops/metrics_impl.py
@@ -1295,7 +1295,7 @@ def mean_squared_error(labels,
 
   predictions, labels, weights = _remove_squeezable_dimensions(
       predictions=predictions, labels=labels, weights=weights)
-  squared_error = math_ops.square(labels - predictions)
+  squared_error = math_ops.squared_difference(labels, predictions)
   return mean(squared_error, weights, metrics_collections, updates_collections,
               name or 'mean_squared_error')
 
-- 
GitLab


From e4a09e402b3f57ad29d8dfee29b4106347cac8a0 Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Mon, 28 Jan 2019 05:02:40 -0800
Subject: [PATCH 1353/2345] Add integration test for a generative RNN exported
 as SavedModel.

This tests the following:
 - @tf.function with and without input_signature,
 - several Keras layers including layers.LSTMCell, layers.RNN, layers.Dense,
 - capturing of a directly declared tf.Variable,
 - text preprocessing ops,
 - differentiation w.r.t. all of the above variables.

PiperOrigin-RevId: 231199593
---
 .../saved_model/integration_tests/BUILD       |  36 ++++
 .../export_text_rnn_model.py                  | 193 ++++++++++++++++++
 .../integration_tests/saved_model_test.py     |  55 +++++
 .../integration_tests/use_text_rnn_model.py   |  50 +++++
 4 files changed, 334 insertions(+)
 create mode 100644 tensorflow/examples/saved_model/integration_tests/BUILD
 create mode 100644 tensorflow/examples/saved_model/integration_tests/export_text_rnn_model.py
 create mode 100644 tensorflow/examples/saved_model/integration_tests/saved_model_test.py
 create mode 100644 tensorflow/examples/saved_model/integration_tests/use_text_rnn_model.py

diff --git a/tensorflow/examples/saved_model/integration_tests/BUILD b/tensorflow/examples/saved_model/integration_tests/BUILD
new file mode 100644
index 0000000000..3e7a2b3f73
--- /dev/null
+++ b/tensorflow/examples/saved_model/integration_tests/BUILD
@@ -0,0 +1,36 @@
+licenses(["notice"])  # Apache 2.0
+
+exports_files(["LICENSE"])
+
+load("//tensorflow:tensorflow.bzl", "py_test")
+
+py_binary(
+    name = "export_text_rnn_model",
+    srcs = ["export_text_rnn_model.py"],
+    deps = [
+        "//tensorflow:tensorflow_py",
+    ],
+)
+
+py_binary(
+    name = "use_text_rnn_model",
+    srcs = ["use_text_rnn_model.py"],
+    deps = [
+        "//tensorflow:tensorflow_py",
+    ],
+)
+
+py_test(
+    name = "saved_model_test",
+    srcs = [
+        "saved_model_test.py",
+    ],
+    data = [
+        ":export_text_rnn_model",
+        ":use_text_rnn_model",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow:tensorflow_py",
+    ],
+)
diff --git a/tensorflow/examples/saved_model/integration_tests/export_text_rnn_model.py b/tensorflow/examples/saved_model/integration_tests/export_text_rnn_model.py
new file mode 100644
index 0000000000..32bdb284f2
--- /dev/null
+++ b/tensorflow/examples/saved_model/integration_tests/export_text_rnn_model.py
@@ -0,0 +1,193 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Text RNN model stored as a SavedModel."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl import app
+from absl import flags
+
+import tensorflow as tf
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_string("export_dir", None, "Directory to export SavedModel.")
+
+
+class TextRnnModel(tf.train.Checkpoint):
+  """Text RNN model.
+
+  A full generative text RNN model that can train and decode sentences from a
+  starting word.
+  """
+
+  def __init__(self, vocab, emb_dim, buckets, state_size):
+    super(TextRnnModel, self).__init__()
+    self._buckets = buckets
+    self._lstm_cell = tf.keras.layers.LSTMCell(units=state_size)
+    self._rnn_layer = tf.keras.layers.RNN(
+        self._lstm_cell, return_sequences=True)
+    self._embeddings = tf.Variable(tf.random.uniform(shape=[buckets, emb_dim]))
+    self._logit_layer = tf.keras.layers.Dense(buckets)
+    self._set_up_vocab(vocab)
+
+  def _tokenize(self, sentences):
+    # Perform a minimalistic text preprocessing by removing punctuation and
+    # splitting on spaces.
+    normalized_sentences = tf.strings.regex_replace(
+        input=sentences, pattern=r"\pP", rewrite="")
+    sparse_tokens = tf.string_split(normalized_sentences, " ")
+
+    # Deal with a corner case: there is one empty sentence.
+    sparse_tokens, _ = tf.sparse.fill_empty_rows(sparse_tokens, tf.constant(""))
+    # Deal with a corner case: all sentences are empty.
+    sparse_tokens = tf.sparse.reset_shape(sparse_tokens)
+
+    return (sparse_tokens.indices, sparse_tokens.values,
+            sparse_tokens.dense_shape)
+
+  def _set_up_vocab(self, vocab_tokens):
+    # TODO(vbardiovsky): Currently there is no real vocabulary, because
+    # saved_model serialization does not support trackable resources. Add a real
+    # vocabulary when it does.
+    vocab_list = ["UNK"] * self._buckets
+    for vocab_token in vocab_tokens:
+      index = self._words_to_indices(vocab_token).numpy()
+      vocab_list[index] = vocab_token
+    # This is a variable representing an inverse index.
+    self._vocab_tensor = tf.Variable(vocab_list)
+
+  def _indices_to_words(self, indices):
+    return tf.gather(self._vocab_tensor, indices)
+
+  def _words_to_indices(self, words):
+    return tf.strings.to_hash_bucket(words, self._buckets)
+
+  @tf.function(input_signature=[tf.TensorSpec([None], tf.dtypes.string)])
+  def train(self, sentences):
+    token_ids, token_values, token_dense_shape = self._tokenize(sentences)
+    tokens_sparse = tf.sparse.SparseTensor(
+        indices=token_ids, values=token_values, dense_shape=token_dense_shape)
+    tokens = tf.sparse.to_dense(tokens_sparse, default_value="")
+
+    sparse_lookup_ids = tf.sparse.SparseTensor(
+        indices=tokens_sparse.indices,
+        values=self._words_to_indices(tokens_sparse.values),
+        dense_shape=tokens_sparse.dense_shape)
+    lookup_ids = tf.sparse.to_dense(sparse_lookup_ids, default_value=0)
+
+    # Targets are the next word for each word of the sentence.
+    tokens_ids_seq = lookup_ids[:, 0:-1]
+    tokens_ids_target = lookup_ids[:, 1:]
+
+    tokens_prefix = tokens[:, 0:-1]
+
+    # Mask determining which positions we care about for a loss: all positions
+    # that have a valid non-terminal token.
+    mask = tf.logical_and(
+        tf.logical_not(tf.equal(tokens_prefix, "")),
+        tf.logical_not(tf.equal(tokens_prefix, "<E>")))
+
+    input_mask = tf.cast(mask, tf.int32)
+
+    with tf.GradientTape() as t:
+      sentence_embeddings = tf.nn.embedding_lookup(self._embeddings,
+                                                   tokens_ids_seq)
+
+      lstm_initial_state = self._lstm_cell.get_initial_state(
+          sentence_embeddings)
+
+      lstm_output = self._rnn_layer(
+          inputs=sentence_embeddings, initial_state=lstm_initial_state)
+
+      # Stack LSTM outputs into a batch instead of a 2D array.
+      lstm_output = tf.reshape(lstm_output, [-1, self._lstm_cell.output_size])
+
+      logits = self._logit_layer(lstm_output)
+
+      targets = tf.reshape(tokens_ids_target, [-1])
+      weights = tf.cast(tf.reshape(input_mask, [-1]), tf.float32)
+
+      losses = tf.nn.sparse_softmax_cross_entropy_with_logits(
+          labels=targets, logits=logits)
+
+      # Final loss is the mean loss for all token losses.
+      final_loss = tf.math.divide(
+          tf.reduce_sum(tf.multiply(losses, weights)),
+          tf.reduce_sum(weights),
+          name="final_loss")
+
+    watched = t.watched_variables()
+    gradients = t.gradient(final_loss, watched)
+
+    for w, g in zip(watched, gradients):
+      w.assign_sub(g)
+
+    return final_loss
+
+  @tf.function
+  def decode_greedy(self, sequence_length, first_word):
+    initial_state = self._lstm_cell.get_initial_state(
+        dtype=tf.float32, batch_size=1)
+
+    sequence = [first_word]
+    current_word = first_word
+    current_id = tf.expand_dims(self._words_to_indices(current_word), 0)
+    current_state = initial_state
+
+    for _ in range(sequence_length):
+      token_embeddings = tf.nn.embedding_lookup(self._embeddings, current_id)
+      lstm_outputs, current_state = self._lstm_cell(token_embeddings,
+                                                    current_state)
+      lstm_outputs = tf.reshape(lstm_outputs, [-1, self._lstm_cell.output_size])
+      logits = self._logit_layer(lstm_outputs)
+      softmax = tf.nn.softmax(logits)
+
+      next_ids = tf.math.argmax(softmax, axis=1)
+      next_words = self._indices_to_words(next_ids)[0]
+
+      current_id = next_ids
+      current_word = next_words
+      sequence.append(current_word)
+
+    return sequence
+
+
+def main(argv):
+  del argv
+
+  sentences = ["<S> hello there <E>", "<S> how are you doing today <E>"]
+  vocab = [
+      "<S>", "<E>", "hello", "there", "how", "are", "you", "doing", "today"
+  ]
+
+  module = TextRnnModel(vocab=vocab, emb_dim=10, buckets=100, state_size=128)
+
+  for _ in range(100):
+    _ = module.train(tf.constant(sentences))
+
+  # We have to call this function explicitly if we want it exported, because it
+  # has no input_signature in the @tf.function decorator.
+  decoded = module.decode_greedy(
+      sequence_length=10, first_word=tf.constant("<S>"))
+  _ = [d.numpy() for d in decoded]
+
+  tf.saved_model.save(module, FLAGS.export_dir)
+
+
+if __name__ == "__main__":
+  app.run(main)
diff --git a/tensorflow/examples/saved_model/integration_tests/saved_model_test.py b/tensorflow/examples/saved_model/integration_tests/saved_model_test.py
new file mode 100644
index 0000000000..20cf09fa06
--- /dev/null
+++ b/tensorflow/examples/saved_model/integration_tests/saved_model_test.py
@@ -0,0 +1,55 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""SavedModel integration tests."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import subprocess
+
+import tensorflow as tf
+
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import resource_loader
+from tensorflow.python.platform import tf_logging as logging
+
+
+class SavedModelTest(tf.test.TestCase):
+
+  def assertCommandSucceeded(self, binary, **flags):
+    command_parts = [binary]
+    for flag_key, flag_value in flags.items():
+      command_parts.append("--%s=%s" % (flag_key, flag_value))
+
+    logging.info("Running: %s" % command_parts)
+    subprocess.check_call(
+        command_parts, env=dict(os.environ, TF2_BEHAVIOR="enabled"))
+
+  @test_util.run_v2_only
+  def test_text_rnn(self):
+    export_dir = self.get_temp_dir()
+    export_binary = resource_loader.get_path_to_datafile(
+        "export_text_rnn_model")
+    self.assertCommandSucceeded(export_binary, export_dir=export_dir)
+
+    use_binary = resource_loader.get_path_to_datafile("use_text_rnn_model")
+    self.assertCommandSucceeded(use_binary, model_dir=export_dir)
+
+
+if __name__ == "__main__":
+  # tf.enable_v2_behavior()
+  tf.test.main()
diff --git a/tensorflow/examples/saved_model/integration_tests/use_text_rnn_model.py b/tensorflow/examples/saved_model/integration_tests/use_text_rnn_model.py
new file mode 100644
index 0000000000..3f75c49a32
--- /dev/null
+++ b/tensorflow/examples/saved_model/integration_tests/use_text_rnn_model.py
@@ -0,0 +1,50 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Load and use RNN model stored as a SavedModel."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl import app
+from absl import flags
+
+import tensorflow as tf
+# TODO(vbardiovsky): Remove when load is available.
+from tensorflow.python.saved_model.load import load
+
+tf.saved_model.load = load
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_string("model_dir", None, "Directory to load SavedModel from.")
+
+
+def main(argv):
+  del argv
+
+  sentences = [
+      "<S> sentence <E>", "<S> second sentence <E>", "<S> third sentence<E>"
+  ]
+
+  model = tf.saved_model.load(FLAGS.model_dir)
+  model.train(tf.constant(sentences))
+  decoded = model.decode_greedy(
+      sequence_length=10, first_word=tf.constant("<S>"))
+  _ = [d.numpy() for d in decoded]
+
+
+if __name__ == "__main__":
+  app.run(main)
-- 
GitLab


From 15971827de11886f662a91849d6b9097426b5c28 Mon Sep 17 00:00:00 2001
From: Grzegorz Pawelczak <grzegorzp@graphcore.ai>
Date: Mon, 28 Jan 2019 13:40:58 +0000
Subject: [PATCH 1354/2345] Create the epsilon with the right type

---
 tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc b/tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc
index 0e2f335f33..4664ab0507 100644
--- a/tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc
@@ -159,8 +159,8 @@ class FusedBatchNormGradOp : public XlaOpKernel {
 
       // scratch1 = rsqrt(pop_var + epsilon)
       auto neg_half = XlaHelpers::FloatLiteral(b, scale_dtype, -0.5);
-      auto scratch1 = xla::Pow(
-          xla::Add(var, xla::ConstantR0<float>(b, epsilon_)), neg_half);
+      auto epsilon = XlaHelpers::FloatLiteral(b, scale_dtype, epsilon_);
+      auto scratch1 = xla::Pow(xla::Add(var, epsilon), neg_half);
 
       // scratch2 = sum(y_backprop * (x - mean))
       auto mul =
-- 
GitLab


From e89b1a0999a9c1ececded21dbf7079a92ace6bee Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 28 Jan 2019 06:36:00 -0800
Subject: [PATCH 1355/2345] Fix estimator dependency.

This needs to be cherry-picked to TF r1.13 branch.

PiperOrigin-RevId: 231209093
---
 tensorflow/tools/pip_package/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index 678531643c..0137f9da43 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -58,7 +58,7 @@ REQUIRED_PACKAGES = [
     'six >= 1.10.0',
     'protobuf >= 3.6.1',
     'tensorboard >= 1.12.0, < 1.13.0',
-    'tensorflow_estimator >= 1.13.0 < 1.14.0',
+    'tensorflow_estimator >= 1.13.0, < 1.14.0',
     'termcolor >= 1.1.0',
 ]
 
-- 
GitLab


From d14effce2c9c45a92b0f3871bd73ba921bb2c4f9 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Mon, 28 Jan 2019 06:44:05 -0800
Subject: [PATCH 1356/2345] Replace unordered_map with absl::flat_hash_map.

PiperOrigin-RevId: 231209960
---
 tensorflow/compiler/xla/service/BUILD                |  1 +
 tensorflow/compiler/xla/service/hlo_computation.cc   | 12 ++++++------
 tensorflow/compiler/xla/service/hlo_computation.h    |  4 ++--
 .../compiler/xla/service/hlo_computation_test.cc     |  4 ++--
 .../compiler/xla/service/while_loop_simplifier.cc    |  4 ++--
 5 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 3c123fd5c6..8a92a7005e 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -2164,6 +2164,7 @@ tf_cc_test(
         "//tensorflow/compiler/xla:test_helpers",
         "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+        "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/container:flat_hash_set",
     ],
 )
diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc
index 98164752b9..40fe91398b 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation.cc
@@ -857,15 +857,15 @@ Status HloComputation::Accept(
 std::unique_ptr<HloComputation> HloComputation::Clone(
     const string& suffix, HloCloneContext* context) {
   return CloneWithReplacements(
-      /*replacements=*/std::unordered_map<const HloInstruction*,
-                                          std::unique_ptr<HloInstruction>>(),
+      /*replacements=*/absl::flat_hash_map<const HloInstruction*,
+                                           std::unique_ptr<HloInstruction>>(),
       /*extra_parameters=*/{}, context, suffix);
 }
 
 std::unique_ptr<HloComputation> HloComputation::CloneWithReplacementPairs(
     std::pair<const HloInstruction*, std::unique_ptr<HloInstruction>> r1,
     HloCloneContext* context, const string& suffix) {
-  std::unordered_map<const HloInstruction*, std::unique_ptr<HloInstruction>>
+  absl::flat_hash_map<const HloInstruction*, std::unique_ptr<HloInstruction>>
       replacements;
   replacements.emplace(std::move(r1));
   return CloneWithReplacements(std::move(replacements), /*extra_parameters=*/{},
@@ -876,7 +876,7 @@ std::unique_ptr<HloComputation> HloComputation::CloneWithReplacementPairs(
     std::pair<const HloInstruction*, std::unique_ptr<HloInstruction>> r1,
     std::pair<const HloInstruction*, std::unique_ptr<HloInstruction>> r2,
     HloCloneContext* context, const string& suffix) {
-  std::unordered_map<const HloInstruction*, std::unique_ptr<HloInstruction>>
+  absl::flat_hash_map<const HloInstruction*, std::unique_ptr<HloInstruction>>
       replacements;
   replacements.emplace(std::move(r1));
   replacements.emplace(std::move(r2));
@@ -889,7 +889,7 @@ std::unique_ptr<HloComputation> HloComputation::CloneWithReplacementPairs(
     std::pair<const HloInstruction*, std::unique_ptr<HloInstruction>> r2,
     std::pair<const HloInstruction*, std::unique_ptr<HloInstruction>> r3,
     HloCloneContext* context, const string& suffix) {
-  std::unordered_map<const HloInstruction*, std::unique_ptr<HloInstruction>>
+  absl::flat_hash_map<const HloInstruction*, std::unique_ptr<HloInstruction>>
       replacements;
   replacements.emplace(std::move(r1));
   replacements.emplace(std::move(r2));
@@ -899,7 +899,7 @@ std::unique_ptr<HloComputation> HloComputation::CloneWithReplacementPairs(
 }
 
 std::unique_ptr<HloComputation> HloComputation::CloneWithReplacements(
-    std::unordered_map<const HloInstruction*, std::unique_ptr<HloInstruction>>
+    absl::flat_hash_map<const HloInstruction*, std::unique_ptr<HloInstruction>>
         replacements,
     absl::Span<const HloInstruction* const> extra_parameters,
     HloCloneContext* context, const string& suffix) {
diff --git a/tensorflow/compiler/xla/service/hlo_computation.h b/tensorflow/compiler/xla/service/hlo_computation.h
index e6a1eb89cf..0cb9caddd0 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.h
+++ b/tensorflow/compiler/xla/service/hlo_computation.h
@@ -20,7 +20,6 @@ limitations under the License.
 #include <list>
 #include <memory>
 #include <string>
-#include <unordered_map>
 #include <unordered_set>
 #include <utility>
 #include <vector>
@@ -329,7 +328,8 @@ class HloComputation {
   // All relevant instructions are cloned, *including* unique_ptr in the
   // `replacements` map.
   std::unique_ptr<HloComputation> CloneWithReplacements(
-      std::unordered_map<const HloInstruction*, std::unique_ptr<HloInstruction>>
+      absl::flat_hash_map<const HloInstruction*,
+                          std::unique_ptr<HloInstruction>>
           replacements,
       absl::Span<const HloInstruction* const> extra_parameters = {},
       HloCloneContext* context = nullptr, const string& suffix = "clone");
diff --git a/tensorflow/compiler/xla/service/hlo_computation_test.cc b/tensorflow/compiler/xla/service/hlo_computation_test.cc
index 35ee532704..3b88e9745c 100644
--- a/tensorflow/compiler/xla/service/hlo_computation_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation_test.cc
@@ -17,9 +17,9 @@ limitations under the License.
 
 #include <memory>
 #include <set>
-#include <unordered_map>
 #include <vector>
 
+#include "absl/container/flat_hash_map.h"
 #include "absl/container/flat_hash_set.h"
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h"
@@ -511,7 +511,7 @@ TEST_F(HloComputationTest, CloneWithReplacements) {
   auto module = CreateNewVerifiedModule();
   auto computation =
       module->AddEntryComputation(builder.Build(/*root_instruction=*/lt));
-  std::unordered_map<const HloInstruction*, std::unique_ptr<HloInstruction>>
+  absl::flat_hash_map<const HloInstruction*, std::unique_ptr<HloInstruction>>
       replacements;
   replacements.emplace(param2,
                        HloInstruction::CreateParameter(2, r0s32, "p.1"));
diff --git a/tensorflow/compiler/xla/service/while_loop_simplifier.cc b/tensorflow/compiler/xla/service/while_loop_simplifier.cc
index 09d5409571..386ffb9954 100644
--- a/tensorflow/compiler/xla/service/while_loop_simplifier.cc
+++ b/tensorflow/compiler/xla/service/while_loop_simplifier.cc
@@ -180,7 +180,7 @@ static StatusOr<bool> TryRemoveDeadWhileParams(HloInstruction* while_op) {
   // replace the old instructions after we remove unused elements from the while
   // tuple.
   auto make_while_computation_replacements = [&](const HloComputation* comp) {
-    std::unordered_map<const HloInstruction*, std::unique_ptr<HloInstruction>>
+    absl::flat_hash_map<const HloInstruction*, std::unique_ptr<HloInstruction>>
         replacements;
 
     auto* param = comp->parameter_instruction(0);
@@ -232,7 +232,7 @@ static StatusOr<bool> TryRemoveDeadWhileParams(HloInstruction* while_op) {
       while_cond->CloneWithReplacements(
           make_while_computation_replacements(while_cond));
 
-  std::unordered_map<const HloInstruction*, std::unique_ptr<HloInstruction>>
+  absl::flat_hash_map<const HloInstruction*, std::unique_ptr<HloInstruction>>
       while_body_replacements = make_while_computation_replacements(while_body);
   std::vector<HloInstruction*> new_while_body_root_elems;
   new_while_body_root_elems.reserve(new_to_old_tuple_idx.size());
-- 
GitLab


From 505382d99af4bcdef608d9d3e80f776470e9a53e Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Mon, 28 Jan 2019 07:44:06 -0800
Subject: [PATCH 1357/2345] Actually use dynamic dispatch for the print
 overload.

PiperOrigin-RevId: 231217350
---
 .../autograph/converters/builtin_functions_test.py  |  4 +++-
 .../python/autograph/operators/py_builtins.py       | 13 +++++++++++--
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/autograph/converters/builtin_functions_test.py b/tensorflow/python/autograph/converters/builtin_functions_test.py
index 2683be16ec..2e6cf16b9c 100644
--- a/tensorflow/python/autograph/converters/builtin_functions_test.py
+++ b/tensorflow/python/autograph/converters/builtin_functions_test.py
@@ -55,7 +55,9 @@ class BuiltinFunctionsTest(converter_testing.TestCase):
     with self.converted(test_fn, builtin_functions, {'print': print}) as result:
       with self.session() as sess:
         with self.assertPrints('a\n'):
-          sess.run(result.test_fn('a'))
+          sess.run(result.test_fn(constant_op.constant('a')))
+      with self.assertPrints('a\n'):
+        result.test_fn('a')
 
   @test_util.run_deprecated_v1
   def test_print_multiple_values(self):
diff --git a/tensorflow/python/autograph/operators/py_builtins.py b/tensorflow/python/autograph/operators/py_builtins.py
index 9f54cf4fe5..fe9486ca1e 100644
--- a/tensorflow/python/autograph/operators/py_builtins.py
+++ b/tensorflow/python/autograph/operators/py_builtins.py
@@ -155,14 +155,23 @@ def _py_len(s):
 
 
 def print_(*objects, **kwargs):
+  """Overload of the print builtin."""
   # Note: Python 2.6 doesn't support explicit keywords after starargs.
   unknown_kwargs = tuple(
       set(kwargs.keys()) - set(('sep', 'end', 'file', 'flush')))
   if unknown_kwargs:
     raise ValueError('invalid keyword arguments: {}'.format(unknown_kwargs))
 
-  # TODO(mdan): use logging_ops.Print when py_func is not supported.
-  return _tf_py_func_print(objects, kwargs)
+  # TODO(mdan): Use next.flatten(objects) instead?
+  if any(tensor_util.is_tensor(o) for o in objects):
+    # TODO(mdan): use tf.print instead.
+    return _tf_py_func_print(objects, kwargs)
+  else:
+    _py_print(*objects, **kwargs)
+
+
+def _py_print(*objects, **kwargs):
+  print(*objects, **kwargs)
 
 
 def _tf_py_func_print(objects, kwargs):
-- 
GitLab


From 80a2144b58ac183ea72f86f1918254fb1a38fb4a Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Mon, 28 Jan 2019 07:48:27 -0800
Subject: [PATCH 1358/2345] Make it easier to disable specific features.

PiperOrigin-RevId: 231217872
---
 tensorflow/python/autograph/core/converter.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tensorflow/python/autograph/core/converter.py b/tensorflow/python/autograph/core/converter.py
index 79f96fa90e..870c213e63 100644
--- a/tensorflow/python/autograph/core/converter.py
+++ b/tensorflow/python/autograph/core/converter.py
@@ -129,6 +129,18 @@ class Feature(enum.Enum):
   LISTS = 'LISTS'
   NAME_SCOPES = 'NAME_SCOPES'
 
+  @classmethod
+  def all(cls):
+    """Returns a tuple that enables all options."""
+    return tuple(cls.__members__.values())
+
+  @classmethod
+  def all_but(cls, exclude):
+    """Returns a tuple that enables all but the excluded options."""
+    if not isinstance(exclude, (list, tuple, set)):
+      exclude = (exclude,)
+    return tuple(set(cls.all()) - set(exclude) - {cls.ALL})
+
 
 class ConversionOptions(object):
   """Immutable container for global conversion flags.
-- 
GitLab


From 12114318861cfc71c8b382b66e48438add149504 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Mon, 28 Jan 2019 16:00:38 +0000
Subject: [PATCH 1359/2345] Add jupyter_http_over_ws as part of jupyter install
 in Dockerfile

This fix tries to address the issue raised in 25247 to add
jupyter_http_over_ws as part of jupyter install in Dockerfile.

This fix fixes 25247.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .../tools/dockerfiles/partials/jupyter.partial.Dockerfile       | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/tools/dockerfiles/partials/jupyter.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/jupyter.partial.Dockerfile
index c4ec6095c0..c056d915d6 100644
--- a/tensorflow/tools/dockerfiles/partials/jupyter.partial.Dockerfile
+++ b/tensorflow/tools/dockerfiles/partials/jupyter.partial.Dockerfile
@@ -1,4 +1,6 @@
 RUN ${PIP} install jupyter matplotlib
+RUN ${PIP} install jupyter_http_over_ws
+RUN jupyter serverextension enable --py jupyter_http_over_ws
 
 RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/
 RUN mkdir /.local && chmod a+rwx /.local
-- 
GitLab


From 8c84164c095fa193403733c4de646fa5e84d5071 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Mon, 28 Jan 2019 16:09:29 +0000
Subject: [PATCH 1360/2345] Rebuild dockerfiles from assembly

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .../tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile      | 2 ++
 .../dockerfiles/dockerfiles/devel-cpu-jupyter.Dockerfile      | 4 +++-
 tensorflow/tools/dockerfiles/dockerfiles/devel-cpu.Dockerfile | 2 +-
 .../dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile      | 2 ++
 .../tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile      | 2 ++
 5 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile
index d8fabadec2..c09a95858b 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile
@@ -54,6 +54,8 @@ COPY bashrc /etc/bash.bashrc
 RUN chmod a+rwx /etc/bash.bashrc
 
 RUN ${PIP} install jupyter matplotlib
+RUN ${PIP} install jupyter_http_over_ws
+RUN jupyter serverextension enable --py jupyter_http_over_ws
 
 RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/
 RUN mkdir /.local && chmod a+rwx /.local
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu-jupyter.Dockerfile
index c1f6dafbe0..099c2dc62e 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu-jupyter.Dockerfile
@@ -46,7 +46,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
  
 ENV CI_BUILD_PYTHON python
 
-# Check out TensorFlow source code if --build_arg CHECKOUT_TENSORFLOW=1
+# Check out TensorFlow source code if --build-arg CHECKOUT_TF_SRC=1
 ARG CHECKOUT_TF_SRC=0
 RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src
 
@@ -105,6 +105,8 @@ COPY bashrc /etc/bash.bashrc
 RUN chmod a+rwx /etc/bash.bashrc
 
 RUN ${PIP} install jupyter matplotlib
+RUN ${PIP} install jupyter_http_over_ws
+RUN jupyter serverextension enable --py jupyter_http_over_ws
 
 RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/
 RUN mkdir /.local && chmod a+rwx /.local
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu.Dockerfile
index b4dfc8b099..a4017634fe 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu.Dockerfile
@@ -46,7 +46,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
  
 ENV CI_BUILD_PYTHON python
 
-# Check out TensorFlow source code if --build_arg CHECKOUT_TENSORFLOW=1
+# Check out TensorFlow source code if --build-arg CHECKOUT_TF_SRC=1
 ARG CHECKOUT_TF_SRC=0
 RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src
 
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile
index 6d76c06332..44481eb16a 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile
@@ -125,6 +125,8 @@ COPY bashrc /etc/bash.bashrc
 RUN chmod a+rwx /etc/bash.bashrc
 
 RUN ${PIP} install jupyter matplotlib
+RUN ${PIP} install jupyter_http_over_ws
+RUN jupyter serverextension enable --py jupyter_http_over_ws
 
 RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/
 RUN mkdir /.local && chmod a+rwx /.local
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile
index 46252c5413..088310755e 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile
@@ -82,6 +82,8 @@ COPY bashrc /etc/bash.bashrc
 RUN chmod a+rwx /etc/bash.bashrc
 
 RUN ${PIP} install jupyter matplotlib
+RUN ${PIP} install jupyter_http_over_ws
+RUN jupyter serverextension enable --py jupyter_http_over_ws
 
 RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/
 RUN mkdir /.local && chmod a+rwx /.local
-- 
GitLab


From cb49fa39e5ac2a5ea66720ae60b8b20aebad39a1 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Mon, 28 Jan 2019 08:30:00 -0800
Subject: [PATCH 1361/2345] Always raise internal error. In a subsequent change
 we can raise syntax errors separately. Add more logging.

PiperOrigin-RevId: 231223885
---
 .../python/autograph/impl/conversion.py       | 13 ++++---
 tensorflow/python/autograph/pyct/BUILD        |  1 +
 tensorflow/python/autograph/pyct/errors.py    | 37 +++++++++++++++++++
 .../python/autograph/pyct/origin_info.py      |  6 +++
 .../python/autograph/utils/ag_logging.py      |  7 ++++
 5 files changed, 58 insertions(+), 6 deletions(-)
 create mode 100644 tensorflow/python/autograph/pyct/errors.py

diff --git a/tensorflow/python/autograph/impl/conversion.py b/tensorflow/python/autograph/impl/conversion.py
index 3a901dfd38..1cd5671eae 100644
--- a/tensorflow/python/autograph/impl/conversion.py
+++ b/tensorflow/python/autograph/impl/conversion.py
@@ -43,11 +43,12 @@ from tensorflow.python.autograph.converters import side_effect_guards
 from tensorflow.python.autograph.converters import slices
 from tensorflow.python.autograph.core import config
 from tensorflow.python.autograph.core import converter
-from tensorflow.python.autograph.core import errors
+from tensorflow.python.autograph.core import errors as ag_errors
 from tensorflow.python.autograph.core import function_wrapping
 from tensorflow.python.autograph.lang import special_functions
 from tensorflow.python.autograph.pyct import ast_util
 from tensorflow.python.autograph.pyct import compiler
+from tensorflow.python.autograph.pyct import errors
 from tensorflow.python.autograph.pyct import inspect_utils
 from tensorflow.python.autograph.pyct import origin_info
 from tensorflow.python.autograph.pyct import parser
@@ -322,7 +323,7 @@ def _add_self_references(namespace, autograph_module):
     ag_internal.utils = utils
     ag_internal.function_scope = function_wrapping.function_scope
     ag_internal.rewrite_graph_construction_error = (
-        errors.rewrite_graph_construction_error)
+        ag_errors.rewrite_graph_construction_error)
     # TODO(mdan): Add safeguards against name clashes.
     # We don't want to create a submodule because we want the operators to be
     # accessible as ag__.<operator>
@@ -340,6 +341,7 @@ def function_to_graph(f,
   """Specialization of `entity_to_graph` for callable functions."""
 
   node, source = parser.parse_entity(f)
+  logging.log(3, 'Source code of %s:\n%s', f, source)
   node = node.body[0]
 
   # In general, the output of inspect.getsource is inexact because it uses
@@ -380,10 +382,9 @@ def function_to_graph(f,
   try:
     node = node_to_graph(node, context)
   except (ValueError, AttributeError, KeyError, NotImplementedError) as e:
-    # TODO(znado): raise InternalError
-    if not context.current_origin:
-      raise e
-    raise transformer.AutoGraphParseError(str(e), context.current_origin)
+    logging.error(1, 'Error converting %s', f, exc_info=True)
+    raise errors.InternalError('conversion', e)
+    # TODO(mdan): Catch and rethrow syntax errors.
 
   if isinstance(node, gast.Lambda):
     new_name = namer.new_symbol('tf__lambda', ())
diff --git a/tensorflow/python/autograph/pyct/BUILD b/tensorflow/python/autograph/pyct/BUILD
index ba8ec27139..e6bff2d719 100644
--- a/tensorflow/python/autograph/pyct/BUILD
+++ b/tensorflow/python/autograph/pyct/BUILD
@@ -24,6 +24,7 @@ py_library(
         "ast_util.py",
         "cfg.py",
         "compiler.py",
+        "errors.py",
         "inspect_utils.py",
         "origin_info.py",
         "parser.py",
diff --git a/tensorflow/python/autograph/pyct/errors.py b/tensorflow/python/autograph/pyct/errors.py
new file mode 100644
index 0000000000..4fc9c168c0
--- /dev/null
+++ b/tensorflow/python/autograph/pyct/errors.py
@@ -0,0 +1,37 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Code transformation exceptions."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+class AutoGraphError(Exception):
+  pass
+
+
+class InternalError(AutoGraphError):
+
+  def __init__(self, message, original_exc):
+    super(InternalError, self).__init__()
+    self.message = message
+    self.original_exc = original_exc
+
+  def __str__(self):
+    return '{} during {}: {}'.format(
+        type(self.original_exc).__name__, self.message, self.original_exc)
+
+
diff --git a/tensorflow/python/autograph/pyct/origin_info.py b/tensorflow/python/autograph/pyct/origin_info.py
index 102bd42c91..230468eab9 100644
--- a/tensorflow/python/autograph/pyct/origin_info.py
+++ b/tensorflow/python/autograph/pyct/origin_info.py
@@ -18,6 +18,7 @@ from __future__ import division
 from __future__ import print_function
 
 import collections
+import os
 import tokenize
 
 import gast
@@ -73,6 +74,11 @@ class OriginInfo(
     return (self.loc.filename, self.loc.lineno, self.function_name,
             self.source_code_line)
 
+  def __repr__(self):
+    return '{}:{}:{}'.format(
+        os.path.split(self.loc.filename)[1], self.loc.lineno,
+        self.loc.col_offset)
+
 
 # TODO(mdan): This source map should be a class - easier to refer to.
 def create_source_map(nodes, code, filename, indices_in_code):
diff --git a/tensorflow/python/autograph/utils/ag_logging.py b/tensorflow/python/autograph/utils/ag_logging.py
index 17fc8f58be..cc51e341f4 100644
--- a/tensorflow/python/autograph/utils/ag_logging.py
+++ b/tensorflow/python/autograph/utils/ag_logging.py
@@ -117,6 +117,13 @@ def has_verbosity(level):
   return get_verbosity() >= level
 
 
+def error(level, msg, *args, **kwargs):
+  if has_verbosity(level):
+    logging.error(msg, *args, **kwargs)
+    if echo_log_to_stdout:
+      print(msg % args)
+
+
 def log(level, msg, *args, **kwargs):
   if has_verbosity(level):
     logging.info(msg, *args, **kwargs)
-- 
GitLab


From 7fa94aa9a79f151cc758d16fc6a293c4026ae4f7 Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Mon, 28 Jan 2019 08:39:29 -0800
Subject: [PATCH 1362/2345] Add integration test for an RNN cell exported as
 SavedModel.

PiperOrigin-RevId: 231225341
---
 .../saved_model/integration_tests/BUILD       | 18 ++++++
 .../integration_tests/export_rnn_cell.py      | 63 +++++++++++++++++++
 .../integration_tests/saved_model_test.py     |  9 +++
 .../integration_tests/use_rnn_cell.py         | 49 +++++++++++++++
 4 files changed, 139 insertions(+)
 create mode 100644 tensorflow/examples/saved_model/integration_tests/export_rnn_cell.py
 create mode 100644 tensorflow/examples/saved_model/integration_tests/use_rnn_cell.py

diff --git a/tensorflow/examples/saved_model/integration_tests/BUILD b/tensorflow/examples/saved_model/integration_tests/BUILD
index 3e7a2b3f73..f40c6d9d28 100644
--- a/tensorflow/examples/saved_model/integration_tests/BUILD
+++ b/tensorflow/examples/saved_model/integration_tests/BUILD
@@ -20,13 +20,31 @@ py_binary(
     ],
 )
 
+py_binary(
+    name = "export_rnn_cell",
+    srcs = ["export_rnn_cell.py"],
+    deps = [
+        "//tensorflow:tensorflow_py",
+    ],
+)
+
+py_binary(
+    name = "use_rnn_cell",
+    srcs = ["use_rnn_cell.py"],
+    deps = [
+        "//tensorflow:tensorflow_py",
+    ],
+)
+
 py_test(
     name = "saved_model_test",
     srcs = [
         "saved_model_test.py",
     ],
     data = [
+        ":export_rnn_cell",
         ":export_text_rnn_model",
+        ":use_rnn_cell",
         ":use_text_rnn_model",
     ],
     srcs_version = "PY2AND3",
diff --git a/tensorflow/examples/saved_model/integration_tests/export_rnn_cell.py b/tensorflow/examples/saved_model/integration_tests/export_rnn_cell.py
new file mode 100644
index 0000000000..f1ae38b813
--- /dev/null
+++ b/tensorflow/examples/saved_model/integration_tests/export_rnn_cell.py
@@ -0,0 +1,63 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Export an RNN cell in SavedModel format."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl import app
+from absl import flags
+import numpy as np
+
+import tensorflow as tf
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_string("export_dir", None, "Directory to export SavedModel.")
+
+
+def main(argv):
+  del argv
+
+  root = tf.train.Checkpoint()
+  # Create a cell and attach to our checkpointable.
+  root.rnn_cell = tf.keras.layers.LSTMCell(units=10, recurrent_initializer=None)
+
+  # Wrap the rnn_cell.__call__ function and assign to next_state.
+  root.next_state = tf.function(root.rnn_cell.__call__, autograph=False)
+
+  # Wrap the rnn_cell.get_initial_function using a decorator and assign to an
+  # attribute with the same name.
+  @tf.function(input_signature=[tf.TensorSpec([None, None], tf.float32)])
+  def get_initial_state(tensor):
+    return root.rnn_cell.get_initial_state(tensor, None, None)
+
+  root.get_initial_state = get_initial_state
+
+  # Construct an initial_state, then call next_state explicitly to trigger a
+  # trace for serialization (we need an explicit call, because next_state has
+  # not been annotated with an input_signature).
+  initial_state = root.get_initial_state(
+      tf.constant(np.random.uniform(size=[3, 10]).astype(np.float32)))
+  root.next_state(
+      tf.constant(np.random.uniform(size=[3, 19]).astype(np.float32)),
+      initial_state)
+
+  tf.saved_model.save(root, FLAGS.export_dir)
+
+
+if __name__ == "__main__":
+  app.run(main)
diff --git a/tensorflow/examples/saved_model/integration_tests/saved_model_test.py b/tensorflow/examples/saved_model/integration_tests/saved_model_test.py
index 20cf09fa06..770e809ffc 100644
--- a/tensorflow/examples/saved_model/integration_tests/saved_model_test.py
+++ b/tensorflow/examples/saved_model/integration_tests/saved_model_test.py
@@ -49,6 +49,15 @@ class SavedModelTest(tf.test.TestCase):
     use_binary = resource_loader.get_path_to_datafile("use_text_rnn_model")
     self.assertCommandSucceeded(use_binary, model_dir=export_dir)
 
+  @test_util.run_v2_only
+  def test_rnn_cell(self):
+    export_dir = self.get_temp_dir()
+    export_binary = resource_loader.get_path_to_datafile(
+        "export_rnn_cell")
+    self.assertCommandSucceeded(export_binary, export_dir=export_dir)
+
+    use_binary = resource_loader.get_path_to_datafile("use_rnn_cell")
+    self.assertCommandSucceeded(use_binary, model_dir=export_dir)
 
 if __name__ == "__main__":
   # tf.enable_v2_behavior()
diff --git a/tensorflow/examples/saved_model/integration_tests/use_rnn_cell.py b/tensorflow/examples/saved_model/integration_tests/use_rnn_cell.py
new file mode 100644
index 0000000000..798033517c
--- /dev/null
+++ b/tensorflow/examples/saved_model/integration_tests/use_rnn_cell.py
@@ -0,0 +1,49 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Load and use an RNN cell stored as a SavedModel."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl import app
+from absl import flags
+import numpy as np
+
+import tensorflow as tf
+# TODO(vbardiovsky): Remove when load is available.
+from tensorflow.python.saved_model.load import load
+
+tf.saved_model.load = load
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_string("model_dir", None, "Directory to load SavedModel from.")
+
+
+def main(argv):
+  del argv
+  cell = tf.saved_model.load(FLAGS.model_dir)
+
+  initial_state = cell.get_initial_state(
+      tf.constant(np.random.uniform(size=[3, 10]).astype(np.float32)))
+
+  cell.next_state(
+      tf.constant(np.random.uniform(size=[3, 19]).astype(np.float32)),
+      initial_state)
+
+
+if __name__ == "__main__":
+  app.run(main)
-- 
GitLab


From 8d3dda24f00a457678eb5f986bce80ec8fc6e2a5 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Mon, 28 Jan 2019 09:01:02 -0800
Subject: [PATCH 1363/2345] Fix bug in handling of return statements. Relax the
 restriction that all code paths have a return value, allowing for None
 returns when the control flow creating them is not staged.

PiperOrigin-RevId: 231228548
---
 .../autograph/converters/return_statements.py | 526 +++++++++---------
 .../converters/return_statements_test.py      |  45 +-
 .../python/autograph/impl/conversion_test.py  |   1 -
 tensorflow/python/autograph/pyct/templates.py |   1 +
 4 files changed, 301 insertions(+), 272 deletions(-)

diff --git a/tensorflow/python/autograph/converters/return_statements.py b/tensorflow/python/autograph/converters/return_statements.py
index 496c99e3b5..80a555385d 100644
--- a/tensorflow/python/autograph/converters/return_statements.py
+++ b/tensorflow/python/autograph/converters/return_statements.py
@@ -22,310 +22,326 @@ import gast
 
 from tensorflow.python.autograph.core import converter
 from tensorflow.python.autograph.pyct import anno
-from tensorflow.python.autograph.pyct import ast_util
+from tensorflow.python.autograph.pyct import parser
 from tensorflow.python.autograph.pyct import templates
 from tensorflow.python.autograph.pyct.static_analysis.annos import NodeAnno
 
 
-# TODO(mdan): Move this logic into transformer_base.
-class BodyVisitor(converter.Base):
-  """Walks breadth- or depth-first the list-of-nodes bodies of AST nodes."""
+BODY_DEFINITELY_RETURNS = 'BODY_DEFINITELY_RETURNS'
+ORELSE_DEFINITELY_RETURNS = 'ORELSE_DEFINITELY_RETURNS'
+STMT_DEFINITELY_RETURNS = 'STMT_DEFINITELY_RETURNS'
 
-  def __init__(self, ctx, depth_first=False):
-    super(BodyVisitor, self).__init__(ctx)
-    self.depth_first = depth_first
-    self.changes_made = False
 
-  def visit_nodelist(self, nodelist):
-    for node in nodelist:
-      if isinstance(node, list):
-        node = self.visit_nodelist(node)
+class _Block(object):
+
+  def __init__(self):
+    self.definitely_returns = False
+
+
+class ConditionalReturnLowering(converter.Base):
+  """Rewrites a a pattern where it's unbovious that all paths return a value.
+
+  This rewrite allows avoiding intermediate None return values.
+
+  The following pattern:
+
+      if cond:
+        <block 1>
+        return
       else:
-        node = self.generic_visit(node)
-    return nodelist
+        <block 2>
+      <block 3>
 
-  def visit_If(self, node):
-    if self.depth_first:
-      node = self.generic_visit(node)
-    node.body = self.visit_nodelist(node.body)
-    node.orelse = self.visit_nodelist(node.orelse)
-    if not self.depth_first:
-      node = self.generic_visit(node)
-    return node
+  is converted to:
 
-  def visit_For(self, node):
-    if self.depth_first:
-      node = self.generic_visit(node)
-    node.body = self.visit_nodelist(node.body)
-    node.orelse = self.visit_nodelist(node.orelse)
-    if not self.depth_first:
-      node = self.generic_visit(node)
+      if cond:
+        <block 1>
+        return
+      else:
+        <block 2>
+        <block 3>
+
+  and vice-versa (if the else returns, subsequent statements are moved unthe the
+  if branch).
+  """
+
+  def visit_Return(self, node):
+    self.state[_Block].definitely_returns = True
     return node
 
+  def _postprocess_statement(self, node):
+    # If the node definitely returns (e.g. it's a with statement with a
+    # return stateent in it), then the current block also definitely returns.
+    if anno.getanno(node, STMT_DEFINITELY_RETURNS, default=False):
+      self.state[_Block].definitely_returns = True
+
+    # The special case: collapse a typical conditional return pattern into
+    # a single conditional with possibly returns on both branches. This
+    # reduces the use of None return values, which don't work with TF
+    # conditionals.
+    if (isinstance(node, gast.If)
+        and anno.getanno(node, BODY_DEFINITELY_RETURNS, default=False)):
+      return node, node.orelse
+    elif (isinstance(node, gast.If)
+          and anno.getanno(node, ORELSE_DEFINITELY_RETURNS, default=False)):
+      return node, node.body
+
+    return node, None
+
+  def _visit_statement_block(self, node, nodes):
+    self.state[_Block].enter()
+    new_nodes = self.visit_block(nodes, after_visit=self._postprocess_statement)
+    block_definitely_returns = self.state[_Block].definitely_returns
+    self.state[_Block].exit()
+    return new_nodes, block_definitely_returns
+
   def visit_While(self, node):
-    if self.depth_first:
-      node = self.generic_visit(node)
-    node.body = self.visit_nodelist(node.body)
-    node.orelse = self.visit_nodelist(node.orelse)
-    if not self.depth_first:
-      node = self.generic_visit(node)
+    node.test = self.visit(node.test)
+    node.body, _ = self._visit_statement_block(node, node.body)
+    node.orelse, _ = self._visit_statement_block(node, node.orelse)
     return node
 
-  def visit_Try(self, node):
-    if self.depth_first:
-      node = self.generic_visit(node)
-    node.body = self.visit_nodelist(node.body)
-    node.orelse = self.visit_nodelist(node.orelse)
-    node.finalbody = self.visit_nodelist(node.finalbody)
-    for i in range(len(node.handlers)):
-      node.handlers[i].body = self.visit_nodelist(node.handlers[i].body)
-    if not self.depth_first:
-      node = self.generic_visit(node)
+  def visit_For(self, node):
+    node.iter = self.visit(node.iter)
+    node.target = self.visit(node.target)
+    node.body, _ = self._visit_statement_block(node, node.body)
+    node.orelse, _ = self._visit_statement_block(node, node.orelse)
     return node
 
-  def visit_With(self, node):
-    if self.depth_first:
-      node = self.generic_visit(node)
-    node.body = self.visit_nodelist(node.body)
-    if not self.depth_first:
-      node = self.generic_visit(node)
-    return node
+  def visit_If(self, node):
+    node.test = self.visit(node.test)
 
-  def visit_FunctionDef(self, node):
-    if self.depth_first:
-      node = self.generic_visit(node)
-    node.body = self.visit_nodelist(node.body)
-    self.generic_visit(node)
-    if not self.depth_first:
-      node = self.generic_visit(node)
-    return node
+    node.body, body_definitely_returns = self._visit_statement_block(
+        node, node.body)
+    if body_definitely_returns:
+      anno.setanno(node, BODY_DEFINITELY_RETURNS, True)
 
+    node.orelse, orelse_definitely_returns = self._visit_statement_block(
+        node, node.orelse)
+    if orelse_definitely_returns:
+      anno.setanno(node, ORELSE_DEFINITELY_RETURNS, True)
 
-class FoldElse(BodyVisitor):
-
-  def visit_nodelist(self, nodelist):
-    for i in range(len(nodelist)):
-      node = nodelist[i]
-      if isinstance(node, gast.If):
-        true_branch_returns = isinstance(node.body[-1], gast.Return)
-        false_branch_returns = len(node.orelse) and isinstance(
-            node.orelse[-1], gast.Return)
-        # If the last node in the if body is a return,
-        # then every line after this if statement effectively
-        # belongs in the else.
-        if true_branch_returns and not false_branch_returns:
-          for j in range(i + 1, len(nodelist)):
-            nodelist[i].orelse.append(ast_util.copy_clean(nodelist[j]))
-          if nodelist[i + 1:]:
-            self.changes_made = True
-          return nodelist[:i + 1]
-        elif not true_branch_returns and false_branch_returns:
-          for j in range(i + 1, len(nodelist)):
-            nodelist[i].body.append(ast_util.copy_clean(nodelist[j]))
-          if nodelist[i + 1:]:
-            self.changes_made = True
-          return nodelist[:i + 1]
-        elif true_branch_returns and false_branch_returns:
-          if nodelist[i + 1:]:
-            raise ValueError(
-                'Unreachable code after conditional where both branches return.'
-            )
-          return nodelist
-      elif isinstance(node, gast.Return) and nodelist[i + 1:]:
-        raise ValueError(
-            'Cannot have statements after a return in the same basic block')
-    return nodelist
-
-
-def contains_return(node):
-  for n in gast.walk(node):
-    if isinstance(n, gast.Return):
-      return True
-  return False
-
-
-class LiftReturn(converter.Base):
-  """Move return statements out of If and With blocks."""
-
-  def __init__(self, ctx):
-    super(LiftReturn, self).__init__(ctx)
-    self.changes_made = False
-    self.common_return_name = None
+    if body_definitely_returns and orelse_definitely_returns:
+      self.state[_Block].definitely_returns = True
 
-  def visit_If(self, node):
-    # Depth-first traversal of if statements
-    node = self.generic_visit(node)
-
-    # We check if both branches return, and if so, lift the return out of the
-    # conditional. We don't enforce that the true and false branches either
-    # both return or both do not, because FoldElse might move a return
-    # into a branch after this transform completes. FoldElse and LiftReturn
-    # are alternately run until the code reaches a fixed point.
-    true_branch_returns = isinstance(node.body[-1], gast.Return)
-    false_branch_returns = len(node.orelse) and isinstance(
-        node.orelse[-1], gast.Return)
-    if true_branch_returns and false_branch_returns:
-      node.body[-1] = templates.replace(
-          'a = b', a=self.common_return_name, b=node.body[-1].value)[0]
-      node.orelse[-1] = templates.replace(
-          'a = b', a=self.common_return_name, b=node.orelse[-1].value)[0]
-      return_node = templates.replace('return a', a=self.common_return_name)[0]
-      self.changes_made = True
-      return [node, return_node]
-    else:
-      return node
-
-  def visit_With(self, node):
-    # Depth-first traversal of syntax
-    node = self.generic_visit(node)
-
-    # If the with statement returns, lift the return
-    if isinstance(node.body[-1], gast.Return):
-      node.body[-1] = templates.replace(
-          'a = b', a=self.common_return_name, b=node.body[-1].value)[0]
-      return_node = templates.replace('return a', a=self.common_return_name)[0]
-      node = self.generic_visit(node)
-      self.changes_made = True
-      return [node, return_node]
-    else:
-      return node
+    return node
 
   def visit_FunctionDef(self, node):
-    # Ensure we're doing depth-first traversal
-    last_return_name = self.common_return_name
-    body_scope = anno.getanno(node, NodeAnno.BODY_SCOPE)
-    referenced_names = body_scope.referenced
-    self.common_return_name = self.ctx.namer.new_symbol('return_',
-                                                        referenced_names)
-    node = self.generic_visit(node)
-    self.common_return_name = last_return_name
+    node.args = self.visit(node.args)
+    node.body, _ = self._visit_statement_block(node, node.body)
     return node
 
 
-class DetectReturnInUnsupportedControlFlow(gast.NodeVisitor):
-  """Throws an error if code returns inside loops or try/except."""
+class _Return(object):
 
-  # First, throw an error if we detect a return statement in a loop.
-  # TODO(alexbw): we need to learn to handle returns inside a loop,
-  # but don't currently have the TF constructs to do so (need something
-  # that looks vaguely like a goto).
+  def __init__(self):
+    self.used = False
+    self.create_guard = False
+    self.guard_created = False
+
+  def __repr__(self):
+    return 'used: {}'.format(
+        self.used)
+
+
+class _Function(object):
 
   def __init__(self):
-    self.cant_return = False
-    self.function_level = 0
-    super(DetectReturnInUnsupportedControlFlow, self).__init__()
+    self.do_return_var_name = None
+    self.retval_var_name = None
 
-  def visit_While(self, node):
-    self.cant_return = True
-    self.generic_visit(node)
-    self.cant_return = False
+  def __repr__(self):
+    return 'return control: {}, return value: {}'.format(
+        self.do_return_var_name, self.retval_var_name)
 
-  def visit_For(self, node):
-    self.cant_return = True
-    self.generic_visit(node)
-    self.cant_return = False
 
-  def visit_Try(self, node):
-    self.cant_return = True
-    self.generic_visit(node)
-    self.cant_return = False
+class ReturnStatementsTransformer(converter.Base):
+  """Lowers return statements into variables and conditionals.
 
-  def visit_FunctionDef(self, node):
-    if not self.function_level:
-      self.function_level += 1
-      self.generic_visit(node)
-      self.function_level -= 1
+  Specifically, the following pattern:
 
-  def visit_Return(self, node):
-    if self.cant_return:
-      raise ValueError(
-          '`return` statements are not supported in loops. '
-          'Try assigning to a variable in the while loop, and returning '
-          'outside of the loop')
+      <block 1>
+      return val
+      <block 2>
 
+  is converted to:
 
-class DetectReturnInConditional(gast.NodeVisitor):
-  """Assert that no return statements are present in conditionals."""
+      do_return = False
+      retval = None
 
-  def __init__(self):
-    self.cant_return = False
-    self.function_level = 0
-    super(DetectReturnInConditional, self).__init__()
+      <block 1>
 
-  def visit_If(self, node):
-    self.cant_return = True
-    self.generic_visit(node)
-    self.cant_return = False
+      do_return = True
+      retval = val
 
-  def visit_FunctionDef(self, node):
-    if not self.function_level:
-      self.function_level += 1
-      self.generic_visit(node)
-      self.function_level -= 1
+      if not do_return:
+        <block 2>
+
+      return retval
+
+  The conversion adjusts loops as well:
+
+      <block 1>
+      while cond:
+        <block 2>
+        return retval
+
+  is converted to:
+
+      <block 1>
+      while not do_return and cond:
+        <block 2>
+        do_return = True
+        retval = val
+  """
 
   def visit_Return(self, node):
-    if self.cant_return:
-      raise ValueError(
-          'After transforms, a conditional contained a `return `statement, '
-          'which is not allowed. This is a bug, and should not happen.')
+    self.state[_Return].used = True
 
+    retval = node.value if node.value else parser.parse_expresison('None')
 
-class DetectReturnInFunctionDef(gast.NodeVisitor):
+    template = """
+      do_return_var_name = True
+      retval_var_name = retval
+    """
+    node = templates.replace(
+        template,
+        do_return_var_name=self.state[_Function].do_return_var_name,
+        retval_var_name=self.state[_Function].retval_var_name,
+        retval=retval)
 
-  def visit_FunctionDef(self, node):
-    self.generic_visit(node)
-    if not contains_return(node):
-      raise ValueError(
-          'Each function definition should contain at least one return.')
+    return node
 
+  def _postprocess_statement(self, node):
+    # Example of how the state machine below works:
+    #
+    #   1| stmt           # State: _Return.used = False
+    #    |                # Action: none
+    #   3| return         # State: _Return.used = True,
+    #    |                #        _Return.guard_created = False,
+    #    |                #        _Return.create_guard = False
+    #    |                # Action: _Return.create_guard = True
+    #   4| stmt           # State: _Return.used = True,
+    #    |                #        _Return.guard_created = False,
+    #    |                #        _Return.create_guard = True
+    #    |                # Action: create `if not return_used`,
+    #    |                #         set _Return.guard_created = True
+    #   5| stmt           # State: _Return.used = True,
+    #    |                #        _Return.guard_created = True
+    #    |                # Action: none (will be wrapped under previously
+    #    |                #         created if node)
+    if self.state[_Return].used:
+      if self.state[_Return].guard_created:
+        return node, None
+
+      elif not self.state[_Return].create_guard:
+        self.state[_Return].create_guard = True
+        return node, None
+
+      elif (not self.state[_Return].guard_created and
+            self.state[_Return].create_guard):
+        self.state[_Return].guard_created = True
+        template = """
+          if ag__.not_(do_return_var_name):
+            original_node
+        """
+        cond, = templates.replace(
+            template,
+            do_return_var_name=self.state[_Function].do_return_var_name,
+            original_node=node)
+        return cond, cond.body
 
-def transform(node, ctx):
-  """Ensure a function has only a single return.
-
-  This transforms an AST node with multiple returns successively into containing
-  only a single return node.
-  There are a few restrictions on what we can handle:
-   - An AST being transformed must contain at least one return.
-   - No returns allowed in loops. We have to know the type of the return value,
-   and we currently don't have either a type inference system to discover it,
-   nor do we have a mechanism for late type binding in TensorFlow.
-   - After all transformations are finished, a Return node is not allowed inside
-   control flow. If we were unable to move a return outside of control flow,
-   this is an error.
-
-  Args:
-     node: ast.AST
-     ctx: converter.EntityContext
-
-  Returns:
-     new_node: an AST with a single return value
-
-  Raises:
-    ValueError: if the AST is structured so that we can't perform the
-   transform.
-  """
-  # Make sure that the function has at least one return statement
-  # TODO(alexbw): turning off this assertion for now --
-  # we need to not require this in e.g. class constructors.
-  # DetectReturnInFunctionDef().visit(node)
+      else:
+        assert False, 'should handle all states'
 
-  # Make sure there's no returns in unsupported locations (loops, try/except)
-  DetectReturnInUnsupportedControlFlow().visit(node)
+    return node, None
 
-  while True:
+  def _visit_statement_block(self, node, nodes):
+    self.state[_Return].enter()
+    nodes = self.visit_block(nodes, after_visit=self._postprocess_statement)
+    return_used = self.state[_Return].used
+    self.state[_Return].exit()
+    if return_used:
+      self.state[_Return].used = True
+    return nodes
 
-    # Try to lift all returns out of if statements and with blocks
-    lr = LiftReturn(ctx)
-    node = lr.visit(node)
-    changes_made = lr.changes_made
-    fe = FoldElse(ctx)
-    node = fe.visit(node)
-    changes_made = changes_made or fe.changes_made
+  def visit_While(self, node):
+    node.test = self.visit(node.test)
+
+    # Add the check for return to the loop condition.
+    node.body = self._visit_statement_block(node, node.body)
+    if self.state[_Return].used:
+      node.test = templates.replace_as_expression(
+          'ag__.and_(lambda: ag__.not_(control_var), lambda: test)',
+          test=node.test,
+          control_var=self.state[_Function].do_return_var_name)
+
+    node.orelse = self._visit_statement_block(node, node.orelse)
+    return node
 
-    if not changes_made:
-      break
+  def visit_For(self, node):
+    node.iter = self.visit(node.iter)
+    node.target = self.visit(node.target)
+
+    # Add the check for return to the loop condition.
+    node.body = self._visit_statement_block(node, node.body)
+    if self.state[_Return].used:
+      extra_test = anno.getanno(node, 'extra_test', default=None)
+      if extra_test is not None:
+        extra_test = templates.replace_as_expression(
+            'ag__.and_(lambda: ag__.not_(control_var), lambda: extra_test)',
+            extra_test=extra_test,
+            control_var=self.state[_Function].do_return_var_name)
+      else:
+        extra_test = templates.replace_as_expression(
+            'ag__.not_(control_var)',
+            control_var=self.state[_Function].do_return_var_name)
+      anno.setanno(node, 'extra_test', extra_test)
 
-  # Make sure we've scrubbed all returns from conditionals
-  DetectReturnInConditional().visit(node)
+    node.orelse = self._visit_statement_block(node, node.orelse)
+    return node
 
+  def visit_If(self, node):
+    node.test = self.visit(node.test)
+    node.body = self._visit_statement_block(node, node.body)
+    node.orelse = self._visit_statement_block(node, node.orelse)
+    return node
+
+  def visit_FunctionDef(self, node):
+    self.state[_Function].enter()
+    self.state[_Return].enter()
+
+    scope = anno.getanno(node, NodeAnno.BODY_SCOPE)
+    do_return_var_name = self.ctx.namer.new_symbol(
+        'do_return', scope.referenced)
+    retval_var_name = self.ctx.namer.new_symbol('retval_', scope.referenced)
+    self.state[_Function].do_return_var_name = do_return_var_name
+    self.state[_Function].retval_var_name = retval_var_name
+
+    converted_body = self._visit_statement_block(node, node.body)
+
+    if self.state[_Return].used:
+      template = """
+        do_return_var_name = False
+        retval_var_name = None
+        body
+        return retval_var_name
+      """
+      node.body = templates.replace(
+          template,
+          body=converted_body,
+          do_return_var_name=do_return_var_name,
+          retval_var_name=retval_var_name)
+
+    self.state[_Return].exit()
+    self.state[_Function].exit()
+    return node
+
+
+def transform(node, ctx):
+  """Ensure a function has only a single return."""
+  # Note: Technically, these two could be merged into a single walk, but
+  # keeping them separate helps with readability.
+  node = ConditionalReturnLowering(ctx).visit(node)
+  node = ReturnStatementsTransformer(ctx).visit(node)
   return node
diff --git a/tensorflow/python/autograph/converters/return_statements_test.py b/tensorflow/python/autograph/converters/return_statements_test.py
index 762fbc6f60..2b160d5ac7 100644
--- a/tensorflow/python/autograph/converters/return_statements_test.py
+++ b/tensorflow/python/autograph/converters/return_statements_test.py
@@ -49,17 +49,16 @@ class SingleReturnTest(converter_testing.TestCase):
     self.assertTransformedEquivalent(test_fn, 2)
     self.assertTransformedEquivalent(test_fn, -2)
 
-  def test_missing_orelse(self):
+  def test_missing_else(self):
 
     def test_fn(x):
       if x > 0:
         return x
 
-    node, ctx = self.prepare(test_fn, {})
-    with self.assertRaises(ValueError):
-      return_statements.transform(node, ctx)
+    self.assertTransformedEquivalent(test_fn, 2)
+    self.assertTransformedEquivalent(test_fn, -2)
 
-  def test_missing_orelse_recovrable(self):
+  def test_missing_else_then_default(self):
 
     def test_fn(x):
       if x > 0:
@@ -69,7 +68,7 @@ class SingleReturnTest(converter_testing.TestCase):
     self.assertTransformedEquivalent(test_fn, 2)
     self.assertTransformedEquivalent(test_fn, -2)
 
-  def test_missing_branch_return_recoverable(self):
+  def test_else_only_then_default(self):
 
     def test_fn(x):
       if x < 0:
@@ -136,7 +135,7 @@ class SingleReturnTest(converter_testing.TestCase):
 
     self.assertTransformedEquivalent(test_fn, 2)
 
-  def test_nested_functions(self):
+  def test_nested_function(self):
 
     def test_fn(x):
 
@@ -151,7 +150,7 @@ class SingleReturnTest(converter_testing.TestCase):
     self.assertTransformedEquivalent(test_fn, 2)
     self.assertTransformedEquivalent(test_fn, -2)
 
-  def test_nested_functions_in_control_flow(self):
+  def test_nested_function_in_control_flow(self):
 
     def test_fn(x):
 
@@ -163,16 +162,30 @@ class SingleReturnTest(converter_testing.TestCase):
     self.assertTransformedEquivalent(test_fn, 2)
     self.assertTransformedEquivalent(test_fn, -2)
 
-  def test_loop(self):
+  def test_for_loop(self):
 
-    def test_fn(x):
-      for _ in range(10):
-        return x
-      return x
+    def test_fn(n):
+      for _ in range(n):
+        return 1
 
-    node, ctx = self.prepare(test_fn, {})
-    with self.assertRaises(ValueError):
-      return_statements.transform(node, ctx)
+    self.assertTransformedEquivalent(test_fn, 2)
+    self.assertTransformedEquivalent(test_fn, 0)
+
+  def test_while_loop(self):
+
+    def test_fn(n):
+      i = 0
+      s = 0
+      while i < n:
+        i += 1
+        s += i
+        if s > 4:
+          return s
+      return -1
+
+    self.assertTransformedEquivalent(test_fn, 0)
+    self.assertTransformedEquivalent(test_fn, 2)
+    self.assertTransformedEquivalent(test_fn, 4)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/autograph/impl/conversion_test.py b/tensorflow/python/autograph/impl/conversion_test.py
index 9a4fbdad8c..cd893e3ff1 100644
--- a/tensorflow/python/autograph/impl/conversion_test.py
+++ b/tensorflow/python/autograph/impl/conversion_test.py
@@ -96,7 +96,6 @@ class ConversionTest(test.TestCase):
     f_node = program_ctx.dependency_cache[f][0]
     g_node = program_ctx.dependency_cache[g][0]
     self.assertEqual('tf__f', f_node.name)
-    self.assertEqual('tf__g', f_node.body[0].body[0].body[0].value.func.id)
     self.assertEqual('tf__g', g_node.name)
 
   def test_entity_to_graph_class_hierarchy(self):
diff --git a/tensorflow/python/autograph/pyct/templates.py b/tensorflow/python/autograph/pyct/templates.py
index 43279b3ca0..0f0b8613ac 100644
--- a/tensorflow/python/autograph/pyct/templates.py
+++ b/tensorflow/python/autograph/pyct/templates.py
@@ -108,6 +108,7 @@ class ReplaceTransformer(gast.NodeTransformer):
         anno.Basic.ORIGIN,
         anno.Basic.SKIP_PROCESSING,
         anno.Static.ORIG_DEFINITIONS,
+        'extra_test',
     }
 
   def _prepare_replacement(self, replaced, key):
-- 
GitLab


From 3a4952b6c9208092582af31e2bb597fed7fae260 Mon Sep 17 00:00:00 2001
From: Guozhong Zhuang <guozhong.zhuang@intel.com>
Date: Mon, 28 Jan 2019 09:55:54 -0800
Subject: [PATCH 1364/2345] Change per code review suggestions

---
 tensorflow/core/util/mkl_util.h | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/core/util/mkl_util.h b/tensorflow/core/util/mkl_util.h
index 34284f1e1e..c897c90124 100644
--- a/tensorflow/core/util/mkl_util.h
+++ b/tensorflow/core/util/mkl_util.h
@@ -2031,11 +2031,11 @@ class MklPrimitive {
 const mkldnn::memory::dims NONE_DIMS = {};
 
 //
-// LRUCache is a class which implements LRU (Least Recently Use) cache.
+// LRUCache is a class which implements LRU (Least Recently Used) cache.
 // The implementation is similar to that of
 //    tensorflow/core/platform/cloud/expiring_lru_cache.h
 // without its thread-safe part because the cache is supposed to be
-// used as thread local (for instance, MKLPrimitive caching).
+// used as thread local (for instance, MklPrimitive caching).
 //
 // The LRU list maintains objects in chronological order based on
 // creation time, with the least recently accessed object at the
@@ -2084,7 +2084,7 @@ class LRUCache {
     // delete the cached objects
     for (auto& key : lru_list_) {
       auto it = cache_.find(key);
-      DCHECK(it == cache_.end());
+      DCHECK(it != cache_.end());
       delete it->second.op;
     }
 
@@ -2107,12 +2107,12 @@ class LRUCache {
   };
 
   // Remove the least recently accessed entry from LRU list, which
-  // is the tail of lru_list_. Correspondingly cache_ is updated.
+  // is the tail of lru_list_. Update cache_ correspondingly.
   bool Delete() {
     if (lru_list_.empty()) return false;
     string key = lru_list_.back();
     auto it = cache_.find(key);
-    DCHECK(it == cache_.end());
+    DCHECK(it != cache_.end());
     lru_list_.pop_back();
     delete it->second.op;  // delete the object
     cache_.erase(it);
-- 
GitLab


From 4b433090add559ce0dbe1db42ae73bba233c1363 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 28 Jan 2019 09:51:01 -0800
Subject: [PATCH 1365/2345] [XLA] Adding custom_call support to the
 interpreter. This allows the interpreter backend to use the same global
 custom call target registry as the CPU backend. The C ABI and calling
 convention of the CPU JIT/AOT code is matched to enable the same functions to
 be used.

Users of the Service/Backend API can now register their target functions and interchangeably use the interpreter and CPU backend at runtime.

PiperOrigin-RevId: 231236724
---
 tensorflow/compiler/xla/service/BUILD         |  2 +-
 .../compiler/xla/service/hlo_evaluator.cc     | 22 +++++-
 .../compiler/xla/service/hlo_evaluator.h      | 26 +++++++
 .../xla/service/hlo_evaluator_test.cc         | 78 +++++++++++++++++++
 .../compiler/xla/service/interpreter/BUILD    |  1 +
 .../xla/service/interpreter/compiler.cc       | 37 ++++++++-
 6 files changed, 163 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 8a92a7005e..34af6b3597 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -272,6 +272,7 @@ tf_cc_test(
     srcs = ["hlo_evaluator_test.cc"],
     deps = [
         ":hlo",
+        ":hlo_element_type_converter",
         ":hlo_evaluator",
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:reference_util",
@@ -284,7 +285,6 @@ tf_cc_test(
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:xla_builder",
-        "//tensorflow/compiler/xla/service:hlo_element_type_converter",
         "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
         "//tensorflow/compiler/xla/tests:test_utils",
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index a40ded76cf..ecde37be58 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -29,7 +29,6 @@ limitations under the License.
 #include "absl/strings/string_view.h"
 #include "tensorflow/compiler/xla/index_util.h"
 #include "tensorflow/compiler/xla/layout_util.h"
-#include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/literal_util.h"
 #include "tensorflow/compiler/xla/map_util.h"
 #include "tensorflow/compiler/xla/primitive_util.h"
@@ -1507,6 +1506,27 @@ Status HloEvaluator::HandleReduce(HloInstruction* reduce) {
   }
 }
 
+Status HloEvaluator::HandleCustomCall(HloInstruction* custom_call) {
+  if (!custom_call_handler_) {
+    // No handler is registered; this means custom-calls are not allowed.
+    return DefaultAction(custom_call);
+  }
+
+  // Evaluate input operands so the handler has access to the operand data.
+  std::vector<const Literal*> operands;
+  operands.reserve(custom_call->operand_count());
+  for (const HloInstruction* operand : custom_call->operands()) {
+    operands.push_back(&GetEvaluatedLiteralFor(operand));
+  }
+
+  // Synchronously issue the handler to populate the instruction output literal.
+  TF_ASSIGN_OR_RETURN(
+      auto output, custom_call_handler_(custom_call, absl::MakeSpan(operands)));
+
+  evaluated_[custom_call] = std::move(output);
+  return Status::OK();
+}
+
 Status HloEvaluator::Preprocess(HloInstruction* hlo) {
   VLOG(2) << "About to visit HLO: " << hlo->ToString();
   return ShapeUtil::ValidateShape(hlo->shape());
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.h b/tensorflow/compiler/xla/service/hlo_evaluator.h
index ccb8af4fb0..72ea40bcd7 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.h
@@ -16,12 +16,14 @@ limitations under the License.
 #ifndef TENSORFLOW_COMPILER_XLA_SERVICE_HLO_EVALUATOR_H_
 #define TENSORFLOW_COMPILER_XLA_SERVICE_HLO_EVALUATOR_H_
 
+#include <functional>
 #include <memory>
 
 #include "absl/container/node_hash_map.h"
 #include "absl/memory/memory.h"
 #include "absl/types/span.h"
 #include "tensorflow/compiler/xla/array2d.h"
+#include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h"
 #include "tensorflow/compiler/xla/service/dynamic_dimension_inference.h"
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
@@ -132,6 +134,23 @@ class HloEvaluator : public DfsHloVisitorWithDefault {
   // Enable the fast path for certain operations like dot or convolution.
   void set_use_fast_path(bool value) { use_fast_path_ = value; }
 
+  // Handles evaluation of a custom-call op.
+  // Operand literals are provided in |operands| and implementations must
+  // populate |output| before returning.
+  using CustomCallHandler = std::function<StatusOr<Literal>(
+      HloInstruction* custom_call, absl::Span<const Literal*> operands)>;
+
+  // Sets a handler that is called during evaluation for custom-call ops.
+  // If no handler is defined the default error behavior will occur. The handler
+  // will be provided evaluated literals for all operands and is expected to
+  // return an output literal of the appropriate shape.
+  void set_custom_call_handler(
+      std::function<StatusOr<Literal>(HloInstruction* custom_call,
+                                      absl::Span<const Literal*> operands)>
+          handler) {
+    custom_call_handler_ = std::move(handler);
+  }
+
   // Returns the result of a matrix multiply `lhs x rhs`.
   static std::unique_ptr<Array2D<Eigen::half>> MatmulArray2D(
       const Array2D<Eigen::half>& lhs, const Array2D<Eigen::half>& rhs);
@@ -219,6 +238,8 @@ class HloEvaluator : public DfsHloVisitorWithDefault {
 
   Status HandleReduce(HloInstruction* reduce) override;
 
+  Status HandleCustomCall(HloInstruction* custom_call) override;
+
   // Unsupported HLOs, note some of them (such as BatchNorm*) are typically
   // expanded in a semantic-preserving way into other HLOs by adding exanpsion
   // HLO pass to the HLO optimization pass during compilation, which can then be
@@ -321,6 +342,11 @@ class HloEvaluator : public DfsHloVisitorWithDefault {
   // returns the dynamic dimension size of its operand.
   DynamicDimensionInference* dynamic_dimension_inference_;
 
+  // Optional handler for custom_call ops.
+  std::function<StatusOr<Literal>(HloInstruction* custom_call,
+                                  absl::Span<const Literal*> operands)>
+      custom_call_handler_;
+
   TF_DISALLOW_COPY_AND_ASSIGN(HloEvaluator);
 };
 
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
index 590f76f472..9bc71c9d6c 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
@@ -3041,5 +3041,83 @@ TEST_F(HloEvaluatorTest, PreserveMOFusionOutputLayout) {
       absl::c_equal(args[0].data<float>(), actual_literals[0].data<float>()));
 }
 
+// Tests that custom_calls fail to evaluate when no handler is specified.
+TEST_F(HloEvaluatorTest, EvaluateCustomCall_NoHandler) {
+  constexpr absl::string_view hlo_text = R"(
+    HloModule EvaluateCustomCall_NoHandler
+    ENTRY kernel_entry {
+      parameter.0 = u32[2,2]{1,0} parameter(0)
+      ROOT test_root = (u32[2,2]{1,0}) custom-call(parameter.0),
+          custom_call_target="_my_custom_call"
+    }
+  )";
+
+  TF_ASSERT_OK_AND_ASSIGN(m_, ParseAndReturnVerifiedModule(hlo_text));
+  auto args = MakeFakeArguments(m_.get()).ConsumeValueOrDie();
+  EXPECT_EQ(HloEvaluator().Evaluate(*m_, {&args[0]}).status().code(),
+            ::tensorflow::error::UNIMPLEMENTED);
+}
+
+// Tests when a custom_call handler returns an error.
+TEST_F(HloEvaluatorTest, EvaluateCustomCall_HandlerError) {
+  constexpr absl::string_view hlo_text = R"(
+    HloModule EvaluateCustomCall_HandlerError
+    ENTRY kernel_entry {
+      parameter.0 = u32[2,2]{1,0} parameter(0)
+      ROOT test_root = (u32[2,2]{1,0}) custom-call(parameter.0),
+          custom_call_target="_my_custom_call"
+    }
+  )";
+
+  TF_ASSERT_OK_AND_ASSIGN(m_, ParseAndReturnVerifiedModule(hlo_text));
+  auto args = MakeFakeArguments(m_.get()).ConsumeValueOrDie();
+  HloEvaluator evaluator;
+  evaluator.set_custom_call_handler(
+      [](HloInstruction* custom_call, absl::Span<const Literal*> operands) {
+        return InternalError("Test error");
+      });
+  EXPECT_EQ(evaluator.Evaluate(*m_, {&args[0]}).status().code(),
+            ::tensorflow::error::INTERNAL);
+}
+
+// Tests the custom_call handler on calls with many inputs.
+// We sum the operands so that we can verify the operand and output literals
+// are properly mapped for access.
+TEST_F(HloEvaluatorTest, EvaluateCustomCall_ManyInputs) {
+  constexpr absl::string_view hlo_text = R"(
+    HloModule EvaluateCustomCall_ManyInputs
+    ENTRY kernel_entry {
+      parameter.0 = u32[1]{0} parameter(0)
+      parameter.1 = u32[1]{0} parameter(1)
+      ROOT test_root = u32[1]{0} custom-call(parameter.0, parameter.1),
+          custom_call_target="_my_custom_call"
+    }
+  )";
+
+  TF_ASSERT_OK_AND_ASSIGN(m_, ParseAndReturnVerifiedModule(hlo_text));
+  auto args = MakeFakeArguments(m_.get()).ConsumeValueOrDie();
+  HloEvaluator evaluator;
+  evaluator.set_custom_call_handler(
+      [](HloInstruction* custom_call, absl::Span<const Literal*> operands) {
+        EXPECT_EQ(HloOpcode::kCustomCall, custom_call->opcode());
+        EXPECT_EQ("_my_custom_call", custom_call->custom_call_target());
+        EXPECT_EQ(2, custom_call->operand_count());
+        EXPECT_EQ(2, operands.size());
+        auto output = Literal::CreateFromShape(custom_call->shape());
+        auto operand0_data = operands[0]->data<uint32>();
+        auto operand1_data = operands[1]->data<uint32>();
+        auto output_data = output.data<uint32>();
+        output_data[0] = operand0_data[0] + operand1_data[0];
+        return output;
+      });
+  TF_ASSERT_OK_AND_ASSIGN(
+      Literal actual_literal,
+      evaluator.Evaluate(*m_->entry_computation(), {&args[0], &args[1]}));
+  auto arg0_data = args[0].data<uint32>();
+  auto arg1_data = args[1].data<uint32>();
+  std::vector<uint32> expected_data = {arg0_data[0] + arg1_data[0]};
+  EXPECT_TRUE(absl::c_equal(expected_data, actual_literal.data<uint32>()));
+}
+
 }  // namespace
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/interpreter/BUILD b/tensorflow/compiler/xla/service/interpreter/BUILD
index 545662543c..a305c6e800 100644
--- a/tensorflow/compiler/xla/service/interpreter/BUILD
+++ b/tensorflow/compiler/xla/service/interpreter/BUILD
@@ -51,6 +51,7 @@ cc_library(
         "//tensorflow/compiler/xla/service:reduce_precision_insertion",
         "//tensorflow/compiler/xla/service:reshape_mover",
         "//tensorflow/compiler/xla/service:while_loop_simplifier",
+        "//tensorflow/compiler/xla/service/cpu:custom_call_target_registry",
         "//tensorflow/core:lib",
         "//tensorflow/stream_executor",
         "@com_google_absl//absl/memory",
diff --git a/tensorflow/compiler/xla/service/interpreter/compiler.cc b/tensorflow/compiler/xla/service/interpreter/compiler.cc
index 4818b2dae0..0827b1daf8 100644
--- a/tensorflow/compiler/xla/service/interpreter/compiler.cc
+++ b/tensorflow/compiler/xla/service/interpreter/compiler.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include "absl/memory/memory.h"
 #include "tensorflow/compiler/xla/service/algebraic_simplifier.h"
 #include "tensorflow/compiler/xla/service/computation_placer.h"
+#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h"
 #include "tensorflow/compiler/xla/service/dynamic_index_splitter.h"
 #include "tensorflow/compiler/xla/service/flatten_call_graph.h"
 #include "tensorflow/compiler/xla/service/hlo_constant_folding.h"
@@ -42,6 +43,38 @@ limitations under the License.
 namespace xla {
 namespace interpreter {
 
+namespace {
+
+// Handles custom_call ops during evaluation by routing them through the global
+// CPU registry used by other CPU-based backends.
+StatusOr<Literal> HandleEvaluatorCustomCall(
+    HloInstruction* custom_call, absl::Span<const Literal*> operands) {
+  // Find the target C function in the global registry.
+  auto* registry = xla::cpu::CustomCallTargetRegistry::Global();
+  void* target_fn = registry->Lookup(custom_call->custom_call_target());
+  if (!target_fn) {
+    return NotFound("Custom call target '%s' was not registered",
+                    custom_call->custom_call_target());
+  }
+
+  // Populate pointers to operand and output literal data.
+  std::vector<const void*> operand_data;
+  operand_data.reserve(operands.size());
+  for (const auto* literal : operands) {
+    operand_data.push_back(literal->untyped_data());
+  }
+  auto output = Literal::CreateFromShape(custom_call->shape());
+  void* output_data = output.untyped_data();
+
+  // Call the target function matching the C ABI used by the CPU backends.
+  auto* typed_fn = reinterpret_cast<void (*)(void*, const void**)>(target_fn);
+  (*typed_fn)(output_data, operand_data.data());
+
+  return std::move(output);
+}
+
+}  // namespace
+
 Status InterpreterCompiler::RunHloOptimization(HloModule* hlo_module) {
   HloPassPipeline pipeline("Interpreter");
 
@@ -83,10 +116,12 @@ StatusOr<std::unique_ptr<Executable>> InterpreterCompiler::RunBackend(
   // In this case we are using an HloEvaluator at execution time, so we don't
   // need to compile anything
 
-  // Create executable from only the Hlo module.
   auto evaluator = absl::make_unique<HloEvaluator>();
   evaluator->set_use_fast_path(
       hlo_module->config().debug_options().xla_hlo_evaluator_use_fast_path());
+  evaluator->set_custom_call_handler(HandleEvaluatorCustomCall);
+
+  // Create executable from only the Hlo module.
   std::unique_ptr<Executable> executable =
       absl::make_unique<InterpreterExecutable>(std::move(hlo_module),
                                                std::move(evaluator));
-- 
GitLab


From e8359f630e3bbb70bdf978ba7e943c5c4286feba Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 28 Jan 2019 10:19:07 -0800
Subject: [PATCH 1366/2345] Support lists of strings in custom op attributes.

PiperOrigin-RevId: 231242613
---
 tensorflow/lite/toco/tflite/operator.cc      | 17 ++++++++++++++++-
 tensorflow/lite/toco/tflite/operator_test.cc | 17 +++++++++++++++++
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index 24f2675731..f61488ef35 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -1728,7 +1728,14 @@ class TensorFlowUnsupported : public BaseOperator {
           has_valid_attr = true;
           break;
         case tensorflow::AttrValue::kList:
-          if (attr.list().i_size() > 0) {
+          if (attr.list().s_size() > 0) {
+            auto start = fbb->StartVector(key);
+            for (const string& v : attr.list().s()) {
+              fbb->Add(v);
+            }
+            fbb->EndVector(start, /*typed=*/true, /*fixed=*/false);
+            has_valid_attr = true;
+          } else if (attr.list().i_size() > 0) {
             auto start = fbb->StartVector(key);
             for (const int64_t v : attr.list().i()) {
               fbb->Add(v);
@@ -1810,6 +1817,14 @@ class TensorFlowUnsupported : public BaseOperator {
           }
           break;
         }
+        case 15: {  // flexbuffers::FBT_VECTOR_STRING: {
+          auto* list = (*attr)[key].mutable_list();
+          const auto& vector = value.AsTypedVector();
+          for (size_t i = 0; i < vector.size(); i++) {
+            list->add_s(vector[i].AsString().str());
+          }
+          break;
+        }
         default:
           LOG(WARNING) << "Ignoring unsupported attribute type with key '"
                        << key << "'";
diff --git a/tensorflow/lite/toco/tflite/operator_test.cc b/tensorflow/lite/toco/tflite/operator_test.cc
index 52f10c2032..88f68f7ebf 100644
--- a/tensorflow/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/lite/toco/tflite/operator_test.cc
@@ -572,6 +572,14 @@ TEST_F(OperatorTest, TensorFlowUnsupported) {
   (*attr)["str_attr"].set_s("Hello World");
   (*attr)["int_attr"].set_i(17);
   (*attr)["bool_attr"].set_b(true);
+  {
+    auto* list = (*attr)["list_string_attr"].mutable_list();
+    list->add_s("abcde");
+    list->add_s("1234");
+    list->add_s("");
+    list->add_s("zyxwv");
+    list->add_s("!-.");
+  }
   {
     auto* list = (*attr)["list_float_attr"].mutable_list();
     list->add_f(std::numeric_limits<float>::min());
@@ -597,6 +605,15 @@ TEST_F(OperatorTest, TensorFlowUnsupported) {
   EXPECT_EQ("Hello World", output_attr.at("str_attr").s());
   EXPECT_EQ(17, output_attr.at("int_attr").i());
   EXPECT_EQ(true, output_attr.at("bool_attr").b());
+  {
+    const auto& list = output_attr.at("list_string_attr").list();
+    ASSERT_EQ(5, list.s_size());
+    EXPECT_EQ("abcde", list.s(0));
+    EXPECT_EQ("1234", list.s(1));
+    EXPECT_EQ("", list.s(2));
+    EXPECT_EQ("zyxwv", list.s(3));
+    EXPECT_EQ("!-.", list.s(4));
+  }
   {
     const auto& list = output_attr.at("list_float_attr").list();
     ASSERT_EQ(3, list.f_size());
-- 
GitLab


From 3b2be16d1045bd6fe2307e4982fd744e2731459d Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Mon, 28 Jan 2019 10:22:35 -0800
Subject: [PATCH 1367/2345] Creating HashTable (to be part of the TF2.0 API)
 that doesn't have a shared_name argument and instead uses the shared_name
 from the initializer. Also modifying the factory methods that create tables
 to use the new table.

The old HashTable is now moved to contrib.

Also updated tests so that they work both in V1 and V2 mode.

PiperOrigin-RevId: 231243269
---
 tensorflow/contrib/lookup/lookup_ops.py       |  78 +++-
 tensorflow/python/data/kernel_tests/BUILD     |   1 +
 .../kernel_tests/iterator_cluster_test.py     |   2 +-
 .../python/kernel_tests/lookup_ops_test.py    | 439 +++++++++---------
 tensorflow/python/ops/lookup_ops.py           |  87 ++--
 5 files changed, 314 insertions(+), 293 deletions(-)

diff --git a/tensorflow/contrib/lookup/lookup_ops.py b/tensorflow/contrib/lookup/lookup_ops.py
index 229a72a780..c2916b82a1 100644
--- a/tensorflow/contrib/lookup/lookup_ops.py
+++ b/tensorflow/contrib/lookup/lookup_ops.py
@@ -28,7 +28,6 @@ from tensorflow.python.ops import lookup_ops
 # pylint: disable=unused-import
 from tensorflow.python.ops.lookup_ops import FastHashSpec
 from tensorflow.python.ops.lookup_ops import HasherSpec
-from tensorflow.python.ops.lookup_ops import HashTable
 from tensorflow.python.ops.lookup_ops import IdTableWithHashBuckets
 from tensorflow.python.ops.lookup_ops import index_table_from_file
 from tensorflow.python.ops.lookup_ops import index_to_string_table_from_file
@@ -288,6 +287,83 @@ def index_to_string(tensor, mapping, default_value="UNK", name=None):
   return table.lookup(tensor)
 
 
+class HashTable(InitializableLookupTableBase):
+  """A generic hash table implementation.
+
+  Example usage:
+
+  ```python
+  table = tf.HashTable(
+      tf.KeyValueTensorInitializer(keys, values), -1)
+  out = table.lookup(input_tensor)
+  table.init.run()
+  print(out.eval())
+  ```
+  """
+
+  def __init__(self, initializer, default_value, shared_name=None, name=None):
+    """Creates a non-initialized `HashTable` object.
+
+    Creates a table, the type of its keys and values are specified by the
+    initializer.
+    Before using the table you will have to initialize it. After initialization
+    the table will be immutable.
+
+    Args:
+      initializer: The table initializer to use. See `HashTable` kernel for
+        supported key and value types.
+      default_value: The value to use if a key is missing in the table.
+      shared_name: If non-empty, this table will be shared under the given name
+        across multiple sessions.
+      name: A name for the operation (optional).
+
+    Returns:
+      A `HashTable` object.
+    """
+    self._initializer = initializer
+    self._default_value = default_value
+    self._shared_name = shared_name
+    self._name = name or "hash_table"
+    self._table_name = None
+    super(HashTable, self).__init__(default_value, initializer)
+    self._value_shape = self._default_value.get_shape()
+
+  def create_resource(self):
+    table_ref = gen_lookup_ops.hash_table_v2(
+        shared_name=self._shared_name,
+        key_dtype=self._initializer.key_dtype,
+        value_dtype=self._initializer.value_dtype,
+        name=self._name)
+    if context.executing_eagerly():
+      self._table_name = None
+    else:
+      self._table_name = table_ref.op.name.split("/")[-1]
+    return table_ref
+
+  @property
+  def name(self):
+    return self._table_name
+
+  def export(self, name=None):
+    """Returns tensors of all keys and values in the table.
+
+    Args:
+      name: A name for the operation (optional).
+
+    Returns:
+      A pair of tensors with the first tensor containing all keys and the
+        second tensors containing all values in the table.
+    """
+    with ops.name_scope(name, "%s_Export" % self.name,
+                        [self.resource_handle]) as name:
+      exported_keys, exported_values = gen_lookup_ops.lookup_table_export_v2(
+          self.resource_handle, self._key_dtype, self._value_dtype, name=name)
+
+    exported_values.set_shape(exported_keys.get_shape().concatenate(
+        self._value_shape))
+    return exported_keys, exported_values
+
+
 class MutableHashTable(LookupInterface):
   """A generic mutable hash table implementation.
 
diff --git a/tensorflow/python/data/kernel_tests/BUILD b/tensorflow/python/data/kernel_tests/BUILD
index 8206ce382c..b134a8deae 100644
--- a/tensorflow/python/data/kernel_tests/BUILD
+++ b/tensorflow/python/data/kernel_tests/BUILD
@@ -287,6 +287,7 @@ tf_py_test(
     size = "small",
     srcs = ["iterator_cluster_test.py"],
     additional_deps = [
+        "//tensorflow/contrib/lookup:lookup_py",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python/data/ops:iterator_ops",
diff --git a/tensorflow/python/data/kernel_tests/iterator_cluster_test.py b/tensorflow/python/data/kernel_tests/iterator_cluster_test.py
index 2008823495..23d3b6a439 100644
--- a/tensorflow/python/data/kernel_tests/iterator_cluster_test.py
+++ b/tensorflow/python/data/kernel_tests/iterator_cluster_test.py
@@ -19,6 +19,7 @@ from __future__ import print_function
 
 import numpy as np
 
+from tensorflow.contrib import lookup as lookup_ops
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.client import session
 from tensorflow.python.data.ops import dataset_ops
@@ -31,7 +32,6 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import functional_ops
-from tensorflow.python.ops import lookup_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import string_ops
 from tensorflow.python.platform import test
diff --git a/tensorflow/python/kernel_tests/lookup_ops_test.py b/tensorflow/python/kernel_tests/lookup_ops_test.py
index 3dd1ee33d9..8711923491 100644
--- a/tensorflow/python/kernel_tests/lookup_ops_test.py
+++ b/tensorflow/python/kernel_tests/lookup_ops_test.py
@@ -35,9 +35,8 @@ from tensorflow.python.platform import test
 from tensorflow.python.training import server_lib
 
 
-class HashTableOpTest(test.TestCase):
+class HashTableTest(test.TestCase):
 
-  @test_util.run_deprecated_v1
   def testHashTable(self):
     with self.cached_session():
       default_val = -1
@@ -45,9 +44,9 @@ class HashTableOpTest(test.TestCase):
       values = constant_op.constant([0, 1, 2], dtypes.int64)
       table = lookup_ops.HashTable(
           lookup_ops.KeyValueTensorInitializer(keys, values), default_val)
-      table.initializer.run()
+      self.evaluate(table.initializer)
 
-      self.assertAllEqual(3, table.size().eval())
+      self.assertAllEqual(3, self.evaluate(table.size()))
 
       input_string = constant_op.constant(["brain", "salad", "tank"])
       output = table.lookup(input_string)
@@ -62,7 +61,6 @@ class HashTableOpTest(test.TestCase):
                             self.evaluate(exported_keys_tensor))
       self.assertItemsEqual([0, 1, 2], self.evaluate(exported_values_tensor))
 
-  @test_util.run_deprecated_v1
   def testHashTableFindHighRank(self):
     with self.cached_session():
       default_val = -1
@@ -70,18 +68,17 @@ class HashTableOpTest(test.TestCase):
       values = constant_op.constant([0, 1, 2], dtypes.int64)
       table = lookup_ops.HashTable(
           lookup_ops.KeyValueTensorInitializer(keys, values), default_val)
-      table.initializer.run()
+      self.evaluate(table.initializer)
 
-      self.assertAllEqual(3, table.size().eval())
+      self.assertAllEqual(3, self.evaluate(table.size()))
 
-      input_string = constant_op.constant(
-          [["brain", "salad"], ["tank", "tarkus"]])
+      input_string = constant_op.constant([["brain", "salad"],
+                                           ["tank", "tarkus"]])
       output = table.lookup(input_string)
 
       result = self.evaluate(output)
       self.assertAllEqual([[0, 1], [-1, -1]], result)
 
-  @test_util.run_deprecated_v1
   def testHashTableInitWithPythonArrays(self):
     with self.cached_session():
       default_val = -1
@@ -90,9 +87,9 @@ class HashTableOpTest(test.TestCase):
       table = lookup_ops.HashTable(
           lookup_ops.KeyValueTensorInitializer(
               keys, values, value_dtype=dtypes.int64), default_val)
-      table.initializer.run()
+      self.evaluate(table.initializer)
 
-      self.assertAllEqual(3, table.size().eval())
+      self.assertAllEqual(3, self.evaluate(table.size()))
 
       input_string = constant_op.constant(["brain", "salad", "tank"])
       output = table.lookup(input_string)
@@ -100,7 +97,6 @@ class HashTableOpTest(test.TestCase):
       result = self.evaluate(output)
       self.assertAllEqual([0, 1, -1], result)
 
-  @test_util.run_deprecated_v1
   def testHashTableInitWithNumPyArrays(self):
     with self.cached_session():
       default_val = -1
@@ -108,9 +104,9 @@ class HashTableOpTest(test.TestCase):
       values = np.array([0, 1, 2], dtype=np.int64)
       table = lookup_ops.HashTable(
           lookup_ops.KeyValueTensorInitializer(keys, values), default_val)
-      table.initializer.run()
+      self.evaluate(table.initializer)
 
-      self.assertAllEqual(3, table.size().eval())
+      self.assertAllEqual(3, self.evaluate(table.size()))
 
       input_string = constant_op.constant(["brain", "salad", "tank"])
       output = table.lookup(input_string)
@@ -118,9 +114,9 @@ class HashTableOpTest(test.TestCase):
       result = self.evaluate(output)
       self.assertAllEqual([0, 1, -1], result)
 
-  @test_util.run_deprecated_v1
   def testMultipleHashTables(self):
-    with self.cached_session() as sess:
+    with self.cached_session():
+
       default_val = -1
       keys = constant_op.constant(["brain", "salad", "surgery"])
       values = constant_op.constant([0, 1, 2], dtypes.int64)
@@ -132,10 +128,12 @@ class HashTableOpTest(test.TestCase):
       table3 = lookup_ops.HashTable(
           lookup_ops.KeyValueTensorInitializer(keys, values), default_val)
 
-      lookup_ops.tables_initializer().run()
-      self.assertAllEqual(3, table1.size().eval())
-      self.assertAllEqual(3, table2.size().eval())
-      self.assertAllEqual(3, table3.size().eval())
+      self.evaluate(table1.initializer)
+      self.evaluate(table2.initializer)
+      self.evaluate(table3.initializer)
+      self.assertAllEqual(3, self.evaluate(table1.size()))
+      self.assertAllEqual(3, self.evaluate(table2.size()))
+      self.assertAllEqual(3, self.evaluate(table3.size()))
 
       input_string = constant_op.constant(["brain", "salad", "tank"])
       output1 = table1.lookup(input_string)
@@ -147,7 +145,6 @@ class HashTableOpTest(test.TestCase):
       self.assertAllEqual([0, 1, -1], out2)
       self.assertAllEqual([0, 1, -1], out3)
 
-  @test_util.run_deprecated_v1
   def testHashTableWithTensorDefault(self):
     with self.cached_session():
       default_val = constant_op.constant(-1, dtypes.int64)
@@ -155,7 +152,7 @@ class HashTableOpTest(test.TestCase):
       values = constant_op.constant([0, 1, 2], dtypes.int64)
       table = lookup_ops.HashTable(
           lookup_ops.KeyValueTensorInitializer(keys, values), default_val)
-      table.initializer.run()
+      self.evaluate(table.initializer)
 
       input_string = constant_op.constant(["brain", "salad", "tank"])
       output = table.lookup(input_string)
@@ -163,15 +160,14 @@ class HashTableOpTest(test.TestCase):
       result = self.evaluate(output)
       self.assertAllEqual([0, 1, -1], result)
 
-  @test_util.run_deprecated_v1
   def testHashTableWithSparseTensorInput(self):
-    with self.cached_session() as sess:
+    with self.cached_session():
       default_val = constant_op.constant(-1, dtypes.int64)
       keys = constant_op.constant(["brain", "salad", "surgery"])
       values = constant_op.constant([0, 1, 2], dtypes.int64)
       table = lookup_ops.HashTable(
           lookup_ops.KeyValueTensorInitializer(keys, values), default_val)
-      table.initializer.run()
+      self.evaluate(table.initializer)
 
       sp_indices = [[0, 0], [0, 1], [1, 0]]
       sp_shape = [2, 2]
@@ -187,7 +183,6 @@ class HashTableOpTest(test.TestCase):
       self.assertAllEqual(sp_indices, out_indices)
       self.assertAllEqual(sp_shape, out_shape)
 
-  @test_util.run_deprecated_v1
   def testSignatureMismatch(self):
     with self.cached_session():
       default_val = -1
@@ -195,12 +190,12 @@ class HashTableOpTest(test.TestCase):
       values = constant_op.constant([0, 1, 2], dtypes.int64)
       table = lookup_ops.HashTable(
           lookup_ops.KeyValueTensorInitializer(keys, values), default_val)
-      table.initializer.run()
+      self.evaluate(table.initializer)
 
       # Ref types do not produce a lookup signature mismatch.
       input_string_ref = variables.Variable("brain")
-      variables.global_variables_initializer().run()
-      self.assertEqual(0, table.lookup(input_string_ref).eval())
+      self.evaluate(input_string_ref.initializer)
+      self.assertEqual(0, self.evaluate(table.lookup(input_string_ref)))
 
       input_string = constant_op.constant([1, 2, 3], dtypes.int64)
       with self.assertRaises(TypeError):
@@ -223,8 +218,9 @@ class HashTableOpTest(test.TestCase):
     with self.cached_session():
       default_val = -1
       table = lookup_ops.HashTable(
-          lookup_ops.KeyValueTensorInitializer(
-              ["a"], [1], value_dtype=dtypes.int64), default_val)
+          lookup_ops.KeyValueTensorInitializer(["a"], [1],
+                                               value_dtype=dtypes.int64),
+          default_val)
 
       input_string = constant_op.constant(["brain", "salad", "surgery"])
       output = table.lookup(input_string)
@@ -240,10 +236,10 @@ class HashTableOpTest(test.TestCase):
       values = constant_op.constant([0, 1, 2], dtypes.int64)
       table = lookup_ops.HashTable(
           lookup_ops.KeyValueTensorInitializer(keys, values), default_val)
-      table.initializer.run()
+      self.evaluate(table.initializer)
 
       with self.assertRaisesOpError("Table already initialized"):
-        table.initializer.run()
+        self.evaluate(table.initializer)
 
   @test_util.run_deprecated_v1
   def testInitializationWithInvalidDimensions(self):
@@ -259,10 +255,9 @@ class HashTableOpTest(test.TestCase):
   @test_util.run_deprecated_v1
   def testMultipleSessions(self):
     # Start a server
-    server = server_lib.Server(
-        {
-            "local0": ["localhost:0"]
-        }, protocol="grpc", start=True)
+    server = server_lib.Server({"local0": ["localhost:0"]},
+                               protocol="grpc",
+                               start=True)
     # Create two sessions sharing the same state
     session1 = session.Session(server.target)
     session2 = session.Session(server.target)
@@ -277,16 +272,15 @@ class HashTableOpTest(test.TestCase):
 
     # Init the table in the first session.
     with session1:
-      table.initializer.run()
-      self.assertAllEqual(3, table.size().eval())
+      self.evaluate(table.initializer)
+      self.assertAllEqual(3, self.evaluate(table.size()))
 
     # Init the table in the second session and verify that we do not get a
     # "Table already initialized" error.
     with session2:
       table.initializer.run()
-      self.assertAllEqual(3, table.size().eval())
+      self.assertAllEqual(3, self.evaluate(table.size()))
 
-  @test_util.run_deprecated_v1
   def testHashTableInt32String(self):
     with self.cached_session():
       default_val = "n/a"
@@ -294,7 +288,7 @@ class HashTableOpTest(test.TestCase):
       values = constant_op.constant(["brain", "salad", "surgery"])
       table = lookup_ops.HashTable(
           lookup_ops.KeyValueTensorInitializer(keys, values), default_val)
-      table.initializer.run()
+      self.evaluate(table.initializer)
 
       input_tensor = constant_op.constant([0, 1, -1])
       output = table.lookup(input_tensor)
@@ -311,7 +305,6 @@ class IndexTableFromFile(test.TestCase):
       f.write("\n".join(values) + "\n")
     return vocabulary_file
 
-  @test_util.run_deprecated_v1
   def test_string_index_table_from_file(self):
     vocabulary_file = self._createVocabFile("f2i_vocab1.txt")
     with self.cached_session():
@@ -319,12 +312,12 @@ class IndexTableFromFile(test.TestCase):
           vocabulary_file=vocabulary_file, num_oov_buckets=1)
       ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"]))
 
-      with self.assertRaises(errors_impl.OpError):
-        self.evaluate(ids)
-      lookup_ops.tables_initializer().run()
+      if not context.executing_eagerly():
+        with self.assertRaises(errors_impl.OpError):
+          self.evaluate(ids)
+      self.evaluate(lookup_ops.tables_initializer())
       self.assertAllEqual((1, 2, 3), self.evaluate(ids))
 
-  @test_util.run_deprecated_v1
   def test_string_index_table_from_multicolumn_file(self):
     vocabulary_file = self._createVocabFile(
         "f2i_vocab1.txt", values=("brain\t300", "salad\t20", "surgery\t1"))
@@ -336,12 +329,12 @@ class IndexTableFromFile(test.TestCase):
           value_column_index=lookup_ops.TextFileIndex.LINE_NUMBER)
       ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"]))
 
-      with self.assertRaises(errors_impl.OpError):
-        self.evaluate(ids)
-      lookup_ops.tables_initializer().run()
+      if not context.executing_eagerly():
+        with self.assertRaises(errors_impl.OpError):
+          self.evaluate(ids)
+      self.evaluate(lookup_ops.tables_initializer())
       self.assertAllEqual((1, 2, 3), self.evaluate(ids))
 
-  @test_util.run_deprecated_v1
   def test_string_index_table_from_multicolumn_file_custom_delimiter(self):
     vocabulary_file = self._createVocabFile(
         "f2i_vocab1.txt", values=("brain 300", "salad 20", "surgery 1"))
@@ -354,12 +347,12 @@ class IndexTableFromFile(test.TestCase):
           delimiter=" ")
       ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"]))
 
-      with self.assertRaises(errors_impl.OpError):
-        self.evaluate(ids)
-      lookup_ops.tables_initializer().run()
+      if not context.executing_eagerly():
+        with self.assertRaises(errors_impl.OpError):
+          self.evaluate(ids)
+      self.evaluate(lookup_ops.tables_initializer())
       self.assertAllEqual((1, 2, 3), self.evaluate(ids))
 
-  @test_util.run_deprecated_v1
   def test_string_index_table_from_file_tensor_filename(self):
     vocabulary_file = self._createVocabFile("f2i_vocab1.txt")
     with self.cached_session():
@@ -368,14 +361,16 @@ class IndexTableFromFile(test.TestCase):
           vocabulary_file=vocabulary_file, num_oov_buckets=1)
       ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"]))
 
-      with self.assertRaises(errors_impl.OpError):
-        self.evaluate(ids)
-      lookup_ops.tables_initializer().run()
+      if not context.executing_eagerly():
+        with self.assertRaises(errors_impl.OpError):
+          self.evaluate(ids)
+      self.evaluate(lookup_ops.tables_initializer())
       self.assertAllEqual((1, 2, 3), self.evaluate(ids))
-      self.assertEqual(1,
-                       len(ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS)))
+      if not context.executing_eagerly():
+        self.assertEqual(1,
+                         len(ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS)))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("placeholder usage")
   def test_string_index_table_from_file_placeholder_filename(self):
     vocabulary_file = self._createVocabFile("f2i_vocab1.txt")
     with self.cached_session():
@@ -393,7 +388,6 @@ class IndexTableFromFile(test.TestCase):
       self.assertEqual(0,
                        len(ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS)))
 
-  @test_util.run_deprecated_v1
   def test_int32_index_table_from_file(self):
     vocabulary_file = self._createVocabFile(
         "f2i_vocab2.txt", values=("42", "1", "-1000"))
@@ -405,12 +399,12 @@ class IndexTableFromFile(test.TestCase):
       ids = table.lookup(
           constant_op.constant((1, -1000, 11), dtype=dtypes.int32))
 
-      with self.assertRaises(errors_impl.OpError):
-        self.evaluate(ids)
-      lookup_ops.tables_initializer().run()
+      if not context.executing_eagerly():
+        with self.assertRaises(errors_impl.OpError):
+          self.evaluate(ids)
+      self.evaluate(lookup_ops.tables_initializer())
       self.assertAllEqual((1, 2, 3), self.evaluate(ids))
 
-  @test_util.run_deprecated_v1
   def test_int64_index_table_from_file(self):
     vocabulary_file = self._createVocabFile(
         "f2i_vocab3.txt", values=("42", "1", "-1000"))
@@ -422,12 +416,12 @@ class IndexTableFromFile(test.TestCase):
       ids = table.lookup(
           constant_op.constant((1, -1000, 11), dtype=dtypes.int64))
 
-      with self.assertRaises(errors_impl.OpError):
-        self.evaluate(ids)
-      lookup_ops.tables_initializer().run()
+      if not context.executing_eagerly():
+        with self.assertRaises(errors_impl.OpError):
+          self.evaluate(ids)
+      self.evaluate(lookup_ops.tables_initializer())
       self.assertAllEqual((1, 2, 3), self.evaluate(ids))
 
-  @test_util.run_deprecated_v1
   def test_index_table_from_file_with_default_value(self):
     default_value = -42
     vocabulary_file = self._createVocabFile("f2i_vocab4.txt")
@@ -436,12 +430,12 @@ class IndexTableFromFile(test.TestCase):
           vocabulary_file=vocabulary_file, default_value=default_value)
       ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"]))
 
-      with self.assertRaises(errors_impl.OpError):
-        self.evaluate(ids)
-      lookup_ops.tables_initializer().run()
+      if not context.executing_eagerly():
+        with self.assertRaises(errors_impl.OpError):
+          self.evaluate(ids)
+      self.evaluate(lookup_ops.tables_initializer())
       self.assertAllEqual((1, 2, default_value), self.evaluate(ids))
 
-  @test_util.run_deprecated_v1
   def test_index_table_from_file_with_oov_buckets(self):
     vocabulary_file = self._createVocabFile("f2i_vocab5.txt")
     with self.cached_session():
@@ -450,9 +444,10 @@ class IndexTableFromFile(test.TestCase):
       ids = table.lookup(
           constant_op.constant(["salad", "surgery", "tarkus", "toccata"]))
 
-      with self.assertRaises(errors_impl.OpError):
-        self.evaluate(ids)
-      lookup_ops.tables_initializer().run()
+      if not context.executing_eagerly():
+        with self.assertRaises(errors_impl.OpError):
+          self.evaluate(ids)
+      self.evaluate(lookup_ops.tables_initializer())
       self.assertAllEqual(
           (
               1,  # From vocabulary file.
@@ -490,7 +485,6 @@ class IndexTableFromFile(test.TestCase):
         vocabulary_file=vocabulary_file,
         vocab_size=0)
 
-  @test_util.run_deprecated_v1
   def test_index_table_from_file_with_vocab_size_too_small(self):
     vocabulary_file = self._createVocabFile("f2i_vocab6.txt")
     with self.cached_session():
@@ -498,22 +492,22 @@ class IndexTableFromFile(test.TestCase):
           vocabulary_file=vocabulary_file, vocab_size=2)
       ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"]))
 
-      with self.assertRaises(errors_impl.OpError):
-        self.evaluate(ids)
-      lookup_ops.tables_initializer().run()
+      if not context.executing_eagerly():
+        with self.assertRaises(errors_impl.OpError):
+          self.evaluate(ids)
+      self.evaluate(lookup_ops.tables_initializer())
       self.assertAllEqual((1, -1, -1), self.evaluate(ids))
-      self.assertEqual(2, table.size().eval())
+      self.assertEqual(2, self.evaluate(table.size()))
 
-  @test_util.run_deprecated_v1
   def test_index_table_from_file_with_vocab_size_too_large(self):
     vocabulary_file = self._createVocabFile("f2i_vocab7.txt")
     with self.cached_session():
-      table = lookup_ops.index_table_from_file(
-          vocabulary_file=vocabulary_file, vocab_size=4)
-      self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
-                              "Invalid vocab_size", table.initializer.run)
+      with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
+                                   "Invalid vocab_size"):
+        table = lookup_ops.index_table_from_file(
+            vocabulary_file=vocabulary_file, vocab_size=4)
+        self.evaluate(table.initializer)
 
-  @test_util.run_deprecated_v1
   def test_index_table_from_file_with_vocab_size(self):
     vocabulary_file = self._createVocabFile("f2i_vocab8.txt")
 
@@ -528,11 +522,12 @@ class IndexTableFromFile(test.TestCase):
           vocabulary_file=vocabulary_file, vocab_size=3)
       ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"]))
 
-      with self.assertRaises(errors_impl.OpError):
-        self.evaluate(ids)
-      lookup_ops.tables_initializer().run()
+      if not context.executing_eagerly():
+        with self.assertRaises(errors_impl.OpError):
+          self.evaluate(ids)
+      self.evaluate(lookup_ops.tables_initializer())
       self.assertAllEqual((1, 2, -1), self.evaluate(ids))
-      self.assertEqual(3, table.size().eval())
+      self.assertEqual(3, self.evaluate(table.size()))
 
   def test_index_table_from_file_with_invalid_hashers(self):
     vocabulary_file = self._createVocabFile("invalid_hasher.txt")
@@ -606,22 +601,20 @@ class KeyValueTensorInitializerTest(test.TestCase):
       table = lookup_ops.HashTable(init, default_value=-1)
       table.initializer.run()
 
-  @test_util.run_deprecated_v1
   def test_int32(self):
     with ops.Graph().as_default(), self.cached_session():
       init = lookup_ops.KeyValueTensorInitializer((42, 1, -1000), (0, 1, 2),
                                                   dtypes.int32, dtypes.int64)
       self.assertEqual("", init._shared_name)
       table = lookup_ops.HashTable(init, default_value=-1)
-      with self.assertRaisesRegexp(
-          errors_impl.OpError, "No OpKernel was registered"):
+      with self.assertRaisesRegexp(errors_impl.OpError,
+                                   "No OpKernel was registered"):
         table.initializer.run()
 
 
 class IndexTableFromTensor(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
-  @test_util.run_deprecated_v1
   def test_index_table_from_tensor_with_tensor_init(self):
     table = lookup_ops.index_table_from_tensor(
         vocabulary_list=("brain", "salad", "surgery"), num_oov_buckets=1)
@@ -638,7 +631,6 @@ class IndexTableFromTensor(test.TestCase):
     ids = table.lookup(constant_op.constant(("salad", "surgery", "tarkus")))
     self.assertAllEqual((1, 2, 3), self.evaluate(ids))
 
-  @test_util.run_deprecated_v1
   def test_int32_index_table_from_tensor_with_tensor_init(self):
     with self.cached_session():
       table = lookup_ops.index_table_from_tensor(
@@ -646,12 +638,12 @@ class IndexTableFromTensor(test.TestCase):
       ids = table.lookup(
           constant_op.constant((1, -1000, 11), dtype=dtypes.int32))
 
-      with self.assertRaises(errors_impl.FailedPreconditionError):
-        self.evaluate(ids)
-      lookup_ops.tables_initializer().run()
+      if not context.executing_eagerly():
+        with self.assertRaises(errors_impl.FailedPreconditionError):
+          self.evaluate(ids)
+      self.evaluate(lookup_ops.tables_initializer())
       self.assertAllEqual((1, 2, 3), self.evaluate(ids))
 
-  @test_util.run_deprecated_v1
   def test_int64_index_table_from_tensor_with_tensor_init(self):
     with self.cached_session():
       table = lookup_ops.index_table_from_tensor(
@@ -659,12 +651,12 @@ class IndexTableFromTensor(test.TestCase):
       ids = table.lookup(
           constant_op.constant((1, -1000, 11), dtype=dtypes.int64))
 
-      with self.assertRaises(errors_impl.FailedPreconditionError):
-        self.evaluate(ids)
-      lookup_ops.tables_initializer().run()
+      if not context.executing_eagerly():
+        with self.assertRaises(errors_impl.FailedPreconditionError):
+          self.evaluate(ids)
+      self.evaluate(lookup_ops.tables_initializer())
       self.assertAllEqual((1, 2, 3), self.evaluate(ids))
 
-  @test_util.run_deprecated_v1
   def test_index_table_from_tensor_with_default_value(self):
     default_value = -42
     with self.cached_session():
@@ -673,9 +665,10 @@ class IndexTableFromTensor(test.TestCase):
           default_value=default_value)
       ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"]))
 
-      with self.assertRaises(errors_impl.FailedPreconditionError):
-        self.evaluate(ids)
-      lookup_ops.tables_initializer().run()
+      if not context.executing_eagerly():
+        with self.assertRaises(errors_impl.FailedPreconditionError):
+          self.evaluate(ids)
+      self.evaluate(lookup_ops.tables_initializer())
       self.assertAllEqual((1, 2, default_value), self.evaluate(ids))
 
   def test_index_table_from_tensor_missing_vocabulary_list(self):
@@ -685,17 +678,13 @@ class IndexTableFromTensor(test.TestCase):
         lookup_ops.index_table_from_tensor(
             vocabulary_list=None, num_oov_buckets=1)
 
-  @test_util.run_deprecated_v1
   def test_index_table_from_tensor_empty_vocabulary_list(self):
     with self.cached_session():
-      table = lookup_ops.index_table_from_tensor(
-          vocabulary_list=np.array([], dtype=np.str_), num_oov_buckets=1)
-      ids = table.lookup(constant_op.constant(["salad", "surgery", "brain"]))
-      with self.assertRaises(errors_impl.OpError):
-        self.evaluate(ids)
       with self.assertRaisesRegexp(
           errors_impl.OpError, "keys and values cannot be empty"):
-        lookup_ops.tables_initializer().run()
+        _ = lookup_ops.index_table_from_tensor(
+            vocabulary_list=np.array([], dtype=np.str_), num_oov_buckets=1)
+        self.evaluate(lookup_ops.tables_initializer())
 
   def test_index_table_from_tensor_with_invalid_hashers(self):
     with self.cached_session():
@@ -722,7 +711,6 @@ class IndexToStringTableFromFileTest(test.TestCase):
       f.write("\n".join(values) + "\n")
     return vocabulary_file
 
-  @test_util.run_deprecated_v1
   def test_index_to_string_table(self):
     vocabulary_path = self._createVocabFile("i2f_vocab1.txt")
     # vocabulary_file supports string and tensor
@@ -734,13 +722,13 @@ class IndexToStringTableFromFileTest(test.TestCase):
             vocabulary_file=vocabulary_file)
         features = table.lookup(
             constant_op.constant([0, 1, 2, 3], dtypes.int64))
-        with self.assertRaises(errors_impl.OpError):
-          self.evaluate(features)
-        lookup_ops.tables_initializer().run()
+        if not context.executing_eagerly():
+          with self.assertRaises(errors_impl.OpError):
+            self.evaluate(features)
+        self.evaluate(lookup_ops.tables_initializer())
         self.assertAllEqual((b"brain", b"salad", b"surgery", b"UNK"),
                             self.evaluate(features))
 
-  @test_util.run_deprecated_v1
   def test_index_to_string_table_from_multicolumn_file(self):
     vocabulary_file = self._createVocabFile(
         "f2i_vocab1.txt", values=("brain\t300", "salad\t20", "surgery\t1"))
@@ -750,13 +738,13 @@ class IndexToStringTableFromFileTest(test.TestCase):
           key_column_index=lookup_ops.TextFileIndex.LINE_NUMBER,
           value_column_index=0)
       features = table.lookup(constant_op.constant([0, 1, 2, 3], dtypes.int64))
-      with self.assertRaises(errors_impl.OpError):
-        self.evaluate(features)
-      lookup_ops.tables_initializer().run()
+      if not context.executing_eagerly():
+        with self.assertRaises(errors_impl.OpError):
+          self.evaluate(features)
+      self.evaluate(lookup_ops.tables_initializer())
       self.assertAllEqual((b"brain", b"salad", b"surgery", b"UNK"),
                           self.evaluate(features))
 
-  @test_util.run_deprecated_v1
   def test_index_to_string_table_from_multicolumn_file_custom_delimiter(self):
     vocabulary_file = self._createVocabFile(
         "f2i_vocab1.txt", values=("brain 300", "salad 20", "surgery 1"))
@@ -767,13 +755,13 @@ class IndexToStringTableFromFileTest(test.TestCase):
           value_column_index=0,
           delimiter=" ")
       features = table.lookup(constant_op.constant([0, 1, 2, 3], dtypes.int64))
-      with self.assertRaises(errors_impl.OpError):
-        self.evaluate(features)
-      lookup_ops.tables_initializer().run()
+      if not context.executing_eagerly():
+        with self.assertRaises(errors_impl.OpError):
+          self.evaluate(features)
+      self.evaluate(lookup_ops.tables_initializer())
       self.assertAllEqual((b"brain", b"salad", b"surgery", b"UNK"),
                           self.evaluate(features))
 
-  @test_util.run_deprecated_v1
   def test_index_to_string_table_with_default_value(self):
     default_value = b"NONE"
     vocabulary_file = self._createVocabFile("f2i_vocab2.txt")
@@ -781,13 +769,13 @@ class IndexToStringTableFromFileTest(test.TestCase):
       table = lookup_ops.index_to_string_table_from_file(
           vocabulary_file=vocabulary_file, default_value=default_value)
       features = table.lookup(constant_op.constant([1, 2, 4], dtypes.int64))
-      with self.assertRaises(errors_impl.OpError):
-        self.evaluate(features)
-      lookup_ops.tables_initializer().run()
+      if not context.executing_eagerly():
+        with self.assertRaises(errors_impl.OpError):
+          self.evaluate(features)
+      self.evaluate(lookup_ops.tables_initializer())
       self.assertAllEqual((b"salad", b"surgery", default_value),
                           self.evaluate(features))
 
-  @test_util.run_deprecated_v1
   def test_index_to_string_table_with_vocab_size_too_small(self):
     default_value = b"NONE"
     vocabulary_file = self._createVocabFile("f2i_vocab2.txt")
@@ -797,27 +785,22 @@ class IndexToStringTableFromFileTest(test.TestCase):
           vocab_size=2,
           default_value=default_value)
       features = table.lookup(constant_op.constant([1, 2, 4], dtypes.int64))
-      with self.assertRaises(errors_impl.OpError):
-        self.evaluate(features)
-      lookup_ops.tables_initializer().run()
+      if not context.executing_eagerly():
+        with self.assertRaises(errors_impl.OpError):
+          self.evaluate(features)
+      self.evaluate(lookup_ops.tables_initializer())
       self.assertAllEqual((b"salad", default_value, default_value),
                           self.evaluate(features))
 
-  @test_util.run_deprecated_v1
   def test_index_to_string_table_with_vocab_size_too_large(self):
     vocabulary_file = self._createVocabFile("f2i_vocab6.txt")
     with self.cached_session():
-      table = lookup_ops.index_to_string_table_from_file(
-          vocabulary_file=vocabulary_file, vocab_size=4)
-      features = table.lookup(constant_op.constant([1, 2, 4], dtypes.int64))
+      with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
+                                   "Invalid vocab_size"):
+        _ = lookup_ops.index_to_string_table_from_file(
+            vocabulary_file=vocabulary_file, vocab_size=4)
+        self.evaluate(lookup_ops.tables_initializer())
 
-      with self.assertRaises(errors_impl.OpError):
-        self.evaluate(features)
-      init = lookup_ops.tables_initializer()
-      self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
-                              "Invalid vocab_size", init.run)
-
-  @test_util.run_deprecated_v1
   def test_index_to_string_table_with_vocab_size(self):
     vocabulary_file = self._createVocabFile("f2i_vocab7.txt")
     with self.cached_session():
@@ -825,16 +808,16 @@ class IndexToStringTableFromFileTest(test.TestCase):
           vocabulary_file=vocabulary_file, vocab_size=3)
       features = table.lookup(constant_op.constant([1, 2, 4], dtypes.int64))
 
-      with self.assertRaises(errors_impl.OpError):
-        self.evaluate(features)
-      lookup_ops.tables_initializer().run()
+      if not context.executing_eagerly():
+        with self.assertRaises(errors_impl.OpError):
+          self.evaluate(features)
+      self.evaluate(lookup_ops.tables_initializer())
       self.assertAllEqual((b"salad", b"surgery", b"UNK"),
                           self.evaluate(features))
 
 
 class IndexToStringTableFromTensorTest(test.TestCase):
 
-  @test_util.run_deprecated_v1
   def test_index_to_string_table_from_tensor(self):
     with self.cached_session():
       vocabulary_list = constant_op.constant(["brain", "salad", "surgery"])
@@ -843,14 +826,14 @@ class IndexToStringTableFromTensorTest(test.TestCase):
 
       indices = constant_op.constant([0, 1, 2, 3], dtypes.int64)
       features = table.lookup(indices)
-      with self.assertRaises(errors_impl.OpError):
-        self.evaluate(features)
-      lookup_ops.tables_initializer().run()
+      if not context.executing_eagerly():
+        with self.assertRaises(errors_impl.OpError):
+          self.evaluate(features)
+      self.evaluate(lookup_ops.tables_initializer())
 
       self.assertAllEqual((b"brain", b"salad", b"surgery", b"UNK"),
                           self.evaluate(features))
 
-  @test_util.run_deprecated_v1
   def test_duplicate_entries(self):
     with self.cached_session():
       vocabulary_list = constant_op.constant(["hello", "hello"])
@@ -858,10 +841,9 @@ class IndexToStringTableFromTensorTest(test.TestCase):
           vocabulary_list=vocabulary_list)
       indices = constant_op.constant([0, 1, 4], dtypes.int64)
       features = table.lookup(indices)
-      lookup_ops.tables_initializer().run()
+      self.evaluate(lookup_ops.tables_initializer())
       self.assertAllEqual((b"hello", b"hello", b"UNK"), self.evaluate(features))
 
-  @test_util.run_deprecated_v1
   def test_index_to_string_with_default_value(self):
     default_value = b"NONE"
     with self.cached_session():
@@ -870,10 +852,10 @@ class IndexToStringTableFromTensorTest(test.TestCase):
           vocabulary_list=vocabulary_list, default_value=default_value)
       indices = constant_op.constant([1, 2, 4], dtypes.int64)
       features = table.lookup(indices)
-      with self.assertRaises(errors_impl.OpError):
-        self.evaluate(features)
-
-      lookup_ops.tables_initializer().run()
+      if not context.executing_eagerly():
+        with self.assertRaises(errors_impl.OpError):
+          self.evaluate(features)
+      self.evaluate(lookup_ops.tables_initializer())
       self.assertAllEqual((b"salad", b"surgery", default_value),
                           self.evaluate(features))
 
@@ -902,7 +884,6 @@ class InitializeTableFromFileOpTest(test.TestCase):
     result = self.evaluate(output)
     self.assertAllEqual([0, 1, -1], result)
 
-  @test_util.run_deprecated_v1
   def testInitializeInt64Table(self):
     vocabulary_file = self._createVocabFile(
         "one_column_int64.txt", values=("42", "1", "-1000"))
@@ -914,7 +895,7 @@ class InitializeTableFromFileOpTest(test.TestCase):
           dtypes.int64, lookup_ops.TextFileIndex.LINE_NUMBER)
       self.assertTrue("one_column_int64.txt_-2_-1" in init._shared_name)
       table = lookup_ops.HashTable(init, default_value)
-      table.initializer.run()
+      self.evaluate(table.initializer)
 
       output = table.lookup(
           constant_op.constant((42, 1, 11), dtype=dtypes.int64))
@@ -922,7 +903,6 @@ class InitializeTableFromFileOpTest(test.TestCase):
       result = self.evaluate(output)
       self.assertAllEqual([0, 1, -1], result)
 
-  @test_util.run_deprecated_v1
   def testInitializeIndexTable(self):
     vocabulary_file = self._createVocabFile("one_column_2.txt")
 
@@ -930,12 +910,11 @@ class InitializeTableFromFileOpTest(test.TestCase):
       default_value = "UNK"
       key_index = lookup_ops.TextFileIndex.LINE_NUMBER
       value_index = lookup_ops.TextFileIndex.WHOLE_LINE
-      init = lookup_ops.TextFileInitializer(vocabulary_file, dtypes.int64,
-                                            key_index, dtypes.string,
-                                            value_index)
+      init = lookup_ops.TextFileInitializer(
+          vocabulary_file, dtypes.int64, key_index, dtypes.string, value_index)
       self.assertTrue("one_column_2.txt_-1_-2" in init._shared_name)
       table = lookup_ops.HashTable(init, default_value)
-      table.initializer.run()
+      self.evaluate(table.initializer)
 
       input_values = constant_op.constant([0, 1, 2, 3], dtypes.int64)
       output = table.lookup(input_values)
@@ -943,7 +922,6 @@ class InitializeTableFromFileOpTest(test.TestCase):
       result = self.evaluate(output)
       self.assertAllEqual([b"brain", b"salad", b"surgery", b"UNK"], result)
 
-  @test_util.run_deprecated_v1
   def testMultiColumn(self):
     vocabulary_file = os.path.join(self.get_temp_dir(), "three_columns.txt")
     with open(vocabulary_file, "w") as f:
@@ -954,12 +932,11 @@ class InitializeTableFromFileOpTest(test.TestCase):
       key_index = 1
       value_index = 2
 
-      init = lookup_ops.TextFileInitializer(vocabulary_file, dtypes.string,
-                                            key_index, dtypes.int64,
-                                            value_index)
+      init = lookup_ops.TextFileInitializer(
+          vocabulary_file, dtypes.string, key_index, dtypes.int64, value_index)
       self.assertTrue("three_columns.txt_1_2" in init._shared_name)
       table = lookup_ops.HashTable(init, default_value)
-      table.initializer.run()
+      self.evaluate(table.initializer)
 
       input_string = constant_op.constant(["brain", "salad", "surgery"])
       output = table.lookup(input_string)
@@ -967,7 +944,6 @@ class InitializeTableFromFileOpTest(test.TestCase):
       result = self.evaluate(output)
       self.assertAllEqual([1, 5, 6], result)
 
-  @test_util.run_deprecated_v1
   def testInvalidDataTypeInMultiColumn(self):
     vocabulary_file = os.path.join(self.get_temp_dir(), "three_columns.txt")
     with open(vocabulary_file, "w") as f:
@@ -977,13 +953,12 @@ class InitializeTableFromFileOpTest(test.TestCase):
       default_value = -1
       key_index = 2
       value_index = 1
-      init = lookup_ops.TextFileInitializer(vocabulary_file, dtypes.string,
-                                            key_index, dtypes.int64,
-                                            value_index)
+      init = lookup_ops.TextFileInitializer(
+          vocabulary_file, dtypes.string, key_index, dtypes.int64, value_index)
       self.assertTrue("three_columns.txt_2_1" in init._shared_name)
-      table = lookup_ops.HashTable(init, default_value)
       with self.assertRaisesOpError("is not a valid"):
-        table.initializer.run()
+        table = lookup_ops.HashTable(init, default_value)
+        self.evaluate(table.initializer)
 
   def testInvalidDataType(self):
     vocabulary_file = self._createVocabFile("one_column_3.txt")
@@ -1000,49 +975,42 @@ class InitializeTableFromFileOpTest(test.TestCase):
         self.assertTrue("one_column_3.txt_-2_-1" in init._shared_name)
         lookup_ops.HashTable(init, default_value)
 
-  @test_util.run_deprecated_v1
   def testInvalidIndex(self):
     vocabulary_file = self._createVocabFile("one_column_4.txt")
     with self.cached_session():
       default_value = -1
       key_index = 1  # second column of the line
       value_index = lookup_ops.TextFileIndex.LINE_NUMBER
-      init = lookup_ops.TextFileInitializer(vocabulary_file, dtypes.string,
-                                            key_index, dtypes.int64,
-                                            value_index)
+      init = lookup_ops.TextFileInitializer(
+          vocabulary_file, dtypes.string, key_index, dtypes.int64, value_index)
       self.assertTrue("one_column_4.txt_1_-1" in init._shared_name)
-      table = lookup_ops.HashTable(init, default_value)
 
       with self.assertRaisesOpError("Invalid number of columns"):
-        table.initializer.run()
+        table = lookup_ops.HashTable(init, default_value)
+        self.evaluate(table.initializer)
 
-  @test_util.run_deprecated_v1
   def testInitializeSameTableWithMultipleNodes(self):
     vocabulary_file = self._createVocabFile("one_column_5.txt")
 
     with self.cached_session():
-      shared_name = "shared-one-columm"
       default_value = -1
       init1 = lookup_ops.TextFileInitializer(
           vocabulary_file, dtypes.string, lookup_ops.TextFileIndex.WHOLE_LINE,
           dtypes.int64, lookup_ops.TextFileIndex.LINE_NUMBER)
       self.assertTrue("one_column_5.txt_-2_-1" in init1._shared_name)
-      table1 = lookup_ops.HashTable(init1, default_value,
-                                    shared_name=shared_name)
+      table1 = lookup_ops.HashTable(init1, default_value)
       init2 = lookup_ops.TextFileInitializer(
           vocabulary_file, dtypes.string, lookup_ops.TextFileIndex.WHOLE_LINE,
           dtypes.int64, lookup_ops.TextFileIndex.LINE_NUMBER)
       self.assertTrue("one_column_5.txt_-2_-1" in init2._shared_name)
-      table2 = lookup_ops.HashTable(init2, default_value,
-                                    shared_name=shared_name)
+      table2 = lookup_ops.HashTable(init2, default_value)
       init3 = lookup_ops.TextFileInitializer(
           vocabulary_file, dtypes.string, lookup_ops.TextFileIndex.WHOLE_LINE,
           dtypes.int64, lookup_ops.TextFileIndex.LINE_NUMBER)
       self.assertTrue("one_column_5.txt_-2_-1" in init3._shared_name)
-      table3 = lookup_ops.HashTable(init3, default_value,
-                                    shared_name=shared_name)
+      table3 = lookup_ops.HashTable(init3, default_value)
 
-      lookup_ops.tables_initializer().run()
+      self.evaluate(lookup_ops.tables_initializer())
 
       input_string = constant_op.constant(["brain", "salad", "tank"])
 
@@ -1065,48 +1033,56 @@ class InitializeTableFromFileOpTest(test.TestCase):
                 dtypes.int64, lookup_ops.TextFileIndex.LINE_NUMBER),
             default_value)
 
-  @test_util.run_deprecated_v1
   def testInitializeWithVocabSize(self):
     with self.cached_session():
       default_value = -1
       vocab_size = 3
       vocabulary_file1 = self._createVocabFile("one_column6.txt")
       init1 = lookup_ops.TextFileInitializer(
-          vocabulary_file1, dtypes.string, lookup_ops.TextFileIndex.WHOLE_LINE,
-          dtypes.int64, lookup_ops.TextFileIndex.LINE_NUMBER,
+          vocabulary_file1,
+          dtypes.string,
+          lookup_ops.TextFileIndex.WHOLE_LINE,
+          dtypes.int64,
+          lookup_ops.TextFileIndex.LINE_NUMBER,
           vocab_size=vocab_size)
       self.assertTrue("one_column6.txt_3_-2_-1" in init1._shared_name)
       table1 = lookup_ops.HashTable(init1, default_value)
 
       # Initialize from file.
-      table1.initializer.run()
-      self.assertEquals(vocab_size, table1.size().eval())
+      self.evaluate(table1.initializer)
+      self.assertEquals(vocab_size, self.evaluate(table1.size()))
 
       vocabulary_file2 = self._createVocabFile("one_column7.txt")
       vocab_size = 5
       init2 = lookup_ops.TextFileInitializer(
-          vocabulary_file2, dtypes.string, lookup_ops.TextFileIndex.WHOLE_LINE,
-          dtypes.int64, lookup_ops.TextFileIndex.LINE_NUMBER,
+          vocabulary_file2,
+          dtypes.string,
+          lookup_ops.TextFileIndex.WHOLE_LINE,
+          dtypes.int64,
+          lookup_ops.TextFileIndex.LINE_NUMBER,
           vocab_size=vocab_size)
       self.assertTrue("one_column7.txt_5_-2_-1" in init2._shared_name)
-      table2 = lookup_ops.HashTable(init2, default_value)
       with self.assertRaisesOpError("Invalid vocab_size"):
-        table2.initializer.run()
+        table2 = lookup_ops.HashTable(init2, default_value)
+        self.evaluate(table2.initializer)
 
       vocab_size = 1
       vocabulary_file3 = self._createVocabFile("one_column3.txt")
       init3 = lookup_ops.TextFileInitializer(
-          vocabulary_file3, dtypes.string, lookup_ops.TextFileIndex.WHOLE_LINE,
-          dtypes.int64, lookup_ops.TextFileIndex.LINE_NUMBER,
+          vocabulary_file3,
+          dtypes.string,
+          lookup_ops.TextFileIndex.WHOLE_LINE,
+          dtypes.int64,
+          lookup_ops.TextFileIndex.LINE_NUMBER,
           vocab_size=vocab_size)
       self.assertTrue("one_column3.txt_1_-2_-1" in init3._shared_name)
       table3 = lookup_ops.HashTable(init3, default_value)
 
       # Smaller vocab size reads only vocab_size records.
-      table3.initializer.run()
-      self.assertEquals(vocab_size, table3.size().eval())
+      self.evaluate(table3.initializer)
+      self.assertEquals(vocab_size, self.evaluate(table3.size()))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("placeholder usage")
   def testFeedVocabularyName(self):
     vocabulary_file = self._createVocabFile("feed_vocabulary.txt")
 
@@ -1133,7 +1109,6 @@ class InitializeTableFromFileOpTest(test.TestCase):
       result = self.evaluate(output)
       self.assertAllEqual([0, 1, -1], result)
 
-  @test_util.run_deprecated_v1
   def testInvalidFilenames(self):
     vocabulary_file = self._createVocabFile("filename_shape.txt")
 
@@ -1151,14 +1126,21 @@ class InitializeTableFromFileOpTest(test.TestCase):
 
       # Non-scalar filename
       filenames = constant_op.constant([vocabulary_file, vocabulary_file])
-      with self.assertRaises(ValueError):
-        lookup_ops.HashTable(
-            lookup_ops.TextFileInitializer(
-                filenames, dtypes.string, lookup_ops.TextFileIndex.WHOLE_LINE,
-                dtypes.int64, lookup_ops.TextFileIndex.LINE_NUMBER),
-            default_value)
+      if not context.executing_eagerly():
+        with self.assertRaises(ValueError):
+          lookup_ops.HashTable(
+              lookup_ops.TextFileInitializer(
+                  filenames, dtypes.string, lookup_ops.TextFileIndex.WHOLE_LINE,
+                  dtypes.int64, lookup_ops.TextFileIndex.LINE_NUMBER),
+              default_value)
+      else:
+        with self.assertRaises(errors_impl.InvalidArgumentError):
+          lookup_ops.HashTable(
+              lookup_ops.TextFileInitializer(
+                  filenames, dtypes.string, lookup_ops.TextFileIndex.WHOLE_LINE,
+                  dtypes.int64, lookup_ops.TextFileIndex.LINE_NUMBER),
+              default_value)
 
-  @test_util.run_deprecated_v1
   def testIdToStringTable(self):
     vocab_file = self._createVocabFile("feat_to_id_1.txt")
     with self.cached_session():
@@ -1169,16 +1151,15 @@ class InitializeTableFromFileOpTest(test.TestCase):
       self.assertTrue("feat_to_id_1.txt_3_-1_-2", init._shared_name)
       table = lookup_ops.HashTable(init, default_value)
 
-      table.initializer.run()
+      self.evaluate(table.initializer)
 
       input_values = constant_op.constant([0, 1, 2, 3], dtypes.int64)
 
       out = table.lookup(input_values)
       self.assertAllEqual([b"brain", b"salad", b"surgery", b"UNK"],
                           self.evaluate(out))
-      self.assertEquals(vocab_size, table.size().eval())
+      self.assertEquals(vocab_size, self.evaluate(table.size()))
 
-  @test_util.run_deprecated_v1
   def testStringToIdTable(self):
     vocab_file = self._createVocabFile("feat_to_id_2.txt")
     with self.cached_session():
@@ -1188,15 +1169,14 @@ class InitializeTableFromFileOpTest(test.TestCase):
           vocab_file, vocab_size=vocab_size)
       self.assertTrue("feat_to_id_2.txt_3_-1_-2", init._shared_name)
       table = lookup_ops.HashTable(init, default_value)
-      table.initializer.run()
+      self.evaluate(table.initializer)
 
       input_string = constant_op.constant(["brain", "salad", "surgery", "UNK"])
 
       out = table.lookup(input_string)
       self.assertAllEqual([0, 1, 2, -1], self.evaluate(out))
-      self.assertEquals(vocab_size, table.size().eval())
+      self.assertEquals(vocab_size, self.evaluate(table.size()))
 
-  @test_util.run_deprecated_v1
   def testInt64ToIdTable(self):
     vocab_file = self._createVocabFile(
         "feat_to_id_3.txt", values=("42", "1", "-1000"))
@@ -1207,12 +1187,12 @@ class InitializeTableFromFileOpTest(test.TestCase):
           vocab_file, vocab_size=vocab_size, key_dtype=dtypes.int64)
       self.assertTrue("feat_to_id_3.txt_3_-1_-2", init._shared_name)
       table = lookup_ops.HashTable(init, default_value)
-      table.initializer.run()
+      self.evaluate(table.initializer)
 
       out = table.lookup(
           constant_op.constant((42, 1, -1000, 11), dtype=dtypes.int64))
       self.assertAllEqual((0, 1, 2, -1), self.evaluate(out))
-      self.assertEquals(vocab_size, table.size().eval())
+      self.assertEquals(vocab_size, self.evaluate(table.size()))
 
 
 class IdTableWithHashBucketsTest(test.TestCase):
@@ -1389,7 +1369,6 @@ class IdTableWithHashBucketsTest(test.TestCase):
   @test_util.run_deprecated_v1
   def testIdTableWithHashBucketsInitializationAcrossSessions(self):
     vocab_file = self._createVocabFile("feat_to_id_5.txt")
-    shared_name = "across-sessions"
     with self.cached_session():
       default_value = -1
       vocab_size = 3
@@ -1397,9 +1376,8 @@ class IdTableWithHashBucketsTest(test.TestCase):
       table1 = lookup_ops.IdTableWithHashBuckets(
           lookup_ops.HashTable(
               lookup_ops.TextFileIdTableInitializer(
-                  vocab_file, vocab_size=vocab_size),
-              default_value,
-              shared_name=shared_name), oov_buckets)
+                  vocab_file, vocab_size=vocab_size), default_value),
+          oov_buckets)
 
       table1.initializer.run()
 
@@ -1421,9 +1399,8 @@ class IdTableWithHashBucketsTest(test.TestCase):
       table2 = lookup_ops.IdTableWithHashBuckets(
           lookup_ops.HashTable(
               lookup_ops.TextFileIdTableInitializer(
-                  vocab_file, vocab_size=vocab_size),
-              default_value,
-              shared_name=shared_name), oov_buckets)
+                  vocab_file, vocab_size=vocab_size), default_value),
+          oov_buckets)
 
       input_string_2 = constant_op.constant(["fruit", "salad", "UNK"])
 
diff --git a/tensorflow/python/ops/lookup_ops.py b/tensorflow/python/ops/lookup_ops.py
index 6a7057fafd..50e14297e3 100644
--- a/tensorflow/python/ops/lookup_ops.py
+++ b/tensorflow/python/ops/lookup_ops.py
@@ -252,7 +252,7 @@ class HashTable(InitializableLookupTableBase):
   ```
   """
 
-  def __init__(self, initializer, default_value, shared_name=None, name=None):
+  def __init__(self, initializer, default_value, name=None):
     """Creates a non-initialized `HashTable` object.
 
     Creates a table, the type of its keys and values are specified by the
@@ -264,8 +264,6 @@ class HashTable(InitializableLookupTableBase):
       initializer: The table initializer to use. See `HashTable` kernel for
         supported key and value types.
       default_value: The value to use if a key is missing in the table.
-      shared_name: If non-empty, this table will be shared under
-        the given name across multiple sessions.
       name: A name for the operation (optional).
 
     Returns:
@@ -273,21 +271,22 @@ class HashTable(InitializableLookupTableBase):
     """
     self._initializer = initializer
     self._default_value = default_value
-    self._shared_name = shared_name
-    self._name = name
-    self._table_name = ""
+    self._shared_name = self._initializer._shared_name  # pylint: disable=protected-access
+    self._name = name or "hash_table"
+    self._table_name = None
     super(HashTable, self).__init__(default_value, initializer)
     self._value_shape = self._default_value.get_shape()
 
   def create_resource(self):
-    with ops.name_scope(self._name, "hash_table",
-                        (self._initializer, self._default_value)) as scope:
-      table_ref = gen_lookup_ops.hash_table_v2(
-          shared_name=self._shared_name,
-          key_dtype=self._initializer.key_dtype,
-          value_dtype=self._initializer.value_dtype,
-          name=scope)
-      self._table_name = scope.split("/")[-2]
+    table_ref = gen_lookup_ops.hash_table_v2(
+        shared_name=self._shared_name,
+        key_dtype=self._initializer.key_dtype,
+        value_dtype=self._initializer.value_dtype,
+        name=self._name)
+    if context.executing_eagerly():
+      self._table_name = None
+    else:
+      self._table_name = table_ref.op.name.split("/")[-1]
     return table_ref
 
   @property
@@ -306,9 +305,8 @@ class HashTable(InitializableLookupTableBase):
     """
     with ops.name_scope(name, "%s_Export" % self.name,
                         [self.resource_handle]) as name:
-      with ops.colocate_with(self.resource_handle):
-        exported_keys, exported_values = gen_lookup_ops.lookup_table_export_v2(
-            self.resource_handle, self._key_dtype, self._value_dtype, name=name)
+      exported_keys, exported_values = gen_lookup_ops.lookup_table_export_v2(
+          self.resource_handle, self._key_dtype, self._value_dtype, name=name)
 
     exported_values.set_shape(exported_keys.get_shape().concatenate(
         self._value_shape))
@@ -588,16 +586,14 @@ class TextFileInitializer(TableInitializerBase):
     if self._vocab_size:
       # Keep the shared_name:
       # <table_type>_<filename>_<vocab_size>_<key_index>_<value_index>
-      shared_name = "hash_table_%s_%d_%s_%s" % (self._filename_arg,
-                                                self._vocab_size,
-                                                self._key_index,
-                                                self._value_index)
+      shared_name = "hash_table_%s_%d_%s_%s" % (
+          self._filename_arg, self._vocab_size, self._key_index,
+          self._value_index)
     else:
       # Keep the shared_name
       # <table_type>_<filename>_<key_index>_<value_index>
       shared_name = "hash_table_%s_%s_%s" % (self._filename_arg,
-                                             self._key_index,
-                                             self._value_index)
+                                             self._key_index, self._value_index)
     return shared_name
 
 
@@ -850,7 +846,10 @@ class IdTableWithHashBuckets(LookupInterface):
       raise TypeError(
           "hasher_spec must be of type HasherSpec, got %s" % hasher_spec)
     self._hasher_spec = hasher_spec
-    self._table_name = name.split("/")[-1]
+    if name:
+      self._table_name = name.split("/")[-1]
+    else:
+      self._table_name = None
     super(IdTableWithHashBuckets, self).__init__(key_dtype, dtypes.int64)
 
   def create_resource(self):
@@ -1053,20 +1052,7 @@ def index_table_from_file(vocabulary_file=None,
 
   with ops.name_scope(name, "string_to_index") as feat_to_id_scope:
     table = None
-    shared_name = ""
     with ops.name_scope(None, "hash_table") as hash_table_scope:
-      if vocab_size:
-        # Keep the shared_name:
-        # <table_type>_<filename>_<vocab_size>_<key_index>_<value_index>
-        shared_name = "hash_table_%s_%d_%s_%s" % (vocabulary_file, vocab_size,
-                                                  key_column_index,
-                                                  value_column_index)
-      else:
-        # Keep the shared_name
-        # <table_type>_<filename>_<key_index>_<value_index>
-        shared_name = "hash_table_%s_%s_%s" % (vocabulary_file,
-                                               key_column_index,
-                                               value_column_index)
       init = TextFileIdTableInitializer(
           vocabulary_file,
           vocab_size=vocab_size,
@@ -1076,8 +1062,7 @@ def index_table_from_file(vocabulary_file=None,
           value_column_index=value_column_index,
           delimiter=delimiter)
 
-      table = HashTable(
-          init, default_value, shared_name=shared_name, name=hash_table_scope)
+      table = HashTable(init, default_value, name=hash_table_scope)
     if num_oov_buckets:
       table = IdTableWithHashBuckets(
           table,
@@ -1167,12 +1152,7 @@ def index_table_from_tensor(vocabulary_list,
     num_elements = array_ops.size(keys)
     values = math_ops.to_int64(math_ops.range(num_elements))
 
-    shared_name = ""
     with ops.name_scope(None, "hash_table") as hash_table_scope:
-      if context.executing_eagerly():
-        # Ensure a unique name when eager execution is enabled to avoid spurious
-        # sharing issues.
-        shared_name += str(ops.uid())
       table_keys = math_ops.to_int64(keys) if keys.dtype.is_integer else keys
       init = KeyValueTensorInitializer(
           table_keys,
@@ -1180,8 +1160,7 @@ def index_table_from_tensor(vocabulary_list,
           table_keys.dtype.base_dtype,
           dtypes.int64,
           name="table_init")
-      table = HashTable(
-          init, default_value, shared_name=shared_name, name=hash_table_scope)
+      table = HashTable(init, default_value, name=hash_table_scope)
     if num_oov_buckets:
       table = IdTableWithHashBuckets(
           table,
@@ -1270,17 +1249,6 @@ def index_to_string_table_from_file(vocabulary_file,
     raise ValueError("vocab_size must be greater than 0, got %d." % vocab_size)
 
   with ops.name_scope(name, "index_to_string") as scope:
-    shared_name = ""
-    if vocab_size:
-      # Keep a shared_name
-      # <table_type>_<filename>_<vocab_size>_<key_index>_<value_index>
-      shared_name = "hash_table_%s_%d_%s_%s" % (vocabulary_file, vocab_size,
-                                                key_column_index,
-                                                value_column_index)
-    else:
-      # Keep a shared_name <table_type>_<filename>_<key_index>_<value_index>
-      shared_name = "hash_table_%s_%s_%s" % (vocabulary_file, key_column_index,
-                                             value_column_index)
     init = TextFileStringTableInitializer(
         vocabulary_file,
         vocab_size=vocab_size,
@@ -1290,7 +1258,7 @@ def index_to_string_table_from_file(vocabulary_file,
         delimiter=delimiter)
 
     # TODO(yleon): Use a more effienct structure.
-    return HashTable(init, default_value, shared_name=shared_name, name=scope)
+    return HashTable(init, default_value, name=scope)
 
 
 def index_to_string_table_from_tensor(vocabulary_list,
@@ -1348,11 +1316,10 @@ def index_to_string_table_from_tensor(vocabulary_list,
     num_elements = array_ops.size(vocabulary_list)
     keys = math_ops.to_int64(math_ops.range(num_elements))
 
-    shared_name = ""
     init = KeyValueTensorInitializer(
         keys, vocabulary_list, dtypes.int64, dtypes.string, name="table_init")
     # TODO(yleon): Use a more effienct structure.
-    return HashTable(init, default_value, shared_name=shared_name, name=scope)
+    return HashTable(init, default_value, name=scope)
 
 
 ops.NotDifferentiable("LookupTableFind")
-- 
GitLab


From 1fc63acf51901911183d40c531f471a9eb26e3a7 Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Mon, 28 Jan 2019 10:28:37 -0800
Subject: [PATCH 1368/2345] Remove the resize op from the gradient of
 TensorListGather to avoid implementing TensorListResize in XLA. Instead we
 directly pass a num_elements arg to TensorListScatter.

PiperOrigin-RevId: 231244477
---
 .../api_def_TensorListScatterV2.pbtxt         | 17 ++++++++++
 .../api_def_TensorListScatterV2.pbtxt         |  4 +++
 tensorflow/core/kernels/list_kernels.cc       |  4 +++
 tensorflow/core/kernels/list_kernels.cu.cc    |  7 ++++
 tensorflow/core/kernels/list_kernels.h        | 25 ++++++++++----
 tensorflow/core/ops/list_ops.cc               | 20 +++++++++++
 .../python/kernel_tests/list_ops_test.py      | 33 +++++++++++++++---
 tensorflow/python/ops/list_ops.py             | 34 +++++++++++++------
 tensorflow/python/ops/tensor_array_ops.py     |  3 +-
 9 files changed, 125 insertions(+), 22 deletions(-)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorListScatterV2.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_TensorListScatterV2.pbtxt

diff --git a/tensorflow/core/api_def/base_api/api_def_TensorListScatterV2.pbtxt b/tensorflow/core/api_def/base_api/api_def_TensorListScatterV2.pbtxt
new file mode 100644
index 0000000000..1f520900fc
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_TensorListScatterV2.pbtxt
@@ -0,0 +1,17 @@
+op {
+  graph_op_name: "TensorListScatterV2"
+  summary: "Creates a TensorList by indexing into a Tensor."
+  description: <<END
+Each member of the TensorList corresponds to one row of the input tensor,
+specified by the given index (see `tf.gather`).
+
+tensor: The input tensor.
+indices: The indices used to index into the list.
+element_shape: The shape of the elements in the list (can be less specified than
+  the shape of the tensor).
+num_elements: The size of the output list. Must be large enough to accommodate
+  the largest index in indices. If -1, the list is just large enough to include
+  the largest index in indices.
+output_handle: The TensorList.
+END
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_TensorListScatterV2.pbtxt b/tensorflow/core/api_def/python_api/api_def_TensorListScatterV2.pbtxt
new file mode 100644
index 0000000000..d2a92a529d
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_TensorListScatterV2.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "TensorListScatterV2"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/kernels/list_kernels.cc b/tensorflow/core/kernels/list_kernels.cc
index 55c2facc3c..1e449ddb2c 100644
--- a/tensorflow/core/kernels/list_kernels.cc
+++ b/tensorflow/core/kernels/list_kernels.cc
@@ -652,6 +652,10 @@ REGISTER_KERNEL_BUILDER(Name("TensorListConcatLists").Device(DEVICE_GPU),
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_CPU),                \
                           TensorListScatter<CPUDevice, T>)        \
+  REGISTER_KERNEL_BUILDER(Name("TensorListScatterV2")             \
+                              .TypeConstraint<T>("element_dtype") \
+                              .Device(DEVICE_CPU),                \
+                          TensorListScatter<CPUDevice, T>)        \
   REGISTER_KERNEL_BUILDER(Name("TensorListSplit")                 \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_CPU),                \
diff --git a/tensorflow/core/kernels/list_kernels.cu.cc b/tensorflow/core/kernels/list_kernels.cu.cc
index 1d5d747610..5259389712 100644
--- a/tensorflow/core/kernels/list_kernels.cu.cc
+++ b/tensorflow/core/kernels/list_kernels.cu.cc
@@ -79,6 +79,13 @@ typedef Eigen::GpuDevice GPUDevice;
                               .HostMemory("element_shape")        \
                               .HostMemory("indices"),             \
                           TensorListScatter<GPUDevice, T>)        \
+  REGISTER_KERNEL_BUILDER(Name("TensorListScatterV2")             \
+                              .TypeConstraint<T>("element_dtype") \
+                              .Device(DEVICE_GPU)                 \
+                              .HostMemory("element_shape")        \
+                              .HostMemory("num_elements")         \
+                              .HostMemory("indices"),             \
+                          TensorListScatter<GPUDevice, T>)        \
   REGISTER_KERNEL_BUILDER(Name("TensorListSplit")                 \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_GPU)                 \
diff --git a/tensorflow/core/kernels/list_kernels.h b/tensorflow/core/kernels/list_kernels.h
index f74a88a768..7b3ff078c8 100644
--- a/tensorflow/core/kernels/list_kernels.h
+++ b/tensorflow/core/kernels/list_kernels.h
@@ -663,6 +663,13 @@ class TensorListScatter : public OpKernel {
     Tensor indices = c->input(1);
     PartialTensorShape element_shape;
     OP_REQUIRES_OK(c, TensorShapeFromTensor(c->input(2), &element_shape));
+    // TensorListScatterV2 passes the num_elements input, TensorListScatter does
+    // not.
+    int num_elements = c->num_inputs() >= 4 ? c->input(3).scalar<int>()() : -1;
+    OP_REQUIRES(c, num_elements >= -1,
+                errors::InvalidArgument(
+                    "TensorListScatter expects num_elements >= -1, found: ",
+                    num_elements));
     TensorList output_list;
     const Tensor& input_tensor = c->input(0);
     output_list.element_dtype = input_tensor.dtype();
@@ -686,17 +693,23 @@ class TensorListScatter : public OpKernel {
 
     // Validate indices and resize output_list.tensors to fit the highest index.
     {
-      size_t list_size = 0;
+      int highest_index = -1;
       for (int index = 0; index < indices.NumElements(); ++index) {
         const int i = indices.flat<int32>()(index);
-        OP_REQUIRES(c, i >= 0,
+        OP_REQUIRES(
+            c, i >= 0,
+            errors::InvalidArgument(
+                "Indices in TensorListScatter must all be non-negative."));
+        OP_REQUIRES(c, num_elements == -1 || i < num_elements,
                     errors::InvalidArgument(
-                        "Indices in TensorListScatter must all be positive."));
-        if (i >= list_size) {
-          list_size = i + 1;
+                        "TensorListScatter: Trying to scatter at index ", i,
+                        " in list with size ", num_elements));
+        if (i > highest_index) {
+          highest_index = i;
         }
       }
-      output_list.tensors.resize(list_size, Tensor(DT_INVALID));
+      output_list.tensors.resize(std::max(highest_index + 1, num_elements),
+                                 Tensor(DT_INVALID));
     }
 
     for (int index = 0; index < indices.NumElements(); ++index) {
diff --git a/tensorflow/core/ops/list_ops.cc b/tensorflow/core/ops/list_ops.cc
index 48b6e016c9..fdaa5a2129 100644
--- a/tensorflow/core/ops/list_ops.cc
+++ b/tensorflow/core/ops/list_ops.cc
@@ -465,6 +465,26 @@ REGISTER_OP("TensorListScatter")
       return Status::OK();
     });
 
+REGISTER_OP("TensorListScatterV2")
+    .Input("tensor: element_dtype")
+    .Input("indices: int32")
+    .Input("element_shape: shape_type")
+    .Input("num_elements: int32")
+    .Output("output_handle: variant")
+    .Attr("element_dtype: type")
+    .Attr("shape_type: {int32, int64}")
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      DataType element_dtype;
+      TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype));
+      shape_inference::ShapeHandle element_shape;
+      TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensorTreatScalarAsUnknownShape(
+          2, &element_shape));
+      c->set_output_handle_shapes_and_types(0,
+                                            {{element_shape, element_dtype}});
+      c->set_output(0, c->Scalar());
+      return Status::OK();
+    });
+
 REGISTER_OP("TensorListConcatLists")
     .Input("input_a: variant")
     .Input("input_b: variant")
diff --git a/tensorflow/python/kernel_tests/list_ops_test.py b/tensorflow/python/kernel_tests/list_ops_test.py
index 88ca6e85df..e203d1b909 100644
--- a/tensorflow/python/kernel_tests/list_ops_test.py
+++ b/tensorflow/python/kernel_tests/list_ops_test.py
@@ -463,25 +463,48 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
   def testScatterOutputListSize(self):
     c0 = constant_op.constant([1.0, 2.0])
-    l = list_ops.tensor_list_scatter(
-        c0, [1, 3], ops.convert_to_tensor([], dtype=dtypes.int32))
+    l = list_ops.tensor_list_scatter(c0, [1, 3], [])
     # TensorListScatter should return a list with size largest index + 1.
     self.assertAllEqual(list_ops.tensor_list_length(l), 4)
 
+  def testScatterOutputListSizeWithNumElementsSpecified(self):
+    c0 = constant_op.constant([1.0, 2.0])
+    l = gen_list_ops.tensor_list_scatter_v2(
+        c0, [1, 3], list_ops._build_element_shape([]), num_elements=5)
+    # TensorListScatter should return a list with size num_elements.
+    self.assertAllEqual(list_ops.tensor_list_length(l), 5)
+
+  def testScatterFailsWhenIndexLargerThanNumElements(self):
+    c0 = constant_op.constant([1.0, 2.0])
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        "TensorListScatter: Trying to scatter at index 3 in list with size 3"):
+      l = gen_list_ops.tensor_list_scatter_v2(
+          c0, [1, 3], list_ops._build_element_shape([]), num_elements=3)
+      self.evaluate(l)
+
+  def testScatterFailsWithInvalidNumElements(self):
+    c0 = constant_op.constant([1.0, 2.0])
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        "TensorListScatter expects num_elements >= -1, found: -2"):
+      l = gen_list_ops.tensor_list_scatter_v2(
+          c0, [1, 3], list_ops._build_element_shape([]), num_elements=-2)
+      self.evaluate(l)
+
   def testScatterWithInvalidRowsInInputTensorFails(self):
     c0 = constant_op.constant([1.0, 2.0])
     with self.assertRaisesRegexp(
         errors.InvalidArgumentError,
         "Invalid number of rows in input tensor. Expected: 3 Actual: 2"):
-      l = list_ops.tensor_list_scatter(
-          c0, [1, 0, 2], ops.convert_to_tensor([], dtype=dtypes.int32))
+      l = list_ops.tensor_list_scatter(c0, [1, 0, 2], [])
       self.evaluate(l)
 
   def testScatterWithNegativeIndicesFails(self):
     c0 = constant_op.constant([1.0, 2.0])
     with self.assertRaisesRegexp(
         errors.InvalidArgumentError,
-        "Indices in TensorListScatter must all be positive."):
+        "Indices in TensorListScatter must all be non-negative."):
       l = list_ops.tensor_list_scatter(
           c0, [-1, -2], ops.convert_to_tensor([], dtype=dtypes.int32))
       self.evaluate(l)
diff --git a/tensorflow/python/ops/list_ops.py b/tensorflow/python/ops/list_ops.py
index af059a541e..fcd5b816fe 100644
--- a/tensorflow/python/ops/list_ops.py
+++ b/tensorflow/python/ops/list_ops.py
@@ -97,6 +97,15 @@ def tensor_list_gather(input_handle, indices, element_dtype, name=None):
       name=name)
 
 
+def tensor_list_scatter(tensor, indices, element_shape, name=None):
+  return gen_list_ops.tensor_list_scatter_v2(
+      tensor=tensor,
+      indices=indices,
+      element_shape=_build_element_shape(element_shape),
+      num_elements=-1,
+      name=name)
+
+
 def tensor_list_stack(input_handle, element_dtype, num_elements=-1, name=None):
   return gen_list_ops.tensor_list_stack(
       input_handle=input_handle,
@@ -248,25 +257,30 @@ def _TensorListResizeGrad(op, dlist):
 def _TensorListGatherGrad(op, dtensor):
   """Gradient function for TensorListGather."""
   input_list, indices, _ = op.inputs
-  dlist = gen_list_ops.tensor_list_scatter(
+  dlist = gen_list_ops.tensor_list_scatter_v2(
       tensor=dtensor,
       indices=indices,
-      element_shape=ops.convert_to_tensor(-1, dtype=dtypes.int32))
-  # TensorListScatter returns a list with size `max(indices) + 1`
-  # so we manually resize it to match the size of the input list.
-  input_list_size = gen_list_ops.tensor_list_length(input_list)
-  dlist = gen_list_ops.tensor_list_resize(dlist, input_list_size)
+      element_shape=gen_list_ops.tensor_list_element_shape(
+          input_list, shape_type=dtypes.int32),
+      num_elements=gen_list_ops.tensor_list_length(input_list))
   return dlist, None, None
 
 
 @ops.RegisterGradient("TensorListScatter")
+@ops.RegisterGradient("TensorListScatterV2")
 def _TensorListScatterGrad(op, dlist):
-  t, indices, _ = op.inputs
-  return gen_list_ops.tensor_list_gather(
+  """Gradient function for TensorListScatter."""
+  tensor = op.inputs[0]
+  indices = op.inputs[1]
+  dtensor = gen_list_ops.tensor_list_gather(
       dlist,
       indices,
-      element_shape=array_ops.slice(array_ops.shape(t), [1], [-1]),
-      element_dtype=t.dtype), None, None
+      element_shape=array_ops.slice(array_ops.shape(tensor), [1], [-1]),
+      element_dtype=tensor.dtype)
+  if op.type == "TensorListScatterV2":
+    return dtensor, None, None, None
+  else:
+    return dtensor, None, None
 
 
 def _build_element_shape(shape):
diff --git a/tensorflow/python/ops/tensor_array_ops.py b/tensorflow/python/ops/tensor_array_ops.py
index 90a8b0af46..41a481430f 100644
--- a/tensorflow/python/ops/tensor_array_ops.py
+++ b/tensorflow/python/ops/tensor_array_ops.py
@@ -619,8 +619,9 @@ class _GraphTensorArrayV2(object):
       value = ops.convert_to_tensor(value, name="value")
       if self._infer_shape and not context.executing_eagerly():
         self._merge_element_shape(value.shape[1:])
+      element_shape = self._element_shape[0] if self._element_shape else None
       flow_out = list_ops.tensor_list_scatter(
-          tensor=value, indices=indices, element_shape=-1)
+          tensor=value, indices=indices, element_shape=element_shape)
       return build_ta_with_new_flow(self, flow_out)
 
   @tf_should_use.should_use_result
-- 
GitLab


From 5a5e93e4f245146e6c2b3eac5173b48d15755a27 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 28 Jan 2019 10:32:27 -0800
Subject: [PATCH 1369/2345] Add OpKernels for some stateless maps

PiperOrigin-RevId: 231245207
---
 tensorflow/core/kernels/lookup_tables/BUILD   |  17 +++
 .../lookup_tables/fingerprint64_map_ops.cc    | 144 ++++++++++++++++++
 .../lookup_tables/lookup_table_interface.h    | 108 +++++--------
 .../kernels/lookup_tables/table_op_utils.h    |  47 +++---
 tensorflow/core/ops/fingerprint64_map_ops.cc  |  37 +++++
 5 files changed, 258 insertions(+), 95 deletions(-)
 create mode 100644 tensorflow/core/kernels/lookup_tables/fingerprint64_map_ops.cc
 create mode 100644 tensorflow/core/ops/fingerprint64_map_ops.cc

diff --git a/tensorflow/core/kernels/lookup_tables/BUILD b/tensorflow/core/kernels/lookup_tables/BUILD
index 53074f0819..f4a41a5cfc 100644
--- a/tensorflow/core/kernels/lookup_tables/BUILD
+++ b/tensorflow/core/kernels/lookup_tables/BUILD
@@ -10,6 +10,8 @@ package(
 
 licenses(["notice"])  # Apache 2.0
 
+load("//tensorflow:tensorflow.bzl", "tf_kernel_library")
+
 cc_library(
     name = "lookup_table_interface",
     hdrs = ["lookup_table_interface.h"],
@@ -17,6 +19,7 @@ cc_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "@com_google_absl//absl/types:optional",
         "@com_google_absl//absl/types:span",
     ],
 )
@@ -41,3 +44,17 @@ cc_library(
         "@com_google_absl//absl/base:core_headers",
     ],
 )
+
+tf_kernel_library(
+    name = "fingerprint64_map_ops",
+    srcs = [
+        "fingerprint64_map_ops.cc",
+    ],
+    deps = [
+        ":table_op_utils",
+        ":table_resource_utils",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "@com_google_absl//absl/strings",
+    ],
+)
diff --git a/tensorflow/core/kernels/lookup_tables/fingerprint64_map_ops.cc b/tensorflow/core/kernels/lookup_tables/fingerprint64_map_ops.cc
new file mode 100644
index 0000000000..a000828c4b
--- /dev/null
+++ b/tensorflow/core/kernels/lookup_tables/fingerprint64_map_ops.cc
@@ -0,0 +1,144 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "absl/strings/string_view.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/kernels/lookup_tables/table_op_utils.h"
+#include "tensorflow/core/kernels/lookup_tables/table_resource_utils.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/platform/fingerprint.h"
+#include "tensorflow/core/platform/macros.h"
+
+namespace tensorflow {
+namespace tables {
+
+// Map x -> (Fingerprint64(x) % num_oov_buckets) + offset.
+// num_oov_buckets and offset are node attributes provided at construction
+// time.
+template <class HeterogeneousKeyType, class ValueType>
+class Fingerprint64Map final
+    : public LookupTableInterface<HeterogeneousKeyType, ValueType> {
+ public:
+  Fingerprint64Map(int64 num_oov_buckets, int64 offset)
+      : num_oov_buckets_(num_oov_buckets), offset_(offset) {}
+
+  mutex* GetMutex() const override { return nullptr; }
+
+  bool UnsafeInsertOrAssign(const HeterogeneousKeyType& key,
+                            const ValueType& value) override {
+    return true;
+  }
+
+  Status TableUnbatchedInsertStatus() const override {
+    return errors::Unimplemented("Fingerprint64Map does not support inserts.");
+  }
+
+  Status BatchInsertOrAssign(absl::Span<const HeterogeneousKeyType> keys,
+                             absl::Span<const ValueType> values) override {
+    return errors::Unimplemented("Fingerprint64Map does not support inserts.");
+  }
+
+  ValueType UnsafeLookupKey(
+      const HeterogeneousKeyType& key_to_find) const override {
+    // This can cause a downcast.
+    return static_cast<ValueType>(Fingerprint64(key_to_find) %
+                                  num_oov_buckets_) +
+           offset_;
+  }
+
+  Status TableUnbatchedLookupStatus() const override { return Status::OK(); }
+
+  Status BatchLookup(absl::Span<const HeterogeneousKeyType> keys,
+                     absl::Span<ValueType> values,
+                     int64 prefetch_lookahead) const override {
+    if (ABSL_PREDICT_FALSE(keys.size() != values.size())) {
+      return errors::InvalidArgument(
+          "keys and values do not have the same number of elements (found ",
+          keys.size(), " vs ", values.size(), ").");
+    }
+    for (size_t i = 0; i < keys.size(); ++i) {
+      values[i] = Fingerprint64Map::UnsafeLookupKey(keys[i]);
+    }
+    return Status::OK();
+  }
+
+  const absl::optional<const ValueType> DefaultValue() const override {
+    return {};
+  }
+
+  void UnsafePrefetchKey(
+      const HeterogeneousKeyType& key_to_find) const override {}
+
+  size_t UnsafeSize() const override { return 0; }
+
+  Status SizeStatus() const override {
+    return errors::Unimplemented(
+        "Fingerprint64Map does not have a concept of size.");
+  }
+
+  bool UnsafeContainsKey(
+      const HeterogeneousKeyType& key_to_find) const override {
+    return true;
+  }
+
+ private:
+  const int64 num_oov_buckets_;
+  const int64 offset_;
+  TF_DISALLOW_COPY_AND_ASSIGN(Fingerprint64Map);
+};
+
+template <typename Fingerprint64Map>
+struct Fingerprint64MapFactory {
+  struct Functor {
+    template <typename ContainerBase>
+    static Status AllocateContainer(OpKernelContext* ctx, OpKernel* kernel,
+                                    ContainerBase** container) {
+      int64 num_oov_buckets;
+      int64 offset;
+      TF_RETURN_IF_ERROR(
+          GetNodeAttr(kernel->def(), "num_oov_buckets", &num_oov_buckets));
+      TF_RETURN_IF_ERROR(GetNodeAttr(kernel->def(), "offset", &offset));
+      *container = new Fingerprint64Map(num_oov_buckets, offset);
+      return Status::OK();
+    }
+  };
+};
+
+#define REGISTER_STRING_KERNEL(table_value_dtype)                             \
+  REGISTER_KERNEL_BUILDER(                                                    \
+      Name("Fingerprint64Map")                                                \
+          .Device(DEVICE_CPU)                                                 \
+          .TypeConstraint<Variant>("heterogeneous_key_dtype")                 \
+          .TypeConstraint<table_value_dtype>("table_value_dtype"),            \
+      ResourceConstructionOp<                                                 \
+          LookupTableInterface<absl::string_view, table_value_dtype>,         \
+          Fingerprint64MapFactory<Fingerprint64Map<                           \
+              absl::string_view, table_value_dtype>>::Functor>);              \
+  REGISTER_KERNEL_BUILDER(                                                    \
+      Name("Fingerprint64Map")                                                \
+          .Device(DEVICE_CPU)                                                 \
+          .TypeConstraint<string>("heterogeneous_key_dtype")                  \
+          .TypeConstraint<table_value_dtype>("table_value_dtype"),            \
+      ResourceConstructionOp<LookupTableInterface<string, table_value_dtype>, \
+                             Fingerprint64MapFactory<Fingerprint64Map<        \
+                                 string, table_value_dtype>>::Functor>);
+
+REGISTER_STRING_KERNEL(int32);
+REGISTER_STRING_KERNEL(int64);
+
+#undef REGISTER_STRING_KERNEL
+
+}  // namespace tables
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/lookup_tables/lookup_table_interface.h b/tensorflow/core/kernels/lookup_tables/lookup_table_interface.h
index 62121de618..0cfe44eda7 100644
--- a/tensorflow/core/kernels/lookup_tables/lookup_table_interface.h
+++ b/tensorflow/core/kernels/lookup_tables/lookup_table_interface.h
@@ -19,6 +19,7 @@ limitations under the License.
 #include <cstddef>
 #include <string>
 
+#include "absl/types/optional.h"
 #include "absl/types/span.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/lib/core/status.h"
@@ -28,26 +29,10 @@ namespace tensorflow {
 namespace tables {
 
 // Interface for key-value pair lookups with support for heterogeneous keys.
-// One of the key high performance abstractions introduced here is the
-// hierarchy of a primary and secondary (or fallback) table.
-// The primary table can be considered the main data structure into which
-// key-value pairs may be inserted.
-// When a key is requested, LookupTableInterface::LookupKey semantics expect
-// that the primary table be queried first. The secondary table is queried
-// when the primary table lookup fails (eg. does not contain a requested key).
-// Lookups in the secondary table must always return well defined values in
-// this case.
-// LookupTableInterface features batch and serial table insert/lookup
-// methods. Serial methods have corresponding status methods which say
-// whether the table supports serial inserts/lookups which are guaranteed to
-// succeed.
-// For example, an implementation may only support a one-time batch population
-// after which further insertions are undefined. Another may not permit lookups
-// before its size is non zero.
-// In each of these cases, the success or failure of inserts/lookups is not
-// dependent on the parameter values. On the other hand, once the table is in
-// ready state, LookupTableInterface semantics guarantee that all lookups and
-// inserts will succeed.
+// This class contains two main kinds of methods: methods which operate on
+// a batch of inputs and methods which do not. The latter have the prefix
+// 'Unsafe'. Clients must call the corresponding status methods to determine
+// whether they are safe to call within a code block.
 // Implementations must guarantee thread-safety when GetMutex is used to
 // synchronize method access.
 template <typename HeterogeneousKeyType, typename ValueType>
@@ -64,17 +49,17 @@ class LookupTableInterface : public ResourceBase {
   // Insert the KV pair into the underlying table. If a key equivalent to key
   // already exists in the underlying table, its corresponding value is
   // overridden. Returns true only if the key was inserted for the first time.
-  // Undefined if TableInsertStatus() != OK.
-  virtual bool InsertOrAssign(const HeterogeneousKeyType& key,
-                              const ValueType& value) = 0;
+  // Undefined if TableUnbatchedInsertStatus() != OK.
+  virtual bool UnsafeInsertOrAssign(const HeterogeneousKeyType& key,
+                                    const ValueType& value) = 0;
 
   // Returns OK if it is safe to call InsertOrAssign.
   // Once OK is returned, it is safe to call InsertOrAssign for the rest of the
   // program.
-  virtual Status TableInsertStatus() const TF_MUST_USE_RESULT = 0;
+  virtual Status TableUnbatchedInsertStatus() const TF_MUST_USE_RESULT = 0;
 
   // Stores each KV pair {keys[i], values[i]} in the underlying map, overriding
-  // pre-existing pairs with equivalent keys.
+  // pre-existing pairs which have equivalent keys.
   // keys and values should have the same size.
   virtual Status BatchInsertOrAssign(
       absl::Span<const HeterogeneousKeyType> keys,
@@ -82,64 +67,49 @@ class LookupTableInterface : public ResourceBase {
 
   // Prefetch key_to_find into implementation defined data caches.
   // Implementations are free to leave this a no-op.
-  // Undefined if TableLookupStatus() != OK.
-  virtual void PrefetchKey(const HeterogeneousKeyType& key_to_find) const {}
-
-  // Returns true if and only if the primary table contains key_to_find.
-  // What constitutes the primary table is implementation defined.
-  // Undefined if TableLookupStatus() != OK.
-  virtual bool ContainsKey(const HeterogeneousKeyType& key_to_find) const = 0;
-
-  // Lookup the value for key_to_find in the primary. If this lookup fails
-  // (for eg. the key does not exist), the value in the secondary table is
-  // returned.
-  // Undefined if TableLookupStatus() != OK.
-  virtual ValueType LookupKey(
-      const HeterogeneousKeyType& key_to_find) const = 0;
+  // Undefined if TableUnbatchedLookupStatus() != OK.
+  virtual void UnsafePrefetchKey(
+      const HeterogeneousKeyType& key_to_find) const {}
 
-  // Lookup the value for key_to_find in the primary table and return it.
-  // If the lookup fails, the return value is undefined unless
-  // PrimaryTableDefaultValue() != NULL in which case
-  // *PrimaryTableDefaultValue() is returned.
-  // Undefined if TableLookupStatus() != OK.
-  virtual ValueType LookupKeyInPrimaryTable(
+  // Returns true if and only if the table contains key_to_find.
+  // Undefined if TableUnbatchedLookupStatus() != OK.
+  virtual bool UnsafeContainsKey(
       const HeterogeneousKeyType& key_to_find) const = 0;
 
-  // Lookup the value for key_to_find in the secondary table and return it.
-  // This value may not be well defined if key_to_find is in the primary table.
-  // Undefined if TableLookupStatus() != OK.
-  virtual ValueType LookupKeyInSecondaryTable(
+  // Lookup the value for key_to_find. This value must always be well-defined,
+  // even when ContainsKey(key_to_find) == false. When
+  // dv = DefaultValue() != absl::nullopt and ContainsKey(key_to_find) == false,
+  // dv is returned.
+  // Undefined if TableUnbatchedLookupStatus() != OK.
+  virtual ValueType UnsafeLookupKey(
       const HeterogeneousKeyType& key_to_find) const = 0;
 
-  // Lookup the values for keys in the primary or secondary table and store
-  // them in values. prefetch_lookahead is used to prefetch the key at index
+  // Returns OK if it is safe to call PrefetchKey, ContainsKey, and
+  // UnsafeLookupKey.
+  // If OK is returned, it is safe to call these methods until the next
+  // non-const method of this class is called.
+  virtual Status TableUnbatchedLookupStatus() const TF_MUST_USE_RESULT = 0;
+
+  // Lookup the values for keys and store them in values.
+  // prefetch_lookahead is used to prefetch the key at index
   // i + prefetch_lookahead at the ith iteration of the implemented loop.
-  // Undefined if TableLookupStatus() != OK.
+  // keys and values must have the same size.
   virtual Status BatchLookup(absl::Span<const HeterogeneousKeyType> keys,
                              absl::Span<ValueType> values,
                              int64 prefetch_lookahead) const = 0;
 
-  // Returns OK if it is safe to call PrefetchKey, ContainsKey, LookupKey,
-  // LookupKeyInPrimaryTable, and LookupKeyInSecondaryTable.
-  // Once OK is returned, it is safe to call these methods until the next
-  // non-const method of this class is called.
-  virtual Status TableLookupStatus() const TF_MUST_USE_RESULT = 0;
-
-  // If non-null value is returned, *value is guaranteed to not be a value in
-  // the primary table. LookupKeyInPrimaryTable returns *value for keys
-  // which are not in the primary table.
-  virtual const ValueType* PrimaryTableDefaultValue() const = 0;
-
-  // Returns the number of elements in the primary table.
+  // Returns the number of elements in the table.
   // Undefined if SizeStatus() != OK.
-  virtual size_t size() const = 0;
+  virtual size_t UnsafeSize() const = 0;
 
-  // Returns OK if the return value of size() is well defined.
+  // Returns OK if the return value of UnsafeSize() is always well-defined.
   virtual Status SizeStatus() const TF_MUST_USE_RESULT = 0;
 
-  string DebugString() const override {
-    return strings::StrCat("A lookup table of size: ", size());
-  }
+  // If non-null value is returned, LookupKey returns that value only for keys
+  // which satisfy ContainsKey(key_to_find) == false.
+  virtual const absl::optional<const ValueType> DefaultValue() const = 0;
+
+  string DebugString() const override { return "A lookup table"; }
 
   ~LookupTableInterface() override = default;
 };
diff --git a/tensorflow/core/kernels/lookup_tables/table_op_utils.h b/tensorflow/core/kernels/lookup_tables/table_op_utils.h
index fd850574b5..4bc18c7dc2 100644
--- a/tensorflow/core/kernels/lookup_tables/table_op_utils.h
+++ b/tensorflow/core/kernels/lookup_tables/table_op_utils.h
@@ -35,16 +35,15 @@ limitations under the License.
 namespace tensorflow {
 namespace tables {
 
-// Lookup or create resources of type Container but treat them as having type
-// ContainerBase for the purpose of dynamic dispatching.
-// Container must have constructor Container(OpKernelContext*, OpKernel*)
-template <class ContainerBase, class ContainerChild>
+// Create resources of type ContainerBase using the static method
+// Functor::AllocateContainer(OpKernelConstruction*, OpKernel*,
+// ContainerBase**)
+// If the resource has already been created it will be looked up.
+template <class ContainerBase, typename Functor>
 class ResourceConstructionOp : public OpKernel {
  public:
   explicit ResourceConstructionOp(OpKernelConstruction* ctx)
       : OpKernel(ctx), table_handle_set_(false) {
-    static_assert(std::is_base_of<ContainerBase, ContainerChild>::value,
-                  "ContainerChild is not derived from ContainerBase");
     OP_REQUIRES_OK(
         ctx, ctx->GetAttr("use_node_name_sharing", &use_node_name_sharing_));
   }
@@ -59,10 +58,11 @@ class ResourceConstructionOp : public OpKernel {
 
     auto creator = [ctx,
                     this](ContainerBase** ret) EXCLUSIVE_LOCKS_REQUIRED(mu_) {
-      ContainerBase* container = new ContainerChild(ctx, this);
-      if (!ctx->status().ok()) {
+      ContainerBase* container;
+      auto status = Functor::AllocateContainer(ctx, this, &container);
+      if (TF_PREDICT_FALSE(!status.ok())) {
         container->Unref();
-        return ctx->status();
+        return status;
       }
       if (ctx->track_allocations()) {
         ctx->record_persistent_memory_allocation(container->MemoryUsed());
@@ -98,7 +98,6 @@ class ResourceConstructionOp : public OpKernel {
 
  private:
   mutex mu_;
-  PersistentTensor table_handle_ GUARDED_BY(mu_);
   bool table_handle_set_ GUARDED_BY(mu_);
   ContainerInfo cinfo_;
   bool use_node_name_sharing_;
@@ -106,23 +105,18 @@ class ResourceConstructionOp : public OpKernel {
   TF_DISALLOW_COPY_AND_ASSIGN(ResourceConstructionOp);
 };
 
-// Lookup or create resources of type Container but treat them as having type
-// ContainerBase for the purpose of dynamic dispatching.
-// Container must have constructor
-// Container(OpKernelContext*, OpKernel*, FallbackTableBaseType*)
+// Create resources of type ContainerBase using the static method
+// Functor::AllocateContainer(OpKernelConstruction*, OpKernel*,
+// FallbackTableBaseType*, ContainerBase**)
+// If the resource has already been created it will be looked up.
 // Container must decrease the reference count of the FallbackTableBaseType*
 // constructor argument before its destructor completes.
-template <class ContainerBase, class ContainerChild,
+template <class ContainerBase, class Functor,
           class FallbackTableBaseType = ContainerBase>
 class TableWithFallbackConstructionOp : public OpKernel {
  public:
   explicit TableWithFallbackConstructionOp(OpKernelConstruction* ctx)
       : OpKernel(ctx), table_handle_set_(false) {
-    static_assert(std::is_base_of<ContainerBase, ContainerChild>::value,
-                  "ContainerChild is not derived from ContainerBase");
-    OP_REQUIRES_OK(ctx, ctx->allocate_persistent(tensorflow::DT_STRING,
-                                                 tensorflow::TensorShape({2}),
-                                                 &table_handle_, nullptr));
     OP_REQUIRES_OK(
         ctx, ctx->GetAttr("use_node_name_sharing", &use_node_name_sharing_));
   }
@@ -158,10 +152,12 @@ class TableWithFallbackConstructionOp : public OpKernel {
       // ResourceConstructionOp because Container constructor requires an
       // input which can only be constructed if the resource manager
       // internal lock is not already held.
-      ContainerBase* container = new ContainerChild(ctx, this, fallback_table);
-      if (!ctx->status().ok()) {
+      ContainerBase* container;
+      auto status =
+          Functor::AllocateContainer(ctx, this, fallback_table, &container);
+      if (TF_PREDICT_FALSE(!status.ok())) {
         container->Unref();
-        return ctx->status();
+        return status;
       }
       if (ctx->track_allocations()) {
         ctx->record_persistent_memory_allocation(container->MemoryUsed());
@@ -197,7 +193,6 @@ class TableWithFallbackConstructionOp : public OpKernel {
 
  private:
   mutex mu_;
-  PersistentTensor table_handle_ GUARDED_BY(mu_);
   bool table_handle_set_ GUARDED_BY(mu_);
   ContainerInfo cinfo_;
   bool use_node_name_sharing_;
@@ -226,9 +221,9 @@ class ContainerSizeOp : public OpKernel {
     auto* mutex = container->GetMutex();
     if (mutex != nullptr) {
       tf_shared_lock lock(*mutex);
-      out->scalar<int64>()() = container->size();
+      out->scalar<int64>()() = container->UnsafeSize();
     } else {
-      out->scalar<int64>()() = container->size();
+      out->scalar<int64>()() = container->UnsafeSize();
     }
   }
 };
diff --git a/tensorflow/core/ops/fingerprint64_map_ops.cc b/tensorflow/core/ops/fingerprint64_map_ops.cc
new file mode 100644
index 0000000000..91b24b4017
--- /dev/null
+++ b/tensorflow/core/ops/fingerprint64_map_ops.cc
@@ -0,0 +1,37 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/shape_inference.h"
+
+namespace tensorflow {
+
+using shape_inference::InferenceContext;
+
+REGISTER_OP("Fingerprint64Map")
+    .Output("table_handle: resource")
+    .Attr("heterogeneous_key_dtype: type")
+    .Attr("table_value_dtype: type = DT_INT64")
+    .Attr("num_oov_buckets: int >= 1")
+    .Attr("offset: int >= 0 = 0")
+    .Attr("use_node_name_sharing: bool = false")
+    .Attr("container: string = ''")
+    .Attr("shared_name: string = ''")
+    .SetShapeFn([](InferenceContext* c) {
+      c->set_output(0, c->Scalar());
+      return Status::OK();
+    });
+
+}  // namespace tensorflow
-- 
GitLab


From 59cf62cc475651e75fc8d2948daf2444cc0e8c15 Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Mon, 28 Jan 2019 10:38:35 -0800
Subject: [PATCH 1370/2345] Change estimator dep to >1.13rc0.

PiperOrigin-RevId: 231246464
---
 tensorflow/tools/pip_package/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index 0137f9da43..368cb171ba 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -58,7 +58,7 @@ REQUIRED_PACKAGES = [
     'six >= 1.10.0',
     'protobuf >= 3.6.1',
     'tensorboard >= 1.12.0, < 1.13.0',
-    'tensorflow_estimator >= 1.13.0, < 1.14.0',
+    'tensorflow_estimator >= 1.13.0rc0, < 1.14.0rc0',
     'termcolor >= 1.1.0',
 ]
 
-- 
GitLab


From 3623f9ae518e0e356a39f3a2aa7a8b8146f4ae4d Mon Sep 17 00:00:00 2001
From: Lukas Geiger <lukas.geiger94@gmail.com>
Date: Thu, 24 Jan 2019 13:59:46 +0100
Subject: [PATCH 1371/2345] Replace deprecated usage of np.asscalar with
 np.ndarray.item()

[`numpy.asscalar()` is deprecated since version 1.16](https://github.com/numpy/numpy/blob/master/numpy/lib/type_check.py#L519-L548).

This PR replaces its usage with [`numpy.ndarray.item()`](https://www.numpy.org/devdocs/reference/generated/numpy.ndarray.item.html)
---
 .../compiler/xla/python_api/xla_literal.py    |  2 +-
 .../python/framework/fast_tensor_util.pyx     |  2 +-
 tensorflow/python/framework/tensor_util.py    | 26 +++++++++----------
 .../python/kernel_tests/cwise_ops_test.py     |  2 +-
 tensorflow/python/ops/nn_ops.py               |  2 +-
 5 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/tensorflow/compiler/xla/python_api/xla_literal.py b/tensorflow/compiler/xla/python_api/xla_literal.py
index 757e41a78a..19bd685ab2 100644
--- a/tensorflow/compiler/xla/python_api/xla_literal.py
+++ b/tensorflow/compiler/xla/python_api/xla_literal.py
@@ -69,7 +69,7 @@ def _ConvertNumpyArrayToLiteral(ndarray):
 
   if ndarray.ndim == 0:
     getattr(literal, type_record.literal_field_name).append(
-        _np.asscalar(ndarray.astype(type_record.literal_field_type)))
+        ndarray.astype(type_record.literal_field_type).item())
   else:
     # Ndarrays with boolean dtypes need special type conversion with protobufs
     if ndarray.dtype in {_np.bool_, _np.dtype('bool')}:
diff --git a/tensorflow/python/framework/fast_tensor_util.pyx b/tensorflow/python/framework/fast_tensor_util.pyx
index 2e3e15f53a..dc14361d66 100644
--- a/tensorflow/python/framework/fast_tensor_util.pyx
+++ b/tensorflow/python/framework/fast_tensor_util.pyx
@@ -131,4 +131,4 @@ def AppendBoolArrayToTensorProto(tensor_proto, nparray):
   cdef long i, n
   n = nparray.size
   for i in range(n):
-    tensor_proto.bool_val.append(np.asscalar(nparray[i]))
+    tensor_proto.bool_val.append(nparray.item(i))
diff --git a/tensorflow/python/framework/tensor_util.py b/tensorflow/python/framework/tensor_util.py
index ca8b067935..47e2437e20 100644
--- a/tensorflow/python/framework/tensor_util.py
+++ b/tensorflow/python/framework/tensor_util.py
@@ -42,7 +42,7 @@ from tensorflow.python.util.tf_export import tf_export
 
 
 def ExtractBitsFromFloat16(x):
-  return np.asscalar(np.asarray(x, dtype=np.float16).view(np.uint16))
+  return np.asarray(x, dtype=np.float16).view(np.uint16).item()
 
 
 def SlowAppendFloat16ArrayToTensorProto(tensor_proto, proto_values):
@@ -58,8 +58,8 @@ def _MediumAppendFloat16ArrayToTensorProto(tensor_proto, proto_values):
 
 
 def ExtractBitsFromBFloat16(x):
-  return np.asscalar(
-      np.asarray(x, dtype=dtypes.bfloat16.as_numpy_dtype).view(np.uint16))
+  return np.asarray(
+      x, dtype=dtypes.bfloat16.as_numpy_dtype).view(np.uint16).item()
 
 
 def SlowAppendBFloat16ArrayToTensorProto(tensor_proto, proto_values):
@@ -122,39 +122,39 @@ if _FAST_TENSOR_UTIL_AVAILABLE:
 else:
 
   def SlowAppendFloat32ArrayToTensorProto(tensor_proto, proto_values):
-    tensor_proto.float_val.extend([np.asscalar(x) for x in proto_values])
+    tensor_proto.float_val.extend([x.item() for x in proto_values])
 
   def SlowAppendFloat64ArrayToTensorProto(tensor_proto, proto_values):
-    tensor_proto.double_val.extend([np.asscalar(x) for x in proto_values])
+    tensor_proto.double_val.extend([x.item() for x in proto_values])
 
   def SlowAppendIntArrayToTensorProto(tensor_proto, proto_values):
-    tensor_proto.int_val.extend([np.asscalar(x) for x in proto_values])
+    tensor_proto.int_val.extend([x.item() for x in proto_values])
 
   def SlowAppendInt64ArrayToTensorProto(tensor_proto, proto_values):
-    tensor_proto.int64_val.extend([np.asscalar(x) for x in proto_values])
+    tensor_proto.int64_val.extend([x.item() for x in proto_values])
 
   def SlowAppendQIntArrayToTensorProto(tensor_proto, proto_values):
-    tensor_proto.int_val.extend([np.asscalar(x[0]) for x in proto_values])
+    tensor_proto.int_val.extend([x.item(0) for x in proto_values])
 
   def SlowAppendUInt32ArrayToTensorProto(tensor_proto, proto_values):
-    tensor_proto.uint32_val.extend([np.asscalar(x) for x in proto_values])
+    tensor_proto.uint32_val.extend([x.item() for x in proto_values])
 
   def SlowAppendUInt64ArrayToTensorProto(tensor_proto, proto_values):
-    tensor_proto.uint64_val.extend([np.asscalar(x) for x in proto_values])
+    tensor_proto.uint64_val.extend([x.item() for x in proto_values])
 
   def SlowAppendComplex64ArrayToTensorProto(tensor_proto, proto_values):
     tensor_proto.scomplex_val.extend(
-        [np.asscalar(v) for x in proto_values for v in [x.real, x.imag]])
+        [v.item() for x in proto_values for v in [x.real, x.imag]])
 
   def SlowAppendComplex128ArrayToTensorProto(tensor_proto, proto_values):
     tensor_proto.dcomplex_val.extend(
-        [np.asscalar(v) for x in proto_values for v in [x.real, x.imag]])
+        [v.item() for x in proto_values for v in [x.real, x.imag]])
 
   def SlowAppendObjectArrayToTensorProto(tensor_proto, proto_values):
     tensor_proto.string_val.extend([compat.as_bytes(x) for x in proto_values])
 
   def SlowAppendBoolArrayToTensorProto(tensor_proto, proto_values):
-    tensor_proto.bool_val.extend([np.asscalar(x) for x in proto_values])
+    tensor_proto.bool_val.extend([x.item() for x in proto_values])
 
   _NP_TO_APPEND_FN = {
       dtypes.bfloat16.as_numpy_dtype: SlowAppendBFloat16ArrayToTensorProto,
diff --git a/tensorflow/python/kernel_tests/cwise_ops_test.py b/tensorflow/python/kernel_tests/cwise_ops_test.py
index 70f19f9d2f..29d335d68c 100644
--- a/tensorflow/python/kernel_tests/cwise_ops_test.py
+++ b/tensorflow/python/kernel_tests/cwise_ops_test.py
@@ -595,7 +595,7 @@ class MinMaxOpTest(test.TestCase):
 
   def testScalar(self):
     x = np.random.rand(1, 3, 2) * 100.
-    y = np.asscalar(np.random.rand(1) * 100.)  # should broadcast
+    y = np.random.rand(1).item() * 100.  # should broadcast
     # dropped np.float64, int64 because TF automatically converts to 32 bit
     for t in [np.float32, np.int32]:
       self._compare(x.astype(t), t(y), use_gpu=False)
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index e0ef9e5e34..25e9ffd34f 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -2334,7 +2334,7 @@ def leaky_relu(features, alpha=0.2, name=None):
       features = math_ops.to_float(features)
     if compat.forward_compatible(2018, 11, 1):
       if isinstance(alpha, np.ndarray):
-        alpha = np.asscalar(alpha)
+        alpha = alpha.item()
       return gen_nn_ops.leaky_relu(features, alpha=alpha, name=name)
     alpha = ops.convert_to_tensor(alpha, dtype=features.dtype, name="alpha")
     return math_ops.maximum(alpha * features, features, name=name)
-- 
GitLab


From 367f16435994a6070a852b9613198922a8a6ca5e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 28 Jan 2019 10:46:54 -0800
Subject: [PATCH 1372/2345] [TF:XLA] Increase tolerance testing XLA's
 AtrousConv2d gradient.

Auto tuning on the GPU caused this error tolerance to be exceeded very often with XLA.

PiperOrigin-RevId: 231248158
---
 tensorflow/python/kernel_tests/atrous_conv2d_test.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/python/kernel_tests/atrous_conv2d_test.py b/tensorflow/python/kernel_tests/atrous_conv2d_test.py
index 5d7b2dd30f..b84e764723 100644
--- a/tensorflow/python/kernel_tests/atrous_conv2d_test.py
+++ b/tensorflow/python/kernel_tests/atrous_conv2d_test.py
@@ -139,7 +139,6 @@ class AtrousConv2DTest(test.TestCase):
                   y1.eval(), self.evaluate(y2), rtol=1e-2, atol=1e-2)
 
   @test_util.run_deprecated_v1
-  @test_util.disable_xla("This test never passed for XLA")  # larger error range
   def testGradient(self):
     with self.session(use_gpu=True):
       # Input: [batch, height, width, input_depth]
@@ -161,7 +160,7 @@ class AtrousConv2DTest(test.TestCase):
                                                       [x_shape, f_shape],
                                                       output, y_shape)
         print("atrous_conv2d gradient err = %g " % err)
-        err_tolerance = 1e-3
+        err_tolerance = 4e-3 if test_util.is_xla_enabled() else 1e-3
         self.assertLess(err, err_tolerance)
 
 
-- 
GitLab


From 28f9fb6129dba2a1e54b819fc0eb753386666fbf Mon Sep 17 00:00:00 2001
From: Guozhong Zhuang <guozhong.zhuang@intel.com>
Date: Mon, 28 Jan 2019 11:04:52 -0800
Subject: [PATCH 1373/2345] More Change per code review suggestions

---
 tensorflow/core/util/mkl_util.h       |  4 ++--
 tensorflow/core/util/mkl_util_test.cc | 18 ++++++++----------
 2 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/tensorflow/core/util/mkl_util.h b/tensorflow/core/util/mkl_util.h
index c897c90124..037a53a642 100644
--- a/tensorflow/core/util/mkl_util.h
+++ b/tensorflow/core/util/mkl_util.h
@@ -2126,8 +2126,8 @@ class LRUCache {
   std::unordered_map<string, Entry> cache_;
 
   // The LRU list of entries.
-  // The front of the list identifies the most recently accessed entry,
-  // while the back of the list is the least recently accessed entry.
+  // The front of the list contians the key of the most recently accessed 
+  // entry, while the back of the list is the least recently accessed entry.
   std::list<string> lru_list_;
 };
 
diff --git a/tensorflow/core/util/mkl_util_test.cc b/tensorflow/core/util/mkl_util_test.cc
index 7ac2e510fc..bed6febe37 100644
--- a/tensorflow/core/util/mkl_util_test.cc
+++ b/tensorflow/core/util/mkl_util_test.cc
@@ -96,26 +96,24 @@ TEST(MklUtilTest, LRUCacheTest) {
   }
 
   // Test GetOp and capacity:
-  //   -- Less recently accessed objects should not be
-  //      in cache any more.
-  for (int k = 0; k < num_objects - capacity; k++) {
+  // Least recently accessed objects should not be in cache any more.
+  for (int k = 0; k < num_objects - capacity; ++k) {
     EXPECT_EQ(nullptr, lru_cache.GetOp(std::to_string(k)));
   }
 
   // Test GetOp and capacity:
-  //   -- most recently accessed objects should still
-  //      be in cache.
-  for (int k = num_objects - capacity; k < num_objects; k++) {
-    int* pint = lru_cache.GetOp(std::to_string(k));
-    EXPECT_NE(nullptr, pint);
-    EXPECT_EQ(*pint, k);
+  // Most recently accessed objects should still be in cache.
+  for (int k = num_objects - capacity; k < num_objects; ++k) {
+    int* int_ptr = lru_cache.GetOp(std::to_string(k));
+    EXPECT_NE(nullptr, int_ptr);
+    EXPECT_EQ(*int_ptr, k);
   }
 
   // Clean up the cache
   lru_cache.Clear();
 
   // After clean up, there should be no cached object.
-  for (int k = 0; k < num_objects; k++) {
+  for (int k = 0; k < num_objects; ++k) {
     EXPECT_EQ(nullptr, lru_cache.GetOp(std::to_string(k)));
   }
 }
-- 
GitLab


From 33f28b7d485e5c1f33061ee08678241b55f04631 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 28 Jan 2019 10:53:43 -0800
Subject: [PATCH 1374/2345] [TF:XLA] Delete unused variable in
 deadness_analysis.

PiperOrigin-RevId: 231249575
---
 tensorflow/compiler/jit/deadness_analysis.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/compiler/jit/deadness_analysis.cc b/tensorflow/compiler/jit/deadness_analysis.cc
index 08fabb3e30..0ef0d3db8c 100644
--- a/tensorflow/compiler/jit/deadness_analysis.cc
+++ b/tensorflow/compiler/jit/deadness_analysis.cc
@@ -1071,7 +1071,6 @@ Status DeadnessAnalysisImpl::PopulateWithReversePostOrder(
     absl::Span<Node* const> rpo) {
   std::vector<string> unreachable_nodes;
   // Compute the loop structure of the graph.
-  std::vector<ControlFlowInfo> control_flow_info;
   TF_RETURN_IF_ERROR(
       BuildControlFlowInfo(&graph_, &control_flow_info_, &unreachable_nodes));
 
-- 
GitLab


From 819e55e2992878f0854886c5f6940d9fe4a725fb Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 28 Jan 2019 10:57:08 -0800
Subject: [PATCH 1375/2345] Add AdaptiveSharedBatchScheduler queue option
 batch_timeout_micros which forces the batch to wait for some time before
 being scheduled. This can improve performance by limiting the scheduling of
 almost empty batches.

PiperOrigin-RevId: 231250232
---
 .../adaptive_shared_batch_scheduler.h         | 39 +++++++++++++------
 1 file changed, 28 insertions(+), 11 deletions(-)

diff --git a/tensorflow/core/kernels/batching_util/adaptive_shared_batch_scheduler.h b/tensorflow/core/kernels/batching_util/adaptive_shared_batch_scheduler.h
index 656b6ced6d..bef73b0574 100644
--- a/tensorflow/core/kernels/batching_util/adaptive_shared_batch_scheduler.h
+++ b/tensorflow/core/kernels/batching_util/adaptive_shared_batch_scheduler.h
@@ -125,6 +125,10 @@ class AdaptiveSharedBatchScheduler
     int max_batch_size = 1000;
     // Maximum number of enqueued (i.e. non-scheduled) batches.
     int max_enqueued_batches = 10;
+    // Amount of time non-full batches must wait before becoming schedulable.
+    // A non-zero value can improve performance by limiting the scheduling of
+    // nearly empty batches.
+    int64 batch_timeout_micros = 0;
   };
 
   using BatchProcessor = std::function<void(std::unique_ptr<Batch<TaskType>>)>;
@@ -267,8 +271,11 @@ class ASBSQueue : public BatchScheduler<TaskType> {
 template <typename TaskType>
 class ASBSBatch : public Batch<TaskType> {
  public:
-  ASBSBatch(ASBSQueue<TaskType>* queue, int64 creation_time_micros)
-      : queue_(queue), creation_time_micros_(creation_time_micros) {}
+  ASBSBatch(ASBSQueue<TaskType>* queue, int64 creation_time_micros,
+            int64 batch_timeout_micros)
+      : queue_(queue),
+        creation_time_micros_(creation_time_micros),
+        schedulable_time_micros_(creation_time_micros + batch_timeout_micros) {}
 
   ~ASBSBatch() override {}
 
@@ -276,9 +283,12 @@ class ASBSBatch : public Batch<TaskType> {
 
   int64 creation_time_micros() const { return creation_time_micros_; }
 
+  int64 schedulable_time_micros() const { return schedulable_time_micros_; }
+
  private:
   ASBSQueue<TaskType>* queue_;
   const int64 creation_time_micros_;
+  const int64 schedulable_time_micros_;
   TF_DISALLOW_COPY_AND_ASSIGN(ASBSBatch);
 };
 }  // namespace internal
@@ -377,7 +387,12 @@ void AdaptiveSharedBatchScheduler<TaskType>::AddBatch(
     bool also_schedule_closed_batch) {
   mutex_lock l(mu_);
   batches_.push_back(batch);
-  MaybeScheduleNextBatch();
+  // Maybe schedule this batch once it becomes schedulable.
+  GetEnv()->SchedClosureAfter(
+      batch->schedulable_time_micros() - batch->creation_time_micros(), [this] {
+        mutex_lock l(mu_);
+        MaybeScheduleNextBatch();
+      });
   if (also_schedule_closed_batch) {
     MaybeScheduleClosedBatch();
   }
@@ -400,21 +415,22 @@ void AdaptiveSharedBatchScheduler<TaskType>::MaybeScheduleNextBatch() {
           in_flight_batches_limit_ - in_flight_batches_) {
     return;
   }
-  auto best_it = batches_.begin();
-  double best_score =
-      (*best_it)->creation_time_micros() -
-      options_.full_batch_scheduling_boost_micros * (*best_it)->size() /
-          static_cast<double>((*best_it)->queue()->max_task_size());
-  for (auto it = batches_.begin() + 1; it != batches_.end(); it++) {
+  auto best_it = batches_.end();
+  double best_score;
+  int64 now_micros = GetEnv()->NowMicros();
+  for (auto it = batches_.begin(); it != batches_.end(); it++) {
+    if ((*it)->schedulable_time_micros() > now_micros) continue;
     const double score =
         (*it)->creation_time_micros() -
         options_.full_batch_scheduling_boost_micros * (*it)->size() /
             static_cast<double>((*it)->queue()->max_task_size());
-    if (score < best_score) {
+    if (best_it == batches_.end() || score < best_score) {
       best_score = score;
       best_it = it;
     }
   }
+  // No schedulable batches.
+  if (best_it == batches_.end()) return;
   const internal::ASBSBatch<TaskType>* batch = *best_it;
   batches_.erase(best_it);
   // Queue may destroy itself after ReleaseBatch is called.
@@ -552,7 +568,8 @@ Status ASBSQueue<TaskType>::Schedule(std::unique_ptr<TaskType>* task) {
     if (!current_batch_) {
       num_enqueued_batches_++;
       current_batch_ = new_batch =
-          new ASBSBatch<TaskType>(this, scheduler_->GetEnv()->NowMicros());
+          new ASBSBatch<TaskType>(this, scheduler_->GetEnv()->NowMicros(),
+                                  options_.batch_timeout_micros);
     }
     current_batch_->AddTask(std::move(*task));
     num_enqueued_tasks_++;
-- 
GitLab


From 56c3ac7d23a6016d646fb69ca44da7b8280acb09 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 28 Jan 2019 10:58:11 -0800
Subject: [PATCH 1376/2345] [TF:XLA] Change error type in XLA for uninitialized
 ResourceVariable.

This is to match TensorFlow Classic's implementation and matches the actual error condition more closely.

PiperOrigin-RevId: 231250467
---
 tensorflow/compiler/tests/variable_ops_test.py               | 2 +-
 tensorflow/compiler/tf2xla/xla_op_kernel.cc                  | 4 ++--
 tensorflow/python/kernel_tests/resource_variable_ops_test.py | 1 -
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/tests/variable_ops_test.py b/tensorflow/compiler/tests/variable_ops_test.py
index fcd7ac5ba1..18c5870e0d 100644
--- a/tensorflow/compiler/tests/variable_ops_test.py
+++ b/tensorflow/compiler/tests/variable_ops_test.py
@@ -485,7 +485,7 @@ class SliceAssignTest(xla_test.XLATestCase):
       checker2[None] = [6]  # new axis
 
   def testUninitialized(self):
-    with self.assertRaisesRegexp(errors.InvalidArgumentError,
+    with self.assertRaisesRegexp(errors.FailedPreconditionError,
                                  "uninitialized variable"):
       with self.test_session() as sess, self.test_scope():
         v = resource_variable_ops.ResourceVariable([1, 2])
diff --git a/tensorflow/compiler/tf2xla/xla_op_kernel.cc b/tensorflow/compiler/tf2xla/xla_op_kernel.cc
index 1147d0c726..78bc2c9442 100644
--- a/tensorflow/compiler/tf2xla/xla_op_kernel.cc
+++ b/tensorflow/compiler/tf2xla/xla_op_kernel.cc
@@ -354,8 +354,8 @@ Status ReadVariableInputTensor(const Tensor& tensor, DataType type,
   TF_RET_CHECK(variable != nullptr);
   TF_RET_CHECK(variable->kind() == XlaResource::kVariable);
   if (!variable->initialized()) {
-    return errors::InvalidArgument("Read of uninitialized variable ",
-                                   variable->name());
+    return errors::FailedPrecondition("Read of uninitialized variable ",
+                                      variable->name());
   }
   if (variable->type() != type) {
     return errors::InvalidArgument(
diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index f90c090ac7..163d5a316c 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -745,7 +745,6 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
 
   @test_util.run_in_graph_and_eager_modes
   @test_util.run_v1_only("b/120545219")
-  @test_util.disable_xla("This test never passed for XLA")
   def testDestroyResource(self):
     v = resource_variable_ops.ResourceVariable(3.0, name="var0")
     self.evaluate(variables.global_variables_initializer())
-- 
GitLab


From aba83497f568b15f9b86d3571c1f9acfcb0c1a0e Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Mon, 28 Jan 2019 11:00:39 -0800
Subject: [PATCH 1377/2345] PR #25011: [GPU][ROCm][CUDA] StreamExecutor logic
 for ROCm / CUDA platform (PR 20709 / 22669 / 24156 continued)

Please approve this CL. It will be submitted automatically, and its GitHub pull request will be marked as merged.

Imported from GitHub PR #25011

New PR to continue the efforts started by @deven-amd in #20709 / #22669 / #24156.

This PR aims to refactor StreamExecutor GPU interfaces so it can be shared among CUDA and ROCm. The PR would be the first part of a series of PRs.

Based on @timshen91 's inputs, I've refactored logic in #214156 so :

- only contains changes in stream_executor/....
- does not remove any stream_executor/cuda/*.h, so that things outside of stream_executor don't break. All the types and functions in the namespace cuda now alias to namespace gpu counterparts. For example, namespace cuda { using CUDADriver = gpu::GpuDriver; }.
- all stream_executor/gpu/BUILD targets should be only visible to //third_party/tensorflow/stream_executor:__subpackages__.
- target stream_executor/gpu:X should be only used by stream_executor/cuda:cuda_X or stream_executor/rocm:rocm_X, not cuda_Y. For example, cuda:cuda_platform should depend on cuda:cuda_driver, not gpu:gpu_driver.

Copybara import of the project:

  - 267affbb73df9164baf4e62142fe7201e6a305ee [ROCm][CUDA] StreamExecutor logic for ROCm / CUDA platform by Wen-Heng (Jack) Chung <whchung@gmail.com>
  - 04fac5bf358059bdb2cd4a3e092e52dc982ea7b0 Merge 267affbb73df9164baf4e62142fe7201e6a305ee into 5f8ea... by Wen-Heng (Jack) Chung <whchung@gmail.com>

COPYBARA_INTEGRATE_REVIEW=https://github.com/tensorflow/tensorflow/pull/25011 from ROCmSoftwarePlatform:google-upstream-pr-stream-executor-alt 267affbb73df9164baf4e62142fe7201e6a305ee
PiperOrigin-RevId: 231250990
---
 tensorflow/BUILD                              |    7 +
 tensorflow/core/BUILD                         |    8 +
 .../core/platform/default/build_config.bzl    |    6 +
 .../core/platform/default/build_config/BUILD  |   27 +
 tensorflow/core/platform/stream_executor.h    |    1 +
 .../core/platform/stream_executor_no_cuda.h   |    1 +
 tensorflow/stream_executor/BUILD              |    5 +
 tensorflow/stream_executor/cuda/BUILD         |   36 +-
 .../stream_executor/cuda/cuda_activation.h    |   28 +-
 tensorflow/stream_executor/cuda/cuda_blas.cc  |  690 ++++-----
 tensorflow/stream_executor/cuda/cuda_blas.h   |   14 +-
 .../stream_executor/cuda/cuda_diagnostics.cc  |   42 +-
 .../stream_executor/cuda/cuda_diagnostics.h   |   64 +-
 tensorflow/stream_executor/cuda/cuda_dnn.cc   |   86 +-
 tensorflow/stream_executor/cuda/cuda_dnn.h    |   10 +-
 .../stream_executor/cuda/cuda_driver.cc       |  410 ++---
 tensorflow/stream_executor/cuda/cuda_driver.h |  498 +-----
 .../cuda/cuda_driver_wrapper.h                |    1 +
 tensorflow/stream_executor/cuda/cuda_event.cc |   31 +-
 tensorflow/stream_executor/cuda/cuda_event.h  |   37 +-
 tensorflow/stream_executor/cuda/cuda_fft.cc   |   48 +-
 tensorflow/stream_executor/cuda/cuda_fft.h    |   28 +-
 .../stream_executor/cuda/cuda_gpu_executor.cc |  499 +++---
 .../stream_executor/cuda/cuda_gpu_executor.h  |  281 +---
 .../stream_executor/cuda/cuda_helpers.h       |   81 +-
 .../stream_executor/cuda/cuda_kernel.cc       |   38 +
 tensorflow/stream_executor/cuda/cuda_kernel.h |   96 +-
 .../stream_executor/cuda/cuda_platform.cc     |   14 +-
 .../stream_executor/cuda/cuda_platform.h      |    8 +-
 tensorflow/stream_executor/cuda/cuda_rng.cc   |  116 +-
 tensorflow/stream_executor/cuda/cuda_rng.h    |   76 +-
 tensorflow/stream_executor/cuda/cuda_stream.h |   69 +-
 tensorflow/stream_executor/cuda/cuda_timer.h  |   64 +-
 .../stream_executor/cuda/cudnn_version.cc     |    4 +-
 .../stream_executor/cuda/cudnn_version.h      |    4 +-
 .../cuda/cudnn_version_test.cc                |    4 +-
 .../stream_executor/device_description.cc     |   10 +
 .../stream_executor/device_description.h      |   12 +
 tensorflow/stream_executor/gpu/BUILD          |  209 +++
 .../gpu_activation.cc}                        |   28 +-
 .../stream_executor/gpu/gpu_activation.h      |   61 +
 .../stream_executor/gpu/gpu_diagnostics.h     |   99 ++
 tensorflow/stream_executor/gpu/gpu_driver.h   |  525 +++++++
 tensorflow/stream_executor/gpu/gpu_event.cc   |   47 +
 tensorflow/stream_executor/gpu/gpu_event.h    |   62 +
 tensorflow/stream_executor/gpu/gpu_executor.h |  347 +++++
 tensorflow/stream_executor/gpu/gpu_helpers.h  |  107 ++
 tensorflow/stream_executor/gpu/gpu_kernel.h   |  105 ++
 tensorflow/stream_executor/gpu/gpu_rng.h      |  125 ++
 .../cuda_stream.cc => gpu/gpu_stream.cc}      |   36 +-
 tensorflow/stream_executor/gpu/gpu_stream.h   |   96 ++
 .../{cuda/cuda_timer.cc => gpu/gpu_timer.cc}  |   55 +-
 tensorflow/stream_executor/gpu/gpu_timer.h    |   90 ++
 tensorflow/stream_executor/gpu/gpu_types.h    |   84 +
 tensorflow/stream_executor/platform.cc        |    4 +
 tensorflow/stream_executor/platform.h         |    1 +
 tensorflow/stream_executor/rocm/BUILD         |  267 ++++
 .../stream_executor/rocm/rocm_diagnostics.cc  |  234 +++
 .../stream_executor/rocm/rocm_driver.cc       | 1365 +++++++++++++++++
 tensorflow/stream_executor/rocm/rocm_event.cc |   46 +
 .../stream_executor/rocm/rocm_gpu_executor.cc |  976 ++++++++++++
 .../stream_executor/rocm/rocm_kernel.cc       |   38 +
 .../stream_executor/rocm/rocm_platform.cc     |  180 +++
 .../stream_executor/rocm/rocm_platform.h      |  110 ++
 .../stream_executor/rocm/rocm_platform_id.cc  |   24 +
 .../stream_executor/rocm/rocm_platform_id.h   |   34 +
 tensorflow/stream_executor/rocm/rocm_rng.cc   |  284 ++++
 .../stream_executor_internal.cc               |    7 +
 .../stream_executor_internal.h                |    6 +-
 .../stream_executor/stream_executor_pimpl.cc  |    5 +
 third_party/gpus/rocm/BUILD.tpl               |    1 +
 71 files changed, 6789 insertions(+), 2323 deletions(-)
 create mode 100644 tensorflow/stream_executor/cuda/cuda_kernel.cc
 create mode 100644 tensorflow/stream_executor/gpu/BUILD
 rename tensorflow/stream_executor/{cuda/cuda_activation.cc => gpu/gpu_activation.cc} (62%)
 create mode 100644 tensorflow/stream_executor/gpu/gpu_activation.h
 create mode 100644 tensorflow/stream_executor/gpu/gpu_diagnostics.h
 create mode 100644 tensorflow/stream_executor/gpu/gpu_driver.h
 create mode 100644 tensorflow/stream_executor/gpu/gpu_event.cc
 create mode 100644 tensorflow/stream_executor/gpu/gpu_event.h
 create mode 100644 tensorflow/stream_executor/gpu/gpu_executor.h
 create mode 100644 tensorflow/stream_executor/gpu/gpu_helpers.h
 create mode 100644 tensorflow/stream_executor/gpu/gpu_kernel.h
 create mode 100644 tensorflow/stream_executor/gpu/gpu_rng.h
 rename tensorflow/stream_executor/{cuda/cuda_stream.cc => gpu/gpu_stream.cc} (51%)
 create mode 100644 tensorflow/stream_executor/gpu/gpu_stream.h
 rename tensorflow/stream_executor/{cuda/cuda_timer.cc => gpu/gpu_timer.cc} (51%)
 create mode 100644 tensorflow/stream_executor/gpu/gpu_timer.h
 create mode 100644 tensorflow/stream_executor/gpu/gpu_types.h
 create mode 100644 tensorflow/stream_executor/rocm/BUILD
 create mode 100644 tensorflow/stream_executor/rocm/rocm_diagnostics.cc
 create mode 100644 tensorflow/stream_executor/rocm/rocm_driver.cc
 create mode 100644 tensorflow/stream_executor/rocm/rocm_event.cc
 create mode 100644 tensorflow/stream_executor/rocm/rocm_gpu_executor.cc
 create mode 100644 tensorflow/stream_executor/rocm/rocm_kernel.cc
 create mode 100644 tensorflow/stream_executor/rocm/rocm_platform.cc
 create mode 100644 tensorflow/stream_executor/rocm/rocm_platform.h
 create mode 100644 tensorflow/stream_executor/rocm/rocm_platform_id.cc
 create mode 100644 tensorflow/stream_executor/rocm/rocm_platform_id.h
 create mode 100644 tensorflow/stream_executor/rocm/rocm_rng.cc

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 0be7920d11..413806fac1 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -343,6 +343,13 @@ config_setting(
     },
 )
 
+config_setting(
+    name = "using_rocm_hipcc",
+    define_values = {
+        "using_rocm_hipcc": "true",
+    },
+)
+
 config_setting(
     name = "with_mpi_support",
     values = {"define": "with_mpi_support=true"},
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index ebc8c309ec..a932974270 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1964,6 +1964,14 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "rocm",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tensorflow/core/platform/default/build_config:rocm",
+    ],
+)
+
 # -----------------------------------------------------------------------------
 # Clif-related proto libraries.
 
diff --git a/tensorflow/core/platform/default/build_config.bzl b/tensorflow/core/platform/default/build_config.bzl
index e26828c75e..e949008403 100644
--- a/tensorflow/core/platform/default/build_config.bzl
+++ b/tensorflow/core/platform/default/build_config.bzl
@@ -6,6 +6,7 @@ load("//tensorflow:tensorflow.bzl", "if_windows")
 load("//tensorflow:tensorflow.bzl", "if_not_windows")
 load("//tensorflow/core:platform/default/build_config_root.bzl", "if_static")
 load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda")
+load("@local_config_rocm//rocm:build_defs.bzl", "if_rocm")
 load(
     "//third_party/mkl:build_defs.bzl",
     "if_mkl_ml",
@@ -735,6 +736,11 @@ def tf_additional_binary_deps():
             "//tensorflow/stream_executor:cuda_platform",
             "//tensorflow/core/platform/default/build_config:cuda",
         ],
+    ) + if_rocm(
+        [
+            "//tensorflow/stream_executor:rocm_platform",
+            "//tensorflow/core/platform/default/build_config:rocm",
+        ],
     ) + [
         # TODO(allenl): Split these out into their own shared objects (they are
         # here because they are shared between contrib/ op shared objects and
diff --git a/tensorflow/core/platform/default/build_config/BUILD b/tensorflow/core/platform/default/build_config/BUILD
index 664218d11b..6faf5c512f 100644
--- a/tensorflow/core/platform/default/build_config/BUILD
+++ b/tensorflow/core/platform/default/build_config/BUILD
@@ -8,6 +8,7 @@ licenses(["notice"])  # Apache 2.0
 exports_files(["LICENSE"])
 
 load("//tensorflow:tensorflow.bzl", "if_cuda")
+load("//tensorflow:tensorflow.bzl", "if_rocm")
 load("//tensorflow:tensorflow.bzl", "tf_copts")
 load("//tensorflow:tensorflow.bzl", "tf_cuda_library")
 load("//tensorflow/core:platform/default/build_config_root.bzl", "if_static")
@@ -42,6 +43,7 @@ tf_cuda_library(
         "//tensorflow/stream_executor/cuda:cuda_platform_id",
         "//tensorflow/stream_executor/host:host_platform_id",
         "//tensorflow/stream_executor/platform:dso_loader",
+        "//tensorflow/stream_executor/rocm:rocm_platform_id",
     ] + select({
         "@local_config_cuda//cuda:darwin": ["IOKit"],
         "//conditions:default": [],
@@ -50,6 +52,7 @@ tf_cuda_library(
         "//tensorflow:using_cuda_nvcc": ["//tensorflow/stream_executor/cuda:all_runtime"],
         "//tensorflow:using_cuda_clang_with_dynamic_build": [],
         "//tensorflow:using_cuda_nvcc_with_dynamic_build": [],
+        "//tensorflow:using_rocm_hipcc": ["//tensorflow/stream_executor/rocm:all_runtime"],
         "//conditions:default": [],
     }),
 )
@@ -67,6 +70,18 @@ cc_library(
     }),
 )
 
+cc_library(
+    name = "stream_executor_rocm",
+    deps = [
+        ":stream_executor_no_cuda",
+        ":rocm",
+    ] + if_static(
+        ["//tensorflow/stream_executor/rocm:all_runtime"],
+    ) + select({
+        "//conditions:default": [],
+    }),
+)
+
 cc_library(
     name = "stream_executor_no_cuda",
     deps = [
@@ -79,6 +94,7 @@ cc_library(
         "//tensorflow/stream_executor/host:host_platform",
         "//tensorflow/stream_executor/host:host_platform_id",
         "//tensorflow/stream_executor/platform:dso_loader",
+        "//tensorflow/stream_executor/rocm:rocm_platform_id",
     ],
 )
 
@@ -267,6 +283,17 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "rocm",
+    data = [],
+    linkopts = select({
+        "//conditions:default": [
+            "-Wl,-rpath,../local_config_rocm/rocm/rocm/lib",
+        ],
+    }),
+    deps = [],
+)
+
 cc_library(
     name = "sycl",
     data = if_ccpp([
diff --git a/tensorflow/core/platform/stream_executor.h b/tensorflow/core/platform/stream_executor.h
index 42822859f6..437e8a1c95 100644
--- a/tensorflow/core/platform/stream_executor.h
+++ b/tensorflow/core/platform/stream_executor.h
@@ -27,6 +27,7 @@ limitations under the License.
 #include "tensorflow/stream_executor/multi_platform_manager.h"
 #include "tensorflow/stream_executor/platform.h"
 #include "tensorflow/stream_executor/platform/dso_loader.h"
+#include "tensorflow/stream_executor/rocm/rocm_platform_id.h"
 #include "tensorflow/stream_executor/scratch_allocator.h"
 #include "tensorflow/stream_executor/stream.h"
 #include "tensorflow/stream_executor/stream_executor.h"
diff --git a/tensorflow/core/platform/stream_executor_no_cuda.h b/tensorflow/core/platform/stream_executor_no_cuda.h
index 123035cc8a..129ee6c7a7 100644
--- a/tensorflow/core/platform/stream_executor_no_cuda.h
+++ b/tensorflow/core/platform/stream_executor_no_cuda.h
@@ -27,6 +27,7 @@ limitations under the License.
 #include "tensorflow/stream_executor/multi_platform_manager.h"
 #include "tensorflow/stream_executor/platform.h"
 #include "tensorflow/stream_executor/platform/dso_loader.h"
+#include "tensorflow/stream_executor/rocm/rocm_platform_id.h"
 #include "tensorflow/stream_executor/scratch_allocator.h"
 #include "tensorflow/stream_executor/stream.h"
 #include "tensorflow/stream_executor/stream_executor.h"
diff --git a/tensorflow/stream_executor/BUILD b/tensorflow/stream_executor/BUILD
index fca28e016a..980c92aaca 100644
--- a/tensorflow/stream_executor/BUILD
+++ b/tensorflow/stream_executor/BUILD
@@ -654,3 +654,8 @@ alias(
     name = "cuda_platform",
     actual = "//tensorflow/stream_executor/cuda:all_runtime",
 )
+
+alias(
+    name = "rocm_platform",
+    actual = "//tensorflow/stream_executor/rocm:all_runtime",
+)
diff --git a/tensorflow/stream_executor/cuda/BUILD b/tensorflow/stream_executor/cuda/BUILD
index 43b865a90f..9dc3be45e7 100644
--- a/tensorflow/stream_executor/cuda/BUILD
+++ b/tensorflow/stream_executor/cuda/BUILD
@@ -66,6 +66,7 @@ cc_library(
     deps = if_cuda_is_configured([
         "@com_google_absl//absl/container:inlined_vector",
         "@com_google_absl//absl/strings",
+        "//tensorflow/stream_executor/gpu:gpu_diagnostics_header",
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
     ]),
@@ -85,6 +86,7 @@ cc_library(
         "@com_google_absl//absl/strings",
         "@local_config_cuda//cuda:cuda_headers",
         "//tensorflow/stream_executor:device_options",
+        "//tensorflow/stream_executor/gpu:gpu_driver_header",
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
         "//tensorflow/stream_executor/platform:dso_loader",
@@ -97,18 +99,22 @@ cc_library(
     name = "cuda_activation_header",
     hdrs = ["cuda_activation.h"],
     visibility = ["//visibility:public"],
-    deps = ["//tensorflow/stream_executor/platform"],
+    deps = [
+        "//tensorflow/stream_executor/gpu:gpu_activation_header",
+        "//tensorflow/stream_executor/platform",
+    ],
 )
 
 cc_library(
     name = "cuda_activation",
-    srcs = if_cuda_is_configured(["cuda_activation.cc"]),
+    srcs = [],
     hdrs = if_cuda_is_configured(["cuda_activation.h"]),
     deps = if_cuda_is_configured([
         ":cuda_driver",
         "@local_config_cuda//cuda:cuda_headers",
         "//tensorflow/stream_executor",
         "//tensorflow/stream_executor:stream_executor_internal",
+        "//tensorflow/stream_executor/gpu:gpu_activation",
         "//tensorflow/stream_executor/platform",
     ]),
 )
@@ -120,6 +126,7 @@ cc_library(
     deps = if_cuda_is_configured([
         ":cuda_kernel",
         "//tensorflow/stream_executor:event",
+        "//tensorflow/stream_executor/gpu:gpu_executor_header",
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
     ]),
@@ -133,10 +140,10 @@ cc_library(
     deps = if_cuda_is_configured([
         ":cuda_activation",
         ":cuda_gpu_executor",
-        ":cuda_helpers",
         ":cuda_platform_id",
         ":cuda_stream",
         ":cuda_timer",
+        ":cuda_helpers",
         "@com_google_absl//absl/strings",
         "//third_party/eigen3",
         "@local_config_cuda//cuda:cuda_headers",
@@ -147,6 +154,7 @@ cc_library(
         "//tensorflow/stream_executor:plugin_registry",
         "//tensorflow/stream_executor:scratch_allocator",
         "//tensorflow/stream_executor:timer",
+        "//tensorflow/stream_executor/gpu:gpu_helpers_header",
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
         "//tensorflow/stream_executor/platform:dso_loader",
@@ -162,14 +170,15 @@ cc_library(
     deps = if_cuda_is_configured([
         ":cuda_activation_header",
         ":cuda_gpu_executor_header",
-        ":cuda_helpers",
         ":cuda_platform_id",
         ":cuda_stream",
+        ":cuda_helpers",
         "@local_config_cuda//cuda:cuda_headers",
         "//tensorflow/stream_executor:event",
         "//tensorflow/stream_executor:fft",
         "//tensorflow/stream_executor:plugin_registry",
         "//tensorflow/stream_executor:scratch_allocator",
+        "//tensorflow/stream_executor/gpu:gpu_helpers_header",
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
         "//tensorflow/stream_executor/platform:dso_loader",
@@ -223,13 +232,15 @@ cc_library(
     deps = if_cuda_is_configured([
         ":cuda_activation",
         ":cuda_gpu_executor",
-        ":cuda_helpers",
         ":cuda_platform_id",
         ":cuda_stream",
+        ":cuda_helpers",
         "@local_config_cuda//cuda:cuda_headers",
         "//tensorflow/stream_executor:event",
         "//tensorflow/stream_executor:plugin_registry",
         "//tensorflow/stream_executor:rng",
+        "//tensorflow/stream_executor/gpu:gpu_helpers_header",
+        "//tensorflow/stream_executor/gpu:gpu_rng_header",
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
         "//tensorflow/stream_executor/platform:dso_loader",
@@ -239,12 +250,14 @@ cc_library(
 
 cc_library(
     name = "cuda_kernel",
+    srcs = if_cuda_is_configured(["cuda_kernel.cc"]),
     hdrs = if_cuda_is_configured(["cuda_kernel.h"]),
     deps = if_cuda_is_configured([
         ":cuda_driver",
         "@local_config_cuda//cuda:cuda_headers",
         "//tensorflow/stream_executor:event",
         "//tensorflow/stream_executor:stream_executor_pimpl_header",
+        "//tensorflow/stream_executor/gpu:gpu_kernel_header",
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
     ]),
@@ -254,6 +267,9 @@ cc_library(
 cc_library(
     name = "cuda_helpers",
     textual_hdrs = if_cuda_is_configured(["cuda_helpers.h"]),
+    deps = if_cuda_is_configured([
+        "//tensorflow/stream_executor/gpu:gpu_helpers_header",
+    ]),
 )
 
 cc_library(
@@ -265,19 +281,22 @@ cc_library(
         ":cuda_gpu_executor_header",
         ":cuda_stream",
         "//tensorflow/stream_executor:stream_executor_headers",
+        "//tensorflow/stream_executor/gpu:gpu_event",
+        "//tensorflow/stream_executor/gpu:gpu_stream_header",
         "//tensorflow/stream_executor/lib",
     ]),
 )
 
 cc_library(
     name = "cuda_stream",
-    srcs = if_cuda_is_configured(["cuda_stream.cc"]),
+    srcs = [],
     hdrs = if_cuda_is_configured(["cuda_stream.h"]),
     deps = if_cuda_is_configured([
         ":cuda_driver",
         ":cuda_gpu_executor_header",
         "//tensorflow/stream_executor:stream_executor_headers",
         "//tensorflow/stream_executor:stream_header",
+        "//tensorflow/stream_executor/gpu:gpu_stream",
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
     ]),
@@ -285,18 +304,18 @@ cc_library(
 
 cc_library(
     name = "cuda_timer",
-    srcs = if_cuda_is_configured(["cuda_timer.cc"]),
+    srcs = [],
     hdrs = if_cuda_is_configured(["cuda_timer.h"]),
     deps = if_cuda_is_configured([
         ":cuda_driver",
         ":cuda_gpu_executor_header",
         ":cuda_stream",
         "//tensorflow/stream_executor:stream_executor_headers",
+        "//tensorflow/stream_executor/gpu:gpu_timer",
         "//tensorflow/stream_executor/lib",
     ]),
 )
 
-# It implements :cuda_gpu_executor_header
 cc_library(
     name = "cuda_gpu_executor",
     srcs = if_cuda_is_configured(["cuda_gpu_executor.cc"]),
@@ -316,6 +335,7 @@ cc_library(
         "//tensorflow/stream_executor:stream_executor_internal",
         "//tensorflow/stream_executor:stream_executor_pimpl_header",
         "//tensorflow/stream_executor:timer",
+        "//tensorflow/stream_executor/gpu:gpu_executor_header",
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
         "//tensorflow/stream_executor/platform:dso_loader",
diff --git a/tensorflow/stream_executor/cuda/cuda_activation.h b/tensorflow/stream_executor/cuda/cuda_activation.h
index ef9807820f..2b80ae094d 100644
--- a/tensorflow/stream_executor/cuda/cuda_activation.h
+++ b/tensorflow/stream_executor/cuda/cuda_activation.h
@@ -17,13 +17,13 @@ limitations under the License.
 // It reaches into the CUDA implementation to activate an underlying CUDA
 // context.
 //
-// Having this file separate from cuda_gpu_executor.h means that dependent
+// Having this file separate from cuda/cuda_gpu_executor.h means that dependent
 // code does not also have to depend on cuda.h.
 
 #ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_ACTIVATION_H_
 #define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_ACTIVATION_H_
 
-#include "tensorflow/stream_executor/platform/port.h"
+#include "tensorflow/stream_executor/gpu/gpu_activation.h"
 
 namespace stream_executor {
 
@@ -31,29 +31,7 @@ class StreamExecutor;
 
 namespace cuda {
 
-class CUDAExecutor;
-class ScopedActivateContext;
-
-// Activates a CUDA context within an enclosing scope.
-class ScopedActivateExecutorContext {
- public:
-  // Form that takes a CUDA executor implementation.
-  explicit ScopedActivateExecutorContext(CUDAExecutor* cuda_exec);
-
-  // Form that takes a pImpl executor and extracts a CUDA implementation --
-  // fatal failure if it is not CUDA inside.
-  explicit ScopedActivateExecutorContext(StreamExecutor* stream_exec);
-
-  ScopedActivateExecutorContext(ScopedActivateExecutorContext&& other);
-
-  ~ScopedActivateExecutorContext();
-
- private:
-  // The cuda.h-using datatype that we wrap.
-  ScopedActivateContext* driver_scoped_activate_context_;
-
-  SE_DISALLOW_COPY_AND_ASSIGN(ScopedActivateExecutorContext);
-};
+using ScopedActivateExecutorContext = gpu::ScopedActivateExecutorContext;
 
 }  // namespace cuda
 }  // namespace stream_executor
diff --git a/tensorflow/stream_executor/cuda/cuda_blas.cc b/tensorflow/stream_executor/cuda/cuda_blas.cc
index 7325476ef1..5bbb98664e 100644
--- a/tensorflow/stream_executor/cuda/cuda_blas.cc
+++ b/tensorflow/stream_executor/cuda/cuda_blas.cc
@@ -71,7 +71,7 @@ limitations under the License.
 #include "tensorflow/stream_executor/stream_executor.h"
 
 namespace stream_executor {
-namespace cuda {
+namespace gpu {
 
 PLUGIN_REGISTRY_DEFINE_PLUGIN_ID(kCuBlasPlugin);
 
@@ -257,8 +257,8 @@ namespace wrap {
   struct WrapperShim__##__name {                                    \
     static const char *kName;                                       \
     template <typename... Args>                                     \
-    cublasStatus_t operator()(CUDAExecutor *parent, Args... args) { \
-      cuda::ScopedActivateExecutorContext sac{parent};              \
+    cublasStatus_t operator()(GpuExecutor *parent, Args... args) { \
+      gpu::ScopedActivateExecutorContext sac{parent};              \
       return ::__name(args...);                                     \
     }                                                               \
   } __name;                                                         \
@@ -290,8 +290,8 @@ namespace wrap {
       return f;                                                           \
     }                                                                     \
     template <typename... Args>                                           \
-    cublasStatus_t operator()(CUDAExecutor* parent, Args... args) {       \
-      cuda::ScopedActivateExecutorContext sac{parent};                    \
+    cublasStatus_t operator()(GpuExecutor* parent, Args... args) {        \
+      gpu::ScopedActivateExecutorContext sac{parent};                     \
       return DynLoad()(args...);                                          \
     }                                                                     \
   } __name;                                                               \
@@ -395,7 +395,7 @@ class ScopedCublasPointerMode {
   //
   // Parameters:
   //  handle: The cublas library handle to act upon in setting the pointer mode.
-  explicit ScopedCublasPointerMode(CUDAExecutor *parent, cublasHandle_t handle)
+  explicit ScopedCublasPointerMode(GpuExecutor *parent, cublasHandle_t handle)
       : parent_(parent), handle_(handle), ok_(false) {}
 
   // Attempts the switch to the requested scoped pointer mode, new_mode.
@@ -433,7 +433,7 @@ class ScopedCublasPointerMode {
   }
 
  private:
-  CUDAExecutor *parent_;   // Executor establishing this pointer mode for.
+  GpuExecutor *parent_;   // Executor establishing this pointer mode for.
   cublasHandle_t handle_;  // Handle to the cuBLAS instance of interest.
   cublasPointerMode_t old_mode_;  // Prior cuBLAS pointer mode, to be restored.
   bool ok_;                       // Whether the change was successful.
@@ -456,7 +456,7 @@ class ScopedCublasMathMode {
   //
   // Parameters:
   //  handle: The cublas library handle to act upon in setting the math mode.
-  explicit ScopedCublasMathMode(CUDAExecutor *parent, cublasHandle_t handle)
+  explicit ScopedCublasMathMode(GpuExecutor *parent, cublasHandle_t handle)
       : parent_(parent), handle_(handle), ok_(false) {}
 
   // Attempts the switch to the requested scoped math mode, new_mode.
@@ -493,7 +493,7 @@ class ScopedCublasMathMode {
   }
 
  private:
-  CUDAExecutor *parent_;   // Executor establishing this math mode for.
+  GpuExecutor *parent_;   // Executor establishing this math mode for.
   cublasHandle_t handle_;  // Handle to the cuBLAS instance of interest.
   cublasMath_t old_mode_;  // Prior cuBLAS math mode, to be restored.
   bool ok_;                // Whether the change was successful.
@@ -510,7 +510,7 @@ bool CUDABlas::Init() {
   return true;
 }
 
-CUDABlas::CUDABlas(cuda::CUDAExecutor *parent)
+CUDABlas::CUDABlas(gpu::GpuExecutor *parent)
     : parent_(CHECK_NOTNULL(parent)), blas_(nullptr) {}
 
 CUDABlas::~CUDABlas() {
@@ -521,10 +521,10 @@ CUDABlas::~CUDABlas() {
 
 bool CUDABlas::SetStream(Stream *stream) {
   CHECK(stream != nullptr);
-  CHECK(AsCUDAStreamValue(stream) != nullptr);
+  CHECK(AsGpuStreamValue(stream) != nullptr);
   CHECK(blas_ != nullptr);
   cublasStatus_t ret =
-      wrap::cublasSetStream(parent_, blas_, AsCUDAStreamValue(stream));
+      wrap::cublasSetStream(parent_, blas_, AsGpuStreamValue(stream));
   if (ret != CUBLAS_STATUS_SUCCESS) {
     LOG(ERROR) << "failed to set stream for cuBLAS calls: " << ToString(ret);
     return false;
@@ -702,7 +702,7 @@ bool CUDABlas::DoBlasAsum(Stream *stream, uint64 elem_count,
                           DeviceMemory<float> *result) {
   return DoBlasInternal(wrap::cublasSasum, stream,
                         false /* = pointer_mode_host */, elem_count,
-                        CUDAMemory(x), incx, CUDAMemoryMutable(result));
+                        GpuMemory(x), incx, GpuMemoryMutable(result));
 }
 
 bool CUDABlas::DoBlasAsum(Stream *stream, uint64 elem_count,
@@ -710,7 +710,7 @@ bool CUDABlas::DoBlasAsum(Stream *stream, uint64 elem_count,
                           DeviceMemory<double> *result) {
   return DoBlasInternal(wrap::cublasDasum, stream,
                         false /* = pointer_mode_host */, elem_count,
-                        CUDAMemory(x), incx, CUDAMemoryMutable(result));
+                        GpuMemory(x), incx, GpuMemoryMutable(result));
 }
 
 bool CUDABlas::DoBlasAsum(Stream *stream, uint64 elem_count,
@@ -718,7 +718,7 @@ bool CUDABlas::DoBlasAsum(Stream *stream, uint64 elem_count,
                           DeviceMemory<float> *result) {
   return DoBlasInternal(
       wrap::cublasScasum, stream, false /* = pointer_mode_host */,
-      elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
+      elem_count, GpuComplex(GpuMemory(x)), incx, GpuMemoryMutable(result));
 }
 
 bool CUDABlas::DoBlasAsum(Stream *stream, uint64 elem_count,
@@ -726,7 +726,7 @@ bool CUDABlas::DoBlasAsum(Stream *stream, uint64 elem_count,
                           DeviceMemory<double> *result) {
   return DoBlasInternal(
       wrap::cublasDzasum, stream, false /* = pointer_mode_host */,
-      elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
+      elem_count, GpuComplex(GpuMemory(x)), incx, GpuMemoryMutable(result));
 }
 
 bool CUDABlas::DoBlasAxpy(Stream *stream, uint64 elem_count, float alpha,
@@ -734,7 +734,7 @@ bool CUDABlas::DoBlasAxpy(Stream *stream, uint64 elem_count, float alpha,
                           DeviceMemory<float> *y, int incy) {
   return DoBlasInternal(wrap::cublasSaxpy, stream,
                         true /* = pointer_mode_host */, elem_count, &alpha,
-                        CUDAMemory(x), incx, CUDAMemoryMutable(y), incy);
+                        GpuMemory(x), incx, GpuMemoryMutable(y), incy);
 }
 
 bool CUDABlas::DoBlasAxpy(Stream *stream, uint64 elem_count, double alpha,
@@ -742,7 +742,7 @@ bool CUDABlas::DoBlasAxpy(Stream *stream, uint64 elem_count, double alpha,
                           DeviceMemory<double> *y, int incy) {
   return DoBlasInternal(wrap::cublasDaxpy, stream,
                         true /* = pointer_mode_host */, elem_count, &alpha,
-                        CUDAMemory(x), incx, CUDAMemoryMutable(y), incy);
+                        GpuMemory(x), incx, GpuMemoryMutable(y), incy);
 }
 
 bool CUDABlas::DoBlasAxpy(Stream *stream, uint64 elem_count,
@@ -751,8 +751,8 @@ bool CUDABlas::DoBlasAxpy(Stream *stream, uint64 elem_count,
                           DeviceMemory<std::complex<float>> *y, int incy) {
   return DoBlasInternal(wrap::cublasCaxpy, stream,
                         true /* = pointer_mode_host */, elem_count,
-                        CUDAComplex(&alpha), CUDAComplex(CUDAMemory(x)), incx,
-                        CUDAComplex(CUDAMemoryMutable(y)), incy);
+                        GpuComplex(&alpha), GpuComplex(GpuMemory(x)), incx,
+                        GpuComplex(GpuMemoryMutable(y)), incy);
 }
 
 bool CUDABlas::DoBlasAxpy(Stream *stream, uint64 elem_count,
@@ -761,8 +761,8 @@ bool CUDABlas::DoBlasAxpy(Stream *stream, uint64 elem_count,
                           DeviceMemory<std::complex<double>> *y, int incy) {
   return DoBlasInternal(wrap::cublasZaxpy, stream,
                         true /* = pointer_mode_host */, elem_count,
-                        CUDAComplex(&alpha), CUDAComplex(CUDAMemory(x)), incx,
-                        CUDAComplex(CUDAMemoryMutable(y)), incy);
+                        GpuComplex(&alpha), GpuComplex(GpuMemory(x)), incx,
+                        GpuComplex(GpuMemoryMutable(y)), incy);
 }
 
 bool CUDABlas::DoBlasCopy(Stream *stream, uint64 elem_count,
@@ -770,7 +770,7 @@ bool CUDABlas::DoBlasCopy(Stream *stream, uint64 elem_count,
                           DeviceMemory<float> *y, int incy) {
   return DoBlasInternal(wrap::cublasScopy, stream,
                         true /* = pointer_mode_host */, elem_count,
-                        CUDAMemory(x), incx, CUDAMemoryMutable(y), incy);
+                        GpuMemory(x), incx, GpuMemoryMutable(y), incy);
 }
 
 bool CUDABlas::DoBlasCopy(Stream *stream, uint64 elem_count,
@@ -778,7 +778,7 @@ bool CUDABlas::DoBlasCopy(Stream *stream, uint64 elem_count,
                           DeviceMemory<double> *y, int incy) {
   return DoBlasInternal(wrap::cublasDcopy, stream,
                         true /* = pointer_mode_host */, elem_count,
-                        CUDAMemory(x), incx, CUDAMemoryMutable(y), incy);
+                        GpuMemory(x), incx, GpuMemoryMutable(y), incy);
 }
 
 bool CUDABlas::DoBlasCopy(Stream *stream, uint64 elem_count,
@@ -786,8 +786,8 @@ bool CUDABlas::DoBlasCopy(Stream *stream, uint64 elem_count,
                           DeviceMemory<std::complex<float>> *y, int incy) {
   return DoBlasInternal(wrap::cublasCcopy, stream,
                         true /* = pointer_mode_host */, elem_count,
-                        CUDAComplex(CUDAMemory(x)), incx,
-                        CUDAComplex(CUDAMemoryMutable(y)), incy);
+                        GpuComplex(GpuMemory(x)), incx,
+                        GpuComplex(GpuMemoryMutable(y)), incy);
 }
 
 bool CUDABlas::DoBlasCopy(Stream *stream, uint64 elem_count,
@@ -795,8 +795,8 @@ bool CUDABlas::DoBlasCopy(Stream *stream, uint64 elem_count,
                           DeviceMemory<std::complex<double>> *y, int incy) {
   return DoBlasInternal(wrap::cublasZcopy, stream,
                         true /* = pointer_mode_host */, elem_count,
-                        CUDAComplex(CUDAMemory(x)), incx,
-                        CUDAComplex(CUDAMemoryMutable(y)), incy);
+                        GpuComplex(GpuMemory(x)), incx,
+                        GpuComplex(GpuMemoryMutable(y)), incy);
 }
 
 bool CUDABlas::DoBlasDot(Stream *stream, uint64 elem_count,
@@ -805,7 +805,7 @@ bool CUDABlas::DoBlasDot(Stream *stream, uint64 elem_count,
                          DeviceMemory<float> *result) {
   return DoBlasInternal(
       wrap::cublasSdot, stream, false /* = pointer_mode_host */, elem_count,
-      CUDAMemory(x), incx, CUDAMemory(y), incy, CUDAMemoryMutable(result));
+      GpuMemory(x), incx, GpuMemory(y), incy, GpuMemoryMutable(result));
 }
 
 bool CUDABlas::DoBlasDot(Stream *stream, uint64 elem_count,
@@ -814,7 +814,7 @@ bool CUDABlas::DoBlasDot(Stream *stream, uint64 elem_count,
                          DeviceMemory<double> *result) {
   return DoBlasInternal(
       wrap::cublasDdot, stream, false /* = pointer_mode_host */, elem_count,
-      CUDAMemory(x), incx, CUDAMemory(y), incy, CUDAMemoryMutable(result));
+      GpuMemory(x), incx, GpuMemory(y), incy, GpuMemoryMutable(result));
 }
 
 bool CUDABlas::DoBlasDotc(Stream *stream, uint64 elem_count,
@@ -823,8 +823,8 @@ bool CUDABlas::DoBlasDotc(Stream *stream, uint64 elem_count,
                           DeviceMemory<std::complex<float>> *result) {
   return DoBlasInternal(
       wrap::cublasCdotc, stream, false /* = pointer_mode_host */, elem_count,
-      CUDAComplex(CUDAMemory(x)), incx, CUDAComplex(CUDAMemory(y)), incy,
-      CUDAComplex(CUDAMemoryMutable(result)));
+      GpuComplex(GpuMemory(x)), incx, GpuComplex(GpuMemory(y)), incy,
+      GpuComplex(GpuMemoryMutable(result)));
 }
 
 bool CUDABlas::DoBlasDotc(Stream *stream, uint64 elem_count,
@@ -833,8 +833,8 @@ bool CUDABlas::DoBlasDotc(Stream *stream, uint64 elem_count,
                           DeviceMemory<std::complex<double>> *result) {
   return DoBlasInternal(
       wrap::cublasZdotc, stream, false /* = pointer_mode_host */, elem_count,
-      CUDAComplex(CUDAMemory(x)), incx, CUDAComplex(CUDAMemory(y)), incy,
-      CUDAComplex(CUDAMemoryMutable(result)));
+      GpuComplex(GpuMemory(x)), incx, GpuComplex(GpuMemory(y)), incy,
+      GpuComplex(GpuMemoryMutable(result)));
 }
 
 bool CUDABlas::DoBlasDotu(Stream *stream, uint64 elem_count,
@@ -843,8 +843,8 @@ bool CUDABlas::DoBlasDotu(Stream *stream, uint64 elem_count,
                           DeviceMemory<std::complex<float>> *result) {
   return DoBlasInternal(
       wrap::cublasCdotu, stream, false /* = pointer_mode_host */, elem_count,
-      CUDAComplex(CUDAMemory(x)), incx, CUDAComplex(CUDAMemory(y)), incy,
-      CUDAComplex(CUDAMemoryMutable(result)));
+      GpuComplex(GpuMemory(x)), incx, GpuComplex(GpuMemory(y)), incy,
+      GpuComplex(GpuMemoryMutable(result)));
 }
 
 bool CUDABlas::DoBlasDotu(Stream *stream, uint64 elem_count,
@@ -853,8 +853,8 @@ bool CUDABlas::DoBlasDotu(Stream *stream, uint64 elem_count,
                           DeviceMemory<std::complex<double>> *result) {
   return DoBlasInternal(
       wrap::cublasZdotu, stream, false /* = pointer_mode_host */, elem_count,
-      CUDAComplex(CUDAMemory(x)), incx, CUDAComplex(CUDAMemory(y)), incy,
-      CUDAComplex(CUDAMemoryMutable(result)));
+      GpuComplex(GpuMemory(x)), incx, GpuComplex(GpuMemory(y)), incy,
+      GpuComplex(GpuMemoryMutable(result)));
 }
 
 bool CUDABlas::DoBlasNrm2(Stream *stream, uint64 elem_count,
@@ -862,7 +862,7 @@ bool CUDABlas::DoBlasNrm2(Stream *stream, uint64 elem_count,
                           DeviceMemory<float> *result) {
   return DoBlasInternal(wrap::cublasSnrm2, stream,
                         false /* = pointer_mode_host */, elem_count,
-                        CUDAMemory(x), incx, CUDAMemoryMutable(result));
+                        GpuMemory(x), incx, GpuMemoryMutable(result));
 }
 
 bool CUDABlas::DoBlasNrm2(Stream *stream, uint64 elem_count,
@@ -870,7 +870,7 @@ bool CUDABlas::DoBlasNrm2(Stream *stream, uint64 elem_count,
                           DeviceMemory<double> *result) {
   return DoBlasInternal(wrap::cublasDnrm2, stream,
                         false /* = pointer_mode_host */, elem_count,
-                        CUDAMemory(x), incx, CUDAMemoryMutable(result));
+                        GpuMemory(x), incx, GpuMemoryMutable(result));
 }
 
 bool CUDABlas::DoBlasNrm2(Stream *stream, uint64 elem_count,
@@ -878,7 +878,7 @@ bool CUDABlas::DoBlasNrm2(Stream *stream, uint64 elem_count,
                           DeviceMemory<float> *result) {
   return DoBlasInternal(
       wrap::cublasScnrm2, stream, false /* = pointer_mode_host */,
-      elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
+      elem_count, GpuComplex(GpuMemory(x)), incx, GpuMemoryMutable(result));
 }
 
 bool CUDABlas::DoBlasNrm2(Stream *stream, uint64 elem_count,
@@ -886,7 +886,7 @@ bool CUDABlas::DoBlasNrm2(Stream *stream, uint64 elem_count,
                           DeviceMemory<double> *result) {
   return DoBlasInternal(
       wrap::cublasDznrm2, stream, false /* = pointer_mode_host */,
-      elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
+      elem_count, GpuComplex(GpuMemory(x)), incx, GpuMemoryMutable(result));
 }
 
 bool CUDABlas::DoBlasRot(Stream *stream, uint64 elem_count,
@@ -894,7 +894,7 @@ bool CUDABlas::DoBlasRot(Stream *stream, uint64 elem_count,
                          DeviceMemory<float> *y, int incy, float c, float s) {
   return DoBlasInternal(
       wrap::cublasSrot, stream, true /* = pointer_mode_host */, elem_count,
-      CUDAMemoryMutable(x), incx, CUDAMemoryMutable(y), incy, &c, &s);
+      GpuMemoryMutable(x), incx, GpuMemoryMutable(y), incy, &c, &s);
 }
 
 bool CUDABlas::DoBlasRot(Stream *stream, uint64 elem_count,
@@ -903,7 +903,7 @@ bool CUDABlas::DoBlasRot(Stream *stream, uint64 elem_count,
                          double s) {
   return DoBlasInternal(
       wrap::cublasDrot, stream, true /* = pointer_mode_host */, elem_count,
-      CUDAMemoryMutable(x), incx, CUDAMemoryMutable(y), incy, &c, &s);
+      GpuMemoryMutable(x), incx, GpuMemoryMutable(y), incy, &c, &s);
 }
 
 bool CUDABlas::DoBlasRot(Stream *stream, uint64 elem_count,
@@ -912,8 +912,8 @@ bool CUDABlas::DoBlasRot(Stream *stream, uint64 elem_count,
                          float c, float s) {
   return DoBlasInternal(wrap::cublasCsrot, stream,
                         true /* = pointer_mode_host */, elem_count,
-                        CUDAComplex(CUDAMemoryMutable(x)), incx,
-                        CUDAComplex(CUDAMemoryMutable(y)), incy, &c, &s);
+                        GpuComplex(GpuMemoryMutable(x)), incx,
+                        GpuComplex(GpuMemoryMutable(y)), incy, &c, &s);
 }
 
 bool CUDABlas::DoBlasRot(Stream *stream, uint64 elem_count,
@@ -922,17 +922,17 @@ bool CUDABlas::DoBlasRot(Stream *stream, uint64 elem_count,
                          double c, double s) {
   return DoBlasInternal(wrap::cublasZdrot, stream,
                         true /* = pointer_mode_host */, elem_count,
-                        CUDAComplex(CUDAMemoryMutable(x)), incx,
-                        CUDAComplex(CUDAMemoryMutable(y)), incy, &c, &s);
+                        GpuComplex(GpuMemoryMutable(x)), incx,
+                        GpuComplex(GpuMemoryMutable(y)), incy, &c, &s);
 }
 
 bool CUDABlas::DoBlasRotg(Stream *stream, DeviceMemory<float> *a,
                           DeviceMemory<float> *b, DeviceMemory<float> *c,
                           DeviceMemory<float> *s) {
   return DoBlasInternal(wrap::cublasSrotg, stream,
-                        false /* = pointer_mode_host */, CUDAMemoryMutable(a),
-                        CUDAMemoryMutable(b), CUDAMemoryMutable(c),
-                        CUDAMemoryMutable(s));
+                        false /* = pointer_mode_host */, GpuMemoryMutable(a),
+                        GpuMemoryMutable(b), GpuMemoryMutable(c),
+                        GpuMemoryMutable(s));
 }
 
 bool CUDABlas::DoBlasRotg(Stream *stream, DeviceMemory<double> *a,
@@ -940,8 +940,8 @@ bool CUDABlas::DoBlasRotg(Stream *stream, DeviceMemory<double> *a,
                           DeviceMemory<double> *s) {
   return DoBlasInternal(wrap::cublasDrotg, stream,
                         false /* = pointer_mode_host */,
-                        CUDAComplex(CUDAMemoryMutable(a)), CUDAMemoryMutable(b),
-                        CUDAMemoryMutable(c), CUDAMemoryMutable(s));
+                        GpuComplex(GpuMemoryMutable(a)), GpuMemoryMutable(b),
+                        GpuMemoryMutable(c), GpuMemoryMutable(s));
 }
 
 bool CUDABlas::DoBlasRotg(Stream *stream, DeviceMemory<std::complex<float>> *a,
@@ -950,8 +950,8 @@ bool CUDABlas::DoBlasRotg(Stream *stream, DeviceMemory<std::complex<float>> *a,
                           DeviceMemory<std::complex<float>> *s) {
   return DoBlasInternal(
       wrap::cublasCrotg, stream, false /* = pointer_mode_host */,
-      CUDAComplex(CUDAMemoryMutable(a)), CUDAComplex(CUDAMemoryMutable(b)),
-      CUDAComplex(CUDAMemoryMutable(c)), CUDAComplex(CUDAMemoryMutable(s)));
+      GpuComplex(GpuMemoryMutable(a)), GpuComplex(GpuMemoryMutable(b)),
+      GpuComplex(GpuMemoryMutable(c)), GpuComplex(GpuMemoryMutable(s)));
 }
 
 bool CUDABlas::DoBlasRotg(Stream *stream, DeviceMemory<std::complex<double>> *a,
@@ -960,8 +960,8 @@ bool CUDABlas::DoBlasRotg(Stream *stream, DeviceMemory<std::complex<double>> *a,
                           DeviceMemory<std::complex<double>> *s) {
   return DoBlasInternal(
       wrap::cublasZrotg, stream, false /* = pointer_mode_host */,
-      CUDAComplex(CUDAMemoryMutable(a)), CUDAComplex(CUDAMemoryMutable(b)),
-      CUDAComplex(CUDAMemoryMutable(c)), CUDAComplex(CUDAMemoryMutable(s)));
+      GpuComplex(GpuMemoryMutable(a)), GpuComplex(GpuMemoryMutable(b)),
+      GpuComplex(GpuMemoryMutable(c)), GpuComplex(GpuMemoryMutable(s)));
 }
 
 bool CUDABlas::DoBlasRotm(Stream *stream, uint64 elem_count,
@@ -970,8 +970,8 @@ bool CUDABlas::DoBlasRotm(Stream *stream, uint64 elem_count,
                           const DeviceMemory<float> &param) {
   return DoBlasInternal(wrap::cublasSrotm, stream,
                         false /* = pointer_mode_host */, elem_count,
-                        CUDAMemoryMutable(x), incx, CUDAMemoryMutable(y), incy,
-                        CUDAMemory(param));
+                        GpuMemoryMutable(x), incx, GpuMemoryMutable(y), incy,
+                        GpuMemory(param));
 }
 
 bool CUDABlas::DoBlasRotm(Stream *stream, uint64 elem_count,
@@ -980,8 +980,8 @@ bool CUDABlas::DoBlasRotm(Stream *stream, uint64 elem_count,
                           const DeviceMemory<double> &param) {
   return DoBlasInternal(wrap::cublasDrotm, stream,
                         false /* = pointer_mode_host */, elem_count,
-                        CUDAMemoryMutable(x), incx, CUDAMemoryMutable(y), incy,
-                        CUDAMemory(param));
+                        GpuMemoryMutable(x), incx, GpuMemoryMutable(y), incy,
+                        GpuMemory(param));
 }
 
 bool CUDABlas::DoBlasRotmg(Stream *stream, DeviceMemory<float> *d1,
@@ -989,9 +989,9 @@ bool CUDABlas::DoBlasRotmg(Stream *stream, DeviceMemory<float> *d1,
                            const DeviceMemory<float> &y1,
                            DeviceMemory<float> *param) {
   return DoBlasInternal(wrap::cublasSrotmg, stream,
-                        false /* = pointer_mode_host */, CUDAMemoryMutable(d1),
-                        CUDAMemoryMutable(d2), CUDAMemoryMutable(x1),
-                        CUDAMemory(y1), CUDAMemoryMutable(param));
+                        false /* = pointer_mode_host */, GpuMemoryMutable(d1),
+                        GpuMemoryMutable(d2), GpuMemoryMutable(x1),
+                        GpuMemory(y1), GpuMemoryMutable(param));
 }
 
 bool CUDABlas::DoBlasRotmg(Stream *stream, DeviceMemory<double> *d1,
@@ -999,37 +999,37 @@ bool CUDABlas::DoBlasRotmg(Stream *stream, DeviceMemory<double> *d1,
                            const DeviceMemory<double> &y1,
                            DeviceMemory<double> *param) {
   return DoBlasInternal(wrap::cublasDrotmg, stream,
-                        false /* = pointer_mode_host */, CUDAMemoryMutable(d1),
-                        CUDAMemoryMutable(d2), CUDAMemoryMutable(x1),
-                        CUDAMemory(y1), CUDAMemoryMutable(param));
+                        false /* = pointer_mode_host */, GpuMemoryMutable(d1),
+                        GpuMemoryMutable(d2), GpuMemoryMutable(x1),
+                        GpuMemory(y1), GpuMemoryMutable(param));
 }
 
 bool CUDABlas::DoBlasScal(Stream *stream, uint64 elem_count, float alpha,
                           DeviceMemory<float> *x, int incx) {
   return DoBlasInternal(wrap::cublasSscal, stream,
                         true /* = pointer_mode_host */, elem_count, &alpha,
-                        CUDAMemoryMutable(x), incx);
+                        GpuMemoryMutable(x), incx);
 }
 
 bool CUDABlas::DoBlasScal(Stream *stream, uint64 elem_count, double alpha,
                           DeviceMemory<double> *x, int incx) {
   return DoBlasInternal(wrap::cublasDscal, stream,
                         true /* = pointer_mode_host */, elem_count, &alpha,
-                        CUDAMemoryMutable(x), incx);
+                        GpuMemoryMutable(x), incx);
 }
 
 bool CUDABlas::DoBlasScal(Stream *stream, uint64 elem_count, float alpha,
                           DeviceMemory<std::complex<float>> *x, int incx) {
   return DoBlasInternal(
       wrap::cublasCsscal, stream, true /* = pointer_mode_host */, elem_count,
-      CUDAComplex(&alpha), CUDAComplex(CUDAMemoryMutable(x)), incx);
+      GpuComplex(&alpha), GpuComplex(GpuMemoryMutable(x)), incx);
 }
 
 bool CUDABlas::DoBlasScal(Stream *stream, uint64 elem_count, double alpha,
                           DeviceMemory<std::complex<double>> *x, int incx) {
   return DoBlasInternal(
       wrap::cublasZdscal, stream, true /* = pointer_mode_host */, elem_count,
-      CUDAComplex(&alpha), CUDAComplex(CUDAMemoryMutable(x)), incx);
+      GpuComplex(&alpha), GpuComplex(GpuMemoryMutable(x)), incx);
 }
 
 bool CUDABlas::DoBlasScal(Stream *stream, uint64 elem_count,
@@ -1037,7 +1037,7 @@ bool CUDABlas::DoBlasScal(Stream *stream, uint64 elem_count,
                           DeviceMemory<std::complex<float>> *x, int incx) {
   return DoBlasInternal(
       wrap::cublasCscal, stream, true /* = pointer_mode_host */, elem_count,
-      CUDAComplex(&alpha), CUDAComplex(CUDAMemoryMutable(x)), incx);
+      GpuComplex(&alpha), GpuComplex(GpuMemoryMutable(x)), incx);
 }
 
 bool CUDABlas::DoBlasScal(Stream *stream, uint64 elem_count,
@@ -1045,7 +1045,7 @@ bool CUDABlas::DoBlasScal(Stream *stream, uint64 elem_count,
                           DeviceMemory<std::complex<double>> *x, int incx) {
   return DoBlasInternal(
       wrap::cublasZscal, stream, true /* = pointer_mode_host */, elem_count,
-      CUDAComplex(&alpha), CUDAComplex(CUDAMemoryMutable(x)), incx);
+      GpuComplex(&alpha), GpuComplex(GpuMemoryMutable(x)), incx);
 }
 
 bool CUDABlas::DoBlasSwap(Stream *stream, uint64 elem_count,
@@ -1053,7 +1053,7 @@ bool CUDABlas::DoBlasSwap(Stream *stream, uint64 elem_count,
                           DeviceMemory<float> *y, int incy) {
   return DoBlasInternal(wrap::cublasSswap, stream,
                         true /* = pointer_mode_host */, elem_count,
-                        CUDAMemoryMutable(x), incx, CUDAMemoryMutable(y), incy);
+                        GpuMemoryMutable(x), incx, GpuMemoryMutable(y), incy);
 }
 
 bool CUDABlas::DoBlasSwap(Stream *stream, uint64 elem_count,
@@ -1061,7 +1061,7 @@ bool CUDABlas::DoBlasSwap(Stream *stream, uint64 elem_count,
                           DeviceMemory<double> *y, int incy) {
   return DoBlasInternal(wrap::cublasDswap, stream,
                         true /* = pointer_mode_host */, elem_count,
-                        CUDAMemoryMutable(x), incx, CUDAMemoryMutable(y), incy);
+                        GpuMemoryMutable(x), incx, GpuMemoryMutable(y), incy);
 }
 
 bool CUDABlas::DoBlasSwap(Stream *stream, uint64 elem_count,
@@ -1069,8 +1069,8 @@ bool CUDABlas::DoBlasSwap(Stream *stream, uint64 elem_count,
                           DeviceMemory<std::complex<float>> *y, int incy) {
   return DoBlasInternal(wrap::cublasCswap, stream,
                         true /* = pointer_mode_host */, elem_count,
-                        CUDAComplex(CUDAMemoryMutable(x)), incx,
-                        CUDAComplex(CUDAMemoryMutable(y)), incy);
+                        GpuComplex(GpuMemoryMutable(x)), incx,
+                        GpuComplex(GpuMemoryMutable(y)), incy);
 }
 
 bool CUDABlas::DoBlasSwap(Stream *stream, uint64 elem_count,
@@ -1078,8 +1078,8 @@ bool CUDABlas::DoBlasSwap(Stream *stream, uint64 elem_count,
                           DeviceMemory<std::complex<double>> *y, int incy) {
   return DoBlasInternal(wrap::cublasZswap, stream,
                         true /* = pointer_mode_host */, elem_count,
-                        CUDAComplex(CUDAMemoryMutable(x)), incx,
-                        CUDAComplex(CUDAMemoryMutable(y)), incy);
+                        GpuComplex(GpuMemoryMutable(x)), incx,
+                        GpuComplex(GpuMemoryMutable(y)), incy);
 }
 
 bool CUDABlas::DoBlasIamax(Stream *stream, uint64 elem_count,
@@ -1087,7 +1087,7 @@ bool CUDABlas::DoBlasIamax(Stream *stream, uint64 elem_count,
                            DeviceMemory<int> *result) {
   return DoBlasInternal(wrap::cublasIsamax, stream,
                         false /* = pointer_mode_host */, elem_count,
-                        CUDAMemory(x), incx, CUDAMemoryMutable(result));
+                        GpuMemory(x), incx, GpuMemoryMutable(result));
 }
 
 bool CUDABlas::DoBlasIamax(Stream *stream, uint64 elem_count,
@@ -1095,7 +1095,7 @@ bool CUDABlas::DoBlasIamax(Stream *stream, uint64 elem_count,
                            DeviceMemory<int> *result) {
   return DoBlasInternal(wrap::cublasIdamax, stream,
                         false /* = pointer_mode_host */, elem_count,
-                        CUDAMemory(x), incx, CUDAMemoryMutable(result));
+                        GpuMemory(x), incx, GpuMemoryMutable(result));
 }
 
 bool CUDABlas::DoBlasIamax(Stream *stream, uint64 elem_count,
@@ -1103,7 +1103,7 @@ bool CUDABlas::DoBlasIamax(Stream *stream, uint64 elem_count,
                            DeviceMemory<int> *result) {
   return DoBlasInternal(
       wrap::cublasIcamax, stream, false /* = pointer_mode_host */,
-      elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
+      elem_count, GpuComplex(GpuMemory(x)), incx, GpuMemoryMutable(result));
 }
 
 bool CUDABlas::DoBlasIamax(Stream *stream, uint64 elem_count,
@@ -1111,7 +1111,7 @@ bool CUDABlas::DoBlasIamax(Stream *stream, uint64 elem_count,
                            int incx, DeviceMemory<int> *result) {
   return DoBlasInternal(
       wrap::cublasIzamax, stream, false /* = pointer_mode_host */,
-      elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
+      elem_count, GpuComplex(GpuMemory(x)), incx, GpuMemoryMutable(result));
 }
 
 bool CUDABlas::DoBlasIamin(Stream *stream, uint64 elem_count,
@@ -1119,7 +1119,7 @@ bool CUDABlas::DoBlasIamin(Stream *stream, uint64 elem_count,
                            DeviceMemory<int> *result) {
   return DoBlasInternal(
       wrap::cublasIsamin, stream, false /* = pointer_mode_host */,
-      elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
+      elem_count, GpuComplex(GpuMemory(x)), incx, GpuMemoryMutable(result));
 }
 
 bool CUDABlas::DoBlasIamin(Stream *stream, uint64 elem_count,
@@ -1127,7 +1127,7 @@ bool CUDABlas::DoBlasIamin(Stream *stream, uint64 elem_count,
                            DeviceMemory<int> *result) {
   return DoBlasInternal(
       wrap::cublasIdamin, stream, false /* = pointer_mode_host */,
-      elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
+      elem_count, GpuComplex(GpuMemory(x)), incx, GpuMemoryMutable(result));
 }
 
 bool CUDABlas::DoBlasIamin(Stream *stream, uint64 elem_count,
@@ -1135,7 +1135,7 @@ bool CUDABlas::DoBlasIamin(Stream *stream, uint64 elem_count,
                            DeviceMemory<int> *result) {
   return DoBlasInternal(
       wrap::cublasIcamin, stream, false /* = pointer_mode_host */,
-      elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
+      elem_count, GpuComplex(GpuMemory(x)), incx, GpuMemoryMutable(result));
 }
 
 bool CUDABlas::DoBlasIamin(Stream *stream, uint64 elem_count,
@@ -1143,7 +1143,7 @@ bool CUDABlas::DoBlasIamin(Stream *stream, uint64 elem_count,
                            int incx, DeviceMemory<int> *result) {
   return DoBlasInternal(
       wrap::cublasIzamin, stream, false /* = pointer_mode_host */,
-      elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
+      elem_count, GpuComplex(GpuMemory(x)), incx, GpuMemoryMutable(result));
 }
 
 bool CUDABlas::DoBlasGbmv(Stream *stream, blas::Transpose trans, uint64 m,
@@ -1153,8 +1153,8 @@ bool CUDABlas::DoBlasGbmv(Stream *stream, blas::Transpose trans, uint64 m,
                           DeviceMemory<float> *y, int incy) {
   return DoBlasInternal(
       wrap::cublasSgbmv, stream, true /* = pointer_mode_host */,
-      CUDABlasTranspose(trans), m, n, kl, ku, &alpha, CUDAMemory(a), lda,
-      CUDAMemory(x), incx, &beta, CUDAMemoryMutable(y), incy);
+      CUDABlasTranspose(trans), m, n, kl, ku, &alpha, GpuMemory(a), lda,
+      GpuMemory(x), incx, &beta, GpuMemoryMutable(y), incy);
 }
 
 bool CUDABlas::DoBlasGbmv(Stream *stream, blas::Transpose trans, uint64 m,
@@ -1164,8 +1164,8 @@ bool CUDABlas::DoBlasGbmv(Stream *stream, blas::Transpose trans, uint64 m,
                           DeviceMemory<double> *y, int incy) {
   return DoBlasInternal(
       wrap::cublasDgbmv, stream, true /* = pointer_mode_host */,
-      CUDABlasTranspose(trans), m, n, kl, ku, &alpha, CUDAMemory(a), lda,
-      CUDAMemory(x), incx, &beta, CUDAMemoryMutable(y), incy);
+      CUDABlasTranspose(trans), m, n, kl, ku, &alpha, GpuMemory(a), lda,
+      GpuMemory(x), incx, &beta, GpuMemoryMutable(y), incy);
 }
 
 bool CUDABlas::DoBlasGbmv(Stream *stream, blas::Transpose trans, uint64 m,
@@ -1177,9 +1177,9 @@ bool CUDABlas::DoBlasGbmv(Stream *stream, blas::Transpose trans, uint64 m,
                           DeviceMemory<std::complex<float>> *y, int incy) {
   return DoBlasInternal(
       wrap::cublasCgbmv, stream, true /* = pointer_mode_host */,
-      CUDABlasTranspose(trans), m, n, kl, ku, CUDAComplex(&alpha),
-      CUDAComplex(CUDAMemory(a)), lda, CUDAComplex(CUDAMemory(x)), incx,
-      CUDAComplex(&beta), CUDAComplex(CUDAMemoryMutable(y)), incy);
+      CUDABlasTranspose(trans), m, n, kl, ku, GpuComplex(&alpha),
+      GpuComplex(GpuMemory(a)), lda, GpuComplex(GpuMemory(x)), incx,
+      GpuComplex(&beta), GpuComplex(GpuMemoryMutable(y)), incy);
 }
 
 bool CUDABlas::DoBlasGbmv(Stream *stream, blas::Transpose trans, uint64 m,
@@ -1191,9 +1191,9 @@ bool CUDABlas::DoBlasGbmv(Stream *stream, blas::Transpose trans, uint64 m,
                           DeviceMemory<std::complex<double>> *y, int incy) {
   return DoBlasInternal(
       wrap::cublasZgbmv, stream, true /* = pointer_mode_host */,
-      CUDABlasTranspose(trans), m, n, kl, ku, CUDAComplex(&alpha),
-      CUDAComplex(CUDAMemory(a)), lda, CUDAComplex(CUDAMemory(x)), incx,
-      CUDAComplex(&beta), CUDAComplex(CUDAMemoryMutable(y)), incy);
+      CUDABlasTranspose(trans), m, n, kl, ku, GpuComplex(&alpha),
+      GpuComplex(GpuMemory(a)), lda, GpuComplex(GpuMemory(x)), incx,
+      GpuComplex(&beta), GpuComplex(GpuMemoryMutable(y)), incy);
 }
 
 bool CUDABlas::DoBlasGemv(Stream *stream, blas::Transpose trans, uint64 m,
@@ -1202,8 +1202,8 @@ bool CUDABlas::DoBlasGemv(Stream *stream, blas::Transpose trans, uint64 m,
                           float beta, DeviceMemory<float> *y, int incy) {
   return DoBlasInternal(
       wrap::cublasSgemv, stream, true /* = pointer_mode_host */,
-      CUDABlasTranspose(trans), m, n, &alpha, CUDAMemory(a), lda, CUDAMemory(x),
-      incx, &beta, CUDAMemoryMutable(y), incy);
+      CUDABlasTranspose(trans), m, n, &alpha, GpuMemory(a), lda, GpuMemory(x),
+      incx, &beta, GpuMemoryMutable(y), incy);
 }
 
 bool CUDABlas::DoBlasGemv(Stream *stream, blas::Transpose trans, uint64 m,
@@ -1212,8 +1212,8 @@ bool CUDABlas::DoBlasGemv(Stream *stream, blas::Transpose trans, uint64 m,
                           double beta, DeviceMemory<double> *y, int incy) {
   return DoBlasInternal(
       wrap::cublasDgemv, stream, true /* = pointer_mode_host */,
-      CUDABlasTranspose(trans), m, n, &alpha, CUDAMemory(a), lda, CUDAMemory(x),
-      incx, &beta, CUDAMemoryMutable(y), incy);
+      CUDABlasTranspose(trans), m, n, &alpha, GpuMemory(a), lda, GpuMemory(x),
+      incx, &beta, GpuMemoryMutable(y), incy);
 }
 
 bool CUDABlas::DoBlasGemv(Stream *stream, blas::Transpose trans, uint64 m,
@@ -1224,9 +1224,9 @@ bool CUDABlas::DoBlasGemv(Stream *stream, blas::Transpose trans, uint64 m,
                           DeviceMemory<std::complex<float>> *y, int incy) {
   return DoBlasInternal(
       wrap::cublasCgemv, stream, true /* = pointer_mode_host */,
-      CUDABlasTranspose(trans), m, n, CUDAComplex(&alpha),
-      CUDAComplex(CUDAMemory(a)), lda, CUDAComplex(CUDAMemory(x)), incx,
-      CUDAComplex(&beta), CUDAComplex(CUDAMemoryMutable(y)), incy);
+      CUDABlasTranspose(trans), m, n, GpuComplex(&alpha),
+      GpuComplex(GpuMemory(a)), lda, GpuComplex(GpuMemory(x)), incx,
+      GpuComplex(&beta), GpuComplex(GpuMemoryMutable(y)), incy);
 }
 
 bool CUDABlas::DoBlasGemv(Stream *stream, blas::Transpose trans, uint64 m,
@@ -1237,9 +1237,9 @@ bool CUDABlas::DoBlasGemv(Stream *stream, blas::Transpose trans, uint64 m,
                           DeviceMemory<std::complex<double>> *y, int incy) {
   return DoBlasInternal(
       wrap::cublasZgemv, stream, true /* = pointer_mode_host */,
-      CUDABlasTranspose(trans), m, n, CUDAComplex(&alpha),
-      CUDAComplex(CUDAMemory(a)), lda, CUDAComplex(CUDAMemory(x)), incx,
-      CUDAComplex(&beta), CUDAComplex(CUDAMemoryMutable(y)), incy);
+      CUDABlasTranspose(trans), m, n, GpuComplex(&alpha),
+      GpuComplex(GpuMemory(a)), lda, GpuComplex(GpuMemory(x)), incx,
+      GpuComplex(&beta), GpuComplex(GpuMemoryMutable(y)), incy);
 }
 
 bool CUDABlas::DoBlasGer(Stream *stream, uint64 m, uint64 n, float alpha,
@@ -1248,7 +1248,7 @@ bool CUDABlas::DoBlasGer(Stream *stream, uint64 m, uint64 n, float alpha,
                          DeviceMemory<float> *a, int lda) {
   return DoBlasInternal(
       wrap::cublasSger, stream, true /* = pointer_mode_host */, m, n, &alpha,
-      CUDAMemory(x), incx, CUDAMemory(y), incy, CUDAMemoryMutable(a), lda);
+      GpuMemory(x), incx, GpuMemory(y), incy, GpuMemoryMutable(a), lda);
 }
 
 bool CUDABlas::DoBlasGer(Stream *stream, uint64 m, uint64 n, double alpha,
@@ -1257,7 +1257,7 @@ bool CUDABlas::DoBlasGer(Stream *stream, uint64 m, uint64 n, double alpha,
                          DeviceMemory<double> *a, int lda) {
   return DoBlasInternal(
       wrap::cublasDger, stream, true /* = pointer_mode_host */, m, n, &alpha,
-      CUDAMemory(x), incx, CUDAMemory(y), incy, CUDAMemoryMutable(a), lda);
+      GpuMemory(x), incx, GpuMemory(y), incy, GpuMemoryMutable(a), lda);
 }
 
 bool CUDABlas::DoBlasGerc(Stream *stream, uint64 m, uint64 n,
@@ -1267,8 +1267,8 @@ bool CUDABlas::DoBlasGerc(Stream *stream, uint64 m, uint64 n,
                           DeviceMemory<std::complex<float>> *a, int lda) {
   return DoBlasInternal(
       wrap::cublasCgerc, stream, true /* = pointer_mode_host */, m, n,
-      CUDAComplex(&alpha), CUDAComplex(CUDAMemory(x)), incx,
-      CUDAComplex(CUDAMemory(y)), incy, CUDAComplex(CUDAMemoryMutable(a)), lda);
+      GpuComplex(&alpha), GpuComplex(GpuMemory(x)), incx,
+      GpuComplex(GpuMemory(y)), incy, GpuComplex(GpuMemoryMutable(a)), lda);
 }
 
 bool CUDABlas::DoBlasGerc(Stream *stream, uint64 m, uint64 n,
@@ -1278,8 +1278,8 @@ bool CUDABlas::DoBlasGerc(Stream *stream, uint64 m, uint64 n,
                           DeviceMemory<std::complex<double>> *a, int lda) {
   return DoBlasInternal(
       wrap::cublasZgerc, stream, true /* = pointer_mode_host */, m, n,
-      CUDAComplex(&alpha), CUDAComplex(CUDAMemory(x)), incx,
-      CUDAComplex(CUDAMemory(y)), incy, CUDAComplex(CUDAMemoryMutable(a)), lda);
+      GpuComplex(&alpha), GpuComplex(GpuMemory(x)), incx,
+      GpuComplex(GpuMemory(y)), incy, GpuComplex(GpuMemoryMutable(a)), lda);
 }
 
 bool CUDABlas::DoBlasGeru(Stream *stream, uint64 m, uint64 n,
@@ -1289,8 +1289,8 @@ bool CUDABlas::DoBlasGeru(Stream *stream, uint64 m, uint64 n,
                           DeviceMemory<std::complex<float>> *a, int lda) {
   return DoBlasInternal(
       wrap::cublasCgeru, stream, true /* = pointer_mode_host */, m, n,
-      CUDAComplex(&alpha), CUDAComplex(CUDAMemory(x)), incx,
-      CUDAComplex(CUDAMemory(y)), incy, CUDAComplex(CUDAMemoryMutable(a)), lda);
+      GpuComplex(&alpha), GpuComplex(GpuMemory(x)), incx,
+      GpuComplex(GpuMemory(y)), incy, GpuComplex(GpuMemoryMutable(a)), lda);
 }
 
 bool CUDABlas::DoBlasGeru(Stream *stream, uint64 m, uint64 n,
@@ -1300,8 +1300,8 @@ bool CUDABlas::DoBlasGeru(Stream *stream, uint64 m, uint64 n,
                           DeviceMemory<std::complex<double>> *a, int lda) {
   return DoBlasInternal(
       wrap::cublasZgeru, stream, true /* = pointer_mode_host */, m, n,
-      CUDAComplex(&alpha), CUDAComplex(CUDAMemory(x)), incx,
-      CUDAComplex(CUDAMemory(y)), incy, CUDAComplex(CUDAMemoryMutable(a)), lda);
+      GpuComplex(&alpha), GpuComplex(GpuMemory(x)), incx,
+      GpuComplex(GpuMemory(y)), incy, GpuComplex(GpuMemoryMutable(a)), lda);
 }
 
 bool CUDABlas::DoBlasHbmv(Stream *stream, blas::UpperLower uplo, uint64 n,
@@ -1312,9 +1312,9 @@ bool CUDABlas::DoBlasHbmv(Stream *stream, blas::UpperLower uplo, uint64 n,
                           DeviceMemory<std::complex<float>> *y, int incy) {
   return DoBlasInternal(
       wrap::cublasChbmv, stream, true /* = pointer_mode_host */,
-      CUDABlasUpperLower(uplo), n, k, CUDAComplex(&alpha),
-      CUDAComplex(CUDAMemory(a)), lda, CUDAComplex(CUDAMemory(x)), incx,
-      CUDAComplex(&beta), CUDAComplex(CUDAMemoryMutable(y)), incy);
+      CUDABlasUpperLower(uplo), n, k, GpuComplex(&alpha),
+      GpuComplex(GpuMemory(a)), lda, GpuComplex(GpuMemory(x)), incx,
+      GpuComplex(&beta), GpuComplex(GpuMemoryMutable(y)), incy);
 }
 
 bool CUDABlas::DoBlasHbmv(Stream *stream, blas::UpperLower uplo, uint64 n,
@@ -1325,9 +1325,9 @@ bool CUDABlas::DoBlasHbmv(Stream *stream, blas::UpperLower uplo, uint64 n,
                           DeviceMemory<std::complex<double>> *y, int incy) {
   return DoBlasInternal(
       wrap::cublasZhbmv, stream, true /* = pointer_mode_host */,
-      CUDABlasUpperLower(uplo), n, k, CUDAComplex(&alpha),
-      CUDAComplex(CUDAMemory(a)), lda, CUDAComplex(CUDAMemory(x)), incx,
-      CUDAComplex(&beta), CUDAComplex(CUDAMemoryMutable(y)), incy);
+      CUDABlasUpperLower(uplo), n, k, GpuComplex(&alpha),
+      GpuComplex(GpuMemory(a)), lda, GpuComplex(GpuMemory(x)), incx,
+      GpuComplex(&beta), GpuComplex(GpuMemoryMutable(y)), incy);
 }
 
 bool CUDABlas::DoBlasHemv(Stream *stream, blas::UpperLower uplo, uint64 n,
@@ -1338,9 +1338,9 @@ bool CUDABlas::DoBlasHemv(Stream *stream, blas::UpperLower uplo, uint64 n,
                           DeviceMemory<std::complex<float>> *y, int incy) {
   return DoBlasInternal(
       wrap::cublasChemv, stream, true /* = pointer_mode_host */,
-      CUDABlasUpperLower(uplo), n, CUDAComplex(&alpha),
-      CUDAComplex(CUDAMemory(a)), lda, CUDAComplex(CUDAMemory(x)), incx,
-      CUDAComplex(&beta), CUDAComplex(CUDAMemoryMutable(y)), incy);
+      CUDABlasUpperLower(uplo), n, GpuComplex(&alpha),
+      GpuComplex(GpuMemory(a)), lda, GpuComplex(GpuMemory(x)), incx,
+      GpuComplex(&beta), GpuComplex(GpuMemoryMutable(y)), incy);
 }
 
 bool CUDABlas::DoBlasHemv(Stream *stream, blas::UpperLower uplo, uint64 n,
@@ -1351,9 +1351,9 @@ bool CUDABlas::DoBlasHemv(Stream *stream, blas::UpperLower uplo, uint64 n,
                           DeviceMemory<std::complex<double>> *y, int incy) {
   return DoBlasInternal(
       wrap::cublasZhemv, stream, true /* = pointer_mode_host */,
-      CUDABlasUpperLower(uplo), n, CUDAComplex(&alpha),
-      CUDAComplex(CUDAMemory(a)), lda, CUDAComplex(CUDAMemory(x)), incx,
-      CUDAComplex(&beta), CUDAComplex(CUDAMemoryMutable(y)), incy);
+      CUDABlasUpperLower(uplo), n, GpuComplex(&alpha),
+      GpuComplex(GpuMemory(a)), lda, GpuComplex(GpuMemory(x)), incx,
+      GpuComplex(&beta), GpuComplex(GpuMemoryMutable(y)), incy);
 }
 
 bool CUDABlas::DoBlasHer(Stream *stream, blas::UpperLower uplo, uint64 n,
@@ -1362,8 +1362,8 @@ bool CUDABlas::DoBlasHer(Stream *stream, blas::UpperLower uplo, uint64 n,
                          DeviceMemory<std::complex<float>> *a, int lda) {
   return DoBlasInternal(
       wrap::cublasCher, stream, true /* = pointer_mode_host */,
-      CUDABlasUpperLower(uplo), n, &alpha, CUDAComplex(CUDAMemory(x)), incx,
-      CUDAComplex(CUDAMemoryMutable(a)), lda);
+      CUDABlasUpperLower(uplo), n, &alpha, GpuComplex(GpuMemory(x)), incx,
+      GpuComplex(GpuMemoryMutable(a)), lda);
 }
 
 bool CUDABlas::DoBlasHer(Stream *stream, blas::UpperLower uplo, uint64 n,
@@ -1372,8 +1372,8 @@ bool CUDABlas::DoBlasHer(Stream *stream, blas::UpperLower uplo, uint64 n,
                          DeviceMemory<std::complex<double>> *a, int lda) {
   return DoBlasInternal(
       wrap::cublasZher, stream, true /* = pointer_mode_host */,
-      CUDABlasUpperLower(uplo), n, &alpha, CUDAComplex(CUDAMemory(x)), incx,
-      CUDAComplex(CUDAMemoryMutable(a)), lda);
+      CUDABlasUpperLower(uplo), n, &alpha, GpuComplex(GpuMemory(x)), incx,
+      GpuComplex(GpuMemoryMutable(a)), lda);
 }
 
 bool CUDABlas::DoBlasHer2(Stream *stream, blas::UpperLower uplo, uint64 n,
@@ -1383,9 +1383,9 @@ bool CUDABlas::DoBlasHer2(Stream *stream, blas::UpperLower uplo, uint64 n,
                           DeviceMemory<std::complex<float>> *a, int lda) {
   return DoBlasInternal(
       wrap::cublasCher2, stream, true /* = pointer_mode_host */,
-      CUDABlasUpperLower(uplo), n, CUDAComplex(&alpha),
-      CUDAComplex(CUDAMemory(x)), incx, CUDAComplex(CUDAMemory(y)), incy,
-      CUDAComplex(CUDAMemoryMutable(a)), lda);
+      CUDABlasUpperLower(uplo), n, GpuComplex(&alpha),
+      GpuComplex(GpuMemory(x)), incx, GpuComplex(GpuMemory(y)), incy,
+      GpuComplex(GpuMemoryMutable(a)), lda);
 }
 
 bool CUDABlas::DoBlasHer2(Stream *stream, blas::UpperLower uplo, uint64 n,
@@ -1395,9 +1395,9 @@ bool CUDABlas::DoBlasHer2(Stream *stream, blas::UpperLower uplo, uint64 n,
                           DeviceMemory<std::complex<double>> *a, int lda) {
   return DoBlasInternal(
       wrap::cublasZher2, stream, true /* = pointer_mode_host */,
-      CUDABlasUpperLower(uplo), n, CUDAComplex(&alpha),
-      CUDAComplex(CUDAMemory(x)), incx, CUDAComplex(CUDAMemory(y)), incy,
-      CUDAComplex(CUDAMemoryMutable(a)), lda);
+      CUDABlasUpperLower(uplo), n, GpuComplex(&alpha),
+      GpuComplex(GpuMemory(x)), incx, GpuComplex(GpuMemory(y)), incy,
+      GpuComplex(GpuMemoryMutable(a)), lda);
 }
 
 bool CUDABlas::DoBlasHpmv(Stream *stream, blas::UpperLower uplo, uint64 n,
@@ -1408,9 +1408,9 @@ bool CUDABlas::DoBlasHpmv(Stream *stream, blas::UpperLower uplo, uint64 n,
                           DeviceMemory<std::complex<float>> *y, int incy) {
   return DoBlasInternal(
       wrap::cublasChpmv, stream, true /* = pointer_mode_host */,
-      CUDABlasUpperLower(uplo), n, CUDAComplex(&alpha),
-      CUDAComplex(CUDAMemory(ap)), CUDAComplex(CUDAMemory(x)), incx,
-      CUDAComplex(&beta), CUDAComplex(CUDAMemoryMutable(y)), incy);
+      CUDABlasUpperLower(uplo), n, GpuComplex(&alpha),
+      GpuComplex(GpuMemory(ap)), GpuComplex(GpuMemory(x)), incx,
+      GpuComplex(&beta), GpuComplex(GpuMemoryMutable(y)), incy);
 }
 
 bool CUDABlas::DoBlasHpmv(Stream *stream, blas::UpperLower uplo, uint64 n,
@@ -1421,9 +1421,9 @@ bool CUDABlas::DoBlasHpmv(Stream *stream, blas::UpperLower uplo, uint64 n,
                           DeviceMemory<std::complex<double>> *y, int incy) {
   return DoBlasInternal(
       wrap::cublasZhpmv, stream, true /* = pointer_mode_host */,
-      CUDABlasUpperLower(uplo), n, CUDAComplex(&alpha),
-      CUDAComplex(CUDAMemory(ap)), CUDAComplex(CUDAMemory(x)), incx,
-      CUDAComplex(&beta), CUDAComplex(CUDAMemoryMutable(y)), incy);
+      CUDABlasUpperLower(uplo), n, GpuComplex(&alpha),
+      GpuComplex(GpuMemory(ap)), GpuComplex(GpuMemory(x)), incx,
+      GpuComplex(&beta), GpuComplex(GpuMemoryMutable(y)), incy);
 }
 
 bool CUDABlas::DoBlasHpr(Stream *stream, blas::UpperLower uplo, uint64 n,
@@ -1432,8 +1432,8 @@ bool CUDABlas::DoBlasHpr(Stream *stream, blas::UpperLower uplo, uint64 n,
                          DeviceMemory<std::complex<float>> *ap) {
   return DoBlasInternal(
       wrap::cublasChpr, stream, true /* = pointer_mode_host */,
-      CUDABlasUpperLower(uplo), n, CUDAComplex(&alpha),
-      CUDAComplex(CUDAMemory(x)), incx, CUDAComplex(CUDAMemoryMutable(ap)));
+      CUDABlasUpperLower(uplo), n, GpuComplex(&alpha),
+      GpuComplex(GpuMemory(x)), incx, GpuComplex(GpuMemoryMutable(ap)));
 }
 
 bool CUDABlas::DoBlasHpr(Stream *stream, blas::UpperLower uplo, uint64 n,
@@ -1442,8 +1442,8 @@ bool CUDABlas::DoBlasHpr(Stream *stream, blas::UpperLower uplo, uint64 n,
                          DeviceMemory<std::complex<double>> *ap) {
   return DoBlasInternal(
       wrap::cublasZhpr, stream, true /* = pointer_mode_host */,
-      CUDABlasUpperLower(uplo), n, CUDAComplex(&alpha),
-      CUDAComplex(CUDAMemory(x)), incx, CUDAComplex(CUDAMemoryMutable(ap)));
+      CUDABlasUpperLower(uplo), n, GpuComplex(&alpha),
+      GpuComplex(GpuMemory(x)), incx, GpuComplex(GpuMemoryMutable(ap)));
 }
 
 bool CUDABlas::DoBlasHpr2(Stream *stream, blas::UpperLower uplo, uint64 n,
@@ -1453,9 +1453,9 @@ bool CUDABlas::DoBlasHpr2(Stream *stream, blas::UpperLower uplo, uint64 n,
                           DeviceMemory<std::complex<float>> *ap) {
   return DoBlasInternal(
       wrap::cublasChpr2, stream, true /* = pointer_mode_host */,
-      CUDABlasUpperLower(uplo), n, CUDAComplex(&alpha),
-      CUDAComplex(CUDAMemory(x)), incx, CUDAComplex(CUDAMemory(y)), incy,
-      CUDAComplex(CUDAMemoryMutable(ap)));
+      CUDABlasUpperLower(uplo), n, GpuComplex(&alpha),
+      GpuComplex(GpuMemory(x)), incx, GpuComplex(GpuMemory(y)), incy,
+      GpuComplex(GpuMemoryMutable(ap)));
 }
 
 bool CUDABlas::DoBlasHpr2(Stream *stream, blas::UpperLower uplo, uint64 n,
@@ -1465,9 +1465,9 @@ bool CUDABlas::DoBlasHpr2(Stream *stream, blas::UpperLower uplo, uint64 n,
                           DeviceMemory<std::complex<double>> *ap) {
   return DoBlasInternal(
       wrap::cublasZhpr2, stream, true /* = pointer_mode_host */,
-      CUDABlasUpperLower(uplo), n, CUDAComplex(&alpha),
-      CUDAComplex(CUDAMemory(x)), incx, CUDAComplex(CUDAMemory(y)), incy,
-      CUDAComplex(CUDAMemoryMutable(ap)));
+      CUDABlasUpperLower(uplo), n, GpuComplex(&alpha),
+      GpuComplex(GpuMemory(x)), incx, GpuComplex(GpuMemory(y)), incy,
+      GpuComplex(GpuMemoryMutable(ap)));
 }
 
 bool CUDABlas::DoBlasSbmv(Stream *stream, blas::UpperLower uplo, uint64 n,
@@ -1476,8 +1476,8 @@ bool CUDABlas::DoBlasSbmv(Stream *stream, blas::UpperLower uplo, uint64 n,
                           float beta, DeviceMemory<float> *y, int incy) {
   return DoBlasInternal(
       wrap::cublasSsbmv, stream, true /* = pointer_mode_host */,
-      CUDABlasUpperLower(uplo), n, k, &alpha, CUDAMemory(a), lda, CUDAMemory(x),
-      incx, &beta, CUDAMemoryMutable(y), incy);
+      CUDABlasUpperLower(uplo), n, k, &alpha, GpuMemory(a), lda, GpuMemory(x),
+      incx, &beta, GpuMemoryMutable(y), incy);
 }
 
 bool CUDABlas::DoBlasSbmv(Stream *stream, blas::UpperLower uplo, uint64 n,
@@ -1486,8 +1486,8 @@ bool CUDABlas::DoBlasSbmv(Stream *stream, blas::UpperLower uplo, uint64 n,
                           double beta, DeviceMemory<double> *y, int incy) {
   return DoBlasInternal(
       wrap::cublasDsbmv, stream, true /* = pointer_mode_host */,
-      CUDABlasUpperLower(uplo), n, k, &alpha, CUDAMemory(a), lda, CUDAMemory(x),
-      incx, &beta, CUDAMemoryMutable(y), incy);
+      CUDABlasUpperLower(uplo), n, k, &alpha, GpuMemory(a), lda, GpuMemory(x),
+      incx, &beta, GpuMemoryMutable(y), incy);
 }
 
 bool CUDABlas::DoBlasSpmv(Stream *stream, blas::UpperLower uplo, uint64 n,
@@ -1496,8 +1496,8 @@ bool CUDABlas::DoBlasSpmv(Stream *stream, blas::UpperLower uplo, uint64 n,
                           DeviceMemory<float> *y, int incy) {
   return DoBlasInternal(wrap::cublasSspmv, stream,
                         true /* = pointer_mode_host */,
-                        CUDABlasUpperLower(uplo), n, &alpha, CUDAMemory(ap),
-                        CUDAMemory(x), incx, &beta, CUDAMemoryMutable(y), incy);
+                        CUDABlasUpperLower(uplo), n, &alpha, GpuMemory(ap),
+                        GpuMemory(x), incx, &beta, GpuMemoryMutable(y), incy);
 }
 
 bool CUDABlas::DoBlasSpmv(Stream *stream, blas::UpperLower uplo, uint64 n,
@@ -1506,8 +1506,8 @@ bool CUDABlas::DoBlasSpmv(Stream *stream, blas::UpperLower uplo, uint64 n,
                           DeviceMemory<double> *y, int incy) {
   return DoBlasInternal(wrap::cublasDspmv, stream,
                         true /* = pointer_mode_host */,
-                        CUDABlasUpperLower(uplo), n, &alpha, CUDAMemory(ap),
-                        CUDAMemory(x), incx, &beta, CUDAMemoryMutable(y), incy);
+                        CUDABlasUpperLower(uplo), n, &alpha, GpuMemory(ap),
+                        GpuMemory(x), incx, &beta, GpuMemoryMutable(y), incy);
 }
 
 bool CUDABlas::DoBlasSpr(Stream *stream, blas::UpperLower uplo, uint64 n,
@@ -1515,8 +1515,8 @@ bool CUDABlas::DoBlasSpr(Stream *stream, blas::UpperLower uplo, uint64 n,
                          DeviceMemory<float> *ap) {
   return DoBlasInternal(wrap::cublasSspr, stream,
                         true /* = pointer_mode_host */,
-                        CUDABlasUpperLower(uplo), n, &alpha, CUDAMemory(x),
-                        incx, CUDAMemoryMutable(ap));
+                        CUDABlasUpperLower(uplo), n, &alpha, GpuMemory(x),
+                        incx, GpuMemoryMutable(ap));
 }
 
 bool CUDABlas::DoBlasSpr(Stream *stream, blas::UpperLower uplo, uint64 n,
@@ -1524,8 +1524,8 @@ bool CUDABlas::DoBlasSpr(Stream *stream, blas::UpperLower uplo, uint64 n,
                          DeviceMemory<double> *ap) {
   return DoBlasInternal(wrap::cublasDspr, stream,
                         true /* = pointer_mode_host */,
-                        CUDABlasUpperLower(uplo), n, &alpha, CUDAMemory(x),
-                        incx, CUDAMemoryMutable(ap));
+                        CUDABlasUpperLower(uplo), n, &alpha, GpuMemory(x),
+                        incx, GpuMemoryMutable(ap));
 }
 
 bool CUDABlas::DoBlasSpr2(Stream *stream, blas::UpperLower uplo, uint64 n,
@@ -1534,8 +1534,8 @@ bool CUDABlas::DoBlasSpr2(Stream *stream, blas::UpperLower uplo, uint64 n,
                           DeviceMemory<float> *ap) {
   return DoBlasInternal(wrap::cublasSspr2, stream,
                         true /* = pointer_mode_host */,
-                        CUDABlasUpperLower(uplo), n, &alpha, CUDAMemory(x),
-                        incx, CUDAMemory(y), incy, CUDAMemoryMutable(ap));
+                        CUDABlasUpperLower(uplo), n, &alpha, GpuMemory(x),
+                        incx, GpuMemory(y), incy, GpuMemoryMutable(ap));
 }
 
 bool CUDABlas::DoBlasSpr2(Stream *stream, blas::UpperLower uplo, uint64 n,
@@ -1544,8 +1544,8 @@ bool CUDABlas::DoBlasSpr2(Stream *stream, blas::UpperLower uplo, uint64 n,
                           DeviceMemory<double> *ap) {
   return DoBlasInternal(wrap::cublasDspr2, stream,
                         true /* = pointer_mode_host */,
-                        CUDABlasUpperLower(uplo), n, &alpha, CUDAMemory(x),
-                        incx, CUDAMemory(y), incy, CUDAMemoryMutable(ap));
+                        CUDABlasUpperLower(uplo), n, &alpha, GpuMemory(x),
+                        incx, GpuMemory(y), incy, GpuMemoryMutable(ap));
 }
 
 bool CUDABlas::DoBlasSymv(Stream *stream, blas::UpperLower uplo, uint64 n,
@@ -1554,8 +1554,8 @@ bool CUDABlas::DoBlasSymv(Stream *stream, blas::UpperLower uplo, uint64 n,
                           DeviceMemory<float> *y, int incy) {
   return DoBlasInternal(wrap::cublasSsymv, stream,
                         true /* = pointer_mode_host */,
-                        CUDABlasUpperLower(uplo), n, &alpha, CUDAMemory(a), lda,
-                        CUDAMemory(x), incx, &beta, CUDAMemoryMutable(y), incy);
+                        CUDABlasUpperLower(uplo), n, &alpha, GpuMemory(a), lda,
+                        GpuMemory(x), incx, &beta, GpuMemoryMutable(y), incy);
 }
 
 bool CUDABlas::DoBlasSymv(Stream *stream, blas::UpperLower uplo, uint64 n,
@@ -1564,8 +1564,8 @@ bool CUDABlas::DoBlasSymv(Stream *stream, blas::UpperLower uplo, uint64 n,
                           DeviceMemory<double> *y, int incy) {
   return DoBlasInternal(wrap::cublasDsymv, stream,
                         true /* = pointer_mode_host */,
-                        CUDABlasUpperLower(uplo), n, &alpha, CUDAMemory(a), lda,
-                        CUDAMemory(x), incx, &beta, CUDAMemoryMutable(y), incy);
+                        CUDABlasUpperLower(uplo), n, &alpha, GpuMemory(a), lda,
+                        GpuMemory(x), incx, &beta, GpuMemoryMutable(y), incy);
 }
 
 bool CUDABlas::DoBlasSyr(Stream *stream, blas::UpperLower uplo, uint64 n,
@@ -1573,8 +1573,8 @@ bool CUDABlas::DoBlasSyr(Stream *stream, blas::UpperLower uplo, uint64 n,
                          DeviceMemory<float> *a, int lda) {
   return DoBlasInternal(wrap::cublasSsyr, stream,
                         true /* = pointer_mode_host */,
-                        CUDABlasUpperLower(uplo), n, &alpha, CUDAMemory(x),
-                        incx, CUDAMemoryMutable(a), lda);
+                        CUDABlasUpperLower(uplo), n, &alpha, GpuMemory(x),
+                        incx, GpuMemoryMutable(a), lda);
 }
 
 bool CUDABlas::DoBlasSyr(Stream *stream, blas::UpperLower uplo, uint64 n,
@@ -1582,8 +1582,8 @@ bool CUDABlas::DoBlasSyr(Stream *stream, blas::UpperLower uplo, uint64 n,
                          DeviceMemory<double> *a, int lda) {
   return DoBlasInternal(wrap::cublasDsyr, stream,
                         true /* = pointer_mode_host */,
-                        CUDABlasUpperLower(uplo), n, &alpha, CUDAMemory(x),
-                        incx, CUDAMemoryMutable(a), lda);
+                        CUDABlasUpperLower(uplo), n, &alpha, GpuMemory(x),
+                        incx, GpuMemoryMutable(a), lda);
 }
 
 bool CUDABlas::DoBlasSyr2(Stream *stream, blas::UpperLower uplo, uint64 n,
@@ -1592,8 +1592,8 @@ bool CUDABlas::DoBlasSyr2(Stream *stream, blas::UpperLower uplo, uint64 n,
                           DeviceMemory<float> *a, int lda) {
   return DoBlasInternal(wrap::cublasSsyr2, stream,
                         true /* = pointer_mode_host */,
-                        CUDABlasUpperLower(uplo), n, &alpha, CUDAMemory(x),
-                        incx, CUDAMemory(y), incy, CUDAMemoryMutable(a), lda);
+                        CUDABlasUpperLower(uplo), n, &alpha, GpuMemory(x),
+                        incx, GpuMemory(y), incy, GpuMemoryMutable(a), lda);
 }
 
 bool CUDABlas::DoBlasSyr2(Stream *stream, blas::UpperLower uplo, uint64 n,
@@ -1602,8 +1602,8 @@ bool CUDABlas::DoBlasSyr2(Stream *stream, blas::UpperLower uplo, uint64 n,
                           DeviceMemory<double> *a, int lda) {
   return DoBlasInternal(wrap::cublasDsyr2, stream,
                         true /* = pointer_mode_host */,
-                        CUDABlasUpperLower(uplo), n, &alpha, CUDAMemory(x),
-                        incx, CUDAMemory(y), incy, CUDAMemoryMutable(a), lda);
+                        CUDABlasUpperLower(uplo), n, &alpha, GpuMemory(x),
+                        incx, GpuMemory(y), incy, GpuMemoryMutable(a), lda);
 }
 
 bool CUDABlas::DoBlasTbmv(Stream *stream, blas::UpperLower uplo,
@@ -1613,8 +1613,8 @@ bool CUDABlas::DoBlasTbmv(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(wrap::cublasStbmv, stream,
                         true /* = pointer_mode_host */,
                         CUDABlasUpperLower(uplo), CUDABlasTranspose(trans),
-                        CUDABlasDiagonal(diag), n, k, CUDAMemory(a), lda,
-                        CUDAMemoryMutable(x), incx);
+                        CUDABlasDiagonal(diag), n, k, GpuMemory(a), lda,
+                        GpuMemoryMutable(x), incx);
 }
 
 bool CUDABlas::DoBlasTbmv(Stream *stream, blas::UpperLower uplo,
@@ -1624,8 +1624,8 @@ bool CUDABlas::DoBlasTbmv(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(wrap::cublasDtbmv, stream,
                         true /* = pointer_mode_host */,
                         CUDABlasUpperLower(uplo), CUDABlasTranspose(trans),
-                        CUDABlasDiagonal(diag), n, k, CUDAMemory(a), lda,
-                        CUDAMemoryMutable(x), incx);
+                        CUDABlasDiagonal(diag), n, k, GpuMemory(a), lda,
+                        GpuMemoryMutable(x), incx);
 }
 
 bool CUDABlas::DoBlasTbmv(Stream *stream, blas::UpperLower uplo,
@@ -1636,8 +1636,8 @@ bool CUDABlas::DoBlasTbmv(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(
       wrap::cublasCtbmv, stream, true /* = pointer_mode_host */,
       CUDABlasUpperLower(uplo), CUDABlasTranspose(trans),
-      CUDABlasDiagonal(diag), n, k, CUDAComplex(CUDAMemory(a)), lda,
-      CUDAComplex(CUDAMemoryMutable(x)), incx);
+      CUDABlasDiagonal(diag), n, k, GpuComplex(GpuMemory(a)), lda,
+      GpuComplex(GpuMemoryMutable(x)), incx);
 }
 
 bool CUDABlas::DoBlasTbmv(Stream *stream, blas::UpperLower uplo,
@@ -1648,8 +1648,8 @@ bool CUDABlas::DoBlasTbmv(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(
       wrap::cublasZtbmv, stream, true /* = pointer_mode_host */,
       CUDABlasUpperLower(uplo), CUDABlasTranspose(trans),
-      CUDABlasDiagonal(diag), n, k, CUDAComplex(CUDAMemory(a)), lda,
-      CUDAComplex(CUDAMemoryMutable(x)), incx);
+      CUDABlasDiagonal(diag), n, k, GpuComplex(GpuMemory(a)), lda,
+      GpuComplex(GpuMemoryMutable(x)), incx);
 }
 
 bool CUDABlas::DoBlasTbsv(Stream *stream, blas::UpperLower uplo,
@@ -1659,8 +1659,8 @@ bool CUDABlas::DoBlasTbsv(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(wrap::cublasStbsv, stream,
                         true /* = pointer_mode_host */,
                         CUDABlasUpperLower(uplo), CUDABlasTranspose(trans),
-                        CUDABlasDiagonal(diag), n, k, CUDAMemory(a), lda,
-                        CUDAMemoryMutable(x), incx);
+                        CUDABlasDiagonal(diag), n, k, GpuMemory(a), lda,
+                        GpuMemoryMutable(x), incx);
 }
 
 bool CUDABlas::DoBlasTbsv(Stream *stream, blas::UpperLower uplo,
@@ -1670,8 +1670,8 @@ bool CUDABlas::DoBlasTbsv(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(wrap::cublasDtbsv, stream,
                         true /* = pointer_mode_host */,
                         CUDABlasUpperLower(uplo), CUDABlasTranspose(trans),
-                        CUDABlasDiagonal(diag), n, k, CUDAMemory(a), lda,
-                        CUDAMemoryMutable(x), incx);
+                        CUDABlasDiagonal(diag), n, k, GpuMemory(a), lda,
+                        GpuMemoryMutable(x), incx);
 }
 
 bool CUDABlas::DoBlasTbsv(Stream *stream, blas::UpperLower uplo,
@@ -1682,8 +1682,8 @@ bool CUDABlas::DoBlasTbsv(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(
       wrap::cublasCtbsv, stream, true /* = pointer_mode_host */,
       CUDABlasUpperLower(uplo), CUDABlasTranspose(trans),
-      CUDABlasDiagonal(diag), n, k, CUDAComplex(CUDAMemory(a)), lda,
-      CUDAComplex(CUDAMemoryMutable(x)), incx);
+      CUDABlasDiagonal(diag), n, k, GpuComplex(GpuMemory(a)), lda,
+      GpuComplex(GpuMemoryMutable(x)), incx);
 }
 
 bool CUDABlas::DoBlasTbsv(Stream *stream, blas::UpperLower uplo,
@@ -1694,8 +1694,8 @@ bool CUDABlas::DoBlasTbsv(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(
       wrap::cublasZtbsv, stream, true /* = pointer_mode_host */,
       CUDABlasUpperLower(uplo), CUDABlasTranspose(trans),
-      CUDABlasDiagonal(diag), n, k, CUDAComplex(CUDAMemory(a)), lda,
-      CUDAComplex(CUDAMemoryMutable(x)), incx);
+      CUDABlasDiagonal(diag), n, k, GpuComplex(GpuMemory(a)), lda,
+      GpuComplex(GpuMemoryMutable(x)), incx);
 }
 
 bool CUDABlas::DoBlasTpmv(Stream *stream, blas::UpperLower uplo,
@@ -1705,7 +1705,7 @@ bool CUDABlas::DoBlasTpmv(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(
       wrap::cublasStpmv, stream, true /* = pointer_mode_host */,
       CUDABlasUpperLower(uplo), CUDABlasTranspose(trans),
-      CUDABlasDiagonal(diag), n, CUDAMemory(ap), CUDAMemoryMutable(x), incx);
+      CUDABlasDiagonal(diag), n, GpuMemory(ap), GpuMemoryMutable(x), incx);
 }
 
 bool CUDABlas::DoBlasTpmv(Stream *stream, blas::UpperLower uplo,
@@ -1715,7 +1715,7 @@ bool CUDABlas::DoBlasTpmv(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(
       wrap::cublasDtpmv, stream, true /* = pointer_mode_host */,
       CUDABlasUpperLower(uplo), CUDABlasTranspose(trans),
-      CUDABlasDiagonal(diag), n, CUDAMemory(ap), CUDAMemoryMutable(x), incx);
+      CUDABlasDiagonal(diag), n, GpuMemory(ap), GpuMemoryMutable(x), incx);
 }
 
 bool CUDABlas::DoBlasTpmv(Stream *stream, blas::UpperLower uplo,
@@ -1725,8 +1725,8 @@ bool CUDABlas::DoBlasTpmv(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(wrap::cublasCtpmv, stream,
                         true /* = pointer_mode_host */,
                         CUDABlasUpperLower(uplo), CUDABlasTranspose(trans),
-                        CUDABlasDiagonal(diag), n, CUDAComplex(CUDAMemory(ap)),
-                        CUDAComplex(CUDAMemoryMutable(x)), incx);
+                        CUDABlasDiagonal(diag), n, GpuComplex(GpuMemory(ap)),
+                        GpuComplex(GpuMemoryMutable(x)), incx);
 }
 
 bool CUDABlas::DoBlasTpmv(Stream *stream, blas::UpperLower uplo,
@@ -1736,8 +1736,8 @@ bool CUDABlas::DoBlasTpmv(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(wrap::cublasZtpmv, stream,
                         true /* = pointer_mode_host */,
                         CUDABlasUpperLower(uplo), CUDABlasTranspose(trans),
-                        CUDABlasDiagonal(diag), n, CUDAComplex(CUDAMemory(ap)),
-                        CUDAComplex(CUDAMemoryMutable(x)), incx);
+                        CUDABlasDiagonal(diag), n, GpuComplex(GpuMemory(ap)),
+                        GpuComplex(GpuMemoryMutable(x)), incx);
 }
 
 bool CUDABlas::DoBlasTpsv(Stream *stream, blas::UpperLower uplo,
@@ -1747,7 +1747,7 @@ bool CUDABlas::DoBlasTpsv(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(
       wrap::cublasStpsv, stream, true /* = pointer_mode_host */,
       CUDABlasUpperLower(uplo), CUDABlasTranspose(trans),
-      CUDABlasDiagonal(diag), n, CUDAMemory(ap), CUDAMemoryMutable(x), incx);
+      CUDABlasDiagonal(diag), n, GpuMemory(ap), GpuMemoryMutable(x), incx);
 }
 
 bool CUDABlas::DoBlasTpsv(Stream *stream, blas::UpperLower uplo,
@@ -1757,7 +1757,7 @@ bool CUDABlas::DoBlasTpsv(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(
       wrap::cublasDtpsv, stream, true /* = pointer_mode_host */,
       CUDABlasUpperLower(uplo), CUDABlasTranspose(trans),
-      CUDABlasDiagonal(diag), n, CUDAMemory(ap), CUDAMemoryMutable(x), incx);
+      CUDABlasDiagonal(diag), n, GpuMemory(ap), GpuMemoryMutable(x), incx);
 }
 
 bool CUDABlas::DoBlasTpsv(Stream *stream, blas::UpperLower uplo,
@@ -1767,8 +1767,8 @@ bool CUDABlas::DoBlasTpsv(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(wrap::cublasCtpsv, stream,
                         true /* = pointer_mode_host */,
                         CUDABlasUpperLower(uplo), CUDABlasTranspose(trans),
-                        CUDABlasDiagonal(diag), n, CUDAComplex(CUDAMemory(ap)),
-                        CUDAComplex(CUDAMemoryMutable(x)), incx);
+                        CUDABlasDiagonal(diag), n, GpuComplex(GpuMemory(ap)),
+                        GpuComplex(GpuMemoryMutable(x)), incx);
 }
 
 bool CUDABlas::DoBlasTpsv(Stream *stream, blas::UpperLower uplo,
@@ -1778,8 +1778,8 @@ bool CUDABlas::DoBlasTpsv(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(wrap::cublasZtpsv, stream,
                         true /* = pointer_mode_host */,
                         CUDABlasUpperLower(uplo), CUDABlasTranspose(trans),
-                        CUDABlasDiagonal(diag), n, CUDAComplex(CUDAMemory(ap)),
-                        CUDAComplex(CUDAMemoryMutable(x)), incx);
+                        CUDABlasDiagonal(diag), n, GpuComplex(GpuMemory(ap)),
+                        GpuComplex(GpuMemoryMutable(x)), incx);
 }
 
 bool CUDABlas::DoBlasTrmv(Stream *stream, blas::UpperLower uplo,
@@ -1789,8 +1789,8 @@ bool CUDABlas::DoBlasTrmv(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(wrap::cublasStrmv, stream,
                         true /* = pointer_mode_host */,
                         CUDABlasUpperLower(uplo), CUDABlasTranspose(trans),
-                        CUDABlasDiagonal(diag), n, CUDAMemory(a), lda,
-                        CUDAMemoryMutable(x), incx);
+                        CUDABlasDiagonal(diag), n, GpuMemory(a), lda,
+                        GpuMemoryMutable(x), incx);
 }
 
 bool CUDABlas::DoBlasTrmv(Stream *stream, blas::UpperLower uplo,
@@ -1800,8 +1800,8 @@ bool CUDABlas::DoBlasTrmv(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(wrap::cublasDtrmv, stream,
                         true /* = pointer_mode_host */,
                         CUDABlasUpperLower(uplo), CUDABlasTranspose(trans),
-                        CUDABlasDiagonal(diag), n, CUDAMemory(a), lda,
-                        CUDAMemoryMutable(x), incx);
+                        CUDABlasDiagonal(diag), n, GpuMemory(a), lda,
+                        GpuMemoryMutable(x), incx);
 }
 
 bool CUDABlas::DoBlasTrmv(Stream *stream, blas::UpperLower uplo,
@@ -1811,8 +1811,8 @@ bool CUDABlas::DoBlasTrmv(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(wrap::cublasCtrmv, stream,
                         true /* = pointer_mode_host */,
                         CUDABlasUpperLower(uplo), CUDABlasTranspose(trans),
-                        CUDABlasDiagonal(diag), n, CUDAComplex(CUDAMemory(a)),
-                        lda, CUDAComplex(CUDAMemoryMutable(x)), incx);
+                        CUDABlasDiagonal(diag), n, GpuComplex(GpuMemory(a)),
+                        lda, GpuComplex(GpuMemoryMutable(x)), incx);
 }
 
 bool CUDABlas::DoBlasTrmv(Stream *stream, blas::UpperLower uplo,
@@ -1822,8 +1822,8 @@ bool CUDABlas::DoBlasTrmv(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(wrap::cublasZtrmv, stream,
                         true /* = pointer_mode_host */,
                         CUDABlasUpperLower(uplo), CUDABlasTranspose(trans),
-                        CUDABlasDiagonal(diag), n, CUDAComplex(CUDAMemory(a)),
-                        lda, CUDAComplex(CUDAMemoryMutable(x)), incx);
+                        CUDABlasDiagonal(diag), n, GpuComplex(GpuMemory(a)),
+                        lda, GpuComplex(GpuMemoryMutable(x)), incx);
 }
 
 bool CUDABlas::DoBlasTrsv(Stream *stream, blas::UpperLower uplo,
@@ -1833,8 +1833,8 @@ bool CUDABlas::DoBlasTrsv(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(wrap::cublasStrsv, stream,
                         true /* = pointer_mode_host */,
                         CUDABlasUpperLower(uplo), CUDABlasTranspose(trans),
-                        CUDABlasDiagonal(diag), n, CUDAMemory(a), lda,
-                        CUDAMemoryMutable(x), incx);
+                        CUDABlasDiagonal(diag), n, GpuMemory(a), lda,
+                        GpuMemoryMutable(x), incx);
 }
 
 bool CUDABlas::DoBlasTrsv(Stream *stream, blas::UpperLower uplo,
@@ -1844,8 +1844,8 @@ bool CUDABlas::DoBlasTrsv(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(wrap::cublasDtrsv, stream,
                         true /* = pointer_mode_host */,
                         CUDABlasUpperLower(uplo), CUDABlasTranspose(trans),
-                        CUDABlasDiagonal(diag), n, CUDAMemory(a), lda,
-                        CUDAMemoryMutable(x), incx);
+                        CUDABlasDiagonal(diag), n, GpuMemory(a), lda,
+                        GpuMemoryMutable(x), incx);
 }
 
 bool CUDABlas::DoBlasTrsv(Stream *stream, blas::UpperLower uplo,
@@ -1855,8 +1855,8 @@ bool CUDABlas::DoBlasTrsv(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(wrap::cublasCtrsv, stream,
                         true /* = pointer_mode_host */,
                         CUDABlasUpperLower(uplo), CUDABlasTranspose(trans),
-                        CUDABlasDiagonal(diag), n, CUDAComplex(CUDAMemory(a)),
-                        lda, CUDAComplex(CUDAMemoryMutable(x)), incx);
+                        CUDABlasDiagonal(diag), n, GpuComplex(GpuMemory(a)),
+                        lda, GpuComplex(GpuMemoryMutable(x)), incx);
 }
 
 bool CUDABlas::DoBlasTrsv(Stream *stream, blas::UpperLower uplo,
@@ -1866,8 +1866,8 @@ bool CUDABlas::DoBlasTrsv(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(wrap::cublasZtrsv, stream,
                         true /* = pointer_mode_host */,
                         CUDABlasUpperLower(uplo), CUDABlasTranspose(trans),
-                        CUDABlasDiagonal(diag), n, CUDAComplex(CUDAMemory(a)),
-                        lda, CUDAComplex(CUDAMemoryMutable(x)), incx);
+                        CUDABlasDiagonal(diag), n, GpuComplex(GpuMemory(a)),
+                        lda, GpuComplex(GpuMemoryMutable(x)), incx);
 }
 
 bool CUDABlas::DoBlasGemm(
@@ -1921,9 +1921,9 @@ bool CUDABlas::DoBlasGemm(
   return DoBlasInternalImpl(
       wrap::cublasSgemmEx, stream, true /* = pointer_mode_host */,
       true /* = err_on_failure= */, use_tensor_ops, CUDABlasTranspose(transa),
-      CUDABlasTranspose(transb), m, n, k, &alpha, CUDAMemory(a),
-      SE_CUDA_DATA_HALF, lda, CUDAMemory(b), SE_CUDA_DATA_HALF, ldb, &beta,
-      CUDAMemoryMutable(c), SE_CUDA_DATA_HALF, ldc);
+      CUDABlasTranspose(transb), m, n, k, &alpha, GpuMemory(a),
+      SE_CUDA_DATA_HALF, lda, GpuMemory(b), SE_CUDA_DATA_HALF, ldb, &beta,
+      GpuMemoryMutable(c), SE_CUDA_DATA_HALF, ldc);
 
 #else
   LOG(ERROR) << "fp16 sgemm is not implemented in this cuBLAS version "
@@ -1968,7 +1968,7 @@ bool CUDABlas::DoBlasGemm(Stream *stream, blas::Transpose transa,
   return DoBlasInternal(
       wrap::cublasSgemm, stream, true /* = pointer_mode_host */,
       CUDABlasTranspose(transa), CUDABlasTranspose(transb), m, n, k, &alpha,
-      CUDAMemory(a), lda, CUDAMemory(b), ldb, &beta, CUDAMemoryMutable(c), ldc);
+      GpuMemory(a), lda, GpuMemory(b), ldb, &beta, GpuMemoryMutable(c), ldc);
 }
 
 bool CUDABlas::DoBlasGemm(Stream *stream, blas::Transpose transa,
@@ -1979,7 +1979,7 @@ bool CUDABlas::DoBlasGemm(Stream *stream, blas::Transpose transa,
   return DoBlasInternal(
       wrap::cublasDgemm, stream, true /* = pointer_mode_host */,
       CUDABlasTranspose(transa), CUDABlasTranspose(transb), m, n, k, &alpha,
-      CUDAMemory(a), lda, CUDAMemory(b), ldb, &beta, CUDAMemoryMutable(c), ldc);
+      GpuMemory(a), lda, GpuMemory(b), ldb, &beta, GpuMemoryMutable(c), ldc);
 }
 
 bool CUDABlas::DoBlasGemm(Stream *stream, blas::Transpose transa,
@@ -1992,9 +1992,9 @@ bool CUDABlas::DoBlasGemm(Stream *stream, blas::Transpose transa,
   return DoBlasInternal(
       wrap::cublasCgemm, stream, true /* = pointer_mode_host */,
       CUDABlasTranspose(transa), CUDABlasTranspose(transb), m, n, k,
-      CUDAComplex(&alpha), CUDAComplex(CUDAMemory(a)), lda,
-      CUDAComplex(CUDAMemory(b)), ldb, CUDAComplex(&beta),
-      CUDAComplex(CUDAMemoryMutable(c)), ldc);
+      GpuComplex(&alpha), GpuComplex(GpuMemory(a)), lda,
+      GpuComplex(GpuMemory(b)), ldb, GpuComplex(&beta),
+      GpuComplex(GpuMemoryMutable(c)), ldc);
 }
 
 bool CUDABlas::DoBlasGemm(Stream *stream, blas::Transpose transa,
@@ -2007,9 +2007,9 @@ bool CUDABlas::DoBlasGemm(Stream *stream, blas::Transpose transa,
   return DoBlasInternal(
       wrap::cublasZgemm, stream, true /* = pointer_mode_host */,
       CUDABlasTranspose(transa), CUDABlasTranspose(transb), m, n, k,
-      CUDAComplex(&alpha), CUDAComplex(CUDAMemory(a)), lda,
-      CUDAComplex(CUDAMemory(b)), ldb, CUDAComplex(&beta),
-      CUDAComplex(CUDAMemoryMutable(c)), ldc);
+      GpuComplex(&alpha), GpuComplex(GpuMemory(a)), lda,
+      GpuComplex(GpuMemory(b)), ldb, GpuComplex(&beta),
+      GpuComplex(GpuMemoryMutable(c)), ldc);
 }
 
 bool CUDABlas::DoBlasGemvWithProfiling(
@@ -2116,10 +2116,10 @@ bool CUDABlas::DoBlasGemvWithProfilingImpl(
     const DeviceMemory<T> &a, int lda, const DeviceMemory<T> &x, int incx,
     const T &beta, DeviceMemory<T> *y, int incy,
     blas::ProfileResult *output_profile_result) {
-  std::unique_ptr<CUDATimer, TimerDeleter> timer;
+  std::unique_ptr<GpuTimer, GpuTimerDeleter> timer;
   if (output_profile_result != nullptr) {
-    timer.reset(new CUDATimer(parent_));
-    if (!timer->Init() || !timer->Start(AsCUDAStream(stream))) {
+    timer.reset(new GpuTimer(parent_));
+    if (!timer->Init() || !timer->Start(AsGpuStream(stream))) {
       return false;
     }
   }
@@ -2129,9 +2129,9 @@ bool CUDABlas::DoBlasGemvWithProfilingImpl(
       DoBlasGemv(stream, trans, m, n, alpha, a, lda, x, incx, beta, y, incy);
 
   if (timer != nullptr && result) {
-    // CUDATimer will CHECK-fail if we Stop() it while the stream is in an error
+    // GpuTimer will CHECK-fail if we Stop() it while the stream is in an error
     // state.
-    if (!timer->Stop(AsCUDAStream(stream))) {
+    if (!timer->Stop(AsGpuStream(stream))) {
       return false;
     }
     output_profile_result->set_is_valid(true);
@@ -2148,10 +2148,10 @@ bool CUDABlas::DoBlasGemmWithProfilingImpl(
     uint64 n, uint64 k, const ParamType &alpha, const DeviceMemory<T> &a,
     int lda, const DeviceMemory<T> &b, int ldb, const ParamType &beta,
     DeviceMemory<T> *c, int ldc, blas::ProfileResult *output_profile_result) {
-  std::unique_ptr<CUDATimer, TimerDeleter> timer;
+  std::unique_ptr<GpuTimer, GpuTimerDeleter> timer;
   if (output_profile_result != nullptr) {
-    timer.reset(new CUDATimer(parent_));
-    if (!timer->Init() || !timer->Start(AsCUDAStream(stream))) {
+    timer.reset(new GpuTimer(parent_));
+    if (!timer->Init() || !timer->Start(AsGpuStream(stream))) {
       return false;
     }
   }
@@ -2161,9 +2161,9 @@ bool CUDABlas::DoBlasGemmWithProfilingImpl(
                            ldb, beta, c, ldc);
 
   if (timer != nullptr && result) {
-    // CUDATimer will CHECK-fail if we Stop() it while the stream is in an error
+    // GpuTimer will CHECK-fail if we Stop() it while the stream is in an error
     // state.
-    if (!timer->Stop(AsCUDAStream(stream))) {
+    if (!timer->Stop(AsGpuStream(stream))) {
       return false;
     }
     output_profile_result->set_is_valid(true);
@@ -2238,13 +2238,13 @@ bool CUDABlas::DoBlasGemmWithAlgorithmImpl(
     return false;
   }
 
-  std::unique_ptr<CUDATimer, TimerDeleter> timer;
+  std::unique_ptr<GpuTimer, GpuTimerDeleter> timer;
   if (output_profile_result != nullptr) {
-    timer.reset(new CUDATimer(parent_));
-    if (!timer->Init() || !timer->Start(AsCUDAStream(stream))) {
+    timer.reset(new GpuTimer(parent_));
+    if (!timer->Init() || !timer->Start(AsGpuStream(stream))) {
       VLOG(2) << "DoBlasGemmWithAlgorithm returning false because "
                  "output_profile_result was given, but we were unable to "
-                 "create a CUDATimer.";
+                 "create a GpuTimer.";
       return false;
     }
   }
@@ -2270,19 +2270,19 @@ bool CUDABlas::DoBlasGemmWithAlgorithmImpl(
   bool result = DoBlasInternalFailureOK(
       wrap::cublasGemmEx, stream, /* pointer_mode_host = */ !alpha.is_pointer(),
       CUDABlasTranspose(transa), CUDABlasTranspose(transb), m, n, k,
-      alpha.is_pointer() ? CUDAMemory(alpha.pointer()) : &alpha.value(),
-      CUDAMemory(a), cuda_in_type, lda, CUDAMemory(b), cuda_in_type, ldb,
-      beta.is_pointer() ? CUDAMemory(beta.pointer()) : &beta.value(),
-      CUDAMemoryMutable(c), CUDADataType<OutT>::type, ldc,
+      alpha.is_pointer() ? GpuMemory(alpha.pointer()) : &alpha.value(),
+      GpuMemory(a), cuda_in_type, lda, GpuMemory(b), cuda_in_type, ldb,
+      beta.is_pointer() ? GpuMemory(beta.pointer()) : &beta.value(),
+      GpuMemoryMutable(c), CUDADataType<OutT>::type, ldc,
       CUDAComputationType(computation_type),
       static_cast<cublasGemmAlgo_t>(algorithm));
 
   if (timer != nullptr && result) {
-    // CUDATimer will CHECK-fail if we Stop() it while the stream is in an error
+    // GpuTimer will CHECK-fail if we Stop() it while the stream is in an error
     // state.
-    if (!timer->Stop(AsCUDAStream(stream))) {
+    if (!timer->Stop(AsGpuStream(stream))) {
       VLOG(2) << "DoBlasGemmWithAlgorithm returning false; unable to stop "
-                 "CUDATimer.";
+                 "GpuTimer.";
       return false;
     }
     output_profile_result->set_is_valid(true);
@@ -2470,7 +2470,7 @@ port::Status CUDABlas::DoBlasGemmBatchedInternal(
     c_raw_ptrs.push_back(static_cast<T *>(c_ptrs_to_wrappers[i]->opaque()));
   }
 
-  typedef typename HalfAsFloat<typename CUDAComplexT<T>::type>::type CUDA_T;
+  typedef typename HalfAsFloat<typename GpuComplexT<T>::type>::type CUDA_T;
 
   const size_t size = batch_count * sizeof(CUDA_T *);
 
@@ -2535,11 +2535,11 @@ port::Status CUDABlas::DoBlasGemmBatchedInternal(
     cudaDataType_t compute_type =
         (data_type == CUDA_R_16F ? CUDA_R_32F : data_type);
     const void **a_void_ptrs = reinterpret_cast<const void **>(
-        const_cast<const CUDA_T **>(CUDAMemory(a)));
+        const_cast<const CUDA_T **>(GpuMemory(a)));
     const void **b_void_ptrs = reinterpret_cast<const void **>(
-        const_cast<const CUDA_T **>(CUDAMemory(b)));
+        const_cast<const CUDA_T **>(GpuMemory(b)));
     void **c_void_ptrs =
-        reinterpret_cast<void **>(const_cast<CUDA_T **>(CUDAMemory(c)));
+        reinterpret_cast<void **>(const_cast<CUDA_T **>(GpuMemory(c)));
     bool ok;
     ok = DoBlasInternalImpl(
         wrap::cublasGemmBatchedEx, stream, true /* = pointer_mode_host */,
@@ -2559,9 +2559,9 @@ port::Status CUDABlas::DoBlasGemmBatchedInternal(
     bool ok = DoBlasInternal(
         cublas_func, stream, true /* = pointer_mode_host */,
         CUDABlasTranspose(transa), CUDABlasTranspose(transb), m, n, k,
-        CUDAComplex(&alpha), const_cast<const CUDA_T **>(CUDAMemory(a)), lda,
-        const_cast<const CUDA_T **>(CUDAMemory(b)), ldb, CUDAComplex(&beta),
-        const_cast<CUDA_T **>(CUDAMemory(c)), ldc, batch_count);
+        GpuComplex(&alpha), const_cast<const CUDA_T **>(GpuMemory(a)), lda,
+        const_cast<const CUDA_T **>(GpuMemory(b)), ldb, GpuComplex(&beta),
+        const_cast<CUDA_T **>(GpuMemory(c)), ldc, batch_count);
     if (ok) {
       return port::Status::OK();
     }
@@ -2693,8 +2693,8 @@ bool CUDABlas::DoBlasGemmStridedBatched(
           wrap::cublasGemmStridedBatchedEx, stream,
           true /* = pointer_mode_host */, true /* = err_on_failure */,
           use_tensor_ops, CUDABlasTranspose(transa), CUDABlasTranspose(transb),
-          m, n, k, &alpha, CUDAMemory(a), CUDA_R_16F, lda, stride_a,
-          CUDAMemory(b), CUDA_R_16F, ldb, stride_b, &beta, CUDAMemoryMutable(c),
+          m, n, k, &alpha, GpuMemory(a), CUDA_R_16F, lda, stride_a,
+          GpuMemory(b), CUDA_R_16F, ldb, stride_b, &beta, GpuMemoryMutable(c),
           CUDA_R_16F, ldc, stride_c, batch_count, CUDA_R_32F, algo);
       if (ok) {
         return true;
@@ -2708,11 +2708,11 @@ bool CUDABlas::DoBlasGemmStridedBatched(
   // Either CUDA_VERSION < 9.1 or SM < 5.0. Fall back to a loop.
   for (int batch = 0; batch < batch_count; ++batch) {
     const auto *a_matrix =
-        reinterpret_cast<const __half *>(CUDAMemory(a) + batch * stride_a);
+        reinterpret_cast<const __half *>(GpuMemory(a) + batch * stride_a);
     const auto *b_matrix =
-        reinterpret_cast<const __half *>(CUDAMemory(b) + batch * stride_b);
+        reinterpret_cast<const __half *>(GpuMemory(b) + batch * stride_b);
     auto *c_matrix =
-        reinterpret_cast<__half *>(CUDAMemoryMutable(c) + batch * stride_c);
+        reinterpret_cast<__half *>(GpuMemoryMutable(c) + batch * stride_c);
     bool ok = DoBlasInternalImpl(
         wrap::cublasSgemmEx, stream, true /* = pointer_mode_host */,
         true /* = err_on_failure= */, use_tensor_ops, CUDABlasTranspose(transa),
@@ -2736,8 +2736,8 @@ bool CUDABlas::DoBlasGemmStridedBatched(
   return DoBlasInternal(
       wrap::cublasSgemmStridedBatched, stream, true /* = pointer_mode_host */,
       CUDABlasTranspose(transa), CUDABlasTranspose(transb), m, n, k, &alpha,
-      CUDAMemory(a), lda, stride_a, CUDAMemory(b), ldb, stride_b, &beta,
-      CUDAMemoryMutable(c), ldc, stride_c, batch_count);
+      GpuMemory(a), lda, stride_a, GpuMemory(b), ldb, stride_b, &beta,
+      GpuMemoryMutable(c), ldc, stride_c, batch_count);
 }
 
 bool CUDABlas::DoBlasGemmStridedBatched(
@@ -2749,8 +2749,8 @@ bool CUDABlas::DoBlasGemmStridedBatched(
   return DoBlasInternal(
       wrap::cublasDgemmStridedBatched, stream, true /* = pointer_mode_host */,
       CUDABlasTranspose(transa), CUDABlasTranspose(transb), m, n, k, &alpha,
-      CUDAMemory(a), lda, stride_a, CUDAMemory(b), ldb, stride_b, &beta,
-      CUDAMemoryMutable(c), ldc, stride_c, batch_count);
+      GpuMemory(a), lda, stride_a, GpuMemory(b), ldb, stride_b, &beta,
+      GpuMemoryMutable(c), ldc, stride_c, batch_count);
 }
 
 bool CUDABlas::DoBlasGemmStridedBatched(
@@ -2763,9 +2763,9 @@ bool CUDABlas::DoBlasGemmStridedBatched(
   return DoBlasInternal(
       wrap::cublasCgemmStridedBatched, stream, true /* = pointer_mode_host */,
       CUDABlasTranspose(transa), CUDABlasTranspose(transb), m, n, k,
-      CUDAComplex(&alpha), CUDAComplex(CUDAMemory(a)), lda, stride_a,
-      CUDAComplex(CUDAMemory(b)), ldb, stride_b, CUDAComplex(&beta),
-      CUDAComplex(CUDAMemoryMutable(c)), ldc, stride_c, batch_count);
+      GpuComplex(&alpha), GpuComplex(GpuMemory(a)), lda, stride_a,
+      GpuComplex(GpuMemory(b)), ldb, stride_b, GpuComplex(&beta),
+      GpuComplex(GpuMemoryMutable(c)), ldc, stride_c, batch_count);
 }
 
 bool CUDABlas::DoBlasGemmStridedBatched(
@@ -2778,9 +2778,9 @@ bool CUDABlas::DoBlasGemmStridedBatched(
   return DoBlasInternal(
       wrap::cublasZgemmStridedBatched, stream, true /* = pointer_mode_host */,
       CUDABlasTranspose(transa), CUDABlasTranspose(transb), m, n, k,
-      CUDAComplex(&alpha), CUDAComplex(CUDAMemory(a)), lda, stride_a,
-      CUDAComplex(CUDAMemory(b)), ldb, stride_b, CUDAComplex(&beta),
-      CUDAComplex(CUDAMemoryMutable(c)), ldc, stride_c, batch_count);
+      GpuComplex(&alpha), GpuComplex(GpuMemory(a)), lda, stride_a,
+      GpuComplex(GpuMemory(b)), ldb, stride_b, GpuComplex(&beta),
+      GpuComplex(GpuMemoryMutable(c)), ldc, stride_c, batch_count);
 }
 
 bool CUDABlas::DoBlasHemm(Stream *stream, blas::Side side,
@@ -2792,9 +2792,9 @@ bool CUDABlas::DoBlasHemm(Stream *stream, blas::Side side,
                           DeviceMemory<std::complex<float>> *c, int ldc) {
   return DoBlasInternal(
       wrap::cublasChemm, stream, true /* = pointer_mode_host */,
-      CUDABlasSide(side), CUDABlasUpperLower(uplo), m, n, CUDAComplex(&alpha),
-      CUDAComplex(CUDAMemory(a)), lda, CUDAComplex(CUDAMemory(b)), ldb,
-      CUDAComplex(&beta), CUDAComplex(CUDAMemoryMutable(c)), ldc);
+      CUDABlasSide(side), CUDABlasUpperLower(uplo), m, n, GpuComplex(&alpha),
+      GpuComplex(GpuMemory(a)), lda, GpuComplex(GpuMemory(b)), ldb,
+      GpuComplex(&beta), GpuComplex(GpuMemoryMutable(c)), ldc);
 }
 
 bool CUDABlas::DoBlasHemm(Stream *stream, blas::Side side,
@@ -2806,9 +2806,9 @@ bool CUDABlas::DoBlasHemm(Stream *stream, blas::Side side,
                           DeviceMemory<std::complex<double>> *c, int ldc) {
   return DoBlasInternal(
       wrap::cublasZhemm, stream, true /* = pointer_mode_host */,
-      CUDABlasSide(side), CUDABlasUpperLower(uplo), m, n, CUDAComplex(&alpha),
-      CUDAComplex(CUDAMemory(a)), lda, CUDAComplex(CUDAMemory(b)), ldb,
-      CUDAComplex(&beta), CUDAComplex(CUDAMemoryMutable(c)), ldc);
+      CUDABlasSide(side), CUDABlasUpperLower(uplo), m, n, GpuComplex(&alpha),
+      GpuComplex(GpuMemory(a)), lda, GpuComplex(GpuMemory(b)), ldb,
+      GpuComplex(&beta), GpuComplex(GpuMemoryMutable(c)), ldc);
 }
 
 bool CUDABlas::DoBlasHerk(Stream *stream, blas::UpperLower uplo,
@@ -2820,8 +2820,8 @@ bool CUDABlas::DoBlasHerk(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(wrap::cublasCherk, stream,
                         true /* = pointer_mode_host */,
                         CUDABlasUpperLower(uplo), CUDABlasTranspose(trans), n,
-                        k, CUDAComplex(&alpha), CUDAComplex(CUDAMemory(a)), lda,
-                        &beta, CUDAComplex(CUDAMemoryMutable(c)), ldc);
+                        k, GpuComplex(&alpha), GpuComplex(GpuMemory(a)), lda,
+                        &beta, GpuComplex(GpuMemoryMutable(c)), ldc);
 }
 
 bool CUDABlas::DoBlasHerk(Stream *stream, blas::UpperLower uplo,
@@ -2833,8 +2833,8 @@ bool CUDABlas::DoBlasHerk(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(wrap::cublasZherk, stream,
                         true /* = pointer_mode_host */,
                         CUDABlasUpperLower(uplo), CUDABlasTranspose(trans), n,
-                        k, CUDAComplex(&alpha), CUDAComplex(CUDAMemory(a)), lda,
-                        &beta, CUDAComplex(CUDAMemoryMutable(c)), ldc);
+                        k, GpuComplex(&alpha), GpuComplex(GpuMemory(a)), lda,
+                        &beta, GpuComplex(GpuMemoryMutable(c)), ldc);
 }
 
 bool CUDABlas::DoBlasHer2k(Stream *stream, blas::UpperLower uplo,
@@ -2847,9 +2847,9 @@ bool CUDABlas::DoBlasHer2k(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(wrap::cublasCher2k, stream,
                         true /* = pointer_mode_host */,
                         CUDABlasUpperLower(uplo), CUDABlasTranspose(trans), n,
-                        k, CUDAComplex(&alpha), CUDAComplex(CUDAMemory(a)), lda,
-                        CUDAComplex(CUDAMemory(b)), ldb, &beta,
-                        CUDAComplex(CUDAMemoryMutable(c)), ldc);
+                        k, GpuComplex(&alpha), GpuComplex(GpuMemory(a)), lda,
+                        GpuComplex(GpuMemory(b)), ldb, &beta,
+                        GpuComplex(GpuMemoryMutable(c)), ldc);
 }
 
 bool CUDABlas::DoBlasHer2k(Stream *stream, blas::UpperLower uplo,
@@ -2862,9 +2862,9 @@ bool CUDABlas::DoBlasHer2k(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(wrap::cublasZher2k, stream,
                         true /* = pointer_mode_host */,
                         CUDABlasUpperLower(uplo), CUDABlasTranspose(trans), n,
-                        k, CUDAComplex(&alpha), CUDAComplex(CUDAMemory(a)), lda,
-                        CUDAComplex(CUDAMemory(b)), ldb, &beta,
-                        CUDAComplex(CUDAMemoryMutable(c)), ldc);
+                        k, GpuComplex(&alpha), GpuComplex(GpuMemory(a)), lda,
+                        GpuComplex(GpuMemory(b)), ldb, &beta,
+                        GpuComplex(GpuMemoryMutable(c)), ldc);
 }
 
 bool CUDABlas::DoBlasSymm(Stream *stream, blas::Side side,
@@ -2874,8 +2874,8 @@ bool CUDABlas::DoBlasSymm(Stream *stream, blas::Side side,
                           DeviceMemory<float> *c, int ldc) {
   return DoBlasInternal(
       wrap::cublasSsymm, stream, true /* = pointer_mode_host */,
-      CUDABlasSide(side), CUDABlasUpperLower(uplo), m, n, &alpha, CUDAMemory(a),
-      lda, CUDAMemory(b), ldb, &beta, CUDAMemoryMutable(c), ldc);
+      CUDABlasSide(side), CUDABlasUpperLower(uplo), m, n, &alpha, GpuMemory(a),
+      lda, GpuMemory(b), ldb, &beta, GpuMemoryMutable(c), ldc);
 }
 
 bool CUDABlas::DoBlasSymm(Stream *stream, blas::Side side,
@@ -2885,8 +2885,8 @@ bool CUDABlas::DoBlasSymm(Stream *stream, blas::Side side,
                           DeviceMemory<double> *c, int ldc) {
   return DoBlasInternal(
       wrap::cublasDsymm, stream, true /* = pointer_mode_host */,
-      CUDABlasSide(side), CUDABlasUpperLower(uplo), m, n, &alpha, CUDAMemory(a),
-      lda, CUDAMemory(b), ldb, &beta, CUDAMemoryMutable(c), ldc);
+      CUDABlasSide(side), CUDABlasUpperLower(uplo), m, n, &alpha, GpuMemory(a),
+      lda, GpuMemory(b), ldb, &beta, GpuMemoryMutable(c), ldc);
 }
 
 bool CUDABlas::DoBlasSymm(Stream *stream, blas::Side side,
@@ -2898,9 +2898,9 @@ bool CUDABlas::DoBlasSymm(Stream *stream, blas::Side side,
                           DeviceMemory<std::complex<float>> *c, int ldc) {
   return DoBlasInternal(
       wrap::cublasCsymm, stream, true /* = pointer_mode_host */,
-      CUDABlasSide(side), CUDABlasUpperLower(uplo), m, n, CUDAComplex(&alpha),
-      CUDAComplex(CUDAMemory(a)), lda, CUDAComplex(CUDAMemory(b)), ldb,
-      CUDAComplex(&beta), CUDAComplex(CUDAMemoryMutable(c)), ldc);
+      CUDABlasSide(side), CUDABlasUpperLower(uplo), m, n, GpuComplex(&alpha),
+      GpuComplex(GpuMemory(a)), lda, GpuComplex(GpuMemory(b)), ldb,
+      GpuComplex(&beta), GpuComplex(GpuMemoryMutable(c)), ldc);
 }
 
 bool CUDABlas::DoBlasSymm(Stream *stream, blas::Side side,
@@ -2912,9 +2912,9 @@ bool CUDABlas::DoBlasSymm(Stream *stream, blas::Side side,
                           DeviceMemory<std::complex<double>> *c, int ldc) {
   return DoBlasInternal(
       wrap::cublasZsymm, stream, true /* = pointer_mode_host */,
-      CUDABlasSide(side), CUDABlasUpperLower(uplo), m, n, CUDAComplex(&alpha),
-      CUDAComplex(CUDAMemory(a)), lda, CUDAComplex(CUDAMemory(b)), ldb,
-      CUDAComplex(&beta), CUDAComplex(CUDAMemoryMutable(c)), ldc);
+      CUDABlasSide(side), CUDABlasUpperLower(uplo), m, n, GpuComplex(&alpha),
+      GpuComplex(GpuMemory(a)), lda, GpuComplex(GpuMemory(b)), ldb,
+      GpuComplex(&beta), GpuComplex(GpuMemoryMutable(c)), ldc);
 }
 
 bool CUDABlas::DoBlasSyrk(Stream *stream, blas::UpperLower uplo,
@@ -2924,7 +2924,7 @@ bool CUDABlas::DoBlasSyrk(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(
       wrap::cublasSsyrk, stream, true /* = pointer_mode_host */,
       CUDABlasUpperLower(uplo), CUDABlasTranspose(trans), n, k, &alpha,
-      CUDAMemory(a), lda, &beta, CUDAMemoryMutable(c), ldc);
+      GpuMemory(a), lda, &beta, GpuMemoryMutable(c), ldc);
 }
 
 bool CUDABlas::DoBlasSyrk(Stream *stream, blas::UpperLower uplo,
@@ -2934,7 +2934,7 @@ bool CUDABlas::DoBlasSyrk(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(
       wrap::cublasDsyrk, stream, true /* = pointer_mode_host */,
       CUDABlasUpperLower(uplo), CUDABlasTranspose(trans), n, k, &alpha,
-      CUDAMemory(a), lda, &beta, CUDAMemoryMutable(c), ldc);
+      GpuMemory(a), lda, &beta, GpuMemoryMutable(c), ldc);
 }
 
 bool CUDABlas::DoBlasSyrk(Stream *stream, blas::UpperLower uplo,
@@ -2946,8 +2946,8 @@ bool CUDABlas::DoBlasSyrk(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(
       wrap::cublasCsyrk, stream, true /* = pointer_mode_host */,
       CUDABlasUpperLower(uplo), CUDABlasTranspose(trans), n, k,
-      CUDAComplex(&alpha), CUDAComplex(CUDAMemory(a)), lda, CUDAComplex(&beta),
-      CUDAComplex(CUDAMemoryMutable(c)), ldc);
+      GpuComplex(&alpha), GpuComplex(GpuMemory(a)), lda, GpuComplex(&beta),
+      GpuComplex(GpuMemoryMutable(c)), ldc);
 }
 
 bool CUDABlas::DoBlasSyrk(Stream *stream, blas::UpperLower uplo,
@@ -2959,8 +2959,8 @@ bool CUDABlas::DoBlasSyrk(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(
       wrap::cublasZsyrk, stream, true /* = pointer_mode_host */,
       CUDABlasUpperLower(uplo), CUDABlasTranspose(trans), n, k,
-      CUDAComplex(&alpha), CUDAComplex(CUDAMemory(a)), lda, CUDAComplex(&beta),
-      CUDAComplex(CUDAMemoryMutable(c)), ldc);
+      GpuComplex(&alpha), GpuComplex(GpuMemory(a)), lda, GpuComplex(&beta),
+      GpuComplex(GpuMemoryMutable(c)), ldc);
 }
 
 bool CUDABlas::DoBlasSyr2k(Stream *stream, blas::UpperLower uplo,
@@ -2971,7 +2971,7 @@ bool CUDABlas::DoBlasSyr2k(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(
       wrap::cublasSsyr2k, stream, true /* = pointer_mode_host */,
       CUDABlasUpperLower(uplo), CUDABlasTranspose(trans), n, k, &alpha,
-      CUDAMemory(a), lda, CUDAMemory(b), ldb, &beta, CUDAMemoryMutable(c), ldc);
+      GpuMemory(a), lda, GpuMemory(b), ldb, &beta, GpuMemoryMutable(c), ldc);
 }
 
 bool CUDABlas::DoBlasSyr2k(Stream *stream, blas::UpperLower uplo,
@@ -2982,7 +2982,7 @@ bool CUDABlas::DoBlasSyr2k(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(
       wrap::cublasDsyr2k, stream, true /* = pointer_mode_host */,
       CUDABlasUpperLower(uplo), CUDABlasTranspose(trans), n, k, &alpha,
-      CUDAMemory(a), lda, CUDAMemory(b), ldb, &beta, CUDAMemoryMutable(c), ldc);
+      GpuMemory(a), lda, GpuMemory(b), ldb, &beta, GpuMemoryMutable(c), ldc);
 }
 
 bool CUDABlas::DoBlasSyr2k(Stream *stream, blas::UpperLower uplo,
@@ -2995,9 +2995,9 @@ bool CUDABlas::DoBlasSyr2k(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(wrap::cublasCsyr2k, stream,
                         true /* = pointer_mode_host */,
                         CUDABlasUpperLower(uplo), CUDABlasTranspose(trans), n,
-                        k, CUDAComplex(&alpha), CUDAComplex(CUDAMemory(a)), lda,
-                        CUDAComplex(CUDAMemory(b)), ldb, CUDAComplex(&beta),
-                        CUDAComplex(CUDAMemoryMutable(c)), ldc);
+                        k, GpuComplex(&alpha), GpuComplex(GpuMemory(a)), lda,
+                        GpuComplex(GpuMemory(b)), ldb, GpuComplex(&beta),
+                        GpuComplex(GpuMemoryMutable(c)), ldc);
 }
 
 bool CUDABlas::DoBlasSyr2k(Stream *stream, blas::UpperLower uplo,
@@ -3010,9 +3010,9 @@ bool CUDABlas::DoBlasSyr2k(Stream *stream, blas::UpperLower uplo,
   return DoBlasInternal(wrap::cublasZsyr2k, stream,
                         true /* = pointer_mode_host */,
                         CUDABlasUpperLower(uplo), CUDABlasTranspose(trans), n,
-                        k, CUDAComplex(&alpha), CUDAComplex(CUDAMemory(a)), lda,
-                        CUDAComplex(CUDAMemory(b)), ldb, CUDAComplex(&beta),
-                        CUDAComplex(CUDAMemoryMutable(c)), ldc);
+                        k, GpuComplex(&alpha), GpuComplex(GpuMemory(a)), lda,
+                        GpuComplex(GpuMemory(b)), ldb, GpuComplex(&beta),
+                        GpuComplex(GpuMemoryMutable(c)), ldc);
 }
 
 bool CUDABlas::DoBlasTrmm(Stream *stream, blas::Side side,
@@ -3023,8 +3023,8 @@ bool CUDABlas::DoBlasTrmm(Stream *stream, blas::Side side,
   return DoBlasInternal(
       wrap::cublasStrmm, stream, true /* = pointer_mode_host */,
       CUDABlasSide(side), CUDABlasUpperLower(uplo), CUDABlasTranspose(transa),
-      CUDABlasDiagonal(diag), m, n, &alpha, CUDAMemory(a), lda,
-      CUDAMemoryMutable(b), ldb, CUDAMemoryMutable(b), ldb);
+      CUDABlasDiagonal(diag), m, n, &alpha, GpuMemory(a), lda,
+      GpuMemoryMutable(b), ldb, GpuMemoryMutable(b), ldb);
 }
 
 bool CUDABlas::DoBlasTrmm(Stream *stream, blas::Side side,
@@ -3035,8 +3035,8 @@ bool CUDABlas::DoBlasTrmm(Stream *stream, blas::Side side,
   return DoBlasInternal(
       wrap::cublasDtrmm, stream, true /* = pointer_mode_host */,
       CUDABlasSide(side), CUDABlasUpperLower(uplo), CUDABlasTranspose(transa),
-      CUDABlasDiagonal(diag), m, n, &alpha, CUDAMemory(a), lda,
-      CUDAMemoryMutable(b), ldb, CUDAMemoryMutable(b), ldb);
+      CUDABlasDiagonal(diag), m, n, &alpha, GpuMemory(a), lda,
+      GpuMemoryMutable(b), ldb, GpuMemoryMutable(b), ldb);
 }
 
 bool CUDABlas::DoBlasTrmm(Stream *stream, blas::Side side,
@@ -3048,9 +3048,9 @@ bool CUDABlas::DoBlasTrmm(Stream *stream, blas::Side side,
   return DoBlasInternal(
       wrap::cublasCtrmm, stream, true /* = pointer_mode_host */,
       CUDABlasSide(side), CUDABlasUpperLower(uplo), CUDABlasTranspose(transa),
-      CUDABlasDiagonal(diag), m, n, CUDAComplex(&alpha),
-      CUDAComplex(CUDAMemory(a)), lda, CUDAComplex(CUDAMemoryMutable(b)), ldb,
-      CUDAComplex(CUDAMemoryMutable(b)), ldb);
+      CUDABlasDiagonal(diag), m, n, GpuComplex(&alpha),
+      GpuComplex(GpuMemory(a)), lda, GpuComplex(GpuMemoryMutable(b)), ldb,
+      GpuComplex(GpuMemoryMutable(b)), ldb);
 }
 
 bool CUDABlas::DoBlasTrmm(Stream *stream, blas::Side side,
@@ -3062,9 +3062,9 @@ bool CUDABlas::DoBlasTrmm(Stream *stream, blas::Side side,
   return DoBlasInternal(
       wrap::cublasZtrmm, stream, true /* = pointer_mode_host */,
       CUDABlasSide(side), CUDABlasUpperLower(uplo), CUDABlasTranspose(transa),
-      CUDABlasDiagonal(diag), m, n, CUDAComplex(&alpha),
-      CUDAComplex(CUDAMemory(a)), lda, CUDAComplex(CUDAMemoryMutable(b)), ldb,
-      CUDAComplex(CUDAMemoryMutable(b)), ldb);
+      CUDABlasDiagonal(diag), m, n, GpuComplex(&alpha),
+      GpuComplex(GpuMemory(a)), lda, GpuComplex(GpuMemoryMutable(b)), ldb,
+      GpuComplex(GpuMemoryMutable(b)), ldb);
 }
 
 bool CUDABlas::DoBlasTrsm(Stream *stream, blas::Side side,
@@ -3075,8 +3075,8 @@ bool CUDABlas::DoBlasTrsm(Stream *stream, blas::Side side,
   return DoBlasInternal(wrap::cublasStrsm, stream,
                         true /* = pointer_mode_host */, CUDABlasSide(side),
                         CUDABlasUpperLower(uplo), CUDABlasTranspose(transa),
-                        CUDABlasDiagonal(diag), m, n, &alpha, CUDAMemory(a),
-                        lda, CUDAMemoryMutable(b), ldb);
+                        CUDABlasDiagonal(diag), m, n, &alpha, GpuMemory(a),
+                        lda, GpuMemoryMutable(b), ldb);
 }
 
 bool CUDABlas::DoBlasTrsm(Stream *stream, blas::Side side,
@@ -3087,8 +3087,8 @@ bool CUDABlas::DoBlasTrsm(Stream *stream, blas::Side side,
   return DoBlasInternal(wrap::cublasDtrsm, stream,
                         true /* = pointer_mode_host */, CUDABlasSide(side),
                         CUDABlasUpperLower(uplo), CUDABlasTranspose(transa),
-                        CUDABlasDiagonal(diag), m, n, &alpha, CUDAMemory(a),
-                        lda, CUDAMemoryMutable(b), ldb);
+                        CUDABlasDiagonal(diag), m, n, &alpha, GpuMemory(a),
+                        lda, GpuMemoryMutable(b), ldb);
 }
 
 bool CUDABlas::DoBlasTrsm(Stream *stream, blas::Side side,
@@ -3100,8 +3100,8 @@ bool CUDABlas::DoBlasTrsm(Stream *stream, blas::Side side,
   return DoBlasInternal(
       wrap::cublasCtrsm, stream, true /* = pointer_mode_host */,
       CUDABlasSide(side), CUDABlasUpperLower(uplo), CUDABlasTranspose(transa),
-      CUDABlasDiagonal(diag), m, n, CUDAComplex(&alpha),
-      CUDAComplex(CUDAMemory(a)), lda, CUDAComplex(CUDAMemoryMutable(b)), ldb);
+      CUDABlasDiagonal(diag), m, n, GpuComplex(&alpha),
+      GpuComplex(GpuMemory(a)), lda, GpuComplex(GpuMemoryMutable(b)), ldb);
 }
 
 bool CUDABlas::DoBlasTrsm(Stream *stream, blas::Side side,
@@ -3113,19 +3113,19 @@ bool CUDABlas::DoBlasTrsm(Stream *stream, blas::Side side,
   return DoBlasInternal(
       wrap::cublasZtrsm, stream, true /* = pointer_mode_host */,
       CUDABlasSide(side), CUDABlasUpperLower(uplo), CUDABlasTranspose(transa),
-      CUDABlasDiagonal(diag), m, n, CUDAComplex(&alpha),
-      CUDAComplex(CUDAMemory(a)), lda, CUDAComplex(CUDAMemoryMutable(b)), ldb);
+      CUDABlasDiagonal(diag), m, n, GpuComplex(&alpha),
+      GpuComplex(GpuMemory(a)), lda, GpuComplex(GpuMemoryMutable(b)), ldb);
 }
 
-}  // namespace cuda
+}  // namespace gpu
 
 void initialize_cublas() {
   port::Status status =
       PluginRegistry::Instance()->RegisterFactory<PluginRegistry::BlasFactory>(
-          cuda::kCudaPlatformId, cuda::kCuBlasPlugin, "cuBLAS",
+          cuda::kCudaPlatformId, gpu::kCuBlasPlugin, "cuBLAS",
           [](internal::StreamExecutorInterface *parent) -> blas::BlasSupport * {
-            cuda::CUDAExecutor *cuda_executor =
-                dynamic_cast<cuda::CUDAExecutor *>(parent);
+            gpu::GpuExecutor *cuda_executor =
+                dynamic_cast<gpu::GpuExecutor *>(parent);
             if (cuda_executor == nullptr) {
               LOG(ERROR)
                   << "Attempting to initialize an instance of the cuBLAS "
@@ -3133,7 +3133,7 @@ void initialize_cublas() {
               return nullptr;
             }
 
-            cuda::CUDABlas *blas = new cuda::CUDABlas(cuda_executor);
+            gpu::CUDABlas *blas = new gpu::CUDABlas(cuda_executor);
             if (!blas->Init()) {
               // Note: Init() will log a more specific error.
               delete blas;
@@ -3148,7 +3148,7 @@ void initialize_cublas() {
   }
 
   PluginRegistry::Instance()->SetDefaultFactory(
-      cuda::kCudaPlatformId, PluginKind::kBlas, cuda::kCuBlasPlugin);
+      cuda::kCudaPlatformId, PluginKind::kBlas, gpu::kCuBlasPlugin);
 }
 
 }  // namespace stream_executor
diff --git a/tensorflow/stream_executor/cuda/cuda_blas.h b/tensorflow/stream_executor/cuda/cuda_blas.h
index 0fb05089d7..63d03056d9 100644
--- a/tensorflow/stream_executor/cuda/cuda_blas.h
+++ b/tensorflow/stream_executor/cuda/cuda_blas.h
@@ -33,26 +33,26 @@ namespace stream_executor {
 
 class Stream;
 
-namespace cuda {
+namespace gpu {
 
 // Opaque and unique identifier for the cuBLAS plugin.
 extern const PluginId kCuBlasPlugin;
 
-class CUDAExecutor;
+class GpuExecutor;
 
 // BLAS plugin for CUDA platform via cuBLAS library.
 //
 // This satisfies the platform-agnostic BlasSupport interface.
 //
 // Note that the cuBLAS handle that this encapsulates is implicitly tied to the
-// context (and, as a result, the device) that the parent CUDAExecutor is tied
+// context (and, as a result, the device) that the parent GpuExecutor is tied
 // to. This simply happens as an artifact of creating the cuBLAS handle when a
 // CUDA context is active.
 //
 // Thread-safe post-initialization.
 class CUDABlas : public blas::BlasSupport {
  public:
-  explicit CUDABlas(CUDAExecutor *parent);
+  explicit CUDABlas(GpuExecutor *parent);
 
   // Allocates a cuBLAS handle.
   bool Init();
@@ -145,9 +145,9 @@ class CUDABlas : public blas::BlasSupport {
   // mutex that guards the cuBLAS handle for this device.
   mutex mu_;
 
-  // CUDAExecutor which instantiated this CUDABlas.
+  // GpuExecutor which instantiated this CUDABlas.
   // Immutable post-initialization.
-  CUDAExecutor *parent_;
+  GpuExecutor *parent_;
 
   // cuBLAS library handle on the device.
   cublasHandle_t blas_ GUARDED_BY(mu_);
@@ -155,7 +155,7 @@ class CUDABlas : public blas::BlasSupport {
   SE_DISALLOW_COPY_AND_ASSIGN(CUDABlas);
 };
 
-}  // namespace cuda
+}  // namespace gpu
 }  // namespace stream_executor
 
 #endif  // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_BLAS_H_
diff --git a/tensorflow/stream_executor/cuda/cuda_diagnostics.cc b/tensorflow/stream_executor/cuda/cuda_diagnostics.cc
index 6af71b6c9d..e58ebee80d 100644
--- a/tensorflow/stream_executor/cuda/cuda_diagnostics.cc
+++ b/tensorflow/stream_executor/cuda/cuda_diagnostics.cc
@@ -52,13 +52,6 @@ limitations under the License.
 namespace stream_executor {
 namespace cuda {
 
-#ifdef __APPLE__
-static const CFStringRef kDriverKextIdentifier = CFSTR("com.nvidia.CUDA");
-#elif !defined(PLATFORM_WINDOWS)
-static const char *kDriverVersionPath = "/proc/driver/nvidia/version";
-#endif
-
-
 string DriverVersionToString(DriverVersion version) {
   return port::Printf("%d.%d.%d", std::get<0>(version), std::get<1>(version), std::get<2>(version));
 }
@@ -112,6 +105,18 @@ port::StatusOr<DriverVersion> StringToDriverVersion(const string &value) {
   return result;
 }
 
+}  // namespace cuda
+}  // namespace stream_executor
+
+namespace stream_executor {
+namespace gpu {
+
+#ifdef __APPLE__
+static const CFStringRef kDriverKextIdentifier = CFSTR("com.nvidia.CUDA");
+#elif !defined(PLATFORM_WINDOWS)
+static const char *kDriverVersionPath = "/proc/driver/nvidia/version";
+#endif
+
 // -- class Diagnostician
 
 string Diagnostician::GetDevNodePath(int dev_node_ordinal) {
@@ -190,11 +195,11 @@ void Diagnostician::LogDiagnosticInformation() {
   }
   port::StatusOr<DriverVersion> dso_version = FindDsoVersion();
   LOG(INFO) << "libcuda reported version is: "
-            << DriverVersionStatusToString(dso_version);
+            << cuda::DriverVersionStatusToString(dso_version);
 
   port::StatusOr<DriverVersion> kernel_version = FindKernelDriverVersion();
   LOG(INFO) << "kernel reported version is: "
-	  << DriverVersionStatusToString(kernel_version);
+            << cuda::DriverVersionStatusToString(kernel_version);
 #endif
 
   // OS X kernel driver does not report version accurately
@@ -232,7 +237,7 @@ port::StatusOr<DriverVersion> Diagnostician::FindDsoVersion() {
     }
     const size_t length = suffix_pos - start;
     const string version = path.substr(start, length);
-    result = StringToDriverVersion(version);
+    result = cuda::StringToDriverVersion(version);
   }
 #else
 #if !defined(PLATFORM_WINDOWS) && !defined(ANDROID_TEGRA)
@@ -260,7 +265,7 @@ port::StatusOr<DriverVersion> Diagnostician::FindDsoVersion() {
       // TODO(b/22689637): Eliminate the explicit namespace if possible.
       auto stripped_dso_version = port::StripSuffixString(dso_version, ".ld64");
       auto result = static_cast<port::StatusOr<DriverVersion> *>(data);
-      *result = StringToDriverVersion(stripped_dso_version);
+      *result = cuda::StringToDriverVersion(stripped_dso_version);
       return 1;
     }
     return 0;
@@ -292,7 +297,7 @@ port::StatusOr<DriverVersion> Diagnostician::FindKernelModuleVersion(
   // TODO(b/22689637): Eliminate the explicit namespace if possible.
   auto stripped_kernel_version =
       port::StripSuffixString(kernel_version, ".ld64");
-  return StringToDriverVersion(stripped_kernel_version);
+  return cuda::StringToDriverVersion(stripped_kernel_version);
 }
 
 void Diagnostician::WarnOnDsoKernelMismatch(
@@ -301,12 +306,12 @@ void Diagnostician::WarnOnDsoKernelMismatch(
   if (kernel_version.ok() && dso_version.ok() &&
       dso_version.ValueOrDie() == kernel_version.ValueOrDie()) {
     LOG(INFO) << "kernel version seems to match DSO: "
-              << DriverVersionToString(kernel_version.ValueOrDie());
+              << cuda::DriverVersionToString(kernel_version.ValueOrDie());
   } else {
     LOG(ERROR) << "kernel version "
-               << DriverVersionStatusToString(kernel_version)
+               << cuda::DriverVersionStatusToString(kernel_version)
                << " does not match DSO version "
-               << DriverVersionStatusToString(dso_version)
+               << cuda::DriverVersionStatusToString(dso_version)
                << " -- cannot find working devices in this configuration";
   }
 }
@@ -336,9 +341,9 @@ port::StatusOr<DriverVersion> Diagnostician::FindKernelDriverVersion() {
     // see
     // https://developer.apple.com/library/mac/documentation/CoreFoundation/Conceptual/CFStrings/Articles/AccessingContents.html#//apple_ref/doc/uid/20001184-100980-TPXREF112
     if (version == NULL) {
-      return StringToDriverVersion("");
+      return cuda::StringToDriverVersion("");
     }
-    return StringToDriverVersion(version);
+    return cuda::StringToDriverVersion(version);
   }
   CFRelease(kext_infos);
   auto status = port::Status(
@@ -387,6 +392,5 @@ port::StatusOr<DriverVersion> Diagnostician::FindKernelDriverVersion() {
 #endif
 }
 
-
-}  // namespace cuda
+}  // namespace gpu
 }  // namespace stream_executor
diff --git a/tensorflow/stream_executor/cuda/cuda_diagnostics.h b/tensorflow/stream_executor/cuda/cuda_diagnostics.h
index f2db2eb20a..0837e136fd 100644
--- a/tensorflow/stream_executor/cuda/cuda_diagnostics.h
+++ b/tensorflow/stream_executor/cuda/cuda_diagnostics.h
@@ -16,17 +16,13 @@ limitations under the License.
 #ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DIAGNOSTICS_H_
 #define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DIAGNOSTICS_H_
 
-#include "tensorflow/stream_executor/platform/port.h"
-#include <tuple>
-
-#include "tensorflow/stream_executor/lib/statusor.h"
-#include "tensorflow/stream_executor/platform/port.h"
+#include "tensorflow/stream_executor/gpu/gpu_diagnostics.h"
 
 namespace stream_executor {
 namespace cuda {
 
 // e.g. DriverVersion{346, 3, 4}
-using DriverVersion = std::tuple<int, int, int>;
+using DriverVersion = gpu::DriverVersion;
 
 // Converts a parsed driver version to string form.
 string DriverVersionToString(DriverVersion version);
@@ -35,61 +31,9 @@ string DriverVersionToString(DriverVersion version);
 string DriverVersionStatusToString(port::StatusOr<DriverVersion> version);
 
 // Converts a string of a form like "331.79" to a DriverVersion{331, 79}.
-port::StatusOr<DriverVersion> StringToDriverVersion(const string &value);
-
-class Diagnostician {
- public:
-  // Logs diagnostic information when CUDA appears to be misconfigured (e.g. is
-  // not initializing).
-  //
-  // Note: if we're running on a machine that has no GPUs, we don't want to
-  // produce very much log spew beyond saying, "looks like there's no CUDA
-  // kernel
-  // module running".
-  //
-  // Note: we use non-Google-File:: API here because we may be called before
-  // InitGoogle has completed.
-  static void LogDiagnosticInformation();
-
-  // Given the driver version file contents, finds the kernel module version and
-  // returns it as a string.
-  //
-  // This is solely used for more informative log messages when the user is
-  // running on a machine that happens to have a libcuda/kernel driver mismatch.
-  static port::StatusOr<DriverVersion> FindKernelModuleVersion(
-      const string &driver_version_file_contents);
-
-  // Extracts the kernel driver version from the current host.
-  static port::StatusOr<DriverVersion> FindKernelDriverVersion();
-
-  // Iterates through loaded DSOs with DlIteratePhdrCallback to find the
-  // driver-interfacing DSO version number. Returns it as a string.
-  static port::StatusOr<DriverVersion> FindDsoVersion();
-
-  // Logs information about the kernel driver version and userspace driver
-  // library version.
-  static void LogDriverVersionInformation();
-
- private:
-
-  // Given the DSO version number and the driver version file contents, extracts
-  // the driver version and compares, warning the user in the case of
-  // incompatibility.
-  //
-  // This is solely used for more informative log messages when the user is
-  // running on a machine that happens to have a libcuda/kernel driver mismatch.
-  static void WarnOnDsoKernelMismatch(
-      port::StatusOr<DriverVersion> dso_version,
-      port::StatusOr<DriverVersion> kernel_version);
-
-  // Logs information about the dev nodes present on this machine: their
-  // existence, permissions, accessibility from this uid/gid.
-  static void LogDevNodeDiagnosticInformation();
-
-  static string GetDevNodePath(int dev_node_ordinal);
+port::StatusOr<DriverVersion> StringToDriverVersion(const string& value);
 
-  SE_DISALLOW_COPY_AND_ASSIGN(Diagnostician);
-};
+using Diagnostician = gpu::Diagnostician;
 
 }  // namespace cuda
 }  // namespace stream_executor
diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index ef2e13d314..06739e8655 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -58,7 +58,7 @@ limitations under the License.
 #pragma clang diagnostic warning "-Wmismatched-tags"
 
 namespace stream_executor {
-namespace cuda {
+namespace gpu {
 
 PLUGIN_REGISTRY_DEFINE_PLUGIN_ID(kCuDnnPlugin);
 
@@ -137,7 +137,7 @@ class CudnnHandle {
  public:
   // Takes ownership of the executor context and the lock to access cuDNN
   // using handle.
-  CudnnHandle(cuda::ScopedActivateExecutorContext context, mutex_lock lock,
+  CudnnHandle(gpu::ScopedActivateExecutorContext context, mutex_lock lock,
               cudnnHandle_t handle)
       : context_(std::move(context)), lock_(std::move(lock)), handle_(handle) {}
 
@@ -146,7 +146,7 @@ class CudnnHandle {
   cudnnHandle_t handle() const { return handle_; }
 
  private:
-  cuda::ScopedActivateExecutorContext context_;
+  gpu::ScopedActivateExecutorContext context_;
   mutex_lock lock_;
   cudnnHandle_t handle_;  // Not owned.
 };
@@ -334,10 +334,10 @@ class CudnnAccess {
   // The legacy default stream synchronizes with all other streams and it is
   // therefore a bad idea (performance wise) to call any cuDNN APIs that
   // enqueue work in the stream.
-  CudnnHandle GetHandle(CUDAExecutor* executor, Stream* stream) {
+  CudnnHandle GetHandle(GpuExecutor* executor, Stream* stream) {
     mutex_lock lock(mutex_);
-    cuda::ScopedActivateExecutorContext context(executor);
-    CUstream cu_stream = stream ? AsCUDAStreamValue(stream) : cudaStreamLegacy;
+    gpu::ScopedActivateExecutorContext context(executor);
+    CUstream cu_stream = stream ? AsGpuStreamValue(stream) : cudaStreamLegacy;
     auto status = cudnnSetStream(handle_, cu_stream);
     CHECK_EQ(status, CUDNN_STATUS_SUCCESS) << "Failed to set cuDNN stream.";
     return CudnnHandle(std::move(context), std::move(lock), handle_);
@@ -448,7 +448,7 @@ port::Status GetLoadedCudnnVersion(CudnnVersion* version) {
 
 }  // namespace
 
-CudnnSupport::CudnnSupport(CUDAExecutor* parent) : parent_(parent) {}
+CudnnSupport::CudnnSupport(GpuExecutor* parent) : parent_(parent) {}
 
 port::Status CudnnSupport::Init() {
   ScopedActivateExecutorContext context(parent_);
@@ -481,14 +481,14 @@ port::Status CudnnSupport::Init() {
   CHECK_EQ(cudnn_handle, nullptr);
   LOG(ERROR) << "Could not create cudnn handle: " << ToString(status);
   if (status == CUDNN_STATUS_NOT_INITIALIZED) {
-    auto result = cuda::Diagnostician::FindKernelDriverVersion();
+    auto result = gpu::Diagnostician::FindKernelDriverVersion();
     if (!result.ok()) {
       LOG(ERROR) << "Error retrieving driver version: "
-                 << DriverVersionStatusToString(result);
+                 << cuda::DriverVersionStatusToString(result);
     } else {
       const auto& version = result.ValueOrDie();
       LOG(ERROR) << "Possibly insufficient driver version: "
-                 << DriverVersionToString(version);
+                 << cuda::DriverVersionToString(version);
     }
   }
 
@@ -1151,7 +1151,7 @@ class CudnnRnnParamsDescriptor {
 }  // namespace
 
 class CudnnRnnDescriptor : public dnn::RnnDescriptor {
-  CudnnRnnDescriptor(const CudnnHandle& cudnn, cuda::RnnDescriptor rnn_desc,
+  CudnnRnnDescriptor(const CudnnHandle& cudnn, gpu::RnnDescriptor rnn_desc,
                      PersistentRnnPlan rnn_plan, int num_layers,
                      int hidden_size, int input_size, int batch_size,
                      cudnnRNNInputMode_t input_mode,
@@ -1191,7 +1191,7 @@ class CudnnRnnDescriptor : public dnn::RnnDescriptor {
         CudnnDropoutDescriptor dropout_desc,
         CudnnDropoutDescriptor::Create(cudnn, dropout, seed, state_allocator));
 
-    cuda::RnnDescriptor rnn_desc = CreateRnnDescriptor();
+    gpu::RnnDescriptor rnn_desc = CreateRnnDescriptor();
     cudnnRNNAlgo_t rnn_algo = ToCudnnRNNAlgo(algorithm_config.algorithm());
 
     // TODO: allow the user to choose an algorithm.
@@ -1282,7 +1282,7 @@ class CudnnRnnDescriptor : public dnn::RnnDescriptor {
   }
 
  private:
-  cuda::RnnDescriptor rnn_desc_;
+  gpu::RnnDescriptor rnn_desc_;
   PersistentRnnPlan rnn_plan_;
   int num_layers_;
   int hidden_size_;
@@ -1401,15 +1401,14 @@ port::StatusOr<CudnnRnnParamsDescriptor> CudnnRnnParamsDescriptor::Create(
 
 class CudnnRnnSequenceTensorDescriptor
     : public dnn::RnnSequenceTensorDescriptor {
-  CudnnRnnSequenceTensorDescriptor(CUDAExecutor* parent, int max_seq_length,
+  CudnnRnnSequenceTensorDescriptor(GpuExecutor* parent, int max_seq_length,
                                    int batch_size, int data_size,
                                    cudnnDataType_t data_type,
 #if CUDNN_VERSION >= 7201
                                    RNNDataDescriptor data_handle,
 #endif
                                    TensorDescriptor handle)
-      : parent_(parent),
-        max_seq_length_(max_seq_length),
+      : max_seq_length_(max_seq_length),
         batch_size_(batch_size),
         data_size_(data_size),
         data_type_(data_type),
@@ -1425,7 +1424,7 @@ class CudnnRnnSequenceTensorDescriptor
       default;
 
   static port::StatusOr<CudnnRnnSequenceTensorDescriptor> Create(
-      CUDAExecutor* parent, int max_seq_length, int batch_size, int data_size,
+      GpuExecutor* parent, int max_seq_length, int batch_size, int data_size,
       cudnnDataType_t data_type) {
     CHECK_GT(max_seq_length, 0);
     int dims[] = {batch_size, data_size, 1};
@@ -1444,7 +1443,7 @@ class CudnnRnnSequenceTensorDescriptor
   }
 
   static port::StatusOr<CudnnRnnSequenceTensorDescriptor> Create(
-      CUDAExecutor* parent, int max_seq_length, int batch_size, int data_size,
+      GpuExecutor* parent, int max_seq_length, int batch_size, int data_size,
       const absl::Span<const int>& seq_lengths, cudnnDataType_t data_type) {
 #if CUDNN_VERSION >= 7201
     CHECK_GT(max_seq_length, 0);
@@ -1496,7 +1495,6 @@ class CudnnRnnSequenceTensorDescriptor
   }
 
  private:
-  CUDAExecutor* parent_;
   int max_seq_length_;
   int batch_size_;
   int data_size_;
@@ -1511,11 +1509,10 @@ class CudnnRnnSequenceTensorDescriptor
 
 class CudnnRnnStateTensorDescriptor : public dnn::RnnStateTensorDescriptor {
  public:
-  CudnnRnnStateTensorDescriptor(CUDAExecutor* parent, int num_layers,
+  CudnnRnnStateTensorDescriptor(GpuExecutor* parent, int num_layers,
                                 int batch_size, int data_size,
                                 cudnnDataType_t data_type)
-      : parent_(parent),
-        handle_(CreateTensorDescriptor()),
+      : handle_(CreateTensorDescriptor()),
         num_layers_(num_layers),
         batch_size_(batch_size),
         data_size_(data_size),
@@ -1535,7 +1532,6 @@ class CudnnRnnStateTensorDescriptor : public dnn::RnnStateTensorDescriptor {
   int data_size() const { return data_size_; }
 
  private:
-  CUDAExecutor* parent_;
   TensorDescriptor handle_;
   int num_layers_;
   int batch_size_;
@@ -1699,14 +1695,14 @@ port::Status CudnnSupport::DoRnnForwardImpl(
     }
   }
 
-  std::unique_ptr<CUDATimer, TimerDeleter> timer;
+  std::unique_ptr<GpuTimer, GpuTimerDeleter> timer;
   const bool is_profiling = output_profile_result != nullptr;
   if (is_profiling) {
-    timer.reset(new CUDATimer(parent_));
+    timer.reset(new GpuTimer(parent_));
     // The start and stop of the timer should be as close to the Cudnn call as
     // possible. It is still possible for other threads to issue workload on
     // to this stream. So it could take multiple profiling measurements.
-    if (!timer->Init() || !timer->Start(AsCUDAStream(stream))) {
+    if (!timer->Init() || !timer->Start(AsGpuStream(stream))) {
       return port::Status(port::error::INTERNAL, "Failed to start timer");
     }
   }
@@ -1791,7 +1787,7 @@ port::Status CudnnSupport::DoRnnForwardImpl(
   }
 
   if (is_profiling) {
-    if (!timer->Stop(AsCUDAStream(stream))) {
+    if (!timer->Stop(AsGpuStream(stream))) {
       return port::Status(port::error::INTERNAL, "Failed to stop timer");
     }
     auto algo_desc = *rnn_desc.algorithm_config().algorithm();
@@ -1842,14 +1838,14 @@ port::Status CudnnSupport::DoRnnBackwardImpl(
                       CreateRnnWorkspace(stream, cudnn, rnn_desc, input_desc,
                                          workspace_allocator));
 
-  std::unique_ptr<CUDATimer, TimerDeleter> timer;
+  std::unique_ptr<GpuTimer, GpuTimerDeleter> timer;
   const bool is_profiling = output_profile_result != nullptr;
   if (is_profiling) {
-    timer.reset(new CUDATimer(parent_));
+    timer.reset(new GpuTimer(parent_));
     // The start and stop of the timer should be as close to the Cudnn call as
     // possible. It is still possible for other threads to issue workload on
     // to this stream. So it could take multiple profiling measurements.
-    if (!timer->Init() || !timer->Start(AsCUDAStream(stream))) {
+    if (!timer->Init() || !timer->Start(AsGpuStream(stream))) {
       return port::Status(port::error::INTERNAL, "Failed to start timer");
     }
   }
@@ -1948,7 +1944,7 @@ port::Status CudnnSupport::DoRnnBackwardImpl(
   }
 
   if (is_profiling) {
-    if (!timer->Stop(AsCUDAStream(stream))) {
+    if (!timer->Stop(AsGpuStream(stream))) {
       return port::Status(port::error::INTERNAL, "Failed to stop timer");
     }
     auto algo_desc = *rnn_desc.algorithm_config().algorithm();
@@ -2915,13 +2911,13 @@ port::Status CudnnSupport::DoConvolve(
 
   const bool is_profiling = output_profile_result != nullptr;
 
-  std::unique_ptr<CUDATimer, TimerDeleter> timer;
+  std::unique_ptr<GpuTimer, GpuTimerDeleter> timer;
   if (is_profiling) {
-    timer.reset(new CUDATimer(parent_));  // NOLINT
+    timer.reset(new GpuTimer(parent_));  // NOLINT
     // The start and stop of the timer should be as close to the Cudnn call as
     // possible. It is still possible for other threads to issue workload on
     // to this stream. So it could take multiple profiling measurements.
-    if (!timer->Init() || !timer->Start(AsCUDAStream(stream))) {
+    if (!timer->Init() || !timer->Start(AsGpuStream(stream))) {
       return port::Status(port::error::INTERNAL, "Failed to start timer");
     }
   }
@@ -3110,7 +3106,7 @@ port::Status CudnnSupport::DoConvolve(
   }
 
   if (is_profiling) {
-    if (!timer->Stop(AsCUDAStream(stream))) {
+    if (!timer->Stop(AsGpuStream(stream))) {
       return port::Status(port::error::INTERNAL, "Failed to stop timer");
     }
     output_profile_result->set_algorithm(algorithm_desc);
@@ -3175,13 +3171,13 @@ port::Status CudnnSupport::DoFusedConvolveImpl(
           stream, cudnn, algorithm_config, conv_input_nd, filter, conv,
           output_nd, scratch_allocator, &scratch));
 
-  std::unique_ptr<CUDATimer, TimerDeleter> timer;
+  std::unique_ptr<GpuTimer, GpuTimerDeleter> timer;
   if (is_profiling) {
-    timer.reset(new CUDATimer(parent_));  // NOLINT
+    timer.reset(new GpuTimer(parent_));  // NOLINT
     // The start and stop of the timer should be as close to the Cudnn call as
     // possible. It is still possible for other threads to issue workload on
     // to this stream. So it could take multiple profiling measurements.
-    if (!timer->Init() || !timer->Start(AsCUDAStream(stream))) {
+    if (!timer->Init() || !timer->Start(AsGpuStream(stream))) {
       return port::Status(port::error::INTERNAL, "Failed to start timer");
     }
   }
@@ -3234,7 +3230,7 @@ port::Status CudnnSupport::DoFusedConvolveImpl(
       /*yDesc=*/output_nd.handle(), /*y=*/output_data->opaque()));
 
   if (is_profiling) {
-    if (!timer->Stop(AsCUDAStream(stream))) {
+    if (!timer->Stop(AsGpuStream(stream))) {
       return port::Status(port::error::INTERNAL, "Failed to stop timer");
     }
     output_profile_result->set_algorithm(algo_desc);
@@ -4339,22 +4335,22 @@ bool CudnnSupport::DeriveOutputBatchDescriptor(
   return IsStatusOk(status, /*report_error=*/true);
 }
 
-}  // namespace cuda
+}  // namespace gpu
 
 void initialize_cudnn() {
   port::Status status =
       PluginRegistry::Instance()->RegisterFactory<PluginRegistry::DnnFactory>(
-          cuda::kCudaPlatformId, cuda::kCuDnnPlugin, "cuDNN",
+          cuda::kCudaPlatformId, gpu::kCuDnnPlugin, "cuDNN",
           [](internal::StreamExecutorInterface* parent) -> dnn::DnnSupport* {
-            cuda::CUDAExecutor* cuda_executor =
-                dynamic_cast<cuda::CUDAExecutor*>(parent);
+            gpu::GpuExecutor* cuda_executor =
+                dynamic_cast<gpu::GpuExecutor*>(parent);
             if (cuda_executor == nullptr) {
               LOG(ERROR) << "Attempting to initialize an instance of the cuDNN "
                          << "support library with a non-CUDA StreamExecutor";
               return nullptr;
             }
 
-            cuda::CudnnSupport* dnn = new cuda::CudnnSupport(cuda_executor);
+            gpu::CudnnSupport* dnn = new gpu::CudnnSupport(cuda_executor);
             if (!dnn->Init().ok()) {
               // Note: Init() will log a more specific error.
               delete dnn;
@@ -4369,7 +4365,7 @@ void initialize_cudnn() {
   }
 
   PluginRegistry::Instance()->SetDefaultFactory(
-      cuda::kCudaPlatformId, PluginKind::kDnn, cuda::kCuDnnPlugin);
+      cuda::kCudaPlatformId, PluginKind::kDnn, gpu::kCuDnnPlugin);
 }
 
 }  // namespace stream_executor
diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.h b/tensorflow/stream_executor/cuda/cuda_dnn.h
index 2fd311e887..80c7c8a06c 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.h
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.h
@@ -28,9 +28,9 @@ limitations under the License.
 #include "tensorflow/stream_executor/temporary_device_memory.h"
 
 namespace stream_executor {
-namespace cuda {
+namespace gpu {
 
-class CUDAExecutor;
+class GpuExecutor;
 class CudnnRnnDescriptor;
 class CudnnRnnSequenceTensorDescriptor;
 class CudnnRnnStateTensorDescriptor;
@@ -42,7 +42,7 @@ extern const PluginId kCuDnnPlugin;
 // functions, see dnn.h.
 class CudnnSupport : public dnn::DnnSupport {
  public:
-  explicit CudnnSupport(CUDAExecutor* parent);
+  explicit CudnnSupport(GpuExecutor* parent);
 
   port::Status Init() override;
   port::StatusOr<perftools::gputools::dnn::VersionInfo> GetVersion() override;
@@ -552,7 +552,7 @@ class CudnnSupport : public dnn::DnnSupport {
                          DeviceMemoryBase* output_data) override;
 
  private:
-  CUDAExecutor* parent_;  // Parent executor object. Not owned.
+  GpuExecutor* parent_;  // Parent executor object. Not owned.
 
   // Provides access to the cuDNN handle.
   std::unique_ptr<class CudnnAccess> cudnn_;
@@ -667,7 +667,7 @@ class CudnnSupport : public dnn::DnnSupport {
   SE_DISALLOW_COPY_AND_ASSIGN(CudnnSupport);
 };
 
-}  // namespace cuda
+}  // namespace gpu
 }  // namespace stream_executor
 
 #endif  // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DNN_H_
diff --git a/tensorflow/stream_executor/cuda/cuda_driver.cc b/tensorflow/stream_executor/cuda/cuda_driver.cc
index c39e4f59eb..34ba7c5b15 100644
--- a/tensorflow/stream_executor/cuda/cuda_driver.cc
+++ b/tensorflow/stream_executor/cuda/cuda_driver.cc
@@ -45,21 +45,20 @@ bool FLAGS_gpuexec_cuda_device_0_only = false;
 
 // Debugging: on each push and pop of a cuda context, verify the current context
 // matches the expected one.
-constexpr bool kVerifyCudaContext = false;
+constexpr bool kVerifyGpuContext = false;
 
 namespace stream_executor {
-namespace cuda {
-
+namespace gpu {
 namespace {
 
 // Manages the singleton map of contexts that we've created, mapping
-// from the CUcontext to the CudaContext* that we pass around internally.
-// This also manages assignment of unique ids to CudaContexts, to allow
+// from the CUcontext to the GpuContext* that we pass around internally.
+// This also manages assignment of unique ids to GpuContexts, to allow
 // for fast comparison of a context against the current context.
 //
 // CUDA-runtime-created contexts are avoided, if triple angle
 // brace launches are required, by using the scoped activations in
-// cuda_activation.h.
+// gpu/gpu_activation.h.
 class CreatedContexts {
  public:
   // Returns whether context is a member of the live set.
@@ -69,14 +68,14 @@ class CreatedContexts {
   }
 
   // Adds context to the live set, or returns it if it's already present.
-  static CudaContext* Add(CUcontext context) {
+  static GpuContext* Add(CUcontext context) {
     CHECK(context != nullptr);
     mutex_lock lock(mu_);
     auto insert_result = Live()->insert(std::make_pair(context, nullptr));
     auto it = insert_result.first;
     if (insert_result.second) {
       // context was not present in the map.  Add it.
-      it->second = MakeUnique<CudaContext>(context, next_id_++);
+      it->second = MakeUnique<GpuContext>(context, next_id_++);
     }
     return it->second.get();
   }
@@ -92,9 +91,9 @@ class CreatedContexts {
 
  private:
   // Returns the live map singleton.
-  static std::map<CUcontext, std::unique_ptr<CudaContext>> *Live() {
+  static std::map<CUcontext, std::unique_ptr<GpuContext>>* Live() {
     static auto singleton =
-        new std::map<CUcontext, std::unique_ptr<CudaContext>>;
+        new std::map<CUcontext, std::unique_ptr<GpuContext>>;
     return singleton;
   }
 
@@ -123,7 +122,7 @@ string ToString(CUresult result) {
 // created by StreamExecutor (to ensure that the CUDA runtime didn't create a
 // context behind our backs).
 CUcontext CurrentContext() {
-  CUcontext current = CUDADriver::CurrentContextOrDie();
+  CUcontext current = cuda::CurrentContextOrDie();
   if (current != nullptr && !CreatedContexts::Has(current)) {
     LOG(FATAL) << "current context was not created by the StreamExecutor "
                   "cuda_driver API: "
@@ -177,7 +176,7 @@ void SynchronizeOrDie() {
 
 struct ThreadLocalData {
   int64 id;
-  CudaContext* context;  // Only valid if id == a known good context.
+  GpuContext* context;  // Only valid if id == a known good context.
   int depth;
 };
 
@@ -185,13 +184,13 @@ SE_STATIC_THREAD_LOCAL_POD(ThreadLocalData, tls_data);
 
 }  // namespace
 
-ScopedActivateContext::ScopedActivateContext(CudaContext* cuda_context) {
+ScopedActivateContext::ScopedActivateContext(GpuContext* cuda_context) {
   if (FLAGS_gpuexec_cuda_sync_around_driver_calls) SynchronizeOrDie();
 
   auto* tls = &tls_data.get();
   tls->depth++;
   if (tls->id == cuda_context->id()) {
-    if (kVerifyCudaContext) {
+    if (kVerifyGpuContext) {
       CHECK_EQ(CurrentContext(), cuda_context->context());
     }
     DCHECK_EQ(CurrentContext(), cuda_context->context());
@@ -215,8 +214,8 @@ ScopedActivateContext::~ScopedActivateContext() {
 
   auto* tls = &tls_data.get();
 
-  if (kVerifyCudaContext) {
-    // Note that if kVerifyCudaContext is used, and contexts are deleted, it's
+  if (kVerifyGpuContext) {
+    // Note that if kVerifyGpuContext is used, and contexts are deleted, it's
     // possible this could fail in the CurrentContext() call.
     CHECK_EQ(CurrentContext(),
              tls->context == nullptr ? nullptr : tls->context->context());
@@ -242,7 +241,7 @@ namespace {
 // logging purposes. Returns "?" if the device could not be successfully
 // queried.
 string CUDAPointerToDeviceString(CUdeviceptr pointer) {
-  auto value = CUDADriver::GetPointerDevice(pointer);
+  auto value = GpuDriver::GetPointerDevice(pointer);
   if (value.ok()) {
     return absl::StrCat(value.ValueOrDie());
   }
@@ -254,7 +253,7 @@ string CUDAPointerToDeviceString(CUdeviceptr pointer) {
 // logging purposes. Returns "?" if the memory space could not be successfully
 // queried.
 string CUDAPointerToMemorySpaceString(CUdeviceptr pointer) {
-  auto value = CUDADriver::GetPointerMemorySpace(pointer);
+  auto value = GpuDriver::GetPointerMemorySpace(pointer);
   if (value.ok()) {
     return MemorySpaceString(value.ValueOrDie());
   }
@@ -267,20 +266,20 @@ string CUDAPointerToMemorySpaceString(CUdeviceptr pointer) {
 // primarily for logging purposes. Returns "error" if an error is encountered
 // in the process of querying.
 string CUDAPointersToCanAccessString(CUdeviceptr from, CUdeviceptr to) {
-  auto from_context = CUDADriver::GetPointerContext(from);
+  auto from_context = GpuDriver::GetPointerContext(from);
   if (!from_context.ok()) {
     LOG(ERROR) << "could not retrieve source pointer's context: "
                << from_context.status();
     return "error";
   }
-  auto to_context = CUDADriver::GetPointerContext(to);
+  auto to_context = GpuDriver::GetPointerContext(to);
   if (!to_context.ok()) {
     LOG(ERROR) << "could not retrieve destination pointer's context: "
                << to_context.status();
     return "error";
   }
-  return CUDADriver::CanEnablePeerAccess(from_context.ValueOrDie(),
-                                         to_context.ValueOrDie())
+  return GpuDriver::CanEnablePeerAccess(from_context.ValueOrDie(),
+                                        to_context.ValueOrDie())
              ? "true"
              : "false";
 }
@@ -308,9 +307,9 @@ static port::Status InternalInit() {
 
 }  // namespace
 
-/* static */ port::Status CUDADriver::Init() {
+/* static */ port::Status GpuDriver::Init() {
   // Cached return value from calling InternalInit(), as cuInit need only be
-  // called once, but CUDADriver::Init may be called many times.
+  // called once, but GpuDriver::Init may be called many times.
   static port::Status init_retval;
   static bool set = false;
   static mutex *init_mu = new mutex;
@@ -324,8 +323,8 @@ static port::Status InternalInit() {
   return init_retval;
 }
 
-/* static */ port::Status CUDADriver::GetDevice(int device_ordinal,
-                                                CUdevice *device) {
+/* static */ port::Status GpuDriver::GetDevice(int device_ordinal,
+                                               CUdevice* device) {
   CUresult res = tensorflow::wrap::cuDeviceGet(device, device_ordinal);
   if (res == CUDA_SUCCESS) {
     return port::Status::OK();
@@ -336,8 +335,8 @@ static port::Status InternalInit() {
       absl::StrCat("failed call to cuDeviceGet: ", ToString(res)));
 }
 
-/* static */ bool CUDADriver::GetDeviceName(CUdevice device,
-                                            string *device_name) {
+/* static */ bool GpuDriver::GetDeviceName(CUdevice device,
+                                           string* device_name) {
   static const size_t kCharLimit = 64;
   absl::InlinedVector<char, 4> chars(kCharLimit);
   CUresult res =
@@ -376,9 +375,9 @@ bool DeviceOptionsToContextFlags(const DeviceOptions &device_options,
   return true;
 }
 
-/* static */ port::Status CUDADriver::CreateContext(
-    CUdevice device, const DeviceOptions &device_options,
-    CudaContext **context) {
+/* static */ port::Status GpuDriver::CreateContext(
+    int device_ordinal, CUdevice device, const DeviceOptions& device_options,
+    GpuContext** context) {
   *context = nullptr;
 
   int flags = 0;
@@ -407,7 +406,7 @@ bool DeviceOptionsToContextFlags(const DeviceOptions &device_options,
     }
   }
 
-  former_context = CUDADriver::CurrentContextOrDie();
+  former_context = cuda::CurrentContextOrDie();
   res = tensorflow::wrap::cuDevicePrimaryCtxRetain(&new_context, device);
   if (former_context != nullptr) {
     CUdevice former_device;
@@ -454,7 +453,7 @@ bool DeviceOptionsToContextFlags(const DeviceOptions &device_options,
   return port::Status(port::error::INTERNAL, message);
 }
 
-/* static */ void CUDADriver::DestroyContext(CudaContext* context) {
+/* static */ void GpuDriver::DestroyContext(GpuContext* context) {
   if (context == nullptr) {
     return;
   }
@@ -473,9 +472,9 @@ bool DeviceOptionsToContextFlags(const DeviceOptions &device_options,
   CreatedContexts::Remove(context->context());
 }
 
-/* static */ bool CUDADriver::FuncGetAttribute(CUfunction_attribute attribute,
-                                               CUfunction func,
-                                               int *attribute_value) {
+/* static */ bool GpuDriver::FuncGetAttribute(CUfunction_attribute attribute,
+                                              CUfunction func,
+                                              int* attribute_value) {
   CUresult res =
       tensorflow::wrap::cuFuncGetAttribute(attribute_value, attribute, func);
   if (res != CUDA_SUCCESS) {
@@ -486,8 +485,8 @@ bool DeviceOptionsToContextFlags(const DeviceOptions &device_options,
   return true;
 }
 
-/* static */ bool CUDADriver::FuncSetCacheConfig(CUfunction function,
-                                                 CUfunc_cache cache_config) {
+/* static */ bool GpuDriver::FuncSetCacheConfig(CUfunction function,
+                                                CUfunc_cache cache_config) {
   CUresult res = tensorflow::wrap::cuFuncSetCacheConfig(function, cache_config);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to set CUDA kernel cache config. kernel: " << function
@@ -499,7 +498,7 @@ bool DeviceOptionsToContextFlags(const DeviceOptions &device_options,
 }
 
 /* static */ port::StatusOr<CUsharedconfig>
-CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
+GpuDriver::ContextGetSharedMemConfig(GpuContext* context) {
   CUsharedconfig shared_mem_config;
   ScopedActivateContext activation(context);
   CUresult result =
@@ -517,8 +516,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return shared_mem_config;
 }
 
-/* static */ port::Status CUDADriver::ContextSetSharedMemConfig(
-    CudaContext* context, CUsharedconfig shared_mem_config) {
+/* static */ port::Status GpuDriver::ContextSetSharedMemConfig(
+    GpuContext* context, CUsharedconfig shared_mem_config) {
   ScopedActivateContext activation(context);
   CUresult result =
       tensorflow::wrap::cuCtxSetSharedMemConfig(shared_mem_config);
@@ -536,12 +535,12 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return port::Status::OK();
 }
 
-/* static */ bool CUDADriver::LaunchKernel(
-    CudaContext* context, CUfunction function, unsigned int grid_dim_x,
+/* static */ bool GpuDriver::LaunchKernel(
+    GpuContext* context, CUfunction function, unsigned int grid_dim_x,
     unsigned int grid_dim_y, unsigned int grid_dim_z, unsigned int block_dim_x,
     unsigned int block_dim_y, unsigned int block_dim_z,
-    unsigned int shared_mem_bytes, CUstream stream, void **kernel_params,
-    void **extra) {
+    unsigned int shared_mem_bytes, CUstream stream, void** kernel_params,
+    void** extra) {
   ScopedActivateContext activation(context);
   VLOG(2) << "launching kernel: " << function << "; gdx: " << grid_dim_x
           << " gdy: " << grid_dim_y << " gdz: " << grid_dim_z
@@ -559,9 +558,9 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return true;
 }
 
-/* static */ port::Status CUDADriver::LoadCubin(CudaContext* context,
-                                                const char *cubin_bytes,
-                                                CUmodule *module) {
+/* static */ port::Status GpuDriver::LoadCubin(GpuContext* context,
+                                               const char* cubin_bytes,
+                                               CUmodule* module) {
   ScopedActivateContext activation(context);
   CUresult result =
       tensorflow::wrap::cuModuleLoadFatBinary(module, cubin_bytes);
@@ -573,9 +572,9 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return port::Status::OK();
 }
 
-/* static */ bool CUDADriver::LoadPtx(CudaContext* context,
-                                      const char *ptx_contents,
-                                      CUmodule *module) {
+/* static */ bool GpuDriver::LoadPtx(GpuContext* context,
+                                     const char* ptx_contents,
+                                     CUmodule* module) {
   port::Notification notification;
   bool ret = true;
   GetDriverExecutor()->Schedule([context, ptx_contents, module, &ret,
@@ -643,9 +642,16 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return ret;
 }
 
-/* static */ bool CUDADriver::SynchronousMemsetUint8(CudaContext* context,
-                                                     CUdeviceptr location,
-                                                     uint8 value, size_t size) {
+/* static */ bool GpuDriver::LoadHsaco(GpuContext* context,
+                                       const char* hsaco_contents,
+                                       CUmodule* module) {
+  LOG(ERROR) << "Feature not supported on CUDA platform (LoadHsaco)";
+  return false;
+}
+
+/* static */ bool GpuDriver::SynchronousMemsetUint8(GpuContext* context,
+                                                    CUdeviceptr location,
+                                                    uint8 value, size_t size) {
   ScopedActivateContext activation(context);
   CUresult res = tensorflow::wrap::cuMemsetD8(location, value, size);
   if (res != CUDA_SUCCESS) {
@@ -655,10 +661,10 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return true;
 }
 
-/* static */ bool CUDADriver::SynchronousMemsetUint32(CudaContext* context,
-                                                      CUdeviceptr location,
-                                                      uint32 value,
-                                                      size_t uint32_count) {
+/* static */ bool GpuDriver::SynchronousMemsetUint32(GpuContext* context,
+                                                     CUdeviceptr location,
+                                                     uint32 value,
+                                                     size_t uint32_count) {
   ScopedActivateContext activation(context);
   CUresult res = tensorflow::wrap::cuMemsetD32(location, value, uint32_count);
   if (res != CUDA_SUCCESS) {
@@ -668,11 +674,11 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return true;
 }
 
-/* static */ bool CUDADriver::AsynchronousMemsetUint8(CudaContext* context,
-                                                      CUdeviceptr location,
-                                                      uint8 value,
-                                                      size_t uint32_count,
-                                                      CUstream stream) {
+/* static */ bool GpuDriver::AsynchronousMemsetUint8(GpuContext* context,
+                                                     CUdeviceptr location,
+                                                     uint8 value,
+                                                     size_t uint32_count,
+                                                     CUstream stream) {
   ScopedActivateContext activation(context);
   CUresult res =
       tensorflow::wrap::cuMemsetD8Async(location, value, uint32_count, stream);
@@ -684,11 +690,11 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return true;
 }
 
-/* static */ bool CUDADriver::AsynchronousMemsetUint32(CudaContext* context,
-                                                       CUdeviceptr location,
-                                                       uint32 value,
-                                                       size_t uint32_count,
-                                                       CUstream stream) {
+/* static */ bool GpuDriver::AsynchronousMemsetUint32(GpuContext* context,
+                                                      CUdeviceptr location,
+                                                      uint32 value,
+                                                      size_t uint32_count,
+                                                      CUstream stream) {
   ScopedActivateContext activation(context);
   CUresult res =
       tensorflow::wrap::cuMemsetD32Async(location, value, uint32_count, stream);
@@ -700,10 +706,10 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return true;
 }
 
-/* static */ bool CUDADriver::AddStreamCallback(CudaContext* context,
-                                                CUstream stream,
-                                                StreamCallback callback,
-                                                void *data) {
+/* static */ bool GpuDriver::AddStreamCallback(GpuContext* context,
+                                               CUstream stream,
+                                               StreamCallback callback,
+                                               void* data) {
   // Note: flags param is required to be zero according to CUDA 6.0.
   CUresult res = tensorflow::wrap::cuStreamAddCallback(stream, callback, data,
                                                        0 /* = flags */);
@@ -714,10 +720,10 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return true;
 }
 
-/* static */ bool CUDADriver::GetModuleFunction(CudaContext *context,
-                                                CUmodule module,
-                                                const char *kernel_name,
-                                                CUfunction *function) {
+/* static */ bool GpuDriver::GetModuleFunction(GpuContext* context,
+                                               CUmodule module,
+                                               const char* kernel_name,
+                                               CUfunction* function) {
   ScopedActivateContext activated{context};
   CHECK(module != nullptr && kernel_name != nullptr);
   CUresult res =
@@ -731,11 +737,10 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return true;
 }
 
-/* static */ bool CUDADriver::GetModuleSymbol(CudaContext* context,
-                                              CUmodule module,
-                                              const char *symbol_name,
-                                              CUdeviceptr *dptr,
-                                              size_t *bytes) {
+/* static */ bool GpuDriver::GetModuleSymbol(GpuContext* context,
+                                             CUmodule module,
+                                             const char* symbol_name,
+                                             CUdeviceptr* dptr, size_t* bytes) {
   ScopedActivateContext activated{context};
   CHECK(module != nullptr && symbol_name != nullptr &&
         (dptr != nullptr || bytes != nullptr));
@@ -752,8 +757,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return true;
 }
 
-/* static */ void CUDADriver::UnloadModule(CudaContext *context,
-                                           CUmodule module) {
+/* static */ void GpuDriver::UnloadModule(GpuContext* context,
+                                          CUmodule module) {
   ScopedActivateContext activated{context};
   CUresult res = tensorflow::wrap::cuModuleUnload(module);
   if (res != CUDA_SUCCESS) {
@@ -762,8 +767,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   }
 }
 
-/* static */ port::StatusOr<CUdevice> CUDADriver::DeviceFromContext(
-    CudaContext* context) {
+/* static */ port::StatusOr<CUdevice> GpuDriver::DeviceFromContext(
+    GpuContext* context) {
   ScopedActivateContext activated{context};
   CUdevice device = -1;
   CUresult result = tensorflow::wrap::cuCtxGetDevice(&device);
@@ -776,26 +781,26 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
       absl::StrCat("failed to get device for context: ", ToString(result)));
 }
 
-/* static */ bool CUDADriver::CreateStream(CudaContext *context,
-                                           CUstream *out) {
+/* static */ bool GpuDriver::CreateStream(GpuContext* context,
+                                          CUstream* stream) {
   // TODO(leary) can we switch this to CU_STREAM_NON_BLOCKING or will that mess
   // up synchronization with respect to memsets and any other things that have
   // to occur on the default stream?
   ScopedActivateContext activated{context};
-  CUresult res = tensorflow::wrap::cuStreamCreate(out, 0);
+  CUresult res = tensorflow::wrap::cuStreamCreate(stream, 0);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "could not allocate CUDA stream for context "
                << context->context() << ": " << ToString(res);
     return false;
   }
 
-  VLOG(2) << "successfully created stream " << *out << " for context "
+  VLOG(2) << "successfully created stream " << *stream << " for context "
           << context->context() << " on thread";
   return true;
 }
 
-/* static */ void CUDADriver::DestroyStream(CudaContext* context,
-                                            CUstream *stream) {
+/* static */ void GpuDriver::DestroyStream(GpuContext* context,
+                                           CUstream* stream) {
   if (*stream == nullptr) {
     return;
   }
@@ -812,8 +817,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   }
 }
 
-/* static */ void *CUDADriver::DeviceAllocate(CudaContext *context,
-                                              uint64 bytes) {
+/* static */ void* GpuDriver::DeviceAllocate(GpuContext* context,
+                                             uint64 bytes) {
   ScopedActivateContext activated{context};
   CUdeviceptr result = 0;
   CUresult res = tensorflow::wrap::cuMemAlloc(&result, bytes);
@@ -829,8 +834,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return ptr;
 }
 
-/* static */ void CUDADriver::DeviceDeallocate(CudaContext* context,
-                                               void *location) {
+/* static */ void GpuDriver::DeviceDeallocate(GpuContext* context,
+                                              void* location) {
   ScopedActivateContext activation(context);
   CUdeviceptr pointer = absl::bit_cast<CUdeviceptr>(location);
   CUresult res = tensorflow::wrap::cuMemFree(pointer);
@@ -843,8 +848,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   }
 }
 
-/* static */ void *CUDADriver::UnifiedMemoryAllocate(CudaContext *context,
-                                                     uint64 bytes) {
+/* static */ void* GpuDriver::UnifiedMemoryAllocate(GpuContext* context,
+                                                    uint64 bytes) {
   ScopedActivateContext activation(context);
   CUdeviceptr result = 0;
   // "Portable" memory is visible to all CUDA contexts. Safe for our use model.
@@ -861,8 +866,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return ptr;
 }
 
-/* static */ void CUDADriver::UnifiedMemoryDeallocate(CudaContext *context,
-                                                      void *location) {
+/* static */ void GpuDriver::UnifiedMemoryDeallocate(GpuContext* context,
+                                                     void* location) {
   ScopedActivateContext activation(context);
   CUdeviceptr pointer = absl::bit_cast<CUdeviceptr>(location);
   CUresult res = tensorflow::wrap::cuMemFree(pointer);
@@ -875,8 +880,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   }
 }
 
-/* static */ void *CUDADriver::HostAllocate(CudaContext *context,
-                                            uint64 bytes) {
+/* static */ void* GpuDriver::HostAllocate(GpuContext* context, uint64 bytes) {
   ScopedActivateContext activation(context);
   void *host_mem = nullptr;
   // "Portable" memory is visible to all CUDA contexts. Safe for our use model.
@@ -889,8 +893,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return host_mem;
 }
 
-/* static */ void CUDADriver::HostDeallocate(CudaContext* context,
-                                             void *location) {
+/* static */ void GpuDriver::HostDeallocate(GpuContext* context,
+                                            void* location) {
   ScopedActivateContext activation(context);
   CUresult res = tensorflow::wrap::cuMemFreeHost(location);
   if (res != CUDA_SUCCESS) {
@@ -899,8 +903,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   }
 }
 
-/* static */ bool CUDADriver::HostRegister(CudaContext* context, void *location,
-                                           uint64 bytes) {
+/* static */ bool GpuDriver::HostRegister(GpuContext* context, void* location,
+                                          uint64 bytes) {
   ScopedActivateContext activation(context);
   // "Portable" memory is visible to all CUDA contexts. Safe for our use model.
   CUresult res = tensorflow::wrap::cuMemHostRegister(
@@ -913,8 +917,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return true;
 }
 
-/* static */ bool CUDADriver::HostUnregister(CudaContext* context,
-                                             void *location) {
+/* static */ bool GpuDriver::HostUnregister(GpuContext* context,
+                                            void* location) {
   ScopedActivateContext activation(context);
   CUresult res = tensorflow::wrap::cuMemHostUnregister(location);
   if (res != CUDA_SUCCESS) {
@@ -925,8 +929,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return true;
 }
 
-/* static */ port::Status CUDADriver::DestroyEvent(CudaContext* context,
-                                                   CUevent *event) {
+/* static */ port::Status GpuDriver::DestroyEvent(GpuContext* context,
+                                                  CUevent* event) {
   if (*event == nullptr) {
     return port::Status(port::error::INVALID_ARGUMENT,
                         "input event cannot be null");
@@ -953,9 +957,9 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   }
 }
 
-/* static */ port::Status CUDADriver::RecordEvent(CudaContext* context,
-                                                  CUevent event,
-                                                  CUstream stream) {
+/* static */ port::Status GpuDriver::RecordEvent(GpuContext* context,
+                                                 CUevent event,
+                                                 CUstream stream) {
   ScopedActivateContext activated{context};
   CUresult res = tensorflow::wrap::cuEventRecord(event, stream);
   switch (res) {
@@ -975,8 +979,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   }
 }
 
-/* static */ port::StatusOr<CUresult> CUDADriver::QueryEvent(
-    CudaContext *context, CUevent event) {
+/* static */ port::StatusOr<CUresult> GpuDriver::QueryEvent(GpuContext* context,
+                                                            CUevent event) {
   ScopedActivateContext activated{context};
   CUresult res = tensorflow::wrap::cuEventQuery(event);
   if (res != CUDA_SUCCESS && res != CUDA_ERROR_NOT_READY) {
@@ -988,9 +992,9 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return res;
 }
 
-/* static */ bool CUDADriver::GetEventElapsedTime(CudaContext* context,
-                                                  float *elapsed_milliseconds,
-                                                  CUevent start, CUevent stop) {
+/* static */ bool GpuDriver::GetEventElapsedTime(GpuContext* context,
+                                                 float* elapsed_milliseconds,
+                                                 CUevent start, CUevent stop) {
   ScopedActivateContext activated{context};
   // The stop event must have completed in order for cuEventElapsedTime to
   // work.
@@ -1009,9 +1013,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return true;
 }
 
-/* static */ bool CUDADriver::WaitStreamOnEvent(CudaContext* context,
-                                                CUstream stream,
-                                                CUevent event) {
+/* static */ bool GpuDriver::WaitStreamOnEvent(GpuContext* context,
+                                               CUstream stream, CUevent event) {
   ScopedActivateContext activation(context);
   CUresult res =
       tensorflow::wrap::cuStreamWaitEvent(stream, event, 0 /* = flags */);
@@ -1023,7 +1026,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return true;
 }
 
-/* static */ bool CUDADriver::SynchronizeContext(CudaContext* context) {
+/* static */ bool GpuDriver::SynchronizeContext(GpuContext* context) {
   ScopedActivateContext activation(context);
   CUresult res = tensorflow::wrap::cuCtxSynchronize();
   if (res != CUDA_SUCCESS) {
@@ -1035,8 +1038,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return true;
 }
 
-/* static */ port::Status CUDADriver::SynchronizeStream(CudaContext *context,
-                                                        CUstream stream) {
+/* static */ port::Status GpuDriver::SynchronizeStream(GpuContext* context,
+                                                       CUstream stream) {
   ScopedActivateContext activated{context};
   CHECK(stream != nullptr);
   CUresult res = tensorflow::wrap::cuStreamSynchronize(stream);
@@ -1051,8 +1054,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return port::Status::OK();
 }
 
-/* static */ bool CUDADriver::IsStreamIdle(CudaContext *context,
-                                           CUstream stream) {
+/* static */ bool GpuDriver::IsStreamIdle(GpuContext* context,
+                                          CUstream stream) {
   ScopedActivateContext activated{context};
   CHECK(stream != nullptr);
   CUresult res = tensorflow::wrap::cuStreamQuery(stream);
@@ -1066,10 +1069,10 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return false;
 }
 
-/* static */ port::Status CUDADriver::SynchronousMemcpyD2H(CudaContext *context,
-                                                           void *host_dst,
-                                                           CUdeviceptr gpu_src,
-                                                           uint64 size) {
+/* static */ port::Status GpuDriver::SynchronousMemcpyD2H(GpuContext* context,
+                                                          void* host_dst,
+                                                          CUdeviceptr gpu_src,
+                                                          uint64 size) {
   ScopedActivateContext activation(context);
   CUresult res = tensorflow::wrap::cuMemcpyDtoH(host_dst, gpu_src, size);
   if (res != CUDA_SUCCESS) {
@@ -1084,10 +1087,10 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return port::Status::OK();
 }
 
-/* static */ port::Status CUDADriver::SynchronousMemcpyH2D(CudaContext *context,
-                                                           CUdeviceptr gpu_dst,
-                                                           const void *host_src,
-                                                           uint64 size) {
+/* static */ port::Status GpuDriver::SynchronousMemcpyH2D(GpuContext* context,
+                                                          CUdeviceptr gpu_dst,
+                                                          const void* host_src,
+                                                          uint64 size) {
   ScopedActivateContext activation(context);
   CUresult res = tensorflow::wrap::cuMemcpyHtoD(gpu_dst, host_src, size);
   if (res != CUDA_SUCCESS) {
@@ -1101,10 +1104,10 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return port::Status::OK();
 }
 
-/* static */ port::Status CUDADriver::SynchronousMemcpyD2D(CudaContext *context,
-                                                           CUdeviceptr gpu_dst,
-                                                           CUdeviceptr gpu_src,
-                                                           uint64 size) {
+/* static */ port::Status GpuDriver::SynchronousMemcpyD2D(GpuContext* context,
+                                                          CUdeviceptr gpu_dst,
+                                                          CUdeviceptr gpu_src,
+                                                          uint64 size) {
   ScopedActivateContext activation(context);
   CUresult res = tensorflow::wrap::cuMemcpyDtoD(gpu_dst, gpu_src, size);
   if (res != CUDA_SUCCESS) {
@@ -1118,11 +1121,11 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return port::Status::OK();
 }
 
-/* static */ bool CUDADriver::AsynchronousMemcpyD2H(CudaContext* context,
-                                                    void *host_dst,
-                                                    CUdeviceptr gpu_src,
-                                                    uint64 size,
-                                                    CUstream stream) {
+/* static */ bool GpuDriver::AsynchronousMemcpyD2H(GpuContext* context,
+                                                   void* host_dst,
+                                                   CUdeviceptr gpu_src,
+                                                   uint64 size,
+                                                   CUstream stream) {
   ScopedActivateContext activation(context);
   CUresult res =
       tensorflow::wrap::cuMemcpyDtoHAsync(host_dst, gpu_src, size, stream);
@@ -1140,11 +1143,11 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return true;
 }
 
-/* static */ bool CUDADriver::AsynchronousMemcpyH2D(CudaContext* context,
-                                                    CUdeviceptr gpu_dst,
-                                                    const void *host_src,
-                                                    uint64 size,
-                                                    CUstream stream) {
+/* static */ bool GpuDriver::AsynchronousMemcpyH2D(GpuContext* context,
+                                                   CUdeviceptr gpu_dst,
+                                                   const void* host_src,
+                                                   uint64 size,
+                                                   CUstream stream) {
   ScopedActivateContext activation(context);
   CUresult res =
       tensorflow::wrap::cuMemcpyHtoDAsync(gpu_dst, host_src, size, stream);
@@ -1161,11 +1164,11 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return true;
 }
 
-/* static */ bool CUDADriver::AsynchronousMemcpyD2D(CudaContext* context,
-                                                    CUdeviceptr gpu_dst,
-                                                    CUdeviceptr gpu_src,
-                                                    uint64 size,
-                                                    CUstream stream) {
+/* static */ bool GpuDriver::AsynchronousMemcpyD2D(GpuContext* context,
+                                                   CUdeviceptr gpu_dst,
+                                                   CUdeviceptr gpu_src,
+                                                   uint64 size,
+                                                   CUstream stream) {
   ScopedActivateContext activation(context);
   CUresult result =
       tensorflow::wrap::cuMemcpyDtoDAsync(gpu_dst, gpu_src, size, stream);
@@ -1189,9 +1192,9 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return true;
 }
 
-/* static */ port::Status CUDADriver::CreateEvent(CudaContext* context,
-                                                  CUevent *result,
-                                                  EventFlags flags) {
+/* static */ port::Status GpuDriver::CreateEvent(GpuContext* context,
+                                                 CUevent* result,
+                                                 EventFlags flags) {
   int cuflags;
   switch (flags) {
     case EventFlags::kDefault:
@@ -1219,7 +1222,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   }
 }
 
-/* static */ int CUDADriver::GetDeviceCount() {
+/* static */ int GpuDriver::GetDeviceCount() {
   int device_count = 0;
   CUresult res = tensorflow::wrap::cuDeviceGetCount(&device_count);
   if (res != CUDA_SUCCESS) {
@@ -1233,9 +1236,9 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return device_count;
 }
 
-/* static */ port::StatusOr<CudaContext*> CUDADriver::GetPointerContext(
+/* static */ port::StatusOr<GpuContext*> GpuDriver::GetPointerContext(
     CUdeviceptr pointer) {
-  CudaContext* context = nullptr;
+  GpuContext* context = nullptr;
   CUresult result = tensorflow::wrap::cuPointerGetAttribute(
       &context, CU_POINTER_ATTRIBUTE_CONTEXT, pointer);
   if (result == CUDA_SUCCESS) {
@@ -1249,7 +1252,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                    ToString(result)));
 }
 
-/* static */ port::StatusOr<MemorySpace> CUDADriver::GetPointerMemorySpace(
+/* static */ port::StatusOr<MemorySpace> GpuDriver::GetPointerMemorySpace(
     CUdeviceptr pointer) {
   unsigned int value;
   CUresult result = tensorflow::wrap::cuPointerGetAttribute(
@@ -1273,9 +1276,9 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                    ToString(result)));
 }
 
-/* static */ port::Status CUDADriver::GetPointerAddressRange(CUdeviceptr dptr,
-                                                             CUdeviceptr *base,
-                                                             size_t *size) {
+/* static */ port::Status GpuDriver::GetPointerAddressRange(CUdeviceptr dptr,
+                                                            CUdeviceptr* base,
+                                                            size_t* size) {
   CUresult result = tensorflow::wrap::cuMemGetAddressRange(base, size, dptr);
   if (result == CUDA_SUCCESS) {
     return port::Status::OK();
@@ -1295,7 +1298,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                    reinterpret_cast<void *>(dptr), ToString(result).c_str()));
 }
 
-/* static */ port::StatusOr<CUdevice> CUDADriver::GetPointerDevice(
+/* static */ port::StatusOr<CUdevice> GpuDriver::GetPointerDevice(
     CUdeviceptr pointer) {
   auto result = GetPointerContext(pointer);
   if (!result.ok()) {
@@ -1305,9 +1308,9 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return DeviceFromContext(result.ValueOrDie());
 }
 
-/* static */ port::Status CUDADriver::GetComputeCapability(int *cc_major,
-                                                           int *cc_minor,
-                                                           CUdevice device) {
+/* static */ port::Status GpuDriver::GetComputeCapability(int* cc_major,
+                                                          int* cc_minor,
+                                                          CUdevice device) {
   *cc_major = 0;
   *cc_minor = 0;
 
@@ -1334,6 +1337,13 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   return port::Status::OK();
 }
 
+/* static */ port::Status GpuDriver::GetGpuISAVersion(int* version,
+                                                      CUdevice device) {
+  return port::Status{
+      port::error::INTERNAL,
+      "Feature not supported on CUDA platform (GetGpuISAVersion)"};
+}
+
 // Helper function that turns the integer output of cuDeviceGetAttribute to type
 // T and wraps it in a StatusOr.
 template <typename T>
@@ -1352,49 +1362,49 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
   return converted;
 }
 
-/* static */ port::StatusOr<int> CUDADriver::GetMultiprocessorCount(
+/* static */ port::StatusOr<int> GpuDriver::GetMultiprocessorCount(
     CUdevice device) {
   return GetSimpleAttribute<int>(device,
                                  CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT);
 }
 
-/* static */ port::StatusOr<int64> CUDADriver::GetMaxSharedMemoryPerCore(
+/* static */ port::StatusOr<int64> GpuDriver::GetMaxSharedMemoryPerCore(
     CUdevice device) {
   return GetSimpleAttribute<int64>(
       device, CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_MULTIPROCESSOR);
 }
 
-/* static */ port::StatusOr<int64> CUDADriver::GetMaxSharedMemoryPerBlock(
+/* static */ port::StatusOr<int64> GpuDriver::GetMaxSharedMemoryPerBlock(
     CUdevice device) {
   return GetSimpleAttribute<int64>(
       device, CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK);
 }
 
-/* static */ port::StatusOr<int64> CUDADriver::GetMaxThreadsPerMultiprocessor(
+/* static */ port::StatusOr<int64> GpuDriver::GetMaxThreadsPerMultiprocessor(
     CUdevice device) {
   return GetSimpleAttribute<int64>(
       device, CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_MULTIPROCESSOR);
 }
 
-/* static */ port::StatusOr<int64> CUDADriver::GetMaxThreadsPerBlock(
+/* static */ port::StatusOr<int64> GpuDriver::GetMaxThreadsPerBlock(
     CUdevice device) {
   return GetSimpleAttribute<int64>(device,
                                    CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_BLOCK);
 }
 
-/* static */ port::StatusOr<int64> CUDADriver::GetMaxRegistersPerBlock(
+/* static */ port::StatusOr<int64> GpuDriver::GetMaxRegistersPerBlock(
     CUdevice device) {
   return GetSimpleAttribute<int64>(device,
                                    CU_DEVICE_ATTRIBUTE_MAX_REGISTERS_PER_BLOCK);
 }
 
-/* static */ port::StatusOr<int64> CUDADriver::GetThreadsPerWarp(
+/* static */ port::StatusOr<int64> GpuDriver::GetThreadsPerWarp(
     CUdevice device) {
   return GetSimpleAttribute<int64>(device, CU_DEVICE_ATTRIBUTE_WARP_SIZE);
 }
 
-/* static */ bool CUDADriver::GetGridLimits(int *x, int *y, int *z,
-                                            CUdevice device) {
+/* static */ bool GpuDriver::GetGridLimits(int* x, int* y, int* z,
+                                           CUdevice device) {
   int value;
   CUresult res = tensorflow::wrap::cuDeviceGetAttribute(
       &value, CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_X, device);
@@ -1422,7 +1432,7 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
   return true;
 }
 
-/* static */ bool CUDADriver::GetDriverVersion(int *driver_version) {
+/* static */ bool GpuDriver::GetDriverVersion(int* driver_version) {
   CUresult res = tensorflow::wrap::cuDriverGetVersion(driver_version);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to query driver version: " << ToString(res);
@@ -1432,7 +1442,19 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
   return true;
 }
 
-/* static */ port::StatusOr<int> CUDADriver::GetDeviceAttribute(
+/* static */ bool GpuDriver::GetDeviceProperties(CUdevprop* device_properties,
+                                                 int device_ordinal) {
+  CUresult res = tensorflow::wrap::cuDeviceGetProperties(device_properties,
+                                                         device_ordinal);
+  if (res != CUDA_SUCCESS) {
+    LOG(ERROR) << "failed to query device properties: " << ToString(res);
+    return false;
+  }
+
+  return true;
+}
+
+/* static */ port::StatusOr<int> GpuDriver::GetDeviceAttribute(
     CUdevice_attribute attribute, CUdevice device) {
   int val;
   CUresult res =
@@ -1446,7 +1468,7 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
   return val;
 }
 
-/* static */ bool CUDADriver::IsEccEnabled(CUdevice device, bool *result) {
+/* static */ bool GpuDriver::IsEccEnabled(CUdevice device, bool* result) {
   int value = -1;
   CUresult res = tensorflow::wrap::cuDeviceGetAttribute(
       &value, CU_DEVICE_ATTRIBUTE_ECC_ENABLED, device);
@@ -1459,9 +1481,9 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
   return true;
 }
 
-/* static */ bool CUDADriver::GetDeviceMemoryInfo(CudaContext* context,
-                                                  int64 *free_out,
-                                                  int64 *total_out) {
+/* static */ bool GpuDriver::GetDeviceMemoryInfo(GpuContext* context,
+                                                 int64* free_out,
+                                                 int64* total_out) {
   ScopedActivateContext activation(context);
   size_t free = 0;
   size_t total = 0;
@@ -1476,8 +1498,8 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
   return true;
 }
 
-/* static */ bool CUDADriver::GetDeviceTotalMemory(CUdevice device,
-                                                   uint64 *result) {
+/* static */ bool GpuDriver::GetDeviceTotalMemory(CUdevice device,
+                                                  uint64* result) {
   size_t value = -1;
   CUresult res = tensorflow::wrap::cuDeviceTotalMem(&value, device);
   if (res != CUDA_SUCCESS) {
@@ -1489,7 +1511,7 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
   return true;
 }
 
-/* static */ string CUDADriver::GetPCIBusID(CUdevice device) {
+/* static */ string GpuDriver::GetPCIBusID(CUdevice device) {
   string pci_bus_id;
   static const int kBufferSize = 64;
   absl::InlinedVector<char, 4> chars(kBufferSize);
@@ -1504,8 +1526,8 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
   return pci_bus_id;
 }
 
-/* static */ bool CUDADriver::CanEnablePeerAccess(CudaContext* from,
-                                                  CudaContext* to) {
+/* static */ bool GpuDriver::CanEnablePeerAccess(GpuContext* from,
+                                                 GpuContext* to) {
   if (from == to) {
     return true;  // A context can always access its own memory.
   }
@@ -1533,8 +1555,8 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
   return can_access_peer;
 }
 
-/* static */ port::Status CUDADriver::EnablePeerAccess(CudaContext* from,
-                                                       CudaContext* to) {
+/* static */ port::Status GpuDriver::EnablePeerAccess(GpuContext* from,
+                                                      GpuContext* to) {
   if (from == to) {
     return port::Status::OK();  // A context can always access its own memory.
   }
@@ -1553,8 +1575,8 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
   return port::Status::OK();
 }
 
-/* static */ port::StatusOr<int> CUDADriver::GetMaxOccupiedBlocksPerCore(
-    CudaContext* context, CUfunction kernel, int threads_per_block,
+/* static */ port::StatusOr<int> GpuDriver::GetMaxOccupiedBlocksPerCore(
+    GpuContext* context, CUfunction kernel, int threads_per_block,
     size_t dynamic_shared_memory_bytes) {
   ScopedActivateContext activation(context);
 
@@ -1572,11 +1594,15 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
   return max_blocks;
 }
 
-/* static */ CUcontext CUDADriver::CurrentContextOrDie() {
+}  // namespace gpu
+
+namespace cuda {
+
+CUcontext CurrentContextOrDie() {
   CUcontext current = nullptr;
   CUresult result = tensorflow::wrap::cuCtxGetCurrent(&current);
   if (result != CUDA_SUCCESS) {
-    LOG(FATAL) << "failed to query current context: " << ToString(result);
+    LOG(FATAL) << "failed to query current context: " << gpu::ToString(result);
   }
   return current;
 }
diff --git a/tensorflow/stream_executor/cuda/cuda_driver.h b/tensorflow/stream_executor/cuda/cuda_driver.h
index bee4c8af27..5bbe6f6e62 100644
--- a/tensorflow/stream_executor/cuda/cuda_driver.h
+++ b/tensorflow/stream_executor/cuda/cuda_driver.h
@@ -18,495 +18,45 @@ limitations under the License.
 #ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DRIVER_H_
 #define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DRIVER_H_
 
-#include <stddef.h>
-#include "tensorflow/stream_executor/platform/port.h"
-
-#include "cuda/include/cuda.h"
-#include "tensorflow/stream_executor/device_options.h"
-#include "tensorflow/stream_executor/lib/status.h"
-#include "tensorflow/stream_executor/lib/statusor.h"
-#include "tensorflow/stream_executor/platform/port.h"
+#include "tensorflow/stream_executor/gpu/gpu_driver.h"
 
 namespace stream_executor {
-namespace cuda {
-
-// Identifies the memory space where an allocation resides. See
-// CUDADriver::GetPointerMemorySpace().
-enum class MemorySpace { kHost, kDevice };
-
-// Returns a casual string, such as "host" for the provided memory space.
-string MemorySpaceString(MemorySpace memory_space);
-
-class CudaContext;
-
-// CUDADriver contains wrappers for calls to the userspace library driver. It's
-// useful to isolate these calls and put basic wrappers around them to separate
-// userspace library driver behaviors from the rest of the program.
-//
-// At the moment it's simply used as a namespace.
-//
-// The calls log any specific errors internally and return whether the operation
-// was successful to the caller.
-//
-// The order of parameters is generally kept symmetric with the underlying CUDA
-// driver API.
-//
-// Links on functions are to specific documentation under
-// http://docs.nvidia.com/cuda/cuda-driver-api/
-//
-// Thread safety: these functions should not be used from signal handlers.
-class CUDADriver {
- public:
-  // Wraps a call to cuInit with logging to help indicate what has gone wrong in
-  // the case of failure. Safe to call multiple times; will be fast on all calls
-  // after the first.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__INITIALIZE.html#group__CUDA__INITIALIZE_1g0a2f1517e1bd8502c7194c3a8c134bc3
-  static port::Status Init();
-
-  // Returns the device associated with the given context.
-  // device is an outparam owned by the caller, must not be null.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g4e84b109eba36cdaaade167f34ae881e
-  static port::StatusOr<CUdevice> DeviceFromContext(CudaContext* context);
-
-  // Creates a new CUDA stream associated with the given context via
-  // cuStreamCreate.
-  // stream is an outparam owned by the caller, must not be null.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#group__CUDA__STREAM_1ga581f0c5833e21ded8b5a56594e243f4
-  static bool CreateStream(CudaContext* context, CUstream* stream);
-
-  // Destroys a CUDA stream associated with the given context.
-  // stream is owned by the caller, must not be null, and *stream is set to null
-  // if the stream is successfully destroyed.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#group__CUDA__STREAM_1g244c8833de4596bcd31a06cdf21ee758
-  static void DestroyStream(CudaContext* context, CUstream* stream);
-
-  // CUDA events can explicitly disable event TSC retrieval for some presumed
-  // performance improvement if timing is unnecessary.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g450687e75f3ff992fe01662a43d9d3db
-  enum class EventFlags { kDefault, kDisableTiming };
-
-  // Creates a new event associated with the given context.
-  // result is an outparam owned by the caller and must not be null.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g450687e75f3ff992fe01662a43d9d3db
-  static port::Status CreateEvent(CudaContext* context, CUevent* result,
-                                  EventFlags flags);
-
-  // Destroys *event and turns it into a nullptr. event may not be null, but
-  // *event may be, via cuEventDestroy
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g593ec73a8ec5a5fc031311d3e4dca1ef
-  static port::Status DestroyEvent(CudaContext* context, CUevent* event);
-
-  // Allocates a GPU memory space of size bytes associated with the given
-  // context via cuMemAlloc.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gb82d2a09844a58dd9e744dc31e8aa467
-  static void* DeviceAllocate(CudaContext* context, uint64 bytes);
-
-  // Deallocates a GPU memory space of size bytes associated with the given
-  // context via cuMemFree.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g89b3f154e17cc89b6eea277dbdf5c93a
-  static void DeviceDeallocate(CudaContext* context, void* location);
-
-  // Allocates a unified memory space of size bytes associated with the given
-  // context via cuMemAllocManaged.
-  // https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gb347ded34dc326af404aa02af5388a32
-  static void* UnifiedMemoryAllocate(CudaContext* context, uint64 bytes);
-
-  // Deallocates a unified memory space of size bytes associated with the given
-  // context via cuMemFree.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g89b3f154e17cc89b6eea277dbdf5c93a
-  static void UnifiedMemoryDeallocate(CudaContext* context, void* location);
-
-  // Allocates page-locked and CUDA-registered memory on the host via
-  // cuMemAllocHost.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gdd8311286d2c2691605362c689bc64e0
-  static void* HostAllocate(CudaContext* context, uint64 bytes);
-
-  // Deallocates a location created by HostAllocate, via cuMemFreeHost.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g62e0fdbe181dab6b1c90fa1a51c7b92c
-  static void HostDeallocate(CudaContext* context, void* location);
-
-  // Registers a memory region at location of size bytes via cuMemHostRegister.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gf0a9fe11544326dabd743b7aa6b54223
-  static bool HostRegister(CudaContext* context, void* location, uint64 bytes);
-
-  // Unregisters a memory region that was previously registered at location via
-  // cuMemHostUnregister.
-  //
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g63f450c8125359be87b7623b1c0b2a14
-  //
-  // TODO(leary) verify an error will be returned if the location wasn't
-  // previously registered.
-  static bool HostUnregister(CudaContext* context, void* location);
-
-  // Given a device ordinal, returns a device handle into the device outparam,
-  // which must not be null.
-  //
-  // N.B. these device handles do not have a corresponding destroy function in
-  // the CUDA driver API.
-  static port::Status GetDevice(int device_ordinal, CUdevice* device);
-
-  // Given a device handle, returns the name reported by the driver for the
-  // device.
-  static bool GetDeviceName(CUdevice device, string* name_out);
-
-  // Given a device to create a context for, returns a context handle into the
-  // context outparam, which must not be null.
-  //
-  // N.B. CUDA contexts are weird. They are implicitly associated with the
-  // calling thread. Current documentation on contexts and their influence on
-  // userspace processes is given here:
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g65dc0012348bc84810e2103a40d8e2cf
-  static port::Status CreateContext(CUdevice device,
-                                    const DeviceOptions& device_options,
-                                    CudaContext** context);
-
-  // Destroys the provided context via cuCtxDestroy.
-  // Don't do this while clients could still be using the context, per the docs
-  // bad things will happen.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g27a365aebb0eb548166309f58a1e8b8e
-  static void DestroyContext(CudaContext* context);
-
-  // Queries the runtime for the specified attribute of the specified function.
-  // cuFuncGetAttribute (the underlying CUDA driver API routine) only operates
-  // in terms of integer-sized values, so there's no potential for overrun (as
-  // of CUDA 5.5).
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EXEC.html#group__CUDA__EXEC_1g5e92a1b0d8d1b82cb00dcfb2de15961b
-  static bool FuncGetAttribute(CUfunction_attribute attribute,
-                               CUfunction function, int* attribute_value);
-
-  // Sets the preferred cache configuration for the specified function.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EXEC.html#group__CUDA__EXEC_1g40f8c11e81def95dc0072a375f965681
-  static bool FuncSetCacheConfig(CUfunction function,
-                                 CUfunc_cache cache_config);
-
-  // Gets the preferred shared memory bank configuration for the specified
-  // CONTEXT (not function!), either default or four- or eight-byte bank size.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g17153a1b8b8c756f7ab8505686a4ad74
-  static port::StatusOr<CUsharedconfig> ContextGetSharedMemConfig(
-      CudaContext* context);
-
-  // Sets the preferred shared memory bank configuration for the specified
-  // CONTEXT (not function!), either default or four- or eight-byte bank size.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g2574235fa643f8f251bf7bc28fac3692
-  static port::Status ContextSetSharedMemConfig(
-      CudaContext* context, CUsharedconfig shared_mem_config);
-
-  // Launches a CUDA kernel via cuLaunchKernel.
-  // TODO(leary) describe the structure of kernel_params and extra in a readable
-  // way.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EXEC.html#group__CUDA__EXEC_1gb8f3dc3031b40da29d5f9a7139e52e15
-  static bool LaunchKernel(CudaContext* context, CUfunction function,
-                           unsigned int grid_dim_x, unsigned int grid_dim_y,
-                           unsigned int grid_dim_z, unsigned int block_dim_x,
-                           unsigned int block_dim_y, unsigned int block_dim_z,
-                           unsigned int shared_mem_bytes, CUstream stream,
-                           void** kernel_params, void** extra);
-
-  // Loads ptx_contents with the CUDA driver's PTX JIT and stores the resulting
-  // handle in "module". Any error logs that are produced are logged internally.
-  static bool LoadPtx(CudaContext* context, const char* ptx_contents,
-                      CUmodule* module);
-
-  // Loads cubin_bytes with the CUDA driver's blob loading interface and stores
-  // the resulting handle in "module".
-  static port::Status LoadCubin(CudaContext* context, const char* cubin_bytes,
-                                CUmodule* module);
-
-  // Retrieves a named kernel from a loaded module, and places the resulting
-  // handle into function (outparam) on success. Neither kernel_name nor
-  // function may be null. No ownership is taken of kernel_name.
-  static bool GetModuleFunction(CudaContext* context, CUmodule module,
-                                const char* kernel_name, CUfunction* function);
-
-  // Retrieves a named global/constant symbol from a loaded module, and returns
-  // a device pointer and size of the symbol on success. symbol_name may not be
-  // null. At least one of dptr or bytes should not be null. No ownership is
-  // taken of symbol_name.
-  static bool GetModuleSymbol(CudaContext* context, CUmodule module,
-                              const char* symbol_name, CUdeviceptr* dptr,
-                              size_t* bytes);
-
-  // Unloads module from the current context via cuModuleUnload.
-  // TODO(leary) the documentation doesn't say what kind of disasters happen
-  // if you try to unload a module while its CUfunctions are in use.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MODULE.html#group__CUDA__MODULE_1g8ea3d716524369de3763104ced4ea57b
-  static void UnloadModule(CudaContext* context, CUmodule module);
-
-  // Performs a synchronous memset of the device memory segment via cuMemsetD8.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g6e582bf866e9e2fb014297bfaf354d7b
-  static bool SynchronousMemsetUint8(CudaContext* context, CUdeviceptr location,
-                                     uint8 value, size_t size);
-
-  // Performs a synchronous memset of the device memory segment via cuMemsetD32.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g983e8d8759acd1b64326317481fbf132
-  static bool SynchronousMemsetUint32(CudaContext* context,
-                                      CUdeviceptr location, uint32 value,
-                                      size_t uint32_count);
-
-  // Performs an asynchronous memset of the device memory segment via
-  // cuMemsetD8Async.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gaef08a7ccd61112f94e82f2b30d43627
-  static bool AsynchronousMemsetUint8(CudaContext* context,
-                                      CUdeviceptr location, uint8 value,
-                                      size_t uint32_count, CUstream stream);
-
-  // Performs an asynchronous memset of the device memory segment via
-  // cuMemsetD32Async.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g58229da5d30f1c0cdf667b320ec2c0f5
-  static bool AsynchronousMemsetUint32(CudaContext* context,
-                                       CUdeviceptr location, uint32 value,
-                                       size_t uint32_count, CUstream stream);
-
-  // -- Synchronous memcopies.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g4d32266788c440b0220b1a9ba5795169
-
-  static port::Status SynchronousMemcpyD2H(CudaContext* context, void* host_dst,
-                                           CUdeviceptr gpu_src, uint64 size);
-  static port::Status SynchronousMemcpyH2D(CudaContext* context,
-                                           CUdeviceptr gpu_dst,
-                                           const void* host_src, uint64 size);
-  static port::Status SynchronousMemcpyD2D(CudaContext* context,
-                                           CUdeviceptr gpu_dst,
-                                           CUdeviceptr gpu_src, uint64 size);
-
-  // -- Asynchronous memcopies.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g56f30236c7c5247f8e061b59d3268362
-
-  static bool AsynchronousMemcpyD2H(CudaContext* context, void* host_dst,
-                                    CUdeviceptr gpu_src, uint64 size,
-                                    CUstream stream);
-  static bool AsynchronousMemcpyH2D(CudaContext* context, CUdeviceptr gpu_dst,
-                                    const void* host_src, uint64 size,
-                                    CUstream stream);
-  static bool AsynchronousMemcpyD2D(CudaContext* context, CUdeviceptr gpu_dst,
-                                    CUdeviceptr gpu_src, uint64 size,
-                                    CUstream stream);
-
-  // The CUDA stream callback type signature.
-  // The data passed to AddStreamCallback is subsequently passed to this
-  // callback when it fires.
-  //
-  // Some notable things:
-  // * Callbacks must not make any CUDA API calls.
-  // * Callbacks from independent streams execute in an undefined order and may
-  //   be serialized.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#group__CUDA__STREAM_1g613d97a277d7640f4cb1c03bd51c2483
-  typedef void (*StreamCallback)(CUstream stream, CUresult status, void* data);
-
-  // Enqueues a callback operation into stream.
-  // See StreamCallback above and the NVIDIA documentation for additional
-  // details.
-  static bool AddStreamCallback(CudaContext* context, CUstream stream,
-                                StreamCallback callback, void* data);
-
-  // Causes stream to wait for event to trigger before proceeding via
-  // cuStreamWaitEvent.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#axzz334nAXAhM
-  static bool WaitStreamOnEvent(CudaContext* context, CUstream stream,
-                                CUevent event);
-
-  // Blocks the calling thread until the operations enqueued onto stream have
-  // been completed, via cuStreamSynchronize.
-  //
-  // TODO(leary) if a pathological thread enqueues operations onto the stream
-  // while another thread blocks like this, can you wind up waiting an unbounded
-  // amount of time?
-  //
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#group__CUDA__STREAM_1g15e49dd91ec15991eb7c0a741beb7dad
-  static port::Status SynchronizeStream(CudaContext* context, CUstream stream);
-
-  // Blocks the calling thread until the operations associated with the context
-  // have been completed, via cuCtxSynchronize.
-  //
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g7a54725f28d34b8c6299f0c6ca579616
-  static bool SynchronizeContext(CudaContext* context);
-
-  // Returns true if all stream tasks have completed at time of the call. Note
-  // the potential for races around this call (if another thread adds work to
-  // the stream immediately after this returns).
-  static bool IsStreamIdle(CudaContext* context, CUstream stream);
-
-  // Returns whether code in the from context can access memory in the to
-  // context via cuDeviceCanAccessPeer.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__PEER__ACCESS.html#group__CUDA__PEER__ACCESS_1g496bdaae1f632ebfb695b99d2c40f19e
-  static bool CanEnablePeerAccess(CudaContext* from, CudaContext* to);
-
-  // Enables peer access per CanEnablePeerAccess, via cuCtxEnablePeerAccess.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__PEER__ACCESS.html#group__CUDA__PEER__ACCESS_1g0889ec6728e61c05ed359551d67b3f5a
-  static port::Status EnablePeerAccess(CudaContext* from, CudaContext* to);
-
-  // Returns the elapsed milliseconds between start and stop via
-  // cuEventElapsedTime.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1gdfb1178807353bbcaa9e245da497cf97
-  static bool GetEventElapsedTime(CudaContext* context,
-                                  float* elapsed_milliseconds, CUevent start,
-                                  CUevent stop);
-
-  // Records that an event occurred when execution reaches the current point in
-  // thestream via cuEventRecord.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g95424d3be52c4eb95d83861b70fb89d1
-  static port::Status RecordEvent(CudaContext* context, CUevent event,
-                                  CUstream stream);
-
-  // Polls (without blocking) to determine the status of an event - pending or
-  // complete (or an error status).
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g6f0704d755066b0ee705749ae911deef
-  static port::StatusOr<CUresult> QueryEvent(CudaContext* context,
-                                             CUevent event);
-
-  // -- Pointer-specific calls.
-
-  // Returns the context in which pointer was allocated or registered.
-  static port::StatusOr<CudaContext*> GetPointerContext(CUdeviceptr pointer);
-
-  // Returns the device associated with the context from GetPointerContext().
-  static port::StatusOr<CUdevice> GetPointerDevice(CUdeviceptr pointer);
-
-  // Returns the memory space addressed by pointer.
-  static port::StatusOr<MemorySpace> GetPointerMemorySpace(CUdeviceptr pointer);
-
-  // Returns the base address and size of the device pointer dptr.
-  static port::Status GetPointerAddressRange(CUdeviceptr dptr,
-                                             CUdeviceptr* base, size_t* size);
-
-  // -- Device-specific calls.
-
-  // Returns the compute capability for the device; i.e (3, 5).
-  // This is currently done via the deprecated device API.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE__DEPRECATED.html#group__CUDA__DEVICE__DEPRECATED_1ge2091bbac7e1fb18c2821612115607ea
-  static port::Status GetComputeCapability(int* cc_major, int* cc_minor,
-                                           CUdevice device);
-
-  // Returns the number of multiprocessors on the device (note that the device
-  // may be multi-GPU-per-board).
-  static port::StatusOr<int> GetMultiprocessorCount(CUdevice device);
-
-  // Returns the limit on number of threads that can be resident in a single
-  // multiprocessor.
-  static port::StatusOr<int64> GetMaxThreadsPerMultiprocessor(CUdevice device);
-
-  // Returns the limit on number of threads which may be resident for a single
-  // block (cooperative thread array).
-  static port::StatusOr<int64> GetMaxThreadsPerBlock(CUdevice device);
-
-  // Returns the amount of shared memory available on a single GPU core (i.e.
-  // SM on NVIDIA devices).
-  static port::StatusOr<int64> GetMaxSharedMemoryPerCore(CUdevice device);
-
-  // Returns the amount of shared memory available for a single block
-  // (cooperative thread array).
-  static port::StatusOr<int64> GetMaxSharedMemoryPerBlock(CUdevice device);
-
-  // Returns the maximum supported number of registers per block.
-  static port::StatusOr<int64> GetMaxRegistersPerBlock(CUdevice device);
-
-  // Returns the number of threads per warp.
-  static port::StatusOr<int64> GetThreadsPerWarp(CUdevice device);
-
-  // Queries the grid limits for device with cuDeviceGetAttribute calls.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g9c3e1414f0ad901d3278a4d6645fc266
-  static bool GetGridLimits(int* x, int* y, int* z, CUdevice device);
-
-  // Gets a specific integer-valued property about the given device.
-  //
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g9c3e1414f0ad901d3278a4d6645fc266
-  static port::StatusOr<int> GetDeviceAttribute(CUdevice_attribute attribute,
-                                                CUdevice device);
-
-  // Returns whether ECC is enabled for the given CUdevice via
-  // cuDeviceGetattribute with CU_DEVICE_ATTRIBUTE_ECC_ENABLED.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g9c3e1414f0ad901d3278a4d6645fc266
-  static bool IsEccEnabled(CUdevice device, bool* result);
-
-  // Returns the total amount of memory available for allocation by the CUDA
-  // context, in bytes, via cuDeviceTotalMem.
-  static bool GetDeviceTotalMemory(CUdevice device, uint64* result);
-
-  // Returns the free amount of memory and total amount of memory, as reported
-  // by cuMemGetInfo.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g808f555540d0143a331cc42aa98835c0
-  static bool GetDeviceMemoryInfo(CudaContext* context, int64* free,
-                                  int64* total);
-
-  // Returns a PCI bus id string for the device.
-  // [domain]:[bus]:[device].[function]
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g85295e7d9745ab8f0aa80dd1e172acfc
-  static string GetPCIBusID(CUdevice device);
-
-  // -- Context- and device-independent calls.
-
-  // Returns the number of visible CUDA device via cuDeviceGetCount.
-  // This should correspond to the set of device ordinals available.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g52b5ce05cb8c5fb6831b2c0ff2887c74
-  static int GetDeviceCount();
-
-  // Returns the driver version number via cuDriverGetVersion.
-  // This is, surprisingly, NOT the actual driver version (e.g. 331.79) but,
-  // instead, the CUDA toolkit release number that this driver is compatible
-  // with; e.g. 6000 (for a CUDA 6.0 compatible driver) or 6050 (for a CUDA 6.5
-  // compatible driver).
-  //
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__VERSION.html#group__CUDA__VERSION_1g8b7a10395392e049006e61bcdc8ebe71
-  static bool GetDriverVersion(int* driver_version);
-
-  // -- Other calls
-
-  // Returns the maximum number of blocks (per multiprocessor) occupied by the
-  // specified kernel/CUfunction when launched with the specified parameters.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__OCCUPANCY.html#group__CUDA__OCCUPANCY_1gcc6e1094d05cba2cee17fe33ddd04a98
-  static port::StatusOr<int> GetMaxOccupiedBlocksPerCore(
-      CudaContext* context, CUfunction kernel, int threads_per_block,
-      size_t dynamic_shared_memory_bytes);
-
-  // Returns the current context set in CUDA. This is done by calling the cuda
-  // driver (e.g., this value is not our cached view of the current context).
-  static CUcontext CurrentContextOrDie();
-
-  // Seam for injecting an error at CUDA initialization time for testing
-  // purposes.
-  static bool driver_inject_init_error_;
-};
-
-// Ensures a context is activated within a scope.
-class ScopedActivateContext {
- public:
-  // Activates the context via cuCtxSetCurrent, if it is not the currently
-  // active context (a la cuCtxGetCurrent). Note the alternative push/pop
-  // mechanism is said by NVIDIA to be relatively slow and deprecated.
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1gbe562ee6258b4fcc272ca6478ca2a2f7
-  explicit ScopedActivateContext(CudaContext* context);
-
-  // Checks that the context has remained activated for the duration of the
-  // scope.
-  ~ScopedActivateContext();
-
- private:
-  CudaContext* to_restore_ = nullptr;
-};
-
-// CudaContext wraps a cuda CUcontext handle, and includes a unique id. The
+namespace gpu {
+// CUDAContext wraps a cuda CUcontext handle, and includes a unique id. The
 // unique id is positive, and ids are not repeated within the process.
-class CudaContext {
+class GpuContext {
  public:
-  CudaContext(CUcontext context, int64 id) : context_(context), id_(id) {}
+  GpuContext(CUcontext context, int64 id) : context_(context), id_(id) {}
 
   CUcontext context() const { return context_; }
   int64 id() const { return id_; }
 
   // Disallow copying and moving.
-  CudaContext(CudaContext&&) = delete;
-  CudaContext(const CudaContext&) = delete;
-  CudaContext& operator=(CudaContext&&) = delete;
-  CudaContext& operator=(const CudaContext&) = delete;
+  GpuContext(GpuContext&&) = delete;
+  GpuContext(const GpuContext&) = delete;
+  GpuContext& operator=(GpuContext&&) = delete;
+  GpuContext& operator=(const GpuContext&) = delete;
 
  private:
   CUcontext const context_;
   const int64 id_;
 };
 
-inline CUcontext CurrentContextOrDie() {
-  return CUDADriver::CurrentContextOrDie();
-}
+}  // namespace gpu
+
+namespace cuda {
+
+using MemorySpace = gpu::MemorySpace;
+
+using CUDADriver = gpu::GpuDriver;
+
+using ScopedActivateContext = gpu::ScopedActivateContext;
+
+using CudaContext = gpu::GpuContext;
+
+// Returns the current context set in CUDA. This is done by calling the cuda
+// driver (e.g., this value is not our cached view of the current context).
+CUcontext CurrentContextOrDie();
 
 }  // namespace cuda
 }  // namespace stream_executor
diff --git a/tensorflow/stream_executor/cuda/cuda_driver_wrapper.h b/tensorflow/stream_executor/cuda/cuda_driver_wrapper.h
index ee99908bd5..657eea8171 100644
--- a/tensorflow/stream_executor/cuda/cuda_driver_wrapper.h
+++ b/tensorflow/stream_executor/cuda/cuda_driver_wrapper.h
@@ -78,6 +78,7 @@ namespace wrap {
   __macro(cuDeviceGetCount)                             \
   __macro(cuDeviceGetName)                              \
   __macro(cuDeviceGetPCIBusId)                          \
+  __macro(cuDeviceGetProperties)                        \
   __macro(cuDevicePrimaryCtxGetState)                   \
   __macro(cuDevicePrimaryCtxRelease)                    \
   __macro(cuDevicePrimaryCtxRetain)                     \
diff --git a/tensorflow/stream_executor/cuda/cuda_event.cc b/tensorflow/stream_executor/cuda/cuda_event.cc
index 96dcf17356..fd9d4741e0 100644
--- a/tensorflow/stream_executor/cuda/cuda_event.cc
+++ b/tensorflow/stream_executor/cuda/cuda_event.cc
@@ -20,30 +20,11 @@ limitations under the License.
 #include "tensorflow/stream_executor/lib/statusor.h"
 
 namespace stream_executor {
-namespace cuda {
+namespace gpu {
 
-CUDAEvent::CUDAEvent(CUDAExecutor* parent)
-    : parent_(parent), cuda_event_(nullptr) {}
-
-CUDAEvent::~CUDAEvent() {}
-
-port::Status CUDAEvent::Init() {
-  return CUDADriver::CreateEvent(parent_->cuda_context(), &cuda_event_,
-                                 CUDADriver::EventFlags::kDisableTiming);
-}
-
-port::Status CUDAEvent::Destroy() {
-  return CUDADriver::DestroyEvent(parent_->cuda_context(), &cuda_event_);
-}
-
-port::Status CUDAEvent::Record(CUDAStream* stream) {
-  return CUDADriver::RecordEvent(parent_->cuda_context(), cuda_event_,
-                                 stream->cuda_stream());
-}
-
-Event::Status CUDAEvent::PollForStatus() {
+Event::Status GpuEvent::PollForStatus() {
   port::StatusOr<CUresult> status =
-      CUDADriver::QueryEvent(parent_->cuda_context(), cuda_event_);
+      GpuDriver::QueryEvent(parent_->gpu_context(), gpu_event_);
   if (!status.ok()) {
     LOG(ERROR) << "Error polling for event status: "
                << status.status().error_message();
@@ -62,9 +43,5 @@ Event::Status CUDAEvent::PollForStatus() {
   }
 }
 
-const CUevent& CUDAEvent::cuda_event() {
-  return cuda_event_;
-}
-
-}  // namespace cuda
+}  // namespace gpu
 }  // namespace stream_executor
diff --git a/tensorflow/stream_executor/cuda/cuda_event.h b/tensorflow/stream_executor/cuda/cuda_event.h
index f62344672e..e3596e0261 100644
--- a/tensorflow/stream_executor/cuda/cuda_event.h
+++ b/tensorflow/stream_executor/cuda/cuda_event.h
@@ -16,45 +16,12 @@ limitations under the License.
 #ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_EVENT_H_
 #define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_EVENT_H_
 
-#include "tensorflow/stream_executor/cuda/cuda_driver.h"
-#include "tensorflow/stream_executor/cuda/cuda_stream.h"
-#include "tensorflow/stream_executor/event.h"
-#include "tensorflow/stream_executor/lib/status.h"
+#include "tensorflow/stream_executor/gpu/gpu_event.h"
 
 namespace stream_executor {
 namespace cuda {
 
-// CUDAEvent wraps a CUevent in the platform-independent EventInterface
-// interface.
-class CUDAEvent : public internal::EventInterface {
- public:
-  explicit CUDAEvent(CUDAExecutor* parent);
-
-  ~CUDAEvent() override;
-
-  // Populates the CUDA-platform-specific elements of this object.
-  port::Status Init();
-
-  // Deallocates any platform-specific elements of this object. This is broken
-  // out (not part of the destructor) to allow for error reporting.
-  port::Status Destroy();
-
-  // Inserts the event at the current position into the specified stream.
-  port::Status Record(CUDAStream* stream);
-
-  // Polls the CUDA platform for the event's current status.
-  Event::Status PollForStatus();
-
-  // The underlying CUDA event element.
-  const CUevent& cuda_event();
-
- private:
-  // The Executor used to which this object and CUevent are bound.
-  CUDAExecutor* parent_;
-
-  // The underlying CUDA event element.
-  CUevent cuda_event_;
-};
+using CUDAEvent = gpu::GpuEvent;
 
 }  // namespace cuda
 }  // namespace stream_executor
diff --git a/tensorflow/stream_executor/cuda/cuda_fft.cc b/tensorflow/stream_executor/cuda/cuda_fft.cc
index 776719bc35..054b43b5b7 100644
--- a/tensorflow/stream_executor/cuda/cuda_fft.cc
+++ b/tensorflow/stream_executor/cuda/cuda_fft.cc
@@ -33,7 +33,7 @@ limitations under the License.
 #include "tensorflow/stream_executor/stream_executor_internal.h"
 
 namespace stream_executor {
-namespace cuda {
+namespace gpu {
 
 PLUGIN_REGISTRY_DEFINE_PLUGIN_ID(kCuFftPlugin);
 
@@ -45,13 +45,13 @@ namespace wrap {
 // manner on first use. This dynamic loading technique is used to avoid DSO
 // dependencies on vendor libraries which may or may not be available in the
 // deployed binary environment.
-#define STREAM_EXECUTOR_CUFFT_WRAP(__name)                       \
-  struct WrapperShim__##__name {                                 \
-    template <typename... Args>                                  \
-    cufftResult operator()(CUDAExecutor *parent, Args... args) { \
-      cuda::ScopedActivateExecutorContext sac{parent};           \
-      return ::__name(args...);                                  \
-    }                                                            \
+#define STREAM_EXECUTOR_CUFFT_WRAP(__name)                      \
+  struct WrapperShim__##__name {                                \
+    template <typename... Args>                                 \
+    cufftResult operator()(GpuExecutor *parent, Args... args) { \
+      gpu::ScopedActivateExecutorContext sac{parent};           \
+      return ::__name(args...);                                 \
+    }                                                           \
   } __name;
 
 #else
@@ -77,8 +77,8 @@ namespace wrap {
       return f;                                                           \
     }                                                                     \
     template <typename... Args>                                           \
-    cufftResult operator()(CUDAExecutor *parent, Args... args) {          \
-      cuda::ScopedActivateExecutorContext sac{parent};                    \
+    cufftResult operator()(GpuExecutor *parent, Args... args) {           \
+      gpu::ScopedActivateExecutorContext sac{parent};                     \
       return DynLoad()(args...);                                          \
     }                                                                     \
   } __name;                                                               \
@@ -145,8 +145,8 @@ cufftType CUDAFftType(fft::Type type) {
 }
 
 // Associates the given stream with the given cuFFT plan.
-bool SetStream(CUDAExecutor *parent, cufftHandle plan, Stream *stream) {
-  auto ret = wrap::cufftSetStream(parent, plan, AsCUDAStreamValue(stream));
+bool SetStream(GpuExecutor *parent, cufftHandle plan, Stream *stream) {
+  auto ret = wrap::cufftSetStream(parent, plan, AsGpuStreamValue(stream));
   if (ret != CUFFT_SUCCESS) {
     LOG(ERROR) << "failed to run cuFFT routine cufftSetStream: " << ret;
     return false;
@@ -157,7 +157,7 @@ bool SetStream(CUDAExecutor *parent, cufftHandle plan, Stream *stream) {
 }  // namespace
 
 port::Status CUDAFftPlan::Initialize(
-    CUDAExecutor *parent, Stream *stream, int rank, uint64 *elem_count,
+    GpuExecutor *parent, Stream *stream, int rank, uint64 *elem_count,
     uint64 *input_embed, uint64 input_stride, uint64 input_distance,
     uint64 *output_embed, uint64 output_stride, uint64 output_distance,
     fft::Type type, int batch_count, ScratchAllocator *scratch_allocator) {
@@ -317,7 +317,7 @@ port::Status CUDAFftPlan::Initialize(
   return port::Status::OK();
 }
 
-port::Status CUDAFftPlan::Initialize(CUDAExecutor *parent, Stream *stream,
+port::Status CUDAFftPlan::Initialize(GpuExecutor *parent, Stream *stream,
                                      int rank, uint64 *elem_count,
                                      fft::Type type,
                                      ScratchAllocator *scratch_allocator) {
@@ -549,8 +549,8 @@ bool CUDAFft::DoFftInternal(Stream *stream, fft::Plan *plan, FuncT cufftExec,
   }
 
   auto ret = cufftExec(parent_, cuda_fft_plan->GetPlan(),
-                       CUDAComplex(const_cast<InputT *>(CUDAMemory(input))),
-                       CUDAComplex(CUDAMemoryMutable(output)));
+                       GpuComplex(const_cast<InputT *>(GpuMemory(input))),
+                       GpuComplex(GpuMemoryMutable(output)));
 
   if (ret != CUFFT_SUCCESS) {
     LOG(ERROR) << "failed to run cuFFT routine: " << ret;
@@ -576,8 +576,8 @@ bool CUDAFft::DoFftWithDirectionInternal(Stream *stream, fft::Plan *plan,
   }
 
   auto ret = cufftExec(parent_, cuda_fft_plan->GetPlan(),
-                       CUDAComplex(const_cast<InputT *>(CUDAMemory(input))),
-                       CUDAComplex(CUDAMemoryMutable(output)),
+                       GpuComplex(const_cast<InputT *>(GpuMemory(input))),
+                       GpuComplex(GpuMemoryMutable(output)),
                        cuda_fft_plan->GetFftDirection());
 
   if (ret != CUFFT_SUCCESS) {
@@ -614,22 +614,22 @@ STREAM_EXECUTOR_CUDA_DEFINE_FFT(double, Z2Z, D2Z, Z2D)
 
 #undef STREAM_EXECUTOR_CUDA_DEFINE_FFT
 
-}  // namespace cuda
+}  // namespace gpu
 
 void initialize_cufft() {
   port::Status status =
       PluginRegistry::Instance()->RegisterFactory<PluginRegistry::FftFactory>(
-          cuda::kCudaPlatformId, cuda::kCuFftPlugin, "cuFFT",
+          cuda::kCudaPlatformId, gpu::kCuFftPlugin, "cuFFT",
           [](internal::StreamExecutorInterface *parent) -> fft::FftSupport * {
-            cuda::CUDAExecutor *cuda_executor =
-                dynamic_cast<cuda::CUDAExecutor *>(parent);
+            gpu::GpuExecutor *cuda_executor =
+                dynamic_cast<gpu::GpuExecutor *>(parent);
             if (cuda_executor == nullptr) {
               LOG(ERROR) << "Attempting to initialize an instance of the cuFFT "
                          << "support library with a non-CUDA StreamExecutor";
               return nullptr;
             }
 
-            return new cuda::CUDAFft(cuda_executor);
+            return new gpu::CUDAFft(cuda_executor);
           });
   if (!status.ok()) {
     LOG(ERROR) << "Unable to register cuFFT factory: "
@@ -637,7 +637,7 @@ void initialize_cufft() {
   }
 
   PluginRegistry::Instance()->SetDefaultFactory(
-      cuda::kCudaPlatformId, PluginKind::kFft, cuda::kCuFftPlugin);
+      cuda::kCudaPlatformId, PluginKind::kFft, gpu::kCuFftPlugin);
 }
 
 }  // namespace stream_executor
diff --git a/tensorflow/stream_executor/cuda/cuda_fft.h b/tensorflow/stream_executor/cuda/cuda_fft.h
index 8171e61418..0f3baeab6f 100644
--- a/tensorflow/stream_executor/cuda/cuda_fft.h
+++ b/tensorflow/stream_executor/cuda/cuda_fft.h
@@ -30,9 +30,9 @@ namespace stream_executor {
 
 class Stream;
 
-namespace cuda {
+namespace gpu {
 
-class CUDAExecutor;
+class GpuExecutor;
 
 // Opaque and unique indentifier for the cuFFT plugin.
 extern const PluginId kCuFftPlugin;
@@ -64,17 +64,17 @@ class CUDAFftPlan : public fft::Plan {
   }
 
   // Initialize function for batched plan
-  port::Status Initialize(CUDAExecutor *parent, Stream *stream, int rank,
-                          uint64 *elem_count, uint64 *input_embed,
+  port::Status Initialize(GpuExecutor* parent, Stream* stream, int rank,
+                          uint64* elem_count, uint64* input_embed,
                           uint64 input_stride, uint64 input_distance,
-                          uint64 *output_embed, uint64 output_stride,
+                          uint64* output_embed, uint64 output_stride,
                           uint64 output_distance, fft::Type type,
-                          int batch_count, ScratchAllocator *scratch_allocator);
+                          int batch_count, ScratchAllocator* scratch_allocator);
 
   // Initialize function for 1d,2d, and 3d plan
-  port::Status Initialize(CUDAExecutor *parent, Stream *stream, int rank,
-                          uint64 *elem_count, fft::Type type,
-                          ScratchAllocator *scratch_allocator);
+  port::Status Initialize(GpuExecutor* parent, Stream* stream, int rank,
+                          uint64* elem_count, fft::Type type,
+                          ScratchAllocator* scratch_allocator);
 
   port::Status UpdateScratchAllocator(Stream *stream,
                                       ScratchAllocator *scratch_allocator);
@@ -83,7 +83,7 @@ class CUDAFftPlan : public fft::Plan {
   bool IsInitialized() const { return is_initialized_; }
 
  private:
-  CUDAExecutor *parent_;
+  GpuExecutor* parent_;
   cufftHandle plan_;
   fft::Type fft_type_;
   DeviceMemory<uint8> scratch_;
@@ -96,7 +96,7 @@ class CUDAFftPlan : public fft::Plan {
 // This satisfies the platform-agnostic FftSupport interface.
 //
 // Note that the cuFFT handle that this encapsulates is implicitly tied to the
-// context (and, as a result, the device) that the parent CUDAExecutor is tied
+// context (and, as a result, the device) that the parent GpuExecutor is tied
 // to. This simply happens as an artifact of creating the cuFFT handle when a
 // CUDA context is active.
 //
@@ -104,13 +104,13 @@ class CUDAFftPlan : public fft::Plan {
 // context of parent_, so all context is explicit.
 class CUDAFft : public fft::FftSupport {
  public:
-  explicit CUDAFft(CUDAExecutor *parent) : parent_(parent) {}
+  explicit CUDAFft(GpuExecutor* parent) : parent_(parent) {}
   ~CUDAFft() override {}
 
   TENSORFLOW_STREAM_EXECUTOR_GPU_FFT_SUPPORT_OVERRIDES
 
  private:
-  CUDAExecutor *parent_;
+  GpuExecutor* parent_;
 
   // Two helper functions that execute dynload::cufftExec?2?.
 
@@ -131,7 +131,7 @@ class CUDAFft : public fft::FftSupport {
   SE_DISALLOW_COPY_AND_ASSIGN(CUDAFft);
 };
 
-}  // namespace cuda
+}  // namespace gpu
 }  // namespace stream_executor
 
 #endif  // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_FFT_H_
diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
index 51dec6b246..420f2591b8 100644
--- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
+++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
@@ -72,7 +72,7 @@ extern bool FLAGS_check_gpu_leaks;
 bool FLAGS_prefer_cubin_to_ptx = true;
 
 namespace stream_executor {
-namespace cuda {
+namespace gpu {
 
 // Hook that can be used to CUBIN-ate PTX before it is loaded into the driver.
 // It has been observed that loading both PTX and cubins into the driver library
@@ -84,17 +84,16 @@ namespace cuda {
 // variable with extern linkage and populate it from another translation unit.
 std::function<string(const string &)> g_cubinate;
 
-static CUDAEvent *AsCUDAEvent(Event *event) {
+static GpuEvent* AsGpuEvent(Event* event) {
   DCHECK(event != nullptr);
-  return static_cast<CUDAEvent *>(event->implementation());
+  return static_cast<GpuEvent*>(event->implementation());
 }
 
-
 // Given a platform-independent timer datatype, returns the internal CUDA
 // platform implementation pointer.
-static CUDATimer *AsCUDATimer(Timer *timer) {
+static GpuTimer* AsGpuTimer(Timer* timer) {
   DCHECK(timer != nullptr);
-  return static_cast<CUDATimer *>(timer->implementation());
+  return static_cast<GpuTimer*>(timer->implementation());
 }
 
 // Given const GPU memory, returns a libcuda device pointer datatype, suitable
@@ -112,48 +111,49 @@ static CUdeviceptr AsCudaDevicePtr(DeviceMemoryBase *gpu_mem) {
   return AsCudaDevicePtr(*gpu_mem);
 }
 
-CudaContext* ExtractCudaContext(CUDAExecutor *cuda_exec) {
+GpuContext* ExtractGpuContext(GpuExecutor* cuda_exec) {
   CHECK(cuda_exec != nullptr);
-  return cuda_exec->cuda_context();
+  return cuda_exec->gpu_context();
 }
 
-CUDAExecutor *ExtractCudaExecutor(StreamExecutor *stream_exec) {
-  return static_cast<CUDAExecutor *>(stream_exec->implementation());
+GpuExecutor* ExtractGpuExecutor(StreamExecutor* stream_exec) {
+  return static_cast<GpuExecutor*>(stream_exec->implementation());
 }
 
-CUDAExecutor::~CUDAExecutor() {
-  CHECK(kernel_to_gpu_binary_.empty()) << "CUDAExecutor has live kernels.";
-  CHECK(gpu_binary_to_module_.empty()) << "CUDAExecutor has loaded modules.";
+GpuExecutor::~GpuExecutor() {
+  CHECK(kernel_to_gpu_binary_.empty()) << "GpuExecutor has live kernels.";
+  CHECK(gpu_binary_to_module_.empty()) << "GpuExecutor has loaded modules.";
   if (context_ != nullptr) {
-    CUDADriver::DestroyContext(context_);
+    GpuDriver::DestroyContext(context_);
   }
 }
 
-port::Status CUDAExecutor::Init(int device_ordinal,
-                                DeviceOptions device_options) {
+port::Status GpuExecutor::Init(int device_ordinal,
+                               DeviceOptions device_options) {
   device_ordinal_ = device_ordinal;
 
-  auto status = CUDADriver::Init();
+  auto status = GpuDriver::Init();
   if (!status.ok()) {
     return status;
   }
 
-  status = CUDADriver::GetDevice(device_ordinal_, &device_);
+  status = GpuDriver::GetDevice(device_ordinal_, &device_);
   if (!status.ok()) {
     return status;
   }
 
-  status = CUDADriver::CreateContext(device_, device_options, &context_);
+  status = GpuDriver::CreateContext(device_ordinal_, device_, device_options,
+                                    &context_);
   if (!status.ok()) {
     return status;
   }
 
-  return CUDADriver::GetComputeCapability(&cc_major_, &cc_minor_, device_);
+  return GpuDriver::GetComputeCapability(&cc_major_, &cc_minor_, device_);
 }
 
-bool CUDAExecutor::FindOnDiskForComputeCapability(
+bool GpuExecutor::FindOnDiskForComputeCapability(
     absl::string_view filename, absl::string_view canonical_suffix,
-    string *found_filename) const {
+    string* found_filename) const {
   if (cc_major_ == 0 && cc_minor_ == 0) {
     return false;
   }
@@ -177,6 +177,13 @@ bool CUDAExecutor::FindOnDiskForComputeCapability(
   return false;
 }
 
+bool GpuExecutor::FindOnDiskForISAVersion(absl::string_view filename,
+                                          absl::string_view canonical_suffix,
+                                          string* found_filename) const {
+  LOG(ERROR)
+      << "Feature not supported on CUDA platform (FindOnDiskForISAVersion)";
+  return false;
+}
 // Returns the path to the running executable.
 // N.B. Derived from //knowledge/smalltalk/background_kb.cc
 // Arg: strip_exe: if true, remove the name of the executable itself from the
@@ -211,12 +218,12 @@ static string GetBinaryDir(bool strip_exe) {
   return exe_path;
 }
 
-bool CUDAExecutor::LoadModuleFromCuBin(const char *cubin, CUmodule *module) {
+bool GpuExecutor::LoadModuleFromCuBin(const char* cubin, CUmodule* module) {
   uint64_t module_refcount;
   std::tie(*module, module_refcount) = gpu_binary_to_module_[cubin];
 
   if (*module == nullptr) {
-    auto load_status = CUDADriver::LoadCubin(context_, cubin, module);
+    auto load_status = GpuDriver::LoadCubin(context_, cubin, module);
     if (!load_status.ok()) {
       LOG(ERROR) << "failed to load CUBIN: " << load_status;
       return false;
@@ -233,12 +240,12 @@ bool CUDAExecutor::LoadModuleFromCuBin(const char *cubin, CUmodule *module) {
   return true;
 }
 
-bool CUDAExecutor::LoadModuleFromPtx(const char *ptx, CUmodule *module) {
+bool GpuExecutor::LoadModuleFromPtx(const char* ptx, CUmodule* module) {
   uint64_t module_refcount;
   std::tie(*module, module_refcount) = gpu_binary_to_module_[ptx];
 
   if (*module == nullptr) {
-    if (!CUDADriver::LoadPtx(context_, ptx, module)) {
+    if (!GpuDriver::LoadPtx(context_, ptx, module)) {
       return false;
     }
     VLOG(3) << "Loaded PTX " << static_cast<const void *>(ptx) << " as module "
@@ -253,9 +260,14 @@ bool CUDAExecutor::LoadModuleFromPtx(const char *ptx, CUmodule *module) {
   return true;
 }
 
-bool CUDAExecutor::GetKernel(const MultiKernelLoaderSpec &spec,
-                             KernelBase *kernel) {
-  CUDAKernel *cuda_kernel = AsCUDAKernel(kernel);
+bool GpuExecutor::LoadModuleFromHsaco(const char* hsaco, CUmodule* module) {
+  LOG(ERROR) << "Feature not supported on CUDA platform (LoadModuleFromHsaco)";
+  return false;
+}
+
+bool GpuExecutor::GetKernel(const MultiKernelLoaderSpec& spec,
+                            KernelBase* kernel) {
+  GpuKernel* cuda_kernel = AsGpuKernel(kernel);
   CUmodule module;
   const string *kernelname;
 
@@ -295,8 +307,8 @@ bool CUDAExecutor::GetKernel(const MultiKernelLoaderSpec &spec,
     return false;
   }
   VLOG(2) << "getting function " << *kernelname << " from module " << module;
-  if (!CUDADriver::GetModuleFunction(context_, module, kernelname->c_str(),
-                                     cuda_kernel->cuda_function_ptr())) {
+  if (!GpuDriver::GetModuleFunction(context_, module, kernelname->c_str(),
+                                    cuda_kernel->gpu_function_ptr())) {
     return false;
   }
 
@@ -313,7 +325,7 @@ bool CUDAExecutor::GetKernel(const MultiKernelLoaderSpec &spec,
   return true;
 }
 
-bool CUDAExecutor::UnloadGpuBinary(const void *gpu_binary) {
+bool GpuExecutor::UnloadGpuBinary(const void* gpu_binary) {
   auto module_it = gpu_binary_to_module_.find(gpu_binary);
   if (gpu_binary_to_module_.end() == module_it) {
     VLOG(3) << "No loaded CUDA module for " << gpu_binary;
@@ -324,13 +336,13 @@ bool CUDAExecutor::UnloadGpuBinary(const void *gpu_binary) {
   VLOG(3) << "Found CUDA module " << module << " with refcount " << refcount;
   if (--refcount == 0) {
     VLOG(3) << "Unloading CUDA module " << module;
-    CUDADriver::UnloadModule(context_, module);
+    GpuDriver::UnloadModule(context_, module);
     gpu_binary_to_module_.erase(module_it);
   }
   return true;
 }
 
-void CUDAExecutor::UnloadKernel(const KernelBase *kernel) {
+void GpuExecutor::UnloadKernel(const KernelBase* kernel) {
   VLOG(3) << "Unloading kernel " << kernel << " : " << kernel->name();
 
   mutex_lock lock{in_memory_modules_mu_};
@@ -346,9 +358,9 @@ void CUDAExecutor::UnloadKernel(const KernelBase *kernel) {
   kernel_to_gpu_binary_.erase(gpu_binary_it);
 }
 
-bool CUDAExecutor::LoadModule(const MultiModuleLoaderSpec &spec,
-                              ModuleHandle *module_handle) {
-  // In CUDAExecutor we store the pointer to the GPU binary (PTX or CUBIN) as
+bool GpuExecutor::LoadModule(const MultiModuleLoaderSpec& spec,
+                             ModuleHandle* module_handle) {
+  // In GpuExecutor we store the pointer to the GPU binary (PTX or CUBIN) as
   // ModuleHandle::id().
   CUmodule cu_module;
   if (spec.has_cuda_cubin_in_memory()) {
@@ -382,25 +394,23 @@ bool CUDAExecutor::LoadModule(const MultiModuleLoaderSpec &spec,
   return false;
 }
 
-bool CUDAExecutor::UnloadModule(ModuleHandle module_handle) {
+bool GpuExecutor::UnloadModule(ModuleHandle module_handle) {
   const char *gpu_binary = reinterpret_cast<const char *>(module_handle.id());
   mutex_lock lock{in_memory_modules_mu_};
   return UnloadGpuBinary(gpu_binary);
 }
 
-bool CUDAExecutor::GetKernelMetadata(CUDAKernel *cuda_kernel,
-                                     KernelMetadata *kernel_metadata) {
+bool GpuExecutor::GetKernelMetadata(GpuKernel* cuda_kernel,
+                                    KernelMetadata* kernel_metadata) {
   int value;
-  if (!CUDADriver::FuncGetAttribute(CU_FUNC_ATTRIBUTE_NUM_REGS,
-                                    *cuda_kernel->cuda_function_ptr(),
-                                    &value)) {
+  if (!GpuDriver::FuncGetAttribute(CU_FUNC_ATTRIBUTE_NUM_REGS,
+                                   *cuda_kernel->gpu_function_ptr(), &value)) {
     return false;
   }
   kernel_metadata->set_registers_per_thread(value);
 
-  if (!CUDADriver::FuncGetAttribute(CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES,
-                                    *cuda_kernel->cuda_function_ptr(),
-                                    &value)) {
+  if (!GpuDriver::FuncGetAttribute(CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES,
+                                   *cuda_kernel->gpu_function_ptr(), &value)) {
     return false;
   }
   kernel_metadata->set_shared_memory_bytes(value);
@@ -408,13 +418,13 @@ bool CUDAExecutor::GetKernelMetadata(CUDAKernel *cuda_kernel,
   return true;
 }
 
-bool CUDAExecutor::Launch(Stream *stream, const ThreadDim &thread_dims,
-                          const BlockDim &block_dims, const KernelBase &kernel,
-                          const KernelArgsArrayBase &args) {
+bool GpuExecutor::Launch(Stream* stream, const ThreadDim& thread_dims,
+                         const BlockDim& block_dims, const KernelBase& kernel,
+                         const KernelArgsArrayBase& args) {
   CHECK_EQ(kernel.Arity(), args.number_of_arguments());
-  CUstream custream = AsCUDAStreamValue(stream);
-  const CUDAKernel *cuda_kernel = AsCUDAKernel(&kernel);
-  CUfunction cufunc = cuda_kernel->AsCUDAFunctionValue();
+  CUstream custream = AsGpuStreamValue(stream);
+  const GpuKernel* cuda_kernel = AsGpuKernel(&kernel);
+  CUfunction cufunc = cuda_kernel->AsGpuFunctionHandle();
 
   // Only perform/print the occupancy check once.  Even just checking to see
   // whether we've done an occupancy check on this kernel before isn't free
@@ -431,16 +441,16 @@ bool CUDAExecutor::Launch(Stream *stream, const ThreadDim &thread_dims,
 
   if (cuda_kernel->GetPreferredCacheConfig() !=
       KernelCacheConfig::kNoPreference) {
-    CUDADriver::FuncSetCacheConfig(cufunc, cuda_kernel->GetCUDACacheConfig());
+    GpuDriver::FuncSetCacheConfig(cufunc, cuda_kernel->GetGpuCacheConfig());
   }
 
   void **kernel_params = const_cast<void **>(args.argument_addresses().data());
 
-  if (!CUDADriver::LaunchKernel(context_, cufunc, block_dims.x, block_dims.y,
-                                block_dims.z, thread_dims.x, thread_dims.y,
-                                thread_dims.z, args.number_of_shared_bytes(),
-                                custream, kernel_params,
-                                nullptr /* = extra */)) {
+  if (!GpuDriver::LaunchKernel(context_, cufunc, block_dims.x, block_dims.y,
+                               block_dims.z, thread_dims.x, thread_dims.y,
+                               thread_dims.z, args.number_of_shared_bytes(),
+                               custream, kernel_params,
+                               nullptr /* = extra */)) {
     LOG(ERROR) << "failed to launch CUDA kernel " << kernel.name() << " with "
                << args.number_of_arguments()
                << " args; thread dim: " << thread_dims.ToString()
@@ -454,9 +464,9 @@ bool CUDAExecutor::Launch(Stream *stream, const ThreadDim &thread_dims,
 // This is a non-essential operation; if there's a failure, proceed without
 // logging an error. It's nearly certain that in case of failures, we'd never
 // get here in the first place; these are very low-impact routines.
-void CUDAExecutor::VlogOccupancyInfo(const KernelBase &kernel,
-                                     const ThreadDim &thread_dims,
-                                     const BlockDim &block_dims) {
+void GpuExecutor::VlogOccupancyInfo(const KernelBase& kernel,
+                                    const ThreadDim& thread_dims,
+                                    const BlockDim& block_dims) {
   VLOG(2) << "Computing kernel occupancy for kernel "
           << kernel.demangled_name();
   VLOG(2) << "Thread dimensions (" << thread_dims.x << ", " << thread_dims.y
@@ -475,8 +485,8 @@ void CUDAExecutor::VlogOccupancyInfo(const KernelBase &kernel,
   const DeviceDescription &device_description =
       kernel.parent()->GetDeviceDescription();
 
-  const CUDAKernel *cuda_kernel = AsCUDAKernel(&kernel);
-  CUfunction cufunc = cuda_kernel->AsCUDAFunctionValue();
+  const GpuKernel* cuda_kernel = AsGpuKernel(&kernel);
+  CUfunction cufunc = cuda_kernel->AsGpuFunctionHandle();
 
   int blocks_per_sm = CalculateOccupancy(device_description, regs_per_thread,
                                          smem_per_block, thread_dims, cufunc);
@@ -496,10 +506,11 @@ void CUDAExecutor::VlogOccupancyInfo(const KernelBase &kernel,
 // Compute and return maximum blocks per core (occupancy) based on the
 // device description, some kernel characteristics and the number of threads per
 // block.  If unable to compute occupancy, zero is returned.
-int CUDAExecutor::CalculateOccupancy(
-    const DeviceDescription &device_description, uint64 registers_per_thread,
-    uint64 shared_memory_per_block, const ThreadDim &thread_dims,
-    CUfunction func) {
+int GpuExecutor::CalculateOccupancy(const DeviceDescription& device_description,
+                                    uint64 registers_per_thread,
+                                    uint64 shared_memory_per_block,
+                                    const ThreadDim& thread_dims,
+                                    CUfunction func) {
   int suggested_blocks = 0;
   int suggested_threads = 0;
   CUresult err = tensorflow::wrap::cuOccupancyMaxPotentialBlockSize(
@@ -511,12 +522,12 @@ int CUDAExecutor::CalculateOccupancy(
 
 // Compute and return the suggested thread count to achieve ideal occupancy.
 // If the provided thread dimensions match this number, zero is returned.
-int CUDAExecutor::CompareOccupancy(int *initial_blocks,
-                                   const DeviceDescription &device_description,
-                                   uint64 registers_per_thread,
-                                   uint64 shared_memory_per_block,
-                                   const ThreadDim &thread_dims,
-                                   CUfunction func) {
+int GpuExecutor::CompareOccupancy(int* initial_blocks,
+                                  const DeviceDescription& device_description,
+                                  uint64 registers_per_thread,
+                                  uint64 shared_memory_per_block,
+                                  const ThreadDim& thread_dims,
+                                  CUfunction func) {
   int suggested_blocks = 0;
   int suggested_threads = 0;
   CUresult err = tensorflow::wrap::cuOccupancyMaxPotentialBlockSize(
@@ -531,88 +542,87 @@ int CUDAExecutor::CompareOccupancy(int *initial_blocks,
   }
 }
 
-void *CUDAExecutor::Allocate(uint64 size) {
-  return CUDADriver::DeviceAllocate(context_, size);
+void* GpuExecutor::Allocate(uint64 size) {
+  return GpuDriver::DeviceAllocate(context_, size);
 }
 
-void *CUDAExecutor::AllocateSubBuffer(DeviceMemoryBase *mem,
-                                      uint64 offset_bytes, uint64 size_bytes) {
+void* GpuExecutor::AllocateSubBuffer(DeviceMemoryBase* mem, uint64 offset_bytes,
+                                     uint64 size_bytes) {
   // offset and size are in bytes, so char* works as the pointer type.
   return reinterpret_cast<char *>(mem->opaque()) + offset_bytes;
 }
 
-void CUDAExecutor::Deallocate(DeviceMemoryBase *mem) {
+void GpuExecutor::Deallocate(DeviceMemoryBase* mem) {
   // CUDA "sub-buffers" are just pointer + offset, so no dealloc is necessary.
   if (!mem->is_sub_buffer()) {
-    CUDADriver::DeviceDeallocate(context_, mem->opaque());
+    GpuDriver::DeviceDeallocate(context_, mem->opaque());
   }
 }
 
-bool CUDAExecutor::HostMemoryRegister(void *location, uint64 size) {
+bool GpuExecutor::HostMemoryRegister(void* location, uint64 size) {
   if (location == nullptr || size == 0) {
     LOG(WARNING) << "attempting to register null or zero-sized memory: "
                  << location << "; size " << size;
   }
   VLOG(2) << "registering " << location << " size " << size;
-  return CUDADriver::HostRegister(context_, location, size);
+  return GpuDriver::HostRegister(context_, location, size);
 }
 
-bool CUDAExecutor::HostMemoryUnregister(void *location) {
+bool GpuExecutor::HostMemoryUnregister(void* location) {
   VLOG(2) << "unregistering " << location;
-  return CUDADriver::HostUnregister(context_, location);
+  return GpuDriver::HostUnregister(context_, location);
 }
 
-bool CUDAExecutor::SynchronizeAllActivity() {
-  return CUDADriver::SynchronizeContext(context_);
+bool GpuExecutor::SynchronizeAllActivity() {
+  return GpuDriver::SynchronizeContext(context_);
 }
 
-bool CUDAExecutor::SynchronousMemZero(DeviceMemoryBase *location, uint64 size) {
+bool GpuExecutor::SynchronousMemZero(DeviceMemoryBase* location, uint64 size) {
   if (reinterpret_cast<uintptr_t>(location->opaque()) % 4 == 0 &&
       size % 4 == 0) {
-    return CUDADriver::SynchronousMemsetUint32(
+    return GpuDriver::SynchronousMemsetUint32(
         context_, AsCudaDevicePtr(location), 0x0, size / 4);
   }
-  return CUDADriver::SynchronousMemsetUint8(context_, AsCudaDevicePtr(location),
-                                            0x0, size);
+  return GpuDriver::SynchronousMemsetUint8(context_, AsCudaDevicePtr(location),
+                                           0x0, size);
 }
 
-bool CUDAExecutor::SynchronousMemSet(DeviceMemoryBase *location, int value,
-                                     uint64 size) {
+bool GpuExecutor::SynchronousMemSet(DeviceMemoryBase* location, int value,
+                                    uint64 size) {
   if (reinterpret_cast<uintptr_t>(location->opaque()) % 4 == 0 &&
       size % 4 == 0) {
     // cudaMemset reinterprets "value" as a uint8.
     uint8 byte_value = static_cast<uint8>(value);
     uint32 pattern = (byte_value << 24) | (byte_value << 16) |
                      (byte_value << 8) | byte_value;
-    return CUDADriver::SynchronousMemsetUint32(
+    return GpuDriver::SynchronousMemsetUint32(
         context_, AsCudaDevicePtr(location), pattern, size / 4);
   }
-  return CUDADriver::SynchronousMemsetUint8(context_, AsCudaDevicePtr(location),
-                                            value, size);
+  return GpuDriver::SynchronousMemsetUint8(context_, AsCudaDevicePtr(location),
+                                           value, size);
 }
 
-port::Status CUDAExecutor::SynchronousMemcpy(DeviceMemoryBase *gpu_dst,
-                                             const void *host_src,
-                                             uint64 size) {
-  return CUDADriver::SynchronousMemcpyH2D(context_, AsCudaDevicePtr(gpu_dst),
-                                          host_src, size);
+port::Status GpuExecutor::SynchronousMemcpy(DeviceMemoryBase* gpu_dst,
+                                            const void* host_src, uint64 size) {
+  return GpuDriver::SynchronousMemcpyH2D(context_, AsCudaDevicePtr(gpu_dst),
+                                         host_src, size);
 }
 
-port::Status CUDAExecutor::SynchronousMemcpy(void *host_dst,
-                                             const DeviceMemoryBase &gpu_src,
-                                             uint64 size) {
-  return CUDADriver::SynchronousMemcpyD2H(context_, host_dst,
-                                          AsCudaDevicePtr(gpu_src), size);
+port::Status GpuExecutor::SynchronousMemcpy(void* host_dst,
+                                            const DeviceMemoryBase& gpu_src,
+                                            uint64 size) {
+  return GpuDriver::SynchronousMemcpyD2H(context_, host_dst,
+                                         AsCudaDevicePtr(gpu_src), size);
 }
 
-port::Status CUDAExecutor::SynchronousMemcpyDeviceToDevice(
-    DeviceMemoryBase *gpu_dst, const DeviceMemoryBase &gpu_src, uint64 size) {
-  return CUDADriver::SynchronousMemcpyD2D(context_, AsCudaDevicePtr(gpu_dst),
-                                          AsCudaDevicePtr(gpu_src), size);
+port::Status GpuExecutor::SynchronousMemcpyDeviceToDevice(
+    DeviceMemoryBase* gpu_dst, const DeviceMemoryBase& gpu_src, uint64 size) {
+  return GpuDriver::SynchronousMemcpyD2D(context_, AsCudaDevicePtr(gpu_dst),
+                                         AsCudaDevicePtr(gpu_src), size);
 }
 
-bool CUDAExecutor::MemZero(Stream *stream, DeviceMemoryBase *location,
-                           uint64 size) {
+bool GpuExecutor::MemZero(Stream* stream, DeviceMemoryBase* location,
+                          uint64 size) {
   if (reinterpret_cast<uintptr_t>(location->opaque()) % 4 == 0 &&
       size % 4 == 0) {
     return Memset32(stream, location, 0x0, size);
@@ -621,88 +631,87 @@ bool CUDAExecutor::MemZero(Stream *stream, DeviceMemoryBase *location,
   }
 }
 
-bool CUDAExecutor::Memset(Stream *stream, DeviceMemoryBase *location,
-                           uint8 pattern, uint64 size) {
+bool GpuExecutor::Memset(Stream* stream, DeviceMemoryBase* location,
+                         uint8 pattern, uint64 size) {
   VLOG(2) << "enqueueing memset8 operation onto stream " << stream
           << " at location " << location << " with size " << size
           << " and pattern " << std::hex << pattern;
-  return CUDADriver::AsynchronousMemsetUint8(
-      context_, AsCudaDevicePtr(location), pattern, size,
-      AsCUDAStreamValue(stream));
+  return GpuDriver::AsynchronousMemsetUint8(context_, AsCudaDevicePtr(location),
+                                            pattern, size,
+                                            AsGpuStreamValue(stream));
 }
 
-bool CUDAExecutor::Memset32(Stream *stream, DeviceMemoryBase *location,
-                            uint32 pattern, uint64 size) {
+bool GpuExecutor::Memset32(Stream* stream, DeviceMemoryBase* location,
+                           uint32 pattern, uint64 size) {
   VLOG(2) << "enqueueing memset32 operation onto stream " << stream
           << " at location " << location << " with size " << size
           << " and pattern " << std::hex << pattern;
   CHECK(reinterpret_cast<uintptr_t>(location->opaque()) % 4 == 0 &&
         size % 4 == 0);
-  return CUDADriver::AsynchronousMemsetUint32(
+  return GpuDriver::AsynchronousMemsetUint32(
       context_, AsCudaDevicePtr(location), pattern, size / 4,
-      AsCUDAStreamValue(stream));
+      AsGpuStreamValue(stream));
 }
 
-bool CUDAExecutor::Memcpy(Stream *stream, void *host_dst,
-                          const DeviceMemoryBase &gpu_src, uint64 size) {
-  return CUDADriver::AsynchronousMemcpyD2H(context_, host_dst,
-                                           AsCudaDevicePtr(gpu_src), size,
-                                           AsCUDAStreamValue(stream));
+bool GpuExecutor::Memcpy(Stream* stream, void* host_dst,
+                         const DeviceMemoryBase& gpu_src, uint64 size) {
+  return GpuDriver::AsynchronousMemcpyD2H(context_, host_dst,
+                                          AsCudaDevicePtr(gpu_src), size,
+                                          AsGpuStreamValue(stream));
 }
 
-bool CUDAExecutor::Memcpy(Stream *stream, DeviceMemoryBase *gpu_dst,
-                          const void *host_src, uint64 size) {
-  return CUDADriver::AsynchronousMemcpyH2D(context_, AsCudaDevicePtr(gpu_dst),
-                                           host_src, size,
-                                           AsCUDAStreamValue(stream));
+bool GpuExecutor::Memcpy(Stream* stream, DeviceMemoryBase* gpu_dst,
+                         const void* host_src, uint64 size) {
+  return GpuDriver::AsynchronousMemcpyH2D(context_, AsCudaDevicePtr(gpu_dst),
+                                          host_src, size,
+                                          AsGpuStreamValue(stream));
 }
 
-bool CUDAExecutor::MemcpyDeviceToDevice(Stream *stream,
-                                        DeviceMemoryBase *gpu_dst,
-                                        const DeviceMemoryBase &gpu_src,
-                                        uint64 size) {
-  return CUDADriver::AsynchronousMemcpyD2D(context_, AsCudaDevicePtr(gpu_dst),
-                                           AsCudaDevicePtr(gpu_src), size,
-                                           AsCUDAStreamValue(stream));
+bool GpuExecutor::MemcpyDeviceToDevice(Stream* stream,
+                                       DeviceMemoryBase* gpu_dst,
+                                       const DeviceMemoryBase& gpu_src,
+                                       uint64 size) {
+  return GpuDriver::AsynchronousMemcpyD2D(context_, AsCudaDevicePtr(gpu_dst),
+                                          AsCudaDevicePtr(gpu_src), size,
+                                          AsGpuStreamValue(stream));
 }
 
-bool CUDAExecutor::HostCallback(Stream *stream,
-                                std::function<port::Status()> callback) {
+bool GpuExecutor::HostCallback(Stream* stream,
+                               std::function<port::Status()> callback) {
   auto callback_ptr = new std::function<void()>([callback]() {
     port::Status s = callback();
     if (!s.ok()) {
       LOG(WARNING) << "Host callback failed: " << s;
     }
   });
-  return CUDADriver::AddStreamCallback(context_, AsCUDAStreamValue(stream),
-                                       InternalHostCallback, callback_ptr);
+  return GpuDriver::AddStreamCallback(context_, AsGpuStreamValue(stream),
+                                      InternalHostCallback, callback_ptr);
 }
 
-/* static */ void CUDAExecutor::InternalHostCallback(CUstream stream,
-                                                     CUresult status,
-                                                     void *data) {
+/* static */ void GpuExecutor::InternalHostCallback(CUstream stream,
+                                                    CUresult status,
+                                                    void* data) {
   std::function<void()> *callback =
       reinterpret_cast<std::function<void()> *>(data);
   (*callback)();
   delete callback;
 }
 
-port::Status CUDAExecutor::AllocateEvent(Event *event) {
-  return AsCUDAEvent(event)->Init();
+port::Status GpuExecutor::AllocateEvent(Event* event) {
+  return AsGpuEvent(event)->Init();
 }
 
-port::Status CUDAExecutor::DeallocateEvent(Event *event) {
-  return AsCUDAEvent(event)->Destroy();
+port::Status GpuExecutor::DeallocateEvent(Event* event) {
+  return AsGpuEvent(event)->Destroy();
 }
 
-port::Status CUDAExecutor::RecordEvent(Stream *stream, Event *event) {
-  return AsCUDAEvent(event)->Record(AsCUDAStream(stream));
+port::Status GpuExecutor::RecordEvent(Stream* stream, Event* event) {
+  return AsGpuEvent(event)->Record(AsGpuStream(stream));
 }
 
-port::Status CUDAExecutor::WaitForEvent(Stream *stream, Event *event) {
-  if (CUDADriver::WaitStreamOnEvent(context_,
-                                    AsCUDAStream(stream)->cuda_stream(),
-                                    AsCUDAEvent(event)->cuda_event())) {
+port::Status GpuExecutor::WaitForEvent(Stream* stream, Event* event) {
+  if (GpuDriver::WaitStreamOnEvent(context_, AsGpuStream(stream)->gpu_stream(),
+                                   AsGpuEvent(event)->gpu_event())) {
     return port::Status::OK();
   } else {
     return port::Status(
@@ -712,61 +721,61 @@ port::Status CUDAExecutor::WaitForEvent(Stream *stream, Event *event) {
   }
 }
 
-Event::Status CUDAExecutor::PollForEventStatus(Event *event) {
-  return AsCUDAEvent(event)->PollForStatus();
+Event::Status GpuExecutor::PollForEventStatus(Event* event) {
+  return AsGpuEvent(event)->PollForStatus();
 }
 
-bool CUDAExecutor::AllocateStream(Stream *stream) {
-  return AsCUDAStream(stream)->Init();
+bool GpuExecutor::AllocateStream(Stream* stream) {
+  return AsGpuStream(stream)->Init();
 }
 
-void CUDAExecutor::DeallocateStream(Stream *stream) {
-  CUDAStream *cuda_stream = AsCUDAStream(stream);
+void GpuExecutor::DeallocateStream(Stream* stream) {
+  GpuStream* cuda_stream = AsGpuStream(stream);
   if (!cuda_stream->IsIdle()) {
     LOG(ERROR) << "Deallocating stream with pending work";
   }
   cuda_stream->Destroy();
 }
 
-bool CUDAExecutor::AllocateTimer(Timer *timer) {
-  return AsCUDATimer(timer)->Init();
+bool GpuExecutor::AllocateTimer(Timer* timer) {
+  return AsGpuTimer(timer)->Init();
 }
 
-void CUDAExecutor::DeallocateTimer(Timer *timer) {
-  AsCUDATimer(timer)->Destroy();
+void GpuExecutor::DeallocateTimer(Timer* timer) {
+  AsGpuTimer(timer)->Destroy();
 }
 
-bool CUDAExecutor::CreateStreamDependency(Stream *dependent, Stream *other) {
-  CUevent other_completed_event = *AsCUDAStream(other)->completed_event();
-  bool ok = CUDADriver::RecordEvent(context_, other_completed_event,
-                                    AsCUDAStreamValue(other))
-      .ok();
+bool GpuExecutor::CreateStreamDependency(Stream* dependent, Stream* other) {
+  CUevent other_completed_event = *AsGpuStream(other)->completed_event();
+  bool ok = GpuDriver::RecordEvent(context_, other_completed_event,
+                                   AsGpuStreamValue(other))
+                .ok();
   if (!ok) {
     LOG(ERROR) << "failed to record completion event; "
                   "therefore, failed to create inter-stream dependency";
     return false;
   }
 
-  return CUDADriver::WaitStreamOnEvent(context_, AsCUDAStreamValue(dependent),
-                                       other_completed_event);
+  return GpuDriver::WaitStreamOnEvent(context_, AsGpuStreamValue(dependent),
+                                      other_completed_event);
 }
 
-bool CUDAExecutor::StartTimer(Stream *stream, Timer *timer) {
-  return AsCUDATimer(timer)->Start(AsCUDAStream(stream));
+bool GpuExecutor::StartTimer(Stream* stream, Timer* timer) {
+  return AsGpuTimer(timer)->Start(AsGpuStream(stream));
 }
 
-bool CUDAExecutor::StopTimer(Stream *stream, Timer *timer) {
-  return AsCUDATimer(timer)->Stop(AsCUDAStream(stream));
+bool GpuExecutor::StopTimer(Stream* stream, Timer* timer) {
+  return AsGpuTimer(timer)->Stop(AsGpuStream(stream));
 }
 
-port::Status CUDAExecutor::BlockHostUntilDone(Stream *stream) {
-  return CUDADriver::SynchronizeStream(context_, AsCUDAStreamValue(stream));
+port::Status GpuExecutor::BlockHostUntilDone(Stream* stream) {
+  return GpuDriver::SynchronizeStream(context_, AsGpuStreamValue(stream));
 }
 
-blas::BlasSupport *CUDAExecutor::CreateBlas() {
+blas::BlasSupport* GpuExecutor::CreateBlas() {
   PluginRegistry *registry = PluginRegistry::Instance();
   port::StatusOr<PluginRegistry::BlasFactory> status =
-      registry->GetFactory<PluginRegistry::BlasFactory>(kCudaPlatformId,
+      registry->GetFactory<PluginRegistry::BlasFactory>(cuda::kCudaPlatformId,
                                                         plugin_config_.blas());
   if (!status.ok()) {
     LOG(ERROR) << "Unable to retrieve BLAS factory: "
@@ -777,10 +786,10 @@ blas::BlasSupport *CUDAExecutor::CreateBlas() {
   return status.ValueOrDie()(this);
 }
 
-dnn::DnnSupport *CUDAExecutor::CreateDnn() {
+dnn::DnnSupport* GpuExecutor::CreateDnn() {
   PluginRegistry *registry = PluginRegistry::Instance();
   port::StatusOr<PluginRegistry::DnnFactory> status =
-      registry->GetFactory<PluginRegistry::DnnFactory>(kCudaPlatformId,
+      registry->GetFactory<PluginRegistry::DnnFactory>(cuda::kCudaPlatformId,
                                                        plugin_config_.dnn());
   if (!status.ok()) {
     LOG(ERROR) << "Unable to retrieve DNN factory: "
@@ -791,10 +800,10 @@ dnn::DnnSupport *CUDAExecutor::CreateDnn() {
   return status.ValueOrDie()(this);
 }
 
-fft::FftSupport *CUDAExecutor::CreateFft() {
+fft::FftSupport* GpuExecutor::CreateFft() {
   PluginRegistry *registry = PluginRegistry::Instance();
   port::StatusOr<PluginRegistry::FftFactory> status =
-      registry->GetFactory<PluginRegistry::FftFactory>(kCudaPlatformId,
+      registry->GetFactory<PluginRegistry::FftFactory>(cuda::kCudaPlatformId,
                                                        plugin_config_.fft());
   if (!status.ok()) {
     LOG(ERROR) << "Unable to retrieve FFT factory: "
@@ -805,10 +814,10 @@ fft::FftSupport *CUDAExecutor::CreateFft() {
   return status.ValueOrDie()(this);
 }
 
-rng::RngSupport *CUDAExecutor::CreateRng() {
+rng::RngSupport* GpuExecutor::CreateRng() {
   PluginRegistry *registry = PluginRegistry::Instance();
   port::StatusOr<PluginRegistry::RngFactory> status =
-      registry->GetFactory<PluginRegistry::RngFactory>(kCudaPlatformId,
+      registry->GetFactory<PluginRegistry::RngFactory>(cuda::kCudaPlatformId,
                                                        plugin_config_.rng());
   if (!status.ok()) {
     LOG(ERROR) << "Unable to retrieve RNG factory: "
@@ -820,23 +829,21 @@ rng::RngSupport *CUDAExecutor::CreateRng() {
 }
 
 // TODO(rspringer): Remove in b/18544742.
-bool CUDAExecutor::SupportsDnn() const {
-  return true;
-}
+bool GpuExecutor::SupportsDnn() const { return true; }
 
-bool CUDAExecutor::CanEnablePeerAccessTo(StreamExecutorInterface *other) {
-  CUDAExecutor *cuda_other = static_cast<CUDAExecutor *>(other);
-  return CUDADriver::CanEnablePeerAccess(context_, cuda_other->context_);
+bool GpuExecutor::CanEnablePeerAccessTo(StreamExecutorInterface* other) {
+  GpuExecutor* cuda_other = static_cast<GpuExecutor*>(other);
+  return GpuDriver::CanEnablePeerAccess(context_, cuda_other->context_);
 }
 
-port::Status CUDAExecutor::EnablePeerAccessTo(StreamExecutorInterface *other) {
-  CUDAExecutor *cuda_other = static_cast<CUDAExecutor *>(other);
-  return CUDADriver::EnablePeerAccess(context_, cuda_other->context_);
+port::Status GpuExecutor::EnablePeerAccessTo(StreamExecutorInterface* other) {
+  GpuExecutor* cuda_other = static_cast<GpuExecutor*>(other);
+  return GpuDriver::EnablePeerAccess(context_, cuda_other->context_);
 }
 
-SharedMemoryConfig CUDAExecutor::GetDeviceSharedMemoryConfig() {
+SharedMemoryConfig GpuExecutor::GetDeviceSharedMemoryConfig() {
   port::StatusOr<CUsharedconfig> cuda_config =
-      CUDADriver::ContextGetSharedMemConfig(context_);
+      GpuDriver::ContextGetSharedMemConfig(context_);
   if (!cuda_config.ok()) {
     // Don't log; the failed call will log necessary output.
     return SharedMemoryConfig::kDefault;
@@ -855,7 +862,7 @@ SharedMemoryConfig CUDAExecutor::GetDeviceSharedMemoryConfig() {
   }
 }
 
-port::Status CUDAExecutor::SetDeviceSharedMemoryConfig(
+port::Status GpuExecutor::SetDeviceSharedMemoryConfig(
     SharedMemoryConfig config) {
   CUsharedconfig cuda_config;
   switch (config) {
@@ -872,21 +879,21 @@ port::Status CUDAExecutor::SetDeviceSharedMemoryConfig(
       LOG(FATAL) << "Invalid shared memory configuration specified: "
                  << static_cast<int>(config);
   }
-  return CUDADriver::ContextSetSharedMemConfig(context_, cuda_config);
+  return GpuDriver::ContextSetSharedMemConfig(context_, cuda_config);
 }
 
-bool CUDAExecutor::DeviceMemoryUsage(int64 *free, int64 *total) const {
-  return CUDADriver::GetDeviceMemoryInfo(context_, free, total);
+bool GpuExecutor::DeviceMemoryUsage(int64* free, int64* total) const {
+  return GpuDriver::GetDeviceMemoryInfo(context_, free, total);
 }
 
-bool CUDAExecutor::GetSymbol(const string &symbol_name,
-                             ModuleHandle module_handle, void **mem,
-                             size_t *bytes) {
+bool GpuExecutor::GetSymbol(const string& symbol_name,
+                            ModuleHandle module_handle, void** mem,
+                            size_t* bytes) {
   auto lookup_in_module = [&](CUmodule module) {
     CHECK(module != nullptr);
-    return CUDADriver::GetModuleSymbol(context_, module, symbol_name.c_str(),
-                                       reinterpret_cast<CUdeviceptr *>(mem),
-                                       bytes);
+    return GpuDriver::GetModuleSymbol(context_, module, symbol_name.c_str(),
+                                      reinterpret_cast<CUdeviceptr*>(mem),
+                                      bytes);
   };
 
   {  // give limited scope to mutex_lock
@@ -908,13 +915,13 @@ bool CUDAExecutor::GetSymbol(const string &symbol_name,
   return false;
 }
 
-bool CUDAExecutor::FillBlockDimLimit(BlockDim *block_dim_limit) const {
+bool GpuExecutor::FillBlockDimLimit(BlockDim* block_dim_limit) const {
   // The BlockDim name is a mismatch against these GRID_DIM_* queries because
   // we use BlockDims to express the dimensions of blocks within a grid
   // (as opposed to ThreadDim which expresses the dimensions of threads
   // within a block).
   int x, y, z;
-  if (!CUDADriver::GetGridLimits(&x, &y, &z, device_)) {
+  if (!GpuDriver::GetGridLimits(&x, &y, &z, device_)) {
     return false;
   }
 
@@ -924,35 +931,35 @@ bool CUDAExecutor::FillBlockDimLimit(BlockDim *block_dim_limit) const {
   return true;
 }
 
-bool CUDAExecutor::SupportsBlas() const { return true; }
+bool GpuExecutor::SupportsBlas() const { return true; }
 
-bool CUDAExecutor::SupportsFft() const { return true; }
+bool GpuExecutor::SupportsFft() const { return true; }
 
-bool CUDAExecutor::SupportsRng() const { return true; }
+bool GpuExecutor::SupportsRng() const { return true; }
 
 std::unique_ptr<internal::EventInterface>
-CUDAExecutor::CreateEventImplementation() {
-  return std::unique_ptr<internal::EventInterface>(new CUDAEvent(this));
+GpuExecutor::CreateEventImplementation() {
+  return std::unique_ptr<internal::EventInterface>(new GpuEvent(this));
 }
 
 std::unique_ptr<internal::KernelInterface>
-CUDAExecutor::CreateKernelImplementation() {
-  return std::unique_ptr<internal::KernelInterface>(new CUDAKernel());
+GpuExecutor::CreateKernelImplementation() {
+  return std::unique_ptr<internal::KernelInterface>(new GpuKernel());
 }
 
 std::unique_ptr<internal::StreamInterface>
-CUDAExecutor::GetStreamImplementation() {
-  return std::unique_ptr<internal::StreamInterface>(new CUDAStream(this));
+GpuExecutor::GetStreamImplementation() {
+  return std::unique_ptr<internal::StreamInterface>(new GpuStream(this));
 }
 
 std::unique_ptr<internal::TimerInterface>
-CUDAExecutor::GetTimerImplementation() {
-  return std::unique_ptr<internal::TimerInterface>(new CUDATimer(this));
+GpuExecutor::GetTimerImplementation() {
+  return std::unique_ptr<internal::TimerInterface>(new GpuTimer(this));
 }
 
-void *CUDAExecutor::GpuContextHack() { return context_; }
+void* GpuExecutor::GpuContextHack() { return context_; }
 
-CudaContext* CUDAExecutor::cuda_context() { return context_; }
+GpuContext* GpuExecutor::gpu_context() { return context_; }
 
 // Attempts to read the NUMA node corresponding to the GPU device's PCI bus out
 // of SysFS. Returns -1 if it cannot.
@@ -1019,21 +1026,21 @@ static int TryToReadNumaNode(const string &pci_bus_id, int device_ordinal) {
 #endif
 }
 
-
-DeviceDescription *CUDAExecutor::PopulateDeviceDescription() const {
+DeviceDescription* GpuExecutor::PopulateDeviceDescription() const {
   internal::DeviceDescriptionBuilder builder;
 
   {
     int driver_version = 0;
-    (void)CUDADriver::GetDriverVersion(&driver_version);
+    (void)GpuDriver::GetDriverVersion(&driver_version);
     string augmented_driver_version = port::Printf(
         "%d (%s)", driver_version,
-        DriverVersionStatusToString(Diagnostician::FindDsoVersion()).c_str());
+        cuda::DriverVersionStatusToString(Diagnostician::FindDsoVersion())
+            .c_str());
     builder.set_driver_version(augmented_driver_version);
   }
 
   {
-    string pci_bus_id = CUDADriver::GetPCIBusID(device_);
+    string pci_bus_id = GpuDriver::GetPCIBusID(device_);
 
     // Lower the hex characters to match sysfs.
     pci_bus_id = port::Lowercase(pci_bus_id);
@@ -1046,43 +1053,43 @@ DeviceDescription *CUDAExecutor::PopulateDeviceDescription() const {
 
   {
     builder.set_threads_per_block_limit(
-        CUDADriver::GetDeviceAttribute(
-            CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_BLOCK, device_)
+        GpuDriver::GetDeviceAttribute(CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_BLOCK,
+                                      device_)
             .ValueOrDie());
 
     ThreadDim thread_dim_limit;
-    thread_dim_limit.x = CUDADriver::GetDeviceAttribute(
+    thread_dim_limit.x = GpuDriver::GetDeviceAttribute(
                              CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_X, device_)
                              .ValueOrDie();
-    thread_dim_limit.y = CUDADriver::GetDeviceAttribute(
+    thread_dim_limit.y = GpuDriver::GetDeviceAttribute(
                              CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_Y, device_)
                              .ValueOrDie();
-    thread_dim_limit.z = CUDADriver::GetDeviceAttribute(
+    thread_dim_limit.z = GpuDriver::GetDeviceAttribute(
                              CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_Z, device_)
                              .ValueOrDie();
     builder.set_thread_dim_limit(thread_dim_limit);
 
     int clock_rate =
-        CUDADriver::GetDeviceAttribute(CU_DEVICE_ATTRIBUTE_CLOCK_RATE, device_)
+        GpuDriver::GetDeviceAttribute(CU_DEVICE_ATTRIBUTE_CLOCK_RATE, device_)
             .ValueOrDie();
     builder.set_clock_rate_ghz(static_cast<float>(clock_rate) / 1e6);
   }
 
   {
     bool ecc_enabled = false;
-    (void)CUDADriver::IsEccEnabled(device_, &ecc_enabled);
+    (void)GpuDriver::IsEccEnabled(device_, &ecc_enabled);
     builder.set_ecc_enabled(ecc_enabled);
   }
 
   {
     uint64 device_memory_size = -1;
-    (void)CUDADriver::GetDeviceTotalMemory(device_, &device_memory_size);
+    (void)GpuDriver::GetDeviceTotalMemory(device_, &device_memory_size);
     builder.set_device_memory_size(device_memory_size);
   }
 
-  port::StatusOr<int> mem_clock_khz = CUDADriver::GetDeviceAttribute(
+  port::StatusOr<int> mem_clock_khz = GpuDriver::GetDeviceAttribute(
       CU_DEVICE_ATTRIBUTE_MEMORY_CLOCK_RATE, device_ordinal_);
-  port::StatusOr<int> mem_bus_width_bits = CUDADriver::GetDeviceAttribute(
+  port::StatusOr<int> mem_bus_width_bits = GpuDriver::GetDeviceAttribute(
       CU_DEVICE_ATTRIBUTE_GLOBAL_MEMORY_BUS_WIDTH, device_ordinal_);
   if (mem_clock_khz.ok() && mem_bus_width_bits.ok()) {
     // Times 2 because HBM is DDR memory; it gets two data bits per each data
@@ -1100,7 +1107,7 @@ DeviceDescription *CUDAExecutor::PopulateDeviceDescription() const {
 
   {
     string device_name;
-    (void)CUDADriver::GetDeviceName(device_, &device_name);
+    (void)GpuDriver::GetDeviceName(device_, &device_name);
     builder.set_name(device_name);
   }
 
@@ -1114,19 +1121,19 @@ DeviceDescription *CUDAExecutor::PopulateDeviceDescription() const {
   builder.set_device_vendor("NVIDIA Corporation");
   builder.set_cuda_compute_capability(cc_major_, cc_minor_);
   builder.set_shared_memory_per_core(
-      CUDADriver::GetMaxSharedMemoryPerCore(device_).ValueOrDie());
+      GpuDriver::GetMaxSharedMemoryPerCore(device_).ValueOrDie());
   builder.set_shared_memory_per_block(
-      CUDADriver::GetMaxSharedMemoryPerBlock(device_).ValueOrDie());
+      GpuDriver::GetMaxSharedMemoryPerBlock(device_).ValueOrDie());
   builder.set_core_count(
-      CUDADriver::GetMultiprocessorCount(device_).ValueOrDie());
+      GpuDriver::GetMultiprocessorCount(device_).ValueOrDie());
   builder.set_threads_per_core_limit(
-      CUDADriver::GetMaxThreadsPerMultiprocessor(device_).ValueOrDie());
+      GpuDriver::GetMaxThreadsPerMultiprocessor(device_).ValueOrDie());
   builder.set_registers_per_block_limit(
-      CUDADriver::GetMaxRegistersPerBlock(device_).ValueOrDie());
+      GpuDriver::GetMaxRegistersPerBlock(device_).ValueOrDie());
   builder.set_threads_per_warp(
-      CUDADriver::GetThreadsPerWarp(device_).ValueOrDie());
+      GpuDriver::GetThreadsPerWarp(device_).ValueOrDie());
   builder.set_registers_per_core_limit(
-      CUDADriver::GetDeviceAttribute(
+      GpuDriver::GetDeviceAttribute(
           CU_DEVICE_ATTRIBUTE_MAX_REGISTERS_PER_MULTIPROCESSOR, device_)
           .ValueOrDie());
 
@@ -1134,11 +1141,11 @@ DeviceDescription *CUDAExecutor::PopulateDeviceDescription() const {
   return built.release();
 }
 
-}  // namespace cuda
+}  // namespace gpu
 
 void initialize_cuda_gpu_executor() {
-  *internal::MakeCUDAExecutorImplementation() = [](const PluginConfig &config) {
-    return new cuda::CUDAExecutor{config};
+  *internal::MakeCUDAExecutorImplementation() = [](const PluginConfig& config) {
+    return new gpu::GpuExecutor{config};
   };
 }
 
diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.h b/tensorflow/stream_executor/cuda/cuda_gpu_executor.h
index ae8e4abf92..9d02c7516c 100644
--- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.h
+++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.h
@@ -22,289 +22,12 @@ limitations under the License.
 #ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_GPU_EXECUTOR_H_
 #define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_GPU_EXECUTOR_H_
 
-#include <set>
-#include <unordered_map>
-
-#include "absl/strings/string_view.h"
-#include "tensorflow/stream_executor/cuda/cuda_kernel.h"
-#include "tensorflow/stream_executor/event.h"
-#include "tensorflow/stream_executor/lib/status.h"
-#include "tensorflow/stream_executor/lib/statusor.h"
-#include "tensorflow/stream_executor/platform.h"
-#include "tensorflow/stream_executor/platform/mutex.h"
-#include "tensorflow/stream_executor/platform/port.h"
-#include "tensorflow/stream_executor/platform/thread_annotations.h"
-#include "tensorflow/stream_executor/stream_executor_internal.h"
+#include "tensorflow/stream_executor/gpu/gpu_executor.h"
 
 namespace stream_executor {
 namespace cuda {
 
-// CUDA-platform implementation of the platform-agnostic
-// StreamExecutorInferface.
-class CUDAExecutor : public internal::StreamExecutorInterface {
- public:
-  // sub_platform indicates the subplatform used in this executor; it must
-  // be a CUDA type.
-  explicit CUDAExecutor(const PluginConfig &plugin_config)
-      : device_(0),
-        context_(nullptr),
-        device_ordinal_(0),
-        cc_major_(0),
-        cc_minor_(0),
-        plugin_config_(plugin_config) {}
-
-  // See the corresponding StreamExecutor methods for method comments on the
-  // following overrides.
-
-  ~CUDAExecutor() override;
-
-  port::Status Init(int device_ordinal, DeviceOptions device_options) override;
-
-  bool GetKernel(const MultiKernelLoaderSpec &spec,
-                 KernelBase *kernel) override;
-  void UnloadKernel(const KernelBase *kernel) override;
-  bool LoadModule(const MultiModuleLoaderSpec &spec,
-                  ModuleHandle *module_handle) override;
-  bool UnloadModule(ModuleHandle module_handle) override;
-
-  bool Launch(Stream *stream, const ThreadDim &thread_dims,
-              const BlockDim &block_dims, const KernelBase &k,
-              const KernelArgsArrayBase &args) override;
-
-  int CalculateOccupancy(const DeviceDescription &device_description,
-                         uint64 registers_per_thread,
-                         uint64 shared_memory_per_block,
-                         const ThreadDim &thread_dims, CUfunction func);
-
-  int CompareOccupancy(int *initial_blocks,
-                       const DeviceDescription &device_description,
-                       uint64 registers_per_thread,
-                       uint64 shared_memory_per_block,
-                       const ThreadDim &thread_dims, CUfunction func);
-
-  void *Allocate(uint64 size) override;
-
-  void *AllocateSubBuffer(DeviceMemoryBase *mem, uint64 offset_bytes,
-                          uint64 size_bytes) override;
-
-  void Deallocate(DeviceMemoryBase *mem) override;
-
-  void *UnifiedMemoryAllocate(uint64 size) override {
-    return CUDADriver::UnifiedMemoryAllocate(context_, size);
-  }
-
-  void UnifiedMemoryDeallocate(void *location) override {
-    return CUDADriver::UnifiedMemoryDeallocate(context_, location);
-  }
-
-  // CUDA allocation/registration functions are necessary because the driver
-  // internally sets up buffers for DMA operations (and page locks them).
-  // There's no external interface for us to otherwise control these DMA
-  // settings.
-  void *HostMemoryAllocate(uint64 size) override {
-    return CUDADriver::HostAllocate(context_, size);
-  }
-
-  void HostMemoryDeallocate(void *location) override {
-    return CUDADriver::HostDeallocate(context_, location);
-  }
-
-  bool HostMemoryRegister(void *location, uint64 size) override;
-
-  bool HostMemoryUnregister(void *location) override;
-
-  bool SynchronizeAllActivity() override;
-
-  bool SynchronousMemZero(DeviceMemoryBase *location, uint64 size) override;
-
-  bool SynchronousMemSet(DeviceMemoryBase *location, int value,
-                         uint64 size) override;
-
-  port::Status SynchronousMemcpy(DeviceMemoryBase *gpu_dst,
-                                 const void *host_src, uint64 size) override;
-
-  port::Status SynchronousMemcpy(void *host_dst,
-                                 const DeviceMemoryBase &gpu_src,
-                                 uint64 size) override;
-
-  port::Status SynchronousMemcpyDeviceToDevice(DeviceMemoryBase *gpu_dst,
-                                               const DeviceMemoryBase &gpu_src,
-                                               uint64 size) override;
-
-  bool MemZero(Stream *stream, DeviceMemoryBase *location,
-               uint64 size) override;
-  bool Memset(Stream *stream, DeviceMemoryBase *location, uint8 pattern,
-              uint64 size) override;
-  bool Memset32(Stream *stream, DeviceMemoryBase *location, uint32 pattern,
-                uint64 size) override;
-
-  bool Memcpy(Stream *stream, void *host_dst, const DeviceMemoryBase &gpu_src,
-              uint64 size) override;
-
-  bool Memcpy(Stream *stream, DeviceMemoryBase *gpu_dst, const void *host_src,
-              uint64 size) override;
-
-  bool MemcpyDeviceToDevice(Stream *stream, DeviceMemoryBase *gpu_dst,
-                            const DeviceMemoryBase &gpu_src,
-                            uint64 size) override;
-
-  bool HostCallback(Stream *stream,
-                    std::function<port::Status()> callback) override;
-
-  bool AllocateStream(Stream *stream) override;
-
-  void DeallocateStream(Stream *stream) override;
-
-  bool CreateStreamDependency(Stream *dependent, Stream *other) override;
-
-  bool AllocateTimer(Timer *timer) override;
-
-  void DeallocateTimer(Timer *timer) override;
-
-  bool StartTimer(Stream *stream, Timer *timer) override;
-
-  bool StopTimer(Stream *stream, Timer *timer) override;
-
-  port::Status AllocateEvent(Event *event) override;
-
-  port::Status DeallocateEvent(Event *event) override;
-
-  port::Status RecordEvent(Stream *stream, Event *event) override;
-
-  port::Status WaitForEvent(Stream *stream, Event *event) override;
-
-  Event::Status PollForEventStatus(Event *event) override;
-
-  port::Status BlockHostUntilDone(Stream *stream) override;
-
-  int PlatformDeviceCount() override { return CUDADriver::GetDeviceCount(); }
-
-  port::Status EnablePeerAccessTo(StreamExecutorInterface *other) override;
-
-  bool CanEnablePeerAccessTo(StreamExecutorInterface *other) override;
-
-  SharedMemoryConfig GetDeviceSharedMemoryConfig() override;
-
-  port::Status SetDeviceSharedMemoryConfig(SharedMemoryConfig config) override;
-
-  bool DeviceMemoryUsage(int64 *free, int64 *total) const override;
-
-  // Search for the symbol and returns a device pointer and size.
-  // Returns false if symbol does not exist.
-  bool GetSymbol(const string &symbol_name, ModuleHandle module_handle,
-                 void **mem, size_t *bytes) override;
-
-  DeviceDescription *PopulateDeviceDescription() const override;
-
-  // Populates the block_dim_limit by querying the device driver API. If an
-  // error occurs at any point while asking the driver for block dim limits, it
-  // will be only partially populated as a result, and an error will be logged.
-  bool FillBlockDimLimit(BlockDim *block_dim_limit) const;
-
-  bool SupportsBlas() const override;
-
-  blas::BlasSupport *CreateBlas() override;
-
-  bool SupportsFft() const override;
-
-  fft::FftSupport *CreateFft() override;
-
-  bool SupportsRng() const override;
-
-  rng::RngSupport *CreateRng() override;
-
-  bool SupportsDnn() const override;
-
-  dnn::DnnSupport *CreateDnn() override;
-
-  std::unique_ptr<internal::EventInterface> CreateEventImplementation()
-      override;
-
-  std::unique_ptr<internal::KernelInterface> CreateKernelImplementation()
-      override;
-
-  std::unique_ptr<internal::StreamInterface> GetStreamImplementation() override;
-
-  std::unique_ptr<internal::TimerInterface> GetTimerImplementation() override;
-
-  void *GpuContextHack() override;
-
-  CudaContext* cuda_context();
-
- private:
-  // Attempts to find a more specific version of the file indicated by
-  // filename by looking for compute-capability-specific suffixed versions; i.e.
-  // looking for "foo.ptx" will check to see if "foo.ptx.cc30.ptx" is present if
-  // we're on a compute capability 3.0 machine.
-  bool FindOnDiskForComputeCapability(absl::string_view filename,
-                                      absl::string_view canonical_suffix,
-                                      string *found_filename) const;
-
-  // Host callback landing routine invoked by CUDA.
-  // data: User-provided callback provided to HostCallback() above, captured
-  //       as a std::function<void()>. Allocated/initialized inside
-  //       HostCallback() and owned and deleted by this call.
-  static void InternalHostCallback(CUstream stream, CUresult status,
-                                   void *data);
-
-  // Collects metadata for the specified kernel.
-  bool GetKernelMetadata(CUDAKernel *cuda_kernel,
-                         KernelMetadata *kernel_metadata);
-
-  // Prints to VLOG(2) information about the kernel's occupancy and how it might
-  // be improved.
-  void VlogOccupancyInfo(const KernelBase &kernel, const ThreadDim &thread_dims,
-                         const BlockDim &block_dims);
-
-  bool LoadModuleFromCuBin(const char *cubin, CUmodule *module)
-      EXCLUSIVE_LOCKS_REQUIRED(in_memory_modules_mu_);
-
-  // Loads the PTX text `ptx` as a CUDA module.  `ptx` must be null terminated.
-  bool LoadModuleFromPtx(const char *ptx, CUmodule *module)
-      EXCLUSIVE_LOCKS_REQUIRED(in_memory_modules_mu_);
-
-  bool UnloadGpuBinary(const void *gpu_binary)
-      EXCLUSIVE_LOCKS_REQUIRED(in_memory_modules_mu_);
-
-  // Guards the in-memory-module mapping.
-  mutex in_memory_modules_mu_;
-
-  // Kernel -> loaded GPU binary. Many kernels may load the same binary.
-  std::unordered_map<const KernelBase *, const void *> kernel_to_gpu_binary_
-      GUARDED_BY(in_memory_modules_mu_);
-  // GPU binary (PTX or CUBIN) -> {CUDA module, reference count}.
-  std::unordered_map<const void *, std::pair<CUmodule, uint64>>
-      gpu_binary_to_module_ GUARDED_BY(in_memory_modules_mu_);
-
-  // Guards the launched kernel set.
-  mutex launched_kernels_mu_;
-
-  // Keeps track of the set of launched kernels. Currently used to suppress the
-  // occupancy check on subsequent launches.
-  std::set<CUfunction> launched_kernels_ GUARDED_BY(launched_kernels_mu_);
-
-  // Handle for the CUDA device being operated on. Immutable
-  // post-initialization.
-  CUdevice device_;
-
-  // Handle for session with the library/driver. Immutable post-initialization.
-  CudaContext* context_;
-
-  // The device ordinal value that this executor was initialized with; recorded
-  // for use in getting device metadata. Immutable post-initialization.
-  int device_ordinal_;
-
-  // The major verion of the compute capability for device_.
-  int cc_major_;
-
-  // The minor verion of the compute capability for device_.
-  int cc_minor_;
-
-  // The plugin configuration associated with this instance.
-  PluginConfig plugin_config_;
-
-  SE_DISALLOW_COPY_AND_ASSIGN(CUDAExecutor);
-};
+using CUDAExecutor = gpu::GpuExecutor;
 
 }  // namespace cuda
 }  // namespace stream_executor
diff --git a/tensorflow/stream_executor/cuda/cuda_helpers.h b/tensorflow/stream_executor/cuda/cuda_helpers.h
index dc0dc694cd..af6dcf3549 100644
--- a/tensorflow/stream_executor/cuda/cuda_helpers.h
+++ b/tensorflow/stream_executor/cuda/cuda_helpers.h
@@ -17,88 +17,9 @@ limitations under the License.
 //
 // These are typically placed here for use by multiple source components (for
 // example, BLAS and executor components).
-
 #ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_HELPERS_H_
 #define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_HELPERS_H_
 
-#include <stddef.h>
-#include <complex>
-
-#include "cuda/include/cuComplex.h"
-
-namespace stream_executor {
-
-template <typename ElemT>
-class DeviceMemory;
-
-namespace cuda {
-
-// Converts a const DeviceMemory reference to its underlying typed pointer in
-// CUDA
-// device memory.
-template <typename T>
-const T *CUDAMemory(const DeviceMemory<T> &mem) {
-  return static_cast<const T *>(mem.opaque());
-}
-
-// Converts a (non-const) DeviceMemory pointer reference to its underlying typed
-// pointer in CUDA device memory.
-template <typename T>
-T *CUDAMemoryMutable(DeviceMemory<T> *mem) {
-  return static_cast<T *>(mem->opaque());
-}
-
-static_assert(sizeof(std::complex<float>) == sizeof(cuComplex),
-              "std::complex<float> and cuComplex should have the same size");
-static_assert(offsetof(cuComplex, x) == 0,
-              "The real part of cuComplex should appear first.");
-static_assert(sizeof(std::complex<double>) == sizeof(cuDoubleComplex),
-              "std::complex<double> and cuDoubleComplex should have the same "
-              "size");
-static_assert(offsetof(cuDoubleComplex, x) == 0,
-              "The real part of cuDoubleComplex should appear first.");
-
-// Type traits to get CUDA complex types from std::complex<>.
-
-template <typename T>
-struct CUDAComplexT {
-  typedef T type;
-};
-
-template <>
-struct CUDAComplexT<std::complex<float>> {
-  typedef cuComplex type;
-};
-
-template <>
-struct CUDAComplexT<std::complex<double>> {
-  typedef cuDoubleComplex type;
-};
-
-// Converts pointers of std::complex<> to pointers of
-// cuComplex/cuDoubleComplex. No type conversion for non-complex types.
-
-template <typename T>
-inline const typename CUDAComplexT<T>::type *CUDAComplex(const T *p) {
-  return reinterpret_cast<const typename CUDAComplexT<T>::type *>(p);
-}
-
-template <typename T>
-inline typename CUDAComplexT<T>::type *CUDAComplex(T *p) {
-  return reinterpret_cast<typename CUDAComplexT<T>::type *>(p);
-}
-
-// Converts values of std::complex<float/double> to values of
-// cuComplex/cuDoubleComplex.
-inline cuComplex CUDAComplexValue(std::complex<float> val) {
-  return {val.real(), val.imag()};
-}
-
-inline cuDoubleComplex CUDAComplexValue(std::complex<double> val) {
-  return {val.real(), val.imag()};
-}
-
-}  // namespace cuda
-}  // namespace stream_executor
+#include "tensorflow/stream_executor/gpu/gpu_helpers.h"
 
 #endif  // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_HELPERS_H_
diff --git a/tensorflow/stream_executor/cuda/cuda_kernel.cc b/tensorflow/stream_executor/cuda/cuda_kernel.cc
new file mode 100644
index 0000000000..1b4e9a178f
--- /dev/null
+++ b/tensorflow/stream_executor/cuda/cuda_kernel.cc
@@ -0,0 +1,38 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/stream_executor/cuda/cuda_kernel.h"
+
+namespace stream_executor {
+namespace gpu {
+
+CUfunc_cache GpuKernel::GetGpuCacheConfig() const {
+  switch (preferred_cache_config_) {
+    case KernelCacheConfig::kNoPreference:
+      return CU_FUNC_CACHE_PREFER_NONE;
+    case KernelCacheConfig::kPreferShared:
+      return CU_FUNC_CACHE_PREFER_SHARED;
+    case KernelCacheConfig::kPreferL1:
+      return CU_FUNC_CACHE_PREFER_L1;
+    case KernelCacheConfig::kPreferEqual:
+      return CU_FUNC_CACHE_PREFER_EQUAL;
+    default:
+      LOG(FATAL) << "Unknown KernelCacheConfig"
+                 << static_cast<int32>(preferred_cache_config_);
+  }
+}
+
+}  // namespace gpu
+}  // namespace stream_executor
diff --git a/tensorflow/stream_executor/cuda/cuda_kernel.h b/tensorflow/stream_executor/cuda/cuda_kernel.h
index ec1dc51e57..a8a18d200d 100644
--- a/tensorflow/stream_executor/cuda/cuda_kernel.h
+++ b/tensorflow/stream_executor/cuda/cuda_kernel.h
@@ -22,104 +22,12 @@ limitations under the License.
 #ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_KERNEL_H_
 #define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_KERNEL_H_
 
-#include "tensorflow/stream_executor/kernel_cache_config.h"
-#include "tensorflow/stream_executor/stream_executor_internal.h"
-#include "tensorflow/stream_executor/cuda/cuda_driver.h"
-#include "tensorflow/stream_executor/platform/port.h"
-#include "tensorflow/stream_executor/platform/logging.h"
-#include "cuda/include/cuda.h"
-
-#ifdef PLATFORMS_GPUS_CUDA_DYNAMIC_LIBCUDA_DYNAMIC_LIBCUDA_H_
-#error \
-    "No driver calls in this file, wrap driver functionality in cuda_driver.cc."
-#endif
-
-#ifdef __CUDA_RUNTIME_H__
-#error \
-    "CUDA runtime being included into CUDA GPU executor; should be driver only."
-#endif
+#include "tensorflow/stream_executor/gpu/gpu_kernel.h"
 
 namespace stream_executor {
 namespace cuda {
 
-// Wraps a CUfunction to implement the platform-independent KernelInterface.
-class CUDAKernel : public internal::KernelInterface {
- public:
-  CUDAKernel() : cuda_function_(nullptr), arity_(0),
-                 preferred_cache_config_(KernelCacheConfig::kNoPreference) {}
-
-  // Note that the function is unloaded when the module is unloaded, and the
-  // module that the function is contained in is owned by the CUDAExecutor.
-  ~CUDAKernel() override {}
-
-  // As arity cannot be reflected upon using the CUDA API, the arity is
-  // explicitly set during the CUDAExecutor::GetKernel initialization process.
-  void set_arity(unsigned arity) { arity_ = arity; }
-  unsigned Arity() const override { return arity_; }
-
-  // Returns the CUfunction value for passing to the CUDA API.
-  CUfunction AsCUDAFunctionValue() const {
-    DCHECK(cuda_function_ != nullptr);
-    return const_cast<CUfunction>(cuda_function_);
-  }
-
-  // Returns the slot that the CUfunction is stored within for this object,
-  // for the CUDA API which wants to load into a CUfunction*.
-  CUfunction *cuda_function_ptr() { return &cuda_function_; }
-
-  // CUDA supports setting the preferred cache configuration of a CUfunction
-  // (more-or-less equivalent to a CUDAKernel). We support this via the below
-  // functions; users can set a preference, and that is applied when the kernel
-  // is [lazy-]loaded (in CUDAExecutor::Launch). The alternative would be to
-  // load the kernel & set the preference when the user calls the setter below;
-  // either approach is valid.
-  // Sets the current kernel cache configuration preference.
-  void SetPreferredCacheConfig(KernelCacheConfig config) override {
-    preferred_cache_config_ = config;
-  }
-
-  // Returns the current kernel cache configuration preference.
-  KernelCacheConfig GetPreferredCacheConfig() const override {
-    return preferred_cache_config_;
-  }
-
-  // Returns the current kernel cache configuration preference as a
-  // CUfunc_cache.
-  CUfunc_cache GetCUDACacheConfig() const {
-    switch (preferred_cache_config_) {
-      case KernelCacheConfig::kNoPreference:
-        return CU_FUNC_CACHE_PREFER_NONE;
-      case KernelCacheConfig::kPreferShared:
-        return CU_FUNC_CACHE_PREFER_SHARED;
-      case KernelCacheConfig::kPreferL1:
-        return CU_FUNC_CACHE_PREFER_L1;
-      case KernelCacheConfig::kPreferEqual:
-        return CU_FUNC_CACHE_PREFER_EQUAL;
-      default:
-        LOG(FATAL) << "Unknown KernelCacheConfig"
-                   << static_cast<int32>(preferred_cache_config_);
-    }
-  }
-
- private:
-  CUfunction cuda_function_;  // Wrapped CUDA kernel handle.
-  unsigned arity_;            // Number of formal parameters the kernel takes.
-
-  // Preferred (but not required) cache configuration for this kernel.
-  KernelCacheConfig preferred_cache_config_;
-};
-
-// Given a platform-independent kernel datatype, returns the (const) internal
-// CUDA platform implementation pointer.
-inline const CUDAKernel *AsCUDAKernel(const KernelBase *kernel) {
-  return static_cast<const CUDAKernel *>(kernel->implementation());
-}
-
-// Given a platform-independent kernel datatype, returns the (non-const)
-// internal CUDA platform implementation pointer.
-inline CUDAKernel *AsCUDAKernel(KernelBase *kernel) {
-  return static_cast<CUDAKernel *>(kernel->implementation());
-}
+using CUDAKernel = gpu::GpuKernel;
 
 }  // namespace cuda
 }  // namespace stream_executor
diff --git a/tensorflow/stream_executor/cuda/cuda_platform.cc b/tensorflow/stream_executor/cuda/cuda_platform.cc
index b342e71bdd..54aba01278 100644
--- a/tensorflow/stream_executor/cuda/cuda_platform.cc
+++ b/tensorflow/stream_executor/cuda/cuda_platform.cc
@@ -25,7 +25,7 @@ limitations under the License.
 #include "tensorflow/stream_executor/lib/stringprintf.h"
 
 namespace stream_executor {
-namespace cuda {
+namespace gpu {
 namespace {
 
 // Synchronize with spinlocks.
@@ -129,16 +129,16 @@ port::StatusOr<StreamExecutor*> CudaPlatform::FirstExecutorForBus(
       port::Printf("Executor for bus %d not found.", bus_ordinal));
 }
 
-Platform::Id CudaPlatform::id() const { return kCudaPlatformId; }
+Platform::Id CudaPlatform::id() const { return cuda::kCudaPlatformId; }
 
 int CudaPlatform::VisibleDeviceCount() const {
   // Throw away the result - it logs internally, and this [containing] function
   // isn't in the path of user control. It's safe to call this > 1x.
-  if (!cuda::CUDADriver::Init().ok()) {
+  if (!gpu::GpuDriver::Init().ok()) {
     return -1;
   }
 
-  return CUDADriver::GetDeviceCount();
+  return GpuDriver::GetDeviceCount();
 }
 
 const string& CudaPlatform::Name() const { return name_; }
@@ -169,7 +169,7 @@ port::StatusOr<StreamExecutor*> CudaPlatform::GetExecutor(
 port::StatusOr<std::unique_ptr<StreamExecutor>>
 CudaPlatform::GetUncachedExecutor(const StreamExecutorConfig& config) {
   auto executor = MakeUnique<StreamExecutor>(
-      this, MakeUnique<CUDAExecutor>(config.plugin_config));
+      this, MakeUnique<GpuExecutor>(config.plugin_config));
   auto init_status = executor->Init(config.ordinal, config.device_options);
   if (!init_status.ok()) {
     return port::Status(
@@ -191,13 +191,13 @@ void CudaPlatform::UnregisterTraceListener(TraceListener* listener) {
   LOG(FATAL) << "not yet implemented: unregister CUDA trace listener";
 }
 
-}  // namespace cuda
+}  // namespace gpu
 
 static void InitializeCudaPlatform() {
   // Disabling leak checking, MultiPlatformManager does not destroy its
   // registered platforms.
 
-  std::unique_ptr<cuda::CudaPlatform> platform(new cuda::CudaPlatform);
+  std::unique_ptr<gpu::CudaPlatform> platform(new gpu::CudaPlatform);
   SE_CHECK_OK(MultiPlatformManager::RegisterPlatform(std::move(platform)));
 }
 
diff --git a/tensorflow/stream_executor/cuda/cuda_platform.h b/tensorflow/stream_executor/cuda/cuda_platform.h
index fc0e15d5a6..1ab09d690f 100644
--- a/tensorflow/stream_executor/cuda/cuda_platform.h
+++ b/tensorflow/stream_executor/cuda/cuda_platform.h
@@ -32,7 +32,7 @@ limitations under the License.
 #include "tensorflow/stream_executor/trace_listener.h"
 
 namespace stream_executor {
-namespace cuda {
+namespace gpu {
 
 // Opaque and unique identifier for the CUDA platform plugin.
 // This is needed so that plugins can refer to/identify this platform without
@@ -102,6 +102,12 @@ class CudaPlatform : public Platform {
   SE_DISALLOW_COPY_AND_ASSIGN(CudaPlatform);
 };
 
+}  // namespace gpu
+
+namespace cuda {
+
+using CudaPlatform = gpu::CudaPlatform;
+
 }  // namespace cuda
 }  // namespace stream_executor
 
diff --git a/tensorflow/stream_executor/cuda/cuda_rng.cc b/tensorflow/stream_executor/cuda/cuda_rng.cc
index 022ee17ff7..395b30b491 100644
--- a/tensorflow/stream_executor/cuda/cuda_rng.cc
+++ b/tensorflow/stream_executor/cuda/cuda_rng.cc
@@ -58,33 +58,33 @@ std::ostream &operator<<(std::ostream &in, const curandStatus_t &status) {
 }
 
 namespace stream_executor {
-namespace cuda {
+namespace gpu {
 
-PLUGIN_REGISTRY_DEFINE_PLUGIN_ID(kCuRandPlugin);
+PLUGIN_REGISTRY_DEFINE_PLUGIN_ID(kGpuRandPlugin);
 
 namespace wrap {
 
 #ifdef PLATFORM_GOOGLE
-#define STREAM_EXECUTOR_CURAND_WRAP(__name)                         \
-  struct WrapperShim__##__name {                                    \
-    template <typename... Args>                                     \
-    curandStatus_t operator()(CUDAExecutor *parent, Args... args) { \
-      cuda::ScopedActivateExecutorContext sac{parent};              \
-      return ::__name(args...);                                     \
-    }                                                               \
+#define STREAM_EXECUTOR_CURAND_WRAP(__name)                        \
+  struct WrapperShim__##__name {                                   \
+    template <typename... Args>                                    \
+    curandStatus_t operator()(GpuExecutor* parent, Args... args) { \
+      gpu::ScopedActivateExecutorContext sac{parent};              \
+      return ::__name(args...);                                    \
+    }                                                              \
   } __name;
 
 #else
 #define STREAM_EXECUTOR_CURAND_WRAP(__name)                               \
   struct DynLoadShim__##__name {                                          \
-    static const char *kName;                                             \
+    static const char* kName;                                             \
     using FuncPtrT = std::add_pointer<decltype(::__name)>::type;          \
-    static void *GetDsoHandle() {                                         \
+    static void* GetDsoHandle() {                                         \
       auto s = internal::CachedDsoLoader::GetCurandDsoHandle();           \
       return s.ValueOrDie();                                              \
     }                                                                     \
     static FuncPtrT LoadOrDie() {                                         \
-      void *f;                                                            \
+      void* f;                                                            \
       auto s = port::Env::Default()->GetSymbolFromLibrary(GetDsoHandle(), \
                                                           kName, &f);     \
       CHECK(s.ok()) << "could not find " << kName                         \
@@ -96,12 +96,12 @@ namespace wrap {
       return f;                                                           \
     }                                                                     \
     template <typename... Args>                                           \
-    curandStatus_t operator()(CUDAExecutor *parent, Args... args) {       \
-      cuda::ScopedActivateExecutorContext sac{parent};                    \
+    curandStatus_t operator()(GpuExecutor* parent, Args... args) {        \
+      gpu::ScopedActivateExecutorContext sac{parent};                     \
       return DynLoad()(args...);                                          \
     }                                                                     \
   } __name;                                                               \
-  const char *DynLoadShim__##__name::kName = #__name;
+  const char* DynLoadShim__##__name::kName = #__name;
 #endif
 
 STREAM_EXECUTOR_CURAND_WRAP(curandCreateGenerator);
@@ -116,38 +116,15 @@ STREAM_EXECUTOR_CURAND_WRAP(curandGenerateNormalDouble);
 
 }  // namespace wrap
 
-template <typename T>
-string TypeString();
-
-template <>
-string TypeString<float>() {
-  return "float";
-}
-
-template <>
-string TypeString<double>() {
-  return "double";
-}
-
-template <>
-string TypeString<std::complex<float>>() {
-  return "std::complex<float>";
-}
-
-template <>
-string TypeString<std::complex<double>>() {
-  return "std::complex<double>";
-}
-
-CUDARng::CUDARng(CUDAExecutor *parent) : parent_(parent), rng_(nullptr) {}
+GpuRng::GpuRng(GpuExecutor* parent) : parent_(parent), rng_(nullptr) {}
 
-CUDARng::~CUDARng() {
+GpuRng::~GpuRng() {
   if (rng_ != nullptr) {
     wrap::curandDestroyGenerator(parent_, rng_);
   }
 }
 
-bool CUDARng::Init() {
+bool GpuRng::Init() {
   mutex_lock lock(mu_);
   CHECK(rng_ == nullptr);
 
@@ -162,9 +139,9 @@ bool CUDARng::Init() {
   return true;
 }
 
-bool CUDARng::SetStream(Stream *stream) {
+bool GpuRng::SetStream(Stream* stream) {
   curandStatus_t ret =
-      wrap::curandSetStream(parent_, rng_, AsCUDAStreamValue(stream));
+      wrap::curandSetStream(parent_, rng_, AsGpuStreamValue(stream));
   if (ret != CURAND_STATUS_SUCCESS) {
     LOG(ERROR) << "failed to set stream for random generation: " << ret;
     return false;
@@ -182,8 +159,7 @@ constexpr bool ComplexIsConsecutiveFloats() {
 }
 
 template <typename T>
-bool CUDARng::DoPopulateRandUniformInternal(Stream *stream,
-                                            DeviceMemory<T> *v) {
+bool GpuRng::DoPopulateRandUniformInternal(Stream* stream, DeviceMemory<T>* v) {
   mutex_lock lock(mu_);
   static_assert(ComplexIsConsecutiveFloats(),
                 "std::complex values are not stored as consecutive values");
@@ -203,11 +179,11 @@ bool CUDARng::DoPopulateRandUniformInternal(Stream *stream,
   if (std::is_same<T, float>::value ||
       std::is_same<T, std::complex<float>>::value) {
     ret = wrap::curandGenerateUniform(
-        parent_, rng_, reinterpret_cast<float *>(CUDAMemoryMutable(v)),
+        parent_, rng_, reinterpret_cast<float*>(GpuMemoryMutable(v)),
         element_count);
   } else {
     ret = wrap::curandGenerateUniformDouble(
-        parent_, rng_, reinterpret_cast<double *>(CUDAMemoryMutable(v)),
+        parent_, rng_, reinterpret_cast<double*>(GpuMemoryMutable(v)),
         element_count);
   }
   if (ret != CURAND_STATUS_SUCCESS) {
@@ -220,29 +196,29 @@ bool CUDARng::DoPopulateRandUniformInternal(Stream *stream,
   return true;
 }
 
-bool CUDARng::DoPopulateRandUniform(Stream *stream, DeviceMemory<float> *v) {
+bool GpuRng::DoPopulateRandUniform(Stream* stream, DeviceMemory<float>* v) {
   return DoPopulateRandUniformInternal(stream, v);
 }
 
-bool CUDARng::DoPopulateRandUniform(Stream *stream, DeviceMemory<double> *v) {
+bool GpuRng::DoPopulateRandUniform(Stream* stream, DeviceMemory<double>* v) {
   return DoPopulateRandUniformInternal(stream, v);
 }
 
-bool CUDARng::DoPopulateRandUniform(Stream *stream,
-                                    DeviceMemory<std::complex<float>> *v) {
+bool GpuRng::DoPopulateRandUniform(Stream* stream,
+                                   DeviceMemory<std::complex<float>>* v) {
   return DoPopulateRandUniformInternal(stream, v);
 }
 
-bool CUDARng::DoPopulateRandUniform(Stream *stream,
-                                    DeviceMemory<std::complex<double>> *v) {
+bool GpuRng::DoPopulateRandUniform(Stream* stream,
+                                   DeviceMemory<std::complex<double>>* v) {
   return DoPopulateRandUniformInternal(stream, v);
 }
 
 template <typename ElemT, typename FuncT>
-bool CUDARng::DoPopulateRandGaussianInternal(Stream *stream, ElemT mean,
-                                             ElemT stddev,
-                                             DeviceMemory<ElemT> *v,
-                                             FuncT func) {
+bool GpuRng::DoPopulateRandGaussianInternal(Stream* stream, ElemT mean,
+                                            ElemT stddev,
+                                            DeviceMemory<ElemT>* v,
+                                            FuncT func) {
   mutex_lock lock(mu_);
 
   if (!SetStream(stream)) {
@@ -251,7 +227,7 @@ bool CUDARng::DoPopulateRandGaussianInternal(Stream *stream, ElemT mean,
 
   uint64 element_count = v->ElementCount();
   curandStatus_t ret =
-      func(parent_, rng_, CUDAMemoryMutable(v), element_count, mean, stddev);
+      func(parent_, rng_, GpuMemoryMutable(v), element_count, mean, stddev);
 
   if (ret != CURAND_STATUS_SUCCESS) {
     LOG(ERROR) << "failed to do gaussian generation of " << v->ElementCount()
@@ -262,19 +238,19 @@ bool CUDARng::DoPopulateRandGaussianInternal(Stream *stream, ElemT mean,
   return true;
 }
 
-bool CUDARng::DoPopulateRandGaussian(Stream *stream, float mean, float stddev,
-                                     DeviceMemory<float> *v) {
+bool GpuRng::DoPopulateRandGaussian(Stream* stream, float mean, float stddev,
+                                    DeviceMemory<float>* v) {
   return DoPopulateRandGaussianInternal(stream, mean, stddev, v,
                                         wrap::curandGenerateNormal);
 }
 
-bool CUDARng::DoPopulateRandGaussian(Stream *stream, double mean, double stddev,
-                                     DeviceMemory<double> *v) {
+bool GpuRng::DoPopulateRandGaussian(Stream* stream, double mean, double stddev,
+                                    DeviceMemory<double>* v) {
   return DoPopulateRandGaussianInternal(stream, mean, stddev, v,
                                         wrap::curandGenerateNormalDouble);
 }
 
-bool CUDARng::SetSeed(Stream *stream, const uint8 *seed, uint64 seed_bytes) {
+bool GpuRng::SetSeed(Stream* stream, const uint8* seed, uint64 seed_bytes) {
   mutex_lock lock(mu_);
   CHECK(rng_ != nullptr);
 
@@ -303,15 +279,15 @@ bool CUDARng::SetSeed(Stream *stream, const uint8 *seed, uint64 seed_bytes) {
   return true;
 }
 
-}  // namespace cuda
+}  // namespace gpu
 
 void initialize_curand() {
   port::Status status =
       PluginRegistry::Instance()->RegisterFactory<PluginRegistry::RngFactory>(
-          cuda::kCudaPlatformId, cuda::kCuRandPlugin, "cuRAND",
-          [](internal::StreamExecutorInterface *parent) -> rng::RngSupport * {
-            cuda::CUDAExecutor *cuda_executor =
-                dynamic_cast<cuda::CUDAExecutor *>(parent);
+          cuda::kCudaPlatformId, gpu::kGpuRandPlugin, "cuRAND",
+          [](internal::StreamExecutorInterface* parent) -> rng::RngSupport* {
+            gpu::GpuExecutor* cuda_executor =
+                dynamic_cast<gpu::GpuExecutor*>(parent);
             if (cuda_executor == nullptr) {
               LOG(ERROR)
                   << "Attempting to initialize an instance of the cuRAND "
@@ -319,7 +295,7 @@ void initialize_curand() {
               return nullptr;
             }
 
-            cuda::CUDARng *rng = new cuda::CUDARng(cuda_executor);
+            gpu::GpuRng* rng = new gpu::GpuRng(cuda_executor);
             if (!rng->Init()) {
               // Note: Init() will log a more specific error.
               delete rng;
@@ -334,7 +310,7 @@ void initialize_curand() {
   }
 
   PluginRegistry::Instance()->SetDefaultFactory(
-      cuda::kCudaPlatformId, PluginKind::kRng, cuda::kCuRandPlugin);
+      cuda::kCudaPlatformId, PluginKind::kRng, gpu::kGpuRandPlugin);
 }
 
 }  // namespace stream_executor
diff --git a/tensorflow/stream_executor/cuda/cuda_rng.h b/tensorflow/stream_executor/cuda/cuda_rng.h
index 57ef398aaa..d7f6b0e8e0 100644
--- a/tensorflow/stream_executor/cuda/cuda_rng.h
+++ b/tensorflow/stream_executor/cuda/cuda_rng.h
@@ -16,85 +16,13 @@ limitations under the License.
 #ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_RNG_H_
 #define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_RNG_H_
 
-#include "tensorflow/stream_executor/platform/mutex.h"
-#include "tensorflow/stream_executor/platform/port.h"
-#include "tensorflow/stream_executor/platform/thread_annotations.h"
-#include "tensorflow/stream_executor/plugin_registry.h"
-#include "tensorflow/stream_executor/rng.h"
-
-typedef struct curandGenerator_st *curandGenerator_t;
+#include "tensorflow/stream_executor/gpu/gpu_rng.h"
 
 namespace stream_executor {
 
-class Stream;
-template <typename ElemT>
-class DeviceMemory;
-
 namespace cuda {
 
-// Opaque and unique identifier for the cuRAND plugin.
-extern const PluginId kCuRandPlugin;
-
-class CUDAExecutor;
-
-// CUDA-platform implementation of the random number generation support
-// interface.
-//
-// Thread-safe post-initialization.
-class CUDARng : public rng::RngSupport {
- public:
-  explicit CUDARng(CUDAExecutor *parent);
-
-  // Retrieves a curand library generator handle. This is necessary for
-  // enqueuing random number generation work onto the device.
-  // TODO(leary) provide a way for users to select the RNG algorithm.
-  bool Init();
-
-  // Releases a curand library generator handle, if one was acquired.
-  ~CUDARng() override;
-
-  // See rng::RngSupport for details on the following overrides.
-  bool DoPopulateRandUniform(Stream *stream, DeviceMemory<float> *v) override;
-  bool DoPopulateRandUniform(Stream *stream, DeviceMemory<double> *v) override;
-  bool DoPopulateRandUniform(Stream *stream,
-                             DeviceMemory<std::complex<float>> *v) override;
-  bool DoPopulateRandUniform(Stream *stream,
-                             DeviceMemory<std::complex<double>> *v) override;
-  bool DoPopulateRandGaussian(Stream *stream, float mean, float stddev,
-                              DeviceMemory<float> *v) override;
-  bool DoPopulateRandGaussian(Stream *stream, double mean, double stddev,
-                              DeviceMemory<double> *v) override;
-
-  bool SetSeed(Stream *stream, const uint8 *seed, uint64 seed_bytes) override;
-
- private:
-  // Actually performs the work of generating random numbers - the public
-  // methods are thin wrappers to this interface.
-  template <typename T>
-  bool DoPopulateRandUniformInternal(Stream *stream, DeviceMemory<T> *v);
-  template <typename ElemT, typename FuncT>
-  bool DoPopulateRandGaussianInternal(Stream *stream, ElemT mean, ElemT stddev,
-                                      DeviceMemory<ElemT> *v, FuncT func);
-
-  // Sets the stream for the internal curand generator.
-  //
-  // This is a stateful operation, as the handle can only have one stream set at
-  // a given time, so it is usually performed right before enqueuing work to do
-  // with random number generation.
-  bool SetStream(Stream *stream) EXCLUSIVE_LOCKS_REQUIRED(mu_);
-
-  // mutex that guards the cuRAND handle for this device.
-  mutex mu_;
-
-  // CUDAExecutor which instantiated this CUDARng.
-  // Immutable post-initialization.
-  CUDAExecutor *parent_;
-
-  // cuRANDalibrary handle on the device.
-  curandGenerator_t rng_ GUARDED_BY(mu_);
-
-  SE_DISALLOW_COPY_AND_ASSIGN(CUDARng);
-};
+using CUDARng = gpu::GpuRng;
 
 }  // namespace cuda
 }  // namespace stream_executor
diff --git a/tensorflow/stream_executor/cuda/cuda_stream.h b/tensorflow/stream_executor/cuda/cuda_stream.h
index bb8bda4755..4460351368 100644
--- a/tensorflow/stream_executor/cuda/cuda_stream.h
+++ b/tensorflow/stream_executor/cuda/cuda_stream.h
@@ -13,79 +13,22 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-// Defines the CUDAStream type - the CUDA-specific implementation of the generic
+// Defines the GpuStream type - the CUDA-specific implementation of the generic
 // StreamExecutor Stream interface.
 
 #ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_STREAM_H_
 #define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_STREAM_H_
 
-#include "tensorflow/stream_executor/cuda/cuda_driver.h"
-#include "tensorflow/stream_executor/platform/thread_annotations.h"
-#include "tensorflow/stream_executor/stream_executor_internal.h"
+#include "tensorflow/stream_executor/gpu/gpu_stream.h"
 
 namespace stream_executor {
 namespace cuda {
 
-class CUDAExecutor;
+using CUDAStream = gpu::GpuStream;
 
-// Wraps a CUstream in order to satisfy the platform-independent
-// StreamInterface.
-//
-// Thread-safe post-initialization.
-class CUDAStream : public internal::StreamInterface {
- public:
-  explicit CUDAStream(CUDAExecutor *parent)
-      : parent_(parent), cuda_stream_(nullptr), completed_event_(nullptr) {}
-
-  // Note: teardown is handled by a parent's call to DeallocateStream.
-  ~CUDAStream() override {}
-
-  void *GpuStreamHack() override { return cuda_stream_; }
-  void **GpuStreamMemberHack() override {
-    return reinterpret_cast<void **>(&cuda_stream_);
-  }
-
-  // Explicitly initialize the CUDA resources associated with this stream, used
-  // by StreamExecutor::AllocateStream().
-  bool Init();
-
-  // Explicitly destroy the CUDA resources associated with this stream, used by
-  // StreamExecutor::DeallocateStream().
-  void Destroy();
-
-  // Returns true if no work is pending or executing on the stream.
-  bool IsIdle() const;
-
-  // Retrieves an event which indicates that all work enqueued into the stream
-  // has completed. Ownership of the event is not transferred to the caller, the
-  // event is owned by this stream.
-  CUevent* completed_event() { return &completed_event_; }
-
-  // Returns the CUstream value for passing to the CUDA API.
-  //
-  // Precond: this CUDAStream has been allocated (otherwise passing a nullptr
-  // into the NVIDIA library causes difficult-to-understand faults).
-  CUstream cuda_stream() const {
-    DCHECK(cuda_stream_ != nullptr);
-    return const_cast<CUstream>(cuda_stream_);
-  }
-
-  CUDAExecutor *parent() const { return parent_; }
-
- private:
-  CUDAExecutor *parent_;  // Executor that spawned this stream.
-  CUstream cuda_stream_;  // Wrapped CUDA stream handle.
-
-  // Event that indicates this stream has completed.
-  CUevent completed_event_ = nullptr;
-};
-
-// Helper functions to simplify extremely common flows.
-// Converts a Stream to the underlying CUDAStream implementation.
-CUDAStream *AsCUDAStream(Stream *stream);
-
-// Extracts a CUstream from a CUDAStream-backed Stream object.
-CUstream AsCUDAStreamValue(Stream *stream);
+inline CUDAStream* AsCUDAStream(Stream* stream) {
+  return gpu::AsGpuStream(stream);
+}
 
 }  // namespace cuda
 }  // namespace stream_executor
diff --git a/tensorflow/stream_executor/cuda/cuda_timer.h b/tensorflow/stream_executor/cuda/cuda_timer.h
index e040cf86fa..01b722e888 100644
--- a/tensorflow/stream_executor/cuda/cuda_timer.h
+++ b/tensorflow/stream_executor/cuda/cuda_timer.h
@@ -13,76 +13,18 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-// Defines the CUDATimer type - the CUDA-specific implementation of the generic
+// Defines the GpuTimer type - the CUDA-specific implementation of the generic
 // StreamExecutor Timer interface.
 
 #ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_TIMER_H_
 #define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_TIMER_H_
 
-#include "tensorflow/stream_executor/stream_executor_internal.h"
-#include "tensorflow/stream_executor/cuda/cuda_driver.h"
-#include "tensorflow/stream_executor/cuda/cuda_gpu_executor.h"
+#include "tensorflow/stream_executor/gpu/gpu_timer.h"
 
 namespace stream_executor {
 namespace cuda {
 
-class CUDAExecutor;
-class CUDAStream;
-
-// Wraps a pair of CUevents in order to satisfy the platform-independent
-// TimerInferface -- both a start and a stop event are present which may be
-// recorded in a stream.
-class CUDATimer : public internal::TimerInterface {
- public:
-  explicit CUDATimer(CUDAExecutor *parent)
-      : parent_(parent), start_event_(nullptr), stop_event_(nullptr) {}
-
-  // Note: teardown needs to be explicitly handled in this API by a call to
-  // StreamExecutor::DeallocateTimer(), which invokes Destroy().
-  // TODO(csigg): Change to RAII.
-  ~CUDATimer() override {}
-
-  // Allocates the platform-specific pieces of the timer, called as part of
-  // StreamExecutor::AllocateTimer().
-  bool Init();
-
-  // Deallocates the platform-specific pieces of the timer, called as part of
-  // StreamExecutor::DeallocateTimer().
-  void Destroy();
-
-  // Records the "timer start" event at the current point in the stream.
-  bool Start(CUDAStream *stream);
-
-  // Records the "timer stop" event at the current point in the stream.
-  bool Stop(CUDAStream *stream);
-
-  // Returns the elapsed time, in milliseconds, between the start and stop
-  // events.
-  float GetElapsedMilliseconds() const;
-
-  // See Timer::Microseconds().
-  // TODO(leary) make this into an error code interface...
-  uint64 Microseconds() const override {
-    return GetElapsedMilliseconds() * 1e3;
-  }
-
-  // See Timer::Nanoseconds().
-  uint64 Nanoseconds() const override { return GetElapsedMilliseconds() * 1e6; }
-
- private:
-  CUDAExecutor *parent_;
-  CUevent start_event_;  // Event recorded to indicate the "start" timestamp
-                         // executing in a stream.
-  CUevent stop_event_;   // Event recorded to indicate the "stop" timestamp
-                         // executing in a stream.
-};
-
-struct TimerDeleter {
-  void operator()(CUDATimer *t) {
-    t->Destroy();
-    delete t;
-  }
-};
+using CUDATimer = gpu::GpuTimer;
 
 }  // namespace cuda
 }  // namespace stream_executor
diff --git a/tensorflow/stream_executor/cuda/cudnn_version.cc b/tensorflow/stream_executor/cuda/cudnn_version.cc
index e8fcc03618..9ef8bc95e5 100644
--- a/tensorflow/stream_executor/cuda/cudnn_version.cc
+++ b/tensorflow/stream_executor/cuda/cudnn_version.cc
@@ -16,7 +16,7 @@ limitations under the License.
 #include "tensorflow/stream_executor/cuda/cudnn_version.h"
 
 namespace stream_executor {
-namespace cuda {
+namespace gpu {
 
 bool IsSourceCompatibleWithCudnnLibrary(CudnnVersion source_version,
                                         CudnnVersion loaded_version) {
@@ -36,5 +36,5 @@ bool IsSourceCompatibleWithCudnnLibrary(CudnnVersion source_version,
            loaded_version.minor_version >= source_version.minor_version));
 }
 
-}  // namespace cuda
+}  // namespace gpu
 }  // namespace stream_executor
diff --git a/tensorflow/stream_executor/cuda/cudnn_version.h b/tensorflow/stream_executor/cuda/cudnn_version.h
index 6464e7f8e8..4607a9bff8 100644
--- a/tensorflow/stream_executor/cuda/cudnn_version.h
+++ b/tensorflow/stream_executor/cuda/cudnn_version.h
@@ -21,7 +21,7 @@ limitations under the License.
 #include "tensorflow/core/lib/strings/strcat.h"
 
 namespace stream_executor {
-namespace cuda {
+namespace gpu {
 
 struct CudnnVersion {
   CudnnVersion() = default;
@@ -44,7 +44,7 @@ struct CudnnVersion {
 bool IsSourceCompatibleWithCudnnLibrary(CudnnVersion source_version,
                                         CudnnVersion loaded_version);
 
-}  // namespace cuda
+}  // namespace gpu
 }  // namespace stream_executor
 
 #endif  // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDNN_VERSION_H_
diff --git a/tensorflow/stream_executor/cuda/cudnn_version_test.cc b/tensorflow/stream_executor/cuda/cudnn_version_test.cc
index 7d4c6399d0..cfe114662d 100644
--- a/tensorflow/stream_executor/cuda/cudnn_version_test.cc
+++ b/tensorflow/stream_executor/cuda/cudnn_version_test.cc
@@ -18,7 +18,7 @@ limitations under the License.
 #include "tensorflow/core/platform/test.h"
 
 namespace stream_executor {
-namespace cuda {
+namespace gpu {
 namespace {
 
 TEST(CuDNNVersion, ToString) {
@@ -68,5 +68,5 @@ TEST(IsSourceCompatibleWithCudnnLibraryTest, Basic) {
 }
 
 }  // namespace
-}  // namespace cuda
+}  // namespace gpu
 }  // namespace stream_executor
diff --git a/tensorflow/stream_executor/device_description.cc b/tensorflow/stream_executor/device_description.cc
index 0b991b7ba8..2595d216b4 100644
--- a/tensorflow/stream_executor/device_description.cc
+++ b/tensorflow/stream_executor/device_description.cc
@@ -50,6 +50,7 @@ DeviceDescription::DeviceDescription()
       clock_rate_ghz_(-1.0),
       cuda_compute_capability_major_(-1),
       cuda_compute_capability_minor_(-1),
+      rocm_amdgpu_isa_version_(-1),
       numa_node_(-1),
       core_count_(-1),
       ecc_enabled_(false) {}
@@ -112,6 +113,15 @@ bool DeviceDescription::cuda_compute_capability(int *major, int *minor) const {
   return cuda_compute_capability_major_ != 0;
 }
 
+bool DeviceDescription::rocm_amdgpu_isa_version(int *version) const {
+  bool status = false;
+  if (rocm_amdgpu_isa_version_ > 0) {
+    *version = rocm_amdgpu_isa_version_;
+    status = true;
+  }
+  return status;
+}
+
 bool ThreadDimOk(const DeviceDescription &device_description,
                  const ThreadDim &thread_dim) {
   auto total_threads = thread_dim.x * thread_dim.y * thread_dim.z;
diff --git a/tensorflow/stream_executor/device_description.h b/tensorflow/stream_executor/device_description.h
index 8ddf18629d..cccc209e1c 100644
--- a/tensorflow/stream_executor/device_description.h
+++ b/tensorflow/stream_executor/device_description.h
@@ -133,6 +133,11 @@ class DeviceDescription {
   // zero, and the return value will be false.
   bool cuda_compute_capability(int *major, int *minor) const;
 
+  // Returns the AMDGPU ISA version if we're running on the ROCm platform.
+  // If the information is not available, the version is not modified,
+  // and the return value will be false.
+  bool rocm_amdgpu_isa_version(int *version) const;
+
   // Returns the maximum amount of shared memory present on a single core
   // (i.e. Streaming Multiprocessor on NVIDIA GPUs; Compute Unit for OpenCL
   // devices). Note that some devices, such as NVIDIA's have a configurable
@@ -195,6 +200,9 @@ class DeviceDescription {
   int cuda_compute_capability_major_;
   int cuda_compute_capability_minor_;
 
+  // ROCM AMDGPU ISA version, 0 if not available.
+  int rocm_amdgpu_isa_version_;
+
   int numa_node_;
   int core_count_;
   bool ecc_enabled_;
@@ -280,6 +288,10 @@ class DeviceDescriptionBuilder {
     device_description_->cuda_compute_capability_minor_ = minor;
   }
 
+  void set_rocm_amdgpu_isa_version(int version) {
+    device_description_->rocm_amdgpu_isa_version_ = version;
+  }
+
   void set_numa_node(int value) { device_description_->numa_node_ = value; }
   void set_core_count(int value) { device_description_->core_count_ = value; }
   void set_ecc_enabled(bool value) {
diff --git a/tensorflow/stream_executor/gpu/BUILD b/tensorflow/stream_executor/gpu/BUILD
new file mode 100644
index 0000000000..26a9c6426d
--- /dev/null
+++ b/tensorflow/stream_executor/gpu/BUILD
@@ -0,0 +1,209 @@
+# Description:
+#   GPU-platform specific StreamExecutor support code.
+
+licenses(["notice"])  # Apache 2.0
+
+load(
+    "//tensorflow/stream_executor:build_defs.bzl",
+    "stream_executor_friends",
+)
+load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda_is_configured")
+load("@local_config_rocm//rocm:build_defs.bzl", "if_rocm_is_configured")
+
+package_group(
+    name = "friends",
+    packages = stream_executor_friends(),
+)
+
+package(
+    default_visibility = [":friends"],
+)
+
+# Filegroup used to collect source files for the dependency check.
+filegroup(
+    name = "c_srcs",
+    data = glob([
+        "**/*.cc",
+        "**/*.h",
+    ]),
+)
+
+cc_library(
+    name = "gpu_activation_header",
+    hdrs = ["gpu_activation.h"],
+    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    deps = ["//tensorflow/stream_executor/platform"],
+)
+
+cc_library(
+    name = "gpu_activation",
+    srcs = ["gpu_activation.cc"],
+    hdrs = ["gpu_activation.h"],
+    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    deps = [
+        ":gpu_activation_header",
+        ":gpu_driver_header",
+        "//tensorflow/stream_executor",
+        "//tensorflow/stream_executor:stream_executor_internal",
+        "//tensorflow/stream_executor/platform",
+    ],
+)
+
+cc_library(
+    name = "gpu_diagnostics_header",
+    hdrs = ["gpu_diagnostics.h"],
+    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    deps = [
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+    ],
+)
+
+cc_library(
+    name = "gpu_driver_header",
+    hdrs = ["gpu_driver.h"],
+    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    deps = [
+        ":gpu_types_header",
+        "//tensorflow/stream_executor:device_options",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@local_config_cuda//cuda:cuda_headers",
+    ],
+)
+
+cc_library(
+    name = "gpu_event_header",
+    hdrs = ["gpu_event.h"],
+    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    deps = [
+        ":gpu_driver_header",
+        ":gpu_stream_header",
+        "//tensorflow/stream_executor:event",
+        "//tensorflow/stream_executor/lib",
+    ],
+)
+
+cc_library(
+    name = "gpu_event",
+    srcs = ["gpu_event.cc"],
+    hdrs = ["gpu_event.h"],
+    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    deps = [
+        ":gpu_driver_header",
+        ":gpu_executor_header",
+        ":gpu_stream",
+        "//tensorflow/stream_executor:stream_executor_headers",
+        "//tensorflow/stream_executor/lib",
+    ],
+)
+
+cc_library(
+    name = "gpu_executor_header",
+    hdrs = ["gpu_executor.h"],
+    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    deps = [
+        ":gpu_kernel_header",
+        "//tensorflow/stream_executor:event",
+        "//tensorflow/stream_executor:platform",
+        "//tensorflow/stream_executor:stream_executor_internal",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "gpu_helpers_header",
+    hdrs = ["gpu_helpers.h"],
+    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    deps = [":gpu_types_header"],
+)
+
+cc_library(
+    name = "gpu_kernel_header",
+    hdrs = ["gpu_kernel.h"],
+    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    deps = [
+        ":gpu_driver_header",
+        "//tensorflow/stream_executor:event",
+        "//tensorflow/stream_executor:stream_executor_pimpl_header",
+        "//tensorflow/stream_executor/platform",
+    ],
+)
+
+cc_library(
+    name = "gpu_rng_header",
+    hdrs = ["gpu_rng.h"],
+    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    deps = [
+        ":gpu_types_header",
+        "//tensorflow/stream_executor:plugin_registry",
+        "//tensorflow/stream_executor:rng",
+        "//tensorflow/stream_executor/platform",
+    ],
+)
+
+cc_library(
+    name = "gpu_stream_header",
+    hdrs = ["gpu_stream.h"],
+    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    deps = [
+        ":gpu_driver_header",
+        "//tensorflow/stream_executor:stream_executor_internal",
+        "//tensorflow/stream_executor/platform",
+    ],
+)
+
+cc_library(
+    name = "gpu_stream",
+    srcs = ["gpu_stream.cc"],
+    hdrs = ["gpu_stream.h"],
+    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    deps = [
+        ":gpu_driver_header",
+        ":gpu_executor_header",
+        "//tensorflow/stream_executor:stream_executor_headers",
+        "//tensorflow/stream_executor:stream_header",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+    ],
+)
+
+cc_library(
+    name = "gpu_timer_header",
+    hdrs = ["gpu_timer.h"],
+    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    deps = [
+        ":gpu_driver_header",
+        ":gpu_executor_header",
+        "//tensorflow/stream_executor:stream_executor_internal",
+    ],
+)
+
+cc_library(
+    name = "gpu_timer",
+    srcs = ["gpu_timer.cc"],
+    hdrs = ["gpu_timer.h"],
+    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    deps = [
+        ":gpu_driver_header",
+        ":gpu_executor_header",
+        ":gpu_stream",
+        "//tensorflow/stream_executor:stream_executor_headers",
+        "//tensorflow/stream_executor/lib",
+    ],
+)
+
+cc_library(
+    name = "gpu_types_header",
+    hdrs = ["gpu_types.h"],
+    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    deps = [
+        "//tensorflow/stream_executor/platform",
+    ] + if_cuda_is_configured([
+        "@local_config_cuda//cuda:cuda_headers",
+    ]) + if_rocm_is_configured([
+        "@local_config_rocm//rocm:rocm_headers",
+    ]),
+)
diff --git a/tensorflow/stream_executor/cuda/cuda_activation.cc b/tensorflow/stream_executor/gpu/gpu_activation.cc
similarity index 62%
rename from tensorflow/stream_executor/cuda/cuda_activation.cc
rename to tensorflow/stream_executor/gpu/gpu_activation.cc
index 02371c3c3a..6f74eef2db 100644
--- a/tensorflow/stream_executor/cuda/cuda_activation.cc
+++ b/tensorflow/stream_executor/gpu/gpu_activation.cc
@@ -1,4 +1,4 @@
-/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -13,36 +13,36 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/stream_executor/cuda/cuda_activation.h"
+#include "tensorflow/stream_executor/gpu/gpu_activation.h"
 
-#include "tensorflow/stream_executor/cuda/cuda_driver.h"
+#include "tensorflow/stream_executor/gpu/gpu_driver.h"
 #include "tensorflow/stream_executor/stream_executor.h"
 #include "tensorflow/stream_executor/stream_executor_internal.h"
 
 namespace stream_executor {
-namespace cuda {
+namespace gpu {
 
-CudaContext* ExtractCudaContext(CUDAExecutor *cuda_exec);
-CUDAExecutor *ExtractCudaExecutor(StreamExecutor *stream_exec);
+GpuContext* ExtractGpuContext(GpuExecutor* gpu_exec);
+GpuExecutor* ExtractGpuExecutor(StreamExecutor* stream_exec);
 
 ScopedActivateExecutorContext::ScopedActivateExecutorContext(
-    CUDAExecutor *cuda_exec):
-      driver_scoped_activate_context_(
-          new ScopedActivateContext{ExtractCudaContext(cuda_exec)}) { }
+    GpuExecutor* gpu_exec)
+    : driver_scoped_activate_context_(
+          new ScopedActivateContext{ExtractGpuContext(gpu_exec)}) {}
 
 ScopedActivateExecutorContext::ScopedActivateExecutorContext(
-    StreamExecutor *stream_exec)
-    : ScopedActivateExecutorContext(ExtractCudaExecutor(stream_exec)) {}
+    StreamExecutor* stream_exec)
+    : ScopedActivateExecutorContext(ExtractGpuExecutor(stream_exec)) {}
 
 ScopedActivateExecutorContext::~ScopedActivateExecutorContext() {
-  delete static_cast<ScopedActivateContext *>(driver_scoped_activate_context_);
+  delete static_cast<ScopedActivateContext*>(driver_scoped_activate_context_);
 }
 
 ScopedActivateExecutorContext::ScopedActivateExecutorContext(
-    ScopedActivateExecutorContext &&other)
+    ScopedActivateExecutorContext&& other)
     : driver_scoped_activate_context_(other.driver_scoped_activate_context_) {
   other.driver_scoped_activate_context_ = nullptr;
 }
 
-}  // namespace cuda
+}  // namespace gpu
 }  // namespace stream_executor
diff --git a/tensorflow/stream_executor/gpu/gpu_activation.h b/tensorflow/stream_executor/gpu/gpu_activation.h
new file mode 100644
index 0000000000..3409304d77
--- /dev/null
+++ b/tensorflow/stream_executor/gpu/gpu_activation.h
@@ -0,0 +1,61 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// This file contains APIs that assume a StreamExecutor is backed by CUDA.
+// It reaches into the CUDA implementation to activate an underlying CUDA
+// context.
+//
+// Having this file separate from gpu/gpu_executor.h means that dependent
+// code does not also have to depend on cuda.h.
+
+#ifndef TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_ACTIVATION_H_
+#define TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_ACTIVATION_H_
+
+#include "tensorflow/stream_executor/platform/port.h"
+
+namespace stream_executor {
+
+class StreamExecutor;
+
+namespace gpu {
+
+class GpuExecutor;
+class ScopedActivateContext;
+
+// Activates a CUDA context within an enclosing scope.
+class ScopedActivateExecutorContext {
+ public:
+  // Form that takes a CUDA executor implementation.
+  explicit ScopedActivateExecutorContext(GpuExecutor* gpu_exec);
+
+  // Form that takes a pImpl executor and extracts a CUDA implementation --
+  // fatal failure if it is not CUDA inside.
+  explicit ScopedActivateExecutorContext(StreamExecutor* stream_exec);
+
+  ScopedActivateExecutorContext(ScopedActivateExecutorContext&& other);
+
+  ~ScopedActivateExecutorContext();
+
+ private:
+  // The cuda.h-using datatype that we wrap.
+  ScopedActivateContext* driver_scoped_activate_context_;
+
+  SE_DISALLOW_COPY_AND_ASSIGN(ScopedActivateExecutorContext);
+};
+
+}  // namespace gpu
+}  // namespace stream_executor
+
+#endif  // TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_ACTIVATION_H_
diff --git a/tensorflow/stream_executor/gpu/gpu_diagnostics.h b/tensorflow/stream_executor/gpu/gpu_diagnostics.h
new file mode 100644
index 0000000000..71642109b5
--- /dev/null
+++ b/tensorflow/stream_executor/gpu/gpu_diagnostics.h
@@ -0,0 +1,99 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_DIAGNOSTICS_H_
+#define TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_DIAGNOSTICS_H_
+
+#include <tuple>
+#include "tensorflow/stream_executor/platform/port.h"
+
+#include "tensorflow/stream_executor/lib/statusor.h"
+#include "tensorflow/stream_executor/platform/port.h"
+
+namespace stream_executor {
+namespace gpu {
+
+// e.g. DriverVersion{346, 3, 4}
+using DriverVersion = std::tuple<int, int, int>;
+
+// FIXME: These functions are in stream_executor::cuda namespaces for now
+// Will move to stream_executor::gpu namespace in the near future
+//
+//// Converts a parsed driver version to string form.
+// string DriverVersionToString(DriverVersion version);
+//
+//// Converts a parsed driver version or status value to natural string form.
+// string DriverVersionStatusToString(port::StatusOr<DriverVersion> version);
+//
+//// Converts a string of a form like "331.79" to a DriverVersion{331, 79}.
+// port::StatusOr<DriverVersion> StringToDriverVersion(const string& value);
+
+class Diagnostician {
+ public:
+  // Logs diagnostic information when CUDA appears to be misconfigured (e.g. is
+  // not initializing).
+  //
+  // Note: if we're running on a machine that has no GPUs, we don't want to
+  // produce very much log spew beyond saying, "looks like there's no CUDA
+  // kernel
+  // module running".
+  //
+  // Note: we use non-Google-File:: API here because we may be called before
+  // InitGoogle has completed.
+  static void LogDiagnosticInformation();
+
+  // Given the driver version file contents, finds the kernel module version and
+  // returns it as a string.
+  //
+  // This is solely used for more informative log messages when the user is
+  // running on a machine that happens to have a libcuda/kernel driver mismatch.
+  static port::StatusOr<DriverVersion> FindKernelModuleVersion(
+      const string& driver_version_file_contents);
+
+  // Extracts the kernel driver version from the current host.
+  static port::StatusOr<DriverVersion> FindKernelDriverVersion();
+
+  // Iterates through loaded DSOs with DlIteratePhdrCallback to find the
+  // driver-interfacing DSO version number. Returns it as a string.
+  static port::StatusOr<DriverVersion> FindDsoVersion();
+
+  // Logs information about the kernel driver version and userspace driver
+  // library version.
+  static void LogDriverVersionInformation();
+
+ private:
+  // Given the DSO version number and the driver version file contents, extracts
+  // the driver version and compares, warning the user in the case of
+  // incompatibility.
+  //
+  // This is solely used for more informative log messages when the user is
+  // running on a machine that happens to have a libcuda/kernel driver mismatch.
+  static void WarnOnDsoKernelMismatch(
+      port::StatusOr<DriverVersion> dso_version,
+      port::StatusOr<DriverVersion> kernel_version);
+
+  // Logs information about the dev nodes present on this machine: their
+  // existence, permissions, accessibility from this uid/gid.
+  static void LogDevNodeDiagnosticInformation();
+
+  static string GetDevNodePath(int dev_node_ordinal);
+
+  SE_DISALLOW_COPY_AND_ASSIGN(Diagnostician);
+};
+
+}  // namespace gpu
+}  // namespace stream_executor
+
+#endif  // TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_DIAGNOSTICS_H_
diff --git a/tensorflow/stream_executor/gpu/gpu_driver.h b/tensorflow/stream_executor/gpu/gpu_driver.h
new file mode 100644
index 0000000000..a5ef48db47
--- /dev/null
+++ b/tensorflow/stream_executor/gpu/gpu_driver.h
@@ -0,0 +1,525 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// CUDA userspace driver library wrapper functionality.
+
+#ifndef TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_DRIVER_H_
+#define TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_DRIVER_H_
+
+#include <stddef.h>
+#include "tensorflow/stream_executor/platform/port.h"
+
+#include "cuda/include/cuda.h"
+#include "tensorflow/stream_executor/device_options.h"
+#include "tensorflow/stream_executor/lib/status.h"
+#include "tensorflow/stream_executor/lib/statusor.h"
+#include "tensorflow/stream_executor/platform/port.h"
+
+#include "tensorflow/stream_executor/gpu/gpu_types.h"
+
+namespace stream_executor {
+namespace gpu {
+
+// Identifies the memory space where an allocation resides. See
+// GpuDriver::GetPointerMemorySpace().
+enum class MemorySpace { kHost, kDevice };
+
+// Returns a casual string, such as "host" for the provided memory space.
+string MemorySpaceString(MemorySpace memory_space);
+
+class GpuContext;
+
+// GpuDriver contains wrappers for calls to the userspace library driver. It's
+// useful to isolate these calls and put basic wrappers around them to separate
+// userspace library driver behaviors from the rest of the program.
+//
+// At the moment it's simply used as a namespace.
+//
+// The calls log any specific errors internally and return whether the operation
+// was successful to the caller.
+//
+// The order of parameters is generally kept symmetric with the underlying CUDA
+// driver API.
+//
+// Links on functions are to specific documentation under
+// http://docs.nvidia.com/cuda/cuda-driver-api/
+//
+// Thread safety: these functions should not be used from signal handlers.
+class GpuDriver {
+ public:
+  // Wraps a call to cuInit with logging to help indicate what has gone wrong in
+  // the case of failure. Safe to call multiple times; will be fast on all calls
+  // after the first.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__INITIALIZE.html#group__CUDA__INITIALIZE_1g0a2f1517e1bd8502c7194c3a8c134bc3
+  static port::Status Init();
+
+  // Returns the device associated with the given context.
+  // device is an outparam owned by the caller, must not be null.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g4e84b109eba36cdaaade167f34ae881e
+  static port::StatusOr<GpuDeviceHandle> DeviceFromContext(GpuContext* context);
+
+  // Creates a new CUDA stream associated with the given context via
+  // cuStreamCreate.
+  // stream is an outparam owned by the caller, must not be null.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#group__CUDA__STREAM_1ga581f0c5833e21ded8b5a56594e243f4
+  static bool CreateStream(GpuContext* context, GpuStreamHandle* stream);
+
+  // Destroys a CUDA stream associated with the given context.
+  // stream is owned by the caller, must not be null, and *stream is set to null
+  // if the stream is successfully destroyed.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#group__CUDA__STREAM_1g244c8833de4596bcd31a06cdf21ee758
+  static void DestroyStream(GpuContext* context, GpuStreamHandle* stream);
+
+  // CUDA events can explicitly disable event TSC retrieval for some presumed
+  // performance improvement if timing is unnecessary.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g450687e75f3ff992fe01662a43d9d3db
+  enum class EventFlags { kDefault, kDisableTiming };
+
+  // Creates a new event associated with the given context.
+  // result is an outparam owned by the caller and must not be null.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g450687e75f3ff992fe01662a43d9d3db
+  static port::Status CreateEvent(GpuContext* context, GpuEventHandle* result,
+                                  EventFlags flags);
+
+  // Destroys *event and turns it into a nullptr. event may not be null, but
+  // *event may be, via cuEventDestroy
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g593ec73a8ec5a5fc031311d3e4dca1ef
+  static port::Status DestroyEvent(GpuContext* context, GpuEventHandle* event);
+
+  // Allocates a GPU memory space of size bytes associated with the given
+  // context via cuMemAlloc.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gb82d2a09844a58dd9e744dc31e8aa467
+  static void* DeviceAllocate(GpuContext* context, uint64 bytes);
+
+  // Deallocates a GPU memory space of size bytes associated with the given
+  // context via cuMemFree.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g89b3f154e17cc89b6eea277dbdf5c93a
+  static void DeviceDeallocate(GpuContext* context, void* location);
+
+  // Allocates a unified memory space of size bytes associated with the given
+  // context via cuMemAllocManaged.
+  // https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gb347ded34dc326af404aa02af5388a32
+  // (supported on CUDA only)
+  static void* UnifiedMemoryAllocate(GpuContext* context, uint64 bytes);
+
+  // Deallocates a unified memory space of size bytes associated with the given
+  // context via cuMemFree.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g89b3f154e17cc89b6eea277dbdf5c93a
+  // (supported on CUDA only)
+  static void UnifiedMemoryDeallocate(GpuContext* context, void* location);
+
+  // Allocates page-locked and CUDA-registered memory on the host via
+  // cuMemAllocHost.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gdd8311286d2c2691605362c689bc64e0
+  static void* HostAllocate(GpuContext* context, uint64 bytes);
+
+  // Deallocates a location created by HostAllocate, via cuMemFreeHost.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g62e0fdbe181dab6b1c90fa1a51c7b92c
+  static void HostDeallocate(GpuContext* context, void* location);
+
+  // Registers a memory region at location of size bytes via cuMemHostRegister.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gf0a9fe11544326dabd743b7aa6b54223
+  static bool HostRegister(GpuContext* context, void* location, uint64 bytes);
+
+  // Unregisters a memory region that was previously registered at location via
+  // cuMemHostUnregister.
+  //
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g63f450c8125359be87b7623b1c0b2a14
+  //
+  // TODO(leary) verify an error will be returned if the location wasn't
+  // previously registered.
+  static bool HostUnregister(GpuContext* context, void* location);
+
+  // Given a device ordinal, returns a device handle into the device outparam,
+  // which must not be null.
+  //
+  // N.B. these device handles do not have a corresponding destroy function in
+  // the CUDA driver API.
+  static port::Status GetDevice(int device_ordinal, GpuDeviceHandle* device);
+
+  // Given a device handle, returns the name reported by the driver for the
+  // device.
+  static bool GetDeviceName(GpuDeviceHandle device, string* device_name);
+
+  // Given a device to create a context for, returns a context handle into the
+  // context outparam, which must not be null.
+  //
+  // N.B. CUDA contexts are weird. They are implicitly associated with the
+  // calling thread. Current documentation on contexts and their influence on
+  // userspace processes is given here:
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g65dc0012348bc84810e2103a40d8e2cf
+  static port::Status CreateContext(int device_ordinal, GpuDeviceHandle device,
+                                    const DeviceOptions& device_options,
+                                    GpuContext** context);
+
+  // Destroys the provided context via cuCtxDestroy.
+  // Don't do this while clients could still be using the context, per the docs
+  // bad things will happen.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g27a365aebb0eb548166309f58a1e8b8e
+  static void DestroyContext(GpuContext* context);
+
+  // Queries the runtime for the specified attribute of the specified function.
+  // cuFuncGetAttribute (the underlying CUDA driver API routine) only operates
+  // in terms of integer-sized values, so there's no potential for overrun (as
+  // of CUDA 5.5).
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EXEC.html#group__CUDA__EXEC_1g5e92a1b0d8d1b82cb00dcfb2de15961b
+  static bool FuncGetAttribute(GpuFunctionAttribute attribute,
+                               GpuFunctionHandle function,
+                               int* attribute_value);
+
+  // Sets the preferred cache configuration for the specified function.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EXEC.html#group__CUDA__EXEC_1g40f8c11e81def95dc0072a375f965681
+  static bool FuncSetCacheConfig(GpuFunctionHandle function,
+                                 GpuFuncCachePreference cache_config);
+
+  // Gets the preferred shared memory bank configuration for the specified
+  // CONTEXT (not function!), either default or four- or eight-byte bank size.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g17153a1b8b8c756f7ab8505686a4ad74
+  static port::StatusOr<GpuSharedMemConfig> ContextGetSharedMemConfig(
+      GpuContext* context);
+
+  // Sets the preferred shared memory bank configuration for the specified
+  // CONTEXT (not function!), either default or four- or eight-byte bank size.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g2574235fa643f8f251bf7bc28fac3692
+  static port::Status ContextSetSharedMemConfig(
+      GpuContext* context, GpuSharedMemConfig shared_mem_config);
+
+  // Launches a CUDA kernel via cuLaunchKernel.
+  // TODO(leary) describe the structure of kernel_params and extra in a readable
+  // way.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EXEC.html#group__CUDA__EXEC_1gb8f3dc3031b40da29d5f9a7139e52e15
+  static bool LaunchKernel(GpuContext* context, GpuFunctionHandle function,
+                           unsigned int grid_dim_x, unsigned int grid_dim_y,
+                           unsigned int grid_dim_z, unsigned int block_dim_x,
+                           unsigned int block_dim_y, unsigned int block_dim_z,
+                           unsigned int shared_mem_bytes,
+                           GpuStreamHandle stream, void** kernel_params,
+                           void** extra);
+
+  // Loads ptx_contents with the CUDA driver's PTX JIT and stores the resulting
+  // handle in "module". Any error logs that are produced are logged internally.
+  // (supported on CUDA only)
+  static bool LoadPtx(GpuContext* context, const char* ptx_contents,
+                      GpuModuleHandle* module);
+
+  // Loads cubin_bytes with the CUDA driver's blob loading interface and stores
+  // the resulting handle in "module".
+  // (supported on CUDA only)
+  static port::Status LoadCubin(GpuContext* context, const char* cubin_bytes,
+                                GpuModuleHandle* module);
+
+  // Loads HSACO with the ROCM runtime and stores the resulting handle in
+  // "module". Any error logs that are produced are logged internally.
+  // (supported on ROCm only)
+  static bool LoadHsaco(GpuContext* context, const char* hsaco_contents,
+                        GpuModuleHandle* module);
+
+  // Retrieves a named kernel from a loaded module, and places the resulting
+  // handle into function (outparam) on success. Neither kernel_name nor
+  // function may be null. No ownership is taken of kernel_name.
+  static bool GetModuleFunction(GpuContext* context, GpuModuleHandle module,
+                                const char* kernel_name,
+                                GpuFunctionHandle* function);
+
+  // Retrieves a named global/constant symbol from a loaded module, and returns
+  // a device pointer and size of the symbol on success. symbol_name may not be
+  // null. At least one of dptr or bytes should not be null. No ownership is
+  // taken of symbol_name.
+  static bool GetModuleSymbol(GpuContext* context, GpuModuleHandle module,
+                              const char* symbol_name, GpuDevicePtr* dptr,
+                              size_t* bytes);
+
+  // Unloads module from the current context via cuModuleUnload.
+  // TODO(leary) the documentation doesn't say what kind of disasters happen
+  // if you try to unload a module while its GpuFunctionHandles are in use.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MODULE.html#group__CUDA__MODULE_1g8ea3d716524369de3763104ced4ea57b
+  static void UnloadModule(GpuContext* context, GpuModuleHandle module);
+
+  // Performs a synchronous memset of the device memory segment via cuMemsetD8.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g6e582bf866e9e2fb014297bfaf354d7b
+  static bool SynchronousMemsetUint8(GpuContext* context, GpuDevicePtr location,
+                                     uint8 value, size_t size);
+
+  // Performs a synchronous memset of the device memory segment via cuMemsetD32.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g983e8d8759acd1b64326317481fbf132
+  static bool SynchronousMemsetUint32(GpuContext* context,
+                                      GpuDevicePtr location, uint32 value,
+                                      size_t uint32_count);
+
+  // Performs an asynchronous memset of the device memory segment via
+  // cuMemsetD8Async.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gaef08a7ccd61112f94e82f2b30d43627
+  static bool AsynchronousMemsetUint8(GpuContext* context,
+                                      GpuDevicePtr location, uint8 value,
+                                      size_t uint32_count,
+                                      GpuStreamHandle stream);
+
+  // Performs an asynchronous memset of the device memory segment via
+  // cuMemsetD32Async.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g58229da5d30f1c0cdf667b320ec2c0f5
+  static bool AsynchronousMemsetUint32(GpuContext* context,
+                                       GpuDevicePtr location, uint32 value,
+                                       size_t uint32_count,
+                                       GpuStreamHandle stream);
+
+  // -- Synchronous memcopies.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g4d32266788c440b0220b1a9ba5795169
+
+  static port::Status SynchronousMemcpyD2H(GpuContext* context, void* host_dst,
+                                           GpuDevicePtr gpu_src, uint64 size);
+  static port::Status SynchronousMemcpyH2D(GpuContext* context,
+                                           GpuDevicePtr gpu_dst,
+                                           const void* host_src, uint64 size);
+  static port::Status SynchronousMemcpyD2D(GpuContext* context,
+                                           GpuDevicePtr gpu_dst,
+                                           GpuDevicePtr gpu_src, uint64 size);
+
+  // -- Asynchronous memcopies.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g56f30236c7c5247f8e061b59d3268362
+
+  static bool AsynchronousMemcpyD2H(GpuContext* context, void* host_dst,
+                                    GpuDevicePtr gpu_src, uint64 size,
+                                    GpuStreamHandle stream);
+  static bool AsynchronousMemcpyH2D(GpuContext* context, GpuDevicePtr gpu_dst,
+                                    const void* host_src, uint64 size,
+                                    GpuStreamHandle stream);
+  static bool AsynchronousMemcpyD2D(GpuContext* context, GpuDevicePtr gpu_dst,
+                                    GpuDevicePtr gpu_src, uint64 size,
+                                    GpuStreamHandle stream);
+
+  // The CUDA stream callback type signature.
+  // The data passed to AddStreamCallback is subsequently passed to this
+  // callback when it fires.
+  //
+  // Some notable things:
+  // * Callbacks must not make any CUDA API calls.
+  // * Callbacks from independent streams execute in an undefined order and may
+  //   be serialized.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#group__CUDA__STREAM_1g613d97a277d7640f4cb1c03bd51c2483
+  typedef void (*StreamCallback)(GpuStreamHandle stream, GpuStatus status,
+                                 void* data);
+
+  // Enqueues a callback operation into stream.
+  // See StreamCallback above and the NVIDIA documentation for additional
+  // details.
+  static bool AddStreamCallback(GpuContext* context, GpuStreamHandle stream,
+                                StreamCallback callback, void* data);
+
+  // Causes stream to wait for event to trigger before proceeding via
+  // cuStreamWaitEvent.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#axzz334nAXAhM
+  static bool WaitStreamOnEvent(GpuContext* context, GpuStreamHandle stream,
+                                GpuEventHandle event);
+
+  // Blocks the calling thread until the operations enqueued onto stream have
+  // been completed, via cuStreamSynchronize.
+  //
+  // TODO(leary) if a pathological thread enqueues operations onto the stream
+  // while another thread blocks like this, can you wind up waiting an unbounded
+  // amount of time?
+  //
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#group__CUDA__STREAM_1g15e49dd91ec15991eb7c0a741beb7dad
+  static port::Status SynchronizeStream(GpuContext* context,
+                                        GpuStreamHandle stream);
+
+  // Blocks the calling thread until the operations associated with the context
+  // have been completed, via cuCtxSynchronize.
+  //
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g7a54725f28d34b8c6299f0c6ca579616
+  static bool SynchronizeContext(GpuContext* context);
+
+  // Returns true if all stream tasks have completed at time of the call. Note
+  // the potential for races around this call (if another thread adds work to
+  // the stream immediately after this returns).
+  static bool IsStreamIdle(GpuContext* context, GpuStreamHandle stream);
+
+  // Returns whether code in the from context can access memory in the to
+  // context via cuDeviceCanAccessPeer.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__PEER__ACCESS.html#group__CUDA__PEER__ACCESS_1g496bdaae1f632ebfb695b99d2c40f19e
+  static bool CanEnablePeerAccess(GpuContext* from, GpuContext* to);
+
+  // Enables peer access per CanEnablePeerAccess, via cuCtxEnablePeerAccess.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__PEER__ACCESS.html#group__CUDA__PEER__ACCESS_1g0889ec6728e61c05ed359551d67b3f5a
+  static port::Status EnablePeerAccess(GpuContext* from, GpuContext* to);
+
+  // Returns the elapsed milliseconds between start and stop via
+  // cuEventElapsedTime.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1gdfb1178807353bbcaa9e245da497cf97
+  static bool GetEventElapsedTime(GpuContext* context,
+                                  float* elapsed_milliseconds,
+                                  GpuEventHandle start, GpuEventHandle stop);
+
+  // Records that an event occurred when execution reaches the current point in
+  // thestream via cuEventRecord.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g95424d3be52c4eb95d83861b70fb89d1
+  static port::Status RecordEvent(GpuContext* context, GpuEventHandle event,
+                                  GpuStreamHandle stream);
+
+  // Polls (without blocking) to determine the status of an event - pending or
+  // complete (or an error status).
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g6f0704d755066b0ee705749ae911deef
+  static port::StatusOr<GpuStatus> QueryEvent(GpuContext* context,
+                                              GpuEventHandle event);
+
+  // -- Pointer-specific calls.
+
+  // Returns the context in which pointer was allocated or registered.
+  static port::StatusOr<GpuContext*> GetPointerContext(GpuDevicePtr pointer);
+
+  // Returns the device associated with the context from GetPointerContext().
+  static port::StatusOr<GpuDeviceHandle> GetPointerDevice(GpuDevicePtr pointer);
+
+  // Returns the memory space addressed by pointer.
+  static port::StatusOr<MemorySpace> GetPointerMemorySpace(
+      GpuDevicePtr pointer);
+
+  // Returns the base address and size of the device pointer dptr.
+  static port::Status GetPointerAddressRange(GpuDevicePtr dptr,
+                                             GpuDevicePtr* base, size_t* size);
+
+  // -- Device-specific calls.
+
+  // Returns the compute capability for the device; i.e (3, 5).
+  // This is currently done via the deprecated device API.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE__DEPRECATED.html#group__CUDA__DEVICE__DEPRECATED_1ge2091bbac7e1fb18c2821612115607ea
+  // (supported on CUDA only)
+  static port::Status GetComputeCapability(int* cc_major, int* cc_minor,
+                                           GpuDeviceHandle device);
+
+  // Returns Gpu ISA version for the device; i.e 803, 900.
+  // (supported on ROCm only)
+  static port::Status GetGpuISAVersion(int* version, GpuDeviceHandle device);
+
+  // Returns the number of multiprocessors on the device (note that the device
+  // may be multi-GPU-per-board).
+  static port::StatusOr<int> GetMultiprocessorCount(GpuDeviceHandle device);
+
+  // Returns the limit on number of threads that can be resident in a single
+  // multiprocessor.
+  static port::StatusOr<int64> GetMaxThreadsPerMultiprocessor(
+      GpuDeviceHandle device);
+
+  // Returns the limit on number of threads which may be resident for a single
+  // block (cooperative thread array).
+  static port::StatusOr<int64> GetMaxThreadsPerBlock(GpuDeviceHandle device);
+
+  // Returns the amount of shared memory available on a single GPU core (i.e.
+  // SM on NVIDIA devices).
+  static port::StatusOr<int64> GetMaxSharedMemoryPerCore(
+      GpuDeviceHandle device);
+
+  // Returns the amount of shared memory available for a single block
+  // (cooperative thread array).
+  static port::StatusOr<int64> GetMaxSharedMemoryPerBlock(
+      GpuDeviceHandle device);
+
+  // Returns the maximum supported number of registers per block.
+  static port::StatusOr<int64> GetMaxRegistersPerBlock(GpuDeviceHandle device);
+
+  // Returns the number of threads per warp.
+  static port::StatusOr<int64> GetThreadsPerWarp(GpuDeviceHandle device);
+
+  // Queries the grid limits for device with cuDeviceGetAttribute calls.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g9c3e1414f0ad901d3278a4d6645fc266
+  static bool GetGridLimits(int* x, int* y, int* z, GpuDeviceHandle device);
+
+  // Returns a grab-bag of device properties in a caller-owned device_properties
+  // structure for device_ordinal via cuDeviceGetProperties.
+  //
+  // This call is deprecated in the NVIDIA driver API; its replacement is
+  // GetDeviceAttribute
+  //
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE__DEPRECATED.html#group__CUDA__DEVICE__DEPRECATED_1g65a5b4e25186bd257df80b98c98cffe6
+  static bool GetDeviceProperties(GpuDeviceProperty* device_properties,
+                                  int device_ordinal);
+
+  // Gets a specific integer-valued property about the given device.
+  //
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g9c3e1414f0ad901d3278a4d6645fc266
+  static port::StatusOr<int> GetDeviceAttribute(GpuDeviceAttribute attribute,
+                                                GpuDeviceHandle device);
+
+  // Returns whether ECC is enabled for the given GpuDeviceHandle via
+  // cuDeviceGetattribute with CU_DEVICE_ATTRIBUTE_ECC_ENABLED.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g9c3e1414f0ad901d3278a4d6645fc266
+  static bool IsEccEnabled(GpuDeviceHandle device, bool* result);
+
+  // Returns the total amount of memory available for allocation by the CUDA
+  // context, in bytes, via cuDeviceTotalMem.
+  static bool GetDeviceTotalMemory(GpuDeviceHandle device, uint64* result);
+
+  // Returns the free amount of memory and total amount of memory, as reported
+  // by cuMemGetInfo.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g808f555540d0143a331cc42aa98835c0
+  static bool GetDeviceMemoryInfo(GpuContext* context, int64* free,
+                                  int64* total);
+
+  // Returns a PCI bus id string for the device.
+  // [domain]:[bus]:[device].[function]
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g85295e7d9745ab8f0aa80dd1e172acfc
+  static string GetPCIBusID(GpuDeviceHandle device);
+
+  // -- Context- and device-independent calls.
+
+  // Returns the number of visible CUDA device via cuDeviceGetCount.
+  // This should correspond to the set of device ordinals available.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g52b5ce05cb8c5fb6831b2c0ff2887c74
+  static int GetDeviceCount();
+
+  // Returns the driver version number via cuDriverGetVersion.
+  // This is, surprisingly, NOT the actual driver version (e.g. 331.79) but,
+  // instead, the CUDA toolkit release number that this driver is compatible
+  // with; e.g. 6000 (for a CUDA 6.0 compatible driver) or 6050 (for a CUDA 6.5
+  // compatible driver).
+  //
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__VERSION.html#group__CUDA__VERSION_1g8b7a10395392e049006e61bcdc8ebe71
+  static bool GetDriverVersion(int* driver_version);
+
+  // -- Other calls
+
+  // Returns the maximum number of blocks (per multiprocessor) occupied by the
+  // specified kernel/GpuFunctionHandle when launched with the specified
+  // parameters.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__OCCUPANCY.html#group__CUDA__OCCUPANCY_1gcc6e1094d05cba2cee17fe33ddd04a98
+  static port::StatusOr<int> GetMaxOccupiedBlocksPerCore(
+      GpuContext* context, GpuFunctionHandle kernel, int threads_per_block,
+      size_t dynamic_shared_memory_bytes);
+
+  // Seam for injecting an error at CUDA initialization time for testing
+  // purposes.
+  static bool driver_inject_init_error_;
+};
+
+// Ensures a context is activated within a scope.
+class ScopedActivateContext {
+ public:
+  // Activates the context via cuCtxSetCurrent, if it is not the currently
+  // active context (a la cuCtxGetCurrent). Note the alternative push/pop
+  // mechanism is said by NVIDIA to be relatively slow and deprecated.
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1gbe562ee6258b4fcc272ca6478ca2a2f7
+  explicit ScopedActivateContext(GpuContext* context);
+
+  // Checks that the context has remained activated for the duration of the
+  // scope.
+  ~ScopedActivateContext();
+
+ private:
+  GpuContext* to_restore_ = nullptr;
+};
+
+}  // namespace gpu
+}  // namespace stream_executor
+
+#endif  // TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_DRIVER_H_
diff --git a/tensorflow/stream_executor/gpu/gpu_event.cc b/tensorflow/stream_executor/gpu/gpu_event.cc
new file mode 100644
index 0000000000..a523958550
--- /dev/null
+++ b/tensorflow/stream_executor/gpu/gpu_event.cc
@@ -0,0 +1,47 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/stream_executor/gpu/gpu_event.h"
+
+#include "tensorflow/stream_executor/gpu/gpu_executor.h"
+#include "tensorflow/stream_executor/gpu/gpu_stream.h"
+#include "tensorflow/stream_executor/lib/statusor.h"
+
+namespace stream_executor {
+namespace gpu {
+
+GpuEvent::GpuEvent(GpuExecutor* parent)
+    : parent_(parent), gpu_event_(nullptr) {}
+
+GpuEvent::~GpuEvent() {}
+
+port::Status GpuEvent::Init() {
+  return GpuDriver::CreateEvent(parent_->gpu_context(), &gpu_event_,
+                                GpuDriver::EventFlags::kDisableTiming);
+}
+
+port::Status GpuEvent::Destroy() {
+  return GpuDriver::DestroyEvent(parent_->gpu_context(), &gpu_event_);
+}
+
+port::Status GpuEvent::Record(GpuStream* stream) {
+  return GpuDriver::RecordEvent(parent_->gpu_context(), gpu_event_,
+                                stream->gpu_stream());
+}
+
+GpuEventHandle GpuEvent::gpu_event() { return gpu_event_; }
+
+}  // namespace gpu
+}  // namespace stream_executor
diff --git a/tensorflow/stream_executor/gpu/gpu_event.h b/tensorflow/stream_executor/gpu/gpu_event.h
new file mode 100644
index 0000000000..61f39d42fe
--- /dev/null
+++ b/tensorflow/stream_executor/gpu/gpu_event.h
@@ -0,0 +1,62 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_EVENT_H_
+#define TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_EVENT_H_
+
+#include "tensorflow/stream_executor/event.h"
+#include "tensorflow/stream_executor/gpu/gpu_driver.h"
+#include "tensorflow/stream_executor/gpu/gpu_stream.h"
+#include "tensorflow/stream_executor/lib/status.h"
+
+namespace stream_executor {
+namespace gpu {
+
+// GpuEvent wraps a GpuEventHandle in the platform-independent EventInterface
+// interface.
+class GpuEvent : public internal::EventInterface {
+ public:
+  explicit GpuEvent(GpuExecutor* parent);
+
+  ~GpuEvent() override;
+
+  // Populates the CUDA-platform-specific elements of this object.
+  port::Status Init();
+
+  // Deallocates any platform-specific elements of this object. This is broken
+  // out (not part of the destructor) to allow for error reporting.
+  port::Status Destroy();
+
+  // Inserts the event at the current position into the specified stream.
+  port::Status Record(GpuStream* stream);
+
+  // Polls the CUDA platform for the event's current status.
+  Event::Status PollForStatus();
+
+  // The underlying CUDA event element.
+  GpuEventHandle gpu_event();
+
+ private:
+  // The Executor used to which this object and GpuEventHandle are bound.
+  GpuExecutor* parent_;
+
+  // The underlying CUDA event element.
+  GpuEventHandle gpu_event_;
+};
+
+}  // namespace gpu
+}  // namespace stream_executor
+
+#endif  // TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_EVENT_H_
diff --git a/tensorflow/stream_executor/gpu/gpu_executor.h b/tensorflow/stream_executor/gpu/gpu_executor.h
new file mode 100644
index 0000000000..6f969a98d2
--- /dev/null
+++ b/tensorflow/stream_executor/gpu/gpu_executor.h
@@ -0,0 +1,347 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// The CUDA implementation of the StreamExecutorInterface functionality.
+// CUDA inclusions are ideally confined to this implementation file.
+//
+// The notions from the StreamExecutor basically correspond to the CUDA streams
+// programming model provided by the libcuda.so driver APIs, so we don't have
+// to do much more than wrap the calls to the libraries appropriately.
+#ifndef TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_EXECUTOR_H_
+#define TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_EXECUTOR_H_
+
+#include <set>
+#include <unordered_map>
+
+#include "absl/strings/string_view.h"
+#include "tensorflow/stream_executor/event.h"
+#include "tensorflow/stream_executor/gpu/gpu_kernel.h"
+#include "tensorflow/stream_executor/lib/status.h"
+#include "tensorflow/stream_executor/lib/statusor.h"
+#include "tensorflow/stream_executor/platform.h"
+#include "tensorflow/stream_executor/platform/mutex.h"
+#include "tensorflow/stream_executor/platform/port.h"
+#include "tensorflow/stream_executor/platform/thread_annotations.h"
+#include "tensorflow/stream_executor/stream_executor_internal.h"
+
+namespace stream_executor {
+namespace gpu {
+
+// CUDA-platform implementation of the platform-agnostic
+// StreamExecutorInferface.
+class GpuExecutor : public internal::StreamExecutorInterface {
+ public:
+  // sub_platform indicates the subplatform used in this executor; it must
+  // be a CUDA type.
+  explicit GpuExecutor(const PluginConfig& plugin_config)
+      : device_(0),
+        context_(nullptr),
+        device_ordinal_(0),
+        cc_major_(0),
+        cc_minor_(0),
+        version_(0),
+        plugin_config_(plugin_config) {}
+
+  // See the corresponding StreamExecutor methods for method comments on the
+  // following overrides.
+
+  ~GpuExecutor() override;
+
+  port::Status Init(int device_ordinal, DeviceOptions device_options) override;
+
+  bool GetKernel(const MultiKernelLoaderSpec& spec,
+                 KernelBase* kernel) override;
+  // (supported on CUDA only)
+  void UnloadKernel(const KernelBase* kernel) override;
+  bool LoadModule(const MultiModuleLoaderSpec& spec,
+                  ModuleHandle* module_handle) override;
+  bool UnloadModule(ModuleHandle module_handle) override;
+
+  bool Launch(Stream* stream, const ThreadDim& thread_dims,
+              const BlockDim& block_dims, const KernelBase& k,
+              const KernelArgsArrayBase& args) override;
+
+  // (supported on CUDA only)
+  int CalculateOccupancy(const DeviceDescription& device_description,
+                         uint64 registers_per_thread,
+                         uint64 shared_memory_per_block,
+                         const ThreadDim& thread_dims, GpuFunctionHandle func);
+
+  // (supported on CUDA only)
+  int CompareOccupancy(int* initial_blocks,
+                       const DeviceDescription& device_description,
+                       uint64 registers_per_thread,
+                       uint64 shared_memory_per_block,
+                       const ThreadDim& thread_dims, GpuFunctionHandle func);
+
+  void* Allocate(uint64 size) override;
+
+  void* AllocateSubBuffer(DeviceMemoryBase* mem, uint64 offset_bytes,
+                          uint64 size_bytes) override;
+
+  void Deallocate(DeviceMemoryBase* mem) override;
+
+  void* UnifiedMemoryAllocate(uint64 size) override {
+    return GpuDriver::UnifiedMemoryAllocate(context_, size);
+  }
+
+  void UnifiedMemoryDeallocate(void* location) override {
+    return GpuDriver::UnifiedMemoryDeallocate(context_, location);
+  }
+
+  // CUDA allocation/registration functions are necessary because the driver
+  // internally sets up buffers for DMA operations (and page locks them).
+  // There's no external interface for us to otherwise control these DMA
+  // settings.
+  void* HostMemoryAllocate(uint64 size) override {
+    return GpuDriver::HostAllocate(context_, size);
+  }
+
+  void HostMemoryDeallocate(void* location) override {
+    return GpuDriver::HostDeallocate(context_, location);
+  }
+
+  bool HostMemoryRegister(void* location, uint64 size) override;
+
+  bool HostMemoryUnregister(void* location) override;
+
+  bool SynchronizeAllActivity() override;
+
+  bool SynchronousMemZero(DeviceMemoryBase* location, uint64 size) override;
+
+  bool SynchronousMemSet(DeviceMemoryBase* location, int value,
+                         uint64 size) override;
+
+  port::Status SynchronousMemcpy(DeviceMemoryBase* gpu_dst,
+                                 const void* host_src, uint64 size) override;
+
+  port::Status SynchronousMemcpy(void* host_dst,
+                                 const DeviceMemoryBase& gpu_src,
+                                 uint64 size) override;
+
+  port::Status SynchronousMemcpyDeviceToDevice(DeviceMemoryBase* gpu_dst,
+                                               const DeviceMemoryBase& gpu_src,
+                                               uint64 size) override;
+
+  bool MemZero(Stream* stream, DeviceMemoryBase* location,
+               uint64 size) override;
+  bool Memset(Stream* stream, DeviceMemoryBase* location, uint8 pattern,
+              uint64 size) override;
+  bool Memset32(Stream* stream, DeviceMemoryBase* location, uint32 pattern,
+                uint64 size) override;
+
+  bool Memcpy(Stream* stream, void* host_dst, const DeviceMemoryBase& gpu_src,
+              uint64 size) override;
+
+  bool Memcpy(Stream* stream, DeviceMemoryBase* gpu_dst, const void* host_src,
+              uint64 size) override;
+
+  bool MemcpyDeviceToDevice(Stream* stream, DeviceMemoryBase* gpu_dst,
+                            const DeviceMemoryBase& gpu_src,
+                            uint64 size) override;
+
+  bool HostCallback(Stream* stream,
+                    std::function<port::Status()> callback) override;
+
+  bool AllocateStream(Stream* stream) override;
+
+  void DeallocateStream(Stream* stream) override;
+
+  bool CreateStreamDependency(Stream* dependent, Stream* other) override;
+
+  bool AllocateTimer(Timer* timer) override;
+
+  void DeallocateTimer(Timer* timer) override;
+
+  bool StartTimer(Stream* stream, Timer* timer) override;
+
+  bool StopTimer(Stream* stream, Timer* timer) override;
+
+  port::Status AllocateEvent(Event* event) override;
+
+  port::Status DeallocateEvent(Event* event) override;
+
+  port::Status RecordEvent(Stream* stream, Event* event) override;
+
+  port::Status WaitForEvent(Stream* stream, Event* event) override;
+
+  Event::Status PollForEventStatus(Event* event) override;
+
+  port::Status BlockHostUntilDone(Stream* stream) override;
+
+  int PlatformDeviceCount() override { return GpuDriver::GetDeviceCount(); }
+
+  port::Status EnablePeerAccessTo(StreamExecutorInterface* other) override;
+
+  bool CanEnablePeerAccessTo(StreamExecutorInterface* other) override;
+
+  SharedMemoryConfig GetDeviceSharedMemoryConfig() override;
+
+  port::Status SetDeviceSharedMemoryConfig(SharedMemoryConfig config) override;
+
+  bool DeviceMemoryUsage(int64* free, int64* total) const override;
+
+  // Search for the symbol and returns a device pointer and size.
+  // Returns false if symbol does not exist.
+  bool GetSymbol(const string& symbol_name, ModuleHandle module_handle,
+                 void** mem, size_t* bytes) override;
+
+  DeviceDescription* PopulateDeviceDescription() const override;
+
+  // Populates the block_dim_limit by querying the device driver API. If an
+  // error occurs at any point while asking the driver for block dim limits, it
+  // will be only partially populated as a result, and an error will be logged.
+  bool FillBlockDimLimit(BlockDim* block_dim_limit) const;
+
+  bool SupportsBlas() const override;
+
+  blas::BlasSupport* CreateBlas() override;
+
+  bool SupportsFft() const override;
+
+  fft::FftSupport* CreateFft() override;
+
+  bool SupportsRng() const override;
+
+  rng::RngSupport* CreateRng() override;
+
+  bool SupportsDnn() const override;
+
+  dnn::DnnSupport* CreateDnn() override;
+
+  std::unique_ptr<internal::EventInterface> CreateEventImplementation()
+      override;
+
+  std::unique_ptr<internal::KernelInterface> CreateKernelImplementation()
+      override;
+
+  std::unique_ptr<internal::StreamInterface> GetStreamImplementation() override;
+
+  std::unique_ptr<internal::TimerInterface> GetTimerImplementation() override;
+
+  void* GpuContextHack() override;
+
+  GpuContext* gpu_context();
+
+ private:
+  // Attempts to find a more specific version of the file indicated by
+  // filename by looking for compute-capability-specific suffixed versions; i.e.
+  // looking for "foo.ptx" will check to see if "foo.ptx.cc30.ptx" is present if
+  // we're on a compute capability 3.0 machine.
+  // (supported on CUDA only)
+  bool FindOnDiskForComputeCapability(absl::string_view filename,
+                                      absl::string_view canonical_suffix,
+                                      string* found_filename) const;
+
+  // Attempts to find a more specific version of the file indicated by
+  // filename by looking for AMDGPU ISA-specific suffixed versions.
+  // (supported on ROCm only)
+
+  bool FindOnDiskForISAVersion(absl::string_view filename,
+                               absl::string_view canonical_suffix,
+                               string* found_filename) const;
+
+  // Host callback landing routine invoked by CUDA.
+  // data: User-provided callback provided to HostCallback() above, captured
+  //       as a std::function<void()>. Allocated/initialized inside
+  //       HostCallback() and owned and deleted by this call.
+  static void InternalHostCallback(GpuStreamHandle stream, GpuStatus status,
+                                   void* data);
+
+  // Collects metadata for the specified kernel.
+  bool GetKernelMetadata(GpuKernel* cuda_kernel,
+                         KernelMetadata* kernel_metadata);
+
+  // Prints to VLOG(2) information about the kernel's occupancy and how it might
+  // be improved.
+  void VlogOccupancyInfo(const KernelBase& kernel, const ThreadDim& thread_dims,
+                         const BlockDim& block_dims);
+
+  // (supported on CUDA only)
+  bool LoadModuleFromCuBin(const char* cubin, GpuModuleHandle* module)
+      EXCLUSIVE_LOCKS_REQUIRED(in_memory_modules_mu_);
+
+  // Loads the PTX text `ptx` as a CUDA module.  `ptx` must be null terminated.
+  // (supported on CUDA only)
+  bool LoadModuleFromPtx(const char* ptx, GpuModuleHandle* module)
+      EXCLUSIVE_LOCKS_REQUIRED(in_memory_modules_mu_);
+
+  // (supported on ROCm only)
+  bool LoadModuleFromHsaco(const char* hsaco, GpuModuleHandle* module)
+      EXCLUSIVE_LOCKS_REQUIRED(in_memory_modules_mu_);
+
+  bool UnloadGpuBinary(const void* gpu_binary)
+      EXCLUSIVE_LOCKS_REQUIRED(in_memory_modules_mu_);
+
+  // Guards the on-disk-module mapping.
+  mutex disk_modules_mu_;
+
+  // Mapping from filename to GPUModuleHandle, if it was already retrieved.
+  // Multiple GPUFunctionHandle are usually obtained from a single
+  // GPUModuleHandle so we attempt to hit in this mapping first, before
+  // retrieving it.
+  std::map<string, GpuModuleHandle> disk_modules_ GUARDED_BY(disk_modules_mu_);
+
+  // Guards the in-memory-module mapping.
+  mutex in_memory_modules_mu_;
+
+  std::map<const char*, GpuModuleHandle> in_memory_modules_
+      GUARDED_BY(in_memory_modules_mu_);
+
+  // Kernel -> loaded GPU binary. Many kernels may load the same binary.
+  std::unordered_map<const KernelBase*, const void*> kernel_to_gpu_binary_
+      GUARDED_BY(in_memory_modules_mu_);
+  // GPU binary (PTX or CUBIN or HSACO) -> {CUDA module, reference count}.
+  std::unordered_map<const void*, std::pair<GpuModuleHandle, uint64>>
+      gpu_binary_to_module_ GUARDED_BY(in_memory_modules_mu_);
+
+  // Guards the launched kernel set.
+  mutex launched_kernels_mu_;
+
+  // Keeps track of the set of launched kernels. Currently used to suppress the
+  // occupancy check on subsequent launches.
+  std::set<GpuFunctionHandle> launched_kernels_
+      GUARDED_BY(launched_kernels_mu_);
+
+  // Handle for the CUDA device being operated on. Immutable
+  // post-initialization.
+  GpuDeviceHandle device_;
+
+  // Handle for session with the library/driver. Immutable post-initialization.
+  GpuContext* context_;
+
+  // The device ordinal value that this executor was initialized with; recorded
+  // for use in getting device metadata. Immutable post-initialization.
+  int device_ordinal_;
+
+  // The major verion of the compute capability for device_.
+  int cc_major_;
+
+  // The minor verion of the compute capability for device_.
+  int cc_minor_;
+
+  // GPU ISA version for device_.
+  int version_;
+
+  // The plugin configuration associated with this instance.
+  PluginConfig plugin_config_;
+
+  SE_DISALLOW_COPY_AND_ASSIGN(GpuExecutor);
+};
+
+}  // namespace gpu
+}  // namespace stream_executor
+
+#endif  // TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_EXECUTOR_H_
diff --git a/tensorflow/stream_executor/gpu/gpu_helpers.h b/tensorflow/stream_executor/gpu/gpu_helpers.h
new file mode 100644
index 0000000000..117a71718f
--- /dev/null
+++ b/tensorflow/stream_executor/gpu/gpu_helpers.h
@@ -0,0 +1,107 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Common helper functions used for dealing with CUDA API datatypes.
+//
+// These are typically placed here for use by multiple source components (for
+// example, BLAS and executor components).
+
+#ifndef TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_HELPERS_H_
+#define TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_HELPERS_H_
+
+#include <stddef.h>
+#include <complex>
+
+#include "tensorflow/stream_executor/gpu/gpu_types.h"
+
+namespace stream_executor {
+
+template <typename ElemT>
+class DeviceMemory;
+
+namespace gpu {
+
+// Converts a const DeviceMemory reference to its underlying typed pointer in
+// CUDA
+// device memory.
+template <typename T>
+const T* GpuMemory(const DeviceMemory<T>& mem) {
+  return static_cast<const T*>(mem.opaque());
+}
+
+// Converts a (non-const) DeviceMemory pointer reference to its underlying typed
+// pointer in CUDA device memory.
+template <typename T>
+T* GpuMemoryMutable(DeviceMemory<T>* mem) {
+  return static_cast<T*>(mem->opaque());
+}
+
+static_assert(
+    sizeof(std::complex<float>) == sizeof(GpuComplexType),
+    "std::complex<float> and GpuComplexType should have the same size");
+static_assert(offsetof(GpuComplexType, x) == 0,
+              "The real part of GpuComplexType should appear first.");
+static_assert(
+    sizeof(std::complex<double>) == sizeof(GpuDoubleComplexType),
+    "std::complex<double> and GpuDoubleComplexType should have the same "
+    "size");
+static_assert(offsetof(GpuDoubleComplexType, x) == 0,
+              "The real part of GpuDoubleComplexType should appear first.");
+
+// Type traits to get CUDA complex types from std::complex<>.
+
+template <typename T>
+struct GpuComplexT {
+  typedef T type;
+};
+
+template <>
+struct GpuComplexT<std::complex<float>> {
+  typedef GpuComplexType type;
+};
+
+template <>
+struct GpuComplexT<std::complex<double>> {
+  typedef GpuDoubleComplexType type;
+};
+
+// Converts pointers of std::complex<> to pointers of
+// GpuComplexType/GpuDoubleComplexType. No type conversion for non-complex
+// types.
+
+template <typename T>
+inline const typename GpuComplexT<T>::type* GpuComplex(const T* p) {
+  return reinterpret_cast<const typename GpuComplexT<T>::type*>(p);
+}
+
+template <typename T>
+inline typename GpuComplexT<T>::type* GpuComplex(T* p) {
+  return reinterpret_cast<typename GpuComplexT<T>::type*>(p);
+}
+
+// Converts values of std::complex<float/double> to values of
+// GpuComplexType/GpuDoubleComplexType.
+inline GpuComplexType GpuComplexValue(std::complex<float> val) {
+  return {val.real(), val.imag()};
+}
+
+inline GpuDoubleComplexType GpuComplexValue(std::complex<double> val) {
+  return {val.real(), val.imag()};
+}
+
+}  // namespace gpu
+}  // namespace stream_executor
+
+#endif  // TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_HELPERS_H_
diff --git a/tensorflow/stream_executor/gpu/gpu_kernel.h b/tensorflow/stream_executor/gpu/gpu_kernel.h
new file mode 100644
index 0000000000..5b89b20097
--- /dev/null
+++ b/tensorflow/stream_executor/gpu/gpu_kernel.h
@@ -0,0 +1,105 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// The CUDA implementation of the StreamExecutorInterface functionality.
+// CUDA inclusions are ideally confined to this implementation file.
+//
+// The notions from the StreamExecutor basically correspond to the CUDA streams
+// programming model provided by the libcuda.so driver APIs, so we don't have
+// to do much more than wrap the calls to the libraries appropriately.
+#ifndef TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_KERNEL_H_
+#define TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_KERNEL_H_
+
+#include "tensorflow/stream_executor/gpu/gpu_driver.h"
+#include "tensorflow/stream_executor/kernel_cache_config.h"
+#include "tensorflow/stream_executor/platform/logging.h"
+#include "tensorflow/stream_executor/platform/port.h"
+#include "tensorflow/stream_executor/stream_executor_internal.h"
+
+namespace stream_executor {
+namespace gpu {
+
+// Wraps a GpuFunctionHandle to implement the platform-independent
+// KernelInterface.
+class GpuKernel : public internal::KernelInterface {
+ public:
+  GpuKernel()
+      : gpu_function_(nullptr),
+        arity_(0),
+        preferred_cache_config_(KernelCacheConfig::kNoPreference) {}
+
+  // Note that the function is unloaded when the module is unloaded, and the
+  // module that the function is contained in is owned by the GpuExecutor.
+  ~GpuKernel() override {}
+
+  // As arity cannot be reflected upon using the CUDA API, the arity is
+  // explicitly set during the GpuExecutor::GetKernel initialization process.
+  void set_arity(unsigned arity) { arity_ = arity; }
+  unsigned Arity() const override { return arity_; }
+
+  // Returns the GpuFunctionHandle value for passing to the CUDA API.
+  GpuFunctionHandle AsGpuFunctionHandle() const {
+    DCHECK(gpu_function_ != nullptr);
+    return const_cast<GpuFunctionHandle>(gpu_function_);
+  }
+
+  // Returns the slot that the GpuFunctionHandle is stored within for this
+  // object, for the CUDA API which wants to load into a GpuFunctionHandle*.
+  GpuFunctionHandle* gpu_function_ptr() { return &gpu_function_; }
+
+  // CUDA supports setting the preferred cache configuration of a
+  // GpuFunctionHandle (more-or-less equivalent to a GpuKernel). We support this
+  // via the below functions; users can set a preference, and that is applied
+  // when the kernel is [lazy-]loaded (in GpuExecutor::Launch). The alternative
+  // would be to load the kernel & set the preference when the user calls the
+  // setter below; either approach is valid. Sets the current kernel cache
+  // configuration preference.
+  void SetPreferredCacheConfig(KernelCacheConfig config) override {
+    preferred_cache_config_ = config;
+  }
+
+  // Returns the current kernel cache configuration preference.
+  KernelCacheConfig GetPreferredCacheConfig() const override {
+    return preferred_cache_config_;
+  }
+
+  // Returns the current kernel cache configuration preference as a
+  // CUfunc_cache.
+  GpuFuncCachePreference GetGpuCacheConfig() const;
+
+ private:
+  GpuFunctionHandle gpu_function_;  // Wrapped CUDA kernel handle.
+  unsigned arity_;  // Number of formal parameters the kernel takes.
+
+  // Preferred (but not required) cache configuration for this kernel.
+  KernelCacheConfig preferred_cache_config_;
+};
+
+// Given a platform-independent kernel datatype, returns the (const) internal
+// CUDA platform implementation pointer.
+inline const GpuKernel* AsGpuKernel(const KernelBase* kernel) {
+  return static_cast<const GpuKernel*>(kernel->implementation());
+}
+
+// Given a platform-independent kernel datatype, returns the (non-const)
+// internal CUDA platform implementation pointer.
+inline GpuKernel* AsGpuKernel(KernelBase* kernel) {
+  return static_cast<GpuKernel*>(kernel->implementation());
+}
+
+}  // namespace gpu
+}  // namespace stream_executor
+
+#endif  // TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_KERNEL_H_
diff --git a/tensorflow/stream_executor/gpu/gpu_rng.h b/tensorflow/stream_executor/gpu/gpu_rng.h
new file mode 100644
index 0000000000..d4bf1e1963
--- /dev/null
+++ b/tensorflow/stream_executor/gpu/gpu_rng.h
@@ -0,0 +1,125 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_RNG_H_
+#define TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_RNG_H_
+
+#include "tensorflow/stream_executor/platform/mutex.h"
+#include "tensorflow/stream_executor/platform/port.h"
+#include "tensorflow/stream_executor/platform/thread_annotations.h"
+#include "tensorflow/stream_executor/plugin_registry.h"
+#include "tensorflow/stream_executor/rng.h"
+
+#include "tensorflow/stream_executor/gpu/gpu_types.h"
+
+namespace stream_executor {
+
+class Stream;
+template <typename ElemT>
+class DeviceMemory;
+
+namespace gpu {
+
+// Opaque and unique identifier for the GPU RNG plugin.
+extern const PluginId kGpuRandPlugin;
+
+class GpuExecutor;
+
+// GPU-platform implementation of the random number generation support
+// interface.
+//
+// Thread-safe post-initialization.
+class GpuRng : public rng::RngSupport {
+ public:
+  explicit GpuRng(GpuExecutor* parent);
+
+  // Retrieves a gpu rng library generator handle. This is necessary for
+  // enqueuing random number generation work onto the device.
+  // TODO(leary) provide a way for users to select the RNG algorithm.
+  bool Init();
+
+  // Releases a gpu rng library generator handle, if one was acquired.
+  ~GpuRng() override;
+
+  // See rng::RngSupport for details on the following overrides.
+  bool DoPopulateRandUniform(Stream* stream, DeviceMemory<float>* v) override;
+  bool DoPopulateRandUniform(Stream* stream, DeviceMemory<double>* v) override;
+  bool DoPopulateRandUniform(Stream* stream,
+                             DeviceMemory<std::complex<float>>* v) override;
+  bool DoPopulateRandUniform(Stream* stream,
+                             DeviceMemory<std::complex<double>>* v) override;
+  bool DoPopulateRandGaussian(Stream* stream, float mean, float stddev,
+                              DeviceMemory<float>* v) override;
+  bool DoPopulateRandGaussian(Stream* stream, double mean, double stddev,
+                              DeviceMemory<double>* v) override;
+
+  bool SetSeed(Stream* stream, const uint8* seed, uint64 seed_bytes) override;
+
+ private:
+  // Actually performs the work of generating random numbers - the public
+  // methods are thin wrappers to this interface.
+  template <typename T>
+  bool DoPopulateRandUniformInternal(Stream* stream, DeviceMemory<T>* v);
+  template <typename ElemT, typename FuncT>
+  bool DoPopulateRandGaussianInternal(Stream* stream, ElemT mean, ElemT stddev,
+                                      DeviceMemory<ElemT>* v, FuncT func);
+
+  // Sets the stream for the internal gpu rng generator.
+  //
+  // This is a stateful operation, as the handle can only have one stream set at
+  // a given time, so it is usually performed right before enqueuing work to do
+  // with random number generation.
+  bool SetStream(Stream* stream) EXCLUSIVE_LOCKS_REQUIRED(mu_);
+
+  // mutex that guards the gpu rng library handle for this device.
+  mutex mu_;
+
+  // GpuExecutor which instantiated this GpuRng.
+  // Immutable post-initialization.
+  GpuExecutor* parent_;
+
+  // gpu rng library handle on the device.
+  GpuRngHandle rng_ GUARDED_BY(mu_);
+
+  SE_DISALLOW_COPY_AND_ASSIGN(GpuRng);
+};
+
+template <typename T>
+string TypeString();
+
+template <>
+string TypeString<float>() {
+  return "float";
+}
+
+template <>
+string TypeString<double>() {
+  return "double";
+}
+
+template <>
+string TypeString<std::complex<float>>() {
+  return "std::complex<float>";
+}
+
+template <>
+string TypeString<std::complex<double>>() {
+  return "std::complex<double>";
+}
+
+}  // namespace gpu
+}  // namespace stream_executor
+
+#endif  // TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_RNG_H_
diff --git a/tensorflow/stream_executor/cuda/cuda_stream.cc b/tensorflow/stream_executor/gpu/gpu_stream.cc
similarity index 51%
rename from tensorflow/stream_executor/cuda/cuda_stream.cc
rename to tensorflow/stream_executor/gpu/gpu_stream.cc
index b5aa7694f7..f43500370f 100644
--- a/tensorflow/stream_executor/cuda/cuda_stream.cc
+++ b/tensorflow/stream_executor/gpu/gpu_stream.cc
@@ -1,4 +1,4 @@
-/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -13,49 +13,49 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/stream_executor/cuda/cuda_stream.h"
+#include "tensorflow/stream_executor/gpu/gpu_stream.h"
 
-#include "tensorflow/stream_executor/cuda/cuda_gpu_executor.h"
+#include "tensorflow/stream_executor/gpu/gpu_executor.h"
 #include "tensorflow/stream_executor/lib/status.h"
 #include "tensorflow/stream_executor/stream.h"
 
 namespace stream_executor {
-namespace cuda {
+namespace gpu {
 
-bool CUDAStream::Init() {
-  if (!CUDADriver::CreateStream(parent_->cuda_context(), &cuda_stream_)) {
+bool GpuStream::Init() {
+  if (!GpuDriver::CreateStream(parent_->gpu_context(), &gpu_stream_)) {
     return false;
   }
-  return CUDADriver::CreateEvent(parent_->cuda_context(), &completed_event_,
-                                 CUDADriver::EventFlags::kDisableTiming)
+  return GpuDriver::CreateEvent(parent_->gpu_context(), &completed_event_,
+                                GpuDriver::EventFlags::kDisableTiming)
       .ok();
 }
 
-void CUDAStream::Destroy() {
+void GpuStream::Destroy() {
   if (completed_event_ != nullptr) {
     port::Status status =
-        CUDADriver::DestroyEvent(parent_->cuda_context(), &completed_event_);
+        GpuDriver::DestroyEvent(parent_->gpu_context(), &completed_event_);
     if (!status.ok()) {
       LOG(ERROR) << status.error_message();
     }
   }
 
-  CUDADriver::DestroyStream(parent_->cuda_context(), &cuda_stream_);
+  GpuDriver::DestroyStream(parent_->gpu_context(), &gpu_stream_);
 }
 
-bool CUDAStream::IsIdle() const {
-  return CUDADriver::IsStreamIdle(parent_->cuda_context(), cuda_stream_);
+bool GpuStream::IsIdle() const {
+  return GpuDriver::IsStreamIdle(parent_->gpu_context(), gpu_stream_);
 }
 
-CUDAStream *AsCUDAStream(Stream *stream) {
+GpuStream* AsGpuStream(Stream* stream) {
   DCHECK(stream != nullptr);
-  return static_cast<CUDAStream *>(stream->implementation());
+  return static_cast<GpuStream*>(stream->implementation());
 }
 
-CUstream AsCUDAStreamValue(Stream *stream) {
+GpuStreamHandle AsGpuStreamValue(Stream* stream) {
   DCHECK(stream != nullptr);
-  return AsCUDAStream(stream)->cuda_stream();
+  return AsGpuStream(stream)->gpu_stream();
 }
 
-}  // namespace cuda
+}  // namespace gpu
 }  // namespace stream_executor
diff --git a/tensorflow/stream_executor/gpu/gpu_stream.h b/tensorflow/stream_executor/gpu/gpu_stream.h
new file mode 100644
index 0000000000..c38f6c132a
--- /dev/null
+++ b/tensorflow/stream_executor/gpu/gpu_stream.h
@@ -0,0 +1,96 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Defines the GpuStream type - the CUDA-specific implementation of the generic
+// StreamExecutor Stream interface.
+
+#ifndef TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_STREAM_H_
+#define TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_STREAM_H_
+
+#include "tensorflow/stream_executor/gpu/gpu_driver.h"
+#include "tensorflow/stream_executor/platform/thread_annotations.h"
+#include "tensorflow/stream_executor/stream_executor_internal.h"
+
+namespace stream_executor {
+namespace gpu {
+
+class GpuExecutor;
+
+// Wraps a GpuStreamHandle in order to satisfy the platform-independent
+// StreamInterface.
+//
+// Thread-safe post-initialization.
+class GpuStream : public internal::StreamInterface {
+ public:
+  explicit GpuStream(GpuExecutor* parent)
+      : parent_(parent), gpu_stream_(nullptr), completed_event_(nullptr) {}
+
+  // Note: teardown is handled by a parent's call to DeallocateStream.
+  ~GpuStream() override {}
+
+  void* GpuStreamHack() override { return gpu_stream_; }
+  void** GpuStreamMemberHack() override {
+    return reinterpret_cast<void**>(&gpu_stream_);
+  }
+
+  // Explicitly initialize the CUDA resources associated with this stream, used
+  // by StreamExecutor::AllocateStream().
+  bool Init();
+
+  // Explicitly destroy the CUDA resources associated with this stream, used by
+  // StreamExecutor::DeallocateStream().
+  void Destroy();
+
+  // Returns true if no work is pending or executing on the stream.
+  bool IsIdle() const;
+
+  // Retrieves an event which indicates that all work enqueued into the stream
+  // has completed. Ownership of the event is not transferred to the caller, the
+  // event is owned by this stream.
+  GpuEventHandle* completed_event() { return &completed_event_; }
+
+  // Returns the GpuStreamHandle value for passing to the CUDA API.
+  //
+  // Precond: this GpuStream has been allocated (otherwise passing a nullptr
+  // into the NVIDIA library causes difficult-to-understand faults).
+  GpuStreamHandle gpu_stream() const {
+    DCHECK(gpu_stream_ != nullptr);
+    return const_cast<GpuStreamHandle>(gpu_stream_);
+  }
+
+  // TODO(timshen): Migrate away and remove this function.
+  GpuStreamHandle cuda_stream() const { return gpu_stream(); }
+
+  GpuExecutor* parent() const { return parent_; }
+
+ private:
+  GpuExecutor* parent_;         // Executor that spawned this stream.
+  GpuStreamHandle gpu_stream_;  // Wrapped CUDA stream handle.
+
+  // Event that indicates this stream has completed.
+  GpuEventHandle completed_event_ = nullptr;
+};
+
+// Helper functions to simplify extremely common flows.
+// Converts a Stream to the underlying GpuStream implementation.
+GpuStream* AsGpuStream(Stream* stream);
+
+// Extracts a GpuStreamHandle from a GpuStream-backed Stream object.
+GpuStreamHandle AsGpuStreamValue(Stream* stream);
+
+}  // namespace gpu
+}  // namespace stream_executor
+
+#endif  // TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_STREAM_H_
diff --git a/tensorflow/stream_executor/cuda/cuda_timer.cc b/tensorflow/stream_executor/gpu/gpu_timer.cc
similarity index 51%
rename from tensorflow/stream_executor/cuda/cuda_timer.cc
rename to tensorflow/stream_executor/gpu/gpu_timer.cc
index 991a12a23d..cc4b50d9ba 100644
--- a/tensorflow/stream_executor/cuda/cuda_timer.cc
+++ b/tensorflow/stream_executor/gpu/gpu_timer.cc
@@ -1,4 +1,4 @@
-/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -13,31 +13,31 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/stream_executor/cuda/cuda_timer.h"
+#include "tensorflow/stream_executor/gpu/gpu_timer.h"
 
-#include "tensorflow/stream_executor/cuda/cuda_driver.h"
-#include "tensorflow/stream_executor/cuda/cuda_gpu_executor.h"
-#include "tensorflow/stream_executor/cuda/cuda_stream.h"
+#include "tensorflow/stream_executor/gpu/gpu_driver.h"
+#include "tensorflow/stream_executor/gpu/gpu_executor.h"
+#include "tensorflow/stream_executor/gpu/gpu_stream.h"
 #include "tensorflow/stream_executor/lib/status.h"
 
 namespace stream_executor {
-namespace cuda {
+namespace gpu {
 
-bool CUDATimer::Init() {
+bool GpuTimer::Init() {
   CHECK(start_event_ == nullptr && stop_event_ == nullptr);
-  CudaContext* context = parent_->cuda_context();
-  port::Status status = CUDADriver::CreateEvent(
-      context, &start_event_, CUDADriver::EventFlags::kDefault);
+  GpuContext* context = parent_->gpu_context();
+  port::Status status = GpuDriver::CreateEvent(context, &start_event_,
+                                               GpuDriver::EventFlags::kDefault);
   if (!status.ok()) {
     LOG(ERROR) << status;
     return false;
   }
 
-  status = CUDADriver::CreateEvent(context, &stop_event_,
-                                   CUDADriver::EventFlags::kDefault);
+  status = GpuDriver::CreateEvent(context, &stop_event_,
+                                  GpuDriver::EventFlags::kDefault);
   if (!status.ok()) {
     LOG(ERROR) << status;
-    status = CUDADriver::DestroyEvent(context, &start_event_);
+    status = GpuDriver::DestroyEvent(context, &start_event_);
     if (!status.ok()) {
       LOG(ERROR) << status;
     }
@@ -48,47 +48,46 @@ bool CUDATimer::Init() {
   return true;
 }
 
-void CUDATimer::Destroy() {
-  CudaContext* context = parent_->cuda_context();
-  port::Status status = CUDADriver::DestroyEvent(context, &start_event_);
+void GpuTimer::Destroy() {
+  GpuContext* context = parent_->gpu_context();
+  port::Status status = GpuDriver::DestroyEvent(context, &start_event_);
   if (!status.ok()) {
     LOG(ERROR) << status;
   }
 
-  status = CUDADriver::DestroyEvent(context, &stop_event_);
+  status = GpuDriver::DestroyEvent(context, &stop_event_);
   if (!status.ok()) {
     LOG(ERROR) << status;
   }
 }
 
-float CUDATimer::GetElapsedMilliseconds() const {
+float GpuTimer::GetElapsedMilliseconds() const {
   CHECK(start_event_ != nullptr && stop_event_ != nullptr);
   // TODO(leary) provide a way to query timer resolution?
   // CUDA docs say a resolution of about 0.5us
   float elapsed_milliseconds = NAN;
-  (void)CUDADriver::GetEventElapsedTime(parent_->cuda_context(),
-                                        &elapsed_milliseconds, start_event_,
-                                        stop_event_);
+  (void)GpuDriver::GetEventElapsedTime(
+      parent_->gpu_context(), &elapsed_milliseconds, start_event_, stop_event_);
   return elapsed_milliseconds;
 }
 
-bool CUDATimer::Start(CUDAStream* stream) {
-  port::Status status = CUDADriver::RecordEvent(
-      parent_->cuda_context(), start_event_, stream->cuda_stream());
+bool GpuTimer::Start(GpuStream* stream) {
+  port::Status status = GpuDriver::RecordEvent(
+      parent_->gpu_context(), start_event_, stream->gpu_stream());
   if (!status.ok()) {
     LOG(ERROR) << status;
   }
   return status.ok();
 }
 
-bool CUDATimer::Stop(CUDAStream* stream) {
-  port::Status status = CUDADriver::RecordEvent(
-      parent_->cuda_context(), stop_event_, stream->cuda_stream());
+bool GpuTimer::Stop(GpuStream* stream) {
+  port::Status status = GpuDriver::RecordEvent(
+      parent_->gpu_context(), stop_event_, stream->gpu_stream());
   if (!status.ok()) {
     LOG(ERROR) << status;
   }
   return status.ok();
 }
 
-}  // namespace cuda
+}  // namespace gpu
 }  // namespace stream_executor
diff --git a/tensorflow/stream_executor/gpu/gpu_timer.h b/tensorflow/stream_executor/gpu/gpu_timer.h
new file mode 100644
index 0000000000..886f0c2d57
--- /dev/null
+++ b/tensorflow/stream_executor/gpu/gpu_timer.h
@@ -0,0 +1,90 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Defines the GpuTimer type - the CUDA-specific implementation of the generic
+// StreamExecutor Timer interface.
+
+#ifndef TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_TIMER_H_
+#define TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_TIMER_H_
+
+#include "tensorflow/stream_executor/gpu/gpu_driver.h"
+#include "tensorflow/stream_executor/gpu/gpu_executor.h"
+#include "tensorflow/stream_executor/stream_executor_internal.h"
+
+namespace stream_executor {
+namespace gpu {
+
+class GpuExecutor;
+class GpuStream;
+
+// Wraps a pair of GpuEventHandles in order to satisfy the platform-independent
+// TimerInferface -- both a start and a stop event are present which may be
+// recorded in a stream.
+class GpuTimer : public internal::TimerInterface {
+ public:
+  explicit GpuTimer(GpuExecutor* parent)
+      : parent_(parent), start_event_(nullptr), stop_event_(nullptr) {}
+
+  // Note: teardown needs to be explicitly handled in this API by a call to
+  // StreamExecutor::DeallocateTimer(), which invokes Destroy().
+  // TODO(csigg): Change to RAII.
+  ~GpuTimer() override {}
+
+  // Allocates the platform-specific pieces of the timer, called as part of
+  // StreamExecutor::AllocateTimer().
+  bool Init();
+
+  // Deallocates the platform-specific pieces of the timer, called as part of
+  // StreamExecutor::DeallocateTimer().
+  void Destroy();
+
+  // Records the "timer start" event at the current point in the stream.
+  bool Start(GpuStream* stream);
+
+  // Records the "timer stop" event at the current point in the stream.
+  bool Stop(GpuStream* stream);
+
+  // Returns the elapsed time, in milliseconds, between the start and stop
+  // events.
+  float GetElapsedMilliseconds() const;
+
+  // See Timer::Microseconds().
+  // TODO(leary) make this into an error code interface...
+  uint64 Microseconds() const override {
+    return GetElapsedMilliseconds() * 1e3;
+  }
+
+  // See Timer::Nanoseconds().
+  uint64 Nanoseconds() const override { return GetElapsedMilliseconds() * 1e6; }
+
+ private:
+  GpuExecutor* parent_;
+  GpuEventHandle start_event_;  // Event recorded to indicate the "start"
+                                // timestamp executing in a stream.
+  GpuEventHandle stop_event_;   // Event recorded to indicate the "stop"
+                                // timestamp executing in a stream.
+};
+
+struct GpuTimerDeleter {
+  void operator()(GpuTimer* t) {
+    t->Destroy();
+    delete t;
+  }
+};
+
+}  // namespace gpu
+}  // namespace stream_executor
+
+#endif  // TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_TIMER_H_
diff --git a/tensorflow/stream_executor/gpu/gpu_types.h b/tensorflow/stream_executor/gpu/gpu_types.h
new file mode 100644
index 0000000000..c69177d076
--- /dev/null
+++ b/tensorflow/stream_executor/gpu/gpu_types.h
@@ -0,0 +1,84 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// GPU (ROCm / CUDA) specific type handle resolution
+
+#ifndef TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_TYPES_H_
+#define TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_TYPES_H_
+
+#if TENSORFLOW_USE_ROCM
+
+#include "rocm/include/hip/hip_complex.h"
+#include "rocm/include/hip/hip_runtime.h"
+#include "rocm/include/hiprand/hiprand.h"
+
+#else  // CUDA
+
+#include "cuda/include/cuComplex.h"
+#include "cuda/include/cuda.h"
+
+// cannot include curand.h here
+//   because it triggers the #error in cuda/cuda_gpu_executor.cc
+//     (because curand.h includes cuda_runtime.h)
+// so explicitly adding the lone typedef we need from that file
+typedef struct curandGenerator_st* curandGenerator_t;
+
+#endif
+
+namespace stream_executor {
+namespace gpu {
+
+#if TENSORFLOW_USE_ROCM
+
+using GpuStreamHandle = hipStream_t;
+using GpuEventHandle = hipEvent_t;
+using GpuFunctionHandle = hipFunction_t;
+using GpuFunctionAttribute = hipDeviceAttribute_t;  // not a typo!
+using GpuDeviceHandle = hipDevice_t;
+using GpuDevicePtr = hipDeviceptr_t;
+using GpuDeviceAttribute = hipDeviceAttribute_t;
+using GpuDeviceProperty = hipDeviceProp_t;
+using GpuModuleHandle = hipModule_t;
+using GpuStatus = hipError_t;
+using GpuFuncCachePreference = hipFuncCache_t;
+using GpuSharedMemConfig = hipSharedMemConfig;
+using GpuComplexType = hipComplex;
+using GpuDoubleComplexType = hipDoubleComplex;
+using GpuRngHandle = hiprandGenerator_t;
+
+#else  // CUDA
+
+using GpuStreamHandle = CUstream;
+using GpuEventHandle = CUevent;
+using GpuFunctionHandle = CUfunction;
+using GpuFunctionAttribute = CUfunction_attribute;
+using GpuDeviceHandle = CUdevice;
+using GpuDevicePtr = CUdeviceptr;
+using GpuDeviceAttribute = CUdevice_attribute;
+using GpuDeviceProperty = CUdevprop;
+using GpuModuleHandle = CUmodule;
+using GpuStatus = CUresult;
+using GpuFuncCachePreference = CUfunc_cache;
+using GpuSharedMemConfig = CUsharedconfig;
+using GpuComplexType = cuComplex;
+using GpuDoubleComplexType = cuDoubleComplex;
+using GpuRngHandle = curandGenerator_t;
+
+#endif
+
+}  // namespace gpu
+}  // namespace stream_executor
+
+#endif  // TENSORFLOW_STREAM_EXECUTOR_GPU_GPU_TYPES_H_
diff --git a/tensorflow/stream_executor/platform.cc b/tensorflow/stream_executor/platform.cc
index c0205abbee..9c99581438 100644
--- a/tensorflow/stream_executor/platform.cc
+++ b/tensorflow/stream_executor/platform.cc
@@ -28,6 +28,8 @@ string PlatformKindString(PlatformKind kind) {
   switch (kind) {
     case PlatformKind::kCuda:
       return "CUDA";
+    case PlatformKind::kROCm:
+      return "ROCm";
     case PlatformKind::kOpenCL:
       return "OpenCL";
     case PlatformKind::kHost:
@@ -52,6 +54,7 @@ PlatformKind PlatformKindFromString(string kind) {
 bool PlatformIsRunnable(PlatformKind kind) {
   switch (kind) {
     case PlatformKind::kCuda:
+    case PlatformKind::kROCm:
     case PlatformKind::kOpenCL:
     case PlatformKind::kHost:
       return true;
@@ -63,6 +66,7 @@ bool PlatformIsRunnable(PlatformKind kind) {
 bool PlatformIsRunnableOnDevice(PlatformKind kind) {
   switch (kind) {
     case PlatformKind::kCuda:
+    case PlatformKind::kROCm:
     case PlatformKind::kOpenCL:
       return true;
     default:
diff --git a/tensorflow/stream_executor/platform.h b/tensorflow/stream_executor/platform.h
index 5cb7047b6f..2c2cd77ad2 100644
--- a/tensorflow/stream_executor/platform.h
+++ b/tensorflow/stream_executor/platform.h
@@ -40,6 +40,7 @@ class StreamExecutor;
 enum class PlatformKind {
   kInvalid,
   kCuda,
+  kROCm,
   kOpenCL,
   kHost,
   kMock,
diff --git a/tensorflow/stream_executor/rocm/BUILD b/tensorflow/stream_executor/rocm/BUILD
new file mode 100644
index 0000000000..dd08b36308
--- /dev/null
+++ b/tensorflow/stream_executor/rocm/BUILD
@@ -0,0 +1,267 @@
+# Description:
+#   ROCm-platform specific StreamExecutor support code.
+
+licenses(["notice"])  # Apache 2.0
+
+load("//tensorflow:tensorflow.bzl", "tf_cc_test")
+load(
+    "//tensorflow/stream_executor:build_defs.bzl",
+    "stream_executor_friends",
+)
+load("//tensorflow:tensorflow.bzl", "tf_copts")
+load("@local_config_rocm//rocm:build_defs.bzl", "if_rocm_is_configured")
+load("//tensorflow/core:platform/default/build_config_root.bzl", "if_static")
+
+package_group(
+    name = "friends",
+    packages = stream_executor_friends(),
+)
+
+package(
+    default_visibility = [":friends"],
+)
+
+# Filegroup used to collect source files for the dependency check.
+filegroup(
+    name = "c_srcs",
+    data = glob([
+        "**/*.cc",
+        "**/*.h",
+    ]),
+)
+
+cc_library(
+    name = "rocm_diagnostics",
+    srcs = if_rocm_is_configured(["rocm_diagnostics.cc"]),
+    hdrs = [],
+    deps = if_rocm_is_configured([
+        "@com_google_absl//absl/container:inlined_vector",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/strings:str_format",
+        "//tensorflow/stream_executor/gpu:gpu_diagnostics_header",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+    ]),
+)
+
+cc_library(
+    name = "rocm_driver",
+    srcs = if_rocm_is_configured(["rocm_driver.cc"]),
+    hdrs = [],
+    deps = if_rocm_is_configured([
+        ":rocm_diagnostics",
+        "@com_google_absl//absl/base",
+        "@com_google_absl//absl/container:inlined_vector",
+        "@com_google_absl//absl/strings",
+        "//tensorflow/stream_executor:device_options",
+        "//tensorflow/stream_executor/gpu:gpu_driver_header",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@local_config_rocm//rocm:rocm_headers",
+    ]),
+)
+
+cc_library(
+    name = "rocm_event",
+    srcs = if_rocm_is_configured(["rocm_event.cc"]),
+    hdrs = [],
+    deps = if_rocm_is_configured([
+        ":rocm_driver",
+        "//tensorflow/stream_executor:stream_executor_headers",
+        "//tensorflow/stream_executor/gpu:gpu_event_header",
+        "//tensorflow/stream_executor/gpu:gpu_executor_header",
+        "//tensorflow/stream_executor/gpu:gpu_stream_header",
+        "//tensorflow/stream_executor/lib",
+    ]),
+)
+
+cc_library(
+    name = "rocm_gpu_executor",
+    srcs = if_rocm_is_configured(["rocm_gpu_executor.cc"]),
+    hdrs = [],
+    deps = if_rocm_is_configured([
+        ":rocm_diagnostics",
+        ":rocm_driver",
+        ":rocm_event",
+        ":rocm_kernel",
+        ":rocm_platform_id",
+        "@com_google_absl//absl/strings",
+        "//tensorflow/stream_executor:event",
+        "//tensorflow/stream_executor:plugin_registry",
+        "//tensorflow/stream_executor:stream_executor_internal",
+        "//tensorflow/stream_executor:stream_executor_pimpl_header",
+        "//tensorflow/stream_executor:timer",
+        "//tensorflow/stream_executor/gpu:gpu_activation_header",
+        "//tensorflow/stream_executor/gpu:gpu_event",
+        "//tensorflow/stream_executor/gpu:gpu_kernel_header",
+        "//tensorflow/stream_executor/gpu:gpu_stream",
+        "//tensorflow/stream_executor/gpu:gpu_timer",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "//tensorflow/stream_executor/platform:dso_loader",
+    ]),
+    alwayslink = True,
+)
+
+cc_library(
+    name = "rocm_kernel",
+    srcs = if_rocm_is_configured(["rocm_kernel.cc"]),
+    hdrs = [],
+    visibility = ["//visibility:public"],
+    deps = if_rocm_is_configured([
+        "//tensorflow/stream_executor/gpu:gpu_kernel_header",
+    ]),
+    alwayslink = True,
+)
+
+cc_library(
+    name = "rocm_platform",
+    srcs = if_rocm_is_configured(["rocm_platform.cc"]),
+    hdrs = if_rocm_is_configured(["rocm_platform.h"]),
+    visibility = ["//visibility:public"],
+    deps = if_rocm_is_configured([
+        ":rocm_driver",
+        ":rocm_gpu_executor",
+        ":rocm_platform_id",
+        "//tensorflow/stream_executor",  # buildcleaner: keep
+        "//tensorflow/stream_executor:executor_cache",
+        "//tensorflow/stream_executor:multi_platform_manager",
+        "//tensorflow/stream_executor:stream_executor_pimpl_header",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+    ]),
+    alwayslink = True,  # Registers itself with the MultiPlatformManager.
+)
+
+cc_library(
+    name = "rocm_platform_id",
+    srcs = ["rocm_platform_id.cc"],
+    hdrs = ["rocm_platform_id.h"],
+    deps = ["//tensorflow/stream_executor:platform"],
+)
+
+# FIXME: enable in future PRs
+#cc_library(
+#    name = "rocblas_plugin",
+#    srcs = ["rocm_blas.cc"],
+#    hdrs = ["rocm_blas.h"],
+#    visibility = ["//visibility:public"],
+#    deps = [
+#        ":rocm_gpu_executor",
+#        ":rocm_platform_id",
+#        "//third_party/eigen3",
+#        "//tensorflow/core:lib_internal",
+#        "//tensorflow/stream_executor",
+#        "//tensorflow/stream_executor:event",
+#        "//tensorflow/stream_executor:host_or_device_scalar",
+#        "//tensorflow/stream_executor:plugin_registry",
+#        "//tensorflow/stream_executor:scratch_allocator",
+#        "//tensorflow/stream_executor:timer",
+#        "//tenosrflow/stream_executor/gpu:gpu_activation_header",
+#        "//tenosrflow/stream_executor/gpu:gpu_stream_header",
+#        "//tenosrflow/stream_executor/gpu:gpu_timer_header",
+#        "//tensorflow/stream_executor/lib",
+#        "//tensorflow/stream_executor/platform",
+#        "//tensorflow/stream_executor/platform:dso_loader",
+#        "@com_google_absl//absl/strings",
+#        "@local_config_rocm//rocm:rocm_headers",
+#    ] + if_static(["@local_config_rocm//rocm:rocblas"]),
+#    alwayslink = True,
+#)
+
+# FIXME: enable in future PRs
+#cc_library(
+#    name = "rocfft_plugin",
+#    srcs = ["rocm_fft.cc"],
+#    hdrs = [],
+#    visibility = ["//visibility:public"],
+#    deps = [
+#        ":rocm_platform_id",
+#        "//tensorflow/stream_executor:event",
+#        "//tensorflow/stream_executor:fft",
+#        "//tensorflow/stream_executor:plugin_registry",
+#        "//tensorflow/stream_executor:scratch_allocator",
+#        "//tenosrflow/stream_executor/gpu:gpu_stream_header",
+#        "//tensorflow/stream_executor/lib",
+#        "//tensorflow/stream_executor/platform",
+#        "//tensorflow/stream_executor/platform:dso_loader",
+#        "@local_config_rocm//rocm:rocm_headers",
+#    ] + if_static(["@local_config_rocm//rocm:rocfft"]),
+#    alwayslink = True,
+#)
+
+# FIXME: enable in future PRs
+#cc_library(
+#    name = "miopen_plugin",
+#    srcs = ["rocm_dnn.cc"],
+#    hdrs = [],
+#    copts = [
+#        # STREAM_EXECUTOR_CUDNN_WRAP would fail on Clang with the default
+#        # setting of template depth 256
+#        "-ftemplate-depth-512",
+#    ],
+#    visibility = ["//visibility:public"],
+#    deps = [
+#        ":rocm_diagnostics",
+#        ":rocm_driver",
+#        ":rocm_gpu_executor",
+#        ":rocm_platform_id",
+#        "//third_party/eigen3",
+#        "//tensorflow/core:lib",
+#        "//tensorflow/core:lib_internal",
+#        "//tensorflow/core:logger",
+#        "//tensorflow/stream_executor:dnn",
+#        "//tensorflow/stream_executor:event",
+#        "//tensorflow/stream_executor:logging_proto_cc",
+#        "//tensorflow/stream_executor:plugin_registry",
+#        "//tensorflow/stream_executor:scratch_allocator",
+#        "//tensorflow/stream_executor:stream_executor_pimpl_header",
+#        "//tensorflow/stream_executor:temporary_device_memory",
+#        "//tenosrflow/stream_executor/gpu:gpu_activation_header",
+#        "//tenosrflow/stream_executor/gpu:gpu_stream_header",
+#        "//tenosrflow/stream_executor/gpu:gpu_timer_header",
+#        "//tensorflow/stream_executor/lib",
+#        "//tensorflow/stream_executor/platform",
+#        "//tensorflow/stream_executor/platform:dso_loader",
+#        "@com_google_absl//absl/strings",
+#        "@local_config_rocm//rocm:rocm_headers",
+#    ] + tf_additional_miopen_plugin_deps() + if_static(["@local_config_rocm//rocm:miopen"]),
+#    alwayslink = True,
+#)
+
+# FIXME: enable in future PRs
+#cc_library(
+#    name = "rocrand_plugin",
+#    srcs = ["rocm_rng.cc"],
+#    hdrs = [],
+#    deps = [
+#        ":rocm_gpu_executor",
+#        ":rocm_platform_id",
+#        "@local_config_rocm//rocm:rocm_headers",
+#        "//tensorflow/stream_executor:event",
+#        "//tensorflow/stream_executor:plugin_registry",
+#        "//tensorflow/stream_executor:rng",
+#        "//tenosrflow/stream_executor/gpu:gpu_activation_header",
+#        "//tenosrflow/stream_executor/gpu:gpu_stream_header",
+#        "//tensorflow/stream_executor/lib",
+#        "//tensorflow/stream_executor/platform",
+#        "//tensorflow/stream_executor/platform:dso_loader",
+#    ] + if_static(["@local_config_rocm//rocm:curand"]),
+#    alwayslink = True,
+#)
+
+cc_library(
+    name = "all_runtime",
+    copts = tf_copts(),
+    visibility = ["//visibility:public"],
+    deps = if_rocm_is_configured([
+        # FIXME: enable in future PRs
+        #":miopen_plugin",
+        #":rocfft_plugin",
+        #":rocblas_plugin",
+        #":rocrand_plugin",
+        ":rocm_driver",
+        ":rocm_platform",
+    ]),
+    alwayslink = 1,
+)
diff --git a/tensorflow/stream_executor/rocm/rocm_diagnostics.cc b/tensorflow/stream_executor/rocm/rocm_diagnostics.cc
new file mode 100644
index 0000000000..c6da7f9e3d
--- /dev/null
+++ b/tensorflow/stream_executor/rocm/rocm_diagnostics.cc
@@ -0,0 +1,234 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <dirent.h>
+
+#include <limits.h>
+#include <link.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/sysmacros.h>
+#include <unistd.h>
+#include <algorithm>
+#include <memory>
+#include <vector>
+
+#include "absl/container/inlined_vector.h"
+#include "absl/strings/str_cat.h"
+#include "absl/strings/str_format.h"
+#include "tensorflow/stream_executor/gpu/gpu_diagnostics.h"
+#include "tensorflow/stream_executor/lib/error.h"
+#include "tensorflow/stream_executor/lib/numbers.h"
+#include "tensorflow/stream_executor/lib/process_state.h"
+#include "tensorflow/stream_executor/lib/status.h"
+#include "tensorflow/stream_executor/lib/str_util.h"
+#include "tensorflow/stream_executor/lib/stringprintf.h"
+#include "tensorflow/stream_executor/platform/logging.h"
+
+namespace stream_executor {
+namespace gpu {
+
+string DriverVersionToString(DriverVersion version) {
+  return absl::StrFormat("%d.%d.%d", std::get<0>(version), std::get<1>(version),
+                         std::get<2>(version));
+}
+
+string DriverVersionStatusToString(port::StatusOr<DriverVersion> version) {
+  if (!version.ok()) {
+    return version.status().ToString();
+  }
+
+  return DriverVersionToString(version.ValueOrDie());
+}
+
+port::StatusOr<DriverVersion> StringToDriverVersion(const string& value) {
+  std::vector<string> pieces = port::Split(value, '.');
+  if (pieces.size() != 2 && pieces.size() != 3) {
+    return port::Status{port::error::INVALID_ARGUMENT,
+                        absl::StrFormat("expected %%d.%%d or %%d.%%d.%%d form "
+                                        "for driver version; got \"%s\"",
+                                        value.c_str())};
+  }
+
+  int major;
+  int minor;
+  int patch = 0;
+  if (!port::safe_strto32(pieces[0], &major)) {
+    return port::Status{
+        port::error::INVALID_ARGUMENT,
+        absl::StrFormat("could not parse major version number \"%s\" as an "
+                        "integer from string \"%s\"",
+                        pieces[0].c_str(), value.c_str())};
+  }
+  if (!port::safe_strto32(pieces[1], &minor)) {
+    return port::Status{
+        port::error::INVALID_ARGUMENT,
+        absl::StrFormat("could not parse minor version number \"%s\" as an "
+                        "integer from string \"%s\"",
+                        pieces[1].c_str(), value.c_str())};
+  }
+  if (pieces.size() == 3 && !port::safe_strto32(pieces[2], &patch)) {
+    return port::Status{
+        port::error::INVALID_ARGUMENT,
+        absl::StrFormat("could not parse patch version number \"%s\" as an "
+                        "integer from string \"%s\"",
+                        pieces[2].c_str(), value.c_str())};
+  }
+
+  DriverVersion result{major, minor, patch};
+  VLOG(2) << "version string \"" << value << "\" made value "
+          << DriverVersionToString(result);
+  return result;
+}
+
+// -- class Diagnostician
+
+string Diagnostician::GetDevNodePath(int dev_node_ordinal) {
+  return absl::StrCat("/dev/kfd", dev_node_ordinal);
+}
+
+void Diagnostician::LogDiagnosticInformation() {
+  LOG(INFO) << "retrieving ROCM diagnostic information for host: "
+            << port::Hostname();
+
+  LogDriverVersionInformation();
+}
+
+/* static */ void Diagnostician::LogDriverVersionInformation() {
+  LOG(INFO) << "hostname: " << port::Hostname();
+  if (VLOG_IS_ON(1)) {
+    const char* value = getenv("LD_LIBRARY_PATH");
+    string library_path = value == nullptr ? "" : value;
+    VLOG(1) << "LD_LIBRARY_PATH is: \"" << library_path << "\"";
+
+    std::vector<string> pieces = port::Split(library_path, ':');
+    for (const auto& piece : pieces) {
+      if (piece.empty()) {
+        continue;
+      }
+      DIR* dir = opendir(piece.c_str());
+      if (dir == nullptr) {
+        VLOG(1) << "could not open \"" << piece << "\"";
+        continue;
+      }
+      while (dirent* entity = readdir(dir)) {
+        VLOG(1) << piece << " :: " << entity->d_name;
+      }
+      closedir(dir);
+    }
+  }
+  port::StatusOr<DriverVersion> dso_version = FindDsoVersion();
+  LOG(INFO) << "librocm reported version is: "
+            << DriverVersionStatusToString(dso_version);
+
+  port::StatusOr<DriverVersion> kernel_version = FindKernelDriverVersion();
+  LOG(INFO) << "kernel reported version is: "
+            << DriverVersionStatusToString(kernel_version);
+
+  if (kernel_version.ok() && dso_version.ok()) {
+    WarnOnDsoKernelMismatch(dso_version, kernel_version);
+  }
+}
+
+// Iterates through loaded DSOs with DlIteratePhdrCallback to find the
+// driver-interfacing DSO version number. Returns it as a string.
+port::StatusOr<DriverVersion> Diagnostician::FindDsoVersion() {
+  port::StatusOr<DriverVersion> result{port::Status{
+      port::error::NOT_FOUND,
+      "was unable to find librocm.so DSO loaded into this program"}};
+
+  // Callback used when iterating through DSOs. Looks for the driver-interfacing
+  // DSO and yields its version number into the callback data, when found.
+  auto iterate_phdr = [](struct dl_phdr_info* info, size_t size,
+                         void* data) -> int {
+    if (strstr(info->dlpi_name, "librocm.so.1")) {
+      VLOG(1) << "found DLL info with name: " << info->dlpi_name;
+      char resolved_path[PATH_MAX] = {0};
+      if (realpath(info->dlpi_name, resolved_path) == nullptr) {
+        return 0;
+      }
+      VLOG(1) << "found DLL info with resolved path: " << resolved_path;
+      const char* slash = rindex(resolved_path, '/');
+      if (slash == nullptr) {
+        return 0;
+      }
+      const char* so_suffix = ".so.";
+      const char* dot = strstr(slash, so_suffix);
+      if (dot == nullptr) {
+        return 0;
+      }
+      string dso_version = dot + strlen(so_suffix);
+      // TODO(b/22689637): Eliminate the explicit namespace if possible.
+      auto stripped_dso_version = port::StripSuffixString(dso_version, ".ld64");
+      auto result = static_cast<port::StatusOr<DriverVersion>*>(data);
+      *result = StringToDriverVersion(stripped_dso_version);
+      return 1;
+    }
+    return 0;
+  };
+
+  dl_iterate_phdr(iterate_phdr, &result);
+
+  return result;
+}
+
+port::StatusOr<DriverVersion> Diagnostician::FindKernelModuleVersion(
+    const string& driver_version_file_contents) {
+  static const char* kDriverFilePrelude = "Kernel Module  ";
+  size_t offset = driver_version_file_contents.find(kDriverFilePrelude);
+  if (offset == string::npos) {
+    return port::Status{
+        port::error::NOT_FOUND,
+        absl::StrCat("could not find kernel module information in "
+                     "driver version file contents: \"",
+                     driver_version_file_contents, "\"")};
+  }
+
+  string version_and_rest = driver_version_file_contents.substr(
+      offset + strlen(kDriverFilePrelude), string::npos);
+  size_t space_index = version_and_rest.find(" ");
+  auto kernel_version = version_and_rest.substr(0, space_index);
+  // TODO(b/22689637): Eliminate the explicit namespace if possible.
+  auto stripped_kernel_version =
+      port::StripSuffixString(kernel_version, ".ld64");
+  return StringToDriverVersion(stripped_kernel_version);
+}
+
+void Diagnostician::WarnOnDsoKernelMismatch(
+    port::StatusOr<DriverVersion> dso_version,
+    port::StatusOr<DriverVersion> kernel_version) {
+  if (kernel_version.ok() && dso_version.ok() &&
+      dso_version.ValueOrDie() == kernel_version.ValueOrDie()) {
+    LOG(INFO) << "kernel version seems to match DSO: "
+              << DriverVersionToString(kernel_version.ValueOrDie());
+  } else {
+    LOG(ERROR) << "kernel version "
+               << DriverVersionStatusToString(kernel_version)
+               << " does not match DSO version "
+               << DriverVersionStatusToString(dso_version)
+               << " -- cannot find working devices in this configuration";
+  }
+}
+
+port::StatusOr<DriverVersion> Diagnostician::FindKernelDriverVersion() {
+  auto status = port::Status{port::error::UNIMPLEMENTED,
+                             "kernel reported driver version not implemented"};
+  return status;
+}
+
+}  // namespace gpu
+}  // namespace stream_executor
diff --git a/tensorflow/stream_executor/rocm/rocm_driver.cc b/tensorflow/stream_executor/rocm/rocm_driver.cc
new file mode 100644
index 0000000000..39d52d2830
--- /dev/null
+++ b/tensorflow/stream_executor/rocm/rocm_driver.cc
@@ -0,0 +1,1365 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <map>
+#include <set>
+#include <utility>
+
+#include "absl/base/casts.h"
+#include "absl/container/inlined_vector.h"
+#include "absl/strings/str_cat.h"
+#include "absl/strings/str_format.h"
+#include "tensorflow/stream_executor/gpu/gpu_diagnostics.h"
+#include "tensorflow/stream_executor/gpu/gpu_driver.h"
+#include "tensorflow/stream_executor/lib/env.h"
+#include "tensorflow/stream_executor/lib/error.h"
+#include "tensorflow/stream_executor/lib/human_readable.h"
+#include "tensorflow/stream_executor/lib/notification.h"
+#include "tensorflow/stream_executor/lib/stacktrace.h"
+#include "tensorflow/stream_executor/lib/static_threadlocal.h"
+#include "tensorflow/stream_executor/lib/stringprintf.h"
+#include "tensorflow/stream_executor/lib/threadpool.h"
+#include "tensorflow/stream_executor/platform/logging.h"
+#include "tensorflow/stream_executor/platform/mutex.h"
+#include "tensorflow/stream_executor/platform/port.h"
+
+bool FLAGS_gpuexec_rocm_driver_inject_init_error = false;
+bool FLAGS_gpuexec_rocm_sync_around_driver_calls = false;
+bool FLAGS_gpuexec_rocm_device_0_only = false;
+
+// Debugging: on each push and pop of a rocm context, verify the current device
+// matches the expected one.
+constexpr bool kVerifyGpuContext = false;
+
+namespace stream_executor {
+namespace gpu {
+
+// GpuContext wraps the device_ordinal.
+// Only reason we need this wrapper class is to make the GpuDriver* API
+class GpuContext {
+ public:
+  GpuContext(const int v) : device_ordinal_(v) {}
+
+  int device_ordinal() const { return device_ordinal_; }
+
+  // Disallow copying and moving.
+  GpuContext(GpuContext&&) = delete;
+  GpuContext(const GpuContext&) = delete;
+  GpuContext& operator=(GpuContext&&) = delete;
+  GpuContext& operator=(const GpuContext&) = delete;
+
+ private:
+  const int device_ordinal_;
+};
+
+namespace {
+
+// Formats hipError_t to output prettified values into a log stream.
+// Error summaries taken from:
+//
+// TODO(leary) switch to cuGetErrorName when updated rocm.h is available.
+string ToString(hipError_t result) {
+#define OSTREAM_ROCM_ERROR(__name) \
+  case hipError##__name:           \
+    return "HIP_ERROR_" #__name;
+
+  switch (result) {
+    OSTREAM_ROCM_ERROR(InvalidValue)
+    OSTREAM_ROCM_ERROR(OutOfMemory)
+    OSTREAM_ROCM_ERROR(NotInitialized)
+    OSTREAM_ROCM_ERROR(Deinitialized)
+    OSTREAM_ROCM_ERROR(NoDevice)
+    OSTREAM_ROCM_ERROR(InvalidDevice)
+    OSTREAM_ROCM_ERROR(InvalidImage)
+    OSTREAM_ROCM_ERROR(InvalidContext)
+    OSTREAM_ROCM_ERROR(InvalidHandle)
+    OSTREAM_ROCM_ERROR(NotFound)
+    OSTREAM_ROCM_ERROR(NotReady)
+    OSTREAM_ROCM_ERROR(NoBinaryForGpu)
+
+    // Encountered an uncorrectable ECC error during execution.
+    OSTREAM_ROCM_ERROR(ECCNotCorrectable)
+
+    // Load/store on an invalid address. Must reboot all context.
+    case 700:
+      return "ROCM_ERROR_ILLEGAL_ADDRESS";
+    // Passed too many / wrong arguments, too many threads for register count.
+    case 701:
+      return "ROCM_ERROR_LAUNCH_OUT_OF_RESOURCES";
+
+      OSTREAM_ROCM_ERROR(ContextAlreadyInUse)
+      OSTREAM_ROCM_ERROR(PeerAccessUnsupported)
+      OSTREAM_ROCM_ERROR(Unknown)  // Unknown internal error to ROCM.
+    default:
+      return absl::StrCat("hipError_t(", static_cast<int>(result), ")");
+  }
+}
+
+// ROCM driver routines may require a large amount of stack (particularly
+// hipModuleLoadDataEx, in our experience). To avoid stack overflow when using
+// stack-limited threads (such as those spawned by a default-argument
+// thread::ThreadPool on some platforms), we run certain routines in this pool
+// and wait for completion.
+static mutex driver_executor_threadpool_mu(LINKER_INITIALIZED);
+static port::ThreadPool* InitializeDriverExecutor() {
+  return new port::ThreadPool(port::Env::Default(), port::ThreadOptions(),
+                              "rocm_driver", 1);
+}
+
+port::ThreadPool* GetDriverExecutor() {
+  mutex_lock lock(driver_executor_threadpool_mu);
+  static port::ThreadPool* thread_pool = InitializeDriverExecutor();
+  return thread_pool;
+}
+
+}  // namespace
+
+string MemorySpaceString(MemorySpace memory_space) {
+  switch (memory_space) {
+    case MemorySpace::kHost:
+      return "host";
+    case MemorySpace::kDevice:
+      return "device";
+    default:
+      LOG(FATAL) << "impossible memory space";
+  }
+}
+
+// Returns the current device set in HIP. This is done by calling the
+// HIP driver (e.g., this value is not our cached view of the current device).
+static int CurrentDeviceOrDie() {
+  int current = -1;
+  hipError_t result = hipGetDevice(&current);
+  if (result != hipSuccess) {
+    LOG(FATAL) << "failed to query current device: " << ToString(result);
+  }
+  return current;
+}
+
+namespace {
+
+// Call hipDeviceSynchronize and crash if it doesn't succeed.
+void SynchronizeOrDie() {
+  auto res = hipDeviceSynchronize();
+  if (res != hipSuccess) {
+    LOG(FATAL) << "Synchronize found " << ToString(res)
+               << " :: " << port::CurrentStackTrace();
+  }
+}
+
+struct ThreadLocalData {
+  int current_device_ordinal;
+  int depth;
+};
+
+SE_STATIC_THREAD_LOCAL_POD(ThreadLocalData, tls_data);
+
+}  // namespace
+
+ScopedActivateContext::ScopedActivateContext(GpuContext* context) {
+  if (FLAGS_gpuexec_rocm_sync_around_driver_calls) {
+    SynchronizeOrDie();
+  }
+
+  auto* tls = &tls_data.get();
+  if (tls->depth == 0) {
+    tls->current_device_ordinal = CurrentDeviceOrDie();
+  }
+
+  if (kVerifyGpuContext) {
+    CHECK_EQ(CurrentDeviceOrDie(), tls->current_device_ordinal);
+  }
+
+  tls->depth++;
+
+  to_restore_ = context;
+
+  if (context->device_ordinal() == tls->current_device_ordinal) {
+    DCHECK_EQ(CurrentDeviceOrDie(), context->device_ordinal());
+    return;
+  }
+
+  VLOG(3) << "ScopedActivateContext switching device from "
+          << tls->current_device_ordinal << " to " << context->device_ordinal();
+
+  // Set the device and update thread local.
+  CHECK_EQ(hipSuccess, hipSetDevice(context->device_ordinal()));
+  tls->current_device_ordinal = context->device_ordinal();
+}
+
+ScopedActivateContext::~ScopedActivateContext() {
+  if (FLAGS_gpuexec_rocm_sync_around_driver_calls) {
+    SynchronizeOrDie();
+  }
+
+  auto* tls = &tls_data.get();
+
+  if (kVerifyGpuContext) {
+    CHECK_EQ(CurrentDeviceOrDie(), tls->current_device_ordinal);
+  }
+
+  tls->depth--;
+  DCHECK_GE(tls->depth, 0);
+
+  if (to_restore_->device_ordinal() == tls->current_device_ordinal) {
+    DCHECK_EQ(CurrentDeviceOrDie(), to_restore_->device_ordinal());
+    return;
+  }
+
+  VLOG(3) << "ScopedActivateContext switching device from "
+          << tls->current_device_ordinal << " to "
+          << to_restore_->device_ordinal();
+
+  // Set context and update thread local.
+  CHECK_EQ(hipSuccess, hipSetDevice(to_restore_->device_ordinal()));
+  tls->current_device_ordinal = to_restore_->device_ordinal();
+}
+
+namespace {
+
+// Returns a stringified device number associated with pointer, primarily for
+// logging purposes. Returns "?" if the device could not be successfully
+// queried.
+string ROCMPointerToDeviceString(hipDeviceptr_t pointer) {
+  auto value = GpuDriver::GetPointerDevice(pointer);
+  if (value.ok()) {
+    return absl::StrCat(value.ValueOrDie());
+  }
+  LOG(ERROR) << "could not query device: " << value.status();
+  return "?";
+}
+
+// Returns a stringified memory space associated with pointer, primarily for
+// logging purposes. Returns "?" if the memory space could not be successfully
+// queried.
+string ROCMPointerToMemorySpaceString(hipDeviceptr_t pointer) {
+  auto value = GpuDriver::GetPointerMemorySpace(pointer);
+  if (value.ok()) {
+    return MemorySpaceString(value.ValueOrDie());
+  }
+  LOG(ERROR) << "could not query device: " << value.status();
+  return "?";
+}
+
+// Returns a stringified representation of whether or not peer access is
+// permitted between the "from" and "to" pointers' associated contexts,
+// primarily for logging purposes. Returns "error" if an error is encountered
+// in the process of querying.
+string ROCMPointersToCanAccessString(hipDeviceptr_t from, hipDeviceptr_t to) {
+  hipPointerAttribute_t from_pointerAttributes;
+  hipError_t result = hipPointerGetAttributes(&from_pointerAttributes, from);
+  if (result != hipSuccess) {
+    LOG(ERROR) << "could not retrieve source pointer's device: "
+               << ToString(result);
+    return "error";
+  }
+
+  hipPointerAttribute_t to_pointerAttributes;
+  result = hipPointerGetAttributes(&to_pointerAttributes, to);
+  if (result != hipSuccess) {
+    LOG(ERROR) << "could not retrieve destination pointer's device: "
+               << ToString(result);
+    return "error";
+  }
+
+  GpuContext fromCtx(from_pointerAttributes.device);
+  GpuContext toCtx(to_pointerAttributes.device);
+
+  return GpuDriver::CanEnablePeerAccess(&fromCtx, &toCtx) ? "true" : "false";
+}
+
+// Actually performs the work of ROCM initialization. Wrapped up in one-time
+// execution guard.
+static port::Status InternalInit() {
+  hipError_t res = hipErrorNoDevice;
+  if (FLAGS_gpuexec_rocm_driver_inject_init_error) {
+    LOG(ERROR) << "injecting ROCM init error; initialization will fail";
+  } else {
+    res = hipInit(0 /* = flags */);
+  }
+
+  if (res == hipSuccess) {
+    return port::Status::OK();
+  }
+
+  LOG(ERROR) << "failed call to hipInit: " << ToString(res);
+  Diagnostician::LogDiagnosticInformation();
+  return port::Status{port::error::ABORTED,
+                      absl::StrCat("failed call to hipInit: ", ToString(res))};
+}
+
+}  // namespace
+
+/* static */ port::Status GpuDriver::Init() {
+  // Cached return value from calling InternalInit(), as hipInit need only be
+  // called once, but GpuDriver::Init may be called many times.
+  static port::Status init_retval;
+  static bool set = false;
+  static mutex* init_mu = new mutex;
+
+  mutex_lock lock(*init_mu);
+  if (!set) {
+    init_retval = InternalInit();
+    set = true;
+  }
+
+  return init_retval;
+}
+
+/* static */ port::Status GpuDriver::GetDevice(int device_ordinal,
+                                               hipDevice_t* device) {
+  hipError_t res = hipDeviceGet(device, device_ordinal);
+  if (res == hipSuccess) {
+    return port::Status::OK();
+  }
+
+  return port::Status{
+      port::error::INTERNAL,
+      absl::StrCat("failed call to hipDeviceGet: ", ToString(res))};
+}
+
+/* static */ bool GpuDriver::GetDeviceName(hipDevice_t device,
+                                           string* device_name) {
+  static const size_t kCharLimit = 64;
+  absl::InlinedVector<char, 4> chars(kCharLimit);
+  hipError_t res = hipDeviceGetName(chars.begin(), kCharLimit - 1, device);
+  if (res != hipSuccess) {
+    LOG(ERROR) << "failed to get device name for " << device << ": "
+               << ToString(res);
+    return false;
+  }
+  chars[kCharLimit - 1] = '\0';
+  *device_name = chars.begin();
+  return true;
+}
+
+bool DeviceOptionsToContextFlags(const DeviceOptions& device_options,
+                                 int* flags) {
+  static_assert(DeviceOptions::kMask == 0xf,
+                "needs update for new device options");
+  return true;
+}
+
+/* static */ port::Status GpuDriver::CreateContext(
+    int device_ordinal, hipDevice_t device, const DeviceOptions& device_options,
+    GpuContext** context) {
+  *context = new GpuContext(device_ordinal);
+  return port::Status::OK();
+}
+/* static */ void GpuDriver::DestroyContext(GpuContext* context) {
+  if (context == nullptr) {
+    return;
+  }
+  delete context;
+}
+
+/* static */ bool GpuDriver::FuncGetAttribute(hipDeviceAttribute_t attribute,
+                                              hipFunction_t func,
+                                              int* attribute_value) {
+  // TODO(ROCm) properly implement this feature in HIP
+  hipError_t res = hipSuccess;
+  if (res != hipSuccess) {
+    LOG(ERROR) << "failed to query kernel attribute. kernel: " << func
+               << ", attribute: " << attribute;
+    return false;
+  }
+  return true;
+}
+
+/* static */ bool GpuDriver::FuncSetCacheConfig(hipFunction_t function,
+                                                hipFuncCache_t cache_config) {
+  hipError_t res = hipFuncSetCacheConfig(function, cache_config);
+  if (res != hipSuccess) {
+    LOG(ERROR) << "failed to set ROCM kernel cache config. kernel: " << function
+               << ", config: " << cache_config << ", result: " << ToString(res);
+    return false;
+  }
+
+  return true;
+}
+
+/* static */ port::StatusOr<hipSharedMemConfig>
+GpuDriver::ContextGetSharedMemConfig(GpuContext* context) {
+  hipSharedMemConfig shared_mem_config;
+  ScopedActivateContext activation{context};
+  hipError_t result = hipDeviceGetSharedMemConfig(&shared_mem_config);
+  if (result != hipSuccess) {
+    LOG(ERROR) << "failed to get ROCM device shared memory config. "
+               << "Context device ID: " << context->device_ordinal()
+               << ", result: " << ToString(result);
+    return port::Status{
+        port::error::INTERNAL,
+        absl::StrCat("failed to get shared memory config: ", ToString(result))};
+  }
+  return shared_mem_config;
+}
+
+/* static */ port::Status GpuDriver::ContextSetSharedMemConfig(
+    GpuContext* context, hipSharedMemConfig shared_mem_config) {
+  ScopedActivateContext activation{context};
+  hipError_t result = hipDeviceSetSharedMemConfig(shared_mem_config);
+  if (result != hipSuccess) {
+    LOG(ERROR) << "failed to set ROCM device shared memory config. "
+               << "Context device ID: " << context->device_ordinal()
+               << ", config: " << shared_mem_config
+               << ", result: " << ToString(result);
+    return port::Status{
+        port::error::INTERNAL,
+        absl::StrCat("failed to set shared memory config: ", ToString(result))};
+  }
+  return port::Status::OK();
+}
+
+/* static */ bool GpuDriver::LaunchKernel(
+    GpuContext* context, hipFunction_t function, unsigned int grid_dim_x,
+    unsigned int grid_dim_y, unsigned int grid_dim_z, unsigned int block_dim_x,
+    unsigned int block_dim_y, unsigned int block_dim_z,
+    unsigned int shared_mem_bytes, GpuStreamHandle stream, void** kernel_params,
+    void** extra) {
+  ScopedActivateContext activation{context};
+  VLOG(2) << "launching kernel: " << function << "; gdx: " << grid_dim_x
+          << " gdy: " << grid_dim_y << " gdz: " << grid_dim_z
+          << " bdx: " << block_dim_x << " bdy: " << block_dim_y
+          << " bdz: " << block_dim_z << " smem: " << shared_mem_bytes;
+  hipError_t res = hipModuleLaunchKernel(
+      function, grid_dim_x, grid_dim_y, grid_dim_z, block_dim_x, block_dim_y,
+      block_dim_z, shared_mem_bytes, stream, kernel_params, extra);
+  if (res != hipSuccess) {
+    LOG(ERROR) << "failed to launch ROCM kernel: " << function
+               << "; result: " << ToString(res);
+    return false;
+  }
+  VLOG(2) << "successfully launched kernel";
+  return true;
+}
+
+/* static */ bool GpuDriver::LoadPtx(GpuContext* context,
+                                     const char* ptx_contents,
+                                     hipModule_t* module) {
+  LOG(ERROR) << "Feature not supported on ROCm platform (LoadPtx)";
+  return false;
+}
+
+/* static */ port::Status GpuDriver::LoadCubin(GpuContext* context,
+                                               const char* cubin_bytes,
+                                               hipModule_t* module) {
+  return port::Status{port::error::INTERNAL,
+                      "Feature not supported on ROCm platform (LoadCubin)"};
+}
+
+/* static */ bool GpuDriver::LoadHsaco(GpuContext* context,
+                                       const char* hsaco_contents,
+                                       hipModule_t* module) {
+  port::Notification notification;
+  bool ret = true;
+  GetDriverExecutor()->Schedule(
+      [context, hsaco_contents, module, &ret, &notification]() {
+        ScopedActivateContext activation{context};
+        void* hsaco_data = const_cast<char*>(hsaco_contents);
+
+        hipError_t res = hipModuleLoadData(module, hsaco_data);
+
+        if (res != hipSuccess) {
+          LOG(ERROR) << "failed to load HSACO: " << ToString(res);
+          ret = false;
+          notification.Notify();
+        }
+
+        CHECK(module != nullptr);
+        notification.Notify();
+      });
+  notification.WaitForNotification();
+
+  return ret;
+}
+
+/* static */ bool GpuDriver::SynchronousMemsetUint8(GpuContext* context,
+                                                    hipDeviceptr_t location,
+                                                    uint8 value, size_t size) {
+  ScopedActivateContext activation{context};
+  hipError_t res = hipMemset(location, value, size);
+  if (res != hipSuccess) {
+    LOG(ERROR) << "failed to memset memory: " << ToString(res);
+    return false;
+  }
+  return true;
+}
+
+/* static */ bool GpuDriver::SynchronousMemsetUint32(GpuContext* context,
+                                                     hipDeviceptr_t location,
+                                                     uint32 value,
+                                                     size_t uint32_count) {
+  ScopedActivateContext activation{context};
+  void* pointer = absl::bit_cast<void*>(location);
+  unsigned char valueC = static_cast<unsigned char>(value);
+  uint32_t value32 = (valueC << 24) | (valueC << 16) | (valueC << 8) | (valueC);
+  if (value32 != value) {
+    //  mismatch indicates case where hipMemsetAsyc can't emulate hipMemSetD32
+    LOG(ERROR) << "failed to memset memory";
+    return false;
+  }
+  hipError_t res =
+      hipMemset(pointer, static_cast<int>(value), uint32_count * 4);
+  if (res != hipSuccess) {
+    LOG(ERROR) << "failed to memset memory: " << ToString(res);
+    return false;
+  }
+  return true;
+}
+
+/* static */ bool GpuDriver::AsynchronousMemsetUint8(GpuContext* context,
+                                                     hipDeviceptr_t location,
+                                                     uint8 value,
+                                                     size_t uint32_count,
+                                                     GpuStreamHandle stream) {
+  ScopedActivateContext activation{context};
+  hipError_t res = hipMemsetAsync(location, value, uint32_count, stream);
+  if (res != hipSuccess) {
+    LOG(ERROR) << "failed to enqueue async memset operation: " << ToString(res);
+    return false;
+  }
+  VLOG(2) << "successfully enqueued async memset operation";
+  return true;
+}
+
+/* static */ bool GpuDriver::AsynchronousMemsetUint32(GpuContext* context,
+                                                      hipDeviceptr_t location,
+                                                      uint32 value,
+                                                      size_t uint32_count,
+                                                      GpuStreamHandle stream) {
+  ScopedActivateContext activation{context};
+  void* pointer = absl::bit_cast<void*>(location);
+
+  // FIXME - need to set a 32-bit value here
+  unsigned char valueC = static_cast<unsigned char>(value);
+  uint32_t value32 = (valueC << 24) | (valueC << 16) | (valueC << 8) | (valueC);
+  if (value32 != value) {
+    // mismatch indicates case where hipMemsetAsyc can't emulate hipMemSetD32
+    LOG(ERROR) << "failed to memset memory";
+    return false;
+  }
+  hipError_t res = hipMemsetAsync(pointer, value, uint32_count * 4, stream);
+  if (res != hipSuccess) {
+    LOG(ERROR) << "failed to enqueue async memset operation: " << ToString(res);
+    return false;
+  }
+  VLOG(2) << "successfully enqueued async memset operation";
+  return true;
+}
+
+/* static */ bool GpuDriver::AddStreamCallback(GpuContext* context,
+                                               GpuStreamHandle stream,
+                                               StreamCallback callback,
+                                               void* data) {
+  hipError_t res = hipStreamAddCallback(stream, (hipStreamCallback_t)callback,
+                                        data, 0 /* = flags */);
+  if (res != hipSuccess) {
+    LOG(ERROR) << "unable to add host callback: " << ToString(res);
+    return false;
+  }
+  return true;
+}
+
+/* static */ bool GpuDriver::GetModuleFunction(GpuContext* context,
+                                               hipModule_t module,
+                                               const char* kernel_name,
+                                               hipFunction_t* function) {
+  ScopedActivateContext activated{context};
+  CHECK(module != nullptr && kernel_name != nullptr);
+  hipError_t res = hipModuleGetFunction(function, module, kernel_name);
+  if (res != hipSuccess) {
+    LOG(ERROR) << "failed to get kernel \"" << kernel_name
+               << "\" from module: " << ToString(res);
+    return false;
+  }
+
+  return true;
+}
+
+/* static */ bool GpuDriver::GetModuleSymbol(GpuContext* context,
+                                             hipModule_t module,
+                                             const char* symbol_name,
+                                             hipDeviceptr_t* dptr,
+                                             size_t* bytes) {
+  ScopedActivateContext activated{context};
+  CHECK(module != nullptr && symbol_name != nullptr &&
+        (dptr != nullptr || bytes != nullptr));
+  hipError_t res = hipModuleGetGlobal(dptr, bytes, module, symbol_name);
+  if (res != hipSuccess) {
+    // symbol may not be found in the current module, but it may reside in
+    // another module.
+    VLOG(2) << "failed to get symbol \"" << symbol_name
+            << "\" from module: " << ToString(res);
+    return false;
+  }
+
+  return true;
+}
+
+/* static */ void GpuDriver::UnloadModule(GpuContext* context,
+                                          hipModule_t module) {
+  ScopedActivateContext activated{context};
+  hipError_t res = hipModuleUnload(module);
+  if (res != hipSuccess) {
+    LOG(ERROR) << "failed to unload module " << module
+               << "; leaking: " << ToString(res);
+  }
+}
+
+/* static */ bool GpuDriver::CreateStream(GpuContext* context,
+                                          GpuStreamHandle* stream) {
+  ScopedActivateContext activated{context};
+  hipError_t res = hipStreamCreateWithFlags(
+      stream, hipStreamDefault);  // switch to hipStreamNonBlocking?
+  if (res != hipSuccess) {
+    LOG(ERROR) << "could not allocate ROCM stream for device "
+               << context->device_ordinal() << ": " << ToString(res);
+    return false;
+  }
+
+  VLOG(2) << "successfully created stream " << *stream << " for device "
+          << context->device_ordinal() << " on thread";
+  return true;
+}
+
+/* static */ void GpuDriver::DestroyStream(GpuContext* context,
+                                           GpuStreamHandle* stream) {
+  if (*stream == nullptr) {
+    return;
+  }
+
+  ScopedActivateContext activated{context};
+  hipError_t res = hipStreamDestroy(*stream);
+  if (res != hipSuccess) {
+    LOG(ERROR) << "failed to destroy ROCM stream for device "
+               << context->device_ordinal() << ": " << ToString(res);
+  } else {
+    VLOG(2) << "successfully destroyed stream " << *stream << " for device "
+            << context->device_ordinal();
+    *stream = nullptr;
+  }
+}
+
+/* static */ void* GpuDriver::DeviceAllocate(GpuContext* context,
+                                             uint64 bytes) {
+  ScopedActivateContext activated{context};
+  hipDeviceptr_t result = 0;
+  hipError_t res = hipMalloc(&result, bytes);
+  if (res != hipSuccess) {
+    LOG(ERROR) << "failed to allocate "
+               << port::HumanReadableNumBytes::ToString(bytes) << " (" << bytes
+               << " bytes) from device: " << ToString(res);
+    return nullptr;
+  }
+  void* ptr = reinterpret_cast<void*>(result);
+  VLOG(2) << "allocated " << ptr << " for device " << context->device_ordinal()
+          << " of " << bytes << " bytes";
+  return ptr;
+}
+
+/* static */ void GpuDriver::DeviceDeallocate(GpuContext* context,
+                                              void* location) {
+  ScopedActivateContext activation{context};
+  hipDeviceptr_t pointer = absl::bit_cast<hipDeviceptr_t>(location);
+  hipError_t res = hipFree(pointer);
+  if (res != hipSuccess) {
+    LOG(ERROR) << "failed to free device memory at " << location
+               << "; result: " << ToString(res);
+  } else {
+    VLOG(2) << "deallocated " << location << " for device "
+            << context->device_ordinal();
+  }
+}
+
+/* static */ void* GpuDriver::UnifiedMemoryAllocate(GpuContext* context,
+                                                    uint64 bytes) {
+  ScopedActivateContext activated{context};
+
+  LOG(ERROR)
+      << "Feature not supported on ROCm platform (UnifiedMemoryAllocate)";
+  return nullptr;
+}
+
+/* static */ void GpuDriver::UnifiedMemoryDeallocate(GpuContext* context,
+                                                     void* location) {
+  LOG(ERROR)
+      << "Feature not supported on ROCm platform (UnifiedMemoryDeallocate)";
+}
+
+/* static */ void* GpuDriver::HostAllocate(GpuContext* context, uint64 bytes) {
+  ScopedActivateContext activation{context};
+  void* host_mem = nullptr;
+  // "Portable" memory is visible to all ROCM contexts. Safe for our use model.
+  hipError_t res = hipHostMalloc(&host_mem, bytes, hipHostMallocPortable);
+  if (res != hipSuccess) {
+    LOG(ERROR) << "failed to alloc " << bytes
+               << " bytes on host: " << ToString(res);
+  }
+  return host_mem;
+}
+
+/* static */ void GpuDriver::HostDeallocate(GpuContext* context,
+                                            void* location) {
+  ScopedActivateContext activation{context};
+  hipError_t res = hipHostFree(location);
+  if (res != hipSuccess) {
+    LOG(ERROR) << "error deallocating host memory at " << location << ": "
+               << ToString(res);
+  }
+}
+
+/* static */ bool GpuDriver::HostRegister(GpuContext* context, void* location,
+                                          uint64 bytes) {
+  ScopedActivateContext activation{context};
+  // "Portable" memory is visible to all ROCM contexts. Safe for our use model.
+  hipError_t res = hipHostRegister(location, bytes, hipHostRegisterPortable);
+  if (res != hipSuccess) {
+    LOG(ERROR) << "error registering host memory at " << location << ": "
+               << ToString(res);
+    return false;
+  }
+  return true;
+}
+
+/* static */ bool GpuDriver::HostUnregister(GpuContext* context,
+                                            void* location) {
+  ScopedActivateContext activation{context};
+  hipError_t res = hipHostUnregister(location);
+  if (res != hipSuccess) {
+    LOG(ERROR) << "error unregistering host memory at " << location << ": "
+               << ToString(res);
+    return false;
+  }
+  return true;
+}
+
+/* static */ port::Status GpuDriver::DestroyEvent(GpuContext* context,
+                                                  GpuEventHandle* event) {
+  if (*event == nullptr) {
+    return port::Status{port::error::INVALID_ARGUMENT,
+                        "input event cannot be null"};
+  }
+
+  ScopedActivateContext activated{context};
+  hipError_t res = hipEventDestroy(*event);
+  *event = nullptr;
+
+  switch (res) {
+    case hipSuccess:
+      return port::Status::OK();
+    case hipErrorDeinitialized:
+    case hipErrorNotInitialized:
+      return port::Status{
+          port::error::FAILED_PRECONDITION,
+          absl::StrFormat("error destroying ROCM event in device %d: %s",
+                          context->device_ordinal(), ToString(res).c_str())};
+    default:
+      return port::Status{
+          port::error::INTERNAL,
+          absl::StrFormat("error destroying ROCM event in device %d: %s",
+                          context->device_ordinal(), ToString(res).c_str())};
+  }
+}
+
+/* static */ port::Status GpuDriver::RecordEvent(GpuContext* context,
+                                                 GpuEventHandle event,
+                                                 GpuStreamHandle stream) {
+  ScopedActivateContext activated{context};
+  hipError_t res = hipEventRecord(event, stream);
+  switch (res) {
+    case hipSuccess:
+      return port::Status::OK();
+    case hipErrorDeinitialized:
+    case hipErrorNotInitialized:
+      return port::Status{
+          port::error::FAILED_PRECONDITION,
+          absl::StrFormat("error recording ROCM event on stream %p: %s", stream,
+                          ToString(res).c_str())};
+    default:
+      return port::Status{
+          port::error::INVALID_ARGUMENT,
+          absl::StrFormat("error recording ROCM event on stream %p: %s", stream,
+                          ToString(res).c_str())};
+  }
+}
+
+/* static */ port::StatusOr<hipError_t> GpuDriver::QueryEvent(
+    GpuContext* context, GpuEventHandle event) {
+  ScopedActivateContext activated{context};
+  hipError_t res = hipEventQuery(event);
+  if (res != hipSuccess && res != hipErrorNotReady) {
+    return port::Status{
+        port::error::INTERNAL,
+        absl::StrFormat("failed to query event: %s", ToString(res).c_str())};
+  }
+
+  return res;
+}
+
+/* static */ bool GpuDriver::GetEventElapsedTime(GpuContext* context,
+                                                 float* elapsed_milliseconds,
+                                                 GpuEventHandle start,
+                                                 GpuEventHandle stop) {
+  ScopedActivateContext activated{context};
+  // The stop event must have completed in order for hipEventElapsedTime to
+  // work.
+  hipError_t res = hipEventSynchronize(stop);
+  if (res != hipSuccess) {
+    LOG(ERROR) << "failed to synchronize the stop event: " << ToString(res);
+    return false;
+  }
+  res = hipEventElapsedTime(elapsed_milliseconds, start, stop);
+  if (res != hipSuccess) {
+    LOG(ERROR) << "failed to get elapsed time between events: "
+               << ToString(res);
+    return false;
+  }
+
+  return true;
+}
+
+/* static */ bool GpuDriver::WaitStreamOnEvent(GpuContext* context,
+                                               GpuStreamHandle stream,
+                                               GpuEventHandle event) {
+  ScopedActivateContext activation{context};
+  hipError_t res = hipStreamWaitEvent(stream, event, 0 /* = flags */);
+  if (res != hipSuccess) {
+    LOG(ERROR) << "could not wait stream on event: " << ToString(res);
+    return false;
+  }
+
+  return true;
+}
+
+/* static */ bool GpuDriver::SynchronizeContext(GpuContext* context) {
+  ScopedActivateContext activation{context};
+  hipError_t res = hipDeviceSynchronize();
+  if (res != hipSuccess) {
+    LOG(ERROR) << "could not synchronize on ROCM device: " << ToString(res)
+               << " :: " << port::CurrentStackTrace();
+    return false;
+  }
+
+  return true;
+}
+
+/* static */ port::Status GpuDriver::SynchronizeStream(GpuContext* context,
+                                                       GpuStreamHandle stream) {
+  ScopedActivateContext activated{context};
+  CHECK(stream != nullptr);
+  hipError_t res = hipStreamSynchronize(stream);
+  if (res != hipSuccess) {
+    port::Status status = port::InternalError(
+        absl::StrCat("could not synchronize on ROCM stream: ", ToString(res)));
+    LOG(ERROR) << status << " :: " << port::CurrentStackTrace();
+    return status;
+  }
+  VLOG(2) << "successfully synchronized stream " << stream << " on device "
+          << context->device_ordinal();
+  return port::Status::OK();
+}
+
+/* static */ bool GpuDriver::IsStreamIdle(GpuContext* context,
+                                          GpuStreamHandle stream) {
+  ScopedActivateContext activated{context};
+  CHECK(stream != nullptr);
+  hipError_t res = hipStreamQuery(stream);
+  if (res == hipSuccess) {
+    return true;
+  }
+
+  if (res != hipErrorNotReady) {
+    LOG(ERROR) << "stream in bad state on status query: " << ToString(res);
+  }
+  return false;
+}
+
+/* static */ port::Status GpuDriver::SynchronousMemcpyD2H(
+    GpuContext* context, void* host_dst, hipDeviceptr_t gpu_src, uint64 size) {
+  ScopedActivateContext activation{context};
+  hipError_t res = hipMemcpyDtoH(host_dst, gpu_src, size);
+  if (res != hipSuccess) {
+    return port::InternalError(
+        absl::StrFormat("failed to synchronous memcpy from device to host: %s; "
+                        "host dst: %p; Gpu src: %p; size: %llu=0x%llx",
+                        ToString(res).c_str(), host_dst,
+                        absl::bit_cast<void*>(gpu_src), size, size));
+  }
+  VLOG(2) << "successfully sync memcpy'd d2h of " << size << " bytes to "
+          << host_dst;
+  return port::Status::OK();
+}
+
+/* static */ port::Status GpuDriver::SynchronousMemcpyH2D(
+    GpuContext* context, hipDeviceptr_t gpu_dst, const void* host_src,
+    uint64 size) {
+  ScopedActivateContext activation{context};
+  hipError_t res = hipMemcpyHtoD(gpu_dst, const_cast<void*>(host_src), size);
+  if (res != hipSuccess) {
+    return port::InternalError(absl::StrFormat(
+        "failed to synchronous memcpy from host to device: %s; Gpu dst: %p;"
+        " host src: %p; size: %llu=0x%llx",
+        ToString(res).c_str(), absl::bit_cast<void*>(gpu_dst), host_src, size,
+        size));
+  }
+  VLOG(2) << "successfully enqueued sync memcpy h2d of " << size << " bytes";
+  return port::Status::OK();
+}
+
+/* static */ port::Status GpuDriver::SynchronousMemcpyD2D(
+    GpuContext* context, hipDeviceptr_t gpu_dst, hipDeviceptr_t gpu_src,
+    uint64 size) {
+  ScopedActivateContext activation{context};
+  hipError_t res = hipMemcpyDtoD(gpu_dst, gpu_src, size);
+  if (res != hipSuccess) {
+    return port::InternalError(absl::StrFormat(
+        "failed to synchronous memcpy from host to device: %s; Gpu dst: %p; "
+        "Gpu src: %p; size: %llu=0x%llx",
+        ToString(res).c_str(), absl::bit_cast<void*>(gpu_dst),
+        absl::bit_cast<void*>(gpu_src), size, size));
+  }
+  VLOG(2) << "successfully sync memcpy'd d2d of " << size << " bytes";
+  return port::Status::OK();
+}
+
+/* static */ bool GpuDriver::AsynchronousMemcpyD2H(GpuContext* context,
+                                                   void* host_dst,
+                                                   hipDeviceptr_t gpu_src,
+                                                   uint64 size,
+                                                   GpuStreamHandle stream) {
+  ScopedActivateContext activation{context};
+  hipError_t res = hipMemcpyDtoHAsync(host_dst, gpu_src, size, stream);
+  if (res != hipSuccess) {
+    LOG(ERROR) << absl::StrFormat(
+        "failed to enqueue async memcpy from device to host: %s; host dst: %p; "
+        "Gpu src: %p; size: %llu=0x%llx",
+        ToString(res).c_str(), host_dst, absl::bit_cast<void*>(gpu_src), size,
+        size);
+    return false;
+  }
+  VLOG(2) << "successfully enqueued async memcpy d2h of " << size
+          << " bytes from " << absl::bit_cast<void*>(gpu_src) << " to "
+          << host_dst << " on stream " << stream;
+  return true;
+}
+
+/* static */ bool GpuDriver::AsynchronousMemcpyH2D(GpuContext* context,
+                                                   hipDeviceptr_t gpu_dst,
+                                                   const void* host_src,
+                                                   uint64 size,
+                                                   GpuStreamHandle stream) {
+  ScopedActivateContext activation{context};
+  hipError_t res =
+      hipMemcpyHtoDAsync(gpu_dst, const_cast<void*>(host_src), size, stream);
+  if (res != hipSuccess) {
+    LOG(ERROR) << absl::StrFormat(
+        "failed to enqueue async memcpy from host to device: %s; Gpu dst: %p; "
+        "host src: %p; size: %llu=0x%llx",
+        ToString(res).c_str(), absl::bit_cast<void*>(gpu_dst), host_src, size,
+        size);
+    return false;
+  }
+  VLOG(2) << "successfully enqueued async memcpy h2d of " << size << " bytes"
+          << " on stream " << stream;
+  return true;
+}
+
+/* static */ bool GpuDriver::AsynchronousMemcpyD2D(GpuContext* context,
+                                                   hipDeviceptr_t gpu_dst,
+                                                   hipDeviceptr_t gpu_src,
+                                                   uint64 size,
+                                                   GpuStreamHandle stream) {
+  ScopedActivateContext activation{context};
+  hipError_t result = hipMemcpyDtoDAsync(gpu_dst, gpu_src, size, stream);
+  if (result != hipSuccess) {
+    LOG(ERROR) << absl::StrFormat(
+        "failed to enqueue async memcpy from device to device: %s"
+        "; Gpu dst: %p on %s %s"
+        "; Gpu src: %p on %s %s"
+        "; can access? %s; size: %llu=0x%llx",
+        ToString(result).c_str(), absl::bit_cast<void*>(gpu_dst),
+        ROCMPointerToMemorySpaceString(gpu_dst).c_str(),
+        ROCMPointerToDeviceString(gpu_dst).c_str(),
+        absl::bit_cast<void*>(gpu_src),
+        ROCMPointerToMemorySpaceString(gpu_src).c_str(),
+        ROCMPointerToDeviceString(gpu_src).c_str(),
+        ROCMPointersToCanAccessString(gpu_src, gpu_dst).c_str(), size, size);
+
+    return false;
+  }
+  VLOG(2) << "successfully enqueued async memcpy d2d of " << size << " bytes";
+  return true;
+}
+
+/* static */ port::Status GpuDriver::CreateEvent(GpuContext* context,
+                                                 GpuEventHandle* event,
+                                                 EventFlags flags) {
+  int hipflags;
+  switch (flags) {
+    case EventFlags::kDefault:
+      hipflags = hipEventDefault;
+      break;
+    case EventFlags::kDisableTiming:
+      hipflags = hipEventDisableTiming | hipEventReleaseToSystem;
+      break;
+    default:
+      LOG(FATAL) << "impossible event flags: " << int(hipflags);
+  }
+
+  ScopedActivateContext activated{context};
+  hipError_t res = hipEventCreateWithFlags(event, hipflags);
+
+  if (res == hipSuccess) {
+    return port::Status::OK();
+  } else if (res == hipErrorMemoryAllocation) {
+    return port::Status{port::error::RESOURCE_EXHAUSTED,
+                        "could not create ROCM event: out of device memory"};
+  } else {
+    return port::Status{
+        port::error::FAILED_PRECONDITION,
+        absl::StrCat("could not create ROCM event: ", ToString(res))};
+  }
+}
+
+/* static */ int GpuDriver::GetDeviceCount() {
+  int device_count = 0;
+  hipError_t res = hipGetDeviceCount(&device_count);
+  if (res != hipSuccess) {
+    LOG(ERROR) << "could not retrieve ROCM device count: " << ToString(res);
+    return 0;
+  }
+
+  if (FLAGS_gpuexec_rocm_device_0_only && device_count > 1) {
+    device_count = 1;
+  }
+  return device_count;
+}
+
+/* static */ port::Status GpuDriver::GetComputeCapability(int* cc_major,
+                                                          int* cc_minor,
+                                                          hipDevice_t device) {
+  return port::Status(
+      port::error::INTERNAL,
+      absl::StrFormat("failed to get compute capability for device: %d "
+                      "(unsupported API on AMD Gpus)",
+                      device));
+}
+
+/* static */ port::Status GpuDriver::GetPointerAddressRange(
+    hipDeviceptr_t dptr, hipDeviceptr_t* base, size_t* size) {
+  hipError_t result = hipMemGetAddressRange(base, size, dptr);
+  if (result == hipSuccess) {
+    return port::Status::OK();
+  } else if (result == hipErrorNotFound) {
+    // We differentiate between "this pointer is unknown" (return here) and
+    // "there was an internal error while performing this operation" (return
+    // below).
+    return port::Status{port::error::NOT_FOUND,
+                        absl::StrFormat("not a device pointer %p; %s",
+                                        reinterpret_cast<void*>(dptr),
+                                        ToString(result).c_str())};
+  }
+
+  return port::Status{
+      port::error::INTERNAL,
+      absl::StrFormat("failed to get pointer into for device pointer %p; %s",
+                      reinterpret_cast<void*>(dptr), ToString(result).c_str())};
+}
+
+/* static */ port::StatusOr<MemorySpace> GpuDriver::GetPointerMemorySpace(
+    hipDeviceptr_t pointer) {
+  unsigned int value;
+  hipError_t result = hipSuccess;
+  if (result == hipSuccess) {
+    switch (value) {
+      case hipMemoryTypeDevice:
+        return MemorySpace::kDevice;
+      case hipMemoryTypeHost:
+        return MemorySpace::kHost;
+      default:
+        return port::Status{
+            port::error::INTERNAL,
+            absl::StrCat("unknown memory space provided by ROCM API: ", value)};
+    }
+  }
+
+  return port::Status{
+      port::error::INTERNAL,
+      absl::StrCat("failed to query device pointer for memory space: ",
+                   ToString(result))};
+}
+
+/* static */ port::StatusOr<hipDevice_t> GpuDriver::GetPointerDevice(
+    hipDeviceptr_t pointer) {
+  hipPointerAttribute_t pointerAttributes;
+  hipError_t result = hipPointerGetAttributes(&pointerAttributes, pointer);
+  if (result != hipSuccess) {
+    return port::Status{
+        port::error::INTERNAL,
+        absl::StrCat("failed to get device for pointer: ", ToString(result))};
+  }
+
+  hipDevice_t device;
+  result = hipDeviceGet(&device, pointerAttributes.device);
+  if (result != hipSuccess) {
+    return port::Status{
+        port::error::INTERNAL,
+        absl::StrCat("failed to get device for pointer: ", ToString(result))};
+  }
+
+  return device;
+}
+
+/* static */ port::Status GpuDriver::GetGpuISAVersion(int* version,
+                                                      hipDevice_t device) {
+  hipDeviceProp_t props;
+  hipError_t result = hipGetDeviceProperties(&props, device);
+  if (result == hipSuccess) {
+    *version = props.gcnArch;
+    return port::Status::OK();
+  }
+  *version = 0;
+  return port::Status{
+      port::error::INTERNAL,
+      absl::StrFormat("failed to determine AMDGpu ISA version for device %d",
+                      device)};
+}
+
+// Helper function that turns the integer output of hipDeviceGetAttribute to
+// type T and wraps it in a StatusOr.
+template <typename T>
+static port::StatusOr<T> GetSimpleAttribute(hipDevice_t device,
+                                            hipDeviceAttribute_t attribute) {
+  int value = -1;
+  hipError_t result = hipDeviceGetAttribute(&value, attribute, device);
+  if (result != hipSuccess) {
+    return port::Status{
+        port::error::NOT_FOUND,
+        absl::StrCat("could not retrieve ROCM device attribute (", attribute,
+                     "): ", ToString(result))};
+  }
+  T converted = value;
+  return converted;
+}
+
+/* static */ port::StatusOr<int> GpuDriver::GetMultiprocessorCount(
+    hipDevice_t device) {
+  return GetSimpleAttribute<int>(device, hipDeviceAttributeMultiprocessorCount);
+}
+
+/* static */ port::StatusOr<int64> GpuDriver::GetMaxSharedMemoryPerCore(
+    hipDevice_t device) {
+  return GetSimpleAttribute<int64>(
+      device, hipDeviceAttributeMaxSharedMemoryPerMultiprocessor);
+}
+
+/* static */ port::StatusOr<int64> GpuDriver::GetMaxSharedMemoryPerBlock(
+    hipDevice_t device) {
+  return GetSimpleAttribute<int64>(device,
+                                   hipDeviceAttributeMaxSharedMemoryPerBlock);
+}
+
+/* static */ port::StatusOr<int64> GpuDriver::GetMaxThreadsPerMultiprocessor(
+    hipDevice_t device) {
+  return GetSimpleAttribute<int64>(
+      device, hipDeviceAttributeMaxThreadsPerMultiProcessor);
+}
+
+/* static */ port::StatusOr<int64> GpuDriver::GetMaxThreadsPerBlock(
+    hipDevice_t device) {
+  return GetSimpleAttribute<int64>(device,
+                                   hipDeviceAttributeMaxThreadsPerBlock);
+}
+
+/* static */ port::StatusOr<int64> GpuDriver::GetMaxRegistersPerBlock(
+    hipDevice_t device) {
+  return GetSimpleAttribute<int64>(device,
+                                   hipDeviceAttributeMaxRegistersPerBlock);
+}
+
+/* static */ port::StatusOr<int64> GpuDriver::GetThreadsPerWarp(
+    hipDevice_t device) {
+  return GetSimpleAttribute<int64>(device, hipDeviceAttributeWarpSize);
+}
+
+/* static */ bool GpuDriver::GetGridLimits(int* x, int* y, int* z,
+                                           hipDevice_t device) {
+  int value;
+  hipError_t res =
+      hipDeviceGetAttribute(&value, hipDeviceAttributeMaxGridDimX, device);
+  if (res != hipSuccess) {
+    LOG(ERROR) << "failed to query max grid dim x: " << ToString(res);
+    return false;
+  }
+  *x = value;
+
+  res = hipDeviceGetAttribute(&value, hipDeviceAttributeMaxGridDimY, device);
+  if (res != hipSuccess) {
+    LOG(ERROR) << "failed to query max grid dim y: " << ToString(res);
+    return false;
+  }
+  *y = value;
+
+  res = hipDeviceGetAttribute(&value, hipDeviceAttributeMaxGridDimZ, device);
+  if (res != hipSuccess) {
+    LOG(ERROR) << "failed to query max grid dim z: " << ToString(res);
+    return false;
+  }
+  *z = value;
+  return true;
+}
+
+/* static */ bool GpuDriver::GetDriverVersion(int* driver_version) {
+  hipError_t res = hipDriverGetVersion(driver_version);
+  if (res != hipSuccess) {
+    LOG(ERROR) << "failed to query driver version: " << ToString(res);
+    return false;
+  }
+
+  return true;
+}
+
+/* static */ bool GpuDriver::GetDeviceProperties(
+    hipDeviceProp_t* device_properties, int device_ordinal) {
+  hipError_t res = hipGetDeviceProperties(device_properties, device_ordinal);
+  if (res != hipSuccess) {
+    LOG(ERROR) << "failed to query device properties: " << ToString(res);
+    return false;
+  }
+
+  return true;
+}
+
+/* static */ port::StatusOr<int> GpuDriver::GetDeviceAttribute(
+    hipDeviceAttribute_t attribute, hipDevice_t device) {
+  return GetSimpleAttribute<int>(device, attribute);
+}
+
+/* static */ bool GpuDriver::IsEccEnabled(hipDevice_t device, bool* result) {
+  int value = -1;
+  hipError_t res = hipSuccess;
+  // TODO(ROCm) implement this feature in HIP
+  if (res != hipSuccess) {
+    LOG(ERROR) << "failed to query ECC status: " << ToString(res);
+    return false;
+  }
+
+  *result = value;
+  return true;
+}
+
+/* static */ bool GpuDriver::GetDeviceMemoryInfo(GpuContext* context,
+                                                 int64* free_out,
+                                                 int64* total_out) {
+  ScopedActivateContext activation{context};
+  size_t free = 0;
+  size_t total = 0;
+  hipError_t res = hipMemGetInfo(&free, &total);
+  if (res != hipSuccess) {
+    LOG(ERROR) << "failed to query device memory info: " << ToString(res);
+    return false;
+  }
+
+  *free_out = free;
+  *total_out = total;
+  return true;
+}
+
+/* static */ bool GpuDriver::GetDeviceTotalMemory(hipDevice_t device,
+                                                  uint64* result) {
+  size_t value = -1;
+  hipError_t res = hipDeviceTotalMem(&value, device);
+  if (res != hipSuccess) {
+    LOG(ERROR) << "failed to query total available memory: " << ToString(res);
+    return false;
+  }
+
+  *result = value;
+  return true;
+}
+
+/* static */ string GpuDriver::GetPCIBusID(hipDevice_t device) {
+  string pci_bus_id;
+  static const int kBufferSize = 64;
+  absl::InlinedVector<char, 4> chars(kBufferSize);
+  chars[kBufferSize - 1] = '\0';
+  hipError_t res = hipDeviceGetPCIBusId(chars.begin(), kBufferSize - 1, device);
+  if (res != hipSuccess) {
+    LOG(ERROR) << "failed to query PCI bus id for device: " << ToString(res);
+    return pci_bus_id;
+  }
+  pci_bus_id = chars.begin();
+  return pci_bus_id;
+}
+
+/* static */ bool GpuDriver::CanEnablePeerAccess(GpuContext* from,
+                                                 GpuContext* to) {
+  if (from->device_ordinal() == to->device_ordinal()) {
+    return true;  // A device can always access its own memory.
+  }
+
+  int can_access_peer = -1;
+  hipError_t res = hipDeviceCanAccessPeer(
+      &can_access_peer, from->device_ordinal(), to->device_ordinal());
+  if (res != hipSuccess) {
+    LOG(ERROR) << "failed to detect peer access capability: " << ToString(res);
+    return false;
+  }
+
+  return can_access_peer;
+}
+
+/* static */ port::Status GpuDriver::EnablePeerAccess(GpuContext* from,
+                                                      GpuContext* to) {
+  if (from->device_ordinal() == to->device_ordinal()) {
+    return port::Status::OK();  // A device can always access its own memory.
+  }
+
+  ScopedActivateContext activated{from};
+  hipError_t result =
+      hipDeviceEnablePeerAccess(to->device_ordinal(), 0 /* = flags */);
+  if (result != hipSuccess && result != hipErrorPeerAccessAlreadyEnabled) {
+    return port::Status{
+        port::error::INTERNAL,
+        absl::StrFormat("failed to enable peer access from %d to %d: %s",
+                        from->device_ordinal(), to->device_ordinal(),
+                        ToString(result).c_str())};
+  }
+
+  return port::Status::OK();
+}
+
+/* static */ port::StatusOr<int> GpuDriver::GetMaxOccupiedBlocksPerCore(
+    GpuContext* context, hipFunction_t kernel, int threads_per_block,
+    size_t dynamic_shared_memory_bytes) {
+  ScopedActivateContext activation{context};
+
+  int max_blocks = 0;
+  hipError_t result = hipSuccess;
+  // TODO(ROCm) implement this feature in HIP
+  if (result != hipSuccess) {
+    return port::Status{
+        port::error::INTERNAL,
+        absl::StrFormat("failed to calculate occupancy of kernel %p: %s",
+                        kernel, ToString(result).c_str())};
+  }
+
+  return max_blocks;
+}
+
+}  // namespace gpu
+}  // namespace stream_executor
diff --git a/tensorflow/stream_executor/rocm/rocm_event.cc b/tensorflow/stream_executor/rocm/rocm_event.cc
new file mode 100644
index 0000000000..c0ffd74c17
--- /dev/null
+++ b/tensorflow/stream_executor/rocm/rocm_event.cc
@@ -0,0 +1,46 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/stream_executor/gpu/gpu_event.h"
+#include "tensorflow/stream_executor/gpu/gpu_executor.h"
+#include "tensorflow/stream_executor/gpu/gpu_stream.h"
+#include "tensorflow/stream_executor/lib/statusor.h"
+
+namespace stream_executor {
+namespace gpu {
+
+Event::Status GpuEvent::PollForStatus() {
+  port::StatusOr<hipError_t> status =
+      GpuDriver::QueryEvent(parent_->gpu_context(), gpu_event_);
+  if (!status.ok()) {
+    LOG(ERROR) << "Error polling for event status: "
+               << status.status().error_message();
+    return Event::Status::kError;
+  }
+
+  switch (status.ValueOrDie()) {
+    case hipSuccess:
+      return Event::Status::kComplete;
+    case hipErrorNotReady:
+      return Event::Status::kPending;
+    default:
+      LOG(INFO) << "Error condition returned for event status: "
+                << status.ValueOrDie();
+      return Event::Status::kError;
+  }
+}
+
+}  // namespace gpu
+}  // namespace stream_executor
diff --git a/tensorflow/stream_executor/rocm/rocm_gpu_executor.cc b/tensorflow/stream_executor/rocm/rocm_gpu_executor.cc
new file mode 100644
index 0000000000..249c7870e7
--- /dev/null
+++ b/tensorflow/stream_executor/rocm/rocm_gpu_executor.cc
@@ -0,0 +1,976 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <unistd.h>
+
+#include "absl/base/casts.h"
+#include "absl/strings/str_cat.h"
+#include "absl/strings/str_format.h"
+#include "tensorflow/stream_executor/gpu/gpu_diagnostics.h"
+#include "tensorflow/stream_executor/gpu/gpu_driver.h"
+#include "tensorflow/stream_executor/gpu/gpu_event.h"
+#include "tensorflow/stream_executor/gpu/gpu_executor.h"
+#include "tensorflow/stream_executor/gpu/gpu_stream.h"
+#include "tensorflow/stream_executor/gpu/gpu_timer.h"
+#include "tensorflow/stream_executor/kernel_cache_config.h"
+#include "tensorflow/stream_executor/lib/env.h"
+#include "tensorflow/stream_executor/lib/error.h"
+#include "tensorflow/stream_executor/lib/initialize.h"
+#include "tensorflow/stream_executor/lib/mathutil.h"
+#include "tensorflow/stream_executor/lib/numbers.h"
+#include "tensorflow/stream_executor/lib/path.h"
+#include "tensorflow/stream_executor/lib/process_state.h"
+#include "tensorflow/stream_executor/lib/ptr_util.h"
+#include "tensorflow/stream_executor/lib/statusor.h"
+#include "tensorflow/stream_executor/lib/str_util.h"
+#include "tensorflow/stream_executor/lib/stringprintf.h"
+#include "tensorflow/stream_executor/platform.h"
+#include "tensorflow/stream_executor/platform/dso_loader.h"
+#include "tensorflow/stream_executor/platform/logging.h"
+#include "tensorflow/stream_executor/platform/port.h"
+#include "tensorflow/stream_executor/plugin_registry.h"
+#include "tensorflow/stream_executor/rocm/rocm_platform_id.h"
+#include "tensorflow/stream_executor/stream.h"
+#include "tensorflow/stream_executor/stream_executor_internal.h"
+#include "tensorflow/stream_executor/stream_executor_pimpl.h"
+#include "tensorflow/stream_executor/timer.h"
+
+#ifdef PLATFORMS_GPUS_ROCM_DYNAMIC_LIBROCM_DYNAMIC_LIBROCM_H_
+#error \
+    "No driver calls in this file, wrap driver functionality in rocm_driver.cc."
+#endif
+
+#ifdef __ROCM_RUNTIME_H__
+#error \
+    "ROCM runtime being included into ROCM GPU executor; should be driver only."
+#endif
+
+namespace stream_executor {
+namespace gpu {
+
+static GpuEvent* AsGpuEvent(Event* event) {
+  DCHECK(event != nullptr);
+  return static_cast<GpuEvent*>(event->implementation());
+}
+
+// Given a platform-independent timer datatype, returns the internal ROCM
+// platform implementation pointer.
+static GpuTimer* AsGpuTimer(Timer* timer) {
+  DCHECK(timer != nullptr);
+  return static_cast<GpuTimer*>(timer->implementation());
+}
+
+// Given const GPU memory, returns a librocm device pointer datatype, suitable
+// for passing directly to librocm APIs.
+//
+// N.B. we must lose constness in order to pass a suitable type to the existing
+// librocm APIs, so the caller should take care to only pass the result of const
+// GPU memory conversions to librocm functions which will honor constness.
+static hipDeviceptr_t AsROCmDevicePtr(const DeviceMemoryBase& gpu_mem) {
+  return const_cast<hipDeviceptr_t>(gpu_mem.opaque());
+}
+
+// See description on const version above.
+static hipDeviceptr_t AsROCmDevicePtr(DeviceMemoryBase* gpu_mem) {
+  return AsROCmDevicePtr(*gpu_mem);
+}
+
+static GpuContext* GetGpuContext(Stream* stream) {
+  return static_cast<GpuExecutor*>(stream->parent()->implementation())
+      ->gpu_context();
+}
+
+GpuContext* ExtractGpuContext(GpuExecutor* rocm_exec) {
+  CHECK(rocm_exec != nullptr);
+  return rocm_exec->gpu_context();
+}
+
+GpuExecutor* ExtractGpuExecutor(StreamExecutor* stream_exec) {
+  return static_cast<GpuExecutor*>(stream_exec->implementation());
+}
+
+GpuExecutor::~GpuExecutor() {
+  for (auto& it : disk_modules_) {
+    GpuDriver::UnloadModule(context_, it.second);
+  }
+  for (auto& it : in_memory_modules_) {
+    GpuDriver::UnloadModule(context_, it.second);
+  }
+  if (context_ != nullptr) {
+    GpuDriver::DestroyContext(context_);
+  }
+  CHECK(gpu_binary_to_module_.empty()) << "GpuExecutor has loaded modules.";
+}
+bool GpuExecutor::UnloadModule(ModuleHandle module_handle) {
+  const char* gpu_binary = reinterpret_cast<const char*>(module_handle.id());
+  mutex_lock lock{in_memory_modules_mu_};
+  return UnloadGpuBinary(gpu_binary);
+}
+
+bool GpuExecutor::UnloadGpuBinary(const void* gpu_binary) {
+  auto module_it = gpu_binary_to_module_.find(gpu_binary);
+  if (gpu_binary_to_module_.end() == module_it) {
+    VLOG(3) << "No loaded  HSACO module for " << gpu_binary;
+    return false;
+  }
+  auto& module = module_it->second.first;
+  auto& refcount = module_it->second.second;
+  VLOG(3) << "Found HSACO module " << module << " with refcount " << refcount;
+  if (--refcount == 0) {
+    VLOG(3) << "Unloading  HSACO module " << module;
+    GpuDriver::UnloadModule(context_, module);
+    gpu_binary_to_module_.erase(module_it);
+  }
+  return true;
+}
+
+void GpuExecutor::UnloadKernel(const KernelBase* kernel) {
+  LOG(FATAL) << "Feature not supported on ROCM platform (UnloadKernel)";
+}
+
+port::Status GpuExecutor::Init(int device_ordinal,
+                               DeviceOptions device_options) {
+  device_ordinal_ = device_ordinal;
+
+  auto status = GpuDriver::Init();
+  if (!status.ok()) {
+    return status;
+  }
+
+  status = GpuDriver::GetDevice(device_ordinal_, &device_);
+  if (!status.ok()) {
+    return status;
+  }
+
+  status = GpuDriver::CreateContext(device_ordinal_, device_, device_options,
+                                    &context_);
+  if (!status.ok()) {
+    return status;
+  }
+
+  return GpuDriver::GetGpuISAVersion(&version_, device_);
+}
+
+bool GpuExecutor::FindOnDiskForComputeCapability(
+    absl::string_view filename, absl::string_view canonical_suffix,
+    string* found_filename) const {
+  LOG(FATAL) << "Feature not supported on ROCM platform "
+                "(FindOnDiskForComputeCapability)";
+  return false;
+}
+
+bool GpuExecutor::FindOnDiskForISAVersion(absl::string_view filename,
+                                          absl::string_view canonical_suffix,
+                                          string* found_filename) const {
+  if (version_ == 0) {
+    return false;
+  }
+
+  string cc_specific =
+      absl::StrCat(filename, ".cc", version_, canonical_suffix);
+  if (port::FileExists(cc_specific).ok()) {
+    VLOG(2) << "found AMDGPU ISA version-specific file, using that: "
+            << cc_specific;
+    *found_filename = cc_specific;
+    return true;
+  }
+
+  VLOG(2) << "could not find AMDGPU ISA version-specific file at: "
+          << cc_specific;
+  if (port::FileExists(string(filename)).ok()) {
+    *found_filename = string(filename);
+    return true;
+  }
+
+  return false;
+}
+
+// Returns the path to the running executable.
+// N.B. Derived from //knowledge/smalltalk/background_kb.cc
+// Arg: strip_exe: if true, remove the name of the executable itself from the
+//                 returned string. Example: calling this from /usr/bin/foo
+//                 would return /usr/bin.
+static string GetBinaryDir(bool strip_exe) {
+  char exe_path[PATH_MAX] = {0};
+  CHECK_ERR(readlink("/proc/self/exe", exe_path, sizeof(exe_path) - 1));
+  // Make sure it's null-terminated:
+  exe_path[sizeof(exe_path) - 1] = 0;
+
+  if (strip_exe) {
+    // The exe is the last component of the path, so remove one component.
+    string ret = exe_path;
+    std::vector<string> components = port::Split(exe_path, '/');
+    components.pop_back();
+    return port::Join(components, "/");
+  }
+  return exe_path;
+}
+
+bool GpuExecutor::GetKernel(const MultiKernelLoaderSpec& spec,
+                            KernelBase* kernel) {
+  GpuKernel* rocm_kernel = AsGpuKernel(kernel);
+  hipModule_t module = nullptr;
+  const string* kernelname;
+
+  const OnDiskKernelLoaderSpec* on_disk_spec = nullptr;
+  bool has_cubin = spec.has_cuda_cubin_on_disk();
+  if (has_cubin) {
+    on_disk_spec = &spec.cuda_cubin_on_disk();
+  }
+
+  if (on_disk_spec != nullptr) {
+    LOG(WARNING) << "loading ROCM kernel from disk is not supported";
+    return false;
+  } else if (spec.has_cuda_cubin_in_memory()) {
+    kernelname = &spec.cuda_cubin_in_memory().kernelname();
+
+    const char* hsaco = spec.cuda_cubin_in_memory().bytes();
+    mutex_lock lock{in_memory_modules_mu_};
+    module = in_memory_modules_[hsaco];
+
+    if (module == nullptr) {
+      if (!GpuDriver::LoadHsaco(context_, hsaco, &module)) {
+        LOG(ERROR) << "failed to load HSACO\n";
+        return false;
+      }
+      in_memory_modules_[hsaco] = module;
+    }
+  } else {
+    LOG(WARNING) << "no method of loading ROCM kernel provided";
+    return false;
+  }
+
+  VLOG(2) << "getting function " << *kernelname << " from module " << module;
+  if (!GpuDriver::GetModuleFunction(context_, module, kernelname->c_str(),
+                                    rocm_kernel->gpu_function_ptr())) {
+    return false;
+  }
+
+  // We have to trust the kernel loader spec arity because there doesn't appear
+  // to be a way to reflect on the number of expected arguments w/the ROCM API.
+  rocm_kernel->set_arity(spec.arity());
+
+  KernelMetadata kernel_metadata;
+  if (!GetKernelMetadata(rocm_kernel, &kernel_metadata)) {
+    LOG(WARNING) << "Unable to get metadata for kernel " << kernelname;
+  }
+  kernel->set_metadata(kernel_metadata);
+  kernel->set_name(*kernelname);
+  return true;
+}
+
+bool GpuExecutor::GetKernelMetadata(GpuKernel* rocm_kernel,
+                                    KernelMetadata* kernel_metadata) {
+  int value = 0;
+  // TODO(ROCm) implement this feature in HIP
+  kernel_metadata->set_registers_per_thread(value);
+
+  // TODO(ROCm) implement this feature in HIP
+  kernel_metadata->set_shared_memory_bytes(value);
+
+  return true;
+}
+
+bool GpuExecutor::Launch(Stream* stream, const ThreadDim& thread_dims,
+                         const BlockDim& block_dims, const KernelBase& kernel,
+                         const KernelArgsArrayBase& args) {
+  CHECK_EQ(kernel.Arity(), args.number_of_arguments());
+  GpuStreamHandle hipstream = AsGpuStreamValue(stream);
+  const GpuKernel* rocm_kernel = AsGpuKernel(&kernel);
+  hipFunction_t hipfunc = rocm_kernel->AsGpuFunctionHandle();
+
+  // Only perform/print the occupancy check once.  Even just checking to see
+  // whether we've done an occupancy check on this kernel before isn't free
+  // (because we have to synchronize), so we only do this at -v 2+.
+  if (VLOG_IS_ON(2)) {
+    mutex_lock lock(launched_kernels_mu_);
+    if (!launched_kernels_.count(hipfunc)) {
+      VlogOccupancyInfo(kernel, thread_dims, block_dims);
+      // TODO(rspringer): Remove elements from launched_kernels_...if we ever
+      // expose a kernel/module deallocation method.
+      launched_kernels_.insert(hipfunc);
+    }
+  }
+
+  if (rocm_kernel->GetPreferredCacheConfig() !=
+      KernelCacheConfig::kNoPreference) {
+    GpuDriver::FuncSetCacheConfig(hipfunc, rocm_kernel->GetGpuCacheConfig());
+  }
+
+  // prepare kernargs
+  // KernelArgsArrayBase keeps the pointer of arguments
+  // deference them here
+  std::vector<void*> kernargs;
+  KernelArgIterator iter = args.arg_iterator();
+  while (iter.has_next()) {
+    KernelArg arg = iter.next();
+    VLOG(2) << "*(arg.address): "
+            << reinterpret_cast<void*>(
+                   *static_cast<const uint64_t*>(arg.address));
+    kernargs.push_back(
+        reinterpret_cast<void*>(*static_cast<const uint64_t*>(arg.address)));
+  }
+
+  size_t size = sizeof(void*) * kernargs.size();
+  void* config[] = {HIP_LAUNCH_PARAM_BUFFER_POINTER, kernargs.data(),
+                    HIP_LAUNCH_PARAM_BUFFER_SIZE, &size, HIP_LAUNCH_PARAM_END};
+
+  if (!GpuDriver::LaunchKernel(
+          GetGpuContext(stream), hipfunc, block_dims.x, block_dims.y,
+          block_dims.z, thread_dims.x, thread_dims.y, thread_dims.z,
+          args.number_of_shared_bytes(), hipstream, nullptr, (void**)&config)) {
+    LOG(ERROR) << "failed to launch ROCM kernel with args: "
+               << args.number_of_arguments()
+               << "; thread dim: " << thread_dims.ToString()
+               << "; block dim: " << block_dims.ToString();
+    return false;
+  }
+
+  return true;
+}
+
+int GpuExecutor::CalculateOccupancy(const DeviceDescription& device_description,
+                                    uint64 registers_per_thread,
+                                    uint64 shared_memory_per_block,
+                                    const ThreadDim& thread_dims,
+                                    GpuFunctionHandle func) {
+  LOG(FATAL) << "Feature not supported on ROCM platform (CalculateOccupancy)";
+  return 0;
+}
+
+int GpuExecutor::CompareOccupancy(int* initial_blocks,
+                                  const DeviceDescription& device_description,
+                                  uint64 registers_per_thread,
+                                  uint64 shared_memory_per_block,
+                                  const ThreadDim& thread_dims,
+                                  GpuFunctionHandle func) {
+  LOG(FATAL) << "Feature not supported on ROCM platform (CompareOccupancy)";
+  return 0;
+}
+
+bool GpuExecutor::LoadModule(const MultiModuleLoaderSpec& spec,
+                             ModuleHandle* module_handle) {
+  // In GpuExecutor we store the pointer to the  HSACO binary  as
+  // ModuleHandle::id().
+  hipModule_t hip_module = nullptr;
+  // TODO(ROCm): Need  generic term instead of cubin/cuda/ptx
+  if (spec.has_cuda_cubin_in_memory()) {
+    mutex_lock lock{in_memory_modules_mu_};
+    if (!LoadModuleFromHsaco(
+            reinterpret_cast<const char*>(spec.cuda_cubin_in_memory().data()),
+            &hip_module)) {
+      return false;
+    }
+    *module_handle = ModuleHandle(const_cast<void*>(
+        static_cast<const void*>(spec.cuda_cubin_in_memory().data())));
+    return true;
+  } else {
+    LOG(ERROR) << "No HSACO binary found \n";
+    return false;
+  }
+}
+
+bool GpuExecutor::LoadModuleFromCuBin(const char* cubin, hipModule_t* module) {
+  LOG(FATAL) << "Feature not supported on ROCM platform (LoadModuleFromCuBin)";
+  return false;
+}
+
+bool GpuExecutor::LoadModuleFromPtx(const char* ptx, hipModule_t* module) {
+  LOG(FATAL) << "Feature not supported on ROCM platform (LoadModuleFromPtx)";
+  return false;
+}
+
+bool GpuExecutor::LoadModuleFromHsaco(const char* hsaco, hipModule_t* module) {
+  uint64_t module_refcount;
+  std::tie(*module, module_refcount) = gpu_binary_to_module_[hsaco];
+
+  if (*module == nullptr) {
+    if (!GpuDriver::LoadHsaco(context_, hsaco, module)) {
+      LOG(ERROR) << "failed to load : HSACO \n";
+      return false;
+    }
+    module_refcount = 1;
+    VLOG(3) << "Loaded HSACO " << static_cast<const void*>(hsaco)
+            << " as module " << *module;
+  } else {
+    ++module_refcount;
+    VLOG(3) << "HSACO " << static_cast<const void*>(hsaco)
+            << " is already loaded as module " << *module;
+  }
+  gpu_binary_to_module_[hsaco] = {*module, module_refcount};
+  return true;
+}
+
+// This is a non-essential operation; if there's a failure, proceed without
+// logging an error. It's nearly certain that in case of failures, we'd never
+// get here in the first place; these are very low-impact routines.
+void GpuExecutor::VlogOccupancyInfo(const KernelBase& kernel,
+                                    const ThreadDim& thread_dims,
+                                    const BlockDim& block_dims) {
+  // TODO(ROCm) implement this feature in HIP
+}
+
+void* GpuExecutor::Allocate(uint64 size) {
+  return GpuDriver::DeviceAllocate(context_, size);
+}
+
+void* GpuExecutor::AllocateSubBuffer(DeviceMemoryBase* mem, uint64 offset_bytes,
+                                     uint64 size_bytes) {
+  // offset and size are in bytes, so char* works as the pointer type.
+  return reinterpret_cast<char*>(mem->opaque()) + offset_bytes;
+}
+
+void GpuExecutor::Deallocate(DeviceMemoryBase* mem) {
+  // ROCM "sub-buffers" are just pointer + offset, so no dealloc is necessary.
+  if (!mem->is_sub_buffer()) {
+    GpuDriver::DeviceDeallocate(context_, mem->opaque());
+  }
+}
+
+bool GpuExecutor::HostMemoryRegister(void* location, uint64 size) {
+  if (location == nullptr || size == 0) {
+    LOG(WARNING) << "attempting to register null or zero-sized memory: "
+                 << location << "; size " << size;
+  }
+  VLOG(2) << "registering " << location << " size " << size;
+  return GpuDriver::HostRegister(context_, location, size);
+}
+
+bool GpuExecutor::HostMemoryUnregister(void* location) {
+  VLOG(2) << "unregistering " << location;
+  return GpuDriver::HostUnregister(context_, location);
+}
+
+bool GpuExecutor::SynchronizeAllActivity() {
+  return GpuDriver::SynchronizeContext(context_);
+}
+
+bool GpuExecutor::SynchronousMemZero(DeviceMemoryBase* location, uint64 size) {
+  if (reinterpret_cast<uintptr_t>(location->opaque()) % 4 == 0 &&
+      size % 4 == 0) {
+    return GpuDriver::SynchronousMemsetUint32(
+        context_, AsROCmDevicePtr(location), 0x0, size / 4);
+  }
+  return GpuDriver::SynchronousMemsetUint8(context_, AsROCmDevicePtr(location),
+                                           0x0, size);
+}
+
+bool GpuExecutor::SynchronousMemSet(DeviceMemoryBase* location, int value,
+                                    uint64 size) {
+  if (reinterpret_cast<uintptr_t>(location->opaque()) % 4 == 0 &&
+      size % 4 == 0) {
+    // hipMemset reinterprets "value" as a uint8.
+    uint8 byte_value = static_cast<uint8>(value);
+    uint32 pattern = (byte_value << 24) | (byte_value << 16) |
+                     (byte_value << 8) | byte_value;
+    return GpuDriver::SynchronousMemsetUint32(
+        context_, AsROCmDevicePtr(location), pattern, size / 4);
+  }
+  return GpuDriver::SynchronousMemsetUint8(context_, AsROCmDevicePtr(location),
+                                           value, size);
+}
+
+port::Status GpuExecutor::SynchronousMemcpy(DeviceMemoryBase* gpu_dst,
+                                            const void* host_src, uint64 size) {
+  return GpuDriver::SynchronousMemcpyH2D(context_, AsROCmDevicePtr(gpu_dst),
+                                         host_src, size);
+}
+
+port::Status GpuExecutor::SynchronousMemcpy(void* host_dst,
+                                            const DeviceMemoryBase& gpu_src,
+                                            uint64 size) {
+  return GpuDriver::SynchronousMemcpyD2H(context_, host_dst,
+                                         AsROCmDevicePtr(gpu_src), size);
+}
+
+port::Status GpuExecutor::SynchronousMemcpyDeviceToDevice(
+    DeviceMemoryBase* gpu_dst, const DeviceMemoryBase& gpu_src, uint64 size) {
+  return GpuDriver::SynchronousMemcpyD2D(context_, AsROCmDevicePtr(gpu_dst),
+                                         AsROCmDevicePtr(gpu_src), size);
+}
+
+bool GpuExecutor::MemZero(Stream* stream, DeviceMemoryBase* location,
+                          uint64 size) {
+  if (reinterpret_cast<uintptr_t>(location->opaque()) % 4 == 0 &&
+      size % 4 == 0) {
+    return Memset32(stream, location, 0x0, size);
+  } else {
+    return Memset(stream, location, 0x0, size);
+  }
+}
+
+bool GpuExecutor::Memset(Stream* stream, DeviceMemoryBase* location,
+                         uint8 pattern, uint64 size) {
+  VLOG(2) << "enqueueing memset8 operation onto stream " << stream
+          << " at location " << location << " with size " << size
+          << " and pattern " << std::hex << pattern;
+  return GpuDriver::AsynchronousMemsetUint8(context_, AsROCmDevicePtr(location),
+                                            pattern, size,
+                                            AsGpuStreamValue(stream));
+}
+
+bool GpuExecutor::Memset32(Stream* stream, DeviceMemoryBase* location,
+                           uint32 pattern, uint64 size) {
+  VLOG(2) << "enqueueing memset32 operation onto stream " << stream
+          << " at location " << location << " with size " << size
+          << " and pattern " << std::hex << pattern;
+  CHECK(reinterpret_cast<uintptr_t>(location->opaque()) % 4 == 0 &&
+        size % 4 == 0);
+  return GpuDriver::AsynchronousMemsetUint32(
+      context_, AsROCmDevicePtr(location), pattern, size / 4,
+      AsGpuStreamValue(stream));
+}
+
+bool GpuExecutor::Memcpy(Stream* stream, void* host_dst,
+                         const DeviceMemoryBase& gpu_src, uint64 size) {
+  return GpuDriver::AsynchronousMemcpyD2H(context_, host_dst,
+                                          AsROCmDevicePtr(gpu_src), size,
+                                          AsGpuStreamValue(stream));
+}
+
+bool GpuExecutor::Memcpy(Stream* stream, DeviceMemoryBase* gpu_dst,
+                         const void* host_src, uint64 size) {
+  return GpuDriver::AsynchronousMemcpyH2D(context_, AsROCmDevicePtr(gpu_dst),
+                                          host_src, size,
+                                          AsGpuStreamValue(stream));
+}
+
+bool GpuExecutor::MemcpyDeviceToDevice(Stream* stream,
+                                       DeviceMemoryBase* gpu_dst,
+                                       const DeviceMemoryBase& gpu_src,
+                                       uint64 size) {
+  return GpuDriver::AsynchronousMemcpyD2D(context_, AsROCmDevicePtr(gpu_dst),
+                                          AsROCmDevicePtr(gpu_src), size,
+                                          AsGpuStreamValue(stream));
+}
+
+bool GpuExecutor::HostCallback(Stream* stream,
+                               std::function<port::Status()> callback) {
+  auto callback_ptr = new std::function<void()>([callback]() {
+    port::Status s = callback();
+    if (!s.ok()) {
+      LOG(WARNING) << "Host callback failed: " << s;
+    }
+  });
+  return GpuDriver::AddStreamCallback(context_, AsGpuStreamValue(stream),
+                                      InternalHostCallback, callback_ptr);
+}
+
+/* static */ void GpuExecutor::InternalHostCallback(GpuStreamHandle stream,
+                                                    hipError_t status,
+                                                    void* data) {
+  std::function<void()>* callback =
+      reinterpret_cast<std::function<void()>*>(data);
+  (*callback)();
+  delete callback;
+}
+
+port::Status GpuExecutor::AllocateEvent(Event* event) {
+  return AsGpuEvent(event)->Init();
+}
+
+port::Status GpuExecutor::DeallocateEvent(Event* event) {
+  return AsGpuEvent(event)->Destroy();
+}
+
+port::Status GpuExecutor::RecordEvent(Stream* stream, Event* event) {
+  return AsGpuEvent(event)->Record(AsGpuStream(stream));
+}
+
+port::Status GpuExecutor::WaitForEvent(Stream* stream, Event* event) {
+  if (GpuDriver::WaitStreamOnEvent(context_, AsGpuStream(stream)->gpu_stream(),
+                                   AsGpuEvent(event)->gpu_event())) {
+    return port::Status::OK();
+  } else {
+    return port::Status{
+        port::error::INTERNAL,
+        absl::StrFormat("error recording waiting for ROCM event on stream %p",
+                        stream)};
+  }
+}
+
+Event::Status GpuExecutor::PollForEventStatus(Event* event) {
+  return AsGpuEvent(event)->PollForStatus();
+}
+
+bool GpuExecutor::AllocateStream(Stream* stream) {
+  return AsGpuStream(stream)->Init();
+}
+
+void GpuExecutor::DeallocateStream(Stream* stream) {
+  GpuStream* rocm_stream = AsGpuStream(stream);
+  if (!rocm_stream->IsIdle()) {
+    LOG(ERROR) << "Deallocating stream with pending work";
+  }
+  rocm_stream->Destroy();
+}
+
+bool GpuExecutor::AllocateTimer(Timer* timer) {
+  return AsGpuTimer(timer)->Init();
+}
+
+void GpuExecutor::DeallocateTimer(Timer* timer) {
+  AsGpuTimer(timer)->Destroy();
+}
+
+bool GpuExecutor::CreateStreamDependency(Stream* dependent, Stream* other) {
+  GpuEventHandle other_completed_event = *AsGpuStream(other)->completed_event();
+  bool ok = GpuDriver::RecordEvent(context_, other_completed_event,
+                                   AsGpuStreamValue(other))
+                .ok();
+  if (!ok) {
+    LOG(ERROR) << "failed to record completion event; "
+                  "therefore, failed to create inter-stream dependency";
+    return false;
+  }
+
+  return GpuDriver::WaitStreamOnEvent(context_, AsGpuStreamValue(dependent),
+                                      other_completed_event);
+}
+
+bool GpuExecutor::StartTimer(Stream* stream, Timer* timer) {
+  return AsGpuTimer(timer)->Start(AsGpuStream(stream));
+}
+
+bool GpuExecutor::StopTimer(Stream* stream, Timer* timer) {
+  return AsGpuTimer(timer)->Stop(AsGpuStream(stream));
+}
+
+port::Status GpuExecutor::BlockHostUntilDone(Stream* stream) {
+  return GpuDriver::SynchronizeStream(context_, AsGpuStreamValue(stream));
+}
+
+blas::BlasSupport* GpuExecutor::CreateBlas() {
+  PluginRegistry* registry = PluginRegistry::Instance();
+  port::StatusOr<PluginRegistry::BlasFactory> status =
+      registry->GetFactory<PluginRegistry::BlasFactory>(kROCmPlatformId,
+                                                        plugin_config_.blas());
+  if (!status.ok()) {
+    LOG(ERROR) << "Unable to retrieve BLAS factory: "
+               << status.status().error_message();
+    return nullptr;
+  }
+
+  return status.ValueOrDie()(this);
+}
+
+dnn::DnnSupport* GpuExecutor::CreateDnn() {
+  PluginRegistry* registry = PluginRegistry::Instance();
+  port::StatusOr<PluginRegistry::DnnFactory> status =
+      registry->GetFactory<PluginRegistry::DnnFactory>(kROCmPlatformId,
+                                                       plugin_config_.dnn());
+  if (!status.ok()) {
+    LOG(ERROR) << "Unable to retrieve DNN factory: "
+               << status.status().error_message();
+    return nullptr;
+  }
+
+  return status.ValueOrDie()(this);
+}
+
+fft::FftSupport* GpuExecutor::CreateFft() {
+  PluginRegistry* registry = PluginRegistry::Instance();
+  port::StatusOr<PluginRegistry::FftFactory> status =
+      registry->GetFactory<PluginRegistry::FftFactory>(kROCmPlatformId,
+                                                       plugin_config_.fft());
+  if (!status.ok()) {
+    LOG(ERROR) << "Unable to retrieve FFT factory: "
+               << status.status().error_message();
+    return nullptr;
+  }
+
+  return status.ValueOrDie()(this);
+}
+
+rng::RngSupport* GpuExecutor::CreateRng() {
+  PluginRegistry* registry = PluginRegistry::Instance();
+  port::StatusOr<PluginRegistry::RngFactory> status =
+      registry->GetFactory<PluginRegistry::RngFactory>(kROCmPlatformId,
+                                                       plugin_config_.rng());
+  if (!status.ok()) {
+    LOG(ERROR) << "Unable to retrieve RNG factory: "
+               << status.status().error_message();
+    return nullptr;
+  }
+
+  return status.ValueOrDie()(this);
+}
+
+// TODO(rspringer): Remove in b/18544742.
+bool GpuExecutor::SupportsDnn() const { return true; }
+
+bool GpuExecutor::CanEnablePeerAccessTo(StreamExecutorInterface* other) {
+  GpuExecutor* rocm_other = static_cast<GpuExecutor*>(other);
+  return GpuDriver::CanEnablePeerAccess(context_, rocm_other->context_);
+}
+
+port::Status GpuExecutor::EnablePeerAccessTo(StreamExecutorInterface* other) {
+  GpuExecutor* rocm_other = static_cast<GpuExecutor*>(other);
+  return GpuDriver::EnablePeerAccess(context_, rocm_other->context_);
+}
+
+SharedMemoryConfig GpuExecutor::GetDeviceSharedMemoryConfig() {
+  port::StatusOr<hipSharedMemConfig> rocm_config =
+      GpuDriver::ContextGetSharedMemConfig(context_);
+  if (!rocm_config.ok()) {
+    // Don't log; the failed call will log necessary output.
+    return SharedMemoryConfig::kDefault;
+  }
+
+  switch (rocm_config.ValueOrDie()) {
+    case hipSharedMemBankSizeDefault:
+      return SharedMemoryConfig::kDefault;
+    case hipSharedMemBankSizeFourByte:
+      return SharedMemoryConfig::kFourByte;
+    case hipSharedMemBankSizeEightByte:
+      return SharedMemoryConfig::kEightByte;
+    default:
+      LOG(FATAL) << "Invalid shared memory configuration returned: "
+                 << rocm_config.ValueOrDie();
+  }
+}
+
+port::Status GpuExecutor::SetDeviceSharedMemoryConfig(
+    SharedMemoryConfig config) {
+  hipSharedMemConfig rocm_config;
+  switch (config) {
+    case SharedMemoryConfig::kDefault:
+      rocm_config = hipSharedMemBankSizeDefault;
+      break;
+    case SharedMemoryConfig::kFourByte:
+      rocm_config = hipSharedMemBankSizeFourByte;
+      break;
+    case SharedMemoryConfig::kEightByte:
+      rocm_config = hipSharedMemBankSizeEightByte;
+      break;
+    default:
+      LOG(FATAL) << "Invalid shared memory configuration specified: "
+                 << static_cast<int>(config);
+  }
+  return GpuDriver::ContextSetSharedMemConfig(context_, rocm_config);
+}
+
+bool GpuExecutor::DeviceMemoryUsage(int64* free, int64* total) const {
+  return GpuDriver::GetDeviceMemoryInfo(context_, free, total);
+}
+
+bool GpuExecutor::GetSymbol(const string& symbol_name,
+                            ModuleHandle module_handle, void** mem,
+                            size_t* bytes) {
+  {  // give limited scope to mutex_lock
+    mutex_lock lock{disk_modules_mu_};
+    for (auto& it : disk_modules_) {
+      if (GpuDriver::GetModuleSymbol(context_, it.second, symbol_name.c_str(),
+                                     reinterpret_cast<hipDeviceptr_t*>(mem),
+                                     bytes)) {
+        return true;
+      }
+    }
+  }
+
+  {  // give limited scope to mutex_lock
+    mutex_lock lock{in_memory_modules_mu_};
+    for (auto& it : in_memory_modules_) {
+      if (GpuDriver::GetModuleSymbol(context_, it.second, symbol_name.c_str(),
+                                     reinterpret_cast<hipDeviceptr_t*>(mem),
+                                     bytes)) {
+        return true;
+      }
+    }
+  }
+
+  {  // give limited scope to mutex_lock
+    mutex_lock lock{in_memory_modules_mu_};
+    if (static_cast<bool>(module_handle)) {
+      auto it = gpu_binary_to_module_.find(module_handle.id());
+      CHECK(it != gpu_binary_to_module_.end());
+      if (GpuDriver::GetModuleSymbol(
+              context_, it->second.first, symbol_name.c_str(),
+              reinterpret_cast<hipDeviceptr_t*>(mem), bytes)) {
+        return true;
+      }
+    }
+
+    for (auto& it : gpu_binary_to_module_) {
+      if (GpuDriver::GetModuleSymbol(
+              context_, it.second.first, symbol_name.c_str(),
+              reinterpret_cast<hipDeviceptr_t*>(mem), bytes)) {
+        return true;
+      }
+    }
+  }
+
+  LOG(INFO) << "Falied to find symbol in any modules: " << symbol_name;
+  return false;
+}
+
+bool GpuExecutor::FillBlockDimLimit(BlockDim* block_dim_limit) const {
+  // The BlockDim name is a mismatch against these GRID_DIM_* queries because
+  // we use BlockDims to express the dimensions of blocks within a grid
+  // (as opposed to ThreadDim which expresses the dimensions of threads
+  // within a block).
+  int x, y, z;
+  if (!GpuDriver::GetGridLimits(&x, &y, &z, device_)) {
+    return false;
+  }
+
+  block_dim_limit->x = x;
+  block_dim_limit->y = y;
+  block_dim_limit->z = z;
+  return true;
+}
+
+bool GpuExecutor::SupportsBlas() const { return true; }
+
+bool GpuExecutor::SupportsFft() const { return true; }
+
+bool GpuExecutor::SupportsRng() const { return true; }
+
+std::unique_ptr<internal::EventInterface>
+GpuExecutor::CreateEventImplementation() {
+  return std::unique_ptr<internal::EventInterface>(new GpuEvent(this));
+}
+
+std::unique_ptr<internal::KernelInterface>
+GpuExecutor::CreateKernelImplementation() {
+  return std::unique_ptr<internal::KernelInterface>(new GpuKernel());
+}
+
+std::unique_ptr<internal::StreamInterface>
+GpuExecutor::GetStreamImplementation() {
+  return std::unique_ptr<internal::StreamInterface>(new GpuStream(this));
+}
+
+std::unique_ptr<internal::TimerInterface>
+GpuExecutor::GetTimerImplementation() {
+  return std::unique_ptr<internal::TimerInterface>(new GpuTimer(this));
+}
+
+void* GpuExecutor::GpuContextHack() { return context_; }
+
+GpuContext* GpuExecutor::gpu_context() { return context_; }
+
+// Attempts to read the NUMA node corresponding to the GPU device's PCI bus out
+// of SysFS. Returns -1 if it cannot.
+//
+// For anything more complicated/prod-focused than this, you'll likely want to
+// turn to gsys' topology modeling.
+static int TryToReadNumaNode(const string& pci_bus_id, int device_ordinal) {
+  // TODO(ROCm) implement this feature in HIP
+  return 1;
+}
+
+DeviceDescription* GpuExecutor::PopulateDeviceDescription() const {
+  internal::DeviceDescriptionBuilder builder;
+
+  {
+    int driver_version = 0;
+    (void)GpuDriver::GetDriverVersion(&driver_version);
+    string augmented_driver_version = absl::StrFormat(
+        "%d (%s)", driver_version,
+        DriverVersionStatusToString(Diagnostician::FindDsoVersion()).c_str());
+    builder.set_driver_version(augmented_driver_version);
+  }
+
+  {
+    string pci_bus_id = GpuDriver::GetPCIBusID(device_);
+
+    // Lower the hex characters to match sysfs.
+    pci_bus_id = port::Lowercase(pci_bus_id);
+    builder.set_pci_bus_id(pci_bus_id);
+
+    // Read the NUMA node corresponding to the PCI bus ID out of sysfs.
+    int numa_node = TryToReadNumaNode(pci_bus_id, device_ordinal_);
+    builder.set_numa_node(numa_node);
+  }
+
+  hipDeviceProp_t prop;
+  if (GpuDriver::GetDeviceProperties(&prop, device_ordinal_)) {
+    builder.set_threads_per_block_limit(prop.maxThreadsPerBlock);
+
+    ThreadDim thread_dim_limit;
+    thread_dim_limit.x = prop.maxThreadsDim[0];
+    thread_dim_limit.y = prop.maxThreadsDim[1];
+    thread_dim_limit.z = prop.maxThreadsDim[2];
+    builder.set_thread_dim_limit(thread_dim_limit);
+
+    float clock_rate_ghz = static_cast<float>(prop.clockRate) / 1e6;
+    builder.set_clock_rate_ghz(clock_rate_ghz);
+  }
+
+  {
+    bool ecc_enabled = false;
+    (void)GpuDriver::IsEccEnabled(device_, &ecc_enabled);
+    builder.set_ecc_enabled(ecc_enabled);
+  }
+
+  {
+    uint64 device_memory_size = -1;
+    (void)GpuDriver::GetDeviceTotalMemory(device_, &device_memory_size);
+    builder.set_device_memory_size(device_memory_size);
+  }
+
+  {
+    BlockDim block_dim_limit;
+    FillBlockDimLimit(&block_dim_limit);
+    builder.set_block_dim_limit(block_dim_limit);
+  }
+
+  {
+    string device_name;
+    (void)GpuDriver::GetDeviceName(device_, &device_name);
+    builder.set_name(device_name);
+  }
+
+  builder.set_platform_version(
+      absl::StrCat("AMDGPU ISA version: gfx", version_));
+
+  // TODO(leary) should be a way to query this from the driver, but this is
+  // unlikely to change for us any time soon.
+  builder.set_device_address_bits(64);
+
+  builder.set_device_vendor("Advanced Micro Devices, Inc");
+  builder.set_rocm_amdgpu_isa_version(version_);
+  builder.set_shared_memory_per_core(
+      GpuDriver::GetMaxSharedMemoryPerCore(device_).ValueOrDie());
+  builder.set_shared_memory_per_block(
+      GpuDriver::GetMaxSharedMemoryPerBlock(device_).ValueOrDie());
+  builder.set_core_count(
+      GpuDriver::GetMultiprocessorCount(device_).ValueOrDie());
+  builder.set_threads_per_core_limit(
+      GpuDriver::GetMaxThreadsPerMultiprocessor(device_).ValueOrDie());
+  builder.set_registers_per_block_limit(
+      GpuDriver::GetMaxRegistersPerBlock(device_).ValueOrDie());
+  builder.set_threads_per_warp(
+      GpuDriver::GetThreadsPerWarp(device_).ValueOrDie());
+  builder.set_registers_per_core_limit(64 * 1024);
+
+  auto built = builder.Build();
+  return built.release();
+}
+
+}  // namespace gpu
+
+void initialize_rocm_gpu_executor() {
+  *internal::MakeROCMExecutorImplementation() = [](const PluginConfig& config) {
+    return new gpu::GpuExecutor{config};
+  };
+}
+
+}  // namespace stream_executor
+
+REGISTER_MODULE_INITIALIZER(rocm_gpu_executor, {
+  stream_executor::initialize_rocm_gpu_executor();
+});
diff --git a/tensorflow/stream_executor/rocm/rocm_kernel.cc b/tensorflow/stream_executor/rocm/rocm_kernel.cc
new file mode 100644
index 0000000000..162b2bdc71
--- /dev/null
+++ b/tensorflow/stream_executor/rocm/rocm_kernel.cc
@@ -0,0 +1,38 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/stream_executor/gpu/gpu_kernel.h"
+
+namespace stream_executor {
+namespace gpu {
+
+hipFuncCache_t GpuKernel::GetGpuCacheConfig() const {
+  switch (preferred_cache_config_) {
+    case KernelCacheConfig::kNoPreference:
+      return hipFuncCachePreferNone;
+    case KernelCacheConfig::kPreferShared:
+      return hipFuncCachePreferShared;
+    case KernelCacheConfig::kPreferL1:
+      return hipFuncCachePreferL1;
+    case KernelCacheConfig::kPreferEqual:
+      return hipFuncCachePreferEqual;
+    default:
+      LOG(FATAL) << "Unknown KernelCacheConfig"
+                 << static_cast<int32>(preferred_cache_config_);
+  }
+}
+
+}  // namespace gpu
+}  // namespace stream_executor
diff --git a/tensorflow/stream_executor/rocm/rocm_platform.cc b/tensorflow/stream_executor/rocm/rocm_platform.cc
new file mode 100644
index 0000000000..113371dd55
--- /dev/null
+++ b/tensorflow/stream_executor/rocm/rocm_platform.cc
@@ -0,0 +1,180 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/stream_executor/rocm/rocm_platform.h"
+
+#include "absl/strings/str_format.h"
+#include "tensorflow/stream_executor/gpu/gpu_driver.h"
+#include "tensorflow/stream_executor/gpu/gpu_executor.h"
+#include "tensorflow/stream_executor/lib/error.h"
+#include "tensorflow/stream_executor/lib/initialize.h"
+#include "tensorflow/stream_executor/lib/ptr_util.h"
+#include "tensorflow/stream_executor/lib/status.h"
+#include "tensorflow/stream_executor/lib/stringprintf.h"
+#include "tensorflow/stream_executor/rocm/rocm_platform_id.h"
+
+namespace stream_executor {
+namespace gpu {
+
+ROCmPlatform::ROCmPlatform()
+    : name_("ROCM"), min_numa_node_(0), limit_numa_node_(0) {}
+
+ROCmPlatform::~ROCmPlatform() {}
+
+// Due to legacy issues in user code, we can't currently call InpectNumaNodes
+// at module initialization time, because non-GPU programs still include this
+// plugin via various methods, so instead, it has to be init-on-reference.
+void ROCmPlatform::InspectNumaNodes() {
+  // To get NUMA node information, we need to create all executors, so we can
+  // examine their device descriptions to see their bus assignments.
+  static bool initialized = false;
+  static mutex numa_mutex(LINKER_INITIALIZED);
+  mutex_lock lock(numa_mutex);
+  if (initialized) {
+    return;
+  }
+
+  StreamExecutorConfig config;
+  for (int i = 0; i < VisibleDeviceCount(); i++) {
+    config.ordinal = i;
+    StreamExecutor* exec = GetExecutor(config).ValueOrDie();
+    if (i == 0) {
+      // NUMA nodes may not start at 0, so set the minimum node  based on the
+      // first executor we see.
+      min_numa_node_ = exec->GetDeviceDescription().numa_node();
+      limit_numa_node_ = min_numa_node_ + 1;
+    } else {
+      min_numa_node_ =
+          std::min(min_numa_node_, exec->GetDeviceDescription().numa_node());
+      limit_numa_node_ = std::max(limit_numa_node_,
+                                  exec->GetDeviceDescription().numa_node() + 1);
+    }
+  }
+  initialized = true;
+}
+
+int ROCmPlatform::BusCount() {
+  InspectNumaNodes();
+  return limit_numa_node_ - min_numa_node_;
+}
+
+int ROCmPlatform::DeviceToBus(int device_ordinal) {
+  StreamExecutorConfig config;
+  config.ordinal = device_ordinal;
+  StreamExecutor* exec = GetExecutor(config).ValueOrDie();
+  return exec->GetDeviceDescription().numa_node() - min_numa_node_;
+}
+
+port::StatusOr<StreamExecutor*> ROCmPlatform::FirstExecutorForBus(
+    int bus_ordinal) {
+  InspectNumaNodes();
+  CHECK_LT(bus_ordinal, BusCount()) << "bus ordinal out of available range";
+  for (int i = 0; i < VisibleDeviceCount(); i++) {
+    if (DeviceToBus(i) == bus_ordinal) {
+      StreamExecutorConfig config;
+      config.ordinal = i;
+      return GetExecutor(config).ValueOrDie();
+    }
+  }
+
+  return port::Status{
+      port::error::NOT_FOUND,
+      absl::StrFormat("Executor for bus %d not found.", bus_ordinal)};
+}
+
+Platform::Id ROCmPlatform::id() const { return kROCmPlatformId; }
+
+int ROCmPlatform::VisibleDeviceCount() const {
+  // Throw away the result - it logs internally, and this [containing] function
+  // isn't in the path of user control. It's safe to call this > 1x.
+
+  if (!gpu::GpuDriver::Init().ok()) {
+    return -1;
+  }
+
+  return GpuDriver::GetDeviceCount();
+}
+
+const string& ROCmPlatform::Name() const { return name_; }
+
+port::StatusOr<StreamExecutor*> ROCmPlatform::ExecutorForDevice(int ordinal) {
+  StreamExecutorConfig config;
+  config.ordinal = ordinal;
+  config.plugin_config = PluginConfig();
+  config.device_options = DeviceOptions::Default();
+  return GetExecutor(config);
+}
+
+port::StatusOr<StreamExecutor*> ROCmPlatform::ExecutorForDeviceWithPluginConfig(
+    int device_ordinal, const PluginConfig& plugin_config) {
+  StreamExecutorConfig config;
+  config.ordinal = device_ordinal;
+  config.plugin_config = plugin_config;
+  config.device_options = DeviceOptions::Default();
+  return GetExecutor(config);
+}
+
+port::StatusOr<StreamExecutor*> ROCmPlatform::GetExecutor(
+    const StreamExecutorConfig& config) {
+  return executor_cache_.GetOrCreate(
+      config, [&]() { return GetUncachedExecutor(config); });
+}
+
+port::StatusOr<std::unique_ptr<StreamExecutor>>
+ROCmPlatform::GetUncachedExecutor(const StreamExecutorConfig& config) {
+  auto executor = MakeUnique<StreamExecutor>(
+      this, MakeUnique<GpuExecutor>(config.plugin_config));
+  auto init_status = executor->Init(config.ordinal, config.device_options);
+  if (!init_status.ok()) {
+    return port::Status{
+        port::error::INTERNAL,
+        absl::StrFormat(
+            "failed initializing StreamExecutor for ROCM device ordinal %d: %s",
+            config.ordinal, init_status.ToString().c_str())};
+  }
+
+  return std::move(executor);
+}
+
+void ROCmPlatform::RegisterTraceListener(
+    std::unique_ptr<TraceListener> listener) {
+  LOG(FATAL) << "not yet implemented: register ROCM trace listener";
+}
+
+void ROCmPlatform::UnregisterTraceListener(TraceListener* listener) {
+  LOG(FATAL) << "not yet implemented: unregister ROCM trace listener";
+}
+
+}  // namespace gpu
+
+static void InitializeROCmPlatform() {
+  // Disabling leak checking, MultiPlatformManager does not destroy its
+  // registered platforms.
+  auto status = MultiPlatformManager::PlatformWithName("ROCM");
+  if (!status.ok()) {
+    std::unique_ptr<gpu::ROCmPlatform> platform(new gpu::ROCmPlatform);
+    SE_CHECK_OK(MultiPlatformManager::RegisterPlatform(std::move(platform)));
+  }
+}
+
+}  // namespace stream_executor
+
+REGISTER_MODULE_INITIALIZER(rocm_platform,
+                            stream_executor::InitializeROCmPlatform());
+
+DECLARE_MODULE_INITIALIZER(multi_platform_manager);
+// Note that module initialization sequencing is not supported in the
+// open-source project, so this will be a no-op there.
+REGISTER_MODULE_INITIALIZER_SEQUENCE(rocm_platform, multi_platform_manager);
diff --git a/tensorflow/stream_executor/rocm/rocm_platform.h b/tensorflow/stream_executor/rocm/rocm_platform.h
new file mode 100644
index 0000000000..d498e5fdb1
--- /dev/null
+++ b/tensorflow/stream_executor/rocm/rocm_platform.h
@@ -0,0 +1,110 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_STREAM_EXECUTOR_ROCM_ROCM_PLATFORM_H_
+#define TENSORFLOW_STREAM_EXECUTOR_ROCM_ROCM_PLATFORM_H_
+
+#include <memory>
+#include <vector>
+
+#include "tensorflow/stream_executor/executor_cache.h"
+#include "tensorflow/stream_executor/lib/statusor.h"
+#include "tensorflow/stream_executor/multi_platform_manager.h"
+#include "tensorflow/stream_executor/platform.h"
+#include "tensorflow/stream_executor/platform/mutex.h"
+#include "tensorflow/stream_executor/platform/port.h"
+#include "tensorflow/stream_executor/platform/thread_annotations.h"
+#include "tensorflow/stream_executor/stream_executor_internal.h"
+#include "tensorflow/stream_executor/stream_executor_pimpl.h"
+#include "tensorflow/stream_executor/trace_listener.h"
+
+namespace stream_executor {
+namespace gpu {
+
+// Opaque and unique identifier for the ROCM platform plugin.
+// This is needed so that plugins can refer to/identify this platform without
+// instantiating a ROCmPlatform object.
+extern const Platform::Id kROCmPlatformId;
+
+// ROCm-specific platform plugin, registered as a singleton value via module
+// initializer.
+class ROCmPlatform : public Platform {
+ public:
+  ROCmPlatform();
+  ~ROCmPlatform() override;
+
+  // ROCmPlatform-specific functionality
+  // Returns the number of distinct buses / NUMA nodes on the machine.
+  int BusCount();
+
+  // Returns the bus/NUMA node for the specified device ordinal.
+  int DeviceToBus(int device_ordinal);
+
+  // Returns the lowest-ordinal-number StreamExecutor on the specified bus.
+  port::StatusOr<StreamExecutor*> FirstExecutorForBus(int bus_ordinal);
+
+  // Platform interface implementation:
+  // Returns the same value as kROCmPlatform above.
+  Platform::Id id() const override;
+
+  // Returns -1 as a sentinel on internal failure (and logs the error).
+  int VisibleDeviceCount() const override;
+
+  const string& Name() const override;
+
+  port::StatusOr<StreamExecutor*> ExecutorForDevice(int ordinal) override;
+
+  port::StatusOr<StreamExecutor*> ExecutorForDeviceWithPluginConfig(
+      int ordinal, const PluginConfig& config) override;
+
+  port::StatusOr<StreamExecutor*> GetExecutor(
+      const StreamExecutorConfig& config) override;
+
+  port::StatusOr<std::unique_ptr<StreamExecutor>> GetUncachedExecutor(
+      const StreamExecutorConfig& config) override;
+
+  void RegisterTraceListener(std::unique_ptr<TraceListener> listener) override;
+
+  void UnregisterTraceListener(TraceListener* listener) override;
+
+ private:
+  // Determines the number of NUMA nodes and the assignment of executor to each.
+  void InspectNumaNodes();
+
+  // This platform's name.
+  string name_;
+
+  // mutex that guards internal state.
+  mutable mutex mu_;
+
+  // Cache of created executors.
+  ExecutorCache executor_cache_;
+
+  // The smallest NUMA node value for any device managed by this machine
+  // manager. Used, along with limit_numa_node_, to convert NUMA nodes into bus
+  // ordinals. The NUMA node space occupied by GPUs is assumed to be dense./
+  int min_numa_node_;
+
+  // Larger than the NUMA node value for any device managed by this machine
+  // manager.
+  int limit_numa_node_;
+
+  SE_DISALLOW_COPY_AND_ASSIGN(ROCmPlatform);
+};
+
+}  // namespace gpu
+}  // namespace stream_executor
+
+#endif  // TENSORFLOW_STREAM_EXECUTOR_ROCM_ROCM_PLATFORM_H_
diff --git a/tensorflow/stream_executor/rocm/rocm_platform_id.cc b/tensorflow/stream_executor/rocm/rocm_platform_id.cc
new file mode 100644
index 0000000000..daa42ab022
--- /dev/null
+++ b/tensorflow/stream_executor/rocm/rocm_platform_id.cc
@@ -0,0 +1,24 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/stream_executor/rocm/rocm_platform_id.h"
+
+namespace stream_executor {
+namespace gpu {
+
+PLATFORM_DEFINE_ID(kROCmPlatformId);
+
+}  // namespace gpu
+}  // namespace stream_executor
diff --git a/tensorflow/stream_executor/rocm/rocm_platform_id.h b/tensorflow/stream_executor/rocm/rocm_platform_id.h
new file mode 100644
index 0000000000..71c760b827
--- /dev/null
+++ b/tensorflow/stream_executor/rocm/rocm_platform_id.h
@@ -0,0 +1,34 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_STREAM_EXECUTOR_ROCM_ROCM_PLATFORM_ID_H_
+#define TENSORFLOW_STREAM_EXECUTOR_ROCM_ROCM_PLATFORM_ID_H_
+
+#include "tensorflow/stream_executor/platform.h"
+
+namespace stream_executor {
+namespace gpu {
+
+// Opaque and unique identifier for the ROCm platform.
+// This is needed so that plugins can refer to/identify this platform without
+// instantiating a ROCmPlatform object.
+// This is broken out here to avoid a circular dependency between ROCmPlatform
+// and GpuExecutor.
+extern const Platform::Id kROCmPlatformId;
+
+}  // namespace gpu
+}  // namespace stream_executor
+
+#endif  // TENSORFLOW_STREAM_EXECUTOR_ROCM_ROCM_PLATFORM_ID_H_
diff --git a/tensorflow/stream_executor/rocm/rocm_rng.cc b/tensorflow/stream_executor/rocm/rocm_rng.cc
new file mode 100644
index 0000000000..2048c8ff64
--- /dev/null
+++ b/tensorflow/stream_executor/rocm/rocm_rng.cc
@@ -0,0 +1,284 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "rocm/include/hiprand/hiprand.h"
+#include "tensorflow/stream_executor/device_memory.h"
+#include "tensorflow/stream_executor/gpu/gpu_activation.h"
+#include "tensorflow/stream_executor/gpu/gpu_executor.h"
+#include "tensorflow/stream_executor/gpu/gpu_helpers.h"
+#include "tensorflow/stream_executor/gpu/gpu_rng.h"
+#include "tensorflow/stream_executor/gpu/gpu_stream.h"
+#include "tensorflow/stream_executor/lib/env.h"
+#include "tensorflow/stream_executor/lib/initialize.h"
+#include "tensorflow/stream_executor/lib/status.h"
+#include "tensorflow/stream_executor/platform/logging.h"
+#include "tensorflow/stream_executor/rng.h"
+#include "tensorflow/stream_executor/rocm/rocm_platform_id.h"
+
+// Formats hiprandStatus_t to output prettified values into a log stream.
+std::ostream& operator<<(std::ostream& in, const hiprandStatus_t& status) {
+#define OSTREAM_HIPRAND_STATUS(__name) \
+  case HIPRAND_STATUS_##__name:        \
+    in << "HIPRAND_STATUS_" #__name;   \
+    return in;
+
+  switch (status) {
+    OSTREAM_HIPRAND_STATUS(SUCCESS)
+    OSTREAM_HIPRAND_STATUS(VERSION_MISMATCH)
+    OSTREAM_HIPRAND_STATUS(NOT_INITIALIZED)
+    OSTREAM_HIPRAND_STATUS(ALLOCATION_FAILED)
+    OSTREAM_HIPRAND_STATUS(TYPE_ERROR)
+    OSTREAM_HIPRAND_STATUS(OUT_OF_RANGE)
+    OSTREAM_HIPRAND_STATUS(LENGTH_NOT_MULTIPLE)
+    OSTREAM_HIPRAND_STATUS(LAUNCH_FAILURE)
+    OSTREAM_HIPRAND_STATUS(PREEXISTING_FAILURE)
+    OSTREAM_HIPRAND_STATUS(INITIALIZATION_FAILED)
+    OSTREAM_HIPRAND_STATUS(ARCH_MISMATCH)
+    OSTREAM_HIPRAND_STATUS(INTERNAL_ERROR)
+    default:
+      in << "hiprandStatus_t(" << static_cast<int>(status) << ")";
+      return in;
+  }
+}
+
+namespace stream_executor {
+namespace gpu {
+
+PLUGIN_REGISTRY_DEFINE_PLUGIN_ID(kGpuRandPlugin);
+
+namespace wrap {
+
+#define PERFTOOLS_GPUTOOLS_HIPRAND_WRAP(__name)                     \
+  struct WrapperShim__##__name {                                    \
+    template <typename... Args>                                     \
+    hiprandStatus_t operator()(GpuExecutor* parent, Args... args) { \
+      gpu::ScopedActivateExecutorContext sac{parent};               \
+      return ::__name(args...);                                     \
+    }                                                               \
+  } __name;
+
+PERFTOOLS_GPUTOOLS_HIPRAND_WRAP(hiprandCreateGenerator);
+PERFTOOLS_GPUTOOLS_HIPRAND_WRAP(hiprandDestroyGenerator);
+PERFTOOLS_GPUTOOLS_HIPRAND_WRAP(hiprandSetStream);
+PERFTOOLS_GPUTOOLS_HIPRAND_WRAP(hiprandGenerateUniform);
+PERFTOOLS_GPUTOOLS_HIPRAND_WRAP(hiprandGenerateUniformDouble);
+PERFTOOLS_GPUTOOLS_HIPRAND_WRAP(hiprandSetPseudoRandomGeneratorSeed);
+PERFTOOLS_GPUTOOLS_HIPRAND_WRAP(hiprandSetGeneratorOffset);
+PERFTOOLS_GPUTOOLS_HIPRAND_WRAP(hiprandGenerateNormal);
+PERFTOOLS_GPUTOOLS_HIPRAND_WRAP(hiprandGenerateNormalDouble);
+
+}  // namespace wrap
+
+GpuRng::GpuRng(GpuExecutor* parent) : parent_(parent), rng_(nullptr) {}
+
+GpuRng::~GpuRng() {
+  if (rng_ != nullptr) {
+    wrap::hiprandDestroyGenerator(parent_, rng_);
+  }
+}
+
+bool GpuRng::Init() {
+  mutex_lock lock{mu_};
+  CHECK(rng_ == nullptr);
+
+  hiprandStatus_t ret =
+      wrap::hiprandCreateGenerator(parent_, &rng_, HIPRAND_RNG_PSEUDO_DEFAULT);
+  if (ret != HIPRAND_STATUS_SUCCESS) {
+    LOG(ERROR) << "failed to create random number generator: " << ret;
+    return false;
+  }
+
+  CHECK(rng_ != nullptr);
+  return true;
+}
+
+bool GpuRng::SetStream(Stream* stream) {
+  hiprandStatus_t ret =
+      wrap::hiprandSetStream(parent_, rng_, AsGpuStreamValue(stream));
+  if (ret != HIPRAND_STATUS_SUCCESS) {
+    LOG(ERROR) << "failed to set stream for random generation: " << ret;
+    return false;
+  }
+
+  return true;
+}
+
+// Returns true if std::complex stores its contents as two consecutive
+// elements. Tests int, float and double, as the last two are independent
+// specializations.
+constexpr bool ComplexIsConsecutiveFloats() {
+  return sizeof(std::complex<int>) == 8 && sizeof(std::complex<float>) == 8 &&
+         sizeof(std::complex<double>) == 16;
+}
+
+template <typename T>
+bool GpuRng::DoPopulateRandUniformInternal(Stream* stream, DeviceMemory<T>* v) {
+  mutex_lock lock{mu_};
+  static_assert(ComplexIsConsecutiveFloats(),
+                "std::complex values are not stored as consecutive values");
+
+  if (!SetStream(stream)) {
+    return false;
+  }
+
+  // std::complex<T> is currently implemented as two consecutive T variables.
+  uint64 element_count = v->ElementCount();
+  if (std::is_same<T, std::complex<float>>::value ||
+      std::is_same<T, std::complex<double>>::value) {
+    element_count *= 2;
+  }
+
+  hiprandStatus_t ret;
+  if (std::is_same<T, float>::value ||
+      std::is_same<T, std::complex<float>>::value) {
+    ret = wrap::hiprandGenerateUniform(
+        parent_, rng_, reinterpret_cast<float*>(GpuMemoryMutable(v)),
+        element_count);
+  } else {
+    ret = wrap::hiprandGenerateUniformDouble(
+        parent_, rng_, reinterpret_cast<double*>(GpuMemoryMutable(v)),
+        element_count);
+  }
+  if (ret != HIPRAND_STATUS_SUCCESS) {
+    LOG(ERROR) << "failed to do uniform generation of " << v->ElementCount()
+               << " " << TypeString<T>() << "s at " << v->opaque() << ": "
+               << ret;
+    return false;
+  }
+
+  return true;
+}
+
+bool GpuRng::DoPopulateRandUniform(Stream* stream, DeviceMemory<float>* v) {
+  return DoPopulateRandUniformInternal(stream, v);
+}
+
+bool GpuRng::DoPopulateRandUniform(Stream* stream, DeviceMemory<double>* v) {
+  return DoPopulateRandUniformInternal(stream, v);
+}
+
+bool GpuRng::DoPopulateRandUniform(Stream* stream,
+                                   DeviceMemory<std::complex<float>>* v) {
+  return DoPopulateRandUniformInternal(stream, v);
+}
+
+bool GpuRng::DoPopulateRandUniform(Stream* stream,
+                                   DeviceMemory<std::complex<double>>* v) {
+  return DoPopulateRandUniformInternal(stream, v);
+}
+
+template <typename ElemT, typename FuncT>
+bool GpuRng::DoPopulateRandGaussianInternal(Stream* stream, ElemT mean,
+                                            ElemT stddev,
+                                            DeviceMemory<ElemT>* v,
+                                            FuncT func) {
+  mutex_lock lock{mu_};
+
+  if (!SetStream(stream)) {
+    return false;
+  }
+
+  uint64 element_count = v->ElementCount();
+  hiprandStatus_t ret =
+      func(parent_, rng_, GpuMemoryMutable(v), element_count, mean, stddev);
+
+  if (ret != HIPRAND_STATUS_SUCCESS) {
+    LOG(ERROR) << "failed to do gaussian generation of " << v->ElementCount()
+               << " floats at " << v->opaque() << ": " << ret;
+    return false;
+  }
+
+  return true;
+}
+
+bool GpuRng::DoPopulateRandGaussian(Stream* stream, float mean, float stddev,
+                                    DeviceMemory<float>* v) {
+  return DoPopulateRandGaussianInternal(stream, mean, stddev, v,
+                                        wrap::hiprandGenerateNormal);
+}
+
+bool GpuRng::DoPopulateRandGaussian(Stream* stream, double mean, double stddev,
+                                    DeviceMemory<double>* v) {
+  return DoPopulateRandGaussianInternal(stream, mean, stddev, v,
+                                        wrap::hiprandGenerateNormalDouble);
+}
+
+bool GpuRng::SetSeed(Stream* stream, const uint8* seed, uint64 seed_bytes) {
+  mutex_lock lock{mu_};
+  CHECK(rng_ != nullptr);
+
+  if (!CheckSeed(seed, seed_bytes)) {
+    return false;
+  }
+
+  if (!SetStream(stream)) {
+    return false;
+  }
+
+  // Requires 8 bytes of seed data; checked in RngSupport::CheckSeed (above)
+  // (which itself requires 16 for API consistency with host RNG fallbacks).
+  hiprandStatus_t ret = wrap::hiprandSetPseudoRandomGeneratorSeed(
+      parent_, rng_, *(reinterpret_cast<const uint64*>(seed)));
+  if (ret != HIPRAND_STATUS_SUCCESS) {
+    LOG(ERROR) << "failed to set rng seed: " << ret;
+    return false;
+  }
+
+  ret = wrap::hiprandSetGeneratorOffset(parent_, rng_, 0);
+  if (ret != HIPRAND_STATUS_SUCCESS) {
+    LOG(ERROR) << "failed to reset rng position: " << ret;
+    return false;
+  }
+  return true;
+}
+
+}  // namespace gpu
+}  // namespace stream_executor
+
+namespace se = ::stream_executor;
+
+REGISTER_MODULE_INITIALIZER(register_hiprand, {
+  se::port::Status status =
+      se::PluginRegistry::Instance()
+          ->RegisterFactory<se::PluginRegistry::RngFactory>(
+              se::gpu::kROCmPlatformId, se::gpu::kGpuRandPlugin, "hipRAND",
+              [](se::internal::StreamExecutorInterface* parent)
+                  -> se::rng::RngSupport* {
+                se::gpu::GpuExecutor* rocm_executor =
+                    dynamic_cast<se::gpu::GpuExecutor*>(parent);
+                if (rocm_executor == nullptr) {
+                  LOG(ERROR)
+                      << "Attempting to initialize an instance of the hipRAND "
+                      << "support library with a non-ROCM StreamExecutor";
+                  return nullptr;
+                }
+
+                se::gpu::GpuRng* rng = new se::gpu::GpuRng(rocm_executor);
+                if (!rng->Init()) {
+                  // Note: Init() will log a more specific error.
+                  delete rng;
+                  return nullptr;
+                }
+                return rng;
+              });
+
+  if (!status.ok()) {
+    LOG(ERROR) << "Unable to register hipRAND factory: "
+               << status.error_message();
+  }
+
+  se::PluginRegistry::Instance()->SetDefaultFactory(
+      se::gpu::kROCmPlatformId, se::PluginKind::kRng, se::gpu::kGpuRandPlugin);
+});
diff --git a/tensorflow/stream_executor/stream_executor_internal.cc b/tensorflow/stream_executor/stream_executor_internal.cc
index 341c6edccd..46afedef33 100644
--- a/tensorflow/stream_executor/stream_executor_internal.cc
+++ b/tensorflow/stream_executor/stream_executor_internal.cc
@@ -25,6 +25,13 @@ StreamExecutorFactory* MakeCUDAExecutorImplementation() {
   return &instance;
 }
 
+// -- ROCm
+
+StreamExecutorFactory* MakeROCMExecutorImplementation() {
+  static StreamExecutorFactory instance;
+  return &instance;
+}
+
 // -- OpenCL
 
 StreamExecutorFactory* MakeOpenCLExecutorImplementation() {
diff --git a/tensorflow/stream_executor/stream_executor_internal.h b/tensorflow/stream_executor/stream_executor_internal.h
index 0c2c33cfca..6138554e9e 100644
--- a/tensorflow/stream_executor/stream_executor_internal.h
+++ b/tensorflow/stream_executor/stream_executor_internal.h
@@ -374,9 +374,11 @@ using StreamFactory = std::function<StreamInterface *(StreamExecutor *)>;
 using TimerFactory = std::function<TimerInterface *(StreamExecutor *)>;
 using KernelFactory = std::function<KernelInterface*()>;
 
-StreamExecutorFactory* MakeCUDAExecutorImplementation();
+StreamExecutorFactory *MakeCUDAExecutorImplementation();
 
-StreamExecutorFactory* MakeOpenCLExecutorImplementation();
+StreamExecutorFactory *MakeROCMExecutorImplementation();
+
+StreamExecutorFactory *MakeOpenCLExecutorImplementation();
 
 extern StreamExecutorFactory MakeHostExecutorImplementation;
 
diff --git a/tensorflow/stream_executor/stream_executor_pimpl.cc b/tensorflow/stream_executor/stream_executor_pimpl.cc
index 439c73ec8f..6f0ba51ec6 100644
--- a/tensorflow/stream_executor/stream_executor_pimpl.cc
+++ b/tensorflow/stream_executor/stream_executor_pimpl.cc
@@ -71,6 +71,9 @@ internal::StreamExecutorInterface *StreamExecutorImplementationFromPlatformKind(
     case PlatformKind::kCuda:
       factory = *internal::MakeCUDAExecutorImplementation();
       break;
+    case PlatformKind::kROCm:
+      factory = *internal::MakeROCMExecutorImplementation();
+      break;
     case PlatformKind::kOpenCL:
       factory = *internal::MakeOpenCLExecutorImplementation();
       break;
@@ -188,6 +191,8 @@ StreamExecutor::StreamExecutor(
       memory_limit_bytes_(GetMemoryLimitBytes()) {
   if (port::Lowercase(platform_->Name()) == "cuda") {
     platform_kind_ = PlatformKind::kCuda;
+  } else if (port::Lowercase(platform_->Name()) == "rocm") {
+    platform_kind_ = PlatformKind::kROCm;
   } else if (port::Lowercase(platform_->Name()) == "opencl") {
     platform_kind_ = PlatformKind::kOpenCL;
   } else if (port::Lowercase(platform_->Name()) == "host") {
diff --git a/third_party/gpus/rocm/BUILD.tpl b/third_party/gpus/rocm/BUILD.tpl
index 655acaec97..502b6b8de2 100644
--- a/third_party/gpus/rocm/BUILD.tpl
+++ b/third_party/gpus/rocm/BUILD.tpl
@@ -18,6 +18,7 @@ cc_library(
     includes = [
         ".",
         "rocm/include",
+        "rocm/include/rocrand",
     ],
     visibility = ["//visibility:public"],
 )
-- 
GitLab


From 2f42814a4175257f3aec46c2985ef60f88cfebb6 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 28 Jan 2019 11:05:26 -0800
Subject: [PATCH 1378/2345] Add a
 --tf_xla_disable_deadness_safety_checks_for_debugging flag; NFC

This flag can be used to (unsoundly) disable deadness analysis to aid analysis.
For instance it can help us understand how much performance we're losing due to
conservatism in deadness analysis.

Clean up the commenting style on MarkForCompilationPassFlags while at it.

PiperOrigin-RevId: 231252312
---
 tensorflow/compiler/jit/flags.cc              |  9 +++-
 tensorflow/compiler/jit/flags.h               | 54 +++++++++++--------
 .../compiler/jit/mark_for_compilation_pass.cc | 20 +++++--
 3 files changed, 58 insertions(+), 25 deletions(-)

diff --git a/tensorflow/compiler/jit/flags.cc b/tensorflow/compiler/jit/flags.cc
index 98e344b3a0..fba69dfccc 100644
--- a/tensorflow/compiler/jit/flags.cc
+++ b/tensorflow/compiler/jit/flags.cc
@@ -68,7 +68,12 @@ void AppendMarkForCompilationPassFlagsInternal(std::vector<Flag>* flag_list) {
       Flag("tf_xla_fusion_only",
            &mark_for_compilation_flags->tf_xla_fusion_only,
            "enable fusion of element-wise operations only using XLA when "
-           "global_jit_level is ON*.")};
+           "global_jit_level is ON*."),
+      Flag("tf_xla_disable_deadness_safety_checks_for_debugging",
+           &mark_for_compilation_flags
+                ->tf_xla_disable_deadness_safety_checks_for_debugging,
+           "Disable deadness related safety checks when clustering (this is "
+           "unsound).")};
   flag_list->insert(flag_list->end(), new_flags.begin(), new_flags.end());
 }
 
@@ -89,6 +94,8 @@ void AllocateAndParseFlags() {
   mark_for_compilation_flags->tf_xla_clustering_fuel =
       std::numeric_limits<int64>::max();
   mark_for_compilation_flags->tf_xla_fusion_only = false;
+  mark_for_compilation_flags
+      ->tf_xla_disable_deadness_safety_checks_for_debugging = false;
 
   device_flags = new XlaDeviceFlags;
   device_flags->tf_xla_compile_on_demand = false;
diff --git a/tensorflow/compiler/jit/flags.h b/tensorflow/compiler/jit/flags.h
index 5ddea588ee..ed7810fcfd 100644
--- a/tensorflow/compiler/jit/flags.h
+++ b/tensorflow/compiler/jit/flags.h
@@ -25,27 +25,39 @@ namespace tensorflow {
 
 // Flags associated with the XLA bridge's mark_for_compilation_pass module.
 struct MarkForCompilationPassFlags {
-  int32 tf_xla_auto_jit;  // Control compilation of operators into XLA
-                          // computations on CPU and GPU devices.  0 = use
-                          // ConfigProto setting; -1 = off; 1 = on for things
-                          // very likely to be improved; 2 = on for everything.
-                          // Experimental.
-  int32 tf_xla_min_cluster_size;  // Minimum number of operators in an XLA
-                                  // compilation. Ignored for operators placed
-                                  // on an XLA device or operators explicitly
-                                  // marked for compilation.
-  int32 tf_xla_max_cluster_size;  // Maximum number of operators in an XLA
-                                  // compilation.
-  bool tf_xla_clustering_debug;   // Dump graphs during XLA compilation.
-  bool tf_xla_cpu_global_jit;     // Enables global JIT compilation for CPU
-                                  // via SessionOptions.
-  int64 tf_xla_clustering_fuel;   // "Compiler fuel" for clustering.  Only this
-                                  // many ops will be marked as eligible for
-                                  // clustering.
-  bool tf_xla_fusion_only;  // This flag is effective only when global_jit_level
-                            // is set to ON* and overrides its behavior. If
-                            // true, enable fusion of element-wise operations
-                            // only using XLA.
+  // Control compilation of operators into XLA computations on CPU and GPU
+  // devices.  0 = use ConfigProto setting; -1 = off; 1 = on for things very
+  // likely to be improved; 2 = on for everything.
+  //
+  // Experimental.
+  int32 tf_xla_auto_jit;
+
+  // Minimum number of operators in an XLA compilation. Ignored for operators
+  // placed on an XLA device or operators explicitly marked for compilation.
+  int32 tf_xla_min_cluster_size;
+
+  // Maximum number of operators in an XLA compilation.
+  int32 tf_xla_max_cluster_size;
+
+  // Dump graphs during XLA compilation.
+  bool tf_xla_clustering_debug;
+
+  // Enables global JIT compilation for CPU via SessionOptions.
+  bool tf_xla_cpu_global_jit;
+
+  // "Compiler fuel" for clustering.  Only this many ops will be marked as
+  // eligible for clustering.
+  int64 tf_xla_clustering_fuel;
+
+  // tf_xla_fusion_only is effective only when global_jit_level is set to ON*
+  // and overrides its behavior. If true, enable fusion of element-wise
+  // operations only using XLA.
+  bool tf_xla_fusion_only;
+
+  // If tf_xla_disable_deadness_safety_checks_for_debugging is set to true then
+  // we do not do deadness related safety checks.  This is unsound in general,
+  // but can be used as a debugging aid.
+  bool tf_xla_disable_deadness_safety_checks_for_debugging;
 };
 
 // Flags associated with the XLA bridge's xla_device module.
diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass.cc b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
index 234e8052d1..20c2cd7e05 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
@@ -689,6 +689,17 @@ Status MarkForCompilationPass::Run(
     TF_RETURN_IF_ERROR(DeadnessAnalysis::Run(**options.graph, &deadness));
   }
 
+  bool deadness_analysis_disabled =
+      GetMarkForCompilationPassFlags()
+          ->tf_xla_disable_deadness_safety_checks_for_debugging;
+
+  if (deadness_analysis_disabled) {
+    LOG(WARNING) << "Deadness analysis was manually disabled via "
+                    "--tf_xla_disable_deadness_safety_checks_for_debugging; "
+                    "auto-clustering "
+                    "is unsound!";
+  }
+
   auto is_compilable = [&](const Node* node, const DeviceType& device_type) {
     const XlaOpRegistry::DeviceRegistration* registration;
     if (!XlaOpRegistry::GetCompilationDevice(device_type.type(),
@@ -721,9 +732,12 @@ Status MarkForCompilationPass::Run(
     // and some are dead) then don't compile it.  XLA cannot represent the
     // deadness semantics of these nodes correctly and auto-clustering these
     // nodes can cause deadness to propagate to nodes that should be live.
-    if (node->IsMerge() || deadness->HasInputsWithMismatchingDeadness(*node)) {
-      VLOG(2) << "Rejecting " << node->name() << ": mismatching deadness.";
-      return false;
+    if (!deadness_analysis_disabled) {
+      if (node->IsMerge() ||
+          deadness->HasInputsWithMismatchingDeadness(*node)) {
+        VLOG(2) << "Rejecting " << node->name() << ": mismatching deadness.";
+        return false;
+      }
     }
 
     // Check for fusable ops only if requested.
-- 
GitLab


From 269082890c7cf02e448d67c815af2efe65ef130a Mon Sep 17 00:00:00 2001
From: Guozhong Zhuang <guozhong.zhuang@intel.com>
Date: Mon, 28 Jan 2019 11:36:30 -0800
Subject: [PATCH 1379/2345] address merge conflicts

---
 tensorflow/core/util/mkl_util.h | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/tensorflow/core/util/mkl_util.h b/tensorflow/core/util/mkl_util.h
index 037a53a642..b04d52b48f 100644
--- a/tensorflow/core/util/mkl_util.h
+++ b/tensorflow/core/util/mkl_util.h
@@ -111,6 +111,36 @@ typedef enum {
   Dim3d_I = 1
 } MklDnnDims3D;
 
+// Enum for the order of dimensions of a TF 2D filter with shape [filter_height,
+// filter_width, in_channels, out_channels]
+typedef enum {
+  TF_2DFILTER_DIM_H = 0,
+  TF_2DFILTER_DIM_W = 1,
+  TF_2DFILTER_DIM_I = 2,
+  TF_2DFILTER_DIM_O = 3
+} TFFilterDims2d;
+
+// Enum for the order of dimensions of a TF 3D filter with shape [filter_depth,
+// filter_height, filter_width, in_channels, out_channels]
+typedef enum {
+  TF_3DFILTER_DIM_P = 0,
+  TF_3DFILTER_DIM_H = 1,
+  TF_3DFILTER_DIM_W = 2,
+  TF_3DFILTER_DIM_I = 3,
+  TF_3DFILTER_DIM_O = 4
+} TFFilterDims3d;
+
+// The dimensions order that MKL DNN requires for the filter in a grouped
+// convolution (2D only)
+typedef enum {
+  MKL_GROUP_FILTER_DIM_G = 0,
+  MKL_GROUP_FILTER_DIM_O = 1,
+  MKL_GROUP_FILTER_DIM_I = 2,
+  MKL_GROUP_FILTER_DIM_H = 3,
+  MKL_GROUP_FILTER_DIM_W = 4
+} MklDnnFilterGroupDims;
+
+
 // Enum used to templatize MklOp kernel implementations
 // that support both fp32 and int8 versions.
 enum class MklQuantization {
@@ -1736,6 +1766,7 @@ class MklDnnData {
   inline void SetUsrMem(const memory::primitive_desc& pd,
                         void* data_buffer = nullptr) {
     CHECK_NOTNULL(cpu_engine_);
+    if (user_memory_) delete user_memory_;
     // TODO(nhasabni): can we remove dynamic memory allocation?
     if (data_buffer) {
       user_memory_ = new memory(pd, data_buffer);
-- 
GitLab


From a0cec6fe3e379ca85045017316b52d88d4bacfef Mon Sep 17 00:00:00 2001
From: Rick Chao <rchao@google.com>
Date: Mon, 28 Jan 2019 11:16:35 -0800
Subject: [PATCH 1380/2345] Remove redundant lines to set
 self._local_sense.value in distribute_coordinator._Barrier.wait(). It is set
 to not self._flag immediately following that anyway.

PiperOrigin-RevId: 231254673
---
 tensorflow/python/distribute/distribute_coordinator.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tensorflow/python/distribute/distribute_coordinator.py b/tensorflow/python/distribute/distribute_coordinator.py
index 70c6409a22..c01c0f0572 100644
--- a/tensorflow/python/distribute/distribute_coordinator.py
+++ b/tensorflow/python/distribute/distribute_coordinator.py
@@ -77,8 +77,6 @@ class _Barrier(object):
 
   def wait(self):
     """Waits until all other callers reach the same wait call."""
-    if not hasattr(self._local_sense, "value"):
-      self._local_sense.value = False
     self._local_sense.value = not self._flag
     with self._lock:
       self._counter += 1
-- 
GitLab


From bb89949e2ec2dcff9864f79183335eeec6c1f64c Mon Sep 17 00:00:00 2001
From: Roy Frostig <frostig@google.com>
Date: Mon, 28 Jan 2019 11:30:34 -0800
Subject: [PATCH 1381/2345] [XLA] Add AllToAll to the Python XLA client

PiperOrigin-RevId: 231257629
---
 .../xla/python/local_computation_builder.cc   | 11 ++++
 .../xla/python/local_computation_builder.h    |  4 ++
 .../xla/python/local_computation_builder.i    |  1 +
 tensorflow/compiler/xla/python/xla_client.py  | 50 ++++++++++++++++---
 .../compiler/xla/python/xla_client_test.py    | 12 +++++
 5 files changed, 71 insertions(+), 7 deletions(-)

diff --git a/tensorflow/compiler/xla/python/local_computation_builder.cc b/tensorflow/compiler/xla/python/local_computation_builder.cc
index 0e898d494e..ce4bd6f681 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.cc
+++ b/tensorflow/compiler/xla/python/local_computation_builder.cc
@@ -691,6 +691,17 @@ LocalOp LocalComputationBuilder::Collapse(const LocalOp& operand,
   return xla::Collapse(operand.op(), dimensions);
 }
 
+LocalOp LocalComputationBuilder::AllToAll(
+    const LocalOp& operand, int64 split_dimension, int64 concat_dimension,
+    int64 split_count, absl::Span<const ReplicaGroup> replica_groups) {
+  std::vector<ReplicaGroup> rg(replica_groups.size());
+  for (int i = 0; i < replica_groups.size(); ++i) {
+    rg.push_back(replica_groups[i]);
+  }
+  return xla::AllToAll(operand.op(), split_dimension, concat_dimension,
+                       split_count, rg);
+}
+
 LocalOp LocalComputationBuilder::CrossReplicaSum(
     const LocalOp& operand, absl::Span<const ReplicaGroup> replica_groups) {
   return xla::CrossReplicaSum(operand.op(), replica_groups);
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.h b/tensorflow/compiler/xla/python/local_computation_builder.h
index 6170567f9f..e3af88f825 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.h
+++ b/tensorflow/compiler/xla/python/local_computation_builder.h
@@ -316,6 +316,10 @@ class LocalComputationBuilder {
 
   LocalOp Collapse(const LocalOp& operand, absl::Span<const int64> dimensions);
 
+  LocalOp AllToAll(const LocalOp& operand, int64 split_dimension,
+                   int64 concat_dimension, int64 split_count,
+                   absl::Span<const ReplicaGroup> replica_groups);
+
   LocalOp CrossReplicaSum(const LocalOp& operand,
                           absl::Span<const ReplicaGroup> replica_groups);
 
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.i b/tensorflow/compiler/xla/python/local_computation_builder.i
index 6a85ed62de..7b2f69d6ec 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.i
+++ b/tensorflow/compiler/xla/python/local_computation_builder.i
@@ -1018,6 +1018,7 @@ tensorflow::ImportNumpy();
 %unignore xla::swig::LocalComputationBuilder::Pad;
 %unignore xla::swig::LocalComputationBuilder::Reshape;
 %unignore xla::swig::LocalComputationBuilder::Collapse;
+%unignore xla::swig::LocalComputationBuilder::AllToAll;
 %unignore xla::swig::LocalComputationBuilder::CrossReplicaSum;
 %unignore xla::swig::LocalComputationBuilder::Slice;
 %unignore xla::swig::LocalComputationBuilder::SliceInDim;
diff --git a/tensorflow/compiler/xla/python/xla_client.py b/tensorflow/compiler/xla/python/xla_client.py
index 8964b15829..37cae0e3b6 100644
--- a/tensorflow/compiler/xla/python/xla_client.py
+++ b/tensorflow/compiler/xla/python/xla_client.py
@@ -675,6 +675,12 @@ class LocalComputation(object):
     self._delete(self._c_computation)
 
 
+def _make_replica_group_proto(replica_group):
+  replica_group_proto = xla_data_pb2.ReplicaGroup()
+  replica_group_proto.replica_ids.extend(replica_group)
+  return replica_group_proto
+
+
 class ComputationBuilder(object):
   """XLA computation builder.
 
@@ -965,6 +971,41 @@ class ComputationBuilder(object):
       dimensions = tuple(range(ndim))
     return self._client.Reshape(operand, dimensions, new_sizes)
 
+  def AllToAll(self,
+               operand,
+               split_dimension,
+               concat_dimension,
+               replica_groups=None):
+    """AllToAll op.
+
+    Args:
+      operand: LocalOp representing the input array
+      split_dimension: the dimension along which the operand is split
+      concat_dimension: the dimension along which the split blocks are
+        concatenated
+      replica_groups: optional, list of lists of ints encoding a partition of
+        the set {0, 1, ..., num_replicas} into equally-sized replica groups
+        within which the all-to-all is performed. If not supplied or None (the
+        default), all replicas belong to the same group.
+
+    Returns:
+      A LocalOp that represents the all-to-all concatenation.
+    """
+    if replica_groups is None:
+      replica_groups_protos = []  # special value for XLA API
+    else:
+      replica_groups = list(replica_groups)
+      replica_groups_protos = [
+          _make_replica_group_proto(group) for group in replica_groups]
+    if not replica_groups:
+      split_count = get_replica_count()
+    else:
+      split_count = len(replica_groups[0])
+      if not all(split_count == len(g) for g in replica_groups):
+        raise ValueError('Replica groups must be equally sized')
+    return self._client.AllToAll(operand, split_dimension, concat_dimension,
+                                 split_count, replica_groups_protos)
+
   def CrossReplicaSum(self, operand, replica_groups=None):
     """CrossReplicaSum op.
 
@@ -978,16 +1019,11 @@ class ComputationBuilder(object):
     Returns:
       A LocalOp that represents on each replica the sum of its group's values.
     """
-
-    def make_proto(replica_group):
-      replica_group_proto = xla_data_pb2.ReplicaGroup()
-      replica_group_proto.replica_ids.extend(replica_group)
-      return replica_group_proto
-
     if replica_groups is None:
       replica_groups = []  # special value for XLA API
     else:
-      replica_groups = [make_proto(group) for group in replica_groups]
+      replica_groups = [
+          _make_replica_group_proto(group) for group in replica_groups]
     return self._client.CrossReplicaSum(operand, replica_groups)
 
   def Collapse(self, operand, dimensions):
diff --git a/tensorflow/compiler/xla/python/xla_client_test.py b/tensorflow/compiler/xla/python/xla_client_test.py
index 54c76241b9..c80e792464 100644
--- a/tensorflow/compiler/xla/python/xla_client_test.py
+++ b/tensorflow/compiler/xla/python/xla_client_test.py
@@ -553,6 +553,18 @@ class SingleOpTest(LocalComputationTest):
       for src_dtype, dst_dtype in itertools.product(xla_types, xla_types):
         _ConvertAndTest(x, src_dtype, dst_dtype, xla_types[dst_dtype])
 
+  # TODO(b/123523486): re-enable when shape check is resolved
+  def DISABLED_testAllToAllOneReplica(self):
+    samples = [
+        NumpyArrayF32([97.0]),
+        NumpyArrayF32([64.0, 117.0]),
+        NumpyArrayF32([[2.0, 3.0], [4.0, 5.0]]),
+    ]
+    for lhs in samples[:1]:
+      c = self._NewComputation()
+      c.AllToAll(c.Constant(lhs), 0, 0)
+      self._ExecuteAndCompareExact(c, expected=lhs)
+
   def testCrossReplicaSumOneReplica(self):
     samples = [
         NumpyArrayF32(42.0),
-- 
GitLab


From 18c47113bd2be90a85b6b0bbb1429b5f5ca46073 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Mon, 28 Jan 2019 11:40:39 -0800
Subject: [PATCH 1382/2345] Fix a number of warnings when building TF on Mac.

Mostly unused member variables.

PiperOrigin-RevId: 231259867
---
 tensorflow/core/grappler/mutable_graph_view.cc            | 4 ----
 .../core/grappler/optimizers/dependency_optimizer.h       | 4 +---
 .../core/grappler/optimizers/pin_to_host_optimizer.h      | 8 ++------
 tensorflow/core/grappler/optimizers/remapper.h            | 5 +----
 tensorflow/core/grappler/optimizers/shape_optimizer.h     | 8 ++------
 tensorflow/core/platform/posix/env.cc                     | 4 ++--
 tensorflow/core/util/strided_slice_op.cc                  | 2 +-
 7 files changed, 9 insertions(+), 26 deletions(-)

diff --git a/tensorflow/core/grappler/mutable_graph_view.cc b/tensorflow/core/grappler/mutable_graph_view.cc
index 724efcd21e..d4f415ae7b 100644
--- a/tensorflow/core/grappler/mutable_graph_view.cc
+++ b/tensorflow/core/grappler/mutable_graph_view.cc
@@ -52,10 +52,6 @@ bool IsTensorIdControlling(const TensorId& tensor_id) {
   return tensor_id.index() == Graph::kControlSlot;
 }
 
-bool IsOutputPortRegular(const MutableGraphView::OutputPort& port) {
-  return port.port_id > Graph::kControlSlot;
-}
-
 bool IsOutputPortControlling(const MutableGraphView::OutputPort& port) {
   return port.port_id == Graph::kControlSlot;
 }
diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer.h b/tensorflow/core/grappler/optimizers/dependency_optimizer.h
index 7b032673fb..99021b955f 100644
--- a/tensorflow/core/grappler/optimizers/dependency_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/dependency_optimizer.h
@@ -30,8 +30,7 @@ namespace grappler {
 class DependencyOptimizer : public GraphOptimizer {
  public:
   DependencyOptimizer() {}
-  explicit DependencyOptimizer(RewriterConfig::Toggle opt_level)
-      : opt_level_(opt_level) {}
+  explicit DependencyOptimizer(RewriterConfig::Toggle opt_level) {}
   ~DependencyOptimizer() override {}
 
   string name() const override { return "dependency_optimizer"; };
@@ -73,7 +72,6 @@ class DependencyOptimizer : public GraphOptimizer {
   // single control edge.
   void GroupCrossDeviceControlEdges();
 
-  RewriterConfig::Toggle opt_level_;
   bool fetch_nodes_known_;
   std::unordered_set<string> nodes_to_preserve_;
   std::unique_ptr<NodeMap> node_map_;
diff --git a/tensorflow/core/grappler/optimizers/pin_to_host_optimizer.h b/tensorflow/core/grappler/optimizers/pin_to_host_optimizer.h
index d557a03463..44f26461c0 100644
--- a/tensorflow/core/grappler/optimizers/pin_to_host_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/pin_to_host_optimizer.h
@@ -38,9 +38,8 @@ string TryFindHostDevice(const gtl::FlatSet<string>& devices,
 // gpu->gpu->gpu may have been better/faster. We should probably fix this.
 class PinToHostOptimizer : public GraphOptimizer {
  public:
-  PinToHostOptimizer() : opt_level_(RewriterConfig::DEFAULT) {}
-  explicit PinToHostOptimizer(RewriterConfig::Toggle opt_level)
-      : opt_level_(opt_level) {}
+  PinToHostOptimizer() {}
+  explicit PinToHostOptimizer(RewriterConfig::Toggle opt_level) {}
 
   ~PinToHostOptimizer() override {}
 
@@ -51,9 +50,6 @@ class PinToHostOptimizer : public GraphOptimizer {
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimized_graph, double result) override {}
-
- private:
-  RewriterConfig::Toggle opt_level_;
 };
 
 }  // end namespace grappler
diff --git a/tensorflow/core/grappler/optimizers/remapper.h b/tensorflow/core/grappler/optimizers/remapper.h
index c18413e4e7..804338f4d2 100644
--- a/tensorflow/core/grappler/optimizers/remapper.h
+++ b/tensorflow/core/grappler/optimizers/remapper.h
@@ -26,7 +26,7 @@ namespace grappler {
 // nodes to decrease the amount of operations needed to perform a computation.
 class Remapper : public GraphOptimizer {
  public:
-  explicit Remapper(RewriterConfig::Toggle opt_level) : opt_level_(opt_level) {}
+  explicit Remapper(RewriterConfig::Toggle opt_level) {}
 
   ~Remapper() override {}
 
@@ -37,9 +37,6 @@ class Remapper : public GraphOptimizer {
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimized_graph, double result) override;
-
- private:
-  RewriterConfig::Toggle opt_level_;
 };
 
 }  // end namespace grappler
diff --git a/tensorflow/core/grappler/optimizers/shape_optimizer.h b/tensorflow/core/grappler/optimizers/shape_optimizer.h
index b7f84a1e5d..d9c1fefb19 100644
--- a/tensorflow/core/grappler/optimizers/shape_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/shape_optimizer.h
@@ -30,9 +30,8 @@ namespace grappler {
 // information.
 class ShapeOptimizer : public GraphOptimizer {
  public:
-  ShapeOptimizer() : opt_level_(RewriterConfig::ON) {}
-  explicit ShapeOptimizer(RewriterConfig::Toggle opt_level)
-      : opt_level_(opt_level) {}
+  ShapeOptimizer() {}
+  explicit ShapeOptimizer(RewriterConfig::Toggle opt_level) {}
 
   ~ShapeOptimizer() override {}
 
@@ -43,9 +42,6 @@ class ShapeOptimizer : public GraphOptimizer {
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimized_graph, double result) override;
-
- private:
-  RewriterConfig::Toggle opt_level_;
 };
 
 }  // end namespace grappler
diff --git a/tensorflow/core/platform/posix/env.cc b/tensorflow/core/platform/posix/env.cc
index 4e9373616e..b02b1f3f48 100644
--- a/tensorflow/core/platform/posix/env.cc
+++ b/tensorflow/core/platform/posix/env.cc
@@ -86,7 +86,7 @@ class PosixEnv : public Env {
     return new StdThread(thread_options, name, fn);
   }
 
-  int32 GetCurrentThreadId() {
+  int32 GetCurrentThreadId() override {
 #ifdef __APPLE__
     uint64_t tid64;
     pthread_threadid_np(nullptr, &tid64);
@@ -101,7 +101,7 @@ class PosixEnv : public Env {
 #endif
   }
 
-  bool GetCurrentThreadName(string* name) {
+  bool GetCurrentThreadName(string* name) override {
 #ifdef __ANDROID__
     return false;
 #else
diff --git a/tensorflow/core/util/strided_slice_op.cc b/tensorflow/core/util/strided_slice_op.cc
index 9db35da6a2..0df810abd0 100644
--- a/tensorflow/core/util/strided_slice_op.cc
+++ b/tensorflow/core/util/strided_slice_op.cc
@@ -272,7 +272,7 @@ Status ValidateStridedSliceOp(
     const std::array<int64, 2> valid_range = {
         {stride_i > 0 ? 0 : -1, stride_i > 0 ? dim_i : dim_i - 1}};
 
-    auto canonical = [stride_i, i, dim_i, masks, valid_range](int64 x, int c) {
+    auto canonical = [stride_i, dim_i, masks, valid_range](int64 x, int c) {
       if (masks[c]) {
         return stride_i > 0 ? valid_range[c] : valid_range[(c + 1) & 1];
       } else {
-- 
GitLab


From 37d538a626814f1a3cb83b7392bc6af9a7e9e1f7 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Mon, 28 Jan 2019 11:42:05 -0800
Subject: [PATCH 1383/2345] Automated rollback of commit
 8d3dda24f00a457678eb5f986bce80ec8fc6e2a5

PiperOrigin-RevId: 231260115
---
 .../autograph/converters/return_statements.py | 526 +++++++++---------
 .../converters/return_statements_test.py      |  45 +-
 .../python/autograph/impl/conversion_test.py  |   1 +
 tensorflow/python/autograph/pyct/templates.py |   1 -
 4 files changed, 272 insertions(+), 301 deletions(-)

diff --git a/tensorflow/python/autograph/converters/return_statements.py b/tensorflow/python/autograph/converters/return_statements.py
index 80a555385d..496c99e3b5 100644
--- a/tensorflow/python/autograph/converters/return_statements.py
+++ b/tensorflow/python/autograph/converters/return_statements.py
@@ -22,326 +22,310 @@ import gast
 
 from tensorflow.python.autograph.core import converter
 from tensorflow.python.autograph.pyct import anno
-from tensorflow.python.autograph.pyct import parser
+from tensorflow.python.autograph.pyct import ast_util
 from tensorflow.python.autograph.pyct import templates
 from tensorflow.python.autograph.pyct.static_analysis.annos import NodeAnno
 
 
-BODY_DEFINITELY_RETURNS = 'BODY_DEFINITELY_RETURNS'
-ORELSE_DEFINITELY_RETURNS = 'ORELSE_DEFINITELY_RETURNS'
-STMT_DEFINITELY_RETURNS = 'STMT_DEFINITELY_RETURNS'
+# TODO(mdan): Move this logic into transformer_base.
+class BodyVisitor(converter.Base):
+  """Walks breadth- or depth-first the list-of-nodes bodies of AST nodes."""
 
+  def __init__(self, ctx, depth_first=False):
+    super(BodyVisitor, self).__init__(ctx)
+    self.depth_first = depth_first
+    self.changes_made = False
 
-class _Block(object):
-
-  def __init__(self):
-    self.definitely_returns = False
-
-
-class ConditionalReturnLowering(converter.Base):
-  """Rewrites a a pattern where it's unbovious that all paths return a value.
-
-  This rewrite allows avoiding intermediate None return values.
-
-  The following pattern:
-
-      if cond:
-        <block 1>
-        return
+  def visit_nodelist(self, nodelist):
+    for node in nodelist:
+      if isinstance(node, list):
+        node = self.visit_nodelist(node)
       else:
-        <block 2>
-      <block 3>
-
-  is converted to:
+        node = self.generic_visit(node)
+    return nodelist
 
-      if cond:
-        <block 1>
-        return
-      else:
-        <block 2>
-        <block 3>
-
-  and vice-versa (if the else returns, subsequent statements are moved unthe the
-  if branch).
-  """
-
-  def visit_Return(self, node):
-    self.state[_Block].definitely_returns = True
+  def visit_If(self, node):
+    if self.depth_first:
+      node = self.generic_visit(node)
+    node.body = self.visit_nodelist(node.body)
+    node.orelse = self.visit_nodelist(node.orelse)
+    if not self.depth_first:
+      node = self.generic_visit(node)
     return node
 
-  def _postprocess_statement(self, node):
-    # If the node definitely returns (e.g. it's a with statement with a
-    # return stateent in it), then the current block also definitely returns.
-    if anno.getanno(node, STMT_DEFINITELY_RETURNS, default=False):
-      self.state[_Block].definitely_returns = True
-
-    # The special case: collapse a typical conditional return pattern into
-    # a single conditional with possibly returns on both branches. This
-    # reduces the use of None return values, which don't work with TF
-    # conditionals.
-    if (isinstance(node, gast.If)
-        and anno.getanno(node, BODY_DEFINITELY_RETURNS, default=False)):
-      return node, node.orelse
-    elif (isinstance(node, gast.If)
-          and anno.getanno(node, ORELSE_DEFINITELY_RETURNS, default=False)):
-      return node, node.body
-
-    return node, None
-
-  def _visit_statement_block(self, node, nodes):
-    self.state[_Block].enter()
-    new_nodes = self.visit_block(nodes, after_visit=self._postprocess_statement)
-    block_definitely_returns = self.state[_Block].definitely_returns
-    self.state[_Block].exit()
-    return new_nodes, block_definitely_returns
+  def visit_For(self, node):
+    if self.depth_first:
+      node = self.generic_visit(node)
+    node.body = self.visit_nodelist(node.body)
+    node.orelse = self.visit_nodelist(node.orelse)
+    if not self.depth_first:
+      node = self.generic_visit(node)
+    return node
 
   def visit_While(self, node):
-    node.test = self.visit(node.test)
-    node.body, _ = self._visit_statement_block(node, node.body)
-    node.orelse, _ = self._visit_statement_block(node, node.orelse)
+    if self.depth_first:
+      node = self.generic_visit(node)
+    node.body = self.visit_nodelist(node.body)
+    node.orelse = self.visit_nodelist(node.orelse)
+    if not self.depth_first:
+      node = self.generic_visit(node)
     return node
 
-  def visit_For(self, node):
-    node.iter = self.visit(node.iter)
-    node.target = self.visit(node.target)
-    node.body, _ = self._visit_statement_block(node, node.body)
-    node.orelse, _ = self._visit_statement_block(node, node.orelse)
+  def visit_Try(self, node):
+    if self.depth_first:
+      node = self.generic_visit(node)
+    node.body = self.visit_nodelist(node.body)
+    node.orelse = self.visit_nodelist(node.orelse)
+    node.finalbody = self.visit_nodelist(node.finalbody)
+    for i in range(len(node.handlers)):
+      node.handlers[i].body = self.visit_nodelist(node.handlers[i].body)
+    if not self.depth_first:
+      node = self.generic_visit(node)
     return node
 
-  def visit_If(self, node):
-    node.test = self.visit(node.test)
-
-    node.body, body_definitely_returns = self._visit_statement_block(
-        node, node.body)
-    if body_definitely_returns:
-      anno.setanno(node, BODY_DEFINITELY_RETURNS, True)
-
-    node.orelse, orelse_definitely_returns = self._visit_statement_block(
-        node, node.orelse)
-    if orelse_definitely_returns:
-      anno.setanno(node, ORELSE_DEFINITELY_RETURNS, True)
-
-    if body_definitely_returns and orelse_definitely_returns:
-      self.state[_Block].definitely_returns = True
-
+  def visit_With(self, node):
+    if self.depth_first:
+      node = self.generic_visit(node)
+    node.body = self.visit_nodelist(node.body)
+    if not self.depth_first:
+      node = self.generic_visit(node)
     return node
 
   def visit_FunctionDef(self, node):
-    node.args = self.visit(node.args)
-    node.body, _ = self._visit_statement_block(node, node.body)
+    if self.depth_first:
+      node = self.generic_visit(node)
+    node.body = self.visit_nodelist(node.body)
+    self.generic_visit(node)
+    if not self.depth_first:
+      node = self.generic_visit(node)
     return node
 
 
-class _Return(object):
+class FoldElse(BodyVisitor):
+
+  def visit_nodelist(self, nodelist):
+    for i in range(len(nodelist)):
+      node = nodelist[i]
+      if isinstance(node, gast.If):
+        true_branch_returns = isinstance(node.body[-1], gast.Return)
+        false_branch_returns = len(node.orelse) and isinstance(
+            node.orelse[-1], gast.Return)
+        # If the last node in the if body is a return,
+        # then every line after this if statement effectively
+        # belongs in the else.
+        if true_branch_returns and not false_branch_returns:
+          for j in range(i + 1, len(nodelist)):
+            nodelist[i].orelse.append(ast_util.copy_clean(nodelist[j]))
+          if nodelist[i + 1:]:
+            self.changes_made = True
+          return nodelist[:i + 1]
+        elif not true_branch_returns and false_branch_returns:
+          for j in range(i + 1, len(nodelist)):
+            nodelist[i].body.append(ast_util.copy_clean(nodelist[j]))
+          if nodelist[i + 1:]:
+            self.changes_made = True
+          return nodelist[:i + 1]
+        elif true_branch_returns and false_branch_returns:
+          if nodelist[i + 1:]:
+            raise ValueError(
+                'Unreachable code after conditional where both branches return.'
+            )
+          return nodelist
+      elif isinstance(node, gast.Return) and nodelist[i + 1:]:
+        raise ValueError(
+            'Cannot have statements after a return in the same basic block')
+    return nodelist
+
+
+def contains_return(node):
+  for n in gast.walk(node):
+    if isinstance(n, gast.Return):
+      return True
+  return False
+
+
+class LiftReturn(converter.Base):
+  """Move return statements out of If and With blocks."""
+
+  def __init__(self, ctx):
+    super(LiftReturn, self).__init__(ctx)
+    self.changes_made = False
+    self.common_return_name = None
 
-  def __init__(self):
-    self.used = False
-    self.create_guard = False
-    self.guard_created = False
-
-  def __repr__(self):
-    return 'used: {}'.format(
-        self.used)
-
-
-class _Function(object):
-
-  def __init__(self):
-    self.do_return_var_name = None
-    self.retval_var_name = None
+  def visit_If(self, node):
+    # Depth-first traversal of if statements
+    node = self.generic_visit(node)
+
+    # We check if both branches return, and if so, lift the return out of the
+    # conditional. We don't enforce that the true and false branches either
+    # both return or both do not, because FoldElse might move a return
+    # into a branch after this transform completes. FoldElse and LiftReturn
+    # are alternately run until the code reaches a fixed point.
+    true_branch_returns = isinstance(node.body[-1], gast.Return)
+    false_branch_returns = len(node.orelse) and isinstance(
+        node.orelse[-1], gast.Return)
+    if true_branch_returns and false_branch_returns:
+      node.body[-1] = templates.replace(
+          'a = b', a=self.common_return_name, b=node.body[-1].value)[0]
+      node.orelse[-1] = templates.replace(
+          'a = b', a=self.common_return_name, b=node.orelse[-1].value)[0]
+      return_node = templates.replace('return a', a=self.common_return_name)[0]
+      self.changes_made = True
+      return [node, return_node]
+    else:
+      return node
+
+  def visit_With(self, node):
+    # Depth-first traversal of syntax
+    node = self.generic_visit(node)
+
+    # If the with statement returns, lift the return
+    if isinstance(node.body[-1], gast.Return):
+      node.body[-1] = templates.replace(
+          'a = b', a=self.common_return_name, b=node.body[-1].value)[0]
+      return_node = templates.replace('return a', a=self.common_return_name)[0]
+      node = self.generic_visit(node)
+      self.changes_made = True
+      return [node, return_node]
+    else:
+      return node
 
-  def __repr__(self):
-    return 'return control: {}, return value: {}'.format(
-        self.do_return_var_name, self.retval_var_name)
+  def visit_FunctionDef(self, node):
+    # Ensure we're doing depth-first traversal
+    last_return_name = self.common_return_name
+    body_scope = anno.getanno(node, NodeAnno.BODY_SCOPE)
+    referenced_names = body_scope.referenced
+    self.common_return_name = self.ctx.namer.new_symbol('return_',
+                                                        referenced_names)
+    node = self.generic_visit(node)
+    self.common_return_name = last_return_name
+    return node
 
 
-class ReturnStatementsTransformer(converter.Base):
-  """Lowers return statements into variables and conditionals.
+class DetectReturnInUnsupportedControlFlow(gast.NodeVisitor):
+  """Throws an error if code returns inside loops or try/except."""
 
-  Specifically, the following pattern:
+  # First, throw an error if we detect a return statement in a loop.
+  # TODO(alexbw): we need to learn to handle returns inside a loop,
+  # but don't currently have the TF constructs to do so (need something
+  # that looks vaguely like a goto).
 
-      <block 1>
-      return val
-      <block 2>
+  def __init__(self):
+    self.cant_return = False
+    self.function_level = 0
+    super(DetectReturnInUnsupportedControlFlow, self).__init__()
 
-  is converted to:
+  def visit_While(self, node):
+    self.cant_return = True
+    self.generic_visit(node)
+    self.cant_return = False
 
-      do_return = False
-      retval = None
+  def visit_For(self, node):
+    self.cant_return = True
+    self.generic_visit(node)
+    self.cant_return = False
 
-      <block 1>
+  def visit_Try(self, node):
+    self.cant_return = True
+    self.generic_visit(node)
+    self.cant_return = False
 
-      do_return = True
-      retval = val
+  def visit_FunctionDef(self, node):
+    if not self.function_level:
+      self.function_level += 1
+      self.generic_visit(node)
+      self.function_level -= 1
 
-      if not do_return:
-        <block 2>
+  def visit_Return(self, node):
+    if self.cant_return:
+      raise ValueError(
+          '`return` statements are not supported in loops. '
+          'Try assigning to a variable in the while loop, and returning '
+          'outside of the loop')
 
-      return retval
 
-  The conversion adjusts loops as well:
+class DetectReturnInConditional(gast.NodeVisitor):
+  """Assert that no return statements are present in conditionals."""
 
-      <block 1>
-      while cond:
-        <block 2>
-        return retval
+  def __init__(self):
+    self.cant_return = False
+    self.function_level = 0
+    super(DetectReturnInConditional, self).__init__()
 
-  is converted to:
+  def visit_If(self, node):
+    self.cant_return = True
+    self.generic_visit(node)
+    self.cant_return = False
 
-      <block 1>
-      while not do_return and cond:
-        <block 2>
-        do_return = True
-        retval = val
-  """
+  def visit_FunctionDef(self, node):
+    if not self.function_level:
+      self.function_level += 1
+      self.generic_visit(node)
+      self.function_level -= 1
 
   def visit_Return(self, node):
-    self.state[_Return].used = True
-
-    retval = node.value if node.value else parser.parse_expresison('None')
-
-    template = """
-      do_return_var_name = True
-      retval_var_name = retval
-    """
-    node = templates.replace(
-        template,
-        do_return_var_name=self.state[_Function].do_return_var_name,
-        retval_var_name=self.state[_Function].retval_var_name,
-        retval=retval)
-
-    return node
-
-  def _postprocess_statement(self, node):
-    # Example of how the state machine below works:
-    #
-    #   1| stmt           # State: _Return.used = False
-    #    |                # Action: none
-    #   3| return         # State: _Return.used = True,
-    #    |                #        _Return.guard_created = False,
-    #    |                #        _Return.create_guard = False
-    #    |                # Action: _Return.create_guard = True
-    #   4| stmt           # State: _Return.used = True,
-    #    |                #        _Return.guard_created = False,
-    #    |                #        _Return.create_guard = True
-    #    |                # Action: create `if not return_used`,
-    #    |                #         set _Return.guard_created = True
-    #   5| stmt           # State: _Return.used = True,
-    #    |                #        _Return.guard_created = True
-    #    |                # Action: none (will be wrapped under previously
-    #    |                #         created if node)
-    if self.state[_Return].used:
-      if self.state[_Return].guard_created:
-        return node, None
-
-      elif not self.state[_Return].create_guard:
-        self.state[_Return].create_guard = True
-        return node, None
-
-      elif (not self.state[_Return].guard_created and
-            self.state[_Return].create_guard):
-        self.state[_Return].guard_created = True
-        template = """
-          if ag__.not_(do_return_var_name):
-            original_node
-        """
-        cond, = templates.replace(
-            template,
-            do_return_var_name=self.state[_Function].do_return_var_name,
-            original_node=node)
-        return cond, cond.body
+    if self.cant_return:
+      raise ValueError(
+          'After transforms, a conditional contained a `return `statement, '
+          'which is not allowed. This is a bug, and should not happen.')
 
-      else:
-        assert False, 'should handle all states'
 
-    return node, None
+class DetectReturnInFunctionDef(gast.NodeVisitor):
 
-  def _visit_statement_block(self, node, nodes):
-    self.state[_Return].enter()
-    nodes = self.visit_block(nodes, after_visit=self._postprocess_statement)
-    return_used = self.state[_Return].used
-    self.state[_Return].exit()
-    if return_used:
-      self.state[_Return].used = True
-    return nodes
+  def visit_FunctionDef(self, node):
+    self.generic_visit(node)
+    if not contains_return(node):
+      raise ValueError(
+          'Each function definition should contain at least one return.')
 
-  def visit_While(self, node):
-    node.test = self.visit(node.test)
 
-    # Add the check for return to the loop condition.
-    node.body = self._visit_statement_block(node, node.body)
-    if self.state[_Return].used:
-      node.test = templates.replace_as_expression(
-          'ag__.and_(lambda: ag__.not_(control_var), lambda: test)',
-          test=node.test,
-          control_var=self.state[_Function].do_return_var_name)
+def transform(node, ctx):
+  """Ensure a function has only a single return.
+
+  This transforms an AST node with multiple returns successively into containing
+  only a single return node.
+  There are a few restrictions on what we can handle:
+   - An AST being transformed must contain at least one return.
+   - No returns allowed in loops. We have to know the type of the return value,
+   and we currently don't have either a type inference system to discover it,
+   nor do we have a mechanism for late type binding in TensorFlow.
+   - After all transformations are finished, a Return node is not allowed inside
+   control flow. If we were unable to move a return outside of control flow,
+   this is an error.
+
+  Args:
+     node: ast.AST
+     ctx: converter.EntityContext
+
+  Returns:
+     new_node: an AST with a single return value
+
+  Raises:
+    ValueError: if the AST is structured so that we can't perform the
+   transform.
+  """
+  # Make sure that the function has at least one return statement
+  # TODO(alexbw): turning off this assertion for now --
+  # we need to not require this in e.g. class constructors.
+  # DetectReturnInFunctionDef().visit(node)
 
-    node.orelse = self._visit_statement_block(node, node.orelse)
-    return node
+  # Make sure there's no returns in unsupported locations (loops, try/except)
+  DetectReturnInUnsupportedControlFlow().visit(node)
 
-  def visit_For(self, node):
-    node.iter = self.visit(node.iter)
-    node.target = self.visit(node.target)
-
-    # Add the check for return to the loop condition.
-    node.body = self._visit_statement_block(node, node.body)
-    if self.state[_Return].used:
-      extra_test = anno.getanno(node, 'extra_test', default=None)
-      if extra_test is not None:
-        extra_test = templates.replace_as_expression(
-            'ag__.and_(lambda: ag__.not_(control_var), lambda: extra_test)',
-            extra_test=extra_test,
-            control_var=self.state[_Function].do_return_var_name)
-      else:
-        extra_test = templates.replace_as_expression(
-            'ag__.not_(control_var)',
-            control_var=self.state[_Function].do_return_var_name)
-      anno.setanno(node, 'extra_test', extra_test)
+  while True:
 
-    node.orelse = self._visit_statement_block(node, node.orelse)
-    return node
+    # Try to lift all returns out of if statements and with blocks
+    lr = LiftReturn(ctx)
+    node = lr.visit(node)
+    changes_made = lr.changes_made
+    fe = FoldElse(ctx)
+    node = fe.visit(node)
+    changes_made = changes_made or fe.changes_made
 
-  def visit_If(self, node):
-    node.test = self.visit(node.test)
-    node.body = self._visit_statement_block(node, node.body)
-    node.orelse = self._visit_statement_block(node, node.orelse)
-    return node
-
-  def visit_FunctionDef(self, node):
-    self.state[_Function].enter()
-    self.state[_Return].enter()
-
-    scope = anno.getanno(node, NodeAnno.BODY_SCOPE)
-    do_return_var_name = self.ctx.namer.new_symbol(
-        'do_return', scope.referenced)
-    retval_var_name = self.ctx.namer.new_symbol('retval_', scope.referenced)
-    self.state[_Function].do_return_var_name = do_return_var_name
-    self.state[_Function].retval_var_name = retval_var_name
-
-    converted_body = self._visit_statement_block(node, node.body)
-
-    if self.state[_Return].used:
-      template = """
-        do_return_var_name = False
-        retval_var_name = None
-        body
-        return retval_var_name
-      """
-      node.body = templates.replace(
-          template,
-          body=converted_body,
-          do_return_var_name=do_return_var_name,
-          retval_var_name=retval_var_name)
-
-    self.state[_Return].exit()
-    self.state[_Function].exit()
-    return node
+    if not changes_made:
+      break
 
+  # Make sure we've scrubbed all returns from conditionals
+  DetectReturnInConditional().visit(node)
 
-def transform(node, ctx):
-  """Ensure a function has only a single return."""
-  # Note: Technically, these two could be merged into a single walk, but
-  # keeping them separate helps with readability.
-  node = ConditionalReturnLowering(ctx).visit(node)
-  node = ReturnStatementsTransformer(ctx).visit(node)
   return node
diff --git a/tensorflow/python/autograph/converters/return_statements_test.py b/tensorflow/python/autograph/converters/return_statements_test.py
index 2b160d5ac7..762fbc6f60 100644
--- a/tensorflow/python/autograph/converters/return_statements_test.py
+++ b/tensorflow/python/autograph/converters/return_statements_test.py
@@ -49,16 +49,17 @@ class SingleReturnTest(converter_testing.TestCase):
     self.assertTransformedEquivalent(test_fn, 2)
     self.assertTransformedEquivalent(test_fn, -2)
 
-  def test_missing_else(self):
+  def test_missing_orelse(self):
 
     def test_fn(x):
       if x > 0:
         return x
 
-    self.assertTransformedEquivalent(test_fn, 2)
-    self.assertTransformedEquivalent(test_fn, -2)
+    node, ctx = self.prepare(test_fn, {})
+    with self.assertRaises(ValueError):
+      return_statements.transform(node, ctx)
 
-  def test_missing_else_then_default(self):
+  def test_missing_orelse_recovrable(self):
 
     def test_fn(x):
       if x > 0:
@@ -68,7 +69,7 @@ class SingleReturnTest(converter_testing.TestCase):
     self.assertTransformedEquivalent(test_fn, 2)
     self.assertTransformedEquivalent(test_fn, -2)
 
-  def test_else_only_then_default(self):
+  def test_missing_branch_return_recoverable(self):
 
     def test_fn(x):
       if x < 0:
@@ -135,7 +136,7 @@ class SingleReturnTest(converter_testing.TestCase):
 
     self.assertTransformedEquivalent(test_fn, 2)
 
-  def test_nested_function(self):
+  def test_nested_functions(self):
 
     def test_fn(x):
 
@@ -150,7 +151,7 @@ class SingleReturnTest(converter_testing.TestCase):
     self.assertTransformedEquivalent(test_fn, 2)
     self.assertTransformedEquivalent(test_fn, -2)
 
-  def test_nested_function_in_control_flow(self):
+  def test_nested_functions_in_control_flow(self):
 
     def test_fn(x):
 
@@ -162,30 +163,16 @@ class SingleReturnTest(converter_testing.TestCase):
     self.assertTransformedEquivalent(test_fn, 2)
     self.assertTransformedEquivalent(test_fn, -2)
 
-  def test_for_loop(self):
-
-    def test_fn(n):
-      for _ in range(n):
-        return 1
-
-    self.assertTransformedEquivalent(test_fn, 2)
-    self.assertTransformedEquivalent(test_fn, 0)
-
-  def test_while_loop(self):
+  def test_loop(self):
 
-    def test_fn(n):
-      i = 0
-      s = 0
-      while i < n:
-        i += 1
-        s += i
-        if s > 4:
-          return s
-      return -1
+    def test_fn(x):
+      for _ in range(10):
+        return x
+      return x
 
-    self.assertTransformedEquivalent(test_fn, 0)
-    self.assertTransformedEquivalent(test_fn, 2)
-    self.assertTransformedEquivalent(test_fn, 4)
+    node, ctx = self.prepare(test_fn, {})
+    with self.assertRaises(ValueError):
+      return_statements.transform(node, ctx)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/autograph/impl/conversion_test.py b/tensorflow/python/autograph/impl/conversion_test.py
index cd893e3ff1..9a4fbdad8c 100644
--- a/tensorflow/python/autograph/impl/conversion_test.py
+++ b/tensorflow/python/autograph/impl/conversion_test.py
@@ -96,6 +96,7 @@ class ConversionTest(test.TestCase):
     f_node = program_ctx.dependency_cache[f][0]
     g_node = program_ctx.dependency_cache[g][0]
     self.assertEqual('tf__f', f_node.name)
+    self.assertEqual('tf__g', f_node.body[0].body[0].body[0].value.func.id)
     self.assertEqual('tf__g', g_node.name)
 
   def test_entity_to_graph_class_hierarchy(self):
diff --git a/tensorflow/python/autograph/pyct/templates.py b/tensorflow/python/autograph/pyct/templates.py
index 0f0b8613ac..43279b3ca0 100644
--- a/tensorflow/python/autograph/pyct/templates.py
+++ b/tensorflow/python/autograph/pyct/templates.py
@@ -108,7 +108,6 @@ class ReplaceTransformer(gast.NodeTransformer):
         anno.Basic.ORIGIN,
         anno.Basic.SKIP_PROCESSING,
         anno.Static.ORIG_DEFINITIONS,
-        'extra_test',
     }
 
   def _prepare_replacement(self, replaced, key):
-- 
GitLab


From 1a1e50ee40f3f9ddf418dfe0c98b13c5db7e0339 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Mon, 28 Jan 2019 11:42:39 -0800
Subject: [PATCH 1384/2345] init_scope uses the innermost nonempty device stack

Since we no longer inherit devices inside functions, using the inner context's device doesn't make much sense. We were also only doing it when graph building.

Should remove one more difference between Defun/(defun/function) for the tf.data migration.

PiperOrigin-RevId: 231260238
---
 tensorflow/python/eager/context.py       | 16 ++++++++++-----
 tensorflow/python/eager/function_test.py | 26 ++++++++++++++++++++++++
 tensorflow/python/framework/ops.py       |  9 +++++---
 tensorflow/python/framework/ops_test.py  | 18 ++++++++++------
 4 files changed, 55 insertions(+), 14 deletions(-)

diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py
index 33551fccd5..fd9be060ac 100644
--- a/tensorflow/python/eager/context.py
+++ b/tensorflow/python/eager/context.py
@@ -158,7 +158,8 @@ class _EagerContext(threading.local):
 
 
 ContextSwitch = collections.namedtuple(
-    "ContextSwitch", ["is_building_function", "enter_context_fn"])
+    "ContextSwitch", ["is_building_function", "enter_context_fn",
+                      "device_stack"])
 
 
 # `_ContextSwitchStack` is a `threading.local` to match the semantics of
@@ -175,9 +176,10 @@ class _ContextSwitchStack(threading.local):
       # across threads, since (1) `enable_eager_execution` modifies a
       # process-level flag (`default_execution_mode`) and (2) `__init__` is
       # called each time a threading.local object is used in a separate thread.
-      self.push(is_building_function=False, enter_context_fn=eager_mode)
+      self.push(is_building_function=False, enter_context_fn=eager_mode,
+                device_stack=None)
 
-  def push(self, is_building_function, enter_context_fn):
+  def push(self, is_building_function, enter_context_fn, device_stack):
     """Push metadata about a context switch onto the stack.
 
     A context switch can take any one of the two forms: installing a graph as
@@ -188,10 +190,14 @@ class _ContextSwitchStack(threading.local):
       is_building_function: (bool.) Whether the context is building a function.
       enter_context_fn: (function.) A callable that executes the context switch.
         For example, `graph.as_default` or `eager_mode`.
+      device_stack: If applicable, the device function stack for this
+        graph. When breaking out of graphs in init_scope, the innermost nonempty
+        device stack is used. Eager contexts put `None` here and the value is
+        never used.
     """
 
     self.stack.append(
-        ContextSwitch(is_building_function, enter_context_fn))
+        ContextSwitch(is_building_function, enter_context_fn, device_stack))
 
   def pop(self):
     """Pop the stack."""
@@ -442,7 +448,7 @@ class Context(object):
       # Entering graph mode does not provide us with sufficient information to
       # record a context switch; graph-based context switches are only logged
       # when a graph is registered as the default graph.
-      self.context_switches.push(False, eager_mode)
+      self.context_switches.push(False, eager_mode, None)
     try:
       yield
     finally:
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index a2da088d63..9c713ddec9 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -954,6 +954,32 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
 
     self.assertAllClose([[[[4.0]]]], self.evaluate(y))
 
+  # Variable lifting is somewhat different between defun/tf.function, so testing
+  # device placement on both makes sense.
+  @parameterized.named_parameters(
+      dict(testcase_name='Defun',
+           function_decorator=function.defun),
+      dict(testcase_name='DefFunction',
+           function_decorator=def_function.function))
+  @test_util.run_in_graph_and_eager_modes
+  def testVariablesPlacedOnOutsideDevice(self, function_decorator):
+
+    class _Obj(object):
+
+      def __init__(self):
+        self.v = None
+
+      @function_decorator
+      def f(self):
+        if self.v is None:
+          self.v = variables.Variable(1.)
+        return self.v + 1.
+
+    has_device = _Obj()
+    with ops.device('cpu:0'):
+      has_device.f()
+    self.assertIn('CPU', has_device.v.device)
+
   @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True)
   def testDefunKerasModelCall(self):
     model = MiniModel()
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 63b7108f66..b7908497df 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -5410,7 +5410,8 @@ class _DefaultGraphStack(_DefaultStack):  # pylint: disable=protected-access
   @tf_contextlib.contextmanager
   def get_controller(self, default):
     context.context().context_switches.push(
-        default.building_function, default.as_default)
+        default.building_function, default.as_default,
+        default._device_function_stack)
     try:
       with super(_DefaultGraphStack, self).get_controller(
           default) as g, context.graph_mode():
@@ -5490,7 +5491,7 @@ def init_scope():
       # Names that end with trailing slashes are treated by `name_scope` as
       # absolute.
       scope = scope + "/"
-    inner_device_stack = default_graph._device_function_stack  # pylint: disable=protected-access
+    innermost_nonempty_device_stack = default_graph._device_function_stack  # pylint: disable=protected-access
 
     outer_context = None
     if not _default_graph_stack.stack:
@@ -5503,6 +5504,8 @@ def init_scope():
     else:
       # Find a context that is not building a function.
       for stack_entry in reversed(context.context().context_switches.stack):
+        if not innermost_nonempty_device_stack:
+          innermost_nonempty_device_stack = stack_entry.device_stack
         if not stack_entry.is_building_function:
           outer_context = stack_entry.enter_context_fn
           break
@@ -5531,7 +5534,7 @@ def init_scope():
           # to do the same when lifting into the eager context.
           outer_graph = get_default_graph()
           outer_device_stack = outer_graph._device_function_stack  # pylint: disable=protected-access
-          outer_graph._device_function_stack = inner_device_stack  # pylint: disable=protected-access
+          outer_graph._device_function_stack = innermost_nonempty_device_stack  # pylint: disable=protected-access
         yield
     finally:
       # If an exception is raised here it may be hiding a related exception in
diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py
index 8347e9d1eb..8872c9f67e 100644
--- a/tensorflow/python/framework/ops_test.py
+++ b/tensorflow/python/framework/ops_test.py
@@ -2153,13 +2153,19 @@ class InitScopeTest(test_util.TensorFlowTestCase):
     with g0.as_default(), ops.device("CPU:0"):
       g1 = ops.Graph()
       g1._building_function = True  # pylint: disable=protected-access
-      with g1.as_default(), ops.device("GPU:0"):
+      with g1.as_default():
+        with ops.device("GPU:0"):
+          with ops.init_scope():
+            # init_scope should preserve device set under `g1`.
+            on_gpu = constant_op.constant(1.0)
+            self.assertEqual(on_gpu.device, "/device:GPU:0")
+          still_on_gpu = constant_op.constant(1.0)
+          self.assertEqual(still_on_gpu.device, "/device:GPU:0")
+        blank = constant_op.constant(1.0)
+        self.assertEqual(blank.device, "")
         with ops.init_scope():
-          # init_scope should preserve device set under `g1`.
-          on_gpu = constant_op.constant(1.0)
-          self.assertEqual(on_gpu.device, "/device:GPU:0")
-        still_on_gpu = constant_op.constant(1.0)
-        self.assertEqual(still_on_gpu.device, "/device:GPU:0")
+          now_on_cpu = constant_op.constant(1.0)
+          self.assertEqual(now_on_cpu.device, "/device:CPU:0")
       on_cpu = constant_op.constant(1.0)
       self.assertEqual(on_cpu.device, "/device:CPU:0")
 
-- 
GitLab


From b24783482f452b9aa0f1756360fd453a24230275 Mon Sep 17 00:00:00 2001
From: Sherry Yang <sherryy@google.com>
Date: Mon, 28 Jan 2019 12:43:03 -0800
Subject: [PATCH 1385/2345] Dump graph after each iteration and optimizer in
 grappler.

PiperOrigin-RevId: 231271377
---
 .../common_runtime/optimization_registry.cc   | 10 +++++--
 .../grappler/optimizers/meta_optimizer.cc     | 30 +++++++++++++++++--
 2 files changed, 35 insertions(+), 5 deletions(-)

diff --git a/tensorflow/core/common_runtime/optimization_registry.cc b/tensorflow/core/common_runtime/optimization_registry.cc
index 9be540b019..8120a2005a 100644
--- a/tensorflow/core/common_runtime/optimization_registry.cc
+++ b/tensorflow/core/common_runtime/optimization_registry.cc
@@ -42,13 +42,17 @@ Status OptimizationPassRegistry::RunGrouping(
         if (!s.ok()) return s;
         if (VLOG_IS_ON(1)) {
           DumpGraphToFile(
-              strings::StrCat("after_phase_", phase.first, "_", pass->name()),
+              strings::StrCat(
+                  "after_phase_", phase.first, "_", pass->name(), "_",
+                  reinterpret_cast<uintptr_t>((*options.graph).get())),
               **options.graph);
           if (options.partition_graphs) {
             for (auto& part : *options.partition_graphs) {
               DumpGraphToFile(
-                  strings::StrCat("after_phase_", phase.first, "_",
-                                  pass->name(), "_partition_", part.first),
+                  strings::StrCat(
+                      "after_phase_", phase.first, "_", pass->name(),
+                      "_partition_", part.first, "_",
+                      reinterpret_cast<uintptr_t>(part.second.get())),
                   *part.second);
             }
           }
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
index aa19d33f46..c309caa98a 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
@@ -376,6 +376,12 @@ Status MetaOptimizer::OptimizeGraph(Cluster* cluster, const GrapplerItem& item,
     }
 
     VLOG(4) << "Starting optimization iteration " << iteration;
+    if (VLOG_IS_ON(4)) {
+      DumpGraphDefToFile(
+          strings::StrCat("before_MetaOptimizer_iteration_", iteration, "_",
+                          reinterpret_cast<uintptr_t>(optimized_graph)),
+          *optimized_graph);
+    }
     for (const auto& optimizer : optimizers) {
       GRAPPLER_RETURN_IF_DEADLINE_EXCEEDED();
       // Some optimizers can run only once.
@@ -390,11 +396,25 @@ Status MetaOptimizer::OptimizeGraph(Cluster* cluster, const GrapplerItem& item,
         continue;
       }
       RUN_OPTIMIZER_OR_RETURN_IF_ERROR(optimizer.get());
+
+      if (VLOG_IS_ON(4)) {
+        DumpGraphDefToFile(
+            strings::StrCat("after_MetaOptimizer_iteration_", iteration, "_",
+                            optimizer->name(), "_",
+                            reinterpret_cast<uintptr_t>(optimized_graph)),
+            *optimized_graph);
+      }
       for (const auto& verifier : inter_optimizer_verifiers) {
         // TODO(ashwinm): Need to enforce verification_deadline.
         TF_RETURN_IF_ERROR(verifier->Verify(*optimized_graph));
       }
     }
+    if (VLOG_IS_ON(4)) {
+      DumpGraphDefToFile(
+          strings::StrCat("after_MetaOptimizer_iteration_", iteration, "_",
+                          reinterpret_cast<uintptr_t>(optimized_graph)),
+          *optimized_graph);
+    }
     // TODO(ashwinm): Need to enforce verification_deadline.
     for (const auto& verifier : post_optimization_verifiers) {
       TF_RETURN_IF_ERROR(verifier->Verify(*optimized_graph));
@@ -510,7 +530,10 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
   if (IsTPUGraphDef(*optimized_graph)) {
     VLOG(2) << "Skipping optimizing funcs for TPU graphs";
     if (VLOG_IS_ON(1)) {
-      DumpGraphDefToFile("after_MetaOptimizer", *optimized_graph);
+      DumpGraphDefToFile(
+          strings::StrCat("after_MetaOptimizer_",
+                          reinterpret_cast<uintptr_t>(optimized_graph)),
+          *optimized_graph);
     }
     return Status::OK();
   }
@@ -616,7 +639,10 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
           << " functions: " << str_util::Join(optimized_funcs, ", ");
 
   if (VLOG_IS_ON(1)) {
-    DumpGraphDefToFile("after_MetaOptimizer", *optimized_graph);
+    DumpGraphDefToFile(
+        strings::StrCat("after_MetaOptimizer_",
+                        reinterpret_cast<uintptr_t>(optimized_graph)),
+        *optimized_graph);
   }
   return Status::OK();
 }
-- 
GitLab


From 6df7679bb0f5c675b2955cde46081d6907993dff Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Mon, 28 Jan 2019 12:54:49 -0800
Subject: [PATCH 1386/2345] Fix TAP tests. Follow up to cl/230995424

PiperOrigin-RevId: 231273328
---
 .../python/keras/engine/training_utils.py     | 36 ++++++++++++-------
 1 file changed, 24 insertions(+), 12 deletions(-)

diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index bbc180be89..ea78a24607 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -844,22 +844,34 @@ def get_metric_function(metric, output_shape=None, loss_fn=None):
   Returns:
       The metric function.
   """
+  if metric not in ['accuracy', 'acc', 'crossentropy', 'ce']:
+    return metrics_module.get(metric)
+
+  is_sparse_categorical_crossentropy = (
+      isinstance(loss_fn, losses.SparseCategoricalCrossentropy) or
+      (isinstance(loss_fn, losses.LossFunctionWrapper) and
+       loss_fn.fn == losses.sparse_categorical_crossentropy))
+
+  is_binary_crossentropy = (
+      isinstance(loss_fn, losses.BinaryCrossentropy) or
+      (isinstance(loss_fn, losses.LossFunctionWrapper) and
+       loss_fn.fn == losses.binary_crossentropy))
+
   if metric in ['accuracy', 'acc']:
-    if output_shape[-1] == 1 or loss_fn == losses.binary_crossentropy:
-      return metrics_module.binary_accuracy  # case: binary accuracy
-    elif loss_fn == losses.sparse_categorical_crossentropy:
-      # case: categorical accuracy with sparse targets
+    if output_shape[-1] == 1 or is_binary_crossentropy:
+      return metrics_module.binary_accuracy
+    elif is_sparse_categorical_crossentropy:
       return metrics_module.sparse_categorical_accuracy
-    return metrics_module.categorical_accuracy  # case: categorical accuracy
-  elif metric in ['crossentropy', 'ce']:
-    if output_shape[-1] == 1 or loss_fn == losses.binary_crossentropy:
-      return metrics_module.binary_crossentropy  # case: binary cross-entropy
-    elif loss_fn == losses.sparse_categorical_crossentropy:
-      # case: categorical cross-entropy with sparse targets
+    # If the output_shape[-1] is not 1, then we know output is `categorical`.
+    # We assume it is sparse categorical only if loss is explicitly given
+    # as sparse categorical crossentropy loss.
+    return metrics_module.categorical_accuracy
+  else:
+    if output_shape[-1] == 1 or is_binary_crossentropy:
+      return metrics_module.binary_crossentropy
+    elif is_sparse_categorical_crossentropy:
       return metrics_module.sparse_categorical_crossentropy
-    # case: categorical cross-entropy
     return metrics_module.categorical_crossentropy
-  return metrics_module.get(metric)
 
 
 def call_metric_function(metric_fn, y_true, y_pred, weights=None, mask=None):
-- 
GitLab


From 3bf5de196abbfc829fdc36a2feeed04b284c9125 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 28 Jan 2019 13:13:38 -0800
Subject: [PATCH 1387/2345] Update ops-related pbtxt files.

PiperOrigin-RevId: 231276628
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 37 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 37 +++++++++++++++++++
 2 files changed, 74 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 9c1a0b0d9b..ee16a58667 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -78785,6 +78785,43 @@ op {
     }
   }
 }
+op {
+  name: "TensorListScatterV2"
+  input_arg {
+    name: "tensor"
+    type_attr: "element_dtype"
+  }
+  input_arg {
+    name: "indices"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "element_shape"
+    type_attr: "shape_type"
+  }
+  input_arg {
+    name: "num_elements"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "output_handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "element_dtype"
+    type: "type"
+  }
+  attr {
+    name: "shape_type"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
 op {
   name: "TensorListSetItem"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index ef990b4ff2..40e15b6138 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -38520,6 +38520,43 @@ op {
     }
   }
 }
+op {
+  name: "TensorListScatterV2"
+  input_arg {
+    name: "tensor"
+    type_attr: "element_dtype"
+  }
+  input_arg {
+    name: "indices"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "element_shape"
+    type_attr: "shape_type"
+  }
+  input_arg {
+    name: "num_elements"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "output_handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "element_dtype"
+    type: "type"
+  }
+  attr {
+    name: "shape_type"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
 op {
   name: "TensorListSetItem"
   input_arg {
-- 
GitLab


From 9a2f94fa9914472cee14502c58364b480e92f7d8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 28 Jan 2019 13:15:10 -0800
Subject: [PATCH 1388/2345] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 231276874

---
 tensorflow/go/op/wrappers.go | 89 +++++++++++++++++++++++-------------
 1 file changed, 58 insertions(+), 31 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index c95c8c90b5..c537cb1ba9 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -6644,6 +6644,37 @@ func TensorArraySizeV2(scope *Scope, handle tf.Output, flow_in tf.Output) (size
 	return op.Output(0)
 }
 
+// TensorArrayConcatV2Attr is an optional argument to TensorArrayConcatV2.
+type TensorArrayConcatV2Attr func(optionalAttr)
+
+// TensorArrayConcatV2ElementShapeExcept0 sets the optional element_shape_except0 attribute to value.
+// If not specified, defaults to <unknown_rank:true >
+func TensorArrayConcatV2ElementShapeExcept0(value tf.Shape) TensorArrayConcatV2Attr {
+	return func(m optionalAttr) {
+		m["element_shape_except0"] = value
+	}
+}
+
+// Deprecated. Use TensorArrayConcatV3
+func TensorArrayConcatV2(scope *Scope, handle tf.Output, flow_in tf.Output, dtype tf.DataType, optional ...TensorArrayConcatV2Attr) (value tf.Output, lengths tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtype": dtype}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorArrayConcatV2",
+		Input: []tf.Input{
+			handle, flow_in,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
 // EditDistanceAttr is an optional argument to EditDistance.
 type EditDistanceAttr func(optionalAttr)
 
@@ -15856,6 +15887,33 @@ func ResourceSparseApplyRMSProp(scope *Scope, var_ tf.Output, ms tf.Output, mom
 	return scope.AddOperation(opspec)
 }
 
+// Creates a TensorList by indexing into a Tensor.
+//
+// Each member of the TensorList corresponds to one row of the input tensor,
+// specified by the given index (see `tf.gather`).
+//
+// tensor: The input tensor.
+// indices: The indices used to index into the list.
+// element_shape: The shape of the elements in the list (can be less specified than
+//   the shape of the tensor).
+// num_elements: The size of the output list. Must be large enough to accommodate
+//   the largest index in indices. If -1, the list is just large enough to include
+//   the largest index in indices.
+// output_handle: The TensorList.
+func TensorListScatterV2(scope *Scope, tensor tf.Output, indices tf.Output, element_shape tf.Output, num_elements tf.Output) (output_handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorListScatterV2",
+		Input: []tf.Input{
+			tensor, indices, element_shape, num_elements,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // SampleDistortedBoundingBoxAttr is an optional argument to SampleDistortedBoundingBox.
 type SampleDistortedBoundingBoxAttr func(optionalAttr)
 
@@ -32855,37 +32913,6 @@ func ResourceScatterNdSub(scope *Scope, ref tf.Output, indices tf.Output, update
 	return scope.AddOperation(opspec)
 }
 
-// TensorArrayConcatV2Attr is an optional argument to TensorArrayConcatV2.
-type TensorArrayConcatV2Attr func(optionalAttr)
-
-// TensorArrayConcatV2ElementShapeExcept0 sets the optional element_shape_except0 attribute to value.
-// If not specified, defaults to <unknown_rank:true >
-func TensorArrayConcatV2ElementShapeExcept0(value tf.Shape) TensorArrayConcatV2Attr {
-	return func(m optionalAttr) {
-		m["element_shape_except0"] = value
-	}
-}
-
-// Deprecated. Use TensorArrayConcatV3
-func TensorArrayConcatV2(scope *Scope, handle tf.Output, flow_in tf.Output, dtype tf.DataType, optional ...TensorArrayConcatV2Attr) (value tf.Output, lengths tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtype": dtype}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "TensorArrayConcatV2",
-		Input: []tf.Input{
-			handle, flow_in,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
-}
-
 // Creates a dataset that batches and pads `batch_size` elements from the input.
 //
 // Arguments:
-- 
GitLab


From ae88f8781a2e4f0c71ebf13d03779e88fdd7cbe5 Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Mon, 28 Jan 2019 13:17:19 -0800
Subject: [PATCH 1389/2345] Re-enable whitelisting of TF Ops in Layers.

PiperOrigin-RevId: 231277190
---
 tensorflow/python/keras/BUILD                 |  12 ++
 tensorflow/python/keras/engine/base_layer.py  |   9 +
 .../python/keras/engine/base_layer_utils.py   |  98 ++++++----
 .../keras/layers/tensorflow_op_layer_test.py  | 169 ++++++++++++++++++
 4 files changed, 255 insertions(+), 33 deletions(-)
 create mode 100644 tensorflow/python/keras/layers/tensorflow_op_layer_test.py

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index e879254dae..38793e9f95 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -485,6 +485,18 @@ tf_py_test(
     ],
 )
 
+tf_py_test(
+    name = "tensorflow_op_layer_test",
+    size = "medium",
+    srcs = ["layers/tensorflow_op_layer_test.py"],
+    additional_deps = [
+        ":keras",
+        "@absl_py//absl/testing:parameterized",
+        "//tensorflow/python:client_testlib",
+    ],
+    shard_count = 3,
+)
+
 tf_py_test(
     name = "convolutional_recurrent_test",
     size = "large",
diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index 5eb2c5a752..a355dda4e5 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -524,6 +524,15 @@ class Layer(checkpointable.Checkpointable):
     # models using the functional API).
     build_graph = tf_utils.are_all_symbolic_tensors(input_list)
 
+    if build_graph:
+      # Only create Keras history if at least one tensor originates from a
+      # `keras.Input`. Otherwise this Layer may be being used outside the Keras
+      # framework.
+      if base_layer_utils.needs_keras_history(inputs):
+        base_layer_utils.create_keras_history(inputs)
+      # Do not track these Tensors in any sublayers invoked during `call`.
+      base_layer_utils.mark_checked(inputs)
+
     # Handle Keras mask propagation from previous layer to current layer.
     previous_mask = None
     if build_graph and (not hasattr(self, '_compute_previous_mask') or
diff --git a/tensorflow/python/keras/engine/base_layer_utils.py b/tensorflow/python/keras/engine/base_layer_utils.py
index 5d39038ddd..cdc861f974 100644
--- a/tensorflow/python/keras/engine/base_layer_utils.py
+++ b/tensorflow/python/keras/engine/base_layer_utils.py
@@ -237,8 +237,7 @@ def create_keras_history(tensors):
   This method checks to see if a Tensor in `tensors` is missing Keras metadata
   and has its origin in a Keras `Input` Layer. If so, this method will replace
   the raw TensorFlow Operations that created this tensor with
-  `TensorFlowOpLayer`
-  instances that create identical operations.
+  `TensorFlowOpLayer` instances that create identical operations.
 
   Any Tensors not originating from a Keras `Input` Layer will be treated as
   constants when constructing `TensorFlowOpLayer` instances.
@@ -247,17 +246,7 @@ def create_keras_history(tensors):
     tensors: A structure of Tensors, some of which come from raw TensorFlow
       operations and need to have Keras metadata assigned to them.
   """
-
-  try:
-    _create_keras_history_helper(tensors, set())
-  except AttributeError:
-    # This can happen with sublayers inside of layers in the Functional API.
-    # The error occurs when a Functional Model is running in V2 function
-    # mode and a non-Keras Tensor is passed as an input to a sublayer inside
-    # of another layer.
-    # TODO(omalleyt): Only run `create_keras_history` during Functional API
-    # creation phase.
-    pass
+  _create_keras_history_helper(tensors, set())
 
 
 def _create_keras_history_helper(tensors, processed_ops=None):
@@ -265,8 +254,8 @@ def _create_keras_history_helper(tensors, processed_ops=None):
 
   Arguments:
     tensors: A structure of Tensors for which to create Keras metadata.
-    processed_ops: TensorFlow operations that have already been wrapped in
-      `TensorFlowOpLayer` instances.
+    processed_ops: Set. TensorFlow operations that have already been wrapped
+      in `TensorFlowOpLayer` instances.
 
   Returns:
     The updated set of TensorFlow Operations that have been wrapped
@@ -287,7 +276,7 @@ def _create_keras_history_helper(tensors, processed_ops=None):
       constants = {}
       layer_inputs = []
       for i, op_input in enumerate(op_inputs):
-        if uses_keras_input_layers(op_input):
+        if uses_keras_history(op_input):
           layer_inputs.append(op_input)
         else:
           # Treat any value not originating from a `keras.Input` as
@@ -306,33 +295,76 @@ def _create_keras_history_helper(tensors, processed_ops=None):
   return processed_ops
 
 
-def uses_keras_input_layers(tensors):
-  """Checks if at least one Tensor in `tensors` originates from a Keras `Input`.
+def needs_keras_history(tensors):
+  """Check if any Tensors need to be wrapped in TensorFlowOpLayers.
 
-  If so, the Functional API is being used.
+  Arguments:
+    tensors: An arbitrary nested structure of Tensors.
+
+  Returns:
+    Bool, whether at least one Tensor needs to be wrapped.
+  """
+  input_tensors = nest.flatten(tensors)
+  if all(
+      getattr(tensor, '_keras_history', None) is not None
+      for tensor in input_tensors):
+    # KerasHistory already set.
+    return False
+  return uses_keras_history(tensors)
+
+
+def uses_keras_history(tensors):
+  """Check if at least one Tensor originates from a `keras.Input`.
+
+  This is `True` if at least one Tensor has its origin in a `keras.Input`.
+  Any Tensor that originates from a `keras.Input` will have a dependency
+  Tensor with a `_keras_history` attribute attached. Tensors that have
+  already been checked to not originate from a `keras.Input`
+  are marked as `_keras_history_checked`.
 
   Arguments:
     tensors: An arbitrary nested structure of Tensors.
 
   Returns:
-    Bool, whether at least one Tensor originates from a Keras `Input`.
+    Bool, whether at least one Tensor originates from a `keras.Input`.
   """
   checked_tensors = set()
-  input_tensors = nest.flatten(tensors)
+  tensors_to_check = nest.flatten(tensors)
+
+  while tensors_to_check:
+    new_tensors_to_check = set()
+    for tensor in tensors_to_check:
+      if getattr(tensor, '_keras_history_checked', None) is not None:
+        continue
+      if getattr(tensor, '_keras_history', None) is not None:
+        return True
 
-  while input_tensors:
-    if any(
-        getattr(tensor, '_keras_history', None) is not None
-        for tensor in input_tensors):
-      return True
-    checked_tensors.update(input_tensors)
-    new_input_tensors = set()
-    for tensor in input_tensors:
       try:
-        new_input_tensors.update(tensor.op.inputs)
+        new_tensors_to_check.update(tensor.op.inputs)
       except AttributeError:
-        # In case `tensor` is a Variable created in an Eager
-        # context
+        # In case `tensor` is a Variable created in an Eager context.
         pass
-    input_tensors = list(new_input_tensors - checked_tensors)
+
+    checked_tensors.update(tensors_to_check)
+    tensors_to_check = list(new_tensors_to_check - checked_tensors)
+
+  # Mark that these Tensors have been checked once for `_keras_history`,
+  # and should not be checked again for performance reasons.
+  mark_checked(tensors)
   return False
+
+
+def mark_checked(tensors):
+  """Marks that these Tensors should not be tracked.
+
+  This prevents Layers from attempting to create TensorFlowOpLayers
+  for these Tensors.
+
+  Arguments:
+    tensors: An arbitrary structure of Tensors.
+  """
+
+  def _mark_checked(tensor):
+    tensor._keras_history_checked = True  # pylint: disable=protected-access
+
+  nest.map_structure(_mark_checked, tensors)
diff --git a/tensorflow/python/keras/layers/tensorflow_op_layer_test.py b/tensorflow/python/keras/layers/tensorflow_op_layer_test.py
new file mode 100644
index 0000000000..b83508ee66
--- /dev/null
+++ b/tensorflow/python/keras/layers/tensorflow_op_layer_test.py
@@ -0,0 +1,169 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test for allowing TF ops to work with Keras Functional API."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import time
+
+from absl.testing import parameterized
+import numpy as np
+
+from tensorflow.python import keras
+from tensorflow.python.framework import ops
+from tensorflow.python.keras import keras_parameterized
+from tensorflow.python.keras import testing_utils
+from tensorflow.python.keras.optimizer_v2 import adam
+from tensorflow.python.ops import gen_nn_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.platform import test
+from tensorflow.python.util import nest
+
+
+def _single_op_at_end():
+  inputs = keras.Input(shape=(10,))
+  x = keras.layers.Dense(10)(inputs)
+  outputs = gen_nn_ops.relu(x)
+  return inputs, outputs
+
+
+def _multiple_ops_at_end():
+  inputs = keras.Input(shape=(10,))
+  x = keras.layers.Dense(10)(inputs)
+  x = gen_nn_ops.relu(x)
+  outputs = gen_nn_ops.relu(x)
+  return inputs, outputs
+
+
+def _single_op_in_middle():
+  inputs = keras.Input(shape=(10,))
+  x = keras.layers.Dense(10)(inputs)
+  x = gen_nn_ops.relu(x)
+  outputs = keras.layers.Dense(10)(x)
+  return inputs, outputs
+
+
+def _multiple_ops_in_middle():
+  inputs = keras.Input(shape=(10,))
+  x = keras.layers.Dense(10)(inputs)
+  x = gen_nn_ops.relu(x)
+  x = gen_nn_ops.relu(x)
+  outputs = keras.layers.Dense(10)(x)
+  return inputs, outputs
+
+
+def _single_standalone_branch():
+  inputs = keras.Input(shape=(10,))
+  x = keras.layers.Dense(10)(inputs)
+  outputs = x * 2
+  return inputs, outputs
+
+
+def _single_op_with_attrs():
+  inputs = keras.Input(shape=(10,))
+  x = math_ops.reduce_mean(inputs, axis=1, keepdims=True)
+  outputs = keras.layers.Dense(10)(x)
+  return inputs, outputs
+
+
+def _multiple_uses():
+  inputs = keras.Input(shape=(10,))
+  x = math_ops.reduce_mean(inputs, axis=1, keepdims=True)
+  x1 = keras.layers.Dense(10)(x)
+  x2 = keras.layers.Dense(10)(x)
+  outputs = x1 + x2
+  return inputs, outputs
+
+
+@keras_parameterized.run_all_keras_modes
+class AutoLambdaTest(keras_parameterized.TestCase):
+
+  @parameterized.named_parameters(
+      ('single_op_at_end', _single_op_at_end),
+      ('multiple_ops_at_end', _multiple_ops_at_end),
+      ('single_op_in_middle', _single_op_in_middle),
+      ('multiple_ops_in_middle', _multiple_ops_in_middle),
+      ('single_standalone_branch', _single_standalone_branch),
+      ('single_op_with_attrs', _single_op_with_attrs),
+      ('multiple_uses', _multiple_uses))
+  def test_autolambda(self, model_fn):
+    inputs, outputs = model_fn()
+    model = keras.Model(inputs, outputs)
+    model.compile(
+        adam.Adam(0.001), 'mse', run_eagerly=testing_utils.should_run_eagerly())
+
+    np_inputs = nest.map_structure(lambda x: np.ones((10, 10), 'float32'),
+                                   inputs)
+    np_outputs = nest.map_structure(lambda x: np.ones((10, 10), 'float32'),
+                                    outputs)
+    model.fit(np_inputs, np_outputs, batch_size=2)
+
+  def test_numerical_correctness_simple(self):
+    x = ops.convert_to_tensor([[-1., 0., -2., 1.]])
+    inputs = keras.Input(shape=(4,))
+    outputs = gen_nn_ops.relu(inputs)
+    model = keras.Model(inputs, outputs)
+    y = self.evaluate(model(x))
+    self.assertAllClose(y, [[0., 0., 0., 1.]])
+
+  def test_numerical_correctness_with_attrs(self):
+    x = ops.convert_to_tensor([[1.5, 1.5], [2.5, 3.5]])
+    inputs = keras.Input(shape=(10,))
+    outputs = math_ops.reduce_mean(inputs, axis=1)
+    model = keras.Model(inputs, outputs)
+    y = self.evaluate(model(x))
+    self.assertAllClose(y, [1.5, 3.])
+
+  def test_serialization(self):
+    x = ops.convert_to_tensor([-1., 0., -2., 1.])
+    inputs = keras.Input(shape=(4,))
+    outputs = gen_nn_ops.relu(inputs)
+    model1 = keras.Model(inputs, outputs)
+    y1 = self.evaluate(model1(x))
+    model2 = model1.from_config(model1.get_config())
+    y2 = self.evaluate(model2(x))
+    self.assertAllClose(y1, y2)
+
+  def test_no_tracking(self):
+    x = keras.backend.placeholder((10, 10))
+    keras.layers.Dense(1)(x)
+    self.assertTrue(x._keras_history_checked)
+
+  def test_timing_scales_linearly(self):
+
+    def _construct_graph_of_size(size):
+      start = time.time()
+      x = keras.backend.placeholder(shape=(10, 4))
+
+      for _ in range(size):
+        x = keras.layers.Dense(4)(x)
+        x = gen_nn_ops.relu(x)
+
+      end = time.time()
+      return end - start
+
+    size_50 = _construct_graph_of_size(50)
+    size_500 = _construct_graph_of_size(500)
+
+    # Check reasonable graph construction time.
+    self.assertLess(size_50, 5)
+    # Check construction time grows approx. linearly with size.
+    e = 1.5  # Fudge factor to prevent flakiness.
+    self.assertLess(size_500, (10 * e) * size_50)
+
+
+if __name__ == '__main__':
+  test.main()
-- 
GitLab


From c30e777661b983d449b02b06453a457913d94148 Mon Sep 17 00:00:00 2001
From: Billy Lamberta <blamb@google.com>
Date: Mon, 28 Jan 2019 13:33:26 -0800
Subject: [PATCH 1390/2345] Fix link for Lite python_api page.

PiperOrigin-RevId: 231279838
---
 tensorflow/lite/g3doc/apis.md                | 2 +-
 tensorflow/lite/g3doc/using_select_tf_ops.md | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/tensorflow/lite/g3doc/apis.md b/tensorflow/lite/g3doc/apis.md
index 1a05142bc4..28af7e25f8 100644
--- a/tensorflow/lite/g3doc/apis.md
+++ b/tensorflow/lite/g3doc/apis.md
@@ -7,7 +7,7 @@ no surprise that the APIs try to avoid unnecessary copies at the expense of
 convenience.  Similarly, consistency with TensorFlow APIs was not an explicit
 goal and some variance is to be expected.
 
-There is also a [Python API for TensorFlow Lite](g3doc/convert/python_api.md).
+There is also a [Python API for TensorFlow Lite](./convert/python_api.md).
 
 ## C++
 
diff --git a/tensorflow/lite/g3doc/using_select_tf_ops.md b/tensorflow/lite/g3doc/using_select_tf_ops.md
index 269774a4b1..1c5f23b7bb 100644
--- a/tensorflow/lite/g3doc/using_select_tf_ops.md
+++ b/tensorflow/lite/g3doc/using_select_tf_ops.md
@@ -49,8 +49,7 @@ partially supported by TensorFlow Lite, and one would like to avoid those
 limitations.
 
 The following example shows how to use `target_ops` in the
-[`TFLiteConverter`](https://www.tensorflow.org/lite/convert/python_api) Python
-API.
+[`TFLiteConverter`](./convert/python_api) Python API.
 
 ```
 import tensorflow as tf
-- 
GitLab


From 3766d18ed705695c8253bf4b35ba87361174a738 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 28 Jan 2019 13:35:56 -0800
Subject: [PATCH 1391/2345] [Grappler] Fix the structure_verifier to take the
 union of the global OpRegistry and the graph's function library.

Without this, the verifier would incorrectly flag ops from the function library as not being registered.

PiperOrigin-RevId: 231280297
---
 .../grappler/optimizers/meta_optimizer_test.cc    | 15 ++++++++-------
 .../core/grappler/verifiers/structure_verifier.cc |  6 +++++-
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc b/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
index 55f14b3b68..0970134ed2 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
@@ -824,7 +824,6 @@ TEST_F(MetaOptimizerTest, RunPostOptimizationVerifiersOnInvalidGraph) {
       {NDef("x0", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
        NDef("x1", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
        NDef("dy", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
-       NDef("x1", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
        // Calls into function library
        NDef("mul_1", "MyMul1", {"x0", "x1"}, {}, kDevice),
        NDef("mul_2", "MyMul2", {"x0", "x1"}, {}, kDevice),
@@ -857,9 +856,10 @@ TEST_F(MetaOptimizerTest, RunPostOptimizationVerifiersOnInvalidGraph) {
   MetaOptimizer optimizer_with_post_verifiers(nullptr, config_proto);
   Status status =
       optimizer_with_post_verifiers.Optimize(nullptr, item, &output);
-  EXPECT_EQ(status.code(), errors::Code::NOT_FOUND);
-  EXPECT_TRUE(
-      absl::StrContains(status.error_message(), "Op type not registered"));
+  EXPECT_EQ(status.code(), errors::Code::INVALID_ARGUMENT);
+  EXPECT_TRUE(absl::StrContains(
+      status.error_message(),
+      "NodeDef expected inputs 'float' do not match 3 inputs specified"));
 }
 
 TEST_F(MetaOptimizerTest, RunInterOptimizerVerifiersOnInvalidGraph) {
@@ -931,9 +931,10 @@ TEST_F(MetaOptimizerTest, RunInterOptimizerVerifiersOnInvalidGraph) {
   MetaOptimizer optimizer_with_inter_verifiers(nullptr, config_proto);
   Status status =
       optimizer_with_inter_verifiers.Optimize(nullptr, item, &output);
-  EXPECT_EQ(status.code(), errors::Code::NOT_FOUND);
-  EXPECT_TRUE(
-      absl::StrContains(status.error_message(), "Op type not registered"));
+  EXPECT_EQ(status.code(), errors::Code::INVALID_ARGUMENT);
+  EXPECT_TRUE(absl::StrContains(
+      status.error_message(),
+      "NodeDef expected inputs 'float' do not match 3 inputs specified"));
 }
 
 }  // namespace
diff --git a/tensorflow/core/grappler/verifiers/structure_verifier.cc b/tensorflow/core/grappler/verifiers/structure_verifier.cc
index ab0a2ed526..2b438b56c4 100644
--- a/tensorflow/core/grappler/verifiers/structure_verifier.cc
+++ b/tensorflow/core/grappler/verifiers/structure_verifier.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include <string>
 #include <vector>
 
+#include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/op.h"
@@ -32,8 +33,11 @@ namespace grappler {
 // TODO(ashwinm): Expand this to add more structural checks.
 Status StructureVerifier::Verify(const GraphDef& graph) {
   StatusGroup status_group;
+
+  FunctionLibraryDefinition function_library(OpRegistry::Global(),
+                                             graph.library());
   status_group.Update(tensorflow::graph::ValidateGraphDefAgainstOpRegistry(
-      graph, *OpRegistry::Global()));
+      graph, function_library));
   status_group.Update(tensorflow::graph::VerifyNoDuplicateNodeNames(graph));
 
   std::vector<const NodeDef*> topo_order;
-- 
GitLab


From 6376dddf050490e750b1a7ef7e7456228c24ca26 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Mon, 28 Jan 2019 13:42:26 -0800
Subject: [PATCH 1392/2345] [tf.data] Avoid printing a spurious warning for
 `tf.compat.v1.data.make_one_shot_iterator` and
 `tf.compat.v1.data.make_initializable_iterator`.

PiperOrigin-RevId: 231281365
---
 tensorflow/python/data/ops/dataset_ops.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index 104f99a3d4..766c6d5395 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -1364,6 +1364,9 @@ class DatasetV1(DatasetV2):
     Returns:
       An `Iterator` over the elements of this dataset.
     """
+    return self._make_one_shot_iterator()
+
+  def _make_one_shot_iterator(self):  # pylint: disable=missing-docstring
     if context.executing_eagerly():
       return iterator_ops.EagerIterator(self)
 
@@ -1441,6 +1444,10 @@ class DatasetV1(DatasetV2):
     Raises:
       RuntimeError: If eager execution is enabled.
     """
+
+    return self._make_initializable_iterator(shared_name)
+
+  def _make_initializable_iterator(self, shared_name=None):  # pylint: disable=missing-docstring
     if context.executing_eagerly():
       raise RuntimeError(
           "dataset.make_initializable_iterator is not supported when eager "
@@ -1725,11 +1732,11 @@ def make_one_shot_iterator(dataset):
     A `tf.data.Iterator` over the elements of this dataset.
   """
   try:
-    # Call the defined `make_one_shot_iterator()` if there is one, because some
+    # Call the defined `_make_one_shot_iterator()` if there is one, because some
     # datasets (e.g. for prefetching) override its behavior.
-    return dataset.make_one_shot_iterator()
+    return dataset._make_one_shot_iterator()  # pylint: disable=protected-access
   except AttributeError:
-    return DatasetV1Adapter(dataset).make_one_shot_iterator()
+    return DatasetV1Adapter(dataset)._make_one_shot_iterator()  # pylint: disable=protected-access
 
 
 @tf_export(v1=["data.make_initializable_iterator"])
@@ -1759,11 +1766,11 @@ def make_initializable_iterator(dataset, shared_name=None):
     RuntimeError: If eager execution is enabled.
   """
   try:
-    # Call the defined `make_initializable_iterator()` if there is one, because
+    # Call the defined `_make_initializable_iterator()` if there is one, because
     # some datasets (e.g. for prefetching) override its behavior.
-    return dataset.make_initializable_iterator(shared_name)
+    return dataset._make_initializable_iterator(shared_name)  # pylint: disable=protected-access
   except AttributeError:
-    return DatasetV1Adapter(dataset).make_initializable_iterator(shared_name)
+    return DatasetV1Adapter(dataset)._make_initializable_iterator(shared_name)  # pylint: disable=protected-access
 
 
 @tf_export("data.Options")
-- 
GitLab


From d8064519ee36e958ab95c1d0df12b1e615194ecf Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Tue, 13 Mar 2018 17:53:11 +0000
Subject: [PATCH 1393/2345] Add zero padding option to
 `tf.image.extract_glimpse`

There has been several feature request in 16663 and 2134
asking for zero padding option to `tf.image.extract_glimpse`.
This fix is an attempt to address the issue through:
1. Add an additional args of `noise` so that it is possible
to specify `uniform`, `gaussian`, or `zero`.
2. In case `zero` is specified, extract_glimpse will fill
padding with zeros.

Note this PR keeps the original arg `uniform_noise` intact
(unless it conflict with `noise`), though it is possible
to deprecate the `uniform_noise` if desired.

This fix fixes 16663.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/kernels/attention_ops.cc  | 16 +++++++++++++---
 tensorflow/core/kernels/eigen_attention.h | 17 ++++++++++-------
 2 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/tensorflow/core/kernels/attention_ops.cc b/tensorflow/core/kernels/attention_ops.cc
index ce2fce92e4..25f2c81ecb 100644
--- a/tensorflow/core/kernels/attention_ops.cc
+++ b/tensorflow/core/kernels/attention_ops.cc
@@ -34,7 +34,17 @@ class ExtractGlimpseOp : public OpKernel {
   explicit ExtractGlimpseOp(OpKernelConstruction* context) : OpKernel(context) {
     OP_REQUIRES_OK(context, context->GetAttr("normalized", &normalized_));
     OP_REQUIRES_OK(context, context->GetAttr("centered", &centered_));
-    OP_REQUIRES_OK(context, context->GetAttr("uniform_noise", &uniform_noise_));
+    bool uniform_noise = false;
+    OP_REQUIRES_OK(context, context->GetAttr("uniform_noise", &uniform_noise));
+    OP_REQUIRES_OK(context, context->GetAttr("noise", &noise_));
+    OP_REQUIRES(context, !uniform_noise || noise_ == "",
+                errors::InvalidArgument("The uniform_noise and noise could not be specified at the same time"));
+    if (noise_ == "") {
+      noise_ = uniform_noise ? "uniform" : "gaussian";
+    } else {
+      OP_REQUIRES(context, noise_ == "uniform" || noise_ == "gaussian" | noise_ == "zero",
+                  errors::InvalidArgument("The noise could only be uniform, gaussian, or zero, got", noise_));
+    }
   }
 
   // Expect input tensor of rank 4 with dimensions (batch_size, height, width,
@@ -98,13 +108,13 @@ class ExtractGlimpseOp : public OpKernel {
         context->eigen_cpu_device()) =
         Eigen::ExtractGlimpses(input.tensor<float, 4>().swap_layout(),
                                output_width, output_height, offset_vec,
-                               normalized_, centered_, uniform_noise_);
+                               normalized_, centered_, noise_);
   }
 
  private:
   bool normalized_;
   bool centered_;
-  bool uniform_noise_;
+  string noise_;
 };
 
 REGISTER_KERNEL_BUILDER(Name("ExtractGlimpse").Device(DEVICE_CPU),
diff --git a/tensorflow/core/kernels/eigen_attention.h b/tensorflow/core/kernels/eigen_attention.h
index 4d86f9deb9..629731c1e5 100644
--- a/tensorflow/core/kernels/eigen_attention.h
+++ b/tensorflow/core/kernels/eigen_attention.h
@@ -48,13 +48,13 @@ struct GlimpseExtractionOp {
   GlimpseExtractionOp(const Index width, const Index height,
                       const std::vector<IndexPair<float> >& offsets,
                       const bool normalized, const bool centered,
-                      const bool uniform_noise)
+                      const std::string &noise)
       : width_(width),
         height_(height),
         offsets_(offsets),
         normalized_(normalized),
         centered_(centered),
-        uniform_noise_(uniform_noise) {}
+        noise_(noise) {}
 
   template <typename Input>
   DSizes<Index, 4> dimensions(const Input& input) const {
@@ -144,8 +144,11 @@ struct GlimpseExtractionOp {
       slice_extent[2] = std::min<Index>(input_height, slice_extent[2]);
 
       if (partial_overlap) {
-        if (uniform_noise_) {
-          // Initialize the glimpse with uniform noise.
+        if (noise_ == "zero") {
+           // Initialize the glimpse with zero noise.
+          output.template chip<3>(i).setZero();
+        } else if (noise_ == "uniform") {
+           // Initialize the glimpse with uniform noise.
           typedef typename internal::remove_const<
               typename internal::traits<Input>::Scalar>::type Scalar;
           TensorFixedSize<Scalar, Sizes<> > mini;
@@ -225,7 +228,7 @@ struct GlimpseExtractionOp {
   const std::vector<IndexPair<float> > offsets_;
   const bool normalized_;
   const bool centered_;
-  const bool uniform_noise_;
+  const std::string noise_;
 };
 }  // namespace
 
@@ -238,7 +241,7 @@ ExtractGlimpses(const Input& input,
                 const typename internal::traits<Input>::Index height,
                 const std::vector<IndexPair<float> >& offsets,
                 const bool normalized = true, const bool centered = true,
-                const bool uniform_noise = true) {
+                const std::string &noise = "uniform") {
   EIGEN_STATIC_ASSERT(internal::traits<Input>::Layout == ColMajor,
                       YOU_MADE_A_PROGRAMMING_MISTAKE);
   EIGEN_STATIC_ASSERT(internal::traits<Input>::NumDimensions == 4,
@@ -246,7 +249,7 @@ ExtractGlimpses(const Input& input,
 
   typedef typename internal::traits<Input>::Index Index;
   const GlimpseExtractionOp<Index> op(width, height, offsets, normalized,
-                                      centered, uniform_noise);
+                                      centered, noise);
   return input.customOp(op);
 }
 
-- 
GitLab


From 2c409af18d58b38385d5b080f7bffc8d435981e1 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Tue, 13 Mar 2018 17:57:25 +0000
Subject: [PATCH 1394/2345] Update tensorflow/core/ops/image_ops.cc

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/ops/image_ops.cc | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tensorflow/core/ops/image_ops.cc b/tensorflow/core/ops/image_ops.cc
index 0f1555f49c..b23200171d 100644
--- a/tensorflow/core/ops/image_ops.cc
+++ b/tensorflow/core/ops/image_ops.cc
@@ -606,6 +606,7 @@ REGISTER_OP("ExtractGlimpse")
     .Attr("centered: bool = true")
     .Attr("normalized: bool = true")
     .Attr("uniform_noise: bool = true")
+    .Attr("noise: string = ''")
     .SetShapeFn([](InferenceContext* c) {
       ShapeHandle input;
       TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 4, &input));
@@ -618,6 +619,14 @@ REGISTER_OP("ExtractGlimpse")
       DimensionHandle unused;
       TF_RETURN_IF_ERROR(c->WithValue(c->Dim(offsets, 1), 2, &unused));
 
+      bool uniform_noise = false;
+      TF_RETURN_IF_ERROR(c->GetAttr("uniform_noise", &uniform_noise));
+      string noise;
+      TF_RETURN_IF_ERROR(c->GetAttr("noise", &noise));
+      if (uniform_noise && noise != "") {
+        return errors::InvalidArgument("The uniform_noise and noise could not be specified at the same time");
+      }
+
       return SetOutputToSizedImage(c, batch_dim, 1 /* size_input_idx */,
                                    c->Dim(input, 3));
     });
-- 
GitLab


From a8105c7d2a9c9dd7052967bb0745e905ef5c65f6 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Tue, 13 Mar 2018 17:58:03 +0000
Subject: [PATCH 1395/2345] Add test case for additional arg with
 extract_glimpse

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/kernel_tests/attention_ops_test.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/tensorflow/python/kernel_tests/attention_ops_test.py b/tensorflow/python/kernel_tests/attention_ops_test.py
index 00dba9996d..a90362645e 100644
--- a/tensorflow/python/kernel_tests/attention_ops_test.py
+++ b/tensorflow/python/kernel_tests/attention_ops_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import image_ops
 from tensorflow.python.platform import test
@@ -195,5 +196,15 @@ class ExtractGlimpseTest(test.TestCase):
         expected_rows=[None, None, None, 1, 2, 3, 4],
         expected_cols=[56, 57, 58, 59, 60])
 
+  def testGlimpseNoOverlapZero(self):
+    img = constant_op.constant(np.arange(25).reshape((1, 5, 5, 1)),
+                               dtype=dtypes.float32)
+    with self.test_session():
+      result = image_ops.extract_glimpse(img, [3, 3], [[-2, 2]],
+                                         centered=False, normalized=False,
+                                         uniform_noise=False, noise="zero")
+      self.assertAllEqual(np.asarray([[0, 0, 0], [0, 0, 0], [0, 0, 0]]),
+                          result.eval()[0, :, :, 0])
+
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From fc8c61ccfae0bea7b8675342875b0e5138418c01 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Tue, 13 Mar 2018 20:02:36 +0000
Subject: [PATCH 1396/2345] Add additional test cases

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .../python/kernel_tests/attention_ops_test.py | 44 ++++++++++++++++---
 1 file changed, 38 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/kernel_tests/attention_ops_test.py b/tensorflow/python/kernel_tests/attention_ops_test.py
index a90362645e..2db74157fe 100644
--- a/tensorflow/python/kernel_tests/attention_ops_test.py
+++ b/tensorflow/python/kernel_tests/attention_ops_test.py
@@ -196,15 +196,47 @@ class ExtractGlimpseTest(test.TestCase):
         expected_rows=[None, None, None, 1, 2, 3, 4],
         expected_cols=[56, 57, 58, 59, 60])
 
-  def testGlimpseNoOverlapZero(self):
+  def testGlimpseNoiseZero(self):
+    # Image:
+    # [  0.   1.   2.   3.   4.]
+    # [  5.   6.   7.   8.   9.]
+    # [ 10.  11.  12.  13.  14.]
+    # [ 15.  16.  17.  18.  19.]
+    # [ 20.  21.  22.  23.  24.]
     img = constant_op.constant(np.arange(25).reshape((1, 5, 5, 1)),
                                dtype=dtypes.float32)
     with self.test_session():
-      result = image_ops.extract_glimpse(img, [3, 3], [[-2, 2]],
-                                         centered=False, normalized=False,
-                                         uniform_noise=False, noise="zero")
-      self.assertAllEqual(np.asarray([[0, 0, 0], [0, 0, 0], [0, 0, 0]]),
-                          result.eval()[0, :, :, 0])
+      # Result 1:
+      # [ 0.  0.  0.]
+      # [ 0.  0.  0.]
+      # [ 0.  0.  0.]
+      result1 = image_ops.extract_glimpse(img, [3, 3], [[-2, 2]],
+                                          centered=False, normalized=False,
+                                          uniform_noise=False, noise="zero")
+      self.assertAllEqual(np.asarray([[0, 0, 0],
+                                      [0, 0, 0],
+                                      [0, 0, 0]]),
+                          result1.eval()[0, :, :, 0])
+
+      # Result 2:
+      # [  0.   0.   0.   0.   0.   0.   0.]
+      # [  0.   0.   1.   2.   3.   4.   0.]
+      # [  0.   5.   6.   7.   8.   9.   0.]
+      # [  0.  10.  11.  12.  13.  14.   0.]
+      # [  0.  15.  16.  17.  18.  19.   0.]
+      # [  0.  20.  21.  22.  23.  24.   0.]
+      # [  0.   0.   0.   0.   0.   0.   0.]]
+      result2 = image_ops.extract_glimpse(img, [7, 7], [[0, 0]],
+                                          normalized=False,
+                                          uniform_noise=False, noise="zero")
+      self.assertAllEqual(np.asarray([[0, 0, 0, 0, 0, 0, 0],
+                                      [0, 0, 1, 2, 3, 4, 0],
+                                      [0, 5, 6, 7, 8, 9, 0],
+                                      [0, 10, 11, 12, 13, 14, 0],
+                                      [0, 15, 16, 17, 18, 19, 0],
+                                      [0, 20, 21, 22, 23, 24, 0],
+                                      [0, 0, 0, 0, 0, 0, 0]]),
+                          result2.eval()[0, :, :, 0])
 
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From d2744e5792f5d6d82b957d8f85c49cd7be6df022 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Tue, 13 Mar 2018 20:20:06 +0000
Subject: [PATCH 1397/2345] Update goldens API

with
```
bazel-bin/tensorflow/tools/api/tests/api_compatibility_test
           --update_goldens True
```

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt
index 15d0e099ba..d2a9182f2c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt
@@ -82,7 +82,7 @@ tf_module {
   }
   member_method {
     name: "extract_glimpse"
-    argspec: "args=[\'input\', \'size\', \'offsets\', \'centered\', \'normalized\', \'uniform_noise\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'True\', \'True\', \'None\'], "
+    argspec: "args=[\'input\', \'size\', \'offsets\', \'centered\', \'normalized\', \'uniform_noise\', \'noise\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'True\', \'True\', \'\', \'None\'], "
   }
   member_method {
     name: "extract_image_patches"
-- 
GitLab


From c9d794ab93d27280f8da6439a6247f5cc57f722d Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Tue, 13 Mar 2018 22:04:19 +0000
Subject: [PATCH 1398/2345] Update the api_def

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .../core/api_def/base_api/api_def_ExtractGlimpse.pbtxt    | 8 ++++++++
 tensorflow/core/ops/image_ops_test.cc                     | 7 +++++++
 2 files changed, 15 insertions(+)

diff --git a/tensorflow/core/api_def/base_api/api_def_ExtractGlimpse.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExtractGlimpse.pbtxt
index c10a1bb778..0ed7ccad26 100644
--- a/tensorflow/core/api_def/base_api/api_def_ExtractGlimpse.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ExtractGlimpse.pbtxt
@@ -48,6 +48,14 @@ END
     description: <<END
 indicates if the noise should be generated using a
 uniform distribution or a Gaussian distribution.
+END
+  }
+  attr {
+    name: "noise"
+    description: <<END
+indicates if the noise should `uniform`, `gaussian`, or
+`zero`. The default is `` which means the the noise type
+will be decided by `uniform_noise`.
 END
   }
   summary: "Extracts a glimpse from the input tensor."
diff --git a/tensorflow/core/ops/image_ops_test.cc b/tensorflow/core/ops/image_ops_test.cc
index 517af26b44..e517e75095 100644
--- a/tensorflow/core/ops/image_ops_test.cc
+++ b/tensorflow/core/ops/image_ops_test.cc
@@ -183,6 +183,13 @@ TEST(ImageOpsTest, ExtractGlimpse_ShapeFn) {
   op.input_tensors.resize(2);
 
   // Inputs are input, size, offsets.
+  TF_ASSERT_OK(NodeDefBuilder("test", "ExtractGlimpse")
+                   .Input({"input", 0, DT_FLOAT})
+                   .Input({"size", 1, DT_INT32})
+                   .Input({"offsets", 2, DT_FLOAT})
+                   .Attr("uniform_noise", true)
+                   .Attr("noise", "")
+                   .Finalize(&op.node_def));
 
   // Rank and size checks.
   INFER_ERROR("Shape must be rank 4 but is rank 5", op, "[1,2,3,4,5];?;?");
-- 
GitLab


From a44bb307d4e60eca4f842f8c78a5824db2ca98a4 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Fri, 23 Mar 2018 01:40:31 +0000
Subject: [PATCH 1399/2345] Preserve the GPU compatibility

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/kernels/eigen_attention.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/kernels/eigen_attention.h b/tensorflow/core/kernels/eigen_attention.h
index 629731c1e5..041ba12594 100644
--- a/tensorflow/core/kernels/eigen_attention.h
+++ b/tensorflow/core/kernels/eigen_attention.h
@@ -145,8 +145,9 @@ struct GlimpseExtractionOp {
 
       if (partial_overlap) {
         if (noise_ == "zero") {
-           // Initialize the glimpse with zero noise.
-          output.template chip<3>(i).setZero();
+          // Initialize the glimpse with zero noise.
+          output.template chip<3>(i).device(device) =
+              output.template chip<3>(i).constant(0);
         } else if (noise_ == "uniform") {
            // Initialize the glimpse with uniform noise.
           typedef typename internal::remove_const<
-- 
GitLab


From ce28d6021b23a471333e4c26faba62ed777724b8 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Fri, 23 Mar 2018 03:30:18 +0000
Subject: [PATCH 1400/2345] Convert string to enum for performance/GPU

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/kernels/attention_ops.cc  |  34 +++--
 tensorflow/core/kernels/eigen_attention.h | 147 ++++++++++++----------
 2 files changed, 103 insertions(+), 78 deletions(-)

diff --git a/tensorflow/core/kernels/attention_ops.cc b/tensorflow/core/kernels/attention_ops.cc
index 25f2c81ecb..58c7aa3506 100644
--- a/tensorflow/core/kernels/attention_ops.cc
+++ b/tensorflow/core/kernels/attention_ops.cc
@@ -35,15 +35,28 @@ class ExtractGlimpseOp : public OpKernel {
     OP_REQUIRES_OK(context, context->GetAttr("normalized", &normalized_));
     OP_REQUIRES_OK(context, context->GetAttr("centered", &centered_));
     bool uniform_noise = false;
+    string noise;
     OP_REQUIRES_OK(context, context->GetAttr("uniform_noise", &uniform_noise));
-    OP_REQUIRES_OK(context, context->GetAttr("noise", &noise_));
-    OP_REQUIRES(context, !uniform_noise || noise_ == "",
-                errors::InvalidArgument("The uniform_noise and noise could not be specified at the same time"));
-    if (noise_ == "") {
-      noise_ = uniform_noise ? "uniform" : "gaussian";
+    OP_REQUIRES_OK(context, context->GetAttr("noise", &noise));
+    OP_REQUIRES(context, !uniform_noise || noise == "",
+                errors::InvalidArgument("The uniform_noise and noise could not "
+                                        "be specified at the same time"));
+    if (noise == "") {
+      noise_ = uniform_noise ? Eigen::ExtractGlimpsesNoiseMode::UNIFORM
+                             : Eigen::ExtractGlimpsesNoiseMode::GAUSSIAN;
     } else {
-      OP_REQUIRES(context, noise_ == "uniform" || noise_ == "gaussian" | noise_ == "zero",
-                  errors::InvalidArgument("The noise could only be uniform, gaussian, or zero, got", noise_));
+      OP_REQUIRES(context,
+                  noise == "uniform" || noise == "gaussian" || noise == "zero",
+                  errors::InvalidArgument(
+                      "The noise could only be uniform, gaussian, or zero, got",
+                      noise));
+      if (noise == "uniform") {
+        noise_ = Eigen::ExtractGlimpsesNoiseMode::UNIFORM;
+      } else if (noise == "gaussian") {
+        noise_ = Eigen::ExtractGlimpsesNoiseMode::GAUSSIAN;
+      } else {
+        noise_ = Eigen::ExtractGlimpsesNoiseMode::ZERO;
+      }
     }
   }
 
@@ -62,9 +75,8 @@ class ExtractGlimpseOp : public OpKernel {
     const int64 batch_size = input_shape.dim_size(0);
 
     const Tensor& window_size = context->input(1);
-    OP_REQUIRES(context,
-                (window_size.shape().dims() == 1) &&
-                    window_size.shape().dim_size(0) == 2,
+    OP_REQUIRES(context, (window_size.shape().dims() == 1) &&
+                             window_size.shape().dim_size(0) == 2,
                 errors::InvalidArgument(
                     "input must be a vector of size 2 (height, width)",
                     window_size.shape().DebugString()));
@@ -114,7 +126,7 @@ class ExtractGlimpseOp : public OpKernel {
  private:
   bool normalized_;
   bool centered_;
-  string noise_;
+  Eigen::ExtractGlimpsesNoiseMode noise_;
 };
 
 REGISTER_KERNEL_BUILDER(Name("ExtractGlimpse").Device(DEVICE_CPU),
diff --git a/tensorflow/core/kernels/eigen_attention.h b/tensorflow/core/kernels/eigen_attention.h
index 041ba12594..c5158e65d8 100644
--- a/tensorflow/core/kernels/eigen_attention.h
+++ b/tensorflow/core/kernels/eigen_attention.h
@@ -20,6 +20,13 @@ limitations under the License.
 
 namespace Eigen {
 
+// Noise mode used when padding.
+enum ExtractGlimpsesNoiseMode {
+  UNIFORM = 0,
+  GAUSSIAN = 1,
+  ZERO = 2,
+};
+
 /** ExtractGlimpses
  * \ingroup CXX11_NeuralNetworks_Module
  *
@@ -43,12 +50,13 @@ namespace Eigen {
  * for width and height which will be equal to the requested glimpse size.
  */
 namespace {
+
 template <typename Index>
 struct GlimpseExtractionOp {
   GlimpseExtractionOp(const Index width, const Index height,
                       const std::vector<IndexPair<float> >& offsets,
                       const bool normalized, const bool centered,
-                      const std::string &noise)
+                      const ExtractGlimpsesNoiseMode noise)
       : width_(width),
         height_(height),
         offsets_(offsets),
@@ -144,68 +152,73 @@ struct GlimpseExtractionOp {
       slice_extent[2] = std::min<Index>(input_height, slice_extent[2]);
 
       if (partial_overlap) {
-        if (noise_ == "zero") {
-          // Initialize the glimpse with zero noise.
-          output.template chip<3>(i).device(device) =
-              output.template chip<3>(i).constant(0);
-        } else if (noise_ == "uniform") {
-           // Initialize the glimpse with uniform noise.
-          typedef typename internal::remove_const<
-              typename internal::traits<Input>::Scalar>::type Scalar;
-          TensorFixedSize<Scalar, Sizes<> > mini;
-          mini.device(device) = input.template chip<3>(i).minimum();
-          TensorFixedSize<float, Sizes<> > range;
-          range.device(device) = (input.template chip<3>(i).maximum() - mini)
-                                     .template cast<float>();
+        switch (noise_) {
+          case ZERO: {
+            // Initialize the glimpse with zero noise.
+            output.template chip<3>(i).device(device) =
+                output.template chip<3>(i).constant(0);
+          } break;
+          case UNIFORM: {
+            // Initialize the glimpse with uniform noise.
+            typedef typename internal::remove_const<
+                typename internal::traits<Input>::Scalar>::type Scalar;
+            TensorFixedSize<Scalar, Sizes<> > mini;
+            mini.device(device) = input.template chip<3>(i).minimum();
+            TensorFixedSize<float, Sizes<> > range;
+            range.device(device) = (input.template chip<3>(i).maximum() - mini)
+                                       .template cast<float>();
 
-          DSizes<Index, 3> glimpse_size(num_channels, width_, height_);
-          TensorMap<Tensor<float, 3> > tmp(NULL, glimpse_size);
-          output.template chip<3>(i).device(device) =
-              mini.reshape(Sizes<1, 1, 1>()).broadcast(glimpse_size) +
-              (tmp.random(unigen) *
-               range.reshape(Sizes<1, 1, 1>()).broadcast(glimpse_size))
-                  .template cast<Scalar>();
-        } else {
-          // Initialize the glimpse with white noise: compute the mean and sigma
-          // of each channel, and use them to shape the gaussian.
-          DSizes<Index, 2> glimpse_size(width_, height_);
-          DSizes<Index, 2> input_size(input_width, input_height);
-          typedef typename internal::remove_const<
-              typename internal::traits<Input>::Scalar>::type Scalar;
+            DSizes<Index, 3> glimpse_size(num_channels, width_, height_);
+            TensorMap<Tensor<float, 3> > tmp(NULL, glimpse_size);
+            output.template chip<3>(i).device(device) =
+                mini.reshape(Sizes<1, 1, 1>()).broadcast(glimpse_size) +
+                (tmp.random(unigen) *
+                 range.reshape(Sizes<1, 1, 1>()).broadcast(glimpse_size))
+                    .template cast<Scalar>();
+          } break;
+          case GAUSSIAN: {
+            // Initialize the glimpse with white noise: compute the mean and
+            // sigma
+            // of each channel, and use them to shape the gaussian.
+            DSizes<Index, 2> glimpse_size(width_, height_);
+            DSizes<Index, 2> input_size(input_width, input_height);
+            typedef typename internal::remove_const<
+                typename internal::traits<Input>::Scalar>::type Scalar;
 
-          for (int j = 0; j < num_channels; ++j) {
-            TensorFixedSize<Scalar, Sizes<> > mean;
-            mean.device(device) = input.template chip<3>(i)
-                                      .template chip<0>(j)
-                                      .template cast<float>()
-                                      .mean();
-            TensorFixedSize<float, Sizes<> > sigma;
-            sigma.device(device) =
-                (input.template chip<3>(i)
-                     .template chip<0>(j)
-                     .template cast<float>() -
-                 mean.reshape(Sizes<1, 1>()).broadcast(input_size))
-                    .square()
-                    .mean()
-                    .sqrt();
-            TensorFixedSize<Scalar, Sizes<> > mini;
-            mini.device(device) =
-                input.template chip<3>(i).template chip<0>(j).minimum();
-            TensorFixedSize<float, Sizes<> > maxi;
-            maxi.device(device) =
-                input.template chip<3>(i).template chip<0>(j).maximum();
+            for (int j = 0; j < num_channels; ++j) {
+              TensorFixedSize<Scalar, Sizes<> > mean;
+              mean.device(device) = input.template chip<3>(i)
+                                        .template chip<0>(j)
+                                        .template cast<float>()
+                                        .mean();
+              TensorFixedSize<float, Sizes<> > sigma;
+              sigma.device(device) =
+                  (input.template chip<3>(i)
+                       .template chip<0>(j)
+                       .template cast<float>() -
+                   mean.reshape(Sizes<1, 1>()).broadcast(input_size))
+                      .square()
+                      .mean()
+                      .sqrt();
+              TensorFixedSize<Scalar, Sizes<> > mini;
+              mini.device(device) =
+                  input.template chip<3>(i).template chip<0>(j).minimum();
+              TensorFixedSize<float, Sizes<> > maxi;
+              maxi.device(device) =
+                  input.template chip<3>(i).template chip<0>(j).maximum();
 
-            TensorMap<Tensor<float, 2> > tmp(NULL, glimpse_size);
-            output.template chip<3>(i).template chip<0>(j).device(device) =
-                (mean.reshape(Sizes<1, 1>()).broadcast(glimpse_size) +
-                 (tmp.random(gen) *
-                  sigma.reshape(Sizes<1, 1>()).broadcast(glimpse_size))
-                     .template cast<Scalar>())
-                    .cwiseMin(
-                        maxi.reshape(Sizes<1, 1>()).broadcast(glimpse_size))
-                    .cwiseMax(
-                        mini.reshape(Sizes<1, 1>()).broadcast(glimpse_size));
-          }
+              TensorMap<Tensor<float, 2> > tmp(NULL, glimpse_size);
+              output.template chip<3>(i).template chip<0>(j).device(device) =
+                  (mean.reshape(Sizes<1, 1>()).broadcast(glimpse_size) +
+                   (tmp.random(gen) *
+                    sigma.reshape(Sizes<1, 1>()).broadcast(glimpse_size))
+                       .template cast<Scalar>())
+                      .cwiseMin(
+                          maxi.reshape(Sizes<1, 1>()).broadcast(glimpse_size))
+                      .cwiseMax(
+                          mini.reshape(Sizes<1, 1>()).broadcast(glimpse_size));
+            }
+          } break;
         }
 
         // Copy the part of the glimpse that cover the input image if any.
@@ -229,7 +242,7 @@ struct GlimpseExtractionOp {
   const std::vector<IndexPair<float> > offsets_;
   const bool normalized_;
   const bool centered_;
-  const std::string noise_;
+  const ExtractGlimpsesNoiseMode noise_;
 };
 }  // namespace
 
@@ -237,12 +250,12 @@ template <typename Input>
 EIGEN_ALWAYS_INLINE static const TensorCustomUnaryOp<
     const GlimpseExtractionOp<typename internal::traits<Input>::Index>,
     const Input>
-ExtractGlimpses(const Input& input,
-                const typename internal::traits<Input>::Index width,
-                const typename internal::traits<Input>::Index height,
-                const std::vector<IndexPair<float> >& offsets,
-                const bool normalized = true, const bool centered = true,
-                const std::string &noise = "uniform") {
+ExtractGlimpses(
+    const Input& input, const typename internal::traits<Input>::Index width,
+    const typename internal::traits<Input>::Index height,
+    const std::vector<IndexPair<float> >& offsets, const bool normalized = true,
+    const bool centered = true,
+    const ExtractGlimpsesNoiseMode noise = ExtractGlimpsesNoiseMode::UNIFORM) {
   EIGEN_STATIC_ASSERT(internal::traits<Input>::Layout == ColMajor,
                       YOU_MADE_A_PROGRAMMING_MISTAKE);
   EIGEN_STATIC_ASSERT(internal::traits<Input>::NumDimensions == 4,
-- 
GitLab


From 84c560138a89d76eed77a1945cc8582563ea8b90 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Mon, 8 Oct 2018 13:48:48 +0000
Subject: [PATCH 1401/2345] Address review comment

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/ops/image_ops.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/ops/image_ops.cc b/tensorflow/core/ops/image_ops.cc
index b23200171d..542f90845d 100644
--- a/tensorflow/core/ops/image_ops.cc
+++ b/tensorflow/core/ops/image_ops.cc
@@ -624,7 +624,7 @@ REGISTER_OP("ExtractGlimpse")
       string noise;
       TF_RETURN_IF_ERROR(c->GetAttr("noise", &noise));
       if (uniform_noise && noise != "") {
-        return errors::InvalidArgument("The uniform_noise and noise could not be specified at the same time");
+        return errors::InvalidArgument("The uniform_noise and noise should not be specified at the same time");
       }
 
       return SetOutputToSizedImage(c, batch_dim, 1 /* size_input_idx */,
-- 
GitLab


From 37a1324ff0070797bc9a401776600044d5ecff3a Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Mon, 8 Oct 2018 14:08:43 +0000
Subject: [PATCH 1402/2345] Update API golden

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
index 3c6ed1cfb8..f8f8007b0d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
@@ -82,7 +82,7 @@ tf_module {
   }
   member_method {
     name: "extract_glimpse"
-    argspec: "args=[\'input\', \'size\', \'offsets\', \'centered\', \'normalized\', \'uniform_noise\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'True\', \'True\', \'None\'], "
+    argspec: "args=[\'input\', \'size\', \'offsets\', \'centered\', \'normalized\', \'uniform_noise\', \'noise\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'True\', \'True\', \'\', \'None\'], "
   }
   member_method {
     name: "extract_image_patches"
-- 
GitLab


From e7b3e0e49845acc123f78ffcb0c22e5364e29adb Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Mon, 8 Oct 2018 14:10:22 +0000
Subject: [PATCH 1403/2345] Fix `Experimental clang-format Check` failure

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/ops/image_ops.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/ops/image_ops.cc b/tensorflow/core/ops/image_ops.cc
index 542f90845d..0ea48e8887 100644
--- a/tensorflow/core/ops/image_ops.cc
+++ b/tensorflow/core/ops/image_ops.cc
@@ -624,7 +624,9 @@ REGISTER_OP("ExtractGlimpse")
       string noise;
       TF_RETURN_IF_ERROR(c->GetAttr("noise", &noise));
       if (uniform_noise && noise != "") {
-        return errors::InvalidArgument("The uniform_noise and noise should not be specified at the same time");
+        return errors::InvalidArgument(
+            "The uniform_noise and noise should not be specified at the same "
+            "time");
       }
 
       return SetOutputToSizedImage(c, batch_dim, 1 /* size_input_idx */,
-- 
GitLab


From 1cef6e15d831e6950687e5c5292f659d839f980a Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Thu, 11 Oct 2018 01:00:27 +0000
Subject: [PATCH 1404/2345] Add wrapper to tf.image.extract_glimpse to maintain
 backward-compatibility

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .../python_api/api_def_ExtractGlimpse.pbtxt   |  4 +---
 tensorflow/python/ops/image_ops_impl.py       | 22 +++++++++++++++++++
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/api_def/python_api/api_def_ExtractGlimpse.pbtxt b/tensorflow/core/api_def/python_api/api_def_ExtractGlimpse.pbtxt
index ed8abdfcd7..f1fc72c4ca 100644
--- a/tensorflow/core/api_def/python_api/api_def_ExtractGlimpse.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_ExtractGlimpse.pbtxt
@@ -1,6 +1,4 @@
 op {
   graph_op_name: "ExtractGlimpse"
-  endpoint {
-    name: "image.extract_glimpse"
-  }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index 8047743cfa..7a9689808e 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -3070,3 +3070,25 @@ crop_and_resize_deprecation = deprecation.deprecated_args(
     None, 'box_ind is deprecated, use box_indices instead', 'box_ind')
 tf_export(v1=['image.crop_and_resize'])(
     crop_and_resize_deprecation(gen_image_ops.crop_and_resize))
+
+@tf_export('image.extract_glimpse')
+def extract_glimpse(
+    input,
+    size,
+    offsets,
+    centered=True,
+    normalized=True,
+    uniform_noise=True,
+    name=None,
+    noise=""):
+  return gen_image_ops.extract_glimpse(
+      input=input,
+      size=size,
+      offsets=offsets,
+      centered=centered,
+      normalized=normalized,
+      uniform_noise=uniform_noise,
+      noise=noise,
+      name=name)
+
+extract_glimpse.__doc__ = gen_image_ops.extract_glimpse.__doc__
-- 
GitLab


From 1895c5dbd87ae0bda87fd1bfe5fac24454cba881 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Thu, 11 Oct 2018 14:53:08 +0000
Subject: [PATCH 1405/2345] Use @tf_export(v1=[..], v2=[..]) to expose
 image.extract_glimpse

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/ops/image_ops_impl.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index 7a9689808e..59003adecf 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -3071,9 +3071,9 @@ crop_and_resize_deprecation = deprecation.deprecated_args(
 tf_export(v1=['image.crop_and_resize'])(
     crop_and_resize_deprecation(gen_image_ops.crop_and_resize))
 
-@tf_export('image.extract_glimpse')
+@tf_export(v1=['image.extract_glimpse'])
 def extract_glimpse(
-    input,
+    input, # pylint: disable=redefined-builtin
     size,
     offsets,
     centered=True,
@@ -3092,3 +3092,6 @@ def extract_glimpse(
       name=name)
 
 extract_glimpse.__doc__ = gen_image_ops.extract_glimpse.__doc__
+
+extract_glimpse_v2=gen_image_ops.extract_glimpse
+tf_export(v2=['image.extract_glimpse'])(extract_glimpse_v2)
-- 
GitLab


From 6edf17c8819ea6e07a264d641c134767f2296bbb Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Thu, 11 Oct 2018 15:04:15 +0000
Subject: [PATCH 1406/2345] Update api goldens

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt | 2 +-
 tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt | 4 ----
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt
index d2a9182f2c..e4fd87ce24 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt
@@ -82,7 +82,7 @@ tf_module {
   }
   member_method {
     name: "extract_glimpse"
-    argspec: "args=[\'input\', \'size\', \'offsets\', \'centered\', \'normalized\', \'uniform_noise\', \'noise\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'True\', \'True\', \'\', \'None\'], "
+    argspec: "args=[\'input\', \'size\', \'offsets\', \'centered\', \'normalized\', \'uniform_noise\', \'name\', \'noise\'], varargs=None, keywords=None, defaults=[\'True\', \'True\', \'True\', \'None\', \'\'], "
   }
   member_method {
     name: "extract_image_patches"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
index f8f8007b0d..3b444f02f0 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
@@ -80,10 +80,6 @@ tf_module {
     name: "encode_png"
     argspec: "args=[\'image\', \'compression\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'None\'], "
   }
-  member_method {
-    name: "extract_glimpse"
-    argspec: "args=[\'input\', \'size\', \'offsets\', \'centered\', \'normalized\', \'uniform_noise\', \'noise\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'True\', \'True\', \'\', \'None\'], "
-  }
   member_method {
     name: "extract_image_patches"
     argspec: "args=[\'images\', \'sizes\', \'strides\', \'rates\', \'padding\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-- 
GitLab


From 196b5aefbb2ff6d0fff434e02699db621a8faf41 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Thu, 11 Oct 2018 17:48:03 +0000
Subject: [PATCH 1407/2345] Pylint fix

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/ops/image_ops_impl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index 59003adecf..f577243ac7 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -3093,5 +3093,5 @@ def extract_glimpse(
 
 extract_glimpse.__doc__ = gen_image_ops.extract_glimpse.__doc__
 
-extract_glimpse_v2=gen_image_ops.extract_glimpse
+extract_glimpse_v2 = gen_image_ops.extract_glimpse
 tf_export(v2=['image.extract_glimpse'])(extract_glimpse_v2)
-- 
GitLab


From 2c5fce22f64f2fb687473d6c04030f60fb5b985c Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Thu, 25 Oct 2018 22:41:40 +0000
Subject: [PATCH 1408/2345] Fix exposed namespace api to no use v2, only
 `tf_export('image.extract_glimpse', v1=[])`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/ops/image_ops_impl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index f577243ac7..70ab1a5526 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -3094,4 +3094,4 @@ def extract_glimpse(
 extract_glimpse.__doc__ = gen_image_ops.extract_glimpse.__doc__
 
 extract_glimpse_v2 = gen_image_ops.extract_glimpse
-tf_export(v2=['image.extract_glimpse'])(extract_glimpse_v2)
+tf_export('image.extract_glimpse', v1=[])(extract_glimpse_v2)
-- 
GitLab


From e624b1b2de98ee1036272003bd582c315e66606c Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sat, 27 Oct 2018 00:27:56 +0000
Subject: [PATCH 1409/2345] Fix api golden

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
index 3b444f02f0..f8f8007b0d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
@@ -80,6 +80,10 @@ tf_module {
     name: "encode_png"
     argspec: "args=[\'image\', \'compression\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'None\'], "
   }
+  member_method {
+    name: "extract_glimpse"
+    argspec: "args=[\'input\', \'size\', \'offsets\', \'centered\', \'normalized\', \'uniform_noise\', \'noise\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'True\', \'True\', \'\', \'None\'], "
+  }
   member_method {
     name: "extract_image_patches"
     argspec: "args=[\'images\', \'sizes\', \'strides\', \'rates\', \'padding\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-- 
GitLab


From 5e9a34b05226f84a5a7bd8ae7e0bd9864306b250 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Fri, 4 Jan 2019 21:28:30 +0000
Subject: [PATCH 1410/2345] Update image.extract_glimpse to take noise in v2,
 and keep v1 api

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/ops/image_ops_impl.py | 122 ++++++++++++++++++++++--
 1 file changed, 116 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index 70ab1a5526..9a1682b39a 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -3079,8 +3079,56 @@ def extract_glimpse(
     centered=True,
     normalized=True,
     uniform_noise=True,
-    name=None,
-    noise=""):
+    name=None):
+  """Extracts a glimpse from the input tensor.
+
+  Returns a set of windows called glimpses extracted at location
+  `offsets` from the input tensor. If the windows only partially
+  overlaps the inputs, the non overlapping areas will be filled with
+  random noise.
+
+  The result is a 4-D tensor of shape `[batch_size, glimpse_height,
+  glimpse_width, channels]`. The channels and batch dimensions are the
+  same as that of the input tensor. The height and width of the output
+  windows are specified in the `size` parameter.
+
+  The argument `normalized` and `centered` controls how the windows are built:
+
+  * If the coordinates are normalized but not centered, 0.0 and 1.0
+    correspond to the minimum and maximum of each height and width
+    dimension.
+  * If the coordinates are both normalized and centered, they range from
+    -1.0 to 1.0. The coordinates (-1.0, -1.0) correspond to the upper
+    left corner, the lower right corner is located at (1.0, 1.0) and the
+    center is at (0, 0).
+  * If the coordinates are not normalized they are interpreted as
+    numbers of pixels.
+
+  Args:
+    input: A `Tensor` of type `float32`.
+      A 4-D float tensor of shape `[batch_size, height, width, channels]`.
+    size: A `Tensor` of type `int32`.
+      A 1-D tensor of 2 elements containing the size of the glimpses
+      to extract.  The glimpse height must be specified first, following
+      by the glimpse width.
+    offsets: A `Tensor` of type `float32`.
+      A 2-D integer tensor of shape `[batch_size, 2]` containing
+      the y, x locations of the center of each window.
+    centered: An optional `bool`. Defaults to `True`.
+      indicates if the offset coordinates are centered relative to
+      the image, in which case the (0, 0) offset is relative to the center
+      of the input images. If false, the (0,0) offset corresponds to the
+      upper left corner of the input images.
+    normalized: An optional `bool`. Defaults to `True`.
+      indicates if the offset coordinates are normalized.
+    uniform_noise: An optional `bool`. Defaults to `True`.
+      indicates if the noise should be generated using a
+      uniform distribution or a Gaussian distribution.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor` of type `float32`.
+  """
   return gen_image_ops.extract_glimpse(
       input=input,
       size=size,
@@ -3088,10 +3136,72 @@ def extract_glimpse(
       centered=centered,
       normalized=normalized,
       uniform_noise=uniform_noise,
-      noise=noise,
       name=name)
 
-extract_glimpse.__doc__ = gen_image_ops.extract_glimpse.__doc__
+@tf_export("image.extract_glimpse", v1=[])
+def extract_glimpse_v2(
+    input, # pylint: disable=redefined-builtin
+    size,
+    offsets,
+    centered=True,
+    normalized=True,
+    noise="uniform",
+    name=None):
+  """Extracts a glimpse from the input tensor.
+
+  Returns a set of windows called glimpses extracted at location
+  `offsets` from the input tensor. If the windows only partially
+  overlaps the inputs, the non overlapping areas will be filled with
+  random noise.
+
+  The result is a 4-D tensor of shape `[batch_size, glimpse_height,
+  glimpse_width, channels]`. The channels and batch dimensions are the
+  same as that of the input tensor. The height and width of the output
+  windows are specified in the `size` parameter.
+
+  The argument `normalized` and `centered` controls how the windows are built:
+
+  * If the coordinates are normalized but not centered, 0.0 and 1.0
+    correspond to the minimum and maximum of each height and width
+    dimension.
+  * If the coordinates are both normalized and centered, they range from
+    -1.0 to 1.0. The coordinates (-1.0, -1.0) correspond to the upper
+    left corner, the lower right corner is located at (1.0, 1.0) and the
+    center is at (0, 0).
+  * If the coordinates are not normalized they are interpreted as
+    numbers of pixels.
 
-extract_glimpse_v2 = gen_image_ops.extract_glimpse
-tf_export('image.extract_glimpse', v1=[])(extract_glimpse_v2)
+  Args:
+    input: A `Tensor` of type `float32`.
+      A 4-D float tensor of shape `[batch_size, height, width, channels]`.
+    size: A `Tensor` of type `int32`.
+      A 1-D tensor of 2 elements containing the size of the glimpses
+      to extract.  The glimpse height must be specified first, following
+      by the glimpse width.
+    offsets: A `Tensor` of type `float32`.
+      A 2-D integer tensor of shape `[batch_size, 2]` containing
+      the y, x locations of the center of each window.
+    centered: An optional `bool`. Defaults to `True`.
+      indicates if the offset coordinates are centered relative to
+      the image, in which case the (0, 0) offset is relative to the center
+      of the input images. If false, the (0,0) offset corresponds to the
+      upper left corner of the input images.
+    normalized: An optional `bool`. Defaults to `True`.
+      indicates if the offset coordinates are normalized.
+    noise: An optional `string`. Defaults to `uniform`.
+      indicates if the noise should be `uniform` (uniform distribution),
+      `gaussian` (gaussian distribution), or `zero` (zero padding).
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor` of type `float32`.
+  """
+  return gen_image_ops.extract_glimpse(
+      input=input,
+      size=size,
+      offsets=offsets,
+      centered=centered,
+      normalized=normalized,
+      noise=noise,
+      uniform_noise=False,
+      name=name)
-- 
GitLab


From eb074cf0d97afcd6a506c989bee173f74a4c4589 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Fri, 4 Jan 2019 21:30:17 +0000
Subject: [PATCH 1411/2345] Update test cases for tf.image.extract_glimpse

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/kernel_tests/attention_ops_test.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/kernel_tests/attention_ops_test.py b/tensorflow/python/kernel_tests/attention_ops_test.py
index 2db74157fe..d3f892cb57 100644
--- a/tensorflow/python/kernel_tests/attention_ops_test.py
+++ b/tensorflow/python/kernel_tests/attention_ops_test.py
@@ -210,9 +210,9 @@ class ExtractGlimpseTest(test.TestCase):
       # [ 0.  0.  0.]
       # [ 0.  0.  0.]
       # [ 0.  0.  0.]
-      result1 = image_ops.extract_glimpse(img, [3, 3], [[-2, 2]],
-                                          centered=False, normalized=False,
-                                          uniform_noise=False, noise="zero")
+      result1 = image_ops.extract_glimpse_v2(img, [3, 3], [[-2, 2]],
+                                             centered=False, normalized=False,
+                                             noise="zero")
       self.assertAllEqual(np.asarray([[0, 0, 0],
                                       [0, 0, 0],
                                       [0, 0, 0]]),
@@ -226,9 +226,9 @@ class ExtractGlimpseTest(test.TestCase):
       # [  0.  15.  16.  17.  18.  19.   0.]
       # [  0.  20.  21.  22.  23.  24.   0.]
       # [  0.   0.   0.   0.   0.   0.   0.]]
-      result2 = image_ops.extract_glimpse(img, [7, 7], [[0, 0]],
-                                          normalized=False,
-                                          uniform_noise=False, noise="zero")
+      result2 = image_ops.extract_glimpse_v2(img, [7, 7], [[0, 0]],
+                                             normalized=False,
+                                             noise="zero")
       self.assertAllEqual(np.asarray([[0, 0, 0, 0, 0, 0, 0],
                                       [0, 0, 1, 2, 3, 4, 0],
                                       [0, 5, 6, 7, 8, 9, 0],
-- 
GitLab


From 921985ff1c0a10ec97690803fb66b78c328e8c0b Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Fri, 4 Jan 2019 21:38:24 +0000
Subject: [PATCH 1412/2345] Update API goldens with v2 change only

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt | 2 +-
 tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt
index e4fd87ce24..15d0e099ba 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt
@@ -82,7 +82,7 @@ tf_module {
   }
   member_method {
     name: "extract_glimpse"
-    argspec: "args=[\'input\', \'size\', \'offsets\', \'centered\', \'normalized\', \'uniform_noise\', \'name\', \'noise\'], varargs=None, keywords=None, defaults=[\'True\', \'True\', \'True\', \'None\', \'\'], "
+    argspec: "args=[\'input\', \'size\', \'offsets\', \'centered\', \'normalized\', \'uniform_noise\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'True\', \'True\', \'None\'], "
   }
   member_method {
     name: "extract_image_patches"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
index f8f8007b0d..990f160b39 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
@@ -82,7 +82,7 @@ tf_module {
   }
   member_method {
     name: "extract_glimpse"
-    argspec: "args=[\'input\', \'size\', \'offsets\', \'centered\', \'normalized\', \'uniform_noise\', \'noise\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'True\', \'True\', \'\', \'None\'], "
+    argspec: "args=[\'input\', \'size\', \'offsets\', \'centered\', \'normalized\', \'noise\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'True\', \'uniform\', \'None\'], "
   }
   member_method {
     name: "extract_image_patches"
-- 
GitLab


From f0dc741f4618290008011b5a742d9c84134b1c51 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Thu, 10 Jan 2019 04:34:03 +0000
Subject: [PATCH 1413/2345] Set default noise as uniform to conform with v1 api

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/api_def/base_api/api_def_ExtractGlimpse.pbtxt | 2 +-
 tensorflow/core/kernels/attention_ops.cc                      | 2 +-
 tensorflow/core/ops/image_ops.cc                              | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/api_def/base_api/api_def_ExtractGlimpse.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExtractGlimpse.pbtxt
index 0ed7ccad26..4b951659a2 100644
--- a/tensorflow/core/api_def/base_api/api_def_ExtractGlimpse.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ExtractGlimpse.pbtxt
@@ -54,7 +54,7 @@ END
     name: "noise"
     description: <<END
 indicates if the noise should `uniform`, `gaussian`, or
-`zero`. The default is `` which means the the noise type
+`zero`. The default is `uniform` which means the the noise type
 will be decided by `uniform_noise`.
 END
   }
diff --git a/tensorflow/core/kernels/attention_ops.cc b/tensorflow/core/kernels/attention_ops.cc
index 58c7aa3506..52ca7705f2 100644
--- a/tensorflow/core/kernels/attention_ops.cc
+++ b/tensorflow/core/kernels/attention_ops.cc
@@ -38,7 +38,7 @@ class ExtractGlimpseOp : public OpKernel {
     string noise;
     OP_REQUIRES_OK(context, context->GetAttr("uniform_noise", &uniform_noise));
     OP_REQUIRES_OK(context, context->GetAttr("noise", &noise));
-    OP_REQUIRES(context, !uniform_noise || noise == "",
+    OP_REQUIRES(context, !(uniform_noise && (noise != "" && noise != "uniform")),
                 errors::InvalidArgument("The uniform_noise and noise could not "
                                         "be specified at the same time"));
     if (noise == "") {
diff --git a/tensorflow/core/ops/image_ops.cc b/tensorflow/core/ops/image_ops.cc
index 0ea48e8887..8eae75058c 100644
--- a/tensorflow/core/ops/image_ops.cc
+++ b/tensorflow/core/ops/image_ops.cc
@@ -606,7 +606,7 @@ REGISTER_OP("ExtractGlimpse")
     .Attr("centered: bool = true")
     .Attr("normalized: bool = true")
     .Attr("uniform_noise: bool = true")
-    .Attr("noise: string = ''")
+    .Attr("noise: string = 'uniform'")
     .SetShapeFn([](InferenceContext* c) {
       ShapeHandle input;
       TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 4, &input));
@@ -623,7 +623,7 @@ REGISTER_OP("ExtractGlimpse")
       TF_RETURN_IF_ERROR(c->GetAttr("uniform_noise", &uniform_noise));
       string noise;
       TF_RETURN_IF_ERROR(c->GetAttr("noise", &noise));
-      if (uniform_noise && noise != "") {
+      if (uniform_noise && (noise != "" && noise != "uniform")) {
         return errors::InvalidArgument(
             "The uniform_noise and noise should not be specified at the same "
             "time");
-- 
GitLab


From ba4ad8d2a03c6b3c0eec44d6fe532f46b8de4b1e Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Thu, 10 Jan 2019 18:06:37 +0000
Subject: [PATCH 1414/2345] Add v2 transform compatibility tool for
 tf.image.extract_glimpse

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .../tools/compatibility/tf_upgrade_v2.py      | 44 ++++++++++++++++++-
 .../tools/compatibility/tf_upgrade_v2_test.py | 44 +++++++++++++++++++
 2 files changed, 87 insertions(+), 1 deletion(-)

diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 03ecf5f303..f5d4dc1da4 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -794,7 +794,7 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.image.resize_bicubic": self._image_resize_transformer,
         "tf.image.resize_bilinear": self._image_resize_transformer,
         "tf.image.resize_nearest_neighbor": self._image_resize_transformer,
-
+        "tf.image.extract_glimpse": self._extract_glimpse_transformer,
     }
 
     decay_function_comment = (
@@ -1266,6 +1266,48 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
 
     return node
 
+  @staticmethod
+  def _extract_glimpse_transformer(parent, node, full_name, name, logs):
+    def _replace_uniform_noise_node(parent, old_value):
+      """Replaces old_value with 'uniform' or 'guassian'."""
+      uniform = ast.Str(s="uniform")
+      gaussian = ast.Str(s="gaussian")
+      new_value = ast.IfExp(body=uniform, test=old_value,
+                            orelse=gaussian)
+      # This copies the prefix and suffix on old_value to new_value.
+      pasta.ast_utils.replace_child(parent, old_value, new_value)
+      ast.copy_location(new_value, old_value)
+      # Put parentheses around noise.value (and remove the old prefix/
+      # suffix, they should only be around new_value).
+      pasta.base.formatting.set(new_value, "prefix", "(")
+      pasta.base.formatting.set(new_value, "suffix", ")")
+
+    # Check if we have a uniform_noise keyword arg
+    for uniform_noise in node.keywords:
+      if uniform_noise.arg == "uniform_noise":
+        logs.append((ast_edits.INFO, node.lineno, node.col_offset,
+                     "Changing uniform_noise arg of tf.image.extract_glimpse "
+                     "to noise, and recomputing value. Please check this "
+                     "transformation.\n"))
+        uniform_noise.arg = "noise"
+        value = "uniform" if uniform_noise.value else "gaussian"
+        _replace_uniform_noise_node(uniform_noise, uniform_noise.value)
+        return node
+
+    # Maybe it was a positional arg
+    if len(node.args) < 5:
+      logs.append((ast_edits.ERROR, node.lineno, node.col_offset,
+                   "tf.image.extract_glimpse called without "
+                   "arguments, so automatic fix was disabled. "
+                   "tf.image.extract_glimpse has changed the semantics "
+                   "of the sxth argument."))
+    else:
+      _replace_uniform_noise_node(node, node.args[5])
+      logs.append((ast_edits.INFO, node.lineno, node.col_offset,
+                   "Changing uniform_noise arg of tf.image.extract_glimpse to "
+                   "noise, and recomputing value.\n"))
+      return node
+
   @staticmethod
   def _dropout_transformer(parent, node, full_name, name, logs):
     def _replace_keep_prob_node(parent, old_value):
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 8ff5d01ae6..380cb22569 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -483,6 +483,50 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
       self.assertIn("%s requires manual check" % ns, errors[0])
       self.assertIn("loss_reduction has been changed", report)
 
+  def testExtractGlimpse(self):
+    text = ("tf.image.extract_glimpse(x, size, off, False, "
+            "False, False, name=\"foo\")\n")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(
+        new_text,
+        "tf.image.extract_glimpse(x, size, off, False, "
+        "False, ('uniform' if False else 'gaussian'), name=\"foo\")\n",
+    )
+
+    text = ("tf.image.extract_glimpse(x, size, off, centered=False, "
+            "normalized=False, uniform_noise=uniform_noise, name=\"foo\")\n")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(
+        new_text,
+        "tf.image.extract_glimpse(x, size, off, centered=False, "
+        "normalized=False, noise=('uniform' if uniform_noise else 'gaussian')"
+        ", name=\"foo\")\n",
+    )
+
+    text = ("tf.image.extract_glimpse(x,\n"
+            "                         size,\n"
+            "                         off,\n"
+            "                         centered=True,\n"
+            "                         normalized=True, # Stuff before\n"
+            "                         uniform_noise=False,\n"
+            "                         name=\"foo\")# Stuff after\n")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(
+        new_text,
+        "tf.image.extract_glimpse(x,\n"
+        "                         size,\n"
+        "                         off,\n"
+        "                         centered=True,\n"
+        "                         normalized=True, # Stuff before\n"
+        "                         noise=('uniform' if False else 'gaussian'),\n"
+        "                         name=\"foo\")# Stuff after\n")
+
+    text = "tf.image.extract_glimpse(x)\n"
+    _, unused_report, errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, text)
+    self.assertIn("tf.image.extract_glimpse called without arguments",
+                  errors[0])
+
   def testDropout(self):
     text = "tf.nn.dropout(x, keep_prob, name=\"foo\")\n"
     _, unused_report, unused_errors, new_text = self._upgrade(text)
-- 
GitLab


From 6f9355a52ce67a0e82d1a2186e31867e3531e97a Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 13 Jan 2019 19:47:00 +0000
Subject: [PATCH 1415/2345] Add parentheses around new_value.test to cover `d
 if (a if t else b) else d`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .../tools/compatibility/tf_upgrade_v2.py      | 20 ++++++++-----------
 .../tools/compatibility/tf_upgrade_v2_test.py | 14 ++++++-------
 2 files changed, 15 insertions(+), 19 deletions(-)

diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index f5d4dc1da4..c6d7da86e0 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -1277,10 +1277,11 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
       # This copies the prefix and suffix on old_value to new_value.
       pasta.ast_utils.replace_child(parent, old_value, new_value)
       ast.copy_location(new_value, old_value)
-      # Put parentheses around noise.value (and remove the old prefix/
-      # suffix, they should only be around new_value).
-      pasta.base.formatting.set(new_value, "prefix", "(")
-      pasta.base.formatting.set(new_value, "suffix", ")")
+      # Put parentheses around noise.value.test (and remove the old prefix/
+      # suffix, they should only be around new_value.test), so that:
+      # "uniform" if (a if b else c) else "gaussian" is valid.
+      pasta.base.formatting.set(new_value.test, "prefix", "(")
+      pasta.base.formatting.set(new_value.test, "suffix", ")")
 
     # Check if we have a uniform_noise keyword arg
     for uniform_noise in node.keywords:
@@ -1294,14 +1295,9 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         _replace_uniform_noise_node(uniform_noise, uniform_noise.value)
         return node
 
-    # Maybe it was a positional arg
-    if len(node.args) < 5:
-      logs.append((ast_edits.ERROR, node.lineno, node.col_offset,
-                   "tf.image.extract_glimpse called without "
-                   "arguments, so automatic fix was disabled. "
-                   "tf.image.extract_glimpse has changed the semantics "
-                   "of the sxth argument."))
-    else:
+    # Since `noise`/`uniform_noise` is optional arg, nothing needs to be
+    # done if len(node.args) < 5.
+    if len(node.args) >= 5:
       _replace_uniform_noise_node(node, node.args[5])
       logs.append((ast_edits.INFO, node.lineno, node.col_offset,
                    "Changing uniform_noise arg of tf.image.extract_glimpse to "
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 380cb22569..55bef93bf6 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -490,17 +490,18 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
     self.assertEqual(
         new_text,
         "tf.image.extract_glimpse(x, size, off, False, "
-        "False, ('uniform' if False else 'gaussian'), name=\"foo\")\n",
+        "False, 'uniform' if (False) else 'gaussian', name=\"foo\")\n",
     )
 
     text = ("tf.image.extract_glimpse(x, size, off, centered=False, "
-            "normalized=False, uniform_noise=uniform_noise, name=\"foo\")\n")
+            "normalized=False, uniform_noise=True if uniform_noise else "
+            "False, name=\"foo\")\n")
     _, unused_report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual(
         new_text,
         "tf.image.extract_glimpse(x, size, off, centered=False, "
-        "normalized=False, noise=('uniform' if uniform_noise else 'gaussian')"
-        ", name=\"foo\")\n",
+        "normalized=False, noise='uniform' if (True if uniform_noise else "
+        "False) else 'gaussian', name=\"foo\")\n",
     )
 
     text = ("tf.image.extract_glimpse(x,\n"
@@ -518,14 +519,13 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
         "                         off,\n"
         "                         centered=True,\n"
         "                         normalized=True, # Stuff before\n"
-        "                         noise=('uniform' if False else 'gaussian'),\n"
+        "                         noise='uniform' if (False) else 'gaussian',\n"
         "                         name=\"foo\")# Stuff after\n")
 
     text = "tf.image.extract_glimpse(x)\n"
     _, unused_report, errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, text)
-    self.assertIn("tf.image.extract_glimpse called without arguments",
-                  errors[0])
+    self.assertEqual(errors, [])
 
   def testDropout(self):
     text = "tf.nn.dropout(x, keep_prob, name=\"foo\")\n"
-- 
GitLab


From 6be09c8686fc4eede452ed8a09a6f31facc64f17 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Mon, 14 Jan 2019 17:49:13 +0000
Subject: [PATCH 1416/2345] Fixed Experimental clang-format Check error

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/kernels/attention_ops.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/attention_ops.cc b/tensorflow/core/kernels/attention_ops.cc
index 52ca7705f2..70e59a90fe 100644
--- a/tensorflow/core/kernels/attention_ops.cc
+++ b/tensorflow/core/kernels/attention_ops.cc
@@ -38,7 +38,8 @@ class ExtractGlimpseOp : public OpKernel {
     string noise;
     OP_REQUIRES_OK(context, context->GetAttr("uniform_noise", &uniform_noise));
     OP_REQUIRES_OK(context, context->GetAttr("noise", &noise));
-    OP_REQUIRES(context, !(uniform_noise && (noise != "" && noise != "uniform")),
+    OP_REQUIRES(context,
+                !(uniform_noise && (noise != "" && noise != "uniform")),
                 errors::InvalidArgument("The uniform_noise and noise could not "
                                         "be specified at the same time"));
     if (noise == "") {
-- 
GitLab


From bf4767c6bafb077fc591107691199a6981c29304 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 28 Jan 2019 13:51:02 -0800
Subject: [PATCH 1417/2345] Add test for using datasets for multi-input
 multi-output datasets.

PiperOrigin-RevId: 231282688
---
 .../keras/engine/training_dataset_test.py     | 60 +++++++++++++++++++
 1 file changed, 60 insertions(+)

diff --git a/tensorflow/python/keras/engine/training_dataset_test.py b/tensorflow/python/keras/engine/training_dataset_test.py
index 5ed07fc131..751c265005 100644
--- a/tensorflow/python/keras/engine/training_dataset_test.py
+++ b/tensorflow/python/keras/engine/training_dataset_test.py
@@ -225,6 +225,66 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
                                  'the `steps` argument'):
       model.predict(dataset, verbose=0)
 
+  # TODO(b/123531973): Include tests using dataset_v1.
+  @keras_parameterized.run_with_all_model_types(exclude_models='sequential')
+  @keras_parameterized.run_all_keras_modes(always_skip_v1=True)
+  def test_training_and_eval_methods_on_multi_input_output_dataset(self):
+    input_a = keras.layers.Input(shape=(3,), name='input_1')
+    input_b = keras.layers.Input(shape=(3,), name='input_2')
+    dense = keras.layers.Dense(4, name='dense')
+    dropout = keras.layers.Dropout(0.5, name='dropout')
+    branch_a = [input_a, dense]
+    branch_b = [input_b, dense, dropout]
+
+    model = testing_utils.get_multi_io_model(branch_a, branch_b)
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
+
+    input_a_np = np.random.random((10, 3)).astype(dtype=np.float32)
+    input_b_np = np.random.random((10, 3)).astype(dtype=np.float32)
+    output_d_np = np.random.random((10, 4)).astype(dtype=np.float32)
+    output_e_np = np.random.random((10, 4)).astype(dtype=np.float32)
+
+    # Test with tuples
+    dataset_tuple = dataset_ops.Dataset.from_tensor_slices((
+        (input_a_np, input_b_np), (output_d_np, output_e_np)))
+    dataset_tuple = dataset_tuple.repeat(100)
+    dataset_tuple = dataset_tuple.batch(10)
+
+    model.fit(dataset_tuple, epochs=1, steps_per_epoch=2, verbose=1)
+    model.evaluate(dataset_tuple, steps=2, verbose=1)
+
+    predict_dataset_tuple = dataset_ops.Dataset.from_tensor_slices(
+        (input_a_np, input_b_np))
+    # TODO(b/123360757): Remove below assertion once predict() supports
+    # muti-input datasets.
+    with self.assertRaisesRegexp(ValueError,
+                                 'Error when checking model input'):
+      model.predict(predict_dataset_tuple, steps=1)
+
+    # Test with dict
+    input_dict = {'input_1': input_a_np, 'input_2': input_b_np}
+    if testing_utils.get_model_type() == 'subclass':
+      output_dict = {'output_1': output_d_np, 'output_2': output_e_np}
+    else:
+      output_dict = {'dense': output_d_np, 'dropout': output_e_np}
+
+    dataset_dict = dataset_ops.Dataset.from_tensor_slices((
+        input_dict, output_dict))
+    dataset_dict = dataset_dict.repeat(100)
+    dataset_dict = dataset_dict.batch(10)
+
+    model.fit(dataset_dict, epochs=1, steps_per_epoch=2, verbose=1)
+    model.evaluate(dataset_dict, steps=2, verbose=1)
+
+    predict_dataset_dict = dataset_ops.Dataset.from_tensor_slices(
+        input_dict)
+    predict_dataset_dict = predict_dataset_dict.repeat(100)
+    predict_dataset_dict = dataset_dict.batch(10)
+    model.predict(predict_dataset_dict, steps=1)
+
   @keras_parameterized.run_with_all_model_types
   @keras_parameterized.run_all_keras_modes
   def test_dataset_with_sample_weights(self):
-- 
GitLab


From ca97d11512236715b4ed5ddd814d2de913dedfd1 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Mon, 28 Jan 2019 13:58:20 -0800
Subject: [PATCH 1418/2345] [XLA:CPU] Add additional assertions to
 __xla_cpu_runtime_ParallelForkJoin.

I got bitten by a null pointer here.

PiperOrigin-RevId: 231283840
---
 tensorflow/compiler/xla/service/cpu/runtime_fork_join.cc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tensorflow/compiler/xla/service/cpu/runtime_fork_join.cc b/tensorflow/compiler/xla/service/cpu/runtime_fork_join.cc
index 2d9492eacf..6f72ddadf9 100644
--- a/tensorflow/compiler/xla/service/cpu/runtime_fork_join.cc
+++ b/tensorflow/compiler/xla/service/cpu/runtime_fork_join.cc
@@ -69,8 +69,13 @@ TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_ParallelForkJoin(
   CHECK_EQ(params, nullptr);
   CHECK_GT(num_partitions, 1);
   CHECK_GT(num_partitioned_dims, 0);
+  CHECK_NE(function_ptr, nullptr);
+  CHECK_NE(partitions, nullptr);
   const xla::ExecutableRunOptions* run_options =
       static_cast<const xla::ExecutableRunOptions*>(run_options_ptr);
+  CHECK_NE(run_options, nullptr);
+  CHECK_NE(run_options->intra_op_thread_pool(), nullptr);
+
   ComputeFunctionType function =
       reinterpret_cast<ComputeFunctionType>(function_ptr);
   // Compute partition stride in 'partitions' array.
-- 
GitLab


From 9e78e8aa87eadeb7abb676be9ff3926a6a3865c5 Mon Sep 17 00:00:00 2001
From: Duncan Riach <duncan@nvidia.com>
Date: Mon, 28 Jan 2019 14:06:44 -0800
Subject: [PATCH 1419/2345] Make cuDNN max-pooling deterministic when
 TF_CUDNN_DETERMINISTIC environment variable is 1/true

---
 tensorflow/stream_executor/cuda/cuda_dnn.cc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index 06739e8655..283c907ab0 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -784,7 +784,6 @@ bool BatchnormSpatialPersistentEnabled() {
 }
 
 // A helper function to decide whether to enable deterministic functionality.
-// TODO(pr/24355): Support all cuDNN functionality (currently only convolution).
 bool RequireDeterminism() {
   static bool is_enabled = [] {
     bool is_enabled = false;
@@ -887,10 +886,13 @@ class CudnnPoolingDescriptor {
     std::transform(shape64.cbegin(), shape64.cend(), shape.begin(),
                    &CheckedNarrowing<int64, int>);
     bool propagate_nans = pooling_descriptor.propagate_nans();
+    auto cudnn_max_pooling_mode = RequireDeterminism()
+                                      ? CUDNN_POOLING_MAX_DETERMINISTIC
+                                      : CUDNN_POOLING_MAX;
     CHECK_CUDNN_OK(cudnnSetPoolingNdDescriptor(
         handle_.get(),
         (pooling_descriptor.mode() == dnn::PoolingMode::kMaximum
-             ? CUDNN_POOLING_MAX
+             ? cudnn_max_pooling_mode
              : CUDNN_POOLING_AVERAGE_COUNT_EXCLUDE_PADDING),
         propagate_nans ? CUDNN_PROPAGATE_NAN : CUDNN_NOT_PROPAGATE_NAN, nd,
         shape.data(), padding.data(), strides.data()));
-- 
GitLab


From f9f590d185afe072272c3e02cd3f3e34013fc85b Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Mon, 28 Jan 2019 14:04:48 -0800
Subject: [PATCH 1420/2345] [XLA] Comment fix in local_client.h.

PiperOrigin-RevId: 231285168
---
 tensorflow/compiler/xla/client/local_client.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/client/local_client.h b/tensorflow/compiler/xla/client/local_client.h
index ddb36680e8..4f4fc8df31 100644
--- a/tensorflow/compiler/xla/client/local_client.h
+++ b/tensorflow/compiler/xla/client/local_client.h
@@ -114,7 +114,7 @@ class LocalClient : public Client {
   // Build and return a LocalExecutable object. The executable is compiled using
   // the given XlaComputation, argument layouts and options.
   //
-  // The given ExecutableBuildOptions override any values from TF_XLA_FLAGS
+  // The given ExecutableBuildOptions overrides any values from XLA_FLAGS
   // environment variable.
   StatusOr<std::unique_ptr<LocalExecutable>> Compile(
       const XlaComputation& computation,
-- 
GitLab


From 4e23c1c778dd1008f4134d4fbc39401c0410e485 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 28 Jan 2019 14:14:48 -0800
Subject: [PATCH 1421/2345] [TF:XLA] Bump open source llvm revision to r352344

PiperOrigin-RevId: 231287114
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 655f0fdb84..1165a1a151 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -514,11 +514,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "d32baf8e3ccfdc689704ee8c9af5f17b9d2b571d3d5b25e857c375d28a435193",
-        strip_prefix = "llvm-ec3165a24ab126615b1766f6c48c0b0de844e4cb",
+        sha256 = "48699c52e64e428388b0c7d2daa5197864f1f2425608a89fc15d494bf4bab0e0",
+        strip_prefix = "llvm-66b8f683ab6b3be52fe606a0db0f377f9f66e170",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/ec3165a24ab126615b1766f6c48c0b0de844e4cb.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/ec3165a24ab126615b1766f6c48c0b0de844e4cb.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/66b8f683ab6b3be52fe606a0db0f377f9f66e170.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/66b8f683ab6b3be52fe606a0db0f377f9f66e170.tar.gz",
         ],
     )
 
-- 
GitLab


From ad0af72a1c071ed53932efc0a3ccde81b27f3992 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 28 Jan 2019 14:46:52 -0800
Subject: [PATCH 1422/2345] Fix non-determinism in test naming when using
 combinations.

PiperOrigin-RevId: 231292935
---
 tensorflow/contrib/distribute/python/combinations.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/combinations.py b/tensorflow/contrib/distribute/python/combinations.py
index db79d6c0d8..f104d950c3 100644
--- a/tensorflow/contrib/distribute/python/combinations.py
+++ b/tensorflow/contrib/distribute/python/combinations.py
@@ -112,12 +112,12 @@ def generate(combinations):
       # We use OrderedDicts in `combine()` and `times()` to ensure stable
       # order of keys in each dictionary.
       assert isinstance(combination, OrderedDict)
-      name = "".join([
+      name = "".join(sorted([
           "_{}_{}".format(
               "".join(filter(str.isalnum, key)),
               "".join(filter(str.isalnum, str(value))))
           for key, value in combination.items()
-      ])
+      ]))
       named_combinations.append(
           OrderedDict(
               list(combination.items()) + [("testcase_name",
-- 
GitLab


From 78fb58843ef13bf8aa98cdf350859faede740473 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Mon, 28 Jan 2019 14:47:43 -0800
Subject: [PATCH 1423/2345] [XLA] Switch ClientLibraryTestBase to use
 LocalClient*s.

No functional change -- the code still uses the vanilla Client API.  But this
allows subclasses to use the LocalClient API if they choose.

PiperOrigin-RevId: 231293067
---
 tensorflow/compiler/xla/tests/client_library_test_base.cc | 5 +++--
 tensorflow/compiler/xla/tests/client_library_test_base.h  | 4 ++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.cc b/tensorflow/compiler/xla/tests/client_library_test_base.cc
index edb95c973b..0e99ede5d0 100644
--- a/tensorflow/compiler/xla/tests/client_library_test_base.cc
+++ b/tensorflow/compiler/xla/tests/client_library_test_base.cc
@@ -41,8 +41,9 @@ constexpr char kInterpreter[] = "interpreter";
 
 // Wrapper function that creates a nicer error message (than a bare
 // ValueOrDie()) if the platform we intend to test is not available.
-Client* GetOrCreateLocalClientOrDie(const LocalClientOptions& client_options) {
-  StatusOr<Client*> result =
+LocalClient* GetOrCreateLocalClientOrDie(
+    const LocalClientOptions& client_options) {
+  StatusOr<LocalClient*> result =
       ClientLibrary::GetOrCreateLocalClient(client_options);
   TF_CHECK_OK(result.status()) << " could not create local client for testing";
   return result.ValueOrDie();
diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.h b/tensorflow/compiler/xla/tests/client_library_test_base.h
index 3f65ed7fce..d700437ed3 100644
--- a/tensorflow/compiler/xla/tests/client_library_test_base.h
+++ b/tensorflow/compiler/xla/tests/client_library_test_base.h
@@ -385,8 +385,8 @@ class ClientLibraryTestBase : public ::testing::Test {
   StatusOr<std::pair<Literal, Literal>> ComputeValueAndReference(
       XlaBuilder* builder, absl::Span<const Literal> arguments);
 
-  Client* client_;
-  Client* ref_client_;  // To compute reference result.
+  LocalClient* client_;
+  LocalClient* ref_client_;  // To compute reference result.
   ExecutionOptions execution_options_;
 
  private:
-- 
GitLab


From a677d1b604ff3df1a5f5499dac148e3bc6d8df57 Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Mon, 28 Jan 2019 15:07:32 -0800
Subject: [PATCH 1424/2345] Add Keil project generation to TensorFlow Lite for
 Microcontrollers

PiperOrigin-RevId: 231296755
---
 .../experimental/micro/tools/make/Makefile    |  13 +-
 .../micro/tools/make/generate_keil_project.py | 117 +++++
 .../tools/make/generate_keil_project_test.sh  |  69 +++
 .../micro/tools/make/helper_functions.inc     |   7 +
 .../tools/make/templates/README_KEIL.md.tpl   |   5 +
 .../make/templates/keil_project.uvprojx.tpl   | 418 ++++++++++++++++++
 6 files changed, 626 insertions(+), 3 deletions(-)
 create mode 100644 tensorflow/lite/experimental/micro/tools/make/generate_keil_project.py
 create mode 100755 tensorflow/lite/experimental/micro/tools/make/generate_keil_project_test.sh
 create mode 100644 tensorflow/lite/experimental/micro/tools/make/templates/README_KEIL.md.tpl
 create mode 100644 tensorflow/lite/experimental/micro/tools/make/templates/keil_project.uvprojx.tpl

diff --git a/tensorflow/lite/experimental/micro/tools/make/Makefile b/tensorflow/lite/experimental/micro/tools/make/Makefile
index 8be57cc18d..89df595b4f 100644
--- a/tensorflow/lite/experimental/micro/tools/make/Makefile
+++ b/tensorflow/lite/experimental/micro/tools/make/Makefile
@@ -39,9 +39,12 @@ INCLUDES := \
 -I$(MAKEFILE_DIR)/downloads/gemmlowp \
 -I$(MAKEFILE_DIR)/downloads/flatbuffers/include \
 -I$(OBJDIR)
-# This is at the end so any globally-installed frameworks like protobuf don't
-# override local versions in the source tree.
-INCLUDES += -I/usr/local/include
+
+# These are the include paths added to any generated project file.
+PROJECT_INCLUDES := \
+. \
+third_party/gemmlowp \
+third_party/flatbuffers/include
 
 TEST_SCRIPT := tensorflow/lite/experimental/micro/testing/test_linux_binary.sh
 
@@ -125,6 +128,10 @@ MBED_PROJECT_FILES := \
   mbed-os.lib \
   mbed_app.json
 
+KEIL_PROJECT_FILES := \
+  README_KEIL.md \
+  keil_project.uvprojx
+
 # These target-specific makefiles should modify or replace options like
 # CXXFLAGS or LIBS to work for a specific targetted architecture. All logic
 # based on platforms or architectures should happen within these files, to
diff --git a/tensorflow/lite/experimental/micro/tools/make/generate_keil_project.py b/tensorflow/lite/experimental/micro/tools/make/generate_keil_project.py
new file mode 100644
index 0000000000..fb0c632e24
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/tools/make/generate_keil_project.py
@@ -0,0 +1,117 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Generates a Keil uVision project file from a template."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import argparse
+import os.path
+import re
+
+
+def sanitize_xml(unsanitized):
+  """Uses a whitelist to avoid generating bad XML."""
+  return re.sub(r'[^a-zA-Z0-9+_\-/\\.]', '', unsanitized)
+
+
+def main(unused_args, flags):
+  """Generates a Keil project file from a template source."""
+  with open(flags.input_template, 'r') as input_template_file:
+    template_file_text = input_template_file.read()
+
+  template_file_text = re.sub(r'%{EXECUTABLE}%', flags.executable,
+                              template_file_text)
+
+  srcs_list = flags.srcs.split(' ')
+  hdrs_list = flags.hdrs.split(' ')
+  all_srcs_list = srcs_list + hdrs_list
+  all_srcs_list.sort()
+
+  replace_srcs = ''
+  for src in all_srcs_list:
+    if not src:
+      continue
+    ext = os.path.splitext(src)[1]
+    # These extension indexes are used by uVision to keep track of the type
+    # of files. I determined them by experimentation, since the file format
+    # isn't documented.
+    if ext == '.h':
+      ext_index = '5'
+    elif ext == '.c':
+      ext_index = '1'
+    elif ext == '.cc' or ext == '.cpp':
+      ext_index = '8'
+    else:
+      ext_index = '5'
+    basename = sanitize_xml(os.path.basename(src))
+    clean_src = sanitize_xml(src)
+    replace_srcs += '            <File>\n'
+    replace_srcs += '              <FileName>' + basename + '</FileName>\n'
+    replace_srcs += '              <FileType>' + ext_index + '</FileType>\n'
+    replace_srcs += '              <FilePath>' + clean_src + '</FilePath>\n'
+    replace_srcs += '            </File>\n'
+  template_file_text = re.sub(r'%{SRCS}%', replace_srcs, template_file_text)
+
+  include_paths = re.sub(' ', ';', flags.include_paths)
+  template_file_text = re.sub(r'%{INCLUDE_PATHS}%', include_paths,
+                              template_file_text)
+
+  with open(flags.output_file, 'w') as output_file:
+    output_file.write(template_file_text)
+
+
+def parse_args():
+  """Converts the raw arguments into accessible flags."""
+  parser = argparse.ArgumentParser()
+  parser.register('type', 'bool', lambda v: v.lower() == 'true')
+  parser.add_argument(
+      '--input_template',
+      type=str,
+      default='',
+      help='Path to template project file to build from.')
+  parser.add_argument(
+      '--output_file',
+      type=str,
+      default='',
+      help='Path to write the completed project file to.')
+  parser.add_argument(
+      '--executable',
+      type=str,
+      default='',
+      help='Name of the executable the project will build.')
+  parser.add_argument(
+      '--hdrs',
+      type=str,
+      default='',
+      help='Space-separated list of C or C++ source files to compile.')
+  parser.add_argument(
+      '--srcs',
+      type=str,
+      default='',
+      help='Space-separated list of C or C++ header files to include.')
+  parser.add_argument(
+      '--include_paths',
+      type=str,
+      default='',
+      help='Space-separated list of paths to look for header files on.')
+  flags, unparsed = parser.parse_known_args()
+
+  main(unparsed, flags)
+
+
+if __name__ == '__main__':
+  parse_args()
diff --git a/tensorflow/lite/experimental/micro/tools/make/generate_keil_project_test.sh b/tensorflow/lite/experimental/micro/tools/make/generate_keil_project_test.sh
new file mode 100755
index 0000000000..22b68e4f68
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/tools/make/generate_keil_project_test.sh
@@ -0,0 +1,69 @@
+#!/bin/bash
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+#
+# Bash unit tests for the TensorFlow Lite Micro project generator.
+
+set -e
+
+INPUT_TEMPLATE=${TEST_SRCDIR}/tensorflow/lite/experimental/micro/tools/make/templates/keil_project.uvprojx.tpl
+OUTPUT_FILE=${TEST_TMPDIR}/keil_project.uvprojx
+EXECUTABLE=test_executable
+
+${TEST_SRCDIR}/tensorflow/lite/experimental/micro/tools/make/generate_keil_project \
+  --input_template=${INPUT_TEMPLATE} \
+  --output_file=${OUTPUT_FILE} \
+  --executable=${EXECUTABLE} \
+  --hdrs="foo.h bar.h" \
+  --srcs="foo.c bar.cc some/bad<xml.cc" \
+  --include_paths=". include"
+
+if ! grep -q "${EXECUTABLE}" ${OUTPUT_FILE}; then
+  echo "ERROR: No executable name '${EXECUTABLE}' found in project file '${OUTPUT_FILE}'."
+  exit 1
+fi
+
+if ! grep -q "foo\.h" ${OUTPUT_FILE}; then
+  echo "ERROR: No header 'foo.h' found in project file '${OUTPUT_FILE}'."
+  exit 1
+fi
+
+if ! grep -q "bar\.h" ${OUTPUT_FILE}; then
+  echo "ERROR: No header 'bar.h' found in project file '${OUTPUT_FILE}'."
+  exit 1
+fi
+
+if ! grep -q "foo\.c" ${OUTPUT_FILE}; then
+  echo "ERROR: No source 'foo.c' found in project file '${OUTPUT_FILE}'."
+  exit 1
+fi
+
+if ! grep -q "bar\.cc" ${OUTPUT_FILE}; then
+  echo "ERROR: No source 'bar.cc' found in project file '${OUTPUT_FILE}'."
+  exit 1
+fi
+
+if ! grep -q "some/badxml\.cc" ${OUTPUT_FILE}; then
+  echo "ERROR: No source 'some/badxml.cc' found in project file '${OUTPUT_FILE}'."
+  exit 1
+fi
+
+if ! grep -q "\.;include" ${OUTPUT_FILE}; then
+  echo "ERROR: No include paths '.;include' found in project file '${OUTPUT_FILE}'."
+  exit 1
+fi
+
+echo
+echo "SUCCESS: generate_keil_project test PASSED"
diff --git a/tensorflow/lite/experimental/micro/tools/make/helper_functions.inc b/tensorflow/lite/experimental/micro/tools/make/helper_functions.inc
index c20cc5cfb3..4d2465b5e9 100644
--- a/tensorflow/lite/experimental/micro/tools/make/helper_functions.inc
+++ b/tensorflow/lite/experimental/micro/tools/make/helper_functions.inc
@@ -70,6 +70,12 @@ $(PRJDIR)$(3)/$(1)/%: tensorflow/lite/experimental/micro/tools/make/templates/%.
 	sed -E 's#\%\{LINKER_FLAGS\}\%#$(6)#g' | \
 	sed -E 's#\%\{CXX_FLAGS\}\%#$(7)#g' > $$@
 
+$(PRJDIR)$(3)/$(1)/keil_project.uvprojx: tensorflow/lite/experimental/micro/tools/make/templates/keil_project.uvprojx.tpl
+	@mkdir -p $$(dir $$@)
+	python tensorflow/lite/experimental/micro/tools/make/generate_keil_project.py \
+        --input_template=$$< --output_file=$$@ --executable=$(3) \
+        --srcs="$(4)" --hdrs="$(5)" --include_paths="$$(PROJECT_INCLUDES)"
+
 generate_$(3)_$(1)_project: $(addprefix $(PRJDIR)$(3)/$(1)/, $(4) $(5) $(2))
 endef
 
@@ -85,6 +91,7 @@ endef
 define generate_microlite_projects
 $(call generate_project,make,$(MAKE_PROJECT_FILES),$(1),$(MICROLITE_CC_SRCS) $(THIRD_PARTY_CC_SRCS) $(2),$(MICROLITE_CC_HDRS) $(THIRD_PARTY_CC_HDRS) $(MICROLITE_TEST_HDRS) $(3),$(MICROLITE_LIBS),$(CXXFLAGS))
 $(call generate_project,mbed,$(MBED_PROJECT_FILES),$(1),$(MICROLITE_CC_SRCS) $(THIRD_PARTY_CC_SRCS) $(2),$(MICROLITE_CC_HDRS) $(THIRD_PARTY_CC_HDRS) $(MICROLITE_TEST_HDRS) $(3),$(MICROLITE_LIBS),$(CXXFLAGS))
+$(call generate_project,keil,$(KEIL_PROJECT_FILES),$(1),$(MICROLITE_CC_SRCS) $(THIRD_PARTY_CC_SRCS) $(2),$(MICROLITE_CC_HDRS) $(THIRD_PARTY_CC_HDRS) $(MICROLITE_TEST_HDRS) $(3),$(MICROLITE_LIBS),$(CXXFLAGS))
 endef
 
 
diff --git a/tensorflow/lite/experimental/micro/tools/make/templates/README_KEIL.md.tpl b/tensorflow/lite/experimental/micro/tools/make/templates/README_KEIL.md.tpl
new file mode 100644
index 0000000000..945b9f9c1a
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/tools/make/templates/README_KEIL.md.tpl
@@ -0,0 +1,5 @@
+# TensorFlow Lite Micro Mbed Project
+
+This folder has been autogenerated by TensorFlow, and contains source, header,
+and project files needed to build a single TensorFlow Lite Micro target using
+the Keil uVision IDE.
diff --git a/tensorflow/lite/experimental/micro/tools/make/templates/keil_project.uvprojx.tpl b/tensorflow/lite/experimental/micro/tools/make/templates/keil_project.uvprojx.tpl
new file mode 100644
index 0000000000..440d4b6b3e
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/tools/make/templates/keil_project.uvprojx.tpl
@@ -0,0 +1,418 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
+<Project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="project_projx.xsd">
+
+  <SchemaVersion>2.1</SchemaVersion>
+
+  <Targets>
+    <Target>
+      <TargetName>%{EXECUTABLE}%</TargetName>
+      <ToolsetNumber>0x4</ToolsetNumber>
+      <ToolsetName>ARM-ADS</ToolsetName>
+      <pCCUsed>6100001::V6.10.1::.\ARMCLANG</pCCUsed>
+      <uAC6>1</uAC6>
+      <TargetOption>
+        <TargetCommonOption>
+          <Device>STM32F746NGHx</Device>
+          <Vendor>STMicroelectronics</Vendor>
+          <PackID>Keil.STM32F7xx_DFP.2.11.0</PackID>
+          <PackURL>http://www.keil.com/pack</PackURL>
+          <Cpu>IRAM(0x20010000,0x40000) IRAM2(0x20000000,0x10000) IROM(0x08000000,0x100000) IROM2(0x00200000,0x100000) CPUTYPE("Cortex-M7") FPU3(SFPU) CLOCK(12000000) ELITTLE</Cpu>
+          <FlashUtilSpec></FlashUtilSpec>
+          <StartupFile></StartupFile>
+          <FlashDriverDll>UL2CM3(-S0 -C0 -P0 -FD20010000 -FC1000 -FN2 -FF0STM32F7x_1024 -FS08000000 -FL0100000 -FF1STM32F7xTCM_1024 -FS1200000 -FL1100000 -FP0($$Device:STM32F746NGHx$CMSIS\Flash\STM32F7x_1024.FLM) -FP1($$Device:STM32F746NGHx$CMSIS\Flash\STM32F7xTCM_1024.FLM))</FlashDriverDll>
+          <DeviceId>0</DeviceId>
+          <RegisterFile>$$Device:STM32F746NGHx$Drivers\CMSIS\Device\ST\STM32F7xx\Include\stm32f7xx.h</RegisterFile>
+          <MemoryEnv></MemoryEnv>
+          <Cmp></Cmp>
+          <Asm></Asm>
+          <Linker></Linker>
+          <OHString></OHString>
+          <InfinionOptionDll></InfinionOptionDll>
+          <SLE66CMisc></SLE66CMisc>
+          <SLE66AMisc></SLE66AMisc>
+          <SLE66LinkerMisc></SLE66LinkerMisc>
+          <SFDFile>$$Device:STM32F746NGHx$CMSIS\SVD\STM32F7x6_v1r1.svd</SFDFile>
+          <bCustSvd>0</bCustSvd>
+          <UseEnv>0</UseEnv>
+          <BinPath></BinPath>
+          <IncludePath></IncludePath>
+          <LibPath></LibPath>
+          <RegisterFilePath></RegisterFilePath>
+          <DBRegisterFilePath></DBRegisterFilePath>
+          <TargetStatus>
+            <Error>0</Error>
+            <ExitCodeStop>0</ExitCodeStop>
+            <ButtonStop>0</ButtonStop>
+            <NotGenerated>0</NotGenerated>
+            <InvalidFlash>1</InvalidFlash>
+          </TargetStatus>
+          <OutputDirectory>.\Objects\</OutputDirectory>
+          <OutputName>%{EXECUTABLE}%</OutputName>
+          <CreateExecutable>1</CreateExecutable>
+          <CreateLib>0</CreateLib>
+          <CreateHexFile>0</CreateHexFile>
+          <DebugInformation>1</DebugInformation>
+          <BrowseInformation>1</BrowseInformation>
+          <ListingPath>.\Listings\</ListingPath>
+          <HexFormatSelection>1</HexFormatSelection>
+          <Merge32K>0</Merge32K>
+          <CreateBatchFile>0</CreateBatchFile>
+          <BeforeCompile>
+            <RunUserProg1>0</RunUserProg1>
+            <RunUserProg2>0</RunUserProg2>
+            <UserProg1Name></UserProg1Name>
+            <UserProg2Name></UserProg2Name>
+            <UserProg1Dos16Mode>0</UserProg1Dos16Mode>
+            <UserProg2Dos16Mode>0</UserProg2Dos16Mode>
+            <nStopU1X>0</nStopU1X>
+            <nStopU2X>0</nStopU2X>
+          </BeforeCompile>
+          <BeforeMake>
+            <RunUserProg1>0</RunUserProg1>
+            <RunUserProg2>0</RunUserProg2>
+            <UserProg1Name></UserProg1Name>
+            <UserProg2Name></UserProg2Name>
+            <UserProg1Dos16Mode>0</UserProg1Dos16Mode>
+            <UserProg2Dos16Mode>0</UserProg2Dos16Mode>
+            <nStopB1X>0</nStopB1X>
+            <nStopB2X>0</nStopB2X>
+          </BeforeMake>
+          <AfterMake>
+            <RunUserProg1>0</RunUserProg1>
+            <RunUserProg2>0</RunUserProg2>
+            <UserProg1Name></UserProg1Name>
+            <UserProg2Name></UserProg2Name>
+            <UserProg1Dos16Mode>0</UserProg1Dos16Mode>
+            <UserProg2Dos16Mode>0</UserProg2Dos16Mode>
+            <nStopA1X>0</nStopA1X>
+            <nStopA2X>0</nStopA2X>
+          </AfterMake>
+          <SelectedForBatchBuild>0</SelectedForBatchBuild>
+          <SVCSIdString></SVCSIdString>
+        </TargetCommonOption>
+        <CommonProperty>
+          <UseCPPCompiler>0</UseCPPCompiler>
+          <RVCTCodeConst>0</RVCTCodeConst>
+          <RVCTZI>0</RVCTZI>
+          <RVCTOtherData>0</RVCTOtherData>
+          <ModuleSelection>0</ModuleSelection>
+          <IncludeInBuild>1</IncludeInBuild>
+          <AlwaysBuild>0</AlwaysBuild>
+          <GenerateAssemblyFile>0</GenerateAssemblyFile>
+          <AssembleAssemblyFile>0</AssembleAssemblyFile>
+          <PublicsOnly>0</PublicsOnly>
+          <StopOnExitCode>3</StopOnExitCode>
+          <CustomArgument></CustomArgument>
+          <IncludeLibraryModules></IncludeLibraryModules>
+          <ComprImg>1</ComprImg>
+        </CommonProperty>
+        <DllOption>
+          <SimDllName>SARMCM3.DLL</SimDllName>
+          <SimDllArguments> -REMAP -MPU</SimDllArguments>
+          <SimDlgDll>DCM.DLL</SimDlgDll>
+          <SimDlgDllArguments>-pCM7</SimDlgDllArguments>
+          <TargetDllName>SARMCM3.DLL</TargetDllName>
+          <TargetDllArguments> -MPU</TargetDllArguments>
+          <TargetDlgDll>TCM.DLL</TargetDlgDll>
+          <TargetDlgDllArguments>-pCM7</TargetDlgDllArguments>
+        </DllOption>
+        <DebugOption>
+          <OPTHX>
+            <HexSelection>1</HexSelection>
+            <HexRangeLowAddress>0</HexRangeLowAddress>
+            <HexRangeHighAddress>0</HexRangeHighAddress>
+            <HexOffset>0</HexOffset>
+            <Oh166RecLen>16</Oh166RecLen>
+          </OPTHX>
+        </DebugOption>
+        <Utilities>
+          <Flash1>
+            <UseTargetDll>1</UseTargetDll>
+            <UseExternalTool>0</UseExternalTool>
+            <RunIndependent>0</RunIndependent>
+            <UpdateFlashBeforeDebugging>1</UpdateFlashBeforeDebugging>
+            <Capability>1</Capability>
+            <DriverSelection>-1</DriverSelection>
+          </Flash1>
+          <bUseTDR>1</bUseTDR>
+          <Flash2>BIN\UL2CM3.DLL</Flash2>
+          <Flash3></Flash3>
+          <Flash4></Flash4>
+          <pFcarmOut></pFcarmOut>
+          <pFcarmGrp></pFcarmGrp>
+          <pFcArmRoot></pFcArmRoot>
+          <FcArmLst>0</FcArmLst>
+        </Utilities>
+        <TargetArmAds>
+          <ArmAdsMisc>
+            <GenerateListings>0</GenerateListings>
+            <asHll>1</asHll>
+            <asAsm>1</asAsm>
+            <asMacX>1</asMacX>
+            <asSyms>1</asSyms>
+            <asFals>1</asFals>
+            <asDbgD>1</asDbgD>
+            <asForm>1</asForm>
+            <ldLst>0</ldLst>
+            <ldmm>1</ldmm>
+            <ldXref>1</ldXref>
+            <BigEnd>0</BigEnd>
+            <AdsALst>1</AdsALst>
+            <AdsACrf>1</AdsACrf>
+            <AdsANop>0</AdsANop>
+            <AdsANot>0</AdsANot>
+            <AdsLLst>1</AdsLLst>
+            <AdsLmap>1</AdsLmap>
+            <AdsLcgr>1</AdsLcgr>
+            <AdsLsym>1</AdsLsym>
+            <AdsLszi>1</AdsLszi>
+            <AdsLtoi>1</AdsLtoi>
+            <AdsLsun>1</AdsLsun>
+            <AdsLven>1</AdsLven>
+            <AdsLsxf>1</AdsLsxf>
+            <RvctClst>0</RvctClst>
+            <GenPPlst>0</GenPPlst>
+            <AdsCpuType>"Cortex-M7"</AdsCpuType>
+            <RvctDeviceName></RvctDeviceName>
+            <mOS>0</mOS>
+            <uocRom>0</uocRom>
+            <uocRam>0</uocRam>
+            <hadIROM>1</hadIROM>
+            <hadIRAM>1</hadIRAM>
+            <hadXRAM>0</hadXRAM>
+            <uocXRam>0</uocXRam>
+            <RvdsVP>2</RvdsVP>
+            <RvdsMve>0</RvdsMve>
+            <hadIRAM2>1</hadIRAM2>
+            <hadIROM2>1</hadIROM2>
+            <StupSel>8</StupSel>
+            <useUlib>0</useUlib>
+            <EndSel>0</EndSel>
+            <uLtcg>0</uLtcg>
+            <nSecure>0</nSecure>
+            <RoSelD>4</RoSelD>
+            <RwSelD>4</RwSelD>
+            <CodeSel>0</CodeSel>
+            <OptFeed>0</OptFeed>
+            <NoZi1>0</NoZi1>
+            <NoZi2>0</NoZi2>
+            <NoZi3>0</NoZi3>
+            <NoZi4>0</NoZi4>
+            <NoZi5>0</NoZi5>
+            <Ro1Chk>0</Ro1Chk>
+            <Ro2Chk>0</Ro2Chk>
+            <Ro3Chk>0</Ro3Chk>
+            <Ir1Chk>1</Ir1Chk>
+            <Ir2Chk>0</Ir2Chk>
+            <Ra1Chk>0</Ra1Chk>
+            <Ra2Chk>0</Ra2Chk>
+            <Ra3Chk>0</Ra3Chk>
+            <Im1Chk>1</Im1Chk>
+            <Im2Chk>1</Im2Chk>
+            <OnChipMemories>
+              <Ocm1>
+                <Type>0</Type>
+                <StartAddress>0x0</StartAddress>
+                <Size>0x0</Size>
+              </Ocm1>
+              <Ocm2>
+                <Type>0</Type>
+                <StartAddress>0x0</StartAddress>
+                <Size>0x0</Size>
+              </Ocm2>
+              <Ocm3>
+                <Type>0</Type>
+                <StartAddress>0x0</StartAddress>
+                <Size>0x0</Size>
+              </Ocm3>
+              <Ocm4>
+                <Type>0</Type>
+                <StartAddress>0x0</StartAddress>
+                <Size>0x0</Size>
+              </Ocm4>
+              <Ocm5>
+                <Type>0</Type>
+                <StartAddress>0x0</StartAddress>
+                <Size>0x0</Size>
+              </Ocm5>
+              <Ocm6>
+                <Type>0</Type>
+                <StartAddress>0x0</StartAddress>
+                <Size>0x0</Size>
+              </Ocm6>
+              <IRAM>
+                <Type>0</Type>
+                <StartAddress>0x20010000</StartAddress>
+                <Size>0x40000</Size>
+              </IRAM>
+              <IROM>
+                <Type>1</Type>
+                <StartAddress>0x8000000</StartAddress>
+                <Size>0x100000</Size>
+              </IROM>
+              <XRAM>
+                <Type>0</Type>
+                <StartAddress>0x0</StartAddress>
+                <Size>0x0</Size>
+              </XRAM>
+              <OCR_RVCT1>
+                <Type>1</Type>
+                <StartAddress>0x0</StartAddress>
+                <Size>0x0</Size>
+              </OCR_RVCT1>
+              <OCR_RVCT2>
+                <Type>1</Type>
+                <StartAddress>0x0</StartAddress>
+                <Size>0x0</Size>
+              </OCR_RVCT2>
+              <OCR_RVCT3>
+                <Type>1</Type>
+                <StartAddress>0x0</StartAddress>
+                <Size>0x0</Size>
+              </OCR_RVCT3>
+              <OCR_RVCT4>
+                <Type>1</Type>
+                <StartAddress>0x8000000</StartAddress>
+                <Size>0x100000</Size>
+              </OCR_RVCT4>
+              <OCR_RVCT5>
+                <Type>1</Type>
+                <StartAddress>0x200000</StartAddress>
+                <Size>0x100000</Size>
+              </OCR_RVCT5>
+              <OCR_RVCT6>
+                <Type>0</Type>
+                <StartAddress>0x0</StartAddress>
+                <Size>0x0</Size>
+              </OCR_RVCT6>
+              <OCR_RVCT7>
+                <Type>0</Type>
+                <StartAddress>0x0</StartAddress>
+                <Size>0x0</Size>
+              </OCR_RVCT7>
+              <OCR_RVCT8>
+                <Type>0</Type>
+                <StartAddress>0x0</StartAddress>
+                <Size>0x0</Size>
+              </OCR_RVCT8>
+              <OCR_RVCT9>
+                <Type>0</Type>
+                <StartAddress>0x20010000</StartAddress>
+                <Size>0x40000</Size>
+              </OCR_RVCT9>
+              <OCR_RVCT10>
+                <Type>0</Type>
+                <StartAddress>0x20000000</StartAddress>
+                <Size>0x10000</Size>
+              </OCR_RVCT10>
+            </OnChipMemories>
+            <RvctStartVector></RvctStartVector>
+          </ArmAdsMisc>
+          <Cads>
+            <interw>1</interw>
+            <Optim>7</Optim>
+            <oTime>0</oTime>
+            <SplitLS>0</SplitLS>
+            <OneElfS>1</OneElfS>
+            <Strict>0</Strict>
+            <EnumInt>0</EnumInt>
+            <PlainCh>0</PlainCh>
+            <Ropi>0</Ropi>
+            <Rwpi>0</Rwpi>
+            <wLevel>3</wLevel>
+            <uThumb>0</uThumb>
+            <uSurpInc>0</uSurpInc>
+            <uC99>0</uC99>
+            <uGnu>1</uGnu>
+            <useXO>0</useXO>
+            <v6Lang>3</v6Lang>
+            <v6LangP>3</v6LangP>
+            <vShortEn>1</vShortEn>
+            <vShortWch>1</vShortWch>
+            <v6Lto>0</v6Lto>
+            <v6WtE>0</v6WtE>
+            <v6Rtti>0</v6Rtti>
+            <VariousControls>
+              <MiscControls></MiscControls>
+              <Define></Define>
+              <Undefine></Undefine>
+              <IncludePath>%{INCLUDE_PATHS}%</IncludePath>
+            </VariousControls>
+          </Cads>
+          <Aads>
+            <interw>1</interw>
+            <Ropi>0</Ropi>
+            <Rwpi>0</Rwpi>
+            <thumb>0</thumb>
+            <SplitLS>0</SplitLS>
+            <SwStkChk>0</SwStkChk>
+            <NoWarn>0</NoWarn>
+            <uSurpInc>0</uSurpInc>
+            <useXO>0</useXO>
+            <uClangAs>0</uClangAs>
+            <VariousControls>
+              <MiscControls></MiscControls>
+              <Define></Define>
+              <Undefine></Undefine>
+              <IncludePath></IncludePath>
+            </VariousControls>
+          </Aads>
+          <LDads>
+            <umfTarg>0</umfTarg>
+            <Ropi>0</Ropi>
+            <Rwpi>0</Rwpi>
+            <noStLib>0</noStLib>
+            <RepFail>1</RepFail>
+            <useFile>0</useFile>
+            <TextAddressRange>0x08000000</TextAddressRange>
+            <DataAddressRange>0x20010000</DataAddressRange>
+            <pXoBase></pXoBase>
+            <ScatterFile></ScatterFile>
+            <IncludeLibs></IncludeLibs>
+            <IncludeLibsPath></IncludeLibsPath>
+            <Misc></Misc>
+            <LinkerInputFile></LinkerInputFile>
+            <DisabledWarnings></DisabledWarnings>
+          </LDads>
+        </TargetArmAds>
+      </TargetOption>
+      <Groups>
+        <Group>
+          <GroupName>Source</GroupName>
+          <Files>
+%{SRCS}%
+          </Files>
+        </Group>
+        <Group>
+          <GroupName>::Compiler</GroupName>
+        </Group>
+      </Groups>
+    </Target>
+  </Targets>
+
+  <RTE>
+    <apis/>
+    <components>
+      <component Cbundle="ARM Compiler" Cclass="Compiler" Cgroup="I/O" Csub="STDERR" Cvariant="ITM" Cvendor="Keil" Cversion="1.2.0" condition="ARMCC Cortex-M with ITM">
+        <package name="ARM_Compiler" schemaVersion="1.4.9" url="http://www.keil.com/pack/" vendor="Keil" version="1.6.0"/>
+        <targetInfos>
+          <targetInfo name="%{EXECUTABLE}%"/>
+        </targetInfos>
+      </component>
+      <component Cbundle="ARM Compiler" Cclass="Compiler" Cgroup="I/O" Csub="STDIN" Cvariant="ITM" Cvendor="Keil" Cversion="1.2.0" condition="ARMCC Cortex-M with ITM">
+        <package name="ARM_Compiler" schemaVersion="1.4.9" url="http://www.keil.com/pack/" vendor="Keil" version="1.6.0"/>
+        <targetInfos>
+          <targetInfo name="%{EXECUTABLE}%"/>
+        </targetInfos>
+      </component>
+      <component Cbundle="ARM Compiler" Cclass="Compiler" Cgroup="I/O" Csub="STDOUT" Cvariant="ITM" Cvendor="Keil" Cversion="1.2.0" condition="ARMCC Cortex-M with ITM">
+        <package name="ARM_Compiler" schemaVersion="1.4.9" url="http://www.keil.com/pack/" vendor="Keil" version="1.6.0"/>
+        <targetInfos>
+          <targetInfo name="%{EXECUTABLE}%"/>
+        </targetInfos>
+      </component>
+    </components>
+    <files/>
+  </RTE>
+
+</Project>
-- 
GitLab


From 33bf4d37ff71950cdcffdfeda36d8e0995cc67b2 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Mon, 28 Jan 2019 15:14:08 -0800
Subject: [PATCH 1425/2345] [XLA:CPU] Remove explicit type whitelist for
 ReduceWindow in CPU backend.

Pretty much any type should work here, and I have a use case for 64-bit integer types.

PiperOrigin-RevId: 231297954
---
 .../compiler/xla/service/cpu/ir_emitter.cc    |  5 -----
 .../compiler/xla/tests/reduce_window_test.cc  | 19 +++++++++++++++++++
 2 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index 0226e8275c..b26bfffc6a 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -765,11 +765,6 @@ StatusOr<llvm::Value*> IrEmitter::EmitTargetElementLoopBodyForReduceWindow(
 }
 
 Status IrEmitter::HandleReduceWindow(HloInstruction* reduce_window) {
-  TF_RETURN_IF_ERROR(ElementTypesSameAndSupported(
-      /*instruction=*/*reduce_window,
-      /*operands=*/{reduce_window->operand(0)},
-      /*supported_types=*/{F32, BF16, S32, F16}));
-
   // Pseudo code for reduce window:
   //
   //   for (coordinates O in the output)
diff --git a/tensorflow/compiler/xla/tests/reduce_window_test.cc b/tensorflow/compiler/xla/tests/reduce_window_test.cc
index 16c67d94c7..30e2d24184 100644
--- a/tensorflow/compiler/xla/tests/reduce_window_test.cc
+++ b/tensorflow/compiler/xla/tests/reduce_window_test.cc
@@ -1532,6 +1532,25 @@ ENTRY %reduce-window (parameter.0: s32[81,8], parameter.1: s32[]) -> s32[82,8] {
   EXPECT_TRUE(RunAndCompare(hlo_string, absl::nullopt));
 }
 
+XLA_TEST_F(HloTestBase, ReduceWindowS64) {
+  const string hlo_string = R"(
+HloModule reduce-window
+
+%identity.pad_to_reduce_window (param0: s64[], param1: s64[]) -> s64[] {
+  %param0 = s64[] parameter(0)
+  ROOT %param1 = s64[] parameter(1)
+}
+
+ENTRY %reduce-window (parameter.0: s64[81,8], parameter.1: s64[]) -> s64[82,8] {
+  %parameter.0 = s64[81,8]{1,0} parameter(0)
+  %parameter.1 = s64[] parameter(1)
+  ROOT %reduce-window = s64[82,8]{1,0} reduce-window(s64[81,8]{1,0} %parameter.0, s64[] %parameter.1), window={size=1x1 pad=0_1x0_0}, to_apply=%identity.pad_to_reduce_window
+}
+
+)";
+  EXPECT_TRUE(RunAndCompare(hlo_string, absl::nullopt));
+}
+
 XLA_TEST_F(HloTestBase, ReduceWindowF16) {
   const string hlo_string = R"(
 HloModule reduce-window
-- 
GitLab


From 83464708268433415bfddc63f9df581c8c7b2420 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Mon, 28 Jan 2019 15:24:02 -0800
Subject: [PATCH 1426/2345] [XLA] Use the LocalClient API in
 exhaustive_f32_elementwise_op_test.

The LocalClient API doesn't have to marshal parameters into protobufs.  This is
a significant speedup: One shard goes from 1s to 800ms.

PiperOrigin-RevId: 231299741
---
 .../exhaustive_f32_elementwise_op_test.cc     | 45 ++++++++++++++-----
 1 file changed, 35 insertions(+), 10 deletions(-)

diff --git a/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc b/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc
index 139d5c70b8..6634655371 100644
--- a/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc
+++ b/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc
@@ -52,20 +52,45 @@ class ExhaustiveF32ElementwiseOpTest
       }
     }
 
-    TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<GlobalData> input_data,
-                            client_->TransferToServer(input_literal));
-
     auto input = Parameter(&builder, 0, input_literal.shape(), "input");
     enqueue_op(&builder, input);
-
-    std::vector<float> expected_result;
-    expected_result.reserve(input_size);
+    TF_ASSERT_OK_AND_ASSIGN(XlaComputation comp, builder.Build());
+
+    // Build and run the computation using the LocalClient API, rather than the
+    // plain Client API, which is used by ClientLibraryTestBase.  This is
+    // because the plain Client API results does more memcpys to/from Literals,
+    // and that's slow given that we're touching a lot of data here.
+    //
+    // Copy debug options from ClientLibraryTestBase.  In particular, we're
+    // interested in disabling constant folding.
+    ExecutableBuildOptions build_opts;
+    *build_opts.mutable_debug_options() = *mutable_debug_options();
+    TF_ASSERT_OK_AND_ASSIGN(
+        auto executable,
+        client_->Compile(comp, {&input_literal.shape()}, build_opts));
+
+    TF_ASSERT_OK_AND_ASSIGN(
+        ScopedShapedBuffer input_data,
+        client_->LiteralToShapedBuffer(input_literal, /*device_ordinal=*/0));
+
+    ExecutableRunOptions run_opts;
+    run_opts.set_allocator(client_->backend().memory_allocator());
+    run_opts.set_intra_op_thread_pool(
+        client_->backend().eigen_intra_op_thread_pool_device());
+    TF_ASSERT_OK_AND_ASSIGN(ScopedShapedBuffer result,
+                            executable->Run({&input_data}, run_opts));
+
+    TF_ASSERT_OK_AND_ASSIGN(Literal result_literal,
+                            client_->ShapedBufferToLiteral(result));
+
+    Literal expected_literal =
+        LiteralUtil::CreateFromDimensions(F32, {input_size});
+    absl::Span<float> expected_arr = expected_literal.data<float>();
     for (int64 i = 0; i < input_size; i++) {
-      expected_result.push_back(evaluate_op(input_arr[i]));
+      expected_arr[i] = evaluate_op(input_arr[i]);
     }
-
-    ComputeAndCompareR1<float>(&builder, expected_result, {input_data.get()},
-                               error_spec_);
+    EXPECT_TRUE(
+        LiteralTestUtil::Near(expected_literal, result_literal, error_spec_));
   }
 };
 
-- 
GitLab


From 7fcc8dab6578f73d69e71c0bc9ce1d546ac19c7b Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Mon, 28 Jan 2019 15:35:55 -0800
Subject: [PATCH 1427/2345] Add while loop support for dynamic dimension
 inference.

- Add while support
- Add tuple support
- Implement ToString function

PiperOrigin-RevId: 231301670
---
 tensorflow/compiler/xla/service/BUILD         |   1 +
 .../service/dynamic_dimension_inference.cc    | 157 ++++++++++++++++++
 .../xla/service/dynamic_dimension_inference.h |   7 +
 .../dynamic_dimension_inference_test.cc       |  90 ++++++++++
 4 files changed, 255 insertions(+)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 34af6b3597..e9a6ce2ade 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -1930,6 +1930,7 @@ cc_library(
     hdrs = ["dynamic_dimension_inference.h"],
     deps = [
         ":hlo",
+        ":while_util",
         "//tensorflow/compiler/xla:status",
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:types",
diff --git a/tensorflow/compiler/xla/service/dynamic_dimension_inference.cc b/tensorflow/compiler/xla/service/dynamic_dimension_inference.cc
index 2b158d7a6e..de3b508064 100644
--- a/tensorflow/compiler/xla/service/dynamic_dimension_inference.cc
+++ b/tensorflow/compiler/xla/service/dynamic_dimension_inference.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/service/while_util.h"
 
 namespace xla {
 
@@ -53,6 +54,8 @@ class DynamicDimensionInferenceVisitor : public DfsHloVisitorWithDefault {
 
   Status HandleDot(HloInstruction* hlo) override;
 
+  Status HandleTuple(HloInstruction* hlo) override;
+
   Status HandleTranspose(HloInstruction* hlo) override;
 
   Status HandleReshape(HloInstruction* hlo) override;
@@ -77,6 +80,8 @@ class DynamicDimensionInferenceVisitor : public DfsHloVisitorWithDefault {
 
   Status HandleElementwiseBinary(HloInstruction* hlo) override;
 
+  Status HandleWhile(HloInstruction* hlo) override;
+
  private:
   using OperandDynamicDimensionFn = std::function<Status(
       HloInstruction* operand, ShapeIndex index, int64 dimension,
@@ -122,6 +127,16 @@ Status DynamicDimensionInferenceVisitor::HandleGetTupleElement(
       });
 }
 
+Status DynamicDimensionInferenceVisitor::HandleTuple(HloInstruction* hlo) {
+  return ForEachOperandDynamicDimension(
+      hlo, [&](HloInstruction*, ShapeIndex index, int64 dimension,
+               int64 operand_index, HloInstruction* dynamic_size) {
+        index.push_front(operand_index);
+        parent_->SetDynamicSize(hlo, index, dimension, dynamic_size);
+        return Status::OK();
+      });
+}
+
 Status DynamicDimensionInferenceVisitor::HandleBroadcast(HloInstruction* hlo) {
   return ForEachOperandDynamicDimension(
       hlo, [&](HloInstruction* operand, ShapeIndex index, int64 dimension,
@@ -383,6 +398,120 @@ Status DynamicDimensionInferenceVisitor::HandleSelectAndScatter(
       });
 }
 
+Status DynamicDimensionInferenceVisitor::HandleWhile(HloInstruction* hlo) {
+  // While loop is handled by passing dynamic size hlos as parameters into the
+  // hlo while loop. This is done by replacing the original while with a new
+  // one.
+  //
+  // Before:
+  //
+  // op1 = ...
+  // op2 = ...
+  // op1_x = ... // dynamic dimension size of op1
+  // while = while(op1, op2)
+  //
+  //
+  // After:
+  //
+  // op1 = ...
+  // op2 = ...
+  // op1_x = ... // dynamic dimension size of op1
+  // while = while(op1, op2, op1_x)
+  //
+  // In the above graph, op_x is the bound of the dynamic dimension size of op1
+  // and is wired into the while loop as new parameter.
+  //
+  // TODO(b/119843103): Once we implement dynamic bounds in XLA backend, dynamic
+  // bound can be propagated through native xla values instead of relying on
+  // additional parameter.
+
+  // dynamic_size_to_operand_id_index_map keeps track of dynamic size operations
+  // to their operand ids in the new while loop.
+  absl::flat_hash_map<HloInstruction*, int64>
+      dynamic_size_to_operand_id_index_map;
+
+  // operands_to_add collects dynamic sizes that need to be added to the while
+  // loop as parameters. Note that a dynamic size is ignored if it is already
+  // part of the parameter. i.e.:
+  //
+  // We don't do:
+  //
+  // op1 = ...
+  // op2 = ...
+  // op_x = ... // dynamic dimension size of both op1 and op2
+  // while = while(op1, op2, op_x, op_x) // 4 parameters
+  //
+  // But we do:
+  //
+  // op1 = ...
+  // op2 = ...
+  // op_x = ... // dynamic dimension size of both op1 and op2
+  // while = while(op1, op2, op_x)
+  //
+  // An alternative is to do this in a while loop CSE pass.
+  //
+  std::vector<HloInstruction*> operands_to_add;
+  int64 operand_count = hlo->shape().tuple_shapes_size();
+  TF_RETURN_IF_ERROR(ForEachOperandDynamicDimension(
+      hlo, [&](HloInstruction*, ShapeIndex, int64, int64,
+               HloInstruction* dynamic_size) {
+        const HloInstruction* tuple_operand = hlo->operand(0);
+        for (int64 i = 0; i < tuple_operand->operand_count(); ++i) {
+          if (dynamic_size == tuple_operand->operand(i)) {
+            dynamic_size_to_operand_id_index_map[dynamic_size] = i;
+            return Status::OK();
+          }
+        }
+        auto iter = dynamic_size_to_operand_id_index_map.find(dynamic_size);
+        if (iter == dynamic_size_to_operand_id_index_map.end()) {
+          operands_to_add.push_back(dynamic_size);
+          dynamic_size_to_operand_id_index_map[dynamic_size] = operand_count++;
+        }
+        return Status::OK();
+      }));
+
+  if (!operands_to_add.empty()) {
+    // Only replace the while loop if there are new parameters to add.
+    HloInstruction* old_tuple_operand = hlo->mutable_operand(0);
+    TF_ASSIGN_OR_RETURN(
+        WhileUtil::MakeInstructionsLiveInResult result,
+        WhileUtil::MakeInstructionsLiveIn(hlo, operands_to_add));
+    // WhileUtil creates a new while hlo and tuple. Update the dynamic size
+    // mapping for the newly created tuple.
+    HloInstruction* new_tuple_operand =
+        result.new_while_instr->mutable_operand(0);
+    parent_->CopyMapping(/*from=*/old_tuple_operand, /*to=*/new_tuple_operand);
+    hlo = result.new_while_instr;
+  }
+
+  // We have replaced the while loop, now set the dynamic dimensions for the
+  // newly created while loop so that the hlos that consumes the while loop can
+  // see the dynamic dimensions. Also sets the dynamic parameter binding for
+  // running inference in the while loop.
+  DynamicParameterBinding binding_for_while;
+  TF_RETURN_IF_ERROR(ForEachOperandDynamicDimension(
+      hlo, [&](HloInstruction*, ShapeIndex index, int64 dimension,
+               int64 operand_index, HloInstruction* dynamic_size) {
+        DynamicParameterBinding::DynamicParameter dynamic_parameter{
+            operand_index,
+            {dynamic_size_to_operand_id_index_map[dynamic_size]}};
+        DynamicParameterBinding::DynamicDimension dynamic_dimension{
+            operand_index, index, dimension};
+        TF_RETURN_IF_ERROR(
+            binding_for_while.Bind(dynamic_parameter, dynamic_dimension));
+        parent_->SetDynamicSize(hlo, index, dimension, dynamic_size);
+        return Status::OK();
+      }));
+
+  // Run inference in while body and condition.
+  TF_RETURN_IF_ERROR(DynamicDimensionInferenceVisitor::Run(
+      hlo->while_body(), binding_for_while, parent_));
+  TF_RETURN_IF_ERROR(DynamicDimensionInferenceVisitor::Run(
+      hlo->while_condition(), binding_for_while, parent_));
+
+  return Status::OK();
+}
+
 Status DynamicDimensionInferenceVisitor::HandleParameter(HloInstruction* hlo) {
   return param_bindings_.ForEachBinding(
       [&](const DynamicParameterBinding::DynamicParameter& dynamic_parameter,
@@ -430,6 +559,20 @@ Status DynamicDimensionInferenceVisitor::ForEachOperandDynamicDimension(
   return Status::OK();
 }
 
+void DynamicDimensionInference::CopyMapping(HloInstruction* from,
+                                            HloInstruction* to) {
+  auto iter = per_hlo_dynamic_dimensions_.find(from);
+  if (iter != per_hlo_dynamic_dimensions_.end()) {
+    for (auto& dynamic_dimension : iter->second) {
+      HloInstruction* dynamic_size =
+          GetDynamicSize(dynamic_dimension.inst, dynamic_dimension.index,
+                         dynamic_dimension.dim);
+      SetDynamicSize(to, dynamic_dimension.index, dynamic_dimension.dim,
+                     dynamic_size);
+    }
+  }
+}
+
 /* static */
 StatusOr<DynamicDimensionInference> DynamicDimensionInference::Run(
     HloModule* module) {
@@ -439,6 +582,20 @@ StatusOr<DynamicDimensionInference> DynamicDimensionInference::Run(
   return inference;
 }
 
+string DynamicDimensionInference::ToString() const {
+  std::vector<string> pieces;
+  pieces.push_back("DynamicDimensionInference: ");
+  for (const auto& mapping : dynamic_mapping_) {
+    const DynamicDimension& dynamic_dimension = mapping.first;
+    pieces.push_back(absl::StrFormat(
+        " -- instruction %s at %s has dim %lld as dynamic"
+        " dimension, which is represented by instruction %s",
+        dynamic_dimension.inst->ToString(), dynamic_dimension.index.ToString(),
+        dynamic_dimension.dim, mapping.second->ToString()));
+  }
+  return absl::StrJoin(pieces, "\n");
+}
+
 DynamicDimensionInference::DynamicDimensionInference(HloModule* module)
     : module_(module) {}
 
diff --git a/tensorflow/compiler/xla/service/dynamic_dimension_inference.h b/tensorflow/compiler/xla/service/dynamic_dimension_inference.h
index 164d15bf11..d0f2998328 100644
--- a/tensorflow/compiler/xla/service/dynamic_dimension_inference.h
+++ b/tensorflow/compiler/xla/service/dynamic_dimension_inference.h
@@ -88,6 +88,11 @@ class DynamicDimensionInference {
     iter.first->second.emplace(DynamicDimension{inst, index, dim});
   }
 
+  // Copies the internal mapping from instruction `from` to instruction `to`.
+  // This is useful when an instruction is replaced by the other during the
+  // inferencing process.
+  void CopyMapping(HloInstruction* from, HloInstruction* to);
+
   // AnalyzeDynamicDimensions starts the analysis of the dynamic dimensions in
   // module_.
   Status AnalyzeDynamicDimensions();
@@ -101,6 +106,8 @@ class DynamicDimensionInference {
   using DynamicMapping = absl::flat_hash_map<DynamicDimension, HloInstruction*>;
   DynamicMapping dynamic_mapping_;
 
+  // A convenient mapping from an hlo to the set of dynamic dimensions that it
+  // holds.
   using PerHloDynamicDimensions =
       absl::flat_hash_map<HloInstruction*,
                           absl::flat_hash_set<DynamicDimension>>;
diff --git a/tensorflow/compiler/xla/service/dynamic_dimension_inference_test.cc b/tensorflow/compiler/xla/service/dynamic_dimension_inference_test.cc
index b42e67b4bb..597cdf27c3 100644
--- a/tensorflow/compiler/xla/service/dynamic_dimension_inference_test.cc
+++ b/tensorflow/compiler/xla/service/dynamic_dimension_inference_test.cc
@@ -445,6 +445,96 @@ TEST_F(DynamicDimensionInferenceTest, BroadcastTest) {
   EXPECT_EQ(inference_->GetDynamicSize(broadcast, {}, 2), nullptr);
 }
 
+TEST_F(DynamicDimensionInferenceTest, WhileTest) {
+  // Test the ability to trace into while loops.
+  auto builder = HloComputation::Builder(TestName());
+  auto input_shape = ShapeUtil::MakeShape(F32, {2, 4, 4});
+  auto output_shape = ShapeUtil::MakeShape(F32, {2, 2, 2});
+  auto tuple_shape = ShapeUtil::MakeTupleShape({input_shape, input_shape});
+
+  // Body:
+  //
+  //   Param
+  //   |  |
+  // GTE1 GTE2
+  //   |  |
+  //    ADD
+  auto body_builder = HloComputation::Builder("body");
+  auto body_param = body_builder.AddInstruction(
+      HloInstruction::CreateParameter(0, tuple_shape, "param"));
+  auto gte_0 = body_builder.AddInstruction(
+      HloInstruction::CreateGetTupleElement(input_shape, body_param, 0));
+  auto gte_1 = body_builder.AddInstruction(
+      HloInstruction::CreateGetTupleElement(input_shape, body_param, 1));
+  auto add = body_builder.AddInstruction(
+      HloInstruction::CreateBinary(input_shape, HloOpcode::kAdd, gte_0, gte_1));
+  body_builder.AddInstruction(HloInstruction::CreateTuple({add, add}));
+
+  HloComputation* body = module_->AddEmbeddedComputation(body_builder.Build());
+
+  auto cond_builder = HloComputation::Builder("condition");
+  cond_builder.AddInstruction(
+      HloInstruction::CreateParameter(0, tuple_shape, "param"));
+  cond_builder.AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<bool>(false)));
+  HloComputation* condition =
+      module_->AddEmbeddedComputation(cond_builder.Build());
+
+  // Entry:
+  //
+  //  Param
+  //   |
+  //  While
+  auto* a_param = builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/0, tuple_shape, "A"));
+  auto* size_param = builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/1, scalar_shape_, "size_param"));
+  builder.AddInstruction(
+      HloInstruction::CreateWhile(tuple_shape, condition, body, a_param));
+
+  module_->AddEntryComputation(builder.Build());
+
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{1, {}},
+      DynamicParameterBinding::DynamicDimension{0, {0}, 0}));
+
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{1, {}},
+      DynamicParameterBinding::DynamicDimension{0, {1}, 0}));
+
+  // Test that dynamic dimension inference does the right thing. A lambda is
+  // used here since we want to test twice by running inference again
+  // (idempotency).
+  auto test_dynamic_dimension = [&]() {
+    HloInstruction* while_hlo = nullptr;
+    // The while hlo has been replaced, find the new one.
+    for (HloInstruction* inst : module_->entry_computation()->instructions()) {
+      if (inst->opcode() == HloOpcode::kWhile) {
+        while_hlo = inst;
+      }
+    }
+    ASSERT_NE(while_hlo, nullptr);
+    // The original while shape has 2 parameters. With dynamic size passed in
+    // as an extra parameter, the tuple should have 3 elements.
+    EXPECT_EQ(while_hlo->shape().tuple_shapes_size(), 3);
+    HloInstruction* add = nullptr;
+    for (HloInstruction* inst : while_hlo->while_body()->instructions()) {
+      if (inst->opcode() == HloOpcode::kAdd) {
+        add = inst;
+      }
+    }
+    EXPECT_NE(add, nullptr);
+    EXPECT_NE(inference_->GetDynamicSize(add, {}, 0), nullptr);
+    EXPECT_EQ(inference_->GetDynamicSize(while_hlo, {0}, 0), size_param);
+    EXPECT_EQ(inference_->GetDynamicSize(while_hlo, {1}, 0), size_param);
+  };
+
+  TF_ASSERT_OK(RunInference());
+  test_dynamic_dimension();
+  TF_ASSERT_OK(RunInference());
+  test_dynamic_dimension();
+}
+
 TEST_F(DynamicDimensionInferenceTest, ReduceWindowBatchTest) {
   // Test the ability to trace reduce window batch dimensions.
   auto builder = HloComputation::Builder(TestName());
-- 
GitLab


From 0013f89a9f0191e446a668f6f955043d4dc66c20 Mon Sep 17 00:00:00 2001
From: Tong Shen <endlessroad@google.com>
Date: Mon, 28 Jan 2019 15:51:37 -0800
Subject: [PATCH 1428/2345] Handle the case where Merge node of Conditional A
 is used as Conditional B's predicate.

For Conditional B's If node, we should use Conditional A's If output as predicate (instead of Conditional A's merge node). Otherwise, Conditional A's Merge node is still connected to Conditional B's If node, and when we call DeleteReachableAndDeadNodes(), Conditional A's Merge node and Conditional B's If node will be removed, which is wrong.

Notice that we cannot just modify merge_to_predicate_; because in all nodes' CondState, we still use Conditional A's Merge node name as CondState predicate.

PiperOrigin-RevId: 231304145
---
 .../compiler/tf2xla/functionalize_cond.cc     | 56 +++++++++++++++----
 .../compiler/tf2xla/functionalize_cond.h      |  3 +
 2 files changed, 47 insertions(+), 12 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/functionalize_cond.cc b/tensorflow/compiler/tf2xla/functionalize_cond.cc
index 7ae96e1d48..e780c93a8f 100644
--- a/tensorflow/compiler/tf2xla/functionalize_cond.cc
+++ b/tensorflow/compiler/tf2xla/functionalize_cond.cc
@@ -247,7 +247,9 @@ class Conditional {
   Status AddMerge(Node* m);
 
   // Constructs an If node from the merge nodes.
-  Status BuildAndReplace(Graph* graph, FunctionLibraryDefinition* library);
+  Status BuildAndReplace(
+      Graph* graph, FunctionLibraryDefinition* library,
+      std::unordered_map<Node*, OutputTensor>* merge_to_replacement);
 
  private:
   // Extracts the then/else bodies: creates new graphs with the nodes
@@ -262,10 +264,15 @@ class Conditional {
   Status BuildIfNode(Graph* graph, FunctionLibraryDefinition* library);
 
   // Adds input edges to If node.
-  Status AddInputEdges(Graph* graph);
+  Status AddInputEdges(
+      Graph* graph,
+      const std::unordered_map<Node*, OutputTensor>& merge_to_replacement);
 
   // Adds output edges from If node.
-  Status AddOutputEdges(Graph* graph);
+  // Record new output tensor for all Merge nodes in 'merge_to_replacement'.
+  Status AddOutputEdges(
+      Graph* graph,
+      std::unordered_map<Node*, OutputTensor>* merge_to_replacement);
 
   // Adds switch node that is part of this conditional.
   Status AddSwitch(Node* s);
@@ -720,12 +727,29 @@ Status Conditional::BuildIfNode(Graph* graph,
   return Status::OK();
 }
 
-Status Conditional::AddInputEdges(Graph* graph) {
+Status Conditional::AddInputEdges(
+    Graph* graph,
+    const std::unordered_map<Node*, OutputTensor>& merge_to_replacement) {
   VLOG(2) << "AddInputEdges for " << if_node_->name();
   int index = 0;
   // Add predicate input.
-  graph->AddEdge(const_cast<Node*>(predicate_.node), predicate_.index, if_node_,
-                 index++);
+  if (predicate_.node->IsMerge()) {
+    // If the predicate is a Merge node, we should not use Merge output as
+    // predicate. Instead, we should use the corresponding If output in
+    // 'merge_to_replacement'. Otherwise, this Conditional's If node is still
+    // connected to the predicate Merge node; and when we call
+    // DeleteReachableAndDeadNodes(), the predicate Merge node and this
+    // Conditional's If node will be removed.
+    auto iter = merge_to_replacement.find(predicate_.node);
+    if (iter == merge_to_replacement.end()) {
+      return errors::Internal("Cannot find replacement for Merge node ",
+                              predicate_.node->name());
+    }
+    graph->AddEdge(iter->second.node, iter->second.index, if_node_, index++);
+  } else {
+    graph->AddEdge(const_cast<Node*>(predicate_.node), predicate_.index,
+                   if_node_, index++);
+  }
   // Add function body inputs.
   for (auto& arg : cond_arg_nodes_) {
     if (arg.src_output == Graph::kControlSlot) {
@@ -740,7 +764,9 @@ Status Conditional::AddInputEdges(Graph* graph) {
   return Status::OK();
 }
 
-Status Conditional::AddOutputEdges(Graph* graph) {
+Status Conditional::AddOutputEdges(
+    Graph* graph,
+    std::unordered_map<Node*, OutputTensor>* merge_to_replacement) {
   VLOG(2) << "AddOutputEdges for " << if_node_->name();
   int i = 0;
   for (Node* node : merges_) {
@@ -764,6 +790,10 @@ Status Conditional::AddOutputEdges(Graph* graph) {
         graph->AddEdge(if_node_, i, dst, dst_input);
       }
     }
+
+    // Record corresponding output tensor in 'merge_to_replacement'.
+    (*merge_to_replacement)[node] = OutputTensor{if_node_, i};
+
     ++i;
   }
   for (Node* n : external_control_outputs_) {
@@ -773,8 +803,9 @@ Status Conditional::AddOutputEdges(Graph* graph) {
   return Status::OK();
 }
 
-Status Conditional::BuildAndReplace(Graph* graph,
-                                    FunctionLibraryDefinition* library) {
+Status Conditional::BuildAndReplace(
+    Graph* graph, FunctionLibraryDefinition* library,
+    std::unordered_map<Node*, OutputTensor>* merge_to_replacement) {
   VLOG(1) << "Build If and replace merge nodes "
           << NodesToString(this->merges_);
   if (replaced_) return Status::OK();
@@ -793,8 +824,8 @@ Status Conditional::BuildAndReplace(Graph* graph,
   }
 
   TF_RETURN_IF_ERROR(BuildIfNode(graph, library));
-  TF_RETURN_IF_ERROR(AddInputEdges(graph));
-  TF_RETURN_IF_ERROR(AddOutputEdges(graph));
+  TF_RETURN_IF_ERROR(AddInputEdges(graph, *merge_to_replacement));
+  TF_RETURN_IF_ERROR(AddOutputEdges(graph, merge_to_replacement));
   TF_RETURN_IF_ERROR(parent_->PropagateUpdatedState(if_node_));
 
   // Check that the if_node doesn't feed into itself.
@@ -1345,7 +1376,8 @@ Status FunctionalizeCond::FunctionalizeInternal() {
     Conditional cond(merge_to_predicate_.at(cluster.front()), this,
                      &state_map_);
     for (Node* merge : cluster) TF_RETURN_IF_ERROR(cond.AddMerge(merge));
-    TF_RETURN_IF_ERROR(cond.BuildAndReplace(graph_, library_));
+    TF_RETURN_IF_ERROR(
+        cond.BuildAndReplace(graph_, library_, &merge_to_predicate_));
 
     if (VLOG_IS_ON(4)) DumpGraphWithCondState("after_extract");
   }
diff --git a/tensorflow/compiler/tf2xla/functionalize_cond.h b/tensorflow/compiler/tf2xla/functionalize_cond.h
index 8525d7af61..9a610a5cbf 100644
--- a/tensorflow/compiler/tf2xla/functionalize_cond.h
+++ b/tensorflow/compiler/tf2xla/functionalize_cond.h
@@ -232,6 +232,9 @@ class FunctionalizeCond {
   // Mapping from merge nodes to predicate.
   std::unordered_map<Node*, OutputTensor> merge_to_predicate_;
 
+  // Mapping from merge nodes to corresponding If node outputs.
+  std::unordered_map<Node*, OutputTensor> merge_to_replacement_;
+
   FunctionLibraryDefinition* library_;
   Graph* graph_;
 
-- 
GitLab


From 0909b15d5dae0c9895e17d46c8e4183db4889230 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Mon, 28 Jan 2019 16:00:53 -0800
Subject: [PATCH 1429/2345] [XLA] Speed up exhaustive_f32_elementwise_op_test
 by using our own implementation of LiteralTestUtil::Near.

LiteralTestUtil::Near does a bunch of stuff that we don't care about, like
keeping track of mean/stddev.  It also operates on literals not 1D arrays, and
so does expensive index calculations on each entry.

Rather than laboriously try to fix it up, it's easy enough just to reeimplement
the relevant part.

Speedup is ~8/5 = 1.6x.

PiperOrigin-RevId: 231305563
---
 .../exhaustive_f32_elementwise_op_test.cc     | 75 ++++++++++++++-----
 1 file changed, 58 insertions(+), 17 deletions(-)

diff --git a/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc b/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc
index 6634655371..11b06c7355 100644
--- a/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc
+++ b/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc
@@ -34,24 +34,38 @@ class ExhaustiveF32ElementwiseOpTest
     int64 begin, end;
     std::tie(begin, end) = GetParam();
     int64 input_size = end - begin;
+
+    if (begin >= known_incorrect_range.first &&
+        end <= known_incorrect_range.second) {
+      LOG(INFO) << absl::StreamFormat(
+          "Skipping this shard, as the range under test, [%d, %d), falls "
+          "entirely within the known-incorrect range [%d, %d).",
+          begin, end, known_incorrect_range.first,
+          known_incorrect_range.second);
+      return;
+    }
+
     LOG(INFO) << "Checking range [" << begin << ", " << end << ")";
 
     XlaBuilder builder(TestName());
 
-    Literal input_literal =
-        LiteralUtil::CreateFromDimensions(F32, {input_size});
-    absl::Span<float> input_arr = input_literal.data<float>();
-    for (int64 i = begin; i < end; i++) {
+    auto ith_input_elem = [&](int64 i) -> float {
+      i += begin;
+      // If the operation is known to be buggy on a specific input clamp that
+      // input to 0 under the assumption that the op is at least correct on 0.
       if (i >= known_incorrect_range.first &&
           i < known_incorrect_range.second) {
-        // If the operation is known to be buggy on a specific input clamp that
-        // input to 0 under the assumption that the op is at least correct on 0.
-        input_arr[i - begin] = 0;
-      } else {
-        input_arr[i - begin] = absl::bit_cast<float, int>(i);
+        return 0;
       }
-    }
+      return absl::bit_cast<float, int32>(i);
+    };
 
+    Literal input_literal =
+        LiteralUtil::CreateFromDimensions(F32, {input_size});
+    absl::Span<float> input_arr = input_literal.data<float>();
+    for (int64 i = 0; i < input_size; i++) {
+      input_arr[i] = ith_input_elem(i);
+    }
     auto input = Parameter(&builder, 0, input_literal.shape(), "input");
     enqueue_op(&builder, input);
     TF_ASSERT_OK_AND_ASSIGN(XlaComputation comp, builder.Build());
@@ -83,14 +97,41 @@ class ExhaustiveF32ElementwiseOpTest
     TF_ASSERT_OK_AND_ASSIGN(Literal result_literal,
                             client_->ShapedBufferToLiteral(result));
 
-    Literal expected_literal =
-        LiteralUtil::CreateFromDimensions(F32, {input_size});
-    absl::Span<float> expected_arr = expected_literal.data<float>();
-    for (int64 i = 0; i < input_size; i++) {
-      expected_arr[i] = evaluate_op(input_arr[i]);
+    // We essentially reimplement LiteralTestUtil::Near here because
+    //  a) this streamlined implementation is much faster, and
+    //  b) we can print out better error messages (namely, we can print out
+    //     which floating-point value input failed, while LiteralTestUtil::Near
+    //     can only print out the input index that failed).
+    absl::Span<float> result_arr = result_literal.data<float>();
+    ASSERT_EQ(result_arr.size(), input_arr.size());
+    int64 mismatches = 0;
+    for (int64 i = 0; i < input_arr.size(); ++i) {
+      float input = ith_input_elem(i);
+      float expected = evaluate_op(input);
+      float actual = result_arr[i];
+      float abs_err = std::abs(expected - actual);
+      float rel_err = abs_err / std::abs(expected);
+      if (abs_err < error_spec_.abs || rel_err < error_spec_.rel ||
+          (std::isnan(expected) && std::isnan(actual)) ||
+          (std::isinf(expected) && std::isinf(actual) &&
+           (expected > 0) == (actual > 0))) {
+        // Successful match!  Nothing to do.
+      } else {
+        constexpr int64 kMaxMismatchesPrinted = 1000;
+        mismatches++;
+        if (mismatches < kMaxMismatchesPrinted || VLOG_IS_ON(2)) {
+          LOG(ERROR) << "Mismatch on " << input << " (0x"
+                     << absl::StrCat(absl::Hex(input, absl::kZeroPad8))
+                     << "). Expected " << expected << ", but got " << actual;
+        }
+        if (mismatches == kMaxMismatchesPrinted && !VLOG_IS_ON(2)) {
+          LOG(ERROR) << "Not printing any more mismatches; pass "
+                        "--vmodule=exhaustive_f32_elementwise_op_test=2 to see "
+                        "all of them.";
+        }
+      }
     }
-    EXPECT_TRUE(
-        LiteralTestUtil::Near(expected_literal, result_literal, error_spec_));
+    EXPECT_EQ(mismatches, 0);
   }
 };
 
-- 
GitLab


From 3071702b159563091b768b48462d04720d6aab0d Mon Sep 17 00:00:00 2001
From: James Ring <sjr@google.com>
Date: Mon, 28 Jan 2019 16:13:32 -0800
Subject: [PATCH 1430/2345] Release Tensorflow Java 1.13rc0

PiperOrigin-RevId: 231307828
---
 tensorflow/java/maven/libtensorflow/pom.xml              | 2 +-
 tensorflow/java/maven/libtensorflow_jni/pom.xml          | 2 +-
 tensorflow/java/maven/libtensorflow_jni_gpu/pom.xml      | 2 +-
 tensorflow/java/maven/pom.xml                            | 2 +-
 tensorflow/java/maven/proto/pom.xml                      | 2 +-
 tensorflow/java/maven/spark-tensorflow-connector/pom.xml | 2 +-
 tensorflow/java/maven/tensorflow-hadoop/pom.xml          | 2 +-
 tensorflow/java/maven/tensorflow/pom.xml                 | 2 +-
 8 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/tensorflow/java/maven/libtensorflow/pom.xml b/tensorflow/java/maven/libtensorflow/pom.xml
index db3a3609f1..4d9f629fa2 100644
--- a/tensorflow/java/maven/libtensorflow/pom.xml
+++ b/tensorflow/java/maven/libtensorflow/pom.xml
@@ -6,7 +6,7 @@
   <parent>
     <groupId>org.tensorflow</groupId>
     <artifactId>parentpom</artifactId>
-    <version>1.12.0</version>
+    <version>1.13.0-rc0</version>
     <relativePath>../</relativePath>
   </parent>
   <artifactId>libtensorflow</artifactId>
diff --git a/tensorflow/java/maven/libtensorflow_jni/pom.xml b/tensorflow/java/maven/libtensorflow_jni/pom.xml
index 53f7a2d63e..49a4359249 100644
--- a/tensorflow/java/maven/libtensorflow_jni/pom.xml
+++ b/tensorflow/java/maven/libtensorflow_jni/pom.xml
@@ -6,7 +6,7 @@
   <parent>
     <groupId>org.tensorflow</groupId>
     <artifactId>parentpom</artifactId>
-    <version>1.12.0</version>
+    <version>1.13.0-rc0</version>
     <relativePath>../</relativePath>
   </parent>
   <artifactId>libtensorflow_jni</artifactId>
diff --git a/tensorflow/java/maven/libtensorflow_jni_gpu/pom.xml b/tensorflow/java/maven/libtensorflow_jni_gpu/pom.xml
index a17724c805..b54da039e5 100644
--- a/tensorflow/java/maven/libtensorflow_jni_gpu/pom.xml
+++ b/tensorflow/java/maven/libtensorflow_jni_gpu/pom.xml
@@ -6,7 +6,7 @@
   <parent>
     <groupId>org.tensorflow</groupId>
     <artifactId>parentpom</artifactId>
-    <version>1.12.0</version>
+    <version>1.13.0-rc0</version>
     <relativePath>../</relativePath>
   </parent>
   <artifactId>libtensorflow_jni_gpu</artifactId>
diff --git a/tensorflow/java/maven/pom.xml b/tensorflow/java/maven/pom.xml
index 30831f90b9..8dfb5d7aa3 100644
--- a/tensorflow/java/maven/pom.xml
+++ b/tensorflow/java/maven/pom.xml
@@ -6,7 +6,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.tensorflow</groupId>
   <artifactId>parentpom</artifactId>
-  <version>1.12.0</version>
+  <version>1.13.0-rc0</version>
   <packaging>pom</packaging>
 
   <url>https://www.tensorflow.org</url>
diff --git a/tensorflow/java/maven/proto/pom.xml b/tensorflow/java/maven/proto/pom.xml
index dd6b52be62..65f545bdde 100644
--- a/tensorflow/java/maven/proto/pom.xml
+++ b/tensorflow/java/maven/proto/pom.xml
@@ -6,7 +6,7 @@
   <parent>
     <groupId>org.tensorflow</groupId>
     <artifactId>parentpom</artifactId>
-    <version>1.12.0</version>
+    <version>1.13.0-rc0</version>
     <relativePath>../</relativePath>
   </parent>
   <artifactId>proto</artifactId>
diff --git a/tensorflow/java/maven/spark-tensorflow-connector/pom.xml b/tensorflow/java/maven/spark-tensorflow-connector/pom.xml
index f47c11809d..1ca1e91891 100644
--- a/tensorflow/java/maven/spark-tensorflow-connector/pom.xml
+++ b/tensorflow/java/maven/spark-tensorflow-connector/pom.xml
@@ -6,7 +6,7 @@
     <groupId>org.tensorflow</groupId>
     <artifactId>spark-tensorflow-connector_2.11</artifactId>
     <packaging>jar</packaging>
-    <version>1.12.0</version>
+    <version>1.13.0-rc0</version>
     <name>spark-tensorflow-connector</name>
     <url>https://www.tensorflow.org</url>
     <description>TensorFlow TFRecord connector for Apache Spark DataFrames</description>
diff --git a/tensorflow/java/maven/tensorflow-hadoop/pom.xml b/tensorflow/java/maven/tensorflow-hadoop/pom.xml
index 11aaba983f..df86c50f86 100644
--- a/tensorflow/java/maven/tensorflow-hadoop/pom.xml
+++ b/tensorflow/java/maven/tensorflow-hadoop/pom.xml
@@ -5,7 +5,7 @@
     <groupId>org.tensorflow</groupId>
     <artifactId>tensorflow-hadoop</artifactId>
     <packaging>jar</packaging>
-    <version>1.12.0</version>
+    <version>1.13.0-rc0</version>
     <name>tensorflow-hadoop</name>
     <url>https://www.tensorflow.org</url>
     <description>TensorFlow TFRecord InputFormat/OutputFormat for Apache Hadoop</description>
diff --git a/tensorflow/java/maven/tensorflow/pom.xml b/tensorflow/java/maven/tensorflow/pom.xml
index 07fcfa5144..a318e7fa08 100644
--- a/tensorflow/java/maven/tensorflow/pom.xml
+++ b/tensorflow/java/maven/tensorflow/pom.xml
@@ -6,7 +6,7 @@
   <parent>
     <groupId>org.tensorflow</groupId>
     <artifactId>parentpom</artifactId>
-    <version>1.12.0</version>
+    <version>1.13.0-rc0</version>
     <relativePath>../</relativePath>
   </parent>
   <artifactId>tensorflow</artifactId>
-- 
GitLab


From c8128f9dbebdb2327bd369cbc7ca774434b41821 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 28 Jan 2019 16:27:48 -0800
Subject: [PATCH 1431/2345] Fix TRANING -> TRAINING typo in v2.0 API.

PiperOrigin-RevId: 231310166
---
 tensorflow/python/saved_model/tag_constants.py              | 2 +-
 tensorflow/tools/api/golden/v2/tensorflow.saved_model.pbtxt | 2 +-
 tensorflow/tools/compatibility/renames_v2.py                | 1 -
 3 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/saved_model/tag_constants.py b/tensorflow/python/saved_model/tag_constants.py
index 8c84c9fbe4..7793d49214 100644
--- a/tensorflow/python/saved_model/tag_constants.py
+++ b/tensorflow/python/saved_model/tag_constants.py
@@ -33,7 +33,7 @@ tf_export(
 # Tag for the `training` graph.
 TRAINING = "train"
 tf_export(
-    "saved_model.TRANING",
+    "saved_model.TRAINING",
     v1=["saved_model.TRAINING",
         "saved_model.tag_constants.TRAINING"]).export_constant(
             __name__, "TRAINING")
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.saved_model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.saved_model.pbtxt
index 63bebb20bc..f6ab7ac0ad 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.saved_model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.saved_model.pbtxt
@@ -77,7 +77,7 @@ tf_module {
     mtype: "<type \'str\'>"
   }
   member {
-    name: "TRANING"
+    name: "TRAINING"
     mtype: "<type \'str\'>"
   }
   member {
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 9c02c20f0f..7c4b72df2d 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -497,7 +497,6 @@ renames = {
     'tf.saved_model.Builder': 'tf.compat.v1.saved_model.Builder',
     'tf.saved_model.LEGACY_INIT_OP_KEY': 'tf.compat.v1.saved_model.LEGACY_INIT_OP_KEY',
     'tf.saved_model.MAIN_OP_KEY': 'tf.compat.v1.saved_model.MAIN_OP_KEY',
-    'tf.saved_model.TRAINING': 'tf.saved_model.TRAINING',
     'tf.saved_model.build_tensor_info': 'tf.compat.v1.saved_model.build_tensor_info',
     'tf.saved_model.builder.SavedModelBuilder': 'tf.compat.v1.saved_model.builder.SavedModelBuilder',
     'tf.saved_model.constants.ASSETS_DIRECTORY': 'tf.saved_model.ASSETS_DIRECTORY',
-- 
GitLab


From 96433794b3988ad4b6ecdfbeb0c23f848ad01513 Mon Sep 17 00:00:00 2001
From: David Majnemer <majnemer@google.com>
Date: Mon, 28 Jan 2019 16:50:30 -0800
Subject: [PATCH 1432/2345] [TF2XLA] Replace a call to Pow with an exponent of
 .5 with Rsqrt

PiperOrigin-RevId: 231313690
---
 tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc b/tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc
index 0e2f335f33..9089cb4fab 100644
--- a/tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_helpers.h"
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
+#include "tensorflow/compiler/xla/client/lib/math.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/core/util/tensor_format.h"
 
@@ -158,9 +159,8 @@ class FusedBatchNormGradOp : public XlaOpKernel {
       offset_backprop = XlaHelpers::ConvertElementType(reduce, scale_dtype);
 
       // scratch1 = rsqrt(pop_var + epsilon)
-      auto neg_half = XlaHelpers::FloatLiteral(b, scale_dtype, -0.5);
-      auto scratch1 = xla::Pow(
-          xla::Add(var, xla::ConstantR0<float>(b, epsilon_)), neg_half);
+      auto scratch1 =
+          xla::Rsqrt(xla::Add(var, xla::ConstantR0<float>(b, epsilon_)));
 
       // scratch2 = sum(y_backprop * (x - mean))
       auto mul =
-- 
GitLab


From eabf2ec12818c4e8ae7008ce14ed12ad38bd1537 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Mon, 28 Jan 2019 17:17:33 -0800
Subject: [PATCH 1433/2345] Work around a buggy get_config by ignoring its
 errors when checkpointing

This is extra "nice to have" metadata, and otherwise it looks like a checkpointing error. Not worth bothering people about.

PiperOrigin-RevId: 231318216
---
 tensorflow/python/training/checkpointable/base.py   | 11 ++++++++---
 .../python/training/checkpointable/base_test.py     | 13 +++++++++++++
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/training/checkpointable/base.py b/tensorflow/python/training/checkpointable/base.py
index 8257693055..9fb251e455 100644
--- a/tensorflow/python/training/checkpointable/base.py
+++ b/tensorflow/python/training/checkpointable/base.py
@@ -846,11 +846,16 @@ class Checkpointable(object):
       return {}
     weak_self = weakref.ref(self)
     def _state_callback():
+      """Serializes `self.get_config()` for saving."""
       dereferenced_self = weak_self()
       if dereferenced_self:
-        return json.dumps(dereferenced_self,
-                          default=serialization.get_json_type,
-                          sort_keys=True).encode("utf8")
+        try:
+          return json.dumps(dereferenced_self,
+                            default=serialization.get_json_type,
+                            sort_keys=True).encode("utf8")
+        except TypeError:
+          # Even if get_config worked objects may have produced garbage.
+          return ""
       else:
         return ""
     return {OBJECT_CONFIG_JSON_KEY: functools.partial(
diff --git a/tensorflow/python/training/checkpointable/base_test.py b/tensorflow/python/training/checkpointable/base_test.py
index 750799f030..5863f5a560 100644
--- a/tensorflow/python/training/checkpointable/base_test.py
+++ b/tensorflow/python/training/checkpointable/base_test.py
@@ -83,6 +83,19 @@ class InterfaceTests(test.TestCase):
     with self.assertRaisesRegexp(AssertionError, "foo_attr"):
       status.assert_consumed()
 
+  def testBuggyGetConfig(self):
+
+    class NotSerializable(object):
+      pass
+
+    class GetConfigRaisesError(base.Checkpointable):
+
+      def get_config(self):
+        return NotSerializable()
+
+    util.Checkpoint(obj=GetConfigRaisesError()).save(
+        os.path.join(self.get_temp_dir(), "ckpt"))
+
 
 if __name__ == "__main__":
   ops.enable_eager_execution()
-- 
GitLab


From 11dcc43a6cacd983f5756bfc33bc2e7d3d7e90fb Mon Sep 17 00:00:00 2001
From: Yanan Cao <ycao@google.com>
Date: Mon, 28 Jan 2019 17:21:48 -0800
Subject: [PATCH 1434/2345] Make checkpoint_utils_test more self-contained.

PiperOrigin-RevId: 231318767
---
 tensorflow/contrib/distribute/python/BUILD    |  1 -
 .../python/checkpoint_utils_test.py           | 22 ++++++++++++++++---
 2 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 1b455a4e64..fca1e29df4 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -774,7 +774,6 @@ cuda_py_test(
     additional_deps = [
         ":combinations",
         "//tensorflow/python:client_testlib",
-        "//tensorflow/python:checkpoint_utils_test",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:training",
         "//tensorflow/python:variable_scope",
diff --git a/tensorflow/contrib/distribute/python/checkpoint_utils_test.py b/tensorflow/contrib/distribute/python/checkpoint_utils_test.py
index 3ef8b9574a..7ee50f0315 100644
--- a/tensorflow/contrib/distribute/python/checkpoint_utils_test.py
+++ b/tensorflow/contrib/distribute/python/checkpoint_utils_test.py
@@ -25,6 +25,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import os
 from absl.testing import parameterized
 
 from tensorflow.contrib.distribute.python import combinations
@@ -33,7 +34,23 @@ from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 from tensorflow.python.training import checkpoint_utils
-from tensorflow.python.training import checkpoint_utils_test
+from tensorflow.python.training import saver as saver_lib
+
+
+def _create_checkpoints(sess, checkpoint_dir):
+  checkpoint_prefix = os.path.join(checkpoint_dir, "model")
+  checkpoint_state_name = "checkpoint"
+  v1 = variable_scope.get_variable("var1", [1, 10])
+  v2 = variable_scope.get_variable("var2", [10, 10])
+  sess.run(variables.global_variables_initializer())
+  v1_value, v2_value = sess.run([v1, v2])
+  saver = saver_lib.Saver()
+  saver.save(
+      sess,
+      checkpoint_prefix,
+      global_step=0,
+      latest_filename=checkpoint_state_name)
+  return v1_value, v2_value
 
 
 class CheckpointUtilsWithDistributionStrategyTest(
@@ -51,8 +68,7 @@ class CheckpointUtilsWithDistributionStrategyTest(
   def testInitFromCheckpoint(self, distribution, in_replica_mode):
     checkpoint_dir = self.get_temp_dir()
     with self.cached_session() as session:
-      v1_value, v2_value, _, _ = checkpoint_utils_test._create_checkpoints(
-          session, checkpoint_dir)
+      v1_value, v2_value = _create_checkpoints(session, checkpoint_dir)
 
     def init_and_verify(g):
       v1 = variable_scope.get_variable("new_var1", [1, 10])
-- 
GitLab


From 80f6a627759395087eba23412d5a4d893970f93e Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Mon, 28 Jan 2019 17:23:24 -0800
Subject: [PATCH 1435/2345] Automated rollback of commit
 7b8464b172fbe96464e7a800f89883e5796b1458

PiperOrigin-RevId: 231318985
---
 .../distribute/distribute_coordinator.py      | 15 ++-
 tensorflow/python/keras/BUILD                 |  1 -
 tensorflow/python/keras/engine/training.py    | 95 ++++---------------
 .../python/keras/engine/training_utils.py     | 10 --
 4 files changed, 27 insertions(+), 94 deletions(-)

diff --git a/tensorflow/python/distribute/distribute_coordinator.py b/tensorflow/python/distribute/distribute_coordinator.py
index c01c0f0572..f1d7d1debd 100644
--- a/tensorflow/python/distribute/distribute_coordinator.py
+++ b/tensorflow/python/distribute/distribute_coordinator.py
@@ -719,8 +719,7 @@ def run_distribute_coordinator(worker_fn,
 
   Returns:
     In the client job, return the value returned by `worker_fn` if
-    it is in-graph replication or INDEPENDENT_WORKER mode; return None
-    otherwise.
+    it is in-graph replication; return None otherwise.
   """
   tf_config = json.loads(os.environ.get("TF_CONFIG", "{}"))
   if not cluster_spec:
@@ -818,19 +817,19 @@ def run_distribute_coordinator(worker_fn,
     if task_type in [_TaskType.CHIEF, _TaskType.WORKER]:
       if strategy.extended.experimental_between_graph:
         # All jobs run `worker_fn` if between-graph.
-        return _run_single_worker(worker_fn, strategy, cluster_spec, task_type,
-                                  task_id, session_config, rpc_layer)
+        _run_single_worker(worker_fn, strategy, cluster_spec, task_type,
+                           task_id, session_config, rpc_layer)
       else:
         # Only one node runs `worker_fn` if in-graph.
         context = _WorkerContext(strategy, cluster_spec, task_type, task_id)
         if context.is_chief:
-          return _run_single_worker(worker_fn, strategy, cluster_spec, None,
-                                    None, session_config, rpc_layer)
+          _run_single_worker(worker_fn, strategy, cluster_spec, None, None,
+                             session_config, rpc_layer)
         else:
           server.join()
     elif task_type == _TaskType.EVALUATOR:
-      return _run_single_worker(eval_fn, eval_strategy, cluster_spec, task_type,
-                                task_id, session_config, rpc_layer)
+      _run_single_worker(eval_fn, eval_strategy, cluster_spec, task_type,
+                         task_id, session_config, rpc_layer)
     else:
       if task_type != _TaskType.PS:
         raise ValueError("Unexpected task_type: %r" % task_type)
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 38793e9f95..46259ead99 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -159,7 +159,6 @@ py_library(
         ":regularizers",
         ":saving",
         "//tensorflow/python/data",
-        "//tensorflow/python/distribute:distribute_coordinator",
         "//tensorflow/python/distribute:reduce_util",
         "//tensorflow/python/training/checkpointable:data_structures",
         "//tensorflow/tools/docs:doc_controls",
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 12a1008153..6ab9935d43 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -24,8 +24,6 @@ import numpy as np
 from tensorflow.python import tf2
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
-from tensorflow.python.distribute import distribute_coordinator as dc
-from tensorflow.python.distribute import distribute_coordinator_context as dc_context
 from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
@@ -130,7 +128,6 @@ class Model(Network):
     # passing distribution strategy to compile rather than creating the model
     # under distribution strategy scope.
     self._compile_distribution = False
-    self._distributed_session_is_configured = False
 
     self.run_eagerly = None
 
@@ -219,7 +216,6 @@ class Model(Network):
                       'create the model under the distribution strategy scope.')
       self._distribution_strategy = distribute
       self._compile_distribution = True
-      self._distributed_session_is_configured = False
     else:
       if distribution_strategy_context.has_strategy():
         # When the user builds the model in the DS scope and cross replica
@@ -278,7 +274,9 @@ class Model(Network):
     # Set DistributionStrategy specific parameters.
     for mode in [ModeKeys.TRAIN, ModeKeys.TEST, ModeKeys.PREDICT]:
       distributed_training_utils.set_distributed_model(self, mode, None)
-
+    if self._distribution_strategy is not None:
+      distributed_training_utils.configure_and_create_session(
+          self._distribution_strategy)
     # Initialize model metric attributes.
     self._init_metric_attributes()
     if not self.built or not self.inputs or not self.outputs:
@@ -788,54 +786,23 @@ class Model(Network):
 
     # Case 1: distribution strategy.
     if self._distribution_strategy:
-      if training_utils.should_run_multi_worker():
-        # Multi-Worker mode runs the Keras training loop on multiple
-        # servers via the Distribute Coordinator.
-        def _worker_fn(_):
-          """Run training inside the distributed coordinator."""
-          self._configure_distributed_session()
-          return training_distributed.fit_distributed(
-              self,
-              x=x,
-              y=y,
-              batch_size=batch_size,
-              epochs=epochs,
-              verbose=verbose,
-              callbacks=callbacks,
-              validation_split=validation_split,
-              validation_data=validation_data,
-              shuffle=shuffle,
-              class_weight=class_weight,
-              sample_weight=sample_weight,
-              initial_epoch=initial_epoch,
-              steps_per_epoch=steps_per_epoch,
-              validation_steps=validation_steps,
-              validation_freq=validation_freq)
-
-        # Independent worker only for now.
-        return dc.run_distribute_coordinator(
-            _worker_fn,
-            self._distribution_strategy,
-            mode=dc.CoordinatorMode.INDEPENDENT_WORKER)
-      else:
-        self._configure_distributed_session()
-        return training_distributed.fit_distributed(
-            self,
-            x=x,
-            y=y,
-            batch_size=batch_size,
-            epochs=epochs,
-            verbose=verbose,
-            callbacks=callbacks,
-            validation_split=validation_split,
-            validation_data=validation_data,
-            shuffle=shuffle,
-            class_weight=class_weight,
-            sample_weight=sample_weight,
-            initial_epoch=initial_epoch,
-            steps_per_epoch=steps_per_epoch,
-            validation_steps=validation_steps,
-            validation_freq=validation_freq)
+      return training_distributed.fit_distributed(
+          self,
+          x=x,
+          y=y,
+          batch_size=batch_size,
+          epochs=epochs,
+          verbose=verbose,
+          callbacks=callbacks,
+          validation_split=validation_split,
+          validation_data=validation_data,
+          shuffle=shuffle,
+          class_weight=class_weight,
+          sample_weight=sample_weight,
+          initial_epoch=initial_epoch,
+          steps_per_epoch=steps_per_epoch,
+          validation_steps=validation_steps,
+          validation_freq=validation_freq)
 
     batch_size = self._validate_or_infer_batch_size(
         batch_size, steps_per_epoch, x)
@@ -1050,7 +1017,6 @@ class Model(Network):
     """
     # Case 1: distribution strategy.
     if self._distribution_strategy:
-      self._configure_distributed_session()
       return training_distributed.evaluate_distributed(
           self,
           x=x,
@@ -1178,7 +1144,6 @@ class Model(Network):
     """
     # Case 1: distribution strategy.
     if self._distribution_strategy:
-      self._configure_distributed_session()
       return training_distributed.predict_distributed(self,
                                                       x=x,
                                                       batch_size=batch_size,
@@ -2715,26 +2680,6 @@ class Model(Network):
     self.output_names = training_utils.generic_output_names(outputs)
     self.built = True
 
-  def _configure_distributed_session(self):
-    """Configure a Session for use with Distribution Strategies.
-
-    Raises:
-      ValueError: If a non-distributed Session has already been created.
-    """
-    if not self._distributed_session_is_configured:
-      if (dc_context.get_current_worker_context() is not None and
-          getattr(K._SESSION, 'session', None) is not None):  # pylint: disable=protected-access
-        raise ValueError('Session was created before `fit`, `evaluate`, '
-                         'or `predict` was called. With Multi-Worker '
-                         'mode, this is not allowed. Please avoid '
-                         'creating a Session outside of these methods. '
-                         'The Session may have been created by a call '
-                         'to `keras.backend.get_session()` or '
-                         'functions that use Sessions, like `load_weights`.')
-      distributed_training_utils.configure_and_create_session(
-          self._distribution_strategy)
-      self._distributed_session_is_configured = True
-
 
 class DistributedCallbackModel(Model):
   """Model that is used for callbacks with DistributionStrategy."""
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index ea78a24607..4c913b5d51 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -22,8 +22,6 @@ import abc
 import collections
 from collections import OrderedDict
 import copy
-import json
-import os
 
 import numpy as np
 import six
@@ -49,7 +47,6 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import weights_broadcast_ops
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.training import server_lib
 from tensorflow.python.util import nest
 
 
@@ -1536,13 +1533,6 @@ def convert_eager_tensors_to_numpy(structure):
   return nest.map_structure(_convert, structure)
 
 
-def should_run_multi_worker():
-  """Whether a model should be run using DistributedCoordinator."""
-  tf_config = json.loads(os.environ.get('TF_CONFIG', '{}'))
-  cluster_spec = server_lib.ClusterSpec(tf_config.get('cluster', {}))
-  return tf_config and 'master' not in cluster_spec.jobs
-
-
 def should_run_validation(validation_freq, epoch):
   """Checks if validation should be run this epoch.
 
-- 
GitLab


From 187bbff3bcaa77afa1d7775fe4cb1e77189b80f5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 28 Jan 2019 17:34:14 -0800
Subject: [PATCH 1436/2345] Dump graph debug information when saving the
 meta_graph.

PiperOrigin-RevId: 231320436
---
 tensorflow/core/BUILD                         |  1 +
 .../core/protobuf/graph_debug_info.proto      | 35 +++++++++
 .../python/framework/error_interpolation.py   | 24 +++++++
 tensorflow/python/framework/meta_graph.py     | 71 ++++++++++++++++++-
 .../python/framework/meta_graph_test.py       | 20 ++++++
 tensorflow/python/training/saver.py           | 24 +++++--
 .../golden/v1/tensorflow.train.-saver.pbtxt   |  4 +-
 .../api/golden/v1/tensorflow.train.pbtxt      |  2 +-
 8 files changed, 173 insertions(+), 8 deletions(-)
 create mode 100644 tensorflow/core/protobuf/graph_debug_info.proto

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index a932974270..3d92a836d1 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -201,6 +201,7 @@ COMMON_PROTO_SRCS = [
     "protobuf/cluster.proto",
     "protobuf/debug.proto",
     "protobuf/device_properties.proto",
+    "protobuf/graph_debug_info.proto",
     "protobuf/queue_runner.proto",
     "protobuf/rewriter_config.proto",
     "protobuf/tensor_bundle.proto",
diff --git a/tensorflow/core/protobuf/graph_debug_info.proto b/tensorflow/core/protobuf/graph_debug_info.proto
new file mode 100644
index 0000000000..cd3cdeba42
--- /dev/null
+++ b/tensorflow/core/protobuf/graph_debug_info.proto
@@ -0,0 +1,35 @@
+syntax = "proto3";
+
+package tensorflow;
+option cc_enable_arenas = true;
+option java_outer_classname = "GraphDebugInfoProtos";
+option java_multiple_files = true;
+option java_package = "org.tensorflow.framework";
+
+message GraphDebugInfo {
+  // This represents a file/line location in the source code.
+  message FileLineCol {
+    // File name index, which can be used to retrive the file name string from
+    // `files`. The value should be between 0 and (len(files)-1)
+    int32 file_index = 1;
+
+    // Line number in the file.
+    int32 line = 2;
+
+    // Col number in the file line.
+    int32 col = 3;
+  }
+
+  // This represents a stack trace which is a ordered list of `FileLineCol`.
+  message StackTrace {
+    // Each line in the stack trace.
+    repeated FileLineCol file_line_cols = 1;
+  }
+
+  // This stores all the source code file names and can be indexed by the
+  // `file_index`.
+  repeated string files = 1;
+
+  // This maps a node name to a stack trace in the source code.
+  map<string, StackTrace> traces = 2;
+}
diff --git a/tensorflow/python/framework/error_interpolation.py b/tensorflow/python/framework/error_interpolation.py
index af83b70a46..2d1b210d2a 100644
--- a/tensorflow/python/framework/error_interpolation.py
+++ b/tensorflow/python/framework/error_interpolation.py
@@ -210,6 +210,30 @@ def _get_defining_frame_from_op(op):
   frame_index = _find_index_of_defining_frame_for_op(op)
   return op.traceback[frame_index]
 
+def compute_useful_stack(op):
+  """Return a list of line name and lineno pairs, which form a 'useful' stack.
+
+  Starting from the defining frame to the outermost one, this method computes
+  the contiguous portion of the 'useful' stack trace and returns each line as
+  a line name and lineno pair.
+
+  Args:
+    op: op.Operation object having a _traceback member.
+
+  Returns:
+    A list of line name and lineno pairs. Below is an example of returned list:
+    [("tool_utils.py", "124"), ("tool_utils.py", "21"), ....]
+  """
+  defining_frame_index = _find_index_of_defining_frame_for_op(op)
+  stack_trace = []
+  # the stack trace is collected from the defining (included) to the outermost.
+  for index in reversed(range(defining_frame_index + 1)):
+    frame = op.traceback[index]
+    filename = frame[tf_stack.TB_FILENAME]
+    lineno = frame[tf_stack.TB_LINENO]
+    stack_trace.append((filename, lineno))
+  return stack_trace
+
 
 def compute_field_dict(op, strip_file_prefix=""):
   """Return a dictionary mapping interpolation tokens to values.
diff --git a/tensorflow/python/framework/meta_graph.py b/tensorflow/python/framework/meta_graph.py
index ddf6f66e8a..0febab506e 100644
--- a/tensorflow/python/framework/meta_graph.py
+++ b/tensorflow/python/framework/meta_graph.py
@@ -29,10 +29,12 @@ from google.protobuf import text_format
 from tensorflow.core.framework import attr_value_pb2
 from tensorflow.core.framework import graph_pb2
 from tensorflow.core.framework import op_def_pb2
+from tensorflow.core.protobuf import graph_debug_info_pb2
 from tensorflow.core.protobuf import meta_graph_pb2
 from tensorflow.core.protobuf import saver_pb2
 from tensorflow.python import pywrap_tensorflow
 from tensorflow.python.eager import context
+from tensorflow.python.framework import error_interpolation
 from tensorflow.python.framework import graph_io
 from tensorflow.python.framework import importer
 from tensorflow.python.framework import op_def_registry
@@ -509,6 +511,52 @@ def strip_graph_default_valued_attrs(meta_graph_def):
   meta_graph_def.meta_info_def.stripped_default_attrs = True
 
 
+def create_graph_debug_info_def(operations):
+  """Construct and returns a `GraphDebugInfo` protocol buffer.
+
+  Args:
+    operations: An iterable of op.Operation objects having _traceback members.
+
+  Returns:
+    GraphDebugInfo protocol buffer.
+
+  Raises:
+    TypeError: If the arguments are not of the correct proto buffer type.
+  """
+  # Creates an empty GraphDebugInfoDef proto.
+  graph_debug_info_def = graph_debug_info_pb2.GraphDebugInfo()
+
+  # Gets the file names and line numbers for the exported node names. Also
+  # collects the unique file names.
+  all_file_names = set()
+  node_to_trace = {}
+  for op in operations:
+    # Gets the stack trace of the operation and then the file location.
+    node_name = op.name
+    node_to_trace[node_name] = error_interpolation.compute_useful_stack(op)
+    for trace in node_to_trace[node_name]:
+      all_file_names.add(trace[0])
+
+  # Sets the `files` field in the GraphDebugInfo proto
+  graph_debug_info_def.files.extend(sorted(all_file_names))
+
+  # Builds a mapping between file names and index of the `files` field, so we
+  # only store the indexes for the nodes in the GraphDebugInfo.
+  file_to_index = dict(
+      [(y, x) for x, y in enumerate(graph_debug_info_def.files)])
+
+  # Creates the FileLineCol proto for each node and sets the value in the
+  # GraphDebugInfo proto. We only store the file name index for each node to
+  # save the storage space.
+  for node_name, trace in node_to_trace.items():
+    trace_def = graph_debug_info_def.traces[node_name]
+    for file_line in trace:
+      file_index = file_to_index[file_line[0]]
+      trace_def.file_line_cols.add(file_index=file_index, line=file_line[1])
+
+  return graph_debug_info_def
+
+
 def create_meta_graph_def(meta_info_def=None,
                           graph_def=None,
                           saver_def=None,
@@ -881,6 +929,7 @@ def export_scoped_meta_graph(filename=None,
                              saver_def=None,
                              clear_extraneous_savers=False,
                              strip_default_attrs=False,
+                             save_debug_info=False,
                              **kwargs):
   """Returns `MetaGraphDef` proto. Optionally writes it to filename.
 
@@ -910,7 +959,10 @@ def export_scoped_meta_graph(filename=None,
         graph (both Save/Restore ops and SaverDefs) that are not associated
         with the provided SaverDef.
     strip_default_attrs: Set to true if default valued attributes must be
-        removed while exporting the GraphDef.
+      removed while exporting the GraphDef.
+    save_debug_info: If `True`, save the GraphDebugInfo to a separate file,
+      which in the same directory of filename and with `_debug` added before the
+      file extension.
     **kwargs: Optional keyed arguments, including meta_info_def and
         collection_list.
 
@@ -1005,6 +1057,23 @@ def export_scoped_meta_graph(filename=None,
         os.path.dirname(filename),
         os.path.basename(filename),
         as_text=as_text)
+    if save_debug_info:
+      name, _ = os.path.splitext(filename)
+      debug_filename = "{name}{ext}".format(name=name, ext=".debug")
+
+      # Gets the operation from the graph by the name.
+      ops_to_export = {}
+      for node in scoped_meta_graph_def.graph_def.node:
+        scoped_op_name = ops.prepend_name_scope(node.name, export_scope)
+        ops_to_export.add(graph.get_operation_by_name(scoped_op_name))
+
+      graph_debug_info = create_graph_debug_info_def(ops_to_export)
+
+      graph_io.write_graph(
+          graph_debug_info,
+          os.path.dirname(debug_filename),
+          os.path.basename(debug_filename),
+          as_text=as_text)
 
   return scoped_meta_graph_def, var_list
 
diff --git a/tensorflow/python/framework/meta_graph_test.py b/tensorflow/python/framework/meta_graph_test.py
index e6e8788164..3a0f338e23 100644
--- a/tensorflow/python/framework/meta_graph_test.py
+++ b/tensorflow/python/framework/meta_graph_test.py
@@ -707,6 +707,26 @@ class ScopedMetaGraphTest(test.TestCase):
     test_util.assert_meta_graph_protos_equal(self, orig_meta_graph,
                                              new_meta_graph)
 
+  def testExportDebugInfo(self):
+    graph1 = ops.Graph()
+    with graph1.as_default():
+      with ops.name_scope("hidden1/hidden2/hidden3"):
+        images = constant_op.constant(
+            1.0, dtypes.float32, shape=[3, 2], name="images")
+        weights1 = variables.Variable([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]],
+                                      name="weights")
+        biases1 = resource_variable_ops.ResourceVariable(
+            [0.1] * 3, name="biases")
+        nn_ops.relu(math_ops.matmul(images, weights1) + biases1, name="relu")
+    debug_info_def = meta_graph.create_graph_debug_info_def(
+        operations=graph1.get_operations())
+
+    # The unique file names in all the stack traces should be larger or equal
+    # than 1.
+    self.assertTrue(len(debug_info_def.files) >= 1)
+    # All the nodes from the exported graphdef are included.
+    self.assertEqual(len(debug_info_def.traces), len(graph1.get_operations()))
+
   # Verifies that we can export a subgraph in a nested name scope containing a
   # "hidden1/hidden2" and import it into "new_hidden1/new_hidden2" in a new
   # graph.
diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py
index 348b8bf1ef..ee0e2154e8 100644
--- a/tensorflow/python/training/saver.py
+++ b/tensorflow/python/training/saver.py
@@ -1078,7 +1078,8 @@ class Saver(object):
            meta_graph_suffix="meta",
            write_meta_graph=True,
            write_state=True,
-           strip_default_attrs=False):
+           strip_default_attrs=False,
+           save_debug_info=False):
     # pylint: disable=line-too-long
     """Saves variables.
 
@@ -1108,6 +1109,10 @@ class Saver(object):
       strip_default_attrs: Boolean. If `True`, default-valued attributes will be
         removed from the NodeDefs. For a detailed guide, see
         [Stripping Default-Valued Attributes](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/README.md#stripping-default-valued-attributes).
+      save_debug_info: If `True`, save the GraphDebugInfo to a separate file,
+        which in the same directory of save_path and with `_debug` added before
+        the file extension. This is only enabled when `write_meta_graph` is
+        `True`
 
     Returns:
       A string: path prefix used for the checkpoint files.  If the saver is
@@ -1193,7 +1198,8 @@ class Saver(object):
       if not context.executing_eagerly():
         with sess.graph.as_default():
           self.export_meta_graph(
-              meta_graph_filename, strip_default_attrs=strip_default_attrs)
+              meta_graph_filename, strip_default_attrs=strip_default_attrs,
+              save_debug_info=save_debug_info)
 
     if self._is_empty:
       return None
@@ -1207,7 +1213,8 @@ class Saver(object):
                         export_scope=None,
                         clear_devices=False,
                         clear_extraneous_savers=False,
-                        strip_default_attrs=False):
+                        strip_default_attrs=False,
+                        save_debug_info=False):
     # pylint: disable=line-too-long
     """Writes `MetaGraphDef` to save_path/filename.
 
@@ -1224,6 +1231,9 @@ class Saver(object):
       strip_default_attrs: Boolean. If `True`, default-valued attributes will be
         removed from the NodeDefs. For a detailed guide, see
         [Stripping Default-Valued Attributes](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/README.md#stripping-default-valued-attributes).
+      save_debug_info: If `True`, save the GraphDebugInfo to a separate file,
+        which in the same directory of filename and with `_debug` added before
+        the file extension.
 
     Returns:
       A `MetaGraphDef` proto.
@@ -1238,7 +1248,8 @@ class Saver(object):
         export_scope=export_scope,
         clear_devices=clear_devices,
         clear_extraneous_savers=clear_extraneous_savers,
-        strip_default_attrs=strip_default_attrs)
+        strip_default_attrs=strip_default_attrs,
+        save_debug_info=save_debug_info)
 
   def restore(self, sess, save_path):
     """Restores previously saved variables.
@@ -1497,6 +1508,7 @@ def export_meta_graph(filename=None,
                       clear_devices=False,
                       clear_extraneous_savers=False,
                       strip_default_attrs=False,
+                      save_debug_info=False,
                       **kwargs):
   # pylint: disable=line-too-long
   """Returns `MetaGraphDef` proto. Optionally writes it to filename.
@@ -1527,6 +1539,9 @@ def export_meta_graph(filename=None,
     strip_default_attrs: Boolean. If `True`, default-valued attributes will be
       removed from the NodeDefs. For a detailed guide, see
       [Stripping Default-Valued Attributes](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/README.md#stripping-default-valued-attributes).
+    save_debug_info: If `True`, save the GraphDebugInfo to a separate file,
+      which in the same directory of filename and with `_debug` added before
+      the file extend.
     **kwargs: Optional keyed arguments.
 
   Returns:
@@ -1558,6 +1573,7 @@ def export_meta_graph(filename=None,
       clear_devices=clear_devices,
       clear_extraneous_savers=clear_extraneous_savers,
       strip_default_attrs=strip_default_attrs,
+      save_debug_info=save_debug_info,
       **kwargs)
   return meta_graph_def
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.-saver.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.-saver.pbtxt
index 2cda458f46..a91ba5b672 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.train.-saver.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.train.-saver.pbtxt
@@ -20,7 +20,7 @@ tf_class {
   }
   member_method {
     name: "export_meta_graph"
-    argspec: "args=[\'self\', \'filename\', \'collection_list\', \'as_text\', \'export_scope\', \'clear_devices\', \'clear_extraneous_savers\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'None\', \'False\', \'False\', \'False\'], "
+    argspec: "args=[\'self\', \'filename\', \'collection_list\', \'as_text\', \'export_scope\', \'clear_devices\', \'clear_extraneous_savers\', \'strip_default_attrs\', \'save_debug_info\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'None\', \'False\', \'False\', \'False\', \'False\'], "
   }
   member_method {
     name: "from_proto"
@@ -36,7 +36,7 @@ tf_class {
   }
   member_method {
     name: "save"
-    argspec: "args=[\'self\', \'sess\', \'save_path\', \'global_step\', \'latest_filename\', \'meta_graph_suffix\', \'write_meta_graph\', \'write_state\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'meta\', \'True\', \'True\', \'False\'], "
+    argspec: "args=[\'self\', \'sess\', \'save_path\', \'global_step\', \'latest_filename\', \'meta_graph_suffix\', \'write_meta_graph\', \'write_state\', \'strip_default_attrs\', \'save_debug_info\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'meta\', \'True\', \'True\', \'False\', \'False\'], "
   }
   member_method {
     name: "set_last_checkpoints"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.pbtxt
index bdb3ea2197..ab7b199d80 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.train.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.train.pbtxt
@@ -298,7 +298,7 @@ tf_module {
   }
   member_method {
     name: "export_meta_graph"
-    argspec: "args=[\'filename\', \'meta_info_def\', \'graph_def\', \'saver_def\', \'collection_list\', \'as_text\', \'graph\', \'export_scope\', \'clear_devices\', \'clear_extraneous_savers\', \'strip_default_attrs\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'False\', \'None\', \'None\', \'False\', \'False\', \'False\'], "
+    argspec: "args=[\'filename\', \'meta_info_def\', \'graph_def\', \'saver_def\', \'collection_list\', \'as_text\', \'graph\', \'export_scope\', \'clear_devices\', \'clear_extraneous_savers\', \'strip_default_attrs\', \'save_debug_info\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'False\', \'None\', \'None\', \'False\', \'False\', \'False\', \'False\'], "
   }
   member_method {
     name: "generate_checkpoint_state_proto"
-- 
GitLab


From d3a2429be4c3d428af047d368e488bddd5627a67 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Mon, 28 Jan 2019 17:46:34 -0800
Subject: [PATCH 1437/2345] [XLA] Add variadic overloads of xla::{And,Or}.

Lets you specify And(x, y, z) instead of And(x, And(y, z)).

PiperOrigin-RevId: 231322047
---
 tensorflow/compiler/xla/client/xla_builder.h  | 24 ++++++++++++++
 .../compiler/xla/client/xla_builder_test.cc   | 32 +++++++++++++++++++
 2 files changed, 56 insertions(+)

diff --git a/tensorflow/compiler/xla/client/xla_builder.h b/tensorflow/compiler/xla/client/xla_builder.h
index 3bd6d42363..e024e2810d 100644
--- a/tensorflow/compiler/xla/client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_builder.h
@@ -1580,9 +1580,33 @@ XlaOp Min(const XlaOp& lhs, const XlaOp& rhs,
 XlaOp And(const XlaOp& lhs, const XlaOp& rhs,
           absl::Span<const int64> broadcast_dimensions = {});
 
+// Overload to call And with 3 or more operands.  We need the following somewhat
+// convoluted overload set to disambiguate with the overload that takes the
+// `broadcast_dimensions` optional param.
+inline XlaOp And(const XlaOp& op1, const XlaOp& op2, const XlaOp& op3) {
+  return And(op1, And(op2, op3));
+}
+template <typename... XlaOpTs>
+XlaOp And(const XlaOp& op1, const XlaOp& op2, const XlaOp& op3,
+          const XlaOpTs&... operands) {
+  return And(op1, And(op2, And(op3, operands...)));
+}
+
 XlaOp Or(const XlaOp& lhs, const XlaOp& rhs,
          absl::Span<const int64> broadcast_dimensions = {});
 
+// Overload to call Or with 3 or more operands.  As with `And`, we need the
+// following complicated overload set to handle the default arg in the `Or`
+// overload above.
+inline XlaOp Or(const XlaOp& op1, const XlaOp& op2, const XlaOp& op3) {
+  return Or(op1, Or(op2, op3));
+}
+template <typename... XlaOpTs>
+XlaOp Or(const XlaOp& op1, const XlaOp& op2, const XlaOp& op3,
+         const XlaOpTs&... operands) {
+  return Or(op1, Or(op2, Or(op3, operands...)));
+}
+
 XlaOp Xor(const XlaOp& lhs, const XlaOp& rhs,
           absl::Span<const int64> broadcast_dimensions = {});
 
diff --git a/tensorflow/compiler/xla/client/xla_builder_test.cc b/tensorflow/compiler/xla/client/xla_builder_test.cc
index 098165000a..743434cba8 100644
--- a/tensorflow/compiler/xla/client/xla_builder_test.cc
+++ b/tensorflow/compiler/xla/client/xla_builder_test.cc
@@ -135,6 +135,38 @@ TEST_F(XlaBuilderTest, BinaryOperatorsBuildExpectedHLO) {
       op::ShiftRightLogical(op::Constant(), op::Constant()));
 }
 
+TEST_F(XlaBuilderTest, VariadicAnd) {
+  XlaBuilder b(TestName());
+  Shape s = ShapeUtil::MakeShape(PRED, {});
+  And(Parameter(&b, 0, s, "p0"), Parameter(&b, 1, s, "p1"),
+      Parameter(&b, 2, s, "p2"));
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  // Don't specify in the test whether And(x, y, z) is right- or
+  // left-associative; accept either one.
+  EXPECT_THAT(
+      module->entry_computation()->root_instruction(),
+      ::testing::AnyOf(op::And(op::Parameter(0),
+                               op::And(op::Parameter(1), op::Parameter(2))),
+                       op::And(op::And(op::Parameter(0), op::Parameter(1)),
+                               op::Parameter(2))));
+}
+
+TEST_F(XlaBuilderTest, VariadicOr) {
+  XlaBuilder b(TestName());
+  Shape s = ShapeUtil::MakeShape(PRED, {});
+  Or(Parameter(&b, 0, s, "p0"), Parameter(&b, 1, s, "p1"),
+     Parameter(&b, 2, s, "p2"));
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  // Don't specify in the test whether Or(x, y, z) is right- or
+  // left-associative; accept either one.
+  EXPECT_THAT(
+      module->entry_computation()->root_instruction(),
+      ::testing::AnyOf(
+          op::Or(op::Parameter(0), op::Or(op::Parameter(1), op::Parameter(2))),
+          op::Or(op::Or(op::Parameter(0), op::Parameter(1)),
+                 op::Parameter(2))));
+}
+
 TEST_F(XlaBuilderTest, ShiftRightOperatorOnNonIntegerProducesError) {
   XlaBuilder b(TestName());
   ConstantR0<float>(&b, 1) >> ConstantR0<float>(&b, 2);
-- 
GitLab


From 3b24c98659e87ff753f09ded2ae5b95c30610812 Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Mon, 28 Jan 2019 17:46:48 -0800
Subject: [PATCH 1438/2345] Expose tf.function in v1

PiperOrigin-RevId: 231322087
---
 tensorflow/python/eager/def_function.py         | 3 +--
 tensorflow/tools/api/golden/v1/tensorflow.pbtxt | 4 ++++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py
index 1175052530..c144bbf270 100644
--- a/tensorflow/python/eager/def_function.py
+++ b/tensorflow/python/eager/def_function.py
@@ -658,8 +658,7 @@ class Function(object):
     return self._descriptor_cache[instance]
 
 
-# In TensorFlow 1.x, exported as tf.contrib.eager.function
-@tf_export("function", v1=[])
+@tf_export("function")
 def function(func=None,
              input_signature=None,
              autograph=True,
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
index 016ae23890..585dc2ea57 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
@@ -1260,6 +1260,10 @@ tf_module {
     name: "foldr"
     argspec: "args=[\'fn\', \'elems\', \'initializer\', \'parallel_iterations\', \'back_prop\', \'swap_memory\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'10\', \'True\', \'False\', \'None\'], "
   }
+  member_method {
+    name: "function"
+    argspec: "args=[\'func\', \'input_signature\', \'autograph\', \'experimental_autograph_options\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\', \'None\'], "
+  }
   member_method {
     name: "gather"
     argspec: "args=[\'params\', \'indices\', \'validate_indices\', \'name\', \'axis\', \'batch_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'0\'], "
-- 
GitLab


From b44e732fb3ac7cfd71608fa126bde11142073893 Mon Sep 17 00:00:00 2001
From: Suharsh Sivakumar <suharshs@google.com>
Date: Mon, 28 Jan 2019 18:16:03 -0800
Subject: [PATCH 1439/2345] Make test_util.h follow functionality of
 quantization param changes in model.cc

PiperOrigin-RevId: 231325828
---
 tensorflow/lite/kernels/test_util.h | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/tensorflow/lite/kernels/test_util.h b/tensorflow/lite/kernels/test_util.h
index c5435b3546..83f0868595 100644
--- a/tensorflow/lite/kernels/test_util.h
+++ b/tensorflow/lite/kernels/test_util.h
@@ -430,6 +430,17 @@ class SingleOpModel {
     // Update quantization params.
     t->params.scale = scaling_factor;
     t->params.zero_point = 0;
+    // Populate the new quantization params.
+    TfLiteQuantizationFree(&t->quantization);
+    t->quantization.type = kTfLiteAffineQuantization;
+    auto* affine_quantization = reinterpret_cast<TfLiteAffineQuantization*>(
+        malloc(sizeof(TfLiteAffineQuantization)));
+    affine_quantization->quantized_dimension = 0;
+    affine_quantization->scale = TfLiteFloatArrayCreate(1);
+    affine_quantization->zero_point = TfLiteIntArrayCreate(1);
+    affine_quantization->scale->data[0] = scaling_factor;
+    affine_quantization->zero_point->data[0] = 0;
+    t->quantization.params = affine_quantization;
     return q;
   }
 
-- 
GitLab


From a3f6654bcea084dd9dd29f2f43a5065607abafe1 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Mon, 28 Jan 2019 18:16:03 -0800
Subject: [PATCH 1440/2345] [Grappler] Add inputs_ready node to inlined
 function body.

Functions inlined inside a body of a while loop can receive tensors with mismatched deadness.

PiperOrigin-RevId: 231325829
---
 .../core/grappler/mutable_graph_view.cc       |   2 +
 .../grappler/optimizers/function_optimizer.cc | 128 +++++++++++-------
 .../optimizers/function_optimizer_test.cc     |  91 +++++++++----
 3 files changed, 144 insertions(+), 77 deletions(-)

diff --git a/tensorflow/core/grappler/mutable_graph_view.cc b/tensorflow/core/grappler/mutable_graph_view.cc
index d4f415ae7b..e56431090f 100644
--- a/tensorflow/core/grappler/mutable_graph_view.cc
+++ b/tensorflow/core/grappler/mutable_graph_view.cc
@@ -89,6 +89,8 @@ bool CanDedupControlWithRegularInput(const MutableGraphView& graph,
 bool CanDedupControlWithRegularInput(const MutableGraphView& graph,
                                      absl::string_view control_node_name) {
   NodeDef* control_node = graph.GetNode(control_node_name);
+  DCHECK(control_node != nullptr)
+      << "Didn't find a node for control dependency: " << control_node_name;
   return CanDedupControlWithRegularInput(graph, *control_node);
 }
 
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index 6913f9b55b..215ffc3b43 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -68,10 +68,13 @@ constexpr char kFuncAttrName[] = "f";
 
 constexpr char kNoInlineAttr[] = "_noinline";
 
-// Names of the nodes that used to anchor incoming/outgoing control edges for
-// inlined function calls (see InlineIndirectFunctionCall).
-constexpr char kControlInputNodeName[] = "control_input";
-constexpr char kControlOutputNodeName[] = "control_output";
+// Name of the node that will have control edges from function input nodes, and
+// also used as a new destination for incoming control edges.
+constexpr char kInputsReadyNodeName[] = "inputs_ready";
+
+// Name of the node that will have control edges from function output nodes, and
+// also used as a new source of outgoing control edges.
+constexpr char kOutputsReadyNodeName[] = "outputs_ready";
 
 bool AttrIsTrue(const FunctionDef& func, const string& attr) {
   return func.attr().count(attr) != 0 && func.attr().at(attr).b();
@@ -1289,11 +1292,14 @@ struct MaybeDeadOutput {
 Status MaybeDeadOutputs(const FunctionOptimizerContext& ctx,
                         const GrapplerFunctionItem& item,
                         std::vector<MaybeDeadOutput>* maybe_dead) {
+  VLOG(3) << "Find function outputs that might return dead tensors: item.id="
+          << item.id;
   DCHECK(maybe_dead->empty()) << "Input argument must be an empty vector";
 
   std::vector<const NodeDef*> dead_tensor_srcs;
   for (const NodeDef& node : item.graph.node()) {
     if (IsSwitch(node)) {
+      VLOG(4) << "Add dead tensors source. Switch node: " << node.name();
       dead_tensor_srcs.push_back(&node);
       continue;
     }
@@ -1310,7 +1316,11 @@ Status MaybeDeadOutputs(const FunctionOptimizerContext& ctx,
       std::vector<MaybeDeadOutput> func_dead_outputs;
       TF_RETURN_IF_ERROR(MaybeDeadOutputs(ctx, func_item, &func_dead_outputs));
 
-      if (!func_dead_outputs.empty()) dead_tensor_srcs.push_back(&node);
+      if (!func_dead_outputs.empty()) {
+        VLOG(4) << "Add dead tensors source. Function call: " << node.op()
+                << " node=" << node.name();
+        dead_tensor_srcs.push_back(&node);
+      }
     }
   }
 
@@ -1425,7 +1435,10 @@ Status CheckThatSideEffectsWillExecute(
 
     // Check if we reached one of the output nodes.
     const auto callbacks = DfsCallbacks::PreOrder([&](const NodeDef* node) {
-      if (output_nodes.count(node->name())) will_execute = true;
+      if (output_nodes.contains(node->name())) {
+        VLOG(4) << "Found a path to output node: " << node->name();
+        will_execute = true;
+      }
     });
 
     // Stop if we already proved that node will execute.
@@ -1644,30 +1657,37 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
   }
 
   // ------------------------------------------------------------------------ //
-  // To guarantee side-effects execution order we add NoOp control_input and
-  // control_output nodes:
-  // 1) 'control_input' node will have incoming control edges from all nodes in
-  //    'happens_before' set.
-  // 2) 'control_output' node will have outgoing control edges to all nodes in
-  //    'happens_after' set.
-
-  NodeDef* control_input = nullptr;
-  NodeDef* control_output = nullptr;
-
-  // IMPORTANT: Actual control inputs will be added to these nodes at the very
+  // IMPORTANT: Actual inputs will be added to the following nodes at the very
   // last stage, because we don't want to have invalid edges in a function body
-  // graph (control edges depend on the nodes in the "outer" optimized graph).
-
-  if (!happens_before.empty()) {
-    control_input = item.graph.add_node();
-    control_input->set_op("NoOp");
-    control_input->set_name(kControlInputNodeName);
-  }
-
-  if (!happens_after.empty()) {
-    control_output = item.graph.add_node();
-    control_output->set_op("NoOp");
-    control_output->set_name(kControlOutputNodeName);
+  // graph (control edges that depend on the nodes in the "outer" optimized
+  // graph).
+
+  // If one of the function inputs is a dead tensor, we must not execute any of
+  // the function body nodes, and let the dead tensor flag propagate through the
+  // inlined function body. We add NoOp inputs_ready node, and add control edges
+  // to it from all input nodes. Inlined function arguments (Identity nodes)
+  // will have a control dependency on it.
+  //
+  // If the function call node has incoming control edges, we will update them
+  // to use this node as destination, to ensure side-effects execution order.
+  NodeDef* inputs_ready_node = nullptr;
+  if (func_node.input_size() > 0) {
+    inputs_ready_node = item.graph.add_node();
+    inputs_ready_node->set_op("NoOp");
+    inputs_ready_node->set_name(kInputsReadyNodeName);
+  }
+
+  // All nodes that have control edge from the function call node, will be
+  // updated to have a control edge from 'outputs_ready_node`. This node will
+  // have control edges from all function outputs. This a "barrier" that
+  // guarantees that all function side effects were executed, and it will also
+  // allow to propagate deadness flag (if there is a deadness mismatch between
+  // output nodes).
+  NodeDef* outputs_ready_node = nullptr;
+  if (item.output_size() > 0 || !happens_after.empty()) {
+    outputs_ready_node = item.graph.add_node();
+    outputs_ready_node->set_op("NoOp");
+    outputs_ready_node->set_name(kOutputsReadyNodeName);
   }
 
   // ------------------------------------------------------------------------ //
@@ -1682,11 +1702,8 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
   // TODO(ezhulenev): Use FrameMap (see grappler/utils/frame.h) to find out if
   // the function is called inside a loop.
   std::vector<string> empty_inputs_hook;
-  if (!item.inputs().empty()) {
-    const InputArgExpansion& arg0 = item.inputs()[0];
-    empty_inputs_hook.push_back(arg0.placeholders[0]);
-  } else if (control_input != nullptr) {
-    empty_inputs_hook.push_back(control_input->name());
+  if (inputs_ready_node != nullptr) {
+    empty_inputs_hook.push_back(inputs_ready_node->name());
   }
 
   // ------------------------------------------------------------------------ //
@@ -1720,20 +1737,31 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
       const int input_idx = input_placeholder_idx->second;
       func_body_node.add_input(inputs[input_idx].ToString());
 
-      // All side effects must happen before inputs can start executing.
-      if (control_input) {
+      // Add a control dependency on 'inputs_ready' node, to guarantee that all
+      // inputs are alive and all side-effects executed before function body.
+      if (inputs_ready_node) {
         func_body_node.add_input(
-            AsControlDependency(inlined_node_name(control_input->name())));
+            AsControlDependency(inlined_node_name(inputs_ready_node->name())));
       }
     } else {
       // Update inputs of the regular function body nodes.
       for (string& input : *func_body_node.mutable_input()) {
         input = inlined_node_name(input);
       }
-      // Add control input to ensure node executed in correct frame.
-      if (func_body_node.input_size() == 0 && !empty_inputs_hook.empty() &&
-          func_body_node.name() != kControlInputNodeName &&
-          func_body_node.name() != kControlOutputNodeName) {
+
+      // Check if we need to ensure node execution in correct loop frame.
+      bool node_needs_empty_inputs_hook =
+          // We have a node to hook and node has no inputs.
+          !empty_inputs_hook.empty() && func_body_node.input_size() == 0 &&
+          // Inputs ready node will always have edge from main graph. If
+          // function call has no regular and control inputs, we will not add
+          // inputs_ready node to the function body graph.
+          node_name != kInputsReadyNodeName &&
+          // Outputs ready node might not have any inputs (in case function has
+          // no outputs), so we must make sure it's executed in correct frame.
+          (node_name != kOutputsReadyNodeName || item.output_size() == 0);
+
+      if (node_needs_empty_inputs_hook) {
         *func_body_node.add_input() =
             AsControlDependency(inlined_node_name(empty_inputs_hook[0]));
       }
@@ -1778,18 +1806,22 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
   // ------------------------------------------------------------------------ //
   // Move all the nodes to the optimized graph after successful preprocessing.
 
-  if (control_input != nullptr) {
-    string inlined_node = inlined_node_name(control_input->name());
+  if (inputs_ready_node != nullptr) {
+    string inlined_node = inlined_node_name(inputs_ready_node->name());
     absl::optional<int> node_idx = placed_topo_view.GetNodeIndex(inlined_node);
 
-    for (const string& node_name : happens_before) {
-      placed_graph_def.mutable_node(*node_idx)->add_input(
-          AsControlDependency(node_name));
+    absl::flat_hash_set<string> input_nodes;
+    for (const string& input : func_node.input()) {
+      const SafeTensorId tensor = ParseTensorName(input);
+      if (input_nodes.insert(tensor.node()).second) {
+        placed_graph_def.mutable_node(*node_idx)->add_input(
+            AsControlDependency(tensor.node()));
+      }
     }
   }
 
-  if (control_output != nullptr) {
-    string inlined_node = inlined_node_name(control_output->name());
+  if (outputs_ready_node != nullptr) {
+    string inlined_node = inlined_node_name(outputs_ready_node->name());
     absl::optional<int> node_idx = placed_topo_view.GetNodeIndex(inlined_node);
 
     // Add control edges from all nodes producing output tensors.
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
index fc7fffe7b2..7e472ec25b 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
@@ -734,9 +734,13 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionSimpleFunction) {
        NDef("b", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
 
        // Function must be inlined and all nodes placed on a valid device.
-       NDef("c/x", "Identity", {"a:0"}, {{"T", DT_FLOAT}}, kDevice),
-       NDef("c/y", "Identity", {"b:0"}, {{"T", DT_FLOAT}}, kDevice),
+       NDef("c/inputs_ready", "NoOp", {"^a", "^b"}, {}, kDevice),
+       NDef("c/x", "Identity", {"a:0", "^c/inputs_ready"}, {{"T", DT_FLOAT}},
+            kDevice),
+       NDef("c/y", "Identity", {"b:0", "^c/inputs_ready"}, {{"T", DT_FLOAT}},
+            kDevice),
        NDef("c/mul", "Mul", {"c/x", "c/y"}, {{"T", DT_FLOAT}}, kDevice),
+       NDef("c/outputs_ready", "NoOp", {"^c/mul"}, {}, kDevice),
 
        NDef("d", "Identity", {"c/mul:0"}, {{"T", DT_FLOAT}}, kDevice)},
       // Function library.
@@ -831,43 +835,51 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithControlDependencies) {
             kDevice),
 
        // Function body of a first function call inlined into the graph.
-       NDef("f1/control_input", "NoOp", {"^init_v"}, {}, kDevice),
-       NDef("f1/x", "Identity", {"a:0", "^f1/control_input"}, {{"T", DT_FLOAT}},
+       NDef("f1/inputs_ready", "NoOp", {"^a", "^b", "^v", "^init_v"}, {},
             kDevice),
-       NDef("f1/y", "Identity", {"b:0", "^f1/control_input"}, {{"T", DT_FLOAT}},
+
+       NDef("f1/x", "Identity", {"a:0", "^f1/inputs_ready"}, {{"T", DT_FLOAT}},
             kDevice),
-       NDef("f1/v", "Identity", {"v:0", "^f1/control_input"},
+       NDef("f1/y", "Identity", {"b:0", "^f1/inputs_ready"}, {{"T", DT_FLOAT}},
+            kDevice),
+       NDef("f1/v", "Identity", {"v:0", "^f1/inputs_ready"},
             {{"T", DT_RESOURCE}}, kDevice),
-       NDef("f1/one", "Const", {"^f1/x"},
+
+       NDef("f1/one", "Const", {"^f1/inputs_ready"},
             {{"dtype", DT_FLOAT}, {"value", kOne}}, kDevice),
        NDef("f1/add", "AssignAddVariableOp", {"f1/v", "f1/one"},
             {{"dtype", DT_FLOAT}}, kDevice),
        NDef("f1/mul", "Mul", {"f1/x", "f1/y", "^f1/add"}, {{"T", DT_FLOAT}},
             kDevice),
-       NDef("f1/control_output", "NoOp", {"^f1/mul"}, {}, kDevice),
+
+       NDef("f1/outputs_ready", "NoOp", {"^f1/mul"}, {}, kDevice),
 
        // Function body of a second function call also inlined into the graph,
        // and input nodes read directly from the inlined nodes of the first
        // function call.
-       NDef("f2/control_input", "NoOp", {"^f1/control_output"}, {}, kDevice),
-       NDef("f2/x", "Identity", {"f1/mul:0", "^f2/control_input"},
+       NDef("f2/inputs_ready", "NoOp", {"^v", "^f1/outputs_ready"}, {},
+            kDevice),
+
+       NDef("f2/x", "Identity", {"f1/mul:0", "^f2/inputs_ready"},
             {{"T", DT_FLOAT}}, kDevice),
-       NDef("f2/y", "Identity", {"f1/mul:0", "^f2/control_input"},
+       NDef("f2/y", "Identity", {"f1/mul:0", "^f2/inputs_ready"},
             {{"T", DT_FLOAT}}, kDevice),
-       NDef("f2/v", "Identity", {"v:0", "^f2/control_input"},
+       NDef("f2/v", "Identity", {"v:0", "^f2/inputs_ready"},
             {{"T", DT_RESOURCE}}, kDevice),
-       NDef("f2/one", "Const", {"^f2/x"},
+
+       NDef("f2/one", "Const", {"^f2/inputs_ready"},
             {{"dtype", DT_FLOAT}, {"value", kOne}}, kDevice),
        NDef("f2/add", "AssignAddVariableOp", {"f2/v", "f2/one"},
             {{"dtype", DT_FLOAT}}, kDevice),
        NDef("f2/mul", "Mul", {"f2/x", "f2/y", "^f2/add"}, {{"T", DT_FLOAT}},
             kDevice),
-       NDef("f2/control_output", "NoOp", {"^f2/mul"}, {}, kDevice),
+
+       NDef("f2/outputs_ready", "NoOp", {"^f2/mul"}, {}, kDevice),
 
        // Return values read directly from inlined nodes.
        NDef("out_1", "Identity", {"f2/mul:0"}, {{"T", DT_FLOAT}}, kDevice),
        NDef("out_2", "ReadVariableOp",
-            {"v", "^f1/control_output", "^f2/control_output"},
+            {"v", "^f1/outputs_ready", "^f2/outputs_ready"},
             {{"dtype", DT_FLOAT}}, kDevice)},
 
       // Function library.
@@ -933,9 +945,13 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithDevicePlacement) {
 
        // Function must be inlined and `mul` node placed on a requested device,
        // and input `Identity` nodes must be colocated with their source nodes.
-       NDef("c/x", "Identity", {"a:0"}, {{"T", DT_FLOAT}}, cpu0),
-       NDef("c/y", "Identity", {"b:0"}, {{"T", DT_FLOAT}}, cpu1),
+       NDef("c/inputs_ready", "NoOp", {"^a", "^b"}, {}, cpu0),
+       NDef("c/x", "Identity", {"a:0", "^c/inputs_ready"}, {{"T", DT_FLOAT}},
+            cpu0),
+       NDef("c/y", "Identity", {"b:0", "^c/inputs_ready"}, {{"T", DT_FLOAT}},
+            cpu1),
        NDef("c/mul", "Mul", {"c/x", "c/y"}, {{"T", DT_FLOAT}}, cpu1),
+       NDef("c/outputs_ready", "NoOp", {"^c/mul"}, {}, cpu0),
 
        NDef("d", "Identity", {"c/mul:0"}, {{"T", DT_FLOAT}}, cpu0)},
       // Function library.
@@ -1003,20 +1019,24 @@ TEST_F(FunctionOptimizerTest,
        NDef("b", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
 
        // Function body of a first function call inlined into the graph.
-       NDef("f1/x", "Identity", {"a:0"}, {{"T", DT_FLOAT}}, kDevice),
-       NDef("f1/y", "Identity", {"b:0"}, {{"T", DT_FLOAT}}, kDevice),
+       NDef("f1/inputs_ready", "NoOp", {"^a", "^b"}, {}, kDevice),
+       NDef("f1/x", "Identity", {"a:0", "^f1/inputs_ready"}, {{"T", DT_FLOAT}},
+            kDevice),
+       NDef("f1/y", "Identity", {"b:0", "^f1/inputs_ready"}, {{"T", DT_FLOAT}},
+            kDevice),
        NDef("f1/mul", "Mul", {"f1/x", "f1/y"}, {{"T", DT_FLOAT}}, kDevice),
-       NDef("f1/control_output", "NoOp", {"^f1/mul"}, {}, kDevice),
+       NDef("f1/outputs_ready", "NoOp", {"^f1/mul"}, {}, kDevice),
 
        // Function body of a second function call also inlined into the graph,
        // and input nodes read directly from the inlined nodes of the first
        // function call, and control dependency edge removed.
-       NDef("f2/control_input", "NoOp", {"^f1/control_output"}, {}, kDevice),
-       NDef("f2/x", "Identity", {"f1/mul:0", "^f2/control_input"},
+       NDef("f2/inputs_ready", "NoOp", {"^f1/outputs_ready"}, {}, kDevice),
+       NDef("f2/x", "Identity", {"f1/mul:0", "^f2/inputs_ready"},
             {{"T", DT_FLOAT}}, kDevice),
-       NDef("f2/y", "Identity", {"f1/mul:0", "^f2/control_input"},
+       NDef("f2/y", "Identity", {"f1/mul:0", "^f2/inputs_ready"},
             {{"T", DT_FLOAT}}, kDevice),
        NDef("f2/mul", "Mul", {"f2/x", "f2/y"}, {{"T", DT_FLOAT}}, kDevice),
+       NDef("f2/outputs_ready", "NoOp", {"^f2/mul"}, {}, kDevice),
 
        // Return directly from inlined node of f2.
        NDef("out", "Identity", {"f2/mul:0"}, {{"T", DT_FLOAT}}, kDevice)},
@@ -1161,8 +1181,11 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithMergedDeadTensors) {
        NDef("b", "Placeholder", {}, {{"dtype", DT_BOOL}}, kDevice),
 
        // Function body of a first function call inlined into the graph.
-       NDef("fn/x", "Identity", {"a:0"}, {{"T", DT_FLOAT}}, kDevice),
-       NDef("fn/cond", "Identity", {"b:0"}, {{"T", DT_BOOL}}, kDevice),
+       NDef("fn/inputs_ready", "NoOp", {"^a", "^b"}, {}, kDevice),
+       NDef("fn/x", "Identity", {"a:0", "^fn/inputs_ready"}, {{"T", DT_FLOAT}},
+            kDevice),
+       NDef("fn/cond", "Identity", {"b:0", "^fn/inputs_ready"},
+            {{"T", DT_BOOL}}, kDevice),
        NDef("fn/switch", "Switch", {"fn/x:0", "fn/cond:0"}, {{"T", DT_FLOAT}},
             kDevice),
        NDef("fn/if_false", "Identity", {"fn/switch:0"}, {{"T", DT_FLOAT}},
@@ -1171,6 +1194,7 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithMergedDeadTensors) {
             kDevice),
        NDef("fn/merge", "Merge", {"fn/if_false:0", "fn/if_true:0"},
             {{"T", DT_FLOAT}, {"N", 2}}, kDevice),
+       NDef("fn/outputs_ready", "NoOp", {"^fn/merge"}, {}, kDevice),
 
        // Return directly from inlined node.
        NDef("out", "Identity", {"fn/merge:0"}, {{"T", DT_FLOAT}}, kDevice)},
@@ -1241,17 +1265,26 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithNestedFunctionCall) {
   GraphDef expected = test::function::GDef(
       {NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
 
-       // Inlined inputs of `c` node.
-       NDef("b/x", "Identity", {"a:0"}, {{"T", DT_FLOAT}}, kDevice),
+       // Inlined inputs of `b` node.
+       NDef("b/inputs_ready", "NoOp", {"^a"}, {}, kDevice),
+       NDef("b/x", "Identity", {"a:0", "^b/inputs_ready"}, {{"T", DT_FLOAT}},
+            kDevice),
 
        // Inlined inputs of `square` node inside inlined `MySquare` function.
-       NDef("b/square/x", "Identity", {"b/x:0"}, {{"T", DT_FLOAT}}, kDevice),
-       NDef("b/square/y", "Identity", {"b/x:0"}, {{"T", DT_FLOAT}}, kDevice),
+       NDef("b/square/inputs_ready", "NoOp", {"^b/x"}, {}, kDevice),
+       NDef("b/square/x", "Identity", {"b/x:0", "^b/square/inputs_ready"},
+            {{"T", DT_FLOAT}}, kDevice),
+       NDef("b/square/y", "Identity", {"b/x:0", "^b/square/inputs_ready"},
+            {{"T", DT_FLOAT}}, kDevice),
 
        // Inlined mul node from the `MyMul` function.
        NDef("b/square/mul", "Mul", {"b/square/x", "b/square/y"},
             {{"T", DT_FLOAT}}, kDevice),
 
+       NDef("b/square/outputs_ready", "NoOp", {"^b/square/mul"}, {}, kDevice),
+       NDef("b/outputs_ready", "NoOp", {"^b/square/outputs_ready"}, {},
+            kDevice),
+
        NDef("c", "Identity", {"b/square/mul:0"}, {{"T", DT_FLOAT}}, kDevice)},
       // Function library.
       {mul_func});
-- 
GitLab


From bf0ef95b31d8d69f9be2c8d660ba3b09721e8fcf Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Mon, 28 Jan 2019 18:24:08 -0800
Subject: [PATCH 1441/2345] Support captured EagerTensors in serialized
 functions.

This is one issue with loading the table test. I've skipped it for now because we also need to recreate generic resources for the table initializer.

PiperOrigin-RevId: 231326745
---
 .../saved_model/function_serialization.py     |  11 +-
 tensorflow/python/saved_model/load.py         |  13 ++
 tensorflow/python/saved_model/load_test.py    |  42 ++++++
 tensorflow/python/saved_model/save.py         | 142 ++++++++++--------
 .../saved_model/saved_object_graph.proto      |   6 +
 5 files changed, 143 insertions(+), 71 deletions(-)

diff --git a/tensorflow/python/saved_model/function_serialization.py b/tensorflow/python/saved_model/function_serialization.py
index 8c65007bae..986614e5d0 100644
--- a/tensorflow/python/saved_model/function_serialization.py
+++ b/tensorflow/python/saved_model/function_serialization.py
@@ -19,7 +19,6 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.framework import func_graph as func_graph_module
-from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.saved_model import nested_structure_coder
 from tensorflow.python.saved_model import saved_object_graph_pb2
 
@@ -45,14 +44,10 @@ def serialize_concrete_function(concrete_function, node_ids, coder):
     for capture in concrete_function.captured_inputs:
       bound_inputs.append(node_ids[capture])
   except KeyError:
-    # TODO(allenl): This warning shadows a real issue in test_table in
-    # save_test.py, where we don't handle captured constants. Fix that and
-    # then make this an exception.
-    logging.warning(
+    raise KeyError(
         "Concrete function %s not added to object based saved model as it "
-        "captures tensor %s which is unsupported or not reachable from root.",
-        concrete_function.name, capture)
-    return None
+        "captures tensor %s which is unsupported or not reachable from root."
+        % (concrete_function.name, capture))
   concrete_function_proto = saved_object_graph_pb2.SavedConcreteFunction()
   structured_outputs = func_graph_module.convert_structure_to_signature(
       concrete_function.structured_outputs)
diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
index ad7a046ba4..4b9ba02656 100644
--- a/tensorflow/python/saved_model/load.py
+++ b/tensorflow/python/saved_model/load.py
@@ -21,6 +21,8 @@ from __future__ import print_function
 import functools
 import os
 
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import tensor_util
 from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import resource_variable_ops
@@ -43,6 +45,8 @@ class _Loader(object):
   def __init__(self, object_graph_proto, saved_model_proto, export_dir):
     meta_graph = saved_model_proto.meta_graphs[0]
     self._asset_file_def = meta_graph.asset_file_def
+    self._operation_attributes = {
+        node.name: node.attr for node in meta_graph.graph_def.node}
     self._proto = object_graph_proto
     self._export_dir = export_dir
     self._concrete_functions = (
@@ -99,6 +103,8 @@ class _Loader(object):
       return obj.handle
     elif isinstance(obj, tracking.TrackableAsset):
       return obj.asset_path.handle
+    elif tensor_util.is_tensor(obj):
+      return obj
     raise ValueError("Can't convert node %s to tensor" % (type(obj)))
 
   def _load_all(self):
@@ -138,6 +144,7 @@ class _Loader(object):
             self._recreate_bare_concrete_function,
             proto.bare_concrete_function),
         "variable": lambda: self._recreate_variable(proto.variable),
+        "constant": lambda: self._recreate_constant(proto.constant),
     }
     kind = proto.WhichOneof("kind")
     if kind not in factory:
@@ -177,6 +184,12 @@ class _Loader(object):
     dummy_value = init_ops.Zeros(dtype=proto.dtype)(shape=proto.shape)
     return variables.Variable(dummy_value, trainable=proto.trainable), setattr
 
+  def _recreate_constant(self, proto):
+    tensor_proto = self._operation_attributes[proto.operation]["value"].tensor
+    imported_constant = constant_op.constant(
+        tensor_util.MakeNdarray(tensor_proto))
+    return imported_constant, setattr
+
 
 def _call_attribute(instance, *args, **kwargs):
   return instance.__call__(*args, **kwargs)
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index ef703f9d29..61fcea3748 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -32,11 +32,13 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import tensor_spec
 from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import lookup_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.saved_model import load
 from tensorflow.python.saved_model import save
 from tensorflow.python.training.checkpointable import tracking
+from tensorflow.python.training.checkpointable import util
 
 
 @parameterized.named_parameters(
@@ -734,5 +736,45 @@ class LoadTest(test.TestCase, parameterized.TestCase):
     imported.variables[1].assign(4.)
     self.assertAllClose([9., 16.], [loss() for loss in imported.losses])
 
+  def test_captured_constant(self, cycles):
+    const = array_ops.zeros([100])
+    root = tracking.AutoCheckpointable()
+    root.f = def_function.function(lambda: const + 1.)
+    root.g = def_function.function(lambda: const + 2.)
+    self.assertAllClose(array_ops.ones([100]), root.f())
+    self.assertAllClose(2. * array_ops.ones([100]), root.g())
+    imported = self.cycle(root, cycles)
+    self.assertAllClose(array_ops.ones([100]), imported.f())
+    self.assertAllClose(2. * array_ops.ones([100]), imported.g())
+    # TODO(b/123408994): Use the public get_concrete_function.
+    f_concrete = imported.f._list_all_concrete_functions_for_serialization()[0]
+    g_concrete = imported.g._list_all_concrete_functions_for_serialization()[0]
+    self.assertLen(f_concrete.captured_inputs, 1)
+    self.assertLen(g_concrete.captured_inputs, 1)
+    # We should be using the same captured EagerTensor in both functions, not
+    # duplicating the constant.
+    self.assertIs(f_concrete.captured_inputs[0],
+                  g_concrete.captured_inputs[0])
+
+  def test_table(self, cycles):
+    # TODO(b/123408779): Handle generic TrackableResources and enable this test
+    self.skipTest("Need to handle generic TrackableResources")
+    vocab_path = self._make_asset("alpha\nbeta\ngamma\n")
+    initializer = lookup_ops.TextFileInitializer(
+        vocab_path,
+        key_dtype=dtypes.string,
+        key_index=lookup_ops.TextFileIndex.WHOLE_LINE,
+        value_dtype=dtypes.int64,
+        value_index=lookup_ops.TextFileIndex.LINE_NUMBER)
+    root = util.Checkpoint(table=lookup_ops.HashTable(
+        initializer, default_value=-1))
+    root.table_user = def_function.function(
+        root.table.lookup,
+        input_signature=[tensor_spec.TensorSpec(None, dtypes.string)])
+    self.assertEqual(2, root.table_user(constant_op.constant("gamma")).numpy())
+    imported = self.cycle(root, cycles)
+    self.assertEqual(
+        2, imported.table_user(constant_op.constant("gamma")).numpy())
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index 20af464edf..a9bf1a7370 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -54,6 +54,13 @@ from tensorflow.python.util import nest
 from tensorflow.python.util.tf_export import tf_export
 
 DEFAULT_SIGNATURE_ATTR = "_default_save_signature"
+_UNCOPIABLE_DTYPES = frozenset((dtypes.resource, dtypes.variant))
+
+
+# A container for an EagerTensor constant which has been copied to the exported
+# Graph.
+_CapturedConstant = collections.namedtuple(
+    "_CapturedConstant", ["eager_tensor", "graph_tensor"])
 
 
 class _SaveableView(object):
@@ -70,6 +77,7 @@ class _SaveableView(object):
     checkpointable_objects, node_ids, slot_variables = util.find_objects(root)
     self.nodes = checkpointable_objects
     self.node_ids = node_ids
+    self.captured_tensor_node_ids = util.ObjectIdentityDictionary()
     self.slot_variables = slot_variables
     self.functions = util.ObjectIdentityDictionary()
     self.concrete_functions = []
@@ -112,7 +120,8 @@ class _SaveableView(object):
       assert self.node_ids[node] == node_id
       object_proto = proto.nodes.add()
       object_proto.slot_variables.extend(self.slot_variables.get(node, ()))
-      if isinstance(node, (def_function.Function, defun.ConcreteFunction)):
+      if isinstance(node, (def_function.Function, defun.ConcreteFunction,
+                           _CapturedConstant)):
         continue
       for child in node._checkpoint_dependencies:  # pylint: disable=protected-access
         child_proto = object_proto.children.add()
@@ -141,6 +150,65 @@ class _SaveableView(object):
         functions[attribute_name] = attribute_value
     return functions
 
+  def map_resources(self):
+    """Makes new resource handle ops corresponding to existing resource tensors.
+
+    Creates resource handle ops in the current default graph, whereas
+    `accessible_objects` will be from an eager context. Resource mapping adds
+    resource handle ops to the main GraphDef of a SavedModel, which allows the
+    C++ loader API to interact with variables.
+
+    Returns:
+      A tuple of (object_map, resource_map, asset_info):
+        object_map: A dictionary mapping from object in `accessible_objects` to
+          replacement objects created to hold the new resource tensors.
+        resource_map: A dictionary mapping from resource tensors extracted from
+          `accessible_objects` to newly created resource tensors.
+        asset_info: An _AssetInfo tuple describing external assets referenced
+          from accessible_objects.
+    """
+    # Only makes sense when adding to the export Graph
+    assert not context.executing_eagerly()
+    # TODO(allenl): Handle MirroredVariables and other types of variables which
+    # may need special casing.
+    object_map = util.ObjectIdentityDictionary()
+    resource_map = {}
+    asset_info = _AssetInfo(
+        asset_defs=[],
+        asset_initializers_by_resource={},
+        asset_filename_map={},
+        asset_index={})
+    for node_id, obj in enumerate(self.nodes):
+      if isinstance(obj, tracking.TrackableResource):
+        new_resource = obj.create_resource()
+        resource_map[obj.resource_handle] = new_resource
+        self.captured_tensor_node_ids[obj.resource_handle] = node_id
+      elif resource_variable_ops.is_resource_variable(obj):
+        new_variable = resource_variable_ops.copy_to_graph_uninitialized(obj)
+        object_map[obj] = new_variable
+        resource_map[obj.handle] = new_variable.handle
+        self.captured_tensor_node_ids[obj.handle] = node_id
+      elif isinstance(obj, tracking.TrackableAsset):
+        _process_asset(obj, asset_info, resource_map)
+        self.captured_tensor_node_ids[obj.asset_path.handle] = node_id
+
+    for concrete_function in self.concrete_functions:
+      for capture in concrete_function.captured_inputs:
+        if (isinstance(capture, ops.EagerTensor)
+            and capture.dtype not in _UNCOPIABLE_DTYPES
+            and capture not in self.captured_tensor_node_ids):
+          copied_tensor = constant_op.constant(capture.numpy())
+          node_id = len(self.nodes)
+          node = _CapturedConstant(
+              eager_tensor=capture, graph_tensor=copied_tensor)
+          self.nodes.append(node)
+          self.node_ids[capture] = node_id
+          self.node_ids[node] = node_id
+          self.captured_tensor_node_ids[capture] = node_id
+          resource_map[capture] = copied_tensor
+
+    return object_map, resource_map, asset_info
+
 
 def _get_signature(function):
   if (isinstance(function, (defun.Function, def_function.Function)) and
@@ -280,10 +348,6 @@ def _map_captures_to_created_tensors(
              "assigning them to an attribute of another tracked object, or to "
              "an attribute of the main object directly.")
             .format(interior))
-      else:
-        # This is a captured Tensor, but it's not a resource. We'll just add it
-        # to the graph as a constant.
-        mapped_resource = constant_op.constant(exterior.numpy())
     export_captures.append(mapped_resource)
   return export_captures
 
@@ -476,49 +540,6 @@ def _process_asset(trackable_asset, asset_info, resource_map):
   resource_map[original_variable.handle] = asset_variable.handle
 
 
-def _map_resources(accessible_objects):
-  """Makes new resource handle ops corresponding to existing resource tensors.
-
-  Creates resource handle ops in the current default graph, whereas
-  `accessible_objects` will be from an eager context. Resource mapping adds
-  resource handle ops to the main GraphDef of a SavedModel, which allows the C++
-  loader API to interact with variables.
-
-  Args:
-    accessible_objects: A list of objects, some of which may contain resources,
-      to create replacements for.
-
-  Returns:
-    A tuple of (object_map, resource_map, asset_info):
-      object_map: A dictionary mapping from object in `accessible_objects` to
-        replacement objects created to hold the new resource tensors.
-      resource_map: A dictionary mapping from resource tensors extracted from
-        `accessible_objects` to newly created resource tensors.
-      asset_info: An _AssetInfo tuple describing external assets referenced from
-        accessible_objects.
-  """
-  # TODO(allenl): Handle MirroredVariables and other types of variables which
-  # may need special casing.
-  object_map = util.ObjectIdentityDictionary()
-  resource_map = {}
-  asset_info = _AssetInfo(
-      asset_defs=[],
-      asset_initializers_by_resource={},
-      asset_filename_map={},
-      asset_index={})
-  for obj in accessible_objects:
-    if isinstance(obj, tracking.TrackableResource):
-      new_resource = obj.create_resource()
-      resource_map[obj.resource_handle] = new_resource
-    elif resource_variable_ops.is_resource_variable(obj):
-      new_variable = resource_variable_ops.copy_to_graph_uninitialized(obj)
-      object_map[obj] = new_variable
-      resource_map[obj.handle] = new_variable.handle
-    elif isinstance(obj, tracking.TrackableAsset):
-      _process_asset(obj, asset_info, resource_map)
-  return object_map, resource_map, asset_info
-
-
 def _fill_meta_graph_def(meta_graph_def, saveable_view, signature_functions,
                          object_saver):
   """Generates a MetaGraph which calls `signature_functions`.
@@ -541,7 +562,7 @@ def _fill_meta_graph_def(meta_graph_def, saveable_view, signature_functions,
   exported_graph = ops.Graph()
   resource_initializer_ops = []
   with exported_graph.as_default():
-    object_map, resource_map, asset_info = _map_resources(accessible_objects)
+    object_map, resource_map, asset_info = saveable_view.map_resources()
     for resource_initializer_function in resource_initializer_functions:
       asset_dependencies = []
       for capture in resource_initializer_function.graph.external_captures:
@@ -577,9 +598,6 @@ def _fill_meta_graph_def(meta_graph_def, saveable_view, signature_functions,
     saver_def = saver.to_proto()
     meta_graph_def.saver_def.CopyFrom(saver_def)
   graph_def = exported_graph.as_graph_def(add_shapes=True)
-  # Clean reference cycles so repeated export()s don't make work for the garbage
-  # collector.
-  ops.dismantle_graph(exported_graph)
 
   meta_graph_def.graph_def.CopyFrom(graph_def)
   meta_graph_def.meta_info_def.tags.append(tag_constants.SERVING)
@@ -587,7 +605,7 @@ def _fill_meta_graph_def(meta_graph_def, saveable_view, signature_functions,
   for signature_key, signature in signatures.items():
     meta_graph_def.signature_def[signature_key].CopyFrom(signature)
   meta_graph.strip_graph_default_valued_attrs(meta_graph_def)
-  return asset_info
+  return asset_info, exported_graph
 
 
 def _write_object_graph(saveable_view, export_dir, asset_file_def_index):
@@ -597,18 +615,10 @@ def _write_object_graph(saveable_view, export_dir, asset_file_def_index):
   proto = saved_object_graph_pb2.SavedObjectGraph()
   saveable_view.fill_object_graph_proto(proto)
 
-  node_ids = util.ObjectIdentityDictionary()
-  for i, obj in enumerate(saveable_view.nodes):
-    node_ids[obj] = i
-    if resource_variable_ops.is_resource_variable(obj):
-      node_ids[obj.handle] = i
-    elif isinstance(obj, tracking.TrackableAsset):
-      node_ids[obj.asset_path.handle] = i
-
   coder = nested_structure_coder.StructureCoder()
   for concrete_function in saveable_view.concrete_functions:
     serialized = function_serialization.serialize_concrete_function(
-        concrete_function, node_ids, coder)
+        concrete_function, saveable_view.captured_tensor_node_ids, coder)
     if serialized is not None:
       proto.concrete_functions[concrete_function.name].CopyFrom(
           serialized)
@@ -641,6 +651,8 @@ def _write_object_proto(obj, proto, asset_file_def_index):
   elif isinstance(obj, defun.ConcreteFunction):
     proto.bare_concrete_function.CopyFrom(
         function_serialization.serialize_bare_concrete_function(obj))
+  elif isinstance(obj, _CapturedConstant):
+    proto.constant.operation = obj.graph_tensor.op.name
   else:
     registered_type_proto = revived_types.serialize(obj)
     if registered_type_proto is None:
@@ -841,7 +853,7 @@ def save(obj, export_dir, signatures=None):
   meta_graph_def = saved_model.meta_graphs.add()
   # TODO(andresp): Should this be using saveable_view?
   object_saver = util.CheckpointableSaver(obj)
-  asset_info = _fill_meta_graph_def(
+  asset_info, exported_graph = _fill_meta_graph_def(
       meta_graph_def, saveable_view, signatures, object_saver)
   saved_model.saved_model_schema_version = (
       constants.SAVED_MODEL_SCHEMA_VERSION)
@@ -857,3 +869,7 @@ def save(obj, export_dir, signatures=None):
       compat.as_bytes(constants.SAVED_MODEL_FILENAME_PB))
   file_io.write_string_to_file(path, saved_model.SerializeToString())
   _write_object_graph(saveable_view, export_dir, asset_info.asset_index)
+  # Clean reference cycles so repeated export()s don't make work for the garbage
+  # collector. Before this point we need to keep references to captured
+  # constants in the saved graph.
+  ops.dismantle_graph(exported_graph)
diff --git a/tensorflow/python/saved_model/saved_object_graph.proto b/tensorflow/python/saved_model/saved_object_graph.proto
index 1e8743f681..c57bf15efa 100644
--- a/tensorflow/python/saved_model/saved_object_graph.proto
+++ b/tensorflow/python/saved_model/saved_object_graph.proto
@@ -59,6 +59,7 @@ message SavedObject {
     SavedFunction function = 6;
     SavedVariable variable = 7;
     SavedBareConcreteFunction bare_concrete_function = 8;
+    SavedConstant constant = 9;
   }
 }
 
@@ -121,6 +122,11 @@ message SavedBareConcreteFunction {
   int64 allowed_positional_arguments = 3;
 }
 
+message SavedConstant {
+  // An Operation name for a ConstantOp in this SavedObjectGraph's MetaGraph.
+  string operation = 1;
+}
+
 // Represents a Variable that is initialized by loading the contents from the
 // SavedModel checkpoint.
 message SavedVariable {
-- 
GitLab


From 8fdf7b0e3b94154dd268acbc1db569c6c7ca3ce3 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Mon, 28 Jan 2019 18:28:27 -0800
Subject: [PATCH 1442/2345] Disable pinning functions to the CPU.

PiperOrigin-RevId: 231327139
---
 .../core/common_runtime/eager/execute.cc      |  4 ++--
 tensorflow/python/eager/function_test.py      | 19 +++++++++++++++++++
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/common_runtime/eager/execute.cc b/tensorflow/core/common_runtime/eager/execute.cc
index 9df1511bc7..423f71f36f 100644
--- a/tensorflow/core/common_runtime/eager/execute.cc
+++ b/tensorflow/core/common_runtime/eager/execute.cc
@@ -744,7 +744,7 @@ bool IsPinnableOp(const string& op_type) {
   static const gtl::FlatSet<string>* unpinnable_ops = new gtl::FlatSet<string>({
       "RandomUniform",
       "RandomUniformInt",
-      "RandomNormal",
+      "RandomStandardNormal",
       "StatelessRandomUniform",
       "StatelessRandomUniformInt",
       "StatelessRandomNormal",
@@ -764,7 +764,7 @@ bool IsPinnableOp(const string& op_type) {
 Status MaybeUpdateOpDevice(EagerOperation* op) {
   EagerContext* ctx = op->EagerContext();
   bool all_inputs_eligible_for_cpu_pinning =
-      ctx->PinSmallOpsToCPU() && IsPinnableOp(op->Name());
+      ctx->PinSmallOpsToCPU() && !op->is_function() && IsPinnableOp(op->Name());
   Device* op_device = op->Device() == nullptr ? ctx->HostCPU() : op->Device();
   for (int i = 0; i < op->Inputs().size(); ++i) {
     TensorHandle* tensor_handle = op->Inputs()[i];
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index 9c713ddec9..d337429db6 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -49,6 +49,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import clip_ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import gen_random_ops
 from tensorflow.python.ops import gen_resource_variable_ops
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import list_ops
@@ -2120,6 +2121,24 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
       self.assertIn('node assert_equal/Assert/Assert (defined at', e.message)
       self.assertNotIn('fn3', e.message)
 
+  def testFunctionIsNotPinned(self):
+    """Tests that functions aren't pinned to the CPU by the eager runtime."""
+    if not context.context().num_gpus():
+      self.skipTest('No GPUs found.')
+    seed1, seed2 = 79, 25
+    shape = constant_op.constant([4, 7])
+    dtype = dtypes.float32
+
+    @def_function.function
+    def func():
+      with ops.device('GPU:0'):
+        return gen_random_ops.random_standard_normal(
+            shape, dtype=dtype, seed=seed1, seed2=seed2)
+
+    with ops.device('GPU:0'):
+      x = func()
+      self.assertRegexpMatches(x.device, 'GPU')
+
 
 class MultiDeviceTest(test.TestCase, parameterized.TestCase):
 
-- 
GitLab


From cbc8e83f641f097011e6f2345cf63c90f354a3cc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 28 Jan 2019 18:31:15 -0800
Subject: [PATCH 1443/2345] Passes through `align_corners` in
 `resize_image_with_pad`.

The change should be backwards compatible, as the default value for the new argument matches the old behavior.

PiperOrigin-RevId: 231327410
---
 tensorflow/python/ops/image_ops_impl.py               | 11 +++++++++--
 tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt |  2 +-
 tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt |  2 +-
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index 8047743cfa..3b0f095d11 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -1100,7 +1100,8 @@ def resize_images_v2(images,
 def resize_image_with_pad(image,
                           target_height,
                           target_width,
-                          method=ResizeMethod.BILINEAR):
+                          method=ResizeMethod.BILINEAR,
+                          align_corners=False):
   """Resizes and pads an image to a target width and height.
 
   Resizes an image to a target width and height by keeping
@@ -1115,6 +1116,9 @@ def resize_image_with_pad(image,
     target_height: Target height.
     target_width: Target width.
     method: Method to use for resizing image. See `resize_images()`
+    align_corners: bool.  If True, the centers of the 4 corner pixels of the
+        input and output tensors are aligned, preserving the values at the
+        corner pixels. Defaults to `False`.
 
   Raises:
     ValueError: if `target_height` or `target_width` are zero or negative.
@@ -1180,7 +1184,10 @@ def resize_image_with_pad(image,
     p_width = max_(0, math_ops.cast(f_padding_width, dtype=dtypes.int32))
 
     # Resize first, then pad to meet requested dimensions
-    resized = resize_images(image, [resized_height, resized_width], method)
+    resized = resize_images(
+        image, [resized_height, resized_width],
+        method,
+        align_corners=align_corners)
 
     padded = pad_to_bounding_box(resized, p_height, p_width, target_height,
                                  target_width)
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt
index 15d0e099ba..fd6bbcd53a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt
@@ -194,7 +194,7 @@ tf_module {
   }
   member_method {
     name: "resize_image_with_pad"
-    argspec: "args=[\'image\', \'target_height\', \'target_width\', \'method\'], varargs=None, keywords=None, defaults=[\'0\'], "
+    argspec: "args=[\'image\', \'target_height\', \'target_width\', \'method\', \'align_corners\'], varargs=None, keywords=None, defaults=[\'0\', \'False\'], "
   }
   member_method {
     name: "resize_images"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
index 3c6ed1cfb8..a5dd619511 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
@@ -182,7 +182,7 @@ tf_module {
   }
   member_method {
     name: "resize_image_with_pad"
-    argspec: "args=[\'image\', \'target_height\', \'target_width\', \'method\'], varargs=None, keywords=None, defaults=[\'0\'], "
+    argspec: "args=[\'image\', \'target_height\', \'target_width\', \'method\', \'align_corners\'], varargs=None, keywords=None, defaults=[\'0\', \'False\'], "
   }
   member_method {
     name: "rgb_to_grayscale"
-- 
GitLab


From e7f1a44a5f24fc017c7ab6f05fa421610ed08597 Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Mon, 28 Jan 2019 18:43:01 -0800
Subject: [PATCH 1444/2345] Make sure the name of the learning phase tensor is
 independent of any outer name scope.

PiperOrigin-RevId: 231328536
---
 tensorflow/python/keras/backend.py      | 10 ++++++----
 tensorflow/python/keras/backend_test.py |  6 ++++++
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index 80921c9761..a99c48f1b5 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -215,8 +215,9 @@ def clear_session():
   _SESSION.session = None
   graph = get_graph()
   with graph.as_default():
-    phase = array_ops.placeholder_with_default(
-        False, shape=(), name='keras_learning_phase')
+    with ops.name_scope(''):
+      phase = array_ops.placeholder_with_default(
+          False, shape=(), name='keras_learning_phase')
     _GRAPH_LEARNING_PHASES = {}
     _GRAPH_LEARNING_PHASES[graph] = phase
     _GRAPH_VARIABLES.pop(graph, None)
@@ -275,8 +276,9 @@ def symbolic_learning_phase():
   graph = get_graph()
   with graph.as_default():
     if graph not in _GRAPH_LEARNING_PHASES:
-      phase = array_ops.placeholder_with_default(
-          False, shape=(), name='keras_learning_phase')
+      with ops.name_scope(''):
+        phase = array_ops.placeholder_with_default(
+            False, shape=(), name='keras_learning_phase')
       _GRAPH_LEARNING_PHASES[graph] = phase
     return _GRAPH_LEARNING_PHASES[graph]
 
diff --git a/tensorflow/python/keras/backend_test.py b/tensorflow/python/keras/backend_test.py
index f43a3055cb..263a1dc461 100644
--- a/tensorflow/python/keras/backend_test.py
+++ b/tensorflow/python/keras/backend_test.py
@@ -119,6 +119,12 @@ class BackendUtilsTest(test.TestCase):
           self.evaluate(variables.global_variables_initializer())
           sess.run(y, feed_dict={x: np.random.random((2, 3))})
 
+  def test_learning_phase_name(self):
+    with ops.name_scope('test_scope'):
+      # Test that outer name scopes do not affect the learning phase's name.
+      lp = keras.backend.symbolic_learning_phase()
+    self.assertEqual(lp.name, 'keras_learning_phase:0')
+
   def test_learning_phase_scope(self):
     initial_learning_phase = keras.backend.learning_phase()
     with keras.backend.learning_phase_scope(1):
-- 
GitLab


From a39f31a94272d2e562a33f28437095014295f9ea Mon Sep 17 00:00:00 2001
From: Frank Chen <frankchn@google.com>
Date: Mon, 28 Jan 2019 18:43:15 -0800
Subject: [PATCH 1445/2345] Making Cluster Resolvers visible by default for
 TensorFlow 2.0 transition

PiperOrigin-RevId: 231328553
---
 .../contrib/cluster_resolver/__init__.py      |  4 +-
 .../python/training/__init__.py               |  4 +-
 .../python/training/gce_cluster_resolver.py   |  7 +-
 .../contrib/distribute/python/tpu_strategy.py |  6 +-
 tensorflow/python/distribute/__init__.py      |  1 +
 .../distribute/cluster_resolver/__init__.py   | 31 +------
 .../cluster_resolver/cluster_resolver.py      | 43 +++++++---
 .../cluster_resolver/cluster_resolver_test.py | 12 ++-
 .../cluster_resolver/gce_cluster_resolver.py  | 25 ++----
 .../gce_cluster_resolver_test.py              | 26 +++---
 .../kubernetes_cluster_resolver.py            | 12 +--
 .../slurm_cluster_resolver.py                 | 12 +--
 .../tfconfig_cluster_resolver.py              |  2 +
 .../cluster_resolver/tpu_cluster_resolver.py  | 72 ++++++++--------
 .../tpu_cluster_resolver_test.py              | 83 +++++++++----------
 .../tools/api/generator/api_init_files.bzl    |  1 +
 .../tools/api/generator/api_init_files_v1.bzl |  1 +
 ...e.cluster_resolver.-cluster-resolver.pbtxt | 24 ++++++
 ...ter_resolver.-g-c-e-cluster-resolver.pbtxt | 38 +++++++++
 ...esolver.-kubernetes-cluster-resolver.pbtxt | 26 ++++++
 ...er_resolver.-simple-cluster-resolver.pbtxt | 38 +++++++++
 ...ter_resolver.-slurm-cluster-resolver.pbtxt | 30 +++++++
 ...esolver.-t-f-config-cluster-resolver.pbtxt | 38 +++++++++
 ...ter_resolver.-t-p-u-cluster-resolver.pbtxt | 34 ++++++++
 ...ute.cluster_resolver.-union-resolver.pbtxt | 38 +++++++++
 ...nsorflow.distribute.cluster_resolver.pbtxt | 35 ++++++++
 .../api/golden/v1/tensorflow.distribute.pbtxt |  4 +
 ...e.cluster_resolver.-cluster-resolver.pbtxt | 24 ++++++
 ...ter_resolver.-g-c-e-cluster-resolver.pbtxt | 38 +++++++++
 ...esolver.-kubernetes-cluster-resolver.pbtxt | 26 ++++++
 ...er_resolver.-simple-cluster-resolver.pbtxt | 38 +++++++++
 ...ter_resolver.-slurm-cluster-resolver.pbtxt | 30 +++++++
 ...esolver.-t-f-config-cluster-resolver.pbtxt | 38 +++++++++
 ...ter_resolver.-t-p-u-cluster-resolver.pbtxt | 34 ++++++++
 ...ute.cluster_resolver.-union-resolver.pbtxt | 38 +++++++++
 ...nsorflow.distribute.cluster_resolver.pbtxt | 35 ++++++++
 .../api/golden/v2/tensorflow.distribute.pbtxt |  4 +
 37 files changed, 768 insertions(+), 184 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-cluster-resolver.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-g-c-e-cluster-resolver.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-kubernetes-cluster-resolver.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-simple-cluster-resolver.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-slurm-cluster-resolver.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-t-f-config-cluster-resolver.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-t-p-u-cluster-resolver.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-union-resolver.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-cluster-resolver.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-g-c-e-cluster-resolver.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-kubernetes-cluster-resolver.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-simple-cluster-resolver.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-slurm-cluster-resolver.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-t-f-config-cluster-resolver.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-t-p-u-cluster-resolver.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-union-resolver.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.pbtxt

diff --git a/tensorflow/contrib/cluster_resolver/__init__.py b/tensorflow/contrib/cluster_resolver/__init__.py
index 390b3e7550..a4dea85efd 100644
--- a/tensorflow/contrib/cluster_resolver/__init__.py
+++ b/tensorflow/contrib/cluster_resolver/__init__.py
@@ -23,7 +23,7 @@ from __future__ import print_function
 from tensorflow.python.distribute.cluster_resolver.cluster_resolver import ClusterResolver
 from tensorflow.python.distribute.cluster_resolver.cluster_resolver import SimpleClusterResolver
 from tensorflow.python.distribute.cluster_resolver.cluster_resolver import UnionClusterResolver
-from tensorflow.python.distribute.cluster_resolver.gce_cluster_resolver import GceClusterResolver
+from tensorflow.python.distribute.cluster_resolver.gce_cluster_resolver import GCEClusterResolver
 from tensorflow.python.distribute.cluster_resolver.kubernetes_cluster_resolver import KubernetesClusterResolver
 from tensorflow.python.distribute.cluster_resolver.slurm_cluster_resolver import SlurmClusterResolver
 from tensorflow.python.distribute.cluster_resolver.tfconfig_cluster_resolver import TFConfigClusterResolver
@@ -36,7 +36,7 @@ _allowed_symbols = [
     'ClusterResolver',
     'SimpleClusterResolver',
     'UnionClusterResolver',
-    'GceClusterResolver',
+    'GCEClusterResolver',
     'KubernetesClusterResolver',
     'TFConfigClusterResolver',
     'TPUClusterResolver',
diff --git a/tensorflow/contrib/cluster_resolver/python/training/__init__.py b/tensorflow/contrib/cluster_resolver/python/training/__init__.py
index 10d93549eb..ef1e9f11a0 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/__init__.py
+++ b/tensorflow/contrib/cluster_resolver/python/training/__init__.py
@@ -25,7 +25,7 @@ from __future__ import print_function
 from tensorflow.python.distribute.cluster_resolver.cluster_resolver import ClusterResolver
 from tensorflow.python.distribute.cluster_resolver.cluster_resolver import SimpleClusterResolver
 from tensorflow.python.distribute.cluster_resolver.cluster_resolver import UnionClusterResolver
-from tensorflow.python.distribute.cluster_resolver.gce_cluster_resolver import GceClusterResolver
+from tensorflow.python.distribute.cluster_resolver.gce_cluster_resolver import GCEClusterResolver
 from tensorflow.python.distribute.cluster_resolver.kubernetes_cluster_resolver import KubernetesClusterResolver
 from tensorflow.python.distribute.cluster_resolver.slurm_cluster_resolver import SlurmClusterResolver
 from tensorflow.python.distribute.cluster_resolver.tfconfig_cluster_resolver import TFConfigClusterResolver
@@ -43,7 +43,7 @@ _allowed_symbols = [
     'ClusterResolver',
     'SimpleClusterResolver',
     'UnionClusterResolver',
-    'GceClusterResolver',
+    'GCEClusterResolver',
     'KubernetesClusterResolver',
     'TFConfigClusterResolver',
     'TPUClusterResolver',
diff --git a/tensorflow/contrib/cluster_resolver/python/training/gce_cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/gce_cluster_resolver.py
index 55e61155c6..5b49116ff6 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/gce_cluster_resolver.py
+++ b/tensorflow/contrib/cluster_resolver/python/training/gce_cluster_resolver.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Stub file for GceClusterResolver to maintain backwards compatibility."""
+"""Stub file for GCEClusterResolver to maintain backwards compatibility."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -23,13 +23,14 @@ from __future__ import print_function
 # existing OSS code will not be broken.
 
 # pylint: disable=unused-import
-from tensorflow.python.distribute.cluster_resolver.gce_cluster_resolver import GceClusterResolver
+from tensorflow.python.distribute.cluster_resolver.gce_cluster_resolver import GCEClusterResolver
 # pylint: enable=unused-import
 
 from tensorflow.python.util.all_util import remove_undocumented
 
+
 _allowed_symbols = [
-    'GceClusterResolver',
+    'GCEClusterResolver',
 ]
 
 remove_undocumented(__name__, _allowed_symbols)
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 4387210062..69ce1141d8 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -39,7 +39,7 @@ from tensorflow.python.distribute import input_lib
 from tensorflow.python.distribute import numpy_dataset
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.distribute import values
-from tensorflow.python.distribute.cluster_resolver import tpu_cluster_resolver as resolver_lib
+from tensorflow.python.distribute.cluster_resolver import TPUClusterResolver
 from tensorflow.python.eager import context
 from tensorflow.python.eager import tape
 from tensorflow.python.framework import constant_op
@@ -65,7 +65,7 @@ def initialize_tpu_system(cluster_resolver=None):
     The tf.contrib.tpu.Topology object for the topology of the TPU cluster.
   """
   if cluster_resolver is None:
-    cluster_resolver = resolver_lib.TPUClusterResolver("")
+    cluster_resolver = TPUClusterResolver("")
   master = cluster_resolver.master()
 
   logging.info("Initializing the TPU system.")
@@ -246,7 +246,7 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
     super(TPUExtended, self).__init__(container_strategy)
 
     if tpu_cluster_resolver is None:
-      tpu_cluster_resolver = resolver_lib.TPUClusterResolver("")
+      tpu_cluster_resolver = TPUClusterResolver("")
 
     if steps_per_run is None:
       # TODO(frankchn): Warn when we are being used by DS/Keras and this is
diff --git a/tensorflow/python/distribute/__init__.py b/tensorflow/python/distribute/__init__.py
index 4ff912ae10..54e539ecdd 100644
--- a/tensorflow/python/distribute/__init__.py
+++ b/tensorflow/python/distribute/__init__.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 # pylint: disable=unused-import
+from tensorflow.python.distribute import cluster_resolver
 from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.distribute import mirrored_strategy
diff --git a/tensorflow/python/distribute/cluster_resolver/__init__.py b/tensorflow/python/distribute/cluster_resolver/__init__.py
index ef87f59b7f..39ea191fb0 100644
--- a/tensorflow/python/distribute/cluster_resolver/__init__.py
+++ b/tensorflow/python/distribute/cluster_resolver/__init__.py
@@ -18,40 +18,11 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.python.distribute.cluster_resolver import cluster_resolver
-from tensorflow.python.distribute.cluster_resolver import gce_cluster_resolver
-from tensorflow.python.distribute.cluster_resolver import kubernetes_cluster_resolver
-from tensorflow.python.distribute.cluster_resolver import slurm_cluster_resolver
-from tensorflow.python.distribute.cluster_resolver import tfconfig_cluster_resolver
-from tensorflow.python.distribute.cluster_resolver import tpu_cluster_resolver
-
 from tensorflow.python.distribute.cluster_resolver.cluster_resolver import ClusterResolver
 from tensorflow.python.distribute.cluster_resolver.cluster_resolver import SimpleClusterResolver
 from tensorflow.python.distribute.cluster_resolver.cluster_resolver import UnionClusterResolver
-from tensorflow.python.distribute.cluster_resolver.gce_cluster_resolver import GceClusterResolver
+from tensorflow.python.distribute.cluster_resolver.gce_cluster_resolver import GCEClusterResolver
 from tensorflow.python.distribute.cluster_resolver.kubernetes_cluster_resolver import KubernetesClusterResolver
 from tensorflow.python.distribute.cluster_resolver.slurm_cluster_resolver import SlurmClusterResolver
 from tensorflow.python.distribute.cluster_resolver.tfconfig_cluster_resolver import TFConfigClusterResolver
 from tensorflow.python.distribute.cluster_resolver.tpu_cluster_resolver import TPUClusterResolver
-
-from tensorflow.python.util.all_util import remove_undocumented
-
-_allowed_symbols = [
-    'cluster_resolver',
-    'gce_cluster_resolver',
-    'kubernetes_cluster_resolver',
-    'slurm_cluster_resolver',
-    'tfconfig_cluster_resolver',
-    'tpu_cluster_resolver',
-    'ClusterResolver',
-    'SimpleClusterResolver',
-    'UnionClusterResolver',
-    'GceClusterResolver',
-    'KubernetesClusterResolver',
-    'TFConfigClusterResolver',
-    'TPUClusterResolver',
-    'SlurmClusterResolver',
-]
-
-remove_undocumented(__name__, _allowed_symbols)
-
diff --git a/tensorflow/python/distribute/cluster_resolver/cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/cluster_resolver.py
index 8cc7cff639..22b93f0330 100644
--- a/tensorflow/python/distribute/cluster_resolver/cluster_resolver.py
+++ b/tensorflow/python/distribute/cluster_resolver/cluster_resolver.py
@@ -20,11 +20,13 @@ from __future__ import print_function
 
 import abc
 
+import collections
 import six
 
 from tensorflow.python.client import session
 from tensorflow.python.framework import ops
 from tensorflow.python.training.server_lib import ClusterSpec
+from tensorflow.python.util.tf_export import tf_export
 
 
 def format_master_url(master, rpc_layer=None):
@@ -42,6 +44,7 @@ def get_accelerator_devices(master, config_proto):
   return devices
 
 
+@tf_export('distribute.cluster_resolver.ClusterResolver')
 @six.add_metaclass(abc.ABCMeta)
 class ClusterResolver(object):
   """Abstract class for all implementations of ClusterResolvers.
@@ -104,17 +107,14 @@ class ClusterResolver(object):
   def num_accelerators(self,
                        task_type=None,
                        task_id=None,
-                       accelerator_type='GPU',
                        config_proto=None):
     """Returns the number of accelerator cores per worker.
 
     This returns the number of accelerator cores (such as GPUs and TPUs)
-    available per worker. If workers only has CPU cores available, then this
-    should return 0. This method will query the master for this information
-    if it is not otherwise known.
+    available per worker.
 
-    Optionally, we allow callers to specify the task_type, task_id, and
-    rpc_layer, if they want to target a specific TensorFlow process to query
+    Optionally, we allow callers to specify the task_type, and task_id, for
+    if they want to target a specific TensorFlow process to query
     the number of accelerators. This is to support heterogenous environments,
     where the number of accelerators cores per host is different.
 
@@ -123,21 +123,39 @@ class ClusterResolver(object):
         want to query.
       task_id: (Optional) The index of the TensorFlow task of the machine we
         want to query.
-      accelerator_type: (Optional) The type of accelerator we are trying to
-        query (defaults to 'GPU').
       config_proto: (Optional) Configuration for starting a new session to
         query how many accelerator cores it has.
+
+    Returns:
+      A map of accelerator types to number of cores.
     """
     master = self.master(task_type, task_id)
     devices = get_accelerator_devices(master, config_proto)
-    return sum(1 for d in devices if d.device_type == accelerator_type)
+    mapping = collections.defaultdict(int)
+    for device in devices:
+      mapping[device.device_type] += 1
+    return mapping
 
-  @abc.abstractproperty
+  @property
   def environment(self):
-    """Returns the current environment which TensorFlow is running in."""
-    raise NotImplementedError()
+    """Returns the current environment which TensorFlow is running in.
+
+    There are two possible return values, "google" (when TensorFlow is running
+    in a Google-internal environment) or an empty string (when TensorFlow is
+    running elsewhere).
+
+    If you are implementing a ClusterResolver that works in both the Google
+    environment and the open-source world (for instance, a TPU ClusterResolver
+    or similar), you will have to return the appropriate string depending on the
+    environment, which you will have to detect.
+
+    Otherwise, if you are implementing a ClusterResolver that will only work
+    in open-source TensorFlow, you do not need to implement this property.
+    """
+    return ''
 
 
+@tf_export('distribute.cluster_resolver.SimpleClusterResolver')
 class SimpleClusterResolver(ClusterResolver):
   """Simple implementation of ClusterResolver that accepts a ClusterSpec."""
 
@@ -237,6 +255,7 @@ class SimpleClusterResolver(ClusterResolver):
     self._rpc_layer = rpc_layer
 
 
+@tf_export('distribute.cluster_resolver.UnionResolver')
 class UnionClusterResolver(ClusterResolver):
   """Performs a union on underlying ClusterResolvers.
 
diff --git a/tensorflow/python/distribute/cluster_resolver/cluster_resolver_test.py b/tensorflow/python/distribute/cluster_resolver/cluster_resolver_test.py
index c1eb29e2fc..019d223eb0 100644
--- a/tensorflow/python/distribute/cluster_resolver/cluster_resolver_test.py
+++ b/tensorflow/python/distribute/cluster_resolver/cluster_resolver_test.py
@@ -57,24 +57,28 @@ class BaseClusterResolverTest(test.TestCase):
     mock_list_devices.return_value = device_list
 
     resolver = MockBaseClusterResolver()
-    self.assertEqual(resolver.num_accelerators(), 4)
+    self.assertEqual(resolver.num_accelerators(), {"GPU": 4})
 
   @mock.patch.object(session.BaseSession, "list_devices")
-  def testNumAcceleratorsFilterSuccess(self, mock_list_devices):
+  def testNumAcceleratorsMultiDeviceSuccess(self, mock_list_devices):
     device_names = [
         "/job:worker/task:0/device:TPU:0",
         "/job:worker/task:0/device:TPU:1",
         "/job:worker/task:0/device:TPU:2",
         "/job:worker/task:0/device:TPU:3",
+        "/job:worker/task:0/device:GPU:0",
+        "/job:worker/task:0/device:GPU:1",
+        "/job:worker/task:0/device:GPU:2",
+        "/job:worker/task:0/device:GPU:3",
     ]
     device_list = [
         session._DeviceAttributes(
-            name, "TPU", 1024, 0) for name in device_names
+            name, name[26:29], 1024, 0) for name in device_names
     ]
     mock_list_devices.return_value = device_list
 
     resolver = MockBaseClusterResolver()
-    self.assertEqual(resolver.num_accelerators(), 0)
+    self.assertEqual(resolver.num_accelerators(), {"TPU": 4, "GPU": 4})
 
 
 class UnionClusterResolverTest(test.TestCase):
diff --git a/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver.py
index 305c53870d..9d7dfdd1ea 100644
--- a/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver.py
+++ b/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver.py
@@ -20,6 +20,8 @@ from __future__ import print_function
 
 from tensorflow.python.distribute.cluster_resolver.cluster_resolver import ClusterResolver
 from tensorflow.python.training.server_lib import ClusterSpec
+from tensorflow.python.util.tf_export import tf_export
+
 
 _GOOGLE_API_CLIENT_INSTALLED = True
 try:
@@ -29,11 +31,8 @@ except ImportError:
   _GOOGLE_API_CLIENT_INSTALLED = False
 
 
-def _format_master_url(master, rpc_layer=None):
-  return '%s://%s' % (rpc_layer, master) if rpc_layer else master
-
-
-class GceClusterResolver(ClusterResolver):
+@tf_export('distribute.cluster_resolver.GCEClusterResolver')
+class GCEClusterResolver(ClusterResolver):
   """Cluster Resolver for Google Compute Engine.
 
   This is an implementation of cluster resolvers for the Google Compute Engine
@@ -53,9 +52,9 @@ class GceClusterResolver(ClusterResolver):
                rpc_layer='grpc',
                credentials='default',
                service=None):
-    """Creates a new GceClusterResolver object.
+    """Creates a new GCEClusterResolver object.
 
-    This takes in a few parameters and creates a GceClusterResolver project. It
+    This takes in a few parameters and creates a GCEClusterResolver project. It
     will then use these parameters to query the GCE API for the IP addresses of
     each instance in the instance group.
 
@@ -173,23 +172,13 @@ class GceClusterResolver(ClusterResolver):
   @task_type.setter
   def task_type(self, task_type):
     raise RuntimeError(
-        'You cannot reset the task_type of the GceClusterResolver after it has '
+        'You cannot reset the task_type of the GCEClusterResolver after it has '
         'been created.')
 
   @task_id.setter
   def task_id(self, task_id):
     self._task_id = task_id
 
-  @property
-  def environment(self):
-    """Returns the current environment which TensorFlow is running in.
-
-    For users in the GCE environment, the environment property is always an
-    empty string, and Google users will not use this ClusterResolver for running
-    on internal systems.
-    """
-    return ''
-
   @property
   def rpc_layer(self):
     return self._rpc_layer
diff --git a/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver_test.py b/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver_test.py
index 07b9eeb08e..47d1cdc0da 100644
--- a/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver_test.py
+++ b/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver_test.py
@@ -12,13 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for GceClusterResolver."""
+"""Tests for GCEClusterResolver."""
 
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.python.distribute.cluster_resolver import GceClusterResolver
+from tensorflow.python.distribute.cluster_resolver import GCEClusterResolver
 from tensorflow.python.distribute.cluster_resolver import UnionClusterResolver
 from tensorflow.python.platform import test
 from tensorflow.python.training import server_lib
@@ -27,7 +27,7 @@ from tensorflow.python.training import server_lib
 mock = test.mock
 
 
-class GceClusterResolverTest(test.TestCase):
+class GCEClusterResolverTest(test.TestCase):
 
   def _verifyClusterSpecEquality(self, cluster_spec, expected_proto):
     self.assertProtoEquals(expected_proto, cluster_spec.as_cluster_def())
@@ -121,7 +121,7 @@ class GceClusterResolverTest(test.TestCase):
     return self.standard_mock_service_client(mock_instance_group, mock_instance)
 
   def testSimpleSuccessfulRetrieval(self):
-    gce_cluster_resolver = GceClusterResolver(
+    gce_cluster_resolver = GCEClusterResolver(
         project='test-project',
         zone='us-east1-d',
         instance_group='test-instance-group',
@@ -136,7 +136,7 @@ class GceClusterResolverTest(test.TestCase):
     self._verifyClusterSpecEquality(actual_cluster_spec, expected_proto)
 
   def testMasterRetrieval(self):
-    gce_cluster_resolver = GceClusterResolver(
+    gce_cluster_resolver = GCEClusterResolver(
         project='test-project',
         zone='us-east1-d',
         instance_group='test-instance-group',
@@ -153,7 +153,7 @@ class GceClusterResolverTest(test.TestCase):
         {'name': 'instance3', 'ip': '10.3.4.5'},
     ]
 
-    gce_cluster_resolver = GceClusterResolver(
+    gce_cluster_resolver = GCEClusterResolver(
         project='test-project',
         zone='us-east1-d',
         instance_group='test-instance-group',
@@ -172,7 +172,7 @@ class GceClusterResolverTest(test.TestCase):
         {'name': 'instance3', 'ip': '10.3.4.5'},
     ]
 
-    gce_cluster_resolver = GceClusterResolver(
+    gce_cluster_resolver = GCEClusterResolver(
         project='test-project',
         zone='us-east1-d',
         instance_group='test-instance-group',
@@ -196,7 +196,7 @@ class GceClusterResolverTest(test.TestCase):
         {'name': 'instance3', 'ip': '10.3.4.5'},
     ]
 
-    gce_cluster_resolver = GceClusterResolver(
+    gce_cluster_resolver = GCEClusterResolver(
         project='test-project',
         zone='us-east1-d',
         instance_group='test-instance-group',
@@ -210,7 +210,7 @@ class GceClusterResolverTest(test.TestCase):
         task_type='', task_id=0), 'grpc://10.1.2.3:8470')
 
   def testCustomJobNameAndPortRetrieval(self):
-    gce_cluster_resolver = GceClusterResolver(
+    gce_cluster_resolver = GCEClusterResolver(
         project='test-project',
         zone='us-east1-d',
         instance_group='test-instance-group',
@@ -232,7 +232,7 @@ class GceClusterResolverTest(test.TestCase):
         {'name': 'instance3', 'ip': '10.3.4.5'},
     ]
 
-    gce_cluster_resolver = GceClusterResolver(
+    gce_cluster_resolver = GCEClusterResolver(
         project='test-project',
         zone='us-east1-d',
         instance_group='test-instance-group',
@@ -266,7 +266,7 @@ class GceClusterResolverTest(test.TestCase):
         {'name': 'ps2', 'ip': '10.100.2.3'},
     ]
 
-    worker1_gce_cluster_resolver = GceClusterResolver(
+    worker1_gce_cluster_resolver = GCEClusterResolver(
         project='test-project',
         zone='us-east1-d',
         instance_group='test-instance-group',
@@ -275,7 +275,7 @@ class GceClusterResolverTest(test.TestCase):
         credentials=None,
         service=self.gen_standard_mock_service_client(worker1_name_to_ip))
 
-    worker2_gce_cluster_resolver = GceClusterResolver(
+    worker2_gce_cluster_resolver = GCEClusterResolver(
         project='test-project',
         zone='us-east1-d',
         instance_group='test-instance-group',
@@ -284,7 +284,7 @@ class GceClusterResolverTest(test.TestCase):
         credentials=None,
         service=self.gen_standard_mock_service_client(worker2_name_to_ip))
 
-    ps_gce_cluster_resolver = GceClusterResolver(
+    ps_gce_cluster_resolver = GCEClusterResolver(
         project='test-project',
         zone='us-east1-d',
         instance_group='test-instance-group',
diff --git a/tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver.py
index 71d48ed2b9..28b2712590 100644
--- a/tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver.py
+++ b/tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 from tensorflow.python.distribute.cluster_resolver.cluster_resolver import ClusterResolver
 from tensorflow.python.distribute.cluster_resolver.cluster_resolver import format_master_url
 from tensorflow.python.training import server_lib
+from tensorflow.python.util.tf_export import tf_export
 
 _KUBERNETES_API_CLIENT_INSTALLED = True
 try:
@@ -30,6 +31,7 @@ except ImportError:
   _KUBERNETES_API_CLIENT_INSTALLED = False
 
 
+@tf_export('distribute.cluster_resolver.KubernetesClusterResolver')
 class KubernetesClusterResolver(ClusterResolver):
   """Cluster Resolver for Kubernetes.
 
@@ -154,13 +156,3 @@ class KubernetesClusterResolver(ClusterResolver):
       cluster_map[tf_job] = all_pods
 
     return server_lib.ClusterSpec(cluster_map)
-
-  @property
-  def environment(self):
-    """Returns the current environment which TensorFlow is running in.
-
-    For users in the Cloud environment, the environment property is always an
-    empty string, and Google users will not use this ClusterResolver for running
-    on internal systems.
-    """
-    return ''
diff --git a/tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver.py
index 0ec566c670..04675f4d17 100644
--- a/tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver.py
+++ b/tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver.py
@@ -25,8 +25,10 @@ import subprocess
 from tensorflow.python.distribute.cluster_resolver.cluster_resolver import ClusterResolver
 from tensorflow.python.distribute.cluster_resolver.cluster_resolver import format_master_url
 from tensorflow.python.training.server_lib import ClusterSpec
+from tensorflow.python.util.tf_export import tf_export
 
 
+@tf_export('distribute.cluster_resolver.SlurmClusterResolver')
 class SlurmClusterResolver(ClusterResolver):
   """Cluster Resolver for system with Slurm workload manager.
 
@@ -215,16 +217,6 @@ class SlurmClusterResolver(ClusterResolver):
 
     return ''
 
-  @property
-  def environment(self):
-    """Returns the current environment which TensorFlow is running in.
-
-    For users in the Slurm environment, the environment property is always an
-    empty string, and Google users will not use this ClusterResolver for running
-    on internal systems.
-    """
-    return ''
-
   def num_accelerators(self,
                        task_type=None,
                        task_id=None,
diff --git a/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver.py
index 6dcbafbc50..48bfc60651 100644
--- a/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver.py
+++ b/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver.py
@@ -24,6 +24,7 @@ import os
 
 from tensorflow.python.distribute.cluster_resolver.cluster_resolver import ClusterResolver
 from tensorflow.python.training.server_lib import ClusterSpec
+from tensorflow.python.util.tf_export import tf_export
 
 _TF_CONFIG_ENV = 'TF_CONFIG'
 _SESSION_MASTER_KEY = 'session_master'
@@ -47,6 +48,7 @@ def _get_value_in_tfconfig(key, default=None):
   return tf_config[key] if key in tf_config else default
 
 
+@tf_export('distribute.cluster_resolver.TFConfigClusterResolver')
 class TFConfigClusterResolver(ClusterResolver):
   """Implementation of a ClusterResolver which reads the TF_CONFIG EnvVar."""
 
diff --git a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
index d02e9b8fe8..b8d2ecc3e2 100644
--- a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
+++ b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
@@ -34,6 +34,7 @@ from tensorflow.python.framework import errors
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import server_lib
 from tensorflow.python.util import compat
+from tensorflow.python.util.tf_export import tf_export
 
 _GOOGLE_API_CLIENT_INSTALLED = True
 try:
@@ -42,7 +43,6 @@ try:
 except ImportError:
   _GOOGLE_API_CLIENT_INSTALLED = False
 
-
 _GKE_ENV_VARIABLE = 'KUBE_GOOGLE_CLOUD_TPU_ENDPOINTS'
 _ENDPOINTS_SEPARATOR = ','
 _DEFAULT_ENV_VARIABLE = 'TPU_NAME'
@@ -56,38 +56,7 @@ DeviceDetails = collections.namedtuple(
     'DeviceDetails', ['device_map', 'total_cores'])
 
 
-def _get_device_dict_and_cores(devices):
-  """Returns a dict of hosts to cores and total cores given devices names.
-
-  Returns a namedtuple with two attributes:
-    device_map: A map of host_ids to a list of core_ids.
-    total_cores: The total number of cores within the TPU system.
-
-  Args:
-    devices: A list of devices returned by session.list_devices()
-  """
-  device_map = collections.defaultdict(list)
-  num_cores = 0
-  for device in devices:
-    match = _TPU_DEVICE_REGEX.match(device.name)
-    if match:
-      host_id = match.group('host_id')
-      core_id = match.group('core_id')
-      device_map[host_id].append(core_id)
-      num_cores += 1
-  return DeviceDetails(device_map, num_cores)
-
-
-def _verify_and_return_same_core_count(device_dict):
-  """Verifies that every device in device_dict has the same number of cores."""
-  num_cores_per_host_set = (
-      {len(core_ids) for core_ids in device_dict.values()})
-  if len(num_cores_per_host_set) != 1:
-    raise RuntimeError('TPU cores on each device is not the same. This '
-                       'should never happen. Devices: {}'.format(device_dict))
-  return num_cores_per_host_set.pop()
-
-
+@tf_export('distribute.cluster_resolver.TPUClusterResolver')
 class TPUClusterResolver(ClusterResolver):
   """Cluster Resolver for Google Cloud TPUs.
 
@@ -143,6 +112,38 @@ class TPUClusterResolver(ClusterResolver):
       return False
     return True
 
+  @staticmethod
+  def _get_device_dict_and_cores(devices):
+    """Returns a dict of hosts to cores and total cores given devices names.
+
+    Returns a namedtuple with two attributes:
+      device_map: A map of host_ids to a list of core_ids.
+      total_cores: The total number of cores within the TPU system.
+
+    Args:
+      devices: A list of devices returned by session.list_devices()
+    """
+    device_map = collections.defaultdict(list)
+    num_cores = 0
+    for device in devices:
+      match = _TPU_DEVICE_REGEX.match(device.name)
+      if match:
+        host_id = match.group('host_id')
+        core_id = match.group('core_id')
+        device_map[host_id].append(core_id)
+        num_cores += 1
+    return DeviceDetails(device_map, num_cores)
+
+  @staticmethod
+  def _verify_and_return_same_core_count(device_dict):
+    """Verifies that every device in device_dict has the same # of cores."""
+    num_cores_per_host_set = (
+        {len(core_ids) for core_ids in device_dict.values()})
+    if len(num_cores_per_host_set) != 1:
+      raise RuntimeError('TPU cores on each device is not the same. This '
+                         'should never happen. Devices: {}'.format(device_dict))
+    return num_cores_per_host_set.pop()
+
   @staticmethod
   def _inGke():
     """When running in GKE, the environment variable will be set."""
@@ -482,7 +483,7 @@ class TPUClusterResolver(ClusterResolver):
     # TODO(b/120564445): Replace with standard library for retries.
     while True:
       try:
-        device_details = _get_device_dict_and_cores(
+        device_details = TPUClusterResolver._get_device_dict_and_cores(
             get_accelerator_devices(self.master(), config_proto=config_proto))
         break
       except errors.DeadlineExceededError:
@@ -497,7 +498,8 @@ class TPUClusterResolver(ClusterResolver):
           raise RuntimeError(error_message)
 
     if device_details.total_cores:
-      return _verify_and_return_same_core_count(device_details.device_map)
+      return TPUClusterResolver._verify_and_return_same_core_count(
+          device_details.device_map)
     return 0
 
   @property
diff --git a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver_test.py b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver_test.py
index 0cb0dc607f..7f06dc168b 100644
--- a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver_test.py
+++ b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver_test.py
@@ -24,8 +24,7 @@ import six
 from six.moves.urllib.error import URLError
 
 from tensorflow.python.client import session
-from tensorflow.python.distribute import cluster_resolver
-from tensorflow.python.distribute.cluster_resolver import tpu_cluster_resolver
+from tensorflow.python.distribute.cluster_resolver import TPUClusterResolver
 from tensorflow.python.framework import errors
 from tensorflow.python.platform import test
 from tensorflow.python.training import server_lib
@@ -129,26 +128,26 @@ class TPUClusterResolverTest(test.TestCase):
 
     return mock_client
 
-  @mock.patch.object(cluster_resolver.TPUClusterResolver,
+  @mock.patch.object(TPUClusterResolver,
                      '_isRunningInGCE',
                      mock_is_running_in_gce)
   def testCheckRunningInGceWithNoTpuName(self):
     with self.assertRaisesRegexp(RuntimeError, '.*Google Cloud.*'):
-      cluster_resolver.TPUClusterResolver(tpu='')
+      TPUClusterResolver(tpu='')
 
   @mock.patch.object(six.moves.urllib.request,
                      'urlopen',
                      mock_running_in_gce_urlopen)
   def testIsRunningInGce(self):
-    self.assertTrue(cluster_resolver.TPUClusterResolver._isRunningInGCE())
+    self.assertTrue(TPUClusterResolver._isRunningInGCE())
 
   @mock.patch.object(six.moves.urllib.request,
                      'urlopen',
                      mock_not_running_in_gce_urlopen)
   def testIsNotRunningInGce(self):
-    self.assertFalse(cluster_resolver.TPUClusterResolver._isRunningInGCE())
+    self.assertFalse(TPUClusterResolver._isRunningInGCE())
 
-  @mock.patch.object(cluster_resolver.TPUClusterResolver,
+  @mock.patch.object(TPUClusterResolver,
                      '_requestComputeMetadata',
                      mock_request_compute_metadata)
   def testRetrieveProjectAndZoneFromMetadata(self):
@@ -160,7 +159,7 @@ class TPUClusterResolverTest(test.TestCase):
         }
     }
 
-    resolver = cluster_resolver.TPUClusterResolver(
+    resolver = TPUClusterResolver(
         project=None,
         zone=None,
         tpu=['test-tpu-1'],
@@ -182,7 +181,7 @@ class TPUClusterResolverTest(test.TestCase):
     self._verifyClusterSpecEquality(actual_cluster_spec, str(expected_proto))
     self.assertEqual(resolver.master(), 'grpc://10.1.2.3:8470')
 
-  @mock.patch.object(cluster_resolver.TPUClusterResolver,
+  @mock.patch.object(TPUClusterResolver,
                      '_requestComputeMetadata',
                      mock_request_compute_metadata)
   def testRetrieveProjectAndZoneFromMetadataNoCoordinator(self):
@@ -194,7 +193,7 @@ class TPUClusterResolverTest(test.TestCase):
         }
     }
 
-    resolver = cluster_resolver.TPUClusterResolver(
+    resolver = TPUClusterResolver(
         project=None,
         zone=None,
         tpu=['test-tpu-1'],
@@ -209,7 +208,7 @@ class TPUClusterResolverTest(test.TestCase):
     self._verifyClusterSpecEquality(actual_cluster_spec, expected_proto)
     self.assertEqual(resolver.master(), 'grpc://10.1.2.3:8470')
 
-  @mock.patch.object(cluster_resolver.TPUClusterResolver,
+  @mock.patch.object(TPUClusterResolver,
                      '_requestComputeMetadata',
                      mock_request_compute_metadata)
   def testUnhealthyCloudTpu(self):
@@ -221,7 +220,7 @@ class TPUClusterResolverTest(test.TestCase):
         }
     }
 
-    resolver = cluster_resolver.TPUClusterResolver(
+    resolver = TPUClusterResolver(
         project=None,
         zone=None,
         tpu='test-tpu-1',
@@ -232,7 +231,7 @@ class TPUClusterResolverTest(test.TestCase):
     with self.assertRaises(RuntimeError):
       resolver.cluster_spec()
 
-  @mock.patch.object(cluster_resolver.TPUClusterResolver,
+  @mock.patch.object(TPUClusterResolver,
                      '_requestComputeMetadata',
                      mock_request_compute_metadata)
   def testNotReadyCloudTpu(self):
@@ -244,7 +243,7 @@ class TPUClusterResolverTest(test.TestCase):
         }
     }
 
-    resolver = cluster_resolver.TPUClusterResolver(
+    resolver = TPUClusterResolver(
         project=None,
         zone=None,
         tpu='test-tpu-1',
@@ -264,7 +263,7 @@ class TPUClusterResolverTest(test.TestCase):
         }
     }
 
-    resolver = cluster_resolver.TPUClusterResolver(
+    resolver = TPUClusterResolver(
         project='test-project',
         zone='us-central1-c',
         tpu=['test-tpu-1'],
@@ -292,7 +291,7 @@ class TPUClusterResolverTest(test.TestCase):
         }
     }
 
-    resolver = cluster_resolver.TPUClusterResolver(
+    resolver = TPUClusterResolver(
         project='test-project',
         zone='us-central1-c',
         tpu='test-tpu-1',
@@ -309,7 +308,7 @@ class TPUClusterResolverTest(test.TestCase):
     self._verifyClusterSpecEquality(actual_cluster_spec, expected_proto)
     self.assertEqual('grpc://10.2.3.4:8470', resolver.master())
 
-  @mock.patch.object(cluster_resolver.TPUClusterResolver,
+  @mock.patch.object(TPUClusterResolver,
                      '_requestComputeMetadata',
                      mock_request_compute_metadata)
   def testPodResolution(self):
@@ -338,7 +337,7 @@ class TPUClusterResolverTest(test.TestCase):
         }
     }
 
-    resolver = cluster_resolver.TPUClusterResolver(
+    resolver = TPUClusterResolver(
         tpu='test-tpu-1',
         credentials=None,
         service=self.mock_service_client(tpu_map=tpu_map),
@@ -387,7 +386,7 @@ class TPUClusterResolverTest(test.TestCase):
         }
     }
 
-    resolver = cluster_resolver.TPUClusterResolver(
+    resolver = TPUClusterResolver(
         project='test-project',
         zone='us-central1-c',
         tpu='test-tpu-1',
@@ -412,7 +411,7 @@ class TPUClusterResolverTest(test.TestCase):
     tpu_map = {}
 
     with self.assertRaises(ValueError):
-      cluster_resolver.TPUClusterResolver(
+      TPUClusterResolver(
           project='test-project',
           zone='us-central1-c',
           tpu=[],
@@ -422,7 +421,7 @@ class TPUClusterResolverTest(test.TestCase):
 
   # TODO(saeta): Convert to parameterized test when included in OSS TF.
   def verifyShouldResolve(self, tpu, should_resolve):
-    resolver = cluster_resolver.TPUClusterResolver(
+    resolver = TPUClusterResolver(
         project='test-project',
         zone='us-central1-c',
         tpu=tpu,
@@ -432,7 +431,7 @@ class TPUClusterResolverTest(test.TestCase):
     self.assertEqual(should_resolve, resolver._shouldResolve(),
                      "TPU: '%s'" % tpu)
 
-  @mock.patch.object(cluster_resolver.TPUClusterResolver,
+  @mock.patch.object(TPUClusterResolver,
                      '_isRunningInGCE',
                      mock_is_not_running_in_gce)
   def testShouldResolveNoName(self):
@@ -457,7 +456,7 @@ class TPUClusterResolverTest(test.TestCase):
     self.verifyShouldResolve('grpctpu', True)
 
   def testNoCallComputeMetadata(self):
-    resolver = cluster_resolver.TPUClusterResolver(
+    resolver = TPUClusterResolver(
         tpu='/bns/foo/bar')
     self.assertEqual('/bns/foo/bar', resolver.master())
     self.assertEqual(None, resolver.cluster_spec())
@@ -466,12 +465,12 @@ class TPUClusterResolverTest(test.TestCase):
     os.environ['KUBE_GOOGLE_CLOUD_TPU_ENDPOINTS'] = 'grpc://10.120.27.5:8470'
 
     self.assertIn('KUBE_GOOGLE_CLOUD_TPU_ENDPOINTS', os.environ)
-    self.assertTrue(cluster_resolver.TPUClusterResolver._inGke())
+    self.assertTrue(TPUClusterResolver._inGke())
     self.assertEqual(
         compat.as_bytes('grpc://10.120.27.5:8470'),
-        compat.as_bytes(cluster_resolver.TPUClusterResolver._gkeEndpoints()))
+        compat.as_bytes(TPUClusterResolver._gkeEndpoints()))
 
-    resolver = cluster_resolver.TPUClusterResolver()
+    resolver = TPUClusterResolver()
     self.assertEqual(
         compat.as_bytes('grpc://10.120.27.5:8470'),
         compat.as_bytes(resolver.master()))
@@ -493,15 +492,15 @@ class TPUClusterResolverTest(test.TestCase):
                                                      'grpc://10.120.27.8:8470')
 
     self.assertIn('KUBE_GOOGLE_CLOUD_TPU_ENDPOINTS', os.environ)
-    self.assertTrue(cluster_resolver.TPUClusterResolver._inGke())
+    self.assertTrue(TPUClusterResolver._inGke())
     self.assertEqual(
         compat.as_bytes('grpc://10.120.27.5:8470,'
                         'grpc://10.120.27.6:8470,'
                         'grpc://10.120.27.7:8470,'
                         'grpc://10.120.27.8:8470'),
-        compat.as_bytes(cluster_resolver.TPUClusterResolver._gkeEndpoints()))
+        compat.as_bytes(TPUClusterResolver._gkeEndpoints()))
 
-    resolver = cluster_resolver.TPUClusterResolver()
+    resolver = TPUClusterResolver()
     self.assertEqual(
         compat.as_bytes('grpc://10.120.27.5:8470'),
         compat.as_bytes(resolver.master()))
@@ -522,17 +521,17 @@ class TPUClusterResolverTest(test.TestCase):
   def testEnvironmentDiscoveryUrl(self):
     os.environ['TPU_API_DISCOVERY_URL'] = 'https://{api}.internal/{apiVersion}'
     self.assertEqual('https://{api}.internal/{apiVersion}',
-                     (cluster_resolver.TPUClusterResolver.
+                     (TPUClusterResolver.
                       _environmentDiscoveryUrl()))
 
   def testEnvironmentAndRpcDetectionForGoogle(self):
-    resolver = cluster_resolver.TPUClusterResolver(
+    resolver = TPUClusterResolver(
         tpu='/bns/ab/cd/ef')
     self.assertEqual(resolver.environment, 'google')
     self.assertEqual(resolver.rpc_layer, None)
 
   def testEnvironmentAndRpcDetectionForGrpcString(self):
-    resolver = cluster_resolver.TPUClusterResolver(
+    resolver = TPUClusterResolver(
         tpu='grpc://10.1.2.3:8470')
     self.assertEqual(resolver.environment, '')
     self.assertEqual(resolver.rpc_layer, 'grpc')
@@ -564,7 +563,7 @@ class TPUClusterResolverTest(test.TestCase):
         }
     }
 
-    resolver = cluster_resolver.TPUClusterResolver(
+    resolver = TPUClusterResolver(
         project='test-project',
         zone='us-central1-c',
         tpu='test-tpu-1',
@@ -599,7 +598,7 @@ class TPUClusterResolverTest(test.TestCase):
             name, 'TPU', 1024, 0) for name in device_names
     ]
 
-    device_details = tpu_cluster_resolver._get_device_dict_and_cores(
+    device_details = TPUClusterResolver._get_device_dict_and_cores(
         device_list)
     self.assertEqual(device_details.total_cores, 8)
     self.assertEqual(device_details.device_map,
@@ -624,24 +623,24 @@ class TPUClusterResolverTest(test.TestCase):
             name, 'XLA', 1024, 0) for name in device_names
     ]
 
-    device_dict, num_cores = tpu_cluster_resolver._get_device_dict_and_cores(
+    device_dict, num_cores = TPUClusterResolver._get_device_dict_and_cores(
         device_list)
     self.assertEqual(num_cores, 0)
     self.assertEqual(device_dict, {})
 
   def testVerifySameCoreCount(self):
     self.assertEqual(
-        tpu_cluster_resolver._verify_and_return_same_core_count(
+        TPUClusterResolver._verify_and_return_same_core_count(
             {0: [0, 1, 2, 3, 4, 5, 6, 7]}), 8)
     self.assertEqual(
-        tpu_cluster_resolver._verify_and_return_same_core_count(
+        TPUClusterResolver._verify_and_return_same_core_count(
             {0: [0, 1], 1: [2, 3]}), 2)
     with self.assertRaises(RuntimeError):
-      tpu_cluster_resolver._verify_and_return_same_core_count(
+      TPUClusterResolver._verify_and_return_same_core_count(
           {0: [0], 1: [1, 2]})
 
   @mock.patch.object(session.BaseSession, 'list_devices')
-  @mock.patch.object(cluster_resolver.TPUClusterResolver,
+  @mock.patch.object(TPUClusterResolver,
                      '_isRunningInGCE',
                      mock_is_not_running_in_gce)
   def testNumAcceleratorsSuccess(self, mock_list_devices):
@@ -661,15 +660,15 @@ class TPUClusterResolverTest(test.TestCase):
     ]
     mock_list_devices.return_value = device_list
 
-    resolver = cluster_resolver.TPUClusterResolver(tpu='')
+    resolver = TPUClusterResolver(tpu='')
     self.assertEqual(resolver.num_accelerators(), 2)
 
   @mock.patch.object(session.BaseSession, 'list_devices')
-  @mock.patch.object(cluster_resolver.TPUClusterResolver,
+  @mock.patch.object(TPUClusterResolver,
                      '_isRunningInGCE',
                      mock_is_not_running_in_gce)
   def testNumAcceleratorsRetryFailure(self, mock_list_devices):
-    resolver = cluster_resolver.TPUClusterResolver(tpu='')
+    resolver = TPUClusterResolver(tpu='')
     mock_list_devices.side_effect = errors.DeadlineExceededError(
         None, None, 'timeout')
     with self.assertRaises(RuntimeError):
diff --git a/tensorflow/python/tools/api/generator/api_init_files.bzl b/tensorflow/python/tools/api/generator/api_init_files.bzl
index 7637cbd584..2bac83c423 100644
--- a/tensorflow/python/tools/api/generator/api_init_files.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files.bzl
@@ -14,6 +14,7 @@ TENSORFLOW_API_INIT_FILES = [
     "data/experimental/__init__.py",
     "debugging/__init__.py",
     "distribute/__init__.py",
+    "distribute/cluster_resolver/__init__.py",
     "dtypes/__init__.py",
     "errors/__init__.py",
     "experimental/__init__.py",
diff --git a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
index 5db84b1d0b..21e5951010 100644
--- a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
@@ -15,6 +15,7 @@ TENSORFLOW_API_INIT_FILES_V1 = [
     "data/experimental/__init__.py",
     "debugging/__init__.py",
     "distribute/__init__.py",
+    "distribute/cluster_resolver/__init__.py",
     "distributions/__init__.py",
     "dtypes/__init__.py",
     "errors/__init__.py",
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-cluster-resolver.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-cluster-resolver.pbtxt
new file mode 100644
index 0000000000..0b35b61b4c
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-cluster-resolver.pbtxt
@@ -0,0 +1,24 @@
+path: "tensorflow.distribute.cluster_resolver.ClusterResolver"
+tf_class {
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.cluster_resolver.ClusterResolver\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "environment"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+  }
+  member_method {
+    name: "cluster_spec"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "master"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'rpc_layer\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "num_accelerators"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'config_proto\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-g-c-e-cluster-resolver.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-g-c-e-cluster-resolver.pbtxt
new file mode 100644
index 0000000000..5c2cc522f1
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-g-c-e-cluster-resolver.pbtxt
@@ -0,0 +1,38 @@
+path: "tensorflow.distribute.cluster_resolver.GCEClusterResolver"
+tf_class {
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.gce_cluster_resolver.GCEClusterResolver\'>"
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.cluster_resolver.ClusterResolver\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "environment"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "rpc_layer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "task_id"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "task_type"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'project\', \'zone\', \'instance_group\', \'port\', \'task_type\', \'task_id\', \'rpc_layer\', \'credentials\', \'service\'], varargs=None, keywords=None, defaults=[\'worker\', \'0\', \'grpc\', \'default\', \'None\'], "
+  }
+  member_method {
+    name: "cluster_spec"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "master"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'rpc_layer\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "num_accelerators"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'config_proto\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-kubernetes-cluster-resolver.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-kubernetes-cluster-resolver.pbtxt
new file mode 100644
index 0000000000..3220d68e05
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-kubernetes-cluster-resolver.pbtxt
@@ -0,0 +1,26 @@
+path: "tensorflow.distribute.cluster_resolver.KubernetesClusterResolver"
+tf_class {
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.kubernetes_cluster_resolver.KubernetesClusterResolver\'>"
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.cluster_resolver.ClusterResolver\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "environment"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'job_to_label_mapping\', \'tf_server_port\', \'rpc_layer\', \'override_client\'], varargs=None, keywords=None, defaults=[\'None\', \'8470\', \'grpc\', \'None\'], "
+  }
+  member_method {
+    name: "cluster_spec"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "master"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'rpc_layer\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "num_accelerators"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'config_proto\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-simple-cluster-resolver.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-simple-cluster-resolver.pbtxt
new file mode 100644
index 0000000000..4e80e3af30
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-simple-cluster-resolver.pbtxt
@@ -0,0 +1,38 @@
+path: "tensorflow.distribute.cluster_resolver.SimpleClusterResolver"
+tf_class {
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.cluster_resolver.SimpleClusterResolver\'>"
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.cluster_resolver.ClusterResolver\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "environment"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "rpc_layer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "task_id"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "task_type"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'cluster_spec\', \'master\', \'task_type\', \'task_id\', \'environment\', \'num_accelerators\', \'rpc_layer\'], varargs=None, keywords=None, defaults=[\'\', \'None\', \'None\', \'\', \'0\', \'None\'], "
+  }
+  member_method {
+    name: "cluster_spec"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "master"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'rpc_layer\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "num_accelerators"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'accelerator_type\', \'config_proto\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'GPU\', \'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-slurm-cluster-resolver.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-slurm-cluster-resolver.pbtxt
new file mode 100644
index 0000000000..971ea3dca4
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-slurm-cluster-resolver.pbtxt
@@ -0,0 +1,30 @@
+path: "tensorflow.distribute.cluster_resolver.SlurmClusterResolver"
+tf_class {
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.slurm_cluster_resolver.SlurmClusterResolver\'>"
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.cluster_resolver.ClusterResolver\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "environment"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'jobs\', \'port_base\', \'gpus_per_node\', \'gpus_per_task\', \'tasks_per_node\', \'auto_set_gpu\', \'rpc_layer\'], varargs=None, keywords=None, defaults=[\'8888\', \'1\', \'1\', \'None\', \'True\', \'grpc\'], "
+  }
+  member_method {
+    name: "cluster_spec"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_task_info"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "master"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'rpc_layer\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "num_accelerators"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'accelerator_type\', \'config_proto\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'GPU\', \'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-t-f-config-cluster-resolver.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-t-f-config-cluster-resolver.pbtxt
new file mode 100644
index 0000000000..5f9a430c0f
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-t-f-config-cluster-resolver.pbtxt
@@ -0,0 +1,38 @@
+path: "tensorflow.distribute.cluster_resolver.TFConfigClusterResolver"
+tf_class {
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.tfconfig_cluster_resolver.TFConfigClusterResolver\'>"
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.cluster_resolver.ClusterResolver\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "environment"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "rpc_layer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "task_id"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "task_type"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'rpc_layer\', \'environment\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "cluster_spec"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "master"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'rpc_layer\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "num_accelerators"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'config_proto\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-t-p-u-cluster-resolver.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-t-p-u-cluster-resolver.pbtxt
new file mode 100644
index 0000000000..ca22c85ac0
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-t-p-u-cluster-resolver.pbtxt
@@ -0,0 +1,34 @@
+path: "tensorflow.distribute.cluster_resolver.TPUClusterResolver"
+tf_class {
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.tpu_cluster_resolver.TPUClusterResolver\'>"
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.cluster_resolver.ClusterResolver\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "environment"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'tpu\', \'zone\', \'project\', \'job_name\', \'coordinator_name\', \'coordinator_address\', \'credentials\', \'service\', \'discovery_url\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'worker\', \'None\', \'None\', \'default\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "cluster_spec"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_job_name"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_master"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "master"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'rpc_layer\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "num_accelerators"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'accelerator_type\', \'config_proto\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'TPU\', \'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-union-resolver.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-union-resolver.pbtxt
new file mode 100644
index 0000000000..179848aca3
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.-union-resolver.pbtxt
@@ -0,0 +1,38 @@
+path: "tensorflow.distribute.cluster_resolver.UnionResolver"
+tf_class {
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.cluster_resolver.UnionClusterResolver\'>"
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.cluster_resolver.ClusterResolver\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "environment"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "rpc_layer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "task_id"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "task_type"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "cluster_spec"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "master"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'rpc_layer\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "num_accelerators"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'accelerator_type\', \'config_proto\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'GPU\', \'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.pbtxt
new file mode 100644
index 0000000000..5906ffa850
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.cluster_resolver.pbtxt
@@ -0,0 +1,35 @@
+path: "tensorflow.distribute.cluster_resolver"
+tf_module {
+  member {
+    name: "ClusterResolver"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "GCEClusterResolver"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "KubernetesClusterResolver"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "SimpleClusterResolver"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "SlurmClusterResolver"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "TFConfigClusterResolver"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "TPUClusterResolver"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "UnionResolver"
+    mtype: "<type \'type\'>"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.pbtxt
index 31dc6e0716..5b5c9e2df1 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.pbtxt
@@ -32,6 +32,10 @@ tf_module {
     name: "StrategyExtended"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "cluster_resolver"
+    mtype: "<type \'module\'>"
+  }
   member_method {
     name: "get_loss_reduction"
     argspec: "args=[], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-cluster-resolver.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-cluster-resolver.pbtxt
new file mode 100644
index 0000000000..0b35b61b4c
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-cluster-resolver.pbtxt
@@ -0,0 +1,24 @@
+path: "tensorflow.distribute.cluster_resolver.ClusterResolver"
+tf_class {
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.cluster_resolver.ClusterResolver\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "environment"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+  }
+  member_method {
+    name: "cluster_spec"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "master"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'rpc_layer\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "num_accelerators"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'config_proto\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-g-c-e-cluster-resolver.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-g-c-e-cluster-resolver.pbtxt
new file mode 100644
index 0000000000..5c2cc522f1
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-g-c-e-cluster-resolver.pbtxt
@@ -0,0 +1,38 @@
+path: "tensorflow.distribute.cluster_resolver.GCEClusterResolver"
+tf_class {
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.gce_cluster_resolver.GCEClusterResolver\'>"
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.cluster_resolver.ClusterResolver\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "environment"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "rpc_layer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "task_id"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "task_type"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'project\', \'zone\', \'instance_group\', \'port\', \'task_type\', \'task_id\', \'rpc_layer\', \'credentials\', \'service\'], varargs=None, keywords=None, defaults=[\'worker\', \'0\', \'grpc\', \'default\', \'None\'], "
+  }
+  member_method {
+    name: "cluster_spec"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "master"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'rpc_layer\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "num_accelerators"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'config_proto\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-kubernetes-cluster-resolver.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-kubernetes-cluster-resolver.pbtxt
new file mode 100644
index 0000000000..3220d68e05
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-kubernetes-cluster-resolver.pbtxt
@@ -0,0 +1,26 @@
+path: "tensorflow.distribute.cluster_resolver.KubernetesClusterResolver"
+tf_class {
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.kubernetes_cluster_resolver.KubernetesClusterResolver\'>"
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.cluster_resolver.ClusterResolver\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "environment"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'job_to_label_mapping\', \'tf_server_port\', \'rpc_layer\', \'override_client\'], varargs=None, keywords=None, defaults=[\'None\', \'8470\', \'grpc\', \'None\'], "
+  }
+  member_method {
+    name: "cluster_spec"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "master"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'rpc_layer\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "num_accelerators"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'config_proto\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-simple-cluster-resolver.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-simple-cluster-resolver.pbtxt
new file mode 100644
index 0000000000..4e80e3af30
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-simple-cluster-resolver.pbtxt
@@ -0,0 +1,38 @@
+path: "tensorflow.distribute.cluster_resolver.SimpleClusterResolver"
+tf_class {
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.cluster_resolver.SimpleClusterResolver\'>"
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.cluster_resolver.ClusterResolver\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "environment"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "rpc_layer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "task_id"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "task_type"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'cluster_spec\', \'master\', \'task_type\', \'task_id\', \'environment\', \'num_accelerators\', \'rpc_layer\'], varargs=None, keywords=None, defaults=[\'\', \'None\', \'None\', \'\', \'0\', \'None\'], "
+  }
+  member_method {
+    name: "cluster_spec"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "master"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'rpc_layer\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "num_accelerators"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'accelerator_type\', \'config_proto\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'GPU\', \'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-slurm-cluster-resolver.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-slurm-cluster-resolver.pbtxt
new file mode 100644
index 0000000000..971ea3dca4
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-slurm-cluster-resolver.pbtxt
@@ -0,0 +1,30 @@
+path: "tensorflow.distribute.cluster_resolver.SlurmClusterResolver"
+tf_class {
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.slurm_cluster_resolver.SlurmClusterResolver\'>"
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.cluster_resolver.ClusterResolver\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "environment"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'jobs\', \'port_base\', \'gpus_per_node\', \'gpus_per_task\', \'tasks_per_node\', \'auto_set_gpu\', \'rpc_layer\'], varargs=None, keywords=None, defaults=[\'8888\', \'1\', \'1\', \'None\', \'True\', \'grpc\'], "
+  }
+  member_method {
+    name: "cluster_spec"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_task_info"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "master"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'rpc_layer\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "num_accelerators"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'accelerator_type\', \'config_proto\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'GPU\', \'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-t-f-config-cluster-resolver.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-t-f-config-cluster-resolver.pbtxt
new file mode 100644
index 0000000000..5f9a430c0f
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-t-f-config-cluster-resolver.pbtxt
@@ -0,0 +1,38 @@
+path: "tensorflow.distribute.cluster_resolver.TFConfigClusterResolver"
+tf_class {
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.tfconfig_cluster_resolver.TFConfigClusterResolver\'>"
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.cluster_resolver.ClusterResolver\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "environment"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "rpc_layer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "task_id"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "task_type"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'rpc_layer\', \'environment\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "cluster_spec"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "master"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'rpc_layer\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "num_accelerators"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'config_proto\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-t-p-u-cluster-resolver.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-t-p-u-cluster-resolver.pbtxt
new file mode 100644
index 0000000000..ca22c85ac0
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-t-p-u-cluster-resolver.pbtxt
@@ -0,0 +1,34 @@
+path: "tensorflow.distribute.cluster_resolver.TPUClusterResolver"
+tf_class {
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.tpu_cluster_resolver.TPUClusterResolver\'>"
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.cluster_resolver.ClusterResolver\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "environment"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'tpu\', \'zone\', \'project\', \'job_name\', \'coordinator_name\', \'coordinator_address\', \'credentials\', \'service\', \'discovery_url\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'worker\', \'None\', \'None\', \'default\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "cluster_spec"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_job_name"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_master"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "master"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'rpc_layer\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "num_accelerators"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'accelerator_type\', \'config_proto\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'TPU\', \'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-union-resolver.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-union-resolver.pbtxt
new file mode 100644
index 0000000000..179848aca3
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.-union-resolver.pbtxt
@@ -0,0 +1,38 @@
+path: "tensorflow.distribute.cluster_resolver.UnionResolver"
+tf_class {
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.cluster_resolver.UnionClusterResolver\'>"
+  is_instance: "<class \'tensorflow.python.distribute.cluster_resolver.cluster_resolver.ClusterResolver\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "environment"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "rpc_layer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "task_id"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "task_type"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "cluster_spec"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "master"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'rpc_layer\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "num_accelerators"
+    argspec: "args=[\'self\', \'task_type\', \'task_id\', \'accelerator_type\', \'config_proto\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'GPU\', \'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.pbtxt
new file mode 100644
index 0000000000..5906ffa850
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.cluster_resolver.pbtxt
@@ -0,0 +1,35 @@
+path: "tensorflow.distribute.cluster_resolver"
+tf_module {
+  member {
+    name: "ClusterResolver"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "GCEClusterResolver"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "KubernetesClusterResolver"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "SimpleClusterResolver"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "SlurmClusterResolver"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "TFConfigClusterResolver"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "TPUClusterResolver"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "UnionResolver"
+    mtype: "<type \'type\'>"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.pbtxt
index 31dc6e0716..5b5c9e2df1 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.pbtxt
@@ -32,6 +32,10 @@ tf_module {
     name: "StrategyExtended"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "cluster_resolver"
+    mtype: "<type \'module\'>"
+  }
   member_method {
     name: "get_loss_reduction"
     argspec: "args=[], varargs=None, keywords=None, defaults=None"
-- 
GitLab


From 79548341efa79db02c2f2e8685ef0a4c96b2d4ee Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 28 Jan 2019 18:46:53 -0800
Subject: [PATCH 1446/2345] Improve quantized (uint8/int16) subtraction
 support.

Subtraction always took the slow path on uint8, changed to
support the non-4D activation path for non-broadcast.

Added support for int16 subtraction including broadcast and activation fns.

PiperOrigin-RevId: 231328853
---
 .../internal/optimized/optimized_ops.h        |   1 +
 .../internal/reference/reference_ops.h        |  48 ++++
 tensorflow/lite/kernels/sub.cc                | 225 ++++++++++++++----
 tensorflow/lite/kernels/sub_test.cc           | 111 +++++++++
 4 files changed, 333 insertions(+), 52 deletions(-)

diff --git a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
index 744f5cc20b..5dc03e7d53 100644
--- a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
+++ b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
@@ -85,6 +85,7 @@ using reference_ops::Select;
 using reference_ops::SpaceToBatchND;
 using reference_ops::Split;
 using reference_ops::StridedSlice;
+using reference_ops::Sub16;
 using reference_ops::Transpose;
 
 // TODO(b/80247582) Remove this constant.
diff --git a/tensorflow/lite/kernels/internal/reference/reference_ops.h b/tensorflow/lite/kernels/internal/reference/reference_ops.h
index 125fa95674..2eb294f734 100644
--- a/tensorflow/lite/kernels/internal/reference/reference_ops.h
+++ b/tensorflow/lite/kernels/internal/reference/reference_ops.h
@@ -1688,6 +1688,54 @@ inline void SubWithActivation(const ArithmeticParams& params,
   }
 }
 
+inline void Sub16(const ArithmeticParams& params,
+                  const RuntimeShape& input1_shape, const int16_t* input1_data,
+                  const RuntimeShape& input2_shape, const int16_t* input2_data,
+                  const RuntimeShape& output_shape, int16_t* output_data) {
+  gemmlowp::ScopedProfilingLabel label("Sub/Int16");
+  const int input1_shift = params.input1_shift;
+  const int flat_size =
+      MatchingFlatSize(output_shape, input1_shape, input2_shape);
+  const int16 output_activation_min = params.quantized_activation_min;
+  const int16 output_activation_max = params.quantized_activation_max;
+
+  TFLITE_DCHECK(input1_shift == 0 || params.input2_shift == 0);
+  TFLITE_DCHECK_LE(input1_shift, 0);
+  TFLITE_DCHECK_LE(params.input2_shift, 0);
+  const int16* not_shift_input = input1_shift == 0 ? input1_data : input2_data;
+  const int16* shift_input = input1_shift == 0 ? input2_data : input1_data;
+  const int input_right_shift =
+      input1_shift == 0 ? -params.input2_shift : -input1_shift;
+
+  if (input1_shift == 0) {
+    // F0 uses 0 integer bits, range [-1, 1].
+    using F0 = gemmlowp::FixedPoint<std::int16_t, 0>;
+    for (int i = 0; i < flat_size; ++i) {
+      F0 input_ready_scaled = F0::FromRaw(not_shift_input[i]);
+      F0 scaled_input = F0::FromRaw(
+          gemmlowp::RoundingDivideByPOT(shift_input[i], input_right_shift));
+      F0 result = SaturatingSub(input_ready_scaled, scaled_input);
+      const int16 raw_output = result.raw();
+      const int16 clamped_output = std::min(
+          output_activation_max, std::max(output_activation_min, raw_output));
+      output_data[i] = clamped_output;
+    }
+  } else {
+    // F0 uses 0 integer bits, range [-1, 1].
+    using F0 = gemmlowp::FixedPoint<std::int16_t, 0>;
+    for (int i = 0; i < flat_size; ++i) {
+      F0 input_ready_scaled = F0::FromRaw(not_shift_input[i]);
+      F0 scaled_input = F0::FromRaw(
+          gemmlowp::RoundingDivideByPOT(shift_input[i], input_right_shift));
+      F0 result = SaturatingSub(scaled_input, input_ready_scaled);
+      const int16 raw_output = result.raw();
+      const int16 clamped_output = std::min(
+          output_activation_max, std::max(output_activation_min, raw_output));
+      output_data[i] = clamped_output;
+    }
+  }
+}
+
 template <typename Scalar>
 inline void Concatenation(const ConcatenationParams& params,
                           const RuntimeShape* const* input_shapes,
diff --git a/tensorflow/lite/kernels/sub.cc b/tensorflow/lite/kernels/sub.cc
index 06a3b3499a..9144144e16 100644
--- a/tensorflow/lite/kernels/sub.cc
+++ b/tensorflow/lite/kernels/sub.cc
@@ -39,6 +39,23 @@ constexpr int kOutputTensor = 0;
 
 struct OpData {
   bool requires_broadcast;
+
+  // These fields are used in both the general 8-bit -> 8bit quantized path,
+  // and the special 16-bit -> 16bit quantized path
+  int input1_shift;
+  int input2_shift;
+  int32 output_activation_min;
+  int32 output_activation_max;
+
+  // These fields are used only in the general 8-bit -> 8bit quantized path
+  int32 input1_multiplier;
+  int32 input2_multiplier;
+  int32 output_multiplier;
+  int output_shift;
+  int left_shift;
+  int32 input1_offset;
+  int32 input2_offset;
+  int32 output_offset;
 };
 
 void* Init(TfLiteContext* context, const char* buffer, size_t length) {
@@ -51,8 +68,103 @@ void Free(TfLiteContext* context, void* buffer) {
   delete reinterpret_cast<OpData*>(buffer);
 }
 
+TfLiteStatus PrepareUint8SubOp(TfLiteContext* context,
+                               const TfLiteTensor* input_1,
+                               const TfLiteTensor* input_2,
+                               TfLiteTensor* output, TfLiteSubParams* params,
+                               OpData* op_params, int op_sign) {
+  const auto& input1_quantization_params = input_1->params;
+  const auto& input2_quantization_params = input_2->params;
+  const auto& output_quantization_params = output->params;
+
+  TF_LITE_ENSURE(context, input1_quantization_params.zero_point >= 0);
+  TF_LITE_ENSURE(context, input1_quantization_params.zero_point <= 255);
+  TF_LITE_ENSURE(context, input2_quantization_params.zero_point >= 0);
+  TF_LITE_ENSURE(context, input2_quantization_params.zero_point <= 255);
+  TF_LITE_ENSURE(context, output_quantization_params.zero_point >= 0);
+  TF_LITE_ENSURE(context, output_quantization_params.zero_point <= 255);
+
+  op_params->input1_offset = -input1_quantization_params.zero_point;
+  op_params->input2_offset = -input2_quantization_params.zero_point;
+  op_params->output_offset = output_quantization_params.zero_point;
+  op_params->left_shift = 20;
+  const double twice_max_input_scale =
+      2 * std::max(input1_quantization_params.scale,
+                   input2_quantization_params.scale);
+  const double real_input1_multiplier =
+      input1_quantization_params.scale / twice_max_input_scale;
+  const double real_input2_multiplier =
+      input2_quantization_params.scale / twice_max_input_scale;
+  const double real_output_multiplier =
+      twice_max_input_scale /
+      ((1 << op_params->left_shift) * output_quantization_params.scale);
+
+  tflite::QuantizeMultiplierSmallerThanOneExp(real_input1_multiplier,
+                                              &op_params->input1_multiplier,
+                                              &op_params->input1_shift);
+  tflite::QuantizeMultiplierSmallerThanOneExp(real_input2_multiplier,
+                                              &op_params->input2_multiplier,
+                                              &op_params->input2_shift);
+  op_params->input2_multiplier *= op_sign;
+  tflite::QuantizeMultiplierSmallerThanOneExp(real_output_multiplier,
+                                              &op_params->output_multiplier,
+                                              &op_params->output_shift);
+
+  CalculateActivationRangeUint8(params->activation, output,
+                                &op_params->output_activation_min,
+                                &op_params->output_activation_max);
+  return kTfLiteOk;
+}
+
+TfLiteStatus PrepareInt16SubOp(TfLiteContext* context,
+                               const TfLiteTensor* input1,
+                               const TfLiteTensor* input2, TfLiteTensor* output,
+                               TfLiteSubParams* params, OpData* data) {
+  // 16bit -> 16bit special quantized path, supporting only a rather
+  // narrow case of quantization parameters: zero_points must all be 0
+  // ("symmetric quantization") and scales must be power-of-two (which
+  // we abbreviate as "POT" below). The intended use case for this path
+  // is in LSTM cells, where, due to the constraints of implementing
+  // some of the math in these LSTM cells in fixed-point arithmetic,
+  // we need to have such symmetric, power-of-two quantization
+  // (Fixed-point formats are inherently symmetric, power-of-two).
+  TF_LITE_ENSURE_EQ(context, input1->params.zero_point, 0);
+  TF_LITE_ENSURE_EQ(context, input2->params.zero_point, 0);
+  TF_LITE_ENSURE_EQ(context, output->params.zero_point, 0);
+
+  int input1_scale_log2_rounded;
+  bool input1_scale_is_pot =
+      CheckedLog2(input1->params.scale, &input1_scale_log2_rounded);
+  TF_LITE_ENSURE(context, input1_scale_is_pot);
+
+  int input2_scale_log2_rounded;
+  bool input2_scale_is_pot =
+      CheckedLog2(input2->params.scale, &input2_scale_log2_rounded);
+  TF_LITE_ENSURE(context, input2_scale_is_pot);
+
+  int output_scale_log2_rounded;
+  bool output_scale_is_pot =
+      CheckedLog2(output->params.scale, &output_scale_log2_rounded);
+  TF_LITE_ENSURE(context, output_scale_is_pot);
+
+  data->input1_shift = input1_scale_log2_rounded - output_scale_log2_rounded;
+  data->input2_shift = input2_scale_log2_rounded - output_scale_log2_rounded;
+
+  // Shifting of one input is supported. The graph quantization should ensure
+  // that the other input matches the output.
+  TF_LITE_ENSURE(context, data->input1_shift == 0 || data->input2_shift == 0);
+  TF_LITE_ENSURE(context, data->input1_shift <= 0);
+  TF_LITE_ENSURE(context, data->input2_shift <= 0);
+
+  CalculateActivationRangeQuantized(context, params->activation, output,
+                                    &data->output_activation_min,
+                                    &data->output_activation_max);
+  return kTfLiteOk;
+}
+
 TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   OpData* data = reinterpret_cast<OpData*>(node->user_data);
+  auto* params = reinterpret_cast<TfLiteSubParams*>(node->builtin_data);
 
   TF_LITE_ENSURE_EQ(context, NumInputs(node), 2);
   TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);
@@ -74,6 +186,14 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
     output_size = TfLiteIntArrayCopy(input1->dims);
   }
 
+  if (output->type == kTfLiteUInt8) {
+    TF_LITE_ENSURE_OK(context, PrepareUint8SubOp(context, input1, input2,
+                                                 output, params, data, -1));
+  } else if (output->type == kTfLiteInt16) {
+    TF_LITE_ENSURE_OK(context, PrepareInt16SubOp(context, input1, input2,
+                                                 output, params, data));
+  }
+
   return context->ResizeTensor(context, output, output_size);
 }
 
@@ -129,60 +249,61 @@ void EvalQuantized(TfLiteContext* context, TfLiteNode* node,
                    TfLiteSubParams* params, const OpData* data,
                    const TfLiteTensor* input1, const TfLiteTensor* input2,
                    TfLiteTensor* output) {
-  auto input1_offset = -input1->params.zero_point;
-  auto input2_offset = -input2->params.zero_point;
-  auto output_offset = output->params.zero_point;
-  const int left_shift = 20;
-  const double twice_max_input_scale =
-      2 * std::max(input1->params.scale, input2->params.scale);
-  const double real_input1_multiplier =
-      input1->params.scale / twice_max_input_scale;
-  const double real_input2_multiplier =
-      input2->params.scale / twice_max_input_scale;
-  const double real_output_multiplier =
-      twice_max_input_scale / ((1 << left_shift) * output->params.scale);
+  tflite::ArithmeticParams op_params;
+  op_params.left_shift = data->left_shift;
+  op_params.input1_offset = data->input1_offset;
+  op_params.input1_multiplier = data->input1_multiplier;
+  op_params.input1_shift = data->input1_shift;
+  op_params.input2_offset = data->input2_offset;
+  op_params.input2_multiplier = data->input2_multiplier;
+  op_params.input2_shift = data->input2_shift;
+  op_params.output_offset = data->output_offset;
+  op_params.output_multiplier = data->output_multiplier;
+  op_params.output_shift = data->output_shift;
+  SetActivationParams(data->output_activation_min, data->output_activation_max,
+                      &op_params);
 
-  int32 input1_multiplier;
-  int input1_shift;
-  QuantizeMultiplierSmallerThanOneExp(real_input1_multiplier,
-                                      &input1_multiplier, &input1_shift);
-  int32 input2_multiplier;
-  int input2_shift;
-  QuantizeMultiplierSmallerThanOneExp(real_input2_multiplier,
-                                      &input2_multiplier, &input2_shift);
-  int32 output_multiplier;
-  int output_shift;
-  QuantizeMultiplierSmallerThanOneExp(real_output_multiplier,
-                                      &output_multiplier, &output_shift);
+  const bool need_broadcast = optimized_ops::ProcessBroadcastShapes(
+      GetTensorShape(input1), GetTensorShape(input2), &op_params);
 
-  int32 output_activation_min, output_activation_max;
-  CalculateActivationRangeUint8(params->activation, output,
-                                &output_activation_min, &output_activation_max);
-
-#define TF_LITE_SUB(type, opname)                                      \
-  tflite::ArithmeticParams op_params;                                  \
-  op_params.left_shift = left_shift;                                   \
-  op_params.input1_offset = input1_offset;                             \
-  op_params.input1_multiplier = input1_multiplier;                     \
-  op_params.input1_shift = input1_shift;                               \
-  op_params.input2_offset = input2_offset;                             \
-  op_params.input2_multiplier = input2_multiplier;                     \
-  op_params.input2_shift = input2_shift;                               \
-  op_params.output_offset = output_offset;                             \
-  op_params.output_multiplier = output_multiplier;                     \
-  op_params.output_shift = output_shift;                               \
-  SetActivationParams(output_activation_min, output_activation_max,    \
-                      &op_params);                                     \
-  type::opname(op_params, GetTensorShape(input1),                      \
-               GetTensorData<uint8_t>(input1), GetTensorShape(input2), \
-               GetTensorData<uint8_t>(input2), GetTensorShape(output), \
-               GetTensorData<uint8_t>(output))
-  // The quantized version of Sub doesn't support activations, so we
-  // always use BroadcastSub.
-  if (kernel_type == kReference) {
-    TF_LITE_SUB(reference_ops, BroadcastSub4DSlow);
+#define TF_LITE_SUB(type, opname, data_type)                             \
+  type::opname(op_params, GetTensorShape(input1),                        \
+               GetTensorData<data_type>(input1), GetTensorShape(input2), \
+               GetTensorData<data_type>(input2), GetTensorShape(output), \
+               GetTensorData<data_type>(output))
+  if (output->type == kTfLiteUInt8) {
+    // NOTE: We are using the add kernels. This is possible as the second values
+    // multiplier is negated before being passed down.
+    if (kernel_type == kReference) {
+      if (need_broadcast) {
+        TF_LITE_SUB(reference_ops, BroadcastAdd4DSlow, uint8_t);
+      } else {
+        TF_LITE_SUB(reference_ops, Add, uint8_t);
+      }
+    } else {
+      if (op_params.broadcast_category ==
+          BroadcastableOpCategory::kGenericBroadcast) {
+        TF_LITE_SUB(optimized_ops, BroadcastAdd4DSlow, uint8_t);
+      } else if (need_broadcast) {
+        TF_LITE_SUB(optimized_ops, BroadcastAddFivefold, uint8_t);
+      } else {
+        TF_LITE_SUB(optimized_ops, Add, uint8_t);
+      }
+    }
   } else {
-    TF_LITE_SUB(optimized_ops, BroadcastSub4DSlow);
+    if (kernel_type == kReference) {
+      if (need_broadcast) {
+        TF_LITE_SUB(reference_ops, BroadcastSub4DSlow, int16_t);
+      } else {
+        TF_LITE_SUB(reference_ops, Sub16, int16_t);
+      }
+    } else {
+      if (need_broadcast) {
+        TF_LITE_SUB(optimized_ops, BroadcastSub4DSlow, int16_t);
+      } else {
+        TF_LITE_SUB(optimized_ops, Sub16, int16_t);
+      }
+    }
   }
 #undef TF_LITE_SUB
 }
@@ -198,7 +319,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
 
   if (output->type == kTfLiteFloat32 || output->type == kTfLiteInt32) {
     EvalSub<kernel_type>(context, node, params, data, input1, input2, output);
-  } else if (output->type == kTfLiteUInt8) {
+  } else if (output->type == kTfLiteUInt8 || output->type == kTfLiteInt16) {
     EvalQuantized<kernel_type>(context, node, params, data, input1, input2,
                                output);
   } else {
diff --git a/tensorflow/lite/kernels/sub_test.cc b/tensorflow/lite/kernels/sub_test.cc
index 41503300ab..23927c6554 100644
--- a/tensorflow/lite/kernels/sub_test.cc
+++ b/tensorflow/lite/kernels/sub_test.cc
@@ -67,6 +67,11 @@ class QuantizedSubOpModel : public BaseSubOpModel {
     return Dequantize<uint8_t>(ExtractVector<uint8_t>(output_),
                                GetScale(output_), GetZeroPoint(output_));
   }
+
+  std::vector<float> GetDequantizedOutputInt16() {
+    return Dequantize<int16_t>(ExtractVector<int16_t>(output_),
+                               GetScale(output_), GetZeroPoint(output_));
+  }
 };
 
 // for quantized Sub, the error shouldn't exceed 2*step
@@ -76,6 +81,12 @@ float GetTolerance(int min, int max) {
   return kQuantizedTolerance;
 }
 
+float GetToleranceInt16(float min, float max) {
+  float kQuantizedStep = (max - min) / std::numeric_limits<int16_t>::max();
+  float kQuantizedTolerance = 2.0 * kQuantizedStep;
+  return kQuantizedTolerance;
+}
+
 TEST(FloatSubOpModel, NoActivation) {
   FloatSubOpModel m({TensorType_FLOAT32, {1, 2, 2, 1}},
                     {TensorType_FLOAT32, {1, 2, 2, 1}},
@@ -266,6 +277,106 @@ TEST(QuantizedSubOpModel, QuantizedWithBroadcast) {
   }
 }
 
+TEST(QuantizedSubOpModel, QuantizedTestsNoActivationInt16) {
+  const float kMin = -1.f;
+  const float kMax =
+      static_cast<float>(std::numeric_limits<int16_t>::max() - 1) /
+      std::numeric_limits<int16_t>::max();
+  float kQuantizedTolerance = GetToleranceInt16(kMin, kMax);
+  std::vector<std::vector<float>> inputs1 = {
+      {0.7, 0.6, 0.6, 0.5}, {-0.2, 0.6, 0.9, -0.1}, {-0.2, 0.6, -0.3, 0.8}};
+  std::vector<std::vector<float>> inputs2 = {
+      {0.6, 0.4, 0.3, 0.1}, {0.6, 0.4, 0.5, -0.8}, {0.6, 0.4, 0.8, 0.5}};
+  std::vector<std::vector<float>> results = {
+      {0.1, 0.2, 0.3, 0.4}, {-0.8, 0.2, 0.4, 0.7}, {-0.8, 0.2, -1.0, 0.3}};
+  for (int i = 0; i < inputs1.size(); ++i) {
+    QuantizedSubOpModel m({TensorType_INT16, {1, 2, 2, 1}, kMin, kMax},
+                          {TensorType_INT16, {1, 2, 2, 1}, kMin, kMax},
+                          {TensorType_INT16, {}, kMin, kMax},
+                          ActivationFunctionType_NONE);
+    m.QuantizeAndPopulate<int16_t>(m.input1(), inputs1[i]);
+    m.QuantizeAndPopulate<int16_t>(m.input2(), inputs2[i]);
+    m.Invoke();
+    EXPECT_THAT(
+        m.GetDequantizedOutputInt16(),
+        ElementsAreArray(ArrayFloatNear(results[i], kQuantizedTolerance)))
+        << "With test number " << i;
+  }
+}
+
+TEST(QuantizedSubOpModel, QuantizedTestsReluActivationInt16) {
+  const float kMin = -2.f;
+  const float kMax = 2.0 * (std::numeric_limits<int16_t>::max() - 1) /
+                     std::numeric_limits<int16_t>::max();
+  float kQuantizedTolerance = GetToleranceInt16(kMin, kMax);
+  std::vector<std::vector<float>> inputs1 = {{-0.8, 0.2, 0.9, 0.7},
+                                             {-0.8, 0.2, 0.7, 0.5}};
+  std::vector<std::vector<float>> inputs2 = {{0.6, 0.4, 0.9, -0.8},
+                                             {0.6, 0.4, -0.8, 0.3}};
+  std::vector<std::vector<float>> results = {{-1.0, -0.2, 0.0, 1.0},
+                                             {-1.0, -0.2, 1.0, 0.2}};
+  for (int i = 0; i < inputs1.size(); ++i) {
+    QuantizedSubOpModel m({TensorType_INT16, {1, 2, 2, 1}, kMin, kMax},
+                          {TensorType_INT16, {1, 2, 2, 1}, kMin, kMax},
+                          {TensorType_INT16, {}, kMin, kMax},
+                          ActivationFunctionType_RELU_N1_TO_1);
+    m.QuantizeAndPopulate<int16_t>(m.input1(), inputs1[i]);
+    m.QuantizeAndPopulate<int16_t>(m.input2(), inputs2[i]);
+    m.Invoke();
+    EXPECT_THAT(
+        m.GetDequantizedOutputInt16(),
+        ElementsAreArray(ArrayFloatNear(results[i], kQuantizedTolerance)))
+        << "With test number " << i;
+  }
+}
+
+TEST(QuantizedSubOpModel, QuantizedTestsNoActivationBroadcastInt16) {
+  const float kMin = -1.f;
+  const float kMax =
+      static_cast<float>(std::numeric_limits<int16_t>::max() - 1) /
+      std::numeric_limits<int16_t>::max();
+  float kQuantizedTolerance = GetToleranceInt16(kMin, kMax);
+  std::vector<std::vector<int>> test_shapes = {
+      {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}};
+  for (int i = 0; i < test_shapes.size(); ++i) {
+    QuantizedSubOpModel m({TensorType_INT16, test_shapes[i], kMin, kMax},
+                          {TensorType_INT16, {}, kMin, kMax},
+                          {TensorType_INT16, {}, kMin, kMax},
+                          ActivationFunctionType_NONE);
+    m.QuantizeAndPopulate<int16_t>(m.input1(),
+                                   {-0.9, -0.7, -0.3, 0.0, 0.3, 0.5});
+    m.QuantizeAndPopulate<int16_t>(m.input2(), {0.2});
+    m.Invoke();
+    EXPECT_THAT(m.GetDequantizedOutputInt16(),
+                ElementsAreArray(ArrayFloatNear(
+                    {-1.0, -0.9, -0.5, -0.2, 0.1, 0.3}, kQuantizedTolerance)))
+        << "With shape number " << i;
+  }
+}
+
+TEST(QuantizedSubOpModel, QuantizedTestsReluActivationBroadcastInt16) {
+  const float kMin = -2.f;
+  const float kMax = 2.0 * (std::numeric_limits<int16_t>::max() - 1) /
+                     std::numeric_limits<int16_t>::max();
+  float kQuantizedTolerance = GetToleranceInt16(kMin, kMax);
+  std::vector<std::vector<int>> test_shapes = {
+      {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}};
+  for (int i = 0; i < test_shapes.size(); ++i) {
+    QuantizedSubOpModel m({TensorType_INT16, test_shapes[i], kMin, kMax},
+                          {TensorType_INT16, {}, kMin, kMax},
+                          {TensorType_INT16, {}, kMin, kMax},
+                          ActivationFunctionType_RELU_N1_TO_1);
+    m.QuantizeAndPopulate<int16_t>(m.input1(),
+                                   {-0.9, -0.7, -0.3, 0.0, 0.3, 0.5});
+    m.QuantizeAndPopulate<int16_t>(m.input2(), {0.2});
+    m.Invoke();
+    EXPECT_THAT(m.GetDequantizedOutputInt16(),
+                ElementsAreArray(ArrayFloatNear(
+                    {-1.0, -0.9, -0.5, -0.2, 0.1, 0.3}, kQuantizedTolerance)))
+        << "With shape number " << i;
+  }
+}
+
 }  // namespace
 }  // namespace tflite
 int main(int argc, char** argv) {
-- 
GitLab


From 2597b96b727fb951b642a79b64f097151a575362 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Mon, 28 Jan 2019 18:57:17 -0800
Subject: [PATCH 1447/2345] Add a method_name and a tf.saved_model.save

Previously method_name was blank. We'll never be able to set "classify" or "regress", so everything will be predict.

I will follow up with a tensorflow_serving integration test (in that repo) due to sync issues.

Fixes #25234.

PiperOrigin-RevId: 231329696
---
 tensorflow/python/saved_model/save.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index a9bf1a7370..7eeb14608f 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -479,7 +479,8 @@ def _generate_signatures(signature_functions, resource_map):
         function.name, signature_key)
     signatures[signature_key] = signature_def_utils.build_signature_def(
         _tensor_dict_to_tensorinfo(exterior_argument_placeholders),
-        _tensor_dict_to_tensorinfo(outputs))
+        _tensor_dict_to_tensorinfo(outputs),
+        method_name=signature_constants.PREDICT_METHOD_NAME)
   return signatures
 
 
-- 
GitLab


From 1fc57fd1bd59f892f340701272013ca816d9d28c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 28 Jan 2019 18:57:22 -0800
Subject: [PATCH 1448/2345] Update ops-related pbtxt files.

PiperOrigin-RevId: 231329705
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 482 ++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 |  15 +
 2 files changed, 497 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index ee16a58667..503d4b557f 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -12624,6 +12624,30 @@ op {
     }
   }
 }
+op {
+  name: "Cholesky"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_DOUBLE
+        type: DT_FLOAT
+        type: DT_HALF
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+      }
+    }
+  }
+}
 op {
   name: "CholeskyGrad"
   input_arg {
@@ -12649,6 +12673,32 @@ op {
     }
   }
 }
+op {
+  name: "CholeskyGrad"
+  input_arg {
+    name: "l"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+}
 op {
   name: "ClipByValue"
   input_arg {
@@ -31710,6 +31760,34 @@ op {
     }
   }
 }
+op {
+  name: "LogMatrixDeterminant"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "sign"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "log_abs_determinant"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+      }
+    }
+  }
+}
 op {
   name: "LogSoftmax"
   input_arg {
@@ -32232,6 +32310,47 @@ op {
     }
   }
 }
+op {
+  name: "Lu"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "lu"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "p"
+    type_attr: "output_idx_type"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_DOUBLE
+        type: DT_FLOAT
+        type: DT_HALF
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+      }
+    }
+  }
+  attr {
+    name: "output_idx_type"
+    type: "type"
+    default_value {
+      type: DT_INT32
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
 op {
   name: "MakeIterator"
   input_arg {
@@ -33113,6 +33232,30 @@ op {
     }
   }
 }
+op {
+  name: "MatrixDeterminant"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+      }
+    }
+  }
+}
 op {
   name: "MatrixDiag"
   input_arg {
@@ -33192,6 +33335,61 @@ op {
     version: 27
   }
 }
+op {
+  name: "MatrixExponential"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_DOUBLE
+        type: DT_FLOAT
+        type: DT_HALF
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+      }
+    }
+  }
+  deprecation {
+    version: 27
+  }
+}
+op {
+  name: "MatrixInverse"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "adjoint"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_DOUBLE
+        type: DT_FLOAT
+      }
+    }
+  }
+}
 op {
   name: "MatrixInverse"
   input_arg {
@@ -33216,6 +33414,8 @@ op {
       list {
         type: DT_DOUBLE
         type: DT_FLOAT
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
       }
     }
   }
@@ -33244,6 +33444,7 @@ op {
       list {
         type: DT_DOUBLE
         type: DT_FLOAT
+        type: DT_HALF
         type: DT_COMPLEX64
         type: DT_COMPLEX128
       }
@@ -33324,6 +33525,77 @@ op {
     }
   }
 }
+op {
+  name: "MatrixSolve"
+  input_arg {
+    name: "matrix"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "rhs"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "adjoint"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_DOUBLE
+        type: DT_FLOAT
+        type: DT_HALF
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+      }
+    }
+  }
+}
+op {
+  name: "MatrixSolveLs"
+  input_arg {
+    name: "matrix"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "rhs"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "l2_regularizer"
+    type: DT_DOUBLE
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_DOUBLE
+        type: DT_FLOAT
+      }
+    }
+  }
+  attr {
+    name: "fast"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+}
 op {
   name: "MatrixSolveLs"
   input_arg {
@@ -33349,6 +33621,8 @@ op {
       list {
         type: DT_DOUBLE
         type: DT_FLOAT
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
       }
     }
   }
@@ -33385,6 +33659,7 @@ op {
       list {
         type: DT_DOUBLE
         type: DT_FLOAT
+        type: DT_HALF
         type: DT_COMPLEX64
         type: DT_COMPLEX128
       }
@@ -33421,6 +33696,30 @@ op {
     }
   }
 }
+op {
+  name: "MatrixSquareRoot"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_DOUBLE
+        type: DT_FLOAT
+        type: DT_HALF
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+      }
+    }
+  }
+}
 op {
   name: "MatrixTriangularSolve"
   input_arg {
@@ -33501,6 +33800,48 @@ op {
     }
   }
 }
+op {
+  name: "MatrixTriangularSolve"
+  input_arg {
+    name: "matrix"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "rhs"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "lower"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  attr {
+    name: "adjoint"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_DOUBLE
+        type: DT_FLOAT
+        type: DT_HALF
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+      }
+    }
+  }
+}
 op {
   name: "Max"
   input_arg {
@@ -42659,6 +43000,41 @@ op {
     }
   }
 }
+op {
+  name: "Qr"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "q"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "r"
+    type_attr: "T"
+  }
+  attr {
+    name: "full_matrices"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_DOUBLE
+        type: DT_FLOAT
+        type: DT_HALF
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+      }
+    }
+  }
+}
 op {
   name: "QuantizeAndDequantize"
   input_arg {
@@ -63578,6 +63954,63 @@ op {
     version: 11
   }
 }
+op {
+  name: "SelfAdjointEig"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_DOUBLE
+        type: DT_FLOAT
+        type: DT_HALF
+      }
+    }
+  }
+  deprecation {
+    version: 11
+  }
+}
+op {
+  name: "SelfAdjointEigV2"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "e"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "v"
+    type_attr: "T"
+  }
+  attr {
+    name: "compute_v"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_DOUBLE
+        type: DT_FLOAT
+      }
+    }
+  }
+}
 op {
   name: "SelfAdjointEigV2"
   input_arg {
@@ -63606,6 +64039,8 @@ op {
       list {
         type: DT_DOUBLE
         type: DT_FLOAT
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
       }
     }
   }
@@ -63638,6 +64073,7 @@ op {
       list {
         type: DT_DOUBLE
         type: DT_FLOAT
+        type: DT_HALF
         type: DT_COMPLEX64
         type: DT_COMPLEX128
       }
@@ -76581,6 +77017,52 @@ op {
     }
   }
 }
+op {
+  name: "Svd"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "s"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "u"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "v"
+    type_attr: "T"
+  }
+  attr {
+    name: "compute_uv"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  attr {
+    name: "full_matrices"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_DOUBLE
+        type: DT_FLOAT
+        type: DT_HALF
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+      }
+    }
+  }
+}
 op {
   name: "Switch"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 40e15b6138..66dc4be00d 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -5234,6 +5234,7 @@ op {
       list {
         type: DT_DOUBLE
         type: DT_FLOAT
+        type: DT_HALF
         type: DT_COMPLEX64
         type: DT_COMPLEX128
       }
@@ -5259,6 +5260,7 @@ op {
     type: "type"
     allowed_values {
       list {
+        type: DT_HALF
         type: DT_FLOAT
         type: DT_DOUBLE
       }
@@ -15830,6 +15832,7 @@ op {
     type: "type"
     allowed_values {
       list {
+        type: DT_HALF
         type: DT_FLOAT
         type: DT_DOUBLE
         type: DT_COMPLEX64
@@ -16264,6 +16267,7 @@ op {
       list {
         type: DT_DOUBLE
         type: DT_FLOAT
+        type: DT_HALF
         type: DT_COMPLEX64
         type: DT_COMPLEX128
       }
@@ -16832,6 +16836,7 @@ op {
     type: "type"
     allowed_values {
       list {
+        type: DT_HALF
         type: DT_FLOAT
         type: DT_DOUBLE
         type: DT_COMPLEX64
@@ -16887,6 +16892,7 @@ op {
       list {
         type: DT_DOUBLE
         type: DT_FLOAT
+        type: DT_HALF
         type: DT_COMPLEX64
         type: DT_COMPLEX128
       }
@@ -16921,6 +16927,7 @@ op {
       list {
         type: DT_DOUBLE
         type: DT_FLOAT
+        type: DT_HALF
         type: DT_COMPLEX64
         type: DT_COMPLEX128
       }
@@ -16995,6 +17002,7 @@ op {
       list {
         type: DT_DOUBLE
         type: DT_FLOAT
+        type: DT_HALF
         type: DT_COMPLEX64
         type: DT_COMPLEX128
       }
@@ -17026,6 +17034,7 @@ op {
       list {
         type: DT_DOUBLE
         type: DT_FLOAT
+        type: DT_HALF
         type: DT_COMPLEX64
         type: DT_COMPLEX128
       }
@@ -17056,6 +17065,7 @@ op {
       list {
         type: DT_DOUBLE
         type: DT_FLOAT
+        type: DT_HALF
         type: DT_COMPLEX64
         type: DT_COMPLEX128
       }
@@ -17097,6 +17107,7 @@ op {
       list {
         type: DT_DOUBLE
         type: DT_FLOAT
+        type: DT_HALF
         type: DT_COMPLEX64
         type: DT_COMPLEX128
       }
@@ -21632,6 +21643,7 @@ op {
       list {
         type: DT_DOUBLE
         type: DT_FLOAT
+        type: DT_HALF
         type: DT_COMPLEX64
         type: DT_COMPLEX128
       }
@@ -31328,6 +31340,7 @@ op {
       list {
         type: DT_DOUBLE
         type: DT_FLOAT
+        type: DT_HALF
       }
     }
   }
@@ -31364,6 +31377,7 @@ op {
       list {
         type: DT_DOUBLE
         type: DT_FLOAT
+        type: DT_HALF
         type: DT_COMPLEX64
         type: DT_COMPLEX128
       }
@@ -36826,6 +36840,7 @@ op {
       list {
         type: DT_DOUBLE
         type: DT_FLOAT
+        type: DT_HALF
         type: DT_COMPLEX64
         type: DT_COMPLEX128
       }
-- 
GitLab


From 820857604fe3648d6fb90b4ac0d1130633c80f32 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Mon, 28 Jan 2019 19:12:46 -0800
Subject: [PATCH 1449/2345] Fix bug in handling of return statements. Relax the
 restriction that all code paths have a return value, allowing for None
 returns when the control flow creating them is not staged.

PiperOrigin-RevId: 231331284
---
 .../autograph/converters/return_statements.py | 557 ++++++++++--------
 .../converters/return_statements_test.py      |  45 +-
 .../python/autograph/impl/conversion_test.py  |   1 -
 tensorflow/python/autograph/pyct/templates.py |   1 +
 .../python/autograph/pyct/transformer.py      |   8 +-
 5 files changed, 337 insertions(+), 275 deletions(-)

diff --git a/tensorflow/python/autograph/converters/return_statements.py b/tensorflow/python/autograph/converters/return_statements.py
index 496c99e3b5..39b4df313a 100644
--- a/tensorflow/python/autograph/converters/return_statements.py
+++ b/tensorflow/python/autograph/converters/return_statements.py
@@ -22,310 +22,353 @@ import gast
 
 from tensorflow.python.autograph.core import converter
 from tensorflow.python.autograph.pyct import anno
-from tensorflow.python.autograph.pyct import ast_util
+from tensorflow.python.autograph.pyct import parser
 from tensorflow.python.autograph.pyct import templates
 from tensorflow.python.autograph.pyct.static_analysis.annos import NodeAnno
 
 
-# TODO(mdan): Move this logic into transformer_base.
-class BodyVisitor(converter.Base):
-  """Walks breadth- or depth-first the list-of-nodes bodies of AST nodes."""
+BODY_DEFINITELY_RETURNS = 'BODY_DEFINITELY_RETURNS'
+ORELSE_DEFINITELY_RETURNS = 'ORELSE_DEFINITELY_RETURNS'
+STMT_DEFINITELY_RETURNS = 'STMT_DEFINITELY_RETURNS'
 
-  def __init__(self, ctx, depth_first=False):
-    super(BodyVisitor, self).__init__(ctx)
-    self.depth_first = depth_first
-    self.changes_made = False
 
-  def visit_nodelist(self, nodelist):
-    for node in nodelist:
-      if isinstance(node, list):
-        node = self.visit_nodelist(node)
+class _Block(object):
+
+  def __init__(self):
+    self.definitely_returns = False
+
+
+class ConditionalReturnRewriter(converter.Base):
+  """Rewrites a a pattern where it's unbovious that all paths return a value.
+
+  This rewrite allows avoiding intermediate None return values.
+
+  The following pattern:
+
+      if cond:
+        <block 1>
+        return
       else:
-        node = self.generic_visit(node)
-    return nodelist
+        <block 2>
+      <block 3>
 
-  def visit_If(self, node):
-    if self.depth_first:
-      node = self.generic_visit(node)
-    node.body = self.visit_nodelist(node.body)
-    node.orelse = self.visit_nodelist(node.orelse)
-    if not self.depth_first:
-      node = self.generic_visit(node)
-    return node
+  is converted to:
 
-  def visit_For(self, node):
-    if self.depth_first:
-      node = self.generic_visit(node)
-    node.body = self.visit_nodelist(node.body)
-    node.orelse = self.visit_nodelist(node.orelse)
-    if not self.depth_first:
-      node = self.generic_visit(node)
+      if cond:
+        <block 1>
+        return
+      else:
+        <block 2>
+        <block 3>
+
+  and vice-versa (if the else returns, subsequent statements are moved under the
+  if branch).
+  """
+
+  def visit_Return(self, node):
+    self.state[_Block].definitely_returns = True
     return node
 
+  def _postprocess_statement(self, node):
+    # If the node definitely returns (e.g. it's a with statement with a
+    # return stateent in it), then the current block also definitely returns.
+    if anno.getanno(node, STMT_DEFINITELY_RETURNS, default=False):
+      self.state[_Block].definitely_returns = True
+
+    # The special case: collapse a typical conditional return pattern into
+    # a single conditional with possibly returns on both branches. This
+    # reduces the use of None return values, which don't work with TF
+    # conditionals.
+    if (isinstance(node, gast.If)
+        and anno.getanno(node, BODY_DEFINITELY_RETURNS, default=False)):
+      return node, node.orelse
+    elif (isinstance(node, gast.If)
+          and anno.getanno(node, ORELSE_DEFINITELY_RETURNS, default=False)):
+      return node, node.body
+
+    return node, None
+
+  def _visit_statement_block(self, node, nodes):
+    self.state[_Block].enter()
+    new_nodes = self.visit_block(nodes, after_visit=self._postprocess_statement)
+    block_definitely_returns = self.state[_Block].definitely_returns
+    self.state[_Block].exit()
+    return new_nodes, block_definitely_returns
+
   def visit_While(self, node):
-    if self.depth_first:
-      node = self.generic_visit(node)
-    node.body = self.visit_nodelist(node.body)
-    node.orelse = self.visit_nodelist(node.orelse)
-    if not self.depth_first:
-      node = self.generic_visit(node)
+    node.test = self.visit(node.test)
+    node.body, _ = self._visit_statement_block(node, node.body)
+    node.orelse, _ = self._visit_statement_block(node, node.orelse)
     return node
 
-  def visit_Try(self, node):
-    if self.depth_first:
-      node = self.generic_visit(node)
-    node.body = self.visit_nodelist(node.body)
-    node.orelse = self.visit_nodelist(node.orelse)
-    node.finalbody = self.visit_nodelist(node.finalbody)
-    for i in range(len(node.handlers)):
-      node.handlers[i].body = self.visit_nodelist(node.handlers[i].body)
-    if not self.depth_first:
-      node = self.generic_visit(node)
+  def visit_For(self, node):
+    node.iter = self.visit(node.iter)
+    node.target = self.visit(node.target)
+    node.body, _ = self._visit_statement_block(node, node.body)
+    node.orelse, _ = self._visit_statement_block(node, node.orelse)
     return node
 
-  def visit_With(self, node):
-    if self.depth_first:
-      node = self.generic_visit(node)
-    node.body = self.visit_nodelist(node.body)
-    if not self.depth_first:
-      node = self.generic_visit(node)
-    return node
+  def visit_If(self, node):
+    node.test = self.visit(node.test)
 
-  def visit_FunctionDef(self, node):
-    if self.depth_first:
-      node = self.generic_visit(node)
-    node.body = self.visit_nodelist(node.body)
-    self.generic_visit(node)
-    if not self.depth_first:
-      node = self.generic_visit(node)
-    return node
+    node.body, body_definitely_returns = self._visit_statement_block(
+        node, node.body)
+    if body_definitely_returns:
+      anno.setanno(node, BODY_DEFINITELY_RETURNS, True)
 
+    node.orelse, orelse_definitely_returns = self._visit_statement_block(
+        node, node.orelse)
+    if orelse_definitely_returns:
+      anno.setanno(node, ORELSE_DEFINITELY_RETURNS, True)
 
-class FoldElse(BodyVisitor):
-
-  def visit_nodelist(self, nodelist):
-    for i in range(len(nodelist)):
-      node = nodelist[i]
-      if isinstance(node, gast.If):
-        true_branch_returns = isinstance(node.body[-1], gast.Return)
-        false_branch_returns = len(node.orelse) and isinstance(
-            node.orelse[-1], gast.Return)
-        # If the last node in the if body is a return,
-        # then every line after this if statement effectively
-        # belongs in the else.
-        if true_branch_returns and not false_branch_returns:
-          for j in range(i + 1, len(nodelist)):
-            nodelist[i].orelse.append(ast_util.copy_clean(nodelist[j]))
-          if nodelist[i + 1:]:
-            self.changes_made = True
-          return nodelist[:i + 1]
-        elif not true_branch_returns and false_branch_returns:
-          for j in range(i + 1, len(nodelist)):
-            nodelist[i].body.append(ast_util.copy_clean(nodelist[j]))
-          if nodelist[i + 1:]:
-            self.changes_made = True
-          return nodelist[:i + 1]
-        elif true_branch_returns and false_branch_returns:
-          if nodelist[i + 1:]:
-            raise ValueError(
-                'Unreachable code after conditional where both branches return.'
-            )
-          return nodelist
-      elif isinstance(node, gast.Return) and nodelist[i + 1:]:
-        raise ValueError(
-            'Cannot have statements after a return in the same basic block')
-    return nodelist
-
-
-def contains_return(node):
-  for n in gast.walk(node):
-    if isinstance(n, gast.Return):
-      return True
-  return False
-
-
-class LiftReturn(converter.Base):
-  """Move return statements out of If and With blocks."""
-
-  def __init__(self, ctx):
-    super(LiftReturn, self).__init__(ctx)
-    self.changes_made = False
-    self.common_return_name = None
+    if body_definitely_returns and orelse_definitely_returns:
+      self.state[_Block].definitely_returns = True
 
-  def visit_If(self, node):
-    # Depth-first traversal of if statements
-    node = self.generic_visit(node)
-
-    # We check if both branches return, and if so, lift the return out of the
-    # conditional. We don't enforce that the true and false branches either
-    # both return or both do not, because FoldElse might move a return
-    # into a branch after this transform completes. FoldElse and LiftReturn
-    # are alternately run until the code reaches a fixed point.
-    true_branch_returns = isinstance(node.body[-1], gast.Return)
-    false_branch_returns = len(node.orelse) and isinstance(
-        node.orelse[-1], gast.Return)
-    if true_branch_returns and false_branch_returns:
-      node.body[-1] = templates.replace(
-          'a = b', a=self.common_return_name, b=node.body[-1].value)[0]
-      node.orelse[-1] = templates.replace(
-          'a = b', a=self.common_return_name, b=node.orelse[-1].value)[0]
-      return_node = templates.replace('return a', a=self.common_return_name)[0]
-      self.changes_made = True
-      return [node, return_node]
-    else:
-      return node
-
-  def visit_With(self, node):
-    # Depth-first traversal of syntax
-    node = self.generic_visit(node)
-
-    # If the with statement returns, lift the return
-    if isinstance(node.body[-1], gast.Return):
-      node.body[-1] = templates.replace(
-          'a = b', a=self.common_return_name, b=node.body[-1].value)[0]
-      return_node = templates.replace('return a', a=self.common_return_name)[0]
-      node = self.generic_visit(node)
-      self.changes_made = True
-      return [node, return_node]
-    else:
-      return node
+    return node
 
   def visit_FunctionDef(self, node):
-    # Ensure we're doing depth-first traversal
-    last_return_name = self.common_return_name
-    body_scope = anno.getanno(node, NodeAnno.BODY_SCOPE)
-    referenced_names = body_scope.referenced
-    self.common_return_name = self.ctx.namer.new_symbol('return_',
-                                                        referenced_names)
-    node = self.generic_visit(node)
-    self.common_return_name = last_return_name
+    node.args = self.visit(node.args)
+    node.body, _ = self._visit_statement_block(node, node.body)
     return node
 
 
-class DetectReturnInUnsupportedControlFlow(gast.NodeVisitor):
-  """Throws an error if code returns inside loops or try/except."""
+class _Return(object):
+
+  def __init__(self):
+    self.used = False
+    self.create_guard = False
+    self.guard_created = False
+
+  def __repr__(self):
+    return 'used: {}'.format(
+        self.used)
+
 
-  # First, throw an error if we detect a return statement in a loop.
-  # TODO(alexbw): we need to learn to handle returns inside a loop,
-  # but don't currently have the TF constructs to do so (need something
-  # that looks vaguely like a goto).
+class _Function(object):
 
   def __init__(self):
-    self.cant_return = False
-    self.function_level = 0
-    super(DetectReturnInUnsupportedControlFlow, self).__init__()
+    self.do_return_var_name = None
+    self.retval_var_name = None
 
-  def visit_While(self, node):
-    self.cant_return = True
-    self.generic_visit(node)
-    self.cant_return = False
+  def __repr__(self):
+    return 'return control: {}, return value: {}'.format(
+        self.do_return_var_name, self.retval_var_name)
 
-  def visit_For(self, node):
-    self.cant_return = True
-    self.generic_visit(node)
-    self.cant_return = False
 
-  def visit_Try(self, node):
-    self.cant_return = True
-    self.generic_visit(node)
-    self.cant_return = False
+class ReturnStatementsTransformer(converter.Base):
+  """Lowers return statements into variables and conditionals.
 
-  def visit_FunctionDef(self, node):
-    if not self.function_level:
-      self.function_level += 1
-      self.generic_visit(node)
-      self.function_level -= 1
+  Specifically, the following pattern:
 
-  def visit_Return(self, node):
-    if self.cant_return:
-      raise ValueError(
-          '`return` statements are not supported in loops. '
-          'Try assigning to a variable in the while loop, and returning '
-          'outside of the loop')
+      <block 1>
+      return val
+      <block 2>
 
+  is converted to:
 
-class DetectReturnInConditional(gast.NodeVisitor):
-  """Assert that no return statements are present in conditionals."""
+      do_return = False
+      retval = None
 
-  def __init__(self):
-    self.cant_return = False
-    self.function_level = 0
-    super(DetectReturnInConditional, self).__init__()
+      <block 1>
 
-  def visit_If(self, node):
-    self.cant_return = True
-    self.generic_visit(node)
-    self.cant_return = False
+      do_return = True
+      retval = val
 
-  def visit_FunctionDef(self, node):
-    if not self.function_level:
-      self.function_level += 1
-      self.generic_visit(node)
-      self.function_level -= 1
+      if not do_return:
+        <block 2>
 
-  def visit_Return(self, node):
-    if self.cant_return:
-      raise ValueError(
-          'After transforms, a conditional contained a `return `statement, '
-          'which is not allowed. This is a bug, and should not happen.')
+      return retval
 
+  The conversion adjusts loops as well:
 
-class DetectReturnInFunctionDef(gast.NodeVisitor):
+      <block 1>
+      while cond:
+        <block 2>
+        return retval
 
-  def visit_FunctionDef(self, node):
-    self.generic_visit(node)
-    if not contains_return(node):
-      raise ValueError(
-          'Each function definition should contain at least one return.')
-
-
-def transform(node, ctx):
-  """Ensure a function has only a single return.
-
-  This transforms an AST node with multiple returns successively into containing
-  only a single return node.
-  There are a few restrictions on what we can handle:
-   - An AST being transformed must contain at least one return.
-   - No returns allowed in loops. We have to know the type of the return value,
-   and we currently don't have either a type inference system to discover it,
-   nor do we have a mechanism for late type binding in TensorFlow.
-   - After all transformations are finished, a Return node is not allowed inside
-   control flow. If we were unable to move a return outside of control flow,
-   this is an error.
-
-  Args:
-     node: ast.AST
-     ctx: converter.EntityContext
-
-  Returns:
-     new_node: an AST with a single return value
-
-  Raises:
-    ValueError: if the AST is structured so that we can't perform the
-   transform.
+  is converted to:
+
+      <block 1>
+      while not do_return and cond:
+        <block 2>
+        do_return = True
+        retval = val
   """
-  # Make sure that the function has at least one return statement
-  # TODO(alexbw): turning off this assertion for now --
-  # we need to not require this in e.g. class constructors.
-  # DetectReturnInFunctionDef().visit(node)
 
-  # Make sure there's no returns in unsupported locations (loops, try/except)
-  DetectReturnInUnsupportedControlFlow().visit(node)
+  def __init__(self, ctx, default_to_null_return):
+    super(ReturnStatementsTransformer, self).__init__(ctx)
+    self.default_to_null_return = default_to_null_return
+
+  def visit_Return(self, node):
+    self.state[_Return].used = True
+
+    retval = node.value if node.value else parser.parse_expresison('None')
+
+    template = """
+      do_return_var_name = True
+      retval_var_name = retval
+    """
+    node = templates.replace(
+        template,
+        do_return_var_name=self.state[_Function].do_return_var_name,
+        retval_var_name=self.state[_Function].retval_var_name,
+        retval=retval)
+
+    return node
+
+  def _postprocess_statement(self, node):
+    # Example of how the state machine below works:
+    #
+    #   1| stmt           # State: _Return.used = False
+    #    |                # Action: none
+    #   3| return         # State: _Return.used = True,
+    #    |                #        _Return.guard_created = False,
+    #    |                #        _Return.create_guard = False
+    #    |                # Action: _Return.create_guard = True
+    #   4| stmt           # State: _Return.used = True,
+    #    |                #        _Return.guard_created = False,
+    #    |                #        _Return.create_guard = True
+    #    |                # Action: create `if not return_used`,
+    #    |                #         set _Return.guard_created = True
+    #   5| stmt           # State: _Return.used = True,
+    #    |                #        _Return.guard_created = True
+    #    |                # Action: none (will be wrapped under previously
+    #    |                #         created if node)
+    if self.state[_Return].used:
+      if self.state[_Return].guard_created:
+        return node, None
+
+      elif not self.state[_Return].create_guard:
+        self.state[_Return].create_guard = True
+        return node, None
+
+      elif (not self.state[_Return].guard_created and
+            self.state[_Return].create_guard):
+        self.state[_Return].guard_created = True
+        template = """
+          if ag__.not_(do_return_var_name):
+            original_node
+        """
+        cond, = templates.replace(
+            template,
+            do_return_var_name=self.state[_Function].do_return_var_name,
+            original_node=node)
+        return cond, cond.body
+
+      else:
+        assert False, 'should handle all states'
+
+    return node, None
+
+  def _visit_statement_block(self, node, nodes):
+    self.state[_Return].enter()
+    nodes = self.visit_block(nodes, after_visit=self._postprocess_statement)
+    return_used = self.state[_Return].used
+    self.state[_Return].exit()
+    if return_used:
+      self.state[_Return].used = True
+    return nodes
+
+  def visit_While(self, node):
+    node.test = self.visit(node.test)
+
+    # Add the check for return to the loop condition.
+    node.body = self._visit_statement_block(node, node.body)
+    if self.state[_Return].used:
+      node.test = templates.replace_as_expression(
+          'ag__.and_(lambda: ag__.not_(control_var), lambda: test)',
+          test=node.test,
+          control_var=self.state[_Function].do_return_var_name)
+
+    node.orelse = self._visit_statement_block(node, node.orelse)
+    return node
+
+  def visit_For(self, node):
+    node.iter = self.visit(node.iter)
+    node.target = self.visit(node.target)
+
+    # Add the check for return to the loop condition.
+    node.body = self._visit_statement_block(node, node.body)
+    if self.state[_Return].used:
+      extra_test = anno.getanno(node, 'extra_test', default=None)
+      if extra_test is not None:
+        extra_test = templates.replace_as_expression(
+            'ag__.and_(lambda: ag__.not_(control_var), lambda: extra_test)',
+            extra_test=extra_test,
+            control_var=self.state[_Function].do_return_var_name)
+      else:
+        extra_test = templates.replace_as_expression(
+            'ag__.not_(control_var)',
+            control_var=self.state[_Function].do_return_var_name)
+      anno.setanno(node, 'extra_test', extra_test)
+
+    node.orelse = self._visit_statement_block(node, node.orelse)
+    return node
+
+  def visit_If(self, node):
+    node.test = self.visit(node.test)
+    node.body = self._visit_statement_block(node, node.body)
+    node.orelse = self._visit_statement_block(node, node.orelse)
+    return node
+
+  def visit_FunctionDef(self, node):
+    self.state[_Function].enter()
+    self.state[_Return].enter()
+
+    scope = anno.getanno(node, NodeAnno.BODY_SCOPE)
+    do_return_var_name = self.ctx.namer.new_symbol(
+        'do_return', scope.referenced)
+    retval_var_name = self.ctx.namer.new_symbol('retval_', scope.referenced)
+    self.state[_Function].do_return_var_name = do_return_var_name
+    self.state[_Function].retval_var_name = retval_var_name
+
+    converted_body = self._visit_statement_block(node, node.body)
+
+    # Avoid placing statements before any eventual docstring.
+    # TODO(mdan): Should a docstring even be included in the output?
+    docstring = None
+    if converted_body:
+      if (isinstance(converted_body[0], gast.Expr) and
+          isinstance(converted_body[0].value, gast.Str)):
+        docstring = converted_body[0]
+        converted_body = converted_body[1:]
+
+    if self.state[_Return].used:
+      if self.default_to_null_return:
+        template = """
+          do_return_var_name = False
+          retval_var_name = None
+          body
+          return retval_var_name
+        """
+      else:
+        template = """
+          body
+          return retval_var_name
+        """
+      node.body = templates.replace(
+          template,
+          body=converted_body,
+          do_return_var_name=do_return_var_name,
+          retval_var_name=retval_var_name)
+
+      if docstring:
+        node.body.insert(0, docstring)
+
+    self.state[_Return].exit()
+    self.state[_Function].exit()
+    return node
 
-  while True:
 
-    # Try to lift all returns out of if statements and with blocks
-    lr = LiftReturn(ctx)
-    node = lr.visit(node)
-    changes_made = lr.changes_made
-    fe = FoldElse(ctx)
-    node = fe.visit(node)
-    changes_made = changes_made or fe.changes_made
+def transform(node, ctx, default_to_null_return=True):
+  """Ensure a function has only a single return."""
+  # Note: Technically, these two could be merged into a single walk, but
+  # keeping them separate helps with readability.
 
-    if not changes_made:
-      break
+  node = ConditionalReturnRewriter(ctx).visit(node)
 
-  # Make sure we've scrubbed all returns from conditionals
-  DetectReturnInConditional().visit(node)
+  transformer = ReturnStatementsTransformer(
+      ctx, default_to_null_return=default_to_null_return)
+  node = transformer.visit(node)
 
   return node
diff --git a/tensorflow/python/autograph/converters/return_statements_test.py b/tensorflow/python/autograph/converters/return_statements_test.py
index 762fbc6f60..2b160d5ac7 100644
--- a/tensorflow/python/autograph/converters/return_statements_test.py
+++ b/tensorflow/python/autograph/converters/return_statements_test.py
@@ -49,17 +49,16 @@ class SingleReturnTest(converter_testing.TestCase):
     self.assertTransformedEquivalent(test_fn, 2)
     self.assertTransformedEquivalent(test_fn, -2)
 
-  def test_missing_orelse(self):
+  def test_missing_else(self):
 
     def test_fn(x):
       if x > 0:
         return x
 
-    node, ctx = self.prepare(test_fn, {})
-    with self.assertRaises(ValueError):
-      return_statements.transform(node, ctx)
+    self.assertTransformedEquivalent(test_fn, 2)
+    self.assertTransformedEquivalent(test_fn, -2)
 
-  def test_missing_orelse_recovrable(self):
+  def test_missing_else_then_default(self):
 
     def test_fn(x):
       if x > 0:
@@ -69,7 +68,7 @@ class SingleReturnTest(converter_testing.TestCase):
     self.assertTransformedEquivalent(test_fn, 2)
     self.assertTransformedEquivalent(test_fn, -2)
 
-  def test_missing_branch_return_recoverable(self):
+  def test_else_only_then_default(self):
 
     def test_fn(x):
       if x < 0:
@@ -136,7 +135,7 @@ class SingleReturnTest(converter_testing.TestCase):
 
     self.assertTransformedEquivalent(test_fn, 2)
 
-  def test_nested_functions(self):
+  def test_nested_function(self):
 
     def test_fn(x):
 
@@ -151,7 +150,7 @@ class SingleReturnTest(converter_testing.TestCase):
     self.assertTransformedEquivalent(test_fn, 2)
     self.assertTransformedEquivalent(test_fn, -2)
 
-  def test_nested_functions_in_control_flow(self):
+  def test_nested_function_in_control_flow(self):
 
     def test_fn(x):
 
@@ -163,16 +162,30 @@ class SingleReturnTest(converter_testing.TestCase):
     self.assertTransformedEquivalent(test_fn, 2)
     self.assertTransformedEquivalent(test_fn, -2)
 
-  def test_loop(self):
+  def test_for_loop(self):
 
-    def test_fn(x):
-      for _ in range(10):
-        return x
-      return x
+    def test_fn(n):
+      for _ in range(n):
+        return 1
 
-    node, ctx = self.prepare(test_fn, {})
-    with self.assertRaises(ValueError):
-      return_statements.transform(node, ctx)
+    self.assertTransformedEquivalent(test_fn, 2)
+    self.assertTransformedEquivalent(test_fn, 0)
+
+  def test_while_loop(self):
+
+    def test_fn(n):
+      i = 0
+      s = 0
+      while i < n:
+        i += 1
+        s += i
+        if s > 4:
+          return s
+      return -1
+
+    self.assertTransformedEquivalent(test_fn, 0)
+    self.assertTransformedEquivalent(test_fn, 2)
+    self.assertTransformedEquivalent(test_fn, 4)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/autograph/impl/conversion_test.py b/tensorflow/python/autograph/impl/conversion_test.py
index 9a4fbdad8c..cd893e3ff1 100644
--- a/tensorflow/python/autograph/impl/conversion_test.py
+++ b/tensorflow/python/autograph/impl/conversion_test.py
@@ -96,7 +96,6 @@ class ConversionTest(test.TestCase):
     f_node = program_ctx.dependency_cache[f][0]
     g_node = program_ctx.dependency_cache[g][0]
     self.assertEqual('tf__f', f_node.name)
-    self.assertEqual('tf__g', f_node.body[0].body[0].body[0].value.func.id)
     self.assertEqual('tf__g', g_node.name)
 
   def test_entity_to_graph_class_hierarchy(self):
diff --git a/tensorflow/python/autograph/pyct/templates.py b/tensorflow/python/autograph/pyct/templates.py
index 43279b3ca0..0f0b8613ac 100644
--- a/tensorflow/python/autograph/pyct/templates.py
+++ b/tensorflow/python/autograph/pyct/templates.py
@@ -108,6 +108,7 @@ class ReplaceTransformer(gast.NodeTransformer):
         anno.Basic.ORIGIN,
         anno.Basic.SKIP_PROCESSING,
         anno.Static.ORIG_DEFINITIONS,
+        'extra_test',
     }
 
   def _prepare_replacement(self, replaced, key):
diff --git a/tensorflow/python/autograph/pyct/transformer.py b/tensorflow/python/autograph/pyct/transformer.py
index f32d609388..90213da782 100644
--- a/tensorflow/python/autograph/pyct/transformer.py
+++ b/tensorflow/python/autograph/pyct/transformer.py
@@ -305,11 +305,17 @@ class Base(gast.NodeTransformer):
     return self._local_scope_state[-1].get(name, default)
 
   def debug_print(self, node):
-    """Helper method useful for debugging."""
+    """Helper method useful for debugging. Prints the AST."""
     if __debug__:
       print(pretty_printer.fmt(node))
     return node
 
+  def debug_print_src(self, node):
+    """Helper method useful for debugging. Prints the AST as code."""
+    if __debug__:
+      print(compiler.ast_to_source(node))
+    return node
+
   def create_assignment(self, target, expression):
     template = """
       target = expression
-- 
GitLab


From e98a897f188af80d1fdc04676283aaa649bcffbe Mon Sep 17 00:00:00 2001
From: Shining Sun <shiningsun@google.com>
Date: Mon, 28 Jan 2019 19:37:42 -0800
Subject: [PATCH 1450/2345] Move one_device_strategy from contrib to core.

PiperOrigin-RevId: 231333409
---
 tensorflow/contrib/distribute/python/BUILD    |  16 +-
 .../contrib/distribute/python/monitor_test.py |   2 +-
 .../distribute/python/one_device_strategy.py  | 193 +---------------
 .../python/one_device_strategy_test.py        |   2 +-
 tensorflow/python/distribute/BUILD            |  19 ++
 .../python/distribute/one_device_strategy.py  | 214 ++++++++++++++++++
 6 files changed, 240 insertions(+), 206 deletions(-)
 create mode 100644 tensorflow/python/distribute/one_device_strategy.py

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index fca1e29df4..b43cd04c5e 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -127,17 +127,7 @@ py_library(
     srcs = ["one_device_strategy.py"],
     visibility = ["//tensorflow:internal"],
     deps = [
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:dtypes",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python/distribute:distribute_lib",
-        "//tensorflow/python/distribute:input_lib",
-        "//tensorflow/python/distribute:numpy_dataset",
-        "//tensorflow/python/distribute:reduce_util",
-        "//tensorflow/python/distribute:values",
-        "//tensorflow/python/eager:context",
-        "@six_archive//:six",
+        "//tensorflow/python/distribute:one_device_strategy",
     ],
 )
 
@@ -216,9 +206,9 @@ py_test(
     srcs = ["one_device_strategy_test.py"],
     srcs_version = "PY2AND3",
     deps = [
-        ":one_device_strategy",
         ":strategy_test_lib",
         "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python/distribute:one_device_strategy",
         "//tensorflow/python/eager:test",
     ],
 )
@@ -502,10 +492,10 @@ cuda_py_test(
     additional_deps = [
         ":combinations",
         ":monitor",
-        ":one_device_strategy",
         ":single_loss_example",
         "@absl_py//absl/testing:parameterized",
         "//third_party/py/numpy",
+        "//tensorflow/python/distribute:one_device_strategy",
         "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:test",
         "//tensorflow/python:framework_ops",
diff --git a/tensorflow/contrib/distribute/python/monitor_test.py b/tensorflow/contrib/distribute/python/monitor_test.py
index 16be839e1d..c0651610ca 100644
--- a/tensorflow/contrib/distribute/python/monitor_test.py
+++ b/tensorflow/contrib/distribute/python/monitor_test.py
@@ -23,9 +23,9 @@ import numpy
 
 from tensorflow.contrib.distribute.python import combinations
 from tensorflow.contrib.distribute.python import monitor as monitor_lib
-from tensorflow.contrib.distribute.python import one_device_strategy
 from tensorflow.contrib.distribute.python.single_loss_example import single_loss_example
 from tensorflow.python.client import session
+from tensorflow.python.distribute import one_device_strategy
 from tensorflow.python.eager import context
 from tensorflow.python.eager import test
 from tensorflow.python.framework import ops
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py
index 24d6a443fe..13a501394e 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy.py
@@ -18,195 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.python.distribute import device_util
-from tensorflow.python.distribute import distribute_lib
-from tensorflow.python.distribute import input_lib
-from tensorflow.python.distribute import numpy_dataset
-from tensorflow.python.distribute import values
-from tensorflow.python.framework import constant_op
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.util import nest
+from tensorflow.python.distribute import one_device_strategy
 
-
-# TODO(josh11b): Replace asserts in this file with if ...: raise ...
-
-
-class OneDeviceStrategy(distribute_lib.DistributionStrategy):
-  """A distribution strategy for running on a single device."""
-  # TODO(josh11b): Do we wrap values in types to generate errors if you are
-  # doing something that won't work with other DistributionStrategy
-  # implementations?
-
-  def __init__(self, device):
-    super(OneDeviceStrategy, self).__init__(OneDeviceExtended(self, device))
-
-
-class OneDeviceExtended(distribute_lib.DistributionStrategyExtended):
-  """Implementation of OneDeviceStrategy."""
-
-  def __init__(self, container_strategy, device):
-    super(OneDeviceExtended, self).__init__(container_strategy)
-    self._device = device
-    self._default_device = device
-    self._input_device = device_util.canonicalize("/device:CPU:0")
-    worker_device_pairs = [(self._input_device, [self._device])]
-    device_map = values.SingleDeviceMap(device)
-    self._input_workers = input_lib.InputWorkers(
-        device_map, worker_device_pairs)
-
-  def _create_variable(self, next_creator, *args, **kwargs):
-    colocate_with = kwargs.pop("colocate_with", None)
-    if colocate_with is None:
-      with ops.device(self._device):
-        return next_creator(*args, **kwargs)
-    with ops.colocate_with(colocate_with):
-      return next_creator(*args, **kwargs)
-
-  def _validate_colocate_with_variable(self, colocate_with_variable):
-    values.validate_colocate(colocate_with_variable, self)
-
-  def _make_dataset_iterator(self, dataset):
-    """Make iterator from dataset without splitting the batch."""
-    return input_lib.DatasetIterator(dataset, self._input_workers)
-
-  def _make_input_fn_iterator(
-      self,
-      input_fn,
-      replication_mode=distribute_lib.InputReplicationMode.PER_WORKER):
-    return input_lib.InputFunctionIterator(
-        input_fn, self._input_workers, [distribute_lib.InputContext()])
-
-  def _experimental_make_numpy_dataset(self, numpy_input, session):
-    return numpy_dataset.one_host_numpy_dataset(
-        numpy_input, self._input_device, session)
-
-  def _broadcast_to(self, tensor, destinations):
-    del destinations
-    return tensor
-
-  # TODO(priyag): Deal with OutOfRange errors  once b/111349762 is fixed.
-  def _experimental_run_steps_on_iterator(self, fn, iterator, iterations,
-                                          initial_loop_values=None):
-    if initial_loop_values is None:
-      initial_loop_values = {}
-    initial_loop_values = nest.flatten(initial_loop_values)
-
-    ctx = input_lib.MultiStepContext()
-    def body(i, *args):
-      """A wrapper around `fn` to create the while loop body."""
-      del args
-      fn_result = fn(ctx, iterator.get_next())
-      flat_last_step_outputs = nest.flatten(ctx.last_step_outputs)
-      with ops.control_dependencies([fn_result]):
-        return [i + 1] + flat_last_step_outputs
-
-    # We capture the control_flow_context at this point, before we run `fn`
-    # inside a while_loop. This is useful in cases where we might need to exit
-    # these contexts and get back to the outer context to do some things, for
-    # e.g. create an op which should be evaluated only once at the end of the
-    # loop on the host. One such usage is in creating metrics' value op.
-    self._outer_control_flow_context = (
-        ops.get_default_graph()._get_control_flow_context())  # pylint: disable=protected-access
-
-    # TODO(priyag): Use max_iterations instead of an explicit counter.
-    cond = lambda i, *args: i < iterations
-    i = constant_op.constant(0)
-    loop_result = control_flow_ops.while_loop(
-        cond, body, [i] + initial_loop_values, name="",
-        parallel_iterations=1, back_prop=False, swap_memory=False,
-        return_same_structure=True)
-    del self._outer_control_flow_context
-
-    ctx.run_op = control_flow_ops.group(loop_result)
-
-    # Convert the last_step_outputs from a list to the original dict structure
-    # of last_step_outputs.
-    last_step_tensor_outputs = loop_result[1:]
-    last_step_tensor_outputs_dict = nest.pack_sequence_as(
-        ctx.last_step_outputs, last_step_tensor_outputs)
-
-    ctx._set_last_step_outputs(last_step_tensor_outputs_dict)  # pylint: disable=protected-access
-    return ctx
-
-  def _call_for_each_replica(self, fn, args, kwargs):
-    strategy = self._container_strategy()
-    with ops.device(self._device), _OneDeviceReplicaContext(strategy):
-      return fn(*args, **kwargs)
-
-  def _reduce_to(self, reduce_op, value, destinations):
-    del reduce_op, destinations
-    return value
-
-  def _update(self, var, fn, args, kwargs, group):
-    # The implementations of _update() and _update_non_slot() are identical
-    # except _update() passes `var` as the first argument to `fn()`.
-    return self._update_non_slot(var, fn, (var,) + tuple(args), kwargs, group)
-
-  def _update_non_slot(self, colocate_with, fn, args, kwargs, group):
-    del colocate_with
-    with ops.device(self._device), distribute_lib.UpdateContext(self._device):
-      result = fn(*args, **kwargs)
-      if group:
-        return result
-      else:
-        return nest.map_structure(self._unwrap, result)
-
-  def read_var(self, replica_local_var):
-    """Read the aggregate value of a replica-local variable."""
-    return array_ops.identity(replica_local_var)
-
-  def _unwrap(self, value):
-    return (value,)
-
-  def value_container(self, value):
-    return value
-
-  @property
-  def _num_replicas_in_sync(self):
-    return 1
-
-  @property
-  def worker_devices(self):
-    return (self._device,)
-
-  @property
-  def parameter_devices(self):
-    return (self._device,)
-
-  def non_slot_devices(self, var_list):
-    del var_list
-    return (self._device,)
-
-  @property
-  def experimental_should_init(self):
-    return True
-
-  @property
-  def should_checkpoint(self):
-    return True
-
-  @property
-  def should_save_summary(self):
-    return True
-
-  # TODO(priyag): Delete this once all strategies use global batch size.
-  @property
-  def _global_batch_size(self):
-    """Global and per-replica batching are equivalent for OneDeviceStrategy."""
-    return True
-
-
-class _OneDeviceReplicaContext(distribute_lib.ReplicaContext):
-  """ReplicaContext for OneDeviceStrategy."""
-
-  def __init__(self, strategy):
-    zero = constant_op.constant(0, dtypes.int32)
-    distribute_lib.ReplicaContext.__init__(
-        self, strategy, replica_id_in_sync_group=zero)
-
-  @property
-  def devices(self):
-    return self._strategy.extended.worker_devices
+OneDeviceStrategy = one_device_strategy.OneDeviceStrategy
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy_test.py b/tensorflow/contrib/distribute/python/one_device_strategy_test.py
index f81466a6c7..683ed86f4f 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy_test.py
@@ -18,9 +18,9 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.distribute.python import one_device_strategy
 from tensorflow.contrib.distribute.python import strategy_test_lib
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.distribute import one_device_strategy
 from tensorflow.python.eager import test
 from tensorflow.python.framework import test_util
 
diff --git a/tensorflow/python/distribute/BUILD b/tensorflow/python/distribute/BUILD
index 36e8557126..77ec4a5be3 100644
--- a/tensorflow/python/distribute/BUILD
+++ b/tensorflow/python/distribute/BUILD
@@ -264,6 +264,25 @@ py_library(
     ],
 )
 
+py_library(
+    name = "one_device_strategy",
+    srcs = ["one_device_strategy.py"],
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        ":distribute_lib",
+        ":input_lib",
+        ":numpy_dataset",
+        ":reduce_util",
+        ":values",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python/eager:context",
+        "@six_archive//:six",
+    ],
+)
+
 py_library(
     name = "multi_worker_util",
     srcs = [
diff --git a/tensorflow/python/distribute/one_device_strategy.py b/tensorflow/python/distribute/one_device_strategy.py
new file mode 100644
index 0000000000..b2d2f03618
--- /dev/null
+++ b/tensorflow/python/distribute/one_device_strategy.py
@@ -0,0 +1,214 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Class OneDeviceStrategy implementing DistributionStrategy."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.distribute import device_util
+from tensorflow.python.distribute import distribute_lib
+from tensorflow.python.distribute import input_lib
+from tensorflow.python.distribute import numpy_dataset
+from tensorflow.python.distribute import values
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.util import nest
+
+
+# TODO(josh11b): Replace asserts in this file with if ...: raise ...
+
+
+class OneDeviceStrategy(distribute_lib.DistributionStrategy):
+  """A distribution strategy for running on a single device."""
+  # TODO(josh11b): Do we wrap values in types to generate errors if you are
+  # doing something that won't work with other DistributionStrategy
+  # implementations?
+
+  def __init__(self, device):
+    super(OneDeviceStrategy, self).__init__(OneDeviceExtended(self, device))
+
+
+class OneDeviceExtended(distribute_lib.DistributionStrategyExtended):
+  """Implementation of OneDeviceStrategy."""
+
+  def __init__(self, container_strategy, device):
+    super(OneDeviceExtended, self).__init__(container_strategy)
+    self._device = device
+    self._default_device = device
+    self._input_device = device_util.canonicalize("/device:CPU:0")
+    worker_device_pairs = [(self._input_device, [self._device])]
+    device_map = values.SingleDeviceMap(device)
+    self._input_workers = input_lib.InputWorkers(
+        device_map, worker_device_pairs)
+
+  def _create_variable(self, next_creator, *args, **kwargs):
+    colocate_with = kwargs.pop("colocate_with", None)
+    if colocate_with is None:
+      with ops.device(self._device):
+        return next_creator(*args, **kwargs)
+    with ops.colocate_with(colocate_with):
+      return next_creator(*args, **kwargs)
+
+  def _validate_colocate_with_variable(self, colocate_with_variable):
+    values.validate_colocate(colocate_with_variable, self)
+
+  def _make_dataset_iterator(self, dataset):
+    """Make iterator from dataset without splitting the batch."""
+    # Note that split_batch_by argument is not passed because it is always 1 in
+    # this strategy, and adding it adds unnecessary overhead to the dataset.
+    return input_lib.DatasetIterator(dataset, self._input_workers)
+
+  def _make_input_fn_iterator(
+      self,
+      input_fn,
+      replication_mode=distribute_lib.InputReplicationMode.PER_WORKER):
+    return input_lib.InputFunctionIterator(
+        input_fn, self._input_workers, [distribute_lib.InputContext()])
+
+  def _experimental_make_numpy_dataset(self, numpy_input, session):
+    return numpy_dataset.one_host_numpy_dataset(
+        numpy_input, self._input_device, session)
+
+  def _broadcast_to(self, tensor, destinations):
+    del destinations
+    return tensor
+
+  # TODO(priyag): Deal with OutOfRange errors  once b/111349762 is fixed.
+  def _experimental_run_steps_on_iterator(self, fn, iterator, iterations,
+                                          initial_loop_values=None):
+    if initial_loop_values is None:
+      initial_loop_values = {}
+    initial_loop_values = nest.flatten(initial_loop_values)
+
+    ctx = input_lib.MultiStepContext()
+    def body(i, *args):
+      """A wrapper around `fn` to create the while loop body."""
+      del args
+      fn_result = fn(ctx, iterator.get_next())
+      flat_last_step_outputs = nest.flatten(ctx.last_step_outputs)
+      with ops.control_dependencies([fn_result]):
+        return [i + 1] + flat_last_step_outputs
+
+    # We capture the control_flow_context at this point, before we run `fn`
+    # inside a while_loop. This is useful in cases where we might need to exit
+    # these contexts and get back to the outer context to do some things, for
+    # e.g. create an op which should be evaluated only once at the end of the
+    # loop on the host. One such usage is in creating metrics' value op.
+    self._outer_control_flow_context = (
+        ops.get_default_graph()._get_control_flow_context())  # pylint: disable=protected-access
+
+    # TODO(priyag): Use max_iterations instead of an explicit counter.
+    cond = lambda i, *args: i < iterations
+    i = constant_op.constant(0)
+    loop_result = control_flow_ops.while_loop(
+        cond, body, [i] + initial_loop_values, name="",
+        parallel_iterations=1, back_prop=False, swap_memory=False,
+        return_same_structure=True)
+    del self._outer_control_flow_context
+
+    ctx.run_op = control_flow_ops.group(loop_result)
+
+    # Convert the last_step_outputs from a list to the original dict structure
+    # of last_step_outputs.
+    last_step_tensor_outputs = loop_result[1:]
+    last_step_tensor_outputs_dict = nest.pack_sequence_as(
+        ctx.last_step_outputs, last_step_tensor_outputs)
+
+    ctx._set_last_step_outputs(last_step_tensor_outputs_dict)  # pylint: disable=protected-access
+    return ctx
+
+  def _call_for_each_replica(self, fn, args, kwargs):
+    strategy = self._container_strategy()
+    with ops.device(self._device), _OneDeviceReplicaContext(strategy):
+      return fn(*args, **kwargs)
+
+  def _reduce_to(self, reduce_op, value, destinations):
+    del reduce_op, destinations
+    return value
+
+  def _update(self, var, fn, args, kwargs, group):
+    # The implementations of _update() and _update_non_slot() are identical
+    # except _update() passes `var` as the first argument to `fn()`.
+    return self._update_non_slot(var, fn, (var,) + tuple(args), kwargs, group)
+
+  def _update_non_slot(self, colocate_with, fn, args, kwargs, group):
+    del colocate_with
+    with ops.device(self._device), distribute_lib.UpdateContext(self._device):
+      result = fn(*args, **kwargs)
+      if group:
+        return result
+      else:
+        return nest.map_structure(self._unwrap, result)
+
+  def read_var(self, replica_local_var):
+    """Read the aggregate value of a replica-local variable."""
+    return array_ops.identity(replica_local_var)
+
+  def _unwrap(self, value):
+    return (value,)
+
+  def value_container(self, value):
+    return value
+
+  @property
+  def _num_replicas_in_sync(self):
+    return 1
+
+  @property
+  def worker_devices(self):
+    return (self._device,)
+
+  @property
+  def parameter_devices(self):
+    return (self._device,)
+
+  def non_slot_devices(self, var_list):
+    del var_list
+    return (self._device,)
+
+  @property
+  def experimental_should_init(self):
+    return True
+
+  @property
+  def should_checkpoint(self):
+    return True
+
+  @property
+  def should_save_summary(self):
+    return True
+
+  # TODO(priyag): Delete this once all strategies use global batch size.
+  @property
+  def _global_batch_size(self):
+    """Global and per-replica batching are equivalent for OneDeviceStrategy."""
+    return True
+
+
+class _OneDeviceReplicaContext(distribute_lib.ReplicaContext):
+  """ReplicaContext for OneDeviceStrategy."""
+
+  def __init__(self, strategy):
+    zero = constant_op.constant(0, dtypes.int32)
+    distribute_lib.ReplicaContext.__init__(
+        self, strategy, replica_id_in_sync_group=zero)
+
+  @property
+  def devices(self):
+    return self._strategy.extended.worker_devices
-- 
GitLab


From 601921925d6c0dc413af1215e14b967468a05162 Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Mon, 28 Jan 2019 19:41:44 -0800
Subject: [PATCH 1451/2345] Renaming V2 shared embedding columns so as to not
 have a collision with the V1 name.

PiperOrigin-RevId: 231333721
---
 tensorflow/python/feature_column/feature_column_v2.py          | 2 +-
 tensorflow/tools/api/golden/v2/tensorflow.feature_column.pbtxt | 2 +-
 tensorflow/tools/compatibility/renames_v2.py                   | 1 +
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/feature_column/feature_column_v2.py b/tensorflow/python/feature_column/feature_column_v2.py
index 7c28fe2467..a94245612b 100644
--- a/tensorflow/python/feature_column/feature_column_v2.py
+++ b/tensorflow/python/feature_column/feature_column_v2.py
@@ -1091,7 +1091,7 @@ def shared_embedding_columns(categorical_columns,
   return result
 
 
-@tf_export('feature_column.shared_embedding_columns', v1=[])
+@tf_export('feature_column.shared_embeddings', v1=[])
 def shared_embedding_columns_v2(categorical_columns,
                                 dimension,
                                 combiner='mean',
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.feature_column.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.feature_column.pbtxt
index 3aadd7dc34..d6d5fc903f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.feature_column.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.feature_column.pbtxt
@@ -41,7 +41,7 @@ tf_module {
     argspec: "args=[\'key\', \'shape\', \'default_value\', \'dtype\', \'normalizer_fn\'], varargs=None, keywords=None, defaults=[\'(1,)\', \'None\', \"<dtype: \'float32\'>\", \'None\'], "
   }
   member_method {
-    name: "shared_embedding_columns"
+    name: "shared_embeddings"
     argspec: "args=[\'categorical_columns\', \'dimension\', \'combiner\', \'initializer\', \'shared_embedding_collection_name\', \'ckpt_to_load_from\', \'tensor_name_in_ckpt\', \'max_norm\', \'trainable\'], varargs=None, keywords=None, defaults=[\'mean\', \'None\', \'None\', \'None\', \'None\', \'None\', \'True\'], "
   }
   member_method {
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 7c4b72df2d..5eb3d35527 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -187,6 +187,7 @@ renames = {
     'tf.fake_quant_with_min_max_vars_per_channel_gradient': 'tf.quantization.fake_quant_with_min_max_vars_per_channel_gradient',
     'tf.feature_column.input_layer': 'tf.compat.v1.feature_column.input_layer',
     'tf.feature_column.linear_model': 'tf.compat.v1.feature_column.linear_model',
+    'tf.feature_column.shared_embedding_columns': 'tf.compat.v1.feature_column.shared_embedding_columns',
     'tf.fft': 'tf.signal.fft',
     'tf.fft2d': 'tf.signal.fft2d',
     'tf.fft3d': 'tf.signal.fft3d',
-- 
GitLab


From 9b3e4371e805052c07be4a1a525925aa74f60ca2 Mon Sep 17 00:00:00 2001
From: Vikram Tiwari <vikramtheone1@gmail.com>
Date: Mon, 28 Jan 2019 19:55:31 -0800
Subject: [PATCH 1452/2345] add decode_wav api endpoint

---
 tensorflow/core/api_def/python_api/api_def_DecodeWav.pbtxt | 4 +++-
 tensorflow/tools/api/golden/v1/tensorflow.audio.pbtxt      | 4 ++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/api_def/python_api/api_def_DecodeWav.pbtxt b/tensorflow/core/api_def/python_api/api_def_DecodeWav.pbtxt
index d6fd4691f7..28f4514bd8 100644
--- a/tensorflow/core/api_def/python_api/api_def_DecodeWav.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_DecodeWav.pbtxt
@@ -1,4 +1,6 @@
 op {
   graph_op_name: "DecodeWav"
-  visibility: HIDDEN
+  endpoint {
+    name: "audio.decode_wav"
+  }
 }
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.audio.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.audio.pbtxt
index ce29615f72..db1ff06efb 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.audio.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.audio.pbtxt
@@ -4,4 +4,8 @@ tf_module {
     name: "encode_wav"
     argspec: "args=[\'audio\', \'sample_rate\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "decode_wav"
+    argspec: "args=[\'contents\', \'audio\', \'sample_rate\', \'desired_channels\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
 }
-- 
GitLab


From ad65038c9081b258f70dc6ff6115e829afe23156 Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Mon, 28 Jan 2019 19:45:17 -0800
Subject: [PATCH 1453/2345] Fixing MultiDeviceIterator memory leak issue in
 eager mode.

In Graph mode, we rely on MultiDeviceIteratorHandleOp destruction to decrement the ref count for the resource. Since we don't destroy kernels in Eager mode, we explicitly added in a destroy_resource_op to mitigate this.

The problem is that this isn't enough. The ResourceMgr.LookupOrCreate method ends up increasing the ref count of the resource by 2 and we were effectively doing two Unref's in graph mode in the destructor. So even with the destroy resource op, the refcount remained 1 and didn't go down to zero.

The fix here is to handle the Eager mode case separately, similar to what we've done with the AnonymousIteratorHandleOp. Instead of creating a whole new kernel, we re-use the existing kernel and use a special shared_name argument to identify when to switch the behavior. Now in Eager mode, after running the HandleOp kernel, the refcount of the resource is 1.

PiperOrigin-RevId: 231333966
---
 .../kernels/data/multi_device_iterator_ops.cc | 65 +++++++++++++------
 .../multi_device_iterator_test.py             | 50 ++++++++++++++
 .../data/ops/multi_device_iterator_ops.py     |  4 +-
 3 files changed, 95 insertions(+), 24 deletions(-)

diff --git a/tensorflow/core/kernels/data/multi_device_iterator_ops.cc b/tensorflow/core/kernels/data/multi_device_iterator_ops.cc
index 167276032b..f0c6f5807b 100644
--- a/tensorflow/core/kernels/data/multi_device_iterator_ops.cc
+++ b/tensorflow/core/kernels/data/multi_device_iterator_ops.cc
@@ -359,6 +359,9 @@ class MultiDeviceIterator : public ResourceBase {
   std::unique_ptr<MultiDeviceBuffer> multi_device_buffer_ GUARDED_BY(mu_);
 };
 
+// Used to generate unique names for anonymous multi device iterators.
+static std::atomic<int64> current_id_;
+
 // Just creates a MultiDeviceIterator and returns it.
 class MultiDeviceIteratorHandleOp : public OpKernel {
  public:
@@ -388,6 +391,8 @@ class MultiDeviceIteratorHandleOp : public OpKernel {
   }
 
   void Compute(OpKernelContext* context) override LOCKS_EXCLUDED(mu_) {
+    string unique_name = cinfo_.name();
+    string container_name = cinfo_.container();
     {
       mutex_lock l(mu_);
       if (resource_ == nullptr) {
@@ -402,31 +407,49 @@ class MultiDeviceIteratorHandleOp : public OpKernel {
         OP_REQUIRES_OK(context, cinfo_.Init(mgr, def()));
 
         MultiDeviceIterator* resource;
-        OP_REQUIRES_OK(context,
-                       mgr->LookupOrCreate<MultiDeviceIterator>(
-                           cinfo_.container(), cinfo_.name(), &resource,
-                           [this, lib, &flib_def, &pflr,
-                            &function_handle_cache](MultiDeviceIterator** ret)
-                               EXCLUSIVE_LOCKS_REQUIRED(mu_) {
-                                 *ret = new MultiDeviceIterator(
-                                     output_types_, output_shapes_, devices_,
-                                     std::move(flib_def), std::move(pflr), lib,
-                                     std::move(function_handle_cache));
-                                 return Status::OK();
-                               }));
-
-        Status s = VerifyResource(resource);
-        if (TF_PREDICT_FALSE(!s.ok())) {
-          resource->Unref();
-          context->SetStatus(s);
-          return;
-        }
 
-        resource_ = resource;
+        if (name_ == ResourceHandle::ANONYMOUS_NAME) {
+          unique_name = strings::StrCat("_AnonymousMultiDeviceIterator",
+                                        current_id_.fetch_add(1));
+          container_name = "AnonymousMultiDeviceIterator";
+          resource = new MultiDeviceIterator(
+              output_types_, output_shapes_, devices_, std::move(flib_def),
+              std::move(pflr), lib, std::move(function_handle_cache));
+          OP_REQUIRES_OK(context, mgr->Create<MultiDeviceIterator>(
+                                      container_name, unique_name, resource));
+          Status s = VerifyResource(resource);
+          if (TF_PREDICT_FALSE(!s.ok())) {
+            resource->Unref();
+            context->SetStatus(s);
+            return;
+          }
+        } else {
+          unique_name = cinfo_.name();
+          container_name = cinfo_.container();
+          OP_REQUIRES_OK(context,
+                         mgr->LookupOrCreate<MultiDeviceIterator>(
+                             container_name, unique_name, &resource,
+                             [this, lib, &flib_def, &pflr,
+                              &function_handle_cache](MultiDeviceIterator** ret)
+                                 EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+                                   *ret = new MultiDeviceIterator(
+                                       output_types_, output_shapes_, devices_,
+                                       std::move(flib_def), std::move(pflr),
+                                       lib, std::move(function_handle_cache));
+                                   return Status::OK();
+                                 }));
+          Status s = VerifyResource(resource);
+          if (TF_PREDICT_FALSE(!s.ok())) {
+            resource->Unref();
+            context->SetStatus(s);
+            return;
+          }
+          resource_ = resource;
+        }
       }
     }
     OP_REQUIRES_OK(context, MakeResourceHandleToOutput(
-                                context, 0, cinfo_.container(), cinfo_.name(),
+                                context, 0, container_name, unique_name,
                                 MakeTypeIndex<MultiDeviceIterator>()));
   }
 
diff --git a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
index 2bc0d8f90d..110e3b9660 100644
--- a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
+++ b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
@@ -18,7 +18,9 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import time
 from absl.testing import parameterized
+import six
 
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.data.experimental.ops import optimization
@@ -34,10 +36,40 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
 
+# memory_profiler might not be available in the OSS version of TensorFlow.
+try:
+  import memory_profiler  # pylint:disable=g-import-not-at-top
+except ImportError:
+  memory_profiler = None
+
+
 @test_util.run_all_in_graph_and_eager_modes
 class MultiDeviceIteratorTest(test_base.DatasetTestBase,
                               parameterized.TestCase):
 
+  def assertNotIncreasingMemory(self,
+                                f,
+                                num_iters=100000,
+                                increase_threshold_absolute_mb=10):
+    """Assert memory usage doesn't increase beyond given threshold for f."""
+
+    with context.eager_mode():
+      # Warm up.
+      f()
+
+      # Wait for background threads to start up and take over memory.
+      # FIXME: The nature of this test leaves few other options. Maybe there
+      # is a better way to do this.
+      time.sleep(4)
+      initial = memory_profiler.memory_usage(-1)[0]
+      for _ in six.moves.range(num_iters):
+        f()
+      increase = memory_profiler.memory_usage(-1)[0] - initial
+      assert increase < increase_threshold_absolute_mb, (
+          "Increase is too high. Initial memory usage: %f MB. Increase: %f MB. "
+          "Maximum allowed increase: %f") % (initial, increase,
+                                             increase_threshold_absolute_mb)
+
   @parameterized.parameters(0, 1, 42,)
   @test_util.run_v1_only("b/121264236")
   def testInitOnly(self, num_inits):
@@ -68,6 +100,24 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase,
         self.evaluate(elem_on_1)
         self.evaluate(elem_on_2)
 
+  @test_util.run_v1_only("b/121264236")
+  def testEagerNoMemoryLeak(self):
+    if not context.executing_eagerly():
+      self.skipTest("Only eager mode test")
+    if memory_profiler is None:
+      self.skipTest("memory_profiler required to run this test")
+
+    def f():
+      dataset = dataset_ops.Dataset.range(10)
+      multi_device_iterator = multi_device_iterator_ops.MultiDeviceIterator(
+          dataset, ["/cpu:1", "/cpu:2"])
+      self.evaluate(multi_device_iterator.get_next())
+      del multi_device_iterator
+      del dataset
+
+    self.assertNotIncreasingMemory(
+        f, num_iters=100, increase_threshold_absolute_mb=175)
+
   @test_util.run_v1_only("b/121264236")
   def testOneOnSameDevice(self):
     with ops.device("/cpu:0"):
diff --git a/tensorflow/python/data/ops/multi_device_iterator_ops.py b/tensorflow/python/data/ops/multi_device_iterator_ops.py
index e9a26807ae..0a74b9be06 100644
--- a/tensorflow/python/data/ops/multi_device_iterator_ops.py
+++ b/tensorflow/python/data/ops/multi_device_iterator_ops.py
@@ -158,9 +158,7 @@ class MultiDeviceIterator(object):
       # TODO(b/121378567): Get rid of this shared_name hack.
       shared_name = ""
       if context.executing_eagerly():
-        # Ensure a unique name when eager execution is enabled to avoid spurious
-        # sharing issues.
-        shared_name += str(ops.uid())
+        shared_name = context.shared_name()
       self._multi_device_iterator_resource = (
           gen_dataset_ops.multi_device_iterator(
               devices=self._devices,
-- 
GitLab


From 2ac6545b3a5c2ed2b7d6f4e554492469e97555e0 Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Mon, 28 Jan 2019 19:52:50 -0800
Subject: [PATCH 1454/2345] Clean up conversion warning language, and use the
 ability to distinguish info, warning, and errors better.

Also improves the conversion of some symbols (changes a warning to a conversion for more cases that is now possible).

PiperOrigin-RevId: 231334490
---
 tensorflow/tools/compatibility/ast_edits.py   |  71 +-
 .../tools/compatibility/ast_edits_test.py     |   2 +-
 tensorflow/tools/compatibility/reorders_v2.py |   9 +
 tensorflow/tools/compatibility/tf_upgrade.py  |   7 +-
 .../tools/compatibility/tf_upgrade_v2.py      | 837 +++++++++++-------
 .../tools/compatibility/tf_upgrade_v2_main.py |   2 +-
 .../tools/compatibility/tf_upgrade_v2_test.py |  85 +-
 7 files changed, 641 insertions(+), 372 deletions(-)

diff --git a/tensorflow/tools/compatibility/ast_edits.py b/tensorflow/tools/compatibility/ast_edits.py
index eabb0be4e6..572bf384e4 100644
--- a/tensorflow/tools/compatibility/ast_edits.py
+++ b/tensorflow/tools/compatibility/ast_edits.py
@@ -62,6 +62,45 @@ def full_name_node(name, ctx=ast.Load()):
   return node
 
 
+def get_arg_value(node, arg_name, arg_pos=None):
+  """Get the value of an argument from a ast.Call node.
+
+  This function goes through the positional and keyword arguments to check
+  whether a given argument was used, and if so, returns its value (the node
+  representing its value).
+
+  This cannot introspect *args or **args, but it safely handles *args in
+  Python3.5+.
+
+  Args:
+    node: The ast.Call node to extract arg values from.
+    arg_name: The name of the argument to extract.
+    arg_pos: The position of the argument (in case it's passed as a positional
+      argument).
+
+  Returns:
+    A tuple (arg_present, arg_value) containing a boolean indicating whether
+    the argument is present, and its value in case it is.
+  """
+  # Check keyword args
+  if arg_name is not None:
+    for kw in node.keywords:
+      if kw.arg == arg_name:
+        return (True, kw.value)
+
+  # Check positional args
+  if arg_pos is not None:
+    idx = 0
+    for arg in node.args:
+      if sys.version_info[:2] >= (3, 5) and isinstance(arg, ast.Starred):
+        continue  # Can't parse Starred
+      if idx == arg_pos:
+        return (True, arg)
+      idx += 1
+
+  return (False, None)
+
+
 class APIChangeSpec(object):
   """This class defines the transformations that need to happen.
 
@@ -195,11 +234,10 @@ class _PastaEditVisitor(ast.NodeVisitor):
     """Adds an error to be printed about full_name at node."""
     function_warnings = self._api_change_spec.function_warnings
     if full_name in function_warnings:
-      warning_message = function_warnings[full_name]
-      warning_message = warning_message.replace("<function name>", full_name)
-      self.add_log(WARNING, node.lineno, node.col_offset,
-                   "%s requires manual check. %s" % (full_name,
-                                                     warning_message))
+      level, message = function_warnings[full_name]
+      message = message.replace("<function name>", full_name)
+      self.add_log(level, node.lineno, node.col_offset,
+                   "%s requires manual check. %s" % (full_name, message))
       return True
     else:
       return False
@@ -232,13 +270,13 @@ class _PastaEditVisitor(ast.NodeVisitor):
     arg_warnings = self._get_applicable_dict("function_arg_warnings",
                                              full_name, name)
 
-    used_args = [kw.arg for kw in node.keywords]
-    for (kwarg, arg), warning in arg_warnings.items():
-      if kwarg in used_args or len(node.args) > arg:
+    for (kwarg, arg), (level, warning) in arg_warnings.items():
+      present, _ = get_arg_value(node, kwarg, arg)
+      if present:
         warned = True
         warning_message = warning.replace("<function name>", full_name or name)
-        self.add_log(WARNING, node.lineno, node.col_offset,
-                     "%s called with %s argument requires manual check: %s." %
+        self.add_log(level, node.lineno, node.col_offset,
+                     "%s called with %s argument requires manual check: %s" %
                      (full_name or name, kwarg, warning_message))
 
     return warned
@@ -280,12 +318,14 @@ class _PastaEditVisitor(ast.NodeVisitor):
     if full_name in function_reorders:
       reordered = function_reorders[full_name]
       new_keywords = []
-      for idx, arg in enumerate(node.args):
+      idx = 0
+      for arg in node.args:
         if sys.version_info[:2] >= (3, 5) and isinstance(arg, ast.Starred):
           continue  # Can't move Starred to keywords
         keyword_arg = reordered[idx]
         keyword = ast.keyword(arg=keyword_arg, value=arg)
         new_keywords.append(keyword)
+        idx += 1
 
       if new_keywords:
         self.add_log(INFO, node.lineno, node.col_offset,
@@ -374,11 +414,10 @@ class _PastaEditVisitor(ast.NodeVisitor):
       logs = []
       new_node = transformer(parent, node, full_name, name, logs)
       self.add_logs(logs)
-      if new_node:
-        if new_node is not node:
-          pasta.ast_utils.replace_child(parent, node, new_node)
-          node = new_node
-          self._stack[-1] = node
+      if new_node and new_node is not node:
+        pasta.ast_utils.replace_child(parent, node, new_node)
+        node = new_node
+        self._stack[-1] = node
 
     self.generic_visit(node)
 
diff --git a/tensorflow/tools/compatibility/ast_edits_test.py b/tensorflow/tools/compatibility/ast_edits_test.py
index 7049479152..366ea2cb72 100644
--- a/tensorflow/tools/compatibility/ast_edits_test.py
+++ b/tensorflow/tools/compatibility/ast_edits_test.py
@@ -417,7 +417,7 @@ class TestAstEdits(test_util.TensorFlowTestCase):
 
       def __init__(self):
         NoUpdateSpec.__init__(self)
-        self.function_warnings = {"*.foo": "not good"}
+        self.function_warnings = {"*.foo": (ast_edits.WARNING, "not good")}
 
     texts = ["object.foo()", "get_object().foo()",
              "get_object().foo()", "object.foo().bar()"]
diff --git a/tensorflow/tools/compatibility/reorders_v2.py b/tensorflow/tools/compatibility/reorders_v2.py
index f9b0e3f9d8..01556b1225 100644
--- a/tensorflow/tools/compatibility/reorders_v2.py
+++ b/tensorflow/tools/compatibility/reorders_v2.py
@@ -37,6 +37,7 @@ reorders = {
     'tf.decode_csv': ['records', 'record_defaults', 'field_delim', 'use_quote_delim', 'name', 'na_value', 'select_cols'],
     'tf.depth_to_space': ['input', 'block_size', 'name', 'data_format'],
     'tf.feature_column.categorical_column_with_vocabulary_file': ['key', 'vocabulary_file', 'vocabulary_size', 'num_oov_buckets', 'default_value', 'dtype'],
+    'tf.image.sample_distorted_bounding_box': ['image_size', 'bounding_boxes', 'seed', 'seed2', 'min_object_covered', 'aspect_ratio_range', 'area_range', 'max_attempts', 'use_image_if_no_bounding_boxes', 'name'],
     'tf.io.decode_csv': ['records', 'record_defaults', 'field_delim', 'use_quote_delim', 'name', 'na_value', 'select_cols'],
     'tf.io.parse_example': ['serialized', 'features', 'name', 'example_names'],
     'tf.io.parse_single_example': ['serialized', 'features', 'name', 'example_names'],
@@ -56,12 +57,19 @@ reorders = {
     'tf.math.reduce_prod': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
     'tf.math.reduce_sum': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
     'tf.multinomial': ['logits', 'num_samples', 'seed', 'name', 'output_dtype'],
+    'tf.nn.conv1d': ['value', 'filters', 'stride', 'padding', 'use_cudnn_on_gpu', 'data_format', 'name'],
+    'tf.nn.conv2d': ['input', 'filter', 'strides', 'padding', 'use_cudnn_on_gpu', 'data_format', 'dilations', 'name'],
+    'tf.nn.conv2d_backprop_filter': ['input', 'filter_sizes', 'out_backprop', 'strides', 'padding', 'use_cudnn_on_gpu', 'data_format', 'dilations', 'name'],
+    'tf.nn.conv2d_backprop_input': ['input_sizes', 'filter', 'out_backprop', 'strides', 'padding', 'use_cudnn_on_gpu', 'data_format', 'dilations', 'name'],
     'tf.nn.convolution': ['input', 'filter', 'padding', 'strides', 'dilation_rate', 'name', 'data_format'],
     'tf.nn.crelu': ['features', 'name', 'axis'],
+    'tf.nn.ctc_beam_search_decoder': ['inputs', 'sequence_length', 'beam_width', 'top_paths', 'merge_repeated'],
     'tf.nn.depth_to_space': ['input', 'block_size', 'name', 'data_format'],
     'tf.nn.depthwise_conv2d': ['input', 'filter', 'strides', 'padding', 'rate', 'name', 'data_format'],
     'tf.nn.embedding_lookup': ['params', 'ids', 'partition_strategy', 'name', 'validate_indices', 'max_norm'],
     'tf.nn.embedding_lookup_sparse': ['params', 'sp_ids', 'sp_weights', 'partition_strategy', 'name', 'combiner', 'max_norm'],
+    'tf.nn.fractional_avg_pool': ['value', 'pooling_ratio', 'pseudo_random', 'overlapping', 'deterministic', 'seed', 'seed2', 'name'],
+    'tf.nn.fractional_max_pool': ['value', 'pooling_ratio', 'pseudo_random', 'overlapping', 'deterministic', 'seed', 'seed2', 'name'],
     'tf.nn.in_top_k': ['predictions', 'targets', 'k', 'name'],
     'tf.nn.moments': ['x', 'axes', 'shift', 'name', 'keep_dims'],
     'tf.nn.pool': ['input', 'window_shape', 'pooling_type', 'padding', 'dilation_rate', 'strides', 'name', 'data_format'],
@@ -113,6 +121,7 @@ reorders = {
     'tf.strings.reduce_join': ['inputs', 'axis', 'keep_dims', 'separator', 'name', 'reduction_indices'],
     'tf.strings.substr': ['input', 'pos', 'len', 'name', 'unit'],
     'tf.substr': ['input', 'pos', 'len', 'name', 'unit'],
+    'tf.test.assert_equal_graph_def': ['actual', 'expected', 'checkpoint_v2'],
     'tf.transpose': ['a', 'perm', 'name', 'conjugate'],
     'tf.tuple': ['tensors', 'name', 'control_inputs'],
     'tf.while_loop': ['cond', 'body', 'loop_vars', 'shape_invariants', 'parallel_iterations', 'back_prop', 'swap_memory', 'name', 'maximum_iterations', 'return_same_structure']
diff --git a/tensorflow/tools/compatibility/tf_upgrade.py b/tensorflow/tools/compatibility/tf_upgrade.py
index 241b08510f..8d16c679b5 100644
--- a/tensorflow/tools/compatibility/tf_upgrade.py
+++ b/tensorflow/tools/compatibility/tf_upgrade.py
@@ -177,10 +177,11 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
 
     # Warnings that should be printed if corresponding functions are used.
     self.function_warnings = {
-        "tf.reverse":
-            "ERROR: tf.reverse has had its argument semantics changed "
+        "tf.reverse": (
+            ast_edits.ERROR,
+            "tf.reverse has had its argument semantics changed "
             "significantly. The converter cannot detect this reliably, so "
-            "you need to inspect this usage manually.\n",
+            "you need to inspect this usage manually.\n"),
     }
 
 
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 03ecf5f303..2abd3eac1f 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -19,6 +19,8 @@ from __future__ import division
 from __future__ import print_function
 
 import ast
+import functools
+import sys
 
 import pasta
 import six
@@ -27,6 +29,9 @@ from tensorflow.tools.compatibility import ast_edits
 from tensorflow.tools.compatibility import renames_v2
 from tensorflow.tools.compatibility import reorders_v2
 
+# These pylint warnings are a mistake.
+# pylint: disable=g-explicit-bool-comparison,g-bool-id-comparison
+
 
 class TFAPIChangeSpec(ast_edits.APIChangeSpec):
   """List of maps that describe what changed in the API."""
@@ -38,6 +43,15 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     # Only keyword args are handled, so make sure to also put any function in
     # function_reorders to ensure that all args are made into keywords first.
     self.function_keyword_renames = {
+        "tf.test.assert_equal_graph_def": {
+            "checkpoint_v2": None,
+        },
+        "tf.nn.embedding_lookup": {
+            "validate_indices": None,
+        },
+        "tf.image.sample_distorted_bounding_box": {
+            "seed2": None,
+        },
         "tf.gradients": {
             "colocate_gradients_with_ops": None,
         },
@@ -380,6 +394,21 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.nn.weighted_moments": {
             "keep_dims": "keepdims"
         },
+        "tf.nn.conv1d": {
+            "value": "input",
+            "use_cudnn_on_gpu": None,
+        },
+        "tf.nn.conv2d": {
+            "filter": "filters",
+            "use_cudnn_on_gpu": None,
+        },
+        "tf.nn.conv2d_backprop_filter": {
+            "use_cudnn_on_gpu": None,
+        },
+        "tf.nn.conv2d_backprop_input": {
+            "filter": "filters",
+            "use_cudnn_on_gpu": None,
+        },
     }
 
     # pylint: disable=line-too-long
@@ -662,6 +691,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     # Mapping from function to the new name of the function
     self.symbol_renames = renames_v2.renames
     self.symbol_renames.update(self.manual_symbol_renames)
+    self.symbol_renames = {
+        name: new_name
+        for name, new_name in self.symbol_renames.items()
+    }
 
     # Variables that should be changed to functions.
     self.change_to_function = {}
@@ -684,6 +717,11 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.nn.space_to_batch",
         "tf.boolean_mask",
         "tf.convert_to_tensor",
+        "tf.nn.conv1d",
+        "tf.nn.conv2d",
+        "tf.nn.conv2d_backprop_filter",
+        "tf.nn.conv2d_backprop_input",
+        "tf.nn.ctc_beam_search_decoder",
         "tf.nn.moments",
         "tf.nn.convolution",
         "tf.nn.crelu",
@@ -746,6 +784,7 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.nn.embedding_lookup_sparse",
         "tf.nn.in_top_k",
         "tf.nn.space_to_depth",
+        "tf.test.assert_equal_graph_def",
         "tf.linalg.norm",
         "tf.norm",
         "tf.reverse_sequence",
@@ -754,6 +793,9 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         # keyword arguments. Add keyword arguments in rare case when they
         # are not specified.
         "tf.nn.softmax_cross_entropy_with_logits",
+        "tf.nn.fractional_avg_pool",
+        "tf.nn.fractional_max_pool",
+        "tf.image.sample_distorted_bounding_box",
     }
 
     # Functions that were reordered should be changed to the new keyword args
@@ -761,43 +803,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     # positional arguments yourself, this could do the wrong thing.
     self.function_reorders = reorders_v2.reorders
 
-    # Specially handled functions (pasta version)
-    # Each transformer is a callable which will be called with the arguments
-    #   transformer(parent, node, full_name, name, logs, errors)
-    # Where logs and errors are lists to which (line, col, msg) tuples can be
-    # appended, full_name is the FQN of the function called (or None if that is
-    # unknown), name is the name of the function called (or None is that is
-    # unknown). node is an ast.Call node representing this function call, and
-    # parent is its parent in the AST.
-    # The function may modify node (but not parent), and must return
-    # - none, if nothing was modified
-    # - node, if node was modified in place (make sure to use
-    #   pasta.ast_utils.replace_child to swap out children, otherwise formatting
-    #   may get messy)
-    # - a replacement for node, if the whole call node was replaced. The caller
-    #   will take care of changing parent.
-    self.function_transformers = {
-        "*.make_initializable_iterator": self._iterator_transformer,
-        "*.make_one_shot_iterator": self._iterator_transformer,
-        "tf.nn.dropout": self._dropout_transformer,
-        "tf.batch_gather": self._batch_gather_transformer,
-        "tf.to_bfloat16": self._cast_transformer,
-        "tf.to_complex128": self._cast_transformer,
-        "tf.to_complex64": self._cast_transformer,
-        "tf.to_double": self._cast_transformer,
-        "tf.to_float": self._cast_transformer,
-        "tf.to_int32": self._cast_transformer,
-        "tf.to_int64": self._cast_transformer,
-        "tf.nn.softmax_cross_entropy_with_logits":
-            self._softmax_cross_entropy_with_logits_transformer,
-        "tf.image.resize_area": self._image_resize_transformer,
-        "tf.image.resize_bicubic": self._image_resize_transformer,
-        "tf.image.resize_bilinear": self._image_resize_transformer,
-        "tf.image.resize_nearest_neighbor": self._image_resize_transformer,
-
-    }
-
     decay_function_comment = (
+        ast_edits.INFO,
         "<function name> has been changed to return a callable instead "
         "of a tensor when graph building, but its functionality remains "
         "unchanged during eager execution (returns a callable like "
@@ -806,64 +813,66 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         " be correct).\n"
     )
 
-    # TODO(b/118888586): add default value change to update script.
-    default_loss_reduction_changed = (
-        "default value of loss_reduction has been changed to "
-        "SUM_OVER_BATCH_SIZE.\n"
-    )
-
     assert_return_type_comment = (
-        "assert_* functions have been changed to return None, the "
+        ast_edits.INFO,
+        "<function name> has been changed to return None, the "
         "data argument has been removed, and arguments have been reordered."
         "\nThe calls have been converted to compat.v1 for safety (even though "
         " they may already have been correct)."
     )
 
     assert_rank_comment = (
-        "assert_rank_* functions have been changed to return None, and"
+        ast_edits.INFO,
+        "<function name> has been changed to return None, and"
         " the data and summarize arguments have been removed."
         "\nThe calls have been converted to compat.v1 for safety (even though "
         " they may already have been correct)."
     )
 
-    tf_01s_like_no_optimize_comment = (
-        "tf.zeros_like and tf.ones_like no longer have the optimize "
-        "argument in TF 2.0 or after (also, `tensor' argument is renamed to "
-        "`input')."
-        "\nThe calls have been converted to compat.v1 for safety (even though "
-        " they may already have been correct)."
-    )
-
-    deprecate_partition_strategy_comment = (
-        "`partition_strategy` has been removed from `%s` "
-        " The 'div' strategy is used by default.")
-
     initializers_no_dtype_comment = (
-        "tf.initializers and tf.keras.initializers no longer have the "
+        ast_edits.INFO,
+        "Initializers no longer have the "
         "dtype argument in the constructor or partition_info argument in the "
-        "call method in TF 2.0 and after. The only API symbols are now "
-        "tf.keras.initializers.* or tf.initializers.*."
-        "\nThe calls have been converted to compat.v1 for safety (even though "
-        "they may already have been correct).")
-
-    uniform_unit_scaling_initializer_comment = (
-        "uniform_unit_scaling_initializer has been removed. Please use"
-        " tf.initializers.variance_scaling instead with distribution=uniform "
-        "to get equivalent behaviour.")
+        "__call__ method.\nThe calls have been converted to compat.v1 for"
+        "safety (even though they may already have been correct).")
 
     metrics_comment = (
-        "tf.metrics have been converted to object oriented versions in"
+        ast_edits.INFO,
+        "tf.metrics have been replaced with object oriented versions in"
         " TF 2.0 and after. The metric function calls have been converted to "
         "compat.v1 for backward compatibility. Please update these calls to "
         "the TF 2.0 versions.")
 
     losses_comment = (
-        "tf.losses have been converted to object oriented versions in"
+        ast_edits.INFO,
+        "tf.losses have been replaced with object oriented versions in"
         " TF 2.0 and after. The loss function calls have been converted to "
         "compat.v1 for backward compatibility. Please update these calls to "
         "the TF 2.0 versions.")
 
+# This could be done with a _rename_if_arg_not_found_transformer
+    deprecate_partition_strategy_comment = (
+        ast_edits.WARNING,
+        "`partition_strategy` has been removed from <function name>. "
+        " The 'div' strategy will be used by default.")
+
+# TODO(b/118888586): add default value change to update script.
+    default_loss_reduction_changed = (
+        ast_edits.WARNING,
+        "default value of loss_reduction has been changed to "
+        "SUM_OVER_BATCH_SIZE.\n"
+    )
+
+# make change instead
+    uniform_unit_scaling_initializer_comment = (
+        ast_edits.ERROR,
+        "uniform_unit_scaling_initializer has been removed. Please use"
+        " tf.initializers.variance_scaling instead with distribution=uniform "
+        "to get equivalent behaviour.")
+
+# Make change instead (issue warning about strip_...)
     export_saved_model_renamed = (
+        ast_edits.ERROR,
         "(Manual edit required) Please rename the method export_savedmodel() "
         "to export_saved_model(). Two things to note:\n\t(1) The argument "
         "strip_default_attributes has been removed. The function will always "
@@ -947,13 +956,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             assert_rank_comment,
         "tf.debugging.assert_rank_in":
             assert_rank_comment,
-        "tf.device":
-            "tf.device no longer takes function as an argument. "
-            "'device_name_or_function' argument has been renamed to "
-            "'device_name'.",
-        "tf.flags":
+        "tf.flags": (
+            ast_edits.ERROR,
             "tf.flags has been removed, please use the argparse or absl"
-            " module if you need command line parsing.",
+            " modules if you need command line parsing."),
         "tf.train.exponential_decay":
             decay_function_comment,
         "tf.train.piecewise_constant_decay":
@@ -988,55 +994,12 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             default_loss_reduction_changed,
         "tf.estimator.BaselineRegressor":
             default_loss_reduction_changed,
-        "tf.nn.conv1d":
-            "WARNING: use_cudnn_on_gpu argument has been removed and \"value\""
-            " was renamed to \"input\"",
-        "tf.nn.conv2d":
-            "WARNING: use_cudnn_on_gpu argument has been removed and "
-            "\"filter\" was renamed to \"filters\"",
-        "tf.nn.conv2d_backprop_filter":
-            "WARNING: use_cudnn_on_gpu argument has been removed",
-        "tf.nn.conv2d_backprop_input":
-            "WARNING: use_cudnn_on_gpu argument has been removed and "
-            "\"filter\" was renamed to \"filters\"",
-        "tf.nn.erosion2d":
-            "WARNING: <function name> now requires a data_format argument",
         "tf.nn.nce_loss":
-            deprecate_partition_strategy_comment % "tf.nn.nce_loss",
+            deprecate_partition_strategy_comment,
         "tf.nn.safe_embedding_lookup_sparse":
-            deprecate_partition_strategy_comment %
-            "tf.nn.safe_embedding_lookup_sparse",
+            deprecate_partition_strategy_comment,
         "tf.nn.sampled_softmax_loss":
-            deprecate_partition_strategy_comment % "tf.nn.sampled_softmax_loss",
-        "tf.zeros_like":
-            tf_01s_like_no_optimize_comment,
-        "tf.ones_like":
-            tf_01s_like_no_optimize_comment,
-        "tf.nn.embedding_lookup":
-            "WARNING: validate_indices argument has been removed.",
-        "tf.while_loop":
-            "tf.while_loop no longer takes 'return_same_structure' argument. "
-            "'return_same_structure' now defaults to True. Also, 'name'"
-            "argument is now the last argument.",
-        "tf.image.sample_distorted_bounding_box":
-            "tf.image.sample_distorted_bounding_box no longer takes 'seed2' "
-            "argument.",
-        "tf.nn.ctc_beam_search_decoder":
-            "tf.nn.ctc_beam_search_decoder no longer takes 'merge_repeated' "
-            "argument. 'merge_repeated' now defaults to False.",
-        "tf.nn.fractional_avg_pool":
-            "tf.nn.fractional_avg_pool no longer takes 'seed2' and "
-            "'deterministic' arguments. Now it takes a single 'seed' arg. If "
-            "'seed' is zero, the execution is random and deterministic "
-            "otherwise",
-        "tf.nn.fractional_max_pool":
-            "tf.nn.fractional_max_pool no longer takes 'seed2' and "
-            "'deterministic' arguments. Now it takes a single 'seed' arg. If "
-            "'seed' is zero, the execution is random and deterministic "
-            "otherwise",
-        "tf.test.assert_equal_graph_def":
-            "tf.assert_equal_graph_def no longer takes 'checkpoint_v2' "
-            "argument. 'checkpoint_v2' now defaults to True.",
+            deprecate_partition_strategy_comment,
         "tf.keras.initializers.Zeros":
             initializers_no_dtype_comment,
         "tf.keras.initializers.zeros":
@@ -1211,227 +1174,489 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
 
     # Warnings that are emitted only if a specific arg is found.
     self.function_arg_warnings = {
+        "tf.nn.conv1d": {
+            ("use_cudnn_on_gpu", 4): (
+                ast_edits.WARNING,
+                "use_cudnn_on_gpu has been removed, behavior is now equivalent"
+                "to setting it to True."),
+        },
+        "tf.nn.conv2d": {
+            ("use_cudnn_on_gpu", 4): (
+                ast_edits.WARNING,
+                "use_cudnn_on_gpu has been removed, behavior is now equivalent"
+                "to setting it to True."),
+        },
+        "tf.nn.conv2d_backprop_filter": {
+            ("use_cudnn_on_gpu", 5): (
+                ast_edits.WARNING,
+                "use_cudnn_on_gpu has been removed, behavior is now equivalent"
+                "to setting it to True."),
+        },
+        "tf.nn.conv2d_backprop_input": {
+            ("use_cudnn_on_gpu", 5): (
+                ast_edits.WARNING,
+                "use_cudnn_on_gpu has been removed, behavior is now equivalent"
+                "to setting it to True."),
+        },
         "tf.gradients": {
-            ("colocate_gradients_with_ops", 4):
+            ("colocate_gradients_with_ops", 4): (
+                ast_edits.INFO,
                 "tf.gradients no longer takes "
                 "'colocate_gradients_with_ops' argument, it behaves as if it "
-                "was set to True.",
+                "was set to True."),
         },
         "*.minimize": {
-            ("colocate_gradients_with_ops", 5):
+            ("colocate_gradients_with_ops", 5): (
+                ast_edits.INFO,
                 "Optimizer.minimize no longer takes "
                 "'colocate_gradients_with_ops' argument, it behaves as if it "
-                "was set to True.",
+                "was set to True."),
         },
         "*.compute_gradients": {
-            ("colocate_gradients_with_ops", 4):
+            ("colocate_gradients_with_ops", 4): (
+                ast_edits.INFO,
                 "Optimizer.compute_gradients no "
                 "longer takes 'colocate_gradients_with_ops' argument, it "
-                "behaves as if it was set to True.",
+                "behaves as if it was set to True."),
         },
         "tf.cond": {
-            ("strict", 3):
+            ("strict", 3): (
+                ast_edits.WARNING,
                 "tf.cond no longer takes 'strict' argument, it behaves as "
-                "if was set to True."
+                "if was set to True.")
         },
     }
 
-    self.symbol_renames = {
-        name: new_name
-        for name, new_name in self.symbol_renames.items()
+    # Specially handled functions
+    # Each transformer is a callable which will be called with the arguments
+    #   transformer(parent, node, full_name, name, logs, errors)
+    # Where logs is a list to which (level, line, col, msg) tuples can be
+    # appended, full_name is the FQN of the function called (or None if that is
+    # unknown), name is the name of the function called (or None is that is
+    # unknown). node is an ast.Call node representing this function call, and
+    # parent is its parent in the AST.
+    # The function may modify node (but not parent), and must return
+    # - none, if nothing was modified
+    # - node, if node was modified in place (make sure to use
+    #   pasta.ast_utils.replace_child to swap out children, otherwise formatting
+    #   may get messy)
+    # - a replacement for node, if the whole call node was replaced. The caller
+    #   will take care of changing parent.
+    self.function_transformers = {
+        "*.make_initializable_iterator": _iterator_transformer,
+        "*.make_one_shot_iterator": _iterator_transformer,
+        "tf.nn.dropout": _dropout_transformer,
+        "tf.batch_gather": _batch_gather_transformer,
+        "tf.to_bfloat16": _cast_transformer,
+        "tf.to_complex128": _cast_transformer,
+        "tf.to_complex64": _cast_transformer,
+        "tf.to_double": _cast_transformer,
+        "tf.to_float": _cast_transformer,
+        "tf.to_int32": _cast_transformer,
+        "tf.to_int64": _cast_transformer,
+        "tf.nn.softmax_cross_entropy_with_logits":
+            _softmax_cross_entropy_with_logits_transformer,
+        "tf.image.resize_area": _image_resize_transformer,
+        "tf.image.resize_bicubic": _image_resize_transformer,
+        "tf.image.resize_bilinear": _image_resize_transformer,
+        "tf.image.resize_nearest_neighbor": _image_resize_transformer,
+        "tf.nn.fractional_avg_pool": _pool_seed_transformer,
+        "tf.nn.fractional_max_pool": _pool_seed_transformer,
+        "tf.device": functools.partial(
+            _rename_if_arg_found_transformer, arg_name="device_name",
+            arg_ok_predicate=_is_ast_str, remove_if_ok=False,
+            message="tf.device no longer takes functions as an argument. "
+            "We could not determine that the argument value is a string, so "
+            "the call was converted to compat.v1."),
+        "tf.zeros_like": functools.partial(
+            _rename_if_arg_found_transformer, arg_name="optimize",
+            arg_ok_predicate=_is_ast_true, remove_if_ok=True,
+            message="tf.zeros_like no longer takes an optimize argument, and "
+            "behaves as if optimize=True. This call site specifies something "
+            "other than optimize=True, so it was converted to compat.v1."),
+        "tf.ones_like": functools.partial(
+            _rename_if_arg_found_transformer, arg_name="optimize",
+            arg_ok_predicate=_is_ast_true, remove_if_ok=True,
+            message="tf.ones_like no longer takes an optimize argument, and "
+            "behaves as if optimize=True. This call site specifies something "
+            "other than optimize=True, so it was converted to compat.v1."),
+        "tf.while_loop": functools.partial(
+            _rename_if_arg_found_transformer,
+            arg_name="return_same_structure",
+            arg_ok_predicate=_is_ast_true, remove_if_ok=True,
+            message="tf.while_loop no longer takes 'return_same_structure' "
+            "argument and behaves as if return_same_structure=True. This call "
+            "site specifies something other than return_same_structure=True, "
+            "so it was converted to compat.v1."),
+        "tf.nn.ctc_beam_search_decoder": functools.partial(
+            _rename_if_arg_found_transformer,
+            arg_name="merge_repeated",
+            arg_ok_predicate=_is_ast_false, remove_if_ok=True,
+            message="tf.nn.ctc_beam_search_decoder no longer takes the "
+            "'merge_repeated' argument and behaves as if merge_repeated=False. "
+            "This call site specifies something other than "
+            "merge_repeated=False, so it was converted to compat.v1."),
+        "tf.nn.erosion2d": functools.partial(
+            _add_argument_transformer,
+            arg_name="data_format",
+            arg_value_ast=ast.Str("NHWC")),
     }
 
-  @staticmethod
-  def _iterator_transformer(parent, node, full_name, name, logs):
-    # First, check that node.func.value is not already something we like
-    # (tf.compat.v1.data), or something which is handled in the rename
-    # (tf.data). This transformer only handles the method call to function call
-    # conversion.
-    if full_name and (full_name.startswith("tf.compat.v1.data") or
-                      full_name.startswith("tf.data")):
-      return
 
-    # This should never happen, since we're only called for Attribute nodes.
-    if not isinstance(node.func, ast.Attribute):
-      return
+def _is_ast_str(node):
+  """Determine whether this node represents a string."""
+  allowed_types = [ast.Str]
+  if hasattr(ast, "Bytes"):
+    allowed_types += [ast.Bytes]
+  if hasattr(ast, "JoinedStr"):
+    allowed_types += [ast.JoinedStr]
+  if hasattr(ast, "FormattedValue"):
+    allowed_types += [ast.FormattedValue]
+  return isinstance(node, allowed_types)
 
-    # Transform from x.f(y) to tf.compat.v1.data.f(x, y)
-    # Fortunately, node.func.value should already have valid position info
-    node.args = [node.func.value] + node.args
-    node.func.value = ast_edits.full_name_node("tf.compat.v1.data")
 
-    logs.append((ast_edits.WARNING, node.lineno, node.col_offset,
-                 "Changing dataset.%s() to tf.compat.v1.data.%s(dataset). "
-                 "Please check this transformation.\n" % (name, name)))
+def _is_ast_true(node):
+  if hasattr(ast, "NameConstant"):
+    return isinstance(node, ast.NameConstant) and node.value is True
+  else:
+    return isinstance(node, ast.Name) and node.id == "True"
 
-    return node
 
-  @staticmethod
-  def _dropout_transformer(parent, node, full_name, name, logs):
-    def _replace_keep_prob_node(parent, old_value):
-      """Replaces old_value with 1-(old_value)."""
-      one = ast.Num(n=1)
-      one.lineno = 0
-      one.col_offset = 0
-      new_value = ast.BinOp(left=one, op=ast.Sub(),
-                            right=old_value)
-      # This copies the prefix and suffix on old_value to new_value.
-      pasta.ast_utils.replace_child(parent, old_value, new_value)
-      ast.copy_location(new_value, old_value)
-      # Put parentheses around keep_prob.value (and remove the old prefix/
-      # suffix, they should only be around new_value).
-      pasta.base.formatting.set(old_value, "prefix", "(")
-      pasta.base.formatting.set(old_value, "suffix", ")")
-
-    # Check if we have a keep_prob keyword arg
-    for keep_prob in node.keywords:
-      if keep_prob.arg == "keep_prob":
-        logs.append((ast_edits.INFO, node.lineno, node.col_offset,
-                     "Changing keep_prob arg of tf.nn.dropout to rate\n"))
-        keep_prob.arg = "rate"
-        _replace_keep_prob_node(keep_prob, keep_prob.value)
-        return node
-
-    # Maybe it was a positional arg
-    if len(node.args) < 2:
-      logs.append((ast_edits.ERROR, node.lineno, node.col_offset,
-                   "tf.nn.dropout called without arguments, so "
-                   "automatic fix was disabled. tf.nn.dropout has changed "
-                   "the semantics of the second argument."))
-    else:
-      _replace_keep_prob_node(node, node.args[1])
-      logs.append((ast_edits.INFO, node.lineno, node.col_offset,
-                   "Changing keep_prob arg of tf.nn.dropout to rate, and "
-                   "recomputing value.\n"))
+def _is_ast_false(node):
+  if hasattr(ast, "NameConstant"):
+    return isinstance(node, ast.NameConstant) and node.value is False
+  else:
+    return isinstance(node, ast.Name) and node.id == "False"
 
-      return node
 
-  @staticmethod
-  def _cast_transformer(parent, node, full_name, name, logs):
-    """Transforms to_int and to_float to cast(..., dtype=...)."""
-
-    # Find out the dtype to cast to from the function name
-    dtype_str = name[3:]
-    # Special cases where the full dtype is not given
-    if dtype_str == "float":
-      dtype_str = "float32"
-    elif dtype_str == "double":
-      dtype_str = "float64"
-    new_arg = ast.keyword(arg="dtype",
-                          value=ast.Attribute(value=ast.Name(id="tf",
-                                                             ctx=ast.Load()),
-                                              attr=dtype_str, ctx=ast.Load()))
-    # Ensures a valid transformation when a positional name arg is given
-    if len(node.args) == 2:
-      name_arg = ast.keyword(arg="name",
-                             value=node.args[-1])
-      node.args = node.args[:-1]
-      node.keywords.append(name_arg)
-
-    # Python3 ast requires the args for the Attribute, but codegen will mess up
-    # the arg order if we just set them to 0.
-    new_arg.value.lineno = node.lineno
-    new_arg.value.col_offset = node.col_offset+100
-
-    node.keywords.append(new_arg)
-    if isinstance(node.func, ast.Attribute):
-      node.func.attr = "cast"
+# Lots of unused arguments below, since these are called in a standard manner.
+# pylint: disable=unused-argument
+
+
+def _rename_if_arg_found_transformer(parent, node, full_name, name, logs,
+                                     arg_name=None,
+                                     arg_ok_predicate=None,
+                                     remove_if_ok=False,
+                                     message=None):
+  """Replaces the given call with tf.compat.v1 if the given arg is found.
+
+  This requires the function to be called with all named args, so for using
+  this transformer, the function should also be added to renames.
+
+  If the arg is not found, the call site is left alone.
+
+  If the arg is found, and if arg_ok_predicate is given, it is called with
+  the ast Expression representing the argument value found. If it returns
+  True, the function is left alone.
+
+  If the arg is found, arg_ok_predicate is not None and returns ok, and
+  remove_if_ok is True, the argument is removed from the call.
+
+  Otherwise, `compat.v1` is inserted between tf and the function name.
+
+  Args:
+    parent: Parent of node.
+    node: ast.Call node to maybe modify.
+    full_name: full name of function to modify
+    name: name of function to modify
+    logs: list of logs to append to
+    arg_name: name of the argument to look for
+    arg_ok_predicate: predicate callable with the ast of the argument value,
+      returns whether the argument value is allowed.
+    remove_if_ok: remove the argument if present and ok as determined by
+      arg_ok_predicate.
+    message: message to print if a non-ok arg is found (and hence, the function
+      is renamed to its compat.v1 version).
+
+  Returns:
+    node, if it was modified, else None.
+  """
+  # Check whether arg is there.
+  arg_present, arg_value = ast_edits.get_arg_value(node, arg_name)
+  if not arg_present:
+    return
+
+  # Check whether arg is problematic (and if not, maybe remove it).
+  if arg_ok_predicate and arg_ok_predicate(arg_value):
+    if remove_if_ok:
+      for i, kw in enumerate(node.keywords):
+        if kw.arg == arg_name:
+          node.keywords.pop(i)
+          logs.append((ast_edits.INFO, node.lineno, node.col_offset,
+                       "Removed argument %s for function %s" % (
+                           arg_name, full_name or name)))
+          break
+      return node
     else:
-      assert isinstance(node.func, ast.Name)
-      node.func.id = "cast"
+      return
 
-    logs.append((ast_edits.INFO, node.lineno, node.col_offset,
-                 "Changed %s call to tf.cast(..., dtype=tf.%s)." % (full_name,
-                                                                    dtype_str)))
-    return node
+  # All conditions met, insert v1 and log what we did.
+  # We must have a full name, so the func is an attribute.
+  new_name = full_name.replace("tf.", "tf.compat.v1.", 1)
+  node.func = ast_edits.full_name_node(new_name)
+  logs.append((
+      ast_edits.INFO, node.lineno, node.col_offset,
+      "Renaming %s to %s because argument %s is present. %s" %
+      (full_name, new_name, arg_name, message if message is not None else "")
+  ))
+  return node
 
-  @staticmethod
-  def _softmax_cross_entropy_with_logits_transformer(
-      parent, node, full_name, name, logs):
-    def _wrap_label(parent, old_value):
-      """Wrap labels with tf.stop_gradient."""
-      if six.PY3:
-        new_value = ast.Call(
-            ast.Name(id="tf.stop_gradient", ctx=ast.Load()),
-            [old_value], [])
-      else:
-        new_value = ast.Call(
-            ast.Name(id="tf.stop_gradient", ctx=ast.Load()),
-            [old_value], [], None, None)
-
-      # This copies the prefix and suffix on old_value to new_value.
-      pasta.ast_utils.replace_child(parent, old_value, new_value)
-      ast.copy_location(new_value, old_value)
-
-    # Check if we have a labels keyword arg
-    for karg in node.keywords:
-      if karg.arg == "labels":
-        logs.append((ast_edits.INFO, node.lineno, node.col_offset,
-                     "Changing labels arg of "
-                     "tf.nn.softmax_cross_entropy_with_logits to "
-                     "tf.stop_gradient(labels). Please check this "
-                     "transformation.\n"))
-        _wrap_label(karg, karg.value)
-        return node
-    return node
 
-  @staticmethod
-  def _batch_gather_transformer(parent, node, full_name, name, logs):
-    # Check if the call already has a batch_dims argument
-    if any([kw.arg == "batch_dims" for kw in node.keywords]):
+def _add_argument_transformer(parent, node, full_name, name, logs,
+                              arg_name, arg_value_ast):
+  """Adds an argument (as a final kwarg arg_name=arg_value_ast)."""
+  node.keywords.append(ast.keyword(arg=arg_name, value=arg_value_ast))
+  logs.add((
+      ast_edits.INFO, node.lineno, node.col_offset,
+      "Adding argument '%s' to call to %s." % (pasta.dump(node.keywords[-1],
+                                                          full_name or name))
+  ))
+  return node
+
+
+def _iterator_transformer(parent, node, full_name, name, logs):
+  """Transform iterator methods to compat function calls."""
+  # First, check that node.func.value is not already something we like
+  # (tf.compat.v1.data), or something which is handled in the rename
+  # (tf.data). This transformer only handles the method call to function call
+  # conversion.
+  if full_name and (full_name.startswith("tf.compat.v1.data") or
+                    full_name.startswith("tf.data")):
+    return
+
+  # This should never happen, since we're only called for Attribute nodes.
+  if not isinstance(node.func, ast.Attribute):
+    return
+
+  # Transform from x.f(y) to tf.compat.v1.data.f(x, y)
+  # Fortunately, node.func.value should already have valid position info
+  node.args = [node.func.value] + node.args
+  node.func.value = ast_edits.full_name_node("tf.compat.v1.data")
+
+  logs.append((ast_edits.WARNING, node.lineno, node.col_offset,
+               "Changing dataset.%s() to tf.compat.v1.data.%s(dataset). "
+               "Please check this transformation.\n" % (name, name)))
+
+  return node
+
+
+def _dropout_transformer(parent, node, full_name, name, logs):
+  """Replace keep_prob with 1-rate."""
+  def _replace_keep_prob_node(parent, old_value):
+    """Replaces old_value with 1-(old_value)."""
+    one = ast.Num(n=1)
+    one.lineno = 0
+    one.col_offset = 0
+    new_value = ast.BinOp(left=one, op=ast.Sub(),
+                          right=old_value)
+    # This copies the prefix and suffix on old_value to new_value.
+    pasta.ast_utils.replace_child(parent, old_value, new_value)
+    ast.copy_location(new_value, old_value)
+    # Put parentheses around keep_prob.value (and remove the old prefix/
+    # suffix, they should only be around new_value).
+    pasta.base.formatting.set(old_value, "prefix", "(")
+    pasta.base.formatting.set(old_value, "suffix", ")")
+
+  # Check if we have a keep_prob keyword arg
+  for keep_prob in node.keywords:
+    if keep_prob.arg == "keep_prob":
       logs.append((ast_edits.INFO, node.lineno, node.col_offset,
-                   "tf.batch_gather already has batch_dims argument. Neat."))
-      return None
-
-    minus_one = ast.Num(n=-1)
-    minus_one.lineno = 0
-    minus_one.col_offset = 0
-    new_arg = ast.keyword("batch_dims", minus_one)
-    node.keywords.append(new_arg)
+                   "Changing keep_prob arg of tf.nn.dropout to rate\n"))
+      keep_prob.arg = "rate"
+      _replace_keep_prob_node(keep_prob, keep_prob.value)
+      return node
+
+  # Maybe it was a positional arg
+  if len(node.args) < 2:
+    logs.append((ast_edits.ERROR, node.lineno, node.col_offset,
+                 "tf.nn.dropout called without arguments, so "
+                 "automatic fix was disabled. tf.nn.dropout has changed "
+                 "the semantics of the second argument."))
+  else:
+    _replace_keep_prob_node(node, node.args[1])
     logs.append((ast_edits.INFO, node.lineno, node.col_offset,
-                 "Added keyword argument batch_dims=-1 to tf.batch_gather."))
+                 "Changing keep_prob arg of tf.nn.dropout to rate, and "
+                 "recomputing value.\n"))
+
     return node
 
-  @staticmethod
-  def _image_resize_transformer(parent, node, full_name, name, logs):
-    """Transforms image.resize_* to image.resize(..., method=*, ...)."""
-
-    resize_method = name[7:].upper()
-    new_arg = ast.keyword(arg="method",
-                          value=ast.Attribute(
-                              value=ast.Attribute(
-                                  value=ast.Attribute(
-                                      value=ast.Name(id="tf", ctx=ast.Load()),
-                                      attr="image", ctx=ast.Load()),
-                                  attr="ResizeMethod", ctx=ast.Load()),
-                              attr=resize_method, ctx=ast.Load()))
-
-    # Ensures a valid transformation when a positional name arg is given
-    if len(node.args) == 4:
-      pos_arg = ast.keyword(arg="preserve_aspect_ratio",
-                            value=node.args[-1])
-      node.args = node.args[:-1]
-      node.keywords.append(pos_arg)
-    if len(node.args) == 3:
-      pos_arg = ast.keyword(arg="align_corners",
-                            value=node.args[-1])
-      node.args = node.args[:-1]
-      node.keywords.append(pos_arg)
-
-    # Python3 ast requires the args for the Attribute, but codegen will mess up
-    # the arg order if we just set them to 0.
-    new_arg.value.lineno = node.lineno
-    new_arg.value.col_offset = node.col_offset+100
-
-    node.keywords.append(new_arg)
-    if isinstance(node.func, ast.Attribute):
-      node.func.attr = "resize"
+
+def _cast_transformer(parent, node, full_name, name, logs):
+  """Transforms to_int and to_float to cast(..., dtype=...)."""
+
+  # Find out the dtype to cast to from the function name
+  dtype_str = name[3:]
+  # Special cases where the full dtype is not given
+  if dtype_str == "float":
+    dtype_str = "float32"
+  elif dtype_str == "double":
+    dtype_str = "float64"
+  new_arg = ast.keyword(arg="dtype",
+                        value=ast.Attribute(value=ast.Name(id="tf",
+                                                           ctx=ast.Load()),
+                                            attr=dtype_str, ctx=ast.Load()))
+  # Ensures a valid transformation when a positional name arg is given
+  if len(node.args) == 2:
+    name_arg = ast.keyword(arg="name",
+                           value=node.args[-1])
+    node.args = node.args[:-1]
+    node.keywords.append(name_arg)
+
+  # Python3 ast requires the args for the Attribute, but codegen will mess up
+  # the arg order if we just set them to 0.
+  new_arg.value.lineno = node.lineno
+  new_arg.value.col_offset = node.col_offset+100
+
+  node.keywords.append(new_arg)
+  if isinstance(node.func, ast.Attribute):
+    node.func.attr = "cast"
+  else:
+    assert isinstance(node.func, ast.Name)
+    node.func.id = "cast"
+
+  logs.append((ast_edits.INFO, node.lineno, node.col_offset,
+               "Changed %s call to tf.cast(..., dtype=tf.%s)." % (full_name,
+                                                                  dtype_str)))
+  return node
+
+
+def _softmax_cross_entropy_with_logits_transformer(
+    parent, node, full_name, name, logs):
+  """Wrap labels argument with stop_gradients."""
+  def _wrap_label(parent, old_value):
+    """Wrap labels with tf.stop_gradient."""
+    if six.PY3:
+      new_value = ast.Call(
+          ast.Name(id="tf.stop_gradient", ctx=ast.Load()),
+          [old_value], [])
     else:
-      assert isinstance(node.func, ast.Name)
-      node.func.id = "resize"
+      new_value = ast.Call(
+          ast.Name(id="tf.stop_gradient", ctx=ast.Load()),
+          [old_value], [], None, None)
+
+    # This copies the prefix and suffix on old_value to new_value.
+    pasta.ast_utils.replace_child(parent, old_value, new_value)
+    ast.copy_location(new_value, old_value)
 
+  # Check if we have a labels keyword arg
+  for karg in node.keywords:
+    if karg.arg == "labels":
+      logs.append((ast_edits.INFO, node.lineno, node.col_offset,
+                   "Changing labels arg of "
+                   "tf.nn.softmax_cross_entropy_with_logits to "
+                   "tf.stop_gradient(labels). Please check this "
+                   "transformation.\n"))
+      _wrap_label(karg, karg.value)
+      return node
+  return node
+
+
+def _batch_gather_transformer(parent, node, full_name, name, logs):
+  """Add batch_dims argument for gather calls."""
+  # Check if the call already has a batch_dims argument
+  if any([kw.arg == "batch_dims" for kw in node.keywords]):
     logs.append((ast_edits.INFO, node.lineno, node.col_offset,
-                 "Changed %s call to tf.image.resize(..., "
-                 "method=tf.image.ResizeMethod.%s)." % (full_name,
-                                                        resize_method)))
+                 "tf.batch_gather already has batch_dims argument. Neat."))
+    return None
+
+  minus_one = ast.Num(n=-1)
+  minus_one.lineno = 0
+  minus_one.col_offset = 0
+  new_arg = ast.keyword("batch_dims", minus_one)
+  node.keywords.append(new_arg)
+  logs.append((ast_edits.INFO, node.lineno, node.col_offset,
+               "Added keyword argument batch_dims=-1 to tf.batch_gather."))
+  return node
+
+
+def _image_resize_transformer(parent, node, full_name, name, logs):
+  """Transforms image.resize_* to image.resize(..., method=*, ...)."""
+  resize_method = name[7:].upper()
+  new_arg = ast.keyword(arg="method",
+                        value=ast.Attribute(
+                            value=ast.Attribute(
+                                value=ast.Attribute(
+                                    value=ast.Name(id="tf", ctx=ast.Load()),
+                                    attr="image", ctx=ast.Load()),
+                                attr="ResizeMethod", ctx=ast.Load()),
+                            attr=resize_method, ctx=ast.Load()))
+
+  # Ensures a valid transformation when a positional name arg is given
+  if len(node.args) == 4:
+    pos_arg = ast.keyword(arg="preserve_aspect_ratio",
+                          value=node.args[-1])
+    node.args = node.args[:-1]
+    node.keywords.append(pos_arg)
+  if len(node.args) == 3:
+    pos_arg = ast.keyword(arg="align_corners",
+                          value=node.args[-1])
+    node.args = node.args[:-1]
+    node.keywords.append(pos_arg)
+
+  # Python3 ast requires the args for the Attribute, but codegen will mess up
+  # the arg order if we just set them to 0.
+  new_arg.value.lineno = node.lineno
+  new_arg.value.col_offset = node.col_offset+100
+
+  node.keywords.append(new_arg)
+  if isinstance(node.func, ast.Attribute):
+    node.func.attr = "resize"
+  else:
+    assert isinstance(node.func, ast.Name)
+    node.func.id = "resize"
+
+  logs.append((ast_edits.INFO, node.lineno, node.col_offset,
+               "Changed %s call to tf.image.resize(..., "
+               "method=tf.image.ResizeMethod.%s)." % (full_name,
+                                                      resize_method)))
+  return node
+
+
+def _pool_seed_transformer(parent, node, full_name, name, logs):
+  """Removes seed2 and deterministic, and adds non-zero seed if needed."""
+  # This requires that this function uses all kwargs (add to renames!).
+  seed_arg = None
+  deterministic = False
+  modified = False
+  new_keywords = []
+
+  for kw in node.keywords:
+    if sys.version_info[:2] >= (3, 5) and isinstance(kw, ast.Starred):
+      pass
+    elif kw.arg == "seed":
+      seed_arg = kw
+    elif kw.arg == "seed2" or kw.arg == "deterministic":
+      lineno = getattr(kw, "lineno", node.lineno)
+      col_offset = getattr(kw, "col_offset", node.col_offset)
+      logs.append((ast_edits.INFO, lineno, col_offset,
+                   "Removed argument %s for function %s" % (
+                       kw.arg, full_name or name)))
+      if kw.arg == "deterministic":
+        if not _is_ast_false(kw.value):
+          deterministic = True
+      modified = True
+      continue
+    new_keywords.append(kw)
+
+  if deterministic:
+    if seed_arg is None:
+      new_keywords.append(ast.keyword(arg="seed", value=ast.Num(42)))
+      logs.add((
+          ast_edits.INFO, node.lineno, node.col_offset,
+          "Adding seed=42 to call to %s since determinism was requested" % (
+              full_name or name)
+      ))
+    else:
+      logs.add((
+          ast_edits.WARNING, node.lineno, node.col_offset,
+          "The deterministic argument is deprecated for %s, pass a "
+          "non-zero seed for determinism. The deterministic argument is "
+          "present, possibly not False, and the seed is already set. The "
+          "converter cannot determine whether it is nonzero, please check."
+      ))
+
+  if modified:
+    node.keywords = new_keywords
     return node
+  else:
+    return
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_main.py b/tensorflow/tools/compatibility/tf_upgrade_v2_main.py
index b446452cfe..aecd74e71c 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_main.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_main.py
@@ -112,7 +112,7 @@ Simple usage:
     report = ("TensorFlow 2.0 Upgrade Script\n"
               "-----------------------------\n"
               "Converted %d files\n" % files_processed +
-              "Detected %d errors that require attention" % num_errors + "\n" +
+              "Detected %d issues that require attention" % num_errors + "\n" +
               "-" * 80 + "\n") + "".join(report)
     with open(report_filename, "w") as report_file:
       report_file.write(report)
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 8ff5d01ae6..2acaaacc18 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -376,19 +376,13 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
                   "tf.train.inverse_time_decay", "tf.train.cosine_decay",
                   "tf.train.cosine_decay_restarts",
                   "tf.train.linear_cosine_decay",
-                  "tf.train.noisy_linear_cosine_decay"]:
+                  "tf.train.noisy_linear_cosine_decay",
+                  "tf.train.piecewise_constant_decay",
+                 ]:
 
       text = "%s(a, b)\n" % decay
-      _, report, errors, _ = self._upgrade(text)
-      self.assertIn("%s requires manual check" % decay, errors[0])
-      self.assertIn("%s has been changed" % decay, report)
-
-  def testPiecewiseDecay(self):
-    text = "tf.train.piecewise_constant_decay(a, b)\n"
-    _, report, errors, _ = self._upgrade(text)
-    self.assertIn("tf.train.piecewise_constant_decay requires manual check",
-                  errors[0])
-    self.assertIn("tf.train.piecewise_constant_decay has been changed", report)
+      _, report, unused_errors, _ = self._upgrade(text)
+      self.assertIn("%s has been changed to return a callable" % decay, report)
 
   def testMetrics(self):
     metrics = [
@@ -427,16 +421,11 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
         "true_positives_at_thresholds",
     ]
     for m in metrics:
-      ns = "tf.metrics." + m
-      text = ns + "(a, b)"
-      _, report, errors, new_text = self._upgrade(text)
+      text = "tf.metrics." + m + "(a, b)"
+      _, report, unused_errors, new_text = self._upgrade(text)
       self.assertEqual("tf.compat.v1.metrics." + m + "(a, b)", new_text)
-      self.assertIn("%s requires manual check" % ns, errors[0])
       self.assertIn(
-          "tf.metrics have been converted to object oriented"
-          " versions in TF 2.0 and after. The metric function calls have been "
-          "converted to compat.v1 for backward compatibility. Please update "
-          "these calls to the TF 2.0 versions.", report)
+          "tf.metrics have been replaced with object oriented versions", report)
 
   def testLosses(self):
     losses = [
@@ -458,16 +447,11 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
         "sparse_softmax_cross_entropy",
     ]
     for l in losses:
-      ns = "tf.losses." + l
-      text = ns + "(a, b)"
-      _, report, errors, new_text = self._upgrade(text)
+      text = "tf.losses." + l + "(a, b)"
+      _, report, unused_errors, new_text = self._upgrade(text)
       self.assertEqual("tf.compat.v1.losses." + l + "(a, b)", new_text)
-      self.assertIn("%s requires manual check" % ns, errors[0])
       self.assertIn(
-          "tf.losses have been converted to object oriented"
-          " versions in TF 2.0 and after. The loss function calls have been "
-          "converted to compat.v1 for backward compatibility. Please update "
-          "these calls to the TF 2.0 versions.", report)
+          "tf.losses have been replaced with object oriented versions", report)
 
   def testEstimatorLossReductionChange(self):
     classes = [
@@ -605,10 +589,9 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
     self.assertEqual(errors, [])
 
     text = "tf.gradients(a, colocate_gradients_with_ops=False)\n"
-    _, unused_report, errors, new_text = self._upgrade(text)
+    _, report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual("tf.gradients(a)\n", new_text)
-    self.assertIn("tf.gradients", errors[0])
-    self.assertIn("requires manual check", errors[0])
+    self.assertIn("tf.gradients no longer takes", report)
 
   def testColocateGradientsWithOpsMinimize(self):
     text = "optimizer.minimize(a, foo=False)\n"
@@ -617,10 +600,9 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
     self.assertEqual(errors, [])
 
     text = "optimizer.minimize(a, colocate_gradients_with_ops=False)\n"
-    _, unused_report, errors, new_text = self._upgrade(text)
+    _, report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual("optimizer.minimize(a)\n", new_text)
-    self.assertIn("requires manual check", errors[0])
-    self.assertIn("minimize", errors[0])
+    self.assertIn("Optimizer.minimize no longer takes", report)
 
   def testColocateGradientsWithOpsComputeGradients(self):
     text = "optimizer.compute_gradients(a, foo=False)\n"
@@ -629,10 +611,9 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
     self.assertEqual(errors, [])
 
     text = "optimizer.compute_gradients(a, colocate_gradients_with_ops=False)\n"
-    _, unused_report, errors, new_text = self._upgrade(text)
+    _, report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual("optimizer.compute_gradients(a)\n", new_text)
-    self.assertIn("requires manual check", errors[0])
-    self.assertIn("compute_gradients", errors[0])
+    self.assertIn("Optimizer.compute_gradients no longer takes", report)
 
   def testExportSavedModelRename(self):
     text = "self.est.export_savedmodel(path)"
@@ -831,7 +812,7 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
             "validate_indices, max_norm)")
     expected_text = ("tf.nn.embedding_lookup(params=params, ids=ids, "
                      "partition_strategy=partition_strategy, name=name, "
-                     "validate_indices=validate_indices, max_norm=max_norm)")
+                     "max_norm=max_norm)")
     _, unused_report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, expected_text)
 
@@ -1029,29 +1010,43 @@ def _log_prob(self, x):
                  "assert_scalar"]:
       text = "tf.%s(a)" % name
       expected_text = "tf.compat.v1.%s(a)" % name
-      _, unused_report, errors, new_text = self._upgrade(text)
+      _, report, unused_errors, new_text = self._upgrade(text)
       self.assertEqual(expected_text, new_text)
-      self.assertIn("assert_* functions", errors[0])
+      self.assertIn("%s has been" % name, report)
 
       text = "tf.debugging.%s(a)" % name
       expected_text = "tf.compat.v1.debugging.%s(a)" % name
-      _, unused_report, errors, new_text = self._upgrade(text)
+      _, report, unused_errors, new_text = self._upgrade(text)
       self.assertEqual(expected_text, new_text)
-      self.assertIn("assert_* functions", errors[0])
+      self.assertIn("%s has been" % name, report)
 
   def testAssertRankStatements(self):
     for name in ["assert_rank", "assert_rank_at_least", "assert_rank_in"]:
       text = "tf.%s(a)" % name
       expected_text = "tf.compat.v1.%s(a)" % name
-      _, unused_report, errors, new_text = self._upgrade(text)
+      _, report, unused_errors, new_text = self._upgrade(text)
       self.assertEqual(expected_text, new_text)
-      self.assertIn("assert_rank_* functions", errors[0])
+      self.assertIn("%s has been" % name, report)
 
       text = "tf.debugging.%s(a)" % name
       expected_text = "tf.compat.v1.debugging.%s(a)" % name
-      _, unused_report, errors, new_text = self._upgrade(text)
+      _, report, unused_errors, new_text = self._upgrade(text)
       self.assertEqual(expected_text, new_text)
-      self.assertIn("assert_rank_* functions", errors[0])
+      self.assertIn("%s has been" % name, report)
+
+  def test_assert_equal_graph_def(self):
+    text = "tf.test.assert_equal_graph_def(a, b, checkpoint_v2=x)"
+    expected = "tf.test.assert_equal_graph_def(actual=a, expected=b)"
+    _, _, _, new_text = self._upgrade(text)
+    self.assertEqual(expected, new_text)
+
+  def test_sample_distorted_bounding_box(self):
+    # pylint: disable=line-too-long
+    text = "tf.image.sample_distorted_bounding_box(a, b, c, d, e, f, g, h, i, j)"
+    expected = "tf.image.sample_distorted_bounding_box(image_size=a, bounding_boxes=b, seed=c, min_object_covered=e, aspect_ratio_range=f, area_range=g, max_attempts=h, use_image_if_no_bounding_boxes=i, name=j)"
+    # pylint: enable=line-too-long
+    _, _, _, new_text = self._upgrade(text)
+    self.assertEqual(expected, new_text)
 
 
 class TestUpgradeFiles(test_util.TensorFlowTestCase):
-- 
GitLab


From 90901d44547e99f2cef0349a220032f9cee40762 Mon Sep 17 00:00:00 2001
From: Yanan Cao <ycao@google.com>
Date: Mon, 28 Jan 2019 20:11:41 -0800
Subject: [PATCH 1455/2345] Internal change

PiperOrigin-RevId: 231336248
---
 tensorflow/python/keras/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 46259ead99..c353590415 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -1021,6 +1021,7 @@ tf_py_test(
     shard_count = 6,
     tags = [
         "no_oss",
+        "notap",  #TODO(b/123544294): Re-enable this test.
         "notsan",
     ],
 )
-- 
GitLab


From d2cb5c33085c9a16e9f2f516a78ad280a5881558 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 28 Jan 2019 20:30:28 -0800
Subject: [PATCH 1456/2345] Support stringmaplist parsing.

PiperOrigin-RevId: 231337720
---
 tensorflow/lite/build_def.bzl                 |   2 +-
 tensorflow/lite/toco/BUILD                    |  14 ++
 tensorflow/lite/toco/args.cc                  | 169 ++++++++++++++++++
 tensorflow/lite/toco/args.h                   |  77 +-------
 .../lite/toco/model_cmdline_flags_test.cc     |  67 +++++++
 5 files changed, 257 insertions(+), 72 deletions(-)
 create mode 100644 tensorflow/lite/toco/args.cc
 create mode 100644 tensorflow/lite/toco/model_cmdline_flags_test.cc

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index bfac073d87..f4c9e4ea04 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -265,7 +265,7 @@ def generated_test_models():
         "logical_and",
         "logical_or",
         "logical_xor",
-        #"lstm", TODO(b/122889684): Resolve toco structured line parsing in oss.
+        "lstm",
         "max_pool",
         "maximum",
         "mean",
diff --git a/tensorflow/lite/toco/BUILD b/tensorflow/lite/toco/BUILD
index 47880c1f0b..3150c48c73 100644
--- a/tensorflow/lite/toco/BUILD
+++ b/tensorflow/lite/toco/BUILD
@@ -133,6 +133,7 @@ cc_library(
 cc_library(
     name = "model_cmdline_flags",
     srcs = [
+        "args.cc",
         "model_cmdline_flags.cc",
     ],
     hdrs = [
@@ -478,3 +479,16 @@ tf_cc_test(
         "@com_google_googletest//:gtest",
     ],
 )
+
+tf_cc_test(
+    name = "model_cmdline_flags_test",
+    srcs = [
+        "model_cmdline_flags_test.cc",
+    ],
+    deps = [
+        ":model_cmdline_flags",
+        ":model_flags_proto_cc",
+        "//tensorflow/lite/testing:util",
+        "@com_google_googletest//:gtest",
+    ],
+)
diff --git a/tensorflow/lite/toco/args.cc b/tensorflow/lite/toco/args.cc
new file mode 100644
index 0000000000..da8debc49a
--- /dev/null
+++ b/tensorflow/lite/toco/args.cc
@@ -0,0 +1,169 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/toco/args.h"
+#include "absl/strings/str_split.h"
+
+namespace toco {
+namespace {
+
+// Helper class for SplitStructuredLine parsing.
+class ClosingSymbolLookup {
+ public:
+  explicit ClosingSymbolLookup(const char* symbol_pairs)
+      : closing_(), valid_closing_() {
+    // Initialize the opening/closing arrays.
+    for (const char* symbol = symbol_pairs; *symbol != 0; ++symbol) {
+      unsigned char opening = *symbol;
+      ++symbol;
+      // If the string ends before the closing character has been found,
+      // use the opening character as the closing character.
+      unsigned char closing = *symbol != 0 ? *symbol : opening;
+      closing_[opening] = closing;
+      valid_closing_[closing] = true;
+      if (*symbol == 0) break;
+    }
+  }
+
+  ClosingSymbolLookup(const ClosingSymbolLookup&) = delete;
+  ClosingSymbolLookup& operator=(const ClosingSymbolLookup&) = delete;
+
+  // Returns the closing character corresponding to an opening one,
+  // or 0 if the argument is not an opening character.
+  char GetClosingChar(char opening) const {
+    return closing_[static_cast<unsigned char>(opening)];
+  }
+
+  // Returns true if the argument is a closing character.
+  bool IsClosing(char c) const {
+    return valid_closing_[static_cast<unsigned char>(c)];
+  }
+
+ private:
+  // Maps an opening character to its closing. If the entry contains 0,
+  // the character is not in the opening set.
+  char closing_[256];
+  // Valid closing characters.
+  bool valid_closing_[256];
+};
+
+bool SplitStructuredLine(absl::string_view line, char delimiter,
+                         const char* symbol_pairs,
+                         std::vector<absl::string_view>* cols) {
+  ClosingSymbolLookup lookup(symbol_pairs);
+
+  // Stack of symbols expected to close the current opened expressions.
+  std::vector<char> expected_to_close;
+
+  ABSL_RAW_CHECK(cols != nullptr, "");
+  cols->push_back(line);
+  for (size_t i = 0; i < line.size(); ++i) {
+    char c = line[i];
+    if (expected_to_close.empty() && c == delimiter) {
+      // We don't have any open expression, this is a valid separator.
+      cols->back().remove_suffix(line.size() - i);
+      cols->push_back(line.substr(i + 1));
+    } else if (!expected_to_close.empty() && c == expected_to_close.back()) {
+      // Can we close the currently open expression?
+      expected_to_close.pop_back();
+    } else if (lookup.GetClosingChar(c)) {
+      // If this is an opening symbol, we open a new expression and push
+      // the expected closing symbol on the stack.
+      expected_to_close.push_back(lookup.GetClosingChar(c));
+    } else if (lookup.IsClosing(c)) {
+      // Error: mismatched closing symbol.
+      return false;
+    }
+  }
+  if (!expected_to_close.empty()) {
+    return false;  // Missing closing symbol(s)
+  }
+  return true;  // Success
+}
+
+inline bool TryStripPrefixString(absl::string_view str,
+                                 absl::string_view prefix, string* result) {
+  bool res = absl::ConsumePrefix(&str, prefix);
+  result->assign(str.begin(), str.end());
+  return res;
+}
+
+inline bool TryStripSuffixString(absl::string_view str,
+                                 absl::string_view suffix, string* result) {
+  bool res = absl::ConsumeSuffix(&str, suffix);
+  result->assign(str.begin(), str.end());
+  return res;
+}
+
+}  // namespace
+
+bool Arg<toco::IntList>::Parse(string text) {
+  parsed_value_.elements.clear();
+  specified_ = true;
+  // strings::Split("") produces {""}, but we need {} on empty input.
+  // TODO(aselle): Moved this from elsewhere, but ahentz recommends we could
+  // use absl::SplitLeadingDec32Values(text.c_str(), &parsed_values_.elements)
+  if (!text.empty()) {
+    int32 element;
+    for (absl::string_view part : absl::StrSplit(text, ',')) {
+      if (!SimpleAtoi(part, &element)) return false;
+      parsed_value_.elements.push_back(element);
+    }
+  }
+  return true;
+}
+
+bool Arg<toco::StringMapList>::Parse(string text) {
+  parsed_value_.elements.clear();
+  specified_ = true;
+
+  if (text.empty()) {
+    return true;
+  }
+
+  std::vector<absl::string_view> outer_vector;
+  absl::string_view text_disposable_copy = text;
+  // TODO(aselle): Change argument parsing when absl supports structuredline.
+  SplitStructuredLine(text_disposable_copy, ',', "{}", &outer_vector);
+  for (const absl::string_view& outer_member_stringpiece : outer_vector) {
+    string outer_member(outer_member_stringpiece);
+    if (outer_member.empty()) {
+      continue;
+    }
+    string outer_member_copy = outer_member;
+    absl::StripAsciiWhitespace(&outer_member);
+    if (!TryStripPrefixString(outer_member, "{", &outer_member)) return false;
+    if (!TryStripSuffixString(outer_member, "}", &outer_member)) return false;
+    const std::vector<string> inner_fields_vector =
+        absl::StrSplit(outer_member, ',');
+
+    std::unordered_map<string, string> element;
+    for (const string& member_field : inner_fields_vector) {
+      std::vector<string> outer_member_key_value =
+          absl::StrSplit(member_field, ':');
+      if (outer_member_key_value.size() != 2) return false;
+      string& key = outer_member_key_value[0];
+      string& value = outer_member_key_value[1];
+      absl::StripAsciiWhitespace(&key);
+      absl::StripAsciiWhitespace(&value);
+      if (element.count(key) != 0) return false;
+      element[key] = value;
+    }
+    parsed_value_.elements.push_back(element);
+  }
+  return true;
+}
+
+}  // namespace toco
diff --git a/tensorflow/lite/toco/args.h b/tensorflow/lite/toco/args.h
index 188f2f7e7a..c6eeb2859a 100644
--- a/tensorflow/lite/toco/args.h
+++ b/tensorflow/lite/toco/args.h
@@ -22,10 +22,6 @@ limitations under the License.
 #include <unordered_map>
 #include <vector>
 #include "tensorflow/lite/toco/toco_port.h"
-#if defined(PLATFORM_GOOGLE)
-#include "strings/split.h"
-#include "strings/strip.h"
-#endif
 #include "absl/strings/numbers.h"
 #include "absl/strings/str_split.h"
 #include "tensorflow/lite/toco/toco_types.h"
@@ -64,7 +60,7 @@ class Arg final {
   const T& value() const { return value_; }
 
   // Parsing callback for the tensorflow::Flags code
-  bool parse(T value_in) {
+  bool Parse(T value_in) {
     value_ = value_in;
     specified_ = true;
     return true;
@@ -72,7 +68,7 @@ class Arg final {
 
   // Bind the parse member function so tensorflow::Flags can call it.
   std::function<bool(T)> bind() {
-    return std::bind(&Arg::parse, this, std::placeholders::_1);
+    return std::bind(&Arg::Parse, this, std::placeholders::_1);
   }
 
  private:
@@ -90,24 +86,10 @@ class Arg<toco::IntList> final {
   // Return true if the command line argument was specified on the command line.
   bool specified() const { return specified_; }
   // Bind the parse member function so tensorflow::Flags can call it.
-  bool parse(string text) {
-    parsed_value_.elements.clear();
-    specified_ = true;
-    // strings::Split("") produces {""}, but we need {} on empty input.
-    // TODO(aselle): Moved this from elsewhere, but ahentz recommends we could
-    // use absl::SplitLeadingDec32Values(text.c_str(), &parsed_values_.elements)
-    if (!text.empty()) {
-      int32 element;
-      for (absl::string_view part : absl::StrSplit(text, ',')) {
-        if (!SimpleAtoi(part, &element)) return false;
-        parsed_value_.elements.push_back(element);
-      }
-    }
-    return true;
-  }
+  bool Parse(string text);
 
   std::function<bool(string)> bind() {
-    return std::bind(&Arg::parse, this, std::placeholders::_1);
+    return std::bind(&Arg::Parse, this, std::placeholders::_1);
   }
 
   const toco::IntList& value() const { return parsed_value_; }
@@ -126,57 +108,10 @@ class Arg<toco::StringMapList> final {
   bool specified() const { return specified_; }
   // Bind the parse member function so tensorflow::Flags can call it.
 
-  bool parse(string text) {
-    parsed_value_.elements.clear();
-    specified_ = true;
-
-    if (text.empty()) {
-      return true;
-    }
-
-#if defined(PLATFORM_GOOGLE)
-    std::vector<absl::string_view> outer_vector;
-    absl::string_view text_disposable_copy = text;
-    SplitStructuredLine(text_disposable_copy, ',', "{}", &outer_vector);
-    for (const absl::string_view& outer_member_stringpiece : outer_vector) {
-      string outer_member(outer_member_stringpiece);
-      if (outer_member.empty()) {
-        continue;
-      }
-      string outer_member_copy = outer_member;
-      absl::StripAsciiWhitespace(&outer_member);
-      if (!strings::TryStripPrefixString(outer_member, "{", &outer_member))
-        return false;
-      if (!strings::TryStripSuffixString(outer_member, "}", &outer_member))
-        return false;
-      const std::vector<string> inner_fields_vector =
-          absl::StrSplit(outer_member, ',');
-
-      std::unordered_map<string, string> element;
-      for (const string& member_field : inner_fields_vector) {
-        std::vector<string> outer_member_key_value =
-            absl::StrSplit(member_field, ':');
-        if (outer_member_key_value.size() != 2) return false;
-        string& key = outer_member_key_value[0];
-        string& value = outer_member_key_value[1];
-        absl::StripAsciiWhitespace(&key);
-        absl::StripAsciiWhitespace(&value);
-        if (element.count(key) != 0) return false;
-        element[key] = value;
-      }
-      parsed_value_.elements.push_back(element);
-    }
-    return true;
-#else
-    // TODO(aselle): Fix argument parsing when absl supports structuredline
-    fprintf(stderr, "%s:%d StringMapList arguments not supported\n", __FILE__,
-            __LINE__);
-    abort();
-#endif
-  }
+  bool Parse(string text);
 
   std::function<bool(string)> bind() {
-    return std::bind(&Arg::parse, this, std::placeholders::_1);
+    return std::bind(&Arg::Parse, this, std::placeholders::_1);
   }
 
   const toco::StringMapList& value() const { return parsed_value_; }
diff --git a/tensorflow/lite/toco/model_cmdline_flags_test.cc b/tensorflow/lite/toco/model_cmdline_flags_test.cc
new file mode 100644
index 0000000000..b2f8dc5985
--- /dev/null
+++ b/tensorflow/lite/toco/model_cmdline_flags_test.cc
@@ -0,0 +1,67 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <string>
+#include <unordered_map>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+#include "tensorflow/lite/testing/util.h"
+#include "tensorflow/lite/toco/args.h"
+#include "tensorflow/lite/toco/model_cmdline_flags.h"
+
+namespace toco {
+namespace {
+
+TEST(ModelCmdlineFlagsTest, ParseArgsStringMapList) {
+  int args_count = 3;
+  const char* args[] = {
+      "toco",
+      "--input_arrays=input_1",
+      "--rnn_states={state_array:rnn/BasicLSTMCellZeroState/zeros,"
+      "back_edge_source_array:rnn/basic_lstm_cell/Add_1,size:4},"
+      "{state_array:rnn/BasicLSTMCellZeroState/zeros_1,"
+      "back_edge_source_array:rnn/basic_lstm_cell/Mul_2,size:4}",
+  };
+
+  string expected_input_arrays = "input_1";
+  std::vector<std::unordered_map<string, string>> expected_rnn_states;
+  expected_rnn_states.push_back(
+      {{"state_array", "rnn/BasicLSTMCellZeroState/zeros"},
+       {"back_edge_source_array", "rnn/basic_lstm_cell/Add_1"},
+       {"size", "4"}});
+  expected_rnn_states.push_back(
+      {{"state_array", "rnn/BasicLSTMCellZeroState/zeros_1"},
+       {"back_edge_source_array", "rnn/basic_lstm_cell/Mul_2"},
+       {"size", "4"}});
+
+  string message;
+  ParsedModelFlags result_flags;
+
+  EXPECT_TRUE(ParseModelFlagsFromCommandLineFlags(
+      &args_count, const_cast<char**>(args), &message, &result_flags));
+  EXPECT_EQ(result_flags.input_arrays.value(), expected_input_arrays);
+  EXPECT_EQ(result_flags.rnn_states.value().elements, expected_rnn_states);
+}
+
+}  // namespace
+}  // namespace toco
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  ::toco::port::InitGoogleWasDoneElsewhere();
+  return RUN_ALL_TESTS();
+}
-- 
GitLab


From 232b66c4b1193ffdaf2064198f04ff0a4d019cf3 Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Mon, 28 Jan 2019 20:42:52 -0800
Subject: [PATCH 1457/2345] Python bindings for calibration.

PiperOrigin-RevId: 231338705
---
 tensorflow/lite/python/optimize/BUILD         |  66 +++++
 .../python/optimize/calibration_wrapper.cc    | 226 ++++++++++++++++++
 .../python/optimize/calibration_wrapper.h     |  90 +++++++
 .../python/optimize/calibration_wrapper.i     |  27 +++
 tensorflow/lite/python/optimize/calibrator.py |  68 ++++++
 .../lite/python/optimize/calibrator_test.py   |  93 +++++++
 .../test_data/mobilenet_like_model.bin        | Bin 0 -> 2484 bytes
 7 files changed, 570 insertions(+)
 create mode 100644 tensorflow/lite/python/optimize/BUILD
 create mode 100644 tensorflow/lite/python/optimize/calibration_wrapper.cc
 create mode 100644 tensorflow/lite/python/optimize/calibration_wrapper.h
 create mode 100644 tensorflow/lite/python/optimize/calibration_wrapper.i
 create mode 100644 tensorflow/lite/python/optimize/calibrator.py
 create mode 100644 tensorflow/lite/python/optimize/calibrator_test.py
 create mode 100644 tensorflow/lite/python/optimize/test_data/mobilenet_like_model.bin

diff --git a/tensorflow/lite/python/optimize/BUILD b/tensorflow/lite/python/optimize/BUILD
new file mode 100644
index 0000000000..74b573b81f
--- /dev/null
+++ b/tensorflow/lite/python/optimize/BUILD
@@ -0,0 +1,66 @@
+package(
+    default_visibility = ["//visibility:public"],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+load("//tensorflow:tensorflow.bzl", "tf_py_wrap_cc")
+
+cc_library(
+    name = "calibration_wrapper_lib",
+    srcs = ["calibration_wrapper.cc"],
+    hdrs = ["calibration_wrapper.h"],
+    deps = [
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite/kernels:builtin_ops",
+        "//tensorflow/lite/python/interpreter_wrapper:python_error_reporter",
+        "//tensorflow/lite/python/interpreter_wrapper:python_utils",
+        "//tensorflow/lite/tools/optimize:calibration_reader",
+        "//tensorflow/lite/tools/optimize:calibrator_lib",
+        "//tensorflow/lite/tools/optimize:quantize_model",
+        "//third_party/py/numpy:headers",
+        "//third_party/python_runtime:headers",
+        "@com_google_absl//absl/memory",
+    ],
+)
+
+tf_py_wrap_cc(
+    name = "tensorflow_lite_wrap_calibration_wrapper",
+    srcs = [
+        "calibration_wrapper.i",
+    ],
+    deps = [
+        ":calibration_wrapper_lib",
+        "//third_party/python_runtime:headers",
+    ],
+)
+
+py_library(
+    name = "calibrator",
+    srcs = [
+        "calibrator.py",
+    ],
+    srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tensorflow/lite/python/optimize:tensorflow_lite_wrap_calibration_wrapper",
+        "//tensorflow/python:util",
+        "//third_party/py/numpy",
+    ],
+)
+
+py_test(
+    name = "calibrator_test",
+    srcs = ["calibrator_test.py"],
+    data = [":test_data"],
+    srcs_version = "PY2AND3",
+    tags = ["no_oss"],
+    deps = [
+        ":calibrator",
+        "//tensorflow/lite/python/optimize:tensorflow_lite_wrap_calibration_wrapper",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:platform",
+        "//third_party/py/numpy",
+    ],
+)
diff --git a/tensorflow/lite/python/optimize/calibration_wrapper.cc b/tensorflow/lite/python/optimize/calibration_wrapper.cc
new file mode 100644
index 0000000000..d6fe1fbdad
--- /dev/null
+++ b/tensorflow/lite/python/optimize/calibration_wrapper.cc
@@ -0,0 +1,226 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/python/optimize/calibration_wrapper.h"
+
+#include <memory>
+#include <sstream>
+#include <string>
+
+#include "absl/memory/memory.h"
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/model.h"
+#include "tensorflow/lite/python/interpreter_wrapper/python_error_reporter.h"
+#include "tensorflow/lite/python/interpreter_wrapper/python_utils.h"
+#include "tensorflow/lite/tools/optimize/calibration_reader.h"
+#include "tensorflow/lite/tools/optimize/calibrator.h"
+#include "tensorflow/lite/tools/optimize/quantize_model.h"
+
+// Disallow Numpy 1.7 deprecated symbols.
+#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
+
+#include <Python.h>
+
+#include "numpy/arrayobject.h"
+#include "numpy/ufuncobject.h"
+
+#if PY_MAJOR_VERSION >= 3
+#define PY_TO_CPPSTRING PyBytes_AsStringAndSize
+#define CPP_TO_PYSTRING PyBytes_FromStringAndSize
+#else
+#define PY_TO_CPPSTRING PyString_AsStringAndSize
+#define CPP_TO_PYSTRING PyString_FromStringAndSize
+#endif
+
+#define TFLITE_PY_CHECK(x)               \
+  if ((x) != kTfLiteOk) {                \
+    return error_reporter_->exception(); \
+  }
+
+#define TFLITE_PY_ENSURE_VALID_INTERPRETER()                               \
+  if (!interpreter_) {                                                     \
+    PyErr_SetString(PyExc_ValueError, "Interpreter was not initialized."); \
+    return nullptr;                                                        \
+  }
+
+namespace tflite {
+namespace calibration_wrapper {
+
+namespace {
+
+struct PyDecrefDeleter {
+  void operator()(PyObject* p) const { Py_DECREF(p); }
+};
+
+std::unique_ptr<tflite::ModelT> CreateMutableModel(const tflite::Model& model) {
+  std::unique_ptr<tflite::ModelT> copied_model =
+      absl::make_unique<tflite::ModelT>();
+  model.UnPackTo(copied_model.get(), nullptr);
+  return copied_model;
+}
+
+}  // namespace
+
+CalibrationWrapper::CalibrationWrapper(
+    std::unique_ptr<tflite::Interpreter> interpreter,
+    std::unique_ptr<tflite::ops::builtin::BuiltinOpResolver> resolver,
+    std::unique_ptr<tflite::interpreter_wrapper::PythonErrorReporter>
+        error_reporter,
+    std::unique_ptr<tflite::FlatBufferModel> model,
+    std::unique_ptr<tflite::optimize::calibration::CalibrationReader> reader)
+    : interpreter_(std::move(interpreter)),
+      error_reporter_(std::move(error_reporter)),
+      resolver_(std::move(resolver)),
+      model_(std::move(model)),
+      reader_(std::move(reader)) {}
+
+CalibrationWrapper::~CalibrationWrapper() {}
+
+PyObject* CalibrationWrapper::Prepare() {
+  TFLITE_PY_ENSURE_VALID_INTERPRETER();
+  TFLITE_PY_CHECK(interpreter_->AllocateTensors());
+  TFLITE_PY_CHECK(interpreter_->ResetVariableTensors());
+  Py_RETURN_NONE;
+}
+
+PyObject* CalibrationWrapper::FeedTensor(PyObject* input_value) {
+  TFLITE_PY_ENSURE_VALID_INTERPRETER();
+  if (!PyList_Check(input_value)) {
+    PyErr_Format(PyExc_ValueError,
+                 "Invalid input type: expected input to be a list.");
+    return nullptr;
+  }
+
+  const size_t inputs_size = PyList_Size(input_value);
+
+  if (inputs_size != interpreter_->inputs().size()) {
+    PyErr_Format(PyExc_ValueError,
+                 "Invalid input size: expected %ld items got %ld items.",
+                 interpreter_->inputs().size(), inputs_size);
+    return nullptr;
+  }
+
+  for (size_t i = 0; i < inputs_size; i++) {
+    PyObject* input = PyList_GetItem(input_value, i);
+    if (!input) {
+      return nullptr;
+    }
+    int input_tensor_idx = interpreter_->inputs()[i];
+    if (!SetTensor(input_tensor_idx, input)) {
+      return nullptr;
+    }
+  }
+
+  TFLITE_PY_CHECK(interpreter_->Invoke());
+  Py_RETURN_NONE;
+}
+
+PyObject* CalibrationWrapper::SetTensor(int index, PyObject* value) {
+  TFLITE_PY_ENSURE_VALID_INTERPRETER();
+
+  std::unique_ptr<PyObject, PyDecrefDeleter> array_safe(
+      PyArray_FromAny(value, nullptr, 0, 0, NPY_ARRAY_CARRAY, nullptr));
+  if (!array_safe) {
+    PyErr_SetString(PyExc_ValueError,
+                    "Failed to convert value into readable tensor.");
+    return nullptr;
+  }
+
+  PyArrayObject* array = reinterpret_cast<PyArrayObject*>(array_safe.get());
+  const TfLiteTensor* tensor = interpreter_->tensor(index);
+
+  if (python_utils::TfLiteTypeFromPyArray(array) != tensor->type) {
+    PyErr_Format(PyExc_ValueError,
+                 "Cannot set tensor:"
+                 " Got tensor of type %d"
+                 " but expected type %d for input %d, name: %s ",
+                 python_utils::TfLiteTypeFromPyArray(array), tensor->type,
+                 index, tensor->name);
+    return nullptr;
+  }
+
+  if (PyArray_NDIM(array) != tensor->dims->size) {
+    PyErr_SetString(PyExc_ValueError, "Cannot set tensor: Dimension mismatch");
+    return nullptr;
+  }
+
+  for (int j = 0; j < PyArray_NDIM(array); j++) {
+    if (tensor->dims->data[j] != PyArray_SHAPE(array)[j]) {
+      PyErr_SetString(PyExc_ValueError,
+                      "Cannot set tensor: Dimension mismatch");
+      return nullptr;
+    }
+  }
+
+  size_t size = PyArray_NBYTES(array);
+  if (size != tensor->bytes) {
+    PyErr_Format(PyExc_ValueError,
+                 "numpy array had %zu bytes but expected %zu bytes.", size,
+                 tensor->bytes);
+    return nullptr;
+  }
+  memcpy(tensor->data.raw, PyArray_DATA(array), size);
+  Py_RETURN_NONE;
+}
+
+PyObject* CalibrationWrapper::QuantizeModel() {
+  auto tflite_model = CreateMutableModel(*model_->GetModel());
+  reader_->AddCalibrationToModel(tflite_model.get());
+  flatbuffers::FlatBufferBuilder builder;
+  auto status = tflite::optimize::QuantizeModel(&builder, tflite_model.get(),
+                                                error_reporter_.get());
+  if (status != kTfLiteOk) {
+    error_reporter_->exception();
+    return nullptr;
+  }
+  return CPP_TO_PYSTRING(
+      reinterpret_cast<const char*>(builder.GetCurrentBufferPointer()),
+      builder.GetSize());
+}
+
+/*static*/ CalibrationWrapper* CalibrationWrapper::CreateWrapperCPPFromBuffer(
+    PyObject* data) {
+  using tflite::interpreter_wrapper::PythonErrorReporter;
+  char* buf = nullptr;
+  Py_ssize_t length;
+  std::unique_ptr<PythonErrorReporter> error_reporter(new PythonErrorReporter);
+  if (PY_TO_CPPSTRING(data, &buf, &length) == -1) {
+    return nullptr;
+  }
+  std::unique_ptr<tflite::FlatBufferModel> model =
+      tflite::FlatBufferModel::BuildFromBuffer(buf, length,
+                                               error_reporter.get());
+  if (!model) {
+    PyErr_Format(PyExc_ValueError, "Invalid model");
+    return nullptr;
+  }
+  auto resolver = absl::make_unique<tflite::ops::builtin::BuiltinOpResolver>();
+  std::unique_ptr<tflite::Interpreter> interpreter;
+  std::unique_ptr<tflite::optimize::calibration::CalibrationReader> reader;
+  auto status = tflite::optimize::calibration::BuildLoggingInterpreter(
+      *model, *resolver, &interpreter, &reader);
+  if (status != kTfLiteOk) {
+    error_reporter->exception();
+    return nullptr;
+  }
+
+  auto wrapper = new CalibrationWrapper(
+      std::move(interpreter), std::move(resolver), std::move(error_reporter),
+      std::move(model), std::move(reader));
+  return wrapper;
+}
+
+}  // namespace calibration_wrapper
+}  // namespace tflite
diff --git a/tensorflow/lite/python/optimize/calibration_wrapper.h b/tensorflow/lite/python/optimize/calibration_wrapper.h
new file mode 100644
index 0000000000..213bc4a182
--- /dev/null
+++ b/tensorflow/lite/python/optimize/calibration_wrapper.h
@@ -0,0 +1,90 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_PYTHON_OPTIMIZE_CALIBRATION_WRAPPER_H_
+#define TENSORFLOW_LITE_PYTHON_OPTIMIZE_CALIBRATION_WRAPPER_H_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+// Place `<locale>` before <Python.h> to avoid build failures in macOS.
+#include <locale>
+
+// The empty line above is on purpose as otherwise clang-format will
+// automatically move <Python.h> before <locale>.
+#include <Python.h>
+
+// We forward declare TFLite classes here to avoid exposing them to SWIG.
+namespace tflite {
+namespace ops {
+namespace builtin {
+class BuiltinOpResolver;
+}  // namespace builtin
+}  // namespace ops
+
+class FlatBufferModel;
+class Interpreter;
+
+namespace interpreter_wrapper {
+class PythonErrorReporter;
+}  // namespace interpreter_wrapper
+
+namespace optimize {
+namespace calibration {
+class CalibrationReader;
+}  // namespace calibration
+}  // namespace optimize
+
+namespace calibration_wrapper {
+
+class CalibrationWrapper {
+ public:
+  // SWIG caller takes ownership of pointer.
+  static CalibrationWrapper* CreateWrapperCPPFromBuffer(PyObject* data);
+  ~CalibrationWrapper();
+
+  PyObject* Prepare();
+
+  PyObject* FeedTensor(PyObject* input_value);
+
+  PyObject* QuantizeModel();
+
+ private:
+  // CalibrationWrapper is not copyable or assignable. We avoid the use of
+  // CalibrationWrapper() = delete here for SWIG compatibility.
+  CalibrationWrapper(
+      std::unique_ptr<tflite::Interpreter> interpreter,
+      std::unique_ptr<tflite::ops::builtin::BuiltinOpResolver> resolver,
+      std::unique_ptr<tflite::interpreter_wrapper::PythonErrorReporter>
+          error_reporter,
+      std::unique_ptr<tflite::FlatBufferModel> model,
+      std::unique_ptr<tflite::optimize::calibration::CalibrationReader> reader);
+
+  CalibrationWrapper(const CalibrationWrapper& rhs);
+
+  PyObject* SetTensor(int index, PyObject* value);
+
+  std::unique_ptr<tflite::Interpreter> interpreter_;
+  std::unique_ptr<tflite::interpreter_wrapper::PythonErrorReporter>
+      error_reporter_;
+  std::unique_ptr<tflite::ops::builtin::BuiltinOpResolver> resolver_;
+  std::unique_ptr<tflite::FlatBufferModel> model_;
+  std::unique_ptr<tflite::optimize::calibration::CalibrationReader> reader_;
+};
+
+}  // namespace calibration_wrapper
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_PYTHON_OPTIMIZE_CALIBRATION_WRAPPER_H_
diff --git a/tensorflow/lite/python/optimize/calibration_wrapper.i b/tensorflow/lite/python/optimize/calibration_wrapper.i
new file mode 100644
index 0000000000..094ac20733
--- /dev/null
+++ b/tensorflow/lite/python/optimize/calibration_wrapper.i
@@ -0,0 +1,27 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+%include "std_string.i"
+
+
+%{
+#define SWIG_FILE_WITH_INIT
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/model.h"
+#include "tensorflow/lite/python/optimize/calibration_wrapper.h"
+%}
+
+
+%include "tensorflow/lite/python/optimize/calibration_wrapper.h"
\ No newline at end of file
diff --git a/tensorflow/lite/python/optimize/calibrator.py b/tensorflow/lite/python/optimize/calibrator.py
new file mode 100644
index 0000000000..18aabba96e
--- /dev/null
+++ b/tensorflow/lite/python/optimize/calibrator.py
@@ -0,0 +1,68 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Python wrapper for post training quantization with calibration."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.util.lazy_loader import LazyLoader
+
+# Lazy load since some of the performance benchmark skylark rules
+# break dependencies. Must use double quotes to match code internal rewrite
+# rule.
+_calibration_wrapper = LazyLoader(
+    "_calibration_wrapper", globals(),
+    "tensorflow.lite.python.optimize."
+    "tensorflow_lite_wrap_calibration_wrapper")
+
+
+class Calibrator(object):
+  """Calibrates a floating point model and then quantizes it.
+
+  This is an internal class, not a public interface.
+  """
+
+  def __init__(self, model_content):
+    """Constructor.
+
+    Args:
+      model_content: Content of a TF-Lite Flatbuffer file.
+
+    Raises:
+      ValueError: If the calibrator was unable to open the model.
+    """
+    if not model_content:
+      raise ValueError("`model_content` must be specified.")
+    try:
+      self._calibrator = (_calibration_wrapper.CalibrationWrapper
+                          .CreateWrapperCPPFromBuffer(model_content))
+    except Exception as e:
+      raise ValueError("Failed to parse the model: %s." % e)
+    if not self._calibrator:
+      raise ValueError("Failed to parse the model.")
+
+  def calibrate_and_quantize(self, dataset_gen):
+    """Calibrates the model with specified generator and then quantizes it.
+
+    Returns:
+      A quantized model.
+
+    Args:
+      dataset_gen: A generator that generates calibration samples.
+    """
+    self._calibrator.Prepare()
+    for calibration_sample in dataset_gen():
+      self._calibrator.FeedTensor([calibration_sample])
+    return self._calibrator.QuantizeModel()
diff --git a/tensorflow/lite/python/optimize/calibrator_test.py b/tensorflow/lite/python/optimize/calibrator_test.py
new file mode 100644
index 0000000000..230b46b3df
--- /dev/null
+++ b/tensorflow/lite/python/optimize/calibrator_test.py
@@ -0,0 +1,93 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for Calibrator."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+import numpy as np
+
+from tensorflow.lite.python.optimize import calibrator as _calibrator
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import resource_loader
+from tensorflow.python.platform import test
+
+
+class CalibratorTest(test_util.TensorFlowTestCase):
+
+  def test_calibration_with_quantization(self):
+    model_path = resource_loader.get_path_to_datafile(
+        'test_data/mobilenet_like_model.bin')
+    float_model = open(model_path, 'rb').read()
+    quantizer = _calibrator.Calibrator(float_model)
+
+    # Input generator for the model.
+    def input_gen():
+      for _ in range(10):
+        yield np.ones(shape=(1, 5, 5, 3), dtype=np.float32)
+
+    quantized_model = quantizer.calibrate_and_quantize(input_gen)
+    self.assertIsNotNone(quantized_model)
+
+  def test_invalid_model_buffer(self):
+    float_model = b'\0' * 100
+    with self.assertRaisesWithRegexpMatch(ValueError,
+                                          'Failed to parse the model'):
+      _calibrator.Calibrator(float_model)
+
+  def test_empty_calibrator_gen(self):
+    model_path = resource_loader.get_path_to_datafile(
+        'test_data/mobilenet_like_model.bin')
+    float_model = open(model_path, 'rb').read()
+    quantizer = _calibrator.Calibrator(float_model)
+
+    def empty_input_gen():
+      for i in ():
+        yield i
+
+    with self.assertRaises(RuntimeError):
+      quantizer.calibrate_and_quantize(empty_input_gen)
+
+  def test_invalid_shape_calibrator_gen(self):
+    model_path = resource_loader.get_path_to_datafile(
+        'test_data/mobilenet_like_model.bin')
+    float_model = open(model_path, 'rb').read()
+    quantizer = _calibrator.Calibrator(float_model)
+
+    # Input generator with incorrect shape.
+    def input_gen():
+      for _ in range(10):
+        yield np.ones(shape=(1, 2, 2, 3), dtype=np.float32)
+
+    with self.assertRaisesWithRegexpMatch(ValueError, 'Dimension mismatch'):
+      quantizer.calibrate_and_quantize(input_gen)
+
+  def test_invalid_type_calibrator_gen(self):
+    model_path = resource_loader.get_path_to_datafile(
+        'test_data/mobilenet_like_model.bin')
+    float_model = open(model_path, 'rb').read()
+    quantizer = _calibrator.Calibrator(float_model)
+
+    # Input generator with incorrect shape.
+    def input_gen():
+      for _ in range(10):
+        yield np.ones(shape=(1, 5, 5, 3), dtype=np.int32)
+
+    with self.assertRaises(ValueError):
+      quantizer.calibrate_and_quantize(input_gen)
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/lite/python/optimize/test_data/mobilenet_like_model.bin b/tensorflow/lite/python/optimize/test_data/mobilenet_like_model.bin
new file mode 100644
index 0000000000000000000000000000000000000000..a2909249ffd2675fad9c0cd60a6ff75f940b3fb0
GIT binary patch
literal 2484
zcmb1OU|<Mw^D$;%;A4<rU}4~3;9(G85MkhBU|?WoU|?|JWMJT7U|<knU|>*SU|?Wm
zU|=}F#K6E0(&z8&ui%`YSC(2-lA59i)-#2PfuVwlf#DA$1H%di28JF628I>}28J32
z28I#_28J9428I*{1_lEL1_liVh}kR*3=AGW|NsB?<NtpU8)P2?5=i;^|G&^rh@1)o
z14DyByzPXGvi1z$=kBwRyu6PgOm%NTmf${<2oC!V=BxGzTu9nC<LCeXbAJB+zu@Qp
z|4V-U|G(nr|Nmi7GtkY?gNnN_GB9M?&$Mf_nPm6Es($Z`=gao(Slqf#%=FIQ7nuw8
z@u~Lio3X5BZ^Qy!+cV-5Y`^`!y?23(!@kLOv-YuFk+=I6EwV4$V(z{ND~|6iF6`O2
zs*lO8c*e|qVqwMmE~GW@(K0cz6RYXk$KBX!=diBE)}e0R-sYz>_KBIV+562~e4kJ2
zlD#~?&e~o(S+g&@xoY1b{S-UZ_4DiwwoJE6@Xy^N_P1l7UQGKwhDAO5Hq<<}`7P+V
z@3U{yK8CefwmuRE_6A67w_$J#wOJE>Y;S{c&|V>tSi1+c_WMk{CfMFO&1ZLd-by=%
zY;oIezqozRbOY@U^kmpp|DR#kbN#yQx4S-e4e}57h%~p_-ZNgc@3v0Vz5|=&_8oW;
zyO*sn%TCAdg6+54b#@uI4EN^B%&<H0X3;(~aV0w~OD4Opk017$^>^)CmakzK7MgCi
z?-wMlKyiJ<bAer=tkJ&GviN;dZoAkqD7xFlM{L?RCuoVCQp@yx9~!jwZF|JAZ^c_p
zJG;$$><*~T+auvJ)y`&`>b@1+jkZ1=hwOT^=G(rp$+Rmd5VaHNPTbqikhjlA^}$|=
z?Kf-%PCeZ7Ap6dqHYfLeX1^i!qQ~!&-~a#fFo?iYH%K0omPHsC7&t&_oCT7uL1`O=
zK}nAnlwKJb7&b65Fi0>lFkE0@V7SA?z#ziRz!1X3z_5dnfx&~Bf#DA{4KHD4U`SwK
zU?}?Y|34_Nfb`3N(lrAE11L>{%mw8q7}jQBU?@&4NGwWB%1Mn+&d)0|O3}v<NJ%Xy
z$tce(PG!jW^Z)-jsM#R*gY2+?=0}j3YQ&fqpOl$c%)qea&;S2Ye<AMVW?*0dxf7Hh
zVeVvQU|;}YkhvgxAg+tI(09sAEOtx*WnYFLfByfEgPOqwHNyewSCBrCe?b_;R>bDE
zg8a<95|G#8vr~)mQbB3O=kNdjo1kWLLd`URnh7!s<R%aXnF%VN0&)_QQ#0~&Qc{cJ
z4GlnX$pemKP@IC?4f78JL%lOdKPX;>{z3c=QUmfgC=6lhK<2_Qs9eN|4rKrT{r^7>
zY8E@xt;l9EF)%QIFvvVf1_p-2vecr)^wju*{QR8EymX@!eaEu&fc$)rzg_<Q|Gx@q
z78}$oWPgL)4#FU_L>U+uauX|%%<)aEfSS|u@BjZ#P;)@x1o9uKyoC7=WCjR>%mKMM
zI6ti<H?e}@$iM&pwf;lG1*8@fcA&BurWTY>;AsF<-W$0<QxXFM!~g&P)1Z1m=7Q|h
zfyNn#kBmWe2HYHItULVw|9=@&KPbFF`dy&<VfMq!z?zQWSp<=eSQyyAc@9*kfpR@0
zWe9-lCy?Jj=7MMsr2GR){}}O&A%L(9l9NDrn+0qaDBeymGB8Ad42RZzAoD@tney-d
ze+C8yP7oVpFDN`g@dBbj%0YTTGzf#zC<6l<0|x`h7a(4P{St_NR!}&=^n=0)rXLh;
zAiW@SKsrF-3Gy0941_`QAS}WFO4Xnmo*i7DGcbrSfXo18cbK`LxC4oS^nua{hz98e
znF|U}2?lnQHUc(#LjHsO%Lvj7G8g1lQ22sqkRL$h1t@=m$}x}{kUbFdxfleX=7ZEA
zo4o>JCKJdl*vtf(3(C`=@(g4qR6hf#+=sasWFNAbAax=PGEo1pL&HJ>N;BYeKgg}1
z`~`|nkQpGmL1GLHT;R46s1*gWA0!9DAhn?Q2IVPGIDp&_(t8c!C&_ydTJRZ!X8Z-B
Xnb{cF7+4v=amoTp59|;&0|PSvTB#`M

literal 0
HcmV?d00001

-- 
GitLab


From 01cf6233d62d476676ffbea0fad46660a0ba9a16 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Mon, 28 Jan 2019 20:47:19 -0800
Subject: [PATCH 1458/2345] Add v2 APIs for AUCCurve and AUCSummationMethod
 enums.

PiperOrigin-RevId: 231339023
---
 tensorflow/python/keras/metrics.py            | 20 ++++------
 .../keras/metrics_confusion_matrix_test.py    | 39 ++++++++++++-------
 .../python/keras/utils/metrics_utils.py       | 34 ++++++++++++++++
 .../v1/tensorflow.keras.metrics.-a-u-c.pbtxt  |  2 +-
 .../v2/tensorflow.keras.metrics.-a-u-c.pbtxt  |  2 +-
 5 files changed, 68 insertions(+), 29 deletions(-)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 6aed0e7ac6..4662535fcd 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -1607,8 +1607,8 @@ class AUC(Metric):
 
   def __init__(self,
                num_thresholds=200,
-               curve=metrics_utils.AUCCurve.ROC,
-               summation_method=metrics_utils.AUCSummationMethod.INTERPOLATION,
+               curve='ROC',
+               summation_method='interpolation',
                name=None,
                dtype=None):
     """Creates an `AUC` instance.
@@ -1631,18 +1631,12 @@ class AUC(Metric):
     # Validate configurations.
     if num_thresholds <= 1:
       raise ValueError('`num_thresholds` must be > 1.')
-    if curve not in list(metrics_utils.AUCCurve):
-      raise ValueError('Invalid curve: "{}". Valid options are: "{}"'.format(
-          curve, list(metrics_utils.AUCCurve)))
-    if summation_method not in list(metrics_utils.AUCSummationMethod):
-      raise ValueError(
-          'Invalid summation method: "{}". Valid options are: "{}"'.format(
-              summation_method, list(metrics_utils.AUCSummationMethod)))
 
     # Update properties.
     self.num_thresholds = num_thresholds
-    self.curve = curve
-    self.summation_method = summation_method
+    self.curve = metrics_utils.AUCCurve.from_str(curve)
+    self.summation_method = metrics_utils.AUCSummationMethod.from_str(
+        summation_method)
     super(AUC, self).__init__(name=name, dtype=dtype)
 
     # Create metric variables
@@ -1802,8 +1796,8 @@ class AUC(Metric):
   def get_config(self):
     config = {
         'num_thresholds': self.num_thresholds,
-        'curve': self.curve,
-        'summation_method': self.summation_method,
+        'curve': self.curve.value,
+        'summation_method': self.summation_method.value,
     }
     base_config = super(AUC, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
diff --git a/tensorflow/python/keras/metrics_confusion_matrix_test.py b/tensorflow/python/keras/metrics_confusion_matrix_test.py
index 6a0fcfed07..972f7b6de7 100644
--- a/tensorflow/python/keras/metrics_confusion_matrix_test.py
+++ b/tensorflow/python/keras/metrics_confusion_matrix_test.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import json
+
 from absl.testing import parameterized
 import numpy as np
 
@@ -949,8 +951,8 @@ class AUCTest(test.TestCase):
   def test_config(self):
     auc_obj = metrics.AUC(
         num_thresholds=100,
-        curve=metrics_utils.AUCCurve.PR,
-        summation_method=metrics_utils.AUCSummationMethod.MAJORING,
+        curve='PR',
+        summation_method='majoring',
         name='auc_1')
     self.assertEqual(auc_obj.name, 'auc_1')
     self.assertEqual(len(auc_obj.variables), 4)
@@ -958,6 +960,8 @@ class AUCTest(test.TestCase):
     self.assertEqual(auc_obj.curve, metrics_utils.AUCCurve.PR)
     self.assertEqual(auc_obj.summation_method,
                      metrics_utils.AUCSummationMethod.MAJORING)
+    old_config = auc_obj.get_config()
+    self.assertDictEqual(old_config, json.loads(json.dumps(old_config)))
 
     # Check save and restore config
     auc_obj2 = metrics.AUC.from_config(auc_obj.get_config())
@@ -967,6 +971,8 @@ class AUCTest(test.TestCase):
     self.assertEqual(auc_obj2.curve, metrics_utils.AUCCurve.PR)
     self.assertEqual(auc_obj2.summation_method,
                      metrics_utils.AUCSummationMethod.MAJORING)
+    new_config = auc_obj2.get_config()
+    self.assertDictEqual(old_config, new_config)
 
   def test_value_is_idempotent(self):
     self.setup()
@@ -1021,8 +1027,7 @@ class AUCTest(test.TestCase):
   def test_weighted_roc_majoring(self):
     self.setup()
     auc_obj = metrics.AUC(
-        num_thresholds=self.num_thresholds,
-        summation_method=metrics_utils.AUCSummationMethod.MAJORING)
+        num_thresholds=self.num_thresholds, summation_method='majoring')
     self.evaluate(variables.variables_initializer(auc_obj.variables))
     result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)
 
@@ -1037,8 +1042,7 @@ class AUCTest(test.TestCase):
   def test_weighted_roc_minoring(self):
     self.setup()
     auc_obj = metrics.AUC(
-        num_thresholds=self.num_thresholds,
-        summation_method=metrics_utils.AUCSummationMethod.MINORING)
+        num_thresholds=self.num_thresholds, summation_method='minoring')
     self.evaluate(variables.variables_initializer(auc_obj.variables))
     result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)
 
@@ -1054,8 +1058,8 @@ class AUCTest(test.TestCase):
     self.setup()
     auc_obj = metrics.AUC(
         num_thresholds=self.num_thresholds,
-        curve=metrics_utils.AUCCurve.PR,
-        summation_method=metrics_utils.AUCSummationMethod.MAJORING)
+        curve='PR',
+        summation_method='majoring')
     self.evaluate(variables.variables_initializer(auc_obj.variables))
     result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)
 
@@ -1071,8 +1075,8 @@ class AUCTest(test.TestCase):
     self.setup()
     auc_obj = metrics.AUC(
         num_thresholds=self.num_thresholds,
-        curve=metrics_utils.AUCCurve.PR,
-        summation_method=metrics_utils.AUCSummationMethod.MINORING)
+        curve='PR',
+        summation_method='minoring')
     self.evaluate(variables.variables_initializer(auc_obj.variables))
     result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)
 
@@ -1086,10 +1090,7 @@ class AUCTest(test.TestCase):
 
   def test_weighted_pr_interpolation(self):
     self.setup()
-    auc_obj = metrics.AUC(
-        num_thresholds=self.num_thresholds,
-        curve=metrics_utils.AUCCurve.PR,
-        summation_method=metrics_utils.AUCSummationMethod.INTERPOLATION)
+    auc_obj = metrics.AUC(num_thresholds=self.num_thresholds, curve='PR')
     self.evaluate(variables.variables_initializer(auc_obj.variables))
     result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)
 
@@ -1115,6 +1116,16 @@ class AUCTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, '`num_thresholds` must be > 1.'):
       metrics.AUC(num_thresholds=1)
 
+  def test_invalid_curve(self):
+    with self.assertRaisesRegexp(ValueError,
+                                 'Invalid AUC curve value "Invalid".'):
+      metrics.AUC(curve='Invalid')
+
+  def test_invalid_summation_method(self):
+    with self.assertRaisesRegexp(
+        ValueError, 'Invalid AUC summation method value "Invalid".'):
+      metrics.AUC(summation_method='Invalid')
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/utils/metrics_utils.py b/tensorflow/python/keras/utils/metrics_utils.py
index c5ff617ad7..329cb0b278 100644
--- a/tensorflow/python/keras/utils/metrics_utils.py
+++ b/tensorflow/python/keras/utils/metrics_utils.py
@@ -161,15 +161,49 @@ class ConfusionMatrix(Enum):
 
 
 class AUCCurve(Enum):
+  """Type of AUC Curve (ROC or PR)."""
   ROC = 'ROC'
   PR = 'PR'
 
+  @staticmethod
+  def from_str(key):
+    if key in ('pr', 'PR'):
+      return AUCCurve.PR
+    elif key in ('roc', 'ROC'):
+      return AUCCurve.ROC
+    else:
+      raise ValueError('Invalid AUC curve value "%s".' % key)
+
 
 class AUCSummationMethod(Enum):
+  """Type of AUC summation method.
+
+  https://en.wikipedia.org/wiki/Riemann_sum)
+
+  Contains the following values:
+  * 'interpolation': Applies mid-point summation scheme for `ROC` curve. For
+    `PR` curve, interpolates (true/false) positives but not the ratio that is
+    precision (see Davis & Goadrich 2006 for details).
+  * 'minoring': Applies left summation for increasing intervals and right
+    summation for decreasing intervals.
+  * 'majoring': Applies right summation for increasing intervals and left
+    summation for decreasing intervals.
+  """
   INTERPOLATION = 'interpolation'
   MAJORING = 'majoring'
   MINORING = 'minoring'
 
+  @staticmethod
+  def from_str(key):
+    if key in ('interpolation', 'Interpolation'):
+      return AUCSummationMethod.INTERPOLATION
+    elif key in ('majoring', 'Majoring'):
+      return AUCSummationMethod.MAJORING
+    elif key in ('minoring', 'Minoring'):
+      return AUCSummationMethod.MINORING
+    else:
+      raise ValueError('Invalid AUC summation method value "%s".' % key)
+
 
 def update_confusion_matrix_variables(variables_to_update,
                                       y_true,
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-a-u-c.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-a-u-c.pbtxt
index 8019d43c78..beaa6f0596 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-a-u-c.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-a-u-c.pbtxt
@@ -87,7 +87,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'num_thresholds\', \'curve\', \'summation_method\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'200\', \'AUCCurve.ROC\', \'AUCSummationMethod.INTERPOLATION\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'num_thresholds\', \'curve\', \'summation_method\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'200\', \'ROC\', \'interpolation\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_loss"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-a-u-c.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-a-u-c.pbtxt
index 8019d43c78..beaa6f0596 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-a-u-c.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-a-u-c.pbtxt
@@ -87,7 +87,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'num_thresholds\', \'curve\', \'summation_method\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'200\', \'AUCCurve.ROC\', \'AUCSummationMethod.INTERPOLATION\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'num_thresholds\', \'curve\', \'summation_method\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'200\', \'ROC\', \'interpolation\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_loss"
-- 
GitLab


From dd7de7bbac0a227166d11004e57ceb5522dba7ec Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 28 Jan 2019 21:18:14 -0800
Subject: [PATCH 1459/2345] Add support for final partial batch in predict()
 for TPUStrategy

PiperOrigin-RevId: 231341652
---
 tensorflow/contrib/distribute/python/BUILD    |   1 +
 .../contrib/distribute/python/keras_test.py   | 125 ++++++++++++++++++
 tensorflow/python/distribute/input_lib.py     |  61 +++++----
 tensorflow/python/keras/BUILD                 |   3 +
 .../engine/distributed_training_utils.py      |  46 +++++--
 .../engine/partial_batch_padding_handler.py   | 111 ++++++++++++++++
 tensorflow/python/keras/engine/training.py    |  15 ++-
 .../keras/engine/training_distributed.py      |  59 +++++++--
 8 files changed, 371 insertions(+), 50 deletions(-)
 create mode 100644 tensorflow/python/keras/engine/partial_batch_padding_handler.py

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index b43cd04c5e..c546b83859 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -566,6 +566,7 @@ py_library(
 distribute_py_test(
     name = "keras_test",
     srcs = ["keras_test.py"],
+    full_precision = True,
     main = "keras_test.py",
     shard_count = 16,
     tags = [
diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 17ed871459..4c900e27a4 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -71,6 +71,18 @@ def simple_functional_model():
   return model
 
 
+def simple_multi_inputs_multi_outputs_model():
+  input_a = keras.layers.Input(shape=(16,), name='input_a')
+  input_b = keras.layers.Input(shape=(16,), name='input_b')
+
+  merged = keras.layers.concatenate([input_a, input_b], name='merge')
+  output_c = keras.layers.Dense(3, activation='softmax', name='dense_2')(merged)
+  output_d = keras.layers.Dense(2, activation='softmax', name='dense_3')(merged)
+  model = keras.models.Model(
+      inputs=[input_a, input_b], outputs=[output_c, output_d])
+  return model
+
+
 def multi_inputs_multi_outputs_model():
   input_a = keras.layers.Input(shape=(16,), name='input_a')
   input_b = keras.layers.Input(shape=(16,), name='input_b')
@@ -671,6 +683,61 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
       self.assertAllEqual([6, 7], outs[0].shape)
       self.assertAllEqual([6, 7], outs[1].shape)
 
+  @combinations.generate(tpu_strategy_combinations())
+  def test_predict_with_partial_batch(self, distribution):
+    with self.cached_session():
+      optimizer = gradient_descent.GradientDescentOptimizer(0.001)
+      loss = 'mse'
+
+      with distribution.scope():
+        model_with_ds_strategy = get_model()
+        model_with_ds_strategy.compile(optimizer, loss)
+
+      cpu_model = get_model()
+      cpu_model.compile(optimizer, loss)
+
+      inputs = np.zeros((10, 3), dtype=np.float32)
+
+      # As sample size is 10, we batch by 4 so that the last batch is
+      # a partial batch. Also `fit()` using numpy array as inputs without
+      # distribution strategy uses entire sample as a single batch. As so,
+      # we remove parameters `batch_size` and `steps`.
+      cpu_model.set_weights(model_with_ds_strategy.get_weights())
+      self.assertAllClose(
+          model_with_ds_strategy.predict(inputs, batch_size=4, steps=3),
+          cpu_model.predict(inputs),
+          atol=1e-5, rtol=1e-5)
+
+  @combinations.generate(tpu_strategy_combinations())
+  def test_predict_multi_output_model_with_partial_batch(
+      self, distribution):
+    with self.cached_session():
+      optimizer = gradient_descent.GradientDescentOptimizer(0.001)
+      loss = 'mse'
+
+      with distribution.scope():
+        model_with_ds_strategy = simple_multi_inputs_multi_outputs_model()
+        model_with_ds_strategy.compile(optimizer, loss)
+
+      cpu_model = simple_multi_inputs_multi_outputs_model()
+      cpu_model.compile(optimizer, loss)
+
+      input_data, _ = get_multi_inputs_multi_outputs_data()
+      input_dict = {
+          'input_a': input_data['input_a'],
+          'input_b': input_data['input_b'],
+      }
+
+      # As sample size is 200, we batch by 18 so that the last batch is
+      # a partial batch. Also `fit()` using numpy array as inputs without
+      # distribution strategy uses entire sample as a single batch. As so,
+      # we remove parameters `batch_size` and `steps`.
+      cpu_model.set_weights(model_with_ds_strategy.get_weights())
+      self.assertAllClose(
+          model_with_ds_strategy.predict(input_dict, batch_size=18, steps=12),
+          cpu_model.predict(input_dict),
+          atol=1e-4, rtol=1e-4)
+
 
 class TestDistributionStrategyWithDatasets(test.TestCase,
                                            parameterized.TestCase):
@@ -961,6 +1028,64 @@ class TestDistributionStrategyWithDatasets(test.TestCase,
                 callbacks=[keras.callbacks.LearningRateScheduler(schedule)])
       self.assertAllClose(0.001, keras.backend.get_value(model.optimizer.lr))
 
+  @combinations.generate(tpu_strategy_combinations())
+  def test_predict_with_dataset_with_partial_batch(self, distribution):
+    with self.cached_session():
+      optimizer = gradient_descent.GradientDescentOptimizer(0.001)
+      loss = 'mse'
+
+      with distribution.scope():
+        model_with_ds_strategy = get_model()
+        model_with_ds_strategy.compile(optimizer, loss)
+
+      cpu_model = get_model()
+      cpu_model.compile(optimizer, loss)
+
+      inputs = np.zeros((10, 3), dtype=np.float32)
+      dataset = dataset_ops.Dataset.from_tensor_slices((inputs))
+
+      # As sample size is 10, we batch by 4 so that the last batch is
+      # a partial batch.
+      dataset_with_partial_batch = dataset.batch(4)
+      cpu_model.set_weights(model_with_ds_strategy.get_weights())
+
+      self.assertAllClose(
+          model_with_ds_strategy.predict(dataset_with_partial_batch, steps=3),
+          cpu_model.predict(dataset_with_partial_batch, steps=3),
+          atol=1e-5, rtol=1e-5)
+
+  @combinations.generate(tpu_strategy_combinations())
+  def test_predict_multi_output_model_with_dataset_with_partial_batch(
+      self, distribution):
+    with self.cached_session():
+      optimizer = gradient_descent.GradientDescentOptimizer(0.001)
+      loss = 'mse'
+
+      with distribution.scope():
+        model_with_ds_strategy = simple_multi_inputs_multi_outputs_model()
+        model_with_ds_strategy.compile(optimizer, loss)
+
+      cpu_model = simple_multi_inputs_multi_outputs_model()
+      cpu_model.compile(optimizer, loss)
+
+      input_data, _ = get_multi_inputs_multi_outputs_data()
+      input_dict = {
+          'input_a': input_data['input_a'],
+          'input_b': input_data['input_b'],
+      }
+
+      dataset = dataset_ops.Dataset.from_tensor_slices(input_dict)
+
+      # As sample size is 200, we batch by 18 using 12 steps per epoch so
+      # that the last batch is a partial batch.
+      dataset_with_partial_batch = dataset.batch(18)
+      cpu_model.set_weights(model_with_ds_strategy.get_weights())
+
+      self.assertAllClose(
+          model_with_ds_strategy.predict(dataset_with_partial_batch, steps=12),
+          cpu_model.predict(dataset_with_partial_batch, steps=12),
+          atol=1e-4, rtol=1e-4)
+
 
 class TestDistributionStrategyErrorCases(test.TestCase, parameterized.TestCase):
 
diff --git a/tensorflow/python/distribute/input_lib.py b/tensorflow/python/distribute/input_lib.py
index 43915917d6..0a389e19a8 100644
--- a/tensorflow/python/distribute/input_lib.py
+++ b/tensorflow/python/distribute/input_lib.py
@@ -321,31 +321,35 @@ class _SingleWorkerDatasetIterator(object):
     return self._iterator.output_types
 
 
-def _split_dataset_batch(dataset, split_batch_by):
-  """Divide a batch-ed dataset's batches into smaller batches."""
-  # TODO(sourabhbajaj): Remove this in lieu of distributed datasets
+# TODO(sourabhbajaj): Remove this in lieu of distributed datasets
+def _get_batched_dataset(d):
+  """Get the underlying batch dataset from the dataset object."""
   # pylint: disable=protected-access
-  def _get_batch_dataset(d):
-    """Get the underlying batch dataset from the dataset object."""
-    if isinstance(d, dataset_ops.DatasetV1Adapter):
-      d = d._dataset
-
-    if isinstance(d, (dataset_ops.BatchDataset, batching._MapAndBatchDataset)):
-      return d
-    elif isinstance(d, dataset_ops.PrefetchDataset):
-      return _get_batch_dataset(d._input_dataset)
-    raise ValueError(
-        "Unable to get batched dataset from the input dataset. `batch` "
-        "`map_and_batch` need to be the last operations on the dataset. "
-        "The batch operations can be followed by a prefetch.")
-
-  batched_dataset = _get_batch_dataset(dataset)
-  if isinstance(batched_dataset, dataset_ops.BatchDataset):
-    batch_size = batched_dataset._batch_size
-    drop_remainder = batched_dataset._drop_remainder
-  elif isinstance(batched_dataset, batching._MapAndBatchDataset):
-    batch_size = batched_dataset._batch_size_t
-    drop_remainder = batched_dataset._drop_remainder_t
+  if isinstance(d, dataset_ops.DatasetV1Adapter):
+    d = d._dataset
+
+  if isinstance(d, (dataset_ops.BatchDataset, batching._MapAndBatchDataset)):
+    return d
+  elif isinstance(d, dataset_ops.PrefetchDataset):
+    return _get_batched_dataset(d._input_dataset)
+
+  raise ValueError(
+      "Unable to get batched dataset from the input dataset. `batch` "
+      "`map_and_batch` need to be the last operations on the dataset. "
+      "The batch operations can be followed by a prefetch.")
+
+
+def _get_batched_dataset_attributes(dataset):
+  """Get `batch_size`, `drop_remainder`, and `prefetch_buffer` of dataset."""
+  # pylint: disable=protected-access
+  assert isinstance(dataset,
+                    (dataset_ops.BatchDataset, batching._MapAndBatchDataset))
+  if isinstance(dataset, dataset_ops.BatchDataset):
+    batch_size = dataset._batch_size
+    drop_remainder = dataset._drop_remainder
+  elif isinstance(dataset, batching._MapAndBatchDataset):
+    batch_size = dataset._batch_size_t
+    drop_remainder = dataset._drop_remainder_t
 
   prefetch_buffer = None
   if isinstance(dataset, dataset_ops.PrefetchDataset):
@@ -361,6 +365,15 @@ def _split_dataset_batch(dataset, split_batch_by):
   if tensor_util.is_tensor(drop_remainder):
     drop_remainder = tensor_util.constant_value(drop_remainder)
 
+  return batch_size, drop_remainder, prefetch_buffer
+
+
+def _split_dataset_batch(dataset, split_batch_by):
+  """Divide a batch-ed dataset's batches into smaller batches."""
+  batched_dataset = _get_batched_dataset(dataset)
+  batch_size, drop_remainder, prefetch_buffer = (
+      _get_batched_dataset_attributes(batched_dataset))
+
   if batch_size % split_batch_by:
     raise ValueError(
         "Batch size %s cannot be sharded evenly across replicas %s" % (
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index c353590415..108187aafe 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -134,6 +134,7 @@ py_library(
         "engine/input_layer.py",
         "engine/input_spec.py",
         "engine/network.py",
+        "engine/partial_batch_padding_handler.py",
         "engine/saving.py",
         "engine/sequential.py",
         "engine/training.py",
@@ -159,6 +160,8 @@ py_library(
         ":regularizers",
         ":saving",
         "//tensorflow/python/data",
+        "//tensorflow/python/distribute:distribute_lib",
+        "//tensorflow/python/distribute:input_lib",
         "//tensorflow/python/distribute:reduce_util",
         "//tensorflow/python/training/checkpointable:data_structures",
         "//tensorflow/tools/docs:doc_controls",
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index 224231278b..70de386245 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -379,7 +379,7 @@ def configure_and_create_session(distribution_strategy):
   K.set_session(session)
 
 
-def validate_inputs(x, y, distribution_strategy):
+def validate_inputs(x, y, distribution_strategy, allow_partial_batch=False):
   """Validate inputs when using DistributionStrategy.
 
   Args:
@@ -387,6 +387,8 @@ def validate_inputs(x, y, distribution_strategy):
     y: Model Targets.
     distribution_strategy: The DistributionStrategy with which the model is
       compiled.
+    allow_partial_batch: Boolean. If false, datasets must have fully
+      defined shapes.
 
   Raises:
     ValueError: if input is not a Dataset or a numpy array(when we use
@@ -400,18 +402,13 @@ def validate_inputs(x, y, distribution_strategy):
 
   if is_tpu_strategy(distribution_strategy):
     for i in [x, y]:
-      if isinstance(i, dataset_ops.DatasetV2):
-        shapes = nest.flatten(i.output_shapes)
-        try:
-          s = next(s for s in shapes if not s.is_fully_defined())
-        except StopIteration:
-          continue
-        else:
+      if (isinstance(i, dataset_ops.DatasetV2) and not allow_partial_batch):
+        if not is_dataset_shape_fully_defined(i):
           raise ValueError(
               'Using TPUs currently requires fully defined shapes. Either use '
               'set_shape() on the input tensors or use '
               'dataset.batch(..., drop_remainder=True).'
-              'Found unknown shape {} in input {}.'.format(s, i))
+              'Found unknown shape in input {}.'.format(i))
 
 
 # TODO(b/118776054): Currently we support global batch size for TPUStrategy and
@@ -427,8 +424,15 @@ def is_tpu_strategy(strategy):
   return strategy is not None and strategy.__class__.__name__ == 'TPUStrategy'
 
 
+def is_dataset_shape_fully_defined(dataset):
+  """Returns whether a dataset contains a final partial batch."""
+  shapes = nest.flatten(dataset.output_shapes)
+  unknown_shapes = [s for s in shapes if not s.is_fully_defined()]
+  return not unknown_shapes
+
+
 def get_input_params(distribution_strategy, first_x_value, steps, batch_size,
-                     is_training=False):
+                     mode=None):
   """Calculate the number of batches and steps/steps_per_epoch.
 
   Args:
@@ -437,8 +441,10 @@ def get_input_params(distribution_strategy, first_x_value, steps, batch_size,
       model input.
     steps:  The specified number of steps.
     batch_size: The specified batch_size.
-    is_training: Boolean to relax the constraints on consuming all the training
-      samples to keep compatibility till we support partial batches.
+    mode: ModeKey representing whether input will be used for training,
+      evaluation, or prediction. This is used to relax the constraints on
+      consuming all the training samples to keep compatibility till we
+      support partial batches. If none, then partial batches are not allowed.
 
   Returns:
     steps: The steps or steps_per_epoch argument depending on if a user is
@@ -456,6 +462,14 @@ def get_input_params(distribution_strategy, first_x_value, steps, batch_size,
   use_per_replica_batch = not global_batch_size_supported(
       distribution_strategy)
 
+  # Partial batches are allowed for training as we repeat the
+  # dataset when converting numpy arrays into a dataset.
+  # For other modes uneven batch sizes are not allowed except
+  # for `predict()` on TPUStrategy.
+  allow_partial_batch = (mode == ModeKeys.TRAIN or
+                         (mode == ModeKeys.PREDICT
+                          and is_tpu_strategy(distribution_strategy)))
+
   if steps is None:
     if batch_size is None:
       # If neither the batch size or number of steps are set. We choose the
@@ -468,7 +482,7 @@ def get_input_params(distribution_strategy, first_x_value, steps, batch_size,
       global_batch_size = batch_size
       if use_per_replica_batch:
         global_batch_size *= distribution_strategy.num_replicas_in_sync
-    if not is_training and num_samples % global_batch_size:
+    if not allow_partial_batch and num_samples % global_batch_size:
       raise ValueError('The number of samples %s is not divisible by '
                        'batch size %s.' % (num_samples, global_batch_size))
     steps = num_samples // global_batch_size
@@ -488,7 +502,11 @@ def get_input_params(distribution_strategy, first_x_value, steps, batch_size,
       if use_per_replica_batch:
         global_batch_size *= distribution_strategy.num_replicas_in_sync
 
-      if num_samples < (global_batch_size * steps):
+      min_num_samples = global_batch_size * steps
+      if allow_partial_batch:
+        min_num_samples = global_batch_size * (steps-1) + 1 if steps > 1 else 0
+
+      if num_samples < min_num_samples:
         raise ValueError('Number of samples %s is less than samples required '
                          'for specified batch_size %s and steps %s' % (
                              num_samples, global_batch_size, steps))
diff --git a/tensorflow/python/keras/engine/partial_batch_padding_handler.py b/tensorflow/python/keras/engine/partial_batch_padding_handler.py
new file mode 100644
index 0000000000..c3951ead6e
--- /dev/null
+++ b/tensorflow/python/keras/engine/partial_batch_padding_handler.py
@@ -0,0 +1,111 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Utility object to handler partial batches for TPUStrategy."""
+# pylint: disable=protected-access
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+import six
+
+from tensorflow.python.framework import tensor_util
+from tensorflow.python.keras import backend as K
+from tensorflow.python.ops import array_ops
+from tensorflow.python.util import nest
+
+
+class PartialBatchPaddingHandler(object):
+  """A container that holds info about partial batches for `predict()`."""
+
+  def __init__(self, output_shape):
+    self.padded_batch_size = 0
+    self.padding_mask = array_ops.zeros(0)
+    self.output_shape = output_shape
+
+  def get_real_batch_size(self, dataset_batch):
+    """Returns the number of elements in a potentially partial batch."""
+    if isinstance(dataset_batch, (tuple, list)):
+      dataset_batch = dataset_batch[0]
+
+    assert nest.flatten(dataset_batch)
+
+    def _find_any_tensor(batch_features):
+      tensors = [
+          x for x in nest.flatten(batch_features) if tensor_util.is_tensor(x)
+      ]
+      if not tensors:
+        raise ValueError('Cannot find any Tensor in features dict.')
+      return tensors[0]
+
+    return K.cast(K.shape(_find_any_tensor(dataset_batch))[0],
+                  dtype='int64')
+
+  def update_mask(self, padding_mask, dataset_batch):
+    """Calculate and cache the amount of padding required for a batch."""
+    original_batch_size = self.get_real_batch_size(dataset_batch)
+    missing_count = self.padded_batch_size - original_batch_size
+    mask = K.concatenate([array_ops.ones(original_batch_size),
+                          array_ops.zeros(missing_count)], axis=0)
+    return K.concatenate([padding_mask, mask], axis=0)
+
+  def pad_batch(self, *dataset_batch_elements):
+    """Pads out the batch dimension of a tensor to the complete batch size."""
+    def _pad(batch):
+      """Helper function to pad nested data within each batch elements."""
+      padded_dict_batch = {}
+      if isinstance(batch, dict):
+        for key, value in six.iteritems(batch):
+          padded_dict_batch[key] = _pad(value)
+        return padded_dict_batch
+
+      rank = len(batch.shape)
+      assert rank > 0
+      missing_count = (self.padded_batch_size -
+                       self.get_real_batch_size(batch))
+      padding = K.stack([[0, missing_count]] + [[0, 0]] * (rank - 1))
+      return array_ops.pad(batch, padding, 'constant')
+
+    if len(dataset_batch_elements) == 1:
+      return _pad(dataset_batch_elements[0])
+
+    batch_elements = []
+    for batch_element in dataset_batch_elements:
+      batch_elements.append(_pad(batch_element))
+    return tuple(batch_elements)
+
+  def apply_mask(self, prediction_result):
+    """Removes prediction output that corresponds to padded input."""
+    padding_mask = K.get_value(self.padding_mask)
+    assert len(padding_mask.shape) == 1
+
+    if len(self.output_shape) == 1:
+      prediction = np.take(prediction_result,
+                           np.nonzero(
+                               padding_mask[:len(prediction_result)]),
+                           axis=0)
+      if prediction.shape[0] == 1:
+        prediction = np.squeeze(prediction, axis=0)
+      return prediction
+
+    else:
+      predictions = []
+      for i in range(len(self.output_shape)):
+        prediction = prediction_result[i]
+        prediction = np.take(prediction, np.nonzero(
+            padding_mask[:len(prediction)]), axis=0)
+        predictions.append(np.squeeze(prediction))
+
+      return predictions
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 6ab9935d43..53c6a8896a 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -2136,7 +2136,9 @@ class Model(Network):
                                           steps_name='steps',
                                           steps=None,
                                           validation_split=0,
-                                          shuffle=False):
+                                          shuffle=False,
+                                          repeat=True,
+                                          allow_partial_batch=False):
     """Runs validation checks on input and target data passed by the user.
 
     This is called when using DistributionStrategy to train, evaluate or serve
@@ -2160,6 +2162,10 @@ class Model(Network):
       validation_split: Float between 0 and 1.
         Fraction of the training data to be used as validation data.
       shuffle: Boolean whether to shuffle the training data before each epoch.
+      repeat: Boolean whether to repeat the numpy training data when converting
+        to training dataset.
+      allow_partial_batch: Boolean whether to enforce that all batches have the
+        same size.
 
     Returns:
       Dataset instance.
@@ -2239,10 +2245,13 @@ class Model(Network):
                                                                session=session)
         if shuffle_buffer:
           ds = ds.shuffle(shuffle_buffer)
-        ds = ds.repeat()
+        if repeat:
+          ds = ds.repeat()
+
         # We need to use the drop_remainder argument to get a known static
         # input shape which is required for TPUs.
-        drop_remainder = strategy.extended.experimental_require_static_shapes
+        drop_remainder = (not allow_partial_batch and
+                          strategy.extended.experimental_require_static_shapes)
         x = ds.batch(batch_size, drop_remainder=drop_remainder)
       else:
         assert isinstance(x, dataset_ops.DatasetV2)
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 57b665a3da..c5ee2bf0ff 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -21,7 +21,9 @@ from __future__ import print_function
 
 import numpy as np
 
+from tensorflow.python.data.experimental.ops import batching
 from tensorflow.python.distribute import distribute_lib
+from tensorflow.python.distribute import input_lib
 from tensorflow.python.distribute import reduce_util as ds_reduce_util
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import errors
@@ -29,6 +31,7 @@ from tensorflow.python.framework import tensor_shape
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras import callbacks as cbks
 from tensorflow.python.keras.engine import distributed_training_utils
+from tensorflow.python.keras.engine import partial_batch_padding_handler as padding_util
 from tensorflow.python.keras.engine import training_arrays
 from tensorflow.python.keras.engine import training_utils
 from tensorflow.python.keras.utils.generic_utils import Progbar
@@ -61,10 +64,13 @@ def fit_distributed(model,
 
   first_x_value = nest.flatten(x)[0]
   if isinstance(first_x_value, np.ndarray):
+    # Until support for partial batch is implemented across all
+    # functions and distribution strategy, we pass `mode` to selectively
+    # relax the costraint to consume all the training samples.
     steps_per_epoch, batch_size = (
         distributed_training_utils.get_input_params(
             model._distribution_strategy, first_x_value, steps_per_epoch,
-            batch_size, is_training=True))
+            batch_size, mode=ModeKeys.TRAIN))
   batch_size = model._validate_or_infer_batch_size(
       batch_size, steps_per_epoch, x)
   dataset = model._distribution_standardize_user_data(
@@ -176,18 +182,21 @@ def predict_distributed(model,
                         callbacks=None):
   """Predict loop for Distribution Strategies."""
   distributed_training_utils.validate_inputs(
-      x, None, model._distribution_strategy)
+      x, None, model._distribution_strategy, allow_partial_batch=True)
   first_x_value = nest.flatten(x)[0]
   if isinstance(first_x_value, np.ndarray):
     steps, batch_size = distributed_training_utils.get_input_params(
-        model._distribution_strategy, first_x_value, steps, batch_size)
+        model._distribution_strategy, first_x_value, steps,
+        batch_size, mode=ModeKeys.PREDICT)
   batch_size = model._validate_or_infer_batch_size(batch_size, steps, x)
   dataset = model._distribution_standardize_user_data(
       x,
       batch_size=batch_size,
       check_steps=True,
       steps_name='steps',
-      steps=steps)
+      steps=steps,
+      repeat=False,
+      allow_partial_batch=True)
   if distributed_training_utils.is_tpu_strategy(model._distribution_strategy):
     # TODO(fchollet): why aren't callbacks supported here?
     return experimental_tpu_predict_loop(
@@ -537,6 +546,32 @@ def experimental_tpu_predict_loop(model, dataset, verbose=0, steps=None):
       or list of arrays of predictions
       (if the model has multiple outputs).
   """
+  dataset_fully_shaped = (distributed_training_utils.
+                          is_dataset_shape_fully_defined(dataset))
+  padding_handler = None
+  if not dataset_fully_shaped:
+    # TODO(hongjunchoi): Investigate whether operations from
+    # PartialBatchPaddingHandler are unnecessarily pruned out
+    # during graph optimization.
+    padding_handler = padding_util.PartialBatchPaddingHandler(
+        model._feed_output_shapes)
+    batched_dataset = input_lib._get_batched_dataset(dataset)
+    batch_size, _, prefetch_buffer = input_lib._get_batched_dataset_attributes(
+        batched_dataset)
+    padding_handler.padded_batch_size = batch_size
+    padding_handler.padding_mask = dataset.reduce(padding_handler.padding_mask,
+                                                  padding_handler.update_mask)
+
+    dataset = dataset.map(padding_handler.pad_batch)
+    dataset = dataset.apply(batching.unbatch())
+    # Upon this point, it is guaranteed that the dataset does not
+    # have partial batches. Thus, we set `drop_remainder=True` to
+    # get static shape information about the elements in the dataset.
+    dataset = dataset.batch(batch_size, drop_remainder=True)
+
+    if prefetch_buffer is not None:
+      dataset = dataset.prefetch(prefetch_buffer)
+
   current_strategy = model._distribution_strategy
   iterator = distributed_training_utils.get_iterator(dataset, current_strategy)
 
@@ -623,8 +658,14 @@ def experimental_tpu_predict_loop(model, dataset, verbose=0, steps=None):
   scope.__exit__(None, None, None)
 
   if len(unconcatenated_outs) == 1:
-    return np.concatenate(unconcatenated_outs[0], axis=0)
-  return [
-      np.concatenate(unconcatenated_outs[i], axis=0)
-      for i in range(len(unconcatenated_outs))
-  ]
+    prediction_result = np.concatenate(unconcatenated_outs[0], axis=0)
+  else:
+    prediction_result = [
+        np.concatenate(unconcatenated_outs[i], axis=0)
+        for i in range(len(unconcatenated_outs))
+    ]
+
+  if padding_handler:
+    prediction_result = padding_handler.apply_mask(prediction_result)
+
+  return prediction_result
-- 
GitLab


From fc4dbb6ab580776cfe0c5e88374c425f49728621 Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Mon, 28 Jan 2019 21:25:53 -0800
Subject: [PATCH 1460/2345] [XLA:GPU] Fix a bug related to GPU constant buffer
 name collision.

Having a hyphen or a dot in the name of an LLVM global variable for a GPU
constant buffer can crash the LLVM PTX backend. This happens when a constant
buffer is created for an HLO constant instruction that contains such
characters in its name. Previously, we sanitize the names of the HLO constant
instruction by replacing such characters with underscores during LLVM IR
emission. However, this approach has a problem in handling name collision due
to name sanitization. In particular, when the same sanitized name is used to
create global variables for the constant buffers of two different HLO constant
instructions, LLVM resolves the name conflict by appending a unique number to
the sanitized name. Later on, during LLVM IR emission or GPU kernel execution,
we simplify use the sanitized name of an HLO constant instruction to look up
the global buffer for the constant and find a wrong buffer. This change adds
an HLO pass to sanitize the HLO constant instruction names before LLVM IR
emission.

Relocate gpu_buffer_assignment_test.cc to gpu_sanitize_constant_names_test.cc
and modify the tests for the purpose of testing the new pass that sanitizes HLO
constant instruction names.

PiperOrigin-RevId: 231342192
---
 tensorflow/compiler/xla/service/gpu/BUILD     | 39 ++++++++
 .../gpu/gpu_sanitize_constant_names.cc        | 70 +++++++++++++++
 .../service/gpu/gpu_sanitize_constant_names.h | 38 ++++++++
 .../gpu/gpu_sanitize_constant_names_test.cc   | 82 +++++++++++++++++
 .../xla/service/gpu/nvptx_compiler.cc         |  2 +
 .../compiler/xla/service/gpu/tests/BUILD      | 15 ----
 .../gpu/tests/gpu_buffer_assignment_test.cc   | 90 -------------------
 .../service/llvm_ir/buffer_assignment_util.cc | 21 +++--
 .../service/llvm_ir/buffer_assignment_util.h  |  4 +
 9 files changed, 250 insertions(+), 111 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/gpu/gpu_sanitize_constant_names.cc
 create mode 100644 tensorflow/compiler/xla/service/gpu/gpu_sanitize_constant_names.h
 create mode 100644 tensorflow/compiler/xla/service/gpu/gpu_sanitize_constant_names_test.cc
 delete mode 100644 tensorflow/compiler/xla/service/gpu/tests/gpu_buffer_assignment_test.cc

diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD
index dc17aa4426..85fb2dd47a 100644
--- a/tensorflow/compiler/xla/service/gpu/BUILD
+++ b/tensorflow/compiler/xla/service/gpu/BUILD
@@ -555,6 +555,44 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "gpu_sanitize_constant_names",
+    srcs = ["gpu_sanitize_constant_names.cc"],
+    hdrs = ["gpu_sanitize_constant_names.h"],
+    deps = [
+        "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/compiler/xla/service:hlo_pass",
+        "//tensorflow/compiler/xla/service/llvm_ir:buffer_assignment_util",
+        "//tensorflow/core:lib",
+    ],
+)
+
+tf_cc_test(
+    name = "gpu_sanitize_constant_names_test",
+    srcs = ["gpu_sanitize_constant_names_test.cc"],
+    tags = tf_cuda_tests_tags(),
+    deps = [
+        ":gpu_sanitize_constant_names",
+        ":ir_emission_utils",
+        "//tensorflow/compiler/xla:shape_layout",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:status_macros",
+        "//tensorflow/compiler/xla:test_helpers",
+        "//tensorflow/compiler/xla:util",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/service:computation_layout",
+        "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/compiler/xla/service:hlo_matchers",
+        "//tensorflow/compiler/xla/service:hlo_module_config",
+        "//tensorflow/compiler/xla/service:hlo_parser",
+        "//tensorflow/compiler/xla/tests:hlo_test_base",
+        "//tensorflow/compiler/xla/tests:test_utils",
+        "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+        "//tensorflow/core:test",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
 cc_library(
     name = "fusion_merger",
     srcs = ["fusion_merger.cc"],
@@ -679,6 +717,7 @@ cc_library(
         ":gpu_hlo_schedule",
         ":gpu_hlo_support_checker",
         ":gpu_layout_assignment",
+        ":gpu_sanitize_constant_names",
         ":instruction_fusion",
         ":ir_emission_utils",
         ":ir_emitter",
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_sanitize_constant_names.cc b/tensorflow/compiler/xla/service/gpu/gpu_sanitize_constant_names.cc
new file mode 100644
index 0000000000..5e38ceca18
--- /dev/null
+++ b/tensorflow/compiler/xla/service/gpu/gpu_sanitize_constant_names.cc
@@ -0,0 +1,70 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/gpu/gpu_sanitize_constant_names.h"
+
+#include <memory>
+#include <set>
+#include <vector>
+
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_opcode.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/buffer_assignment_util.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/platform/logging.h"
+
+namespace xla {
+
+namespace gpu {
+
+StatusOr<bool> GpuSanitizeConstantNames::Run(HloModule* module) {
+  bool changed = false;
+
+  NameUniquer instr_name_uniquer(/*separator=*/"_");
+  // Collect the names used for the non-constant HLO instructions.+
+  for (HloComputation* computation : module->computations()) {
+    for (HloInstruction* instr : computation->instructions()) {
+      if (instr->opcode() == HloOpcode::kConstant) {
+        continue;
+      }
+
+      const string& old_name = instr->name();
+      instr->UniquifyName(&instr_name_uniquer);
+      CHECK_EQ(old_name, instr->name());
+    }
+  }
+
+  // Sanitize the names for the constant HLO instructions and make them unique.
+  // This is not merged into the above loop because we don't want this pass to
+  // change the names of non-constant instructions, that is, if a constant HLO
+  // conflicts with a non-constant HLO, we change the name of the constant HLO
+  // even though the non-constant HLO comes after in the HLO module.
+  for (HloComputation* computation : module->computations()) {
+    for (HloInstruction* instr : computation->instructions()) {
+      if (instr->opcode() != HloOpcode::kConstant) {
+        continue;
+      }
+      string sanitized_name = llvm_ir::SanitizeConstantName(*instr);
+      instr->SetAndSanitizeName(sanitized_name);
+      instr->UniquifyName(&instr_name_uniquer);
+      changed = true;
+    }
+  }
+
+  return changed;
+}  // namespace gpu
+
+}  // namespace gpu
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_sanitize_constant_names.h b/tensorflow/compiler/xla/service/gpu/gpu_sanitize_constant_names.h
new file mode 100644
index 0000000000..8d583d047e
--- /dev/null
+++ b/tensorflow/compiler/xla/service/gpu/gpu_sanitize_constant_names.h
@@ -0,0 +1,38 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_GPU_GPU_SANITIZE_CONSTANT_NAMES_H_
+#define TENSORFLOW_COMPILER_XLA_SERVICE_GPU_GPU_SANITIZE_CONSTANT_NAMES_H_
+
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/service/hlo_pass_interface.h"
+
+namespace xla {
+namespace gpu {
+
+// Sanitizes HLO instruction names for the GPU backend. Currently, it only
+// replaces . and - in the HLO constant instruction names with _ to please the
+// LLVM PTX backend.
+class GpuSanitizeConstantNames : public HloModulePass {
+ public:
+  absl::string_view name() const override { return "sanitize-constant-names"; }
+
+  StatusOr<bool> Run(HloModule* module) override;
+};
+
+}  // namespace gpu
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_GPU_GPU_SANITIZE_CONSTANT_NAMES_H_
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_sanitize_constant_names_test.cc b/tensorflow/compiler/xla/service/gpu/gpu_sanitize_constant_names_test.cc
new file mode 100644
index 0000000000..f5adee8cc6
--- /dev/null
+++ b/tensorflow/compiler/xla/service/gpu/gpu_sanitize_constant_names_test.cc
@@ -0,0 +1,82 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <utility>
+
+#include "tensorflow/compiler/xla/service/gpu/gpu_sanitize_constant_names.h"
+
+#include "tensorflow/compiler/xla/service/hlo_matchers.h"
+#include "tensorflow/compiler/xla/service/hlo_module_config.h"
+#include "tensorflow/compiler/xla/service/hlo_parser.h"
+#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace xla {
+namespace gpu {
+namespace {
+
+namespace op = xla::testing::opcode_matchers;
+using SanitizeConstantNamesTest = HloTestBase;
+
+TEST_F(SanitizeConstantNamesTest, InstructionNameWithHyphenSanitized) {
+  const char *const kHloString = R"(
+    HloModule HyphenInInstructionName
+      ENTRY kernelEntry {
+        ROOT equal-to = s32[2]{0} constant({42, 73})
+    })";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseHloString(kHloString));
+
+  EXPECT_TRUE(GpuSanitizeConstantNames().Run(module.get()).ValueOrDie());
+  HloInstruction *root = module->entry_computation()->root_instruction();
+  EXPECT_EQ(root->name(), "equal_to");
+}
+
+TEST_F(SanitizeConstantNamesTest, InstructionNameWithDotSanitized) {
+  const char *const kHloString = R"(
+    HloModule HyphenInInstructionName
+      ENTRY kernelEntry {
+        ROOT equal.to = s32[2]{0} constant({42, 73})
+    })";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseHloString(kHloString));
+
+  EXPECT_TRUE(GpuSanitizeConstantNames().Run(module.get()).ValueOrDie());
+  HloInstruction *root = module->entry_computation()->root_instruction();
+  EXPECT_EQ(root->name(), "equal_to");
+}
+
+TEST_F(SanitizeConstantNamesTest, BufferSanitizedNameCollisionResolved) {
+  const char *const kHloString = R"(
+    HloModule BufferSanitizedName
+      ENTRY kernelEntry {
+      equal.to = s32[2]{0} constant({42, 73})
+      equal-to = s32[2]{0} constant({67, 3})
+      ROOT equal_to = s32[2]{0} add(equal.to, equal-to)
+    })";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseHloString(kHloString));
+
+  EXPECT_TRUE(GpuSanitizeConstantNames().Run(module.get()).ValueOrDie());
+  EXPECT_THAT(FindInstruction(module.get(), "equal_to_1"), op::Constant());
+  EXPECT_THAT(FindInstruction(module.get(), "equal_to_2"), op::Constant());
+}
+
+}  // namespace
+}  // namespace gpu
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
index 48f718b514..e1b91dfb1a 100644
--- a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
+++ b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
@@ -53,6 +53,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.h"
 #include "tensorflow/compiler/xla/service/gpu/gpu_hlo_support_checker.h"
 #include "tensorflow/compiler/xla/service/gpu/gpu_layout_assignment.h"
+#include "tensorflow/compiler/xla/service/gpu/gpu_sanitize_constant_names.h"
 #include "tensorflow/compiler/xla/service/gpu/instruction_fusion.h"
 #include "tensorflow/compiler/xla/service/gpu/ir_emission_utils.h"
 #include "tensorflow/compiler/xla/service/gpu/ir_emitter_context.h"
@@ -377,6 +378,7 @@ Status PrepareHloModuleForIrEmitting(HloModule* hlo_module) {
   pipeline.AddPass<HloDCE>();
   pipeline.AddPass<FlattenCallGraph>();
   pipeline.AddPass<GpuCopyInsertion>();
+  pipeline.AddPass<GpuSanitizeConstantNames>();
   return pipeline.Run(hlo_module).status();
 }
 
diff --git a/tensorflow/compiler/xla/service/gpu/tests/BUILD b/tensorflow/compiler/xla/service/gpu/tests/BUILD
index d8bd9f7f6d..d798b31643 100644
--- a/tensorflow/compiler/xla/service/gpu/tests/BUILD
+++ b/tensorflow/compiler/xla/service/gpu/tests/BUILD
@@ -47,21 +47,6 @@ cc_library(
     ],
 )
 
-tf_cc_test(
-    name = "gpu_buffer_assignment_test",
-    srcs = ["gpu_buffer_assignment_test.cc"],
-    tags = tf_cuda_tests_tags(),
-    deps = [
-        ":gpu_codegen_test",
-        "//tensorflow/compiler/xla/service:hlo",
-        "//tensorflow/compiler/xla/service:hlo_module_config",
-        "//tensorflow/compiler/xla/service:hlo_parser",
-        "//tensorflow/compiler/xla/tests:hlo_test_base",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-    ],
-)
-
 tf_cc_test(
     name = "gpu_copy_test",
     srcs = ["gpu_copy_test.cc"],
diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_buffer_assignment_test.cc b/tensorflow/compiler/xla/service/gpu/tests/gpu_buffer_assignment_test.cc
deleted file mode 100644
index 1335d73494..0000000000
--- a/tensorflow/compiler/xla/service/gpu/tests/gpu_buffer_assignment_test.cc
+++ /dev/null
@@ -1,90 +0,0 @@
-/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include <utility>
-
-#include "tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.h"
-#include "tensorflow/compiler/xla/service/hlo_module_config.h"
-#include "tensorflow/compiler/xla/service/hlo_parser.h"
-#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
-#include "tensorflow/core/platform/test.h"
-
-namespace xla {
-namespace gpu {
-namespace {
-
-class GpuBufferAssignmentTest : public GpuCodegenTest {
- public:
-  HloModuleConfig ConfigWithoutHloPasses() {
-    HloModuleConfig config;
-    auto debug_options = HloTestBase::GetDebugOptionsForTest();
-    // Disable layout_assignment to use the preassigned layouts.
-    debug_options.xla_disable_all_hlo_passes();
-    config.set_debug_options(debug_options);
-    return config;
-  }
-};
-
-TEST_F(GpuBufferAssignmentTest, InstructionNameWithHyphenSanitized) {
-  const char *const kHloString = R"(
-    HloModule HyphenInInstructionName
-      ENTRY kernelEntry {
-        ROOT equal-to = s32[2]{0} constant({42, 73})
-    })";
-
-  // Check that '-' in the instruction name is changed to '_'.
-  auto hlo_module = ParseHloString(kHloString).ValueOrDie();
-  CompileAndVerifyIr(std::move(hlo_module),
-                     R"(
-; CHECK: buffer_for_equal_to =
-)",
-                     /*match_optimized_ir=*/true);
-
-  // TODO(bixia): The run fails randomly.
-  // Check that the kernel runs correctly.
-  // EXPECT_TRUE(RunAndCompareNoHloPasses(kHloString, ErrorSpec{0.0}));
-}
-
-TEST_F(GpuBufferAssignmentTest, BufferSanitizedNameCollisionResolved) {
-  const char *const kHloString = R"(
-    HloModule BufferSanitizedName
-      ENTRY kernelEntry {
-      equal.to = s32[2]{0} constant({42, 73})
-      equal-to = s32[2]{0} constant({67, 3})
-      ROOT add = s32[2]{0} add(equal.to, equal-to)
-    })";
-
-  // Turn of Hlo passes to prevent constant folding.
-  //
-  // Check that '-' and '.' in the instruction name are changed to '_', and
-  // name collision is resolved by LLVM.
-  auto hlo_module =
-      ParseHloString(kHloString, ConfigWithoutHloPasses()).ValueOrDie();
-  CompileAndVerifyIr(std::move(hlo_module),
-                     R"(
-; CHECK: buffer_for_equal_to =
-; CHECK: buffer_for_equal_to1 =
-)",
-                     /*match_optimized_ir=*/false);
-
-  // TODO(bixia): There is another bug that prevents this from running
-  //              correctly.
-  // Check that the kernel runs correctly.
-  // EXPECT_TRUE(RunAndCompareNoHloPasses(kHloString, ErrorSpec{0.0}));
-}
-
-}  // namespace
-}  // namespace gpu
-}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/llvm_ir/buffer_assignment_util.cc b/tensorflow/compiler/xla/service/llvm_ir/buffer_assignment_util.cc
index c2c6405cda..1ea5a42b0b 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/buffer_assignment_util.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/buffer_assignment_util.cc
@@ -41,17 +41,26 @@ static const HloInstruction& InstrForConstantBufferAllocation(
   return *const_instr;
 }
 
-string ConstantBufferAllocationToGlobalName(
-    const BufferAllocation& allocation) {
-  string instr_name = InstrForConstantBufferAllocation(allocation).name();
+string SanitizeConstantName(const HloInstruction& instr) {
+  CHECK_EQ(instr.opcode(), HloOpcode::kConstant);
+  string instr_name = instr.name();
   for (char& c : instr_name) {
-    // Having a hyphen in a global variable name can crash the LLVM PTX backend.
-    // LLVM is able to generate unique global variable names using the string
-    // returned from here as name prefix.
+    // Having a hyphen or a dot in a global variable name can crash the LLVM PTX
+    // backend.
     if (c == '.' || c == '-') {
       c = '_';
     }
   }
+  return instr_name;
+}
+
+string ConstantBufferAllocationToGlobalName(
+    const BufferAllocation& allocation) {
+  const HloInstruction& instr = InstrForConstantBufferAllocation(allocation);
+  string instr_name = instr.name();
+  // Check that names are sanitized and stored in the HLO instructions
+  // before constant buffer allocation.
+  DCHECK_EQ(instr_name, SanitizeConstantName(instr));
   return absl::StrCat("buffer_for_", instr_name);
 }
 
diff --git a/tensorflow/compiler/xla/service/llvm_ir/buffer_assignment_util.h b/tensorflow/compiler/xla/service/llvm_ir/buffer_assignment_util.h
index bfb6eecb87..03e98a6690 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/buffer_assignment_util.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/buffer_assignment_util.h
@@ -20,6 +20,10 @@ limitations under the License.
 
 namespace xla {
 namespace llvm_ir {
+// Sanitizes the HLO constant instruction name so that it can be used for the
+// name of the corresponding constant buffer. In particular, it replaces . and
+// - with _.
+string SanitizeConstantName(const HloInstruction& instr);
 // In XLA:GPU we map constant buffer allocations to globals in the generated
 // LLVM IR.  This function gives us the name of the global variable a constant
 // buffer is mapped to.  Not used on XLA:CPU.
-- 
GitLab


From 4be94e9e266fc363974d5945b8ca66d4977cfa98 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 28 Jan 2019 23:26:19 -0800
Subject: [PATCH 1461/2345] internal changes

PiperOrigin-RevId: 231352178
---
 tensorflow/lite/experimental/examples/lstm/tflite_lstm.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/experimental/examples/lstm/tflite_lstm.py b/tensorflow/lite/experimental/examples/lstm/tflite_lstm.py
index 461345060b..0038e542a0 100644
--- a/tensorflow/lite/experimental/examples/lstm/tflite_lstm.py
+++ b/tensorflow/lite/experimental/examples/lstm/tflite_lstm.py
@@ -230,10 +230,10 @@ class TFLiteLSTMCell(rnn_cell_impl.LayerRNNCell):
     # f stands for forget, i stands for input and o stands for output.
     if self._use_peepholes:
       self._w_f_diag = add_variable_wrapped("w_f_diag", [self._num_units],
-                                            self._initializer, 9,
+                                            self._initializer, 10,
                                             maybe_partitioner)
       self._w_i_diag = add_variable_wrapped("w_i_diag", [self._num_units],
-                                            self._initializer, 10,
+                                            self._initializer, 9,
                                             maybe_partitioner)
       self._w_o_diag = add_variable_wrapped("w_o_diag", [self._num_units],
                                             self._initializer, 11,
-- 
GitLab


From 967a0feae32776687a6ba15df985ece6db80e7c6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 29 Jan 2019 01:02:26 -0800
Subject: [PATCH 1462/2345] compat: Update forward compatibility horizon to
 2019-01-29

PiperOrigin-RevId: 231362634
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 03147e3581..fa8b68078e 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 28)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 29)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 48d79ff437c73c2fb3bdfebcf41d37fe36cb3d63 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 29 Jan 2019 01:14:35 -0800
Subject: [PATCH 1463/2345] [TF:XLA] Make matrix_square_root test more
 permissive.

This test was for concurrency, but also made the assumption that floating point operations are deterministic which is not true with autotuning.

PiperOrigin-RevId: 231364445
---
 tensorflow/python/kernel_tests/matrix_square_root_op_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/matrix_square_root_op_test.py b/tensorflow/python/kernel_tests/matrix_square_root_op_test.py
index 3edb390c72..e09530b891 100644
--- a/tensorflow/python/kernel_tests/matrix_square_root_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_square_root_op_test.py
@@ -114,7 +114,7 @@ class SquareRootOpTest(test.TestCase):
       sqrt2 = gen_linalg_ops.matrix_square_root(square2)
       all_ops = [sqrt1, sqrt2]
       sqrt = self.evaluate(all_ops)
-      self.assertAllEqual(sqrt[0], sqrt[1])
+      self.assertAllClose(sqrt[0], sqrt[1])
 
 
 if __name__ == "__main__":
-- 
GitLab


From 823b694639a3f49b6adbf9e73a08c529d583878e Mon Sep 17 00:00:00 2001
From: James Keeling <jtkeeling@google.com>
Date: Tue, 29 Jan 2019 01:42:32 -0800
Subject: [PATCH 1464/2345] Add bounds checking when looking at the stack in TF
 Registry.

This was breaking integration with R, fixes #25262.

PiperOrigin-RevId: 231367381
---
 tensorflow/python/framework/registry.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/framework/registry.py b/tensorflow/python/framework/registry.py
index 4357c76bd6..53c68b0461 100644
--- a/tensorflow/python/framework/registry.py
+++ b/tensorflow/python/framework/registry.py
@@ -64,8 +64,12 @@ class Registry(object):
     # stack trace is [this_function, Register(), user_function,...]
     # so the user function is #2.
     stack = tf_stack.extract_stack()
-    user_function = stack[2]
-    location_tag = tf_stack.convert_stack([user_function])[0]
+    stack_index = min(2, len(stack)-1)
+    if stack_index >= 0:
+      user_function = stack[stack_index]
+      location_tag = tf_stack.convert_stack([user_function])[0]
+    else:
+      location_tag = "UNKNOWN"
     self._registry[name] = {_TYPE_TAG: candidate, _LOCATION_TAG: location_tag}
 
   def list(self):
-- 
GitLab


From 3bc0eda0ba5d9d7b52e497c216a354de69af81a5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 29 Jan 2019 03:45:16 -0800
Subject: [PATCH 1465/2345] Improve the performance of FastParseSingleExample
 for large float lists

The new version adds a fast path for little endian machines with 4 byte
floats where we can just do a memcpy from the serialized proto to the
result buffer instead of iterating over the data with a loop and doing
an endian repacking.

PiperOrigin-RevId: 231379145
---
 .../core/util/example_proto_fast_parsing.cc   | 46 +++++++++++++++++--
 1 file changed, 41 insertions(+), 5 deletions(-)

diff --git a/tensorflow/core/util/example_proto_fast_parsing.cc b/tensorflow/core/util/example_proto_fast_parsing.cc
index 3cc75bbd1f..8877cd0d3f 100644
--- a/tensorflow/core/util/example_proto_fast_parsing.cc
+++ b/tensorflow/core/util/example_proto_fast_parsing.cc
@@ -27,6 +27,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/threadpool.h"
 #include "tensorflow/core/lib/gtl/inlined_vector.h"
 #include "tensorflow/core/lib/monitoring/counter.h"
+#include "tensorflow/core/platform/byte_order.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/protobuf.h"
 #include "tensorflow/core/util/presized_cuckoo_map.h"
@@ -161,10 +162,30 @@ class Feature {
         if (!stream.ReadVarint32(&packed_length)) return false;
         auto packed_limit = stream.PushLimit(packed_length);
 
-        while (!stream.ExpectAtEnd()) {
-          uint32 buffer32;
-          if (!stream.ReadLittleEndian32(&buffer32)) return false;
-          float_list->push_back(absl::bit_cast<float>(buffer32));
+        // If the result data type is float and we are on a little endian
+        // machine then we can simply memcpy the data from the proto into the
+        // result vector.
+        constexpr int32 kNumFloatBytes = 4;
+        if (port::kLittleEndian &&
+            sizeof(typename Result::value_type) == kNumFloatBytes) {
+          // Store the initial size to know the offset we have to start writing
+          // data from before resizing the output "vector".
+          const size_t initial_size = float_list->size();
+          float_list->resize(initial_size + packed_length / kNumFloatBytes);
+          // Calculate the length of the buffer available what can be less than
+          // what we requested in resize in case of a LimitedArraySlice.
+          const uint32 bytes_to_copy =
+              std::min(static_cast<uint32>((float_list->size() - initial_size) *
+                                           kNumFloatBytes),
+                       packed_length);
+          if (!stream.ReadRaw(float_list->data() + initial_size, bytes_to_copy))
+            return false;
+        } else {
+          while (!stream.ExpectAtEnd()) {
+            uint32 buffer32;
+            if (!stream.ReadLittleEndian32(&buffer32)) return false;
+            float_list->push_back(absl::bit_cast<float>(buffer32));
+          }
         }
 
         stream.PopLimit(packed_limit);
@@ -448,8 +469,10 @@ struct SeededHasher {
 template <typename T>
 class LimitedArraySlice {
  public:
+  using value_type = T;
+
   LimitedArraySlice(T* begin, size_t num_elements)
-      : current_(begin), end_(begin + num_elements) {}
+      : current_(begin), begin_(begin), end_(begin + num_elements) {}
 
   // May return negative if there were push_back calls after slice was filled.
   int64 EndDistance() const { return end_ - current_; }
@@ -462,8 +485,21 @@ class LimitedArraySlice {
     ++current_;
   }
 
+  // Returns the number of elements in the slice.
+  size_t size() const { return std::min(current_ - begin_, end_ - begin_); }
+
+  // Attempts to resize the vector to the given size. It does so by advancing
+  // the pointer to the current element, possibly beyond the end of the slice.
+  // As a consequence, calling `size()` after `resize(x)` was called might
+  // return a value less than `x`.
+  void resize(size_t size) { current_ = begin_ + size; }
+
+  // Returns the pointer to the underlying data buffer.
+  T* data() { return begin_; }
+
  private:
   T* current_;
+  T* begin_;
   T* end_;
 };
 
-- 
GitLab


From 827d9b2c2abe25518ea7cf37aa54b5a39123f297 Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Tue, 29 Jan 2019 05:01:32 -0800
Subject: [PATCH 1466/2345] Remove duplicate function which is now natively in
 nest.

PiperOrigin-RevId: 231385811
---
 tensorflow/python/framework/func_graph.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/tensorflow/python/framework/func_graph.py b/tensorflow/python/framework/func_graph.py
index e7718cfef1..fa83a5f5c5 100644
--- a/tensorflow/python/framework/func_graph.py
+++ b/tensorflow/python/framework/func_graph.py
@@ -63,12 +63,6 @@ class UnknownArgument(object):
   pass
 
 
-# TODO(vbardiovsky): Remove this when nest is updated with new
-# flatten_with_tuple_paths.
-def flatten_with_tuple_paths(structure):
-  return list(zip(nest.yield_flat_paths(structure), nest.flatten(structure)))
-
-
 def convert_structure_to_signature(structure, arg_names=None):
   """Convert a potentially nested structure to a signature.
 
@@ -100,7 +94,7 @@ def convert_structure_to_signature(structure, arg_names=None):
 
   # We are using the flattened paths to name the TensorSpecs. We need an
   # explicit name for them downstream.
-  flattened = flatten_with_tuple_paths(structure)
+  flattened = nest.flatten_with_tuple_paths(structure)
   if arg_names:
     if len(arg_names) != len(structure):
       raise ValueError(
-- 
GitLab


From 512621ac80d66c00e6a4ae57fd3e8fcff86d3de9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 29 Jan 2019 06:16:29 -0800
Subject: [PATCH 1467/2345] [TF:XLA] Update TODO with a bug number for
 StridedSlice.

PiperOrigin-RevId: 231393227
---
 tensorflow/compiler/tf2xla/kernels/strided_slice_op.cc | 2 +-
 tensorflow/python/kernel_tests/array_ops_test.py       | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/strided_slice_op.cc b/tensorflow/compiler/tf2xla/kernels/strided_slice_op.cc
index 2273b59246..e8846fbe88 100644
--- a/tensorflow/compiler/tf2xla/kernels/strided_slice_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/strided_slice_op.cc
@@ -291,7 +291,7 @@ class StridedSliceAssignOp : public XlaOpKernel {
     absl::InlinedVector<xla::XlaOp, 4> slice_begin;
     absl::InlinedVector<int64, 4> slice_dims;
     for (int i = 0; i < begin.size(); ++i) {
-      // TODO(phawkins): implement strides != 1
+      // TODO(b/121179231): implement strides != 1
       OP_REQUIRES(
           ctx, strides[i] == 1 || strides[i] == -1,
           errors::Unimplemented("Strides != 1 or -1 are not yet implemented"));
diff --git a/tensorflow/python/kernel_tests/array_ops_test.py b/tensorflow/python/kernel_tests/array_ops_test.py
index 5ed549f268..0bc533a773 100644
--- a/tensorflow/python/kernel_tests/array_ops_test.py
+++ b/tensorflow/python/kernel_tests/array_ops_test.py
@@ -1050,12 +1050,12 @@ class SliceAssignTest(test_util.TensorFlowTestCase):
     checker2[None] = [6]  # new axis
 
   @test_util.run_deprecated_v1
-  @test_util.disable_xla("This test never passed for XLA")
+  @test_util.disable_xla("b/123559667")
   def testSliceAssign(self):
     self.doTestSliceAssign(use_resource=False)
 
   @test_util.run_deprecated_v1
-  @test_util.disable_xla("This test never passed for XLA")
+  @test_util.disable_xla("b/123559667")
   def testSliceAssignResource(self):
     self.doTestSliceAssign(use_resource=True)
 
-- 
GitLab


From c956c9d38f07eb96cb00a054e9c268b91c27cbff Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Tue, 29 Jan 2019 07:28:13 -0800
Subject: [PATCH 1468/2345] [tf.data] Add a meta benchmark that compares
 various ways of running tf.data benchmarks.

PiperOrigin-RevId: 231401043
---
 tensorflow/python/data/benchmarks/BUILD       |  11 ++
 .../python/data/benchmarks/meta_benchmark.py  | 151 ++++++++++++++++++
 2 files changed, 162 insertions(+)
 create mode 100644 tensorflow/python/data/benchmarks/meta_benchmark.py

diff --git a/tensorflow/python/data/benchmarks/BUILD b/tensorflow/python/data/benchmarks/BUILD
index fd0eca9dd7..739e5bab19 100644
--- a/tensorflow/python/data/benchmarks/BUILD
+++ b/tensorflow/python/data/benchmarks/BUILD
@@ -6,6 +6,17 @@ exports_files(["LICENSE"])
 
 load("//tensorflow:tensorflow.bzl", "py_test")
 
+py_test(
+    name = "meta_benchmark",
+    srcs = ["meta_benchmark.py"],
+    deps = [
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:session",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//third_party/py/numpy",
+    ],
+)
+
 py_test(
     name = "batch_benchmark",
     srcs = ["batch_benchmark.py"],
diff --git a/tensorflow/python/data/benchmarks/meta_benchmark.py b/tensorflow/python/data/benchmarks/meta_benchmark.py
new file mode 100644
index 0000000000..c6d888b2df
--- /dev/null
+++ b/tensorflow/python/data/benchmarks/meta_benchmark.py
@@ -0,0 +1,151 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test utilities for tf.data benchmarking functionality."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import timeit
+import numpy as np
+
+from tensorflow.python.client import session
+from tensorflow.python.data.experimental.ops import sleep
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.eager import context
+from tensorflow.python.platform import test
+
+
+class MetaBenchmark(test.Benchmark):
+  """Benchmark that compares various ways of running tf.data benchmarks."""
+
+  # Note that each of these benchmarks is a separate method so that we can
+  # run them independently and collect a performance profile.
+
+  def setup_fast_dataset(self):
+    self.num_reps = 15
+    self.iters = 100000
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    return dataset_ops.Dataset.range(10000**2).with_options(options)
+
+  def benchmarkFastDatasetWithOnlyCppIterations(self):
+    dataset = self.setup_fast_dataset()
+    self.run_benchmark_with_only_cpp_iterations(dataset)
+
+  def benchmarkFastDatasetWithSessionRun(self):
+    dataset = self.setup_fast_dataset()
+    self.run_benchmark_with_session_run(dataset)
+
+  def benchmarkFastDatasetWithSessionCallable(self):
+    dataset = self.setup_fast_dataset()
+    self.run_benchmark_with_session_run(dataset, make_callable=True)
+
+  def benchmarkFastDatasetInEager(self):
+    with context.eager_mode():
+      dataset = self.setup_fast_dataset()
+      self.run_benchmark_in_eager(dataset)
+
+  def setup_slow_dataset(self):
+    dataset = self.setup_fast_dataset()
+    self.iters = 1000
+    # sleep for 1e-3s per iteration
+    return dataset.apply(sleep.sleep(1000))
+
+  def benchmarkSlowDatasetWithOnlyCppIterations(self):
+    dataset = self.setup_slow_dataset()
+    self.run_benchmark_with_only_cpp_iterations(dataset)
+
+  def benchmarkSlowDatasetWithSessionRun(self):
+    dataset = self.setup_slow_dataset()
+    self.run_benchmark_with_session_run(dataset)
+
+  def benchmarkSlowDatasetWithSessionCallable(self):
+    dataset = self.setup_slow_dataset()
+    self.run_benchmark_with_session_run(dataset, make_callable=True)
+
+  def benchmarkSlowDatasetInEager(self):
+    with context.eager_mode():
+      dataset = self.setup_slow_dataset()
+      self.run_benchmark_in_eager(dataset)
+
+  def report(self, deltas):
+    # Each `delta` is the time taken for `self.iters` iterations. Divide by the
+    # number of iterations here to get per-element iteration time.
+    deltas = np.array(deltas) / self.iters
+    # Discard the first 5 results from "warming up" the session.
+    deltas = deltas[5:]
+
+    median = np.median(deltas)
+    mean = np.mean(deltas)
+    min_val = np.min(deltas)
+    max_val = np.max(deltas)
+    extras = {
+        "iters_per_second": 1 / median,
+        "median": median,
+        "mean": mean,
+        "min": min_val,
+        "max": max_val,
+        "num_reps": self.num_reps - 5,
+    }
+    self.report_benchmark(wall_time=median, iters=self.iters, extras=extras)
+
+  def run_benchmark_in_eager(self, dataset):
+    deltas = []
+    for _ in range(self.num_reps):
+      iterator = iter(dataset)
+      deltas.append(timeit.timeit(lambda: next(iterator), number=self.iters))  # pylint: disable=cell-var-from-loop
+
+    self.report(deltas)
+
+  def run_benchmark_with_session_run(self, dataset, make_callable=False):
+    iterator = dataset_ops.make_initializable_iterator(dataset)
+    next_element = iterator.get_next()
+
+    with session.Session() as sess:
+      deltas = []
+      for _ in range(self.num_reps):
+        if make_callable:
+          get_next_element = sess.make_callable(next_element)
+        else:
+          # Note: session.run(next_element.op) is more performant than
+          # session.run(next_element) because we avoid the cost of copying the
+          # tensor from C++ to python.
+          get_next_element = lambda: sess.run(next_element.op)
+
+        sess.run(iterator.initializer)
+        deltas.append(timeit.timeit(get_next_element, number=self.iters))
+    self.report(deltas)
+
+  def run_benchmark_with_only_cpp_iterations(self, dataset):
+    """Benchmarks the dataset with the iterations performed in C++."""
+    # NOTE: We use `dataset.skip()` to perform the iterations in C++, avoiding
+    # the overhead of multiple `session.run()` calls. Note that this relies on
+    # the underlying implementation of `skip`: if it is optimized in the future,
+    # we will have to change this code.
+    dataset = dataset.skip(self.iters - 1)
+    iterator = dataset_ops.make_initializable_iterator(dataset)
+    next_element = iterator.get_next()
+
+    with session.Session() as sess:
+      deltas = []
+      for _ in range(self.num_reps):
+        sess.run(iterator.initializer)
+        deltas.append(
+            timeit.timeit(lambda: sess.run(next_element.op), number=1))
+    self.report(deltas)
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From 470a757843314113a55bdfe3319f77f02bb3b77d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 29 Jan 2019 09:15:14 -0800
Subject: [PATCH 1469/2345] [TF:XLA] Correctly handle BiasAdd in tf2xla with
 varying space dimensions.

This is adding the functionality added for Tensorflow in #24413

PiperOrigin-RevId: 231414786
---
 tensorflow/compiler/tests/unary_ops_test.py    |  2 +-
 tensorflow/compiler/tf2xla/kernels/bias_ops.cc | 10 +++++++---
 tensorflow/python/kernel_tests/bias_op_test.py |  5 -----
 3 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/tensorflow/compiler/tests/unary_ops_test.py b/tensorflow/compiler/tests/unary_ops_test.py
index 3c2875ba47..083e2e58ae 100644
--- a/tensorflow/compiler/tests/unary_ops_test.py
+++ b/tensorflow/compiler/tests/unary_ops_test.py
@@ -760,7 +760,7 @@ class UnaryOpsTest(xla_test.XLATestCase):
         lambda x: gen_nn_ops.bias_add_grad(x, data_format="NCHW"),
         np.array(
             [[[1., 2.], [3., 4.]], [[5., 6.], [7., 8.]]], dtype=np.float32),
-        expected=np.array([10., 26.], dtype=np.float32))
+        expected=np.array([14., 22.], dtype=np.float32))
 
   def testCast(self):
     shapes = [[], [4], [2, 3], [2, 0, 4]]
diff --git a/tensorflow/compiler/tf2xla/kernels/bias_ops.cc b/tensorflow/compiler/tf2xla/kernels/bias_ops.cc
index e7f369b761..33bdf9aec3 100644
--- a/tensorflow/compiler/tf2xla/kernels/bias_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/bias_ops.cc
@@ -48,8 +48,11 @@ class BiasOp : public XlaOpKernel {
     OP_REQUIRES(ctx, TensorShapeUtils::IsVector(bias_shape),
                 errors::InvalidArgument("Biases must be 1D: ",
                                         bias_shape.DebugString()));
-    int feature_dim = (data_format_ == FORMAT_NHWC) ? input_shape.dims() - 1
-                                                    : input_shape.dims() - 3;
+
+    // feature_dim is the channel (C) dimension of the data.
+    int feature_dim = (data_format_ == FORMAT_NHWC)
+                          ? input_shape.dims() - 1
+                          : /*data_format == FORMAT_NCHW*/ 1;
     OP_REQUIRES(
         ctx, feature_dim >= 0,
         errors::InvalidArgument("Input tensor does not have enough dimensions "
@@ -91,9 +94,10 @@ class BiasAddGradOp : public XlaOpKernel {
                 errors::InvalidArgument("Input tensor must be at least 2D: ",
                                         out_backprop_shape.DebugString()));
 
+    // feature_dim is the channel (C) dimension of the data.
     int feature_dim = (data_format_ == FORMAT_NHWC)
                           ? out_backprop_shape.dims() - 1
-                          : out_backprop_shape.dims() - 3;
+                          : /*data_format == FORMAT_NCHW*/ 1;
     OP_REQUIRES(
         ctx, feature_dim >= 0,
         errors::InvalidArgument("Input tensor does not have enough dimensions "
diff --git a/tensorflow/python/kernel_tests/bias_op_test.py b/tensorflow/python/kernel_tests/bias_op_test.py
index e89e2400f4..94e20d9301 100644
--- a/tensorflow/python/kernel_tests/bias_op_test.py
+++ b/tensorflow/python/kernel_tests/bias_op_test.py
@@ -35,8 +35,6 @@ class BiasAddTest(test.TestCase):
 
   def _npBias(self, inputs, bias):
     assert len(bias.shape) == 1
-    print(inputs.shape)
-    print(bias.shape)
     assert inputs.shape[-1] == bias.shape[0]
     return inputs + bias.reshape(([1] * (len(inputs.shape) - 1)) +
                                  [bias.shape[0]])
@@ -206,7 +204,6 @@ class BiasAddTest(test.TestCase):
         self._testGradient(np_input, bias, dtype, data_format, use_gpu)
 
   @test_util.run_deprecated_v1
-  @test_util.disable_xla("This test never passed for XLA")
   def testGradientTensor3D(self):
     for (data_format, use_gpu) in [("NHWC", False), ("NHWC", True),
                                    ("NCHW", False), ("NCHW", True)]:
@@ -228,7 +225,6 @@ class BiasAddTest(test.TestCase):
         self._testGradient(np_input, bias, dtype, data_format, use_gpu)
 
   @test_util.run_deprecated_v1
-  @test_util.disable_xla("This test never passed for XLA")
   def testGradientTensor5D(self):
     for (data_format, use_gpu) in [("NHWC", False), ("NHWC", True),
                                    ("NCHW", False), ("NCHW", True)]:
@@ -246,7 +242,6 @@ class BiasAddTest(test.TestCase):
       self._testAll(np.random.randn(*shape), np.random.randn(shape[-1]))
 
   @test_util.run_deprecated_v1
-  @test_util.disable_xla("This test never passed for XLA")
   def testEmptyGradient(self):
     for (data_format, use_gpu) in ("NHWC", False), ("NHWC", True):
       for shape in (0, 0), (2, 0), (0, 2):
-- 
GitLab


From 9628503d8e7350fd1f3214429d8ecd3b07ffdcf2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 29 Jan 2019 09:15:29 -0800
Subject: [PATCH 1470/2345] Handle variables defined in a single conditional
 branch. They will still not work in TensorFlow, but this is sometimes valid
 in pure Python code. The error for TensorFlow or misbehaving Python code will
 now happen at runtime rather than compile-time which allows for well-behaved
 Python to execute as expected.

PiperOrigin-RevId: 231414805
---
 .../autograph/converters/control_flow.py      | 33 +++++++++++++------
 .../autograph/converters/control_flow_test.py | 11 -------
 tensorflow/python/autograph/operators/BUILD   | 11 +++++++
 .../python/autograph/operators/__init__.py    |  1 +
 .../autograph/operators/special_values.py     | 23 +++++++++++++
 .../operators/special_values_test.py          | 32 ++++++++++++++++++
 .../python/autograph/utils/ag_logging.py      |  2 +-
 7 files changed, 91 insertions(+), 22 deletions(-)
 create mode 100644 tensorflow/python/autograph/operators/special_values.py
 create mode 100644 tensorflow/python/autograph/operators/special_values_test.py

diff --git a/tensorflow/python/autograph/converters/control_flow.py b/tensorflow/python/autograph/converters/control_flow.py
index 4f38a2926b..0416820594 100644
--- a/tensorflow/python/autograph/converters/control_flow.py
+++ b/tensorflow/python/autograph/converters/control_flow.py
@@ -173,14 +173,12 @@ class ControlFlowTransformer(converter.Base):
         s for s in created_in_body if not s.is_composite())
     basic_created_in_orelse = tuple(
         s for s in created_in_orelse if not s.is_composite())
-    if basic_created_in_body != basic_created_in_orelse:
-      raise ValueError(
-          'if statement may not initialize all variables: the true branch'
-          ' creates %s, while the false branch creates %s. Make sure all'
-          ' these variables are initialized either in both'
-          ' branches or before the if statement.' %
-          (self._fmt_symbols(basic_created_in_body),
-           self._fmt_symbols(basic_created_in_orelse)))
+
+    # These variables are defined only in a single branch. This is fine in
+    # Python so we pass them through. Another backend, e.g. Tensorflow, may need
+    # to handle these cases specially or throw an Error.
+    possibly_undefined = (set(basic_created_in_body) ^
+                          set(basic_created_in_orelse))
 
     # Alias the closure variables inside the conditional functions, to allow
     # the functions access to the respective variables.
@@ -247,10 +245,25 @@ class ControlFlowTransformer(converter.Base):
         aliased_new_names=aliased_orelse_new_names,
         body=node_orelse,
         returns=returned_from_orelse)
+    undefined_assigns = self._create_undefined_assigns(possibly_undefined)
+
     cond_expr = self._create_cond_expr(cond_results, cond_var_name, body_name,
                                        orelse_name)
 
-    return cond_assign + body_def + orelse_def + cond_expr
+    return (undefined_assigns
+            + cond_assign
+            + body_def
+            + orelse_def
+            + cond_expr)
+
+  def _create_undefined_assigns(self, undefined_symbols):
+    assignments = []
+    for s in undefined_symbols:
+      template = '''
+        var = ag__.UNDEFINED
+      '''
+      assignments += templates.replace(template, var=s)
+    return assignments
 
   def _get_loop_state(self, node):
     body_scope = anno.getanno(node, annos.NodeAnno.BODY_SCOPE)
@@ -326,7 +339,7 @@ class ControlFlowTransformer(converter.Base):
     cond_scope = anno.getanno(node, annos.NodeAnno.COND_SCOPE)
     cond_closure = set()
     for s in cond_scope.read:
-      cond_closure.update(s.support_set)
+      cond_closure |= s.support_set
     cond_closure -= loop_state
 
     loop_state, state_ssf, state_ast_tuple, ssf_map = self._state_constructs(
diff --git a/tensorflow/python/autograph/converters/control_flow_test.py b/tensorflow/python/autograph/converters/control_flow_test.py
index 9009e67ef4..1e73dcafa1 100644
--- a/tensorflow/python/autograph/converters/control_flow_test.py
+++ b/tensorflow/python/autograph/converters/control_flow_test.py
@@ -175,17 +175,6 @@ class ControlFlowTest(converter_testing.TestCase):
     self.assertTransformedResult(test_fn, constant_op.constant(1), 1)
     self.assertTransformedResult(test_fn, constant_op.constant(-1), -1)
 
-  def test_if_imbalanced_outputs(self):
-
-    def test_fn(n):
-      if n > 0:
-        b = 4
-      return b
-
-    node, ctx = self.prepare(test_fn, {})
-    with self.assertRaises(ValueError):
-      control_flow.transform(node, ctx)
-
   @test_util.run_deprecated_v1
   def test_simple_for(self):
 
diff --git a/tensorflow/python/autograph/operators/BUILD b/tensorflow/python/autograph/operators/BUILD
index 21a66c86b7..07c860aa3f 100644
--- a/tensorflow/python/autograph/operators/BUILD
+++ b/tensorflow/python/autograph/operators/BUILD
@@ -26,6 +26,7 @@ py_library(
         "logical.py",
         "py_builtins.py",
         "slices.py",
+        "special_values.py",
     ],
     srcs_version = "PY2AND3",
     visibility = ["//tensorflow:__subpackages__"],
@@ -105,3 +106,13 @@ py_test(
         "//tensorflow/python:client_testlib",
     ],
 )
+
+py_test(
+    name = "special_values_test",
+    srcs = ["special_values_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":operators",
+        "//tensorflow/python:client_testlib",
+    ],
+)
diff --git a/tensorflow/python/autograph/operators/__init__.py b/tensorflow/python/autograph/operators/__init__.py
index 35f8028c29..58ed72b700 100644
--- a/tensorflow/python/autograph/operators/__init__.py
+++ b/tensorflow/python/autograph/operators/__init__.py
@@ -71,3 +71,4 @@ from tensorflow.python.autograph.operators.py_builtins import range_
 from tensorflow.python.autograph.operators.slices import get_item
 from tensorflow.python.autograph.operators.slices import GetItemOpts
 from tensorflow.python.autograph.operators.slices import set_item
+from tensorflow.python.autograph.operators.special_values import UNDEFINED
diff --git a/tensorflow/python/autograph/operators/special_values.py b/tensorflow/python/autograph/operators/special_values.py
new file mode 100644
index 0000000000..08a1013524
--- /dev/null
+++ b/tensorflow/python/autograph/operators/special_values.py
@@ -0,0 +1,23 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Utilities used to capture Python idioms."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+# Used to reify undefined Python symbols so they can be used during staging.
+UNDEFINED = object()
diff --git a/tensorflow/python/autograph/operators/special_values_test.py b/tensorflow/python/autograph/operators/special_values_test.py
new file mode 100644
index 0000000000..45fcf45067
--- /dev/null
+++ b/tensorflow/python/autograph/operators/special_values_test.py
@@ -0,0 +1,32 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for python_lang_utils module."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.autograph.operators import special_values
+from tensorflow.python.platform import test
+
+
+class PythonLangUtilsTest(test.TestCase):
+
+  def test_undefined(self):
+    self.assertIs(special_values.UNDEFINED, special_values.UNDEFINED)
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/python/autograph/utils/ag_logging.py b/tensorflow/python/autograph/utils/ag_logging.py
index cc51e341f4..847000aa25 100644
--- a/tensorflow/python/autograph/utils/ag_logging.py
+++ b/tensorflow/python/autograph/utils/ag_logging.py
@@ -132,4 +132,4 @@ def log(level, msg, *args, **kwargs):
 
 
 def warn_first_n(msg, *args, **kwargs):
-  logging.log_first_n(logging.WARNING, msg, *args, **kwargs)
+  logging.log_first_n(logging.WARN, msg, *args, **kwargs)
-- 
GitLab


From 20e867058e4318ef24b8549875657c0c1c93e787 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Tue, 29 Jan 2019 09:25:11 -0800
Subject: [PATCH 1471/2345] Add callbacks to training_distributed.

PiperOrigin-RevId: 231415978
---
 tensorflow/contrib/distribute/python/BUILD    |   2 +-
 .../contrib/distribute/python/keras_test.py   | 120 ++++++++++++++++++
 .../keras/engine/training_distributed.py      |  41 +++++-
 3 files changed, 155 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index c546b83859..1d7123b3e3 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -568,7 +568,7 @@ distribute_py_test(
     srcs = ["keras_test.py"],
     full_precision = True,
     main = "keras_test.py",
-    shard_count = 16,
+    shard_count = 32,
     tags = [
         "multi_and_single_gpu",
         "no_oss",  # TODO(b/117919883): Fix python error.
diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 4c900e27a4..2da71893dc 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -17,6 +17,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import collections
 import os
 import tempfile
 from absl.testing import parameterized
@@ -1087,6 +1088,125 @@ class TestDistributionStrategyWithDatasets(test.TestCase,
           atol=1e-4, rtol=1e-4)
 
 
+class Counter(keras.callbacks.Callback):
+  """Counts the number of times each callback method was run.
+
+  Attributes:
+    method_counts: dict. Contains the counts of time  each callback method was
+      run.
+  """
+
+  def __init__(self):
+    self.method_counts = collections.defaultdict(int)
+    methods_to_count = [
+        'on_batch_begin', 'on_batch_end', 'on_epoch_begin', 'on_epoch_end',
+        'on_predict_batch_begin', 'on_predict_batch_end', 'on_predict_begin',
+        'on_predict_end', 'on_test_batch_begin', 'on_test_batch_end',
+        'on_test_begin', 'on_test_end', 'on_train_batch_begin',
+        'on_train_batch_end', 'on_train_begin', 'on_train_end'
+    ]
+    for method_name in methods_to_count:
+      setattr(self, method_name,
+              self.wrap_with_counts(method_name, getattr(self, method_name)))
+
+  def wrap_with_counts(self, method_name, method):
+
+    def _call_and_count(*args, **kwargs):
+      self.method_counts[method_name] += 1
+      return method(*args, **kwargs)
+
+    return _call_and_count
+
+
+class TestDistributionStrategyWithCallbacks(test.TestCase,
+                                            parameterized.TestCase):
+
+  def _check_counts(self, counter, expected_counts):
+    """Checks that the counts registered by `counter` are those expected."""
+    for method_name, expected_count in expected_counts.items():
+      self.assertEqual(
+          counter.method_counts[method_name],
+          expected_count,
+          msg='For method {}: expected {}, got: {}'.format(
+              method_name, expected_count, counter.method_counts[method_name]))
+
+  @combinations.generate(strategy_minus_tpu_combinations())
+  def test_callbacks_in_fit(self, distribution):
+    with distribution.scope():
+      model = get_model()
+      model.compile(optimizer='sgd', loss='mse', metrics=['mae'])
+
+    dataset = get_dataset(distribution)
+    counter = Counter()
+
+    model.fit(
+        dataset,
+        epochs=2,
+        steps_per_epoch=5,
+        verbose=0,
+        validation_data=dataset,
+        validation_steps=2,
+        callbacks=[counter])
+
+    self._check_counts(
+        counter, {
+            'on_batch_begin': 10,
+            'on_batch_end': 10,
+            'on_epoch_begin': 2,
+            'on_epoch_end': 2,
+            'on_predict_batch_begin': 0,
+            'on_predict_batch_end': 0,
+            'on_predict_begin': 0,
+            'on_predict_end': 0,
+            'on_test_batch_begin': 4,
+            'on_test_batch_end': 4,
+            'on_test_begin': 2,
+            'on_test_end': 2,
+            'on_train_batch_begin': 10,
+            'on_train_batch_end': 10,
+            'on_train_begin': 1,
+            'on_train_end': 1
+        })
+
+  @combinations.generate(strategy_minus_tpu_combinations())
+  def test_callbacks_in_eval(self, distribution):
+    with distribution.scope():
+      model = get_model()
+      model.compile(optimizer='sgd', loss='mse', metrics=['mae'])
+
+    dataset = get_dataset(distribution)
+    counter = Counter()
+
+    model.evaluate(dataset, steps=5, callbacks=[counter])
+
+    self._check_counts(
+        counter, {
+            'on_test_batch_begin': 5,
+            'on_test_batch_end': 5,
+            'on_test_begin': 1,
+            'on_test_end': 1
+        })
+
+  @combinations.generate(strategy_minus_tpu_combinations())
+  def test_callbacks_in_predict(self, distribution):
+    with distribution.scope():
+      model = get_model()
+      model.compile(optimizer='sgd', loss='mse', metrics=['mae'])
+
+    dataset = get_dataset(distribution)
+    counter = Counter()
+
+    model.predict(get_predict_dataset(dataset), steps=5, callbacks=[counter])
+
+    self._check_counts(
+        counter, {
+            'on_predict_batch_begin': 5,
+            'on_predict_batch_end': 5,
+            'on_predict_begin': 1,
+            'on_predict_end': 1
+        })
+
+
 class TestDistributionStrategyErrorCases(test.TestCase, parameterized.TestCase):
 
   @combinations.generate(combinations.combine(
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index c5ee2bf0ff..68160580e6 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -161,9 +161,8 @@ def evaluate_distributed(model,
       steps=steps)
 
   if distributed_training_utils.is_tpu_strategy(model._distribution_strategy):
-    # TODO(fchollet): why aren't callbacks supported here?
     return experimental_tpu_test_loop(
-        model, dataset, verbose=verbose, steps=steps)
+        model, dataset, verbose=verbose, steps=steps, callbacks=callbacks)
   else:
     return training_arrays.test_loop(
         model,
@@ -198,9 +197,8 @@ def predict_distributed(model,
       repeat=False,
       allow_partial_batch=True)
   if distributed_training_utils.is_tpu_strategy(model._distribution_strategy):
-    # TODO(fchollet): why aren't callbacks supported here?
     return experimental_tpu_predict_loop(
-        model, dataset, verbose=verbose, steps=steps)
+        model, dataset, verbose=verbose, steps=steps, callbacks=callbacks)
   else:
     return training_arrays.predict_loop(
         model,
@@ -340,7 +338,9 @@ def experimental_tpu_fit_loop(model,
       do_validation=do_validation,
       epochs=epochs,
       steps_per_epoch=steps_per_epoch,
-      verbose=verbose)
+      verbose=verbose,
+      count_mode='steps',
+      mode=ModeKeys.TRAIN)
 
   # Calculate the steps each time on the device.
   steps_to_run = [current_strategy.extended.steps_per_run] * (
@@ -415,7 +415,8 @@ def experimental_tpu_fit_loop(model,
 def experimental_tpu_test_loop(model,
                                dataset,
                                verbose=0,
-                               steps=None):
+                               steps=None,
+                               callbacks=None):
   """Test loop for evaluating with TPU DistributionStrategy.
 
   Arguments:
@@ -425,6 +426,7 @@ def experimental_tpu_test_loop(model,
       steps: Total number of steps (batches of samples)
           before declaring predictions finished.
           Ignored with the default value of `None`.
+      callbacks: List of callbacks to be called during training
 
   Returns:
       Scalar loss (if the model has a single output and no metrics)
@@ -506,6 +508,16 @@ def experimental_tpu_test_loop(model,
 
   distributed_training_utils._reset_metrics(model)
 
+  callbacks = cbks.configure_callbacks(
+      callbacks,
+      model,
+      do_validation=False,
+      epochs=1,
+      steps_per_epoch=steps,
+      verbose=verbose,
+      count_mode='steps',
+      mode=ModeKeys.TEST)
+
   assert steps is not None
   outs = [0.] * len(model.metrics_names)
   for step in range(steps):
@@ -530,7 +542,11 @@ def experimental_tpu_test_loop(model,
   return outs
 
 
-def experimental_tpu_predict_loop(model, dataset, verbose=0, steps=None):
+def experimental_tpu_predict_loop(model,
+                                  dataset,
+                                  verbose=0,
+                                  steps=None,
+                                  callbacks=None):
   """Predict loop for predicting with TPU DistributionStrategy.
 
   Arguments:
@@ -540,6 +556,7 @@ def experimental_tpu_predict_loop(model, dataset, verbose=0, steps=None):
       steps: Total number of steps (batches of samples)
           before declaring `_predict_loop` finished.
           Ignored with the default value of `None`.
+      callbacks: List of callbacks to be called during training
 
   Returns:
       Array of predictions (if the model has a single output)
@@ -642,6 +659,16 @@ def experimental_tpu_predict_loop(model, dataset, verbose=0, steps=None):
 
   distributed_training_utils._reset_metrics(model)
 
+  callbacks = cbks.configure_callbacks(
+      callbacks,
+      model,
+      do_validation=False,
+      epochs=1,
+      steps_per_epoch=steps,
+      verbose=verbose,
+      count_mode='steps',
+      mode=ModeKeys.PREDICT)
+
   assert steps is not None
   # Since we do not know how many samples we will see, we cannot pre-allocate
   # the returned Numpy arrays. Instead, we store one array per batch seen
-- 
GitLab


From c31b9d4c083bfc44aa0415821788b8eb7abf6b35 Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Tue, 29 Jan 2019 09:37:02 -0800
Subject: [PATCH 1472/2345] Change arg order for assert_equal_graph_def to be
 consistent with the rest of the assert* functions.

PiperOrigin-RevId: 231417691
---
 tensorflow/python/framework/test_util.py             | 4 ++--
 tensorflow/tools/api/golden/v2/tensorflow.test.pbtxt | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index 2bff3e457f..d27572f056 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -132,7 +132,7 @@ def assert_ops_in_graph(expected_ops, graph):
 
 
 @tf_export("test.assert_equal_graph_def", v1=[])
-def assert_equal_graph_def_v2(actual, expected):
+def assert_equal_graph_def_v2(expected, actual):
   """Asserts that two `GraphDef`s are (mostly) the same.
 
   Compares two `GraphDef` protos for equality, ignoring versions and ordering of
@@ -141,8 +141,8 @@ def assert_equal_graph_def_v2(actual, expected):
   ignores randomized attribute values that may appear in V2 checkpoints.
 
   Args:
-    actual: The `GraphDef` we have.
     expected: The `GraphDef` we expected.
+    actual: The `GraphDef` we have.
 
   Raises:
     AssertionError: If the `GraphDef`s do not match.
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.test.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.test.pbtxt
index 980e96ac25..3c77788cb0 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.test.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.test.pbtxt
@@ -10,7 +10,7 @@ tf_module {
   }
   member_method {
     name: "assert_equal_graph_def"
-    argspec: "args=[\'actual\', \'expected\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'expected\', \'actual\'], varargs=None, keywords=None, defaults=None"
   }
   member_method {
     name: "benchmark_config"
-- 
GitLab


From 140efb1567884abd65608b1deb17ebf16f8f276e Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Tue, 29 Jan 2019 09:40:18 -0800
Subject: [PATCH 1473/2345] Moving MutableHashTable and MutableDenseHashTable
 from contrib to core. Also removing the shared_name argument

PiperOrigin-RevId: 231418302
---
 tensorflow/contrib/lookup/lookup_ops.py       |  501 +------
 tensorflow/contrib/lookup/lookup_ops_test.py  | 1299 ----------------
 .../python/kernel_tests/lookup_ops_test.py    | 1303 +++++++++++++++++
 tensorflow/python/ops/lookup_ops.py           |  495 ++++++-
 4 files changed, 1799 insertions(+), 1799 deletions(-)

diff --git a/tensorflow/contrib/lookup/lookup_ops.py b/tensorflow/contrib/lookup/lookup_ops.py
index c2916b82a1..3d21fb68a1 100644
--- a/tensorflow/contrib/lookup/lookup_ops.py
+++ b/tensorflow/contrib/lookup/lookup_ops.py
@@ -18,8 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import functools
-
 from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -41,7 +39,6 @@ from tensorflow.python.ops.lookup_ops import TextFileIndex
 from tensorflow.python.ops.lookup_ops import TextFileInitializer
 from tensorflow.python.ops.lookup_ops import TextFileStringTableInitializer
 # pylint: enable=unused-import
-from tensorflow.python.training.saver import BaseSaverBuilder
 from tensorflow.python.util.deprecation import deprecated
 
 
@@ -364,499 +361,5 @@ class HashTable(InitializableLookupTableBase):
     return exported_keys, exported_values
 
 
-class MutableHashTable(LookupInterface):
-  """A generic mutable hash table implementation.
-
-  Data can be inserted by calling the insert method and removed by calling the
-  remove method. It does not support initialization via the init method.
-
-  Example usage:
-
-  ```python
-  table = tf.contrib.lookup.MutableHashTable(key_dtype=tf.string,
-                                             value_dtype=tf.int64,
-                                             default_value=-1)
-  sess.run(table.insert(keys, values))
-  out = table.lookup(query_keys)
-  print(out.eval())
-  ```
-  """
-
-  def __init__(self,
-               key_dtype,
-               value_dtype,
-               default_value,
-               shared_name=None,
-               name="MutableHashTable",
-               checkpoint=True):
-    """Creates an empty `MutableHashTable` object.
-
-    Creates a table, the type of its keys and values are specified by key_dtype
-    and value_dtype, respectively.
-
-    Args:
-      key_dtype: the type of the key tensors.
-      value_dtype: the type of the value tensors.
-      default_value: The value to use if a key is missing in the table.
-      shared_name: If non-empty, this table will be shared under
-        the given name across multiple sessions.
-      name: A name for the operation (optional).
-      checkpoint: if True, the contents of the table are saved to and restored
-        from checkpoints. If `shared_name` is empty for a checkpointed table, it
-        is shared using the table node name.
-
-    Returns:
-      A `MutableHashTable` object.
-
-    Raises:
-      ValueError: If checkpoint is True and no name was specified.
-    """
-    self._default_value = ops.convert_to_tensor(default_value,
-                                                dtype=value_dtype)
-    self._value_shape = self._default_value.get_shape()
-    self._checkpoint = checkpoint
-    self._key_dtype = key_dtype
-    self._value_dtype = value_dtype
-    self._name = name
-
-    if context.executing_eagerly() and shared_name is None:
-      # TODO(allenl): This will leak memory due to kernel caching by the
-      # shared_name attribute value (but is better than the alternative of
-      # sharing everything by default when executing eagerly; hopefully creating
-      # tables in a loop is uncommon).
-      shared_name = "table_%d" % (ops.uid(),)
-    self._shared_name = shared_name
-    super(MutableHashTable, self).__init__(key_dtype, value_dtype)
-
-    self._resource_handle = self.create_resource()
-    if checkpoint:
-      saveable = MutableHashTable._Saveable(self, name)
-      if not context.executing_eagerly():
-        ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable)
-
-  def create_resource(self):
-    # The table must be shared if checkpointing is requested for multi-worker
-    # training to work correctly. Use the node name if no shared_name has been
-    # explicitly specified.
-    use_node_name_sharing = self._checkpoint and self._shared_name is None
-    if self._default_value.get_shape().ndims == 0:
-      table_ref = gen_lookup_ops.mutable_hash_table_v2(
-          shared_name=self._shared_name,
-          use_node_name_sharing=use_node_name_sharing,
-          key_dtype=self._key_dtype,
-          value_dtype=self._value_dtype,
-          name=self._name)
-    else:
-      table_ref = gen_lookup_ops.mutable_hash_table_of_tensors_v2(
-          shared_name=self._shared_name,
-          use_node_name_sharing=use_node_name_sharing,
-          key_dtype=self._key_dtype,
-          value_dtype=self._value_dtype,
-          value_shape=self._default_value.get_shape(),
-          name=self._name)
-
-    if context.executing_eagerly():
-      self._table_name = None
-    else:
-      self._table_name = table_ref.op.name.split("/")[-1]
-    return table_ref
-
-  @property
-  def name(self):
-    return self._table_name
-
-  def size(self, name=None):
-    """Compute the number of elements in this table.
-
-    Args:
-      name: A name for the operation (optional).
-
-    Returns:
-      A scalar tensor containing the number of elements in this table.
-    """
-    with ops.name_scope(name, "%s_Size" % self.name,
-                        [self.resource_handle]) as name:
-      with ops.colocate_with(self.resource_handle):
-        return gen_lookup_ops.lookup_table_size_v2(
-            self.resource_handle, name=name)
-
-  def remove(self, keys, name=None):
-    """Removes `keys` and its associated values from the table.
-
-    If a key is not present in the table, it is silently ignored.
-
-    Args:
-      keys: Keys to remove. Can be a tensor of any shape. Must match the table's
-        key type.
-      name: A name for the operation (optional).
-
-    Returns:
-      The created Operation.
-
-    Raises:
-      TypeError: when `keys` do not match the table data types.
-    """
-    if keys.dtype != self._key_dtype:
-      raise TypeError("Signature mismatch. Keys must be dtype %s, got %s." %
-                      (self._key_dtype, keys.dtype))
-
-    with ops.name_scope(
-        name, "%s_lookup_table_remove" % self.name,
-        (self.resource_handle, keys, self._default_value)) as name:
-      # pylint: disable=protected-access
-      op = gen_lookup_ops.lookup_table_remove_v2(
-          self.resource_handle, keys, name=name)
-
-    return op
-
-  def lookup(self, keys, name=None):
-    """Looks up `keys` in a table, outputs the corresponding values.
-
-    The `default_value` is used for keys not present in the table.
-
-    Args:
-      keys: Keys to look up. Can be a tensor of any shape. Must match the
-        table's key_dtype.
-      name: A name for the operation (optional).
-
-    Returns:
-      A tensor containing the values in the same shape as `keys` using the
-        table's value type.
-
-    Raises:
-      TypeError: when `keys` do not match the table data types.
-    """
-    with ops.name_scope(
-        name, "%s_lookup_table_find" % self.name,
-        (self.resource_handle, keys, self._default_value)) as name:
-      keys = ops.convert_to_tensor(keys, dtype=self._key_dtype, name="keys")
-      with ops.colocate_with(self.resource_handle):
-        values = gen_lookup_ops.lookup_table_find_v2(
-            self.resource_handle, keys, self._default_value, name=name)
-    return values
-
-  def insert(self, keys, values, name=None):
-    """Associates `keys` with `values`.
-
-    Args:
-      keys: Keys to insert. Can be a tensor of any shape. Must match the
-        table's key type.
-      values: Values to be associated with keys. Must be a tensor of the same
-        shape as `keys` and match the table's value type.
-      name: A name for the operation (optional).
-
-    Returns:
-      The created Operation.
-
-    Raises:
-      TypeError: when `keys` or `values` doesn't match the table data
-        types.
-    """
-    with ops.name_scope(name, "%s_lookup_table_insert" % self.name,
-                        [self.resource_handle, keys, values]) as name:
-      keys = ops.convert_to_tensor(keys, self._key_dtype, name="keys")
-      values = ops.convert_to_tensor(values, self._value_dtype, name="values")
-      with ops.colocate_with(self.resource_handle):
-        # pylint: disable=protected-access
-        op = gen_lookup_ops.lookup_table_insert_v2(
-            self.resource_handle, keys, values, name=name)
-    return op
-
-  def export(self, name=None):
-    """Returns tensors of all keys and values in the table.
-
-    Args:
-      name: A name for the operation (optional).
-
-    Returns:
-      A pair of tensors with the first tensor containing all keys and the
-        second tensors containing all values in the table.
-    """
-    with ops.name_scope(name, "%s_lookup_table_export_values" % self.name,
-                        [self.resource_handle]) as name:
-      with ops.colocate_with(self.resource_handle):
-        exported_keys, exported_values = gen_lookup_ops.lookup_table_export_v2(
-            self.resource_handle, self._key_dtype, self._value_dtype, name=name)
-    return exported_keys, exported_values
-
-  def _gather_saveables_for_checkpoint(self):
-    """For object-based checkpointing."""
-    return {"table": functools.partial(MutableHashTable._Saveable, table=self)}
-
-  class _Saveable(BaseSaverBuilder.SaveableObject):
-    """SaveableObject implementation for MutableHashTable."""
-
-    def __init__(self, table, name):
-      tensors = table.export()
-      specs = [
-          BaseSaverBuilder.SaveSpec(tensors[0], "", name + "-keys"),
-          BaseSaverBuilder.SaveSpec(tensors[1], "", name + "-values")
-      ]
-      # pylint: disable=protected-access
-      super(MutableHashTable._Saveable, self).__init__(table, specs, name)
-
-    def restore(self, restored_tensors, restored_shapes):
-      del restored_shapes  # unused
-      # pylint: disable=protected-access
-      with ops.colocate_with(self.op.resource_handle):
-        return gen_lookup_ops.lookup_table_import_v2(
-            self.op.resource_handle, restored_tensors[0], restored_tensors[1])
-
-
-class MutableDenseHashTable(LookupInterface):
-  """A generic mutable hash table implementation using tensors as backing store.
-
-  Data can be inserted by calling the insert method and removed by calling the
-  remove method. It does not support initialization via the init method.
-
-  It uses "open addressing" with quadratic reprobing to resolve collisions.
-  Compared to `MutableHashTable` the insert, remove and lookup operations in a
-  `MutableDenseHashTable` are typically faster, but memory usage can be higher.
-  However, `MutableDenseHashTable` does not require additional memory for
-  temporary tensors created during checkpointing and restore operations.
-
-  Example usage:
-
-  ```python
-  table = tf.contrib.lookup.MutableDenseHashTable(key_dtype=tf.int64,
-                                                  value_dtype=tf.int64,
-                                                  default_value=-1,
-                                                  empty_key=0,
-                                                  deleted_key=-1)
-
-  sess.run(table.insert(keys, values))
-  out = table.lookup(query_keys)
-  print(out.eval())
-  ```
-  """
-
-  # TODO(andreasst): consider extracting common code with MutableHashTable into
-  # a common superclass.
-  def __init__(self,
-               key_dtype,
-               value_dtype,
-               default_value,
-               empty_key,
-               deleted_key,
-               initial_num_buckets=None,
-               shared_name=None,
-               name="MutableDenseHashTable",
-               checkpoint=True):
-    """Creates an empty `MutableDenseHashTable` object.
-
-    Creates a table, the type of its keys and values are specified by key_dtype
-    and value_dtype, respectively.
-
-    Args:
-      key_dtype: the type of the key tensors.
-      value_dtype: the type of the value tensors.
-      default_value: The value to use if a key is missing in the table.
-      empty_key: the key to use to represent empty buckets internally. Must not
-        be used in insert, remove or lookup operations.
-      initial_num_buckets: the initial number of buckets.
-      shared_name: If non-empty, this table will be shared under
-        the given name across multiple sessions.
-      name: A name for the operation (optional).
-      checkpoint: if True, the contents of the table are saved to and restored
-        from checkpoints. If `shared_name` is empty for a checkpointed table, it
-        is shared using the table node name.
-      deleted_key: the key to use to represent deleted buckets internally. Must
-        not be used in insert, remove or lookup operations and be different from
-        the empty_key.
-
-    Returns:
-      A `MutableDenseHashTable` object.
-
-    Raises:
-      ValueError: If checkpoint is True and no name was specified.
-    """
-    self._default_value = ops.convert_to_tensor(
-        default_value, dtype=value_dtype, name="default_value")
-    self._key_dtype = key_dtype
-    self._value_dtype = value_dtype
-    self._initial_num_buckets = initial_num_buckets
-    self._value_shape = self._default_value.get_shape()
-    self._checkpoint = checkpoint
-    self._name = name
-
-    self._empty_key = ops.convert_to_tensor(
-        empty_key, dtype=key_dtype, name="empty_key")
-    self._deleted_key = ops.convert_to_tensor(
-        deleted_key, dtype=key_dtype, name="deleted_key")
-    if context.executing_eagerly() and shared_name is None:
-      # TODO(allenl): This will leak memory due to kernel caching by the
-      # shared_name attribute value (but is better than the alternative of
-      # sharing everything by default when executing eagerly; hopefully creating
-      # tables in a loop is uncommon).
-      shared_name = "table_%d" % (ops.uid(),)
-    self._shared_name = shared_name
-    super(MutableDenseHashTable, self).__init__(key_dtype, value_dtype)
-
-    self._resource_handle = self.create_resource()
-    if checkpoint:
-      saveable = MutableDenseHashTable._Saveable(self, name)
-      if not context.executing_eagerly():
-        ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable)
-
-  def create_resource(self):
-    # The table must be shared if checkpointing is requested for multi-worker
-    # training to work correctly. Use the node name if no shared_name has been
-    # explicitly specified.
-    use_node_name_sharing = self._checkpoint and self._shared_name is None
-    table_ref = gen_lookup_ops.mutable_dense_hash_table_v2(
-        empty_key=self._empty_key,
-        deleted_key=self._deleted_key,
-        shared_name=self._shared_name,
-        use_node_name_sharing=use_node_name_sharing,
-        value_dtype=self._value_dtype,
-        value_shape=self._value_shape,
-        initial_num_buckets=self._initial_num_buckets,
-        name=self._name)
-    if context.executing_eagerly():
-      self._table_name = None
-    else:
-      self._table_name = table_ref.op.name.split("/")[-1]
-    return table_ref
-
-  @property
-  def name(self):
-    return self._table_name
-
-  def size(self, name=None):
-    """Compute the number of elements in this table.
-
-    Args:
-      name: A name for the operation (optional).
-
-    Returns:
-      A scalar tensor containing the number of elements in this table.
-    """
-    with ops.name_scope(name, "%s_Size" % self.name,
-                        [self.resource_handle]) as name:
-      with ops.colocate_with(self.resource_handle):
-        return gen_lookup_ops.lookup_table_size_v2(
-            self.resource_handle, name=name)
-
-  def lookup(self, keys, name=None):
-    """Looks up `keys` in a table, outputs the corresponding values.
-
-    The `default_value` is used for keys not present in the table.
-
-    Args:
-      keys: Keys to look up. Can be a tensor of any shape. Must match the
-        table's key_dtype.
-      name: A name for the operation (optional).
-
-    Returns:
-      A tensor containing the values in the same shape as `keys` using the
-        table's value type.
-
-    Raises:
-      TypeError: when `keys` do not match the table data types.
-    """
-    with ops.name_scope(name, "%s_lookup_table_find" % self.name,
-                        [self.resource_handle, keys]) as name:
-      keys = ops.convert_to_tensor(keys, dtype=self._key_dtype, name="keys")
-      with ops.colocate_with(self.resource_handle):
-        values = gen_lookup_ops.lookup_table_find_v2(
-            self.resource_handle, keys, self._default_value, name=name)
-
-    return values
-
-  def insert(self, keys, values, name=None):
-    """Associates `keys` with `values`.
-
-    Args:
-      keys: Keys to insert. Can be a tensor of any shape. Must match the
-        table's key type.
-      values: Values to be associated with keys. Must be a tensor of the same
-        shape as `keys` and match the table's value type.
-      name: A name for the operation (optional).
-
-    Returns:
-      The created Operation.
-
-    Raises:
-      TypeError: when `keys` or `values` doesn't match the table data
-        types.
-    """
-    with ops.name_scope(name, "%s_lookup_table_insert" % self.name,
-                        [self.resource_handle, keys, values]) as name:
-      keys = ops.convert_to_tensor(keys, dtype=self._key_dtype, name="keys")
-      values = ops.convert_to_tensor(
-          values, dtype=self._value_dtype, name="values")
-      with ops.colocate_with(self.resource_handle):
-        op = gen_lookup_ops.lookup_table_insert_v2(
-            self.resource_handle, keys, values, name=name)
-      return op
-
-  def remove(self, keys, name=None):
-    """Removes `keys` and its associated values from the table.
-
-    If a key is not present in the table, it is silently ignored.
-
-    Args:
-      keys: Keys to remove. Can be a tensor of any shape. Must match the table's
-        key type.
-      name: A name for the operation (optional).
-
-    Returns:
-      The created Operation.
-
-    Raises:
-      TypeError: when `keys` do not match the table data types.
-    """
-    if keys.dtype != self._key_dtype:
-      raise TypeError("Signature mismatch. Keys must be dtype %s, got %s." %
-                      (self._key_dtype, keys.dtype))
-
-    with ops.name_scope(
-        name, "%s_lookup_table_remove" % self.name,
-        (self.resource_handle, keys, self._default_value)) as name:
-      # pylint: disable=protected-access
-      op = gen_lookup_ops.lookup_table_remove_v2(
-          self.resource_handle, keys, name=name)
-
-    return op
-
-  def export(self, name=None):
-    """Returns tensors of all keys and values in the table.
-
-    Args:
-      name: A name for the operation (optional).
-
-    Returns:
-      A pair of tensors with the first tensor containing all keys and the
-        second tensors containing all values in the table.
-    """
-    with ops.name_scope(name, "%s_lookup_table_export_values" % self.name,
-                        [self.resource_handle]) as name:
-      with ops.colocate_with(self.resource_handle):
-        exported_keys, exported_values = gen_lookup_ops.lookup_table_export_v2(
-            self.resource_handle, self._key_dtype, self._value_dtype, name=name)
-
-    return exported_keys, exported_values
-
-  def _gather_saveables_for_checkpoint(self):
-    """For object-based checkpointing."""
-    return {"table": functools.partial(
-        MutableDenseHashTable._Saveable, table=self)}
-
-  class _Saveable(BaseSaverBuilder.SaveableObject):
-    """SaveableObject implementation for MutableDenseHashTable."""
-
-    def __init__(self, table, name):
-      tensors = table.export()
-      specs = [
-          BaseSaverBuilder.SaveSpec(tensors[0], "", name + "-keys"),
-          BaseSaverBuilder.SaveSpec(tensors[1], "", name + "-values")
-      ]
-      # pylint: disable=protected-access
-      super(MutableDenseHashTable._Saveable, self).__init__(table, specs, name)
-
-    def restore(self, restored_tensors, restored_shapes):
-      del restored_shapes  # unused
-      # pylint: disable=protected-access
-      with ops.colocate_with(self.op.resource_handle):
-        return gen_lookup_ops.lookup_table_import_v2(
-            self.op.resource_handle, restored_tensors[0], restored_tensors[1])
+MutableHashTable = lookup_ops.MutableHashTable
+MutableDenseHashTable = lookup_ops.MutableDenseHashTable
diff --git a/tensorflow/contrib/lookup/lookup_ops_test.py b/tensorflow/contrib/lookup/lookup_ops_test.py
index 9b2c2dd87c..591eabc66c 100644
--- a/tensorflow/contrib/lookup/lookup_ops_test.py
+++ b/tensorflow/contrib/lookup/lookup_ops_test.py
@@ -18,14 +18,10 @@ from __future__ import division
 from __future__ import print_function
 
 import os
-import tempfile
 import numpy as np
-import six
 
 from tensorflow.contrib import lookup
 from tensorflow.python.client import session
-from tensorflow.python.data.experimental.ops import counter
-from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -37,9 +33,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import lookup_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
-from tensorflow.python.training import saver
 from tensorflow.python.training import server_lib
-from tensorflow.python.training.checkpointable import util as checkpointable
 
 
 class HashTableOpTest(test.TestCase):
@@ -299,1240 +293,6 @@ class HashTableOpTest(test.TestCase):
       self.assertAllEqual([b"brain", b"salad", b"n/a"], result)
 
 
-class MutableHashTableOpTest(test.TestCase):
-
-  def testMutableHashTable(self):
-    with self.cached_session():
-      default_val = -1
-      keys = constant_op.constant(["brain", "salad", "surgery", "tarkus"])
-      values = constant_op.constant([0, 1, 2, 3], dtypes.int64)
-      table = lookup.MutableHashTable(dtypes.string, dtypes.int64,
-                                      default_val)
-      self.assertAllEqual(0, table.size().eval())
-
-      table.insert(keys, values).run()
-      self.assertAllEqual(4, table.size().eval())
-
-      remove_string = constant_op.constant(["tarkus", "tank"])
-      table.remove(remove_string).run()
-      self.assertAllEqual(3, table.size().eval())
-
-      input_string = constant_op.constant(["brain", "salad", "tank"])
-      output = table.lookup(input_string)
-      self.assertAllEqual([3], output.get_shape())
-
-      result = output.eval()
-      self.assertAllEqual([0, 1, -1], result)
-
-      exported_keys, exported_values = table.export()
-      self.assertAllEqual([None], exported_keys.get_shape().as_list())
-      self.assertAllEqual([None], exported_values.get_shape().as_list())
-
-      # exported data is in the order of the internal map, i.e. undefined
-      sorted_keys = np.sort(exported_keys.eval())
-      sorted_values = np.sort(exported_values.eval())
-      self.assertAllEqual([b"brain", b"salad", b"surgery"], sorted_keys)
-      self.assertAllEqual([0, 1, 2], sorted_values)
-
-  def testSaveRestore(self):
-    save_dir = os.path.join(self.get_temp_dir(), "save_restore")
-    save_path = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash")
-
-    with self.session(graph=ops.Graph()) as sess:
-      v0 = variables.Variable(10.0, name="v0")
-      v1 = variables.Variable(20.0, name="v1")
-
-      default_val = -1
-      keys = constant_op.constant(["b", "c", "d"], dtypes.string)
-      values = constant_op.constant([0, 1, 2], dtypes.int64)
-      table = lookup.MutableHashTable(
-          dtypes.string, dtypes.int64, default_val, name="t1", checkpoint=True)
-
-      save = saver.Saver()
-      variables.global_variables_initializer().run()
-
-      # Check that the parameter nodes have been initialized.
-      self.assertEqual(10.0, v0.eval())
-      self.assertEqual(20.0, v1.eval())
-
-      self.assertAllEqual(0, table.size().eval())
-      table.insert(keys, values).run()
-      self.assertAllEqual(3, table.size().eval())
-
-      val = save.save(sess, save_path)
-      self.assertTrue(isinstance(val, six.string_types))
-      self.assertEqual(save_path, val)
-
-    with self.session(graph=ops.Graph()) as sess:
-      v0 = variables.Variable(-1.0, name="v0")
-      v1 = variables.Variable(-1.0, name="v1")
-      default_val = -1
-      table = lookup.MutableHashTable(
-          dtypes.string, dtypes.int64, default_val, name="t1", checkpoint=True)
-      table.insert(
-          constant_op.constant(["a", "c"], dtypes.string),
-          constant_op.constant([12, 24], dtypes.int64)).run()
-      self.assertAllEqual(2, table.size().eval())
-
-      save = saver.Saver()
-
-      # Restore the saved values in the parameter nodes.
-      save.restore(sess, save_path)
-      # Check that the parameter nodes have been restored.
-      self.assertEqual(10.0, v0.eval())
-      self.assertEqual(20.0, v1.eval())
-
-      self.assertAllEqual(3, table.size().eval())
-
-      input_string = constant_op.constant(["a", "b", "c", "d", "e"],
-                                          dtypes.string)
-      output = table.lookup(input_string)
-      self.assertAllEqual([-1, 0, 1, 2, -1], output.eval())
-
-  @test_util.run_in_graph_and_eager_modes
-  def testObjectSaveRestore(self):
-    save_dir = os.path.join(self.get_temp_dir(), "save_restore")
-    save_prefix = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash")
-
-    v0 = variables.Variable(10.0, name="v0")
-    v1 = variables.Variable(20.0, name="v1")
-
-    default_val = -1
-    keys = constant_op.constant(["b", "c", "d"], dtypes.string)
-    values = constant_op.constant([0, 1, 2], dtypes.int64)
-    table = lookup.MutableHashTable(
-        dtypes.string, dtypes.int64, default_val, name="t1", checkpoint=True)
-
-    checkpoint = checkpointable.Checkpoint(table=table, v0=v0, v1=v1)
-    self.evaluate([v0.initializer, v1.initializer])
-
-    # Check that the parameter nodes have been initialized.
-    self.assertEqual(10.0, self.evaluate(v0))
-    self.assertEqual(20.0, self.evaluate(v1))
-
-    self.assertAllEqual(0, self.evaluate(table.size()))
-    self.evaluate(table.insert(keys, values))
-    self.assertAllEqual(3, self.evaluate(table.size()))
-
-    save_path = checkpoint.save(save_prefix)
-    del table, checkpoint, v0, v1
-
-    v0 = variables.Variable(-1.0, name="v0")
-    v1 = variables.Variable(-1.0, name="v1")
-    default_val = -1
-    table = lookup.MutableHashTable(
-        dtypes.string, dtypes.int64, default_val, name="t1", checkpoint=True)
-    self.evaluate(table.insert(
-        constant_op.constant(["a", "c"], dtypes.string),
-        constant_op.constant([12, 24], dtypes.int64)))
-    self.assertAllEqual(2, self.evaluate(table.size()))
-
-    checkpoint = checkpointable.Checkpoint(table=table, v0=v0, v1=v1)
-
-    # Restore the saved values in the parameter nodes.
-    checkpoint.restore(save_path).run_restore_ops()
-    # Check that the parameter nodes have been restored.
-    self.assertEqual(10.0, self.evaluate(v0))
-    self.assertEqual(20.0, self.evaluate(v1))
-
-    self.assertAllEqual(3, self.evaluate(table.size()))
-
-    input_string = constant_op.constant(["a", "b", "c", "d", "e"],
-                                        dtypes.string)
-    output = table.lookup(input_string)
-    self.assertAllEqual([-1, 0, 1, 2, -1], self.evaluate(output))
-
-  def testSharing(self):
-    # Start a server to store the table state
-    server = server_lib.Server(
-        {
-            "local0": ["localhost:0"]
-        }, protocol="grpc", start=True)
-    # Create two sessions sharing the same state
-    session1 = session.Session(server.target)
-    session2 = session.Session(server.target)
-
-    table = lookup.MutableHashTable(
-        dtypes.int64, dtypes.string, "-", name="t1")
-
-    # Populate the table in the first session
-    with session1:
-      self.assertAllEqual(0, table.size().eval())
-
-      keys = constant_op.constant([11, 12], dtypes.int64)
-      values = constant_op.constant(["a", "b"])
-      table.insert(keys, values).run()
-      self.assertAllEqual(2, table.size().eval())
-
-      output = table.lookup(constant_op.constant([11, 12, 13], dtypes.int64))
-      self.assertAllEqual([b"a", b"b", b"-"], output.eval())
-
-    # Verify that we can access the shared data from the second session
-    with session2:
-      self.assertAllEqual(2, table.size().eval())
-
-      output = table.lookup(constant_op.constant([10, 11, 12], dtypes.int64))
-      self.assertAllEqual([b"-", b"a", b"b"], output.eval())
-
-  def testMutableHashTableOfTensors(self):
-    with self.cached_session():
-      default_val = constant_op.constant([-1, -1], dtypes.int64)
-      keys = constant_op.constant(["brain", "salad", "surgery", "tarkus"])
-      values = constant_op.constant([[0, 1], [2, 3], [4, 5], [6, 7]],
-                                    dtypes.int64)
-      table = lookup.MutableHashTable(dtypes.string, dtypes.int64,
-                                      default_val)
-      self.assertAllEqual(0, table.size().eval())
-
-      table.insert(keys, values).run()
-      self.assertAllEqual(4, table.size().eval())
-
-      remove_string = constant_op.constant(["tarkus", "tank"])
-      table.remove(remove_string).run()
-      self.assertAllEqual(3, table.size().eval())
-
-      input_string = constant_op.constant(["brain", "salad", "tank"])
-      output = table.lookup(input_string)
-      self.assertAllEqual([3, 2], output.get_shape())
-
-      result = output.eval()
-      self.assertAllEqual([[0, 1], [2, 3], [-1, -1]], result)
-
-      exported_keys, exported_values = table.export()
-      self.assertAllEqual([None], exported_keys.get_shape().as_list(),
-                          msg="Saw shape %s" % exported_keys.shape)
-      self.assertAllEqual([None, 2], exported_values.get_shape().as_list(),
-                          msg="Saw shape %s" % exported_values.shape)
-      # exported data is in the order of the internal map, i.e. undefined
-      sorted_keys = np.sort(exported_keys.eval())
-      sorted_values = np.sort(exported_values.eval())
-      self.assertAllEqual([b"brain", b"salad", b"surgery"], sorted_keys)
-      self.assertAllEqual([[4, 5], [2, 3], [0, 1]], sorted_values)
-
-  def testMutableHashTableExportInsert(self):
-    with self.cached_session():
-      default_val = constant_op.constant([-1, -1], dtypes.int64)
-      keys = constant_op.constant(["brain", "salad", "surgery"])
-      values = constant_op.constant([[0, 1], [2, 3], [4, 5]], dtypes.int64)
-      table1 = lookup.MutableHashTable(dtypes.string, dtypes.int64,
-                                       default_val)
-      self.assertAllEqual(0, table1.size().eval())
-      table1.insert(keys, values).run()
-      self.assertAllEqual(3, table1.size().eval())
-
-      input_string = constant_op.constant(["brain", "salad", "tank"])
-      expected_output = [[0, 1], [2, 3], [-1, -1]]
-      output1 = table1.lookup(input_string)
-      self.assertAllEqual(expected_output, output1.eval())
-
-      exported_keys, exported_values = table1.export()
-      self.assertAllEqual(3, exported_keys.eval().size)
-      self.assertAllEqual(6, exported_values.eval().size)
-
-      # Populate a second table from the exported data
-      table2 = lookup.MutableHashTable(dtypes.string, dtypes.int64,
-                                       default_val)
-      self.assertAllEqual(0, table2.size().eval())
-      table2.insert(exported_keys, exported_values).run()
-      self.assertAllEqual(3, table2.size().eval())
-
-      # Verify lookup result is still the same
-      output2 = table2.lookup(input_string)
-      self.assertAllEqual(expected_output, output2.eval())
-
-  def testMutableHashTableOfTensorsInvalidShape(self):
-    with self.cached_session():
-      default_val = constant_op.constant([-1, -1], dtypes.int64)
-      keys = constant_op.constant(["brain", "salad", "surgery"])
-      table = lookup.MutableHashTable(dtypes.string, dtypes.int64,
-                                      default_val)
-
-      # Shape [6] instead of [3, 2]
-      values = constant_op.constant([0, 1, 2, 3, 4, 5], dtypes.int64)
-      with self.assertRaisesOpError("Expected shape"):
-        table.insert(keys, values).run()
-
-      # Shape [2,3] instead of [3, 2]
-      values = constant_op.constant([[0, 1, 2], [3, 4, 5]], dtypes.int64)
-      with self.assertRaisesOpError("Expected shape"):
-        table.insert(keys, values).run()
-
-      # Shape [2, 2] instead of [3, 2]
-      values = constant_op.constant([[0, 1], [2, 3]], dtypes.int64)
-      with self.assertRaisesOpError("Expected shape"):
-        table.insert(keys, values).run()
-
-      # Shape [3, 1] instead of [3, 2]
-      values = constant_op.constant([[0], [2], [4]], dtypes.int64)
-      with self.assertRaisesOpError("Expected shape"):
-        table.insert(keys, values).run()
-
-      # Valid Insert
-      values = constant_op.constant([[0, 1], [2, 3], [4, 5]], dtypes.int64)
-      table.insert(keys, values).run()
-      self.assertAllEqual(3, table.size().eval())
-
-  def testMutableHashTableInvalidDefaultValue(self):
-    with self.cached_session():
-      default_val = constant_op.constant([[-1, -1]], dtypes.int64)
-      table = lookup.MutableHashTable(dtypes.string, dtypes.int64,
-                                      default_val)
-      with self.assertRaisesOpError("Default value must be a vector"):
-        self.assertAllEqual(0, table.size().eval())
-
-  def testMutableHashTableDuplicateInsert(self):
-    with self.cached_session():
-      default_val = -1
-      keys = constant_op.constant(["brain", "salad", "surgery", "brain"])
-      values = constant_op.constant([0, 1, 2, 3], dtypes.int64)
-      table = lookup.MutableHashTable(dtypes.string, dtypes.int64,
-                                      default_val)
-      self.assertAllEqual(0, table.size().eval())
-
-      table.insert(keys, values).run()
-      self.assertAllEqual(3, table.size().eval())
-
-      input_string = constant_op.constant(["brain", "salad", "tank"])
-      output = table.lookup(input_string)
-
-      result = output.eval()
-      self.assertAllEqual([3, 1, -1], result)
-
-  def testMutableHashTableFindHighRank(self):
-    with self.cached_session():
-      default_val = -1
-      keys = constant_op.constant(["brain", "salad", "surgery"])
-      values = constant_op.constant([0, 1, 2], dtypes.int64)
-      table = lookup.MutableHashTable(dtypes.string, dtypes.int64,
-                                      default_val)
-
-      table.insert(keys, values).run()
-      self.assertAllEqual(3, table.size().eval())
-
-      input_string = constant_op.constant(
-          [["brain", "salad"], ["tank", "tarkus"]])
-      output = table.lookup(input_string)
-      self.assertAllEqual([2, 2], output.get_shape())
-
-      result = output.eval()
-      self.assertAllEqual([[0, 1], [-1, -1]], result)
-
-  def testMutableHashTableInsertHighRank(self):
-    with self.cached_session():
-      default_val = -1
-      keys = constant_op.constant([["brain", "salad"], ["surgery", "tank"]])
-      values = constant_op.constant([[0, 1], [2, 3]], dtypes.int64)
-      table = lookup.MutableHashTable(dtypes.string, dtypes.int64,
-                                      default_val)
-
-      table.insert(keys, values).run()
-      self.assertAllEqual(4, table.size().eval())
-
-      input_string = constant_op.constant(["brain", "salad", "tank", "tarkus"])
-      output = table.lookup(input_string)
-
-      result = output.eval()
-      self.assertAllEqual([0, 1, 3, -1], result)
-
-  def testMutableHashTableRemoveHighRank(self):
-    with self.test_session():
-      default_val = -1
-      keys = constant_op.constant([["brain", "salad"], ["surgery", "tank"]])
-      values = constant_op.constant([[0, 1], [2, 3]], dtypes.int64)
-      table = lookup.MutableHashTable(dtypes.string, dtypes.int64, default_val)
-
-      table.insert(keys, values).run()
-      self.assertAllEqual(4, table.size().eval())
-
-      remove_string = constant_op.constant(["salad", "tarkus"])
-      table.remove(remove_string).run()
-      self.assertAllEqual(3, table.size().eval())
-
-      input_string = constant_op.constant(["brain", "salad", "tank", "tarkus"])
-      output = table.lookup(input_string)
-
-      result = output.eval()
-      self.assertAllEqual([0, -1, 3, -1], result)
-
-  def testMutableHashTableOfTensorsFindHighRank(self):
-    with self.cached_session():
-      default_val = constant_op.constant([-1, -1, -1], dtypes.int64)
-      keys = constant_op.constant(["brain", "salad", "surgery"])
-      values = constant_op.constant([[0, 1, 2], [2, 3, 4], [4, 5, 6]],
-                                    dtypes.int64)
-      table = lookup.MutableHashTable(dtypes.string, dtypes.int64,
-                                      default_val)
-
-      table.insert(keys, values).run()
-      self.assertAllEqual(3, table.size().eval())
-
-      input_string = constant_op.constant(
-          [["brain", "salad"], ["tank", "tarkus"]])
-      output = table.lookup(input_string)
-      self.assertAllEqual([2, 2, 3], output.get_shape())
-
-      result = output.eval()
-      self.assertAllEqual(
-          [[[0, 1, 2], [2, 3, 4]], [[-1, -1, -1], [-1, -1, -1]]], result)
-
-  def testMutableHashTableOfTensorsRemoveHighRank(self):
-    with self.test_session():
-      default_val = constant_op.constant([-1, -1, -1], dtypes.int64)
-      keys = constant_op.constant(["brain", "salad", "surgery"])
-      values = constant_op.constant([[0, 1, 2], [2, 3, 4], [4, 5, 6]],
-                                    dtypes.int64)
-      table = lookup.MutableHashTable(dtypes.string, dtypes.int64, default_val)
-
-      table.insert(keys, values).run()
-      self.assertAllEqual(3, table.size().eval())
-
-      remove_string = constant_op.constant([["brain", "tank"]])
-      table.remove(remove_string).run()
-      self.assertAllEqual(2, table.size().eval())
-
-      input_string = constant_op.constant([["brain", "salad"],
-                                           ["surgery", "tank"]])
-      output = table.lookup(input_string)
-      self.assertAllEqual([2, 2, 3], output.get_shape())
-
-      result = output.eval()
-      self.assertAllEqual(
-          [[[-1, -1, -1], [2, 3, 4]], [[4, 5, 6], [-1, -1, -1]]], result)
-
-  def testMultipleMutableHashTables(self):
-    with self.cached_session() as sess:
-      default_val = -1
-      keys = constant_op.constant(["brain", "salad", "surgery"])
-      values = constant_op.constant([0, 1, 2], dtypes.int64)
-
-      table1 = lookup.MutableHashTable(dtypes.string, dtypes.int64,
-                                       default_val)
-      table2 = lookup.MutableHashTable(dtypes.string, dtypes.int64,
-                                       default_val)
-      table3 = lookup.MutableHashTable(dtypes.string, dtypes.int64,
-                                       default_val)
-      table1.insert(keys, values).run()
-      table2.insert(keys, values).run()
-      table3.insert(keys, values).run()
-
-      self.assertAllEqual(3, table1.size().eval())
-      self.assertAllEqual(3, table2.size().eval())
-      self.assertAllEqual(3, table3.size().eval())
-
-      input_string = constant_op.constant(["brain", "salad", "tank"])
-      output1 = table1.lookup(input_string)
-      output2 = table2.lookup(input_string)
-      output3 = table3.lookup(input_string)
-
-      out1, out2, out3 = sess.run([output1, output2, output3])
-      self.assertAllEqual([0, 1, -1], out1)
-      self.assertAllEqual([0, 1, -1], out2)
-      self.assertAllEqual([0, 1, -1], out3)
-
-  def testMutableHashTableWithTensorDefault(self):
-    with self.cached_session():
-      default_val = constant_op.constant(-1, dtypes.int64)
-      keys = constant_op.constant(["brain", "salad", "surgery"])
-      values = constant_op.constant([0, 1, 2], dtypes.int64)
-      table = lookup.MutableHashTable(dtypes.string, dtypes.int64,
-                                      default_val)
-
-      table.insert(keys, values).run()
-      self.assertAllEqual(3, table.size().eval())
-
-      input_string = constant_op.constant(["brain", "salad", "tank"])
-      output = table.lookup(input_string)
-
-      result = output.eval()
-      self.assertAllEqual([0, 1, -1], result)
-
-  def testSignatureMismatch(self):
-    with self.cached_session():
-      default_val = -1
-      keys = constant_op.constant(["brain", "salad", "surgery"])
-      values = constant_op.constant([0, 1, 2], dtypes.int64)
-      table = lookup.MutableHashTable(dtypes.string, dtypes.int64,
-                                      default_val)
-
-      # insert with keys of the wrong type
-      with self.assertRaises(ValueError):
-        table.insert(constant_op.constant([4, 5, 6]), values).run()
-
-      # insert with values of the wrong type
-      with self.assertRaises(ValueError):
-        table.insert(keys, constant_op.constant(["a", "b", "c"])).run()
-
-      self.assertAllEqual(0, table.size().eval())
-
-      table.insert(keys, values).run()
-      self.assertAllEqual(3, table.size().eval())
-
-      input_string_ref = variables.Variable("brain")
-      input_int64_ref = variables.Variable(-1, dtype=dtypes.int64)
-      variables.global_variables_initializer().run()
-
-      # Ref types do not produce an insert signature mismatch.
-      table.insert(input_string_ref, input_int64_ref).run()
-      self.assertAllEqual(3, table.size().eval())
-
-      # Ref types do not produce a lookup signature mismatch.
-      self.assertEqual(-1, table.lookup(input_string_ref).eval())
-
-      # lookup with keys of the wrong type
-      input_string = constant_op.constant([1, 2, 3], dtypes.int64)
-      with self.assertRaises(ValueError):
-        table.lookup(input_string).eval()
-
-      # default value of the wrong type
-      with self.assertRaises(TypeError):
-        lookup.MutableHashTable(dtypes.string, dtypes.int64, "UNK")
-
-  def testMutableHashTableStringFloat(self):
-    with self.cached_session():
-      default_val = -1.5
-      keys = constant_op.constant(["brain", "salad", "surgery"])
-      values = constant_op.constant([0, 1.1, 2.2], dtypes.float32)
-      table = lookup.MutableHashTable(dtypes.string, dtypes.float32,
-                                      default_val)
-      self.assertAllEqual(0, table.size().eval())
-
-      table.insert(keys, values).run()
-      self.assertAllEqual(3, table.size().eval())
-
-      input_string = constant_op.constant(["brain", "salad", "tank"])
-      output = table.lookup(input_string)
-
-      result = output.eval()
-      self.assertAllClose([0, 1.1, default_val], result)
-
-  def testMutableHashTableIntFloat(self):
-    with self.cached_session():
-      default_val = -1.0
-      keys = constant_op.constant([3, 7, 0], dtypes.int64)
-      values = constant_op.constant([7.5, -1.2, 9.9], dtypes.float32)
-      table = lookup.MutableHashTable(dtypes.int64, dtypes.float32,
-                                      default_val)
-      self.assertAllEqual(0, table.size().eval())
-
-      table.insert(keys, values).run()
-      self.assertAllEqual(3, table.size().eval())
-
-      input_string = constant_op.constant([7, 0, 11], dtypes.int64)
-      output = table.lookup(input_string)
-
-      result = output.eval()
-      self.assertAllClose([-1.2, 9.9, default_val], result)
-
-  def testMutableHashTableInt64String(self):
-    with self.cached_session():
-      default_val = "n/a"
-      keys = constant_op.constant([0, 1, 2], dtypes.int64)
-      values = constant_op.constant(["brain", "salad", "surgery"])
-      table = lookup.MutableHashTable(dtypes.int64, dtypes.string,
-                                      default_val)
-      self.assertAllEqual(0, table.size().eval())
-
-      table.insert(keys, values).run()
-      self.assertAllEqual(3, table.size().eval())
-
-      input_string = constant_op.constant([0, 1, 3], dtypes.int64)
-      output = table.lookup(input_string)
-
-      result = output.eval()
-      self.assertAllEqual((b"brain", b"salad", b"n/a"), result)
-
-
-class MutableDenseHashTableOpTest(test.TestCase):
-
-  def testBasic(self):
-    with self.cached_session():
-
-      keys = constant_op.constant([11, 12, 13, 14], dtypes.int64)
-      values = constant_op.constant([0, 1, 2, 3], dtypes.int64)
-      table = lookup.MutableDenseHashTable(
-          dtypes.int64,
-          dtypes.int64,
-          default_value=-1,
-          empty_key=0,
-          deleted_key=-1)
-      self.assertAllEqual(0, table.size().eval())
-
-      table.insert(keys, values).run()
-      self.assertAllEqual(4, table.size().eval())
-
-      remove_string = constant_op.constant([12, 15], dtypes.int64)
-      table.remove(remove_string).run()
-      self.assertAllEqual(3, table.size().eval())
-
-      input_string = constant_op.constant([11, 12, 15], dtypes.int64)
-      output = table.lookup(input_string)
-      self.assertAllEqual([3], output.get_shape())
-
-      result = output.eval()
-      self.assertAllEqual([0, -1, -1], result)
-
-  def testBasicBool(self):
-    with self.cached_session():
-
-      keys = constant_op.constant([11, 12, 13, 14], dtypes.int64)
-      values = constant_op.constant([True, True, True, True], dtypes.bool)
-      table = lookup.MutableDenseHashTable(
-          dtypes.int64,
-          dtypes.bool,
-          default_value=False,
-          empty_key=0,
-          deleted_key=-1)
-      self.assertAllEqual(0, table.size().eval())
-
-      table.insert(keys, values).run()
-      self.assertAllEqual(4, table.size().eval())
-
-      remove_string = constant_op.constant([11, 15], dtypes.int64)
-      table.remove(remove_string).run()
-      self.assertAllEqual(3, table.size().eval())
-
-      input_string = constant_op.constant([11, 12, 15], dtypes.int64)
-      output = table.lookup(input_string)
-      self.assertAllEqual([3], output.get_shape())
-
-      result = output.eval()
-      self.assertAllEqual([False, True, False], result)
-
-  def testSameEmptyAndDeletedKey(self):
-    with self.cached_session():
-      with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
-                                   "deleted_key"):
-        table = lookup.MutableDenseHashTable(
-            dtypes.int64,
-            dtypes.int64,
-            default_value=-1,
-            empty_key=42,
-            deleted_key=42)
-        self.assertAllEqual(0, table.size().eval())
-
-  def testLookupUnknownShape(self):
-    with self.cached_session():
-      keys = constant_op.constant([11, 12, 13], dtypes.int64)
-      values = constant_op.constant([0, 1, 2], dtypes.int64)
-      table = lookup.MutableDenseHashTable(
-          dtypes.int64,
-          dtypes.int64,
-          default_value=-1,
-          empty_key=0,
-          deleted_key=-1)
-
-      table.insert(keys, values).run()
-      self.assertAllEqual(3, table.size().eval())
-
-      placeholder_keys = array_ops.placeholder(dtypes.int64)
-      output = table.lookup(placeholder_keys)
-      self.assertAllEqual(None, output.get_shape())
-      result = output.eval({placeholder_keys: [11, 12, 15]})
-      self.assertAllEqual([0, 1, -1], result)
-
-  def testMapStringToFloat(self):
-    with self.cached_session():
-
-      keys = constant_op.constant(["a", "b", "c", "d"], dtypes.string)
-      values = constant_op.constant([0.0, 1.1, 2.2, 3.3], dtypes.float32)
-      default_value = constant_op.constant(-1.5, dtypes.float32)
-      table = lookup.MutableDenseHashTable(
-          dtypes.string,
-          dtypes.float32,
-          default_value=default_value,
-          empty_key="",
-          deleted_key="$")
-      self.assertAllEqual(0, table.size().eval())
-
-      table.insert(keys, values).run()
-      self.assertAllEqual(4, table.size().eval())
-
-      remove_string = constant_op.constant(["b", "e"])
-      table.remove(remove_string).run()
-      self.assertAllEqual(3, table.size().eval())
-
-      input_string = constant_op.constant(["a", "b", "d", "e"], dtypes.string)
-      output = table.lookup(input_string)
-      self.assertAllEqual([4], output.get_shape())
-
-      result = output.eval()
-      self.assertAllClose([0, -1.5, 3.3, -1.5], result)
-
-  def testMapInt64ToFloat(self):
-    for float_dtype in [dtypes.float32, dtypes.float64]:
-      with self.cached_session():
-
-        keys = constant_op.constant([11, 12, 13, 14], dtypes.int64)
-        values = constant_op.constant([0.0, 1.1, 2.2, 3.3], float_dtype)
-        default_value = constant_op.constant(-1.5, float_dtype)
-        table = lookup.MutableDenseHashTable(
-            dtypes.int64,
-            float_dtype,
-            default_value=default_value,
-            empty_key=0,
-            deleted_key=-1)
-        self.assertAllEqual(0, table.size().eval())
-
-        table.insert(keys, values).run()
-        self.assertAllEqual(4, table.size().eval())
-
-        remove_string = constant_op.constant([12, 15], dtypes.int64)
-        table.remove(remove_string).run()
-        self.assertAllEqual(3, table.size().eval())
-
-        input_string = constant_op.constant([11, 12, 14, 15], dtypes.int64)
-        output = table.lookup(input_string)
-        self.assertAllEqual([4], output.get_shape())
-
-        result = output.eval()
-        self.assertAllClose([0, -1.5, 3.3, -1.5], result)
-
-  def testVectorValues(self):
-    with self.cached_session():
-      keys = constant_op.constant([11, 12, 13], dtypes.int64)
-      values = constant_op.constant([[0, 1, 2, 3], [3, 4, 5, 6], [6, 7, 8, 9]],
-                                    dtypes.int64)
-      default_value = constant_op.constant([-1, -2, -3, -4], dtypes.int64)
-      table = lookup.MutableDenseHashTable(
-          dtypes.int64,
-          dtypes.int64,
-          default_value=default_value,
-          empty_key=0,
-          deleted_key=-1,
-          initial_num_buckets=4)
-      self.assertAllEqual(0, table.size().eval())
-
-      table.insert(keys, values).run()
-      self.assertAllEqual(3, table.size().eval())
-      self.assertAllEqual(4, len(table.export()[0].eval()))
-
-      table.insert(
-          constant_op.constant([14], dtypes.int64),
-          constant_op.constant([[2, 3, 4, 5]], dtypes.int64)).run()
-      self.assertAllEqual(4, table.size().eval())
-      self.assertAllEqual(8, len(table.export()[0].eval()))
-
-      remove_string = constant_op.constant([12, 16], dtypes.int64)
-      table.remove(remove_string).run()
-      self.assertAllEqual(3, table.size().eval())
-      self.assertAllEqual(8, len(table.export()[0].eval()))
-
-      input_string = constant_op.constant([11, 12, 14, 15], dtypes.int64)
-      output = table.lookup(input_string)
-      self.assertAllEqual([4, 4],
-                          output.shape,
-                          msg="Saw shape: %s" % output.shape)
-
-      result = output.eval()
-      self.assertAllEqual(
-          [[0, 1, 2, 3], [-1, -2, -3, -4], [2, 3, 4, 5], [-1, -2, -3, -4]],
-          result)
-
-  def testVectorKeys(self):
-    with self.cached_session():
-      keys = constant_op.constant([[0, 1], [1, 2], [1, 3]], dtypes.int64)
-      values = constant_op.constant([10, 11, 12], dtypes.int64)
-      empty_key = constant_op.constant([0, 3], dtypes.int64)
-      deleted_key = constant_op.constant([-1, -1], dtypes.int64)
-      default_value = constant_op.constant(-1, dtypes.int64)
-      table = lookup.MutableDenseHashTable(
-          dtypes.int64,
-          dtypes.int64,
-          default_value=default_value,
-          empty_key=empty_key,
-          deleted_key=deleted_key,
-          initial_num_buckets=8)
-      self.assertAllEqual(0, table.size().eval())
-
-      table.insert(keys, values).run()
-      self.assertAllEqual(3, table.size().eval())
-
-      table.insert(
-          constant_op.constant([[0, 0]], dtypes.int64),
-          constant_op.constant([13], dtypes.int64)).run()
-      self.assertAllEqual(4, table.size().eval())
-      self.assertAllEqual(8, len(table.export()[0].eval()))
-
-      remove_string = constant_op.constant([[1, 2], [7, 8]], dtypes.int64)
-      table.remove(remove_string).run()
-      self.assertAllEqual(3, table.size().eval())
-      self.assertAllEqual(8, len(table.export()[0].eval()))
-
-      input_string = constant_op.constant([[0, 1], [1, 2], [1, 3], [0, 2]],
-                                          dtypes.int64)
-      output = table.lookup(input_string)
-      self.assertAllEqual([4], output.get_shape())
-
-      result = output.eval()
-      self.assertAllEqual([10, -1, 12, -1], result)
-
-  def testResize(self):
-    with self.cached_session():
-      keys = constant_op.constant([11, 12, 13], dtypes.int64)
-      values = constant_op.constant([0, 1, 2], dtypes.int64)
-      table = lookup.MutableDenseHashTable(
-          dtypes.int64,
-          dtypes.int64,
-          default_value=-1,
-          empty_key=0,
-          deleted_key=-1,
-          initial_num_buckets=4)
-      self.assertAllEqual(0, table.size().eval())
-
-      table.insert(keys, values).run()
-      self.assertAllEqual(3, table.size().eval())
-      self.assertAllEqual(4, len(table.export()[0].eval()))
-
-      keys2 = constant_op.constant([12, 99], dtypes.int64)
-      table.remove(keys2).run()
-      self.assertAllEqual(2, table.size().eval())
-      self.assertAllEqual(4, len(table.export()[0].eval()))
-
-      keys3 = constant_op.constant([13, 14, 15, 16, 17], dtypes.int64)
-      values3 = constant_op.constant([3, 4, 5, 6, 7], dtypes.int64)
-
-      table.insert(keys3, values3).run()
-      self.assertAllEqual(6, table.size().eval())
-      self.assertAllEqual(16, len(table.export()[0].eval()))
-
-      keys4 = constant_op.constant([10, 11, 12, 13, 14, 15, 16, 17, 18],
-                                   dtypes.int64)
-      output = table.lookup(keys4)
-      self.assertAllEqual([-1, 0, -1, 3, 4, 5, 6, 7, -1], output.eval())
-
-  def testExport(self):
-    with self.cached_session():
-
-      keys = constant_op.constant([11, 12, 13, 14], dtypes.int64)
-      values = constant_op.constant([1, 2, 3, 4], dtypes.int64)
-      table = lookup.MutableDenseHashTable(
-          dtypes.int64,
-          dtypes.int64,
-          default_value=-1,
-          empty_key=100,
-          deleted_key=200,
-          initial_num_buckets=8)
-      self.assertAllEqual(0, table.size().eval())
-
-      table.insert(keys, values).run()
-      self.assertAllEqual(4, table.size().eval())
-
-      keys2 = constant_op.constant([12, 15], dtypes.int64)
-      table.remove(keys2).run()
-      self.assertAllEqual(3, table.size().eval())
-
-      exported_keys, exported_values = table.export()
-      self.assertAllEqual([None], exported_keys.get_shape().as_list())
-      self.assertAllEqual([None], exported_values.get_shape().as_list())
-
-      np_keys = exported_keys.eval()
-      np_values = exported_values.eval()
-
-      self.assertAllEqual(8, len(np_keys))
-      self.assertAllEqual(8, len(np_values))
-
-      # pair up keys and values, drop extra added dimension
-      pairs = np.dstack((np_keys.flatten(), np_values.flatten()))[0]
-      # sort by key
-      pairs = pairs[pairs[:, 0].argsort()]
-      self.assertAllEqual([[11, 1], [13, 3], [14, 4], [100, 0], [100, 0],
-                           [100, 0], [100, 0], [200, 2]], pairs)
-
-  def testSaveRestore(self):
-    save_dir = os.path.join(self.get_temp_dir(), "save_restore")
-    save_path = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash")
-
-    with self.session(graph=ops.Graph()) as sess:
-      default_value = -1
-      empty_key = 0
-      deleted_key = -1
-      keys = constant_op.constant([11, 12, 13, 14], dtypes.int64)
-      values = constant_op.constant([0, 1, 2, 3], dtypes.int64)
-      table = lookup.MutableDenseHashTable(
-          dtypes.int64,
-          dtypes.int64,
-          default_value=default_value,
-          empty_key=empty_key,
-          deleted_key=deleted_key,
-          name="t1",
-          checkpoint=True,
-          initial_num_buckets=32)
-
-      save = saver.Saver()
-
-      self.assertAllEqual(0, table.size().eval())
-      table.insert(keys, values).run()
-      self.assertAllEqual(4, table.size().eval())
-      self.assertAllEqual(32, len(table.export()[0].eval()))
-
-      keys2 = constant_op.constant([12, 15], dtypes.int64)
-      table.remove(keys2).run()
-      self.assertAllEqual(3, table.size().eval())
-      self.assertAllEqual(32, len(table.export()[0].eval()))
-
-      val = save.save(sess, save_path)
-      self.assertTrue(isinstance(val, six.string_types))
-      self.assertEqual(save_path, val)
-
-    with self.session(graph=ops.Graph()) as sess:
-      table = lookup.MutableDenseHashTable(
-          dtypes.int64,
-          dtypes.int64,
-          default_value=default_value,
-          empty_key=empty_key,
-          deleted_key=deleted_key,
-          name="t1",
-          checkpoint=True,
-          initial_num_buckets=64)
-      table.insert(
-          constant_op.constant([11, 14], dtypes.int64),
-          constant_op.constant([12, 24], dtypes.int64)).run()
-      self.assertAllEqual(2, table.size().eval())
-      self.assertAllEqual(64, len(table.export()[0].eval()))
-
-      save = saver.Saver()
-
-      # Restore the saved values in the parameter nodes.
-      save.restore(sess, save_path)
-
-      self.assertAllEqual(3, table.size().eval())
-      self.assertAllEqual(32, len(table.export()[0].eval()))
-
-      input_string = constant_op.constant([10, 11, 12, 13, 14], dtypes.int64)
-      output = table.lookup(input_string)
-      self.assertAllEqual([-1, 0, -1, 2, 3], output.eval())
-
-  @test_util.run_in_graph_and_eager_modes
-  def testObjectSaveRestore(self):
-    save_dir = os.path.join(self.get_temp_dir(), "save_restore")
-    save_prefix = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash")
-
-    default_value = -1
-    empty_key = 0
-    deleted_key = -1
-    keys = constant_op.constant([11, 12, 13], dtypes.int64)
-    values = constant_op.constant([0, 1, 2], dtypes.int64)
-    save_table = lookup.MutableDenseHashTable(
-        dtypes.int64,
-        dtypes.int64,
-        default_value=default_value,
-        empty_key=empty_key,
-        deleted_key=deleted_key,
-        name="t1",
-        checkpoint=True,
-        initial_num_buckets=32)
-
-    save_checkpoint = checkpointable.Checkpoint(table=save_table)
-
-    self.assertAllEqual(0, self.evaluate(save_table.size()))
-    self.evaluate(save_table.insert(keys, values))
-    self.assertAllEqual(3, self.evaluate(save_table.size()))
-    self.assertAllEqual(32, len(self.evaluate(save_table.export()[0])))
-
-    save_path = save_checkpoint.save(save_prefix)
-    del save_table, save_checkpoint
-
-    load_table = lookup.MutableDenseHashTable(
-        dtypes.int64,
-        dtypes.int64,
-        default_value=default_value,
-        empty_key=empty_key,
-        deleted_key=deleted_key,
-        name="t1",
-        checkpoint=True,
-        initial_num_buckets=64)
-    self.evaluate(load_table.insert(
-        constant_op.constant([11, 14], dtypes.int64),
-        constant_op.constant([12, 24], dtypes.int64)))
-    self.assertAllEqual(2, self.evaluate(load_table.size()))
-    self.assertAllEqual(64, len(self.evaluate(load_table.export()[0])))
-
-    restore_checkpoint = checkpointable.Checkpoint(table=load_table)
-
-    # Restore the saved values in the parameter nodes.
-    restore_checkpoint.restore(save_path).run_restore_ops()
-
-    self.assertAllEqual(3, self.evaluate(load_table.size()))
-    self.assertAllEqual(32, len(self.evaluate(load_table.export()[0])))
-
-    input_string = constant_op.constant([10, 11, 12, 13, 14], dtypes.int64)
-    output = load_table.lookup(input_string)
-    self.assertAllEqual([-1, 0, 1, 2, -1], self.evaluate(output))
-
-  def testVectorSaveRestore(self):
-    save_dir = os.path.join(self.get_temp_dir(), "vector_save_restore")
-    save_path = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash")
-
-    with self.session(graph=ops.Graph()) as sess:
-      empty_key = constant_op.constant([11, 13], dtypes.int64)
-      deleted_key = constant_op.constant([-2, -3], dtypes.int64)
-      default_value = constant_op.constant([-1, -2], dtypes.int64)
-      keys = constant_op.constant([[11, 12], [11, 14], [12, 13], [13, 14]],
-                                  dtypes.int64)
-      values = constant_op.constant([[0, 1], [2, 3], [2, 4], [4, 5]],
-                                    dtypes.int64)
-      table = lookup.MutableDenseHashTable(
-          dtypes.int64,
-          dtypes.int64,
-          default_value=default_value,
-          empty_key=empty_key,
-          deleted_key=deleted_key,
-          name="t1",
-          checkpoint=True,
-          initial_num_buckets=32)
-
-      save = saver.Saver()
-
-      self.assertAllEqual(0, table.size().eval())
-      table.insert(keys, values).run()
-      self.assertAllEqual(4, table.size().eval())
-      self.assertAllEqual(32, len(table.export()[0].eval()))
-
-      keys2 = constant_op.constant([[12, 13], [16, 17]], dtypes.int64)
-      table.remove(keys2).run()
-      self.assertAllEqual(3, table.size().eval())
-      self.assertAllEqual(32, len(table.export()[0].eval()))
-
-      val = save.save(sess, save_path)
-      self.assertTrue(isinstance(val, six.string_types))
-      self.assertEqual(save_path, val)
-
-    with self.session(graph=ops.Graph()) as sess:
-      empty_key = constant_op.constant([11, 13], dtypes.int64)
-      deleted_key = constant_op.constant([-2, -3], dtypes.int64)
-      default_value = constant_op.constant([-1, -2], dtypes.int64)
-      table = lookup.MutableDenseHashTable(
-          dtypes.int64,
-          dtypes.int64,
-          default_value=default_value,
-          empty_key=empty_key,
-          deleted_key=deleted_key,
-          name="t1",
-          checkpoint=True,
-          initial_num_buckets=64)
-      table.insert(
-          constant_op.constant([[11, 12], [13, 15]], dtypes.int64),
-          constant_op.constant([[21, 22], [23, 24]], dtypes.int64)).run()
-      self.assertAllEqual(2, table.size().eval())
-      self.assertAllEqual(64, len(table.export()[0].eval()))
-
-      save = saver.Saver()
-
-      # Restore the saved values in the parameter nodes.
-      save.restore(sess, save_path)
-
-      self.assertAllEqual(3, table.size().eval())
-      self.assertAllEqual(32, len(table.export()[0].eval()))
-
-      input_string = constant_op.constant(
-          [[11, 12], [11, 14], [11, 15], [13, 14], [13, 15]], dtypes.int64)
-      output = table.lookup(input_string)
-      self.assertAllEqual([[0, 1], [2, 3], [-1, -2], [4, 5], [-1, -2]],
-                          output.eval())
-
-  def testVectorScalarSaveRestore(self):
-    save_dir = os.path.join(self.get_temp_dir(), "vector_scalar_save_restore")
-    save_path = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash")
-
-    with self.session(graph=ops.Graph()) as sess:
-      empty_key = constant_op.constant([11, 13], dtypes.int64)
-      deleted_key = constant_op.constant([-1, -1], dtypes.int64)
-      default_value = constant_op.constant(-1, dtypes.int64)
-      keys = constant_op.constant([[11, 12], [11, 14], [12, 13], [13, 14]],
-                                  dtypes.int64)
-      values = constant_op.constant([0, 1, 2, 3], dtypes.int64)
-      table = lookup.MutableDenseHashTable(
-          dtypes.int64,
-          dtypes.int64,
-          default_value=default_value,
-          empty_key=empty_key,
-          deleted_key=deleted_key,
-          name="t2",
-          checkpoint=True,
-          initial_num_buckets=32)
-
-      save = saver.Saver()
-
-      self.assertAllEqual(0, table.size().eval())
-      table.insert(keys, values).run()
-      self.assertAllEqual(4, table.size().eval())
-      self.assertAllEqual(32, len(table.export()[0].eval()))
-
-      keys2 = constant_op.constant([[12, 13], [15, 16]], dtypes.int64)
-      table.remove(keys2).run()
-      self.assertAllEqual(3, table.size().eval())
-      self.assertAllEqual(32, len(table.export()[0].eval()))
-
-      val = save.save(sess, save_path)
-      self.assertTrue(isinstance(val, six.string_types))
-      self.assertEqual(save_path, val)
-
-    with self.session(graph=ops.Graph()) as sess:
-      empty_key = constant_op.constant([11, 13], dtypes.int64)
-      deleted_key = constant_op.constant([-1, -1], dtypes.int64)
-      default_value = constant_op.constant(-1, dtypes.int64)
-      table = lookup.MutableDenseHashTable(
-          dtypes.int64,
-          dtypes.int64,
-          default_value=default_value,
-          empty_key=empty_key,
-          deleted_key=deleted_key,
-          name="t2",
-          checkpoint=True,
-          initial_num_buckets=64)
-      table.insert(
-          constant_op.constant([[11, 12], [13, 15]], dtypes.int64),
-          constant_op.constant([3, 4], dtypes.int64)).run()
-      self.assertAllEqual(2, table.size().eval())
-      self.assertAllEqual(64, len(table.export()[0].eval()))
-
-      save = saver.Saver()
-
-      # Restore the saved values in the parameter nodes.
-      save.restore(sess, save_path)
-
-      self.assertAllEqual(3, table.size().eval())
-      self.assertAllEqual(32, len(table.export()[0].eval()))
-
-      input_string = constant_op.constant(
-          [[11, 12], [11, 14], [11, 15], [13, 14], [13, 15]], dtypes.int64)
-      output = table.lookup(input_string)
-      self.assertAllEqual([0, 1, -1, 3, -1], output.eval())
-
-  def testReprobe(self):
-    with self.cached_session():
-      # Insert 6 keys into a table with 8 buckets.
-      # The values are chosen to make sure collisions occur when using GCC STL
-      keys = constant_op.constant([11, 12, 13, 19, 20, 21], dtypes.int64)
-      values = constant_op.constant([51, 52, 53, 54, 55, 56], dtypes.int64)
-      table = lookup.MutableDenseHashTable(
-          dtypes.int64,
-          dtypes.int64,
-          default_value=-1,
-          empty_key=0,
-          deleted_key=-1,
-          initial_num_buckets=8)
-      self.assertAllEqual(0, table.size().eval())
-
-      table.insert(keys, values).run()
-      self.assertAllEqual(6, table.size().eval())
-
-      input_string = constant_op.constant([10, 11, 12, 13, 14, 19, 20, 21, 22],
-                                          dtypes.int64)
-      output = table.lookup(input_string)
-      self.assertAllEqual([9], output.get_shape())
-
-      result = output.eval()
-      self.assertAllEqual([-1, 51, 52, 53, -1, 54, 55, 56, -1], result)
-
-  def testCustomEmptyKey(self):
-    with self.cached_session():
-      keys = constant_op.constant([11, 0, 13], dtypes.int64)
-      values = constant_op.constant([0, 1, 2], dtypes.int64)
-      table = lookup.MutableDenseHashTable(
-          dtypes.int64,
-          dtypes.int64,
-          default_value=-1,
-          empty_key=12,
-          deleted_key=-1)
-      self.assertAllEqual(0, table.size().eval())
-
-      table.insert(keys, values).run()
-      self.assertAllEqual(3, table.size().eval())
-
-      input_string = constant_op.constant([11, 0, 15], dtypes.int64)
-      output = table.lookup(input_string)
-      self.assertAllEqual([3], output.get_shape())
-
-      result = output.eval()
-      self.assertAllEqual([0, 1, -1], result)
-
-  def testErrors(self):
-    with self.cached_session():
-      table = lookup.MutableDenseHashTable(
-          dtypes.int64,
-          dtypes.int64,
-          default_value=-1,
-          empty_key=0,
-          deleted_key=-1)
-
-      # Inserting the empty key returns an error
-      keys1 = constant_op.constant([11, 0], dtypes.int64)
-      values1 = constant_op.constant([0, 1], dtypes.int64)
-      with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
-                                   "empty_key"):
-        table.insert(keys1, values1).run()
-
-      # Looking up the empty key returns an error
-      with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
-                                   "empty_key"):
-        table.lookup(keys1).eval()
-
-      # Inserting the deleted key returns an error
-      keys2 = constant_op.constant([11, -1], dtypes.int64)
-      values2 = constant_op.constant([0, 1], dtypes.int64)
-      with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
-                                   "deleted_key"):
-        table.insert(keys2, values2).run()
-
-      # Looking up the empty key returns an error
-      with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
-                                   "deleted_key"):
-        table.lookup(keys2).eval()
-
-      # Arbitrary tensors of keys are not supported
-      keys = constant_op.constant([[11, 0], [12, 1]], dtypes.int64)
-      values = constant_op.constant([[11, 0], [12, 1]], dtypes.int64)
-      with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
-                                   "Expected key shape"):
-        table.lookup(keys).eval()
-      with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
-                                   "Expected key shape"):
-        table.insert(keys, values).run()
-
-      table2 = lookup.MutableDenseHashTable(
-          dtypes.int64,
-          dtypes.int64,
-          default_value=-1,
-          empty_key=17,
-          deleted_key=-1,
-          initial_num_buckets=12)
-      with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
-                                   "Number of buckets must be"):
-        self.assertAllEqual(0, table2.size().eval())
-
-      with self.assertRaisesRegexp(
-          errors_impl.InvalidArgumentError,
-          "Empty and deleted keys must have same shape"):
-        table3 = lookup.MutableDenseHashTable(
-            dtypes.int64,
-            dtypes.int64,
-            default_value=-1,
-            empty_key=42,
-            deleted_key=[1, 2])
-        self.assertAllEqual(0, table3.size().eval())
-
-      with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
-                                   "Empty and deleted keys cannot be equal"):
-        table4 = lookup.MutableDenseHashTable(
-            dtypes.int64,
-            dtypes.int64,
-            default_value=-1,
-            empty_key=42,
-            deleted_key=42)
-        self.assertAllEqual(0, table4.size().eval())
-
-      with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
-                                   "Empty and deleted keys cannot be equal"):
-        table5 = lookup.MutableDenseHashTable(
-            dtypes.int64,
-            dtypes.int64,
-            default_value=-1,
-            empty_key=[1, 2, 3],
-            deleted_key=[1, 2, 3])
-        self.assertAllEqual(0, table5.size().eval())
-
-
 class IndexTableFromFile(test.TestCase):
 
   def _createVocabFile(self, basename, values=("brain", "salad", "surgery")):
@@ -2721,64 +1481,5 @@ class IdTableWithHashBucketsTest(test.TestCase):
             hasher_spec=lookup.StrongHashSpec([None, 2]))
 
 
-class MutableHashTableBenchmark(test.Benchmark):
-
-  def _create_table(self):
-    return lookup.MutableHashTable(dtypes.int64, dtypes.float32, 0.0)
-
-  def benchmark_single_repeated_scalar_insert_scalar(self):
-    table = self._create_table()
-    value = variables.Variable(1.0)
-    insert = table.insert(0, value)
-    size = table.size()
-    with session.Session() as sess:
-      sess.run(value.initializer)
-      self.run_op_benchmark(sess, insert, burn_iters=10, min_iters=10000)
-      assert sess.run(size) == 1
-
-  def benchmark_many_repeated_scalar_insert_scalar(self):
-    table = self._create_table()
-    c = dataset_ops.make_one_shot_iterator(counter.Counter()).get_next()
-    value = variables.Variable(1.0)
-    insert = table.insert(c, value)
-    size = table.size()
-    with session.Session() as sess:
-      sess.run(value.initializer)
-      self.run_op_benchmark(sess, insert, burn_iters=10, min_iters=10000)
-      assert sess.run(size) >= 10000
-
-  def benchmark_single_repeated_batch_32_insert_scalar(self):
-    table = self._create_table()
-    value = variables.Variable([1.0] * 32)
-    insert = table.insert(list(range(32)), value)
-    size = table.size()
-    with session.Session() as sess:
-      sess.run(value.initializer)
-      self.run_op_benchmark(sess, insert, burn_iters=10, min_iters=1000)
-      assert sess.run(size) == 32
-
-  def benchmark_many_repeated_batch_32_insert_scalar(self):
-    table = self._create_table()
-    c = dataset_ops.make_one_shot_iterator(counter.Counter()).get_next()
-    value = variables.Variable([1.0] * 32)
-    insert = table.insert(32 * c + list(range(32)), value)
-    size = table.size()
-    with session.Session() as sess:
-      sess.run(value.initializer)
-      self.run_op_benchmark(sess, insert, burn_iters=10, min_iters=1000)
-      assert sess.run(size) >= 1000*32
-
-
-class MutableDenseHashTableBenchmark(MutableHashTableBenchmark):
-
-  def _create_table(self):
-    return lookup.MutableDenseHashTable(
-        dtypes.int64,
-        dtypes.float32,
-        default_value=0.0,
-        empty_key=-1,
-        deleted_key=-2)
-
-
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/kernel_tests/lookup_ops_test.py b/tensorflow/python/kernel_tests/lookup_ops_test.py
index 8711923491..bbadeab8e0 100644
--- a/tensorflow/python/kernel_tests/lookup_ops_test.py
+++ b/tensorflow/python/kernel_tests/lookup_ops_test.py
@@ -18,9 +18,13 @@ from __future__ import division
 from __future__ import print_function
 
 import os
+import tempfile
 import numpy as np
+import six
 
 from tensorflow.python.client import session
+from tensorflow.python.data.experimental.ops import counter
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -32,7 +36,9 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import lookup_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
+from tensorflow.python.training import saver
 from tensorflow.python.training import server_lib
+from tensorflow.python.training.checkpointable import util as checkpointable
 
 
 class HashTableTest(test.TestCase):
@@ -1579,5 +1585,1302 @@ class IdTableWithHashBucketsTest(test.TestCase):
       self.assertIsNone(table.resource_handle)
 
 
+class MutableHashTableOpTest(test.TestCase):
+
+  def testMutableHashTable(self):
+    with self.cached_session():
+      default_val = -1
+      keys = constant_op.constant(["brain", "salad", "surgery", "tarkus"])
+      values = constant_op.constant([0, 1, 2, 3], dtypes.int64)
+      table = lookup_ops.MutableHashTable(dtypes.string, dtypes.int64,
+                                          default_val)
+      self.assertAllEqual(0, self.evaluate(table.size()))
+
+      self.evaluate(table.insert(keys, values))
+      self.assertAllEqual(4, self.evaluate(table.size()))
+
+      remove_string = constant_op.constant(["tarkus", "tank"])
+      self.evaluate(table.remove(remove_string))
+      self.assertAllEqual(3, self.evaluate(table.size()))
+
+      input_string = constant_op.constant(["brain", "salad", "tank"])
+      output = table.lookup(input_string)
+      self.assertAllEqual([3], output.get_shape())
+
+      result = self.evaluate(output)
+      self.assertAllEqual([0, 1, -1], result)
+
+      exported_keys, exported_values = table.export()
+
+      # exported data is in the order of the internal map, i.e. undefined
+      sorted_keys = np.sort(self.evaluate(exported_keys))
+      sorted_values = np.sort(self.evaluate(exported_values))
+      self.assertAllEqual([b"brain", b"salad", b"surgery"], sorted_keys)
+      self.assertAllEqual([0, 1, 2], sorted_values)
+
+  @test_util.run_v1_only("SaverV1")
+  def testSaveRestore(self):
+    save_dir = os.path.join(self.get_temp_dir(), "save_restore")
+    save_path = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash")
+
+    with self.session(graph=ops.Graph()) as sess:
+      v0 = variables.Variable(10.0, name="v0")
+      v1 = variables.Variable(20.0, name="v1")
+
+      default_val = -1
+      keys = constant_op.constant(["b", "c", "d"], dtypes.string)
+      values = constant_op.constant([0, 1, 2], dtypes.int64)
+      table = lookup_ops.MutableHashTable(
+          dtypes.string, dtypes.int64, default_val, name="t1", checkpoint=True)
+
+      save = saver.Saver()
+      self.evaluate(variables.global_variables_initializer())
+
+      # Check that the parameter nodes have been initialized.
+      self.assertEqual(10.0, self.evaluate(v0))
+      self.assertEqual(20.0, self.evaluate(v1))
+
+      self.assertAllEqual(0, self.evaluate(table.size()))
+      self.evaluate(table.insert(keys, values))
+      self.assertAllEqual(3, self.evaluate(table.size()))
+
+      val = save.save(sess, save_path)
+      self.assertTrue(isinstance(val, six.string_types))
+      self.assertEqual(save_path, val)
+
+    with self.session(graph=ops.Graph()) as sess:
+      v0 = variables.Variable(-1.0, name="v0")
+      v1 = variables.Variable(-1.0, name="v1")
+      default_val = -1
+      table = lookup_ops.MutableHashTable(
+          dtypes.string, dtypes.int64, default_val, name="t1", checkpoint=True)
+      self.evaluate(
+          table.insert(
+              constant_op.constant(["a", "c"], dtypes.string),
+              constant_op.constant([12, 24], dtypes.int64)))
+      self.assertAllEqual(2, self.evaluate(table.size()))
+
+      save = saver.Saver()
+
+      # Restore the saved values in the parameter nodes.
+      save.restore(sess, save_path)
+      # Check that the parameter nodes have been restored.
+      self.assertEqual(10.0, self.evaluate(v0))
+      self.assertEqual(20.0, self.evaluate(v1))
+
+      self.assertAllEqual(3, self.evaluate(table.size()))
+
+      input_string = constant_op.constant(["a", "b", "c", "d", "e"],
+                                          dtypes.string)
+      output = table.lookup(input_string)
+      self.assertAllEqual([-1, 0, 1, 2, -1], self.evaluate(output))
+
+  @test_util.run_in_graph_and_eager_modes
+  def testObjectSaveRestore(self):
+    save_dir = os.path.join(self.get_temp_dir(), "save_restore")
+    save_prefix = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash")
+
+    v0 = variables.Variable(10.0, name="v0")
+    v1 = variables.Variable(20.0, name="v1")
+
+    default_val = -1
+    keys = constant_op.constant(["b", "c", "d"], dtypes.string)
+    values = constant_op.constant([0, 1, 2], dtypes.int64)
+    table = lookup_ops.MutableHashTable(
+        dtypes.string, dtypes.int64, default_val, name="t1", checkpoint=True)
+
+    checkpoint = checkpointable.Checkpoint(table=table, v0=v0, v1=v1)
+    self.evaluate([v0.initializer, v1.initializer])
+
+    # Check that the parameter nodes have been initialized.
+    self.assertEqual(10.0, self.evaluate(v0))
+    self.assertEqual(20.0, self.evaluate(v1))
+
+    self.assertAllEqual(0, self.evaluate(table.size()))
+    self.evaluate(table.insert(keys, values))
+    self.assertAllEqual(3, self.evaluate(table.size()))
+
+    save_path = checkpoint.save(save_prefix)
+    del table, checkpoint, v0, v1
+
+    v0 = variables.Variable(-1.0, name="v0")
+    v1 = variables.Variable(-1.0, name="v1")
+    default_val = -1
+    table = lookup_ops.MutableHashTable(
+        dtypes.string, dtypes.int64, default_val, name="t1", checkpoint=True)
+    self.evaluate(
+        table.insert(
+            constant_op.constant(["a", "c"], dtypes.string),
+            constant_op.constant([12, 24], dtypes.int64)))
+    self.assertAllEqual(2, self.evaluate(table.size()))
+
+    checkpoint = checkpointable.Checkpoint(table=table, v0=v0, v1=v1)
+
+    # Restore the saved values in the parameter nodes.
+    checkpoint.restore(save_path).run_restore_ops()
+    # Check that the parameter nodes have been restored.
+    self.assertEqual(10.0, self.evaluate(v0))
+    self.assertEqual(20.0, self.evaluate(v1))
+
+    self.assertAllEqual(3, self.evaluate(table.size()))
+
+    input_string = constant_op.constant(["a", "b", "c", "d", "e"],
+                                        dtypes.string)
+    output = table.lookup(input_string)
+    self.assertAllEqual([-1, 0, 1, 2, -1], self.evaluate(output))
+
+  @test_util.run_v1_only("Multiple sessions")
+  def testSharing(self):
+    # Start a server to store the table state
+    server = server_lib.Server({"local0": ["localhost:0"]},
+                               protocol="grpc",
+                               start=True)
+    # Create two sessions sharing the same state
+    session1 = session.Session(server.target)
+    session2 = session.Session(server.target)
+
+    table = lookup_ops.MutableHashTable(
+        dtypes.int64, dtypes.string, "-", name="t1")
+
+    # Populate the table in the first session
+    with session1:
+      self.assertAllEqual(0, table.size().eval())
+
+      keys = constant_op.constant([11, 12], dtypes.int64)
+      values = constant_op.constant(["a", "b"])
+      table.insert(keys, values).run()
+      self.assertAllEqual(2, table.size().eval())
+
+      output = table.lookup(constant_op.constant([11, 12, 13], dtypes.int64))
+      self.assertAllEqual([b"a", b"b", b"-"], output.eval())
+
+    # Verify that we can access the shared data from the second session
+    with session2:
+      self.assertAllEqual(2, table.size().eval())
+
+      output = table.lookup(constant_op.constant([10, 11, 12], dtypes.int64))
+      self.assertAllEqual([b"-", b"a", b"b"], output.eval())
+
+  def testMutableHashTableOfTensors(self):
+    with self.cached_session():
+      default_val = constant_op.constant([-1, -1], dtypes.int64)
+      keys = constant_op.constant(["brain", "salad", "surgery", "tarkus"])
+      values = constant_op.constant([[0, 1], [2, 3], [4, 5], [6, 7]],
+                                    dtypes.int64)
+      table = lookup_ops.MutableHashTable(dtypes.string, dtypes.int64,
+                                          default_val)
+      self.assertAllEqual(0, self.evaluate(table.size()))
+
+      self.evaluate(table.insert(keys, values))
+      self.assertAllEqual(4, self.evaluate(table.size()))
+
+      remove_string = constant_op.constant(["tarkus", "tank"])
+      self.evaluate(table.remove(remove_string))
+      self.assertAllEqual(3, self.evaluate(table.size()))
+
+      input_string = constant_op.constant(["brain", "salad", "tank"])
+      output = table.lookup(input_string)
+      self.assertAllEqual([3, 2], output.get_shape())
+
+      result = self.evaluate(output)
+      self.assertAllEqual([[0, 1], [2, 3], [-1, -1]], result)
+
+      exported_keys, exported_values = table.export()
+      # exported data is in the order of the internal map, i.e. undefined
+      sorted_keys = np.sort(self.evaluate(exported_keys))
+      sorted_values = np.sort(self.evaluate(exported_values))
+      self.assertAllEqual([b"brain", b"salad", b"surgery"], sorted_keys)
+      self.assertAllEqual([[4, 5], [2, 3], [0, 1]], sorted_values)
+
+  def testMutableHashTableExportInsert(self):
+    with self.cached_session():
+      default_val = constant_op.constant([-1, -1], dtypes.int64)
+      keys = constant_op.constant(["brain", "salad", "surgery"])
+      values = constant_op.constant([[0, 1], [2, 3], [4, 5]], dtypes.int64)
+      table1 = lookup_ops.MutableHashTable(dtypes.string, dtypes.int64,
+                                           default_val)
+      self.assertAllEqual(0, self.evaluate(table1.size()))
+      self.evaluate(table1.insert(keys, values))
+      self.assertAllEqual(3, self.evaluate(table1.size()))
+
+      input_string = constant_op.constant(["brain", "salad", "tank"])
+      expected_output = [[0, 1], [2, 3], [-1, -1]]
+      output1 = table1.lookup(input_string)
+      self.assertAllEqual(expected_output, self.evaluate(output1))
+
+      exported_keys, exported_values = table1.export()
+      self.assertAllEqual(3, self.evaluate(exported_keys).size)
+      self.assertAllEqual(6, self.evaluate(exported_values).size)
+
+      # Populate a second table from the exported data
+      table2 = lookup_ops.MutableHashTable(dtypes.string, dtypes.int64,
+                                           default_val)
+      self.assertAllEqual(0, self.evaluate(table2.size()))
+      self.evaluate(table2.insert(exported_keys, exported_values))
+      self.assertAllEqual(3, self.evaluate(table2.size()))
+
+      # Verify lookup result is still the same
+      output2 = table2.lookup(input_string)
+      self.assertAllEqual(expected_output, self.evaluate(output2))
+
+  def testMutableHashTableOfTensorsInvalidShape(self):
+    with self.cached_session():
+      default_val = constant_op.constant([-1, -1], dtypes.int64)
+      keys = constant_op.constant(["brain", "salad", "surgery"])
+      table = lookup_ops.MutableHashTable(dtypes.string, dtypes.int64,
+                                          default_val)
+
+      # Shape [6] instead of [3, 2]
+      values = constant_op.constant([0, 1, 2, 3, 4, 5], dtypes.int64)
+      with self.assertRaisesOpError("Expected shape"):
+        self.evaluate(table.insert(keys, values))
+
+      # Shape [2,3] instead of [3, 2]
+      values = constant_op.constant([[0, 1, 2], [3, 4, 5]], dtypes.int64)
+      with self.assertRaisesOpError("Expected shape"):
+        self.evaluate(table.insert(keys, values))
+
+      # Shape [2, 2] instead of [3, 2]
+      values = constant_op.constant([[0, 1], [2, 3]], dtypes.int64)
+      with self.assertRaisesOpError("Expected shape"):
+        self.evaluate(table.insert(keys, values))
+
+      # Shape [3, 1] instead of [3, 2]
+      values = constant_op.constant([[0], [2], [4]], dtypes.int64)
+      with self.assertRaisesOpError("Expected shape"):
+        self.evaluate(table.insert(keys, values))
+
+      # Valid Insert
+      values = constant_op.constant([[0, 1], [2, 3], [4, 5]], dtypes.int64)
+      self.evaluate(table.insert(keys, values))
+      self.assertAllEqual(3, self.evaluate(table.size()))
+
+  def testMutableHashTableInvalidDefaultValue(self):
+    with self.cached_session():
+      default_val = constant_op.constant([[-1, -1]], dtypes.int64)
+      with self.assertRaisesOpError("Default value must be a vector"):
+        table = lookup_ops.MutableHashTable(dtypes.string, dtypes.int64,
+                                            default_val)
+        self.assertAllEqual(0, self.evaluate(table.size()))
+
+  def testMutableHashTableDuplicateInsert(self):
+    with self.cached_session():
+      default_val = -1
+      keys = constant_op.constant(["brain", "salad", "surgery", "brain"])
+      values = constant_op.constant([0, 1, 2, 3], dtypes.int64)
+      table = lookup_ops.MutableHashTable(dtypes.string, dtypes.int64,
+                                          default_val)
+      self.assertAllEqual(0, self.evaluate(table.size()))
+
+      self.evaluate(table.insert(keys, values))
+      self.assertAllEqual(3, self.evaluate(table.size()))
+
+      input_string = constant_op.constant(["brain", "salad", "tank"])
+      output = table.lookup(input_string)
+
+      result = self.evaluate(output)
+      self.assertAllEqual([3, 1, -1], result)
+
+  def testMutableHashTableFindHighRank(self):
+    with self.cached_session():
+      default_val = -1
+      keys = constant_op.constant(["brain", "salad", "surgery"])
+      values = constant_op.constant([0, 1, 2], dtypes.int64)
+      table = lookup_ops.MutableHashTable(dtypes.string, dtypes.int64,
+                                          default_val)
+
+      self.evaluate(table.insert(keys, values))
+      self.assertAllEqual(3, self.evaluate(table.size()))
+
+      input_string = constant_op.constant([["brain", "salad"],
+                                           ["tank", "tarkus"]])
+      output = table.lookup(input_string)
+      self.assertAllEqual([2, 2], output.get_shape())
+
+      result = self.evaluate(output)
+      self.assertAllEqual([[0, 1], [-1, -1]], result)
+
+  def testMutableHashTableInsertHighRank(self):
+    with self.cached_session():
+      default_val = -1
+      keys = constant_op.constant([["brain", "salad"], ["surgery", "tank"]])
+      values = constant_op.constant([[0, 1], [2, 3]], dtypes.int64)
+      table = lookup_ops.MutableHashTable(dtypes.string, dtypes.int64,
+                                          default_val)
+
+      self.evaluate(table.insert(keys, values))
+      self.assertAllEqual(4, self.evaluate(table.size()))
+
+      input_string = constant_op.constant(["brain", "salad", "tank", "tarkus"])
+      output = table.lookup(input_string)
+
+      result = self.evaluate(output)
+      self.assertAllEqual([0, 1, 3, -1], result)
+
+  def testMutableHashTableRemoveHighRank(self):
+    with self.test_session():
+      default_val = -1
+      keys = constant_op.constant([["brain", "salad"], ["surgery", "tank"]])
+      values = constant_op.constant([[0, 1], [2, 3]], dtypes.int64)
+      table = lookup_ops.MutableHashTable(dtypes.string, dtypes.int64,
+                                          default_val)
+
+      self.evaluate(table.insert(keys, values))
+      self.assertAllEqual(4, self.evaluate(table.size()))
+
+      remove_string = constant_op.constant(["salad", "tarkus"])
+      self.evaluate(table.remove(remove_string))
+      self.assertAllEqual(3, self.evaluate(table.size()))
+
+      input_string = constant_op.constant(["brain", "salad", "tank", "tarkus"])
+      output = table.lookup(input_string)
+
+      result = self.evaluate(output)
+      self.assertAllEqual([0, -1, 3, -1], result)
+
+  def testMutableHashTableOfTensorsFindHighRank(self):
+    with self.cached_session():
+      default_val = constant_op.constant([-1, -1, -1], dtypes.int64)
+      keys = constant_op.constant(["brain", "salad", "surgery"])
+      values = constant_op.constant([[0, 1, 2], [2, 3, 4], [4, 5, 6]],
+                                    dtypes.int64)
+      table = lookup_ops.MutableHashTable(dtypes.string, dtypes.int64,
+                                          default_val)
+
+      self.evaluate(table.insert(keys, values))
+      self.assertAllEqual(3, self.evaluate(table.size()))
+
+      input_string = constant_op.constant([["brain", "salad"],
+                                           ["tank", "tarkus"]])
+      output = table.lookup(input_string)
+      self.assertAllEqual([2, 2, 3], output.get_shape())
+
+      result = self.evaluate(output)
+      self.assertAllEqual(
+          [[[0, 1, 2], [2, 3, 4]], [[-1, -1, -1], [-1, -1, -1]]], result)
+
+  def testMutableHashTableOfTensorsRemoveHighRank(self):
+    with self.test_session():
+      default_val = constant_op.constant([-1, -1, -1], dtypes.int64)
+      keys = constant_op.constant(["brain", "salad", "surgery"])
+      values = constant_op.constant([[0, 1, 2], [2, 3, 4], [4, 5, 6]],
+                                    dtypes.int64)
+      table = lookup_ops.MutableHashTable(dtypes.string, dtypes.int64,
+                                          default_val)
+
+      self.evaluate(table.insert(keys, values))
+      self.assertAllEqual(3, self.evaluate(table.size()))
+
+      remove_string = constant_op.constant([["brain", "tank"]])
+      self.evaluate(table.remove(remove_string))
+      self.assertAllEqual(2, self.evaluate(table.size()))
+
+      input_string = constant_op.constant([["brain", "salad"],
+                                           ["surgery", "tank"]])
+      output = table.lookup(input_string)
+      self.assertAllEqual([2, 2, 3], output.get_shape())
+
+      result = self.evaluate(output)
+      self.assertAllEqual(
+          [[[-1, -1, -1], [2, 3, 4]], [[4, 5, 6], [-1, -1, -1]]], result)
+
+  def testMultipleMutableHashTables(self):
+    with self.cached_session():
+      default_val = -1
+      keys = constant_op.constant(["brain", "salad", "surgery"])
+      values = constant_op.constant([0, 1, 2], dtypes.int64)
+
+      table1 = lookup_ops.MutableHashTable(dtypes.string, dtypes.int64,
+                                           default_val)
+      table2 = lookup_ops.MutableHashTable(dtypes.string, dtypes.int64,
+                                           default_val)
+      table3 = lookup_ops.MutableHashTable(dtypes.string, dtypes.int64,
+                                           default_val)
+      self.evaluate(table1.insert(keys, values))
+      self.evaluate(table2.insert(keys, values))
+      self.evaluate(table3.insert(keys, values))
+
+      self.assertAllEqual(3, self.evaluate(table1.size()))
+      self.assertAllEqual(3, self.evaluate(table2.size()))
+      self.assertAllEqual(3, self.evaluate(table3.size()))
+
+      input_string = constant_op.constant(["brain", "salad", "tank"])
+      output1 = table1.lookup(input_string)
+      output2 = table2.lookup(input_string)
+      output3 = table3.lookup(input_string)
+
+      out1, out2, out3 = self.evaluate([output1, output2, output3])
+      self.assertAllEqual([0, 1, -1], out1)
+      self.assertAllEqual([0, 1, -1], out2)
+      self.assertAllEqual([0, 1, -1], out3)
+
+  def testMutableHashTableWithTensorDefault(self):
+    with self.cached_session():
+      default_val = constant_op.constant(-1, dtypes.int64)
+      keys = constant_op.constant(["brain", "salad", "surgery"])
+      values = constant_op.constant([0, 1, 2], dtypes.int64)
+      table = lookup_ops.MutableHashTable(dtypes.string, dtypes.int64,
+                                          default_val)
+
+      self.evaluate(table.insert(keys, values))
+      self.assertAllEqual(3, self.evaluate(table.size()))
+
+      input_string = constant_op.constant(["brain", "salad", "tank"])
+      output = table.lookup(input_string)
+
+      result = self.evaluate(output)
+      self.assertAllEqual([0, 1, -1], result)
+
+  def testSignatureMismatch(self):
+    with self.cached_session():
+      default_val = -1
+      keys = constant_op.constant(["brain", "salad", "surgery"])
+      values = constant_op.constant([0, 1, 2], dtypes.int64)
+      table = lookup_ops.MutableHashTable(dtypes.string, dtypes.int64,
+                                          default_val)
+
+      # insert with keys of the wrong type
+      with self.assertRaises(ValueError):
+        self.evaluate(table.insert(constant_op.constant([4, 5, 6]), values))
+
+      # insert with values of the wrong type
+      with self.assertRaises(ValueError):
+        self.evaluate(table.insert(keys, constant_op.constant(["a", "b", "c"])))
+
+      self.assertAllEqual(0, self.evaluate(table.size()))
+
+      self.evaluate(table.insert(keys, values))
+      self.assertAllEqual(3, self.evaluate(table.size()))
+
+      input_string_ref = variables.Variable("brain")
+      input_int64_ref = variables.Variable(-1, dtype=dtypes.int64)
+      self.evaluate(variables.global_variables_initializer())
+
+      # Ref types do not produce an insert signature mismatch.
+      self.evaluate(table.insert(input_string_ref, input_int64_ref))
+      self.assertAllEqual(3, self.evaluate(table.size()))
+
+      # Ref types do not produce a lookup signature mismatch.
+      self.assertEqual(-1, self.evaluate(table.lookup(input_string_ref)))
+
+      # lookup with keys of the wrong type
+      input_string = constant_op.constant([1, 2, 3], dtypes.int64)
+      with self.assertRaises(ValueError):
+        self.evaluate(table.lookup(input_string))
+
+      # default value of the wrong type
+      with self.assertRaises(TypeError):
+        lookup_ops.MutableHashTable(dtypes.string, dtypes.int64, "UNK")
+
+  def testMutableHashTableStringFloat(self):
+    with self.cached_session():
+      default_val = -1.5
+      keys = constant_op.constant(["brain", "salad", "surgery"])
+      values = constant_op.constant([0, 1.1, 2.2], dtypes.float32)
+      table = lookup_ops.MutableHashTable(dtypes.string, dtypes.float32,
+                                          default_val)
+      self.assertAllEqual(0, self.evaluate(table.size()))
+
+      self.evaluate(table.insert(keys, values))
+      self.assertAllEqual(3, self.evaluate(table.size()))
+
+      input_string = constant_op.constant(["brain", "salad", "tank"])
+      output = table.lookup(input_string)
+
+      result = self.evaluate(output)
+      self.assertAllClose([0, 1.1, default_val], result)
+
+  def testMutableHashTableIntFloat(self):
+    with self.cached_session():
+      default_val = -1.0
+      keys = constant_op.constant([3, 7, 0], dtypes.int64)
+      values = constant_op.constant([7.5, -1.2, 9.9], dtypes.float32)
+      table = lookup_ops.MutableHashTable(dtypes.int64, dtypes.float32,
+                                          default_val)
+      self.assertAllEqual(0, self.evaluate(table.size()))
+
+      self.evaluate(table.insert(keys, values))
+      self.assertAllEqual(3, self.evaluate(table.size()))
+
+      input_string = constant_op.constant([7, 0, 11], dtypes.int64)
+      output = table.lookup(input_string)
+
+      result = self.evaluate(output)
+      self.assertAllClose([-1.2, 9.9, default_val], result)
+
+  def testMutableHashTableInt64String(self):
+    with self.cached_session():
+      default_val = "n/a"
+      keys = constant_op.constant([0, 1, 2], dtypes.int64)
+      values = constant_op.constant(["brain", "salad", "surgery"])
+      table = lookup_ops.MutableHashTable(dtypes.int64, dtypes.string,
+                                          default_val)
+      self.assertAllEqual(0, self.evaluate(table.size()))
+
+      self.evaluate(table.insert(keys, values))
+      self.assertAllEqual(3, self.evaluate(table.size()))
+
+      input_string = constant_op.constant([0, 1, 3], dtypes.int64)
+      output = table.lookup(input_string)
+
+      result = self.evaluate(output)
+      self.assertAllEqual((b"brain", b"salad", b"n/a"), result)
+
+
+class MutableDenseHashTableOpTest(test.TestCase):
+
+  def testBasic(self):
+    with self.cached_session():
+
+      keys = constant_op.constant([11, 12, 13, 14], dtypes.int64)
+      values = constant_op.constant([0, 1, 2, 3], dtypes.int64)
+      table = lookup_ops.MutableDenseHashTable(
+          dtypes.int64,
+          dtypes.int64,
+          default_value=-1,
+          empty_key=0,
+          deleted_key=-1)
+      self.assertAllEqual(0, self.evaluate(table.size()))
+
+      self.evaluate(table.insert(keys, values))
+      self.assertAllEqual(4, self.evaluate(table.size()))
+
+      remove_string = constant_op.constant([12, 15], dtypes.int64)
+      self.evaluate(table.remove(remove_string))
+      self.assertAllEqual(3, self.evaluate(table.size()))
+
+      input_string = constant_op.constant([11, 12, 15], dtypes.int64)
+      output = table.lookup(input_string)
+      self.assertAllEqual([3], output.get_shape())
+
+      result = self.evaluate(output)
+      self.assertAllEqual([0, -1, -1], result)
+
+  def testBasicBool(self):
+    with self.cached_session():
+
+      keys = constant_op.constant([11, 12, 13, 14], dtypes.int64)
+      values = constant_op.constant([True, True, True, True], dtypes.bool)
+      table = lookup_ops.MutableDenseHashTable(
+          dtypes.int64,
+          dtypes.bool,
+          default_value=False,
+          empty_key=0,
+          deleted_key=-1)
+      self.assertAllEqual(0, self.evaluate(table.size()))
+
+      self.evaluate(table.insert(keys, values))
+      self.assertAllEqual(4, self.evaluate(table.size()))
+
+      remove_string = constant_op.constant([11, 15], dtypes.int64)
+      self.evaluate(table.remove(remove_string))
+      self.assertAllEqual(3, self.evaluate(table.size()))
+
+      input_string = constant_op.constant([11, 12, 15], dtypes.int64)
+      output = table.lookup(input_string)
+      self.assertAllEqual([3], output.get_shape())
+
+      result = self.evaluate(output)
+      self.assertAllEqual([False, True, False], result)
+
+  def testSameEmptyAndDeletedKey(self):
+    with self.cached_session():
+      with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
+                                   "Empty and deleted keys"):
+        table = lookup_ops.MutableDenseHashTable(
+            dtypes.int64,
+            dtypes.int64,
+            default_value=-1,
+            empty_key=42,
+            deleted_key=42)
+        self.assertAllEqual(0, self.evaluate(table.size()))
+
+  @test_util.run_v1_only("uses placeholders")
+  def testLookupUnknownShape(self):
+    with self.cached_session():
+      keys = constant_op.constant([11, 12, 13], dtypes.int64)
+      values = constant_op.constant([0, 1, 2], dtypes.int64)
+      table = lookup_ops.MutableDenseHashTable(
+          dtypes.int64,
+          dtypes.int64,
+          default_value=-1,
+          empty_key=0,
+          deleted_key=-1)
+
+      self.evaluate(table.insert(keys, values))
+      self.assertAllEqual(3, self.evaluate(table.size()))
+
+      placeholder_keys = array_ops.placeholder(dtypes.int64)
+      output = table.lookup(placeholder_keys)
+      self.assertAllEqual(None, output.get_shape())
+      result = output.eval({placeholder_keys: [11, 12, 15]})
+      self.assertAllEqual([0, 1, -1], result)
+
+  def testMapStringToFloat(self):
+    with self.cached_session():
+
+      keys = constant_op.constant(["a", "b", "c", "d"], dtypes.string)
+      values = constant_op.constant([0.0, 1.1, 2.2, 3.3], dtypes.float32)
+      default_value = constant_op.constant(-1.5, dtypes.float32)
+      table = lookup_ops.MutableDenseHashTable(
+          dtypes.string,
+          dtypes.float32,
+          default_value=default_value,
+          empty_key="",
+          deleted_key="$")
+      self.assertAllEqual(0, self.evaluate(table.size()))
+
+      self.evaluate(table.insert(keys, values))
+      self.assertAllEqual(4, self.evaluate(table.size()))
+
+      remove_string = constant_op.constant(["b", "e"])
+      self.evaluate(table.remove(remove_string))
+      self.assertAllEqual(3, self.evaluate(table.size()))
+
+      input_string = constant_op.constant(["a", "b", "d", "e"], dtypes.string)
+      output = table.lookup(input_string)
+      self.assertAllEqual([4], output.get_shape())
+
+      result = self.evaluate(output)
+      self.assertAllClose([0, -1.5, 3.3, -1.5], result)
+
+  def testMapInt64ToFloat(self):
+    for float_dtype in [dtypes.float32, dtypes.float64]:
+      with self.cached_session():
+
+        keys = constant_op.constant([11, 12, 13, 14], dtypes.int64)
+        values = constant_op.constant([0.0, 1.1, 2.2, 3.3], float_dtype)
+        default_value = constant_op.constant(-1.5, float_dtype)
+        table = lookup_ops.MutableDenseHashTable(
+            dtypes.int64,
+            float_dtype,
+            default_value=default_value,
+            empty_key=0,
+            deleted_key=-1)
+        self.assertAllEqual(0, self.evaluate(table.size()))
+
+        self.evaluate(table.insert(keys, values))
+        self.assertAllEqual(4, self.evaluate(table.size()))
+
+        remove_string = constant_op.constant([12, 15], dtypes.int64)
+        self.evaluate(table.remove(remove_string))
+        self.assertAllEqual(3, self.evaluate(table.size()))
+
+        input_string = constant_op.constant([11, 12, 14, 15], dtypes.int64)
+        output = table.lookup(input_string)
+        self.assertAllEqual([4], output.get_shape())
+
+        result = self.evaluate(output)
+        self.assertAllClose([0, -1.5, 3.3, -1.5], result)
+
+  def testVectorValues(self):
+    with self.cached_session():
+      keys = constant_op.constant([11, 12, 13], dtypes.int64)
+      values = constant_op.constant([[0, 1, 2, 3], [3, 4, 5, 6], [6, 7, 8, 9]],
+                                    dtypes.int64)
+      default_value = constant_op.constant([-1, -2, -3, -4], dtypes.int64)
+      table = lookup_ops.MutableDenseHashTable(
+          dtypes.int64,
+          dtypes.int64,
+          default_value=default_value,
+          empty_key=0,
+          deleted_key=-1,
+          initial_num_buckets=4)
+      self.assertAllEqual(0, self.evaluate(table.size()))
+
+      self.evaluate(table.insert(keys, values))
+      self.assertAllEqual(3, self.evaluate(table.size()))
+      self.assertAllEqual(4, len(self.evaluate(table.export()[0])))
+
+      self.evaluate(
+          table.insert(
+              constant_op.constant([14], dtypes.int64),
+              constant_op.constant([[2, 3, 4, 5]], dtypes.int64)))
+      self.assertAllEqual(4, self.evaluate(table.size()))
+      self.assertAllEqual(8, len(self.evaluate(table.export()[0])))
+
+      remove_string = constant_op.constant([12, 16], dtypes.int64)
+      self.evaluate(table.remove(remove_string))
+      self.assertAllEqual(3, self.evaluate(table.size()))
+      self.assertAllEqual(8, len(self.evaluate(table.export()[0])))
+
+      input_string = constant_op.constant([11, 12, 14, 15], dtypes.int64)
+      output = table.lookup(input_string)
+      self.assertAllEqual([4, 4],
+                          output.shape,
+                          msg="Saw shape: %s" % output.shape)
+
+      result = self.evaluate(output)
+      self.assertAllEqual(
+          [[0, 1, 2, 3], [-1, -2, -3, -4], [2, 3, 4, 5], [-1, -2, -3, -4]],
+          result)
+
+  def testVectorKeys(self):
+    with self.cached_session():
+      keys = constant_op.constant([[0, 1], [1, 2], [1, 3]], dtypes.int64)
+      values = constant_op.constant([10, 11, 12], dtypes.int64)
+      empty_key = constant_op.constant([0, 3], dtypes.int64)
+      deleted_key = constant_op.constant([-1, -1], dtypes.int64)
+      default_value = constant_op.constant(-1, dtypes.int64)
+      table = lookup_ops.MutableDenseHashTable(
+          dtypes.int64,
+          dtypes.int64,
+          default_value=default_value,
+          empty_key=empty_key,
+          deleted_key=deleted_key,
+          initial_num_buckets=8)
+      self.assertAllEqual(0, self.evaluate(table.size()))
+
+      self.evaluate(table.insert(keys, values))
+      self.assertAllEqual(3, self.evaluate(table.size()))
+
+      self.evaluate(
+          table.insert(
+              constant_op.constant([[0, 0]], dtypes.int64),
+              constant_op.constant([13], dtypes.int64)))
+      self.assertAllEqual(4, self.evaluate(table.size()))
+      self.assertAllEqual(8, len(self.evaluate(table.export()[0])))
+
+      remove_string = constant_op.constant([[1, 2], [7, 8]], dtypes.int64)
+      self.evaluate(table.remove(remove_string))
+      self.assertAllEqual(3, self.evaluate(table.size()))
+      self.assertAllEqual(8, len(self.evaluate(table.export()[0])))
+
+      input_string = constant_op.constant([[0, 1], [1, 2], [1, 3], [0, 2]],
+                                          dtypes.int64)
+      output = table.lookup(input_string)
+      self.assertAllEqual([4], output.get_shape())
+
+      result = self.evaluate(output)
+      self.assertAllEqual([10, -1, 12, -1], result)
+
+  def testResize(self):
+    with self.cached_session():
+      keys = constant_op.constant([11, 12, 13], dtypes.int64)
+      values = constant_op.constant([0, 1, 2], dtypes.int64)
+      table = lookup_ops.MutableDenseHashTable(
+          dtypes.int64,
+          dtypes.int64,
+          default_value=-1,
+          empty_key=0,
+          deleted_key=-1,
+          initial_num_buckets=4)
+      self.assertAllEqual(0, self.evaluate(table.size()))
+
+      self.evaluate(table.insert(keys, values))
+      self.assertAllEqual(3, self.evaluate(table.size()))
+      self.assertAllEqual(4, len(self.evaluate(table.export()[0])))
+
+      keys2 = constant_op.constant([12, 99], dtypes.int64)
+      self.evaluate(table.remove(keys2))
+      self.assertAllEqual(2, self.evaluate(table.size()))
+      self.assertAllEqual(4, len(self.evaluate(table.export()[0])))
+
+      keys3 = constant_op.constant([13, 14, 15, 16, 17], dtypes.int64)
+      values3 = constant_op.constant([3, 4, 5, 6, 7], dtypes.int64)
+
+      self.evaluate(table.insert(keys3, values3))
+      self.assertAllEqual(6, self.evaluate(table.size()))
+      self.assertAllEqual(16, len(self.evaluate(table.export()[0])))
+
+      keys4 = constant_op.constant([10, 11, 12, 13, 14, 15, 16, 17, 18],
+                                   dtypes.int64)
+      output = table.lookup(keys4)
+      self.assertAllEqual([-1, 0, -1, 3, 4, 5, 6, 7, -1], self.evaluate(output))
+
+  def testExport(self):
+    with self.cached_session():
+
+      keys = constant_op.constant([11, 12, 13, 14], dtypes.int64)
+      values = constant_op.constant([1, 2, 3, 4], dtypes.int64)
+      table = lookup_ops.MutableDenseHashTable(
+          dtypes.int64,
+          dtypes.int64,
+          default_value=-1,
+          empty_key=100,
+          deleted_key=200,
+          initial_num_buckets=8)
+      self.assertAllEqual(0, self.evaluate(table.size()))
+
+      self.evaluate(table.insert(keys, values))
+      self.assertAllEqual(4, self.evaluate(table.size()))
+
+      keys2 = constant_op.constant([12, 15], dtypes.int64)
+      self.evaluate(table.remove(keys2))
+      self.assertAllEqual(3, self.evaluate(table.size()))
+
+      exported_keys, exported_values = table.export()
+
+      np_keys = self.evaluate(exported_keys)
+      np_values = self.evaluate(exported_values)
+
+      self.assertAllEqual(8, len(np_keys))
+      self.assertAllEqual(8, len(np_values))
+
+      # pair up keys and values, drop extra added dimension
+      pairs = np.dstack((np_keys.flatten(), np_values.flatten()))[0]
+      # sort by key
+      pairs = pairs[pairs[:, 0].argsort()]
+      self.assertAllEqual([[11, 1], [13, 3], [14, 4], [100, 0], [100, 0],
+                           [100, 0], [100, 0], [200, 2]], pairs)
+
+  @test_util.run_v1_only("Saver V1 only")
+  def testSaveRestore(self):
+    save_dir = os.path.join(self.get_temp_dir(), "save_restore")
+    save_path = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash")
+
+    with self.session(graph=ops.Graph()) as sess:
+      default_value = -1
+      empty_key = 0
+      deleted_key = -1
+      keys = constant_op.constant([11, 12, 13, 14], dtypes.int64)
+      values = constant_op.constant([0, 1, 2, 3], dtypes.int64)
+      table = lookup_ops.MutableDenseHashTable(
+          dtypes.int64,
+          dtypes.int64,
+          default_value=default_value,
+          empty_key=empty_key,
+          deleted_key=deleted_key,
+          name="t1",
+          checkpoint=True,
+          initial_num_buckets=32)
+
+      save = saver.Saver()
+
+      self.assertAllEqual(0, table.size().eval())
+      table.insert(keys, values).run()
+      self.assertAllEqual(4, table.size().eval())
+      self.assertAllEqual(32, len(table.export()[0].eval()))
+
+      keys2 = constant_op.constant([12, 15], dtypes.int64)
+      table.remove(keys2).run()
+      self.assertAllEqual(3, table.size().eval())
+      self.assertAllEqual(32, len(table.export()[0].eval()))
+
+      val = save.save(sess, save_path)
+      self.assertTrue(isinstance(val, six.string_types))
+      self.assertEqual(save_path, val)
+
+    with self.session(graph=ops.Graph()) as sess:
+      table = lookup_ops.MutableDenseHashTable(
+          dtypes.int64,
+          dtypes.int64,
+          default_value=default_value,
+          empty_key=empty_key,
+          deleted_key=deleted_key,
+          name="t1",
+          checkpoint=True,
+          initial_num_buckets=64)
+      table.insert(
+          constant_op.constant([11, 14], dtypes.int64),
+          constant_op.constant([12, 24], dtypes.int64)).run()
+      self.assertAllEqual(2, table.size().eval())
+      self.assertAllEqual(64, len(table.export()[0].eval()))
+
+      save = saver.Saver()
+
+      # Restore the saved values in the parameter nodes.
+      save.restore(sess, save_path)
+
+      self.assertAllEqual(3, table.size().eval())
+      self.assertAllEqual(32, len(table.export()[0].eval()))
+
+      input_string = constant_op.constant([10, 11, 12, 13, 14], dtypes.int64)
+      output = table.lookup(input_string)
+      self.assertAllEqual([-1, 0, -1, 2, 3], output.eval())
+
+  @test_util.run_in_graph_and_eager_modes
+  def testObjectSaveRestore(self):
+    save_dir = os.path.join(self.get_temp_dir(), "save_restore")
+    save_prefix = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash")
+
+    default_value = -1
+    empty_key = 0
+    deleted_key = -1
+    keys = constant_op.constant([11, 12, 13], dtypes.int64)
+    values = constant_op.constant([0, 1, 2], dtypes.int64)
+    save_table = lookup_ops.MutableDenseHashTable(
+        dtypes.int64,
+        dtypes.int64,
+        default_value=default_value,
+        empty_key=empty_key,
+        deleted_key=deleted_key,
+        name="t1",
+        checkpoint=True,
+        initial_num_buckets=32)
+
+    save_checkpoint = checkpointable.Checkpoint(table=save_table)
+
+    self.assertAllEqual(0, self.evaluate(save_table.size()))
+    self.evaluate(save_table.insert(keys, values))
+    self.assertAllEqual(3, self.evaluate(save_table.size()))
+    self.assertAllEqual(32, len(self.evaluate(save_table.export()[0])))
+
+    save_path = save_checkpoint.save(save_prefix)
+    del save_table, save_checkpoint
+
+    load_table = lookup_ops.MutableDenseHashTable(
+        dtypes.int64,
+        dtypes.int64,
+        default_value=default_value,
+        empty_key=empty_key,
+        deleted_key=deleted_key,
+        name="t1",
+        checkpoint=True,
+        initial_num_buckets=64)
+    self.evaluate(
+        load_table.insert(
+            constant_op.constant([11, 14], dtypes.int64),
+            constant_op.constant([12, 24], dtypes.int64)))
+    self.assertAllEqual(2, self.evaluate(load_table.size()))
+    self.assertAllEqual(64, len(self.evaluate(load_table.export()[0])))
+
+    restore_checkpoint = checkpointable.Checkpoint(table=load_table)
+
+    # Restore the saved values in the parameter nodes.
+    restore_checkpoint.restore(save_path).run_restore_ops()
+
+    self.assertAllEqual(3, self.evaluate(load_table.size()))
+    self.assertAllEqual(32, len(self.evaluate(load_table.export()[0])))
+
+    input_string = constant_op.constant([10, 11, 12, 13, 14], dtypes.int64)
+    output = load_table.lookup(input_string)
+    self.assertAllEqual([-1, 0, 1, 2, -1], self.evaluate(output))
+
+  @test_util.run_v1_only("Saver V1 only")
+  def testVectorSaveRestore(self):
+    save_dir = os.path.join(self.get_temp_dir(), "vector_save_restore")
+    save_path = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash")
+
+    with self.session(graph=ops.Graph()) as sess:
+      empty_key = constant_op.constant([11, 13], dtypes.int64)
+      deleted_key = constant_op.constant([-2, -3], dtypes.int64)
+      default_value = constant_op.constant([-1, -2], dtypes.int64)
+      keys = constant_op.constant([[11, 12], [11, 14], [12, 13], [13, 14]],
+                                  dtypes.int64)
+      values = constant_op.constant([[0, 1], [2, 3], [2, 4], [4, 5]],
+                                    dtypes.int64)
+      table = lookup_ops.MutableDenseHashTable(
+          dtypes.int64,
+          dtypes.int64,
+          default_value=default_value,
+          empty_key=empty_key,
+          deleted_key=deleted_key,
+          name="t1",
+          checkpoint=True,
+          initial_num_buckets=32)
+
+      save = saver.Saver()
+
+      self.assertAllEqual(0, table.size().eval())
+      table.insert(keys, values).run()
+      self.assertAllEqual(4, table.size().eval())
+      self.assertAllEqual(32, len(table.export()[0].eval()))
+
+      keys2 = constant_op.constant([[12, 13], [16, 17]], dtypes.int64)
+      table.remove(keys2).run()
+      self.assertAllEqual(3, table.size().eval())
+      self.assertAllEqual(32, len(table.export()[0].eval()))
+
+      val = save.save(sess, save_path)
+      self.assertTrue(isinstance(val, six.string_types))
+      self.assertEqual(save_path, val)
+
+    with self.session(graph=ops.Graph()) as sess:
+      empty_key = constant_op.constant([11, 13], dtypes.int64)
+      deleted_key = constant_op.constant([-2, -3], dtypes.int64)
+      default_value = constant_op.constant([-1, -2], dtypes.int64)
+      table = lookup_ops.MutableDenseHashTable(
+          dtypes.int64,
+          dtypes.int64,
+          default_value=default_value,
+          empty_key=empty_key,
+          deleted_key=deleted_key,
+          name="t1",
+          checkpoint=True,
+          initial_num_buckets=64)
+      table.insert(
+          constant_op.constant([[11, 12], [13, 15]], dtypes.int64),
+          constant_op.constant([[21, 22], [23, 24]], dtypes.int64)).run()
+      self.assertAllEqual(2, table.size().eval())
+      self.assertAllEqual(64, len(table.export()[0].eval()))
+
+      save = saver.Saver()
+
+      # Restore the saved values in the parameter nodes.
+      save.restore(sess, save_path)
+
+      self.assertAllEqual(3, table.size().eval())
+      self.assertAllEqual(32, len(table.export()[0].eval()))
+
+      input_string = constant_op.constant(
+          [[11, 12], [11, 14], [11, 15], [13, 14], [13, 15]], dtypes.int64)
+      output = table.lookup(input_string)
+      self.assertAllEqual([[0, 1], [2, 3], [-1, -2], [4, 5], [-1, -2]],
+                          output.eval())
+
+  @test_util.run_v1_only("Saver V1 only")
+  def testVectorScalarSaveRestore(self):
+    save_dir = os.path.join(self.get_temp_dir(), "vector_scalar_save_restore")
+    save_path = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash")
+
+    with self.session(graph=ops.Graph()) as sess:
+      empty_key = constant_op.constant([11, 13], dtypes.int64)
+      deleted_key = constant_op.constant([-1, -1], dtypes.int64)
+      default_value = constant_op.constant(-1, dtypes.int64)
+      keys = constant_op.constant([[11, 12], [11, 14], [12, 13], [13, 14]],
+                                  dtypes.int64)
+      values = constant_op.constant([0, 1, 2, 3], dtypes.int64)
+      table = lookup_ops.MutableDenseHashTable(
+          dtypes.int64,
+          dtypes.int64,
+          default_value=default_value,
+          empty_key=empty_key,
+          deleted_key=deleted_key,
+          name="t2",
+          checkpoint=True,
+          initial_num_buckets=32)
+
+      save = saver.Saver()
+
+      self.assertAllEqual(0, table.size().eval())
+      table.insert(keys, values).run()
+      self.assertAllEqual(4, table.size().eval())
+      self.assertAllEqual(32, len(table.export()[0].eval()))
+
+      keys2 = constant_op.constant([[12, 13], [15, 16]], dtypes.int64)
+      table.remove(keys2).run()
+      self.assertAllEqual(3, table.size().eval())
+      self.assertAllEqual(32, len(table.export()[0].eval()))
+
+      val = save.save(sess, save_path)
+      self.assertTrue(isinstance(val, six.string_types))
+      self.assertEqual(save_path, val)
+
+    with self.session(graph=ops.Graph()) as sess:
+      empty_key = constant_op.constant([11, 13], dtypes.int64)
+      deleted_key = constant_op.constant([-1, -1], dtypes.int64)
+      default_value = constant_op.constant(-1, dtypes.int64)
+      table = lookup_ops.MutableDenseHashTable(
+          dtypes.int64,
+          dtypes.int64,
+          default_value=default_value,
+          empty_key=empty_key,
+          deleted_key=deleted_key,
+          name="t2",
+          checkpoint=True,
+          initial_num_buckets=64)
+      table.insert(
+          constant_op.constant([[11, 12], [13, 15]], dtypes.int64),
+          constant_op.constant([3, 4], dtypes.int64)).run()
+      self.assertAllEqual(2, table.size().eval())
+      self.assertAllEqual(64, len(table.export()[0].eval()))
+
+      save = saver.Saver()
+
+      # Restore the saved values in the parameter nodes.
+      save.restore(sess, save_path)
+
+      self.assertAllEqual(3, table.size().eval())
+      self.assertAllEqual(32, len(table.export()[0].eval()))
+
+      input_string = constant_op.constant(
+          [[11, 12], [11, 14], [11, 15], [13, 14], [13, 15]], dtypes.int64)
+      output = table.lookup(input_string)
+      self.assertAllEqual([0, 1, -1, 3, -1], output.eval())
+
+  def testReprobe(self):
+    with self.cached_session():
+      # Insert 6 keys into a table with 8 buckets.
+      # The values are chosen to make sure collisions occur when using GCC STL
+      keys = constant_op.constant([11, 12, 13, 19, 20, 21], dtypes.int64)
+      values = constant_op.constant([51, 52, 53, 54, 55, 56], dtypes.int64)
+      table = lookup_ops.MutableDenseHashTable(
+          dtypes.int64,
+          dtypes.int64,
+          default_value=-1,
+          empty_key=0,
+          deleted_key=-1,
+          initial_num_buckets=8)
+      self.assertAllEqual(0, self.evaluate(table.size()))
+
+      self.evaluate(table.insert(keys, values))
+      self.assertAllEqual(6, self.evaluate(table.size()))
+
+      input_string = constant_op.constant([10, 11, 12, 13, 14, 19, 20, 21, 22],
+                                          dtypes.int64)
+      output = table.lookup(input_string)
+      self.assertAllEqual([9], output.get_shape())
+
+      result = self.evaluate(output)
+      self.assertAllEqual([-1, 51, 52, 53, -1, 54, 55, 56, -1], result)
+
+  def testCustomEmptyKey(self):
+    with self.cached_session():
+      keys = constant_op.constant([11, 0, 13], dtypes.int64)
+      values = constant_op.constant([0, 1, 2], dtypes.int64)
+      table = lookup_ops.MutableDenseHashTable(
+          dtypes.int64,
+          dtypes.int64,
+          default_value=-1,
+          empty_key=12,
+          deleted_key=-1)
+      self.assertAllEqual(0, self.evaluate(table.size()))
+
+      self.evaluate(table.insert(keys, values))
+      self.assertAllEqual(3, self.evaluate(table.size()))
+
+      input_string = constant_op.constant([11, 0, 15], dtypes.int64)
+      output = table.lookup(input_string)
+      self.assertAllEqual([3], output.get_shape())
+
+      result = self.evaluate(output)
+      self.assertAllEqual([0, 1, -1], result)
+
+  def testErrors(self):
+    with self.cached_session():
+      table = lookup_ops.MutableDenseHashTable(
+          dtypes.int64,
+          dtypes.int64,
+          default_value=-1,
+          empty_key=0,
+          deleted_key=-1)
+
+      # Inserting the empty key returns an error
+      keys1 = constant_op.constant([11, 0], dtypes.int64)
+      values1 = constant_op.constant([0, 1], dtypes.int64)
+      with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
+                                   "empty_key"):
+        self.evaluate(table.insert(keys1, values1))
+
+      # Looking up the empty key returns an error
+      with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
+                                   "empty_key"):
+        self.evaluate(table.lookup(keys1))
+
+      # Inserting the deleted key returns an error
+      keys2 = constant_op.constant([11, -1], dtypes.int64)
+      values2 = constant_op.constant([0, 1], dtypes.int64)
+      with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
+                                   "deleted_key"):
+        self.evaluate(table.insert(keys2, values2))
+
+      # Looking up the empty key returns an error
+      with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
+                                   "deleted_key"):
+        self.evaluate(table.lookup(keys2))
+
+      # Arbitrary tensors of keys are not supported
+      keys = constant_op.constant([[11, 0], [12, 1]], dtypes.int64)
+      values = constant_op.constant([[11, 0], [12, 1]], dtypes.int64)
+      with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
+                                   "Expected key shape"):
+        self.evaluate(table.lookup(keys))
+      with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
+                                   "Expected key shape"):
+        self.evaluate(table.insert(keys, values))
+
+      with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
+                                   "Number of buckets must be"):
+        table2 = lookup_ops.MutableDenseHashTable(
+            dtypes.int64,
+            dtypes.int64,
+            default_value=-1,
+            empty_key=17,
+            deleted_key=-1,
+            initial_num_buckets=12)
+        self.assertAllEqual(0, self.evaluate(table2.size()))
+
+      with self.assertRaisesRegexp(
+          errors_impl.InvalidArgumentError,
+          "Empty and deleted keys must have same shape"):
+        table3 = lookup_ops.MutableDenseHashTable(
+            dtypes.int64,
+            dtypes.int64,
+            default_value=-1,
+            empty_key=42,
+            deleted_key=[1, 2])
+        self.assertAllEqual(0, self.evaluate(table3.size()))
+
+      with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
+                                   "Empty and deleted keys cannot be equal"):
+        table4 = lookup_ops.MutableDenseHashTable(
+            dtypes.int64,
+            dtypes.int64,
+            default_value=-1,
+            empty_key=42,
+            deleted_key=42)
+        self.assertAllEqual(0, self.evaluate(table4.size()))
+
+      with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
+                                   "Empty and deleted keys cannot be equal"):
+        table5 = lookup_ops.MutableDenseHashTable(
+            dtypes.int64,
+            dtypes.int64,
+            default_value=-1,
+            empty_key=[1, 2, 3],
+            deleted_key=[1, 2, 3])
+        self.assertAllEqual(0, self.evaluate(table5.size()))
+
+
+class MutableHashTableBenchmark(test.Benchmark):
+
+  def _create_table(self):
+    return lookup_ops.MutableHashTable(dtypes.int64, dtypes.float32, 0.0)
+
+  def benchmark_single_repeated_scalar_insert_scalar(self):
+    table = self._create_table()
+    value = variables.Variable(1.0)
+    insert = table.insert(0, value)
+    size = table.size()
+    with session.Session() as sess:
+      sess.run(value.initializer)
+      self.run_op_benchmark(sess, insert, burn_iters=10, min_iters=10000)
+      assert sess.run(size) == 1
+
+  def benchmark_many_repeated_scalar_insert_scalar(self):
+    table = self._create_table()
+    c = dataset_ops.make_one_shot_iterator(counter.Counter()).get_next()
+    value = variables.Variable(1.0)
+    insert = table.insert(c, value)
+    size = table.size()
+    with session.Session() as sess:
+      sess.run(value.initializer)
+      self.run_op_benchmark(sess, insert, burn_iters=10, min_iters=10000)
+      assert sess.run(size) >= 10000
+
+  def benchmark_single_repeated_batch_32_insert_scalar(self):
+    table = self._create_table()
+    value = variables.Variable([1.0] * 32)
+    insert = table.insert(list(range(32)), value)
+    size = table.size()
+    with session.Session() as sess:
+      sess.run(value.initializer)
+      self.run_op_benchmark(sess, insert, burn_iters=10, min_iters=1000)
+      assert sess.run(size) == 32
+
+  def benchmark_many_repeated_batch_32_insert_scalar(self):
+    table = self._create_table()
+    c = dataset_ops.make_one_shot_iterator(counter.Counter()).get_next()
+    value = variables.Variable([1.0] * 32)
+    insert = table.insert(32 * c + list(range(32)), value)
+    size = table.size()
+    with session.Session() as sess:
+      sess.run(value.initializer)
+      self.run_op_benchmark(sess, insert, burn_iters=10, min_iters=1000)
+      assert sess.run(size) >= 1000 * 32
+
+
+class MutableDenseHashTableBenchmark(MutableHashTableBenchmark):
+
+  def _create_table(self):
+    return lookup_ops.MutableDenseHashTable(
+        dtypes.int64,
+        dtypes.float32,
+        default_value=0.0,
+        empty_key=-1,
+        deleted_key=-2)
+
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/ops/lookup_ops.py b/tensorflow/python/ops/lookup_ops.py
index 50e14297e3..127565d19d 100644
--- a/tensorflow/python/ops/lookup_ops.py
+++ b/tensorflow/python/ops/lookup_ops.py
@@ -41,6 +41,7 @@ from tensorflow.python.ops.gen_lookup_ops import *
 # pylint: enable=wildcard-import
 from tensorflow.python.training.checkpointable import base as checkpointable_base
 from tensorflow.python.training.checkpointable import tracking as checkpointable
+from tensorflow.python.training.saver import BaseSaverBuilder
 from tensorflow.python.util import compat
 from tensorflow.python.util.deprecation import deprecated
 from tensorflow.python.util.tf_export import tf_export
@@ -64,7 +65,7 @@ def initialize_all_tables(name="init_all_tables"):
 @tf_export(v1=["initializers.tables_initializer", "tables_initializer"])
 def tables_initializer(name="init_all_tables"):
   """Returns an Op that initializes all tables of the default graph.
-  
+
   See the [Low Level Intro](https://www.tensorflow.org/guide/low_level_intro#feature_columns)
   guide, for an example of usage.
 
@@ -1322,6 +1323,498 @@ def index_to_string_table_from_tensor(vocabulary_list,
     return HashTable(init, default_value, name=scope)
 
 
+class MutableHashTable(LookupInterface):
+  """A generic mutable hash table implementation.
+
+  Data can be inserted by calling the insert method and removed by calling the
+  remove method. It does not support initialization via the init method.
+
+  Example usage:
+
+  ```python
+  table = tf.lookup.MutableHashTable(key_dtype=tf.string, value_dtype=tf.int64,
+                                     default_value=-1)
+  sess.run(table.insert(keys, values))
+  out = table.lookup(query_keys)
+  print(out.eval())
+  ```
+  """
+
+  def __init__(self,
+               key_dtype,
+               value_dtype,
+               default_value,
+               name="MutableHashTable",
+               checkpoint=True):
+    """Creates an empty `MutableHashTable` object.
+
+    Creates a table, the type of its keys and values are specified by key_dtype
+    and value_dtype, respectively.
+
+    Args:
+      key_dtype: the type of the key tensors.
+      value_dtype: the type of the value tensors.
+      default_value: The value to use if a key is missing in the table.
+      name: A name for the operation (optional).
+      checkpoint: if True, the contents of the table are saved to and restored
+        from checkpoints. If `shared_name` is empty for a checkpointed table, it
+        is shared using the table node name.
+
+    Returns:
+      A `MutableHashTable` object.
+
+    Raises:
+      ValueError: If checkpoint is True and no name was specified.
+    """
+    self._default_value = ops.convert_to_tensor(
+        default_value, dtype=value_dtype)
+    self._value_shape = self._default_value.get_shape()
+    self._checkpoint = checkpoint
+    self._key_dtype = key_dtype
+    self._value_dtype = value_dtype
+    self._name = name
+
+    self._shared_name = None
+    if context.executing_eagerly():
+      # TODO(allenl): This will leak memory due to kernel caching by the
+      # shared_name attribute value (but is better than the alternative of
+      # sharing everything by default when executing eagerly; hopefully creating
+      # tables in a loop is uncommon).
+      self._shared_name = "table_%d" % (ops.uid(),)
+    super(MutableHashTable, self).__init__(key_dtype, value_dtype)
+
+    self._resource_handle = self.create_resource()
+    if checkpoint:
+      saveable = MutableHashTable._Saveable(self, name)
+      if not context.executing_eagerly():
+        ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable)
+
+  def create_resource(self):
+    # The table must be shared if checkpointing is requested for multi-worker
+    # training to work correctly. Use the node name if no shared_name has been
+    # explicitly specified.
+    use_node_name_sharing = self._checkpoint and self._shared_name is None
+    if self._default_value.get_shape().ndims == 0:
+      table_ref = gen_lookup_ops.mutable_hash_table_v2(
+          shared_name=self._shared_name,
+          use_node_name_sharing=use_node_name_sharing,
+          key_dtype=self._key_dtype,
+          value_dtype=self._value_dtype,
+          name=self._name)
+    else:
+      table_ref = gen_lookup_ops.mutable_hash_table_of_tensors_v2(
+          shared_name=self._shared_name,
+          use_node_name_sharing=use_node_name_sharing,
+          key_dtype=self._key_dtype,
+          value_dtype=self._value_dtype,
+          value_shape=self._default_value.get_shape(),
+          name=self._name)
+
+    if context.executing_eagerly():
+      self._table_name = None
+    else:
+      self._table_name = table_ref.op.name.split("/")[-1]
+    return table_ref
+
+  @property
+  def name(self):
+    return self._table_name
+
+  def size(self, name=None):
+    """Compute the number of elements in this table.
+
+    Args:
+      name: A name for the operation (optional).
+
+    Returns:
+      A scalar tensor containing the number of elements in this table.
+    """
+    with ops.name_scope(name, "%s_Size" % self.name,
+                        [self.resource_handle]) as name:
+      with ops.colocate_with(self.resource_handle):
+        return gen_lookup_ops.lookup_table_size_v2(
+            self.resource_handle, name=name)
+
+  def remove(self, keys, name=None):
+    """Removes `keys` and its associated values from the table.
+
+    If a key is not present in the table, it is silently ignored.
+
+    Args:
+      keys: Keys to remove. Can be a tensor of any shape. Must match the table's
+        key type.
+      name: A name for the operation (optional).
+
+    Returns:
+      The created Operation.
+
+    Raises:
+      TypeError: when `keys` do not match the table data types.
+    """
+    if keys.dtype != self._key_dtype:
+      raise TypeError("Signature mismatch. Keys must be dtype %s, got %s." %
+                      (self._key_dtype, keys.dtype))
+
+    with ops.name_scope(
+        name, "%s_lookup_table_remove" % self.name,
+        (self.resource_handle, keys, self._default_value)) as name:
+      # pylint: disable=protected-access
+      op = gen_lookup_ops.lookup_table_remove_v2(
+          self.resource_handle, keys, name=name)
+
+    return op
+
+  def lookup(self, keys, name=None):
+    """Looks up `keys` in a table, outputs the corresponding values.
+
+    The `default_value` is used for keys not present in the table.
+
+    Args:
+      keys: Keys to look up. Can be a tensor of any shape. Must match the
+        table's key_dtype.
+      name: A name for the operation (optional).
+
+    Returns:
+      A tensor containing the values in the same shape as `keys` using the
+        table's value type.
+
+    Raises:
+      TypeError: when `keys` do not match the table data types.
+    """
+    with ops.name_scope(
+        name, "%s_lookup_table_find" % self.name,
+        (self.resource_handle, keys, self._default_value)) as name:
+      keys = ops.convert_to_tensor(keys, dtype=self._key_dtype, name="keys")
+      with ops.colocate_with(self.resource_handle):
+        values = gen_lookup_ops.lookup_table_find_v2(
+            self.resource_handle, keys, self._default_value, name=name)
+    return values
+
+  def insert(self, keys, values, name=None):
+    """Associates `keys` with `values`.
+
+    Args:
+      keys: Keys to insert. Can be a tensor of any shape. Must match the table's
+        key type.
+      values: Values to be associated with keys. Must be a tensor of the same
+        shape as `keys` and match the table's value type.
+      name: A name for the operation (optional).
+
+    Returns:
+      The created Operation.
+
+    Raises:
+      TypeError: when `keys` or `values` doesn't match the table data
+        types.
+    """
+    with ops.name_scope(name, "%s_lookup_table_insert" % self.name,
+                        [self.resource_handle, keys, values]) as name:
+      keys = ops.convert_to_tensor(keys, self._key_dtype, name="keys")
+      values = ops.convert_to_tensor(values, self._value_dtype, name="values")
+      with ops.colocate_with(self.resource_handle):
+        # pylint: disable=protected-access
+        op = gen_lookup_ops.lookup_table_insert_v2(
+            self.resource_handle, keys, values, name=name)
+    return op
+
+  def export(self, name=None):
+    """Returns tensors of all keys and values in the table.
+
+    Args:
+      name: A name for the operation (optional).
+
+    Returns:
+      A pair of tensors with the first tensor containing all keys and the
+        second tensors containing all values in the table.
+    """
+    with ops.name_scope(name, "%s_lookup_table_export_values" % self.name,
+                        [self.resource_handle]) as name:
+      with ops.colocate_with(self.resource_handle):
+        exported_keys, exported_values = gen_lookup_ops.lookup_table_export_v2(
+            self.resource_handle, self._key_dtype, self._value_dtype, name=name)
+    return exported_keys, exported_values
+
+  def _gather_saveables_for_checkpoint(self):
+    """For object-based checkpointing."""
+    return {"table": functools.partial(MutableHashTable._Saveable, table=self)}
+
+  class _Saveable(BaseSaverBuilder.SaveableObject):
+    """SaveableObject implementation for MutableHashTable."""
+
+    def __init__(self, table, name):
+      tensors = table.export()
+      specs = [
+          BaseSaverBuilder.SaveSpec(tensors[0], "", name + "-keys"),
+          BaseSaverBuilder.SaveSpec(tensors[1], "", name + "-values")
+      ]
+      # pylint: disable=protected-access
+      super(MutableHashTable._Saveable, self).__init__(table, specs, name)
+
+    def restore(self, restored_tensors, restored_shapes):
+      del restored_shapes  # unused
+      # pylint: disable=protected-access
+      with ops.colocate_with(self.op.resource_handle):
+        return gen_lookup_ops.lookup_table_import_v2(
+            self.op.resource_handle, restored_tensors[0], restored_tensors[1])
+
+
+class MutableDenseHashTable(LookupInterface):
+  """A generic mutable hash table implementation using tensors as backing store.
+
+  Data can be inserted by calling the insert method and removed by calling the
+  remove method. It does not support initialization via the init method.
+
+  It uses "open addressing" with quadratic reprobing to resolve collisions.
+  Compared to `MutableHashTable` the insert, remove and lookup operations in a
+  `MutableDenseHashTable` are typically faster, but memory usage can be higher.
+  However, `MutableDenseHashTable` does not require additional memory for
+  temporary tensors created during checkpointing and restore operations.
+
+  Example usage:
+
+  ```python
+  table = tf.lookup.MutableDenseHashTable(key_dtype=tf.int64,
+                                          value_dtype=tf.int64,
+                                          default_value=-1,
+                                          empty_key=0,
+                                          deleted_key=-1)
+
+  sess.run(table.insert(keys, values))
+  out = table.lookup(query_keys)
+  print(out.eval())
+  ```
+  """
+
+  # TODO(andreasst): consider extracting common code with MutableHashTable into
+  # a common superclass.
+  def __init__(self,
+               key_dtype,
+               value_dtype,
+               default_value,
+               empty_key,
+               deleted_key,
+               initial_num_buckets=None,
+               name="MutableDenseHashTable",
+               checkpoint=True):
+    """Creates an empty `MutableDenseHashTable` object.
+
+    Creates a table, the type of its keys and values are specified by key_dtype
+    and value_dtype, respectively.
+
+    Args:
+      key_dtype: the type of the key tensors.
+      value_dtype: the type of the value tensors.
+      default_value: The value to use if a key is missing in the table.
+      empty_key: the key to use to represent empty buckets internally. Must not
+        be used in insert, remove or lookup operations.
+      deleted_key: the key to use to represent deleted buckets internally. Must
+        not be used in insert, remove or lookup operations and be different from
+        the empty_key.
+      initial_num_buckets: the initial number of buckets.
+      name: A name for the operation (optional).
+      checkpoint: if True, the contents of the table are saved to and restored
+        from checkpoints. If `shared_name` is empty for a checkpointed table, it
+        is shared using the table node name.
+
+    Returns:
+      A `MutableDenseHashTable` object.
+
+    Raises:
+      ValueError: If checkpoint is True and no name was specified.
+    """
+    self._default_value = ops.convert_to_tensor(
+        default_value, dtype=value_dtype, name="default_value")
+    self._key_dtype = key_dtype
+    self._value_dtype = value_dtype
+    self._initial_num_buckets = initial_num_buckets
+    self._value_shape = self._default_value.get_shape()
+    self._checkpoint = checkpoint
+    self._name = name
+
+    self._empty_key = ops.convert_to_tensor(
+        empty_key, dtype=key_dtype, name="empty_key")
+    self._deleted_key = ops.convert_to_tensor(
+        deleted_key, dtype=key_dtype, name="deleted_key")
+    self._shared_name = None
+    if context.executing_eagerly():
+      # TODO(allenl): This will leak memory due to kernel caching by the
+      # shared_name attribute value (but is better than the alternative of
+      # sharing everything by default when executing eagerly; hopefully creating
+      # tables in a loop is uncommon).
+      self._shared_name = "table_%d" % (ops.uid(),)
+    super(MutableDenseHashTable, self).__init__(key_dtype, value_dtype)
+
+    self._resource_handle = self.create_resource()
+    if checkpoint:
+      saveable = MutableDenseHashTable._Saveable(self, name)
+      if not context.executing_eagerly():
+        ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable)
+
+  def create_resource(self):
+    # The table must be shared if checkpointing is requested for multi-worker
+    # training to work correctly. Use the node name if no shared_name has been
+    # explicitly specified.
+    use_node_name_sharing = self._checkpoint and self._shared_name is None
+    table_ref = gen_lookup_ops.mutable_dense_hash_table_v2(
+        empty_key=self._empty_key,
+        deleted_key=self._deleted_key,
+        shared_name=self._shared_name,
+        use_node_name_sharing=use_node_name_sharing,
+        value_dtype=self._value_dtype,
+        value_shape=self._value_shape,
+        initial_num_buckets=self._initial_num_buckets,
+        name=self._name)
+    if context.executing_eagerly():
+      self._table_name = None
+    else:
+      self._table_name = table_ref.op.name.split("/")[-1]
+    return table_ref
+
+  @property
+  def name(self):
+    return self._table_name
+
+  def size(self, name=None):
+    """Compute the number of elements in this table.
+
+    Args:
+      name: A name for the operation (optional).
+
+    Returns:
+      A scalar tensor containing the number of elements in this table.
+    """
+    with ops.name_scope(name, "%s_Size" % self.name,
+                        [self.resource_handle]) as name:
+      with ops.colocate_with(self.resource_handle):
+        return gen_lookup_ops.lookup_table_size_v2(
+            self.resource_handle, name=name)
+
+  def lookup(self, keys, name=None):
+    """Looks up `keys` in a table, outputs the corresponding values.
+
+    The `default_value` is used for keys not present in the table.
+
+    Args:
+      keys: Keys to look up. Can be a tensor of any shape. Must match the
+        table's key_dtype.
+      name: A name for the operation (optional).
+
+    Returns:
+      A tensor containing the values in the same shape as `keys` using the
+        table's value type.
+
+    Raises:
+      TypeError: when `keys` do not match the table data types.
+    """
+    with ops.name_scope(name, "%s_lookup_table_find" % self.name,
+                        [self.resource_handle, keys]) as name:
+      keys = ops.convert_to_tensor(keys, dtype=self._key_dtype, name="keys")
+      with ops.colocate_with(self.resource_handle):
+        values = gen_lookup_ops.lookup_table_find_v2(
+            self.resource_handle, keys, self._default_value, name=name)
+
+    return values
+
+  def insert(self, keys, values, name=None):
+    """Associates `keys` with `values`.
+
+    Args:
+      keys: Keys to insert. Can be a tensor of any shape. Must match the table's
+        key type.
+      values: Values to be associated with keys. Must be a tensor of the same
+        shape as `keys` and match the table's value type.
+      name: A name for the operation (optional).
+
+    Returns:
+      The created Operation.
+
+    Raises:
+      TypeError: when `keys` or `values` doesn't match the table data
+        types.
+    """
+    with ops.name_scope(name, "%s_lookup_table_insert" % self.name,
+                        [self.resource_handle, keys, values]) as name:
+      keys = ops.convert_to_tensor(keys, dtype=self._key_dtype, name="keys")
+      values = ops.convert_to_tensor(
+          values, dtype=self._value_dtype, name="values")
+      with ops.colocate_with(self.resource_handle):
+        op = gen_lookup_ops.lookup_table_insert_v2(
+            self.resource_handle, keys, values, name=name)
+      return op
+
+  def remove(self, keys, name=None):
+    """Removes `keys` and its associated values from the table.
+
+    If a key is not present in the table, it is silently ignored.
+
+    Args:
+      keys: Keys to remove. Can be a tensor of any shape. Must match the table's
+        key type.
+      name: A name for the operation (optional).
+
+    Returns:
+      The created Operation.
+
+    Raises:
+      TypeError: when `keys` do not match the table data types.
+    """
+    if keys.dtype != self._key_dtype:
+      raise TypeError("Signature mismatch. Keys must be dtype %s, got %s." %
+                      (self._key_dtype, keys.dtype))
+
+    with ops.name_scope(
+        name, "%s_lookup_table_remove" % self.name,
+        (self.resource_handle, keys, self._default_value)) as name:
+      # pylint: disable=protected-access
+      op = gen_lookup_ops.lookup_table_remove_v2(
+          self.resource_handle, keys, name=name)
+
+    return op
+
+  def export(self, name=None):
+    """Returns tensors of all keys and values in the table.
+
+    Args:
+      name: A name for the operation (optional).
+
+    Returns:
+      A pair of tensors with the first tensor containing all keys and the
+        second tensors containing all values in the table.
+    """
+    with ops.name_scope(name, "%s_lookup_table_export_values" % self.name,
+                        [self.resource_handle]) as name:
+      with ops.colocate_with(self.resource_handle):
+        exported_keys, exported_values = gen_lookup_ops.lookup_table_export_v2(
+            self.resource_handle, self._key_dtype, self._value_dtype, name=name)
+
+    return exported_keys, exported_values
+
+  def _gather_saveables_for_checkpoint(self):
+    """For object-based checkpointing."""
+    return {
+        "table": functools.partial(MutableDenseHashTable._Saveable, table=self)
+    }
+
+  class _Saveable(BaseSaverBuilder.SaveableObject):
+    """SaveableObject implementation for MutableDenseHashTable."""
+
+    def __init__(self, table, name):
+      tensors = table.export()
+      specs = [
+          BaseSaverBuilder.SaveSpec(tensors[0], "", name + "-keys"),
+          BaseSaverBuilder.SaveSpec(tensors[1], "", name + "-values")
+      ]
+      # pylint: disable=protected-access
+      super(MutableDenseHashTable._Saveable, self).__init__(table, specs, name)
+
+    def restore(self, restored_tensors, restored_shapes):
+      del restored_shapes  # unused
+      # pylint: disable=protected-access
+      with ops.colocate_with(self.op.resource_handle):
+        return gen_lookup_ops.lookup_table_import_v2(
+            self.op.resource_handle, restored_tensors[0], restored_tensors[1])
+
+
 ops.NotDifferentiable("LookupTableFind")
 ops.NotDifferentiable("LookupTableFindV2")
 ops.NotDifferentiable("LookupTableInsert")
-- 
GitLab


From d52804582d9f2d1522cff1341d2b17bc83bd7685 Mon Sep 17 00:00:00 2001
From: nammbash <niroopshankar.ammbashankar@intel.com>
Date: Tue, 29 Jan 2019 09:50:37 -0800
Subject: [PATCH 1474/2345] fix formatting manually for pbtxt and ignoring
 clang formatting

---
 .../base_api/api_def_RequantizationRangePerChannel.pbtxt      | 4 ++--
 .../core/api_def/base_api/api_def_RequantizePerChannel.pbtxt  | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/api_def/base_api/api_def_RequantizationRangePerChannel.pbtxt b/tensorflow/core/api_def/base_api/api_def_RequantizationRangePerChannel.pbtxt
index 8874a53ec1..2226027a42 100644
--- a/tensorflow/core/api_def/base_api/api_def_RequantizationRangePerChannel.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_RequantizationRangePerChannel.pbtxt
@@ -1,4 +1,4 @@
 op {
-graph_op_name:
-  "RequantizationRangePerChannel" visibility : HIDDEN
+graph_op_name: "RequantizationRangePerChannel"
+visibility : HIDDEN
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_RequantizePerChannel.pbtxt b/tensorflow/core/api_def/base_api/api_def_RequantizePerChannel.pbtxt
index 0e4cd40a36..c7bfc0a051 100644
--- a/tensorflow/core/api_def/base_api/api_def_RequantizePerChannel.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_RequantizePerChannel.pbtxt
@@ -1,4 +1,4 @@
 op {
-graph_op_name:
-  "RequantizePerChannel" visibility : HIDDEN
+graph_op_name: "RequantizePerChannel"
+visibility : HIDDEN
 }
-- 
GitLab


From dff96fb7ab332a742e1f3e9d85555c5d0bfa5eaa Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Tue, 29 Jan 2019 09:58:14 -0800
Subject: [PATCH 1475/2345] [Grappler] Lower If/While before inlining function
 body

PiperOrigin-RevId: 231421281
---
 tensorflow/core/grappler/optimizers/BUILD     |   1 +
 .../grappler/optimizers/function_optimizer.cc |  65 +++++-----
 .../optimizers/function_optimizer_test.cc     | 122 ++++++++++++++++++
 3 files changed, 157 insertions(+), 31 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 4af8fc9874..cb5d7d6746 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -181,6 +181,7 @@ tf_cuda_cc_test(
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/utils:grappler_test",
+        "@com_google_absl//absl/algorithm:container",
     ],
 )
 
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index 215ffc3b43..b722e6b332 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -27,6 +27,8 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/device_mgr.h"
 #include "tensorflow/core/common_runtime/device_set.h"
 #include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/common_runtime/lower_if_while.h"
+#include "tensorflow/core/common_runtime/optimization_registry.h"
 #include "tensorflow/core/common_runtime/placer.h"
 #include "tensorflow/core/common_runtime/process_function_library_runtime.h"
 #include "tensorflow/core/framework/attr_value_util.h"
@@ -1390,20 +1392,6 @@ Status IsInlinableIndirectFunctionCall(const FunctionOptimizerContext& ctx,
         SummarizeNodeDef(func_node));
   }
 
-  // TODO(b/120991525, b/120986912): We need to lower `If` and `While` nodes to
-  // `Switch` nodes after function inlining (one more PRE_PLACEMENT pass?), but
-  // because of the reason described above we are not sure that it's safe, for
-  // now just disable inlining functions with functional control flow.
-  const auto is_functional_ctrl_flow_op = [](const NodeDef& node) {
-    return IsIf(node) || IsWhile(node);
-  };
-  if (absl::c_any_of(func.node_def(), is_functional_ctrl_flow_op)) {
-    return errors::FailedPrecondition(
-        "Can't inline function with `If` or `While` nodes in the function "
-        "body: ",
-        SummarizeNodeDef(func_node));
-  }
-
   return Status::OK();
 }
 
@@ -1414,7 +1402,8 @@ Status CheckThatSideEffectsWillExecute(
     const FunctionOptimizerContext& ctx,
     const GraphTopologyView& graph_topo_view,
     const absl::flat_hash_set<string> output_nodes) {
-  // We ignore side-effects safety check in aggressive mode.
+  // In aggressive mode we just print a warning for side-effectful nodes that
+  // might not be executed after inlining.
   const bool aggressive = ctx.opt_level() == RewriterConfig::AGGRESSIVE;
 
   for (const NodeDef& func_body_node : graph_topo_view.graph()->node()) {
@@ -1448,18 +1437,17 @@ Status CheckThatSideEffectsWillExecute(
     DfsTraversal(graph_topo_view, {&func_body_node},
                  TraversalDirection::kFollowOutputs, predicates, callbacks);
 
-    if (!will_execute && !aggressive) {
-      return errors::Internal(
+    if (!will_execute) {
+      const string error_message = absl::StrCat(
           "Can't guarantee execution of a side-effectful node, that is not "
           "reachable from function outputs. Function body node: ",
           SummarizeNodeDef(func_body_node));
-    }
 
-    if (!will_execute && aggressive) {
-      LOG(WARNING)
-          << "Can't guarantee execution of a side-effectful node, that is not "
-             "reachable from function outputs. Function body node: "
-          << SummarizeNodeDef(func_body_node);
+      if (aggressive) {
+        LOG(WARNING) << error_message;
+      } else {
+        return errors::Internal(error_message);
+      }
     }
   }
 
@@ -1467,19 +1455,33 @@ Status CheckThatSideEffectsWillExecute(
 }
 
 Status PlaceInlinedFunctionBody(
-    const FunctionOptimizerContext& ctx, const NodeDef& func_node,
-    const GrapplerFunctionItem& item,
+    const NodeDef& func_node, const GrapplerFunctionItem& item,
     const absl::flat_hash_map<absl::string_view, int>& input_placeholders_idx,
-    GraphDef* placed_graph_def) {
+    FunctionOptimizerContext* ctx, GraphDef* placed_graph_def) {
   // Control flow lowering and Placer works with a Graph object.
   std::unique_ptr<Graph> func_body_graph =
-      absl::make_unique<Graph>(ctx.function_library());
+      absl::make_unique<Graph>(ctx->function_library());
 
   GraphConstructorOptions opts;
   TF_RETURN_IF_ERROR(
       ConvertGraphDefToGraph(opts, item.graph, func_body_graph.get()));
 
-  // TODO(ezhulenev): Lower If/While ops.
+  // ------------------------------------------------------------------------ //
+  // Grappler receives the graph after PRE_PLACEMENT, Placer, and POST_PLACEMENT
+  // passes, so each node has a valid device assignment. Also V2 control
+  // flow ops (functional If and While) should have been lowered to V1 control
+  // flow (Switch and Merge nodes). To keep the graph valid for execution we
+  // must assign device to every inlined graph node, and also lower the control
+  // flow.
+
+  GraphOptimizationPassOptions opt_options;
+  opt_options.graph = &func_body_graph;
+  opt_options.flib_def = ctx->mutable_function_library();
+
+  // TODO(ezhulenev): Should we run full PRE_PLACEMENT pass here? And
+  // POST_PLACEMENT after placer?
+  LowerIfWhilePass pass;
+  TF_RETURN_IF_ERROR(pass.Run(opt_options));
 
   // ------------------------------------------------------------------------ //
   // Before placing the function body nodes we pin input placeholders to the
@@ -1492,7 +1494,7 @@ Status PlaceInlinedFunctionBody(
     if (input_placeholder_idx != input_placeholders_idx.end()) {
       const int input_idx = input_placeholder_idx->second;
       const GraphView::OutputPort output_port =
-          ctx.graph_view().GetRegularFanin({&func_node, input_idx});
+          ctx->graph_view().GetRegularFanin({&func_node, input_idx});
 
       VLOG(3) << "Pin inlined function input node '" << func_body_node->name()
               << "' to the '" << output_port.node->device() << "' device.";
@@ -1504,7 +1506,7 @@ Status PlaceInlinedFunctionBody(
   // After placing nodes corresponding to the function inputs, we need to assign
   // device placements to all other function body nodes.
 
-  const DeviceSet* devices = ctx.devices();
+  const DeviceSet* devices = ctx->devices();
 
   if (devices->devices().empty()) {
     // If there are no devices available for placer, we just put all nodes to
@@ -1546,6 +1548,7 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
                                   FunctionOptimizerContext* ctx,
                                   GraphDef* optimized_graph) {
   VLOG(2) << "Inline indirect function call: " << SummarizeNodeDef(func_node);
+  VLOG(4) << "Inlined function definition: " << DebugString(func);
   TF_RETURN_IF_ERROR(IsInlinableIndirectFunctionCall(*ctx, func, func_node));
 
   const AttrSlice func_instantiation_attr =
@@ -1712,7 +1715,7 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
 
   GraphDef placed_graph_def;
   TF_RETURN_IF_ERROR(PlaceInlinedFunctionBody(
-      *ctx, func_node, item, input_placeholders_idx, &placed_graph_def));
+      func_node, item, input_placeholders_idx, ctx, &placed_graph_def));
 
   // ------------------------------------------------------------------------ //
   // After all nodes placed we need to prepare them for inlining into the
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
index 7e472ec25b..c7373989dc 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
@@ -14,6 +14,8 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/grappler/optimizers/function_optimizer.h"
+
+#include "absl/algorithm/container.h"
 #include "tensorflow/cc/ops/functional_ops.h"
 #include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/core/framework/function_testlib.h"
@@ -1302,6 +1304,126 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithNestedFunctionCall) {
   test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
 }
 
+TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithFunctionalControlFlow) {
+  using test::function::NDef;
+  using FDH = FunctionDefHelper;
+
+  FunctionOptimizer optimizer(RewriterConfig::AGGRESSIVE);
+
+  FunctionDef add_func = FunctionDefHelper::Create(
+      "MyAdd", {"x:T", "y:T"}, {"z:T"}, {"T: {float, double}"},
+      {{{"add"}, "Add", {"x", "y"}, {{"T", "$T"}}}},
+      /* Mapping between function returns and function node outputs. */
+      {{"z", "add:z:0"}});
+
+  FunctionDef mul_func = FunctionDefHelper::Create(
+      "MyMul", {"x:T", "y:T"}, {"z:T"}, {"T: {float, double}"},
+      {{{"mul"}, "Mul", {"x", "y"}, {{"T", "$T"}}}},
+      /* Mapping between function returns and function node outputs. */
+      {{"z", "mul:z:0"}});
+
+  // Compute: return cond ? a + b : a * b
+  FunctionDef add_or_mul_func = FunctionDefHelper::Create(
+      "AddOrMul", {"cond:bool", "x:float", "y:float"}, {"z:float"}, {},
+      {
+          {{"if_node"},
+           "If",
+           {"cond", "x", "y"},
+           {
+               {"Tcond", DT_BOOL},
+               {"Tin", DataTypeSlice{DT_FLOAT, DT_FLOAT}},
+               {"Tout", DataTypeSlice{DT_FLOAT}},
+               {"then_branch", FDH::FunctionRef("MyAdd", {{"T", DT_FLOAT}})},
+               {"else_branch", FDH::FunctionRef("MyMul", {{"T", DT_FLOAT}})},
+               {"_lower_using_switch_merge", true},
+           }},
+      },
+      /* Mapping between function returns and function node outputs. */
+      {{"z", "if_node:output:0"}});
+
+  // Build a computation graph for:
+  //   is_add: bool
+  //   a: float
+  //   b: float
+  //   c = AddOrMul(is_add, a, b)  # is_add ? a + b : a * b
+  //   d = Identity(c)
+  //   return d
+
+  // c = MyMul(a, b)
+  GrapplerItem item;
+  item.fetch = {"d"};
+  item.graph = test::function::GDef(
+      {NDef("is_add", "Placeholder", {}, {{"dtype", DT_BOOL}}, kDevice),
+       NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("b", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+
+       NDef("c", "PartitionedCall", {"is_add", "a", "b"},
+            {{"Tin", DataTypeSlice{DT_BOOL, DT_FLOAT, DT_FLOAT}},
+             {"Tout", DataTypeSlice{DT_FLOAT}},
+             {"f", FDH::FunctionRef("AddOrMul")}},
+            kDevice),
+
+       NDef("d", "Identity", {"c"}, {{"T", DT_FLOAT}}, kDevice)},
+      // Function library.
+      {add_or_mul_func, add_func, mul_func});
+
+  GraphDef optimized_graph;
+  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &optimized_graph));
+
+  const auto count_nodes_with_op = [&](const string& op) {
+    return absl::c_count_if(optimized_graph.node(), [&](const NodeDef& node) {
+      return node.op() == op;
+    });
+  };
+
+  // All `PartitionedCall` nodes in the optimized graph must be inlined, and
+  // `If` node must be lowered to `Switch` and `Merge` nodes.
+  EXPECT_EQ(count_nodes_with_op("PartitionedCallOp"), 0);
+  EXPECT_EQ(count_nodes_with_op("If"), 0);
+  EXPECT_EQ(count_nodes_with_op("Switch"), 3);
+  EXPECT_EQ(count_nodes_with_op("Merge"), 1);
+
+  GrapplerItem optimized = item.WithGraph(std::move(optimized_graph));
+
+  Tensor one = test::AsScalar<float>(1.0);
+  Tensor two = test::AsScalar<float>(2.0);
+  Tensor three = test::AsScalar<float>(3.0);
+
+  const auto feed_args = [&](bool is_add) {
+    std::vector<std::pair<string, Tensor>> feed;
+    feed.emplace_back("a", one);
+    feed.emplace_back("b", two);
+    feed.emplace_back("is_add", test::AsScalar<bool>(is_add));
+    return feed;
+  };
+
+  {  // Check 'is_add == true': a + b
+    item.feed = feed_args(true);
+    optimized.feed = feed_args(true);
+
+    auto tensors_expected = EvaluateFetchNodes(item);
+    ASSERT_EQ(tensors_expected.size(), 1);
+    test::ExpectTensorEqual<float>(tensors_expected[0], three);
+
+    auto tensors = EvaluateFetchNodes(optimized);
+    ASSERT_EQ(tensors.size(), tensors_expected.size());
+    test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
+  }
+
+  {  // Check 'is_add == false': a * b
+    item.feed = feed_args(false);
+    optimized.feed = feed_args(false);
+
+    auto tensors_expected = EvaluateFetchNodes(item);
+    ASSERT_EQ(tensors_expected.size(), 1);
+    test::ExpectTensorEqual<float>(tensors_expected[0], two);
+
+    auto tensors = EvaluateFetchNodes(optimized);
+    ASSERT_EQ(tensors.size(), tensors_expected.size());
+    test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
+  }
+}
+
 TEST_F(FunctionOptimizerTest, SpecializeFunctionXTimesTwo) {
   using test::function::NDef;
 
-- 
GitLab


From 25a987044e97d8fff2733ad15b7617b57259e20f Mon Sep 17 00:00:00 2001
From: Yanan Cao <ycao@google.com>
Date: Tue, 29 Jan 2019 09:58:18 -0800
Subject: [PATCH 1476/2345] Automated rollback of commit
 01cf6233d62d476676ffbea0fad46660a0ba9a16

PiperOrigin-RevId: 231421297
---
 tensorflow/python/keras/metrics.py            | 20 ++++++----
 .../keras/metrics_confusion_matrix_test.py    | 39 +++++++------------
 .../python/keras/utils/metrics_utils.py       | 34 ----------------
 .../v1/tensorflow.keras.metrics.-a-u-c.pbtxt  |  2 +-
 .../v2/tensorflow.keras.metrics.-a-u-c.pbtxt  |  2 +-
 5 files changed, 29 insertions(+), 68 deletions(-)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 4662535fcd..6aed0e7ac6 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -1607,8 +1607,8 @@ class AUC(Metric):
 
   def __init__(self,
                num_thresholds=200,
-               curve='ROC',
-               summation_method='interpolation',
+               curve=metrics_utils.AUCCurve.ROC,
+               summation_method=metrics_utils.AUCSummationMethod.INTERPOLATION,
                name=None,
                dtype=None):
     """Creates an `AUC` instance.
@@ -1631,12 +1631,18 @@ class AUC(Metric):
     # Validate configurations.
     if num_thresholds <= 1:
       raise ValueError('`num_thresholds` must be > 1.')
+    if curve not in list(metrics_utils.AUCCurve):
+      raise ValueError('Invalid curve: "{}". Valid options are: "{}"'.format(
+          curve, list(metrics_utils.AUCCurve)))
+    if summation_method not in list(metrics_utils.AUCSummationMethod):
+      raise ValueError(
+          'Invalid summation method: "{}". Valid options are: "{}"'.format(
+              summation_method, list(metrics_utils.AUCSummationMethod)))
 
     # Update properties.
     self.num_thresholds = num_thresholds
-    self.curve = metrics_utils.AUCCurve.from_str(curve)
-    self.summation_method = metrics_utils.AUCSummationMethod.from_str(
-        summation_method)
+    self.curve = curve
+    self.summation_method = summation_method
     super(AUC, self).__init__(name=name, dtype=dtype)
 
     # Create metric variables
@@ -1796,8 +1802,8 @@ class AUC(Metric):
   def get_config(self):
     config = {
         'num_thresholds': self.num_thresholds,
-        'curve': self.curve.value,
-        'summation_method': self.summation_method.value,
+        'curve': self.curve,
+        'summation_method': self.summation_method,
     }
     base_config = super(AUC, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
diff --git a/tensorflow/python/keras/metrics_confusion_matrix_test.py b/tensorflow/python/keras/metrics_confusion_matrix_test.py
index 972f7b6de7..6a0fcfed07 100644
--- a/tensorflow/python/keras/metrics_confusion_matrix_test.py
+++ b/tensorflow/python/keras/metrics_confusion_matrix_test.py
@@ -18,8 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import json
-
 from absl.testing import parameterized
 import numpy as np
 
@@ -951,8 +949,8 @@ class AUCTest(test.TestCase):
   def test_config(self):
     auc_obj = metrics.AUC(
         num_thresholds=100,
-        curve='PR',
-        summation_method='majoring',
+        curve=metrics_utils.AUCCurve.PR,
+        summation_method=metrics_utils.AUCSummationMethod.MAJORING,
         name='auc_1')
     self.assertEqual(auc_obj.name, 'auc_1')
     self.assertEqual(len(auc_obj.variables), 4)
@@ -960,8 +958,6 @@ class AUCTest(test.TestCase):
     self.assertEqual(auc_obj.curve, metrics_utils.AUCCurve.PR)
     self.assertEqual(auc_obj.summation_method,
                      metrics_utils.AUCSummationMethod.MAJORING)
-    old_config = auc_obj.get_config()
-    self.assertDictEqual(old_config, json.loads(json.dumps(old_config)))
 
     # Check save and restore config
     auc_obj2 = metrics.AUC.from_config(auc_obj.get_config())
@@ -971,8 +967,6 @@ class AUCTest(test.TestCase):
     self.assertEqual(auc_obj2.curve, metrics_utils.AUCCurve.PR)
     self.assertEqual(auc_obj2.summation_method,
                      metrics_utils.AUCSummationMethod.MAJORING)
-    new_config = auc_obj2.get_config()
-    self.assertDictEqual(old_config, new_config)
 
   def test_value_is_idempotent(self):
     self.setup()
@@ -1027,7 +1021,8 @@ class AUCTest(test.TestCase):
   def test_weighted_roc_majoring(self):
     self.setup()
     auc_obj = metrics.AUC(
-        num_thresholds=self.num_thresholds, summation_method='majoring')
+        num_thresholds=self.num_thresholds,
+        summation_method=metrics_utils.AUCSummationMethod.MAJORING)
     self.evaluate(variables.variables_initializer(auc_obj.variables))
     result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)
 
@@ -1042,7 +1037,8 @@ class AUCTest(test.TestCase):
   def test_weighted_roc_minoring(self):
     self.setup()
     auc_obj = metrics.AUC(
-        num_thresholds=self.num_thresholds, summation_method='minoring')
+        num_thresholds=self.num_thresholds,
+        summation_method=metrics_utils.AUCSummationMethod.MINORING)
     self.evaluate(variables.variables_initializer(auc_obj.variables))
     result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)
 
@@ -1058,8 +1054,8 @@ class AUCTest(test.TestCase):
     self.setup()
     auc_obj = metrics.AUC(
         num_thresholds=self.num_thresholds,
-        curve='PR',
-        summation_method='majoring')
+        curve=metrics_utils.AUCCurve.PR,
+        summation_method=metrics_utils.AUCSummationMethod.MAJORING)
     self.evaluate(variables.variables_initializer(auc_obj.variables))
     result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)
 
@@ -1075,8 +1071,8 @@ class AUCTest(test.TestCase):
     self.setup()
     auc_obj = metrics.AUC(
         num_thresholds=self.num_thresholds,
-        curve='PR',
-        summation_method='minoring')
+        curve=metrics_utils.AUCCurve.PR,
+        summation_method=metrics_utils.AUCSummationMethod.MINORING)
     self.evaluate(variables.variables_initializer(auc_obj.variables))
     result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)
 
@@ -1090,7 +1086,10 @@ class AUCTest(test.TestCase):
 
   def test_weighted_pr_interpolation(self):
     self.setup()
-    auc_obj = metrics.AUC(num_thresholds=self.num_thresholds, curve='PR')
+    auc_obj = metrics.AUC(
+        num_thresholds=self.num_thresholds,
+        curve=metrics_utils.AUCCurve.PR,
+        summation_method=metrics_utils.AUCSummationMethod.INTERPOLATION)
     self.evaluate(variables.variables_initializer(auc_obj.variables))
     result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)
 
@@ -1116,16 +1115,6 @@ class AUCTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, '`num_thresholds` must be > 1.'):
       metrics.AUC(num_thresholds=1)
 
-  def test_invalid_curve(self):
-    with self.assertRaisesRegexp(ValueError,
-                                 'Invalid AUC curve value "Invalid".'):
-      metrics.AUC(curve='Invalid')
-
-  def test_invalid_summation_method(self):
-    with self.assertRaisesRegexp(
-        ValueError, 'Invalid AUC summation method value "Invalid".'):
-      metrics.AUC(summation_method='Invalid')
-
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/utils/metrics_utils.py b/tensorflow/python/keras/utils/metrics_utils.py
index 329cb0b278..c5ff617ad7 100644
--- a/tensorflow/python/keras/utils/metrics_utils.py
+++ b/tensorflow/python/keras/utils/metrics_utils.py
@@ -161,49 +161,15 @@ class ConfusionMatrix(Enum):
 
 
 class AUCCurve(Enum):
-  """Type of AUC Curve (ROC or PR)."""
   ROC = 'ROC'
   PR = 'PR'
 
-  @staticmethod
-  def from_str(key):
-    if key in ('pr', 'PR'):
-      return AUCCurve.PR
-    elif key in ('roc', 'ROC'):
-      return AUCCurve.ROC
-    else:
-      raise ValueError('Invalid AUC curve value "%s".' % key)
-
 
 class AUCSummationMethod(Enum):
-  """Type of AUC summation method.
-
-  https://en.wikipedia.org/wiki/Riemann_sum)
-
-  Contains the following values:
-  * 'interpolation': Applies mid-point summation scheme for `ROC` curve. For
-    `PR` curve, interpolates (true/false) positives but not the ratio that is
-    precision (see Davis & Goadrich 2006 for details).
-  * 'minoring': Applies left summation for increasing intervals and right
-    summation for decreasing intervals.
-  * 'majoring': Applies right summation for increasing intervals and left
-    summation for decreasing intervals.
-  """
   INTERPOLATION = 'interpolation'
   MAJORING = 'majoring'
   MINORING = 'minoring'
 
-  @staticmethod
-  def from_str(key):
-    if key in ('interpolation', 'Interpolation'):
-      return AUCSummationMethod.INTERPOLATION
-    elif key in ('majoring', 'Majoring'):
-      return AUCSummationMethod.MAJORING
-    elif key in ('minoring', 'Minoring'):
-      return AUCSummationMethod.MINORING
-    else:
-      raise ValueError('Invalid AUC summation method value "%s".' % key)
-
 
 def update_confusion_matrix_variables(variables_to_update,
                                       y_true,
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-a-u-c.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-a-u-c.pbtxt
index beaa6f0596..8019d43c78 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-a-u-c.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-a-u-c.pbtxt
@@ -87,7 +87,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'num_thresholds\', \'curve\', \'summation_method\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'200\', \'ROC\', \'interpolation\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'num_thresholds\', \'curve\', \'summation_method\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'200\', \'AUCCurve.ROC\', \'AUCSummationMethod.INTERPOLATION\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_loss"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-a-u-c.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-a-u-c.pbtxt
index beaa6f0596..8019d43c78 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-a-u-c.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-a-u-c.pbtxt
@@ -87,7 +87,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'num_thresholds\', \'curve\', \'summation_method\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'200\', \'ROC\', \'interpolation\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'num_thresholds\', \'curve\', \'summation_method\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'200\', \'AUCCurve.ROC\', \'AUCSummationMethod.INTERPOLATION\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_loss"
-- 
GitLab


From 0aee82add196a6f0796bc8215c3d4bff90780a4e Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Tue, 29 Jan 2019 09:58:36 -0800
Subject: [PATCH 1477/2345] Exposing tf.get_static_value

PiperOrigin-RevId: 231421348
---
 tensorflow/python/framework/tensor_util.py      | 11 ++++-------
 tensorflow/tools/api/golden/v1/tensorflow.pbtxt |  4 ++++
 tensorflow/tools/api/golden/v2/tensorflow.pbtxt |  4 ++++
 tensorflow/tools/compatibility/tf_upgrade_v2.py |  2 ++
 4 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/tensorflow/python/framework/tensor_util.py b/tensorflow/python/framework/tensor_util.py
index ca8b067935..54d5478aba 100644
--- a/tensorflow/python/framework/tensor_util.py
+++ b/tensorflow/python/framework/tensor_util.py
@@ -781,19 +781,16 @@ def _ConstantValue(tensor, partial):
     return None
 
 
+@tf_export('get_static_value')
 def constant_value(tensor, partial=False):  # pylint: disable=invalid-name
   """Returns the constant value of the given tensor, if efficiently calculable.
 
   This function attempts to partially evaluate the given tensor, and
   returns its value as a numpy ndarray if this succeeds.
 
-  TODO(mrry): Consider whether this function should use a registration
-  mechanism like gradients and ShapeFunctions, so that it is easily
-  extensible.
-
-  NOTE: If `constant_value(tensor)` returns a non-`None` result, it will no
-  longer be possible to feed a different value for `tensor`. This allows the
-  result of this function to influence the graph that is constructed, and
+  Compatibility(V1): If `constant_value(tensor)` returns a non-`None` result, it
+  will no longer be possible to feed a different value for `tensor`. This allows
+  the result of this function to influence the graph that is constructed, and
   permits static shape optimizations.
 
   Args:
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
index 585dc2ea57..0276a81473 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
@@ -1308,6 +1308,10 @@ tf_module {
     name: "get_session_tensor"
     argspec: "args=[\'handle\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "get_static_value"
+    argspec: "args=[\'tensor\', \'partial\'], varargs=None, keywords=None, defaults=[\'False\'], "
+  }
   member_method {
     name: "get_variable"
     argspec: "args=[\'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'collections\', \'caching_device\', \'partitioner\', \'validate_shape\', \'use_resource\', \'custom_getter\', \'constraint\', \'synchronization\', \'aggregation\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 92f4704b49..4af4b7ef55 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -604,6 +604,10 @@ tf_module {
     name: "get_logger"
     argspec: "args=[], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "get_static_value"
+    argspec: "args=[\'tensor\', \'partial\'], varargs=None, keywords=None, defaults=[\'False\'], "
+  }
   member_method {
     name: "gradients"
     argspec: "args=[\'ys\', \'xs\', \'grad_ys\', \'name\', \'gate_gradients\', \'aggregation_method\', \'stop_gradients\', \'unconnected_gradients\'], varargs=None, keywords=None, defaults=[\'None\', \'gradients\', \'False\', \'None\', \'None\', \'UnconnectedGradients.NONE\'], "
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 2abd3eac1f..49de4a189e 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -537,6 +537,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.data.experimental.unbatch",
         "tf.contrib.data.unique":
             "tf.data.experimental.unique",
+        "tf.contrib.framework.constant_value":
+            "tf.get_static_value",
         "tf.contrib.saved_model.load_keras_model":
             "tf.keras.experimental.load_from_saved_model",
         "tf.contrib.saved_model.save_keras_model":
-- 
GitLab


From 55f469ec29bb7f0a019df6043baca47a5f9511fe Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Tue, 29 Jan 2019 10:02:26 -0800
Subject: [PATCH 1478/2345] Better error message for unhashable type in
 tf.function cache key.

We can probably do better if we force an error while building the cache key,
but not dropping the existing error message on the ground will help as it
gives you the offending types.

PiperOrigin-RevId: 231422170
---
 tensorflow/python/eager/function.py      | 7 ++++---
 tensorflow/python/eager/function_test.py | 9 +++++++++
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index ac596654d5..860371dd6c 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -1290,9 +1290,10 @@ class Function(object):
     with self._lock:
       try:
         graph_function = self._function_cache.get(cache_key, None)
-      except TypeError:
-        raise TypeError("Arguments supplied to `defun`-generated functions "
-                        "must be hashable.")
+      except TypeError as e:
+        raise TypeError(
+            "Arguments supplied to `defun`-generated functions must be"
+            " hashable.  Original error: %s" % e)
 
       if graph_function is None:
         logging.vlog(1,
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index d337429db6..c8f7ecbca2 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -112,6 +112,15 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
       t = constant_op.constant(1.0)
       self.assertAllEqual(add(t, t).numpy(), 2.0)
 
+  def testNoHash(self):
+
+    @def_function.function()
+    def f(_):
+      return 1.0
+
+    with self.assertRaisesRegexp(TypeError, 'set'):
+      f(set([]))
+
   def testFuncName(self):
 
     @function.defun_with_attributes(attributes={'func_name': 'multiply'})
-- 
GitLab


From 2d1fbaac92986f267320a02785a49e79c7ace743 Mon Sep 17 00:00:00 2001
From: Peter Buchlovsky <petebu@google.com>
Date: Tue, 29 Jan 2019 10:17:59 -0800
Subject: [PATCH 1479/2345] Add support for input functions that return
 callables that return a nest of tensors.

PiperOrigin-RevId: 231425396
---
 .../collective_all_reduce_strategy_test.py    | 51 ++++++++++++-------
 .../python/mirrored_strategy_multigpu_test.py | 49 ++++++++++++++++--
 .../python/one_device_strategy_test.py        | 20 +++++++-
 .../python/parameter_server_strategy_test.py  | 48 ++++++++++++-----
 .../distribute/python/strategy_test_lib.py    | 21 ++++----
 tensorflow/python/distribute/input_lib.py     | 32 ++++++++++--
 6 files changed, 171 insertions(+), 50 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
index 9b6236fd9f..bba0bce9f2 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
@@ -240,7 +240,7 @@ class CollectiveAllReduceStrategyTestBase(
     return np.allclose(x_value, reduced_x_value, atol=1e-5)
 
   def _test_input_fn_iterator(self, task_type, task_id, num_gpus, input_fn,
-                              expected_values):
+                              expected_values, test_reinitialize=True):
     distribution, master_target, config = self._get_test_object(
         task_type, task_id, num_gpus)
     devices = distribution.extended.worker_devices
@@ -263,13 +263,14 @@ class CollectiveAllReduceStrategyTestBase(
                   for r in range(len(devices))])
 
       # After re-initializing the iterator, should be able to iterate again.
-      sess.run(iterator.initialize())
+      if test_reinitialize:
+        sess.run(iterator.initialize())
 
-      for expected_value in expected_values:
-        next_element = iterator.get_next()
-        computed_value = sess.run([values.select_replica(r, next_element)
-                                   for r in range(len(devices))])
-        self.assertEqual(expected_value, computed_value)
+        for expected_value in expected_values:
+          next_element = iterator.get_next()
+          computed_value = sess.run([values.select_replica(r, next_element)
+                                     for r in range(len(devices))])
+          self.assertEqual(expected_value, computed_value)
 
 
 class DistributedCollectiveAllReduceStrategyTest(
@@ -319,23 +320,31 @@ class DistributedCollectiveAllReduceStrategyTest(
 
   # TODO(yuefengz): Update how we use num_gpus and required_gpus
   @combinations.generate(
-      combinations.combine(mode=['graph'], num_gpus=[0, 1, 2], required_gpus=1))
-  def testMakeInputFnIterator(self, num_gpus):
+      combinations.combine(mode=['graph'], num_gpus=[0, 1, 2], required_gpus=1,
+                           use_dataset=[True, False]))
+  def testMakeInputFnIterator(self, num_gpus, use_dataset):
     if context.num_gpus() < num_gpus:
       self.skipTest('Not enough GPUs')
-    dataset_fn = lambda: dataset_ops.Dataset.range(100)
+    if use_dataset:
+      fn = lambda: dataset_ops.Dataset.range(100)
+    else:
+      def fn():
+        dataset = dataset_ops.Dataset.range(100)
+        it = dataset.make_one_shot_iterator()
+        return it.get_next
     # We use CPU as the device when num_gpus = 0
     devices_per_worker = max(1, num_gpus)
     expected_values = [[i+j for j in range(devices_per_worker)]
                        for i in range(0, 100, devices_per_worker)]
 
     input_fn = self._input_fn_to_test_input_context(
-        dataset_fn,
+        fn,
         expected_num_replicas_in_sync=3*devices_per_worker,
         expected_num_input_pipelines=3,
         expected_input_pipeline_id=1)  # because task_id = 1
     self._test_input_fn_iterator('worker', 1, num_gpus,
-                                 input_fn, expected_values)
+                                 input_fn, expected_values,
+                                 test_reinitialize=use_dataset)
 
   def testUpdateConfigProto(self):
     distribution = collective_all_reduce_strategy.CollectiveAllReduceStrategy(
@@ -425,19 +434,27 @@ class LocalCollectiveAllReduceStrategy(
     self._test_complex_model(None, None, num_gpus)
 
   @combinations.generate(
-      combinations.combine(mode=['graph', 'eager'], required_gpus=2))
-  def testMakeInputFnIterator(self):
+      combinations.combine(mode=['graph', 'eager'], required_gpus=2,
+                           use_dataset=[True, False]))
+  def testMakeInputFnIterator(self, use_dataset):
     num_gpus = 2
-    dataset_fn = lambda: dataset_ops.Dataset.range(5 * num_gpus)
+    if use_dataset:
+      fn = lambda: dataset_ops.Dataset.range(5 * num_gpus)
+    else:
+      def fn():
+        dataset = dataset_ops.Dataset.range(5 * num_gpus)
+        it = dataset.make_one_shot_iterator()
+        return it.get_next
     expected_values = [range(i, i + num_gpus) for i in range(0, 10, num_gpus)]
 
     input_fn = self._input_fn_to_test_input_context(
-        dataset_fn,
+        fn,
         expected_num_replicas_in_sync=num_gpus,
         expected_num_input_pipelines=1,
         expected_input_pipeline_id=0)
     self._test_input_fn_iterator(None, None, num_gpus,
-                                 input_fn, expected_values)
+                                 input_fn, expected_values,
+                                 test_reinitialize=use_dataset)
 
   def testAllReduceSum(self):
     if context.num_gpus() < 2: self.skipTest('Not enough GPUs')
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index d6337d106f..0b8df787e6 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -38,8 +38,8 @@ from tensorflow.python.eager import context
 from tensorflow.python.eager import function
 from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
-from tensorflow.python.framework import func_graph
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import func_graph
 from tensorflow.python.framework import ops
 from tensorflow.python.keras.engine import training as keras_training
 from tensorflow.python.keras.layers import core as keras_core
@@ -103,7 +103,7 @@ class MirroredTwoDeviceDistributionTest(
       expected = sum(range(distribution.num_replicas_in_sync))
       self.assertEqual(expected, self.evaluate(reduced))
 
-  def testMakeInputFnIterator(self, distribution):
+  def testMakeInputFnIteratorWithDataset(self, distribution):
     dataset_fn = lambda: dataset_ops.Dataset.range(10)
     expected_values = [[i, i+1] for i in range(0, 10, 2)]
 
@@ -116,6 +116,23 @@ class MirroredTwoDeviceDistributionTest(
     self._test_input_fn_iterator(iterator, distribution.extended.worker_devices,
                                  expected_values)
 
+  def testMakeInputFnIteratorWithCallable(self, distribution):
+    def fn():
+      dataset = dataset_ops.Dataset.range(2).interleave(
+          (lambda _: dataset_ops.Dataset.range(10)), cycle_length=2)
+      it = dataset.make_one_shot_iterator()
+      return it.get_next
+    expected_values = [[i, i] for i in range(0, 10)]
+
+    input_fn = self._input_fn_to_test_input_context(
+        fn,
+        expected_num_replicas_in_sync=2,
+        expected_num_input_pipelines=1,
+        expected_input_pipeline_id=0)
+    iterator = distribution.make_input_fn_iterator(input_fn)
+    self._test_input_fn_iterator(iterator, distribution.extended.worker_devices,
+                                 expected_values, test_reinitialize=False)
+
   def testNumpyIterator(self, distribution):
     self._test_numpy_iterator(distribution)
 
@@ -1416,7 +1433,7 @@ class MultiWorkerMirroredStrategyTest(
       self.assertEqual(a.device, "/job:worker/task:0")
       self.assertEqual(b.device, "/job:worker/task:0/device:CPU:0")
 
-  def testMakeInputFnIterator(self, distribution):
+  def testMakeInputFnIteratorWithDataset(self, distribution):
     self._configure_distribution_strategy(distribution)
     dataset_fn = lambda: dataset_ops.Dataset.range(100)
     num_gpus = context.num_gpus()
@@ -1437,6 +1454,32 @@ class MultiWorkerMirroredStrategyTest(
       self._test_input_fn_iterator(
           iterator, distribution.extended.worker_devices, expected_values, sess)
 
+  def testMakeInputFnIteratorWithCallable(self, distribution):
+    self._configure_distribution_strategy(distribution)
+    def fn():
+      dataset = dataset_ops.Dataset.range(100)
+      it = dataset.make_one_shot_iterator()
+      return it.get_next
+    num_gpus = context.num_gpus()
+    num_workers = 2
+
+    expected_values = []
+    for i in range(0, 100, num_gpus):
+      expected_values.append([i+j for j in range(num_gpus)] * num_workers)
+
+    with context.graph_mode(), self.cached_session() as sess:
+      # `expected_input_pipeline_id` is None because the input_fn will be called
+      # multiple times, each with a different input_pipeline_id.
+      input_fn = self._input_fn_to_test_input_context(
+          fn,
+          expected_num_replicas_in_sync=num_workers*num_gpus,
+          expected_num_input_pipelines=num_workers,
+          expected_input_pipeline_id=None)
+      iterator = distribution.make_input_fn_iterator(input_fn)
+      self._test_input_fn_iterator(
+          iterator, distribution.extended.worker_devices, expected_values, sess,
+          test_reinitialize=False)
+
   def testUpdateConfigProto(self, distribution):
     distribution.configure(cluster_spec={"worker": ["fake1", "fake2"]})
 
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy_test.py b/tensorflow/contrib/distribute/python/one_device_strategy_test.py
index 683ed86f4f..906bffc852 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy_test.py
@@ -46,7 +46,7 @@ class OneDeviceStrategyTest(
     self._test_call_and_merge_exceptions(self._get_distribution_strategy())
 
   @test_util.run_in_graph_and_eager_modes
-  def testMakeInputFnIterator(self):
+  def testMakeInputFnIteratorWithDataset(self):
     d = one_device_strategy.OneDeviceStrategy("/device:CPU:0")
     dataset_fn = lambda: dataset_ops.Dataset.range(10)
     expected_values = [[i] for i in range(10)]
@@ -59,6 +59,24 @@ class OneDeviceStrategyTest(
     self._test_input_fn_iterator(
         iterator, d.extended.worker_devices, expected_values)
 
+  @test_util.run_in_graph_and_eager_modes
+  def testMakeInputFnIteratorWithCallable(self):
+    d = one_device_strategy.OneDeviceStrategy("/device:CPU:0")
+    def fn():
+      dataset = dataset_ops.Dataset.range(10)
+      it = dataset.make_one_shot_iterator()
+      return it.get_next
+    expected_values = [[i] for i in range(10)]
+    input_fn = self._input_fn_to_test_input_context(
+        fn,
+        expected_num_replicas_in_sync=1,
+        expected_num_input_pipelines=1,
+        expected_input_pipeline_id=0)
+    iterator = d.make_input_fn_iterator(input_fn)
+    self._test_input_fn_iterator(
+        iterator, d.extended.worker_devices, expected_values,
+        test_reinitialize=False)
+
   @test_util.run_in_graph_and_eager_modes
   def testNumpyIterator(self):
     self._test_numpy_iterator(self._get_distribution_strategy())
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
index 89dcdbcfc2..fede253d13 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
@@ -571,6 +571,7 @@ class ParameterServerStrategyTestBase(
                               num_gpus,
                               input_fn,
                               expected_values,
+                              test_reinitialize=True,
                               use_core_strategy=False):
     distribution, master_target, config = self._get_test_objects(
         task_type, task_id, num_gpus, use_core_strategy=use_core_strategy)
@@ -594,13 +595,14 @@ class ParameterServerStrategyTestBase(
                   for r in range(len(devices))])
 
       # After re-initializing the iterator, should be able to iterate again.
-      sess.run(iterator.initialize())
+      if test_reinitialize:
+        sess.run(iterator.initialize())
 
-      for expected_value in expected_values:
-        next_element = iterator.get_next()
-        computed_value = sess.run([values.select_replica(r, next_element)
-                                   for r in range(len(devices))])
-        self.assertEqual(expected_value, computed_value)
+        for expected_value in expected_values:
+          next_element = iterator.get_next()
+          computed_value = sess.run([values.select_replica(r, next_element)
+                                     for r in range(len(devices))])
+          self.assertEqual(expected_value, computed_value)
 
 
 class ParameterServerStrategyTest(
@@ -700,16 +702,24 @@ class ParameterServerStrategyTest(
           mode=['graph'],
           num_gpus=[1, 2],
           required_gpus=1,
-          use_core_strategy=[True, False]))
-  def testMakeInputFnIteratorDistributed(self, num_gpus, use_core_strategy):
+          use_core_strategy=[True, False],
+          use_dataset=[True, False]))
+  def testMakeInputFnIteratorDistributed(self, num_gpus, use_core_strategy,
+                                         use_dataset):
     if context.num_gpus() < num_gpus:
       self.skipTest('Not enough GPUs')
-    dataset_fn = lambda: dataset_ops.Dataset.range(100)
+    if use_dataset:
+      fn = lambda: dataset_ops.Dataset.range(100)
+    else:
+      def fn():
+        dataset = dataset_ops.Dataset.range(100)
+        it = dataset.make_one_shot_iterator()
+        return it.get_next
     expected_values = [[i+j for j in range(num_gpus)]
                        for i in range(0, 100, num_gpus)]
 
     input_fn = self._input_fn_to_test_input_context(
-        dataset_fn,
+        fn,
         expected_num_replicas_in_sync=num_gpus,
         expected_num_input_pipelines=3,
         expected_input_pipeline_id=1)  # because task_id = 1
@@ -719,6 +729,7 @@ class ParameterServerStrategyTest(
         num_gpus,
         input_fn,
         expected_values,
+        test_reinitialize=use_dataset,
         use_core_strategy=use_core_strategy)
 
   @combinations.generate(
@@ -726,16 +737,24 @@ class ParameterServerStrategyTest(
           mode=['graph'],
           num_gpus=[1, 2],
           required_gpus=1,
-          use_core_strategy=[True, False]))
-  def testMakeInputFnIteratorLocal(self, num_gpus, use_core_strategy):
+          use_core_strategy=[True, False],
+          use_dataset=[True, False]))
+  def testMakeInputFnIteratorLocal(self, num_gpus, use_core_strategy,
+                                   use_dataset):
     if context.num_gpus() < num_gpus:
       self.skipTest('Not enough GPUs')
-    dataset_fn = lambda: dataset_ops.Dataset.range(100)
+    if use_dataset:
+      fn = lambda: dataset_ops.Dataset.range(100)
+    else:
+      def fn():
+        dataset = dataset_ops.Dataset.range(100)
+        it = dataset.make_one_shot_iterator()
+        return it.get_next
     expected_values = [[i+j for j in range(num_gpus)]
                        for i in range(0, 100, num_gpus)]
 
     input_fn = self._input_fn_to_test_input_context(
-        dataset_fn,
+        fn,
         expected_num_replicas_in_sync=num_gpus,
         expected_num_input_pipelines=1,
         expected_input_pipeline_id=0)  # only one worker and pipeline for local.
@@ -745,6 +764,7 @@ class ParameterServerStrategyTest(
         num_gpus,
         input_fn,
         expected_values,
+        test_reinitialize=use_dataset,
         use_core_strategy=use_core_strategy)
 
   @combinations.generate(
diff --git a/tensorflow/contrib/distribute/python/strategy_test_lib.py b/tensorflow/contrib/distribute/python/strategy_test_lib.py
index 2e2ee92b6e..90f552eda4 100644
--- a/tensorflow/contrib/distribute/python/strategy_test_lib.py
+++ b/tensorflow/contrib/distribute/python/strategy_test_lib.py
@@ -224,7 +224,7 @@ class DistributionTestBase(test.TestCase):
         dist.extended.call_for_each_replica(_merge_call_merge_raises_fn)
 
   def _input_fn_to_test_input_context(self,
-                                      dataset_fn,
+                                      dataset_or_callable_fn,
                                       expected_num_replicas_in_sync,
                                       expected_num_input_pipelines,
                                       expected_input_pipeline_id):
@@ -248,12 +248,12 @@ class DistributionTestBase(test.TestCase):
         self.assertEqual(worker_id_counter[0], input_context.input_pipeline_id)
         worker_id_counter[0] += 1
 
-      return dataset_fn()
+      return dataset_or_callable_fn()
 
     return _input_fn
 
   def _test_input_fn_iterator(self, iterator, devices, expected_values,
-                              sess=None):
+                              sess=None, test_reinitialize=True):
     evaluate = lambda x: sess.run(x) if sess else self.evaluate(x)
     evaluate(iterator.initialize())
 
@@ -269,13 +269,14 @@ class DistributionTestBase(test.TestCase):
           [values.select_replica(r, next_element) for r in range(len(devices))])
 
     # After re-initializing the iterator, should be able to iterate again.
-    evaluate(iterator.initialize())
-
-    for expected_value in expected_values:
-      next_element = iterator.get_next()
-      computed_value = evaluate(
-          [values.select_replica(r, next_element) for r in range(len(devices))])
-      self.assertEqual(expected_value, computed_value)
+    if test_reinitialize:
+      evaluate(iterator.initialize())
+
+      for expected_value in expected_values:
+        next_element = iterator.get_next()
+        computed_value = evaluate([values.select_replica(r, next_element)
+                                   for r in range(len(devices))])
+        self.assertEqual(expected_value, computed_value)
 
   def _test_global_step_update(self, strategy):
     with strategy.scope():
diff --git a/tensorflow/python/distribute/input_lib.py b/tensorflow/python/distribute/input_lib.py
index 0a389e19a8..be6b713f04 100644
--- a/tensorflow/python/distribute/input_lib.py
+++ b/tensorflow/python/distribute/input_lib.py
@@ -182,8 +182,6 @@ class InputFunctionIterator(InputIteratorImpl):
     once on each worker.
 
     TODO(priyag): Add other replication modes.
-    TODO(priyag): Allow taking input function that returns a callable that
-    returns nest of tensors.
 
     Args:
       input_fn: Input function that returns a `tf.data.Dataset` object.
@@ -204,10 +202,14 @@ class InputFunctionIterator(InputIteratorImpl):
       worker = input_workers.worker_devices[i]
       with ops.device(worker):
         result = input_fn(ctx)
-        if not isinstance(result, dataset_ops.DatasetV2):
-          raise ValueError("input_fn must return a tf.data.Dataset.")
         devices = input_workers.compute_devices_for_worker(i)
-        iterator = _SingleWorkerDatasetIterator(result, worker, devices)
+        if isinstance(result, dataset_ops.DatasetV2):
+          iterator = _SingleWorkerDatasetIterator(result, worker, devices)
+        elif callable(result):
+          iterator = _SingleWorkerCallableIterator(result, worker, devices)
+        else:
+          raise ValueError(
+              "input_fn must return a tf.data.Dataset or a callable.")
         iterators.append(iterator)
 
     super(InputFunctionIterator, self).__init__(input_workers, iterators)
@@ -321,6 +323,26 @@ class _SingleWorkerDatasetIterator(object):
     return self._iterator.output_types
 
 
+class _SingleWorkerCallableIterator(object):
+  """Iterator for a single tensor-returning callable."""
+
+  def __init__(self, fn, worker, devices):
+    self._fn = fn
+    self._worker = worker
+    self._devices = devices
+
+  def get_next_as_list(self, name=None):
+    """Get next element from the callable."""
+    del name
+    with ops.device(self._worker):
+      data_list = [self._fn() for _ in self._devices]
+      return data_list
+
+  def initialize(self):
+    # TODO(petebu) Should this throw an exception instead?
+    return []
+
+
 # TODO(sourabhbajaj): Remove this in lieu of distributed datasets
 def _get_batched_dataset(d):
   """Get the underlying batch dataset from the dataset object."""
-- 
GitLab


From b0f2ac239009906063771409c4165ddcfda70df2 Mon Sep 17 00:00:00 2001
From: Alexey Radul <axch@google.com>
Date: Tue, 29 Jan 2019 10:23:48 -0800
Subject: [PATCH 1480/2345] Allow non-Tensors through v2 losses.

PiperOrigin-RevId: 231426607
---
 tensorflow/python/keras/losses.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index 9fbd28fcf3..aaea9162a2 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -155,8 +155,8 @@ class LossFunctionWrapper(Loss):
     Returns:
       Loss values per sample.
     """
-    y_pred = ops.convert_to_tensor(y_pred)
-    y_true = math_ops.cast(y_true, y_pred.dtype)
+    if hasattr(y_pred, 'dtype'):
+      y_true = math_ops.cast(y_true, y_pred.dtype)
     return self.fn(y_true, y_pred, **self._fn_kwargs)
 
   def get_config(self):
-- 
GitLab


From f0c49a7f457740334e9231edd11688d5662f9d2b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 29 Jan 2019 10:26:43 -0800
Subject: [PATCH 1481/2345] [XLA] Adding support for compiling EmptyOp kernels.

PiperOrigin-RevId: 231427140
---
 tensorflow/compiler/tf2xla/kernels/BUILD      |  1 +
 .../compiler/tf2xla/kernels/empty_op.cc       | 66 +++++++++++++++++++
 2 files changed, 67 insertions(+)
 create mode 100644 tensorflow/compiler/tf2xla/kernels/empty_op.cc

diff --git a/tensorflow/compiler/tf2xla/kernels/BUILD b/tensorflow/compiler/tf2xla/kernels/BUILD
index 52d2901e73..bfe668f793 100644
--- a/tensorflow/compiler/tf2xla/kernels/BUILD
+++ b/tensorflow/compiler/tf2xla/kernels/BUILD
@@ -34,6 +34,7 @@ tf_kernel_library(
         "dynamic_slice_ops.cc",
         "dynamic_stitch_op.cc",
         "elu_op.cc",
+        "empty_op.cc",
         "extract_image_patches_op.cc",
         "fake_param_op.cc",
         "fake_quantize_ops.cc",
diff --git a/tensorflow/compiler/tf2xla/kernels/empty_op.cc b/tensorflow/compiler/tf2xla/kernels/empty_op.cc
new file mode 100644
index 0000000000..00d2ce7c12
--- /dev/null
+++ b/tensorflow/compiler/tf2xla/kernels/empty_op.cc
@@ -0,0 +1,66 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// XLA-specific Empty Op.
+
+#include "tensorflow/compiler/tf2xla/type_util.h"
+#include "tensorflow/compiler/tf2xla/xla_helpers.h"
+#include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
+#include "tensorflow/compiler/tf2xla/xla_op_registry.h"
+#include "tensorflow/compiler/xla/client/lib/constants.h"
+#include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/core/framework/kernel_def_builder.h"
+#include "tensorflow/core/framework/register_types.h"
+#include "tensorflow/core/framework/tensor_shape.h"
+
+namespace tensorflow {
+namespace {
+
+class EmptyOp : public XlaOpKernel {
+ public:
+  explicit EmptyOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("dtype", &dtype_));
+    OP_REQUIRES_OK(ctx, DataTypeToPrimitiveType(dtype_, &type_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("init", &init_));
+  }
+
+  void Compile(XlaOpKernelContext* ctx) override {
+    // The output of this Op is a tensor of shape 'shape' with each
+    // element set to the default value of 'dtype'. If 'init' is false then
+    // the result values may be left undefined, though we don't do that here.
+    const TensorShape shape_shape = ctx->InputShape("shape");
+    OP_REQUIRES(
+        ctx, TensorShapeUtils::IsVector(shape_shape),
+        errors::InvalidArgument("shape must be a vector of int32, got shape ",
+                                shape_shape.DebugString()));
+
+    std::vector<int64> shape;
+    OP_REQUIRES_OK(ctx, ctx->ConstantInputAsIntVector("shape", &shape));
+
+    auto default_value = xla::Zero(ctx->builder(), type_);
+    auto result = xla::Broadcast(default_value, shape);
+    ctx->SetOutput(0, result);
+  }
+
+ private:
+  DataType dtype_;
+  xla::PrimitiveType type_;
+  bool init_;
+};
+
+REGISTER_XLA_OP(Name("Empty").CompileTimeConstantInput("shape"), EmptyOp);
+
+}  // namespace
+}  // namespace tensorflow
-- 
GitLab


From 5b16e7079aa7bb8ae6b6db9d74f21afa92bfccbf Mon Sep 17 00:00:00 2001
From: Tong Shen <endlessroad@google.com>
Date: Tue, 29 Jan 2019 10:28:24 -0800
Subject: [PATCH 1482/2345] Fix control input handling for XLA/TPU.

Issue with host loop + TPU rewrite + device loop, before cl/219724472:
a) For device loop, tf.while_loop python code selects a certain Merge node as "pivot", and will add control edges between the "pivot" and nodes in loop cond ("Less" node).
b) AddOp("Less" node) in device loop context ---> AddInnerOp("Less" node) in TPU rewrite context ---> AddOp("Less" node) in TPU rewrite context ---> _RemoveExternalControlEdges("Less" node). The _RemoveExternalControlEdges() function checks "Merge" node's control flow context (which is the device loop context), and "Less" node's control flow context (which is TPU rewrite context at the moment), so it decides "Merge" node is an external control input for "Less" node. It adds an Identity node for the "Merge", and adds control edge between the new Identity node and "Less" node. We use control flow context escaping on the new Identity node, so it does not have any control flow context other than TPU rewrite context, thus is *not* visible to device loop context.
c) At TF execution time, TF complains that the new Identity node has control edge to "Less" node, but they are not in the same while loop context: Identity node does not have any loop context, and "Less" node has device loop context.

How cl/219724472 fixed the issue: in step b), remove the control flow context escaping, so the device loop context sees the new Identity node. Now both Identity node and "Less" node belong to device loop context. But it causes another issue: if there is an control edge between host loop node and device loop node, then we get into an infinite loop: TPU rewrite context adds an Identity -> AddOp() in device loop context -> AddInnerOp() in TPU rewrite context -> AddOp() in TPU rewrite context -> TPU rewrite contet adds an Identity.

How this CL fixed the issue:
1) rollback cl/219724472;
2) in step b), do not let TPU rewrite context think Merge node is an external control input. As a result, the Identity node won't be created, and TF won't complain about the control edge between Identity node and "Less" node.

PiperOrigin-RevId: 231427455
---
 tensorflow/contrib/compiler/xla.py       | 37 ++++++++++++++++++++----
 tensorflow/contrib/tpu/python/tpu/tpu.py | 37 ++++++++++++++++++++----
 2 files changed, 64 insertions(+), 10 deletions(-)

diff --git a/tensorflow/contrib/compiler/xla.py b/tensorflow/contrib/compiler/xla.py
index 0f1be500f4..238c6ab136 100644
--- a/tensorflow/contrib/compiler/xla.py
+++ b/tensorflow/contrib/compiler/xla.py
@@ -144,6 +144,30 @@ class XLACompileContext(control_flow_ops.XLAControlFlowContext):
         logging.warning('... and %d more',
                         len(self._unsupported_ops) - _MAX_WARNING_LINES)
 
+  def _RemoveExternalControlEdges(self, op):
+    """Remove any external control dependency on this op."""
+    internal_control_inputs = []
+    external_control_inputs = []
+    for x in op.control_inputs:
+      # pylint: disable=protected-access
+      is_internal_op = False
+      ctxt = x._get_control_flow_context()
+      while ctxt is not None:
+        if ctxt == self:
+          is_internal_op = True
+          break
+        ctxt = ctxt._outer_context
+      if is_internal_op:
+        internal_control_inputs.append(x)
+      else:
+        external_control_inputs.append(x)
+      # pylint: enable=protected-access
+    # pylint: disable=protected-access
+    op._remove_all_control_inputs()
+    op._add_control_inputs(internal_control_inputs)
+    # pylint: enable=protected-access
+    return internal_control_inputs, external_control_inputs
+
   def AddOp(self, op):
     """Create op in XLACompileContext and notifies outer context recursively."""
     # pylint: disable=protected-access
@@ -193,11 +217,14 @@ class XLACompileContext(control_flow_ops.XLAControlFlowContext):
     if external_control_inputs:
       # Use an identity to pull control inputs as data inputs. Note that we
       # ignore ops which don't have outputs. TODO(phawkins): fix that.
-      external_control_inputs = [
-          array_ops.identity(x.outputs[0]).op
-          for x in external_control_inputs
-          if x.outputs
-      ]
+      with ops.control_dependencies(None):
+        self.Enter()
+        external_control_inputs = [
+            array_ops.identity(x.outputs[0]).op
+            for x in external_control_inputs
+            if x.outputs
+        ]
+        self.Exit()
       # pylint: disable=protected-access
       op._add_control_inputs(external_control_inputs)
       # pylint: enable=protected-access
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu.py b/tensorflow/contrib/tpu/python/tpu/tpu.py
index ebbccea02c..de2bfd49ec 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu.py
@@ -328,6 +328,30 @@ class TPUReplicateContext(control_flow_ops.XLAControlFlowContext):
   def HostComputeCore(self):
     return self._host_compute_core
 
+  def _RemoveExternalControlEdges(self, op):
+    """Remove any external control dependency on this op."""
+    internal_control_inputs = []
+    external_control_inputs = []
+    for x in op.control_inputs:
+      # pylint: disable=protected-access
+      is_internal_op = False
+      ctxt = x._get_control_flow_context()
+      while ctxt is not None:
+        if ctxt == self:
+          is_internal_op = True
+          break
+        ctxt = ctxt._outer_context
+      if is_internal_op:
+        internal_control_inputs.append(x)
+      else:
+        external_control_inputs.append(x)
+      # pylint: enable=protected-access
+    # pylint: disable=protected-access
+    op._remove_all_control_inputs()
+    op._add_control_inputs(internal_control_inputs)
+    # pylint: enable=protected-access
+    return internal_control_inputs, external_control_inputs
+
   def AddOp(self, op):
     # pylint: disable=protected-access
     if op.type in _BLACKLISTED_OPS:
@@ -378,11 +402,14 @@ class TPUReplicateContext(control_flow_ops.XLAControlFlowContext):
     if external_control_inputs:
       # Use an identity to pull control inputs as data inputs. Note that we
       # ignore ops which don't have outputs. TODO(phawkins): fix that.
-      external_control_inputs = [
-          array_ops.identity(x.outputs[0]).op
-          for x in external_control_inputs
-          if x.outputs
-      ]
+      with ops.control_dependencies(None):
+        self.Enter()
+        external_control_inputs = [
+            array_ops.identity(x.outputs[0]).op
+            for x in external_control_inputs
+            if x.outputs
+        ]
+        self.Exit()
       # pylint: disable=protected-access
       op._add_control_inputs(external_control_inputs)
       # pylint: enable=protected-access
-- 
GitLab


From 603e11f5e7b8e16977df27b98db187247e33e572 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Tue, 29 Jan 2019 10:36:32 -0800
Subject: [PATCH 1483/2345] Fix mispelled function name.

PiperOrigin-RevId: 231429189
---
 .../python/autograph/converters/return_statements.py   |  2 +-
 .../autograph/converters/return_statements_test.py     | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/autograph/converters/return_statements.py b/tensorflow/python/autograph/converters/return_statements.py
index 39b4df313a..c6c6c3ba5a 100644
--- a/tensorflow/python/autograph/converters/return_statements.py
+++ b/tensorflow/python/autograph/converters/return_statements.py
@@ -202,7 +202,7 @@ class ReturnStatementsTransformer(converter.Base):
   def visit_Return(self, node):
     self.state[_Return].used = True
 
-    retval = node.value if node.value else parser.parse_expresison('None')
+    retval = node.value if node.value else parser.parse_expression('None')
 
     template = """
       do_return_var_name = True
diff --git a/tensorflow/python/autograph/converters/return_statements_test.py b/tensorflow/python/autograph/converters/return_statements_test.py
index 2b160d5ac7..2942555d17 100644
--- a/tensorflow/python/autograph/converters/return_statements_test.py
+++ b/tensorflow/python/autograph/converters/return_statements_test.py
@@ -187,6 +187,16 @@ class SingleReturnTest(converter_testing.TestCase):
     self.assertTransformedEquivalent(test_fn, 2)
     self.assertTransformedEquivalent(test_fn, 4)
 
+  def test_null_return(self):
+
+    def test_fn(n):
+      if n > 4:
+        return
+      return
+
+    self.assertTransformedEquivalent(test_fn, 4)
+    self.assertTransformedEquivalent(test_fn, 5)
+
 
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 41653fee7e6b0ba1e5979c237cac3c217c941bb0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 29 Jan 2019 10:40:31 -0800
Subject: [PATCH 1484/2345] Adds Android Q types and functions to
 NeuralNetworksShim.h and nnapi_implementation*.

PiperOrigin-RevId: 231430008
---
 tensorflow/lite/nnapi/NeuralNetworksShim.h    | 465 ++++++++++++++++++
 tensorflow/lite/nnapi/NeuralNetworksTypes.h   | 129 +++++
 tensorflow/lite/nnapi/nnapi_implementation.cc |  44 +-
 tensorflow/lite/nnapi/nnapi_implementation.h  | 416 ++++++++++++++++
 .../lite/nnapi/nnapi_implementation_test.cc   |  22 +
 5 files changed, 1071 insertions(+), 5 deletions(-)

diff --git a/tensorflow/lite/nnapi/NeuralNetworksShim.h b/tensorflow/lite/nnapi/NeuralNetworksShim.h
index 3a4e15006e..1800c70685 100644
--- a/tensorflow/lite/nnapi/NeuralNetworksShim.h
+++ b/tensorflow/lite/nnapi/NeuralNetworksShim.h
@@ -255,6 +255,32 @@ inline int ANeuralNetworksModel_setOperandValue(ANeuralNetworksModel* model,
   EXECUTE_FUNCTION_RETURN(model, index, buffer, length);
 }
 
+/**
+ * Sets an operand's per channel quantization parameters.
+ *
+ * Sets parameters required by a tensor of type
+ * {@link ANEURALNETWORKS_TENSOR_QUANT8_SYMM_PER_CHANNEL}.
+ * This function must be called for every tensor of type
+ * {@link ANEURALNETWORKS_TENSOR_QUANT8_SYMM_PER_CHANNEL} before
+ * calling {@link ANeuralNetworksModel_finish}.
+ *
+ * Available since API level 29.
+ *
+ * @param model The model to be modified.
+ * @param index The index of the model operand we're setting.
+ * @param channelQuant The per channel quantization parameters for the operand.
+ *                    No memory in this struct needs to outlive the call to
+ *                    this function.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful.
+ */
+inline int ANeuralNetworksModel_setOperandSymmPerChannelQuantParams(
+    ANeuralNetworksModel* model, int32_t index,
+    const ANeuralNetworksSymmPerChannelQuantParams* channelQuant) {
+  LOAD_FUNCTION(ANeuralNetworksModel_setOperandSymmPerChannelQuantParams);
+  EXECUTE_FUNCTION_RETURN(model, index, channelQuant);
+}
+
 /**
  * Sets an operand to a value stored in a memory object.
  *
@@ -686,6 +712,445 @@ inline void ANeuralNetworksEvent_free(ANeuralNetworksEvent* event) {
   EXECUTE_FUNCTION(event);
 }
 
+/**
+ * Get the number of available devices.
+ *
+ * @param numDevices Used to return the number of devices.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful.
+ *
+ * Available since API level 29.
+ */
+inline int ANeuralNetworks_getDeviceCount(uint32_t* numDevices) {
+  LOAD_FUNCTION(ANeuralNetworks_getDeviceCount);
+  EXECUTE_FUNCTION_RETURN(numDevices);
+}
+
+/**
+ * Get the representation of the specified device.
+ *
+ * @param devIndex The index of the specified device. Must be less than the
+ *                 number of available devices.
+ * @param device The representation of the specified device.
+ *               The same representation will always be returned for the
+ *               specified device.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful.
+ *
+ * Available since API level 29.
+ */
+
+inline int ANeuralNetworks_getDevice(uint32_t devIndex,
+                                     ANeuralNetworksDevice** device) {
+  LOAD_FUNCTION(ANeuralNetworks_getDevice);
+  EXECUTE_FUNCTION_RETURN(devIndex, device);
+}
+
+/**
+ * Get the name of the specified device.
+ *
+ * @param device The representation of the specified device.
+ * @param name   The returned name of the specified device. The name will be in
+ *               UTF-8 and will be null-terminated. It will be recognizable as a
+ *               known device name rather than a cryptic string. For devices
+ *               with API level 29 and above, the format of the name is
+ *               {VENDOR}-{DEVICE}, e.g. “google-ipu”. For devices with feature
+ *               level 28 or lower, the name will always be “unknown-device”.
+ *               The name will remain valid for the duration of the application.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful.
+ *
+ * Available since API level 29.
+ */
+inline int ANeuralNetworksDevice_getName(const ANeuralNetworksDevice* device,
+                                         const char** name) {
+  LOAD_FUNCTION(ANeuralNetworksDevice_getName);
+  EXECUTE_FUNCTION_RETURN(device, name);
+}
+
+/**
+ * Get the version of the driver implementation of the specified device.
+ *
+ * It’s the responsibility of the driver implementor to insure that this version
+ * string uniquely distinguishes this implementation from all previous
+ * implementations.
+ *
+ * This version string must not be confused with the feature level which is
+ * solely defined by {@link ANeuralNetworksDevice_getFeatureLevel}. There is no
+ * implicit ordering of the versions. For example, it is not possible to filter
+ * all drivers older than a certain version.
+ *
+ * Application developers may use this version string to avoid or prefer
+ * specific driver implementations. For example, an application may want to do
+ * so because:
+ *     - A specific version of the driver does not provide the required
+ * performance, perhaps because of a performance regression.
+ *     - A specific version of the driver has a bug or returns results that
+ * don’t match the minimum precision requirement for the application.
+ *
+ * @param device  The representation of the specified device.
+ * @param version The returned version string of the driver for the specified
+ *                device. The string will be in UTF-8 and will be
+ *                null-terminated. For devices with feature level 28 or lower,
+ *                "UNKNOWN" will be returned. The version string will remain
+ *                valid for the duration of the application.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful.
+ *
+ * Available since API level 29.
+ */
+inline int ANeuralNetworksDevice_getVersion(const ANeuralNetworksDevice* device,
+                                            const char** version) {
+  LOAD_FUNCTION(ANeuralNetworksDevice_getVersion);
+  EXECUTE_FUNCTION_RETURN(device, version);
+}
+
+/**
+ * Get the supported NNAPI version of the specified device.
+ *
+ * Each device has a supported feature level, which is the most advanced feature
+ * this driver implements. For example, if the driver implements the features
+ * introduced in Android P, but does not implement the features introduced after
+ * Android P, the value would be 28. Developers could decide whether or not the
+ * specified device should be used for a Model that has certain feature
+ * requirements.
+ *
+ * @param device       The representation of the specified device.
+ * @param featureLevel The API level of the most advanced feature this driver
+ *                     implements.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful.
+ *
+ * Available since API level 29.
+ */
+inline int ANeuralNetworksDevice_getFeatureLevel(
+    const ANeuralNetworksDevice* device, int64_t* featureLevel) {
+  LOAD_FUNCTION(ANeuralNetworksDevice_getFeatureLevel);
+  EXECUTE_FUNCTION_RETURN(device, featureLevel);
+}
+
+/**
+ * Get the supported operations for a specified set of devices. If multiple
+ * devices are selected, the supported operation list is a union of supported
+ * operations of all selected devices.
+ *
+ * @param model        The model to be queried.
+ * @param devices      The set of devices. Must not contain duplicates.
+ * @param numDevices   The number of devices in the set.
+ * @param supportedOps The boolean array to be filled. True means supported. The
+ *                     size of the boolean array must be at least as large as
+ *                     the number of operations in the model. The order of
+ *                     elements in the supportedOps array matches the order in
+ *                     which the corresponding operations were added to the
+ *                     model.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful.
+ *
+ * Available since API level 29.
+ */
+inline int ANeuralNetworksModel_getSupportedOperationsForDevices(
+    const ANeuralNetworksModel* model,
+    const ANeuralNetworksDevice* const* devices, uint32_t numDevices,
+    bool* supportedOps) {
+  LOAD_FUNCTION(ANeuralNetworksModel_getSupportedOperationsForDevices);
+  EXECUTE_FUNCTION_RETURN(model, devices, numDevices, supportedOps);
+}
+
+/**
+ * Create a {@link ANeuralNetworksCompilation} to compile the given model for a
+ * specified set of devices. If more than one device is specified, the
+ * compilation will distribute the workload automatically across the devices.
+ * The model must be fully supported by the specified set of devices. This means
+ * that ANeuralNetworksModel_getSupportedOperationsForDevices() must have
+ * returned true for every operation for that model/devices pair.
+ *
+ * @param model       The {@link ANeuralNetworksModel} to be compiled.
+ * @param devices     The set of devices. Must not contain duplicates.
+ * @param numDevices  The number of devices in the set.
+ * @param compilation The newly created object or NULL if unsuccessful.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA
+ *         if the model is invalid.
+ *
+ * Available since API level 29.
+ */
+inline int ANeuralNetworksCompilation_createForDevices(
+    ANeuralNetworksModel* model, const ANeuralNetworksDevice* const* devices,
+    uint32_t numDevices, ANeuralNetworksCompilation** compilation) {
+  LOAD_FUNCTION(ANeuralNetworksCompilation_createForDevices);
+  EXECUTE_FUNCTION_RETURN(model, devices, numDevices, compilation);
+}
+
+/**
+ * Sets the compilation caching signature and the cache directory.
+ *
+ * Provides optional caching information to the runtime for faster repeated
+ * compilation.
+ *
+ * See {@link ANeuralNetworksCompilation} for information on multithreaded
+ * usage.
+ *
+ * @param compilation The compilation to be modified.
+ * @param cacheDir The cache directory to store and retrieve caching data. It is
+ *                 recommended to use the code_cache provided by the Android
+ *                 runtime. If not using the code_cache, the user should choose
+ *                 a directory local to the application, and is responsible to
+ *                 manage and clean the cache entries.
+ * @param token The token provided by the user to specify a model, must be of
+ *              length ANEURALNETWORKS_BYTE_SIZE_OF_CACHE_TOKEN. The user should
+ *              ensure that the token is unique to a model within the
+ *              application. The NNAPI runtime will not detected token
+ *              collisions. If there is a collision, the compilation outcome may
+ *              be incorrect without notifying with error.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful.
+ *
+ * Available since API level 29.
+ */
+inline int ANeuralNetworksCompilation_setCaching(
+    ANeuralNetworksCompilation* compilation, const char* cacheDir,
+    const uint8_t* token) {
+  LOAD_FUNCTION(ANeuralNetworksCompilation_setCaching);
+  EXECUTE_FUNCTION_RETURN(compilation, cacheDir, token);
+}
+
+/**
+ * Schedule synchronous evaluation of the execution.
+ *
+ * <p>Schedules synchronous evaluation of the execution. Returns once the
+ * execution has completed and the outputs are ready to be consumed.
+ * </p>
+ *
+ * See {@link ANeuralNetworksExecution} for information on multithreaded usage.
+ *
+ * See {@link ANeuralNetworksExecution_startCompute} for asynchronous execution.
+ * Synchronous execution incurs lower overhead than asynchronous execution.
+ *
+ * Available since API level 29.
+ *
+ * @param execution The execution to be scheduled and executed.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if the execution completed normally.
+ *         ANEURALNETWORKS_UNMAPPABLE if the execution input or output memory
+ *         cannot be properly mapped.
+ */
+inline int ANeuralNetworksExecution_compute(
+    ANeuralNetworksExecution* execution) {
+  LOAD_FUNCTION(ANeuralNetworksExecution_compute);
+  EXECUTE_FUNCTION_RETURN(execution);
+}
+
+/**
+ * Get the dimensional information of the specified output operand of the model
+ * of the
+ * {@link ANeuralNetworksExecution}.
+ *
+ * On asynchronous execution initiated by {@link
+ * ANeuralNetworksExecution_startCompute},
+ * {@link ANeuralNetworksEvent_wait} must be called prior to this function to
+ * recuperate the resources used by the execution.
+ *
+ * @param execution The execution to be queried.
+ * @param index The index of the output argument we are querying. It is
+ *              an index into the lists passed to
+ *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is not
+ *              the index associated with {@link
+ * ANeuralNetworksModel_addOperand}.
+ * @param rank The rank of the output operand.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful,
+ * ANEURALNETWORKS_OUTPUT_INSUFFICIENT_SIZE if the target output is provided an
+ * insufficient buffer at execution time, ANEURALNETWORKS_BAD_DATA if the index
+ * is invalid.
+ *
+ * Available since API level 29.
+ */
+inline int ANeuralNetworksExecution_getOutputOperandRank(
+    ANeuralNetworksExecution* execution, int32_t index, uint32_t* rank) {
+  LOAD_FUNCTION(ANeuralNetworksExecution_getOutputOperandRank);
+  EXECUTE_FUNCTION_RETURN(execution, index, rank);
+}
+
+/**
+ * Get the dimensional information of the specified output operand of the model
+ * of the
+ * {@link ANeuralNetworksExecution}. The target output operand cannot be a
+ * scalar.
+ *
+ * On asynchronous execution initiated by
+ * {@link ANeuralNetworksExecution_startCompute},
+ * {@link ANeuralNetworksEvent_wait} must be called prior to this function to
+ * recuperate the resources used by the execution.
+ *
+ * @param execution The execution to be queried.
+ * @param index The index of the output argument we are querying. It is an index
+ *              into the lists passed to
+ *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is not
+ *              the index associated with
+ *              {@link ANeuralNetworksModel_addOperand}.
+ * @param dimensions The dimension array to be filled. The size of the array
+ *                   must be exactly as large as the rank of the output operand
+ *                   to be queried in the model.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful,
+ * ANEURALNETWORKS_OUTPUT_INSUFFICIENT_SIZE if the target output is provided an
+ * insufficient buffer at execution time, ANEURALNETWORKS_BAD_DATA if the index
+ * is invalid or if the target is a scalar.
+ *
+ * Available since API level 29.
+ */
+inline int ANeuralNetworksExecution_getOutputOperandDimensions(
+    ANeuralNetworksExecution* execution, int32_t index, uint32_t* dimensions) {
+  LOAD_FUNCTION(ANeuralNetworksExecution_getOutputOperandDimensions);
+  EXECUTE_FUNCTION_RETURN(execution, index, dimensions);
+}
+
+/**
+ * Create a {@link ANeuralNetworksBurst} to apply the given compilation.
+ * This only creates the burst object. Computation is only performed once
+ * {@link ANeuralNetworksExecution_burstCompute} is invoked with a valid
+ * {@link ANeuralNetworksExecution} and {@link ANeuralNetworksBurst}.
+ *
+ * <p>The provided compilation must outlive the burst object.</p>
+ *
+ * Available since API level 29.
+ *
+ * @param compilation The {@link ANeuralNetworksCompilation} to be evaluated.
+ * @param burst The newly created object or NULL if unsuccessful.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA
+ *         if the compilation is invalid.
+ */
+inline int ANeuralNetworksBurst_create(ANeuralNetworksCompilation* compilation,
+                                       ANeuralNetworksBurst** burst) {
+  LOAD_FUNCTION(ANeuralNetworksBurst_create);
+  EXECUTE_FUNCTION_RETURN(compilation, burst);
+}
+
+/**
+ * Destroys the burst object.
+ *
+ * Available since API level 29.
+ *
+ * @param burst The burst object to be destroyed. Passing NULL is acceptable and
+ *              results in no operation.
+ */
+inline void ANeuralNetworksBurst_free(ANeuralNetworksBurst* burst) {
+  LOAD_FUNCTION(ANeuralNetworksBurst_free);
+  EXECUTE_FUNCTION(burst);
+}
+
+/**
+ * Schedule synchronous evaluation of the execution on a burst object.
+ *
+ * <p>Schedules synchronous evaluation of the execution. Returns once the
+ * execution has completed and the outputs are ready to be consumed.</p>
+ *
+ * <p>There must be at most one {@link ANeuralNetworksExecution} processing at
+ * any given time for any given burst object. Any
+ * {@link ANeuralNetworksExecution} launched before the previous has finished
+ * will result in ANEURALNETWORKS_BAD_STATE.</p>
+ *
+ * Available since API level 29.
+ *
+ * @param burst The burst object to execute on.
+ * @param execution The execution to be scheduled and executed. The execution
+ *                  must be created from the same {@link
+ *                  ANeuralNetworksCompilation} as the burst object.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if the execution completed normally.
+ */
+inline int ANeuralNetworksExecution_burstCompute(
+    ANeuralNetworksExecution* execution, ANeuralNetworksBurst* burst) {
+  LOAD_FUNCTION(ANeuralNetworksExecution_burstCompute);
+  EXECUTE_FUNCTION_RETURN(execution, burst);
+}
+
+/**
+ * Creates a shared memory object from an AHardwareBuffer handle.
+ *
+ * If the shared memory is backed by an AHardwareBuffer of
+ * AHARDWAREBUFFER_FORMAT_BLOB format, it can be used the same way as shared
+ * memory created from a file handle. See
+ * {@link ANeuralNetworksMemory} for a description on how to use this shared
+ * memory.
+ *
+ * If the shared memory is backed by an AHardwareBuffer of a format other than
+ * AHARDWAREBUFFER_FORMAT_BLOB, it can only be used for Model inputs and
+ * outputs. When calling {@link ANeuralNetworksExecution_setInputFromMemory} or
+ * {@link ANeuralNetworksExecution_setOutputFromMemory} with the shared memory,
+ * both offset and length must be set to zero and the entire memory region will
+ * be associated with the specified input or output operand. There is no
+ * guarantee that an arbitrary AHardwareBuffer_Format and
+ * AHardwareBuffer_UsageFlags combination can be used by arbitrary devices. The
+ * execution will fail if selected set of devices cannot consume the buffer.
+ *
+ * Calling {@link ANeuralNetworksModel_setOperandValueFromMemory} with shared
+ * memory backed by an AHardwareBuffer of a format other than
+ * AHARDWAREBUFFER_FORMAT_BLOB is disallowed.
+ *
+ * TODO(miaowang): add documentation about intended usage with introspection
+ * API.
+ *
+ * Available since API level 29.
+ *
+ * @param ahwb The AHardwareBuffer handle.
+ * @param memory The memory object to be created.
+ *               Set to NULL if unsuccessful.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if the request completed normally.
+ *
+ * @see AHardwareBuffer
+ */
+inline int ANeuralNetworksMemory_createFromAHardwareBuffer(
+    const AHardwareBuffer* ahwb, ANeuralNetworksMemory** memory) {
+  LOAD_FUNCTION(ANeuralNetworksMemory_createFromAHardwareBuffer);
+  EXECUTE_FUNCTION_RETURN(ahwb, memory);
+}
+
+/**
+ * Specifies whether duration of the {@link ANeuralNetworksExecution} is to be
+ * measured. By default, duration is not measured.
+ *
+ * The {@link ANeuralNetworksExecution} must have been created with
+ * {@link ANeuralNetworksCompilation_createForDevices} with numDevices = 1.
+ *
+ * See {@link ANeuralNetworksExecution} for information on multithreaded usage.
+ *
+ * Available since API level 29.
+ *
+ * @param execution The execution to be modified.
+ * @param measure 'true' if duration is to be measured, 'false' if not.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful.
+ */
+inline int ANeuralNetworksExecution_setMeasureTiming(
+    ANeuralNetworksExecution* execution, bool measure) {
+  LOAD_FUNCTION(ANeuralNetworksExecution_setMeasureTiming);
+  EXECUTE_FUNCTION_RETURN(execution, measure);
+}
+
+/**
+ * Get the time spent in the specified {@link ANeuralNetworksExecution}, in
+ * nanoseconds. The execution must have completed.
+ *
+ * @param execution The execution to be queried.
+ * @param durationCode The measurement to be queried, specified by {@link
+ * DurationCode}.
+ * @param duration The returned duration. If no measurement was requested by
+ *                 {@link ANeuralNetworksExecution_setMeasureTiming}, or for
+ * some other reason the duration is not available, UINT64_MAX will be returned.
+ *                 A particular device need not support any given measurement.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful.
+ */
+inline int ANeuralNetworksExecution_getDuration(
+    const ANeuralNetworksExecution* execution, int32_t durationCode,
+    uint64_t* duration) {
+  LOAD_FUNCTION(ANeuralNetworksExecution_getDuration);
+  EXECUTE_FUNCTION_RETURN(execution, durationCode, duration);
+}
+
 /**/
 
 #endif  // TENSORFLOW_LITE_NNAPI_NEURALNETWORKSSHIM_H_
diff --git a/tensorflow/lite/nnapi/NeuralNetworksTypes.h b/tensorflow/lite/nnapi/NeuralNetworksTypes.h
index 9291391491..573500dbb3 100644
--- a/tensorflow/lite/nnapi/NeuralNetworksTypes.h
+++ b/tensorflow/lite/nnapi/NeuralNetworksTypes.h
@@ -18,6 +18,8 @@ limitations under the License.
 #include <stdint.h>
 #include <stdio.h>
 
+typedef struct AHardwareBuffer AHardwareBuffer;
+
 // NN api types based on NNAPI header file
 // https://developer.android.com/ndk/reference/group/neural-networks
 
@@ -241,6 +243,53 @@ typedef struct ANeuralNetworksCompilation ANeuralNetworksCompilation;
  */
 typedef struct ANeuralNetworksExecution ANeuralNetworksExecution;
 
+/**
+ * Parameters for ANEURALNETWORKS_TENSOR_QUANT8_SYMM_PER_CHANNEL operand.
+ */
+typedef struct ANeuralNetworksSymmPerChannelQuantParams {
+  /* The index of the channel dimension. */
+  uint32_t channelDim;
+  /** The size of the scale array. Should be equal to dimension[channelDim] of
+   * the Operand. */
+  uint32_t scaleCount;
+  /** The array of scaling values for each channel. Each value must be greater
+   * than zero. */
+  const float* scales;
+} ANeuralNetworksSymmPerChannelQuantParams;
+
+/**
+ * ANeuralNetworksBurst is an opaque type that can be used to reduce the latency
+ * of a rapid sequence of executions. It will likely cause overhead if only used
+ * for a single execution.
+ *
+ * ANeuralNetworksBurst serves as a context object for any number of inferences
+ * using {@link ANeuralNetworksExecution} objects. An ANeuralNetworksBurst
+ * object and the {@link ANeuralNetworksExecution} objects used with it must all
+ * have been created from the same {@link ANeuralNetworksCompilation} object.
+ *
+ * This object is also used as a hint to drivers, providing insight to the
+ * lifetime of a rapid sequence of executions. For example, a driver may choose
+ * to increase the clock frequency of its accelerator for the lifetime of a
+ * burst object.
+ *
+ * <p>To use:<ul>
+ *    <li>Create a new burst object by calling the
+ *        {@link ANeuralNetworksBurst_create} function.</li>
+ *    <li>For each execution:</li><ul>
+ *        <li>Create {@link ANeuralNetworksExecution} and configure its
+ *            properties (see {@link ANeuralNetworksExecution} for
+ * details).</li> <li>Apply the model synchronously with
+ *            {@link ANeuralNetworksExecution_burstCompute}, reusing the same
+ *            {@link ANeuralNetworksBurst} with the new
+ *            {@link ANeuralNetworksExecution}.</li>
+ *        <li>Use and free the {@link ANeuralNetworksExecution}.</li></ul>
+ *    <li>Destroy the burst with
+ *        {@link ANeuralNetworksBurst_free}.</li></ul></p>
+ *
+ * Available since API level 29.
+ */
+typedef struct ANeuralNetworksBurst ANeuralNetworksBurst;
+
 /**
  * ANeuralNetworksOperandType describes the type of an operand.
  * This structure is used to describe both scalars and tensors.
@@ -268,6 +317,16 @@ typedef struct ANeuralNetworksEvent ANeuralNetworksEvent;
 
 typedef int32_t ANeuralNetworksOperationType;
 
+/**
+ * ANeuralNetworksDevice is an opaque type that represents a device.
+ *
+ * This type is used to query basic properties and supported operations of the
+ * corresponding device, and control which device(s) a model is to be run on.
+ *
+ * Available since API level 29.
+ */
+typedef struct ANeuralNetworksDevice ANeuralNetworksDevice;
+
 // nn api function types
 
 typedef int (*ANeuralNetworksMemory_createFromFd_fn)(
@@ -301,6 +360,10 @@ typedef int (*ANeuralNetworksModel_setOperandValue_fn)(
     ANeuralNetworksModel* model, int32_t index, const void* buffer,
     size_t length);
 
+typedef int (*ANeuralNetworksModel_setOperandSymmPerChannelQuantParams_fn)(
+    ANeuralNetworksModel* model, int32_t index,
+    const ANeuralNetworksSymmPerChannelQuantParams* channelQuant);
+
 typedef int (*ANeuralNetworksModel_setOperandValueFromMemory_fn)(
     ANeuralNetworksModel* model, int32_t index,
     const ANeuralNetworksMemory* memory, size_t offset, size_t length);
@@ -351,4 +414,70 @@ typedef void (*ANeuralNetworksEvent_free_fn)(ANeuralNetworksEvent* event);
 
 typedef int (*ASharedMemory_create_fn)(const char* name, size_t size);
 
+typedef int (*ANeuralNetworks_getDeviceCount_fn)(uint32_t* numDevices);
+
+typedef int (*ANeuralNetworks_getDevice_fn)(uint32_t devIndex,
+                                            ANeuralNetworksDevice** device);
+
+typedef int (*ANeuralNetworksDevice_getName_fn)(
+    const ANeuralNetworksDevice* device, const char** name);
+
+typedef int (*ANeuralNetworksDevice_getType_fn)(
+    const ANeuralNetworksDevice* device, int32_t* type);
+
+typedef int (*ANeuralNetworksDevice_getVersion_fn)(
+    const ANeuralNetworksDevice* device, const char** version);
+
+typedef int (*ANeuralNetworksDevice_getFeatureLevel_fn)(
+    const ANeuralNetworksDevice* device, int64_t* featureLevel);
+
+typedef int (*ANeuralNetworksModel_getSupportedOperationsForDevices_fn)(
+    const ANeuralNetworksModel* model,
+    const ANeuralNetworksDevice* const* devices, uint32_t numDevices,
+    bool* supportedOps);
+
+typedef int (*ANeuralNetworksCompilation_createForDevices_fn)(
+    ANeuralNetworksModel* model, const ANeuralNetworksDevice* const* devices,
+    uint32_t numDevices, ANeuralNetworksCompilation** compilation);
+
+typedef int (*ANeuralNetworksCompilation_setCaching_fn)(
+    ANeuralNetworksCompilation* compilation, const char* cacheDir,
+    const uint8_t* token);
+
+typedef int (*ANeuralNetworksExecution_compute_fn)(
+    ANeuralNetworksExecution* execution);
+
+typedef int (*ANeuralNetworksExecution_getOutputOperandRank_fn)(
+    ANeuralNetworksExecution* execution, int32_t index, uint32_t* rank);
+
+typedef int (*ANeuralNetworksExecution_getOutputOperandDimensions_fn)(
+    ANeuralNetworksExecution* execution, int32_t index, uint32_t* dimensions);
+
+typedef int (*ANeuralNetworksBurst_create_fn)(
+    ANeuralNetworksCompilation* compilation, ANeuralNetworksBurst** burst);
+
+typedef void (*ANeuralNetworksBurst_free_fn)(ANeuralNetworksBurst* burst);
+
+typedef int (*ANeuralNetworksExecution_burstCompute_fn)(
+    ANeuralNetworksExecution* execution, ANeuralNetworksBurst* burst);
+
+typedef int (*ANeuralNetworksMemory_createFromAHardwareBuffer_fn)(
+    const AHardwareBuffer* ahwb, ANeuralNetworksMemory** memory);
+
+typedef int (*ANeuralNetworksExecution_setMeasureTiming_fn)(
+    ANeuralNetworksExecution* execution, bool measure);
+
+typedef enum {
+  // Execution time on hardware (not driver, which runs on host processor).
+  ANEURALNETWORKS_DURATION_ON_HARDWARE = 0,
+  // Execution time in driver (including time on hardware).  Excludes overhead
+  // such as that of the runtime itself and the IPC needed for the runtime to
+  // communicate with the driver.
+  ANEURALNETWORKS_DURATION_IN_DRIVER = 1,
+} DurationCode;
+
+typedef int (*ANeuralNetworksExecution_getDuration_fn)(
+    const ANeuralNetworksExecution* execution, int32_t durationCode,
+    uint64_t* duration);
+
 #endif  // TENSORFLOW_LITE_NNAPI_NEURALNETWORKSTYPES_H_
diff --git a/tensorflow/lite/nnapi/nnapi_implementation.cc b/tensorflow/lite/nnapi/nnapi_implementation.cc
index e8b9aed422..36301f1a83 100644
--- a/tensorflow/lite/nnapi/nnapi_implementation.cc
+++ b/tensorflow/lite/nnapi/nnapi_implementation.cc
@@ -51,12 +51,12 @@ int32_t GetAndroidSdkVersion() {
 }
 #endif  // __ANDROID__
 
-void* LoadFunction(void* handle, const char* name) {
+void* LoadFunction(void* handle, const char* name, bool optional) {
   if (handle == nullptr) {
     return nullptr;
   }
   void* fn = dlsym(handle, name);
-  if (fn == nullptr) {
+  if (fn == nullptr && !optional) {
     NNAPI_LOG("nnapi error: unable to open function %s", name);
   }
   return fn;
@@ -78,8 +78,13 @@ int ASharedMemory_create(const char* name, size_t size) {
 }
 #endif  // __ANDROID__
 
-#define LOAD_FUNCTION(handle, name) \
-  nnapi.name = reinterpret_cast<name##_fn>(LoadFunction(handle, #name));
+#define LOAD_FUNCTION(handle, name)         \
+  nnapi.name = reinterpret_cast<name##_fn>( \
+      LoadFunction(handle, #name, /*optional*/ false));
+
+#define LOAD_FUNCTION_OPTIONAL(handle, name) \
+  nnapi.name = reinterpret_cast<name##_fn>(  \
+      LoadFunction(handle, #name, /*optional*/ true));
 
 const NnApi LoadNnApi() {
   NnApi nnapi = {};
@@ -117,6 +122,9 @@ const NnApi LoadNnApi() {
   LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_finish);
   LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_addOperand);
   LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_setOperandValue);
+  LOAD_FUNCTION_OPTIONAL(
+      libneuralnetworks,
+      ANeuralNetworksModel_setOperandSymmPerChannelQuantParams);
   LOAD_FUNCTION(libneuralnetworks,
                 ANeuralNetworksModel_setOperandValueFromMemory);
   LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_addOperation);
@@ -143,7 +151,33 @@ const NnApi LoadNnApi() {
 #else
   nnapi.ASharedMemory_create = ASharedMemory_create;
 #endif  // __ANDROID__
-
+  LOAD_FUNCTION_OPTIONAL(libneuralnetworks, ANeuralNetworks_getDeviceCount);
+  LOAD_FUNCTION_OPTIONAL(libneuralnetworks, ANeuralNetworks_getDevice);
+  LOAD_FUNCTION_OPTIONAL(libneuralnetworks, ANeuralNetworksDevice_getName);
+  LOAD_FUNCTION_OPTIONAL(libneuralnetworks, ANeuralNetworksDevice_getVersion);
+  LOAD_FUNCTION_OPTIONAL(libneuralnetworks,
+                         ANeuralNetworksDevice_getFeatureLevel);
+  LOAD_FUNCTION_OPTIONAL(libneuralnetworks,
+                         ANeuralNetworksModel_getSupportedOperationsForDevices);
+  LOAD_FUNCTION_OPTIONAL(libneuralnetworks,
+                         ANeuralNetworksCompilation_createForDevices);
+  LOAD_FUNCTION_OPTIONAL(libneuralnetworks,
+                         ANeuralNetworksCompilation_setCaching);
+  LOAD_FUNCTION_OPTIONAL(libneuralnetworks, ANeuralNetworksExecution_compute);
+  LOAD_FUNCTION_OPTIONAL(libneuralnetworks,
+                         ANeuralNetworksExecution_getOutputOperandRank);
+  LOAD_FUNCTION_OPTIONAL(libneuralnetworks,
+                         ANeuralNetworksExecution_getOutputOperandDimensions);
+  LOAD_FUNCTION_OPTIONAL(libneuralnetworks, ANeuralNetworksBurst_create);
+  LOAD_FUNCTION_OPTIONAL(libneuralnetworks, ANeuralNetworksBurst_free);
+  LOAD_FUNCTION_OPTIONAL(libneuralnetworks,
+                         ANeuralNetworksExecution_burstCompute);
+  LOAD_FUNCTION_OPTIONAL(libneuralnetworks,
+                         ANeuralNetworksMemory_createFromAHardwareBuffer);
+  LOAD_FUNCTION_OPTIONAL(libneuralnetworks,
+                         ANeuralNetworksExecution_setMeasureTiming);
+  LOAD_FUNCTION_OPTIONAL(libneuralnetworks,
+                         ANeuralNetworksExecution_getDuration);
   return nnapi;
 }
 
diff --git a/tensorflow/lite/nnapi/nnapi_implementation.h b/tensorflow/lite/nnapi/nnapi_implementation.h
index 82d7cc75c1..66a36dbbc3 100644
--- a/tensorflow/lite/nnapi/nnapi_implementation.h
+++ b/tensorflow/lite/nnapi/nnapi_implementation.h
@@ -173,6 +173,29 @@ struct NnApi {
                                               int32_t index, const void* buffer,
                                               size_t length);
 
+  /**
+   * Sets an operand's per channel quantization parameters.
+   *
+   * Sets parameters required by a tensor of type
+   * {@link ANEURALNETWORKS_TENSOR_QUANT8_SYMM_PER_CHANNEL}.
+   * This function must be called for every tensor of type
+   * {@link ANEURALNETWORKS_TENSOR_QUANT8_SYMM_PER_CHANNEL} before
+   * calling {@link ANeuralNetworksModel_finish}.
+   *
+   * Available since API level 29.
+   *
+   * @param model The model to be modified.
+   * @param index The index of the model operand we're setting.
+   * @param channelQuant The per channel quantization parameters for the
+   *                     operand. No memory in this struct needs to outlive the
+   *                     call to this function.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   */
+  int (*ANeuralNetworksModel_setOperandSymmPerChannelQuantParams)(
+      ANeuralNetworksModel* model, int32_t index,
+      const ANeuralNetworksSymmPerChannelQuantParams* channelQuant);
+
   /**
    * Sets an operand to a value stored in a memory object.
    *
@@ -569,6 +592,399 @@ struct NnApi {
   // which was added in 8.1.
   int (*ASharedMemory_create)(const char* name, size_t size);
 
+  /**
+   * Get the number of available devices.
+   *
+   * @param numDevices Used to return the number of devices.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   *
+   * Available since API level 29.
+   */
+  int (*ANeuralNetworks_getDeviceCount)(uint32_t* numDevices);
+
+  /**
+   * Get the representation of the specified device.
+   *
+   * @param devIndex The index of the specified device. Must be less than the
+   *                 number of available devices.
+   * @param device The representation of the specified device.
+   *               The same representation will always be returned for the
+   *               specified device.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   *
+   * Available since API level 29.
+   */
+
+  int (*ANeuralNetworks_getDevice)(uint32_t devIndex,
+                                   ANeuralNetworksDevice** device);
+
+  /**
+   * Get the name of the specified device.
+   *
+   * @param device The representation of the specified device.
+   * @param name The returned name of the specified device. The name will be
+   *             in UTF-8 and will be null-terminated. It will be recognizable
+   *             as a known device name rather than a cryptic string. For
+   *             devices with API level 29 and above, the format of the name is
+   *             {VENDOR}-{DEVICE}, e.g. “google-ipu”. For devices with feature
+   *             level 28 or lower, the name will always be “unknown-device”.
+   *             The name will remain valid for the duration of the application.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   *
+   * Available since API level 29.
+   */
+  int (*ANeuralNetworksDevice_getName)(const ANeuralNetworksDevice* device,
+                                       const char** name);
+
+  /**
+   * Get the version of the driver implementation of the specified device.
+   *
+   * It’s the responsibility of the driver implementor to insure that this
+   * version string uniquely distinguishes this implementation from all previous
+   * implementations.
+   *
+   * This version string must not be confused with the feature level which is
+   * solely defined by {@link ANeuralNetworksDevice_getFeatureLevel}. There is
+   * no implicit ordering of the versions. For example, it is not possible to
+   * filter all drivers older than a certain version.
+   *
+   * Application developers may use this version string to avoid or prefer
+   * specific driver implementations. For example, an application may want to do
+   * so because:
+   *     - A specific version of the driver does not provide the required
+   * performance, perhaps because of a performance regression.
+   *     - A specific version of the driver has a bug or returns results that
+   * don’t match the minimum precision requirement for the application.
+   *
+   * @param device  The representation of the specified device.
+   * @param version The returned version string of the driver for the specified
+   *                device. The string will be in UTF-8 and will be
+   *                null-terminated. For devices with feature level 28 or lower,
+   *                "UNKNOWN" will be returned. The version string will remain
+   *                valid for the duration of the application.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   *
+   * Available since API level 29.
+   */
+  int (*ANeuralNetworksDevice_getVersion)(const ANeuralNetworksDevice* device,
+                                          const char** version);
+
+  /**
+   * Get the supported NNAPI version of the specified device.
+   *
+   * Each device has a supported feature level, which is the most advanced
+   * feature this driver implements. For example, if the driver implements the
+   * features introduced in Android P, but does not implement the features
+   * introduced after Android P, the value would be 28. Developers could decide
+   * whether or not the specified device should be used for a Model that has
+   * certain feature requirements.
+   *
+   * @param device       The representation of the specified device.
+   * @param featureLevel The API level of the most advanced feature this driver
+   *                     implements.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   *
+   * Available since API level 29.
+   */
+  int (*ANeuralNetworksDevice_getFeatureLevel)(
+      const ANeuralNetworksDevice* device, int64_t* featureLevel);
+
+  /**
+   * Get the supported operations for a specified set of devices. If multiple
+   * devices are selected, the supported operation list is a union of supported
+   * operations of all selected devices.
+   *
+   * @param model        The model to be queried.
+   * @param devices      The set of devices. Must not contain duplicates.
+   * @param numDevices   The number of devices in the set.
+   * @param supportedOps The boolean array to be filled. True means supported.
+   *                     The size of the boolean array must be at least as large
+   *                     as the number of operations in the model. The order of
+   *                     elements in the supportedOps array matches the order in
+   *                     which the corresponding operations were added to the
+   *                     model.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   *
+   * Available since API level 29.
+   */
+  int (*ANeuralNetworksModel_getSupportedOperationsForDevices)(
+      const ANeuralNetworksModel* model,
+      const ANeuralNetworksDevice* const* devices, uint32_t numDevices,
+      bool* supportedOps);
+
+  /**
+   * Create a {@link ANeuralNetworksCompilation} to compile the given model for
+   * a specified set of devices. If more than one device is specified, the
+   * compilation will distribute the workload automatically across the devices.
+   * The model must be fully supported by the specified set of devices. This
+   * means that ANeuralNetworksModel_getSupportedOperationsForDevices() must
+   * have returned true for every operation for that model/devices pair.
+   *
+   * @param model       The {@link ANeuralNetworksModel} to be compiled.
+   * @param devices     The set of devices. Must not contain duplicates.
+   * @param numDevices  The number of devices in the set.
+   * @param compilation The newly created object or NULL if unsuccessful.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA
+   *         if the model is invalid.
+   *
+   * Available since API level 29.
+   */
+  int (*ANeuralNetworksCompilation_createForDevices)(
+      ANeuralNetworksModel* model, const ANeuralNetworksDevice* const* devices,
+      uint32_t numDevices, ANeuralNetworksCompilation** compilation);
+
+  /**
+   * Sets the compilation caching signature and the cache directory.
+   *
+   * Provides optional caching information to the runtime for faster repeated
+   * compilation.
+   *
+   * See {@link ANeuralNetworksCompilation} for information on multithreaded
+   * usage.
+   *
+   * @param compilation The compilation to be modified.
+   * @param cacheDir The cache directory to store and retrieve caching data. It
+   *                 is recommended to use the code_cache provided by the
+   *                 Android runtime. If not using the code_cache, the user
+   *                 should choose a directory local to the application, and is
+   *                 responsible to manage and clean the cache entries.
+   * @param token The token provided by the user to specify a model, must be of
+   *              length ANEURALNETWORKS_BYTE_SIZE_OF_CACHE_TOKEN. The user
+   *              should ensure that the token is unique to a model within the
+   *              application. The NNAPI runtime will not detected token
+   *              collisions. If there is a collision, the compilation outcome
+   *              may be incorrect without notifying with error.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   *
+   * Available since API level 29.
+   */
+  int (*ANeuralNetworksCompilation_setCaching)(
+      ANeuralNetworksCompilation* compilation, const char* cacheDir,
+      const uint8_t* token);
+
+  /**
+   * Schedule synchronous evaluation of the execution.
+   *
+   * <p>Schedules synchronous evaluation of the execution. Returns once the
+   * execution has completed and the outputs are ready to be consumed.
+   * </p>
+   *
+   * See {@link ANeuralNetworksExecution} for information on multithreaded
+   * usage.
+   *
+   * See {@link ANeuralNetworksExecution_startCompute} for asynchronous
+   * execution. Synchronous execution incurs lower overhead than asynchronous
+   * execution.
+   *
+   * Available since API level 29.
+   *
+   * @param execution The execution to be scheduled and executed.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if the execution completed normally.
+   *         ANEURALNETWORKS_UNMAPPABLE if the execution input or output memory
+   *         cannot be properly mapped.
+   */
+  int (*ANeuralNetworksExecution_compute)(ANeuralNetworksExecution* execution);
+
+  /**
+   * Get the dimensional information of the specified output operand of the
+   * model of the
+   * {@link ANeuralNetworksExecution}.
+   *
+   * On asynchronous execution initiated by {@link
+   * ANeuralNetworksExecution_startCompute},
+   * {@link ANeuralNetworksEvent_wait} must be called prior to this function to
+   * recuperate the resources used by the execution.
+   *
+   * @param execution The execution to be queried.
+   * @param index The index of the output argument we are querying. It is
+   *              an index into the lists passed to
+   *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is
+   *              not the index associated with
+   *              {@link ANeuralNetworksModel_addOperand}.
+   * @param rank The rank of the output operand.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful,
+   *         ANEURALNETWORKS_OUTPUT_INSUFFICIENT_SIZE if the target output is
+   *         provided an insufficient buffer at execution time,
+   *         ANEURALNETWORKS_BAD_DATA if the index is invalid.
+   *
+   * Available since API level 29.
+   */
+  int (*ANeuralNetworksExecution_getOutputOperandRank)(
+      ANeuralNetworksExecution* execution, int32_t index, uint32_t* rank);
+
+  /**
+   * Get the dimensional information of the specified output operand of the
+   * model of the
+   * {@link ANeuralNetworksExecution}. The target output operand cannot be a
+   * scalar.
+   *
+   * On asynchronous execution initiated by {@link
+   * ANeuralNetworksExecution_startCompute},
+   * {@link ANeuralNetworksEvent_wait} must be called prior to this function to
+   * recuperate the resources used by the execution.
+   *
+   * @param execution The execution to be queried.
+   * @param index The index of the output argument we are querying. It is an
+   *              index into the lists passed to
+   *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is
+   *              not the index associated with
+   *              {@link ANeuralNetworksModel_addOperand}.
+   * @param dimensions The dimension array to be filled. The size of the array
+   *                   must be exactly as large as the rank of the output
+   *                   operand to be queried in the model.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful,
+   *         ANEURALNETWORKS_OUTPUT_INSUFFICIENT_SIZE if the target output is
+   *         provided an insufficient buffer at execution time,
+   *         ANEURALNETWORKS_BAD_DATA if the index is invalid or if the target
+   *         is a scalar.
+   *
+   * Available since API level 29.
+   */
+  int (*ANeuralNetworksExecution_getOutputOperandDimensions)(
+      ANeuralNetworksExecution* execution, int32_t index, uint32_t* dimensions);
+
+  /**
+   * Create a {@link ANeuralNetworksBurst} to apply the given compilation.
+   * This only creates the burst object. Computation is only performed once
+   * {@link ANeuralNetworksExecution_burstCompute} is invoked with a valid
+   * {@link ANeuralNetworksExecution} and {@link ANeuralNetworksBurst}.
+   *
+   * <p>The provided compilation must outlive the burst object.</p>
+   *
+   * Available since API level 29.
+   *
+   * @param compilation The {@link ANeuralNetworksCompilation} to be evaluated.
+   * @param burst The newly created object or NULL if unsuccessful.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA
+   *         if the compilation is invalid.
+   */
+  int (*ANeuralNetworksBurst_create)(ANeuralNetworksCompilation* compilation,
+                                     ANeuralNetworksBurst** burst);
+
+  /**
+   * Destroys the burst object.
+   *
+   * Available since API level 29.
+   *
+   * @param burst The burst object to be destroyed. Passing NULL is acceptable
+   * and results in no operation.
+   */
+  void (*ANeuralNetworksBurst_free)(ANeuralNetworksBurst* burst);
+
+  /**
+   * Schedule synchronous evaluation of the execution on a burst object.
+   *
+   * <p>Schedules synchronous evaluation of the execution. Returns once the
+   * execution has completed and the outputs are ready to be consumed.</p>
+   *
+   * <p>There must be at most one {@link ANeuralNetworksExecution} processing at
+   * any given time for any given burst object. Any
+   * {@link ANeuralNetworksExecution} launched before the previous has finished
+   * will result in ANEURALNETWORKS_BAD_STATE.</p>
+   *
+   * Available since API level 29.
+   *
+   * @param burst The burst object to execute on.
+   * @param execution The execution to be scheduled and executed. The execution
+   *                  must be created from the same {@link
+   *                  ANeuralNetworksCompilation} as the burst object.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if the execution completed normally.
+   */
+  int (*ANeuralNetworksExecution_burstCompute)(
+      ANeuralNetworksExecution* execution, ANeuralNetworksBurst* burst);
+
+  /**
+   * Creates a shared memory object from an AHardwareBuffer handle.
+   *
+   * If the shared memory is backed by an AHardwareBuffer of
+   * AHARDWAREBUFFER_FORMAT_BLOB format, it can be used the same way as
+   * shared memory created from a file handle. See
+   * {@link ANeuralNetworksMemory} for a description on how to use this
+   * shared memory.
+   *
+   * If the shared memory is backed by an AHardwareBuffer of a format other
+   * than AHARDWAREBUFFER_FORMAT_BLOB, it can only be used for Model inputs
+   * and outputs. When calling
+   * {@link ANeuralNetworksExecution_setInputFromMemory} or
+   * {@link ANeuralNetworksExecution_setOutputFromMemory} with the shared
+   * memory, both offset and length must be set to zero and the entire
+   * memory region will be associated with the specified input or output
+   * operand. There is no guarantee that an arbitrary AHardwareBuffer_Format
+   * and AHardwareBuffer_UsageFlags combination can be used by arbitrary
+   * devices. The execution will fail if selected set of devices cannot
+   * consume the buffer.
+   *
+   * Calling {@link ANeuralNetworksModel_setOperandValueFromMemory} with
+   * shared memory backed by an AHardwareBuffer of a format other than
+   * AHARDWAREBUFFER_FORMAT_BLOB is disallowed.
+   *
+   * TODO(miaowang): add documentation about intended usage with
+   * introspection API.
+   *
+   * Available since API level 29.
+   *
+   * @param ahwb The AHardwareBuffer handle.
+   * @param memory The memory object to be created.
+   *               Set to NULL if unsuccessful.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if the request completed normally.
+   *
+   * @see AHardwareBuffer
+   */
+  int (*ANeuralNetworksMemory_createFromAHardwareBuffer)(
+      const AHardwareBuffer* ahwb, ANeuralNetworksMemory** memory);
+
+  /**
+   * Specifies whether duration of the {@link ANeuralNetworksExecution} is to be
+   * measured. By default, duration is not measured.
+   *
+   * The {@link ANeuralNetworksExecution} must have been created with
+   * {@link ANeuralNetworksCompilation_createForDevices} with numDevices = 1.
+   *
+   * See {@link ANeuralNetworksExecution} for information on multithreaded
+   * usage.
+   *
+   * Available since API level 29.
+   *
+   * @param execution The execution to be modified.
+   * @param measure 'true' if duration is to be measured, 'false' if not.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   */
+  int (*ANeuralNetworksExecution_setMeasureTiming)(
+      ANeuralNetworksExecution* execution, bool measure);
+
+  /**
+   * Get the time spent in the specified {@link ANeuralNetworksExecution}, in
+   * nanoseconds. The execution must have completed.
+   *
+   * @param execution The execution to be queried.
+   * @param durationCode The measurement to be queried, specified by {@link
+   * DurationCode}.
+   * @param duration The returned duration. If no measurement was requested by
+   *                 {@link ANeuralNetworksExecution_setMeasureTiming}, or for
+   * some other reason the duration is not available, UINT64_MAX will be
+   * returned. A particular device need not support any given measurement.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   */
+  int (*ANeuralNetworksExecution_getDuration)(
+      const ANeuralNetworksExecution* execution, int32_t durationCode,
+      uint64_t* duration);
+
   /**/
 };
 
diff --git a/tensorflow/lite/nnapi/nnapi_implementation_test.cc b/tensorflow/lite/nnapi/nnapi_implementation_test.cc
index 51fc404ff8..9f30b95ec3 100644
--- a/tensorflow/lite/nnapi/nnapi_implementation_test.cc
+++ b/tensorflow/lite/nnapi/nnapi_implementation_test.cc
@@ -84,6 +84,7 @@ TEST(NnapiLibTest, NnApiImplementation) {
     EXPECT_NE(nnapi->ANeuralNetworksEvent_wait, nullptr);
     EXPECT_NE(nnapi->ANeuralNetworksEvent_free, nullptr);
     EXPECT_NE(nnapi->ASharedMemory_create, nullptr);
+    // TODO(b/123423795): Test Q-specific APIs after release.
   }
 #else
   EXPECT_FALSE(nnapi->nnapi_exists);
@@ -95,6 +96,8 @@ TEST(NnapiLibTest, NnApiImplementation) {
   EXPECT_EQ(nnapi->ANeuralNetworksModel_finish, nullptr);
   EXPECT_EQ(nnapi->ANeuralNetworksModel_addOperand, nullptr);
   EXPECT_EQ(nnapi->ANeuralNetworksModel_setOperandValue, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksModel_setOperandSymmPerChannelQuantParams,
+            nullptr);
   EXPECT_EQ(nnapi->ANeuralNetworksModel_setOperandValueFromMemory, nullptr);
   EXPECT_EQ(nnapi->ANeuralNetworksModel_addOperation, nullptr);
   EXPECT_EQ(nnapi->ANeuralNetworksModel_identifyInputsAndOutputs, nullptr);
@@ -114,6 +117,25 @@ TEST(NnapiLibTest, NnApiImplementation) {
   EXPECT_EQ(nnapi->ANeuralNetworksEvent_wait, nullptr);
   EXPECT_EQ(nnapi->ANeuralNetworksEvent_free, nullptr);
   EXPECT_NE(nnapi->ASharedMemory_create, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworks_getDeviceCount, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworks_getDevice, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksDevice_getName, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksDevice_getVersion, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksDevice_getFeatureLevel, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksModel_getSupportedOperationsForDevices,
+            nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksCompilation_createForDevices, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksCompilation_setCaching, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksExecution_compute, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksExecution_getOutputOperandRank, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksExecution_getOutputOperandDimensions,
+            nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksBurst_create, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksBurst_free, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksExecution_burstCompute, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksMemory_createFromAHardwareBuffer, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksExecution_setMeasureTiming, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksExecution_getDuration, nullptr);
 #endif
 }
 
-- 
GitLab


From f2a2d779a5eba5f27ceef5a11f685a8cb63dad36 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Tue, 29 Jan 2019 10:41:13 -0800
Subject: [PATCH 1485/2345] Move asserts, builtins and logical expression
 overloads to experimental until they are stable enough. Pipe
 experimental_autograph_options through tf.function.

PiperOrigin-RevId: 231430143
---
 .../autograph/converters/break_statements.py  |  6 +--
 .../converters/continue_statements.py         |  2 +-
 tensorflow/python/autograph/core/converter.py | 10 ++++-
 .../python/autograph/impl/conversion.py       |  9 ++--
 .../python/autograph/operators/logical.py     | 44 ++++++++++++++++---
 tensorflow/python/eager/def_function.py       |  5 +--
 tensorflow/python/eager/function.py           | 29 +++++++++---
 tensorflow/python/framework/func_graph.py     |  5 ++-
 ...flow.autograph.experimental.-feature.pbtxt | 12 +++++
 ...flow.autograph.experimental.-feature.pbtxt | 12 +++++
 10 files changed, 108 insertions(+), 26 deletions(-)

diff --git a/tensorflow/python/autograph/converters/break_statements.py b/tensorflow/python/autograph/converters/break_statements.py
index e4e32ab976..c2ced26d8d 100644
--- a/tensorflow/python/autograph/converters/break_statements.py
+++ b/tensorflow/python/autograph/converters/break_statements.py
@@ -53,7 +53,7 @@ class BreakTransformer(converter.Base):
       return block
 
     template = """
-        if not var_name:
+        if ag__.not_(var_name):
           block
       """
     node = templates.replace(
@@ -86,7 +86,7 @@ class BreakTransformer(converter.Base):
 
       template = """
         var_name = False
-        while test and not var_name:
+        while ag__.and_(lambda: test, lambda: ag__.not_(var_name)):
           body
         else:
           orelse
@@ -115,7 +115,7 @@ class BreakTransformer(converter.Base):
       # break did not trigger).
       guarded_orelse = self._guard_if_present(node.orelse, break_var)
       extra_test = templates.replace_as_expression(
-          'not var_name', var_name=break_var)
+          'ag__.not_(var_name)', var_name=break_var)
 
       # The extra test is hidden in the AST, which will confuse the static
       # analysis. To mitigate that, we insert a no-op statement that ensures
diff --git a/tensorflow/python/autograph/converters/continue_statements.py b/tensorflow/python/autograph/converters/continue_statements.py
index 05e19e59fc..c3b6679e38 100644
--- a/tensorflow/python/autograph/converters/continue_statements.py
+++ b/tensorflow/python/autograph/converters/continue_statements.py
@@ -78,7 +78,7 @@ class ContinueCanonicalizationTransformer(converter.Base):
       else:
         self.state[_Continue].guard_created = True
         template = """
-          if not var_name:
+          if ag__.not_(var_name):
             original_node
         """
         cond, = templates.replace(
diff --git a/tensorflow/python/autograph/core/converter.py b/tensorflow/python/autograph/core/converter.py
index 870c213e63..c8db2ca427 100644
--- a/tensorflow/python/autograph/core/converter.py
+++ b/tensorflow/python/autograph/core/converter.py
@@ -113,11 +113,14 @@ class Feature(enum.Enum):
   Attributes:
     ALL: Enable all features.
     AUTO_CONTROL_DEPS: Insert of control dependencies in the generated code.
-    DECORATORS: Allow decorators in local functions. Note that special
-      decorators, like `tf.function`, are allowed regardless of this toggle.
+    ASSERT_STATEMENTS: Convert Tensor-dependent assert statements to tf.Assert.
+    BUILTIN_FUNCTIONS: Convert builtin functions applied to Tensors to
+      their TF counterparts.
     ERROR_REWRITING: Rewrite errors that occur in the generated code to
       indicate the source code to which the failing code corresponds.
     LISTS: Convert list idioms, like initializers, slices, append, etc.
+    LOGICAL_EXPRESSIONS: Convert data-dependent logical expressions applied to
+      Tensors to their TF counterparts.
     NAME_SCOPES: Insert name scopes that name ops according to context, like the
       function they were defined in.
   """
@@ -125,8 +128,11 @@ class Feature(enum.Enum):
   ALL = 'ALL'
 
   AUTO_CONTROL_DEPS = 'AUTO_CONTROL_DEPS'
+  ASSERT_STATEMENTS = 'ASSERT_STATEMENTS'
+  BUILTIN_FUNCTIONS = 'BUILTIN_FUNCTIONS'
   ERROR_REWRITING = 'ERROR_REWRITING'
   LISTS = 'LISTS'
+  LOGICAL_EXPRESSIONS = 'LOGICAL_EXPRESSIONS'
   NAME_SCOPES = 'NAME_SCOPES'
 
   @classmethod
diff --git a/tensorflow/python/autograph/impl/conversion.py b/tensorflow/python/autograph/impl/conversion.py
index 1cd5671eae..901cfb2679 100644
--- a/tensorflow/python/autograph/impl/conversion.py
+++ b/tensorflow/python/autograph/impl/conversion.py
@@ -430,7 +430,8 @@ def node_to_graph(node, context):
   node = converter.apply_(node, context, arg_defaults)
   node = converter.apply_(node, context, directives)
   node = converter.apply_(node, context, break_statements)
-  node = converter.apply_(node, context, asserts)
+  if context.program.options.uses(converter.Feature.ASSERT_STATEMENTS):
+    node = converter.apply_(node, context, asserts)
   # Note: sequencing continue canonicalization before for loop one avoids
   # dealing with the extra loop increment operation that the for
   # canonicalization creates.
@@ -439,11 +440,13 @@ def node_to_graph(node, context):
   if context.program.options.uses(converter.Feature.LISTS):
     node = converter.apply_(node, context, lists)
     node = converter.apply_(node, context, slices)
-  node = converter.apply_(node, context, builtin_functions)
+  if context.program.options.uses(converter.Feature.BUILTIN_FUNCTIONS):
+    node = converter.apply_(node, context, builtin_functions)
   node = converter.apply_(node, context, call_trees)
   node = converter.apply_(node, context, control_flow)
   node = converter.apply_(node, context, conditional_expressions)
-  node = converter.apply_(node, context, logical_expressions)
+  if context.program.options.uses(converter.Feature.LOGICAL_EXPRESSIONS):
+    node = converter.apply_(node, context, logical_expressions)
   if context.program.options.uses(converter.Feature.AUTO_CONTROL_DEPS):
     node = converter.apply_(node, context, side_effect_guards)
   # TODO(mdan): If function scopes ever does more, the toggle will need moving.
diff --git a/tensorflow/python/autograph/operators/logical.py b/tensorflow/python/autograph/operators/logical.py
index dadb0daf1a..cafb0583e8 100644
--- a/tensorflow/python/autograph/operators/logical.py
+++ b/tensorflow/python/autograph/operators/logical.py
@@ -25,10 +25,25 @@ from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gen_math_ops
 
 
+# Note: the implementations in this file are split into very small-grained
+# functions in preparation for the factoring out the more generic pyct library.
+# At that time, the py_* and tf_* functions will reside in different libraries.
+
+
 def not_(a):
   """Functional form of "not"."""
   if tensor_util.is_tensor(a):
-    return gen_math_ops.logical_not(a)
+    return _tf_not(a)
+  return _py_not(a)
+
+
+def _tf_not(a):
+  """Implementation of the "not_" operator for TensorFlow."""
+  return gen_math_ops.logical_not(a)
+
+
+def _py_not(a):
+  """Default Python implementation of the "not_" operator."""
   return not a
 
 
@@ -37,7 +52,7 @@ def and_(a, b):
   a_val = a()
   if tensor_util.is_tensor(a_val):
     return _tf_lazy_and(a_val, b)
-  return a_val and b()
+  return _py_lazy_and(a_val, b)
 
 
 def _tf_lazy_and(cond, b):
@@ -46,12 +61,17 @@ def _tf_lazy_and(cond, b):
   return control_flow_ops.cond(cond, b, lambda: cond)
 
 
+def _py_lazy_and(cond, b):
+  """Lazy-eval equivalent of "and" in Python."""
+  return cond and b()
+
+
 def or_(a, b):
   """Functional form of "or". Uses lazy evaluation semantics."""
   a_val = a()
   if tensor_util.is_tensor(a_val):
     return _tf_lazy_or(a_val, b)
-  return a_val or b()
+  return _py_lazy_or(a_val, b)
 
 
 def _tf_lazy_or(cond, b):
@@ -60,11 +80,16 @@ def _tf_lazy_or(cond, b):
   return control_flow_ops.cond(cond, lambda: cond, b)
 
 
+def _py_lazy_or(cond, b):
+  """Lazy-eval equivalent of "or" in Python."""
+  return cond or b()
+
+
 def eq(a, b):
   """Functional form of "equal"."""
   if tensor_util.is_tensor(a) or tensor_util.is_tensor(b):
     return _tf_equal(a, b)
-  return a == b
+  return _py_equal(a, b)
 
 
 def _tf_equal(a, b):
@@ -72,25 +97,30 @@ def _tf_equal(a, b):
   return gen_math_ops.equal(a, b)
 
 
+def _py_equal(a, b):
+  """Overload of "equal" that falls back to Python's default implementation."""
+  return a == b
+
+
 def not_eq(a, b):
   """Functional form of "not-equal"."""
   return not_(eq(a, b))
 
 
-# Default implementation for the remainings.
+# Default implementation for the rest.
 
 is_ = operator.is_
 is_not = operator.is_not
 
 
 def in_(a, b):
-  """Functional form of "less-than"."""
+  """Functional form of "in"."""
   # TODO(mdan): in and not_in should probably be convertible for some types.
   return a in b
 
 
 def not_in(a, b):
-  """Functional form of "less-than"."""
+  """Functional form of "not-in"."""
   return a not in b
 
 gt = operator.gt
diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py
index c144bbf270..dea71282f5 100644
--- a/tensorflow/python/eager/def_function.py
+++ b/tensorflow/python/eager/def_function.py
@@ -278,8 +278,6 @@ class Function(object):
         python_function, input_signature)
     self._autograph = autograph
     self._experimental_autograph_options = experimental_autograph_options
-    if self._experimental_autograph_options is not None:
-      raise NotImplementedError()
     self._created_variables = None
     self._stateful_fn = None
     self._stateless_fn = None
@@ -301,7 +299,8 @@ class Function(object):
     return function_lib.defun(
         tf_decorator.make_decorator(self._python_function, wrapped_fn),
         input_signature=self._input_signature,
-        autograph=self._autograph)
+        autograph=self._autograph,
+        experimental_autograph_options=self._experimental_autograph_options)
 
   def _canonicalize_function_inputs(self, args, kwds):
     """Canonicalize the inputs to the Python function."""
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 860371dd6c..00932c4c4f 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -1029,7 +1029,8 @@ class Function(object):
                name,
                input_signature=None,
                attributes=None,
-               autograph=True):
+               autograph=True,
+               autograph_options=None):
     """Initializes a `Function`.
 
     Args:
@@ -1042,7 +1043,10 @@ class Function(object):
         of the function.
       autograph: whether to use autograph to compile
         `python_function`. See https://www.tensorflow.org/guide/autograph for
-          more information.
+        more information.
+      autograph_options: Experimental knobs to control behavior
+        `when autograph=True`. See https://www.tensorflow.org/guide/autograph
+        for more information.
 
     Raises:
       ValueError: if `input_signature` is not None and the `python_function`'s
@@ -1056,6 +1060,7 @@ class Function(object):
         python_function, input_signature)
     self._name = name
     self._autograph = autograph
+    self._autograph_options = autograph_options
     self._function_cache = collections.OrderedDict()
     self._garbage_collector = _FunctionGarbageCollector(self._function_cache)
     self._function_attributes = attributes or {}
@@ -1320,6 +1325,7 @@ class Function(object):
                 kwargs,
                 self._input_signature,
                 autograph=self._autograph,
+                autograph_options=self._autograph_options,
                 arg_names=arg_names), self._function_attributes)
         # pylint: disable=protected-access
         # Tell the ConcreteFunction to clean up its graph once it goes out of
@@ -1366,7 +1372,10 @@ def validate_signature(signature):
                     "a possibly nested sequence of TensorSpec objects.")
 
 
-def defun(func=None, input_signature=None, autograph=True):
+def defun(func=None,
+          input_signature=None,
+          autograph=True,
+          experimental_autograph_options=None):
   """Compiles a Python function into a callable TensorFlow graph.
 
   `defun` (short for "define function") compiles a Python function
@@ -1680,6 +1689,9 @@ def defun(func=None, input_signature=None, autograph=True):
     autograph: Whether `func` should be compiled before
       constructing the graph. See https://www.tensorflow.org/guide/autograph
       for more information.
+    experimental_autograph_options: Experimental knobs (in the form of a tuple
+      of tensorflow.autograph.Feature values) to control behavior when
+      autograph=True.
 
 
   Returns:
@@ -1695,13 +1707,15 @@ def defun(func=None, input_signature=None, autograph=True):
   return defun_with_attributes(
       func=func,
       input_signature=input_signature,
-      autograph=autograph)
+      autograph=autograph,
+      experimental_autograph_options=experimental_autograph_options)
 
 
 def defun_with_attributes(func=None,
                           input_signature=None,
                           attributes=None,
-                          autograph=True):
+                          autograph=True,
+                          experimental_autograph_options=None):
   """Compiles a Python function into a callable TensorFlow graph.
 
   This function supports adding extra function attributes. See detailed
@@ -1719,6 +1733,8 @@ def defun_with_attributes(func=None,
       the whitelisted argument which is a python string, and sets the name for
       this `ConcreteFunction` in the graph.
     autograph: same as defun()'s autograph.
+    experimental_autograph_options: same as defun()'s
+      experimental_autograph_options.
 
   Returns:
     Same as the return value of defun, with attributes added to the function in
@@ -1743,7 +1759,8 @@ def defun_with_attributes(func=None,
             name,
             input_signature=input_signature,
             attributes=attributes,
-            autograph=autograph))
+            autograph=autograph,
+            autograph_options=experimental_autograph_options))
 
   # This code path is for the `foo = tfe.defun(foo, ...)` use case
   if func is not None:
diff --git a/tensorflow/python/framework/func_graph.py b/tensorflow/python/framework/func_graph.py
index fa83a5f5c5..fe968480b2 100644
--- a/tensorflow/python/framework/func_graph.py
+++ b/tensorflow/python/framework/func_graph.py
@@ -398,6 +398,7 @@ def func_graph_from_py_func(name,
                             signature=None,
                             func_graph=None,
                             autograph=False,
+                            autograph_options=None,
                             add_control_dependencies=True,
                             arg_names=None,
                             op_return_value=None,
@@ -420,6 +421,8 @@ def func_graph_from_py_func(name,
       this graph else a new one is built and returned.
     autograph: whether to use autograph to compile `python_func`.
       See https://www.tensorflow.org/guide/autograph for more information.
+    autograph_options: additional knobs to control when `autograph=True`.
+      See https://www.tensorflow.org/guide/autograph for more information.
     add_control_dependencies: If True, automatically adds control dependencies
       to ensure program order matches execution order and stateful ops always
       execute.
@@ -522,7 +525,7 @@ def func_graph_from_py_func(name,
                   verbose=autograph.Verbosity.BRIEF,
                   recursive=True,
                   strip_decorators=(def_function.function,),
-                  optional_features=(),
+                  optional_features=autograph_options,
                   force_conversion=True,
               ), *args, **kwargs)
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.autograph.experimental.-feature.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.autograph.experimental.-feature.pbtxt
index 359530f69c..1f04d028ef 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.autograph.experimental.-feature.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.autograph.experimental.-feature.pbtxt
@@ -5,10 +5,18 @@ tf_class {
     name: "ALL"
     mtype: "<enum \'Feature\'>"
   }
+  member {
+    name: "ASSERT_STATEMENTS"
+    mtype: "<enum \'Feature\'>"
+  }
   member {
     name: "AUTO_CONTROL_DEPS"
     mtype: "<enum \'Feature\'>"
   }
+  member {
+    name: "BUILTIN_FUNCTIONS"
+    mtype: "<enum \'Feature\'>"
+  }
   member {
     name: "ERROR_REWRITING"
     mtype: "<enum \'Feature\'>"
@@ -17,6 +25,10 @@ tf_class {
     name: "LISTS"
     mtype: "<enum \'Feature\'>"
   }
+  member {
+    name: "LOGICAL_EXPRESSIONS"
+    mtype: "<enum \'Feature\'>"
+  }
   member {
     name: "NAME_SCOPES"
     mtype: "<enum \'Feature\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.autograph.experimental.-feature.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.autograph.experimental.-feature.pbtxt
index 359530f69c..1f04d028ef 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.autograph.experimental.-feature.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.autograph.experimental.-feature.pbtxt
@@ -5,10 +5,18 @@ tf_class {
     name: "ALL"
     mtype: "<enum \'Feature\'>"
   }
+  member {
+    name: "ASSERT_STATEMENTS"
+    mtype: "<enum \'Feature\'>"
+  }
   member {
     name: "AUTO_CONTROL_DEPS"
     mtype: "<enum \'Feature\'>"
   }
+  member {
+    name: "BUILTIN_FUNCTIONS"
+    mtype: "<enum \'Feature\'>"
+  }
   member {
     name: "ERROR_REWRITING"
     mtype: "<enum \'Feature\'>"
@@ -17,6 +25,10 @@ tf_class {
     name: "LISTS"
     mtype: "<enum \'Feature\'>"
   }
+  member {
+    name: "LOGICAL_EXPRESSIONS"
+    mtype: "<enum \'Feature\'>"
+  }
   member {
     name: "NAME_SCOPES"
     mtype: "<enum \'Feature\'>"
-- 
GitLab


From 42bf2c7307dd69e8728aacaabfc8a417fc1a008b Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Tue, 29 Jan 2019 10:42:51 -0800
Subject: [PATCH 1486/2345] Change max_pool operators to allow ksize and stride
 inputs or either 1, n or n+2

PiperOrigin-RevId: 231430489
---
 tensorflow/python/ops/nn_ops.py | 59 ++++++++++++++++++++++++++-------
 1 file changed, 47 insertions(+), 12 deletions(-)

diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index e0ef9e5e34..bfa640a688 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import collections
 import numbers
 
 import numpy as np
@@ -52,6 +53,28 @@ local_response_normalization = gen_nn_ops.lrn
 # pylint: disable=protected-access
 
 
+def _get_sequence(value, n, channel_index, name):
+  """Formats a value input for gen_nn_ops."""
+  if value is None:
+    value = [1]
+  elif not isinstance(value, collections.Sized):
+    value = [value]
+
+  current_n = len(value)
+  if current_n == n + 2:
+    return value
+  elif current_n == 1:
+    value = list((value[0],) * n)
+  elif current_n != n:
+    raise ValueError("{} should be of length 1, {} or {} but was {}".format(
+        name, n, n + 2, current_n))
+
+  if channel_index == 1:
+    return [1, 1] + value
+  else:
+    return [1] + value + [1]
+
+
 def _non_atrous_convolution(
     input,  # pylint: disable=redefined-builtin
     filter,  # pylint: disable=redefined-builtin
@@ -2976,10 +2999,10 @@ def max_pool(value, ksize, strides, padding, data_format="NHWC", name=None):
 
   Args:
     value: A 4-D `Tensor` of the format specified by `data_format`.
-    ksize: A list or tuple of 4 ints. The size of the window for each dimension
-      of the input tensor.
-    strides: A list or tuple of 4 ints. The stride of the sliding window for
-      each dimension of the input tensor.
+    ksize: An int or list of `ints` that has length `1`, `2` or `4`.
+      The size of the window for each dimension of the input tensor.
+    strides: An int or list of `ints` that has length `1`, `2` or `4`.
+      The stride of the sliding window for each dimension of the input tensor.
     padding: A string, either `'VALID'` or `'SAME'`. The padding algorithm.
       See the "returns" section of `tf.nn.convolution` for details.
     data_format: A string. 'NHWC', 'NCHW' and 'NCHW_VECT_C' are supported.
@@ -2991,6 +3014,13 @@ def max_pool(value, ksize, strides, padding, data_format="NHWC", name=None):
   """
   with ops.name_scope(name, "MaxPool", [value]) as name:
     value = ops.convert_to_tensor(value, name="input")
+    if data_format is None:
+      data_format = "NHWC"
+
+    channel_index = 1 if data_format.startswith("NC") else 3
+    ksize = _get_sequence(ksize, 2, channel_index, "ksize")
+    strides = _get_sequence(strides, 2, channel_index, "strides")
+
     return gen_nn_ops.max_pool(
         value,
         ksize=ksize,
@@ -3026,9 +3056,9 @@ def max_pool_with_argmax_v2(input,
       `int32`, `uint8`, `int16`, `int8`, `int64`, `bfloat16`, `uint16`, `half`,
       `uint32`, `uint64`.
       4-D with shape `[batch, height, width, channels]`.  Input to pool over.
-    ksize: A list of `ints` that has length `>= 4`.
+    ksize: An int or list of `ints` that has length `1`, `2` or `4`.
       The size of the window for each dimension of the input tensor.
-    strides: A list of `ints` that has length `>= 4`.
+    strides: An int or list of `ints` that has length `1`, `2` or `4`.
       The stride of the sliding window for each dimension of the
       input tensor.
     padding: A `string` from: `"SAME", "VALID"`.
@@ -3051,6 +3081,9 @@ def max_pool_with_argmax_v2(input,
   if data_format != "NHWC":
     raise ValueError("Data formats other than 'NHWC' are not yet supported")
 
+  ksize = _get_sequence(ksize, 2, 3, "ksize")
+  strides = _get_sequence(strides, 2, 3, "strides")
+
   return gen_nn_ops.max_pool_with_argmax(input=input,
                                          ksize=ksize,
                                          strides=strides,
@@ -3493,12 +3526,12 @@ def fractional_max_pool_v2(value,
 
   Args:
     value: A `Tensor`. 4-D with shape `[batch, height, width, channels]`.
-    pooling_ratio: A list of `floats` that has length >= 4.  Pooling ratio for
-      each dimension of `value`, currently only supports row and col dimension
-      and should be >= 1.0. For example, a valid pooling ratio looks like [1.0,
-      1.44, 1.73, 1.0]. The first and last elements must be 1.0 because we don't
-      allow pooling on batch and channels dimensions.  1.44 and 1.73 are pooling
-      ratio on height and width dimensions respectively.
+    pooling_ratio: An int or list of `ints` that has length `1`, `2` or `4`.
+      Pooling ratio for each dimension of `value`, currently only supports row
+      and col dimension and should be >= 1.0. For example, a valid pooling ratio
+      looks like [1.0, 1.44, 1.73, 1.0]. The first and last elements must be 1.0
+      because we don't allow pooling on batch and channels dimensions.  1.44 and
+      1.73 are pooling ratio on height and width dimensions respectively.
     pseudo_random: An optional `bool`.  Defaults to `False`. When set to `True`,
       generates the pooling sequence in a pseudorandom fashion, otherwise, in a
       random fashion. Check paper [Benjamin Graham, Fractional
@@ -3524,6 +3557,8 @@ def fractional_max_pool_v2(value,
     row_pooling_sequence: A `Tensor` of type `int64`.
     col_pooling_sequence: A `Tensor` of type `int64`.
   """
+  pooling_ratio = _get_sequence(pooling_ratio, 2, 3, "pooling_ratio")
+
   if seed == 0:
     return gen_nn_ops.fractional_max_pool(value, pooling_ratio, pseudo_random,
                                           overlapping, deterministic=False,
-- 
GitLab


From e8dacba2bdfc9175d2645be51843011350c103b1 Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Tue, 29 Jan 2019 12:04:36 -0800
Subject: [PATCH 1487/2345] Fix stream_executor/gpu/... build breakage when GPU
 is not configuered.

This is similar to having if_cuda_is_configured() guards in
stream_executor/cuda/...

PiperOrigin-RevId: 231447396
---
 tensorflow/stream_executor/build_defs.bzl | 15 ++++--
 tensorflow/stream_executor/gpu/BUILD      | 62 ++++++++---------------
 2 files changed, 33 insertions(+), 44 deletions(-)

diff --git a/tensorflow/stream_executor/build_defs.bzl b/tensorflow/stream_executor/build_defs.bzl
index 717c13d113..1d8a7ec62a 100644
--- a/tensorflow/stream_executor/build_defs.bzl
+++ b/tensorflow/stream_executor/build_defs.bzl
@@ -1,12 +1,21 @@
+load("@local_config_cuda//cuda:build_defs.bzl", "cuda_is_configured")
+load("@local_config_rocm//rocm:build_defs.bzl", "rocm_is_configured")
+
 def stream_executor_friends():
     return ["//tensorflow/..."]
 
 def tf_additional_cuda_platform_deps():
-  return []
+    return []
 
 # Use dynamic loading, therefore should be empty.
 def tf_additional_cuda_driver_deps():
-  return []
+    return []
 
 def tf_additional_cudnn_plugin_deps():
-  return []
+    return []
+
+# Returns whether any GPU backend is configuered.
+def if_gpu_is_configured(x):
+    if cuda_is_configured() or rocm_is_configured():
+      return x
+    return []
diff --git a/tensorflow/stream_executor/gpu/BUILD b/tensorflow/stream_executor/gpu/BUILD
index 26a9c6426d..78fe478679 100644
--- a/tensorflow/stream_executor/gpu/BUILD
+++ b/tensorflow/stream_executor/gpu/BUILD
@@ -5,18 +5,13 @@ licenses(["notice"])  # Apache 2.0
 
 load(
     "//tensorflow/stream_executor:build_defs.bzl",
-    "stream_executor_friends",
+    "if_gpu_is_configured",
 )
 load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda_is_configured")
 load("@local_config_rocm//rocm:build_defs.bzl", "if_rocm_is_configured")
 
-package_group(
-    name = "friends",
-    packages = stream_executor_friends(),
-)
-
 package(
-    default_visibility = [":friends"],
+    default_visibility = ["//tensorflow/stream_executor:__subpackages__"],
 )
 
 # Filegroup used to collect source files for the dependency check.
@@ -30,16 +25,14 @@ filegroup(
 
 cc_library(
     name = "gpu_activation_header",
-    hdrs = ["gpu_activation.h"],
-    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    hdrs = if_gpu_is_configured(["gpu_activation.h"]),
     deps = ["//tensorflow/stream_executor/platform"],
 )
 
 cc_library(
     name = "gpu_activation",
-    srcs = ["gpu_activation.cc"],
-    hdrs = ["gpu_activation.h"],
-    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    srcs = if_gpu_is_configured(["gpu_activation.cc"]),
+    hdrs = if_gpu_is_configured(["gpu_activation.h"]),
     deps = [
         ":gpu_activation_header",
         ":gpu_driver_header",
@@ -51,8 +44,7 @@ cc_library(
 
 cc_library(
     name = "gpu_diagnostics_header",
-    hdrs = ["gpu_diagnostics.h"],
-    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    hdrs = if_gpu_is_configured(["gpu_diagnostics.h"]),
     deps = [
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
@@ -61,8 +53,7 @@ cc_library(
 
 cc_library(
     name = "gpu_driver_header",
-    hdrs = ["gpu_driver.h"],
-    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    hdrs = if_gpu_is_configured(["gpu_driver.h"]),
     deps = [
         ":gpu_types_header",
         "//tensorflow/stream_executor:device_options",
@@ -74,8 +65,7 @@ cc_library(
 
 cc_library(
     name = "gpu_event_header",
-    hdrs = ["gpu_event.h"],
-    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    hdrs = if_gpu_is_configured(["gpu_event.h"]),
     deps = [
         ":gpu_driver_header",
         ":gpu_stream_header",
@@ -86,9 +76,8 @@ cc_library(
 
 cc_library(
     name = "gpu_event",
-    srcs = ["gpu_event.cc"],
-    hdrs = ["gpu_event.h"],
-    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    srcs = if_gpu_is_configured(["gpu_event.cc"]),
+    hdrs = if_gpu_is_configured(["gpu_event.h"]),
     deps = [
         ":gpu_driver_header",
         ":gpu_executor_header",
@@ -100,8 +89,7 @@ cc_library(
 
 cc_library(
     name = "gpu_executor_header",
-    hdrs = ["gpu_executor.h"],
-    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    hdrs = if_gpu_is_configured(["gpu_executor.h"]),
     deps = [
         ":gpu_kernel_header",
         "//tensorflow/stream_executor:event",
@@ -115,15 +103,13 @@ cc_library(
 
 cc_library(
     name = "gpu_helpers_header",
-    hdrs = ["gpu_helpers.h"],
-    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    hdrs = if_gpu_is_configured(["gpu_helpers.h"]),
     deps = [":gpu_types_header"],
 )
 
 cc_library(
     name = "gpu_kernel_header",
-    hdrs = ["gpu_kernel.h"],
-    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    hdrs = if_gpu_is_configured(["gpu_kernel.h"]),
     deps = [
         ":gpu_driver_header",
         "//tensorflow/stream_executor:event",
@@ -134,8 +120,7 @@ cc_library(
 
 cc_library(
     name = "gpu_rng_header",
-    hdrs = ["gpu_rng.h"],
-    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    hdrs = if_gpu_is_configured(["gpu_rng.h"]),
     deps = [
         ":gpu_types_header",
         "//tensorflow/stream_executor:plugin_registry",
@@ -146,8 +131,7 @@ cc_library(
 
 cc_library(
     name = "gpu_stream_header",
-    hdrs = ["gpu_stream.h"],
-    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    hdrs = if_gpu_is_configured(["gpu_stream.h"]),
     deps = [
         ":gpu_driver_header",
         "//tensorflow/stream_executor:stream_executor_internal",
@@ -157,9 +141,8 @@ cc_library(
 
 cc_library(
     name = "gpu_stream",
-    srcs = ["gpu_stream.cc"],
-    hdrs = ["gpu_stream.h"],
-    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    srcs = if_gpu_is_configured(["gpu_stream.cc"]),
+    hdrs = if_gpu_is_configured(["gpu_stream.h"]),
     deps = [
         ":gpu_driver_header",
         ":gpu_executor_header",
@@ -172,8 +155,7 @@ cc_library(
 
 cc_library(
     name = "gpu_timer_header",
-    hdrs = ["gpu_timer.h"],
-    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    hdrs = if_gpu_is_configured(["gpu_timer.h"]),
     deps = [
         ":gpu_driver_header",
         ":gpu_executor_header",
@@ -183,9 +165,8 @@ cc_library(
 
 cc_library(
     name = "gpu_timer",
-    srcs = ["gpu_timer.cc"],
-    hdrs = ["gpu_timer.h"],
-    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    srcs = if_gpu_is_configured(["gpu_timer.cc"]),
+    hdrs = if_gpu_is_configured(["gpu_timer.h"]),
     deps = [
         ":gpu_driver_header",
         ":gpu_executor_header",
@@ -197,8 +178,7 @@ cc_library(
 
 cc_library(
     name = "gpu_types_header",
-    hdrs = ["gpu_types.h"],
-    visibility = ["//tensorflow/stream_executor:__subpackages__"],
+    hdrs = if_gpu_is_configured(["gpu_types.h"]),
     deps = [
         "//tensorflow/stream_executor/platform",
     ] + if_cuda_is_configured([
-- 
GitLab


From 038675489fbf35542f6b0e1b6192876a83e8c5b6 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Tue, 29 Jan 2019 12:04:55 -0800
Subject: [PATCH 1488/2345] Automated rollback of commit
 01cf6233d62d476676ffbea0fad46660a0ba9a16

PiperOrigin-RevId: 231447473
---
 tensorflow/python/keras/metrics.py            | 27 +++++++++----
 .../keras/metrics_confusion_matrix_test.py    | 39 ++++++++++++-------
 .../python/keras/utils/metrics_utils.py       | 34 ++++++++++++++++
 .../v1/tensorflow.keras.metrics.-a-u-c.pbtxt  |  2 +-
 .../v2/tensorflow.keras.metrics.-a-u-c.pbtxt  |  2 +-
 5 files changed, 80 insertions(+), 24 deletions(-)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 6aed0e7ac6..95837e4d8d 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -1607,8 +1607,8 @@ class AUC(Metric):
 
   def __init__(self,
                num_thresholds=200,
-               curve=metrics_utils.AUCCurve.ROC,
-               summation_method=metrics_utils.AUCSummationMethod.INTERPOLATION,
+               curve='ROC',
+               summation_method='interpolation',
                name=None,
                dtype=None):
     """Creates an `AUC` instance.
@@ -1631,18 +1631,29 @@ class AUC(Metric):
     # Validate configurations.
     if num_thresholds <= 1:
       raise ValueError('`num_thresholds` must be > 1.')
-    if curve not in list(metrics_utils.AUCCurve):
+    if isinstance(curve, metrics_utils.AUCCurve) and curve not in list(
+        metrics_utils.AUCCurve):
       raise ValueError('Invalid curve: "{}". Valid options are: "{}"'.format(
           curve, list(metrics_utils.AUCCurve)))
-    if summation_method not in list(metrics_utils.AUCSummationMethod):
+    if isinstance(
+        summation_method,
+        metrics_utils.AUCSummationMethod) and summation_method not in list(
+            metrics_utils.AUCSummationMethod):
       raise ValueError(
           'Invalid summation method: "{}". Valid options are: "{}"'.format(
               summation_method, list(metrics_utils.AUCSummationMethod)))
 
     # Update properties.
     self.num_thresholds = num_thresholds
-    self.curve = curve
-    self.summation_method = summation_method
+    if isinstance(curve, metrics_utils.AUCCurve):
+      self.curve = curve
+    else:
+      self.curve = metrics_utils.AUCCurve.from_str(curve)
+    if isinstance(summation_method, metrics_utils.AUCSummationMethod):
+      self.summation_method = summation_method
+    else:
+      self.summation_method = metrics_utils.AUCSummationMethod.from_str(
+          summation_method)
     super(AUC, self).__init__(name=name, dtype=dtype)
 
     # Create metric variables
@@ -1802,8 +1813,8 @@ class AUC(Metric):
   def get_config(self):
     config = {
         'num_thresholds': self.num_thresholds,
-        'curve': self.curve,
-        'summation_method': self.summation_method,
+        'curve': self.curve.value,
+        'summation_method': self.summation_method.value,
     }
     base_config = super(AUC, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
diff --git a/tensorflow/python/keras/metrics_confusion_matrix_test.py b/tensorflow/python/keras/metrics_confusion_matrix_test.py
index 6a0fcfed07..972f7b6de7 100644
--- a/tensorflow/python/keras/metrics_confusion_matrix_test.py
+++ b/tensorflow/python/keras/metrics_confusion_matrix_test.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import json
+
 from absl.testing import parameterized
 import numpy as np
 
@@ -949,8 +951,8 @@ class AUCTest(test.TestCase):
   def test_config(self):
     auc_obj = metrics.AUC(
         num_thresholds=100,
-        curve=metrics_utils.AUCCurve.PR,
-        summation_method=metrics_utils.AUCSummationMethod.MAJORING,
+        curve='PR',
+        summation_method='majoring',
         name='auc_1')
     self.assertEqual(auc_obj.name, 'auc_1')
     self.assertEqual(len(auc_obj.variables), 4)
@@ -958,6 +960,8 @@ class AUCTest(test.TestCase):
     self.assertEqual(auc_obj.curve, metrics_utils.AUCCurve.PR)
     self.assertEqual(auc_obj.summation_method,
                      metrics_utils.AUCSummationMethod.MAJORING)
+    old_config = auc_obj.get_config()
+    self.assertDictEqual(old_config, json.loads(json.dumps(old_config)))
 
     # Check save and restore config
     auc_obj2 = metrics.AUC.from_config(auc_obj.get_config())
@@ -967,6 +971,8 @@ class AUCTest(test.TestCase):
     self.assertEqual(auc_obj2.curve, metrics_utils.AUCCurve.PR)
     self.assertEqual(auc_obj2.summation_method,
                      metrics_utils.AUCSummationMethod.MAJORING)
+    new_config = auc_obj2.get_config()
+    self.assertDictEqual(old_config, new_config)
 
   def test_value_is_idempotent(self):
     self.setup()
@@ -1021,8 +1027,7 @@ class AUCTest(test.TestCase):
   def test_weighted_roc_majoring(self):
     self.setup()
     auc_obj = metrics.AUC(
-        num_thresholds=self.num_thresholds,
-        summation_method=metrics_utils.AUCSummationMethod.MAJORING)
+        num_thresholds=self.num_thresholds, summation_method='majoring')
     self.evaluate(variables.variables_initializer(auc_obj.variables))
     result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)
 
@@ -1037,8 +1042,7 @@ class AUCTest(test.TestCase):
   def test_weighted_roc_minoring(self):
     self.setup()
     auc_obj = metrics.AUC(
-        num_thresholds=self.num_thresholds,
-        summation_method=metrics_utils.AUCSummationMethod.MINORING)
+        num_thresholds=self.num_thresholds, summation_method='minoring')
     self.evaluate(variables.variables_initializer(auc_obj.variables))
     result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)
 
@@ -1054,8 +1058,8 @@ class AUCTest(test.TestCase):
     self.setup()
     auc_obj = metrics.AUC(
         num_thresholds=self.num_thresholds,
-        curve=metrics_utils.AUCCurve.PR,
-        summation_method=metrics_utils.AUCSummationMethod.MAJORING)
+        curve='PR',
+        summation_method='majoring')
     self.evaluate(variables.variables_initializer(auc_obj.variables))
     result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)
 
@@ -1071,8 +1075,8 @@ class AUCTest(test.TestCase):
     self.setup()
     auc_obj = metrics.AUC(
         num_thresholds=self.num_thresholds,
-        curve=metrics_utils.AUCCurve.PR,
-        summation_method=metrics_utils.AUCSummationMethod.MINORING)
+        curve='PR',
+        summation_method='minoring')
     self.evaluate(variables.variables_initializer(auc_obj.variables))
     result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)
 
@@ -1086,10 +1090,7 @@ class AUCTest(test.TestCase):
 
   def test_weighted_pr_interpolation(self):
     self.setup()
-    auc_obj = metrics.AUC(
-        num_thresholds=self.num_thresholds,
-        curve=metrics_utils.AUCCurve.PR,
-        summation_method=metrics_utils.AUCSummationMethod.INTERPOLATION)
+    auc_obj = metrics.AUC(num_thresholds=self.num_thresholds, curve='PR')
     self.evaluate(variables.variables_initializer(auc_obj.variables))
     result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)
 
@@ -1115,6 +1116,16 @@ class AUCTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, '`num_thresholds` must be > 1.'):
       metrics.AUC(num_thresholds=1)
 
+  def test_invalid_curve(self):
+    with self.assertRaisesRegexp(ValueError,
+                                 'Invalid AUC curve value "Invalid".'):
+      metrics.AUC(curve='Invalid')
+
+  def test_invalid_summation_method(self):
+    with self.assertRaisesRegexp(
+        ValueError, 'Invalid AUC summation method value "Invalid".'):
+      metrics.AUC(summation_method='Invalid')
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/utils/metrics_utils.py b/tensorflow/python/keras/utils/metrics_utils.py
index c5ff617ad7..329cb0b278 100644
--- a/tensorflow/python/keras/utils/metrics_utils.py
+++ b/tensorflow/python/keras/utils/metrics_utils.py
@@ -161,15 +161,49 @@ class ConfusionMatrix(Enum):
 
 
 class AUCCurve(Enum):
+  """Type of AUC Curve (ROC or PR)."""
   ROC = 'ROC'
   PR = 'PR'
 
+  @staticmethod
+  def from_str(key):
+    if key in ('pr', 'PR'):
+      return AUCCurve.PR
+    elif key in ('roc', 'ROC'):
+      return AUCCurve.ROC
+    else:
+      raise ValueError('Invalid AUC curve value "%s".' % key)
+
 
 class AUCSummationMethod(Enum):
+  """Type of AUC summation method.
+
+  https://en.wikipedia.org/wiki/Riemann_sum)
+
+  Contains the following values:
+  * 'interpolation': Applies mid-point summation scheme for `ROC` curve. For
+    `PR` curve, interpolates (true/false) positives but not the ratio that is
+    precision (see Davis & Goadrich 2006 for details).
+  * 'minoring': Applies left summation for increasing intervals and right
+    summation for decreasing intervals.
+  * 'majoring': Applies right summation for increasing intervals and left
+    summation for decreasing intervals.
+  """
   INTERPOLATION = 'interpolation'
   MAJORING = 'majoring'
   MINORING = 'minoring'
 
+  @staticmethod
+  def from_str(key):
+    if key in ('interpolation', 'Interpolation'):
+      return AUCSummationMethod.INTERPOLATION
+    elif key in ('majoring', 'Majoring'):
+      return AUCSummationMethod.MAJORING
+    elif key in ('minoring', 'Minoring'):
+      return AUCSummationMethod.MINORING
+    else:
+      raise ValueError('Invalid AUC summation method value "%s".' % key)
+
 
 def update_confusion_matrix_variables(variables_to_update,
                                       y_true,
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-a-u-c.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-a-u-c.pbtxt
index 8019d43c78..beaa6f0596 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-a-u-c.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-a-u-c.pbtxt
@@ -87,7 +87,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'num_thresholds\', \'curve\', \'summation_method\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'200\', \'AUCCurve.ROC\', \'AUCSummationMethod.INTERPOLATION\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'num_thresholds\', \'curve\', \'summation_method\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'200\', \'ROC\', \'interpolation\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_loss"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-a-u-c.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-a-u-c.pbtxt
index 8019d43c78..beaa6f0596 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-a-u-c.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-a-u-c.pbtxt
@@ -87,7 +87,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'num_thresholds\', \'curve\', \'summation_method\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'200\', \'AUCCurve.ROC\', \'AUCSummationMethod.INTERPOLATION\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'num_thresholds\', \'curve\', \'summation_method\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'200\', \'ROC\', \'interpolation\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_loss"
-- 
GitLab


From 1d8a762da5d608a0f513f22023b356792bfb92fe Mon Sep 17 00:00:00 2001
From: Supriya Rao <supriyar@nvidia.com>
Date: Wed, 24 Oct 2018 13:35:39 -0700
Subject: [PATCH 1489/2345] First Implementation of basic batched
 non_max_suppression kernel and unit tests. Works for all unit tests with
 batch size 1 and 2.

---
 .../core/kernels/non_max_suppression_op.cc    | 300 +++++++++++++
 .../kernels/non_max_suppression_op_test.cc    | 413 ++++++++++++++++++
 tensorflow/core/ops/image_ops.cc              |  14 +
 3 files changed, 727 insertions(+)

diff --git a/tensorflow/core/kernels/non_max_suppression_op.cc b/tensorflow/core/kernels/non_max_suppression_op.cc
index 2f4e51272f..8733961857 100644
--- a/tensorflow/core/kernels/non_max_suppression_op.cc
+++ b/tensorflow/core/kernels/non_max_suppression_op.cc
@@ -74,6 +74,35 @@ static inline void ParseAndCheckBoxSizes(OpKernelContext* context,
               errors::InvalidArgument("boxes must have 4 columns"));
 }
 
+static inline void CheckNMSLiteScoreSizes(OpKernelContext* context, 
+                                          int num_boxes,
+                                          const Tensor& scores) {
+  // The shape of 'scores' is [batch_size, num_boxes, num_classes]
+  OP_REQUIRES(context, scores.dims() == 3,
+              errors::InvalidArgument("scores must be 3-D",
+                                      scores.shape().DebugString()));
+  OP_REQUIRES(context, scores.dim_size(1) == num_boxes,
+              errors::InvalidArgument("scores has incompatible shape"));
+}
+
+static inline void ParseAndCheckNMSLiteBoxSizes(OpKernelContext* context,
+                                         const Tensor& boxes, int* num_boxes, 
+                                         const int num_classes) {
+  // The shape of 'boxes' is [batch_size, num_boxes, q, 4]
+  OP_REQUIRES(context, boxes.dims() == 4,
+              errors::InvalidArgument("boxes must be 4-D",
+                                      boxes.shape().DebugString()));
+
+  bool box_check = boxes.dim_size(2) == 1 || boxes.dim_size(2) == num_classes;
+  OP_REQUIRES(context, box_check,
+              errors::InvalidArgument(
+                  "third dimension of boxes must be either 1 or num classes"));
+  // Num_boxes * q where q is either 1 or num_classes
+  // TODO check if this should take num_classes into account
+  *num_boxes = boxes.dim_size(1); //*boxes.dim_size(2);
+  OP_REQUIRES(context, boxes.dim_size(3) == 4,
+              errors::InvalidArgument("boxes must have 4 columns"));
+}
 // Return intersection-over-union overlap between boxes i and j
 template <typename T>
 static inline bool IOUGreaterThanThreshold(
@@ -195,6 +224,215 @@ void DoNonMaxSuppressionOp(
   std::copy_n(selected.begin(), selected.size(), output_indices_data.data());
 }
 
+void BatchedNonMaxSuppressionOp(OpKernelContext* context, 
+                           const Tensor& inp_boxes, const Tensor& inp_scores, 
+                           int num_boxes, 
+                           const Tensor& max_size_per_class, 
+                           const Tensor& max_total_size, 
+                           const float score_threshold, 
+                           const float iou_threshold) {
+
+  int q = inp_boxes.dim_size(2);
+  int num_classes = inp_scores.dim_size(2);
+  //unpack along batch dimension
+  const int num_batches = inp_boxes.dim_size(0);
+
+  // [num_batches, max_detections, 4]
+  std::vector <std::vector<float>> nmsed_boxes(num_batches);
+  // [num_batches, max_detections]
+  std::vector <std::vector<float>> nmsed_scores(num_batches); 
+  // [num_batches, max_detections]
+  std::vector <std::vector<float>> nmsed_classes(num_batches); 
+  // [num_batches]
+  std::vector <int> final_valid_detections;
+  // [num_batches, max_detections]
+  std::vector <std::vector<int>> selected_indices(num_batches);
+
+  int max_detections = 0;
+  //perform non_max_suppression operation for each batch independently
+  for (int batch = 0; batch < num_batches; ++batch) {
+    std::cout << " iteration " << batch <<std::endl;
+    // dims of per_batch_boxes [num_boxes, q, 4]
+    //Tensor per_batch_boxes = inp_boxes.SubSlice(batch);
+    Tensor per_batch_boxes = inp_boxes.Slice(batch, batch+1);
+    // dims of per_batch_scores [num_boxes, num_classes]
+    Tensor per_batch_scores = inp_scores.Slice(batch, batch+1);
+
+    struct ResultCandidate {
+          int box_index;
+          float score;
+          int class_idx;
+          float box_coord[4];
+    };
+
+    auto rc_cmp = [](const ResultCandidate rc_i, const ResultCandidate rc_j) {
+        return rc_i.score < rc_j.score;
+    };
+    std::priority_queue<ResultCandidate, std::vector<ResultCandidate>, 
+      decltype(rc_cmp)> result_candidate_pq(rc_cmp);
+
+    float * scoresData = per_batch_scores.unaligned_flat<float>().data();
+    float * boxesData = per_batch_boxes.unaligned_flat<float>().data();
+
+    //Iterate through all classes
+    for (int class_idx = 0; class_idx < num_classes; ++class_idx) {
+      std::vector<float> scores_data;
+      std::vector<float> boxes_data_vec;
+
+      for (int box = 0; box < num_boxes; ++box) {
+        //Get the scores per class 
+        //scores_data dim is [num_boxes].
+        scores_data.push_back(scoresData[box * num_classes + class_idx]);
+        for(int cid = 0; cid < 4; ++cid){
+          if(q > 1){
+            //Get the boxes per class. boxes_data_vec dims is [num_boxes, 4]
+            boxes_data_vec.push_back(boxesData[box * q * 4 + 
+                  class_idx * 4 + cid]);
+            //std::cout << " Pushing box_data " << boxes_data_vec[boxes_data_vec.size()-1] << std::endl;
+          }
+          else 
+              boxes_data_vec.push_back(boxesData[box * 4 + cid]);
+        }
+      }
+        
+      //Copy boxes_data_vec to a tensor
+      TensorShape boxesShape({num_boxes, 4});
+      Tensor boxes(per_batch_boxes.dtype(), boxesShape);
+      std::copy_n(boxes_data_vec.begin(), boxes_data_vec.size(), 
+          boxes.unaligned_flat<float>().data());
+
+      const int output_size = std::min(max_size_per_class.scalar<int>()(), 
+                                   num_boxes);
+      // Do NMS, get the candidate indices of form vector<int>
+      // Data structure for selection candidate in NMS.
+      struct Candidate {
+        int box_index;
+        float score;
+      };
+      auto cmp = [](const Candidate bs_i, const Candidate bs_j) {
+        return bs_i.score < bs_j.score;
+      };
+      std::priority_queue<Candidate, std::deque<Candidate>, decltype(cmp)>
+        candidate_priority_queue(cmp);
+      for (int i = 0; i < scores_data.size(); ++i) {
+          //std::cout << "scores_data "<< scores_data[i] << std::endl;
+        if (scores_data[i] > score_threshold) {
+          candidate_priority_queue.emplace(Candidate({i, scores_data[i]}));
+        }
+      }
+
+      std::vector<int> selected;
+      std::vector<float> selected_boxes;
+      Candidate next_candidate;
+
+      const Tensor const_boxes = boxes;
+      typename TTypes<float, 2>::ConstTensor boxes_data = 
+        const_boxes.tensor<float, 2>();
+      LOG(ERROR) << " boxes data " << boxes.DebugString();
+      while (selected.size() < output_size && !candidate_priority_queue.empty())
+      {
+        next_candidate = candidate_priority_queue.top();
+        //std::cout << " next_candidate " << next_candidate.box_index << std::endl;
+        candidate_priority_queue.pop();
+
+        // Overlapping boxes are likely to have similar scores,
+        // therefore we iterate through the previously selected boxes backwards
+        // in order to see if `next_candidate` should be suppressed.
+        bool should_select = true;
+        for (int j = selected.size() - 1; j >= 0; --j) {
+          if (IOUGreaterThanThreshold(boxes_data, next_candidate.box_index, 
+              selected[j], iou_threshold)) {
+              //std::cout << " Dont select. box id is " << selected[j] << " idx " << j <<std::endl;  
+            should_select = false;
+            break;
+          }
+        }
+
+        if (should_select) {
+          selected.push_back(next_candidate.box_index);
+          //std::cout << " select box_index " << next_candidate.box_index << std::endl;
+          //Add the selected box to the result candidate. Sorted by score
+          int id = next_candidate.box_index;
+          ResultCandidate rc = {next_candidate.box_index, next_candidate.score,
+            class_idx, {boxes_data(id, 0), boxes_data(id, 1), boxes_data(id, 2),
+            boxes_data(id, 3)}}; 
+          result_candidate_pq.push(rc);
+        }
+      }
+
+    }
+    int total_size_per_batch = max_total_size.scalar<int>()();
+    std::cout << " pq size " << result_candidate_pq.size() << std::endl;
+
+    if (total_size_per_batch > 0)
+      max_detections = std::min((int) result_candidate_pq.size(), 
+        total_size_per_batch);
+    else max_detections = (int) result_candidate_pq.size();
+
+    std::cout << " max_detections " << max_detections << std::endl;
+    final_valid_detections.push_back(max_detections);
+    // Pick the top max_total_size values 
+    while(total_size_per_batch > 0 && !result_candidate_pq.empty())
+    {
+      ResultCandidate next_candidate = result_candidate_pq.top();
+      result_candidate_pq.pop();
+      // Add to final output vectors
+      nmsed_boxes[batch].push_back(next_candidate.box_coord[0]);
+      nmsed_boxes[batch].push_back(next_candidate.box_coord[1]);
+      nmsed_boxes[batch].push_back(next_candidate.box_coord[2]);
+      nmsed_boxes[batch].push_back(next_candidate.box_coord[3]);
+      nmsed_scores[batch].push_back(next_candidate.score);
+      nmsed_classes[batch].push_back(next_candidate.class_idx);
+      selected_indices[batch].push_back(next_candidate.box_index);
+      total_size_per_batch--;
+    }
+            
+  }
+
+  Tensor* nmsed_boxes_t = nullptr;
+  TensorShape boxes_shape({num_batches, max_detections, 4});
+  OP_REQUIRES_OK(context,
+                 context->allocate_output(0, boxes_shape, &nmsed_boxes_t));
+  auto nmsed_boxes_flat = nmsed_boxes_t->template flat<float>();
+
+  Tensor* nmsed_scores_t = nullptr;
+  TensorShape scores_shape({num_batches, max_detections});
+  OP_REQUIRES_OK(context,
+                 context->allocate_output(1, scores_shape, &nmsed_scores_t));
+  auto nmsed_scores_flat = nmsed_scores_t->template flat<float>();
+
+  Tensor* nmsed_classes_t = nullptr;
+  OP_REQUIRES_OK(context,
+                 context->allocate_output(2, scores_shape, &nmsed_classes_t));
+  auto nmsed_classes_flat = nmsed_classes_t->template flat<float>();
+
+  Tensor* valid_detections_t = nullptr;
+  TensorShape valid_detections_shape({num_batches});
+  OP_REQUIRES_OK(context, context->allocate_output(3, valid_detections_shape, 
+                 &valid_detections_t));
+  auto valid_detections_flat = valid_detections_t->template flat<int>();
+
+  Tensor* selected_indices_t = nullptr;
+  OP_REQUIRES_OK(context,
+                 context->allocate_output(4, scores_shape, &selected_indices_t));
+  auto selected_indices_flat = selected_indices_t->template flat<int>();
+  for (int i = 0; i < num_batches; ++i) {
+    valid_detections_flat(i) = final_valid_detections[i];
+    for (int j = 0; j < max_detections; ++j) {
+      nmsed_scores_flat(i * max_detections + j) = nmsed_scores[i][j];
+      nmsed_classes_flat(i * max_detections + j) = nmsed_classes[i][j];
+      selected_indices_flat(i * max_detections + j) = selected_indices[i][j];
+      for (int k = 0; k < 4; ++k) {
+        nmsed_boxes_flat(i * max_detections * 4 + j * 4 + k) = 
+          nmsed_boxes[i][j * 4 + k]; 
+      }
+    }
+  }
+  LOG(ERROR) << " boxes " << nmsed_boxes_t->DebugString();
+  LOG(ERROR) << " Scores " << nmsed_scores_t->DebugString();
+  LOG(ERROR) << " classes " << nmsed_classes_t->DebugString();
+}
+
 }  // namespace
 
 template <typename Device>
@@ -435,6 +673,65 @@ class NonMaxSuppressionWithOverlapsOp : public OpKernel {
   }
 };
 
+template <typename Device>
+class NonMaxSuppressionLiteOp : public OpKernel {
+ public:
+  explicit NonMaxSuppressionLiteOp(OpKernelConstruction* context)
+      : OpKernel(context) {}
+
+  void Compute(OpKernelContext* context) override {
+    // boxes: [batch_size, num_anchors, q, 4]
+    const Tensor& boxes = context->input(0);
+    LOG(ERROR) << " inp_boxes " << boxes.DebugString();
+    // scores: [batch_size, num_anchors, num_classes]
+    const Tensor& scores = context->input(1);
+    OP_REQUIRES(context, (boxes.dim_size(0) == scores.dim_size(0)),
+              errors::InvalidArgument(
+                "boxes and scores must have same batch size"));
+
+    // max_output_size: scalar
+    const Tensor& max_output_size = context->input(2);
+    OP_REQUIRES(
+        context, TensorShapeUtils::IsScalar(max_output_size.shape()),
+        errors::InvalidArgument("max_size_per_class must be 0-D, got shape ",
+                                max_output_size.shape().DebugString()));
+    // max_total_size: scalar
+    const Tensor& max_total_size = context->input(3);
+    OP_REQUIRES(
+        context, TensorShapeUtils::IsScalar(max_total_size.shape()),
+        errors::InvalidArgument("max_total_size must be 0-D, got shape ",
+                                max_total_size.shape().DebugString()));
+    // iou_threshold: scalar
+    const Tensor& iou_threshold = context->input(4);
+    OP_REQUIRES(context, TensorShapeUtils::IsScalar(iou_threshold.shape()),
+                errors::InvalidArgument("iou_threshold must be 0-D, got shape ",
+                                        iou_threshold.shape().DebugString()));
+    const float iou_threshold_val = iou_threshold.scalar<float>()();
+
+    // score_threshold: scalar
+    const Tensor& score_threshold = context->input(5);
+    OP_REQUIRES(
+        context, TensorShapeUtils::IsScalar(score_threshold.shape()),
+        errors::InvalidArgument("score_threshold must be 0-D, got shape ",
+                                score_threshold.shape().DebugString()));
+    const float score_threshold_val = score_threshold.scalar<float>()();
+
+    OP_REQUIRES(context, iou_threshold_val >= 0 && iou_threshold_val <= 1,
+                errors::InvalidArgument("iou_threshold must be in [0, 1]"));
+    int num_boxes = 0;
+    const int num_classes = scores.dim_size(2);
+    ParseAndCheckNMSLiteBoxSizes(context, boxes, &num_boxes, num_classes);
+    CheckNMSLiteScoreSizes(context, num_boxes, scores);
+    if (!context->status().ok()) {
+      return;
+    }
+    BatchedNonMaxSuppressionOp(context, boxes, scores, num_boxes, 
+                               max_output_size, max_total_size, 
+                               score_threshold_val, iou_threshold_val);
+  }
+};
+
+
 REGISTER_KERNEL_BUILDER(Name("NonMaxSuppression").Device(DEVICE_CPU),
                         NonMaxSuppressionOp<CPUDevice>);
 
@@ -466,4 +763,7 @@ REGISTER_KERNEL_BUILDER(
     Name("NonMaxSuppressionWithOverlaps").Device(DEVICE_CPU),
     NonMaxSuppressionWithOverlapsOp<CPUDevice>);
 
+REGISTER_KERNEL_BUILDER(Name("NonMaxSuppressionLite").Device(DEVICE_CPU),
+        NonMaxSuppressionLiteOp<CPUDevice>);
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/non_max_suppression_op_test.cc b/tensorflow/core/kernels/non_max_suppression_op_test.cc
index c321849f40..bffe388129 100644
--- a/tensorflow/core/kernels/non_max_suppression_op_test.cc
+++ b/tensorflow/core/kernels/non_max_suppression_op_test.cc
@@ -861,4 +861,417 @@ TEST_F(NonMaxSuppressionWithOverlapsOpTest, TestEmptyInput) {
   test::ExpectTensorEqual<int>(expected, *GetOutput(0));
 }
 
+class NonMaxSuppressionLiteOpTest : public OpsTestBase {
+ protected:
+  void MakeOp() {
+    TF_EXPECT_OK(NodeDefBuilder("non_max_suppression_lite_op", 
+                                "NonMaxSuppressionLite")
+                     .Input(FakeInput(DT_FLOAT))
+                     .Input(FakeInput(DT_FLOAT))
+                     .Input(FakeInput(DT_INT32))
+                     .Input(FakeInput(DT_INT32))
+                     .Input(FakeInput(DT_FLOAT))
+                     .Input(FakeInput(DT_FLOAT))
+                     .Finalize(node_def()));
+    TF_EXPECT_OK(InitOp());
+  }
+};
+
+TEST_F(NonMaxSuppressionLiteOpTest, TestEmptyInput) {
+  MakeOp();
+  AddInputFromArray<float>(TensorShape({0, 0, 0, 4}), {});
+  AddInputFromArray<float>(TensorShape({0, 0, 0}), {});
+  AddInputFromArray<int>(TensorShape({}), {30});
+  AddInputFromArray<int>(TensorShape({}), {30});
+  AddInputFromArray<float>(TensorShape({}), {.5f});
+  AddInputFromArray<float>(TensorShape({}), {0.0f});
+  TF_ASSERT_OK(RunOpKernel());
+
+  //boxes
+  Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({0, 0, 4}));
+  test::FillValues<float>(&expected_boxes, {});
+  test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
+  
+  //scores
+  Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({0, 0}));
+  test::FillValues<float>(&expected_scores, {});
+  test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
+  
+  //classes
+  Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({0, 0}));
+  test::FillValues<float>(&expected_classes, {});
+  test::ExpectTensorEqual<float>(expected_classes, *GetOutput(2));
+  
+  // valid
+  Tensor expected_valid_d(allocator(), DT_INT32, TensorShape({0}));
+  test::FillValues<int>(&expected_valid_d, {});
+  test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
+  
+  // indices
+  Tensor expected_indices(allocator(), DT_INT32, TensorShape({0, 0}));
+  test::FillValues<int>(&expected_indices, {});
+  test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
+}
+
+TEST_F(NonMaxSuppressionLiteOpTest, TestSelectFromThreeClusters) {
+  MakeOp();
+  AddInputFromArray<float>(
+      TensorShape({1, 6, 1, 4}),
+      {0, 0,  1, 1,  0, 0.1f,  1, 1.1f,  0, -0.1f, 1, 0.9f,
+       0, 10, 1, 11, 0, 10.1f, 1, 11.1f, 0, 100,   1, 101});
+  AddInputFromArray<float>(TensorShape({1, 6, 1}), 
+       {.9f, .75f, .6f, .95f, .5f, .3f});
+  AddInputFromArray<int>(TensorShape({}), {3});
+  AddInputFromArray<int>(TensorShape({}), {3});
+  AddInputFromArray<float>(TensorShape({}), {.5f});
+  AddInputFromArray<float>(TensorShape({}), {0.0f});
+  TF_ASSERT_OK(RunOpKernel());
+  
+  //boxes
+  Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({1, 3, 4}));
+  test::FillValues<float>(&expected_boxes, {0, 10, 1, 11 ,0, 0, 1, 1, 
+          0, 100, 1, 101});
+  test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
+  //scores
+  Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({1, 3}));
+  test::FillValues<float>(&expected_scores, {0.95, 0.9, 0.3});
+  test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
+  //classes
+  Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({1, 3}));
+  test::FillValues<float>(&expected_classes, {0, 0, 0});
+  test::ExpectTensorEqual<float>(expected_classes, *GetOutput(2));
+  // valid
+  Tensor expected_valid_d(allocator(), DT_INT32, TensorShape({1}));
+  test::FillValues<int>(&expected_valid_d, {3});
+  test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
+  // indices
+  Tensor expected_indices(allocator(), DT_INT32, TensorShape({1, 3}));
+  test::FillValues<int>(&expected_indices, {3, 0, 5});
+  test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
+}
+
+TEST_F(NonMaxSuppressionLiteOpTest, 
+        TestSelectFromThreeClustersWithScoreThreshold) {
+  MakeOp();
+  AddInputFromArray<float>(
+      TensorShape({1, 6, 1, 4}),
+      {0, 0,  1, 1,  0, 0.1f,  1, 1.1f,  0, -0.1f, 1, 0.9f,
+       0, 10, 1, 11, 0, 10.1f, 1, 11.1f, 0, 100,   1, 101});
+  AddInputFromArray<float>(TensorShape({1, 6, 1}), 
+       {.9f, .75f, .6f, .95f, .5f, .3f});
+  AddInputFromArray<int>(TensorShape({}), {3});
+  AddInputFromArray<int>(TensorShape({}), {3});
+  AddInputFromArray<float>(TensorShape({}), {.5f});
+  AddInputFromArray<float>(TensorShape({}), {0.4f});
+  TF_ASSERT_OK(RunOpKernel());
+  
+  //boxes
+  Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({1, 2, 4}));
+  test::FillValues<float>(&expected_boxes, {0, 10, 1, 11, 0, 0, 1, 1});
+  test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
+  //scores
+  Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({1, 2}));
+  test::FillValues<float>(&expected_scores, {0.95, 0.9});
+  test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
+  //classes
+  Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({1, 2}));
+  test::FillValues<float>(&expected_classes, {0, 0});
+  test::ExpectTensorEqual<float>(expected_classes, *GetOutput(2));
+  // valid
+  Tensor expected_valid_d(allocator(), DT_INT32, TensorShape({1}));
+  test::FillValues<int>(&expected_valid_d, {2});
+  test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
+  // indices
+  Tensor expected_indices(allocator(), DT_INT32, TensorShape({1, 2}));
+  test::FillValues<int>(&expected_indices, {3, 0});
+  test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
+}
+
+TEST_F(NonMaxSuppressionLiteOpTest, 
+        TestSelectFromThreeClustersWithScoreThresholdZeroScores) {
+  MakeOp();
+  AddInputFromArray<float>(
+      TensorShape({1, 6, 1, 4}),
+      {0, 0,  1, 1,  0, 0.1f,  1, 1.1f,  0, -0.1f, 1, 0.9f,
+       0, 10, 1, 11, 0, 10.1f, 1, 11.1f, 0, 100,   1, 101});
+  AddInputFromArray<float>(TensorShape({1, 6, 1}), 
+       {.1f, 0, 0, .3f, .2f, -5.0f});
+  // If we ask for more boxes than we actually expect to get back;
+  // should still only get 2 boxes back.
+  AddInputFromArray<int>(TensorShape({}), {6});
+  AddInputFromArray<int>(TensorShape({}), {7});
+  AddInputFromArray<float>(TensorShape({}), {.5f});
+  AddInputFromArray<float>(TensorShape({}), {-3.0f});
+  TF_ASSERT_OK(RunOpKernel());
+  
+  //boxes
+  Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({1, 2, 4}));
+  test::FillValues<float>(&expected_boxes, {0, 10, 1, 11, 0, 0, 1, 1});
+  test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
+  //scores
+  Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({1, 2}));
+  test::FillValues<float>(&expected_scores, {0.3, 0.1});
+  test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
+  //classes
+  Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({1, 2}));
+  test::FillValues<float>(&expected_classes, {0, 0});
+  test::ExpectTensorEqual<float>(expected_classes, *GetOutput(2));
+  // valid
+  Tensor expected_valid_d(allocator(), DT_INT32, TensorShape({1}));
+  test::FillValues<int>(&expected_valid_d, {2});
+  test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
+  // indices
+  Tensor expected_indices(allocator(), DT_INT32, TensorShape({1, 2}));
+  test::FillValues<int>(&expected_indices, {3, 0});
+  test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
+}
+
+TEST_F(NonMaxSuppressionLiteOpTest, TestSelectSingleBox) {
+  MakeOp();
+  AddInputFromArray<float>(TensorShape({1, 1, 1, 4}), {0, 0, 1, 1});  
+  AddInputFromArray<float>(TensorShape({1, 1, 1}), {.9f});
+  AddInputFromArray<int>(TensorShape({}), {3});
+  AddInputFromArray<int>(TensorShape({}), {3});
+  AddInputFromArray<float>(TensorShape({}), {.5f});
+  AddInputFromArray<float>(TensorShape({}), {0.0f});
+  TF_ASSERT_OK(RunOpKernel());
+  
+  //boxes
+  Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({1, 1, 4}));
+  test::FillValues<float>(&expected_boxes, {0, 0, 1, 1});
+  test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
+  //scores
+  Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({1, 1}));
+  test::FillValues<float>(&expected_scores, {0.9});
+  test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
+  //classes
+  Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({1, 1}));
+  test::FillValues<float>(&expected_classes, {0});
+  test::ExpectTensorEqual<float>(expected_classes, *GetOutput(2));
+  // valid
+  Tensor expected_valid_d(allocator(), DT_INT32, TensorShape({1}));
+  test::FillValues<int>(&expected_valid_d, {1});
+  test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
+  // indices
+  Tensor expected_indices(allocator(), DT_INT32, TensorShape({1, 1}));
+  test::FillValues<int>(&expected_indices, {0});
+  test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
+}
+
+TEST_F(NonMaxSuppressionLiteOpTest, 
+        TestSelectFromTwoBatchesWithScoreThreshold) {
+  MakeOp();
+  AddInputFromArray<float>(
+      TensorShape({2, 6, 1, 4}),
+      {0, 0,  1, 1,  0, 0.1f,  1, 1.1f,  0, -0.1f, 1, 0.9f,
+       0, 10, 1, 11, 0, 10.1f, 1, 11.1f, 0, 100,   1, 101,
+       0, 0,  2, 2,  0, 0.2f,  2, 2.2f,  0, -0.2f, 2, 1.9f,
+       0, 20, 2, 22, 0, 20.2f, 2, 22.2f, 0, 200,   2, 202
+       });
+  AddInputFromArray<float>(TensorShape({2, 6, 1}), 
+       {.9f, .75f, .6f, .95f, .5f, .3f, .9f, .75f, .6f, .95f, .5f, .3f});
+  AddInputFromArray<int>(TensorShape({}), {3});
+  AddInputFromArray<int>(TensorShape({}), {9});
+  AddInputFromArray<float>(TensorShape({}), {.5f});
+  AddInputFromArray<float>(TensorShape({}), {0.4f});
+  TF_ASSERT_OK(RunOpKernel());
+  
+  //boxes
+  Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 2, 4}));
+  test::FillValues<float>(&expected_boxes, {0, 10, 1, 11 ,0, 0, 1, 1, 
+          0, 20, 2, 22 ,0, 0, 2, 2});
+  test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
+  //scores
+  Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 2}));
+  test::FillValues<float>(&expected_scores, {0.95, 0.9, 0.95, 0.9});
+  test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
+  //classes
+  Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({2, 2}));
+  test::FillValues<float>(&expected_classes, {0, 0, 0, 0});
+  test::ExpectTensorEqual<float>(expected_classes, *GetOutput(2));
+  // valid
+  Tensor expected_valid_d(allocator(), DT_INT32, TensorShape({2}));
+  test::FillValues<int>(&expected_valid_d, {2, 2});
+  test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
+  // indices
+  Tensor expected_indices(allocator(), DT_INT32, TensorShape({2, 2}));
+  test::FillValues<int>(&expected_indices, {3, 0, 3, 0});
+  test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
+}
+
+TEST_F(NonMaxSuppressionLiteOpTest, 
+        TestSelectFromTwoBatchesTwoClasses) {
+  MakeOp();
+  AddInputFromArray<float>(
+      TensorShape({2, 6, 1, 4}),
+      {0, 0,  1, 1,  0, 0.1f,  1, 1.1f,  0, -0.1f, 1, 0.9f,
+       0, 10, 1, 11, 0, 10.1f, 1, 11.1f, 0, 100,   1, 101,
+       0, 0,  2, 2,  0, 0.2f,  2, 2.2f,  0, -0.2f, 2, 1.9f,
+       0, 20, 2, 22, 0, 20.2f, 2, 22.2f, 0, 200,   2, 202
+       });
+  AddInputFromArray<float>(TensorShape({2, 6, 2}), 
+       {0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f, 
+       0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f}); 
+  AddInputFromArray<int>(TensorShape({}), {3});
+  AddInputFromArray<int>(TensorShape({}), {3});
+  AddInputFromArray<float>(TensorShape({}), {.5f});
+  AddInputFromArray<float>(TensorShape({}), {0.0f});
+  TF_ASSERT_OK(RunOpKernel());
+  
+  //boxes
+  Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 3, 4}));
+  test::FillValues<float>(&expected_boxes, {0, 10, 1, 11 ,0, 0, 1, 1, 
+          0, 0.1f, 1, 1.1f, 0, 20, 2, 22 ,0, 0, 2, 2, 0, 0.2f, 2, 2.2f});
+  test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
+  //scores
+  Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 3}));
+  test::FillValues<float>(&expected_scores, {0.95, 0.9, 0.75, 0.95, 0.9, 0.75});
+  test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
+  //classes
+  Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({2, 3}));
+  test::FillValues<float>(&expected_classes, {0, 1, 0, 0, 1, 0});
+  test::ExpectTensorEqual<float>(expected_classes, *GetOutput(2));
+  // valid
+  Tensor expected_valid_d(allocator(), DT_INT32, TensorShape({2}));
+  test::FillValues<int>(&expected_valid_d, {3, 3});
+  test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
+  // indices
+  Tensor expected_indices(allocator(), DT_INT32, TensorShape({2, 3}));
+  test::FillValues<int>(&expected_indices, {3, 0, 1, 3, 0, 1});
+  test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
+}
+
+TEST_F(NonMaxSuppressionLiteOpTest, 
+        TestSelectFromTwoBatchesTwoClassesWithScoreThreshold) {
+  MakeOp();
+  AddInputFromArray<float>(
+      TensorShape({2, 6, 1, 4}),
+      {0, 0,  1, 1,  0, 0.1f,  1, 1.1f,  0, -0.1f, 1, 0.9f,
+       0, 10, 1, 11, 0, 10.1f, 1, 11.1f, 0, 100,   1, 101,
+       0, 0,  2, 2,  0, 0.2f,  2, 2.2f,  0, -0.2f, 2, 1.9f,
+       0, 20, 2, 22, 0, 20.2f, 2, 22.2f, 0, 200,   2, 202
+       });
+  AddInputFromArray<float>(TensorShape({2, 6, 2}), 
+       {0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f, 
+       0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f}); 
+  AddInputFromArray<int>(TensorShape({}), {3});
+  AddInputFromArray<int>(TensorShape({}), {3});
+  AddInputFromArray<float>(TensorShape({}), {.5f});
+  AddInputFromArray<float>(TensorShape({}), {0.8f});
+  TF_ASSERT_OK(RunOpKernel());
+  
+  //boxes
+  Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 2, 4}));
+  test::FillValues<float>(&expected_boxes, {0, 10, 1, 11 ,0, 0, 1, 1, 
+          0, 20, 2, 22, 0, 0, 2, 2});
+  test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
+  //scores
+  Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 2}));
+  test::FillValues<float>(&expected_scores, {0.95, 0.9, 0.95, 0.9});
+  test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
+  //classes
+  Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({2, 2}));
+  test::FillValues<float>(&expected_classes, {0, 1, 0, 1});
+  test::ExpectTensorEqual<float>(expected_classes, *GetOutput(2));
+  // valid
+  Tensor expected_valid_d(allocator(), DT_INT32, TensorShape({2}));
+  test::FillValues<int>(&expected_valid_d, {2, 2});
+  test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
+  // indices
+  Tensor expected_indices(allocator(), DT_INT32, TensorShape({2, 2}));
+  test::FillValues<int>(&expected_indices, {3, 0, 3, 0});
+  test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
+}
+
+TEST_F(NonMaxSuppressionLiteOpTest, 
+        TestSelectFromTwoBatchesTwoClassesTotalSize) {
+  MakeOp();
+  AddInputFromArray<float>(
+      TensorShape({2, 6, 1, 4}),
+      {0, 0,  1, 1,  0, 0.1f,  1, 1.1f,  0, -0.1f, 1, 0.9f,
+       0, 10, 1, 11, 0, 10.1f, 1, 11.1f, 0, 100,   1, 101,
+       0, 0,  2, 2,  0, 0.2f,  2, 2.2f,  0, -0.2f, 2, 1.9f,
+       0, 20, 2, 22, 0, 20.2f, 2, 22.2f, 0, 200,   2, 202
+       });
+  AddInputFromArray<float>(TensorShape({2, 6, 2}), 
+      {0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f, 
+      0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f}); 
+  AddInputFromArray<int>(TensorShape({}), {3});
+  //Total size per batch is more than size per class
+  AddInputFromArray<int>(TensorShape({}), {5});
+  AddInputFromArray<float>(TensorShape({}), {.5f});
+  AddInputFromArray<float>(TensorShape({}), {0.1f});
+  TF_ASSERT_OK(RunOpKernel());
+  
+  //boxes
+  Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 5, 4}));
+  test::FillValues<float>(&expected_boxes, {0, 10, 1, 11, 0, 0, 1, 1, 
+          0, 0.1f, 1, 1.1f, 0, 10.1f, 1, 11.1f, 0, 100, 1, 101,
+          0, 20, 2, 22, 0, 0, 2, 2, 0, 0.2f, 2, 2.2f, 
+          0, 20.2f, 2, 22.2f, 0, 200, 2, 202});
+  test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
+  //scores
+  Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 5}));
+  test::FillValues<float>(&expected_scores, {0.95, 0.9, 0.75, 0.5, 0.3, 
+          0.95, 0.9, 0.75, 0.5, 0.3});
+  test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
+  //classes
+  Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({2, 5}));
+  test::FillValues<float>(&expected_classes, {0, 1, 0, 1, 0, 0, 1, 0, 1, 0});
+  test::ExpectTensorEqual<float>(expected_classes, *GetOutput(2));
+  // valid
+  Tensor expected_valid_d(allocator(), DT_INT32, TensorShape({2}));
+  test::FillValues<int>(&expected_valid_d, {5, 5});
+  test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
+  // indices
+  Tensor expected_indices(allocator(), DT_INT32, TensorShape({2, 5}));
+  test::FillValues<int>(&expected_indices, {3, 0, 1, 4, 5, 3, 0, 1, 4, 5});
+  test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
+}
+
+TEST_F(NonMaxSuppressionLiteOpTest, 
+        TestSelectFromTwoBatchesTwoClassesForBoxesAndScores) {
+  MakeOp();
+  AddInputFromArray<float>(
+      TensorShape({2, 6, 2, 4}),
+      //batch 0, box1 of class 1 should get selected
+      {0, 0, 1, 1, 0, 0,  1, 1, 0, 0.1f, 1, 1.1f, 0, 0.1f, 4, 4.1f, 
+      0, -0.1f, 1, 0.9f, 0, -0.1f, 1, 0.9f, 0, 10, 1, 11, 0, 10, 1, 11, 
+      0, 10.1f, 1, 11.1f, 0, 10.1f, 1, 11.1f, 0, 100, 1, 101, 0, 100, 1, 101, 
+      //batch 1, box1 of class 0 should get selected
+      0, 0, 2, 2, 0, 0, 2, 2, 0, 0.2f, 2, 2.2f, 0, 0.2f, 2, 2.2f, 
+      0, -0.2f, 2, 1.9f, 0, -0.2f, 2, 1.9f, 0, 20, 2, 22, 0, 20, 2, 22, 
+      0, 20.2f, 2, 22.2f, 0, 20.2f, 2, 22.2f, 0, 200, 2, 202, 0, 200, 2, 202});
+
+  AddInputFromArray<float>(TensorShape({2, 6, 2}), 
+       {0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f, 
+       0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f}); 
+  AddInputFromArray<int>(TensorShape({}), {3});
+  AddInputFromArray<int>(TensorShape({}), {3});
+  AddInputFromArray<float>(TensorShape({}), {.5f});
+  AddInputFromArray<float>(TensorShape({}), {0.0f});
+  TF_ASSERT_OK(RunOpKernel());
+  
+  //boxes
+  Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 3, 4}));
+  test::FillValues<float>(&expected_boxes, {0, 10, 1, 11, 0, 0, 1, 1, 
+          0, 0.1f, 4, 4.1f, 0, 20, 2, 22 ,0, 0, 2, 2, 0, 0.2f, 2, 2.2f});
+  test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
+  //scores
+  Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 3}));
+  test::FillValues<float>(&expected_scores, {0.95, 0.9, 0.8, 0.95, 0.9, 0.75});
+  test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
+  //classes
+  Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({2, 3}));
+  test::FillValues<float>(&expected_classes, {0, 1, 1, 0, 1, 0});
+  test::ExpectTensorEqual<float>(expected_classes, *GetOutput(2));
+  // valid
+  Tensor expected_valid_d(allocator(), DT_INT32, TensorShape({2}));
+  test::FillValues<int>(&expected_valid_d, {3, 3});
+  test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
+  // indices
+  Tensor expected_indices(allocator(), DT_INT32, TensorShape({2, 3}));
+  test::FillValues<int>(&expected_indices, {3, 0, 1, 3, 0, 1});
+  test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
+}
 }  // namespace tensorflow
diff --git a/tensorflow/core/ops/image_ops.cc b/tensorflow/core/ops/image_ops.cc
index 0077f0ffb5..b816cd61c5 100644
--- a/tensorflow/core/ops/image_ops.cc
+++ b/tensorflow/core/ops/image_ops.cc
@@ -819,4 +819,18 @@ REGISTER_OP("NonMaxSuppressionWithOverlaps")
       return Status::OK();
     });
 
+REGISTER_OP("NonMaxSuppressionLite")
+    .Input("boxes: float")
+    .Input("scores: float")
+    .Input("max_output_size_per_class: int32")
+    .Input("max_total_size: int32")
+    .Input("iou_threshold: float")
+    .Input("score_threshold: float")
+    .Output("nmsed_boxes: float")
+    .Output("nmsed_scores: float")
+    .Output("nmsed_classes: float")
+    .Output("valid_detections: int32")
+    .Output("selected_indices: int32")
+    .SetShapeFn(NMSLiteShapeFn);   
+
 }  // namespace tensorflow
-- 
GitLab


From 2d72aeb030b77bde5b376fd93476793a34b29220 Mon Sep 17 00:00:00 2001
From: Supriya Rao <supriyar@nvidia.com>
Date: Wed, 24 Oct 2018 15:19:59 -0700
Subject: [PATCH 1490/2345] Add NMSLite shape function

---
 tensorflow/core/ops/image_ops.cc | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/tensorflow/core/ops/image_ops.cc b/tensorflow/core/ops/image_ops.cc
index b816cd61c5..f12105f693 100644
--- a/tensorflow/core/ops/image_ops.cc
+++ b/tensorflow/core/ops/image_ops.cc
@@ -131,6 +131,35 @@ Status NMSShapeFn(InferenceContext* c) {
   return Status::OK();
 }
 
+Status NMSLiteShapeFn(InferenceContext* c) {
+  //Get inputs and validate ranks
+  ShapeHandle boxes;
+  //boxes is a tensor of Dimensions [batch_size, num_anchors, q, 4]
+  TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 4, &boxes));
+  ShapeHandle scores;
+  //scores is a tensor of Dimensions [batch_size, num_anchors, num_classes]
+  TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 3, &scores));
+  ShapeHandle max_output_size_per_class;
+  TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &max_output_size_per_class));
+  ShapeHandle iou_threshold;
+  TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 0, &iou_threshold));
+  ShapeHandle score_threshold;
+  TF_RETURN_IF_ERROR(c->WithRank(c->input(4), 0, &score_threshold));
+  DimensionHandle unused;
+  //boxes[3] is unused
+  TF_RETURN_IF_ERROR(c->WithValue(c->Dim(boxes, 3), 4, &unused));
+
+  //TODO SR Check dims of outputs here
+  c->set_output(0, c->Vector(c->UnknownDim()));
+  c->set_output(1, c->Vector(c->UnknownDim()));
+  c->set_output(2, c->Vector(c->UnknownDim()));
+  c->set_output(3, c->Vector(c->UnknownDim()));
+  c->set_output(4, c->Vector(c->UnknownDim()));
+
+  return Status::OK();
+}
+
+
 }  // namespace
 
 // --------------------------------------------------------------------------
-- 
GitLab


From a8af48eb1d4b4a3114c6140b4a6923ceae709338 Mon Sep 17 00:00:00 2001
From: Supriya Rao <supriyar@nvidia.com>
Date: Thu, 25 Oct 2018 16:05:44 -0700
Subject: [PATCH 1491/2345] Added support for padding to the max_total_size
 (per batch). Output is padded to zeros. The pad_to_max_total_size is an
 optional bool argument. The valid_detections output must be checked to get
 the number of valid outputs per batch.

---
 .../core/kernels/non_max_suppression_op.cc    | 64 +++++++++++--------
 .../kernels/non_max_suppression_op_test.cc    | 59 +++++++++++++++--
 tensorflow/core/ops/image_ops.cc              | 39 ++++++++---
 3 files changed, 118 insertions(+), 44 deletions(-)

diff --git a/tensorflow/core/kernels/non_max_suppression_op.cc b/tensorflow/core/kernels/non_max_suppression_op.cc
index 8733961857..01c592f838 100644
--- a/tensorflow/core/kernels/non_max_suppression_op.cc
+++ b/tensorflow/core/kernels/non_max_suppression_op.cc
@@ -97,9 +97,7 @@ static inline void ParseAndCheckNMSLiteBoxSizes(OpKernelContext* context,
   OP_REQUIRES(context, box_check,
               errors::InvalidArgument(
                   "third dimension of boxes must be either 1 or num classes"));
-  // Num_boxes * q where q is either 1 or num_classes
-  // TODO check if this should take num_classes into account
-  *num_boxes = boxes.dim_size(1); //*boxes.dim_size(2);
+  *num_boxes = boxes.dim_size(1); 
   OP_REQUIRES(context, boxes.dim_size(3) == 4,
               errors::InvalidArgument("boxes must have 4 columns"));
 }
@@ -228,9 +226,10 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
                            const Tensor& inp_boxes, const Tensor& inp_scores, 
                            int num_boxes, 
                            const Tensor& max_size_per_class, 
-                           const Tensor& max_total_size, 
+                           const int total_size_per_batch, 
                            const float score_threshold, 
-                           const float iou_threshold) {
+                           const float iou_threshold,
+                           bool pad_to_max_total_size = false) {
 
   int q = inp_boxes.dim_size(2);
   int num_classes = inp_scores.dim_size(2);
@@ -251,7 +250,6 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
   int max_detections = 0;
   //perform non_max_suppression operation for each batch independently
   for (int batch = 0; batch < num_batches; ++batch) {
-    std::cout << " iteration " << batch <<std::endl;
     // dims of per_batch_boxes [num_boxes, q, 4]
     //Tensor per_batch_boxes = inp_boxes.SubSlice(batch);
     Tensor per_batch_boxes = inp_boxes.Slice(batch, batch+1);
@@ -288,7 +286,6 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
             //Get the boxes per class. boxes_data_vec dims is [num_boxes, 4]
             boxes_data_vec.push_back(boxesData[box * q * 4 + 
                   class_idx * 4 + cid]);
-            //std::cout << " Pushing box_data " << boxes_data_vec[boxes_data_vec.size()-1] << std::endl;
           }
           else 
               boxes_data_vec.push_back(boxesData[box * 4 + cid]);
@@ -315,7 +312,6 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
       std::priority_queue<Candidate, std::deque<Candidate>, decltype(cmp)>
         candidate_priority_queue(cmp);
       for (int i = 0; i < scores_data.size(); ++i) {
-          //std::cout << "scores_data "<< scores_data[i] << std::endl;
         if (scores_data[i] > score_threshold) {
           candidate_priority_queue.emplace(Candidate({i, scores_data[i]}));
         }
@@ -328,11 +324,10 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
       const Tensor const_boxes = boxes;
       typename TTypes<float, 2>::ConstTensor boxes_data = 
         const_boxes.tensor<float, 2>();
-      LOG(ERROR) << " boxes data " << boxes.DebugString();
+      //LOG(ERROR) << " boxes data " << boxes.DebugString();
       while (selected.size() < output_size && !candidate_priority_queue.empty())
       {
         next_candidate = candidate_priority_queue.top();
-        //std::cout << " next_candidate " << next_candidate.box_index << std::endl;
         candidate_priority_queue.pop();
 
         // Overlapping boxes are likely to have similar scores,
@@ -342,7 +337,6 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
         for (int j = selected.size() - 1; j >= 0; --j) {
           if (IOUGreaterThanThreshold(boxes_data, next_candidate.box_index, 
               selected[j], iou_threshold)) {
-              //std::cout << " Dont select. box id is " << selected[j] << " idx " << j <<std::endl;  
             should_select = false;
             break;
           }
@@ -350,7 +344,6 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
 
         if (should_select) {
           selected.push_back(next_candidate.box_index);
-          //std::cout << " select box_index " << next_candidate.box_index << std::endl;
           //Add the selected box to the result candidate. Sorted by score
           int id = next_candidate.box_index;
           ResultCandidate rc = {next_candidate.box_index, next_candidate.score,
@@ -361,18 +354,16 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
       }
 
     }
-    int total_size_per_batch = max_total_size.scalar<int>()();
-    std::cout << " pq size " << result_candidate_pq.size() << std::endl;
-
     if (total_size_per_batch > 0)
       max_detections = std::min((int) result_candidate_pq.size(), 
         total_size_per_batch);
     else max_detections = (int) result_candidate_pq.size();
 
-    std::cout << " max_detections " << max_detections << std::endl;
     final_valid_detections.push_back(max_detections);
-    // Pick the top max_total_size values 
-    while(total_size_per_batch > 0 && !result_candidate_pq.empty())
+
+    int curr_total_size = total_size_per_batch;
+    // Pick the top total_size_per_batch values 
+    while(curr_total_size > 0 && !result_candidate_pq.empty())
     {
       ResultCandidate next_candidate = result_candidate_pq.top();
       result_candidate_pq.pop();
@@ -384,9 +375,19 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
       nmsed_scores[batch].push_back(next_candidate.score);
       nmsed_classes[batch].push_back(next_candidate.class_idx);
       selected_indices[batch].push_back(next_candidate.box_index);
-      total_size_per_batch--;
+      curr_total_size--;
+    }
+
+    if(pad_to_max_total_size) {
+      nmsed_boxes[batch].resize(total_size_per_batch * 4, 0);
+      nmsed_scores[batch].resize(total_size_per_batch, 0);
+      nmsed_classes[batch].resize(total_size_per_batch, 0);
+      selected_indices[batch].resize(total_size_per_batch, 0);
     }
-            
+  }
+
+  if(pad_to_max_total_size) { 
+    max_detections = total_size_per_batch;
   }
 
   Tensor* nmsed_boxes_t = nullptr;
@@ -414,7 +415,7 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
 
   Tensor* selected_indices_t = nullptr;
   OP_REQUIRES_OK(context,
-                 context->allocate_output(4, scores_shape, &selected_indices_t));
+                context->allocate_output(4, scores_shape, &selected_indices_t));
   auto selected_indices_flat = selected_indices_t->template flat<int>();
   for (int i = 0; i < num_batches; ++i) {
     valid_detections_flat(i) = final_valid_detections[i];
@@ -428,9 +429,9 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
       }
     }
   }
-  LOG(ERROR) << " boxes " << nmsed_boxes_t->DebugString();
-  LOG(ERROR) << " Scores " << nmsed_scores_t->DebugString();
-  LOG(ERROR) << " classes " << nmsed_classes_t->DebugString();
+  //LOG(ERROR) << " boxes " << nmsed_boxes_t->DebugString();
+  //LOG(ERROR) << " Scores " << nmsed_scores_t->DebugString();
+  //LOG(ERROR) << " classes " << nmsed_classes_t->DebugString();
 }
 
 }  // namespace
@@ -677,12 +678,14 @@ template <typename Device>
 class NonMaxSuppressionLiteOp : public OpKernel {
  public:
   explicit NonMaxSuppressionLiteOp(OpKernelConstruction* context)
-      : OpKernel(context) {}
+      : OpKernel(context) {
+    OP_REQUIRES_OK(context, context->GetAttr("pad_to_max_total_size",
+                                             &pad_to_max_total_size_));
+  }
 
   void Compute(OpKernelContext* context) override {
     // boxes: [batch_size, num_anchors, q, 4]
     const Tensor& boxes = context->input(0);
-    LOG(ERROR) << " inp_boxes " << boxes.DebugString();
     // scores: [batch_size, num_anchors, num_classes]
     const Tensor& scores = context->input(1);
     OP_REQUIRES(context, (boxes.dim_size(0) == scores.dim_size(0)),
@@ -701,6 +704,7 @@ class NonMaxSuppressionLiteOp : public OpKernel {
         context, TensorShapeUtils::IsScalar(max_total_size.shape()),
         errors::InvalidArgument("max_total_size must be 0-D, got shape ",
                                 max_total_size.shape().DebugString()));
+    const int max_total_size_per_batch = max_total_size.scalar<int>()();
     // iou_threshold: scalar
     const Tensor& iou_threshold = context->input(4);
     OP_REQUIRES(context, TensorShapeUtils::IsScalar(iou_threshold.shape()),
@@ -722,13 +726,17 @@ class NonMaxSuppressionLiteOp : public OpKernel {
     const int num_classes = scores.dim_size(2);
     ParseAndCheckNMSLiteBoxSizes(context, boxes, &num_boxes, num_classes);
     CheckNMSLiteScoreSizes(context, num_boxes, scores);
+
     if (!context->status().ok()) {
       return;
     }
     BatchedNonMaxSuppressionOp(context, boxes, scores, num_boxes, 
-                               max_output_size, max_total_size, 
-                               score_threshold_val, iou_threshold_val);
+                               max_output_size, max_total_size_per_batch, 
+                               score_threshold_val, iou_threshold_val, 
+                               pad_to_max_total_size_);
   }
+ private:
+  bool pad_to_max_total_size_;
 };
 
 
diff --git a/tensorflow/core/kernels/non_max_suppression_op_test.cc b/tensorflow/core/kernels/non_max_suppression_op_test.cc
index bffe388129..0d99d9887e 100644
--- a/tensorflow/core/kernels/non_max_suppression_op_test.cc
+++ b/tensorflow/core/kernels/non_max_suppression_op_test.cc
@@ -863,7 +863,7 @@ TEST_F(NonMaxSuppressionWithOverlapsOpTest, TestEmptyInput) {
 
 class NonMaxSuppressionLiteOpTest : public OpsTestBase {
  protected:
-  void MakeOp() {
+  void MakeOp(bool pad_to_max_total_size = false) {
     TF_EXPECT_OK(NodeDefBuilder("non_max_suppression_lite_op", 
                                 "NonMaxSuppressionLite")
                      .Input(FakeInput(DT_FLOAT))
@@ -872,6 +872,7 @@ class NonMaxSuppressionLiteOpTest : public OpsTestBase {
                      .Input(FakeInput(DT_INT32))
                      .Input(FakeInput(DT_FLOAT))
                      .Input(FakeInput(DT_FLOAT))
+                     .Attr("pad_to_max_total_size", pad_to_max_total_size)
                      .Finalize(node_def()));
     TF_EXPECT_OK(InitOp());
   }
@@ -1152,8 +1153,9 @@ TEST_F(NonMaxSuppressionLiteOpTest,
        0, 20, 2, 22, 0, 20.2f, 2, 22.2f, 0, 200,   2, 202
        });
   AddInputFromArray<float>(TensorShape({2, 6, 2}), 
-       {0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f, 
-       0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f}); 
+      {0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f, 
+       0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f
+       }); 
   AddInputFromArray<int>(TensorShape({}), {3});
   AddInputFromArray<int>(TensorShape({}), {3});
   AddInputFromArray<float>(TensorShape({}), {.5f});
@@ -1183,6 +1185,49 @@ TEST_F(NonMaxSuppressionLiteOpTest,
   test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
 
+TEST_F(NonMaxSuppressionLiteOpTest, 
+        TestSelectFromTwoBatchesTwoClassesWithScoreThresholdPadded) {
+  MakeOp(true);
+  AddInputFromArray<float>(
+      TensorShape({2, 6, 1, 4}),
+      {0, 0,  1, 1,  0, 0.1f,  1, 1.1f,  0, -0.1f, 1, 0.9f,
+       0, 10, 1, 11, 0, 10.1f, 1, 11.1f, 0, 100,   1, 101,
+       0, 0,  2, 2,  0, 0.2f,  2, 2.2f,  0, -0.2f, 2, 1.9f,
+       0, 20, 2, 22, 0, 20.2f, 2, 22.2f, 0, 200,   2, 202
+       });
+  AddInputFromArray<float>(TensorShape({2, 6, 2}), 
+      {0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f, 
+       0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f
+       }); 
+  AddInputFromArray<int>(TensorShape({}), {3});
+  AddInputFromArray<int>(TensorShape({}), {3});
+  AddInputFromArray<float>(TensorShape({}), {.5f});
+  AddInputFromArray<float>(TensorShape({}), {0.8f});
+  TF_ASSERT_OK(RunOpKernel());
+  
+  //boxes
+  Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 3, 4}));
+  test::FillValues<float>(&expected_boxes, {0, 10, 1, 11 ,0, 0, 1, 1, 
+          0, 0, 0, 0, 0, 20, 2, 22, 0, 0, 2, 2, 0, 0, 0, 0});
+  test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
+  //scores
+  Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 3}));
+  test::FillValues<float>(&expected_scores, {0.95, 0.9, 0, 0.95, 0.9, 0});
+  test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
+  //classes
+  Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({2, 3}));
+  test::FillValues<float>(&expected_classes, {0, 1, 0, 0, 1, 0});
+  test::ExpectTensorEqual<float>(expected_classes, *GetOutput(2));
+  // valid
+  Tensor expected_valid_d(allocator(), DT_INT32, TensorShape({2}));
+  test::FillValues<int>(&expected_valid_d, {2, 2});
+  test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
+  // indices
+  Tensor expected_indices(allocator(), DT_INT32, TensorShape({2, 3}));
+  test::FillValues<int>(&expected_indices, {3, 0, 0, 3, 0, 0});
+  test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
+}
+
 TEST_F(NonMaxSuppressionLiteOpTest, 
         TestSelectFromTwoBatchesTwoClassesTotalSize) {
   MakeOp();
@@ -1195,7 +1240,8 @@ TEST_F(NonMaxSuppressionLiteOpTest,
        });
   AddInputFromArray<float>(TensorShape({2, 6, 2}), 
       {0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f, 
-      0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f}); 
+       0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f
+       }); 
   AddInputFromArray<int>(TensorShape({}), {3});
   //Total size per batch is more than size per class
   AddInputFromArray<int>(TensorShape({}), {5});
@@ -1244,8 +1290,9 @@ TEST_F(NonMaxSuppressionLiteOpTest,
       0, 20.2f, 2, 22.2f, 0, 20.2f, 2, 22.2f, 0, 200, 2, 202, 0, 200, 2, 202});
 
   AddInputFromArray<float>(TensorShape({2, 6, 2}), 
-       {0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f, 
-       0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f}); 
+      {0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f, 
+       0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f
+       }); 
   AddInputFromArray<int>(TensorShape({}), {3});
   AddInputFromArray<int>(TensorShape({}), {3});
   AddInputFromArray<float>(TensorShape({}), {.5f});
diff --git a/tensorflow/core/ops/image_ops.cc b/tensorflow/core/ops/image_ops.cc
index f12105f693..6e4d27f9eb 100644
--- a/tensorflow/core/ops/image_ops.cc
+++ b/tensorflow/core/ops/image_ops.cc
@@ -141,21 +141,39 @@ Status NMSLiteShapeFn(InferenceContext* c) {
   TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 3, &scores));
   ShapeHandle max_output_size_per_class;
   TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &max_output_size_per_class));
+  ShapeHandle max_total_size;
+  TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 0, &max_total_size));
   ShapeHandle iou_threshold;
-  TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 0, &iou_threshold));
+  TF_RETURN_IF_ERROR(c->WithRank(c->input(4), 0, &iou_threshold));
   ShapeHandle score_threshold;
-  TF_RETURN_IF_ERROR(c->WithRank(c->input(4), 0, &score_threshold));
+  TF_RETURN_IF_ERROR(c->WithRank(c->input(5), 0, &score_threshold));
+
   DimensionHandle unused;
-  //boxes[3] is unused
+  //boxes[1] and scores[1] are both num_anchors
+  TF_RETURN_IF_ERROR(c->Merge(c->Dim(boxes, 1), c->Dim(scores, 1), &unused));
+  // The boxes[3] is 4.
   TF_RETURN_IF_ERROR(c->WithValue(c->Dim(boxes, 3), 4, &unused));
 
-  //TODO SR Check dims of outputs here
-  c->set_output(0, c->Vector(c->UnknownDim()));
-  c->set_output(1, c->Vector(c->UnknownDim()));
-  c->set_output(2, c->Vector(c->UnknownDim()));
-  c->set_output(3, c->Vector(c->UnknownDim()));
-  c->set_output(4, c->Vector(c->UnknownDim()));
-
+  bool pad_to_max;
+  TF_RETURN_IF_ERROR(c->GetAttr("pad_to_max_total_size", &pad_to_max));
+  if(pad_to_max) {
+    DimensionHandle output_dim;
+    DimensionHandle batch_dim = c->UnknownDim();
+
+    TF_RETURN_IF_ERROR(c->MakeDimForScalarInput(3, &output_dim));
+    c->set_output(0, c->MakeShape({batch_dim, output_dim, 4}));
+    c->set_output(1, c->MakeShape({batch_dim, output_dim}));
+    c->set_output(2, c->MakeShape({batch_dim, output_dim}));
+    c->set_output(4, c->MakeShape({batch_dim, output_dim}));
+  }
+  else {
+    //TODO SR Can all the outputs be unknowndim?
+    c->set_output(0, c->Vector(c->UnknownDim()));
+    c->set_output(1, c->Vector(c->UnknownDim()));
+    c->set_output(2, c->Vector(c->UnknownDim()));
+    c->set_output(3, c->Vector(c->UnknownDim()));
+    c->set_output(4, c->Vector(c->UnknownDim()));
+  }
   return Status::OK();
 }
 
@@ -860,6 +878,7 @@ REGISTER_OP("NonMaxSuppressionLite")
     .Output("nmsed_classes: float")
     .Output("valid_detections: int32")
     .Output("selected_indices: int32")
+    .Attr("pad_to_max_total_size: bool = false")
     .SetShapeFn(NMSLiteShapeFn);   
 
 }  // namespace tensorflow
-- 
GitLab


From ee3b59caeaaffcffa2e3ef2b1e5a9ece67783d76 Mon Sep 17 00:00:00 2001
From: Supriya Rao <supriyar@nvidia.com>
Date: Tue, 30 Oct 2018 15:35:24 -0700
Subject: [PATCH 1492/2345] Updated with Debug messages (for dev effort only)
 and added clip box param

---
 .../core/kernels/non_max_suppression_op.cc    | 102 +++++++++++++++---
 .../kernels/non_max_suppression_op_test.cc    |   2 +-
 tensorflow/python/ops/image_ops_impl.py       |  65 +++++++++++
 3 files changed, 153 insertions(+), 16 deletions(-)

diff --git a/tensorflow/core/kernels/non_max_suppression_op.cc b/tensorflow/core/kernels/non_max_suppression_op.cc
index 01c592f838..88ad63e534 100644
--- a/tensorflow/core/kernels/non_max_suppression_op.cc
+++ b/tensorflow/core/kernels/non_max_suppression_op.cc
@@ -194,6 +194,7 @@ void DoNonMaxSuppressionOp(
     for (int j = static_cast<int>(selected.size()) - 1; j >= 0; --j) {
       if (suppress_check_fn(next_candidate.box_index, selected[j])) {
         should_select = false;
+        //std::cout << " Dont select " << std::endl;
         break;
       }
     }
@@ -201,6 +202,8 @@ void DoNonMaxSuppressionOp(
     if (should_select) {
       selected.push_back(next_candidate.box_index);
       selected_scores.push_back(next_candidate.score);
+      //std::cout << " Selected index " << next_candidate.box_index << std::endl;
+      //std::cout << " Selected score " << next_candidate.score << std::endl;
     }
   }
 
@@ -229,13 +232,28 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
                            const int total_size_per_batch, 
                            const float score_threshold, 
                            const float iou_threshold,
-                           bool pad_to_max_total_size = false) {
+                           bool pad_to_max_total_size = false, 
+                           const Tensor* clip_box_window = nullptr) {
 
   int q = inp_boxes.dim_size(2);
   int num_classes = inp_scores.dim_size(2);
   //unpack along batch dimension
   const int num_batches = inp_boxes.dim_size(0);
 
+  // Default clip window of [0, 0, 1, 1] if none specified
+  bool share_clip_window = true;
+  std::vector <float> clip_window{0, 0, 1, 1};
+  
+  if (clip_box_window != nullptr) {
+    if (clip_box_window->dims() == 2) {
+      share_clip_window = false;
+    }
+    else {
+      auto value_clip_box = clip_box_window->unaligned_flat<float>();
+      std::copy_n(&value_clip_box(0), 4, clip_window.begin()); 
+    }
+  }
+
   // [num_batches, max_detections, 4]
   std::vector <std::vector<float>> nmsed_boxes(num_batches);
   // [num_batches, max_detections]
@@ -256,11 +274,17 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
     // dims of per_batch_scores [num_boxes, num_classes]
     Tensor per_batch_scores = inp_scores.Slice(batch, batch+1);
 
+    if (!share_clip_window) {
+      auto value_clip_box = 
+          clip_box_window->Slice(batch, batch+1).unaligned_flat<float>();
+      std::copy_n(&value_clip_box(0), 4, clip_window.begin());
+    }
+
     struct ResultCandidate {
-          int box_index;
-          float score;
-          int class_idx;
-          float box_coord[4];
+      int box_index;
+      float score;
+      int class_idx;
+      float box_coord[4];
     };
 
     auto rc_cmp = [](const ResultCandidate rc_i, const ResultCandidate rc_j) {
@@ -338,6 +362,7 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
           if (IOUGreaterThanThreshold(boxes_data, next_candidate.box_index, 
               selected[j], iou_threshold)) {
             should_select = false;
+            //std::cout << " Dont select  for class id" << class_idx << std::endl;
             break;
           }
         }
@@ -361,35 +386,78 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
 
     final_valid_detections.push_back(max_detections);
 
-    int curr_total_size = total_size_per_batch;
-    // Pick the top total_size_per_batch values 
+    int curr_total_size = max_detections;
+    // Pick the top max_detections values 
     while(curr_total_size > 0 && !result_candidate_pq.empty())
     {
       ResultCandidate next_candidate = result_candidate_pq.top();
       result_candidate_pq.pop();
       // Add to final output vectors
-      nmsed_boxes[batch].push_back(next_candidate.box_coord[0]);
-      nmsed_boxes[batch].push_back(next_candidate.box_coord[1]);
-      nmsed_boxes[batch].push_back(next_candidate.box_coord[2]);
-      nmsed_boxes[batch].push_back(next_candidate.box_coord[3]);
+      nmsed_boxes[batch].push_back(
+              std::max(next_candidate.box_coord[0], clip_window[0]));
+      nmsed_boxes[batch].push_back(
+              std::max(next_candidate.box_coord[1], clip_window[1]));
+      nmsed_boxes[batch].push_back(
+              std::min(next_candidate.box_coord[2], clip_window[2]));
+      nmsed_boxes[batch].push_back(
+              std::min(next_candidate.box_coord[3], clip_window[3]));
       nmsed_scores[batch].push_back(next_candidate.score);
       nmsed_classes[batch].push_back(next_candidate.class_idx);
       selected_indices[batch].push_back(next_candidate.box_index);
+      //std::cout << " selected index BS " << batch << " is " << next_candidate.box_index << std::endl;
+      //std::cout << " selected score is " << next_candidate.score << std::endl;
+      //std::cout << " selected class is " << next_candidate.class_idx << std::endl;
+      //std::cout << " box ymin " << next_candidate.box_coord[0] << std::endl;
+      //std::cout << " box xmin " << next_candidate.box_coord[1] << std::endl;
+      //std::cout << " box ymax " << next_candidate.box_coord[2] << std::endl;
+      //std::cout << " box xmax " << next_candidate.box_coord[3] << std::endl;
       curr_total_size--;
     }
 
-    if(pad_to_max_total_size) {
+    if(pad_to_max_total_size && max_detections < total_size_per_batch) {
+    //if(max_detections < total_size_per_batch) {
       nmsed_boxes[batch].resize(total_size_per_batch * 4, 0);
       nmsed_scores[batch].resize(total_size_per_batch, 0);
       nmsed_classes[batch].resize(total_size_per_batch, 0);
       selected_indices[batch].resize(total_size_per_batch, 0);
+      //max_detections = total_size_per_batch;
     }
   }
+  //Current TF implementation :
+  // If use_static_shapes is true, max_total_size is ignored and we pad to max_size_per_class
+  // Else always pad to max_total_size
 
-  if(pad_to_max_total_size) { 
+  //TODO make max_detections an array
+  //bool pad_to_max_detections = false;
+  if (pad_to_max_total_size) { 
     max_detections = total_size_per_batch;
   }
-
+  //For the case where padding is set to false, and each batch has a different
+  //max_detections value, we need to pad it anyway to the max among them.
+  //else {
+  //  for (auto detect : final_valid_detections) {
+  //    if (max_detections < detect) {
+  //      max_detections = detect;
+  //      pad_to_max_detections = true;
+  //    }
+  //  }
+  //}
+  // 
+  ////TODO this is still broken because, if pad = 0 and max_detect is 
+  //// 2 and 3 for BS 0 and 1, we will try to copy 3 values
+  //if (pad_to_max_detections) {
+  //  std::cout << " **** Never tested this before!! **** " << std::endl;
+  //  for (int b = 0; b < num_batches; ++b) {
+  //    if (final_valid_detections[b] < max_detections) {
+  //      nmsed_boxes[b].resize(max_detections * 4, 0);
+  //      nmsed_scores[b].resize(max_detections, 0);
+  //      nmsed_classes[b].resize(max_detections, 0);
+  //      selected_indices[b].resize(max_detections,  0);
+  //    }
+  //  }
+  //}
+
+  //std::cout << " Max detections "<< max_detections << std::endl;
   Tensor* nmsed_boxes_t = nullptr;
   TensorShape boxes_shape({num_batches, max_detections, 4});
   OP_REQUIRES_OK(context,
@@ -430,8 +498,10 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
     }
   }
   //LOG(ERROR) << " boxes " << nmsed_boxes_t->DebugString();
-  //LOG(ERROR) << " Scores " << nmsed_scores_t->DebugString();
+  //std::cout << " Scores " << nmsed_scores_t->DebugString() << std::endl;
   //LOG(ERROR) << " classes " << nmsed_classes_t->DebugString();
+  //LOG(ERROR) << " selected_indices_t " << selected_indices_t->DebugString();
+  //LOG(ERROR) << " valid_detections_t " << valid_detections_t->DebugString();
 }
 
 }  // namespace
@@ -705,6 +775,8 @@ class NonMaxSuppressionLiteOp : public OpKernel {
         errors::InvalidArgument("max_total_size must be 0-D, got shape ",
                                 max_total_size.shape().DebugString()));
     const int max_total_size_per_batch = max_total_size.scalar<int>()();
+    OP_REQUIRES(context, max_total_size_per_batch > 0,
+                errors::InvalidArgument("max_total_size must be > 0"));
     // iou_threshold: scalar
     const Tensor& iou_threshold = context->input(4);
     OP_REQUIRES(context, TensorShapeUtils::IsScalar(iou_threshold.shape()),
diff --git a/tensorflow/core/kernels/non_max_suppression_op_test.cc b/tensorflow/core/kernels/non_max_suppression_op_test.cc
index 0d99d9887e..cc13bb0830 100644
--- a/tensorflow/core/kernels/non_max_suppression_op_test.cc
+++ b/tensorflow/core/kernels/non_max_suppression_op_test.cc
@@ -883,7 +883,7 @@ TEST_F(NonMaxSuppressionLiteOpTest, TestEmptyInput) {
   AddInputFromArray<float>(TensorShape({0, 0, 0, 4}), {});
   AddInputFromArray<float>(TensorShape({0, 0, 0}), {});
   AddInputFromArray<int>(TensorShape({}), {30});
-  AddInputFromArray<int>(TensorShape({}), {30});
+  AddInputFromArray<int>(TensorShape({}), {10});
   AddInputFromArray<float>(TensorShape({}), {.5f});
   AddInputFromArray<float>(TensorShape({}), {0.0f});
   TF_ASSERT_OK(RunOpKernel());
diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index 56ef52e883..e48ac0db06 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -3210,3 +3210,68 @@ def extract_glimpse_v2(
       noise=noise,
       uniform_noise=False,
       name=name)
+
+@tf_export('image.non_max_suppression_lite')
+def non_max_suppression_lite(boxes,
+                             scores,
+                             max_output_size_per_class,
+                             max_total_size,
+                             iou_threshold=0.5,
+                             score_threshold=float('-inf'),
+                             pad_to_max_total_size=False,
+                             name=None):
+  """Greedily selects a subset of bounding boxes in descending order of score.
+
+  This operation performs non_max_suppression on the inputs per batch, across
+  all classes. 
+  Prunes away boxes that have high intersection-over-union (IOU) overlap
+  with previously selected boxes.  Bounding boxes are supplied as
+  [y1, x1, y2, x2], where (y1, x1) and (y2, x2) are the coordinates of any
+  diagonal pair of box corners and the coordinates can be provided as normalized
+  (i.e., lying in the interval [0, 1]) or absolute.  Note that this algorithm
+  is agnostic to where the origin is in the coordinate system.  Note that this
+  algorithm is invariant to orthogonal transformations and translations
+  of the coordinate system; thus translating or reflections of the coordinate
+  system result in the same boxes being selected by the algorithm.
+  The output of this operation is the final boxes, scores and classes tensor 
+  returned after performing non_max_suppression.
+
+  Args:
+    boxes: A 4-D float `Tensor` of shape `[batch_size, num_boxes, q, 4]`.
+    scores: A 3-D float `Tensor` of shape `[batch_size, num_boxes, num_classes]` 
+    representing a single score corresponding to each box (each row of boxes).
+    max_output_size_per_class: A scalar integer `Tensor` representing the 
+    maximum number of boxes to be selected by non max suppression per class
+    max_total_size: A scalar representing maximum number of boxes retained over
+    all classes.
+    iou_threshold: A float representing the threshold for deciding whether boxes
+      overlap too much with respect to IOU.
+    score_threshold: A float representing the threshold for deciding when to
+      remove boxes based on score.
+    pad_to_max_total_size: If true the size of outputs is padded to
+    max_total_size per batch.
+    name: A name for the operation (optional).
+
+  Returns:
+    'nmsed_boxes': A [batch_size, max_detections, 4] float32 tensor
+      containing the non-max suppressed boxes.
+    'nmsed_scores': A [batch_size, max_detections] float32 tensor containing
+      the scores for the boxes.
+    'nmsed_classes': A [batch_size, max_detections] float32 tensor
+      containing the class for boxes.
+    'num_detections': A [batch_size] int32 tensor indicating the number of
+      valid detections per batch item. Only the top num_detections[i] entries in
+      nms_boxes[i], nms_scores[i] and nms_class[i] are valid. The rest of the
+      entries are zero paddings.
+    'selected_indices':  A [batch_size, max_detections] int32 tensor
+      containing the indices of the selected boxes
+  """
+  with ops.name_scope(name, 'non_max_suppression_lite'):
+    print(" *** calling actual non_max_suppression_lite *** ")
+    iou_threshold = ops.convert_to_tensor(iou_threshold, name='iou_threshold')
+    score_threshold = ops.convert_to_tensor(
+        score_threshold, name='score_threshold')
+    return gen_image_ops.non_max_suppression_lite(boxes, scores,
+            max_output_size_per_class, max_total_size, iou_threshold, 
+            score_threshold, pad_to_max_total_size);
+
-- 
GitLab


From a930749fc97ca14e4f9ad262f5b74631b89be916 Mon Sep 17 00:00:00 2001
From: Supriya Rao <supriyar@nvidia.com>
Date: Tue, 30 Oct 2018 15:57:46 -0700
Subject: [PATCH 1493/2345]  Remove debug prints

---
 .../core/kernels/non_max_suppression_op.cc    | 52 +------------------
 tensorflow/python/ops/image_ops_impl.py       |  1 -
 2 files changed, 1 insertion(+), 52 deletions(-)

diff --git a/tensorflow/core/kernels/non_max_suppression_op.cc b/tensorflow/core/kernels/non_max_suppression_op.cc
index 88ad63e534..c67e17bef2 100644
--- a/tensorflow/core/kernels/non_max_suppression_op.cc
+++ b/tensorflow/core/kernels/non_max_suppression_op.cc
@@ -194,7 +194,6 @@ void DoNonMaxSuppressionOp(
     for (int j = static_cast<int>(selected.size()) - 1; j >= 0; --j) {
       if (suppress_check_fn(next_candidate.box_index, selected[j])) {
         should_select = false;
-        //std::cout << " Dont select " << std::endl;
         break;
       }
     }
@@ -202,8 +201,6 @@ void DoNonMaxSuppressionOp(
     if (should_select) {
       selected.push_back(next_candidate.box_index);
       selected_scores.push_back(next_candidate.score);
-      //std::cout << " Selected index " << next_candidate.box_index << std::endl;
-      //std::cout << " Selected score " << next_candidate.score << std::endl;
     }
   }
 
@@ -269,7 +266,6 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
   //perform non_max_suppression operation for each batch independently
   for (int batch = 0; batch < num_batches; ++batch) {
     // dims of per_batch_boxes [num_boxes, q, 4]
-    //Tensor per_batch_boxes = inp_boxes.SubSlice(batch);
     Tensor per_batch_boxes = inp_boxes.Slice(batch, batch+1);
     // dims of per_batch_scores [num_boxes, num_classes]
     Tensor per_batch_scores = inp_scores.Slice(batch, batch+1);
@@ -348,7 +344,6 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
       const Tensor const_boxes = boxes;
       typename TTypes<float, 2>::ConstTensor boxes_data = 
         const_boxes.tensor<float, 2>();
-      //LOG(ERROR) << " boxes data " << boxes.DebugString();
       while (selected.size() < output_size && !candidate_priority_queue.empty())
       {
         next_candidate = candidate_priority_queue.top();
@@ -362,7 +357,6 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
           if (IOUGreaterThanThreshold(boxes_data, next_candidate.box_index, 
               selected[j], iou_threshold)) {
             should_select = false;
-            //std::cout << " Dont select  for class id" << class_idx << std::endl;
             break;
           }
         }
@@ -404,60 +398,21 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
       nmsed_scores[batch].push_back(next_candidate.score);
       nmsed_classes[batch].push_back(next_candidate.class_idx);
       selected_indices[batch].push_back(next_candidate.box_index);
-      //std::cout << " selected index BS " << batch << " is " << next_candidate.box_index << std::endl;
-      //std::cout << " selected score is " << next_candidate.score << std::endl;
-      //std::cout << " selected class is " << next_candidate.class_idx << std::endl;
-      //std::cout << " box ymin " << next_candidate.box_coord[0] << std::endl;
-      //std::cout << " box xmin " << next_candidate.box_coord[1] << std::endl;
-      //std::cout << " box ymax " << next_candidate.box_coord[2] << std::endl;
-      //std::cout << " box xmax " << next_candidate.box_coord[3] << std::endl;
       curr_total_size--;
     }
 
     if(pad_to_max_total_size && max_detections < total_size_per_batch) {
-    //if(max_detections < total_size_per_batch) {
       nmsed_boxes[batch].resize(total_size_per_batch * 4, 0);
       nmsed_scores[batch].resize(total_size_per_batch, 0);
       nmsed_classes[batch].resize(total_size_per_batch, 0);
       selected_indices[batch].resize(total_size_per_batch, 0);
-      //max_detections = total_size_per_batch;
     }
   }
-  //Current TF implementation :
-  // If use_static_shapes is true, max_total_size is ignored and we pad to max_size_per_class
-  // Else always pad to max_total_size
 
-  //TODO make max_detections an array
-  //bool pad_to_max_detections = false;
   if (pad_to_max_total_size) { 
     max_detections = total_size_per_batch;
   }
-  //For the case where padding is set to false, and each batch has a different
-  //max_detections value, we need to pad it anyway to the max among them.
-  //else {
-  //  for (auto detect : final_valid_detections) {
-  //    if (max_detections < detect) {
-  //      max_detections = detect;
-  //      pad_to_max_detections = true;
-  //    }
-  //  }
-  //}
-  // 
-  ////TODO this is still broken because, if pad = 0 and max_detect is 
-  //// 2 and 3 for BS 0 and 1, we will try to copy 3 values
-  //if (pad_to_max_detections) {
-  //  std::cout << " **** Never tested this before!! **** " << std::endl;
-  //  for (int b = 0; b < num_batches; ++b) {
-  //    if (final_valid_detections[b] < max_detections) {
-  //      nmsed_boxes[b].resize(max_detections * 4, 0);
-  //      nmsed_scores[b].resize(max_detections, 0);
-  //      nmsed_classes[b].resize(max_detections, 0);
-  //      selected_indices[b].resize(max_detections,  0);
-  //    }
-  //  }
-  //}
-
-  //std::cout << " Max detections "<< max_detections << std::endl;
+
   Tensor* nmsed_boxes_t = nullptr;
   TensorShape boxes_shape({num_batches, max_detections, 4});
   OP_REQUIRES_OK(context,
@@ -497,11 +452,6 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
       }
     }
   }
-  //LOG(ERROR) << " boxes " << nmsed_boxes_t->DebugString();
-  //std::cout << " Scores " << nmsed_scores_t->DebugString() << std::endl;
-  //LOG(ERROR) << " classes " << nmsed_classes_t->DebugString();
-  //LOG(ERROR) << " selected_indices_t " << selected_indices_t->DebugString();
-  //LOG(ERROR) << " valid_detections_t " << valid_detections_t->DebugString();
 }
 
 }  // namespace
diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index e48ac0db06..4d23509d8b 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -3267,7 +3267,6 @@ def non_max_suppression_lite(boxes,
       containing the indices of the selected boxes
   """
   with ops.name_scope(name, 'non_max_suppression_lite'):
-    print(" *** calling actual non_max_suppression_lite *** ")
     iou_threshold = ops.convert_to_tensor(iou_threshold, name='iou_threshold')
     score_threshold = ops.convert_to_tensor(
         score_threshold, name='score_threshold')
-- 
GitLab


From 6b300cbc9c8b4c77df40923102bef989048c9ece Mon Sep 17 00:00:00 2001
From: Supriya Rao <supriyar@nvidia.com>
Date: Thu, 1 Nov 2018 16:39:00 -0700
Subject: [PATCH 1494/2345] Updated kernel unit tests to account for clipped
 bounding box

---
 .../core/kernels/non_max_suppression_op.cc    |  20 +---
 .../kernels/non_max_suppression_op_test.cc    | 108 ++++++++++--------
 tensorflow/python/ops/image_ops_impl.py       |   5 +-
 3 files changed, 61 insertions(+), 72 deletions(-)

diff --git a/tensorflow/core/kernels/non_max_suppression_op.cc b/tensorflow/core/kernels/non_max_suppression_op.cc
index c67e17bef2..1e7fa863a3 100644
--- a/tensorflow/core/kernels/non_max_suppression_op.cc
+++ b/tensorflow/core/kernels/non_max_suppression_op.cc
@@ -229,8 +229,7 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
                            const int total_size_per_batch, 
                            const float score_threshold, 
                            const float iou_threshold,
-                           bool pad_to_max_total_size = false, 
-                           const Tensor* clip_box_window = nullptr) {
+                           bool pad_to_max_total_size = false) {
 
   int q = inp_boxes.dim_size(2);
   int num_classes = inp_scores.dim_size(2);
@@ -238,19 +237,8 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
   const int num_batches = inp_boxes.dim_size(0);
 
   // Default clip window of [0, 0, 1, 1] if none specified
-  bool share_clip_window = true;
   std::vector <float> clip_window{0, 0, 1, 1};
   
-  if (clip_box_window != nullptr) {
-    if (clip_box_window->dims() == 2) {
-      share_clip_window = false;
-    }
-    else {
-      auto value_clip_box = clip_box_window->unaligned_flat<float>();
-      std::copy_n(&value_clip_box(0), 4, clip_window.begin()); 
-    }
-  }
-
   // [num_batches, max_detections, 4]
   std::vector <std::vector<float>> nmsed_boxes(num_batches);
   // [num_batches, max_detections]
@@ -270,12 +258,6 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
     // dims of per_batch_scores [num_boxes, num_classes]
     Tensor per_batch_scores = inp_scores.Slice(batch, batch+1);
 
-    if (!share_clip_window) {
-      auto value_clip_box = 
-          clip_box_window->Slice(batch, batch+1).unaligned_flat<float>();
-      std::copy_n(&value_clip_box(0), 4, clip_window.begin());
-    }
-
     struct ResultCandidate {
       int box_index;
       float score;
diff --git a/tensorflow/core/kernels/non_max_suppression_op_test.cc b/tensorflow/core/kernels/non_max_suppression_op_test.cc
index cc13bb0830..d5662a15a0 100644
--- a/tensorflow/core/kernels/non_max_suppression_op_test.cc
+++ b/tensorflow/core/kernels/non_max_suppression_op_test.cc
@@ -918,8 +918,8 @@ TEST_F(NonMaxSuppressionLiteOpTest, TestSelectFromThreeClusters) {
   MakeOp();
   AddInputFromArray<float>(
       TensorShape({1, 6, 1, 4}),
-      {0, 0,  1, 1,  0, 0.1f,  1, 1.1f,  0, -0.1f, 1, 0.9f,
-       0, 10, 1, 11, 0, 10.1f, 1, 11.1f, 0, 100,   1, 101});
+      {0, 0,    0.1, 0.1, 0, 0.01f, 0.1, 0.11f, 0, -0.01, 0.1, 0.09f,
+       0, 0.11, 0.1, 0.2, 0, 0.12f, 0.1, 0.21f, 0,  0.3, 1,   0.4});
   AddInputFromArray<float>(TensorShape({1, 6, 1}), 
        {.9f, .75f, .6f, .95f, .5f, .3f});
   AddInputFromArray<int>(TensorShape({}), {3});
@@ -930,8 +930,8 @@ TEST_F(NonMaxSuppressionLiteOpTest, TestSelectFromThreeClusters) {
   
   //boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({1, 3, 4}));
-  test::FillValues<float>(&expected_boxes, {0, 10, 1, 11 ,0, 0, 1, 1, 
-          0, 100, 1, 101});
+  test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 
+          0, 0.3, 1, 0.4});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
   //scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({1, 3}));
@@ -956,8 +956,8 @@ TEST_F(NonMaxSuppressionLiteOpTest,
   MakeOp();
   AddInputFromArray<float>(
       TensorShape({1, 6, 1, 4}),
-      {0, 0,  1, 1,  0, 0.1f,  1, 1.1f,  0, -0.1f, 1, 0.9f,
-       0, 10, 1, 11, 0, 10.1f, 1, 11.1f, 0, 100,   1, 101});
+      {0, 0,    0.1, 0.1, 0, 0.01f, 0.1, 0.11f, 0, -0.01, 0.1, 0.09f,
+       0, 0.11, 0.1, 0.2, 0, 0.12f, 0.1, 0.21f, 0,  0.3, 1,   0.4});
   AddInputFromArray<float>(TensorShape({1, 6, 1}), 
        {.9f, .75f, .6f, .95f, .5f, .3f});
   AddInputFromArray<int>(TensorShape({}), {3});
@@ -968,7 +968,7 @@ TEST_F(NonMaxSuppressionLiteOpTest,
   
   //boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({1, 2, 4}));
-  test::FillValues<float>(&expected_boxes, {0, 10, 1, 11, 0, 0, 1, 1});
+  test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
   //scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({1, 2}));
@@ -993,8 +993,8 @@ TEST_F(NonMaxSuppressionLiteOpTest,
   MakeOp();
   AddInputFromArray<float>(
       TensorShape({1, 6, 1, 4}),
-      {0, 0,  1, 1,  0, 0.1f,  1, 1.1f,  0, -0.1f, 1, 0.9f,
-       0, 10, 1, 11, 0, 10.1f, 1, 11.1f, 0, 100,   1, 101});
+      {0, 0,    0.1, 0.1, 0, 0.01f, 0.1, 0.11f, 0, -0.01, 0.1, 0.09f,
+       0, 0.11, 0.1, 0.2, 0, 0.12f, 0.1, 0.21f, 0,  0.3, 1,   0.4});
   AddInputFromArray<float>(TensorShape({1, 6, 1}), 
        {.1f, 0, 0, .3f, .2f, -5.0f});
   // If we ask for more boxes than we actually expect to get back;
@@ -1007,7 +1007,7 @@ TEST_F(NonMaxSuppressionLiteOpTest,
   
   //boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({1, 2, 4}));
-  test::FillValues<float>(&expected_boxes, {0, 10, 1, 11, 0, 0, 1, 1});
+  test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
   //scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({1, 2}));
@@ -1064,10 +1064,10 @@ TEST_F(NonMaxSuppressionLiteOpTest,
   MakeOp();
   AddInputFromArray<float>(
       TensorShape({2, 6, 1, 4}),
-      {0, 0,  1, 1,  0, 0.1f,  1, 1.1f,  0, -0.1f, 1, 0.9f,
-       0, 10, 1, 11, 0, 10.1f, 1, 11.1f, 0, 100,   1, 101,
-       0, 0,  2, 2,  0, 0.2f,  2, 2.2f,  0, -0.2f, 2, 1.9f,
-       0, 20, 2, 22, 0, 20.2f, 2, 22.2f, 0, 200,   2, 202
+      {0, 0,    0.1, 0.1, 0, 0.01f, 0.1, 0.11f, 0, -0.01, 0.1, 0.09f,
+       0, 0.11, 0.1, 0.2, 0, 0.12f, 0.1, 0.21f, 0,  0.3,  1,   0.4,
+       0, 0,    0.2, 0.2, 0, 0.02f, 0.2, 0.22f, 0, -0.02, 0.2, 0.19f,
+       0, 0.21, 0.2, 0.3, 0, 0.22f, 0.2, 0.31f, 0,  0.4,  1,   0.5
        });
   AddInputFromArray<float>(TensorShape({2, 6, 1}), 
        {.9f, .75f, .6f, .95f, .5f, .3f, .9f, .75f, .6f, .95f, .5f, .3f});
@@ -1079,8 +1079,8 @@ TEST_F(NonMaxSuppressionLiteOpTest,
   
   //boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 2, 4}));
-  test::FillValues<float>(&expected_boxes, {0, 10, 1, 11 ,0, 0, 1, 1, 
-          0, 20, 2, 22 ,0, 0, 2, 2});
+  test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 
+          0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
   //scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 2}));
@@ -1105,10 +1105,10 @@ TEST_F(NonMaxSuppressionLiteOpTest,
   MakeOp();
   AddInputFromArray<float>(
       TensorShape({2, 6, 1, 4}),
-      {0, 0,  1, 1,  0, 0.1f,  1, 1.1f,  0, -0.1f, 1, 0.9f,
-       0, 10, 1, 11, 0, 10.1f, 1, 11.1f, 0, 100,   1, 101,
-       0, 0,  2, 2,  0, 0.2f,  2, 2.2f,  0, -0.2f, 2, 1.9f,
-       0, 20, 2, 22, 0, 20.2f, 2, 22.2f, 0, 200,   2, 202
+      {0, 0,    0.1, 0.1, 0, 0.01f, 0.1, 0.11f, 0, -0.01, 0.1, 0.09f,
+       0, 0.11, 0.1, 0.2, 0, 0.12f, 0.1, 0.21f, 0,  0.3,  1,   0.4,
+       0, 0,    0.2, 0.2, 0, 0.02f, 0.2, 0.22f, 0, -0.02, 0.2, 0.19f,
+       0, 0.21, 0.2, 0.3, 0, 0.22f, 0.2, 0.31f, 0,  0.4,  1,   0.5
        });
   AddInputFromArray<float>(TensorShape({2, 6, 2}), 
        {0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f, 
@@ -1121,8 +1121,9 @@ TEST_F(NonMaxSuppressionLiteOpTest,
   
   //boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 3, 4}));
-  test::FillValues<float>(&expected_boxes, {0, 10, 1, 11 ,0, 0, 1, 1, 
-          0, 0.1f, 1, 1.1f, 0, 20, 2, 22 ,0, 0, 2, 2, 0, 0.2f, 2, 2.2f});
+  test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 
+          0, 0.01f, 0.1, 0.11f, 0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2, 
+          0, 0.02f, 0.2, 0.22f});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
   //scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 3}));
@@ -1147,10 +1148,10 @@ TEST_F(NonMaxSuppressionLiteOpTest,
   MakeOp();
   AddInputFromArray<float>(
       TensorShape({2, 6, 1, 4}),
-      {0, 0,  1, 1,  0, 0.1f,  1, 1.1f,  0, -0.1f, 1, 0.9f,
-       0, 10, 1, 11, 0, 10.1f, 1, 11.1f, 0, 100,   1, 101,
-       0, 0,  2, 2,  0, 0.2f,  2, 2.2f,  0, -0.2f, 2, 1.9f,
-       0, 20, 2, 22, 0, 20.2f, 2, 22.2f, 0, 200,   2, 202
+      {0, 0,    0.1, 0.1, 0, 0.01f, 0.1, 0.11f, 0, -0.01, 0.1, 0.09f,
+       0, 0.11, 0.1, 0.2, 0, 0.12f, 0.1, 0.21f, 0,  0.3,  1,   0.4,
+       0, 0,    0.2, 0.2, 0, 0.02f, 0.2, 0.22f, 0, -0.02, 0.2, 0.19f,
+       0, 0.21, 0.2, 0.3, 0, 0.22f, 0.2, 0.31f, 0,  0.4,  1,   0.5
        });
   AddInputFromArray<float>(TensorShape({2, 6, 2}), 
       {0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f, 
@@ -1164,8 +1165,8 @@ TEST_F(NonMaxSuppressionLiteOpTest,
   
   //boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 2, 4}));
-  test::FillValues<float>(&expected_boxes, {0, 10, 1, 11 ,0, 0, 1, 1, 
-          0, 20, 2, 22, 0, 0, 2, 2});
+  test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 
+          0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
   //scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 2}));
@@ -1190,10 +1191,10 @@ TEST_F(NonMaxSuppressionLiteOpTest,
   MakeOp(true);
   AddInputFromArray<float>(
       TensorShape({2, 6, 1, 4}),
-      {0, 0,  1, 1,  0, 0.1f,  1, 1.1f,  0, -0.1f, 1, 0.9f,
-       0, 10, 1, 11, 0, 10.1f, 1, 11.1f, 0, 100,   1, 101,
-       0, 0,  2, 2,  0, 0.2f,  2, 2.2f,  0, -0.2f, 2, 1.9f,
-       0, 20, 2, 22, 0, 20.2f, 2, 22.2f, 0, 200,   2, 202
+      {0, 0,    0.1, 0.1, 0, 0.01f, 0.1, 0.11f, 0, -0.01, 0.1, 0.09f,
+       0, 0.11, 0.1, 0.2, 0, 0.12f, 0.1, 0.21f, 0,  0.3,  1,   0.4,
+       0, 0,    0.2, 0.2, 0, 0.02f, 0.2, 0.22f, 0, -0.02, 0.2, 0.19f,
+       0, 0.21, 0.2, 0.3, 0, 0.22f, 0.2, 0.31f, 0,  0.4,  1,   0.5
        });
   AddInputFromArray<float>(TensorShape({2, 6, 2}), 
       {0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f, 
@@ -1207,8 +1208,8 @@ TEST_F(NonMaxSuppressionLiteOpTest,
   
   //boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 3, 4}));
-  test::FillValues<float>(&expected_boxes, {0, 10, 1, 11 ,0, 0, 1, 1, 
-          0, 0, 0, 0, 0, 20, 2, 22, 0, 0, 2, 2, 0, 0, 0, 0});
+  test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 
+          0, 0, 0, 0, 0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2, 0, 0, 0, 0});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
   //scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 3}));
@@ -1233,10 +1234,10 @@ TEST_F(NonMaxSuppressionLiteOpTest,
   MakeOp();
   AddInputFromArray<float>(
       TensorShape({2, 6, 1, 4}),
-      {0, 0,  1, 1,  0, 0.1f,  1, 1.1f,  0, -0.1f, 1, 0.9f,
-       0, 10, 1, 11, 0, 10.1f, 1, 11.1f, 0, 100,   1, 101,
-       0, 0,  2, 2,  0, 0.2f,  2, 2.2f,  0, -0.2f, 2, 1.9f,
-       0, 20, 2, 22, 0, 20.2f, 2, 22.2f, 0, 200,   2, 202
+      {0, 0,    0.1, 0.1, 0, 0.01f, 0.1, 0.11f, 0, -0.01, 0.1, 0.09f,
+       0, 0.11, 0.1, 0.2, 0, 0.12f, 0.1, 0.21f, 0,  0.3,  1,   0.4,
+       0, 0,    0.2, 0.2, 0, 0.02f, 0.2, 0.22f, 0, -0.02, 0.2, 0.19f,
+       0, 0.21, 0.2, 0.3, 0, 0.22f, 0.2, 0.31f, 0,  0.4,  1,   0.5
        });
   AddInputFromArray<float>(TensorShape({2, 6, 2}), 
       {0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f, 
@@ -1251,10 +1252,10 @@ TEST_F(NonMaxSuppressionLiteOpTest,
   
   //boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 5, 4}));
-  test::FillValues<float>(&expected_boxes, {0, 10, 1, 11, 0, 0, 1, 1, 
-          0, 0.1f, 1, 1.1f, 0, 10.1f, 1, 11.1f, 0, 100, 1, 101,
-          0, 20, 2, 22, 0, 0, 2, 2, 0, 0.2f, 2, 2.2f, 
-          0, 20.2f, 2, 22.2f, 0, 200, 2, 202});
+  test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 
+          0, 0.01f, 0.1, 0.11f, 0, 0.12f, 0.1, 0.21f, 0,  0.3, 1, 0.4,
+          0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2, 0, 0.02f, 0.2, 0.22f,
+          0, 0.22f, 0.2, 0.31f, 0, 0.4, 1, 0.5});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
   //scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 5}));
@@ -1281,13 +1282,19 @@ TEST_F(NonMaxSuppressionLiteOpTest,
   AddInputFromArray<float>(
       TensorShape({2, 6, 2, 4}),
       //batch 0, box1 of class 1 should get selected
-      {0, 0, 1, 1, 0, 0,  1, 1, 0, 0.1f, 1, 1.1f, 0, 0.1f, 4, 4.1f, 
-      0, -0.1f, 1, 0.9f, 0, -0.1f, 1, 0.9f, 0, 10, 1, 11, 0, 10, 1, 11, 
-      0, 10.1f, 1, 11.1f, 0, 10.1f, 1, 11.1f, 0, 100, 1, 101, 0, 100, 1, 101, 
+      {0, 0,     0.1, 0.1,   0, 0,     0.1, 0.1, 
+       0, 0.01f, 0.1, 0.11f, 0, 0.6f,  0.1, 0.7f,
+       0, -0.01, 0.1, 0.09f, 0, -0.01, 0.1, 0.09f, 
+       0, 0.11,  0.1, 0.2,   0, 0.11,  0.1, 0.2, 
+       0, 0.12f, 0.1, 0.21f, 0, 0.12f, 0.1, 0.21f, 
+       0, 0.3,   1,   0.4,   0, 0.3,   1,   0.4, 
       //batch 1, box1 of class 0 should get selected
-      0, 0, 2, 2, 0, 0, 2, 2, 0, 0.2f, 2, 2.2f, 0, 0.2f, 2, 2.2f, 
-      0, -0.2f, 2, 1.9f, 0, -0.2f, 2, 1.9f, 0, 20, 2, 22, 0, 20, 2, 22, 
-      0, 20.2f, 2, 22.2f, 0, 20.2f, 2, 22.2f, 0, 200, 2, 202, 0, 200, 2, 202});
+       0, 0,     0.2, 0.2,   0, 0,     0.2, 0.2, 
+       0, 0.02f, 0.2, 0.22f, 0, 0.02f, 0.2, 0.22f,
+       0, -0.02, 0.2, 0.19f, 0, -0.02, 0.2, 0.19f,
+       0, 0.21,  0.2, 0.3,   0, 0.21,  0.2, 0.3, 
+       0, 0.22f, 0.2, 0.31f, 0, 0.22f, 0.2, 0.31f,
+       0,  0.4,  1,   0.5,   0, 0.4,   1,   0.5});
 
   AddInputFromArray<float>(TensorShape({2, 6, 2}), 
       {0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f, 
@@ -1301,8 +1308,8 @@ TEST_F(NonMaxSuppressionLiteOpTest,
   
   //boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 3, 4}));
-  test::FillValues<float>(&expected_boxes, {0, 10, 1, 11, 0, 0, 1, 1, 
-          0, 0.1f, 4, 4.1f, 0, 20, 2, 22 ,0, 0, 2, 2, 0, 0.2f, 2, 2.2f});
+  test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 
+           0, 0.6f, 0.1, 0.7f, 0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2, 0, 0.02f, 0.2, 0.22f});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
   //scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 3}));
@@ -1321,4 +1328,5 @@ TEST_F(NonMaxSuppressionLiteOpTest,
   test::FillValues<int>(&expected_indices, {3, 0, 1, 3, 0, 1});
   test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
+
 }  // namespace tensorflow
diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index 4d23509d8b..96ff6ef400 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -3270,7 +3270,6 @@ def non_max_suppression_lite(boxes,
     iou_threshold = ops.convert_to_tensor(iou_threshold, name='iou_threshold')
     score_threshold = ops.convert_to_tensor(
         score_threshold, name='score_threshold')
-    return gen_image_ops.non_max_suppression_lite(boxes, scores,
+    return gen_image_ops.non_max_suppression_lite(boxes, scores, 
             max_output_size_per_class, max_total_size, iou_threshold, 
-            score_threshold, pad_to_max_total_size);
-
+            score_threshold, pad_to_max_total_size)
-- 
GitLab


From ae64047b435aafd6c91f85ca7462ddb5b2a6c6f4 Mon Sep 17 00:00:00 2001
From: Supriya Rao <supriyar@nvidia.com>
Date: Fri, 2 Nov 2018 17:15:19 -0700
Subject: [PATCH 1495/2345] Add support for use_static_shapes. If false, the
 output is always padded to max_total_size If true, the output is padded to
 the minimum of max_total_size and max_size_per_class * num_classes Updated
 unit tests to take this into account

---
 .../core/kernels/non_max_suppression_op.cc    | 70 ++++++++-------
 .../kernels/non_max_suppression_op_test.cc    | 86 ++++++++++---------
 tensorflow/core/ops/image_ops.cc              |  8 +-
 tensorflow/python/ops/image_ops_impl.py       |  9 +-
 4 files changed, 90 insertions(+), 83 deletions(-)

diff --git a/tensorflow/core/kernels/non_max_suppression_op.cc b/tensorflow/core/kernels/non_max_suppression_op.cc
index 1e7fa863a3..a18f3ee785 100644
--- a/tensorflow/core/kernels/non_max_suppression_op.cc
+++ b/tensorflow/core/kernels/non_max_suppression_op.cc
@@ -225,11 +225,11 @@ void DoNonMaxSuppressionOp(
 void BatchedNonMaxSuppressionOp(OpKernelContext* context, 
                            const Tensor& inp_boxes, const Tensor& inp_scores, 
                            int num_boxes, 
-                           const Tensor& max_size_per_class, 
+                           const int max_size_per_class, 
                            const int total_size_per_batch, 
                            const float score_threshold, 
                            const float iou_threshold,
-                           bool pad_to_max_total_size = false) {
+                           bool use_static_shapes = false) {
 
   int q = inp_boxes.dim_size(2);
   int num_classes = inp_scores.dim_size(2);
@@ -239,18 +239,19 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
   // Default clip window of [0, 0, 1, 1] if none specified
   std::vector <float> clip_window{0, 0, 1, 1};
   
-  // [num_batches, max_detections, 4]
+  // [num_batches, per_batch_size, 4]
   std::vector <std::vector<float>> nmsed_boxes(num_batches);
-  // [num_batches, max_detections]
+  // [num_batches, per_batch_size]
   std::vector <std::vector<float>> nmsed_scores(num_batches); 
-  // [num_batches, max_detections]
+  // [num_batches, per_batch_size]
   std::vector <std::vector<float>> nmsed_classes(num_batches); 
   // [num_batches]
   std::vector <int> final_valid_detections;
-  // [num_batches, max_detections]
+  // [num_batches, per_batch_size]
   std::vector <std::vector<int>> selected_indices(num_batches);
 
-  int max_detections = 0;
+  int per_batch_size = total_size_per_batch;
+
   //perform non_max_suppression operation for each batch independently
   for (int batch = 0; batch < num_batches; ++batch) {
     // dims of per_batch_boxes [num_boxes, q, 4]
@@ -300,8 +301,7 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
       std::copy_n(boxes_data_vec.begin(), boxes_data_vec.size(), 
           boxes.unaligned_flat<float>().data());
 
-      const int output_size = std::min(max_size_per_class.scalar<int>()(), 
-                                   num_boxes);
+      const int output_size = std::min(max_size_per_class, num_boxes);
       // Do NMS, get the candidate indices of form vector<int>
       // Data structure for selection candidate in NMS.
       struct Candidate {
@@ -355,10 +355,19 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
       }
 
     }
-    if (total_size_per_batch > 0)
+    int max_detections = 0;
+    // If use_static_shapes is false, we always pad to max_total_size
+    if (!use_static_shapes) {
       max_detections = std::min((int) result_candidate_pq.size(), 
         total_size_per_batch);
-    else max_detections = (int) result_candidate_pq.size();
+      per_batch_size = total_size_per_batch;
+    }
+    else {
+      // max_detections will be <= max_size_per_class * num_classes
+      max_detections = (int) result_candidate_pq.size();
+      per_batch_size = std::min(total_size_per_batch, 
+                            max_size_per_class * num_classes);
+    }
 
     final_valid_detections.push_back(max_detections);
 
@@ -383,26 +392,20 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
       curr_total_size--;
     }
 
-    if(pad_to_max_total_size && max_detections < total_size_per_batch) {
-      nmsed_boxes[batch].resize(total_size_per_batch * 4, 0);
-      nmsed_scores[batch].resize(total_size_per_batch, 0);
-      nmsed_classes[batch].resize(total_size_per_batch, 0);
-      selected_indices[batch].resize(total_size_per_batch, 0);
-    }
-  }
-
-  if (pad_to_max_total_size) { 
-    max_detections = total_size_per_batch;
+    nmsed_boxes[batch].resize(per_batch_size * 4, 0);
+    nmsed_scores[batch].resize(per_batch_size, 0);
+    nmsed_classes[batch].resize(per_batch_size, 0);
+    selected_indices[batch].resize(per_batch_size, 0);
   }
 
   Tensor* nmsed_boxes_t = nullptr;
-  TensorShape boxes_shape({num_batches, max_detections, 4});
+  TensorShape boxes_shape({num_batches, per_batch_size, 4});
   OP_REQUIRES_OK(context,
                  context->allocate_output(0, boxes_shape, &nmsed_boxes_t));
   auto nmsed_boxes_flat = nmsed_boxes_t->template flat<float>();
 
   Tensor* nmsed_scores_t = nullptr;
-  TensorShape scores_shape({num_batches, max_detections});
+  TensorShape scores_shape({num_batches, per_batch_size});
   OP_REQUIRES_OK(context,
                  context->allocate_output(1, scores_shape, &nmsed_scores_t));
   auto nmsed_scores_flat = nmsed_scores_t->template flat<float>();
@@ -424,12 +427,12 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
   auto selected_indices_flat = selected_indices_t->template flat<int>();
   for (int i = 0; i < num_batches; ++i) {
     valid_detections_flat(i) = final_valid_detections[i];
-    for (int j = 0; j < max_detections; ++j) {
-      nmsed_scores_flat(i * max_detections + j) = nmsed_scores[i][j];
-      nmsed_classes_flat(i * max_detections + j) = nmsed_classes[i][j];
-      selected_indices_flat(i * max_detections + j) = selected_indices[i][j];
+    for (int j = 0; j < per_batch_size; ++j) {
+      nmsed_scores_flat(i * per_batch_size + j) = nmsed_scores[i][j];
+      nmsed_classes_flat(i * per_batch_size + j) = nmsed_classes[i][j];
+      selected_indices_flat(i * per_batch_size + j) = selected_indices[i][j];
       for (int k = 0; k < 4; ++k) {
-        nmsed_boxes_flat(i * max_detections * 4 + j * 4 + k) = 
+        nmsed_boxes_flat(i * per_batch_size * 4 + j * 4 + k) = 
           nmsed_boxes[i][j * 4 + k]; 
       }
     }
@@ -681,8 +684,8 @@ class NonMaxSuppressionLiteOp : public OpKernel {
  public:
   explicit NonMaxSuppressionLiteOp(OpKernelConstruction* context)
       : OpKernel(context) {
-    OP_REQUIRES_OK(context, context->GetAttr("pad_to_max_total_size",
-                                             &pad_to_max_total_size_));
+    OP_REQUIRES_OK(context, context->GetAttr("use_static_shapes",
+                                             &use_static_shapes_));
   }
 
   void Compute(OpKernelContext* context) override {
@@ -700,6 +703,7 @@ class NonMaxSuppressionLiteOp : public OpKernel {
         context, TensorShapeUtils::IsScalar(max_output_size.shape()),
         errors::InvalidArgument("max_size_per_class must be 0-D, got shape ",
                                 max_output_size.shape().DebugString()));
+    const int max_size_per_class = max_output_size.scalar<int>()();
     // max_total_size: scalar
     const Tensor& max_total_size = context->input(3);
     OP_REQUIRES(
@@ -735,12 +739,12 @@ class NonMaxSuppressionLiteOp : public OpKernel {
       return;
     }
     BatchedNonMaxSuppressionOp(context, boxes, scores, num_boxes, 
-                               max_output_size, max_total_size_per_batch, 
+                               max_size_per_class, max_total_size_per_batch, 
                                score_threshold_val, iou_threshold_val, 
-                               pad_to_max_total_size_);
+                               use_static_shapes_);
   }
  private:
-  bool pad_to_max_total_size_;
+  bool use_static_shapes_;
 };
 
 
diff --git a/tensorflow/core/kernels/non_max_suppression_op_test.cc b/tensorflow/core/kernels/non_max_suppression_op_test.cc
index d5662a15a0..b5d947db90 100644
--- a/tensorflow/core/kernels/non_max_suppression_op_test.cc
+++ b/tensorflow/core/kernels/non_max_suppression_op_test.cc
@@ -863,7 +863,7 @@ TEST_F(NonMaxSuppressionWithOverlapsOpTest, TestEmptyInput) {
 
 class NonMaxSuppressionLiteOpTest : public OpsTestBase {
  protected:
-  void MakeOp(bool pad_to_max_total_size = false) {
+  void MakeOp(bool use_static_shapes = false) {
     TF_EXPECT_OK(NodeDefBuilder("non_max_suppression_lite_op", 
                                 "NonMaxSuppressionLite")
                      .Input(FakeInput(DT_FLOAT))
@@ -872,7 +872,7 @@ class NonMaxSuppressionLiteOpTest : public OpsTestBase {
                      .Input(FakeInput(DT_INT32))
                      .Input(FakeInput(DT_FLOAT))
                      .Input(FakeInput(DT_FLOAT))
-                     .Attr("pad_to_max_total_size", pad_to_max_total_size)
+                     .Attr("use_static_shapes", use_static_shapes)
                      .Finalize(node_def()));
     TF_EXPECT_OK(InitOp());
   }
@@ -889,17 +889,17 @@ TEST_F(NonMaxSuppressionLiteOpTest, TestEmptyInput) {
   TF_ASSERT_OK(RunOpKernel());
 
   //boxes
-  Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({0, 0, 4}));
+  Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({0, 10, 4}));
   test::FillValues<float>(&expected_boxes, {});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
   
   //scores
-  Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({0, 0}));
+  Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({0, 10}));
   test::FillValues<float>(&expected_scores, {});
   test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
   
   //classes
-  Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({0, 0}));
+  Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({0, 10}));
   test::FillValues<float>(&expected_classes, {});
   test::ExpectTensorEqual<float>(expected_classes, *GetOutput(2));
   
@@ -909,7 +909,7 @@ TEST_F(NonMaxSuppressionLiteOpTest, TestEmptyInput) {
   test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
   
   // indices
-  Tensor expected_indices(allocator(), DT_INT32, TensorShape({0, 0}));
+  Tensor expected_indices(allocator(), DT_INT32, TensorShape({0, 10}));
   test::FillValues<int>(&expected_indices, {});
   test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
@@ -967,24 +967,25 @@ TEST_F(NonMaxSuppressionLiteOpTest,
   TF_ASSERT_OK(RunOpKernel());
   
   //boxes
-  Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({1, 2, 4}));
-  test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1});
+  Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({1, 3, 4}));
+  test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1,
+      0, 0, 0, 0});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
   //scores
-  Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({1, 2}));
-  test::FillValues<float>(&expected_scores, {0.95, 0.9});
+  Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({1, 3}));
+  test::FillValues<float>(&expected_scores, {0.95, 0.9, 0});
   test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
   //classes
-  Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({1, 2}));
-  test::FillValues<float>(&expected_classes, {0, 0});
+  Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({1, 3}));
+  test::FillValues<float>(&expected_classes, {0, 0, 0});
   test::ExpectTensorEqual<float>(expected_classes, *GetOutput(2));
   // valid
   Tensor expected_valid_d(allocator(), DT_INT32, TensorShape({1}));
   test::FillValues<int>(&expected_valid_d, {2});
   test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
   // indices
-  Tensor expected_indices(allocator(), DT_INT32, TensorShape({1, 2}));
-  test::FillValues<int>(&expected_indices, {3, 0});
+  Tensor expected_indices(allocator(), DT_INT32, TensorShape({1, 3}));
+  test::FillValues<int>(&expected_indices, {3, 0, 0});
   test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
 
@@ -999,31 +1000,32 @@ TEST_F(NonMaxSuppressionLiteOpTest,
        {.1f, 0, 0, .3f, .2f, -5.0f});
   // If we ask for more boxes than we actually expect to get back;
   // should still only get 2 boxes back.
-  AddInputFromArray<int>(TensorShape({}), {6});
-  AddInputFromArray<int>(TensorShape({}), {7});
+  AddInputFromArray<int>(TensorShape({}), {4});
+  AddInputFromArray<int>(TensorShape({}), {5});
   AddInputFromArray<float>(TensorShape({}), {.5f});
   AddInputFromArray<float>(TensorShape({}), {-3.0f});
   TF_ASSERT_OK(RunOpKernel());
   
   //boxes
-  Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({1, 2, 4}));
-  test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1});
+  Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({1, 5, 4}));
+  test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
   //scores
-  Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({1, 2}));
-  test::FillValues<float>(&expected_scores, {0.3, 0.1});
+  Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({1, 5}));
+  test::FillValues<float>(&expected_scores, {0.3, 0.1, 0, 0, 0});
   test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
   //classes
-  Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({1, 2}));
-  test::FillValues<float>(&expected_classes, {0, 0});
+  Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({1, 5}));
+  test::FillValues<float>(&expected_classes, {0, 0, 0, 0, 0});
   test::ExpectTensorEqual<float>(expected_classes, *GetOutput(2));
   // valid
   Tensor expected_valid_d(allocator(), DT_INT32, TensorShape({1}));
   test::FillValues<int>(&expected_valid_d, {2});
   test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
   // indices
-  Tensor expected_indices(allocator(), DT_INT32, TensorShape({1, 2}));
-  test::FillValues<int>(&expected_indices, {3, 0});
+  Tensor expected_indices(allocator(), DT_INT32, TensorShape({1, 5}));
+  test::FillValues<int>(&expected_indices, {3, 0, 0, 0, 0});
   test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
 
@@ -1032,7 +1034,7 @@ TEST_F(NonMaxSuppressionLiteOpTest, TestSelectSingleBox) {
   AddInputFromArray<float>(TensorShape({1, 1, 1, 4}), {0, 0, 1, 1});  
   AddInputFromArray<float>(TensorShape({1, 1, 1}), {.9f});
   AddInputFromArray<int>(TensorShape({}), {3});
-  AddInputFromArray<int>(TensorShape({}), {3});
+  AddInputFromArray<int>(TensorShape({}), {1});
   AddInputFromArray<float>(TensorShape({}), {.5f});
   AddInputFromArray<float>(TensorShape({}), {0.0f});
   TF_ASSERT_OK(RunOpKernel());
@@ -1072,31 +1074,31 @@ TEST_F(NonMaxSuppressionLiteOpTest,
   AddInputFromArray<float>(TensorShape({2, 6, 1}), 
        {.9f, .75f, .6f, .95f, .5f, .3f, .9f, .75f, .6f, .95f, .5f, .3f});
   AddInputFromArray<int>(TensorShape({}), {3});
-  AddInputFromArray<int>(TensorShape({}), {9});
+  AddInputFromArray<int>(TensorShape({}), {3});
   AddInputFromArray<float>(TensorShape({}), {.5f});
   AddInputFromArray<float>(TensorShape({}), {0.4f});
   TF_ASSERT_OK(RunOpKernel());
   
   //boxes
-  Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 2, 4}));
+  Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 3, 4}));
   test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 
-          0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2});
+          0, 0, 0, 0, 0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2, 0, 0, 0, 0});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
   //scores
-  Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 2}));
-  test::FillValues<float>(&expected_scores, {0.95, 0.9, 0.95, 0.9});
+  Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 3}));
+  test::FillValues<float>(&expected_scores, {0.95, 0.9, 0, 0.95, 0.9, 0});
   test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
   //classes
-  Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({2, 2}));
-  test::FillValues<float>(&expected_classes, {0, 0, 0, 0});
+  Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({2, 3}));
+  test::FillValues<float>(&expected_classes, {0, 0, 0, 0, 0, 0});
   test::ExpectTensorEqual<float>(expected_classes, *GetOutput(2));
   // valid
   Tensor expected_valid_d(allocator(), DT_INT32, TensorShape({2}));
   test::FillValues<int>(&expected_valid_d, {2, 2});
   test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
   // indices
-  Tensor expected_indices(allocator(), DT_INT32, TensorShape({2, 2}));
-  test::FillValues<int>(&expected_indices, {3, 0, 3, 0});
+  Tensor expected_indices(allocator(), DT_INT32, TensorShape({2, 3}));
+  test::FillValues<int>(&expected_indices, {3, 0, 0, 3, 0, 0});
   test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
 
@@ -1164,25 +1166,25 @@ TEST_F(NonMaxSuppressionLiteOpTest,
   TF_ASSERT_OK(RunOpKernel());
   
   //boxes
-  Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 2, 4}));
+  Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 3, 4}));
   test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 
-          0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2});
+          0, 0, 0, 0, 0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2, 0, 0, 0, 0});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
   //scores
-  Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 2}));
-  test::FillValues<float>(&expected_scores, {0.95, 0.9, 0.95, 0.9});
+  Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 3}));
+  test::FillValues<float>(&expected_scores, {0.95, 0.9, 0, 0.95, 0.9, 0});
   test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
   //classes
-  Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({2, 2}));
-  test::FillValues<float>(&expected_classes, {0, 1, 0, 1});
+  Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({2, 3}));
+  test::FillValues<float>(&expected_classes, {0, 1, 0, 0, 1, 0});
   test::ExpectTensorEqual<float>(expected_classes, *GetOutput(2));
   // valid
   Tensor expected_valid_d(allocator(), DT_INT32, TensorShape({2}));
   test::FillValues<int>(&expected_valid_d, {2, 2});
   test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
   // indices
-  Tensor expected_indices(allocator(), DT_INT32, TensorShape({2, 2}));
-  test::FillValues<int>(&expected_indices, {3, 0, 3, 0});
+  Tensor expected_indices(allocator(), DT_INT32, TensorShape({2, 3}));
+  test::FillValues<int>(&expected_indices, {3, 0, 0, 3, 0, 0});
   test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
 
diff --git a/tensorflow/core/ops/image_ops.cc b/tensorflow/core/ops/image_ops.cc
index 6e4d27f9eb..5d4a40ac24 100644
--- a/tensorflow/core/ops/image_ops.cc
+++ b/tensorflow/core/ops/image_ops.cc
@@ -154,9 +154,9 @@ Status NMSLiteShapeFn(InferenceContext* c) {
   // The boxes[3] is 4.
   TF_RETURN_IF_ERROR(c->WithValue(c->Dim(boxes, 3), 4, &unused));
 
-  bool pad_to_max;
-  TF_RETURN_IF_ERROR(c->GetAttr("pad_to_max_total_size", &pad_to_max));
-  if(pad_to_max) {
+  bool use_static_shapes;
+  TF_RETURN_IF_ERROR(c->GetAttr("use_static_shapes", &use_static_shapes));
+  if(!use_static_shapes) {
     DimensionHandle output_dim;
     DimensionHandle batch_dim = c->UnknownDim();
 
@@ -878,7 +878,7 @@ REGISTER_OP("NonMaxSuppressionLite")
     .Output("nmsed_classes: float")
     .Output("valid_detections: int32")
     .Output("selected_indices: int32")
-    .Attr("pad_to_max_total_size: bool = false")
+    .Attr("use_static_shapes: bool = false")
     .SetShapeFn(NMSLiteShapeFn);   
 
 }  // namespace tensorflow
diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index 96ff6ef400..dadee6dc2b 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -3218,7 +3218,7 @@ def non_max_suppression_lite(boxes,
                              max_total_size,
                              iou_threshold=0.5,
                              score_threshold=float('-inf'),
-                             pad_to_max_total_size=False,
+                             use_static_shapes=False,
                              name=None):
   """Greedily selects a subset of bounding boxes in descending order of score.
 
@@ -3248,8 +3248,9 @@ def non_max_suppression_lite(boxes,
       overlap too much with respect to IOU.
     score_threshold: A float representing the threshold for deciding when to
       remove boxes based on score.
-    pad_to_max_total_size: If true the size of outputs is padded to
-    max_total_size per batch.
+    use_static_shapes: If true, the output nmsed boxes, scores, classes and 
+      selected_indices are padded to be of length `max_size_per_class` and 
+      it doesn't clip boxes to max_total_size. Defaults to false.
     name: A name for the operation (optional).
 
   Returns:
@@ -3272,4 +3273,4 @@ def non_max_suppression_lite(boxes,
         score_threshold, name='score_threshold')
     return gen_image_ops.non_max_suppression_lite(boxes, scores, 
             max_output_size_per_class, max_total_size, iou_threshold, 
-            score_threshold, pad_to_max_total_size)
+            score_threshold, use_static_shapes)
-- 
GitLab


From 914780644b07721d75c478adcc46457a1e7064b4 Mon Sep 17 00:00:00 2001
From: Supriya Rao <supriyar@nvidia.com>
Date: Fri, 2 Nov 2018 18:07:00 -0700
Subject: [PATCH 1496/2345] Fix a bug in calculating max_detections

---
 tensorflow/core/kernels/non_max_suppression_op.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/kernels/non_max_suppression_op.cc b/tensorflow/core/kernels/non_max_suppression_op.cc
index a18f3ee785..da4c56f8ce 100644
--- a/tensorflow/core/kernels/non_max_suppression_op.cc
+++ b/tensorflow/core/kernels/non_max_suppression_op.cc
@@ -363,10 +363,10 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
       per_batch_size = total_size_per_batch;
     }
     else {
-      // max_detections will be <= max_size_per_class * num_classes
-      max_detections = (int) result_candidate_pq.size();
       per_batch_size = std::min(total_size_per_batch, 
                             max_size_per_class * num_classes);
+      max_detections = std::min(per_batch_size, 
+                            (int) result_candidate_pq.size());
     }
 
     final_valid_detections.push_back(max_detections);
-- 
GitLab


From 724b6bfc5fa5b57a32f82111f3b0c3544c535a23 Mon Sep 17 00:00:00 2001
From: Supriya Rao <supriyar@nvidia.com>
Date: Tue, 6 Nov 2018 09:07:11 -0800
Subject: [PATCH 1497/2345] Added argument description, removed trailing
 whitespaces

---
 tensorflow/python/ops/image_ops_impl.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index dadee6dc2b..a2e50ba37a 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -3250,7 +3250,8 @@ def non_max_suppression_lite(boxes,
       remove boxes based on score.
     use_static_shapes: If true, the output nmsed boxes, scores, classes and 
       selected_indices are padded to be of length `max_size_per_class` and 
-      it doesn't clip boxes to max_total_size. Defaults to false.
+      it doesn't clip boxes to max_total_size. If false, the outputs are
+      padded/clipped to max_total_size. Defaults to false.
     name: A name for the operation (optional).
 
   Returns:
@@ -3271,6 +3272,6 @@ def non_max_suppression_lite(boxes,
     iou_threshold = ops.convert_to_tensor(iou_threshold, name='iou_threshold')
     score_threshold = ops.convert_to_tensor(
         score_threshold, name='score_threshold')
-    return gen_image_ops.non_max_suppression_lite(boxes, scores, 
-            max_output_size_per_class, max_total_size, iou_threshold, 
+    return gen_image_ops.non_max_suppression_lite(boxes, scores,
+            max_output_size_per_class, max_total_size, iou_threshold,
             score_threshold, use_static_shapes)
-- 
GitLab


From 742c2bdca2e7d9a9b4a27a218d3ced17398677b1 Mon Sep 17 00:00:00 2001
From: Supriya Rao <supriyar@nvidia.com>
Date: Fri, 9 Nov 2018 13:48:22 -0800
Subject: [PATCH 1498/2345] Align function parameters, remove trailing
 whitespace

---
 .../core/kernels/non_max_suppression_op.cc    | 21 ++++++++++---------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/tensorflow/core/kernels/non_max_suppression_op.cc b/tensorflow/core/kernels/non_max_suppression_op.cc
index da4c56f8ce..950b24958c 100644
--- a/tensorflow/core/kernels/non_max_suppression_op.cc
+++ b/tensorflow/core/kernels/non_max_suppression_op.cc
@@ -222,14 +222,15 @@ void DoNonMaxSuppressionOp(
   std::copy_n(selected.begin(), selected.size(), output_indices_data.data());
 }
 
-void BatchedNonMaxSuppressionOp(OpKernelContext* context, 
-                           const Tensor& inp_boxes, const Tensor& inp_scores, 
-                           int num_boxes, 
-                           const int max_size_per_class, 
-                           const int total_size_per_batch, 
-                           const float score_threshold, 
-                           const float iou_threshold,
-                           bool use_static_shapes = false) {
+void BatchedNonMaxSuppressionOp(OpKernelContext* context,
+                                const Tensor& inp_boxes,
+                                const Tensor& inp_scores,
+                                int num_boxes,
+                                const int max_size_per_class,
+                                const int total_size_per_batch,
+                                const float score_threshold,
+                                const float iou_threshold,
+                                bool use_static_shapes = false) {
 
   int q = inp_boxes.dim_size(2);
   int num_classes = inp_scores.dim_size(2);
@@ -242,9 +243,9 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
   // [num_batches, per_batch_size, 4]
   std::vector <std::vector<float>> nmsed_boxes(num_batches);
   // [num_batches, per_batch_size]
-  std::vector <std::vector<float>> nmsed_scores(num_batches); 
+  std::vector <std::vector<float>> nmsed_scores(num_batches);
   // [num_batches, per_batch_size]
-  std::vector <std::vector<float>> nmsed_classes(num_batches); 
+  std::vector <std::vector<float>> nmsed_classes(num_batches);
   // [num_batches]
   std::vector <int> final_valid_detections;
   // [num_batches, per_batch_size]
-- 
GitLab


From 2ac1242ba9b4708c4be985de296892926f0d33e9 Mon Sep 17 00:00:00 2001
From: Supriya Rao <supriyar@nvidia.com>
Date: Mon, 12 Nov 2018 15:05:33 -0800
Subject: [PATCH 1499/2345] Addressed review comments. Made the following main
 changes 1. Added calculation of batch_dim in NMSLiteShapeFn 2. Added a check
 for max_total_size to be > 0 3. Updated clip calculation to account for all
 clip co-ordinates 4. Other code style related changes

---
 .../core/kernels/non_max_suppression_op.cc    | 33 +++++++++++--------
 tensorflow/core/ops/image_ops.cc              |  7 ++--
 tensorflow/python/ops/image_ops_impl.py       |  9 ++---
 3 files changed, 29 insertions(+), 20 deletions(-)

diff --git a/tensorflow/core/kernels/non_max_suppression_op.cc b/tensorflow/core/kernels/non_max_suppression_op.cc
index 950b24958c..5da83fbf5f 100644
--- a/tensorflow/core/kernels/non_max_suppression_op.cc
+++ b/tensorflow/core/kernels/non_max_suppression_op.cc
@@ -238,18 +238,18 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
   const int num_batches = inp_boxes.dim_size(0);
 
   // Default clip window of [0, 0, 1, 1] if none specified
-  std::vector <float> clip_window{0, 0, 1, 1};
+  std::vector<float> clip_window{0, 0, 1, 1};
   
   // [num_batches, per_batch_size, 4]
-  std::vector <std::vector<float>> nmsed_boxes(num_batches);
+  std::vector<std::vector<float>> nmsed_boxes(num_batches);
   // [num_batches, per_batch_size]
-  std::vector <std::vector<float>> nmsed_scores(num_batches);
+  std::vector<std::vector<float>> nmsed_scores(num_batches);
   // [num_batches, per_batch_size]
-  std::vector <std::vector<float>> nmsed_classes(num_batches);
+  std::vector<std::vector<float>> nmsed_classes(num_batches);
   // [num_batches]
-  std::vector <int> final_valid_detections;
+  std::vector<int> final_valid_detections;
   // [num_batches, per_batch_size]
-  std::vector <std::vector<int>> selected_indices(num_batches);
+  std::vector<std::vector<int>> selected_indices(num_batches);
 
   int per_batch_size = total_size_per_batch;
 
@@ -291,8 +291,9 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
             boxes_data_vec.push_back(boxesData[box * q * 4 + 
                   class_idx * 4 + cid]);
           }
-          else 
+          else {
               boxes_data_vec.push_back(boxesData[box * 4 + cid]);
+          }
         }
       }
         
@@ -302,7 +303,7 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
       std::copy_n(boxes_data_vec.begin(), boxes_data_vec.size(), 
           boxes.unaligned_flat<float>().data());
 
-      const int output_size = std::min(max_size_per_class, num_boxes);
+      const int size_per_class = std::min(max_size_per_class, num_boxes);
       // Do NMS, get the candidate indices of form vector<int>
       // Data structure for selection candidate in NMS.
       struct Candidate {
@@ -327,8 +328,8 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
       const Tensor const_boxes = boxes;
       typename TTypes<float, 2>::ConstTensor boxes_data = 
         const_boxes.tensor<float, 2>();
-      while (selected.size() < output_size && !candidate_priority_queue.empty())
-      {
+      while (selected.size() < size_per_class &&
+             !candidate_priority_queue.empty()) {
         next_candidate = candidate_priority_queue.top();
         candidate_priority_queue.pop();
 
@@ -380,13 +381,17 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
       result_candidate_pq.pop();
       // Add to final output vectors
       nmsed_boxes[batch].push_back(
-              std::max(next_candidate.box_coord[0], clip_window[0]));
+              std::max(std::min(next_candidate.box_coord[0], clip_window[2]),
+                       clip_window[0]));
       nmsed_boxes[batch].push_back(
-              std::max(next_candidate.box_coord[1], clip_window[1]));
+              std::max(std::min(next_candidate.box_coord[1], clip_window[3]),
+                       clip_window[1]));
       nmsed_boxes[batch].push_back(
-              std::min(next_candidate.box_coord[2], clip_window[2]));
+              std::max(std::min(next_candidate.box_coord[2], clip_window[2]),
+                       clip_window[0]));
       nmsed_boxes[batch].push_back(
-              std::min(next_candidate.box_coord[3], clip_window[3]));
+              std::max(std::min(next_candidate.box_coord[3], clip_window[3]),
+                       clip_window[1]));
       nmsed_scores[batch].push_back(next_candidate.score);
       nmsed_classes[batch].push_back(next_candidate.class_idx);
       selected_indices[batch].push_back(next_candidate.box_index);
diff --git a/tensorflow/core/ops/image_ops.cc b/tensorflow/core/ops/image_ops.cc
index 5d4a40ac24..43ab748bcc 100644
--- a/tensorflow/core/ops/image_ops.cc
+++ b/tensorflow/core/ops/image_ops.cc
@@ -158,9 +158,12 @@ Status NMSLiteShapeFn(InferenceContext* c) {
   TF_RETURN_IF_ERROR(c->GetAttr("use_static_shapes", &use_static_shapes));
   if(!use_static_shapes) {
     DimensionHandle output_dim;
-    DimensionHandle batch_dim = c->UnknownDim();
-
+    DimensionHandle batch_dim = c->Dim(boxes, 0);
+    
     TF_RETURN_IF_ERROR(c->MakeDimForScalarInput(3, &output_dim));
+    if(c->Value(output_dim) <= 0) {
+      return errors::InvalidArgument("max_total_size should be > 0 ");
+    }
     c->set_output(0, c->MakeShape({batch_dim, output_dim, 4}));
     c->set_output(1, c->MakeShape({batch_dim, output_dim}));
     c->set_output(2, c->MakeShape({batch_dim, output_dim}));
diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index a2e50ba37a..da850cee01 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -3248,10 +3248,11 @@ def non_max_suppression_lite(boxes,
       overlap too much with respect to IOU.
     score_threshold: A float representing the threshold for deciding when to
       remove boxes based on score.
-    use_static_shapes: If true, the output nmsed boxes, scores, classes and 
-      selected_indices are padded to be of length `max_size_per_class` and 
-      it doesn't clip boxes to max_total_size. If false, the outputs are
-      padded/clipped to max_total_size. Defaults to false.
+    use_static_shapes: If false, the output nmsed boxes, scores, classes and
+      selected_indices are padded/clipped to `max_total_size`. If true, the
+      output nmsed boxes, scores, classes and selected_indices are
+      padded/clipped to be of length `max_size_per_class` or `max_total_size`.
+      Defaults to false.
     name: A name for the operation (optional).
 
   Returns:
-- 
GitLab


From 5f321983717db87cbf6cb80985cd1c7b84a2049a Mon Sep 17 00:00:00 2001
From: Supriya Rao <supriyar@nvidia.com>
Date: Thu, 15 Nov 2018 13:37:32 -0800
Subject: [PATCH 1500/2345] Updated name of use_static_shapes to pad_per_class
 to better reflect the operation

---
 tensorflow/core/kernels/non_max_suppression_op.cc  | 14 +++++++-------
 .../core/kernels/non_max_suppression_op_test.cc    |  4 ++--
 tensorflow/core/ops/image_ops.cc                   | 10 +++++-----
 tensorflow/python/ops/image_ops_impl.py            | 10 +++++-----
 4 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/tensorflow/core/kernels/non_max_suppression_op.cc b/tensorflow/core/kernels/non_max_suppression_op.cc
index 5da83fbf5f..91f441ab8e 100644
--- a/tensorflow/core/kernels/non_max_suppression_op.cc
+++ b/tensorflow/core/kernels/non_max_suppression_op.cc
@@ -230,7 +230,7 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
                                 const int total_size_per_batch,
                                 const float score_threshold,
                                 const float iou_threshold,
-                                bool use_static_shapes = false) {
+                                bool pad_per_class = false) {
 
   int q = inp_boxes.dim_size(2);
   int num_classes = inp_scores.dim_size(2);
@@ -358,8 +358,8 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
 
     }
     int max_detections = 0;
-    // If use_static_shapes is false, we always pad to max_total_size
-    if (!use_static_shapes) {
+    // If pad_per_class is false, we always pad to max_total_size
+    if (!pad_per_class) {
       max_detections = std::min((int) result_candidate_pq.size(), 
         total_size_per_batch);
       per_batch_size = total_size_per_batch;
@@ -690,8 +690,8 @@ class NonMaxSuppressionLiteOp : public OpKernel {
  public:
   explicit NonMaxSuppressionLiteOp(OpKernelConstruction* context)
       : OpKernel(context) {
-    OP_REQUIRES_OK(context, context->GetAttr("use_static_shapes",
-                                             &use_static_shapes_));
+    OP_REQUIRES_OK(context, context->GetAttr("pad_per_class",
+                                             &pad_per_class_));
   }
 
   void Compute(OpKernelContext* context) override {
@@ -747,10 +747,10 @@ class NonMaxSuppressionLiteOp : public OpKernel {
     BatchedNonMaxSuppressionOp(context, boxes, scores, num_boxes, 
                                max_size_per_class, max_total_size_per_batch, 
                                score_threshold_val, iou_threshold_val, 
-                               use_static_shapes_);
+                               pad_per_class_);
   }
  private:
-  bool use_static_shapes_;
+  bool pad_per_class_;
 };
 
 
diff --git a/tensorflow/core/kernels/non_max_suppression_op_test.cc b/tensorflow/core/kernels/non_max_suppression_op_test.cc
index b5d947db90..5cc04dcc17 100644
--- a/tensorflow/core/kernels/non_max_suppression_op_test.cc
+++ b/tensorflow/core/kernels/non_max_suppression_op_test.cc
@@ -863,7 +863,7 @@ TEST_F(NonMaxSuppressionWithOverlapsOpTest, TestEmptyInput) {
 
 class NonMaxSuppressionLiteOpTest : public OpsTestBase {
  protected:
-  void MakeOp(bool use_static_shapes = false) {
+  void MakeOp(bool pad_per_class = false) {
     TF_EXPECT_OK(NodeDefBuilder("non_max_suppression_lite_op", 
                                 "NonMaxSuppressionLite")
                      .Input(FakeInput(DT_FLOAT))
@@ -872,7 +872,7 @@ class NonMaxSuppressionLiteOpTest : public OpsTestBase {
                      .Input(FakeInput(DT_INT32))
                      .Input(FakeInput(DT_FLOAT))
                      .Input(FakeInput(DT_FLOAT))
-                     .Attr("use_static_shapes", use_static_shapes)
+                     .Attr("pad_per_class", pad_per_class)
                      .Finalize(node_def()));
     TF_EXPECT_OK(InitOp());
   }
diff --git a/tensorflow/core/ops/image_ops.cc b/tensorflow/core/ops/image_ops.cc
index 43ab748bcc..c2869aec20 100644
--- a/tensorflow/core/ops/image_ops.cc
+++ b/tensorflow/core/ops/image_ops.cc
@@ -154,14 +154,14 @@ Status NMSLiteShapeFn(InferenceContext* c) {
   // The boxes[3] is 4.
   TF_RETURN_IF_ERROR(c->WithValue(c->Dim(boxes, 3), 4, &unused));
 
-  bool use_static_shapes;
-  TF_RETURN_IF_ERROR(c->GetAttr("use_static_shapes", &use_static_shapes));
-  if(!use_static_shapes) {
+  bool pad_per_class;
+  TF_RETURN_IF_ERROR(c->GetAttr("pad_per_class", &pad_per_class));
+  if(!pad_per_class) {
     DimensionHandle output_dim;
     DimensionHandle batch_dim = c->Dim(boxes, 0);
     
     TF_RETURN_IF_ERROR(c->MakeDimForScalarInput(3, &output_dim));
-    if(c->Value(output_dim) <= 0) {
+    if(c->ValueKnown(output_dim) && c->Value(output_dim) <= 0) {
       return errors::InvalidArgument("max_total_size should be > 0 ");
     }
     c->set_output(0, c->MakeShape({batch_dim, output_dim, 4}));
@@ -881,7 +881,7 @@ REGISTER_OP("NonMaxSuppressionLite")
     .Output("nmsed_classes: float")
     .Output("valid_detections: int32")
     .Output("selected_indices: int32")
-    .Attr("use_static_shapes: bool = false")
+    .Attr("pad_per_class: bool = false")
     .SetShapeFn(NMSLiteShapeFn);   
 
 }  // namespace tensorflow
diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index da850cee01..8f3150d109 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -3218,7 +3218,7 @@ def non_max_suppression_lite(boxes,
                              max_total_size,
                              iou_threshold=0.5,
                              score_threshold=float('-inf'),
-                             use_static_shapes=False,
+                             pad_per_class=False,
                              name=None):
   """Greedily selects a subset of bounding boxes in descending order of score.
 
@@ -3248,11 +3248,11 @@ def non_max_suppression_lite(boxes,
       overlap too much with respect to IOU.
     score_threshold: A float representing the threshold for deciding when to
       remove boxes based on score.
-    use_static_shapes: If false, the output nmsed boxes, scores, classes and
+    pad_per_class: If false, the output nmsed boxes, scores, classes and
       selected_indices are padded/clipped to `max_total_size`. If true, the
       output nmsed boxes, scores, classes and selected_indices are
-      padded/clipped to be of length `max_size_per_class` or `max_total_size`.
-      Defaults to false.
+      padded/clipped to be of length `max_size_per_class` or `max_total_size`
+      (smaller of the two). Defaults to false.
     name: A name for the operation (optional).
 
   Returns:
@@ -3275,4 +3275,4 @@ def non_max_suppression_lite(boxes,
         score_threshold, name='score_threshold')
     return gen_image_ops.non_max_suppression_lite(boxes, scores,
             max_output_size_per_class, max_total_size, iou_threshold,
-            score_threshold, use_static_shapes)
+            score_threshold, pad_per_class)
-- 
GitLab


From 4f5aa1fee071eb95cf5b6e31465228b20a7274d1 Mon Sep 17 00:00:00 2001
From: Supriya Rao <supriyar@nvidia.com>
Date: Mon, 26 Nov 2018 13:58:24 -0800
Subject: [PATCH 1501/2345] Added more checks for NMS function. Updated name of
 the new NMS op to CombinedNonMaxSuppression

---
 .../core/kernels/non_max_suppression_op.cc    | 16 ++---
 .../kernels/non_max_suppression_op_test.cc    | 28 ++++----
 tensorflow/core/ops/image_ops.cc              | 65 ++++++++++++-------
 tensorflow/python/ops/image_ops_impl.py       |  8 +--
 4 files changed, 66 insertions(+), 51 deletions(-)

diff --git a/tensorflow/core/kernels/non_max_suppression_op.cc b/tensorflow/core/kernels/non_max_suppression_op.cc
index 91f441ab8e..7971f3d932 100644
--- a/tensorflow/core/kernels/non_max_suppression_op.cc
+++ b/tensorflow/core/kernels/non_max_suppression_op.cc
@@ -74,7 +74,7 @@ static inline void ParseAndCheckBoxSizes(OpKernelContext* context,
               errors::InvalidArgument("boxes must have 4 columns"));
 }
 
-static inline void CheckNMSLiteScoreSizes(OpKernelContext* context, 
+static inline void CheckCombinedNMSScoreSizes(OpKernelContext* context, 
                                           int num_boxes,
                                           const Tensor& scores) {
   // The shape of 'scores' is [batch_size, num_boxes, num_classes]
@@ -85,7 +85,7 @@ static inline void CheckNMSLiteScoreSizes(OpKernelContext* context,
               errors::InvalidArgument("scores has incompatible shape"));
 }
 
-static inline void ParseAndCheckNMSLiteBoxSizes(OpKernelContext* context,
+static inline void ParseAndCheckCombinedNMSBoxSizes(OpKernelContext* context,
                                          const Tensor& boxes, int* num_boxes, 
                                          const int num_classes) {
   // The shape of 'boxes' is [batch_size, num_boxes, q, 4]
@@ -686,9 +686,9 @@ class NonMaxSuppressionWithOverlapsOp : public OpKernel {
 };
 
 template <typename Device>
-class NonMaxSuppressionLiteOp : public OpKernel {
+class CombinedNonMaxSuppressionOp : public OpKernel {
  public:
-  explicit NonMaxSuppressionLiteOp(OpKernelConstruction* context)
+  explicit CombinedNonMaxSuppressionOp(OpKernelConstruction* context)
       : OpKernel(context) {
     OP_REQUIRES_OK(context, context->GetAttr("pad_per_class",
                                              &pad_per_class_));
@@ -738,8 +738,8 @@ class NonMaxSuppressionLiteOp : public OpKernel {
                 errors::InvalidArgument("iou_threshold must be in [0, 1]"));
     int num_boxes = 0;
     const int num_classes = scores.dim_size(2);
-    ParseAndCheckNMSLiteBoxSizes(context, boxes, &num_boxes, num_classes);
-    CheckNMSLiteScoreSizes(context, num_boxes, scores);
+    ParseAndCheckCombinedNMSBoxSizes(context, boxes, &num_boxes, num_classes);
+    CheckCombinedNMSScoreSizes(context, num_boxes, scores);
 
     if (!context->status().ok()) {
       return;
@@ -785,7 +785,7 @@ REGISTER_KERNEL_BUILDER(
     Name("NonMaxSuppressionWithOverlaps").Device(DEVICE_CPU),
     NonMaxSuppressionWithOverlapsOp<CPUDevice>);
 
-REGISTER_KERNEL_BUILDER(Name("NonMaxSuppressionLite").Device(DEVICE_CPU),
-        NonMaxSuppressionLiteOp<CPUDevice>);
+REGISTER_KERNEL_BUILDER(Name("CombinedNonMaxSuppression").Device(DEVICE_CPU),
+        CombinedNonMaxSuppressionOp<CPUDevice>);
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/non_max_suppression_op_test.cc b/tensorflow/core/kernels/non_max_suppression_op_test.cc
index 5cc04dcc17..afb96a9c82 100644
--- a/tensorflow/core/kernels/non_max_suppression_op_test.cc
+++ b/tensorflow/core/kernels/non_max_suppression_op_test.cc
@@ -861,11 +861,11 @@ TEST_F(NonMaxSuppressionWithOverlapsOpTest, TestEmptyInput) {
   test::ExpectTensorEqual<int>(expected, *GetOutput(0));
 }
 
-class NonMaxSuppressionLiteOpTest : public OpsTestBase {
+class CombinedNonMaxSuppressionOpTest : public OpsTestBase {
  protected:
   void MakeOp(bool pad_per_class = false) {
-    TF_EXPECT_OK(NodeDefBuilder("non_max_suppression_lite_op", 
-                                "NonMaxSuppressionLite")
+    TF_EXPECT_OK(NodeDefBuilder("combined_non_max_suppression_op",
+                                "CombinedNonMaxSuppression")
                      .Input(FakeInput(DT_FLOAT))
                      .Input(FakeInput(DT_FLOAT))
                      .Input(FakeInput(DT_INT32))
@@ -878,7 +878,7 @@ class NonMaxSuppressionLiteOpTest : public OpsTestBase {
   }
 };
 
-TEST_F(NonMaxSuppressionLiteOpTest, TestEmptyInput) {
+TEST_F(CombinedNonMaxSuppressionOpTest, TestEmptyInput) {
   MakeOp();
   AddInputFromArray<float>(TensorShape({0, 0, 0, 4}), {});
   AddInputFromArray<float>(TensorShape({0, 0, 0}), {});
@@ -914,7 +914,7 @@ TEST_F(NonMaxSuppressionLiteOpTest, TestEmptyInput) {
   test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
 
-TEST_F(NonMaxSuppressionLiteOpTest, TestSelectFromThreeClusters) {
+TEST_F(CombinedNonMaxSuppressionOpTest, TestSelectFromThreeClusters) {
   MakeOp();
   AddInputFromArray<float>(
       TensorShape({1, 6, 1, 4}),
@@ -951,7 +951,7 @@ TEST_F(NonMaxSuppressionLiteOpTest, TestSelectFromThreeClusters) {
   test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
 
-TEST_F(NonMaxSuppressionLiteOpTest, 
+TEST_F(CombinedNonMaxSuppressionOpTest,
         TestSelectFromThreeClustersWithScoreThreshold) {
   MakeOp();
   AddInputFromArray<float>(
@@ -989,7 +989,7 @@ TEST_F(NonMaxSuppressionLiteOpTest,
   test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
 
-TEST_F(NonMaxSuppressionLiteOpTest, 
+TEST_F(CombinedNonMaxSuppressionOpTest,
         TestSelectFromThreeClustersWithScoreThresholdZeroScores) {
   MakeOp();
   AddInputFromArray<float>(
@@ -1029,7 +1029,7 @@ TEST_F(NonMaxSuppressionLiteOpTest,
   test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
 
-TEST_F(NonMaxSuppressionLiteOpTest, TestSelectSingleBox) {
+TEST_F(CombinedNonMaxSuppressionOpTest, TestSelectSingleBox) {
   MakeOp();
   AddInputFromArray<float>(TensorShape({1, 1, 1, 4}), {0, 0, 1, 1});  
   AddInputFromArray<float>(TensorShape({1, 1, 1}), {.9f});
@@ -1061,7 +1061,7 @@ TEST_F(NonMaxSuppressionLiteOpTest, TestSelectSingleBox) {
   test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
 
-TEST_F(NonMaxSuppressionLiteOpTest, 
+TEST_F(CombinedNonMaxSuppressionOpTest,
         TestSelectFromTwoBatchesWithScoreThreshold) {
   MakeOp();
   AddInputFromArray<float>(
@@ -1102,7 +1102,7 @@ TEST_F(NonMaxSuppressionLiteOpTest,
   test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
 
-TEST_F(NonMaxSuppressionLiteOpTest, 
+TEST_F(CombinedNonMaxSuppressionOpTest,
         TestSelectFromTwoBatchesTwoClasses) {
   MakeOp();
   AddInputFromArray<float>(
@@ -1145,7 +1145,7 @@ TEST_F(NonMaxSuppressionLiteOpTest,
   test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
 
-TEST_F(NonMaxSuppressionLiteOpTest, 
+TEST_F(CombinedNonMaxSuppressionOpTest,
         TestSelectFromTwoBatchesTwoClassesWithScoreThreshold) {
   MakeOp();
   AddInputFromArray<float>(
@@ -1188,7 +1188,7 @@ TEST_F(NonMaxSuppressionLiteOpTest,
   test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
 
-TEST_F(NonMaxSuppressionLiteOpTest, 
+TEST_F(CombinedNonMaxSuppressionOpTest,
         TestSelectFromTwoBatchesTwoClassesWithScoreThresholdPadded) {
   MakeOp(true);
   AddInputFromArray<float>(
@@ -1231,7 +1231,7 @@ TEST_F(NonMaxSuppressionLiteOpTest,
   test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
 
-TEST_F(NonMaxSuppressionLiteOpTest, 
+TEST_F(CombinedNonMaxSuppressionOpTest,
         TestSelectFromTwoBatchesTwoClassesTotalSize) {
   MakeOp();
   AddInputFromArray<float>(
@@ -1278,7 +1278,7 @@ TEST_F(NonMaxSuppressionLiteOpTest,
   test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
 
-TEST_F(NonMaxSuppressionLiteOpTest, 
+TEST_F(CombinedNonMaxSuppressionOpTest,
         TestSelectFromTwoBatchesTwoClassesForBoxesAndScores) {
   MakeOp();
   AddInputFromArray<float>(
diff --git a/tensorflow/core/ops/image_ops.cc b/tensorflow/core/ops/image_ops.cc
index c2869aec20..87405d2611 100644
--- a/tensorflow/core/ops/image_ops.cc
+++ b/tensorflow/core/ops/image_ops.cc
@@ -131,7 +131,7 @@ Status NMSShapeFn(InferenceContext* c) {
   return Status::OK();
 }
 
-Status NMSLiteShapeFn(InferenceContext* c) {
+Status CombinedNMSShapeFn(InferenceContext* c) {
   //Get inputs and validate ranks
   ShapeHandle boxes;
   //boxes is a tensor of Dimensions [batch_size, num_anchors, q, 4]
@@ -143,40 +143,55 @@ Status NMSLiteShapeFn(InferenceContext* c) {
   TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &max_output_size_per_class));
   ShapeHandle max_total_size;
   TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 0, &max_total_size));
-  ShapeHandle iou_threshold;
-  TF_RETURN_IF_ERROR(c->WithRank(c->input(4), 0, &iou_threshold));
-  ShapeHandle score_threshold;
-  TF_RETURN_IF_ERROR(c->WithRank(c->input(5), 0, &score_threshold));
+  ShapeHandle unused_shape;
+  TF_RETURN_IF_ERROR(c->WithRank(c->input(4), 0, &unused_shape));
+  TF_RETURN_IF_ERROR(c->WithRank(c->input(5), 0, &unused_shape));
 
   DimensionHandle unused;
+  //boxes[0] and scores[0] are both batch_size
+  TF_RETURN_IF_ERROR(c->Merge(c->Dim(boxes, 0), c->Dim(scores, 0), &unused));
   //boxes[1] and scores[1] are both num_anchors
   TF_RETURN_IF_ERROR(c->Merge(c->Dim(boxes, 1), c->Dim(scores, 1), &unused));
   // The boxes[3] is 4.
   TF_RETURN_IF_ERROR(c->WithValue(c->Dim(boxes, 3), 4, &unused));
 
+  DimensionHandle d = c->Dim(boxes, 2);
+  auto num_classes = c->Value(c->Dim(scores, 2));
+  if (c->ValueKnown(d)) {
+    if (c->Value(d) != 1 && c->Value(d) != num_classes) {
+       return errors::InvalidArgument("third dimension of boxes must be either "
+           "1 or equal to the third dimension of scores");
+    }
+  }
+  DimensionHandle output_dim;
+  DimensionHandle batch_dim = c->Dim(boxes, 0);
+
+  TF_RETURN_IF_ERROR(c->MakeDimForScalarInput(3, &output_dim));
+  if (c->ValueKnown(output_dim) && c->Value(output_dim) <= 0) {
+    return errors::InvalidArgument("max_total_size should be > 0 ");
+  }
+  DimensionHandle size_per_class;
+  TF_RETURN_IF_ERROR(c->MakeDimForScalarInput(2, &size_per_class));
+
+  int64 output_size;
   bool pad_per_class;
   TF_RETURN_IF_ERROR(c->GetAttr("pad_per_class", &pad_per_class));
-  if(!pad_per_class) {
-    DimensionHandle output_dim;
-    DimensionHandle batch_dim = c->Dim(boxes, 0);
-    
-    TF_RETURN_IF_ERROR(c->MakeDimForScalarInput(3, &output_dim));
-    if(c->ValueKnown(output_dim) && c->Value(output_dim) <= 0) {
-      return errors::InvalidArgument("max_total_size should be > 0 ");
-    }
-    c->set_output(0, c->MakeShape({batch_dim, output_dim, 4}));
-    c->set_output(1, c->MakeShape({batch_dim, output_dim}));
-    c->set_output(2, c->MakeShape({batch_dim, output_dim}));
-    c->set_output(4, c->MakeShape({batch_dim, output_dim}));
+  if (!pad_per_class) {
+    output_size =  c->Value(output_dim);
   }
   else {
-    //TODO SR Can all the outputs be unknowndim?
-    c->set_output(0, c->Vector(c->UnknownDim()));
-    c->set_output(1, c->Vector(c->UnknownDim()));
-    c->set_output(2, c->Vector(c->UnknownDim()));
-    c->set_output(3, c->Vector(c->UnknownDim()));
-    c->set_output(4, c->Vector(c->UnknownDim()));
+    if (c->ValueKnown(size_per_class) && c->Value(size_per_class) <= 0) {
+      return errors::InvalidArgument("max_output_size_per_class must be > 0 "
+             "if pad_per_class is set to true ");
+    }
+    output_size = std::min(c->Value(output_dim),
+        c->Value(size_per_class) * num_classes);
   }
+  c->set_output(0, c->MakeShape({batch_dim, output_size, 4}));
+  c->set_output(1, c->MakeShape({batch_dim, output_size}));
+  c->set_output(2, c->MakeShape({batch_dim, output_size}));
+  c->set_output(3, c->Vector(batch_dim));
+  c->set_output(4, c->MakeShape({batch_dim, output_size}));
   return Status::OK();
 }
 
@@ -869,7 +884,7 @@ REGISTER_OP("NonMaxSuppressionWithOverlaps")
       return Status::OK();
     });
 
-REGISTER_OP("NonMaxSuppressionLite")
+REGISTER_OP("CombinedNonMaxSuppression")
     .Input("boxes: float")
     .Input("scores: float")
     .Input("max_output_size_per_class: int32")
@@ -882,6 +897,6 @@ REGISTER_OP("NonMaxSuppressionLite")
     .Output("valid_detections: int32")
     .Output("selected_indices: int32")
     .Attr("pad_per_class: bool = false")
-    .SetShapeFn(NMSLiteShapeFn);   
+    .SetShapeFn(CombinedNMSShapeFn);
 
 }  // namespace tensorflow
diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index 8f3150d109..ef65789e22 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -3211,8 +3211,8 @@ def extract_glimpse_v2(
       uniform_noise=False,
       name=name)
 
-@tf_export('image.non_max_suppression_lite')
-def non_max_suppression_lite(boxes,
+@tf_export('image.combined_non_max_suppression')
+def combined_non_max_suppression(boxes,
                              scores,
                              max_output_size_per_class,
                              max_total_size,
@@ -3269,10 +3269,10 @@ def non_max_suppression_lite(boxes,
     'selected_indices':  A [batch_size, max_detections] int32 tensor
       containing the indices of the selected boxes
   """
-  with ops.name_scope(name, 'non_max_suppression_lite'):
+  with ops.name_scope(name, 'combined_non_max_suppression'):
     iou_threshold = ops.convert_to_tensor(iou_threshold, name='iou_threshold')
     score_threshold = ops.convert_to_tensor(
         score_threshold, name='score_threshold')
-    return gen_image_ops.non_max_suppression_lite(boxes, scores,
+    return gen_image_ops.combined_non_max_suppression(boxes, scores,
             max_output_size_per_class, max_total_size, iou_threshold,
             score_threshold, pad_per_class)
-- 
GitLab


From ab8402702110fa7ba1d12104102a7b23967d7f32 Mon Sep 17 00:00:00 2001
From: Supriya Rao <supriyar@nvidia.com>
Date: Wed, 5 Dec 2018 17:26:08 -0800
Subject: [PATCH 1502/2345] Addressed review comments from PR

---
 .../core/kernels/non_max_suppression_op.cc    | 51 +++++++++----------
 tensorflow/core/ops/image_ops.cc              |  8 +--
 tensorflow/python/ops/image_ops_impl.py       | 18 +++----
 3 files changed, 38 insertions(+), 39 deletions(-)

diff --git a/tensorflow/core/kernels/non_max_suppression_op.cc b/tensorflow/core/kernels/non_max_suppression_op.cc
index 7971f3d932..821135d2f6 100644
--- a/tensorflow/core/kernels/non_max_suppression_op.cc
+++ b/tensorflow/core/kernels/non_max_suppression_op.cc
@@ -234,13 +234,12 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
 
   int q = inp_boxes.dim_size(2);
   int num_classes = inp_scores.dim_size(2);
-  //unpack along batch dimension
   const int num_batches = inp_boxes.dim_size(0);
 
   // Default clip window of [0, 0, 1, 1] if none specified
   std::vector<float> clip_window{0, 0, 1, 1};
   
-  // [num_batches, per_batch_size, 4]
+  // [num_batches, per_batch_size * 4]
   std::vector<std::vector<float>> nmsed_boxes(num_batches);
   // [num_batches, per_batch_size]
   std::vector<std::vector<float>> nmsed_scores(num_batches);
@@ -253,12 +252,12 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
 
   int per_batch_size = total_size_per_batch;
 
-  //perform non_max_suppression operation for each batch independently
+  // perform non_max_suppression operation for each batch independently
   for (int batch = 0; batch < num_batches; ++batch) {
     // dims of per_batch_boxes [num_boxes, q, 4]
-    Tensor per_batch_boxes = inp_boxes.Slice(batch, batch+1);
+    Tensor per_batch_boxes = inp_boxes.Slice(batch, batch + 1);
     // dims of per_batch_scores [num_boxes, num_classes]
-    Tensor per_batch_scores = inp_scores.Slice(batch, batch+1);
+    Tensor per_batch_scores = inp_scores.Slice(batch, batch + 1);
 
     struct ResultCandidate {
       int box_index;
@@ -273,34 +272,34 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
     std::priority_queue<ResultCandidate, std::vector<ResultCandidate>, 
       decltype(rc_cmp)> result_candidate_pq(rc_cmp);
 
-    float * scoresData = per_batch_scores.unaligned_flat<float>().data();
-    float * boxesData = per_batch_boxes.unaligned_flat<float>().data();
+    float* scores_data = per_batch_scores.unaligned_flat<float>().data();
+    float* boxes_data = per_batch_boxes.unaligned_flat<float>().data();
 
-    //Iterate through all classes
+    // Iterate through all classes
     for (int class_idx = 0; class_idx < num_classes; ++class_idx) {
-      std::vector<float> scores_data;
-      std::vector<float> boxes_data_vec;
+      std::vector<float> class_scores_data;
+      std::vector<float> class_boxes_data;
 
       for (int box = 0; box < num_boxes; ++box) {
-        //Get the scores per class 
-        //scores_data dim is [num_boxes].
-        scores_data.push_back(scoresData[box * num_classes + class_idx]);
-        for(int cid = 0; cid < 4; ++cid){
-          if(q > 1){
-            //Get the boxes per class. boxes_data_vec dims is [num_boxes, 4]
-            boxes_data_vec.push_back(boxesData[box * q * 4 + 
-                  class_idx * 4 + cid]);
+        // Get the scores per class
+        // class_scores_data dim is [num_boxes].
+        class_scores_data.push_back(scores_data[box * num_classes + class_idx]);
+        for (int cid = 0; cid < 4; ++cid){
+          if (q > 1){
+            // Get the boxes per class. class_boxes_data dims is [num_boxes, 4]
+            class_boxes_data.push_back(boxes_data[(box * q + class_idx) * 4
+                                                  + cid]);
           }
           else {
-              boxes_data_vec.push_back(boxesData[box * 4 + cid]);
+              class_boxes_data.push_back(boxes_data[box * 4 + cid]);
           }
         }
       }
         
-      //Copy boxes_data_vec to a tensor
+      // Copy class_boxes_data to a tensor
       TensorShape boxesShape({num_boxes, 4});
       Tensor boxes(per_batch_boxes.dtype(), boxesShape);
-      std::copy_n(boxes_data_vec.begin(), boxes_data_vec.size(), 
+      std::copy_n(class_boxes_data.begin(), class_boxes_data.size(),
           boxes.unaligned_flat<float>().data());
 
       const int size_per_class = std::min(max_size_per_class, num_boxes);
@@ -315,9 +314,9 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
       };
       std::priority_queue<Candidate, std::deque<Candidate>, decltype(cmp)>
         candidate_priority_queue(cmp);
-      for (int i = 0; i < scores_data.size(); ++i) {
-        if (scores_data[i] > score_threshold) {
-          candidate_priority_queue.emplace(Candidate({i, scores_data[i]}));
+      for (int i = 0; i < class_scores_data.size(); ++i) {
+        if (class_scores_data[i] > score_threshold) {
+          candidate_priority_queue.emplace(Candidate({i, class_scores_data[i]}));
         }
       }
 
@@ -347,7 +346,7 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
 
         if (should_select) {
           selected.push_back(next_candidate.box_index);
-          //Add the selected box to the result candidate. Sorted by score
+          // Add the selected box to the result candidate. Sorted by score
           int id = next_candidate.box_index;
           ResultCandidate rc = {next_candidate.box_index, next_candidate.score,
             class_idx, {boxes_data(id, 0), boxes_data(id, 1), boxes_data(id, 2),
@@ -375,7 +374,7 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
 
     int curr_total_size = max_detections;
     // Pick the top max_detections values 
-    while(curr_total_size > 0 && !result_candidate_pq.empty())
+    while (curr_total_size > 0 && !result_candidate_pq.empty())
     {
       ResultCandidate next_candidate = result_candidate_pq.top();
       result_candidate_pq.pop();
diff --git a/tensorflow/core/ops/image_ops.cc b/tensorflow/core/ops/image_ops.cc
index 87405d2611..39263de6ba 100644
--- a/tensorflow/core/ops/image_ops.cc
+++ b/tensorflow/core/ops/image_ops.cc
@@ -156,9 +156,9 @@ Status CombinedNMSShapeFn(InferenceContext* c) {
   TF_RETURN_IF_ERROR(c->WithValue(c->Dim(boxes, 3), 4, &unused));
 
   DimensionHandle d = c->Dim(boxes, 2);
-  auto num_classes = c->Value(c->Dim(scores, 2));
-  if (c->ValueKnown(d)) {
-    if (c->Value(d) != 1 && c->Value(d) != num_classes) {
+  DimensionHandle class_dim = c->Dim(scores, 2);
+  if (c->ValueKnown(d) && c->ValueKnown(class_dim)) {
+    if (c->Value(d) != 1 && c->Value(d) != c->Value(class_dim)) {
        return errors::InvalidArgument("third dimension of boxes must be either "
            "1 or equal to the third dimension of scores");
     }
@@ -185,7 +185,7 @@ Status CombinedNMSShapeFn(InferenceContext* c) {
              "if pad_per_class is set to true ");
     }
     output_size = std::min(c->Value(output_dim),
-        c->Value(size_per_class) * num_classes);
+        c->Value(size_per_class) * c->Value(class_dim));
   }
   c->set_output(0, c->MakeShape({batch_dim, output_size, 4}));
   c->set_output(1, c->MakeShape({batch_dim, output_size}));
diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index ef65789e22..3bf693daaf 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -3213,13 +3213,13 @@ def extract_glimpse_v2(
 
 @tf_export('image.combined_non_max_suppression')
 def combined_non_max_suppression(boxes,
-                             scores,
-                             max_output_size_per_class,
-                             max_total_size,
-                             iou_threshold=0.5,
-                             score_threshold=float('-inf'),
-                             pad_per_class=False,
-                             name=None):
+                                 scores,
+                                 max_output_size_per_class,
+                                 max_total_size,
+                                 iou_threshold=0.5,
+                                 score_threshold=float('-inf'),
+                                 pad_per_class=False,
+                                 name=None):
   """Greedily selects a subset of bounding boxes in descending order of score.
 
   This operation performs non_max_suppression on the inputs per batch, across
@@ -3229,8 +3229,8 @@ def combined_non_max_suppression(boxes,
   [y1, x1, y2, x2], where (y1, x1) and (y2, x2) are the coordinates of any
   diagonal pair of box corners and the coordinates can be provided as normalized
   (i.e., lying in the interval [0, 1]) or absolute.  Note that this algorithm
-  is agnostic to where the origin is in the coordinate system.  Note that this
-  algorithm is invariant to orthogonal transformations and translations
+  is agnostic to where the origin is in the coordinate system. Also note that
+  this algorithm is invariant to orthogonal transformations and translations
   of the coordinate system; thus translating or reflections of the coordinate
   system result in the same boxes being selected by the algorithm.
   The output of this operation is the final boxes, scores and classes tensor 
-- 
GitLab


From 1d36052a8e1a3adf42c524f7cde5a0fbedafdb6a Mon Sep 17 00:00:00 2001
From: Supriya Rao <supriyar@nvidia.com>
Date: Wed, 12 Dec 2018 13:32:01 -0800
Subject: [PATCH 1503/2345] Use vector instead of priority_queue in kernel op

---
 .../core/kernels/non_max_suppression_op.cc    | 39 ++++++++++---------
 1 file changed, 20 insertions(+), 19 deletions(-)

diff --git a/tensorflow/core/kernels/non_max_suppression_op.cc b/tensorflow/core/kernels/non_max_suppression_op.cc
index 821135d2f6..1615efab54 100644
--- a/tensorflow/core/kernels/non_max_suppression_op.cc
+++ b/tensorflow/core/kernels/non_max_suppression_op.cc
@@ -266,11 +266,7 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
       float box_coord[4];
     };
 
-    auto rc_cmp = [](const ResultCandidate rc_i, const ResultCandidate rc_j) {
-        return rc_i.score < rc_j.score;
-    };
-    std::priority_queue<ResultCandidate, std::vector<ResultCandidate>, 
-      decltype(rc_cmp)> result_candidate_pq(rc_cmp);
+    std::vector<ResultCandidate> result_candidate_vec;
 
     float* scores_data = per_batch_scores.unaligned_flat<float>().data();
     float* boxes_data = per_batch_boxes.unaligned_flat<float>().data();
@@ -310,13 +306,12 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
         float score;
       };
       auto cmp = [](const Candidate bs_i, const Candidate bs_j) {
-        return bs_i.score < bs_j.score;
+        return bs_i.score > bs_j.score;
       };
-      std::priority_queue<Candidate, std::deque<Candidate>, decltype(cmp)>
-        candidate_priority_queue(cmp);
+      std::vector<Candidate> candidate_vector;
       for (int i = 0; i < class_scores_data.size(); ++i) {
         if (class_scores_data[i] > score_threshold) {
-          candidate_priority_queue.emplace(Candidate({i, class_scores_data[i]}));
+          candidate_vector.emplace_back(Candidate({i, class_scores_data[i]}));
         }
       }
 
@@ -324,13 +319,14 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
       std::vector<float> selected_boxes;
       Candidate next_candidate;
 
+      std::sort(candidate_vector.begin(), candidate_vector.end(), cmp);
       const Tensor const_boxes = boxes;
       typename TTypes<float, 2>::ConstTensor boxes_data = 
         const_boxes.tensor<float, 2>();
+      int candidate_idx = 0;
       while (selected.size() < size_per_class &&
-             !candidate_priority_queue.empty()) {
-        next_candidate = candidate_priority_queue.top();
-        candidate_priority_queue.pop();
+             candidate_idx < candidate_vector.size()) {
+        next_candidate = candidate_vector[candidate_idx++];
 
         // Overlapping boxes are likely to have similar scores,
         // therefore we iterate through the previously selected boxes backwards
@@ -351,15 +347,20 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
           ResultCandidate rc = {next_candidate.box_index, next_candidate.score,
             class_idx, {boxes_data(id, 0), boxes_data(id, 1), boxes_data(id, 2),
             boxes_data(id, 3)}}; 
-          result_candidate_pq.push(rc);
+          result_candidate_vec.push_back(rc);
         }
       }
-
     }
+
+    auto rc_cmp = [](const ResultCandidate rc_i, const ResultCandidate rc_j) {
+        return rc_i.score > rc_j.score;
+    };
+    std::sort(result_candidate_vec.begin(), result_candidate_vec.end(), rc_cmp);
+
     int max_detections = 0;
     // If pad_per_class is false, we always pad to max_total_size
     if (!pad_per_class) {
-      max_detections = std::min((int) result_candidate_pq.size(), 
+      max_detections = std::min((int) result_candidate_vec.size(),
         total_size_per_batch);
       per_batch_size = total_size_per_batch;
     }
@@ -367,17 +368,17 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
       per_batch_size = std::min(total_size_per_batch, 
                             max_size_per_class * num_classes);
       max_detections = std::min(per_batch_size, 
-                            (int) result_candidate_pq.size());
+                            (int) result_candidate_vec.size());
     }
 
     final_valid_detections.push_back(max_detections);
 
     int curr_total_size = max_detections;
+    int result_idx = 0;
     // Pick the top max_detections values 
-    while (curr_total_size > 0 && !result_candidate_pq.empty())
+    while (curr_total_size > 0 && result_idx < result_candidate_vec.size())
     {
-      ResultCandidate next_candidate = result_candidate_pq.top();
-      result_candidate_pq.pop();
+      ResultCandidate next_candidate = result_candidate_vec[result_idx++];
       // Add to final output vectors
       nmsed_boxes[batch].push_back(
               std::max(std::min(next_candidate.box_coord[0], clip_window[2]),
-- 
GitLab


From c93e0b5daecc72ccf7961d6fe801551816b9df57 Mon Sep 17 00:00:00 2001
From: Supriya Rao <supriyar@nvidia.com>
Date: Thu, 13 Dec 2018 10:00:23 -0800
Subject: [PATCH 1504/2345] Addressed last set of review comments. Added test
 for pad_per_class which does padding based on max_total_size and
 max_size_per_class

---
 .../core/kernels/non_max_suppression_op.cc    |   3 +-
 .../kernels/non_max_suppression_op_test.cc    | 120 ++++++++++++------
 2 files changed, 83 insertions(+), 40 deletions(-)

diff --git a/tensorflow/core/kernels/non_max_suppression_op.cc b/tensorflow/core/kernels/non_max_suppression_op.cc
index 1615efab54..9e56a14704 100644
--- a/tensorflow/core/kernels/non_max_suppression_op.cc
+++ b/tensorflow/core/kernels/non_max_suppression_op.cc
@@ -376,8 +376,7 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
     int curr_total_size = max_detections;
     int result_idx = 0;
     // Pick the top max_detections values 
-    while (curr_total_size > 0 && result_idx < result_candidate_vec.size())
-    {
+    while (curr_total_size > 0 && result_idx < result_candidate_vec.size()) {
       ResultCandidate next_candidate = result_candidate_vec[result_idx++];
       // Add to final output vectors
       nmsed_boxes[batch].push_back(
diff --git a/tensorflow/core/kernels/non_max_suppression_op_test.cc b/tensorflow/core/kernels/non_max_suppression_op_test.cc
index afb96a9c82..07ad0405d6 100644
--- a/tensorflow/core/kernels/non_max_suppression_op_test.cc
+++ b/tensorflow/core/kernels/non_max_suppression_op_test.cc
@@ -888,17 +888,17 @@ TEST_F(CombinedNonMaxSuppressionOpTest, TestEmptyInput) {
   AddInputFromArray<float>(TensorShape({}), {0.0f});
   TF_ASSERT_OK(RunOpKernel());
 
-  //boxes
+  // boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({0, 10, 4}));
   test::FillValues<float>(&expected_boxes, {});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
   
-  //scores
+  // scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({0, 10}));
   test::FillValues<float>(&expected_scores, {});
   test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
   
-  //classes
+  // classes
   Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({0, 10}));
   test::FillValues<float>(&expected_classes, {});
   test::ExpectTensorEqual<float>(expected_classes, *GetOutput(2));
@@ -928,16 +928,16 @@ TEST_F(CombinedNonMaxSuppressionOpTest, TestSelectFromThreeClusters) {
   AddInputFromArray<float>(TensorShape({}), {0.0f});
   TF_ASSERT_OK(RunOpKernel());
   
-  //boxes
+  // boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({1, 3, 4}));
   test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 
           0, 0.3, 1, 0.4});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
-  //scores
+  // scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({1, 3}));
   test::FillValues<float>(&expected_scores, {0.95, 0.9, 0.3});
   test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
-  //classes
+  // classes
   Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({1, 3}));
   test::FillValues<float>(&expected_classes, {0, 0, 0});
   test::ExpectTensorEqual<float>(expected_classes, *GetOutput(2));
@@ -966,16 +966,16 @@ TEST_F(CombinedNonMaxSuppressionOpTest,
   AddInputFromArray<float>(TensorShape({}), {0.4f});
   TF_ASSERT_OK(RunOpKernel());
   
-  //boxes
+  // boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({1, 3, 4}));
   test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1,
       0, 0, 0, 0});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
-  //scores
+  // scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({1, 3}));
   test::FillValues<float>(&expected_scores, {0.95, 0.9, 0});
   test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
-  //classes
+  // classes
   Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({1, 3}));
   test::FillValues<float>(&expected_classes, {0, 0, 0});
   test::ExpectTensorEqual<float>(expected_classes, *GetOutput(2));
@@ -1006,16 +1006,16 @@ TEST_F(CombinedNonMaxSuppressionOpTest,
   AddInputFromArray<float>(TensorShape({}), {-3.0f});
   TF_ASSERT_OK(RunOpKernel());
   
-  //boxes
+  // boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({1, 5, 4}));
   test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 
       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
-  //scores
+  // scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({1, 5}));
   test::FillValues<float>(&expected_scores, {0.3, 0.1, 0, 0, 0});
   test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
-  //classes
+  // classes
   Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({1, 5}));
   test::FillValues<float>(&expected_classes, {0, 0, 0, 0, 0});
   test::ExpectTensorEqual<float>(expected_classes, *GetOutput(2));
@@ -1039,15 +1039,15 @@ TEST_F(CombinedNonMaxSuppressionOpTest, TestSelectSingleBox) {
   AddInputFromArray<float>(TensorShape({}), {0.0f});
   TF_ASSERT_OK(RunOpKernel());
   
-  //boxes
+  // boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({1, 1, 4}));
   test::FillValues<float>(&expected_boxes, {0, 0, 1, 1});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
-  //scores
+  // scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({1, 1}));
   test::FillValues<float>(&expected_scores, {0.9});
   test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
-  //classes
+  // classes
   Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({1, 1}));
   test::FillValues<float>(&expected_classes, {0});
   test::ExpectTensorEqual<float>(expected_classes, *GetOutput(2));
@@ -1079,16 +1079,16 @@ TEST_F(CombinedNonMaxSuppressionOpTest,
   AddInputFromArray<float>(TensorShape({}), {0.4f});
   TF_ASSERT_OK(RunOpKernel());
   
-  //boxes
+  // boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 3, 4}));
   test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 
           0, 0, 0, 0, 0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2, 0, 0, 0, 0});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
-  //scores
+  // scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 3}));
   test::FillValues<float>(&expected_scores, {0.95, 0.9, 0, 0.95, 0.9, 0});
   test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
-  //classes
+  // classes
   Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({2, 3}));
   test::FillValues<float>(&expected_classes, {0, 0, 0, 0, 0, 0});
   test::ExpectTensorEqual<float>(expected_classes, *GetOutput(2));
@@ -1121,17 +1121,17 @@ TEST_F(CombinedNonMaxSuppressionOpTest,
   AddInputFromArray<float>(TensorShape({}), {0.0f});
   TF_ASSERT_OK(RunOpKernel());
   
-  //boxes
+  // boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 3, 4}));
   test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 
           0, 0.01f, 0.1, 0.11f, 0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2, 
           0, 0.02f, 0.2, 0.22f});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
-  //scores
+  // scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 3}));
   test::FillValues<float>(&expected_scores, {0.95, 0.9, 0.75, 0.95, 0.9, 0.75});
   test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
-  //classes
+  // classes
   Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({2, 3}));
   test::FillValues<float>(&expected_classes, {0, 1, 0, 0, 1, 0});
   test::ExpectTensorEqual<float>(expected_classes, *GetOutput(2));
@@ -1165,16 +1165,16 @@ TEST_F(CombinedNonMaxSuppressionOpTest,
   AddInputFromArray<float>(TensorShape({}), {0.8f});
   TF_ASSERT_OK(RunOpKernel());
   
-  //boxes
+  // boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 3, 4}));
   test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 
           0, 0, 0, 0, 0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2, 0, 0, 0, 0});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
-  //scores
+  // scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 3}));
   test::FillValues<float>(&expected_scores, {0.95, 0.9, 0, 0.95, 0.9, 0});
   test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
-  //classes
+  // classes
   Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({2, 3}));
   test::FillValues<float>(&expected_classes, {0, 1, 0, 0, 1, 0});
   test::ExpectTensorEqual<float>(expected_classes, *GetOutput(2));
@@ -1189,7 +1189,7 @@ TEST_F(CombinedNonMaxSuppressionOpTest,
 }
 
 TEST_F(CombinedNonMaxSuppressionOpTest,
-        TestSelectFromTwoBatchesTwoClassesWithScoreThresholdPadded) {
+        TestSelectFromTwoBatchesTwoClassesWithScoreThresholdPaddedTotalSize) {
   MakeOp(true);
   AddInputFromArray<float>(
       TensorShape({2, 6, 1, 4}),
@@ -1202,22 +1202,22 @@ TEST_F(CombinedNonMaxSuppressionOpTest,
       {0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f, 
        0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f
        }); 
-  AddInputFromArray<int>(TensorShape({}), {3});
+  AddInputFromArray<int>(TensorShape({}), {10});
   AddInputFromArray<int>(TensorShape({}), {3});
   AddInputFromArray<float>(TensorShape({}), {.5f});
   AddInputFromArray<float>(TensorShape({}), {0.8f});
   TF_ASSERT_OK(RunOpKernel());
   
-  //boxes
+  // boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 3, 4}));
   test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 
           0, 0, 0, 0, 0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2, 0, 0, 0, 0});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
-  //scores
+  // scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 3}));
   test::FillValues<float>(&expected_scores, {0.95, 0.9, 0, 0.95, 0.9, 0});
   test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
-  //classes
+  // classes
   Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({2, 3}));
   test::FillValues<float>(&expected_classes, {0, 1, 0, 0, 1, 0});
   test::ExpectTensorEqual<float>(expected_classes, *GetOutput(2));
@@ -1231,6 +1231,50 @@ TEST_F(CombinedNonMaxSuppressionOpTest,
   test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
 
+TEST_F(CombinedNonMaxSuppressionOpTest,
+        TestSelectFromTwoBatchesTwoClassesWithScoreThresholdPaddedPerClass) {
+  MakeOp(true);
+  AddInputFromArray<float>(
+      TensorShape({2, 6, 1, 4}),
+      {0, 0,    0.1, 0.1, 0, 0.01f, 0.1, 0.11f, 0, -0.01, 0.1, 0.09f,
+       0, 0.11, 0.1, 0.2, 0, 0.12f, 0.1, 0.21f, 0,  0.3,  1,   0.4,
+       0, 0,    0.2, 0.2, 0, 0.02f, 0.2, 0.22f, 0, -0.02, 0.2, 0.19f,
+       0, 0.21, 0.2, 0.3, 0, 0.22f, 0.2, 0.31f, 0,  0.4,  1,   0.5
+       });
+  AddInputFromArray<float>(TensorShape({2, 6, 2}),
+      {0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f,
+       0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f
+       });
+  AddInputFromArray<int>(TensorShape({}), {2});
+  AddInputFromArray<int>(TensorShape({}), {50});
+  AddInputFromArray<float>(TensorShape({}), {.5f});
+  AddInputFromArray<float>(TensorShape({}), {0.8f});
+  TF_ASSERT_OK(RunOpKernel());
+
+  // boxes
+  Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 4, 4}));
+  test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1,
+          0, 0, 0, 0, 0, 0, 0, 0, 0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2, 0, 0, 0, 0,
+          0, 0, 0, 0});
+  test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
+  // scores
+  Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 4}));
+  test::FillValues<float>(&expected_scores, {0.95, 0.9, 0, 0, 0.95, 0.9, 0, 0});
+  test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
+  // classes
+  Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({2, 4}));
+  test::FillValues<float>(&expected_classes, {0, 1, 0, 0, 0, 1, 0, 0});
+  test::ExpectTensorEqual<float>(expected_classes, *GetOutput(2));
+  // valid
+  Tensor expected_valid_d(allocator(), DT_INT32, TensorShape({2}));
+  test::FillValues<int>(&expected_valid_d, {2, 2});
+  test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
+  // indices
+  Tensor expected_indices(allocator(), DT_INT32, TensorShape({2, 4}));
+  test::FillValues<int>(&expected_indices, {3, 0, 0, 0, 3, 0, 0, 0});
+  test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
+}
+
 TEST_F(CombinedNonMaxSuppressionOpTest,
         TestSelectFromTwoBatchesTwoClassesTotalSize) {
   MakeOp();
@@ -1246,25 +1290,25 @@ TEST_F(CombinedNonMaxSuppressionOpTest,
        0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f
        }); 
   AddInputFromArray<int>(TensorShape({}), {3});
-  //Total size per batch is more than size per class
+  // Total size per batch is more than size per class
   AddInputFromArray<int>(TensorShape({}), {5});
   AddInputFromArray<float>(TensorShape({}), {.5f});
   AddInputFromArray<float>(TensorShape({}), {0.1f});
   TF_ASSERT_OK(RunOpKernel());
   
-  //boxes
+  // boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 5, 4}));
   test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 
           0, 0.01f, 0.1, 0.11f, 0, 0.12f, 0.1, 0.21f, 0,  0.3, 1, 0.4,
           0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2, 0, 0.02f, 0.2, 0.22f,
           0, 0.22f, 0.2, 0.31f, 0, 0.4, 1, 0.5});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
-  //scores
+  // scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 5}));
   test::FillValues<float>(&expected_scores, {0.95, 0.9, 0.75, 0.5, 0.3, 
           0.95, 0.9, 0.75, 0.5, 0.3});
   test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
-  //classes
+  // classes
   Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({2, 5}));
   test::FillValues<float>(&expected_classes, {0, 1, 0, 1, 0, 0, 1, 0, 1, 0});
   test::ExpectTensorEqual<float>(expected_classes, *GetOutput(2));
@@ -1283,14 +1327,14 @@ TEST_F(CombinedNonMaxSuppressionOpTest,
   MakeOp();
   AddInputFromArray<float>(
       TensorShape({2, 6, 2, 4}),
-      //batch 0, box1 of class 1 should get selected
+      // batch 0, box1 of class 1 should get selected
       {0, 0,     0.1, 0.1,   0, 0,     0.1, 0.1, 
        0, 0.01f, 0.1, 0.11f, 0, 0.6f,  0.1, 0.7f,
        0, -0.01, 0.1, 0.09f, 0, -0.01, 0.1, 0.09f, 
        0, 0.11,  0.1, 0.2,   0, 0.11,  0.1, 0.2, 
        0, 0.12f, 0.1, 0.21f, 0, 0.12f, 0.1, 0.21f, 
        0, 0.3,   1,   0.4,   0, 0.3,   1,   0.4, 
-      //batch 1, box1 of class 0 should get selected
+      // batch 1, box1 of class 0 should get selected
        0, 0,     0.2, 0.2,   0, 0,     0.2, 0.2, 
        0, 0.02f, 0.2, 0.22f, 0, 0.02f, 0.2, 0.22f,
        0, -0.02, 0.2, 0.19f, 0, -0.02, 0.2, 0.19f,
@@ -1308,16 +1352,16 @@ TEST_F(CombinedNonMaxSuppressionOpTest,
   AddInputFromArray<float>(TensorShape({}), {0.0f});
   TF_ASSERT_OK(RunOpKernel());
   
-  //boxes
+  // boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 3, 4}));
   test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 
            0, 0.6f, 0.1, 0.7f, 0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2, 0, 0.02f, 0.2, 0.22f});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
-  //scores
+  // scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 3}));
   test::FillValues<float>(&expected_scores, {0.95, 0.9, 0.8, 0.95, 0.9, 0.75});
   test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
-  //classes
+  // classes
   Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({2, 3}));
   test::FillValues<float>(&expected_classes, {0, 1, 1, 0, 1, 0});
   test::ExpectTensorEqual<float>(expected_classes, *GetOutput(2));
-- 
GitLab


From 487eae69e1e4d0bc4df61ebe2420c5551d2985f9 Mon Sep 17 00:00:00 2001
From: Supriya Rao <supriyar@nvidia.com>
Date: Thu, 13 Dec 2018 11:33:07 -0800
Subject: [PATCH 1505/2345] Clang and pylint formatting fixes

---
 .../core/kernels/non_max_suppression_op.cc    | 129 +++++-----
 .../kernels/non_max_suppression_op_test.cc    | 234 +++++++++---------
 tensorflow/core/ops/image_ops.cc              |  28 +--
 tensorflow/python/ops/image_ops_impl.py       |  15 +-
 4 files changed, 203 insertions(+), 203 deletions(-)

diff --git a/tensorflow/core/kernels/non_max_suppression_op.cc b/tensorflow/core/kernels/non_max_suppression_op.cc
index 9e56a14704..cc525331f2 100644
--- a/tensorflow/core/kernels/non_max_suppression_op.cc
+++ b/tensorflow/core/kernels/non_max_suppression_op.cc
@@ -74,9 +74,9 @@ static inline void ParseAndCheckBoxSizes(OpKernelContext* context,
               errors::InvalidArgument("boxes must have 4 columns"));
 }
 
-static inline void CheckCombinedNMSScoreSizes(OpKernelContext* context, 
-                                          int num_boxes,
-                                          const Tensor& scores) {
+static inline void CheckCombinedNMSScoreSizes(OpKernelContext* context,
+                                              int num_boxes,
+                                              const Tensor& scores) {
   // The shape of 'scores' is [batch_size, num_boxes, num_classes]
   OP_REQUIRES(context, scores.dims() == 3,
               errors::InvalidArgument("scores must be 3-D",
@@ -86,8 +86,9 @@ static inline void CheckCombinedNMSScoreSizes(OpKernelContext* context,
 }
 
 static inline void ParseAndCheckCombinedNMSBoxSizes(OpKernelContext* context,
-                                         const Tensor& boxes, int* num_boxes, 
-                                         const int num_classes) {
+                                                    const Tensor& boxes,
+                                                    int* num_boxes,
+                                                    const int num_classes) {
   // The shape of 'boxes' is [batch_size, num_boxes, q, 4]
   OP_REQUIRES(context, boxes.dims() == 4,
               errors::InvalidArgument("boxes must be 4-D",
@@ -97,7 +98,7 @@ static inline void ParseAndCheckCombinedNMSBoxSizes(OpKernelContext* context,
   OP_REQUIRES(context, box_check,
               errors::InvalidArgument(
                   "third dimension of boxes must be either 1 or num classes"));
-  *num_boxes = boxes.dim_size(1); 
+  *num_boxes = boxes.dim_size(1);
   OP_REQUIRES(context, boxes.dim_size(3) == 4,
               errors::InvalidArgument("boxes must have 4 columns"));
 }
@@ -222,23 +223,18 @@ void DoNonMaxSuppressionOp(
   std::copy_n(selected.begin(), selected.size(), output_indices_data.data());
 }
 
-void BatchedNonMaxSuppressionOp(OpKernelContext* context,
-                                const Tensor& inp_boxes,
-                                const Tensor& inp_scores,
-                                int num_boxes,
-                                const int max_size_per_class,
-                                const int total_size_per_batch,
-                                const float score_threshold,
-                                const float iou_threshold,
-                                bool pad_per_class = false) {
-
+void BatchedNonMaxSuppressionOp(
+    OpKernelContext* context, const Tensor& inp_boxes, const Tensor& inp_scores,
+    int num_boxes, const int max_size_per_class, const int total_size_per_batch,
+    const float score_threshold, const float iou_threshold,
+    bool pad_per_class = false) {
   int q = inp_boxes.dim_size(2);
   int num_classes = inp_scores.dim_size(2);
   const int num_batches = inp_boxes.dim_size(0);
 
   // Default clip window of [0, 0, 1, 1] if none specified
   std::vector<float> clip_window{0, 0, 1, 1};
-  
+
   // [num_batches, per_batch_size * 4]
   std::vector<std::vector<float>> nmsed_boxes(num_batches);
   // [num_batches, per_batch_size]
@@ -280,23 +276,22 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
         // Get the scores per class
         // class_scores_data dim is [num_boxes].
         class_scores_data.push_back(scores_data[box * num_classes + class_idx]);
-        for (int cid = 0; cid < 4; ++cid){
-          if (q > 1){
+        for (int cid = 0; cid < 4; ++cid) {
+          if (q > 1) {
             // Get the boxes per class. class_boxes_data dims is [num_boxes, 4]
-            class_boxes_data.push_back(boxes_data[(box * q + class_idx) * 4
-                                                  + cid]);
-          }
-          else {
-              class_boxes_data.push_back(boxes_data[box * 4 + cid]);
+            class_boxes_data.push_back(
+                boxes_data[(box * q + class_idx) * 4 + cid]);
+          } else {
+            class_boxes_data.push_back(boxes_data[box * 4 + cid]);
           }
         }
       }
-        
+
       // Copy class_boxes_data to a tensor
       TensorShape boxesShape({num_boxes, 4});
       Tensor boxes(per_batch_boxes.dtype(), boxesShape);
       std::copy_n(class_boxes_data.begin(), class_boxes_data.size(),
-          boxes.unaligned_flat<float>().data());
+                  boxes.unaligned_flat<float>().data());
 
       const int size_per_class = std::min(max_size_per_class, num_boxes);
       // Do NMS, get the candidate indices of form vector<int>
@@ -321,8 +316,8 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
 
       std::sort(candidate_vector.begin(), candidate_vector.end(), cmp);
       const Tensor const_boxes = boxes;
-      typename TTypes<float, 2>::ConstTensor boxes_data = 
-        const_boxes.tensor<float, 2>();
+      typename TTypes<float, 2>::ConstTensor boxes_data =
+          const_boxes.tensor<float, 2>();
       int candidate_idx = 0;
       while (selected.size() < size_per_class &&
              candidate_idx < candidate_vector.size()) {
@@ -333,8 +328,8 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
         // in order to see if `next_candidate` should be suppressed.
         bool should_select = true;
         for (int j = selected.size() - 1; j >= 0; --j) {
-          if (IOUGreaterThanThreshold(boxes_data, next_candidate.box_index, 
-              selected[j], iou_threshold)) {
+          if (IOUGreaterThanThreshold(boxes_data, next_candidate.box_index,
+                                      selected[j], iou_threshold)) {
             should_select = false;
             break;
           }
@@ -344,53 +339,54 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
           selected.push_back(next_candidate.box_index);
           // Add the selected box to the result candidate. Sorted by score
           int id = next_candidate.box_index;
-          ResultCandidate rc = {next_candidate.box_index, next_candidate.score,
-            class_idx, {boxes_data(id, 0), boxes_data(id, 1), boxes_data(id, 2),
-            boxes_data(id, 3)}}; 
+          ResultCandidate rc = {next_candidate.box_index,
+                                next_candidate.score,
+                                class_idx,
+                                {boxes_data(id, 0), boxes_data(id, 1),
+                                 boxes_data(id, 2), boxes_data(id, 3)}};
           result_candidate_vec.push_back(rc);
         }
       }
     }
 
     auto rc_cmp = [](const ResultCandidate rc_i, const ResultCandidate rc_j) {
-        return rc_i.score > rc_j.score;
+      return rc_i.score > rc_j.score;
     };
     std::sort(result_candidate_vec.begin(), result_candidate_vec.end(), rc_cmp);
 
     int max_detections = 0;
     // If pad_per_class is false, we always pad to max_total_size
     if (!pad_per_class) {
-      max_detections = std::min((int) result_candidate_vec.size(),
-        total_size_per_batch);
+      max_detections =
+          std::min((int)result_candidate_vec.size(), total_size_per_batch);
       per_batch_size = total_size_per_batch;
-    }
-    else {
-      per_batch_size = std::min(total_size_per_batch, 
-                            max_size_per_class * num_classes);
-      max_detections = std::min(per_batch_size, 
-                            (int) result_candidate_vec.size());
+    } else {
+      per_batch_size =
+          std::min(total_size_per_batch, max_size_per_class * num_classes);
+      max_detections =
+          std::min(per_batch_size, (int)result_candidate_vec.size());
     }
 
     final_valid_detections.push_back(max_detections);
 
     int curr_total_size = max_detections;
     int result_idx = 0;
-    // Pick the top max_detections values 
+    // Pick the top max_detections values
     while (curr_total_size > 0 && result_idx < result_candidate_vec.size()) {
       ResultCandidate next_candidate = result_candidate_vec[result_idx++];
       // Add to final output vectors
       nmsed_boxes[batch].push_back(
-              std::max(std::min(next_candidate.box_coord[0], clip_window[2]),
-                       clip_window[0]));
+          std::max(std::min(next_candidate.box_coord[0], clip_window[2]),
+                   clip_window[0]));
       nmsed_boxes[batch].push_back(
-              std::max(std::min(next_candidate.box_coord[1], clip_window[3]),
-                       clip_window[1]));
+          std::max(std::min(next_candidate.box_coord[1], clip_window[3]),
+                   clip_window[1]));
       nmsed_boxes[batch].push_back(
-              std::max(std::min(next_candidate.box_coord[2], clip_window[2]),
-                       clip_window[0]));
+          std::max(std::min(next_candidate.box_coord[2], clip_window[2]),
+                   clip_window[0]));
       nmsed_boxes[batch].push_back(
-              std::max(std::min(next_candidate.box_coord[3], clip_window[3]),
-                       clip_window[1]));
+          std::max(std::min(next_candidate.box_coord[3], clip_window[3]),
+                   clip_window[1]));
       nmsed_scores[batch].push_back(next_candidate.score);
       nmsed_classes[batch].push_back(next_candidate.class_idx);
       selected_indices[batch].push_back(next_candidate.box_index);
@@ -422,13 +418,13 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
 
   Tensor* valid_detections_t = nullptr;
   TensorShape valid_detections_shape({num_batches});
-  OP_REQUIRES_OK(context, context->allocate_output(3, valid_detections_shape, 
-                 &valid_detections_t));
+  OP_REQUIRES_OK(context, context->allocate_output(3, valid_detections_shape,
+                                                   &valid_detections_t));
   auto valid_detections_flat = valid_detections_t->template flat<int>();
 
   Tensor* selected_indices_t = nullptr;
-  OP_REQUIRES_OK(context,
-                context->allocate_output(4, scores_shape, &selected_indices_t));
+  OP_REQUIRES_OK(
+      context, context->allocate_output(4, scores_shape, &selected_indices_t));
   auto selected_indices_flat = selected_indices_t->template flat<int>();
   for (int i = 0; i < num_batches; ++i) {
     valid_detections_flat(i) = final_valid_detections[i];
@@ -437,8 +433,8 @@ void BatchedNonMaxSuppressionOp(OpKernelContext* context,
       nmsed_classes_flat(i * per_batch_size + j) = nmsed_classes[i][j];
       selected_indices_flat(i * per_batch_size + j) = selected_indices[i][j];
       for (int k = 0; k < 4; ++k) {
-        nmsed_boxes_flat(i * per_batch_size * 4 + j * 4 + k) = 
-          nmsed_boxes[i][j * 4 + k]; 
+        nmsed_boxes_flat(i * per_batch_size * 4 + j * 4 + k) =
+            nmsed_boxes[i][j * 4 + k];
       }
     }
   }
@@ -689,8 +685,7 @@ class CombinedNonMaxSuppressionOp : public OpKernel {
  public:
   explicit CombinedNonMaxSuppressionOp(OpKernelConstruction* context)
       : OpKernel(context) {
-    OP_REQUIRES_OK(context, context->GetAttr("pad_per_class",
-                                             &pad_per_class_));
+    OP_REQUIRES_OK(context, context->GetAttr("pad_per_class", &pad_per_class_));
   }
 
   void Compute(OpKernelContext* context) override {
@@ -698,9 +693,9 @@ class CombinedNonMaxSuppressionOp : public OpKernel {
     const Tensor& boxes = context->input(0);
     // scores: [batch_size, num_anchors, num_classes]
     const Tensor& scores = context->input(1);
-    OP_REQUIRES(context, (boxes.dim_size(0) == scores.dim_size(0)),
-              errors::InvalidArgument(
-                "boxes and scores must have same batch size"));
+    OP_REQUIRES(
+        context, (boxes.dim_size(0) == scores.dim_size(0)),
+        errors::InvalidArgument("boxes and scores must have same batch size"));
 
     // max_output_size: scalar
     const Tensor& max_output_size = context->input(2);
@@ -743,16 +738,16 @@ class CombinedNonMaxSuppressionOp : public OpKernel {
     if (!context->status().ok()) {
       return;
     }
-    BatchedNonMaxSuppressionOp(context, boxes, scores, num_boxes, 
-                               max_size_per_class, max_total_size_per_batch, 
-                               score_threshold_val, iou_threshold_val, 
+    BatchedNonMaxSuppressionOp(context, boxes, scores, num_boxes,
+                               max_size_per_class, max_total_size_per_batch,
+                               score_threshold_val, iou_threshold_val,
                                pad_per_class_);
   }
+
  private:
   bool pad_per_class_;
 };
 
-
 REGISTER_KERNEL_BUILDER(Name("NonMaxSuppression").Device(DEVICE_CPU),
                         NonMaxSuppressionOp<CPUDevice>);
 
@@ -785,6 +780,6 @@ REGISTER_KERNEL_BUILDER(
     NonMaxSuppressionWithOverlapsOp<CPUDevice>);
 
 REGISTER_KERNEL_BUILDER(Name("CombinedNonMaxSuppression").Device(DEVICE_CPU),
-        CombinedNonMaxSuppressionOp<CPUDevice>);
+                        CombinedNonMaxSuppressionOp<CPUDevice>);
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/non_max_suppression_op_test.cc b/tensorflow/core/kernels/non_max_suppression_op_test.cc
index 07ad0405d6..f5c3bc64e1 100644
--- a/tensorflow/core/kernels/non_max_suppression_op_test.cc
+++ b/tensorflow/core/kernels/non_max_suppression_op_test.cc
@@ -892,22 +892,22 @@ TEST_F(CombinedNonMaxSuppressionOpTest, TestEmptyInput) {
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({0, 10, 4}));
   test::FillValues<float>(&expected_boxes, {});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
-  
+
   // scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({0, 10}));
   test::FillValues<float>(&expected_scores, {});
   test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
-  
+
   // classes
   Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({0, 10}));
   test::FillValues<float>(&expected_classes, {});
   test::ExpectTensorEqual<float>(expected_classes, *GetOutput(2));
-  
+
   // valid
   Tensor expected_valid_d(allocator(), DT_INT32, TensorShape({0}));
   test::FillValues<int>(&expected_valid_d, {});
   test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
-  
+
   // indices
   Tensor expected_indices(allocator(), DT_INT32, TensorShape({0, 10}));
   test::FillValues<int>(&expected_indices, {});
@@ -919,19 +919,19 @@ TEST_F(CombinedNonMaxSuppressionOpTest, TestSelectFromThreeClusters) {
   AddInputFromArray<float>(
       TensorShape({1, 6, 1, 4}),
       {0, 0,    0.1, 0.1, 0, 0.01f, 0.1, 0.11f, 0, -0.01, 0.1, 0.09f,
-       0, 0.11, 0.1, 0.2, 0, 0.12f, 0.1, 0.21f, 0,  0.3, 1,   0.4});
-  AddInputFromArray<float>(TensorShape({1, 6, 1}), 
-       {.9f, .75f, .6f, .95f, .5f, .3f});
+       0, 0.11, 0.1, 0.2, 0, 0.12f, 0.1, 0.21f, 0, 0.3,   1,   0.4});
+  AddInputFromArray<float>(TensorShape({1, 6, 1}),
+                           {.9f, .75f, .6f, .95f, .5f, .3f});
   AddInputFromArray<int>(TensorShape({}), {3});
   AddInputFromArray<int>(TensorShape({}), {3});
   AddInputFromArray<float>(TensorShape({}), {.5f});
   AddInputFromArray<float>(TensorShape({}), {0.0f});
   TF_ASSERT_OK(RunOpKernel());
-  
+
   // boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({1, 3, 4}));
-  test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 
-          0, 0.3, 1, 0.4});
+  test::FillValues<float>(&expected_boxes,
+                          {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 0, 0.3, 1, 0.4});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
   // scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({1, 3}));
@@ -952,24 +952,24 @@ TEST_F(CombinedNonMaxSuppressionOpTest, TestSelectFromThreeClusters) {
 }
 
 TEST_F(CombinedNonMaxSuppressionOpTest,
-        TestSelectFromThreeClustersWithScoreThreshold) {
+       TestSelectFromThreeClustersWithScoreThreshold) {
   MakeOp();
   AddInputFromArray<float>(
       TensorShape({1, 6, 1, 4}),
       {0, 0,    0.1, 0.1, 0, 0.01f, 0.1, 0.11f, 0, -0.01, 0.1, 0.09f,
-       0, 0.11, 0.1, 0.2, 0, 0.12f, 0.1, 0.21f, 0,  0.3, 1,   0.4});
-  AddInputFromArray<float>(TensorShape({1, 6, 1}), 
-       {.9f, .75f, .6f, .95f, .5f, .3f});
+       0, 0.11, 0.1, 0.2, 0, 0.12f, 0.1, 0.21f, 0, 0.3,   1,   0.4});
+  AddInputFromArray<float>(TensorShape({1, 6, 1}),
+                           {.9f, .75f, .6f, .95f, .5f, .3f});
   AddInputFromArray<int>(TensorShape({}), {3});
   AddInputFromArray<int>(TensorShape({}), {3});
   AddInputFromArray<float>(TensorShape({}), {.5f});
   AddInputFromArray<float>(TensorShape({}), {0.4f});
   TF_ASSERT_OK(RunOpKernel());
-  
+
   // boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({1, 3, 4}));
-  test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1,
-      0, 0, 0, 0});
+  test::FillValues<float>(&expected_boxes,
+                          {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 0, 0, 0, 0});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
   // scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({1, 3}));
@@ -990,14 +990,14 @@ TEST_F(CombinedNonMaxSuppressionOpTest,
 }
 
 TEST_F(CombinedNonMaxSuppressionOpTest,
-        TestSelectFromThreeClustersWithScoreThresholdZeroScores) {
+       TestSelectFromThreeClustersWithScoreThresholdZeroScores) {
   MakeOp();
   AddInputFromArray<float>(
       TensorShape({1, 6, 1, 4}),
       {0, 0,    0.1, 0.1, 0, 0.01f, 0.1, 0.11f, 0, -0.01, 0.1, 0.09f,
-       0, 0.11, 0.1, 0.2, 0, 0.12f, 0.1, 0.21f, 0,  0.3, 1,   0.4});
-  AddInputFromArray<float>(TensorShape({1, 6, 1}), 
-       {.1f, 0, 0, .3f, .2f, -5.0f});
+       0, 0.11, 0.1, 0.2, 0, 0.12f, 0.1, 0.21f, 0, 0.3,   1,   0.4});
+  AddInputFromArray<float>(TensorShape({1, 6, 1}),
+                           {.1f, 0, 0, .3f, .2f, -5.0f});
   // If we ask for more boxes than we actually expect to get back;
   // should still only get 2 boxes back.
   AddInputFromArray<int>(TensorShape({}), {4});
@@ -1005,11 +1005,14 @@ TEST_F(CombinedNonMaxSuppressionOpTest,
   AddInputFromArray<float>(TensorShape({}), {.5f});
   AddInputFromArray<float>(TensorShape({}), {-3.0f});
   TF_ASSERT_OK(RunOpKernel());
-  
+
   // boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({1, 5, 4}));
-  test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 
-      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,});
+  test::FillValues<float>(
+      &expected_boxes,
+      {
+          0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      });
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
   // scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({1, 5}));
@@ -1031,14 +1034,14 @@ TEST_F(CombinedNonMaxSuppressionOpTest,
 
 TEST_F(CombinedNonMaxSuppressionOpTest, TestSelectSingleBox) {
   MakeOp();
-  AddInputFromArray<float>(TensorShape({1, 1, 1, 4}), {0, 0, 1, 1});  
+  AddInputFromArray<float>(TensorShape({1, 1, 1, 4}), {0, 0, 1, 1});
   AddInputFromArray<float>(TensorShape({1, 1, 1}), {.9f});
   AddInputFromArray<int>(TensorShape({}), {3});
   AddInputFromArray<int>(TensorShape({}), {1});
   AddInputFromArray<float>(TensorShape({}), {.5f});
   AddInputFromArray<float>(TensorShape({}), {0.0f});
   TF_ASSERT_OK(RunOpKernel());
-  
+
   // boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({1, 1, 4}));
   test::FillValues<float>(&expected_boxes, {0, 0, 1, 1});
@@ -1062,27 +1065,28 @@ TEST_F(CombinedNonMaxSuppressionOpTest, TestSelectSingleBox) {
 }
 
 TEST_F(CombinedNonMaxSuppressionOpTest,
-        TestSelectFromTwoBatchesWithScoreThreshold) {
+       TestSelectFromTwoBatchesWithScoreThreshold) {
   MakeOp();
   AddInputFromArray<float>(
       TensorShape({2, 6, 1, 4}),
       {0, 0,    0.1, 0.1, 0, 0.01f, 0.1, 0.11f, 0, -0.01, 0.1, 0.09f,
-       0, 0.11, 0.1, 0.2, 0, 0.12f, 0.1, 0.21f, 0,  0.3,  1,   0.4,
+       0, 0.11, 0.1, 0.2, 0, 0.12f, 0.1, 0.21f, 0, 0.3,   1,   0.4,
        0, 0,    0.2, 0.2, 0, 0.02f, 0.2, 0.22f, 0, -0.02, 0.2, 0.19f,
-       0, 0.21, 0.2, 0.3, 0, 0.22f, 0.2, 0.31f, 0,  0.4,  1,   0.5
-       });
-  AddInputFromArray<float>(TensorShape({2, 6, 1}), 
-       {.9f, .75f, .6f, .95f, .5f, .3f, .9f, .75f, .6f, .95f, .5f, .3f});
+       0, 0.21, 0.2, 0.3, 0, 0.22f, 0.2, 0.31f, 0, 0.4,   1,   0.5});
+  AddInputFromArray<float>(
+      TensorShape({2, 6, 1}),
+      {.9f, .75f, .6f, .95f, .5f, .3f, .9f, .75f, .6f, .95f, .5f, .3f});
   AddInputFromArray<int>(TensorShape({}), {3});
   AddInputFromArray<int>(TensorShape({}), {3});
   AddInputFromArray<float>(TensorShape({}), {.5f});
   AddInputFromArray<float>(TensorShape({}), {0.4f});
   TF_ASSERT_OK(RunOpKernel());
-  
+
   // boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 3, 4}));
-  test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 
-          0, 0, 0, 0, 0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2, 0, 0, 0, 0});
+  test::FillValues<float>(&expected_boxes,
+                          {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 0, 0, 0, 0,
+                           0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2, 0, 0, 0, 0});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
   // scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 3}));
@@ -1102,30 +1106,30 @@ TEST_F(CombinedNonMaxSuppressionOpTest,
   test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
 
-TEST_F(CombinedNonMaxSuppressionOpTest,
-        TestSelectFromTwoBatchesTwoClasses) {
+TEST_F(CombinedNonMaxSuppressionOpTest, TestSelectFromTwoBatchesTwoClasses) {
   MakeOp();
   AddInputFromArray<float>(
       TensorShape({2, 6, 1, 4}),
       {0, 0,    0.1, 0.1, 0, 0.01f, 0.1, 0.11f, 0, -0.01, 0.1, 0.09f,
-       0, 0.11, 0.1, 0.2, 0, 0.12f, 0.1, 0.21f, 0,  0.3,  1,   0.4,
+       0, 0.11, 0.1, 0.2, 0, 0.12f, 0.1, 0.21f, 0, 0.3,   1,   0.4,
        0, 0,    0.2, 0.2, 0, 0.02f, 0.2, 0.22f, 0, -0.02, 0.2, 0.19f,
-       0, 0.21, 0.2, 0.3, 0, 0.22f, 0.2, 0.31f, 0,  0.4,  1,   0.5
-       });
-  AddInputFromArray<float>(TensorShape({2, 6, 2}), 
-       {0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f, 
-       0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f}); 
+       0, 0.21, 0.2, 0.3, 0, 0.22f, 0.2, 0.31f, 0, 0.4,   1,   0.5});
+  AddInputFromArray<float>(TensorShape({2, 6, 2}),
+                           {0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f,
+                            0.5f, 0.5f, 0.3f,  0.1f, 0.1f, 0.9f, 0.75f, 0.8f,
+                            0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f,  0.1f});
   AddInputFromArray<int>(TensorShape({}), {3});
   AddInputFromArray<int>(TensorShape({}), {3});
   AddInputFromArray<float>(TensorShape({}), {.5f});
   AddInputFromArray<float>(TensorShape({}), {0.0f});
   TF_ASSERT_OK(RunOpKernel());
-  
+
   // boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 3, 4}));
-  test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 
-          0, 0.01f, 0.1, 0.11f, 0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2, 
-          0, 0.02f, 0.2, 0.22f});
+  test::FillValues<float>(
+      &expected_boxes,
+      {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 0, 0.01f, 0.1, 0.11f,
+       0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2, 0, 0.02f, 0.2, 0.22f});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
   // scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 3}));
@@ -1146,29 +1150,29 @@ TEST_F(CombinedNonMaxSuppressionOpTest,
 }
 
 TEST_F(CombinedNonMaxSuppressionOpTest,
-        TestSelectFromTwoBatchesTwoClassesWithScoreThreshold) {
+       TestSelectFromTwoBatchesTwoClassesWithScoreThreshold) {
   MakeOp();
   AddInputFromArray<float>(
       TensorShape({2, 6, 1, 4}),
       {0, 0,    0.1, 0.1, 0, 0.01f, 0.1, 0.11f, 0, -0.01, 0.1, 0.09f,
-       0, 0.11, 0.1, 0.2, 0, 0.12f, 0.1, 0.21f, 0,  0.3,  1,   0.4,
+       0, 0.11, 0.1, 0.2, 0, 0.12f, 0.1, 0.21f, 0, 0.3,   1,   0.4,
        0, 0,    0.2, 0.2, 0, 0.02f, 0.2, 0.22f, 0, -0.02, 0.2, 0.19f,
-       0, 0.21, 0.2, 0.3, 0, 0.22f, 0.2, 0.31f, 0,  0.4,  1,   0.5
-       });
-  AddInputFromArray<float>(TensorShape({2, 6, 2}), 
-      {0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f, 
-       0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f
-       }); 
+       0, 0.21, 0.2, 0.3, 0, 0.22f, 0.2, 0.31f, 0, 0.4,   1,   0.5});
+  AddInputFromArray<float>(TensorShape({2, 6, 2}),
+                           {0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f,
+                            0.5f, 0.5f, 0.3f,  0.1f, 0.1f, 0.9f, 0.75f, 0.8f,
+                            0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f,  0.1f});
   AddInputFromArray<int>(TensorShape({}), {3});
   AddInputFromArray<int>(TensorShape({}), {3});
   AddInputFromArray<float>(TensorShape({}), {.5f});
   AddInputFromArray<float>(TensorShape({}), {0.8f});
   TF_ASSERT_OK(RunOpKernel());
-  
+
   // boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 3, 4}));
-  test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 
-          0, 0, 0, 0, 0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2, 0, 0, 0, 0});
+  test::FillValues<float>(&expected_boxes,
+                          {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 0, 0, 0, 0,
+                           0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2, 0, 0, 0, 0});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
   // scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 3}));
@@ -1189,29 +1193,29 @@ TEST_F(CombinedNonMaxSuppressionOpTest,
 }
 
 TEST_F(CombinedNonMaxSuppressionOpTest,
-        TestSelectFromTwoBatchesTwoClassesWithScoreThresholdPaddedTotalSize) {
+       TestSelectFromTwoBatchesTwoClassesWithScoreThresholdPaddedTotalSize) {
   MakeOp(true);
   AddInputFromArray<float>(
       TensorShape({2, 6, 1, 4}),
       {0, 0,    0.1, 0.1, 0, 0.01f, 0.1, 0.11f, 0, -0.01, 0.1, 0.09f,
-       0, 0.11, 0.1, 0.2, 0, 0.12f, 0.1, 0.21f, 0,  0.3,  1,   0.4,
+       0, 0.11, 0.1, 0.2, 0, 0.12f, 0.1, 0.21f, 0, 0.3,   1,   0.4,
        0, 0,    0.2, 0.2, 0, 0.02f, 0.2, 0.22f, 0, -0.02, 0.2, 0.19f,
-       0, 0.21, 0.2, 0.3, 0, 0.22f, 0.2, 0.31f, 0,  0.4,  1,   0.5
-       });
-  AddInputFromArray<float>(TensorShape({2, 6, 2}), 
-      {0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f, 
-       0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f
-       }); 
+       0, 0.21, 0.2, 0.3, 0, 0.22f, 0.2, 0.31f, 0, 0.4,   1,   0.5});
+  AddInputFromArray<float>(TensorShape({2, 6, 2}),
+                           {0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f,
+                            0.5f, 0.5f, 0.3f,  0.1f, 0.1f, 0.9f, 0.75f, 0.8f,
+                            0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f,  0.1f});
   AddInputFromArray<int>(TensorShape({}), {10});
   AddInputFromArray<int>(TensorShape({}), {3});
   AddInputFromArray<float>(TensorShape({}), {.5f});
   AddInputFromArray<float>(TensorShape({}), {0.8f});
   TF_ASSERT_OK(RunOpKernel());
-  
+
   // boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 3, 4}));
-  test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 
-          0, 0, 0, 0, 0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2, 0, 0, 0, 0});
+  test::FillValues<float>(&expected_boxes,
+                          {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 0, 0, 0, 0,
+                           0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2, 0, 0, 0, 0});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
   // scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 3}));
@@ -1232,19 +1236,18 @@ TEST_F(CombinedNonMaxSuppressionOpTest,
 }
 
 TEST_F(CombinedNonMaxSuppressionOpTest,
-        TestSelectFromTwoBatchesTwoClassesWithScoreThresholdPaddedPerClass) {
+       TestSelectFromTwoBatchesTwoClassesWithScoreThresholdPaddedPerClass) {
   MakeOp(true);
   AddInputFromArray<float>(
       TensorShape({2, 6, 1, 4}),
       {0, 0,    0.1, 0.1, 0, 0.01f, 0.1, 0.11f, 0, -0.01, 0.1, 0.09f,
-       0, 0.11, 0.1, 0.2, 0, 0.12f, 0.1, 0.21f, 0,  0.3,  1,   0.4,
+       0, 0.11, 0.1, 0.2, 0, 0.12f, 0.1, 0.21f, 0, 0.3,   1,   0.4,
        0, 0,    0.2, 0.2, 0, 0.02f, 0.2, 0.22f, 0, -0.02, 0.2, 0.19f,
-       0, 0.21, 0.2, 0.3, 0, 0.22f, 0.2, 0.31f, 0,  0.4,  1,   0.5
-       });
+       0, 0.21, 0.2, 0.3, 0, 0.22f, 0.2, 0.31f, 0, 0.4,   1,   0.5});
   AddInputFromArray<float>(TensorShape({2, 6, 2}),
-      {0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f,
-       0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f
-       });
+                           {0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f,
+                            0.5f, 0.5f, 0.3f,  0.1f, 0.1f, 0.9f, 0.75f, 0.8f,
+                            0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f,  0.1f});
   AddInputFromArray<int>(TensorShape({}), {2});
   AddInputFromArray<int>(TensorShape({}), {50});
   AddInputFromArray<float>(TensorShape({}), {.5f});
@@ -1253,9 +1256,10 @@ TEST_F(CombinedNonMaxSuppressionOpTest,
 
   // boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 4, 4}));
-  test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1,
-          0, 0, 0, 0, 0, 0, 0, 0, 0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2, 0, 0, 0, 0,
-          0, 0, 0, 0});
+  test::FillValues<float>(
+      &expected_boxes,
+      {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 0, 0, 0, 0, 0, 0, 0, 0,
+       0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2, 0, 0, 0, 0, 0, 0, 0, 0});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
   // scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 4}));
@@ -1276,37 +1280,37 @@ TEST_F(CombinedNonMaxSuppressionOpTest,
 }
 
 TEST_F(CombinedNonMaxSuppressionOpTest,
-        TestSelectFromTwoBatchesTwoClassesTotalSize) {
+       TestSelectFromTwoBatchesTwoClassesTotalSize) {
   MakeOp();
   AddInputFromArray<float>(
       TensorShape({2, 6, 1, 4}),
       {0, 0,    0.1, 0.1, 0, 0.01f, 0.1, 0.11f, 0, -0.01, 0.1, 0.09f,
-       0, 0.11, 0.1, 0.2, 0, 0.12f, 0.1, 0.21f, 0,  0.3,  1,   0.4,
+       0, 0.11, 0.1, 0.2, 0, 0.12f, 0.1, 0.21f, 0, 0.3,   1,   0.4,
        0, 0,    0.2, 0.2, 0, 0.02f, 0.2, 0.22f, 0, -0.02, 0.2, 0.19f,
-       0, 0.21, 0.2, 0.3, 0, 0.22f, 0.2, 0.31f, 0,  0.4,  1,   0.5
-       });
-  AddInputFromArray<float>(TensorShape({2, 6, 2}), 
-      {0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f, 
-       0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f
-       }); 
+       0, 0.21, 0.2, 0.3, 0, 0.22f, 0.2, 0.31f, 0, 0.4,   1,   0.5});
+  AddInputFromArray<float>(TensorShape({2, 6, 2}),
+                           {0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f,
+                            0.5f, 0.5f, 0.3f,  0.1f, 0.1f, 0.9f, 0.75f, 0.8f,
+                            0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f,  0.1f});
   AddInputFromArray<int>(TensorShape({}), {3});
   // Total size per batch is more than size per class
   AddInputFromArray<int>(TensorShape({}), {5});
   AddInputFromArray<float>(TensorShape({}), {.5f});
   AddInputFromArray<float>(TensorShape({}), {0.1f});
   TF_ASSERT_OK(RunOpKernel());
-  
+
   // boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 5, 4}));
-  test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 
-          0, 0.01f, 0.1, 0.11f, 0, 0.12f, 0.1, 0.21f, 0,  0.3, 1, 0.4,
-          0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2, 0, 0.02f, 0.2, 0.22f,
-          0, 0.22f, 0.2, 0.31f, 0, 0.4, 1, 0.5});
+  test::FillValues<float>(
+      &expected_boxes, {0,   0.11,  0.1, 0.2,   0,   0,     0.1, 0.1, 0, 0.01f,
+                        0.1, 0.11f, 0,   0.12f, 0.1, 0.21f, 0,   0.3, 1, 0.4,
+                        0,   0.21,  0.2, 0.3,   0,   0,     0.2, 0.2, 0, 0.02f,
+                        0.2, 0.22f, 0,   0.22f, 0.2, 0.31f, 0,   0.4, 1, 0.5});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
   // scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 5}));
-  test::FillValues<float>(&expected_scores, {0.95, 0.9, 0.75, 0.5, 0.3, 
-          0.95, 0.9, 0.75, 0.5, 0.3});
+  test::FillValues<float>(
+      &expected_scores, {0.95, 0.9, 0.75, 0.5, 0.3, 0.95, 0.9, 0.75, 0.5, 0.3});
   test::ExpectTensorEqual<float>(expected_scores, *GetOutput(1));
   // classes
   Tensor expected_classes(allocator(), DT_FLOAT, TensorShape({2, 5}));
@@ -1323,39 +1327,37 @@ TEST_F(CombinedNonMaxSuppressionOpTest,
 }
 
 TEST_F(CombinedNonMaxSuppressionOpTest,
-        TestSelectFromTwoBatchesTwoClassesForBoxesAndScores) {
+       TestSelectFromTwoBatchesTwoClassesForBoxesAndScores) {
   MakeOp();
   AddInputFromArray<float>(
       TensorShape({2, 6, 2, 4}),
       // batch 0, box1 of class 1 should get selected
-      {0, 0,     0.1, 0.1,   0, 0,     0.1, 0.1, 
-       0, 0.01f, 0.1, 0.11f, 0, 0.6f,  0.1, 0.7f,
-       0, -0.01, 0.1, 0.09f, 0, -0.01, 0.1, 0.09f, 
-       0, 0.11,  0.1, 0.2,   0, 0.11,  0.1, 0.2, 
-       0, 0.12f, 0.1, 0.21f, 0, 0.12f, 0.1, 0.21f, 
-       0, 0.3,   1,   0.4,   0, 0.3,   1,   0.4, 
-      // batch 1, box1 of class 0 should get selected
-       0, 0,     0.2, 0.2,   0, 0,     0.2, 0.2, 
-       0, 0.02f, 0.2, 0.22f, 0, 0.02f, 0.2, 0.22f,
-       0, -0.02, 0.2, 0.19f, 0, -0.02, 0.2, 0.19f,
-       0, 0.21,  0.2, 0.3,   0, 0.21,  0.2, 0.3, 
-       0, 0.22f, 0.2, 0.31f, 0, 0.22f, 0.2, 0.31f,
-       0,  0.4,  1,   0.5,   0, 0.4,   1,   0.5});
-
-  AddInputFromArray<float>(TensorShape({2, 6, 2}), 
-      {0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f, 
-       0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f, 0.1f
-       }); 
+      {0, 0, 0.1, 0.1, 0, 0, 0.1, 0.1, 0, 0.01f, 0.1, 0.11f, 0, 0.6f, 0.1, 0.7f,
+       0, -0.01, 0.1, 0.09f, 0, -0.01, 0.1, 0.09f, 0, 0.11, 0.1, 0.2, 0, 0.11,
+       0.1, 0.2, 0, 0.12f, 0.1, 0.21f, 0, 0.12f, 0.1, 0.21f, 0, 0.3, 1, 0.4, 0,
+       0.3, 1, 0.4,
+       // batch 1, box1 of class 0 should get selected
+       0, 0, 0.2, 0.2, 0, 0, 0.2, 0.2, 0, 0.02f, 0.2, 0.22f, 0, 0.02f, 0.2,
+       0.22f, 0, -0.02, 0.2, 0.19f, 0, -0.02, 0.2, 0.19f, 0, 0.21, 0.2, 0.3, 0,
+       0.21, 0.2, 0.3, 0, 0.22f, 0.2, 0.31f, 0, 0.22f, 0.2, 0.31f, 0, 0.4, 1,
+       0.5, 0, 0.4, 1, 0.5});
+
+  AddInputFromArray<float>(TensorShape({2, 6, 2}),
+                           {0.1f, 0.9f, 0.75f, 0.8f, 0.6f, 0.3f, 0.95f, 0.1f,
+                            0.5f, 0.5f, 0.3f,  0.1f, 0.1f, 0.9f, 0.75f, 0.8f,
+                            0.6f, 0.3f, 0.95f, 0.1f, 0.5f, 0.5f, 0.3f,  0.1f});
   AddInputFromArray<int>(TensorShape({}), {3});
   AddInputFromArray<int>(TensorShape({}), {3});
   AddInputFromArray<float>(TensorShape({}), {.5f});
   AddInputFromArray<float>(TensorShape({}), {0.0f});
   TF_ASSERT_OK(RunOpKernel());
-  
+
   // boxes
   Tensor expected_boxes(allocator(), DT_FLOAT, TensorShape({2, 3, 4}));
-  test::FillValues<float>(&expected_boxes, {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 
-           0, 0.6f, 0.1, 0.7f, 0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2, 0, 0.02f, 0.2, 0.22f});
+  test::FillValues<float>(
+      &expected_boxes,
+      {0, 0.11, 0.1, 0.2, 0, 0, 0.1, 0.1, 0, 0.6f,  0.1, 0.7f,
+       0, 0.21, 0.2, 0.3, 0, 0, 0.2, 0.2, 0, 0.02f, 0.2, 0.22f});
   test::ExpectTensorEqual<float>(expected_boxes, *GetOutput(0));
   // scores
   Tensor expected_scores(allocator(), DT_FLOAT, TensorShape({2, 3}));
diff --git a/tensorflow/core/ops/image_ops.cc b/tensorflow/core/ops/image_ops.cc
index 39263de6ba..f46fdb713b 100644
--- a/tensorflow/core/ops/image_ops.cc
+++ b/tensorflow/core/ops/image_ops.cc
@@ -132,12 +132,12 @@ Status NMSShapeFn(InferenceContext* c) {
 }
 
 Status CombinedNMSShapeFn(InferenceContext* c) {
-  //Get inputs and validate ranks
+  // Get inputs and validate ranks
   ShapeHandle boxes;
-  //boxes is a tensor of Dimensions [batch_size, num_anchors, q, 4]
+  // boxes is a tensor of Dimensions [batch_size, num_anchors, q, 4]
   TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 4, &boxes));
   ShapeHandle scores;
-  //scores is a tensor of Dimensions [batch_size, num_anchors, num_classes]
+  // scores is a tensor of Dimensions [batch_size, num_anchors, num_classes]
   TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 3, &scores));
   ShapeHandle max_output_size_per_class;
   TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &max_output_size_per_class));
@@ -148,9 +148,9 @@ Status CombinedNMSShapeFn(InferenceContext* c) {
   TF_RETURN_IF_ERROR(c->WithRank(c->input(5), 0, &unused_shape));
 
   DimensionHandle unused;
-  //boxes[0] and scores[0] are both batch_size
+  // boxes[0] and scores[0] are both batch_size
   TF_RETURN_IF_ERROR(c->Merge(c->Dim(boxes, 0), c->Dim(scores, 0), &unused));
-  //boxes[1] and scores[1] are both num_anchors
+  // boxes[1] and scores[1] are both num_anchors
   TF_RETURN_IF_ERROR(c->Merge(c->Dim(boxes, 1), c->Dim(scores, 1), &unused));
   // The boxes[3] is 4.
   TF_RETURN_IF_ERROR(c->WithValue(c->Dim(boxes, 3), 4, &unused));
@@ -159,8 +159,9 @@ Status CombinedNMSShapeFn(InferenceContext* c) {
   DimensionHandle class_dim = c->Dim(scores, 2);
   if (c->ValueKnown(d) && c->ValueKnown(class_dim)) {
     if (c->Value(d) != 1 && c->Value(d) != c->Value(class_dim)) {
-       return errors::InvalidArgument("third dimension of boxes must be either "
-           "1 or equal to the third dimension of scores");
+      return errors::InvalidArgument(
+          "third dimension of boxes must be either "
+          "1 or equal to the third dimension of scores");
     }
   }
   DimensionHandle output_dim;
@@ -177,15 +178,15 @@ Status CombinedNMSShapeFn(InferenceContext* c) {
   bool pad_per_class;
   TF_RETURN_IF_ERROR(c->GetAttr("pad_per_class", &pad_per_class));
   if (!pad_per_class) {
-    output_size =  c->Value(output_dim);
-  }
-  else {
+    output_size = c->Value(output_dim);
+  } else {
     if (c->ValueKnown(size_per_class) && c->Value(size_per_class) <= 0) {
-      return errors::InvalidArgument("max_output_size_per_class must be > 0 "
-             "if pad_per_class is set to true ");
+      return errors::InvalidArgument(
+          "max_output_size_per_class must be > 0 "
+          "if pad_per_class is set to true ");
     }
     output_size = std::min(c->Value(output_dim),
-        c->Value(size_per_class) * c->Value(class_dim));
+                           c->Value(size_per_class) * c->Value(class_dim));
   }
   c->set_output(0, c->MakeShape({batch_dim, output_size, 4}));
   c->set_output(1, c->MakeShape({batch_dim, output_size}));
@@ -195,7 +196,6 @@ Status CombinedNMSShapeFn(InferenceContext* c) {
   return Status::OK();
 }
 
-
 }  // namespace
 
 // --------------------------------------------------------------------------
diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index 3bf693daaf..4e73728b53 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -3223,7 +3223,7 @@ def combined_non_max_suppression(boxes,
   """Greedily selects a subset of bounding boxes in descending order of score.
 
   This operation performs non_max_suppression on the inputs per batch, across
-  all classes. 
+  all classes.
   Prunes away boxes that have high intersection-over-union (IOU) overlap
   with previously selected boxes.  Bounding boxes are supplied as
   [y1, x1, y2, x2], where (y1, x1) and (y2, x2) are the coordinates of any
@@ -3233,14 +3233,14 @@ def combined_non_max_suppression(boxes,
   this algorithm is invariant to orthogonal transformations and translations
   of the coordinate system; thus translating or reflections of the coordinate
   system result in the same boxes being selected by the algorithm.
-  The output of this operation is the final boxes, scores and classes tensor 
+  The output of this operation is the final boxes, scores and classes tensor
   returned after performing non_max_suppression.
 
   Args:
     boxes: A 4-D float `Tensor` of shape `[batch_size, num_boxes, q, 4]`.
-    scores: A 3-D float `Tensor` of shape `[batch_size, num_boxes, num_classes]` 
+    scores: A 3-D float `Tensor` of shape `[batch_size, num_boxes, num_classes]`
     representing a single score corresponding to each box (each row of boxes).
-    max_output_size_per_class: A scalar integer `Tensor` representing the 
+    max_output_size_per_class: A scalar integer `Tensor` representing the
     maximum number of boxes to be selected by non max suppression per class
     max_total_size: A scalar representing maximum number of boxes retained over
     all classes.
@@ -3274,5 +3274,8 @@ def combined_non_max_suppression(boxes,
     score_threshold = ops.convert_to_tensor(
         score_threshold, name='score_threshold')
     return gen_image_ops.combined_non_max_suppression(boxes, scores,
-            max_output_size_per_class, max_total_size, iou_threshold,
-            score_threshold, pad_per_class)
+                                                      max_output_size_per_class,
+                                                      max_total_size,
+                                                      iou_threshold,
+                                                      score_threshold,
+                                                      pad_per_class)
-- 
GitLab


From 6c335a51ece3821b4b4c8be37f502c0b57b0c747 Mon Sep 17 00:00:00 2001
From: Supriya Rao <supriyar@nvidia.com>
Date: Thu, 13 Dec 2018 13:13:16 -0800
Subject: [PATCH 1506/2345] Added api_def_CombinedNonMaxSuppression.pbtxt based
 on test requirement in //tensorflow/core/api_def:api_test

---
 .../api_def_CombinedNonMaxSuppression.pbtxt   | 106 ++++++++++++++++++
 tensorflow/python/ops/image_ops_impl.py       |   6 +-
 2 files changed, 109 insertions(+), 3 deletions(-)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_CombinedNonMaxSuppression.pbtxt

diff --git a/tensorflow/core/api_def/base_api/api_def_CombinedNonMaxSuppression.pbtxt b/tensorflow/core/api_def/base_api/api_def_CombinedNonMaxSuppression.pbtxt
new file mode 100644
index 0000000000..e2b6ed8248
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_CombinedNonMaxSuppression.pbtxt
@@ -0,0 +1,106 @@
+op {
+  graph_op_name: "CombinedNonMaxSuppression"
+  in_arg {
+    name: "boxes"
+    description: <<END
+A 4-D float tensor of shape `[batch_size, num_boxes, q, 4]`.
+END
+  }
+  in_arg {
+    name: "scores"
+    description: <<END
+A 3-D float tensor of shape `[batch_size, num_boxes, num_classes]`
+representing a single score corresponding to each box (each row of boxes).
+END
+  }
+  in_arg {
+    name: "max_output_size_per_class"
+    description: <<END
+A scalar integer tensor representing the maximum number of 
+boxes to be selected by non max suppression per class
+END
+  }
+  in_arg {
+    name: "max_total_size"
+    description: <<END
+A scalar representing maximum number of boxes retained over all classes.
+END
+  }
+  in_arg {
+    name: "iou_threshold"
+    description: <<END
+A 0-D float tensor representing the threshold for deciding whether
+boxes overlap too much with respect to IOU.
+END
+  }
+  in_arg {
+    name: "score_threshold"
+    description: <<END
+A 0-D float tensor representing the threshold for deciding when to remove
+boxes based on score.
+END
+  }
+  attr {
+    name: "pad_per_class"
+    description: <<END
+If false, the output nmsed boxes, scores, classes and
+selected_indices are padded/clipped to `max_total_size`. If true, the
+output nmsed boxes, scores, classes and selected_indices are
+padded/clipped to be of length `max_size_per_class` or `max_total_size`
+(smaller of the two). Defaults to false.
+END
+  }
+  out_arg {
+    name: "nmsed_boxes"
+    description: <<END
+A [batch_size, max_detections, 4] float32 tensor 
+containing the non-max suppressed boxes.
+END
+  }
+  out_arg {
+    name: "nmsed_scores"
+    description: <<END
+A [batch_size, max_detections] float32 tensor 
+containing the scores for the boxes.
+END
+  }
+  out_arg {
+    name: "nmsed_classes"
+    description: <<END
+A [batch_size, max_detections] float32 tensor 
+containing the classes for the boxes.
+END
+  }
+  out_arg {
+    name: "valid_detections"
+    description: <<END
+A [batch_size] int32 tensor indicating the number of
+valid detections per batch item. Only the top num_detections[i] entries in
+nms_boxes[i], nms_scores[i] and nms_class[i] are valid. The rest of the
+entries are zero paddings.
+END
+  }
+  out_arg {
+    name: "selected_indices"
+    description: <<END
+A [batch_size, max_detections] int32 tensor
+containing the indices of the selected boxes
+END
+  }
+  summary: "Greedily selects a subset of bounding boxes in descending order of score,"
+  description: <<END
+This operation performs non_max_suppression on the inputs per batch, across
+all classes.
+Prunes away boxes that have high intersection-over-union (IOU) overlap
+with previously selected boxes.  Bounding boxes are supplied as
+[y1, x1, y2, x2], where (y1, x1) and (y2, x2) are the coordinates of any
+diagonal pair of box corners and the coordinates can be provided as normalized
+(i.e., lying in the interval [0, 1]) or absolute.  Note that this algorithm
+is agnostic to where the origin is in the coordinate system. Also note that
+this algorithm is invariant to orthogonal transformations and translations
+of the coordinate system; thus translating or reflections of the coordinate
+system result in the same boxes being selected by the algorithm.
+The output of this operation is the final boxes, scores and classes tensor
+returned after performing non_max_suppression.
+END
+}
diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index 4e73728b53..583522ea54 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -3262,9 +3262,9 @@ def combined_non_max_suppression(boxes,
       the scores for the boxes.
     'nmsed_classes': A [batch_size, max_detections] float32 tensor
       containing the class for boxes.
-    'num_detections': A [batch_size] int32 tensor indicating the number of
-      valid detections per batch item. Only the top num_detections[i] entries in
-      nms_boxes[i], nms_scores[i] and nms_class[i] are valid. The rest of the
+    'valid_detections': A [batch_size] int32 tensor indicating the number of
+      valid detections per batch item. Only the top valid_detections[i] entries
+      in nms_boxes[i], nms_scores[i] and nms_class[i] are valid. The rest of the
       entries are zero paddings.
     'selected_indices':  A [batch_size, max_detections] int32 tensor
       containing the indices of the selected boxes
-- 
GitLab


From ef532283d7a0ba3e1e565b83b1572a414abdf5b0 Mon Sep 17 00:00:00 2001
From: Supriya Rao <supriyar@nvidia.com>
Date: Thu, 13 Dec 2018 13:40:54 -0800
Subject: [PATCH 1507/2345] Added a comment describing 'q'

---
 .../api_def/base_api/api_def_CombinedNonMaxSuppression.pbtxt  | 4 +++-
 tensorflow/python/ops/image_ops_impl.py                       | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/api_def/base_api/api_def_CombinedNonMaxSuppression.pbtxt b/tensorflow/core/api_def/base_api/api_def_CombinedNonMaxSuppression.pbtxt
index e2b6ed8248..a635b8b164 100644
--- a/tensorflow/core/api_def/base_api/api_def_CombinedNonMaxSuppression.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_CombinedNonMaxSuppression.pbtxt
@@ -3,7 +3,9 @@ op {
   in_arg {
     name: "boxes"
     description: <<END
-A 4-D float tensor of shape `[batch_size, num_boxes, q, 4]`.
+A 4-D float tensor of shape `[batch_size, num_boxes, q, 4]`. If `q` is 1 then 
+same boxes are used for all classes otherwise, if `q` is equal to number of 
+classes, class-specific boxes are used.
 END
   }
   in_arg {
diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index 583522ea54..80a9eae529 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -3237,7 +3237,9 @@ def combined_non_max_suppression(boxes,
   returned after performing non_max_suppression.
 
   Args:
-    boxes: A 4-D float `Tensor` of shape `[batch_size, num_boxes, q, 4]`.
+    boxes: A 4-D float `Tensor` of shape `[batch_size, num_boxes, q, 4]`. If `q`
+    is 1 then same boxes are used for all classes otherwise, if `q` is equal to
+    number of classes, class-specific boxes are used.
     scores: A 3-D float `Tensor` of shape `[batch_size, num_boxes, num_classes]`
     representing a single score corresponding to each box (each row of boxes).
     max_output_size_per_class: A scalar integer `Tensor` representing the
-- 
GitLab


From b1262df96398822f04223bc43ef4f586126aecb4 Mon Sep 17 00:00:00 2001
From: Supriya Rao <supriyar@nvidia.com>
Date: Thu, 13 Dec 2018 18:33:46 -0800
Subject: [PATCH 1508/2345] Add float dtype to the python API

---
 tensorflow/python/ops/image_ops_impl.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index 80a9eae529..321cedc8c2 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -3272,9 +3272,10 @@ def combined_non_max_suppression(boxes,
       containing the indices of the selected boxes
   """
   with ops.name_scope(name, 'combined_non_max_suppression'):
-    iou_threshold = ops.convert_to_tensor(iou_threshold, name='iou_threshold')
+    iou_threshold = ops.convert_to_tensor(
+        iou_threshold, dtype=dtypes.float32, name='iou_threshold')
     score_threshold = ops.convert_to_tensor(
-        score_threshold, name='score_threshold')
+        score_threshold, dtype=dtypes.float32, name='score_threshold')
     return gen_image_ops.combined_non_max_suppression(boxes, scores,
                                                       max_output_size_per_class,
                                                       max_total_size,
-- 
GitLab


From 26a337d86a40bfd7530bafcbef670ffbe7e8e26e Mon Sep 17 00:00:00 2001
From: Pooya Davoodi <pdavoodi@nvidia.com>
Date: Tue, 8 Jan 2019 14:22:53 -0800
Subject: [PATCH 1509/2345] Change golden APIs

---
 tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt | 4 ++++
 tensorflow/tools/api/golden/v2/tensorflow.pbtxt       | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
index 8dc2ae3a5c..80c0cb7ae8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
@@ -32,6 +32,10 @@ tf_module {
     name: "central_crop"
     argspec: "args=[\'image\', \'central_fraction\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "combined_non_max_suppression"
+    argspec: "args=[\'boxes\', \'scores\', \'max_output_size_per_class\', \'max_total_size\', \'iou_threshold\', \'score_threshold\', \'pad_per_class\', \'name\'], varargs=None, keywords=None, defaults=[\'0.5\', \'-inf\', \'False\', \'None\'], "
+  }
   member_method {
     name: "convert_image_dtype"
     argspec: "args=[\'image\', \'dtype\', \'saturate\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 4af4b7ef55..a9d6799d69 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -468,6 +468,10 @@ tf_module {
     name: "clip_by_value"
     argspec: "args=[\'t\', \'clip_value_min\', \'clip_value_max\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "combined_non_max_suppression"
+    argspec: "args=[\'boxes\', \'scores\', \'max_output_size_per_class\', \'max_total_size\', \'iou_threshold\', \'score_threshold\', \'pad_per_class\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
+  }
   member_method {
     name: "complex"
     argspec: "args=[\'real\', \'imag\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-- 
GitLab


From 9ec17972c8c32e2c737b2fe033a2889e7c3553d2 Mon Sep 17 00:00:00 2001
From: Supriya Rao <supriyar@nvidia.com>
Date: Thu, 17 Jan 2019 11:30:48 -0800
Subject: [PATCH 1510/2345] Removed selected_indices return value for NMS op

---
 .../api_def_CombinedNonMaxSuppression.pbtxt   | 13 ++---
 .../core/kernels/non_max_suppression_op.cc    |  9 ----
 .../kernels/non_max_suppression_op_test.cc    | 49 -------------------
 tensorflow/core/ops/image_ops.cc              |  1 -
 tensorflow/python/ops/image_ops_impl.py       | 12 ++---
 5 files changed, 8 insertions(+), 76 deletions(-)

diff --git a/tensorflow/core/api_def/base_api/api_def_CombinedNonMaxSuppression.pbtxt b/tensorflow/core/api_def/base_api/api_def_CombinedNonMaxSuppression.pbtxt
index a635b8b164..3291a85c8a 100644
--- a/tensorflow/core/api_def/base_api/api_def_CombinedNonMaxSuppression.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_CombinedNonMaxSuppression.pbtxt
@@ -45,9 +45,9 @@ END
   attr {
     name: "pad_per_class"
     description: <<END
-If false, the output nmsed boxes, scores, classes and
-selected_indices are padded/clipped to `max_total_size`. If true, the
-output nmsed boxes, scores, classes and selected_indices are
+If false, the output nmsed boxes, scores and classes
+are padded/clipped to `max_total_size`. If true, the
+output nmsed boxes, scores and classes are
 padded/clipped to be of length `max_size_per_class` or `max_total_size`
 (smaller of the two). Defaults to false.
 END
@@ -80,13 +80,6 @@ A [batch_size] int32 tensor indicating the number of
 valid detections per batch item. Only the top num_detections[i] entries in
 nms_boxes[i], nms_scores[i] and nms_class[i] are valid. The rest of the
 entries are zero paddings.
-END
-  }
-  out_arg {
-    name: "selected_indices"
-    description: <<END
-A [batch_size, max_detections] int32 tensor
-containing the indices of the selected boxes
 END
   }
   summary: "Greedily selects a subset of bounding boxes in descending order of score,"
diff --git a/tensorflow/core/kernels/non_max_suppression_op.cc b/tensorflow/core/kernels/non_max_suppression_op.cc
index cc525331f2..f7dd41c8f7 100644
--- a/tensorflow/core/kernels/non_max_suppression_op.cc
+++ b/tensorflow/core/kernels/non_max_suppression_op.cc
@@ -243,8 +243,6 @@ void BatchedNonMaxSuppressionOp(
   std::vector<std::vector<float>> nmsed_classes(num_batches);
   // [num_batches]
   std::vector<int> final_valid_detections;
-  // [num_batches, per_batch_size]
-  std::vector<std::vector<int>> selected_indices(num_batches);
 
   int per_batch_size = total_size_per_batch;
 
@@ -389,14 +387,12 @@ void BatchedNonMaxSuppressionOp(
                    clip_window[1]));
       nmsed_scores[batch].push_back(next_candidate.score);
       nmsed_classes[batch].push_back(next_candidate.class_idx);
-      selected_indices[batch].push_back(next_candidate.box_index);
       curr_total_size--;
     }
 
     nmsed_boxes[batch].resize(per_batch_size * 4, 0);
     nmsed_scores[batch].resize(per_batch_size, 0);
     nmsed_classes[batch].resize(per_batch_size, 0);
-    selected_indices[batch].resize(per_batch_size, 0);
   }
 
   Tensor* nmsed_boxes_t = nullptr;
@@ -422,16 +418,11 @@ void BatchedNonMaxSuppressionOp(
                                                    &valid_detections_t));
   auto valid_detections_flat = valid_detections_t->template flat<int>();
 
-  Tensor* selected_indices_t = nullptr;
-  OP_REQUIRES_OK(
-      context, context->allocate_output(4, scores_shape, &selected_indices_t));
-  auto selected_indices_flat = selected_indices_t->template flat<int>();
   for (int i = 0; i < num_batches; ++i) {
     valid_detections_flat(i) = final_valid_detections[i];
     for (int j = 0; j < per_batch_size; ++j) {
       nmsed_scores_flat(i * per_batch_size + j) = nmsed_scores[i][j];
       nmsed_classes_flat(i * per_batch_size + j) = nmsed_classes[i][j];
-      selected_indices_flat(i * per_batch_size + j) = selected_indices[i][j];
       for (int k = 0; k < 4; ++k) {
         nmsed_boxes_flat(i * per_batch_size * 4 + j * 4 + k) =
             nmsed_boxes[i][j * 4 + k];
diff --git a/tensorflow/core/kernels/non_max_suppression_op_test.cc b/tensorflow/core/kernels/non_max_suppression_op_test.cc
index f5c3bc64e1..242e41b265 100644
--- a/tensorflow/core/kernels/non_max_suppression_op_test.cc
+++ b/tensorflow/core/kernels/non_max_suppression_op_test.cc
@@ -907,11 +907,6 @@ TEST_F(CombinedNonMaxSuppressionOpTest, TestEmptyInput) {
   Tensor expected_valid_d(allocator(), DT_INT32, TensorShape({0}));
   test::FillValues<int>(&expected_valid_d, {});
   test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
-
-  // indices
-  Tensor expected_indices(allocator(), DT_INT32, TensorShape({0, 10}));
-  test::FillValues<int>(&expected_indices, {});
-  test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
 
 TEST_F(CombinedNonMaxSuppressionOpTest, TestSelectFromThreeClusters) {
@@ -945,10 +940,6 @@ TEST_F(CombinedNonMaxSuppressionOpTest, TestSelectFromThreeClusters) {
   Tensor expected_valid_d(allocator(), DT_INT32, TensorShape({1}));
   test::FillValues<int>(&expected_valid_d, {3});
   test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
-  // indices
-  Tensor expected_indices(allocator(), DT_INT32, TensorShape({1, 3}));
-  test::FillValues<int>(&expected_indices, {3, 0, 5});
-  test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
 
 TEST_F(CombinedNonMaxSuppressionOpTest,
@@ -983,10 +974,6 @@ TEST_F(CombinedNonMaxSuppressionOpTest,
   Tensor expected_valid_d(allocator(), DT_INT32, TensorShape({1}));
   test::FillValues<int>(&expected_valid_d, {2});
   test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
-  // indices
-  Tensor expected_indices(allocator(), DT_INT32, TensorShape({1, 3}));
-  test::FillValues<int>(&expected_indices, {3, 0, 0});
-  test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
 
 TEST_F(CombinedNonMaxSuppressionOpTest,
@@ -1026,10 +1013,6 @@ TEST_F(CombinedNonMaxSuppressionOpTest,
   Tensor expected_valid_d(allocator(), DT_INT32, TensorShape({1}));
   test::FillValues<int>(&expected_valid_d, {2});
   test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
-  // indices
-  Tensor expected_indices(allocator(), DT_INT32, TensorShape({1, 5}));
-  test::FillValues<int>(&expected_indices, {3, 0, 0, 0, 0});
-  test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
 
 TEST_F(CombinedNonMaxSuppressionOpTest, TestSelectSingleBox) {
@@ -1058,10 +1041,6 @@ TEST_F(CombinedNonMaxSuppressionOpTest, TestSelectSingleBox) {
   Tensor expected_valid_d(allocator(), DT_INT32, TensorShape({1}));
   test::FillValues<int>(&expected_valid_d, {1});
   test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
-  // indices
-  Tensor expected_indices(allocator(), DT_INT32, TensorShape({1, 1}));
-  test::FillValues<int>(&expected_indices, {0});
-  test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
 
 TEST_F(CombinedNonMaxSuppressionOpTest,
@@ -1100,10 +1079,6 @@ TEST_F(CombinedNonMaxSuppressionOpTest,
   Tensor expected_valid_d(allocator(), DT_INT32, TensorShape({2}));
   test::FillValues<int>(&expected_valid_d, {2, 2});
   test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
-  // indices
-  Tensor expected_indices(allocator(), DT_INT32, TensorShape({2, 3}));
-  test::FillValues<int>(&expected_indices, {3, 0, 0, 3, 0, 0});
-  test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
 
 TEST_F(CombinedNonMaxSuppressionOpTest, TestSelectFromTwoBatchesTwoClasses) {
@@ -1143,10 +1118,6 @@ TEST_F(CombinedNonMaxSuppressionOpTest, TestSelectFromTwoBatchesTwoClasses) {
   Tensor expected_valid_d(allocator(), DT_INT32, TensorShape({2}));
   test::FillValues<int>(&expected_valid_d, {3, 3});
   test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
-  // indices
-  Tensor expected_indices(allocator(), DT_INT32, TensorShape({2, 3}));
-  test::FillValues<int>(&expected_indices, {3, 0, 1, 3, 0, 1});
-  test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
 
 TEST_F(CombinedNonMaxSuppressionOpTest,
@@ -1186,10 +1157,6 @@ TEST_F(CombinedNonMaxSuppressionOpTest,
   Tensor expected_valid_d(allocator(), DT_INT32, TensorShape({2}));
   test::FillValues<int>(&expected_valid_d, {2, 2});
   test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
-  // indices
-  Tensor expected_indices(allocator(), DT_INT32, TensorShape({2, 3}));
-  test::FillValues<int>(&expected_indices, {3, 0, 0, 3, 0, 0});
-  test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
 
 TEST_F(CombinedNonMaxSuppressionOpTest,
@@ -1229,10 +1196,6 @@ TEST_F(CombinedNonMaxSuppressionOpTest,
   Tensor expected_valid_d(allocator(), DT_INT32, TensorShape({2}));
   test::FillValues<int>(&expected_valid_d, {2, 2});
   test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
-  // indices
-  Tensor expected_indices(allocator(), DT_INT32, TensorShape({2, 3}));
-  test::FillValues<int>(&expected_indices, {3, 0, 0, 3, 0, 0});
-  test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
 
 TEST_F(CombinedNonMaxSuppressionOpTest,
@@ -1273,10 +1236,6 @@ TEST_F(CombinedNonMaxSuppressionOpTest,
   Tensor expected_valid_d(allocator(), DT_INT32, TensorShape({2}));
   test::FillValues<int>(&expected_valid_d, {2, 2});
   test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
-  // indices
-  Tensor expected_indices(allocator(), DT_INT32, TensorShape({2, 4}));
-  test::FillValues<int>(&expected_indices, {3, 0, 0, 0, 3, 0, 0, 0});
-  test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
 
 TEST_F(CombinedNonMaxSuppressionOpTest,
@@ -1320,10 +1279,6 @@ TEST_F(CombinedNonMaxSuppressionOpTest,
   Tensor expected_valid_d(allocator(), DT_INT32, TensorShape({2}));
   test::FillValues<int>(&expected_valid_d, {5, 5});
   test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
-  // indices
-  Tensor expected_indices(allocator(), DT_INT32, TensorShape({2, 5}));
-  test::FillValues<int>(&expected_indices, {3, 0, 1, 4, 5, 3, 0, 1, 4, 5});
-  test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
 
 TEST_F(CombinedNonMaxSuppressionOpTest,
@@ -1371,10 +1326,6 @@ TEST_F(CombinedNonMaxSuppressionOpTest,
   Tensor expected_valid_d(allocator(), DT_INT32, TensorShape({2}));
   test::FillValues<int>(&expected_valid_d, {3, 3});
   test::ExpectTensorEqual<int>(expected_valid_d, *GetOutput(3));
-  // indices
-  Tensor expected_indices(allocator(), DT_INT32, TensorShape({2, 3}));
-  test::FillValues<int>(&expected_indices, {3, 0, 1, 3, 0, 1});
-  test::ExpectTensorEqual<int>(expected_indices, *GetOutput(4));
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/ops/image_ops.cc b/tensorflow/core/ops/image_ops.cc
index f46fdb713b..e6aae16f01 100644
--- a/tensorflow/core/ops/image_ops.cc
+++ b/tensorflow/core/ops/image_ops.cc
@@ -895,7 +895,6 @@ REGISTER_OP("CombinedNonMaxSuppression")
     .Output("nmsed_scores: float")
     .Output("nmsed_classes: float")
     .Output("valid_detections: int32")
-    .Output("selected_indices: int32")
     .Attr("pad_per_class: bool = false")
     .SetShapeFn(CombinedNMSShapeFn);
 
diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index 321cedc8c2..f9a9b80e78 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -3250,11 +3250,11 @@ def combined_non_max_suppression(boxes,
       overlap too much with respect to IOU.
     score_threshold: A float representing the threshold for deciding when to
       remove boxes based on score.
-    pad_per_class: If false, the output nmsed boxes, scores, classes and
-      selected_indices are padded/clipped to `max_total_size`. If true, the
-      output nmsed boxes, scores, classes and selected_indices are
-      padded/clipped to be of length `max_size_per_class` or `max_total_size`
-      (smaller of the two). Defaults to false.
+    pad_per_class: If false, the output nmsed boxes, scores and classes
+      are padded/clipped to `max_total_size`. If true, the
+      output nmsed boxes, scores and classes are padded/clipped to be of length
+      `max_size_per_class` or `max_total_size` (smaller of the two).
+      Defaults to false.
     name: A name for the operation (optional).
 
   Returns:
@@ -3268,8 +3268,6 @@ def combined_non_max_suppression(boxes,
       valid detections per batch item. Only the top valid_detections[i] entries
       in nms_boxes[i], nms_scores[i] and nms_class[i] are valid. The rest of the
       entries are zero paddings.
-    'selected_indices':  A [batch_size, max_detections] int32 tensor
-      containing the indices of the selected boxes
   """
   with ops.name_scope(name, 'combined_non_max_suppression'):
     iou_threshold = ops.convert_to_tensor(
-- 
GitLab


From fbe21f4bf1e3e0aae115d7d94c23d97559cbc203 Mon Sep 17 00:00:00 2001
From: Supriya Rao <supriyar@nvidia.com>
Date: Thu, 17 Jan 2019 14:58:53 -0800
Subject: [PATCH 1511/2345] Update comment for pad_per_class

---
 .../base_api/api_def_CombinedNonMaxSuppression.pbtxt        | 6 +++---
 tensorflow/python/ops/image_ops_impl.py                     | 5 +++--
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/tensorflow/core/api_def/base_api/api_def_CombinedNonMaxSuppression.pbtxt b/tensorflow/core/api_def/base_api/api_def_CombinedNonMaxSuppression.pbtxt
index 3291a85c8a..50e72a2446 100644
--- a/tensorflow/core/api_def/base_api/api_def_CombinedNonMaxSuppression.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_CombinedNonMaxSuppression.pbtxt
@@ -47,9 +47,9 @@ END
     description: <<END
 If false, the output nmsed boxes, scores and classes
 are padded/clipped to `max_total_size`. If true, the
-output nmsed boxes, scores and classes are
-padded/clipped to be of length `max_size_per_class` or `max_total_size`
-(smaller of the two). Defaults to false.
+output nmsed boxes, scores and classes are padded to be of length
+`max_size_per_class`*`num_classes`, unless it exceeds `max_total_size` in
+which case it is clipped to `max_total_size`. Defaults to false.
 END
   }
   out_arg {
diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index f9a9b80e78..8026da112c 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -3252,8 +3252,9 @@ def combined_non_max_suppression(boxes,
       remove boxes based on score.
     pad_per_class: If false, the output nmsed boxes, scores and classes
       are padded/clipped to `max_total_size`. If true, the
-      output nmsed boxes, scores and classes are padded/clipped to be of length
-      `max_size_per_class` or `max_total_size` (smaller of the two).
+      output nmsed boxes, scores and classes are padded to be of length
+      `max_size_per_class`*`num_classes`, unless it exceeds `max_total_size` in
+      which case it is clipped to `max_total_size`.
       Defaults to false.
     name: A name for the operation (optional).
 
-- 
GitLab


From b1ee8b90be335158efa4c954b1537d46d953459f Mon Sep 17 00:00:00 2001
From: Pooya Davoodi <pdavoodi@nvidia.com>
Date: Thu, 24 Jan 2019 13:37:08 -0800
Subject: [PATCH 1512/2345] Use std::vector::reserve to optimize allocation

---
 tensorflow/core/kernels/non_max_suppression_op.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/core/kernels/non_max_suppression_op.cc b/tensorflow/core/kernels/non_max_suppression_op.cc
index f7dd41c8f7..482b227ccd 100644
--- a/tensorflow/core/kernels/non_max_suppression_op.cc
+++ b/tensorflow/core/kernels/non_max_suppression_op.cc
@@ -268,7 +268,9 @@ void BatchedNonMaxSuppressionOp(
     // Iterate through all classes
     for (int class_idx = 0; class_idx < num_classes; ++class_idx) {
       std::vector<float> class_scores_data;
+      class_scores_data.reserve(num_boxes);
       std::vector<float> class_boxes_data;
+      class_boxes_data.reserve(num_boxes * 4);
 
       for (int box = 0; box < num_boxes; ++box) {
         // Get the scores per class
-- 
GitLab


From bc857b4c5424ca6e72b58d0700b7c8f57c6a5a5d Mon Sep 17 00:00:00 2001
From: Pooya Davoodi <pdavoodi@nvidia.com>
Date: Fri, 25 Jan 2019 17:29:33 -0800
Subject: [PATCH 1513/2345] Change golden APIs for V1

---
 tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt | 4 ++++
 tensorflow/tools/api/golden/v1/tensorflow.pbtxt       | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt
index fd6bbcd53a..e74a74030f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt
@@ -32,6 +32,10 @@ tf_module {
     name: "central_crop"
     argspec: "args=[\'image\', \'central_fraction\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "combined_non_max_suppression"
+    argspec: "args=[\'boxes\', \'scores\', \'max_output_size_per_class\', \'max_total_size\', \'iou_threshold\', \'score_threshold\', \'pad_per_class\', \'name\'], varargs=None, keywords=None, defaults=[\'0.5\', \'-inf\', \'False\', \'None\'], "
+  }
   member_method {
     name: "convert_image_dtype"
     argspec: "args=[\'image\', \'dtype\', \'saturate\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
index 0276a81473..8b014a73a0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
@@ -932,6 +932,10 @@ tf_module {
     name: "clip_by_value"
     argspec: "args=[\'t\', \'clip_value_min\', \'clip_value_max\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "combined_non_max_suppression"
+    argspec: "args=[\'boxes\', \'scores\', \'max_output_size_per_class\', \'max_total_size\', \'iou_threshold\', \'score_threshold\', \'pad_per_class\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
+  }
   member_method {
     name: "colocate_with"
     argspec: "args=[\'op\', \'ignore_existing\'], varargs=None, keywords=None, defaults=[\'False\'], "
-- 
GitLab


From 3115b4513a898ba7a86b0955850b1cb87abce1ac Mon Sep 17 00:00:00 2001
From: Pooya Davoodi <pdavoodi@nvidia.com>
Date: Mon, 28 Jan 2019 10:18:38 -0800
Subject: [PATCH 1514/2345] Move added NMS op in V1 golden API to keep the list
 sorted

---
 tensorflow/tools/api/golden/v1/tensorflow.pbtxt | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
index 8b014a73a0..a4b46d4260 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
@@ -932,14 +932,14 @@ tf_module {
     name: "clip_by_value"
     argspec: "args=[\'t\', \'clip_value_min\', \'clip_value_max\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "combined_non_max_suppression"
-    argspec: "args=[\'boxes\', \'scores\', \'max_output_size_per_class\', \'max_total_size\', \'iou_threshold\', \'score_threshold\', \'pad_per_class\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
-  }
   member_method {
     name: "colocate_with"
     argspec: "args=[\'op\', \'ignore_existing\'], varargs=None, keywords=None, defaults=[\'False\'], "
   }
+  member_method {
+    name: "combined_non_max_suppression"
+    argspec: "args=[\'boxes\', \'scores\', \'max_output_size_per_class\', \'max_total_size\', \'iou_threshold\', \'score_threshold\', \'pad_per_class\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
+  }
   member_method {
     name: "complex"
     argspec: "args=[\'real\', \'imag\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-- 
GitLab


From 2d3fa90c47af1fd0d893a837f6c50d311060cd58 Mon Sep 17 00:00:00 2001
From: Pooya Davoodi <pdavoodi@nvidia.com>
Date: Mon, 28 Jan 2019 13:40:25 -0800
Subject: [PATCH 1515/2345] Do not set 5th output in CombinedNMSShapeFn

We already removed the 5th output from the op definition. This part
was missed.
---
 tensorflow/core/ops/image_ops.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/core/ops/image_ops.cc b/tensorflow/core/ops/image_ops.cc
index e6aae16f01..f451987a9b 100644
--- a/tensorflow/core/ops/image_ops.cc
+++ b/tensorflow/core/ops/image_ops.cc
@@ -192,7 +192,6 @@ Status CombinedNMSShapeFn(InferenceContext* c) {
   c->set_output(1, c->MakeShape({batch_dim, output_size}));
   c->set_output(2, c->MakeShape({batch_dim, output_size}));
   c->set_output(3, c->Vector(batch_dim));
-  c->set_output(4, c->MakeShape({batch_dim, output_size}));
   return Status::OK();
 }
 
-- 
GitLab


From dcea1dde241d41010bd6e5073563e16243a031fa Mon Sep 17 00:00:00 2001
From: Yanan Cao <ycao@google.com>
Date: Tue, 29 Jan 2019 12:45:25 -0800
Subject: [PATCH 1516/2345] Internal change

PiperOrigin-RevId: 231454603
---
 tensorflow/python/kernel_tests/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index aef4354ee0..8fccb0f141 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -521,6 +521,7 @@ tf_py_test(
         "//tensorflow/python:io_ops",
         "//tensorflow/python:util",
     ],
+    tags = ["notap"],  #TODO(b/123583863): Re-enable this test.
 )
 
 tf_py_test(
-- 
GitLab


From e6b2a5743bf4db9d2579e520da5ad83e5f9fa0c5 Mon Sep 17 00:00:00 2001
From: Deven Desai <deven.desai.amd@gmail.com>
Date: Tue, 29 Jan 2019 20:55:46 +0000
Subject: [PATCH 1517/2345] fix for the compiler error with --config=rocm

---
 tensorflow/stream_executor/rocm/rocm_gpu_executor.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/stream_executor/rocm/rocm_gpu_executor.cc b/tensorflow/stream_executor/rocm/rocm_gpu_executor.cc
index 249c7870e7..4087262620 100644
--- a/tensorflow/stream_executor/rocm/rocm_gpu_executor.cc
+++ b/tensorflow/stream_executor/rocm/rocm_gpu_executor.cc
@@ -880,7 +880,11 @@ DeviceDescription* GpuExecutor::PopulateDeviceDescription() const {
     (void)GpuDriver::GetDriverVersion(&driver_version);
     string augmented_driver_version = absl::StrFormat(
         "%d (%s)", driver_version,
-        DriverVersionStatusToString(Diagnostician::FindDsoVersion()).c_str());
+	"__FIXME__");
+	// FIXME:
+	// uncomment the line below once the "DriverVersionStatusToString"
+	// routine is moved from the "cuda" namespace to the "gpu" naemspace
+        // DriverVersionStatusToString(Diagnostician::FindDsoVersion()).c_str());
     builder.set_driver_version(augmented_driver_version);
   }
 
-- 
GitLab


From e91db6cebef4eb78377c87a6c664d2909c0653ae Mon Sep 17 00:00:00 2001
From: Yanan Cao <ycao@google.com>
Date: Tue, 29 Jan 2019 12:48:31 -0800
Subject: [PATCH 1518/2345] Internal change

PiperOrigin-RevId: 231455101
---
 tensorflow/python/keras/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 108187aafe..ca01d85465 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -731,7 +731,7 @@ cuda_py_test(
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
     ],
-    shard_count = 6,
+    shard_count = 8,
 )
 
 tf_py_test(
-- 
GitLab


From 8a1dcea0acb7fde84b43d7c922b2b1bffd6824d8 Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Tue, 29 Jan 2019 12:56:08 -0800
Subject: [PATCH 1519/2345] For the multi device iterator, making sure that the
 prefetch_buffer_size doesn't exceed the max_buffer_size.

This was causing deadlocks during shutdown when one iterator (corresponding to say device0) would get destroyed and so that wouldn't be requesting any more data. As a result the MultiDeviceIterator would get blocked on device0 now. device1 might still be alive and requesting data and now its blocked on device0 to fetch (which it won't anymore since its destroyed).

PiperOrigin-RevId: 231456490
---
 .../python/data/kernel_tests/multi_device_iterator_test.py | 3 +--
 tensorflow/python/data/ops/multi_device_iterator_ops.py    | 7 +++++++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
index 110e3b9660..e6dfc338eb 100644
--- a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
+++ b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
@@ -263,7 +263,6 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase,
 
   @test_util.run_v1_only("b/121264236")
   def testMultipleInitializationsEager(self):
-    self.skipTest("b/123023614")
     if not context.executing_eagerly():
       return
 
@@ -272,7 +271,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase,
       dataset2 = dataset_ops.Dataset.range(1000)
       dataset = dataset_ops.Dataset.zip((dataset1, dataset2))
 
-    for _ in range(1000):
+    for _ in range(5):
       multi_device_iterator = multi_device_iterator_ops.MultiDeviceIterator(
           dataset, ["/cpu:1", "/cpu:2"], prefetch_buffer_size=4)
       elem_on_1, elem_on_2 = multi_device_iterator.get_next()
diff --git a/tensorflow/python/data/ops/multi_device_iterator_ops.py b/tensorflow/python/data/ops/multi_device_iterator_ops.py
index 0a74b9be06..051bc45843 100644
--- a/tensorflow/python/data/ops/multi_device_iterator_ops.py
+++ b/tensorflow/python/data/ops/multi_device_iterator_ops.py
@@ -145,6 +145,10 @@ class MultiDeviceIterator(object):
         to prefetch into.
       source_device: The host device to place the `dataset` on.
 
+      In order to prevent deadlocks, if the prefetch_buffer_size is greater
+      than the max_buffer_size, we set the max_buffer_size to
+      prefetch_buffer_size.
+
     Raises:
       RuntimeError: If run in Eager mode.
     """
@@ -153,6 +157,9 @@ class MultiDeviceIterator(object):
     self._source_device = source_device
     self._source_device_tensor = ops.convert_to_tensor(source_device)
 
+    if prefetch_buffer_size > max_buffer_size:
+      max_buffer_size = prefetch_buffer_size
+
     # Create the MultiDeviceIterator.
     with ops.device(self._source_device):
       # TODO(b/121378567): Get rid of this shared_name hack.
-- 
GitLab


From 121cb2614eb172c1932c89f9aa92722b0b47ae75 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 29 Jan 2019 13:10:52 -0800
Subject: [PATCH 1520/2345] [XLA] Changing the vlog of the entire HloModule
 proto on load to -v=3.

PiperOrigin-RevId: 231459245
---
 tensorflow/compiler/xla/service/hlo_module.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/hlo_module.cc b/tensorflow/compiler/xla/service/hlo_module.cc
index 258f918f47..759ce6541e 100644
--- a/tensorflow/compiler/xla/service/hlo_module.cc
+++ b/tensorflow/compiler/xla/service/hlo_module.cc
@@ -250,7 +250,7 @@ HloModuleProto HloModule::ToProto() const {
 StatusOr<std::unique_ptr<HloModule>> HloModule::CreateFromProto(
     const HloModuleProto& proto, const HloModuleConfig& module_config) {
   VLOG(2) << "CreateFromProto()";
-  XLA_VLOG_LINES(2, proto.DebugString());
+  XLA_VLOG_LINES(3, proto.DebugString());
 
   // The ProgramShape in the passed in module config must match the shapes of
   // the entry parameters and root.
-- 
GitLab


From 0c8347957826cd5d170ff9a300354ba59cd1dc7b Mon Sep 17 00:00:00 2001
From: Guozhong Zhuang <guozhong.zhuang@intel.com>
Date: Tue, 29 Jan 2019 13:37:36 -0800
Subject: [PATCH 1521/2345] Fix issues with comments and coding style

---
 tensorflow/core/util/mkl_util.h | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/tensorflow/core/util/mkl_util.h b/tensorflow/core/util/mkl_util.h
index 499a24b3dc..8f341061d5 100644
--- a/tensorflow/core/util/mkl_util.h
+++ b/tensorflow/core/util/mkl_util.h
@@ -86,7 +86,7 @@ namespace tensorflow {
 // For use with MKL ML, has been deprecated
 typedef enum { W = 0, H = 1, C = 2, N = 3 } MklDims;
 
-// The dimensions order that MKL DNN internally uses for 2D activations
+// The dimensions order that MKL-DNN internally uses for 2D activations
 // [Batch, Channel, Height, Width] and
 // for 2D filters [Out_Channel, In_Channel, Height, Width].
 typedef enum {
@@ -98,7 +98,7 @@ typedef enum {
   Dim_I = 1
 } MklDnnDims;
 
-// The dimensions order that MKL DNN internally uses for 3D activations
+// The dimensions order that MKL-DNN internally uses for 3D activations
 // [Batch, Channel, Depth, Height, Width] and
 // for 3D filters [Out_Channel, In_Channel, Depth, Height, Width].
 typedef enum {
@@ -130,7 +130,7 @@ typedef enum {
   TF_3DFILTER_DIM_O = 4
 } TFFilterDims3d;
 
-// The dimensions order that MKL DNN requires for the filter in a grouped
+// The dimensions order that MKL-DNN requires for the filter in a grouped
 // convolution (2D only)
 typedef enum {
   MKL_GROUP_FILTER_DIM_G = 0,
@@ -141,7 +141,6 @@ typedef enum {
 } MklDnnFilterGroupDims;
 
 // Enum used to templatize MklOp kernel implementations
-
 // that support both fp32 and int8 versions.
 enum class MklQuantization {
   QUANTIZED_VERSION,
@@ -2157,7 +2156,7 @@ class LRUCache {
   std::unordered_map<string, Entry> cache_;
 
   // The LRU list of entries.
-  // The front of the list contians the key of the most recently accessed 
+  // The front of the list contains the key of the most recently accessed 
   // entry, while the back of the list is the least recently accessed entry.
   std::list<string> lru_list_;
 };
-- 
GitLab


From 40351b4e51710491b7304bd26900787c68823970 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 29 Jan 2019 13:34:59 -0800
Subject: [PATCH 1522/2345] [MacOs] Skip a failing test for MacOs

PiperOrigin-RevId: 231463501
---
 tensorflow/core/platform/env_test.cc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/platform/env_test.cc b/tensorflow/core/platform/env_test.cc
index 5a5d133c96..ea1f123424 100644
--- a/tensorflow/core/platform/env_test.cc
+++ b/tensorflow/core/platform/env_test.cc
@@ -394,13 +394,15 @@ TEST_F(DefaultEnvTest, CreateUniqueFileName) {
 
 TEST_F(DefaultEnvTest, GetThreadInformation) {
   Env* env = Env::Default();
-
+  // TODO(fishx): Turn on this test for Apple.
+#if !defined(__APPLE__)
   EXPECT_NE(env->GetCurrentThreadId(), 0);
+#endif
   string thread_name;
   bool res = env->GetCurrentThreadName(&thread_name);
 #if defined(PLATFORM_WINDOWS) || defined(__ANDROID__)
   EXPECT_FALSE(res);
-#else
+#elif !defined(__APPLE__)
   EXPECT_TRUE(res);
   EXPECT_GT(thread_name.size(), 0);
 #endif
-- 
GitLab


From 6813be8a45f17913cc9463019bb5a79bde0b4e74 Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Tue, 29 Jan 2019 13:49:12 -0800
Subject: [PATCH 1523/2345] Eliminate `_call_and_compute_mask` and instead
 always implicitly pass masks and always `__call__` Layers in Keras Graph
 Networks.

PiperOrigin-RevId: 231466249
---
 tensorflow/python/keras/engine/base_layer.py  |  78 +++++++------
 .../python/keras/engine/base_layer_utils.py   |  18 +--
 tensorflow/python/keras/engine/input_layer.py |   1 +
 tensorflow/python/keras/engine/network.py     | 106 ++++++------------
 tensorflow/python/keras/engine/sequential.py  |  69 ++++--------
 .../python/keras/engine/training_eager.py     |  19 ++--
 tensorflow/python/keras/layers/core_test.py   |   7 ++
 .../python/keras/utils/generic_utils.py       |   7 +-
 8 files changed, 119 insertions(+), 186 deletions(-)

diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index a355dda4e5..688fdd0101 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -473,15 +473,10 @@ class Layer(checkpointable.Checkpointable):
             one per output tensor of the layer).
     """
     if not self.supports_masking:
-      if mask is not None:
-        if isinstance(mask, list):
-          if any(m is not None for m in mask):
-            raise TypeError('Layer ' + self.name + ' does not support masking, '
-                            'but was passed an input_mask: ' + str(mask))
-        else:
-          raise TypeError('Layer ' + self.name + ' does not support masking, '
-                          'but was passed an input_mask: ' + str(mask))
-      # masking not explicitly supported: return None as mask
+      if any(m is not None for m in nest.flatten(mask)):
+        raise TypeError('Layer ' + self.name + ' does not support masking, '
+                        'but was passed an input_mask: ' + str(mask))
+      # masking not explicitly supported: return None as mask.
       return None
     # if masking is explicitly supported, by default
     # carry over the input mask
@@ -535,15 +530,15 @@ class Layer(checkpointable.Checkpointable):
 
     # Handle Keras mask propagation from previous layer to current layer.
     previous_mask = None
-    if build_graph and (not hasattr(self, '_compute_previous_mask') or
-                        self._compute_previous_mask):
+    if (not hasattr(self, '_compute_previous_mask') or
+        self._compute_previous_mask):
       previous_mask = base_layer_utils.collect_previous_mask(inputs)
       if not hasattr(self, '_call_fn_args'):
         self._call_fn_args = function_utils.fn_args(self.call)
       if ('mask' in self._call_fn_args and 'mask' not in kwargs and
           not generic_utils.is_all_none(previous_mask)):
-        # The previous layer generated a mask, and mask was not explicitly pass
-        # to __call__, hence we set previous_mask as the default value.
+        # The previous layer generated a mask, and mask was not explicitly
+        # pass to __call__, hence we set previous_mask as the default value.
         kwargs['mask'] = previous_mask
 
     with ops.name_scope(self._name_scope()):
@@ -589,11 +584,11 @@ class Layer(checkpointable.Checkpointable):
             raise ValueError('A layer\'s `call` method should return a '
                              'Tensor or a list of Tensors, not None '
                              '(layer: ' + self.name + ').')
-          self._handle_activity_regularization(inputs, outputs)
-          self._set_mask_metadata(inputs, outputs, previous_mask)
           if base_layer_utils.have_all_keras_metadata(inputs):
             inputs, outputs = self._set_connectivity_metadata_(
                 inputs, outputs, args, kwargs)
+          self._handle_activity_regularization(inputs, outputs)
+          self._set_mask_metadata(inputs, outputs, previous_mask)
           if hasattr(self, '_set_inputs') and not self.inputs:
             # Subclassed network: explicitly set metadata normally set by
             # a call to self._set_inputs().
@@ -605,7 +600,7 @@ class Layer(checkpointable.Checkpointable):
         # Eager execution on data tensors.
         outputs = self.call(inputs, *args, **kwargs)
         self._handle_activity_regularization(inputs, outputs)
-        return outputs
+        self._set_mask_metadata(inputs, outputs, previous_mask)
 
     if not context.executing_eagerly():
       # Optionally load weight values specified at layer instantiation.
@@ -1352,27 +1347,32 @@ class Layer(checkpointable.Checkpointable):
           self.add_loss(mean_activity_loss, inputs=inputs)
 
   def _set_mask_metadata(self, inputs, outputs, previous_mask):
-    # In some cases the mask of the outputs has already been computed by
-    # inner layers and does not need to be recomputed by this layer.
-    mask_already_computed = all(
-        hasattr(x, '_keras_mask') for x in generic_utils.to_list(outputs))
-    if hasattr(self, 'compute_mask') and not mask_already_computed:
-      output_mask = self.compute_mask(inputs, previous_mask)
-    else:
-      output_mask = None
-    if isinstance(outputs, (list, tuple)):
-      if output_mask is None:
-        output_mask = [None for _ in range(len(outputs))]
-      for x, m in zip(outputs, output_mask):
-        try:
-          x._keras_mask = m  # pylint: disable=protected-access
-        except AttributeError:
-          pass  # C type such as dict. Masking not supported in this case.
+    if getattr(self, '_compute_output_and_mask_jointly', False):
+      # Mask is already computed for Keras Graph Networks.
+      return
+
+    flat_outputs = nest.flatten(outputs)
+    if all(getattr(x, '_keras_mask', None) is not None for x in flat_outputs):
+      # Mask is already computed by sublayers.
+      return
+
+    if hasattr(self, 'compute_mask'):
+      output_masks = self.compute_mask(inputs, previous_mask)
+      # `compute_mask` can return a single `None` even when a Layer
+      # has multiple outputs.
+      if output_masks is None:
+        flat_masks = [None for _ in flat_outputs]
+      else:
+        flat_masks = nest.flatten(output_masks)
     else:
+      flat_masks = [None for _ in flat_outputs]
+
+    for output, mask in zip(flat_outputs, flat_masks):
       try:
-        outputs._keras_mask = output_mask  # pylint: disable=protected-access
+        output._keras_mask = mask
       except AttributeError:
-        pass  # C type such as dict. Masking not supported in this case.
+        # C Type such as np.ndarray.
+        pass
 
   def _set_connectivity_metadata_(self, inputs, outputs, args, kwargs):
     call_convention = getattr(
@@ -1585,9 +1585,13 @@ class Layer(checkpointable.Checkpointable):
   def _symbolic_call(self, inputs):
     input_shapes = nest.map_structure(lambda x: x.shape, inputs)
     output_shapes = self.compute_output_shape(input_shapes)
-    return nest.map_structure(
-        lambda shape: backend.placeholder(shape, dtype=self.dtype),
-        output_shapes)
+
+    def _make_placeholder_like(shape):
+      ph = backend.placeholder(shape, self.dtype)
+      ph._keras_mask = None
+      return ph
+
+    return nest.map_structure(_make_placeholder_like, output_shapes)
 
   def __setattr__(self, name, value):
     if (not getattr(self, '_setattr_tracking', True) or
diff --git a/tensorflow/python/keras/engine/base_layer_utils.py b/tensorflow/python/keras/engine/base_layer_utils.py
index cdc861f974..211e8482e4 100644
--- a/tensorflow/python/keras/engine/base_layer_utils.py
+++ b/tensorflow/python/keras/engine/base_layer_utils.py
@@ -205,22 +205,16 @@ def collect_previous_mask(input_tensors):
   """Retrieves the output mask(s) of the previous node.
 
   Arguments:
-      input_tensors: A tensor or list of tensors.
+      input_tensors: An arbitrary structure of Tensors.
 
   Returns:
       A mask tensor or list of mask tensors.
   """
-  input_tensors = nest.flatten(input_tensors)
-  masks = []
-  for x in input_tensors:
-    if hasattr(x, '_keras_mask'):
-      mask = x._keras_mask  # pylint: disable=protected-access
-      masks.append(mask)
-    else:
-      masks.append(None)
-  if len(masks) == 1:
-    return masks[0]
-  return masks
+
+  def _collect_previous_mask(x):
+    return getattr(x, '_keras_mask', None)
+
+  return nest.map_structure(_collect_previous_mask, input_tensors)
 
 
 def have_all_keras_metadata(tensors):
diff --git a/tensorflow/python/keras/engine/input_layer.py b/tensorflow/python/keras/engine/input_layer.py
index c6dcedfce2..32fbbea8a1 100644
--- a/tensorflow/python/keras/engine/input_layer.py
+++ b/tensorflow/python/keras/engine/input_layer.py
@@ -123,6 +123,7 @@ class InputLayer(base_layer.Layer):
     # Create an input node to add to self.outbound_node
     # and set output_tensors' _keras_history.
     input_tensor._keras_history = (self, 0, 0)  # pylint: disable=protected-access
+    input_tensor._keras_mask = None
     base_layer.Node(
         self,
         inbound_layers=[],
diff --git a/tensorflow/python/keras/engine/network.py b/tensorflow/python/keras/engine/network.py
index 41f5f319bc..0835e7c083 100644
--- a/tensorflow/python/keras/engine/network.py
+++ b/tensorflow/python/keras/engine/network.py
@@ -170,14 +170,7 @@ class Network(base_layer.Layer):
     self.trainable = True
     self._is_compiled = False
     self._expects_training_arg = False
-    # In many internal cases one needs to compute both the model's output
-    # and its output mask without relying on `__call__` (which would do both and
-    # set mask metadata), but for models, computing the mask requires to
-    # recompute the output.
-    # Hence the pattern `output = model.call(); mask = model.compute_mask()`
-    # would be redundant, and internal logic
-    # (susceptible to use `call` directly) should prefer using the
-    # internal method `output, mask = _call_and_compute_mask()`.
+
     # This is True for Sequential networks and graph networks.
     self._compute_output_and_mask_jointly = False
 
@@ -234,9 +227,9 @@ class Network(base_layer.Layer):
     if any(not hasattr(tensor, '_keras_history') for tensor in self.outputs):
       base_layer_utils.create_keras_history(self._nested_outputs)
 
+    self._base_init(name=name)
     self._validate_graph_inputs_and_outputs()
 
-    self._base_init(name=name)
     self._compute_previous_mask = (
         'mask' in tf_inspect.getfullargspec(self.call).args or
         hasattr(self, 'compute_mask'))
@@ -478,8 +471,11 @@ class Network(base_layer.Layer):
     if not self._is_graph_network:
       return None
 
-    _, output_masks = self._run_internal_graph(inputs, mask=mask)
-    return output_masks
+    # TODO(omalleyt): b/123540974 This function is not really safe to call
+    # by itself because it will duplicate any updates and losses in graph
+    # mode by `call`ing the Layers again.
+    output_tensors = self._run_internal_graph(inputs, mask=mask)
+    return nest.map_structure(lambda t: t._keras_mask, output_tensors)
 
   @property
   def layers(self):
@@ -866,10 +862,6 @@ class Network(base_layer.Layer):
       raise NotImplementedError('When subclassing the `Model` class, you should'
                                 ' implement a `call` method.')
 
-    outputs, _ = self._run_internal_graph(inputs, training=training, mask=mask)
-    return outputs
-
-  def _call_and_compute_mask(self, inputs, training=None, mask=None):
     return self._run_internal_graph(inputs, training=training, mask=mask)
 
   def compute_output_shape(self, input_shape):
@@ -972,92 +964,59 @@ class Network(base_layer.Layer):
     else:
       masks = nest.flatten(mask)
 
+    for input_t, mask in zip(inputs, masks):
+      input_t._keras_mask = mask
+
     # Dictionary mapping reference tensors to computed tensors.
     tensor_dict = {}
-    # Dictionary mapping reference tensors to computed masks.
-    mask_dict = {}
 
     for x, y, mask in zip(self.inputs, inputs, masks):
       tensor_dict[str(id(x))] = y
-      mask_dict[str(id(x))] = mask
 
     depth_keys = list(self._nodes_by_depth.keys())
     depth_keys.sort(reverse=True)
+    # Ignore the InputLayers when computing the graph.
+    depth_keys = depth_keys[1:]
+
     for depth in depth_keys:
       nodes = self._nodes_by_depth[depth]
       for node in nodes:
         # This is always a single layer, never a list.
         layer = node.outbound_layer
-        # node_input_tensors = node.input_tensors
-        # node_output_tensors = node.output_tensors
 
         if all(
             str(id(tensor)) in tensor_dict
             for tensor in nest.flatten(node.input_tensors)):
+
           # Call layer (reapplying ops to new inputs).
-          with ops.name_scope(layer.name):
-            computed_tensors = nest.map_structure(
-                lambda t: tensor_dict[str(id(t))], node.input_tensors)
-            computed_masks = nest.map_structure(lambda t: mask_dict[str(id(t))],
-                                                node.input_tensors)
-            kwargs = node.arguments or {}
-            # Ensure `training` arg propagation if applicable.
-            argspec = self._layer_call_argspecs[layer].args
-            if 'training' in argspec:
-              kwargs.setdefault('training', training)
-            if 'mask' in argspec:
-              kwargs.setdefault('mask', computed_masks)
-
-            # Compute outputs and masks.
-            output_masks = None
-            if (isinstance(layer, Network) and
-                layer._compute_output_and_mask_jointly):
-              output_tensors, output_masks = layer._call_and_compute_mask(
-                  computed_tensors, **kwargs)
-            else:
-              if context.executing_eagerly():
-                output_tensors = layer(computed_tensors, **kwargs)
-              elif layer.dynamic:
-                output_tensors = layer._symbolic_call(computed_tensors)  # pylint: disable=protected-call
-              else:
-                output_tensors = layer.call(computed_tensors, **kwargs)
-              if hasattr(layer, 'compute_mask'):
-                output_masks = layer.compute_mask(computed_tensors,
-                                                  computed_masks)
-            if output_masks is None:
-              output_masks = nest.pack_sequence_as(
-                  output_tensors, [None for _ in nest.flatten(output_tensors)])
-
-            if not context.executing_eagerly():
-              # Set mask metadata.
-              for x, m in zip(
-                  nest.flatten(output_tensors), nest.flatten(output_masks)):
-                try:
-                  x._keras_mask = m
-                except AttributeError:
-                  pass
-
-              # Apply activity regularizer if any.
-              layer._handle_activity_regularization(computed_tensors,
-                                                    output_tensors)
+          computed_tensors = nest.map_structure(
+              lambda t: tensor_dict[str(id(t))], node.input_tensors)
+
+          # Ensure `training` and `mask` arg propagation if applicable.
+          kwargs = node.arguments or {}
+          argspec = self._layer_call_argspecs[layer].args
+          if 'training' in argspec:
+            kwargs.setdefault('training', training)
+          if 'mask' in argspec:
+            computed_masks = nest.map_structure(lambda t: t._keras_mask,
+                                                computed_tensors)
+            kwargs.setdefault('mask', computed_masks)
+
+          # Compute outputs.
+          output_tensors = layer(computed_tensors, **kwargs)
 
           # Update tensor_dict.
-          for x, y, mask in zip(
-              nest.flatten(node.output_tensors), nest.flatten(output_tensors),
-              nest.flatten(output_masks)):
+          for x, y in zip(
+              nest.flatten(node.output_tensors), nest.flatten(output_tensors)):
             tensor_dict[str(id(x))] = y
-            mask_dict[str(id(x))] = mask
 
     output_tensors = []
-    output_masks = []
     output_shapes = []
     for x in self.outputs:
       assert str(id(x)) in tensor_dict, 'Could not compute output ' + str(x)
       tensor = tensor_dict[str(id(x))]
-      mask = mask_dict[str(id(x))]
       output_shapes.append(x.shape)
       output_tensors.append(tensor)
-      output_masks.append(mask)
 
     if output_shapes is not None:
       input_shapes = [x.shape for x in inputs]
@@ -1066,8 +1025,7 @@ class Network(base_layer.Layer):
           self._nested_outputs, output_shapes)
 
     output_tensors = nest.pack_sequence_as(self._nested_outputs, output_tensors)
-    output_masks = nest.pack_sequence_as(self._nested_outputs, output_masks)
-    return output_tensors, output_masks
+    return output_tensors
 
   def get_config(self):
     if not self._is_graph_network:
diff --git a/tensorflow/python/keras/engine/sequential.py b/tensorflow/python/keras/engine/sequential.py
index 37eb2840b3..671bfe9c28 100644
--- a/tensorflow/python/keras/engine/sequential.py
+++ b/tensorflow/python/keras/engine/sequential.py
@@ -21,15 +21,11 @@ from __future__ import print_function
 
 import copy
 
-from tensorflow.python.eager import context
-from tensorflow.python.framework import ops
 from tensorflow.python.keras import layers as layer_module
 from tensorflow.python.keras.engine import base_layer
+from tensorflow.python.keras.engine import input_layer
+from tensorflow.python.keras.engine import training
 from tensorflow.python.keras.engine import training_utils
-from tensorflow.python.keras.engine.input_layer import Input
-from tensorflow.python.keras.engine.input_layer import InputLayer
-from tensorflow.python.keras.engine.network import Network
-from tensorflow.python.keras.engine.training import Model
 from tensorflow.python.keras.utils import layer_utils
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training.checkpointable import base as checkpointable
@@ -39,7 +35,7 @@ from tensorflow.python.util.tf_export import keras_export
 
 
 @keras_export('keras.models.Sequential', 'keras.Sequential')
-class Sequential(Model):
+class Sequential(training.Model):
   """Linear stack of layers.
 
   Arguments:
@@ -119,7 +115,7 @@ class Sequential(Model):
     # `CheckpointableBase` manages the `_layers` attributes and does filtering
     # over it.
     layers = super(Sequential, self).layers
-    if layers and isinstance(layers[0], InputLayer):
+    if layers and isinstance(layers[0], input_layer.InputLayer):
       return layers[1:]
     return layers[:]
 
@@ -149,7 +145,7 @@ class Sequential(Model):
     self.built = False
     set_inputs = False
     if not self._layers:
-      if isinstance(layer, InputLayer):
+      if isinstance(layer, input_layer.InputLayer):
         # Corner case where the user passes an InputLayer layer via `add`.
         assert len(nest.flatten(layer._inbound_nodes[-1].output_tensors)) == 1
         set_inputs = True
@@ -157,10 +153,8 @@ class Sequential(Model):
         batch_shape, dtype = training_utils.get_input_shape_and_dtype(layer)
         if batch_shape:
           # Instantiate an input layer.
-          x = Input(
-              batch_shape=batch_shape,
-              dtype=dtype,
-              name=layer.name + '_input')
+          x = input_layer.Input(
+              batch_shape=batch_shape, dtype=dtype, name=layer.name + '_input')
           # This will build the current layer
           # and create the node connecting the current layer
           # to the input layer we just created.
@@ -233,18 +227,12 @@ class Sequential(Model):
       super(Sequential, self).build(input_shape)
     self.built = True
 
-  def call(self, inputs, training=None, mask=None):
+  def call(self, inputs, training=None, mask=None):  # pylint: disable=redefined-outer-name
     if self._is_graph_network:
+      if not self.built:
+        self._init_graph_network(self.inputs, self.outputs, name=self.name)
       return super(Sequential, self).call(inputs, training=training, mask=mask)
 
-    outputs, _ = self._call_and_compute_mask(
-        inputs, training=training, mask=mask)
-    return outputs
-
-  def _call_and_compute_mask(self, inputs, training=None, mask=None):
-    if not self.built and self._is_graph_network:
-      self._init_graph_network(self.inputs, self.outputs, name=self.name)
-
     outputs = inputs  # handle the corner case where self.layers is empty
     for layer in self.layers:
       # During each iteration, `inputs` are the inputs to `layer`, and `outputs`
@@ -257,35 +245,13 @@ class Sequential(Model):
       if 'training' in argspec:
         kwargs['training'] = training
 
-      if isinstance(layer, Network) and layer._compute_output_and_mask_jointly:
-        outputs, mask = layer._call_and_compute_mask(inputs, **kwargs)
-      else:
-        if not layer.built:
-          # Build layer if applicable.
-          with ops.name_scope(layer._name_scope()):
-            layer._maybe_build(inputs)
-          layer.built = True
-        if layer.supports_masking:
-          mask = layer.compute_mask(inputs, mask)
-        else:
-          mask = None
-
-        if context.executing_eagerly():
-          # __call__ handles activity regularization.
-          outputs = layer(inputs, **kwargs)
-        elif layer.dynamic:
-          outputs = layer._symbolic_call(inputs)
-          layer._handle_activity_regularization(inputs, outputs)
-        else:
-          outputs = layer.call(inputs, **kwargs)
-          layer._handle_activity_regularization(inputs, outputs)
-
-      if not context.executing_eagerly():
-        outputs._keras_mask = mask
+      outputs = layer(inputs, **kwargs)
 
       # `outputs` will be the inputs to the next layer.
       inputs = outputs
-    return outputs, mask
+      mask = outputs._keras_mask
+
+    return outputs
 
   def compute_output_shape(self, input_shape):
     shape = input_shape
@@ -294,8 +260,11 @@ class Sequential(Model):
     return shape
 
   def compute_mask(self, inputs, mask):
-    _, mask = self._call_and_compute_mask(inputs, mask=mask)
-    return mask
+    # TODO(omalleyt): b/123540974 This function is not really safe to call
+    # by itself because it will duplicate any updates and losses in graph
+    # mode by `call`ing the Layers again.
+    outputs = self.call(inputs, mask=mask)
+    return outputs._keras_mask
 
   def predict_proba(self, x, batch_size=32, verbose=0):
     """Generates class probability predictions for the input samples.
diff --git a/tensorflow/python/keras/engine/training_eager.py b/tensorflow/python/keras/engine/training_eager.py
index 6242b54c0c..7f80f94d48 100644
--- a/tensorflow/python/keras/engine/training_eager.py
+++ b/tensorflow/python/keras/engine/training_eager.py
@@ -21,6 +21,8 @@ from __future__ import print_function
 
 import collections
 
+import numpy as np
+
 from tensorflow.python.eager.backprop import GradientTape
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
@@ -101,16 +103,17 @@ def _model_loss(model,
   if len(inputs) == 1 and not isinstance(inputs, dict):
     inputs = inputs[0]
 
-  if model._compute_output_and_mask_jointly:
-    outs, masks = model._call_and_compute_mask(inputs, **kwargs)
-    masks = nest.flatten(masks)
-  else:
-    outs = model.call(inputs, **kwargs)
-    masks = None
+  # Allow mixed `NumPy` and `EagerTensor` input here.
+  if any(
+      isinstance(input_t, (np.ndarray, float, int))
+      for input_t in nest.flatten(inputs)):
+    inputs = nest.map_structure(ops.convert_to_tensor, inputs)
+
+  outs = model(inputs, **kwargs)
 
   outs = nest.flatten(outs)
-  if masks is None:
-    masks = [None for _ in outs]
+  # `None` by default for `EagerTensors`.
+  masks = [t._keras_mask for t in outs]
   targets = nest.flatten(targets)
 
   loss_metrics = []
diff --git a/tensorflow/python/keras/layers/core_test.py b/tensorflow/python/keras/layers/core_test.py
index d61c8e3d08..6ba59b2ff3 100644
--- a/tensorflow/python/keras/layers/core_test.py
+++ b/tensorflow/python/keras/layers/core_test.py
@@ -190,6 +190,13 @@ class CoreLayersTest(keras_parameterized.TestCase):
     testing_utils.layer_test(
         keras.layers.Masking, kwargs={}, input_shape=(3, 2, 3))
 
+  def test_keras_mask(self):
+    x = np.ones((10, 10))
+    y = keras.layers.Masking(1.)(x)
+    self.assertTrue(hasattr(y, '_keras_mask'))
+    self.assertTrue(y._keras_mask is not None)
+    self.assertAllClose(self.evaluate(y._keras_mask), np.zeros((10,)))
+
   def test_activation(self):
     # with string argument
     testing_utils.layer_test(
diff --git a/tensorflow/python/keras/utils/generic_utils.py b/tensorflow/python/keras/utils/generic_utils.py
index 8b5cdadd45..85457720dc 100644
--- a/tensorflow/python/keras/utils/generic_utils.py
+++ b/tensorflow/python/keras/utils/generic_utils.py
@@ -572,11 +572,8 @@ def to_snake_case(name):
   return 'private' + insecure
 
 
-def is_all_none(iterable_or_element):
-  if not isinstance(iterable_or_element, (list, tuple)):
-    iterable = [iterable_or_element]
-  else:
-    iterable = iterable_or_element
+def is_all_none(structure):
+  iterable = nest.flatten(structure)
   # We cannot use Python's `any` because the iterable may return Tensors.
   for element in iterable:
     if element is not None:
-- 
GitLab


From 0863d6286c91aba585849266851a08af657c4f91 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 29 Jan 2019 13:54:28 -0800
Subject: [PATCH 1524/2345] This change natively implements batch group counts
 in TPU lowering.

PiperOrigin-RevId: 231467270
---
 .../service/convolution_group_converter.cc    | 69 +------------------
 .../convolution_group_converter_test.cc       | 12 ++--
 .../compiler/xla/service/hlo_instructions.cc  | 14 ++++
 tensorflow/compiler/xla/service/hlo_parser.cc |  6 ++
 .../conv_depthwise_backprop_filter_test.cc    | 33 ++++-----
 5 files changed, 45 insertions(+), 89 deletions(-)

diff --git a/tensorflow/compiler/xla/service/convolution_group_converter.cc b/tensorflow/compiler/xla/service/convolution_group_converter.cc
index 1c1f543170..2ef723febd 100644
--- a/tensorflow/compiler/xla/service/convolution_group_converter.cc
+++ b/tensorflow/compiler/xla/service/convolution_group_converter.cc
@@ -215,81 +215,15 @@ Status ConvolutionVisitor::HandleBatchGroupCount(HloInstruction* convolution) {
   };
 
   int64 input_batch_dimension = dim_numbers.input_batch_dimension();
-  int64 input_feature_dimension = dim_numbers.input_feature_dimension();
   int64 output_batch_dimension = dim_numbers.output_batch_dimension();
   int64 output_feature_dimension = dim_numbers.output_feature_dimension();
-  int64 kernel_input_feature_dimension =
-      dim_numbers.kernel_input_feature_dimension();
 
   int64 input_batch = activation->shape().dimensions(input_batch_dimension);
 
   // We are not yet supporting batch_group of sizes greater than 1.
   TF_RET_CHECK(input_batch == batch_group_count);
 
-  if (is_cost_viable_(convolution)) {
-    // Add a dimension to the activation, and reshape.
-    Shape reshaped_activation_shape = activation->shape();
-    ShapeUtil::AppendMajorDimension(1, &reshaped_activation_shape);
-
-    activation = add(
-        HloInstruction::CreateReshape(reshaped_activation_shape, activation));
-
-    // Add a dimension to the filter, and reshape.
-    Shape reshaped_filter_shape = filter->shape();
-    ShapeUtil::AppendMajorDimension(1, &reshaped_filter_shape);
-
-    filter = add(HloInstruction::CreateReshape(reshaped_filter_shape, filter));
-
-    int64 new_spatial_dim = reshaped_activation_shape.dimensions().size() - 1;
-
-    Shape new_output_shape = convolution->shape();
-    ShapeUtil::AppendMajorDimension(1, &new_output_shape);
-
-    int64 input_feature =
-        activation->shape().dimensions(input_feature_dimension);
-
-    // The code below edits convolution dimension numbers. Please refer to
-    // conv_op_helpers.cc to find how the dimensions were set up originally.
-
-    // Effectively, the new input batch becomes 1, and so does the kernel
-    // input feature. The original input batch now becomes a spatial dimension.
-    // The output batch (remember that the output is the new kernel for in
-    // backprop) becomes a spatial dimension too.
-
-    dim_numbers.set_input_batch_dimension(new_spatial_dim);
-    dim_numbers.set_input_feature_dimension(input_batch_dimension);
-    dim_numbers.set_kernel_input_feature_dimension(new_spatial_dim);
-
-    dim_numbers.add_input_spatial_dimensions(input_feature_dimension);
-    dim_numbers.add_kernel_spatial_dimensions(kernel_input_feature_dimension);
-
-    dim_numbers.add_output_spatial_dimensions(output_batch_dimension);
-    dim_numbers.set_output_batch_dimension(new_spatial_dim);
-
-    // Add window for the new spatial dimension.
-    Window new_window = convolution->window();
-    auto* dim = new_window.add_dimensions();
-    dim->set_window_dilation(1);
-    dim->set_base_dilation(1);
-    dim->set_stride(1);
-    dim->set_size(input_feature);
-
-    auto new_convolution = add(HloInstruction::CreateConvolve(
-        new_output_shape, activation, filter,
-        /*feature_group_count=*/batch_group_count, /*batch_group_count=*/1,
-        new_window, dim_numbers, convolution->precision_config()));
-
-    // Delete the extra spatial dimension, and reshape.
-    Shape reshaped_convolution_shape = ShapeUtil::DeleteDimension(
-        new_spatial_dim - 1, new_convolution->shape());
-    auto reshaped_convolution = HloInstruction::CreateReshape(
-        reshaped_convolution_shape, new_convolution);
-
-    TF_RETURN_IF_ERROR(computation_->ReplaceWithNewInstruction(
-        convolution, std::move(reshaped_convolution)));
-
-    changed_ = true;
-  } else {
+  if (!is_cost_viable_(convolution)) {
     // We first obtain the expanded the filter (which is the convolution
     // output). The batch dimension is the expanded one (which originally
     // represents kernel input feature dimension). We mask the filter to zero
@@ -375,6 +309,7 @@ Status ConvolutionVisitor::HandleBatchGroupCount(HloInstruction* convolution) {
 
     TF_RETURN_IF_ERROR(computation_->ReplaceWithNewInstruction(
         convolution, std::move(reduce_window_converted)));
+    changed_ = true;
   }
 
   return Status::OK();
diff --git a/tensorflow/compiler/xla/service/convolution_group_converter_test.cc b/tensorflow/compiler/xla/service/convolution_group_converter_test.cc
index 585b81a5db..9cee3eda95 100644
--- a/tensorflow/compiler/xla/service/convolution_group_converter_test.cc
+++ b/tensorflow/compiler/xla/service/convolution_group_converter_test.cc
@@ -109,16 +109,16 @@ ENTRY %Convolve1D1Window_0.v3 (input: f32[16,19,19,512]{3,2,1,0}, filter: f32[16
   auto computation = module->entry_computation();
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root->opcode(), HloOpcode::kConvolution);
-  auto cost_model = [](HloInstruction* conv) { return true; };
+  auto cost_model = [](HloInstruction* conv) { return false; };
   ConvolutionGroupConverter converter(cost_model, /*convert_batch_groups_only=*/
                                       true);
   ASSERT_TRUE(converter.Run(module.get()).ValueOrDie());
   root = computation->root_instruction();
-  // Make sure the convolution is converted to one with batch_group_count = 1.
-  EXPECT_EQ(root->operand(0)->opcode(), HloOpcode::kConvolution);
-  EXPECT_EQ(root->operand(0)->batch_group_count(), 1);
-  // Verify that the convolution is replaced by a reshape.
-  EXPECT_EQ(root->opcode(), HloOpcode::kReshape);
+
+  // Verify that the convolution is replaced by a convert.
+  EXPECT_EQ(root->opcode(), HloOpcode::kConvert);
+  // Make sure the convert is being fed by a reduce window.
+  EXPECT_EQ(root->operand(0)->opcode(), HloOpcode::kReduceWindow);
 }
 
 }  // namespace
diff --git a/tensorflow/compiler/xla/service/hlo_instructions.cc b/tensorflow/compiler/xla/service/hlo_instructions.cc
index b01f01ef01..785206bf77 100644
--- a/tensorflow/compiler/xla/service/hlo_instructions.cc
+++ b/tensorflow/compiler/xla/service/hlo_instructions.cc
@@ -1695,6 +1695,7 @@ HloInstructionProto HloConvolutionInstruction::ToProto() const {
   *proto.mutable_convolution_dimension_numbers() =
       convolution_dimension_numbers_;
   proto.set_feature_group_count(feature_group_count_);
+  proto.set_batch_group_count(batch_group_count_);
   *proto.mutable_precision_config() = precision_config_;
   return proto;
 }
@@ -1732,6 +1733,9 @@ bool HloConvolutionInstruction::IdenticalSlowPath(
   if (feature_group_count_ != other.feature_group_count()) {
     return false;
   }
+  if (batch_group_count_ != other.batch_group_count()) {
+    return false;
+  }
   return protobuf_util::ProtobufEquals(window(), casted_other.window()) &&
          protobuf_util::ProtobufEquals(
              convolution_dimension_numbers(),
@@ -1850,6 +1854,7 @@ HloCustomCallInstruction::HloCustomCallInstruction(
       custom_call_target_(custom_call_target.begin(), custom_call_target.end()),
       opaque_(opaque.begin(), opaque.end()),
       feature_group_count_(1),
+      batch_group_count_(1),
       layout_constrained_(false) {
   for (auto operand : operands) {
     AppendOperand(operand);
@@ -1864,6 +1869,7 @@ HloCustomCallInstruction::HloCustomCallInstruction(
       custom_call_target_(custom_call_target.begin(), custom_call_target.end()),
       opaque_(opaque.begin(), opaque.end()),
       feature_group_count_(1),
+      batch_group_count_(1),
       layout_constrained_(true),
       operand_shapes_with_layout_(operand_shapes_with_layout.begin(),
                                   operand_shapes_with_layout.end()) {
@@ -1884,6 +1890,7 @@ HloInstructionProto HloCustomCallInstruction::ToProto() const {
   proto.set_custom_call_target(custom_call_target_);
   proto.set_custom_call_opaque(opaque_);
   proto.set_feature_group_count(feature_group_count_);
+  proto.set_batch_group_count(batch_group_count_);
   if (layout_constrained()) {
     proto.set_constrain_layout(true);
     for (const Shape& shape : operand_shapes_with_layout_) {
@@ -1907,6 +1914,9 @@ std::vector<string> HloCustomCallInstruction::ExtraAttributesToStringImpl(
   if (feature_group_count_ != 1) {
     extra.push_back(StrCat("feature_group_count=", feature_group_count_));
   }
+  if (batch_group_count_ != 1) {
+    extra.push_back(StrCat("batch_group_count=", batch_group_count_));
+  }
   // By contract, we print the custom call target even if
   // options.print_subcomputation_mode() == kOff, because the call target is not
   // an HloComputation.
@@ -1950,6 +1960,9 @@ bool HloCustomCallInstruction::IdenticalSlowPath(
   if (feature_group_count_ != casted_other.feature_group_count_) {
     return false;
   }
+  if (batch_group_count_ != casted_other.batch_group_count_) {
+    return false;
+  }
   return custom_call_target_ == casted_other.custom_call_target_ &&
          opaque_ == casted_other.opaque_;
 }
@@ -1967,6 +1980,7 @@ HloCustomCallInstruction::CloneWithNewOperandsImpl(
     cloned->set_convolution_dimension_numbers(*convolution_dimension_numbers_);
   }
   cloned->set_feature_group_count(feature_group_count_);
+  cloned->set_batch_group_count(batch_group_count_);
   return std::move(cloned);
 }
 
diff --git a/tensorflow/compiler/xla/service/hlo_parser.cc b/tensorflow/compiler/xla/service/hlo_parser.cc
index 638396308c..bef1e067a4 100644
--- a/tensorflow/compiler/xla/service/hlo_parser.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser.cc
@@ -1375,6 +1375,7 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
       optional<Window> window;
       optional<ConvolutionDimensionNumbers> dnums;
       optional<int64> feature_group_count;
+      optional<int64> batch_group_count;
       optional<std::vector<Shape>> operand_layout_constraints;
       attrs["custom_call_target"] = {/*required=*/true, AttrTy::kString,
                                      &custom_call_target};
@@ -1384,6 +1385,8 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
                              AttrTy::kConvolutionDimensionNumbers, &dnums};
       attrs["feature_group_count"] = {/*required=*/false, AttrTy::kInt64,
                                       &feature_group_count};
+      attrs["batch_group_count"] = {/*required=*/false, AttrTy::kInt64,
+                                    &batch_group_count};
       attrs["operand_layout_constraints"] = {
           /*required=*/false, AttrTy::kShapeList, &operand_layout_constraints};
       if (!ParseOperands(&operands) || !ParseAttributes(attrs)) {
@@ -1439,6 +1442,9 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
       if (feature_group_count.has_value()) {
         instruction->set_feature_group_count(*feature_group_count);
       }
+      if (batch_group_count.has_value()) {
+        instruction->set_batch_group_count(*batch_group_count);
+      }
       break;
     }
     case HloOpcode::kDot: {
diff --git a/tensorflow/compiler/xla/tests/conv_depthwise_backprop_filter_test.cc b/tensorflow/compiler/xla/tests/conv_depthwise_backprop_filter_test.cc
index c78d3f3d9e..96f4aedf8b 100644
--- a/tensorflow/compiler/xla/tests/conv_depthwise_backprop_filter_test.cc
+++ b/tensorflow/compiler/xla/tests/conv_depthwise_backprop_filter_test.cc
@@ -32,20 +32,20 @@ string GetFloatDataType(bool use_bfloat16) {
   return use_bfloat16 ? "bf16" : "f32";
 }
 
-struct DepthwiseConvolution2DSpec {
+struct BatchGroupedConvolution2DSpec {
   int64 output_batch, window, window_dilation;
   std::vector<int64> activation_dims;
   std::vector<int64> kernel_dims;
   std::vector<int64> output_dims;
 };
 
-class DepthwiseConvolution2DTest
+class BatchGroupedConvolution2DTest
     : public HloTestBase,
       public ::testing::WithParamInterface<
-          ::testing::tuple<DepthwiseConvolution2DSpec, bool>> {};
+          ::testing::tuple<BatchGroupedConvolution2DSpec, bool>> {};
 
-static std::vector<DepthwiseConvolution2DSpec> GetConv2DTestCases() {
-  std::vector<DepthwiseConvolution2DSpec> config_set;
+static std::vector<BatchGroupedConvolution2DSpec> GetConv2DTestCases() {
+  std::vector<BatchGroupedConvolution2DSpec> config_set;
   std::vector<std::vector<int64>> config_options = {
       {8, 5, 3, 2},   {4, 5, 5, 2},   {8, 7, 4, 128},  {16, 20, 20, 256},
       {256, 7, 5, 4}, {256, 6, 6, 4}, {256, 8, 8, 512}};
@@ -56,7 +56,7 @@ static std::vector<DepthwiseConvolution2DSpec> GetConv2DTestCases() {
     int64 kernel_size = option[2];
     int64 batch = option[0];
 
-    DepthwiseConvolution2DSpec config;
+    BatchGroupedConvolution2DSpec config;
     config.window_dilation = 1;
     config.output_batch = feature;
     config.window = kernel_size;
@@ -72,7 +72,7 @@ static std::vector<DepthwiseConvolution2DSpec> GetConv2DTestCases() {
 
     // Add configurations for window dilation cases.
     if (activation_size % 2 == 0 && activation_size == kernel_size) {
-      DepthwiseConvolution2DSpec config;
+      BatchGroupedConvolution2DSpec config;
       config.window_dilation = 2;
       config.output_batch = feature;
       config.window = kernel_size / 2;
@@ -90,9 +90,9 @@ static std::vector<DepthwiseConvolution2DSpec> GetConv2DTestCases() {
   return config_set;
 }
 
-string DepthwiseConvolution2DTestDataToString(
+string BatchGroupedConvolution2DTestDataToString(
     const ::testing::TestParamInfo<
-        ::testing::tuple<DepthwiseConvolution2DSpec, bool>>& data) {
+        ::testing::tuple<BatchGroupedConvolution2DSpec, bool>>& data) {
   const auto& spec = ::testing::get<0>(data.param);
   const string data_type = GetFloatDataType(::testing::get<1>(data.param));
   string str = absl::StrCat(
@@ -105,8 +105,8 @@ string DepthwiseConvolution2DTestDataToString(
   return str;
 }
 
-string BuildHloTextDepthwiseConvolution2D(
-    const DepthwiseConvolution2DSpec& spec, bool use_bfloat16) {
+string BuildHloTextBatchGroupedConvolution2D(
+    const BatchGroupedConvolution2DSpec& spec, bool use_bfloat16) {
   const string data_type = GetFloatDataType(use_bfloat16);
   return absl::StrFormat(
       R"(
@@ -129,11 +129,11 @@ string BuildHloTextDepthwiseConvolution2D(
       spec.window_dilation, spec.output_batch);
 }
 
-XLA_TEST_P(DepthwiseConvolution2DTest, DoIt) {
-  const DepthwiseConvolution2DSpec& spec = ::testing::get<0>(GetParam());
+XLA_TEST_P(BatchGroupedConvolution2DTest, DoIt) {
+  const BatchGroupedConvolution2DSpec& spec = ::testing::get<0>(GetParam());
   bool use_bfloat16 = ::testing::get<1>(GetParam());
   const string hlo_text =
-      BuildHloTextDepthwiseConvolution2D(spec, use_bfloat16);
+      BuildHloTextBatchGroupedConvolution2D(spec, use_bfloat16);
 
   EXPECT_TRUE(RunAndCompare(hlo_text, ErrorSpec{0.01, 0.01},
                             [](HloModule* module) -> Status {
@@ -145,10 +145,11 @@ XLA_TEST_P(DepthwiseConvolution2DTest, DoIt) {
 }
 
 INSTANTIATE_TEST_CASE_P(
-    DepthwiseConvolution2DTestWithRandomIndices, DepthwiseConvolution2DTest,
+    BatchGroupedConvolution2DTestWithRandomIndices,
+    BatchGroupedConvolution2DTest,
     ::testing::Combine(::testing::ValuesIn(GetConv2DTestCases()),
                        ::testing::Bool()),
-    DepthwiseConvolution2DTestDataToString);
+    BatchGroupedConvolution2DTestDataToString);
 
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 5217790d44c721a857d1e099c5acbbdb56e27d1f Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Tue, 29 Jan 2019 14:23:05 -0800
Subject: [PATCH 1525/2345] Capture new logging level introduced by TRT 5.1

---
 tensorflow/compiler/tf2tensorrt/utils/trt_logger.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/compiler/tf2tensorrt/utils/trt_logger.cc b/tensorflow/compiler/tf2tensorrt/utils/trt_logger.cc
index c48bd6bf77..f454f55f2c 100644
--- a/tensorflow/compiler/tf2tensorrt/utils/trt_logger.cc
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_logger.cc
@@ -26,6 +26,9 @@ namespace tensorrt {
 void Logger::log(Severity severity, const char* msg) {
   // Suppress info-level messages
   switch (severity) {
+#if NV_TENSORRT_MAJOR >= 5 && NV_TENSORRT_MINOR >= 1
+    case Severity::kVERBOSE:
+#endif
     case Severity::kINFO: {  // Mark TRT info messages as debug!
       VLOG(2) << name_ << " " << msg;
       break;
-- 
GitLab


From bcf8586bd326e26f373f040843e92bab195a266a Mon Sep 17 00:00:00 2001
From: Guozhong Zhuang <guozhong.zhuang@intel.com>
Date: Tue, 29 Jan 2019 14:27:32 -0800
Subject: [PATCH 1526/2345] Remove a unnecessary DCHECK statement

---
 tensorflow/core/util/mkl_util.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/core/util/mkl_util.h b/tensorflow/core/util/mkl_util.h
index 8f341061d5..016cc98f33 100644
--- a/tensorflow/core/util/mkl_util.h
+++ b/tensorflow/core/util/mkl_util.h
@@ -2142,7 +2142,6 @@ class LRUCache {
     if (lru_list_.empty()) return false;
     string key = lru_list_.back();
     auto it = cache_.find(key);
-    DCHECK(it != cache_.end());
     lru_list_.pop_back();
     delete it->second.op;  // delete the object
     cache_.erase(it);
-- 
GitLab


From 94edaf8a79a6c2bccf701b2c59dfd5836df885ef Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Tue, 29 Jan 2019 14:27:09 -0800
Subject: [PATCH 1527/2345] Move input_output_buffer_alias to third_party

- Open sources the pass that automatically sets buffer alias config to achieve max buffer aliasing.

PiperOrigin-RevId: 231473601
---
 tensorflow/compiler/xla/service/BUILD         |  41 +++++
 .../service/hlo_input_output_alias_config.h   |   3 +-
 .../optimize_input_output_buffer_alias.cc     | 106 +++++++++++++
 .../optimize_input_output_buffer_alias.h      |  71 +++++++++
 ...optimize_input_output_buffer_alias_test.cc | 145 ++++++++++++++++++
 5 files changed, 365 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/compiler/xla/service/optimize_input_output_buffer_alias.cc
 create mode 100644 tensorflow/compiler/xla/service/optimize_input_output_buffer_alias.h
 create mode 100644 tensorflow/compiler/xla/service/optimize_input_output_buffer_alias_test.cc

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index e9a6ce2ade..b01234fe26 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -3749,6 +3749,47 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "optimize_input_output_buffer_alias",
+    srcs = ["optimize_input_output_buffer_alias.cc"],
+    hdrs = ["optimize_input_output_buffer_alias.h"],
+    deps = [
+        ":hlo",
+        ":hlo_pass",
+        "//tensorflow/compiler/xla:shape_tree",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:status",
+        "//tensorflow/compiler/xla:status_macros",
+        "//tensorflow/compiler/xla:statusor",
+        "//tensorflow/compiler/xla:util",
+        "//tensorflow/core:lib",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/strings:str_format",
+    ],
+)
+
+tf_cc_test(
+    name = "optimize_input_output_buffer_alias_test",
+    srcs = ["optimize_input_output_buffer_alias_test.cc"],
+    deps = [
+        ":optimize_input_output_buffer_alias",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:status_macros",
+        "//tensorflow/compiler/xla:test",
+        "//tensorflow/compiler/xla:test_helpers",
+        "//tensorflow/compiler/xla:util",
+        "//tensorflow/compiler/xla/tests:hlo_test_base",
+        "//tensorflow/compiler/xla/tests:test_utils",
+        "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:test",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/strings:str_format",
+    ],
+)
+
 cc_library(
     name = "ar_crs_combiner",
     srcs = ["ar_crs_combiner.cc"],
diff --git a/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h b/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h
index b0b71dece8..cd13c7a3ac 100644
--- a/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h
+++ b/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h
@@ -56,7 +56,8 @@ class HloInputOutputAliasConfig {
 
   HloInputOutputAliasConfig() = default;
 
-  explicit HloInputOutputAliasConfig(Shape shape) : alias_(shape) {}
+  explicit HloInputOutputAliasConfig(Shape output_shape)
+      : alias_(output_shape) {}
 
   virtual ~HloInputOutputAliasConfig() = default;
 
diff --git a/tensorflow/compiler/xla/service/optimize_input_output_buffer_alias.cc b/tensorflow/compiler/xla/service/optimize_input_output_buffer_alias.cc
new file mode 100644
index 0000000000..00e5a9186e
--- /dev/null
+++ b/tensorflow/compiler/xla/service/optimize_input_output_buffer_alias.cc
@@ -0,0 +1,106 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/compiler/xla/service/optimize_input_output_buffer_alias.h"
+
+#include <queue>
+#include <string>
+
+#include "absl/container/flat_hash_map.h"
+#include "absl/memory/memory.h"
+#include "absl/strings/str_format.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/status_macros.h"
+#include "tensorflow/compiler/xla/util.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace xla {
+namespace {
+
+// Returns true if the given shape is a non-nested tuple.
+bool IsNonNestedTuple(const Shape& shape) {
+  return shape.IsTuple() && !ShapeUtil::IsNestedTuple(shape);
+}
+
+}  // namespace
+
+StatusOr<bool> OptimizeInputOutputBufferAlias::Build(
+    const Shape& input_shape, const Shape& output_shape,
+    HloInputOutputAliasConfig* alias_config) {
+  bool changed = false;
+  TF_RET_CHECK(LayoutUtil::HasLayout(input_shape));
+  TF_RET_CHECK(LayoutUtil::HasLayout(output_shape));
+  VLOG(0) << "input_shape:" << input_shape.ToString();
+  VLOG(0) << "output_shape:" << output_shape.ToString();
+
+  // For all buffers defined by the parameter, build a map from the byte
+  // size to the list of the buffers of that size.
+  absl::flat_hash_map<int64, std::queue<ShapeIndex>> size_to_input_index;
+  ShapeUtil::ForEachSubshape(
+      input_shape, [&](const Shape& subshape, const ShapeIndex& index) {
+        if (subshape.IsTuple()) {
+          return;
+        }
+        int64 bytes = size_func_(subshape);
+        size_to_input_index[bytes].push(index);
+      });
+
+  // For each result buffer shape index, take the first unused parameter
+  // buffer that matches the size.
+  TF_RETURN_IF_ERROR(ShapeUtil::ForEachSubshapeWithStatus(
+      output_shape, [&](const Shape& subshape, const ShapeIndex& index) {
+        if (subshape.IsTuple()) {
+          return Status::OK();
+        }
+        int64 bytes = size_func_(subshape);
+
+        auto it = size_to_input_index.find(bytes);
+        if (it != size_to_input_index.end() && !it->second.empty()) {
+          changed = true;
+          const ShapeIndex& input_index = it->second.front();
+          const ShapeIndex& output_index = index;
+          if (!alias_config->ParameterHasAlias(0, input_index) &&
+              !alias_config->OutputHasAlias(output_index)) {
+            TF_RETURN_IF_ERROR(alias_config->SetUpAlias(
+                output_index, 0, input_index,
+                HloInputOutputAliasConfig::AliasKind::kSystemAlias));
+          }
+          VLOG(3) << "Set up alias from with param index "
+                  << it->second.front().ToString() << ", shape size " << bytes
+                  << " and result subshape "
+                  << ShapeUtil::HumanStringWithLayout(subshape) << " at index "
+                  << index.ToString();
+          it->second.pop();
+        }
+        return Status::OK();
+      }));
+  return changed;
+}
+
+StatusOr<bool> OptimizeInputOutputBufferAlias::Run(HloModule* module) {
+  // User buffer alias only work for modules with 1 parameter.
+  if (module->entry_computation()->num_parameters() != 1) {
+    return false;
+  }
+
+  HloInputOutputAliasConfig* alias_config =
+      &module->input_output_alias_config();
+
+  return Build(module->entry_computation()->parameter_instruction(0)->shape(),
+               module->entry_computation()->root_instruction()->shape(),
+               alias_config);
+}
+
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/optimize_input_output_buffer_alias.h b/tensorflow/compiler/xla/service/optimize_input_output_buffer_alias.h
new file mode 100644
index 0000000000..79ce468e97
--- /dev/null
+++ b/tensorflow/compiler/xla/service/optimize_input_output_buffer_alias.h
@@ -0,0 +1,71 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_OPTIMIZE_INPUT_OUTPUT_BUFFER_ALIAS_H_
+#define TENSORFLOW_COMPILER_XLA_SERVICE_OPTIMIZE_INPUT_OUTPUT_BUFFER_ALIAS_H_
+
+#include <memory>
+
+#include "absl/container/flat_hash_map.h"
+#include "tensorflow/compiler/xla/service/hlo_input_output_alias_config.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_pass_interface.h"
+#include "tensorflow/compiler/xla/shape_tree.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/status.h"
+#include "tensorflow/compiler/xla/statusor.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/gtl/flatmap.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace xla {
+
+// This pass opportunistically finds input and output buffers that can be
+// aliased, and writes the alias config into the HloModule.
+//
+// The input and the output buffers can be in any shape, and each output buffer
+// can alias with an input buffer with the same size. Each input buffer may only
+// alias with a single output buffer. For example, for the following parameter
+// and the output buffers,
+//
+//  Parameters : { P1(2MiB), P2(4MiB), P3(8MiB), P4(4MiB), P5(4MiB), ... }
+//  Outputs    : { O1(4MiB), O2(2MiB), O3(4MiB), O4(6MiB), O5(4MiB), ... }
+//
+// one potential aliasing would be (O1, P2), (O2, P1), (O3, P4), (O5, P5), ..
+class OptimizeInputOutputBufferAlias : public HloModulePass {
+  using ShapeSizeFunction = std::function<int64(const Shape&)>;
+
+ public:
+  OptimizeInputOutputBufferAlias(ShapeSizeFunction size_func)
+      : size_func_(size_func) {}
+  ~OptimizeInputOutputBufferAlias() override = default;
+
+  absl::string_view name() const override {
+    return "optimize_input_output_buffer_alias.h";
+  }
+
+  StatusOr<bool> Run(HloModule* module) override;
+
+ private:
+  friend class OptimizeInputOutputBufferAliasTest;
+
+  StatusOr<bool> Build(const Shape& input_shape, const Shape& output_shape,
+                       HloInputOutputAliasConfig* alias_config);
+  ShapeSizeFunction size_func_ = nullptr;
+};
+
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_OPTIMIZE_INPUT_OUTPUT_BUFFER_ALIAS_H_
diff --git a/tensorflow/compiler/xla/service/optimize_input_output_buffer_alias_test.cc b/tensorflow/compiler/xla/service/optimize_input_output_buffer_alias_test.cc
new file mode 100644
index 0000000000..41e90f9b69
--- /dev/null
+++ b/tensorflow/compiler/xla/service/optimize_input_output_buffer_alias_test.cc
@@ -0,0 +1,145 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/optimize_input_output_buffer_alias.h"
+
+#include <memory>
+
+#include "absl/memory/memory.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/test.h"
+#include "tensorflow/compiler/xla/test_helpers.h"
+#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+#include "tensorflow/compiler/xla/tests/test_utils.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace xla {
+
+// Tests that UserBufferAlias properly maps input and output buffer indices of
+// various shapes for aliasing.
+class OptimizeInputOutputBufferAliasTest : public HloTestBase {
+ protected:
+  OptimizeInputOutputBufferAliasTest() {
+    r1f32_ = ShapeUtil::MakeShape(F32, {4});
+    r2f32_ = ShapeUtil::MakeShape(F32, {4, 5});
+    r3f32_ = ShapeUtil::MakeShape(F32, {4, 5, 6});
+    r4f32_ = ShapeUtil::MakeShape(F32, {4, 5, 6, 7});
+
+    auto size_func = [](const Shape& shape) {
+      return ShapeUtil::ByteSizeOf(shape);
+    };
+
+    optimize_pass_ =
+        absl::make_unique<OptimizeInputOutputBufferAlias>(size_func);
+  }
+
+  // Returns the number of output indices that aliases with the input.
+  int64 AliasCount() {
+    int64 count = 0;
+
+    config_.ForEachAlias(
+        [&](const ShapeIndex&, const HloInputOutputAliasConfig::Alias&) {
+          count++;
+        });
+    return count;
+  }
+
+  bool BuildAliasConfig(const Shape& input_shape, const Shape& output_shape) {
+    config_ = HloInputOutputAliasConfig(output_shape);
+    auto changed = optimize_pass_->Build(input_shape, output_shape, &config_);
+    TF_CHECK_OK(changed.status());
+
+    return changed.ValueOrDie();
+  }
+
+  std::unique_ptr<OptimizeInputOutputBufferAlias> optimize_pass_;
+
+  HloInputOutputAliasConfig config_;
+
+  Shape r1f32_;
+  Shape r2f32_;
+  Shape r3f32_;
+  Shape r4f32_;
+};
+
+// All shapes are different, so no aliasing is available.
+TEST_F(OptimizeInputOutputBufferAliasTest, AllDifferentBufferSizes) {
+  Shape input = ShapeUtil::MakeTupleShape({r1f32_, r2f32_});
+  Shape output = ShapeUtil::MakeTupleShape({r3f32_, r4f32_});
+  bool changed = BuildAliasConfig(input, output);
+  EXPECT_FALSE(changed);
+  EXPECT_EQ(AliasCount(), 0);
+}
+
+// Input and output shapes are equal, so buffers can alias at the same index.
+TEST_F(OptimizeInputOutputBufferAliasTest, OrderedNonNestedTuple) {
+  Shape input = ShapeUtil::MakeTupleShape({r1f32_, r2f32_, r3f32_, r4f32_});
+  Shape output = ShapeUtil::MakeTupleShape({r1f32_, r2f32_, r3f32_, r4f32_});
+  bool changed = BuildAliasConfig(input, output);
+  EXPECT_TRUE(changed);
+  EXPECT_EQ(AliasCount(), 4);
+
+  EXPECT_EQ(config_.GetAliasedOutput(0, {0}), ShapeIndex{0});
+  EXPECT_EQ(config_.GetAliasedOutput(0, {1}), ShapeIndex{1});
+  EXPECT_EQ(config_.GetAliasedOutput(0, {2}), ShapeIndex{2});
+  EXPECT_EQ(config_.GetAliasedOutput(0, {3}), ShapeIndex{3});
+}
+
+// Only a subset of the tuple element shapes match between the input and the
+// output.
+TEST_F(OptimizeInputOutputBufferAliasTest, PartialReuseNonNestedTuple) {
+  Shape input = ShapeUtil::MakeTupleShape({r1f32_, r1f32_, r2f32_, r2f32_});
+  Shape output = ShapeUtil::MakeTupleShape({r1f32_, r2f32_, r3f32_, r4f32_});
+  bool changed = BuildAliasConfig(input, output);
+  EXPECT_TRUE(changed);
+
+  EXPECT_EQ(AliasCount(), 2);
+
+  EXPECT_EQ(config_.GetAliasedOutput(0, {0}), ShapeIndex{0});
+  EXPECT_EQ(config_.GetAliasedOutput(0, {2}), ShapeIndex{1});
+}
+
+// The output shape is reverse of the input shape, but we can still reuse all
+// the buffers.
+TEST_F(OptimizeInputOutputBufferAliasTest, UnorderedNonNestedTuple) {
+  Shape input = ShapeUtil::MakeTupleShape({r1f32_, r2f32_, r3f32_, r4f32_});
+  Shape output = ShapeUtil::MakeTupleShape({r4f32_, r3f32_, r2f32_, r1f32_});
+  bool changed = BuildAliasConfig(input, output);
+  EXPECT_TRUE(changed);
+
+  EXPECT_EQ(AliasCount(), 4);
+
+  EXPECT_EQ(config_.GetAliasedOutput(0, {0}), ShapeIndex{3});
+  EXPECT_EQ(config_.GetAliasedOutput(0, {1}), ShapeIndex{2});
+  EXPECT_EQ(config_.GetAliasedOutput(0, {2}), ShapeIndex{1});
+  EXPECT_EQ(config_.GetAliasedOutput(0, {3}), ShapeIndex{0});
+}
+
+TEST_F(OptimizeInputOutputBufferAliasTest, UnorderedNestedTuple) {
+  Shape input = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeTupleShape({r1f32_}), r2f32_, r3f32_, r4f32_});
+  Shape output = ShapeUtil::MakeTupleShape(
+      {r1f32_, ShapeUtil::MakeTupleShape({r3f32_, r2f32_}), r2f32_});
+  bool changed = BuildAliasConfig(input, output);
+  EXPECT_TRUE(changed);
+
+  EXPECT_EQ(AliasCount(), 3);
+
+  EXPECT_EQ(config_.GetAliasedOutput(0, {0, 0}), ShapeIndex{0});
+  EXPECT_EQ(config_.GetAliasedOutput(0, {1}), ShapeIndex({1, 1}));
+  EXPECT_EQ(config_.GetAliasedOutput(0, {2}), ShapeIndex({1, 0}));
+}
+
+}  // namespace xla
-- 
GitLab


From baafd483600673d5f935b3df46493e34f34bf494 Mon Sep 17 00:00:00 2001
From: Guozhong Zhuang <guozhong.zhuang@intel.com>
Date: Tue, 29 Jan 2019 14:35:49 -0800
Subject: [PATCH 1528/2345] Upcase the lead character of a statement

---
 tensorflow/core/util/mkl_util.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/util/mkl_util.h b/tensorflow/core/util/mkl_util.h
index 016cc98f33..5abe34cba2 100644
--- a/tensorflow/core/util/mkl_util.h
+++ b/tensorflow/core/util/mkl_util.h
@@ -2148,7 +2148,7 @@ class LRUCache {
     return true;
   }
 
-  // cache capacity
+  // Cache capacity
   size_t capacity_;
 
   // The cache, a map from string key to a LRU entry.
-- 
GitLab


From 5870add5d332b8bb1c69f0f1f64797649ccd8902 Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Tue, 29 Jan 2019 14:46:11 -0800
Subject: [PATCH 1529/2345] [tf.data] Add `ChooseFastestDataset` behind map
 vectorization optimization.

PiperOrigin-RevId: 231477334
---
 .../optimizers/data/map_vectorization.cc      |  38 ++++-
 .../optimizers/data/map_vectorization.h       |  11 ++
 .../optimizers/data/map_vectorization_test.cc | 147 ++++++++++++++----
 .../optimization/map_vectorization_test.py    |  10 +-
 4 files changed, 161 insertions(+), 45 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
index 3da238fb7d..13933b6947 100644
--- a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
@@ -50,6 +50,7 @@ constexpr char kBatchV2Op[] = "BatchDatasetV2";
 constexpr char kExperimentalMapAndBatchOp[] = "ExperimentalMapAndBatchDataset";
 constexpr char kMapOp[] = "MapDataset";
 constexpr char kParallelMapOp[] = "ParallelMapDataset";
+constexpr char kChooseFastestOp[] = "ExperimentalChooseFastestDataset";
 constexpr int kAutotune = -1;
 
 // Returns a FunctionDef containing a MapDefun op that wraps the original
@@ -445,6 +446,29 @@ Status AddNewMapNode(const NodeDef& old_map_node, const NodeDef& old_batch_node,
   return Status::OK();
 }
 
+Status AddNewChooseFastestNode(gtl::ArraySlice<NodeDef> input_nodes,
+                               MutableGraphView* graph,
+                               NodeDef** new_choose_fastest_node) {
+  NodeDef choose_fastest_node;
+  choose_fastest_node.set_op(kChooseFastestOp);
+  graph_utils::SetUniqueGraphNodeName(choose_fastest_node.op(), graph->graph(),
+                                      &choose_fastest_node);
+
+  // Set the `input_datasets` input argument.
+  for (const auto& node_def : input_nodes) {
+    choose_fastest_node.add_input(node_def.name());
+  }
+  AddNodeAttr("N", static_cast<int>(input_nodes.size()), &choose_fastest_node);
+  AddNodeAttr("num_experiments", 10, &choose_fastest_node);
+
+  for (auto key : {"output_shapes", "output_types"}) {
+    graph_utils::CopyAttribute(key, input_nodes[0], &choose_fastest_node);
+  }
+
+  *new_choose_fastest_node = graph->AddNode(std::move(choose_fastest_node));
+  return Status::OK();
+}
+
 // Given an input pipeline graph and a query node, tries to the node to the
 // 'batch' node in a input_dataset->map->batch pattern, or the 'map_and_batch'
 // node in an input_dataset->map_and_batch pattern.
@@ -537,15 +561,15 @@ Status MapVectorization::OptimizeAndCollectStats(Cluster* cluster,
     TF_RETURN_IF_ERROR(AddNewMapNode(*map_node, *batch_node, *new_batch_node,
                                      *vectorized_func, &graph, &new_map_node));
 
-    // Make output of Batch point to Map instead.
-    TF_RETURN_IF_ERROR(
-        graph.UpdateFanouts(batch_node->name(), new_map_node->name()));
-    // Mark the `Map` and `Batch` nodes for removal.
-    nodes_to_delete.insert(map_node->name());
-    nodes_to_delete.insert(batch_node->name());
+    NodeDef* new_choose_fastest_node;
+    TF_RETURN_IF_ERROR(AddNewChooseFastestNode(
+        {*new_map_node, *batch_node}, &graph, &new_choose_fastest_node));
+
+    // Make output of Batch point to ChooseFastest instead.
+    TF_RETURN_IF_ERROR(graph.UpdateFanouts(batch_node->name(),
+                                           new_choose_fastest_node->name()));
     stats->num_changes++;
   }
-  TF_RETURN_IF_ERROR(graph.DeleteNodes(nodes_to_delete));
   return Status::OK();
 }
 
diff --git a/tensorflow/core/grappler/optimizers/data/map_vectorization.h b/tensorflow/core/grappler/optimizers/data/map_vectorization.h
index 3e170aae27..88ec9cfec6 100644
--- a/tensorflow/core/grappler/optimizers/data/map_vectorization.h
+++ b/tensorflow/core/grappler/optimizers/data/map_vectorization.h
@@ -27,6 +27,17 @@ namespace grappler {
 // rewritten as dataset.batch().map(vectorized_map_fn). This is more performant
 // when the map_fn is cheap, because it amortizes the cost of running a map
 // function over a larger batch.
+//
+// From:
+//      input --> map --> batch --> output
+//              (or map_and_batch)
+//
+// To:
+//      input --> map --> batch --------+
+//        |     (or map_and_batch)      |
+//        |                             v
+//        +-----> batch --> map --> choose_fastest --> output
+//
 class MapVectorization : public TFDataOptimizerBase {
  public:
   MapVectorization() = default;
diff --git a/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc b/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc
index 0565d5f643..30385c7d45 100644
--- a/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc
@@ -38,6 +38,7 @@ constexpr char kBatchV2Op[] = "BatchDatasetV2";
 constexpr char kExperimentalMapAndBatchOp[] = "ExperimentalMapAndBatchDataset";
 constexpr char kMapOp[] = "MapDataset";
 constexpr char kParallelMapOp[] = "ParallelMapDataset";
+constexpr char kChooseFastestOp[] = "ExperimentalChooseFastestDataset";
 constexpr char kAttrNameF[] = "f";
 constexpr char kAttrNameTarguments[] = "Targuments";
 constexpr char kAttrNameOutputTypes[] = "output_types";
@@ -156,20 +157,92 @@ void CheckNotVectorized(const GraphDef& output, const string& map_op,
   EXPECT_EQ(batch_node.input(0), map_node.name());
 }
 
+void CheckBranch(const GraphDef& graph, string input_name,
+                 gtl::ArraySlice<string> ops, const string& terminal_input) {
+  for (int i = 0, size = ops.size(); i < size; ++i) {
+    const NodeDef& input_node =
+        graph.node(graph_utils::FindGraphNodeWithName(input_name, graph));
+    EXPECT_EQ(input_node.op(), ops[size - i - 1]);
+    input_name = input_node.input(0);
+  }
+  EXPECT_EQ(input_name, terminal_input);
+}
+
+// Checks that a graph has undergone the map_vectorization transformation
+// successfully, whereby the new graph has the shape:
+//
+//    input_node --> new batch --> new map -------+
+//         |                                      |
+//         |                                      v
+//         +-------> old map --> old batch ---> choose_fastest
+//
 void CheckVectorized(const GraphDef& output, const string& map_op,
-                     const string& batch_op, const string& map_input_name) {
-  ASSERT_EQ(graph_utils::FindAllGraphNodesWithOp(map_op, output).size(), 1);
-  ASSERT_EQ(graph_utils::FindAllGraphNodesWithOp(batch_op, output).size(), 1);
-  const NodeDef& map_node =
-      output.node(graph_utils::FindGraphNodeWithOp(map_op, output));
-  const NodeDef& batch_node =
-      output.node(graph_utils::FindGraphNodeWithOp(batch_op, output));
-  EXPECT_EQ(map_node.input(0), batch_node.name());
-  EXPECT_EQ(batch_node.input(0), map_input_name);
+                     const string& batch_op, const string& map_input_name,
+                     bool fused = false) {
+  ASSERT_EQ(graph_utils::FindAllGraphNodesWithOp(map_op, output).size(), 2);
+  ASSERT_EQ(graph_utils::FindAllGraphNodesWithOp(batch_op, output).size(), 2);
+  ASSERT_EQ(
+      graph_utils::FindAllGraphNodesWithOp(kChooseFastestOp, output).size(), 1);
+  const NodeDef& choose_fastest_node =
+      output.node(graph_utils::FindGraphNodeWithOp(kChooseFastestOp, output));
+
+  // Branch 0: vectorized
+  CheckBranch(output, choose_fastest_node.input(0), {batch_op, map_op},
+              map_input_name);
+
+  // Branch 1: original
+  CheckBranch(output, choose_fastest_node.input(1), {map_op, batch_op},
+              map_input_name);
+
+  const NodeDef& vectorized_map_node = output.node(
+      graph_utils::FindGraphNodeWithName(choose_fastest_node.input(0), output));
+  // Check that the function is actually vectorized.
+  // The vectorization of the identity function is itself.
+  string function_name =
+      vectorized_map_node.attr().at(kAttrNameF).func().name();
+  int found =
+      graph_utils::FindGraphFunctionWithName(function_name, output.library());
+  ASSERT_NE(found, -1);
+  const auto& function = output.library().function(found);
+  EXPECT_EQ(function.node_def(0).op(), "Identity");
+}
 
+// Checks that a graph has undergone the map_vectorization transformation
+// successfully, whereby the new graph has the shape:
+//
+//    input_node --> new batch -> new map --------+
+//         |                                      |
+//         |                                      v
+//         +-------> old map_and_batch ---> choose_fastest
+//
+void CheckVectorizedFused(const GraphDef& output,
+                          const string& map_input_name) {
+  ASSERT_EQ(graph_utils::FindAllGraphNodesWithOp(kParallelMapOp, output).size(),
+            1);
+  ASSERT_EQ(graph_utils::FindAllGraphNodesWithOp(kBatchV2Op, output).size(), 1);
+  ASSERT_EQ(
+      graph_utils::FindAllGraphNodesWithOp(kExperimentalMapAndBatchOp, output)
+          .size(),
+      1);
+  ASSERT_EQ(
+      graph_utils::FindAllGraphNodesWithOp(kChooseFastestOp, output).size(), 1);
+  const NodeDef& choose_fastest_node =
+      output.node(graph_utils::FindGraphNodeWithOp(kChooseFastestOp, output));
+
+  // Branch 0: vectorized
+  CheckBranch(output, choose_fastest_node.input(0),
+              {kBatchV2Op, kParallelMapOp}, map_input_name);
+
+  // Branch 1: original
+  CheckBranch(output, choose_fastest_node.input(1),
+              {kExperimentalMapAndBatchOp}, map_input_name);
+
+  const NodeDef& vectorized_map_node = output.node(
+      graph_utils::FindGraphNodeWithName(choose_fastest_node.input(0), output));
   // Check that the function is actually vectorized.
   // The vectorization of the identity function is itself.
-  string function_name = map_node.attr().at(kAttrNameF).func().name();
+  string function_name =
+      vectorized_map_node.attr().at(kAttrNameF).func().name();
   int found =
       graph_utils::FindGraphFunctionWithName(function_name, output.library());
   ASSERT_NE(found, -1);
@@ -238,7 +311,7 @@ TEST(MapVectorizationTest, VectorizeExperimentalMapAndBatch) {
   MapVectorization optimizer;
   GraphDef output;
   TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output));
-  CheckVectorized(output, kParallelMapOp, kBatchV2Op, "range");
+  CheckVectorizedFused(output, "range");
 }
 
 void EvaluateNodes(const GraphDef& graph,
@@ -252,9 +325,11 @@ void EvaluateNodes(const GraphDef& graph,
 void CheckNumParallelCalls(const GraphDef& output,
                            int expected_num_parallel_calls) {
   // Run the graph to see that the new num_parallel_calls is computed correctly.
-  const NodeDef& map_node =
-      output.node(graph_utils::FindGraphNodeWithOp(kParallelMapOp, output));
-  const string& num_parallel_calls = map_node.input(1);
+  const NodeDef& choose_fastest_node =
+      output.node(graph_utils::FindGraphNodeWithOp(kChooseFastestOp, output));
+  const NodeDef& vectorized_map_node = output.node(
+      graph_utils::FindGraphNodeWithName(choose_fastest_node.input(0), output));
+  const string& num_parallel_calls = vectorized_map_node.input(1);
   std::vector<Tensor> output_tensors;
   EvaluateNodes(output, {num_parallel_calls}, &output_tensors);
 
@@ -306,7 +381,7 @@ TEST_P(NumParallelCallsTest, TestCorrectNumParallelCallsFused) {
   MapVectorization optimizer;
   GraphDef output;
   TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output));
-  CheckVectorized(output, kParallelMapOp, kBatchV2Op, range_node->name());
+  CheckVectorizedFused(output, range_node->name());
 
   CheckNumParallelCalls(output, params.expected_num_parallel_calls);
 }
@@ -343,33 +418,37 @@ TEST_P(ChainedMapAndBatchTest, IsVectorized) {
     return batch_node;
   };
 
-  auto map_and_batch_0 =
-      make_map_and_batch(input_node, std::get<0>(GetParam()));
-  auto map_and_batch_1 =
-      make_map_and_batch(map_and_batch_0, std::get<1>(GetParam()));
+  bool fuse_0 = std::get<0>(GetParam());
+  bool fuse_1 = std::get<1>(GetParam());
+  auto map_and_batch_0 = make_map_and_batch(input_node, fuse_0);
+  auto map_and_batch_1 = make_map_and_batch(map_and_batch_0, fuse_1);
   ASSERT_NE(map_and_batch_1, nullptr);
 
   MapVectorization optimizer;
   GraphDef output;
   TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output));
   TF_ASSERT_OK(TopologicalSort(&output));
-  std::vector<int> map_nodes =
-      graph_utils::FindAllGraphNodesWithOp(kParallelMapOp, output);
-  std::vector<int> batch_nodes =
-      graph_utils::FindAllGraphNodesWithOp(kBatchV2Op, output);
-  ASSERT_EQ(map_nodes.size(), 2);
-  ASSERT_EQ(batch_nodes.size(), 2);
+
+  std::vector<int> choose_fastest_nodes =
+      graph_utils::FindAllGraphNodesWithOp(kChooseFastestOp, output);
+  ASSERT_EQ(choose_fastest_nodes.size(), 2);
+
+  std::vector<string> fused_sequence({kExperimentalMapAndBatchOp});
+  std::vector<string> unfused_sequence({kParallelMapOp, kBatchV2Op});
   const NodeDef& range_node =
       output.node(graph_utils::FindGraphNodeWithOp(kRangeOp, output));
-
-  const NodeDef& batch_node_0 = output.node(batch_nodes[0]);
-  EXPECT_EQ(batch_node_0.input(0), range_node.name());
-  const NodeDef& map_node_0 = output.node(map_nodes[0]);
-  EXPECT_EQ(map_node_0.input(0), batch_node_0.name());
-  const NodeDef& batch_node_1 = output.node(batch_nodes[1]);
-  EXPECT_EQ(batch_node_1.input(0), map_node_0.name());
-  const NodeDef& map_node_1 = output.node(map_nodes[1]);
-  EXPECT_EQ(map_node_1.input(0), batch_node_1.name());
+  const NodeDef& choose_fastest_0 = output.node(choose_fastest_nodes[0]);
+  CheckBranch(output, choose_fastest_0.input(0), {kBatchV2Op, kParallelMapOp},
+              range_node.name());
+  CheckBranch(output, choose_fastest_0.input(1),
+              fuse_0 ? fused_sequence : unfused_sequence, range_node.name());
+
+  const NodeDef& choose_fastest_1 = output.node(choose_fastest_nodes[1]);
+  CheckBranch(output, choose_fastest_1.input(0), {kBatchV2Op, kParallelMapOp},
+              choose_fastest_0.name());
+  CheckBranch(output, choose_fastest_1.input(1),
+              fuse_1 ? fused_sequence : unfused_sequence,
+              choose_fastest_0.name());
 }
 
 INSTANTIATE_TEST_SUITE_P(ChainedMapAndBatchTest, ChainedMapAndBatchTest,
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
index 5e748a78ef..ef4dbc8a71 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
@@ -356,8 +356,10 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
       return dataset
 
     unoptimized = _make_dataset([map_node_name, "Batch"])
-    optimized = _make_dataset(["Batch", map_node_name]
-                              if expect_optimized else [map_node_name, "Batch"])
+    # Note that because of the `ChooseDataset` fork, we can't use `assert_next`
+    # to verify the optimization result.
+    optimized = _make_dataset(
+        [] if expect_optimized else [map_node_name, "Batch"])
     options = dataset_ops.Options()
     options.experimental_optimization.apply_default_optimizations = False
     options.experimental_optimization.map_vectorization = True
@@ -418,7 +420,7 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
       return dataset
 
     unoptimized = _make_dataset(["MapAndBatch"])
-    optimized = _make_dataset(["Batch", "ParallelMap"])
+    optimized = _make_dataset([])
     options = dataset_ops.Options()
     options.experimental_optimization.map_vectorization = True
     optimized = optimized.with_options(options)
@@ -469,7 +471,7 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
       return dataset
 
     unoptimized = make_dataset(unoptimized_seq)
-    optimized = make_dataset(["Batch", "ParallelMap", "Batch", "ParallelMap"])
+    optimized = make_dataset([])
     options = dataset_ops.Options()
     options.experimental_optimization.map_vectorization = True
     optimized = optimized.with_options(options)
-- 
GitLab


From cab792dae6e2622990671d7ac4b92d584645beaf Mon Sep 17 00:00:00 2001
From: Smit Hinsu <hinsu@google.com>
Date: Tue, 29 Jan 2019 14:52:43 -0800
Subject: [PATCH 1530/2345] Use ScopedUnref in convert_graph.cc to unref
 resource on error

PiperOrigin-RevId: 231478532
---
 tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc
index 1fdb099cc1..8b902dd90e 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc
@@ -212,6 +212,7 @@ tensorflow::Status ConvertCalibGraphToInferGraph(
         return tensorflow::errors::FailedPrecondition(
             "Need to run graph with calibration data first!");
       }
+      tensorflow::core::ScopedUnref calib_sc(cres);
       if (cres->calibrator_) {
         cres->calibrator_->waitAndSetDone();
         cres->thr_->join();
@@ -228,7 +229,6 @@ tensorflow::Status ConvertCalibGraphToInferGraph(
         return tensorflow::errors::Unknown(
             "Can't get TRTCalibrator from resource manager!");
       }
-      cres->Unref();
       TF_RETURN_IF_ERROR(calib_rm->Cleanup(container_name));
     }
   }
-- 
GitLab


From 6cd5f063a129a2cbcd58f82662d6a4688bb2fbb0 Mon Sep 17 00:00:00 2001
From: Cong Liu <congliu@google.com>
Date: Tue, 29 Jan 2019 14:54:30 -0800
Subject: [PATCH 1531/2345] [XLA] Allow dynamism mismatch in degenerate
 broadcast.

For example, in add(f32[<=x,1], f32[1,<=y]), the size-1 dimensions don't have to
be dynamic.

PiperOrigin-RevId: 231478874
---
 .../compiler/xla/client/xla_builder_test.cc   | 21 +++++++++++++++++++
 .../compiler/xla/service/shape_inference.cc   | 13 ++++++++----
 2 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_builder_test.cc b/tensorflow/compiler/xla/client/xla_builder_test.cc
index 743434cba8..368a9452ca 100644
--- a/tensorflow/compiler/xla/client/xla_builder_test.cc
+++ b/tensorflow/compiler/xla/client/xla_builder_test.cc
@@ -590,6 +590,27 @@ TEST_F(XlaBuilderTest, DynamicBroadcast) {
       << result_shape;
 }
 
+TEST_F(XlaBuilderTest, DynamicBinaryHasDegenerateBroadcast) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {10}), ShapeUtil::MakeShape(F32, {1, 15}),
+       ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{1},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/0));
+  auto gte0 = GetTupleElement(p0, 0);
+  auto gte1 = GetTupleElement(p0, 1);
+  Add(gte0, gte1, /*broadcast_dimensions=*/{0});  // f32[<=10, 15]
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  const Shape& result_shape =
+      module->entry_computation()->root_instruction()->shape();
+  EXPECT_TRUE(ContainersEqual(result_shape.dynamic_dimensions(), {true, false}))
+      << result_shape;
+}
+
 TEST_F(XlaBuilderTest, DynamicPad) {
   XlaBuilder b(TestName());
   Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index 946577d55d..4680c924cc 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -832,10 +832,15 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
           ShapeUtil::HumanString(larger_shape));
     }
     if (small_is_dynamic != large_is_dynamic) {
-      return InvalidArgument(
-          "Broadcast dimension %d dynamism mismatch: %s and %s.", i,
-          ShapeUtil::HumanString(smaller_shape),
-          ShapeUtil::HumanString(larger_shape));
+      if ((small_dimension_size == 1 && !small_is_dynamic) ||
+          (large_dimension_size == 1 && !large_is_dynamic)) {
+        // Do nothing. It's OK when the size-1 dimension is not static.
+      } else {
+        return InvalidArgument(
+            "Broadcast dimension %d dynamism mismatch: %s and %s.", i,
+            ShapeUtil::HumanString(smaller_shape),
+            ShapeUtil::HumanString(larger_shape));
+      }
     }
     // Make sure the broadcast dimensions are listed in a strictly increasing
     // order.
-- 
GitLab


From fc1c990df1d44eb8a72d7f934f951571420721f7 Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Tue, 29 Jan 2019 14:56:34 -0800
Subject: [PATCH 1532/2345] [XLA] Enable some reduce simplifications for
 multi-output reduce.

PiperOrigin-RevId: 231479222
---
 .../xla/service/algebraic_simplifier.cc       | 94 +++++++++++++------
 .../xla/service/algebraic_simplifier_test.cc  | 68 ++++++++++++++
 2 files changed, 132 insertions(+), 30 deletions(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index da15ff7d7a..e456b92500 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -2714,28 +2714,72 @@ Status AlgebraicSimplifierVisitor::HandleDynamicUpdateSlice(
   return Status::OK();
 }
 
-Status AlgebraicSimplifierVisitor::HandleReduce(HloInstruction* reduce) {
-  // TODO(b/112040122): Most of those optimizations can be done for multi-output
-  // reduces.
-  if (reduce->shape().IsTuple()) {
-    return Status::OK();
-  }
+Status AlgebraicSimplifierVisitor::HandleReduce(HloInstruction* hlo) {
+  HloReduceInstruction* reduce = Cast<HloReduceInstruction>(hlo);
+  bool multi_output_reduce = reduce->shape().IsTuple();
+
+  // For tuple reduce, we require all reduce shapes to be the same, up to the
+  // element types, so we can just the first operand and the first result as a
+  // representative.
+  auto arg = reduce->inputs()[0];
+  auto init_value = reduce->init_values()[0];
+  const Shape& reduce_result_shape =
+      multi_output_reduce ? reduce->shape().tuple_shapes(0) : reduce->shape();
 
-  auto arg = reduce->mutable_operand(0);
-  auto init_value = reduce->mutable_operand(1);
   absl::Span<const int64> dimensions(reduce->dimensions());
   HloComputation* function = reduce->to_apply();
   if (ShapeUtil::IsZeroElementArray(arg->shape()) ||
-      ShapeUtil::IsZeroElementArray(reduce->shape())) {
-    return ReplaceWithNewInstruction(
-        reduce,
-        HloInstruction::CreateBroadcast(reduce->shape(), init_value, {}));
+      ShapeUtil::IsZeroElementArray(reduce_result_shape)) {
+    if (multi_output_reduce) {
+      std::vector<HloInstruction*> broadcast_inits;
+      int64 inputs = reduce->input_count();
+      for (int64 i = 0; i < inputs; ++i) {
+        broadcast_inits.push_back(computation_->AddInstruction(
+            HloInstruction::CreateBroadcast(reduce->shape().tuple_shapes(i),
+                                            reduce->init_values()[i], {})));
+      }
+      return ReplaceWithNewInstruction(
+          reduce, HloInstruction::CreateTuple(broadcast_inits));
+    } else {
+      return ReplaceWithNewInstruction(
+          reduce,
+          HloInstruction::CreateBroadcast(reduce_result_shape, init_value, {}));
+    }
+  }
+
+  // If the reduction results in the same number of elements, then the only
+  // possible side effect would be a reshape. Since the init_value is an
+  // identity of the reduction function, we can therefore replace the reduce
+  // with a simple reshape, ignoring the reduction function completely.
+  if (ShapeUtil::ElementsIn(reduce_result_shape) ==
+      ShapeUtil::ElementsIn(arg->shape())) {
+    if (multi_output_reduce) {
+      std::vector<HloInstruction*> reshaped_args;
+      int64 inputs = reduce->input_count();
+      for (int64 i = 0; i < inputs; ++i) {
+        reshaped_args.push_back(
+            computation_->AddInstruction(HloInstruction::CreateReshape(
+                reduce->shape().tuple_shapes(i), reduce->inputs()[i])));
+      }
+      return ReplaceWithNewInstruction(
+          reduce, HloInstruction::CreateTuple(reshaped_args));
+    } else {
+      return ReplaceWithNewInstruction(
+          reduce, HloInstruction::CreateReshape(reduce_result_shape, arg));
+    }
+  }
+
+  // TODO(b/112040122): Most of those optimizations below can be done for
+  // multi-output reduces.
+  if (multi_output_reduce) {
+    return Status::OK();
   }
 
   // A Transpose feeding a reduce can simply permute the reduction dimensions
   // field if the output of the reduce is a vector or scalar. Higher ranked
   // result may require a transpose of the output.
-  if (reduce->shape().rank() <= 1 && arg->opcode() == HloOpcode::kTranspose) {
+  if (reduce_result_shape.rank() <= 1 &&
+      arg->opcode() == HloOpcode::kTranspose) {
     auto transpose_dimensions = arg->dimensions();
     std::vector<int64> new_reduce_dimensions;
     for (auto dim : dimensions) {
@@ -2743,20 +2787,10 @@ Status AlgebraicSimplifierVisitor::HandleReduce(HloInstruction* reduce) {
     }
     return ReplaceWithNewInstruction(
         reduce, HloInstruction::CreateReduce(
-                    reduce->shape(), arg->mutable_operand(0), init_value,
+                    reduce_result_shape, arg->mutable_operand(0), init_value,
                     new_reduce_dimensions, function));
   }
 
-  // If the reduction results in the same number of elements, then the only
-  // possible side effect would be a reshape. Since the init_value is an
-  // identity of the reduction function, we can therefore replace the reduce
-  // with a simple reshape, ignoring the reduction function completely.
-  if (ShapeUtil::ElementsIn(reduce->shape()) ==
-      ShapeUtil::ElementsIn(arg->shape())) {
-    return ReplaceWithNewInstruction(
-        reduce, HloInstruction::CreateReshape(reduce->shape(), arg));
-  }
-
   // If a reduce feeds a reduce with the same computation and initial value,
   // they can be combined into a single reduce.
   if (arg->opcode() == HloOpcode::kReduce &&
@@ -2782,9 +2816,9 @@ Status AlgebraicSimplifierVisitor::HandleReduce(HloInstruction* reduce) {
     std::merge(arg_dims.begin(), arg_dims.end(), reduce_dims.begin(),
                reduce_dims.end(), std::back_inserter(new_dimensions));
     return ReplaceWithNewInstruction(
-        reduce,
-        HloInstruction::CreateReduce(reduce->shape(), arg->mutable_operand(0),
-                                     init_value, new_dimensions, function));
+        reduce, HloInstruction::CreateReduce(
+                    reduce_result_shape, arg->mutable_operand(0), init_value,
+                    new_dimensions, function));
   }
 
   // A reshape that collapses multiple dimensions into a dimension being
@@ -2827,7 +2861,7 @@ Status AlgebraicSimplifierVisitor::HandleReduce(HloInstruction* reduce) {
       }
       return ReplaceWithNewInstruction(
           reduce, HloInstruction::CreateReduce(
-                      reduce->shape(), arg->mutable_operand(0), init_value,
+                      reduce_result_shape, arg->mutable_operand(0), init_value,
                       new_reduce_dimensions, function));
     }
   }
@@ -2842,11 +2876,11 @@ Status AlgebraicSimplifierVisitor::HandleReduce(HloInstruction* reduce) {
     HloInstruction* old_reduce = nullptr;
     for (HloInstruction* operand : arg->operands()) {
       HloInstruction* new_reduce = computation_->AddInstruction(
-          HloInstruction::CreateReduce(reduce->shape(), operand, init_value,
+          HloInstruction::CreateReduce(reduce_result_shape, operand, init_value,
                                        reduce->dimensions(), function));
       if (old_reduce != nullptr) {
         new_reduce = computation_->AddInstruction(HloInstruction::CreateMap(
-            reduce->shape(), {old_reduce, new_reduce}, function));
+            reduce_result_shape, {old_reduce, new_reduce}, function));
       }
       old_reduce = new_reduce;
     }
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index 3602ab82b2..089312211e 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -4770,5 +4770,73 @@ INSTANTIATE_TEST_SUITE_P(
     DotOfGatherSimplificationTestInstantiation, DotOfGatherSimplificationTest,
     ::testing::ValuesIn(DotOfGatherPositiveNegativeTests()));
 
+TEST_F(AlgebraicSimplifierTest, TupleReduceReshape) {
+  const char* hlo_string = R"(
+HloModule module
+
+reducer {
+  parameter.1 = f32[] parameter(0)
+  parameter.3 = f32[] parameter(2)
+  add.2 = f32[] add(parameter.1, parameter.3)
+  parameter.0 = f32[] parameter(1)
+  parameter.2 = f32[] parameter(3)
+  add.3 = f32[] add(parameter.0, parameter.2)
+  ROOT tuple.4 = (f32[], f32[]) tuple(add.2, add.3)
+}
+
+ENTRY entry {
+  parameter.6 = (f32[], f32[]) parameter(0)
+  get-tuple-element.10 = f32[] get-tuple-element(parameter.6), index=0
+  get-tuple-element.11 = f32[] get-tuple-element(parameter.6), index=1
+  constant = f32[] constant(0)
+  ROOT reduce = (f32[], f32[]) reduce(get-tuple-element.10, get-tuple-element.11, constant, constant), dimensions={}, to_apply=reducer
+}
+)";
+  TF_ASSERT_OK_AND_ASSIGN(auto module,
+                          ParseAndReturnVerifiedModule(hlo_string));
+
+  AlgebraicSimplifierOptions options;
+  AlgebraicSimplifier simplifier(options);
+  EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
+  auto root = module->entry_computation()->root_instruction();
+  EXPECT_THAT(root, GmockMatch(m::Tuple(
+                        m::Reshape(m::GetTupleElement(m::Parameter(), 0)),
+                        m::Reshape(m::GetTupleElement(m::Parameter(), 1)))));
+}
+
+TEST_F(AlgebraicSimplifierTest, TupleReduceBroadcast) {
+  const char* hlo_string = R"(
+HloModule module
+
+reducer {
+  parameter.1 = f32[] parameter(0)
+  parameter.3 = f32[] parameter(2)
+  mul.2 = f32[] add(parameter.1, parameter.3)
+  parameter.0 = f32[] parameter(1)
+  parameter.2 = f32[] parameter(3)
+  add.3 = f32[] add(parameter.0, parameter.2)
+  ROOT tuple.4 = (f32[], f32[]) tuple(mul.2, add.3)
+}
+
+ENTRY entry {
+  parameter.6 = (f32[0, 10, 10], f32[0, 10, 10]) parameter(0)
+  get-tuple-element.10 = f32[0, 10, 10] get-tuple-element(parameter.6), index=0
+  get-tuple-element.11 = f32[0, 10, 10] get-tuple-element(parameter.6), index=1
+  constant.0 = f32[] constant(0)
+  constant.1 = f32[] constant(1)
+  ROOT reduce = (f32[10, 10], f32[10, 10]) reduce(get-tuple-element.10, get-tuple-element.11, constant.0, constant.1), dimensions={0}, to_apply=reducer
+}
+)";
+  TF_ASSERT_OK_AND_ASSIGN(auto module,
+                          ParseAndReturnVerifiedModule(hlo_string));
+
+  AlgebraicSimplifierOptions options;
+  AlgebraicSimplifier simplifier(options);
+  EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
+  auto root = module->entry_computation()->root_instruction();
+  EXPECT_THAT(root, GmockMatch(m::Tuple(m::Broadcast(m::ConstantScalar(0)),
+                                        m::Broadcast(m::ConstantScalar(1)))));
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 6895544450e74a775597c4dc90eeaf68bd3ad73c Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Tue, 29 Jan 2019 15:24:32 -0800
Subject: [PATCH 1533/2345] Adding the experimental_ints_on_device attribute to
 function defined in CopyToDeviceDataset. Similar to the problem with other
 functions that might run on a GPU, we want int32's in device memory instead
 because the downstream code expects it as so.

PiperOrigin-RevId: 231484444
---
 tensorflow/python/data/experimental/ops/prefetching_ops.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/data/experimental/ops/prefetching_ops.py b/tensorflow/python/data/experimental/ops/prefetching_ops.py
index ef9db2f2d0..e21ff8e9da 100644
--- a/tensorflow/python/data/experimental/ops/prefetching_ops.py
+++ b/tensorflow/python/data/experimental/ops/prefetching_ops.py
@@ -147,7 +147,9 @@ class _CopyToDeviceDataset(dataset_ops.UnaryUnchangedStructureDataset):
 
     next_func_concrete = _next_func._get_concrete_function_internal()  # pylint: disable=protected-access
 
-    @function.defun(input_signature=[tensor_spec.TensorSpec([], dtypes.string)])
+    @function.defun_with_attributes(
+        input_signature=[tensor_spec.TensorSpec([], dtypes.string)],
+        attributes={"experimental_ints_on_device": True})
     def _remote_next_func(string_handle):
       return functional_ops.remote_call(
           target=self._source_device,
-- 
GitLab


From 157f8fdbafddfe260ecaa5ccf30bddc24c4b3b06 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Tue, 29 Jan 2019 15:26:55 -0800
Subject: [PATCH 1534/2345] Move convert_to_tensor logic from the
 `LossFunctionWrapper` class to the individual loss functions to support use
 cases where custom loss functions have non-tensor params.

PiperOrigin-RevId: 231484908
---
 tensorflow/python/keras/losses.py             | 322 +++++++-----------
 tensorflow/python/keras/metrics.py            |  10 +-
 tensorflow/python/keras/utils/tf_utils.py     |   4 +
 ...ow.keras.losses.-binary-crossentropy.pbtxt |   3 +-
 ...ras.losses.-categorical-crossentropy.pbtxt |   3 +-
 ...flow.keras.losses.-categorical-hinge.pbtxt |   3 +-
 .../v1/tensorflow.keras.losses.-hinge.pbtxt   |   1 +
 .../v1/tensorflow.keras.losses.-huber.pbtxt   |   3 +-
 ...sorflow.keras.losses.-k-l-divergence.pbtxt |   3 +-
 .../tensorflow.keras.losses.-log-cosh.pbtxt   |   3 +-
 .../tensorflow.keras.losses.-log-loss.pbtxt   |   3 +-
 ...ow.keras.losses.-mean-absolute-error.pbtxt |   3 +-
 ...sses.-mean-absolute-percentage-error.pbtxt |   3 +-
 ...low.keras.losses.-mean-squared-error.pbtxt |   3 +-
 ...sses.-mean-squared-logarithmic-error.pbtxt |   3 +-
 .../v1/tensorflow.keras.losses.-poisson.pbtxt |   3 +-
 ...ses.-sparse-categorical-crossentropy.pbtxt |   1 +
 ...nsorflow.keras.losses.-squared-hinge.pbtxt |   3 +-
 ...ow.keras.losses.-binary-crossentropy.pbtxt |   3 +-
 ...ras.losses.-categorical-crossentropy.pbtxt |   3 +-
 ...flow.keras.losses.-categorical-hinge.pbtxt |   3 +-
 .../v2/tensorflow.keras.losses.-hinge.pbtxt   |   1 +
 .../v2/tensorflow.keras.losses.-huber.pbtxt   |   3 +-
 ...sorflow.keras.losses.-k-l-divergence.pbtxt |   3 +-
 .../tensorflow.keras.losses.-log-cosh.pbtxt   |   3 +-
 .../tensorflow.keras.losses.-log-loss.pbtxt   |   3 +-
 ...ow.keras.losses.-mean-absolute-error.pbtxt |   3 +-
 ...sses.-mean-absolute-percentage-error.pbtxt |   3 +-
 ...low.keras.losses.-mean-squared-error.pbtxt |   3 +-
 ...sses.-mean-squared-logarithmic-error.pbtxt |   3 +-
 .../v2/tensorflow.keras.losses.-poisson.pbtxt |   3 +-
 ...ses.-sparse-categorical-crossentropy.pbtxt |   1 +
 ...nsorflow.keras.losses.-squared-hinge.pbtxt |   3 +-
 33 files changed, 185 insertions(+), 233 deletions(-)

diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index aaea9162a2..01268a1f76 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -29,6 +29,7 @@ from tensorflow.python.keras import backend as K
 from tensorflow.python.keras.utils.generic_utils import deserialize_keras_object
 from tensorflow.python.keras.utils.generic_utils import serialize_keras_object
 from tensorflow.python.keras.utils.losses_utils import compute_weighted_loss
+from tensorflow.python.keras.utils.tf_utils import is_tensor_or_variable
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
@@ -155,19 +156,18 @@ class LossFunctionWrapper(Loss):
     Returns:
       Loss values per sample.
     """
-    if hasattr(y_pred, 'dtype'):
-      y_true = math_ops.cast(y_true, y_pred.dtype)
     return self.fn(y_true, y_pred, **self._fn_kwargs)
 
   def get_config(self):
-    config = {'fn': self.fn}
-    config.update(self._fn_kwargs)
+    config = {}
+    for k, v in six.iteritems(self._fn_kwargs):
+      config[k] = K.eval(v) if is_tensor_or_variable(v) else v
     base_config = super(LossFunctionWrapper, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
 
 
 @keras_export('keras.losses.MeanSquaredError')
-class MeanSquaredError(Loss):
+class MeanSquaredError(LossFunctionWrapper):
   """Computes the mean of squares of errors between labels and predictions.
 
   For example, if `y_true` is [0., 0., 1., 1.] and `y_pred` is [1., 1., 1., 0.]
@@ -189,23 +189,15 @@ class MeanSquaredError(Loss):
   ```
   """
 
-  def call(self, y_true, y_pred):
-    """Invokes the `MeanSquaredError` instance.
-
-    Args:
-      y_true: Ground truth values.
-      y_pred: The predicted values.
-
-    Returns:
-      Mean squared error losses.
-    """
-    y_pred = ops.convert_to_tensor(y_pred)
-    y_true = math_ops.cast(y_true, y_pred.dtype)
-    return mean_squared_error(y_true, y_pred)
+  def __init__(self,
+               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               name='mean_squared_error'):
+    super(MeanSquaredError, self).__init__(
+        mean_squared_error, name=name, reduction=reduction)
 
 
 @keras_export('keras.losses.MeanAbsoluteError')
-class MeanAbsoluteError(Loss):
+class MeanAbsoluteError(LossFunctionWrapper):
   """Computes the mean of absolute difference between labels and predictions.
 
   For example, if `y_true` is [0., 0., 1., 1.] and `y_pred` is [1., 1., 1., 0.]
@@ -227,23 +219,15 @@ class MeanAbsoluteError(Loss):
   ```
   """
 
-  def call(self, y_true, y_pred):
-    """Invokes the `MeanAbsoluteError` instance.
-
-    Args:
-      y_true: Ground truth values.
-      y_pred: The predicted values.
-
-    Returns:
-      Mean absolute error losses.
-    """
-    y_pred = ops.convert_to_tensor(y_pred)
-    y_true = math_ops.cast(y_true, y_pred.dtype)
-    return mean_absolute_error(y_true, y_pred)
+  def __init__(self,
+               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               name='mean_absolute_error'):
+    super(MeanAbsoluteError, self).__init__(
+        mean_absolute_error, name=name, reduction=reduction)
 
 
 @keras_export('keras.losses.MeanAbsolutePercentageError')
-class MeanAbsolutePercentageError(Loss):
+class MeanAbsolutePercentageError(LossFunctionWrapper):
   """Computes the mean absolute percentage error between `y_true` and `y_pred`.
 
   For example, if `y_true` is [0., 0., 1., 1.] and `y_pred` is [1., 1., 1., 0.]
@@ -265,23 +249,15 @@ class MeanAbsolutePercentageError(Loss):
   ```
   """
 
-  def call(self, y_true, y_pred):
-    """Invokes the `MeanAbsolutePercentageError` instance.
-
-    Args:
-      y_true: Ground truth values.
-      y_pred: The predicted values.
-
-    Returns:
-      Mean absolute percentage error losses.
-    """
-    y_pred = ops.convert_to_tensor(y_pred)
-    y_true = math_ops.cast(y_true, y_pred.dtype)
-    return mean_absolute_percentage_error(y_true, y_pred)
+  def __init__(self,
+               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               name='mean_absolute_percentage_error'):
+    super(MeanAbsolutePercentageError, self).__init__(
+        mean_absolute_percentage_error, name=name, reduction=reduction)
 
 
 @keras_export('keras.losses.MeanSquaredLogarithmicError')
-class MeanSquaredLogarithmicError(Loss):
+class MeanSquaredLogarithmicError(LossFunctionWrapper):
   """Computes the mean squared logarithmic error between `y_true` and `y_pred`.
 
   For example, if `y_true` is [0., 0., 1., 1.] and `y_pred` is [1., 1., 1., 0.]
@@ -303,23 +279,15 @@ class MeanSquaredLogarithmicError(Loss):
   ```
   """
 
-  def call(self, y_true, y_pred):
-    """Invokes the `MeanSquaredLogarithmicError` instance.
-
-    Args:
-      y_true: Ground truth values.
-      y_pred: The predicted values.
-
-    Returns:
-      Mean squared logarithmic error losses.
-    """
-    y_pred = ops.convert_to_tensor(y_pred)
-    y_true = math_ops.cast(y_true, y_pred.dtype)
-    return mean_squared_logarithmic_error(y_true, y_pred)
+  def __init__(self,
+               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               name='mean_squared_logarithmic_error'):
+    super(MeanSquaredLogarithmicError, self).__init__(
+        mean_squared_logarithmic_error, name=name, reduction=reduction)
 
 
 @keras_export('keras.losses.BinaryCrossentropy')
-class BinaryCrossentropy(Loss):
+class BinaryCrossentropy(LossFunctionWrapper):
   """Computes the crossentropy loss between the labels and predictions.
 
   Use this crossentropy loss function when there are only two label classes
@@ -357,33 +325,18 @@ class BinaryCrossentropy(Loss):
                from_logits=False,
                label_smoothing=0,
                reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
-               name=None):
-    super(BinaryCrossentropy, self).__init__(reduction=reduction, name=name)
+               name='binary_crossentropy'):
+    super(BinaryCrossentropy, self).__init__(
+        binary_crossentropy,
+        name=name,
+        reduction=reduction,
+        from_logits=from_logits,
+        label_smoothing=label_smoothing)
     self.from_logits = from_logits
-    self.label_smoothing = ops.convert_to_tensor(
-        label_smoothing, dtype=K.floatx())
-
-  def call(self, y_true, y_pred):
-    """Invokes the `BinaryCrossentropy` instance.
-
-    Args:
-      y_true: Ground truth values.
-      y_pred: The predicted values.
-
-    Returns:
-      Binary cross entropy losses.
-    """
-    y_pred = ops.convert_to_tensor(y_pred)
-    y_true = math_ops.cast(y_true, y_pred.dtype)
-    return binary_crossentropy(
-        y_true,
-        y_pred,
-        from_logits=self.from_logits,
-        label_smoothing=self.label_smoothing)
 
 
 @keras_export('keras.losses.CategoricalCrossentropy')
-class CategoricalCrossentropy(Loss):
+class CategoricalCrossentropy(LossFunctionWrapper):
   """Computes the crossentropy loss between the labels and predictions.
 
   Use this crossentropy loss function when there are two or more label classes.
@@ -428,34 +381,17 @@ class CategoricalCrossentropy(Loss):
                from_logits=False,
                label_smoothing=0,
                reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
-               name=None):
+               name='categorical_crossentropy'):
     super(CategoricalCrossentropy, self).__init__(
-        reduction=reduction, name=name)
-    self.from_logits = from_logits
-    self.label_smoothing = ops.convert_to_tensor(
-        label_smoothing, dtype=K.floatx())
-
-  def call(self, y_true, y_pred):
-    """Invokes the `CategoricalCrossentropy` instance.
-
-    Args:
-      y_true: Ground truth values.
-      y_pred: The predicted values.
-
-    Returns:
-      Categorical cross entropy losses.
-    """
-    y_pred = ops.convert_to_tensor(y_pred)
-    y_true = math_ops.cast(y_true, y_pred.dtype)
-    return categorical_crossentropy(
-        y_true,
-        y_pred,
-        from_logits=self.from_logits,
-        label_smoothing=self.label_smoothing)
+        categorical_crossentropy,
+        name=name,
+        reduction=reduction,
+        from_logits=from_logits,
+        label_smoothing=label_smoothing)
 
 
 @keras_export('keras.losses.SparseCategoricalCrossentropy')
-class SparseCategoricalCrossentropy(Loss):
+class SparseCategoricalCrossentropy(LossFunctionWrapper):
   """Computes the crossentropy loss between the labels and predictions.
 
   Use this crossentropy loss function when there are two or more label classes.
@@ -499,27 +435,14 @@ class SparseCategoricalCrossentropy(Loss):
                reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
                name=None):
     super(SparseCategoricalCrossentropy, self).__init__(
-        reduction=reduction, name=name)
-    self.from_logits = from_logits
-
-  def call(self, y_true, y_pred):
-    """Invokes the `SparseCategoricalCrossentropy` instance.
-
-    Args:
-      y_true: Ground truth values.
-      y_pred: The predicted values.
-
-    Returns:
-      Sparse categorical cross entropy losses.
-    """
-    y_pred = ops.convert_to_tensor(y_pred)
-    y_true = ops.convert_to_tensor(y_true)
-    return sparse_categorical_crossentropy(
-        y_true, y_pred, from_logits=self.from_logits)
+        sparse_categorical_crossentropy,
+        name=name,
+        reduction=reduction,
+        from_logits=from_logits)
 
 
 @keras_export('keras.losses.Hinge')
-class Hinge(Loss):
+class Hinge(LossFunctionWrapper):
   """Computes the hinge loss between `y_true` and `y_pred`.
 
   Usage:
@@ -538,23 +461,14 @@ class Hinge(Loss):
   ```
   """
 
-  def call(self, y_true, y_pred):
-    """Calculates the hinge loss.
-
-    Args:
-      y_true: Ground truth values.
-      y_pred: The predicted values.
-
-    Returns:
-      Hinge loss.
-    """
-    y_pred = ops.convert_to_tensor(y_pred)
-    y_true = math_ops.cast(y_true, y_pred.dtype)
-    return hinge(y_true, y_pred)
+  def __init__(self,
+               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               name=None):
+    super(Hinge, self).__init__(hinge, name=name, reduction=reduction)
 
 
 @keras_export('keras.losses.SquaredHinge')
-class SquaredHinge(Loss):
+class SquaredHinge(LossFunctionWrapper):
   """Computes the squared hinge loss between `y_true` and `y_pred`.
 
   Usage:
@@ -573,23 +487,15 @@ class SquaredHinge(Loss):
   ```
   """
 
-  def call(self, y_true, y_pred):
-    """Calculates the squared hinge loss.
-
-    Args:
-      y_true: Ground truth values.
-      y_pred: The predicted values.
-
-    Returns:
-      Squared hinge loss.
-    """
-    y_pred = ops.convert_to_tensor(y_pred)
-    y_true = math_ops.cast(y_true, y_pred.dtype)
-    return squared_hinge(y_true, y_pred)
+  def __init__(self,
+               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               name='squared_hinge'):
+    super(SquaredHinge, self).__init__(
+        squared_hinge, name=name, reduction=reduction)
 
 
 @keras_export('keras.losses.CategoricalHinge')
-class CategoricalHinge(Loss):
+class CategoricalHinge(LossFunctionWrapper):
   """Computes the categorical hinge loss between `y_true` and `y_pred`.
 
   Usage:
@@ -608,23 +514,15 @@ class CategoricalHinge(Loss):
   ```
   """
 
-  def call(self, y_true, y_pred):
-    """Calculates the categorical hinge loss.
-
-    Args:
-      y_true: Ground truth values.
-      y_pred: The predicted values.
-
-    Returns:
-      Categorical hinge loss.
-    """
-    y_pred = ops.convert_to_tensor(y_pred)
-    y_true = math_ops.cast(y_true, y_pred.dtype)
-    return categorical_hinge(y_true, y_pred)
+  def __init__(self,
+               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               name='categorical_hinge'):
+    super(CategoricalHinge, self).__init__(
+        categorical_hinge, name=name, reduction=reduction)
 
 
 @keras_export('keras.losses.LogLoss')
-class LogLoss(Loss):
+class LogLoss(LossFunctionWrapper):
   """Computes the log loss between `y_true` and `y_pred`.
 
   `logloss = - y_true * log(y_pred) - (1 - y_true) * log(1 - y_pred)`
@@ -645,14 +543,14 @@ class LogLoss(Loss):
   ```
   """
 
-  def call(self, y_true, y_pred):
-    y_pred = ops.convert_to_tensor(y_pred)
-    y_true = math_ops.cast(y_true, y_pred.dtype)
-    return logloss(y_true, y_pred)
+  def __init__(self,
+               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               name='logloss'):
+    super(LogLoss, self).__init__(logloss, name=name, reduction=reduction)
 
 
 @keras_export('keras.losses.Poisson')
-class Poisson(Loss):
+class Poisson(LossFunctionWrapper):
   """Computes the Poisson loss between `y_true` and `y_pred`.
 
   `loss = y_pred - y_true * log(y_pred)`
@@ -673,14 +571,14 @@ class Poisson(Loss):
   ```
   """
 
-  def call(self, y_true, y_pred):
-    y_pred = ops.convert_to_tensor(y_pred)
-    y_true = math_ops.cast(y_true, y_pred.dtype)
-    return poisson(y_true, y_pred)
+  def __init__(self,
+               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               name='poisson'):
+    super(Poisson, self).__init__(poisson, name=name, reduction=reduction)
 
 
 @keras_export('keras.losses.LogCosh')
-class LogCosh(Loss):
+class LogCosh(LossFunctionWrapper):
   """Computes the logarithm of the hyperbolic cosine of the prediction error.
 
   `logcosh = log((exp(x) + exp(-x))/2)`, where x is the error (y_pred - y_true)
@@ -701,14 +599,14 @@ class LogCosh(Loss):
   ```
   """
 
-  def call(self, y_true, y_pred):
-    y_pred = ops.convert_to_tensor(y_pred)
-    y_true = math_ops.cast(y_true, y_pred.dtype)
-    return logcosh(y_true, y_pred)
+  def __init__(self,
+               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               name='logcosh'):
+    super(LogCosh, self).__init__(logcosh, name=name, reduction=reduction)
 
 
 @keras_export('keras.losses.KLDivergence')
-class KLDivergence(Loss):
+class KLDivergence(LossFunctionWrapper):
   """Computes Kullback Leibler divergence loss between `y_true` and `y_pred`.
 
   `loss = y_true * log(y_true / y_pred)`
@@ -729,14 +627,15 @@ class KLDivergence(Loss):
   ```
   """
 
-  def call(self, y_true, y_pred):
-    y_pred = ops.convert_to_tensor(y_pred)
-    y_true = math_ops.cast(y_true, y_pred.dtype)
-    return kullback_leibler_divergence(y_true, y_pred)
+  def __init__(self,
+               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               name='kullback_leibler_divergence'):
+    super(KLDivergence, self).__init__(
+        kullback_leibler_divergence, name=name, reduction=reduction)
 
 
 @keras_export('keras.losses.Huber')
-class Huber(Loss):
+class Huber(LossFunctionWrapper):
   """Computes the Huber loss between `y_true` and `y_pred`.
 
   For each value x in `error=y_true-y_pred`, the following is calculated:
@@ -773,14 +672,9 @@ class Huber(Loss):
   def __init__(self,
                delta=1.0,
                reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
-               name=None):
-    super(Huber, self).__init__(reduction=reduction, name=name)
-    self.delta = delta
-
-  def call(self, y_true, y_pred):
-    y_pred = ops.convert_to_tensor(y_pred)
-    y_true = math_ops.cast(y_true, y_pred.dtype)
-    return huber_loss(y_true, y_pred, delta=self.delta)
+               name='huber_loss'):
+    super(Huber, self).__init__(
+        huber_loss, name=name, reduction=reduction, delta=delta)
 
 
 @keras_export('keras.metrics.mean_squared_error',
@@ -790,6 +684,8 @@ class Huber(Loss):
               'keras.losses.mse',
               'keras.losses.MSE')
 def mean_squared_error(y_true, y_pred):
+  y_pred = ops.convert_to_tensor(y_pred)
+  y_true = math_ops.cast(y_true, y_pred.dtype)
   return K.mean(math_ops.squared_difference(y_pred, y_true), axis=-1)
 
 
@@ -800,6 +696,8 @@ def mean_squared_error(y_true, y_pred):
               'keras.losses.mae',
               'keras.losses.MAE')
 def mean_absolute_error(y_true, y_pred):
+  y_pred = ops.convert_to_tensor(y_pred)
+  y_true = math_ops.cast(y_true, y_pred.dtype)
   return K.mean(math_ops.abs(y_pred - y_true), axis=-1)
 
 
@@ -809,7 +707,9 @@ def mean_absolute_error(y_true, y_pred):
               'keras.losses.mean_absolute_percentage_error',
               'keras.losses.mape',
               'keras.losses.MAPE')
-def mean_absolute_percentage_error(y_true, y_pred):
+def mean_absolute_percentage_error(y_true, y_pred):  # pylint: disable=missing-docstring
+  y_pred = ops.convert_to_tensor(y_pred)
+  y_true = math_ops.cast(y_true, y_pred.dtype)
   diff = math_ops.abs(
       (y_true - y_pred) / K.clip(math_ops.abs(y_true), K.epsilon(), None))
   return 100. * K.mean(diff, axis=-1)
@@ -821,7 +721,9 @@ def mean_absolute_percentage_error(y_true, y_pred):
               'keras.losses.mean_squared_logarithmic_error',
               'keras.losses.msle',
               'keras.losses.MSLE')
-def mean_squared_logarithmic_error(y_true, y_pred):
+def mean_squared_logarithmic_error(y_true, y_pred):  # pylint: disable=missing-docstring
+  y_pred = ops.convert_to_tensor(y_pred)
+  y_true = math_ops.cast(y_true, y_pred.dtype)
   first_log = math_ops.log(K.clip(y_pred, K.epsilon(), None) + 1.)
   second_log = math_ops.log(K.clip(y_true, K.epsilon(), None) + 1.)
   return K.mean(math_ops.squared_difference(first_log, second_log), axis=-1)
@@ -829,23 +731,31 @@ def mean_squared_logarithmic_error(y_true, y_pred):
 
 @keras_export('keras.metrics.squared_hinge', 'keras.losses.squared_hinge')
 def squared_hinge(y_true, y_pred):
+  y_pred = ops.convert_to_tensor(y_pred)
+  y_true = math_ops.cast(y_true, y_pred.dtype)
   return K.mean(
       math_ops.square(math_ops.maximum(1. - y_true * y_pred, 0.)), axis=-1)
 
 
 @keras_export('keras.metrics.hinge', 'keras.losses.hinge')
 def hinge(y_true, y_pred):
+  y_pred = ops.convert_to_tensor(y_pred)
+  y_true = math_ops.cast(y_true, y_pred.dtype)
   return K.mean(math_ops.maximum(1. - y_true * y_pred, 0.), axis=-1)
 
 
 @keras_export('keras.losses.categorical_hinge')
 def categorical_hinge(y_true, y_pred):
+  y_pred = ops.convert_to_tensor(y_pred)
+  y_true = math_ops.cast(y_true, y_pred.dtype)
   pos = math_ops.reduce_sum(y_true * y_pred, axis=-1)
   neg = math_ops.reduce_max((1. - y_true) * y_pred, axis=-1)
   return math_ops.maximum(0., neg - pos + 1.)
 
 
 def logloss(y_true, y_pred):
+  y_pred = ops.convert_to_tensor(y_pred)
+  y_true = math_ops.cast(y_true, y_pred.dtype)
   losses = math_ops.multiply(y_true, math_ops.log(y_pred + K.epsilon()))
   losses += math_ops.multiply((1 - y_true),
                               math_ops.log(1 - y_pred + K.epsilon()))
@@ -901,6 +811,8 @@ def logcosh(y_true, y_pred):
   Returns:
       Tensor with one scalar loss entry per sample.
   """
+  y_pred = ops.convert_to_tensor(y_pred)
+  y_true = math_ops.cast(y_true, y_pred.dtype)
 
   def _logcosh(x):
     return x + nn.softplus(-2. * x) - math_ops.log(2.)
@@ -926,6 +838,9 @@ def categorical_crossentropy(y_true,
   Returns:
     Categorical crossentropy loss value.
   """
+  y_pred = ops.convert_to_tensor(y_pred)
+  y_true = math_ops.cast(y_true, y_pred.dtype)
+  label_smoothing = ops.convert_to_tensor(label_smoothing, dtype=K.floatx())
 
   def _smooth_labels():
     num_classes = math_ops.cast(array_ops.shape(y_true)[1], y_pred.dtype)
@@ -945,7 +860,10 @@ def sparse_categorical_crossentropy(y_true, y_pred, from_logits=False):
 
 @keras_export('keras.metrics.binary_crossentropy',
               'keras.losses.binary_crossentropy')
-def binary_crossentropy(y_true, y_pred, from_logits=False, label_smoothing=0):
+def binary_crossentropy(y_true, y_pred, from_logits=False, label_smoothing=0):  # pylint: disable=missing-docstring
+  y_pred = ops.convert_to_tensor(y_pred)
+  y_true = math_ops.cast(y_true, y_pred.dtype)
+  label_smoothing = ops.convert_to_tensor(label_smoothing, dtype=K.floatx())
 
   def _smooth_labels():
     return y_true * (1.0 - label_smoothing) + 0.5 * label_smoothing
@@ -962,7 +880,9 @@ def binary_crossentropy(y_true, y_pred, from_logits=False, label_smoothing=0):
               'keras.losses.kullback_leibler_divergence',
               'keras.losses.kld',
               'keras.losses.KLD')
-def kullback_leibler_divergence(y_true, y_pred):
+def kullback_leibler_divergence(y_true, y_pred):  # pylint: disable=missing-docstring
+  y_pred = ops.convert_to_tensor(y_pred)
+  y_true = math_ops.cast(y_true, y_pred.dtype)
   y_true = K.clip(y_true, K.epsilon(), 1)
   y_pred = K.clip(y_pred, K.epsilon(), 1)
   return math_ops.reduce_sum(y_true * math_ops.log(y_true / y_pred), axis=-1)
@@ -970,6 +890,8 @@ def kullback_leibler_divergence(y_true, y_pred):
 
 @keras_export('keras.metrics.poisson', 'keras.losses.poisson')
 def poisson(y_true, y_pred):
+  y_pred = ops.convert_to_tensor(y_pred)
+  y_true = math_ops.cast(y_true, y_pred.dtype)
   return K.mean(y_pred - y_true * math_ops.log(y_pred + K.epsilon()), axis=-1)
 
 
diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 95837e4d8d..bc7c1bff80 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -29,7 +29,6 @@ from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
-from tensorflow.python.framework import tensor_util
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras.engine.base_layer import Layer
 from tensorflow.python.keras.losses import binary_crossentropy
@@ -51,6 +50,7 @@ from tensorflow.python.keras.utils.generic_utils import deserialize_keras_object
 from tensorflow.python.keras.utils.generic_utils import serialize_keras_object
 from tensorflow.python.keras.utils.generic_utils import to_list
 from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions
+from tensorflow.python.keras.utils.tf_utils import is_tensor_or_variable
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import confusion_matrix
 from tensorflow.python.ops import init_ops
@@ -509,7 +509,7 @@ class MeanRelativeError(Mean):
 
   def get_config(self):
     n = self.normalizer
-    config = {'normalizer': K.eval(n) if _is_tensor_or_variable(n) else n}
+    config = {'normalizer': K.eval(n) if is_tensor_or_variable(n) else n}
     base_config = super(MeanRelativeError, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
 
@@ -558,7 +558,7 @@ class MeanMetricWrapper(Mean):
   def get_config(self):
     config = {}
     for k, v in six.iteritems(self._fn_kwargs):
-      config[k] = K.eval(v) if _is_tensor_or_variable(v) else v
+      config[k] = K.eval(v) if is_tensor_or_variable(v) else v
     base_config = super(MeanMetricWrapper, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
 
@@ -2650,7 +2650,3 @@ def get(identifier):
   else:
     raise ValueError('Could not interpret '
                      'metric function identifier: %s' % identifier)
-
-
-def _is_tensor_or_variable(x):
-  return tensor_util.is_tensor(x) or isinstance(x, tf_variables.Variable)
diff --git a/tensorflow/python/keras/utils/tf_utils.py b/tensorflow/python/keras/utils/tf_utils.py
index dc5c4f1d90..387fbf5767 100644
--- a/tensorflow/python/keras/utils/tf_utils.py
+++ b/tensorflow/python/keras/utils/tf_utils.py
@@ -351,3 +351,7 @@ def register_symbolic_tensor_type(cls):
   """
   global _user_convertible_tensor_types
   _user_convertible_tensor_types.add(cls)
+
+
+def is_tensor_or_variable(x):
+  return tensor_util.is_tensor(x) or isinstance(x, variables.Variable)
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-binary-crossentropy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-binary-crossentropy.pbtxt
index 2f7da93f6f..1242eec68f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-binary-crossentropy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-binary-crossentropy.pbtxt
@@ -1,11 +1,12 @@
 path: "tensorflow.keras.losses.BinaryCrossentropy"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.BinaryCrossentropy\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'from_logits\', \'label_smoothing\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'0\', \'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'from_logits\', \'label_smoothing\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'0\', \'sum_over_batch_size\', \'binary_crossentropy\'], "
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-categorical-crossentropy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-categorical-crossentropy.pbtxt
index b3a7cd8097..cf3c2de840 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-categorical-crossentropy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-categorical-crossentropy.pbtxt
@@ -1,11 +1,12 @@
 path: "tensorflow.keras.losses.CategoricalCrossentropy"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.CategoricalCrossentropy\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'from_logits\', \'label_smoothing\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'0\', \'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'from_logits\', \'label_smoothing\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'0\', \'sum_over_batch_size\', \'categorical_crossentropy\'], "
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-categorical-hinge.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-categorical-hinge.pbtxt
index 4ba9e57bed..fa374afb28 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-categorical-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-categorical-hinge.pbtxt
@@ -1,11 +1,12 @@
 path: "tensorflow.keras.losses.CategoricalHinge"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.CategoricalHinge\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'categorical_hinge\'], "
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-hinge.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-hinge.pbtxt
index 7b3c62d3be..a4c25eefcb 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-hinge.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.keras.losses.Hinge"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.Hinge\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-huber.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-huber.pbtxt
index 2204f0b0f0..1fa8ffa957 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-huber.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-huber.pbtxt
@@ -1,11 +1,12 @@
 path: "tensorflow.keras.losses.Huber"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.Huber\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'delta\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'1.0\', \'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'delta\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'1.0\', \'sum_over_batch_size\', \'huber_loss\'], "
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-k-l-divergence.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-k-l-divergence.pbtxt
index 60706b4035..d950c789eb 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-k-l-divergence.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-k-l-divergence.pbtxt
@@ -1,11 +1,12 @@
 path: "tensorflow.keras.losses.KLDivergence"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.KLDivergence\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'kullback_leibler_divergence\'], "
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-log-cosh.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-log-cosh.pbtxt
index cf1a0f1e80..fbbd5317f8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-log-cosh.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-log-cosh.pbtxt
@@ -1,11 +1,12 @@
 path: "tensorflow.keras.losses.LogCosh"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.LogCosh\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'logcosh\'], "
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-log-loss.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-log-loss.pbtxt
index 466e0b4006..40ca239cce 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-log-loss.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-log-loss.pbtxt
@@ -1,11 +1,12 @@
 path: "tensorflow.keras.losses.LogLoss"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.LogLoss\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'logloss\'], "
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-absolute-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-absolute-error.pbtxt
index 712bb2ecd3..9da6b59ec8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-absolute-error.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-absolute-error.pbtxt
@@ -1,11 +1,12 @@
 path: "tensorflow.keras.losses.MeanAbsoluteError"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.MeanAbsoluteError\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'mean_absolute_error\'], "
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-absolute-percentage-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-absolute-percentage-error.pbtxt
index 7fe362da89..7c3ae9b49a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-absolute-percentage-error.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-absolute-percentage-error.pbtxt
@@ -1,11 +1,12 @@
 path: "tensorflow.keras.losses.MeanAbsolutePercentageError"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.MeanAbsolutePercentageError\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'mean_absolute_percentage_error\'], "
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-squared-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-squared-error.pbtxt
index a571853350..2126ac68d2 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-squared-error.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-squared-error.pbtxt
@@ -1,11 +1,12 @@
 path: "tensorflow.keras.losses.MeanSquaredError"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.MeanSquaredError\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'mean_squared_error\'], "
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-squared-logarithmic-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-squared-logarithmic-error.pbtxt
index 200006db35..6ef9610546 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-squared-logarithmic-error.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-squared-logarithmic-error.pbtxt
@@ -1,11 +1,12 @@
 path: "tensorflow.keras.losses.MeanSquaredLogarithmicError"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.MeanSquaredLogarithmicError\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'mean_squared_logarithmic_error\'], "
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-poisson.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-poisson.pbtxt
index 674258ed8b..61c90c3140 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-poisson.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-poisson.pbtxt
@@ -1,11 +1,12 @@
 path: "tensorflow.keras.losses.Poisson"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.Poisson\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'poisson\'], "
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-sparse-categorical-crossentropy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-sparse-categorical-crossentropy.pbtxt
index 15a59d16c2..c13f9f967d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-sparse-categorical-crossentropy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-sparse-categorical-crossentropy.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.keras.losses.SparseCategoricalCrossentropy"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.SparseCategoricalCrossentropy\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-squared-hinge.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-squared-hinge.pbtxt
index f8badb2b6e..fabe4c7814 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-squared-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-squared-hinge.pbtxt
@@ -1,11 +1,12 @@
 path: "tensorflow.keras.losses.SquaredHinge"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.SquaredHinge\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'squared_hinge\'], "
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-binary-crossentropy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-binary-crossentropy.pbtxt
index 2f7da93f6f..1242eec68f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-binary-crossentropy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-binary-crossentropy.pbtxt
@@ -1,11 +1,12 @@
 path: "tensorflow.keras.losses.BinaryCrossentropy"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.BinaryCrossentropy\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'from_logits\', \'label_smoothing\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'0\', \'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'from_logits\', \'label_smoothing\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'0\', \'sum_over_batch_size\', \'binary_crossentropy\'], "
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-categorical-crossentropy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-categorical-crossentropy.pbtxt
index b3a7cd8097..cf3c2de840 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-categorical-crossentropy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-categorical-crossentropy.pbtxt
@@ -1,11 +1,12 @@
 path: "tensorflow.keras.losses.CategoricalCrossentropy"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.CategoricalCrossentropy\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'from_logits\', \'label_smoothing\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'0\', \'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'from_logits\', \'label_smoothing\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'0\', \'sum_over_batch_size\', \'categorical_crossentropy\'], "
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-categorical-hinge.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-categorical-hinge.pbtxt
index 4ba9e57bed..fa374afb28 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-categorical-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-categorical-hinge.pbtxt
@@ -1,11 +1,12 @@
 path: "tensorflow.keras.losses.CategoricalHinge"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.CategoricalHinge\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'categorical_hinge\'], "
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-hinge.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-hinge.pbtxt
index 7b3c62d3be..a4c25eefcb 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-hinge.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.keras.losses.Hinge"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.Hinge\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-huber.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-huber.pbtxt
index 2204f0b0f0..1fa8ffa957 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-huber.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-huber.pbtxt
@@ -1,11 +1,12 @@
 path: "tensorflow.keras.losses.Huber"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.Huber\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'delta\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'1.0\', \'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'delta\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'1.0\', \'sum_over_batch_size\', \'huber_loss\'], "
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-k-l-divergence.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-k-l-divergence.pbtxt
index 60706b4035..d950c789eb 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-k-l-divergence.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-k-l-divergence.pbtxt
@@ -1,11 +1,12 @@
 path: "tensorflow.keras.losses.KLDivergence"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.KLDivergence\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'kullback_leibler_divergence\'], "
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-log-cosh.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-log-cosh.pbtxt
index cf1a0f1e80..fbbd5317f8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-log-cosh.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-log-cosh.pbtxt
@@ -1,11 +1,12 @@
 path: "tensorflow.keras.losses.LogCosh"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.LogCosh\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'logcosh\'], "
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-log-loss.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-log-loss.pbtxt
index 466e0b4006..40ca239cce 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-log-loss.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-log-loss.pbtxt
@@ -1,11 +1,12 @@
 path: "tensorflow.keras.losses.LogLoss"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.LogLoss\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'logloss\'], "
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-absolute-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-absolute-error.pbtxt
index 712bb2ecd3..9da6b59ec8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-absolute-error.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-absolute-error.pbtxt
@@ -1,11 +1,12 @@
 path: "tensorflow.keras.losses.MeanAbsoluteError"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.MeanAbsoluteError\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'mean_absolute_error\'], "
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-absolute-percentage-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-absolute-percentage-error.pbtxt
index 7fe362da89..7c3ae9b49a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-absolute-percentage-error.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-absolute-percentage-error.pbtxt
@@ -1,11 +1,12 @@
 path: "tensorflow.keras.losses.MeanAbsolutePercentageError"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.MeanAbsolutePercentageError\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'mean_absolute_percentage_error\'], "
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-squared-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-squared-error.pbtxt
index a571853350..2126ac68d2 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-squared-error.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-squared-error.pbtxt
@@ -1,11 +1,12 @@
 path: "tensorflow.keras.losses.MeanSquaredError"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.MeanSquaredError\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'mean_squared_error\'], "
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-squared-logarithmic-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-squared-logarithmic-error.pbtxt
index 200006db35..6ef9610546 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-squared-logarithmic-error.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-squared-logarithmic-error.pbtxt
@@ -1,11 +1,12 @@
 path: "tensorflow.keras.losses.MeanSquaredLogarithmicError"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.MeanSquaredLogarithmicError\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'mean_squared_logarithmic_error\'], "
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-poisson.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-poisson.pbtxt
index 674258ed8b..61c90c3140 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-poisson.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-poisson.pbtxt
@@ -1,11 +1,12 @@
 path: "tensorflow.keras.losses.Poisson"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.Poisson\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'poisson\'], "
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-sparse-categorical-crossentropy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-sparse-categorical-crossentropy.pbtxt
index 15a59d16c2..c13f9f967d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-sparse-categorical-crossentropy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-sparse-categorical-crossentropy.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.keras.losses.SparseCategoricalCrossentropy"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.SparseCategoricalCrossentropy\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-squared-hinge.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-squared-hinge.pbtxt
index f8badb2b6e..fabe4c7814 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-squared-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-squared-hinge.pbtxt
@@ -1,11 +1,12 @@
 path: "tensorflow.keras.losses.SquaredHinge"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.losses.SquaredHinge\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'squared_hinge\'], "
   }
   member_method {
     name: "call"
-- 
GitLab


From 17198dd48951b27def39d653eafa21dac3c72939 Mon Sep 17 00:00:00 2001
From: nammbash <niroopshankar.ammbashankar@intel.com>
Date: Tue, 29 Jan 2019 16:03:26 -0800
Subject: [PATCH 1535/2345] Review changes

---
 tensorflow/core/kernels/BUILD                 | 13 ++--
 .../core/kernels/mkl_quantized_conv_ops.h     | 11 ++--
 ...mkl_requantization_range_per_channel_op.cc | 32 ++++-----
 .../core/kernels/mkl_requantize_ops_test.cc   | 66 +++++++++----------
 .../kernels/mkl_requantize_per_channel_op.cc  | 37 +++++------
 5 files changed, 80 insertions(+), 79 deletions(-)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 26b8c691e1..f170142105 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -6104,9 +6104,7 @@ tf_kernel_library(
 tf_cc_test(
     name = "requantization_range_op_test",
     size = "small",
-    srcs = [
-        "requantization_range_op_test.cc",
-    ],
+    srcs = ["requantization_range_op_test.cc"],
     deps = [
         ":ops_testutil",
         ":ops_util",
@@ -6994,8 +6992,7 @@ tf_mkl_kernel_library(
         "no_op.h",
         "reference_gemm.h",
     ],
-    deps = if_mkl(
-        [
+    deps = if_mkl([
             ":concat_lib_hdrs",
             ":conv_ops",
             ":cwise_op",
@@ -7004,6 +7001,7 @@ tf_mkl_kernel_library(
             ":ops_util",
             ":pooling_ops",
             ":quantization_utils",
+            ":transpose_functor",
             "//tensorflow/core:array_ops_op_lib",
             "//tensorflow/core:core_cpu",
             "//tensorflow/core:framework",
@@ -7011,10 +7009,9 @@ tf_mkl_kernel_library(
             "//tensorflow/core:math_ops_op_lib",
             "//tensorflow/core:nn_ops_op_lib",
             "//third_party/eigen3",
-            "@gemmlowp",
-            ":transpose_functor",
-            "@mkl_dnn",
             "//third_party/mkl:intel_binary_blob",
+            "@gemmlowp",
+            "@mkl_dnn",            
         ],
     ),
 )
diff --git a/tensorflow/core/kernels/mkl_quantized_conv_ops.h b/tensorflow/core/kernels/mkl_quantized_conv_ops.h
index 84a1ccb4c2..985081486b 100644
--- a/tensorflow/core/kernels/mkl_quantized_conv_ops.h
+++ b/tensorflow/core/kernels/mkl_quantized_conv_ops.h
@@ -26,7 +26,10 @@ template <class T>
 float MklFloatForOneQuantizedLevel(float range_min, float range_max) {
   int64 highest = static_cast<int64>(Eigen::NumTraits<T>::highest());
   int64 lowest = static_cast<int64>(Eigen::NumTraits<T>::lowest());
-  if (lowest < -highest) lowest += 1;
+
+  // Adjusting for having a symmetric range. 
+  // for example: for 8-bit [-127, 127] as opposed to [-128, 127].
+  if (lowest < -highest) ++lowest;
 
   const float float_for_one_quantized_level =
       (range_max - range_min) / (highest - lowest);
@@ -57,8 +60,8 @@ void MklQuantizationRangeForMultiplication(float min_a, float max_a,
                                            const Tensor& max_b_vector,
                                            Tensor** min_c_vector,
                                            Tensor** max_c_vector) {
-  CHECK(min_b_vector.NumElements() == (*min_c_vector)->NumElements());
-  CHECK(max_b_vector.NumElements() == (*max_c_vector)->NumElements());
+  DCHECK(min_b_vector.NumElements() == (*min_c_vector)->NumElements());
+  DCHECK(max_b_vector.NumElements() == (*max_c_vector)->NumElements());
   size_t n_channel = min_b_vector.NumElements();
   const int64 c_highest = static_cast<int64>(Eigen::NumTraits<T3>::highest());
   const int64 c_lowest = static_cast<int64>(Eigen::NumTraits<T3>::lowest());
@@ -67,7 +70,7 @@ void MklQuantizationRangeForMultiplication(float min_a, float max_a,
   float* min_c = (*min_c_vector)->flat<float>().data();
   float* max_c = (*max_c_vector)->flat<float>().data();
 #pragma omp parallel for
-  for (size_t n = 0; n < n_channel; n++) {
+  for (size_t n = 0; n < n_channel; ++n) {
     float a_float_for_one_quant_level =
         MklFloatForOneQuantizedLevel<T1>(min_a, max_a);
     float b_float_for_one_quant_level =
diff --git a/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc b/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc
index d23a2d320a..78ef2fc4a5 100644
--- a/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc
+++ b/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc
@@ -1,4 +1,4 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -43,15 +43,15 @@ class MklRequantizationRangePerChannelOp : public OpKernel {
 
   void Compute(OpKernelContext* ctx) override {
     const Tensor& input = ctx->input(kInputTensorIndex);
-    const Tensor& input_min = ctx->input(kInputMin);
-    const Tensor& input_max = ctx->input(kInputMax);
+    const Tensor& input_min = ctx->input(kInputMinIndex);
+    const Tensor& input_max = ctx->input(kInputMaxIndex);
 
     size_t depth = input_max.NumElements();
     OP_REQUIRES(ctx, input_min.dim_size(0) == depth,
-                errors::InvalidArgument("min has incorrect size, expected ",
+                errors::InvalidArgument("input_min has incorrect size, expected ",
                                         depth, " was ", input_min.dim_size(0)));
     OP_REQUIRES(ctx, input_max.dim_size(0) == depth,
-                errors::InvalidArgument("max has incorrect size, expected ",
+                errors::InvalidArgument("input_max has incorrect size, expected ",
                                         depth, " was ", input_max.dim_size(0)));
 
     const float* input_min_data = input_min.flat<float>().data();
@@ -62,8 +62,9 @@ class MklRequantizationRangePerChannelOp : public OpKernel {
     auto input_matrix = input.flat_inner_dims<qint32>();
     auto transposed_input = input_matrix.shuffle(shuffling);
 
+    // Find the ranges of each channel in parallel.
 #pragma omp parallel for
-    for (size_t i = 0; i < depth; i++) {
+    for (size_t i = 0; i < depth; ++i) {
       Eigen::Tensor<qint32, 0, Eigen::RowMajor> min =
           transposed_input.chip<0>(i).minimum();
       Eigen::Tensor<qint32, 0, Eigen::RowMajor> max =
@@ -74,12 +75,13 @@ class MklRequantizationRangePerChannelOp : public OpKernel {
           std::max(std::abs(min_per_channel), std::abs(max_per_channel));
       float scale =
           std::max(std::abs(input_min_data[i]), std::abs(input_max_data[i]));
-      ranges[i] = (scale * (float)abs_max / (float)(1L << 31));
+      ranges[i] = scale * static_cast<float>(abs_max) / static_cast<float>(1L << 31);
       if (min_per_channel < 0) is_non_negative = false;
     }
 
+    // Obtain ranges of out_min_max after all parallel_for openMP threads have joined. 
     float out_min_max = std::numeric_limits<float>::min();
-    for (size_t i = 0; i < depth; i++) {
+    for (size_t i = 0; i < depth; ++i) {
       if (out_min_max < ranges[i]) out_min_max = ranges[i];
     }
     // Fixing max to clip_value_max_ (example 6.0 to support relu6)
@@ -87,19 +89,19 @@ class MklRequantizationRangePerChannelOp : public OpKernel {
 
     Tensor* output_min = nullptr;
     Tensor* output_max = nullptr;
-    OP_REQUIRES_OK(ctx, ctx->allocate_output(kOutputMin, {}, &output_min));
-    OP_REQUIRES_OK(ctx, ctx->allocate_output(kOutputMax, {}, &output_max));
-    output_min->flat<float>()(0) = is_non_negative ? 0.0f : out_min_max * -1.0f;
+    OP_REQUIRES_OK(ctx, ctx->allocate_output(kOutputMinIndex, {}, &output_min));
+    OP_REQUIRES_OK(ctx, ctx->allocate_output(kOutputMaxIndex, {}, &output_max));
+    output_min->flat<float>()(0) = is_non_negative ? 0.0f : -out_min_max;
     output_max->flat<float>()(0) = out_min_max;
   }
 
  private:
   float clip_value_max_ = std::numeric_limits<float>::infinity();
   const int kInputTensorIndex = 0;
-  const int kInputMin = 1;
-  const int kInputMax = 2;
-  const int kOutputMin = 0;
-  const int kOutputMax = 1;
+  const int kInputMinIndex = 1;
+  const int kInputMaxIndexIndex = 2;
+  const int kOutputMinIndex = 0;
+  const int kOutputMaxIndex = 1;
 };
 
 REGISTER_KERNEL_BUILDER(Name("RequantizationRangePerChannel")
diff --git a/tensorflow/core/kernels/mkl_requantize_ops_test.cc b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
index 44ab936471..ebc7687271 100644
--- a/tensorflow/core/kernels/mkl_requantize_ops_test.cc
+++ b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
@@ -13,6 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include <cmath>
+
 #include "tensorflow/core/framework/allocator.h"
 #include "tensorflow/core/framework/fake_input.h"
 #include "tensorflow/core/framework/node_def_builder.h"
@@ -27,8 +29,6 @@ limitations under the License.
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/platform/test_benchmark.h"
 
-#include <cmath>
-
 namespace tensorflow {
 
 class MklRequantizatedOpsTest : public OpsTestBase {};
@@ -43,24 +43,24 @@ class MklRequantizatedOpsTestHelper : public OpsTestBase {
 void MklRequantizatedOpsTestHelper::Setup(Tensor &input_tensor_qint32,
                                           float &range_weights_ch1,
                                           float &range_weights_ch2) {
-  // Step 1: Assumption of inputs
-  // ----------------------------
+  // Step 1: Input range assumptions
+  // -------------------------------
   // Assume input Tensor T (NHWC) in FP32 has range [0, 5.0]   size nt*ht*wt*ct
-  // Assume input Filter W (NHWC) with 2 output channels of    size nw*ht**wt*2
-  // logically,   Filter W has 2 channels W1 and W2 each of    size nw*ht**wt*1
-  // Assume input Filter W1(NHWC) in FP32 has range [-2.0, 2.0]size nw*ht**wt*1
-  // Assume input Filter W2(NHWC) in FP32 has range [-3.0, 3.0]size nw*ht**wt*1
-
-  // Step 2: Assumption of Quantizing inputs and weights (per channel)
-  // ------------------------------------------------------------------
-  // When these 2 Tensors, T and W are quantized using a Quantize Op.
-  // When  the input Tensor T (NHWC) is quantized to unsigned int8.
-  // While the input Filter W (NHWC) is qunatized to signed int8.
-  // hence T max value is mapped to ((2^8-1) = 255) while W to ((2^7)-1 = 127))
-
-  // Range of Quantized T  in int8[0   , 255] maps to orig T  in FP32[0   , 5.0]
-  // Range of Quantized W1 in int8[-127, 127] maps to orig W1 in FP32[-2.0, 2.0]
-  // Range of Quantized W2 in int8[-127, 127] maps to orig W2 in FP32[-3.0, 3.0]
+  // Assume input Filter W (NHWC) with 2 output channels of    size nw*ht*wt*2
+  // logically,   Filter W has 2 channels W1 and W2 each of    size nw*ht*wt*1
+  // Assume input Filter W1(NHWC) in FP32 has range [-2.0, 2.0]size nw*ht*wt*1
+  // Assume input Filter W2(NHWC) in FP32 has range [-3.0, 3.0]size nw*ht*wt*1
+
+  // Step 2: Quantization details (per channel)
+  // ------------------------------------------
+  // T and W are quantized using a Quantize Op.
+  // The input Tensor T (NHWC) is quantized to unsigned int8.
+  // The input Filter W (NHWC) is quantized to signed int8.
+  // Hence T's max value is mapped to ((2^8-1) = 255), while W's to ((2^7)-1 = 127)).
+
+  // Range of Quantized T  in uint8[0   , 255] maps to orig T  in FP32[0   , 5.0]
+  // Range of Quantized W1 in int8[-127, 127]  maps to orig W1 in FP32[-2.0, 2.0]
+  // Range of Quantized W2 in int8[-127, 127]  maps to orig W2 in FP32[-3.0, 3.0]
 
   // Hence the resolution of Quantized T will be 5.0/255
   // Hence the resolution of Quantized W1 will be 2.0/127
@@ -68,20 +68,20 @@ void MklRequantizatedOpsTestHelper::Setup(Tensor &input_tensor_qint32,
 
   // Step 3: Assumption of quantizedconv on quantized input&weights(per channel)
   // ---------------------------------------------------------------------------
-  // The input T and weights W1 (or W2) will be convolved (and multipled)
-  // The output Tensor T is in int32 whose range is [-2^31, 2^31]
-  // The Range of the Convolved T*W1 is 2^31 * 5.0/255 * 2.0/127 = 663110.59
-  // So Range of Convolved T*W1 in int32[-2^31, 22^31] that maps to
+  // The input T and weights W1 (or W2) will be convolved.
+  // The output Tensor T is in int32 whose range is [-(2^31-1), 2^31-1)
+  // The Range of the convolved T*W1 is ((2^31)-1) * 5.0/255 * 2.0/127 = 663110.59
+  // So the range of convolved T*W1 in int32[-(2^31-1), 22^31-1] that maps to
   // orig T Range in FP32[0,5.0] * [-2.0, 2.0] is [-663110.59, 663110.59]
 
-  // The Range of the Convolved T*W2 is 2^31 * 5.0/255 * 3.0/127 = 994665.88
-  // So Range of Convolved T*W2 in int32[-2^31, 22^31] that maps to
-  // orig T Range in FP32[0,5.0] * [-3.0, 3.0]  is [-994665.88, 994665.88]
+  // The Range of the convolved T*W2 is 2^31-1 * 5.0/255 * 3.0/127 = 994665.88
+  // So Range of convolved T*W2 in int32[-(2^31-1), 22^31-1] that maps to
+  // orig T Range in FP32 [0, 5.0] * [-3.0, 3.0]  is [-994665.88, 994665.88]
 
   // Step 4: Assumption output above is fed to Requantization_range_perchannel
   // --------------------------------------------------------------------------
-  // Here we recalculate the new Range for Convolved T*W so that we
-  // make good use in int8 qunatization from int32 to int8.
+  // Here we recalculate the new Range for convolved T*W so that we
+  // make good use in int8 quantization from int32 to int8.
 
   // We assume the above operations are performed and use these values above
   // as ranges for Requantization_range_perchannel_op.
@@ -102,10 +102,10 @@ void MklRequantizatedOpsTestHelper::Setup(Tensor &input_tensor_qint32,
   // test RequantizationRangePerChannelTest_ClipMax
 }
 
-// Following tests the RequantizationRangePerChannel Op wherein the range
+// Tests the RequantizationRangePerChannel Op wherein the range
 // of the weights is calculated per channel.
 TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_Basic) {
-  // Let us setup the tensor and inputs before we run this op.
+  // Let us set up the tensor and inputs before we run this op.
   float clip_max_value = pow(2, 31);
   float range_weights_ch1 = 0.0;
   float range_weights_ch2 = 0.0;
@@ -115,7 +115,7 @@ TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_Basic) {
   const int input_width = 4;
   const int input_channels = 2;
 
-  // define and input tensor T shape.
+  // Define the shape of T.
   Tensor input_tensor_qint32(DT_QINT32,
                              {1, input_height, input_width, input_channels});
 
@@ -202,9 +202,9 @@ TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_ClipMax) {
                             input_tensor_qint32.flat<qint32>());
 
   // Calculate the Min and max from the ranges
-  float ch1_min = -1.0 * range_weights_ch1;
+  float ch1_min = -range_weights_ch1;
   float ch1_max = range_weights_ch1;
-  float ch2_min = -1.0 * range_weights_ch2;
+  float ch2_min = -range_weights_ch2;
   float ch2_max = range_weights_ch2;
 
   // Add the Perchannel range Nodes to the Op.
diff --git a/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc b/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc
index f9aa550866..e531ec8931 100644
--- a/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc
+++ b/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc
@@ -1,4 +1,4 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -48,31 +48,31 @@ class MklRequantizePerChannelOp : public OpKernel {
   void Compute(OpKernelContext* ctx) override {
     try {
       const Tensor& input = ctx->input(kInputTensorIndex);
-      const Tensor& input_min_vec = ctx->input(kInputMinVec);
+      const Tensor& input_min_vec = ctx->input(kInputMinVecIndex);
       float* input_min_vec_data = (float*)const_cast<void*>(
           static_cast<const void*>(input_min_vec.flat<float>().data()));
-      const Tensor& input_max_vec = ctx->input(kInputMaxVec);
+      const Tensor& input_max_vec = ctx->input(kInputMaxVecIndex);
       float* input_max_vec_data = (float*)const_cast<void*>(
           static_cast<const void*>(input_max_vec.flat<float>().data()));
 
-      const Tensor& input_requested_min = ctx->input(this->kRequestMin);
+      const Tensor& input_requested_min = ctx->input(this->kRequestMinIndex);
       const float input_requested_min_float =
           input_requested_min.flat<float>()(0);
-      const Tensor& input_requested_max = ctx->input(this->kRequestMax);
+      const Tensor& input_requested_max = ctx->input(this->kRequestMaxIndex);
       const float input_requested_max_float =
           input_requested_max.flat<float>()(0);
 
       size_t depth = input_min_vec.NumElements();
       OP_REQUIRES(
           ctx, input_min_vec.dim_size(0) == depth,
-          errors::InvalidArgument("min has incorrect size, expected ", depth,
+          errors::InvalidArgument("input_min has incorrect size, expected ", depth,
                                   " was ", input_min_vec.dim_size(0)));
       OP_REQUIRES(
           ctx, input_max_vec.dim_size(0) == depth,
-          errors::InvalidArgument("max has incorrect size, expected ", depth,
+          errors::InvalidArgument("input_max has incorrect size, expected ", depth,
                                   " was ", input_max_vec.dim_size(0)));
 
-      if (out_type_ == DT_QINT8) CHECK(input_requested_min_float < 0.0f);
+      if (out_type_ == DT_QINT8) DCHECK(input_requested_min_float < 0.0f);
 
       const float factor = (out_type_ == DT_QINT8) ? 127.0f : 255.0f;
       float requested_min_max = std::max(std::abs(input_requested_min_float),
@@ -82,12 +82,11 @@ class MklRequantizePerChannelOp : public OpKernel {
                                                input.shape(), &output));
 
       std::vector<float> scales(depth);
-      for (int i = 0; i < depth; i++) {
+      for (int i = 0; i < depth; ++i) {
         float min_max_from_vec = std::max(std::abs(input_min_vec_data[i]),
                                           std::abs(input_max_vec_data[i]));
-        float scale =
+        scales[i] =
             factor * (min_max_from_vec / requested_min_max / (float)(1L << 31));
-        scales[i] = scale;
       }
 
       mkldnn::primitive_attr reorder_attr;
@@ -133,8 +132,8 @@ class MklRequantizePerChannelOp : public OpKernel {
 
       Tensor* output_min = nullptr;
       Tensor* output_max = nullptr;
-      OP_REQUIRES_OK(ctx, ctx->allocate_output(kOutputMin, {}, &output_min));
-      OP_REQUIRES_OK(ctx, ctx->allocate_output(kOutputMax, {}, &output_max));
+      OP_REQUIRES_OK(ctx, ctx->allocate_output(kOutputMinIndex, {}, &output_min));
+      OP_REQUIRES_OK(ctx, ctx->allocate_output(kOutputMaxIndex, {}, &output_max));
 
       output_min->flat<float>()(0) = input_requested_min_float;
       output_max->flat<float>()(0) = input_requested_max_float;
@@ -150,13 +149,13 @@ class MklRequantizePerChannelOp : public OpKernel {
 
  private:
   const int kInputTensorIndex = 0;
-  const int kInputMinVec = 1;
-  const int kInputMaxVec = 2;
-  const int kRequestMin = 3;
-  const int kRequestMax = 4;
+  const int kInputMinVecIndex = 1;
+  const int kInputMaxVecIndex = 2;
+  const int kRequestMinIndex = 3;
+  const int kRequestMaxIndex = 4;
   const int kOutputTensorIndex = 0;
-  const int kOutputMin = 1;
-  const int kOutputMax = 2;
+  const int kOutputMinIndex = 1;
+  const int kOutputMaxIndex = 2;
   DataType out_type_;
   engine cpu_engine_ = engine(engine::cpu, 0);
 };
-- 
GitLab


From a669a87180f348641641ed3f9fb38ff93463c969 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 29 Jan 2019 15:58:52 -0800
Subject: [PATCH 1536/2345] V13 androidx migration

PiperOrigin-RevId: 231490618
---
 .../example/android/tflitecamerademo/Camera2BasicFragment.java | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java b/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java
index 814d236872..d417edbbb5 100644
--- a/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java
+++ b/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java
@@ -45,7 +45,6 @@ import android.os.Bundle;
 import android.os.Handler;
 import android.os.HandlerThread;
 import android.support.annotation.NonNull;
-import android.support.v13.app.FragmentCompat;
 import android.support.v4.content.ContextCompat;
 import android.text.SpannableString;
 import android.text.SpannableStringBuilder;
@@ -62,6 +61,7 @@ import android.widget.ListView;
 import android.widget.NumberPicker;
 import android.widget.TextView;
 import android.widget.Toast;
+import androidx.legacy.app.FragmentCompat;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -71,7 +71,6 @@ import java.util.List;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 
-
 /** Basic fragments for the Camera. */
 public class Camera2BasicFragment extends Fragment
     implements FragmentCompat.OnRequestPermissionsResultCallback {
-- 
GitLab


From 87120ddf5881215bb7bb76169543d1705ca754f4 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Tue, 29 Jan 2019 16:05:40 -0800
Subject: [PATCH 1537/2345] [XLA] Add literal C64/C128 support to HLO parser.

Now the parser can handle complex numbers.

In addition, this patch fixes a few bugs I discovered:

 - We weren't checking for out-of-range values in sparse literals.
 - We were saying that bf16 values > F16_MAX were invalid, but in fact bf16 has
   a higher dynamic range than this.
 - We were accepting invalid sparse indices.  Strangley this doesn't crash or
   anything.

PiperOrigin-RevId: 231492104
---
 tensorflow/compiler/xla/literal.h             |   1 +
 tensorflow/compiler/xla/service/BUILD         |   1 +
 tensorflow/compiler/xla/service/hlo_parser.cc | 503 ++++++++++++------
 .../compiler/xla/service/hlo_parser_test.cc   | 169 +++++-
 4 files changed, 506 insertions(+), 168 deletions(-)

diff --git a/tensorflow/compiler/xla/literal.h b/tensorflow/compiler/xla/literal.h
index 041151fda1..368e460ff5 100644
--- a/tensorflow/compiler/xla/literal.h
+++ b/tensorflow/compiler/xla/literal.h
@@ -958,6 +958,7 @@ template <typename NativeT>
 void MutableLiteralBase::AppendSparseElement(
     absl::Span<const int64> multi_index, NativeT value,
     const ShapeIndex& shape_index) {
+  // TODO(jlebar): CHECK that multi_index is in range?
   Piece& p = piece(shape_index);
   const Shape& subshape = p.subshape();
   CHECK(LayoutUtil::IsSparseArray(subshape));
diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index b01234fe26..4f6509c1cb 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -3686,6 +3686,7 @@ cc_library(
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/strings:str_format",
+        "@com_google_absl//absl/types:variant",
     ],
 )
 
diff --git a/tensorflow/compiler/xla/service/hlo_parser.cc b/tensorflow/compiler/xla/service/hlo_parser.cc
index bef1e067a4..1e3d7a47ca 100644
--- a/tensorflow/compiler/xla/service/hlo_parser.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/compiler/xla/service/hlo_parser.h"
+#include <type_traits>
 
 #include "absl/algorithm/container.h"
 #include "absl/memory/memory.h"
@@ -21,6 +22,7 @@ limitations under the License.
 #include "absl/strings/str_format.h"
 #include "absl/strings/str_join.h"
 #include "absl/strings/str_split.h"
+#include "absl/types/variant.h"
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/literal_util.h"
 #include "tensorflow/compiler/xla/primitive_util.h"
@@ -44,8 +46,6 @@ using absl::StrCat;
 using absl::StrFormat;
 using absl::StrJoin;
 
-const double kF16max = 65504;
-
 // Creates and returns a schedule created using the order of the instructions in
 // the HloComputation::instructions() vectors in the module.
 HloSchedule ScheduleFromInstructionOrder(HloModule* module) {
@@ -60,6 +60,11 @@ HloSchedule ScheduleFromInstructionOrder(HloModule* module) {
   return schedule;
 }
 
+// Some functions accept either a linear index or a multi-dimensional index
+// (used for indexing into sparse literals).
+using LinearOrMultiIndex =
+    absl::variant<tensorflow::int64, absl::Span<const int64>>;
+
 // Parser for the HloModule::ToString() format text.
 class HloParser {
  public:
@@ -102,7 +107,7 @@ class HloParser {
   // Parse a single instruction worth of text.
   bool ParseSingleInstruction(HloModule* module);
 
-  // ParseXXX returns false if an error occurred.
+  // Parses a module, returning false if an error occurred.
   bool ParseHloModule(HloModule* module);
 
   bool ParseComputations(HloModule* module);
@@ -118,21 +123,30 @@ class HloParser {
   bool ParseNonTupleLiteral(Literal* literal, const Shape& shape);
   bool ParseDenseLiteral(Literal* literal, const Shape& shape);
   bool ParseSparseLiteral(Literal* literal, const Shape& shape);
-  template <typename LiteralNativeT>
-  bool ParseSparseLiteralHelper(Literal* literal, const Shape& shape);
 
-  // Sets the sub-value of literal at the given index to the given value. The
-  // literal's shape must have the default layout.
-  bool SetValueInLiteral(tensorflow::int64 value,
-                         tensorflow::int64 linear_index, Literal* literal);
-  bool SetValueInLiteral(double value, tensorflow::int64 linear_index,
+  // Sets the sub-value of literal at the given linear or sparse index to the
+  // given value. If the literal is dense, it myst have the default layout.
+  //
+  // `loc` should be the source location of the value.
+  bool SetValueInLiteral(LocTy loc, tensorflow::int64 value,
+                         LinearOrMultiIndex index, Literal* literal);
+  bool SetValueInLiteral(LocTy loc, double value, LinearOrMultiIndex index,
                          Literal* literal);
-  bool SetValueInLiteral(bool value, tensorflow::int64 linear_index,
+  bool SetValueInLiteral(LocTy loc, bool value, LinearOrMultiIndex index,
                          Literal* literal);
+  bool SetValueInLiteral(LocTy loc, std::complex<double> value,
+                         LinearOrMultiIndex index, Literal* literal);
+  // `loc` should be the source location of the value.
   template <typename LiteralNativeT, typename ParsedElemT>
-  bool SetValueInLiteralHelper(ParsedElemT value,
-                               tensorflow::int64 linear_index,
-                               Literal* literal);
+  bool SetValueInLiteralHelper(LocTy loc, ParsedElemT value,
+                               LinearOrMultiIndex index, Literal* literal);
+
+  // Checks whether the given value is within the range of LiteralNativeT.
+  // `loc` should be the source location of the value.
+  template <typename LiteralNativeT, typename ParsedElemT>
+  bool CheckParsedValueIsInRange(LocTy loc, ParsedElemT value);
+  template <typename LiteralNativeT>
+  bool CheckParsedValueIsInRange(LocTy loc, std::complex<double> value);
 
   bool ParseOperands(std::vector<HloInstruction*>* operands);
   // Fills parsed operands into 'operands' and expects a certain number of
@@ -268,6 +282,7 @@ class HloParser {
   bool ParsePrecision(PrecisionConfig::Precision* result);
   bool ParseInt64(tensorflow::int64* result);
   bool ParseDouble(double* result);
+  bool ParseComplex(std::complex<double>* result);
   bool ParseBool(bool* result);
   bool ParseToken(TokKind kind, const string& msg);
 
@@ -1822,130 +1837,150 @@ bool HloParser::ParseInstructionNames(
                     "expects '}' at the end of instruction name list");
 }
 
-bool HloParser::SetValueInLiteral(tensorflow::int64 value,
-                                  tensorflow::int64 linear_index,
-                                  Literal* literal) {
+bool HloParser::SetValueInLiteral(LocTy loc, tensorflow::int64 value,
+                                  LinearOrMultiIndex index, Literal* literal) {
   const Shape& shape = literal->shape();
   switch (shape.element_type()) {
     case S8:
-      return SetValueInLiteralHelper<tensorflow::int8>(value, linear_index,
+      return SetValueInLiteralHelper<tensorflow::int8>(loc, value, index,
                                                        literal);
     case S16:
-      return SetValueInLiteralHelper<tensorflow::int16>(value, linear_index,
+      return SetValueInLiteralHelper<tensorflow::int16>(loc, value, index,
                                                         literal);
     case S32:
-      return SetValueInLiteralHelper<tensorflow::int32>(value, linear_index,
+      return SetValueInLiteralHelper<tensorflow::int32>(loc, value, index,
                                                         literal);
     case S64:
-      return SetValueInLiteralHelper<tensorflow::int64>(value, linear_index,
+      return SetValueInLiteralHelper<tensorflow::int64>(loc, value, index,
                                                         literal);
     case U8:
-      return SetValueInLiteralHelper<tensorflow::uint8>(value, linear_index,
+      return SetValueInLiteralHelper<tensorflow::uint8>(loc, value, index,
                                                         literal);
     case U16:
-      return SetValueInLiteralHelper<tensorflow::uint16>(value, linear_index,
+      return SetValueInLiteralHelper<tensorflow::uint16>(loc, value, index,
                                                          literal);
     case U32:
-      return SetValueInLiteralHelper<tensorflow::uint32>(value, linear_index,
+      return SetValueInLiteralHelper<tensorflow::uint32>(loc, value, index,
                                                          literal);
     case U64:
-      return SetValueInLiteralHelper<tensorflow::uint64>(value, linear_index,
+      return SetValueInLiteralHelper<tensorflow::uint64>(loc, value, index,
                                                          literal);
     case PRED:
       // Bool type literals with rank >= 1 are printed in 0s and 1s.
-      return SetValueInLiteralHelper<bool>(static_cast<bool>(value),
-                                           linear_index, literal);
+      return SetValueInLiteralHelper<bool>(loc, static_cast<bool>(value), index,
+                                           literal);
     default:
       LOG(FATAL) << "unknown integral primitive type "
                  << PrimitiveType_Name(shape.element_type());
   }
 }
 
-bool HloParser::SetValueInLiteral(double value, tensorflow::int64 linear_index,
-                                  Literal* literal) {
+bool HloParser::SetValueInLiteral(LocTy loc, double value,
+                                  LinearOrMultiIndex index, Literal* literal) {
   const Shape& shape = literal->shape();
   switch (shape.element_type()) {
     case F16:
-      return SetValueInLiteralHelper<Eigen::half>(value, linear_index, literal);
+      return SetValueInLiteralHelper<Eigen::half>(loc, value, index, literal);
     case BF16:
-      return SetValueInLiteralHelper<tensorflow::bfloat16>(value, linear_index,
+      return SetValueInLiteralHelper<tensorflow::bfloat16>(loc, value, index,
                                                            literal);
     case F32:
-      return SetValueInLiteralHelper<float>(value, linear_index, literal);
+      return SetValueInLiteralHelper<float>(loc, value, index, literal);
     case F64:
-      return SetValueInLiteralHelper<double>(value, linear_index, literal);
+      return SetValueInLiteralHelper<double>(loc, value, index, literal);
     default:
       LOG(FATAL) << "unknown floating point primitive type "
                  << PrimitiveType_Name(shape.element_type());
   }
 }
 
-bool HloParser::SetValueInLiteral(bool value, tensorflow::int64 linear_index,
-                                  Literal* literal) {
+bool HloParser::SetValueInLiteral(LocTy loc, bool value,
+                                  LinearOrMultiIndex index, Literal* literal) {
   const Shape& shape = literal->shape();
   switch (shape.element_type()) {
     case PRED:
-      return SetValueInLiteralHelper<bool>(value, linear_index, literal);
+      return SetValueInLiteralHelper<bool>(loc, value, index, literal);
     default:
       LOG(FATAL) << PrimitiveType_Name(shape.element_type())
                  << " is not PRED type";
   }
 }
 
+bool HloParser::SetValueInLiteral(LocTy loc, std::complex<double> value,
+                                  LinearOrMultiIndex index, Literal* literal) {
+  const Shape& shape = literal->shape();
+  switch (shape.element_type()) {
+    case C64:
+      return SetValueInLiteralHelper<std::complex<float>>(loc, value, index,
+                                                          literal);
+    case C128:
+      return SetValueInLiteralHelper<std::complex<double>>(loc, value, index,
+                                                           literal);
+    default:
+      LOG(FATAL) << PrimitiveType_Name(shape.element_type())
+                 << " is not a complex type type";
+  }
+}
+
+template <typename T>
+string StringifyValue(T val) {
+  return StrCat(val);
+}
+template <>
+string StringifyValue(std::complex<double> val) {
+  return StrFormat("(%f, %f)", std::real(val), std::imag(val));
+}
+
 template <typename LiteralNativeT, typename ParsedElemT>
-bool HloParser::SetValueInLiteralHelper(ParsedElemT value,
-                                        tensorflow::int64 linear_index,
+bool HloParser::SetValueInLiteralHelper(LocTy loc, ParsedElemT value,
+                                        LinearOrMultiIndex index,
                                         Literal* literal) {
-  // Check that linear_index is in range.
-  if (linear_index >= ShapeUtil::ElementsIn(literal->shape())) {
-    return TokenError(
-        StrCat("trys to set value ", value, " to a literal in shape ",
-               ShapeUtil::HumanString(literal->shape()), " at linear index ",
-               linear_index, ", but the index is out of range"));
+  if (!CheckParsedValueIsInRange<LiteralNativeT>(loc, value)) {
+    return false;
   }
 
-  if (std::isnan(value) ||
-      (std::numeric_limits<ParsedElemT>::has_infinity &&
-       (std::numeric_limits<ParsedElemT>::infinity() == value ||
-        -std::numeric_limits<ParsedElemT>::infinity() == value))) {
-    // Skip range checking for non-finite value.
-  } else if (literal->shape().element_type() == F16 ||
-             literal->shape().element_type() == BF16) {
-    if (value > kF16max || value < -kF16max) {
-      return TokenError(StrCat(
-          "value ", value, " is out of range for literal's primitive type ",
-          PrimitiveType_Name(literal->shape().element_type())));
+  // Check that the index is in range and assign into the literal
+  if (auto* linear_index = absl::get_if<tensorflow::int64>(&index)) {
+    if (*linear_index >= ShapeUtil::ElementsIn(literal->shape())) {
+      return Error(loc, StrCat("trys to set value ", StringifyValue(value),
+                               " to a literal in shape ",
+                               ShapeUtil::HumanString(literal->shape()),
+                               " at linear index ", *linear_index,
+                               ", but the index is out of range"));
     }
-  } else if (std::is_unsigned<LiteralNativeT>::value) {
-    CHECK((std::is_same<ParsedElemT, tensorflow::int64>::value ||
-           std::is_same<ParsedElemT, bool>::value))
-        << "Unimplemented checking for ParsedElemT";
+    literal->data<LiteralNativeT>().at(*linear_index) =
+        static_cast<LiteralNativeT>(value);
+  } else {
+    auto* multi_index = absl::get_if<absl::Span<const int64>>(&index);
+    CHECK(multi_index != nullptr);
 
-    ParsedElemT upper_bound;
-    if (sizeof(LiteralNativeT) >= sizeof(ParsedElemT)) {
-      upper_bound = std::numeric_limits<ParsedElemT>::max();
-    } else {
-      upper_bound =
-          static_cast<ParsedElemT>(std::numeric_limits<LiteralNativeT>::max());
+    auto invalid_idx = [&](string msg) {
+      return Error(loc, StrFormat("Invalid sparse index [%s]. %s",
+                                  absl::StrJoin(*multi_index, ", "), msg));
+    };
+
+    const auto& shape = literal->shape();
+    if (shape.rank() != multi_index->size()) {
+      return invalid_idx(
+          StrFormat("Has rank %d, but constant has shape %s, which has rank %d",
+                    multi_index->size(), shape.ToString(), shape.rank()));
     }
-    if (value > upper_bound || value < 0) {
-      // Value is out of range for LiteralNativeT.
-      return TokenError(StrCat(
-          "value ", value, " is out of range for literal's primitive type ",
-          PrimitiveType_Name(literal->shape().element_type())));
-    }
-  } else if (value > static_cast<ParsedElemT>(
-                         std::numeric_limits<LiteralNativeT>::max()) ||
-             value < static_cast<ParsedElemT>(
-                         std::numeric_limits<LiteralNativeT>::lowest())) {
-    // Value is out of range for LiteralNativeT.
-    return TokenError(StrCat(
-        "value ", value, " is out of range for literal's primitive type ",
-        PrimitiveType_Name(literal->shape().element_type())));
+    for (int64 i = 0; i < shape.rank(); ++i) {
+      auto idx = (*multi_index)[i];
+      if (idx < 0) {
+        return invalid_idx(StrFormat(
+            "Sub-index value at %d, namely %d, cannot be negative.", i, idx));
+      }
+      if (idx >= shape.dimensions(i)) {
+        return invalid_idx(
+            StrFormat("Sub-index at %d, namely %d, doesn't fit within shape "
+                      "dimension %d in %s",
+                      i, idx, shape.dimensions(i), shape.ToString()));
+      }
+    }
+    literal->AppendSparseElement(*multi_index,
+                                 static_cast<LiteralNativeT>(value));
   }
-
-  literal->data<LiteralNativeT>().at(linear_index) =
-      static_cast<LiteralNativeT>(value);
   return true;
 }
 
@@ -2002,7 +2037,11 @@ bool HloParser::ParseNonTupleLiteral(Literal* literal, const Shape& shape) {
 }
 
 bool HloParser::ParseDenseLiteral(Literal* literal, const Shape& shape) {
-  const tensorflow::int64 rank = shape.rank();
+  // Cast `rank` to int because we call shape.dimensions(int rank) below, and if
+  // `rank` is an int64, that's an implicit narrowing conversion, which is
+  // implementation-defined behavior.
+  const int rank = static_cast<int>(shape.rank());
+
   // Create a literal with the given shape in default layout.
   *literal = LiteralUtil::CreateFromDimensions(
       shape.element_type(), AsInt64Slice(shape.dimensions()));
@@ -2026,6 +2065,24 @@ bool HloParser::ParseDenseLiteral(Literal* literal, const Shape& shape) {
                           }),
                   "]");
   };
+
+  auto add_one_elem_seen = [&] {
+    if (rank > 0) {
+      if (nest_level != rank) {
+        return TokenError(absl::StrFormat(
+            "expects nested array in rank %d, but sees %d", rank, nest_level));
+      }
+      elems_seen_per_dim[rank - 1]++;
+      if (elems_seen_per_dim[rank - 1] > shape.dimensions(rank - 1)) {
+        return TokenError(absl::StrFormat(
+            "expects %d elements on the minor-most dimension, but "
+            "sees more",
+            shape.dimensions(rank - 1)));
+      }
+    }
+    return true;
+  };
+
   do {
     switch (lexer_.GetKind()) {
       default:
@@ -2061,6 +2118,21 @@ bool HloParser::ParseDenseLiteral(Literal* literal, const Shape& shape) {
         lexer_.Lex();
         break;
       }
+      case TokKind::kLparen: {
+        if (!primitive_util::IsComplexType(shape.element_type())) {
+          return TokenError(
+              absl::StrFormat("unexpected '(' in literal.  Parens are only "
+                              "valid for complex literals"));
+        }
+
+        std::complex<double> value;
+        LocTy loc = lexer_.GetLoc();
+        if (!add_one_elem_seen() || !ParseComplex(&value) ||
+            !SetValueInLiteral(loc, value, linear_index++, literal)) {
+          return false;
+        }
+        break;
+      }
       case TokKind::kComma:
         // Skip.
         lexer_.Lex();
@@ -2072,23 +2144,11 @@ bool HloParser::ParseDenseLiteral(Literal* literal, const Shape& shape) {
       case TokKind::kw_nan:
       case TokKind::kw_inf:
       case TokKind::kNegInf: {
-        if (rank > 0) {
-          if (nest_level != rank) {
-            return TokenError(
-                absl::StrFormat("expects nested array in rank %d, but sees %d",
-                                rank, nest_level));
-          }
-          elems_seen_per_dim[rank - 1]++;
-          if (elems_seen_per_dim[rank - 1] > shape.dimensions(rank - 1)) {
-            return TokenError(absl::StrFormat(
-                "expects %d elements on the minor-most dimension, but "
-                "sees more",
-                shape.dimensions(rank - 1)));
-          }
-        }
+        add_one_elem_seen();
         if (lexer_.GetKind() == TokKind::kw_true ||
             lexer_.GetKind() == TokKind::kw_false) {
-          if (!SetValueInLiteral(lexer_.GetKind() == TokKind::kw_true,
+          if (!SetValueInLiteral(lexer_.GetLoc(),
+                                 lexer_.GetKind() == TokKind::kw_true,
                                  linear_index++, literal)) {
             return false;
           }
@@ -2101,7 +2161,7 @@ bool HloParser::ParseDenseLiteral(Literal* literal, const Shape& shape) {
             return Error(loc, StrCat("expects integer for primitive type: ",
                                      PrimitiveType_Name(shape.element_type())));
           }
-          if (!SetValueInLiteral(value, linear_index++, literal)) {
+          if (!SetValueInLiteral(loc, value, linear_index++, literal)) {
             return false;
           }
         } else if (primitive_util::IsFloatingPointType(shape.element_type())) {
@@ -2112,7 +2172,7 @@ bool HloParser::ParseDenseLiteral(Literal* literal, const Shape& shape) {
                 loc, StrCat("expect floating point value for primitive type: ",
                             PrimitiveType_Name(shape.element_type())));
           }
-          if (!SetValueInLiteral(value, linear_index++, literal)) {
+          if (!SetValueInLiteral(loc, value, linear_index++, literal)) {
             return false;
           }
         } else {
@@ -2129,48 +2189,7 @@ bool HloParser::ParseDenseLiteral(Literal* literal, const Shape& shape) {
 }
 
 bool HloParser::ParseSparseLiteral(Literal* literal, const Shape& shape) {
-  switch (shape.element_type()) {
-    case PRED:
-      return ParseSparseLiteralHelper<tensorflow::uint8>(literal, shape);
-    case S8:
-      return ParseSparseLiteralHelper<tensorflow::int8>(literal, shape);
-    case S16:
-      return ParseSparseLiteralHelper<tensorflow::int16>(literal, shape);
-    case S32:
-      return ParseSparseLiteralHelper<tensorflow::int32>(literal, shape);
-    case S64:
-      return ParseSparseLiteralHelper<tensorflow::int64>(literal, shape);
-    case U8:
-      return ParseSparseLiteralHelper<tensorflow::uint8>(literal, shape);
-    case U16:
-      return ParseSparseLiteralHelper<tensorflow::uint16>(literal, shape);
-    case U32:
-      return ParseSparseLiteralHelper<tensorflow::uint32>(literal, shape);
-    case U64:
-      return ParseSparseLiteralHelper<tensorflow::uint64>(literal, shape);
-    case F16:
-      return ParseSparseLiteralHelper<Eigen::half>(literal, shape);
-    case F32:
-      return ParseSparseLiteralHelper<float>(literal, shape);
-    case BF16:
-      return ParseSparseLiteralHelper<tensorflow::bfloat16>(literal, shape);
-    case F64:
-      return ParseSparseLiteralHelper<double>(literal, shape);
-    default:
-      return Error(lexer_.GetLoc(),
-                   StrCat("invalid primitive type for sparse literal: ",
-                          PrimitiveType_Name(shape.element_type())));
-  }
-}
-
-template <typename LiteralNativeT>
-bool HloParser::ParseSparseLiteralHelper(Literal* literal, const Shape& shape) {
-  std::vector<tensorflow::int64> index;
-
-  tensorflow::int64 rank = shape.rank();
-
   *literal = Literal(shape);
-
   if (!ParseToken(TokKind::kLbrace,
                   "expects '{' at the beginning of a sparse literal")) {
     return false;
@@ -2182,61 +2201,66 @@ bool HloParser::ParseSparseLiteralHelper(Literal* literal, const Shape& shape) {
       break;
     }
 
-    LocTy index_loc = lexer_.GetLoc();
-    index.clear();
+    std::vector<tensorflow::int64> index;
     if (lexer_.GetKind() == TokKind::kInt) {
       tensorflow::int64 single_index = lexer_.GetInt64Val();
       lexer_.Lex();
-      if (rank != 1) {
-        return Error(
-            index_loc,
-            StrCat("invalid single-dimensional index for shape with rank ",
-                   rank, ": ", single_index));
-      }
       index.push_back(single_index);
     } else {
       if (!ParseInt64List(TokKind::kLsquare, TokKind::kRsquare, TokKind::kComma,
                           &index)) {
         return false;
       }
-      if (index.size() != rank) {
-        return Error(
-            index_loc,
-            StrCat("invalid multi-dimension index for shape with rank ", rank,
-                   ": [", StrJoin(index, ", "), "]"));
-      }
     }
     if (!ParseToken(TokKind::kColon,
                     "expects ':' after after the sparse array index and before "
                     "the sparse array value")) {
       return false;
     }
+
     LocTy value_loc = lexer_.GetLoc();
-    LiteralNativeT value;
     if (lexer_.GetKind() == TokKind::kw_true ||
         lexer_.GetKind() == TokKind::kw_false) {
-      value = static_cast<LiteralNativeT>(lexer_.GetKind() == TokKind::kw_true);
+      bool value = lexer_.GetKind() == TokKind::kw_true;
+      if (!SetValueInLiteral(lexer_.GetLoc(), value, index, literal)) {
+        return false;
+      }
       lexer_.Lex();
     } else if (primitive_util::IsIntegralType(shape.element_type())) {
-      tensorflow::int64 value_s64;
-      if (!ParseInt64(&value_s64)) {
+      tensorflow::int64 value;
+      if (!ParseInt64(&value)) {
         return Error(value_loc,
                      StrCat("expects integer for primitive type: ",
                             PrimitiveType_Name(shape.element_type())));
       }
-      value = static_cast<LiteralNativeT>(value_s64);
+      if (!SetValueInLiteral(value_loc, value, index, literal)) {
+        return false;
+      }
     } else if (primitive_util::IsFloatingPointType(shape.element_type())) {
-      double value_f64;
-      if (!ParseDouble(&value_f64)) {
+      double value;
+      if (!ParseDouble(&value)) {
         return Error(value_loc,
                      StrCat("expects floating point value for primitive type: ",
                             PrimitiveType_Name(shape.element_type())));
       }
-      value = static_cast<LiteralNativeT>(value_f64);
+      if (!SetValueInLiteral(value_loc, value, index, literal)) {
+        return false;
+      }
+    } else if (primitive_util::IsComplexType(shape.element_type())) {
+      std::complex<double> value;
+      if (!ParseComplex(&value)) {
+        return Error(value_loc,
+                     StrCat("expects complex value for primitive type: ",
+                            PrimitiveType_Name(shape.element_type())));
+      }
+      if (!SetValueInLiteral(value_loc, value, index, literal)) {
+        return false;
+      }
     } else {
       LOG(FATAL) << "Unexpected element type: "
                  << PrimitiveType_Name(shape.element_type());
     }
+
     if (lexer_.GetKind() != TokKind::kRbrace &&
         !ParseToken(TokKind::kComma,
                     "expects ',' separator between sparse array elements")) {
@@ -2250,14 +2274,114 @@ bool HloParser::ParseSparseLiteralHelper(Literal* literal, const Shape& shape) {
           StrCat("number of sparse elements exceeds maximum for layout: ",
                  ShapeUtil::HumanStringWithLayout(shape)));
     }
-
-    literal->AppendSparseElement(index, value);
   }
 
   literal->SortSparseElements();
   return true;
 }
 
+// MaxFiniteValue is a type-traits helper used by
+// HloParser::CheckParsedValueIsInRange.
+template <typename T>
+struct MinMaxFiniteValue {
+  static T max() { return std::numeric_limits<T>::max(); }
+  static T min() { return std::numeric_limits<T>::lowest(); }
+};
+
+template <>
+struct MinMaxFiniteValue<Eigen::half> {
+  static double max() {
+    // Sadly this is not constexpr, so this forces `value` to be a method.
+    return static_cast<double>(Eigen::NumTraits<Eigen::half>::highest());
+  }
+  static double min() { return -max(); }
+};
+
+template <>
+struct MinMaxFiniteValue<bfloat16> {
+  static double max() { return static_cast<double>(bfloat16::highest()); }
+  static double min() { return -max(); }
+};
+
+template <typename LiteralNativeT, typename ParsedElemT>
+bool HloParser::CheckParsedValueIsInRange(LocTy loc, ParsedElemT value) {
+  PrimitiveType literal_ty =
+      primitive_util::NativeToPrimitiveType<LiteralNativeT>();
+  if (std::isnan(value) ||
+      (std::numeric_limits<ParsedElemT>::has_infinity &&
+       (std::numeric_limits<ParsedElemT>::infinity() == value ||
+        -std::numeric_limits<ParsedElemT>::infinity() == value))) {
+    // Skip range checking for non-finite value.
+  } else if (std::is_unsigned<LiteralNativeT>::value) {
+    CHECK((std::is_same<ParsedElemT, tensorflow::int64>::value ||
+           std::is_same<ParsedElemT, bool>::value))
+        << "Unimplemented checking for ParsedElemT";
+
+    ParsedElemT upper_bound;
+    if (sizeof(LiteralNativeT) >= sizeof(ParsedElemT)) {
+      upper_bound = std::numeric_limits<ParsedElemT>::max();
+    } else {
+      upper_bound =
+          static_cast<ParsedElemT>(std::numeric_limits<LiteralNativeT>::max());
+    }
+    if (value > upper_bound || value < 0) {
+      // Value is out of range for LiteralNativeT.
+      return Error(loc, StrCat("value ", value,
+                               " is out of range for literal's primitive type ",
+                               PrimitiveType_Name(literal_ty), " namely [0, ",
+                               upper_bound, "]."));
+    }
+  } else if (value > MinMaxFiniteValue<LiteralNativeT>::max() ||
+             value < MinMaxFiniteValue<LiteralNativeT>::min()) {
+    // Value is out of range for LiteralNativeT.
+    return Error(loc, StrCat("value ", value,
+                             " is out of range for literal's primitive type ",
+                             PrimitiveType_Name(literal_ty), " namely [",
+                             MinMaxFiniteValue<LiteralNativeT>::min(), ", ",
+                             MinMaxFiniteValue<LiteralNativeT>::max(), "]."));
+  }
+  return true;
+}
+
+template <typename LiteralNativeT>
+bool HloParser::CheckParsedValueIsInRange(LocTy loc,
+                                          std::complex<double> value) {
+  // e.g. `float` for std::complex<float>
+  using LiteralComplexComponentT =
+      decltype(std::real(std::declval<LiteralNativeT>()));
+
+  // We could do simply
+  //
+  //   return CheckParsedValueIsInRange<LiteralNativeT>(std::real(value)) &&
+  //          CheckParsedValueIsInRange<LiteralNativeT>(std::imag(value));
+  //
+  // but this would give bad error messages on failure.
+
+  auto check_component = [&](absl::string_view name, double v) {
+    if (std::isnan(v) || v == std::numeric_limits<double>::infinity() ||
+        v == -std::numeric_limits<double>::infinity()) {
+      // Skip range-checking for non-finite values.
+      return true;
+    }
+
+    double min = MinMaxFiniteValue<LiteralComplexComponentT>::min();
+    double max = MinMaxFiniteValue<LiteralComplexComponentT>::max();
+    if (v < min || v > max) {
+      // Value is out of range for LitearlComplexComponentT.
+      return Error(
+          loc,
+          StrCat(name, " part ", v,
+                 " is out of range for literal's primitive type ",
+                 PrimitiveType_Name(
+                     primitive_util::NativeToPrimitiveType<LiteralNativeT>()),
+                 ", namely [", min, ", ", max, "]."));
+    }
+    return true;
+  };
+  return check_component("real", std::real(value)) &&
+         check_component("imaginary", std::imag(value));
+}
+
 // operands ::= '(' operands1 ')'
 // operands1
 //   ::= /*empty*/
@@ -3352,9 +3476,18 @@ bool HloParser::ParseInt64(tensorflow::int64* result) {
 
 bool HloParser::ParseDouble(double* result) {
   switch (lexer_.GetKind()) {
-    case TokKind::kDecimal:
-      *result = lexer_.GetDecimalVal();
+    case TokKind::kDecimal: {
+      double val = lexer_.GetDecimalVal();
+      // If GetDecimalVal returns +/-inf, that means that we overflowed
+      // `double`.
+      if (std::isinf(val)) {
+        return TokenError(StrCat("Constant is out of range for double (+/-",
+                                 std::numeric_limits<double>::max(),
+                                 ") and so is unparsable."));
+      }
+      *result = val;
       break;
+    }
     case TokKind::kInt:
       *result = static_cast<double>(lexer_.GetInt64Val());
       break;
@@ -3374,6 +3507,42 @@ bool HloParser::ParseDouble(double* result) {
   return true;
 }
 
+bool HloParser::ParseComplex(std::complex<double>* result) {
+  if (lexer_.GetKind() != TokKind::kLparen) {
+    return TokenError("expects '(' before complex number");
+  }
+  lexer_.Lex();
+
+  double real;
+  LocTy loc = lexer_.GetLoc();
+  if (!ParseDouble(&real)) {
+    return Error(loc,
+                 "expect floating-point value for real part of complex number");
+  }
+
+  if (lexer_.GetKind() != TokKind::kComma) {
+    return TokenError(
+        absl::StrFormat("expect comma after real part of complex literal"));
+  }
+  lexer_.Lex();
+
+  double imag;
+  loc = lexer_.GetLoc();
+  if (!ParseDouble(&imag)) {
+    return Error(
+        loc,
+        "expect floating-point value for imaginary part of complex number");
+  }
+
+  if (lexer_.GetKind() != TokKind::kRparen) {
+    return TokenError(absl::StrFormat("expect ')' after complex number"));
+  }
+
+  *result = std::complex<double>(real, imag);
+  lexer_.Lex();
+  return true;
+}
+
 bool HloParser::ParseBool(bool* result) {
   if (lexer_.GetKind() != TokKind::kw_true &&
       lexer_.GetKind() != TokKind::kw_false) {
diff --git a/tensorflow/compiler/xla/service/hlo_parser_test.cc b/tensorflow/compiler/xla/service/hlo_parser_test.cc
index 6ba16cc82a..ff52a89d1b 100644
--- a/tensorflow/compiler/xla/service/hlo_parser_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser_test.cc
@@ -782,7 +782,17 @@ ENTRY %fusion.v3 () -> f32[3,2,1,1] {
 R"(HloModule sparse_f32
 
 ENTRY %sparse () -> f32[2,3,4] {
-  ROOT %foo = f32[2,3,4]sparse{10} constant({[0, 1, 2]: 1, [1, 2, 3]: 2, [2, 3, 4]: 3})
+  ROOT %foo = f32[2,3,4]sparse{10} constant({[0, 1, 2]: 1, [1, 2, 2]: 2, [1, 2, 3]: 3})
+}
+
+)"
+},
+{
+"SparseC128",
+R"(HloModule sparse_c128
+
+ENTRY %sparse () -> c128[2,3,4] {
+  ROOT %foo = c128[2,3,4]sparse{10} constant({[0, 1, 2]: (1, 0), [1, 2, 2]: (2, 5), [1, 2, 3]: (3, 10)})
 }
 
 )"
@@ -894,6 +904,28 @@ ENTRY %CustomCallWithLayoutConstraints (p0: (f32[2,2], f32[42,2,3]), p1: f32[123
   ROOT %custom-call = (f32[1,2,3]{0,2,1}, f32[1,2,3]{1,2,0}) custom-call((f32[2,2]{0,1}, f32[42,2,3]{0,1,2}) %p0, f32[123,4]{0,1} %p1), custom_call_target="baz", operand_layout_constraints={(f32[2,2]{1,0}, f32[42,2,3]{2,0,1}), f32[123,4]{1,0}}
 }
 
+)"
+},
+// Parse c64 literal
+{
+"ParseC64Literal",
+R"(HloModule ParseC64Literal
+
+ENTRY %ParseC64Literal () -> c64[2] {
+  ROOT %c = c64[2]{0} constant({(1, 2), (-inf, nan)})
+}
+
+)"
+},
+// Parse c128 literal
+{
+"ParseC128Literal",
+R"(HloModule ParseC128Literal
+
+ENTRY %ParseC128Literal () -> c128[2] {
+  ROOT %c = c128[2]{0} constant({(1, 2), (-inf, nan)})
+}
+
 )"
 },
   });
@@ -1314,6 +1346,30 @@ ENTRY AddDependency {
   ROOT sum = f32[] add(neg, exp)
 }
 
+)"
+},
+
+// A module containing constants equal to the min/max values of various data
+// types.
+{
+"MinMaxValues",
+R"(HloModule MinMaxValues
+
+ENTRY MinMaxValues {
+  x.s8 = s8[2]{0} constant({-128, 127})
+  x.s16 = s16[2]{0} constant({-32768, 32767})
+  x.s32 = s32[2]{0} constant({-2147483648, 2147483647})
+  x.u8 = u8[2]{0} constant({0, 255})
+  x.u16 = u16[2]{0} constant({0, 65535})
+  x.u32 = u32[2]{0} constant({0, 4294967295})
+  x.f16 = f16[2]{0} constant({-65504, 65504})
+  x.bf16 = bf16[2]{0} constant({-3.38953e+38, 3.38953e+38})
+  x.f32 = f32[2]{0} constant({-3.40282e+38, 3.40282e+38})
+  x.f64 = f64[2]{0} constant({-1.79769e+308, 1.79769e+308})
+  x.c64 = c64[2]{0} constant({(-3.40282e+38, 3.40282e+38), (3.40282e+38, -3.40282e+38)})
+  ROOT c.c128 = c128[2]{0} constant({(-1.79769e+308, 1.79769e+308), (1.79769e+308, -1.79769e+308)})
+}
+
 )"
 },
 });
@@ -1543,6 +1599,37 @@ ENTRY %ConstantF16Overflow.v4 () -> f16[] {
                   "is out of range for literal's primitive type F16");
 }
 
+TEST_F(HloParserTest, ConstantBf16NoOverflow) {
+  // 65505 is in range for bf16.
+  const string original = R"(
+  HloModule test_module
+  ENTRY test {
+    ROOT c = bf16[] constant(-65505)
+  })";
+  EXPECT_EQ(Status::OK(), ParseHloString(original).status());
+}
+
+TEST_F(HloParserTest, ConstantBf16Overflow) {
+  // 1e100 is out of range for bf16.
+  const string original = R"(
+  HloModule test_module
+  ENTRY test {
+    ROOT c = bf16[] constant(1e100)
+  })";
+  ExpectHasSubstr(ParseHloString(original).status().error_message(),
+                  "out of range");
+}
+
+TEST_F(HloParserTest, ConstantF16OverflowInSparseArray) {
+  const string original = R"(
+    HloModule test_module
+    ENTRY test {
+      ROOT c = f16[5]sparse{10} constant({[0]: 0, [1]: -65505})
+    })";
+  ExpectHasSubstr(ParseHloString(original).status().error_message(),
+                  "is out of range for literal's primitive type F16");
+}
+
 TEST_F(HloParserTest, ConstantUnsignedUnderflow) {
   const string original = R"(
       HloModule ConstantUnsignedUnderflow_module
@@ -1577,6 +1664,46 @@ TEST_F(HloParserTest, ConstantUnsignedInt64Overflow) {
   EXPECT_NE(Status::OK(), result.status());
 }
 
+TEST_F(HloParserTest, ConstantC64Overflow) {
+  const string original = R"(
+      HloModule test_module
+      ENTRY test () -> c64[] {
+        ROOT c = c64[] constant((1e100, 0))
+      })";
+  auto result = ParseHloString(original);
+  EXPECT_NE(Status::OK(), result.status());
+}
+
+TEST_F(HloParserTest, ConstantC64Underflow) {
+  const string original = R"(
+      HloModule test_module
+      ENTRY test () -> c64[] {
+        ROOT c = c64[] constant((0, -1e100))
+      })";
+  auto result = ParseHloString(original);
+  EXPECT_NE(Status::OK(), result.status());
+}
+
+TEST_F(HloParserTest, ConstantF64Overflow) {
+  const string original = R"(
+      HloModule test_module
+      ENTRY test {
+        ROOT c = f64[] constant(1.8e308)
+      })";
+  auto result = ParseHloString(original);
+  EXPECT_NE(Status::OK(), result.status());
+}
+
+TEST_F(HloParserTest, ConstantF64Underflow) {
+  const string original = R"(
+      HloModule test_module
+      ENTRY test {
+        ROOT c = f64[] constant(-1.8e308)
+      })";
+  auto result = ParseHloString(original);
+  EXPECT_NE(Status::OK(), result.status());
+}
+
 TEST_F(HloParserTest, ConstantWithExp) {
   const string original = R"(HloModule ConstantWithExp_module
 
@@ -2291,6 +2418,46 @@ ENTRY %entrycomp (p: f32[2,2]) -> f32[2,2] {
                   " with the shape of the operand instruction f32[2,2]{1,0}.");
 }
 
+TEST_F(HloParserTest, OutOfRangeSparseIndex) {
+  const string original = R"(
+    HloModule test_module
+    ENTRY test {
+      ROOT c = f16[5]sparse{10} constant({[100]: 0})
+    })";
+  ExpectHasSubstr(ParseHloString(original).status().error_message(),
+                  "Invalid sparse index");
+}
+
+TEST_F(HloParserTest, NegativeSparseIndex) {
+  const string original = R"(
+    HloModule test_module
+    ENTRY test {
+      ROOT c = f16[5]sparse{10} constant({-1: 0})
+    })";
+  ExpectHasSubstr(ParseHloString(original).status().error_message(),
+                  "Invalid sparse index");
+}
+
+TEST_F(HloParserTest, SparseIndexWithRankTooLarge) {
+  const string original = R"(
+    HloModule test_module
+    ENTRY test {
+      ROOT c = f16[5]sparse{10} constant({[0, 0]: 0})
+    })";
+  ExpectHasSubstr(ParseHloString(original).status().error_message(),
+                  "Invalid sparse index");
+}
+
+TEST_F(HloParserTest, SparseIndexWithRankTooSmall) {
+  const string original = R"(
+    HloModule test_module
+    ENTRY test {
+      ROOT c = f16[5, 5]sparse{10} constant({[0]: 0})
+    })";
+  ExpectHasSubstr(ParseHloString(original).status().error_message(),
+                  "Invalid sparse index");
+}
+
 TEST_F(HloParserTest, ParseShapeStringR2F32) {
   string shape_string = "f32[123,456]";
   TF_ASSERT_OK_AND_ASSIGN(Shape actual, ParseShape(shape_string));
-- 
GitLab


From ec0b21d5631de16380186c15769550bdcbec0865 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 29 Jan 2019 16:08:27 -0800
Subject: [PATCH 1538/2345] [Grappler] Support stateful check of functions
 within functions.

PiperOrigin-RevId: 231492627
---
 tensorflow/core/grappler/grappler_item.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/grappler/grappler_item.cc b/tensorflow/core/grappler/grappler_item.cc
index e27d2b049c..d4bd2cc763 100644
--- a/tensorflow/core/grappler/grappler_item.cc
+++ b/tensorflow/core/grappler/grappler_item.cc
@@ -122,8 +122,10 @@ std::unordered_set<string> GrapplerItem::NodesToPreserve() const {
   // placeholders inside functions, and we want to catch such invalid graphs
   // early.
   if (optimization_options_.is_function_instantiation) {
+    FunctionLibraryDefinition fn_library(OpRegistry::Global(), graph.library());
     for (const NodeDef& node : graph.node()) {
-      if (IsStateful(node) || IsDataset(node) || IsPlaceholder(node)) {
+      if (IsStateful(node, &fn_library) || IsDataset(node) ||
+          IsPlaceholder(node)) {
         result.insert(node.name());
       }
     }
-- 
GitLab


From ec60dbb4aea624ca444b0af7e1024cbea7346931 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Tue, 29 Jan 2019 16:26:38 -0800
Subject: [PATCH 1539/2345] [XLA] Handle IsFinite on f16 and bf16 in
 HloEvaluator.

PiperOrigin-RevId: 231496004
---
 .../compiler/xla/service/hlo_evaluator.cc     | 56 +++++++++++++++----
 .../xla/service/hlo_evaluator_test.cc         | 34 +++++++++++
 2 files changed, 79 insertions(+), 11 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index ecde37be58..29e41433f4 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -490,15 +490,52 @@ Status HloEvaluator::HandleConcatenate(HloInstruction* concatenate) {
 
 Status HloEvaluator::HandleIsFinite(HloInstruction* is_finite) {
   auto operand = is_finite->operand(0);
-  if (!ShapeUtil::ElementIsFloating(operand->shape())) {
-    return InvalidArgument(
-        "expected element type in shape to be float for IsFinite op, got: %s",
-        PrimitiveType_Name(operand->shape().element_type()));
-  }
+  auto elem_ty = operand->shape().element_type();
+  switch (elem_ty) {
+    case PRED:
+    case TUPLE:
+    case OPAQUE:
+    case TOKEN:
+    case S8:
+    case S16:
+    case S32:
+    case S64:
+    case U8:
+    case U16:
+    case U32:
+    case U64:
+    case C64:
+    case C128:
+    // Explicitly enumerate all types in this switch so that when we add a new
+    // type, we'll get a compile error here.
+    case PRIMITIVE_TYPE_INVALID:
+    case PrimitiveType_INT_MIN_SENTINEL_DO_NOT_USE_:
+    case PrimitiveType_INT_MAX_SENTINEL_DO_NOT_USE_:
+      return InvalidArgument(
+          "expected element type in shape to be floating point, but "
+          "got: %s",
+          PrimitiveType_Name(elem_ty));
 
-  switch (operand->shape().element_type()) {
-    case F16:
-      return Unimplemented("unhandled primitive type: F16.");
+    case F16: {
+      auto result_or = ElementWiseUnaryOpImpl<bool, Eigen::half>(
+          is_finite,
+          [](Eigen::half elem_operand) {
+            return std::isfinite(static_cast<float>(elem_operand));
+          },
+          GetEvaluatedLiteralFor(operand));
+      TF_ASSIGN_OR_RETURN(evaluated_[is_finite], std::move(result_or));
+      break;
+    }
+    case BF16: {
+      auto result_or = ElementWiseUnaryOpImpl<bool, bfloat16>(
+          is_finite,
+          [](bfloat16 elem_operand) {
+            return std::isfinite(static_cast<float>(elem_operand));
+          },
+          GetEvaluatedLiteralFor(operand));
+      TF_ASSIGN_OR_RETURN(evaluated_[is_finite], std::move(result_or));
+      break;
+    }
     case F32: {
       auto result_or = ElementWiseUnaryOpImpl<bool, float>(
           is_finite,
@@ -515,9 +552,6 @@ Status HloEvaluator::HandleIsFinite(HloInstruction* is_finite) {
       TF_ASSIGN_OR_RETURN(evaluated_[is_finite], std::move(result_or));
       break;
     }
-    default:
-      LOG(FATAL) << "HandleIsFinite: unknown/unhandled primitive type: "
-                 << PrimitiveType_Name(operand->shape().element_type());
   }
 
   return Status::OK();
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
index 9bc71c9d6c..fb8cd299ce 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
@@ -3119,5 +3119,39 @@ TEST_F(HloEvaluatorTest, EvaluateCustomCall_ManyInputs) {
   EXPECT_TRUE(absl::c_equal(expected_data, actual_literal.data<uint32>()));
 }
 
+TEST_F(HloEvaluatorTest, IsFiniteF16) {
+  constexpr absl::string_view hlo_text = R"(
+  HloModule test
+
+  ENTRY IsFiniteTest {
+    c = f16[6] constant({nan, 7, nan, -1, inf, -inf})
+    ROOT is-finite = pred[6] is-finite(c)
+  })";
+
+  TF_ASSERT_OK_AND_ASSIGN(m_, ParseAndReturnVerifiedModule(hlo_text));
+  TF_ASSERT_OK_AND_ASSIGN(
+      Literal actual_literal,
+      HloEvaluator().Evaluate(*m_->entry_computation(), {}));
+  EXPECT_THAT(actual_literal.data<bool>(),
+              ::testing::ElementsAre(false, true, false, true, false, false));
+}
+
+TEST_F(HloEvaluatorTest, IsFiniteBf16) {
+  constexpr absl::string_view hlo_text = R"(
+  HloModule test
+
+  ENTRY IsFiniteTest {
+    c = bf16[6] constant({nan, 7, nan, -1, inf, -inf})
+    ROOT is-finite = pred[6] is-finite(c)
+  })";
+
+  TF_ASSERT_OK_AND_ASSIGN(m_, ParseAndReturnVerifiedModule(hlo_text));
+  TF_ASSERT_OK_AND_ASSIGN(
+      Literal actual_literal,
+      HloEvaluator().Evaluate(*m_->entry_computation(), {}));
+  EXPECT_THAT(actual_literal.data<bool>(),
+              ::testing::ElementsAre(false, true, false, true, false, false));
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From d4f358a44bec62413cd5b52391a2249015b8ea74 Mon Sep 17 00:00:00 2001
From: Yuanzhong Xu <yuanzx@google.com>
Date: Tue, 29 Jan 2019 16:41:22 -0800
Subject: [PATCH 1540/2345] [XLA] Expose replica ID in HLO

PiperOrigin-RevId: 231498514
---
 tensorflow/compiler/xla/client/xla_builder.cc | 10 ++++++++++
 tensorflow/compiler/xla/client/xla_builder.h  |  6 ++++++
 .../compiler/xla/g3doc/operation_semantics.md | 14 ++++++++++++++
 .../compiler/xla/service/dfs_hlo_visitor.h    |  1 +
 .../service/dfs_hlo_visitor_with_default.h    |  3 +++
 .../compiler/xla/service/hlo_cost_analysis.cc |  4 ++++
 .../compiler/xla/service/hlo_cost_analysis.h  |  1 +
 .../compiler/xla/service/hlo_graph_dumper.cc  |  1 +
 .../compiler/xla/service/hlo_instruction.cc   | 19 +++++++++++++++++++
 .../compiler/xla/service/hlo_instruction.h    |  3 +++
 tensorflow/compiler/xla/service/hlo_opcode.h  |  3 ++-
 tensorflow/compiler/xla/service/hlo_parser.cc | 11 +++++++++++
 .../compiler/xla/service/hlo_verifier.cc      |  4 ++++
 .../compiler/xla/service/hlo_verifier.h       |  1 +
 .../xla/service/instruction_fusion.cc         |  1 +
 .../compiler/xla/service/layout_assignment.cc |  1 +
 16 files changed, 82 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc
index 5c9f9f7088..c00ba26295 100644
--- a/tensorflow/compiler/xla/client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_builder.cc
@@ -2269,6 +2269,14 @@ XlaOp XlaBuilder::CollectivePermute(
   });
 }
 
+XlaOp XlaBuilder::ReplicaId() {
+  return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    HloInstructionProto instr;
+    *instr.mutable_shape() = ShapeUtil::MakeShape(U32, {}).ToProto();
+    return AddInstruction(std::move(instr), HloOpcode::kReplicaId, {});
+  });
+}
+
 XlaOp XlaBuilder::SelectAndScatter(const XlaOp& operand,
                                    const XlaComputation& select,
                                    absl::Span<const int64> window_dimensions,
@@ -3219,6 +3227,8 @@ XlaOp CollectivePermute(
   return operand.builder()->CollectivePermute(operand, source_target_pairs);
 }
 
+XlaOp ReplicaId(XlaBuilder* builder) { return builder->ReplicaId(); }
+
 XlaOp SelectAndScatter(const XlaOp& operand, const XlaComputation& select,
                        absl::Span<const int64> window_dimensions,
                        absl::Span<const int64> window_strides, Padding padding,
diff --git a/tensorflow/compiler/xla/client/xla_builder.h b/tensorflow/compiler/xla/client/xla_builder.h
index e024e2810d..4314747fff 100644
--- a/tensorflow/compiler/xla/client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_builder.h
@@ -562,6 +562,8 @@ class XlaBuilder {
       const XlaOp& operand,
       const std::vector<std::pair<int64, int64>>& source_target_pairs);
 
+  XlaOp ReplicaId();
+
   XlaOp SelectAndScatter(const XlaOp& operand, const XlaComputation& select,
                          absl::Span<const int64> window_dimensions,
                          absl::Span<const int64> window_strides,
@@ -1040,6 +1042,7 @@ class XlaBuilder {
   friend XlaOp CollectivePermute(
       const XlaOp& operand,
       const std::vector<std::pair<int64, int64>>& source_target_pairs);
+  friend XlaOp ReplicaId(XlaBuilder* builder);
   friend XlaOp SelectAndScatter(const XlaOp& operand,
                                 const XlaComputation& select,
                                 absl::Span<const int64> window_dimensions,
@@ -1699,6 +1702,9 @@ XlaOp CollectivePermute(
     const XlaOp& operand,
     const std::vector<std::pair<int64, int64>>& source_target_pairs);
 
+// Enqueues an operation that returns the replica ID.
+XlaOp ReplicaId(XlaBuilder* builder);
+
 // Enqueues an operation that scatters the `source` array to the selected
 // indices of each window.
 XlaOp SelectAndScatter(const XlaOp& operand, const XlaComputation& select,
diff --git a/tensorflow/compiler/xla/g3doc/operation_semantics.md b/tensorflow/compiler/xla/g3doc/operation_semantics.md
index c5f9377f98..363fd17b69 100644
--- a/tensorflow/compiler/xla/g3doc/operation_semantics.md
+++ b/tensorflow/compiler/xla/g3doc/operation_semantics.md
@@ -1873,6 +1873,20 @@ non-deterministic. Therefore, the reduction function should not be overly
 sensitive to reassociation. See the discussion about associativity in the
 context of [`Reduce`](#reduce) for more details.
 
+## ReplicaId
+
+See also
+[`XlaBuilder::ReplicaId`](https://www.tensorflow.org/code/tensorflow/compiler/xla/client/xla_builder.h).
+
+Returns the unique ID (U32 scalar) of the replica.
+
+<b> `ReplicaId()` </b>
+
+The unique ID of each replica is an unsigned integer in the interval `[0, N)`,
+where `N` is the number of replicas. Since all the replicas are running the same
+program, a `ReplicaId()` call in the program will return a different value on
+each replica.
+
 ## Reshape
 
 See also
diff --git a/tensorflow/compiler/xla/service/dfs_hlo_visitor.h b/tensorflow/compiler/xla/service/dfs_hlo_visitor.h
index 2132468b90..c54e954c22 100644
--- a/tensorflow/compiler/xla/service/dfs_hlo_visitor.h
+++ b/tensorflow/compiler/xla/service/dfs_hlo_visitor.h
@@ -108,6 +108,7 @@ class DfsHloVisitorBase {
   virtual Status HandleAllReduce(HloInstructionPtr hlo) = 0;
   virtual Status HandleAllToAll(HloInstructionPtr hlo) = 0;
   virtual Status HandleCollectivePermute(HloInstructionPtr hlo) = 0;
+  virtual Status HandleReplicaId(HloInstructionPtr hlo) = 0;
   virtual Status HandleGetDimensionSize(HloInstructionPtr hlo) = 0;
   virtual Status HandleCompare(HloInstructionPtr hlo) {
     return HandleElementwiseBinary(hlo);
diff --git a/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h b/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h
index 680dd256bb..33b2cc3fb0 100644
--- a/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h
+++ b/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h
@@ -100,6 +100,9 @@ class DfsHloVisitorWithDefaultBase
   Status HandleCollectivePermute(HloInstructionPtr hlo) override {
     return DefaultAction(hlo);
   }
+  Status HandleReplicaId(HloInstructionPtr hlo) override {
+    return DefaultAction(hlo);
+  }
   Status HandleRng(HloInstructionPtr random) override {
     return DefaultAction(random);
   }
diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
index 76fd402b2c..9c7239ff6f 100644
--- a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
+++ b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
@@ -570,6 +570,10 @@ Status HloCostAnalysis::HandleCollectivePermute(const HloInstruction* /*hlo*/) {
   return Status::OK();
 }
 
+Status HloCostAnalysis::HandleReplicaId(const HloInstruction* /*hlo*/) {
+  return Status::OK();
+}
+
 Status HloCostAnalysis::HandleRng(const HloInstruction* random) {
   // TODO(b/26346211): Implement better estimates for the RNG cost, since the
   // cost changes with the implementation and the distribution. For now, assume
diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis.h b/tensorflow/compiler/xla/service/hlo_cost_analysis.h
index b52305626d..421786e20a 100644
--- a/tensorflow/compiler/xla/service/hlo_cost_analysis.h
+++ b/tensorflow/compiler/xla/service/hlo_cost_analysis.h
@@ -74,6 +74,7 @@ class HloCostAnalysis : public ConstDfsHloVisitor {
   Status HandleAllReduce(const HloInstruction* crs) override;
   Status HandleAllToAll(const HloInstruction* hlo) override;
   Status HandleCollectivePermute(const HloInstruction* hlo) override;
+  Status HandleReplicaId(const HloInstruction* hlo) override;
   Status HandleInfeed(const HloInstruction* infeed) override;
   Status HandleOutfeed(const HloInstruction* outfeed) override;
   Status HandleRng(const HloInstruction* random) override;
diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
index 4c7f5e9e7d..46ee99923e 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
@@ -1037,6 +1037,7 @@ ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) {
     case HloOpcode::kRecvDone:
     case HloOpcode::kSend:
     case HloOpcode::kSendDone:
+    case HloOpcode::kReplicaId:
       return kBrown;
     case HloOpcode::kCall:
     case HloOpcode::kConditional:
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index 3c92554ad4..1b677bc25c 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -372,6 +372,13 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
           CreateCollectivePermute(shape, operands(0), source_target_pairs);
       break;
     }
+    case HloOpcode::kReplicaId: {
+      TF_RET_CHECK(proto.operand_ids_size() == 0)
+          << "ReplicaId instruction should have 0 operand but sees "
+          << proto.operand_ids_size();
+      instruction = CreateReplicaId();
+      break;
+    }
     case HloOpcode::kConvolution: {
       TF_RET_CHECK(proto.operand_ids_size() == 2)
           << "Convolution instruction should have 2 operands but sees "
@@ -825,6 +832,11 @@ HloInstruction::CreateCollectivePermute(
       shape, operand, source_target_pairs);
 }
 
+/* static */ std::unique_ptr<HloInstruction> HloInstruction::CreateReplicaId() {
+  return absl::WrapUnique(
+      new HloInstruction(HloOpcode::kReplicaId, ShapeUtil::MakeShape(U32, {})));
+}
+
 /* static */ std::unique_ptr<HloInstruction> HloInstruction::CreateInfeed(
     const Shape& infeed_shape, HloInstruction* token_operand,
     const string& config) {
@@ -1441,6 +1453,10 @@ std::unique_ptr<HloInstruction> HloInstruction::CloneWithNewOperands(
       CHECK_EQ(new_operands.size(), 2);
       clone = CreateAddDependency(new_operands[0], new_operands[1]);
       break;
+    case HloOpcode::kReplicaId:
+      CHECK_EQ(new_operands.size(), 0);
+      clone = CreateReplicaId();
+      break;
   }
   // SetupDerivedInstruction will setup the precision_config_ field.
   SetupDerivedInstruction(clone.get());
@@ -1710,6 +1726,7 @@ bool HloInstruction::IdenticalSlowPath(
     case HloOpcode::kReal:
     case HloOpcode::kRemainder:
     case HloOpcode::kReshape:
+    case HloOpcode::kReplicaId:
     case HloOpcode::kRoundNearestAfz:
     case HloOpcode::kSelect:
     case HloOpcode::kShiftLeft:
@@ -2464,6 +2481,8 @@ Status HloInstruction::Visit(DfsHloVisitorBase<HloInstructionPtr>* visitor) {
       return visitor->HandleAllToAll(this);
     case HloOpcode::kCollectivePermute:
       return visitor->HandleCollectivePermute(this);
+    case HloOpcode::kReplicaId:
+      return visitor->HandleReplicaId(this);
     case HloOpcode::kTuple:
       return visitor->HandleTuple(this);
     case HloOpcode::kMap:
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h
index 2c29b6c243..c11d29d33e 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.h
+++ b/tensorflow/compiler/xla/service/hlo_instruction.h
@@ -494,6 +494,9 @@ class HloInstruction {
       const Shape& shape, HloInstruction* operand,
       const std::vector<std::pair<int64, int64>>& source_target_pairs);
 
+  // Creates an instruction that returns a U32 replica ID.
+  static std::unique_ptr<HloInstruction> CreateReplicaId();
+
   // Creates a conversion instruction, where operand is the data to convert and
   // shape is the target shape for the conversion.
   static std::unique_ptr<HloInstruction> CreateConvert(const Shape& shape,
diff --git a/tensorflow/compiler/xla/service/hlo_opcode.h b/tensorflow/compiler/xla/service/hlo_opcode.h
index 94122ac38f..bf9b3c8117 100644
--- a/tensorflow/compiler/xla/service/hlo_opcode.h
+++ b/tensorflow/compiler/xla/service/hlo_opcode.h
@@ -139,7 +139,8 @@ namespace xla {
   V(kTranspose, "transpose")                                 \
   V(kTuple, "tuple", kHloOpcodeIsVariadic)                   \
   V(kTupleSelect, "tuple-select")                            \
-  V(kWhile, "while")
+  V(kWhile, "while")                                         \
+  V(kReplicaId, "replica-id")
 
 enum class HloOpcode {
 #define DECLARE_ENUM(enum_name, opcode_name, ...) enum_name,
diff --git a/tensorflow/compiler/xla/service/hlo_parser.cc b/tensorflow/compiler/xla/service/hlo_parser.cc
index 1e3d7a47ca..1a930370a9 100644
--- a/tensorflow/compiler/xla/service/hlo_parser.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser.cc
@@ -845,6 +845,17 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
           HloInstruction::CreateCollectivePermute(shape, operands[0], pairs));
       break;
     }
+    case HloOpcode::kReplicaId: {
+      if (!ParseOperands(&operands, /*expected_size=*/1) ||
+          !ParseAttributes(attrs)) {
+        return false;
+      }
+      if (!operands.empty()) {
+        return false;
+      }
+      instruction = builder->AddInstruction(HloInstruction::CreateReplicaId());
+      break;
+    }
     case HloOpcode::kReshape: {
       if (!ParseOperands(&operands, /*expected_size=*/1) ||
           !ParseAttributes(attrs)) {
diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc
index 144c01eac1..4caaa5a32b 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier.cc
@@ -184,6 +184,10 @@ Status ShapeVerifier::HandleAllToAll(HloInstruction* hlo) {
                     ShapeInference::InferAllToAllTupleShape(operand_shapes));
 }
 
+Status ShapeVerifier::HandleReplicaId(HloInstruction* hlo) {
+  return CheckShape(hlo, ShapeUtil::MakeShape(U32, {}));
+}
+
 Status ShapeVerifier::HandleCollectivePermute(HloInstruction* hlo) {
   TF_RETURN_IF_ERROR(CheckOperandCount(hlo, 1));
   return CheckShape(hlo, ShapeInference::InferCollectivePermuteShape(
diff --git a/tensorflow/compiler/xla/service/hlo_verifier.h b/tensorflow/compiler/xla/service/hlo_verifier.h
index 479905b317..facb76a124 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier.h
+++ b/tensorflow/compiler/xla/service/hlo_verifier.h
@@ -55,6 +55,7 @@ class ShapeVerifier : public DfsHloVisitor {
   Status HandleAllReduce(HloInstruction* crs) override;
   Status HandleAllToAll(HloInstruction* hlo) override;
   Status HandleCollectivePermute(HloInstruction* hlo) override;
+  Status HandleReplicaId(HloInstruction* hlo) override;
   Status HandleReducePrecision(HloInstruction* reduce_precision) override;
   Status HandleInfeed(HloInstruction*) override;
   Status HandleOutfeed(HloInstruction*) override;
diff --git a/tensorflow/compiler/xla/service/instruction_fusion.cc b/tensorflow/compiler/xla/service/instruction_fusion.cc
index b97060535d..d4794acb2f 100644
--- a/tensorflow/compiler/xla/service/instruction_fusion.cc
+++ b/tensorflow/compiler/xla/service/instruction_fusion.cc
@@ -95,6 +95,7 @@ bool IsAlwaysDuplicable(const HloInstruction& instruction) {
     case HloOpcode::kPad:
     case HloOpcode::kReal:
     case HloOpcode::kReducePrecision:
+    case HloOpcode::kReplicaId:
     case HloOpcode::kReshape:
     case HloOpcode::kReverse:
     case HloOpcode::kRoundNearestAfz:
diff --git a/tensorflow/compiler/xla/service/layout_assignment.cc b/tensorflow/compiler/xla/service/layout_assignment.cc
index 5d9e3392fd..9376a3c8f8 100644
--- a/tensorflow/compiler/xla/service/layout_assignment.cc
+++ b/tensorflow/compiler/xla/service/layout_assignment.cc
@@ -2095,6 +2095,7 @@ bool LayoutAssignment::InstructionCanChangeLayout(
     case HloOpcode::kRecv:
     case HloOpcode::kRecvDone:
     case HloOpcode::kReduce:
+    case HloOpcode::kReplicaId:
     case HloOpcode::kReshape:
     case HloOpcode::kRng:
     case HloOpcode::kSend:
-- 
GitLab


From c46d383150564c8b72b05acc65182c16f7221694 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 29 Jan 2019 16:48:30 -0800
Subject: [PATCH 1541/2345] Add a version of inspect.getsource that doesn't
 unwrap decorators. Remove the safety check that verified the function's
 source code matches its signature.

PiperOrigin-RevId: 231499768
---
 .../python/autograph/impl/conversion.py       | 24 ++---
 tensorflow/python/autograph/pyct/ast_util.py  | 45 +--------
 .../python/autograph/pyct/ast_util_test.py    | 93 -------------------
 tensorflow/python/util/tf_inspect.py          | 19 ++++
 tensorflow/python/util/tf_inspect_test.py     | 67 +++++++++++++
 5 files changed, 98 insertions(+), 150 deletions(-)

diff --git a/tensorflow/python/autograph/impl/conversion.py b/tensorflow/python/autograph/impl/conversion.py
index 901cfb2679..6912106165 100644
--- a/tensorflow/python/autograph/impl/conversion.py
+++ b/tensorflow/python/autograph/impl/conversion.py
@@ -344,26 +344,20 @@ def function_to_graph(f,
   logging.log(3, 'Source code of %s:\n%s', f, source)
   node = node.body[0]
 
-  # In general, the output of inspect.getsource is inexact because it uses
-  # regex matching to adjust the exact location around the line number that
-  # CPython records. This is particularly problematic for lambda functions,
-  # where the entire containing lines are returned.
-  nodes = ast_util.find_matching_definitions(node, f)
-  if len(nodes) != 1:
-    if f.__name__ == '<lambda>':
+  # In general, the output of inspect.getsource is inexact for lambdas because
+  # it uses regex matching to adjust the exact location around the line number
+  # that CPython records. Then, the entire containing line is returned, which
+  # we may have trouble disambiguating. For example:
+  # x, y = lambda: 1, lambda: 2
+  if f.__name__ == '<lambda>':
+    nodes = ast_util.find_matching_definitions(node, f)
+    if len(nodes) != 1:
       raise ValueError(
           'Unable to identify source code of lambda function {}. It was'
           ' defined on this line: {}, which must contain a single lambda with'
           ' matching signature. To avoid ambiguity, define each lambda'
           ' in a separate expression.'.format(f, source))
-    else:
-      raise ValueError(
-          'Unable to identify source code of function {}({}). The source code'
-          ' reported by Python did not include exactly one matching signature:'
-          '\n{}\n. This is an extremely rare occurrence. Please report it to'
-          ' the TensorFlow team.'.format(f, tf_inspect.getfullargspec(f),
-                                         source))
-  node, = nodes
+    node, = nodes
 
   # TODO(znado): Place inside standard_analysis.
   origin_info.resolve(node, source, f)
diff --git a/tensorflow/python/autograph/pyct/ast_util.py b/tensorflow/python/autograph/pyct/ast_util.py
index f7723412ab..3496c514ee 100644
--- a/tensorflow/python/autograph/pyct/ast_util.py
+++ b/tensorflow/python/autograph/pyct/ast_util.py
@@ -315,8 +315,8 @@ def parallel_walk(node, other):
                 f, n_child, o_child))
 
 
-class FunctionDefMatcher(gast.NodeVisitor):
-  """Finds nodes that match a given function's signature."""
+class LambdaDefinitionMatcher(gast.NodeVisitor):
+  """Finds lambda nodes that match a given lambda's signature."""
 
   def __init__(self, fn):
     self.fn = fn
@@ -349,26 +349,6 @@ class FunctionDefMatcher(gast.NodeVisitor):
 
     return True
 
-  def _argspec_compatible(self, node):
-    arg_spec = tf_inspect.getfullargspec(self.fn)
-
-    node_args = tuple(self._arg_name(arg) for arg in node.args.args)
-    if len(node_args) != len(arg_spec.args) and node.args.vararg is None:
-      return False
-
-    if arg_spec.varargs is not None and node.args.vararg is None:
-      return False
-
-    if arg_spec.varkw is not None and node.args.kwarg is None:
-      return False
-
-    node_kwonlyargs = tuple(self._arg_name(arg) for arg in node.args.kwonlyargs)
-    if (len(node_kwonlyargs) != len(arg_spec.kwonlyargs) and
-        node.args.kwarg is None):
-      return False
-
-    return True
-
   def visit_Lambda(self, node):
     self.generic_visit(node)
 
@@ -379,27 +359,8 @@ class FunctionDefMatcher(gast.NodeVisitor):
 
     self.matching_nodes.append(node)
 
-  def visit_FunctionDef(self, node):
-    self.generic_visit(node)
-
-    if self.fn.__name__ != node.name:
-      return
-
-    # Decorators have the ability to modify a function's signature. They usually
-    # claim that the result is indistinguishable from the original function,
-    # but it's very difficult to fool this test. As a consequence, we relax the
-    # verification and just check that the arguments are compatible.
-    if node.decorator_list:
-      if not self._argspec_compatible(node):
-        return
-    else:
-      if not self._argspec_matches(node):
-        return
-
-    self.matching_nodes.append(node)
-
 
 def find_matching_definitions(node, f):
-  matcher = FunctionDefMatcher(f)
+  matcher = LambdaDefinitionMatcher(f)
   matcher.visit(node)
   return tuple(matcher.matching_nodes)
diff --git a/tensorflow/python/autograph/pyct/ast_util_test.py b/tensorflow/python/autograph/pyct/ast_util_test.py
index d4f1e1c7cd..00ef7f5e05 100644
--- a/tensorflow/python/autograph/pyct/ast_util_test.py
+++ b/tensorflow/python/autograph/pyct/ast_util_test.py
@@ -230,99 +230,6 @@ class AstUtilTest(test.TestCase):
     nodes = ast_util.find_matching_definitions(node, f)
     self.assertLambdaNodes(nodes, ('(2)',))
 
-  def assertFunctionDefNodes(self, matching_nodes, expected_bodies):
-    self.assertEqual(len(matching_nodes), len(expected_bodies))
-    for node in matching_nodes:
-      self.assertIsInstance(node, gast.FunctionDef)
-      self.assertIn(compiler.ast_to_source(node.body).strip(), expected_bodies)
-
-  def test_find_matching_definitions_function(self):
-    node = parser.parse_str(
-        textwrap.dedent("""
-      def f(x):
-        return 1
-    """))
-
-    def f(x):
-      return x
-
-    nodes = ast_util.find_matching_definitions(node, f)
-    self.assertFunctionDefNodes(nodes, ('return 1',))
-
-  def test_find_matching_definitions_nested_functions_same_name(self):
-    node = parser.parse_str(
-        textwrap.dedent("""
-      def f(x, *args, **kwargs):
-        def f(x, y):
-          return 1
-        return 2
-    """))
-
-    def f(x, y):
-      return x + y
-
-    nodes = ast_util.find_matching_definitions(node, f)
-    self.assertFunctionDefNodes(nodes, ('return 1',))
-
-  def test_find_matching_definitions_nested_functions_same_args(self):
-    node = parser.parse_str(
-        textwrap.dedent("""
-      def g(x):
-        def f(x):
-          return 1
-        return 2
-    """))
-
-    def f(x):
-      return x
-
-    nodes = ast_util.find_matching_definitions(node, f)
-    self.assertFunctionDefNodes(nodes, ('return 1',))
-
-  def test_find_matching_definitions_multiple_matches(self):
-    node = parser.parse_str(
-        textwrap.dedent("""
-      def f(x):
-        return 1
-      def f(x):
-        return 2
-    """))
-
-    def f(x):
-      return x
-
-    nodes = ast_util.find_matching_definitions(node, f)
-    self.assertFunctionDefNodes(nodes, ('return 1', 'return 2'))
-
-  def test_find_matching_definitions_decorated_compatible(self):
-    node = parser.parse_str(
-        textwrap.dedent("""
-      @sneaky_decorator
-      def f(x, *args, **kwargs):
-        return 1
-    """))
-
-    def f(a, b, c, d=1):
-      return a + b + c + d
-
-    nodes = ast_util.find_matching_definitions(node, f)
-    self.assertFunctionDefNodes(nodes, ('return 1',))
-
-  def test_find_matching_definitions_decorated_incompatible(self):
-    node = parser.parse_str(
-        textwrap.dedent("""
-      @sneaky_decorator
-      def f(x, y, z):
-        return 1
-    """))
-
-    def f(a, b, c, d, *args):
-      del args
-      return a + b + c + d
-
-    nodes = ast_util.find_matching_definitions(node, f)
-    self.assertFunctionDefNodes(nodes, ())
-
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/util/tf_inspect.py b/tensorflow/python/util/tf_inspect.py
index 5f1e776640..5171cdfdbc 100644
--- a/tensorflow/python/util/tf_inspect.py
+++ b/tensorflow/python/util/tf_inspect.py
@@ -380,3 +380,22 @@ def isroutine(object):  # pylint: disable=redefined-builtin
 def stack(context=1):
   """TFDecorator-aware replacement for inspect.stack."""
   return _inspect.stack(context)[1:]
+
+
+def getsource_no_unwrap(obj):
+  """Return source code for an object. Does not unwrap TFDecorators.
+
+  The source code is returned literally, including indentation for functions not
+  at the top level. This function is analogous to inspect.getsource, with one
+  key difference - it doesn't unwrap decorators. For simplicity, support for
+  some Python object types is dropped (tracebacks, frames, code objects).
+
+  Args:
+      obj: a class, method, or function object.
+
+  Returns:
+      source code as a string
+
+  """
+  lines, lnum = _inspect.findsource(obj)
+  return ''.join(_inspect.getblock(lines[lnum:]))
diff --git a/tensorflow/python/util/tf_inspect_test.py b/tensorflow/python/util/tf_inspect_test.py
index 02d075cdff..910848e67f 100644
--- a/tensorflow/python/util/tf_inspect_test.py
+++ b/tensorflow/python/util/tf_inspect_test.py
@@ -727,6 +727,73 @@ class TfInspectGetCallArgsTest(test.TestCase):
         'c': 'goodbye'
     }, tf_inspect.getcallargs(decorated, 4, c='goodbye'))
 
+  def testGetSourceNoUnwrapHandlesPlainDecorator(self):
+    def dec(f):
+      def wrapper(*args, **kwargs):
+        return f(*args, **kwargs)
+      return wrapper
+
+    @dec
+    def f():
+      return 1
+
+    source = tf_inspect.getsource_no_unwrap(f)
+    self.assertNotIn('dec', source)
+    self.assertIn('wrapper', source)
+    self.assertNotIn('return 1', source)
+
+  def testGetSourceNoUnwrapHandlesFunctoolsDecorator(self):
+    def dec(f):
+      @functools.wraps(f)
+      def wrapper(*args, **kwargs):
+        return f(*args, **kwargs)
+      return wrapper
+
+    @dec
+    def f():
+      return 1
+
+    source = tf_inspect.getsource_no_unwrap(f)
+    self.assertNotIn('dec', source)
+    self.assertIn('wrapper', source)
+    self.assertNotIn('return 1', source)
+
+  def testGetSourceNoUnwrapHandlesPlainDecoratorFactory(self):
+    def dec_factory():
+      def dec(f):
+        def wrapper(*args, **kwargs):
+          return f(*args, **kwargs)
+        return wrapper
+      return dec
+
+    @dec_factory()
+    def f():
+      return 1
+
+    source = tf_inspect.getsource_no_unwrap(f)
+    self.assertNotIn('factory', source)
+    self.assertNotIn('dec', source)
+    self.assertIn('wrapper', source)
+    self.assertNotIn('return 1', source)
+
+  def testGetSourceNoUnwrapHandlesFunctoolsDecoratorFactory(self):
+    def dec_factory():
+      def dec(f):
+        @functools.wraps(f)
+        def wrapper(*args, **kwargs):
+          return f(*args, **kwargs)
+        return wrapper
+      return dec
+
+    @dec_factory()
+    def f():
+      return 1
+
+    source = tf_inspect.getsource_no_unwrap(f)
+    self.assertNotIn('factory', source)
+    self.assertNotIn('dec', source)
+    self.assertIn('wrapper', source)
+    self.assertNotIn('return 1', source)
 
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 7b48d11a5a45ca5d19a994f5a60a53f47339f57a Mon Sep 17 00:00:00 2001
From: Christopher Suter <cgs@google.com>
Date: Tue, 29 Jan 2019 16:52:20 -0800
Subject: [PATCH 1542/2345] Preserve stack trace when conditionally re-raising.

The error handling code in question looks to see if a raised exception is of type TypeError, and if so, if its message matches a pattern, in which case it raises a more informative message about dynamic control flow. However, If it doesn't match the message, it seeks just to reraise the underlying exception. When doing this with "raise e", the underlying stack trace is lost; instead, we should just use "raise", which keeps the source stack trace.

Source for the re-raise behavior: https://stackoverflow.com/questions/8146386/python-conditionally-catching-exceptions

PiperOrigin-RevId: 231500445
---
 tensorflow/python/keras/engine/base_layer.py  | 11 ++++----
 .../python/keras/engine/base_layer_test.py    | 27 +++++++++++++++++++
 2 files changed, 33 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index 688fdd0101..12d3f09bfe 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -561,16 +561,17 @@ class Layer(checkpointable.Checkpointable):
             try:
               outputs = self.call(inputs, *args, **kwargs)
             except TypeError as e:
-              messages = ['`tf.Tensor` as a Python `bool` is not allowed',
-                          'Tensor objects are only iterable when eager']
+              messages = ('`tf.Tensor` as a Python `bool` is not allowed',
+                          'Tensor objects are only iterable when eager')
+              exception_str = str(e)
               for msg in messages:
-                if msg in str(e):
+                if msg in exception_str:
                   raise TypeError('You are attempting to use Python control '
                                   'flow in a layer that was not declared to be '
                                   'dynamic. Pass `dynamic=True` to the class '
                                   'constructor.\nEncountered error:\n"""\n' +
-                                  str(e) + '\n"""')
-              raise e
+                                  exception_str + '\n"""')
+              raise
           else:
             # We will use static shape inference to return symbolic tensors
             # matching the specifications of the layer outputs.
diff --git a/tensorflow/python/keras/engine/base_layer_test.py b/tensorflow/python/keras/engine/base_layer_test.py
index 93ca10ee4e..faeb4c5f40 100644
--- a/tensorflow/python/keras/engine/base_layer_test.py
+++ b/tensorflow/python/keras/engine/base_layer_test.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import sys
+import traceback
 from absl.testing import parameterized
 import numpy as np
 
@@ -320,6 +322,31 @@ class BaseLayerTest(keras_parameterized.TestCase):
                         np.matmul(x_val, y_val),
                         atol=1e-5)
 
+  @test_util.run_in_graph_and_eager_modes
+  def test_reraising_exception(self):
+    # When layer is not dynamic, we have some pattern matching during exception
+    # handling to detect when the user is trying to use python control flow.
+    # When an exception is thrown but the pattern doesn't match, we want to
+    # preserve the originating stack trace. An early implementation of this
+    # logic lost the stack trace. We test the correct behavior here.
+
+    class TypeErrorLayer(base_layer.Layer):
+
+      def call(self, inputs):
+        def easily_identifiable_name():
+          raise TypeError('Non-matching TypeError message.')
+        easily_identifiable_name()
+
+    inputs = keras.Input((3,))
+
+    try:
+      _ = TypeErrorLayer()(inputs)
+    except TypeError:
+      tb = traceback.extract_tb(sys.exc_info()[2])
+      last_entry = tb[-1]
+      function_name = last_entry[2]
+      self.assertEqual(function_name, 'easily_identifiable_name')
+
 
 @test_util.run_all_in_graph_and_eager_modes
 class NestedTrackingTest(test.TestCase):
-- 
GitLab


From 355b1702cca20f5eecb14a9a49227afafb037c36 Mon Sep 17 00:00:00 2001
From: Sachin Joglekar <srjoglekar@google.com>
Date: Tue, 29 Jan 2019 16:53:47 -0800
Subject: [PATCH 1543/2345] Implements quantized version of 4D optimized Mean

PiperOrigin-RevId: 231500668
---
 .../internal/reference/reference_ops.h        | 59 +++++++++++++++++++
 tensorflow/lite/kernels/reduce.cc             | 29 +++++++++
 tensorflow/lite/kernels/reduce_test.cc        | 43 ++++++++++++++
 3 files changed, 131 insertions(+)

diff --git a/tensorflow/lite/kernels/internal/reference/reference_ops.h b/tensorflow/lite/kernels/internal/reference/reference_ops.h
index 2eb294f734..ac6905fcd7 100644
--- a/tensorflow/lite/kernels/internal/reference/reference_ops.h
+++ b/tensorflow/lite/kernels/internal/reference/reference_ops.h
@@ -3845,6 +3845,65 @@ inline void Mean(const tflite::MeanParams& op_params,
   }
 }
 
+inline void Mean(const tflite::MeanParams& op_params,
+                 const RuntimeShape& unextended_input_shape,
+                 const uint8_t* input_data, int32 input_zero_point,
+                 float input_scale, const RuntimeShape& unextended_output_shape,
+                 uint8_t* output_data, int32 output_zero_point,
+                 float output_scale) {
+  gemmlowp::ScopedProfilingLabel label("Mean4D/Uint8");
+
+  // Current implementation only supports dimension equals 4 and simultaneous
+  // reduction over width and height.
+  TFLITE_CHECK_EQ(unextended_input_shape.DimensionsCount(), 4);
+  TFLITE_CHECK_LE(unextended_output_shape.DimensionsCount(), 4);
+  const RuntimeShape input_shape =
+      RuntimeShape::ExtendedShape(4, unextended_input_shape);
+  const RuntimeShape output_shape =
+      RuntimeShape::ExtendedShape(4, unextended_output_shape);
+  const int output_batch = output_shape.Dims(0);
+  const int output_height = output_shape.Dims(1);
+  const int output_width = output_shape.Dims(2);
+  const int output_depth = output_shape.Dims(3);
+  const int input_height = input_shape.Dims(1);
+  const int input_width = input_shape.Dims(2);
+  const float num_elements_in_axis = input_width * input_height;
+
+  TFLITE_DCHECK_EQ(op_params.axis_count, 2);
+  TFLITE_DCHECK((op_params.axis[0] == 1 && op_params.axis[1] == 2) ||
+                (op_params.axis[0] == 2 && op_params.axis[1] == 1));
+  TFLITE_DCHECK_EQ(output_height, 1);
+  TFLITE_DCHECK_EQ(output_width, 1);
+
+  const bool ordinary_mean =
+      (input_zero_point == output_zero_point && input_scale == output_scale);
+  float scale, bias;
+  if (!ordinary_mean) {
+    scale = input_scale / output_scale;
+    bias = -input_zero_point * scale + 0.5;
+  }
+  for (int out_b = 0; out_b < output_batch; ++out_b) {
+    for (int out_d = 0; out_d < output_depth; ++out_d) {
+      float temp_value = 0;
+      for (int in_h = 0; in_h < input_height; ++in_h) {
+        for (int in_w = 0; in_w < input_width; ++in_w) {
+          temp_value +=
+              input_data[Offset(input_shape, out_b, in_h, in_w, out_d)];
+        }
+      }
+      temp_value = temp_value / num_elements_in_axis;
+      if (ordinary_mean) {
+        output_data[Offset(output_shape, out_b, 0, 0, out_d)] =
+            static_cast<uint8_t>(round(temp_value));
+      } else {
+        output_data[Offset(output_shape, out_b, 0, 0, out_d)] =
+            static_cast<uint8_t>(round(temp_value * scale + bias)) +
+            output_zero_point;
+      }
+    }
+  }
+}
+
 // Computes the mean of elements across dimensions given in axis.
 // It does so in two stages, first calculates the sum of elements along the axis
 // then divides it by the number of element in axis for quantized values.
diff --git a/tensorflow/lite/kernels/reduce.cc b/tensorflow/lite/kernels/reduce.cc
index 336e827ca4..1193f84f6f 100644
--- a/tensorflow/lite/kernels/reduce.cc
+++ b/tensorflow/lite/kernels/reduce.cc
@@ -257,6 +257,35 @@ TfLiteStatus EvalMean(TfLiteContext* context, TfLiteNode* node) {
     TF_LITE_ENSURE_OK(context, ResizeTempSum(context, &op_context, temp_sum));
   }
 
+  // Defer to specialized implementation for 4D Mean across axes 1 & 2.
+  if (op_context.input->type == kTfLiteFloat32 ||
+      op_context.input->type == kTfLiteUInt8) {
+    tflite::MeanParams op_params;
+    op_params.axis_count = num_axis;
+    ResolveAxis(GetTensorData<int>(op_context.axis), num_axis, &op_params);
+    const TfLiteTensor* input = op_context.input;
+    if (op_context.params->keep_dims && NumDimensions(input) == 4 &&
+        op_params.axis_count == 2 &&
+        ((op_params.axis[0] == 1 && op_params.axis[1] == 2) ||
+         (op_params.axis[0] == 2 && op_params.axis[1] == 1))) {
+      if (op_context.input->type == kTfLiteUInt8) {
+        reference_ops::Mean(
+            op_params, GetTensorShape(input), GetTensorData<uint8_t>(input),
+            op_context.input->params.zero_point, op_context.input->params.scale,
+            GetTensorShape(op_context.output),
+            GetTensorData<uint8_t>(op_context.output),
+            op_context.output->params.zero_point,
+            op_context.output->params.scale);
+      } else {
+        reference_ops::Mean(op_params, GetTensorShape(input),
+                            GetTensorData<float>(input),
+                            GetTensorShape(op_context.output),
+                            GetTensorData<float>(op_context.output));
+      }
+      return kTfLiteOk;
+    }
+  }
+
 #define TF_LITE_MEAN(kernel_type, data_type, temp_data_type)        \
   kernel_type::Mean<>(                                              \
       GetTensorData<data_type>(op_context.input),                   \
diff --git a/tensorflow/lite/kernels/reduce_test.cc b/tensorflow/lite/kernels/reduce_test.cc
index c1526bddb7..16c59396dd 100644
--- a/tensorflow/lite/kernels/reduce_test.cc
+++ b/tensorflow/lite/kernels/reduce_test.cc
@@ -256,6 +256,49 @@ TEST(ConstFloatMeanOpTest, KeepDims) {
               ElementsAreArray(ArrayFloatNear({10.5, 12.5, 14.5})));
 }
 
+// Uses a set of reduction conditions that trigger the specialized 4D version
+// of Mean.
+TEST(ConstFloatMeanOpTest, KeepDims_4DMean) {
+  std::vector<float> data = {1.0,  2.0,  3.0,  4.0,  5.0,  6.0,  7.0,  8.0,
+                             9.0,  10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0,
+                             17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0};
+  MeanOpConstModel m({TensorType_FLOAT32, {2, 2, 3, 2}},
+                     {TensorType_FLOAT32, {3}}, {2}, {1, 2}, true);
+  m.SetInput(data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({2, 1, 1, 2}));
+  EXPECT_THAT(m.GetOutput<float>(),
+              ElementsAreArray(ArrayFloatNear({6, 7, 18, 19})));
+}
+
+TEST(ConstFloatMeanOpTest, KeepDims_4DMean_UInt8) {
+  float kQuantizedTolerance = GetTolerance(-1.0, 1.0);
+  std::vector<float> data = {0.1, 0.2, 0.3, 0.4, 0.1, 0.2,
+                             0.3, 0.4, 0.1, 0.2, 0.3, 0.4};
+  MeanOpConstModel m({TensorType_UINT8, {1, 2, 2, 3}, -1.0, 1.0},
+                     {TensorType_UINT8, {2}, -1.0, 1.0}, {2}, {1, 2}, true);
+  m.QuantizeAndPopulate<uint8_t>(m.Input(), data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 1, 1, 3}));
+  EXPECT_THAT(m.GetDequantizedOutput(),
+              ElementsAreArray(ArrayFloatNear({0.25098, 0.25098, 0.25098},
+                                              kQuantizedTolerance)));
+}
+
+TEST(ConstFloatMeanOpTest, KeepDims_4DMean_Quantized) {
+  float kQuantizedTolerance = GetTolerance(-5.0, 5.0);
+  std::vector<float> data = {0.1, 0.2, 0.3, 0.4, 0.1, 0.2,
+                             0.3, 0.4, 0.1, 0.2, 0.3, 0.4};
+  MeanOpConstModel m({TensorType_UINT8, {1, 2, 3, 2}, 0.0, 1.0},
+                     {TensorType_UINT8, {3}, -5.0, 5.0}, {2}, {1, 2}, true);
+  m.QuantizeAndPopulate<uint8_t>(m.Input(), data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 1, 1, 2}));
+  EXPECT_THAT(m.GetDequantizedOutput(),
+              ElementsAreArray(
+                  ArrayFloatNear({0.235294, 0.313726}, kQuantizedTolerance)));
+}
+
 TEST(ConstFloatMeanOpTest, Scalar) {
   std::vector<float> data = {3.27};
   MeanOpConstModel m({TensorType_FLOAT32, {}}, {TensorType_FLOAT32, {}}, {},
-- 
GitLab


From fca8f722cb3f92eb36aee009c5016dd679ac0521 Mon Sep 17 00:00:00 2001
From: Tong Shen <endlessroad@google.com>
Date: Tue, 29 Jan 2019 17:00:35 -0800
Subject: [PATCH 1544/2345] Fix misplaced arguments in error message.

PiperOrigin-RevId: 231501652
---
 tensorflow/python/kernel_tests/control_flow_ops_py_test.py | 6 +++---
 tensorflow/python/ops/control_flow_util.py                 | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 3f0b1b3bbc..3ac53ba9c9 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -3912,14 +3912,14 @@ class ControlFlowContextCheckTest(test.TestCase):
     while_tensor = self._getWhileTensor()
     with self.assertRaisesRegexp(
         ValueError,
-        "Cannot use 'while_1/Add' as input to 'while/Const_1' because they are "
+        "Cannot use 'while/Const_1' as input to 'while_1/Add' because they are "
         "in different while loops. See info log for more details."):
       control_flow_ops.while_loop(lambda i: i < 10,
                                   lambda x: math_ops.add(1, while_tensor), [0])
 
     with self.assertRaisesRegexp(
         ValueError,
-        "Cannot use 'while_2/NextIteration' as input to 'while/Const_1' "
+        "Cannot use 'while/Const_1' as input to 'while_2/NextIteration' "
         "because they are in different while loops. See info log for more "
         "details."):
       control_flow_ops.while_loop(lambda i: i < 10, lambda i: while_tensor, [0])
@@ -3976,7 +3976,7 @@ class ControlFlowContextCheckTest(test.TestCase):
 
     with self.assertRaisesRegexp(
         ValueError,
-        "Cannot use 'cond/while_1/add' as input to 'cond/while/Const_1' because"
+        "Cannot use 'cond/while/Const_1' as input to 'cond/while_1/add' because"
         " they are in different while loops. See info log for more details."):
       control_flow_ops.cond(
           math_ops.less(1, 2), true_fn, lambda: constant_op.constant(0))
diff --git a/tensorflow/python/ops/control_flow_util.py b/tensorflow/python/ops/control_flow_util.py
index ff0dff0042..e6fdbe34ec 100644
--- a/tensorflow/python/ops/control_flow_util.py
+++ b/tensorflow/python/ops/control_flow_util.py
@@ -331,7 +331,7 @@ def CheckInputFromValidContext(op, input_op):
     if while_ctxt:
       error_msg = (
           "Cannot use '%s' as input to '%s' because they are in different while"
-          " loops." % (op.name, input_op.name))
+          " loops." % (input_op.name, op.name))
     else:
       error_msg = (
           "Cannot use '%s' as input to '%s' because '%s' is in a while loop."
-- 
GitLab


From 91ebeecc928a9362f5e63595e4191f57d646832e Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Tue, 29 Jan 2019 17:07:38 -0800
Subject: [PATCH 1545/2345] Proper string format on ValueError when MPI library
 is not found.

PiperOrigin-RevId: 231503050
---
 configure.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.py b/configure.py
index adc9ef9cac..8dcd318220 100644
--- a/configure.py
+++ b/configure.py
@@ -1482,7 +1482,7 @@ def set_other_mpi_vars(environ_cp):
   else:
     raise ValueError(
         'Cannot find the MPI library file in %s/lib or %s/lib64 or %s/lib32' %
-        mpi_home, mpi_home, mpi_home)
+        (mpi_home, mpi_home, mpi_home))
 
 
 def set_system_libs_flag(environ_cp):
-- 
GitLab


From cfb819c9cc212467f57e244ee8b4cdb0fd4f937a Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Tue, 29 Jan 2019 17:13:33 -0800
Subject: [PATCH 1546/2345] [XLA] Add xla::IsNan, IsPosInf, IsNegInf,
 IsPosOrNegInf.

Useful numeric helper functions.

PiperOrigin-RevId: 231503933
---
 .../compiler/tf2xla/kernels/unary_ops.cc      |  7 +-
 tensorflow/compiler/xla/client/lib/BUILD      |  2 +
 tensorflow/compiler/xla/client/lib/math.cc    | 91 ++++++++++++-------
 tensorflow/compiler/xla/client/lib/math.h     | 16 +++-
 .../compiler/xla/client/lib/math_test.cc      | 80 +++++++++++++++-
 tensorflow/compiler/xla/client/xla_builder.h  | 12 ++-
 6 files changed, 161 insertions(+), 47 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc
index a0ea6422d7..a7cc8c102a 100644
--- a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc
@@ -65,11 +65,8 @@ XLAJIT_MAKE_UNARY(Exp, xla::Exp(x));
 XLAJIT_MAKE_UNARY(Expm1, xla::Expm1(x));
 XLAJIT_MAKE_UNARY(Floor, xla::Floor(x));
 XLAJIT_MAKE_UNARY(IsFinite, xla::IsFinite(x));
-XLAJIT_MAKE_UNARY(
-    IsInf,
-    xla::Eq(xla::Abs(x),
-            xla::ScalarLike(x, std::numeric_limits<double>::infinity())));
-XLAJIT_MAKE_UNARY(IsNan, xla::Ne(x, x));
+XLAJIT_MAKE_UNARY(IsInf, xla::IsInf(x));
+XLAJIT_MAKE_UNARY(IsNan, xla::IsNan(x));
 // Return 1/x
 XLAJIT_MAKE_UNARY(Inv, xla::ScalarLike(x, 1.0) / x);
 XLAJIT_MAKE_UNARY(Reciprocal, xla::ScalarLike(x, 1.0) / x);
diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD
index 26c5e8eb73..b30ab84240 100644
--- a/tensorflow/compiler/xla/client/lib/BUILD
+++ b/tensorflow/compiler/xla/client/lib/BUILD
@@ -184,6 +184,7 @@ cc_library(
     srcs = ["math.cc"],
     hdrs = ["math.h"],
     deps = [
+        ":arithmetic",
         ":constants",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:status_macros",
@@ -197,6 +198,7 @@ xla_test(
     deps = [
         ":math",
         "//tensorflow/compiler/xla:literal_util",
+        "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:test",
         "//tensorflow/compiler/xla:types",
         "//tensorflow/compiler/xla:xla_data_proto",
diff --git a/tensorflow/compiler/xla/client/lib/math.cc b/tensorflow/compiler/xla/client/lib/math.cc
index 9f03e83c16..675884050e 100644
--- a/tensorflow/compiler/xla/client/lib/math.cc
+++ b/tensorflow/compiler/xla/client/lib/math.cc
@@ -19,6 +19,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/client/lib/math.h"
 
+#include "tensorflow/compiler/xla/client/lib/arithmetic.h"
 #include "tensorflow/compiler/xla/client/lib/constants.h"
 #include "tensorflow/compiler/xla/primitive_util.h"
 #include "tensorflow/compiler/xla/shape_util.h"
@@ -26,6 +27,58 @@ limitations under the License.
 
 namespace xla {
 
+// TODO(jlebar): Use this function in more places in this file to restrict the
+// domain of other functions.
+static Status EnsureOperandIsRealFp(absl::string_view op_name, XlaOp operand) {
+  auto& b = *operand.builder();
+  TF_ASSIGN_OR_RETURN(auto shape, b.GetShape(operand));
+  auto elem_ty = shape.element_type();
+  if (!primitive_util::IsFloatingPointType(elem_ty)) {
+    return InvalidArgument(
+        "Operands to %s must be real-valued floating-point, but got %s",
+        op_name, PrimitiveType_Name(elem_ty));
+  }
+  return Status::OK();
+}
+
+XlaOp IsPosInf(XlaOp operand) {
+  auto& b = *operand.builder();
+  return b.ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    TF_RETURN_IF_ERROR(EnsureOperandIsRealFp("IsPosInf", operand));
+    TF_ASSIGN_OR_RETURN(auto shape, b.GetShape(operand));
+    // Note that this is only correct for floating-point types.  If we wanted it
+    // to be correct for all types, we'd need to Gt(MaxFiniteValue).
+    return Eq(operand, MaxValue(&b, shape.element_type()));
+  });
+}
+
+XlaOp IsNegInf(XlaOp operand) {
+  auto& b = *operand.builder();
+  return b.ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    TF_RETURN_IF_ERROR(EnsureOperandIsRealFp("IsNegInf", operand));
+    TF_ASSIGN_OR_RETURN(auto shape, b.GetShape(operand));
+    // Note that this is only correct for floating-point types.  If we wanted it
+    // to be correct for all types, we'd need to Lt(MinFiniteValue).
+    return Eq(operand, MinValue(&b, shape.element_type()));
+  });
+}
+
+XlaOp IsInf(XlaOp operand) {
+  auto& b = *operand.builder();
+  return b.ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    TF_RETURN_IF_ERROR(EnsureOperandIsRealFp("IsInf", operand));
+    return IsPosInf(Abs(operand));
+  });
+}
+
+XlaOp IsNan(XlaOp operand) {
+  auto& b = *operand.builder();
+  return b.ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    TF_RETURN_IF_ERROR(EnsureOperandIsRealFp("IsNan", operand));
+    return Ne(operand, operand);
+  });
+}
+
 XlaOp Sqrt(XlaOp operand) { return Pow(operand, ScalarLike(operand, 0.5)); }
 
 XlaOp Rsqrt(XlaOp operand) { return Pow(operand, ScalarLike(operand, -0.5)); }
@@ -101,14 +154,8 @@ XlaOp EvaluatePolynomial(XlaOp x, absl::Span<const float> coefficients) {
 XlaOp Erfc(XlaOp x) {
   auto& b = *x.builder();
   return b.ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
-    // Reject non-real non-fp inputs.  (We could extend erfc to accept complex
-    // types, but it doesn't seem necessary at this point.)
-    TF_ASSIGN_OR_RETURN(auto shape, b.GetShape(x));
-    if (!ShapeUtil::ElementIsFloating(shape)) {
-      return InvalidArgument(
-          "erfc only accepts real floating-point arrays or scalars, but got %s",
-          shape.ToString());
-    }
+    TF_RETURN_IF_ERROR(EnsureOperandIsRealFp("Erfc", x));
+
     XlaOp abs_x = Abs(x);
     XlaOp z = Exp(-x * x);
 
@@ -223,15 +270,7 @@ static constexpr std::array<double, 8> kLanczosCoefficients = {
 XlaOp Lgamma(XlaOp input) {
   auto& b = *input.builder();
   return b.ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
-    // Reject non-real non-fp inputs.  (We could extend lgamma to accept complex
-    // types, but it doesn't seem necessary at this point.)
-    TF_ASSIGN_OR_RETURN(auto shape, b.GetShape(input));
-    if (!ShapeUtil::ElementIsFloating(shape)) {
-      return InvalidArgument(
-          "lgamma only accepts real floating-point arrays or scalars, but got "
-          "%s",
-          shape.ToString());
-    }
+    TF_RETURN_IF_ERROR(EnsureOperandIsRealFp("Lgamma", input));
 
     XlaOp one_half = ScalarLike(input, 0.5);
     XlaOp one = ScalarLike(input, 1);
@@ -321,15 +360,7 @@ XlaOp Lgamma(XlaOp input) {
 XlaOp Digamma(XlaOp input) {
   auto& b = *input.builder();
   return b.ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
-    // Reject non-real non-fp inputs.  (We could extend digamma to accept
-    // complex types, but it doesn't seem necessary at this point.)
-    TF_ASSIGN_OR_RETURN(auto shape, b.GetShape(input));
-    if (!ShapeUtil::ElementIsFloating(shape)) {
-      return InvalidArgument(
-          "digamma only accepts real floating-point arrays or scalars, but got "
-          "%s",
-          shape.ToString());
-    }
+    TF_RETURN_IF_ERROR(EnsureOperandIsRealFp("Digamma", input));
 
     XlaOp zero = ScalarLike(input, 0);
     XlaOp one_half = ScalarLike(input, 0.5);
@@ -381,12 +412,8 @@ XlaOp RoundToEven(XlaOp x) {
     // Reject non-real non-fp inputs (What does it even mean to round a complex
     // number?  Do you round each component equally?  In that case, you should
     // just ask for that explicitly.)
-    TF_ASSIGN_OR_RETURN(auto shape, b.GetShape(x));
-    if (ShapeUtil::ElementIsComplex(shape)) {
-      return InvalidArgument(
-          "RoundToEven doesn't accept complex inputs, but got %s",
-          shape.ToString());
-    }
+    TF_RETURN_IF_ERROR(EnsureOperandIsRealFp("RoundToEven", x));
+
     auto half = ScalarLike(x, 0.5);
     auto one = ScalarLike(x, 1.0);
     auto two = ScalarLike(x, 2.0);
diff --git a/tensorflow/compiler/xla/client/lib/math.h b/tensorflow/compiler/xla/client/lib/math.h
index 583481c7f3..907571c9a3 100644
--- a/tensorflow/compiler/xla/client/lib/math.h
+++ b/tensorflow/compiler/xla/client/lib/math.h
@@ -20,6 +20,18 @@ limitations under the License.
 
 namespace xla {
 
+// Determines whether operand is +/-inf or nan.
+//
+// Raises an error if called on integral or complex values.
+XlaOp IsPosInf(XlaOp operand);
+XlaOp IsNegInf(XlaOp operand);
+XlaOp IsInf(XlaOp operand);
+XlaOp IsNan(XlaOp operand);
+
+// Returns the next number after 'from' in the direction of 'to' the same way
+// std::nextafter(from, to) would.
+XlaOp NextAfter(XlaOp from, XlaOp to);
+
 // Computes the square root of 'operand'.
 XlaOp Sqrt(XlaOp operand);
 
@@ -90,10 +102,6 @@ XlaOp Sinh(XlaOp x);
 // is true, otherwise returns its argument.
 xla::XlaOp MaybeConjugate(xla::XlaOp x, bool conjugate);
 
-// Returns the next number after 'from' in the direction of 'to' the same way
-// std::nextafter(from, to) would.
-XlaOp NextAfter(XlaOp from, XlaOp to);
-
 }  // namespace xla
 
 #endif  // TENSORFLOW_COMPILER_XLA_CLIENT_LIB_MATH_H_
diff --git a/tensorflow/compiler/xla/client/lib/math_test.cc b/tensorflow/compiler/xla/client/lib/math_test.cc
index c2e1251fc2..364ac5876a 100644
--- a/tensorflow/compiler/xla/client/lib/math_test.cc
+++ b/tensorflow/compiler/xla/client/lib/math_test.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/lib/math.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/compiler/xla/primitive_util.h"
 #include "tensorflow/compiler/xla/test.h"
 #include "tensorflow/compiler/xla/tests/client_library_test_base.h"
 #include "tensorflow/compiler/xla/tests/test_macros.h"
@@ -55,6 +56,38 @@ class MathTypedTest : public MathTest {
         &b, {T{0.0}, T{-0.0}, -std::numeric_limits<T>::infinity()}, {},
         error_spec_);
   }
+
+  void TestIsInfOrNan() {
+    SetFastMathDisabled(true);
+
+    XlaBuilder b(TestName());
+    auto x =
+        ConstantR1<T>(&b, {
+                              T{0},
+                              T{100},
+                              T{-1000},
+                              T{std::numeric_limits<T>::max()},
+                              T{std::numeric_limits<T>::lowest()},
+                              T{std::numeric_limits<float>::infinity()},
+                              T{-std::numeric_limits<float>::infinity()},
+                              T{std::numeric_limits<float>::quiet_NaN()},
+                              T{std::numeric_limits<float>::signaling_NaN()},
+                          });
+    Tuple(&b, {IsFinite(x), IsInf(x), IsPosInf(x), IsNegInf(x), IsNan(x)});
+
+    auto expected = LiteralUtil::MakeTupleOwned(
+        LiteralUtil::CreateR1<bool>(
+            {true, true, true, true, true, false, false, false, false}),
+        LiteralUtil::CreateR1<bool>(
+            {false, false, false, false, false, true, true, false, false}),
+        LiteralUtil::CreateR1<bool>(
+            {false, false, false, false, false, true, false, false, false}),
+        LiteralUtil::CreateR1<bool>(
+            {false, false, false, false, false, false, true, false, false}),
+        LiteralUtil::CreateR1<bool>(
+            {false, false, false, false, false, false, false, true, true}));
+    ComputeAndCompareLiteral(&b, expected, {});
+  }
 };
 
 // TODO(b/123355973): Add bfloat16 to TestTypes once it's working.
@@ -68,6 +101,50 @@ TYPED_TEST_CASE(MathTypedTest, TestTypes);
 
 XLA_TYPED_TEST(MathTypedTest, LogEdgeCases) { this->TestLogEdgeCases(); }
 XLA_TYPED_TEST(MathTypedTest, Log1pEdgeCases) { this->TestLog1pEdgeCases(); }
+XLA_TYPED_TEST(MathTypedTest, IsInfOrNan) { this->TestIsInfOrNan(); }
+
+// Check that certain ops only support real, floating-point inputs.
+//
+// TODO(jlebar): Expand this test to cover more ops.
+XLA_TEST_F(MathTest, RealFpOnlyOps) {
+  for (int64 i = PrimitiveType_MIN; i <= PrimitiveType_MAX; ++i) {
+    auto ty = static_cast<PrimitiveType>(i);
+    SCOPED_TRACE(PrimitiveType_Name(ty));
+    Shape shape;
+    if (primitive_util::IsArrayType(ty)) {
+      shape = ShapeUtil::MakeShape(ty, {42});
+    } else if (ty == PrimitiveType::TUPLE) {
+      shape = ShapeUtil::MakeTupleShape({});
+    } else if (ty == PrimitiveType::OPAQUE) {
+      shape = ShapeUtil::MakeOpaqueShape();
+    } else if (ty == PrimitiveType::TOKEN) {
+      shape = ShapeUtil::MakeTokenShape();
+    } else {
+      continue;
+    }
+
+    for (const auto& test :
+         std::vector<std::pair<std::function<XlaOp(XlaOp)>, string>>({
+             {IsFinite, "is_finite"},
+             {IsInf, "is_inf"},
+             {IsPosInf, "is_pos_inf"},
+             {IsNegInf, "is_neg_inf"},
+             {IsNan, "is_nan"},
+             {Erf, "erf"},
+             {Erfc, "erfc"},
+             {Lgamma, "lgamma"},
+             {Digamma, "digamma"},
+             {RoundToEven, "round_to_even"},
+         })) {
+      SCOPED_TRACE(test.second);
+      XlaBuilder b(TestName());
+      XlaOp p = Parameter(&b, 0, shape, "p0");
+      test.first(p);
+
+      EXPECT_EQ(b.first_error().ok(), primitive_util::IsFloatingPointType(ty));
+    }
+  }
+}
 
 XLA_TEST_F(MathTest, SqrtF32) {
   XlaBuilder builder(TestName());
@@ -145,8 +222,7 @@ XLA_TEST_F(MathTest, Lgamma) {
   ComputeAndCompareR1<float>(&builder, expected, {}, error_spec_);
 }
 
-// TODO(jlebar): Fails on interpreter due to unimplemented operation.
-XLA_TEST_F(MathTest, DISABLED_ON_INTERPRETER(LgammaF16)) {
+XLA_TEST_F(MathTest, LgammaF16) {
   SetFastMathDisabled(true);
 
   XlaBuilder b(TestName());
diff --git a/tensorflow/compiler/xla/client/xla_builder.h b/tensorflow/compiler/xla/client/xla_builder.h
index 4314747fff..c429035ad0 100644
--- a/tensorflow/compiler/xla/client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_builder.h
@@ -1776,10 +1776,14 @@ XlaOp Imag(const XlaOp& operand);
 XlaOp Pow(const XlaOp& lhs, const XlaOp& rhs,
           absl::Span<const int64> broadcast_dimensions = {});
 
-// Enqueues an operator that tests if the operand's values are finite, i.e.,
-// not Inf or NaN. Defined only for floating-point types. Returns an array of
-// booleans with the same shape where entries are true iff the corresponding
-// entry was NaN.
+// Enqueues an operator that tests if the operand's values are finite, i.e., not
+// +/-Inf or NaN.  Returns an array of booleans with the same shape where
+// entries are true iff the corresponding entry was not infinite or NaN.
+//
+// Defined only for real-valued (i.e. not complex) floating-point types; raises
+// an error for other types.
+//
+// See also IsInf, IsPosInf, IsNegInf, and IsNan in lib/math.h.
 XlaOp IsFinite(const XlaOp& operand);
 
 // Enqueues an iota operation onto the computation.
-- 
GitLab


From 91c36e137f0185b936174f078f8c93549c0ecbcc Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Tue, 29 Jan 2019 17:15:48 -0800
Subject: [PATCH 1547/2345] Add temporary patch to third_party/eigen3 to fix
 builds of Arm/NEON code with older versions of GCC.

PiperOrigin-RevId: 231504293
---
 tensorflow/lite/tools/make/download_dependencies.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/lite/tools/make/download_dependencies.sh b/tensorflow/lite/tools/make/download_dependencies.sh
index fa3d5d3d3b..8c4992a843 100755
--- a/tensorflow/lite/tools/make/download_dependencies.sh
+++ b/tensorflow/lite/tools/make/download_dependencies.sh
@@ -100,5 +100,6 @@ replace_by_sed 's#static uint32x2_t p2ui_CONJ_XOR = vld1_u32( conj_XOR_DATA );#s
   "${DOWNLOADS_DIR}/eigen/Eigen/src/Core/arch/NEON/Complex.h"
 replace_by_sed 's#static uint64x2_t p2ul_CONJ_XOR = vld1q_u64( p2ul_conj_XOR_DATA );#static uint64x2_t p2ul_CONJ_XOR;// = vld1q_u64( p2ul_conj_XOR_DATA ); - Removed by script#' \
   "${DOWNLOADS_DIR}/eigen/Eigen/src/Core/arch/NEON/Complex.h"
+cat "$SCRIPT_DIR/../../../../third_party/eigen3/gebp_neon.patch" | patch "${DOWNLOADS_DIR}/eigen/Eigen/src/Core/products/GeneralBlockPanelKernel.h"
 
 echo "download_dependencies.sh completed successfully." >&2
-- 
GitLab


From 8148037cdeb3436ee9d6c3dce99eee1e75865ade Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Tue, 29 Jan 2019 17:17:08 -0800
Subject: [PATCH 1548/2345] Fix the namespace for kCudaPlatformId, so
 cuda_platform.h is consistent with cuda_platform_id.h

PiperOrigin-RevId: 231504507
---
 tensorflow/stream_executor/cuda/cuda_platform.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/cuda_platform.h b/tensorflow/stream_executor/cuda/cuda_platform.h
index 1ab09d690f..b21e9797be 100644
--- a/tensorflow/stream_executor/cuda/cuda_platform.h
+++ b/tensorflow/stream_executor/cuda/cuda_platform.h
@@ -32,13 +32,14 @@ limitations under the License.
 #include "tensorflow/stream_executor/trace_listener.h"
 
 namespace stream_executor {
-namespace gpu {
-
+namespace cuda {
 // Opaque and unique identifier for the CUDA platform plugin.
 // This is needed so that plugins can refer to/identify this platform without
 // instantiating a CudaPlatform object.
 extern const Platform::Id kCudaPlatformId;
+}  // namespace cuda
 
+namespace gpu {
 // Cuda-specific platform plugin, registered as a singleton value via module
 // initializer.
 class CudaPlatform : public Platform {
-- 
GitLab


From 26b9784074da0858b5ee836d452253400898a23c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 29 Jan 2019 17:18:02 -0800
Subject: [PATCH 1549/2345] Return -1 for ASharedMemory_create if the function
 is not available.

PiperOrigin-RevId: 231504648
---
 tensorflow/lite/nnapi/NeuralNetworksShim.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/nnapi/NeuralNetworksShim.h b/tensorflow/lite/nnapi/NeuralNetworksShim.h
index 1800c70685..c48528fa2d 100644
--- a/tensorflow/lite/nnapi/NeuralNetworksShim.h
+++ b/tensorflow/lite/nnapi/NeuralNetworksShim.h
@@ -59,7 +59,8 @@ inline int ASharedMemory_create(const char* name, size_t size) {
       handle != nullptr ? reinterpret_cast<ASharedMemory_create_fn>(
                               dlsym(handle, "ASharedMemory_create"))
                         : nullptr;
-  return fn(name, size);
+  int fd = fn != nullptr ? fn(name, size) : -1;
+  return fd;
 }
 
 inline void* getLibraryHandle() {
-- 
GitLab


From fa6c6d82afb364efa772f8eb21b160039c53a17f Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Tue, 29 Jan 2019 17:19:34 -0800
Subject: [PATCH 1550/2345] Speed up masking in `Masking` Layer by computing it
 during `call`.

PiperOrigin-RevId: 231504870
---
 tensorflow/python/keras/layers/core.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/layers/core.py b/tensorflow/python/keras/layers/core.py
index fdc0a76877..221a9ba782 100644
--- a/tensorflow/python/keras/layers/core.py
+++ b/tensorflow/python/keras/layers/core.py
@@ -82,6 +82,7 @@ class Masking(Layer):
     super(Masking, self).__init__(**kwargs)
     self.supports_masking = True
     self.mask_value = mask_value
+    self._compute_output_and_mask_jointly = True
 
   def compute_mask(self, inputs, mask=None):
     return K.any(math_ops.not_equal(inputs, self.mask_value), axis=-1)
@@ -89,7 +90,10 @@ class Masking(Layer):
   def call(self, inputs):
     boolean_mask = K.any(
         math_ops.not_equal(inputs, self.mask_value), axis=-1, keepdims=True)
-    return inputs * math_ops.cast(boolean_mask, inputs.dtype)
+    outputs = inputs * math_ops.cast(boolean_mask, inputs.dtype)
+    # Compute the mask and outputs simultaneously.
+    outputs._keras_mask = array_ops.squeeze(boolean_mask, axis=-1)  # pylint: disable=protected-access
+    return outputs
 
   def compute_output_shape(self, input_shape):
     return input_shape
-- 
GitLab


From 3407ea014870d64d30ebd6a5d663826dbeb6e3fd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 29 Jan 2019 17:35:32 -0800
Subject: [PATCH 1551/2345] Fix use-after-free bug in Swisstable usage.

PiperOrigin-RevId: 231507067
---
 tensorflow/core/graph/collective_order.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/core/graph/collective_order.cc b/tensorflow/core/graph/collective_order.cc
index c24166cb8e..e835259d64 100644
--- a/tensorflow/core/graph/collective_order.cc
+++ b/tensorflow/core/graph/collective_order.cc
@@ -47,6 +47,9 @@ Status DiscoverDataDependencies(
       instance_keys->push_back(instance_key);
       VLOG(2) << "collective node " << node->DebugString();
     }
+    // Avoid reference invalidation of `node_deps`.
+    data_dependencies->reserve(data_dependencies->size() + 1 +
+                               node->out_edges().size());
     const auto& node_deps = (*data_dependencies)[node];
     for (const Edge* out_edge : node->out_edges()) {
       auto& child_deps = (*data_dependencies)[out_edge->dst()];
-- 
GitLab


From c6c87456681b88eb9b664fdeaf7e0b735c345ff0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 29 Jan 2019 17:38:05 -0800
Subject: [PATCH 1552/2345] Fix a minor typo in math_ops.py

PiperOrigin-RevId: 231507427
---
 tensorflow/python/ops/math_ops.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index cb7fecf7c7..79133331fc 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -2659,11 +2659,11 @@ def add_n(inputs, name=None):
     cannot be inferred.
   """
   if not inputs or not isinstance(inputs, (list, tuple)):
-    raise ValueError("inputs must be a list of at least one"
+    raise ValueError("inputs must be a list of at least one "
                      "Tensor/IndexedSlices with the same dtype and shape")
   inputs = ops.convert_n_to_tensor_or_indexed_slices(inputs)
   if not all(isinstance(x, (ops.Tensor, ops.IndexedSlices)) for x in inputs):
-    raise ValueError("inputs must be a list of at least one"
+    raise ValueError("inputs must be a list of at least one "
                      "Tensor/IndexedSlices with the same dtype and shape")
 
   if len(inputs) == 1:
-- 
GitLab


From 4a5273567f4a7f950af23647a6dff4932e393835 Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Tue, 29 Jan 2019 17:41:07 -0800
Subject: [PATCH 1553/2345] remove device on loop in all keras tests.

PiperOrigin-RevId: 231507863
---
 .../contrib/distribute/python/combinations.py | 23 +++++--------------
 .../python/keras_correctness_test_base.py     | 10 +++-----
 ...as_stateful_lstm_model_correctness_test.py |  4 ++--
 3 files changed, 11 insertions(+), 26 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/combinations.py b/tensorflow/contrib/distribute/python/combinations.py
index f104d950c3..6122e71632 100644
--- a/tensorflow/contrib/distribute/python/combinations.py
+++ b/tensorflow/contrib/distribute/python/combinations.py
@@ -357,24 +357,13 @@ tpu_strategy = NamedDistribution(
 tpu_strategy_one_step = NamedDistribution(
     "TPUOneStep", _get_tpu_strategy_creator(steps_per_run=1),
     required_tpu=True)
-tpu_strategy_loop_on_device_one_core = NamedDistribution(
-    "TPULoopOnDeviceOneCore", _get_tpu_strategy_creator(
-        steps_per_run=2, use_single_core=True,
-        _disable_training_loop_on_host=True),
+tpu_strategy_one_core = NamedDistribution(
+    "TPUOneCore", _get_tpu_strategy_creator(
+        steps_per_run=2, use_single_core=True),
     required_tpu=True)
-tpu_strategy_one_step_loop_on_device_one_core = NamedDistribution(
-    "TPUOneStepLoopOnDeviceOneCore", _get_tpu_strategy_creator(
-        steps_per_run=1, use_single_core=True,
-        _disable_training_loop_on_host=True),
-    required_tpu=True)
-# TODO(b/122327153): Remove below two NamedDistributions.
-tpu_strategy_loop_on_device = NamedDistribution(
-    "TPULoopOnDevice", _get_tpu_strategy_creator(
-        steps_per_run=2, _disable_training_loop_on_host=True),
-    required_tpu=True)
-tpu_strategy_one_step_loop_on_device = NamedDistribution(
-    "TPUOneStepLoopOnDevice", _get_tpu_strategy_creator(
-        steps_per_run=1, _disable_training_loop_on_host=True),
+tpu_strategy_one_step_one_core = NamedDistribution(
+    "TPUOneStepOneCore", _get_tpu_strategy_creator(
+        steps_per_run=1, use_single_core=True),
     required_tpu=True)
 
 mirrored_strategy_with_one_cpu = NamedDistribution(
diff --git a/tensorflow/contrib/distribute/python/keras_correctness_test_base.py b/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
index 0c783099b5..0fb7a18c40 100644
--- a/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
+++ b/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
@@ -78,11 +78,7 @@ def strategies_for_embedding_models():
   and DefaultStrategy in order to prevent testing timeouts.
   """
 
-  strategies = [s for s in all_strategies
-                if not s.required_tpu and s.required_gpus is not None]
-  strategies.append(combinations.tpu_strategy_loop_on_device)
-  strategies.append(combinations.tpu_strategy_one_step_loop_on_device)
-  return strategies
+  return [s for s in all_strategies if s.required_tpu or s.required_gpus]
 
 
 def test_combinations_for_embedding_model():
@@ -95,8 +91,8 @@ def test_combinations_for_embedding_model():
 
 
 def test_combinations_with_tpu_strategies():
-  tpu_strategies = [combinations.tpu_strategy_loop_on_device,
-                    combinations.tpu_strategy_one_step_loop_on_device]
+  tpu_strategies = [combinations.tpu_strategy,
+                    combinations.tpu_strategy_one_step]
 
   return (
       combinations.times(
diff --git a/tensorflow/contrib/distribute/python/keras_stateful_lstm_model_correctness_test.py b/tensorflow/contrib/distribute/python/keras_stateful_lstm_model_correctness_test.py
index ab56c01d86..f5faf6c36b 100644
--- a/tensorflow/contrib/distribute/python/keras_stateful_lstm_model_correctness_test.py
+++ b/tensorflow/contrib/distribute/python/keras_stateful_lstm_model_correctness_test.py
@@ -29,8 +29,8 @@ from tensorflow.python.training import gradient_descent
 def strategies_for_stateful_embedding_model():
   """Returns TPUStrategy with single core device assignment."""
 
-  return [combinations.tpu_strategy_loop_on_device_one_core,
-          combinations.tpu_strategy_one_step_loop_on_device_one_core]
+  return [combinations.tpu_strategy_one_core,
+          combinations.tpu_strategy_one_step_one_core]
 
 
 def test_combinations_for_stateful_embedding_model():
-- 
GitLab


From e55946355883c75a9d306e6a456e42ac6fe1f0dc Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Tue, 29 Jan 2019 18:01:38 -0800
Subject: [PATCH 1554/2345] Create "PopulateConvolutionQuantizationParams"

PiperOrigin-RevId: 231510449
---
 tensorflow/lite/kernels/BUILD               |   1 +
 tensorflow/lite/kernels/kernel_util.cc      |  68 +++++++++++++
 tensorflow/lite/kernels/kernel_util.h       |   8 ++
 tensorflow/lite/kernels/kernel_util_test.cc | 107 ++++++++++++++++++++
 4 files changed, 184 insertions(+)

diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index 8f7860242a..d338bc5499 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -111,6 +111,7 @@ cc_library(
     ],
     deps = [
         "//tensorflow/lite/c:c_api_internal",
+        "//tensorflow/lite/kernels/internal:quantization_util",
         "//tensorflow/lite/kernels/internal:round",
     ],
 )
diff --git a/tensorflow/lite/kernels/kernel_util.cc b/tensorflow/lite/kernels/kernel_util.cc
index 57f4bfa9fa..eafc1e4676 100644
--- a/tensorflow/lite/kernels/kernel_util.cc
+++ b/tensorflow/lite/kernels/kernel_util.cc
@@ -18,10 +18,78 @@ limitations under the License.
 #include <cmath>
 #include <memory>
 
+#include "tensorflow/lite/kernels/internal/quantization_util.h"
 #include "tensorflow/lite/kernels/internal/round.h"
 
 namespace tflite {
 
+TfLiteStatus PopulateConvolutionQuantizationParams(
+    TfLiteContext* context, const TfLiteTensor* input,
+    const TfLiteTensor* filter, const TfLiteTensor* bias, TfLiteTensor* output,
+    const TfLiteFusedActivation& activation, int32_t* multiplier, int* shift,
+    int32_t* output_activation_min, int32_t* output_activation_max,
+    int32_t* per_channel_multiplier, int* per_channel_shift) {
+  TF_LITE_ENSURE_EQ(context, input->quantization.type,
+                    kTfLiteAffineQuantization);
+  TF_LITE_ENSURE_EQ(context, filter->quantization.type,
+                    kTfLiteAffineQuantization);
+  // TODO(jianlijianli): Enable bias type check and bias scale == input scale
+  // * filter scale for each channel in affine quantization once bias
+  // quantization is properly populated.
+  // TF_LITE_ENSURE_EQ(context, bias->quantization.type,
+  // kTfLiteAffineQuantization);
+
+  // Check data type.
+  const auto* affine_quantization =
+      reinterpret_cast<TfLiteAffineQuantization*>(filter->quantization.params);
+  TF_LITE_ENSURE(context, affine_quantization);
+  TF_LITE_ENSURE(context, affine_quantization->scale);
+  const bool is_per_channel = affine_quantization->scale->size > 1;
+  if (is_per_channel) {
+    //  Currently only Int8 is supported for per channel quantization.
+    TF_LITE_ENSURE_EQ(context, input->type, kTfLiteInt8);
+    TF_LITE_ENSURE_EQ(context, filter->type, kTfLiteInt8);
+    TF_LITE_ENSURE_EQ(
+        context, affine_quantization->scale->size,
+        filter->dims->data[affine_quantization->quantized_dimension]);
+  }
+
+  // Populate multiplier and shift using affine quantization.
+  const int num_channels = affine_quantization->scale->size;
+  const float input_scale = input->params.scale;
+  const float output_scale = output->params.scale;
+  const float* filter_scales = affine_quantization->scale->data;
+  for (int i = 0; i < num_channels; ++i) {
+    const double filter_scale = static_cast<double>(filter_scales[i]);
+    const double effective_output_scale = static_cast<double>(input_scale) *
+                                          filter_scale /
+                                          static_cast<double>(output_scale);
+    int32_t significand;
+    int shift;
+    QuantizeMultiplier(effective_output_scale, &significand, &shift);
+    per_channel_multiplier[i] = significand;
+    per_channel_shift[i] = shift;
+  }
+
+  // Populate scalar quantization parameters.
+  // This check on legacy quantization parameters is kept only for backward
+  // compatibility.
+  if (input->type == kTfLiteUInt8) {
+    // Check bias scale == input scale * filter scale.
+    double real_multiplier = 0.0;
+    TF_LITE_ENSURE_STATUS(GetQuantizedConvolutionMultipler(
+        context, input, filter, bias, output, &real_multiplier));
+    int exponent;
+
+    // Populate quantization parameteters with multiplier and shift.
+    QuantizeMultiplier(real_multiplier, multiplier, &exponent);
+    *shift = -exponent;
+    CalculateActivationRangeUint8(activation, output, output_activation_min,
+                                  output_activation_max);
+  }
+  return kTfLiteOk;
+}
+
 TfLiteStatus GetQuantizedConvolutionMultipler(TfLiteContext* context,
                                               const TfLiteTensor* input,
                                               const TfLiteTensor* filter,
diff --git a/tensorflow/lite/kernels/kernel_util.h b/tensorflow/lite/kernels/kernel_util.h
index 4cfc885f89..57ff65fcea 100644
--- a/tensorflow/lite/kernels/kernel_util.h
+++ b/tensorflow/lite/kernels/kernel_util.h
@@ -84,6 +84,14 @@ inline void SetTensorToDynamic(TfLiteTensor* tensor) {
   }
 }
 
+// Check dimensionality match and populate OpData for Conv and DepthwiseConv.
+TfLiteStatus PopulateConvolutionQuantizationParams(
+    TfLiteContext* context, const TfLiteTensor* input,
+    const TfLiteTensor* filter, const TfLiteTensor* bias, TfLiteTensor* output,
+    const TfLiteFusedActivation& activation, int32_t* multiplier, int* shift,
+    int32_t* output_activation_min, int32_t* output_activation_max,
+    int32_t* per_channel_multiplier, int* per_channel_shift);
+
 // Calculates the multiplication factor for a quantized convolution (or
 // quantized depthwise convolution) involving the given tensors. Returns an
 // error if the scales of the tensors are not compatible.
diff --git a/tensorflow/lite/kernels/kernel_util_test.cc b/tensorflow/lite/kernels/kernel_util_test.cc
index 185ddfaa9d..4e792542a1 100644
--- a/tensorflow/lite/kernels/kernel_util_test.cc
+++ b/tensorflow/lite/kernels/kernel_util_test.cc
@@ -144,6 +144,113 @@ TEST_F(KernelUtilTest, BroadcastShapeDifferentSizes) {
   TfLiteIntArrayFree(output);
 }
 
+// TODO(jianlijianli): Add more test cases.
+TEST_F(KernelUtilTest, CheckAndPopulate) {
+  // Create input.
+  TfLiteTensor input;
+  input.type = kTfLiteInt8;
+  input.allocation_type = kTfLiteArenaRw;
+  input.dims = TfLiteIntArrayCreate(1);
+  input.dims->data[0] = 2;
+  TfLiteQuantizationParams input_quant = {0.5, 5};
+  input.params = input_quant;
+  input.quantization.type = kTfLiteAffineQuantization;
+  auto* input_params = reinterpret_cast<TfLiteAffineQuantization*>(
+      malloc(sizeof(TfLiteAffineQuantization)));
+  input_params->scale = TfLiteFloatArrayCreate(1);
+  input_params->scale->data[0] = 0.5;
+  input_params->zero_point = TfLiteIntArrayCreate(1);
+  input_params->zero_point->data[0] = 5;
+  input.quantization.params = reinterpret_cast<void*>(input_params);
+
+  // Create filter.
+  TfLiteTensor filter;
+  filter.type = kTfLiteInt8;
+  filter.allocation_type = kTfLiteArenaRw;
+  filter.dims = TfLiteIntArrayCreate(4);
+  filter.dims->data[0] = 3;
+  filter.dims->data[1] = 4;
+  filter.dims->data[2] = 5;
+  filter.dims->data[3] = 6;
+  TfLiteQuantizationParams filter_quant = {0.25, 0};
+  filter.params = filter_quant;
+  filter.quantization.type = kTfLiteAffineQuantization;
+  auto* filter_params = reinterpret_cast<TfLiteAffineQuantization*>(
+      malloc(sizeof(TfLiteAffineQuantization)));
+  filter_params->scale = TfLiteFloatArrayCreate(3);
+  filter_params->scale->data[0] = 0.25;
+  filter_params->scale->data[1] = 0.125;
+  filter_params->scale->data[2] = 0.25;
+  filter_params->zero_point = TfLiteIntArrayCreate(3);
+  filter_params->zero_point->data[0] = 0;
+  filter_params->zero_point->data[1] = 0;
+  filter_params->zero_point->data[2] = 0;
+  filter_params->quantized_dimension = 0;
+  filter.quantization.params = reinterpret_cast<void*>(filter_params);
+
+  // Create bias.
+  TfLiteTensor bias;
+  bias.type = kTfLiteInt32;
+  bias.allocation_type = kTfLiteArenaRw;
+  bias.dims = TfLiteIntArrayCreate(4);
+  TfLiteQuantizationParams bias_quant = {0.125, 9};
+  bias.params = bias_quant;
+  bias.quantization.type = kTfLiteAffineQuantization;
+  auto* bias_params = reinterpret_cast<TfLiteAffineQuantization*>(
+      malloc(sizeof(TfLiteAffineQuantization)));
+  bias_params->scale = TfLiteFloatArrayCreate(3);
+  bias_params->scale->data[0] = 0.125;
+  bias_params->scale->data[1] = 0.0625;
+  bias_params->scale->data[2] = 0.125;
+  bias_params->zero_point = TfLiteIntArrayCreate(3);
+  bias_params->zero_point->data[0] = 11;
+  bias_params->zero_point->data[1] = 12;
+  bias_params->zero_point->data[2] = 15;
+  bias.quantization.params = reinterpret_cast<void*>(bias_params);
+
+  // Create output.
+  TfLiteTensor output;
+  output.type = kTfLiteInt8;
+  output.allocation_type = kTfLiteArenaRw;
+  output.dims = nullptr;
+  TfLiteQuantizationParams output_quant = {0.5, -128};
+  output.params = output_quant;
+  output.quantization.type = kTfLiteAffineQuantization;
+  auto* output_params = reinterpret_cast<TfLiteAffineQuantization*>(
+      malloc(sizeof(TfLiteAffineQuantization)));
+  output_params->scale = TfLiteFloatArrayCreate(1);
+  output_params->scale->data[0] = 0.5;
+  output_params->zero_point = TfLiteIntArrayCreate(1);
+  output_params->zero_point->data[0] = -128;
+  output.quantization.params = reinterpret_cast<void*>(output_params);
+
+  // Create call parameters.
+  TfLiteContext context;
+  int32_t multiplier;
+  int shift;
+  int32_t output_activation_min;
+  int32_t output_activation_max;
+  std::vector<int32_t> per_channel_multiplier(3);
+  std::vector<int> per_channel_shift(3);
+
+  // Call and verify results for per channel case.
+  EXPECT_EQ(
+      kTfLiteOk,
+      PopulateConvolutionQuantizationParams(
+          &context, &input, &filter, &bias, &output, kTfLiteActRelu,
+          &multiplier, &shift, &output_activation_min, &output_activation_max,
+          per_channel_multiplier.data(), per_channel_shift.data()));
+  EXPECT_THAT(per_channel_multiplier,
+              ::testing::ElementsAre(1073741824, 1073741824, 1073741824));
+  EXPECT_THAT(per_channel_shift, ::testing::ElementsAre(-1, -2, -1));
+
+  // Release.
+  TfLiteTensorFree(&input);
+  TfLiteTensorFree(&filter);
+  TfLiteTensorFree(&bias);
+  TfLiteTensorFree(&output);
+}
+
 }  // namespace
 }  // namespace tflite
 
-- 
GitLab


From f226ce78897067ecac6fc0b423da6bc270fe21f5 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Tue, 29 Jan 2019 18:02:07 -0800
Subject: [PATCH 1555/2345] Rework seq2seq decoder to be keras layer like and
 compatible with 2.0.

1. Adding new class for decoderV2, which is a keras layer. All the tensor input has been update to be passed in via call(). The other abstract method has not been updated much, only initialize() is update to take more tensor input.
2. The existing dynamic_decode method has been copied to be the call() body of DecoderV2, with some update of removing deprecated variable_scope.
3. Helper has been rename to Sampler to better describe what it does.
4. Change the Sampler to also take tensor input during initializer(), instead of __init__().
5. Existing test has been ported to cover DecoderV2, with eager enabled.

PiperOrigin-RevId: 231510508
---
 tensorflow/contrib/seq2seq/BUILD              |  42 +
 .../kernel_tests/basic_decoder_v2_test.py     | 670 +++++++++++++++
 .../python/kernel_tests/decoder_v2_test.py    | 169 ++++
 .../seq2seq/python/ops/basic_decoder.py       | 101 +++
 .../contrib/seq2seq/python/ops/decoder.py     | 141 +++-
 .../contrib/seq2seq/python/ops/sampler.py     | 765 ++++++++++++++++++
 6 files changed, 1884 insertions(+), 4 deletions(-)
 create mode 100644 tensorflow/contrib/seq2seq/python/kernel_tests/basic_decoder_v2_test.py
 create mode 100644 tensorflow/contrib/seq2seq/python/kernel_tests/decoder_v2_test.py
 create mode 100644 tensorflow/contrib/seq2seq/python/ops/sampler.py

diff --git a/tensorflow/contrib/seq2seq/BUILD b/tensorflow/contrib/seq2seq/BUILD
index 7d5ba90ded..2a70b08f5c 100644
--- a/tensorflow/contrib/seq2seq/BUILD
+++ b/tensorflow/contrib/seq2seq/BUILD
@@ -139,6 +139,27 @@ cuda_py_test(
     ],
 )
 
+cuda_py_test(
+    name = "basic_decoder_v2_test",
+    size = "medium",
+    srcs = ["python/kernel_tests/basic_decoder_v2_test.py"],
+    additional_deps = [
+        ":seq2seq_py",
+        "@absl_py//absl/testing:parameterized",
+        "//tensorflow/contrib/layers:layers_py",
+        "//tensorflow/contrib/rnn:rnn_py",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:init_ops",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:rnn",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python:variables",
+    ],
+)
+
 cuda_py_test(
     name = "beam_search_ops_test",
     size = "medium",
@@ -173,6 +194,27 @@ cuda_py_test(
     ],
 )
 
+cuda_py_test(
+    name = "decoder_v2_test",
+    size = "medium",
+    srcs = ["python/kernel_tests/decoder_v2_test.py"],
+    additional_deps = [
+        ":seq2seq_py",
+        "@absl_py//absl/testing:parameterized",
+        "//tensorflow/contrib/layers:layers_py",
+        "//tensorflow/contrib/rnn:rnn_py",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:init_ops",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:rnn",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python:variables",
+    ],
+)
+
 cuda_py_test(
     name = "beam_search_decoder_test",
     size = "medium",
diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/basic_decoder_v2_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/basic_decoder_v2_test.py
new file mode 100644
index 0000000000..15fb688fc4
--- /dev/null
+++ b/tensorflow/contrib/seq2seq/python/kernel_tests/basic_decoder_v2_test.py
@@ -0,0 +1,670 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for contrib.seq2seq.python.seq2seq.basic_decoder_v2."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+import numpy as np
+
+from tensorflow.contrib.seq2seq.python.ops import basic_decoder
+from tensorflow.contrib.seq2seq.python.ops import sampler as sampler_py
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.keras import keras_parameterized
+from tensorflow.python.layers import core as layers_core
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import rnn_cell
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+
+
+@keras_parameterized.run_all_keras_modes
+class BasicDecoderTest(keras_parameterized.TestCase):
+  """Unit test for basic_decoder.BasicDecoderV2."""
+
+  @parameterized.named_parameters(
+      ("use_output_layer", True),
+      ("without_output_layer", False))
+  def testStepWithTrainingHelperOutputLayer(self, use_output_layer):
+    sequence_length = [3, 4, 3, 1, 0]
+    batch_size = 5
+    max_time = 8
+    input_depth = 7
+    cell_depth = 10
+    output_layer_depth = 3
+
+    with self.cached_session(use_gpu=True):
+      inputs = np.random.randn(batch_size, max_time,
+                               input_depth).astype(np.float32)
+      input_t = constant_op.constant(inputs)
+      cell = rnn_cell.LSTMCell(cell_depth)
+      sampler = sampler_py.TrainingSampler(time_major=False)
+      if use_output_layer:
+        output_layer = layers_core.Dense(output_layer_depth, use_bias=False)
+        expected_output_depth = output_layer_depth
+      else:
+        output_layer = None
+        expected_output_depth = cell_depth
+      initial_state = cell.zero_state(dtype=dtypes.float32,
+                                      batch_size=batch_size)
+      my_decoder = basic_decoder.BasicDecoderV2(
+          cell=cell,
+          sampler=sampler,
+          output_layer=output_layer)
+
+      (first_finished,
+       first_inputs,
+       first_state) = my_decoder.initialize(input_t,
+                                            initial_state=initial_state,
+                                            sequence_length=sequence_length)
+      output_size = my_decoder.output_size
+      output_dtype = my_decoder.output_dtype
+      self.assertEqual(
+          basic_decoder.BasicDecoderOutput(expected_output_depth,
+                                           tensor_shape.TensorShape([])),
+          output_size)
+      self.assertEqual(
+          basic_decoder.BasicDecoderOutput(dtypes.float32, dtypes.int32),
+          output_dtype)
+
+      (step_outputs, step_state, step_next_inputs,
+       step_finished) = my_decoder.step(
+           constant_op.constant(0), first_inputs, first_state)
+      batch_size_t = my_decoder.batch_size
+
+      self.assertTrue(isinstance(first_state, rnn_cell.LSTMStateTuple))
+      self.assertTrue(isinstance(step_state, rnn_cell.LSTMStateTuple))
+      self.assertTrue(
+          isinstance(step_outputs, basic_decoder.BasicDecoderOutput))
+      self.assertEqual((batch_size, expected_output_depth),
+                       step_outputs[0].get_shape())
+      self.assertEqual((batch_size,), step_outputs[1].get_shape())
+      self.assertEqual((batch_size, cell_depth), first_state[0].get_shape())
+      self.assertEqual((batch_size, cell_depth), first_state[1].get_shape())
+      self.assertEqual((batch_size, cell_depth), step_state[0].get_shape())
+      self.assertEqual((batch_size, cell_depth), step_state[1].get_shape())
+
+      if use_output_layer:
+        # The output layer was accessed
+        self.assertEqual(len(output_layer.variables), 1)
+
+      self.evaluate(variables.global_variables_initializer())
+      eval_result = self.evaluate({
+          "batch_size": batch_size_t,
+          "first_finished": first_finished,
+          "first_inputs": first_inputs,
+          "first_state": first_state,
+          "step_outputs": step_outputs,
+          "step_state": step_state,
+          "step_next_inputs": step_next_inputs,
+          "step_finished": step_finished
+      })
+
+      self.assertAllEqual([False, False, False, False, True],
+                          eval_result["first_finished"])
+      self.assertAllEqual([False, False, False, True, True],
+                          eval_result["step_finished"])
+      self.assertEqual(output_dtype.sample_id,
+                       eval_result["step_outputs"].sample_id.dtype)
+      self.assertAllEqual(
+          np.argmax(eval_result["step_outputs"].rnn_output, -1),
+          eval_result["step_outputs"].sample_id)
+
+  def testStepWithGreedyEmbeddingHelper(self):
+    batch_size = 5
+    vocabulary_size = 7
+    cell_depth = vocabulary_size  # cell's logits must match vocabulary size
+    input_depth = 10
+    start_tokens = np.random.randint(0, vocabulary_size, size=batch_size)
+    end_token = 1
+
+    with self.cached_session(use_gpu=True):
+      embeddings = np.random.randn(vocabulary_size,
+                                   input_depth).astype(np.float32)
+      embeddings_t = constant_op.constant(embeddings)
+      cell = rnn_cell.LSTMCell(vocabulary_size)
+      sampler = sampler_py.GreedyEmbeddingSampler()
+      initial_state = cell.zero_state(
+          dtype=dtypes.float32, batch_size=batch_size)
+      my_decoder = basic_decoder.BasicDecoderV2(
+          cell=cell,
+          sampler=sampler)
+      (first_finished, first_inputs, first_state) = my_decoder.initialize(
+          embeddings_t,
+          start_tokens=start_tokens,
+          end_token=end_token,
+          initial_state=initial_state)
+      output_size = my_decoder.output_size
+      output_dtype = my_decoder.output_dtype
+      self.assertEqual(
+          basic_decoder.BasicDecoderOutput(cell_depth,
+                                           tensor_shape.TensorShape([])),
+          output_size)
+      self.assertEqual(
+          basic_decoder.BasicDecoderOutput(dtypes.float32, dtypes.int32),
+          output_dtype)
+
+      (step_outputs, step_state, step_next_inputs,
+       step_finished) = my_decoder.step(
+           constant_op.constant(0), first_inputs, first_state)
+      batch_size_t = my_decoder.batch_size
+
+      self.assertTrue(isinstance(first_state, rnn_cell.LSTMStateTuple))
+      self.assertTrue(isinstance(step_state, rnn_cell.LSTMStateTuple))
+      self.assertTrue(
+          isinstance(step_outputs, basic_decoder.BasicDecoderOutput))
+      self.assertEqual((batch_size, cell_depth), step_outputs[0].get_shape())
+      self.assertEqual((batch_size,), step_outputs[1].get_shape())
+      self.assertEqual((batch_size, cell_depth), first_state[0].get_shape())
+      self.assertEqual((batch_size, cell_depth), first_state[1].get_shape())
+      self.assertEqual((batch_size, cell_depth), step_state[0].get_shape())
+      self.assertEqual((batch_size, cell_depth), step_state[1].get_shape())
+
+      self.evaluate(variables.global_variables_initializer())
+      eval_result = self.evaluate({
+          "batch_size": batch_size_t,
+          "first_finished": first_finished,
+          "first_inputs": first_inputs,
+          "first_state": first_state,
+          "step_outputs": step_outputs,
+          "step_state": step_state,
+          "step_next_inputs": step_next_inputs,
+          "step_finished": step_finished
+      })
+
+      expected_sample_ids = np.argmax(
+          eval_result["step_outputs"].rnn_output, -1)
+      expected_step_finished = (expected_sample_ids == end_token)
+      expected_step_next_inputs = embeddings[expected_sample_ids]
+      self.assertAllEqual([False, False, False, False, False],
+                          eval_result["first_finished"])
+      self.assertAllEqual(expected_step_finished, eval_result["step_finished"])
+      self.assertEqual(output_dtype.sample_id,
+                       eval_result["step_outputs"].sample_id.dtype)
+      self.assertAllEqual(expected_sample_ids,
+                          eval_result["step_outputs"].sample_id)
+      self.assertAllEqual(expected_step_next_inputs,
+                          eval_result["step_next_inputs"])
+
+  def testStepWithSampleEmbeddingHelper(self):
+    batch_size = 5
+    vocabulary_size = 7
+    cell_depth = vocabulary_size  # cell's logits must match vocabulary size
+    input_depth = 10
+    np.random.seed(0)
+    start_tokens = np.random.randint(0, vocabulary_size, size=batch_size)
+    end_token = 1
+
+    with self.cached_session(use_gpu=True):
+      embeddings = np.random.randn(vocabulary_size,
+                                   input_depth).astype(np.float32)
+      embeddings_t = constant_op.constant(embeddings)
+      cell = rnn_cell.LSTMCell(vocabulary_size)
+      sampler = sampler_py.SampleEmbeddingSampler(seed=0)
+      initial_state = cell.zero_state(
+          dtype=dtypes.float32, batch_size=batch_size)
+      my_decoder = basic_decoder.BasicDecoderV2(cell=cell, sampler=sampler)
+      (first_finished,
+       first_inputs,
+       first_state) = my_decoder.initialize(embeddings_t,
+                                            start_tokens=start_tokens,
+                                            end_token=end_token,
+                                            initial_state=initial_state)
+      output_size = my_decoder.output_size
+      output_dtype = my_decoder.output_dtype
+      self.assertEqual(
+          basic_decoder.BasicDecoderOutput(cell_depth,
+                                           tensor_shape.TensorShape([])),
+          output_size)
+      self.assertEqual(
+          basic_decoder.BasicDecoderOutput(dtypes.float32, dtypes.int32),
+          output_dtype)
+
+      (step_outputs, step_state, step_next_inputs,
+       step_finished) = my_decoder.step(
+           constant_op.constant(0), first_inputs, first_state)
+      batch_size_t = my_decoder.batch_size
+
+      self.assertTrue(isinstance(first_state, rnn_cell.LSTMStateTuple))
+      self.assertTrue(isinstance(step_state, rnn_cell.LSTMStateTuple))
+      self.assertTrue(
+          isinstance(step_outputs, basic_decoder.BasicDecoderOutput))
+      self.assertEqual((batch_size, cell_depth), step_outputs[0].get_shape())
+      self.assertEqual((batch_size,), step_outputs[1].get_shape())
+      self.assertEqual((batch_size, cell_depth), first_state[0].get_shape())
+      self.assertEqual((batch_size, cell_depth), first_state[1].get_shape())
+      self.assertEqual((batch_size, cell_depth), step_state[0].get_shape())
+      self.assertEqual((batch_size, cell_depth), step_state[1].get_shape())
+
+      self.evaluate(variables.global_variables_initializer())
+      eval_result = self.evaluate({
+          "batch_size": batch_size_t,
+          "first_finished": first_finished,
+          "first_inputs": first_inputs,
+          "first_state": first_state,
+          "step_outputs": step_outputs,
+          "step_state": step_state,
+          "step_next_inputs": step_next_inputs,
+          "step_finished": step_finished
+      })
+
+      sample_ids = eval_result["step_outputs"].sample_id
+      self.assertEqual(output_dtype.sample_id, sample_ids.dtype)
+      expected_step_finished = (sample_ids == end_token)
+      expected_step_next_inputs = embeddings[sample_ids]
+      self.assertAllEqual(expected_step_finished,
+                          eval_result["step_finished"])
+      self.assertAllEqual(expected_step_next_inputs,
+                          eval_result["step_next_inputs"])
+
+  def testStepWithScheduledEmbeddingTrainingHelper(self):
+    sequence_length = [3, 4, 3, 1, 0]
+    batch_size = 5
+    max_time = 8
+    input_depth = 7
+    vocabulary_size = 10
+
+    with self.cached_session(use_gpu=True):
+      inputs = np.random.randn(
+          batch_size, max_time, input_depth).astype(np.float32)
+      input_t = constant_op.constant(inputs)
+      embeddings = np.random.randn(
+          vocabulary_size, input_depth).astype(np.float32)
+      half = constant_op.constant(0.5)
+      cell = rnn_cell.LSTMCell(vocabulary_size)
+      sampler = sampler_py.ScheduledEmbeddingTrainingSampler(
+          sampling_probability=half,
+          time_major=False)
+      initial_state = cell.zero_state(
+          dtype=dtypes.float32, batch_size=batch_size)
+      my_decoder = basic_decoder.BasicDecoderV2(
+          cell=cell,
+          sampler=sampler)
+      (first_finished, first_inputs, first_state) = my_decoder.initialize(
+          input_t, sequence_length=sequence_length, embedding=embeddings,
+          initial_state=initial_state)
+      output_size = my_decoder.output_size
+      output_dtype = my_decoder.output_dtype
+      self.assertEqual(
+          basic_decoder.BasicDecoderOutput(vocabulary_size,
+                                           tensor_shape.TensorShape([])),
+          output_size)
+      self.assertEqual(
+          basic_decoder.BasicDecoderOutput(dtypes.float32, dtypes.int32),
+          output_dtype)
+
+      (step_outputs, step_state, step_next_inputs,
+       step_finished) = my_decoder.step(
+           constant_op.constant(0), first_inputs, first_state)
+      batch_size_t = my_decoder.batch_size
+
+      self.assertTrue(isinstance(first_state, rnn_cell.LSTMStateTuple))
+      self.assertTrue(isinstance(step_state, rnn_cell.LSTMStateTuple))
+      self.assertTrue(
+          isinstance(step_outputs, basic_decoder.BasicDecoderOutput))
+      self.assertEqual((batch_size, vocabulary_size),
+                       step_outputs[0].get_shape())
+      self.assertEqual((batch_size,), step_outputs[1].get_shape())
+      self.assertEqual((batch_size, vocabulary_size),
+                       first_state[0].get_shape())
+      self.assertEqual((batch_size, vocabulary_size),
+                       first_state[1].get_shape())
+      self.assertEqual((batch_size, vocabulary_size),
+                       step_state[0].get_shape())
+      self.assertEqual((batch_size, vocabulary_size),
+                       step_state[1].get_shape())
+      self.assertEqual((batch_size, input_depth),
+                       step_next_inputs.get_shape())
+
+      self.evaluate(variables.global_variables_initializer())
+      eval_result = self.evaluate({
+          "batch_size": batch_size_t,
+          "first_finished": first_finished,
+          "first_inputs": first_inputs,
+          "first_state": first_state,
+          "step_outputs": step_outputs,
+          "step_state": step_state,
+          "step_next_inputs": step_next_inputs,
+          "step_finished": step_finished
+      })
+
+      self.assertAllEqual([False, False, False, False, True],
+                          eval_result["first_finished"])
+      self.assertAllEqual([False, False, False, True, True],
+                          eval_result["step_finished"])
+      sample_ids = eval_result["step_outputs"].sample_id
+      self.assertEqual(output_dtype.sample_id, sample_ids.dtype)
+      batch_where_not_sampling = np.where(sample_ids == -1)
+      batch_where_sampling = np.where(sample_ids > -1)
+      self.assertAllClose(
+          eval_result["step_next_inputs"][batch_where_sampling],
+          embeddings[sample_ids[batch_where_sampling]])
+      self.assertAllClose(
+          eval_result["step_next_inputs"][batch_where_not_sampling],
+          np.squeeze(inputs[batch_where_not_sampling, 1], axis=0))
+
+  def _testStepWithScheduledOutputTrainingHelper(
+      self, sampling_probability, use_next_inputs_fn, use_auxiliary_inputs):
+    sequence_length = [3, 4, 3, 1, 0]
+    batch_size = 5
+    max_time = 8
+    input_depth = 7
+    cell_depth = input_depth
+    if use_auxiliary_inputs:
+      auxiliary_input_depth = 4
+      auxiliary_inputs = np.random.randn(
+          batch_size, max_time, auxiliary_input_depth).astype(np.float32)
+    else:
+      auxiliary_inputs = None
+
+    with self.cached_session(use_gpu=True):
+      inputs = np.random.randn(batch_size, max_time,
+                               input_depth).astype(np.float32)
+      input_t = constant_op.constant(inputs)
+      cell = rnn_cell.LSTMCell(cell_depth)
+      sampling_probability = constant_op.constant(sampling_probability)
+
+      if use_next_inputs_fn:
+        def next_inputs_fn(outputs):
+          # Use deterministic function for test.
+          samples = math_ops.argmax(outputs, axis=1)
+          return array_ops.one_hot(samples, cell_depth, dtype=dtypes.float32)
+      else:
+        next_inputs_fn = None
+
+      sampler = sampler_py.ScheduledOutputTrainingSampler(
+          sampling_probability=sampling_probability,
+          time_major=False,
+          next_inputs_fn=next_inputs_fn)
+      initial_state = cell.zero_state(
+          dtype=dtypes.float32, batch_size=batch_size)
+
+      my_decoder = basic_decoder.BasicDecoderV2(
+          cell=cell,
+          sampler=sampler)
+
+      (first_finished,
+       first_inputs,
+       first_state) = my_decoder.initialize(input_t,
+                                            sequence_length=sequence_length,
+                                            initial_state=initial_state,
+                                            auxiliary_inputs=auxiliary_inputs)
+      output_size = my_decoder.output_size
+      output_dtype = my_decoder.output_dtype
+      self.assertEqual(
+          basic_decoder.BasicDecoderOutput(cell_depth,
+                                           tensor_shape.TensorShape([])),
+          output_size)
+      self.assertEqual(
+          basic_decoder.BasicDecoderOutput(dtypes.float32, dtypes.int32),
+          output_dtype)
+
+      (step_outputs, step_state, step_next_inputs,
+       step_finished) = my_decoder.step(
+           constant_op.constant(0), first_inputs, first_state)
+
+      if use_next_inputs_fn:
+        output_after_next_inputs_fn = next_inputs_fn(step_outputs.rnn_output)
+
+      batch_size_t = my_decoder.batch_size
+
+      self.assertTrue(isinstance(first_state, rnn_cell.LSTMStateTuple))
+      self.assertTrue(isinstance(step_state, rnn_cell.LSTMStateTuple))
+      self.assertTrue(
+          isinstance(step_outputs, basic_decoder.BasicDecoderOutput))
+      self.assertEqual((batch_size, cell_depth), step_outputs[0].get_shape())
+      self.assertEqual((batch_size,), step_outputs[1].get_shape())
+      self.assertEqual((batch_size, cell_depth), first_state[0].get_shape())
+      self.assertEqual((batch_size, cell_depth), first_state[1].get_shape())
+      self.assertEqual((batch_size, cell_depth), step_state[0].get_shape())
+      self.assertEqual((batch_size, cell_depth), step_state[1].get_shape())
+
+      self.evaluate(variables.global_variables_initializer())
+
+      fetches = {
+          "batch_size": batch_size_t,
+          "first_finished": first_finished,
+          "first_inputs": first_inputs,
+          "first_state": first_state,
+          "step_outputs": step_outputs,
+          "step_state": step_state,
+          "step_next_inputs": step_next_inputs,
+          "step_finished": step_finished
+      }
+      if use_next_inputs_fn:
+        fetches["output_after_next_inputs_fn"] = output_after_next_inputs_fn
+
+      eval_result = self.evaluate(fetches)
+
+      self.assertAllEqual([False, False, False, False, True],
+                          eval_result["first_finished"])
+      self.assertAllEqual([False, False, False, True, True],
+                          eval_result["step_finished"])
+
+      sample_ids = eval_result["step_outputs"].sample_id
+      self.assertEqual(output_dtype.sample_id, sample_ids.dtype)
+      batch_where_not_sampling = np.where(np.logical_not(sample_ids))
+      batch_where_sampling = np.where(sample_ids)
+
+      auxiliary_inputs_to_concat = (
+          auxiliary_inputs[:, 1] if use_auxiliary_inputs else
+          np.array([]).reshape(batch_size, 0).astype(np.float32))
+
+      expected_next_sampling_inputs = np.concatenate(
+          (eval_result["output_after_next_inputs_fn"][batch_where_sampling]
+           if use_next_inputs_fn else
+           eval_result["step_outputs"].rnn_output[batch_where_sampling],
+           auxiliary_inputs_to_concat[batch_where_sampling]),
+          axis=-1)
+      self.assertAllClose(
+          eval_result["step_next_inputs"][batch_where_sampling],
+          expected_next_sampling_inputs)
+
+      self.assertAllClose(
+          eval_result["step_next_inputs"][batch_where_not_sampling],
+          np.concatenate(
+              (np.squeeze(inputs[batch_where_not_sampling, 1], axis=0),
+               auxiliary_inputs_to_concat[batch_where_not_sampling]),
+              axis=-1))
+
+  def testStepWithScheduledOutputTrainingHelperWithoutNextInputsFnOrAuxInputs(
+      self):
+    self._testStepWithScheduledOutputTrainingHelper(
+        sampling_probability=0.5, use_next_inputs_fn=False,
+        use_auxiliary_inputs=False)
+
+  def testStepWithScheduledOutputTrainingHelperWithNextInputsFn(self):
+    self._testStepWithScheduledOutputTrainingHelper(
+        sampling_probability=0.5, use_next_inputs_fn=True,
+        use_auxiliary_inputs=False)
+
+  def testStepWithScheduledOutputTrainingHelperWithAuxiliaryInputs(self):
+    self._testStepWithScheduledOutputTrainingHelper(
+        sampling_probability=0.5, use_next_inputs_fn=False,
+        use_auxiliary_inputs=True)
+
+  def testStepWithScheduledOutputTrainingHelperWithNextInputsFnAndAuxInputs(
+      self):
+    self._testStepWithScheduledOutputTrainingHelper(
+        sampling_probability=0.5, use_next_inputs_fn=True,
+        use_auxiliary_inputs=True)
+
+  def testStepWithScheduledOutputTrainingHelperWithNoSampling(self):
+    self._testStepWithScheduledOutputTrainingHelper(
+        sampling_probability=0.0, use_next_inputs_fn=True,
+        use_auxiliary_inputs=True)
+
+  def testStepWithInferenceHelperCategorical(self):
+    batch_size = 5
+    vocabulary_size = 7
+    cell_depth = vocabulary_size
+    start_token = 0
+    end_token = 6
+
+    start_inputs = array_ops.one_hot(
+        np.ones(batch_size, dtype=np.int32) * start_token,
+        vocabulary_size)
+
+    # The sample function samples categorically from the logits.
+    sample_fn = lambda x: sampler_py.categorical_sample(logits=x)
+    # The next inputs are a one-hot encoding of the sampled labels.
+    next_inputs_fn = (
+        lambda x: array_ops.one_hot(x, vocabulary_size, dtype=dtypes.float32))
+    end_fn = lambda sample_ids: math_ops.equal(sample_ids, end_token)
+
+    with self.cached_session(use_gpu=True):
+      cell = rnn_cell.LSTMCell(vocabulary_size)
+      sampler = sampler_py.InferenceSampler(
+          sample_fn, sample_shape=(), sample_dtype=dtypes.int32, end_fn=end_fn,
+          next_inputs_fn=next_inputs_fn)
+      initial_state = cell.zero_state(
+          dtype=dtypes.float32, batch_size=batch_size)
+      my_decoder = basic_decoder.BasicDecoderV2(
+          cell=cell,
+          sampler=sampler)
+      (first_finished, first_inputs, first_state) = my_decoder.initialize(
+          start_inputs, initial_state=initial_state)
+
+      output_size = my_decoder.output_size
+      output_dtype = my_decoder.output_dtype
+      self.assertEqual(
+          basic_decoder.BasicDecoderOutput(cell_depth,
+                                           tensor_shape.TensorShape([])),
+          output_size)
+      self.assertEqual(
+          basic_decoder.BasicDecoderOutput(dtypes.float32, dtypes.int32),
+          output_dtype)
+
+      (step_outputs, step_state, step_next_inputs,
+       step_finished) = my_decoder.step(
+           constant_op.constant(0), first_inputs, first_state)
+      batch_size_t = my_decoder.batch_size
+
+      self.assertTrue(isinstance(first_state, rnn_cell.LSTMStateTuple))
+      self.assertTrue(isinstance(step_state, rnn_cell.LSTMStateTuple))
+      self.assertTrue(
+          isinstance(step_outputs, basic_decoder.BasicDecoderOutput))
+      self.assertEqual((batch_size, cell_depth), step_outputs[0].get_shape())
+      self.assertEqual((batch_size,), step_outputs[1].get_shape())
+      self.assertEqual((batch_size, cell_depth), first_state[0].get_shape())
+      self.assertEqual((batch_size, cell_depth), first_state[1].get_shape())
+      self.assertEqual((batch_size, cell_depth), step_state[0].get_shape())
+      self.assertEqual((batch_size, cell_depth), step_state[1].get_shape())
+
+      self.evaluate(variables.global_variables_initializer())
+      eval_result = self.evaluate({
+          "batch_size": batch_size_t,
+          "first_finished": first_finished,
+          "first_inputs": first_inputs,
+          "first_state": first_state,
+          "step_outputs": step_outputs,
+          "step_state": step_state,
+          "step_next_inputs": step_next_inputs,
+          "step_finished": step_finished
+      })
+
+      sample_ids = eval_result["step_outputs"].sample_id
+      self.assertEqual(output_dtype.sample_id, sample_ids.dtype)
+      expected_step_finished = (sample_ids == end_token)
+      expected_step_next_inputs = np.zeros((batch_size, vocabulary_size))
+      expected_step_next_inputs[np.arange(batch_size), sample_ids] = 1.0
+      self.assertAllEqual(expected_step_finished,
+                          eval_result["step_finished"])
+      self.assertAllEqual(expected_step_next_inputs,
+                          eval_result["step_next_inputs"])
+
+  def testStepWithInferenceHelperMultilabel(self):
+    batch_size = 5
+    vocabulary_size = 7
+    cell_depth = vocabulary_size
+    start_token = 0
+    end_token = 6
+
+    start_inputs = array_ops.one_hot(
+        np.ones(batch_size, dtype=np.int32) * start_token,
+        vocabulary_size)
+
+    # The sample function samples independent bernoullis from the logits.
+    sample_fn = (
+        lambda x: sampler_py.bernoulli_sample(logits=x, dtype=dtypes.bool))
+    # The next inputs are a one-hot encoding of the sampled labels.
+    next_inputs_fn = math_ops.to_float
+    end_fn = lambda sample_ids: sample_ids[:, end_token]
+
+    with self.cached_session(use_gpu=True):
+      cell = rnn_cell.LSTMCell(vocabulary_size)
+      sampler = sampler_py.InferenceSampler(
+          sample_fn, sample_shape=[cell_depth], sample_dtype=dtypes.bool,
+          end_fn=end_fn, next_inputs_fn=next_inputs_fn)
+      initial_state = cell.zero_state(
+          dtype=dtypes.float32, batch_size=batch_size)
+      my_decoder = basic_decoder.BasicDecoderV2(
+          cell=cell,
+          sampler=sampler)
+      (first_finished, first_inputs, first_state) = my_decoder.initialize(
+          start_inputs, initial_state=initial_state)
+      output_size = my_decoder.output_size
+      output_dtype = my_decoder.output_dtype
+      self.assertEqual(
+          basic_decoder.BasicDecoderOutput(cell_depth, cell_depth),
+          output_size)
+      self.assertEqual(
+          basic_decoder.BasicDecoderOutput(dtypes.float32, dtypes.bool),
+          output_dtype)
+
+      (step_outputs, step_state, step_next_inputs,
+       step_finished) = my_decoder.step(
+           constant_op.constant(0), first_inputs, first_state)
+      batch_size_t = my_decoder.batch_size
+
+      self.assertTrue(isinstance(first_state, rnn_cell.LSTMStateTuple))
+      self.assertTrue(isinstance(step_state, rnn_cell.LSTMStateTuple))
+      self.assertTrue(
+          isinstance(step_outputs, basic_decoder.BasicDecoderOutput))
+      self.assertEqual((batch_size, cell_depth), step_outputs[0].get_shape())
+      self.assertEqual((batch_size, cell_depth), step_outputs[1].get_shape())
+      self.assertEqual((batch_size, cell_depth), first_state[0].get_shape())
+      self.assertEqual((batch_size, cell_depth), first_state[1].get_shape())
+      self.assertEqual((batch_size, cell_depth), step_state[0].get_shape())
+      self.assertEqual((batch_size, cell_depth), step_state[1].get_shape())
+
+      self.evaluate(variables.global_variables_initializer())
+      eval_result = self.evaluate({
+          "batch_size": batch_size_t,
+          "first_finished": first_finished,
+          "first_inputs": first_inputs,
+          "first_state": first_state,
+          "step_outputs": step_outputs,
+          "step_state": step_state,
+          "step_next_inputs": step_next_inputs,
+          "step_finished": step_finished
+      })
+
+      sample_ids = eval_result["step_outputs"].sample_id
+      self.assertEqual(output_dtype.sample_id, sample_ids.dtype)
+      expected_step_finished = sample_ids[:, end_token]
+      expected_step_next_inputs = sample_ids.astype(np.float32)
+      self.assertAllEqual(expected_step_finished,
+                          eval_result["step_finished"])
+      self.assertAllEqual(expected_step_next_inputs,
+                          eval_result["step_next_inputs"])
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/decoder_v2_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/decoder_v2_test.py
new file mode 100644
index 0000000000..b5bba2b32e
--- /dev/null
+++ b/tensorflow/contrib/seq2seq/python/kernel_tests/decoder_v2_test.py
@@ -0,0 +1,169 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for contrib.seq2seq.python.seq2seq.decoder."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.seq2seq.python.ops import basic_decoder
+from tensorflow.contrib.seq2seq.python.ops import sampler as sampler_py
+from tensorflow.python.eager import context
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.keras import keras_parameterized
+from tensorflow.python.ops import rnn
+from tensorflow.python.ops import rnn_cell
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+
+
+@keras_parameterized.run_all_keras_modes
+class DecodeV2RNNTest(keras_parameterized.TestCase, test.TestCase):
+  """Tests for DecoderV2."""
+
+  def _testDecodeRNN(self, time_major, maximum_iterations=None):
+
+    sequence_length = [3, 4, 3, 1, 0]
+    batch_size = 5
+    max_time = 8
+    input_depth = 7
+    cell_depth = 10
+    max_out = max(sequence_length)
+
+    with self.cached_session(use_gpu=True):
+      if time_major:
+        inputs = np.random.randn(max_time, batch_size,
+                                 input_depth).astype(np.float32)
+      else:
+        inputs = np.random.randn(batch_size, max_time,
+                                 input_depth).astype(np.float32)
+      input_t = constant_op.constant(inputs)
+      cell = rnn_cell.LSTMCell(cell_depth)
+      sampler = sampler_py.TrainingSampler(time_major=time_major)
+      my_decoder = basic_decoder.BasicDecoderV2(
+          cell=cell,
+          sampler=sampler,
+          output_time_major=time_major,
+          maximum_iterations=maximum_iterations)
+
+      initial_state = cell.zero_state(
+          dtype=dtypes.float32, batch_size=batch_size)
+      (final_outputs, unused_final_state, final_sequence_length) = my_decoder(
+          input_t, initial_state=initial_state, sequence_length=sequence_length)
+
+      def _t(shape):
+        if time_major:
+          return (shape[1], shape[0]) + shape[2:]
+        return shape
+
+      if not context.executing_eagerly():
+        self.assertEqual((batch_size,),
+                         tuple(final_sequence_length.get_shape().as_list()))
+        self.assertEqual(
+            _t((batch_size, None, cell_depth)),
+            tuple(final_outputs.rnn_output.get_shape().as_list()))
+        self.assertEqual(
+            _t((batch_size, None)),
+            tuple(final_outputs.sample_id.get_shape().as_list()))
+
+      self.evaluate(variables.global_variables_initializer())
+      final_outputs = self.evaluate(final_outputs)
+      final_sequence_length = self.evaluate(final_sequence_length)
+
+      # Mostly a smoke test
+      time_steps = max_out
+      expected_length = sequence_length
+      if maximum_iterations is not None:
+        time_steps = min(max_out, maximum_iterations)
+        expected_length = [min(x, maximum_iterations) for x in expected_length]
+      if context.executing_eagerly() and maximum_iterations != 0:
+        # Only check the shape of output when maximum_iterations > 0, see
+        # b/123431432 for more details.
+        self.assertEqual(
+            _t((batch_size, time_steps, cell_depth)),
+            final_outputs.rnn_output.shape)
+        self.assertEqual(
+            _t((batch_size, time_steps)), final_outputs.sample_id.shape)
+      self.assertItemsEqual(expected_length, final_sequence_length)
+
+  def testDynamicDecodeRNNBatchMajor(self):
+    self._testDecodeRNN(time_major=False)
+
+  def testDynamicDecodeRNNTimeMajor(self):
+    self._testDecodeRNN(time_major=True)
+
+  def testDynamicDecodeRNNZeroMaxIters(self):
+    self._testDecodeRNN(time_major=True, maximum_iterations=0)
+
+  def testDynamicDecodeRNNOneMaxIter(self):
+    self._testDecodeRNN(time_major=True, maximum_iterations=1)
+
+  def _testDynamicDecodeRNNWithTrainingHelperMatchesDynamicRNN(
+      self, use_sequence_length):
+    sequence_length = [3, 4, 3, 1, 0]
+    batch_size = 5
+    max_time = 8
+    input_depth = 7
+    cell_depth = 10
+    max_out = max(sequence_length)
+
+    with self.cached_session(use_gpu=True):
+      inputs = np.random.randn(batch_size, max_time,
+                               input_depth).astype(np.float32)
+      inputs = constant_op.constant(inputs)
+
+      cell = rnn_cell.LSTMCell(cell_depth)
+      zero_state = cell.zero_state(dtype=dtypes.float32, batch_size=batch_size)
+      sampler = sampler_py.TrainingSampler()
+      my_decoder = basic_decoder.BasicDecoderV2(
+          cell=cell, sampler=sampler, impute_finished=use_sequence_length)
+
+      final_decoder_outputs, final_decoder_state, _ = my_decoder(
+          inputs, initial_state=zero_state, sequence_length=sequence_length)
+
+      final_rnn_outputs, final_rnn_state = rnn.dynamic_rnn(
+          cell,
+          inputs,
+          sequence_length=sequence_length if use_sequence_length else None,
+          initial_state=zero_state)
+
+      self.evaluate(variables.global_variables_initializer())
+      eval_result = self.evaluate({
+          "final_decoder_outputs": final_decoder_outputs,
+          "final_decoder_state": final_decoder_state,
+          "final_rnn_outputs": final_rnn_outputs,
+          "final_rnn_state": final_rnn_state
+      })
+
+      # Decoder only runs out to max_out; ensure values are identical
+      # to dynamic_rnn, which also zeros out outputs and passes along state.
+      self.assertAllClose(eval_result["final_decoder_outputs"].rnn_output,
+                          eval_result["final_rnn_outputs"][:, 0:max_out, :])
+      if use_sequence_length:
+        self.assertAllClose(eval_result["final_decoder_state"],
+                            eval_result["final_rnn_state"])
+
+  def testDynamicDecodeRNNWithTrainingHelperMatchesDynamicRNNWithSeqLen(self):
+    self._testDynamicDecodeRNNWithTrainingHelperMatchesDynamicRNN(
+        use_sequence_length=True)
+
+  def testDynamicDecodeRNNWithTrainingHelperMatchesDynamicRNNNoSeqLen(self):
+    self._testDynamicDecodeRNNWithTrainingHelperMatchesDynamicRNN(
+        use_sequence_length=False)
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/seq2seq/python/ops/basic_decoder.py b/tensorflow/contrib/seq2seq/python/ops/basic_decoder.py
index 7eb95e5a70..16dfa7ed82 100644
--- a/tensorflow/contrib/seq2seq/python/ops/basic_decoder.py
+++ b/tensorflow/contrib/seq2seq/python/ops/basic_decoder.py
@@ -23,8 +23,10 @@ import collections
 
 from tensorflow.contrib.seq2seq.python.ops import decoder
 from tensorflow.contrib.seq2seq.python.ops import helper as helper_py
+from tensorflow.contrib.seq2seq.python.ops import sampler as sampler_py
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.keras import layers
 from tensorflow.python.layers import base as layers_base
 from tensorflow.python.ops import rnn_cell_impl
 from tensorflow.python.util import nest
@@ -146,3 +148,102 @@ class BasicDecoder(decoder.Decoder):
           sample_ids=sample_ids)
     outputs = BasicDecoderOutput(cell_outputs, sample_ids)
     return (outputs, next_state, next_inputs, finished)
+
+
+class BasicDecoderV2(decoder.BaseDecoder):
+  """Basic sampling decoder."""
+
+  def __init__(self, cell, sampler, output_layer=None, **kwargs):
+    """Initialize BasicDecoder.
+
+    Args:
+      cell: An `RNNCell` instance.
+      sampler: A `Sampler` instance.
+      output_layer: (Optional) An instance of `tf.layers.Layer`, i.e.,
+        `tf.layers.Dense`. Optional layer to apply to the RNN output prior to
+        storing the result or sampling.
+      **kwargs: Other keyward arguments for layer creation.
+
+    Raises:
+      TypeError: if `cell`, `helper` or `output_layer` have an incorrect type.
+    """
+    rnn_cell_impl.assert_like_rnncell("cell", cell)
+    if not isinstance(sampler, sampler_py.Sampler):
+      raise TypeError("sampler must be a Sampler, received: %s" % (sampler,))
+    if (output_layer is not None and
+        not isinstance(output_layer, layers.Layer)):
+      raise TypeError(
+          "output_layer must be a Layer, received: %s" % (output_layer,))
+    self.cell = cell
+    self.sampler = sampler
+    self.output_layer = output_layer
+    super(BasicDecoderV2, self).__init__(**kwargs)
+
+  def initialize(self, inputs, initial_state=None, **kwargs):
+    """Initialize the decoder."""
+    # Assume the dtype of the cell is the output_size structure
+    # containing the input_state's first component's dtype.
+    self._cell_dtype = nest.flatten(initial_state)[0].dtype
+    return self.sampler.initialize(inputs, **kwargs) + (initial_state,)
+
+  @property
+  def batch_size(self):
+    return self.sampler.batch_size
+
+  def _rnn_output_size(self):
+    size = tensor_shape.TensorShape(self.cell.output_size)
+    if self.output_layer is None:
+      return size
+    else:
+      # To use layer's compute_output_shape, we need to convert the
+      # RNNCell's output_size entries into shapes with an unknown
+      # batch size.  We then pass this through the layer's
+      # compute_output_shape and read off all but the first (batch)
+      # dimensions to get the output size of the rnn with the layer
+      # applied to the top.
+      output_shape_with_unknown_batch = nest.map_structure(
+          lambda s: tensor_shape.TensorShape([None]).concatenate(s), size)
+      layer_output_shape = self.output_layer.compute_output_shape(
+          output_shape_with_unknown_batch)
+      return nest.map_structure(lambda s: s[1:], layer_output_shape)
+
+  @property
+  def output_size(self):
+    # Return the cell output and the id
+    return BasicDecoderOutput(
+        rnn_output=self._rnn_output_size(),
+        sample_id=self.sampler.sample_ids_shape)
+
+  @property
+  def output_dtype(self):
+    # Assume the dtype of the cell is the output_size structure
+    # containing the input_state's first component's dtype.
+    # Return that structure and the sample_ids_dtype from the helper.
+    dtype = self._cell_dtype
+    return BasicDecoderOutput(
+        nest.map_structure(lambda _: dtype, self._rnn_output_size()),
+        self.sampler.sample_ids_dtype)
+
+  def step(self, time, inputs, state):
+    """Perform a decoding step.
+
+    Args:
+      time: scalar `int32` tensor.
+      inputs: A (structure of) input tensors.
+      state: A (structure of) state tensors and TensorArrays.
+
+    Returns:
+      `(outputs, next_state, next_inputs, finished)`.
+    """
+    cell_outputs, cell_state = self.cell(inputs, state)
+    if self.output_layer is not None:
+      cell_outputs = self.output_layer(cell_outputs)
+    sample_ids = self.sampler.sample(
+        time=time, outputs=cell_outputs, state=cell_state)
+    (finished, next_inputs, next_state) = self.sampler.next_inputs(
+        time=time,
+        outputs=cell_outputs,
+        state=cell_state,
+        sample_ids=sample_ids)
+    outputs = BasicDecoderOutput(cell_outputs, sample_ids)
+    return (outputs, next_state, next_inputs, finished)
diff --git a/tensorflow/contrib/seq2seq/python/ops/decoder.py b/tensorflow/contrib/seq2seq/python/ops/decoder.py
index f58268eff5..33f7bac815 100644
--- a/tensorflow/contrib/seq2seq/python/ops/decoder.py
+++ b/tensorflow/contrib/seq2seq/python/ops/decoder.py
@@ -27,6 +27,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import tensor_util
+from tensorflow.python.keras import layers
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import control_flow_util
@@ -135,6 +136,127 @@ class Decoder(object):
     return False
 
 
+class BaseDecoder(layers.Layer):
+  """An RNN Decoder that is based on a Keras layer.
+
+  Concepts used by this interface:
+  - `inputs`: (structure of) tensors and TensorArrays that is passed as input to
+    the RNNCell composing the decoder, at each time step.
+  - `state`: (structure of) tensors and TensorArrays that is passed to the
+    RNNCell instance as the state.
+  - `memory`: (sturecute of) tensors that is usually the full output of the
+    encoder, which will be used for the attention wrapper for the RNNCell.
+  - `finished`: boolean tensor telling whether each sequence in the batch is
+    finished.
+  - `outputs`: Instance of BasicDecoderOutput. Result of the decoding, at each
+    time step.
+  """
+
+  def __init__(self,
+               output_time_major=False,
+               impute_finished=False,
+               maximum_iterations=None,
+               parallel_iterations=32,
+               swap_memory=False,
+               **kwargs):
+    self.output_time_major = output_time_major
+    self.impute_finished = impute_finished
+    self.maximum_iterations = maximum_iterations
+    self.parallel_iterations = parallel_iterations
+    self.swap_memory = swap_memory
+    super(BaseDecoder, self).__init__(**kwargs)
+
+  def call(self, inputs, initial_state=None, **kwargs):
+    init_kwargs = kwargs
+    init_kwargs["initial_state"] = initial_state
+    return dynamic_decode(self,
+                          output_time_major=self.output_time_major,
+                          impute_finished=self.impute_finished,
+                          maximum_iterations=self.maximum_iterations,
+                          parallel_iterations=self.parallel_iterations,
+                          swap_memory=self.swap_memory,
+                          decoder_init_input=inputs,
+                          decoder_init_kwargs=init_kwargs)
+
+  @property
+  def batch_size(self):
+    """The batch size of input values."""
+    raise NotImplementedError
+
+  @property
+  def output_size(self):
+    """A (possibly nested tuple of...) integer[s] or `TensorShape` object[s]."""
+    raise NotImplementedError
+
+  @property
+  def output_dtype(self):
+    """A (possibly nested tuple of...) dtype[s]."""
+    raise NotImplementedError
+
+  def initialize(self, inputs, initial_state=None, **kwargs):
+    """Called before any decoding iterations.
+
+    This methods must compute initial input values and initial state.
+
+    Args:
+      inputs: (structure of) tensors that contains the input for the decoder. In
+        the normal case, its a tensor with shape [batch, timestep, embedding].
+      initial_state: (structure of) tensors that contains the initial state for
+        the RNNCell.
+      **kwargs: Other arguments that are passed in from layer.call() method. It
+        could contains item like input sequence_length, or masking for input.
+
+    Returns:
+      `(finished, initial_inputs, initial_state)`: initial values of
+      'finished' flags, inputs and state.
+    """
+    raise NotImplementedError
+
+  def step(self, time, inputs, state):
+    """Called per step of decoding (but only once for dynamic decoding).
+
+    Args:
+      time: Scalar `int32` tensor. Current step number.
+      inputs: RNNCell input (possibly nested tuple of) tensor[s] for this time
+        step.
+      state: RNNCell state (possibly nested tuple of) tensor[s] from previous
+        time step.
+
+    Returns:
+      `(outputs, next_state, next_inputs, finished)`: `outputs` is an object
+      containing the decoder output, `next_state` is a (structure of) state
+      tensors and TensorArrays, `next_inputs` is the tensor that should be used
+      as input for the next step, `finished` is a boolean tensor telling whether
+      the sequence is complete, for each sequence in the batch.
+    """
+    raise NotImplementedError
+
+  def finalize(self, outputs, final_state, sequence_lengths):
+    raise NotImplementedError
+
+  @property
+  def tracks_own_finished(self):
+    """Describes whether the Decoder keeps track of finished states.
+
+    Most decoders will emit a true/false `finished` value independently
+    at each time step.  In this case, the `dynamic_decode` function keeps track
+    of which batch entries are already finished, and performs a logical OR to
+    insert new batches to the finished set.
+
+    Some decoders, however, shuffle batches / beams between time steps and
+    `dynamic_decode` will mix up the finished state across these entries because
+    it does not track the reshuffle across time steps.  In this case, it is
+    up to the decoder to declare that it will keep track of its own finished
+    state by setting this property to `True`.
+
+    Returns:
+      Python bool.
+    """
+    return False
+
+  # TODO(scottzhu): Add build/get_config/from_config and other layer methods.
+
+
 def _create_zero_outputs(size, dtype, batch_size):
   """Create a zero outputs Tensor structure."""
   def _create(s, d):
@@ -149,7 +271,8 @@ def dynamic_decode(decoder,
                    maximum_iterations=None,
                    parallel_iterations=32,
                    swap_memory=False,
-                   scope=None):
+                   scope=None,
+                   **kwargs):
   """Perform dynamic decoding with `decoder`.
 
   Calls initialize() once and step() repeatedly on the Decoder object.
@@ -171,6 +294,9 @@ def dynamic_decode(decoder,
     parallel_iterations: Argument passed to `tf.while_loop`.
     swap_memory: Argument passed to `tf.while_loop`.
     scope: Optional variable scope to use.
+    **kwargs: dict, other keyword arguments for dynamic_decode. It might contain
+      arguments for `BaseDecoder` to initialize, which takes all tensor inputs
+      during call().
 
   Returns:
     `(final_outputs, final_state, final_sequence_lengths)`.
@@ -179,7 +305,7 @@ def dynamic_decode(decoder,
     TypeError: if `decoder` is not an instance of `Decoder`.
     ValueError: if `maximum_iterations` is provided but is not a scalar.
   """
-  if not isinstance(decoder, Decoder):
+  if not isinstance(decoder, (Decoder, BaseDecoder)):
     raise TypeError("Expected decoder to be type Decoder, but saw: %s" %
                     type(decoder))
 
@@ -204,7 +330,14 @@ def dynamic_decode(decoder,
       if maximum_iterations.get_shape().ndims != 0:
         raise ValueError("maximum_iterations must be a scalar")
 
-    initial_finished, initial_inputs, initial_state = decoder.initialize()
+    if isinstance(decoder, Decoder):
+      initial_finished, initial_inputs, initial_state = decoder.initialize()
+    else:
+      # For BaseDecoder that takes tensor inputs during call.
+      decoder_init_input = kwargs.pop("decoder_init_input", None)
+      decoder_init_kwargs = kwargs.pop("decoder_init_kwargs", {})
+      initial_finished, initial_inputs, initial_state = decoder.initialize(
+          decoder_init_input, **decoder_init_kwargs)
 
     zero_outputs = _create_zero_outputs(decoder.output_size,
                                         decoder.output_dtype,
@@ -222,7 +355,7 @@ def dynamic_decode(decoder,
     def _shape(batch_size, from_shape):
       if (not isinstance(from_shape, tensor_shape.TensorShape) or
           from_shape.ndims == 0):
-        return tensor_shape.TensorShape(None)
+        return None
       else:
         batch_size = tensor_util.constant_value(
             ops.convert_to_tensor(
diff --git a/tensorflow/contrib/seq2seq/python/ops/sampler.py b/tensorflow/contrib/seq2seq/python/ops/sampler.py
new file mode 100644
index 0000000000..9e3e48b3bc
--- /dev/null
+++ b/tensorflow/contrib/seq2seq/python/ops/sampler.py
@@ -0,0 +1,765 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""A library of sampler for use with SamplingDecoders."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import abc
+
+import six
+
+from tensorflow.contrib.seq2seq.python.ops import decoder
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import embedding_ops
+from tensorflow.python.ops import gen_array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import random_ops
+from tensorflow.python.ops import tensor_array_ops
+from tensorflow.python.util import nest
+
+__all__ = [
+    "Sampler",
+    "TrainingSampler",
+    "GreedyEmbeddingSampler",
+    "SampleEmbeddingSampler",
+    "CustomSampler",
+    "ScheduledEmbeddingTrainingSampler",
+    "ScheduledOutputTrainingSampler",
+    "InferenceSampler",
+]
+
+_transpose_batch_time = decoder._transpose_batch_time  # pylint: disable=protected-access
+
+
+@six.add_metaclass(abc.ABCMeta)
+class Sampler(object):
+  """Interface for implementing sampling in seq2seq decoders.
+
+  Sampler instances are used by `BasicDecoder`. The normal usage of a sampler is
+  like below:
+  sampler = Sampler(init_args)
+  (initial_finished, initial_inputs) = sampler.initialize(input_tensors)
+  for time_step in range(time):
+    cell_output, cell_state = cell.call(cell_input, previous_state)
+    sample_ids = sampler.sample(time_step, cell_output, cell_state)
+    (finished, next_inputs, next_state) = sampler.next_inputs(
+        time_step,cell_output, cell_state)
+
+  Note that all the tensor input should not be feed to Sampler as __init__()
+  parameters, instead, they should be feed by decoders via initialize().
+  """
+
+  @abc.abstractmethod
+  def initialize(self, inputs, **kwargs):
+    """initialize the sampler with the input tensors.
+
+    This method suppose to be only invoke once before the calling other methods
+    of the Sampler.
+
+    Args:
+      inputs: A (structure of) input tensors, it could be a nested tuple or a
+        single tensor.
+      **kwargs: Other kwargs for initialization. It could contain tensors like
+        mask for inputs, or non tensor parameter.
+
+    Returns:
+      `(initial_finished, initial_inputs)`.
+    """
+    pass
+
+  @abc.abstractmethod
+  def sample(self, time, outputs, state):
+    """Returns `sample_ids`."""
+    pass
+
+  @abc.abstractmethod
+  def next_inputs(self, time, outputs, state, sample_ids):
+    """Returns `(finished, next_inputs, next_state)`."""
+    pass
+
+  @abc.abstractproperty
+  def batch_size(self):
+    """Batch size of tensor returned by `sample`.
+
+    Returns a scalar int32 tensor. The return value might not available before
+    the invocation of initialize(), in this case, ValueError is raised.
+    """
+    raise NotImplementedError("batch_size has not been implemented")
+
+  @abc.abstractproperty
+  def sample_ids_shape(self):
+    """Shape of tensor returned by `sample`, excluding the batch dimension.
+
+    Returns a `TensorShape`. The return value might not available before the
+    invocation of initialize().
+    """
+    raise NotImplementedError("sample_ids_shape has not been implemented")
+
+  @abc.abstractproperty
+  def sample_ids_dtype(self):
+    """DType of tensor returned by `sample`.
+
+    Returns a DType. The return value might not available before the
+    invocation of initialize().
+    """
+    raise NotImplementedError("sample_ids_dtype has not been implemented")
+
+
+class CustomSampler(Sampler):
+  """Base abstract class that allows the user to customize sampling."""
+
+  def __init__(self,
+               initialize_fn,
+               sample_fn,
+               next_inputs_fn,
+               sample_ids_shape=None,
+               sample_ids_dtype=None):
+    """Initializer.
+
+    Args:
+      initialize_fn: callable that returns `(finished, next_inputs)` for the
+        first iteration.
+      sample_fn: callable that takes `(time, outputs, state)` and emits tensor
+        `sample_ids`.
+      next_inputs_fn: callable that takes `(time, outputs, state, sample_ids)`
+        and emits `(finished, next_inputs, next_state)`.
+      sample_ids_shape: Either a list of integers, or a 1-D Tensor of type
+        `int32`, the shape of each value in the `sample_ids` batch. Defaults to
+        a scalar.
+      sample_ids_dtype: The dtype of the `sample_ids` tensor. Defaults to int32.
+    """
+    self._initialize_fn = initialize_fn
+    self._sample_fn = sample_fn
+    self._next_inputs_fn = next_inputs_fn
+    self._batch_size = None
+    self._sample_ids_shape = tensor_shape.TensorShape(sample_ids_shape or [])
+    self._sample_ids_dtype = sample_ids_dtype or dtypes.int32
+
+  @property
+  def batch_size(self):
+    if self._batch_size is None:
+      raise ValueError("batch_size accessed before initialize was called")
+    return self._batch_size
+
+  @property
+  def sample_ids_shape(self):
+    return self._sample_ids_shape
+
+  @property
+  def sample_ids_dtype(self):
+    return self._sample_ids_dtype
+
+  def initialize(self, inputs, **kwargs):
+    (finished, next_inputs) = self._initialize_fn(inputs, **kwargs)
+    if self._batch_size is None:
+      self._batch_size = array_ops.size(finished)
+    return (finished, next_inputs)
+
+  def sample(self, time, outputs, state):
+    return self._sample_fn(time=time, outputs=outputs, state=state)
+
+  def next_inputs(self, time, outputs, state, sample_ids):
+    return self._next_inputs_fn(
+        time=time, outputs=outputs, state=state, sample_ids=sample_ids)
+
+
+class TrainingSampler(Sampler):
+  """A Sampler for use during training.
+
+  Only reads inputs.
+
+  Returned sample_ids are the argmax of the RNN output logits.
+  """
+
+  def __init__(self, time_major=False):
+    """Initializer.
+
+    Args:
+      time_major: Python bool.  Whether the tensors in `inputs` are time major.
+        If `False` (default), they are assumed to be batch major.
+
+    Raises:
+      ValueError: if `sequence_length` is not a 1D tensor.
+    """
+    self.time_major = time_major
+    self._batch_size = None
+
+  @property
+  def batch_size(self):
+    if self._batch_size is None:
+      raise ValueError("batch_size accessed before initialize was called")
+    return self._batch_size
+
+  @property
+  def sample_ids_shape(self):
+    return tensor_shape.TensorShape([])
+
+  @property
+  def sample_ids_dtype(self):
+    return dtypes.int32
+
+  def initialize(self, inputs, sequence_length=None):
+    """Initialize the TrainSampler.
+
+    Args:
+      inputs: A (structure of) input tensors.
+      sequence_length: An int32 vector tensor.
+
+    Returns:
+      (finished, next_inputs), a tuple of two items. The first item is a boolean
+        vector to indicate whether the item in the batch has finished. The
+        second item is the first slide of input data based on the timestep
+        dimension (usually the second dim of the input).
+    """
+    self.inputs = ops.convert_to_tensor(inputs, name="inputs")
+    if not self.time_major:
+      inputs = nest.map_structure(_transpose_batch_time, inputs)
+
+    self.input_tas = nest.map_structure(_unstack_ta, inputs)
+    if sequence_length is None:
+      raise ValueError("sequence_length is required for TrainingSampler")
+    self.sequence_length = ops.convert_to_tensor(
+        sequence_length, name="sequence_length")
+    if self.sequence_length.get_shape().ndims != 1:
+      raise ValueError(
+          "Expected sequence_length to be a vector, but received shape: %s" %
+          self._sequence_length.get_shape())
+
+    self.zero_inputs = nest.map_structure(
+        lambda inp: array_ops.zeros_like(inp[0, :]), inputs)
+
+    self._batch_size = array_ops.size(self.sequence_length)
+
+    finished = math_ops.equal(0, self.sequence_length)
+    all_finished = math_ops.reduce_all(finished)
+    next_inputs = control_flow_ops.cond(
+        all_finished,
+        lambda: self.zero_inputs,
+        lambda: nest.map_structure(lambda inp: inp.read(0), self.input_tas))
+    return (finished, next_inputs)
+
+  def sample(self, time, outputs, state):
+    del state
+    sample_ids = math_ops.cast(math_ops.argmax(outputs, axis=-1), dtypes.int32)
+    return sample_ids
+
+  def next_inputs(self, time, outputs, state, sample_ids):
+    del sample_ids
+    next_time = time + 1
+    finished = (next_time >= self.sequence_length)
+    all_finished = math_ops.reduce_all(finished)
+
+    def read_from_ta(inp):
+      return inp.read(next_time)
+
+    next_inputs = control_flow_ops.cond(
+        all_finished,
+        lambda: self.zero_inputs,
+        lambda: nest.map_structure(read_from_ta, self.input_tas))
+    return (finished, next_inputs, state)
+
+
+class ScheduledEmbeddingTrainingSampler(TrainingSampler):
+  """A training sampler that adds scheduled sampling.
+
+  Returns -1s for sample_ids where no sampling took place; valid sample id
+  values elsewhere.
+  """
+
+  def __init__(self,
+               sampling_probability,
+               embedding_fn=None,
+               time_major=False,
+               seed=None,
+               scheduling_seed=None):
+    """Initializer.
+
+    Args:
+      sampling_probability: A `float32` 0-D or 1-D tensor: the probability of
+        sampling categorically from the output ids instead of reading directly
+        from the inputs.
+      embedding_fn: A callable that takes a vector tensor of `ids` (argmax ids),
+         or the `params` argument for `embedding_lookup`.
+      time_major: Python bool. Whether the tensors in `inputs` are time major.
+        If `False` (default), they are assumed to be batch major.
+      seed: The sampling seed.
+      scheduling_seed: The schedule decision rule sampling seed.
+
+    Raises:
+      ValueError: if `sampling_probability` is not a scalar or vector.
+    """
+    if callable(embedding_fn) or embedding_fn is None:
+      self.embedding_fn = embedding_fn
+    else:
+      raise ValueError("embedding_fn is expected to be callable, got %s"
+                       % type(embedding_fn))
+    self.sampling_probability = ops.convert_to_tensor(
+        sampling_probability, name="sampling_probability")
+    if self.sampling_probability.get_shape().ndims not in (0, 1):
+      raise ValueError(
+          "sampling_probability must be either a scalar or a vector. "
+          "saw shape: %s" % (self.sampling_probability.get_shape()))
+    self.seed = seed
+    self.scheduling_seed = scheduling_seed
+    super(ScheduledEmbeddingTrainingSampler,
+          self).__init__(time_major=time_major)
+
+  def initialize(self, inputs, sequence_length=None, embedding=None):
+    if self.embedding_fn is None:
+      if embedding is None:
+        raise ValueError("embedding is required as a keyword argument for "
+                         "ScheduledEmbeddingTrainingSampler")
+      self.embedding_fn = (
+          lambda ids: embedding_ops.embedding_lookup(embedding, ids))
+    return super(ScheduledEmbeddingTrainingSampler, self).initialize(
+        inputs, sequence_length=sequence_length)
+
+  def sample(self, time, outputs, state):
+    del state
+    # Return -1s where we did not sample, and sample_ids elsewhere
+    select_sample = bernoulli_sample(
+        probs=self.sampling_probability,
+        dtype=dtypes.bool,
+        sample_shape=self.batch_size,
+        seed=self.scheduling_seed)
+    return array_ops.where(select_sample,
+                           categorical_sample(logits=outputs, seed=self.seed),
+                           gen_array_ops.fill([self.batch_size], -1))
+
+  def next_inputs(self, time, outputs, state, sample_ids):
+    (finished, base_next_inputs, state) = (
+        super(ScheduledEmbeddingTrainingSampler, self).next_inputs(
+            time=time, outputs=outputs, state=state, sample_ids=sample_ids))
+
+    def maybe_sample():
+      """Perform scheduled sampling."""
+      where_sampling = math_ops.cast(
+          array_ops.where(sample_ids > -1), dtypes.int32)
+      where_not_sampling = math_ops.cast(
+          array_ops.where(sample_ids <= -1), dtypes.int32)
+      sample_ids_sampling = array_ops.gather_nd(sample_ids, where_sampling)
+      inputs_not_sampling = array_ops.gather_nd(base_next_inputs,
+                                                where_not_sampling)
+      sampled_next_inputs = self.embedding_fn(sample_ids_sampling)
+      base_shape = array_ops.shape(base_next_inputs)
+      return (array_ops.scatter_nd(
+          indices=where_sampling, updates=sampled_next_inputs, shape=base_shape)
+              + array_ops.scatter_nd(
+                  indices=where_not_sampling,
+                  updates=inputs_not_sampling,
+                  shape=base_shape))
+
+    all_finished = math_ops.reduce_all(finished)
+    next_inputs = control_flow_ops.cond(all_finished, lambda: base_next_inputs,
+                                        maybe_sample)
+    return (finished, next_inputs, state)
+
+
+class ScheduledOutputTrainingSampler(TrainingSampler):
+  """A training sampler that adds scheduled sampling directly to outputs.
+
+  Returns False for sample_ids where no sampling took place; True elsewhere.
+  """
+
+  def __init__(self,
+               sampling_probability,
+               time_major=False,
+               seed=None,
+               next_inputs_fn=None):
+    """Initializer.
+
+    Args:
+      sampling_probability: A `float32` scalar tensor: the probability of
+        sampling from the outputs instead of reading directly from the inputs.
+      time_major: Python bool. Whether the tensors in `inputs` are time major.
+        If `False` (default), they are assumed to be batch major.
+      seed: The sampling seed.
+      next_inputs_fn: (Optional) callable to apply to the RNN outputs to create
+        the next input when sampling. If `None` (default), the RNN outputs will
+        be used as the next inputs.
+
+    Raises:
+      ValueError: if `sampling_probability` is not a scalar or vector.
+    """
+    self.sampling_probability = ops.convert_to_tensor(
+        sampling_probability, name="sampling_probability")
+    if self.sampling_probability.get_shape().ndims not in (0, 1):
+      raise ValueError(
+          "sampling_probability must be either a scalar or a vector. "
+          "saw shape: %s" % (self._sampling_probability.get_shape()))
+
+    self.seed = seed
+    self.next_inputs_fn = next_inputs_fn
+
+    super(ScheduledOutputTrainingSampler, self).__init__(time_major=time_major)
+
+  def initialize(self, inputs, sequence_length=None, auxiliary_inputs=None):
+    if auxiliary_inputs is None:
+      maybe_concatenated_inputs = inputs
+    else:
+      inputs = ops.convert_to_tensor(inputs)
+      auxiliary_inputs = ops.convert_to_tensor(auxiliary_inputs)
+      maybe_concatenated_inputs = nest.map_structure(
+          lambda x, y: array_ops.concat((x, y), -1), inputs, auxiliary_inputs)
+      if not self.time_major:
+        auxiliary_inputs = nest.map_structure(_transpose_batch_time,
+                                              auxiliary_inputs)
+    if auxiliary_inputs is not None:
+      self._auxiliary_input_tas = nest.map_structure(_unstack_ta,
+                                                     auxiliary_inputs)
+    else:
+      self._auxiliary_input_tas = None
+
+    return super(ScheduledOutputTrainingSampler, self).initialize(
+        maybe_concatenated_inputs, sequence_length=sequence_length)
+
+  def sample(self, time, outputs, state):
+    del state
+    return bernoulli_sample(
+        probs=self.sampling_probability,
+        sample_shape=self.batch_size,
+        seed=self.seed)
+
+  def next_inputs(self, time, outputs, state, sample_ids):
+    (finished, base_next_inputs, state) = (
+        super(ScheduledOutputTrainingSampler, self).next_inputs(
+            time=time, outputs=outputs, state=state, sample_ids=sample_ids))
+    sample_ids = math_ops.cast(sample_ids, dtypes.bool)
+
+    def maybe_sample():
+      """Perform scheduled sampling."""
+
+      def maybe_concatenate_auxiliary_inputs(outputs_, indices=None):
+        """Concatenate outputs with auxiliary inputs, if they exist."""
+        if self._auxiliary_input_tas is None:
+          return outputs_
+
+        next_time = time + 1
+        auxiliary_inputs = nest.map_structure(lambda ta: ta.read(next_time),
+                                              self._auxiliary_input_tas)
+        if indices is not None:
+          auxiliary_inputs = array_ops.gather_nd(auxiliary_inputs, indices)
+        return nest.map_structure(lambda x, y: array_ops.concat((x, y), -1),
+                                  outputs_, auxiliary_inputs)
+
+      if self.next_inputs_fn is None:
+        return array_ops.where(sample_ids,
+                               maybe_concatenate_auxiliary_inputs(outputs),
+                               base_next_inputs)
+
+      where_sampling = math_ops.cast(array_ops.where(sample_ids), dtypes.int32)
+      where_not_sampling = math_ops.cast(
+          array_ops.where(math_ops.logical_not(sample_ids)), dtypes.int32)
+      outputs_sampling = array_ops.gather_nd(outputs, where_sampling)
+      inputs_not_sampling = array_ops.gather_nd(base_next_inputs,
+                                                where_not_sampling)
+      sampled_next_inputs = maybe_concatenate_auxiliary_inputs(
+          self.next_inputs_fn(outputs_sampling), where_sampling)
+
+      base_shape = array_ops.shape(base_next_inputs)
+      return (array_ops.scatter_nd(
+          indices=where_sampling, updates=sampled_next_inputs, shape=base_shape)
+              + array_ops.scatter_nd(
+                  indices=where_not_sampling,
+                  updates=inputs_not_sampling,
+                  shape=base_shape))
+
+    all_finished = math_ops.reduce_all(finished)
+    no_samples = math_ops.logical_not(math_ops.reduce_any(sample_ids))
+    next_inputs = control_flow_ops.cond(
+        math_ops.logical_or(all_finished, no_samples), lambda: base_next_inputs,
+        maybe_sample)
+    return (finished, next_inputs, state)
+
+
+class GreedyEmbeddingSampler(Sampler):
+  """A sampler for use during inference.
+
+  Uses the argmax of the output (treated as logits) and passes the
+  result through an embedding layer to get the next input.
+  """
+
+  def __init__(self, embedding_fn=None):
+    """Initializer.
+
+    Args:
+      embedding_fn: A optional callable that takes a vector tensor of `ids`
+        (argmax ids), or the `params` argument for `embedding_lookup`. The
+        returned tensor will be passed to the decoder input. Default to use
+        `embedding_ops.embedding_lookup`.
+    """
+    if embedding_fn is None or callable(embedding_fn):
+      self.embedding_fn = embedding_fn
+    else:
+      raise ValueError("embedding_fn is expected to be a callable, got %s" %
+                       type(embedding_fn))
+    self._batch_size = None
+
+  @property
+  def batch_size(self):
+    if self._batch_size is None:
+      raise ValueError("batch_size accessed before initialize was called")
+    return self._batch_size
+
+  @property
+  def sample_ids_shape(self):
+    return tensor_shape.TensorShape([])
+
+  @property
+  def sample_ids_dtype(self):
+    return dtypes.int32
+
+  def initialize(self, embedding, start_tokens=None, end_token=None):
+    """Initialize the GreedyEmbeddingSampler.
+
+    Args:
+      embedding: tensor that contains embedding states matrix. It will be used
+        to generate generate outputs with start_tokens and end_tokens. The
+        embedding will be ignored if the embedding_fn has been provided at
+        __init__().
+      start_tokens: `int32` vector shaped `[batch_size]`, the start tokens.
+      end_token: `int32` scalar, the token that marks end of decoding.
+
+    Returns:
+      Tuple of two items: `(finished, self.start_inputs)`.
+    Raises:
+      ValueError: if `start_tokens` is not a 1D tensor or `end_token` is not a
+        scalar.
+    """
+    if self.embedding_fn is None:
+      self.embedding_fn = (
+          lambda ids: embedding_ops.embedding_lookup(embedding, ids))
+
+    self.start_tokens = ops.convert_to_tensor(
+        start_tokens, dtype=dtypes.int32, name="start_tokens")
+    self.end_token = ops.convert_to_tensor(
+        end_token, dtype=dtypes.int32, name="end_token")
+    if self.start_tokens.get_shape().ndims != 1:
+      raise ValueError("start_tokens must be a vector")
+    self._batch_size = array_ops.size(start_tokens)
+    if self.end_token.get_shape().ndims != 0:
+      raise ValueError("end_token must be a scalar")
+    self.start_inputs = self.embedding_fn(self.start_tokens)
+
+    finished = array_ops.tile([False], [self._batch_size])
+    return (finished, self.start_inputs)
+
+  def sample(self, time, outputs, state):
+    """sample for GreedyEmbeddingHelper."""
+    del time, state  # unused by sample_fn
+    # Outputs are logits, use argmax to get the most probable id
+    if not isinstance(outputs, ops.Tensor):
+      raise TypeError(
+          "Expected outputs to be a single Tensor, got: %s" % type(outputs))
+    sample_ids = math_ops.argmax(outputs, axis=-1, output_type=dtypes.int32)
+    return sample_ids
+
+  def next_inputs(self, time, outputs, state, sample_ids):
+    """next_inputs_fn for GreedyEmbeddingHelper."""
+    del time, outputs  # unused by next_inputs_fn
+    finished = math_ops.equal(sample_ids, self.end_token)
+    all_finished = math_ops.reduce_all(finished)
+    next_inputs = control_flow_ops.cond(
+        all_finished,
+        # If we're finished, the next_inputs value doesn't matter
+        lambda: self.start_inputs,
+        lambda: self.embedding_fn(sample_ids))
+    return (finished, next_inputs, state)
+
+
+class SampleEmbeddingSampler(GreedyEmbeddingSampler):
+  """A sampler for use during inference.
+
+  Uses sampling (from a distribution) instead of argmax and passes the
+  result through an embedding layer to get the next input.
+  """
+
+  def __init__(self, embedding_fn=None, softmax_temperature=None, seed=None):
+    """Initializer.
+
+    Args:
+      embedding_fn: (Optional) A callable that takes a vector tensor of `ids`
+        (argmax ids), or the `params` argument for `embedding_lookup`. The
+        returned tensor will be passed to the decoder input.
+      softmax_temperature: (Optional) `float32` scalar, value to divide the
+        logits by before computing the softmax. Larger values (above 1.0) result
+        in more random samples, while smaller values push the sampling
+        distribution towards the argmax. Must be strictly greater than 0.
+        Defaults to 1.0.
+      seed: (Optional) The sampling seed.
+
+    Raises:
+      ValueError: if `start_tokens` is not a 1D tensor or `end_token` is not a
+        scalar.
+    """
+    super(SampleEmbeddingSampler, self).__init__(embedding_fn)
+    self.softmax_temperature = softmax_temperature
+    self.seed = seed
+
+  def sample(self, time, outputs, state):
+    """sample for SampleEmbeddingHelper."""
+    del time, state  # unused by sample_fn
+    # Outputs are logits, we sample instead of argmax (greedy).
+    if not isinstance(outputs, ops.Tensor):
+      raise TypeError(
+          "Expected outputs to be a single Tensor, got: %s" % type(outputs))
+    if self.softmax_temperature is None:
+      logits = outputs
+    else:
+      logits = outputs / self.softmax_temperature
+
+    return categorical_sample(logits=logits, seed=self.seed)
+
+
+class InferenceSampler(Sampler):
+  """A helper to use during inference with a custom sampling function."""
+
+  def __init__(self,
+               sample_fn,
+               sample_shape,
+               sample_dtype,
+               end_fn,
+               next_inputs_fn=None):
+    """Initializer.
+
+    Args:
+      sample_fn: A callable that takes `outputs` and emits tensor `sample_ids`.
+      sample_shape: Either a list of integers, or a 1-D Tensor of type `int32`,
+        the shape of the each sample in the batch returned by `sample_fn`.
+      sample_dtype: the dtype of the sample returned by `sample_fn`.
+      end_fn: A callable that takes `sample_ids` and emits a `bool` vector
+        shaped `[batch_size]` indicating whether each sample is an end token.
+      next_inputs_fn: (Optional) A callable that takes `sample_ids` and returns
+        the next batch of inputs. If not provided, `sample_ids` is used as the
+        next batch of inputs.
+    """
+    self.sample_fn = sample_fn
+    self.sample_shape = tensor_shape.TensorShape(sample_shape)
+    self.sample_dtype = sample_dtype
+    self.end_fn = end_fn
+    self.next_inputs_fn = next_inputs_fn
+    self._batch_size = None
+
+  @property
+  def batch_size(self):
+    if self._batch_size is None:
+      raise ValueError("batch_size accessed before initialize was called")
+    return self._batch_size
+
+  @property
+  def sample_ids_shape(self):
+    return self.sample_shape
+
+  @property
+  def sample_ids_dtype(self):
+    return self.sample_dtype
+
+  def initialize(self, start_inputs):
+    self.start_inputs = ops.convert_to_tensor(start_inputs, name="start_inputs")
+    self._batch_size = array_ops.shape(start_inputs)[0]
+    finished = array_ops.tile([False], [self._batch_size])
+    return (finished, self.start_inputs)
+
+  def sample(self, time, outputs, state):
+    del time, state  # unused by sample
+    return self.sample_fn(outputs)
+
+  def next_inputs(self, time, outputs, state, sample_ids):
+    del time, outputs  # unused by next_inputs
+    if self.next_inputs_fn is None:
+      next_inputs = sample_ids
+    else:
+      next_inputs = self.next_inputs_fn(sample_ids)
+    finished = self.end_fn(sample_ids)
+    return (finished, next_inputs, state)
+
+
+# The following sample functions (_call_sampler, bernoulli_sample,
+# categorical_sample) mimic TensorFlow Probability distribution semantics.
+def _call_sampler(sample_n_fn, sample_shape, name=None):
+  """Reshapes vector of samples."""
+  with ops.name_scope(name, "call_sampler", values=[sample_shape]):
+    sample_shape = ops.convert_to_tensor(
+        sample_shape, dtype=dtypes.int32, name="sample_shape")
+    # Ensure sample_shape is a vector (vs just a scalar).
+    pad = math_ops.cast(
+        math_ops.equal(array_ops.rank(sample_shape), 0), dtypes.int32)
+    sample_shape = array_ops.reshape(
+        sample_shape,
+        array_ops.pad(
+            array_ops.shape(sample_shape),
+            paddings=[[pad, 0]],
+            constant_values=1))
+    samples = sample_n_fn(math_ops.reduce_prod(sample_shape))
+    batch_event_shape = array_ops.shape(samples)[1:]
+    final_shape = array_ops.concat([sample_shape, batch_event_shape], 0)
+    return array_ops.reshape(samples, final_shape)
+
+
+def bernoulli_sample(probs=None,
+                     logits=None,
+                     dtype=dtypes.int32,
+                     sample_shape=(),
+                     seed=None):
+  """Samples from Bernoulli distribution."""
+  if probs is None:
+    probs = math_ops.sigmoid(logits, name="probs")
+  else:
+    probs = ops.convert_to_tensor(probs, name="probs")
+  batch_shape_tensor = array_ops.shape(probs)
+
+  def _sample_n(n):
+    """Sample vector of Bernoullis."""
+    new_shape = array_ops.concat([[n], batch_shape_tensor], 0)
+    uniform = random_ops.random_uniform(new_shape, seed=seed, dtype=probs.dtype)
+    return math_ops.cast(math_ops.less(uniform, probs), dtype)
+
+  return _call_sampler(_sample_n, sample_shape)
+
+
+def categorical_sample(logits, dtype=dtypes.int32, sample_shape=(), seed=None):
+  """Samples from categorical distribution."""
+  logits = ops.convert_to_tensor(logits, name="logits")
+  event_size = array_ops.shape(logits)[-1]
+  batch_shape_tensor = array_ops.shape(logits)[:-1]
+
+  def _sample_n(n):
+    """Sample vector of categoricals."""
+    if logits.shape.ndims == 2:
+      logits_2d = logits
+    else:
+      logits_2d = array_ops.reshape(logits, [-1, event_size])
+    sample_dtype = dtypes.int64 if logits.dtype.size > 4 else dtypes.int32
+    draws = random_ops.multinomial(
+        logits_2d, n, seed=seed, output_dtype=sample_dtype)
+    draws = array_ops.reshape(
+        array_ops.transpose(draws),
+        array_ops.concat([[n], batch_shape_tensor], 0))
+    return math_ops.cast(draws, dtype)
+
+  return _call_sampler(_sample_n, sample_shape)
+
+
+def _unstack_ta(inp):
+  return tensor_array_ops.TensorArray(
+      dtype=inp.dtype,
+      size=array_ops.shape(inp)[0],
+      element_shape=inp.get_shape()[1:]).unstack(inp)
-- 
GitLab


From 6161367d8942561f2eeb640b3798cfd8397ea35c Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Tue, 29 Jan 2019 18:18:50 -0800
Subject: [PATCH 1556/2345] Blacklist XRT ops in TF Eager's small Tensor pin to
 host optimization.

PiperOrigin-RevId: 231513080
---
 tensorflow/core/common_runtime/eager/BUILD      | 1 +
 tensorflow/core/common_runtime/eager/execute.cc | 6 +++++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD
index 1df55d9c88..6f2680bf8f 100644
--- a/tensorflow/core/common_runtime/eager/BUILD
+++ b/tensorflow/core/common_runtime/eager/BUILD
@@ -214,6 +214,7 @@ cc_library(
         ":eager_operation",
         ":kernel_and_device",
         ":tensor_handle",
+        "@com_google_absl//absl/strings",
     ] + select({
         "//tensorflow:android": [
             "//tensorflow/core:android_tensorflow_lib_lite",
diff --git a/tensorflow/core/common_runtime/eager/execute.cc b/tensorflow/core/common_runtime/eager/execute.cc
index 423f71f36f..53e9ba232a 100644
--- a/tensorflow/core/common_runtime/eager/execute.cc
+++ b/tensorflow/core/common_runtime/eager/execute.cc
@@ -17,6 +17,7 @@ limitations under the License.
 
 #include <vector>
 
+#include "absl/strings/match.h"
 #include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/common_runtime/device_set.h"
 #include "tensorflow/core/common_runtime/eager/context.h"
@@ -750,7 +751,10 @@ bool IsPinnableOp(const string& op_type) {
       "StatelessRandomNormal",
   });
 
-  return unpinnable_ops->find(op_type) == unpinnable_ops->end();
+  // XRT ops refer to per-device handles that are not safe to move between
+  // devices.
+  return unpinnable_ops->find(op_type) == unpinnable_ops->end() &&
+         !absl::StartsWith(op_type, "XRT");
 }
 
 // The Op device may be updated if:
-- 
GitLab


From 2218088d09277bac875b855ba6ee590d79793f31 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Tue, 29 Jan 2019 19:04:43 -0800
Subject: [PATCH 1557/2345] Update var from grads inside merge call and
 strategy.extended.update

PiperOrigin-RevId: 231518119
---
 tensorflow/contrib/distribute/python/BUILD    |   2 +
 .../contrib/distribute/python/combinations.py |   6 +
 .../python/keras_optimizer_v2_test.py         | 148 +++++++----------
 .../contrib/distribute/python/keras_test.py   |  11 +-
 .../python/keras/optimizer_v2/optimizer_v2.py | 157 ++++++++++--------
 .../keras/optimizer_v2/optimizer_v2_test.py   |  29 ++++
 6 files changed, 184 insertions(+), 169 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 1d7123b3e3..c67af0e46c 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -179,6 +179,7 @@ py_library(
     deps = [
         ":mirrored_strategy",
         ":one_device_strategy",
+        ":parameter_server_strategy",
         ":tpu_strategy",
         "//tensorflow/contrib/cluster_resolver:cluster_resolver_pip",
         "//tensorflow/contrib/optimizer_v2:training",
@@ -404,6 +405,7 @@ cuda_py_test(
     additional_deps = [
         ":keras_test_lib",
     ],
+    shard_count = 4,
     tags = [
         "multi_and_single_gpu",
         "no_oss",  # http://b/119349471
diff --git a/tensorflow/contrib/distribute/python/combinations.py b/tensorflow/contrib/distribute/python/combinations.py
index 6122e71632..104c63f6b0 100644
--- a/tensorflow/contrib/distribute/python/combinations.py
+++ b/tensorflow/contrib/distribute/python/combinations.py
@@ -49,6 +49,7 @@ import six
 from tensorflow.contrib import cluster_resolver
 from tensorflow.contrib.distribute.python import mirrored_strategy as mirrored_lib
 from tensorflow.contrib.distribute.python import one_device_strategy as one_device_lib
+from tensorflow.contrib.distribute.python import parameter_server_strategy
 from tensorflow.contrib.distribute.python import tpu_strategy as tpu_lib
 from tensorflow.contrib.optimizer_v2 import adagrad as adagrad_v2
 from tensorflow.contrib.optimizer_v2 import adam as adam_v2
@@ -396,6 +397,11 @@ core_mirrored_strategy_with_two_gpus = NamedDistribution(
     "CoreMirrored2GPUs",
     lambda: mirrored_lib.CoreMirroredStrategy(["/gpu:0", "/gpu:1"]),
     required_gpus=2)
+parameter_server_strategy_with_two_gpus = NamedDistribution(
+    "ParameterServer2GPUs",
+    lambda: parameter_server_strategy.ParameterServerStrategy(
+        num_gpus_per_worker=2),
+    required_gpus=2)
 
 
 gradient_descent_optimizer_v1_fn = NamedObject(
diff --git a/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py b/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
index 5349794334..952b11932b 100644
--- a/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
+++ b/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
@@ -24,6 +24,7 @@ import numpy as np
 from tensorflow.contrib.distribute.python import combinations
 from tensorflow.python import keras
 from tensorflow.python.distribute import distribution_strategy_context as ds_context
+from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -44,108 +45,71 @@ def get_model():
 
 class MirroredStrategyOptimizerV2Test(test.TestCase, parameterized.TestCase):
 
-  @combinations.generate(combinations.combine(
-      distribution=[
-          combinations.mirrored_strategy_with_gpu_and_cpu,
-          combinations.core_mirrored_strategy_with_gpu_and_cpu],
-      mode=['graph']))
+  @combinations.generate(
+      combinations.combine(
+          distribution=[
+              combinations.core_mirrored_strategy_with_gpu_and_cpu,
+              combinations.core_mirrored_strategy_with_two_gpus,
+              combinations.parameter_server_strategy_with_two_gpus,
+          ],
+          mode=['graph', 'eager']))
   def testKerasOptimizerWithUnequalInput(self, distribution):
-    def create_fn():
+    with distribution.scope():
       var = variables.Variable(
           2.0, name='var', aggregation=variable_scope.VariableAggregation.SUM)
-      # grad for cpu is 1, grad for gpu is 2, avg grad is 1.5.
-      def loss():
-        return math_ops.cast(_replica_id() + 1, dtype=dtypes.float32) * var
-
       optimizer = adam.Adam(learning_rate=0.01, beta_1=0.2, beta_2=0.2)
-      train_op = optimizer.minimize(loss, var_list=[var])
-      m = optimizer.get_slot(var, 'm')
-      v = optimizer.get_slot(var, 'v')
-      return (var, m, v, train_op, optimizer.iterations)
+      all_vars = []
 
-    devices = ['/device:GPU:0', '/device:CPU:0']
-    with distribution.scope():
-      (var, m, v, op,
-       counter) = distribution.extended.call_for_each_replica(create_fn)
+      def model_fn():
+
+        def loss_fn():
+          replica_id = _replica_id()
+          return math_ops.cast(replica_id + 1, dtype=dtypes.float32) * var
+
+        train_op = optimizer.minimize(loss_fn, var_list=[var])
+
+        return train_op, optimizer
+
+      def train_fn():
+        train_op, optimizer = distribution.extended.call_for_each_replica(
+            model_fn)
+        if not all_vars:
+          all_vars.append(var)
+          all_vars.append(optimizer.get_slot(var, 'm'))
+          all_vars.append(optimizer.get_slot(var, 'v'))
+        return distribution.group(train_op)
+
+      if not context.executing_eagerly():
+        with self.cached_session() as sess:
+          train_fn = sess.make_callable(train_fn())
       self.evaluate(variables.global_variables_initializer())
-      var_val = [2.0, 2.0, 2.0]
-      self.assertAllClose(
-          var_val,
-          self.evaluate(
-              [distribution.extended.read_var(var),
-               var.get(devices[0]),
-               var.get(devices[1])]))
-      self.assertAllClose([0, 0, 0],
-                          self.evaluate([
-                              distribution.extended.read_var(counter),
-                              counter.get(devices[0]),
-                              counter.get(devices[1])
-                          ]))
-
-      train_op = distribution.unwrap(op)
-      self.evaluate(train_op)
-      # m(1) = beta1 * m(0) + (1-beta1) * grad = 0.2 * 0 + 0.8 * (1 + 2) / 2
-      m_val = [1.2, 1.2, 1.2]
-      # assert slot variables in both replicas are the same.
-      self.assertAllClose(
-          m_val,
-          self.evaluate(
-              [distribution.extended.read_var(m),
-               m.get(devices[0]),
-               m.get(devices[1])]))
-      # v(1) = beta2 * v(0) + (1-beta2) * grad^2 = 0.2 * 0 + 0.8 * 2.25
-      v_val = [1.8, 1.8, 1.8]
-      self.assertAllClose(
-          v_val,
-          self.evaluate(
-              [distribution.extended.read_var(v),
-               v.get(devices[0]),
-               v.get(devices[1])]))
+
+      # first step.
+      train_fn()
       # var(1) = var(0) - lr * m(1) * sqrt(1 - beta2) / sqrt(v(1)) / (1 - beta1)
       #        = 2.0 - 0.01 * 1.2 * sqrt(0.8) / sqrt(1.8) / 0.8
-      var_val = [1.99, 1.99, 1.99]
-      self.assertAllClose(
-          var_val,
-          self.evaluate(
-              [distribution.extended.read_var(var),
-               var.get(devices[0]),
-               var.get(devices[1])]))
-      self.assertAllClose([1, 1, 1],
-                          self.evaluate([
-                              distribution.extended.read_var(counter),
-                              counter.get(devices[0]),
-                              counter.get(devices[1])
-                          ]))
-
-      self.evaluate(train_op)
+      self.assertAllClose(1.99, self.evaluate(all_vars[0]))
+      # m(1) = beta1 * m(0) + (1-beta1) * grad = 0.2 * 0 + 0.8 * (1 + 2) / 2
+      self.assertAllClose(1.2, self.evaluate(all_vars[1]))
+      # v(1) = beta2 * v(0) + (1-beta2) * grad^2 = 0.2 * 0 + 0.8 * 2.25
+      self.assertAllClose(1.8, self.evaluate(all_vars[2]))
+
+      # second step.
+      train_fn()
+      # var(1) = var(0) - lr * 2 = 1.98
+      self.assertAllClose(1.98, self.evaluate(all_vars[0]))
       # m(2) = beta1 * m(1) + (1-beta1) * grad = 0.2 * 1.2 + 0.8 * 1.5
-      m_val = [1.44, 1.44, 1.44]
-      self.assertAllClose(
-          m_val,
-          self.evaluate(
-              [distribution.extended.read_var(m),
-               m.get(devices[0]),
-               m.get(devices[1])]))
+      self.assertAllClose(1.44, self.evaluate(all_vars[1]))
       # v(2) = beta2 * v(1) + (1-beta2) * grad^2 = 0.2 * 1.8 + 0.8 * 2.25
-      v_val = [2.16, 2.16, 2.16]
-      self.assertAllClose(
-          v_val,
-          self.evaluate(
-              [distribution.extended.read_var(v),
-               v.get(devices[0]),
-               v.get(devices[1])]))
-      self.assertAllClose([2, 2, 2],
-                          self.evaluate([
-                              distribution.extended.read_var(counter),
-                              counter.get(devices[0]),
-                              counter.get(devices[1])
-                          ]))
-
-  @combinations.generate(combinations.combine(
-      distribution=[
-          combinations.mirrored_strategy_with_gpu_and_cpu,
-          combinations.core_mirrored_strategy_with_gpu_and_cpu],
-      mode=['graph']))
+      self.assertAllClose(2.16, self.evaluate(all_vars[2]))
+
+  @combinations.generate(
+      combinations.combine(
+          distribution=[
+              combinations.core_mirrored_strategy_with_gpu_and_cpu,
+              combinations.parameter_server_strategy_with_two_gpus,
+          ],
+          mode=['graph', 'eager']))
   def testOptimizerWithKerasModelAndNumpyArrays(self, distribution):
 
     with self.cached_session():
diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 2da71893dc..54d4ff5bdc 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -272,18 +272,15 @@ def all_strategy_combinations_minus_default():
 
 
 def strategy_and_optimizer_combinations():
-  # TODO(b/122372746): Uncomment optimizers after they pass tests.
   return combinations.times(
       all_strategy_combinations(),
       combinations.combine(optimizer=[
-          combinations.adagrad_optimizer_v1_fn,
-          # combinations.adagrad_optimizer_keras_v2_fn,
-          combinations.adam_optimizer_v1_fn,
+          combinations.adagrad_optimizer_v1_fn, combinations
+          .adagrad_optimizer_keras_v2_fn, combinations.adam_optimizer_v1_fn,
           combinations.adam_optimizer_keras_v2_fn,
           combinations.gradient_descent_optimizer_v1_fn,
-          combinations.gradient_descent_optimizer_keras_v2_fn,
-          combinations.rmsprop_optimizer_v1_fn,
-          # combinations.rmsprop_optimizer_keras_v2_fn
+          combinations.gradient_descent_optimizer_keras_v2_fn, combinations
+          .rmsprop_optimizer_v1_fn, combinations.rmsprop_optimizer_keras_v2_fn
       ]))
 
 
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
index 6cd6cf0a8d..9a62d16855 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
@@ -75,38 +75,31 @@ class OptimizerV2(checkpointable.Checkpointable):
 
   This class defines the API to add Ops to train a model.  You never use this
   class directly, but instead instantiate one of its subclasses such as
-  `GradientDescentOptimizer`, `AdagradOptimizer`, or `MomentumOptimizer`.
+  `tf.keras.optimizers.SGD`, `tf.keras.optimizers.Adam`.
 
   ### Usage
 
   ```python
   # Create an optimizer with the desired parameters.
-  opt = GradientDescentOptimizer(learning_rate=0.1)
-  # Add Ops to the graph to minimize a cost by updating a list of variables.
-  # "cost" is a Tensor, and the list of variables contains tf.Variable
-  # objects.
-  opt_op = opt.minimize(cost, var_list=<list of variables>)
-  ```
-
-  In the training program you will just have to run the returned Op.
-
-  ```python
-  # Execute opt_op to do one step of training:
+  opt = tf.keras.optimizers.SGD(learning_rate=0.1)
+  # `loss` is a callable that takes no argument and returns the value
+  # to minimize.
+  loss = lambda: 3 * var1 * var1 + 2 * var2 * var2
+  # In graph mode, returns op that minimizes the loss by updating the listed
+  # variables.
+  opt_op = opt.minimize(loss, var_list=[var1, var2])
   opt_op.run()
+  # In eager mode, simply call minimize to update the list of variables.
+  opt.minimize(loss, var_list=[var1, var2])
   ```
 
-  ### Thread Compatibility
-
-  The entire optimizer is currently thread compatible, not thread-safe. The user
-  needs to perform synchronization if necessary.
-
   ### Processing gradients before applying them.
 
   Calling `minimize()` takes care of both computing the gradients and
   applying them to the variables.  If you want to process the gradients
   before applying them you can instead use the optimizer in three steps:
 
-  1.  Compute the gradients with `compute_gradients()`.
+  1.  Compute the gradients with `tf.GradientTape`.
   2.  Process the gradients as you wish.
   3.  Apply the processed gradients with `apply_gradients()`.
 
@@ -114,10 +107,15 @@ class OptimizerV2(checkpointable.Checkpointable):
 
   ```python
   # Create an optimizer.
-  opt = GradientDescentOptimizer(learning_rate=0.1)
+  opt = tf.keras.optimizers.SGD(learning_rate=0.1)
 
   # Compute the gradients for a list of variables.
-  grads_and_vars = opt.compute_gradients(loss, <list of variables>)
+  with tf.GradientTape() as tape:
+    loss = <call_loss_function>
+  vars = <list_of_variables>
+  grads = tape.gradient(loss, vars)
+  processed_grads = [process_gradient(g) for g in grads]
+  grads_and_vars = zip(processed_grads, var_list)
 
   # grads_and_vars is a list of tuples (gradient, variable).  Do whatever you
   # need to the 'gradient' part, for example cap them, etc.
@@ -127,13 +125,24 @@ class OptimizerV2(checkpointable.Checkpointable):
   opt.apply_gradients(capped_grads_and_vars)
   ```
 
+  ### Variable Constraint
+  All Keras optimizers respect variable constraints. If constraint function is
+  passed to any variable, the constraint will be applied to the variable after
+  the gradient has been applied to the variable.
+  Important: If gradient is sparse tensor, variable constraint is not supported.
+
+  ### Thread Compatibility
+
+  The entire optimizer is currently thread compatible, not thread-safe. The user
+  needs to perform synchronization if necessary.
+
   ### Slots
 
-  Some optimizer subclasses, such as `MomentumOptimizer` and `AdagradOptimizer`
-  allocate and manage additional variables associated with the variables to
-  train.  These are called <i>Slots</i>.  Slots have names and you can ask the
-  optimizer for the names of the slots that it uses.  Once you have a slot name
-  you can ask the optimizer for the variable it created to hold the slot value.
+  Many optimizer subclasses, such as `Adam` and `Adagrad` allocate and manage
+  additional variables associated with the variables to train.  These are called
+  <i>Slots</i>.  Slots have names and you can ask the optimizer for the names of
+  the slots that it uses.  Once you have a slot name you can ask the optimizer
+  for the variable it created to hold the slot value.
 
   This can be useful if you want to log debug a training algorithm, report stats
   about the slots, etc.
@@ -146,6 +155,33 @@ class OptimizerV2(checkpointable.Checkpointable):
   callables. If they are callable, the callable will be called during
   `apply_gradients()` to get the value for the hyper parameter.
 
+  Hyper parameters can be overwritten through user code:
+
+  Example:
+
+  ```python
+  # Create an optimizer with the desired parameters.
+  opt = tf.keras.optimizers.SGD(learning_rate=0.1)
+  # `loss` is a callable that takes no argument and returns the value
+  # to minimize.
+  loss = lambda: 3 * var1 + 2 * var2
+  # In eager mode, simply call minimize to update the list of variables.
+  opt.minimize(loss, var_list=[var1, var2])
+  # update learning rate
+  opt.learning_rate = 0.05
+  opt.minimize(loss, var_list=[var1, var2])
+  ```
+
+  ### Write a customized optimizer.
+
+  This optimizer class updates variables from gradients and is
+  tf.distribute.Strategy aware. If you intend to create your own optimization
+  algorithm, simply inherit from this class and override the following methods:
+    - resource_apply_dense (update variable given gradient tensor is dense)
+    - resource_apply_sparse (update variable given gradient tensor is sparse)
+    - create_slots (if your optimizer algorithm requires additional variables)
+    - get_config (serialization of the optimizer, include all hyper parameters)
+
   """
 
   def __init__(self, name, **kwargs):
@@ -213,9 +249,9 @@ class OptimizerV2(checkpointable.Checkpointable):
   def minimize(self, loss, var_list, grad_loss=None, name=None):
     """Add operations to minimize `loss` by updating `var_list`.
 
-    This method simply combines calls `compute_gradients()` and
+    This method simply computes gradient using `tf.GradientTape` and calls
     `apply_gradients()`. If you want to process the gradient before applying
-    them call `compute_gradients()` and `apply_gradients()` explicitly instead
+    then call `tf.GradientTape` and `apply_gradients()` explicitly instead
     of using this function.
 
     Args:
@@ -338,8 +374,7 @@ class OptimizerV2(checkpointable.Checkpointable):
     applies gradients.
 
     Args:
-      grads_and_vars: List of (gradient, variable) pairs as returned by
-        `compute_gradients()`.
+      grads_and_vars: List of (gradient, variable) pairs.
       name: Optional name for the returned operation.  Default to the name
         passed to the `Optimizer` constructor.
 
@@ -353,18 +388,24 @@ class OptimizerV2(checkpointable.Checkpointable):
     """
     grads_and_vars = _filter_grads(grads_and_vars)
     var_list = [v for (_, v) in grads_and_vars]
-    if distribute_ctx.has_strategy():
-      reduced_grads = merge_grads(grads_and_vars)
-      grads_and_vars = zip(reduced_grads, var_list)
 
     self._create_hypers()
     with ops.init_scope():
       self._create_slots(var_list)
-    update_ops = []
 
     self._prepare(var_list)
 
-    def update_grad_to_var(grad, var):
+    return distribute_ctx.get_replica_context().merge_call(
+        self._distributed_apply, args=(grads_and_vars,), kwargs={"name": name})
+
+  def _distributed_apply(self, distribution, grads_and_vars, name):
+    """`apply_gradients` using a `DistributionStrategy`."""
+    reduced_grads = distribution.extended.batch_reduce_to(
+        ds_reduce_util.ReduceOp.SUM, grads_and_vars)
+    var_list = [v for _, v in grads_and_vars]
+    grads_and_vars = zip(reduced_grads, var_list)
+
+    def apply_grad_to_update_var(var, grad):
       """Apply gradient to variable."""
       if isinstance(var, ops.Tensor):
         raise NotImplementedError("Trying to update a Tensor ", var)
@@ -381,17 +422,19 @@ class OptimizerV2(checkpointable.Checkpointable):
       else:
         return update_op
 
+    update_ops = []
     with ops.name_scope(name, self._name) as name:
       for grad, var in grads_and_vars:
         scope_name = ("" if ops.executing_eagerly_outside_functions() else
                       "_" + var.op.name)
         with ops.name_scope("update" + scope_name):
-          update_ops.append(update_grad_to_var(grad, var))
-      # control dependencies does not work in per replica mode, please change
-      # this once b/118841692 is fixed.
-      # with ops.control_dependencies(update_ops):
-      #   apply_updates = self._iterations.assign_add(1).op
-      apply_updates = merge_update_step(update_ops, self.iterations)
+          update_ops.extend(
+              distribution.extended.update(
+                  var, apply_grad_to_update_var, args=(grad,), group=False))
+      with ops.control_dependencies(update_ops):
+        apply_updates = self._iterations.assign_add(1)
+      if not context.executing_eagerly():
+        apply_updates = apply_updates.op
       return apply_updates
 
   def get_updates(self, loss, params):
@@ -864,33 +907,6 @@ def _filter_grads(grads_and_vars):
   return filtered
 
 
-def merge_update_step(update_ops, local_step):
-  """Merge local step counter update from different replicas."""
-
-  def merge_update_step_fn(strategy, update_ops, local_step):
-    merged_ops = []
-    for update_op in update_ops:
-      merged_ops.append(strategy.group(update_op))
-    with ops.control_dependencies(merged_ops):
-      incre_op = local_step.assign_add(1).op
-    return incre_op
-
-  return distribute_ctx.get_replica_context().merge_call(
-      merge_update_step_fn, args=(update_ops, local_step))
-
-
-def merge_grads(grads_and_vars):
-  """Merge gradients from different replicas."""
-
-  def merge_grad_fn(strategy, grads_and_vars):
-    reduced_grads = strategy.extended.batch_reduce_to(
-        ds_reduce_util.ReduceOp.SUM, grads_and_vars)
-    return reduced_grads
-
-  return distribute_ctx.get_replica_context().merge_call(
-      merge_grad_fn, args=(grads_and_vars,))
-
-
 def _var_key(var):
   """Key for representing a primary variable, for looking up slots.
 
@@ -906,9 +922,10 @@ def _var_key(var):
   """
 
   # pylint: disable=protected-access
-  if distribute_ctx.has_strategy() and hasattr(var, "_primary_var"):
-    var = var._primary_var
-  if hasattr(var, "op"):
+  # Get the distributed variable if it exists.
+  if getattr(var, "_distributed_container", None) is not None:
+    var = var._distributed_container()
+  if var._in_graph_mode:
     return var._shared_name
   return var._unique_id
 
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
index 8069703b7a..57bd9439ed 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
@@ -42,6 +42,7 @@ from tensorflow.python.keras.optimizer_v2 import adam
 from tensorflow.python.keras.optimizer_v2 import adamax
 from tensorflow.python.keras.optimizer_v2 import gradient_descent
 from tensorflow.python.keras.optimizer_v2 import nadam
+from tensorflow.python.keras.optimizer_v2 import optimizer_v2
 from tensorflow.python.keras.optimizer_v2 import rmsprop
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import clip_ops
@@ -500,6 +501,17 @@ class OptimizerTest(test.TestCase):
     new_step_value = self.evaluate(global_step)
     self.assertEqual(new_step_value, init_step_value + 1)
 
+  def testVarKey(self):
+    with context.graph_mode():
+      a = variables.Variable([1., 2.], name='var')
+      b = variables.Variable([1.], name='var')
+      self.assertTrue(a._in_graph_mode)
+      self.assertTrue(b._in_graph_mode)
+      var_key = optimizer_v2._var_key(a)
+      self.assertEqual('var', var_key)
+      var_key = optimizer_v2._var_key(b)
+      self.assertEqual('var_1', var_key)
+
 
 @keras_parameterized.run_with_all_model_types
 class OptimizersCompatibilityTest(keras_parameterized.TestCase):
@@ -673,6 +685,23 @@ class OptimizerWithFunctionTest(test.TestCase):
       self.assertAllClose([0., 1.], fn(), atol=1e-4)
       self.assertAllClose([-1, 0.], fn(), atol=1e-4)
 
+  def testVarKeyWithVarCreatedInEager(self):
+    with context.eager_mode():
+      a = variables.Variable([1., 2.], name='var')
+      b = variables.Variable([1.], name='var')
+
+      @test_util.also_run_as_tf_function
+      def var_key_test():
+        self.assertFalse(a._in_graph_mode)
+        self.assertFalse(b._in_graph_mode)
+        var_key_a = optimizer_v2._var_key(a)
+        self.assertStartsWith(var_key_a, 'var_')
+        var_key_b = optimizer_v2._var_key(b)
+        self.assertStartsWith(var_key_b, 'var_')
+        self.assertNotEquals(var_key_a, var_key_b)
+
+      var_key_test()
+
 
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 7eff9dfb0eaaad318d5f05e0a1b7e3f7180ea993 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 29 Jan 2019 19:09:13 -0800
Subject: [PATCH 1558/2345] Update ops-related pbtxt files.

PiperOrigin-RevId: 231518570
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 47 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 |  7 +++
 2 files changed, 54 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 503d4b557f..8c05ad0e25 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -24035,6 +24035,53 @@ op {
     }
   }
 }
+op {
+  name: "ExtractGlimpse"
+  input_arg {
+    name: "input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "size"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "offsets"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "glimpse"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "centered"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  attr {
+    name: "normalized"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  attr {
+    name: "uniform_noise"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  attr {
+    name: "noise"
+    type: "string"
+    default_value {
+      s: "uniform"
+    }
+  }
+}
 op {
   name: "ExtractImagePatches"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 66dc4be00d..03868f1c67 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -11704,6 +11704,13 @@ op {
       b: true
     }
   }
+  attr {
+    name: "noise"
+    type: "string"
+    default_value {
+      s: "uniform"
+    }
+  }
 }
 op {
   name: "ExtractImagePatches"
-- 
GitLab


From a3afe4309acfb86a9d6acc205f22d946f3790067 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 29 Jan 2019 19:10:03 -0800
Subject: [PATCH 1559/2345] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 231518633

---
 tensorflow/go/op/wrappers.go | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index c537cb1ba9..2430063994 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -31765,6 +31765,18 @@ func ExtractGlimpseUniformNoise(value bool) ExtractGlimpseAttr {
 	}
 }
 
+// ExtractGlimpseNoise sets the optional noise attribute to value.
+//
+// value: indicates if the noise should `uniform`, `gaussian`, or
+// `zero`. The default is `uniform` which means the the noise type
+// will be decided by `uniform_noise`.
+// If not specified, defaults to "uniform"
+func ExtractGlimpseNoise(value string) ExtractGlimpseAttr {
+	return func(m optionalAttr) {
+		m["noise"] = value
+	}
+}
+
 // Extracts a glimpse from the input tensor.
 //
 // Returns a set of windows called glimpses extracted at location
-- 
GitLab


From 1afd37f416f9cafd37ca9b3652384855cb472a3c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 29 Jan 2019 19:45:32 -0800
Subject: [PATCH 1560/2345] [XLA] Update cost model to account for batch group
 counts.

PiperOrigin-RevId: 231521874
---
 tensorflow/compiler/xla/service/hlo_cost_analysis.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
index 9c7239ff6f..1ee958114e 100644
--- a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
+++ b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
@@ -524,7 +524,8 @@ Status HloCostAnalysis::HandleConvolution(const HloInstruction* convolution) {
   }
 
   const int64 fma_count = (input_feature / convolution->feature_group_count()) *
-                          output_feature * batch *
+                          output_feature *
+                          (batch / convolution->batch_group_count()) *
                           Product(valid_position_counts);
   current_properties_[kFlopsKey] = fma_count * kFmaFlops;
   return Status::OK();
-- 
GitLab


From c27f792d682af935160a681fb3a435d849c571e6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 29 Jan 2019 20:18:17 -0800
Subject: [PATCH 1561/2345] Update ops-related pbtxt files.

PiperOrigin-RevId: 231524653
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 50 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 50 +++++++++++++++++++
 2 files changed, 100 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 8c05ad0e25..f67895755a 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -12963,6 +12963,56 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "CombinedNonMaxSuppression"
+  input_arg {
+    name: "boxes"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "scores"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_output_size_per_class"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "max_total_size"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "iou_threshold"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "score_threshold"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "nmsed_boxes"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "nmsed_scores"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "nmsed_classes"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "valid_detections"
+    type: DT_INT32
+  }
+  attr {
+    name: "pad_per_class"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+}
 op {
   name: "CompareAndBitpack"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 03868f1c67..7fd8a36795 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -5468,6 +5468,56 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "CombinedNonMaxSuppression"
+  input_arg {
+    name: "boxes"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "scores"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_output_size_per_class"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "max_total_size"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "iou_threshold"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "score_threshold"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "nmsed_boxes"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "nmsed_scores"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "nmsed_classes"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "valid_detections"
+    type: DT_INT32
+  }
+  attr {
+    name: "pad_per_class"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+}
 op {
   name: "CompareAndBitpack"
   input_arg {
-- 
GitLab


From 09696450acc1a69dcac4717de87ee4f30544865a Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Tue, 29 Jan 2019 20:22:35 -0800
Subject: [PATCH 1562/2345] Add v2 sparse categorical crossentropy metric.

PiperOrigin-RevId: 231525003
---
 tensorflow/python/keras/losses.py             |   4 +-
 tensorflow/python/keras/metrics.py            |  71 +++++++
 tensorflow/python/keras/metrics_test.py       | 152 +++++++++++++
 .../golden/v1/tensorflow.keras.losses.pbtxt   |   2 +-
 ...ics.-sparse-categorical-crossentropy.pbtxt | 199 ++++++++++++++++++
 .../golden/v1/tensorflow.keras.metrics.pbtxt  |   6 +-
 .../golden/v2/tensorflow.keras.losses.pbtxt   |   2 +-
 ...ics.-sparse-categorical-crossentropy.pbtxt | 199 ++++++++++++++++++
 .../golden/v2/tensorflow.keras.metrics.pbtxt  |   6 +-
 9 files changed, 635 insertions(+), 6 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-crossentropy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-crossentropy.pbtxt

diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index 01268a1f76..e83b805ce4 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -853,9 +853,9 @@ def categorical_crossentropy(y_true,
 
 @keras_export('keras.metrics.sparse_categorical_crossentropy',
               'keras.losses.sparse_categorical_crossentropy')
-def sparse_categorical_crossentropy(y_true, y_pred, from_logits=False):
+def sparse_categorical_crossentropy(y_true, y_pred, from_logits=False, axis=-1):
   return K.sparse_categorical_crossentropy(
-      y_true, y_pred, from_logits=from_logits)
+      y_true, y_pred, from_logits=from_logits, axis=axis)
 
 
 @keras_export('keras.metrics.binary_crossentropy',
diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index bc7c1bff80..f7ecbefa3a 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -2543,6 +2543,77 @@ class CategoricalCrossentropy(MeanMetricWrapper):
         label_smoothing=label_smoothing)
 
 
+@keras_export('keras.metrics.SparseCategoricalCrossentropy')
+class SparseCategoricalCrossentropy(MeanMetricWrapper):
+  """Computes the crossentropy metric between the labels and predictions.
+
+  Use this crossentropy metric when there are two or more label classes.
+  We expect labels to be provided as integers. If you want to provide labels
+  using `one-hot` representation, please use `CategoricalCrossentropy` metric.
+  There should be `# classes` floating point values per feature for `y_pred`
+  and a single floating point value per feature for `y_true`.
+
+  In the snippet below, there is a single floating point value per example for
+  `y_true` and `# classes` floating pointing values per example for `y_pred`.
+  The shape of `y_true` is `[batch_size]` and the shape of `y_pred` is
+  `[batch_size, num_classes]`.
+
+  Usage:
+
+  ```python
+  m = tf.keras.metrics.SparseCategoricalCrossentropy()
+  m.update_state(
+    [1, 2],
+    [[0.05, 0.95, 0], [0.1, 0.8, 0.1]])
+
+  # y_true = one_hot(y_true) = [[0, 1, 0], [0, 0, 1]]
+  # logits = log(y_pred)
+  # softmax = exp(logits) / sum(exp(logits), axis=-1)
+  # softmax = [[0.05, 0.95, EPSILON], [0.1, 0.8, 0.1]]
+
+  # xent = -sum(y * log(softmax), 1)
+  # log(softmax) = [[-2.9957, -0.0513, -16.1181], [-2.3026, -0.2231, -2.3026]]
+  # y_true * log(softmax) = [[0, -0.0513, 0], [0, 0, -2.3026]]
+
+  # xent = [0.0513, 2.3026]
+  # Reduced xent = (0.0513 + 2.3026) / 2
+
+  print('Final result: ', m.result().numpy())  # Final result: 1.176
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile(
+    'sgd',
+    loss='mse',
+    metrics=[tf.keras.metrics.SparseCategoricalCrossentropy()])
+  ```
+
+  Args:
+    name: (Optional) string name of the metric instance.
+    dtype: (Optional) data type of the metric result.
+    from_logits: (Optional ) Whether `y_pred` is expected to be a logits tensor.
+      By default, we assume that `y_pred` encodes a probability distribution.
+    axis: (Optional) Defaults to -1. The dimension along which the metric is
+      computed.
+  """
+
+  def __init__(self,
+               name='sparse_categorical_crossentropy',
+               dtype=None,
+               from_logits=False,
+               axis=-1):
+
+    super(SparseCategoricalCrossentropy, self).__init__(
+        sparse_categorical_crossentropy,
+        name,
+        dtype=dtype,
+        from_logits=from_logits,
+        axis=axis)
+
+
 def accuracy(y_true, y_pred):
   y_pred.get_shape().assert_is_compatible_with(y_true.get_shape())
   if y_true.dtype != y_pred.dtype:
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index 35732ad307..7d1f88877c 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import json
 import math
 import os
 import numpy as np
@@ -1563,6 +1564,157 @@ class CategoricalCrossentropyTest(test.TestCase):
     self.assertAllClose(self.evaluate(loss), 3.667, atol=1e-3)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class SparseCategoricalCrossentropyTest(test.TestCase):
+
+  def test_config(self):
+    scce_obj = metrics.SparseCategoricalCrossentropy(
+        name='scce', dtype=dtypes.int32)
+    self.assertEqual(scce_obj.name, 'scce')
+    self.assertEqual(scce_obj.dtype, dtypes.int32)
+    old_config = scce_obj.get_config()
+    self.assertDictEqual(old_config, json.loads(json.dumps(old_config)))
+
+    # Check save and restore config
+    scce_obj2 = metrics.SparseCategoricalCrossentropy.from_config(old_config)
+    self.assertEqual(scce_obj2.name, 'scce')
+    self.assertEqual(scce_obj2.dtype, dtypes.int32)
+    new_config = scce_obj2.get_config()
+    self.assertDictEqual(old_config, new_config)
+
+  def test_unweighted(self):
+    scce_obj = metrics.SparseCategoricalCrossentropy()
+    self.evaluate(variables.variables_initializer(scce_obj.variables))
+
+    y_true = np.asarray([1, 2])
+    y_pred = np.asarray([[0.05, 0.95, 0], [0.1, 0.8, 0.1]])
+    result = scce_obj(y_true, y_pred)
+
+    # EPSILON = 1e-7, y = y_true, y` = y_pred
+    # y` = clip_ops.clip_by_value(output, EPSILON, 1. - EPSILON)
+    # y` = [[0.05, 0.95, EPSILON], [0.1, 0.8, 0.1]]
+    # logits = log(y`) =  [[-2.9957, -0.0513, -16.1181],
+    #                      [-2.3026, -0.2231, -2.3026]]
+
+    # softmax = exp(logits) / sum(exp(logits), axis=-1)
+    # y = one_hot(y) = [[0, 1, 0], [0, 0, 1]]
+    # xent = -sum(y * log(softmax), 1)
+
+    # exp(logits) = [[0.05, 0.95, EPSILON], [0.1, 0.8, 0.1]]
+    # sum(exp(logits), axis=-1) = [1, 1]
+    # softmax = [[0.05, 0.95, EPSILON], [0.1, 0.8, 0.1]]
+    # log(softmax) = [[-2.9957, -0.0513, -16.1181],
+    #                 [-2.3026, -0.2231, -2.3026]]
+    # y * log(softmax) = [[0, -0.0513, 0], [0, 0, -2.3026]]
+    # xent = [0.0513, 2.3026]
+    # Reduced xent = (0.0513 + 2.3026) / 2
+
+    self.assertAllClose(self.evaluate(result), 1.176, atol=1e-3)
+
+  def test_unweighted_from_logits(self):
+    scce_obj = metrics.SparseCategoricalCrossentropy(from_logits=True)
+    self.evaluate(variables.variables_initializer(scce_obj.variables))
+
+    y_true = np.asarray([1, 2])
+    logits = np.asarray([[1, 9, 0], [1, 8, 1]], dtype=np.float32)
+    result = scce_obj(y_true, logits)
+
+    # softmax = exp(logits) / sum(exp(logits), axis=-1)
+    # y_true = one_hot(y_true) = [[0, 1, 0], [0, 0, 1]]
+    # xent = -sum(y_true * log(softmax), 1)
+
+    # exp(logits) = [[2.718, 8103.084, 1], [2.718, 2980.958, 2.718]]
+    # sum(exp(logits), axis=-1) = [8106.802, 2986.394]
+    # softmax = [[0.00033, 0.99954, 0.00012], [0.00091, 0.99817, 0.00091]]
+    # log(softmax) = [[-8.00045, -0.00045, -9.00045],
+    #                 [-7.00182, -0.00182, -7.00182]]
+    # y_true * log(softmax) = [[0, -0.00045, 0], [0, 0, -7.00182]]
+    # xent = [0.00045, 7.00182]
+    # Reduced xent = (0.00045 + 7.00182) / 2
+
+    self.assertAllClose(self.evaluate(result), 3.5011, atol=1e-3)
+
+  def test_weighted(self):
+    scce_obj = metrics.SparseCategoricalCrossentropy()
+    self.evaluate(variables.variables_initializer(scce_obj.variables))
+
+    y_true = np.asarray([1, 2])
+    y_pred = np.asarray([[0.05, 0.95, 0], [0.1, 0.8, 0.1]])
+    sample_weight = constant_op.constant([1.5, 2.])
+    result = scce_obj(y_true, y_pred, sample_weight=sample_weight)
+
+    # EPSILON = 1e-7, y = y_true, y` = y_pred
+    # y` = clip_ops.clip_by_value(output, EPSILON, 1. - EPSILON)
+    # y` = [[0.05, 0.95, EPSILON], [0.1, 0.8, 0.1]]
+    # logits = log(y`) =  [[-2.9957, -0.0513, -16.1181],
+    #                      [-2.3026, -0.2231, -2.3026]]
+
+    # softmax = exp(logits) / sum(exp(logits), axis=-1)
+    # y = one_hot(y) = [[0, 1, 0], [0, 0, 1]]
+    # xent = -sum(y * log(softmax), 1)
+
+    # exp(logits) = [[0.05, 0.95, EPSILON], [0.1, 0.8, 0.1]]
+    # sum(exp(logits), axis=-1) = [1, 1]
+    # softmax = [[0.05, 0.95, EPSILON], [0.1, 0.8, 0.1]]
+    # log(softmax) = [[-2.9957, -0.0513, -16.1181],
+    #                 [-2.3026, -0.2231, -2.3026]]
+    # y * log(softmax) = [[0, -0.0513, 0], [0, 0, -2.3026]]
+    # xent = [0.0513, 2.3026]
+    # Weighted xent = [0.051 * 1.5, 2.302 * 2.]
+    # Reduced xent = (0.051 * 1.5 + 2.302 * 2.) / 3.5
+
+    self.assertAllClose(self.evaluate(result), 1.338, atol=1e-3)
+
+  def test_weighted_from_logits(self):
+    scce_obj = metrics.SparseCategoricalCrossentropy(from_logits=True)
+    self.evaluate(variables.variables_initializer(scce_obj.variables))
+
+    y_true = np.asarray([1, 2])
+    logits = np.asarray([[1, 9, 0], [1, 8, 1]], dtype=np.float32)
+    sample_weight = constant_op.constant([1.5, 2.])
+    result = scce_obj(y_true, logits, sample_weight=sample_weight)
+
+    # softmax = exp(logits) / sum(exp(logits), axis=-1)
+    # y_true = one_hot(y_true) = [[0, 1, 0], [0, 0, 1]]
+    # xent = -sum(y_true * log(softmax), 1)
+    # xent = [0.00045, 7.00182]
+    # weighted xent = [0.000675, 14.00364]
+    # Reduced xent = (0.000675 + 14.00364) / (1.5 + 2)
+
+    self.assertAllClose(self.evaluate(result), 4.0012, atol=1e-3)
+
+  def test_axis(self):
+    scce_obj = metrics.SparseCategoricalCrossentropy(axis=0)
+    self.evaluate(variables.variables_initializer(scce_obj.variables))
+
+    y_true = np.asarray([1, 2])
+    y_pred = np.asarray([[0.05, 0.1], [0.95, 0.8], [0, 0.1]])
+    result = scce_obj(y_true, y_pred)
+
+    # EPSILON = 1e-7, y = y_true, y` = y_pred
+    # y` = clip_ops.clip_by_value(output, EPSILON, 1. - EPSILON)
+    # y` = [[0.05, 0.1], [0.95, 0.8], [EPSILON, 0.1]]
+    # logits = log(y`) =  [[-2.9957, -2.3026],
+    #                      [-0.0513, -0.2231],
+    #                      [-16.1181, -2.3026]]
+
+    # softmax = exp(logits) / sum(exp(logits), axis=-1)
+    # y = one_hot(y) = [[0, 0], [1, 0], [0, 1]]
+    # xent = -sum(y * log(softmax), 1)
+
+    # exp(logits) = [[0.05, 0.1], [0.95, 0.8], [EPSILON, 0.1]]
+    # sum(exp(logits)) = [1, 1]
+    # softmax = [[0.05, 0.1], [0.95, 0.8], [EPSILON, 0.1]]
+    # log(softmax) = [[-2.9957, -2.3026],
+    #                 [-0.0513, -0.2231],
+    #                 [-16.1181, -2.3026]]
+    # y * log(softmax) = [[0, 0], [-0.0513, 0], [0, -2.3026]]
+    # xent = [0.0513, 2.3026]
+    # Reduced xent = (0.0513 + 2.3026) / 2
+
+    self.assertAllClose(self.evaluate(result), 1.176, atol=1e-3)
+
+
 def _get_model(compile_metrics):
   model_layers = [
       layers.Dense(3, activation='relu', kernel_initializer='ones'),
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
index 7381fe3918..4240ff69a5 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
@@ -174,7 +174,7 @@ tf_module {
   }
   member_method {
     name: "sparse_categorical_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\', \'axis\'], varargs=None, keywords=None, defaults=[\'False\', \'-1\'], "
   }
   member_method {
     name: "squared_hinge"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-crossentropy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-crossentropy.pbtxt
new file mode 100644
index 0000000000..c968768168
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-crossentropy.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.keras.metrics.SparseCategoricalCrossentropy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.SparseCategoricalCrossentropy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\', \'from_logits\', \'axis\'], varargs=None, keywords=None, defaults=[\'sparse_categorical_crossentropy\', \'None\', \'False\', \'-1\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
index 7ebf43b362..92dddacf2c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
@@ -112,6 +112,10 @@ tf_module {
     name: "SparseCategoricalAccuracy"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "SparseCategoricalCrossentropy"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "SparseTopKCategoricalAccuracy"
     mtype: "<type \'type\'>"
@@ -250,7 +254,7 @@ tf_module {
   }
   member_method {
     name: "sparse_categorical_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\', \'axis\'], varargs=None, keywords=None, defaults=[\'False\', \'-1\'], "
   }
   member_method {
     name: "sparse_top_k_categorical_accuracy"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
index 090ab7ee3f..af9d3c12e3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
@@ -178,7 +178,7 @@ tf_module {
   }
   member_method {
     name: "sparse_categorical_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\', \'axis\'], varargs=None, keywords=None, defaults=[\'False\', \'-1\'], "
   }
   member_method {
     name: "squared_hinge"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-crossentropy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-crossentropy.pbtxt
new file mode 100644
index 0000000000..c968768168
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-crossentropy.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.keras.metrics.SparseCategoricalCrossentropy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.SparseCategoricalCrossentropy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\', \'from_logits\', \'axis\'], varargs=None, keywords=None, defaults=[\'sparse_categorical_crossentropy\', \'None\', \'False\', \'-1\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
index 7ebf43b362..92dddacf2c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
@@ -112,6 +112,10 @@ tf_module {
     name: "SparseCategoricalAccuracy"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "SparseCategoricalCrossentropy"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "SparseTopKCategoricalAccuracy"
     mtype: "<type \'type\'>"
@@ -250,7 +254,7 @@ tf_module {
   }
   member_method {
     name: "sparse_categorical_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\', \'axis\'], varargs=None, keywords=None, defaults=[\'False\', \'-1\'], "
   }
   member_method {
     name: "sparse_top_k_categorical_accuracy"
-- 
GitLab


From 24ce9f5bb519a10a6f431a425487561ffbceda2e Mon Sep 17 00:00:00 2001
From: Dayananda-V <dayanandasiet@gmail.com>
Date: Mon, 28 Jan 2019 16:25:20 +0530
Subject: [PATCH 1563/2345] TF Framework error_test missing test case add
 1-error_code_from_exception_type api test case

---
 tensorflow/python/framework/errors_impl.py |  6 +++++-
 tensorflow/python/framework/errors_test.py | 12 ++++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/framework/errors_impl.py b/tensorflow/python/framework/errors_impl.py
index 922b9e2bd3..c473dfeedf 100644
--- a/tensorflow/python/framework/errors_impl.py
+++ b/tensorflow/python/framework/errors_impl.py
@@ -511,7 +511,11 @@ def exception_type_from_error_code(error_code):
 
 @tf_export("errors.error_code_from_exception_type")
 def error_code_from_exception_type(cls):
-  return _EXCEPTION_CLASS_TO_CODE[cls]
+  try:
+    return _EXCEPTION_CLASS_TO_CODE[cls]
+  except KeyError:
+    warnings.warn("Unknown class exception")
+    return UnknownError(None, None, "Unknown class exception", None)
 
 
 def _make_specific_exception(node_def, op, message, error_code):
diff --git a/tensorflow/python/framework/errors_test.py b/tensorflow/python/framework/errors_test.py
index 574b126cae..c044202d92 100644
--- a/tensorflow/python/framework/errors_test.py
+++ b/tensorflow/python/framework/errors_test.py
@@ -70,6 +70,10 @@ class ErrorsTest(test.TestCase):
           isinstance(
               errors_impl._make_specific_exception(None, None, None,
                                                    error_code), exc_type))
+      # error_code_from_exception_type and exception_type_from_error_code should
+      # be consistent with operation result.
+      self.assertEqual(error_code,
+                       errors_impl.error_code_from_exception_type(exc_type))
       # pylint: enable=protected-access
 
   def testKnownErrorClassForEachErrorCodeInProto(self):
@@ -98,6 +102,14 @@ class ErrorsTest(test.TestCase):
     self.assertTrue("Unknown error code: 37" in str(w[0].message))
     self.assertTrue(isinstance(exc, errors_impl.OpError))
 
+    with warnings.catch_warnings(record=True) as w:
+      # pylint: disable=protected-access
+      exc = errors_impl.error_code_from_exception_type("Unknown")
+      # pylint: enable=protected-access
+    self.assertEqual(1, len(w))
+    self.assertTrue("Unknown class exception" in str(w[0].message))
+    self.assertTrue(isinstance(exc, errors_impl.OpError))
+
   def testStatusDoesNotLeak(self):
     try:
       with errors.raise_exception_on_not_ok_status() as status:
-- 
GitLab


From 724a264c4e1a86879feabf2fa43b9d2c42320058 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 29 Jan 2019 20:46:03 -0800
Subject: [PATCH 1564/2345] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 231526760

---
 tensorflow/go/op/wrappers.go | 119 ++++++++++++++++++++++++++++-------
 1 file changed, 96 insertions(+), 23 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 2430063994..97b6d4bba6 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -25974,29 +25974,6 @@ func SegmentMin(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.
 	return op.Output(0)
 }
 
-// Removes keys and its associated values from a table.
-//
-// The tensor `keys` must of the same type as the keys of the table. Keys not
-// already in the table are silently ignored.
-//
-// Arguments:
-//	table_handle: Handle to the table.
-//	keys: Any shape.  Keys of the elements to remove.
-//
-// Returns the created operation.
-func LookupTableRemoveV2(scope *Scope, table_handle tf.Output, keys tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "LookupTableRemoveV2",
-		Input: []tf.Input{
-			table_handle, keys,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
 // Computes the sum along segments of a tensor.
 //
 // Read
@@ -32184,6 +32161,102 @@ func NonMaxSuppressionV4(scope *Scope, boxes tf.Output, scores tf.Output, max_ou
 	return op.Output(0), op.Output(1)
 }
 
+// Removes keys and its associated values from a table.
+//
+// The tensor `keys` must of the same type as the keys of the table. Keys not
+// already in the table are silently ignored.
+//
+// Arguments:
+//	table_handle: Handle to the table.
+//	keys: Any shape.  Keys of the elements to remove.
+//
+// Returns the created operation.
+func LookupTableRemoveV2(scope *Scope, table_handle tf.Output, keys tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "LookupTableRemoveV2",
+		Input: []tf.Input{
+			table_handle, keys,
+		},
+	}
+	return scope.AddOperation(opspec)
+}
+
+// CombinedNonMaxSuppressionAttr is an optional argument to CombinedNonMaxSuppression.
+type CombinedNonMaxSuppressionAttr func(optionalAttr)
+
+// CombinedNonMaxSuppressionPadPerClass sets the optional pad_per_class attribute to value.
+//
+// value: If false, the output nmsed boxes, scores and classes
+// are padded/clipped to `max_total_size`. If true, the
+// output nmsed boxes, scores and classes are padded to be of length
+// `max_size_per_class`*`num_classes`, unless it exceeds `max_total_size` in
+// which case it is clipped to `max_total_size`. Defaults to false.
+// If not specified, defaults to false
+func CombinedNonMaxSuppressionPadPerClass(value bool) CombinedNonMaxSuppressionAttr {
+	return func(m optionalAttr) {
+		m["pad_per_class"] = value
+	}
+}
+
+// Greedily selects a subset of bounding boxes in descending order of score,
+//
+// This operation performs non_max_suppression on the inputs per batch, across
+// all classes.
+// Prunes away boxes that have high intersection-over-union (IOU) overlap
+// with previously selected boxes.  Bounding boxes are supplied as
+// [y1, x1, y2, x2], where (y1, x1) and (y2, x2) are the coordinates of any
+// diagonal pair of box corners and the coordinates can be provided as normalized
+// (i.e., lying in the interval [0, 1]) or absolute.  Note that this algorithm
+// is agnostic to where the origin is in the coordinate system. Also note that
+// this algorithm is invariant to orthogonal transformations and translations
+// of the coordinate system; thus translating or reflections of the coordinate
+// system result in the same boxes being selected by the algorithm.
+// The output of this operation is the final boxes, scores and classes tensor
+// returned after performing non_max_suppression.
+//
+// Arguments:
+//	boxes: A 4-D float tensor of shape `[batch_size, num_boxes, q, 4]`. If `q` is 1 then
+// same boxes are used for all classes otherwise, if `q` is equal to number of
+// classes, class-specific boxes are used.
+//	scores: A 3-D float tensor of shape `[batch_size, num_boxes, num_classes]`
+// representing a single score corresponding to each box (each row of boxes).
+//	max_output_size_per_class: A scalar integer tensor representing the maximum number of
+// boxes to be selected by non max suppression per class
+//	max_total_size: A scalar representing maximum number of boxes retained over all classes.
+//	iou_threshold: A 0-D float tensor representing the threshold for deciding whether
+// boxes overlap too much with respect to IOU.
+//	score_threshold: A 0-D float tensor representing the threshold for deciding when to remove
+// boxes based on score.
+//
+// Returns A [batch_size, max_detections, 4] float32 tensor
+// containing the non-max suppressed boxes.A [batch_size, max_detections] float32 tensor
+// containing the scores for the boxes.A [batch_size, max_detections] float32 tensor
+// containing the classes for the boxes.A [batch_size] int32 tensor indicating the number of
+// valid detections per batch item. Only the top num_detections[i] entries in
+// nms_boxes[i], nms_scores[i] and nms_class[i] are valid. The rest of the
+// entries are zero paddings.
+func CombinedNonMaxSuppression(scope *Scope, boxes tf.Output, scores tf.Output, max_output_size_per_class tf.Output, max_total_size tf.Output, iou_threshold tf.Output, score_threshold tf.Output, optional ...CombinedNonMaxSuppressionAttr) (nmsed_boxes tf.Output, nmsed_scores tf.Output, nmsed_classes tf.Output, valid_detections tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "CombinedNonMaxSuppression",
+		Input: []tf.Input{
+			boxes, scores, max_output_size_per_class, max_total_size, iou_threshold, score_threshold,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2), op.Output(3)
+}
+
 // Computes the matrix logarithm of one or more square matrices:
 //
 //
-- 
GitLab


From 79d86aca647b5a3b8300eb104519332871d00947 Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Tue, 29 Jan 2019 21:19:32 -0800
Subject: [PATCH 1565/2345] Move map_fn to separate file

PiperOrigin-RevId: 231529445
---
 tensorflow/compiler/tests/BUILD               |   1 +
 .../eval/python/classifier_metrics_impl.py    |  20 +-
 .../gan/python/eval/python/summaries_impl.py  |   4 +-
 tensorflow/contrib/labeled_tensor/BUILD       |   2 +-
 .../contrib/labeled_tensor/python/ops/ops.py  |   5 +-
 .../estimators/dynamic_rnn_estimator_test.py  |   4 +-
 .../python/slim/data/tfexample_decoder.py     |   6 +-
 tensorflow/python/BUILD                       |  21 +-
 .../benchmarks/map_defun_benchmark.py         |   6 +-
 .../python/data/kernel_tests/filter_test.py   |   4 +-
 .../python/data/kernel_tests/map_test.py      |   8 +-
 .../python/grappler/layout_optimizer_test.py  |  10 +-
 tensorflow/python/keras/BUILD                 |   1 +
 tensorflow/python/keras/backend.py            |   3 +-
 tensorflow/python/kernel_tests/BUILD          |  23 ++
 .../kernel_tests/control_flow_ops_py_test.py  |   9 +-
 .../kernel_tests/functional_ops_test.py       | 167 ----------
 tensorflow/python/kernel_tests/lu_op_test.py  |   6 +-
 tensorflow/python/kernel_tests/map_fn_test.py | 224 ++++++++++++++
 .../python/kernel_tests/while_v2_test.py      |   4 +-
 tensorflow/python/ops/ctc_ops.py              |   3 +-
 .../python/ops/distributions/multinomial.py   |   4 +-
 tensorflow/python/ops/functional_ops.py       | 251 ---------------
 tensorflow/python/ops/linalg_ops.py           |   6 +-
 tensorflow/python/ops/map_fn.py               | 285 ++++++++++++++++++
 tensorflow/python/ops/parallel_for/BUILD      |   3 +-
 .../ops/parallel_for/control_flow_ops_test.py |   8 +-
 tensorflow/python/ops/parallel_for/pfor.py    |   8 +-
 28 files changed, 617 insertions(+), 479 deletions(-)
 create mode 100644 tensorflow/python/kernel_tests/map_fn_test.py
 create mode 100644 tensorflow/python/ops/map_fn.py

diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD
index 2b9f5d8dbd..63878d880b 100644
--- a/tensorflow/compiler/tests/BUILD
+++ b/tensorflow/compiler/tests/BUILD
@@ -243,6 +243,7 @@ tf_xla_py_test(
         ":xla_test",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:framework",
+        "//tensorflow/python:map_fn",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform_test",
         "//tensorflow/python:training",
diff --git a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
index 31f0d34ed6..ff19ce2f78 100644
--- a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
+++ b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
@@ -41,9 +41,9 @@ from tensorflow.python.framework import importer
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.ops import functional_ops
 from tensorflow.python.ops import image_ops
 from tensorflow.python.ops import linalg_ops
+from tensorflow.python.ops import map_fn
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn_impl
 from tensorflow.python.ops import nn_ops
@@ -346,7 +346,7 @@ def classifier_score(images, classifier_fn, num_batches=1):
       images, num_or_size_splits=num_batches)
 
   # Compute the classifier splits using the memory-efficient `map_fn`.
-  logits = functional_ops.map_fn(
+  logits = map_fn.map_fn(
       fn=classifier_fn,
       elems=array_ops.stack(generated_images_list),
       parallel_iterations=1,
@@ -505,12 +505,12 @@ def frechet_classifier_distance(real_images,
 
   # Compute the activations using the memory-efficient `map_fn`.
   def compute_activations(elems):
-    return functional_ops.map_fn(fn=classifier_fn,
-                                 elems=elems,
-                                 parallel_iterations=1,
-                                 back_prop=False,
-                                 swap_memory=True,
-                                 name='RunClassifier')
+    return map_fn.map_fn(fn=classifier_fn,
+                         elems=elems,
+                         parallel_iterations=1,
+                         back_prop=False,
+                         swap_memory=True,
+                         name='RunClassifier')
 
   real_a = compute_activations(real_imgs)
   gen_a = compute_activations(generated_imgs)
@@ -895,7 +895,7 @@ def kernel_classifier_distance_and_std(real_images,
 
   # Compute the activations using the memory-efficient `map_fn`.
   def compute_activations(elems):
-    return functional_ops.map_fn(
+    return map_fn.map_fn(
         fn=classifier_fn,
         elems=elems,
         parallel_iterations=1,
@@ -1099,7 +1099,7 @@ def kernel_classifier_distance_and_std_from_activations(real_activations,
             (math_ops.reduce_sum(k_rr) - math_ops.trace(k_rr)) / (m * (m - 1)) +
             (math_ops.reduce_sum(k_gg) - math_ops.trace(k_gg)) / (n * (n - 1)))
 
-  ests = functional_ops.map_fn(
+  ests = map_fn.map_fn(
       compute_kid_block, math_ops.range(n_blocks), dtype=dtype, back_prop=False)
 
   mn = math_ops.reduce_mean(ests)
diff --git a/tensorflow/contrib/gan/python/eval/python/summaries_impl.py b/tensorflow/contrib/gan/python/eval/python/summaries_impl.py
index 9f448d3a16..c7bbd65bbf 100644
--- a/tensorflow/contrib/gan/python/eval/python/summaries_impl.py
+++ b/tensorflow/contrib/gan/python/eval/python/summaries_impl.py
@@ -22,7 +22,7 @@ from tensorflow.contrib.gan.python import namedtuples
 from tensorflow.contrib.gan.python.eval.python import eval_utils
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import functional_ops
+from tensorflow.python.ops import map_fn
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops.losses import util as loss_util
@@ -261,7 +261,7 @@ def add_stargan_image_summaries(stargan_model,
 
   summary.image(
       'stargan_image_generation',
-      functional_ops.map_fn(
+      map_fn.map_fn(
           _build_image,
           stargan_model.input_data[:num_images],
           parallel_iterations=num_images,
diff --git a/tensorflow/contrib/labeled_tensor/BUILD b/tensorflow/contrib/labeled_tensor/BUILD
index 588f15b867..7e19ae7c13 100644
--- a/tensorflow/contrib/labeled_tensor/BUILD
+++ b/tensorflow/contrib/labeled_tensor/BUILD
@@ -155,7 +155,7 @@ py_library(
         ":core",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:functional_ops",
+        "//tensorflow/python:map_fn",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:numerics",
         "//tensorflow/python:random_ops",
diff --git a/tensorflow/contrib/labeled_tensor/python/ops/ops.py b/tensorflow/contrib/labeled_tensor/python/ops/ops.py
index 2ede5daee7..a65f045cc8 100644
--- a/tensorflow/contrib/labeled_tensor/python/ops/ops.py
+++ b/tensorflow/contrib/labeled_tensor/python/ops/ops.py
@@ -29,6 +29,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import functional_ops
+from tensorflow.python.ops import map_fn as map_fn_lib
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import numerics
 from tensorflow.python.ops import random_ops
@@ -629,7 +630,7 @@ def map_fn(fn, labeled_tensor, name=None):
 
     # TODO(ericmc): Fix this upstream.
     if labeled_tensor.dtype == dtypes.string:
-      # We must construct the full graph here, because functional_ops.map_fn
+      # We must construct the full graph here, because map_fn_lib.map_fn
       # doesn't work for string-valued tensors.
       # Constructing the full graph may be slow.
       map_lts = [fn(t) for t in unpack_lts]
@@ -652,7 +653,7 @@ def map_fn(fn, labeled_tensor, name=None):
         tensor_lt = core.LabeledTensor(tensor, original_axes)
         return fn(tensor_lt).tensor
 
-      map_op = functional_ops.map_fn(
+      map_op = map_fn_lib.map_fn(
           tf_fn, labeled_tensor.tensor, dtype=first_map_lt.dtype)
       map_lt = core.LabeledTensor(map_op, final_axes)
 
diff --git a/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator_test.py b/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator_test.py
index 28c4964527..c3e9e3af94 100644
--- a/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator_test.py
+++ b/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator_test.py
@@ -37,8 +37,8 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import random_seed
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import functional_ops
 from tensorflow.python.ops import lookup_ops
+from tensorflow.python.ops import map_fn
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import rnn_cell
@@ -524,7 +524,7 @@ class DynamicRNNEstimatorLearningTest(test.TestCase):
       def input_fn():
         starts = random_ops.random_uniform(
             [batch_size], maxval=(2 * np.pi), seed=seed)
-        sin_curves = functional_ops.map_fn(
+        sin_curves = map_fn.map_fn(
             _sin_fn, (starts,), dtype=dtypes.float32)
         inputs = array_ops.expand_dims(
             array_ops.slice(sin_curves, [0, 0], [batch_size, sequence_length]),
diff --git a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py
index 1b2b6acacc..c63a3ca19b 100644
--- a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py
+++ b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py
@@ -32,7 +32,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.ops import functional_ops
+from tensorflow.python.ops import map_fn
 from tensorflow.python.ops import image_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import parsing_ops
@@ -396,8 +396,8 @@ class Image(ItemHandler):
     image_format = keys_to_tensors[self._format_key]
 
     if self._repeated:
-      return functional_ops.map_fn(lambda x: self._decode(x, image_format),
-                                   image_buffer, dtype=self._dtype)
+      return map_fn.map_fn(lambda x: self._decode(x, image_format),
+                           image_buffer, dtype=self._dtype)
     else:
       return self._decode(image_buffer, image_format)
 
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index c89027582c..813040177d 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -140,6 +140,7 @@ py_library(
         ":lib",
         ":list_ops",
         ":manip_ops",
+        ":map_fn",
         ":math_ops",
         ":metrics",
         ":mode_keys",
@@ -915,6 +916,24 @@ py_library(
     ],
 )
 
+py_library(
+    name = "map_fn",
+    srcs = ["ops/map_fn.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":array_ops",
+        ":constant_op",
+        ":control_flow_ops",
+        ":framework_ops",
+        ":sparse_tensor",
+        ":tensor_array_ops",
+        ":tensor_shape",
+        ":util",
+        ":variable_scope",
+        "//tensorflow/python/eager:context",
+    ],
+)
+
 py_library(
     name = "func_graph",
     srcs = ["framework/func_graph.py"],
@@ -2546,9 +2565,9 @@ py_library(
         ":array_ops",
         ":dtypes",
         ":framework_ops",
-        ":functional_ops",
         ":linalg_ops_gen",
         ":linalg_ops_impl",
+        ":map_fn",
         ":math_ops",
         "//third_party/py/numpy",
     ],
diff --git a/tensorflow/python/data/experimental/benchmarks/map_defun_benchmark.py b/tensorflow/python/data/experimental/benchmarks/map_defun_benchmark.py
index 49297ca7c5..ac6d7d0360 100644
--- a/tensorflow/python/data/experimental/benchmarks/map_defun_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/map_defun_benchmark.py
@@ -25,7 +25,7 @@ from tensorflow.python.eager import function
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import tensor_spec
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import functional_ops
+from tensorflow.python.ops import map_fn
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
@@ -56,14 +56,14 @@ class MapDefunBenchmark(test.Benchmark):
     def defun(x):
       return array_ops.identity(x)
 
-    def map_fn(x):
+    def fn(x):
       return array_ops.identity(x)
 
     base = math_ops.range(100)
     for input_size in [10, 100, 1000, 10000]:
       num_iters = 100000 // input_size
       map_defun_op = map_defun.map_defun(defun, [base], [dtypes.int32], [()])
-      map_fn_op = functional_ops.map_fn(map_fn, base)
+      map_fn_op = map_fn.map_fn(fn, base)
 
       self._run(
           map_defun_op, "with_defun_size_%d" % input_size, num_iters=num_iters)
diff --git a/tensorflow/python/data/kernel_tests/filter_test.py b/tensorflow/python/data/kernel_tests/filter_test.py
index afaf954cbc..cd8f526e72 100644
--- a/tensorflow/python/data/kernel_tests/filter_test.py
+++ b/tensorflow/python/data/kernel_tests/filter_test.py
@@ -24,7 +24,7 @@ from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import test_util
-from tensorflow.python.ops import functional_ops
+from tensorflow.python.ops import map_fn
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
@@ -83,7 +83,7 @@ class FilterTest(test_base.DatasetTestBase):
     # Define a predicate that returns true for the first element of
     # the sequence and not the second, and uses `tf.map_fn()`.
     def _predicate(xs):
-      squared_xs = functional_ops.map_fn(lambda x: x * x, xs)
+      squared_xs = map_fn.map_fn(lambda x: x * x, xs)
       summed = math_ops.reduce_sum(squared_xs)
       return math_ops.equal(summed, 1 + 4 + 9)
 
diff --git a/tensorflow/python/data/kernel_tests/map_test.py b/tensorflow/python/data/kernel_tests/map_test.py
index 97ab6b21bc..8d98d650e1 100644
--- a/tensorflow/python/data/kernel_tests/map_test.py
+++ b/tensorflow/python/data/kernel_tests/map_test.py
@@ -39,8 +39,8 @@ from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import data_flow_ops
-from tensorflow.python.ops import functional_ops
 from tensorflow.python.ops import lookup_ops
+from tensorflow.python.ops import map_fn
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import script_ops
@@ -522,7 +522,7 @@ class MapTest(test_base.DatasetTestBase, parameterized.TestCase):
   def testUseStepContainerInMap(self):
     row = np.arange(6)
     dataset = dataset_ops.Dataset.from_tensors(
-        row).map(lambda elems: functional_ops.map_fn(lambda x: x * x, elems))
+        row).map(lambda elems: map_fn.map_fn(lambda x: x * x, elems))
     self.assertDatasetProduces(dataset, expected_output=[row**2])
 
   def testCaseAndCondInMap(self):
@@ -586,7 +586,7 @@ class MapTest(test_base.DatasetTestBase, parameterized.TestCase):
     def build_dataset(row, num):
       # pylint: disable=g-long-lambda
       dataset = dataset_ops.Dataset.from_tensors(
-          row).map(lambda elems: functional_ops.map_fn(
+          row).map(lambda elems: map_fn.map_fn(
               lambda x: control_map_fn(x, num), elems))
       return self.getNext(dataset)
 
@@ -628,7 +628,7 @@ class MapTest(test_base.DatasetTestBase, parameterized.TestCase):
     num = 2
     # pylint: disable=g-long-lambda
     dataset = dataset_ops.Dataset.from_tensors(
-        row).map(lambda elems: functional_ops.map_fn(
+        row).map(lambda elems: map_fn.map_fn(
             lambda x: control_map_fn(x, num), elems))
     # pylint: enable=g-long-lambda
     get_next = self.getNext(dataset)
diff --git a/tensorflow/python/grappler/layout_optimizer_test.py b/tensorflow/python/grappler/layout_optimizer_test.py
index 98f2e6d718..b6aee941b0 100644
--- a/tensorflow/python/grappler/layout_optimizer_test.py
+++ b/tensorflow/python/grappler/layout_optimizer_test.py
@@ -34,10 +34,10 @@ from tensorflow.python.grappler import cluster as gcluster
 from tensorflow.python.grappler import tf_optimizer
 from tensorflow.python.layers import convolutional as conv_layers
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import functional_ops
 from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import gen_math_ops
 from tensorflow.python.ops import gen_nn_ops
+from tensorflow.python.ops import map_fn
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
 from tensorflow.python.ops import random_ops
@@ -120,7 +120,7 @@ def _loop():
   x3 = random_ops.truncated_normal([1, 784], seed=0)
   x4 = random_ops.truncated_normal([1, 784], seed=0)
   elems = (x1, x2, x3, x4)
-  outputs = functional_ops.map_fn(_two_layer_model, elems, dtype=dtypes.float32)
+  outputs = map_fn.map_fn(_two_layer_model, elems, dtype=dtypes.float32)
   return outputs
 
 
@@ -131,8 +131,7 @@ def _loop_with_branch():
   x3 = random_ops.truncated_normal([1, 784], seed=0)
   x4 = random_ops.truncated_normal([1, 784], seed=0)
   elems = (x1, x2, x3, x4)
-  outputs = functional_ops.map_fn(
-      _model_with_branch, elems, dtype=dtypes.float32)
+  outputs = map_fn.map_fn(_model_with_branch, elems, dtype=dtypes.float32)
   return outputs
 
 
@@ -143,8 +142,7 @@ def _loop_with_vec_and_4d():
   x3 = random_ops.truncated_normal([1, 784], seed=0)
   x4 = random_ops.truncated_normal([1, 784], seed=0)
   elems = (x1, x2, x3, x4)
-  outputs = functional_ops.map_fn(
-      _model_with_vec_and_4d, elems, dtype=dtypes.float32)
+  outputs = map_fn.map_fn(_model_with_vec_and_4d, elems, dtype=dtypes.float32)
   return outputs
 
 
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index ca01d85465..b37bce9057 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -100,6 +100,7 @@ py_library(
         "//tensorflow/python:init_ops",
         "//tensorflow/python:init_ops_v2",
         "//tensorflow/python:logging_ops",
+        "//tensorflow/python:map_fn",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:metrics",
         "//tensorflow/python:nn",
diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index a99c48f1b5..837fd2643a 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -51,6 +51,7 @@ from tensorflow.python.ops import image_ops
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import logging_ops
+from tensorflow.python.ops import map_fn as map_fn_lib
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
 from tensorflow.python.ops import random_ops
@@ -5118,7 +5119,7 @@ def map_fn(fn, elems, name=None, dtype=None):
   Returns:
       Tensor with dtype `dtype`.
   """
-  return functional_ops.map_fn(fn, elems, name=name, dtype=dtype)
+  return map_fn_lib.map_fn(fn, elems, name=name, dtype=dtype)
 
 
 @keras_export('keras.backend.foldl')
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 8fccb0f141..60126bb9fa 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -2077,6 +2077,29 @@ cuda_py_test(
     xla_enable_strict_auto_jit = True,
 )
 
+cuda_py_test(
+    name = "map_fn_test",
+    size = "small",
+    srcs = ["map_fn_test.py"],
+    additional_deps = [
+        "//third_party/py/numpy",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:gradients",
+        "//tensorflow/python:init_ops",
+        "//tensorflow/python:map_fn",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:tensor_array_grad",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python:variables",
+    ],
+    grpc_enabled = True,
+    shard_count = 2,
+    tags = ["no_windows"],
+    xla_enable_strict_auto_jit = True,
+)
+
 cuda_py_test(
     name = "pad_op_test",
     size = "small",
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 3ac53ba9c9..c5b7a95c61 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -58,6 +58,7 @@ from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import logging_ops
+from tensorflow.python.ops import map_fn
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn_grad  # pylint: disable=unused-import
 from tensorflow.python.ops import nn_ops
@@ -497,7 +498,7 @@ class ControlFlowTest(test.TestCase):
         pred = array_ops.placeholder(dtypes.bool, [])
         x = constant_op.constant([1.0, 2.0, 3.0])
         y = control_flow_ops.cond(
-            pred, lambda: functional_ops.map_fn(lambda z: z * 2.0, x),
+            pred, lambda: map_fn.map_fn(lambda z: z * 2.0, x),
             lambda: constant_op.constant([1.0, 1.0, 1.0]))
         g = gradients_impl.gradients(y, x)[0]
 
@@ -2933,10 +2934,10 @@ class ControlFlowTest(test.TestCase):
 
       def inner_loop(t):
         fn = lambda n: n + math_ops.square(var)
-        return functional_ops.map_fn(fn=fn, elems=t, parallel_iterations=10)
+        return map_fn.map_fn(fn=fn, elems=t, parallel_iterations=10)
 
       def outer_loop(inp):
-        return functional_ops.map_fn(
+        return map_fn.map_fn(
             fn=inner_loop, elems=inp, parallel_iterations=10)
 
       var = variables.Variable(constant_op.constant(3.0))
@@ -3122,7 +3123,7 @@ class ControlFlowTest(test.TestCase):
       def b(i, y):
         return [
             i + 1,
-            functional_ops.map_fn(lambda x: math_ops.multiply(x, param), y)
+            map_fn.map_fn(lambda x: math_ops.multiply(x, param), y)
         ]
 
       r = control_flow_ops.while_loop(c, b, [n0, y0], parallel_iterations=1)
diff --git a/tensorflow/python/kernel_tests/functional_ops_test.py b/tensorflow/python/kernel_tests/functional_ops_test.py
index 92f4e7b39e..13f9acbe1b 100644
--- a/tensorflow/python/kernel_tests/functional_ops_test.py
+++ b/tensorflow/python/kernel_tests/functional_ops_test.py
@@ -24,13 +24,11 @@ from tensorflow.core.framework import attr_value_pb2
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.client import session
 from tensorflow.python.data.ops import iterator_ops
-from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import function
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import functional_ops
@@ -193,140 +191,6 @@ class FunctionalOpsTest(test.TestCase):
       self.assertAllEqual(720.0, self.evaluate(r))
   # pylint: enable=unnecessary-lambda
 
-  @test_util.run_in_graph_and_eager_modes
-  def testMap_Simple(self):
-    nums = [1, 2, 3, 4, 5, 6]
-    elems = constant_op.constant(nums, name="data")
-    r = functional_ops.map_fn(
-        lambda x: math_ops.multiply(math_ops.add(x, 3), 2), elems)
-    self.assertAllEqual(
-        np.array([(x + 3) * 2 for x in nums]), self.evaluate(r))
-
-  def testMapDtypeEager(self):
-    with context.eager_mode():
-      dtype = functional_ops.map_fn(lambda x: constant_op.constant(""),
-                                    constant_op.constant([]),
-                                    dtype=dtypes.string).dtype
-      self.assertEqual(dtype, dtypes.string)
-
-  def testMapSparseTensor(self):
-    with self.cached_session():
-      with self.assertRaises(TypeError):
-        functional_ops.map_fn(
-            lambda x: x,
-            sparse_tensor.SparseTensor(
-                indices=[[0, 0], [0, 1], [1, 0]],
-                values=constant_op.constant([0, 1, 2]),
-                dense_shape=[2, 2]))
-
-  @test_util.run_in_graph_and_eager_modes
-  def testMapOverScalarErrors(self):
-    with self.assertRaisesRegexp(ValueError, "not scalars"):
-      functional_ops.map_fn(lambda x: x, [1, 2])
-    with self.assertRaisesRegexp(ValueError, "not a scalar"):
-      functional_ops.map_fn(lambda x: x, 1)
-
-  @test_util.run_deprecated_v1
-  def testMap_Scoped(self):
-    with self.cached_session() as sess:
-
-      def double_scoped(x):
-        """2x with a dummy 2 that is scoped."""
-        with variable_scope.variable_scope("body"):
-          # Dummy variable, just to check that scoping works as intended.
-          two = variable_scope.get_variable(
-              "two", [],
-              dtype=dtypes.int32,
-              initializer=init_ops.constant_initializer(2))
-          return math_ops.multiply(x, two)
-
-      with variable_scope.variable_scope("root") as varscope:
-        elems = constant_op.constant([1, 2, 3, 4, 5, 6], name="data")
-        doubles = np.array([2 * x for x in [1, 2, 3, 4, 5, 6]])
-
-        r = functional_ops.map_fn(double_scoped, elems)
-        # Check that we have the one variable we asked for here.
-        self.assertEqual(len(variables.trainable_variables()), 1)
-        self.assertEqual(variables.trainable_variables()[0].name,
-                         "root/body/two:0")
-        sess.run([variables.global_variables_initializer()])
-        self.assertAllEqual(doubles, self.evaluate(r))
-
-        # Now let's reuse our single variable.
-        varscope.reuse_variables()
-        r = functional_ops.map_fn(double_scoped, elems)
-        self.assertEqual(len(variables.trainable_variables()), 1)
-        self.assertAllEqual(doubles, self.evaluate(r))
-
-  @test_util.run_deprecated_v1
-  def testMap_Grad(self):
-    with self.cached_session():
-      param = constant_op.constant(2.0)
-      elems = constant_op.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], name="elems")
-      y = functional_ops.map_fn(
-          lambda x: math_ops.multiply(math_ops.square(x), param), elems)
-      r = gradients_impl.gradients(y, param)[0]
-      self.assertAllEqual(91.0, self.evaluate(r))
-      r = gradients_impl.gradients(y, elems)[0]
-      self.assertAllEqual([4.0, 8.0, 12.0, 16.0, 20.0, 24.0], self.evaluate(r))
-
-  @test_util.run_in_graph_and_eager_modes
-  def testMap_SimpleNotTensor(self):
-    nums = np.array([1, 2, 3, 4, 5, 6])
-    r = functional_ops.map_fn(
-        lambda x: math_ops.multiply(math_ops.add(x, 3), 2), nums)
-    self.assertAllEqual(
-        np.array([(x + 3) * 2 for x in nums]), self.evaluate(r))
-
-  @test_util.run_in_graph_and_eager_modes
-  def testMap_SingleInputMultiOutput(self):
-    nums = np.array([1, 2, 3, 4, 5, 6])
-    r = functional_ops.map_fn(
-        lambda x: ((x + 3) * 2, -(x + 3) * 2),
-        nums,
-        dtype=(dtypes.int64, dtypes.int64))
-    self.assertEqual(2, len(r))
-    self.assertEqual((6,), r[0].get_shape())
-    self.assertEqual((6,), r[1].get_shape())
-    received = self.evaluate(r)
-    self.assertAllEqual((nums + 3) * 2, received[0])
-    self.assertAllEqual(-(nums + 3) * 2, received[1])
-
-  @test_util.run_in_graph_and_eager_modes
-  def testMap_MultiOutputMismatchedDtype(self):
-    nums = np.array([1, 2, 3, 4, 5, 6])
-    with self.assertRaisesRegexp(
-        TypeError, r"two structures don't have the same nested structure"):
-      # lambda emits tuple, but dtype is a list
-      functional_ops.map_fn(
-          lambda x: ((x + 3) * 2, -(x + 3) * 2),
-          nums,
-          dtype=[dtypes.int64, dtypes.int64])
-
-  @test_util.run_in_graph_and_eager_modes
-  def testMap_MultiInputSingleOutput(self):
-    nums = np.array([1, 2, 3, 4, 5, 6])
-    r = functional_ops.map_fn(
-        lambda x: x[0] * x[1][0] + x[1][1], (nums, (nums, -nums)),
-        dtype=dtypes.int64)
-    self.assertEqual((6,), r.get_shape())
-    received = self.evaluate(r)
-    self.assertAllEqual(nums * nums + (-nums), received)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testMap_MultiInputSameStructureOutput(self):
-    nums = np.array([1, 2, 3, 4, 5, 6])
-    r = functional_ops.map_fn(lambda x: (x[1][0], (x[1][1], x[0])),
-                              (nums, (2 * nums, -nums)))
-    r = [r[0], r[1][0], r[1][1]]
-    self.assertEqual((6,), r[0].get_shape())
-    self.assertEqual((6,), r[1].get_shape())
-    self.assertEqual((6,), r[2].get_shape())
-    received = self.evaluate(r)
-    self.assertAllEqual(2 * nums, received[0])
-    self.assertAllEqual(-nums, received[1])
-    self.assertAllEqual(nums, received[2])
-
   @test_util.run_in_graph_and_eager_modes
   def testScan_Simple(self):
     elems = constant_op.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], name="data")
@@ -489,37 +353,6 @@ class FunctionalOpsTest(test.TestCase):
     y = functional_ops.foldl(fn, x, initializer=initializer)
     self.assertAllEqual(y.get_shape(), self.evaluate(y).shape)
 
-  @test_util.run_in_graph_and_eager_modes
-  def testMapShape(self):
-    x = constant_op.constant([[1, 2, 3], [4, 5, 6]])
-    y = functional_ops.map_fn(lambda e: e, x)
-    self.assertAllEqual(y.get_shape(), self.evaluate(y).shape)
-
-  @test_util.run_deprecated_v1
-  def testMapUnknownShape(self):
-    x = array_ops.placeholder(dtypes.float32)
-    y = functional_ops.map_fn(lambda e: e, x)
-    self.assertIs(None, y.get_shape().dims)
-
-  @test_util.disable_control_flow_v2("b/119323354")
-  @test_util.run_in_graph_and_eager_modes
-  @test_util.run_v1_only("b/120545219")
-  def testMapEmptyScalar(self):
-    map_return = functional_ops.map_fn(lambda x: 1, constant_op.constant([]))
-    self.assertAllEqual([0], map_return.get_shape().dims)
-    self.assertAllEqual([0], self.evaluate(map_return).shape)
-
-  # TODO(akshayka): this test fails in eager: the iterable is of length 0 so
-  # so the body of the while loop never executes
-  @test_util.disable_control_flow_v2("b/119323354")
-  @test_util.run_v1_only("b/120545219")
-  def testMapEmptyTensor(self):
-    with self.cached_session():
-      map_return = functional_ops.map_fn(lambda x: array_ops.zeros([3, 2]),
-                                         constant_op.constant([]))
-      self.assertAllEqual([0, 3, 2], map_return.get_shape().dims)
-      self.assertAllEqual([0, 3, 2], self.evaluate(map_return).shape)
-
   @test_util.run_in_graph_and_eager_modes
   def testScanShape(self):
     x = constant_op.constant([[1, 2, 3], [4, 5, 6]])
diff --git a/tensorflow/python/kernel_tests/lu_op_test.py b/tensorflow/python/kernel_tests/lu_op_test.py
index 06deb0e1c8..951af020fe 100644
--- a/tensorflow/python/kernel_tests/lu_op_test.py
+++ b/tensorflow/python/kernel_tests/lu_op_test.py
@@ -27,8 +27,8 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.ops import functional_ops
 from tensorflow.python.ops import linalg_ops
+from tensorflow.python.ops import map_fn
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import variables
@@ -104,8 +104,8 @@ class LuOpTest(test.TestCase):
         verification_reshaped = array_ops.reshape(verification,
                                                   [-1, num_rows, num_cols])
         # Invert the permutation in each batch.
-        inv_perm_reshaped = functional_ops.map_fn(array_ops.invert_permutation,
-                                                  perm_reshaped)
+        inv_perm_reshaped = map_fn.map_fn(array_ops.invert_permutation,
+                                          perm_reshaped)
         batch_size = perm_reshaped.shape.as_list()[0]
         # Prepare the batch indices with the same shape as the permutation.
         # The corresponding batch index is paired with each of the `num_rows`
diff --git a/tensorflow/python/kernel_tests/map_fn_test.py b/tensorflow/python/kernel_tests/map_fn_test.py
new file mode 100644
index 0000000000..a31db416a2
--- /dev/null
+++ b/tensorflow/python/kernel_tests/map_fn_test.py
@@ -0,0 +1,224 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for tensorflow.kernels.functional_ops."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.eager import context
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gradients_impl
+from tensorflow.python.ops import init_ops
+from tensorflow.python.ops import map_fn
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.ops import variables
+import tensorflow.python.ops.tensor_array_grad  # pylint: disable=unused-import
+from tensorflow.python.platform import test
+
+
+# pylint: disable=invalid-name
+def simple_scoped_fn(a, x):
+  """Simple function: (a, x) -> 2(x+a), but with "2" as a variable in scope."""
+  with variable_scope.variable_scope("body"):
+    # Dummy variable, just to check that scoping works as intended.
+    two = variable_scope.get_variable(
+        "two", [],
+        dtype=dtypes.int32,
+        initializer=init_ops.constant_initializer(2))
+    return math_ops.multiply(math_ops.add(a, x), two)
+
+
+@test_util.with_control_flow_v2
+@test_util.disable_all_xla("This test never passed for XLA")
+class MapFnTest(test.TestCase):
+
+  @test_util.run_in_graph_and_eager_modes
+  def testMap_Simple(self):
+    nums = [1, 2, 3, 4, 5, 6]
+    elems = constant_op.constant(nums, name="data")
+    r = map_fn.map_fn(
+        lambda x: math_ops.multiply(math_ops.add(x, 3), 2), elems)
+    self.assertAllEqual(
+        np.array([(x + 3) * 2 for x in nums]), self.evaluate(r))
+
+  def testMapDtypeEager(self):
+    with context.eager_mode():
+      dtype = map_fn.map_fn(lambda x: constant_op.constant(""),
+                            constant_op.constant([]),
+                            dtype=dtypes.string).dtype
+      self.assertEqual(dtype, dtypes.string)
+
+  def testMapSparseTensor(self):
+    with self.cached_session():
+      with self.assertRaises(TypeError):
+        map_fn.map_fn(
+            lambda x: x,
+            sparse_tensor.SparseTensor(
+                indices=[[0, 0], [0, 1], [1, 0]],
+                values=constant_op.constant([0, 1, 2]),
+                dense_shape=[2, 2]))
+
+  @test_util.run_in_graph_and_eager_modes
+  def testMapOverScalarErrors(self):
+    with self.assertRaisesRegexp(ValueError, "not scalars"):
+      map_fn.map_fn(lambda x: x, [1, 2])
+    with self.assertRaisesRegexp(ValueError, "not a scalar"):
+      map_fn.map_fn(lambda x: x, 1)
+
+  @test_util.run_deprecated_v1
+  def testMap_Scoped(self):
+    with self.cached_session() as sess:
+
+      def double_scoped(x):
+        """2x with a dummy 2 that is scoped."""
+        with variable_scope.variable_scope("body"):
+          # Dummy variable, just to check that scoping works as intended.
+          two = variable_scope.get_variable(
+              "two", [],
+              dtype=dtypes.int32,
+              initializer=init_ops.constant_initializer(2))
+          return math_ops.multiply(x, two)
+
+      with variable_scope.variable_scope("root") as varscope:
+        elems = constant_op.constant([1, 2, 3, 4, 5, 6], name="data")
+        doubles = np.array([2 * x for x in [1, 2, 3, 4, 5, 6]])
+
+        r = map_fn.map_fn(double_scoped, elems)
+        # Check that we have the one variable we asked for here.
+        self.assertEqual(len(variables.trainable_variables()), 1)
+        self.assertEqual(variables.trainable_variables()[0].name,
+                         "root/body/two:0")
+        sess.run([variables.global_variables_initializer()])
+        self.assertAllEqual(doubles, self.evaluate(r))
+
+        # Now let's reuse our single variable.
+        varscope.reuse_variables()
+        r = map_fn.map_fn(double_scoped, elems)
+        self.assertEqual(len(variables.trainable_variables()), 1)
+        self.assertAllEqual(doubles, self.evaluate(r))
+
+  @test_util.run_deprecated_v1
+  def testMap_Grad(self):
+    with self.cached_session():
+      param = constant_op.constant(2.0)
+      elems = constant_op.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], name="elems")
+      y = map_fn.map_fn(
+          lambda x: math_ops.multiply(math_ops.square(x), param), elems)
+      r = gradients_impl.gradients(y, param)[0]
+      self.assertAllEqual(91.0, self.evaluate(r))
+      r = gradients_impl.gradients(y, elems)[0]
+      self.assertAllEqual([4.0, 8.0, 12.0, 16.0, 20.0, 24.0], self.evaluate(r))
+
+  @test_util.run_in_graph_and_eager_modes
+  def testMap_SimpleNotTensor(self):
+    nums = np.array([1, 2, 3, 4, 5, 6])
+    r = map_fn.map_fn(
+        lambda x: math_ops.multiply(math_ops.add(x, 3), 2), nums)
+    self.assertAllEqual(
+        np.array([(x + 3) * 2 for x in nums]), self.evaluate(r))
+
+  @test_util.run_in_graph_and_eager_modes
+  def testMap_SingleInputMultiOutput(self):
+    nums = np.array([1, 2, 3, 4, 5, 6])
+    r = map_fn.map_fn(
+        lambda x: ((x + 3) * 2, -(x + 3) * 2),
+        nums,
+        dtype=(dtypes.int64, dtypes.int64))
+    self.assertEqual(2, len(r))
+    self.assertEqual((6,), r[0].get_shape())
+    self.assertEqual((6,), r[1].get_shape())
+    received = self.evaluate(r)
+    self.assertAllEqual((nums + 3) * 2, received[0])
+    self.assertAllEqual(-(nums + 3) * 2, received[1])
+
+  @test_util.run_in_graph_and_eager_modes
+  def testMap_MultiOutputMismatchedDtype(self):
+    nums = np.array([1, 2, 3, 4, 5, 6])
+    with self.assertRaisesRegexp(
+        TypeError, r"two structures don't have the same nested structure"):
+      # lambda emits tuple, but dtype is a list
+      map_fn.map_fn(
+          lambda x: ((x + 3) * 2, -(x + 3) * 2),
+          nums,
+          dtype=[dtypes.int64, dtypes.int64])
+
+  @test_util.run_in_graph_and_eager_modes
+  def testMap_MultiInputSingleOutput(self):
+    nums = np.array([1, 2, 3, 4, 5, 6])
+    r = map_fn.map_fn(
+        lambda x: x[0] * x[1][0] + x[1][1], (nums, (nums, -nums)),
+        dtype=dtypes.int64)
+    self.assertEqual((6,), r.get_shape())
+    received = self.evaluate(r)
+    self.assertAllEqual(nums * nums + (-nums), received)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testMap_MultiInputSameStructureOutput(self):
+    nums = np.array([1, 2, 3, 4, 5, 6])
+    r = map_fn.map_fn(lambda x: (x[1][0], (x[1][1], x[0])),
+                      (nums, (2 * nums, -nums)))
+    r = [r[0], r[1][0], r[1][1]]
+    self.assertEqual((6,), r[0].get_shape())
+    self.assertEqual((6,), r[1].get_shape())
+    self.assertEqual((6,), r[2].get_shape())
+    received = self.evaluate(r)
+    self.assertAllEqual(2 * nums, received[0])
+    self.assertAllEqual(-nums, received[1])
+    self.assertAllEqual(nums, received[2])
+
+  @test_util.run_in_graph_and_eager_modes
+  def testMapShape(self):
+    x = constant_op.constant([[1, 2, 3], [4, 5, 6]])
+    y = map_fn.map_fn(lambda e: e, x)
+    self.assertAllEqual(y.get_shape(), self.evaluate(y).shape)
+
+  @test_util.run_deprecated_v1
+  def testMapUnknownShape(self):
+    x = array_ops.placeholder(dtypes.float32)
+    y = map_fn.map_fn(lambda e: e, x)
+    self.assertIs(None, y.get_shape().dims)
+
+  @test_util.disable_control_flow_v2("b/119323354")
+  @test_util.run_in_graph_and_eager_modes
+  @test_util.run_v1_only("b/120545219")
+  def testMapEmptyScalar(self):
+    map_return = map_fn.map_fn(lambda x: 1, constant_op.constant([]))
+    self.assertAllEqual([0], map_return.get_shape().dims)
+    self.assertAllEqual([0], self.evaluate(map_return).shape)
+
+  # TODO(akshayka): this test fails in eager: the iterable is of length 0 so
+  # so the body of the while loop never executes
+  @test_util.disable_control_flow_v2("b/119323354")
+  @test_util.run_v1_only("b/120545219")
+  def testMapEmptyTensor(self):
+    with self.cached_session():
+      map_return = map_fn.map_fn(lambda x: array_ops.zeros([3, 2]),
+                                 constant_op.constant([]))
+      self.assertAllEqual([0, 3, 2], map_return.get_shape().dims)
+      self.assertAllEqual([0, 3, 2], self.evaluate(map_return).shape)
+
+
+if __name__ == "__main__":
+  test.main()
+
+# pylint: enable=invalid-name
diff --git a/tensorflow/python/kernel_tests/while_v2_test.py b/tensorflow/python/kernel_tests/while_v2_test.py
index 061d787760..2c6d275900 100644
--- a/tensorflow/python/kernel_tests/while_v2_test.py
+++ b/tensorflow/python/kernel_tests/while_v2_test.py
@@ -32,9 +32,9 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.grappler import tf_optimizer
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import functional_ops
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import list_ops
+from tensorflow.python.ops import map_fn
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.ops import while_v2
@@ -394,7 +394,7 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     param = constant_op.constant(2.0)
     y0 = constant_op.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], name="elems")
     # map_fn uses TensorArray internally.
-    r = functional_ops.map_fn(lambda x: math_ops.multiply(x, param), y0)
+    r = map_fn.map_fn(lambda x: math_ops.multiply(x, param), y0)
     grad = gradients_impl.gradients(r, param)[0]
     self.assertAllClose([2.0, 4.0, 6.0, 8.0, 10.0, 12.0], self.evaluate(r))
     self.assertAllClose(21.0, self.evaluate(grad))
diff --git a/tensorflow/python/ops/ctc_ops.py b/tensorflow/python/ops/ctc_ops.py
index 45286f7c18..359719f053 100644
--- a/tensorflow/python/ops/ctc_ops.py
+++ b/tensorflow/python/ops/ctc_ops.py
@@ -31,6 +31,7 @@ from tensorflow.python.ops import functional_ops
 from tensorflow.python.ops import gen_ctc_ops
 from tensorflow.python.ops import inplace_ops
 from tensorflow.python.ops import linalg_ops
+from tensorflow.python.ops import map_fn
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import sparse_ops
@@ -904,7 +905,7 @@ def ctc_unique_labels(labels, name=None):
           u.y, [[0, _get_dim(u.idx, 0) - _get_dim(u.y, 0)]])
       y = math_ops.cast(y, dtypes.int64)
       return [y, u.idx]
-    return functional_ops.map_fn(
+    return map_fn.map_fn(
         _unique, labels, dtype=[dtypes.int64, dtypes.int32])
 
 
diff --git a/tensorflow/python/ops/distributions/multinomial.py b/tensorflow/python/ops/distributions/multinomial.py
index 97d2b1b26c..1b2dbfaf9f 100644
--- a/tensorflow/python/ops/distributions/multinomial.py
+++ b/tensorflow/python/ops/distributions/multinomial.py
@@ -23,7 +23,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.ops import functional_ops
+from tensorflow.python.ops import map_fn
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import random_ops
@@ -266,7 +266,7 @@ class Multinomial(distribution.Distribution):
       x = math_ops.reduce_sum(array_ops.one_hot(x, depth=k), axis=-2)  # [n, k]
       return x
 
-    x = functional_ops.map_fn(
+    x = map_fn.map_fn(
         _sample_single, [flat_logits, flat_ndraws],
         dtype=self.dtype)  # [B1B2...Bm, n, k]
 
diff --git a/tensorflow/python/ops/functional_ops.py b/tensorflow/python/ops/functional_ops.py
index b240d1e465..324f55a89f 100644
--- a/tensorflow/python/ops/functional_ops.py
+++ b/tensorflow/python/ops/functional_ops.py
@@ -25,7 +25,6 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import function
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
@@ -37,7 +36,6 @@ from tensorflow.python.ops import variable_scope as vs
 from tensorflow.python.ops.gen_functional_ops import remote_call
 # pylint: enable=unused-import
 from tensorflow.python.ops.gen_functional_ops import symbolic_gradient
-from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import compat
 from tensorflow.python.util import function_utils
 from tensorflow.python.util import nest
@@ -265,255 +263,6 @@ def foldr(fn, elems, initializer=None, parallel_iterations=10, back_prop=True,
     return r_a
 
 
-@tf_export("map_fn")
-def map_fn(fn, elems, dtype=None, parallel_iterations=None, back_prop=True,
-           swap_memory=False, infer_shape=True, name=None):
-  """map on the list of tensors unpacked from `elems` on dimension 0.
-
-  The simplest version of `map_fn` repeatedly applies the callable `fn` to a
-  sequence of elements from first to last. The elements are made of the
-  tensors unpacked from `elems`. `dtype` is the data type of the return
-  value of `fn`. Users must provide `dtype` if it is different from
-  the data type of `elems`.
-
-  Suppose that `elems` is unpacked into `values`, a list of tensors. The shape
-  of the result tensor is `[values.shape[0]] + fn(values[0]).shape`.
-
-  This method also allows multi-arity `elems` and output of `fn`.  If `elems`
-  is a (possibly nested) list or tuple of tensors, then each of these tensors
-  must have a matching first (unpack) dimension.  The signature of `fn` may
-  match the structure of `elems`.  That is, if `elems` is
-  `(t1, [t2, t3, [t4, t5]])`, then an appropriate signature for `fn` is:
-  `fn = lambda (t1, [t2, t3, [t4, t5]]):`.
-
-  Furthermore, `fn` may emit a different structure than its input.  For example,
-  `fn` may look like: `fn = lambda t1: return (t1 + 1, t1 - 1)`.  In this case,
-  the `dtype` parameter is not optional: `dtype` must be a type or (possibly
-  nested) tuple of types matching the output of `fn`.
-
-  To apply a functional operation to the nonzero elements of a SparseTensor
-  one of the following methods is recommended. First, if the function is
-  expressible as TensorFlow ops, use
-
-  ```python
-    result = SparseTensor(input.indices, fn(input.values), input.dense_shape)
-  ```
-
-  If, however, the function is not expressible as a TensorFlow op, then use
-
-  ```python
-  result = SparseTensor(
-    input.indices, map_fn(fn, input.values), input.dense_shape)
-  ```
-
-  instead.
-
-  When executing eagerly, map_fn does not execute in parallel even if
-  `parallel_iterations` is set to a value > 1. You can still get the
-  performance benefits of running a function in parallel by using the
-  `tf.contrib.eager.defun` decorator,
-
-  ```python
-  # Assume the function being used in map_fn is fn.
-  # To ensure map_fn calls fn in parallel, use the defun decorator.
-  @tf.contrib.eager.defun
-  def func(tensor):
-    return tf.map_fn(fn, tensor)
-  ```
-
-  Note that if you use the defun decorator, any non-TensorFlow Python code
-  that you may have written in your function won't get executed. See
-  `tf.contrib.eager.defun` for more details. The recommendation would be to
-  debug without defun but switch to defun to get performance benefits of
-  running map_fn in parallel.
-
-  Args:
-    fn: The callable to be performed.  It accepts one argument, which will
-      have the same (possibly nested) structure as `elems`.  Its output
-      must have the same structure as `dtype` if one is provided, otherwise
-      it must have the same structure as `elems`.
-    elems: A tensor or (possibly nested) sequence of tensors, each of which
-      will be unpacked along their first dimension.  The nested sequence
-      of the resulting slices will be applied to `fn`.
-    dtype: (optional) The output type(s) of `fn`.  If `fn` returns a structure
-      of Tensors differing from the structure of `elems`, then `dtype` is not
-      optional and must have the same structure as the output of `fn`.
-    parallel_iterations: (optional) The number of iterations allowed to run
-      in parallel. When graph building, the default value is 10. While executing
-      eagerly, the default value is set to 1.
-    back_prop: (optional) True enables support for back propagation.
-    swap_memory: (optional) True enables GPU-CPU memory swapping.
-    infer_shape: (optional) False disables tests for consistent output shapes.
-    name: (optional) Name prefix for the returned tensors.
-
-  Returns:
-    A tensor or (possibly nested) sequence of tensors.  Each tensor packs the
-    results of applying `fn` to tensors unpacked from `elems` along the first
-    dimension, from first to last.
-
-  Raises:
-    TypeError: if `fn` is not callable or the structure of the output of
-      `fn` and `dtype` do not match, or if elems is a SparseTensor.
-    ValueError: if the lengths of the output of `fn` and `dtype` do not match.
-
-  Examples:
-    ```python
-    elems = np.array([1, 2, 3, 4, 5, 6])
-    squares = map_fn(lambda x: x * x, elems)
-    # squares == [1, 4, 9, 16, 25, 36]
-    ```
-
-    ```python
-    elems = (np.array([1, 2, 3]), np.array([-1, 1, -1]))
-    alternate = map_fn(lambda x: x[0] * x[1], elems, dtype=tf.int64)
-    # alternate == [-1, 2, -3]
-    ```
-
-    ```python
-    elems = np.array([1, 2, 3])
-    alternates = map_fn(lambda x: (x, -x), elems, dtype=(tf.int64, tf.int64))
-    # alternates[0] == [1, 2, 3]
-    # alternates[1] == [-1, -2, -3]
-    ```
-  """
-  if not callable(fn):
-    raise TypeError("fn must be callable.")
-
-  if isinstance(elems, sparse_tensor.SparseTensor):
-    raise TypeError(
-        "To perform a map on the values of a sparse tensor use either "
-        " SparseTensor(input.indices, fn(input.values), input.dense_shape) or "
-        " SparseTensor(input.indices, map_fn(fn, input.values), "
-        "input.dense_shape)")
-
-  in_graph_mode = not context.executing_eagerly()
-  # Set the default number of parallel_iterations depending on graph/eager mode.
-  if in_graph_mode and not parallel_iterations:
-    parallel_iterations = 10
-  elif not in_graph_mode and not parallel_iterations:
-    parallel_iterations = 1
-
-  if not in_graph_mode and parallel_iterations > 1:
-    logging.log_first_n(logging.WARN, "Setting parallel_iterations > 1 has no "
-                        "effect when executing eagerly. Consider calling map_fn"
-                        " with tf.contrib.eager.defun to execute fn in "
-                        "parallel.", 1)
-    parallel_iterations = 1
-
-  input_is_sequence = nest.is_sequence(elems)
-  input_flatten = lambda x: nest.flatten(x) if input_is_sequence else [x]
-  def input_pack(x):
-    return nest.pack_sequence_as(elems, x) if input_is_sequence else x[0]
-
-  if dtype is None:
-    output_is_sequence = input_is_sequence
-    output_flatten = input_flatten
-    output_pack = input_pack
-  else:
-    output_is_sequence = nest.is_sequence(dtype)
-    output_flatten = lambda x: nest.flatten(x) if output_is_sequence else [x]
-    def output_pack(x):
-      return (nest.pack_sequence_as(dtype, x)
-              if output_is_sequence else x[0])
-
-  elems_flat = input_flatten(elems)
-
-  with ops.name_scope(name, "map", elems_flat):
-    # TODO(akshayka): Remove the in_graph_mode check once caching devices are
-    # supported in Eager
-    if in_graph_mode:
-      # Any get_variable calls in fn will cache the first call locally
-      # and not issue repeated network I/O requests for each iteration.
-      varscope = vs.get_variable_scope()
-      varscope_caching_device_was_none = False
-      if varscope.caching_device is None:
-        # TODO(ebrevdo): Change to using colocate_with here and in other
-        # methods.
-        varscope.set_caching_device(lambda op: op.device)
-        varscope_caching_device_was_none = True
-
-    elems_flat = [
-        ops.convert_to_tensor(elem, name="elem") for elem in elems_flat]
-
-    dtype = dtype or input_pack([elem.dtype for elem in elems_flat])
-    dtype_flat = output_flatten(dtype)
-
-    # Convert elems to tensor array. n may be known statically.
-    static_shape = elems_flat[0].shape
-    if static_shape.ndims is not None and static_shape.ndims < 1:
-      if len(elems_flat) == 1:
-        raise ValueError("elems must be a 1+ dimensional Tensor, not a scalar")
-      else:
-        raise ValueError(
-            "elements in elems must be 1+ dimensional Tensors, not scalars"
-        )
-    n = (tensor_shape.dimension_value(static_shape[0])
-         or array_ops.shape(elems_flat[0])[0])
-
-    # TensorArrays are always flat
-    elems_ta = [
-        tensor_array_ops.TensorArray(dtype=elem.dtype, size=n,
-                                     dynamic_size=False,
-                                     infer_shape=True)
-        for elem in elems_flat]
-    # Unpack elements
-    elems_ta = [
-        elem_ta.unstack(elem) for elem_ta, elem in zip(elems_ta, elems_flat)]
-
-    i = constant_op.constant(0)
-
-    accs_ta = [
-        tensor_array_ops.TensorArray(dtype=dt, size=n,
-                                     dynamic_size=False,
-                                     infer_shape=infer_shape)
-        for dt in dtype_flat]
-
-    def compute(i, tas):
-      """The loop body of map_fn.
-
-      Args:
-        i: the loop counter
-        tas: the flat TensorArray accumulator list
-
-      Returns:
-        (i + 1, tas): the updated counter + updated TensorArrays
-
-      Raises:
-        TypeError: if dtype and packed_fn_values structure do not match
-        ValueType: if dtype and packed_fn_values lengths do not match
-      """
-      packed_values = input_pack([elem_ta.read(i) for elem_ta in elems_ta])
-      packed_fn_values = fn(packed_values)
-      nest.assert_same_structure(dtype or elems, packed_fn_values)
-      flat_fn_values = output_flatten(packed_fn_values)
-      tas = [ta.write(i, value) for (ta, value) in zip(tas, flat_fn_values)]
-      return (i + 1, tas)
-
-    _, r_a = control_flow_ops.while_loop(
-        lambda i, _: i < n, compute, (i, accs_ta),
-        parallel_iterations=parallel_iterations,
-        back_prop=back_prop,
-        swap_memory=swap_memory,
-        maximum_iterations=n)
-    results_flat = [r.stack() for r in r_a]
-
-    n_static = tensor_shape.Dimension(tensor_shape.dimension_value(
-        elems_flat[0].get_shape().with_rank_at_least(1)[0]))
-    for elem in elems_flat[1:]:
-      n_static.merge_with(tensor_shape.Dimension(tensor_shape.dimension_value(
-          elem.get_shape().with_rank_at_least(1)[0])))
-    for r in results_flat:
-      r.set_shape(tensor_shape.TensorShape(n_static).concatenate(
-          r.get_shape()[1:]))
-
-    # TODO(akshayka): Remove the in_graph_mode check once caching devices are
-    # supported in Eager
-    if in_graph_mode and varscope_caching_device_was_none:
-      varscope.set_caching_device(None)
-
-    return output_pack(results_flat)
-
-
 @tf_export("scan")
 def scan(fn, elems, initializer=None, parallel_iterations=10, back_prop=True,
          swap_memory=False, infer_shape=True, reverse=False, name=None):
diff --git a/tensorflow/python/ops/linalg_ops.py b/tensorflow/python/ops/linalg_ops.py
index 1a9e7112b4..e8652ba44a 100644
--- a/tensorflow/python/ops/linalg_ops.py
+++ b/tensorflow/python/ops/linalg_ops.py
@@ -24,9 +24,9 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.ops import functional_ops
 from tensorflow.python.ops import gen_linalg_ops
 from tensorflow.python.ops import linalg_ops_impl
+from tensorflow.python.ops import map_fn
 from tensorflow.python.ops import math_ops
 # pylint: disable=wildcard-import
 from tensorflow.python.ops.gen_linalg_ops import *
@@ -595,14 +595,14 @@ def norm(tensor,
     if ord in ['fro', 'euclidean', 2, 2.0]:
       if is_matrix_norm and ord in [2, 2.0]:
         rank = array_ops.rank(tensor)
-        positive_axis = functional_ops.map_fn(
+        positive_axis = map_fn.map_fn(
             lambda i: control_flow_ops.cond(i >= 0, lambda: i, lambda: i + rank),
             ops.convert_to_tensor(axis))
         axes = math_ops.range(rank)
         perm_before = array_ops.concat(
             [array_ops.setdiff1d(axes, positive_axis)[0], positive_axis],
             axis=0)
-        perm_after = functional_ops.map_fn(
+        perm_after = map_fn.map_fn(
             lambda i: math_ops.cast(
                 array_ops.squeeze(
                     array_ops.where(math_ops.equal(perm_before, i))),
diff --git a/tensorflow/python/ops/map_fn.py b/tensorflow/python/ops/map_fn.py
new file mode 100644
index 0000000000..6767e3eb22
--- /dev/null
+++ b/tensorflow/python/ops/map_fn.py
@@ -0,0 +1,285 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =============================================================================
+
+"""Functional operations."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+from tensorflow.python.eager import context
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import tensor_array_ops
+from tensorflow.python.ops import variable_scope as vs
+from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.util import nest
+from tensorflow.python.util.tf_export import tf_export
+
+
+@tf_export("map_fn")
+def map_fn(fn, elems, dtype=None, parallel_iterations=None, back_prop=True,
+           swap_memory=False, infer_shape=True, name=None):
+  """map on the list of tensors unpacked from `elems` on dimension 0.
+
+  The simplest version of `map_fn` repeatedly applies the callable `fn` to a
+  sequence of elements from first to last. The elements are made of the
+  tensors unpacked from `elems`. `dtype` is the data type of the return
+  value of `fn`. Users must provide `dtype` if it is different from
+  the data type of `elems`.
+
+  Suppose that `elems` is unpacked into `values`, a list of tensors. The shape
+  of the result tensor is `[values.shape[0]] + fn(values[0]).shape`.
+
+  This method also allows multi-arity `elems` and output of `fn`.  If `elems`
+  is a (possibly nested) list or tuple of tensors, then each of these tensors
+  must have a matching first (unpack) dimension.  The signature of `fn` may
+  match the structure of `elems`.  That is, if `elems` is
+  `(t1, [t2, t3, [t4, t5]])`, then an appropriate signature for `fn` is:
+  `fn = lambda (t1, [t2, t3, [t4, t5]]):`.
+
+  Furthermore, `fn` may emit a different structure than its input.  For example,
+  `fn` may look like: `fn = lambda t1: return (t1 + 1, t1 - 1)`.  In this case,
+  the `dtype` parameter is not optional: `dtype` must be a type or (possibly
+  nested) tuple of types matching the output of `fn`.
+
+  To apply a functional operation to the nonzero elements of a SparseTensor
+  one of the following methods is recommended. First, if the function is
+  expressible as TensorFlow ops, use
+
+  ```python
+    result = SparseTensor(input.indices, fn(input.values), input.dense_shape)
+  ```
+
+  If, however, the function is not expressible as a TensorFlow op, then use
+
+  ```python
+  result = SparseTensor(
+    input.indices, map_fn(fn, input.values), input.dense_shape)
+  ```
+
+  instead.
+
+  When executing eagerly, map_fn does not execute in parallel even if
+  `parallel_iterations` is set to a value > 1. You can still get the
+  performance benefits of running a function in parallel by using the
+  `tf.contrib.eager.defun` decorator,
+
+  ```python
+  # Assume the function being used in map_fn is fn.
+  # To ensure map_fn calls fn in parallel, use the defun decorator.
+  @tf.contrib.eager.defun
+  def func(tensor):
+    return tf.map_fn(fn, tensor)
+  ```
+
+  Note that if you use the defun decorator, any non-TensorFlow Python code
+  that you may have written in your function won't get executed. See
+  `tf.contrib.eager.defun` for more details. The recommendation would be to
+  debug without defun but switch to defun to get performance benefits of
+  running map_fn in parallel.
+
+  Args:
+    fn: The callable to be performed.  It accepts one argument, which will
+      have the same (possibly nested) structure as `elems`.  Its output
+      must have the same structure as `dtype` if one is provided, otherwise
+      it must have the same structure as `elems`.
+    elems: A tensor or (possibly nested) sequence of tensors, each of which
+      will be unpacked along their first dimension.  The nested sequence
+      of the resulting slices will be applied to `fn`.
+    dtype: (optional) The output type(s) of `fn`.  If `fn` returns a structure
+      of Tensors differing from the structure of `elems`, then `dtype` is not
+      optional and must have the same structure as the output of `fn`.
+    parallel_iterations: (optional) The number of iterations allowed to run
+      in parallel. When graph building, the default value is 10. While executing
+      eagerly, the default value is set to 1.
+    back_prop: (optional) True enables support for back propagation.
+    swap_memory: (optional) True enables GPU-CPU memory swapping.
+    infer_shape: (optional) False disables tests for consistent output shapes.
+    name: (optional) Name prefix for the returned tensors.
+
+  Returns:
+    A tensor or (possibly nested) sequence of tensors.  Each tensor packs the
+    results of applying `fn` to tensors unpacked from `elems` along the first
+    dimension, from first to last.
+
+  Raises:
+    TypeError: if `fn` is not callable or the structure of the output of
+      `fn` and `dtype` do not match, or if elems is a SparseTensor.
+    ValueError: if the lengths of the output of `fn` and `dtype` do not match.
+
+  Examples:
+    ```python
+    elems = np.array([1, 2, 3, 4, 5, 6])
+    squares = map_fn(lambda x: x * x, elems)
+    # squares == [1, 4, 9, 16, 25, 36]
+    ```
+
+    ```python
+    elems = (np.array([1, 2, 3]), np.array([-1, 1, -1]))
+    alternate = map_fn(lambda x: x[0] * x[1], elems, dtype=tf.int64)
+    # alternate == [-1, 2, -3]
+    ```
+
+    ```python
+    elems = np.array([1, 2, 3])
+    alternates = map_fn(lambda x: (x, -x), elems, dtype=(tf.int64, tf.int64))
+    # alternates[0] == [1, 2, 3]
+    # alternates[1] == [-1, -2, -3]
+    ```
+  """
+  if not callable(fn):
+    raise TypeError("fn must be callable.")
+
+  if isinstance(elems, sparse_tensor.SparseTensor):
+    raise TypeError(
+        "To perform a map on the values of a sparse tensor use either "
+        " SparseTensor(input.indices, fn(input.values), input.dense_shape) or "
+        " SparseTensor(input.indices, map_fn(fn, input.values), "
+        "input.dense_shape)")
+
+  in_graph_mode = not context.executing_eagerly()
+  # Set the default number of parallel_iterations depending on graph/eager mode.
+  if in_graph_mode and not parallel_iterations:
+    parallel_iterations = 10
+  elif not in_graph_mode and not parallel_iterations:
+    parallel_iterations = 1
+
+  if not in_graph_mode and parallel_iterations > 1:
+    logging.log_first_n(logging.WARN, "Setting parallel_iterations > 1 has no "
+                        "effect when executing eagerly. Consider calling map_fn"
+                        " with tf.contrib.eager.defun to execute fn in "
+                        "parallel.", 1)
+    parallel_iterations = 1
+
+  input_is_sequence = nest.is_sequence(elems)
+  input_flatten = lambda x: nest.flatten(x) if input_is_sequence else [x]
+  def input_pack(x):
+    return nest.pack_sequence_as(elems, x) if input_is_sequence else x[0]
+
+  if dtype is None:
+    output_is_sequence = input_is_sequence
+    output_flatten = input_flatten
+    output_pack = input_pack
+  else:
+    output_is_sequence = nest.is_sequence(dtype)
+    output_flatten = lambda x: nest.flatten(x) if output_is_sequence else [x]
+    def output_pack(x):
+      return (nest.pack_sequence_as(dtype, x)
+              if output_is_sequence else x[0])
+
+  elems_flat = input_flatten(elems)
+
+  with ops.name_scope(name, "map", elems_flat):
+    # TODO(akshayka): Remove the in_graph_mode check once caching devices are
+    # supported in Eager
+    if in_graph_mode:
+      # Any get_variable calls in fn will cache the first call locally
+      # and not issue repeated network I/O requests for each iteration.
+      varscope = vs.get_variable_scope()
+      varscope_caching_device_was_none = False
+      if varscope.caching_device is None:
+        # TODO(ebrevdo): Change to using colocate_with here and in other
+        # methods.
+        varscope.set_caching_device(lambda op: op.device)
+        varscope_caching_device_was_none = True
+
+    elems_flat = [
+        ops.convert_to_tensor(elem, name="elem") for elem in elems_flat]
+
+    dtype = dtype or input_pack([elem.dtype for elem in elems_flat])
+    dtype_flat = output_flatten(dtype)
+
+    # Convert elems to tensor array. n may be known statically.
+    static_shape = elems_flat[0].shape
+    if static_shape.ndims is not None and static_shape.ndims < 1:
+      if len(elems_flat) == 1:
+        raise ValueError("elems must be a 1+ dimensional Tensor, not a scalar")
+      else:
+        raise ValueError(
+            "elements in elems must be 1+ dimensional Tensors, not scalars"
+        )
+    n = (tensor_shape.dimension_value(static_shape[0])
+         or array_ops.shape(elems_flat[0])[0])
+
+    # TensorArrays are always flat
+    elems_ta = [
+        tensor_array_ops.TensorArray(dtype=elem.dtype,
+                                     size=n,
+                                     dynamic_size=False,
+                                     infer_shape=True)
+        for elem in elems_flat]
+    # Unpack elements
+    elems_ta = [
+        elem_ta.unstack(elem) for elem_ta, elem in zip(elems_ta, elems_flat)]
+
+    i = constant_op.constant(0)
+
+    accs_ta = [
+        tensor_array_ops.TensorArray(dtype=dt,
+                                     size=n,
+                                     dynamic_size=False,
+                                     infer_shape=infer_shape)
+        for dt in dtype_flat]
+
+    def compute(i, tas):
+      """The loop body of map_fn.
+
+      Args:
+        i: the loop counter
+        tas: the flat TensorArray accumulator list
+
+      Returns:
+        (i + 1, tas): the updated counter + updated TensorArrays
+
+      Raises:
+        TypeError: if dtype and packed_fn_values structure do not match
+        ValueType: if dtype and packed_fn_values lengths do not match
+      """
+      packed_values = input_pack([elem_ta.read(i) for elem_ta in elems_ta])
+      packed_fn_values = fn(packed_values)
+      nest.assert_same_structure(dtype or elems, packed_fn_values)
+      flat_fn_values = output_flatten(packed_fn_values)
+      tas = [ta.write(i, value) for (ta, value) in zip(tas, flat_fn_values)]
+      return (i + 1, tas)
+
+    _, r_a = control_flow_ops.while_loop(
+        lambda i, _: i < n, compute, (i, accs_ta),
+        parallel_iterations=parallel_iterations,
+        back_prop=back_prop,
+        swap_memory=swap_memory,
+        maximum_iterations=n)
+    results_flat = [r.stack() for r in r_a]
+
+    n_static = tensor_shape.Dimension(tensor_shape.dimension_value(
+        elems_flat[0].get_shape().with_rank_at_least(1)[0]))
+    for elem in elems_flat[1:]:
+      n_static.merge_with(tensor_shape.Dimension(tensor_shape.dimension_value(
+          elem.get_shape().with_rank_at_least(1)[0])))
+    for r in results_flat:
+      r.set_shape(tensor_shape.TensorShape(n_static).concatenate(
+          r.get_shape()[1:]))
+
+    # TODO(akshayka): Remove the in_graph_mode check once caching devices are
+    # supported in Eager
+    if in_graph_mode and varscope_caching_device_was_none:
+      varscope.set_caching_device(None)
+
+    return output_pack(results_flat)
diff --git a/tensorflow/python/ops/parallel_for/BUILD b/tensorflow/python/ops/parallel_for/BUILD
index fd89d8ad45..05d2e4c7fc 100644
--- a/tensorflow/python/ops/parallel_for/BUILD
+++ b/tensorflow/python/ops/parallel_for/BUILD
@@ -31,6 +31,7 @@ py_library(
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:functional_ops",
         "//tensorflow/python:gradients",
+        "//tensorflow/python:map_fn",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:nn_ops",
         "//tensorflow/python:platform",
@@ -56,7 +57,7 @@ py_library(
         "//tensorflow/python:data_flow_ops",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
-        "//tensorflow/python:functional_ops",
+        "//tensorflow/python:map_fn",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:nn_ops",
         "//tensorflow/python:platform",
diff --git a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
index 8a5830e28f..1b1b336bd0 100644
--- a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
+++ b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
@@ -36,9 +36,9 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import bitwise_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import data_flow_ops
-from tensorflow.python.ops import functional_ops
 from tensorflow.python.ops import gradients as gradient_ops
 from tensorflow.python.ops import logging_ops
+from tensorflow.python.ops import map_fn
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
 from tensorflow.python.ops import parsing_ops
@@ -928,15 +928,15 @@ class Benchmarks(test.Benchmark):
       b = 256
       params = 1000
       inp = random_ops.random_normal((b, params))
-      map_fn = lambda x: x * x
+      fn = lambda x: x * x
 
       def pfor_map_fn(f, x):
         return pfor_control_flow_ops.pfor(
             lambda i: f(array_ops.gather(x, i)),
             array_ops.shape(x)[0])
 
-      map_output = functional_ops.map_fn(map_fn, inp)
-      pfor_output = pfor_map_fn(map_fn, inp)
+      map_output = map_fn.map_fn(fn, inp)
+      pfor_output = pfor_map_fn(fn, inp)
 
       self._run(map_output, 100, name="tf_map_fn")
       self._run(pfor_output, 100, name="pfor_map_fn")
diff --git a/tensorflow/python/ops/parallel_for/pfor.py b/tensorflow/python/ops/parallel_for/pfor.py
index a22c1126c9..b0f6a6a443 100644
--- a/tensorflow/python/ops/parallel_for/pfor.py
+++ b/tensorflow/python/ops/parallel_for/pfor.py
@@ -32,9 +32,9 @@ from tensorflow.python.ops import bitwise_ops
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import data_flow_ops
-from tensorflow.python.ops import functional_ops
 from tensorflow.python.ops import gen_parsing_ops
 from tensorflow.python.ops import gen_sparse_ops
+from tensorflow.python.ops import map_fn
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import parsing_ops
@@ -1037,7 +1037,7 @@ class PFor(object):
     if sparse_tensor_rank is not None:
       sparse_tensor_rank += 1
 
-    def map_fn(args):
+    def fn(args):
       res = gen_sparse_ops.serialize_sparse(
           args[0], args[1], args[2], out_type=dtypes.variant)
       return res
@@ -1046,8 +1046,8 @@ class PFor(object):
     # sparse tensor element and batch them all, then deserializes the batch.
     # TODO(rachelim): Try to do this without map_fn -- add the right offsets
     # to shape and indices tensors instead.
-    result = functional_ops.map_fn(
-        map_fn, [indices, values, shape], dtype=dtypes.variant)
+    result = map_fn.map_fn(
+        fn, [indices, values, shape], dtype=dtypes.variant)
     return sparse_ops.deserialize_sparse(
         result, dtype=values.dtype, rank=sparse_tensor_rank)
 
-- 
GitLab


From 799acd00e2d317f8ad4ae2cd9a27c502577d9a8c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 29 Jan 2019 22:14:56 -0800
Subject: [PATCH 1566/2345] Update numpy version >=1.14.5

PiperOrigin-RevId: 231534268
---
 tensorflow/tools/pip_package/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index 368cb171ba..bbe3050513 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -54,7 +54,7 @@ REQUIRED_PACKAGES = [
     'google_pasta >= 0.1.1',
     'keras_applications >= 1.0.6',
     'keras_preprocessing >= 1.0.5',
-    'numpy >= 1.13.3',
+    'numpy >= 1.14.5, < 2.0',
     'six >= 1.10.0',
     'protobuf >= 3.6.1',
     'tensorboard >= 1.12.0, < 1.13.0',
-- 
GitLab


From 30e8b66c22cf031babdf81b977cc57ac5cc74192 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 29 Jan 2019 23:53:44 -0800
Subject: [PATCH 1567/2345] Optimize SparseMatMulOp::Compute in two ways:  - A
 significant portion if the time in  is spent comparing bfloat16s to zero.
 This is costly because operator==(bfloat16, bfloat16) does two conversions to
 float. Avoid conversions to float to do the comparison by adding a fast test
 for equality of bfloat16 to zero. This gains 0% to 7% for bfloat16. To get an
 idea of why this is better, see: https://godbolt.org/z/T1zqvx.  - Avoid a
 modulo computation + a switch which is redundant with internal loop checks +
 resize calls. This gains 0% to 1.5% depending on the benchmark.

This is never slower than the previous code and is up to 7% faster.

PiperOrigin-RevId: 231541527
---
 tensorflow/core/kernels/sparse_matmul_op.cc | 72 +++++++++++----------
 tensorflow/core/lib/bfloat16/bfloat16.h     |  2 +
 2 files changed, 41 insertions(+), 33 deletions(-)

diff --git a/tensorflow/core/kernels/sparse_matmul_op.cc b/tensorflow/core/kernels/sparse_matmul_op.cc
index 2ea7a1ed3b..2c1f613c18 100644
--- a/tensorflow/core/kernels/sparse_matmul_op.cc
+++ b/tensorflow/core/kernels/sparse_matmul_op.cc
@@ -48,6 +48,8 @@ limitations under the License.
 #include "tensorflow/core/kernels/eigen_contraction_kernel.h"
 #endif
 
+#define ALWAYS_INLINE EIGEN_ALWAYS_INLINE
+
 namespace tensorflow {
 namespace {
 
@@ -161,6 +163,19 @@ struct SparseSlice {
   const int block_size;
 };
 
+template <typename T>
+bool IsZero(T v);
+
+template <>
+ALWAYS_INLINE bool IsZero(bfloat16 v) {
+  return v.IsZero();
+}
+
+template <>
+ALWAYS_INLINE bool IsZero(float v) {
+  return v == 0.0f;
+}
+
 template <typename T>
 template <bool Transpose>
 void SparseSlice<T>::Initialize(
@@ -182,9 +197,8 @@ void SparseSlice<T>::Initialize(
   index.reserve(num_blocks * num_rows * 2);
 
   Index3 idx3;
-  Index idx;
-  int data3_size = 0;
-  static const T zero(0);
+  const int stride = Transpose ? mat.dimension(1) : 1;
+
   for (int i = 0; i < num_blocks; ++i) {
     int num_block_cols = std::min(block_size, num_cols - block_size * i);
     for (int row = 0; row < num_rows; ++row) {
@@ -196,54 +210,48 @@ void SparseSlice<T>::Initialize(
       const auto* start =
           Transpose ? &mat(col_offset, row) : &mat(row, col_offset);
       const auto* curr = start;
-      const int stride = Transpose ? mat.dimension(1) : 1;
       const auto* end = start + stride * num_block_cols;
       uint8 k = 0;
 #define NEXT_ELEM \
   curr += stride; \
   ++k;
+#define EAT_ZEROS                          \
+  while (curr < end && IsZero<T>(*curr)) { \
+    NEXT_ELEM;                             \
+  }
       while (true) {
-        while (curr < end && (*curr == zero)) {
-          NEXT_ELEM;
-        }
+        EAT_ZEROS
         if (curr >= end) break;
         idx3.k1 = k;
-        data3.push_back(*curr);
+        const T value1 = *curr;
         NEXT_ELEM;
 
-        while (curr < end && (*curr == zero)) {
-          NEXT_ELEM;
+        EAT_ZEROS
+        if (curr >= end) {
+          data.push_back(value1);
+          index.push_back({idx3.m, idx3.k1});
+          break;
         }
-        if (curr >= end) break;
         idx3.k2 = k;
-        data3.push_back(*curr);
+        const T value2 = *curr;
         NEXT_ELEM;
 
-        while (curr < end && (*curr == zero)) {
-          NEXT_ELEM;
+        EAT_ZEROS
+        if (curr >= end) {
+          data.push_back(value2);
+          index.push_back({idx3.m, idx3.k2});
+          data.push_back(value1);
+          index.push_back({idx3.m, idx3.k1});
+          break;
         }
-        if (curr >= end) break;
         idx3.k3 = k;
+        data3.push_back(value1);
+        data3.push_back(value2);
         data3.push_back(*curr);
         NEXT_ELEM;
         index3.push_back(idx3);
 #undef NEXT_ELEM
-      }
-      int num_inserted_mod = data3.size() % 3;
-      // Move some elements to index and data if needed.
-      data3_size = data3.size() - num_inserted_mod;
-      idx.m = idx3.m;
-      switch (num_inserted_mod) {
-        case 2:
-          idx.k = idx3.k2;
-          data.push_back(data3[data3_size + 1]);
-          index.push_back(idx);
-          TF_FALLTHROUGH_INTENDED;
-        case 1:
-          idx.k = idx3.k1;
-          data.push_back(data3[data3_size]);
-          index.push_back(idx);
-          data3.resize(data3_size);
+#undef EAT_ZEROS
       }
     }
     col_offset += block_size;
@@ -276,8 +284,6 @@ const int kNumOperands = (sizeof(Packet) / sizeof(float));
 #define STORE(x, y) Eigen::internal::pstore<float>(x, y);
 #define FMA(a, b, c, d) d = Eigen::internal::pmadd<Packet>(a, b, c);
 
-#define ALWAYS_INLINE EIGEN_ALWAYS_INLINE
-
 ALWAYS_INLINE float ConvertBfloat16ToFloat(const bfloat16* src) {
   float out = 0;
   auto tmp = reinterpret_cast<bfloat16*>(&out);
diff --git a/tensorflow/core/lib/bfloat16/bfloat16.h b/tensorflow/core/lib/bfloat16/bfloat16.h
index 4408546580..9a7c635c6e 100644
--- a/tensorflow/core/lib/bfloat16/bfloat16.h
+++ b/tensorflow/core/lib/bfloat16/bfloat16.h
@@ -372,6 +372,8 @@ struct bfloat16 {
     return x;
   }
 
+  bool IsZero() const { return (value & 0x7FFF) == ZERO_VALUE; }
+
   uint16_t value;
 
   // A value that represents "not a number".
-- 
GitLab


From 9ffbf1aa5e90ed33defc1ea5cf1eaa45bf8f587b Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Wed, 30 Jan 2019 00:43:25 -0800
Subject: [PATCH 1568/2345] Support functions without a return value. Do this
 by filtering out ops.Operation if returned.

PiperOrigin-RevId: 231546655
---
 .../saved_model/function_deserialization.py   |  6 ++++-
 tensorflow/python/saved_model/load_test.py    | 26 +++++++++++++++++++
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py
index 3ad289e275..e82e64260c 100644
--- a/tensorflow/python/saved_model/function_deserialization.py
+++ b/tensorflow/python/saved_model/function_deserialization.py
@@ -160,7 +160,11 @@ def recreate_function(saved_function, concrete_functions):
                             canonicalized_original_inputs):
         flattened_inputs = nest.flatten(canonicalized_inputs)
         filtered_inputs = [t for t in flattened_inputs if _is_tensor(t)]
-        return function_obj._call_flat(filtered_inputs)  # pylint: disable=protected-access
+
+        result = function_obj._call_flat(filtered_inputs)  # pylint: disable=protected-access
+        if isinstance(result, ops.Operation):
+          return None
+        return result
 
     raise AssertionError(
         "Could not find matching function to call for canonicalized inputs %r. "
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index 61fcea3748..299b6aefa0 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -257,6 +257,32 @@ class LoadTest(test.TestCase, parameterized.TestCase):
                             constant_op.constant([1.0, 2.0, 3.0]),
                             dtype=dtypes.int32).numpy())
 
+  def test_function_no_return(self, cycles):
+
+    class CheckpointableWithOneVariable(tracking.AutoCheckpointable):
+
+      def __init__(self, initial_value=0.0):
+        super(CheckpointableWithOneVariable, self).__init__()
+        self.variable = variables.Variable(initial_value)
+
+      @def_function.function
+      def increase(self, by=1.0):
+        self.variable.assign_add(by)
+
+    obj = CheckpointableWithOneVariable(5.0)
+
+    obj.increase(constant_op.constant(10.0))
+    self.assertEqual(15.0, obj.variable.numpy())
+    obj.increase()
+    self.assertEqual(16.0, obj.variable.numpy())
+
+    imported = self.cycle(obj, cycles)
+
+    imported.increase(constant_op.constant(10.0))
+    self.assertEqual(26.0, imported.variable.numpy())
+    imported.increase(constant_op.constant(1.0))
+    self.assertEqual(27.0, imported.variable.numpy())
+
   def test_structured_inputs(self, cycles):
 
     def func(x, training=True):
-- 
GitLab


From c5c997ffa1b65ca85666c7083acbf9c64a5e935d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 30 Jan 2019 01:02:29 -0800
Subject: [PATCH 1569/2345] compat: Update forward compatibility horizon to
 2019-01-30

PiperOrigin-RevId: 231548585
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index fa8b68078e..4f84bcba37 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 29)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 30)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From a93f06d160955c99bc279a419902ac40824a2cab Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Wed, 30 Jan 2019 01:13:07 -0800
Subject: [PATCH 1570/2345] Add a test and improve error message for variables
 that don't have a checkpointable name.

PiperOrigin-RevId: 231550245
---
 .../saved_model/function_serialization.py     |  8 ++++++--
 tensorflow/python/saved_model/save_test.py    | 19 +++++++++++++++++++
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/saved_model/function_serialization.py b/tensorflow/python/saved_model/function_serialization.py
index 986614e5d0..95dfec7b69 100644
--- a/tensorflow/python/saved_model/function_serialization.py
+++ b/tensorflow/python/saved_model/function_serialization.py
@@ -45,8 +45,12 @@ def serialize_concrete_function(concrete_function, node_ids, coder):
       bound_inputs.append(node_ids[capture])
   except KeyError:
     raise KeyError(
-        "Concrete function %s not added to object based saved model as it "
-        "captures tensor %s which is unsupported or not reachable from root."
+        "Failed to add concrete function %s to object based saved model as it "
+        "captures tensor %s which is unsupported or not reachable from root. "
+        "One reason could be that a stateful object or a variable that the "
+        "function depends on is not assigned to an attribute of the serialized "
+        "checkpointable object "
+        "(see SaveTest.test_captures_unreachable_variable)."
         % (concrete_function.name, capture))
   concrete_function_proto = saved_object_graph_pb2.SavedConcreteFunction()
   structured_outputs = func_graph_module.convert_structure_to_signature(
diff --git a/tensorflow/python/saved_model/save_test.py b/tensorflow/python/saved_model/save_test.py
index fa79e45bf1..cbca51aea7 100644
--- a/tensorflow/python/saved_model/save_test.py
+++ b/tensorflow/python/saved_model/save_test.py
@@ -122,6 +122,25 @@ class SaveTest(test.TestCase):
         ValueError, "Expected a TensorFlow function"):
       save.save(root, save_dir, root.f)
 
+  def test_captures_unreachable_variable(self):
+    root = tracking.AutoCheckpointable()
+    unreachable_variable = variables.Variable([5.0, 2.0])
+    root.reachable_variable = variables.Variable([1.0, 3.0])
+
+    @def_function.function
+    def increase_variable(x):
+      return 2 * unreachable_variable * x + root.reachable_variable
+
+    root.f = increase_variable
+
+    self.assertAllEqual([101.0, 83.0],
+                        root.f(constant_op.constant([10.0, 20.0])).numpy())
+
+    save_dir = os.path.join(self.get_temp_dir(), "saved_model")
+
+    with self.assertRaisesRegexp(KeyError, "not reachable from root"):
+      save.save(root, save_dir)
+
   def test_nested_inputs(self):
     root = tracking.AutoCheckpointable()
     root.f = def_function.function(
-- 
GitLab


From 24a7280bd173a15437a7e52e2544ca4d009f16f4 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Wed, 30 Jan 2019 02:26:37 -0800
Subject: [PATCH 1571/2345] Remove the optimization to replace sort with
 scatter.

We will switch to allowing to specify a computation for sorting.
This will make it even less likely that this optimization will kick in.
TopK will specify a computation to sort into descending order, and
the current logic assumes that the sorting is done into ascending order.

PiperOrigin-RevId: 231557946
---
 .../xla/service/algebraic_simplifier.cc       | 103 ------------------
 .../xla/service/algebraic_simplifier.h        |  12 --
 .../xla/service/algebraic_simplifier_test.cc  |  73 -------------
 .../xla/service/gpu/nvptx_compiler.cc         |   2 -
 4 files changed, 190 deletions(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index e456b92500..549e675d65 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -3142,109 +3142,6 @@ Status AlgebraicSimplifierVisitor::HandleSort(HloInstruction* sort) {
     return ReplaceWithNewInstruction(
         sort, HloInstruction::CreateTuple(sort->operands()));
   }
-
-  if (!options_.enable_permutation_sort_replacement()) {
-    return Status::OK();
-  }
-  // Check if we are sorting a permutation. In that case, we know that the keys
-  // will be sorted to the identity permutation, and we can represent the
-  // changes to the 'values' parameter as a scatter.
-  if (sort->operand_count() == 2 &&
-      operand->opcode() == HloOpcode::kGetTupleElement) {
-    const HloInstruction* other_sort = operand->operand(0);
-    // Check whether the 'values' parameter is the result of another sort with
-    // the same sort dimension.
-    if (other_sort->opcode() == HloOpcode::kSort &&
-        other_sort->operand_count() >= 2 &&
-        other_sort->dimensions(0) == dimension_to_sort &&
-        other_sort->operand(operand->tuple_index())->opcode() ==
-            HloOpcode::kIota) {
-      auto* iota =
-          Cast<HloIotaInstruction>(other_sort->operand(operand->tuple_index()));
-      // The sort operand needs to be an integral iota, and the iota dimension
-      // needs to be the dimension that was sorted.
-      if (iota->iota_dimension() == dimension_to_sort &&
-          ShapeUtil::ElementIsIntegral(iota->shape())) {
-        // We use the following construction method for a Scatter that applies
-        // the permutation from 'keys' to the 'values' parameter.
-        // - Take the "keys" parameter of the second sort and reshape it to have
-        //   another "1" dimension at the end.
-        // - Concatenate it with iotas of the same extended shape with all
-        //   different iota_dimensions except the dimension_to_sort in the order
-        //   of iota_dimensions/dimension_to_sort, so e.g. with rank 3 and
-        //   dimension_to_sort = 1, we would have concatenate of (iota with
-        //   iota_dimension=0, keys, iota with iota_dimension = 2)
-        // - Use this as the indices parameter of scatter, and set updates
-        //   of the scatter to be a reshaped 'values' parameter of sort (adding
-        //   'rank' many 1 dimensions at the end).
-        int64 rank = operand->shape().rank();
-        Shape extended_shape = operand->shape();
-        extended_shape.add_dimensions(1);
-        extended_shape.mutable_layout()->add_minor_to_major(rank);
-        auto reshaped_permutation = computation_->AddInstruction(
-            HloInstruction::CreateReshape(extended_shape, operand));
-        std::vector<HloInstruction*> concat_operands;
-        for (int64 i = 0; i < rank; ++i) {
-          if (i == dimension_to_sort) {
-            concat_operands.push_back(reshaped_permutation);
-          } else {
-            concat_operands.push_back(computation_->AddInstruction(
-                HloInstruction::CreateIota(extended_shape, i)));
-          }
-        }
-        Shape concat_shape = operand->shape();
-        concat_shape.add_dimensions(rank);
-        concat_shape.mutable_layout()->add_minor_to_major(rank);
-        auto scatter_indices =
-            rank > 1 ? computation_->AddInstruction(
-                           HloInstruction::CreateConcatenate(
-                               concat_shape, concat_operands, rank))
-                     : reshaped_permutation;
-
-        // We don't care about the operand, it will be completely overridden by
-        // the updates.
-        auto scatter_operand = computation_->AddInstruction(
-            HloInstruction::CreateIota(sort->operand(1)->shape(), 0));
-
-        // Construct the updates operand of scatter.
-        Shape update_shape = sort->operand(1)->shape();
-        for (int64 i = 0; i < rank; ++i) {
-          update_shape.add_dimensions(1);
-          update_shape.mutable_layout()->add_minor_to_major(rank + i);
-        }
-        auto scatter_updates =
-            computation_->AddInstruction(HloInstruction::CreateReshape(
-                update_shape, sort->mutable_operand(1)));
-
-        // Construct the updates computation, which simply replaces the operand
-        // values with the update values.
-        HloComputation::Builder b("update_replace_computation");
-        Shape scalar_shape = ShapeUtil::MakeShape(S32, {});
-        b.AddInstruction(
-            HloInstruction::CreateParameter(0, scalar_shape, "scalar_lhs"));
-        auto scalar_rhs = b.AddInstruction(
-            HloInstruction::CreateParameter(1, scalar_shape, "scalar_rhs"));
-        auto update_replace_computation =
-            computation_->parent()->AddEmbeddedComputation(b.Build(scalar_rhs));
-
-        ScatterDimensionNumbers dim_numbers;
-        dim_numbers.set_index_vector_dim(rank);
-        for (int64 i = 0; i < rank; ++i) {
-          dim_numbers.add_update_window_dims(rank + i);
-          dim_numbers.add_scatter_dims_to_operand_dims(i);
-        }
-        auto scatter =
-            computation_->AddInstruction(HloInstruction::CreateScatter(
-                sort->operand(1)->shape(), scatter_operand, scatter_indices,
-                scatter_updates, update_replace_computation, dim_numbers));
-        return ReplaceWithNewInstruction(
-            sort, HloInstruction::CreateTuple(
-                      {computation_->AddInstruction(HloInstruction::CreateIota(
-                           operand->shape(), dimension_to_sort)),
-                       scatter}));
-      }
-    }
-  }
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.h b/tensorflow/compiler/xla/service/algebraic_simplifier.h
index ff3f638b22..df5a8c2ec1 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.h
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.h
@@ -71,17 +71,6 @@ class AlgebraicSimplifierOptions {
     return enable_conv_simplification_;
   }
 
-  // If enable_permutation_sort_replacement is true, a sort op that is known to
-  // sort a permutation will be replaced with a scatter op.
-  void set_enable_permutation_sort_replacement(
-      bool enable_permutation_sort_replacement) {
-    enable_permutation_sort_replacement_ = enable_permutation_sort_replacement;
-  }
-
-  bool enable_permutation_sort_replacement() const {
-    return enable_permutation_sort_replacement_;
-  }
-
   // If enable_window_reduce_replacement is true, the kReduceWindow instruction
   // can be optimized by replacement with simpler operations.
   void set_enable_window_reduce_to_reduce_replacement(
@@ -99,7 +88,6 @@ class AlgebraicSimplifierOptions {
   bool is_layout_sensitive_{false};
   bool enable_dot_strength_reduction_{true};
   bool enable_conv_simplification_{true};
-  bool enable_permutation_sort_replacement_{false};
   bool enable_window_reduce_to_reduce_replacement_{true};
 };
 
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index 089312211e..b5fed239d8 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -2759,79 +2759,6 @@ TEST_F(AlgebraicSimplifierTest, RemoveNoopSort) {
   EXPECT_THAT(computation->root_instruction(), keys);
 }
 
-TEST_F(AlgebraicSimplifierTest, ReplacePermutationSortWithScatter) {
-  const char* hlo_string = R"(
-    HloModule permutation_sort
-
-    ENTRY sort_computation {
-      keys = f32[64,8732]{1,0} parameter(0)
-      values = s32[64,8732]{1,0} iota(), iota_dimension=1
-      sort = (f32[64,8732]{1,0}, s32[64,8732]{1,0}) sort(keys, values), dimensions={1}
-      gte = s32[64,8732]{1,0} get-tuple-element(sort), index=1
-      ROOT sort2 = (s32[64,8732]{1,0}, s32[64,8732]{1,0}) sort(gte, values), dimensions={1}
-    }
-  )";
-  TF_ASSERT_OK_AND_ASSIGN(auto module,
-                          ParseAndReturnVerifiedModule(hlo_string));
-
-  AlgebraicSimplifierOptions options;
-  options.set_enable_permutation_sort_replacement(true);
-  AlgebraicSimplifier simplifier(options);
-  EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
-  auto root = module->entry_computation()->root_instruction();
-  EXPECT_THAT(root,
-              GmockMatch(m::Tuple(
-                  m::Iota(),
-                  m::Scatter(m::Iota(), m::Concatenate(m::Iota(), m::Reshape()),
-                             m::Reshape()))));
-}
-
-TEST_F(AlgebraicSimplifierTest, DontReplacePermutationSortIfNonIntegral) {
-  // Same as ReplacePermutationSortWithScatter except that the iota has F32
-  // type.
-  const char* hlo_string = R"(
-    HloModule permutation_sort
-
-    ENTRY sort_computation {
-      keys = f32[64,8732]{1,0} parameter(0)
-      values = f32[64,8732]{1,0} iota(), iota_dimension=1
-      sort = (f32[64,8732]{1,0}, f32[64,8732]{1,0}) sort(keys, values), dimensions={1}
-      gte = f32[64,8732]{1,0} get-tuple-element(sort), index=1
-      ROOT sort2 = (f32[64,8732]{1,0}, f32[64,8732]{1,0}) sort(gte, values), dimensions={1}
-    }
-  )";
-  TF_ASSERT_OK_AND_ASSIGN(auto module,
-                          ParseAndReturnVerifiedModule(hlo_string));
-
-  AlgebraicSimplifierOptions options;
-  options.set_enable_permutation_sort_replacement(true);
-  AlgebraicSimplifier simplifier(options);
-  EXPECT_FALSE(simplifier.Run(module.get()).ValueOrDie());
-}
-
-TEST_F(AlgebraicSimplifierTest, DontReplacePermutationSortWrongDimensions) {
-  // Same as ReplacePermutationSortWithScatter except that the sort dimensions
-  // don't match.
-  const char* hlo_string = R"(
-   HloModule permutation_sort
-
-    ENTRY sort_computation {
-      keys = f32[64,8732]{1,0} parameter(0)
-      values = s32[64,8732]{1,0} iota(), iota_dimension=1
-      sort = (f32[64,8732]{1,0}, s32[64,8732]{1,0}) sort(keys, values), dimensions={1}
-      gte = s32[64,8732]{1,0} get-tuple-element(sort), index=1
-      ROOT sort2 = (s32[64,8732]{1,0}, s32[64,8732]{1,0}) sort(gte, values), dimensions={0}
-    }
-  )";
-  TF_ASSERT_OK_AND_ASSIGN(auto module,
-                          ParseAndReturnVerifiedModule(hlo_string));
-
-  AlgebraicSimplifierOptions options;
-  options.set_enable_permutation_sort_replacement(true);
-  AlgebraicSimplifier simplifier(options);
-  EXPECT_FALSE(simplifier.Run(module.get()).ValueOrDie());
-}
-
 TEST_F(AlgebraicSimplifierTest, ReplaceEffectiveScalarKeyValueSortWithTuple) {
   auto builder = HloComputation::Builder(TestName());
 
diff --git a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
index e1b91dfb1a..1f4f176661 100644
--- a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
+++ b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
@@ -201,7 +201,6 @@ Status OptimizeHloModule(HloModule* hlo_module, se::StreamExecutor* stream_exec,
       pipeline.AddPass<ZeroSizedHloElimination>();
 
       AlgebraicSimplifierOptions options;
-      options.set_enable_permutation_sort_replacement(true);
       pass.AddPass<AlgebraicSimplifier>(options);
       pass.AddPass<SortSimplifier>();
       pass.AddPass<TupleSimplifier>();
@@ -274,7 +273,6 @@ Status OptimizeHloModule(HloModule* hlo_module, se::StreamExecutor* stream_exec,
     // duplicate or NOPs, so remove them with algebraic simplification and CSE.
     AlgebraicSimplifierOptions options;
     options.set_is_layout_sensitive(true);
-    options.set_enable_permutation_sort_replacement(true);
     pipeline.AddPass<HloPassFix<AlgebraicSimplifier>>(options);
 
     // Choose the fastest algorithm for each conv.
-- 
GitLab


From b34901da3f3e052ad7a930bce2084cc5478cbf1d Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Wed, 30 Jan 2019 03:48:43 -0800
Subject: [PATCH 1572/2345] Allow max_pool3d to be more flexible in the inputs
 it allows

PiperOrigin-RevId: 231567605
---
 .../python_api/api_def_MaxPool3D.pbtxt        |  4 +-
 tensorflow/python/ops/nn_ops.py               | 45 +++++++++++++++++++
 2 files changed, 46 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/api_def/python_api/api_def_MaxPool3D.pbtxt b/tensorflow/core/api_def/python_api/api_def_MaxPool3D.pbtxt
index e8576c9ff2..47016b9d69 100644
--- a/tensorflow/core/api_def/python_api/api_def_MaxPool3D.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_MaxPool3D.pbtxt
@@ -1,6 +1,4 @@
 op {
   graph_op_name: "MaxPool3D"
-  endpoint {
-    name: "nn.max_pool3d"
-  }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 11b6690519..67f88995ba 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -3030,6 +3030,51 @@ def max_pool(value, ksize, strides, padding, data_format="NHWC", name=None):
         name=name)
 
 
+# pylint: disable=redefined-builtin
+@tf_export("nn.max_pool3d")
+def max_pool3d(input, ksize, strides, padding, data_format="NDHWC", name=None):
+  """Performs the max pooling on the input.
+
+  Args:
+    input: A 5-D `Tensor` of the format specified by `data_format`.
+    ksize: An int or list of `ints` that has length `1`, `3` or `5`. The size of
+      the window for each dimension of the input tensor.
+    strides: An int or list of `ints` that has length `1`, `3` or `5`. The
+      stride of the sliding window for each dimension of the input tensor.
+    padding: A string, either `'VALID'` or `'SAME'`. The padding algorithm. See
+      the "returns" section of `tf.nn.convolution` for details.
+    data_format: An optional string from: "NDHWC", "NCDHW". Defaults to "NDHWC".
+      The data format of the input and output data. With the default format
+      "NDHWC", the data is stored in the order of: [batch, in_depth, in_height,
+        in_width, in_channels]. Alternatively, the format could be "NCDHW", the
+      data storage order is: [batch, in_channels, in_depth, in_height,
+        in_width].
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor` of format specified by `data_format`.
+    The max pooled output tensor.
+  """
+  with ops.name_scope(name, "MaxPool3D", [input]) as name:
+    if data_format is None:
+      data_format = "NDHWC"
+    channel_index = 1 if data_format.startswith("NC") else 4
+
+    ksize = _get_sequence(ksize, 3, channel_index, "ksize")
+    strides = _get_sequence(strides, 3, channel_index, "strides")
+
+    return gen_nn_ops.max_pool3d(
+        input,
+        ksize=ksize,
+        strides=strides,
+        padding=padding,
+        data_format=data_format,
+        name=name)
+
+
+# pylint: enable=redefined-builtin
+
+
 # pylint: disable=redefined-builtin
 @tf_export("nn.max_pool_with_argmax", v1=[])
 def max_pool_with_argmax_v2(input,
-- 
GitLab


From 5044ab1763bf4a72a03865715c1b2edd525ecca7 Mon Sep 17 00:00:00 2001
From: Amit <30853054+amitsrivastava78@users.noreply.github.com>
Date: Wed, 30 Jan 2019 18:34:39 +0530
Subject: [PATCH 1573/2345] Updated the READMe.md file

typo error fixed
---
 tensorflow/lite/tools/accuracy/ilsvrc/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/tools/accuracy/ilsvrc/README.md b/tensorflow/lite/tools/accuracy/ilsvrc/README.md
index ac3a1566e2..2b0d4ed579 100644
--- a/tensorflow/lite/tools/accuracy/ilsvrc/README.md
+++ b/tensorflow/lite/tools/accuracy/ilsvrc/README.md
@@ -27,7 +27,7 @@ The binary takes the following parameters:
     1001 dimension tensor.
 
 *   `output_path`: `string` \
-    This is the path to the output file. The output is a CSV file that has top-10 accuracies in each row. Each line of output file is the cumulative accuracy after processing images in a sorted order. So first line is accuracy after processing the first image, second line is accuracy after procesing first two images. The last line of the file is accuracy after processing the entire validation set.
+    This is the path to the output file. The output is a CSV file that has top-10 accuracies in each row. Each line of output file is the cumulative accuracy after processing images in a sorted order. So first line is accuracy after processing the first image, second line is accuracy after processing first two images. The last line of the file is accuracy after processing the entire validation set.
 
 and the following optional parameters:
 
-- 
GitLab


From e114d880ac993b4af6428be4aa7772a947a1ad86 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 30 Jan 2019 05:27:42 -0800
Subject: [PATCH 1574/2345] [TF:XLA] Disable XLA test for large concats on CPU.

This allows the test time to become shorter and was originally for a feature that is only needed for GPUs and CPUs.

PiperOrigin-RevId: 231576743
---
 tensorflow/compiler/tests/BUILD              | 3 +--
 tensorflow/compiler/tests/concat_ops_test.py | 3 +++
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD
index 63878d880b..139c92728d 100644
--- a/tensorflow/compiler/tests/BUILD
+++ b/tensorflow/compiler/tests/BUILD
@@ -279,10 +279,9 @@ tf_xla_py_test(
     ],
 )
 
-# This test is large because occasionally the cpu test is long for testConcatLargeNumberOfTensors
 tf_xla_py_test(
     name = "concat_ops_test",
-    size = "large",
+    size = "medium",
     srcs = ["concat_ops_test.py"],
     deps = [
         ":xla_test",
diff --git a/tensorflow/compiler/tests/concat_ops_test.py b/tensorflow/compiler/tests/concat_ops_test.py
index 2187f57960..76750decd2 100644
--- a/tensorflow/compiler/tests/concat_ops_test.py
+++ b/tensorflow/compiler/tests/concat_ops_test.py
@@ -294,6 +294,9 @@ class ConcatTest(xla_test.XLATestCase):
   # The purpose of this is to ensure that XLA on GPU will not run out of memory
   # with too many arguments.
   def testConcatLargeNumberOfTensors(self):
+    if "CPU" in self.device:
+      self.skipTest("This test can time out on CPU, so we will just allow "
+                    "other backends to catch this specific error.")
     with self.cached_session():
       with self.test_scope():
         for concat_dim in range(2):
-- 
GitLab


From 6631cd5e0e43e672d2de3cf1f0e29da17b57824e Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Wed, 30 Jan 2019 07:34:51 -0800
Subject: [PATCH 1575/2345] Cleanup: remove trt_engine_op.h.

PiperOrigin-RevId: 231591236
---
 tensorflow/compiler/tf2tensorrt/BUILD         |   6 +-
 .../tf2tensorrt/kernels/trt_engine_op.cc      |  89 +++++++++++-
 .../tf2tensorrt/kernels/trt_engine_op.h       | 130 ------------------
 3 files changed, 88 insertions(+), 137 deletions(-)
 delete mode 100644 tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.h

diff --git a/tensorflow/compiler/tf2tensorrt/BUILD b/tensorflow/compiler/tf2tensorrt/BUILD
index a67e511826..9ad27c9801 100644
--- a/tensorflow/compiler/tf2tensorrt/BUILD
+++ b/tensorflow/compiler/tf2tensorrt/BUILD
@@ -65,9 +65,6 @@ cc_library(
         "kernels/get_serialized_resource_op.cc",
         "kernels/trt_engine_op.cc",
     ],
-    hdrs = [
-        "kernels/trt_engine_op.h",
-    ],
     copts = tf_copts(),
     visibility = ["//visibility:public"],
     deps = [
@@ -87,8 +84,7 @@ cc_library(
     ] + if_tensorrt([
         "@local_config_tensorrt//:tensorrt",
     ]) + tf_custom_op_library_additional_deps(),
-    # TODO(laigd): fix this by merging header file in cc file.
-    alwayslink = 1,  # buildozer: disable=alwayslink-with-hdrs
+    alwayslink = 1,
 )
 
 tf_cuda_cc_test(
diff --git a/tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.cc b/tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.cc
index 198d68b609..ec9dc08c12 100644
--- a/tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.cc
@@ -12,9 +12,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.h"
-
 #include <algorithm>
+#include <memory>
+#include <vector>
 
 #include "absl/memory/memory.h"
 #include "absl/strings/str_cat.h"
@@ -22,20 +22,28 @@ limitations under the License.
 #include "tensorflow/compiler/tf2tensorrt/convert/utils.h"
 #include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.h"
 #include "tensorflow/compiler/tf2tensorrt/utils/test_utils.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_allocator.h"
 #include "tensorflow/compiler/tf2tensorrt/utils/trt_logger.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_lru_cache.h"
 #include "tensorflow/compiler/tf2tensorrt/utils/trt_resource_manager.h"
 #include "tensorflow/compiler/tf2tensorrt/utils/trt_resources.h"
+#include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/framework/graph_to_functiondef.h"
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/lib/core/refcount.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/platform/stream_executor.h"
+#include "tensorflow/core/platform/thread_annotations.h"
 #include "tensorflow/core/platform/types.h"
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
 #include "cuda/include/cuda_runtime_api.h"
+#include "tensorrt/include/NvInfer.h"
 
 namespace tensorflow {
 namespace tensorrt {
@@ -55,6 +63,83 @@ class AsyncHelper : public tensorflow::core::RefCounted {
   AsyncOpKernel::DoneCallback done_;
 };
 
+//  This OP can construct TRTEngine on the fly and if construction of engine
+//  fails, executes equivalent subgraph as a TensorFlow function.
+class TRTEngineOp : public AsyncOpKernel {
+ public:
+  explicit TRTEngineOp(OpKernelConstruction* context);
+
+  void ComputeAsync(OpKernelContext* context,
+                    AsyncOpKernel::DoneCallback done) override;
+
+ private:
+  // Execute calibration
+  void ExecuteCalibration(OpKernelContext* ctx, AsyncHelper* helper);
+
+  // Construct a function handle for executing native funcdef graph
+  Status ConstructFunctionHandle(OpKernelContext* ctx);
+
+  // Execute replaced native segment as function Op.
+  void ExecuteNativeSegment(OpKernelContext* ctx, AsyncHelper* helper);
+
+  // Execute the tensorrt engine. Returns whether we need to retry by running
+  // the native segment.
+  bool ExecuteTrtEngine(OpKernelContext* ctx, EngineContext* engine_context);
+
+  // Allocate necessary resources for calibration
+  Status AllocateCalibrationResources(OpKernelContext* ctx,
+                                      SerializableResourceBase** cr);
+
+  // Get engine for the input shape
+  EngineContext* GetEngine(const std::vector<TensorShape>& input_shapes,
+                           OpKernelContext* ctx);
+
+  // Return engine batch in cached_engne_batch_sizes_ which is closest to input
+  // batch.
+  bool GetCompatibleCachedEngine(
+      const std::vector<TensorShape>& actual_input_shapes,
+      std::vector<TensorShape>* engine_input_shapes);
+
+  std::vector<string> input_nodes_;
+  std::vector<string> output_nodes_;
+
+  // serialized protobuf segment or trt engine depending on static_engine_ flag.
+  string serialized_segment_;
+
+  // Name of the function for TF native execution of the segment.
+  string funcdef_name_;
+
+  // GraphDef representation of the segment.
+  GraphDef segment_graph_;
+
+  // Engine Precision mode.
+  int precision_mode_;
+
+  // Whether engine is constructed during the conversion or needs to be
+  // constructed from protobuf segment.
+  bool static_engine_;
+
+  // Whether to calibrate INT8 engine.
+  bool calibration_mode_;
+
+  // Batches of the cached engines
+  std::vector<int> cached_engine_batches_;
+
+  // Maximum number of cached engines
+  int max_cached_engines_;
+
+  int64 workspace_size_;
+  mutex engine_mutex_;
+  FunctionLibraryRuntime::Handle native_func_;
+
+  // The finalized calibrator for inference.
+  std::unique_ptr<TRTInt8Calibrator> calibrator_;
+
+  // If true, create calibration graph for INT8 mode. Otherwise, we are using
+  // user-provided quantization ranges.
+  bool use_calibration_;
+};
+
 #define TYPECASE(dt, X, Y)                                                \
   case dt: {                                                              \
     return (void*)X->flat<tensorflow::EnumToDataType<dt>::Type>().data(); \
diff --git a/tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.h b/tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.h
deleted file mode 100644
index 64f8c97a74..0000000000
--- a/tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.h
+++ /dev/null
@@ -1,130 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_KERNELS_TRT_ENGINE_OP_H_
-#define TENSORFLOW_COMPILER_TF2TENSORRT_KERNELS_TRT_ENGINE_OP_H_
-
-#include <memory>
-#include <vector>
-
-#include "tensorflow/compiler/tf2tensorrt/convert/utils.h"
-#include "tensorflow/compiler/tf2tensorrt/utils/trt_allocator.h"
-#include "tensorflow/compiler/tf2tensorrt/utils/trt_logger.h"
-#include "tensorflow/compiler/tf2tensorrt/utils/trt_lru_cache.h"
-#include "tensorflow/compiler/tf2tensorrt/utils/trt_resources.h"
-#include "tensorflow/core/framework/function.h"
-#include "tensorflow/core/framework/graph.pb.h"
-#include "tensorflow/core/framework/op.h"
-#include "tensorflow/core/framework/op_kernel.h"
-#include "tensorflow/core/platform/mutex.h"
-#include "tensorflow/core/platform/thread_annotations.h"
-
-#if GOOGLE_CUDA
-#if GOOGLE_TENSORRT
-#include "cuda/include/cuda_runtime_api.h"
-#include "tensorrt/include/NvInfer.h"
-
-namespace tensorflow {
-namespace tensorrt {
-struct TRTInt8Calibrator;
-class AsyncHelper;
-//  TODO(Sami): Remove this file?
-
-//  This OP can construct TRTEngine on the fly and if construction of engine
-//  fails, executes equivalent subgraph as a TensorFlow function.
-class TRTEngineOp : public AsyncOpKernel {
- public:
-  explicit TRTEngineOp(OpKernelConstruction* context);
-
-  void ComputeAsync(OpKernelContext* context,
-                    AsyncOpKernel::DoneCallback done) override;
-
- private:
-  // TODO(samikama): context should go to a resource manager!
-
-  // Execute calibration
-  void ExecuteCalibration(OpKernelContext* ctx, AsyncHelper* helper);
-
-  // Construct a function handle for executing native funcdef graph
-  Status ConstructFunctionHandle(OpKernelContext* ctx);
-
-  // Execute replaced native segment as function Op.
-  void ExecuteNativeSegment(OpKernelContext* ctx, AsyncHelper* helper);
-
-  // Execute the tensorrt engine. Returns whether we need to retry by running
-  // the native segment.
-  bool ExecuteTrtEngine(OpKernelContext* ctx, EngineContext* engine_context);
-
-  // Allocate necessary resources for calibration
-  Status AllocateCalibrationResources(OpKernelContext* ctx,
-                                      SerializableResourceBase** cr);
-
-  // Get engine for the input shape
-  EngineContext* GetEngine(const std::vector<TensorShape>& input_shapes,
-                           OpKernelContext* ctx);
-
-  // Return engine batch in cached_engne_batch_sizes_ which is closest to input
-  // batch.
-  bool GetCompatibleCachedEngine(
-      const std::vector<TensorShape>& actual_input_shapes,
-      std::vector<TensorShape>* engine_input_shapes);
-
-  std::vector<string> input_nodes_;
-  std::vector<string> output_nodes_;
-
-  // serialized protobuf segment or trt engine depending on static_engine_ flag.
-  string serialized_segment_;
-
-  // Name of the function for TF native execution of the segment.
-  string funcdef_name_;
-
-  // GraphDef representation of the segment.
-  GraphDef segment_graph_;
-
-  // Engine Precision mode.
-  int precision_mode_;
-
-  // Whether engine is constructed during the conversion or needs to be
-  // constructed from protobuf segment.
-  bool static_engine_;
-
-  // Whether to calibrate INT8 engine.
-  bool calibration_mode_;
-
-  // Batches of the cached engines
-  std::vector<int> cached_engine_batches_;
-
-  // Maximum number of cached engines
-  int max_cached_engines_;
-
-  int64 workspace_size_;
-  mutex engine_mutex_;
-  FunctionLibraryRuntime::Handle native_func_;
-
-  // The finalized calibrator for inference.
-  std::unique_ptr<TRTInt8Calibrator> calibrator_;
-
-  // If true, create calibration graph for INT8 mode. Otherwise, we are using
-  // user-provided quantization ranges.
-  bool use_calibration_;
-};
-
-}  // namespace tensorrt
-}  // namespace tensorflow
-
-#endif  // GOOGLE_TENSORRT
-#endif  // GOOGLE_CUDA
-
-#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_KERNELS_TRT_ENGINE_OP_H_
-- 
GitLab


From 75dfa41fabaddba3536acf778348b007f315a67d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 30 Jan 2019 08:21:09 -0800
Subject: [PATCH 1576/2345] Adding a try/except wrapping impl.api.to_graph that
 throws an InternalError.

PiperOrigin-RevId: 231598248
---
 tensorflow/python/autograph/__init__.py    |   2 +
 tensorflow/python/autograph/impl/api.py    | 125 +++++++++++----------
 tensorflow/python/autograph/pyct/errors.py |  11 ++
 3 files changed, 78 insertions(+), 60 deletions(-)

diff --git a/tensorflow/python/autograph/__init__.py b/tensorflow/python/autograph/__init__.py
index a1956d1d8a..5e187ab20f 100644
--- a/tensorflow/python/autograph/__init__.py
+++ b/tensorflow/python/autograph/__init__.py
@@ -50,6 +50,7 @@ from tensorflow.python.autograph.lang.directives import set_element_type
 from tensorflow.python.autograph.lang.directives import set_loop_options
 from tensorflow.python.autograph.lang.special_functions import stack
 from tensorflow.python.autograph.lang.special_functions import tensor_list
+from tensorflow.python.autograph.pyct.errors import AutoGraphError
 from tensorflow.python.autograph.pyct.transformer import AutoGraphParseError
 from tensorflow.python.autograph.utils import ag_logging
 from tensorflow.python.util.all_util import remove_undocumented
@@ -79,6 +80,7 @@ _allowed_symbols = [
     'tensor_list',
     # Exceptions
     'AutoGraphParseError',
+    'AutoGraphError',
     # Utilities: to be removed
     'utils',
 ]
diff --git a/tensorflow/python/autograph/impl/api.py b/tensorflow/python/autograph/impl/api.py
index 122ea1726b..3ad69689f7 100644
--- a/tensorflow/python/autograph/impl/api.py
+++ b/tensorflow/python/autograph/impl/api.py
@@ -33,6 +33,7 @@ from tensorflow.python.autograph.core import converter
 from tensorflow.python.autograph.impl import conversion
 from tensorflow.python.autograph.operators import py_builtins
 from tensorflow.python.autograph.pyct import compiler
+from tensorflow.python.autograph.pyct import errors
 from tensorflow.python.autograph.pyct import inspect_utils
 from tensorflow.python.autograph.utils import py_func
 from tensorflow.python.framework import tensor_util
@@ -398,66 +399,70 @@ def to_graph(entity,
   Raises:
     ValueError: If the entity could not be converted.
   """
-  if experimental_strip_decorators is None:
-    experimental_strip_decorators = ()
-  experimental_strip_decorators += (convert, do_not_convert, converted_call)
-
-  program_ctx = converter.ProgramContext(
-      options=converter.ConversionOptions(
-          recursive=recursive,
-          verbose=experimental_verbose,
-          strip_decorators=experimental_strip_decorators,
-          optional_features=experimental_optional_features),
-      partial_types=experimental_partial_types,
-      autograph_module=tf_inspect.getmodule(to_graph),
-      uncompiled_modules=config.DEFAULT_UNCOMPILED_MODULES)
-  _, name, namespace = conversion.entity_to_graph(entity, program_ctx,
-                                                  arg_values, arg_types)
-
-  nodes = []
-  for dep in reversed(program_ctx.conversion_order):
-    nodes.extend(program_ctx.dependency_cache[dep])
-
-  compiled_module, _ = compiler.ast_to_object(
-      nodes,
-      source_prefix=program_ctx.required_imports,
-      include_source_map=True)
-
-  # The compiled code should see everything the entry entity saw.
-  # TODO(mdan): This might not work well if the call tree spans modules?
-  for key, val in namespace.items():
-    # Avoid overwriting entities that have been transformed.
-    if key not in compiled_module.__dict__:
-      compiled_module.__dict__[key] = val
-  for key, val in program_ctx.additional_symbols.items():
-    if key not in compiled_module.__dict__:
-      compiled_module.__dict__[key] = val
-  compiled = getattr(compiled_module, name)
-
-  if tf_inspect.isfunction(entity):
-    compiled.__defaults__ = entity.__defaults__
-
-  if hasattr(compiled, '__globals__'):
-    # Remove self to avoid circular references. This will probably only work
-    # so long as the function is not reentrant.
-    del compiled.__globals__[name]
-
-  # Need this so the source_mapping attribute is available for the context
-  # manager to access for runtime errors.
-  #
-  # Note that compiler.ast_to_object attaches the source map 'ag_source_map__'
-  # symbol to the compiled module.
-  # TODO(mdan): Record this statically in the generated code.
-  # TODO(mdan): Rename this attribute to 'autograph_info__'
-  source_map_attribute_name = 'ag_source_map'
-  if getattr(compiled, source_map_attribute_name, None) is not None:
-    raise ValueError('cannot convert %s because is has an attribute '
-                     '"%s", which is reserved for AutoGraph.' %
-                     (compiled, source_map_attribute_name))
-  setattr(compiled, source_map_attribute_name,
-          compiled_module.__dict__['ag_source_map__'])
-
-  return compiled
+  try:
+    if experimental_strip_decorators is None:
+      experimental_strip_decorators = ()
+    experimental_strip_decorators += (convert, do_not_convert, converted_call)
+
+    program_ctx = converter.ProgramContext(
+        options=converter.ConversionOptions(
+            recursive=recursive,
+            verbose=experimental_verbose,
+            strip_decorators=experimental_strip_decorators,
+            optional_features=experimental_optional_features),
+        partial_types=experimental_partial_types,
+        autograph_module=tf_inspect.getmodule(to_graph),
+        uncompiled_modules=config.DEFAULT_UNCOMPILED_MODULES)
+    _, name, namespace = conversion.entity_to_graph(entity, program_ctx,
+                                                    arg_values, arg_types)
+
+    nodes = []
+    for dep in reversed(program_ctx.conversion_order):
+      nodes.extend(program_ctx.dependency_cache[dep])
+
+    compiled_module, _ = compiler.ast_to_object(
+        nodes,
+        source_prefix=program_ctx.required_imports,
+        include_source_map=True)
+
+    # The compiled code should see everything the entry entity saw.
+    # TODO(mdan): This might not work well if the call tree spans modules?
+    for key, val in namespace.items():
+      # Avoid overwriting entities that have been transformed.
+      if key not in compiled_module.__dict__:
+        compiled_module.__dict__[key] = val
+    for key, val in program_ctx.additional_symbols.items():
+      if key not in compiled_module.__dict__:
+        compiled_module.__dict__[key] = val
+    compiled = getattr(compiled_module, name)
+
+    if tf_inspect.isfunction(entity):
+      compiled.__defaults__ = entity.__defaults__
+
+    if hasattr(compiled, '__globals__'):
+      # Remove self to avoid circular references. This will probably only work
+      # so long as the function is not reentrant.
+      del compiled.__globals__[name]
+
+    # Need this so the source_mapping attribute is available for the context
+    # manager to access for runtime errors.
+    #
+    # Note that compiler.ast_to_object attaches the source map 'ag_source_map__'
+    # symbol to the compiled module.
+    # TODO(mdan): Record this statically in the generated code.
+    # TODO(mdan): Rename this attribute to 'autograph_info__'
+    source_map_attribute_name = 'ag_source_map'
+    if getattr(compiled, source_map_attribute_name, None) is not None:
+      # TODO(znado): change input problem errors into TransformError
+      raise ValueError('cannot convert %s because is has an attribute '
+                       '"%s", which is reserved for AutoGraph.' %
+                       (compiled, source_map_attribute_name))
+    setattr(compiled, source_map_attribute_name,
+            compiled_module.__dict__['ag_source_map__'])
+
+    return compiled
+  except (ValueError, AttributeError, KeyError, NameError, AssertionError) as e:
+    errors.report_internal_error(entity, e)
 
 
 @tf_export('autograph.to_code')
diff --git a/tensorflow/python/autograph/pyct/errors.py b/tensorflow/python/autograph/pyct/errors.py
index 4fc9c168c0..3de3fadc88 100644
--- a/tensorflow/python/autograph/pyct/errors.py
+++ b/tensorflow/python/autograph/pyct/errors.py
@@ -18,12 +18,15 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.autograph.utils import ag_logging
+
 
 class AutoGraphError(Exception):
   pass
 
 
 class InternalError(AutoGraphError):
+  """Raised when AutoGraph finds an unexpected error."""
 
   def __init__(self, message, original_exc):
     super(InternalError, self).__init__()
@@ -35,3 +38,11 @@ class InternalError(AutoGraphError):
         type(self.original_exc).__name__, self.message, self.original_exc)
 
 
+def report_internal_error(entity, exception):
+  ag_logging.log(1, 'Error transforming %s', entity, exc_info=True)
+  # TODO(znado): Add external bug reporting instructions.
+  raise AutoGraphError(
+      'Unexpected error transforming %s. If you believe this is due to a bug,'
+      ' please set the verbosity to 10 (on Linux, `export '
+      'AUTOGRAPH_VERBOSITY=1`) and attach the full output when filing the bug '
+      'report. Caused by %s' % (entity, exception))
-- 
GitLab


From 00d159b19863b570dba216d0311c723c1dcccc6a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 30 Jan 2019 08:25:50 -0800
Subject: [PATCH 1577/2345] Remove hardwired selection of M4 microprocessor
 architecture from CMSIS preprocessor.

PiperOrigin-RevId: 231598881
---
 .../micro/examples/micro_speech/CMSIS/preprocessor.cc            | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/preprocessor.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/preprocessor.cc
index 5c6978b5ed..78155cc2ba 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/preprocessor.cc
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/preprocessor.cc
@@ -14,7 +14,6 @@ limitations under the License.
 ==============================================================================*/
 
 extern "C" {
-#define ARM_MATH_CM4
 #define IFFT_FLAG_R 0
 #define BIT_REVERSE_FLAG 1
 #define FFT_SIZE 512
-- 
GitLab


From 71c9f15ea2f953fcdb4ff33316547c71930ed4d7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 30 Jan 2019 08:44:09 -0800
Subject: [PATCH 1578/2345] Delete AutoGraphParseError now that it is no longer
 used because we are moving to AutoGraphError, InternalError, etc.

PiperOrigin-RevId: 231601621
---
 tensorflow/python/autograph/__init__.py         |  4 +---
 tensorflow/python/autograph/pyct/transformer.py | 12 ------------
 2 files changed, 1 insertion(+), 15 deletions(-)

diff --git a/tensorflow/python/autograph/__init__.py b/tensorflow/python/autograph/__init__.py
index 5e187ab20f..3009cfffd4 100644
--- a/tensorflow/python/autograph/__init__.py
+++ b/tensorflow/python/autograph/__init__.py
@@ -49,9 +49,8 @@ from tensorflow.python.autograph.impl.api import to_graph
 from tensorflow.python.autograph.lang.directives import set_element_type
 from tensorflow.python.autograph.lang.directives import set_loop_options
 from tensorflow.python.autograph.lang.special_functions import stack
-from tensorflow.python.autograph.lang.special_functions import tensor_list
 from tensorflow.python.autograph.pyct.errors import AutoGraphError
-from tensorflow.python.autograph.pyct.transformer import AutoGraphParseError
+from tensorflow.python.autograph.lang.special_functions import tensor_list
 from tensorflow.python.autograph.utils import ag_logging
 from tensorflow.python.util.all_util import remove_undocumented
 
@@ -79,7 +78,6 @@ _allowed_symbols = [
     'stack',
     'tensor_list',
     # Exceptions
-    'AutoGraphParseError',
     'AutoGraphError',
     # Utilities: to be removed
     'utils',
diff --git a/tensorflow/python/autograph/pyct/transformer.py b/tensorflow/python/autograph/pyct/transformer.py
index 90213da782..d8e093c3eb 100644
--- a/tensorflow/python/autograph/pyct/transformer.py
+++ b/tensorflow/python/autograph/pyct/transformer.py
@@ -27,18 +27,6 @@ from tensorflow.python.autograph.pyct import pretty_printer
 from tensorflow.python.autograph.pyct import templates
 
 
-class AutoGraphParseError(SyntaxError):
-  """Error for graph construction errors from AutoGraph generated code."""
-
-  def __init__(self, error, origin_info):
-    file_path = origin_info.loc.filename
-    line_number = origin_info.loc.lineno
-    col_offset = origin_info.loc.col_offset
-    source_line = origin_info.source_code_line
-    super(AutoGraphParseError, self).__init__(
-        error, (file_path, line_number, col_offset, source_line))
-
-
 # TODO(znado): Use namedtuple.
 class Context(object):
   """Contains information about a source code transformation.
-- 
GitLab


From f7c6065865d50eda39e059b19a4f12dabe3f1872 Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Wed, 30 Jan 2019 08:58:18 -0800
Subject: [PATCH 1579/2345] Fix Micro makefile build breakage

PiperOrigin-RevId: 231603748
---
 tensorflow/lite/experimental/micro/tools/make/Makefile | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/tensorflow/lite/experimental/micro/tools/make/Makefile b/tensorflow/lite/experimental/micro/tools/make/Makefile
index 89df595b4f..9c62d8a36d 100644
--- a/tensorflow/lite/experimental/micro/tools/make/Makefile
+++ b/tensorflow/lite/experimental/micro/tools/make/Makefile
@@ -32,13 +32,9 @@ ALL_TAGS := $(TAGS) $(TARGET)
 
 INCLUDES := \
 -I. \
--I$(MAKEFILE_DIR)/../../../../../ \
--I$(MAKEFILE_DIR)/../../../../../../ \
--I$(MAKEFILE_DIR)/../../../../../../../ \
 -I$(MAKEFILE_DIR)/downloads/ \
 -I$(MAKEFILE_DIR)/downloads/gemmlowp \
--I$(MAKEFILE_DIR)/downloads/flatbuffers/include \
--I$(OBJDIR)
+-I$(MAKEFILE_DIR)/downloads/flatbuffers/include
 
 # These are the include paths added to any generated project file.
 PROJECT_INCLUDES := \
-- 
GitLab


From 13679b875aa87a3d28cf224388ee6a255d3f8844 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Wed, 30 Jan 2019 09:16:22 -0800
Subject: [PATCH 1580/2345] Adds all v2-only keywords added by the converter to
 v1 ops.

This has the sad side effect of needing to add default values
to a lot of arguments.

PiperOrigin-RevId: 231606771
---
 .../python_api/api_def_Dilation2D.pbtxt       |   5 +-
 .../api_def_MaxPoolWithArgmax.pbtxt           |   5 +-
 tensorflow/python/framework/ops.py            |   6 +-
 tensorflow/python/ops/array_ops.py            |  30 +++-
 tensorflow/python/ops/ctc_ops.py              |   9 +-
 tensorflow/python/ops/image_ops_impl.py       |  20 ++-
 tensorflow/python/ops/math_ops.py             |   8 +-
 tensorflow/python/ops/nn_impl.py              |  53 +++++--
 tensorflow/python/ops/nn_ops.py               | 143 ++++++++++++++----
 tensorflow/python/ops/numerics.py             |   9 +-
 tensorflow/python/ops/sparse_ops.py           |   9 +-
 tensorflow/python/ops/string_ops.py           |  37 ++++-
 .../api/golden/v1/tensorflow.debugging.pbtxt  |   2 +-
 .../api/golden/v1/tensorflow.image.pbtxt      |   4 +-
 .../tools/api/golden/v1/tensorflow.math.pbtxt |   2 +-
 .../tools/api/golden/v1/tensorflow.nn.pbtxt   |  38 ++---
 .../tools/api/golden/v1/tensorflow.pbtxt      |  20 +--
 .../api/golden/v1/tensorflow.sparse.pbtxt     |   2 +-
 .../api/golden/v1/tensorflow.strings.pbtxt    |   6 +-
 .../tools/compatibility/tf_upgrade_v2_test.py |  44 ++++--
 20 files changed, 334 insertions(+), 118 deletions(-)

diff --git a/tensorflow/core/api_def/python_api/api_def_Dilation2D.pbtxt b/tensorflow/core/api_def/python_api/api_def_Dilation2D.pbtxt
index 1bd83d9061..97af07e001 100644
--- a/tensorflow/core/api_def/python_api/api_def_Dilation2D.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_Dilation2D.pbtxt
@@ -1,7 +1,4 @@
 op {
   graph_op_name: "Dilation2D"
-  endpoint {
-    name: "nn.dilation2d"
-    deprecation_version: 2
-  }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_MaxPoolWithArgmax.pbtxt b/tensorflow/core/api_def/python_api/api_def_MaxPoolWithArgmax.pbtxt
index 13a1a0b5df..c57cfc7727 100644
--- a/tensorflow/core/api_def/python_api/api_def_MaxPoolWithArgmax.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_MaxPoolWithArgmax.pbtxt
@@ -1,7 +1,4 @@
 op {
   graph_op_name: "MaxPoolWithArgmax"
-  endpoint {
-    name: "nn.max_pool_with_argmax"
-    deprecation_version: 2
-  }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index b7908497df..07d1e72e29 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -991,7 +991,8 @@ register_dense_tensor_like_type(Tensor)
 
 
 @tf_export(v1=["convert_to_tensor"])
-def convert_to_tensor(value, dtype=None, name=None, preferred_dtype=None):
+def convert_to_tensor(value, dtype=None, name=None, preferred_dtype=None,
+                      dtype_hint=None):
   """Converts the given `value` to a `Tensor`.
 
   This function converts Python objects of various types to `Tensor`
@@ -1031,6 +1032,7 @@ def convert_to_tensor(value, dtype=None, name=None, preferred_dtype=None):
       dtype in mind when converting to a tensor, so preferred_dtype
       can be used as a soft preference.  If the conversion to
       `preferred_dtype` is not possible, this argument has no effect.
+    dtype_hint: same meaning as preferred_dtype, and overrides it.
 
   Returns:
     A `Tensor` based on `value`.
@@ -1040,6 +1042,8 @@ def convert_to_tensor(value, dtype=None, name=None, preferred_dtype=None):
     RuntimeError: If a registered conversion function returns an invalid value.
     ValueError: If the `value` is a tensor not of given `dtype` in graph mode.
   """
+  preferred_dtype = deprecation.deprecated_argument_lookup(
+      "dtype_hint", dtype_hint, "preferred_dtype", preferred_dtype)
   return convert_to_tensor_v2(value, dtype, preferred_dtype, name)
 
 
diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 287510342e..89d6de6dff 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -2666,7 +2666,10 @@ def required_space_to_batch_paddings(input_shape,
 
 @tf_export(v1=["nn.space_to_batch", "space_to_batch"])
 @deprecation.deprecated_endpoints("space_to_batch")
-def space_to_batch(input, paddings, block_size, name=None):  # pylint: disable=redefined-builtin
+def space_to_batch(  # pylint: disable=missing-docstring
+    input, paddings, block_size=None, name=None, block_shape=None):  # pylint: disable=redefined-builtin
+  block_size = deprecation.deprecated_argument_lookup(
+      "block_shape", block_shape, "block_size", block_size)
   result = space_to_batch_nd(
       input,
       paddings=paddings,
@@ -2722,7 +2725,9 @@ depth_to_space_v2.__doc__ = gen_array_ops.depth_to_space.__doc__
 
 
 @tf_export(v1=["batch_to_space"])
-def batch_to_space(input, crops, block_size, name=None):  # pylint: disable=redefined-builtin
+def batch_to_space(input, crops, block_size, name=None, block_shape=None):  # pylint: disable=redefined-builtin,missing-docstring
+  block_size = deprecation.deprecated_argument_lookup(
+      "block_shape", block_shape, "block_size", block_size)
   result = batch_to_space_nd(
       input,
       crops=crops,
@@ -3654,7 +3659,22 @@ def extract_image_patches_v2(
   return gen_array_ops.extract_image_patches(
       images, sizes, strides, rates, padding, name)
 
-extract_image_patches_deprecation = deprecation.deprecated_args(
+
+@tf_export(v1=["image.extract_image_patches", "extract_image_patches"])
+@deprecation.deprecated_args(
     None, "ksizes is deprecated, use sizes instead", "ksizes")
-tf_export(v1=["image.extract_image_patches", "extract_image_patches"])(
-    extract_image_patches_deprecation(gen_array_ops.extract_image_patches))
+def extract_image_patches(  # pylint: disable=missing-docstring
+    images,
+    ksizes=None,
+    strides=None,
+    rates=None,
+    padding=None,
+    name=None,
+    sizes=None):
+  ksizes = deprecation.deprecated_argument_lookup(
+      "sizes", sizes, "ksizes", ksizes)
+  return gen_array_ops.extract_image_patches(
+      images, ksizes, strides, rates, padding, name)
+
+
+extract_image_patches.__doc__ = gen_array_ops.extract_image_patches.__doc__
diff --git a/tensorflow/python/ops/ctc_ops.py b/tensorflow/python/ops/ctc_ops.py
index 359719f053..80502daaac 100644
--- a/tensorflow/python/ops/ctc_ops.py
+++ b/tensorflow/python/ops/ctc_ops.py
@@ -36,16 +36,18 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import sparse_ops
 from tensorflow.python.ops.nn_grad import _BroadcastMul
+from tensorflow.python.util import deprecation
 from tensorflow.python.util import nest
 from tensorflow.python.util.tf_export import tf_export
 
 
 # pylint: disable=protected-access, invalid-name
 @tf_export(v1=["nn.ctc_loss"])
-def ctc_loss(labels, inputs, sequence_length,
+def ctc_loss(labels, inputs=None, sequence_length=None,
              preprocess_collapse_repeated=False,
              ctc_merge_repeated=True,
-             ignore_longer_outputs_than_inputs=False, time_major=True):
+             ignore_longer_outputs_than_inputs=False, time_major=True,
+             logits=None):
   """Computes the CTC (Connectionist Temporal Classification) Loss.
 
   This op implements the CTC loss as presented in the article:
@@ -143,6 +145,7 @@ def ctc_loss(labels, inputs, sequence_length,
       avoids transposes at the beginning of the ctc_loss calculation.  However,
       most TensorFlow data is batch-major, so by this function also accepts
       inputs in batch-major form.
+    logits: Alias for inputs.
 
   Returns:
     A 1-D `float` `Tensor`, size `[batch]`, containing the negative log
@@ -157,6 +160,8 @@ def ctc_loss(labels, inputs, sequence_length,
     raise TypeError("Expected labels (first argument) to be a SparseTensor")
 
   # For internal calculations, we transpose to [time, batch, num_classes]
+  inputs = deprecation.deprecated_argument_lookup(
+      "logits", logits, "inputs", inputs)
   if not time_major:
     inputs = array_ops.transpose(inputs, [1, 0, 2])  # (B,T,N) => (T,B,N)
 
diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index 0cb15fce1f..44bd92f422 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -3073,10 +3073,24 @@ def crop_and_resize_v2(
       image, boxes, box_indices, crop_size, method, extrapolation_value, name)
 
 
-crop_and_resize_deprecation = deprecation.deprecated_args(
+@tf_export(v1=['image.crop_and_resize'])
+@deprecation.deprecated_args(
     None, 'box_ind is deprecated, use box_indices instead', 'box_ind')
-tf_export(v1=['image.crop_and_resize'])(
-    crop_and_resize_deprecation(gen_image_ops.crop_and_resize))
+def crop_and_resize_v1(   # pylint: disable=missing-docstring
+    image,
+    boxes,
+    box_ind=None,
+    crop_size=None,
+    method='bilinear',
+    extrapolation_value=0,
+    name=None,
+    box_indices=None):
+  box_ind = deprecation.deprecated_argument_lookup(
+      "box_indices", box_indices, "box_ind", box_ind)
+  return gen_image_ops.crop_and_resize(
+      image, boxes, box_ind, crop_size, method, extrapolation_value, name)
+
+crop_and_resize_v1.__doc__ = gen_image_ops.crop_and_resize.__doc__
 
 
 @tf_export(v1=['image.extract_glimpse'])
diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index 79133331fc..c620712b1d 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -1341,13 +1341,14 @@ def reduce_sum(input_tensor, axis=None, keepdims=False, name=None):
     None, "keep_dims is deprecated, use keepdims instead", "keep_dims")
 @deprecation.deprecated_args(
     None, "reduction_indices is deprecated, use axis instead", "axis")
-def count_nonzero(input_tensor,
+def count_nonzero(input_tensor=None,
                   axis=None,
                   keepdims=None,
                   dtype=dtypes.int64,
                   name=None,
                   reduction_indices=None,
-                  keep_dims=None):
+                  keep_dims=None,
+                  input=None):  # pylint: disable=redefined-builtin
   """Computes number of nonzero elements across dimensions of a tensor.
 
   Reduces `input_tensor` along the dimensions given in `axis`.
@@ -1393,12 +1394,15 @@ def count_nonzero(input_tensor,
     name: A name for the operation (optional).
     reduction_indices: The old (deprecated) name for axis.
     keep_dims: Deprecated alias for `keepdims`.
+    input: Overrides input_tensor. For compatibility.
 
   Returns:
     The reduced tensor (number of nonzero values).
   """
   keepdims = deprecation.deprecated_argument_lookup("keepdims", keepdims,
                                                     "keep_dims", keep_dims)
+  input_tensor = deprecation.deprecated_argument_lookup(
+      "input", input, "input_tensor", input_tensor)
   axis = deprecation.deprecated_argument_lookup(
       "axis", axis,
       "reduction_indices", reduction_indices
diff --git a/tensorflow/python/ops/nn_impl.py b/tensorflow/python/ops/nn_impl.py
index 17174a313d..1853323cad 100644
--- a/tensorflow/python/ops/nn_impl.py
+++ b/tensorflow/python/ops/nn_impl.py
@@ -456,7 +456,8 @@ def depthwise_conv2d(input,
                      padding,
                      rate=None,
                      name=None,
-                     data_format=None):
+                     data_format=None,
+                     dilations=None):
   """Depthwise 2-D convolution.
 
   Given a 4D input tensor ('NHWC' or 'NCHW' data formats)
@@ -492,12 +493,14 @@ def depthwise_conv2d(input,
       greater than 1, then all values of strides must be 1.
     name: A name for this operation (optional).
     data_format: The data format for input. Either "NHWC" (default) or "NCHW".
+    dilations: Alias of rate.
 
   Returns:
     A 4-D `Tensor` with shape according to `data_format`.  E.g., for
     "NHWC" format, shape is
     `[batch, out_height, out_width, in_channels * channel_multiplier].`
   """
+  rate = deprecated_argument_lookup("dilations", dilations, "rate", rate)
   with ops.name_scope(name, "depthwise", [input, filter]) as name:
     input = ops.convert_to_tensor(input, name="tensor_in")
     filter = ops.convert_to_tensor(filter, name="filter_in")
@@ -591,7 +594,8 @@ def separable_conv2d(input,
                      padding,
                      rate=None,
                      name=None,
-                     data_format=None):
+                     data_format=None,
+                     dilations=None):
   """2-D convolution with separable filters.
 
   Performs a depthwise convolution that acts separately on channels followed by
@@ -631,12 +635,14 @@ def separable_conv2d(input,
       greater than 1, then all values of strides must be 1.
     name: A name for this operation (optional).
     data_format: The data format for input. Either "NHWC" (default) or "NCHW".
+    dilations: Alias of rate.
 
   Returns:
     A 4-D `Tensor` with shape according to 'data_format'. For
       example, with data_format="NHWC", shape is [batch, out_height,
       out_width, out_channels].
   """
+  rate = deprecated_argument_lookup("dilations", dilations, "rate", rate)
   with ops.name_scope(name, "separable_conv2d",
                       [input, depthwise_filter, pointwise_filter]) as name:
     input = ops.convert_to_tensor(input, name="tensor_in")
@@ -751,7 +757,8 @@ def separable_conv2d_v2(
 
 
 @tf_export(v1=["nn.sufficient_statistics"])
-def sufficient_statistics(x, axes, shift=None, keep_dims=False, name=None):
+def sufficient_statistics(x, axes, shift=None, keep_dims=False, name=None,
+                          keepdims=None):
   """Calculate the sufficient statistics for the mean and variance of `x`.
 
   These sufficient statistics are computed using the one pass algorithm on
@@ -766,6 +773,7 @@ def sufficient_statistics(x, axes, shift=None, keep_dims=False, name=None):
       close to the true mean provides the most numerically stable results.
     keep_dims: produce statistics with the same dimensionality as the input.
     name: Name used to scope the operations that compute the sufficient stats.
+    keepdims: Alias for keep_dims.
 
   Returns:
     Four `Tensor` objects of the same type as `x`:
@@ -776,6 +784,8 @@ def sufficient_statistics(x, axes, shift=None, keep_dims=False, name=None):
     * the shift by which the mean must be corrected or None if `shift` is None.
   """
   axes = list(set(axes))
+  keep_dims = deprecated_argument_lookup(
+      "keepdims", keepdims, "keep_dims", keep_dims)
   with ops.name_scope(name, "sufficient_statistics", [x, shift]):
     x = ops.convert_to_tensor(x, name="x")
     x_shape = x.get_shape()
@@ -867,7 +877,8 @@ def moments(
     axes,
     shift=None,  # pylint: disable=unused-argument
     name=None,
-    keep_dims=False):
+    keep_dims=False,
+    keepdims=None):
   """Calculate the mean and variance of `x`.
 
   The mean and variance are calculated by aggregating the contents of `x`
@@ -890,10 +901,13 @@ def moments(
     shift: Not used in the current implementation
     name: Name used to scope the operations that compute the moments.
     keep_dims: produce moments with the same dimensionality as the input.
+    keepdims: Alias to keep_dims.
 
   Returns:
     Two `Tensor` objects: `mean` and `variance`.
   """
+  keep_dims = deprecated_argument_lookup(
+      "keepdims", keepdims, "keep_dims", keep_dims)
   with ops.name_scope(name, "moments", [x, axes]):
     # The dynamic range of fp16 is too limited to support the collection of
     # sufficient statistics. As a workaround we simply perform the operations
@@ -957,7 +971,8 @@ def moments_v2(
 
 
 @tf_export(v1=["nn.weighted_moments"])
-def weighted_moments(x, axes, frequency_weights, name=None, keep_dims=False):
+def weighted_moments(x, axes, frequency_weights, name=None, keep_dims=False,
+                     keepdims=None):
   """Returns the frequency-weighted mean and variance of `x`.
 
   Args:
@@ -968,10 +983,13 @@ def weighted_moments(x, axes, frequency_weights, name=None, keep_dims=False):
       broadcast with x.
     name: Name used to scope the operation.
     keep_dims: Produce moments with the same dimensionality as the input.
+    keepdims: Alias of keep_dims.
 
   Returns:
     Two tensors: `weighted_mean` and `weighted_variance`.
   """
+  keep_dims = deprecated_argument_lookup(
+      "keepdims", keepdims, "keep_dims", keep_dims)
   with ops.name_scope(name, "weighted_moments", [x, frequency_weights, axes]):
     x = ops.convert_to_tensor(x, name="x")
     frequency_weights = ops.convert_to_tensor(
@@ -1184,14 +1202,17 @@ def fused_batch_norm(
 
 
 @tf_export(v1=["nn.batch_norm_with_global_normalization"])
-def batch_norm_with_global_normalization(t,
-                                         m,
-                                         v,
-                                         beta,
-                                         gamma,
-                                         variance_epsilon,
-                                         scale_after_normalization,
-                                         name=None):
+def batch_norm_with_global_normalization(t=None,
+                                         m=None,
+                                         v=None,
+                                         beta=None,
+                                         gamma=None,
+                                         variance_epsilon=None,
+                                         scale_after_normalization=None,
+                                         name=None,
+                                         input=None,  # pylint: disable=redefined-builtin
+                                         mean=None,
+                                         variance=None):
   """Batch normalization.
 
   This op is deprecated. See `tf.nn.batch_normalization`.
@@ -1213,10 +1234,16 @@ def batch_norm_with_global_normalization(t,
     scale_after_normalization: A bool indicating whether the resulted tensor
       needs to be multiplied with gamma.
     name: A name for this operation (optional).
+    input: Alias for t.
+    mean: Alias for m.
+    variance: Alias for v.
 
   Returns:
      A batch-normalized `t`.
   """
+  t = deprecated_argument_lookup("input", input, "t", t)
+  m = deprecated_argument_lookup("mean", mean, "m", m)
+  v = deprecated_argument_lookup("variance", variance, "v", v)
   return batch_normalization(t, m, v, beta, gamma if scale_after_normalization
                              else None, variance_epsilon, name)
 
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 67f88995ba..fd1173bc3a 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -298,6 +298,24 @@ def dilation2d_v2(
                                name=name)
 
 
+@tf_export(v1=["nn.dilation2d"])
+def dilation2d_v1(  # pylint: disable=missing-docstring
+    input,  # pylint: disable=redefined-builtin
+    filter=None,  # pylint: disable=redefined-builtin
+    strides=None,
+    rates=None,
+    padding=None,
+    name=None,
+    filters=None,
+    dilations=None):
+  filter = deprecated_argument_lookup("filters", filters, "filter", filter)
+  rates = deprecated_argument_lookup("dilations", dilations, "rates", rates)
+  return gen_nn_ops.dilation2d(input, filter, strides, rates, padding, name)
+
+
+dilation2d_v1.__doc__ = gen_nn_ops.dilation2d.__doc__
+
+
 @tf_export("nn.with_space_to_batch")
 def with_space_to_batch(
     input,  # pylint: disable=redefined-builtin
@@ -744,7 +762,9 @@ def convolution(
     strides=None,
     dilation_rate=None,
     name=None,
-    data_format=None):
+    data_format=None,
+    filters=None,
+    dilations=None):
   # pylint: disable=line-too-long
   """Computes sums of N-D convolutions (actually cross-correlation).
 
@@ -830,6 +850,7 @@ def convolution(
       starts with "NC").  For N=1, the valid values are "NWC" (default) and
       "NCW".  For N=2, the valid values are "NHWC" (default) and "NCHW".
       For N=3, the valid values are "NDHWC" (default) and "NCDHW".
+    filters: Alias of filter.
 
   Returns:
     A `Tensor` with the same type as `input` of shape
@@ -857,6 +878,9 @@ def convolution(
       is other than `"VALID"` or `"SAME"`, or if data_format is invalid.
 
   """
+  filter = deprecated_argument_lookup("filters", filters, "filter", filter)
+  dilation_rate = deprecated_argument_lookup(
+      "dilations", dilations, "dilation_rate", dilation_rate)
   # pylint: enable=line-too-long
   with ops.name_scope(name, "convolution", [input, filter]) as name:
     input = ops.convert_to_tensor(input, name="input")  # pylint: disable=redefined-builtin
@@ -998,7 +1022,8 @@ def pool(
     dilation_rate=None,
     strides=None,
     name=None,
-    data_format=None):
+    data_format=None,
+    dilations=None):
   # pylint: disable=line-too-long
   """Performs an N-D pooling operation.
 
@@ -1055,6 +1080,7 @@ def pool(
       starts with "NC").  For N=1, the valid values are "NWC" (default) and
       "NCW".  For N=2, the valid values are "NHWC" (default) and "NCHW".
       For N=3, the valid values are "NDHWC" (default) and "NCDHW".
+    dilations: Alias for dilation_rate
 
   Returns:
     Tensor of rank N+2, of shape
@@ -1079,6 +1105,8 @@ def pool(
     ValueError: if arguments are invalid.
 
   """
+  dilation_rate = deprecated_argument_lookup(
+      "dilations", dilations, "dilation_rate", dilation_rate)
   # pylint: enable=line-too-long
   with ops.name_scope(name, "%s_pool" % (pooling_type.lower()),
                       [input]) as scope:
@@ -1520,13 +1548,14 @@ def conv2d_v2(input,  # pylint: disable=redefined-builtin
 @tf_export(v1=["nn.conv2d"])
 def conv2d(  # pylint: disable=redefined-builtin,dangerous-default-value
     input,
-    filter,
-    strides,
-    padding,
+    filter=None,
+    strides=None,
+    padding=None,
     use_cudnn_on_gpu=True,
     data_format="NHWC",
     dilations=[1, 1, 1, 1],
-    name=None):
+    name=None,
+    filters=None):
   r"""Computes a 2-D convolution given 4-D `input` and `filter` tensors.
 
   Given an input tensor of shape `[batch, in_height, in_width, in_channels]`
@@ -1585,10 +1614,13 @@ def conv2d(  # pylint: disable=redefined-builtin,dangerous-default-value
       value of `data_format`, see above for details. Dilations in the batch and
       depth dimensions must be 1.
     name: A name for the operation (optional).
+    filters: Alias for filter.
 
   Returns:
     A `Tensor`. Has the same type as `input`.
   """
+  filter = deprecation.deprecated_argument_lookup(
+      "filters", filters, "filter", filter)
   padding, explicit_paddings = _convert_padding(padding)
   return gen_nn_ops.conv2d(input,  # pylint: disable=redefined-builtin
                            filter,
@@ -1792,14 +1824,15 @@ def conv2d_backprop_input_v2(input_sizes,
 @tf_export(v1=["nn.conv2d_backprop_input"])
 def conv2d_backprop_input(  # pylint: disable=redefined-builtin,dangerous-default-value
     input_sizes,
-    filter,
-    out_backprop,
-    strides,
-    padding,
+    filter=None,
+    out_backprop=None,
+    strides=None,
+    padding=None,
     use_cudnn_on_gpu=True,
     data_format="NHWC",
     dilations=[1, 1, 1, 1],
-    name=None):
+    name=None,
+    filters=None):
   r"""Computes the gradients of convolution with respect to the input.
 
   Args:
@@ -1839,10 +1872,13 @@ def conv2d_backprop_input(  # pylint: disable=redefined-builtin,dangerous-defaul
       the value of `data_format`, see above for details. Dilations in the batch
       and depth dimensions must be 1.
     name: A name for the operation (optional).
+    filters: Alias for filter.
 
   Returns:
     A `Tensor`. Has the same type as `filter`.
   """
+  filter = deprecation.deprecated_argument_lookup(
+      "filters", filters, "filter", filter)
   padding, explicit_paddings = _convert_padding(padding)
   return gen_nn_ops.conv2d_backprop_input(
       input_sizes, filter, out_backprop, strides, padding, use_cudnn_on_gpu,
@@ -1851,13 +1887,15 @@ def conv2d_backprop_input(  # pylint: disable=redefined-builtin,dangerous-defaul
 
 @tf_export(v1=["nn.conv2d_transpose"])
 def conv2d_transpose(
-    value,
-    filter,  # pylint: disable=redefined-builtin
-    output_shape,
-    strides,
+    value=None,
+    filter=None,  # pylint: disable=redefined-builtin
+    output_shape=None,
+    strides=None,
     padding="SAME",
     data_format="NHWC",
-    name=None):
+    name=None,
+    input=None,  # pylint: disable=redefined-builtin
+    filters=None):
   """The transpose of `conv2d`.
 
   This operation is sometimes called "deconvolution" after [Deconvolutional
@@ -1880,6 +1918,8 @@ def conv2d_transpose(
       See the "returns" section of `tf.nn.convolution` for details.
     data_format: A string. 'NHWC' and 'NCHW' are supported.
     name: Optional name for the returned tensor.
+    input: Alias for value.
+    filters: Alias for filter.
 
   Returns:
     A `Tensor` with the same type as `value`.
@@ -1888,6 +1928,8 @@ def conv2d_transpose(
     ValueError: If input/output depth does not match `filter`'s shape, or if
       padding is other than `'VALID'` or `'SAME'`.
   """
+  value = deprecated_argument_lookup("input", input, "value", value)
+  filter = deprecated_argument_lookup("filters", filters, "filter", filter)
   with ops.name_scope(name, "conv2d_transpose",
                       [value, filter, output_shape]) as name:
     if data_format not in ("NCHW", "NHWC"):
@@ -2119,20 +2161,39 @@ def conv3d_v2(input,  # pylint: disable=redefined-builtin,missing-docstring
                            data_format=data_format,
                            dilations=dilations,
                            name=name)
-tf_export(v1=["nn.conv3d"])(gen_nn_ops.conv3d)
+
+
+@tf_export(v1=["nn.conv3d"])
+def conv3d_v1(  # pylint: disable=missing-docstring,dangerous-default-value
+    input,  # pylint: disable=redefined-builtin
+    filter=None,  # pylint: disable=redefined-builtin
+    strides=None,
+    padding=None,
+    data_format="NDHWC",
+    dilations=[1, 1, 1, 1, 1],
+    name=None,
+    filters=None):
+  filter = deprecated_argument_lookup("filters", filters, "filter", filter)
+  return gen_nn_ops.conv3d(
+      input, filter, strides, padding, data_format, dilations, name)
+
+
 conv3d_v2.__doc__ = deprecation.rewrite_argument_docstring(
     gen_nn_ops.conv3d.__doc__, "filter", "filters")
+conv3d_v1.__doc__ = gen_nn_ops.conv3d.__doc__
 
 
 @tf_export(v1=["nn.conv3d_transpose"])
 def conv3d_transpose(
     value,
-    filter,  # pylint: disable=redefined-builtin
-    output_shape,
-    strides,
+    filter=None,  # pylint: disable=redefined-builtin
+    output_shape=None,
+    strides=None,
     padding="SAME",
     data_format="NDHWC",
-    name=None):
+    name=None,
+    input=None,  # pylint: disable=redefined-builtin
+    filters=None):
   """The transpose of `conv3d`.
 
   This operation is sometimes called "deconvolution" after [Deconvolutional
@@ -2155,6 +2216,8 @@ def conv3d_transpose(
     data_format: A string, either `'NDHWC'` or `'NCDHW`' specifying the layout
       of the input and output tensors. Defaults to `'NDHWC'`.
     name: Optional name for the returned tensor.
+    input: Alias of value.
+    filters: Alias of filter.
 
   Returns:
     A `Tensor` with the same type as `value`.
@@ -2163,6 +2226,8 @@ def conv3d_transpose(
     ValueError: If input/output depth does not match `filter`'s shape, or if
       padding is other than `'VALID'` or `'SAME'`.
   """
+  filter = deprecated_argument_lookup("filters", filters, "filter", filter)
+  value = deprecated_argument_lookup("input", input, "value", value)
   with ops.name_scope(name, "conv3d_transpose",
                       [value, filter, output_shape]) as name:
     value = ops.convert_to_tensor(value, name="value")
@@ -2779,7 +2844,8 @@ def softmax_cross_entropy_with_logits(
     labels=None,
     logits=None,
     dim=-1,
-    name=None):
+    name=None,
+    axis=None):
   """Computes softmax cross entropy between `logits` and `labels`.
 
   Measures the probability error in discrete classification tasks in which the
@@ -2819,12 +2885,14 @@ def softmax_cross_entropy_with_logits(
     logits: Unscaled log probabilities.
     dim: The class dimension. Defaulted to -1 which is the last dimension.
     name: A name for the operation (optional).
+    axis: Alias for dim.
 
   Returns:
     A `Tensor` that contains the softmax cross entropy loss. Its type is the
     same as `logits` and its shape is the same as `labels` except that it does
     not have the last dimension of `labels`.
   """
+  dim = deprecated_argument_lookup("axis", axis, "dim", dim)
   _ensure_xent_args("softmax_cross_entropy_with_logits", _sentinel, labels,
                     logits)
 
@@ -3136,6 +3204,24 @@ def max_pool_with_argmax_v2(input,
                                          Targmax=output_dtype,
                                          name=name)
 
+
+@tf_export(v1=["nn.max_pool_with_argmax"])
+def max_pool_with_argmax_v1(input,  # pylint: disable=missing-docstring,invalid-name
+                            ksize,
+                            strides,
+                            padding,
+                            data_format="NHWC",
+                            Targmax=dtypes.int64,  # pylint: disable=invalid-name
+                            name=None,
+                            output_dtype=None):
+  Targmax = deprecated_argument_lookup(
+      "output_dtype", output_dtype, "Targmax", Targmax)
+  if output_dtype is not None:
+    Targmax = output_dtype
+  return gen_nn_ops.max_pool_with_argmax(
+      input, ksize, strides, padding, data_format, Targmax, name)
+
+max_pool_with_argmax_v1.__doc__ = gen_nn_ops.max_pool_with_argmax.__doc__
 # pylint: enable=redefined-builtin
 
 
@@ -3745,13 +3831,14 @@ def fractional_avg_pool_v2(value,
     "`NHWC` for data_format is deprecated, use `NWC` instead",
     warn_once=True,
     data_format="NHWC")
-def conv1d(value,
-           filters,
-           stride,
-           padding,
+def conv1d(value=None,
+           filters=None,
+           stride=None,
+           padding=None,
            use_cudnn_on_gpu=None,
            data_format=None,
-           name=None):
+           name=None,
+           input=None):  # pylint: disable=redefined-builtin
   r"""Computes a 1-D convolution given 3-D input and filter tensors.
 
   Given an input tensor of shape
@@ -3788,6 +3875,7 @@ def conv1d(value,
       [batch, in_width, in_channels].  The `"NCW"` format stores
       data as [batch, in_channels, in_width].
     name: A name for the operation (optional).
+    input: Alias for value.
 
   Returns:
     A `Tensor`.  Has the same type as input.
@@ -3795,6 +3883,7 @@ def conv1d(value,
   Raises:
     ValueError: if `data_format` is invalid.
   """
+  value = deprecation.deprecated_argument_lookup("input", input, "value", value)
   with ops.name_scope(name, "conv1d", [value, filters]) as name:
     # Reshape the input tensor to [batch, 1, in_width, in_channels]
     if data_format is None or data_format == "NHWC" or data_format == "NWC":
diff --git a/tensorflow/python/ops/numerics.py b/tensorflow/python/ops/numerics.py
index 0ab39ad0a8..2aba42ef89 100644
--- a/tensorflow/python/ops/numerics.py
+++ b/tensorflow/python/ops/numerics.py
@@ -30,18 +30,23 @@ from tensorflow.python.util.tf_export import tf_export
 
 @tf_export(v1=["debugging.assert_all_finite", "verify_tensor_all_finite"])
 @deprecation.deprecated_endpoints("verify_tensor_all_finite")
-def verify_tensor_all_finite(t, msg, name=None):
+def verify_tensor_all_finite(t=None, msg=None, name=None, x=None, message=None):
   """Assert that the tensor does not contain any NaN's or Inf's.
 
   Args:
     t: Tensor to check.
     msg: Message to log on failure.
     name: A name for this operation (optional).
+    x: Alias for t.
+    message: Alias for msg.
 
   Returns:
     Same tensor as `t`.
   """
-  return verify_tensor_all_finite_v2(t, msg, name)
+  x = deprecation.deprecated_argument_lookup("x", x, "t", t)
+  message = deprecation.deprecated_argument_lookup(
+      "message", message, "msg", msg)
+  return verify_tensor_all_finite_v2(x, message, name)
 
 
 @tf_export("debugging.assert_all_finite", v1=[])
diff --git a/tensorflow/python/ops/sparse_ops.py b/tensorflow/python/ops/sparse_ops.py
index a149d98730..db05eb1315 100644
--- a/tensorflow/python/ops/sparse_ops.py
+++ b/tensorflow/python/ops/sparse_ops.py
@@ -197,7 +197,8 @@ def sparse_concat(axis,
                   sp_inputs,
                   name=None,
                   expand_nonconcat_dim=False,
-                  concat_dim=None):
+                  concat_dim=None,
+                  expand_nonconcat_dims=None):
   """Concatenates a list of `SparseTensor` along the specified dimension.
 
   Concatenation is with respect to the dense versions of each sparse input.
@@ -286,6 +287,7 @@ def sparse_concat(axis,
     expand_nonconcat_dim: Whether to allow the expansion in the non-concat
       dimensions. Defaulted to False.
     concat_dim: The old (deprecated) name for axis.
+    expand_nonconcat_dims: alias for expand_nonconcat_dim
 
   Returns:
     A `SparseTensor` with the concatenated output.
@@ -293,6 +295,11 @@ def sparse_concat(axis,
   Raises:
     TypeError: If `sp_inputs` is not a list of `SparseTensor`.
   """
+  expand_nonconcat_dim = deprecation.deprecated_argument_lookup(
+      "expand_nonconcat_dims", expand_nonconcat_dims,
+      "expand_nonconcat_dim", expand_nonconcat_dim)
+  if expand_nonconcat_dims is not None:
+    expand_nonconcat_dim = expand_nonconcat_dims
   axis = deprecation.deprecated_argument_lookup("axis", axis, "concat_dim",
                                                 concat_dim)
   return sparse_concat_v2(axis, sp_inputs, expand_nonconcat_dim, name)
diff --git a/tensorflow/python/ops/string_ops.py b/tensorflow/python/ops/string_ops.py
index 9967f48060..f75634d856 100644
--- a/tensorflow/python/ops/string_ops.py
+++ b/tensorflow/python/ops/string_ops.py
@@ -321,7 +321,10 @@ def reduce_join(inputs, axis=None,  # pylint: disable=missing-docstring
                 keep_dims=False,
                 separator="",
                 name=None,
-                reduction_indices=None):
+                reduction_indices=None,
+                keepdims=None):
+  keep_dims = deprecation.deprecated_argument_lookup(
+      "keepdims", keepdims, "keep_dims", keep_dims)
   inputs_t = ops.convert_to_tensor(inputs)
   reduction_indices = _reduce_join_reduction_dims(
       inputs_t, axis, reduction_indices)
@@ -422,9 +425,19 @@ def string_to_number(input, out_type=dtypes.float32, name=None):
     A `Tensor` of type `out_type`.
   """
   return gen_parsing_ops.string_to_number(input, out_type, name)
-tf_export(v1=["strings.to_number", "string_to_number"])(
-    gen_parsing_ops.string_to_number
-    )
+
+
+@tf_export(v1=["strings.to_number", "string_to_number"])
+def string_to_number_v1(
+    string_tensor=None,
+    out_type=dtypes.float32,
+    name=None,
+    input=None):
+  string_tensor = deprecation.deprecated_argument_lookup(
+      "input", input, "string_tensor", string_tensor)
+  return gen_parsing_ops.string_to_number(string_tensor, out_type, name)
+
+string_to_number_v1.__doc__ = gen_parsing_ops.string_to_number.__doc__
 
 
 @tf_export("strings.to_hash_bucket", v1=[])
@@ -450,6 +463,16 @@ def string_to_hash_bucket(input, num_buckets, name=None):
   """
   # pylint: enable=line-too-long
   return gen_string_ops.string_to_hash_bucket(input, num_buckets, name)
-tf_export(v1=["strings.to_hash_bucket", "string_to_hash_bucket"])(
-    gen_string_ops.string_to_hash_bucket
-    )
+
+
+@tf_export(v1=["strings.to_hash_bucket", "string_to_hash_bucket"])
+def string_to_hash_bucket_v1(
+    string_tensor=None,
+    num_buckets=None,
+    name=None,
+    input=None):
+  string_tensor = deprecation.deprecated_argument_lookup(
+      "input", input, "string_tensor", string_tensor)
+  return gen_string_ops.string_to_hash_bucket(string_tensor, num_buckets, name)
+
+string_to_hash_bucket_v1.__doc__ = gen_string_ops.string_to_hash_bucket.__doc__
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.debugging.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.debugging.pbtxt
index 8a7f1e9363..bf6816a0ab 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.debugging.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.debugging.pbtxt
@@ -6,7 +6,7 @@ tf_module {
   }
   member_method {
     name: "assert_all_finite"
-    argspec: "args=[\'t\', \'msg\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'t\', \'msg\', \'name\', \'x\', \'message\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "assert_equal"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt
index e74a74030f..102ca132a2 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt
@@ -42,7 +42,7 @@ tf_module {
   }
   member_method {
     name: "crop_and_resize"
-    argspec: "args=[\'image\', \'boxes\', \'box_ind\', \'crop_size\', \'method\', \'extrapolation_value\', \'name\'], varargs=None, keywords=None, defaults=[\'bilinear\', \'0\', \'None\'], "
+    argspec: "args=[\'image\', \'boxes\', \'box_ind\', \'crop_size\', \'method\', \'extrapolation_value\', \'name\', \'box_indices\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'bilinear\', \'0\', \'None\', \'None\'], "
   }
   member_method {
     name: "crop_to_bounding_box"
@@ -90,7 +90,7 @@ tf_module {
   }
   member_method {
     name: "extract_image_patches"
-    argspec: "args=[\'images\', \'ksizes\', \'strides\', \'rates\', \'padding\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'images\', \'ksizes\', \'strides\', \'rates\', \'padding\', \'name\', \'sizes\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "extract_jpeg_shape"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.math.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.math.pbtxt
index 2e298d8cb6..4e211be968 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.math.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.math.pbtxt
@@ -102,7 +102,7 @@ tf_module {
   }
   member_method {
     name: "count_nonzero"
-    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'dtype\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \"<dtype: \'int64\'>\", \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'dtype\', \'name\', \'reduction_indices\', \'keep_dims\', \'input\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \"<dtype: \'int64\'>\", \'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "cumprod"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
index 40e20f8c91..4a55224248 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
@@ -30,7 +30,7 @@ tf_module {
   }
   member_method {
     name: "batch_norm_with_global_normalization"
-    argspec: "args=[\'t\', \'m\', \'v\', \'beta\', \'gamma\', \'variance_epsilon\', \'scale_after_normalization\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'t\', \'m\', \'v\', \'beta\', \'gamma\', \'variance_epsilon\', \'scale_after_normalization\', \'name\', \'input\', \'mean\', \'variance\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "batch_normalization"
@@ -54,11 +54,11 @@ tf_module {
   }
   member_method {
     name: "conv1d"
-    argspec: "args=[\'value\', \'filters\', \'stride\', \'padding\', \'use_cudnn_on_gpu\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'value\', \'filters\', \'stride\', \'padding\', \'use_cudnn_on_gpu\', \'data_format\', \'name\', \'input\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "conv2d"
-    argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'use_cudnn_on_gpu\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'NHWC\', \'[1, 1, 1, 1]\', \'None\'], "
+    argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'use_cudnn_on_gpu\', \'data_format\', \'dilations\', \'name\', \'filters\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'NHWC\', \'[1, 1, 1, 1]\', \'None\', \'None\'], "
   }
   member_method {
     name: "conv2d_backprop_filter"
@@ -66,15 +66,15 @@ tf_module {
   }
   member_method {
     name: "conv2d_backprop_input"
-    argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'use_cudnn_on_gpu\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'NHWC\', \'[1, 1, 1, 1]\', \'None\'], "
+    argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'use_cudnn_on_gpu\', \'data_format\', \'dilations\', \'name\', \'filters\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'True\', \'NHWC\', \'[1, 1, 1, 1]\', \'None\', \'None\'], "
   }
   member_method {
     name: "conv2d_transpose"
-    argspec: "args=[\'value\', \'filter\', \'output_shape\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'SAME\', \'NHWC\', \'None\'], "
+    argspec: "args=[\'value\', \'filter\', \'output_shape\', \'strides\', \'padding\', \'data_format\', \'name\', \'input\', \'filters\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'SAME\', \'NHWC\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "conv3d"
-    argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NDHWC\', \'[1, 1, 1, 1, 1]\', \'None\'], "
+    argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\', \'filters\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'NDHWC\', \'[1, 1, 1, 1, 1]\', \'None\', \'None\'], "
   }
   member_method {
     name: "conv3d_backprop_filter"
@@ -86,11 +86,11 @@ tf_module {
   }
   member_method {
     name: "conv3d_transpose"
-    argspec: "args=[\'value\', \'filter\', \'output_shape\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'SAME\', \'NDHWC\', \'None\'], "
+    argspec: "args=[\'value\', \'filter\', \'output_shape\', \'strides\', \'padding\', \'data_format\', \'name\', \'input\', \'filters\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'SAME\', \'NDHWC\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "convolution"
-    argspec: "args=[\'input\', \'filter\', \'padding\', \'strides\', \'dilation_rate\', \'name\', \'data_format\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input\', \'filter\', \'padding\', \'strides\', \'dilation_rate\', \'name\', \'data_format\', \'filters\', \'dilations\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "crelu"
@@ -110,7 +110,7 @@ tf_module {
   }
   member_method {
     name: "ctc_loss"
-    argspec: "args=[\'labels\', \'inputs\', \'sequence_length\', \'preprocess_collapse_repeated\', \'ctc_merge_repeated\', \'ignore_longer_outputs_than_inputs\', \'time_major\'], varargs=None, keywords=None, defaults=[\'False\', \'True\', \'False\', \'True\'], "
+    argspec: "args=[\'labels\', \'inputs\', \'sequence_length\', \'preprocess_collapse_repeated\', \'ctc_merge_repeated\', \'ignore_longer_outputs_than_inputs\', \'time_major\', \'logits\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'True\', \'False\', \'True\', \'None\'], "
   }
   member_method {
     name: "ctc_loss_v2"
@@ -126,7 +126,7 @@ tf_module {
   }
   member_method {
     name: "depthwise_conv2d"
-    argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'rate\', \'name\', \'data_format\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'rate\', \'name\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "depthwise_conv2d_backprop_filter"
@@ -150,7 +150,7 @@ tf_module {
   }
   member_method {
     name: "dilation2d"
-    argspec: "args=[\'input\', \'filter\', \'strides\', \'rates\', \'padding\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'input\', \'filter\', \'strides\', \'rates\', \'padding\', \'name\', \'filters\', \'dilations\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "dropout"
@@ -242,11 +242,11 @@ tf_module {
   }
   member_method {
     name: "max_pool_with_argmax"
-    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'Targmax\', \'name\'], varargs=None, keywords=None, defaults=[\"<dtype: \'int64\'>\", \'None\'], "
+    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\', \'Targmax\', \'name\', \'output_dtype\'], varargs=None, keywords=None, defaults=[\'NHWC\', \"<dtype: \'int64\'>\", \'None\', \'None\'], "
   }
   member_method {
     name: "moments"
-    argspec: "args=[\'x\', \'axes\', \'shift\', \'name\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\'], "
+    argspec: "args=[\'x\', \'axes\', \'shift\', \'name\', \'keep_dims\', \'keepdims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'None\'], "
   }
   member_method {
     name: "nce_loss"
@@ -258,7 +258,7 @@ tf_module {
   }
   member_method {
     name: "pool"
-    argspec: "args=[\'input\', \'window_shape\', \'pooling_type\', \'padding\', \'dilation_rate\', \'strides\', \'name\', \'data_format\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input\', \'window_shape\', \'pooling_type\', \'padding\', \'dilation_rate\', \'strides\', \'name\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "quantized_avg_pool"
@@ -306,7 +306,7 @@ tf_module {
   }
   member_method {
     name: "separable_conv2d"
-    argspec: "args=[\'input\', \'depthwise_filter\', \'pointwise_filter\', \'strides\', \'padding\', \'rate\', \'name\', \'data_format\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input\', \'depthwise_filter\', \'pointwise_filter\', \'strides\', \'padding\', \'rate\', \'name\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "sigmoid"
@@ -322,7 +322,7 @@ tf_module {
   }
   member_method {
     name: "softmax_cross_entropy_with_logits"
-    argspec: "args=[\'_sentinel\', \'labels\', \'logits\', \'dim\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'-1\', \'None\'], "
+    argspec: "args=[\'_sentinel\', \'labels\', \'logits\', \'dim\', \'name\', \'axis\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'-1\', \'None\', \'None\'], "
   }
   member_method {
     name: "softmax_cross_entropy_with_logits_v2"
@@ -338,7 +338,7 @@ tf_module {
   }
   member_method {
     name: "space_to_batch"
-    argspec: "args=[\'input\', \'paddings\', \'block_size\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'input\', \'paddings\', \'block_size\', \'name\', \'block_shape\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "space_to_depth"
@@ -362,7 +362,7 @@ tf_module {
   }
   member_method {
     name: "sufficient_statistics"
-    argspec: "args=[\'x\', \'axes\', \'shift\', \'keep_dims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'x\', \'axes\', \'shift\', \'keep_dims\', \'name\', \'keepdims\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'None\'], "
   }
   member_method {
     name: "tanh"
@@ -382,7 +382,7 @@ tf_module {
   }
   member_method {
     name: "weighted_moments"
-    argspec: "args=[\'x\', \'axes\', \'frequency_weights\', \'name\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'False\'], "
+    argspec: "args=[\'x\', \'axes\', \'frequency_weights\', \'name\', \'keep_dims\', \'keepdims\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
   member_method {
     name: "with_space_to_batch"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
index a4b46d4260..1409a3276e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
@@ -858,7 +858,7 @@ tf_module {
   }
   member_method {
     name: "batch_to_space"
-    argspec: "args=[\'input\', \'crops\', \'block_size\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'input\', \'crops\', \'block_size\', \'name\', \'block_shape\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
   member_method {
     name: "batch_to_space_nd"
@@ -974,7 +974,7 @@ tf_module {
   }
   member_method {
     name: "convert_to_tensor"
-    argspec: "args=[\'value\', \'dtype\', \'name\', \'preferred_dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'value\', \'dtype\', \'name\', \'preferred_dtype\', \'dtype_hint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "convert_to_tensor_or_indexed_slices"
@@ -994,7 +994,7 @@ tf_module {
   }
   member_method {
     name: "count_nonzero"
-    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'dtype\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \"<dtype: \'int64\'>\", \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'dtype\', \'name\', \'reduction_indices\', \'keep_dims\', \'input\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \"<dtype: \'int64\'>\", \'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "count_up_to"
@@ -1186,7 +1186,7 @@ tf_module {
   }
   member_method {
     name: "extract_image_patches"
-    argspec: "args=[\'images\', \'ksizes\', \'strides\', \'rates\', \'padding\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'images\', \'ksizes\', \'strides\', \'rates\', \'padding\', \'name\', \'sizes\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "extract_volume_patches"
@@ -1798,7 +1798,7 @@ tf_module {
   }
   member_method {
     name: "reduce_join"
-    argspec: "args=[\'inputs\', \'axis\', \'keep_dims\', \'separator\', \'name\', \'reduction_indices\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'\', \'None\', \'None\'], "
+    argspec: "args=[\'inputs\', \'axis\', \'keep_dims\', \'separator\', \'name\', \'reduction_indices\', \'keepdims\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "reduce_logsumexp"
@@ -2026,7 +2026,7 @@ tf_module {
   }
   member_method {
     name: "space_to_batch"
-    argspec: "args=[\'input\', \'paddings\', \'block_size\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'input\', \'paddings\', \'block_size\', \'name\', \'block_shape\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "space_to_batch_nd"
@@ -2042,7 +2042,7 @@ tf_module {
   }
   member_method {
     name: "sparse_concat"
-    argspec: "args=[\'axis\', \'sp_inputs\', \'name\', \'expand_nonconcat_dim\', \'concat_dim\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'axis\', \'sp_inputs\', \'name\', \'expand_nonconcat_dim\', \'concat_dim\', \'expand_nonconcat_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'None\'], "
   }
   member_method {
     name: "sparse_fill_empty_rows"
@@ -2194,7 +2194,7 @@ tf_module {
   }
   member_method {
     name: "string_to_hash_bucket"
-    argspec: "args=[\'string_tensor\', \'num_buckets\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'string_tensor\', \'num_buckets\', \'name\', \'input\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "string_to_hash_bucket_fast"
@@ -2206,7 +2206,7 @@ tf_module {
   }
   member_method {
     name: "string_to_number"
-    argspec: "args=[\'string_tensor\', \'out_type\', \'name\'], varargs=None, keywords=None, defaults=[\"<dtype: \'float32\'>\", \'None\'], "
+    argspec: "args=[\'string_tensor\', \'out_type\', \'name\', \'input\'], varargs=None, keywords=None, defaults=[\'None\', \"<dtype: \'float32\'>\", \'None\', \'None\'], "
   }
   member_method {
     name: "substr"
@@ -2374,7 +2374,7 @@ tf_module {
   }
   member_method {
     name: "verify_tensor_all_finite"
-    argspec: "args=[\'t\', \'msg\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'t\', \'msg\', \'name\', \'x\', \'message\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "where"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.sparse.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.sparse.pbtxt
index 33e342bc75..d3543e2e19 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.sparse.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.sparse.pbtxt
@@ -14,7 +14,7 @@ tf_module {
   }
   member_method {
     name: "concat"
-    argspec: "args=[\'axis\', \'sp_inputs\', \'name\', \'expand_nonconcat_dim\', \'concat_dim\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'axis\', \'sp_inputs\', \'name\', \'expand_nonconcat_dim\', \'concat_dim\', \'expand_nonconcat_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'None\'], "
   }
   member_method {
     name: "cross"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.strings.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.strings.pbtxt
index 6f4f4c2f31..ada8be9145 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.strings.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.strings.pbtxt
@@ -18,7 +18,7 @@ tf_module {
   }
   member_method {
     name: "reduce_join"
-    argspec: "args=[\'inputs\', \'axis\', \'keep_dims\', \'separator\', \'name\', \'reduction_indices\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'\', \'None\', \'None\'], "
+    argspec: "args=[\'inputs\', \'axis\', \'keep_dims\', \'separator\', \'name\', \'reduction_indices\', \'keepdims\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "regex_full_match"
@@ -42,7 +42,7 @@ tf_module {
   }
   member_method {
     name: "to_hash_bucket"
-    argspec: "args=[\'string_tensor\', \'num_buckets\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'string_tensor\', \'num_buckets\', \'name\', \'input\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "to_hash_bucket_fast"
@@ -54,7 +54,7 @@ tf_module {
   }
   member_method {
     name: "to_number"
-    argspec: "args=[\'string_tensor\', \'out_type\', \'name\'], varargs=None, keywords=None, defaults=[\"<dtype: \'float32\'>\", \'None\'], "
+    argspec: "args=[\'string_tensor\', \'out_type\', \'name\', \'input\'], varargs=None, keywords=None, defaults=[\'None\', \"<dtype: \'float32\'>\", \'None\', \'None\'], "
   }
   member_method {
     name: "unicode_decode"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index aacd5bd716..2bd0853709 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -86,18 +86,30 @@ class TestUpgrade(test_util.TensorFlowTestCase):
   @classmethod
   def setUpClass(cls):
     cls.v2_symbols = {}
-    if not hasattr(tf.compat, "v2"):
-      return
+    cls.v1_symbols = {}
+    if hasattr(tf.compat, "v2"):
 
-    def symbol_collector(unused_path, unused_parent, children):
-      for child in children:
-        _, attr = tf_decorator.unwrap(child[1])
-        api_names_v2 = tf_export.get_v2_names(attr)
-        for name in api_names_v2:
-          cls.v2_symbols["tf." + name] = attr
+      def symbol_collector(unused_path, unused_parent, children):
+        for child in children:
+          _, attr = tf_decorator.unwrap(child[1])
+          api_names_v2 = tf_export.get_v2_names(attr)
+          for name in api_names_v2:
+            cls.v2_symbols["tf." + name] = attr
+
+      visitor = public_api.PublicAPIVisitor(symbol_collector)
+      traverse.traverse(tf.compat.v2, visitor)
+
+    if hasattr(tf.compat, "v1"):
 
-    visitor = public_api.PublicAPIVisitor(symbol_collector)
-    traverse.traverse(tf.compat.v2, visitor)
+      def symbol_collector_v1(unused_path, unused_parent, children):
+        for child in children:
+          _, attr = tf_decorator.unwrap(child[1])
+          api_names_v1 = tf_export.get_v1_names(attr)
+          for name in api_names_v1:
+            cls.v1_symbols["tf." + name] = attr
+
+      visitor = public_api.PublicAPIVisitor(symbol_collector_v1)
+      traverse.traverse(tf.compat.v1, visitor)
 
   def _upgrade(self, old_file_text):
     in_file = six.StringIO(old_file_text)
@@ -287,6 +299,18 @@ class TestUpgrade(test_util.TensorFlowTestCase):
                 "Invalid argument '%s' in 2.0 when converting\n%s\nto\n%s.\n"
                 "Supported arguments: %s" % (
                     new_arg, text_input, text, str(args_v2)))
+          # 4. Verify that the argument exists in v1 as well.
+          if new_function_name in set(["tf.nn.ctc_loss",
+                                       "tf.saved_model.save"]):
+            continue
+          args_v1 = get_args(self.v1_symbols[new_function_name])
+          args_v1.extend(v2_arg_exceptions)
+          for new_arg in new_args:
+            self.assertIn(
+                new_arg, args_v1,
+                "Invalid argument '%s' in 1.0 when converting\n%s\nto\n%s.\n"
+                "Supported arguments: %s" % (
+                    new_arg, text_input, text, str(args_v1)))
 
     visitor = public_api.PublicAPIVisitor(conversion_visitor)
     visitor.do_not_descend_map["tf"].append("contrib")
-- 
GitLab


From 1e9e2a70031dc0dd244857bc64d2f272ba63aec2 Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne <reedwm@google.com>
Date: Wed, 30 Jan 2019 09:24:38 -0800
Subject: [PATCH 1581/2345] Remove conv2d grad kernels tagged with
 "eigen_tensor".

I do not think anyone uses these kernels, and it is not worth supporting them.

PiperOrigin-RevId: 231608033
---
 .../core/kernels/conv_grad_filter_ops.cc      | 132 +----------------
 .../core/kernels/conv_grad_input_ops.cc       | 133 +-----------------
 2 files changed, 4 insertions(+), 261 deletions(-)

diff --git a/tensorflow/core/kernels/conv_grad_filter_ops.cc b/tensorflow/core/kernels/conv_grad_filter_ops.cc
index 0df05ceb02..5eaddec768 100644
--- a/tensorflow/core/kernels/conv_grad_filter_ops.cc
+++ b/tensorflow/core/kernels/conv_grad_filter_ops.cc
@@ -181,130 +181,6 @@ struct LaunchXsmmBackwardFilter<CPUDevice, float> {
 };
 #endif
 
-template <typename Device, class T>
-class Conv2DFastBackpropFilterOp : public OpKernel {
- public:
-  explicit Conv2DFastBackpropFilterOp(OpKernelConstruction* context)
-      : OpKernel(context) {
-    string data_format;
-    OP_REQUIRES_OK(context, context->GetAttr("data_format", &data_format));
-    OP_REQUIRES(context, FormatFromString(data_format, &data_format_),
-                errors::InvalidArgument("Invalid data format"));
-    OP_REQUIRES(context, data_format_ == FORMAT_NHWC,
-                errors::InvalidArgument(
-                    "Conv2DFastBackpropFilterOp only supports NHWC."));
-    OP_REQUIRES_OK(context, context->GetAttr("strides", &strides_));
-    OP_REQUIRES(context, strides_.size() == 4,
-                errors::InvalidArgument("Sliding window strides field must "
-                                        "specify 4 dimensions"));
-    OP_REQUIRES(
-        context, (strides_[0] == 1 && strides_[3] == 1),
-        errors::InvalidArgument("Current implementation does not yet support "
-                                "strides in the batch and depth dimensions."));
-    OP_REQUIRES(context, strides_[1] > 0 && strides_[2] > 0,
-                errors::InvalidArgument(
-                    "Row and column strides should be larger than 0."));
-    OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
-    OP_REQUIRES(
-        context, padding_ != Padding::EXPLICIT,
-        errors::Unimplemented("Current CPU implementation does not support "
-                              "EXPLICIT padding yet."));
-    std::vector<int64> explicit_paddings;
-    OP_REQUIRES_OK(context,
-                   context->GetAttr("explicit_paddings", &explicit_paddings));
-    OP_REQUIRES_OK(context, CheckValidPadding(padding_, explicit_paddings,
-                                              /*num_dims=*/4, data_format_));
-    OP_REQUIRES_OK(context, context->GetAttr("dilations", &dilations_));
-    OP_REQUIRES(context, dilations_.size() == 4,
-                errors::InvalidArgument("Sliding window dilations field must "
-                                        "specify 4 dimensions"));
-    OP_REQUIRES(context, (dilations_[0] == 1 && dilations_[3] == 1),
-                errors::InvalidArgument(
-                    "Current implementation does not yet support "
-                    "dilations in the batch and depth dimensions."));
-    // TODO(yangzihao): Add a CPU implementation for dilated convolution.
-    OP_REQUIRES(context, (dilations_[1] == 1 && dilations_[2] == 1),
-                errors::InvalidArgument(
-                    "Current Eigen and libxsmm implementations do not "
-                    "yet support dilation rates larger than 1."));
-  }
-
-  void Compute(OpKernelContext* context) override {
-    const Tensor& input = context->input(0);
-    const Tensor& filter_sizes = context->input(1);
-    const Tensor& out_backprop = context->input(2);
-    OP_REQUIRES(
-        context, TensorShapeUtils::IsVector(filter_sizes.shape()),
-        errors::InvalidArgument(
-            "Conv2DBackpropFilter: filter_sizes input must be 1-dim, not ",
-            filter_sizes.dims()));
-    TensorShape filter_shape;
-    OP_REQUIRES_OK(context, TensorShapeUtils::MakeShape(
-                                filter_sizes.vec<int32>(), &filter_shape));
-
-    ConvBackpropDimensions dims;
-    OP_REQUIRES_OK(
-        context,
-        ConvBackpropComputeDimensions(
-            type_string(), /*num_spatial_dims=*/2, input.shape(), filter_shape,
-            out_backprop.shape(), strides_, padding_, data_format_, &dims));
-
-    Tensor* filter_backprop = nullptr;
-    OP_REQUIRES_OK(context,
-                   context->allocate_output(0, filter_shape, &filter_backprop));
-
-    // If there is nothing to compute, return.
-    if (filter_shape.num_elements() == 0) {
-      return;
-    }
-
-#if defined TENSORFLOW_USE_LIBXSMM_CONVOLUTIONS && \
-    defined TENSORFLOW_USE_LIBXSMM_BACKWARD_CONVOLUTIONS
-    int64 pad_top, pad_bottom;
-    int64 pad_left, pad_right;
-    OP_REQUIRES_OK(
-        context,
-        GetWindowedOutputSizeVerbose(
-            dims.spatial_dims[0].input_size, dims.spatial_dims[0].filter_size,
-            dims.spatial_dims[0].stride, padding_,
-            &dims.spatial_dims[0].output_size, &pad_top, &pad_bottom));
-    OP_REQUIRES_OK(
-        context,
-        GetWindowedOutputSizeVerbose(
-            dims.spatial_dims[1].input_size, dims.spatial_dims[1].filter_size,
-            dims.spatial_dims[1].stride, padding_,
-            &dims.spatial_dims[1].output_size, &pad_left, &pad_right));
-
-    if (pad_left == pad_right && pad_top == pad_bottom) {
-      if (LaunchXsmmBackwardFilter<Device, T>()(
-              context, context->eigen_device<Device>(), input.tensor<T, 4>(),
-              filter_backprop->tensor<T, 4>(), out_backprop.tensor<T, 4>(),
-              dims.spatial_dims[0].input_size, dims.spatial_dims[1].input_size,
-              static_cast<int>(dims.spatial_dims[0].stride),
-              static_cast<int>(dims.spatial_dims[1].stride),
-              static_cast<int>(pad_top), static_cast<int>(pad_left),
-              data_format_)) {
-        return;
-      }
-    }
-#endif
-
-    LaunchConv2DBackpropFilterOp<Device, T>()(
-        context, false, false, out_backprop, input,
-        /*row_dilation=*/1, /*col_dilation=*/1, dims.spatial_dims[0].stride,
-        dims.spatial_dims[1].stride, padding_, /*explicit_paddings=*/{},
-        filter_backprop, data_format_);
-  }
-
- private:
-  std::vector<int32> dilations_;
-  std::vector<int32> strides_;
-  Padding padding_;
-  TensorFormat data_format_;
-
-  TF_DISALLOW_COPY_AND_ASSIGN(Conv2DFastBackpropFilterOp);
-};
-
 // Based on implementation written by Yangqing Jia (jiayq).
 template <typename Device, class T>
 class Conv2DCustomBackpropFilterOp : public OpKernel {
@@ -537,12 +413,7 @@ class Conv2DCustomBackpropFilterOp : public OpKernel {
                               .Device(DEVICE_CPU)                             \
                               .Label("custom")                                \
                               .TypeConstraint<T>("T"),                        \
-                          Conv2DCustomBackpropFilterOp<CPUDevice, T>);        \
-  REGISTER_KERNEL_BUILDER(Name("Conv2DBackpropFilter")                        \
-                              .Device(DEVICE_CPU)                             \
-                              .Label("eigen_tensor")                          \
-                              .TypeConstraint<T>("T"),                        \
-                          Conv2DFastBackpropFilterOp<CPUDevice, T>);
+                          Conv2DCustomBackpropFilterOp<CPUDevice, T>);
 
 TF_CALL_half(REGISTER_CPU_KERNELS);
 TF_CALL_float(REGISTER_CPU_KERNELS);
@@ -1098,6 +969,7 @@ REGISTER_KERNEL_BUILDER(Name("Conv2DBackpropFilter")
                         Conv2DSlowBackpropFilterOp<GPUDevice, Eigen::half>);
 
 // To be used inside depthwise_conv_grad_op.cc.
+// TODO(reedwm): Move this and the definition to depthwise_conv_grad_op.cc.
 template struct LaunchConv2DBackpropFilterOp<GPUDevice, float>;
 template struct LaunchConv2DBackpropFilterOp<GPUDevice, Eigen::half>;
 template struct LaunchConv2DBackpropFilterOp<GPUDevice, double>;
diff --git a/tensorflow/core/kernels/conv_grad_input_ops.cc b/tensorflow/core/kernels/conv_grad_input_ops.cc
index 74b97b9864..5520cf7524 100644
--- a/tensorflow/core/kernels/conv_grad_input_ops.cc
+++ b/tensorflow/core/kernels/conv_grad_input_ops.cc
@@ -184,131 +184,6 @@ struct LaunchXsmmBackwardInputConvolution<CPUDevice, float> {
 };
 #endif
 
-template <typename Device, class T>
-class Conv2DFastBackpropInputOp : public OpKernel {
- public:
-  explicit Conv2DFastBackpropInputOp(OpKernelConstruction* context)
-      : OpKernel(context) {
-    string data_format;
-    OP_REQUIRES_OK(context, context->GetAttr("data_format", &data_format));
-    OP_REQUIRES(context, FormatFromString(data_format, &data_format_),
-                errors::InvalidArgument("Invalid data format"));
-    OP_REQUIRES(context, data_format_ == FORMAT_NHWC,
-                errors::InvalidArgument(
-                    "Eigen Conv2DFastBackpropInputOp only supports NHWC."));
-    OP_REQUIRES_OK(context, context->GetAttr("strides", &strides_));
-    OP_REQUIRES(context, strides_.size() == 4,
-                errors::InvalidArgument("Sliding window strides field must "
-                                        "specify 4 dimensions"));
-    OP_REQUIRES(
-        context, (strides_[0] == 1 && strides_[3] == 1),
-        errors::InvalidArgument("Current implementation does not yet support "
-                                "strides in the batch and depth dimensions."));
-    OP_REQUIRES(context, strides_[1] > 0 && strides_[2] > 0,
-                errors::InvalidArgument(
-                    "Row and column strides should be larger than 0."));
-    OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
-    OP_REQUIRES_OK(context, context->GetAttr("dilations", &dilations_));
-    OP_REQUIRES(context, dilations_.size() == 4,
-                errors::InvalidArgument("Sliding window dilations field must "
-                                        "specify 4 dimensions"));
-    OP_REQUIRES(context, (dilations_[0] && dilations_[3]),
-                errors::InvalidArgument(
-                    "Current implementation does not yet support "
-                    "dilations in the batch and depth dimensions."));
-    // TODO(yangzihao): Add a CPU implementation for dilated convolution.
-    OP_REQUIRES(context, (dilations_[1] == 1 && dilations_[2] == 1),
-                errors::InvalidArgument(
-                    "Current Eigen and libxsmm implementations do not "
-                    "yet support dilation rates larger than 1."));
-    OP_REQUIRES(
-        context, padding_ != Padding::EXPLICIT,
-        errors::Unimplemented("Current CPU implementation does not support "
-                              "EXPLICIT padding yet."));
-    std::vector<int64> explicit_paddings;
-    OP_REQUIRES_OK(context,
-                   context->GetAttr("explicit_paddings", &explicit_paddings));
-    OP_REQUIRES_OK(context, CheckValidPadding(padding_, explicit_paddings,
-                                              /*num_dims=*/4, data_format_));
-  }
-
-  void Compute(OpKernelContext* context) override {
-    const Tensor& input_sizes = context->input(0);
-    const Tensor& filter = context->input(1);
-    const Tensor& out_backprop = context->input(2);
-    OP_REQUIRES(
-        context, TensorShapeUtils::IsVector(input_sizes.shape()),
-        errors::InvalidArgument(
-            "Conv2DBackpropInput: input_sizes input must be 1-dim, not ",
-            input_sizes.dims()));
-    TensorShape input_shape;
-    OP_REQUIRES_OK(context, TensorShapeUtils::MakeShape(
-                                input_sizes.vec<int32>(), &input_shape));
-
-    ConvBackpropDimensions dims;
-    OP_REQUIRES_OK(context,
-                   ConvBackpropComputeDimensions(
-                       "Conv2DFastBackpropInput", /*num_spatial_dims=*/2,
-                       input_shape, filter.shape(), out_backprop.shape(),
-                       strides_, padding_, data_format_, &dims));
-
-    Tensor* in_backprop = nullptr;
-    OP_REQUIRES_OK(context,
-                   context->allocate_output(0, input_shape, &in_backprop));
-
-    // If there is nothing to compute, return.
-    if (input_shape.num_elements() == 0) {
-      return;
-    }
-
-#if defined TENSORFLOW_USE_LIBXSMM_CONVOLUTIONS && \
-    defined TENSORFLOW_USE_LIBXSMM_BACKWARD_CONVOLUTIONS
-    int64 pad_top, pad_bottom;
-    int64 pad_left, pad_right;
-    OP_REQUIRES_OK(
-        context,
-        GetWindowedOutputSizeVerbose(
-            dims.spatial_dims[0].input_size, dims.spatial_dims[0].filter_size,
-            dims.spatial_dims[0].stride, padding_,
-            &dims.spatial_dims[0].output_size, &pad_top, &pad_bottom));
-    OP_REQUIRES_OK(
-        context,
-        GetWindowedOutputSizeVerbose(
-            dims.spatial_dims[1].input_size, dims.spatial_dims[1].filter_size,
-            dims.spatial_dims[1].stride, padding_,
-            &dims.spatial_dims[1].output_size, &pad_left, &pad_right));
-
-    if (pad_left == pad_right && pad_top == pad_bottom) {
-      if (LaunchXsmmBackwardInputConvolution<Device, T>()(
-              context, context->eigen_device<Device>(),
-              in_backprop->tensor<T, 4>(), filter.tensor<T, 4>(),
-              out_backprop.tensor<T, 4>(), dims.spatial_dims[0].input_size,
-              dims.spatial_dims[1].input_size,
-              static_cast<int>(dims.spatial_dims[0].stride),
-              static_cast<int>(dims.spatial_dims[1].stride),
-              static_cast<int>(pad_top), static_cast<int>(pad_left),
-              data_format_)) {
-        return;
-      }
-    }
-#endif
-
-    LaunchConv2DBackpropInputOp<Device, T>()(
-        context, false, false, out_backprop, filter,
-        /*row_dilation=*/1, /*col_dilation=*/1, dims.spatial_dims[0].stride,
-        dims.spatial_dims[1].stride, padding_, /*explicit_paddings=*/{},
-        in_backprop, data_format_);
-  }
-
- private:
-  std::vector<int32> dilations_;
-  std::vector<int32> strides_;
-  Padding padding_;
-  TensorFormat data_format_;
-
-  TF_DISALLOW_COPY_AND_ASSIGN(Conv2DFastBackpropInputOp);
-};
-
 // Based on implementation written by Yangqing Jia (jiayq).
 template <typename Device, class T>
 class Conv2DCustomBackpropInputOp : public OpKernel {
@@ -608,12 +483,7 @@ class Conv2DCustomBackpropInputOp : public OpKernel {
                               .Device(DEVICE_CPU)                            \
                               .Label("custom")                               \
                               .TypeConstraint<T>("T"),                       \
-                          Conv2DCustomBackpropInputOp<CPUDevice, T>);        \
-  REGISTER_KERNEL_BUILDER(Name("Conv2DBackpropInput")                        \
-                              .Device(DEVICE_CPU)                            \
-                              .Label("eigen_tensor")                         \
-                              .TypeConstraint<T>("T"),                       \
-                          Conv2DFastBackpropInputOp<CPUDevice, T>);
+                          Conv2DCustomBackpropInputOp<CPUDevice, T>);
 
 TF_CALL_half(REGISTER_CPU_KERNELS);
 TF_CALL_float(REGISTER_CPU_KERNELS);
@@ -1178,6 +1048,7 @@ REGISTER_KERNEL_BUILDER(Name("Conv2DBackpropInput")
                         Conv2DSlowBackpropInputOp<GPUDevice, Eigen::half>);
 
 // To be used inside depthwise_conv_grad_op.cc.
+// TODO(reedwm): Move this and the definition to depthwise_conv_grad_op.cc.
 template struct LaunchConv2DBackpropInputOp<GPUDevice, float>;
 template struct LaunchConv2DBackpropInputOp<GPUDevice, Eigen::half>;
 template struct LaunchConv2DBackpropInputOp<GPUDevice, double>;
-- 
GitLab


From 040a993db329aa98bc17f9d5f744d14753b36f1d Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Wed, 30 Jan 2019 09:44:48 -0800
Subject: [PATCH 1582/2345] More thoroughly search for print functions when
 deciding whether to add from __import__ import print_function.

PiperOrigin-RevId: 231611485
---
 tensorflow/python/autograph/pyct/parser.py      | 3 ++-
 tensorflow/python/autograph/pyct/parser_test.py | 9 +++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/autograph/pyct/parser.py b/tensorflow/python/autograph/pyct/parser.py
index a14966e470..011d80dd62 100644
--- a/tensorflow/python/autograph/pyct/parser.py
+++ b/tensorflow/python/autograph/pyct/parser.py
@@ -21,6 +21,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import re
 import textwrap
 
 import gast
@@ -105,7 +106,7 @@ def parse_str(src):
   """Returns the AST of given piece of code."""
   # TODO(mdan): This should exclude the module things are autowrapped in.
 
-  if six.PY2 and '.print(' in src:
+  if six.PY2 and re.search('\\Wprint\\(', src):
     # This special treatment is required because gast.parse is not aware of
     # whether print_function was present in the original context.
     src = 'from __future__ import print_function\n' + src
diff --git a/tensorflow/python/autograph/pyct/parser_test.py b/tensorflow/python/autograph/pyct/parser_test.py
index d3a7b7a014..cc0123408d 100644
--- a/tensorflow/python/autograph/pyct/parser_test.py
+++ b/tensorflow/python/autograph/pyct/parser_test.py
@@ -42,6 +42,15 @@ class ParserTest(test.TestCase):
     """))
     self.assertEqual('f', mod.body[0].name)
 
+  def test_parse_str_print(self):
+    mod = parser.parse_str(
+        textwrap.dedent("""
+            def f(x):
+              print(x)
+              return x + 1
+    """))
+    self.assertEqual('f', mod.body[0].name)
+
   def test_parse_comments(self):
     def f():
 # unindented comment
-- 
GitLab


From edfb0bba7f6d93c3179c395f78cd938e52a988da Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Wed, 30 Jan 2019 09:46:55 -0800
Subject: [PATCH 1583/2345] Fix androidx dependency for Android example

PiperOrigin-RevId: 231611822
---
 .../example/android/tflitecamerademo/Camera2BasicFragment.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java b/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java
index d417edbbb5..c6f315b545 100644
--- a/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java
+++ b/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java
@@ -61,7 +61,7 @@ import android.widget.ListView;
 import android.widget.NumberPicker;
 import android.widget.TextView;
 import android.widget.Toast;
-import androidx.legacy.app.FragmentCompat;
+import android.support.v13.app.FragmentCompat;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
-- 
GitLab


From cf3825d17cc6f56a7e3f7694114c7663db8522a9 Mon Sep 17 00:00:00 2001
From: Lukas Geiger <lukas.geiger94@gmail.com>
Date: Wed, 30 Jan 2019 17:01:21 +0100
Subject: [PATCH 1584/2345] Generalize MinMax monotonic optimizer to any MinMax
 op

---
 tensorflow/core/grappler/op_types.cc              | 10 ++++++++++
 tensorflow/core/grappler/op_types.h               |  2 ++
 .../grappler/optimizers/arithmetic_optimizer.cc   | 15 +++++++++++++--
 3 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc
index 27fba4fd11..5d24bf07db 100644
--- a/tensorflow/core/grappler/op_types.cc
+++ b/tensorflow/core/grappler/op_types.cc
@@ -47,6 +47,16 @@ bool IsAnyDiv(const NodeDef& node) {
          node.op() == "FloorDiv" || node.op() == "TruncateDiv";
 }
 
+bool IsAnyMax(const NodeDef& node) {
+  const auto& op = node.op();
+  return op == "Max" || op == "SegmentMax" || op == "UnsortedSegmentMax";
+}
+
+bool IsAnyMin(const NodeDef& node) {
+  const auto& op = node.op();
+  return op == "Min" || op == "SegmentMin" || op == "UnsortedSegmentMin";
+}
+
 bool IsApproximateEqual(const NodeDef& node) {
   return node.op() == "ApproximateEqual";
 }
diff --git a/tensorflow/core/grappler/op_types.h b/tensorflow/core/grappler/op_types.h
index a1ee253369..5fde80a3f2 100644
--- a/tensorflow/core/grappler/op_types.h
+++ b/tensorflow/core/grappler/op_types.h
@@ -28,6 +28,8 @@ bool IsAll(const NodeDef& node);
 bool IsAngle(const NodeDef& node);
 bool IsAny(const NodeDef& node);
 bool IsAnyDiv(const NodeDef& node);
+bool IsAnyMax(const NodeDef& node);
+bool IsAnyMin(const NodeDef& node);
 bool IsApproximateEqual(const NodeDef& node);
 bool IsAvgPoolGrad(const NodeDef& node);
 bool IsAssert(const NodeDef& node);
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index 2168dbd623..f59e63c5ae 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -53,6 +53,7 @@ limitations under the License.
 #include "tensorflow/core/util/strided_slice_op.h"
 
 using tensorflow::strings::StrCat;
+using tensorflow::str_util::StringReplace;
 
 namespace tensorflow {
 namespace grappler {
@@ -2721,7 +2722,7 @@ class OptimizeMaxOrMinOfMonotonicStage : public ArithmeticOptimizerStage {
   ~OptimizeMaxOrMinOfMonotonicStage() override = default;
 
   bool IsSupported(const NodeDef* node) const override {
-    return IsMax(*node) || IsMin(*node);
+    return IsAnyMax(*node) || IsAnyMin(*node);
   }
 
   Status TrySimplify(NodeDef* reduction_node,
@@ -2752,7 +2753,7 @@ class OptimizeMaxOrMinOfMonotonicStage : public ArithmeticOptimizerStage {
       if (!is_non_decreasing) {
         // Flip Min<->Max if the function is non-increasing, e.g.
         // Max(Neg(x)) = Neg(Min(x)).
-        const string opposite = IsMax(*reduction_node) ? "Min" : "Max";
+        const string opposite = FlipMinMax(*reduction_node);
         reduction_node->set_op(opposite);
       }
       AddToOptimizationQueue(reduction_node);
@@ -2775,6 +2776,16 @@ class OptimizeMaxOrMinOfMonotonicStage : public ArithmeticOptimizerStage {
       AddToOptimizationQueue(consumer);
     }
   }
+
+ private:
+  string FlipMinMax(const NodeDef& node) {
+    const string& op = node.op();
+    if (IsAnyMax(node)) {
+      return str_util::StringReplace(op, "Max", "Min", false);
+    } else {
+      return str_util::StringReplace(op, "Min", "Max", false);
+    }
+  }
 };
 
 // Replace a chain of type&shape preserving unary ops with a
-- 
GitLab


From bc695da1b4c90164c51b5e875ff285e1d5ee3ce3 Mon Sep 17 00:00:00 2001
From: Lukas Geiger <lukas.geiger94@gmail.com>
Date: Wed, 30 Jan 2019 19:06:30 +0100
Subject: [PATCH 1585/2345] Generalize MinMax monotonic optimizer to ArgMax

---
 tensorflow/core/grappler/op_types.cc          |  4 ++
 tensorflow/core/grappler/op_types.h           |  2 +
 .../optimizers/arithmetic_optimizer.cc        | 11 ++++-
 .../optimizers/arithmetic_optimizer_test.cc   | 41 +++++++++++++++++++
 4 files changed, 56 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc
index 5d24bf07db..2236f57f74 100644
--- a/tensorflow/core/grappler/op_types.cc
+++ b/tensorflow/core/grappler/op_types.cc
@@ -61,6 +61,10 @@ bool IsApproximateEqual(const NodeDef& node) {
   return node.op() == "ApproximateEqual";
 }
 
+bool IsArgMax(const NodeDef& node) { return node.op() == "ArgMax"; }
+
+bool IsArgMin(const NodeDef& node) { return node.op() == "ArgMin"; }
+
 bool IsAvgPoolGrad(const NodeDef& node) { return node.op() == "AvgPoolGrad"; }
 
 bool IsAssign(const NodeDef& node) {
diff --git a/tensorflow/core/grappler/op_types.h b/tensorflow/core/grappler/op_types.h
index 5fde80a3f2..fc32749d9f 100644
--- a/tensorflow/core/grappler/op_types.h
+++ b/tensorflow/core/grappler/op_types.h
@@ -31,6 +31,8 @@ bool IsAnyDiv(const NodeDef& node);
 bool IsAnyMax(const NodeDef& node);
 bool IsAnyMin(const NodeDef& node);
 bool IsApproximateEqual(const NodeDef& node);
+bool IsArgMax(const NodeDef& node);
+bool IsArgMin(const NodeDef& node);
 bool IsAvgPoolGrad(const NodeDef& node);
 bool IsAssert(const NodeDef& node);
 bool IsAssign(const NodeDef& node);
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index f59e63c5ae..1a3e7c6f42 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -2722,7 +2722,8 @@ class OptimizeMaxOrMinOfMonotonicStage : public ArithmeticOptimizerStage {
   ~OptimizeMaxOrMinOfMonotonicStage() override = default;
 
   bool IsSupported(const NodeDef* node) const override {
-    return IsAnyMax(*node) || IsAnyMin(*node);
+    return IsAnyMax(*node) || IsAnyMin(*node) ||
+           IsArgMax(*node) || IsArgMin(*node);
   }
 
   Status TrySimplify(NodeDef* reduction_node,
@@ -2756,6 +2757,12 @@ class OptimizeMaxOrMinOfMonotonicStage : public ArithmeticOptimizerStage {
         const string opposite = FlipMinMax(*reduction_node);
         reduction_node->set_op(opposite);
       }
+
+      if (IsArgMax(*reduction_node) || IsArgMin(*reduction_node)) {
+        // ArgMax(Sqrt(x)) = ArgMax(x)
+        inner_function->set_op("Identity");
+      }
+
       AddToOptimizationQueue(reduction_node);
       AddToOptimizationQueue(inner_function);
       AddToOptimizationQueue(inner_input);
@@ -2780,7 +2787,7 @@ class OptimizeMaxOrMinOfMonotonicStage : public ArithmeticOptimizerStage {
  private:
   string FlipMinMax(const NodeDef& node) {
     const string& op = node.op();
-    if (IsAnyMax(node)) {
+    if (IsAnyMax(node) || IsArgMax(node)) {
       return str_util::StringReplace(op, "Max", "Min", false);
     } else {
       return str_util::StringReplace(op, "Min", "Max", false);
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
index 94c59c68c8..dd5ee909b6 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
@@ -3466,6 +3466,47 @@ TEST_F(ArithmeticOptimizerTest, OptimizeMaxOrMinOfMonotonicElementWise) {
   EXPECT_EQ(2, required_node_count);
 }
 
+TEST_F(ArithmeticOptimizerTest, OptimizeArgMaxOrArgMinOfMonotonicElementWise) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+  const auto x = ops::Const(s.WithOpName("x"), {1.0f, 2.0f}, {1, 2});
+  Output sqrt = ops::Sqrt(s.WithOpName("sqrt"), x);
+  Output arg_max = ops::ArgMax(s.WithOpName("arg_max"), sqrt, 1);
+  Output final_out = ops::Identity(s.WithOpName("final_out"), arg_max);
+
+  GrapplerItem item;
+  item.fetch = {"final_out"};
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+  const auto tensors_expected = EvaluateNodes(item.graph, item.fetch);
+  EXPECT_EQ(1, tensors_expected.size());
+
+  GraphDef output;
+  ArithmeticOptimizer optimizer;
+  EnableOnlyOptimizeMaxOrMinOfMonotonic(&optimizer);
+  OptimizeAndPrune(&optimizer, &item, &output);
+  const auto tensors = EvaluateNodes(output, item.fetch);
+  EXPECT_EQ(1, tensors.size());
+
+  test::ExpectTensorEqual<int64>(tensors_expected[0], tensors[0]);
+  EXPECT_EQ(item.graph.node_size() - 1, output.node_size());
+  // Check if the inputs are switched
+  int required_node_count = 0;
+  for (int i = 0; i < output.node_size(); ++i) {
+    const NodeDef& node = output.node(i);
+    if (node.name() == "final_out") {
+      EXPECT_EQ("Identity", node.op());
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("arg_max", node.input(0));
+      ++required_node_count;
+    } else if (node.name() == "arg_max") {
+      EXPECT_EQ("ArgMax", node.op());
+      EXPECT_EQ(2, node.input_size());
+      EXPECT_EQ("x", node.input(0));
+      ++required_node_count;
+    }
+  }
+  EXPECT_EQ(2, required_node_count);
+}
+
 TEST_F(ArithmeticOptimizerTest,
        OptimizeMaxOrMinOfMonotonicElementWise_DoNotChangeFetchNode) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
-- 
GitLab


From 2cb0880812a9a99485feb2bbbfa9b8aca430a6c6 Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Wed, 30 Jan 2019 09:52:02 -0800
Subject: [PATCH 1586/2345] [XLA] Fix problems in handling kParameter HLO
 instruction with negative parameter number.

When the input is HLO text that contains kParameter instructions with negative
parameter numbers, an HLO tool such as run_hlo_module, crashes in creating the
HloComptation. We fix the HLO parser to report errors instead. Add an HLO
parser test case.

When the input is a binary HLO proto that contains kParameter instructions with
negative parameter numbers, run_hlo_module crashes in verifying the module. We
fix the DynamicParameterBinding verifier to report errors instead. Add an HLO
proto corpus for fuzzing.

PiperOrigin-RevId: 231612816
---
 .../compiler/xla/service/dynamic_parameter_binding.cc  |  3 ++-
 tensorflow/compiler/xla/service/hlo_parser.cc          | 10 ++++++++--
 tensorflow/compiler/xla/service/hlo_parser_test.cc     |  8 ++++++++
 3 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/service/dynamic_parameter_binding.cc b/tensorflow/compiler/xla/service/dynamic_parameter_binding.cc
index e9c8aa03e2..7f0ae692f7 100644
--- a/tensorflow/compiler/xla/service/dynamic_parameter_binding.cc
+++ b/tensorflow/compiler/xla/service/dynamic_parameter_binding.cc
@@ -112,7 +112,8 @@ Status DynamicParameterBinding::Verify(const HloModule& module) const {
   return ForEachBinding([&](const DynamicParameter& dynamic_parameter,
                             const DynamicDimension& dynamic_dimension)
                             -> Status {
-    TF_RET_CHECK(dynamic_parameter.parameter_num < entry->num_parameters());
+    TF_RET_CHECK(dynamic_parameter.parameter_num >= 0 &&
+                 dynamic_parameter.parameter_num < entry->num_parameters());
     TF_RET_CHECK(dynamic_dimension.parameter_num < entry->num_parameters());
     TF_RET_CHECK(ShapeUtil::IndexIsValid(
         entry->parameter_instruction(dynamic_parameter.parameter_num)->shape(),
diff --git a/tensorflow/compiler/xla/service/hlo_parser.cc b/tensorflow/compiler/xla/service/hlo_parser.cc
index 1a930370a9..d7fa6b4a64 100644
--- a/tensorflow/compiler/xla/service/hlo_parser.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser.cc
@@ -658,8 +658,14 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
       tensorflow::int64 parameter_number;
       if (!ParseToken(TokKind::kLparen,
                       "expects '(' before parameter number") ||
-          !ParseInt64(&parameter_number) ||
-          !ParseToken(TokKind::kRparen, "expects ')' after parameter number") ||
+          !ParseInt64(&parameter_number)) {
+        return false;
+      }
+      if (parameter_number < 0) {
+        Error(lexer_.GetLoc(), "parameter number must be >= 0");
+        return false;
+      }
+      if (!ParseToken(TokKind::kRparen, "expects ')' after parameter number") ||
           !ParseAttributes(attrs)) {
         return false;
       }
diff --git a/tensorflow/compiler/xla/service/hlo_parser_test.cc b/tensorflow/compiler/xla/service/hlo_parser_test.cc
index ff52a89d1b..6eee767900 100644
--- a/tensorflow/compiler/xla/service/hlo_parser_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser_test.cc
@@ -2558,5 +2558,13 @@ TEST_F(HloParserTest, ParseDynamicTuple) {
       << "actual:   " << ShapeUtil::HumanString(actual);
 }
 
+TEST_F(HloParserTest, NegativeParameterNumber) {
+  const string hlo_string = "par0 = f32[3,5] parameter(-1)";
+  auto result = ParseHloString(hlo_string);
+  ASSERT_FALSE(result.status().ok());
+  EXPECT_THAT(result.status().error_message(),
+              ::testing::HasSubstr("parameter number must be >= 0"));
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 8feadedbf5eea4760c67036e51e18c757481085c Mon Sep 17 00:00:00 2001
From: Igor Saprykin <isaprykin@google.com>
Date: Wed, 30 Jan 2019 09:59:11 -0800
Subject: [PATCH 1587/2345] Sort the keyword arguments to combinations.py using
 more than one letter.

If two or more arguments start with the same letter, then the test case names might differ across the test runs.

PiperOrigin-RevId: 231614003
---
 tensorflow/contrib/distribute/python/combinations.py      | 6 +++---
 tensorflow/contrib/distribute/python/combinations_test.py | 8 ++++++++
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/combinations.py b/tensorflow/contrib/distribute/python/combinations.py
index 104c63f6b0..798a1591c7 100644
--- a/tensorflow/contrib/distribute/python/combinations.py
+++ b/tensorflow/contrib/distribute/python/combinations.py
@@ -113,12 +113,12 @@ def generate(combinations):
       # We use OrderedDicts in `combine()` and `times()` to ensure stable
       # order of keys in each dictionary.
       assert isinstance(combination, OrderedDict)
-      name = "".join(sorted([
+      name = "".join([
           "_{}_{}".format(
               "".join(filter(str.isalnum, key)),
               "".join(filter(str.isalnum, str(value))))
           for key, value in combination.items()
-      ]))
+      ])
       named_combinations.append(
           OrderedDict(
               list(combination.items()) + [("testcase_name",
@@ -232,7 +232,7 @@ def combine(**kwargs):
   if not kwargs:
     return [OrderedDict()]
 
-  sort_by_key = lambda k: k[0][0]
+  sort_by_key = lambda k: k[0]
   kwargs = OrderedDict(sorted(kwargs.items(), key=sort_by_key))
   first = list(kwargs.items())[0]
 
diff --git a/tensorflow/contrib/distribute/python/combinations_test.py b/tensorflow/contrib/distribute/python/combinations_test.py
index 86aa48cea8..9f3deadbec 100644
--- a/tensorflow/contrib/distribute/python/combinations_test.py
+++ b/tensorflow/contrib/distribute/python/combinations_test.py
@@ -42,6 +42,14 @@ class TestingCombinationsTest(test.TestCase):
         "b": 3
     }], combinations.combine(a=[1, 2], b=[2, 3]))
 
+  def test_arguments_sorted(self):
+    self.assertEqual([
+        OrderedDict([("aa", 1), ("ab", 2)]),
+        OrderedDict([("aa", 1), ("ab", 3)]),
+        OrderedDict([("aa", 2), ("ab", 2)]),
+        OrderedDict([("aa", 2), ("ab", 3)])
+    ], combinations.combine(ab=[2, 3], aa=[1, 2]))
+
   def test_combine_single_parameter(self):
     self.assertEqual([{
         "a": 1,
-- 
GitLab


From 31fd1ffbad50b742cc69123ce146b59b8d79f2b3 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Wed, 30 Jan 2019 09:59:30 -0800
Subject: [PATCH 1588/2345] Exposes
 tf.nest.{flatten,map_structure,pack_sequence_as,assert_same_structure}

PiperOrigin-RevId: 231614063
---
 .../tools/api/generator/api_init_files.bzl    |  1 +
 .../tools/api/generator/api_init_files_v1.bzl |  1 +
 tensorflow/python/util/nest.py                | 38 ++++++++++++++++++-
 .../tools/api/golden/v1/tensorflow.nest.pbtxt | 19 ++++++++++
 .../tools/api/golden/v1/tensorflow.pbtxt      |  4 ++
 .../tools/api/golden/v2/tensorflow.nest.pbtxt | 19 ++++++++++
 .../tools/api/golden/v2/tensorflow.pbtxt      |  4 ++
 .../tools/compatibility/tf_upgrade_v2.py      |  8 ++++
 8 files changed, 92 insertions(+), 2 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.nest.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.nest.pbtxt

diff --git a/tensorflow/python/tools/api/generator/api_init_files.bzl b/tensorflow/python/tools/api/generator/api_init_files.bzl
index 2bac83c423..d18776eab7 100644
--- a/tensorflow/python/tools/api/generator/api_init_files.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files.bzl
@@ -29,6 +29,7 @@ TENSORFLOW_API_INIT_FILES = [
     "lite/constants/__init__.py",
     "losses/__init__.py",
     "math/__init__.py",
+    "nest/__init__.py",
     "nn/__init__.py",
     "nn/rnn_cell/__init__.py",
     "quantization/__init__.py",
diff --git a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
index 21e5951010..5213d5eed8 100644
--- a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
@@ -38,6 +38,7 @@ TENSORFLOW_API_INIT_FILES_V1 = [
     "manip/__init__.py",
     "math/__init__.py",
     "metrics/__init__.py",
+    "nest/__init__.py",
     "nn/__init__.py",
     "nn/rnn_cell/__init__.py",
     "profiler/__init__.py",
diff --git a/tensorflow/python/util/nest.py b/tensorflow/python/util/nest.py
index 436cf82ea5..a43ec48589 100644
--- a/tensorflow/python/util/nest.py
+++ b/tensorflow/python/util/nest.py
@@ -39,6 +39,7 @@ import collections as _collections
 import six as _six
 
 from tensorflow.python import pywrap_tensorflow as _pywrap_tensorflow
+from tensorflow.python.util.tf_export import tf_export
 
 
 _SHALLOW_TREE_HAS_INVALID_KEYS = (
@@ -183,8 +184,38 @@ is_sequence = _pywrap_tensorflow.IsSequence
 is_sequence_or_composite = _pywrap_tensorflow.IsSequenceOrComposite
 
 
-# See the swig file (util.i) for documentation.
-flatten = _pywrap_tensorflow.Flatten
+@tf_export("nest.flatten")
+def flatten(structure, expand_composites=False):
+  """Returns a flat list from a given nested structure.
+
+  If nest is not a sequence, tuple, or dict, then returns a single-element list:
+  [nest].
+
+  In the case of dict instances, the sequence consists of the values, sorted by
+  key to ensure deterministic behavior. This is true also for OrderedDict
+  instances: their sequence order is ignored, the sorting order of keys is used
+  instead. The same convention is followed in pack_sequence_as. This correctly
+  repacks dicts and OrderedDicts after they have been flattened, and also allows
+  flattening an OrderedDict and then repacking it back using a corresponding
+  plain dict, or vice-versa. Dictionaries with non-sortable keys cannot be
+  flattened.
+
+  Users must not modify any collections used in nest while this function is
+  running.
+
+  Args:
+    structure: an arbitrarily nested structure or a scalar object. Note, numpy
+      arrays are considered scalars.
+    expand_composites: If true, then composite tensors such as tf.SparseTensor
+       and tf.RaggedTensor are expanded into their component tensors.
+
+  Returns:
+    A Python list, the flattened version of the input.
+
+  Raises:
+    TypeError: The nest is or contains a dict with non-sortable keys.
+  """
+  return _pywrap_tensorflow.Flatten(structure, expand_composites)
 
 
 # See the swig file (util.i) for documentation.
@@ -203,6 +234,7 @@ class _DotString(object):
 _DOT = _DotString()
 
 
+@tf_export("nest.assert_same_structure")
 def assert_same_structure(nest1, nest2, check_types=True,
                           expand_composites=False):
   """Asserts that two structures are nested in the same way.
@@ -337,6 +369,7 @@ def _packed_nest_with_indices(structure, flat, index, is_seq):
   return index, packed
 
 
+@tf_export("nest.pack_sequence_as")
 def pack_sequence_as(structure, flat_sequence, expand_composites=False):
   """Returns a given flattened sequence packed into a given structure.
 
@@ -394,6 +427,7 @@ def pack_sequence_as(structure, flat_sequence, expand_composites=False):
   return _sequence_like(structure, packed)
 
 
+@tf_export("nest.map_structure")
 def map_structure(func, *structure, **kwargs):
   """Applies `func` to each entry in `structure` and returns a new structure.
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nest.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nest.pbtxt
new file mode 100644
index 0000000000..ad5360e335
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nest.pbtxt
@@ -0,0 +1,19 @@
+path: "tensorflow.nest"
+tf_module {
+  member_method {
+    name: "assert_same_structure"
+    argspec: "args=[\'nest1\', \'nest2\', \'check_types\', \'expand_composites\'], varargs=None, keywords=None, defaults=[\'True\', \'False\'], "
+  }
+  member_method {
+    name: "flatten"
+    argspec: "args=[\'structure\', \'expand_composites\'], varargs=None, keywords=None, defaults=[\'False\'], "
+  }
+  member_method {
+    name: "map_structure"
+    argspec: "args=[\'func\'], varargs=structure, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "pack_sequence_as"
+    argspec: "args=[\'structure\', \'flat_sequence\', \'expand_composites\'], varargs=None, keywords=None, defaults=[\'False\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
index 1409a3276e..c8ec8da259 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
@@ -476,6 +476,10 @@ tf_module {
     name: "name_scope"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "nest"
+    mtype: "<type \'module\'>"
+  }
   member {
     name: "newaxis"
     mtype: "<type \'NoneType\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nest.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nest.pbtxt
new file mode 100644
index 0000000000..ad5360e335
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nest.pbtxt
@@ -0,0 +1,19 @@
+path: "tensorflow.nest"
+tf_module {
+  member_method {
+    name: "assert_same_structure"
+    argspec: "args=[\'nest1\', \'nest2\', \'check_types\', \'expand_composites\'], varargs=None, keywords=None, defaults=[\'True\', \'False\'], "
+  }
+  member_method {
+    name: "flatten"
+    argspec: "args=[\'structure\', \'expand_composites\'], varargs=None, keywords=None, defaults=[\'False\'], "
+  }
+  member_method {
+    name: "map_structure"
+    argspec: "args=[\'func\'], varargs=structure, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "pack_sequence_as"
+    argspec: "args=[\'structure\', \'flat_sequence\', \'expand_composites\'], varargs=None, keywords=None, defaults=[\'False\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index a9d6799d69..e1eef058f4 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -212,6 +212,10 @@ tf_module {
     name: "name_scope"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "nest"
+    mtype: "<type \'module\'>"
+  }
   member {
     name: "newaxis"
     mtype: "<type \'NoneType\'>"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 1292eb4224..002a924f10 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -537,6 +537,14 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.data.experimental.unbatch",
         "tf.contrib.data.unique":
             "tf.data.experimental.unique",
+        "tf.contrib.framework.nest.assert_same_structure":
+            "tf.nest.assert_same_structure",
+        "tf.contrib.framework.nest.flatten":
+            "tf.nest.flatten",
+        "tf.contrib.framework.nest.map_structure":
+            "tf.nest.map_structure",
+        "tf.contrib.framework.nest.pack_sequence_as":
+            "tf.nest.pack_sequence_as",
         "tf.contrib.framework.constant_value":
             "tf.get_static_value",
         "tf.contrib.saved_model.load_keras_model":
-- 
GitLab


From 3599af7b153fd942d056e9abba910b5aa29c67a3 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Wed, 30 Jan 2019 10:11:22 -0800
Subject: [PATCH 1589/2345] Internal Cleanup.

PiperOrigin-RevId: 231616671
---
 tensorflow/contrib/distribute/python/keras_test.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 54d4ff5bdc..73c27d680a 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -275,12 +275,14 @@ def strategy_and_optimizer_combinations():
   return combinations.times(
       all_strategy_combinations(),
       combinations.combine(optimizer=[
-          combinations.adagrad_optimizer_v1_fn, combinations
-          .adagrad_optimizer_keras_v2_fn, combinations.adam_optimizer_v1_fn,
+          combinations.adagrad_optimizer_v1_fn,
+          combinations.adagrad_optimizer_keras_v2_fn,
+          combinations.adam_optimizer_v1_fn,
           combinations.adam_optimizer_keras_v2_fn,
           combinations.gradient_descent_optimizer_v1_fn,
-          combinations.gradient_descent_optimizer_keras_v2_fn, combinations
-          .rmsprop_optimizer_v1_fn, combinations.rmsprop_optimizer_keras_v2_fn
+          combinations.gradient_descent_optimizer_keras_v2_fn,
+          combinations.rmsprop_optimizer_v1_fn,
+          combinations.rmsprop_optimizer_keras_v2_fn
       ]))
 
 
-- 
GitLab


From 609992a3a0c20757c0a839b0c2bb9de8a42ab5e5 Mon Sep 17 00:00:00 2001
From: Daryl Ng <darylng@google.com>
Date: Wed, 30 Jan 2019 10:14:58 -0800
Subject: [PATCH 1590/2345] Adding uint32 to the the supported infeed dtypes

PiperOrigin-RevId: 231617395
---
 tensorflow/contrib/tpu/python/ops/tpu_ops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tpu/python/ops/tpu_ops.py b/tensorflow/contrib/tpu/python/ops/tpu_ops.py
index 9260e7b8a8..500dd2cd39 100644
--- a/tensorflow/contrib/tpu/python/ops/tpu_ops.py
+++ b/tensorflow/contrib/tpu/python/ops/tpu_ops.py
@@ -157,7 +157,7 @@ if platform.system() != "Windows":
 
   _SUPPORTED_INFEED_DTYPES = set([
       dtypes.bool, dtypes.int32, dtypes.int64, dtypes.bfloat16, dtypes.float32,
-      dtypes.complex64
+      dtypes.complex64, dtypes.uint32
   ])
 
   def infeed_dequeue(dtype, shape, name=None):
-- 
GitLab


From 32387f09285735f2f8c90832435a75ab753622b7 Mon Sep 17 00:00:00 2001
From: Guozhong Zhuang <guozhong.zhuang@intel.com>
Date: Wed, 30 Jan 2019 10:50:41 -0800
Subject: [PATCH 1591/2345] Remove code which can NOT be executed

---
 tensorflow/core/util/mkl_util.h | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/tensorflow/core/util/mkl_util.h b/tensorflow/core/util/mkl_util.h
index 5abe34cba2..1a5c6d8e56 100644
--- a/tensorflow/core/util/mkl_util.h
+++ b/tensorflow/core/util/mkl_util.h
@@ -2111,13 +2111,6 @@ class LRUCache {
   void Clear() {
     if (lru_list_.empty()) return;
 
-    // delete the cached objects
-    for (auto& key : lru_list_) {
-      auto it = cache_.find(key);
-      DCHECK(it != cache_.end());
-      delete it->second.op;
-    }
-
     // clean up the cache
     cache_.clear();
     lru_list_.clear();
-- 
GitLab


From 1924fe1c4df33f818ef2d2eb6582d07995f70b94 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 30 Jan 2019 10:31:50 -0800
Subject: [PATCH 1592/2345] [TF:XLA] Re-enable MaxPool test for XLA.

PiperOrigin-RevId: 231620880
---
 tensorflow/python/kernel_tests/pooling_ops_test.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/python/kernel_tests/pooling_ops_test.py b/tensorflow/python/kernel_tests/pooling_ops_test.py
index 27a7190434..94a861f554 100644
--- a/tensorflow/python/kernel_tests/pooling_ops_test.py
+++ b/tensorflow/python/kernel_tests/pooling_ops_test.py
@@ -730,7 +730,7 @@ class PoolingTest(test.TestCase):
         t = nn_ops.max_pool(
             t, ksize=ksize, strides=strides, padding="SAME").eval()
 
-  @test_util.disable_xla("This test never passed for XLA")
+  @test_util.disable_xla("b/123338077")
   def testDepthwiseMaxPoolInvalidConfigs(self):
     self._testDepthwiseMaxPoolInvalidConfig(
         [1, 2, 2, 4], [1, 2, 2, 2], [1, 1, 1, 2],
@@ -1175,7 +1175,6 @@ class PoolingTest(test.TestCase):
           use_gpu=use_gpu)
 
   @test_util.run_deprecated_v1
-  @test_util.disable_xla("This test never passed for XLA")
   def testMaxPoolGrad(self):
     for (data_format, use_gpu) in GetTestConfigs():
       self._testMaxPoolGradValidPadding1_1(data_format, use_gpu)
-- 
GitLab


From 19c79e944bc8b5f05fc8f0e449dc0e7aa24f1aca Mon Sep 17 00:00:00 2001
From: Tong Shen <endlessroad@google.com>
Date: Wed, 30 Jan 2019 10:32:00 -0800
Subject: [PATCH 1593/2345] Do not assume number of outputs for While node is
 always the same as number of inputs.

PiperOrigin-RevId: 231620915
---
 tensorflow/compiler/tf2xla/BUILD              |  2 ++
 tensorflow/compiler/tf2xla/tf2xla_util.cc     |  7 ++++
 .../compiler/tf2xla/tf2xla_util_test.cc       | 34 +++++++++++++++++++
 3 files changed, 43 insertions(+)

diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD
index 92ba474fbc..14bd9d4b13 100644
--- a/tensorflow/compiler/tf2xla/BUILD
+++ b/tensorflow/compiler/tf2xla/BUILD
@@ -317,11 +317,13 @@ tf_cc_test(
         ":tf2xla_util",
         "//tensorflow/cc:cc_ops",
         "//tensorflow/cc:function_ops",
+        "//tensorflow/cc:functional_ops",
         "//tensorflow/cc:ops",
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:ops",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
diff --git a/tensorflow/compiler/tf2xla/tf2xla_util.cc b/tensorflow/compiler/tf2xla/tf2xla_util.cc
index 18d87727c5..c64f78e1a1 100644
--- a/tensorflow/compiler/tf2xla/tf2xla_util.cc
+++ b/tensorflow/compiler/tf2xla/tf2xla_util.cc
@@ -265,6 +265,13 @@ Status PropagateConstIntoWhileNode(Graph* g, Node* while_node,
     }
 
     // Check if i-th retval's input comes from i-th arg directly.
+    // For resource variable input of While nodes, TF2XLA convention is to place
+    // them at the end of all inputs (after all data inputs), and *not* return
+    // them. So number of While node inputs might be larger than number of its
+    // outputs.
+    if (i >= body_func->signature().output_arg_size()) {
+      continue;
+    }
     const OpDef_ArgDef& output_arg = body_func->signature().output_arg(i);
     auto output_arg_input = body_func->ret().find(output_arg.name());
     if (output_arg_input == body_func->ret().end()) {
diff --git a/tensorflow/compiler/tf2xla/tf2xla_util_test.cc b/tensorflow/compiler/tf2xla/tf2xla_util_test.cc
index 202e929315..9e9c3cecee 100644
--- a/tensorflow/compiler/tf2xla/tf2xla_util_test.cc
+++ b/tensorflow/compiler/tf2xla/tf2xla_util_test.cc
@@ -21,11 +21,13 @@ limitations under the License.
 #include "tensorflow/cc/framework/ops.h"
 #include "tensorflow/cc/ops/data_flow_ops.h"
 #include "tensorflow/cc/ops/function_ops.h"
+#include "tensorflow/cc/ops/functional_ops.h"
 #include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/compiler/tf2xla/sharding_util.h"
 #include "tensorflow/core/common_runtime/graph_optimizer.h"
 #include "tensorflow/core/common_runtime/process_function_library_runtime.h"
 #include "tensorflow/core/framework/function.h"
+#include "tensorflow/core/framework/graph_to_functiondef.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/graph/graph.h"
 #include "tensorflow/core/lib/core/status.h"
@@ -329,5 +331,37 @@ TEST(CachedFunctionHandles, Basic) {
   TF_EXPECT_OK(cached_function_handles.ReleaseAllHandles());
 }
 
+TEST(PropagateConstIntoFunctionalNodes, WhileLoopWithResourceInput) {
+  FunctionLibraryDefinition fld(OpRegistry::Global(), {});
+  {
+    // Cond graph & body graph.
+    Scope scope = Scope::NewRootScope().ExitOnError();
+    auto pred = ops::_Arg(scope.WithOpName("pred"), DT_BOOL, 0);
+    auto input = ops::_Arg(scope.WithOpName("input"), DT_RESOURCE, 1);
+    auto ret = ops::_Retval(scope.WithOpName("ret"), pred, 0);
+    Graph graph(OpRegistry::Global());
+    TF_ASSERT_OK(scope.ToGraph(&graph));
+    FunctionDef cond_fdef;
+    TF_ASSERT_OK(GraphToFunctionDef(graph, "cond", &cond_fdef));
+    TF_ASSERT_OK(fld.AddFunctionDef(cond_fdef));
+    FunctionDef body_fdef;
+    TF_ASSERT_OK(GraphToFunctionDef(graph, "body", &body_fdef));
+    TF_ASSERT_OK(fld.AddFunctionDef(body_fdef));
+  }
+  Scope scope = Scope::NewRootScope().ExitOnError();
+  auto pred = ops::Const(scope.WithOpName("pred"), false, TensorShape({}));
+  auto input = ops::Const(scope.WithOpName("input"), 0, TensorShape({}));
+  NameAttrList cond_fn, body_fn;
+  cond_fn.set_name("cond");
+  body_fn.set_name("body");
+  auto while_op =
+      ops::While(scope.WithOpName("while"),
+                 std::initializer_list<Input>{pred, input}, cond_fn, body_fn);
+  Graph graph(OpRegistry::Global());
+  TF_ASSERT_OK(scope.ToGraph(&graph));
+
+  TF_EXPECT_OK(PropagateConstIntoFunctionalNodes(&graph, &fld, &fld));
+}
+
 }  // namespace
 }  // namespace tensorflow
-- 
GitLab


From 234c738bcd513f5747925c87efb9ed4175a2aa5e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 30 Jan 2019 10:42:44 -0800
Subject: [PATCH 1594/2345] This CL introduces serializable/deserializable
 learning rate decay schedules for the Keras v2 optimizers.

PiperOrigin-RevId: 231623483
---
 tensorflow/python/BUILD                       |    2 +-
 tensorflow/python/keras/BUILD                 |   14 +-
 tensorflow/python/keras/optimizer_v2/BUILD    |   29 +
 .../python/keras/optimizer_v2/adagrad_test.py |   47 +
 .../python/keras/optimizer_v2/adam_test.py    |   50 +
 .../optimizer_v2/gradient_descent_test.py     |   84 +-
 .../optimizer_v2/learning_rate_schedule.py    | 1031 +++++++++++++++++
 .../learning_rate_schedule_test.py            |  527 +++++++++
 tensorflow/python/keras/optimizer_v2/nadam.py |    7 +
 .../python/keras/optimizer_v2/optimizer_v2.py |   19 +-
 .../keras/optimizer_v2/optimizer_v2_test.py   |   35 +
 .../python/keras/optimizer_v2/rmsprop_test.py |   73 ++
 .../tools/api/generator/api_init_files.bzl    |    1 +
 .../tools/api/generator/api_init_files_v1.bzl |    1 +
 .../python/training/learning_rate_decay.py    |   90 +-
 .../python/training/learning_rate_decay_v2.py |  898 --------------
 .../training/learning_rate_decay_v2_test.py   |  497 --------
 ...low.keras.experimental.-cosine-decay.pbtxt |   18 +
 .../v1/tensorflow.keras.experimental.pbtxt    |    4 +
 .../v1/tensorflow.keras.optimizers.pbtxt      |    4 +
 ...imizers.schedules.-exponential-decay.pbtxt |   18 +
 ...mizers.schedules.-inverse-time-decay.pbtxt |   18 +
 ...rs.schedules.-learning-rate-schedule.pbtxt |   16 +
 ....schedules.-piecewise-constant-decay.pbtxt |   18 +
 ...timizers.schedules.-polynomial-decay.pbtxt |   18 +
 ...ensorflow.keras.optimizers.schedules.pbtxt |   31 +
 ....experimental.-cosine-decay-restarts.pbtxt |   18 +
 ...low.keras.experimental.-cosine-decay.pbtxt |   18 +
 ...as.experimental.-linear-cosine-decay.pbtxt |   18 +
 ...erimental.-noisy-linear-cosine-decay.pbtxt |   18 +
 .../v2/tensorflow.keras.experimental.pbtxt    |   16 +
 .../v2/tensorflow.keras.optimizers.pbtxt      |    4 +
 ...imizers.schedules.-exponential-decay.pbtxt |   18 +
 ...mizers.schedules.-inverse-time-decay.pbtxt |   18 +
 ...rs.schedules.-learning-rate-schedule.pbtxt |   16 +
 ....schedules.-piecewise-constant-decay.pbtxt |   18 +
 ...timizers.schedules.-polynomial-decay.pbtxt |   18 +
 ...ensorflow.keras.optimizers.schedules.pbtxt |   31 +
 .../api/golden/v2/tensorflow.train.pbtxt      |   36 -
 tensorflow/tools/compatibility/renames_v2.py  |    9 +
 40 files changed, 2298 insertions(+), 1508 deletions(-)
 create mode 100644 tensorflow/python/keras/optimizer_v2/learning_rate_schedule.py
 create mode 100644 tensorflow/python/keras/optimizer_v2/learning_rate_schedule_test.py
 delete mode 100644 tensorflow/python/training/learning_rate_decay_v2.py
 delete mode 100644 tensorflow/python/training/learning_rate_decay_v2_test.py
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-cosine-decay.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.schedules.-exponential-decay.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.schedules.-inverse-time-decay.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.schedules.-learning-rate-schedule.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.schedules.-piecewise-constant-decay.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.schedules.-polynomial-decay.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.schedules.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-cosine-decay-restarts.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-cosine-decay.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-linear-cosine-decay.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-noisy-linear-cosine-decay.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.schedules.-exponential-decay.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.schedules.-inverse-time-decay.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.schedules.-learning-rate-schedule.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.schedules.-piecewise-constant-decay.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.schedules.-polynomial-decay.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.schedules.pbtxt

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 813040177d..109beace58 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3722,6 +3722,7 @@ py_library(
         "//tensorflow/python/distribute:reduce_util",
         "//tensorflow/python/eager:backprop",
         "//tensorflow/python/eager:context",
+        "//tensorflow/python/keras/optimizer_v2:learning_rate_schedule",
         "//tensorflow/python/ops/losses",
         "//tensorflow/python/training/checkpointable:base",
         "//tensorflow/python/training/checkpointable:util",
@@ -4807,7 +4808,6 @@ cuda_py_tests(
         "training/ftrl_test.py",
         "training/gradient_descent_test.py",
         "training/learning_rate_decay_test.py",
-        "training/learning_rate_decay_v2_test.py",
         "training/momentum_test.py",
         "training/optimizer_test.py",
         "training/proximal_adagrad_test.py",
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index b37bce9057..ad895d21dc 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -310,7 +310,6 @@ py_library(
         "layers/recurrent.py",
         "layers/serialization.py",
         "layers/wrappers.py",
-        "utils/generic_utils.py",
         "utils/kernelized_utils.py",
         "utils/layer_utils.py",
         "utils/tf_utils.py",
@@ -318,6 +317,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":engine",
+        ":generic_utils",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:cudnn_rnn_ops_gen",
         "//tensorflow/python:dtypes",
@@ -339,6 +339,18 @@ py_library(
     ],
 )
 
+py_library(
+    name = "generic_utils",
+    srcs = [
+        "utils/generic_utils.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:util",
+        "//third_party/py/numpy",
+    ],
+)
+
 tf_py_test(
     name = "integration_test",
     size = "medium",
diff --git a/tensorflow/python/keras/optimizer_v2/BUILD b/tensorflow/python/keras/optimizer_v2/BUILD
index 45afe2a134..88f2521d5e 100644
--- a/tensorflow/python/keras/optimizer_v2/BUILD
+++ b/tensorflow/python/keras/optimizer_v2/BUILD
@@ -25,6 +25,7 @@ py_library(
     ],
     srcs_version = "PY2AND3",
     deps = [
+        ":learning_rate_schedule",
         "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:distribute",
         "//tensorflow/python:framework",
@@ -39,6 +40,21 @@ py_library(
     ],
 )
 
+py_library(
+    name = "learning_rate_schedule",
+    srcs = [
+        "learning_rate_schedule.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:control_flow_ops",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:random_ops",
+        "//tensorflow/python/keras:generic_utils",
+    ],
+)
+
 cuda_py_test(
     name = "adagrad_test",
     size = "medium",
@@ -197,6 +213,19 @@ py_test(
     ],
 )
 
+py_test(
+    name = "learning_rate_schedule_test",
+    size = "small",
+    srcs = ["learning_rate_schedule_test.py"],
+    deps = [
+        ":optimizer_v2",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python/keras",
+        "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
+    ],
+)
+
 cuda_py_test(
     name = "rmsprop_test",
     size = "medium",
diff --git a/tensorflow/python/keras/optimizer_v2/adagrad_test.py b/tensorflow/python/keras/optimizer_v2/adagrad_test.py
index 864aefaf70..9c8d3ff8a4 100644
--- a/tensorflow/python/keras/optimizer_v2/adagrad_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adagrad_test.py
@@ -28,6 +28,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.keras.optimizer_v2 import adagrad
+from tensorflow.python.keras.optimizer_v2 import learning_rate_schedule
 from tensorflow.python.ops import embedding_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
@@ -160,6 +161,52 @@ class AdagradOptimizerTest(test.TestCase):
           self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
           self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
+  def testBasicWithLearningRateInverseTimeDecay(self):
+    for dtype in [dtypes.float32, dtypes.float64]:
+      with self.cached_session():
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
+        var0 = resource_variable_ops.ResourceVariable(var0_np)
+        var1 = resource_variable_ops.ResourceVariable(var1_np)
+        grads0 = constant_op.constant(grads0_np)
+        grads1 = constant_op.constant(grads1_np)
+
+        learning_rate = 3.0
+        decay = 0.5
+        lr_schedule = learning_rate_schedule.InverseTimeDecay(
+            learning_rate, decay_steps=1.0, decay_rate=decay)
+
+        ada_opt = adagrad.Adagrad(lr_schedule)
+
+        accum0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        accum1_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+
+        if not context.executing_eagerly():
+          ada_update = ada_opt.apply_gradients(
+              zip([grads0, grads1], [var0, var1]))
+          self.evaluate(variables.global_variables_initializer())
+
+        # Fetch params to validate initial values
+        v0_val, v1_val = self.evaluate([var0, var1])
+        self.assertAllClose([1.0, 2.0], v0_val)
+        self.assertAllClose([3.0, 4.0], v1_val)
+
+        # Run 3 steps of adagrad
+        for t in range(3):
+          if not context.executing_eagerly():
+            self.evaluate(ada_update)
+          else:
+            ada_opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+          lr_np = learning_rate / (1 + decay * t)
+          var0_np, accum0_np = adagrad_update_numpy(var0_np, accum0_np,
+                                                    grads0_np, lr_np)
+          var1_np, accum1_np = adagrad_update_numpy(var1_np, accum1_np,
+                                                    grads1_np, lr_np)
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
+
   @test_util.run_deprecated_v1
   def testMinimizeSparseResourceVariable(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
diff --git a/tensorflow/python/keras/optimizer_v2/adam_test.py b/tensorflow/python/keras/optimizer_v2/adam_test.py
index 7918c09b7e..761b6a0854 100644
--- a/tensorflow/python/keras/optimizer_v2/adam_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adam_test.py
@@ -27,6 +27,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.keras import optimizers
 from tensorflow.python.keras.optimizer_v2 import adam
+from tensorflow.python.keras.optimizer_v2 import learning_rate_schedule
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
@@ -399,6 +400,55 @@ class AdamOptimizerTest(test.TestCase):
           self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
           self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
+  def testBasicWithLearningRateInverseTimeDecay(self):
+    for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]):
+      with self.session(graph=ops.Graph()):
+        # Initialize variables for numpy implementation.
+        m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
+
+        var0 = resource_variable_ops.ResourceVariable(
+            var0_np, name="var0_%d" % i)
+        var1 = resource_variable_ops.ResourceVariable(
+            var1_np, name="var1_%d" % i)
+        grads0 = constant_op.constant(grads0_np)
+        grads1 = constant_op.constant(grads1_np)
+
+        learning_rate = 0.001
+        decay = 0.5
+        lr_schedule = learning_rate_schedule.InverseTimeDecay(
+            learning_rate, decay_steps=1.0, decay_rate=decay)
+        beta_1 = 0.9
+        beta_2 = 0.999
+        epsilon = 1e-7
+
+        opt = adam.Adam(
+            learning_rate=lr_schedule,
+            beta_1=beta_1,
+            beta_2=beta_2,
+            epsilon=epsilon)
+        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+
+        self.evaluate(variables.global_variables_initializer())
+        # Run 3 steps of Adam
+        for t in range(3):
+          self.evaluate(update)
+
+          lr_np = learning_rate / (1 + decay * t)
+
+          var0_np, m0, v0 = adam_update_numpy(
+              var0_np, grads0_np, t, m0, v0, lr=lr_np)
+          var1_np, m1, v1 = adam_update_numpy(
+              var1_np, grads1_np, t, m1, v1, lr=lr_np)
+
+          # Validate updated params
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
+
   @test_util.run_deprecated_v1
   def testTensorLearningRate(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
diff --git a/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py b/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py
index 333a6f288e..6bd56372b9 100644
--- a/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py
+++ b/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py
@@ -28,6 +28,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.keras.optimizer_v2 import gradient_descent
+from tensorflow.python.keras.optimizer_v2 import learning_rate_schedule
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import embedding_ops
 from tensorflow.python.ops import math_ops
@@ -57,42 +58,61 @@ class GradientDescentOptimizerTest(test.TestCase):
         self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01],
                                            self.evaluate(var1))
 
+  def _test_basic_sgd_with_learning_rate_decay(self, sgd, dtype):
+    var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype)
+    var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype)
+    grads0 = constant_op.constant([0.1, 0.1], dtype=dtype)
+    grads1 = constant_op.constant([0.01, 0.01], dtype=dtype)
+    if not context.executing_eagerly():
+      sgd_op = sgd.apply_gradients(zip([grads0, grads1], [var0, var1]))
+    self.evaluate(variables.global_variables_initializer())
+    # Run 2 steps of sgd
+    if not context.executing_eagerly():
+      self.evaluate(sgd_op)
+    else:
+      sgd.apply_gradients(zip([grads0, grads1], [var0, var1]))
+    # Validate updated params
+    self.assertAllCloseAccordingToType([1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1],
+                                       self.evaluate(var0))
+    self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01],
+                                       self.evaluate(var1))
+
+    if not context.executing_eagerly():
+      self.evaluate(sgd_op)
+    else:
+      sgd.apply_gradients(zip([grads0, grads1], [var0, var1]))
+    # Validate updated params
+    self.assertAllCloseAccordingToType(
+        [1.0 - 3.0 * 0.1 - 2.0 * 0.1, 2.0 - 3.0 * 0.1 - 2.0 * 0.1],
+        self.evaluate(var0))
+    self.assertAllCloseAccordingToType(
+        [3.0 - 3.0 * 0.01 - 2.0 * 0.01, 4.0 - 3.0 * 0.01 - 2.0 * 0.01],
+        self.evaluate(var1))
+
   @test_util.run_in_graph_and_eager_modes
   def testBasicWithLearningRateDecay(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
-      with self.cached_session():
-        var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype)
-        var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype)
-        grads0 = constant_op.constant([0.1, 0.1], dtype=dtype)
-        grads1 = constant_op.constant([0.01, 0.01], dtype=dtype)
-        learning_rate = 3.0
-        decay = 0.5
-        sgd = gradient_descent.SGD(learning_rate=learning_rate, decay=decay)
-        if not context.executing_eagerly():
-          sgd_op = sgd.apply_gradients(zip([grads0, grads1], [var0, var1]))
-        self.evaluate(variables.global_variables_initializer())
-        # Run 2 steps of sgd
-        if not context.executing_eagerly():
-          self.evaluate(sgd_op)
-        else:
-          sgd.apply_gradients(zip([grads0, grads1], [var0, var1]))
-        # Validate updated params
-        self.assertAllCloseAccordingToType([1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1],
-                                           self.evaluate(var0))
-        self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01],
-                                           self.evaluate(var1))
+      learning_rate = 3.0
+      decay = 0.5
+      sgd = gradient_descent.SGD(learning_rate=learning_rate, decay=decay)
+      self._test_basic_sgd_with_learning_rate_decay(sgd, dtype)
 
-        if not context.executing_eagerly():
-          self.evaluate(sgd_op)
-        else:
-          sgd.apply_gradients(zip([grads0, grads1], [var0, var1]))
-        # Validate updated params
-        self.assertAllCloseAccordingToType(
-            [1.0 - 3.0 * 0.1 - 2.0 * 0.1, 2.0 - 3.0 * 0.1 - 2.0 * 0.1],
-            self.evaluate(var0))
-        self.assertAllCloseAccordingToType(
-            [3.0 - 3.0 * 0.01 - 2.0 * 0.01, 4.0 - 3.0 * 0.01 - 2.0 * 0.01],
-            self.evaluate(var1))
+  @test_util.run_in_graph_and_eager_modes
+  def testBasicWithLearningRateInverseTimeDecay(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      learning_rate = learning_rate_schedule.InverseTimeDecay(
+          3.0, decay_steps=1.0, decay_rate=0.5)
+      sgd = gradient_descent.SGD(learning_rate=learning_rate)
+      self._test_basic_sgd_with_learning_rate_decay(sgd, dtype)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testBasicWithLearningRateInverseTimeDecaySerializeAndDeserialize(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      learning_rate = learning_rate_schedule.InverseTimeDecay(
+          3.0, decay_steps=1.0, decay_rate=0.5)
+      sgd = gradient_descent.SGD(learning_rate=learning_rate)
+      sgd = gradient_descent.SGD.from_config(sgd.get_config())
+      self._test_basic_sgd_with_learning_rate_decay(sgd, dtype)
 
   @test_util.run_in_graph_and_eager_modes
   def testBasicCallableParams(self):
diff --git a/tensorflow/python/keras/optimizer_v2/learning_rate_schedule.py b/tensorflow/python/keras/optimizer_v2/learning_rate_schedule.py
new file mode 100644
index 0000000000..a182d74e56
--- /dev/null
+++ b/tensorflow/python/keras/optimizer_v2/learning_rate_schedule.py
@@ -0,0 +1,1031 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Various learning rate decay functions."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import abc
+import math
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.keras.utils import generic_utils
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import random_ops
+from tensorflow.python.util.tf_export import keras_export
+
+
+@keras_export("keras.optimizers.schedules.LearningRateSchedule")
+class LearningRateSchedule(object):
+  """A serializable learning rate decay schedule.
+
+  `LearningRateSchedule`s can be passed in as the learning rate of optimizers in
+  `tf.keras.optimizers`. They can be serialized and deserialized using
+  `tf.keras.optimizers.schedules.serialize` and
+  `tf.keras.optimizers.schedules.deserialize`.
+  """
+
+  @abc.abstractmethod
+  def __call__(self, step):
+    raise NotImplementedError("Learning rate schedule must override __call__")
+
+  @abc.abstractmethod
+  def get_config(self):
+    raise NotImplementedError("Learning rate schedule must override get_config")
+
+  @classmethod
+  def from_config(cls, config):
+    """Instantiates a `LearningRateSchedule` from its config.
+
+    Args:
+        config: Output of `get_config()`.
+
+    Returns:
+        A `LearningRateSchedule` instance.
+    """
+    return cls(**config)
+
+
+@keras_export("keras.optimizers.schedules.ExponentialDecay")
+class ExponentialDecay(LearningRateSchedule):
+  """A LearningRateSchedule that uses an exponential decay schedule."""
+
+  def __init__(
+      self,
+      initial_learning_rate,
+      decay_steps,
+      decay_rate,
+      staircase=False,
+      name=None):
+    """Applies exponential decay to the learning rate.
+
+    When training a model, it is often recommended to lower the learning rate as
+    the training progresses. This schedule applies an exponential decay function
+    to an optimizer step, given a provided initial learning rate.
+
+    The schedule a 1-arg callable that produces a decayed learning
+    rate when passed the current optimizer step. This can be useful for changing
+    the learning rate value across different invocations of optimizer functions.
+    It is computed as:
+
+    ```python
+    def decayed_learning_rate(step):
+      return initial_learning_rate * decay_rate ^ (step / decay_steps)
+    ```
+
+    If the argument `staircase` is `True`, then `step / decay_steps` is
+    an integer division and the decayed learning rate follows a
+    staircase function.
+
+    You can pass this schedule directly into a `tf.keras.optimizers.Optimizer`
+    as the learning rate.
+    Example: When fitting a Keras model, decay every 100000 steps with a base
+    of 0.96:
+
+    ```python
+    initial_learning_rate = 0.1
+    lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(
+        initial_learning_rate,
+        decay_steps=100000,
+        decay_rate=0.96,
+        staircase=True)
+
+    model.compile(optimizer=tf.keras.optimizers.SGD(learning_rate=lr_schedule),
+                  loss='sparse_categorical_crossentropy',
+                  metrics=['accuracy'])
+
+    model.fit(data, labels, epochs=5)
+    ```
+
+    The learning rate schedule is also serializable and deserializable using
+    `tf.keras.optimizers.schedules.serialize` and
+    `tf.keras.optimizers.schedules.deserialize`.
+
+    Args:
+      initial_learning_rate: A scalar `float32` or `float64` `Tensor` or a
+        Python number.  The initial learning rate.
+      decay_steps: A scalar `int32` or `int64` `Tensor` or a Python number.
+        Must be positive.  See the decay computation above.
+      decay_rate: A scalar `float32` or `float64` `Tensor` or a
+        Python number.  The decay rate.
+      staircase: Boolean.  If `True` decay the learning rate at discrete
+        intervals
+      name: String.  Optional name of the operation.  Defaults to
+        'ExponentialDecay'.
+
+    Returns:
+      A 1-arg callable learning rate schedule that takes the current optimizer
+      step and outputs the decayed learning rate, a scalar `Tensor` of the same
+      type as `initial_learning_rate`.
+    """
+    super(ExponentialDecay, self).__init__()
+    self.initial_learning_rate = initial_learning_rate
+    self.decay_steps = decay_steps
+    self.decay_rate = decay_rate
+    self.staircase = staircase
+    self.name = name
+
+  def __call__(self, step):
+    with ops.name_scope(
+        self.name, "ExponentialDecay",
+        [self.initial_learning_rate, step, self.decay_steps, self.decay_rate]
+    ) as name:
+      initial_learning_rate = ops.convert_to_tensor(
+          self.initial_learning_rate, name="initial_learning_rate")
+      dtype = initial_learning_rate.dtype
+      decay_steps = math_ops.cast(self.decay_steps, dtype)
+      decay_rate = math_ops.cast(self.decay_rate, dtype)
+
+      global_step_recomp = math_ops.cast(step, dtype)
+      p = global_step_recomp / decay_steps
+      if self.staircase:
+        p = math_ops.floor(p)
+      return math_ops.multiply(
+          initial_learning_rate, math_ops.pow(decay_rate, p), name=name)
+
+  def get_config(self):
+    return {
+        "initial_learning_rate": self.initial_learning_rate,
+        "decay_steps": self.decay_steps,
+        "decay_rate": self.decay_rate,
+        "staircase": self.staircase,
+        "name": self.name
+    }
+
+
+@keras_export("keras.optimizers.schedules.PiecewiseConstantDecay")
+class PiecewiseConstantDecay(LearningRateSchedule):
+  """A LearningRateSchedule that uses a piecewise constant decay schedule."""
+
+  def __init__(
+      self,
+      boundaries,
+      values,
+      name=None):
+    """Piecewise constant from boundaries and interval values.
+
+    The function returns a 1-arg callable to compute the piecewise constant
+    when passed the current optimizer step. This can be useful for changing the
+    learning rate value across different invocations of optimizer functions.
+
+    Example: use a learning rate that's 1.0 for the first 100001 steps, 0.5
+      for the next 10000 steps, and 0.1 for any additional steps.
+
+    ```python
+    step = tf.Variable(0, trainable=False)
+    boundaries = [100000, 110000]
+    values = [1.0, 0.5, 0.1]
+    learning_rate_fn = keras.optimizers.schedules.PiecewiseConstantDecay(
+        boundaries, values)
+
+    # Later, whenever we perform an optimization step, we pass in the step.
+    learning_rate = learning_rate_fn(step)
+    ```
+
+    You can pass this schedule directly into a `tf.keras.optimizers.Optimizer`
+    as the learning rate. The learning rate schedule is also serializable and
+    deserializable using `tf.keras.optimizers.schedules.serialize` and
+    `tf.keras.optimizers.schedules.deserialize`.
+
+    Args:
+      boundaries: A list of `Tensor`s or `int`s or `float`s with strictly
+        increasing entries, and with all elements having the same type as the
+        optimizer step.
+      values: A list of `Tensor`s or `float`s or `int`s that specifies the
+        values for the intervals defined by `boundaries`. It should have one
+        more element than `boundaries`, and all elements should have the same
+        type.
+      name: A string. Optional name of the operation. Defaults to
+        'PiecewiseConstant'.
+
+    Returns:
+      A 1-arg callable learning rate schedule that takes the current optimizer
+      step and outputs the decayed learning rate, a scalar `Tensor` of the same
+      type as the boundary tensors.
+
+      The output of the 1-arg function that takes the `step`
+      is `values[0]` when `step <= boundaries[0]`,
+      `values[1]` when `step > boundaries[0]` and `step <= boundaries[1]`, ...,
+      and values[-1] when `step > boundaries[-1]`.
+
+    Raises:
+      ValueError: if types of all `values` do not match or
+          the number of elements in the lists does not match.
+    """
+    super(PiecewiseConstantDecay, self).__init__()
+
+    if len(boundaries) != len(values) - 1:
+      raise ValueError(
+          "The length of boundaries should be 1 less than the length of values")
+
+    self.boundaries = boundaries
+    self.values = values
+    self.name = name
+
+  def __call__(self, step):
+    with ops.name_scope(self.name, "PiecewiseConstant",
+                        [step, self.boundaries, self.values, self.name]):
+      boundaries = ops.convert_n_to_tensor(self.boundaries)
+      values = ops.convert_n_to_tensor(self.values)
+      x_recomp = ops.convert_to_tensor(step)
+      # Avoid explicit conversion to x's dtype. This could result in faulty
+      # comparisons, for example if floats are converted to integers.
+      for i, b in enumerate(boundaries):
+        if b.dtype.base_dtype != x_recomp.dtype.base_dtype:
+          # We can promote int32 boundaries to int64 without loss of precision.
+          # This covers the most common case where the user passes in boundaries
+          # as an array of Python integers.
+          if (b.dtype.base_dtype == dtypes.int32 and
+              x_recomp.dtype.base_dtype == dtypes.int64):
+            b = math_ops.cast(b, x_recomp.dtype.base_dtype)
+            boundaries[i] = b
+          else:
+            raise ValueError(
+                "Boundaries (%s) must have the same dtype as x (%s)." %
+                (b.dtype.base_dtype, x_recomp.dtype.base_dtype))
+      # TODO(rdipietro): Ensure that boundaries' elements strictly increases.
+      for v in values[1:]:
+        if v.dtype.base_dtype != values[0].dtype.base_dtype:
+          raise ValueError(
+              "Values must have elements all with the same dtype (%s vs %s)." %
+              (values[0].dtype.base_dtype, v.dtype.base_dtype))
+      pred_fn_pairs = []
+      pred_fn_pairs.append((x_recomp <= boundaries[0], lambda: values[0]))
+      pred_fn_pairs.append((x_recomp > boundaries[-1], lambda: values[-1]))
+      for low, high, v in zip(boundaries[:-1], boundaries[1:], values[1:-1]):
+        # Need to bind v here; can do this with lambda v=v: ...
+        pred = (x_recomp > low) & (x_recomp <= high)
+        pred_fn_pairs.append((pred, lambda v=v: v))
+
+      # The default isn't needed here because our conditions are mutually
+      # exclusive and exhaustive, but tf.case requires it.
+      default = lambda: values[0]
+      return control_flow_ops.case(pred_fn_pairs, default, exclusive=True)
+
+  def get_config(self):
+    return {
+        "boundaries": self.boundaries,
+        "values": self.values,
+        "name": self.name
+    }
+
+
+@keras_export("keras.optimizers.schedules.PolynomialDecay")
+class PolynomialDecay(LearningRateSchedule):
+  """A LearningRateSchedule that uses a polynomial decay schedule."""
+
+  def __init__(
+      self,
+      initial_learning_rate,
+      decay_steps,
+      end_learning_rate=0.0001,
+      power=1.0,
+      cycle=False,
+      name=None):
+    """Applies a polynomial decay to the learning rate.
+
+    It is commonly observed that a monotonically decreasing learning rate, whose
+    degree of change is carefully chosen, results in a better performing model.
+    This schedule applies a polynomial decay function to an optimizer step,
+    given a provided `initial_learning_rate`, to reach an `end_learning_rate`
+    in the given `decay_steps`.
+
+    It requires a `step` value to compute the decayed learning rate. You
+    can just pass a TensorFlow variable that you increment at each training
+    step.
+
+    The schedule is a 1-arg callable that produces a decayed learning rate
+    when passed the current optimizer step. This can be useful for changing the
+    learning rate value across different invocations of optimizer functions.
+    It is computed as:
+
+    ```python
+    def decayed_learning_rate(step):
+      step = min(step, decay_steps)
+      return ((initial_learning_rate - end_learning_rate) *
+              (1 - step / decay_steps) ^ (power)
+             ) + end_learning_rate
+    ```
+
+    If `cycle` is True then a multiple of `decay_steps` is used, the first one
+    that is bigger than `step`.
+
+    ```python
+    def decayed_learning_rate(step):
+      decay_steps = decay_steps * ceil(step / decay_steps)
+      return ((initial_learning_rate - end_learning_rate) *
+              (1 - step / decay_steps) ^ (power)
+             ) + end_learning_rate
+    ```
+
+    You can pass this schedule directly into a `tf.keras.optimizers.Optimizer`
+    as the learning rate.
+    Example: Fit a model while decaying from 0.1 to 0.01 in 10000 steps using
+    sqrt (i.e. power=0.5):
+
+    ```python
+    ...
+    starter_learning_rate = 0.1
+    end_learning_rate = 0.01
+    decay_steps = 10000
+    learning_rate_fn = tf.keras.optimizers.schedules.PolynomialDecay(
+        starter_learning_rate,
+        decay_steps,
+        end_learning_rate,
+        power=0.5)
+
+    model.compile(optimizer=tf.keras.optimizers.SGD(
+                      learning_rate=learning_rate_fn),
+                  loss='sparse_categorical_crossentropy',
+                  metrics=['accuracy'])
+
+    model.fit(data, labels, epochs=5)
+    ```
+
+    The learning rate schedule is also serializable and deserializable using
+    `tf.keras.optimizers.schedules.serialize` and
+    `tf.keras.optimizers.schedules.deserialize`.
+
+    Args:
+      initial_learning_rate: A scalar `float32` or `float64` `Tensor` or a
+        Python number.  The initial learning rate.
+      decay_steps: A scalar `int32` or `int64` `Tensor` or a Python number.
+        Must be positive.  See the decay computation above.
+      end_learning_rate: A scalar `float32` or `float64` `Tensor` or a
+        Python number.  The minimal end learning rate.
+      power: A scalar `float32` or `float64` `Tensor` or a
+        Python number.  The power of the polynomial. Defaults to linear, 1.0.
+      cycle: A boolean, whether or not it should cycle beyond decay_steps.
+      name: String.  Optional name of the operation. Defaults to
+        'PolynomialDecay'.
+
+    Returns:
+      A 1-arg callable learning rate schedule that takes the current optimizer
+      step and outputs the decayed learning rate, a scalar `Tensor` of the same
+      type as `initial_learning_rate`.
+    """
+    super(PolynomialDecay, self).__init__()
+
+    self.initial_learning_rate = initial_learning_rate
+    self.decay_steps = decay_steps
+    self.end_learning_rate = end_learning_rate
+    self.power = power
+    self.cycle = cycle
+    self.name = name
+
+  def __call__(self, step):
+    with ops.name_scope(
+        self.name, "PolynomialDecay",
+        [self.initial_learning_rate, step, self.decay_steps,
+         self.end_learning_rate, self.power]
+    ) as name:
+      initial_learning_rate = ops.convert_to_tensor(
+          self.initial_learning_rate, name="initial_learning_rate")
+      dtype = initial_learning_rate.dtype
+      end_learning_rate = math_ops.cast(self.end_learning_rate, dtype)
+      power = math_ops.cast(self.power, dtype)
+
+      global_step_recomp = math_ops.cast(step, dtype)
+      decay_steps_recomp = math_ops.cast(self.decay_steps, dtype)
+      if self.cycle:
+        # Find the first multiple of decay_steps that is bigger than
+        # global_step. If global_step is zero set the multiplier to 1
+        multiplier = control_flow_ops.cond(
+            math_ops.equal(global_step_recomp, 0), lambda: 1.0,
+            lambda: math_ops.ceil(global_step_recomp / self.decay_steps))
+        decay_steps_recomp = math_ops.multiply(decay_steps_recomp, multiplier)
+      else:
+        # Make sure that the global_step used is not bigger than decay_steps.
+        global_step_recomp = math_ops.minimum(global_step_recomp,
+                                              self.decay_steps)
+
+      p = math_ops.div(global_step_recomp, decay_steps_recomp)
+      return math_ops.add(
+          math_ops.multiply(initial_learning_rate - end_learning_rate,
+                            math_ops.pow(1 - p, power)),
+          end_learning_rate,
+          name=name)
+
+  def get_config(self):
+    return {
+        "initial_learning_rate": self.initial_learning_rate,
+        "decay_steps": self.decay_steps,
+        "end_learning_rate": self.end_learning_rate,
+        "power": self.power,
+        "cycle": self.cycle,
+        "name": self.name
+    }
+
+
+@keras_export("keras.optimizers.schedules.InverseTimeDecay")
+class InverseTimeDecay(LearningRateSchedule):
+  """A LearningRateSchedule that uses an inverse time decay schedule."""
+
+  def __init__(
+      self,
+      initial_learning_rate,
+      decay_steps,
+      decay_rate,
+      staircase=False,
+      name=None):
+    """Applies inverse time decay to the initial learning rate.
+
+    When training a model, it is often recommended to lower the learning rate as
+    the training progresses. This schedule applies the inverse decay function
+    to an optimizer step, given a provided initial learning rate.
+    It requires a `step` value to compute the decayed learning rate. You can
+    just pass a TensorFlow variable that you increment at each training step.
+
+    The schedule a 1-arg callable that produces a decayed learning
+    rate when passed the current optimizer step. This can be useful for changing
+    the learning rate value across different invocations of optimizer functions.
+    It is computed as:
+
+    ```python
+    def decayed_learning_rate(step):
+      return initial_learning_rate / (1 + decay_rate * step / decay_step)
+    ```
+
+    or, if `staircase` is `True`, as:
+
+    ```python
+    def decayed_learning_rate(step):
+      return initial_learning_rate / (1 + decay_rate * floor(step / decay_step))
+    ```
+
+    You can pass this schedule directly into a `tf.keras.optimizers.Optimizer`
+    as the learning rate.
+    Example: Fit a Keras model when decaying 1/t with a rate of 0.5:
+
+    ```python
+    ...
+    initial_learning_rate = 0.1
+    decay_steps = 1.0
+    decay_rate = 0.5
+    learning_rate_fn = keras.optimizers.schedules.InverseTimeDecay(
+      initial_learning_rate, global_step, decay_steps, decay_rate)
+
+    model.compile(optimizer=tf.keras.optimizers.SGD(
+                      learning_rate=learning_rate_fn),
+                  loss='sparse_categorical_crossentropy',
+                  metrics=['accuracy'])
+
+    model.fit(data, labels, epochs=5)
+    ```
+
+    Args:
+      initial_learning_rate: A scalar `float32` or `float64` `Tensor` or a
+        Python number.  The initial learning rate.
+      decay_steps: How often to apply decay.
+      decay_rate: A Python number.  The decay rate.
+      staircase: Whether to apply decay in a discrete staircase, as opposed to
+        continuous, fashion.
+      name: String.  Optional name of the operation.  Defaults to
+        'InverseTimeDecay'.
+
+    Returns:
+      A 1-arg callable learning rate schedule that takes the current optimizer
+      step and outputs the decayed learning rate, a scalar `Tensor` of the same
+      type as `initial_learning_rate`.
+    """
+    super(InverseTimeDecay, self).__init__()
+
+    self.initial_learning_rate = initial_learning_rate
+    self.decay_steps = decay_steps
+    self.decay_rate = decay_rate
+    self.staircase = staircase
+    self.name = name
+
+  def __call__(self, step):
+    with ops.name_scope(self.name, "InverseTimeDecay",
+                        [self.initial_learning_rate, step, self.decay_rate]
+                       ) as name:
+      initial_learning_rate = ops.convert_to_tensor(
+          self.initial_learning_rate, name="initial_learning_rate")
+      dtype = initial_learning_rate.dtype
+      decay_steps = math_ops.cast(self.decay_steps, dtype)
+      decay_rate = math_ops.cast(self.decay_rate, dtype)
+
+      global_step_recomp = math_ops.cast(step, dtype)
+      p = global_step_recomp / decay_steps
+      if self.staircase:
+        p = math_ops.floor(p)
+      const = math_ops.cast(constant_op.constant(1), dtype)
+      denom = math_ops.add(const, math_ops.multiply(decay_rate, p))
+      return math_ops.div(initial_learning_rate, denom, name=name)
+
+  def get_config(self):
+    return {
+        "initial_learning_rate": self.initial_learning_rate,
+        "decay_steps": self.decay_steps,
+        "decay_rate": self.decay_rate,
+        "staircase": self.staircase,
+        "name": self.name
+    }
+
+
+@keras_export("keras.experimental.CosineDecay")
+class CosineDecay(LearningRateSchedule):
+  """A LearningRateSchedule that uses a cosine decay schedule."""
+
+  def __init__(
+      self,
+      initial_learning_rate,
+      decay_steps,
+      alpha=0.0,
+      name=None):
+    """Applies cosine decay to the learning rate.
+
+    See [Loshchilov & Hutter, ICLR2016], SGDR: Stochastic Gradient Descent
+    with Warm Restarts. https://arxiv.org/abs/1608.03983
+
+    When training a model, it is often recommended to lower the learning rate as
+    the training progresses. This schedule applies a cosine decay function
+    to an optimizer step, given a provided initial learning rate.
+    It requires a `step` value to compute the decayed learning rate. You can
+    just pass a TensorFlow variable that you increment at each training step.
+
+    The schedule a 1-arg callable that produces a decayed learning
+    rate when passed the current optimizer step. This can be useful for changing
+    the learning rate value across different invocations of optimizer functions.
+    It is computed as:
+
+    ```python
+    def decayed_learning_rate(step):
+      step = min(step, decay_steps)
+      cosine_decay = 0.5 * (1 + cos(pi * step / decay_steps))
+      decayed = (1 - alpha) * cosine_decay + alpha
+      return initial_learning_rate * decayed
+    ```
+
+    Example usage:
+    ```python
+    decay_steps = 1000
+    lr_decayed_fn = tf.keras.experimental.CosineDecay(
+        initial_learning_rate, global_step, decay_steps)
+    ```
+
+    You can pass this schedule directly into a `tf.keras.optimizers.Optimizer`
+    as the learning rate. The learning rate schedule is also serializable and
+    deserializable using `tf.keras.optimizers.schedules.serialize` and
+    `tf.keras.optimizers.schedules.deserialize`.
+
+    Args:
+      initial_learning_rate: A scalar `float32` or `float64` Tensor or a
+        Python number. The initial learning rate.
+      decay_steps: A scalar `int32` or `int64` `Tensor` or a Python number.
+        Number of steps to decay over.
+      alpha: A scalar `float32` or `float64` Tensor or a Python number.
+        Minimum learning rate value as a fraction of initial_learning_rate.
+      name: String. Optional name of the operation.  Defaults to 'CosineDecay'.
+    Returns:
+      A 1-arg callable learning rate schedule that takes the current optimizer
+      step and outputs the decayed learning rate, a scalar `Tensor` of the same
+      type as `initial_learning_rate`.
+    """
+    super(CosineDecay, self).__init__()
+
+    self.initial_learning_rate = initial_learning_rate
+    self.decay_steps = decay_steps
+    self.alpha = alpha
+    self.name = name
+
+  def __call__(self, step):
+    with ops.name_scope(self.name, "CosineDecay",
+                        [self.initial_learning_rate, step]):
+      initial_learning_rate = ops.convert_to_tensor(
+          self.initial_learning_rate, name="initial_learning_rate")
+      dtype = initial_learning_rate.dtype
+      decay_steps = math_ops.cast(self.decay_steps, dtype)
+
+      global_step_recomp = math_ops.cast(step, dtype)
+      global_step_recomp = math_ops.minimum(global_step_recomp, decay_steps)
+      completed_fraction = global_step_recomp / decay_steps
+      cosine_decayed = 0.5 * (1.0 + math_ops.cos(
+          constant_op.constant(math.pi) * completed_fraction))
+
+      decayed = (1 - self.alpha) * cosine_decayed + self.alpha
+      return math_ops.multiply(initial_learning_rate, decayed)
+
+  def get_config(self):
+    return {
+        "initial_learning_rate": self.initial_learning_rate,
+        "decay_steps": self.decay_steps,
+        "alpha": self.alpha,
+        "name": self.name
+    }
+
+
+@keras_export("keras.experimental.CosineDecayRestarts",
+              v1=[])
+class CosineDecayRestarts(LearningRateSchedule):
+  """A LearningRateSchedule that uses a cosine decay schedule with restarts."""
+
+  def __init__(
+      self,
+      initial_learning_rate,
+      first_decay_steps,
+      t_mul=2.0,
+      m_mul=1.0,
+      alpha=0.0,
+      name=None):
+    """Applies cosine decay with restarts to the learning rate.
+
+    See [Loshchilov & Hutter, ICLR2016], SGDR: Stochastic Gradient Descent
+    with Warm Restarts. https://arxiv.org/abs/1608.03983
+
+    When training a model, it is often recommended to lower the learning rate as
+    the training progresses. This schedule applies a cosine decay function with
+    restarts to an optimizer step, given a provided initial learning rate.
+    It requires a `step` value to compute the decayed learning rate. You can
+    just pass a TensorFlow variable that you increment at each training step.
+
+    The schedule a 1-arg callable that produces a decayed learning
+    rate when passed the current optimizer step. This can be useful for changing
+    the learning rate value across different invocations of optimizer functions.
+
+    The learning rate multiplier first decays
+    from 1 to `alpha` for `first_decay_steps` steps. Then, a warm
+    restart is performed. Each new warm restart runs for `t_mul` times more
+    steps and with `m_mul` times smaller initial learning rate.
+
+    Example usage:
+    ```python
+    first_decay_steps = 1000
+    lr_decayed_fn = (
+      tf.keras.experimental.CosineDecayRestarts(
+          initial_learning_rate,
+          global_step,
+          first_decay_steps))
+    ```
+
+    You can pass this schedule directly into a `tf.keras.optimizers.Optimizer`
+    as the learning rate. The learning rate schedule is also serializable and
+    deserializable using `tf.keras.optimizers.schedules.serialize` and
+    `tf.keras.optimizers.schedules.deserialize`.
+
+    Args:
+      initial_learning_rate: A scalar `float32` or `float64` Tensor or a Python
+        number. The initial learning rate.
+      first_decay_steps: A scalar `int32` or `int64` `Tensor` or a Python
+        number. Number of steps to decay over.
+      t_mul: A scalar `float32` or `float64` `Tensor` or a Python number.
+        Used to derive the number of iterations in the i-th period
+      m_mul: A scalar `float32` or `float64` `Tensor` or a Python number.
+        Used to derive the initial learning rate of the i-th period:
+      alpha: A scalar `float32` or `float64` Tensor or a Python number.
+        Minimum learning rate value as a fraction of the initial_learning_rate.
+      name: String. Optional name of the operation.  Defaults to 'SGDRDecay'.
+    Returns:
+      A 1-arg callable learning rate schedule that takes the current optimizer
+      step and outputs the decayed learning rate, a scalar `Tensor` of the same
+      type as `initial_learning_rate`.
+    Raises:
+      ValueError: if `global_step` is not supplied.
+    """
+    super(CosineDecayRestarts, self).__init__()
+
+    self.initial_learning_rate = initial_learning_rate
+    self.first_decay_steps = first_decay_steps
+    self._t_mul = t_mul
+    self._m_mul = m_mul
+    self.alpha = alpha
+    self.name = name
+
+  def __call__(self, step):
+    with ops.name_scope(self.name, "SGDRDecay",
+                        [self.initial_learning_rate, step]
+                       ) as name:
+      initial_learning_rate = ops.convert_to_tensor(
+          self.initial_learning_rate, name="initial_learning_rate")
+      dtype = initial_learning_rate.dtype
+      first_decay_steps = math_ops.cast(self.first_decay_steps, dtype)
+      alpha = math_ops.cast(self.alpha, dtype)
+      t_mul = math_ops.cast(self._t_mul, dtype)
+      m_mul = math_ops.cast(self._m_mul, dtype)
+
+      global_step_recomp = math_ops.cast(step, dtype)
+      completed_fraction = global_step_recomp / first_decay_steps
+
+      def compute_step(completed_fraction, geometric=False):
+        """Helper for `cond` operation."""
+        if geometric:
+          i_restart = math_ops.floor(
+              math_ops.log(1.0 - completed_fraction * (1.0 - t_mul)) /
+              math_ops.log(t_mul))
+
+          sum_r = (1.0 - t_mul**i_restart) / (1.0 - t_mul)
+          completed_fraction = (completed_fraction - sum_r) / t_mul**i_restart
+
+        else:
+          i_restart = math_ops.floor(completed_fraction)
+          completed_fraction -= i_restart
+
+        return i_restart, completed_fraction
+
+      i_restart, completed_fraction = control_flow_ops.cond(
+          math_ops.equal(t_mul, 1.0),
+          lambda: compute_step(completed_fraction, geometric=False),
+          lambda: compute_step(completed_fraction, geometric=True))
+
+      m_fac = m_mul**i_restart
+      cosine_decayed = 0.5 * m_fac * (1.0 + math_ops.cos(
+          constant_op.constant(math.pi) * completed_fraction))
+      decayed = (1 - alpha) * cosine_decayed + alpha
+
+      return math_ops.multiply(initial_learning_rate, decayed, name=name)
+
+  def get_config(self):
+    return {
+        "initial_learning_rate": self.initial_learning_rate,
+        "first_decay_steps": self.first_decay_steps,
+        "t_mul": self._t_mul,
+        "m_mul": self._m_mul,
+        "alpha": self.alpha,
+        "name": self.name
+    }
+
+
+@keras_export("keras.experimental.LinearCosineDecay",
+              v1=[])
+class LinearCosineDecay(LearningRateSchedule):
+  """A LearningRateSchedule that uses a linear cosine decay schedule."""
+
+  def __init__(
+      self,
+      initial_learning_rate,
+      decay_steps,
+      num_periods=0.5,
+      alpha=0.0,
+      beta=0.001,
+      name=None):
+    """Applies linear cosine decay to the learning rate.
+
+    See [Bello et al., ICML2017] Neural Optimizer Search with RL.
+    https://arxiv.org/abs/1709.07417
+
+    For the idea of warm starts here controlled by `num_periods`,
+    see [Loshchilov & Hutter, ICLR2016] SGDR: Stochastic Gradient Descent
+    with Warm Restarts. https://arxiv.org/abs/1608.03983
+
+    Note that linear cosine decay is more aggressive than cosine decay and
+    larger initial learning rates can typically be used.
+
+    When training a model, it is often recommended to lower the learning rate as
+    the training progresses. This schedule applies a linear cosine decay
+    function to an optimizer step, given a provided initial learning rate.
+    It requires a `step` value to compute the decayed learning rate. You can
+    just pass a TensorFlow variable that you increment at each training step.
+
+    The schedule a 1-arg callable that produces a decayed learning
+    rate when passed the current optimizer step. This can be useful for changing
+    the learning rate value across different invocations of optimizer functions.
+    It is computed as:
+
+    ```python
+    def decayed_learning_rate(step):
+      step = min(step, decay_steps)
+      linear_decay = (decay_steps - step) / decay_steps
+      cosine_decay = 0.5 * (
+          1 + cos(pi * 2 * num_periods * step / decay_steps))
+      decayed = (alpha + linear_decay) * cosine_decay + beta
+      return initial_learning_rate * decayed
+    ```
+
+    Example usage:
+    ```python
+    decay_steps = 1000
+    lr_decayed_fn = (
+      tf.keras.experimental.LinearCosineDecay(
+        initial_learning_rate, global_step, decay_steps))
+    ```
+
+    You can pass this schedule directly into a `tf.keras.optimizers.Optimizer`
+    as the learning rate. The learning rate schedule is also serializable and
+    deserializable using `tf.keras.optimizers.schedules.serialize` and
+    `tf.keras.optimizers.schedules.deserialize`.
+
+    Args:
+      initial_learning_rate: A scalar `float32` or `float64` Tensor or a Python
+        number. The initial learning rate.
+      decay_steps: A scalar `int32` or `int64` `Tensor` or a Python number.
+        Number of steps to decay over.
+      num_periods: Number of periods in the cosine part of the decay.
+        See computation above.
+      alpha: See computation above.
+      beta: See computation above.
+      name: String.  Optional name of the operation.  Defaults to
+        'LinearCosineDecay'.
+    Returns:
+      A 1-arg callable learning rate schedule that takes the current optimizer
+      step and outputs the decayed learning rate, a scalar `Tensor` of the same
+      type as `initial_learning_rate`.
+    """
+    super(LinearCosineDecay, self).__init__()
+
+    self.initial_learning_rate = initial_learning_rate
+    self.decay_steps = decay_steps
+    self.num_periods = num_periods
+    self.alpha = alpha
+    self.beta = beta
+    self.name = name
+
+  def __call__(self, step):
+    with ops.name_scope(self.name, "LinearCosineDecay",
+                        [self.initial_learning_rate, step]) as name:
+      initial_learning_rate = ops.convert_to_tensor(
+          self.initial_learning_rate, name="initial_learning_rate")
+      dtype = initial_learning_rate.dtype
+      decay_steps = math_ops.cast(self.decay_steps, dtype)
+      num_periods = math_ops.cast(self.num_periods, dtype)
+      alpha = math_ops.cast(self.alpha, dtype)
+      beta = math_ops.cast(self.beta, dtype)
+
+      global_step_recomp = math_ops.cast(step, dtype)
+      global_step_recomp = math_ops.minimum(global_step_recomp, decay_steps)
+      linear_decayed = (decay_steps - global_step_recomp) / decay_steps
+      completed_fraction = global_step_recomp / decay_steps
+      fraction = 2.0 * num_periods * completed_fraction
+      cosine_decayed = 0.5 * (
+          1.0 + math_ops.cos(constant_op.constant(math.pi) * fraction))
+
+      linear_cosine_decayed = (alpha + linear_decayed) * cosine_decayed + beta
+      return math_ops.multiply(initial_learning_rate, linear_cosine_decayed,
+                               name=name)
+
+  def get_config(self):
+    return {
+        "initial_learning_rate": self.initial_learning_rate,
+        "decay_steps": self.decay_steps,
+        "num_periods": self.num_periods,
+        "alpha": self.alpha,
+        "beta": self.beta,
+        "name": self.name
+    }
+
+
+@keras_export("keras.experimental.NoisyLinearCosineDecay",
+              v1=[])
+class NoisyLinearCosineDecay(LearningRateSchedule):
+  """A LearningRateSchedule that uses a noisy linear cosine decay schedule."""
+
+  def __init__(
+      self,
+      initial_learning_rate,
+      decay_steps,
+      initial_variance=1.0,
+      variance_decay=0.55,
+      num_periods=0.5,
+      alpha=0.0,
+      beta=0.001,
+      name=None):
+    """Applies noisy linear cosine decay to the learning rate.
+
+    See [Bello et al., ICML2017] Neural Optimizer Search with RL.
+    https://arxiv.org/abs/1709.07417
+
+    For the idea of warm starts here controlled by `num_periods`,
+    see [Loshchilov & Hutter, ICLR2016] SGDR: Stochastic Gradient Descent
+    with Warm Restarts. https://arxiv.org/abs/1608.03983
+
+    Note that linear cosine decay is more aggressive than cosine decay and
+    larger initial learning rates can typically be used.
+
+    When training a model, it is often recommended to lower the learning rate as
+    the training progresses. This schedule applies a noisy linear cosine decay
+    function to an optimizer step, given a provided initial learning rate.
+    It requires a `step` value to compute the decayed learning rate. You can
+    just pass a TensorFlow variable that you increment at each training step.
+
+    The schedule a 1-arg callable that produces a decayed learning
+    rate when passed the current optimizer step. This can be useful for changing
+    the learning rate value across different invocations of optimizer functions.
+    It is computed as:
+
+    ```python
+    def decayed_learning_rate(step):
+      step = min(step, decay_steps)
+      linear_decay = (decay_steps - step) / decay_steps)
+      cosine_decay = 0.5 * (
+          1 + cos(pi * 2 * num_periods * step / decay_steps))
+      decayed = (alpha + linear_decay + eps_t) * cosine_decay + beta
+      return initial_learning_rate * decayed
+    ```
+    where eps_t is 0-centered gaussian noise with variance
+    initial_variance / (1 + global_step) ** variance_decay
+
+    Example usage:
+    ```python
+    decay_steps = 1000
+    lr_decayed_fn = (
+      tf.keras.experimental.NoisyLinearCosineDecay(
+        initial_learning_rate, global_step, decay_steps))
+    ```
+
+    You can pass this schedule directly into a `tf.keras.optimizers.Optimizer`
+    as the learning rate. The learning rate schedule is also serializable and
+    deserializable using `tf.keras.optimizers.schedules.serialize` and
+    `tf.keras.optimizers.schedules.deserialize`.
+
+    Args:
+      initial_learning_rate: A scalar `float32` or `float64` Tensor or a Python
+        number. The initial learning rate.
+      decay_steps: A scalar `int32` or `int64` `Tensor` or a Python number.
+        Number of steps to decay over.
+      initial_variance: initial variance for the noise. See computation above.
+      variance_decay: decay for the noise's variance. See computation above.
+      num_periods: Number of periods in the cosine part of the decay.
+        See computation above.
+      alpha: See computation above.
+      beta: See computation above.
+      name: String.  Optional name of the operation.  Defaults to
+        'NoisyLinearCosineDecay'.
+    Returns:
+      A 1-arg callable learning rate schedule that takes the current optimizer
+      step and outputs the decayed learning rate, a scalar `Tensor` of the same
+      type as `initial_learning_rate`.
+    """
+    super(NoisyLinearCosineDecay, self).__init__()
+
+    self.initial_learning_rate = initial_learning_rate
+    self.decay_steps = decay_steps
+    self.initial_variance = initial_variance
+    self.variance_decay = variance_decay
+    self.num_periods = num_periods
+    self.alpha = alpha
+    self.beta = beta
+    self.name = name
+
+  def __call__(self, step):
+    with ops.name_scope(self.name, "NoisyLinearCosineDecay",
+                        [self.initial_learning_rate, step]) as name:
+      initial_learning_rate = ops.convert_to_tensor(
+          self.initial_learning_rate, name="initial_learning_rate")
+      dtype = initial_learning_rate.dtype
+      decay_steps = math_ops.cast(self.decay_steps, dtype)
+      initial_variance = math_ops.cast(self.initial_variance, dtype)
+      variance_decay = math_ops.cast(self.variance_decay, dtype)
+      num_periods = math_ops.cast(self.num_periods, dtype)
+      alpha = math_ops.cast(self.alpha, dtype)
+      beta = math_ops.cast(self.beta, dtype)
+
+      global_step_recomp = math_ops.cast(step, dtype)
+      global_step_recomp = math_ops.minimum(global_step_recomp, decay_steps)
+      linear_decayed = (decay_steps - global_step_recomp) / decay_steps
+      variance = initial_variance / (
+          math_ops.pow(1.0 + global_step_recomp, variance_decay))
+      std = math_ops.sqrt(variance)
+      noisy_linear_decayed = (
+          linear_decayed + random_ops.random_normal(
+              linear_decayed.shape, stddev=std))
+
+      completed_fraction = global_step_recomp / decay_steps
+      fraction = 2.0 * num_periods * completed_fraction
+      cosine_decayed = 0.5 * (
+          1.0 + math_ops.cos(constant_op.constant(math.pi) * fraction))
+      noisy_linear_cosine_decayed = (
+          (alpha + noisy_linear_decayed) * cosine_decayed + beta)
+
+      return math_ops.multiply(
+          initial_learning_rate, noisy_linear_cosine_decayed, name=name)
+
+  def get_config(self):
+    return {
+        "initial_learning_rate": self.initial_learning_rate,
+        "decay_steps": self.decay_steps,
+        "initial_variance": self.initial_variance,
+        "variance_decay": self.variance_decay,
+        "num_periods": self.num_periods,
+        "alpha": self.alpha,
+        "beta": self.beta,
+        "name": self.name
+    }
+
+
+@keras_export("keras.optimizers.schedules.serialize")
+def serialize(learning_rate_schedule):
+  return generic_utils.serialize_keras_object(learning_rate_schedule)
+
+
+@keras_export("keras.optimizers.schedules.deserialize")
+def deserialize(config, custom_objects=None):
+  return generic_utils.deserialize_keras_object(
+      config,
+      module_objects=globals(),
+      custom_objects=custom_objects,
+      printable_module_name="decay")
diff --git a/tensorflow/python/keras/optimizer_v2/learning_rate_schedule_test.py b/tensorflow/python/keras/optimizer_v2/learning_rate_schedule_test.py
new file mode 100644
index 0000000000..87b97fa76c
--- /dev/null
+++ b/tensorflow/python/keras/optimizer_v2/learning_rate_schedule_test.py
@@ -0,0 +1,527 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Functional test for learning rate decay."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import math
+from absl.testing import parameterized
+
+from tensorflow.python.eager import context
+from tensorflow.python.framework import test_util
+from tensorflow.python.keras.optimizer_v2 import learning_rate_schedule
+# Import resource_variable_ops for the variables-to-tensor implicit conversion.
+from tensorflow.python.ops import resource_variable_ops  # pylint: disable=unused-import
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import googletest
+
+
+def _maybe_serialized(lr_decay, serialize_and_deserialize):
+  if serialize_and_deserialize:
+    serialized = learning_rate_schedule.serialize(lr_decay)
+    return learning_rate_schedule.deserialize(serialized)
+  else:
+    return lr_decay
+
+
+@parameterized.named_parameters(
+    ("NotSerialized", False),
+    ("Serialized", True))
+class LRDecayTestV2(test_util.TensorFlowTestCase, parameterized.TestCase):
+
+  @test_util.run_in_graph_and_eager_modes
+  def testContinuous(self, serialize):
+    self.evaluate(variables.global_variables_initializer())
+    step = 5
+    decayed_lr = learning_rate_schedule.ExponentialDecay(0.05, 10, 0.96)
+    decayed_lr = _maybe_serialized(decayed_lr, serialize)
+    expected = .05 * 0.96**(5.0 / 10.0)
+    self.assertAllClose(self.evaluate(decayed_lr(step)), expected, 1e-6)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testStaircase(self, serialize):
+    if context.executing_eagerly():
+      step = resource_variable_ops.ResourceVariable(0)
+      self.evaluate(variables.global_variables_initializer())
+      decayed_lr = learning_rate_schedule.ExponentialDecay(
+          .1, 3, 0.96, staircase=True)
+      decayed_lr = _maybe_serialized(decayed_lr, serialize)
+
+      # No change to learning rate due to staircase
+      expected = .1
+      self.evaluate(step.assign(1))
+      self.assertAllClose(self.evaluate(decayed_lr(step)), expected, 1e-6)
+
+      expected = .1
+      self.evaluate(step.assign(2))
+      self.assertAllClose(self.evaluate(decayed_lr(step)), expected, 1e-6)
+
+      # Decayed learning rate
+      expected = .1 * 0.96 ** (100 // 3)
+      self.evaluate(step.assign(100))
+      self.assertAllClose(self.evaluate(decayed_lr(step)), expected, 1e-6)
+
+  @test_util.run_deprecated_v1
+  def testVariables(self, serialize):
+    step = variables.Variable(1)
+    assign_1 = step.assign(1)
+    assign_2 = step.assign(2)
+    assign_100 = step.assign(100)
+    decayed_lr = learning_rate_schedule.ExponentialDecay(
+        .1, 3, 0.96, staircase=True)
+    decayed_lr = _maybe_serialized(decayed_lr, serialize)
+
+    self.evaluate(variables.global_variables_initializer())
+    # No change to learning rate
+    self.evaluate(assign_1.op)
+    self.assertAllClose(self.evaluate(decayed_lr(step)), .1, 1e-6)
+    self.evaluate(assign_2.op)
+    self.assertAllClose(self.evaluate(decayed_lr(step)), .1, 1e-6)
+    # Decayed learning rate
+    self.evaluate(assign_100.op)
+    expected = .1 * 0.96**(100 // 3)
+    self.assertAllClose(self.evaluate(decayed_lr(step)), expected, 1e-6)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testPiecewiseConstant(self, serialize):
+    x = resource_variable_ops.ResourceVariable(-999)
+    decayed_lr = learning_rate_schedule.PiecewiseConstantDecay(
+        [100, 110, 120], [1.0, 0.1, 0.01, 0.001])
+    decayed_lr = _maybe_serialized(decayed_lr, serialize)
+
+    self.evaluate(variables.global_variables_initializer())
+
+    self.assertAllClose(self.evaluate(decayed_lr(x)), 1.0, 1e-6)
+    self.evaluate(x.assign(100))
+    self.assertAllClose(self.evaluate(decayed_lr(x)), 1.0, 1e-6)
+    self.evaluate(x.assign(105))
+    self.assertAllClose(self.evaluate(decayed_lr(x)), 0.1, 1e-6)
+    self.evaluate(x.assign(110))
+    self.assertAllClose(self.evaluate(decayed_lr(x)), 0.1, 1e-6)
+    self.evaluate(x.assign(120))
+    self.assertAllClose(self.evaluate(decayed_lr(x)), 0.01, 1e-6)
+    self.evaluate(x.assign(999))
+    self.assertAllClose(self.evaluate(decayed_lr(x)), 0.001, 1e-6)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testPiecewiseConstantEdgeCases(self, serialize):
+    x_int = resource_variable_ops.ResourceVariable(
+        0, dtype=variables.dtypes.int32)
+    boundaries, values = [-1.0, 1.0], [1, 2, 3]
+    with self.assertRaises(ValueError):
+      decayed_lr = learning_rate_schedule.PiecewiseConstantDecay(
+          boundaries, values)
+      decayed_lr = _maybe_serialized(decayed_lr, serialize)
+      decayed_lr(x_int)
+
+    x = resource_variable_ops.ResourceVariable(0.0)
+    boundaries, values = [-1.0, 1.0], [1.0, 2, 3]
+    with self.assertRaises(ValueError):
+      decayed_lr = learning_rate_schedule.PiecewiseConstantDecay(
+          boundaries, values)
+      decayed_lr = _maybe_serialized(decayed_lr, serialize)
+      decayed_lr(x)
+
+    # Test casting boundaries from int32 to int64.
+    x_int64 = resource_variable_ops.ResourceVariable(
+        0, dtype=variables.dtypes.int64)
+    boundaries, values = [1, 2, 3], [0.4, 0.5, 0.6, 0.7]
+    decayed_lr = learning_rate_schedule.PiecewiseConstantDecay(
+        boundaries, values)
+    decayed_lr = _maybe_serialized(decayed_lr, serialize)
+
+    self.evaluate(variables.global_variables_initializer())
+    self.assertAllClose(self.evaluate(decayed_lr(x_int64)), 0.4, 1e-6)
+    self.evaluate(x_int64.assign(1))
+    self.assertAllClose(self.evaluate(decayed_lr(x_int64)), 0.4, 1e-6)
+    self.evaluate(x_int64.assign(2))
+    self.assertAllClose(self.evaluate(decayed_lr(x_int64)), 0.5, 1e-6)
+    self.evaluate(x_int64.assign(3))
+    self.assertAllClose(self.evaluate(decayed_lr(x_int64)), 0.6, 1e-6)
+    self.evaluate(x_int64.assign(4))
+    self.assertAllClose(self.evaluate(decayed_lr(x_int64)), 0.7, 1e-6)
+
+
+@parameterized.named_parameters(
+    ("NotSerialized", False),
+    ("Serialized", True))
+class LinearDecayTestV2(test_util.TensorFlowTestCase, parameterized.TestCase):
+
+  @test_util.run_in_graph_and_eager_modes
+  def testHalfWay(self, serialize):
+    step = 5
+    lr = 0.05
+    end_lr = 0.0
+    decayed_lr = learning_rate_schedule.PolynomialDecay(lr, 10, end_lr)
+    decayed_lr = _maybe_serialized(decayed_lr, serialize)
+    expected = lr * 0.5
+    self.assertAllClose(self.evaluate(decayed_lr(step)), expected, 1e-6)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testEnd(self, serialize):
+    step = 10
+    lr = 0.05
+    end_lr = 0.001
+    decayed_lr = learning_rate_schedule.PolynomialDecay(lr, 10, end_lr)
+    decayed_lr = _maybe_serialized(decayed_lr, serialize)
+    expected = end_lr
+    self.assertAllClose(self.evaluate(decayed_lr(step)), expected, 1e-6)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testHalfWayWithEnd(self, serialize):
+    step = 5
+    lr = 0.05
+    end_lr = 0.001
+    decayed_lr = learning_rate_schedule.PolynomialDecay(lr, 10, end_lr)
+    decayed_lr = _maybe_serialized(decayed_lr, serialize)
+    expected = (lr + end_lr) * 0.5
+    self.assertAllClose(self.evaluate(decayed_lr(step)), expected, 1e-6)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testBeyondEnd(self, serialize):
+    step = 15
+    lr = 0.05
+    end_lr = 0.001
+    decayed_lr = learning_rate_schedule.PolynomialDecay(lr, 10, end_lr)
+    decayed_lr = _maybe_serialized(decayed_lr, serialize)
+    expected = end_lr
+    self.assertAllClose(self.evaluate(decayed_lr(step)), expected, 1e-6)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testBeyondEndWithCycle(self, serialize):
+    step = 15
+    lr = 0.05
+    end_lr = 0.001
+    decayed_lr = learning_rate_schedule.PolynomialDecay(
+        lr, 10, end_lr, cycle=True)
+    decayed_lr = _maybe_serialized(decayed_lr, serialize)
+    expected = (lr - end_lr) * 0.25 + end_lr
+    self.assertAllClose(self.evaluate(decayed_lr(step)), expected, 1e-6)
+
+
+@parameterized.named_parameters(
+    ("NotSerialized", False),
+    ("Serialized", True))
+class SqrtDecayTestV2(test_util.TensorFlowTestCase,
+                      parameterized.TestCase):
+
+  @test_util.run_in_graph_and_eager_modes
+  def testHalfWay(self, serialize):
+    step = 5
+    lr = 0.05
+    end_lr = 0.0
+    power = 0.5
+    decayed_lr = learning_rate_schedule.PolynomialDecay(
+        lr, 10, end_lr, power=power)
+    decayed_lr = _maybe_serialized(decayed_lr, serialize)
+    expected = lr * 0.5**power
+    self.assertAllClose(self.evaluate(decayed_lr(step)), expected, 1e-6)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testEnd(self, serialize):
+    step = 10
+    lr = 0.05
+    end_lr = 0.001
+    power = 0.5
+    decayed_lr = learning_rate_schedule.PolynomialDecay(
+        lr, 10, end_lr, power=power)
+    decayed_lr = _maybe_serialized(decayed_lr, serialize)
+    expected = end_lr
+    self.assertAllClose(self.evaluate(decayed_lr(step)), expected, 1e-6)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testHalfWayWithEnd(self, serialize):
+    step = 5
+    lr = 0.05
+    end_lr = 0.001
+    power = 0.5
+    decayed_lr = learning_rate_schedule.PolynomialDecay(
+        lr, 10, end_lr, power=power)
+    decayed_lr = _maybe_serialized(decayed_lr, serialize)
+    expected = (lr - end_lr) * 0.5**power + end_lr
+    self.assertAllClose(self.evaluate(decayed_lr(step)), expected, 1e-6)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testBeyondEnd(self, serialize):
+    step = 15
+    lr = 0.05
+    end_lr = 0.001
+    power = 0.5
+    decayed_lr = learning_rate_schedule.PolynomialDecay(
+        lr, 10, end_lr, power=power)
+    decayed_lr = _maybe_serialized(decayed_lr, serialize)
+    expected = end_lr
+    self.assertAllClose(self.evaluate(decayed_lr(step)), expected, 1e-6)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testBeyondEndWithCycle(self, serialize):
+    step = 15
+    lr = 0.05
+    end_lr = 0.001
+    power = 0.5
+    decayed_lr = learning_rate_schedule.PolynomialDecay(
+        lr, 10, end_lr, power=power, cycle=True)
+    decayed_lr = _maybe_serialized(decayed_lr, serialize)
+    expected = (lr - end_lr) * 0.25**power + end_lr
+    self.assertAllClose(self.evaluate(decayed_lr(step)), expected, 1e-6)
+
+
+@parameterized.named_parameters(
+    ("NotSerialized", False),
+    ("Serialized", True))
+class PolynomialDecayTestV2(test_util.TensorFlowTestCase,
+                            parameterized.TestCase):
+
+  @test_util.run_in_graph_and_eager_modes
+  def testBeginWithCycle(self, serialize):
+    lr = 0.001
+    decay_steps = 10
+    step = 0
+    decayed_lr = learning_rate_schedule.PolynomialDecay(
+        lr, decay_steps, cycle=True)
+    decayed_lr = _maybe_serialized(decayed_lr, serialize)
+    expected = lr
+    self.assertAllClose(self.evaluate(decayed_lr(step)), expected, 1e-6)
+
+
+@parameterized.named_parameters(
+    ("NotSerialized", False),
+    ("Serialized", True))
+class InverseDecayTestV2(test_util.TensorFlowTestCase, parameterized.TestCase):
+
+  @test_util.run_in_graph_and_eager_modes
+  def testDecay(self, serialize):
+    initial_lr = 0.1
+    k = 10
+    decay_rate = 0.96
+    step = resource_variable_ops.ResourceVariable(0)
+    decayed_lr = learning_rate_schedule.InverseTimeDecay(initial_lr, k,
+                                                         decay_rate)
+    decayed_lr = _maybe_serialized(decayed_lr, serialize)
+
+    self.evaluate(variables.global_variables_initializer())
+    for i in range(k + 1):
+      expected = initial_lr / (1 + i / k * decay_rate)
+      self.assertAllClose(self.evaluate(decayed_lr(step)), expected, 1e-6)
+      self.evaluate(step.assign_add(1))
+
+  @test_util.run_in_graph_and_eager_modes
+  def testStaircase(self, serialize):
+    initial_lr = 0.1
+    k = 10
+    decay_rate = 0.96
+    step = resource_variable_ops.ResourceVariable(0)
+    decayed_lr = learning_rate_schedule.InverseTimeDecay(
+        initial_lr, k, decay_rate, staircase=True)
+    decayed_lr = _maybe_serialized(decayed_lr, serialize)
+
+    self.evaluate(variables.global_variables_initializer())
+    for i in range(k + 1):
+      expected = initial_lr / (1 + decay_rate * (i // k))
+      self.assertAllClose(self.evaluate(decayed_lr(step)), expected, 1e-6)
+      self.evaluate(step.assign_add(1))
+
+
+@parameterized.named_parameters(
+    ("NotSerialized", False),
+    ("Serialized", True))
+class CosineDecayTestV2(test_util.TensorFlowTestCase, parameterized.TestCase):
+
+  def np_cosine_decay(self, step, decay_steps, alpha=0.0):
+    step = min(step, decay_steps)
+    completed_fraction = step / decay_steps
+    decay = 0.5 * (1.0 + math.cos(math.pi * completed_fraction))
+    return (1.0 - alpha) * decay + alpha
+
+  @test_util.run_in_graph_and_eager_modes
+  def testDecay(self, serialize):
+    num_training_steps = 1000
+    initial_lr = 1.0
+    for step in range(0, 1500, 250):
+      decayed_lr = learning_rate_schedule.CosineDecay(initial_lr,
+                                                      num_training_steps)
+      decayed_lr = _maybe_serialized(decayed_lr, serialize)
+      expected = self.np_cosine_decay(step, num_training_steps)
+      self.assertAllClose(self.evaluate(decayed_lr(step)), expected, 1e-6)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testAlpha(self, serialize):
+    num_training_steps = 1000
+    initial_lr = 1.0
+    alpha = 0.1
+    for step in range(0, 1500, 250):
+      decayed_lr = learning_rate_schedule.CosineDecay(initial_lr,
+                                                      num_training_steps,
+                                                      alpha)
+      decayed_lr = _maybe_serialized(decayed_lr, serialize)
+      expected = self.np_cosine_decay(step, num_training_steps, alpha)
+      self.assertAllClose(self.evaluate(decayed_lr(step)), expected, 1e-6)
+
+
+@parameterized.named_parameters(
+    ("NotSerialized", False),
+    ("Serialized", True))
+class CosineDecayRestartsTestV2(test_util.TensorFlowTestCase,
+                                parameterized.TestCase):
+
+  def np_cosine_decay_restarts(self, step, decay_steps, t_mul=2.0, m_mul=1.0,
+                               alpha=0.0):
+    fac = 1.0
+    while step >= decay_steps:
+      step -= decay_steps
+      decay_steps *= t_mul
+      fac *= m_mul
+
+    completed_fraction = step / decay_steps
+    decay = fac * 0.5 * (1.0 + math.cos(math.pi * completed_fraction))
+    return (1.0 - alpha) * decay + alpha
+
+  @test_util.run_in_graph_and_eager_modes
+  def testDecay(self, serialize):
+    num_training_steps = 1000
+    initial_lr = 1.0
+    for step in range(0, 1500, 250):
+      decayed_lr = learning_rate_schedule.CosineDecayRestarts(
+          initial_lr, num_training_steps)
+      decayed_lr = _maybe_serialized(decayed_lr, serialize)
+      expected = self.np_cosine_decay_restarts(step, num_training_steps)
+      self.assertAllClose(self.evaluate(decayed_lr(step)), expected, 1e-6)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testAlpha(self, serialize):
+    num_training_steps = 1000
+    initial_lr = 1.0
+    alpha = 0.1
+    for step in range(0, 1500, 250):
+      decayed_lr = learning_rate_schedule.CosineDecayRestarts(
+          initial_lr, num_training_steps, alpha=alpha)
+      decayed_lr = _maybe_serialized(decayed_lr, serialize)
+      expected = self.np_cosine_decay_restarts(
+          step, num_training_steps, alpha=alpha)
+      self.assertAllClose(self.evaluate(decayed_lr(step)), expected, 1e-6)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testMMul(self, serialize):
+    num_training_steps = 1000
+    initial_lr = 1.0
+    m_mul = 0.9
+    for step in range(0, 1500, 250):
+      decayed_lr = learning_rate_schedule.CosineDecayRestarts(
+          initial_lr, num_training_steps, m_mul=m_mul)
+      decayed_lr = _maybe_serialized(decayed_lr, serialize)
+      expected = self.np_cosine_decay_restarts(
+          step, num_training_steps, m_mul=m_mul)
+      self.assertAllClose(self.evaluate(decayed_lr(step)), expected, 1e-6)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testTMul(self, serialize):
+    num_training_steps = 1000
+    initial_lr = 1.0
+    t_mul = 1.0
+    for step in range(0, 1500, 250):
+      decayed_lr = learning_rate_schedule.CosineDecayRestarts(
+          initial_lr, num_training_steps, t_mul=t_mul)
+      decayed_lr = _maybe_serialized(decayed_lr, serialize)
+      expected = self.np_cosine_decay_restarts(
+          step, num_training_steps, t_mul=t_mul)
+      self.assertAllClose(self.evaluate(decayed_lr(step)), expected, 1e-6)
+
+
+@parameterized.named_parameters(
+    ("NotSerialized", False),
+    ("Serialized", True))
+class LinearCosineDecayTestV2(test_util.TensorFlowTestCase,
+                              parameterized.TestCase):
+
+  def np_linear_cosine_decay(self,
+                             step,
+                             decay_steps,
+                             alpha=0.0,
+                             beta=0.001,
+                             num_periods=0.5):
+    step = min(step, decay_steps)
+    linear_decayed = float(decay_steps - step) / decay_steps
+    fraction = 2.0 * num_periods * step / float(decay_steps)
+    cosine_decayed = 0.5 * (1.0 + math.cos(math.pi * fraction))
+    return (alpha + linear_decayed) * cosine_decayed + beta
+
+  @test_util.run_in_graph_and_eager_modes
+  def testDefaultDecay(self, serialize):
+    num_training_steps = 1000
+    initial_lr = 1.0
+    for step in range(0, 1500, 250):
+      decayed_lr = learning_rate_schedule.LinearCosineDecay(
+          initial_lr, num_training_steps)
+      decayed_lr = _maybe_serialized(decayed_lr, serialize)
+      expected = self.np_linear_cosine_decay(step, num_training_steps)
+      self.assertAllClose(self.evaluate(decayed_lr(step)), expected, 1e-6)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testNonDefaultDecay(self, serialize):
+    num_training_steps = 1000
+    initial_lr = 1.0
+    for step in range(0, 1500, 250):
+      decayed_lr = learning_rate_schedule.LinearCosineDecay(
+          initial_lr,
+          num_training_steps,
+          alpha=0.1,
+          beta=1e-4,
+          num_periods=5)
+      decayed_lr = _maybe_serialized(decayed_lr, serialize)
+      expected = self.np_linear_cosine_decay(
+          step, num_training_steps, alpha=0.1, beta=1e-4, num_periods=5)
+      self.assertAllClose(self.evaluate(decayed_lr(step)), expected, 1e-6)
+
+
+@parameterized.named_parameters(
+    ("NotSerialized", False),
+    ("Serialized", True))
+class NoisyLinearCosineDecayTestV2(test_util.TensorFlowTestCase,
+                                   parameterized.TestCase):
+
+  @test_util.run_in_graph_and_eager_modes
+  def testDefaultNoisyLinearCosine(self, serialize):
+    num_training_steps = 1000
+    initial_lr = 1.0
+    for step in range(0, 1500, 250):
+      # No numerical check because of noise
+      decayed_lr = learning_rate_schedule.NoisyLinearCosineDecay(
+          initial_lr, num_training_steps)
+      decayed_lr = _maybe_serialized(decayed_lr, serialize)
+      # Cannot be deterministically tested
+      self.evaluate(decayed_lr(step))
+
+  @test_util.run_in_graph_and_eager_modes
+  def testNonDefaultNoisyLinearCosine(self, serialize):
+    num_training_steps = 1000
+    initial_lr = 1.0
+    for step in range(0, 1500, 250):
+      # No numerical check because of noise
+      decayed_lr = learning_rate_schedule.NoisyLinearCosineDecay(
+          initial_lr,
+          num_training_steps,
+          initial_variance=0.5,
+          variance_decay=0.1,
+          alpha=0.1,
+          beta=1e-4,
+          num_periods=5)
+      decayed_lr = _maybe_serialized(decayed_lr, serialize)
+      # Cannot be deterministically tested
+      self.evaluate(decayed_lr(step))
+
+if __name__ == "__main__":
+  googletest.main()
diff --git a/tensorflow/python/keras/optimizer_v2/nadam.py b/tensorflow/python/keras/optimizer_v2/nadam.py
index d515f98725..77a897124b 100644
--- a/tensorflow/python/keras/optimizer_v2/nadam.py
+++ b/tensorflow/python/keras/optimizer_v2/nadam.py
@@ -19,6 +19,7 @@ from __future__ import print_function
 
 from tensorflow.python.framework import ops
 from tensorflow.python.keras import backend_config
+from tensorflow.python.keras.optimizer_v2 import learning_rate_schedule
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
@@ -86,6 +87,12 @@ class Nadam(optimizer_v2.OptimizerV2):
 
     # Backwards compatiblity with keras NAdam optimizer.
     kwargs['decay'] = kwargs.pop('schedule_decay', 0.004)
+    learning_rate = kwargs.get('lr', learning_rate)
+    if isinstance(learning_rate, learning_rate_schedule.LearningRateSchedule):
+      raise ValueError('The Nadam optimizer does not support '
+                       'tf.keras.optimizers.LearningRateSchedules as the '
+                       'learning rate.')
+
     if epsilon is None:
       epsilon = backend_config.epsilon()
     super(Nadam, self).__init__(name, **kwargs)
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
index 9a62d16855..bf6dcaad12 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
@@ -36,6 +36,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.keras import backend
 from tensorflow.python.keras import initializers
 from tensorflow.python.keras.engine import base_layer_utils
+from tensorflow.python.keras.optimizer_v2 import learning_rate_schedule
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import clip_ops
 from tensorflow.python.ops import gradients
@@ -452,8 +453,11 @@ class OptimizerV2(checkpointable.Checkpointable):
       self._hyper[name] = value
     else:
       prev_value = self._hyper[name]
-      if callable(prev_value) or isinstance(prev_value,
-                                            (ops.Tensor, int, float)):
+      if (callable(prev_value)
+          or isinstance(prev_value,
+                        (ops.Tensor, int, float,
+                         learning_rate_schedule.LearningRateSchedule))
+          or isinstance(value, learning_rate_schedule.LearningRateSchedule)):
         self._hyper[name] = value
       else:
         backend.set_value(self._hyper[name], value)
@@ -462,6 +466,8 @@ class OptimizerV2(checkpointable.Checkpointable):
     if not self._hypers_created:
       self._create_hypers()
     value = self._hyper[name]
+    if isinstance(value, learning_rate_schedule.LearningRateSchedule):
+      return value
     if callable(value):
       value = value()
     if dtype:
@@ -575,6 +581,9 @@ class OptimizerV2(checkpointable.Checkpointable):
   def _decayed_lr(self, var_dtype):
     """Get decayed learning rate as a Tensor with dtype=var_dtype."""
     lr_t = self._get_hyper("learning_rate", var_dtype)
+    if isinstance(lr_t, learning_rate_schedule.LearningRateSchedule):
+      local_step = math_ops.cast(self.iterations, var_dtype)
+      lr_t = math_ops.cast(lr_t(local_step), var_dtype)
     if self._initial_decay > 0.:
       local_step = math_ops.cast(self.iterations, var_dtype)
       decay_t = self._get_hyper("decay", var_dtype)
@@ -619,11 +628,17 @@ class OptimizerV2(checkpointable.Checkpointable):
     """
     if "lr" in config:
       config["learning_rate"] = config.pop("lr")
+    if "learning_rate" in config:
+      if isinstance(config["learning_rate"], dict):
+        config["learning_rate"] = learning_rate_schedule.deserialize(
+            config["learning_rate"])
     return cls(**config)
 
   def _serialize_hyperparameter(self, hyperparameter_name):
     """Serialize a hyperparameter that can be a float, callable, or Tensor."""
     value = self._hyper[hyperparameter_name]
+    if isinstance(value, learning_rate_schedule.LearningRateSchedule):
+      return learning_rate_schedule.serialize(value)
     if callable(value):
       return value()
     if isinstance(value, (ops.Tensor, tf_variables.Variable,
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
index 57bd9439ed..2d4c182716 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
@@ -41,6 +41,7 @@ from tensorflow.python.keras.optimizer_v2 import adagrad
 from tensorflow.python.keras.optimizer_v2 import adam
 from tensorflow.python.keras.optimizer_v2 import adamax
 from tensorflow.python.keras.optimizer_v2 import gradient_descent
+from tensorflow.python.keras.optimizer_v2 import learning_rate_schedule
 from tensorflow.python.keras.optimizer_v2 import nadam
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
 from tensorflow.python.keras.optimizer_v2 import rmsprop
@@ -113,6 +114,13 @@ class OptimizerTest(test.TestCase):
       # var1 = [0., 1.] - 0.5 * [3, 3]
       self.assertAllClose([-1.5, -0.5], self.evaluate(var1))
 
+      sgd.learning_rate = learning_rate_schedule.InverseTimeDecay(
+          0.5, decay_steps=1.0, decay_rate=0.5)
+      if context.executing_eagerly():
+        sgd.minimize(loss, [var0, var1])
+      else:
+        self.evaluate(opt_op)
+
   @test_util.run_in_graph_and_eager_modes
   def testPrecomputedGradient(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
@@ -281,6 +289,33 @@ class OptimizerTest(test.TestCase):
       self.evaluate(variables.global_variables_initializer())
       self.assertEqual(self.evaluate(lr), self.evaluate(lr3))
 
+  @test_util.run_in_graph_and_eager_modes
+  def testConfigWithLearningRateDecay(self):
+    with self.cached_session():
+      decay_schedule = learning_rate_schedule.InverseTimeDecay(
+          0.5, decay_steps=1.0, decay_rate=0.1)
+      step = 10
+      opt = gradient_descent.SGD(decay_schedule)
+      config = opt.get_config()
+      opt2 = gradient_descent.SGD.from_config(config)
+      # assert both are equal float values.
+      self.assertAllEqual(
+          decay_schedule(step),
+          opt._get_hyper('learning_rate')(step))
+      self.assertAllEqual(
+          decay_schedule(step),
+          opt2._get_hyper('learning_rate')(step))
+      var0 = variables.Variable([[1.0], [2.0]], dtype=dtypes.float32)
+      loss = lambda: 3 * var0
+      # learning rate variable created when calling minimize.
+      opt.minimize(loss, [var0])
+      self.evaluate(variables.global_variables_initializer())
+      config = opt.get_config()
+      opt3 = gradient_descent.SGD.from_config(config)
+      self.assertAllEqual(
+          self.evaluate(opt._get_hyper('learning_rate')(step)),
+          opt3._get_hyper('learning_rate')(step))
+
   @test_util.run_in_graph_and_eager_modes
   def testGradClipValue(self):
     with self.cached_session():
diff --git a/tensorflow/python/keras/optimizer_v2/rmsprop_test.py b/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
index a9ddc2155a..ab80526676 100644
--- a/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
+++ b/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
@@ -29,6 +29,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
+from tensorflow.python.keras.optimizer_v2 import learning_rate_schedule
 from tensorflow.python.keras.optimizer_v2 import rmsprop
 from tensorflow.python.ops import embedding_ops
 from tensorflow.python.ops import math_ops
@@ -244,6 +245,78 @@ class RMSpropOptimizerTest(test.TestCase):
       self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
       self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
+  def testDenseWithLearningRateInverseTimeDecay(self):
+    var0_np = np.array([1.0, 2.0])
+    grads0_np = np.array([0.1, 0.2])
+    var1_np = np.array([3.0, 4.0])
+    grads1_np = np.array([0.01, 0.2])
+
+    var0 = resource_variable_ops.ResourceVariable(var0_np)
+    var1 = resource_variable_ops.ResourceVariable(var1_np)
+    grads0 = constant_op.constant(grads0_np)
+    grads1 = constant_op.constant(grads1_np)
+    learning_rate = 0.01
+    rho = 0.9
+    momentum = 0.0
+    epsilon = 1e-7
+    centered = False
+    decay = 0.5
+    lr_schedule = learning_rate_schedule.InverseTimeDecay(
+        learning_rate, decay_steps=1.0, decay_rate=decay)
+    opt = rmsprop.RMSprop(
+        learning_rate=lr_schedule,
+        rho=rho,
+        momentum=momentum,
+        epsilon=epsilon,
+        centered=centered)
+
+    update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+    self.evaluate(variables.global_variables_initializer())
+
+    rms0 = opt.get_slot(var0, "rms")
+    self.assertTrue(rms0 is not None)
+    rms1 = opt.get_slot(var1, "rms")
+    self.assertTrue(rms1 is not None)
+    if momentum > 0.:
+      mom0 = opt.get_slot(var0, "momentum")
+      mom1 = opt.get_slot(var1, "momentum")
+    else:
+      mom0 = None
+      mom1 = None
+
+    mg0_np = np.array([0.0, 0.0])
+    mg1_np = np.array([0.0, 0.0])
+    rms0_np = np.array([0.0, 0.0])
+    rms1_np = np.array([0.0, 0.0])
+    mom0_np = np.array([0.0, 0.0])
+    mom1_np = np.array([0.0, 0.0])
+
+    # Fetch params to validate initial values
+    self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+    self.assertAllClose([3.0, 4.0], self.evaluate(var1))
+
+    # Run 4 steps of RMSprop
+    for t in range(2):
+      self.evaluate(update)
+
+      lr = learning_rate / (1 + decay * t)
+      var0_np, mg0_np, rms0_np, mom0_np = self._rmsprop_update_numpy(
+          var0_np, grads0_np, mg0_np, rms0_np, mom0_np, lr, rho, momentum,
+          epsilon, centered)
+      var1_np, mg1_np, rms1_np, mom1_np = self._rmsprop_update_numpy(
+          var1_np, grads1_np, mg1_np, rms1_np, mom1_np, lr, rho, momentum,
+          epsilon, centered)
+
+      # Validate updated params
+      self.assertAllCloseAccordingToType(rms0_np, self.evaluate(rms0))
+      self.assertAllCloseAccordingToType(rms1_np, self.evaluate(rms1))
+      if momentum > 0.:
+        self.assertAllCloseAccordingToType(mom0_np, self.evaluate(mom0))
+        self.assertAllCloseAccordingToType(mom1_np, self.evaluate(mom1))
+      self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+      self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
+
   @test_util.run_deprecated_v1
   def testMinimizeSparseResourceVariable(self):
     for dtype in [dtypes.float32, dtypes.float64]:
diff --git a/tensorflow/python/tools/api/generator/api_init_files.bzl b/tensorflow/python/tools/api/generator/api_init_files.bzl
index d18776eab7..245380287a 100644
--- a/tensorflow/python/tools/api/generator/api_init_files.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files.bzl
@@ -84,6 +84,7 @@ KERAS_API_INIT_FILES = [
     "keras/metrics/__init__.py",
     "keras/models/__init__.py",
     "keras/optimizers/__init__.py",
+    "keras/optimizers/schedules/__init__.py",
     "keras/preprocessing/__init__.py",
     "keras/preprocessing/image/__init__.py",
     "keras/preprocessing/sequence/__init__.py",
diff --git a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
index 5213d5eed8..c8c7c51b40 100644
--- a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
@@ -107,6 +107,7 @@ KERAS_API_INIT_FILES_V1 = [
     "keras/metrics/__init__.py",
     "keras/models/__init__.py",
     "keras/optimizers/__init__.py",
+    "keras/optimizers/schedules/__init__.py",
     "keras/preprocessing/__init__.py",
     "keras/preprocessing/image/__init__.py",
     "keras/preprocessing/sequence/__init__.py",
diff --git a/tensorflow/python/training/learning_rate_decay.py b/tensorflow/python/training/learning_rate_decay.py
index c52e89db1f..ab9d923bed 100644
--- a/tensorflow/python/training/learning_rate_decay.py
+++ b/tensorflow/python/training/learning_rate_decay.py
@@ -17,8 +17,11 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import functools
+
 from tensorflow.python.eager import context
-from tensorflow.python.training import learning_rate_decay_v2
+from tensorflow.python.keras.optimizer_v2 import learning_rate_schedule
+from tensorflow.python.ops import math_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -88,15 +91,15 @@ def exponential_decay(learning_rate,
   the learning rate value across different invocations of optimizer functions.
   @end_compatibility
   """
-  decayed_lr = learning_rate_decay_v2.exponential_decay(learning_rate,
-                                                        global_step,
-                                                        decay_steps,
-                                                        decay_rate,
-                                                        staircase=staircase,
-                                                        name=name)
+  decayed_lr = learning_rate_schedule.ExponentialDecay(learning_rate,
+                                                       decay_steps,
+                                                       decay_rate,
+                                                       staircase=staircase,
+                                                       name=name)
   if not context.executing_eagerly():
-    decayed_lr = decayed_lr()
-
+    decayed_lr = decayed_lr(global_step)
+  else:
+    decayed_lr = functools.partial(decayed_lr, global_step)
   return decayed_lr
 
 
@@ -143,11 +146,12 @@ def piecewise_constant(x, boundaries, values, name=None):
   the learning rate value across different invocations of optimizer functions.
   @end_compatibility
   """
-  decayed_lr = learning_rate_decay_v2.piecewise_constant(x, boundaries, values,
-                                                         name=name)
+  decayed_lr = learning_rate_schedule.PiecewiseConstantDecay(
+      boundaries, values, name=name)
   if not context.executing_eagerly():
-    decayed_lr = decayed_lr()
-
+    decayed_lr = decayed_lr(x)
+  else:
+    decayed_lr = functools.partial(decayed_lr, x)
   return decayed_lr
 
 
@@ -236,9 +240,8 @@ def polynomial_decay(learning_rate,
   the learning rate value across different invocations of optimizer functions.
   @end_compatibility
   """
-  decayed_lr = learning_rate_decay_v2.polynomial_decay(
+  decayed_lr = learning_rate_schedule.PolynomialDecay(
       learning_rate,
-      global_step,
       decay_steps,
       end_learning_rate=end_learning_rate,
       power=power,
@@ -246,8 +249,9 @@ def polynomial_decay(learning_rate,
       name=name)
 
   if not context.executing_eagerly():
-    decayed_lr = decayed_lr()
-
+    decayed_lr = decayed_lr(global_step)
+  else:
+    decayed_lr = functools.partial(decayed_lr, global_step)
   return decayed_lr
 
 
@@ -323,13 +327,15 @@ def natural_exp_decay(learning_rate,
   the learning rate value across different invocations of optimizer functions.
   @end_compatibility
   """
-  decayed_lr = learning_rate_decay_v2.natural_exp_decay(
-      learning_rate, global_step, decay_steps, decay_rate, staircase=staircase,
+  natural_exp_rate = math_ops.exp(math_ops.negative(decay_rate))
+  decayed_lr = learning_rate_schedule.ExponentialDecay(
+      learning_rate, decay_steps, natural_exp_rate, staircase=staircase,
       name=name)
 
   if not context.executing_eagerly():
-    decayed_lr = decayed_lr()
-
+    decayed_lr = decayed_lr(global_step)
+  else:
+    decayed_lr = functools.partial(decayed_lr, global_step)
   return decayed_lr
 
 
@@ -405,17 +411,17 @@ def inverse_time_decay(learning_rate,
   the learning rate value across different invocations of optimizer functions.
   @end_compatibility
   """
-  decayed_lr = learning_rate_decay_v2.inverse_time_decay(
+  decayed_lr = learning_rate_schedule.InverseTimeDecay(
       learning_rate,
-      global_step,
       decay_steps,
       decay_rate,
       staircase=staircase,
       name=name)
 
   if not context.executing_eagerly():
-    decayed_lr = decayed_lr()
-
+    decayed_lr = decayed_lr(global_step)
+  else:
+    decayed_lr = functools.partial(decayed_lr, global_step)
   return decayed_lr
 
 
@@ -468,12 +474,13 @@ def cosine_decay(learning_rate, global_step, decay_steps, alpha=0.0, name=None):
   the learning rate value across different invocations of optimizer functions.
   @end_compatibility
   """
-  decayed_lr = learning_rate_decay_v2.cosine_decay(
-      learning_rate, global_step, decay_steps, alpha=alpha, name=name)
+  decayed_lr = learning_rate_schedule.CosineDecay(
+      learning_rate, decay_steps, alpha=alpha, name=name)
 
   if not context.executing_eagerly():
-    decayed_lr = decayed_lr()
-
+    decayed_lr = decayed_lr(global_step)
+  else:
+    decayed_lr = functools.partial(decayed_lr, global_step)
   return decayed_lr
 
 
@@ -535,9 +542,8 @@ def cosine_decay_restarts(learning_rate,
   the learning rate value across different invocations of optimizer functions.
   @end_compatibility
   """
-  decayed_lr = learning_rate_decay_v2.cosine_decay_restarts(
+  decayed_lr = learning_rate_schedule.CosineDecayRestarts(
       learning_rate,
-      global_step,
       first_decay_steps,
       t_mul=t_mul,
       m_mul=m_mul,
@@ -545,8 +551,9 @@ def cosine_decay_restarts(learning_rate,
       name=name)
 
   if not context.executing_eagerly():
-    decayed_lr = decayed_lr()
-
+    decayed_lr = decayed_lr(global_step)
+  else:
+    decayed_lr = functools.partial(decayed_lr, global_step)
   return decayed_lr
 
 
@@ -617,9 +624,8 @@ def linear_cosine_decay(learning_rate,
   the learning rate value across different invocations of optimizer functions.
   @end_compatibility
   """
-  decayed_lr = learning_rate_decay_v2.linear_cosine_decay(
+  decayed_lr = learning_rate_schedule.LinearCosineDecay(
       learning_rate,
-      global_step,
       decay_steps,
       num_periods=num_periods,
       alpha=alpha,
@@ -627,8 +633,9 @@ def linear_cosine_decay(learning_rate,
       name=name)
 
   if not context.executing_eagerly():
-    decayed_lr = decayed_lr()
-
+    decayed_lr = decayed_lr(global_step)
+  else:
+    decayed_lr = functools.partial(decayed_lr, global_step)
   return decayed_lr
 
 
@@ -707,8 +714,8 @@ def noisy_linear_cosine_decay(learning_rate,
   the learning rate value across different invocations of optimizer functions.
   @end_compatibility
   """
-  decayed_lr = learning_rate_decay_v2.noisy_linear_cosine_decay(
-      learning_rate, global_step,
+  decayed_lr = learning_rate_schedule.NoisyLinearCosineDecay(
+      learning_rate,
       decay_steps,
       initial_variance=initial_variance,
       variance_decay=variance_decay,
@@ -718,6 +725,7 @@ def noisy_linear_cosine_decay(learning_rate,
       name=name)
 
   if not context.executing_eagerly():
-    decayed_lr = decayed_lr()
-
+    decayed_lr = decayed_lr(global_step)
+  else:
+    decayed_lr = functools.partial(decayed_lr, global_step)
   return decayed_lr
diff --git a/tensorflow/python/training/learning_rate_decay_v2.py b/tensorflow/python/training/learning_rate_decay_v2.py
deleted file mode 100644
index eb69feb17d..0000000000
--- a/tensorflow/python/training/learning_rate_decay_v2.py
+++ /dev/null
@@ -1,898 +0,0 @@
-# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Various learning rate decay functions."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import functools
-import math
-
-from tensorflow.python.framework import constant_op
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops
-from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import random_ops
-from tensorflow.python.util.tf_export import tf_export
-
-
-@tf_export("train.exponential_decay", v1=[])
-def exponential_decay(learning_rate,
-                      global_step,
-                      decay_steps,
-                      decay_rate,
-                      staircase=False,
-                      name=None):
-  """Applies exponential decay to the learning rate.
-
-  When training a model, it is often recommended to lower the learning rate as
-  the training progresses.  This function applies an exponential decay function
-  to a provided initial learning rate.  It requires a `global_step` value to
-  compute the decayed learning rate.  You can just pass a TensorFlow variable
-  that you increment at each training step.
-
-  The function returns a no-arg function that produces the decayed learning
-  rate. This can be useful for changing the learning rate value across
-  different invocations of optimizer functions.
-  It is computed as:
-
-  ```python
-  decayed_learning_rate = learning_rate *
-                          decay_rate ^ (global_step / decay_steps)
-  ```
-
-  If the argument `staircase` is `True`, then `global_step / decay_steps` is an
-  integer division and the decayed learning rate follows a staircase function.
-
-  Example: decay every 100000 steps with a base of 0.96:
-
-  ```python
-  ...
-  global_step = tf.Variable(0, trainable=False)
-  starter_learning_rate = 0.1
-  learning_rate_fn = tf.train.exponential_decay(starter_learning_rate,
-                                                global_step, 100000, 0.96,
-                                                staircase=True)
-  # Passing global_step to minimize() will increment it at each step.
-  learning_step = (
-      tf.train.GradientDescentOptimizer(learning_rate_fn)
-      .minimize(...my loss..., global_step=global_step)
-  )
-  ```
-
-  Args:
-    learning_rate: A scalar `float32` or `float64` `Tensor` or a
-      Python number.  The initial learning rate.
-    global_step: A scalar `int32` or `int64` `Tensor` or a Python number.
-      Global step to use for the decay computation.  Must not be negative.
-    decay_steps: A scalar `int32` or `int64` `Tensor` or a Python number.
-      Must be positive.  See the decay computation above.
-    decay_rate: A scalar `float32` or `float64` `Tensor` or a
-      Python number.  The decay rate.
-    staircase: Boolean.  If `True` decay the learning rate at discrete intervals
-    name: String.  Optional name of the operation.  Defaults to
-      'ExponentialDecay'.
-
-  Returns:
-    A no-arg function that outputs the decayed learning rate, a scalar `Tensor`
-    of the same type as `learning_rate`.
-
-  Raises:
-    ValueError: if `global_step` is not supplied.
-  """
-  if global_step is None:
-    raise ValueError("global_step is required for exponential_decay.")
-  def decayed_lr(learning_rate, global_step, decay_steps, decay_rate,
-                 staircase, name):
-    """Helper to recompute learning rate; most helpful in eager-mode."""
-    with ops.name_scope(
-        name, "ExponentialDecay",
-        [learning_rate, global_step, decay_steps, decay_rate]) as name:
-      learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate")
-      dtype = learning_rate.dtype
-      decay_steps = math_ops.cast(decay_steps, dtype)
-      decay_rate = math_ops.cast(decay_rate, dtype)
-
-      global_step_recomp = math_ops.cast(global_step, dtype)
-      p = global_step_recomp / decay_steps
-      if staircase:
-        p = math_ops.floor(p)
-      return math_ops.multiply(
-          learning_rate, math_ops.pow(decay_rate, p), name=name)
-
-  return functools.partial(decayed_lr, learning_rate, global_step, decay_steps,
-                           decay_rate, staircase, name)
-
-
-@tf_export("train.piecewise_constant_decay", v1=[])
-def piecewise_constant(x, boundaries, values, name=None):
-  """Piecewise constant from boundaries and interval values.
-
-  This function returns a no-arg callable to compute the piecewise constant.
-  This can be useful for changing the learning rate value across
-  different invocations of optimizer functions.
-
-  Example: use a learning rate that's 1.0 for the first 100001 steps, 0.5
-    for the next 10000 steps, and 0.1 for any additional steps.
-
-  ```python
-  global_step = tf.Variable(0, trainable=False)
-  boundaries = [100000, 110000]
-  values = [1.0, 0.5, 0.1]
-  learning_rate_fn = tf.train.piecewise_constant(global_step, boundaries,
-    values)
-  learning_rate = learning_rate_fn()
-
-  # Later, whenever we perform an optimization step, we increment global_step.
-  ```
-
-  Args:
-    x: A 0-D scalar `Tensor`. Must be one of the following types: `float32`,
-      `float64`, `uint8`, `int8`, `int16`, `int32`, `int64`.
-    boundaries: A list of `Tensor`s or `int`s or `float`s with strictly
-      increasing entries, and with all elements having the same type as `x`.
-    values: A list of `Tensor`s or `float`s or `int`s that specifies the values
-      for the intervals defined by `boundaries`. It should have one more element
-      than `boundaries`, and all elements should have the same type.
-    name: A string. Optional name of the operation. Defaults to
-      'PiecewiseConstant'.
-
-  Returns:
-    A no-arg function that outputs a 0-D Tensor. The output of the no-arg
-    function is `values[0]` when `x <= boundaries[0]`,
-    `values[1]` when `x > boundaries[0]` and `x <= boundaries[1]`, ...,
-    and values[-1] when `x > boundaries[-1]`.
-
-  Raises:
-    ValueError: if types of `x` and `boundaries` do not match, or types of all
-        `values` do not match or
-        the number of elements in the lists does not match.
-  """
-  if len(boundaries) != len(values) - 1:
-    raise ValueError(
-        "The length of boundaries should be 1 less than the length of values")
-  def decayed_lr(x, boundaries, values, name):
-    """Helper to recompute learning rate; most helpful in eager-mode."""
-    with ops.name_scope(name, "PiecewiseConstant",
-                        [x, boundaries, values, name]) as name:
-      boundaries = ops.convert_n_to_tensor(boundaries)
-      values = ops.convert_n_to_tensor(values)
-      x_recomp = ops.convert_to_tensor(x)
-      # Avoid explicit conversion to x's dtype. This could result in faulty
-      # comparisons, for example if floats are converted to integers.
-      for i, b in enumerate(boundaries):
-        if b.dtype.base_dtype != x_recomp.dtype.base_dtype:
-          # We can promote int32 boundaries to int64 without loss of precision.
-          # This covers the most common case where the user passes in boundaries
-          # as an array of Python integers.
-          if (b.dtype.base_dtype == dtypes.int32 and
-              x_recomp.dtype.base_dtype == dtypes.int64):
-            b = math_ops.cast(b, x_recomp.dtype.base_dtype)
-            boundaries[i] = b
-          else:
-            raise ValueError(
-                "Boundaries (%s) must have the same dtype as x (%s)." %
-                (b.dtype.base_dtype, x_recomp.dtype.base_dtype))
-      # TODO(rdipietro): Ensure that boundaries' elements strictly increases.
-      for v in values[1:]:
-        if v.dtype.base_dtype != values[0].dtype.base_dtype:
-          raise ValueError(
-              "Values must have elements all with the same dtype (%s vs %s)." %
-              (values[0].dtype.base_dtype, v.dtype.base_dtype))
-      pred_fn_pairs = []
-      pred_fn_pairs.append((x_recomp <= boundaries[0], lambda: values[0]))
-      pred_fn_pairs.append((x_recomp > boundaries[-1], lambda: values[-1]))
-      for low, high, v in zip(boundaries[:-1], boundaries[1:], values[1:-1]):
-        # Need to bind v here; can do this with lambda v=v: ...
-        pred = (x_recomp > low) & (x_recomp <= high)
-        pred_fn_pairs.append((pred, lambda v=v: v))
-
-      # The default isn't needed here because our conditions are mutually
-      # exclusive and exhaustive, but tf.case requires it.
-      default = lambda: values[0]
-      return control_flow_ops.case(pred_fn_pairs, default, exclusive=True)
-
-  return functools.partial(decayed_lr, x, boundaries, values, name)
-
-
-@tf_export("train.polynomial_decay", v1=[])
-def polynomial_decay(learning_rate,
-                     global_step,
-                     decay_steps,
-                     end_learning_rate=0.0001,
-                     power=1.0,
-                     cycle=False,
-                     name=None):
-  """Applies a polynomial decay to the learning rate.
-
-  It is commonly observed that a monotonically decreasing learning rate, whose
-  degree of change is carefully chosen, results in a better performing model.
-  This function applies a polynomial decay function to a provided initial
-  `learning_rate` to reach an `end_learning_rate` in the given `decay_steps`.
-
-  It requires a `global_step` value to compute the decayed learning rate.  You
-  can just pass a TensorFlow variable that you increment at each training step.
-
-  The function returns a no-arg callable that outputs the decayed learning
-  rate. This can be useful for changing the learning rate value across
-  different invocations of optimizer functions. It is computed as:
-
-  ```python
-  global_step = min(global_step, decay_steps)
-  decayed_learning_rate = (learning_rate - end_learning_rate) *
-                          (1 - global_step / decay_steps) ^ (power) +
-                          end_learning_rate
-
-  ```
-
-  If `cycle` is True then a multiple of `decay_steps` is used, the first one
-  that is bigger than `global_steps`.
-
-  ```python
-  decay_steps = decay_steps * ceil(global_step / decay_steps)
-  decayed_learning_rate_fn = (learning_rate - end_learning_rate) *
-                          (1 - global_step / decay_steps) ^ (power) +
-                          end_learning_rate
-  decayed_learning_rate = decayed_learning_rate_fn()
-
-  ```
-
-  Example: decay from 0.1 to 0.01 in 10000 steps using sqrt (i.e. power=0.5):
-
-  ```python
-  ...
-  global_step = tf.Variable(0, trainable=False)
-  starter_learning_rate = 0.1
-  end_learning_rate = 0.01
-  decay_steps = 10000
-  learning_rate_fn = tf.train.polynomial_decay(starter_learning_rate,
-                                               global_step, decay_steps,
-                                               end_learning_rate,
-                                               power=0.5)
-  # Passing global_step to minimize() will increment it at each step.
-  learning_step = (
-      tf.train.GradientDescentOptimizer(learning_rate_fn)
-      .minimize(...my loss..., global_step=global_step)
-  )
-  ```
-
-  Args:
-    learning_rate: A scalar `float32` or `float64` `Tensor` or a
-      Python number.  The initial learning rate.
-    global_step: A scalar `int32` or `int64` `Tensor` or a Python number.
-      Global step to use for the decay computation.  Must not be negative.
-    decay_steps: A scalar `int32` or `int64` `Tensor` or a Python number.
-      Must be positive.  See the decay computation above.
-    end_learning_rate: A scalar `float32` or `float64` `Tensor` or a
-      Python number.  The minimal end learning rate.
-    power: A scalar `float32` or `float64` `Tensor` or a
-      Python number.  The power of the polynomial. Defaults to linear, 1.0.
-    cycle: A boolean, whether or not it should cycle beyond decay_steps.
-    name: String.  Optional name of the operation. Defaults to
-      'PolynomialDecay'.
-
-  Returns:
-    A no-arg function that outputs the decayed learning rate, a scalar `Tensor`
-    of the same type as `learning_rate`.
-
-  Raises:
-    ValueError: if `global_step` is not supplied.
-  """
-  if global_step is None:
-    raise ValueError("global_step is required for polynomial_decay.")
-  def decayed_lr(learning_rate, global_step, decay_steps, end_learning_rate,
-                 power, cycle, name):
-    """Helper to recompute learning rate; most helpful in eager-mode."""
-    with ops.name_scope(
-        name, "PolynomialDecay",
-        [learning_rate, global_step, decay_steps, end_learning_rate, power]
-    ) as name:
-      learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate")
-      dtype = learning_rate.dtype
-      end_learning_rate = math_ops.cast(end_learning_rate, dtype)
-      power = math_ops.cast(power, dtype)
-
-      global_step_recomp = math_ops.cast(global_step, dtype)
-      decay_steps_recomp = math_ops.cast(decay_steps, dtype)
-      if cycle:
-        # Find the first multiple of decay_steps that is bigger than
-        # global_step. If global_step is zero set the multiplier to 1
-        multiplier = control_flow_ops.cond(
-            math_ops.equal(global_step_recomp, 0), lambda: 1.0,
-            lambda: math_ops.ceil(global_step_recomp / decay_steps))
-        decay_steps_recomp = math_ops.multiply(decay_steps_recomp, multiplier)
-      else:
-        # Make sure that the global_step used is not bigger than decay_steps.
-        global_step_recomp = math_ops.minimum(global_step_recomp, decay_steps)
-
-      p = math_ops.div(global_step_recomp, decay_steps_recomp)
-      return math_ops.add(
-          math_ops.multiply(learning_rate - end_learning_rate,
-                            math_ops.pow(1 - p, power)),
-          end_learning_rate,
-          name=name)
-
-  return functools.partial(
-      decayed_lr, learning_rate, global_step, decay_steps, end_learning_rate,
-      power, cycle, name)
-
-
-@tf_export("train.natural_exp_decay", v1=[])
-def natural_exp_decay(learning_rate,
-                      global_step,
-                      decay_steps,
-                      decay_rate,
-                      staircase=False,
-                      name=None):
-  """Applies natural exponential decay to the initial learning rate.
-
-  When training a model, it is often recommended to lower the learning rate as
-  the training progresses.  This function applies an exponential decay function
-  to a provided initial learning rate.  It requires an `global_step` value to
-  compute the decayed learning rate.  You can just pass a TensorFlow variable
-  that you increment at each training step.
-
-  The function returns a no-arg callable that produces the decayed learning
-  rate. This can be useful for changing the learning rate value across
-  different invocations of optimizer functions. It is computed as:
-
-  ```python
-  decayed_learning_rate = learning_rate * exp(-decay_rate * global_step /
-  decay_step)
-  ```
-
-  or, if `staircase` is `True`, as:
-
-  ```python
-  decayed_learning_rate = learning_rate * exp(-decay_rate * floor(global_step /
-  decay_step))
-  ```
-
-  Example: decay exponentially with a base of 0.96:
-
-  ```python
-  ...
-  global_step = tf.Variable(0, trainable=False)
-  learning_rate = 0.1
-  decay_steps = 5
-  k = 0.5
-  learning_rate_fn = tf.train.natural_exp_decay(learning_rate, global_step,
-                                                decay_steps, k)
-
-  # Passing global_step to minimize() will increment it at each step.
-  learning_step = (
-      tf.train.GradientDescentOptimizer(learning_rate_fn)
-      .minimize(...my loss..., global_step=global_step)
-  )
-  ```
-
-  Args:
-    learning_rate: A scalar `float32` or `float64` `Tensor` or a
-      Python number.  The initial learning rate.
-    global_step: A Python number.
-      Global step to use for the decay computation.  Must not be negative.
-    decay_steps: How often to apply decay.
-    decay_rate: A Python number.  The decay rate.
-    staircase: Whether to apply decay in a discrete staircase, as opposed to
-      continuous, fashion.
-    name: String.  Optional name of the operation.  Defaults to
-      'ExponentialTimeDecay'.
-
-  Returns:
-    A no-arg function that outputs the decayed learning rate, a scalar `Tensor`
-    of the same type as `learning_rate`.
-
-  Raises:
-    ValueError: if `global_step` is not supplied.
-  """
-  if global_step is None:
-    raise ValueError("global_step is required for natural_exp_decay.")
-  def decayed_lr(learning_rate, global_step, decay_steps, decay_rate, staircase,
-                 name):
-    """Helper to recompute learning rate; most helpful in eager-mode."""
-    with ops.name_scope(name, "NaturalExpDecay",
-                        [learning_rate, global_step, decay_rate]) as name:
-      learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate")
-      dtype = learning_rate.dtype
-      decay_steps = math_ops.cast(decay_steps, dtype)
-      decay_rate = math_ops.cast(decay_rate, dtype)
-
-      global_step_recomp = math_ops.cast(global_step, dtype)
-      p = global_step_recomp / decay_steps
-      if staircase:
-        p = math_ops.floor(p)
-      exponent = math_ops.exp(
-          math_ops.multiply(math_ops.negative(decay_rate), p))
-      return math_ops.multiply(learning_rate, exponent, name=name)
-
-  return functools.partial(decayed_lr, learning_rate, global_step, decay_steps,
-                           decay_rate, staircase, name)
-
-
-@tf_export("train.inverse_time_decay", v1=[])
-def inverse_time_decay(learning_rate,
-                       global_step,
-                       decay_steps,
-                       decay_rate,
-                       staircase=False,
-                       name=None):
-  """Applies inverse time decay to the initial learning rate.
-
-  When training a model, it is often recommended to lower the learning rate as
-  the training progresses.  This function applies an inverse decay function
-  to a provided initial learning rate.  It requires an `global_step` value to
-  compute the decayed learning rate.  You can just pass a TensorFlow variable
-  that you increment at each training step.
-
-  The function returns a no-arg callable that produces the decayed learning
-  rate. This can be useful for changing the learning rate value across
-  different invocations of optimizer functions. It is computed as:
-
-  ```python
-  decayed_learning_rate = learning_rate / (1 + decay_rate * global_step /
-  decay_step)
-  ```
-
-  or, if `staircase` is `True`, as:
-
-  ```python
-  decayed_learning_rate = learning_rate / (1 + decay_rate * floor(global_step /
-  decay_step))
-  ```
-
-  Example: decay 1/t with a rate of 0.5:
-
-  ```python
-  ...
-  global_step = tf.Variable(0, trainable=False)
-  learning_rate = 0.1
-  decay_steps = 1.0
-  decay_rate = 0.5
-  learning_rate_fn = tf.train.inverse_time_decay(learning_rate, global_step,
-  decay_steps, decay_rate)
-
-  # Passing global_step to minimize() will increment it at each step.
-  learning_step = (
-      tf.train.GradientDescentOptimizer(learning_rate_fn)
-      .minimize(...my loss..., global_step=global_step)
-  )
-  ```
-
-  Args:
-    learning_rate: A scalar `float32` or `float64` `Tensor` or a
-      Python number.  The initial learning rate.
-    global_step: A Python number.
-      Global step to use for the decay computation.  Must not be negative.
-    decay_steps: How often to apply decay.
-    decay_rate: A Python number.  The decay rate.
-    staircase: Whether to apply decay in a discrete staircase, as opposed to
-      continuous, fashion.
-    name: String.  Optional name of the operation.  Defaults to
-      'InverseTimeDecay'.
-
-  Returns:
-    A no-arg function that outputs the decayed learning rate, a scalar `Tensor`
-    of the same type as `learning_rate`.
-
-  Raises:
-    ValueError: if `global_step` is not supplied.
-  """
-  if global_step is None:
-    raise ValueError("global_step is required for inverse_time_decay.")
-  def decayed_lr(learning_rate, global_step, decay_steps, decay_rate, staircase,
-                 name):
-    """Helper to recompute learning rate; most helpful in eager-mode."""
-    with ops.name_scope(name, "InverseTimeDecay",
-                        [learning_rate, global_step, decay_rate]) as name:
-      learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate")
-      dtype = learning_rate.dtype
-      decay_steps = math_ops.cast(decay_steps, dtype)
-      decay_rate = math_ops.cast(decay_rate, dtype)
-
-      global_step_recomp = math_ops.cast(global_step, dtype)
-      p = global_step_recomp / decay_steps
-      if staircase:
-        p = math_ops.floor(p)
-      const = math_ops.cast(constant_op.constant(1), dtype)
-      denom = math_ops.add(const, math_ops.multiply(decay_rate, p))
-      return math_ops.div(learning_rate, denom, name=name)
-
-  return functools.partial(decayed_lr, learning_rate, global_step, decay_steps,
-                           decay_rate, staircase, name)
-
-
-@tf_export("train.cosine_decay", v1=[])
-def cosine_decay(learning_rate, global_step, decay_steps, alpha=0.0,
-                 name=None):
-  """Applies cosine decay to the learning rate.
-
-  See [Loshchilov & Hutter, ICLR2016], SGDR: Stochastic Gradient Descent
-  with Warm Restarts. https://arxiv.org/abs/1608.03983
-
-  When training a model, it is often recommended to lower the learning rate as
-  the training progresses.  This function applies a cosine decay function
-  to a provided initial learning rate.  It requires a `global_step` value to
-  compute the decayed learning rate.  You can just pass a TensorFlow variable
-  that you increment at each training step.
-
-  The function returns a no-arg callable that produces the decayed learning
-  rate. This can be useful for changing the learning rate value across
-  different invocations of optimizer functions. It is computed as:
-
-  ```python
-  global_step = min(global_step, decay_steps)
-  cosine_decay = 0.5 * (1 + cos(pi * global_step / decay_steps))
-  decayed = (1 - alpha) * cosine_decay + alpha
-  decayed_learning_rate = learning_rate * decayed
-  ```
-
-  Example usage:
-  ```python
-  decay_steps = 1000
-  lr_decayed_fn = tf.train.cosine_decay(learning_rate, global_step, decay_steps)
-  ```
-
-  Args:
-    learning_rate: A scalar `float32` or `float64` Tensor or a Python number.
-      The initial learning rate.
-    global_step: A scalar `int32` or `int64` `Tensor` or a Python number.
-      Global step to use for the decay computation.
-    decay_steps: A scalar `int32` or `int64` `Tensor` or a Python number.
-      Number of steps to decay over.
-    alpha: A scalar `float32` or `float64` Tensor or a Python number.
-      Minimum learning rate value as a fraction of learning_rate.
-    name: String. Optional name of the operation.  Defaults to 'CosineDecay'.
-  Returns:
-    A no-arg function that outputs the decayed learning rate, a scalar `Tensor`
-    of the same type as `learning_rate`.
-  Raises:
-    ValueError: if `global_step` is not supplied.
-  """
-  if global_step is None:
-    raise ValueError("cosine decay requires global_step")
-  def decayed_lr(learning_rate, global_step, decay_steps, alpha, name):
-    """Helper to recompute learning rate; most helpful in eager-mode."""
-    with ops.name_scope(name, "CosineDecay",
-                        [learning_rate, global_step]) as name:
-      learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate")
-      dtype = learning_rate.dtype
-      decay_steps = math_ops.cast(decay_steps, dtype)
-
-      global_step_recomp = math_ops.cast(global_step, dtype)
-      global_step_recomp = math_ops.minimum(global_step_recomp, decay_steps)
-      completed_fraction = global_step_recomp / decay_steps
-      cosine_decayed = 0.5 * (1.0 + math_ops.cos(
-          constant_op.constant(math.pi) * completed_fraction))
-
-      decayed = (1 - alpha) * cosine_decayed + alpha
-      return math_ops.multiply(learning_rate, decayed)
-
-  return functools.partial(decayed_lr, learning_rate, global_step, decay_steps,
-                           alpha, name)
-
-
-@tf_export("train.cosine_decay_restarts", v1=[])
-def cosine_decay_restarts(learning_rate,
-                          global_step,
-                          first_decay_steps,
-                          t_mul=2.0,
-                          m_mul=1.0,
-                          alpha=0.0,
-                          name=None):
-  """Applies cosine decay with restarts to the learning rate.
-
-  See [Loshchilov & Hutter, ICLR2016], SGDR: Stochastic Gradient Descent
-  with Warm Restarts. https://arxiv.org/abs/1608.03983
-
-  When training a model, it is often recommended to lower the learning rate as
-  the training progresses.  This function applies a cosine decay function with
-  restarts to a provided initial learning rate.  It requires a `global_step`
-  value to compute the decayed learning rate.  You can just pass a TensorFlow
-  variable that you increment at each training step.
-
-  The function returns a no-arg callable that produces the decayed learning
-  rate while taking into account possible warm restarts. This can be useful for
-  changing the learning rate value across different invocations of optimizer
-  functions.
-
-  The learning rate multiplier first decays
-  from 1 to `alpha` for `first_decay_steps` steps. Then, a warm
-  restart is performed. Each new warm restart runs for `t_mul` times more steps
-  and with `m_mul` times smaller initial learning rate.
-
-  Example usage:
-  ```python
-  first_decay_steps = 1000
-  lr_decayed_fn = tf.train.cosine_decay_restarts(learning_rate, global_step,
-                                     first_decay_steps)
-  ```
-
-  Args:
-    learning_rate: A scalar `float32` or `float64` Tensor or a Python number.
-      The initial learning rate.
-    global_step: A scalar `int32` or `int64` `Tensor` or a Python number.
-      Global step to use for the decay computation.
-    first_decay_steps: A scalar `int32` or `int64` `Tensor` or a Python number.
-      Number of steps to decay over.
-    t_mul: A scalar `float32` or `float64` `Tensor` or a Python number.
-      Used to derive the number of iterations in the i-th period
-    m_mul: A scalar `float32` or `float64` `Tensor` or a Python number.
-      Used to derive the initial learning rate of the i-th period:
-    alpha: A scalar `float32` or `float64` Tensor or a Python number.
-      Minimum learning rate value as a fraction of the learning_rate.
-    name: String. Optional name of the operation.  Defaults to 'SGDRDecay'.
-  Returns:
-    A no-arg function that outputs the decayed learning rate, a scalar `Tensor`
-    of the same type as `learning_rate`.
-
-  Raises:
-    ValueError: if `global_step` is not supplied.
-  """
-  if global_step is None:
-    raise ValueError("cosine decay restarts requires global_step")
-  def decayed_lr(learning_rate, global_step, first_decay_steps, t_mul, m_mul,
-                 alpha, name):
-    """Helper to recompute learning rate; most helpful in eager-mode."""
-    with ops.name_scope(name, "SGDRDecay", [learning_rate, global_step]
-                       ) as name:
-      learning_rate = ops.convert_to_tensor(
-          learning_rate, name="initial_learning_rate")
-      dtype = learning_rate.dtype
-      first_decay_steps = math_ops.cast(first_decay_steps, dtype)
-      alpha = math_ops.cast(alpha, dtype)
-      t_mul = math_ops.cast(t_mul, dtype)
-      m_mul = math_ops.cast(m_mul, dtype)
-
-      global_step_recomp = math_ops.cast(global_step, dtype)
-      completed_fraction = global_step_recomp / first_decay_steps
-
-      def compute_step(completed_fraction, geometric=False):
-        """Helper for `cond` operation."""
-        if geometric:
-          i_restart = math_ops.floor(
-              math_ops.log(1.0 - completed_fraction * (1.0 - t_mul)) /
-              math_ops.log(t_mul))
-
-          sum_r = (1.0 - t_mul**i_restart) / (1.0 - t_mul)
-          completed_fraction = (completed_fraction - sum_r) / t_mul**i_restart
-
-        else:
-          i_restart = math_ops.floor(completed_fraction)
-          completed_fraction -= i_restart
-
-        return i_restart, completed_fraction
-
-      i_restart, completed_fraction = control_flow_ops.cond(
-          math_ops.equal(t_mul, 1.0),
-          lambda: compute_step(completed_fraction, geometric=False),
-          lambda: compute_step(completed_fraction, geometric=True))
-
-      m_fac = m_mul**i_restart
-      cosine_decayed = 0.5 * m_fac * (1.0 + math_ops.cos(
-          constant_op.constant(math.pi) * completed_fraction))
-      decayed = (1 - alpha) * cosine_decayed + alpha
-
-      return math_ops.multiply(learning_rate, decayed, name=name)
-
-  return functools.partial(decayed_lr, learning_rate, global_step,
-                           first_decay_steps, t_mul, m_mul, alpha, name)
-
-
-@tf_export("train.linear_cosine_decay", v1=[])
-def linear_cosine_decay(learning_rate,
-                        global_step,
-                        decay_steps,
-                        num_periods=0.5,
-                        alpha=0.0,
-                        beta=0.001,
-                        name=None):
-  """Applies linear cosine decay to the learning rate.
-
-  See [Bello et al., ICML2017] Neural Optimizer Search with RL.
-  https://arxiv.org/abs/1709.07417
-
-  For the idea of warm starts here controlled by `num_periods`,
-  see [Loshchilov & Hutter, ICLR2016] SGDR: Stochastic Gradient Descent
-  with Warm Restarts. https://arxiv.org/abs/1608.03983
-
-  Note that linear cosine decay is more aggressive than cosine decay and
-  larger initial learning rates can typically be used.
-
-  When training a model, it is often recommended to lower the learning rate as
-  the training progresses.  This function applies a linear cosine decay function
-  to a provided initial learning rate.  It requires a `global_step` value to
-  compute the decayed learning rate.  You can just pass a TensorFlow variable
-  that you increment at each training step.
-
-  The function returns a no-arg callable that produces the decayed learning
-  rate. This can be useful for changing the learning rate value across
-  different invocations of optimizer functions. It is computed as:
-
-  ```python
-  global_step = min(global_step, decay_steps)
-  linear_decay = (decay_steps - global_step) / decay_steps)
-  cosine_decay = 0.5 * (
-      1 + cos(pi * 2 * num_periods * global_step / decay_steps))
-  decayed = (alpha + linear_decay) * cosine_decay + beta
-  decayed_learning_rate = learning_rate * decayed
-  ```
-
-  Example usage:
-  ```python
-  decay_steps = 1000
-  lr_decayed_fn = tf.train.linear_cosine_decay(learning_rate, global_step,
-                                               decay_steps)
-  ```
-
-  Args:
-    learning_rate: A scalar `float32` or `float64` Tensor or a Python number.
-      The initial learning rate.
-    global_step: A scalar `int32` or `int64` `Tensor` or a Python number.
-      Global step to use for the decay computation.
-    decay_steps: A scalar `int32` or `int64` `Tensor` or a Python number.
-      Number of steps to decay over.
-    num_periods: Number of periods in the cosine part of the decay.
-      See computation above.
-    alpha: See computation above.
-    beta: See computation above.
-    name: String.  Optional name of the operation.  Defaults to
-      'LinearCosineDecay'.
-  Returns:
-    A no-arg function that outputs the decayed learning rate, a scalar `Tensor`
-    of the same type as `learning_rate`.
-  Raises:
-    ValueError: if `global_step` is not supplied.
-  """
-  if global_step is None:
-    raise ValueError("linear cosine decay requires global_step")
-  def decayed_lr(learning_rate, global_step, decay_steps, num_periods, alpha,
-                 beta, name):
-    """Helper to recompute learning rate; most helpful in eager-mode."""
-    with ops.name_scope(name, "LinearCosineDecay",
-                        [learning_rate, global_step]) as name:
-      learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate")
-      dtype = learning_rate.dtype
-      decay_steps = math_ops.cast(decay_steps, dtype)
-      num_periods = math_ops.cast(num_periods, dtype)
-      alpha = math_ops.cast(alpha, dtype)
-      beta = math_ops.cast(beta, dtype)
-
-      global_step_recomp = math_ops.cast(global_step, dtype)
-      global_step_recomp = math_ops.minimum(global_step_recomp, decay_steps)
-      linear_decayed = (decay_steps - global_step_recomp) / decay_steps
-      completed_fraction = global_step_recomp / decay_steps
-      fraction = 2.0 * num_periods * completed_fraction
-      cosine_decayed = 0.5 * (
-          1.0 + math_ops.cos(constant_op.constant(math.pi) * fraction))
-
-      linear_cosine_decayed = (alpha + linear_decayed) * cosine_decayed + beta
-      return math_ops.multiply(learning_rate, linear_cosine_decayed, name=name)
-
-  return functools.partial(decayed_lr, learning_rate, global_step, decay_steps,
-                           num_periods, alpha, beta, name)
-
-
-@tf_export("train.noisy_linear_cosine_decay", v1=[])
-def noisy_linear_cosine_decay(learning_rate,
-                              global_step,
-                              decay_steps,
-                              initial_variance=1.0,
-                              variance_decay=0.55,
-                              num_periods=0.5,
-                              alpha=0.0,
-                              beta=0.001,
-                              name=None):
-  """Applies noisy linear cosine decay to the learning rate.
-
-  See [Bello et al., ICML2017] Neural Optimizer Search with RL.
-  https://arxiv.org/abs/1709.07417
-
-  For the idea of warm starts here controlled by `num_periods`,
-  see [Loshchilov & Hutter, ICLR2016] SGDR: Stochastic Gradient Descent
-  with Warm Restarts. https://arxiv.org/abs/1608.03983
-
-  Note that linear cosine decay is more aggressive than cosine decay and
-  larger initial learning rates can typically be used.
-
-  When training a model, it is often recommended to lower the learning rate as
-  the training progresses.  This function applies a noisy linear
-  cosine decay function to a provided initial learning rate.
-  It requires a `global_step` value to compute the decayed learning rate.
-  You can just pass a TensorFlow variable that you increment at each
-  training step.
-
-  The function returns a no-arg callable that produces the decayed learning
-  rate. This can be useful for changing the learning rate value across
-  different invocations of optimizer functions. It is computed as:
-
-  ```python
-  global_step = min(global_step, decay_steps)
-  linear_decay = (decay_steps - global_step) / decay_steps)
-  cosine_decay = 0.5 * (
-      1 + cos(pi * 2 * num_periods * global_step / decay_steps))
-  decayed = (alpha + linear_decay + eps_t) * cosine_decay + beta
-  decayed_learning_rate = learning_rate * decayed
-  ```
-  where eps_t is 0-centered gaussian noise with variance
-  initial_variance / (1 + global_step) ** variance_decay
-
-  Example usage:
-  ```python
-  decay_steps = 1000
-  lr_decayed_fn = tf.train.noisy_linear_cosine_decay(learning_rate, global_step,
-                                                     decay_steps)
-  ```
-
-  Args:
-    learning_rate: A scalar `float32` or `float64` Tensor or a Python number.
-      The initial learning rate.
-    global_step: A scalar `int32` or `int64` `Tensor` or a Python number.
-      Global step to use for the decay computation.
-    decay_steps: A scalar `int32` or `int64` `Tensor` or a Python number.
-      Number of steps to decay over.
-    initial_variance: initial variance for the noise. See computation above.
-    variance_decay: decay for the noise's variance. See computation above.
-    num_periods: Number of periods in the cosine part of the decay.
-      See computation above.
-    alpha: See computation above.
-    beta: See computation above.
-    name: String.  Optional name of the operation.  Defaults to
-      'NoisyLinearCosineDecay'.
-  Returns:
-    A no-arg function that outputs the decayed learning rate, a scalar `Tensor`
-    of the same type as `learning_rate`.
-  Raises:
-    ValueError: if `global_step` is not supplied.
-  """
-  if global_step is None:
-    raise ValueError("noisy linear cosine decay requires global_step")
-  def decayed_lr(learning_rate, global_step, decay_steps, initial_variance,
-                 variance_decay, num_periods, alpha, beta, name):
-    """Helper to recompute learning rate; most helpful in eager-mode."""
-    with ops.name_scope(name, "NoisyLinearCosineDecay",
-                        [learning_rate, global_step]) as name:
-      learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate")
-      dtype = learning_rate.dtype
-      decay_steps = math_ops.cast(decay_steps, dtype)
-      initial_variance = math_ops.cast(initial_variance, dtype)
-      variance_decay = math_ops.cast(variance_decay, dtype)
-      num_periods = math_ops.cast(num_periods, dtype)
-      alpha = math_ops.cast(alpha, dtype)
-      beta = math_ops.cast(beta, dtype)
-
-      global_step_recomp = math_ops.cast(global_step, dtype)
-      global_step_recomp = math_ops.minimum(global_step_recomp, decay_steps)
-      linear_decayed = (decay_steps - global_step_recomp) / decay_steps
-      variance = initial_variance / (
-          math_ops.pow(1.0 + global_step_recomp, variance_decay))
-      std = math_ops.sqrt(variance)
-      noisy_linear_decayed = (
-          linear_decayed + random_ops.random_normal(
-              linear_decayed.shape, stddev=std))
-
-      completed_fraction = global_step_recomp / decay_steps
-      fraction = 2.0 * num_periods * completed_fraction
-      cosine_decayed = 0.5 * (
-          1.0 + math_ops.cos(constant_op.constant(math.pi) * fraction))
-      noisy_linear_cosine_decayed = (
-          (alpha + noisy_linear_decayed) * cosine_decayed + beta)
-
-      return math_ops.multiply(
-          learning_rate, noisy_linear_cosine_decayed, name=name)
-
-  return functools.partial(decayed_lr, learning_rate, global_step, decay_steps,
-                           initial_variance, variance_decay, num_periods, alpha,
-                           beta, name)
diff --git a/tensorflow/python/training/learning_rate_decay_v2_test.py b/tensorflow/python/training/learning_rate_decay_v2_test.py
deleted file mode 100644
index cb96773e29..0000000000
--- a/tensorflow/python/training/learning_rate_decay_v2_test.py
+++ /dev/null
@@ -1,497 +0,0 @@
-# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Functional test for learning rate decay."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import math
-
-from tensorflow.python.eager import context
-from tensorflow.python.framework import test_util
-# Import resource_variable_ops for the variables-to-tensor implicit conversion.
-from tensorflow.python.ops import resource_variable_ops  # pylint: disable=unused-import
-from tensorflow.python.ops import variables
-from tensorflow.python.platform import googletest
-from tensorflow.python.training import learning_rate_decay_v2
-
-
-class LRDecayTestV2(test_util.TensorFlowTestCase):
-
-  @test_util.run_in_graph_and_eager_modes
-  def testContinuous(self):
-    self.evaluate(variables.global_variables_initializer())
-    step = 5
-    decayed_lr = learning_rate_decay_v2.exponential_decay(0.05, step, 10, 0.96)
-    expected = .05 * 0.96**(5.0 / 10.0)
-    self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testStaircase(self):
-    if context.executing_eagerly():
-      step = resource_variable_ops.ResourceVariable(0)
-      self.evaluate(variables.global_variables_initializer())
-      decayed_lr = learning_rate_decay_v2.exponential_decay(
-          .1, step, 3, 0.96, staircase=True)
-
-      # No change to learning rate due to staircase
-      expected = .1
-      self.evaluate(step.assign(1))
-      self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
-
-      expected = .1
-      self.evaluate(step.assign(2))
-      self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
-
-      # Decayed learning rate
-      expected = .1 * 0.96 ** (100 // 3)
-      self.evaluate(step.assign(100))
-      self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
-
-  @test_util.run_deprecated_v1
-  def testVariables(self):
-    step = variables.Variable(1)
-    assign_1 = step.assign(1)
-    assign_2 = step.assign(2)
-    assign_100 = step.assign(100)
-    decayed_lr = learning_rate_decay_v2.exponential_decay(
-        .1, step, 3, 0.96, staircase=True)
-    self.evaluate(variables.global_variables_initializer())
-    # No change to learning rate
-    self.evaluate(assign_1.op)
-    self.assertAllClose(self.evaluate(decayed_lr()), .1, 1e-6)
-    self.evaluate(assign_2.op)
-    self.assertAllClose(self.evaluate(decayed_lr()), .1, 1e-6)
-    # Decayed learning rate
-    self.evaluate(assign_100.op)
-    expected = .1 * 0.96**(100 // 3)
-    self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testPiecewiseConstant(self):
-    x = resource_variable_ops.ResourceVariable(-999)
-    decayed_lr = learning_rate_decay_v2.piecewise_constant(
-        x, [100, 110, 120], [1.0, 0.1, 0.01, 0.001])
-
-    self.evaluate(variables.global_variables_initializer())
-
-    self.assertAllClose(self.evaluate(decayed_lr()), 1.0, 1e-6)
-    self.evaluate(x.assign(100))
-    self.assertAllClose(self.evaluate(decayed_lr()), 1.0, 1e-6)
-    self.evaluate(x.assign(105))
-    self.assertAllClose(self.evaluate(decayed_lr()), 0.1, 1e-6)
-    self.evaluate(x.assign(110))
-    self.assertAllClose(self.evaluate(decayed_lr()), 0.1, 1e-6)
-    self.evaluate(x.assign(120))
-    self.assertAllClose(self.evaluate(decayed_lr()), 0.01, 1e-6)
-    self.evaluate(x.assign(999))
-    self.assertAllClose(self.evaluate(decayed_lr()), 0.001, 1e-6)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testPiecewiseConstantEdgeCases(self):
-    x_int = resource_variable_ops.ResourceVariable(
-        0, dtype=variables.dtypes.int32)
-    boundaries, values = [-1.0, 1.0], [1, 2, 3]
-    with self.assertRaises(ValueError):
-      decayed_lr = learning_rate_decay_v2.piecewise_constant(
-          x_int, boundaries, values)
-      decayed_lr()
-
-    x = resource_variable_ops.ResourceVariable(0.0)
-    boundaries, values = [-1.0, 1.0], [1.0, 2, 3]
-    with self.assertRaises(ValueError):
-      decayed_lr = learning_rate_decay_v2.piecewise_constant(
-          x, boundaries, values)()
-      decayed_lr()
-
-    # Test that ref types are valid.
-    if not context.executing_eagerly():
-      x = variables.Variable(0.0)
-      x_ref = x.op.outputs[0]   # float32_ref tensor should be accepted
-      boundaries, values = [1.0, 2.0], [1, 2, 3]
-      learning_rate_decay_v2.piecewise_constant(x_ref, boundaries, values)
-
-    # Test casting boundaries from int32 to int64.
-    x_int64 = resource_variable_ops.ResourceVariable(
-        0, dtype=variables.dtypes.int64)
-    boundaries, values = [1, 2, 3], [0.4, 0.5, 0.6, 0.7]
-    decayed_lr = learning_rate_decay_v2.piecewise_constant(
-        x_int64, boundaries, values)
-
-    self.evaluate(variables.global_variables_initializer())
-    self.assertAllClose(self.evaluate(decayed_lr()), 0.4, 1e-6)
-    self.evaluate(x_int64.assign(1))
-    self.assertAllClose(self.evaluate(decayed_lr()), 0.4, 1e-6)
-    self.evaluate(x_int64.assign(2))
-    self.assertAllClose(self.evaluate(decayed_lr()), 0.5, 1e-6)
-    self.evaluate(x_int64.assign(3))
-    self.assertAllClose(self.evaluate(decayed_lr()), 0.6, 1e-6)
-    self.evaluate(x_int64.assign(4))
-    self.assertAllClose(self.evaluate(decayed_lr()), 0.7, 1e-6)
-
-
-class LinearDecayTestV2(test_util.TensorFlowTestCase):
-
-  @test_util.run_in_graph_and_eager_modes
-  def testHalfWay(self):
-    step = 5
-    lr = 0.05
-    end_lr = 0.0
-    decayed_lr = learning_rate_decay_v2.polynomial_decay(lr, step, 10, end_lr)
-    expected = lr * 0.5
-    self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testEnd(self):
-    step = 10
-    lr = 0.05
-    end_lr = 0.001
-    decayed_lr = learning_rate_decay_v2.polynomial_decay(lr, step, 10, end_lr)
-    expected = end_lr
-    self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testHalfWayWithEnd(self):
-    step = 5
-    lr = 0.05
-    end_lr = 0.001
-    decayed_lr = learning_rate_decay_v2.polynomial_decay(lr, step, 10, end_lr)
-    expected = (lr + end_lr) * 0.5
-    self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testBeyondEnd(self):
-    step = 15
-    lr = 0.05
-    end_lr = 0.001
-    decayed_lr = learning_rate_decay_v2.polynomial_decay(lr, step, 10, end_lr)
-    expected = end_lr
-    self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testBeyondEndWithCycle(self):
-    step = 15
-    lr = 0.05
-    end_lr = 0.001
-    decayed_lr = learning_rate_decay_v2.polynomial_decay(
-        lr, step, 10, end_lr, cycle=True)
-    expected = (lr - end_lr) * 0.25 + end_lr
-    self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
-
-
-class SqrtDecayTestV2(test_util.TensorFlowTestCase):
-
-  @test_util.run_in_graph_and_eager_modes
-  def testHalfWay(self):
-    step = 5
-    lr = 0.05
-    end_lr = 0.0
-    power = 0.5
-    decayed_lr = learning_rate_decay_v2.polynomial_decay(
-        lr, step, 10, end_lr, power=power)
-    expected = lr * 0.5**power
-    self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testEnd(self):
-    step = 10
-    lr = 0.05
-    end_lr = 0.001
-    power = 0.5
-    decayed_lr = learning_rate_decay_v2.polynomial_decay(
-        lr, step, 10, end_lr, power=power)
-    expected = end_lr
-    self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testHalfWayWithEnd(self):
-    step = 5
-    lr = 0.05
-    end_lr = 0.001
-    power = 0.5
-    decayed_lr = learning_rate_decay_v2.polynomial_decay(
-        lr, step, 10, end_lr, power=power)
-    expected = (lr - end_lr) * 0.5**power + end_lr
-    self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testBeyondEnd(self):
-    step = 15
-    lr = 0.05
-    end_lr = 0.001
-    power = 0.5
-    decayed_lr = learning_rate_decay_v2.polynomial_decay(
-        lr, step, 10, end_lr, power=power)
-    expected = end_lr
-    self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testBeyondEndWithCycle(self):
-    step = 15
-    lr = 0.05
-    end_lr = 0.001
-    power = 0.5
-    decayed_lr = learning_rate_decay_v2.polynomial_decay(
-        lr, step, 10, end_lr, power=power, cycle=True)
-    expected = (lr - end_lr) * 0.25**power + end_lr
-    self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
-
-
-class PolynomialDecayTestV2(test_util.TensorFlowTestCase):
-
-  @test_util.run_in_graph_and_eager_modes
-  def testBeginWithCycle(self):
-    lr = 0.001
-    decay_steps = 10
-    step = 0
-    decayed_lr = learning_rate_decay_v2.polynomial_decay(
-        lr, step, decay_steps, cycle=True)
-    expected = lr
-    self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
-
-
-class ExponentialDecayTestV2(test_util.TensorFlowTestCase):
-
-  @test_util.run_in_graph_and_eager_modes
-  def testDecay(self):
-    initial_lr = 0.1
-    k = 10
-    decay_rate = 0.96
-    step = resource_variable_ops.ResourceVariable(0)
-    decayed_lr = learning_rate_decay_v2.natural_exp_decay(initial_lr, step, k,
-                                                          decay_rate)
-
-    self.evaluate(variables.global_variables_initializer())
-    for i in range(k + 1):
-      expected = initial_lr * math.exp(-i / k * decay_rate)
-      self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
-      self.evaluate(step.assign_add(1))
-
-  @test_util.run_in_graph_and_eager_modes
-  def testStaircase(self):
-    initial_lr = 0.1
-    k = 10
-    decay_rate = 0.96
-    step = resource_variable_ops.ResourceVariable(0)
-    decayed_lr = learning_rate_decay_v2.natural_exp_decay(
-        initial_lr, step, k, decay_rate, staircase=True)
-
-    self.evaluate(variables.global_variables_initializer())
-    for i in range(k + 1):
-      expected = initial_lr * math.exp(-decay_rate * (i // k))
-      self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
-      self.evaluate(step.assign_add(1))
-
-
-class InverseDecayTestV2(test_util.TensorFlowTestCase):
-
-  @test_util.run_in_graph_and_eager_modes
-  def testDecay(self):
-    initial_lr = 0.1
-    k = 10
-    decay_rate = 0.96
-    step = resource_variable_ops.ResourceVariable(0)
-    decayed_lr = learning_rate_decay_v2.inverse_time_decay(initial_lr, step, k,
-                                                           decay_rate)
-
-    self.evaluate(variables.global_variables_initializer())
-    for i in range(k + 1):
-      expected = initial_lr / (1 + i / k * decay_rate)
-      self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
-      self.evaluate(step.assign_add(1))
-
-  @test_util.run_in_graph_and_eager_modes
-  def testStaircase(self):
-    initial_lr = 0.1
-    k = 10
-    decay_rate = 0.96
-    step = resource_variable_ops.ResourceVariable(0)
-    decayed_lr = learning_rate_decay_v2.inverse_time_decay(
-        initial_lr, step, k, decay_rate, staircase=True)
-
-    self.evaluate(variables.global_variables_initializer())
-    for i in range(k + 1):
-      expected = initial_lr / (1 + decay_rate * (i // k))
-      self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
-      self.evaluate(step.assign_add(1))
-
-
-class CosineDecayTestV2(test_util.TensorFlowTestCase):
-
-  def np_cosine_decay(self, step, decay_steps, alpha=0.0):
-    step = min(step, decay_steps)
-    completed_fraction = step / decay_steps
-    decay = 0.5 * (1.0 + math.cos(math.pi * completed_fraction))
-    return (1.0 - alpha) * decay + alpha
-
-  @test_util.run_in_graph_and_eager_modes
-  def testDecay(self):
-    num_training_steps = 1000
-    initial_lr = 1.0
-    for step in range(0, 1500, 250):
-      decayed_lr = learning_rate_decay_v2.cosine_decay(initial_lr, step,
-                                                       num_training_steps)
-      expected = self.np_cosine_decay(step, num_training_steps)
-      self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testAlpha(self):
-    num_training_steps = 1000
-    initial_lr = 1.0
-    alpha = 0.1
-    for step in range(0, 1500, 250):
-      decayed_lr = learning_rate_decay_v2.cosine_decay(initial_lr, step,
-                                                       num_training_steps,
-                                                       alpha)
-      expected = self.np_cosine_decay(step, num_training_steps, alpha)
-      self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
-
-
-class CosineDecayRestartsTestV2(test_util.TensorFlowTestCase):
-
-  def np_cosine_decay_restarts(self, step, decay_steps, t_mul=2.0, m_mul=1.0,
-                               alpha=0.0):
-    fac = 1.0
-    while step >= decay_steps:
-      step -= decay_steps
-      decay_steps *= t_mul
-      fac *= m_mul
-
-    completed_fraction = step / decay_steps
-    decay = fac * 0.5 * (1.0 + math.cos(math.pi * completed_fraction))
-    return (1.0 - alpha) * decay + alpha
-
-  @test_util.run_in_graph_and_eager_modes
-  def testDecay(self):
-    num_training_steps = 1000
-    initial_lr = 1.0
-    for step in range(0, 1500, 250):
-      decayed_lr = learning_rate_decay_v2.cosine_decay_restarts(
-          initial_lr, step, num_training_steps)
-      expected = self.np_cosine_decay_restarts(step, num_training_steps)
-      self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testAlpha(self):
-    num_training_steps = 1000
-    initial_lr = 1.0
-    alpha = 0.1
-    for step in range(0, 1500, 250):
-      decayed_lr = learning_rate_decay_v2.cosine_decay_restarts(
-          initial_lr, step, num_training_steps, alpha=alpha)
-      expected = self.np_cosine_decay_restarts(
-          step, num_training_steps, alpha=alpha)
-      self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testMMul(self):
-    num_training_steps = 1000
-    initial_lr = 1.0
-    m_mul = 0.9
-    for step in range(0, 1500, 250):
-      decayed_lr = learning_rate_decay_v2.cosine_decay_restarts(
-          initial_lr, step, num_training_steps, m_mul=m_mul)
-      expected = self.np_cosine_decay_restarts(
-          step, num_training_steps, m_mul=m_mul)
-      self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testTMul(self):
-    num_training_steps = 1000
-    initial_lr = 1.0
-    t_mul = 1.0
-    for step in range(0, 1500, 250):
-      decayed_lr = learning_rate_decay_v2.cosine_decay_restarts(
-          initial_lr, step, num_training_steps, t_mul=t_mul)
-      expected = self.np_cosine_decay_restarts(
-          step, num_training_steps, t_mul=t_mul)
-      self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
-
-
-class LinearCosineDecayTestV2(test_util.TensorFlowTestCase):
-
-  def np_linear_cosine_decay(self,
-                             step,
-                             decay_steps,
-                             alpha=0.0,
-                             beta=0.001,
-                             num_periods=0.5):
-    step = min(step, decay_steps)
-    linear_decayed = float(decay_steps - step) / decay_steps
-    fraction = 2.0 * num_periods * step / float(decay_steps)
-    cosine_decayed = 0.5 * (1.0 + math.cos(math.pi * fraction))
-    return (alpha + linear_decayed) * cosine_decayed + beta
-
-  @test_util.run_in_graph_and_eager_modes
-  def testDefaultDecay(self):
-    num_training_steps = 1000
-    initial_lr = 1.0
-    for step in range(0, 1500, 250):
-      decayed_lr = learning_rate_decay_v2.linear_cosine_decay(
-          initial_lr, step, num_training_steps)
-      expected = self.np_linear_cosine_decay(step, num_training_steps)
-      self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testNonDefaultDecay(self):
-    num_training_steps = 1000
-    initial_lr = 1.0
-    for step in range(0, 1500, 250):
-      decayed_lr = learning_rate_decay_v2.linear_cosine_decay(
-          initial_lr,
-          step,
-          num_training_steps,
-          alpha=0.1,
-          beta=1e-4,
-          num_periods=5)
-      expected = self.np_linear_cosine_decay(
-          step, num_training_steps, alpha=0.1, beta=1e-4, num_periods=5)
-      self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
-
-
-class NoisyLinearCosineDecayTestV2(test_util.TensorFlowTestCase):
-
-  @test_util.run_in_graph_and_eager_modes
-  def testDefaultNoisyLinearCosine(self):
-    num_training_steps = 1000
-    initial_lr = 1.0
-    for step in range(0, 1500, 250):
-      # No numerical check because of noise
-      decayed_lr = learning_rate_decay_v2.noisy_linear_cosine_decay(
-          initial_lr, step, num_training_steps)
-      # Cannot be deterministically tested
-      self.evaluate(decayed_lr())
-
-  @test_util.run_in_graph_and_eager_modes
-  def testNonDefaultNoisyLinearCosine(self):
-    num_training_steps = 1000
-    initial_lr = 1.0
-    for step in range(0, 1500, 250):
-      # No numerical check because of noise
-      decayed_lr = learning_rate_decay_v2.noisy_linear_cosine_decay(
-          initial_lr,
-          step,
-          num_training_steps,
-          initial_variance=0.5,
-          variance_decay=0.1,
-          alpha=0.1,
-          beta=1e-4,
-          num_periods=5)
-      # Cannot be deterministically tested
-      self.evaluate(decayed_lr())
-
-if __name__ == "__main__":
-  googletest.main()
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-cosine-decay.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-cosine-decay.pbtxt
new file mode 100644
index 0000000000..e2549a2ac6
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-cosine-decay.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.keras.experimental.CosineDecay"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.CosineDecay\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.LearningRateSchedule\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'initial_learning_rate\', \'decay_steps\', \'alpha\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0\', \'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.pbtxt
index 5cd6851278..24684b9f4d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.pbtxt
@@ -1,5 +1,9 @@
 path: "tensorflow.keras.experimental"
 tf_module {
+  member {
+    name: "CosineDecay"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "PeepholeLSTMCell"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.pbtxt
index 7257b02087..97f7c4b8c9 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.pbtxt
@@ -32,6 +32,10 @@ tf_module {
     name: "SGD"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "schedules"
+    mtype: "<type \'module\'>"
+  }
   member_method {
     name: "deserialize"
     argspec: "args=[\'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.schedules.-exponential-decay.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.schedules.-exponential-decay.pbtxt
new file mode 100644
index 0000000000..25ae478cb2
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.schedules.-exponential-decay.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.keras.optimizers.schedules.ExponentialDecay"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.ExponentialDecay\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.LearningRateSchedule\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'initial_learning_rate\', \'decay_steps\', \'decay_rate\', \'staircase\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.schedules.-inverse-time-decay.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.schedules.-inverse-time-decay.pbtxt
new file mode 100644
index 0000000000..b2fe61f4d2
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.schedules.-inverse-time-decay.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.keras.optimizers.schedules.InverseTimeDecay"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.InverseTimeDecay\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.LearningRateSchedule\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'initial_learning_rate\', \'decay_steps\', \'decay_rate\', \'staircase\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.schedules.-learning-rate-schedule.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.schedules.-learning-rate-schedule.pbtxt
new file mode 100644
index 0000000000..3b33bd7526
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.schedules.-learning-rate-schedule.pbtxt
@@ -0,0 +1,16 @@
+path: "tensorflow.keras.optimizers.schedules.LearningRateSchedule"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.LearningRateSchedule\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.schedules.-piecewise-constant-decay.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.schedules.-piecewise-constant-decay.pbtxt
new file mode 100644
index 0000000000..6f1496492a
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.schedules.-piecewise-constant-decay.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.keras.optimizers.schedules.PiecewiseConstantDecay"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.PiecewiseConstantDecay\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.LearningRateSchedule\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'boundaries\', \'values\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.schedules.-polynomial-decay.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.schedules.-polynomial-decay.pbtxt
new file mode 100644
index 0000000000..728436c361
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.schedules.-polynomial-decay.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.keras.optimizers.schedules.PolynomialDecay"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.PolynomialDecay\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.LearningRateSchedule\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'initial_learning_rate\', \'decay_steps\', \'end_learning_rate\', \'power\', \'cycle\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0001\', \'1.0\', \'False\', \'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.schedules.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.schedules.pbtxt
new file mode 100644
index 0000000000..024e472a73
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.schedules.pbtxt
@@ -0,0 +1,31 @@
+path: "tensorflow.keras.optimizers.schedules"
+tf_module {
+  member {
+    name: "ExponentialDecay"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "InverseTimeDecay"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "LearningRateSchedule"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "PiecewiseConstantDecay"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "PolynomialDecay"
+    mtype: "<type \'type\'>"
+  }
+  member_method {
+    name: "deserialize"
+    argspec: "args=[\'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "serialize"
+    argspec: "args=[\'learning_rate_schedule\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-cosine-decay-restarts.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-cosine-decay-restarts.pbtxt
new file mode 100644
index 0000000000..58bede556d
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-cosine-decay-restarts.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.keras.experimental.CosineDecayRestarts"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.CosineDecayRestarts\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.LearningRateSchedule\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'initial_learning_rate\', \'first_decay_steps\', \'t_mul\', \'m_mul\', \'alpha\', \'name\'], varargs=None, keywords=None, defaults=[\'2.0\', \'1.0\', \'0.0\', \'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-cosine-decay.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-cosine-decay.pbtxt
new file mode 100644
index 0000000000..e2549a2ac6
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-cosine-decay.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.keras.experimental.CosineDecay"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.CosineDecay\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.LearningRateSchedule\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'initial_learning_rate\', \'decay_steps\', \'alpha\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0\', \'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-linear-cosine-decay.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-linear-cosine-decay.pbtxt
new file mode 100644
index 0000000000..f083120b52
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-linear-cosine-decay.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.keras.experimental.LinearCosineDecay"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.LinearCosineDecay\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.LearningRateSchedule\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'initial_learning_rate\', \'decay_steps\', \'num_periods\', \'alpha\', \'beta\', \'name\'], varargs=None, keywords=None, defaults=[\'0.5\', \'0.0\', \'0.001\', \'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-noisy-linear-cosine-decay.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-noisy-linear-cosine-decay.pbtxt
new file mode 100644
index 0000000000..8ea3c6beb1
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-noisy-linear-cosine-decay.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.keras.experimental.NoisyLinearCosineDecay"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.NoisyLinearCosineDecay\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.LearningRateSchedule\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'initial_learning_rate\', \'decay_steps\', \'initial_variance\', \'variance_decay\', \'num_periods\', \'alpha\', \'beta\', \'name\'], varargs=None, keywords=None, defaults=[\'1.0\', \'0.55\', \'0.5\', \'0.0\', \'0.001\', \'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.pbtxt
index 5cd6851278..721c18890a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.pbtxt
@@ -1,5 +1,21 @@
 path: "tensorflow.keras.experimental"
 tf_module {
+  member {
+    name: "CosineDecay"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "CosineDecayRestarts"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "LinearCosineDecay"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "NoisyLinearCosineDecay"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "PeepholeLSTMCell"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.pbtxt
index 7257b02087..97f7c4b8c9 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.pbtxt
@@ -32,6 +32,10 @@ tf_module {
     name: "SGD"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "schedules"
+    mtype: "<type \'module\'>"
+  }
   member_method {
     name: "deserialize"
     argspec: "args=[\'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.schedules.-exponential-decay.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.schedules.-exponential-decay.pbtxt
new file mode 100644
index 0000000000..25ae478cb2
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.schedules.-exponential-decay.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.keras.optimizers.schedules.ExponentialDecay"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.ExponentialDecay\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.LearningRateSchedule\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'initial_learning_rate\', \'decay_steps\', \'decay_rate\', \'staircase\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.schedules.-inverse-time-decay.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.schedules.-inverse-time-decay.pbtxt
new file mode 100644
index 0000000000..b2fe61f4d2
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.schedules.-inverse-time-decay.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.keras.optimizers.schedules.InverseTimeDecay"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.InverseTimeDecay\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.LearningRateSchedule\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'initial_learning_rate\', \'decay_steps\', \'decay_rate\', \'staircase\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.schedules.-learning-rate-schedule.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.schedules.-learning-rate-schedule.pbtxt
new file mode 100644
index 0000000000..3b33bd7526
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.schedules.-learning-rate-schedule.pbtxt
@@ -0,0 +1,16 @@
+path: "tensorflow.keras.optimizers.schedules.LearningRateSchedule"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.LearningRateSchedule\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.schedules.-piecewise-constant-decay.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.schedules.-piecewise-constant-decay.pbtxt
new file mode 100644
index 0000000000..6f1496492a
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.schedules.-piecewise-constant-decay.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.keras.optimizers.schedules.PiecewiseConstantDecay"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.PiecewiseConstantDecay\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.LearningRateSchedule\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'boundaries\', \'values\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.schedules.-polynomial-decay.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.schedules.-polynomial-decay.pbtxt
new file mode 100644
index 0000000000..728436c361
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.schedules.-polynomial-decay.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.keras.optimizers.schedules.PolynomialDecay"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.PolynomialDecay\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.LearningRateSchedule\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'initial_learning_rate\', \'decay_steps\', \'end_learning_rate\', \'power\', \'cycle\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0001\', \'1.0\', \'False\', \'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.schedules.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.schedules.pbtxt
new file mode 100644
index 0000000000..024e472a73
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.schedules.pbtxt
@@ -0,0 +1,31 @@
+path: "tensorflow.keras.optimizers.schedules"
+tf_module {
+  member {
+    name: "ExponentialDecay"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "InverseTimeDecay"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "LearningRateSchedule"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "PiecewiseConstantDecay"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "PolynomialDecay"
+    mtype: "<type \'type\'>"
+  }
+  member_method {
+    name: "deserialize"
+    argspec: "args=[\'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "serialize"
+    argspec: "args=[\'learning_rate_schedule\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
index 8b39086ed1..4f293fb40d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
@@ -68,34 +68,14 @@ tf_module {
     name: "ServerDef"
     mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
   }
-  member_method {
-    name: "cosine_decay"
-    argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'alpha\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0\', \'None\'], "
-  }
-  member_method {
-    name: "cosine_decay_restarts"
-    argspec: "args=[\'learning_rate\', \'global_step\', \'first_decay_steps\', \'t_mul\', \'m_mul\', \'alpha\', \'name\'], varargs=None, keywords=None, defaults=[\'2.0\', \'1.0\', \'0.0\', \'None\'], "
-  }
-  member_method {
-    name: "exponential_decay"
-    argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'decay_rate\', \'staircase\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
-  }
   member_method {
     name: "get_checkpoint_state"
     argspec: "args=[\'checkpoint_dir\', \'latest_filename\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "inverse_time_decay"
-    argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'decay_rate\', \'staircase\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
-  }
   member_method {
     name: "latest_checkpoint"
     argspec: "args=[\'checkpoint_dir\', \'latest_filename\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "linear_cosine_decay"
-    argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'num_periods\', \'alpha\', \'beta\', \'name\'], varargs=None, keywords=None, defaults=[\'0.5\', \'0.0\', \'0.001\', \'None\'], "
-  }
   member_method {
     name: "list_variables"
     argspec: "args=[\'ckpt_dir_or_file\'], varargs=None, keywords=None, defaults=None"
@@ -108,22 +88,6 @@ tf_module {
     name: "load_variable"
     argspec: "args=[\'ckpt_dir_or_file\', \'name\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "natural_exp_decay"
-    argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'decay_rate\', \'staircase\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
-  }
-  member_method {
-    name: "noisy_linear_cosine_decay"
-    argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'initial_variance\', \'variance_decay\', \'num_periods\', \'alpha\', \'beta\', \'name\'], varargs=None, keywords=None, defaults=[\'1.0\', \'0.55\', \'0.5\', \'0.0\', \'0.001\', \'None\'], "
-  }
-  member_method {
-    name: "piecewise_constant_decay"
-    argspec: "args=[\'x\', \'boundaries\', \'values\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "polynomial_decay"
-    argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'end_learning_rate\', \'power\', \'cycle\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0001\', \'1.0\', \'False\', \'None\'], "
-  }
   member_method {
     name: "sdca_fprint"
     argspec: "args=[\'input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 5eb3d35527..011be5172d 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -693,8 +693,11 @@ renames = {
     'tf.train.batch': 'tf.compat.v1.train.batch',
     'tf.train.batch_join': 'tf.compat.v1.train.batch_join',
     'tf.train.checkpoint_exists': 'tf.compat.v1.train.checkpoint_exists',
+    'tf.train.cosine_decay': 'tf.compat.v1.train.cosine_decay',
+    'tf.train.cosine_decay_restarts': 'tf.compat.v1.train.cosine_decay_restarts',
     'tf.train.create_global_step': 'tf.compat.v1.train.create_global_step',
     'tf.train.do_quantize_training_on_graphdef': 'tf.compat.v1.train.do_quantize_training_on_graphdef',
+    'tf.train.exponential_decay': 'tf.compat.v1.train.exponential_decay',
     'tf.train.export_meta_graph': 'tf.compat.v1.train.export_meta_graph',
     'tf.train.generate_checkpoint_state_proto': 'tf.compat.v1.train.generate_checkpoint_state_proto',
     'tf.train.get_checkpoint_mtimes': 'tf.compat.v1.train.get_checkpoint_mtimes',
@@ -704,13 +707,19 @@ renames = {
     'tf.train.import_meta_graph': 'tf.compat.v1.train.import_meta_graph',
     'tf.train.init_from_checkpoint': 'tf.compat.v1.train.init_from_checkpoint',
     'tf.train.input_producer': 'tf.compat.v1.train.input_producer',
+    'tf.train.inverse_time_decay': 'tf.compat.v1.train.inverse_time_decay',
     'tf.train.limit_epochs': 'tf.compat.v1.train.limit_epochs',
+    'tf.train.linear_cosine_decay': 'tf.compat.v1.train.linear_cosine_decay',
     'tf.train.match_filenames_once': 'tf.io.match_filenames_once',
     'tf.train.maybe_batch': 'tf.compat.v1.train.maybe_batch',
     'tf.train.maybe_batch_join': 'tf.compat.v1.train.maybe_batch_join',
     'tf.train.maybe_shuffle_batch': 'tf.compat.v1.train.maybe_shuffle_batch',
     'tf.train.maybe_shuffle_batch_join': 'tf.compat.v1.train.maybe_shuffle_batch_join',
+    'tf.train.natural_exp_decay': 'tf.compat.v1.train.natural_exp_decay',
+    'tf.train.noisy_linear_cosine_decay': 'tf.compat.v1.train.noisy_linear_cosine_decay',
     'tf.train.piecewise_constant': 'tf.compat.v1.train.piecewise_constant',
+    'tf.train.piecewise_constant_decay': 'tf.compat.v1.train.piecewise_constant_decay',
+    'tf.train.polynomial_decay': 'tf.compat.v1.train.polynomial_decay',
     'tf.train.queue_runner.QueueRunner': 'tf.compat.v1.train.queue_runner.QueueRunner',
     'tf.train.queue_runner.add_queue_runner': 'tf.compat.v1.train.queue_runner.add_queue_runner',
     'tf.train.queue_runner.start_queue_runners': 'tf.compat.v1.train.queue_runner.start_queue_runners',
-- 
GitLab


From 429ec381328836790c4afac0ce46261a3c2862da Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Wed, 30 Jan 2019 10:48:16 -0800
Subject: [PATCH 1595/2345] Cleanup: remove manual_test.py. It was added for
 debugging purposes and is not used any more.

PiperOrigin-RevId: 231624711
---
 tensorflow/contrib/tensorrt/BUILD             |   1 -
 .../contrib/tensorrt/test/manual_test.py      | 114 ------------------
 2 files changed, 115 deletions(-)
 delete mode 100644 tensorflow/contrib/tensorrt/test/manual_test.py

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 67461450f8..e13edd1b5e 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -146,7 +146,6 @@ cuda_py_tests(
         "test/identity_output_test.py",
         "test/int32_test.py",
         "test/lru_cache_test.py",
-        "test/manual_test.py",
         "test/memory_alignment_test.py",
         "test/multi_connection_neighbor_engine_test.py",
         "test/neighboring_engine_test.py",
diff --git a/tensorflow/contrib/tensorrt/test/manual_test.py b/tensorflow/contrib/tensorrt/test/manual_test.py
deleted file mode 100644
index aad7b9f307..0000000000
--- a/tensorflow/contrib/tensorrt/test/manual_test.py
+++ /dev/null
@@ -1,114 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Basic tests for TF-TensorRT integration."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import ast
-import os
-
-from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
-from tensorflow.core.framework import graph_pb2
-from tensorflow.python.platform import gfile
-from tensorflow.python.platform import test
-
-
-class ManualTest(trt_test.TfTrtIntegrationTestBase):
-
-  def __init__(self, methodName='runTest'):  # pylint: disable=invalid-name
-    super(ManualTest, self).__init__(methodName)
-    self._params_map = None
-
-  def _GetEnv(self):
-    """Get an environment variable specifying the manual test parameters.
-
-    The value of the environment variable is the string representation of a dict
-    which should contain the following keys:
-    - 'graph_path': the file path to the serialized frozen graphdef
-    - 'input_names': TfTrtIntegrationTestParams.input_names
-    - 'input_dims': TfTrtIntegrationTestParams.input_dims
-    - 'expected_output_dims': TfTrtIntegrationTestParams.expected_output_dims
-    - 'output_name': the name of op to fetch
-    - 'expected_engines_to_run': ExpectedEnginesToRun() will return this
-    - 'expected_engines_to_build': ExpectedEnginesToBuild() will return this
-    - 'max_batch_size': ConversionParams.max_batch_size
-
-    Returns:
-      The value of the environment variable.
-    """
-    return os.getenv('TRT_MANUAL_TEST_PARAMS', '')
-
-  def _GetParamsMap(self):
-    """Parse the environment variable as a dict and return it."""
-    if self._params_map is None:
-      self._params_map = ast.literal_eval(self._GetEnv())
-    return self._params_map
-
-  def GetParams(self):
-    """Testing conversion of manually provided frozen graph."""
-    params_map = self._GetParamsMap()
-    gdef = graph_pb2.GraphDef()
-    with gfile.Open(params_map['graph_path'], 'rb') as f:
-      gdef.ParseFromString(f.read())
-    return trt_test.TfTrtIntegrationTestParams(
-        gdef=gdef,
-        input_names=params_map['input_names'],
-        input_dims=[params_map['input_dims']],
-        output_names=params_map['output_names'],
-        expected_output_dims=[params_map['expected_output_dims']])
-
-  def GetConversionParams(self, run_params):
-    """Return a ConversionParams for test."""
-    conversion_params = super(ManualTest, self).GetConversionParams(run_params)
-    params_map = self._GetParamsMap()
-    if 'max_batch_size' in params_map:
-      conversion_params = conversion_params._replace(
-          max_batch_size=params_map['max_batch_size'])
-    return conversion_params
-
-  def ExpectedEnginesToBuild(self, run_params):
-    """Return the expected engines to build."""
-    return self._GetParamsMap()['expected_engines_to_build']
-
-  def ExpectedEnginesToRun(self, run_params):
-    """Return the expected engines to run."""
-    params_map = self._GetParamsMap()
-    if 'expected_engines_to_run' in params_map:
-      return params_map['expected_engines_to_run']
-    return self.ExpectedEnginesToBuild(run_params)
-
-  def ExpectedAbsoluteTolerance(self, run_params):
-    """The absolute tolerance to compare floating point results."""
-    params_map = self._GetParamsMap()
-    if 'atol' in params_map:
-      return params_map['atol']
-    return 1.e-3
-
-  def ExpectedRelativeTolerance(self, run_params):
-    """The relative tolerance to compare floating point results."""
-    params_map = self._GetParamsMap()
-    if 'rtol' in params_map:
-      return params_map['rtol']
-    return 1.e-3
-
-  def ShouldRunTest(self, run_params):
-    """Whether to run the test."""
-    return len(self._GetEnv())
-
-
-if __name__ == '__main__':
-  test.main()
-- 
GitLab


From 57c8ec08ab02b64427a822d06da6348cf28f7a54 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 30 Jan 2019 10:55:38 -0800
Subject: [PATCH 1596/2345] Change noisy VLOG(0) (which is equivalent to
 LOG(INFO)) to VLOG(1).

PiperOrigin-RevId: 231626459
---
 .../xla/service/optimize_input_output_buffer_alias.cc         | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/optimize_input_output_buffer_alias.cc b/tensorflow/compiler/xla/service/optimize_input_output_buffer_alias.cc
index 00e5a9186e..701c629add 100644
--- a/tensorflow/compiler/xla/service/optimize_input_output_buffer_alias.cc
+++ b/tensorflow/compiler/xla/service/optimize_input_output_buffer_alias.cc
@@ -42,8 +42,8 @@ StatusOr<bool> OptimizeInputOutputBufferAlias::Build(
   bool changed = false;
   TF_RET_CHECK(LayoutUtil::HasLayout(input_shape));
   TF_RET_CHECK(LayoutUtil::HasLayout(output_shape));
-  VLOG(0) << "input_shape:" << input_shape.ToString();
-  VLOG(0) << "output_shape:" << output_shape.ToString();
+  VLOG(1) << "input_shape:" << input_shape.ToString();
+  VLOG(1) << "output_shape:" << output_shape.ToString();
 
   // For all buffers defined by the parameter, build a map from the byte
   // size to the list of the buffers of that size.
-- 
GitLab


From 34b64c3af3b2677115282cb7218aa81e2d505113 Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Wed, 30 Jan 2019 11:05:49 -0800
Subject: [PATCH 1597/2345] Limit colocation group devices after a node is
 assigned

This change makes Placer's code less brittle. A quick recap of Placer:
Placer first groups nodes into colocation groups. Each group is a set of
nodes and a set of possible devices that can be used for the nodes in
the group. The set of possible devices for a group is encoded in the root
Member of the tree representing this group. All the nodes in a group must
be assigned to the same device.

Before this change, the "all nodes in a group must be assigned to the same
device" property was ensured by a relying on specifics of GetDevicesForNode
and Placer's heuristics.

This change explicitly limits the possible set of devices to the single device
that a node in the group was assigned to. All future calls to GetDevicesForNode
will return just this device making it harder for a heuristic to choose
another device.

I did not find an easy way to test this change because there is no behavior
change. My future changes include a test for this that utilizes multi-device
function calls.

PiperOrigin-RevId: 231628964
---
 tensorflow/core/common_runtime/placer.cc | 96 +++++++++++++++++++-----
 tensorflow/core/common_runtime/placer.h  |  5 --
 2 files changed, 77 insertions(+), 24 deletions(-)

diff --git a/tensorflow/core/common_runtime/placer.cc b/tensorflow/core/common_runtime/placer.cc
index 54f03ab858..8c0a3fb0c5 100644
--- a/tensorflow/core/common_runtime/placer.cc
+++ b/tensorflow/core/common_runtime/placer.cc
@@ -408,6 +408,44 @@ class ColocationGraph {
     return Status::OK();
   }
 
+  // Limits the possible devices of `node`'s colocation group to the device
+  // to which `node` is assigned. This makes sure that all nodes in this
+  // colocation group will be assigned to the same device. Without this
+  // explicit restriction, heuristics can choose a different possible device
+  // for other nodes in the group.
+  Status LimitToAssignedDevice(const Node& node) {
+    if (node.assigned_device_name_index() < 0) {
+      return errors::Internal(
+          "Expected an assigned node as argument to LimitToAssignedDevice but "
+          "got: ",
+          node.DebugString());
+    }
+    int root = FindRoot(node.id());
+    Member& root_member = members_[root];
+    if (node.assigned_device_name_index() ==
+        root_member.assigned_device_name_index) {
+      return Status::OK();
+    }
+    DeviceNameUtils::ParsedName parsed;
+    DeviceNameUtils::ParseFullName(node.assigned_device_name(), &parsed);
+    Status s = DeviceNameUtils::MergeDevNames(&root_member.device_name, parsed,
+                                              allow_soft_placement_);
+    if (!s.ok()) {
+      return errors::Internal(
+          "Constraining by assigned device should not cause an error. Original "
+          "root device name: ",
+          DeviceNameUtils::ParsedNameToString(root_member.device_name),
+          " assigned device name \"", node.assigned_device_name(),
+          ". Error: ", s.error_message());
+    }
+
+    root_member.assigned_device_name_index = node.assigned_device_name_index();
+    // Clear cached possible_devices, if any.
+    root_member.possible_devices.clear();
+
+    return Status::OK();
+  }
+
   // For the given node, subject to the constraints previously given
   // to this ColocationGraph, set its assigned_device_name. Returns OK
   // if a satisfying device can be found, otherwise an error.
@@ -587,6 +625,16 @@ class ColocationGraph {
     // those of all of its children.
     DeviceNameUtils::ParsedName device_name;
 
+    // Once colocation groups have been formed and we assigned at least
+    // one node in this group to a device, assigned_device_name_index will
+    // contain this device name's index in the graph. The `device_name` will
+    // contain the parsed name of this device and `possible_devices`, if
+    // computed, will contain just this device.
+    // `assigned_device_name_index` is an optimization to avoid parsing and
+    // comparing device names. The value of -1 signals that a single device
+    // has not been chosen yet.
+    int assigned_device_name_index = -1;
+
     // If this node is a root, stores a list of Devices to which this node
     // and all of its children have been assigned, or nullptr if this
     // has not yet been computed.
@@ -891,6 +939,30 @@ class ColocationGraph {
   const bool allow_soft_placement_;
   const bool log_device_placement_;
 };
+
+void LogDeviceAssignment(const Node* node, bool log_device_placement) {
+  // Log placement if log_device_placement is set.
+  if (log_device_placement) {
+    printf("%s: (%s): %s\n", node->name().c_str(), node->type_string().c_str(),
+           node->assigned_device_name().c_str());
+    LOG(INFO) << node->name() << ": "
+              << "(" << node->type_string() << ")"
+              << node->assigned_device_name();
+  }
+}
+
+Status AssignAndLog(int assigned_device, Node* node,
+                    ColocationGraph* colocation_graph,
+                    bool log_device_placement) {
+  node->set_assigned_device_name_index(assigned_device);
+
+  // Constraint the group of node to the assigned device.
+  TF_RETURN_IF_ERROR(colocation_graph->LimitToAssignedDevice(*node));
+
+  LogDeviceAssignment(node, log_device_placement);
+  return Status::OK();
+}
+
 }  // namespace
 
 Placer::Placer(Graph* graph, const DeviceSet* devices,
@@ -927,7 +999,7 @@ Status Placer::Run() {
     // devices (e.g., for stateful placements), so the placer should not try to
     // place nodes that are already placed.
     if (node->has_assigned_device_name()) {
-      LogDeviceAssignment(node);
+      LogDeviceAssignment(node, log_device_placement_);
       continue;
     }
 
@@ -983,7 +1055,8 @@ Status Placer::Run() {
       assigned_device = graph_->InternDeviceName((*devices)[0]->name());
     }
 
-    AssignAndLog(assigned_device, node);
+    TF_RETURN_IF_ERROR(AssignAndLog(assigned_device, node, &colocation_graph,
+                                    log_device_placement_));
   }
 
   // Perform a second pass assignment for those nodes explicitly
@@ -1022,7 +1095,8 @@ Status Placer::Run() {
       assigned_device = graph_->InternDeviceName((*devices)[0]->name());
     }
 
-    AssignAndLog(assigned_device, node);
+    TF_RETURN_IF_ERROR(AssignAndLog(assigned_device, node, &colocation_graph,
+                                    log_device_placement_));
   }
 
   return Status::OK();
@@ -1045,20 +1119,4 @@ bool Placer::CanAssignToDevice(const string& candidate_device_name,
   return false;
 }
 
-void Placer::AssignAndLog(int assigned_device, Node* node) const {
-  node->set_assigned_device_name_index(assigned_device);
-  LogDeviceAssignment(node);
-}
-
-void Placer::LogDeviceAssignment(const Node* node) const {
-  // Log placement if log_device_placement is set.
-  if (log_device_placement_) {
-    printf("%s: (%s): %s\n", node->name().c_str(), node->type_string().c_str(),
-           node->assigned_device_name().c_str());
-    LOG(INFO) << node->name() << ": "
-              << "(" << node->type_string() << ")"
-              << node->assigned_device_name();
-  }
-}
-
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/placer.h b/tensorflow/core/common_runtime/placer.h
index e3e8f3790c..e6c5a89c71 100644
--- a/tensorflow/core/common_runtime/placer.h
+++ b/tensorflow/core/common_runtime/placer.h
@@ -88,11 +88,6 @@ class Placer {
   bool CanAssignToDevice(const string& candidate_device_name,
                          const std::vector<Device*>& devices) const;
 
-  // Assigns 'node's devices to 'assigned_device', and logs the
-  // placement if the SessionOptions entry in 'options_' requests it.
-  void AssignAndLog(int assigned_device, Node* node) const;
-  void LogDeviceAssignment(const Node* node) const;
-
   Graph* const graph_;              // Not owned.
   const DeviceSet* const devices_;  // Not owned.
   const SessionOptions* options_;   // Not owned.
-- 
GitLab


From 95f8fd4f30b1e59ea3cb64fbf0036bd7474842cc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 30 Jan 2019 11:06:18 -0800
Subject: [PATCH 1598/2345] Add support for attributes in TFE_AddEagerOpToGraph
 API.

PiperOrigin-RevId: 231629067
---
 tensorflow/c/c_api_experimental.cc      |  17 ++--
 tensorflow/c/c_api_experimental.h       |  11 +--
 tensorflow/c/c_api_experimental_test.cc | 125 +++++++++++++++++++-----
 3 files changed, 119 insertions(+), 34 deletions(-)

diff --git a/tensorflow/c/c_api_experimental.cc b/tensorflow/c/c_api_experimental.cc
index 6cc74cfb32..a8325ce494 100644
--- a/tensorflow/c/c_api_experimental.cc
+++ b/tensorflow/c/c_api_experimental.cc
@@ -9053,9 +9053,9 @@ static TF_Output getOrCreateSymbolicTensor(TFE_TraceContext* trace_ctx,
   return ret;
 }
 
-void TFE_AddEagerOpToGraph(TFE_Op* op, TFE_TraceContext* trace_ctx,
-                           TFE_TensorHandle** retvals, int* num_retvals,
-                           TF_Status* status) {
+TF_Operation* TFE_AddEagerOpToGraph(TFE_Op* op, TFE_TraceContext* trace_ctx,
+                                    TFE_TensorHandle** retvals,
+                                    int* num_retvals, TF_Status* status) {
   VLOG(1) << "Calling TFE_AddEagerOpToGraph() with op " << op << ": "
           << op->operation.DebugString();
 
@@ -9066,15 +9066,19 @@ void TFE_AddEagerOpToGraph(TFE_Op* op, TFE_TraceContext* trace_ctx,
       TF_NewOperation(trace_ctx->graph, op_type.c_str(), op_name.c_str());
   for (auto* input : op->operation.Inputs()) {
     auto symbolic_input = getOrCreateSymbolicTensor(trace_ctx, input, status);
-    if (!status->status.ok()) return;
+    if (!status->status.ok()) return nullptr;
     TF_AddInput(desc, symbolic_input);
   }
 
   VLOG(1) << "Adding attrs.";
-  // TODO(hongm): add attrs
+  tensorflow::AttrValueMap attrs;
+  op->operation.Attrs().FillAttrValueMap(&attrs);
+  for (const auto& attr : attrs) {
+    desc->node_builder.Attr(attr.first, attr.second);
+  }
 
   auto* graph_op = TF_FinishOperation(desc, status);
-  if (!status->status.ok()) return;
+  if (!status->status.ok()) return nullptr;
 
   VLOG(1) << "Op finalized; setting return tensors.";
   *num_retvals = TF_OperationNumOutputs(graph_op);
@@ -9084,6 +9088,7 @@ void TFE_AddEagerOpToGraph(TFE_Op* op, TFE_TraceContext* trace_ctx,
     auto dtype = TF_OperationOutputType(output);
     retvals[i] = TFE_NewTensorHandleFromTFOutput(output, dtype);
   }
+  return graph_op;
 }
 
 int TFE_FinalizeInputTensorsFromTraceContext(TFE_TraceContext* trace_ctx) {
diff --git a/tensorflow/c/c_api_experimental.h b/tensorflow/c/c_api_experimental.h
index 48ea0ec1ed..8d1a8b82fb 100644
--- a/tensorflow/c/c_api_experimental.h
+++ b/tensorflow/c/c_api_experimental.h
@@ -294,12 +294,11 @@ TF_CAPI_EXPORT extern void TFE_DeleteTraceContext(TFE_TraceContext* trace_ctx);
 
 // Symbolically executes `op`, by adding a corresponding node to the graph
 // associated with `trace_ctx`. This graph node outputs a set of symbolic
-// tensors in `retvals` and `num_retvals`.
-TF_CAPI_EXPORT extern void TFE_AddEagerOpToGraph(TFE_Op* op,
-                                                 TFE_TraceContext* trace_ctx,
-                                                 TFE_TensorHandle** retvals,
-                                                 int* num_retvals,
-                                                 TF_Status* status);
+// tensors in `retvals` and `num_retvals`. Returns the corresponding graph
+// operation on success, otherwise returns nullptr.
+TF_CAPI_EXPORT extern TF_Operation* TFE_AddEagerOpToGraph(
+    TFE_Op* op, TFE_TraceContext* trace_ctx, TFE_TensorHandle** retvals,
+    int* num_retvals, TF_Status* status);
 
 // Finalizes the trace graph and its inputs, and returns the number of inputs.
 // After this call, the next two APIs can be called to iterate over the input
diff --git a/tensorflow/c/c_api_experimental_test.cc b/tensorflow/c/c_api_experimental_test.cc
index 4cfcf2ef3b..354ee5f49f 100644
--- a/tensorflow/c/c_api_experimental_test.cc
+++ b/tensorflow/c/c_api_experimental_test.cc
@@ -319,50 +319,131 @@ TEST(CAPI_EXPERIMENTAL, SymbolicTensor) {
   TF_DeleteStatus(status);
 }
 
-TEST(CAPI_EXPERIMENTAL, DebugPrintAndSymbolicExecution) {
-  TF_Status* status = TF_NewStatus();
-  TFE_ContextOptions* opts = TFE_NewContextOptions();
-  CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
-  TFE_Context* ctx = TFE_NewContext(opts, status);
-  CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
-  TFE_DeleteContextOptions(opts);
+class AddEagerOpToGraphTest : public ::testing::Test {
+ protected:
+  AddEagerOpToGraphTest()
+      : status_(TF_NewStatus()),
+        eager_ctx_(nullptr),
+        graph_(TF_NewGraph()),
+        trace_ctx_(TFE_NewTraceContext(graph_)) {
+    TFE_ContextOptions* opts = TFE_NewContextOptions();
+    CHECK_EQ(TF_OK, TF_GetCode(status_)) << TF_Message(status_);
+    eager_ctx_ = TFE_NewContext(opts, status_);
+    CHECK_EQ(TF_OK, TF_GetCode(status_)) << TF_Message(status_);
+    TFE_DeleteContextOptions(opts);
+  }
+
+  ~AddEagerOpToGraphTest() override {
+    TFE_DeleteTraceContext(trace_ctx_);
+    TF_DeleteGraph(graph_);
+    TFE_DeleteContext(eager_ctx_);
+    TF_DeleteStatus(status_);
+  }
+
+  template <typename Callable>
+  void AddEagerOpToGraphAndCheck(TFE_Op* op, Callable checker) {
+    TFE_TensorHandle* retvals[5];
+    int num_retvals = 5;
+    // Symbolically execute this op, which adds a graph node to `trace_ctx_`.
+    TF_Operation* graph_op =
+        TFE_AddEagerOpToGraph(op, trace_ctx_, retvals, &num_retvals, status_);
+    CHECK_EQ(TF_OK, TF_GetCode(status_)) << TF_Message(status_);
+    CHECK_NOTNULL(graph_op);
+    // Check the expectations.
+    checker(graph_op);
+    for (int i = 0; i < num_retvals; ++i) {
+      TFE_DeleteTensorHandle(retvals[i]);
+    }
+  }
+
+  TF_Status* status_;
+  TFE_Context* eager_ctx_;
+  TF_Graph* graph_;
+  TFE_TraceContext* trace_ctx_;
+};
 
+TEST_F(AddEagerOpToGraphTest, DebugPrintAndSymbolicExecution) {
   TFE_TensorHandle* m = TestMatrixTensorHandle();
-  TFE_Op* op = MatMulOp(ctx, m, m);
+  TFE_Op* op = MatMulOp(eager_ctx_, m, m);
 
-  CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  CHECK_EQ(TF_OK, TF_GetCode(status_)) << TF_Message(status_);
   TFE_OpPrintDebugString(op);
 
-  auto* graph = TF_NewGraph();
-  auto* trace_ctx = TFE_NewTraceContext(graph);
   TFE_TensorHandle* retvals[5];
   int num_retvals = 5;
   // Symbolically execute this op, which adds a graph node to `trace_ctx`.
-  TFE_AddEagerOpToGraph(op, trace_ctx, retvals, &num_retvals, status);
-  CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  TFE_AddEagerOpToGraph(op, trace_ctx_, retvals, &num_retvals, status_);
+  CHECK_EQ(TF_OK, TF_GetCode(status_)) << TF_Message(status_);
 
-  int num_inputs = TFE_FinalizeInputTensorsFromTraceContext(trace_ctx);
+  int num_inputs = TFE_FinalizeInputTensorsFromTraceContext(trace_ctx_);
   CHECK_EQ(num_inputs, 1);
-  auto input_sym_tensor = TFE_GetInputGraphNodeFromTraceContext(trace_ctx,
+  auto input_sym_tensor = TFE_GetInputGraphNodeFromTraceContext(trace_ctx_,
                                                                 /*idx*/ 0);
 
   LOG(INFO) << tensorflow::getTF_OutputDebugString(input_sym_tensor);
-  auto handle = TFE_ConsumeInputConcreteTensorFromTraceContext(trace_ctx,
+  auto handle = TFE_ConsumeInputConcreteTensorFromTraceContext(trace_ctx_,
                                                                /*idx*/ 0);
   TFE_TensorHandlePrintDebugString(handle);
   TFE_DeleteTensorHandle(handle);
 
   CHECK_EQ(num_retvals, 1);
   CHECK_EQ(TFE_TensorHandleDataType(retvals[0]), TF_FLOAT);
-  TFE_DeleteTensorHandle(retvals[0]);
-
-  TFE_DeleteTraceContext(trace_ctx);
-  TF_DeleteGraph(graph);
 
+  TFE_DeleteTensorHandle(retvals[0]);
   TFE_DeleteTensorHandle(m);
   TFE_DeleteOp(op);
-  TFE_DeleteContext(ctx);
-  TF_DeleteStatus(status);
+}
+
+TEST_F(AddEagerOpToGraphTest, ValueAttributesArePreserved) {
+  // Create MinOp
+  TFE_TensorHandle* axis = TestAxisTensorHandle();
+  TFE_Op* op = MinOp(eager_ctx_, axis, axis);
+  CHECK_EQ(TF_OK, TF_GetCode(status_)) << TF_Message(status_);
+
+  // Check the attributes set by the call to MinOp above.
+  AddEagerOpToGraphAndCheck(op, [this, &axis](TF_Operation* graph_op) {
+    unsigned char value;
+    TF_OperationGetAttrBool(graph_op, "keep_dims", &value, status_);
+    CHECK_EQ(TF_OK, TF_GetCode(status_)) << TF_Message(status_);
+    CHECK_EQ(value, 1);
+    TF_DataType dtype;
+    TF_OperationGetAttrType(graph_op, "Tidx", &dtype, status_);
+    CHECK_EQ(TF_OK, TF_GetCode(status_)) << TF_Message(status_);
+    CHECK_EQ(dtype, TF_INT32);
+    TF_OperationGetAttrType(graph_op, "T", &dtype, status_);
+    CHECK_EQ(TF_OK, TF_GetCode(status_)) << TF_Message(status_);
+    CHECK_EQ(dtype, TFE_TensorHandleDataType(axis));
+  });
+  TFE_DeleteTensorHandle(axis);
+  TFE_DeleteOp(op);
+}
+
+TEST_F(AddEagerOpToGraphTest, ListAttributesArePreserved) {
+  // Create a "Squeeze" operator with list attributes.
+  TFE_TensorHandle* axis = TestAxisTensorHandle();
+  TFE_Op* squeeze = TFE_NewOp(eager_ctx_, "Squeeze", status_);
+  CHECK_EQ(TF_OK, TF_GetCode(status_)) << TF_Message(status_);
+  TFE_OpAddInput(squeeze, axis, status_);
+  TFE_OpSetAttrType(squeeze, "T", TF_INT32);
+  std::vector<int64_t> boundaries = {1, 2, 3, 4};
+  TFE_OpSetAttrIntList(squeeze, "squeeze_dims", boundaries.data(),
+                       boundaries.size());
+  // Check attributes are preserved.
+  AddEagerOpToGraphAndCheck(
+      squeeze, [this, &boundaries](TF_Operation* squeeze_graph_op) {
+        TF_DataType dtype;
+        TF_OperationGetAttrType(squeeze_graph_op, "T", &dtype, status_);
+        CHECK_EQ(TF_OK, TF_GetCode(status_)) << TF_Message(status_);
+        CHECK_EQ(dtype, TF_INT32);
+        std::unique_ptr<int64_t[]> list(new int64_t[boundaries.size()]);
+        TF_OperationGetAttrIntList(squeeze_graph_op, "squeeze_dims", list.get(),
+                                   boundaries.size(), status_);
+        CHECK_EQ(TF_OK, TF_GetCode(status_)) << TF_Message(status_);
+        EXPECT_TRUE(std::equal(list.get(), list.get() + boundaries.size(),
+                               boundaries.begin()));
+      });
+  TFE_DeleteTensorHandle(axis);
+  TFE_DeleteOp(squeeze);
 }
 
 }  // namespace
-- 
GitLab


From 7109e400cb5c5a81f62356fb2b104aaf45a2f66c Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Wed, 30 Jan 2019 11:06:45 -0800
Subject: [PATCH 1599/2345] Reduce the priority of def_function's variable
 creator

Allows creators outside a function (e.g. distribution strategies) to create custom variable classes, even if this prevents tracing the initialization graph.

PiperOrigin-RevId: 231629175
---
 tensorflow/python/eager/def_function.py      | 16 ++++++++++++++--
 tensorflow/python/eager/def_function_test.py | 20 ++++++++++++++++++++
 tensorflow/python/framework/ops.py           | 20 ++++++++++++++++++--
 tensorflow/python/ops/variables.py           |  4 ++--
 4 files changed, 54 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py
index dea71282f5..897a38e7d3 100644
--- a/tensorflow/python/eager/def_function.py
+++ b/tensorflow/python/eager/def_function.py
@@ -31,7 +31,6 @@ from tensorflow.python.framework import ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
-from tensorflow.python.ops import variable_scope
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training.checkpointable import base as checkpointable
 from tensorflow.python.util import nest
@@ -289,7 +288,20 @@ class Function(object):
 
     weak_wrapped_fn = None
     def wrapped_fn(*args, **kwds):
-      with variable_scope.variable_creator_scope(scope):
+      """Wraps `self._python_function` in a variable creator scope."""
+      # We register a variable creator with reduced priority. If an outer
+      # variable creator is just modifying keyword arguments to the variable
+      # constructor, this will work harmoniously. Since the `scope` registered
+      # here actually creates the variable, it taking priority would otherwise
+      # ignore the outer creator.
+      #
+      # If an outer variable creator calls the variable constructor manually,
+      # for example creating a MirroredVariable, then they won't call our
+      # creator. This means we won't be able to trace the initialization graph,
+      # and so variable initializers can't depend on function arguments. This is
+      # better than the alternative, tracing the initialization graph but giving
+      # the user a variable type they didn't want.
+      with ops.get_default_graph()._variable_creator_scope(scope, priority=50):  # pylint: disable=protected-access
         # __wrapped__ allows AutoGraph to swap in a converted function. We give
         # the function a weak reference to itself to avoid a reference cycle.
         return weak_wrapped_fn().__wrapped__(*args, **kwds)
diff --git a/tensorflow/python/eager/def_function_test.py b/tensorflow/python/eager/def_function_test.py
index b777ead913..912198dfcf 100644
--- a/tensorflow/python/eager/def_function_test.py
+++ b/tensorflow/python/eager/def_function_test.py
@@ -31,6 +31,7 @@ from tensorflow.python.keras.engine import training
 from tensorflow.python.keras.layers import core
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 from tensorflow.python.training import adam
@@ -323,6 +324,25 @@ class DefFunctionTest(test.TestCase):
     with self.assertRaisesRegexp(TypeError, MIXING_GRAPH_EAGER_TENSORS_ERROR):
       failing_function()
 
+  def testVariableCreatorScope(self):
+    created_variables = []
+    captured_variables = []
+
+    @def_function.function
+    def f():
+      if not created_variables:
+        created_variables.append(variables.Variable(1.))
+      return created_variables[0] + 1.
+
+    def capture_creator(next_creator, **kwargs):
+      created = next_creator(**kwargs)
+      captured_variables.append(created)
+      return created
+
+    with variable_scope.variable_creator_scope(capture_creator):
+      f()
+    self.assertEqual(created_variables, captured_variables)
+
 
 if __name__ == '__main__':
   ops.enable_eager_execution()
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 07d1e72e29..e74b43f93e 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -3070,11 +3070,27 @@ class Graph(object):
   # Note: this method is private because the API of tf.Graph() is public and
   # frozen, and this functionality is still not ready for public visibility.
   @tf_contextlib.contextmanager
-  def _variable_creator_scope(self, creator):
+  def _variable_creator_scope(self, creator, priority=100):
+    """Scope which defines a variable creation function.
+
+    Args:
+      creator: A callable taking `next_creator` and `kwargs`. See the
+        `tf.variable_creator_scope` docstring.
+      priority: Creators with a higher `priority` are called first. Within the
+        same priority, creators are called inner-to-outer.
+
+    Yields:
+      `_variable_creator_scope` is a context manager with a side effect, but
+      doesn't return a value.
+    """
     # This step makes a copy of the existing stack, and it also initializes
     # self._thread_local._variable_creator_stack if it doesn't exist yet.
     old = list(self._variable_creator_stack)
-    self._thread_local._variable_creator_stack.append(creator)  # pylint: disable=protected-access
+    stack = self._thread_local._variable_creator_stack  # pylint: disable=protected-access
+    stack.append((priority, creator))
+    # Sorting is stable, so we'll put higher-priority creators later in the list
+    # but otherwise maintain registration order.
+    stack.sort(key=lambda item: item[0])
     try:
       yield
     finally:
diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index 236a1cd537..7788f28bc3 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -138,7 +138,7 @@ class VariableMetaclass(type):
                         aggregation=VariableAggregation.NONE):
     """Call on Variable class. Useful to force the signature."""
     previous_getter = lambda **kwargs: default_variable_creator(None, **kwargs)
-    for getter in ops.get_default_graph()._variable_creator_stack:  # pylint: disable=protected-access
+    for _, getter in ops.get_default_graph()._variable_creator_stack:  # pylint: disable=protected-access
       previous_getter = _make_getter(getter, previous_getter)
 
     # Reset `aggregation` that is explicitly set as `None` to the enum NONE.
@@ -174,7 +174,7 @@ class VariableMetaclass(type):
                         aggregation=VariableAggregation.NONE):
     """Call on Variable class. Useful to force the signature."""
     previous_getter = lambda **kws: default_variable_creator_v2(None, **kws)
-    for getter in ops.get_default_graph()._variable_creator_stack:  # pylint: disable=protected-access
+    for _, getter in ops.get_default_graph()._variable_creator_stack:  # pylint: disable=protected-access
       previous_getter = _make_getter(getter, previous_getter)
 
     # Reset `aggregation` that is explicitly set as `None` to the enum NONE.
-- 
GitLab


From f4b2b6876c55a2abc9a6fc2c1f664f8f3951b34f Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Wed, 30 Jan 2019 11:18:32 -0800
Subject: [PATCH 1600/2345] Kill error output from jpeg parsing while fuzzing

PiperOrigin-RevId: 231631615
---
 tensorflow/core/lib/jpeg/jpeg_mem.cc | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/lib/jpeg/jpeg_mem.cc b/tensorflow/core/lib/jpeg/jpeg_mem.cc
index 9e7d1e6410..1b54caf28e 100644
--- a/tensorflow/core/lib/jpeg/jpeg_mem.cc
+++ b/tensorflow/core/lib/jpeg/jpeg_mem.cc
@@ -81,6 +81,12 @@ bool IsCropWindowValid(const UncompressFlags& flags, int input_image_width,
          flags.crop_x + flags.crop_width <= input_image_width;
 }
 
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+// If in fuzzing mode, don't print any error message as that slows down fuzzing.
+// See also http://llvm.org/docs/LibFuzzer.html#fuzzer-friendly-build-mode
+void no_print(j_common_ptr cinfo) {}
+#endif
+
 uint8* UncompressLow(const void* srcdata, FewerArgsForCompiler* argball) {
   // unpack the argball
   const int datasize = argball->datasize_;
@@ -112,9 +118,14 @@ uint8* UncompressLow(const void* srcdata, FewerArgsForCompiler* argball) {
   struct jpeg_decompress_struct cinfo;
   struct jpeg_error_mgr jerr;
   cinfo.err = jpeg_std_error(&jerr);
+  jerr.error_exit = CatchError;
+
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+  jerr.output_message = no_print;
+#endif
+
   jmp_buf jpeg_jmpbuf;
   cinfo.client_data = &jpeg_jmpbuf;
-  jerr.error_exit = CatchError;
   if (setjmp(jpeg_jmpbuf)) {
     delete[] tempdata;
     return nullptr;
-- 
GitLab


From a376c96786ffbce5fece1d301ad42c0537031bb7 Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Wed, 30 Jan 2019 11:18:49 -0800
Subject: [PATCH 1601/2345] Optimize Mirror pad op.

PiperOrigin-RevId: 231631681
---
 tensorflow/lite/kernels/mirror_pad.cc        | 71 ++++++++++++++------
 tensorflow/lite/testing/generate_examples.py |  8 ++-
 2 files changed, 59 insertions(+), 20 deletions(-)

diff --git a/tensorflow/lite/kernels/mirror_pad.cc b/tensorflow/lite/kernels/mirror_pad.cc
index e74e47f7a3..70f7e33928 100644
--- a/tensorflow/lite/kernels/mirror_pad.cc
+++ b/tensorflow/lite/kernels/mirror_pad.cc
@@ -39,8 +39,8 @@ struct PaddedTensor {
   const void* value = nullptr;
   // If this tensor is not one value, then this vector will have
   // all the tensors that belongs to this tensor.
-  // Pointers are owned.
-  std::vector<std::unique_ptr<PaddedTensor>> values;
+  // Pointers are not owned.
+  std::vector<PaddedTensor*> values;
   // Pointers to PaddedTensors that are padded on the left of the current
   // tensor.
   std::vector<PaddedTensor*> left_pad_ptrs;
@@ -55,25 +55,41 @@ struct PaddedTensor {
       if (indices[i] >= result->values.size()) {
         return nullptr;
       }
-      result = result->values[indices[i]].get();
+      result = result->values[indices[i]];
       if (result == nullptr) break;
     }
     return result;
   }
 };
 
+// Wrapper for all intermediate data used by the op.
+struct OpData {
+  // Holds intermediate data structure of the padded tensor.
+  std::vector<PaddedTensor> pad_tensor_buffer;
+  // Total number of intermediate elements in the pad_tensor_buffer.
+  int num_elements;
+};
+
 // Util method to initialize the memory of the padded tensor.
-void InitializeTensorMemory(const TfLiteIntArray* const dims, int dim_index,
-                            int dims_size, PaddedTensor* padded_tensor) {
+// Returns the index of the current item processed in 'padded_tensor_buffer'
+int InitializeTensorMemory(const TfLiteIntArray* const dims, int dim_index,
+                           int dims_size,
+                           std::vector<PaddedTensor>* padded_tensor_buffer,
+                           int element_index, int num_elements) {
   if (dim_index >= dims_size) {
-    return;
+    return element_index;
   }
+  PaddedTensor* padded_tensor = &(*padded_tensor_buffer)[element_index];
+  padded_tensor->values.clear();
   padded_tensor->values.reserve(dims->data[dim_index]);
   for (int i = 0; i < dims->data[dim_index]; ++i) {
-    padded_tensor->values.emplace_back(new PaddedTensor());
-    InitializeTensorMemory(dims, dim_index + 1, dims_size,
-                           padded_tensor->values.back().get());
+    ++element_index;
+    padded_tensor->values.emplace_back(&(*padded_tensor_buffer)[element_index]);
+    element_index = InitializeTensorMemory(dims, dim_index + 1, dims_size,
+                                           padded_tensor_buffer, element_index,
+                                           num_elements);
   }
+  return element_index;
 }
 
 // Returns pointer to the value at the specified index in 'data'.
@@ -191,7 +207,7 @@ TfLiteStatus ValidateTensor(const TfLiteTensor* padding_matrix, int offset,
   }
   if (!padded_tensor->values.empty()) {
     ValidateTensor(padding_matrix, offset, dimension_index + 1,
-                   padded_tensor->values[0].get(), context);
+                   padded_tensor->values[0], context);
   }
   return kTfLiteOk;
 }
@@ -208,18 +224,20 @@ TfLiteStatus PadTensor(const TfLiteTensor* padding_matrix, int offset,
   TF_LITE_ENSURE_STATUS(
       GetPadding(padding_matrix, dimension_index, &left_pad, &right_pad));
 
+  padded_tensor->left_pad_ptrs.clear();
   for (int i = left_pad + offset - 1; i >= offset && left_pad > 0;
        --i, --left_pad) {
-    padded_tensor->left_pad_ptrs.push_back(padded_tensor->values[i].get());
+    padded_tensor->left_pad_ptrs.push_back(padded_tensor->values[i]);
   }
+  padded_tensor->right_pad_ptrs.clear();
   for (int i = padded_tensor->values.size() - (1 + offset);
        i >= 0 && right_pad > 0; --i, --right_pad) {
-    padded_tensor->right_pad_ptrs.push_back(padded_tensor->values[i].get());
+    padded_tensor->right_pad_ptrs.push_back(padded_tensor->values[i]);
   }
 
   for (auto& tensor : padded_tensor->values) {
     TF_LITE_ENSURE_STATUS(PadTensor(padding_matrix, offset, dimension_index + 1,
-                                    tensor.get(), context));
+                                    tensor, context));
   }
   return kTfLiteOk;
 }
@@ -241,7 +259,7 @@ int FillOutput(const PaddedTensor* padded_tensor, T* output_data,
     index_in_output = FillOutput(tensor, output_data, index_in_output);
   }
   for (const auto& tensor : padded_tensor->values) {
-    index_in_output = FillOutput(tensor.get(), output_data, index_in_output);
+    index_in_output = FillOutput(tensor, output_data, index_in_output);
   }
   for (const auto* tensor : padded_tensor->right_pad_ptrs) {
     index_in_output = FillOutput(tensor, output_data, index_in_output);
@@ -271,6 +289,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   const TfLiteTensor* padding_matrix = GetInput(context, node, 1);
   auto* params =
       reinterpret_cast<TfLiteMirrorPaddingParams*>(node->builtin_data);
+  OpData* op_data = reinterpret_cast<OpData*>(node->user_data);
 
   if (params == nullptr) {
     return kTfLiteError;
@@ -287,12 +306,12 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
         context->ResizeTensor(context, output_tensor, output_size.release()));
   }
 
-  PaddedTensor padded_tensor;
+  PaddedTensor& padded_tensor = op_data->pad_tensor_buffer[0];
   // Initialize memory.
-  InitializeTensorMemory(input_tensor->dims, 0, input_dims, &padded_tensor);
+  InitializeTensorMemory(input_tensor->dims, 0, input_dims,
+                         &op_data->pad_tensor_buffer, 0, op_data->num_elements);
   // Set the values from the input_tensor.
   TF_LITE_ENSURE_STATUS(InitFromInputTensor(input_tensor, &padded_tensor));
-
   const int offset =
       params->mode != TfLiteMirrorPaddingMode::kTfLiteMirrorPaddingReflect ? 0
                                                                            : 1;
@@ -335,20 +354,34 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
 }
 
 void* Init(TfLiteContext* context, const char* buffer, size_t length) {
-  return nullptr;
+  return new OpData();
 }
 
-void Free(TfLiteContext* context, void* buffer) {}
+void Free(TfLiteContext* context, void* buffer) {
+  delete reinterpret_cast<OpData*>(buffer);
+}
 
 TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   const TfLiteTensor* input_tensor = GetInput(context, node, 0);
   const TfLiteTensor* padding_matrix = GetInput(context, node, 1);
   TfLiteTensor* output_tensor = GetOutput(context, node, 0);
+  OpData* op_data = reinterpret_cast<OpData*>(node->user_data);
 
   TF_LITE_ENSURE_EQ(context, NumDimensions(padding_matrix), 2);
   TF_LITE_ENSURE_EQ(context, SizeOfDimension(padding_matrix, 0),
                     NumDimensions(input_tensor));
 
+  // Calculate total number of nodes in the tree structure of a tensor
+  // and pre-allocates it.
+  int num_elements = NumElements(input_tensor) + 1;
+  int extra_nodes = 1;
+  for (int i = 0; i < NumDimensions(input_tensor) - 1; ++i) {
+    extra_nodes *= input_tensor->dims->data[i];
+    num_elements += extra_nodes;
+  }
+  op_data->pad_tensor_buffer.resize(num_elements);
+  op_data->num_elements = num_elements;
+
   if (!IsConstantTensor(padding_matrix)) {
     SetTensorToDynamic(output_tensor);
     return kTfLiteOk;
diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index 3aaaf545df..ee68607a0f 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -3757,6 +3757,12 @@ def make_mirror_pad_tests(zip_path):
           "mode": ["REFLECT"],
           "type": ["const"]
       },
+      {
+          "input_shape": [[3, 2, 4, 5]],
+          "padding_matrix": [[[1, 1], [2, 2], [1, 1], [1, 1]]],
+          "mode": ["SYMMETRIC"],
+          "type": ["placeholder"]
+      },
   ]
 
   def build_graph(parameters):
@@ -3790,7 +3796,7 @@ def make_mirror_pad_tests(zip_path):
       test_parameters,
       build_graph,
       build_inputs,
-      expected_tf_success=7)
+      expected_tf_success=8)
 
 
 def make_unroll_batch_matmul_tests(zip_path):
-- 
GitLab


From 9f7901630972ca5e53441c81f628c18d1137ab08 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Wed, 30 Jan 2019 11:42:16 -0800
Subject: [PATCH 1602/2345] [XLA:CPU] Fix fast-math flags in llvm_ir_runtime.cc

Previously we were calling setFastMath(), but we should have been calling
setFastMath(fast_math_enabled).

Also disable fastmath in the exhaustive f32 elementwise op test.

PiperOrigin-RevId: 231636911
---
 tensorflow/compiler/xla/service/cpu/llvm_ir_runtime.cc        | 4 ++--
 .../compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc  | 4 +++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/llvm_ir_runtime.cc b/tensorflow/compiler/xla/service/cpu/llvm_ir_runtime.cc
index f9722ffada..a246494c6c 100644
--- a/tensorflow/compiler/xla/service/cpu/llvm_ir_runtime.cc
+++ b/tensorflow/compiler/xla/service/cpu/llvm_ir_runtime.cc
@@ -83,7 +83,7 @@ llvm::Function* EmitVectorF32ExpIfNeeded(llvm::Module* module,
 
   llvm::IRBuilder<> b(vector_exp_body);
   llvm::FastMathFlags fast_math_flags;
-  fast_math_flags.setFast();
+  fast_math_flags.setFast(enable_fast_math);
   b.setFastMathFlags(fast_math_flags);
 
   VectorSupportLibrary vsl(F32, vector_width, &b, "exp_f32");
@@ -166,7 +166,7 @@ llvm::Function* EmitVectorF32LogIfNeeded(llvm::Module* module,
 
   llvm::IRBuilder<> b(vector_log_body);
   llvm::FastMathFlags fast_math_flags;
-  fast_math_flags.setFast();
+  fast_math_flags.setFast(enable_fast_math);
   b.setFastMathFlags(fast_math_flags);
 
   llvm::Value* input = &*vector_log_function->arg_begin();
diff --git a/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc b/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc
index 11b06c7355..87e912fa28 100644
--- a/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc
+++ b/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc
@@ -25,12 +25,14 @@ class ExhaustiveF32ElementwiseOpTest
     : public ClientLibraryTestBase,
       public ::testing::WithParamInterface<std::pair<int64, int64>> {
  protected:
-  ErrorSpec error_spec_{0.0001, 0.0001, /*relaxed_nans=*/true};
+  ErrorSpec error_spec_{0.0001, 0.0001};
 
   template <typename EnqueueOpTy>
   void ExhaustivelyTestF32Op(EnqueueOpTy enqueue_op,
                              float (*evaluate_op)(float),
                              std::pair<int64, int64> known_incorrect_range) {
+    SetFastMathDisabled(true);
+
     int64 begin, end;
     std::tie(begin, end) = GetParam();
     int64 input_size = end - begin;
-- 
GitLab


From 710b17c12b6cc52a2883f1263c60dc3ce1b6def4 Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Wed, 30 Jan 2019 12:02:48 -0800
Subject: [PATCH 1603/2345] Implement Integer per channel quantization of Conv.

PiperOrigin-RevId: 231640987
---
 tensorflow/lite/kernels/conv.cc               |  69 ++++++++--
 tensorflow/lite/kernels/conv_test.cc          | 102 +++++++++++++-
 tensorflow/lite/kernels/internal/BUILD        |   1 +
 .../internal/reference/integer_ops/conv.h     | 128 ++++++++++++++++++
 4 files changed, 283 insertions(+), 17 deletions(-)
 create mode 100644 tensorflow/lite/kernels/internal/reference/integer_ops/conv.h

diff --git a/tensorflow/lite/kernels/conv.cc b/tensorflow/lite/kernels/conv.cc
index 05368aa5ed..75e75fae6b 100644
--- a/tensorflow/lite/kernels/conv.cc
+++ b/tensorflow/lite/kernels/conv.cc
@@ -27,6 +27,7 @@ limitations under the License.
 #include "tensorflow/lite/kernels/internal/optimized/multithreaded_conv.h"
 #include "tensorflow/lite/kernels/internal/optimized/optimized_ops.h"
 #include "tensorflow/lite/kernels/internal/quantization_util.h"
+#include "tensorflow/lite/kernels/internal/reference/integer_ops/conv.h"
 #include "tensorflow/lite/kernels/internal/reference/reference_ops.h"
 #include "tensorflow/lite/kernels/internal/tensor.h"
 #include "tensorflow/lite/kernels/internal/tensor_utils.h"
@@ -68,6 +69,11 @@ struct OpData {
   // be represented as a fixed point multiplier plus a left shift.
   int32_t output_multiplier;
   int output_shift;
+
+  // Per channel output multiplier and shift.
+  std::vector<int32_t> per_channel_output_multiplier;
+  std::vector<int> per_channel_output_shift;
+
   // The range of the fused activation layer. For example for kNone and
   // uint8_t these would be 0 and 255.
   int32_t output_activation_min;
@@ -226,8 +232,9 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
 
   // Check types. (We assume that UINT8 refers to quantized tensors)
   TfLiteType input_type = input->type;
-  TF_LITE_ENSURE(context,
-                 input_type == kTfLiteFloat32 || input_type == kTfLiteUInt8);
+  TF_LITE_ENSURE(context, input_type == kTfLiteFloat32 ||
+                              input_type == kTfLiteUInt8 ||
+                              input_type == kTfLiteInt8);
   TF_LITE_ENSURE_EQ(context, output->type, input_type);
 
   TfLiteTensor* bias = nullptr;
@@ -238,7 +245,7 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
 
   if (has_bias) {
     bias = &context->tensors[node->inputs->data[2]];
-    if (input_type == kTfLiteUInt8) {
+    if (input_type == kTfLiteUInt8 || input_type == kTfLiteInt8) {
       TF_LITE_ENSURE_EQ(context, bias->type, kTfLiteInt32);
       TF_LITE_ENSURE_EQ(context, bias->params.zero_point, 0);
     } else {
@@ -296,18 +303,25 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   TF_LITE_ENSURE(context, has_bias);
 
   // Note that full fixed-point inference requires that all tensors have their
-  // parameters set. This is usually done during quantized training.
+  // parameters set. This is usually done during quantized training or
+  // calibration.
   if (input_type != kTfLiteFloat32) {
-    double real_multiplier = 0.0;
-    TF_LITE_ENSURE_STATUS(GetQuantizedConvolutionMultipler(
-        context, input, filter, bias, output, &real_multiplier));
-
-    int exponent;
-    QuantizeMultiplier(real_multiplier, &data->output_multiplier, &exponent);
-    data->output_shift = -exponent;
-    CalculateActivationRangeUint8(params->activation, output,
-                                  &data->output_activation_min,
-                                  &data->output_activation_max);
+    TF_LITE_ENSURE_EQ(context, filter->quantization.type,
+                      kTfLiteAffineQuantization);
+    const auto* affine_quantization =
+        reinterpret_cast<TfLiteAffineQuantization*>(
+            filter->quantization.params);
+    TF_LITE_ENSURE(context, affine_quantization);
+    TF_LITE_ENSURE(context, affine_quantization->scale);
+    const int number_channel = affine_quantization->scale->size;
+    data->per_channel_output_multiplier.resize(number_channel);
+    data->per_channel_output_shift.resize(number_channel);
+    TF_LITE_ENSURE_STATUS(tflite::PopulateConvolutionQuantizationParams(
+        context, input, filter, bias, output, params->activation,
+        &data->output_multiplier, &data->output_shift,
+        &data->output_activation_min, &data->output_activation_max,
+        data->per_channel_output_multiplier.data(),
+        data->per_channel_output_shift.data()));
   }
 
   TfLiteIntArray* output_size = TfLiteIntArrayCreate(4);
@@ -481,6 +495,29 @@ void EvalQuantized(TfLiteContext* context, TfLiteNode* node,
   }
 }
 
+void EvalQuantizedPerChannel(TfLiteContext* context, TfLiteNode* node,
+                             TfLiteConvParams* params, OpData* data,
+                             TfLiteTensor* input, TfLiteTensor* filter,
+                             TfLiteTensor* bias, TfLiteTensor* output) {
+  ConvParams op_params;
+  op_params.input_offset = input->params.zero_point;
+  op_params.output_offset = output->params.zero_point;
+  op_params.stride_height = params->stride_height;
+  op_params.stride_width = params->stride_width;
+  op_params.dilation_height_factor = params->dilation_height_factor;
+  op_params.dilation_width_factor = params->dilation_width_factor;
+  op_params.padding_values.height = data->padding.height;
+  op_params.padding_values.width = data->padding.width;
+
+  reference_integer_ops::ConvPerChannel(
+      op_params, data->per_channel_output_multiplier.data(),
+      data->per_channel_output_shift.data(), GetTensorShape(input),
+      GetTensorData<int8>(input), GetTensorShape(filter),
+      GetTensorData<int8>(filter), GetTensorShape(bias),
+      GetTensorData<int32>(bias), GetTensorShape(output),
+      GetTensorData<int8>(output));
+}
+
 template <KernelType kernel_type>
 void EvalFloat(TfLiteContext* context, TfLiteNode* node,
                TfLiteConvParams* params, OpData* data, TfLiteTensor* input,
@@ -665,6 +702,10 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
       EvalQuantized<kernel_type>(context, node, params, data, input, filter,
                                  bias, im2col, hwcn_weights, output);
       break;
+    case kTfLiteInt8:
+      EvalQuantizedPerChannel(context, node, params, data, input, filter, bias,
+                              output);
+      break;
     default:
       context->ReportError(context, "Type %d not currently supported.",
                            input->type);
diff --git a/tensorflow/lite/kernels/conv_test.cc b/tensorflow/lite/kernels/conv_test.cc
index d0350b2fa7..91ad223154 100644
--- a/tensorflow/lite/kernels/conv_test.cc
+++ b/tensorflow/lite/kernels/conv_test.cc
@@ -58,9 +58,35 @@ class BaseConvolutionOpModel : public SingleOpModel {
       // This is a quantized version. The scale of 'bias' depends on the scales
       // of input and filter. Supposedly this is correctly set during quantized
       // training.
-      auto bias_scale = GetScale(input_) * GetScale(filter_);
-      TensorData bias{TensorType_INT32, {bias_size}, 0, 0, bias_scale};
-      bias_ = AddInput(bias);
+      if (filter.per_channel_quantization) {
+        // per channel quantization.
+        std::vector<float> bias_scale(
+            filter.per_channel_quantization_scales.size());
+        std::vector<int64_t> bias_zero_points(
+            filter.per_channel_quantization_scales.size());
+        for (int i = 0; i < filter.per_channel_quantization_scales.size();
+             ++i) {
+          bias_scale[i] =
+              input.scale * filter.per_channel_quantization_scales[i];
+          bias_zero_points[i] = 0;
+        }
+        TensorData bias{TensorType_INT32,
+                        {bias_size},
+                        /*min=*/0,
+                        /*max=*/0,
+                        /*scale=*/0,
+                        /*zero_point=*/0,
+                        true,
+                        /*per_channel_scale=*/bias_scale,
+                        /*per_channel_zero_point=*/bias_zero_points,
+                        /*channel_index==*/0};
+        bias_ = AddInput(bias);
+      } else {
+        // per tensor quantization.
+        auto bias_scale = GetScale(input_) * GetScale(filter_);
+        TensorData bias{TensorType_INT32, {bias_size}, 0, 0, bias_scale};
+        bias_ = AddInput(bias);
+      }
     }
 
     output_ = AddOutput(output);
@@ -1069,6 +1095,76 @@ TEST_P(ConvolutionOpTest, DISABLED_PointwiseMultifilterHybrid) {
                   0.0474)));
 }
 
+class PerChannelQuantizedConvolutionOpModel : public BaseConvolutionOpModel {
+ public:
+  using BaseConvolutionOpModel::BaseConvolutionOpModel;
+
+  void SetInput(std::initializer_list<float> data) {
+    QuantizeAndPopulate<int8_t>(input_, data);
+  }
+
+  void SetFilter(std::initializer_list<float> data) {
+    PerChannelSymmetricQuantizeAndPopulate(filter_, data);
+  }
+
+  void SetBias(std::initializer_list<float> data) {
+    PerChannelQuantizeBias(bias_, data);
+  }
+
+  std::vector<int8_t> GetOutput() { return ExtractVector<int8_t>(output_); }
+  std::vector<float> GetDequantizedOutput() {
+    return Dequantize<int8_t>(ExtractVector<int8_t>(output_), GetScale(output_),
+                              GetZeroPoint(output_));
+  }
+};
+
+TEST_P(ConvolutionOpTest, SimpleTest) {
+  PerChannelQuantizedConvolutionOpModel m(
+      GetRegistration(), {TensorType_INT8, {1, 2, 3, 2}, -63.5, 64, 0.5, -1},
+      {TensorType_INT8,
+       // [2 * 2 * 2 * 2] as [output_channel, y, x, input_channel]
+       {2, 2, 2, 2},
+       0,
+       0,
+       0,
+       0,
+       /*per_channel=*/true,
+       /*per_channel_scales=*/{1, 2},
+       /*per_channel_zeros=*/{0, 0},
+       /*channel_index=*/0},
+      {TensorType_INT8, {}, -63.5, 64, 0.5, -1},
+      /*stride_width=*/1, /*stride_height=*/1);
+  m.SetInput({
+      // [1 * 2 * 3 * 2] as [batch, y, x, input_channel]
+      3, 2,    // batch = 0, y = 0, x = 0
+      1, -1,   // batch = 0, y = 0, x = 1
+      -2, -3,  // batch = 0, y = 0, x = 2
+      4, 3,    // batch = 0, y = 1, x = 0
+      2, -2,   // batch = 0, y = 1, x = 1
+      -3, -4,  // batch = 0, y = 1, x = 2
+  });
+  m.SetFilter(
+      // [2 * 2 * 2 * 2] as [output_channel, y, x, input_channel]
+      {
+          1, 2,  // out channel = 0, y = 0, x = 0
+          3, 4,  // out channel = 0, y = 0, x = 1
+          3, 4,  // out channel = 0, y = 1, x = 0
+          5, 6,  // out channel = 0, y = 1, x = 1
+          7, 8,  // out channel = 1, y = 0, x = 0
+          5, 6,  // out channel = 1, y = 0, x = 1
+          3, 4,  // out channel = 1, y = 1, x = 0
+          1, 2,  // out channel = 1, y = 1, x = 1
+      });
+  m.SetBias({3, -2});
+
+  // Invoke and verify output.
+  // output has dimension [1 * 1 * 2 * 2] as [batch, y, x, output_channel]
+  m.Invoke();
+  EXPECT_THAT(m.GetDequantizedOutput(),
+              ElementsAreArray(ArrayFloatNear({28.5, 64, -59.5, -46})));
+  EXPECT_THAT(m.GetOutput(), ElementsAreArray({56, 127, -120, -93}));
+}
+
 INSTANTIATE_TEST_SUITE_P(
     ConvolutionOpTest, ConvolutionOpTest,
     ::testing::ValuesIn(SingleOpTest::GetKernelTags(*kKernelMap)));
diff --git a/tensorflow/lite/kernels/internal/BUILD b/tensorflow/lite/kernels/internal/BUILD
index 3b64bcce83..9d982a89ea 100644
--- a/tensorflow/lite/kernels/internal/BUILD
+++ b/tensorflow/lite/kernels/internal/BUILD
@@ -306,6 +306,7 @@ cc_library(
         "reference/depthwiseconv_float.h",
         "reference/depthwiseconv_uint8.h",
         "reference/fully_connected.h",
+        "reference/integer_ops/conv.h",
         "reference/integer_ops/dequantize.h",
         "reference/integer_ops/pooling.h",
         "reference/integer_ops/softmax.h",
diff --git a/tensorflow/lite/kernels/internal/reference/integer_ops/conv.h b/tensorflow/lite/kernels/internal/reference/integer_ops/conv.h
new file mode 100644
index 0000000000..14e449e2bb
--- /dev/null
+++ b/tensorflow/lite/kernels/internal/reference/integer_ops/conv.h
@@ -0,0 +1,128 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_CONV_H_
+#define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_CONV_H_
+
+#include "tensorflow/lite/kernels/internal/common.h"
+
+namespace tflite {
+namespace reference_integer_ops {
+
+// Fixed-point per-channel-quantization convolution reference kernel.
+inline void ConvPerChannel(
+    const ConvParams& params, const int32* output_multiplier,
+    const int32* output_shift, const RuntimeShape& input_shape,
+    const int8* input_data, const RuntimeShape& filter_shape,
+    const int8* filter_data, const RuntimeShape& bias_shape,
+    const int32* bias_data, const RuntimeShape& output_shape,
+    int8* output_data) {
+  // Get parameters.
+  const int32 input_offset = params.input_offset;  // r = s(q - Z)
+  const int stride_width = params.stride_width;
+  const int stride_height = params.stride_height;
+  const int dilation_width_factor = params.dilation_width_factor;
+  const int dilation_height_factor = params.dilation_height_factor;
+  const int pad_width = params.padding_values.width;
+  const int pad_height = params.padding_values.height;
+  const int32 output_offset = params.output_offset;
+
+  // Set min and max value of the output.
+  const int32 output_activation_min = std::numeric_limits<int8_t>::min();
+  const int32 output_activation_max = std::numeric_limits<int8_t>::max();
+
+  // Sanity check.
+  TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
+  TFLITE_DCHECK_EQ(input_shape.DimensionsCount(), 4);
+  TFLITE_DCHECK_EQ(filter_shape.DimensionsCount(), 4);
+  TFLITE_DCHECK_EQ(output_shape.DimensionsCount(), 4);
+  const int batches = MatchingDim(input_shape, 0, output_shape, 0);
+  const int input_depth = MatchingDim(input_shape, 3, filter_shape, 3);
+  const int output_depth = MatchingDim(filter_shape, 0, output_shape, 3);
+  if (bias_data) {
+    TFLITE_DCHECK_EQ(bias_shape.FlatSize(), output_depth);
+  }
+
+  // Check dimensions of the tensors.
+  const int input_height = input_shape.Dims(1);
+  const int input_width = input_shape.Dims(2);
+  const int filter_height = filter_shape.Dims(1);
+  const int filter_width = filter_shape.Dims(2);
+  const int output_height = output_shape.Dims(1);
+  const int output_width = output_shape.Dims(2);
+  for (int batch = 0; batch < batches; ++batch) {
+    for (int out_y = 0; out_y < output_height; ++out_y) {
+      for (int out_x = 0; out_x < output_width; ++out_x) {
+        for (int out_channel = 0; out_channel < output_depth; ++out_channel) {
+          const int in_x_origin = (out_x * stride_width) - pad_width;
+          const int in_y_origin = (out_y * stride_height) - pad_height;
+          int32 acc = 0;
+          for (int filter_y = 0; filter_y < filter_height; ++filter_y) {
+            for (int filter_x = 0; filter_x < filter_width; ++filter_x) {
+              for (int in_channel = 0; in_channel < input_depth; ++in_channel) {
+                const int in_x = in_x_origin + dilation_width_factor * filter_x;
+                const int in_y =
+                    in_y_origin + dilation_height_factor * filter_y;
+                // Zero padding by omitting the areas outside the image.
+                const bool is_point_inside_image =
+                    (in_x >= 0) && (in_x < input_width) && (in_y >= 0) &&
+                    (in_y < input_height);
+                if (is_point_inside_image) {
+                  int32 input_val = input_data[Offset(input_shape, batch, in_y,
+                                                      in_x, in_channel)];
+                  int32 filter_val =
+                      filter_data[Offset(filter_shape, out_channel, filter_y,
+                                         filter_x, in_channel)];
+                  // Accumulate with 32 bits accumulator.
+                  // In the nudging process during model quantization, we force
+                  // real value of 0.0 be represented by a quantized value. This
+                  // guarantees that the input_offset is a int8, even though it
+                  // is represented using int32.
+                  // int32 += int8 * (int8 - int8) so the highest value we can
+                  // get from each accumulation is [-127, 127] * ([-128, 127] -
+                  // [-128, 127]), which is [-32512, 32512]. log2(32512)
+                  // = 14.98, which means we can accumulate at least 2^16
+                  // multiplications without overflow. The accumulator is
+                  // applied to a filter so the accumation logic will hold as
+                  // long as the filter size (filter_y * filter_x * in_channel)
+                  // does not exceed 2^16, which is the case in all the models
+                  // we have seen so far.
+                  // TODO(jianlijianli): Add a check to make sure the
+                  // accumulator depth is smaller than 2^16.
+                  acc += filter_val * (input_val - input_offset);
+                }
+              }
+            }
+          }
+
+          if (bias_data) {
+            acc += bias_data[out_channel];
+          }
+          acc = MultiplyByQuantizedMultiplier(
+              acc, output_multiplier[out_channel], output_shift[out_channel]);
+          acc += output_offset;
+          acc = std::max(acc, output_activation_min);
+          acc = std::min(acc, output_activation_max);
+          output_data[Offset(output_shape, batch, out_y, out_x, out_channel)] =
+              static_cast<int8_t>(acc);
+        }
+      }
+    }
+  }
+}
+
+}  // namespace reference_integer_ops
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_CONV_H_
-- 
GitLab


From c6fb02fbb8ffa0cf3b57cc10117683bf3466454f Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Wed, 30 Jan 2019 12:11:55 -0800
Subject: [PATCH 1604/2345] Link all TRT *_op_lib into
 contrib:contrib_ops_op_lib to get rid of the runtime wanring about
 get_serialized_resource_op not registered.

PiperOrigin-RevId: 231642866
---
 tensorflow/compiler/tf2tensorrt/BUILD | 19 +++++++++++++------
 tensorflow/contrib/BUILD              |  2 +-
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/tensorflow/compiler/tf2tensorrt/BUILD b/tensorflow/compiler/tf2tensorrt/BUILD
index 9ad27c9801..35d577e2c1 100644
--- a/tensorflow/compiler/tf2tensorrt/BUILD
+++ b/tensorflow/compiler/tf2tensorrt/BUILD
@@ -97,8 +97,10 @@ tf_cuda_cc_test(
         "nomac",
     ],
     deps = [
-        ":get_serialized_resource_op_op_lib",
+        # TODO(laigd): consider splitting get_serialized_resource_op out from
+        # TF-TRT.
         ":trt_op_kernels",
+        ":trt_op_libs",
         ":trt_resources",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
@@ -117,6 +119,14 @@ tf_gen_op_libs(
     ],
 )
 
+cc_library(
+    name = "trt_op_libs",
+    deps = [
+        ":get_serialized_resource_op_op_lib",
+        ":trt_engine_op_op_lib",
+    ],
+)
+
 tf_cuda_library(
     name = "trt_logging",
     srcs = ["utils/trt_logger.cc"],
@@ -132,9 +142,7 @@ tf_cuda_library(
 tf_gen_op_wrapper_py(
     name = "trt_ops",
     deps = [
-        ":get_serialized_resource_op_op_lib",
-        ":trt_engine_op_op_lib",
-        ":trt_logging",
+        ":trt_op_libs",
     ],
 )
 
@@ -148,8 +156,7 @@ tf_custom_op_py_library(
     ]),
     kernels = [
         ":trt_op_kernels",
-        ":trt_engine_op_op_lib",
-        ":get_serialized_resource_op_op_lib",
+        ":trt_op_libs",
     ],
     srcs_version = "PY2AND3",
     deps = [
diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD
index a4c3d9623a..25f2640e35 100644
--- a/tensorflow/contrib/BUILD
+++ b/tensorflow/contrib/BUILD
@@ -238,7 +238,7 @@ cc_library(
             "//tensorflow/contrib/kinesis:dataset_ops_op_lib",
         ],
     }) + if_not_windows([
-        "//tensorflow/contrib/tensorrt:trt_engine_op_op_lib",
+        "//tensorflow/compiler/tf2tensorrt:trt_op_libs",
     ]) + select({
         "//tensorflow:android": [],
         "//tensorflow:ios": [],
-- 
GitLab


From 5a1865b201d97d4a05bda3238cb1a796d06c7ce7 Mon Sep 17 00:00:00 2001
From: Andrew Cotter <acotter@google.com>
Date: Wed, 30 Jan 2019 12:18:28 -0800
Subject: [PATCH 1605/2345] Adding deprecation notice to
 contrib/constrained_optimization/README.md, pointing to new github repo
 (github.com/google-research/tensorflow_constrained_optimization).

PiperOrigin-RevId: 231644093
---
 tensorflow/contrib/constrained_optimization/README.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tensorflow/contrib/constrained_optimization/README.md b/tensorflow/contrib/constrained_optimization/README.md
index cb1dd7d836..7ffb6894d3 100644
--- a/tensorflow/contrib/constrained_optimization/README.md
+++ b/tensorflow/contrib/constrained_optimization/README.md
@@ -1,5 +1,10 @@
 <!-- TODO(acotter): Add usage example of non-convex optimization and stochastic classification. -->
 
+**NOTE: As tensorflow.contrib is being
+[deprecated](https://github.com/tensorflow/community/pull/18), TFCO is moving to
+its own repository on
+[github](https://github.com/google-research/tensorflow_constrained_optimization).**
+
 # ConstrainedOptimization (TFCO)
 
 TFCO is a library for optimizing inequality-constrained problems in TensorFlow.
-- 
GitLab


From 1944f12a85c42d6172762b4d873fdeb0e73739ce Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 30 Jan 2019 12:19:05 -0800
Subject: [PATCH 1606/2345] Generate testdata from original pbtxt files.

PiperOrigin-RevId: 231644215
---
 tensorflow/lite/python/BUILD                  |  8 +-
 tensorflow/lite/python/testdata/BUILD         | 42 ++++++++
 tensorflow/lite/python/testdata/permute.pbtxt | 98 +++++++++++++++++++
 tensorflow/tools/pip_package/BUILD            |  2 +-
 4 files changed, 142 insertions(+), 8 deletions(-)
 create mode 100644 tensorflow/lite/python/testdata/BUILD
 create mode 100644 tensorflow/lite/python/testdata/permute.pbtxt

diff --git a/tensorflow/lite/python/BUILD b/tensorflow/lite/python/BUILD
index 1fa8494a4c..00366623da 100644
--- a/tensorflow/lite/python/BUILD
+++ b/tensorflow/lite/python/BUILD
@@ -4,12 +4,6 @@ package(default_visibility = ["//tensorflow:internal"])
 
 load("//tensorflow:tensorflow.bzl", "py_test")
 
-filegroup(
-    name = "interpreter_test_data",
-    srcs = glob(["**/testdata/*"]),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 py_library(
     name = "interpreter",
     srcs = [
@@ -27,7 +21,7 @@ py_library(
 py_test(
     name = "interpreter_test",
     srcs = ["interpreter_test.py"],
-    data = [":interpreter_test_data"],
+    data = ["//tensorflow/lite/python/testdata:interpreter_test_data"],
     srcs_version = "PY2AND3",
     tags = ["no_oss"],
     deps = [
diff --git a/tensorflow/lite/python/testdata/BUILD b/tensorflow/lite/python/testdata/BUILD
new file mode 100644
index 0000000000..4689c318b3
--- /dev/null
+++ b/tensorflow/lite/python/testdata/BUILD
@@ -0,0 +1,42 @@
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//tensorflow:internal"])
+
+load("//tensorflow:tensorflow.bzl", "py_test")
+load("//tensorflow/lite:build_def.bzl", "tf_to_tflite")
+
+exports_files(glob(["*.pb"]))
+
+tf_to_tflite(
+    name = "permute_float",
+    src = "permute.pbtxt",
+    out = "permute_float.tflite",
+    options = [
+        "--input_arrays=input",
+        "--output_arrays=output",
+    ],
+)
+
+tf_to_tflite(
+    name = "permute_uint8",
+    src = "permute.pbtxt",
+    out = "permute_uint8.tflite",
+    options = [
+        "--input_arrays=input",
+        "--output_arrays=output",
+        "--inference_type=QUANTIZED_UINT8",
+        "--std_values=1",
+        "--mean_values=0",
+        "--default_ranges_min=0",
+        "--default_ranges_max=255",
+    ],
+)
+
+filegroup(
+    name = "interpreter_test_data",
+    srcs = [
+        ":permute_float",
+        ":permute_uint8",
+    ],
+    visibility = ["//tensorflow:__subpackages__"],
+)
diff --git a/tensorflow/lite/python/testdata/permute.pbtxt b/tensorflow/lite/python/testdata/permute.pbtxt
new file mode 100644
index 0000000000..482b5c6482
--- /dev/null
+++ b/tensorflow/lite/python/testdata/permute.pbtxt
@@ -0,0 +1,98 @@
+node {
+  name: "input"
+  op: "Placeholder"
+  device: "/device:CPU:0"
+  attr {
+    key: "dtype"
+    value {
+      type: DT_FLOAT
+    }
+  }
+  attr {
+    key: "shape"
+    value {
+      shape {
+        dim {
+          size: 1
+        }
+        dim {
+          size: 4
+        }
+      }
+    }
+  }
+}
+node {
+  name: "Const"
+  op: "Const"
+  device: "/device:CPU:0"
+  attr {
+    key: "dtype"
+    value {
+      type: DT_FLOAT
+    }
+  }
+  attr {
+    key: "value"
+    value {
+      tensor {
+        dtype: DT_FLOAT
+        tensor_shape {
+          dim {
+            size: 4
+          }
+          dim {
+            size: 4
+          }
+        }
+        float_val: 0.0
+        float_val: 0.0
+        float_val: 0.0
+        float_val: 1.0
+
+        float_val: 0.0
+        float_val: 0.0
+        float_val: 1.0
+        float_val: 0.0
+
+        float_val: 0.0
+        float_val: 1.0
+        float_val: 0.0
+        float_val: 0.0
+
+        float_val: 1.0
+        float_val: 0.0
+        float_val: 0.0
+        float_val: 0.0
+      }
+    }
+  }
+}
+node {
+  name: "output"
+  op: "MatMul"
+  input: "input"
+  input: "Const"
+  device: "/device:CPU:0"
+  attr {
+    key: "T"
+    value {
+      type: DT_FLOAT
+    }
+  }
+  attr {
+    key: "transpose_a"
+    value {
+      b: false
+    }
+  }
+  attr {
+    key: "transpose_b"
+    value {
+      b: false
+    }
+  }
+}
+versions {
+  producer: 27
+}
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index bc970d0f76..30a31a35c5 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -59,7 +59,7 @@ COMMON_PIP_DEPS = [
     "setup.py",
     ":included_headers",
     "//tensorflow:tensorflow_py",
-    "//tensorflow/lite/python:interpreter_test_data",
+    "//tensorflow/lite/python/testdata:interpreter_test_data",
     "//tensorflow/lite/python:tflite_convert",
     "//tensorflow/lite/toco/python:toco_from_protos",
     # "//tensorflow/python/autograph/converters:converters",
-- 
GitLab


From 8fa8b44eaf92870143c18cf6da719b05bccbdd68 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Wed, 30 Jan 2019 12:21:21 -0800
Subject: [PATCH 1607/2345] Add temporary patch to third_party/eigen3 to fix
 builds of Arm/NEON code with older versions of GCC (for TensorFlow Mobile)

PiperOrigin-RevId: 231644645
---
 tensorflow/contrib/makefile/download_dependencies.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/makefile/download_dependencies.sh b/tensorflow/contrib/makefile/download_dependencies.sh
index 2a5232b476..af3c541dc2 100755
--- a/tensorflow/contrib/makefile/download_dependencies.sh
+++ b/tensorflow/contrib/makefile/download_dependencies.sh
@@ -142,5 +142,6 @@ replace_by_sed 's#static uint64x2_t p2ul_CONJ_XOR = vld1q_u64( p2ul_conj_XOR_DAT
 # TODO(satok): Remove this once protobuf/autogen.sh is fixed.
 replace_by_sed 's#https://googlemock.googlecode.com/files/gmock-1.7.0.zip#http://download.tensorflow.org/deps/gmock-1.7.0.zip#' \
   "${DOWNLOADS_DIR}/protobuf/autogen.sh"
+cat "third_party/eigen3/gebp_neon.patch" | patch "${DOWNLOADS_DIR}/eigen/Eigen/src/Core/products/GeneralBlockPanelKernel.h"
 
 echo "download_dependencies.sh completed successfully." >&2
-- 
GitLab


From 7157690e660676f8b01aec55d42d1e76101e6fa8 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Wed, 30 Jan 2019 12:22:34 -0800
Subject: [PATCH 1608/2345] Export raw ops in v1 and v2.

Ops are exported as tf.raw_ops.Add, and so on.

PiperOrigin-RevId: 231644909
---
 tensorflow/python/BUILD                       |    9 +
 tensorflow/python/framework/python_op_gen.cc  |   64 +
 tensorflow/python/ops/raw_ops_test.py         |   42 +
 .../tools/api/generator/api_init_files.bzl    |    1 +
 .../tools/api/generator/api_init_files_v1.bzl |    1 +
 tensorflow/python/util/tf_export.py           |   10 +
 tensorflow/python/util/tf_inspect.py          |   26 +-
 .../tools/api/golden/v1/tensorflow.pbtxt      |    4 +
 .../api/golden/v1/tensorflow.raw_ops.pbtxt    | 4023 +++++++++++++++++
 .../tools/api/golden/v2/tensorflow.pbtxt      |    4 +
 .../api/golden/v2/tensorflow.raw_ops.pbtxt    | 4023 +++++++++++++++++
 11 files changed, 8196 insertions(+), 11 deletions(-)
 create mode 100644 tensorflow/python/ops/raw_ops_test.py
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 109beace58..98b9518ed4 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -650,6 +650,7 @@ py_library(
         "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:core",
         "//tensorflow/python/eager:execute",
+        "//tensorflow/tools/docs:doc_controls",
     ],
 )
 
@@ -6070,3 +6071,11 @@ py_library(
     srcs = ["tf2.py"],
     srcs_version = "PY2AND3",
 )
+
+cuda_py_test(
+    name = "raw_ops_test",
+    srcs = ["ops/raw_ops_test.py"],
+    additional_deps = [
+        "//tensorflow/python:client_testlib",
+    ],
+)
diff --git a/tensorflow/python/framework/python_op_gen.cc b/tensorflow/python/framework/python_op_gen.cc
index d460168631..0f304385de 100644
--- a/tensorflow/python/framework/python_op_gen.cc
+++ b/tensorflow/python/framework/python_op_gen.cc
@@ -144,6 +144,8 @@ class GenEagerPythonOp : public python_op_gen_internal::GenPythonOp {
                        const string& num_outputs_expr);
   void AddDispatch(const string& prefix);
 
+  void AddRawOpExport();
+
   void AddAttrForArg(const string& attr, int arg_index) {
     gtl::InsertIfNotPresent(&inferred_attrs_, attr,
                             op_def_.input_arg(arg_index).name());
@@ -668,6 +670,7 @@ bool GenEagerPythonOp::AddEagerFastPathAndGraphCode(
   AddEagerFunctionTeardown("  ", output_sizes,
                            true /* execute_record_gradient */);
 
+  AddRawOpExport();
   strings::StrAppend(&result_, "\n\n");
   return true;
 }
@@ -921,6 +924,65 @@ void GenEagerPythonOp::AddDispatch(const string& prefix) {
   strings::StrAppend(&result_, prefix, "  raise\n");
 }
 
+void GenEagerPythonOp::AddRawOpExport() {
+  // Create function for python op.
+  string raw_parameters;
+  string function_call_parameters;
+  string inputs;
+  string attrs;
+  std::map<string, string> renames;
+
+  for (const auto& input_arg : api_def_.in_arg()) {
+    renames.insert({input_arg.name(), input_arg.rename_to()});
+  }
+  for (const auto& attr : api_def_.attr()) {
+    renames.insert({attr.name(), attr.rename_to()});
+  }
+
+  for (const auto& input_arg : op_def_.input_arg()) {
+    if (!raw_parameters.empty()) strings::StrAppend(&raw_parameters, ", ");
+    strings::StrAppend(&raw_parameters, input_arg.name());
+
+    if (!inputs.empty()) strings::StrAppend(&inputs, ", ");
+    strings::StrAppend(&inputs, input_arg.name());
+
+    if (!function_call_parameters.empty()) {
+      strings::StrAppend(&function_call_parameters, ", ");
+    }
+    strings::StrAppend(&function_call_parameters, renames[input_arg.name()],
+                       "=", input_arg.name());
+  }
+  for (const auto& attr : op_def_.attr()) {
+    if (inferred_attrs_.find(attr.name()) != inferred_attrs_.end()) continue;
+
+    if (!raw_parameters.empty()) strings::StrAppend(&raw_parameters, ", ");
+    strings::StrAppend(&raw_parameters, attr.name());
+
+    if (!attrs.empty()) strings::StrAppend(&attrs, ", ");
+    strings::StrAppend(&attrs, "\"", attr.name(), "\", ", attr.name());
+
+    if (!function_call_parameters.empty()) {
+      strings::StrAppend(&function_call_parameters, ", ");
+    }
+    strings::StrAppend(&function_call_parameters, renames[attr.name()], "=",
+                       attr.name());
+  }
+
+  const string raw_function_name =
+      python_op_gen_internal::AvoidPythonReserved(op_def_.name());
+
+  strings::StrAppend(&result_,
+                     "@_doc_controls.do_not_generate_docs\n@_kwarg_only\ndef ",
+                     raw_function_name, "(", raw_parameters, "):\n");
+
+  // Function body.
+  strings::StrAppend(&result_, "  return ", function_name_, "(",
+                     function_call_parameters, ")\n");
+
+  strings::StrAppend(&result_, "tf_export(\"raw_ops.", raw_function_name,
+                     "\")(", raw_function_name, ")\n");
+}
+
 string GetPythonOps(const OpList& ops, const ApiDefMap& api_defs,
                     const std::vector<string>& hidden_ops, bool require_shapes,
                     const string& source_file_name = "") {
@@ -962,6 +1024,8 @@ from tensorflow.python.framework import op_def_library as _op_def_library
 from tensorflow.python.util.deprecation import deprecated_endpoints
 from tensorflow.python.util import dispatch as _dispatch
 from tensorflow.python.util.tf_export import tf_export
+from tensorflow.python.util.tf_export import kwarg_only as _kwarg_only
+from tensorflow.tools.docs import doc_controls as _doc_controls
 
 )");
 
diff --git a/tensorflow/python/ops/raw_ops_test.py b/tensorflow/python/ops/raw_ops_test.py
new file mode 100644
index 0000000000..82ea137354
--- /dev/null
+++ b/tensorflow/python/ops/raw_ops_test.py
@@ -0,0 +1,42 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Raw ops tests."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import gen_math_ops
+from tensorflow.python.platform import test
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class RawOpsTest(test.TestCase):
+
+  def testSimple(self):
+
+    with self.assertRaises(TypeError):
+      _ = gen_math_ops.Add(1., 1.)
+
+    x = constant_op.constant(1)
+    self.assertEqual([2], self.evaluate(gen_math_ops.Add(x=x, y=x)))
+
+
+if __name__ == '__main__':
+  ops.enable_eager_execution()
+  test.main()
diff --git a/tensorflow/python/tools/api/generator/api_init_files.bzl b/tensorflow/python/tools/api/generator/api_init_files.bzl
index 245380287a..5095f9fdbe 100644
--- a/tensorflow/python/tools/api/generator/api_init_files.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files.bzl
@@ -35,6 +35,7 @@ TENSORFLOW_API_INIT_FILES = [
     "quantization/__init__.py",
     "ragged/__init__.py",
     "random/__init__.py",
+    "raw_ops/__init__.py",
     "rnn/__init__.py",
     "saved_model/__init__.py",
     "sets/__init__.py",
diff --git a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
index c8c7c51b40..01bf12ebe7 100644
--- a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
@@ -46,6 +46,7 @@ TENSORFLOW_API_INIT_FILES_V1 = [
     "quantization/__init__.py",
     "ragged/__init__.py",
     "random/__init__.py",
+    "raw_ops/__init__.py",
     "resource_loader/__init__.py",
     "strings/__init__.py",
     "saved_model/__init__.py",
diff --git a/tensorflow/python/util/tf_export.py b/tensorflow/python/util/tf_export.py
index ce1a1a5c30..7b44d72ab9 100644
--- a/tensorflow/python/util/tf_export.py
+++ b/tensorflow/python/util/tf_export.py
@@ -46,6 +46,7 @@ import functools
 import sys
 
 from tensorflow.python.util import tf_decorator
+from tensorflow.python.util import tf_inspect
 
 ESTIMATOR_API_NAME = 'estimator'
 KERAS_API_NAME = 'keras'
@@ -379,6 +380,15 @@ class api_export(object):  # pylint: disable=invalid-name
         (self._names_v1, name))
 
 
+def kwarg_only(f):
+  """A wrapper that throws away all non-kwarg arguments."""
+  def wrapper(**kwargs):
+    return f(**kwargs)
+
+  return tf_decorator.make_decorator(
+      f, wrapper, decorator_argspec=tf_inspect.getargspec(f))
+
+
 tf_export = functools.partial(api_export, api_name=TENSORFLOW_API_NAME)
 estimator_export = functools.partial(api_export, api_name=ESTIMATOR_API_NAME)
 keras_export = functools.partial(api_export, api_name=KERAS_API_NAME)
diff --git a/tensorflow/python/util/tf_inspect.py b/tensorflow/python/util/tf_inspect.py
index 5171cdfdbc..3a1c4a6e12 100644
--- a/tensorflow/python/util/tf_inspect.py
+++ b/tensorflow/python/util/tf_inspect.py
@@ -36,6 +36,19 @@ else:
       'annotations'
   ])
 
+
+def _convert_maybe_argspec_to_fullargspec(argspec):
+  if isinstance(argspec, FullArgSpec):
+    return argspec
+  return FullArgSpec(
+      args=argspec.args,
+      varargs=argspec.varargs,
+      varkw=argspec.keywords,
+      defaults=argspec.defaults,
+      kwonlyargs=[],
+      kwonlydefaults=None,
+      annotations={})
+
 if hasattr(_inspect, 'getfullargspec'):
   _getfullargspec = _inspect.getfullargspec  # pylint: disable=invalid-name
 
@@ -74,16 +87,7 @@ else:
     Returns:
       A FullArgSpec with empty kwonlyargs, kwonlydefaults and annotations.
     """
-    argspecs = getargspec(target)
-    fullargspecs = FullArgSpec(
-        args=argspecs.args,
-        varargs=argspecs.varargs,
-        varkw=argspecs.keywords,
-        defaults=argspecs.defaults,
-        kwonlyargs=[],
-        kwonlydefaults=None,
-        annotations={})
-    return fullargspecs
+    return _convert_maybe_argspec_to_fullargspec(getargspec(target))
 
 
 def currentframe():
@@ -238,7 +242,7 @@ def getfullargspec(obj):
     directly on the callable.
   """
   decorators, target = tf_decorator.unwrap(obj)
-  return next((d.decorator_argspec
+  return next((_convert_maybe_argspec_to_fullargspec(d.decorator_argspec)
                for d in decorators
                if d.decorator_argspec is not None), _getfullargspec(target))
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
index c8ec8da259..56d06053f1 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
@@ -552,6 +552,10 @@ tf_module {
     name: "random_uniform_initializer"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "raw_ops"
+    mtype: "<type \'module\'>"
+  }
   member {
     name: "resource"
     mtype: "<class \'tensorflow.python.framework.dtypes.DType\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt
new file mode 100644
index 0000000000..c52a0cd41f
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt
@@ -0,0 +1,4023 @@
+path: "tensorflow.raw_ops"
+tf_module {
+  member_method {
+    name: "Abort"
+    argspec: "args=[\'error_msg\', \'exit_without_error\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Abs"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AccumulateNV2"
+    argspec: "args=[\'inputs\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AccumulatorApplyGradient"
+    argspec: "args=[\'handle\', \'local_step\', \'gradient\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AccumulatorNumAccumulated"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AccumulatorSetGlobalStep"
+    argspec: "args=[\'handle\', \'new_global_step\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AccumulatorTakeGradient"
+    argspec: "args=[\'handle\', \'num_required\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Acos"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Acosh"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Add"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AddManySparseToTensorsMap"
+    argspec: "args=[\'sparse_indices\', \'sparse_values\', \'sparse_shape\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AddN"
+    argspec: "args=[\'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AddSparseToTensorsMap"
+    argspec: "args=[\'sparse_indices\', \'sparse_values\', \'sparse_shape\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AddV2"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AdjustContrast"
+    argspec: "args=[\'images\', \'contrast_factor\', \'min_value\', \'max_value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AdjustContrastv2"
+    argspec: "args=[\'images\', \'contrast_factor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AdjustHue"
+    argspec: "args=[\'images\', \'delta\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AdjustSaturation"
+    argspec: "args=[\'images\', \'scale\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "All"
+    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AllCandidateSampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Angle"
+    argspec: "args=[\'input\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AnonymousIterator"
+    argspec: "args=[\'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Any"
+    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyAdaMax"
+    argspec: "args=[\'var\', \'m\', \'v\', \'beta1_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyAdadelta"
+    argspec: "args=[\'var\', \'accum\', \'accum_update\', \'lr\', \'rho\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'use_locking\', \'update_slots\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyAdagradDA"
+    argspec: "args=[\'var\', \'gradient_accumulator\', \'gradient_squared_accumulator\', \'grad\', \'lr\', \'l1\', \'l2\', \'global_step\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyAdam"
+    argspec: "args=[\'var\', \'m\', \'v\', \'beta1_power\', \'beta2_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyAddSign"
+    argspec: "args=[\'var\', \'m\', \'lr\', \'alpha\', \'sign_decay\', \'beta\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyCenteredRMSProp"
+    argspec: "args=[\'var\', \'mg\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyFtrl"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'lr\', \'l1\', \'l2\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyFtrlV2"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'lr\', \'l1\', \'l2\', \'l2_shrinkage\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyGradientDescent"
+    argspec: "args=[\'var\', \'alpha\', \'delta\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyMomentum"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyPowerSign"
+    argspec: "args=[\'var\', \'m\', \'lr\', \'logbase\', \'sign_decay\', \'beta\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyProximalAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'l1\', \'l2\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyProximalGradientDescent"
+    argspec: "args=[\'var\', \'alpha\', \'l1\', \'l2\', \'delta\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyRMSProp"
+    argspec: "args=[\'var\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApproximateEqual"
+    argspec: "args=[\'x\', \'y\', \'tolerance\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ArgMax"
+    argspec: "args=[\'input\', \'dimension\', \'output_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ArgMin"
+    argspec: "args=[\'input\', \'dimension\', \'output_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AsString"
+    argspec: "args=[\'input\', \'precision\', \'scientific\', \'shortest\', \'width\', \'fill\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Asin"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Asinh"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Assert"
+    argspec: "args=[\'condition\', \'data\', \'summarize\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Assign"
+    argspec: "args=[\'ref\', \'value\', \'validate_shape\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AssignAdd"
+    argspec: "args=[\'ref\', \'value\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AssignAddVariableOp"
+    argspec: "args=[\'resource\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AssignSub"
+    argspec: "args=[\'ref\', \'value\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AssignSubVariableOp"
+    argspec: "args=[\'resource\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AssignVariableOp"
+    argspec: "args=[\'resource\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Atan"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Atan2"
+    argspec: "args=[\'y\', \'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Atanh"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AudioSpectrogram"
+    argspec: "args=[\'input\', \'window_size\', \'stride\', \'magnitude_squared\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AudioSummary"
+    argspec: "args=[\'tag\', \'tensor\', \'sample_rate\', \'max_outputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AudioSummaryV2"
+    argspec: "args=[\'tag\', \'tensor\', \'sample_rate\', \'max_outputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AvgPool"
+    argspec: "args=[\'value\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AvgPool3D"
+    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AvgPool3DGrad"
+    argspec: "args=[\'orig_input_shape\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AvgPoolGrad"
+    argspec: "args=[\'orig_input_shape\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Barrier"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BarrierClose"
+    argspec: "args=[\'handle\', \'cancel_pending_enqueues\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BarrierIncompleteSize"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BarrierInsertMany"
+    argspec: "args=[\'handle\', \'keys\', \'values\', \'component_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BarrierReadySize"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BarrierTakeMany"
+    argspec: "args=[\'handle\', \'num_elements\', \'component_types\', \'allow_small_batch\', \'wait_for_incomplete\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchCholesky"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchCholeskyGrad"
+    argspec: "args=[\'l\', \'grad\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchDataset"
+    argspec: "args=[\'input_dataset\', \'batch_size\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchDatasetV2"
+    argspec: "args=[\'input_dataset\', \'batch_size\', \'drop_remainder\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchFFT"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchFFT2D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchFFT3D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchIFFT"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchIFFT2D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchIFFT3D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatMul"
+    argspec: "args=[\'x\', \'y\', \'adj_x\', \'adj_y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixBandPart"
+    argspec: "args=[\'input\', \'num_lower\', \'num_upper\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixDeterminant"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixDiag"
+    argspec: "args=[\'diagonal\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixDiagPart"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixInverse"
+    argspec: "args=[\'input\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixSetDiag"
+    argspec: "args=[\'input\', \'diagonal\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixSolve"
+    argspec: "args=[\'matrix\', \'rhs\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixSolveLs"
+    argspec: "args=[\'matrix\', \'rhs\', \'l2_regularizer\', \'fast\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixTriangularSolve"
+    argspec: "args=[\'matrix\', \'rhs\', \'lower\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchNormWithGlobalNormalization"
+    argspec: "args=[\'t\', \'m\', \'v\', \'beta\', \'gamma\', \'variance_epsilon\', \'scale_after_normalization\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchNormWithGlobalNormalizationGrad"
+    argspec: "args=[\'t\', \'m\', \'v\', \'gamma\', \'backprop\', \'variance_epsilon\', \'scale_after_normalization\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchSelfAdjointEig"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchSelfAdjointEigV2"
+    argspec: "args=[\'input\', \'compute_v\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchSvd"
+    argspec: "args=[\'input\', \'compute_uv\', \'full_matrices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchToSpace"
+    argspec: "args=[\'input\', \'crops\', \'block_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchToSpaceND"
+    argspec: "args=[\'input\', \'block_shape\', \'crops\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BesselI0e"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BesselI1e"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Betainc"
+    argspec: "args=[\'a\', \'b\', \'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BiasAdd"
+    argspec: "args=[\'value\', \'bias\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BiasAddGrad"
+    argspec: "args=[\'out_backprop\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BiasAddV1"
+    argspec: "args=[\'value\', \'bias\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Bincount"
+    argspec: "args=[\'arr\', \'size\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Bitcast"
+    argspec: "args=[\'input\', \'type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BitwiseAnd"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BitwiseOr"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BitwiseXor"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesBucketize"
+    argspec: "args=[\'float_values\', \'bucket_boundaries\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesCalculateBestGainsPerFeature"
+    argspec: "args=[\'node_id_range\', \'stats_summary_list\', \'l1\', \'l2\', \'tree_complexity\', \'min_node_weight\', \'max_splits\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesCenterBias"
+    argspec: "args=[\'tree_ensemble_handle\', \'mean_gradients\', \'mean_hessians\', \'l1\', \'l2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesCreateEnsemble"
+    argspec: "args=[\'tree_ensemble_handle\', \'stamp_token\', \'tree_ensemble_serialized\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesCreateQuantileStreamResource"
+    argspec: "args=[\'quantile_stream_resource_handle\', \'epsilon\', \'num_streams\', \'max_elements\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesDeserializeEnsemble"
+    argspec: "args=[\'tree_ensemble_handle\', \'stamp_token\', \'tree_ensemble_serialized\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesEnsembleResourceHandleOp"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesExampleDebugOutputs"
+    argspec: "args=[\'tree_ensemble_handle\', \'bucketized_features\', \'logits_dimension\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesGetEnsembleStates"
+    argspec: "args=[\'tree_ensemble_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesMakeQuantileSummaries"
+    argspec: "args=[\'float_values\', \'example_weights\', \'epsilon\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesMakeStatsSummary"
+    argspec: "args=[\'node_ids\', \'gradients\', \'hessians\', \'bucketized_features_list\', \'max_splits\', \'num_buckets\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesPredict"
+    argspec: "args=[\'tree_ensemble_handle\', \'bucketized_features\', \'logits_dimension\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesQuantileStreamResourceAddSummaries"
+    argspec: "args=[\'quantile_stream_resource_handle\', \'summaries\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesQuantileStreamResourceDeserialize"
+    argspec: "args=[\'quantile_stream_resource_handle\', \'bucket_boundaries\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesQuantileStreamResourceFlush"
+    argspec: "args=[\'quantile_stream_resource_handle\', \'num_buckets\', \'generate_quantiles\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesQuantileStreamResourceGetBucketBoundaries"
+    argspec: "args=[\'quantile_stream_resource_handle\', \'num_features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesQuantileStreamResourceHandleOp"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesSerializeEnsemble"
+    argspec: "args=[\'tree_ensemble_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesTrainingPredict"
+    argspec: "args=[\'tree_ensemble_handle\', \'cached_tree_ids\', \'cached_node_ids\', \'bucketized_features\', \'logits_dimension\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesUpdateEnsemble"
+    argspec: "args=[\'tree_ensemble_handle\', \'feature_ids\', \'node_ids\', \'gains\', \'thresholds\', \'left_node_contribs\', \'right_node_contribs\', \'max_depth\', \'learning_rate\', \'pruning_mode\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BroadcastArgs"
+    argspec: "args=[\'s0\', \'s1\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BroadcastGradientArgs"
+    argspec: "args=[\'s0\', \'s1\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BroadcastTo"
+    argspec: "args=[\'input\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Bucketize"
+    argspec: "args=[\'input\', \'boundaries\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CTCBeamSearchDecoder"
+    argspec: "args=[\'inputs\', \'sequence_length\', \'beam_width\', \'top_paths\', \'merge_repeated\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CTCGreedyDecoder"
+    argspec: "args=[\'inputs\', \'sequence_length\', \'merge_repeated\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CTCLoss"
+    argspec: "args=[\'inputs\', \'labels_indices\', \'labels_values\', \'sequence_length\', \'preprocess_collapse_repeated\', \'ctc_merge_repeated\', \'ignore_longer_outputs_than_inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CacheDataset"
+    argspec: "args=[\'input_dataset\', \'filename\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Cast"
+    argspec: "args=[\'x\', \'DstT\', \'Truncate\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Ceil"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CheckNumerics"
+    argspec: "args=[\'tensor\', \'message\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Cholesky"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CholeskyGrad"
+    argspec: "args=[\'l\', \'grad\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ClipByValue"
+    argspec: "args=[\'t\', \'clip_value_min\', \'clip_value_max\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CloseSummaryWriter"
+    argspec: "args=[\'writer\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CollectiveBcastRecv"
+    argspec: "args=[\'T\', \'group_size\', \'group_key\', \'instance_key\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CollectiveBcastSend"
+    argspec: "args=[\'input\', \'group_size\', \'group_key\', \'instance_key\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CollectiveReduce"
+    argspec: "args=[\'input\', \'group_size\', \'group_key\', \'instance_key\', \'merge_op\', \'final_op\', \'subdiv_offsets\', \'wait_for\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CombinedNonMaxSuppression"
+    argspec: "args=[\'boxes\', \'scores\', \'max_output_size_per_class\', \'max_total_size\', \'iou_threshold\', \'score_threshold\', \'pad_per_class\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CompareAndBitpack"
+    argspec: "args=[\'input\', \'threshold\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Complex"
+    argspec: "args=[\'real\', \'imag\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ComplexAbs"
+    argspec: "args=[\'x\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ComputeAccidentalHits"
+    argspec: "args=[\'true_classes\', \'sampled_candidates\', \'num_true\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Concat"
+    argspec: "args=[\'concat_dim\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ConcatOffset"
+    argspec: "args=[\'concat_dim\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ConcatV2"
+    argspec: "args=[\'values\', \'axis\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ConcatenateDataset"
+    argspec: "args=[\'input_dataset\', \'another_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ConditionalAccumulator"
+    argspec: "args=[\'dtype\', \'shape\', \'container\', \'shared_name\', \'reduction_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conj"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ConjugateTranspose"
+    argspec: "args=[\'x\', \'perm\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Const"
+    argspec: "args=[\'value\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ConsumeMutexLock"
+    argspec: "args=[\'mutex_lock\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ControlTrigger"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv2D"
+    argspec: "args=[\'input\', \'filter\', \'strides\', \'use_cudnn_on_gpu\', \'padding\', \'explicit_paddings\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv2DBackpropFilter"
+    argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'use_cudnn_on_gpu\', \'padding\', \'explicit_paddings\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv2DBackpropInput"
+    argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'use_cudnn_on_gpu\', \'padding\', \'explicit_paddings\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv3D"
+    argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv3DBackpropFilter"
+    argspec: "args=[\'input\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv3DBackpropFilterV2"
+    argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv3DBackpropInput"
+    argspec: "args=[\'input\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv3DBackpropInputV2"
+    argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Cos"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Cosh"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CountUpTo"
+    argspec: "args=[\'ref\', \'limit\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CreateSummaryDbWriter"
+    argspec: "args=[\'writer\', \'db_uri\', \'experiment_name\', \'run_name\', \'user_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CreateSummaryFileWriter"
+    argspec: "args=[\'writer\', \'logdir\', \'max_queue\', \'flush_millis\', \'filename_suffix\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CropAndResize"
+    argspec: "args=[\'image\', \'boxes\', \'box_ind\', \'crop_size\', \'method\', \'extrapolation_value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CropAndResizeGradBoxes"
+    argspec: "args=[\'grads\', \'image\', \'boxes\', \'box_ind\', \'method\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CropAndResizeGradImage"
+    argspec: "args=[\'grads\', \'boxes\', \'box_ind\', \'image_size\', \'T\', \'method\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Cross"
+    argspec: "args=[\'a\', \'b\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNN"
+    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\', \'is_training\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNBackprop"
+    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'output\', \'output_h\', \'output_c\', \'output_backprop\', \'output_h_backprop\', \'output_c_backprop\', \'reserve_space\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNBackpropV2"
+    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'output\', \'output_h\', \'output_c\', \'output_backprop\', \'output_h_backprop\', \'output_c_backprop\', \'reserve_space\', \'host_reserved\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNBackpropV3"
+    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'sequence_lengths\', \'output\', \'output_h\', \'output_c\', \'output_backprop\', \'output_h_backprop\', \'output_c_backprop\', \'reserve_space\', \'host_reserved\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNCanonicalToParams"
+    argspec: "args=[\'num_layers\', \'num_units\', \'input_size\', \'weights\', \'biases\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNParamsSize"
+    argspec: "args=[\'num_layers\', \'num_units\', \'input_size\', \'T\', \'S\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNParamsToCanonical"
+    argspec: "args=[\'num_layers\', \'num_units\', \'input_size\', \'params\', \'num_params\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNV2"
+    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\', \'is_training\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNV3"
+    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'sequence_lengths\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\', \'is_training\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Cumprod"
+    argspec: "args=[\'x\', \'axis\', \'exclusive\', \'reverse\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Cumsum"
+    argspec: "args=[\'x\', \'axis\', \'exclusive\', \'reverse\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DataFormatDimMap"
+    argspec: "args=[\'x\', \'src_format\', \'dst_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DataFormatVecPermute"
+    argspec: "args=[\'x\', \'src_format\', \'dst_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DatasetToGraph"
+    argspec: "args=[\'input_dataset\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DatasetToSingleElement"
+    argspec: "args=[\'dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DebugGradientIdentity"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DebugGradientRefIdentity"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeAndCropJpeg"
+    argspec: "args=[\'contents\', \'crop_window\', \'channels\', \'ratio\', \'fancy_upscaling\', \'try_recover_truncated\', \'acceptable_fraction\', \'dct_method\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeBase64"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeBmp"
+    argspec: "args=[\'contents\', \'channels\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeCSV"
+    argspec: "args=[\'records\', \'record_defaults\', \'field_delim\', \'use_quote_delim\', \'na_value\', \'select_cols\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeCompressed"
+    argspec: "args=[\'bytes\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeGif"
+    argspec: "args=[\'contents\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeJSONExample"
+    argspec: "args=[\'json_examples\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeJpeg"
+    argspec: "args=[\'contents\', \'channels\', \'ratio\', \'fancy_upscaling\', \'try_recover_truncated\', \'acceptable_fraction\', \'dct_method\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodePng"
+    argspec: "args=[\'contents\', \'channels\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeRaw"
+    argspec: "args=[\'bytes\', \'out_type\', \'little_endian\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeWav"
+    argspec: "args=[\'contents\', \'desired_channels\', \'desired_samples\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DeepCopy"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DeleteSessionTensor"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DenseToDenseSetOperation"
+    argspec: "args=[\'set1\', \'set2\', \'set_operation\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DenseToSparseSetOperation"
+    argspec: "args=[\'set1\', \'set2_indices\', \'set2_values\', \'set2_shape\', \'set_operation\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DepthToSpace"
+    argspec: "args=[\'input\', \'block_size\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DepthwiseConv2dNative"
+    argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DepthwiseConv2dNativeBackpropFilter"
+    argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DepthwiseConv2dNativeBackpropInput"
+    argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Dequantize"
+    argspec: "args=[\'input\', \'min_range\', \'max_range\', \'mode\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DeserializeIterator"
+    argspec: "args=[\'resource_handle\', \'serialized\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DeserializeManySparse"
+    argspec: "args=[\'serialized_sparse\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DeserializeSparse"
+    argspec: "args=[\'serialized_sparse\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DestroyResourceOp"
+    argspec: "args=[\'resource\', \'ignore_lookup_error\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DestroyTemporaryVariable"
+    argspec: "args=[\'ref\', \'var_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Diag"
+    argspec: "args=[\'diagonal\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DiagPart"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Digamma"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Dilation2D"
+    argspec: "args=[\'input\', \'filter\', \'strides\', \'rates\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Dilation2DBackpropFilter"
+    argspec: "args=[\'input\', \'filter\', \'out_backprop\', \'strides\', \'rates\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Dilation2DBackpropInput"
+    argspec: "args=[\'input\', \'filter\', \'out_backprop\', \'strides\', \'rates\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Div"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DivNoNan"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DrawBoundingBoxes"
+    argspec: "args=[\'images\', \'boxes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DynamicPartition"
+    argspec: "args=[\'data\', \'partitions\', \'num_partitions\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DynamicStitch"
+    argspec: "args=[\'indices\', \'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EagerPyFunc"
+    argspec: "args=[\'input\', \'token\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EditDistance"
+    argspec: "args=[\'hypothesis_indices\', \'hypothesis_values\', \'hypothesis_shape\', \'truth_indices\', \'truth_values\', \'truth_shape\', \'normalize\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Elu"
+    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EluGrad"
+    argspec: "args=[\'gradients\', \'outputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Empty"
+    argspec: "args=[\'shape\', \'dtype\', \'init\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EmptyTensorList"
+    argspec: "args=[\'element_shape\', \'max_num_elements\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EncodeBase64"
+    argspec: "args=[\'input\', \'pad\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EncodeJpeg"
+    argspec: "args=[\'image\', \'format\', \'quality\', \'progressive\', \'optimize_size\', \'chroma_downsampling\', \'density_unit\', \'x_density\', \'y_density\', \'xmp_metadata\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EncodePng"
+    argspec: "args=[\'image\', \'compression\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EncodeWav"
+    argspec: "args=[\'audio\', \'sample_rate\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EnsureShape"
+    argspec: "args=[\'input\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Enter"
+    argspec: "args=[\'data\', \'frame_name\', \'is_constant\', \'parallel_iterations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Equal"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Erf"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Erfc"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Exit"
+    argspec: "args=[\'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Exp"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExpandDims"
+    argspec: "args=[\'input\', \'dim\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalAssertNextDataset"
+    argspec: "args=[\'input_dataset\', \'transformations\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalBytesProducedStatsDataset"
+    argspec: "args=[\'input_dataset\', \'tag\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalCSVDataset"
+    argspec: "args=[\'filenames\', \'compression_type\', \'buffer_size\', \'header\', \'field_delim\', \'use_quote_delim\', \'na_value\', \'select_cols\', \'record_defaults\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalChooseFastestDataset"
+    argspec: "args=[\'input_datasets\', \'num_experiments\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalDatasetCardinality"
+    argspec: "args=[\'input_dataset\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalDatasetToTFRecord"
+    argspec: "args=[\'input_dataset\', \'filename\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalDenseToSparseBatchDataset"
+    argspec: "args=[\'input_dataset\', \'batch_size\', \'row_shape\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalDirectedInterleaveDataset"
+    argspec: "args=[\'selector_input_dataset\', \'data_input_datasets\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalGroupByReducerDataset"
+    argspec: "args=[\'input_dataset\', \'key_func_other_arguments\', \'init_func_other_arguments\', \'reduce_func_other_arguments\', \'finalize_func_other_arguments\', \'key_func\', \'init_func\', \'reduce_func\', \'finalize_func\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalGroupByWindowDataset"
+    argspec: "args=[\'input_dataset\', \'key_func_other_arguments\', \'reduce_func_other_arguments\', \'window_size_func_other_arguments\', \'key_func\', \'reduce_func\', \'window_size_func\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalIdentityIndexedDataset"
+    argspec: "args=[\'size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalIgnoreErrorsDataset"
+    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalIndexedDatasetGet"
+    argspec: "args=[\'materialized\', \'index\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalIndexedDatasetMaterialize"
+    argspec: "args=[\'dataset\', \'materialized\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalIteratorGetDevice"
+    argspec: "args=[\'resource\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalLMDBDataset"
+    argspec: "args=[\'filenames\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalLatencyStatsDataset"
+    argspec: "args=[\'input_dataset\', \'tag\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalMapAndBatchDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'batch_size\', \'num_parallel_calls\', \'drop_remainder\', \'f\', \'output_types\', \'output_shapes\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalMapDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\', \'use_inter_op_parallelism\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalMatchingFilesDataset"
+    argspec: "args=[\'patterns\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalMaterializedIndexDatasetHandle"
+    argspec: "args=[\'container\', \'shared_name\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalMaxIntraOpParallelismDataset"
+    argspec: "args=[\'input_dataset\', \'max_intra_op_parallelism\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalNonSerializableDataset"
+    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalNumaMapAndBatchDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'batch_size\', \'num_parallel_calls\', \'drop_remainder\', \'f\', \'output_types\', \'output_shapes\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalParallelInterleaveDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'cycle_length\', \'block_length\', \'sloppy\', \'buffer_output_elements\', \'prefetch_input_elements\', \'f\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalParseExampleDataset"
+    argspec: "args=[\'input_dataset\', \'num_parallel_calls\', \'dense_defaults\', \'sparse_keys\', \'dense_keys\', \'sparse_types\', \'dense_shapes\', \'output_types\', \'output_shapes\', \'sloppy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalPrivateThreadPoolDataset"
+    argspec: "args=[\'input_dataset\', \'num_threads\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalRandomDataset"
+    argspec: "args=[\'seed\', \'seed2\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalScanDataset"
+    argspec: "args=[\'input_dataset\', \'initial_state\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalSetStatsAggregatorDataset"
+    argspec: "args=[\'input_dataset\', \'stats_aggregator\', \'tag\', \'counter_prefix\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalSleepDataset"
+    argspec: "args=[\'input_dataset\', \'sleep_microseconds\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalSlidingWindowDataset"
+    argspec: "args=[\'input_dataset\', \'window_size\', \'window_shift\', \'window_stride\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalSqlDataset"
+    argspec: "args=[\'driver_name\', \'data_source_name\', \'query\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalStatsAggregatorHandle"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalStatsAggregatorSummary"
+    argspec: "args=[\'iterator\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalTakeWhileDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'predicate\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalThreadPoolDataset"
+    argspec: "args=[\'input_dataset\', \'thread_pool\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalThreadPoolHandle"
+    argspec: "args=[\'num_threads\', \'max_intra_op_parallelism\', \'display_name\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalUnbatchDataset"
+    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalUniqueDataset"
+    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Expm1"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExtractGlimpse"
+    argspec: "args=[\'input\', \'size\', \'offsets\', \'centered\', \'normalized\', \'uniform_noise\', \'noise\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExtractImagePatches"
+    argspec: "args=[\'images\', \'ksizes\', \'strides\', \'rates\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExtractJpegShape"
+    argspec: "args=[\'contents\', \'output_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExtractVolumePatches"
+    argspec: "args=[\'input\', \'ksizes\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FFT"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FFT2D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FFT3D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FIFOQueue"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FIFOQueueV2"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Fact"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeParam"
+    argspec: "args=[\'dtype\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeQuantWithMinMaxArgs"
+    argspec: "args=[\'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeQuantWithMinMaxArgsGradient"
+    argspec: "args=[\'gradients\', \'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeQuantWithMinMaxVars"
+    argspec: "args=[\'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeQuantWithMinMaxVarsGradient"
+    argspec: "args=[\'gradients\', \'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeQuantWithMinMaxVarsPerChannel"
+    argspec: "args=[\'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeQuantWithMinMaxVarsPerChannelGradient"
+    argspec: "args=[\'gradients\', \'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeQueue"
+    argspec: "args=[\'resource\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Fill"
+    argspec: "args=[\'dims\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FilterByLastComponentDataset"
+    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FilterDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'predicate\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FixedLengthRecordDataset"
+    argspec: "args=[\'filenames\', \'header_bytes\', \'record_bytes\', \'footer_bytes\', \'buffer_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FixedLengthRecordDatasetV2"
+    argspec: "args=[\'filenames\', \'header_bytes\', \'record_bytes\', \'footer_bytes\', \'buffer_size\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FixedLengthRecordReader"
+    argspec: "args=[\'header_bytes\', \'record_bytes\', \'footer_bytes\', \'hop_bytes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FixedLengthRecordReaderV2"
+    argspec: "args=[\'header_bytes\', \'record_bytes\', \'footer_bytes\', \'hop_bytes\', \'container\', \'shared_name\', \'encoding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FixedUnigramCandidateSampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'vocab_file\', \'distortion\', \'num_reserved_ids\', \'num_shards\', \'shard\', \'unigrams\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FlatMapDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Floor"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FloorDiv"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FloorMod"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FlushSummaryWriter"
+    argspec: "args=[\'writer\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "For"
+    argspec: "args=[\'start\', \'limit\', \'delta\', \'input\', \'body\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FractionalAvgPool"
+    argspec: "args=[\'value\', \'pooling_ratio\', \'pseudo_random\', \'overlapping\', \'deterministic\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FractionalAvgPoolGrad"
+    argspec: "args=[\'orig_input_tensor_shape\', \'out_backprop\', \'row_pooling_sequence\', \'col_pooling_sequence\', \'overlapping\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FractionalMaxPool"
+    argspec: "args=[\'value\', \'pooling_ratio\', \'pseudo_random\', \'overlapping\', \'deterministic\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FractionalMaxPoolGrad"
+    argspec: "args=[\'orig_input\', \'orig_output\', \'out_backprop\', \'row_pooling_sequence\', \'col_pooling_sequence\', \'overlapping\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FusedBatchNorm"
+    argspec: "args=[\'x\', \'scale\', \'offset\', \'mean\', \'variance\', \'epsilon\', \'data_format\', \'is_training\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FusedBatchNormGrad"
+    argspec: "args=[\'y_backprop\', \'x\', \'scale\', \'reserve_space_1\', \'reserve_space_2\', \'epsilon\', \'data_format\', \'is_training\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FusedBatchNormGradV2"
+    argspec: "args=[\'y_backprop\', \'x\', \'scale\', \'reserve_space_1\', \'reserve_space_2\', \'epsilon\', \'data_format\', \'is_training\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FusedBatchNormV2"
+    argspec: "args=[\'x\', \'scale\', \'offset\', \'mean\', \'variance\', \'epsilon\', \'data_format\', \'is_training\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FusedPadConv2D"
+    argspec: "args=[\'input\', \'paddings\', \'filter\', \'mode\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FusedResizeAndPadConv2D"
+    argspec: "args=[\'input\', \'size\', \'paddings\', \'filter\', \'resize_align_corners\', \'mode\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Gather"
+    argspec: "args=[\'params\', \'indices\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GatherNd"
+    argspec: "args=[\'params\', \'indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GatherV2"
+    argspec: "args=[\'params\', \'indices\', \'axis\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GenerateVocabRemapping"
+    argspec: "args=[\'new_vocab_file\', \'old_vocab_file\', \'new_vocab_offset\', \'num_new_vocab\', \'old_vocab_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GeneratorDataset"
+    argspec: "args=[\'init_func_other_args\', \'next_func_other_args\', \'finalize_func_other_args\', \'init_func\', \'next_func\', \'finalize_func\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GetSessionHandle"
+    argspec: "args=[\'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GetSessionHandleV2"
+    argspec: "args=[\'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GetSessionTensor"
+    argspec: "args=[\'handle\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Greater"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GreaterEqual"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GuaranteeConst"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "HSVToRGB"
+    argspec: "args=[\'images\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "HashTable"
+    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "HashTableV2"
+    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "HistogramFixedWidth"
+    argspec: "args=[\'values\', \'value_range\', \'nbins\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "HistogramSummary"
+    argspec: "args=[\'tag\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IFFT"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IFFT2D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IFFT3D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IRFFT"
+    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IRFFT2D"
+    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IRFFT3D"
+    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Identity"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IdentityN"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IdentityReader"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IdentityReaderV2"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "If"
+    argspec: "args=[\'cond\', \'input\', \'Tout\', \'then_branch\', \'else_branch\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Igamma"
+    argspec: "args=[\'a\', \'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IgammaGradA"
+    argspec: "args=[\'a\', \'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Igammac"
+    argspec: "args=[\'a\', \'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Imag"
+    argspec: "args=[\'input\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ImageSummary"
+    argspec: "args=[\'tag\', \'tensor\', \'max_images\', \'bad_color\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ImmutableConst"
+    argspec: "args=[\'dtype\', \'shape\', \'memory_region_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ImportEvent"
+    argspec: "args=[\'writer\', \'event\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InTopK"
+    argspec: "args=[\'predictions\', \'targets\', \'k\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InTopKV2"
+    argspec: "args=[\'predictions\', \'targets\', \'k\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InitializeTable"
+    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InitializeTableFromTextFile"
+    argspec: "args=[\'table_handle\', \'filename\', \'key_index\', \'value_index\', \'vocab_size\', \'delimiter\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InitializeTableFromTextFileV2"
+    argspec: "args=[\'table_handle\', \'filename\', \'key_index\', \'value_index\', \'vocab_size\', \'delimiter\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InitializeTableV2"
+    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InplaceAdd"
+    argspec: "args=[\'x\', \'i\', \'v\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InplaceSub"
+    argspec: "args=[\'x\', \'i\', \'v\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InplaceUpdate"
+    argspec: "args=[\'x\', \'i\', \'v\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InterleaveDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'cycle_length\', \'block_length\', \'f\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Inv"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InvGrad"
+    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Invert"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InvertPermutation"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IsBoostedTreesEnsembleInitialized"
+    argspec: "args=[\'tree_ensemble_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IsBoostedTreesQuantileStreamResourceInitialized"
+    argspec: "args=[\'quantile_stream_resource_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IsFinite"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IsInf"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IsNan"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IsVariableInitialized"
+    argspec: "args=[\'ref\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Iterator"
+    argspec: "args=[\'shared_name\', \'container\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IteratorFromStringHandle"
+    argspec: "args=[\'string_handle\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IteratorFromStringHandleV2"
+    argspec: "args=[\'string_handle\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IteratorGetNext"
+    argspec: "args=[\'iterator\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IteratorGetNextAsOptional"
+    argspec: "args=[\'iterator\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IteratorGetNextSync"
+    argspec: "args=[\'iterator\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IteratorToStringHandle"
+    argspec: "args=[\'resource_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IteratorV2"
+    argspec: "args=[\'shared_name\', \'container\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "L2Loss"
+    argspec: "args=[\'t\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LMDBReader"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LRN"
+    argspec: "args=[\'input\', \'depth_radius\', \'bias\', \'alpha\', \'beta\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LRNGrad"
+    argspec: "args=[\'input_grads\', \'input_image\', \'output_image\', \'depth_radius\', \'bias\', \'alpha\', \'beta\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LeakyRelu"
+    argspec: "args=[\'features\', \'alpha\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LeakyReluGrad"
+    argspec: "args=[\'gradients\', \'features\', \'alpha\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LearnedUnigramCandidateSampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LeftShift"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Less"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LessEqual"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Lgamma"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LinSpace"
+    argspec: "args=[\'start\', \'stop\', \'num\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ListDiff"
+    argspec: "args=[\'x\', \'y\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LoadAndRemapMatrix"
+    argspec: "args=[\'ckpt_path\', \'old_tensor_name\', \'row_remapping\', \'col_remapping\', \'initializing_values\', \'num_rows\', \'num_cols\', \'max_rows_in_memory\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Log"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Log1p"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LogMatrixDeterminant"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LogSoftmax"
+    argspec: "args=[\'logits\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LogUniformCandidateSampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LogicalAnd"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LogicalNot"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LogicalOr"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableExport"
+    argspec: "args=[\'table_handle\', \'Tkeys\', \'Tvalues\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableExportV2"
+    argspec: "args=[\'table_handle\', \'Tkeys\', \'Tvalues\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableFind"
+    argspec: "args=[\'table_handle\', \'keys\', \'default_value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableFindV2"
+    argspec: "args=[\'table_handle\', \'keys\', \'default_value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableImport"
+    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableImportV2"
+    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableInsert"
+    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableInsertV2"
+    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableRemoveV2"
+    argspec: "args=[\'table_handle\', \'keys\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableSize"
+    argspec: "args=[\'table_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableSizeV2"
+    argspec: "args=[\'table_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LoopCond"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LowerBound"
+    argspec: "args=[\'sorted_inputs\', \'values\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Lu"
+    argspec: "args=[\'input\', \'output_idx_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MakeIterator"
+    argspec: "args=[\'dataset\', \'iterator\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapClear"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\', \'use_inter_op_parallelism\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapDefun"
+    argspec: "args=[\'arguments\', \'captured_inputs\', \'output_types\', \'output_shapes\', \'f\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapIncompleteSize"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapPeek"
+    argspec: "args=[\'key\', \'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapSize"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapStage"
+    argspec: "args=[\'key\', \'indices\', \'values\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapUnstage"
+    argspec: "args=[\'key\', \'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapUnstageNoKey"
+    argspec: "args=[\'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatMul"
+    argspec: "args=[\'a\', \'b\', \'transpose_a\', \'transpose_b\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatchingFiles"
+    argspec: "args=[\'pattern\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixBandPart"
+    argspec: "args=[\'input\', \'num_lower\', \'num_upper\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixDeterminant"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixDiag"
+    argspec: "args=[\'diagonal\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixDiagPart"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixExponential"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixInverse"
+    argspec: "args=[\'input\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixLogarithm"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixSetDiag"
+    argspec: "args=[\'input\', \'diagonal\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixSolve"
+    argspec: "args=[\'matrix\', \'rhs\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixSolveLs"
+    argspec: "args=[\'matrix\', \'rhs\', \'l2_regularizer\', \'fast\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixSquareRoot"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixTriangularSolve"
+    argspec: "args=[\'matrix\', \'rhs\', \'lower\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Max"
+    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPool"
+    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPool3D"
+    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPool3DGrad"
+    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPool3DGradGrad"
+    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolGrad"
+    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolGradGrad"
+    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolGradGradV2"
+    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolGradGradWithArgmax"
+    argspec: "args=[\'input\', \'grad\', \'argmax\', \'ksize\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolGradV2"
+    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolGradWithArgmax"
+    argspec: "args=[\'input\', \'grad\', \'argmax\', \'ksize\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolV2"
+    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolWithArgmax"
+    argspec: "args=[\'input\', \'ksize\', \'strides\', \'Targmax\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Maximum"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Mean"
+    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Merge"
+    argspec: "args=[\'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MergeSummary"
+    argspec: "args=[\'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MergeV2Checkpoints"
+    argspec: "args=[\'checkpoint_prefixes\', \'destination_prefix\', \'delete_old_dirs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Mfcc"
+    argspec: "args=[\'spectrogram\', \'sample_rate\', \'upper_frequency_limit\', \'lower_frequency_limit\', \'filterbank_channel_count\', \'dct_coefficient_count\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Min"
+    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Minimum"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MirrorPad"
+    argspec: "args=[\'input\', \'paddings\', \'mode\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MirrorPadGrad"
+    argspec: "args=[\'input\', \'paddings\', \'mode\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Mod"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ModelDataset"
+    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Mul"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MultiDeviceIterator"
+    argspec: "args=[\'devices\', \'shared_name\', \'container\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MultiDeviceIteratorFromStringHandle"
+    argspec: "args=[\'string_handle\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MultiDeviceIteratorGetNextFromShard"
+    argspec: "args=[\'multi_device_iterator\', \'shard_num\', \'incarnation_id\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MultiDeviceIteratorInit"
+    argspec: "args=[\'dataset\', \'multi_device_iterator\', \'max_buffer_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MultiDeviceIteratorToStringHandle"
+    argspec: "args=[\'multi_device_iterator\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Multinomial"
+    argspec: "args=[\'logits\', \'num_samples\', \'seed\', \'seed2\', \'output_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutableDenseHashTable"
+    argspec: "args=[\'empty_key\', \'container\', \'shared_name\', \'use_node_name_sharing\', \'value_dtype\', \'value_shape\', \'initial_num_buckets\', \'max_load_factor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutableDenseHashTableV2"
+    argspec: "args=[\'empty_key\', \'deleted_key\', \'container\', \'shared_name\', \'use_node_name_sharing\', \'value_dtype\', \'value_shape\', \'initial_num_buckets\', \'max_load_factor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutableHashTable"
+    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutableHashTableOfTensors"
+    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\', \'value_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutableHashTableOfTensorsV2"
+    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\', \'value_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutableHashTableV2"
+    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutexLock"
+    argspec: "args=[\'mutex\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutexV2"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NcclAllReduce"
+    argspec: "args=[\'input\', \'reduction\', \'num_devices\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NcclBroadcast"
+    argspec: "args=[\'input\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NcclReduce"
+    argspec: "args=[\'input\', \'reduction\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Neg"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NextAfter"
+    argspec: "args=[\'x1\', \'x2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NextIteration"
+    argspec: "args=[\'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NoOp"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NonMaxSuppression"
+    argspec: "args=[\'boxes\', \'scores\', \'max_output_size\', \'iou_threshold\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NonMaxSuppressionV2"
+    argspec: "args=[\'boxes\', \'scores\', \'max_output_size\', \'iou_threshold\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NonMaxSuppressionV3"
+    argspec: "args=[\'boxes\', \'scores\', \'max_output_size\', \'iou_threshold\', \'score_threshold\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NonMaxSuppressionV4"
+    argspec: "args=[\'boxes\', \'scores\', \'max_output_size\', \'iou_threshold\', \'score_threshold\', \'pad_to_max_output_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NonMaxSuppressionWithOverlaps"
+    argspec: "args=[\'overlaps\', \'scores\', \'max_output_size\', \'overlap_threshold\', \'score_threshold\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NotEqual"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NthElement"
+    argspec: "args=[\'input\', \'n\', \'reverse\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OneHot"
+    argspec: "args=[\'indices\', \'depth\', \'on_value\', \'off_value\', \'axis\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OneShotIterator"
+    argspec: "args=[\'dataset_factory\', \'output_types\', \'output_shapes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OnesLike"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OptimizeDataset"
+    argspec: "args=[\'input_dataset\', \'optimizations\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OptionalFromValue"
+    argspec: "args=[\'components\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OptionalGetValue"
+    argspec: "args=[\'optional\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OptionalHasValue"
+    argspec: "args=[\'optional\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OptionalNone"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OrderedMapClear"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OrderedMapIncompleteSize"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OrderedMapPeek"
+    argspec: "args=[\'key\', \'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OrderedMapSize"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OrderedMapStage"
+    argspec: "args=[\'key\', \'indices\', \'values\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OrderedMapUnstage"
+    argspec: "args=[\'key\', \'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OrderedMapUnstageNoKey"
+    argspec: "args=[\'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Pack"
+    argspec: "args=[\'values\', \'axis\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Pad"
+    argspec: "args=[\'input\', \'paddings\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PadV2"
+    argspec: "args=[\'input\', \'paddings\', \'constant_values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PaddedBatchDataset"
+    argspec: "args=[\'input_dataset\', \'batch_size\', \'padded_shapes\', \'padding_values\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PaddedBatchDatasetV2"
+    argspec: "args=[\'input_dataset\', \'batch_size\', \'padded_shapes\', \'padding_values\', \'drop_remainder\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PaddingFIFOQueue"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PaddingFIFOQueueV2"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParallelConcat"
+    argspec: "args=[\'values\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParallelDynamicStitch"
+    argspec: "args=[\'indices\', \'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParallelInterleaveDatasetV2"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'cycle_length\', \'block_length\', \'num_parallel_calls\', \'f\', \'output_types\', \'output_shapes\', \'sloppy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParallelMapDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'num_parallel_calls\', \'f\', \'output_types\', \'output_shapes\', \'use_inter_op_parallelism\', \'sloppy\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParameterizedTruncatedNormal"
+    argspec: "args=[\'shape\', \'means\', \'stdevs\', \'minvals\', \'maxvals\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParseExample"
+    argspec: "args=[\'serialized\', \'names\', \'sparse_keys\', \'dense_keys\', \'dense_defaults\', \'sparse_types\', \'dense_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParseSequenceExample"
+    argspec: "args=[\'serialized\', \'debug_name\', \'context_dense_defaults\', \'feature_list_dense_missing_assumed_empty\', \'context_sparse_keys\', \'context_dense_keys\', \'feature_list_sparse_keys\', \'feature_list_dense_keys\', \'Ncontext_sparse\', \'Ncontext_dense\', \'Nfeature_list_sparse\', \'Nfeature_list_dense\', \'context_sparse_types\', \'feature_list_dense_types\', \'context_dense_shapes\', \'feature_list_sparse_types\', \'feature_list_dense_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParseSingleExample"
+    argspec: "args=[\'serialized\', \'dense_defaults\', \'num_sparse\', \'sparse_keys\', \'dense_keys\', \'sparse_types\', \'dense_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParseSingleSequenceExample"
+    argspec: "args=[\'serialized\', \'feature_list_dense_missing_assumed_empty\', \'context_sparse_keys\', \'context_dense_keys\', \'feature_list_sparse_keys\', \'feature_list_dense_keys\', \'context_dense_defaults\', \'debug_name\', \'context_sparse_types\', \'feature_list_dense_types\', \'context_dense_shapes\', \'feature_list_sparse_types\', \'feature_list_dense_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParseTensor"
+    argspec: "args=[\'serialized\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PartitionedCall"
+    argspec: "args=[\'args\', \'Tout\', \'f\', \'config\', \'config_proto\', \'executor_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Placeholder"
+    argspec: "args=[\'dtype\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PlaceholderV2"
+    argspec: "args=[\'dtype\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PlaceholderWithDefault"
+    argspec: "args=[\'input\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Polygamma"
+    argspec: "args=[\'a\', \'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PopulationCount"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Pow"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PrefetchDataset"
+    argspec: "args=[\'input_dataset\', \'buffer_size\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PreventGradient"
+    argspec: "args=[\'input\', \'message\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Print"
+    argspec: "args=[\'input\', \'data\', \'message\', \'first_n\', \'summarize\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PrintV2"
+    argspec: "args=[\'input\', \'output_stream\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PriorityQueue"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PriorityQueueV2"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Prod"
+    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PyFunc"
+    argspec: "args=[\'input\', \'token\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PyFuncStateless"
+    argspec: "args=[\'input\', \'token\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Qr"
+    argspec: "args=[\'input\', \'full_matrices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizeAndDequantize"
+    argspec: "args=[\'input\', \'signed_input\', \'num_bits\', \'range_given\', \'input_min\', \'input_max\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizeAndDequantizeV2"
+    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'signed_input\', \'num_bits\', \'range_given\', \'round_mode\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizeAndDequantizeV3"
+    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'num_bits\', \'signed_input\', \'range_given\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizeDownAndShrinkRange"
+    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizeV2"
+    argspec: "args=[\'input\', \'min_range\', \'max_range\', \'T\', \'mode\', \'round_mode\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedAdd"
+    argspec: "args=[\'x\', \'y\', \'min_x\', \'max_x\', \'min_y\', \'max_y\', \'Toutput\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedAvgPool"
+    argspec: "args=[\'input\', \'min_input\', \'max_input\', \'ksize\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedBatchNormWithGlobalNormalization"
+    argspec: "args=[\'t\', \'t_min\', \'t_max\', \'m\', \'m_min\', \'m_max\', \'v\', \'v_min\', \'v_max\', \'beta\', \'beta_min\', \'beta_max\', \'gamma\', \'gamma_min\', \'gamma_max\', \'out_type\', \'variance_epsilon\', \'scale_after_normalization\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedBiasAdd"
+    argspec: "args=[\'input\', \'bias\', \'min_input\', \'max_input\', \'min_bias\', \'max_bias\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConcat"
+    argspec: "args=[\'concat_dim\', \'values\', \'input_mins\', \'input_maxes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2D"
+    argspec: "args=[\'input\', \'filter\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DAndRelu"
+    argspec: "args=[\'input\', \'filter\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DAndReluAndRequantize"
+    argspec: "args=[\'input\', \'filter\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DAndRequantize"
+    argspec: "args=[\'input\', \'filter\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DWithBias"
+    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DWithBiasAndRelu"
+    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DWithBiasAndReluAndRequantize"
+    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DWithBiasAndRequantize"
+    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DWithBiasSignedSumAndReluAndRequantize"
+    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'summand\', \'min_summand\', \'max_summand\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DWithBiasSumAndRelu"
+    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'summand\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DWithBiasSumAndReluAndRequantize"
+    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'summand\', \'min_summand\', \'max_summand\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedInstanceNorm"
+    argspec: "args=[\'x\', \'x_min\', \'x_max\', \'output_range_given\', \'given_y_min\', \'given_y_max\', \'variance_epsilon\', \'min_separation\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedMatMul"
+    argspec: "args=[\'a\', \'b\', \'min_a\', \'max_a\', \'min_b\', \'max_b\', \'Toutput\', \'transpose_a\', \'transpose_b\', \'Tactivation\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedMaxPool"
+    argspec: "args=[\'input\', \'min_input\', \'max_input\', \'ksize\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedMul"
+    argspec: "args=[\'x\', \'y\', \'min_x\', \'max_x\', \'min_y\', \'max_y\', \'Toutput\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedRelu"
+    argspec: "args=[\'features\', \'min_features\', \'max_features\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedRelu6"
+    argspec: "args=[\'features\', \'min_features\', \'max_features\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedReluX"
+    argspec: "args=[\'features\', \'max_value\', \'min_features\', \'max_features\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedReshape"
+    argspec: "args=[\'tensor\', \'shape\', \'input_min\', \'input_max\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedResizeBilinear"
+    argspec: "args=[\'images\', \'size\', \'min\', \'max\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueClose"
+    argspec: "args=[\'handle\', \'cancel_pending_enqueues\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueCloseV2"
+    argspec: "args=[\'handle\', \'cancel_pending_enqueues\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueDequeue"
+    argspec: "args=[\'handle\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueDequeueMany"
+    argspec: "args=[\'handle\', \'n\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueDequeueManyV2"
+    argspec: "args=[\'handle\', \'n\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueDequeueUpTo"
+    argspec: "args=[\'handle\', \'n\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueDequeueUpToV2"
+    argspec: "args=[\'handle\', \'n\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueDequeueV2"
+    argspec: "args=[\'handle\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueEnqueue"
+    argspec: "args=[\'handle\', \'components\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueEnqueueMany"
+    argspec: "args=[\'handle\', \'components\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueEnqueueManyV2"
+    argspec: "args=[\'handle\', \'components\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueEnqueueV2"
+    argspec: "args=[\'handle\', \'components\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueIsClosed"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueIsClosedV2"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueSize"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueSizeV2"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RFFT"
+    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RFFT2D"
+    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RFFT3D"
+    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RGBToHSV"
+    argspec: "args=[\'images\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RaggedGather"
+    argspec: "args=[\'params_nested_splits\', \'params_dense_values\', \'indices\', \'OUTPUT_RAGGED_RANK\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RaggedRange"
+    argspec: "args=[\'starts\', \'limits\', \'deltas\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RaggedTensorToSparse"
+    argspec: "args=[\'rt_nested_splits\', \'rt_dense_values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomCrop"
+    argspec: "args=[\'image\', \'size\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomGamma"
+    argspec: "args=[\'shape\', \'alpha\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomGammaGrad"
+    argspec: "args=[\'alpha\', \'sample\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomPoisson"
+    argspec: "args=[\'shape\', \'rate\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomPoissonV2"
+    argspec: "args=[\'shape\', \'rate\', \'seed\', \'seed2\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomShuffle"
+    argspec: "args=[\'value\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomShuffleQueue"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'min_after_dequeue\', \'seed\', \'seed2\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomShuffleQueueV2"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'min_after_dequeue\', \'seed\', \'seed2\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomStandardNormal"
+    argspec: "args=[\'shape\', \'seed\', \'seed2\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomUniform"
+    argspec: "args=[\'shape\', \'seed\', \'seed2\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomUniformInt"
+    argspec: "args=[\'shape\', \'minval\', \'maxval\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Range"
+    argspec: "args=[\'start\', \'limit\', \'delta\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RangeDataset"
+    argspec: "args=[\'start\', \'stop\', \'step\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Rank"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReadFile"
+    argspec: "args=[\'filename\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReadVariableOp"
+    argspec: "args=[\'resource\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderNumRecordsProduced"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderNumRecordsProducedV2"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderNumWorkUnitsCompleted"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderNumWorkUnitsCompletedV2"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderRead"
+    argspec: "args=[\'reader_handle\', \'queue_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderReadUpTo"
+    argspec: "args=[\'reader_handle\', \'queue_handle\', \'num_records\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderReadUpToV2"
+    argspec: "args=[\'reader_handle\', \'queue_handle\', \'num_records\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderReadV2"
+    argspec: "args=[\'reader_handle\', \'queue_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderReset"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderResetV2"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderRestoreState"
+    argspec: "args=[\'reader_handle\', \'state\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderRestoreStateV2"
+    argspec: "args=[\'reader_handle\', \'state\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderSerializeState"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderSerializeStateV2"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Real"
+    argspec: "args=[\'input\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RealDiv"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Reciprocal"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReciprocalGrad"
+    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RecordInput"
+    argspec: "args=[\'file_pattern\', \'file_random_seed\', \'file_shuffle_shift_ratio\', \'file_buffer_size\', \'file_parallelism\', \'batch_size\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReduceDataset"
+    argspec: "args=[\'input_dataset\', \'initial_state\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\', \'use_inter_op_parallelism\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReduceJoin"
+    argspec: "args=[\'inputs\', \'reduction_indices\', \'keep_dims\', \'separator\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RefEnter"
+    argspec: "args=[\'data\', \'frame_name\', \'is_constant\', \'parallel_iterations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RefExit"
+    argspec: "args=[\'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RefIdentity"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RefMerge"
+    argspec: "args=[\'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RefNextIteration"
+    argspec: "args=[\'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RefSelect"
+    argspec: "args=[\'index\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RefSwitch"
+    argspec: "args=[\'data\', \'pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RegexFullMatch"
+    argspec: "args=[\'input\', \'pattern\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RegexReplace"
+    argspec: "args=[\'input\', \'pattern\', \'rewrite\', \'replace_global\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Relu"
+    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Relu6"
+    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Relu6Grad"
+    argspec: "args=[\'gradients\', \'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReluGrad"
+    argspec: "args=[\'gradients\', \'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RemoteCall"
+    argspec: "args=[\'target\', \'args\', \'Tout\', \'f\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RepeatDataset"
+    argspec: "args=[\'input_dataset\', \'count\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RequantizationRange"
+    argspec: "args=[\'input\', \'input_min\', \'input_max\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Requantize"
+    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'requested_output_min\', \'requested_output_max\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Reshape"
+    argspec: "args=[\'tensor\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResizeArea"
+    argspec: "args=[\'images\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResizeBicubic"
+    argspec: "args=[\'images\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResizeBicubicGrad"
+    argspec: "args=[\'grads\', \'original_image\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResizeBilinear"
+    argspec: "args=[\'images\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResizeBilinearGrad"
+    argspec: "args=[\'grads\', \'original_image\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResizeNearestNeighbor"
+    argspec: "args=[\'images\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResizeNearestNeighborGrad"
+    argspec: "args=[\'grads\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyAdaMax"
+    argspec: "args=[\'var\', \'m\', \'v\', \'beta1_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyAdadelta"
+    argspec: "args=[\'var\', \'accum\', \'accum_update\', \'lr\', \'rho\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'use_locking\', \'update_slots\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyAdagradDA"
+    argspec: "args=[\'var\', \'gradient_accumulator\', \'gradient_squared_accumulator\', \'grad\', \'lr\', \'l1\', \'l2\', \'global_step\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyAdam"
+    argspec: "args=[\'var\', \'m\', \'v\', \'beta1_power\', \'beta2_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyAdamWithAmsgrad"
+    argspec: "args=[\'var\', \'m\', \'v\', \'vhat\', \'beta1_power\', \'beta2_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyAddSign"
+    argspec: "args=[\'var\', \'m\', \'lr\', \'alpha\', \'sign_decay\', \'beta\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyCenteredRMSProp"
+    argspec: "args=[\'var\', \'mg\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyFtrl"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'lr\', \'l1\', \'l2\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyFtrlV2"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'lr\', \'l1\', \'l2\', \'l2_shrinkage\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyGradientDescent"
+    argspec: "args=[\'var\', \'alpha\', \'delta\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyKerasMomentum"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyMomentum"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyPowerSign"
+    argspec: "args=[\'var\', \'m\', \'lr\', \'logbase\', \'sign_decay\', \'beta\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyProximalAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'l1\', \'l2\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyProximalGradientDescent"
+    argspec: "args=[\'var\', \'alpha\', \'l1\', \'l2\', \'delta\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyRMSProp"
+    argspec: "args=[\'var\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceCountUpTo"
+    argspec: "args=[\'resource\', \'limit\', \'T\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceGather"
+    argspec: "args=[\'resource\', \'indices\', \'validate_indices\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterAdd"
+    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterDiv"
+    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterMax"
+    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterMin"
+    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterMul"
+    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterNdAdd"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterNdSub"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterNdUpdate"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterSub"
+    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterUpdate"
+    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyAdadelta"
+    argspec: "args=[\'var\', \'accum\', \'accum_update\', \'lr\', \'rho\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'use_locking\', \'update_slots\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyAdagradDA"
+    argspec: "args=[\'var\', \'gradient_accumulator\', \'gradient_squared_accumulator\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'global_step\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyCenteredRMSProp"
+    argspec: "args=[\'var\', \'mg\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyFtrl"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyFtrlV2"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'l2_shrinkage\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyKerasMomentum"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyMomentum"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyProximalAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'l1\', \'l2\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyProximalGradientDescent"
+    argspec: "args=[\'var\', \'alpha\', \'l1\', \'l2\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyRMSProp"
+    argspec: "args=[\'var\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceStridedSliceAssign"
+    argspec: "args=[\'ref\', \'begin\', \'end\', \'strides\', \'value\', \'begin_mask\', \'end_mask\', \'ellipsis_mask\', \'new_axis_mask\', \'shrink_axis_mask\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Restore"
+    argspec: "args=[\'file_pattern\', \'tensor_name\', \'dt\', \'preferred_shard\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RestoreSlice"
+    argspec: "args=[\'file_pattern\', \'tensor_name\', \'shape_and_slice\', \'dt\', \'preferred_shard\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RestoreV2"
+    argspec: "args=[\'prefix\', \'tensor_names\', \'shape_and_slices\', \'dtypes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Reverse"
+    argspec: "args=[\'tensor\', \'dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReverseSequence"
+    argspec: "args=[\'input\', \'seq_lengths\', \'seq_dim\', \'batch_dim\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReverseV2"
+    argspec: "args=[\'tensor\', \'axis\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RightShift"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Rint"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Roll"
+    argspec: "args=[\'input\', \'shift\', \'axis\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Round"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Rsqrt"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RsqrtGrad"
+    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SampleDistortedBoundingBox"
+    argspec: "args=[\'image_size\', \'bounding_boxes\', \'seed\', \'seed2\', \'min_object_covered\', \'aspect_ratio_range\', \'area_range\', \'max_attempts\', \'use_image_if_no_bounding_boxes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SampleDistortedBoundingBoxV2"
+    argspec: "args=[\'image_size\', \'bounding_boxes\', \'min_object_covered\', \'seed\', \'seed2\', \'aspect_ratio_range\', \'area_range\', \'max_attempts\', \'use_image_if_no_bounding_boxes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Save"
+    argspec: "args=[\'filename\', \'tensor_names\', \'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SaveSlices"
+    argspec: "args=[\'filename\', \'tensor_names\', \'shapes_and_slices\', \'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SaveV2"
+    argspec: "args=[\'prefix\', \'tensor_names\', \'shape_and_slices\', \'tensors\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScalarSummary"
+    argspec: "args=[\'tags\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScaleAndTranslate"
+    argspec: "args=[\'images\', \'size\', \'scale\', \'translation\', \'kernel_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScaleAndTranslateGrad"
+    argspec: "args=[\'grads\', \'original_image\', \'scale\', \'translation\', \'kernel_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterAdd"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterDiv"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterMax"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterMin"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterMul"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterNd"
+    argspec: "args=[\'indices\', \'updates\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterNdAdd"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterNdNonAliasingAdd"
+    argspec: "args=[\'input\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterNdSub"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterNdUpdate"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterSub"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterUpdate"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SdcaFprint"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SdcaOptimizer"
+    argspec: "args=[\'sparse_example_indices\', \'sparse_feature_indices\', \'sparse_feature_values\', \'dense_features\', \'example_weights\', \'example_labels\', \'sparse_indices\', \'sparse_weights\', \'dense_weights\', \'example_state_data\', \'loss_type\', \'adaptative\', \'l1\', \'l2\', \'num_loss_partitions\', \'num_inner_iterations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SdcaOptimizerV2"
+    argspec: "args=[\'sparse_example_indices\', \'sparse_feature_indices\', \'sparse_feature_values\', \'dense_features\', \'example_weights\', \'example_labels\', \'sparse_indices\', \'sparse_weights\', \'dense_weights\', \'example_state_data\', \'loss_type\', \'adaptive\', \'l1\', \'l2\', \'num_loss_partitions\', \'num_inner_iterations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SdcaShrinkL1"
+    argspec: "args=[\'weights\', \'l1\', \'l2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SegmentMax"
+    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SegmentMean"
+    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SegmentMin"
+    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SegmentProd"
+    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SegmentSum"
+    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Select"
+    argspec: "args=[\'condition\', \'t\', \'e\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SelfAdjointEig"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SelfAdjointEigV2"
+    argspec: "args=[\'input\', \'compute_v\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Selu"
+    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SeluGrad"
+    argspec: "args=[\'gradients\', \'outputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SerializeIterator"
+    argspec: "args=[\'resource_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SerializeManySparse"
+    argspec: "args=[\'sparse_indices\', \'sparse_values\', \'sparse_shape\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SerializeSparse"
+    argspec: "args=[\'sparse_indices\', \'sparse_values\', \'sparse_shape\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SerializeTensor"
+    argspec: "args=[\'tensor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SetSize"
+    argspec: "args=[\'set_indices\', \'set_values\', \'set_shape\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Shape"
+    argspec: "args=[\'input\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ShapeN"
+    argspec: "args=[\'input\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ShardedFilename"
+    argspec: "args=[\'basename\', \'shard\', \'num_shards\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ShardedFilespec"
+    argspec: "args=[\'basename\', \'num_shards\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ShuffleAndRepeatDataset"
+    argspec: "args=[\'input_dataset\', \'buffer_size\', \'seed\', \'seed2\', \'count\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ShuffleDataset"
+    argspec: "args=[\'input_dataset\', \'buffer_size\', \'seed\', \'seed2\', \'reshuffle_each_iteration\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Sigmoid"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SigmoidGrad"
+    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Sign"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Sin"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Sinh"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Size"
+    argspec: "args=[\'input\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SkipDataset"
+    argspec: "args=[\'input_dataset\', \'count\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Slice"
+    argspec: "args=[\'input\', \'begin\', \'size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Snapshot"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Softmax"
+    argspec: "args=[\'logits\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SoftmaxCrossEntropyWithLogits"
+    argspec: "args=[\'features\', \'labels\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Softplus"
+    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SoftplusGrad"
+    argspec: "args=[\'gradients\', \'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Softsign"
+    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SoftsignGrad"
+    argspec: "args=[\'gradients\', \'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SpaceToBatch"
+    argspec: "args=[\'input\', \'paddings\', \'block_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SpaceToBatchND"
+    argspec: "args=[\'input\', \'block_shape\', \'paddings\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SpaceToDepth"
+    argspec: "args=[\'input\', \'block_size\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseAccumulatorApplyGradient"
+    argspec: "args=[\'handle\', \'local_step\', \'gradient_indices\', \'gradient_values\', \'gradient_shape\', \'has_known_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseAccumulatorTakeGradient"
+    argspec: "args=[\'handle\', \'num_required\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseAdd"
+    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b_indices\', \'b_values\', \'b_shape\', \'thresh\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseAddGrad"
+    argspec: "args=[\'backprop_val_grad\', \'a_indices\', \'b_indices\', \'sum_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyAdadelta"
+    argspec: "args=[\'var\', \'accum\', \'accum_update\', \'lr\', \'rho\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'use_locking\', \'update_slots\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyAdagradDA"
+    argspec: "args=[\'var\', \'gradient_accumulator\', \'gradient_squared_accumulator\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'global_step\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyCenteredRMSProp"
+    argspec: "args=[\'var\', \'mg\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyFtrl"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyFtrlV2"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'l2_shrinkage\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyMomentum"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyProximalAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'l1\', \'l2\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyProximalGradientDescent"
+    argspec: "args=[\'var\', \'alpha\', \'l1\', \'l2\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyRMSProp"
+    argspec: "args=[\'var\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseConcat"
+    argspec: "args=[\'indices\', \'values\', \'shapes\', \'concat_dim\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseConditionalAccumulator"
+    argspec: "args=[\'dtype\', \'shape\', \'container\', \'shared_name\', \'reduction_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseCross"
+    argspec: "args=[\'indices\', \'values\', \'shapes\', \'dense_inputs\', \'hashed_output\', \'num_buckets\', \'hash_key\', \'out_type\', \'internal_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseDenseCwiseAdd"
+    argspec: "args=[\'sp_indices\', \'sp_values\', \'sp_shape\', \'dense\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseDenseCwiseDiv"
+    argspec: "args=[\'sp_indices\', \'sp_values\', \'sp_shape\', \'dense\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseDenseCwiseMul"
+    argspec: "args=[\'sp_indices\', \'sp_values\', \'sp_shape\', \'dense\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseFillEmptyRows"
+    argspec: "args=[\'indices\', \'values\', \'dense_shape\', \'default_value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseFillEmptyRowsGrad"
+    argspec: "args=[\'reverse_index_map\', \'grad_values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseMatMul"
+    argspec: "args=[\'a\', \'b\', \'transpose_a\', \'transpose_b\', \'a_is_sparse\', \'b_is_sparse\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseReduceMax"
+    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseReduceMaxSparse"
+    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseReduceSum"
+    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseReduceSumSparse"
+    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseReorder"
+    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseReshape"
+    argspec: "args=[\'input_indices\', \'input_shape\', \'new_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentMean"
+    argspec: "args=[\'data\', \'indices\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentMeanGrad"
+    argspec: "args=[\'grad\', \'indices\', \'segment_ids\', \'output_dim0\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentMeanWithNumSegments"
+    argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentSqrtN"
+    argspec: "args=[\'data\', \'indices\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentSqrtNGrad"
+    argspec: "args=[\'grad\', \'indices\', \'segment_ids\', \'output_dim0\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentSqrtNWithNumSegments"
+    argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentSum"
+    argspec: "args=[\'data\', \'indices\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentSumWithNumSegments"
+    argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSlice"
+    argspec: "args=[\'indices\', \'values\', \'shape\', \'start\', \'size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSliceGrad"
+    argspec: "args=[\'backprop_val_grad\', \'input_indices\', \'input_start\', \'output_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSoftmax"
+    argspec: "args=[\'sp_indices\', \'sp_values\', \'sp_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSoftmaxCrossEntropyWithLogits"
+    argspec: "args=[\'features\', \'labels\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSparseMaximum"
+    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b_indices\', \'b_values\', \'b_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSparseMinimum"
+    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b_indices\', \'b_values\', \'b_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSplit"
+    argspec: "args=[\'split_dim\', \'indices\', \'values\', \'shape\', \'num_split\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseTensorDenseAdd"
+    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseTensorDenseMatMul"
+    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b\', \'adjoint_a\', \'adjoint_b\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseTensorSliceDataset"
+    argspec: "args=[\'indices\', \'values\', \'dense_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseToDense"
+    argspec: "args=[\'sparse_indices\', \'output_shape\', \'sparse_values\', \'default_value\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseToSparseSetOperation"
+    argspec: "args=[\'set1_indices\', \'set1_values\', \'set1_shape\', \'set2_indices\', \'set2_values\', \'set2_shape\', \'set_operation\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Split"
+    argspec: "args=[\'split_dim\', \'value\', \'num_split\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SplitV"
+    argspec: "args=[\'value\', \'size_splits\', \'split_dim\', \'num_split\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Sqrt"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SqrtGrad"
+    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Square"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SquaredDifference"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Squeeze"
+    argspec: "args=[\'input\', \'squeeze_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Stack"
+    argspec: "args=[\'elem_type\', \'stack_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StackClose"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StackCloseV2"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StackPop"
+    argspec: "args=[\'handle\', \'elem_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StackPopV2"
+    argspec: "args=[\'handle\', \'elem_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StackPush"
+    argspec: "args=[\'handle\', \'elem\', \'swap_memory\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StackPushV2"
+    argspec: "args=[\'handle\', \'elem\', \'swap_memory\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StackV2"
+    argspec: "args=[\'max_size\', \'elem_type\', \'stack_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Stage"
+    argspec: "args=[\'values\', \'capacity\', \'memory_limit\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StageClear"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StagePeek"
+    argspec: "args=[\'index\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StageSize"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatefulPartitionedCall"
+    argspec: "args=[\'args\', \'Tout\', \'f\', \'config\', \'config_proto\', \'executor_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatelessIf"
+    argspec: "args=[\'cond\', \'input\', \'Tout\', \'then_branch\', \'else_branch\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatelessMultinomial"
+    argspec: "args=[\'logits\', \'num_samples\', \'seed\', \'output_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatelessRandomNormal"
+    argspec: "args=[\'shape\', \'seed\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatelessRandomUniform"
+    argspec: "args=[\'shape\', \'seed\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatelessRandomUniformInt"
+    argspec: "args=[\'shape\', \'seed\', \'minval\', \'maxval\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatelessTruncatedNormal"
+    argspec: "args=[\'shape\', \'seed\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatelessWhile"
+    argspec: "args=[\'input\', \'cond\', \'body\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StaticRegexFullMatch"
+    argspec: "args=[\'input\', \'pattern\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StaticRegexReplace"
+    argspec: "args=[\'input\', \'pattern\', \'rewrite\', \'replace_global\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StopGradient"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StridedSlice"
+    argspec: "args=[\'input\', \'begin\', \'end\', \'strides\', \'begin_mask\', \'end_mask\', \'ellipsis_mask\', \'new_axis_mask\', \'shrink_axis_mask\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StridedSliceAssign"
+    argspec: "args=[\'ref\', \'begin\', \'end\', \'strides\', \'value\', \'begin_mask\', \'end_mask\', \'ellipsis_mask\', \'new_axis_mask\', \'shrink_axis_mask\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StridedSliceGrad"
+    argspec: "args=[\'shape\', \'begin\', \'end\', \'strides\', \'dy\', \'begin_mask\', \'end_mask\', \'ellipsis_mask\', \'new_axis_mask\', \'shrink_axis_mask\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringFormat"
+    argspec: "args=[\'inputs\', \'template\', \'placeholder\', \'summarize\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringJoin"
+    argspec: "args=[\'inputs\', \'separator\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringLength"
+    argspec: "args=[\'input\', \'unit\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringSplit"
+    argspec: "args=[\'input\', \'delimiter\', \'skip_empty\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringSplitV2"
+    argspec: "args=[\'input\', \'sep\', \'maxsplit\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringStrip"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringToHashBucket"
+    argspec: "args=[\'string_tensor\', \'num_buckets\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringToHashBucketFast"
+    argspec: "args=[\'input\', \'num_buckets\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringToHashBucketStrong"
+    argspec: "args=[\'input\', \'num_buckets\', \'key\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringToNumber"
+    argspec: "args=[\'string_tensor\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Sub"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Substr"
+    argspec: "args=[\'input\', \'pos\', \'len\', \'unit\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Sum"
+    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SummaryWriter"
+    argspec: "args=[\'shared_name\', \'container\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Svd"
+    argspec: "args=[\'input\', \'compute_uv\', \'full_matrices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Switch"
+    argspec: "args=[\'data\', \'pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SymbolicGradient"
+    argspec: "args=[\'input\', \'Tout\', \'f\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TFRecordDataset"
+    argspec: "args=[\'filenames\', \'compression_type\', \'buffer_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TFRecordReader"
+    argspec: "args=[\'container\', \'shared_name\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TFRecordReaderV2"
+    argspec: "args=[\'container\', \'shared_name\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TakeDataset"
+    argspec: "args=[\'input_dataset\', \'count\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TakeManySparseFromTensorsMap"
+    argspec: "args=[\'sparse_handles\', \'dtype\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Tan"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Tanh"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TanhGrad"
+    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TemporaryVariable"
+    argspec: "args=[\'shape\', \'dtype\', \'var_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArray"
+    argspec: "args=[\'size\', \'dtype\', \'dynamic_size\', \'clear_after_read\', \'tensor_array_name\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayClose"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayCloseV2"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayCloseV3"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayConcat"
+    argspec: "args=[\'handle\', \'flow_in\', \'dtype\', \'element_shape_except0\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayConcatV2"
+    argspec: "args=[\'handle\', \'flow_in\', \'dtype\', \'element_shape_except0\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayConcatV3"
+    argspec: "args=[\'handle\', \'flow_in\', \'dtype\', \'element_shape_except0\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayGather"
+    argspec: "args=[\'handle\', \'indices\', \'flow_in\', \'dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayGatherV2"
+    argspec: "args=[\'handle\', \'indices\', \'flow_in\', \'dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayGatherV3"
+    argspec: "args=[\'handle\', \'indices\', \'flow_in\', \'dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayGrad"
+    argspec: "args=[\'handle\', \'flow_in\', \'source\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayGradV2"
+    argspec: "args=[\'handle\', \'flow_in\', \'source\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayGradV3"
+    argspec: "args=[\'handle\', \'flow_in\', \'source\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayGradWithShape"
+    argspec: "args=[\'handle\', \'flow_in\', \'shape_to_prepend\', \'source\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayPack"
+    argspec: "args=[\'handle\', \'flow_in\', \'dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayRead"
+    argspec: "args=[\'handle\', \'index\', \'flow_in\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayReadV2"
+    argspec: "args=[\'handle\', \'index\', \'flow_in\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayReadV3"
+    argspec: "args=[\'handle\', \'index\', \'flow_in\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayScatter"
+    argspec: "args=[\'handle\', \'indices\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayScatterV2"
+    argspec: "args=[\'handle\', \'indices\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayScatterV3"
+    argspec: "args=[\'handle\', \'indices\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArraySize"
+    argspec: "args=[\'handle\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArraySizeV2"
+    argspec: "args=[\'handle\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArraySizeV3"
+    argspec: "args=[\'handle\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArraySplit"
+    argspec: "args=[\'handle\', \'value\', \'lengths\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArraySplitV2"
+    argspec: "args=[\'handle\', \'value\', \'lengths\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArraySplitV3"
+    argspec: "args=[\'handle\', \'value\', \'lengths\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayUnpack"
+    argspec: "args=[\'handle\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayV2"
+    argspec: "args=[\'size\', \'dtype\', \'element_shape\', \'dynamic_size\', \'clear_after_read\', \'tensor_array_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayV3"
+    argspec: "args=[\'size\', \'dtype\', \'element_shape\', \'dynamic_size\', \'clear_after_read\', \'identical_element_shapes\', \'tensor_array_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayWrite"
+    argspec: "args=[\'handle\', \'index\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayWriteV2"
+    argspec: "args=[\'handle\', \'index\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayWriteV3"
+    argspec: "args=[\'handle\', \'index\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorDataset"
+    argspec: "args=[\'components\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListConcat"
+    argspec: "args=[\'input_handle\', \'element_dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListConcatLists"
+    argspec: "args=[\'input_a\', \'input_b\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListElementShape"
+    argspec: "args=[\'input_handle\', \'shape_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListFromTensor"
+    argspec: "args=[\'tensor\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListGather"
+    argspec: "args=[\'input_handle\', \'indices\', \'element_shape\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListGetItem"
+    argspec: "args=[\'input_handle\', \'index\', \'element_shape\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListLength"
+    argspec: "args=[\'input_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListPopBack"
+    argspec: "args=[\'input_handle\', \'element_shape\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListPushBack"
+    argspec: "args=[\'input_handle\', \'tensor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListPushBackBatch"
+    argspec: "args=[\'input_handles\', \'tensor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListReserve"
+    argspec: "args=[\'element_shape\', \'num_elements\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListResize"
+    argspec: "args=[\'input_handle\', \'size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListScatter"
+    argspec: "args=[\'tensor\', \'indices\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListScatterV2"
+    argspec: "args=[\'tensor\', \'indices\', \'element_shape\', \'num_elements\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListSetItem"
+    argspec: "args=[\'input_handle\', \'index\', \'item\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListSplit"
+    argspec: "args=[\'tensor\', \'element_shape\', \'lengths\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListStack"
+    argspec: "args=[\'input_handle\', \'element_shape\', \'element_dtype\', \'num_elements\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorScatterAdd"
+    argspec: "args=[\'tensor\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorScatterSub"
+    argspec: "args=[\'tensor\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorScatterUpdate"
+    argspec: "args=[\'tensor\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorSliceDataset"
+    argspec: "args=[\'components\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorSummary"
+    argspec: "args=[\'tensor\', \'description\', \'labels\', \'display_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorSummaryV2"
+    argspec: "args=[\'tag\', \'tensor\', \'serialized_summary_metadata\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TextLineDataset"
+    argspec: "args=[\'filenames\', \'compression_type\', \'buffer_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TextLineReader"
+    argspec: "args=[\'skip_header_lines\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TextLineReaderV2"
+    argspec: "args=[\'skip_header_lines\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ThreadUnsafeUnigramCandidateSampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Tile"
+    argspec: "args=[\'input\', \'multiples\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TileGrad"
+    argspec: "args=[\'input\', \'multiples\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Timestamp"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TopK"
+    argspec: "args=[\'input\', \'k\', \'sorted\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TopKV2"
+    argspec: "args=[\'input\', \'k\', \'sorted\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Transpose"
+    argspec: "args=[\'x\', \'perm\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TruncateDiv"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TruncateMod"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TruncatedNormal"
+    argspec: "args=[\'shape\', \'seed\', \'seed2\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnicodeDecode"
+    argspec: "args=[\'input\', \'input_encoding\', \'errors\', \'replacement_char\', \'replace_control_characters\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnicodeDecodeWithOffsets"
+    argspec: "args=[\'input\', \'input_encoding\', \'errors\', \'replacement_char\', \'replace_control_characters\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnicodeEncode"
+    argspec: "args=[\'input_values\', \'input_splits\', \'errors\', \'output_encoding\', \'replacement_char\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnicodeScript"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnicodeTranscode"
+    argspec: "args=[\'input\', \'input_encoding\', \'output_encoding\', \'errors\', \'replacement_char\', \'replace_control_characters\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UniformCandidateSampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Unique"
+    argspec: "args=[\'x\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UniqueV2"
+    argspec: "args=[\'x\', \'axis\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UniqueWithCounts"
+    argspec: "args=[\'x\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UniqueWithCountsV2"
+    argspec: "args=[\'x\', \'axis\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Unpack"
+    argspec: "args=[\'value\', \'num\', \'axis\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnravelIndex"
+    argspec: "args=[\'indices\', \'dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnsortedSegmentMax"
+    argspec: "args=[\'data\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnsortedSegmentMin"
+    argspec: "args=[\'data\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnsortedSegmentProd"
+    argspec: "args=[\'data\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnsortedSegmentSum"
+    argspec: "args=[\'data\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Unstage"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnwrapDatasetVariant"
+    argspec: "args=[\'input_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UpperBound"
+    argspec: "args=[\'sorted_inputs\', \'values\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "VarHandleOp"
+    argspec: "args=[\'container\', \'shared_name\', \'dtype\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "VarIsInitializedOp"
+    argspec: "args=[\'resource\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Variable"
+    argspec: "args=[\'shape\', \'dtype\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "VariableShape"
+    argspec: "args=[\'input\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "VariableV2"
+    argspec: "args=[\'shape\', \'dtype\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Where"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "While"
+    argspec: "args=[\'input\', \'cond\', \'body\', \'output_shapes\', \'parallel_iterations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WholeFileReader"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WholeFileReaderV2"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WindowDataset"
+    argspec: "args=[\'input_dataset\', \'size\', \'shift\', \'stride\', \'drop_remainder\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WrapDatasetVariant"
+    argspec: "args=[\'input_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WriteAudioSummary"
+    argspec: "args=[\'writer\', \'step\', \'tag\', \'tensor\', \'sample_rate\', \'max_outputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WriteFile"
+    argspec: "args=[\'filename\', \'contents\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WriteGraphSummary"
+    argspec: "args=[\'writer\', \'step\', \'tensor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WriteHistogramSummary"
+    argspec: "args=[\'writer\', \'step\', \'tag\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WriteImageSummary"
+    argspec: "args=[\'writer\', \'step\', \'tag\', \'tensor\', \'bad_color\', \'max_images\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WriteScalarSummary"
+    argspec: "args=[\'writer\', \'step\', \'tag\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WriteSummary"
+    argspec: "args=[\'writer\', \'step\', \'tensor\', \'tag\', \'summary_metadata\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Xdivy"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Xlogy"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ZerosLike"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Zeta"
+    argspec: "args=[\'x\', \'q\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ZipDataset"
+    argspec: "args=[\'input_datasets\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index e1eef058f4..bb7ea2e13e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -272,6 +272,10 @@ tf_module {
     name: "random_uniform_initializer"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "raw_ops"
+    mtype: "<type \'module\'>"
+  }
   member {
     name: "resource"
     mtype: "<class \'tensorflow.python.framework.dtypes.DType\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt
new file mode 100644
index 0000000000..c52a0cd41f
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt
@@ -0,0 +1,4023 @@
+path: "tensorflow.raw_ops"
+tf_module {
+  member_method {
+    name: "Abort"
+    argspec: "args=[\'error_msg\', \'exit_without_error\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Abs"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AccumulateNV2"
+    argspec: "args=[\'inputs\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AccumulatorApplyGradient"
+    argspec: "args=[\'handle\', \'local_step\', \'gradient\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AccumulatorNumAccumulated"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AccumulatorSetGlobalStep"
+    argspec: "args=[\'handle\', \'new_global_step\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AccumulatorTakeGradient"
+    argspec: "args=[\'handle\', \'num_required\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Acos"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Acosh"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Add"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AddManySparseToTensorsMap"
+    argspec: "args=[\'sparse_indices\', \'sparse_values\', \'sparse_shape\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AddN"
+    argspec: "args=[\'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AddSparseToTensorsMap"
+    argspec: "args=[\'sparse_indices\', \'sparse_values\', \'sparse_shape\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AddV2"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AdjustContrast"
+    argspec: "args=[\'images\', \'contrast_factor\', \'min_value\', \'max_value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AdjustContrastv2"
+    argspec: "args=[\'images\', \'contrast_factor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AdjustHue"
+    argspec: "args=[\'images\', \'delta\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AdjustSaturation"
+    argspec: "args=[\'images\', \'scale\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "All"
+    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AllCandidateSampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Angle"
+    argspec: "args=[\'input\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AnonymousIterator"
+    argspec: "args=[\'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Any"
+    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyAdaMax"
+    argspec: "args=[\'var\', \'m\', \'v\', \'beta1_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyAdadelta"
+    argspec: "args=[\'var\', \'accum\', \'accum_update\', \'lr\', \'rho\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'use_locking\', \'update_slots\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyAdagradDA"
+    argspec: "args=[\'var\', \'gradient_accumulator\', \'gradient_squared_accumulator\', \'grad\', \'lr\', \'l1\', \'l2\', \'global_step\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyAdam"
+    argspec: "args=[\'var\', \'m\', \'v\', \'beta1_power\', \'beta2_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyAddSign"
+    argspec: "args=[\'var\', \'m\', \'lr\', \'alpha\', \'sign_decay\', \'beta\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyCenteredRMSProp"
+    argspec: "args=[\'var\', \'mg\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyFtrl"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'lr\', \'l1\', \'l2\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyFtrlV2"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'lr\', \'l1\', \'l2\', \'l2_shrinkage\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyGradientDescent"
+    argspec: "args=[\'var\', \'alpha\', \'delta\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyMomentum"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyPowerSign"
+    argspec: "args=[\'var\', \'m\', \'lr\', \'logbase\', \'sign_decay\', \'beta\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyProximalAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'l1\', \'l2\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyProximalGradientDescent"
+    argspec: "args=[\'var\', \'alpha\', \'l1\', \'l2\', \'delta\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyRMSProp"
+    argspec: "args=[\'var\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApproximateEqual"
+    argspec: "args=[\'x\', \'y\', \'tolerance\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ArgMax"
+    argspec: "args=[\'input\', \'dimension\', \'output_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ArgMin"
+    argspec: "args=[\'input\', \'dimension\', \'output_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AsString"
+    argspec: "args=[\'input\', \'precision\', \'scientific\', \'shortest\', \'width\', \'fill\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Asin"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Asinh"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Assert"
+    argspec: "args=[\'condition\', \'data\', \'summarize\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Assign"
+    argspec: "args=[\'ref\', \'value\', \'validate_shape\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AssignAdd"
+    argspec: "args=[\'ref\', \'value\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AssignAddVariableOp"
+    argspec: "args=[\'resource\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AssignSub"
+    argspec: "args=[\'ref\', \'value\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AssignSubVariableOp"
+    argspec: "args=[\'resource\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AssignVariableOp"
+    argspec: "args=[\'resource\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Atan"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Atan2"
+    argspec: "args=[\'y\', \'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Atanh"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AudioSpectrogram"
+    argspec: "args=[\'input\', \'window_size\', \'stride\', \'magnitude_squared\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AudioSummary"
+    argspec: "args=[\'tag\', \'tensor\', \'sample_rate\', \'max_outputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AudioSummaryV2"
+    argspec: "args=[\'tag\', \'tensor\', \'sample_rate\', \'max_outputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AvgPool"
+    argspec: "args=[\'value\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AvgPool3D"
+    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AvgPool3DGrad"
+    argspec: "args=[\'orig_input_shape\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AvgPoolGrad"
+    argspec: "args=[\'orig_input_shape\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Barrier"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BarrierClose"
+    argspec: "args=[\'handle\', \'cancel_pending_enqueues\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BarrierIncompleteSize"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BarrierInsertMany"
+    argspec: "args=[\'handle\', \'keys\', \'values\', \'component_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BarrierReadySize"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BarrierTakeMany"
+    argspec: "args=[\'handle\', \'num_elements\', \'component_types\', \'allow_small_batch\', \'wait_for_incomplete\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchCholesky"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchCholeskyGrad"
+    argspec: "args=[\'l\', \'grad\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchDataset"
+    argspec: "args=[\'input_dataset\', \'batch_size\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchDatasetV2"
+    argspec: "args=[\'input_dataset\', \'batch_size\', \'drop_remainder\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchFFT"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchFFT2D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchFFT3D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchIFFT"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchIFFT2D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchIFFT3D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatMul"
+    argspec: "args=[\'x\', \'y\', \'adj_x\', \'adj_y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixBandPart"
+    argspec: "args=[\'input\', \'num_lower\', \'num_upper\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixDeterminant"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixDiag"
+    argspec: "args=[\'diagonal\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixDiagPart"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixInverse"
+    argspec: "args=[\'input\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixSetDiag"
+    argspec: "args=[\'input\', \'diagonal\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixSolve"
+    argspec: "args=[\'matrix\', \'rhs\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixSolveLs"
+    argspec: "args=[\'matrix\', \'rhs\', \'l2_regularizer\', \'fast\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixTriangularSolve"
+    argspec: "args=[\'matrix\', \'rhs\', \'lower\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchNormWithGlobalNormalization"
+    argspec: "args=[\'t\', \'m\', \'v\', \'beta\', \'gamma\', \'variance_epsilon\', \'scale_after_normalization\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchNormWithGlobalNormalizationGrad"
+    argspec: "args=[\'t\', \'m\', \'v\', \'gamma\', \'backprop\', \'variance_epsilon\', \'scale_after_normalization\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchSelfAdjointEig"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchSelfAdjointEigV2"
+    argspec: "args=[\'input\', \'compute_v\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchSvd"
+    argspec: "args=[\'input\', \'compute_uv\', \'full_matrices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchToSpace"
+    argspec: "args=[\'input\', \'crops\', \'block_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchToSpaceND"
+    argspec: "args=[\'input\', \'block_shape\', \'crops\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BesselI0e"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BesselI1e"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Betainc"
+    argspec: "args=[\'a\', \'b\', \'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BiasAdd"
+    argspec: "args=[\'value\', \'bias\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BiasAddGrad"
+    argspec: "args=[\'out_backprop\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BiasAddV1"
+    argspec: "args=[\'value\', \'bias\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Bincount"
+    argspec: "args=[\'arr\', \'size\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Bitcast"
+    argspec: "args=[\'input\', \'type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BitwiseAnd"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BitwiseOr"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BitwiseXor"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesBucketize"
+    argspec: "args=[\'float_values\', \'bucket_boundaries\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesCalculateBestGainsPerFeature"
+    argspec: "args=[\'node_id_range\', \'stats_summary_list\', \'l1\', \'l2\', \'tree_complexity\', \'min_node_weight\', \'max_splits\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesCenterBias"
+    argspec: "args=[\'tree_ensemble_handle\', \'mean_gradients\', \'mean_hessians\', \'l1\', \'l2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesCreateEnsemble"
+    argspec: "args=[\'tree_ensemble_handle\', \'stamp_token\', \'tree_ensemble_serialized\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesCreateQuantileStreamResource"
+    argspec: "args=[\'quantile_stream_resource_handle\', \'epsilon\', \'num_streams\', \'max_elements\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesDeserializeEnsemble"
+    argspec: "args=[\'tree_ensemble_handle\', \'stamp_token\', \'tree_ensemble_serialized\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesEnsembleResourceHandleOp"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesExampleDebugOutputs"
+    argspec: "args=[\'tree_ensemble_handle\', \'bucketized_features\', \'logits_dimension\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesGetEnsembleStates"
+    argspec: "args=[\'tree_ensemble_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesMakeQuantileSummaries"
+    argspec: "args=[\'float_values\', \'example_weights\', \'epsilon\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesMakeStatsSummary"
+    argspec: "args=[\'node_ids\', \'gradients\', \'hessians\', \'bucketized_features_list\', \'max_splits\', \'num_buckets\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesPredict"
+    argspec: "args=[\'tree_ensemble_handle\', \'bucketized_features\', \'logits_dimension\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesQuantileStreamResourceAddSummaries"
+    argspec: "args=[\'quantile_stream_resource_handle\', \'summaries\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesQuantileStreamResourceDeserialize"
+    argspec: "args=[\'quantile_stream_resource_handle\', \'bucket_boundaries\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesQuantileStreamResourceFlush"
+    argspec: "args=[\'quantile_stream_resource_handle\', \'num_buckets\', \'generate_quantiles\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesQuantileStreamResourceGetBucketBoundaries"
+    argspec: "args=[\'quantile_stream_resource_handle\', \'num_features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesQuantileStreamResourceHandleOp"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesSerializeEnsemble"
+    argspec: "args=[\'tree_ensemble_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesTrainingPredict"
+    argspec: "args=[\'tree_ensemble_handle\', \'cached_tree_ids\', \'cached_node_ids\', \'bucketized_features\', \'logits_dimension\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesUpdateEnsemble"
+    argspec: "args=[\'tree_ensemble_handle\', \'feature_ids\', \'node_ids\', \'gains\', \'thresholds\', \'left_node_contribs\', \'right_node_contribs\', \'max_depth\', \'learning_rate\', \'pruning_mode\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BroadcastArgs"
+    argspec: "args=[\'s0\', \'s1\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BroadcastGradientArgs"
+    argspec: "args=[\'s0\', \'s1\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BroadcastTo"
+    argspec: "args=[\'input\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Bucketize"
+    argspec: "args=[\'input\', \'boundaries\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CTCBeamSearchDecoder"
+    argspec: "args=[\'inputs\', \'sequence_length\', \'beam_width\', \'top_paths\', \'merge_repeated\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CTCGreedyDecoder"
+    argspec: "args=[\'inputs\', \'sequence_length\', \'merge_repeated\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CTCLoss"
+    argspec: "args=[\'inputs\', \'labels_indices\', \'labels_values\', \'sequence_length\', \'preprocess_collapse_repeated\', \'ctc_merge_repeated\', \'ignore_longer_outputs_than_inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CacheDataset"
+    argspec: "args=[\'input_dataset\', \'filename\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Cast"
+    argspec: "args=[\'x\', \'DstT\', \'Truncate\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Ceil"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CheckNumerics"
+    argspec: "args=[\'tensor\', \'message\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Cholesky"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CholeskyGrad"
+    argspec: "args=[\'l\', \'grad\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ClipByValue"
+    argspec: "args=[\'t\', \'clip_value_min\', \'clip_value_max\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CloseSummaryWriter"
+    argspec: "args=[\'writer\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CollectiveBcastRecv"
+    argspec: "args=[\'T\', \'group_size\', \'group_key\', \'instance_key\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CollectiveBcastSend"
+    argspec: "args=[\'input\', \'group_size\', \'group_key\', \'instance_key\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CollectiveReduce"
+    argspec: "args=[\'input\', \'group_size\', \'group_key\', \'instance_key\', \'merge_op\', \'final_op\', \'subdiv_offsets\', \'wait_for\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CombinedNonMaxSuppression"
+    argspec: "args=[\'boxes\', \'scores\', \'max_output_size_per_class\', \'max_total_size\', \'iou_threshold\', \'score_threshold\', \'pad_per_class\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CompareAndBitpack"
+    argspec: "args=[\'input\', \'threshold\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Complex"
+    argspec: "args=[\'real\', \'imag\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ComplexAbs"
+    argspec: "args=[\'x\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ComputeAccidentalHits"
+    argspec: "args=[\'true_classes\', \'sampled_candidates\', \'num_true\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Concat"
+    argspec: "args=[\'concat_dim\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ConcatOffset"
+    argspec: "args=[\'concat_dim\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ConcatV2"
+    argspec: "args=[\'values\', \'axis\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ConcatenateDataset"
+    argspec: "args=[\'input_dataset\', \'another_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ConditionalAccumulator"
+    argspec: "args=[\'dtype\', \'shape\', \'container\', \'shared_name\', \'reduction_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conj"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ConjugateTranspose"
+    argspec: "args=[\'x\', \'perm\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Const"
+    argspec: "args=[\'value\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ConsumeMutexLock"
+    argspec: "args=[\'mutex_lock\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ControlTrigger"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv2D"
+    argspec: "args=[\'input\', \'filter\', \'strides\', \'use_cudnn_on_gpu\', \'padding\', \'explicit_paddings\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv2DBackpropFilter"
+    argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'use_cudnn_on_gpu\', \'padding\', \'explicit_paddings\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv2DBackpropInput"
+    argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'use_cudnn_on_gpu\', \'padding\', \'explicit_paddings\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv3D"
+    argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv3DBackpropFilter"
+    argspec: "args=[\'input\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv3DBackpropFilterV2"
+    argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv3DBackpropInput"
+    argspec: "args=[\'input\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv3DBackpropInputV2"
+    argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Cos"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Cosh"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CountUpTo"
+    argspec: "args=[\'ref\', \'limit\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CreateSummaryDbWriter"
+    argspec: "args=[\'writer\', \'db_uri\', \'experiment_name\', \'run_name\', \'user_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CreateSummaryFileWriter"
+    argspec: "args=[\'writer\', \'logdir\', \'max_queue\', \'flush_millis\', \'filename_suffix\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CropAndResize"
+    argspec: "args=[\'image\', \'boxes\', \'box_ind\', \'crop_size\', \'method\', \'extrapolation_value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CropAndResizeGradBoxes"
+    argspec: "args=[\'grads\', \'image\', \'boxes\', \'box_ind\', \'method\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CropAndResizeGradImage"
+    argspec: "args=[\'grads\', \'boxes\', \'box_ind\', \'image_size\', \'T\', \'method\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Cross"
+    argspec: "args=[\'a\', \'b\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNN"
+    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\', \'is_training\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNBackprop"
+    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'output\', \'output_h\', \'output_c\', \'output_backprop\', \'output_h_backprop\', \'output_c_backprop\', \'reserve_space\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNBackpropV2"
+    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'output\', \'output_h\', \'output_c\', \'output_backprop\', \'output_h_backprop\', \'output_c_backprop\', \'reserve_space\', \'host_reserved\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNBackpropV3"
+    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'sequence_lengths\', \'output\', \'output_h\', \'output_c\', \'output_backprop\', \'output_h_backprop\', \'output_c_backprop\', \'reserve_space\', \'host_reserved\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNCanonicalToParams"
+    argspec: "args=[\'num_layers\', \'num_units\', \'input_size\', \'weights\', \'biases\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNParamsSize"
+    argspec: "args=[\'num_layers\', \'num_units\', \'input_size\', \'T\', \'S\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNParamsToCanonical"
+    argspec: "args=[\'num_layers\', \'num_units\', \'input_size\', \'params\', \'num_params\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNV2"
+    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\', \'is_training\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNV3"
+    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'sequence_lengths\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\', \'is_training\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Cumprod"
+    argspec: "args=[\'x\', \'axis\', \'exclusive\', \'reverse\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Cumsum"
+    argspec: "args=[\'x\', \'axis\', \'exclusive\', \'reverse\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DataFormatDimMap"
+    argspec: "args=[\'x\', \'src_format\', \'dst_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DataFormatVecPermute"
+    argspec: "args=[\'x\', \'src_format\', \'dst_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DatasetToGraph"
+    argspec: "args=[\'input_dataset\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DatasetToSingleElement"
+    argspec: "args=[\'dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DebugGradientIdentity"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DebugGradientRefIdentity"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeAndCropJpeg"
+    argspec: "args=[\'contents\', \'crop_window\', \'channels\', \'ratio\', \'fancy_upscaling\', \'try_recover_truncated\', \'acceptable_fraction\', \'dct_method\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeBase64"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeBmp"
+    argspec: "args=[\'contents\', \'channels\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeCSV"
+    argspec: "args=[\'records\', \'record_defaults\', \'field_delim\', \'use_quote_delim\', \'na_value\', \'select_cols\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeCompressed"
+    argspec: "args=[\'bytes\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeGif"
+    argspec: "args=[\'contents\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeJSONExample"
+    argspec: "args=[\'json_examples\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeJpeg"
+    argspec: "args=[\'contents\', \'channels\', \'ratio\', \'fancy_upscaling\', \'try_recover_truncated\', \'acceptable_fraction\', \'dct_method\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodePng"
+    argspec: "args=[\'contents\', \'channels\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeRaw"
+    argspec: "args=[\'bytes\', \'out_type\', \'little_endian\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeWav"
+    argspec: "args=[\'contents\', \'desired_channels\', \'desired_samples\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DeepCopy"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DeleteSessionTensor"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DenseToDenseSetOperation"
+    argspec: "args=[\'set1\', \'set2\', \'set_operation\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DenseToSparseSetOperation"
+    argspec: "args=[\'set1\', \'set2_indices\', \'set2_values\', \'set2_shape\', \'set_operation\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DepthToSpace"
+    argspec: "args=[\'input\', \'block_size\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DepthwiseConv2dNative"
+    argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DepthwiseConv2dNativeBackpropFilter"
+    argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DepthwiseConv2dNativeBackpropInput"
+    argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Dequantize"
+    argspec: "args=[\'input\', \'min_range\', \'max_range\', \'mode\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DeserializeIterator"
+    argspec: "args=[\'resource_handle\', \'serialized\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DeserializeManySparse"
+    argspec: "args=[\'serialized_sparse\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DeserializeSparse"
+    argspec: "args=[\'serialized_sparse\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DestroyResourceOp"
+    argspec: "args=[\'resource\', \'ignore_lookup_error\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DestroyTemporaryVariable"
+    argspec: "args=[\'ref\', \'var_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Diag"
+    argspec: "args=[\'diagonal\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DiagPart"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Digamma"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Dilation2D"
+    argspec: "args=[\'input\', \'filter\', \'strides\', \'rates\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Dilation2DBackpropFilter"
+    argspec: "args=[\'input\', \'filter\', \'out_backprop\', \'strides\', \'rates\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Dilation2DBackpropInput"
+    argspec: "args=[\'input\', \'filter\', \'out_backprop\', \'strides\', \'rates\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Div"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DivNoNan"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DrawBoundingBoxes"
+    argspec: "args=[\'images\', \'boxes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DynamicPartition"
+    argspec: "args=[\'data\', \'partitions\', \'num_partitions\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DynamicStitch"
+    argspec: "args=[\'indices\', \'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EagerPyFunc"
+    argspec: "args=[\'input\', \'token\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EditDistance"
+    argspec: "args=[\'hypothesis_indices\', \'hypothesis_values\', \'hypothesis_shape\', \'truth_indices\', \'truth_values\', \'truth_shape\', \'normalize\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Elu"
+    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EluGrad"
+    argspec: "args=[\'gradients\', \'outputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Empty"
+    argspec: "args=[\'shape\', \'dtype\', \'init\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EmptyTensorList"
+    argspec: "args=[\'element_shape\', \'max_num_elements\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EncodeBase64"
+    argspec: "args=[\'input\', \'pad\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EncodeJpeg"
+    argspec: "args=[\'image\', \'format\', \'quality\', \'progressive\', \'optimize_size\', \'chroma_downsampling\', \'density_unit\', \'x_density\', \'y_density\', \'xmp_metadata\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EncodePng"
+    argspec: "args=[\'image\', \'compression\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EncodeWav"
+    argspec: "args=[\'audio\', \'sample_rate\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EnsureShape"
+    argspec: "args=[\'input\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Enter"
+    argspec: "args=[\'data\', \'frame_name\', \'is_constant\', \'parallel_iterations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Equal"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Erf"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Erfc"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Exit"
+    argspec: "args=[\'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Exp"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExpandDims"
+    argspec: "args=[\'input\', \'dim\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalAssertNextDataset"
+    argspec: "args=[\'input_dataset\', \'transformations\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalBytesProducedStatsDataset"
+    argspec: "args=[\'input_dataset\', \'tag\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalCSVDataset"
+    argspec: "args=[\'filenames\', \'compression_type\', \'buffer_size\', \'header\', \'field_delim\', \'use_quote_delim\', \'na_value\', \'select_cols\', \'record_defaults\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalChooseFastestDataset"
+    argspec: "args=[\'input_datasets\', \'num_experiments\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalDatasetCardinality"
+    argspec: "args=[\'input_dataset\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalDatasetToTFRecord"
+    argspec: "args=[\'input_dataset\', \'filename\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalDenseToSparseBatchDataset"
+    argspec: "args=[\'input_dataset\', \'batch_size\', \'row_shape\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalDirectedInterleaveDataset"
+    argspec: "args=[\'selector_input_dataset\', \'data_input_datasets\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalGroupByReducerDataset"
+    argspec: "args=[\'input_dataset\', \'key_func_other_arguments\', \'init_func_other_arguments\', \'reduce_func_other_arguments\', \'finalize_func_other_arguments\', \'key_func\', \'init_func\', \'reduce_func\', \'finalize_func\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalGroupByWindowDataset"
+    argspec: "args=[\'input_dataset\', \'key_func_other_arguments\', \'reduce_func_other_arguments\', \'window_size_func_other_arguments\', \'key_func\', \'reduce_func\', \'window_size_func\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalIdentityIndexedDataset"
+    argspec: "args=[\'size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalIgnoreErrorsDataset"
+    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalIndexedDatasetGet"
+    argspec: "args=[\'materialized\', \'index\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalIndexedDatasetMaterialize"
+    argspec: "args=[\'dataset\', \'materialized\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalIteratorGetDevice"
+    argspec: "args=[\'resource\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalLMDBDataset"
+    argspec: "args=[\'filenames\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalLatencyStatsDataset"
+    argspec: "args=[\'input_dataset\', \'tag\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalMapAndBatchDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'batch_size\', \'num_parallel_calls\', \'drop_remainder\', \'f\', \'output_types\', \'output_shapes\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalMapDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\', \'use_inter_op_parallelism\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalMatchingFilesDataset"
+    argspec: "args=[\'patterns\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalMaterializedIndexDatasetHandle"
+    argspec: "args=[\'container\', \'shared_name\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalMaxIntraOpParallelismDataset"
+    argspec: "args=[\'input_dataset\', \'max_intra_op_parallelism\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalNonSerializableDataset"
+    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalNumaMapAndBatchDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'batch_size\', \'num_parallel_calls\', \'drop_remainder\', \'f\', \'output_types\', \'output_shapes\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalParallelInterleaveDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'cycle_length\', \'block_length\', \'sloppy\', \'buffer_output_elements\', \'prefetch_input_elements\', \'f\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalParseExampleDataset"
+    argspec: "args=[\'input_dataset\', \'num_parallel_calls\', \'dense_defaults\', \'sparse_keys\', \'dense_keys\', \'sparse_types\', \'dense_shapes\', \'output_types\', \'output_shapes\', \'sloppy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalPrivateThreadPoolDataset"
+    argspec: "args=[\'input_dataset\', \'num_threads\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalRandomDataset"
+    argspec: "args=[\'seed\', \'seed2\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalScanDataset"
+    argspec: "args=[\'input_dataset\', \'initial_state\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalSetStatsAggregatorDataset"
+    argspec: "args=[\'input_dataset\', \'stats_aggregator\', \'tag\', \'counter_prefix\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalSleepDataset"
+    argspec: "args=[\'input_dataset\', \'sleep_microseconds\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalSlidingWindowDataset"
+    argspec: "args=[\'input_dataset\', \'window_size\', \'window_shift\', \'window_stride\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalSqlDataset"
+    argspec: "args=[\'driver_name\', \'data_source_name\', \'query\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalStatsAggregatorHandle"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalStatsAggregatorSummary"
+    argspec: "args=[\'iterator\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalTakeWhileDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'predicate\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalThreadPoolDataset"
+    argspec: "args=[\'input_dataset\', \'thread_pool\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalThreadPoolHandle"
+    argspec: "args=[\'num_threads\', \'max_intra_op_parallelism\', \'display_name\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalUnbatchDataset"
+    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalUniqueDataset"
+    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Expm1"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExtractGlimpse"
+    argspec: "args=[\'input\', \'size\', \'offsets\', \'centered\', \'normalized\', \'uniform_noise\', \'noise\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExtractImagePatches"
+    argspec: "args=[\'images\', \'ksizes\', \'strides\', \'rates\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExtractJpegShape"
+    argspec: "args=[\'contents\', \'output_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExtractVolumePatches"
+    argspec: "args=[\'input\', \'ksizes\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FFT"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FFT2D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FFT3D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FIFOQueue"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FIFOQueueV2"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Fact"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeParam"
+    argspec: "args=[\'dtype\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeQuantWithMinMaxArgs"
+    argspec: "args=[\'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeQuantWithMinMaxArgsGradient"
+    argspec: "args=[\'gradients\', \'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeQuantWithMinMaxVars"
+    argspec: "args=[\'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeQuantWithMinMaxVarsGradient"
+    argspec: "args=[\'gradients\', \'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeQuantWithMinMaxVarsPerChannel"
+    argspec: "args=[\'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeQuantWithMinMaxVarsPerChannelGradient"
+    argspec: "args=[\'gradients\', \'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeQueue"
+    argspec: "args=[\'resource\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Fill"
+    argspec: "args=[\'dims\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FilterByLastComponentDataset"
+    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FilterDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'predicate\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FixedLengthRecordDataset"
+    argspec: "args=[\'filenames\', \'header_bytes\', \'record_bytes\', \'footer_bytes\', \'buffer_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FixedLengthRecordDatasetV2"
+    argspec: "args=[\'filenames\', \'header_bytes\', \'record_bytes\', \'footer_bytes\', \'buffer_size\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FixedLengthRecordReader"
+    argspec: "args=[\'header_bytes\', \'record_bytes\', \'footer_bytes\', \'hop_bytes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FixedLengthRecordReaderV2"
+    argspec: "args=[\'header_bytes\', \'record_bytes\', \'footer_bytes\', \'hop_bytes\', \'container\', \'shared_name\', \'encoding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FixedUnigramCandidateSampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'vocab_file\', \'distortion\', \'num_reserved_ids\', \'num_shards\', \'shard\', \'unigrams\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FlatMapDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Floor"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FloorDiv"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FloorMod"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FlushSummaryWriter"
+    argspec: "args=[\'writer\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "For"
+    argspec: "args=[\'start\', \'limit\', \'delta\', \'input\', \'body\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FractionalAvgPool"
+    argspec: "args=[\'value\', \'pooling_ratio\', \'pseudo_random\', \'overlapping\', \'deterministic\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FractionalAvgPoolGrad"
+    argspec: "args=[\'orig_input_tensor_shape\', \'out_backprop\', \'row_pooling_sequence\', \'col_pooling_sequence\', \'overlapping\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FractionalMaxPool"
+    argspec: "args=[\'value\', \'pooling_ratio\', \'pseudo_random\', \'overlapping\', \'deterministic\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FractionalMaxPoolGrad"
+    argspec: "args=[\'orig_input\', \'orig_output\', \'out_backprop\', \'row_pooling_sequence\', \'col_pooling_sequence\', \'overlapping\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FusedBatchNorm"
+    argspec: "args=[\'x\', \'scale\', \'offset\', \'mean\', \'variance\', \'epsilon\', \'data_format\', \'is_training\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FusedBatchNormGrad"
+    argspec: "args=[\'y_backprop\', \'x\', \'scale\', \'reserve_space_1\', \'reserve_space_2\', \'epsilon\', \'data_format\', \'is_training\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FusedBatchNormGradV2"
+    argspec: "args=[\'y_backprop\', \'x\', \'scale\', \'reserve_space_1\', \'reserve_space_2\', \'epsilon\', \'data_format\', \'is_training\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FusedBatchNormV2"
+    argspec: "args=[\'x\', \'scale\', \'offset\', \'mean\', \'variance\', \'epsilon\', \'data_format\', \'is_training\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FusedPadConv2D"
+    argspec: "args=[\'input\', \'paddings\', \'filter\', \'mode\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FusedResizeAndPadConv2D"
+    argspec: "args=[\'input\', \'size\', \'paddings\', \'filter\', \'resize_align_corners\', \'mode\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Gather"
+    argspec: "args=[\'params\', \'indices\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GatherNd"
+    argspec: "args=[\'params\', \'indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GatherV2"
+    argspec: "args=[\'params\', \'indices\', \'axis\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GenerateVocabRemapping"
+    argspec: "args=[\'new_vocab_file\', \'old_vocab_file\', \'new_vocab_offset\', \'num_new_vocab\', \'old_vocab_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GeneratorDataset"
+    argspec: "args=[\'init_func_other_args\', \'next_func_other_args\', \'finalize_func_other_args\', \'init_func\', \'next_func\', \'finalize_func\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GetSessionHandle"
+    argspec: "args=[\'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GetSessionHandleV2"
+    argspec: "args=[\'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GetSessionTensor"
+    argspec: "args=[\'handle\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Greater"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GreaterEqual"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GuaranteeConst"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "HSVToRGB"
+    argspec: "args=[\'images\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "HashTable"
+    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "HashTableV2"
+    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "HistogramFixedWidth"
+    argspec: "args=[\'values\', \'value_range\', \'nbins\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "HistogramSummary"
+    argspec: "args=[\'tag\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IFFT"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IFFT2D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IFFT3D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IRFFT"
+    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IRFFT2D"
+    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IRFFT3D"
+    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Identity"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IdentityN"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IdentityReader"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IdentityReaderV2"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "If"
+    argspec: "args=[\'cond\', \'input\', \'Tout\', \'then_branch\', \'else_branch\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Igamma"
+    argspec: "args=[\'a\', \'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IgammaGradA"
+    argspec: "args=[\'a\', \'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Igammac"
+    argspec: "args=[\'a\', \'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Imag"
+    argspec: "args=[\'input\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ImageSummary"
+    argspec: "args=[\'tag\', \'tensor\', \'max_images\', \'bad_color\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ImmutableConst"
+    argspec: "args=[\'dtype\', \'shape\', \'memory_region_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ImportEvent"
+    argspec: "args=[\'writer\', \'event\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InTopK"
+    argspec: "args=[\'predictions\', \'targets\', \'k\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InTopKV2"
+    argspec: "args=[\'predictions\', \'targets\', \'k\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InitializeTable"
+    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InitializeTableFromTextFile"
+    argspec: "args=[\'table_handle\', \'filename\', \'key_index\', \'value_index\', \'vocab_size\', \'delimiter\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InitializeTableFromTextFileV2"
+    argspec: "args=[\'table_handle\', \'filename\', \'key_index\', \'value_index\', \'vocab_size\', \'delimiter\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InitializeTableV2"
+    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InplaceAdd"
+    argspec: "args=[\'x\', \'i\', \'v\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InplaceSub"
+    argspec: "args=[\'x\', \'i\', \'v\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InplaceUpdate"
+    argspec: "args=[\'x\', \'i\', \'v\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InterleaveDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'cycle_length\', \'block_length\', \'f\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Inv"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InvGrad"
+    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Invert"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InvertPermutation"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IsBoostedTreesEnsembleInitialized"
+    argspec: "args=[\'tree_ensemble_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IsBoostedTreesQuantileStreamResourceInitialized"
+    argspec: "args=[\'quantile_stream_resource_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IsFinite"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IsInf"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IsNan"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IsVariableInitialized"
+    argspec: "args=[\'ref\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Iterator"
+    argspec: "args=[\'shared_name\', \'container\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IteratorFromStringHandle"
+    argspec: "args=[\'string_handle\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IteratorFromStringHandleV2"
+    argspec: "args=[\'string_handle\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IteratorGetNext"
+    argspec: "args=[\'iterator\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IteratorGetNextAsOptional"
+    argspec: "args=[\'iterator\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IteratorGetNextSync"
+    argspec: "args=[\'iterator\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IteratorToStringHandle"
+    argspec: "args=[\'resource_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IteratorV2"
+    argspec: "args=[\'shared_name\', \'container\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "L2Loss"
+    argspec: "args=[\'t\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LMDBReader"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LRN"
+    argspec: "args=[\'input\', \'depth_radius\', \'bias\', \'alpha\', \'beta\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LRNGrad"
+    argspec: "args=[\'input_grads\', \'input_image\', \'output_image\', \'depth_radius\', \'bias\', \'alpha\', \'beta\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LeakyRelu"
+    argspec: "args=[\'features\', \'alpha\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LeakyReluGrad"
+    argspec: "args=[\'gradients\', \'features\', \'alpha\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LearnedUnigramCandidateSampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LeftShift"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Less"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LessEqual"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Lgamma"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LinSpace"
+    argspec: "args=[\'start\', \'stop\', \'num\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ListDiff"
+    argspec: "args=[\'x\', \'y\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LoadAndRemapMatrix"
+    argspec: "args=[\'ckpt_path\', \'old_tensor_name\', \'row_remapping\', \'col_remapping\', \'initializing_values\', \'num_rows\', \'num_cols\', \'max_rows_in_memory\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Log"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Log1p"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LogMatrixDeterminant"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LogSoftmax"
+    argspec: "args=[\'logits\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LogUniformCandidateSampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LogicalAnd"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LogicalNot"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LogicalOr"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableExport"
+    argspec: "args=[\'table_handle\', \'Tkeys\', \'Tvalues\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableExportV2"
+    argspec: "args=[\'table_handle\', \'Tkeys\', \'Tvalues\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableFind"
+    argspec: "args=[\'table_handle\', \'keys\', \'default_value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableFindV2"
+    argspec: "args=[\'table_handle\', \'keys\', \'default_value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableImport"
+    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableImportV2"
+    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableInsert"
+    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableInsertV2"
+    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableRemoveV2"
+    argspec: "args=[\'table_handle\', \'keys\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableSize"
+    argspec: "args=[\'table_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableSizeV2"
+    argspec: "args=[\'table_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LoopCond"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LowerBound"
+    argspec: "args=[\'sorted_inputs\', \'values\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Lu"
+    argspec: "args=[\'input\', \'output_idx_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MakeIterator"
+    argspec: "args=[\'dataset\', \'iterator\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapClear"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\', \'use_inter_op_parallelism\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapDefun"
+    argspec: "args=[\'arguments\', \'captured_inputs\', \'output_types\', \'output_shapes\', \'f\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapIncompleteSize"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapPeek"
+    argspec: "args=[\'key\', \'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapSize"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapStage"
+    argspec: "args=[\'key\', \'indices\', \'values\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapUnstage"
+    argspec: "args=[\'key\', \'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapUnstageNoKey"
+    argspec: "args=[\'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatMul"
+    argspec: "args=[\'a\', \'b\', \'transpose_a\', \'transpose_b\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatchingFiles"
+    argspec: "args=[\'pattern\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixBandPart"
+    argspec: "args=[\'input\', \'num_lower\', \'num_upper\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixDeterminant"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixDiag"
+    argspec: "args=[\'diagonal\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixDiagPart"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixExponential"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixInverse"
+    argspec: "args=[\'input\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixLogarithm"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixSetDiag"
+    argspec: "args=[\'input\', \'diagonal\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixSolve"
+    argspec: "args=[\'matrix\', \'rhs\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixSolveLs"
+    argspec: "args=[\'matrix\', \'rhs\', \'l2_regularizer\', \'fast\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixSquareRoot"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixTriangularSolve"
+    argspec: "args=[\'matrix\', \'rhs\', \'lower\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Max"
+    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPool"
+    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPool3D"
+    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPool3DGrad"
+    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPool3DGradGrad"
+    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolGrad"
+    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolGradGrad"
+    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolGradGradV2"
+    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolGradGradWithArgmax"
+    argspec: "args=[\'input\', \'grad\', \'argmax\', \'ksize\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolGradV2"
+    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolGradWithArgmax"
+    argspec: "args=[\'input\', \'grad\', \'argmax\', \'ksize\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolV2"
+    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolWithArgmax"
+    argspec: "args=[\'input\', \'ksize\', \'strides\', \'Targmax\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Maximum"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Mean"
+    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Merge"
+    argspec: "args=[\'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MergeSummary"
+    argspec: "args=[\'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MergeV2Checkpoints"
+    argspec: "args=[\'checkpoint_prefixes\', \'destination_prefix\', \'delete_old_dirs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Mfcc"
+    argspec: "args=[\'spectrogram\', \'sample_rate\', \'upper_frequency_limit\', \'lower_frequency_limit\', \'filterbank_channel_count\', \'dct_coefficient_count\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Min"
+    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Minimum"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MirrorPad"
+    argspec: "args=[\'input\', \'paddings\', \'mode\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MirrorPadGrad"
+    argspec: "args=[\'input\', \'paddings\', \'mode\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Mod"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ModelDataset"
+    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Mul"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MultiDeviceIterator"
+    argspec: "args=[\'devices\', \'shared_name\', \'container\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MultiDeviceIteratorFromStringHandle"
+    argspec: "args=[\'string_handle\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MultiDeviceIteratorGetNextFromShard"
+    argspec: "args=[\'multi_device_iterator\', \'shard_num\', \'incarnation_id\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MultiDeviceIteratorInit"
+    argspec: "args=[\'dataset\', \'multi_device_iterator\', \'max_buffer_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MultiDeviceIteratorToStringHandle"
+    argspec: "args=[\'multi_device_iterator\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Multinomial"
+    argspec: "args=[\'logits\', \'num_samples\', \'seed\', \'seed2\', \'output_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutableDenseHashTable"
+    argspec: "args=[\'empty_key\', \'container\', \'shared_name\', \'use_node_name_sharing\', \'value_dtype\', \'value_shape\', \'initial_num_buckets\', \'max_load_factor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutableDenseHashTableV2"
+    argspec: "args=[\'empty_key\', \'deleted_key\', \'container\', \'shared_name\', \'use_node_name_sharing\', \'value_dtype\', \'value_shape\', \'initial_num_buckets\', \'max_load_factor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutableHashTable"
+    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutableHashTableOfTensors"
+    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\', \'value_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutableHashTableOfTensorsV2"
+    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\', \'value_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutableHashTableV2"
+    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutexLock"
+    argspec: "args=[\'mutex\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutexV2"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NcclAllReduce"
+    argspec: "args=[\'input\', \'reduction\', \'num_devices\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NcclBroadcast"
+    argspec: "args=[\'input\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NcclReduce"
+    argspec: "args=[\'input\', \'reduction\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Neg"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NextAfter"
+    argspec: "args=[\'x1\', \'x2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NextIteration"
+    argspec: "args=[\'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NoOp"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NonMaxSuppression"
+    argspec: "args=[\'boxes\', \'scores\', \'max_output_size\', \'iou_threshold\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NonMaxSuppressionV2"
+    argspec: "args=[\'boxes\', \'scores\', \'max_output_size\', \'iou_threshold\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NonMaxSuppressionV3"
+    argspec: "args=[\'boxes\', \'scores\', \'max_output_size\', \'iou_threshold\', \'score_threshold\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NonMaxSuppressionV4"
+    argspec: "args=[\'boxes\', \'scores\', \'max_output_size\', \'iou_threshold\', \'score_threshold\', \'pad_to_max_output_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NonMaxSuppressionWithOverlaps"
+    argspec: "args=[\'overlaps\', \'scores\', \'max_output_size\', \'overlap_threshold\', \'score_threshold\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NotEqual"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NthElement"
+    argspec: "args=[\'input\', \'n\', \'reverse\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OneHot"
+    argspec: "args=[\'indices\', \'depth\', \'on_value\', \'off_value\', \'axis\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OneShotIterator"
+    argspec: "args=[\'dataset_factory\', \'output_types\', \'output_shapes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OnesLike"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OptimizeDataset"
+    argspec: "args=[\'input_dataset\', \'optimizations\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OptionalFromValue"
+    argspec: "args=[\'components\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OptionalGetValue"
+    argspec: "args=[\'optional\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OptionalHasValue"
+    argspec: "args=[\'optional\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OptionalNone"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OrderedMapClear"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OrderedMapIncompleteSize"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OrderedMapPeek"
+    argspec: "args=[\'key\', \'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OrderedMapSize"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OrderedMapStage"
+    argspec: "args=[\'key\', \'indices\', \'values\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OrderedMapUnstage"
+    argspec: "args=[\'key\', \'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OrderedMapUnstageNoKey"
+    argspec: "args=[\'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Pack"
+    argspec: "args=[\'values\', \'axis\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Pad"
+    argspec: "args=[\'input\', \'paddings\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PadV2"
+    argspec: "args=[\'input\', \'paddings\', \'constant_values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PaddedBatchDataset"
+    argspec: "args=[\'input_dataset\', \'batch_size\', \'padded_shapes\', \'padding_values\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PaddedBatchDatasetV2"
+    argspec: "args=[\'input_dataset\', \'batch_size\', \'padded_shapes\', \'padding_values\', \'drop_remainder\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PaddingFIFOQueue"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PaddingFIFOQueueV2"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParallelConcat"
+    argspec: "args=[\'values\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParallelDynamicStitch"
+    argspec: "args=[\'indices\', \'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParallelInterleaveDatasetV2"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'cycle_length\', \'block_length\', \'num_parallel_calls\', \'f\', \'output_types\', \'output_shapes\', \'sloppy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParallelMapDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'num_parallel_calls\', \'f\', \'output_types\', \'output_shapes\', \'use_inter_op_parallelism\', \'sloppy\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParameterizedTruncatedNormal"
+    argspec: "args=[\'shape\', \'means\', \'stdevs\', \'minvals\', \'maxvals\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParseExample"
+    argspec: "args=[\'serialized\', \'names\', \'sparse_keys\', \'dense_keys\', \'dense_defaults\', \'sparse_types\', \'dense_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParseSequenceExample"
+    argspec: "args=[\'serialized\', \'debug_name\', \'context_dense_defaults\', \'feature_list_dense_missing_assumed_empty\', \'context_sparse_keys\', \'context_dense_keys\', \'feature_list_sparse_keys\', \'feature_list_dense_keys\', \'Ncontext_sparse\', \'Ncontext_dense\', \'Nfeature_list_sparse\', \'Nfeature_list_dense\', \'context_sparse_types\', \'feature_list_dense_types\', \'context_dense_shapes\', \'feature_list_sparse_types\', \'feature_list_dense_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParseSingleExample"
+    argspec: "args=[\'serialized\', \'dense_defaults\', \'num_sparse\', \'sparse_keys\', \'dense_keys\', \'sparse_types\', \'dense_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParseSingleSequenceExample"
+    argspec: "args=[\'serialized\', \'feature_list_dense_missing_assumed_empty\', \'context_sparse_keys\', \'context_dense_keys\', \'feature_list_sparse_keys\', \'feature_list_dense_keys\', \'context_dense_defaults\', \'debug_name\', \'context_sparse_types\', \'feature_list_dense_types\', \'context_dense_shapes\', \'feature_list_sparse_types\', \'feature_list_dense_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParseTensor"
+    argspec: "args=[\'serialized\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PartitionedCall"
+    argspec: "args=[\'args\', \'Tout\', \'f\', \'config\', \'config_proto\', \'executor_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Placeholder"
+    argspec: "args=[\'dtype\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PlaceholderV2"
+    argspec: "args=[\'dtype\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PlaceholderWithDefault"
+    argspec: "args=[\'input\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Polygamma"
+    argspec: "args=[\'a\', \'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PopulationCount"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Pow"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PrefetchDataset"
+    argspec: "args=[\'input_dataset\', \'buffer_size\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PreventGradient"
+    argspec: "args=[\'input\', \'message\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Print"
+    argspec: "args=[\'input\', \'data\', \'message\', \'first_n\', \'summarize\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PrintV2"
+    argspec: "args=[\'input\', \'output_stream\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PriorityQueue"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PriorityQueueV2"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Prod"
+    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PyFunc"
+    argspec: "args=[\'input\', \'token\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PyFuncStateless"
+    argspec: "args=[\'input\', \'token\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Qr"
+    argspec: "args=[\'input\', \'full_matrices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizeAndDequantize"
+    argspec: "args=[\'input\', \'signed_input\', \'num_bits\', \'range_given\', \'input_min\', \'input_max\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizeAndDequantizeV2"
+    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'signed_input\', \'num_bits\', \'range_given\', \'round_mode\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizeAndDequantizeV3"
+    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'num_bits\', \'signed_input\', \'range_given\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizeDownAndShrinkRange"
+    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizeV2"
+    argspec: "args=[\'input\', \'min_range\', \'max_range\', \'T\', \'mode\', \'round_mode\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedAdd"
+    argspec: "args=[\'x\', \'y\', \'min_x\', \'max_x\', \'min_y\', \'max_y\', \'Toutput\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedAvgPool"
+    argspec: "args=[\'input\', \'min_input\', \'max_input\', \'ksize\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedBatchNormWithGlobalNormalization"
+    argspec: "args=[\'t\', \'t_min\', \'t_max\', \'m\', \'m_min\', \'m_max\', \'v\', \'v_min\', \'v_max\', \'beta\', \'beta_min\', \'beta_max\', \'gamma\', \'gamma_min\', \'gamma_max\', \'out_type\', \'variance_epsilon\', \'scale_after_normalization\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedBiasAdd"
+    argspec: "args=[\'input\', \'bias\', \'min_input\', \'max_input\', \'min_bias\', \'max_bias\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConcat"
+    argspec: "args=[\'concat_dim\', \'values\', \'input_mins\', \'input_maxes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2D"
+    argspec: "args=[\'input\', \'filter\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DAndRelu"
+    argspec: "args=[\'input\', \'filter\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DAndReluAndRequantize"
+    argspec: "args=[\'input\', \'filter\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DAndRequantize"
+    argspec: "args=[\'input\', \'filter\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DWithBias"
+    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DWithBiasAndRelu"
+    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DWithBiasAndReluAndRequantize"
+    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DWithBiasAndRequantize"
+    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DWithBiasSignedSumAndReluAndRequantize"
+    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'summand\', \'min_summand\', \'max_summand\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DWithBiasSumAndRelu"
+    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'summand\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DWithBiasSumAndReluAndRequantize"
+    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'summand\', \'min_summand\', \'max_summand\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedInstanceNorm"
+    argspec: "args=[\'x\', \'x_min\', \'x_max\', \'output_range_given\', \'given_y_min\', \'given_y_max\', \'variance_epsilon\', \'min_separation\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedMatMul"
+    argspec: "args=[\'a\', \'b\', \'min_a\', \'max_a\', \'min_b\', \'max_b\', \'Toutput\', \'transpose_a\', \'transpose_b\', \'Tactivation\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedMaxPool"
+    argspec: "args=[\'input\', \'min_input\', \'max_input\', \'ksize\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedMul"
+    argspec: "args=[\'x\', \'y\', \'min_x\', \'max_x\', \'min_y\', \'max_y\', \'Toutput\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedRelu"
+    argspec: "args=[\'features\', \'min_features\', \'max_features\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedRelu6"
+    argspec: "args=[\'features\', \'min_features\', \'max_features\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedReluX"
+    argspec: "args=[\'features\', \'max_value\', \'min_features\', \'max_features\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedReshape"
+    argspec: "args=[\'tensor\', \'shape\', \'input_min\', \'input_max\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedResizeBilinear"
+    argspec: "args=[\'images\', \'size\', \'min\', \'max\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueClose"
+    argspec: "args=[\'handle\', \'cancel_pending_enqueues\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueCloseV2"
+    argspec: "args=[\'handle\', \'cancel_pending_enqueues\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueDequeue"
+    argspec: "args=[\'handle\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueDequeueMany"
+    argspec: "args=[\'handle\', \'n\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueDequeueManyV2"
+    argspec: "args=[\'handle\', \'n\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueDequeueUpTo"
+    argspec: "args=[\'handle\', \'n\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueDequeueUpToV2"
+    argspec: "args=[\'handle\', \'n\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueDequeueV2"
+    argspec: "args=[\'handle\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueEnqueue"
+    argspec: "args=[\'handle\', \'components\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueEnqueueMany"
+    argspec: "args=[\'handle\', \'components\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueEnqueueManyV2"
+    argspec: "args=[\'handle\', \'components\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueEnqueueV2"
+    argspec: "args=[\'handle\', \'components\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueIsClosed"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueIsClosedV2"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueSize"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueSizeV2"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RFFT"
+    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RFFT2D"
+    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RFFT3D"
+    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RGBToHSV"
+    argspec: "args=[\'images\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RaggedGather"
+    argspec: "args=[\'params_nested_splits\', \'params_dense_values\', \'indices\', \'OUTPUT_RAGGED_RANK\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RaggedRange"
+    argspec: "args=[\'starts\', \'limits\', \'deltas\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RaggedTensorToSparse"
+    argspec: "args=[\'rt_nested_splits\', \'rt_dense_values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomCrop"
+    argspec: "args=[\'image\', \'size\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomGamma"
+    argspec: "args=[\'shape\', \'alpha\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomGammaGrad"
+    argspec: "args=[\'alpha\', \'sample\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomPoisson"
+    argspec: "args=[\'shape\', \'rate\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomPoissonV2"
+    argspec: "args=[\'shape\', \'rate\', \'seed\', \'seed2\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomShuffle"
+    argspec: "args=[\'value\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomShuffleQueue"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'min_after_dequeue\', \'seed\', \'seed2\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomShuffleQueueV2"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'min_after_dequeue\', \'seed\', \'seed2\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomStandardNormal"
+    argspec: "args=[\'shape\', \'seed\', \'seed2\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomUniform"
+    argspec: "args=[\'shape\', \'seed\', \'seed2\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomUniformInt"
+    argspec: "args=[\'shape\', \'minval\', \'maxval\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Range"
+    argspec: "args=[\'start\', \'limit\', \'delta\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RangeDataset"
+    argspec: "args=[\'start\', \'stop\', \'step\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Rank"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReadFile"
+    argspec: "args=[\'filename\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReadVariableOp"
+    argspec: "args=[\'resource\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderNumRecordsProduced"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderNumRecordsProducedV2"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderNumWorkUnitsCompleted"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderNumWorkUnitsCompletedV2"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderRead"
+    argspec: "args=[\'reader_handle\', \'queue_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderReadUpTo"
+    argspec: "args=[\'reader_handle\', \'queue_handle\', \'num_records\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderReadUpToV2"
+    argspec: "args=[\'reader_handle\', \'queue_handle\', \'num_records\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderReadV2"
+    argspec: "args=[\'reader_handle\', \'queue_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderReset"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderResetV2"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderRestoreState"
+    argspec: "args=[\'reader_handle\', \'state\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderRestoreStateV2"
+    argspec: "args=[\'reader_handle\', \'state\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderSerializeState"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderSerializeStateV2"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Real"
+    argspec: "args=[\'input\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RealDiv"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Reciprocal"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReciprocalGrad"
+    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RecordInput"
+    argspec: "args=[\'file_pattern\', \'file_random_seed\', \'file_shuffle_shift_ratio\', \'file_buffer_size\', \'file_parallelism\', \'batch_size\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReduceDataset"
+    argspec: "args=[\'input_dataset\', \'initial_state\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\', \'use_inter_op_parallelism\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReduceJoin"
+    argspec: "args=[\'inputs\', \'reduction_indices\', \'keep_dims\', \'separator\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RefEnter"
+    argspec: "args=[\'data\', \'frame_name\', \'is_constant\', \'parallel_iterations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RefExit"
+    argspec: "args=[\'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RefIdentity"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RefMerge"
+    argspec: "args=[\'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RefNextIteration"
+    argspec: "args=[\'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RefSelect"
+    argspec: "args=[\'index\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RefSwitch"
+    argspec: "args=[\'data\', \'pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RegexFullMatch"
+    argspec: "args=[\'input\', \'pattern\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RegexReplace"
+    argspec: "args=[\'input\', \'pattern\', \'rewrite\', \'replace_global\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Relu"
+    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Relu6"
+    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Relu6Grad"
+    argspec: "args=[\'gradients\', \'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReluGrad"
+    argspec: "args=[\'gradients\', \'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RemoteCall"
+    argspec: "args=[\'target\', \'args\', \'Tout\', \'f\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RepeatDataset"
+    argspec: "args=[\'input_dataset\', \'count\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RequantizationRange"
+    argspec: "args=[\'input\', \'input_min\', \'input_max\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Requantize"
+    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'requested_output_min\', \'requested_output_max\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Reshape"
+    argspec: "args=[\'tensor\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResizeArea"
+    argspec: "args=[\'images\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResizeBicubic"
+    argspec: "args=[\'images\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResizeBicubicGrad"
+    argspec: "args=[\'grads\', \'original_image\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResizeBilinear"
+    argspec: "args=[\'images\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResizeBilinearGrad"
+    argspec: "args=[\'grads\', \'original_image\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResizeNearestNeighbor"
+    argspec: "args=[\'images\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResizeNearestNeighborGrad"
+    argspec: "args=[\'grads\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyAdaMax"
+    argspec: "args=[\'var\', \'m\', \'v\', \'beta1_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyAdadelta"
+    argspec: "args=[\'var\', \'accum\', \'accum_update\', \'lr\', \'rho\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'use_locking\', \'update_slots\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyAdagradDA"
+    argspec: "args=[\'var\', \'gradient_accumulator\', \'gradient_squared_accumulator\', \'grad\', \'lr\', \'l1\', \'l2\', \'global_step\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyAdam"
+    argspec: "args=[\'var\', \'m\', \'v\', \'beta1_power\', \'beta2_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyAdamWithAmsgrad"
+    argspec: "args=[\'var\', \'m\', \'v\', \'vhat\', \'beta1_power\', \'beta2_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyAddSign"
+    argspec: "args=[\'var\', \'m\', \'lr\', \'alpha\', \'sign_decay\', \'beta\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyCenteredRMSProp"
+    argspec: "args=[\'var\', \'mg\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyFtrl"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'lr\', \'l1\', \'l2\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyFtrlV2"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'lr\', \'l1\', \'l2\', \'l2_shrinkage\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyGradientDescent"
+    argspec: "args=[\'var\', \'alpha\', \'delta\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyKerasMomentum"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyMomentum"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyPowerSign"
+    argspec: "args=[\'var\', \'m\', \'lr\', \'logbase\', \'sign_decay\', \'beta\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyProximalAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'l1\', \'l2\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyProximalGradientDescent"
+    argspec: "args=[\'var\', \'alpha\', \'l1\', \'l2\', \'delta\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyRMSProp"
+    argspec: "args=[\'var\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceCountUpTo"
+    argspec: "args=[\'resource\', \'limit\', \'T\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceGather"
+    argspec: "args=[\'resource\', \'indices\', \'validate_indices\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterAdd"
+    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterDiv"
+    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterMax"
+    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterMin"
+    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterMul"
+    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterNdAdd"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterNdSub"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterNdUpdate"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterSub"
+    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterUpdate"
+    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyAdadelta"
+    argspec: "args=[\'var\', \'accum\', \'accum_update\', \'lr\', \'rho\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'use_locking\', \'update_slots\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyAdagradDA"
+    argspec: "args=[\'var\', \'gradient_accumulator\', \'gradient_squared_accumulator\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'global_step\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyCenteredRMSProp"
+    argspec: "args=[\'var\', \'mg\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyFtrl"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyFtrlV2"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'l2_shrinkage\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyKerasMomentum"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyMomentum"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyProximalAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'l1\', \'l2\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyProximalGradientDescent"
+    argspec: "args=[\'var\', \'alpha\', \'l1\', \'l2\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyRMSProp"
+    argspec: "args=[\'var\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceStridedSliceAssign"
+    argspec: "args=[\'ref\', \'begin\', \'end\', \'strides\', \'value\', \'begin_mask\', \'end_mask\', \'ellipsis_mask\', \'new_axis_mask\', \'shrink_axis_mask\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Restore"
+    argspec: "args=[\'file_pattern\', \'tensor_name\', \'dt\', \'preferred_shard\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RestoreSlice"
+    argspec: "args=[\'file_pattern\', \'tensor_name\', \'shape_and_slice\', \'dt\', \'preferred_shard\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RestoreV2"
+    argspec: "args=[\'prefix\', \'tensor_names\', \'shape_and_slices\', \'dtypes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Reverse"
+    argspec: "args=[\'tensor\', \'dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReverseSequence"
+    argspec: "args=[\'input\', \'seq_lengths\', \'seq_dim\', \'batch_dim\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReverseV2"
+    argspec: "args=[\'tensor\', \'axis\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RightShift"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Rint"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Roll"
+    argspec: "args=[\'input\', \'shift\', \'axis\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Round"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Rsqrt"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RsqrtGrad"
+    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SampleDistortedBoundingBox"
+    argspec: "args=[\'image_size\', \'bounding_boxes\', \'seed\', \'seed2\', \'min_object_covered\', \'aspect_ratio_range\', \'area_range\', \'max_attempts\', \'use_image_if_no_bounding_boxes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SampleDistortedBoundingBoxV2"
+    argspec: "args=[\'image_size\', \'bounding_boxes\', \'min_object_covered\', \'seed\', \'seed2\', \'aspect_ratio_range\', \'area_range\', \'max_attempts\', \'use_image_if_no_bounding_boxes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Save"
+    argspec: "args=[\'filename\', \'tensor_names\', \'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SaveSlices"
+    argspec: "args=[\'filename\', \'tensor_names\', \'shapes_and_slices\', \'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SaveV2"
+    argspec: "args=[\'prefix\', \'tensor_names\', \'shape_and_slices\', \'tensors\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScalarSummary"
+    argspec: "args=[\'tags\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScaleAndTranslate"
+    argspec: "args=[\'images\', \'size\', \'scale\', \'translation\', \'kernel_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScaleAndTranslateGrad"
+    argspec: "args=[\'grads\', \'original_image\', \'scale\', \'translation\', \'kernel_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterAdd"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterDiv"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterMax"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterMin"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterMul"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterNd"
+    argspec: "args=[\'indices\', \'updates\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterNdAdd"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterNdNonAliasingAdd"
+    argspec: "args=[\'input\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterNdSub"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterNdUpdate"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterSub"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterUpdate"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SdcaFprint"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SdcaOptimizer"
+    argspec: "args=[\'sparse_example_indices\', \'sparse_feature_indices\', \'sparse_feature_values\', \'dense_features\', \'example_weights\', \'example_labels\', \'sparse_indices\', \'sparse_weights\', \'dense_weights\', \'example_state_data\', \'loss_type\', \'adaptative\', \'l1\', \'l2\', \'num_loss_partitions\', \'num_inner_iterations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SdcaOptimizerV2"
+    argspec: "args=[\'sparse_example_indices\', \'sparse_feature_indices\', \'sparse_feature_values\', \'dense_features\', \'example_weights\', \'example_labels\', \'sparse_indices\', \'sparse_weights\', \'dense_weights\', \'example_state_data\', \'loss_type\', \'adaptive\', \'l1\', \'l2\', \'num_loss_partitions\', \'num_inner_iterations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SdcaShrinkL1"
+    argspec: "args=[\'weights\', \'l1\', \'l2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SegmentMax"
+    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SegmentMean"
+    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SegmentMin"
+    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SegmentProd"
+    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SegmentSum"
+    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Select"
+    argspec: "args=[\'condition\', \'t\', \'e\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SelfAdjointEig"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SelfAdjointEigV2"
+    argspec: "args=[\'input\', \'compute_v\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Selu"
+    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SeluGrad"
+    argspec: "args=[\'gradients\', \'outputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SerializeIterator"
+    argspec: "args=[\'resource_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SerializeManySparse"
+    argspec: "args=[\'sparse_indices\', \'sparse_values\', \'sparse_shape\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SerializeSparse"
+    argspec: "args=[\'sparse_indices\', \'sparse_values\', \'sparse_shape\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SerializeTensor"
+    argspec: "args=[\'tensor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SetSize"
+    argspec: "args=[\'set_indices\', \'set_values\', \'set_shape\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Shape"
+    argspec: "args=[\'input\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ShapeN"
+    argspec: "args=[\'input\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ShardedFilename"
+    argspec: "args=[\'basename\', \'shard\', \'num_shards\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ShardedFilespec"
+    argspec: "args=[\'basename\', \'num_shards\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ShuffleAndRepeatDataset"
+    argspec: "args=[\'input_dataset\', \'buffer_size\', \'seed\', \'seed2\', \'count\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ShuffleDataset"
+    argspec: "args=[\'input_dataset\', \'buffer_size\', \'seed\', \'seed2\', \'reshuffle_each_iteration\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Sigmoid"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SigmoidGrad"
+    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Sign"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Sin"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Sinh"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Size"
+    argspec: "args=[\'input\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SkipDataset"
+    argspec: "args=[\'input_dataset\', \'count\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Slice"
+    argspec: "args=[\'input\', \'begin\', \'size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Snapshot"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Softmax"
+    argspec: "args=[\'logits\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SoftmaxCrossEntropyWithLogits"
+    argspec: "args=[\'features\', \'labels\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Softplus"
+    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SoftplusGrad"
+    argspec: "args=[\'gradients\', \'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Softsign"
+    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SoftsignGrad"
+    argspec: "args=[\'gradients\', \'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SpaceToBatch"
+    argspec: "args=[\'input\', \'paddings\', \'block_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SpaceToBatchND"
+    argspec: "args=[\'input\', \'block_shape\', \'paddings\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SpaceToDepth"
+    argspec: "args=[\'input\', \'block_size\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseAccumulatorApplyGradient"
+    argspec: "args=[\'handle\', \'local_step\', \'gradient_indices\', \'gradient_values\', \'gradient_shape\', \'has_known_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseAccumulatorTakeGradient"
+    argspec: "args=[\'handle\', \'num_required\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseAdd"
+    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b_indices\', \'b_values\', \'b_shape\', \'thresh\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseAddGrad"
+    argspec: "args=[\'backprop_val_grad\', \'a_indices\', \'b_indices\', \'sum_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyAdadelta"
+    argspec: "args=[\'var\', \'accum\', \'accum_update\', \'lr\', \'rho\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'use_locking\', \'update_slots\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyAdagradDA"
+    argspec: "args=[\'var\', \'gradient_accumulator\', \'gradient_squared_accumulator\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'global_step\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyCenteredRMSProp"
+    argspec: "args=[\'var\', \'mg\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyFtrl"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyFtrlV2"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'l2_shrinkage\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyMomentum"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyProximalAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'l1\', \'l2\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyProximalGradientDescent"
+    argspec: "args=[\'var\', \'alpha\', \'l1\', \'l2\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyRMSProp"
+    argspec: "args=[\'var\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseConcat"
+    argspec: "args=[\'indices\', \'values\', \'shapes\', \'concat_dim\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseConditionalAccumulator"
+    argspec: "args=[\'dtype\', \'shape\', \'container\', \'shared_name\', \'reduction_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseCross"
+    argspec: "args=[\'indices\', \'values\', \'shapes\', \'dense_inputs\', \'hashed_output\', \'num_buckets\', \'hash_key\', \'out_type\', \'internal_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseDenseCwiseAdd"
+    argspec: "args=[\'sp_indices\', \'sp_values\', \'sp_shape\', \'dense\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseDenseCwiseDiv"
+    argspec: "args=[\'sp_indices\', \'sp_values\', \'sp_shape\', \'dense\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseDenseCwiseMul"
+    argspec: "args=[\'sp_indices\', \'sp_values\', \'sp_shape\', \'dense\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseFillEmptyRows"
+    argspec: "args=[\'indices\', \'values\', \'dense_shape\', \'default_value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseFillEmptyRowsGrad"
+    argspec: "args=[\'reverse_index_map\', \'grad_values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseMatMul"
+    argspec: "args=[\'a\', \'b\', \'transpose_a\', \'transpose_b\', \'a_is_sparse\', \'b_is_sparse\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseReduceMax"
+    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseReduceMaxSparse"
+    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseReduceSum"
+    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseReduceSumSparse"
+    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseReorder"
+    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseReshape"
+    argspec: "args=[\'input_indices\', \'input_shape\', \'new_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentMean"
+    argspec: "args=[\'data\', \'indices\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentMeanGrad"
+    argspec: "args=[\'grad\', \'indices\', \'segment_ids\', \'output_dim0\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentMeanWithNumSegments"
+    argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentSqrtN"
+    argspec: "args=[\'data\', \'indices\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentSqrtNGrad"
+    argspec: "args=[\'grad\', \'indices\', \'segment_ids\', \'output_dim0\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentSqrtNWithNumSegments"
+    argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentSum"
+    argspec: "args=[\'data\', \'indices\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentSumWithNumSegments"
+    argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSlice"
+    argspec: "args=[\'indices\', \'values\', \'shape\', \'start\', \'size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSliceGrad"
+    argspec: "args=[\'backprop_val_grad\', \'input_indices\', \'input_start\', \'output_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSoftmax"
+    argspec: "args=[\'sp_indices\', \'sp_values\', \'sp_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSoftmaxCrossEntropyWithLogits"
+    argspec: "args=[\'features\', \'labels\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSparseMaximum"
+    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b_indices\', \'b_values\', \'b_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSparseMinimum"
+    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b_indices\', \'b_values\', \'b_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSplit"
+    argspec: "args=[\'split_dim\', \'indices\', \'values\', \'shape\', \'num_split\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseTensorDenseAdd"
+    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseTensorDenseMatMul"
+    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b\', \'adjoint_a\', \'adjoint_b\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseTensorSliceDataset"
+    argspec: "args=[\'indices\', \'values\', \'dense_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseToDense"
+    argspec: "args=[\'sparse_indices\', \'output_shape\', \'sparse_values\', \'default_value\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseToSparseSetOperation"
+    argspec: "args=[\'set1_indices\', \'set1_values\', \'set1_shape\', \'set2_indices\', \'set2_values\', \'set2_shape\', \'set_operation\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Split"
+    argspec: "args=[\'split_dim\', \'value\', \'num_split\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SplitV"
+    argspec: "args=[\'value\', \'size_splits\', \'split_dim\', \'num_split\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Sqrt"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SqrtGrad"
+    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Square"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SquaredDifference"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Squeeze"
+    argspec: "args=[\'input\', \'squeeze_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Stack"
+    argspec: "args=[\'elem_type\', \'stack_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StackClose"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StackCloseV2"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StackPop"
+    argspec: "args=[\'handle\', \'elem_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StackPopV2"
+    argspec: "args=[\'handle\', \'elem_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StackPush"
+    argspec: "args=[\'handle\', \'elem\', \'swap_memory\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StackPushV2"
+    argspec: "args=[\'handle\', \'elem\', \'swap_memory\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StackV2"
+    argspec: "args=[\'max_size\', \'elem_type\', \'stack_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Stage"
+    argspec: "args=[\'values\', \'capacity\', \'memory_limit\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StageClear"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StagePeek"
+    argspec: "args=[\'index\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StageSize"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatefulPartitionedCall"
+    argspec: "args=[\'args\', \'Tout\', \'f\', \'config\', \'config_proto\', \'executor_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatelessIf"
+    argspec: "args=[\'cond\', \'input\', \'Tout\', \'then_branch\', \'else_branch\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatelessMultinomial"
+    argspec: "args=[\'logits\', \'num_samples\', \'seed\', \'output_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatelessRandomNormal"
+    argspec: "args=[\'shape\', \'seed\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatelessRandomUniform"
+    argspec: "args=[\'shape\', \'seed\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatelessRandomUniformInt"
+    argspec: "args=[\'shape\', \'seed\', \'minval\', \'maxval\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatelessTruncatedNormal"
+    argspec: "args=[\'shape\', \'seed\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatelessWhile"
+    argspec: "args=[\'input\', \'cond\', \'body\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StaticRegexFullMatch"
+    argspec: "args=[\'input\', \'pattern\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StaticRegexReplace"
+    argspec: "args=[\'input\', \'pattern\', \'rewrite\', \'replace_global\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StopGradient"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StridedSlice"
+    argspec: "args=[\'input\', \'begin\', \'end\', \'strides\', \'begin_mask\', \'end_mask\', \'ellipsis_mask\', \'new_axis_mask\', \'shrink_axis_mask\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StridedSliceAssign"
+    argspec: "args=[\'ref\', \'begin\', \'end\', \'strides\', \'value\', \'begin_mask\', \'end_mask\', \'ellipsis_mask\', \'new_axis_mask\', \'shrink_axis_mask\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StridedSliceGrad"
+    argspec: "args=[\'shape\', \'begin\', \'end\', \'strides\', \'dy\', \'begin_mask\', \'end_mask\', \'ellipsis_mask\', \'new_axis_mask\', \'shrink_axis_mask\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringFormat"
+    argspec: "args=[\'inputs\', \'template\', \'placeholder\', \'summarize\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringJoin"
+    argspec: "args=[\'inputs\', \'separator\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringLength"
+    argspec: "args=[\'input\', \'unit\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringSplit"
+    argspec: "args=[\'input\', \'delimiter\', \'skip_empty\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringSplitV2"
+    argspec: "args=[\'input\', \'sep\', \'maxsplit\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringStrip"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringToHashBucket"
+    argspec: "args=[\'string_tensor\', \'num_buckets\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringToHashBucketFast"
+    argspec: "args=[\'input\', \'num_buckets\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringToHashBucketStrong"
+    argspec: "args=[\'input\', \'num_buckets\', \'key\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringToNumber"
+    argspec: "args=[\'string_tensor\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Sub"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Substr"
+    argspec: "args=[\'input\', \'pos\', \'len\', \'unit\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Sum"
+    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SummaryWriter"
+    argspec: "args=[\'shared_name\', \'container\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Svd"
+    argspec: "args=[\'input\', \'compute_uv\', \'full_matrices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Switch"
+    argspec: "args=[\'data\', \'pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SymbolicGradient"
+    argspec: "args=[\'input\', \'Tout\', \'f\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TFRecordDataset"
+    argspec: "args=[\'filenames\', \'compression_type\', \'buffer_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TFRecordReader"
+    argspec: "args=[\'container\', \'shared_name\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TFRecordReaderV2"
+    argspec: "args=[\'container\', \'shared_name\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TakeDataset"
+    argspec: "args=[\'input_dataset\', \'count\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TakeManySparseFromTensorsMap"
+    argspec: "args=[\'sparse_handles\', \'dtype\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Tan"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Tanh"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TanhGrad"
+    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TemporaryVariable"
+    argspec: "args=[\'shape\', \'dtype\', \'var_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArray"
+    argspec: "args=[\'size\', \'dtype\', \'dynamic_size\', \'clear_after_read\', \'tensor_array_name\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayClose"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayCloseV2"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayCloseV3"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayConcat"
+    argspec: "args=[\'handle\', \'flow_in\', \'dtype\', \'element_shape_except0\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayConcatV2"
+    argspec: "args=[\'handle\', \'flow_in\', \'dtype\', \'element_shape_except0\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayConcatV3"
+    argspec: "args=[\'handle\', \'flow_in\', \'dtype\', \'element_shape_except0\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayGather"
+    argspec: "args=[\'handle\', \'indices\', \'flow_in\', \'dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayGatherV2"
+    argspec: "args=[\'handle\', \'indices\', \'flow_in\', \'dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayGatherV3"
+    argspec: "args=[\'handle\', \'indices\', \'flow_in\', \'dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayGrad"
+    argspec: "args=[\'handle\', \'flow_in\', \'source\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayGradV2"
+    argspec: "args=[\'handle\', \'flow_in\', \'source\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayGradV3"
+    argspec: "args=[\'handle\', \'flow_in\', \'source\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayGradWithShape"
+    argspec: "args=[\'handle\', \'flow_in\', \'shape_to_prepend\', \'source\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayPack"
+    argspec: "args=[\'handle\', \'flow_in\', \'dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayRead"
+    argspec: "args=[\'handle\', \'index\', \'flow_in\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayReadV2"
+    argspec: "args=[\'handle\', \'index\', \'flow_in\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayReadV3"
+    argspec: "args=[\'handle\', \'index\', \'flow_in\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayScatter"
+    argspec: "args=[\'handle\', \'indices\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayScatterV2"
+    argspec: "args=[\'handle\', \'indices\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayScatterV3"
+    argspec: "args=[\'handle\', \'indices\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArraySize"
+    argspec: "args=[\'handle\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArraySizeV2"
+    argspec: "args=[\'handle\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArraySizeV3"
+    argspec: "args=[\'handle\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArraySplit"
+    argspec: "args=[\'handle\', \'value\', \'lengths\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArraySplitV2"
+    argspec: "args=[\'handle\', \'value\', \'lengths\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArraySplitV3"
+    argspec: "args=[\'handle\', \'value\', \'lengths\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayUnpack"
+    argspec: "args=[\'handle\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayV2"
+    argspec: "args=[\'size\', \'dtype\', \'element_shape\', \'dynamic_size\', \'clear_after_read\', \'tensor_array_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayV3"
+    argspec: "args=[\'size\', \'dtype\', \'element_shape\', \'dynamic_size\', \'clear_after_read\', \'identical_element_shapes\', \'tensor_array_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayWrite"
+    argspec: "args=[\'handle\', \'index\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayWriteV2"
+    argspec: "args=[\'handle\', \'index\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayWriteV3"
+    argspec: "args=[\'handle\', \'index\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorDataset"
+    argspec: "args=[\'components\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListConcat"
+    argspec: "args=[\'input_handle\', \'element_dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListConcatLists"
+    argspec: "args=[\'input_a\', \'input_b\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListElementShape"
+    argspec: "args=[\'input_handle\', \'shape_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListFromTensor"
+    argspec: "args=[\'tensor\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListGather"
+    argspec: "args=[\'input_handle\', \'indices\', \'element_shape\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListGetItem"
+    argspec: "args=[\'input_handle\', \'index\', \'element_shape\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListLength"
+    argspec: "args=[\'input_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListPopBack"
+    argspec: "args=[\'input_handle\', \'element_shape\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListPushBack"
+    argspec: "args=[\'input_handle\', \'tensor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListPushBackBatch"
+    argspec: "args=[\'input_handles\', \'tensor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListReserve"
+    argspec: "args=[\'element_shape\', \'num_elements\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListResize"
+    argspec: "args=[\'input_handle\', \'size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListScatter"
+    argspec: "args=[\'tensor\', \'indices\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListScatterV2"
+    argspec: "args=[\'tensor\', \'indices\', \'element_shape\', \'num_elements\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListSetItem"
+    argspec: "args=[\'input_handle\', \'index\', \'item\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListSplit"
+    argspec: "args=[\'tensor\', \'element_shape\', \'lengths\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListStack"
+    argspec: "args=[\'input_handle\', \'element_shape\', \'element_dtype\', \'num_elements\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorScatterAdd"
+    argspec: "args=[\'tensor\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorScatterSub"
+    argspec: "args=[\'tensor\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorScatterUpdate"
+    argspec: "args=[\'tensor\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorSliceDataset"
+    argspec: "args=[\'components\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorSummary"
+    argspec: "args=[\'tensor\', \'description\', \'labels\', \'display_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorSummaryV2"
+    argspec: "args=[\'tag\', \'tensor\', \'serialized_summary_metadata\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TextLineDataset"
+    argspec: "args=[\'filenames\', \'compression_type\', \'buffer_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TextLineReader"
+    argspec: "args=[\'skip_header_lines\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TextLineReaderV2"
+    argspec: "args=[\'skip_header_lines\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ThreadUnsafeUnigramCandidateSampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Tile"
+    argspec: "args=[\'input\', \'multiples\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TileGrad"
+    argspec: "args=[\'input\', \'multiples\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Timestamp"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TopK"
+    argspec: "args=[\'input\', \'k\', \'sorted\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TopKV2"
+    argspec: "args=[\'input\', \'k\', \'sorted\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Transpose"
+    argspec: "args=[\'x\', \'perm\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TruncateDiv"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TruncateMod"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TruncatedNormal"
+    argspec: "args=[\'shape\', \'seed\', \'seed2\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnicodeDecode"
+    argspec: "args=[\'input\', \'input_encoding\', \'errors\', \'replacement_char\', \'replace_control_characters\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnicodeDecodeWithOffsets"
+    argspec: "args=[\'input\', \'input_encoding\', \'errors\', \'replacement_char\', \'replace_control_characters\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnicodeEncode"
+    argspec: "args=[\'input_values\', \'input_splits\', \'errors\', \'output_encoding\', \'replacement_char\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnicodeScript"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnicodeTranscode"
+    argspec: "args=[\'input\', \'input_encoding\', \'output_encoding\', \'errors\', \'replacement_char\', \'replace_control_characters\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UniformCandidateSampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Unique"
+    argspec: "args=[\'x\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UniqueV2"
+    argspec: "args=[\'x\', \'axis\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UniqueWithCounts"
+    argspec: "args=[\'x\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UniqueWithCountsV2"
+    argspec: "args=[\'x\', \'axis\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Unpack"
+    argspec: "args=[\'value\', \'num\', \'axis\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnravelIndex"
+    argspec: "args=[\'indices\', \'dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnsortedSegmentMax"
+    argspec: "args=[\'data\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnsortedSegmentMin"
+    argspec: "args=[\'data\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnsortedSegmentProd"
+    argspec: "args=[\'data\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnsortedSegmentSum"
+    argspec: "args=[\'data\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Unstage"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnwrapDatasetVariant"
+    argspec: "args=[\'input_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UpperBound"
+    argspec: "args=[\'sorted_inputs\', \'values\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "VarHandleOp"
+    argspec: "args=[\'container\', \'shared_name\', \'dtype\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "VarIsInitializedOp"
+    argspec: "args=[\'resource\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Variable"
+    argspec: "args=[\'shape\', \'dtype\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "VariableShape"
+    argspec: "args=[\'input\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "VariableV2"
+    argspec: "args=[\'shape\', \'dtype\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Where"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "While"
+    argspec: "args=[\'input\', \'cond\', \'body\', \'output_shapes\', \'parallel_iterations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WholeFileReader"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WholeFileReaderV2"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WindowDataset"
+    argspec: "args=[\'input_dataset\', \'size\', \'shift\', \'stride\', \'drop_remainder\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WrapDatasetVariant"
+    argspec: "args=[\'input_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WriteAudioSummary"
+    argspec: "args=[\'writer\', \'step\', \'tag\', \'tensor\', \'sample_rate\', \'max_outputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WriteFile"
+    argspec: "args=[\'filename\', \'contents\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WriteGraphSummary"
+    argspec: "args=[\'writer\', \'step\', \'tensor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WriteHistogramSummary"
+    argspec: "args=[\'writer\', \'step\', \'tag\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WriteImageSummary"
+    argspec: "args=[\'writer\', \'step\', \'tag\', \'tensor\', \'bad_color\', \'max_images\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WriteScalarSummary"
+    argspec: "args=[\'writer\', \'step\', \'tag\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WriteSummary"
+    argspec: "args=[\'writer\', \'step\', \'tensor\', \'tag\', \'summary_metadata\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Xdivy"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Xlogy"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ZerosLike"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Zeta"
+    argspec: "args=[\'x\', \'q\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ZipDataset"
+    argspec: "args=[\'input_datasets\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+}
-- 
GitLab


From 0469766d0d085ed9dd8ad3304cf5c95cbbb05509 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Wed, 30 Jan 2019 12:44:34 -0800
Subject: [PATCH 1609/2345] Add compat.v2 module.

PiperOrigin-RevId: 231648934
---
 tensorflow/BUILD                              | 34 +++++++++++----
 tensorflow/compat_template.__init__.py        | 43 +++++++++++++++++++
 tensorflow/opensource_only.files              |  1 +
 .../tools/api/tests/api_compatibility_test.py |  4 +-
 .../tools/compatibility/tf_upgrade_v2_test.py |  5 ++-
 5 files changed, 76 insertions(+), 11 deletions(-)
 create mode 100644 tensorflow/compat_template.__init__.py

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 413806fac1..5a8b97ede9 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -40,12 +40,16 @@ load(
 
 # @unused
 TENSORFLOW_API_INIT_FILES_V2 = (
-    TENSORFLOW_API_INIT_FILES + get_compat_files(TENSORFLOW_API_INIT_FILES_V1, 1)
+    TENSORFLOW_API_INIT_FILES +
+    get_compat_files(TENSORFLOW_API_INIT_FILES, 2) +
+    get_compat_files(TENSORFLOW_API_INIT_FILES_V1, 1)
 )
 
 # @unused
-TENSORFLOW_API_INIT_FILES_V1_WITH_COMPAT = (
-    TENSORFLOW_API_INIT_FILES_V1 + get_compat_files(TENSORFLOW_API_INIT_FILES_V1, 1)
+TENSORFLOW_API_INIT_FILES_V1 = (
+    TENSORFLOW_API_INIT_FILES_V1 +
+    get_compat_files(TENSORFLOW_API_INIT_FILES, 2) +
+    get_compat_files(TENSORFLOW_API_INIT_FILES_V1, 1)
 )
 
 # Config setting used when building for products
@@ -607,13 +611,20 @@ gen_api_init_files(
     name = "tf_python_api_gen_v1",
     srcs = [
         "api_template_v1.__init__.py",
+        "compat_template.__init__.py",
         "compat_template_v1.__init__.py",
     ],
     api_version = 1,
-    compat_api_versions = [1],
-    compat_init_templates = ["compat_template_v1.__init__.py"],
+    compat_api_versions = [
+        1,
+        2,
+    ],
+    compat_init_templates = [
+        "compat_template_v1.__init__.py",
+        "compat_template.__init__.py",
+    ],
     output_dir = "_api/v1/",
-    output_files = TENSORFLOW_API_INIT_FILES_V1_WITH_COMPAT,
+    output_files = TENSORFLOW_API_INIT_FILES_V1,
     output_package = "tensorflow._api.v1",
     root_file_name = "v1.py",
     root_init_template = "api_template_v1.__init__.py",
@@ -623,11 +634,18 @@ gen_api_init_files(
     name = "tf_python_api_gen_v2",
     srcs = [
         "api_template.__init__.py",
+        "compat_template.__init__.py",
         "compat_template_v1.__init__.py",
     ],
     api_version = 2,
-    compat_api_versions = [1],
-    compat_init_templates = ["compat_template_v1.__init__.py"],
+    compat_api_versions = [
+        1,
+        2,
+    ],
+    compat_init_templates = [
+        "compat_template_v1.__init__.py",
+        "compat_template.__init__.py",
+    ],
     output_dir = "_api/v2/",
     output_files = TENSORFLOW_API_INIT_FILES_V2,
     output_package = "tensorflow._api.v2",
diff --git a/tensorflow/compat_template.__init__.py b/tensorflow/compat_template.__init__.py
new file mode 100644
index 0000000000..1833b6a65e
--- /dev/null
+++ b/tensorflow/compat_template.__init__.py
@@ -0,0 +1,43 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Bring in all of the public TensorFlow interface into this module."""
+
+from __future__ import absolute_import as _absolute_import
+from __future__ import division as _division
+from __future__ import print_function as _print_function
+
+import os as _os
+
+# pylint: disable=g-bad-import-order
+from tensorflow.python import pywrap_tensorflow  # pylint: disable=unused-import
+
+# API IMPORTS PLACEHOLDER
+
+from tensorflow.python.tools import component_api_helper as _component_api_helper
+_component_api_helper.package_hook(
+    parent_package_str=__name__,
+    child_package_str=(
+        'tensorflow_estimator.python.estimator.api._v2.estimator'))
+_component_api_helper.package_hook(
+    parent_package_str=__name__,
+    child_package_str=('tensorflow.python.keras.api._v2.keras'))
+
+# We would like the following to work for fully enabling 2.0 in a 1.0 install:
+#
+# import tensorflow.compat.v2 as tf
+# tf.enable_v2_behavior()
+#
+# This make this one symbol available directly.
+from tensorflow.python.compat.v2_compat import enable_v2_behavior  # pylint: disable=g-import-not-at-top
diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index dc98702c08..0312114abe 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -246,5 +246,6 @@ tensorflow/third_party/eigen.BUILD
 tensorflow/stream_executor/build_defs.bzl
 tensorflow/api_template_v1.__init__.py
 tensorflow/compat_template_v1.__init__.py
+tensorflow/compat_template.__init__.py
 tensorflow/api_template.__init__.py
 tensorflow/__init__.py
\ No newline at end of file
diff --git a/tensorflow/tools/api/tests/api_compatibility_test.py b/tensorflow/tools/api/tests/api_compatibility_test.py
index 6b9fc7ddce..9350ebba57 100644
--- a/tensorflow/tools/api/tests/api_compatibility_test.py
+++ b/tensorflow/tools/api/tests/api_compatibility_test.py
@@ -265,7 +265,7 @@ class ApiCompatibilityTest(test.TestCase):
     visitor.do_not_descend_map['tf'].append('contrib')
     if FLAGS.only_test_core_api:
       visitor.do_not_descend_map['tf'].extend(_NON_CORE_PACKAGES)
-    traverse.traverse(tf_v2, visitor)
+    traverse.traverse(tf.compat.v2, visitor)
 
   def _checkBackwardsCompatibility(self,
                                    root,
@@ -351,7 +351,7 @@ class ApiCompatibilityTest(test.TestCase):
         tf_v2,
         golden_file_pattern,
         api_version,
-        additional_private_map={'tf.compat': ['v1']})
+        additional_private_map={'tf.compat': ['v1', 'v2']})
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 2bd0853709..7b15976b59 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -157,7 +157,10 @@ class TestUpgrade(test_util.TensorFlowTestCase):
           _, _, _, text = self._upgrade("tf." + name)
           if (text and
               not text.startswith("tf.compat.v1") and
-              text not in self.v2_symbols):
+              text not in self.v2_symbols and
+              # Builds currently install old version of estimator that doesn't
+              # have some 2.0 symbols.
+              not text.startswith("tf.estimator")):
             self.assertFalse(
                 True, "Symbol %s generated from %s not in v2 API" % (
                     text, name))
-- 
GitLab


From f562ead46078e28061daae125a272e617c576077 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Wed, 30 Jan 2019 12:50:53 -0800
Subject: [PATCH 1610/2345] Disable pinning small ops to CPU by default.

PiperOrigin-RevId: 231650090
---
 tensorflow/contrib/eager/python/tfe_test.py     | 4 ++++
 tensorflow/core/common_runtime/eager/context.cc | 2 +-
 tensorflow/python/eager/core_test.py            | 3 ++-
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/eager/python/tfe_test.py b/tensorflow/contrib/eager/python/tfe_test.py
index 8c35dddb5a..6881fabdc0 100644
--- a/tensorflow/contrib/eager/python/tfe_test.py
+++ b/tensorflow/contrib/eager/python/tfe_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 import tempfile
 
 from tensorflow.contrib.eager.python import tfe
+from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
@@ -40,6 +41,9 @@ class TFETest(test_util.TensorFlowTestCase):
     self.assertAllEqual([[4.]], y.numpy())
 
   def testInstantError(self):
+    if context.num_gpus():
+      # TODO(nareshmodi): make this test better
+      self.skipTest("Gather doesn't do index checking on GPUs")
     with self.assertRaisesRegexp(errors.InvalidArgumentError,
                                  r'indices = 7 is not in \[0, 3\)'):
       array_ops.gather([0, 1, 2], 7)
diff --git a/tensorflow/core/common_runtime/eager/context.cc b/tensorflow/core/common_runtime/eager/context.cc
index 4d8712f4dc..8068905da9 100644
--- a/tensorflow/core/common_runtime/eager/context.cc
+++ b/tensorflow/core/common_runtime/eager/context.cc
@@ -66,7 +66,7 @@ EagerContext::EagerContext(const SessionOptions& opts,
       env_(opts.env),
       use_send_tensor_rpc_(false),
       pin_small_ops_to_cpu_(ReadBoolFromEnvVar(
-          "TF_EAGER_ENABLE_SMALL_TENSOR_CPU_PINNING", true)) {
+          "TF_EAGER_ENABLE_SMALL_TENSOR_CPU_PINNING", false)) {
   if (device_mgr_owned) {
     local_device_manager_.reset(device_mgr);
     local_unowned_device_manager_ = nullptr;
diff --git a/tensorflow/python/eager/core_test.py b/tensorflow/python/eager/core_test.py
index e601aa376f..5432abab60 100644
--- a/tensorflow/python/eager/core_test.py
+++ b/tensorflow/python/eager/core_test.py
@@ -631,7 +631,8 @@ class TFETest(test_util.TensorFlowTestCase):
     for t in tensors:
       self.assertIsInstance(t, ops.EagerTensor)
 
-  def testSmallIntegerOpsForcedToCPU(self):
+  # TODO(b/123637108): re-enable
+  def disabled_testSmallIntegerOpsForcedToCPU(self):
     if not context.context().num_gpus():
       self.skipTest('No GPUs found')
 
-- 
GitLab


From 263788829b33a0f0cb3745b2162943bf852ef2ae Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 30 Jan 2019 12:57:23 -0800
Subject: [PATCH 1611/2345] In TPU embedding mid-level API tpu_embedding.py,
 create embedding variables, slot variables and related ops outside of
 constructor.

This will enable TPUEstimator to provide specific names for the embedding and slot variables. (TPUEstimator only know these names after TPUEmbedding constructor is called.)

PiperOrigin-RevId: 231651332
---
 .../contrib/tpu/python/tpu/tpu_embedding.py   | 200 ++++++++++--------
 .../contrib/tpu/python/tpu/tpu_estimator.py   |   6 +-
 2 files changed, 112 insertions(+), 94 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py b/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py
index 04e7397162..0e4597bd6f 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py
@@ -100,6 +100,25 @@ class TableConfig(
                                            initializer, combiner)
 
 
+AdamSlotVariableNames = collections.namedtuple(
+    'AdamSlotVariableNames', ['m', 'v'])
+
+AdagradSlotVariableName = collections.namedtuple(
+    'AdagradSlotVariableName', ['accumulator'])
+
+AdamSlotVariables = collections.namedtuple(
+    'AdamSlotVariables', ['m', 'v'])
+
+AdagradSlotVariable = collections.namedtuple(
+    'AdagradSlotVariable', ['accumulator'])
+
+VariablesAndOps = collections.namedtuple(
+    'VariablesAndOps',
+    ['embedding_variables_by_table', 'slot_variables_by_table',
+     'load_ops', 'retrieve_ops']
+)
+
+
 # TODO(shizhiw): Factor `use_gradient_accumulation` and
 # `pipeline_execution_with_tensor_core` out of `_OptimizationParameters`.
 class _OptimizationParameters(object):
@@ -236,6 +255,7 @@ class TPUEmbedding(object):
           sparse_features_list.append(sparse_features)
 
     enqueue_ops = embedding.generate_enqueue_ops(sparse_features_list)
+    embedding_variables_and_ops = embedding.create_variables_and_ops()
 
     def computation():
       activations = embedding.get_activations()
@@ -261,6 +281,7 @@ class TPUEmbedding(object):
                                      embedding.config_proto))
       sess.run(variables.global_variables_initializer())
       sess.run(embedding.init_ops)
+      sess.run(embedding_variables_and_ops.load_ops)
       sess.run(enqueue_ops)
       loss_val = sess.run(loss)
     ```
@@ -367,9 +388,6 @@ class TPUEmbedding(object):
 
     self._config_proto = self._create_config_proto()
 
-    self._create_variables_and_ops()
-    self._init_ops.extend(self._load_parameters_ops)
-
   @property
   def hosts(self):
     """A list of device names for CPU hosts.
@@ -435,28 +453,10 @@ class TPUEmbedding(object):
     """
     return self._init_ops
 
-  # TODO(shizhiw): get table variables the same way as getting slot variables.
-  @property
-  def table_to_table_variables_dict(self):
-    return copy.copy(self._table_to_table_variables_dict)
-
   @property
   def feature_to_table_dict(self):
     return copy.copy(self._feature_to_table_dict)
 
-  def get_slot_names(self):
-    """Return a list of the names of slots created by `TPUEmbedding`."""
-    return self._optimizer_handler.get_slot_names()
-
-  def get_slot(self, table, name):
-    """Return a slot named `name` create for `table` by `TPUEmbedding`."""
-    return self._optimizer_handler.get_slot(table, name)
-
-  # TODO(shizhiw): expose load to user too?
-  @property
-  def retrieve_parameters_ops(self):
-    return self._retrieve_parameters_ops
-
   def _create_config_proto(self):
     """Create `TPUEmbeddingConfiguration`."""
     config_proto = elc.TPUEmbeddingConfiguration()
@@ -487,30 +487,63 @@ class TPUEmbedding(object):
 
     return config_proto
 
-  def _create_variables_and_ops(self):
-    """Create embedding variables and return ops to load them into TPU."""
-    self._load_parameters_ops = []
-    self._retrieve_parameters_ops = []
-    self._table_to_table_variables_dict = {}
+  def create_variables_and_ops(self, embedding_variable_name_by_table=None,
+                               slot_variable_names_by_table=None):
+    """Create embedding and slot variables, with ops to load and retrieve them.
+
+    Args:
+      embedding_variable_name_by_table: A dictionary mapping from string of
+        table name to string of embedding variable name. If `None`,
+        defaults from `get_default_slot_variable_names()` will be used.
+      slot_variable_names_by_table: A dictionary mapping from string of table
+        name to `AdamSlotVariableNames`, `AdagradSlotVariableNames` etc. If
+        `None`, defaults from `get_default_slot_variable_names()` will be used.
+
+    Returns:
+      `tpu_embedding.VariablesAndOps` with:
+        A dictionary mapping from string of table name to embedding variables,
+        A dictionary mapping from string of table name to AdagradSlotVariable,
+         AdamSlotVariables etc with slot variables,
+        A list of ops to load embedding and slot variables on CPU to TPU,
+        A list of ops to retrieve embedding and slot variables from TPU to CPU.
+    """
+    embedding_variables_by_table = {}
+    slot_variables_by_table = {}
+    load_ops = []
+    retrieve_ops = []
     for table in self._table_to_config_dict:
+      if embedding_variable_name_by_table:
+        embedding_variable_name = embedding_variable_name_by_table[table]
+      else:
+        embedding_variable_name = table
+      if slot_variable_names_by_table:
+        slot_variable_names = slot_variable_names_by_table[table]
+      else:
+        slot_variable_names = (
+            self._optimizer_handler.get_default_slot_variable_names(table))
+
       device_fn = _create_device_fn(self._hosts)
       with ops.device(device_fn):
-        # TODO(shizhiw): allow user to specify variable name so that
-        # they could make the name consistent with CPU etc.
-        variable_name = table
         table_variables = _create_partitioned_variables(
-            name=variable_name,
+            name=embedding_variable_name,
             num_hosts=self._num_hosts,
             vocabulary_size=self._table_to_config_dict[table].vocabulary_size,
             embedding_dimension=self._table_to_config_dict[table].dimension,
             initializer=self._table_to_config_dict[table].initializer,
             collections=[ops.GraphKeys.GLOBAL_VARIABLES])
-        self._table_to_table_variables_dict[table] = table_variables
-
-        self._optimizer_handler.create_variables_and_ops(
-            table, variable_name, self._num_hosts,
-            self._table_to_config_dict[table], table_variables,
-            self._load_parameters_ops, self._retrieve_parameters_ops)
+        embedding_variables_by_table[table] = table_variables
+
+        slot_variables_for_table, load_ops_for_table, retrieve_ops_for_table = (
+            self._optimizer_handler.create_variables_and_ops(
+                table, slot_variable_names, self._num_hosts,
+                self._table_to_config_dict[table], table_variables)
+        )
+        slot_variables_by_table[table] = slot_variables_for_table
+        load_ops.extend(load_ops_for_table)
+        retrieve_ops.extend(retrieve_ops_for_table)
+    return VariablesAndOps(embedding_variables_by_table,
+                           slot_variables_by_table,
+                           load_ops, retrieve_ops)
 
   def _create_dummy_table_variables(self):
     """Create dummy embedding table variables.
@@ -804,13 +837,11 @@ class _OptimizerHandler(object):
   def set_optimization_parameters(self, table_descriptor):
     raise NotImplementedError()
 
-  def create_variables_and_ops(self, table, variable_name):
+  def get_default_slot_variable_names(self, table):
     raise NotImplementedError()
 
-  def get_slot_names(self):
-    raise NotImplementedError()
-
-  def get_slot(self, table, name):
+  def create_variables_and_ops(self, table, slot_variable_names, num_hosts,
+                               table_config, table_variables):
     raise NotImplementedError()
 
 
@@ -824,21 +855,24 @@ class _AdagradHandler(_OptimizerHandler):
   def set_optimization_parameters(self, table_descriptor):
     table_descriptor.optimization_parameters.adagrad.SetInParent()
 
-  def create_variables_and_ops(self, table, variable_name, num_hosts,
-                               table_config, table_variables,
-                               load_parameters_ops, retrieve_parameters_ops):
-    optimizer_name = 'Adagrad'
+  def get_default_slot_variable_names(self, table):
+    return AdagradSlotVariableName('{}/{}'.format(table, 'Adagrad'))
+
+  def create_variables_and_ops(self, table, slot_variable_names, num_hosts,
+                               table_config, table_variables):
     accumulator_initializer = init_ops.constant_initializer(
         self._optimization_parameters.initial_accumulator)
     accumulator_variables = _create_partitioned_variables(
-        name='%s/%s' % (variable_name, optimizer_name),
+        name=slot_variable_names.accumulator,
         num_hosts=num_hosts,
         vocabulary_size=table_config.vocabulary_size,
         embedding_dimension=table_config.dimension,
         collections=[ops.GraphKeys.GLOBAL_VARIABLES],
         initializer=accumulator_initializer)
+    slot_variables = AdagradSlotVariable(accumulator_variables)
 
-    self._table_to_accumulator_variables_dict[table] = accumulator_variables
+    load_ops = []
+    retrieve_ops = []
     for host_id, table_variable, accumulator_variable in (zip(
         range(num_hosts), table_variables, accumulator_variables)):
       with ops.colocate_with(table_variable):
@@ -858,17 +892,9 @@ class _AdagradHandler(_OptimizerHandler):
             state_ops.assign(table_variable, retrieved_table),
             state_ops.assign(accumulator_variable, retrieved_accumulator))
 
-      load_parameters_ops.append(load_parameters_op)
-      retrieve_parameters_ops.append(retrieve_parameters_op)
-
-  def get_slot_names(self):
-    return ['accumulator']
-
-  def get_slot(self, table, name):
-    if name not in self.get_slot_names():
-      raise ValueError('Adagrad has {} as slot names; got {}.'
-                       .format(self.get_slot_names(), name))
-    return self._table_to_accumulator_variables_dict[table]
+      load_ops.append(load_parameters_op)
+      retrieve_ops.append(retrieve_parameters_op)
+    return slot_variables, load_ops, retrieve_ops
 
 
 class _AdamHandler(_OptimizerHandler):
@@ -891,13 +917,15 @@ class _AdamHandler(_OptimizerHandler):
     table_descriptor.optimization_parameters.adam.use_sum_inside_sqrt = (
         self._optimization_parameters.sum_inside_sqrt)
 
-  def create_variables_and_ops(self, table, variable_name, num_hosts,
-                               table_config, table_variables,
-                               load_parameters_ops, retrieve_parameters_ops):
-    optimizer_name = 'Adam'
+  def get_default_slot_variable_names(self, table):
+    return AdamSlotVariableNames('{}/{}/m'.format(table, 'Adam'),
+                                 '{}/{}/v'.format(table, 'Adam'))
+
+  def create_variables_and_ops(self, table, slot_variable_names, num_hosts,
+                               table_config, table_variables):
     m_initializer = init_ops.zeros_initializer()
     m_variables = _create_partitioned_variables(
-        name='%s/%s/m' % (variable_name, optimizer_name),
+        name=slot_variable_names.m,
         num_hosts=num_hosts,
         vocabulary_size=table_config.vocabulary_size,
         embedding_dimension=table_config.dimension,
@@ -905,16 +933,16 @@ class _AdamHandler(_OptimizerHandler):
         initializer=m_initializer)
     v_initializer = init_ops.zeros_initializer()
     v_variables = _create_partitioned_variables(
-        name='%s/%s/v' % (variable_name, optimizer_name),
+        name=slot_variable_names.v,
         num_hosts=num_hosts,
         vocabulary_size=table_config.vocabulary_size,
         embedding_dimension=table_config.dimension,
         collections=[ops.GraphKeys.GLOBAL_VARIABLES],
         initializer=v_initializer)
+    slot_variables = AdamSlotVariables(m_variables, v_variables)
 
-    self._table_to_m_variables_dict[table] = m_variables
-    self._table_to_v_variables_dict[table] = v_variables
-
+    load_ops = []
+    retrieve_ops = []
     for host_id, table_variable, m_variable, v_variable in (zip(
         range(num_hosts), table_variables,
         m_variables, v_variables)):
@@ -937,20 +965,9 @@ class _AdamHandler(_OptimizerHandler):
             state_ops.assign(m_variable, retrieved_m),
             state_ops.assign(v_variable, retrieved_v))
 
-      load_parameters_ops.append(load_parameters_op)
-      retrieve_parameters_ops.append(retrieve_parameters_op)
-
-  def get_slot_names(self):
-    return ['m', 'v']
-
-  def get_slot(self, table, name):
-    if name == 'm':
-      return self._table_to_m_variables_dict[table]
-    elif name == 'v':
-      return self._table_to_v_variables_dict[table]
-    else:
-      raise ValueError('Adam has {} as slot names; got {}.'
-                       .format(self.get_slot_names(), name))
+      load_ops.append(load_parameters_op)
+      retrieve_ops.append(retrieve_parameters_op)
+    return slot_variables, load_ops, retrieve_ops
 
 
 class _StochasticGradientDescentHandler(_OptimizerHandler):
@@ -960,11 +977,15 @@ class _StochasticGradientDescentHandler(_OptimizerHandler):
     (table_descriptor.optimization_parameters.stochastic_gradient_descent
      .SetInParent())
 
-  def create_variables_and_ops(self, table, variable_name, num_hosts,
-                               table_config, table_variables,
-                               load_parameters_ops, retrieve_parameters_ops):
+  def get_default_slot_variable_names(self, table):
+    return None
+
+  def create_variables_and_ops(self, table, slot_variable_names, num_hosts,
+                               table_config, table_variables):
     del table_config
 
+    load_ops = []
+    retrieve_ops = []
     for host_id, table_variable in (zip(
         range(num_hosts), table_variables)):
       with ops.colocate_with(table_variable):
@@ -984,14 +1005,9 @@ class _StochasticGradientDescentHandler(_OptimizerHandler):
         retrieve_parameters_op = control_flow_ops.group(
             state_ops.assign(table_variable, retrieved_table))
 
-      load_parameters_ops.append(load_parameters_op)
-      retrieve_parameters_ops.append(retrieve_parameters_op)
-
-  def get_slot_names(self):
-    return []
-
-  def get_slot(self, table, name):
-    raise ValueError('Stochastic gradient descent does not have slot variable.')
+      load_ops.append(load_parameters_op)
+      retrieve_ops.append(retrieve_parameters_op)
+    return None, load_ops, retrieve_ops
 
 
 def _get_optimization_handler(optimization_parameters):
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 0620598ea0..10f3451619 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -2779,6 +2779,9 @@ class TPUEstimator(estimator_lib.Estimator):
         tpu_init_ops = []
         if ctx.embedding_config:
           tpu_init_ops.extend(ctx.embedding_config.tpu_embedding.init_ops)
+          embedding_variables_and_ops = (
+              ctx.embedding_config.tpu_embedding.create_variables_and_ops())
+          tpu_init_ops.extend(embedding_variables_and_ops.load_ops)
 
         input_holders = _InputPipeline(input_fn, batch_axis, ctx)
         enqueue_ops, dequeue_fn, input_hooks, run_infeed_loop_on_coordinator = (
@@ -2871,8 +2874,7 @@ class TPUEstimator(estimator_lib.Estimator):
             update_ops = _sync_variables_ops(ctx)
 
           if ctx.embedding_config:
-            update_ops.extend(
-                ctx.embedding_config.tpu_embedding.retrieve_parameters_ops)
+            update_ops.extend(embedding_variables_and_ops.retrieve_ops)
 
           # Validate the TPU training graph to catch basic errors
           _validate_tpu_training_graph()
-- 
GitLab


From d7942058a8b87aee92793bef19e0aeb5e45bb2bc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 30 Jan 2019 12:59:33 -0800
Subject: [PATCH 1612/2345] [TF:XLA] Copy control dependencies in
 increase_dynamism_for_auto_jit.

Previously, new constant nodes would be created, then the FixupSourceAndSinkEdges would create a dependency between these new constants and SOURCE_ which resulted in later failures when constructing control flow information. This change copies the dependency information to keep the same control flow.

PiperOrigin-RevId: 231651748
---
 .../increase_dynamism_for_auto_jit_pass.cc    | 41 ++++++++++++++-----
 ...ncrease_dynamism_for_auto_jit_pass_test.cc | 31 ++++++++++++++
 2 files changed, 62 insertions(+), 10 deletions(-)

diff --git a/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass.cc b/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass.cc
index ce53f70b79..ebfffc3267 100644
--- a/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass.cc
+++ b/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass.h"
+#include <iterator>
 #include "absl/algorithm/container.h"
 #include "absl/container/inlined_vector.h"
 #include "absl/strings/str_cat.h"
@@ -144,7 +145,9 @@ SliceInputs MakeSliceIndexAndSizeInt64(const Scope& host_scope,
 // same constant value.  This helps make the generated GraphDef more readable.
 class ConstantCache {
  public:
-  explicit ConstantCache(const Scope& s) : scope_(s) {}
+  explicit ConstantCache(const Scope& s,
+                         const std::vector<const Edge*>& control_deps)
+      : scope_(s), control_deps_(control_deps) {}
 
   Output Get1DHostConstant(int64 constant) {
     auto it = cache_.find(constant);
@@ -152,6 +155,9 @@ class ConstantCache {
       Output new_const =
           ops::Const(scope_.WithOpName("const_", constant), {constant});
       it = cache_.insert({constant, new_const}).first;
+      for (const Edge* e : control_deps_) {
+        scope_.graph()->AddControlEdge(e->src(), new_const.node());
+      }
     }
     return it->second;
   }
@@ -159,11 +165,13 @@ class ConstantCache {
  private:
   Scope scope_;
   std::unordered_map<int, Output> cache_;
+  std::vector<const Edge*> control_deps_;
 };
 
 // Returns a node computing the size of the Slice op with inputs `slice_inputs`.
 Status ComputeSliceSize(const Scope& host_scope,
-                        const SliceInputs& slice_inputs, Output* size) {
+                        const SliceInputs& slice_inputs,
+                        std::vector<const Edge*> control_deps, Output* size) {
   // If slice_size[i] >= 0 then slice_size[i] = slice_size[i].
   //
   // If slice_size[i] == -1 then slice_size[i] = input_size[i] -
@@ -183,7 +191,7 @@ Status ComputeSliceSize(const Scope& host_scope,
       ops::Shape(host_scope.WithOpName("input_shape"), slice_inputs.input,
                  ops::Shape::OutType(DT_INT64));
 
-  ConstantCache constant_pool(host_scope);
+  ConstantCache constant_pool(host_scope, control_deps);
 
   std::vector<Output> slice_size;
   for (int i = 0; i < slice_inputs.size_as_vector.size(); i++) {
@@ -209,11 +217,16 @@ Status ComputeSliceSize(const Scope& host_scope,
   }
 
   // Trivial ConcatV2 nodes (with exactly one input) are disallowed.
-  *size =
-      slice_size.size() == 1
-          ? slice_size[0]
-          : ops::Concat(host_scope.WithOpName("slice_size"), slice_size,
-                        ops::Const(host_scope.WithOpName("concat_axis"), 0));
+  if (slice_size.size() == 1) {
+    *size = slice_size[0];
+  } else {
+    auto concat_axis = ops::Const(host_scope.WithOpName("concat_axis"), 0);
+    for (const Edge* e : control_deps) {
+      host_scope.graph()->AddControlEdge(e->src(), concat_axis.node());
+    }
+    *size = ops::Concat(host_scope.WithOpName("slice_size"), slice_size,
+                        concat_axis);
+  }
   return Status::OK();
 }
 
@@ -237,9 +250,17 @@ Status ConvertTensorFlowSliceToStaticShapedSlice(
   SliceInputs slice_inputs_int64 =
       MakeSliceIndexAndSizeInt64(host_scope, slice_inputs);
 
+  // Create a list of all control dependencies to be copied when possibly
+  // replacing nodes related to slice_size.
+  Node* old_size;
+  std::vector<const Edge*> old_size_ctrl_deps;
+  TF_RETURN_IF_ERROR(slice->input_node(2, &old_size));
+  absl::c_copy_if(old_size->in_edges(), std::back_inserter(old_size_ctrl_deps),
+                  [](const Edge* e) { return e->IsControlEdge(); });
+
   Output slice_size;
-  TF_RETURN_IF_ERROR(
-      ComputeSliceSize(host_scope, slice_inputs_int64, &slice_size));
+  TF_RETURN_IF_ERROR(ComputeSliceSize(host_scope, slice_inputs_int64,
+                                      old_size_ctrl_deps, &slice_size));
 
   *result =
       ops::Slice(main_scope.WithAssignedDevice(slice->assigned_device_name())
diff --git a/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass_test.cc b/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass_test.cc
index a2f1b831ad..32e30216de 100644
--- a/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass_test.cc
+++ b/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass_test.cc
@@ -401,5 +401,36 @@ TEST(SliceToDynamicSliceRewriteTest, SliceWithSliceBegin) {
                      Name("begin/static_shaped_slice/static_shaped_slice"))),
                  _)));
 }
+
+// New constants being created need to have control dependencies copied to
+// ensure correct control flow analysis in TF V2.
+TEST(SliceToDynamicSliceRewriteTest, WithControlDepsToConstant) {
+  Scope root = Scope::NewRootScope()
+                   .ExitOnError()
+                   .WithAssignedDevice(kDeviceName)
+                   .WithXlaCluster("cluster_0");
+
+  Output input = ops::Placeholder(root.WithOpName("input"), DT_FLOAT);
+  Output begin = ops::Placeholder(root.WithOpName("begin"), DT_INT32);
+  Output size = ops::Const(root.WithOpName("size"), {-1});
+  Output slice = ops::Slice(root.WithOpName("slice"), input, begin, size);
+
+  // Add an additional dependency that should still exist in with the new size
+  // variables.
+  Output dependency = ops::Placeholder(root.WithOpName("dependency"), DT_BOOL);
+  root.graph()->AddControlEdge(dependency.node(), size.node());
+
+  std::unique_ptr<Graph> result;
+  TF_ASSERT_OK(IncreaseDynamismForAutoJit(root, &result));
+
+  // Check that the new constants have control dependencies.
+  Node* const_0 = testing::FindNodeByName(result.get(),
+                                          "slice/static_shaped_slice/const_0");
+  EXPECT_NE(const_0, nullptr);
+  EXPECT_THAT(const_0,
+              NodeWith(Op("Const"), CtrlDeps(NodeWith(Op("Placeholder"),
+                                                      Name("dependency")))));
+}
+
 }  // namespace
 }  // namespace tensorflow
-- 
GitLab


From 529496e2e056c30cc2ca401c7d7e0f4253ecf779 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 30 Jan 2019 13:12:06 -0800
Subject: [PATCH 1613/2345] enable some optimizers in diststrat tests

PiperOrigin-RevId: 231654398
---
 .../contrib/distribute/python/keras_backward_compat_test.py  | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_backward_compat_test.py b/tensorflow/contrib/distribute/python/keras_backward_compat_test.py
index 3bc84dc009..b783ab86a1 100644
--- a/tensorflow/contrib/distribute/python/keras_backward_compat_test.py
+++ b/tensorflow/contrib/distribute/python/keras_backward_compat_test.py
@@ -317,18 +317,17 @@ def all_strategy_combinations():
 
 
 def strategy_and_optimizer_combinations():
-  # TODO(b/122372746): Uncomment optimizers after they pass tests.
   return combinations.times(
       all_strategy_combinations(),
       combinations.combine(optimizer=[
           combinations.adagrad_optimizer_v1_fn,
-          # combinations.adagrad_optimizer_keras_v2_fn,
+          combinations.adagrad_optimizer_keras_v2_fn,
           combinations.adam_optimizer_v1_fn,
           combinations.adam_optimizer_keras_v2_fn,
           combinations.gradient_descent_optimizer_v1_fn,
           combinations.gradient_descent_optimizer_keras_v2_fn,
           combinations.rmsprop_optimizer_v1_fn,
-          # combinations.rmsprop_optimizer_keras_v2_fn
+          combinations.rmsprop_optimizer_keras_v2_fn
       ]))
 
 
-- 
GitLab


From a6031618c3e71ab4a66faf993f4b96e7cc717cca Mon Sep 17 00:00:00 2001
From: Sergii Khomenko <sergii.khomenko@stylight.com>
Date: Wed, 30 Jan 2019 22:41:00 +0100
Subject: [PATCH 1614/2345] Fix a minor typo

---
 tensorflow/python/keras/saving/saved_model.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/saving/saved_model.py b/tensorflow/python/keras/saving/saved_model.py
index fbf0bf68ef..a86555703a 100644
--- a/tensorflow/python/keras/saving/saved_model.py
+++ b/tensorflow/python/keras/saving/saved_model.py
@@ -52,7 +52,7 @@ def export(
   `save_model` generates new files/folders under the `saved_model_path` folder:
   1) a checkpoint containing the model weights.
   2) a saved_model.pb file containing the model's MetaGraphs. The prediction
-     graph is always exported. The evaluaton and training graphs are exported
+     graph is always exported. The evaluation and training graphs are exported
      if the following conditions are met:
      - Evaluation: model loss is defined.
      - Training: model is compiled with an optimizer defined under `tf.train`.
-- 
GitLab


From 81a2ada0943d8d5e301ecb836c486de7dededeba Mon Sep 17 00:00:00 2001
From: Sachin Joglekar <srjoglekar@google.com>
Date: Wed, 30 Jan 2019 13:15:49 -0800
Subject: [PATCH 1615/2345] Internal change

PiperOrigin-RevId: 231655045
---
 tensorflow/lite/tools/evaluation/BUILD        | 118 +++++++++++++
 .../lite/tools/evaluation/evaluation_stage.cc |  77 ++++++++
 .../lite/tools/evaluation/evaluation_stage.h  | 165 ++++++++++++++++++
 .../evaluation/evaluation_stage_factory.h     |  48 +++++
 .../tools/evaluation/evaluation_stage_test.cc | 145 +++++++++++++++
 .../lite/tools/evaluation/identity_stage.cc   |  91 ++++++++++
 .../lite/tools/evaluation/identity_stage.h    |  72 ++++++++
 tensorflow/lite/tools/evaluation/proto/BUILD  |  40 +++++
 .../evaluation/proto/evaluation_config.proto  |  48 +++++
 .../evaluation/proto/evaluation_stages.proto  |  38 ++++
 10 files changed, 842 insertions(+)
 create mode 100644 tensorflow/lite/tools/evaluation/BUILD
 create mode 100644 tensorflow/lite/tools/evaluation/evaluation_stage.cc
 create mode 100644 tensorflow/lite/tools/evaluation/evaluation_stage.h
 create mode 100644 tensorflow/lite/tools/evaluation/evaluation_stage_factory.h
 create mode 100644 tensorflow/lite/tools/evaluation/evaluation_stage_test.cc
 create mode 100644 tensorflow/lite/tools/evaluation/identity_stage.cc
 create mode 100644 tensorflow/lite/tools/evaluation/identity_stage.h
 create mode 100644 tensorflow/lite/tools/evaluation/proto/BUILD
 create mode 100644 tensorflow/lite/tools/evaluation/proto/evaluation_config.proto
 create mode 100644 tensorflow/lite/tools/evaluation/proto/evaluation_stages.proto

diff --git a/tensorflow/lite/tools/evaluation/BUILD b/tensorflow/lite/tools/evaluation/BUILD
new file mode 100644
index 0000000000..927a908ffa
--- /dev/null
+++ b/tensorflow/lite/tools/evaluation/BUILD
@@ -0,0 +1,118 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])  # Apache 2.0
+
+load("//tensorflow:tensorflow.bzl", "tf_cc_test")
+load("//tensorflow/lite:build_def.bzl", "tflite_copts", "tflite_linkopts")
+
+common_linkopts = tflite_linkopts() + select({
+    "//conditions:default": [],
+    "//tensorflow:android": [
+        "-pie",
+        "-llog",
+    ],
+})
+
+cc_library(
+    name = "evaluation_stage",
+    srcs = ["evaluation_stage.cc"],
+    hdrs = ["evaluation_stage.h"],
+    copts = tflite_copts(),
+    deps = [
+        "//tensorflow/core:lib",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_config_proto_cc",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "evaluation_stage_factory",
+    hdrs = ["evaluation_stage_factory.h"],
+    copts = tflite_copts(),
+    deps = [
+        ":evaluation_stage",
+        ":identity_stage",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_config_proto_cc",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_stages_proto_cc",
+        "@com_google_absl//absl/memory",
+    ],
+)
+
+cc_library(
+    name = "identity_stage",
+    srcs = ["identity_stage.cc"],
+    hdrs = ["identity_stage.h"],
+    copts = tflite_copts(),
+    deps = [
+        ":evaluation_stage",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "//tensorflow/cc:cc_ops",
+        "//tensorflow/cc:scope",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_config_proto_cc",
+    ] + select(
+        {
+            "//tensorflow:android": [
+                "//tensorflow/core:android_tensorflow_lib",
+            ],
+            "//conditions:default": [
+                "//tensorflow/core:tensorflow",
+                "//tensorflow/core:protos_all_cc",
+                "//tensorflow/core:core_cpu",
+                "//tensorflow/core:framework",
+                "//tensorflow/core:lib",
+                "//tensorflow/core:ops",
+            ],
+        },
+    ),
+    alwayslink = 1,
+)
+
+tf_cc_test(
+    name = "evaluation_stage_test",
+    srcs = ["evaluation_stage_test.cc"],
+    linkopts = common_linkopts,
+    linkstatic = 1,
+    tags = [
+        "tflite_not_portable_android",
+        "tflite_not_portable_ios",
+    ],
+    deps = [
+        ":evaluation_stage",
+        ":evaluation_stage_factory",
+        ":identity_stage",
+        "@com_google_googletest//:gtest_main",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_config_proto_cc",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_stages_proto_cc",
+    ] + select(
+        {
+            "//tensorflow:android": [
+                "//tensorflow/core:android_tensorflow_lib",
+                "//tensorflow/core:android_tensorflow_test_lib",
+            ],
+            "//conditions:default": [
+                "//tensorflow/core:framework",
+                "//tensorflow/core:core_cpu",
+                "//tensorflow/core:ops",
+                "//tensorflow/core:tensorflow",
+            ],
+        },
+    ),
+)
diff --git a/tensorflow/lite/tools/evaluation/evaluation_stage.cc b/tensorflow/lite/tools/evaluation/evaluation_stage.cc
new file mode 100644
index 0000000000..7c06142cbe
--- /dev/null
+++ b/tensorflow/lite/tools/evaluation/evaluation_stage.cc
@@ -0,0 +1,77 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/evaluation/evaluation_stage.h"
+
+#include <string>
+
+#include "absl/strings/str_split.h"
+
+namespace tflite {
+namespace evaluation {
+
+bool EvaluationStage::Init(
+    absl::flat_hash_map<std::string, void*>& object_map) {
+  // Process & validate configuration of tags.
+  std::vector<std::string> initializers, inputs, outputs;
+  for (const auto& init : config_.initializers()) {
+    initializers.emplace_back(init);
+  }
+  for (const auto& in : config_.inputs()) {
+    inputs.emplace_back(in);
+  }
+  for (const auto& out : config_.outputs()) {
+    outputs.emplace_back(out);
+  }
+  if (!ProcessExpectedTags(GetInitializerTags(), initializers) ||
+      !ProcessExpectedTags(GetInputTags(), inputs) ||
+      !ProcessExpectedTags(GetOutputTags(), outputs)) {
+    return false;
+  }
+  // Class-specific stuff.
+  return DoInit(object_map);
+}
+
+bool EvaluationStage::ProcessExpectedTags(
+    const std::vector<std::string>& expected_tags,
+    std::vector<std::string>& tag_to_name_mappings) {
+  // Validate format of each TAG:name mapping in tag_to_name_mappings, and add
+  // it to tags_to_names_map_.
+  for (const std::string& tag_name_mapping : tag_to_name_mappings) {
+    if (!std::regex_match(tag_name_mapping, kTagNameMappingPattern)) {
+      LOG(ERROR) << "Invalid TAG:name mapping: " << tag_name_mapping;
+      return false;
+    }
+    std::vector<std::string> tag_and_name =
+        absl::StrSplit(tag_name_mapping, ':');
+    tags_to_names_map_[tag_and_name[0]] = tag_and_name[1];
+  }
+
+  // Ensure each expected TAG is valid & has been mapped to a name.
+  for (const std::string& tag : expected_tags) {
+    if (!std::regex_match(std::string(tag), kTagPattern)) {
+      LOG(ERROR) << "Invalid expected TAG: " << tag;
+      return false;
+    }
+    if (tags_to_names_map_.find(tag) == tags_to_names_map_.end()) {
+      LOG(ERROR) << "TAG " << tag << " has not been mapped to a name in config "
+                 << config_.name();
+      return false;
+    }
+  }
+  return true;
+}
+
+}  // namespace evaluation
+}  // namespace tflite
diff --git a/tensorflow/lite/tools/evaluation/evaluation_stage.h b/tensorflow/lite/tools/evaluation/evaluation_stage.h
new file mode 100644
index 0000000000..4ae1810f23
--- /dev/null
+++ b/tensorflow/lite/tools/evaluation/evaluation_stage.h
@@ -0,0 +1,165 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TOOLS_EVALUATION_EVALUATION_STAGE_H_
+#define TENSORFLOW_LITE_TOOLS_EVALUATION_EVALUATION_STAGE_H_
+
+#include <regex>  // NOLINT
+#include <vector>
+
+#include "absl/container/flat_hash_map.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+
+namespace tflite {
+namespace evaluation {
+
+// Superclass for a single stage of an EvaluationPipeline.
+// Provides basic functionality for construction and accessing
+// initializers/inputs/outputs.
+// Every subclass of EvaluationStage will define its own behavior by specifying
+// appropriate accessor TAGs and implementing the Init, Run and Close methods.
+class EvaluationStage {
+ public:
+  // Initializes an EvaluationStage. Returns false if initialization failed,
+  // true otherwise.
+  // Should be called only once, before any call to Run().
+  // object_map should contain {initializer name : object pointer} mappings
+  // required for initialization.
+  //
+  // NOTE: EvaluationStage will not take ownership of any elements of
+  // object_map.
+  bool Init(absl::flat_hash_map<std::string, void*>& object_map);
+
+  // An individual run of the EvaluationStage. Returns false if there was a
+  // failure, true otherwise. Populates metrics into the EvaluationStageMetrics
+  // proto.
+  // Init() should be called before any calls to run().
+  // Inputs are acquired from and outputs are written to the incoming
+  // object_map, using appropriate TAGs.
+  //
+  // NOTE: The EvaluationStage should maintain ownership of outputs it
+  // populates into object_map. Ownership of inputs will be maintained
+  // elsewhere.
+  virtual bool Run(absl::flat_hash_map<std::string, void*>& object_map,
+                   EvaluationStageMetrics& metrics) = 0;
+
+  virtual ~EvaluationStage() = default;
+
+ protected:
+  // Constructs an EvaluationStage.
+  // Each subclass constructor must invoke this constructor.
+  //
+  // NOTE: Do NOT use constructors to obtain new EvaluationStages. Use
+  // tflite::evaluation::GetEvaluationStageFromConfig from
+  // evaluation_stage_factory.h instead.
+  explicit EvaluationStage(const EvaluationStageConfig& config)
+      : config_(config) {}
+
+  // Class-specific initialization, to be overridden by EvaluationStage
+  // sub-classes. Gets called in EvaluationStage::Init().
+  //
+  // NOTE: This object should not take ownership of any elements of object_map.
+  virtual bool DoInit(absl::flat_hash_map<std::string, void*>& object_map) = 0;
+
+  // The three following functions return the initializer/input/output TAGs used
+  // by an EvaluationStage. These should be mapped to meaningful names in the
+  // EvaluationStageConfig, and to required objects during calls to Init/Run.
+  // Format for TAGs: [A-Z0-9_]+ (Uppercase letters, numbers, "_")
+  // Refer docs in tflite.evaluation.EvaluationStageConfig for more information.
+
+  // Returns the expected initializer TAGs.
+  virtual std::vector<std::string> GetInitializerTags() = 0;
+
+  // Returns the expected input TAGs.
+  virtual std::vector<std::string> GetInputTags() = 0;
+
+  // Returns the expected output TAGs.
+  virtual std::vector<std::string> GetOutputTags() = 0;
+
+  // Populates a pointer to the object corresponding to provided TAG.
+  // Returns true if success, false otherwise.
+  // object_map must contain {name : object pointer} mappings, with one of the
+  // names being mapped to the expected TAG in the EvaluationStageConfig.
+  template <class T>
+  bool GetObjectFromTag(const std::string& tag,
+                        absl::flat_hash_map<std::string, void*>& object_map,
+                        T** object_ptr) {
+    *object_ptr = nullptr;
+    // Find name corresponding to TAG.
+    auto mapping_iter = tags_to_names_map_.find(tag);
+    if (mapping_iter == tags_to_names_map_.end()) {
+      LOG(ERROR) << "Unexpected TAG: " << tag;
+      return false;
+    }
+    const std::string& expected_name = mapping_iter->second;
+
+    // Find object from name.
+    auto object_iter = object_map.find(expected_name);
+    if (object_iter == object_map.end()) {
+      LOG(ERROR) << "Could not find object for name: " << expected_name;
+      return false;
+    }
+    *object_ptr = static_cast<T*>(object_iter->second);
+    return true;
+  }
+
+  // Maps the appropriate name to a given object in object_map. The name is
+  // derived from mappings provided in the EvaluationStageConfig.
+  // Returns false if tag is invalid, true otherwise.
+  //
+  // NOTE: The EvaluationStage must maintain ownership of object for the
+  // lifetime of object_map
+  bool AssignObjectToTag(const std::string& tag, void* object_ptr,
+                         absl::flat_hash_map<std::string, void*>& object_map) {
+    // Find name corresponding to TAG.
+    auto mapping_iter = tags_to_names_map_.find(tag);
+    if (mapping_iter == tags_to_names_map_.end()) {
+      LOG(ERROR) << "Unexpected TAG: " << tag;
+      return false;
+    }
+    const std::string& expected_name = mapping_iter->second;
+
+    object_map[expected_name] = object_ptr;
+    return true;
+  }
+
+  const EvaluationStageConfig config_;
+
+ private:
+  // Verifies that all TAGs from expected_tags are present in
+  // tag_to_name_mappings, and then populates tags_to_names_map_ with the
+  // appropriate entries. Returns false in case any TAG/mapping is invalid, true
+  // otherwise.
+  // expected_tags should be a list of TAG-strings.
+  // tag_to_name_mappings should be RepeatedPtrField of strings mapping TAGs to
+  // names in the form "SOME_TAG:some_name".
+  bool ProcessExpectedTags(const std::vector<std::string>& expected_tags,
+                           std::vector<std::string>& tag_to_name_mappings);
+
+  // Maps expected TAGs to their names as defined by the EvaluationStageConfig.
+  absl::flat_hash_map<std::string, std::string> tags_to_names_map_;
+
+  // To ensure correct formatting in the config.
+  const std::regex kTagNameMappingPattern{"^([A-Z0-9_]+):([a-z0-9_]+)$",
+                                          std::regex::optimize};
+
+  // To ensure correct formatting in TAG names.
+  const std::regex kTagPattern{"^[A-Z0-9_]+$", std::regex::optimize};
+};
+
+}  // namespace evaluation
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TOOLS_EVALUATION_EVALUATION_STAGE_H_
diff --git a/tensorflow/lite/tools/evaluation/evaluation_stage_factory.h b/tensorflow/lite/tools/evaluation/evaluation_stage_factory.h
new file mode 100644
index 0000000000..ea70fae614
--- /dev/null
+++ b/tensorflow/lite/tools/evaluation/evaluation_stage_factory.h
@@ -0,0 +1,48 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TOOLS_EVALUATION_EVALUATION_STAGE_FACTORY_H_
+#define TENSORFLOW_LITE_TOOLS_EVALUATION_EVALUATION_STAGE_FACTORY_H_
+
+#include "absl/memory/memory.h"
+#include "tensorflow/lite/tools/evaluation/evaluation_stage.h"
+#include "tensorflow/lite/tools/evaluation/identity_stage.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h"
+
+namespace tflite {
+namespace evaluation {
+
+// The canonical way to generate EvaluationStages.
+// TODO(b/122482115): Implement a Factory class for registration of classes.
+std::unique_ptr<EvaluationStage> CreateEvaluationStageFromConfig(
+    const EvaluationStageConfig& config) {
+  if (!config.has_specification() ||
+      !config.specification().has_process_class()) {
+    LOG(ERROR) << "Process specification not present in config: "
+               << config.name();
+    return nullptr;
+  }
+  switch (config.specification().process_class()) {
+    case UNKNOWN:
+      return nullptr;
+    case IDENTITY:
+      return absl::make_unique<IdentityStage>(config);
+  }
+}
+
+}  // namespace evaluation
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TOOLS_EVALUATION_EVALUATION_STAGE_FACTORY_H_
diff --git a/tensorflow/lite/tools/evaluation/evaluation_stage_test.cc b/tensorflow/lite/tools/evaluation/evaluation_stage_test.cc
new file mode 100644
index 0000000000..79cbf1e5e5
--- /dev/null
+++ b/tensorflow/lite/tools/evaluation/evaluation_stage_test.cc
@@ -0,0 +1,145 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/evaluation/evaluation_stage.h"
+
+#include <memory>
+
+#include <gtest/gtest.h>
+#include "absl/container/flat_hash_map.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/types.pb.h"
+#include "tensorflow/lite/tools/evaluation/evaluation_stage_factory.h"
+#include "tensorflow/lite/tools/evaluation/identity_stage.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h"
+
+namespace tflite {
+namespace evaluation {
+namespace {
+
+using ::tensorflow::DataType;
+using ::tensorflow::Tensor;
+
+constexpr char kIdentityStageName[] = "identity_stage";
+constexpr char kInputTypeName[] = "type";
+constexpr char kInputTensorsName[] = "in";
+constexpr char kOutputTensorsName[] = "out";
+constexpr char kInitializerMapping[] = "INPUT_TYPE:type";
+constexpr char kInputMapping[] = "INPUT_TENSORS:in";
+constexpr char kOutputMapping[] = "OUTPUT_TENSORS:out";
+
+EvaluationStageConfig GetIdentityStageConfig() {
+  EvaluationStageConfig config;
+  config.set_name(kIdentityStageName);
+  config.mutable_specification()->set_process_class(IDENTITY);
+  config.add_initializers(kInitializerMapping);
+  config.add_inputs(kInputMapping);
+  config.add_outputs(kOutputMapping);
+  return config;
+}
+
+TEST(EvaluationStage, IncompleteConfig) {
+  // Construct
+  EvaluationStageConfig config = GetIdentityStageConfig();
+  config.clear_inputs();
+  std::unique_ptr<EvaluationStage> stage_ptr =
+      CreateEvaluationStageFromConfig(config);
+  // Initialize
+  absl::flat_hash_map<std::string, void*> object_map;
+  DataType input_type = tensorflow::DT_FLOAT;
+  object_map[kInputTypeName] = &input_type;
+  EXPECT_FALSE(stage_ptr->Init(object_map));
+}
+
+TEST(EvaluationStage, IncorrectlyFormattedConfig) {
+  // Construct
+  EvaluationStageConfig config = GetIdentityStageConfig();
+  config.clear_initializers();
+  config.add_initializers("INPUT_TYPE-type");
+  std::unique_ptr<EvaluationStage> stage_ptr =
+      CreateEvaluationStageFromConfig(config);
+  // Initialize
+  absl::flat_hash_map<std::string, void*> object_map;
+  DataType input_type = tensorflow::DT_FLOAT;
+  object_map[kInputTypeName] = &input_type;
+  EXPECT_FALSE(stage_ptr->Init(object_map));
+}
+
+TEST(EvaluationStage, ConstructFromConfig_UnknownProcess) {
+  // Construct
+  EvaluationStageConfig config = GetIdentityStageConfig();
+  config.mutable_specification()->clear_process_class();
+  std::unique_ptr<EvaluationStage> stage_ptr =
+      CreateEvaluationStageFromConfig(config);
+  EXPECT_EQ(stage_ptr.get(), nullptr);
+}
+
+TEST(EvaluationStage, NoInitializer) {
+  // Construct
+  EvaluationStageConfig config = GetIdentityStageConfig();
+  std::unique_ptr<EvaluationStage> stage_ptr =
+      CreateEvaluationStageFromConfig(config);
+  // Initialize
+  absl::flat_hash_map<std::string, void*> object_map;
+  EXPECT_FALSE(stage_ptr->Init(object_map));
+}
+
+TEST(EvaluationStage, NoInputs) {
+  // Construct
+  EvaluationStageConfig config = GetIdentityStageConfig();
+  std::unique_ptr<EvaluationStage> stage_ptr =
+      CreateEvaluationStageFromConfig(config);
+  // Initialize
+  absl::flat_hash_map<std::string, void*> object_map;
+  DataType input_type = tensorflow::DT_FLOAT;
+  object_map[kInputTypeName] = &input_type;
+  EXPECT_TRUE(stage_ptr->Init(object_map));
+
+  // Run
+  EvaluationStageMetrics metrics;
+  EXPECT_FALSE(stage_ptr->Run(object_map, metrics));
+}
+
+TEST(EvaluationStage, ExpectedIdentityOutput) {
+  // Construct
+  EvaluationStageConfig config = GetIdentityStageConfig();
+  std::unique_ptr<EvaluationStage> stage_ptr =
+      CreateEvaluationStageFromConfig(config);
+  // Initialize
+  absl::flat_hash_map<std::string, void*> object_map;
+  DataType input_type = tensorflow::DT_FLOAT;
+  object_map[kInputTypeName] = &input_type;
+  EXPECT_TRUE(stage_ptr->Init(object_map));
+
+  // Input Data
+  float float_value = 5.6f;
+  Tensor input_tensor(float_value);
+  std::vector<Tensor> input_tensors = {input_tensor};
+  // Run
+  object_map[kInputTensorsName] = &input_tensors;
+  EvaluationStageMetrics metrics;
+  EXPECT_TRUE(stage_ptr->Run(object_map, metrics));
+
+  // Check output
+  std::vector<Tensor>* output_tensors_ptr =
+      static_cast<std::vector<Tensor>*>(object_map[kOutputTensorsName]);
+  EXPECT_TRUE(output_tensors_ptr != nullptr);
+  EXPECT_FLOAT_EQ(output_tensors_ptr->at(0).scalar<float>()(), float_value);
+  EXPECT_GE(metrics.total_latency_ms(), 0);
+}
+
+}  // namespace
+}  // namespace evaluation
+}  // namespace tflite
diff --git a/tensorflow/lite/tools/evaluation/identity_stage.cc b/tensorflow/lite/tools/evaluation/identity_stage.cc
new file mode 100644
index 0000000000..337372a475
--- /dev/null
+++ b/tensorflow/lite/tools/evaluation/identity_stage.cc
@@ -0,0 +1,91 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/evaluation/identity_stage.h"
+
+#include <ctime>
+
+#include "tensorflow/cc/ops/array_ops.h"
+#include "tensorflow/cc/ops/standard_ops.h"
+#include "tensorflow/core/public/session_options.h"
+
+namespace tflite {
+namespace evaluation {
+
+using ::tensorflow::Scope;
+using ::tensorflow::SessionOptions;
+using ::tensorflow::Tensor;
+using ::tensorflow::ops::Identity;
+using ::tensorflow::ops::Placeholder;
+
+IdentityStage::IdentityStage(const EvaluationStageConfig& config)
+    : EvaluationStage(config) {
+  stage_input_name_ = config_.name() + "_identity_input";
+  stage_output_name_ = config_.name() + "_identity_output";
+}
+
+bool IdentityStage::DoInit(
+    absl::flat_hash_map<std::string, void*>& object_map) {
+  // Initialize TF Graph.
+  const Scope scope = Scope::NewRootScope();
+  if (!GetObjectFromTag(kInputTypeTag, object_map, &input_type_)) {
+    return false;
+  }
+  auto input_placeholder =
+      Placeholder(scope.WithOpName(stage_input_name_), *input_type_);
+  stage_output_ =
+      Identity(scope.WithOpName(stage_output_name_), input_placeholder);
+  if (!scope.status().ok() || !scope.ToGraphDef(&graph_def_).ok()) {
+    return false;
+  }
+
+  // Initialize TF Session.
+  session_.reset(NewSession(SessionOptions()));
+  if (!session_->Create(graph_def_).ok()) {
+    return false;
+  }
+
+  return true;
+}
+
+bool IdentityStage::Run(absl::flat_hash_map<std::string, void*>& object_map,
+                        EvaluationStageMetrics& metrics) {
+  std::vector<Tensor>* input_tensors;
+  if (!GetObjectFromTag(kInputTensorsTag, object_map, &input_tensors)) {
+    return false;
+  }
+  tensor_outputs_.clear();
+  // TODO(b/122482115): Encapsulate timing into its own helper.
+  std::clock_t start = std::clock();
+  if (!session_
+           ->Run({{stage_input_name_, input_tensors->at(0)}},
+                 {stage_output_name_}, {}, &tensor_outputs_)
+           .ok()) {
+    return false;
+  }
+  metrics.set_total_latency_ms(
+      static_cast<float>((std::clock() - start) / (CLOCKS_PER_SEC / 1000)));
+
+  if (!AssignObjectToTag(kOutputTensorsTag, &tensor_outputs_, object_map)) {
+    return false;
+  }
+  return true;
+}
+
+const char IdentityStage::kInputTypeTag[] = "INPUT_TYPE";
+const char IdentityStage::kInputTensorsTag[] = "INPUT_TENSORS";
+const char IdentityStage::kOutputTensorsTag[] = "OUTPUT_TENSORS";
+
+}  // namespace evaluation
+}  // namespace tflite
diff --git a/tensorflow/lite/tools/evaluation/identity_stage.h b/tensorflow/lite/tools/evaluation/identity_stage.h
new file mode 100644
index 0000000000..8bdddb1d74
--- /dev/null
+++ b/tensorflow/lite/tools/evaluation/identity_stage.h
@@ -0,0 +1,72 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TOOLS_EVALUATION_IDENTITY_STAGE_H_
+#define TENSORFLOW_LITE_TOOLS_EVALUATION_IDENTITY_STAGE_H_
+
+#include "absl/container/flat_hash_map.h"
+#include "tensorflow/cc/framework/scope.h"
+#include "tensorflow/core/public/session.h"
+#include "tensorflow/lite/tools/evaluation/evaluation_stage.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+
+namespace tflite {
+namespace evaluation {
+
+// Simple EvaluationStage subclass that encapsulates the functionality of
+// tensorflow::ops::Identity. Primarily used for tests.
+// Initializer TAGs (Object Class): INPUT_TYPE (DataType)
+// Input TAGs (Object Class): INPUT_TENSORS (std::vector<Tensor>)
+// Output TAGs (Object Class): OUTPUT_TENSORS (std::vector<Tensor>)
+// TODO(b/122482115): Migrate common TF-related code into an abstract class.
+class IdentityStage : public EvaluationStage {
+ public:
+  explicit IdentityStage(const EvaluationStageConfig& config);
+
+  bool Run(absl::flat_hash_map<std::string, void*>& object_map,
+           EvaluationStageMetrics& metrics) override;
+
+  ~IdentityStage() {}
+
+ protected:
+  bool DoInit(absl::flat_hash_map<std::string, void*>& object_map) override;
+
+  std::vector<std::string> GetInitializerTags() override {
+    return {kInputTypeTag};
+  }
+  std::vector<std::string> GetInputTags() override {
+    return {kInputTensorsTag};
+  }
+  std::vector<std::string> GetOutputTags() override {
+    return {kOutputTensorsTag};
+  }
+
+ private:
+  ::tensorflow::DataType* input_type_;
+  ::tensorflow::GraphDef graph_def_;
+  ::tensorflow::Output stage_output_;
+  std::unique_ptr<::tensorflow::Session> session_;
+  std::vector<::tensorflow::Tensor> tensor_outputs_;
+  std::string stage_input_name_;
+  std::string stage_output_name_;
+
+  static const char kInputTypeTag[];
+  static const char kInputTensorsTag[];
+  static const char kOutputTensorsTag[];
+};
+
+}  // namespace evaluation
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TOOLS_EVALUATION_IDENTITY_STAGE_H_
diff --git a/tensorflow/lite/tools/evaluation/proto/BUILD b/tensorflow/lite/tools/evaluation/proto/BUILD
new file mode 100644
index 0000000000..6c74735704
--- /dev/null
+++ b/tensorflow/lite/tools/evaluation/proto/BUILD
@@ -0,0 +1,40 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "//tensorflow/core:platform/default/build_config.bzl",
+    "tf_proto_library_cc",
+)
+
+tf_proto_library_cc(
+    name = "evaluation_stages_proto",
+    srcs = [
+        "evaluation_stages.proto",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+tf_proto_library_cc(
+    name = "evaluation_config_proto",
+    srcs = [
+        "evaluation_config.proto",
+    ],
+    protodeps = [":evaluation_stages_proto"],
+    visibility = ["//visibility:public"],
+)
diff --git a/tensorflow/lite/tools/evaluation/proto/evaluation_config.proto b/tensorflow/lite/tools/evaluation/proto/evaluation_config.proto
new file mode 100644
index 0000000000..4f36ac83b8
--- /dev/null
+++ b/tensorflow/lite/tools/evaluation/proto/evaluation_config.proto
@@ -0,0 +1,48 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+syntax = "proto2";
+
+package tflite.evaluation;
+
+import "tensorflow/lite/tools/evaluation/proto/evaluation_stages.proto";
+
+// Next ID: 6
+message EvaluationStageConfig {
+  optional string name = 1;
+
+  // Specification defining what this stage does, and any required parameters.
+  optional ProcessSpecification specification = 2;
+
+  // initializers, inputs and outputs are strings that define colon-separated
+  // mappings between TAGs and their corresponding names.
+  // These names help EvaluationStages communicate with each other during runs.
+  // Format for TAGs: [A-Z0-9_]+ (Uppercase letters, numbers, "_")
+  // Format for names: [a-z0-9_]+ (Lowercase letters, numbers, "_")
+  // Example mapping: "BITMAP1:image_in"
+  // It is up to individual EvaluationStage sub-classes to specify the
+  // initializer/input TAGs they require, and outputs TAGs they provide.
+  repeated string initializers = 3;
+  repeated string inputs = 4;
+  repeated string outputs = 5;
+}
+
+message EvaluationStageMetrics {
+  // Total latency in ms.
+  optional double total_latency_ms = 1;
+
+  // Process-specific numbers such as accuracy, step-latencies, etc.
+  optional ProcessMetrics process_metrics = 2;
+}
diff --git a/tensorflow/lite/tools/evaluation/proto/evaluation_stages.proto b/tensorflow/lite/tools/evaluation/proto/evaluation_stages.proto
new file mode 100644
index 0000000000..f45d96fafc
--- /dev/null
+++ b/tensorflow/lite/tools/evaluation/proto/evaluation_stages.proto
@@ -0,0 +1,38 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+syntax = "proto2";
+
+package tflite.evaluation;
+
+// All EvaluationStage sub-classes must add a value here.
+// A corresponding entry must also be present in EvaluationStage.FromConfig
+enum ProcessClass {
+  // Default/Unknown
+  UNKNOWN = 0;
+  // Identity
+  IDENTITY = 1;
+}
+
+// Defines the functionality executed by an EvaluationStage.
+// TODO(b/122482115): Add stage-specific options using oneof.
+message ProcessSpecification {
+  optional ProcessClass process_class = 1;
+}
+
+// Contains specific metrics, which may differ based on what an EvaluationStage
+// does.
+// TODO(b/122482115): Add stage-specific metrics using oneof.
+message ProcessMetrics {}
-- 
GitLab


From 8970d62487b852c563b98f658d690d32b3869e7c Mon Sep 17 00:00:00 2001
From: hari <hari@etacompute.com>
Date: Wed, 30 Jan 2019 13:49:11 -0800
Subject: [PATCH 1616/2345] removed DebugLog definition fro here and moved it
 to tensorflow/lite/experimental/micro/ecm3531/debug.log

---
 .../experimental/micro/tools/make/targets/ecm3531/_main.c    | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/tensorflow/lite/experimental/micro/tools/make/targets/ecm3531/_main.c b/tensorflow/lite/experimental/micro/tools/make/targets/ecm3531/_main.c
index 2764f3ba50..255eb04f0f 100644
--- a/tensorflow/lite/experimental/micro/tools/make/targets/ecm3531/_main.c
+++ b/tensorflow/lite/experimental/micro/tools/make/targets/ecm3531/_main.c
@@ -51,11 +51,6 @@ void EtaPrintExecutionTime(uint64_t);
 //*****************************************************************************
 extern int main(int argc, char** argv);
 
-void DebugLog(const char* s) { EtaCspIoPrintf("%s", s); }
-void DebugLogInt32(int32_t i) { EtaCspIoPrintf("%d", i); }
-void DebugLogUInt32(uint32_t i) { EtaCspIoPrintf("%d", i); }
-void DebugLogHex(uint32_t i) { EtaCspIoPrintf("0x%8x", i); }
-void DebugLogFloat(float i) { EtaCspIoPrintf("%f", i); }
 
 int _main(void) {
   uint64_t time_ms;
-- 
GitLab


From 2e571c2314c5fd61778cc0d71ca2265dfa18bb29 Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Wed, 30 Jan 2019 13:45:36 -0800
Subject: [PATCH 1617/2345] Add some conversions from contrib to core, and warn
 about contrib symbols that could not be converted.

PiperOrigin-RevId: 231660516
---
 tensorflow/tools/compatibility/ast_edits.py   | 24 +++++++++++++++++++
 .../tools/compatibility/ast_edits_test.py     | 18 ++++++++++++++
 tensorflow/tools/compatibility/tf_upgrade.py  |  2 ++
 .../tools/compatibility/tf_upgrade_v2.py      | 22 +++++++++++++----
 .../tools/compatibility/tf_upgrade_v2_test.py |  7 ++++++
 5 files changed, 69 insertions(+), 4 deletions(-)

diff --git a/tensorflow/tools/compatibility/ast_edits.py b/tensorflow/tools/compatibility/ast_edits.py
index 572bf384e4..7c11cc17b2 100644
--- a/tensorflow/tools/compatibility/ast_edits.py
+++ b/tensorflow/tools/compatibility/ast_edits.py
@@ -116,6 +116,8 @@ class APIChangeSpec(object):
   * `function_warnings`: maps full names of functions to warnings that will be
     printed out if the function is used. (e.g. tf.nn.convolution())
   * `function_transformers`: maps function names to custom handlers
+  * `leftover_warnings`: These warnings are printed if a matching Attribute
+    still exists after all other transformations have run.
 
   For an example, see `TFAPIChangeSpec`.
   """
@@ -242,6 +244,20 @@ class _PastaEditVisitor(ast.NodeVisitor):
     else:
       return False
 
+  def _maybe_add_module_deprecation_warning(self, node, full_name, whole_name):
+    """Adds a warning if full_name is a deprecated module."""
+    warnings = self._api_change_spec.module_deprecations
+    if full_name in warnings:
+      level, message = warnings[full_name]
+      message = message.replace("<function name>", whole_name)
+      self.add_log(level, node.lineno, node.col_offset,
+                   "Using member %s in deprecated module %s. %s" % (whole_name,
+                                                                    full_name,
+                                                                    message))
+      return True
+    else:
+      return False
+
   def _maybe_add_call_warning(self, node, full_name, name):
     """Print a warning when specific functions are called with selected args.
 
@@ -440,6 +456,14 @@ class _PastaEditVisitor(ast.NodeVisitor):
       if self._maybe_change_to_function_call(parent, node, full_name):
         return
 
+      # The isinstance check is enough -- a bare Attribute is never root.
+      i = 2
+      while isinstance(self._stack[-i], ast.Attribute):
+        i += 1
+      whole_name = pasta.dump(self._stack[-(i-1)])
+
+      self._maybe_add_module_deprecation_warning(node, full_name, whole_name)
+
     self.generic_visit(node)
 
 
diff --git a/tensorflow/tools/compatibility/ast_edits_test.py b/tensorflow/tools/compatibility/ast_edits_test.py
index 366ea2cb72..a9307f9f83 100644
--- a/tensorflow/tools/compatibility/ast_edits_test.py
+++ b/tensorflow/tools/compatibility/ast_edits_test.py
@@ -58,6 +58,15 @@ class NoUpdateSpec(ast_edits.APIChangeSpec):
     self.symbol_renames = {}
     self.function_warnings = {}
     self.change_to_function = {}
+    self.module_deprecations = {}
+
+
+class ModuleDeprecationSpec(NoUpdateSpec):
+  """A specification which deprecates 'a.b'."""
+
+  def __init__(self):
+    NoUpdateSpec.__init__(self)
+    self.module_deprecations.update({"a.b": (ast_edits.ERROR, "a.b is evil.")})
 
 
 class RenameKeywordSpec(NoUpdateSpec):
@@ -172,6 +181,15 @@ class TestAstEdits(test_util.TensorFlowTestCase):
                                      "test_out.py", out_file))
     return (count, report, errors), out_file.getvalue()
 
+  def testModuleDeprecation(self):
+    text = "a.b.c(a.b.x)"
+    (_, _, errors), new_text = self._upgrade(ModuleDeprecationSpec(), text)
+    self.assertEqual(text, new_text)
+    self.assertIn("Using member a.b.c", errors[0])
+    self.assertIn("1:0", errors[0])
+    self.assertIn("Using member a.b.c", errors[0])
+    self.assertIn("1:6", errors[1])
+
   def testNoTransformIfNothingIsSupplied(self):
     text = "f(a, b, kw1=c, kw2=d)\n"
     _, new_text = self._upgrade(NoUpdateSpec(), text)
diff --git a/tensorflow/tools/compatibility/tf_upgrade.py b/tensorflow/tools/compatibility/tf_upgrade.py
index 8d16c679b5..5dd548c821 100644
--- a/tensorflow/tools/compatibility/tf_upgrade.py
+++ b/tensorflow/tools/compatibility/tf_upgrade.py
@@ -184,6 +184,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "you need to inspect this usage manually.\n"),
     }
 
+    self.module_deprecations = {}
+
 
 if __name__ == "__main__":
   parser = argparse.ArgumentParser(
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 002a924f10..74d29730b6 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -695,6 +695,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.compat.v1.debugging.assert_rank_in",
         "tf.assert_rank":
             "tf.compat.v1.assert_rank",
+        "tf.contrib.framework.argsort":
+            "tf.argsort",
     }
     # pylint: enable=line-too-long
 
@@ -813,6 +815,14 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     # positional arguments yourself, this could do the wrong thing.
     self.function_reorders = reorders_v2.reorders
 
+    contrib_warning = (
+        ast_edits.ERROR,
+        "<function name> cannot be converted automatically. tf.contrib will not"
+        " be distributed with TensorFlow 2.0, please consider an alternative in"
+        " non-contrib TensorFlow, a community-maintained repository, or fork "
+        "the required code."
+    )
+
     decay_function_comment = (
         ast_edits.INFO,
         "<function name> has been changed to return a callable instead "
@@ -860,27 +870,27 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "compat.v1 for backward compatibility. Please update these calls to "
         "the TF 2.0 versions.")
 
-# This could be done with a _rename_if_arg_not_found_transformer
+    # This could be done with a _rename_if_arg_not_found_transformer
     deprecate_partition_strategy_comment = (
         ast_edits.WARNING,
         "`partition_strategy` has been removed from <function name>. "
         " The 'div' strategy will be used by default.")
 
-# TODO(b/118888586): add default value change to update script.
+    # TODO(b/118888586): add default value change to update script.
     default_loss_reduction_changed = (
         ast_edits.WARNING,
         "default value of loss_reduction has been changed to "
         "SUM_OVER_BATCH_SIZE.\n"
     )
 
-# make change instead
+    # make change instead
     uniform_unit_scaling_initializer_comment = (
         ast_edits.ERROR,
         "uniform_unit_scaling_initializer has been removed. Please use"
         " tf.initializers.variance_scaling instead with distribution=uniform "
         "to get equivalent behaviour.")
 
-# Make change instead (issue warning about strip_...)
+    # Make change instead (issue warning about strip_...)
     export_saved_model_renamed = (
         ast_edits.ERROR,
         "(Manual edit required) Please rename the method export_savedmodel() "
@@ -1313,6 +1323,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             arg_value_ast=ast.Str("NHWC")),
     }
 
+    self.module_deprecations = {
+        "tf.contrib": contrib_warning,
+    }
+
 
 def _is_ast_str(node):
   """Determine whether this node represents a string."""
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 7b15976b59..440c1635e2 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -1118,6 +1118,13 @@ def _log_prob(self, x):
     _, _, _, new_text = self._upgrade(text)
     self.assertEqual(expected, new_text)
 
+  def test_contrib_framework_argsort(self):
+    text = "tf.contrib.framework.argsort"
+    expected = "tf.argsort"
+    # pylint: enable=line-too-long
+    _, _, _, new_text = self._upgrade(text)
+    self.assertEqual(expected, new_text)
+
 
 class TestUpgradeFiles(test_util.TensorFlowTestCase):
 
-- 
GitLab


From 5ca8a9986b5d33ca3b0c023e8fa9dd214ebddb7f Mon Sep 17 00:00:00 2001
From: hari <hari@etacompute.com>
Date: Wed, 30 Jan 2019 13:53:18 -0800
Subject: [PATCH 1618/2345] moved definition of DebugLog here from _main

---
 tensorflow/lite/experimental/micro/ecm3531/debug_log.cc | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 tensorflow/lite/experimental/micro/ecm3531/debug_log.cc

diff --git a/tensorflow/lite/experimental/micro/ecm3531/debug_log.cc b/tensorflow/lite/experimental/micro/ecm3531/debug_log.cc
new file mode 100644
index 0000000000..8a3efac838
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/ecm3531/debug_log.cc
@@ -0,0 +1,7 @@
+#include "tensorflow/lite/experimental/micro/debug_log.h"
+
+#include "eta_csp_io.h"
+
+extern "C" void DebugLog(const char* s) {
+ EtaCspIoPrintf("%s", s);
+}
-- 
GitLab


From 0b92bc4b44e795ec3aa6b66295abec0770e9063f Mon Sep 17 00:00:00 2001
From: Lukas Geiger <lukas.geiger94@gmail.com>
Date: Wed, 30 Jan 2019 23:10:35 +0100
Subject: [PATCH 1619/2345] Bazel: Remove deprecated
 experimental_shortened_obj_file_path option

`experimental_shortened_obj_file_path` is depreacted since [version `0.17.0`](https://blog.bazel.build/2018/09/14/bazel-0.17.html) and activated by default. Since Tensorflow requires [at least bazel `0.19.0`](https://github.com/tensorflow/tensorflow/blob/master/configure.py#L1559), this flag can be savely removed.
---
 configure.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/configure.py b/configure.py
index 8dcd318220..e626082ce0 100644
--- a/configure.py
+++ b/configure.py
@@ -1513,10 +1513,6 @@ def set_windows_build_flags(environ_cp):
   # The host and target platforms are the same in Windows build. So we don't
   # have to distinct them. This avoids building the same targets twice.
   write_to_bazelrc('build --distinct_host_configuration=false')
-  # Enable short object file path to avoid long path issue on Windows.
-  # TODO(pcloudy): Remove this flag when upgrading Bazel to 0.16.0
-  # Short object file path will be enabled by default.
-  write_to_bazelrc('build --experimental_shortened_obj_file_path=true')
 
   if get_var(
       environ_cp, 'TF_OVERRIDE_EIGEN_STRONG_INLINE', 'Eigen strong inline',
-- 
GitLab


From 1e38a0025c9a983bf3229299109b5b1781215c7e Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Wed, 30 Jan 2019 13:55:22 -0800
Subject: [PATCH 1620/2345] [XLA] CHECK that sparse indices are in range in
 MutableLiteralBase::AppendSparseElement.

Previously there was no range-checking on sparse elements' indices.

PiperOrigin-RevId: 231662246
---
 tensorflow/compiler/xla/literal.h | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/literal.h b/tensorflow/compiler/xla/literal.h
index 368e460ff5..c418be895d 100644
--- a/tensorflow/compiler/xla/literal.h
+++ b/tensorflow/compiler/xla/literal.h
@@ -958,12 +958,15 @@ template <typename NativeT>
 void MutableLiteralBase::AppendSparseElement(
     absl::Span<const int64> multi_index, NativeT value,
     const ShapeIndex& shape_index) {
-  // TODO(jlebar): CHECK that multi_index is in range?
   Piece& p = piece(shape_index);
   const Shape& subshape = p.subshape();
   CHECK(LayoutUtil::IsSparseArray(subshape));
   int64 rank = subshape.rank();
   CHECK_EQ(multi_index.size(), rank);
+  for (int64 i = 0; i < rank; ++i) {
+    CHECK_GE(multi_index[i], 0);
+    CHECK_LT(multi_index[i], subshape.dimensions(i));
+  }
   int64 last_element = p.sparse_indices()->index_count();
   CHECK_LT(last_element, LayoutUtil::MaxSparseElements(subshape.layout()));
   p.sparse_indices()->Append(multi_index);
-- 
GitLab


From 91ba16a62a0845b164791162ba294a1782f5dcad Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Wed, 30 Jan 2019 14:27:17 -0800
Subject: [PATCH 1621/2345] Add the test base for tf.data C++ kernels

---
 tensorflow/core/kernels/data/BUILD            |  54 +++++
 .../core/kernels/data/dataset_test_base.cc    | 196 ++++++++++++++++++
 .../core/kernels/data/dataset_test_base.h     | 180 ++++++++++++++++
 3 files changed, 430 insertions(+)
 create mode 100644 tensorflow/core/kernels/data/dataset_test_base.cc
 create mode 100644 tensorflow/core/kernels/data/dataset_test_base.h

diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD
index 3cadb55a21..9833f38768 100644
--- a/tensorflow/core/kernels/data/BUILD
+++ b/tensorflow/core/kernels/data/BUILD
@@ -21,6 +21,26 @@ cc_library(
     deps = ["//tensorflow/core:framework"],
 )
 
+cc_library(
+    name = "dataset_test_base",
+    testonly = 1,
+    srcs = ["dataset_test_base.cc"],
+    hdrs = ["dataset_test_base.h"],
+    deps = [
+        ":iterator_ops",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:core_cpu_internal",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:tensor_testutil",
+        "//tensorflow/core:test",
+        "//tensorflow/core:testlib",
+        "//tensorflow/core/kernels:ops_testutil",
+    ],
+)
+
 cc_library(
     name = "dataset_utils",
     srcs = ["dataset_utils.cc"],
@@ -184,6 +204,26 @@ tf_kernel_library(
     ],
 )
 
+tf_cc_test(
+    name = "map_dataset_op_test",
+    size = "small",
+    srcs = ["map_dataset_op_test.cc"],
+    deps = [
+        ":map_dataset_op",
+        ":range_dataset_op",
+        ":dataset_test_base",
+        ":iterator_ops",
+        "//tensorflow/core:testlib",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:core_cpu_internal",
+        "//tensorflow/core:dataset_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core/kernels:function_ops",
+        "//tensorflow/core/kernels:cwise_op",
+    ],
+)
+
 cc_library(
     name = "parallel_map_iterator",
     srcs = ["parallel_map_iterator.cc"],
@@ -350,6 +390,20 @@ tf_kernel_library(
     ],
 )
 
+tf_cc_test(
+    name = "range_dataset_op_test",
+    size = "small",
+    srcs = ["range_dataset_op_test.cc"],
+    deps = [
+        ":dataset_test_base",
+        ":range_dataset_op",
+        ":iterator_ops",
+        "//tensorflow/core:testlib",
+        "//tensorflow/core:test_main",
+
+    ],
+)
+
 tf_kernel_library(
     name = "shuffle_dataset_op",
     srcs = ["shuffle_dataset_op.cc"],
diff --git a/tensorflow/core/kernels/data/dataset_test_base.cc b/tensorflow/core/kernels/data/dataset_test_base.cc
new file mode 100644
index 0000000000..2212c73367
--- /dev/null
+++ b/tensorflow/core/kernels/data/dataset_test_base.cc
@@ -0,0 +1,196 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/kernels/data/dataset_test_base.h"
+
+namespace tensorflow {
+namespace data {
+
+Status DatasetOpsTestBase::CreateOpKernel(
+    const NodeDef& node_def, std::unique_ptr<OpKernel>* op_kernel) {
+  Status status;
+  *op_kernel =
+      tensorflow::CreateOpKernel(device_type_, device_.get(), allocator_,
+                                 node_def, TF_GRAPH_DEF_VERSION, &status);
+  return status;
+}
+
+Status DatasetOpsTestBase::CreateDataset(OpKernel* kernel,
+                                         OpKernelContext* context,
+                                         DatasetBase** const dataset) {
+  TF_RETURN_IF_ERROR(RunOpKernel(kernel, context));
+  // Assume that DatasetOp has only one output.
+  DCHECK(context->num_outputs() == 1);
+  TF_RETURN_IF_ERROR(GetDatasetFromContext(context, 0, dataset));
+  return Status::OK();
+}
+
+Status DatasetOpsTestBase::CreateIteratorContext(
+    OpKernelContext* const op_context,
+    std::unique_ptr<IteratorContext>* iterator_context) {
+  IteratorContext::Params params(op_context);
+  function_handle_cache_ = absl::make_unique<FunctionHandleCache>(flr_);
+  params.function_handle_cache = function_handle_cache_.get();
+  *iterator_context = absl::make_unique<IteratorContext>(params);
+  return Status::OK();
+}
+
+Status DatasetOpsTestBase::GetDatasetFromContext(OpKernelContext* context,
+                                                 int output_index,
+                                                 DatasetBase** const dataset) {
+  Tensor* output = context->mutable_output(output_index);
+  Status status = GetDatasetFromVariantTensor(*output, dataset);
+  (*dataset)->Ref();
+  return status;
+}
+
+Status DatasetOpsTestBase::InitThreadPool(int thread_num) {
+  if (thread_num < 1) {
+    return errors::InvalidArgument(
+        "The `thread_num` argument should be positive but got: ", thread_num);
+  }
+  thread_pool_ = absl::make_unique<thread::ThreadPool>(
+      Env::Default(), ThreadOptions(), "inter_op", thread_num);
+  return Status::OK();
+}
+
+Status DatasetOpsTestBase::InitFunctionLibraryRuntime(
+    const std::vector<FunctionDef>& flib, int cpu_num) {
+  if (cpu_num < 1) {
+    return errors::InvalidArgument(
+        "The `cpu_num` argument should be positive but got: ", cpu_num);
+  }
+  SessionOptions options;
+  auto* device_count = options.config.mutable_device_count();
+  device_count->insert({"CPU", cpu_num});
+  std::vector<std::unique_ptr<Device>> devices;
+  TF_RETURN_IF_ERROR(DeviceFactory::AddDevices(
+      options, "/job:localhost/replica:0/task:0", &devices));
+  device_mgr_ = absl::make_unique<DeviceMgr>(std::move(devices));
+
+  FunctionDefLibrary proto;
+  for (const auto& fdef : flib) *(proto.add_function()) = fdef;
+  lib_def_ =
+      absl::make_unique<FunctionLibraryDefinition>(OpRegistry::Global(), proto);
+
+  OptimizerOptions opts;
+  pflr_ = absl::make_unique<ProcessFunctionLibraryRuntime>(
+      device_mgr_.get(), Env::Default(), TF_GRAPH_DEF_VERSION, lib_def_.get(),
+      opts, thread_pool_.get(), nullptr /* cluster_flr */);
+  flr_ = pflr_->GetFLR("/job:localhost/replica:0/task:0/cpu:0");
+  if (thread_pool_ == nullptr) {
+    runner_ = [](std::function<void()> fn) { fn(); };
+  } else {
+    runner_ = [this](std::function<void()> fn) {
+      thread_pool_->Schedule(std::move(fn));
+    };
+  }
+  return Status::OK();
+}
+
+Status DatasetOpsTestBase::RunOpKernel(OpKernel* op_kernel,
+                                       OpKernelContext* context) {
+  device_->Compute(op_kernel, context);
+  return context->status();
+}
+
+Status DatasetOpsTestBase::CreateOpKernelContext(
+    OpKernel* kernel, gtl::InlinedVector<TensorValue, 4>* inputs,
+    std::unique_ptr<OpKernelContext>* context) {
+  params_ = absl::make_unique<OpKernelContext::Params>();
+  params_->device = device_.get();
+  params_->resource_manager = device_->resource_manager();
+  params_->frame_iter = FrameAndIter(0, 0);
+  params_->inputs = inputs;
+  params_->op_kernel = kernel;
+  params_->function_library = flr_;
+  params_->runner = &runner_;
+  step_container_ =
+      absl::make_unique<ScopedStepContainer>(0, [](const string&) {});
+  params_->step_container = step_container_.get();
+  checkpoint::TensorSliceReaderCacheWrapper slice_reader_cache_wrapper;
+  slice_reader_cache_ =
+      absl::make_unique<checkpoint::TensorSliceReaderCacheWrapper>();
+  params_->slice_reader_cache = slice_reader_cache_.get();
+
+  // Set the allocator attributes for the outputs.
+  allocator_attrs_.clear();
+  for (int index = 0; index < params_->op_kernel->num_outputs(); index++) {
+    AllocatorAttributes attr;
+    const bool on_host =
+        (params_->op_kernel->output_memory_types()[index] == HOST_MEMORY);
+    attr.set_on_host(on_host);
+    allocator_attrs_.emplace_back(attr);
+  }
+  params_->output_attr_array = gtl::vector_as_array(&allocator_attrs_);
+
+  *context = absl::make_unique<OpKernelContext>(params_.get());
+  return Status::OK();
+}
+
+Status DatasetOpsTestBase::CreateSerializationContext(
+    std::unique_ptr<SerializationContext>* context) {
+  SerializationContext::Params params;
+  params.flib_def = lib_def_.get();
+  *context = absl::make_unique<SerializationContext>(params);
+  return Status::OK();
+}
+
+Status DatasetOpsTestBase::CheckOpKernelInput(
+    const OpKernel& kernel, const gtl::InlinedVector<TensorValue, 4>& inputs) {
+  if (kernel.input_types().size() != inputs.size()) {
+    return errors::Internal("The number of input elements should be ",
+                            kernel.input_types().size(),
+                            ", but got: ", inputs.size());
+  }
+  return Status::OK();
+}
+
+Status DatasetOpsTestBase::AddDatasetInput(
+    gtl::InlinedVector<TensorValue, 4>* inputs, DataTypeVector input_types,
+    DataType dtype, const TensorShape& shape) {
+  if (input_types.size() < inputs->size()) {
+    return errors::InvalidArgument("Adding more inputs than types: ",
+                                   inputs->size(), " vs. ", input_types.size());
+  }
+  bool is_ref = IsRefType(input_types[inputs->size()]);
+  std::unique_ptr<Tensor> input =
+      absl::make_unique<Tensor>(allocator_, dtype, shape);
+
+  if (is_ref) {
+    DataType expected_dtype = RemoveRefType(input_types[inputs->size()]);
+    if (expected_dtype != dtype) {
+      return errors::InvalidArgument("The input data type is ", dtype,
+                                     " , but expected: ", expected_dtype);
+    }
+    inputs->push_back({&lock_for_refs_, input.get()});
+  } else {
+    if (input_types[inputs->size()] != dtype) {
+      return errors::InvalidArgument(
+          "The input data type is ", dtype,
+          " , but expected: ", input_types[inputs->size()]);
+    }
+    inputs->push_back({nullptr, input.get()});
+  }
+
+  // TODO: figure out how to avoid using a member variable to garbage collect
+  // the inputs.
+  tensors_.push_back(std::move(input));
+
+  return Status::OK();
+}
+
+}  // namespace data
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/data/dataset_test_base.h b/tensorflow/core/kernels/data/dataset_test_base.h
new file mode 100644
index 0000000000..e9a352a725
--- /dev/null
+++ b/tensorflow/core/kernels/data/dataset_test_base.h
@@ -0,0 +1,180 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_KERNELS_DATA_DATASET_TEST_BASE_H
+#define TENSORFLOW_CORE_KERNELS_DATA_DATASET_TEST_BASE_H
+
+#include <vector>
+
+#include "tensorflow/core/framework/dataset.h"
+#include "tensorflow/core/framework/function.h"
+#include "tensorflow/core/framework/function_handle_cache.h"
+#include "tensorflow/core/framework/function_testlib.h"
+#include "tensorflow/core/framework/node_def_builder.h"
+#include "tensorflow/core/framework/partial_tensor_shape.h"
+#include "tensorflow/core/framework/variant.h"
+#include "tensorflow/core/framework/variant_tensor_data.h"
+#include "tensorflow/core/kernels/data/dataset_utils.h"
+#include "tensorflow/core/kernels/data/iterator_ops.h"
+#include "tensorflow/core/kernels/ops_testutil.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/platform/types.h"
+#include "tensorflow/core/util/ptr_util.h"
+
+namespace tensorflow {
+namespace data {
+
+// Helpful functions to test Dataset op kernels.
+class DatasetOpsTestBase : public ::testing::Test {
+ public:
+  DatasetOpsTestBase()
+      : device_(DeviceFactory::NewDevice("CPU", {}, "/job:a/replica:0/task:0")),
+        device_type_(DEVICE_CPU) {
+    allocator_ = device_->GetAllocator(AllocatorAttributes());
+  }
+
+  ~DatasetOpsTestBase() {}
+
+  // Creates a new op kernel based on the node definition.
+  Status CreateOpKernel(const NodeDef& node_def,
+                        std::unique_ptr<OpKernel>* op_kernel);
+
+  // Creates a new dataset.
+  Status CreateDataset(OpKernel* kernel, OpKernelContext* context,
+                       DatasetBase** const dataset);
+
+  // Creates a new RangeDataset op kernel. `T` specifies the output dtype of the
+  // op kernel.
+  template <typename T>
+  Status CreateRangeDatasetOpKernel(
+      StringPiece node_name, std::unique_ptr<OpKernel>* range_op_kernel) {
+    DataTypeVector dtypes({tensorflow::DataTypeToEnum<T>::value});
+    std::vector<PartialTensorShape> shapes({{}});
+    NodeDef node_def = test::function::NDef(
+        node_name, "RangeDataset", {"start", "stop", "step"},
+        {{"output_types", dtypes}, {"output_shapes", shapes}});
+
+    TF_RETURN_IF_ERROR(CreateOpKernel(node_def, range_op_kernel));
+    return Status::OK();
+  }
+
+  // Creates a new RangeDataset dataset. `T` specifies the output dtype of the
+  // RangeDataset op kernel.
+  template <typename T>
+  Status CreateRangeDataset(int64 start, int64 end, int64 step,
+                            StringPiece node_name,
+                            DatasetBase** range_dataset) {
+    std::unique_ptr<OpKernel> range_kernel;
+    TF_RETURN_IF_ERROR(CreateRangeDatasetOpKernel<T>(node_name, &range_kernel));
+    gtl::InlinedVector<TensorValue, 4> range_inputs;
+    TF_RETURN_IF_ERROR(AddDatasetInputFromArray<int64>(
+        &range_inputs, range_kernel->input_types(), TensorShape({}), {start}));
+    TF_RETURN_IF_ERROR(AddDatasetInputFromArray<int64>(
+        &range_inputs, range_kernel->input_types(), TensorShape({}), {end}));
+    TF_RETURN_IF_ERROR(AddDatasetInputFromArray<int64>(
+        &range_inputs, range_kernel->input_types(), TensorShape({}), {step}));
+    std::unique_ptr<OpKernelContext> range_context;
+    TF_RETURN_IF_ERROR(CreateOpKernelContext(range_kernel.get(), &range_inputs,
+                                             &range_context));
+    TF_RETURN_IF_ERROR(CheckOpKernelInput(*range_kernel, range_inputs));
+    TF_RETURN_IF_ERROR(RunOpKernel(range_kernel.get(), range_context.get()));
+    TF_RETURN_IF_ERROR(
+        GetDatasetFromContext(range_context.get(), 0, range_dataset));
+    return Status::OK();
+  }
+
+  // Fetches the dataset from the operation context.
+  Status GetDatasetFromContext(OpKernelContext* context, int output_index,
+                               DatasetBase** const dataset);
+
+ protected:
+  // Creates a thread pool for parallel tasks.
+  Status InitThreadPool(int thread_num);
+
+  // Initializes the runtime for computing the dataset operation and registers
+  // the input function definitions. `InitThreadPool()' needs to be called
+  // before this method if we want to run the tasks in parallel.
+  Status InitFunctionLibraryRuntime(const std::vector<FunctionDef>& flib,
+                                    int cpu_num);
+
+  // Runs an operation producing outputs.
+  Status RunOpKernel(OpKernel* op_kernel, OpKernelContext* context);
+
+  // Checks that the size of `inputs` matches the requirement of the op kernel.
+  Status CheckOpKernelInput(const OpKernel& kernel,
+                            const gtl::InlinedVector<TensorValue, 4>& inputs);
+
+  // Creates a new context for running the dataset operation.
+  Status CreateOpKernelContext(OpKernel* kernel,
+                               gtl::InlinedVector<TensorValue, 4>* inputs,
+                               std::unique_ptr<OpKernelContext>* context);
+
+  // Creates a new iterator context for iterating the dataset.
+  Status CreateIteratorContext(
+      OpKernelContext* const op_context,
+      std::unique_ptr<IteratorContext>* iterator_context);
+
+  // Creates a new serialization context for serializing the dataset and
+  // iterator.
+  Status CreateSerializationContext(
+      std::unique_ptr<SerializationContext>* context);
+
+  // Adds an arrayslice of data into the input vector. `input_types` describes
+  // the required data type for each input tensor. `shape` and `data` describes
+  // the shape and values of the current input tensor. `T` specifies the dtype
+  // of the input data.
+  template <typename T>
+  Status AddDatasetInputFromArray(gtl::InlinedVector<TensorValue, 4>* inputs,
+                                  DataTypeVector input_types,
+                                  const TensorShape& shape,
+                                  const gtl::ArraySlice<T>& data) {
+    TF_RETURN_IF_ERROR(
+        AddDatasetInput(inputs, input_types, DataTypeToEnum<T>::v(), shape));
+    test::FillValues<T>(inputs->back().tensor, data);
+    return Status::OK();
+  }
+
+ private:
+  // Adds an empty tensor with the specified dtype and shape to the input
+  // vector.
+  Status AddDatasetInput(gtl::InlinedVector<TensorValue, 4>* inputs,
+                         DataTypeVector input_types, DataType dtype,
+                         const TensorShape& shape);
+
+ protected:
+  std::unique_ptr<Device> device_;
+  DeviceType device_type_;
+  Allocator* allocator_;  // Owned by `AllocatorFactoryRegistry`.
+  std::vector<AllocatorAttributes> allocator_attrs_;
+  std::unique_ptr<ScopedStepContainer> step_container_;
+
+  std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_;
+  FunctionLibraryRuntime* flr_;  // Owned by `pflr_`.
+  std::unique_ptr<FunctionHandleCache> function_handle_cache_;
+  std::function<void(std::function<void()>)> runner_;
+  std::unique_ptr<DeviceMgr> device_mgr_;
+  std::unique_ptr<FunctionLibraryDefinition> lib_def_;
+  std::unique_ptr<OpKernelContext::Params> params_;
+  std::unique_ptr<checkpoint::TensorSliceReaderCacheWrapper>
+      slice_reader_cache_;
+  std::unique_ptr<thread::ThreadPool> thread_pool_;
+  std::vector<std::unique_ptr<Tensor>> tensors_;  // Owns tensors.
+  mutex lock_for_refs_;  // Used as the Mutex for inputs added as refs.
+};
+
+}  // namespace data
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_KERNELS_DATA_DATASET_TEST_BASE_H
-- 
GitLab


From 79ee4c56fcb080c4d6189d6ddf6cad88f26b22d5 Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Wed, 30 Jan 2019 14:27:53 -0800
Subject: [PATCH 1622/2345] Add the test for RangeDataset kernel

---
 .../kernels/data/range_dataset_op_test.cc     | 425 ++++++++++++++++++
 1 file changed, 425 insertions(+)
 create mode 100644 tensorflow/core/kernels/data/range_dataset_op_test.cc

diff --git a/tensorflow/core/kernels/data/range_dataset_op_test.cc b/tensorflow/core/kernels/data/range_dataset_op_test.cc
new file mode 100644
index 0000000000..45a9cff623
--- /dev/null
+++ b/tensorflow/core/kernels/data/range_dataset_op_test.cc
@@ -0,0 +1,425 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/framework/dataset.h"
+#include "tensorflow/core/framework/fake_input.h"
+#include "tensorflow/core/framework/function_testlib.h"
+#include "tensorflow/core/framework/node_def_builder.h"
+#include "tensorflow/core/framework/partial_tensor_shape.h"
+#include "tensorflow/core/framework/variant.h"
+#include "tensorflow/core/framework/variant_tensor_data.h"
+#include "tensorflow/core/kernels/data/dataset_test_base.h"
+#include "tensorflow/core/kernels/data/dataset_utils.h"
+#include "tensorflow/core/kernels/data/iterator_ops.h"
+#include "tensorflow/core/kernels/ops_testutil.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/util/ptr_util.h"
+
+namespace tensorflow {
+namespace data {
+namespace {
+
+class RangeDatasetOpTest : public DatasetOpsTestBase {
+ public:
+  static const char* kNodeName;
+  static const char* kOpName;
+
+ protected:
+  // Creates a new RangeDataset op kernel context.
+  Status CreateRangeDatasetContext(
+      int64 start, int64 end, int64 step, OpKernel* const range_kernel,
+      std::unique_ptr<OpKernelContext>* range_context) {
+    inputs_.clear();
+    TF_RETURN_IF_ERROR(AddDatasetInputFromArray<int64>(
+        &inputs_, range_kernel->input_types(), TensorShape({}), {start}));
+    TF_RETURN_IF_ERROR(AddDatasetInputFromArray<int64>(
+        &inputs_, range_kernel->input_types(), TensorShape({}), {end}));
+    TF_RETURN_IF_ERROR(AddDatasetInputFromArray<int64>(
+        &inputs_, range_kernel->input_types(), TensorShape({}), {step}));
+
+    TF_RETURN_IF_ERROR(
+        CreateOpKernelContext(range_kernel, &inputs_, range_context));
+    TF_RETURN_IF_ERROR(CheckOpKernelInput(*range_kernel, inputs_));
+    return Status::OK();
+  }
+
+ private:
+  gtl::InlinedVector<TensorValue, 4> inputs_;
+};
+
+const char* RangeDatasetOpTest::kNodeName = "range_dataset";
+const char* RangeDatasetOpTest::kOpName = "RangeDataset";
+
+struct GetNextTestParams {
+  explicit GetNextTestParams(int64 input_start, int64 input_end,
+                             int64 input_step)
+      : start(input_start), end(input_end), step(input_step) {}
+
+  int64 start;
+  int64 end;
+  int64 step;
+};
+
+struct DatasetGetNextTest : RangeDatasetOpTest,
+                            ::testing::WithParamInterface<GetNextTestParams> {};
+
+TEST_P(DatasetGetNextTest, GetNext) {
+  int thread_num = 2, cpu_num = 2;
+  GetNextTestParams params = GetParam();
+
+  TF_ASSERT_OK(InitThreadPool(thread_num));
+  TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num));
+
+  std::unique_ptr<OpKernel> range_kernel;
+  TF_ASSERT_OK(CreateRangeDatasetOpKernel<int64>("range", &range_kernel));
+  std::unique_ptr<OpKernelContext> range_context;
+  TF_ASSERT_OK(CreateRangeDatasetContext(params.start, params.end, params.step,
+                                         range_kernel.get(), &range_context));
+  DatasetBase* range_dataset;
+  TF_ASSERT_OK(
+      CreateDataset(range_kernel.get(), range_context.get(), &range_dataset));
+  core::ScopedUnref scored_unref(range_dataset);
+
+  std::unique_ptr<IteratorContext> iterator_context;
+  TF_ASSERT_OK(CreateIteratorContext(range_context.get(), &iterator_context));
+  std::unique_ptr<IteratorBase> iterator;
+  TF_ASSERT_OK(range_dataset->MakeIterator(iterator_context.get(), "Iterator",
+                                           &iterator));
+
+  bool end_of_sequence = false;
+  std::vector<Tensor> out_tensors;
+  while (!end_of_sequence) {
+    TF_EXPECT_OK(iterator->GetNext(iterator_context.get(), &out_tensors,
+                                   &end_of_sequence));
+  }
+  std::vector<int> expected_values;
+  for (int i = params.start; (params.end - i) * params.step > 0;
+       i = i + params.step) {
+    expected_values.reserve(1);
+    expected_values.emplace_back(i);
+  }
+  EXPECT_EQ(out_tensors.size(), expected_values.size());
+  for (size_t i = 0; i < out_tensors.size(); ++i) {
+    int64 actual_value = out_tensors[i].flat<int64>()(0);
+    int64 expect_value = expected_values[i];
+    EXPECT_EQ(actual_value, expect_value);
+  }
+}
+
+INSTANTIATE_TEST_CASE_P(RangeDatasetOpTest, DatasetGetNextTest,
+                        ::testing::Values(GetNextTestParams(0, 10, 1),
+                                          GetNextTestParams(0, 10, 3),
+                                          GetNextTestParams(10, 0, -1),
+                                          GetNextTestParams(10, 0, -3)));
+
+TEST_F(RangeDatasetOpTest, DatasetName) {
+  int64 start = 0, end = 10, step = 1;
+  int thread_num = 2, cpu_num = 2;
+
+  TF_ASSERT_OK(InitThreadPool(thread_num));
+  TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num));
+
+  std::unique_ptr<OpKernel> range_kernel;
+  TF_ASSERT_OK(CreateRangeDatasetOpKernel<int64>("range", &range_kernel));
+  std::unique_ptr<OpKernelContext> range_context;
+  TF_ASSERT_OK(CreateRangeDatasetContext(start, end, step, range_kernel.get(),
+                                         &range_context));
+  DatasetBase* range_dataset;
+  TF_ASSERT_OK(
+      CreateDataset(range_kernel.get(), range_context.get(), &range_dataset));
+  core::ScopedUnref scored_unref(range_dataset);
+
+  EXPECT_EQ(range_dataset->name(), kOpName);
+}
+
+TEST_F(RangeDatasetOpTest, DatasetOutputDtypes) {
+  int64 start = 0, end = 10, step = 1;
+  int thread_num = 2, cpu_num = 2;
+
+  TF_ASSERT_OK(InitThreadPool(thread_num));
+  TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num));
+
+  std::unique_ptr<OpKernel> range_kernel;
+  TF_ASSERT_OK(CreateRangeDatasetOpKernel<int64>("range", &range_kernel));
+  std::unique_ptr<OpKernelContext> range_context;
+  TF_ASSERT_OK(CreateRangeDatasetContext(start, end, step, range_kernel.get(),
+                                         &range_context));
+  DatasetBase* range_dataset;
+  TF_ASSERT_OK(
+      CreateDataset(range_kernel.get(), range_context.get(), &range_dataset));
+  core::ScopedUnref scored_unref(range_dataset);
+
+  DataTypeVector expected_dtypes({DT_INT64});
+  EXPECT_EQ(range_dataset->output_dtypes(), expected_dtypes);
+}
+
+TEST_F(RangeDatasetOpTest, DatasetOutputShapes) {
+  int64 start = 0, end = 10, step = 1;
+  int thread_num = 2, cpu_num = 2;
+
+  TF_ASSERT_OK(InitThreadPool(thread_num));
+  TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num));
+
+  std::unique_ptr<OpKernel> range_kernel;
+  TF_ASSERT_OK(CreateRangeDatasetOpKernel<int64>("range", &range_kernel));
+  std::unique_ptr<OpKernelContext> range_context;
+  TF_ASSERT_OK(CreateRangeDatasetContext(start, end, step, range_kernel.get(),
+                                         &range_context));
+  DatasetBase* range_dataset;
+  TF_ASSERT_OK(
+      CreateDataset(range_kernel.get(), range_context.get(), &range_dataset));
+  core::ScopedUnref scored_unref(range_dataset);
+
+  std::vector<PartialTensorShape> expected_shapes({{}});
+  EXPECT_EQ(range_dataset->output_shapes().size(), expected_shapes.size());
+  for (int i = 0; i < range_dataset->output_shapes().size(); ++i) {
+    range_dataset->output_shapes()[i].IsIdenticalTo(expected_shapes[i]);
+  }
+}
+
+struct CardinalityTestParams {
+  explicit CardinalityTestParams(int64 input_start, int64 input_end,
+                                 int64 input_step,
+                                 int input_expected_cardinality)
+      : start(input_start),
+        end(input_end),
+        step(input_step),
+        expected_cardinality(input_expected_cardinality) {}
+
+  int64 start;
+  int64 end;
+  int64 step;
+  int expected_cardinality;
+};
+
+struct DatasetCardinalityTest
+    : RangeDatasetOpTest,
+      ::testing::WithParamInterface<CardinalityTestParams> {};
+
+TEST_P(DatasetCardinalityTest, Cardinality) {
+  int thread_num = 2, cpu_num = 2;
+  CardinalityTestParams params = GetParam();
+
+  TF_ASSERT_OK(InitThreadPool(thread_num));
+  TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num));
+
+  std::unique_ptr<OpKernel> range_kernel;
+  TF_ASSERT_OK(CreateRangeDatasetOpKernel<int64>("range", &range_kernel));
+  std::unique_ptr<OpKernelContext> range_context;
+  TF_ASSERT_OK(CreateRangeDatasetContext(params.start, params.end, params.step,
+                                         range_kernel.get(), &range_context));
+  DatasetBase* range_dataset;
+  TF_ASSERT_OK(
+      CreateDataset(range_kernel.get(), range_context.get(), &range_dataset));
+  core::ScopedUnref scored_unref(range_dataset);
+
+  EXPECT_EQ(range_dataset->Cardinality(), params.expected_cardinality);
+}
+
+INSTANTIATE_TEST_CASE_P(RangeDatasetOpTest, DatasetCardinalityTest,
+                        ::testing::Values(CardinalityTestParams(0, 10, 1, 10),
+                                          CardinalityTestParams(0, 10, 3, 4),
+                                          CardinalityTestParams(10, 0, -3, 4)));
+
+TEST_F(RangeDatasetOpTest, DatasetSave) {
+  int64 thread_num = 2, cpu_num = 2;
+  int start = 0, end = 10, step = 1;
+
+  TF_ASSERT_OK(InitThreadPool(thread_num));
+  TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num));
+
+  std::unique_ptr<OpKernel> range_kernel;
+  TF_ASSERT_OK(CreateRangeDatasetOpKernel<int64>("range", &range_kernel));
+  std::unique_ptr<OpKernelContext> range_context;
+  TF_ASSERT_OK(CreateRangeDatasetContext(start, end, step, range_kernel.get(),
+                                         &range_context));
+  DatasetBase* range_dataset;
+  TF_ASSERT_OK(
+      CreateDataset(range_kernel.get(), range_context.get(), &range_dataset));
+  core::ScopedUnref scored_unref(range_dataset);
+
+  std::unique_ptr<SerializationContext> serialization_context;
+  TF_ASSERT_OK(CreateSerializationContext(&serialization_context));
+
+  VariantTensorData data;
+  VariantTensorDataWriter writer(&data);
+  TF_ASSERT_OK(range_dataset->Save(serialization_context.get(), &writer));
+  TF_ASSERT_OK(writer.Flush());
+}
+
+TEST_F(RangeDatasetOpTest, IteratorOutputDtypes) {
+  int64 start = 0, end = 10, step = 1;
+  int thread_num = 2, cpu_num = 2;
+
+  TF_ASSERT_OK(InitThreadPool(thread_num));
+  TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num));
+
+  std::unique_ptr<OpKernel> range_kernel;
+  TF_ASSERT_OK(CreateRangeDatasetOpKernel<int64>("range", &range_kernel));
+  std::unique_ptr<OpKernelContext> range_context;
+  TF_ASSERT_OK(CreateRangeDatasetContext(start, end, step, range_kernel.get(),
+                                         &range_context));
+  DatasetBase* range_dataset;
+  TF_ASSERT_OK(
+      CreateDataset(range_kernel.get(), range_context.get(), &range_dataset));
+  core::ScopedUnref scored_unref(range_dataset);
+
+  std::unique_ptr<IteratorContext> iterator_context;
+  TF_ASSERT_OK(CreateIteratorContext(range_context.get(), &iterator_context));
+  std::unique_ptr<IteratorBase> iterator;
+  TF_ASSERT_OK(range_dataset->MakeIterator(iterator_context.get(), "Iterator",
+                                           &iterator));
+
+  DataTypeVector expected_dtypes({DT_INT64});
+  EXPECT_EQ(iterator->output_dtypes(), expected_dtypes);
+}
+
+TEST_F(RangeDatasetOpTest, IteratorOutputShapes) {
+  int64 start = 0, end = 10, step = 1;
+  int thread_num = 2, cpu_num = 2;
+
+  TF_ASSERT_OK(InitThreadPool(thread_num));
+  TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num));
+
+  std::unique_ptr<OpKernel> range_kernel;
+  TF_ASSERT_OK(CreateRangeDatasetOpKernel<int64>("range", &range_kernel));
+  std::unique_ptr<OpKernelContext> range_context;
+  TF_ASSERT_OK(CreateRangeDatasetContext(start, end, step, range_kernel.get(),
+                                         &range_context));
+  DatasetBase* range_dataset;
+  TF_ASSERT_OK(
+      CreateDataset(range_kernel.get(), range_context.get(), &range_dataset));
+  core::ScopedUnref scored_unref(range_dataset);
+
+  std::unique_ptr<IteratorContext> iterator_context;
+  TF_ASSERT_OK(CreateIteratorContext(range_context.get(), &iterator_context));
+  std::unique_ptr<IteratorBase> iterator;
+  TF_ASSERT_OK(range_dataset->MakeIterator(iterator_context.get(), "Iterator",
+                                           &iterator));
+
+  std::vector<PartialTensorShape> expected_shapes({{}});
+  EXPECT_EQ(iterator->output_shapes().size(), expected_shapes.size());
+  for (int i = 0; i < range_dataset->output_shapes().size(); ++i) {
+    iterator->output_shapes()[i].IsIdenticalTo(expected_shapes[i]);
+  }
+}
+
+TEST_F(RangeDatasetOpTest, IteratorOutputPrefix) {
+  int64 start = 0, end = 10, step = 1;
+  int thread_num = 2, cpu_num = 2;
+
+  TF_ASSERT_OK(InitThreadPool(thread_num));
+  TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num));
+
+  std::unique_ptr<OpKernel> range_kernel;
+  TF_ASSERT_OK(CreateRangeDatasetOpKernel<int64>("range", &range_kernel));
+  std::unique_ptr<OpKernelContext> range_context;
+  TF_ASSERT_OK(CreateRangeDatasetContext(start, end, step, range_kernel.get(),
+                                         &range_context));
+  DatasetBase* range_dataset;
+  TF_ASSERT_OK(
+      CreateDataset(range_kernel.get(), range_context.get(), &range_dataset));
+  core::ScopedUnref scored_unref(range_dataset);
+
+  std::unique_ptr<IteratorContext> iterator_context;
+  TF_ASSERT_OK(CreateIteratorContext(range_context.get(), &iterator_context));
+  std::unique_ptr<IteratorBase> iterator;
+  TF_ASSERT_OK(range_dataset->MakeIterator(iterator_context.get(), "Iterator",
+                                           &iterator));
+
+  EXPECT_EQ(iterator->prefix(), "Iterator::Range");
+}
+
+struct RoundtripTestParams {
+  explicit RoundtripTestParams(int64 input_start, int64 input_end,
+                               int64 input_step, int input_breakpoint)
+      : start(input_start),
+        end(input_end),
+        step(input_step),
+        breakpoint(input_breakpoint) {}
+
+  int64 start;
+  int64 end;
+  int64 step;
+  int breakpoint;
+};
+
+struct IteratorRoundtripTest
+    : RangeDatasetOpTest,
+      ::testing::WithParamInterface<RoundtripTestParams> {};
+
+TEST_P(IteratorRoundtripTest, Roundtrip) {
+  int thread_num = 2, cpu_num = 2;
+  RoundtripTestParams params = GetParam();
+
+  TF_ASSERT_OK(InitThreadPool(thread_num));
+  TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num));
+
+  std::unique_ptr<OpKernel> range_kernel;
+  TF_ASSERT_OK(CreateRangeDatasetOpKernel<int64>("range", &range_kernel));
+  std::unique_ptr<OpKernelContext> range_context;
+  TF_ASSERT_OK(CreateRangeDatasetContext(params.start, params.end, params.step,
+                                         range_kernel.get(), &range_context));
+  DatasetBase* range_dataset;
+  TF_ASSERT_OK(
+      CreateDataset(range_kernel.get(), range_context.get(), &range_dataset));
+  core::ScopedUnref scored_unref(range_dataset);
+
+  std::unique_ptr<IteratorContext> iterator_context;
+  TF_ASSERT_OK(CreateIteratorContext(range_context.get(), &iterator_context));
+  std::unique_ptr<IteratorBase> iterator;
+  TF_ASSERT_OK(range_dataset->MakeIterator(iterator_context.get(), "Iterator",
+                                           &iterator));
+
+  std::vector<Tensor> out_tensors;
+  bool end_of_sequence = false;
+  int64 cur_val = params.start - params.step;
+  for (int i = 0; i < params.breakpoint; i++) {
+    if (!end_of_sequence) {
+      TF_EXPECT_OK(iterator->GetNext(iterator_context.get(), &out_tensors,
+                                     &end_of_sequence));
+      cur_val = ((params.end - cur_val - params.step) * params.step > 0)
+                    ? cur_val + params.step
+                    : cur_val;
+    }
+  }
+
+  std::unique_ptr<SerializationContext> serialization_context;
+  TF_ASSERT_OK(CreateSerializationContext(&serialization_context));
+  VariantTensorData data;
+  VariantTensorDataWriter writer(&data);
+  TF_ASSERT_OK(iterator->Save(serialization_context.get(), &writer));
+  TF_ASSERT_OK(writer.Flush());
+  VariantTensorDataReader reader(&data);
+  TF_ASSERT_OK(iterator->Restore(iterator_context.get(), &reader));
+  TF_EXPECT_OK(iterator->GetNext(iterator_context.get(), &out_tensors,
+                                 &end_of_sequence));
+  int64 expect_next = ((params.end - cur_val - params.step) * params.step > 0)
+                        ? cur_val + params.step
+                        : cur_val;
+  EXPECT_EQ(out_tensors.back().flat<int64>()(0), expect_next);
+}
+
+INSTANTIATE_TEST_CASE_P(
+    RangeDatasetOpTest, IteratorRoundtripTest,
+    ::testing::Values(
+        RoundtripTestParams(0, 10, 2, 0),    // unused_iterator
+        RoundtripTestParams(0, 10, 2, 4),    // fully_used_iterator_increase
+        RoundtripTestParams(10, 0, -2, 4),   // fully_used_iterator_decrease
+        RoundtripTestParams(0, 10, 2, 6)));  // exhausted_iterator
+
+}  // namespace
+}  // namespace data
+}  // namespace tensorflow
-- 
GitLab


From d45a7f34bf32c56edf2731b8d310140432fddbc0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 30 Jan 2019 14:12:56 -0800
Subject: [PATCH 1623/2345] Add tests for fit, predict, evaluate without
 specifying steps for all distribution strategies except TPUStrategy.

PiperOrigin-RevId: 231665756
---
 .../python/keras_backward_compat_test.py      | 19 ++++---
 .../contrib/distribute/python/keras_test.py   | 53 +++++++++++++++++--
 .../keras/engine/training_distributed.py      | 26 +++++++--
 .../python/keras/engine/training_utils.py     |  2 +-
 4 files changed, 83 insertions(+), 17 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_backward_compat_test.py b/tensorflow/contrib/distribute/python/keras_backward_compat_test.py
index b783ab86a1..3a7488025b 100644
--- a/tensorflow/contrib/distribute/python/keras_backward_compat_test.py
+++ b/tensorflow/contrib/distribute/python/keras_backward_compat_test.py
@@ -791,16 +791,21 @@ class TestDistributionStrategyErrorCases(test.TestCase, parameterized.TestCase):
             verbose=0,
             sample_weight=sample_weight)
 
-      # Test with not specifying the `steps` argument.
-      with self.assertRaisesRegexp(
-          ValueError, 'the `steps_per_epoch` argument'):
+      # Test with not specifying the `steps` argument for dataset with
+      # infinite cardinality.
+      dataset = dataset.repeat()
+      with self.assertRaisesRegexp(ValueError, 'When passing an infinitely '
+                                   'repeating dataset, you must specify the '
+                                   '`steps_per_epoch` argument'):
         model.fit(dataset, epochs=1, verbose=0)
-      with self.assertRaisesRegexp(ValueError,
-                                   'the `steps` argument'):
+      with self.assertRaisesRegexp(ValueError, 'When passing an infinitely '
+                                   'repeating dataset, you must specify the '
+                                   '`steps` argument'):
         model.evaluate(dataset, verbose=0)
 
-      with self.assertRaisesRegexp(ValueError,
-                                   'the `steps` argument'):
+      with self.assertRaisesRegexp(ValueError, 'When passing an infinitely '
+                                   'repeating dataset, you must specify the '
+                                   '`steps` argument'):
         model.predict(dataset, verbose=0)
 
   @combinations.generate(combinations.combine(
diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 73c27d680a..19e7d848ad 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -841,6 +841,42 @@ class TestDistributionStrategyWithDatasets(test.TestCase,
 
       model.fit(dataset_dict, epochs=1, steps_per_epoch=2, verbose=1)
 
+  # TODO(b/122743976): Include TPUStrategy for this test as well once
+  # step inference is supported.
+  @combinations.generate(strategy_minus_tpu_combinations())
+  def test_fit_eval_and_predict_methods_on_dataset_without_steps(
+      self, distribution):
+    with self.cached_session():
+      with distribution.scope():
+        model = get_model()
+        optimizer = gradient_descent.GradientDescentOptimizer(0.001)
+        loss = 'mse'
+        metrics = ['mae', keras.metrics.CategoricalAccuracy()]
+        model.compile(optimizer, loss, metrics=metrics)
+
+      inputs = np.zeros((1000, 3), dtype=np.float32)
+      targets = np.zeros((1000, 4), dtype=np.float32)
+      # steps/steps_per_epoch are calculated when using numpy arrays as
+      # input data.
+      fit_with_numpy = model.fit(inputs, targets, epochs=1,
+                                 batch_size=10).history
+      eval_with_numpy = model.evaluate(inputs, targets, batch_size=10)
+      predict_with_numpy = model.predict(inputs, batch_size=10)
+
+      dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
+      dataset = dataset.batch(10)
+      fit_with_ds = model.fit(dataset, epochs=1).history
+      eval_with_ds = model.evaluate(dataset)
+      predict_dataset = dataset_ops.Dataset.from_tensor_slices(inputs)
+      predict_dataset = predict_dataset.batch(10, drop_remainder=True)
+      predict_with_ds = model.predict(predict_dataset)
+      self.assertAllClose(
+          fit_with_numpy, fit_with_ds, atol=1e-4, rtol=1e-4)
+      self.assertAllClose(
+          eval_with_numpy, eval_with_ds, atol=1e-4, rtol=1e-4)
+      self.assertAllClose(
+          predict_with_numpy, predict_with_ds, atol=1e-4, rtol=1e-4)
+
   @combinations.generate(all_strategy_combinations())
   def test_fit_eval_and_predict_methods_on_dataset(self, distribution):
     with self.cached_session():
@@ -1293,14 +1329,21 @@ class TestDistributionStrategyErrorCases(test.TestCase, parameterized.TestCase):
             verbose=0,
             sample_weight=sample_weight)
 
-      # Test with not specifying the `steps` argument.
-      with self.assertRaisesRegexp(
-          ValueError, 'the `steps_per_epoch` argument'):
+      # Test with not specifying the `steps` argument for dataset with infinite
+      # cardinality.
+      dataset = dataset.repeat()
+      with self.assertRaisesRegexp(ValueError, 'When passing an infinitely '
+                                   'repeating dataset, you must specify the '
+                                   '`steps_per_epoch` argument'):
         model.fit(dataset, epochs=1, verbose=0)
-      with self.assertRaisesRegexp(ValueError, 'the `steps` argument'):
+      with self.assertRaisesRegexp(ValueError, 'When passing an infinitely '
+                                   'repeating dataset, you must specify the '
+                                   '`steps` argument'):
         model.evaluate(dataset, verbose=0)
 
-      with self.assertRaisesRegexp(ValueError, 'the `steps` argument'):
+      with self.assertRaisesRegexp(ValueError, 'When passing an infinitely '
+                                   'repeating dataset, you must specify the '
+                                   '`steps` argument'):
         model.predict(dataset, verbose=0)
 
   @combinations.generate(combinations.combine(
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 68160580e6..735bd38438 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -22,6 +22,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.data.experimental.ops import batching
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import input_lib
 from tensorflow.python.distribute import reduce_util as ds_reduce_util
@@ -73,13 +74,17 @@ def fit_distributed(model,
             batch_size, mode=ModeKeys.TRAIN))
   batch_size = model._validate_or_infer_batch_size(
       batch_size, steps_per_epoch, x)
+  steps_name = 'steps_per_epoch'
+  if isinstance(x, dataset_ops.DatasetV2):
+    steps_per_epoch = training_utils.infer_steps_for_dataset(
+        x, steps_per_epoch, steps_name=steps_name)
   dataset = model._distribution_standardize_user_data(
       x, y,
       sample_weight=sample_weight,
       class_weight=class_weight,
       batch_size=batch_size,
       check_steps=True,
-      steps_name='steps_per_epoch',
+      steps_name=steps_name,
       steps=steps_per_epoch,
       validation_split=validation_split,
       shuffle=shuffle)
@@ -95,13 +100,17 @@ def fit_distributed(model,
       validation_steps, _ = distributed_training_utils.get_input_params(
           model._distribution_strategy, first_valx_value, validation_steps,
           batch_size)
+    steps_name = 'validation_steps'
+    if isinstance(val_x, dataset_ops.DatasetV2):
+      validation_steps = training_utils.infer_steps_for_dataset(
+          val_x, validation_steps, steps_name=steps_name)
     val_dataset = model._distribution_standardize_user_data(
         val_x, val_y,
         sample_weight=val_sample_weights,
         class_weight=None,
         batch_size=batch_size,
         check_steps=True,
-        steps_name='validation_steps',
+        steps_name=steps_name,
         steps=validation_steps,
         validation_split=validation_split,
         shuffle=shuffle)
@@ -152,12 +161,17 @@ def evaluate_distributed(model,
     steps, batch_size = distributed_training_utils.get_input_params(
         model._distribution_strategy, first_x_value, steps, batch_size)
   batch_size = model._validate_or_infer_batch_size(batch_size, steps, x)
+  steps_name = 'steps'
+
+  if isinstance(x, dataset_ops.DatasetV2):
+    steps = training_utils.infer_steps_for_dataset(x, steps,
+                                                   steps_name=steps_name)
   dataset = model._distribution_standardize_user_data(
       x, y,
       sample_weight=sample_weight,
       batch_size=batch_size,
       check_steps=True,
-      steps_name='steps',
+      steps_name=steps_name,
       steps=steps)
 
   if distributed_training_utils.is_tpu_strategy(model._distribution_strategy):
@@ -188,11 +202,15 @@ def predict_distributed(model,
         model._distribution_strategy, first_x_value, steps,
         batch_size, mode=ModeKeys.PREDICT)
   batch_size = model._validate_or_infer_batch_size(batch_size, steps, x)
+  steps_name = 'steps'
+  if isinstance(x, dataset_ops.DatasetV2):
+    steps = training_utils.infer_steps_for_dataset(x, steps,
+                                                   steps_name=steps_name)
   dataset = model._distribution_standardize_user_data(
       x,
       batch_size=batch_size,
       check_steps=True,
-      steps_name='steps',
+      steps_name=steps_name,
       steps=steps,
       repeat=False,
       allow_partial_batch=True)
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index 4c913b5d51..895a592bd9 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -1342,7 +1342,7 @@ def infer_steps_for_dataset(dataset, steps, epochs=1, steps_name='steps'):
   if size == cardinality.INFINITE and steps is None:
     raise ValueError('When passing an infinitely repeating dataset, you '
                      'must specify the `%s` argument.' % (steps_name,))
-  if size != cardinality.UNKNOWN:
+  if size >= 0:
     if steps is not None and steps * epochs > size:
       if epochs > 1:
         raise ValueError('The dataset you passed contains %s batches, but you '
-- 
GitLab


From 8e9a6cb36d96debff2ff3af65cca53670e529142 Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Wed, 30 Jan 2019 14:28:32 -0800
Subject: [PATCH 1624/2345] Add the test for MapDataset kernel

---
 .../core/kernels/data/map_dataset_op_test.cc  | 537 ++++++++++++++++++
 1 file changed, 537 insertions(+)
 create mode 100644 tensorflow/core/kernels/data/map_dataset_op_test.cc

diff --git a/tensorflow/core/kernels/data/map_dataset_op_test.cc b/tensorflow/core/kernels/data/map_dataset_op_test.cc
new file mode 100644
index 0000000000..660948884d
--- /dev/null
+++ b/tensorflow/core/kernels/data/map_dataset_op_test.cc
@@ -0,0 +1,537 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/framework/dataset.h"
+#include "tensorflow/core/framework/fake_input.h"
+#include "tensorflow/core/framework/function.h"
+#include "tensorflow/core/framework/function_handle_cache.h"
+#include "tensorflow/core/framework/function_testlib.h"
+#include "tensorflow/core/framework/node_def_builder.h"
+#include "tensorflow/core/framework/partial_tensor_shape.h"
+#include "tensorflow/core/framework/variant.h"
+#include "tensorflow/core/framework/variant_tensor_data.h"
+#include "tensorflow/core/kernels/data/dataset_test_base.h"
+#include "tensorflow/core/kernels/data/dataset_utils.h"
+#include "tensorflow/core/kernels/data/iterator_ops.h"
+#include "tensorflow/core/kernels/ops_testutil.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/util/ptr_util.h"
+
+namespace tensorflow {
+namespace data {
+namespace {
+
+class MapDatasetOpTest : public DatasetOpsTestBase {
+ public:
+  static const char* kNodeName;
+  static const char* kOpName;
+
+ protected:
+  // Creates a new MapDataset op kernel. The `input_dataset` parameter should be
+  // same with the node name of the input dataset for the method
+  // `CreateMapDatasetContext()`. `T` specifies the output dtype of MapDataset.
+  template <typename T>
+  Status CreateMapDatasetOpKernel(const string& input_dataset,
+                                  const string& func_name,
+                                  std::unique_ptr<OpKernel>* map_kernel) {
+    DataTypeVector map_output_dtypes({tensorflow::DataTypeToEnum<T>::value});
+    gtl::ArraySlice<TensorShape> map_output_shapes({{}});
+    FunctionDefHelper::AttrValueWrapper func =
+        FunctionDefHelper::FunctionRef(func_name, {{"T", DT_INT64}});
+
+    map_node_def_ = test::function::NDef(kNodeName, kOpName, {input_dataset},
+                                         {{"f", func},
+                                          {"Targuments", {}},
+                                          {"output_shapes", map_output_shapes},
+                                          {"output_types", map_output_dtypes},
+                                          {"use_inter_op_parallelism", true},
+                                          {"preserve_cardinality", false}});
+    TF_CHECK_OK(CreateOpKernel(map_node_def_, map_kernel));
+    return Status::OK();
+  }
+
+  // Creates a new MapDataset op kernel context.
+  Status CreateMapDatasetContext(
+      DatasetBase* const input_dataset, OpKernel* const map_kernel,
+      std::unique_ptr<OpKernelContext>* map_context) {
+    map_inputs_.clear();
+    // Save the input dataset into a variant tensor as the input of MapDataset.
+    Tensor dataset_tensor(DT_VARIANT, TensorShape({}));
+    TF_RETURN_IF_ERROR(
+        StoreDatasetInVariantTensor(input_dataset, &dataset_tensor));
+    Variant variant = dataset_tensor.scalar<Variant>()();
+    TF_RETURN_IF_ERROR(AddDatasetInputFromArray<Variant>(
+        &map_inputs_, map_kernel->input_types(), TensorShape({}), {variant}));
+    input_dataset->Ref();
+    TF_RETURN_IF_ERROR(
+        CreateOpKernelContext(map_kernel, &map_inputs_, map_context));
+    TF_RETURN_IF_ERROR(CheckOpKernelInput(*map_kernel, map_inputs_));
+    return Status::OK();
+  }
+
+ private:
+  NodeDef map_node_def_;
+  gtl::InlinedVector<TensorValue, 4> map_inputs_;
+};
+
+const char* MapDatasetOpTest::kNodeName = "map_dataset";
+const char* MapDatasetOpTest::kOpName = "MapDataset";
+
+struct GetNextTestParams {
+  explicit GetNextTestParams(int64 input_start, int64 input_end,
+                             int64 input_step, string input_func_name,
+                             std::vector<int64> input_expected_values,
+                             std::vector<FunctionDef> input_func_lib)
+      : start(input_start),
+        end(input_end),
+        step(input_step),
+        func_name(std::move(input_func_name)),
+        expected_values(std::move(input_expected_values)),
+        func_lib(std::move(input_func_lib)) {}
+
+  int64 start;
+  int64 end;
+  int64 step;
+  string func_name;
+  std::vector<int64> expected_values;
+  std::vector<FunctionDef> func_lib;
+};
+
+struct DatasetGetNextTest : MapDatasetOpTest,
+                            ::testing::WithParamInterface<GetNextTestParams> {};
+
+TEST_P(DatasetGetNextTest, GetNext) {
+  int thread_num = 2, cpu_num = 2;
+  GetNextTestParams test_params = GetParam();
+
+  TF_ASSERT_OK(InitThreadPool(thread_num));
+  TF_ASSERT_OK(InitFunctionLibraryRuntime(test_params.func_lib, cpu_num));
+
+  DatasetBase* range_dataset;
+  TF_ASSERT_OK(CreateRangeDataset<int64>(test_params.start, test_params.end,
+                                         test_params.step, "range",
+                                         &range_dataset));
+  core::ScopedUnref scored_unref_range_dataset(range_dataset);
+
+  std::unique_ptr<OpKernel> map_kernel;
+  TF_ASSERT_OK(CreateMapDatasetOpKernel<int64>(
+      range_dataset->name(), test_params.func_name, &map_kernel));
+  std::unique_ptr<OpKernelContext> map_context;
+  TF_ASSERT_OK(
+      CreateMapDatasetContext(range_dataset, map_kernel.get(), &map_context));
+  DatasetBase* map_dataset;
+  TF_ASSERT_OK(
+      CreateDataset(map_kernel.get(), map_context.get(), &map_dataset));
+  core::ScopedUnref scored_unref_map_dataset(map_dataset);
+
+  std::unique_ptr<IteratorContext> iterator_context;
+  TF_ASSERT_OK(CreateIteratorContext(map_context.get(), &iterator_context));
+  std::unique_ptr<IteratorBase> iterator;
+  TF_ASSERT_OK(
+      map_dataset->MakeIterator(iterator_context.get(), "Iterator", &iterator));
+  bool end_of_sequence = false;
+  std::vector<Tensor> out_tensors;
+  while (!end_of_sequence) {
+    TF_EXPECT_OK(iterator->GetNext(iterator_context.get(), &out_tensors,
+                                   &end_of_sequence));
+  }
+
+  EXPECT_EQ(out_tensors.size(), test_params.expected_values.size());
+  for (size_t i = 0; i < out_tensors.size(); ++i) {
+    int64 actual_value = out_tensors[i].flat<int64>()(0);
+    int64 expect_value = test_params.expected_values[i];
+    EXPECT_EQ(actual_value, expect_value);
+  }
+}
+
+INSTANTIATE_TEST_CASE_P(
+    MapDatasetOpTest, DatasetGetNextTest,
+    ::testing::Values(
+        GetNextTestParams(
+            0, 10, 3, "XTimesTwo", std::vector<int64>{0, 6, 12, 18},
+            std::vector<FunctionDef>{test::function::XTimesTwo()}),
+        GetNextTestParams(0, 10, 3, "XAddX", std::vector<int64>{0, 6, 12, 18},
+                          std::vector<FunctionDef>{test::function::XAddX()}),
+        GetNextTestParams(
+            10, 0, -3, "XTimesFour", std::vector<int64>{40, 28, 16, 4},
+            std::vector<FunctionDef>{test::function::XTimesTwo(),
+                                     test::function::XTimesFour()})));
+
+TEST_F(MapDatasetOpTest, DatasetName) {
+  int thread_num = 2, cpu_num = 2;
+  int64 start = 0, end = 10, step = 1;
+  FunctionDef func_def = test::function::XTimesTwo();
+
+  TF_ASSERT_OK(InitThreadPool(thread_num));
+  TF_ASSERT_OK(InitFunctionLibraryRuntime({func_def}, cpu_num));
+
+  DatasetBase* range_dataset;
+  TF_ASSERT_OK(
+      CreateRangeDataset<int64>(start, end, step, "range", &range_dataset));
+  core::ScopedUnref scored_unref_range_dataset(range_dataset);
+
+  std::unique_ptr<OpKernel> map_kernel;
+  TF_ASSERT_OK(CreateMapDatasetOpKernel<int64>(
+      range_dataset->name(), func_def.signature().name(), &map_kernel));
+  std::unique_ptr<OpKernelContext> map_context;
+  TF_ASSERT_OK(
+      CreateMapDatasetContext(range_dataset, map_kernel.get(), &map_context));
+  DatasetBase* map_dataset;
+  TF_ASSERT_OK(
+      CreateDataset(map_kernel.get(), map_context.get(), &map_dataset));
+  core::ScopedUnref scored_unref_map_dataset(map_dataset);
+
+  EXPECT_EQ(map_dataset->name(), kOpName);
+}
+
+TEST_F(MapDatasetOpTest, DatasetOutputDtypes) {
+  int thread_num = 2, cpu_num = 2;
+  int64 start = 0, end = 10, step = 1;
+  FunctionDef func_def = test::function::XTimesTwo();
+
+  TF_ASSERT_OK(InitThreadPool(thread_num));
+  TF_ASSERT_OK(InitFunctionLibraryRuntime({func_def}, cpu_num));
+
+  DatasetBase* range_dataset;
+  TF_ASSERT_OK(
+      CreateRangeDataset<int64>(start, end, step, "range", &range_dataset));
+  core::ScopedUnref scored_unref_range_dataset(range_dataset);
+
+  std::unique_ptr<OpKernel> map_kernel;
+  TF_ASSERT_OK(CreateMapDatasetOpKernel<int64>(
+      range_dataset->name(), func_def.signature().name(), &map_kernel));
+  std::unique_ptr<OpKernelContext> map_context;
+  TF_ASSERT_OK(
+      CreateMapDatasetContext(range_dataset, map_kernel.get(), &map_context));
+  DatasetBase* map_dataset;
+  TF_ASSERT_OK(
+      CreateDataset(map_kernel.get(), map_context.get(), &map_dataset));
+  core::ScopedUnref scored_unref_map_dataset(map_dataset);
+
+  DataTypeVector expected_dtypes({DT_INT64});
+  EXPECT_EQ(map_dataset->output_dtypes(), expected_dtypes);
+}
+
+TEST_F(MapDatasetOpTest, DatasetOutputShapes) {
+  int thread_num = 2, cpu_num = 2;
+  int64 start = 0, end = 10, step = 1;
+  FunctionDef func_def = test::function::XTimesTwo();
+
+  TF_ASSERT_OK(InitThreadPool(thread_num));
+  TF_ASSERT_OK(InitFunctionLibraryRuntime({func_def}, cpu_num));
+
+  DatasetBase* range_dataset;
+  TF_ASSERT_OK(
+      CreateRangeDataset<int64>(start, end, step, "range", &range_dataset));
+  core::ScopedUnref scored_unref_range_dataset(range_dataset);
+
+  std::unique_ptr<OpKernel> map_kernel;
+  TF_ASSERT_OK(CreateMapDatasetOpKernel<int64>(
+      range_dataset->name(), func_def.signature().name(), &map_kernel));
+  std::unique_ptr<OpKernelContext> map_context;
+  TF_ASSERT_OK(
+      CreateMapDatasetContext(range_dataset, map_kernel.get(), &map_context));
+  DatasetBase* map_dataset;
+  TF_ASSERT_OK(
+      CreateDataset(map_kernel.get(), map_context.get(), &map_dataset));
+  core::ScopedUnref scored_unref_map_dataset(map_dataset);
+
+  std::vector<PartialTensorShape> expected_shapes({{}});
+  EXPECT_EQ(map_dataset->output_shapes().size(), expected_shapes.size());
+  for (int i = 0; i < map_dataset->output_shapes().size(); ++i) {
+    map_dataset->output_shapes()[i].IsIdenticalTo(expected_shapes[i]);
+  }
+}
+
+struct CardinalityTestParams {
+  explicit CardinalityTestParams(int64 input_start, int64 input_end,
+                                 int64 input_step,
+                                 int input_expected_cardinality)
+      : start(input_start),
+        end(input_end),
+        step(input_step),
+        expected_cardinality(input_expected_cardinality) {}
+
+  int64 start;
+  int64 end;
+  int64 step;
+  int expected_cardinality;
+};
+
+struct DatasetCardinalityTest
+    : MapDatasetOpTest,
+      ::testing::WithParamInterface<CardinalityTestParams> {};
+
+TEST_P(DatasetCardinalityTest, Cardinality) {
+  int thread_num = 2, cpu_num = 2;
+  CardinalityTestParams test_params = GetParam();
+  FunctionDef func_def = test::function::XTimesTwo();
+
+  TF_ASSERT_OK(InitThreadPool(thread_num));
+  TF_ASSERT_OK(InitFunctionLibraryRuntime({func_def}, cpu_num));
+
+  DatasetBase* range_dataset;
+  TF_ASSERT_OK(CreateRangeDataset<int64>(test_params.start, test_params.end,
+                                         test_params.step, "range",
+                                         &range_dataset));
+  core::ScopedUnref scored_unref_range_dataset(range_dataset);
+
+  std::unique_ptr<OpKernel> map_kernel;
+  TF_ASSERT_OK(CreateMapDatasetOpKernel<int64>(
+      range_dataset->name(), func_def.signature().name(), &map_kernel));
+  std::unique_ptr<OpKernelContext> map_context;
+  TF_ASSERT_OK(
+      CreateMapDatasetContext(range_dataset, map_kernel.get(), &map_context));
+  DatasetBase* map_dataset;
+  TF_ASSERT_OK(
+      CreateDataset(map_kernel.get(), map_context.get(), &map_dataset));
+  core::ScopedUnref scored_unref_map_dataset(map_dataset);
+
+  EXPECT_EQ(map_dataset->Cardinality(), test_params.expected_cardinality);
+}
+
+INSTANTIATE_TEST_CASE_P(MapDatasetOpTest, DatasetCardinalityTest,
+                        ::testing::Values(CardinalityTestParams(0, 10, 1, 10),
+                                          CardinalityTestParams(0, 10, 3, 4),
+                                          CardinalityTestParams(10, 0, -3, 4)));
+
+TEST_F(MapDatasetOpTest, DatasetSave) {
+  int thread_num = 2, cpu_num = 2;
+  int64 start = 0, end = 10, step = 1;
+  FunctionDef func_def = test::function::XTimesTwo();
+
+  TF_ASSERT_OK(InitThreadPool(thread_num));
+  TF_ASSERT_OK(InitFunctionLibraryRuntime({func_def}, cpu_num));
+
+  DatasetBase* range_dataset;
+  TF_ASSERT_OK(
+      CreateRangeDataset<int64>(start, end, step, "range", &range_dataset));
+  core::ScopedUnref scored_unref_range_dataset(range_dataset);
+
+  std::unique_ptr<OpKernel> map_kernel;
+  TF_ASSERT_OK(CreateMapDatasetOpKernel<int64>(
+      range_dataset->name(), func_def.signature().name(), &map_kernel));
+  std::unique_ptr<OpKernelContext> map_context;
+  TF_ASSERT_OK(
+      CreateMapDatasetContext(range_dataset, map_kernel.get(), &map_context));
+  DatasetBase* map_dataset;
+  TF_ASSERT_OK(
+      CreateDataset(map_kernel.get(), map_context.get(), &map_dataset));
+  core::ScopedUnref scored_unref_map_dataset(map_dataset);
+
+  std::unique_ptr<SerializationContext> serialization_context;
+  TF_ASSERT_OK(CreateSerializationContext(&serialization_context));
+  VariantTensorData data;
+  VariantTensorDataWriter writer(&data);
+  TF_ASSERT_OK(map_dataset->Save(serialization_context.get(), &writer));
+  TF_ASSERT_OK(writer.Flush());
+}
+
+TEST_F(MapDatasetOpTest, IteratorOutputDtypes) {
+  int64 start = 0, end = 10, step = 1;
+  int thread_num = 2, cpu_num = 2;
+  FunctionDef func_def = test::function::XTimesTwo();
+
+  TF_ASSERT_OK(InitThreadPool(thread_num));
+  TF_ASSERT_OK(InitFunctionLibraryRuntime({func_def}, cpu_num));
+
+  DatasetBase* range_dataset;
+  TF_ASSERT_OK(
+      CreateRangeDataset<int64>(start, end, step, "range", &range_dataset));
+  core::ScopedUnref scored_unref_range_dataset(range_dataset);
+
+  std::unique_ptr<OpKernel> map_kernel;
+  TF_ASSERT_OK(CreateMapDatasetOpKernel<int64>(
+      range_dataset->name(), func_def.signature().name(), &map_kernel));
+  std::unique_ptr<OpKernelContext> map_context;
+  TF_ASSERT_OK(
+      CreateMapDatasetContext(range_dataset, map_kernel.get(), &map_context));
+  DatasetBase* map_dataset;
+  TF_ASSERT_OK(
+      CreateDataset(map_kernel.get(), map_context.get(), &map_dataset));
+  core::ScopedUnref scored_unref_map_dataset(map_dataset);
+
+  std::unique_ptr<IteratorContext> iterator_context;
+  TF_ASSERT_OK(CreateIteratorContext(map_context.get(), &iterator_context));
+  std::unique_ptr<IteratorBase> iterator;
+  TF_ASSERT_OK(
+      map_dataset->MakeIterator(iterator_context.get(), "Iterator", &iterator));
+  DataTypeVector expected_dtypes({DT_INT64});
+  EXPECT_EQ(iterator->output_dtypes(), expected_dtypes);
+}
+
+TEST_F(MapDatasetOpTest, IteratorOutputShapes) {
+  int64 start = 0, end = 10, step = 1;
+  int thread_num = 2, cpu_num = 2;
+  FunctionDef func_def = test::function::XTimesTwo();
+
+  TF_ASSERT_OK(InitThreadPool(thread_num));
+  TF_ASSERT_OK(InitFunctionLibraryRuntime({func_def}, cpu_num));
+
+  DatasetBase* range_dataset;
+  TF_ASSERT_OK(
+      CreateRangeDataset<int64>(start, end, step, "range", &range_dataset));
+  core::ScopedUnref scored_unref_range_dataset(range_dataset);
+
+  std::unique_ptr<OpKernel> map_kernel;
+  TF_ASSERT_OK(CreateMapDatasetOpKernel<int64>(
+      range_dataset->name(), func_def.signature().name(), &map_kernel));
+  std::unique_ptr<OpKernelContext> map_context;
+  TF_ASSERT_OK(
+      CreateMapDatasetContext(range_dataset, map_kernel.get(), &map_context));
+  DatasetBase* map_dataset;
+  TF_ASSERT_OK(
+      CreateDataset(map_kernel.get(), map_context.get(), &map_dataset));
+  core::ScopedUnref scored_unref_map_dataset(map_dataset);
+
+  std::unique_ptr<IteratorContext> iterator_context;
+  TF_ASSERT_OK(CreateIteratorContext(map_context.get(), &iterator_context));
+  std::unique_ptr<IteratorBase> iterator;
+  TF_ASSERT_OK(
+      map_dataset->MakeIterator(iterator_context.get(), "Iterator", &iterator));
+
+  std::vector<PartialTensorShape> expected_shapes({{}});
+  EXPECT_EQ(iterator->output_shapes().size(), expected_shapes.size());
+  for (int i = 0; i < map_dataset->output_shapes().size(); ++i) {
+    iterator->output_shapes()[i].IsIdenticalTo(expected_shapes[i]);
+  }
+}
+
+TEST_F(MapDatasetOpTest, IteratorOutputPrefix) {
+  int64 start = 0, end = 10, step = 1;
+  int thread_num = 2, cpu_num = 2;
+  FunctionDef func_def = test::function::XTimesTwo();
+
+  TF_ASSERT_OK(InitThreadPool(thread_num));
+  TF_ASSERT_OK(InitFunctionLibraryRuntime({func_def}, cpu_num));
+
+  DatasetBase* range_dataset;
+  TF_ASSERT_OK(
+      CreateRangeDataset<int64>(start, end, step, "range", &range_dataset));
+  core::ScopedUnref scored_unref_range_dataset(range_dataset);
+
+  std::unique_ptr<OpKernel> map_kernel;
+  TF_ASSERT_OK(CreateMapDatasetOpKernel<int64>(
+      range_dataset->name(), func_def.signature().name(), &map_kernel));
+  std::unique_ptr<OpKernelContext> map_context;
+  TF_ASSERT_OK(
+      CreateMapDatasetContext(range_dataset, map_kernel.get(), &map_context));
+  DatasetBase* map_dataset;
+  TF_ASSERT_OK(
+      CreateDataset(map_kernel.get(), map_context.get(), &map_dataset));
+  core::ScopedUnref scored_unref_map_dataset(map_dataset);
+
+  std::unique_ptr<IteratorContext> iterator_context;
+  TF_ASSERT_OK(CreateIteratorContext(map_context.get(), &iterator_context));
+  std::unique_ptr<IteratorBase> iterator;
+  TF_ASSERT_OK(
+      map_dataset->MakeIterator(iterator_context.get(), "Iterator", &iterator));
+
+  EXPECT_EQ(iterator->prefix(), "Iterator::Map");
+}
+
+struct RoundtripTestParams {
+  explicit RoundtripTestParams(int64 input_start, int64 input_end,
+                               int64 input_step, int input_breakpoint,
+                               int64 input_expected_value,
+                               string input_func_name,
+                               std::vector<FunctionDef> input_func_lib)
+      : start(input_start),
+        end(input_end),
+        step(input_step),
+        breakpoint(input_breakpoint),
+        expected_value(input_expected_value),
+        func_name(std::move(input_func_name)),
+        func_lib(std::move(input_func_lib)) {}
+
+  int64 start;
+  int64 end;
+  int64 step;
+  int breakpoint;
+  int64 expected_value;
+  string func_name;
+  std::vector<FunctionDef> func_lib;
+};
+
+struct IteratorRoundtripTest
+    : MapDatasetOpTest,
+      ::testing::WithParamInterface<RoundtripTestParams> {};
+
+TEST_P(IteratorRoundtripTest, Roundtrip) {
+  int thread_num = 2, cpu_num = 2;
+  RoundtripTestParams test_params = GetParam();
+
+  TF_ASSERT_OK(InitThreadPool(thread_num));
+  TF_ASSERT_OK(InitFunctionLibraryRuntime(test_params.func_lib, cpu_num));
+
+  DatasetBase* range_dataset;
+  TF_ASSERT_OK(CreateRangeDataset<int64>(test_params.start, test_params.end,
+                                         test_params.step, "range",
+                                         &range_dataset));
+  core::ScopedUnref scored_unref_range_dataset(range_dataset);
+
+  std::unique_ptr<OpKernel> map_kernel;
+  TF_ASSERT_OK(CreateMapDatasetOpKernel<int64>(
+      range_dataset->name(), test_params.func_name, &map_kernel));
+  std::unique_ptr<OpKernelContext> map_context;
+  TF_ASSERT_OK(
+      CreateMapDatasetContext(range_dataset, map_kernel.get(), &map_context));
+  DatasetBase* map_dataset;
+  TF_ASSERT_OK(
+      CreateDataset(map_kernel.get(), map_context.get(), &map_dataset));
+  core::ScopedUnref scored_unref_map_dataset(map_dataset);
+
+  std::unique_ptr<IteratorContext> iterator_context;
+  TF_ASSERT_OK(CreateIteratorContext(map_context.get(), &iterator_context));
+  std::unique_ptr<IteratorBase> iterator;
+  TF_ASSERT_OK(
+      map_dataset->MakeIterator(iterator_context.get(), "Iterator", &iterator));
+
+  std::vector<Tensor> out_tensors;
+  bool end_of_sequence = false;
+  for (int i = 0; i < test_params.breakpoint; i++) {
+    TF_EXPECT_OK(iterator->GetNext(iterator_context.get(), &out_tensors,
+                                   &end_of_sequence));
+  }
+
+  std::unique_ptr<SerializationContext> serialization_context;
+  TF_ASSERT_OK(CreateSerializationContext(&serialization_context));
+  VariantTensorData data;
+  VariantTensorDataWriter writer(&data);
+  TF_ASSERT_OK(iterator->Save(serialization_context.get(), &writer));
+  TF_ASSERT_OK(writer.Flush());
+  VariantTensorDataReader reader(&data);
+  TF_ASSERT_OK(iterator->Restore(iterator_context.get(), &reader));
+  TF_EXPECT_OK(iterator->GetNext(iterator_context.get(), &out_tensors,
+                                 &end_of_sequence));
+  EXPECT_EQ(out_tensors.back().flat<int64>()(0), test_params.expected_value);
+}
+
+INSTANTIATE_TEST_CASE_P(
+    MapDatasetOpTest, IteratorRoundtripTest,
+    ::testing::Values(RoundtripTestParams(0, 10, 2, 0, 0, "XTimesTwo",
+                                          std::vector<FunctionDef>{
+                                              test::function::XTimesTwo()}),
+                      RoundtripTestParams(0, 10, 2, 4, 16, "XAddX",
+                                          std::vector<FunctionDef>{
+                                              test::function::XAddX()}),
+                      RoundtripTestParams(0, 10, 2, 6, 32, "XTimesFour",
+                                          std::vector<FunctionDef>{
+                                              test::function::XTimesTwo(),
+                                              test::function::XTimesFour()})));
+
+}  // namespace
+}  // namespace data
+}  // namespace tensorflow
-- 
GitLab


From 659581f8e7b1d97332c62d5b4f036d3387c7a808 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Wed, 30 Jan 2019 14:34:03 -0800
Subject: [PATCH 1625/2345] Automated rollback of commit
 7157690e660676f8b01aec55d42d1e76101e6fa8

PiperOrigin-RevId: 231669525
---
 tensorflow/python/BUILD                       |    9 -
 tensorflow/python/framework/python_op_gen.cc  |   64 -
 tensorflow/python/ops/raw_ops_test.py         |   42 -
 .../tools/api/generator/api_init_files.bzl    |    1 -
 .../tools/api/generator/api_init_files_v1.bzl |    1 -
 tensorflow/python/util/tf_export.py           |   10 -
 tensorflow/python/util/tf_inspect.py          |   26 +-
 .../tools/api/golden/v1/tensorflow.pbtxt      |    4 -
 .../api/golden/v1/tensorflow.raw_ops.pbtxt    | 4023 -----------------
 .../tools/api/golden/v2/tensorflow.pbtxt      |    4 -
 .../api/golden/v2/tensorflow.raw_ops.pbtxt    | 4023 -----------------
 11 files changed, 11 insertions(+), 8196 deletions(-)
 delete mode 100644 tensorflow/python/ops/raw_ops_test.py
 delete mode 100644 tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 98b9518ed4..109beace58 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -650,7 +650,6 @@ py_library(
         "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:core",
         "//tensorflow/python/eager:execute",
-        "//tensorflow/tools/docs:doc_controls",
     ],
 )
 
@@ -6071,11 +6070,3 @@ py_library(
     srcs = ["tf2.py"],
     srcs_version = "PY2AND3",
 )
-
-cuda_py_test(
-    name = "raw_ops_test",
-    srcs = ["ops/raw_ops_test.py"],
-    additional_deps = [
-        "//tensorflow/python:client_testlib",
-    ],
-)
diff --git a/tensorflow/python/framework/python_op_gen.cc b/tensorflow/python/framework/python_op_gen.cc
index 0f304385de..d460168631 100644
--- a/tensorflow/python/framework/python_op_gen.cc
+++ b/tensorflow/python/framework/python_op_gen.cc
@@ -144,8 +144,6 @@ class GenEagerPythonOp : public python_op_gen_internal::GenPythonOp {
                        const string& num_outputs_expr);
   void AddDispatch(const string& prefix);
 
-  void AddRawOpExport();
-
   void AddAttrForArg(const string& attr, int arg_index) {
     gtl::InsertIfNotPresent(&inferred_attrs_, attr,
                             op_def_.input_arg(arg_index).name());
@@ -670,7 +668,6 @@ bool GenEagerPythonOp::AddEagerFastPathAndGraphCode(
   AddEagerFunctionTeardown("  ", output_sizes,
                            true /* execute_record_gradient */);
 
-  AddRawOpExport();
   strings::StrAppend(&result_, "\n\n");
   return true;
 }
@@ -924,65 +921,6 @@ void GenEagerPythonOp::AddDispatch(const string& prefix) {
   strings::StrAppend(&result_, prefix, "  raise\n");
 }
 
-void GenEagerPythonOp::AddRawOpExport() {
-  // Create function for python op.
-  string raw_parameters;
-  string function_call_parameters;
-  string inputs;
-  string attrs;
-  std::map<string, string> renames;
-
-  for (const auto& input_arg : api_def_.in_arg()) {
-    renames.insert({input_arg.name(), input_arg.rename_to()});
-  }
-  for (const auto& attr : api_def_.attr()) {
-    renames.insert({attr.name(), attr.rename_to()});
-  }
-
-  for (const auto& input_arg : op_def_.input_arg()) {
-    if (!raw_parameters.empty()) strings::StrAppend(&raw_parameters, ", ");
-    strings::StrAppend(&raw_parameters, input_arg.name());
-
-    if (!inputs.empty()) strings::StrAppend(&inputs, ", ");
-    strings::StrAppend(&inputs, input_arg.name());
-
-    if (!function_call_parameters.empty()) {
-      strings::StrAppend(&function_call_parameters, ", ");
-    }
-    strings::StrAppend(&function_call_parameters, renames[input_arg.name()],
-                       "=", input_arg.name());
-  }
-  for (const auto& attr : op_def_.attr()) {
-    if (inferred_attrs_.find(attr.name()) != inferred_attrs_.end()) continue;
-
-    if (!raw_parameters.empty()) strings::StrAppend(&raw_parameters, ", ");
-    strings::StrAppend(&raw_parameters, attr.name());
-
-    if (!attrs.empty()) strings::StrAppend(&attrs, ", ");
-    strings::StrAppend(&attrs, "\"", attr.name(), "\", ", attr.name());
-
-    if (!function_call_parameters.empty()) {
-      strings::StrAppend(&function_call_parameters, ", ");
-    }
-    strings::StrAppend(&function_call_parameters, renames[attr.name()], "=",
-                       attr.name());
-  }
-
-  const string raw_function_name =
-      python_op_gen_internal::AvoidPythonReserved(op_def_.name());
-
-  strings::StrAppend(&result_,
-                     "@_doc_controls.do_not_generate_docs\n@_kwarg_only\ndef ",
-                     raw_function_name, "(", raw_parameters, "):\n");
-
-  // Function body.
-  strings::StrAppend(&result_, "  return ", function_name_, "(",
-                     function_call_parameters, ")\n");
-
-  strings::StrAppend(&result_, "tf_export(\"raw_ops.", raw_function_name,
-                     "\")(", raw_function_name, ")\n");
-}
-
 string GetPythonOps(const OpList& ops, const ApiDefMap& api_defs,
                     const std::vector<string>& hidden_ops, bool require_shapes,
                     const string& source_file_name = "") {
@@ -1024,8 +962,6 @@ from tensorflow.python.framework import op_def_library as _op_def_library
 from tensorflow.python.util.deprecation import deprecated_endpoints
 from tensorflow.python.util import dispatch as _dispatch
 from tensorflow.python.util.tf_export import tf_export
-from tensorflow.python.util.tf_export import kwarg_only as _kwarg_only
-from tensorflow.tools.docs import doc_controls as _doc_controls
 
 )");
 
diff --git a/tensorflow/python/ops/raw_ops_test.py b/tensorflow/python/ops/raw_ops_test.py
deleted file mode 100644
index 82ea137354..0000000000
--- a/tensorflow/python/ops/raw_ops_test.py
+++ /dev/null
@@ -1,42 +0,0 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Raw ops tests."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from tensorflow.python.framework import constant_op
-from tensorflow.python.framework import ops
-from tensorflow.python.framework import test_util
-from tensorflow.python.ops import gen_math_ops
-from tensorflow.python.platform import test
-
-
-@test_util.run_all_in_graph_and_eager_modes
-class RawOpsTest(test.TestCase):
-
-  def testSimple(self):
-
-    with self.assertRaises(TypeError):
-      _ = gen_math_ops.Add(1., 1.)
-
-    x = constant_op.constant(1)
-    self.assertEqual([2], self.evaluate(gen_math_ops.Add(x=x, y=x)))
-
-
-if __name__ == '__main__':
-  ops.enable_eager_execution()
-  test.main()
diff --git a/tensorflow/python/tools/api/generator/api_init_files.bzl b/tensorflow/python/tools/api/generator/api_init_files.bzl
index 5095f9fdbe..245380287a 100644
--- a/tensorflow/python/tools/api/generator/api_init_files.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files.bzl
@@ -35,7 +35,6 @@ TENSORFLOW_API_INIT_FILES = [
     "quantization/__init__.py",
     "ragged/__init__.py",
     "random/__init__.py",
-    "raw_ops/__init__.py",
     "rnn/__init__.py",
     "saved_model/__init__.py",
     "sets/__init__.py",
diff --git a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
index 01bf12ebe7..c8c7c51b40 100644
--- a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
@@ -46,7 +46,6 @@ TENSORFLOW_API_INIT_FILES_V1 = [
     "quantization/__init__.py",
     "ragged/__init__.py",
     "random/__init__.py",
-    "raw_ops/__init__.py",
     "resource_loader/__init__.py",
     "strings/__init__.py",
     "saved_model/__init__.py",
diff --git a/tensorflow/python/util/tf_export.py b/tensorflow/python/util/tf_export.py
index 7b44d72ab9..ce1a1a5c30 100644
--- a/tensorflow/python/util/tf_export.py
+++ b/tensorflow/python/util/tf_export.py
@@ -46,7 +46,6 @@ import functools
 import sys
 
 from tensorflow.python.util import tf_decorator
-from tensorflow.python.util import tf_inspect
 
 ESTIMATOR_API_NAME = 'estimator'
 KERAS_API_NAME = 'keras'
@@ -380,15 +379,6 @@ class api_export(object):  # pylint: disable=invalid-name
         (self._names_v1, name))
 
 
-def kwarg_only(f):
-  """A wrapper that throws away all non-kwarg arguments."""
-  def wrapper(**kwargs):
-    return f(**kwargs)
-
-  return tf_decorator.make_decorator(
-      f, wrapper, decorator_argspec=tf_inspect.getargspec(f))
-
-
 tf_export = functools.partial(api_export, api_name=TENSORFLOW_API_NAME)
 estimator_export = functools.partial(api_export, api_name=ESTIMATOR_API_NAME)
 keras_export = functools.partial(api_export, api_name=KERAS_API_NAME)
diff --git a/tensorflow/python/util/tf_inspect.py b/tensorflow/python/util/tf_inspect.py
index 3a1c4a6e12..5171cdfdbc 100644
--- a/tensorflow/python/util/tf_inspect.py
+++ b/tensorflow/python/util/tf_inspect.py
@@ -36,19 +36,6 @@ else:
       'annotations'
   ])
 
-
-def _convert_maybe_argspec_to_fullargspec(argspec):
-  if isinstance(argspec, FullArgSpec):
-    return argspec
-  return FullArgSpec(
-      args=argspec.args,
-      varargs=argspec.varargs,
-      varkw=argspec.keywords,
-      defaults=argspec.defaults,
-      kwonlyargs=[],
-      kwonlydefaults=None,
-      annotations={})
-
 if hasattr(_inspect, 'getfullargspec'):
   _getfullargspec = _inspect.getfullargspec  # pylint: disable=invalid-name
 
@@ -87,7 +74,16 @@ else:
     Returns:
       A FullArgSpec with empty kwonlyargs, kwonlydefaults and annotations.
     """
-    return _convert_maybe_argspec_to_fullargspec(getargspec(target))
+    argspecs = getargspec(target)
+    fullargspecs = FullArgSpec(
+        args=argspecs.args,
+        varargs=argspecs.varargs,
+        varkw=argspecs.keywords,
+        defaults=argspecs.defaults,
+        kwonlyargs=[],
+        kwonlydefaults=None,
+        annotations={})
+    return fullargspecs
 
 
 def currentframe():
@@ -242,7 +238,7 @@ def getfullargspec(obj):
     directly on the callable.
   """
   decorators, target = tf_decorator.unwrap(obj)
-  return next((_convert_maybe_argspec_to_fullargspec(d.decorator_argspec)
+  return next((d.decorator_argspec
                for d in decorators
                if d.decorator_argspec is not None), _getfullargspec(target))
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
index 56d06053f1..c8ec8da259 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
@@ -552,10 +552,6 @@ tf_module {
     name: "random_uniform_initializer"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "raw_ops"
-    mtype: "<type \'module\'>"
-  }
   member {
     name: "resource"
     mtype: "<class \'tensorflow.python.framework.dtypes.DType\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt
deleted file mode 100644
index c52a0cd41f..0000000000
--- a/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt
+++ /dev/null
@@ -1,4023 +0,0 @@
-path: "tensorflow.raw_ops"
-tf_module {
-  member_method {
-    name: "Abort"
-    argspec: "args=[\'error_msg\', \'exit_without_error\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Abs"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AccumulateNV2"
-    argspec: "args=[\'inputs\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AccumulatorApplyGradient"
-    argspec: "args=[\'handle\', \'local_step\', \'gradient\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AccumulatorNumAccumulated"
-    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AccumulatorSetGlobalStep"
-    argspec: "args=[\'handle\', \'new_global_step\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AccumulatorTakeGradient"
-    argspec: "args=[\'handle\', \'num_required\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Acos"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Acosh"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Add"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AddManySparseToTensorsMap"
-    argspec: "args=[\'sparse_indices\', \'sparse_values\', \'sparse_shape\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AddN"
-    argspec: "args=[\'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AddSparseToTensorsMap"
-    argspec: "args=[\'sparse_indices\', \'sparse_values\', \'sparse_shape\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AddV2"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AdjustContrast"
-    argspec: "args=[\'images\', \'contrast_factor\', \'min_value\', \'max_value\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AdjustContrastv2"
-    argspec: "args=[\'images\', \'contrast_factor\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AdjustHue"
-    argspec: "args=[\'images\', \'delta\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AdjustSaturation"
-    argspec: "args=[\'images\', \'scale\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "All"
-    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AllCandidateSampler"
-    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Angle"
-    argspec: "args=[\'input\', \'Tout\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AnonymousIterator"
-    argspec: "args=[\'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Any"
-    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyAdaMax"
-    argspec: "args=[\'var\', \'m\', \'v\', \'beta1_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyAdadelta"
-    argspec: "args=[\'var\', \'accum\', \'accum_update\', \'lr\', \'rho\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyAdagrad"
-    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'use_locking\', \'update_slots\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyAdagradDA"
-    argspec: "args=[\'var\', \'gradient_accumulator\', \'gradient_squared_accumulator\', \'grad\', \'lr\', \'l1\', \'l2\', \'global_step\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyAdam"
-    argspec: "args=[\'var\', \'m\', \'v\', \'beta1_power\', \'beta2_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyAddSign"
-    argspec: "args=[\'var\', \'m\', \'lr\', \'alpha\', \'sign_decay\', \'beta\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyCenteredRMSProp"
-    argspec: "args=[\'var\', \'mg\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyFtrl"
-    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'lr\', \'l1\', \'l2\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyFtrlV2"
-    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'lr\', \'l1\', \'l2\', \'l2_shrinkage\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyGradientDescent"
-    argspec: "args=[\'var\', \'alpha\', \'delta\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyMomentum"
-    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyPowerSign"
-    argspec: "args=[\'var\', \'m\', \'lr\', \'logbase\', \'sign_decay\', \'beta\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyProximalAdagrad"
-    argspec: "args=[\'var\', \'accum\', \'lr\', \'l1\', \'l2\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyProximalGradientDescent"
-    argspec: "args=[\'var\', \'alpha\', \'l1\', \'l2\', \'delta\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyRMSProp"
-    argspec: "args=[\'var\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApproximateEqual"
-    argspec: "args=[\'x\', \'y\', \'tolerance\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ArgMax"
-    argspec: "args=[\'input\', \'dimension\', \'output_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ArgMin"
-    argspec: "args=[\'input\', \'dimension\', \'output_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AsString"
-    argspec: "args=[\'input\', \'precision\', \'scientific\', \'shortest\', \'width\', \'fill\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Asin"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Asinh"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Assert"
-    argspec: "args=[\'condition\', \'data\', \'summarize\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Assign"
-    argspec: "args=[\'ref\', \'value\', \'validate_shape\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AssignAdd"
-    argspec: "args=[\'ref\', \'value\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AssignAddVariableOp"
-    argspec: "args=[\'resource\', \'value\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AssignSub"
-    argspec: "args=[\'ref\', \'value\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AssignSubVariableOp"
-    argspec: "args=[\'resource\', \'value\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AssignVariableOp"
-    argspec: "args=[\'resource\', \'value\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Atan"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Atan2"
-    argspec: "args=[\'y\', \'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Atanh"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AudioSpectrogram"
-    argspec: "args=[\'input\', \'window_size\', \'stride\', \'magnitude_squared\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AudioSummary"
-    argspec: "args=[\'tag\', \'tensor\', \'sample_rate\', \'max_outputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AudioSummaryV2"
-    argspec: "args=[\'tag\', \'tensor\', \'sample_rate\', \'max_outputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AvgPool"
-    argspec: "args=[\'value\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AvgPool3D"
-    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AvgPool3DGrad"
-    argspec: "args=[\'orig_input_shape\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AvgPoolGrad"
-    argspec: "args=[\'orig_input_shape\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Barrier"
-    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BarrierClose"
-    argspec: "args=[\'handle\', \'cancel_pending_enqueues\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BarrierIncompleteSize"
-    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BarrierInsertMany"
-    argspec: "args=[\'handle\', \'keys\', \'values\', \'component_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BarrierReadySize"
-    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BarrierTakeMany"
-    argspec: "args=[\'handle\', \'num_elements\', \'component_types\', \'allow_small_batch\', \'wait_for_incomplete\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchCholesky"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchCholeskyGrad"
-    argspec: "args=[\'l\', \'grad\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchDataset"
-    argspec: "args=[\'input_dataset\', \'batch_size\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchDatasetV2"
-    argspec: "args=[\'input_dataset\', \'batch_size\', \'drop_remainder\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchFFT"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchFFT2D"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchFFT3D"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchIFFT"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchIFFT2D"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchIFFT3D"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchMatMul"
-    argspec: "args=[\'x\', \'y\', \'adj_x\', \'adj_y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchMatrixBandPart"
-    argspec: "args=[\'input\', \'num_lower\', \'num_upper\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchMatrixDeterminant"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchMatrixDiag"
-    argspec: "args=[\'diagonal\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchMatrixDiagPart"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchMatrixInverse"
-    argspec: "args=[\'input\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchMatrixSetDiag"
-    argspec: "args=[\'input\', \'diagonal\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchMatrixSolve"
-    argspec: "args=[\'matrix\', \'rhs\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchMatrixSolveLs"
-    argspec: "args=[\'matrix\', \'rhs\', \'l2_regularizer\', \'fast\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchMatrixTriangularSolve"
-    argspec: "args=[\'matrix\', \'rhs\', \'lower\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchNormWithGlobalNormalization"
-    argspec: "args=[\'t\', \'m\', \'v\', \'beta\', \'gamma\', \'variance_epsilon\', \'scale_after_normalization\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchNormWithGlobalNormalizationGrad"
-    argspec: "args=[\'t\', \'m\', \'v\', \'gamma\', \'backprop\', \'variance_epsilon\', \'scale_after_normalization\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchSelfAdjointEig"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchSelfAdjointEigV2"
-    argspec: "args=[\'input\', \'compute_v\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchSvd"
-    argspec: "args=[\'input\', \'compute_uv\', \'full_matrices\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchToSpace"
-    argspec: "args=[\'input\', \'crops\', \'block_size\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchToSpaceND"
-    argspec: "args=[\'input\', \'block_shape\', \'crops\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BesselI0e"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BesselI1e"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Betainc"
-    argspec: "args=[\'a\', \'b\', \'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BiasAdd"
-    argspec: "args=[\'value\', \'bias\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BiasAddGrad"
-    argspec: "args=[\'out_backprop\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BiasAddV1"
-    argspec: "args=[\'value\', \'bias\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Bincount"
-    argspec: "args=[\'arr\', \'size\', \'weights\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Bitcast"
-    argspec: "args=[\'input\', \'type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BitwiseAnd"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BitwiseOr"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BitwiseXor"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesBucketize"
-    argspec: "args=[\'float_values\', \'bucket_boundaries\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesCalculateBestGainsPerFeature"
-    argspec: "args=[\'node_id_range\', \'stats_summary_list\', \'l1\', \'l2\', \'tree_complexity\', \'min_node_weight\', \'max_splits\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesCenterBias"
-    argspec: "args=[\'tree_ensemble_handle\', \'mean_gradients\', \'mean_hessians\', \'l1\', \'l2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesCreateEnsemble"
-    argspec: "args=[\'tree_ensemble_handle\', \'stamp_token\', \'tree_ensemble_serialized\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesCreateQuantileStreamResource"
-    argspec: "args=[\'quantile_stream_resource_handle\', \'epsilon\', \'num_streams\', \'max_elements\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesDeserializeEnsemble"
-    argspec: "args=[\'tree_ensemble_handle\', \'stamp_token\', \'tree_ensemble_serialized\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesEnsembleResourceHandleOp"
-    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesExampleDebugOutputs"
-    argspec: "args=[\'tree_ensemble_handle\', \'bucketized_features\', \'logits_dimension\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesGetEnsembleStates"
-    argspec: "args=[\'tree_ensemble_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesMakeQuantileSummaries"
-    argspec: "args=[\'float_values\', \'example_weights\', \'epsilon\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesMakeStatsSummary"
-    argspec: "args=[\'node_ids\', \'gradients\', \'hessians\', \'bucketized_features_list\', \'max_splits\', \'num_buckets\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesPredict"
-    argspec: "args=[\'tree_ensemble_handle\', \'bucketized_features\', \'logits_dimension\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesQuantileStreamResourceAddSummaries"
-    argspec: "args=[\'quantile_stream_resource_handle\', \'summaries\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesQuantileStreamResourceDeserialize"
-    argspec: "args=[\'quantile_stream_resource_handle\', \'bucket_boundaries\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesQuantileStreamResourceFlush"
-    argspec: "args=[\'quantile_stream_resource_handle\', \'num_buckets\', \'generate_quantiles\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesQuantileStreamResourceGetBucketBoundaries"
-    argspec: "args=[\'quantile_stream_resource_handle\', \'num_features\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesQuantileStreamResourceHandleOp"
-    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesSerializeEnsemble"
-    argspec: "args=[\'tree_ensemble_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesTrainingPredict"
-    argspec: "args=[\'tree_ensemble_handle\', \'cached_tree_ids\', \'cached_node_ids\', \'bucketized_features\', \'logits_dimension\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesUpdateEnsemble"
-    argspec: "args=[\'tree_ensemble_handle\', \'feature_ids\', \'node_ids\', \'gains\', \'thresholds\', \'left_node_contribs\', \'right_node_contribs\', \'max_depth\', \'learning_rate\', \'pruning_mode\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BroadcastArgs"
-    argspec: "args=[\'s0\', \'s1\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BroadcastGradientArgs"
-    argspec: "args=[\'s0\', \'s1\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BroadcastTo"
-    argspec: "args=[\'input\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Bucketize"
-    argspec: "args=[\'input\', \'boundaries\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CTCBeamSearchDecoder"
-    argspec: "args=[\'inputs\', \'sequence_length\', \'beam_width\', \'top_paths\', \'merge_repeated\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CTCGreedyDecoder"
-    argspec: "args=[\'inputs\', \'sequence_length\', \'merge_repeated\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CTCLoss"
-    argspec: "args=[\'inputs\', \'labels_indices\', \'labels_values\', \'sequence_length\', \'preprocess_collapse_repeated\', \'ctc_merge_repeated\', \'ignore_longer_outputs_than_inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CacheDataset"
-    argspec: "args=[\'input_dataset\', \'filename\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Cast"
-    argspec: "args=[\'x\', \'DstT\', \'Truncate\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Ceil"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CheckNumerics"
-    argspec: "args=[\'tensor\', \'message\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Cholesky"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CholeskyGrad"
-    argspec: "args=[\'l\', \'grad\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ClipByValue"
-    argspec: "args=[\'t\', \'clip_value_min\', \'clip_value_max\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CloseSummaryWriter"
-    argspec: "args=[\'writer\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CollectiveBcastRecv"
-    argspec: "args=[\'T\', \'group_size\', \'group_key\', \'instance_key\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CollectiveBcastSend"
-    argspec: "args=[\'input\', \'group_size\', \'group_key\', \'instance_key\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CollectiveReduce"
-    argspec: "args=[\'input\', \'group_size\', \'group_key\', \'instance_key\', \'merge_op\', \'final_op\', \'subdiv_offsets\', \'wait_for\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CombinedNonMaxSuppression"
-    argspec: "args=[\'boxes\', \'scores\', \'max_output_size_per_class\', \'max_total_size\', \'iou_threshold\', \'score_threshold\', \'pad_per_class\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CompareAndBitpack"
-    argspec: "args=[\'input\', \'threshold\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Complex"
-    argspec: "args=[\'real\', \'imag\', \'Tout\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ComplexAbs"
-    argspec: "args=[\'x\', \'Tout\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ComputeAccidentalHits"
-    argspec: "args=[\'true_classes\', \'sampled_candidates\', \'num_true\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Concat"
-    argspec: "args=[\'concat_dim\', \'values\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ConcatOffset"
-    argspec: "args=[\'concat_dim\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ConcatV2"
-    argspec: "args=[\'values\', \'axis\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ConcatenateDataset"
-    argspec: "args=[\'input_dataset\', \'another_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ConditionalAccumulator"
-    argspec: "args=[\'dtype\', \'shape\', \'container\', \'shared_name\', \'reduction_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Conj"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ConjugateTranspose"
-    argspec: "args=[\'x\', \'perm\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Const"
-    argspec: "args=[\'value\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ConsumeMutexLock"
-    argspec: "args=[\'mutex_lock\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ControlTrigger"
-    argspec: "args=[], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Conv2D"
-    argspec: "args=[\'input\', \'filter\', \'strides\', \'use_cudnn_on_gpu\', \'padding\', \'explicit_paddings\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Conv2DBackpropFilter"
-    argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'use_cudnn_on_gpu\', \'padding\', \'explicit_paddings\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Conv2DBackpropInput"
-    argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'use_cudnn_on_gpu\', \'padding\', \'explicit_paddings\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Conv3D"
-    argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Conv3DBackpropFilter"
-    argspec: "args=[\'input\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Conv3DBackpropFilterV2"
-    argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Conv3DBackpropInput"
-    argspec: "args=[\'input\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Conv3DBackpropInputV2"
-    argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Cos"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Cosh"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CountUpTo"
-    argspec: "args=[\'ref\', \'limit\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CreateSummaryDbWriter"
-    argspec: "args=[\'writer\', \'db_uri\', \'experiment_name\', \'run_name\', \'user_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CreateSummaryFileWriter"
-    argspec: "args=[\'writer\', \'logdir\', \'max_queue\', \'flush_millis\', \'filename_suffix\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CropAndResize"
-    argspec: "args=[\'image\', \'boxes\', \'box_ind\', \'crop_size\', \'method\', \'extrapolation_value\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CropAndResizeGradBoxes"
-    argspec: "args=[\'grads\', \'image\', \'boxes\', \'box_ind\', \'method\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CropAndResizeGradImage"
-    argspec: "args=[\'grads\', \'boxes\', \'box_ind\', \'image_size\', \'T\', \'method\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Cross"
-    argspec: "args=[\'a\', \'b\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CudnnRNN"
-    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\', \'is_training\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CudnnRNNBackprop"
-    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'output\', \'output_h\', \'output_c\', \'output_backprop\', \'output_h_backprop\', \'output_c_backprop\', \'reserve_space\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CudnnRNNBackpropV2"
-    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'output\', \'output_h\', \'output_c\', \'output_backprop\', \'output_h_backprop\', \'output_c_backprop\', \'reserve_space\', \'host_reserved\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CudnnRNNBackpropV3"
-    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'sequence_lengths\', \'output\', \'output_h\', \'output_c\', \'output_backprop\', \'output_h_backprop\', \'output_c_backprop\', \'reserve_space\', \'host_reserved\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CudnnRNNCanonicalToParams"
-    argspec: "args=[\'num_layers\', \'num_units\', \'input_size\', \'weights\', \'biases\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CudnnRNNParamsSize"
-    argspec: "args=[\'num_layers\', \'num_units\', \'input_size\', \'T\', \'S\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CudnnRNNParamsToCanonical"
-    argspec: "args=[\'num_layers\', \'num_units\', \'input_size\', \'params\', \'num_params\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CudnnRNNV2"
-    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\', \'is_training\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CudnnRNNV3"
-    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'sequence_lengths\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\', \'is_training\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Cumprod"
-    argspec: "args=[\'x\', \'axis\', \'exclusive\', \'reverse\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Cumsum"
-    argspec: "args=[\'x\', \'axis\', \'exclusive\', \'reverse\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DataFormatDimMap"
-    argspec: "args=[\'x\', \'src_format\', \'dst_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DataFormatVecPermute"
-    argspec: "args=[\'x\', \'src_format\', \'dst_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DatasetToGraph"
-    argspec: "args=[\'input_dataset\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DatasetToSingleElement"
-    argspec: "args=[\'dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DebugGradientIdentity"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DebugGradientRefIdentity"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DecodeAndCropJpeg"
-    argspec: "args=[\'contents\', \'crop_window\', \'channels\', \'ratio\', \'fancy_upscaling\', \'try_recover_truncated\', \'acceptable_fraction\', \'dct_method\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DecodeBase64"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DecodeBmp"
-    argspec: "args=[\'contents\', \'channels\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DecodeCSV"
-    argspec: "args=[\'records\', \'record_defaults\', \'field_delim\', \'use_quote_delim\', \'na_value\', \'select_cols\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DecodeCompressed"
-    argspec: "args=[\'bytes\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DecodeGif"
-    argspec: "args=[\'contents\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DecodeJSONExample"
-    argspec: "args=[\'json_examples\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DecodeJpeg"
-    argspec: "args=[\'contents\', \'channels\', \'ratio\', \'fancy_upscaling\', \'try_recover_truncated\', \'acceptable_fraction\', \'dct_method\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DecodePng"
-    argspec: "args=[\'contents\', \'channels\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DecodeRaw"
-    argspec: "args=[\'bytes\', \'out_type\', \'little_endian\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DecodeWav"
-    argspec: "args=[\'contents\', \'desired_channels\', \'desired_samples\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DeepCopy"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DeleteSessionTensor"
-    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DenseToDenseSetOperation"
-    argspec: "args=[\'set1\', \'set2\', \'set_operation\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DenseToSparseSetOperation"
-    argspec: "args=[\'set1\', \'set2_indices\', \'set2_values\', \'set2_shape\', \'set_operation\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DepthToSpace"
-    argspec: "args=[\'input\', \'block_size\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DepthwiseConv2dNative"
-    argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DepthwiseConv2dNativeBackpropFilter"
-    argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DepthwiseConv2dNativeBackpropInput"
-    argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Dequantize"
-    argspec: "args=[\'input\', \'min_range\', \'max_range\', \'mode\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DeserializeIterator"
-    argspec: "args=[\'resource_handle\', \'serialized\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DeserializeManySparse"
-    argspec: "args=[\'serialized_sparse\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DeserializeSparse"
-    argspec: "args=[\'serialized_sparse\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DestroyResourceOp"
-    argspec: "args=[\'resource\', \'ignore_lookup_error\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DestroyTemporaryVariable"
-    argspec: "args=[\'ref\', \'var_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Diag"
-    argspec: "args=[\'diagonal\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DiagPart"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Digamma"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Dilation2D"
-    argspec: "args=[\'input\', \'filter\', \'strides\', \'rates\', \'padding\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Dilation2DBackpropFilter"
-    argspec: "args=[\'input\', \'filter\', \'out_backprop\', \'strides\', \'rates\', \'padding\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Dilation2DBackpropInput"
-    argspec: "args=[\'input\', \'filter\', \'out_backprop\', \'strides\', \'rates\', \'padding\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Div"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DivNoNan"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DrawBoundingBoxes"
-    argspec: "args=[\'images\', \'boxes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DynamicPartition"
-    argspec: "args=[\'data\', \'partitions\', \'num_partitions\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DynamicStitch"
-    argspec: "args=[\'indices\', \'data\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "EagerPyFunc"
-    argspec: "args=[\'input\', \'token\', \'Tout\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "EditDistance"
-    argspec: "args=[\'hypothesis_indices\', \'hypothesis_values\', \'hypothesis_shape\', \'truth_indices\', \'truth_values\', \'truth_shape\', \'normalize\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Elu"
-    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "EluGrad"
-    argspec: "args=[\'gradients\', \'outputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Empty"
-    argspec: "args=[\'shape\', \'dtype\', \'init\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "EmptyTensorList"
-    argspec: "args=[\'element_shape\', \'max_num_elements\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "EncodeBase64"
-    argspec: "args=[\'input\', \'pad\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "EncodeJpeg"
-    argspec: "args=[\'image\', \'format\', \'quality\', \'progressive\', \'optimize_size\', \'chroma_downsampling\', \'density_unit\', \'x_density\', \'y_density\', \'xmp_metadata\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "EncodePng"
-    argspec: "args=[\'image\', \'compression\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "EncodeWav"
-    argspec: "args=[\'audio\', \'sample_rate\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "EnsureShape"
-    argspec: "args=[\'input\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Enter"
-    argspec: "args=[\'data\', \'frame_name\', \'is_constant\', \'parallel_iterations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Equal"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Erf"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Erfc"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Exit"
-    argspec: "args=[\'data\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Exp"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExpandDims"
-    argspec: "args=[\'input\', \'dim\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalAssertNextDataset"
-    argspec: "args=[\'input_dataset\', \'transformations\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalBytesProducedStatsDataset"
-    argspec: "args=[\'input_dataset\', \'tag\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalCSVDataset"
-    argspec: "args=[\'filenames\', \'compression_type\', \'buffer_size\', \'header\', \'field_delim\', \'use_quote_delim\', \'na_value\', \'select_cols\', \'record_defaults\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalChooseFastestDataset"
-    argspec: "args=[\'input_datasets\', \'num_experiments\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalDatasetCardinality"
-    argspec: "args=[\'input_dataset\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalDatasetToTFRecord"
-    argspec: "args=[\'input_dataset\', \'filename\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalDenseToSparseBatchDataset"
-    argspec: "args=[\'input_dataset\', \'batch_size\', \'row_shape\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalDirectedInterleaveDataset"
-    argspec: "args=[\'selector_input_dataset\', \'data_input_datasets\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalGroupByReducerDataset"
-    argspec: "args=[\'input_dataset\', \'key_func_other_arguments\', \'init_func_other_arguments\', \'reduce_func_other_arguments\', \'finalize_func_other_arguments\', \'key_func\', \'init_func\', \'reduce_func\', \'finalize_func\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalGroupByWindowDataset"
-    argspec: "args=[\'input_dataset\', \'key_func_other_arguments\', \'reduce_func_other_arguments\', \'window_size_func_other_arguments\', \'key_func\', \'reduce_func\', \'window_size_func\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalIdentityIndexedDataset"
-    argspec: "args=[\'size\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalIgnoreErrorsDataset"
-    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalIndexedDatasetGet"
-    argspec: "args=[\'materialized\', \'index\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalIndexedDatasetMaterialize"
-    argspec: "args=[\'dataset\', \'materialized\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalIteratorGetDevice"
-    argspec: "args=[\'resource\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalLMDBDataset"
-    argspec: "args=[\'filenames\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalLatencyStatsDataset"
-    argspec: "args=[\'input_dataset\', \'tag\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalMapAndBatchDataset"
-    argspec: "args=[\'input_dataset\', \'other_arguments\', \'batch_size\', \'num_parallel_calls\', \'drop_remainder\', \'f\', \'output_types\', \'output_shapes\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalMapDataset"
-    argspec: "args=[\'input_dataset\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\', \'use_inter_op_parallelism\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalMatchingFilesDataset"
-    argspec: "args=[\'patterns\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalMaterializedIndexDatasetHandle"
-    argspec: "args=[\'container\', \'shared_name\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalMaxIntraOpParallelismDataset"
-    argspec: "args=[\'input_dataset\', \'max_intra_op_parallelism\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalNonSerializableDataset"
-    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalNumaMapAndBatchDataset"
-    argspec: "args=[\'input_dataset\', \'other_arguments\', \'batch_size\', \'num_parallel_calls\', \'drop_remainder\', \'f\', \'output_types\', \'output_shapes\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalParallelInterleaveDataset"
-    argspec: "args=[\'input_dataset\', \'other_arguments\', \'cycle_length\', \'block_length\', \'sloppy\', \'buffer_output_elements\', \'prefetch_input_elements\', \'f\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalParseExampleDataset"
-    argspec: "args=[\'input_dataset\', \'num_parallel_calls\', \'dense_defaults\', \'sparse_keys\', \'dense_keys\', \'sparse_types\', \'dense_shapes\', \'output_types\', \'output_shapes\', \'sloppy\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalPrivateThreadPoolDataset"
-    argspec: "args=[\'input_dataset\', \'num_threads\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalRandomDataset"
-    argspec: "args=[\'seed\', \'seed2\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalScanDataset"
-    argspec: "args=[\'input_dataset\', \'initial_state\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalSetStatsAggregatorDataset"
-    argspec: "args=[\'input_dataset\', \'stats_aggregator\', \'tag\', \'counter_prefix\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalSleepDataset"
-    argspec: "args=[\'input_dataset\', \'sleep_microseconds\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalSlidingWindowDataset"
-    argspec: "args=[\'input_dataset\', \'window_size\', \'window_shift\', \'window_stride\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalSqlDataset"
-    argspec: "args=[\'driver_name\', \'data_source_name\', \'query\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalStatsAggregatorHandle"
-    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalStatsAggregatorSummary"
-    argspec: "args=[\'iterator\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalTakeWhileDataset"
-    argspec: "args=[\'input_dataset\', \'other_arguments\', \'predicate\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalThreadPoolDataset"
-    argspec: "args=[\'input_dataset\', \'thread_pool\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalThreadPoolHandle"
-    argspec: "args=[\'num_threads\', \'max_intra_op_parallelism\', \'display_name\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalUnbatchDataset"
-    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalUniqueDataset"
-    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Expm1"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExtractGlimpse"
-    argspec: "args=[\'input\', \'size\', \'offsets\', \'centered\', \'normalized\', \'uniform_noise\', \'noise\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExtractImagePatches"
-    argspec: "args=[\'images\', \'ksizes\', \'strides\', \'rates\', \'padding\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExtractJpegShape"
-    argspec: "args=[\'contents\', \'output_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExtractVolumePatches"
-    argspec: "args=[\'input\', \'ksizes\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FFT"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FFT2D"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FFT3D"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FIFOQueue"
-    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FIFOQueueV2"
-    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Fact"
-    argspec: "args=[], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FakeParam"
-    argspec: "args=[\'dtype\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FakeQuantWithMinMaxArgs"
-    argspec: "args=[\'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FakeQuantWithMinMaxArgsGradient"
-    argspec: "args=[\'gradients\', \'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FakeQuantWithMinMaxVars"
-    argspec: "args=[\'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FakeQuantWithMinMaxVarsGradient"
-    argspec: "args=[\'gradients\', \'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FakeQuantWithMinMaxVarsPerChannel"
-    argspec: "args=[\'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FakeQuantWithMinMaxVarsPerChannelGradient"
-    argspec: "args=[\'gradients\', \'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FakeQueue"
-    argspec: "args=[\'resource\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Fill"
-    argspec: "args=[\'dims\', \'value\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FilterByLastComponentDataset"
-    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FilterDataset"
-    argspec: "args=[\'input_dataset\', \'other_arguments\', \'predicate\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FixedLengthRecordDataset"
-    argspec: "args=[\'filenames\', \'header_bytes\', \'record_bytes\', \'footer_bytes\', \'buffer_size\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FixedLengthRecordDatasetV2"
-    argspec: "args=[\'filenames\', \'header_bytes\', \'record_bytes\', \'footer_bytes\', \'buffer_size\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FixedLengthRecordReader"
-    argspec: "args=[\'header_bytes\', \'record_bytes\', \'footer_bytes\', \'hop_bytes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FixedLengthRecordReaderV2"
-    argspec: "args=[\'header_bytes\', \'record_bytes\', \'footer_bytes\', \'hop_bytes\', \'container\', \'shared_name\', \'encoding\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FixedUnigramCandidateSampler"
-    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'vocab_file\', \'distortion\', \'num_reserved_ids\', \'num_shards\', \'shard\', \'unigrams\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FlatMapDataset"
-    argspec: "args=[\'input_dataset\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Floor"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FloorDiv"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FloorMod"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FlushSummaryWriter"
-    argspec: "args=[\'writer\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "For"
-    argspec: "args=[\'start\', \'limit\', \'delta\', \'input\', \'body\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FractionalAvgPool"
-    argspec: "args=[\'value\', \'pooling_ratio\', \'pseudo_random\', \'overlapping\', \'deterministic\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FractionalAvgPoolGrad"
-    argspec: "args=[\'orig_input_tensor_shape\', \'out_backprop\', \'row_pooling_sequence\', \'col_pooling_sequence\', \'overlapping\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FractionalMaxPool"
-    argspec: "args=[\'value\', \'pooling_ratio\', \'pseudo_random\', \'overlapping\', \'deterministic\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FractionalMaxPoolGrad"
-    argspec: "args=[\'orig_input\', \'orig_output\', \'out_backprop\', \'row_pooling_sequence\', \'col_pooling_sequence\', \'overlapping\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FusedBatchNorm"
-    argspec: "args=[\'x\', \'scale\', \'offset\', \'mean\', \'variance\', \'epsilon\', \'data_format\', \'is_training\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FusedBatchNormGrad"
-    argspec: "args=[\'y_backprop\', \'x\', \'scale\', \'reserve_space_1\', \'reserve_space_2\', \'epsilon\', \'data_format\', \'is_training\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FusedBatchNormGradV2"
-    argspec: "args=[\'y_backprop\', \'x\', \'scale\', \'reserve_space_1\', \'reserve_space_2\', \'epsilon\', \'data_format\', \'is_training\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FusedBatchNormV2"
-    argspec: "args=[\'x\', \'scale\', \'offset\', \'mean\', \'variance\', \'epsilon\', \'data_format\', \'is_training\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FusedPadConv2D"
-    argspec: "args=[\'input\', \'paddings\', \'filter\', \'mode\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FusedResizeAndPadConv2D"
-    argspec: "args=[\'input\', \'size\', \'paddings\', \'filter\', \'resize_align_corners\', \'mode\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Gather"
-    argspec: "args=[\'params\', \'indices\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "GatherNd"
-    argspec: "args=[\'params\', \'indices\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "GatherV2"
-    argspec: "args=[\'params\', \'indices\', \'axis\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "GenerateVocabRemapping"
-    argspec: "args=[\'new_vocab_file\', \'old_vocab_file\', \'new_vocab_offset\', \'num_new_vocab\', \'old_vocab_size\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "GeneratorDataset"
-    argspec: "args=[\'init_func_other_args\', \'next_func_other_args\', \'finalize_func_other_args\', \'init_func\', \'next_func\', \'finalize_func\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "GetSessionHandle"
-    argspec: "args=[\'value\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "GetSessionHandleV2"
-    argspec: "args=[\'value\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "GetSessionTensor"
-    argspec: "args=[\'handle\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Greater"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "GreaterEqual"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "GuaranteeConst"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "HSVToRGB"
-    argspec: "args=[\'images\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "HashTable"
-    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "HashTableV2"
-    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "HistogramFixedWidth"
-    argspec: "args=[\'values\', \'value_range\', \'nbins\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "HistogramSummary"
-    argspec: "args=[\'tag\', \'values\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IFFT"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IFFT2D"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IFFT3D"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IRFFT"
-    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IRFFT2D"
-    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IRFFT3D"
-    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Identity"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IdentityN"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IdentityReader"
-    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IdentityReaderV2"
-    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "If"
-    argspec: "args=[\'cond\', \'input\', \'Tout\', \'then_branch\', \'else_branch\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Igamma"
-    argspec: "args=[\'a\', \'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IgammaGradA"
-    argspec: "args=[\'a\', \'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Igammac"
-    argspec: "args=[\'a\', \'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Imag"
-    argspec: "args=[\'input\', \'Tout\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ImageSummary"
-    argspec: "args=[\'tag\', \'tensor\', \'max_images\', \'bad_color\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ImmutableConst"
-    argspec: "args=[\'dtype\', \'shape\', \'memory_region_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ImportEvent"
-    argspec: "args=[\'writer\', \'event\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "InTopK"
-    argspec: "args=[\'predictions\', \'targets\', \'k\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "InTopKV2"
-    argspec: "args=[\'predictions\', \'targets\', \'k\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "InitializeTable"
-    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "InitializeTableFromTextFile"
-    argspec: "args=[\'table_handle\', \'filename\', \'key_index\', \'value_index\', \'vocab_size\', \'delimiter\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "InitializeTableFromTextFileV2"
-    argspec: "args=[\'table_handle\', \'filename\', \'key_index\', \'value_index\', \'vocab_size\', \'delimiter\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "InitializeTableV2"
-    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "InplaceAdd"
-    argspec: "args=[\'x\', \'i\', \'v\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "InplaceSub"
-    argspec: "args=[\'x\', \'i\', \'v\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "InplaceUpdate"
-    argspec: "args=[\'x\', \'i\', \'v\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "InterleaveDataset"
-    argspec: "args=[\'input_dataset\', \'other_arguments\', \'cycle_length\', \'block_length\', \'f\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Inv"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "InvGrad"
-    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Invert"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "InvertPermutation"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IsBoostedTreesEnsembleInitialized"
-    argspec: "args=[\'tree_ensemble_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IsBoostedTreesQuantileStreamResourceInitialized"
-    argspec: "args=[\'quantile_stream_resource_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IsFinite"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IsInf"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IsNan"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IsVariableInitialized"
-    argspec: "args=[\'ref\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Iterator"
-    argspec: "args=[\'shared_name\', \'container\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IteratorFromStringHandle"
-    argspec: "args=[\'string_handle\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IteratorFromStringHandleV2"
-    argspec: "args=[\'string_handle\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IteratorGetNext"
-    argspec: "args=[\'iterator\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IteratorGetNextAsOptional"
-    argspec: "args=[\'iterator\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IteratorGetNextSync"
-    argspec: "args=[\'iterator\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IteratorToStringHandle"
-    argspec: "args=[\'resource_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IteratorV2"
-    argspec: "args=[\'shared_name\', \'container\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "L2Loss"
-    argspec: "args=[\'t\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LMDBReader"
-    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LRN"
-    argspec: "args=[\'input\', \'depth_radius\', \'bias\', \'alpha\', \'beta\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LRNGrad"
-    argspec: "args=[\'input_grads\', \'input_image\', \'output_image\', \'depth_radius\', \'bias\', \'alpha\', \'beta\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LeakyRelu"
-    argspec: "args=[\'features\', \'alpha\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LeakyReluGrad"
-    argspec: "args=[\'gradients\', \'features\', \'alpha\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LearnedUnigramCandidateSampler"
-    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LeftShift"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Less"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LessEqual"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Lgamma"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LinSpace"
-    argspec: "args=[\'start\', \'stop\', \'num\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ListDiff"
-    argspec: "args=[\'x\', \'y\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LoadAndRemapMatrix"
-    argspec: "args=[\'ckpt_path\', \'old_tensor_name\', \'row_remapping\', \'col_remapping\', \'initializing_values\', \'num_rows\', \'num_cols\', \'max_rows_in_memory\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Log"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Log1p"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LogMatrixDeterminant"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LogSoftmax"
-    argspec: "args=[\'logits\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LogUniformCandidateSampler"
-    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LogicalAnd"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LogicalNot"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LogicalOr"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LookupTableExport"
-    argspec: "args=[\'table_handle\', \'Tkeys\', \'Tvalues\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LookupTableExportV2"
-    argspec: "args=[\'table_handle\', \'Tkeys\', \'Tvalues\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LookupTableFind"
-    argspec: "args=[\'table_handle\', \'keys\', \'default_value\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LookupTableFindV2"
-    argspec: "args=[\'table_handle\', \'keys\', \'default_value\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LookupTableImport"
-    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LookupTableImportV2"
-    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LookupTableInsert"
-    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LookupTableInsertV2"
-    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LookupTableRemoveV2"
-    argspec: "args=[\'table_handle\', \'keys\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LookupTableSize"
-    argspec: "args=[\'table_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LookupTableSizeV2"
-    argspec: "args=[\'table_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LoopCond"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LowerBound"
-    argspec: "args=[\'sorted_inputs\', \'values\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Lu"
-    argspec: "args=[\'input\', \'output_idx_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MakeIterator"
-    argspec: "args=[\'dataset\', \'iterator\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MapClear"
-    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MapDataset"
-    argspec: "args=[\'input_dataset\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\', \'use_inter_op_parallelism\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MapDefun"
-    argspec: "args=[\'arguments\', \'captured_inputs\', \'output_types\', \'output_shapes\', \'f\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MapIncompleteSize"
-    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MapPeek"
-    argspec: "args=[\'key\', \'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MapSize"
-    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MapStage"
-    argspec: "args=[\'key\', \'indices\', \'values\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MapUnstage"
-    argspec: "args=[\'key\', \'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MapUnstageNoKey"
-    argspec: "args=[\'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MatMul"
-    argspec: "args=[\'a\', \'b\', \'transpose_a\', \'transpose_b\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MatchingFiles"
-    argspec: "args=[\'pattern\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MatrixBandPart"
-    argspec: "args=[\'input\', \'num_lower\', \'num_upper\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MatrixDeterminant"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MatrixDiag"
-    argspec: "args=[\'diagonal\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MatrixDiagPart"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MatrixExponential"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MatrixInverse"
-    argspec: "args=[\'input\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MatrixLogarithm"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MatrixSetDiag"
-    argspec: "args=[\'input\', \'diagonal\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MatrixSolve"
-    argspec: "args=[\'matrix\', \'rhs\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MatrixSolveLs"
-    argspec: "args=[\'matrix\', \'rhs\', \'l2_regularizer\', \'fast\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MatrixSquareRoot"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MatrixTriangularSolve"
-    argspec: "args=[\'matrix\', \'rhs\', \'lower\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Max"
-    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MaxPool"
-    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MaxPool3D"
-    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MaxPool3DGrad"
-    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MaxPool3DGradGrad"
-    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MaxPoolGrad"
-    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MaxPoolGradGrad"
-    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MaxPoolGradGradV2"
-    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MaxPoolGradGradWithArgmax"
-    argspec: "args=[\'input\', \'grad\', \'argmax\', \'ksize\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MaxPoolGradV2"
-    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MaxPoolGradWithArgmax"
-    argspec: "args=[\'input\', \'grad\', \'argmax\', \'ksize\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MaxPoolV2"
-    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MaxPoolWithArgmax"
-    argspec: "args=[\'input\', \'ksize\', \'strides\', \'Targmax\', \'padding\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Maximum"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Mean"
-    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Merge"
-    argspec: "args=[\'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MergeSummary"
-    argspec: "args=[\'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MergeV2Checkpoints"
-    argspec: "args=[\'checkpoint_prefixes\', \'destination_prefix\', \'delete_old_dirs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Mfcc"
-    argspec: "args=[\'spectrogram\', \'sample_rate\', \'upper_frequency_limit\', \'lower_frequency_limit\', \'filterbank_channel_count\', \'dct_coefficient_count\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Min"
-    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Minimum"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MirrorPad"
-    argspec: "args=[\'input\', \'paddings\', \'mode\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MirrorPadGrad"
-    argspec: "args=[\'input\', \'paddings\', \'mode\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Mod"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ModelDataset"
-    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Mul"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MultiDeviceIterator"
-    argspec: "args=[\'devices\', \'shared_name\', \'container\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MultiDeviceIteratorFromStringHandle"
-    argspec: "args=[\'string_handle\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MultiDeviceIteratorGetNextFromShard"
-    argspec: "args=[\'multi_device_iterator\', \'shard_num\', \'incarnation_id\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MultiDeviceIteratorInit"
-    argspec: "args=[\'dataset\', \'multi_device_iterator\', \'max_buffer_size\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MultiDeviceIteratorToStringHandle"
-    argspec: "args=[\'multi_device_iterator\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Multinomial"
-    argspec: "args=[\'logits\', \'num_samples\', \'seed\', \'seed2\', \'output_dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MutableDenseHashTable"
-    argspec: "args=[\'empty_key\', \'container\', \'shared_name\', \'use_node_name_sharing\', \'value_dtype\', \'value_shape\', \'initial_num_buckets\', \'max_load_factor\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MutableDenseHashTableV2"
-    argspec: "args=[\'empty_key\', \'deleted_key\', \'container\', \'shared_name\', \'use_node_name_sharing\', \'value_dtype\', \'value_shape\', \'initial_num_buckets\', \'max_load_factor\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MutableHashTable"
-    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MutableHashTableOfTensors"
-    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\', \'value_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MutableHashTableOfTensorsV2"
-    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\', \'value_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MutableHashTableV2"
-    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MutexLock"
-    argspec: "args=[\'mutex\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MutexV2"
-    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "NcclAllReduce"
-    argspec: "args=[\'input\', \'reduction\', \'num_devices\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "NcclBroadcast"
-    argspec: "args=[\'input\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "NcclReduce"
-    argspec: "args=[\'input\', \'reduction\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Neg"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "NextAfter"
-    argspec: "args=[\'x1\', \'x2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "NextIteration"
-    argspec: "args=[\'data\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "NoOp"
-    argspec: "args=[], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "NonMaxSuppression"
-    argspec: "args=[\'boxes\', \'scores\', \'max_output_size\', \'iou_threshold\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "NonMaxSuppressionV2"
-    argspec: "args=[\'boxes\', \'scores\', \'max_output_size\', \'iou_threshold\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "NonMaxSuppressionV3"
-    argspec: "args=[\'boxes\', \'scores\', \'max_output_size\', \'iou_threshold\', \'score_threshold\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "NonMaxSuppressionV4"
-    argspec: "args=[\'boxes\', \'scores\', \'max_output_size\', \'iou_threshold\', \'score_threshold\', \'pad_to_max_output_size\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "NonMaxSuppressionWithOverlaps"
-    argspec: "args=[\'overlaps\', \'scores\', \'max_output_size\', \'overlap_threshold\', \'score_threshold\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "NotEqual"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "NthElement"
-    argspec: "args=[\'input\', \'n\', \'reverse\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OneHot"
-    argspec: "args=[\'indices\', \'depth\', \'on_value\', \'off_value\', \'axis\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OneShotIterator"
-    argspec: "args=[\'dataset_factory\', \'output_types\', \'output_shapes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OnesLike"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OptimizeDataset"
-    argspec: "args=[\'input_dataset\', \'optimizations\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OptionalFromValue"
-    argspec: "args=[\'components\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OptionalGetValue"
-    argspec: "args=[\'optional\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OptionalHasValue"
-    argspec: "args=[\'optional\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OptionalNone"
-    argspec: "args=[], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OrderedMapClear"
-    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OrderedMapIncompleteSize"
-    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OrderedMapPeek"
-    argspec: "args=[\'key\', \'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OrderedMapSize"
-    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OrderedMapStage"
-    argspec: "args=[\'key\', \'indices\', \'values\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OrderedMapUnstage"
-    argspec: "args=[\'key\', \'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OrderedMapUnstageNoKey"
-    argspec: "args=[\'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Pack"
-    argspec: "args=[\'values\', \'axis\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Pad"
-    argspec: "args=[\'input\', \'paddings\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PadV2"
-    argspec: "args=[\'input\', \'paddings\', \'constant_values\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PaddedBatchDataset"
-    argspec: "args=[\'input_dataset\', \'batch_size\', \'padded_shapes\', \'padding_values\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PaddedBatchDatasetV2"
-    argspec: "args=[\'input_dataset\', \'batch_size\', \'padded_shapes\', \'padding_values\', \'drop_remainder\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PaddingFIFOQueue"
-    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PaddingFIFOQueueV2"
-    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ParallelConcat"
-    argspec: "args=[\'values\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ParallelDynamicStitch"
-    argspec: "args=[\'indices\', \'data\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ParallelInterleaveDatasetV2"
-    argspec: "args=[\'input_dataset\', \'other_arguments\', \'cycle_length\', \'block_length\', \'num_parallel_calls\', \'f\', \'output_types\', \'output_shapes\', \'sloppy\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ParallelMapDataset"
-    argspec: "args=[\'input_dataset\', \'other_arguments\', \'num_parallel_calls\', \'f\', \'output_types\', \'output_shapes\', \'use_inter_op_parallelism\', \'sloppy\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ParameterizedTruncatedNormal"
-    argspec: "args=[\'shape\', \'means\', \'stdevs\', \'minvals\', \'maxvals\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ParseExample"
-    argspec: "args=[\'serialized\', \'names\', \'sparse_keys\', \'dense_keys\', \'dense_defaults\', \'sparse_types\', \'dense_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ParseSequenceExample"
-    argspec: "args=[\'serialized\', \'debug_name\', \'context_dense_defaults\', \'feature_list_dense_missing_assumed_empty\', \'context_sparse_keys\', \'context_dense_keys\', \'feature_list_sparse_keys\', \'feature_list_dense_keys\', \'Ncontext_sparse\', \'Ncontext_dense\', \'Nfeature_list_sparse\', \'Nfeature_list_dense\', \'context_sparse_types\', \'feature_list_dense_types\', \'context_dense_shapes\', \'feature_list_sparse_types\', \'feature_list_dense_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ParseSingleExample"
-    argspec: "args=[\'serialized\', \'dense_defaults\', \'num_sparse\', \'sparse_keys\', \'dense_keys\', \'sparse_types\', \'dense_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ParseSingleSequenceExample"
-    argspec: "args=[\'serialized\', \'feature_list_dense_missing_assumed_empty\', \'context_sparse_keys\', \'context_dense_keys\', \'feature_list_sparse_keys\', \'feature_list_dense_keys\', \'context_dense_defaults\', \'debug_name\', \'context_sparse_types\', \'feature_list_dense_types\', \'context_dense_shapes\', \'feature_list_sparse_types\', \'feature_list_dense_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ParseTensor"
-    argspec: "args=[\'serialized\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PartitionedCall"
-    argspec: "args=[\'args\', \'Tout\', \'f\', \'config\', \'config_proto\', \'executor_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Placeholder"
-    argspec: "args=[\'dtype\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PlaceholderV2"
-    argspec: "args=[\'dtype\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PlaceholderWithDefault"
-    argspec: "args=[\'input\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Polygamma"
-    argspec: "args=[\'a\', \'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PopulationCount"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Pow"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PrefetchDataset"
-    argspec: "args=[\'input_dataset\', \'buffer_size\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PreventGradient"
-    argspec: "args=[\'input\', \'message\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Print"
-    argspec: "args=[\'input\', \'data\', \'message\', \'first_n\', \'summarize\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PrintV2"
-    argspec: "args=[\'input\', \'output_stream\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PriorityQueue"
-    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PriorityQueueV2"
-    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Prod"
-    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PyFunc"
-    argspec: "args=[\'input\', \'token\', \'Tout\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PyFuncStateless"
-    argspec: "args=[\'input\', \'token\', \'Tout\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Qr"
-    argspec: "args=[\'input\', \'full_matrices\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizeAndDequantize"
-    argspec: "args=[\'input\', \'signed_input\', \'num_bits\', \'range_given\', \'input_min\', \'input_max\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizeAndDequantizeV2"
-    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'signed_input\', \'num_bits\', \'range_given\', \'round_mode\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizeAndDequantizeV3"
-    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'num_bits\', \'signed_input\', \'range_given\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizeDownAndShrinkRange"
-    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizeV2"
-    argspec: "args=[\'input\', \'min_range\', \'max_range\', \'T\', \'mode\', \'round_mode\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedAdd"
-    argspec: "args=[\'x\', \'y\', \'min_x\', \'max_x\', \'min_y\', \'max_y\', \'Toutput\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedAvgPool"
-    argspec: "args=[\'input\', \'min_input\', \'max_input\', \'ksize\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedBatchNormWithGlobalNormalization"
-    argspec: "args=[\'t\', \'t_min\', \'t_max\', \'m\', \'m_min\', \'m_max\', \'v\', \'v_min\', \'v_max\', \'beta\', \'beta_min\', \'beta_max\', \'gamma\', \'gamma_min\', \'gamma_max\', \'out_type\', \'variance_epsilon\', \'scale_after_normalization\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedBiasAdd"
-    argspec: "args=[\'input\', \'bias\', \'min_input\', \'max_input\', \'min_bias\', \'max_bias\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedConcat"
-    argspec: "args=[\'concat_dim\', \'values\', \'input_mins\', \'input_maxes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedConv2D"
-    argspec: "args=[\'input\', \'filter\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedConv2DAndRelu"
-    argspec: "args=[\'input\', \'filter\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedConv2DAndReluAndRequantize"
-    argspec: "args=[\'input\', \'filter\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedConv2DAndRequantize"
-    argspec: "args=[\'input\', \'filter\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedConv2DWithBias"
-    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedConv2DWithBiasAndRelu"
-    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedConv2DWithBiasAndReluAndRequantize"
-    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedConv2DWithBiasAndRequantize"
-    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedConv2DWithBiasSignedSumAndReluAndRequantize"
-    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'summand\', \'min_summand\', \'max_summand\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedConv2DWithBiasSumAndRelu"
-    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'summand\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedConv2DWithBiasSumAndReluAndRequantize"
-    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'summand\', \'min_summand\', \'max_summand\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedInstanceNorm"
-    argspec: "args=[\'x\', \'x_min\', \'x_max\', \'output_range_given\', \'given_y_min\', \'given_y_max\', \'variance_epsilon\', \'min_separation\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedMatMul"
-    argspec: "args=[\'a\', \'b\', \'min_a\', \'max_a\', \'min_b\', \'max_b\', \'Toutput\', \'transpose_a\', \'transpose_b\', \'Tactivation\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedMaxPool"
-    argspec: "args=[\'input\', \'min_input\', \'max_input\', \'ksize\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedMul"
-    argspec: "args=[\'x\', \'y\', \'min_x\', \'max_x\', \'min_y\', \'max_y\', \'Toutput\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedRelu"
-    argspec: "args=[\'features\', \'min_features\', \'max_features\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedRelu6"
-    argspec: "args=[\'features\', \'min_features\', \'max_features\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedReluX"
-    argspec: "args=[\'features\', \'max_value\', \'min_features\', \'max_features\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedReshape"
-    argspec: "args=[\'tensor\', \'shape\', \'input_min\', \'input_max\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedResizeBilinear"
-    argspec: "args=[\'images\', \'size\', \'min\', \'max\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueClose"
-    argspec: "args=[\'handle\', \'cancel_pending_enqueues\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueCloseV2"
-    argspec: "args=[\'handle\', \'cancel_pending_enqueues\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueDequeue"
-    argspec: "args=[\'handle\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueDequeueMany"
-    argspec: "args=[\'handle\', \'n\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueDequeueManyV2"
-    argspec: "args=[\'handle\', \'n\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueDequeueUpTo"
-    argspec: "args=[\'handle\', \'n\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueDequeueUpToV2"
-    argspec: "args=[\'handle\', \'n\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueDequeueV2"
-    argspec: "args=[\'handle\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueEnqueue"
-    argspec: "args=[\'handle\', \'components\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueEnqueueMany"
-    argspec: "args=[\'handle\', \'components\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueEnqueueManyV2"
-    argspec: "args=[\'handle\', \'components\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueEnqueueV2"
-    argspec: "args=[\'handle\', \'components\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueIsClosed"
-    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueIsClosedV2"
-    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueSize"
-    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueSizeV2"
-    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RFFT"
-    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RFFT2D"
-    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RFFT3D"
-    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RGBToHSV"
-    argspec: "args=[\'images\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RaggedGather"
-    argspec: "args=[\'params_nested_splits\', \'params_dense_values\', \'indices\', \'OUTPUT_RAGGED_RANK\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RaggedRange"
-    argspec: "args=[\'starts\', \'limits\', \'deltas\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RaggedTensorToSparse"
-    argspec: "args=[\'rt_nested_splits\', \'rt_dense_values\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RandomCrop"
-    argspec: "args=[\'image\', \'size\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RandomGamma"
-    argspec: "args=[\'shape\', \'alpha\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RandomGammaGrad"
-    argspec: "args=[\'alpha\', \'sample\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RandomPoisson"
-    argspec: "args=[\'shape\', \'rate\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RandomPoissonV2"
-    argspec: "args=[\'shape\', \'rate\', \'seed\', \'seed2\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RandomShuffle"
-    argspec: "args=[\'value\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RandomShuffleQueue"
-    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'min_after_dequeue\', \'seed\', \'seed2\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RandomShuffleQueueV2"
-    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'min_after_dequeue\', \'seed\', \'seed2\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RandomStandardNormal"
-    argspec: "args=[\'shape\', \'seed\', \'seed2\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RandomUniform"
-    argspec: "args=[\'shape\', \'seed\', \'seed2\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RandomUniformInt"
-    argspec: "args=[\'shape\', \'minval\', \'maxval\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Range"
-    argspec: "args=[\'start\', \'limit\', \'delta\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RangeDataset"
-    argspec: "args=[\'start\', \'stop\', \'step\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Rank"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReadFile"
-    argspec: "args=[\'filename\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReadVariableOp"
-    argspec: "args=[\'resource\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReaderNumRecordsProduced"
-    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReaderNumRecordsProducedV2"
-    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReaderNumWorkUnitsCompleted"
-    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReaderNumWorkUnitsCompletedV2"
-    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReaderRead"
-    argspec: "args=[\'reader_handle\', \'queue_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReaderReadUpTo"
-    argspec: "args=[\'reader_handle\', \'queue_handle\', \'num_records\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReaderReadUpToV2"
-    argspec: "args=[\'reader_handle\', \'queue_handle\', \'num_records\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReaderReadV2"
-    argspec: "args=[\'reader_handle\', \'queue_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReaderReset"
-    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReaderResetV2"
-    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReaderRestoreState"
-    argspec: "args=[\'reader_handle\', \'state\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReaderRestoreStateV2"
-    argspec: "args=[\'reader_handle\', \'state\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReaderSerializeState"
-    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReaderSerializeStateV2"
-    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Real"
-    argspec: "args=[\'input\', \'Tout\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RealDiv"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Reciprocal"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReciprocalGrad"
-    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RecordInput"
-    argspec: "args=[\'file_pattern\', \'file_random_seed\', \'file_shuffle_shift_ratio\', \'file_buffer_size\', \'file_parallelism\', \'batch_size\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReduceDataset"
-    argspec: "args=[\'input_dataset\', \'initial_state\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\', \'use_inter_op_parallelism\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReduceJoin"
-    argspec: "args=[\'inputs\', \'reduction_indices\', \'keep_dims\', \'separator\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RefEnter"
-    argspec: "args=[\'data\', \'frame_name\', \'is_constant\', \'parallel_iterations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RefExit"
-    argspec: "args=[\'data\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RefIdentity"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RefMerge"
-    argspec: "args=[\'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RefNextIteration"
-    argspec: "args=[\'data\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RefSelect"
-    argspec: "args=[\'index\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RefSwitch"
-    argspec: "args=[\'data\', \'pred\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RegexFullMatch"
-    argspec: "args=[\'input\', \'pattern\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RegexReplace"
-    argspec: "args=[\'input\', \'pattern\', \'rewrite\', \'replace_global\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Relu"
-    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Relu6"
-    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Relu6Grad"
-    argspec: "args=[\'gradients\', \'features\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReluGrad"
-    argspec: "args=[\'gradients\', \'features\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RemoteCall"
-    argspec: "args=[\'target\', \'args\', \'Tout\', \'f\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RepeatDataset"
-    argspec: "args=[\'input_dataset\', \'count\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RequantizationRange"
-    argspec: "args=[\'input\', \'input_min\', \'input_max\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Requantize"
-    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'requested_output_min\', \'requested_output_max\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Reshape"
-    argspec: "args=[\'tensor\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResizeArea"
-    argspec: "args=[\'images\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResizeBicubic"
-    argspec: "args=[\'images\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResizeBicubicGrad"
-    argspec: "args=[\'grads\', \'original_image\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResizeBilinear"
-    argspec: "args=[\'images\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResizeBilinearGrad"
-    argspec: "args=[\'grads\', \'original_image\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResizeNearestNeighbor"
-    argspec: "args=[\'images\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResizeNearestNeighborGrad"
-    argspec: "args=[\'grads\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyAdaMax"
-    argspec: "args=[\'var\', \'m\', \'v\', \'beta1_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyAdadelta"
-    argspec: "args=[\'var\', \'accum\', \'accum_update\', \'lr\', \'rho\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyAdagrad"
-    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'use_locking\', \'update_slots\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyAdagradDA"
-    argspec: "args=[\'var\', \'gradient_accumulator\', \'gradient_squared_accumulator\', \'grad\', \'lr\', \'l1\', \'l2\', \'global_step\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyAdam"
-    argspec: "args=[\'var\', \'m\', \'v\', \'beta1_power\', \'beta2_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyAdamWithAmsgrad"
-    argspec: "args=[\'var\', \'m\', \'v\', \'vhat\', \'beta1_power\', \'beta2_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyAddSign"
-    argspec: "args=[\'var\', \'m\', \'lr\', \'alpha\', \'sign_decay\', \'beta\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyCenteredRMSProp"
-    argspec: "args=[\'var\', \'mg\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyFtrl"
-    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'lr\', \'l1\', \'l2\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyFtrlV2"
-    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'lr\', \'l1\', \'l2\', \'l2_shrinkage\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyGradientDescent"
-    argspec: "args=[\'var\', \'alpha\', \'delta\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyKerasMomentum"
-    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyMomentum"
-    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyPowerSign"
-    argspec: "args=[\'var\', \'m\', \'lr\', \'logbase\', \'sign_decay\', \'beta\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyProximalAdagrad"
-    argspec: "args=[\'var\', \'accum\', \'lr\', \'l1\', \'l2\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyProximalGradientDescent"
-    argspec: "args=[\'var\', \'alpha\', \'l1\', \'l2\', \'delta\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyRMSProp"
-    argspec: "args=[\'var\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceCountUpTo"
-    argspec: "args=[\'resource\', \'limit\', \'T\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceGather"
-    argspec: "args=[\'resource\', \'indices\', \'validate_indices\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceScatterAdd"
-    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceScatterDiv"
-    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceScatterMax"
-    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceScatterMin"
-    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceScatterMul"
-    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceScatterNdAdd"
-    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceScatterNdSub"
-    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceScatterNdUpdate"
-    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceScatterSub"
-    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceScatterUpdate"
-    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceSparseApplyAdadelta"
-    argspec: "args=[\'var\', \'accum\', \'accum_update\', \'lr\', \'rho\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceSparseApplyAdagrad"
-    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'use_locking\', \'update_slots\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceSparseApplyAdagradDA"
-    argspec: "args=[\'var\', \'gradient_accumulator\', \'gradient_squared_accumulator\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'global_step\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceSparseApplyCenteredRMSProp"
-    argspec: "args=[\'var\', \'mg\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceSparseApplyFtrl"
-    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceSparseApplyFtrlV2"
-    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'l2_shrinkage\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceSparseApplyKerasMomentum"
-    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceSparseApplyMomentum"
-    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceSparseApplyProximalAdagrad"
-    argspec: "args=[\'var\', \'accum\', \'lr\', \'l1\', \'l2\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceSparseApplyProximalGradientDescent"
-    argspec: "args=[\'var\', \'alpha\', \'l1\', \'l2\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceSparseApplyRMSProp"
-    argspec: "args=[\'var\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceStridedSliceAssign"
-    argspec: "args=[\'ref\', \'begin\', \'end\', \'strides\', \'value\', \'begin_mask\', \'end_mask\', \'ellipsis_mask\', \'new_axis_mask\', \'shrink_axis_mask\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Restore"
-    argspec: "args=[\'file_pattern\', \'tensor_name\', \'dt\', \'preferred_shard\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RestoreSlice"
-    argspec: "args=[\'file_pattern\', \'tensor_name\', \'shape_and_slice\', \'dt\', \'preferred_shard\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RestoreV2"
-    argspec: "args=[\'prefix\', \'tensor_names\', \'shape_and_slices\', \'dtypes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Reverse"
-    argspec: "args=[\'tensor\', \'dims\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReverseSequence"
-    argspec: "args=[\'input\', \'seq_lengths\', \'seq_dim\', \'batch_dim\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReverseV2"
-    argspec: "args=[\'tensor\', \'axis\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RightShift"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Rint"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Roll"
-    argspec: "args=[\'input\', \'shift\', \'axis\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Round"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Rsqrt"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RsqrtGrad"
-    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SampleDistortedBoundingBox"
-    argspec: "args=[\'image_size\', \'bounding_boxes\', \'seed\', \'seed2\', \'min_object_covered\', \'aspect_ratio_range\', \'area_range\', \'max_attempts\', \'use_image_if_no_bounding_boxes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SampleDistortedBoundingBoxV2"
-    argspec: "args=[\'image_size\', \'bounding_boxes\', \'min_object_covered\', \'seed\', \'seed2\', \'aspect_ratio_range\', \'area_range\', \'max_attempts\', \'use_image_if_no_bounding_boxes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Save"
-    argspec: "args=[\'filename\', \'tensor_names\', \'data\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SaveSlices"
-    argspec: "args=[\'filename\', \'tensor_names\', \'shapes_and_slices\', \'data\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SaveV2"
-    argspec: "args=[\'prefix\', \'tensor_names\', \'shape_and_slices\', \'tensors\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScalarSummary"
-    argspec: "args=[\'tags\', \'values\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScaleAndTranslate"
-    argspec: "args=[\'images\', \'size\', \'scale\', \'translation\', \'kernel_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScaleAndTranslateGrad"
-    argspec: "args=[\'grads\', \'original_image\', \'scale\', \'translation\', \'kernel_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScatterAdd"
-    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScatterDiv"
-    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScatterMax"
-    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScatterMin"
-    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScatterMul"
-    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScatterNd"
-    argspec: "args=[\'indices\', \'updates\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScatterNdAdd"
-    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScatterNdNonAliasingAdd"
-    argspec: "args=[\'input\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScatterNdSub"
-    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScatterNdUpdate"
-    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScatterSub"
-    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScatterUpdate"
-    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SdcaFprint"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SdcaOptimizer"
-    argspec: "args=[\'sparse_example_indices\', \'sparse_feature_indices\', \'sparse_feature_values\', \'dense_features\', \'example_weights\', \'example_labels\', \'sparse_indices\', \'sparse_weights\', \'dense_weights\', \'example_state_data\', \'loss_type\', \'adaptative\', \'l1\', \'l2\', \'num_loss_partitions\', \'num_inner_iterations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SdcaOptimizerV2"
-    argspec: "args=[\'sparse_example_indices\', \'sparse_feature_indices\', \'sparse_feature_values\', \'dense_features\', \'example_weights\', \'example_labels\', \'sparse_indices\', \'sparse_weights\', \'dense_weights\', \'example_state_data\', \'loss_type\', \'adaptive\', \'l1\', \'l2\', \'num_loss_partitions\', \'num_inner_iterations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SdcaShrinkL1"
-    argspec: "args=[\'weights\', \'l1\', \'l2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SegmentMax"
-    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SegmentMean"
-    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SegmentMin"
-    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SegmentProd"
-    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SegmentSum"
-    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Select"
-    argspec: "args=[\'condition\', \'t\', \'e\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SelfAdjointEig"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SelfAdjointEigV2"
-    argspec: "args=[\'input\', \'compute_v\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Selu"
-    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SeluGrad"
-    argspec: "args=[\'gradients\', \'outputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SerializeIterator"
-    argspec: "args=[\'resource_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SerializeManySparse"
-    argspec: "args=[\'sparse_indices\', \'sparse_values\', \'sparse_shape\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SerializeSparse"
-    argspec: "args=[\'sparse_indices\', \'sparse_values\', \'sparse_shape\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SerializeTensor"
-    argspec: "args=[\'tensor\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SetSize"
-    argspec: "args=[\'set_indices\', \'set_values\', \'set_shape\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Shape"
-    argspec: "args=[\'input\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ShapeN"
-    argspec: "args=[\'input\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ShardedFilename"
-    argspec: "args=[\'basename\', \'shard\', \'num_shards\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ShardedFilespec"
-    argspec: "args=[\'basename\', \'num_shards\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ShuffleAndRepeatDataset"
-    argspec: "args=[\'input_dataset\', \'buffer_size\', \'seed\', \'seed2\', \'count\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ShuffleDataset"
-    argspec: "args=[\'input_dataset\', \'buffer_size\', \'seed\', \'seed2\', \'reshuffle_each_iteration\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Sigmoid"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SigmoidGrad"
-    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Sign"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Sin"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Sinh"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Size"
-    argspec: "args=[\'input\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SkipDataset"
-    argspec: "args=[\'input_dataset\', \'count\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Slice"
-    argspec: "args=[\'input\', \'begin\', \'size\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Snapshot"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Softmax"
-    argspec: "args=[\'logits\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SoftmaxCrossEntropyWithLogits"
-    argspec: "args=[\'features\', \'labels\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Softplus"
-    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SoftplusGrad"
-    argspec: "args=[\'gradients\', \'features\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Softsign"
-    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SoftsignGrad"
-    argspec: "args=[\'gradients\', \'features\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SpaceToBatch"
-    argspec: "args=[\'input\', \'paddings\', \'block_size\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SpaceToBatchND"
-    argspec: "args=[\'input\', \'block_shape\', \'paddings\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SpaceToDepth"
-    argspec: "args=[\'input\', \'block_size\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseAccumulatorApplyGradient"
-    argspec: "args=[\'handle\', \'local_step\', \'gradient_indices\', \'gradient_values\', \'gradient_shape\', \'has_known_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseAccumulatorTakeGradient"
-    argspec: "args=[\'handle\', \'num_required\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseAdd"
-    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b_indices\', \'b_values\', \'b_shape\', \'thresh\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseAddGrad"
-    argspec: "args=[\'backprop_val_grad\', \'a_indices\', \'b_indices\', \'sum_indices\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseApplyAdadelta"
-    argspec: "args=[\'var\', \'accum\', \'accum_update\', \'lr\', \'rho\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseApplyAdagrad"
-    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'use_locking\', \'update_slots\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseApplyAdagradDA"
-    argspec: "args=[\'var\', \'gradient_accumulator\', \'gradient_squared_accumulator\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'global_step\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseApplyCenteredRMSProp"
-    argspec: "args=[\'var\', \'mg\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseApplyFtrl"
-    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseApplyFtrlV2"
-    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'l2_shrinkage\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseApplyMomentum"
-    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseApplyProximalAdagrad"
-    argspec: "args=[\'var\', \'accum\', \'lr\', \'l1\', \'l2\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseApplyProximalGradientDescent"
-    argspec: "args=[\'var\', \'alpha\', \'l1\', \'l2\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseApplyRMSProp"
-    argspec: "args=[\'var\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseConcat"
-    argspec: "args=[\'indices\', \'values\', \'shapes\', \'concat_dim\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseConditionalAccumulator"
-    argspec: "args=[\'dtype\', \'shape\', \'container\', \'shared_name\', \'reduction_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseCross"
-    argspec: "args=[\'indices\', \'values\', \'shapes\', \'dense_inputs\', \'hashed_output\', \'num_buckets\', \'hash_key\', \'out_type\', \'internal_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseDenseCwiseAdd"
-    argspec: "args=[\'sp_indices\', \'sp_values\', \'sp_shape\', \'dense\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseDenseCwiseDiv"
-    argspec: "args=[\'sp_indices\', \'sp_values\', \'sp_shape\', \'dense\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseDenseCwiseMul"
-    argspec: "args=[\'sp_indices\', \'sp_values\', \'sp_shape\', \'dense\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseFillEmptyRows"
-    argspec: "args=[\'indices\', \'values\', \'dense_shape\', \'default_value\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseFillEmptyRowsGrad"
-    argspec: "args=[\'reverse_index_map\', \'grad_values\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseMatMul"
-    argspec: "args=[\'a\', \'b\', \'transpose_a\', \'transpose_b\', \'a_is_sparse\', \'b_is_sparse\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseReduceMax"
-    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseReduceMaxSparse"
-    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseReduceSum"
-    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseReduceSumSparse"
-    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseReorder"
-    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseReshape"
-    argspec: "args=[\'input_indices\', \'input_shape\', \'new_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSegmentMean"
-    argspec: "args=[\'data\', \'indices\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSegmentMeanGrad"
-    argspec: "args=[\'grad\', \'indices\', \'segment_ids\', \'output_dim0\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSegmentMeanWithNumSegments"
-    argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSegmentSqrtN"
-    argspec: "args=[\'data\', \'indices\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSegmentSqrtNGrad"
-    argspec: "args=[\'grad\', \'indices\', \'segment_ids\', \'output_dim0\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSegmentSqrtNWithNumSegments"
-    argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSegmentSum"
-    argspec: "args=[\'data\', \'indices\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSegmentSumWithNumSegments"
-    argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSlice"
-    argspec: "args=[\'indices\', \'values\', \'shape\', \'start\', \'size\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSliceGrad"
-    argspec: "args=[\'backprop_val_grad\', \'input_indices\', \'input_start\', \'output_indices\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSoftmax"
-    argspec: "args=[\'sp_indices\', \'sp_values\', \'sp_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSoftmaxCrossEntropyWithLogits"
-    argspec: "args=[\'features\', \'labels\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSparseMaximum"
-    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b_indices\', \'b_values\', \'b_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSparseMinimum"
-    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b_indices\', \'b_values\', \'b_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSplit"
-    argspec: "args=[\'split_dim\', \'indices\', \'values\', \'shape\', \'num_split\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseTensorDenseAdd"
-    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseTensorDenseMatMul"
-    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b\', \'adjoint_a\', \'adjoint_b\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseTensorSliceDataset"
-    argspec: "args=[\'indices\', \'values\', \'dense_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseToDense"
-    argspec: "args=[\'sparse_indices\', \'output_shape\', \'sparse_values\', \'default_value\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseToSparseSetOperation"
-    argspec: "args=[\'set1_indices\', \'set1_values\', \'set1_shape\', \'set2_indices\', \'set2_values\', \'set2_shape\', \'set_operation\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Split"
-    argspec: "args=[\'split_dim\', \'value\', \'num_split\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SplitV"
-    argspec: "args=[\'value\', \'size_splits\', \'split_dim\', \'num_split\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Sqrt"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SqrtGrad"
-    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Square"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SquaredDifference"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Squeeze"
-    argspec: "args=[\'input\', \'squeeze_dims\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Stack"
-    argspec: "args=[\'elem_type\', \'stack_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StackClose"
-    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StackCloseV2"
-    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StackPop"
-    argspec: "args=[\'handle\', \'elem_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StackPopV2"
-    argspec: "args=[\'handle\', \'elem_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StackPush"
-    argspec: "args=[\'handle\', \'elem\', \'swap_memory\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StackPushV2"
-    argspec: "args=[\'handle\', \'elem\', \'swap_memory\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StackV2"
-    argspec: "args=[\'max_size\', \'elem_type\', \'stack_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Stage"
-    argspec: "args=[\'values\', \'capacity\', \'memory_limit\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StageClear"
-    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StagePeek"
-    argspec: "args=[\'index\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StageSize"
-    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StatefulPartitionedCall"
-    argspec: "args=[\'args\', \'Tout\', \'f\', \'config\', \'config_proto\', \'executor_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StatelessIf"
-    argspec: "args=[\'cond\', \'input\', \'Tout\', \'then_branch\', \'else_branch\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StatelessMultinomial"
-    argspec: "args=[\'logits\', \'num_samples\', \'seed\', \'output_dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StatelessRandomNormal"
-    argspec: "args=[\'shape\', \'seed\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StatelessRandomUniform"
-    argspec: "args=[\'shape\', \'seed\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StatelessRandomUniformInt"
-    argspec: "args=[\'shape\', \'seed\', \'minval\', \'maxval\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StatelessTruncatedNormal"
-    argspec: "args=[\'shape\', \'seed\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StatelessWhile"
-    argspec: "args=[\'input\', \'cond\', \'body\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StaticRegexFullMatch"
-    argspec: "args=[\'input\', \'pattern\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StaticRegexReplace"
-    argspec: "args=[\'input\', \'pattern\', \'rewrite\', \'replace_global\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StopGradient"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StridedSlice"
-    argspec: "args=[\'input\', \'begin\', \'end\', \'strides\', \'begin_mask\', \'end_mask\', \'ellipsis_mask\', \'new_axis_mask\', \'shrink_axis_mask\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StridedSliceAssign"
-    argspec: "args=[\'ref\', \'begin\', \'end\', \'strides\', \'value\', \'begin_mask\', \'end_mask\', \'ellipsis_mask\', \'new_axis_mask\', \'shrink_axis_mask\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StridedSliceGrad"
-    argspec: "args=[\'shape\', \'begin\', \'end\', \'strides\', \'dy\', \'begin_mask\', \'end_mask\', \'ellipsis_mask\', \'new_axis_mask\', \'shrink_axis_mask\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StringFormat"
-    argspec: "args=[\'inputs\', \'template\', \'placeholder\', \'summarize\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StringJoin"
-    argspec: "args=[\'inputs\', \'separator\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StringLength"
-    argspec: "args=[\'input\', \'unit\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StringSplit"
-    argspec: "args=[\'input\', \'delimiter\', \'skip_empty\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StringSplitV2"
-    argspec: "args=[\'input\', \'sep\', \'maxsplit\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StringStrip"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StringToHashBucket"
-    argspec: "args=[\'string_tensor\', \'num_buckets\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StringToHashBucketFast"
-    argspec: "args=[\'input\', \'num_buckets\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StringToHashBucketStrong"
-    argspec: "args=[\'input\', \'num_buckets\', \'key\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StringToNumber"
-    argspec: "args=[\'string_tensor\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Sub"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Substr"
-    argspec: "args=[\'input\', \'pos\', \'len\', \'unit\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Sum"
-    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SummaryWriter"
-    argspec: "args=[\'shared_name\', \'container\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Svd"
-    argspec: "args=[\'input\', \'compute_uv\', \'full_matrices\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Switch"
-    argspec: "args=[\'data\', \'pred\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SymbolicGradient"
-    argspec: "args=[\'input\', \'Tout\', \'f\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TFRecordDataset"
-    argspec: "args=[\'filenames\', \'compression_type\', \'buffer_size\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TFRecordReader"
-    argspec: "args=[\'container\', \'shared_name\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TFRecordReaderV2"
-    argspec: "args=[\'container\', \'shared_name\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TakeDataset"
-    argspec: "args=[\'input_dataset\', \'count\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TakeManySparseFromTensorsMap"
-    argspec: "args=[\'sparse_handles\', \'dtype\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Tan"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Tanh"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TanhGrad"
-    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TemporaryVariable"
-    argspec: "args=[\'shape\', \'dtype\', \'var_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArray"
-    argspec: "args=[\'size\', \'dtype\', \'dynamic_size\', \'clear_after_read\', \'tensor_array_name\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayClose"
-    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayCloseV2"
-    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayCloseV3"
-    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayConcat"
-    argspec: "args=[\'handle\', \'flow_in\', \'dtype\', \'element_shape_except0\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayConcatV2"
-    argspec: "args=[\'handle\', \'flow_in\', \'dtype\', \'element_shape_except0\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayConcatV3"
-    argspec: "args=[\'handle\', \'flow_in\', \'dtype\', \'element_shape_except0\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayGather"
-    argspec: "args=[\'handle\', \'indices\', \'flow_in\', \'dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayGatherV2"
-    argspec: "args=[\'handle\', \'indices\', \'flow_in\', \'dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayGatherV3"
-    argspec: "args=[\'handle\', \'indices\', \'flow_in\', \'dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayGrad"
-    argspec: "args=[\'handle\', \'flow_in\', \'source\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayGradV2"
-    argspec: "args=[\'handle\', \'flow_in\', \'source\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayGradV3"
-    argspec: "args=[\'handle\', \'flow_in\', \'source\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayGradWithShape"
-    argspec: "args=[\'handle\', \'flow_in\', \'shape_to_prepend\', \'source\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayPack"
-    argspec: "args=[\'handle\', \'flow_in\', \'dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayRead"
-    argspec: "args=[\'handle\', \'index\', \'flow_in\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayReadV2"
-    argspec: "args=[\'handle\', \'index\', \'flow_in\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayReadV3"
-    argspec: "args=[\'handle\', \'index\', \'flow_in\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayScatter"
-    argspec: "args=[\'handle\', \'indices\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayScatterV2"
-    argspec: "args=[\'handle\', \'indices\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayScatterV3"
-    argspec: "args=[\'handle\', \'indices\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArraySize"
-    argspec: "args=[\'handle\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArraySizeV2"
-    argspec: "args=[\'handle\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArraySizeV3"
-    argspec: "args=[\'handle\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArraySplit"
-    argspec: "args=[\'handle\', \'value\', \'lengths\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArraySplitV2"
-    argspec: "args=[\'handle\', \'value\', \'lengths\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArraySplitV3"
-    argspec: "args=[\'handle\', \'value\', \'lengths\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayUnpack"
-    argspec: "args=[\'handle\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayV2"
-    argspec: "args=[\'size\', \'dtype\', \'element_shape\', \'dynamic_size\', \'clear_after_read\', \'tensor_array_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayV3"
-    argspec: "args=[\'size\', \'dtype\', \'element_shape\', \'dynamic_size\', \'clear_after_read\', \'identical_element_shapes\', \'tensor_array_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayWrite"
-    argspec: "args=[\'handle\', \'index\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayWriteV2"
-    argspec: "args=[\'handle\', \'index\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayWriteV3"
-    argspec: "args=[\'handle\', \'index\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorDataset"
-    argspec: "args=[\'components\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListConcat"
-    argspec: "args=[\'input_handle\', \'element_dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListConcatLists"
-    argspec: "args=[\'input_a\', \'input_b\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListElementShape"
-    argspec: "args=[\'input_handle\', \'shape_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListFromTensor"
-    argspec: "args=[\'tensor\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListGather"
-    argspec: "args=[\'input_handle\', \'indices\', \'element_shape\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListGetItem"
-    argspec: "args=[\'input_handle\', \'index\', \'element_shape\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListLength"
-    argspec: "args=[\'input_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListPopBack"
-    argspec: "args=[\'input_handle\', \'element_shape\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListPushBack"
-    argspec: "args=[\'input_handle\', \'tensor\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListPushBackBatch"
-    argspec: "args=[\'input_handles\', \'tensor\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListReserve"
-    argspec: "args=[\'element_shape\', \'num_elements\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListResize"
-    argspec: "args=[\'input_handle\', \'size\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListScatter"
-    argspec: "args=[\'tensor\', \'indices\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListScatterV2"
-    argspec: "args=[\'tensor\', \'indices\', \'element_shape\', \'num_elements\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListSetItem"
-    argspec: "args=[\'input_handle\', \'index\', \'item\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListSplit"
-    argspec: "args=[\'tensor\', \'element_shape\', \'lengths\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListStack"
-    argspec: "args=[\'input_handle\', \'element_shape\', \'element_dtype\', \'num_elements\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorScatterAdd"
-    argspec: "args=[\'tensor\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorScatterSub"
-    argspec: "args=[\'tensor\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorScatterUpdate"
-    argspec: "args=[\'tensor\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorSliceDataset"
-    argspec: "args=[\'components\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorSummary"
-    argspec: "args=[\'tensor\', \'description\', \'labels\', \'display_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorSummaryV2"
-    argspec: "args=[\'tag\', \'tensor\', \'serialized_summary_metadata\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TextLineDataset"
-    argspec: "args=[\'filenames\', \'compression_type\', \'buffer_size\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TextLineReader"
-    argspec: "args=[\'skip_header_lines\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TextLineReaderV2"
-    argspec: "args=[\'skip_header_lines\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ThreadUnsafeUnigramCandidateSampler"
-    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Tile"
-    argspec: "args=[\'input\', \'multiples\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TileGrad"
-    argspec: "args=[\'input\', \'multiples\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Timestamp"
-    argspec: "args=[], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TopK"
-    argspec: "args=[\'input\', \'k\', \'sorted\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TopKV2"
-    argspec: "args=[\'input\', \'k\', \'sorted\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Transpose"
-    argspec: "args=[\'x\', \'perm\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TruncateDiv"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TruncateMod"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TruncatedNormal"
-    argspec: "args=[\'shape\', \'seed\', \'seed2\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UnicodeDecode"
-    argspec: "args=[\'input\', \'input_encoding\', \'errors\', \'replacement_char\', \'replace_control_characters\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UnicodeDecodeWithOffsets"
-    argspec: "args=[\'input\', \'input_encoding\', \'errors\', \'replacement_char\', \'replace_control_characters\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UnicodeEncode"
-    argspec: "args=[\'input_values\', \'input_splits\', \'errors\', \'output_encoding\', \'replacement_char\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UnicodeScript"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UnicodeTranscode"
-    argspec: "args=[\'input\', \'input_encoding\', \'output_encoding\', \'errors\', \'replacement_char\', \'replace_control_characters\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UniformCandidateSampler"
-    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Unique"
-    argspec: "args=[\'x\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UniqueV2"
-    argspec: "args=[\'x\', \'axis\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UniqueWithCounts"
-    argspec: "args=[\'x\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UniqueWithCountsV2"
-    argspec: "args=[\'x\', \'axis\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Unpack"
-    argspec: "args=[\'value\', \'num\', \'axis\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UnravelIndex"
-    argspec: "args=[\'indices\', \'dims\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UnsortedSegmentMax"
-    argspec: "args=[\'data\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UnsortedSegmentMin"
-    argspec: "args=[\'data\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UnsortedSegmentProd"
-    argspec: "args=[\'data\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UnsortedSegmentSum"
-    argspec: "args=[\'data\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Unstage"
-    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UnwrapDatasetVariant"
-    argspec: "args=[\'input_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UpperBound"
-    argspec: "args=[\'sorted_inputs\', \'values\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "VarHandleOp"
-    argspec: "args=[\'container\', \'shared_name\', \'dtype\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "VarIsInitializedOp"
-    argspec: "args=[\'resource\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Variable"
-    argspec: "args=[\'shape\', \'dtype\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "VariableShape"
-    argspec: "args=[\'input\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "VariableV2"
-    argspec: "args=[\'shape\', \'dtype\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Where"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "While"
-    argspec: "args=[\'input\', \'cond\', \'body\', \'output_shapes\', \'parallel_iterations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "WholeFileReader"
-    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "WholeFileReaderV2"
-    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "WindowDataset"
-    argspec: "args=[\'input_dataset\', \'size\', \'shift\', \'stride\', \'drop_remainder\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "WrapDatasetVariant"
-    argspec: "args=[\'input_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "WriteAudioSummary"
-    argspec: "args=[\'writer\', \'step\', \'tag\', \'tensor\', \'sample_rate\', \'max_outputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "WriteFile"
-    argspec: "args=[\'filename\', \'contents\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "WriteGraphSummary"
-    argspec: "args=[\'writer\', \'step\', \'tensor\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "WriteHistogramSummary"
-    argspec: "args=[\'writer\', \'step\', \'tag\', \'values\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "WriteImageSummary"
-    argspec: "args=[\'writer\', \'step\', \'tag\', \'tensor\', \'bad_color\', \'max_images\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "WriteScalarSummary"
-    argspec: "args=[\'writer\', \'step\', \'tag\', \'value\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "WriteSummary"
-    argspec: "args=[\'writer\', \'step\', \'tensor\', \'tag\', \'summary_metadata\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Xdivy"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Xlogy"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ZerosLike"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Zeta"
-    argspec: "args=[\'x\', \'q\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ZipDataset"
-    argspec: "args=[\'input_datasets\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index bb7ea2e13e..e1eef058f4 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -272,10 +272,6 @@ tf_module {
     name: "random_uniform_initializer"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "raw_ops"
-    mtype: "<type \'module\'>"
-  }
   member {
     name: "resource"
     mtype: "<class \'tensorflow.python.framework.dtypes.DType\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt
deleted file mode 100644
index c52a0cd41f..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt
+++ /dev/null
@@ -1,4023 +0,0 @@
-path: "tensorflow.raw_ops"
-tf_module {
-  member_method {
-    name: "Abort"
-    argspec: "args=[\'error_msg\', \'exit_without_error\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Abs"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AccumulateNV2"
-    argspec: "args=[\'inputs\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AccumulatorApplyGradient"
-    argspec: "args=[\'handle\', \'local_step\', \'gradient\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AccumulatorNumAccumulated"
-    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AccumulatorSetGlobalStep"
-    argspec: "args=[\'handle\', \'new_global_step\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AccumulatorTakeGradient"
-    argspec: "args=[\'handle\', \'num_required\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Acos"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Acosh"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Add"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AddManySparseToTensorsMap"
-    argspec: "args=[\'sparse_indices\', \'sparse_values\', \'sparse_shape\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AddN"
-    argspec: "args=[\'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AddSparseToTensorsMap"
-    argspec: "args=[\'sparse_indices\', \'sparse_values\', \'sparse_shape\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AddV2"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AdjustContrast"
-    argspec: "args=[\'images\', \'contrast_factor\', \'min_value\', \'max_value\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AdjustContrastv2"
-    argspec: "args=[\'images\', \'contrast_factor\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AdjustHue"
-    argspec: "args=[\'images\', \'delta\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AdjustSaturation"
-    argspec: "args=[\'images\', \'scale\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "All"
-    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AllCandidateSampler"
-    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Angle"
-    argspec: "args=[\'input\', \'Tout\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AnonymousIterator"
-    argspec: "args=[\'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Any"
-    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyAdaMax"
-    argspec: "args=[\'var\', \'m\', \'v\', \'beta1_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyAdadelta"
-    argspec: "args=[\'var\', \'accum\', \'accum_update\', \'lr\', \'rho\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyAdagrad"
-    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'use_locking\', \'update_slots\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyAdagradDA"
-    argspec: "args=[\'var\', \'gradient_accumulator\', \'gradient_squared_accumulator\', \'grad\', \'lr\', \'l1\', \'l2\', \'global_step\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyAdam"
-    argspec: "args=[\'var\', \'m\', \'v\', \'beta1_power\', \'beta2_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyAddSign"
-    argspec: "args=[\'var\', \'m\', \'lr\', \'alpha\', \'sign_decay\', \'beta\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyCenteredRMSProp"
-    argspec: "args=[\'var\', \'mg\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyFtrl"
-    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'lr\', \'l1\', \'l2\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyFtrlV2"
-    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'lr\', \'l1\', \'l2\', \'l2_shrinkage\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyGradientDescent"
-    argspec: "args=[\'var\', \'alpha\', \'delta\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyMomentum"
-    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyPowerSign"
-    argspec: "args=[\'var\', \'m\', \'lr\', \'logbase\', \'sign_decay\', \'beta\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyProximalAdagrad"
-    argspec: "args=[\'var\', \'accum\', \'lr\', \'l1\', \'l2\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyProximalGradientDescent"
-    argspec: "args=[\'var\', \'alpha\', \'l1\', \'l2\', \'delta\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApplyRMSProp"
-    argspec: "args=[\'var\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ApproximateEqual"
-    argspec: "args=[\'x\', \'y\', \'tolerance\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ArgMax"
-    argspec: "args=[\'input\', \'dimension\', \'output_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ArgMin"
-    argspec: "args=[\'input\', \'dimension\', \'output_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AsString"
-    argspec: "args=[\'input\', \'precision\', \'scientific\', \'shortest\', \'width\', \'fill\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Asin"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Asinh"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Assert"
-    argspec: "args=[\'condition\', \'data\', \'summarize\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Assign"
-    argspec: "args=[\'ref\', \'value\', \'validate_shape\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AssignAdd"
-    argspec: "args=[\'ref\', \'value\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AssignAddVariableOp"
-    argspec: "args=[\'resource\', \'value\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AssignSub"
-    argspec: "args=[\'ref\', \'value\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AssignSubVariableOp"
-    argspec: "args=[\'resource\', \'value\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AssignVariableOp"
-    argspec: "args=[\'resource\', \'value\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Atan"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Atan2"
-    argspec: "args=[\'y\', \'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Atanh"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AudioSpectrogram"
-    argspec: "args=[\'input\', \'window_size\', \'stride\', \'magnitude_squared\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AudioSummary"
-    argspec: "args=[\'tag\', \'tensor\', \'sample_rate\', \'max_outputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AudioSummaryV2"
-    argspec: "args=[\'tag\', \'tensor\', \'sample_rate\', \'max_outputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AvgPool"
-    argspec: "args=[\'value\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AvgPool3D"
-    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AvgPool3DGrad"
-    argspec: "args=[\'orig_input_shape\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "AvgPoolGrad"
-    argspec: "args=[\'orig_input_shape\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Barrier"
-    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BarrierClose"
-    argspec: "args=[\'handle\', \'cancel_pending_enqueues\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BarrierIncompleteSize"
-    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BarrierInsertMany"
-    argspec: "args=[\'handle\', \'keys\', \'values\', \'component_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BarrierReadySize"
-    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BarrierTakeMany"
-    argspec: "args=[\'handle\', \'num_elements\', \'component_types\', \'allow_small_batch\', \'wait_for_incomplete\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchCholesky"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchCholeskyGrad"
-    argspec: "args=[\'l\', \'grad\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchDataset"
-    argspec: "args=[\'input_dataset\', \'batch_size\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchDatasetV2"
-    argspec: "args=[\'input_dataset\', \'batch_size\', \'drop_remainder\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchFFT"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchFFT2D"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchFFT3D"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchIFFT"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchIFFT2D"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchIFFT3D"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchMatMul"
-    argspec: "args=[\'x\', \'y\', \'adj_x\', \'adj_y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchMatrixBandPart"
-    argspec: "args=[\'input\', \'num_lower\', \'num_upper\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchMatrixDeterminant"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchMatrixDiag"
-    argspec: "args=[\'diagonal\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchMatrixDiagPart"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchMatrixInverse"
-    argspec: "args=[\'input\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchMatrixSetDiag"
-    argspec: "args=[\'input\', \'diagonal\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchMatrixSolve"
-    argspec: "args=[\'matrix\', \'rhs\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchMatrixSolveLs"
-    argspec: "args=[\'matrix\', \'rhs\', \'l2_regularizer\', \'fast\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchMatrixTriangularSolve"
-    argspec: "args=[\'matrix\', \'rhs\', \'lower\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchNormWithGlobalNormalization"
-    argspec: "args=[\'t\', \'m\', \'v\', \'beta\', \'gamma\', \'variance_epsilon\', \'scale_after_normalization\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchNormWithGlobalNormalizationGrad"
-    argspec: "args=[\'t\', \'m\', \'v\', \'gamma\', \'backprop\', \'variance_epsilon\', \'scale_after_normalization\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchSelfAdjointEig"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchSelfAdjointEigV2"
-    argspec: "args=[\'input\', \'compute_v\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchSvd"
-    argspec: "args=[\'input\', \'compute_uv\', \'full_matrices\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchToSpace"
-    argspec: "args=[\'input\', \'crops\', \'block_size\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BatchToSpaceND"
-    argspec: "args=[\'input\', \'block_shape\', \'crops\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BesselI0e"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BesselI1e"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Betainc"
-    argspec: "args=[\'a\', \'b\', \'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BiasAdd"
-    argspec: "args=[\'value\', \'bias\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BiasAddGrad"
-    argspec: "args=[\'out_backprop\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BiasAddV1"
-    argspec: "args=[\'value\', \'bias\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Bincount"
-    argspec: "args=[\'arr\', \'size\', \'weights\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Bitcast"
-    argspec: "args=[\'input\', \'type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BitwiseAnd"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BitwiseOr"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BitwiseXor"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesBucketize"
-    argspec: "args=[\'float_values\', \'bucket_boundaries\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesCalculateBestGainsPerFeature"
-    argspec: "args=[\'node_id_range\', \'stats_summary_list\', \'l1\', \'l2\', \'tree_complexity\', \'min_node_weight\', \'max_splits\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesCenterBias"
-    argspec: "args=[\'tree_ensemble_handle\', \'mean_gradients\', \'mean_hessians\', \'l1\', \'l2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesCreateEnsemble"
-    argspec: "args=[\'tree_ensemble_handle\', \'stamp_token\', \'tree_ensemble_serialized\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesCreateQuantileStreamResource"
-    argspec: "args=[\'quantile_stream_resource_handle\', \'epsilon\', \'num_streams\', \'max_elements\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesDeserializeEnsemble"
-    argspec: "args=[\'tree_ensemble_handle\', \'stamp_token\', \'tree_ensemble_serialized\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesEnsembleResourceHandleOp"
-    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesExampleDebugOutputs"
-    argspec: "args=[\'tree_ensemble_handle\', \'bucketized_features\', \'logits_dimension\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesGetEnsembleStates"
-    argspec: "args=[\'tree_ensemble_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesMakeQuantileSummaries"
-    argspec: "args=[\'float_values\', \'example_weights\', \'epsilon\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesMakeStatsSummary"
-    argspec: "args=[\'node_ids\', \'gradients\', \'hessians\', \'bucketized_features_list\', \'max_splits\', \'num_buckets\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesPredict"
-    argspec: "args=[\'tree_ensemble_handle\', \'bucketized_features\', \'logits_dimension\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesQuantileStreamResourceAddSummaries"
-    argspec: "args=[\'quantile_stream_resource_handle\', \'summaries\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesQuantileStreamResourceDeserialize"
-    argspec: "args=[\'quantile_stream_resource_handle\', \'bucket_boundaries\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesQuantileStreamResourceFlush"
-    argspec: "args=[\'quantile_stream_resource_handle\', \'num_buckets\', \'generate_quantiles\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesQuantileStreamResourceGetBucketBoundaries"
-    argspec: "args=[\'quantile_stream_resource_handle\', \'num_features\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesQuantileStreamResourceHandleOp"
-    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesSerializeEnsemble"
-    argspec: "args=[\'tree_ensemble_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesTrainingPredict"
-    argspec: "args=[\'tree_ensemble_handle\', \'cached_tree_ids\', \'cached_node_ids\', \'bucketized_features\', \'logits_dimension\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BoostedTreesUpdateEnsemble"
-    argspec: "args=[\'tree_ensemble_handle\', \'feature_ids\', \'node_ids\', \'gains\', \'thresholds\', \'left_node_contribs\', \'right_node_contribs\', \'max_depth\', \'learning_rate\', \'pruning_mode\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BroadcastArgs"
-    argspec: "args=[\'s0\', \'s1\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BroadcastGradientArgs"
-    argspec: "args=[\'s0\', \'s1\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "BroadcastTo"
-    argspec: "args=[\'input\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Bucketize"
-    argspec: "args=[\'input\', \'boundaries\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CTCBeamSearchDecoder"
-    argspec: "args=[\'inputs\', \'sequence_length\', \'beam_width\', \'top_paths\', \'merge_repeated\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CTCGreedyDecoder"
-    argspec: "args=[\'inputs\', \'sequence_length\', \'merge_repeated\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CTCLoss"
-    argspec: "args=[\'inputs\', \'labels_indices\', \'labels_values\', \'sequence_length\', \'preprocess_collapse_repeated\', \'ctc_merge_repeated\', \'ignore_longer_outputs_than_inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CacheDataset"
-    argspec: "args=[\'input_dataset\', \'filename\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Cast"
-    argspec: "args=[\'x\', \'DstT\', \'Truncate\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Ceil"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CheckNumerics"
-    argspec: "args=[\'tensor\', \'message\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Cholesky"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CholeskyGrad"
-    argspec: "args=[\'l\', \'grad\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ClipByValue"
-    argspec: "args=[\'t\', \'clip_value_min\', \'clip_value_max\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CloseSummaryWriter"
-    argspec: "args=[\'writer\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CollectiveBcastRecv"
-    argspec: "args=[\'T\', \'group_size\', \'group_key\', \'instance_key\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CollectiveBcastSend"
-    argspec: "args=[\'input\', \'group_size\', \'group_key\', \'instance_key\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CollectiveReduce"
-    argspec: "args=[\'input\', \'group_size\', \'group_key\', \'instance_key\', \'merge_op\', \'final_op\', \'subdiv_offsets\', \'wait_for\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CombinedNonMaxSuppression"
-    argspec: "args=[\'boxes\', \'scores\', \'max_output_size_per_class\', \'max_total_size\', \'iou_threshold\', \'score_threshold\', \'pad_per_class\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CompareAndBitpack"
-    argspec: "args=[\'input\', \'threshold\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Complex"
-    argspec: "args=[\'real\', \'imag\', \'Tout\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ComplexAbs"
-    argspec: "args=[\'x\', \'Tout\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ComputeAccidentalHits"
-    argspec: "args=[\'true_classes\', \'sampled_candidates\', \'num_true\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Concat"
-    argspec: "args=[\'concat_dim\', \'values\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ConcatOffset"
-    argspec: "args=[\'concat_dim\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ConcatV2"
-    argspec: "args=[\'values\', \'axis\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ConcatenateDataset"
-    argspec: "args=[\'input_dataset\', \'another_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ConditionalAccumulator"
-    argspec: "args=[\'dtype\', \'shape\', \'container\', \'shared_name\', \'reduction_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Conj"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ConjugateTranspose"
-    argspec: "args=[\'x\', \'perm\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Const"
-    argspec: "args=[\'value\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ConsumeMutexLock"
-    argspec: "args=[\'mutex_lock\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ControlTrigger"
-    argspec: "args=[], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Conv2D"
-    argspec: "args=[\'input\', \'filter\', \'strides\', \'use_cudnn_on_gpu\', \'padding\', \'explicit_paddings\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Conv2DBackpropFilter"
-    argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'use_cudnn_on_gpu\', \'padding\', \'explicit_paddings\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Conv2DBackpropInput"
-    argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'use_cudnn_on_gpu\', \'padding\', \'explicit_paddings\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Conv3D"
-    argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Conv3DBackpropFilter"
-    argspec: "args=[\'input\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Conv3DBackpropFilterV2"
-    argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Conv3DBackpropInput"
-    argspec: "args=[\'input\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Conv3DBackpropInputV2"
-    argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Cos"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Cosh"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CountUpTo"
-    argspec: "args=[\'ref\', \'limit\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CreateSummaryDbWriter"
-    argspec: "args=[\'writer\', \'db_uri\', \'experiment_name\', \'run_name\', \'user_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CreateSummaryFileWriter"
-    argspec: "args=[\'writer\', \'logdir\', \'max_queue\', \'flush_millis\', \'filename_suffix\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CropAndResize"
-    argspec: "args=[\'image\', \'boxes\', \'box_ind\', \'crop_size\', \'method\', \'extrapolation_value\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CropAndResizeGradBoxes"
-    argspec: "args=[\'grads\', \'image\', \'boxes\', \'box_ind\', \'method\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CropAndResizeGradImage"
-    argspec: "args=[\'grads\', \'boxes\', \'box_ind\', \'image_size\', \'T\', \'method\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Cross"
-    argspec: "args=[\'a\', \'b\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CudnnRNN"
-    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\', \'is_training\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CudnnRNNBackprop"
-    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'output\', \'output_h\', \'output_c\', \'output_backprop\', \'output_h_backprop\', \'output_c_backprop\', \'reserve_space\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CudnnRNNBackpropV2"
-    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'output\', \'output_h\', \'output_c\', \'output_backprop\', \'output_h_backprop\', \'output_c_backprop\', \'reserve_space\', \'host_reserved\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CudnnRNNBackpropV3"
-    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'sequence_lengths\', \'output\', \'output_h\', \'output_c\', \'output_backprop\', \'output_h_backprop\', \'output_c_backprop\', \'reserve_space\', \'host_reserved\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CudnnRNNCanonicalToParams"
-    argspec: "args=[\'num_layers\', \'num_units\', \'input_size\', \'weights\', \'biases\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CudnnRNNParamsSize"
-    argspec: "args=[\'num_layers\', \'num_units\', \'input_size\', \'T\', \'S\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CudnnRNNParamsToCanonical"
-    argspec: "args=[\'num_layers\', \'num_units\', \'input_size\', \'params\', \'num_params\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CudnnRNNV2"
-    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\', \'is_training\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "CudnnRNNV3"
-    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'sequence_lengths\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\', \'is_training\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Cumprod"
-    argspec: "args=[\'x\', \'axis\', \'exclusive\', \'reverse\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Cumsum"
-    argspec: "args=[\'x\', \'axis\', \'exclusive\', \'reverse\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DataFormatDimMap"
-    argspec: "args=[\'x\', \'src_format\', \'dst_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DataFormatVecPermute"
-    argspec: "args=[\'x\', \'src_format\', \'dst_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DatasetToGraph"
-    argspec: "args=[\'input_dataset\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DatasetToSingleElement"
-    argspec: "args=[\'dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DebugGradientIdentity"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DebugGradientRefIdentity"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DecodeAndCropJpeg"
-    argspec: "args=[\'contents\', \'crop_window\', \'channels\', \'ratio\', \'fancy_upscaling\', \'try_recover_truncated\', \'acceptable_fraction\', \'dct_method\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DecodeBase64"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DecodeBmp"
-    argspec: "args=[\'contents\', \'channels\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DecodeCSV"
-    argspec: "args=[\'records\', \'record_defaults\', \'field_delim\', \'use_quote_delim\', \'na_value\', \'select_cols\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DecodeCompressed"
-    argspec: "args=[\'bytes\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DecodeGif"
-    argspec: "args=[\'contents\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DecodeJSONExample"
-    argspec: "args=[\'json_examples\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DecodeJpeg"
-    argspec: "args=[\'contents\', \'channels\', \'ratio\', \'fancy_upscaling\', \'try_recover_truncated\', \'acceptable_fraction\', \'dct_method\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DecodePng"
-    argspec: "args=[\'contents\', \'channels\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DecodeRaw"
-    argspec: "args=[\'bytes\', \'out_type\', \'little_endian\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DecodeWav"
-    argspec: "args=[\'contents\', \'desired_channels\', \'desired_samples\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DeepCopy"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DeleteSessionTensor"
-    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DenseToDenseSetOperation"
-    argspec: "args=[\'set1\', \'set2\', \'set_operation\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DenseToSparseSetOperation"
-    argspec: "args=[\'set1\', \'set2_indices\', \'set2_values\', \'set2_shape\', \'set_operation\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DepthToSpace"
-    argspec: "args=[\'input\', \'block_size\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DepthwiseConv2dNative"
-    argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DepthwiseConv2dNativeBackpropFilter"
-    argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DepthwiseConv2dNativeBackpropInput"
-    argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Dequantize"
-    argspec: "args=[\'input\', \'min_range\', \'max_range\', \'mode\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DeserializeIterator"
-    argspec: "args=[\'resource_handle\', \'serialized\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DeserializeManySparse"
-    argspec: "args=[\'serialized_sparse\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DeserializeSparse"
-    argspec: "args=[\'serialized_sparse\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DestroyResourceOp"
-    argspec: "args=[\'resource\', \'ignore_lookup_error\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DestroyTemporaryVariable"
-    argspec: "args=[\'ref\', \'var_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Diag"
-    argspec: "args=[\'diagonal\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DiagPart"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Digamma"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Dilation2D"
-    argspec: "args=[\'input\', \'filter\', \'strides\', \'rates\', \'padding\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Dilation2DBackpropFilter"
-    argspec: "args=[\'input\', \'filter\', \'out_backprop\', \'strides\', \'rates\', \'padding\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Dilation2DBackpropInput"
-    argspec: "args=[\'input\', \'filter\', \'out_backprop\', \'strides\', \'rates\', \'padding\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Div"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DivNoNan"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DrawBoundingBoxes"
-    argspec: "args=[\'images\', \'boxes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DynamicPartition"
-    argspec: "args=[\'data\', \'partitions\', \'num_partitions\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "DynamicStitch"
-    argspec: "args=[\'indices\', \'data\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "EagerPyFunc"
-    argspec: "args=[\'input\', \'token\', \'Tout\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "EditDistance"
-    argspec: "args=[\'hypothesis_indices\', \'hypothesis_values\', \'hypothesis_shape\', \'truth_indices\', \'truth_values\', \'truth_shape\', \'normalize\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Elu"
-    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "EluGrad"
-    argspec: "args=[\'gradients\', \'outputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Empty"
-    argspec: "args=[\'shape\', \'dtype\', \'init\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "EmptyTensorList"
-    argspec: "args=[\'element_shape\', \'max_num_elements\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "EncodeBase64"
-    argspec: "args=[\'input\', \'pad\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "EncodeJpeg"
-    argspec: "args=[\'image\', \'format\', \'quality\', \'progressive\', \'optimize_size\', \'chroma_downsampling\', \'density_unit\', \'x_density\', \'y_density\', \'xmp_metadata\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "EncodePng"
-    argspec: "args=[\'image\', \'compression\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "EncodeWav"
-    argspec: "args=[\'audio\', \'sample_rate\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "EnsureShape"
-    argspec: "args=[\'input\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Enter"
-    argspec: "args=[\'data\', \'frame_name\', \'is_constant\', \'parallel_iterations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Equal"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Erf"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Erfc"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Exit"
-    argspec: "args=[\'data\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Exp"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExpandDims"
-    argspec: "args=[\'input\', \'dim\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalAssertNextDataset"
-    argspec: "args=[\'input_dataset\', \'transformations\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalBytesProducedStatsDataset"
-    argspec: "args=[\'input_dataset\', \'tag\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalCSVDataset"
-    argspec: "args=[\'filenames\', \'compression_type\', \'buffer_size\', \'header\', \'field_delim\', \'use_quote_delim\', \'na_value\', \'select_cols\', \'record_defaults\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalChooseFastestDataset"
-    argspec: "args=[\'input_datasets\', \'num_experiments\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalDatasetCardinality"
-    argspec: "args=[\'input_dataset\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalDatasetToTFRecord"
-    argspec: "args=[\'input_dataset\', \'filename\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalDenseToSparseBatchDataset"
-    argspec: "args=[\'input_dataset\', \'batch_size\', \'row_shape\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalDirectedInterleaveDataset"
-    argspec: "args=[\'selector_input_dataset\', \'data_input_datasets\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalGroupByReducerDataset"
-    argspec: "args=[\'input_dataset\', \'key_func_other_arguments\', \'init_func_other_arguments\', \'reduce_func_other_arguments\', \'finalize_func_other_arguments\', \'key_func\', \'init_func\', \'reduce_func\', \'finalize_func\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalGroupByWindowDataset"
-    argspec: "args=[\'input_dataset\', \'key_func_other_arguments\', \'reduce_func_other_arguments\', \'window_size_func_other_arguments\', \'key_func\', \'reduce_func\', \'window_size_func\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalIdentityIndexedDataset"
-    argspec: "args=[\'size\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalIgnoreErrorsDataset"
-    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalIndexedDatasetGet"
-    argspec: "args=[\'materialized\', \'index\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalIndexedDatasetMaterialize"
-    argspec: "args=[\'dataset\', \'materialized\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalIteratorGetDevice"
-    argspec: "args=[\'resource\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalLMDBDataset"
-    argspec: "args=[\'filenames\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalLatencyStatsDataset"
-    argspec: "args=[\'input_dataset\', \'tag\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalMapAndBatchDataset"
-    argspec: "args=[\'input_dataset\', \'other_arguments\', \'batch_size\', \'num_parallel_calls\', \'drop_remainder\', \'f\', \'output_types\', \'output_shapes\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalMapDataset"
-    argspec: "args=[\'input_dataset\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\', \'use_inter_op_parallelism\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalMatchingFilesDataset"
-    argspec: "args=[\'patterns\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalMaterializedIndexDatasetHandle"
-    argspec: "args=[\'container\', \'shared_name\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalMaxIntraOpParallelismDataset"
-    argspec: "args=[\'input_dataset\', \'max_intra_op_parallelism\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalNonSerializableDataset"
-    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalNumaMapAndBatchDataset"
-    argspec: "args=[\'input_dataset\', \'other_arguments\', \'batch_size\', \'num_parallel_calls\', \'drop_remainder\', \'f\', \'output_types\', \'output_shapes\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalParallelInterleaveDataset"
-    argspec: "args=[\'input_dataset\', \'other_arguments\', \'cycle_length\', \'block_length\', \'sloppy\', \'buffer_output_elements\', \'prefetch_input_elements\', \'f\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalParseExampleDataset"
-    argspec: "args=[\'input_dataset\', \'num_parallel_calls\', \'dense_defaults\', \'sparse_keys\', \'dense_keys\', \'sparse_types\', \'dense_shapes\', \'output_types\', \'output_shapes\', \'sloppy\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalPrivateThreadPoolDataset"
-    argspec: "args=[\'input_dataset\', \'num_threads\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalRandomDataset"
-    argspec: "args=[\'seed\', \'seed2\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalScanDataset"
-    argspec: "args=[\'input_dataset\', \'initial_state\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalSetStatsAggregatorDataset"
-    argspec: "args=[\'input_dataset\', \'stats_aggregator\', \'tag\', \'counter_prefix\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalSleepDataset"
-    argspec: "args=[\'input_dataset\', \'sleep_microseconds\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalSlidingWindowDataset"
-    argspec: "args=[\'input_dataset\', \'window_size\', \'window_shift\', \'window_stride\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalSqlDataset"
-    argspec: "args=[\'driver_name\', \'data_source_name\', \'query\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalStatsAggregatorHandle"
-    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalStatsAggregatorSummary"
-    argspec: "args=[\'iterator\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalTakeWhileDataset"
-    argspec: "args=[\'input_dataset\', \'other_arguments\', \'predicate\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalThreadPoolDataset"
-    argspec: "args=[\'input_dataset\', \'thread_pool\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalThreadPoolHandle"
-    argspec: "args=[\'num_threads\', \'max_intra_op_parallelism\', \'display_name\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalUnbatchDataset"
-    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExperimentalUniqueDataset"
-    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Expm1"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExtractGlimpse"
-    argspec: "args=[\'input\', \'size\', \'offsets\', \'centered\', \'normalized\', \'uniform_noise\', \'noise\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExtractImagePatches"
-    argspec: "args=[\'images\', \'ksizes\', \'strides\', \'rates\', \'padding\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExtractJpegShape"
-    argspec: "args=[\'contents\', \'output_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ExtractVolumePatches"
-    argspec: "args=[\'input\', \'ksizes\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FFT"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FFT2D"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FFT3D"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FIFOQueue"
-    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FIFOQueueV2"
-    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Fact"
-    argspec: "args=[], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FakeParam"
-    argspec: "args=[\'dtype\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FakeQuantWithMinMaxArgs"
-    argspec: "args=[\'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FakeQuantWithMinMaxArgsGradient"
-    argspec: "args=[\'gradients\', \'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FakeQuantWithMinMaxVars"
-    argspec: "args=[\'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FakeQuantWithMinMaxVarsGradient"
-    argspec: "args=[\'gradients\', \'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FakeQuantWithMinMaxVarsPerChannel"
-    argspec: "args=[\'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FakeQuantWithMinMaxVarsPerChannelGradient"
-    argspec: "args=[\'gradients\', \'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FakeQueue"
-    argspec: "args=[\'resource\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Fill"
-    argspec: "args=[\'dims\', \'value\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FilterByLastComponentDataset"
-    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FilterDataset"
-    argspec: "args=[\'input_dataset\', \'other_arguments\', \'predicate\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FixedLengthRecordDataset"
-    argspec: "args=[\'filenames\', \'header_bytes\', \'record_bytes\', \'footer_bytes\', \'buffer_size\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FixedLengthRecordDatasetV2"
-    argspec: "args=[\'filenames\', \'header_bytes\', \'record_bytes\', \'footer_bytes\', \'buffer_size\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FixedLengthRecordReader"
-    argspec: "args=[\'header_bytes\', \'record_bytes\', \'footer_bytes\', \'hop_bytes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FixedLengthRecordReaderV2"
-    argspec: "args=[\'header_bytes\', \'record_bytes\', \'footer_bytes\', \'hop_bytes\', \'container\', \'shared_name\', \'encoding\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FixedUnigramCandidateSampler"
-    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'vocab_file\', \'distortion\', \'num_reserved_ids\', \'num_shards\', \'shard\', \'unigrams\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FlatMapDataset"
-    argspec: "args=[\'input_dataset\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Floor"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FloorDiv"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FloorMod"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FlushSummaryWriter"
-    argspec: "args=[\'writer\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "For"
-    argspec: "args=[\'start\', \'limit\', \'delta\', \'input\', \'body\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FractionalAvgPool"
-    argspec: "args=[\'value\', \'pooling_ratio\', \'pseudo_random\', \'overlapping\', \'deterministic\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FractionalAvgPoolGrad"
-    argspec: "args=[\'orig_input_tensor_shape\', \'out_backprop\', \'row_pooling_sequence\', \'col_pooling_sequence\', \'overlapping\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FractionalMaxPool"
-    argspec: "args=[\'value\', \'pooling_ratio\', \'pseudo_random\', \'overlapping\', \'deterministic\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FractionalMaxPoolGrad"
-    argspec: "args=[\'orig_input\', \'orig_output\', \'out_backprop\', \'row_pooling_sequence\', \'col_pooling_sequence\', \'overlapping\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FusedBatchNorm"
-    argspec: "args=[\'x\', \'scale\', \'offset\', \'mean\', \'variance\', \'epsilon\', \'data_format\', \'is_training\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FusedBatchNormGrad"
-    argspec: "args=[\'y_backprop\', \'x\', \'scale\', \'reserve_space_1\', \'reserve_space_2\', \'epsilon\', \'data_format\', \'is_training\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FusedBatchNormGradV2"
-    argspec: "args=[\'y_backprop\', \'x\', \'scale\', \'reserve_space_1\', \'reserve_space_2\', \'epsilon\', \'data_format\', \'is_training\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FusedBatchNormV2"
-    argspec: "args=[\'x\', \'scale\', \'offset\', \'mean\', \'variance\', \'epsilon\', \'data_format\', \'is_training\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FusedPadConv2D"
-    argspec: "args=[\'input\', \'paddings\', \'filter\', \'mode\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "FusedResizeAndPadConv2D"
-    argspec: "args=[\'input\', \'size\', \'paddings\', \'filter\', \'resize_align_corners\', \'mode\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Gather"
-    argspec: "args=[\'params\', \'indices\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "GatherNd"
-    argspec: "args=[\'params\', \'indices\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "GatherV2"
-    argspec: "args=[\'params\', \'indices\', \'axis\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "GenerateVocabRemapping"
-    argspec: "args=[\'new_vocab_file\', \'old_vocab_file\', \'new_vocab_offset\', \'num_new_vocab\', \'old_vocab_size\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "GeneratorDataset"
-    argspec: "args=[\'init_func_other_args\', \'next_func_other_args\', \'finalize_func_other_args\', \'init_func\', \'next_func\', \'finalize_func\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "GetSessionHandle"
-    argspec: "args=[\'value\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "GetSessionHandleV2"
-    argspec: "args=[\'value\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "GetSessionTensor"
-    argspec: "args=[\'handle\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Greater"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "GreaterEqual"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "GuaranteeConst"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "HSVToRGB"
-    argspec: "args=[\'images\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "HashTable"
-    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "HashTableV2"
-    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "HistogramFixedWidth"
-    argspec: "args=[\'values\', \'value_range\', \'nbins\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "HistogramSummary"
-    argspec: "args=[\'tag\', \'values\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IFFT"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IFFT2D"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IFFT3D"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IRFFT"
-    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IRFFT2D"
-    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IRFFT3D"
-    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Identity"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IdentityN"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IdentityReader"
-    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IdentityReaderV2"
-    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "If"
-    argspec: "args=[\'cond\', \'input\', \'Tout\', \'then_branch\', \'else_branch\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Igamma"
-    argspec: "args=[\'a\', \'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IgammaGradA"
-    argspec: "args=[\'a\', \'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Igammac"
-    argspec: "args=[\'a\', \'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Imag"
-    argspec: "args=[\'input\', \'Tout\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ImageSummary"
-    argspec: "args=[\'tag\', \'tensor\', \'max_images\', \'bad_color\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ImmutableConst"
-    argspec: "args=[\'dtype\', \'shape\', \'memory_region_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ImportEvent"
-    argspec: "args=[\'writer\', \'event\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "InTopK"
-    argspec: "args=[\'predictions\', \'targets\', \'k\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "InTopKV2"
-    argspec: "args=[\'predictions\', \'targets\', \'k\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "InitializeTable"
-    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "InitializeTableFromTextFile"
-    argspec: "args=[\'table_handle\', \'filename\', \'key_index\', \'value_index\', \'vocab_size\', \'delimiter\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "InitializeTableFromTextFileV2"
-    argspec: "args=[\'table_handle\', \'filename\', \'key_index\', \'value_index\', \'vocab_size\', \'delimiter\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "InitializeTableV2"
-    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "InplaceAdd"
-    argspec: "args=[\'x\', \'i\', \'v\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "InplaceSub"
-    argspec: "args=[\'x\', \'i\', \'v\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "InplaceUpdate"
-    argspec: "args=[\'x\', \'i\', \'v\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "InterleaveDataset"
-    argspec: "args=[\'input_dataset\', \'other_arguments\', \'cycle_length\', \'block_length\', \'f\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Inv"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "InvGrad"
-    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Invert"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "InvertPermutation"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IsBoostedTreesEnsembleInitialized"
-    argspec: "args=[\'tree_ensemble_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IsBoostedTreesQuantileStreamResourceInitialized"
-    argspec: "args=[\'quantile_stream_resource_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IsFinite"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IsInf"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IsNan"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IsVariableInitialized"
-    argspec: "args=[\'ref\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Iterator"
-    argspec: "args=[\'shared_name\', \'container\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IteratorFromStringHandle"
-    argspec: "args=[\'string_handle\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IteratorFromStringHandleV2"
-    argspec: "args=[\'string_handle\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IteratorGetNext"
-    argspec: "args=[\'iterator\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IteratorGetNextAsOptional"
-    argspec: "args=[\'iterator\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IteratorGetNextSync"
-    argspec: "args=[\'iterator\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IteratorToStringHandle"
-    argspec: "args=[\'resource_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IteratorV2"
-    argspec: "args=[\'shared_name\', \'container\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "L2Loss"
-    argspec: "args=[\'t\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LMDBReader"
-    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LRN"
-    argspec: "args=[\'input\', \'depth_radius\', \'bias\', \'alpha\', \'beta\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LRNGrad"
-    argspec: "args=[\'input_grads\', \'input_image\', \'output_image\', \'depth_radius\', \'bias\', \'alpha\', \'beta\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LeakyRelu"
-    argspec: "args=[\'features\', \'alpha\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LeakyReluGrad"
-    argspec: "args=[\'gradients\', \'features\', \'alpha\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LearnedUnigramCandidateSampler"
-    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LeftShift"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Less"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LessEqual"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Lgamma"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LinSpace"
-    argspec: "args=[\'start\', \'stop\', \'num\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ListDiff"
-    argspec: "args=[\'x\', \'y\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LoadAndRemapMatrix"
-    argspec: "args=[\'ckpt_path\', \'old_tensor_name\', \'row_remapping\', \'col_remapping\', \'initializing_values\', \'num_rows\', \'num_cols\', \'max_rows_in_memory\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Log"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Log1p"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LogMatrixDeterminant"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LogSoftmax"
-    argspec: "args=[\'logits\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LogUniformCandidateSampler"
-    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LogicalAnd"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LogicalNot"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LogicalOr"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LookupTableExport"
-    argspec: "args=[\'table_handle\', \'Tkeys\', \'Tvalues\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LookupTableExportV2"
-    argspec: "args=[\'table_handle\', \'Tkeys\', \'Tvalues\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LookupTableFind"
-    argspec: "args=[\'table_handle\', \'keys\', \'default_value\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LookupTableFindV2"
-    argspec: "args=[\'table_handle\', \'keys\', \'default_value\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LookupTableImport"
-    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LookupTableImportV2"
-    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LookupTableInsert"
-    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LookupTableInsertV2"
-    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LookupTableRemoveV2"
-    argspec: "args=[\'table_handle\', \'keys\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LookupTableSize"
-    argspec: "args=[\'table_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LookupTableSizeV2"
-    argspec: "args=[\'table_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LoopCond"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "LowerBound"
-    argspec: "args=[\'sorted_inputs\', \'values\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Lu"
-    argspec: "args=[\'input\', \'output_idx_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MakeIterator"
-    argspec: "args=[\'dataset\', \'iterator\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MapClear"
-    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MapDataset"
-    argspec: "args=[\'input_dataset\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\', \'use_inter_op_parallelism\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MapDefun"
-    argspec: "args=[\'arguments\', \'captured_inputs\', \'output_types\', \'output_shapes\', \'f\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MapIncompleteSize"
-    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MapPeek"
-    argspec: "args=[\'key\', \'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MapSize"
-    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MapStage"
-    argspec: "args=[\'key\', \'indices\', \'values\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MapUnstage"
-    argspec: "args=[\'key\', \'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MapUnstageNoKey"
-    argspec: "args=[\'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MatMul"
-    argspec: "args=[\'a\', \'b\', \'transpose_a\', \'transpose_b\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MatchingFiles"
-    argspec: "args=[\'pattern\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MatrixBandPart"
-    argspec: "args=[\'input\', \'num_lower\', \'num_upper\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MatrixDeterminant"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MatrixDiag"
-    argspec: "args=[\'diagonal\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MatrixDiagPart"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MatrixExponential"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MatrixInverse"
-    argspec: "args=[\'input\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MatrixLogarithm"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MatrixSetDiag"
-    argspec: "args=[\'input\', \'diagonal\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MatrixSolve"
-    argspec: "args=[\'matrix\', \'rhs\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MatrixSolveLs"
-    argspec: "args=[\'matrix\', \'rhs\', \'l2_regularizer\', \'fast\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MatrixSquareRoot"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MatrixTriangularSolve"
-    argspec: "args=[\'matrix\', \'rhs\', \'lower\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Max"
-    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MaxPool"
-    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MaxPool3D"
-    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MaxPool3DGrad"
-    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MaxPool3DGradGrad"
-    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MaxPoolGrad"
-    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MaxPoolGradGrad"
-    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MaxPoolGradGradV2"
-    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MaxPoolGradGradWithArgmax"
-    argspec: "args=[\'input\', \'grad\', \'argmax\', \'ksize\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MaxPoolGradV2"
-    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MaxPoolGradWithArgmax"
-    argspec: "args=[\'input\', \'grad\', \'argmax\', \'ksize\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MaxPoolV2"
-    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MaxPoolWithArgmax"
-    argspec: "args=[\'input\', \'ksize\', \'strides\', \'Targmax\', \'padding\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Maximum"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Mean"
-    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Merge"
-    argspec: "args=[\'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MergeSummary"
-    argspec: "args=[\'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MergeV2Checkpoints"
-    argspec: "args=[\'checkpoint_prefixes\', \'destination_prefix\', \'delete_old_dirs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Mfcc"
-    argspec: "args=[\'spectrogram\', \'sample_rate\', \'upper_frequency_limit\', \'lower_frequency_limit\', \'filterbank_channel_count\', \'dct_coefficient_count\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Min"
-    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Minimum"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MirrorPad"
-    argspec: "args=[\'input\', \'paddings\', \'mode\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MirrorPadGrad"
-    argspec: "args=[\'input\', \'paddings\', \'mode\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Mod"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ModelDataset"
-    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Mul"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MultiDeviceIterator"
-    argspec: "args=[\'devices\', \'shared_name\', \'container\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MultiDeviceIteratorFromStringHandle"
-    argspec: "args=[\'string_handle\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MultiDeviceIteratorGetNextFromShard"
-    argspec: "args=[\'multi_device_iterator\', \'shard_num\', \'incarnation_id\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MultiDeviceIteratorInit"
-    argspec: "args=[\'dataset\', \'multi_device_iterator\', \'max_buffer_size\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MultiDeviceIteratorToStringHandle"
-    argspec: "args=[\'multi_device_iterator\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Multinomial"
-    argspec: "args=[\'logits\', \'num_samples\', \'seed\', \'seed2\', \'output_dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MutableDenseHashTable"
-    argspec: "args=[\'empty_key\', \'container\', \'shared_name\', \'use_node_name_sharing\', \'value_dtype\', \'value_shape\', \'initial_num_buckets\', \'max_load_factor\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MutableDenseHashTableV2"
-    argspec: "args=[\'empty_key\', \'deleted_key\', \'container\', \'shared_name\', \'use_node_name_sharing\', \'value_dtype\', \'value_shape\', \'initial_num_buckets\', \'max_load_factor\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MutableHashTable"
-    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MutableHashTableOfTensors"
-    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\', \'value_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MutableHashTableOfTensorsV2"
-    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\', \'value_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MutableHashTableV2"
-    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MutexLock"
-    argspec: "args=[\'mutex\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MutexV2"
-    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "NcclAllReduce"
-    argspec: "args=[\'input\', \'reduction\', \'num_devices\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "NcclBroadcast"
-    argspec: "args=[\'input\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "NcclReduce"
-    argspec: "args=[\'input\', \'reduction\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Neg"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "NextAfter"
-    argspec: "args=[\'x1\', \'x2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "NextIteration"
-    argspec: "args=[\'data\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "NoOp"
-    argspec: "args=[], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "NonMaxSuppression"
-    argspec: "args=[\'boxes\', \'scores\', \'max_output_size\', \'iou_threshold\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "NonMaxSuppressionV2"
-    argspec: "args=[\'boxes\', \'scores\', \'max_output_size\', \'iou_threshold\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "NonMaxSuppressionV3"
-    argspec: "args=[\'boxes\', \'scores\', \'max_output_size\', \'iou_threshold\', \'score_threshold\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "NonMaxSuppressionV4"
-    argspec: "args=[\'boxes\', \'scores\', \'max_output_size\', \'iou_threshold\', \'score_threshold\', \'pad_to_max_output_size\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "NonMaxSuppressionWithOverlaps"
-    argspec: "args=[\'overlaps\', \'scores\', \'max_output_size\', \'overlap_threshold\', \'score_threshold\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "NotEqual"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "NthElement"
-    argspec: "args=[\'input\', \'n\', \'reverse\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OneHot"
-    argspec: "args=[\'indices\', \'depth\', \'on_value\', \'off_value\', \'axis\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OneShotIterator"
-    argspec: "args=[\'dataset_factory\', \'output_types\', \'output_shapes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OnesLike"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OptimizeDataset"
-    argspec: "args=[\'input_dataset\', \'optimizations\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OptionalFromValue"
-    argspec: "args=[\'components\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OptionalGetValue"
-    argspec: "args=[\'optional\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OptionalHasValue"
-    argspec: "args=[\'optional\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OptionalNone"
-    argspec: "args=[], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OrderedMapClear"
-    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OrderedMapIncompleteSize"
-    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OrderedMapPeek"
-    argspec: "args=[\'key\', \'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OrderedMapSize"
-    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OrderedMapStage"
-    argspec: "args=[\'key\', \'indices\', \'values\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OrderedMapUnstage"
-    argspec: "args=[\'key\', \'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "OrderedMapUnstageNoKey"
-    argspec: "args=[\'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Pack"
-    argspec: "args=[\'values\', \'axis\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Pad"
-    argspec: "args=[\'input\', \'paddings\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PadV2"
-    argspec: "args=[\'input\', \'paddings\', \'constant_values\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PaddedBatchDataset"
-    argspec: "args=[\'input_dataset\', \'batch_size\', \'padded_shapes\', \'padding_values\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PaddedBatchDatasetV2"
-    argspec: "args=[\'input_dataset\', \'batch_size\', \'padded_shapes\', \'padding_values\', \'drop_remainder\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PaddingFIFOQueue"
-    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PaddingFIFOQueueV2"
-    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ParallelConcat"
-    argspec: "args=[\'values\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ParallelDynamicStitch"
-    argspec: "args=[\'indices\', \'data\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ParallelInterleaveDatasetV2"
-    argspec: "args=[\'input_dataset\', \'other_arguments\', \'cycle_length\', \'block_length\', \'num_parallel_calls\', \'f\', \'output_types\', \'output_shapes\', \'sloppy\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ParallelMapDataset"
-    argspec: "args=[\'input_dataset\', \'other_arguments\', \'num_parallel_calls\', \'f\', \'output_types\', \'output_shapes\', \'use_inter_op_parallelism\', \'sloppy\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ParameterizedTruncatedNormal"
-    argspec: "args=[\'shape\', \'means\', \'stdevs\', \'minvals\', \'maxvals\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ParseExample"
-    argspec: "args=[\'serialized\', \'names\', \'sparse_keys\', \'dense_keys\', \'dense_defaults\', \'sparse_types\', \'dense_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ParseSequenceExample"
-    argspec: "args=[\'serialized\', \'debug_name\', \'context_dense_defaults\', \'feature_list_dense_missing_assumed_empty\', \'context_sparse_keys\', \'context_dense_keys\', \'feature_list_sparse_keys\', \'feature_list_dense_keys\', \'Ncontext_sparse\', \'Ncontext_dense\', \'Nfeature_list_sparse\', \'Nfeature_list_dense\', \'context_sparse_types\', \'feature_list_dense_types\', \'context_dense_shapes\', \'feature_list_sparse_types\', \'feature_list_dense_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ParseSingleExample"
-    argspec: "args=[\'serialized\', \'dense_defaults\', \'num_sparse\', \'sparse_keys\', \'dense_keys\', \'sparse_types\', \'dense_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ParseSingleSequenceExample"
-    argspec: "args=[\'serialized\', \'feature_list_dense_missing_assumed_empty\', \'context_sparse_keys\', \'context_dense_keys\', \'feature_list_sparse_keys\', \'feature_list_dense_keys\', \'context_dense_defaults\', \'debug_name\', \'context_sparse_types\', \'feature_list_dense_types\', \'context_dense_shapes\', \'feature_list_sparse_types\', \'feature_list_dense_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ParseTensor"
-    argspec: "args=[\'serialized\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PartitionedCall"
-    argspec: "args=[\'args\', \'Tout\', \'f\', \'config\', \'config_proto\', \'executor_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Placeholder"
-    argspec: "args=[\'dtype\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PlaceholderV2"
-    argspec: "args=[\'dtype\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PlaceholderWithDefault"
-    argspec: "args=[\'input\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Polygamma"
-    argspec: "args=[\'a\', \'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PopulationCount"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Pow"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PrefetchDataset"
-    argspec: "args=[\'input_dataset\', \'buffer_size\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PreventGradient"
-    argspec: "args=[\'input\', \'message\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Print"
-    argspec: "args=[\'input\', \'data\', \'message\', \'first_n\', \'summarize\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PrintV2"
-    argspec: "args=[\'input\', \'output_stream\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PriorityQueue"
-    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PriorityQueueV2"
-    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Prod"
-    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PyFunc"
-    argspec: "args=[\'input\', \'token\', \'Tout\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "PyFuncStateless"
-    argspec: "args=[\'input\', \'token\', \'Tout\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Qr"
-    argspec: "args=[\'input\', \'full_matrices\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizeAndDequantize"
-    argspec: "args=[\'input\', \'signed_input\', \'num_bits\', \'range_given\', \'input_min\', \'input_max\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizeAndDequantizeV2"
-    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'signed_input\', \'num_bits\', \'range_given\', \'round_mode\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizeAndDequantizeV3"
-    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'num_bits\', \'signed_input\', \'range_given\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizeDownAndShrinkRange"
-    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizeV2"
-    argspec: "args=[\'input\', \'min_range\', \'max_range\', \'T\', \'mode\', \'round_mode\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedAdd"
-    argspec: "args=[\'x\', \'y\', \'min_x\', \'max_x\', \'min_y\', \'max_y\', \'Toutput\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedAvgPool"
-    argspec: "args=[\'input\', \'min_input\', \'max_input\', \'ksize\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedBatchNormWithGlobalNormalization"
-    argspec: "args=[\'t\', \'t_min\', \'t_max\', \'m\', \'m_min\', \'m_max\', \'v\', \'v_min\', \'v_max\', \'beta\', \'beta_min\', \'beta_max\', \'gamma\', \'gamma_min\', \'gamma_max\', \'out_type\', \'variance_epsilon\', \'scale_after_normalization\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedBiasAdd"
-    argspec: "args=[\'input\', \'bias\', \'min_input\', \'max_input\', \'min_bias\', \'max_bias\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedConcat"
-    argspec: "args=[\'concat_dim\', \'values\', \'input_mins\', \'input_maxes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedConv2D"
-    argspec: "args=[\'input\', \'filter\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedConv2DAndRelu"
-    argspec: "args=[\'input\', \'filter\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedConv2DAndReluAndRequantize"
-    argspec: "args=[\'input\', \'filter\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedConv2DAndRequantize"
-    argspec: "args=[\'input\', \'filter\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedConv2DWithBias"
-    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedConv2DWithBiasAndRelu"
-    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedConv2DWithBiasAndReluAndRequantize"
-    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedConv2DWithBiasAndRequantize"
-    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedConv2DWithBiasSignedSumAndReluAndRequantize"
-    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'summand\', \'min_summand\', \'max_summand\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedConv2DWithBiasSumAndRelu"
-    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'summand\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedConv2DWithBiasSumAndReluAndRequantize"
-    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'summand\', \'min_summand\', \'max_summand\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedInstanceNorm"
-    argspec: "args=[\'x\', \'x_min\', \'x_max\', \'output_range_given\', \'given_y_min\', \'given_y_max\', \'variance_epsilon\', \'min_separation\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedMatMul"
-    argspec: "args=[\'a\', \'b\', \'min_a\', \'max_a\', \'min_b\', \'max_b\', \'Toutput\', \'transpose_a\', \'transpose_b\', \'Tactivation\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedMaxPool"
-    argspec: "args=[\'input\', \'min_input\', \'max_input\', \'ksize\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedMul"
-    argspec: "args=[\'x\', \'y\', \'min_x\', \'max_x\', \'min_y\', \'max_y\', \'Toutput\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedRelu"
-    argspec: "args=[\'features\', \'min_features\', \'max_features\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedRelu6"
-    argspec: "args=[\'features\', \'min_features\', \'max_features\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedReluX"
-    argspec: "args=[\'features\', \'max_value\', \'min_features\', \'max_features\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedReshape"
-    argspec: "args=[\'tensor\', \'shape\', \'input_min\', \'input_max\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QuantizedResizeBilinear"
-    argspec: "args=[\'images\', \'size\', \'min\', \'max\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueClose"
-    argspec: "args=[\'handle\', \'cancel_pending_enqueues\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueCloseV2"
-    argspec: "args=[\'handle\', \'cancel_pending_enqueues\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueDequeue"
-    argspec: "args=[\'handle\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueDequeueMany"
-    argspec: "args=[\'handle\', \'n\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueDequeueManyV2"
-    argspec: "args=[\'handle\', \'n\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueDequeueUpTo"
-    argspec: "args=[\'handle\', \'n\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueDequeueUpToV2"
-    argspec: "args=[\'handle\', \'n\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueDequeueV2"
-    argspec: "args=[\'handle\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueEnqueue"
-    argspec: "args=[\'handle\', \'components\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueEnqueueMany"
-    argspec: "args=[\'handle\', \'components\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueEnqueueManyV2"
-    argspec: "args=[\'handle\', \'components\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueEnqueueV2"
-    argspec: "args=[\'handle\', \'components\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueIsClosed"
-    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueIsClosedV2"
-    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueSize"
-    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "QueueSizeV2"
-    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RFFT"
-    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RFFT2D"
-    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RFFT3D"
-    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RGBToHSV"
-    argspec: "args=[\'images\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RaggedGather"
-    argspec: "args=[\'params_nested_splits\', \'params_dense_values\', \'indices\', \'OUTPUT_RAGGED_RANK\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RaggedRange"
-    argspec: "args=[\'starts\', \'limits\', \'deltas\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RaggedTensorToSparse"
-    argspec: "args=[\'rt_nested_splits\', \'rt_dense_values\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RandomCrop"
-    argspec: "args=[\'image\', \'size\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RandomGamma"
-    argspec: "args=[\'shape\', \'alpha\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RandomGammaGrad"
-    argspec: "args=[\'alpha\', \'sample\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RandomPoisson"
-    argspec: "args=[\'shape\', \'rate\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RandomPoissonV2"
-    argspec: "args=[\'shape\', \'rate\', \'seed\', \'seed2\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RandomShuffle"
-    argspec: "args=[\'value\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RandomShuffleQueue"
-    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'min_after_dequeue\', \'seed\', \'seed2\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RandomShuffleQueueV2"
-    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'min_after_dequeue\', \'seed\', \'seed2\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RandomStandardNormal"
-    argspec: "args=[\'shape\', \'seed\', \'seed2\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RandomUniform"
-    argspec: "args=[\'shape\', \'seed\', \'seed2\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RandomUniformInt"
-    argspec: "args=[\'shape\', \'minval\', \'maxval\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Range"
-    argspec: "args=[\'start\', \'limit\', \'delta\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RangeDataset"
-    argspec: "args=[\'start\', \'stop\', \'step\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Rank"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReadFile"
-    argspec: "args=[\'filename\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReadVariableOp"
-    argspec: "args=[\'resource\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReaderNumRecordsProduced"
-    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReaderNumRecordsProducedV2"
-    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReaderNumWorkUnitsCompleted"
-    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReaderNumWorkUnitsCompletedV2"
-    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReaderRead"
-    argspec: "args=[\'reader_handle\', \'queue_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReaderReadUpTo"
-    argspec: "args=[\'reader_handle\', \'queue_handle\', \'num_records\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReaderReadUpToV2"
-    argspec: "args=[\'reader_handle\', \'queue_handle\', \'num_records\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReaderReadV2"
-    argspec: "args=[\'reader_handle\', \'queue_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReaderReset"
-    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReaderResetV2"
-    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReaderRestoreState"
-    argspec: "args=[\'reader_handle\', \'state\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReaderRestoreStateV2"
-    argspec: "args=[\'reader_handle\', \'state\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReaderSerializeState"
-    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReaderSerializeStateV2"
-    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Real"
-    argspec: "args=[\'input\', \'Tout\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RealDiv"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Reciprocal"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReciprocalGrad"
-    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RecordInput"
-    argspec: "args=[\'file_pattern\', \'file_random_seed\', \'file_shuffle_shift_ratio\', \'file_buffer_size\', \'file_parallelism\', \'batch_size\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReduceDataset"
-    argspec: "args=[\'input_dataset\', \'initial_state\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\', \'use_inter_op_parallelism\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReduceJoin"
-    argspec: "args=[\'inputs\', \'reduction_indices\', \'keep_dims\', \'separator\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RefEnter"
-    argspec: "args=[\'data\', \'frame_name\', \'is_constant\', \'parallel_iterations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RefExit"
-    argspec: "args=[\'data\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RefIdentity"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RefMerge"
-    argspec: "args=[\'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RefNextIteration"
-    argspec: "args=[\'data\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RefSelect"
-    argspec: "args=[\'index\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RefSwitch"
-    argspec: "args=[\'data\', \'pred\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RegexFullMatch"
-    argspec: "args=[\'input\', \'pattern\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RegexReplace"
-    argspec: "args=[\'input\', \'pattern\', \'rewrite\', \'replace_global\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Relu"
-    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Relu6"
-    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Relu6Grad"
-    argspec: "args=[\'gradients\', \'features\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReluGrad"
-    argspec: "args=[\'gradients\', \'features\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RemoteCall"
-    argspec: "args=[\'target\', \'args\', \'Tout\', \'f\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RepeatDataset"
-    argspec: "args=[\'input_dataset\', \'count\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RequantizationRange"
-    argspec: "args=[\'input\', \'input_min\', \'input_max\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Requantize"
-    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'requested_output_min\', \'requested_output_max\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Reshape"
-    argspec: "args=[\'tensor\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResizeArea"
-    argspec: "args=[\'images\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResizeBicubic"
-    argspec: "args=[\'images\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResizeBicubicGrad"
-    argspec: "args=[\'grads\', \'original_image\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResizeBilinear"
-    argspec: "args=[\'images\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResizeBilinearGrad"
-    argspec: "args=[\'grads\', \'original_image\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResizeNearestNeighbor"
-    argspec: "args=[\'images\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResizeNearestNeighborGrad"
-    argspec: "args=[\'grads\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyAdaMax"
-    argspec: "args=[\'var\', \'m\', \'v\', \'beta1_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyAdadelta"
-    argspec: "args=[\'var\', \'accum\', \'accum_update\', \'lr\', \'rho\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyAdagrad"
-    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'use_locking\', \'update_slots\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyAdagradDA"
-    argspec: "args=[\'var\', \'gradient_accumulator\', \'gradient_squared_accumulator\', \'grad\', \'lr\', \'l1\', \'l2\', \'global_step\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyAdam"
-    argspec: "args=[\'var\', \'m\', \'v\', \'beta1_power\', \'beta2_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyAdamWithAmsgrad"
-    argspec: "args=[\'var\', \'m\', \'v\', \'vhat\', \'beta1_power\', \'beta2_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyAddSign"
-    argspec: "args=[\'var\', \'m\', \'lr\', \'alpha\', \'sign_decay\', \'beta\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyCenteredRMSProp"
-    argspec: "args=[\'var\', \'mg\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyFtrl"
-    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'lr\', \'l1\', \'l2\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyFtrlV2"
-    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'lr\', \'l1\', \'l2\', \'l2_shrinkage\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyGradientDescent"
-    argspec: "args=[\'var\', \'alpha\', \'delta\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyKerasMomentum"
-    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyMomentum"
-    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyPowerSign"
-    argspec: "args=[\'var\', \'m\', \'lr\', \'logbase\', \'sign_decay\', \'beta\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyProximalAdagrad"
-    argspec: "args=[\'var\', \'accum\', \'lr\', \'l1\', \'l2\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyProximalGradientDescent"
-    argspec: "args=[\'var\', \'alpha\', \'l1\', \'l2\', \'delta\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceApplyRMSProp"
-    argspec: "args=[\'var\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceCountUpTo"
-    argspec: "args=[\'resource\', \'limit\', \'T\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceGather"
-    argspec: "args=[\'resource\', \'indices\', \'validate_indices\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceScatterAdd"
-    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceScatterDiv"
-    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceScatterMax"
-    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceScatterMin"
-    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceScatterMul"
-    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceScatterNdAdd"
-    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceScatterNdSub"
-    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceScatterNdUpdate"
-    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceScatterSub"
-    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceScatterUpdate"
-    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceSparseApplyAdadelta"
-    argspec: "args=[\'var\', \'accum\', \'accum_update\', \'lr\', \'rho\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceSparseApplyAdagrad"
-    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'use_locking\', \'update_slots\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceSparseApplyAdagradDA"
-    argspec: "args=[\'var\', \'gradient_accumulator\', \'gradient_squared_accumulator\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'global_step\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceSparseApplyCenteredRMSProp"
-    argspec: "args=[\'var\', \'mg\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceSparseApplyFtrl"
-    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceSparseApplyFtrlV2"
-    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'l2_shrinkage\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceSparseApplyKerasMomentum"
-    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceSparseApplyMomentum"
-    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceSparseApplyProximalAdagrad"
-    argspec: "args=[\'var\', \'accum\', \'lr\', \'l1\', \'l2\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceSparseApplyProximalGradientDescent"
-    argspec: "args=[\'var\', \'alpha\', \'l1\', \'l2\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceSparseApplyRMSProp"
-    argspec: "args=[\'var\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ResourceStridedSliceAssign"
-    argspec: "args=[\'ref\', \'begin\', \'end\', \'strides\', \'value\', \'begin_mask\', \'end_mask\', \'ellipsis_mask\', \'new_axis_mask\', \'shrink_axis_mask\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Restore"
-    argspec: "args=[\'file_pattern\', \'tensor_name\', \'dt\', \'preferred_shard\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RestoreSlice"
-    argspec: "args=[\'file_pattern\', \'tensor_name\', \'shape_and_slice\', \'dt\', \'preferred_shard\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RestoreV2"
-    argspec: "args=[\'prefix\', \'tensor_names\', \'shape_and_slices\', \'dtypes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Reverse"
-    argspec: "args=[\'tensor\', \'dims\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReverseSequence"
-    argspec: "args=[\'input\', \'seq_lengths\', \'seq_dim\', \'batch_dim\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ReverseV2"
-    argspec: "args=[\'tensor\', \'axis\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RightShift"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Rint"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Roll"
-    argspec: "args=[\'input\', \'shift\', \'axis\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Round"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Rsqrt"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "RsqrtGrad"
-    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SampleDistortedBoundingBox"
-    argspec: "args=[\'image_size\', \'bounding_boxes\', \'seed\', \'seed2\', \'min_object_covered\', \'aspect_ratio_range\', \'area_range\', \'max_attempts\', \'use_image_if_no_bounding_boxes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SampleDistortedBoundingBoxV2"
-    argspec: "args=[\'image_size\', \'bounding_boxes\', \'min_object_covered\', \'seed\', \'seed2\', \'aspect_ratio_range\', \'area_range\', \'max_attempts\', \'use_image_if_no_bounding_boxes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Save"
-    argspec: "args=[\'filename\', \'tensor_names\', \'data\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SaveSlices"
-    argspec: "args=[\'filename\', \'tensor_names\', \'shapes_and_slices\', \'data\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SaveV2"
-    argspec: "args=[\'prefix\', \'tensor_names\', \'shape_and_slices\', \'tensors\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScalarSummary"
-    argspec: "args=[\'tags\', \'values\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScaleAndTranslate"
-    argspec: "args=[\'images\', \'size\', \'scale\', \'translation\', \'kernel_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScaleAndTranslateGrad"
-    argspec: "args=[\'grads\', \'original_image\', \'scale\', \'translation\', \'kernel_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScatterAdd"
-    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScatterDiv"
-    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScatterMax"
-    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScatterMin"
-    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScatterMul"
-    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScatterNd"
-    argspec: "args=[\'indices\', \'updates\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScatterNdAdd"
-    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScatterNdNonAliasingAdd"
-    argspec: "args=[\'input\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScatterNdSub"
-    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScatterNdUpdate"
-    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScatterSub"
-    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ScatterUpdate"
-    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SdcaFprint"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SdcaOptimizer"
-    argspec: "args=[\'sparse_example_indices\', \'sparse_feature_indices\', \'sparse_feature_values\', \'dense_features\', \'example_weights\', \'example_labels\', \'sparse_indices\', \'sparse_weights\', \'dense_weights\', \'example_state_data\', \'loss_type\', \'adaptative\', \'l1\', \'l2\', \'num_loss_partitions\', \'num_inner_iterations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SdcaOptimizerV2"
-    argspec: "args=[\'sparse_example_indices\', \'sparse_feature_indices\', \'sparse_feature_values\', \'dense_features\', \'example_weights\', \'example_labels\', \'sparse_indices\', \'sparse_weights\', \'dense_weights\', \'example_state_data\', \'loss_type\', \'adaptive\', \'l1\', \'l2\', \'num_loss_partitions\', \'num_inner_iterations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SdcaShrinkL1"
-    argspec: "args=[\'weights\', \'l1\', \'l2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SegmentMax"
-    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SegmentMean"
-    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SegmentMin"
-    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SegmentProd"
-    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SegmentSum"
-    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Select"
-    argspec: "args=[\'condition\', \'t\', \'e\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SelfAdjointEig"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SelfAdjointEigV2"
-    argspec: "args=[\'input\', \'compute_v\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Selu"
-    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SeluGrad"
-    argspec: "args=[\'gradients\', \'outputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SerializeIterator"
-    argspec: "args=[\'resource_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SerializeManySparse"
-    argspec: "args=[\'sparse_indices\', \'sparse_values\', \'sparse_shape\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SerializeSparse"
-    argspec: "args=[\'sparse_indices\', \'sparse_values\', \'sparse_shape\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SerializeTensor"
-    argspec: "args=[\'tensor\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SetSize"
-    argspec: "args=[\'set_indices\', \'set_values\', \'set_shape\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Shape"
-    argspec: "args=[\'input\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ShapeN"
-    argspec: "args=[\'input\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ShardedFilename"
-    argspec: "args=[\'basename\', \'shard\', \'num_shards\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ShardedFilespec"
-    argspec: "args=[\'basename\', \'num_shards\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ShuffleAndRepeatDataset"
-    argspec: "args=[\'input_dataset\', \'buffer_size\', \'seed\', \'seed2\', \'count\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ShuffleDataset"
-    argspec: "args=[\'input_dataset\', \'buffer_size\', \'seed\', \'seed2\', \'reshuffle_each_iteration\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Sigmoid"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SigmoidGrad"
-    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Sign"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Sin"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Sinh"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Size"
-    argspec: "args=[\'input\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SkipDataset"
-    argspec: "args=[\'input_dataset\', \'count\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Slice"
-    argspec: "args=[\'input\', \'begin\', \'size\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Snapshot"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Softmax"
-    argspec: "args=[\'logits\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SoftmaxCrossEntropyWithLogits"
-    argspec: "args=[\'features\', \'labels\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Softplus"
-    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SoftplusGrad"
-    argspec: "args=[\'gradients\', \'features\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Softsign"
-    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SoftsignGrad"
-    argspec: "args=[\'gradients\', \'features\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SpaceToBatch"
-    argspec: "args=[\'input\', \'paddings\', \'block_size\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SpaceToBatchND"
-    argspec: "args=[\'input\', \'block_shape\', \'paddings\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SpaceToDepth"
-    argspec: "args=[\'input\', \'block_size\', \'data_format\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseAccumulatorApplyGradient"
-    argspec: "args=[\'handle\', \'local_step\', \'gradient_indices\', \'gradient_values\', \'gradient_shape\', \'has_known_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseAccumulatorTakeGradient"
-    argspec: "args=[\'handle\', \'num_required\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseAdd"
-    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b_indices\', \'b_values\', \'b_shape\', \'thresh\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseAddGrad"
-    argspec: "args=[\'backprop_val_grad\', \'a_indices\', \'b_indices\', \'sum_indices\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseApplyAdadelta"
-    argspec: "args=[\'var\', \'accum\', \'accum_update\', \'lr\', \'rho\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseApplyAdagrad"
-    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'use_locking\', \'update_slots\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseApplyAdagradDA"
-    argspec: "args=[\'var\', \'gradient_accumulator\', \'gradient_squared_accumulator\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'global_step\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseApplyCenteredRMSProp"
-    argspec: "args=[\'var\', \'mg\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseApplyFtrl"
-    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseApplyFtrlV2"
-    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'l2_shrinkage\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseApplyMomentum"
-    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseApplyProximalAdagrad"
-    argspec: "args=[\'var\', \'accum\', \'lr\', \'l1\', \'l2\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseApplyProximalGradientDescent"
-    argspec: "args=[\'var\', \'alpha\', \'l1\', \'l2\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseApplyRMSProp"
-    argspec: "args=[\'var\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseConcat"
-    argspec: "args=[\'indices\', \'values\', \'shapes\', \'concat_dim\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseConditionalAccumulator"
-    argspec: "args=[\'dtype\', \'shape\', \'container\', \'shared_name\', \'reduction_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseCross"
-    argspec: "args=[\'indices\', \'values\', \'shapes\', \'dense_inputs\', \'hashed_output\', \'num_buckets\', \'hash_key\', \'out_type\', \'internal_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseDenseCwiseAdd"
-    argspec: "args=[\'sp_indices\', \'sp_values\', \'sp_shape\', \'dense\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseDenseCwiseDiv"
-    argspec: "args=[\'sp_indices\', \'sp_values\', \'sp_shape\', \'dense\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseDenseCwiseMul"
-    argspec: "args=[\'sp_indices\', \'sp_values\', \'sp_shape\', \'dense\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseFillEmptyRows"
-    argspec: "args=[\'indices\', \'values\', \'dense_shape\', \'default_value\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseFillEmptyRowsGrad"
-    argspec: "args=[\'reverse_index_map\', \'grad_values\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseMatMul"
-    argspec: "args=[\'a\', \'b\', \'transpose_a\', \'transpose_b\', \'a_is_sparse\', \'b_is_sparse\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseReduceMax"
-    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseReduceMaxSparse"
-    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseReduceSum"
-    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseReduceSumSparse"
-    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseReorder"
-    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseReshape"
-    argspec: "args=[\'input_indices\', \'input_shape\', \'new_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSegmentMean"
-    argspec: "args=[\'data\', \'indices\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSegmentMeanGrad"
-    argspec: "args=[\'grad\', \'indices\', \'segment_ids\', \'output_dim0\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSegmentMeanWithNumSegments"
-    argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSegmentSqrtN"
-    argspec: "args=[\'data\', \'indices\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSegmentSqrtNGrad"
-    argspec: "args=[\'grad\', \'indices\', \'segment_ids\', \'output_dim0\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSegmentSqrtNWithNumSegments"
-    argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSegmentSum"
-    argspec: "args=[\'data\', \'indices\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSegmentSumWithNumSegments"
-    argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSlice"
-    argspec: "args=[\'indices\', \'values\', \'shape\', \'start\', \'size\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSliceGrad"
-    argspec: "args=[\'backprop_val_grad\', \'input_indices\', \'input_start\', \'output_indices\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSoftmax"
-    argspec: "args=[\'sp_indices\', \'sp_values\', \'sp_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSoftmaxCrossEntropyWithLogits"
-    argspec: "args=[\'features\', \'labels\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSparseMaximum"
-    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b_indices\', \'b_values\', \'b_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSparseMinimum"
-    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b_indices\', \'b_values\', \'b_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseSplit"
-    argspec: "args=[\'split_dim\', \'indices\', \'values\', \'shape\', \'num_split\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseTensorDenseAdd"
-    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseTensorDenseMatMul"
-    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b\', \'adjoint_a\', \'adjoint_b\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseTensorSliceDataset"
-    argspec: "args=[\'indices\', \'values\', \'dense_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseToDense"
-    argspec: "args=[\'sparse_indices\', \'output_shape\', \'sparse_values\', \'default_value\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SparseToSparseSetOperation"
-    argspec: "args=[\'set1_indices\', \'set1_values\', \'set1_shape\', \'set2_indices\', \'set2_values\', \'set2_shape\', \'set_operation\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Split"
-    argspec: "args=[\'split_dim\', \'value\', \'num_split\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SplitV"
-    argspec: "args=[\'value\', \'size_splits\', \'split_dim\', \'num_split\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Sqrt"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SqrtGrad"
-    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Square"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SquaredDifference"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Squeeze"
-    argspec: "args=[\'input\', \'squeeze_dims\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Stack"
-    argspec: "args=[\'elem_type\', \'stack_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StackClose"
-    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StackCloseV2"
-    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StackPop"
-    argspec: "args=[\'handle\', \'elem_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StackPopV2"
-    argspec: "args=[\'handle\', \'elem_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StackPush"
-    argspec: "args=[\'handle\', \'elem\', \'swap_memory\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StackPushV2"
-    argspec: "args=[\'handle\', \'elem\', \'swap_memory\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StackV2"
-    argspec: "args=[\'max_size\', \'elem_type\', \'stack_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Stage"
-    argspec: "args=[\'values\', \'capacity\', \'memory_limit\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StageClear"
-    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StagePeek"
-    argspec: "args=[\'index\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StageSize"
-    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StatefulPartitionedCall"
-    argspec: "args=[\'args\', \'Tout\', \'f\', \'config\', \'config_proto\', \'executor_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StatelessIf"
-    argspec: "args=[\'cond\', \'input\', \'Tout\', \'then_branch\', \'else_branch\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StatelessMultinomial"
-    argspec: "args=[\'logits\', \'num_samples\', \'seed\', \'output_dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StatelessRandomNormal"
-    argspec: "args=[\'shape\', \'seed\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StatelessRandomUniform"
-    argspec: "args=[\'shape\', \'seed\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StatelessRandomUniformInt"
-    argspec: "args=[\'shape\', \'seed\', \'minval\', \'maxval\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StatelessTruncatedNormal"
-    argspec: "args=[\'shape\', \'seed\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StatelessWhile"
-    argspec: "args=[\'input\', \'cond\', \'body\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StaticRegexFullMatch"
-    argspec: "args=[\'input\', \'pattern\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StaticRegexReplace"
-    argspec: "args=[\'input\', \'pattern\', \'rewrite\', \'replace_global\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StopGradient"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StridedSlice"
-    argspec: "args=[\'input\', \'begin\', \'end\', \'strides\', \'begin_mask\', \'end_mask\', \'ellipsis_mask\', \'new_axis_mask\', \'shrink_axis_mask\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StridedSliceAssign"
-    argspec: "args=[\'ref\', \'begin\', \'end\', \'strides\', \'value\', \'begin_mask\', \'end_mask\', \'ellipsis_mask\', \'new_axis_mask\', \'shrink_axis_mask\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StridedSliceGrad"
-    argspec: "args=[\'shape\', \'begin\', \'end\', \'strides\', \'dy\', \'begin_mask\', \'end_mask\', \'ellipsis_mask\', \'new_axis_mask\', \'shrink_axis_mask\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StringFormat"
-    argspec: "args=[\'inputs\', \'template\', \'placeholder\', \'summarize\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StringJoin"
-    argspec: "args=[\'inputs\', \'separator\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StringLength"
-    argspec: "args=[\'input\', \'unit\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StringSplit"
-    argspec: "args=[\'input\', \'delimiter\', \'skip_empty\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StringSplitV2"
-    argspec: "args=[\'input\', \'sep\', \'maxsplit\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StringStrip"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StringToHashBucket"
-    argspec: "args=[\'string_tensor\', \'num_buckets\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StringToHashBucketFast"
-    argspec: "args=[\'input\', \'num_buckets\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StringToHashBucketStrong"
-    argspec: "args=[\'input\', \'num_buckets\', \'key\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StringToNumber"
-    argspec: "args=[\'string_tensor\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Sub"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Substr"
-    argspec: "args=[\'input\', \'pos\', \'len\', \'unit\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Sum"
-    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SummaryWriter"
-    argspec: "args=[\'shared_name\', \'container\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Svd"
-    argspec: "args=[\'input\', \'compute_uv\', \'full_matrices\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Switch"
-    argspec: "args=[\'data\', \'pred\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SymbolicGradient"
-    argspec: "args=[\'input\', \'Tout\', \'f\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TFRecordDataset"
-    argspec: "args=[\'filenames\', \'compression_type\', \'buffer_size\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TFRecordReader"
-    argspec: "args=[\'container\', \'shared_name\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TFRecordReaderV2"
-    argspec: "args=[\'container\', \'shared_name\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TakeDataset"
-    argspec: "args=[\'input_dataset\', \'count\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TakeManySparseFromTensorsMap"
-    argspec: "args=[\'sparse_handles\', \'dtype\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Tan"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Tanh"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TanhGrad"
-    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TemporaryVariable"
-    argspec: "args=[\'shape\', \'dtype\', \'var_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArray"
-    argspec: "args=[\'size\', \'dtype\', \'dynamic_size\', \'clear_after_read\', \'tensor_array_name\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayClose"
-    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayCloseV2"
-    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayCloseV3"
-    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayConcat"
-    argspec: "args=[\'handle\', \'flow_in\', \'dtype\', \'element_shape_except0\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayConcatV2"
-    argspec: "args=[\'handle\', \'flow_in\', \'dtype\', \'element_shape_except0\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayConcatV3"
-    argspec: "args=[\'handle\', \'flow_in\', \'dtype\', \'element_shape_except0\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayGather"
-    argspec: "args=[\'handle\', \'indices\', \'flow_in\', \'dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayGatherV2"
-    argspec: "args=[\'handle\', \'indices\', \'flow_in\', \'dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayGatherV3"
-    argspec: "args=[\'handle\', \'indices\', \'flow_in\', \'dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayGrad"
-    argspec: "args=[\'handle\', \'flow_in\', \'source\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayGradV2"
-    argspec: "args=[\'handle\', \'flow_in\', \'source\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayGradV3"
-    argspec: "args=[\'handle\', \'flow_in\', \'source\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayGradWithShape"
-    argspec: "args=[\'handle\', \'flow_in\', \'shape_to_prepend\', \'source\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayPack"
-    argspec: "args=[\'handle\', \'flow_in\', \'dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayRead"
-    argspec: "args=[\'handle\', \'index\', \'flow_in\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayReadV2"
-    argspec: "args=[\'handle\', \'index\', \'flow_in\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayReadV3"
-    argspec: "args=[\'handle\', \'index\', \'flow_in\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayScatter"
-    argspec: "args=[\'handle\', \'indices\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayScatterV2"
-    argspec: "args=[\'handle\', \'indices\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayScatterV3"
-    argspec: "args=[\'handle\', \'indices\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArraySize"
-    argspec: "args=[\'handle\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArraySizeV2"
-    argspec: "args=[\'handle\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArraySizeV3"
-    argspec: "args=[\'handle\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArraySplit"
-    argspec: "args=[\'handle\', \'value\', \'lengths\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArraySplitV2"
-    argspec: "args=[\'handle\', \'value\', \'lengths\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArraySplitV3"
-    argspec: "args=[\'handle\', \'value\', \'lengths\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayUnpack"
-    argspec: "args=[\'handle\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayV2"
-    argspec: "args=[\'size\', \'dtype\', \'element_shape\', \'dynamic_size\', \'clear_after_read\', \'tensor_array_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayV3"
-    argspec: "args=[\'size\', \'dtype\', \'element_shape\', \'dynamic_size\', \'clear_after_read\', \'identical_element_shapes\', \'tensor_array_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayWrite"
-    argspec: "args=[\'handle\', \'index\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayWriteV2"
-    argspec: "args=[\'handle\', \'index\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorArrayWriteV3"
-    argspec: "args=[\'handle\', \'index\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorDataset"
-    argspec: "args=[\'components\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListConcat"
-    argspec: "args=[\'input_handle\', \'element_dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListConcatLists"
-    argspec: "args=[\'input_a\', \'input_b\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListElementShape"
-    argspec: "args=[\'input_handle\', \'shape_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListFromTensor"
-    argspec: "args=[\'tensor\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListGather"
-    argspec: "args=[\'input_handle\', \'indices\', \'element_shape\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListGetItem"
-    argspec: "args=[\'input_handle\', \'index\', \'element_shape\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListLength"
-    argspec: "args=[\'input_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListPopBack"
-    argspec: "args=[\'input_handle\', \'element_shape\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListPushBack"
-    argspec: "args=[\'input_handle\', \'tensor\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListPushBackBatch"
-    argspec: "args=[\'input_handles\', \'tensor\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListReserve"
-    argspec: "args=[\'element_shape\', \'num_elements\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListResize"
-    argspec: "args=[\'input_handle\', \'size\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListScatter"
-    argspec: "args=[\'tensor\', \'indices\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListScatterV2"
-    argspec: "args=[\'tensor\', \'indices\', \'element_shape\', \'num_elements\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListSetItem"
-    argspec: "args=[\'input_handle\', \'index\', \'item\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListSplit"
-    argspec: "args=[\'tensor\', \'element_shape\', \'lengths\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorListStack"
-    argspec: "args=[\'input_handle\', \'element_shape\', \'element_dtype\', \'num_elements\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorScatterAdd"
-    argspec: "args=[\'tensor\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorScatterSub"
-    argspec: "args=[\'tensor\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorScatterUpdate"
-    argspec: "args=[\'tensor\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorSliceDataset"
-    argspec: "args=[\'components\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorSummary"
-    argspec: "args=[\'tensor\', \'description\', \'labels\', \'display_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TensorSummaryV2"
-    argspec: "args=[\'tag\', \'tensor\', \'serialized_summary_metadata\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TextLineDataset"
-    argspec: "args=[\'filenames\', \'compression_type\', \'buffer_size\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TextLineReader"
-    argspec: "args=[\'skip_header_lines\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TextLineReaderV2"
-    argspec: "args=[\'skip_header_lines\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ThreadUnsafeUnigramCandidateSampler"
-    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Tile"
-    argspec: "args=[\'input\', \'multiples\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TileGrad"
-    argspec: "args=[\'input\', \'multiples\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Timestamp"
-    argspec: "args=[], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TopK"
-    argspec: "args=[\'input\', \'k\', \'sorted\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TopKV2"
-    argspec: "args=[\'input\', \'k\', \'sorted\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Transpose"
-    argspec: "args=[\'x\', \'perm\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TruncateDiv"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TruncateMod"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "TruncatedNormal"
-    argspec: "args=[\'shape\', \'seed\', \'seed2\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UnicodeDecode"
-    argspec: "args=[\'input\', \'input_encoding\', \'errors\', \'replacement_char\', \'replace_control_characters\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UnicodeDecodeWithOffsets"
-    argspec: "args=[\'input\', \'input_encoding\', \'errors\', \'replacement_char\', \'replace_control_characters\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UnicodeEncode"
-    argspec: "args=[\'input_values\', \'input_splits\', \'errors\', \'output_encoding\', \'replacement_char\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UnicodeScript"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UnicodeTranscode"
-    argspec: "args=[\'input\', \'input_encoding\', \'output_encoding\', \'errors\', \'replacement_char\', \'replace_control_characters\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UniformCandidateSampler"
-    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Unique"
-    argspec: "args=[\'x\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UniqueV2"
-    argspec: "args=[\'x\', \'axis\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UniqueWithCounts"
-    argspec: "args=[\'x\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UniqueWithCountsV2"
-    argspec: "args=[\'x\', \'axis\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Unpack"
-    argspec: "args=[\'value\', \'num\', \'axis\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UnravelIndex"
-    argspec: "args=[\'indices\', \'dims\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UnsortedSegmentMax"
-    argspec: "args=[\'data\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UnsortedSegmentMin"
-    argspec: "args=[\'data\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UnsortedSegmentProd"
-    argspec: "args=[\'data\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UnsortedSegmentSum"
-    argspec: "args=[\'data\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Unstage"
-    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UnwrapDatasetVariant"
-    argspec: "args=[\'input_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UpperBound"
-    argspec: "args=[\'sorted_inputs\', \'values\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "VarHandleOp"
-    argspec: "args=[\'container\', \'shared_name\', \'dtype\', \'shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "VarIsInitializedOp"
-    argspec: "args=[\'resource\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Variable"
-    argspec: "args=[\'shape\', \'dtype\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "VariableShape"
-    argspec: "args=[\'input\', \'out_type\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "VariableV2"
-    argspec: "args=[\'shape\', \'dtype\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Where"
-    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "While"
-    argspec: "args=[\'input\', \'cond\', \'body\', \'output_shapes\', \'parallel_iterations\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "WholeFileReader"
-    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "WholeFileReaderV2"
-    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "WindowDataset"
-    argspec: "args=[\'input_dataset\', \'size\', \'shift\', \'stride\', \'drop_remainder\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "WrapDatasetVariant"
-    argspec: "args=[\'input_handle\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "WriteAudioSummary"
-    argspec: "args=[\'writer\', \'step\', \'tag\', \'tensor\', \'sample_rate\', \'max_outputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "WriteFile"
-    argspec: "args=[\'filename\', \'contents\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "WriteGraphSummary"
-    argspec: "args=[\'writer\', \'step\', \'tensor\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "WriteHistogramSummary"
-    argspec: "args=[\'writer\', \'step\', \'tag\', \'values\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "WriteImageSummary"
-    argspec: "args=[\'writer\', \'step\', \'tag\', \'tensor\', \'bad_color\', \'max_images\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "WriteScalarSummary"
-    argspec: "args=[\'writer\', \'step\', \'tag\', \'value\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "WriteSummary"
-    argspec: "args=[\'writer\', \'step\', \'tensor\', \'tag\', \'summary_metadata\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Xdivy"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Xlogy"
-    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ZerosLike"
-    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Zeta"
-    argspec: "args=[\'x\', \'q\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ZipDataset"
-    argspec: "args=[\'input_datasets\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
-  }
-}
-- 
GitLab


From 41bf5782a38a29aafc6fa3856f800540f56c817f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 30 Jan 2019 14:45:25 -0800
Subject: [PATCH 1626/2345] Return the task_type if the Cluster Resolver is in
 Cloud

PiperOrigin-RevId: 231671573
---
 .../python/distribute/cluster_resolver/tpu_cluster_resolver.py  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
index b8d2ecc3e2..abf628be82 100644
--- a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
+++ b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
@@ -380,7 +380,7 @@ class TPUClusterResolver(ClusterResolver):
 
   def get_job_name(self):
     if (self._shouldResolve() or
-        self._tpu.startswith(compat.as_bytes('grpc://'))):
+        self._isRunningInGCE()):
       return self.task_type
 
   def cluster_spec(self):
-- 
GitLab


From 4a18fbb2558cc80bfb754af0cc3602a938a4af17 Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Wed, 30 Jan 2019 15:14:29 -0800
Subject: [PATCH 1627/2345] Fix the style issue in BUILD file

---
 tensorflow/core/kernels/data/BUILD | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD
index 9833f38768..184c61d4b4 100644
--- a/tensorflow/core/kernels/data/BUILD
+++ b/tensorflow/core/kernels/data/BUILD
@@ -9,8 +9,8 @@ licenses(["notice"])  # Apache 2.0
 
 load(
     "//tensorflow:tensorflow.bzl",
-    "tf_kernel_library",
     "tf_cc_test",
+    "tf_kernel_library",
 )
 
 # TODO(mrry): Remove this empty forwarding library.
@@ -209,18 +209,18 @@ tf_cc_test(
     size = "small",
     srcs = ["map_dataset_op_test.cc"],
     deps = [
-        ":map_dataset_op",
-        ":range_dataset_op",
         ":dataset_test_base",
         ":iterator_ops",
-        "//tensorflow/core:testlib",
-        "//tensorflow/core:test_main",
+        ":map_dataset_op",
+        ":range_dataset_op",
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:dataset_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core/kernels:function_ops",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
         "//tensorflow/core/kernels:cwise_op",
+        "//tensorflow/core/kernels:function_ops",
     ],
 )
 
@@ -396,11 +396,10 @@ tf_cc_test(
     srcs = ["range_dataset_op_test.cc"],
     deps = [
         ":dataset_test_base",
-        ":range_dataset_op",
         ":iterator_ops",
-        "//tensorflow/core:testlib",
+        ":range_dataset_op",
         "//tensorflow/core:test_main",
-
+        "//tensorflow/core:testlib",
     ],
 )
 
-- 
GitLab


From f89b9bd4a311088645ee0dd1ed9745cad9574dfa Mon Sep 17 00:00:00 2001
From: hari <hari@etacompute.com>
Date: Wed, 30 Jan 2019 15:18:33 -0800
Subject: [PATCH 1628/2345] added copytight statement

---
 .../lite/experimental/micro/ecm3531/debug_log.cc  | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/tensorflow/lite/experimental/micro/ecm3531/debug_log.cc b/tensorflow/lite/experimental/micro/ecm3531/debug_log.cc
index 8a3efac838..1e66af7068 100644
--- a/tensorflow/lite/experimental/micro/ecm3531/debug_log.cc
+++ b/tensorflow/lite/experimental/micro/ecm3531/debug_log.cc
@@ -1,3 +1,18 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
 #include "tensorflow/lite/experimental/micro/debug_log.h"
 
 #include "eta_csp_io.h"
-- 
GitLab


From bf793a97d1e6973669a719a6d93c5ba7d28530bd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 30 Jan 2019 15:13:13 -0800
Subject: [PATCH 1629/2345] Add third party source files to list of objects to
 compile.

PiperOrigin-RevId: 231676642
---
 tensorflow/lite/experimental/micro/tools/make/Makefile | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/lite/experimental/micro/tools/make/Makefile b/tensorflow/lite/experimental/micro/tools/make/Makefile
index 9c62d8a36d..741d50eb87 100644
--- a/tensorflow/lite/experimental/micro/tools/make/Makefile
+++ b/tensorflow/lite/experimental/micro/tools/make/Makefile
@@ -159,6 +159,9 @@ include $(wildcard tensorflow/lite/experimental/micro/examples/*/Makefile.inc)
 MICROLITE_LIB_OBJS := $(addprefix $(OBJDIR), \
 $(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(MICROLITE_CC_SRCS))))
 
+MICROLITE_LIB_OBJS += $(addprefix $(OBJDIR), \
+$(patsubst %.S,%.o,$(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(THIRD_PARTY_CC_SRCS)))))
+
 # For normal manually-created TensorFlow C++ source files.
 $(OBJDIR)%.o: %.cc
 	@mkdir -p $(dir $@)
-- 
GitLab


From 7b77734cc4e2e62d89f972bd9ea436f8047831ba Mon Sep 17 00:00:00 2001
From: hari <hari@etacompute.com>
Date: Wed, 30 Jan 2019 15:48:35 -0800
Subject: [PATCH 1630/2345] clang-format

---
 tensorflow/lite/experimental/micro/ecm3531/debug_log.cc | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tensorflow/lite/experimental/micro/ecm3531/debug_log.cc b/tensorflow/lite/experimental/micro/ecm3531/debug_log.cc
index 1e66af7068..4d96196396 100644
--- a/tensorflow/lite/experimental/micro/ecm3531/debug_log.cc
+++ b/tensorflow/lite/experimental/micro/ecm3531/debug_log.cc
@@ -17,6 +17,4 @@ limitations under the License.
 
 #include "eta_csp_io.h"
 
-extern "C" void DebugLog(const char* s) {
- EtaCspIoPrintf("%s", s);
-}
+extern "C" void DebugLog(const char* s) { EtaCspIoPrintf("%s", s); }
-- 
GitLab


From 4e19402174593095a0837d6e7a8ca3f686b67543 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 30 Jan 2019 16:05:42 -0800
Subject: [PATCH 1631/2345] Automated rollback of commit
 71d3161aa4151c2c753c28bc31153c3b7ef5459b

PiperOrigin-RevId: 231686985
---
 tensorflow/core/grappler/optimizers/meta_optimizer.cc | 7 ++++---
 tensorflow/core/protobuf/rewriter_config.proto        | 2 +-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
index c309caa98a..c9d519af11 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
@@ -124,7 +124,8 @@ std::unique_ptr<GraphOptimizer> MetaOptimizer::MakeNewOptimizer(
   MK_OPT("scoped_allocator",
          new ScopedAllocatorOptimizer(cfg_.scoped_allocator_optimization(),
                                       cfg_.scoped_allocator_opts()));
-  MK_OPT("small_op", new PinToHostOptimizer(cfg_.pin_to_host_optimization()));
+  MK_OPT("pin_to_host",
+         new PinToHostOptimizer(cfg_.pin_to_host_optimization()));
 
   return std::unique_ptr<GraphOptimizer>();
 }
@@ -164,7 +165,7 @@ Status MetaOptimizer::InitializeOptimizers(
   if (cfg_.remapping() != RewriterConfig::OFF) {
     optimizers->push_back(MakeUnique<Remapper>(cfg_.remapping()));
   }
-  if (cfg_.pin_to_host_optimization() == RewriterConfig::ON) {
+  if (cfg_.pin_to_host_optimization() != RewriterConfig::OFF) {
     optimizers->push_back(MakeUnique<PinToHostOptimizer>());
   }
   if (cfg_.arithmetic_optimization() != RewriterConfig::OFF) {
@@ -679,7 +680,7 @@ bool MetaOptimizerEnabled(const ConfigProto& cfg) {
          rewrite_cfg.memory_optimization() != RewriterConfig::NO_MEM_OPT ||
          rewrite_cfg.debug_stripper() == RewriterConfig::ON ||
          rewrite_cfg.scoped_allocator_optimization() == RewriterConfig::ON ||
-         rewrite_cfg.pin_to_host_optimization() == RewriterConfig::ON ||
+         rewrite_cfg.pin_to_host_optimization() != RewriterConfig::OFF ||
          !rewrite_cfg.optimizers().empty() ||
          !rewrite_cfg.custom_optimizers().empty();
 }
diff --git a/tensorflow/core/protobuf/rewriter_config.proto b/tensorflow/core/protobuf/rewriter_config.proto
index 17e76c4d03..b5c9599872 100644
--- a/tensorflow/core/protobuf/rewriter_config.proto
+++ b/tensorflow/core/protobuf/rewriter_config.proto
@@ -76,7 +76,7 @@ message RewriterConfig {
   // Try to allocate some independent Op outputs contiguously in order to
   // merge or eliminate downstream Ops (off by default).
   Toggle scoped_allocator_optimization = 15;
-  // Force small ops onto the CPU (default is OFF).
+  // Force small ops onto the CPU (default is ON).
   Toggle pin_to_host_optimization = 18;
   // Disable the entire meta optimizer (off by default).
   bool disable_meta_optimizer = 19;
-- 
GitLab


From 7285f3fbddf2054c3971973f62f998579d00d8ec Mon Sep 17 00:00:00 2001
From: Mustafa Ispir <ispir@google.com>
Date: Wed, 30 Jan 2019 16:21:54 -0800
Subject: [PATCH 1632/2345] Documents that users can provide logits instead of
 probabilities in SparceCategoricalAccuracy metric.

PiperOrigin-RevId: 231689841
---
 tensorflow/python/keras/metrics.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index f7ecbefa3a..ca0939e363 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -651,7 +651,8 @@ class CategoricalAccuracy(MeanMetricWrapper):
   For example, if `y_true` is [[0, 0, 1], [0, 1, 0]] and `y_pred` is
   [[0.1, 0.9, 0.8], [0.05, 0.95, 0]] then the categorical accuracy is 1/2 or .5.
   If the weights were specified as [0.7, 0.3] then the categorical accuracy
-  would be .3.
+  would be .3. You can provide logits of classes as `y_pred`, since argmax of
+  logits and probabilities are same.
 
   This metric creates two local variables, `total` and `count` that are used to
   compute the frequency with which `y_pred` matches `y_true`. This frequency is
@@ -701,7 +702,8 @@ class SparseCategoricalAccuracy(MeanMetricWrapper):
   For example, if `y_true` is [[2], [1]] and `y_pred` is
   [[0.1, 0.9, 0.8], [0.05, 0.95, 0]] then the categorical accuracy is 1/2 or .5.
   If the weights were specified as [0.7, 0.3] then the categorical accuracy
-  would be .3.
+  would be .3. You can provide logits of classes as `y_pred`, since argmax of
+  logits and probabilities are same.
 
   This metric creates two local variables, `total` and `count` that are used to
   compute the frequency with which `y_pred` matches `y_true`. This frequency is
-- 
GitLab


From 9f75ff921b59d12b17afdc33597f190b646e6c01 Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Wed, 30 Jan 2019 16:50:05 -0800
Subject: [PATCH 1633/2345] [tf.data] Map vectorization bug fix: prevent
 segfault when BatchDataset has any unknown output shapes.

PiperOrigin-RevId: 231694628
---
 .../grappler/optimizers/data/map_vectorization.cc | 13 +++++++++++--
 .../optimization/map_vectorization_test.py        | 15 +++++++++++++++
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
index 13933b6947..5c8f780533 100644
--- a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
@@ -224,9 +224,18 @@ Status AddNewBatchNode(const NodeDef& old_batch_node, const NodeDef& input_node,
   auto& output_shapes_attr = (*batch_node.mutable_attr())["output_shapes"];
   const auto& input_shapes =
       input_node.attr().at("output_shapes").list().shape();
-  int64 batch_size =
-      old_batch_node.attr().at("output_shapes").list().shape()[0].dim(0).size();
+
+  int64 batch_size = -1;
+  for (const auto& shape :
+       old_batch_node.attr().at("output_shapes").list().shape()) {
+    if (!shape.unknown_rank()) {
+      batch_size = shape.dim(0).size();
+      break;
+    }
+  }
+
   for (size_t i = 0; i < input_shapes.size(); ++i) {
+    // Note: We already checked earlier that input shapes are all fully defined.
     TensorShapeProto* shape = output_shapes_attr.mutable_list()->add_shape();
     TensorShapeProto_Dim* dim = shape->add_dim();
     dim->set_size(batch_size);
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
index ef4dbc8a71..bba2f352e2 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
@@ -519,6 +519,21 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
         [("OneShotIterator", "OneShotIterator_1", 1),
          ("IteratorGetNext", "IteratorGetNext_1", 1)])
 
+  def testOptimizationWithUnknownBatchShape(self):
+    tensor = sparse_tensor.SparseTensor(
+        indices=[[0, 0], [1, 2]], values=[1, 2], dense_shape=[3, 4])
+
+    # Datasets with sparse tensors have unknown output shapes.
+    base_dataset = dataset_ops.Dataset.from_tensors(tensor)
+    unoptimized = base_dataset.apply(batching.map_and_batch(lambda x: x, 2))
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    unoptimized = unoptimized.with_options(options)
+
+    options = dataset_ops.Options()
+    options.experimental_optimization.map_vectorization = True
+    optimized = unoptimized.with_options(options)
+    self.assertDatasetsEqual(unoptimized, optimized)
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 74534b24401305ad6c13ac8f951d555ed638e978 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Wed, 30 Jan 2019 16:51:22 -0800
Subject: [PATCH 1634/2345] Fix a bug for QuantileAccumulator saveable in saved
 model.

PiperOrigin-RevId: 231694849
---
 tensorflow/python/ops/boosted_trees_ops.py | 110 +++++++++++++--------
 1 file changed, 71 insertions(+), 39 deletions(-)

diff --git a/tensorflow/python/ops/boosted_trees_ops.py b/tensorflow/python/ops/boosted_trees_ops.py
index 37d649acf0..f6c37027ae 100644
--- a/tensorflow/python/ops/boosted_trees_ops.py
+++ b/tensorflow/python/ops/boosted_trees_ops.py
@@ -61,7 +61,36 @@ class PruningMode(object):
           sorted(cls._map))))
 
 
-class QuantileAccumulator(saver.BaseSaverBuilder.SaveableObject):
+class QuantileAccumulatorSaveable(saver.BaseSaverBuilder.SaveableObject):
+  """SaveableObject implementation for QuantileAccumulator."""
+
+  def __init__(self, resource_handle, create_op, num_streams, name):
+    self._resource_handle = resource_handle
+    self._num_streams = num_streams
+    self._create_op = create_op
+    bucket_boundaries = get_bucket_boundaries(self._resource_handle,
+                                              self._num_streams)
+    slice_spec = ''
+    specs = []
+
+    def make_save_spec(tensor, suffix):
+      return saver.BaseSaverBuilder.SaveSpec(tensor, slice_spec, name + suffix)
+
+    for i in range(self._num_streams):
+      specs += [
+          make_save_spec(bucket_boundaries[i], '_bucket_boundaries_' + str(i))
+      ]
+    super(QuantileAccumulatorSaveable, self).__init__(self._resource_handle,
+                                                      specs, name)
+
+  def restore(self, restored_tensors, unused_tensor_shapes):
+    bucket_boundaries = restored_tensors
+    with ops.control_dependencies([self._create_op]):
+      return quantile_resource_deserialize(
+          self._resource_handle, bucket_boundaries=bucket_boundaries)
+
+
+class QuantileAccumulator(tracking.TrackableResource):
   """SaveableObject implementation for QuantileAccumulator.
 
      The bucket boundaries are serialized and deserialized from checkpointing.
@@ -73,55 +102,58 @@ class QuantileAccumulator(saver.BaseSaverBuilder.SaveableObject):
                num_quantiles,
                name=None,
                max_elements=None):
+    self._eps = epsilon
+    self._num_streams = num_streams
+    self._num_quantiles = num_quantiles
+    super(QuantileAccumulator, self).__init__()
+
     with ops.name_scope(name, 'QuantileAccumulator') as name:
-      self._eps = epsilon
-      self._num_streams = num_streams
-      self._num_quantiles = num_quantiles
-      self._resource_handle = quantile_resource_handle_op(
-          container='', shared_name=name, name=name)
-      self._create_op = create_quantile_stream_resource(self._resource_handle,
-                                                        epsilon, num_streams)
-      is_initialized_op = is_quantile_resource_initialized(
-          self._resource_handle)
-      resources.register_resource(self._resource_handle, self._create_op,
-                                  is_initialized_op)
-      self._make_saveable(name)
-
-  def _make_saveable(self, name):
-    bucket_boundaries = get_bucket_boundaries(self._resource_handle,
-                                              self._num_streams)
-    slice_spec = ''
-    specs = []
-    for i in range(self._num_streams):
-      specs.append(
-          saver.BaseSaverBuilder.SaveSpec(
-              bucket_boundaries[i], slice_spec,
-              name + '_bucket_boundaries_' + str(i)))
-    super(QuantileAccumulator, self).__init__(self._resource_handle, specs,
-                                              name)
-    ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, self)
+      self._name = name
+      self._resource_handle = self.create_resource()
+      self._init_op = self.initialize()
+      is_initialized_op = self.is_initialized()
+    resources.register_resource(self.resource_handle, self._init_op,
+                                is_initialized_op)
+    self._saveable = QuantileAccumulatorSaveable(
+        self.resource_handle, self._init_op, self._num_streams,
+        self.resource_handle.name)
+    ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, self._saveable)
 
-  def restore(self, restored_tensors, unused_tensor_shapes):
-    bucket_boundaries = restored_tensors
-    with ops.control_dependencies([self._create_op]):
-      return quantile_resource_deserialize(
-          self._resource_handle, bucket_boundaries=bucket_boundaries)
+  def create_resource(self):
+    return quantile_resource_handle_op(
+        container='', shared_name=self._name, name=self._name)
+
+  def initialize(self):
+    return create_quantile_stream_resource(self.resource_handle, self._eps,
+                                           self._num_streams)
+
+  @property
+  def initializer(self):
+    if self._init_op is None:
+      self._init_op = self.initialize()
+    return self._init_op
+
+  def is_initialized(self):
+    return is_quantile_resource_initialized(self.resource_handle)
+
+  @property
+  def saveable(self):
+    return self._saveable
+
+  def _gather_saveables_for_checkpoint(self):
+    return {'quantile_accumulator', self._saveable}
 
   def add_summaries(self, float_columns, example_weights):
     summaries = make_quantile_summaries(float_columns, example_weights,
                                         self._eps)
-    summary_op = quantile_add_summaries(self._resource_handle, summaries)
+    summary_op = quantile_add_summaries(self.resource_handle, summaries)
     return summary_op
 
   def flush(self):
-    return quantile_flush(self._resource_handle, self._num_quantiles)
+    return quantile_flush(self.resource_handle, self._num_quantiles)
 
   def get_bucket_boundaries(self):
-    return get_bucket_boundaries(self._resource_handle, self._num_streams)
-
-  @property
-  def resource(self):
-    return self._resource_handle
+    return get_bucket_boundaries(self.resource_handle, self._num_streams)
 
 
 class _TreeEnsembleSavable(saver.BaseSaverBuilder.SaveableObject):
-- 
GitLab


From 8d76177156a407ace03165339c8438579db81ba3 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Wed, 30 Jan 2019 17:00:45 -0800
Subject: [PATCH 1635/2345] Always copy tensors to the remote CPU when
 executing a remote kernel.

The copy to the appropriate device will happen when the kernel is executing.

Without this, its possible we'll copy tensors to an accelerator device when the
op expects host memory inputs.

PiperOrigin-RevId: 231696485
---
 .../core/common_runtime/eager/execute.cc       | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/common_runtime/eager/execute.cc b/tensorflow/core/common_runtime/eager/execute.cc
index 53e9ba232a..c6e8573cc2 100644
--- a/tensorflow/core/common_runtime/eager/execute.cc
+++ b/tensorflow/core/common_runtime/eager/execute.cc
@@ -252,6 +252,16 @@ bool OnSameTask(EagerContext* ctx, Device* first, Device* second) {
          first->parsed_name().task == second->parsed_name().task;
 }
 
+// Gets the CPU device on the task of device.
+Status CPUDeviceOnTask(EagerContext* ctx, tensorflow::Device* device,
+                       tensorflow::Device** cpu_device) {
+  string cpu_device_name;
+  TF_RETURN_IF_ERROR(DeviceNameUtils::DeviceNameToCpuDeviceName(
+      device->name(), &cpu_device_name));
+
+  return ctx->FindDeviceByName(cpu_device_name, cpu_device);
+}
+
 inline tensorflow::Fprint128 FingerprintCat128(const tensorflow::Fprint128& a,
                                                const tensorflow::Fprint128& b) {
   return {tensorflow::FingerprintCat64(a.low64, b.low64),
@@ -628,10 +638,16 @@ Status EagerRemoteExecute(EagerOperation* op, TensorHandle** retvals,
         // explicitly copy, and instead depend on the copy to happen locally
         // when the op is executed on the device.
         !OnSameTask(ctx, op->Device(), input_device)) {
+      tensorflow::Device* remote_cpu_device;
+      TF_RETURN_IF_ERROR(
+          CPUDeviceOnTask(ctx, op->Device(), &remote_cpu_device));
       // TODO(b/110044833): It's possible the same tensor gets copied to the
       // remote device repeatedly.
+      // Always copy to the remote CPU so that the actual device can be
+      // correctly determined after the kernel is selected/instantiated, since
+      // the op might have its inputs on host memory.
       TF_RETURN_IF_ERROR(MaybeCopyInputToExpectedDevice(
-          op, op->Device(), i, op->Device(), /* run_metadata= */ nullptr,
+          op, op->Device(), i, remote_cpu_device, /* run_metadata= */ nullptr,
           &(*op->MutableInputs())[i]));
     }
 
-- 
GitLab


From ff28e28f75fe296f160d3b17406bd5b446ced267 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 30 Jan 2019 17:16:30 -0800
Subject: [PATCH 1636/2345] Remove ThenNormalize

PiperOrigin-RevId: 231699104
---
 tensorflow/stream_executor/cuda/cuda_dnn.cc |  7 -------
 tensorflow/stream_executor/cuda/cuda_dnn.h  |  5 -----
 tensorflow/stream_executor/dnn.h            | 12 ------------
 tensorflow/stream_executor/stream.cc        | 16 ----------------
 tensorflow/stream_executor/stream.h         |  6 ------
 5 files changed, 46 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index 283c907ab0..bae71b4e8d 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -4132,13 +4132,6 @@ bool CudnnSupport::DoPoolBackward(
   return IsStatusOk(status, /*report_error=*/true);
 }
 
-bool CudnnSupport::DoNormalize(
-    Stream* stream, const dnn::NormalizeDescriptor& normalize_descriptor,
-    const DeviceMemory<float>& input_data, DeviceMemory<float>* output_data) {
-  LOG(FATAL) << "not yet implemented";  // TODO(leary)
-  return false;
-}
-
 bool CudnnSupport::DoNormalizeWithDimensions(
     Stream* stream, const dnn::NormalizeDescriptor& normalize_descriptor,
     const dnn::BatchDescriptor& dimensions,
diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.h b/tensorflow/stream_executor/cuda/cuda_dnn.h
index 80c7c8a06c..d8a8ddf6a5 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.h
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.h
@@ -484,11 +484,6 @@ class CudnnSupport : public dnn::DnnSupport {
                       DeviceMemory<Eigen::half>* output_diff_data,
                       ScratchAllocator* workspace_allocator) override;
 
-  bool DoNormalize(Stream* stream,
-                   const dnn::NormalizeDescriptor& normalize_descriptor,
-                   const DeviceMemory<float>& input_data,
-                   DeviceMemory<float>* output_data) override;
-
   bool DoNormalizeWithDimensions(
       Stream* stream, const dnn::NormalizeDescriptor& normalize_descriptor,
       const dnn::BatchDescriptor& dimensions,
diff --git a/tensorflow/stream_executor/dnn.h b/tensorflow/stream_executor/dnn.h
index 1d9a2be951..24c29486d0 100644
--- a/tensorflow/stream_executor/dnn.h
+++ b/tensorflow/stream_executor/dnn.h
@@ -1648,21 +1648,9 @@ class DnnSupport {
     return false;
   }
 
-  // Applies local response normalization to the values from
-  // input_data and writes the result to output_data. See comments on
-  // NormalizeDescriptor for a description of local response
-  // normalization.
-  virtual bool DoNormalize(Stream* stream,
-                           const dnn::NormalizeDescriptor& normalize_descriptor,
-                           const DeviceMemory<float>& input_data,
-                           DeviceMemory<float>* output_data) = 0;
-
   // Applies local response normalization to the values from input_data and
   // writes the result to output_data.
   //
-  // Similar to DoNormalize, but normalizes across feature maps and allows for
-  // specifying the dimensions of the tensor.
-  //
   // See comments on NormalizeDescriptor for a description of local response
   // normalization.
   virtual bool DoNormalizeWithDimensions(
diff --git a/tensorflow/stream_executor/stream.cc b/tensorflow/stream_executor/stream.cc
index e7485ca426..fde7730cfe 100644
--- a/tensorflow/stream_executor/stream.cc
+++ b/tensorflow/stream_executor/stream.cc
@@ -1742,22 +1742,6 @@ Stream &Stream::ThenPoolBackward(
   return *this;
 }
 
-Stream &Stream::ThenNormalize(
-    const dnn::NormalizeDescriptor &normalize_descriptor,
-    const DeviceMemory<float> &input_data, DeviceMemory<float> *output_data) {
-  VLOG_CALL(PARAM(normalize_descriptor), PARAM(input_data), PARAM(output_data));
-
-  if (ok()) {
-    if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      CheckError(dnn->DoNormalize(this, normalize_descriptor, input_data,
-                                  output_data));
-    } else {
-      SetErrorAndLogNoDnnSupport();
-    }
-  }
-  return *this;
-}
-
 Stream &Stream::ThenNormalizeWithDimensions(
     const dnn::NormalizeDescriptor &normalize_descriptor,
     const dnn::BatchDescriptor &dimensions,
diff --git a/tensorflow/stream_executor/stream.h b/tensorflow/stream_executor/stream.h
index f698d50e2a..65025c2bc8 100644
--- a/tensorflow/stream_executor/stream.h
+++ b/tensorflow/stream_executor/stream.h
@@ -684,12 +684,6 @@ class Stream {
                            DeviceMemory<Eigen::half> *output_diff_data,
                            ScratchAllocator *workspace_allocator = nullptr);
 
-  Stream &ThenNormalize(const dnn::NormalizeDescriptor &normalize_descriptor,
-                        const DeviceMemory<float> &input_data,
-                        DeviceMemory<float> *output_data);
-
-  // Similar to ThenNormalize, but normalizes across feature maps and allows for
-  // specifying the dimensions of the tensor.
   Stream &ThenNormalizeWithDimensions(
       const dnn::NormalizeDescriptor &normalize_descriptor,
       const dnn::BatchDescriptor &dimensions,
-- 
GitLab


From 2b5d703f26f380cc65ec3ac2bf74ce06cc59416e Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Wed, 30 Jan 2019 17:16:40 -0800
Subject: [PATCH 1637/2345] Always convert NumPy to Tensors in `__call__`

PiperOrigin-RevId: 231699124
---
 tensorflow/python/keras/engine/base_layer.py          | 11 +++++++++--
 .../python/keras/layers/tensorflow_op_layer_test.py   |  2 +-
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index 12d3f09bfe..e91dce5e5d 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -509,8 +509,15 @@ class Layer(checkpointable.Checkpointable):
     """
     input_list = nest.flatten(inputs)
     # Accept NumPy inputs by converting to Tensors.
-    if all(isinstance(x, (np.ndarray, float, int)) for x in input_list):
-      inputs = nest.map_structure(ops.convert_to_tensor, inputs)
+    if any(isinstance(x, (np.ndarray, float, int)) for x in input_list):
+      # Don't call `ops.convert_to_tensor` on all `inputs` because
+      # `SparseTensors` can't be converted to `Tensor`.
+      def _convert_non_tensor(x):
+        if isinstance(x, (np.ndarray, float, int)):
+          return ops.convert_to_tensor(x)
+        return x
+
+      inputs = nest.map_structure(_convert_non_tensor, inputs)
       input_list = nest.flatten(inputs)
 
     # We will attempt to build a TF graph if & only if all inputs are symbolic.
diff --git a/tensorflow/python/keras/layers/tensorflow_op_layer_test.py b/tensorflow/python/keras/layers/tensorflow_op_layer_test.py
index b83508ee66..f01c9d4964 100644
--- a/tensorflow/python/keras/layers/tensorflow_op_layer_test.py
+++ b/tensorflow/python/keras/layers/tensorflow_op_layer_test.py
@@ -161,7 +161,7 @@ class AutoLambdaTest(keras_parameterized.TestCase):
     # Check reasonable graph construction time.
     self.assertLess(size_50, 5)
     # Check construction time grows approx. linearly with size.
-    e = 1.5  # Fudge factor to prevent flakiness.
+    e = 2  # Fudge factor to prevent flakiness.
     self.assertLess(size_500, (10 * e) * size_50)
 
 
-- 
GitLab


From 084fac1e3b81ff124770638b6880097765899abe Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Wed, 30 Jan 2019 17:16:52 -0800
Subject: [PATCH 1638/2345] while_v2: Allow capturing the same tensor in both
 cond and body.

PiperOrigin-RevId: 231699147
---
 .../kernel_tests/control_flow_ops_py_test.py  | 19 ++++----
 tensorflow/python/ops/while_v2.py             | 48 ++++++++-----------
 2 files changed, 28 insertions(+), 39 deletions(-)

diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index c5b7a95c61..d8f91034cc 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -1358,7 +1358,7 @@ class ControlFlowTest(test.TestCase):
           r"while loop context '' \(currently defined in 'cond/.+'\)"):
         _ = gradients_impl.gradients(loop, v)
 
-  @test_util.disable_control_flow_v2("b/118457764")
+  @test_util.disable_control_flow_v2("b/123601232")
   @test_util.run_v1_only("b/120545219")
   def testNestedWhileLoopWithMaxItersFromOuterContextInXLAContext(self):
     v = constant_op.constant(1.0)
@@ -3134,25 +3134,24 @@ class ControlFlowTest(test.TestCase):
   def testNestedWhileAndTensorArray(self):
     n = constant_op.constant(3.0)
 
-    def Body(row, ta, n):
+    def Body(row, ta):
 
-      def InnerBody(row, col, ta, n):
+      def InnerBody(row, col, ta):
         # Note: row and col are 1-based.
         ta = ta.write(
             math_ops.cast(n * (row - 1.) + col - 1., dtypes.int32), row * col)
-        return row, col + 1., ta, n
+        return row, col + 1., ta
 
-      # TODO(b/118457764): Remove n from loop_vars from both loops once fixed.
       ta = control_flow_ops.while_loop(
-          lambda _, col, _1, n: col <= n,
-          InnerBody, [row, constant_op.constant(1.), ta, n],
+          lambda _, col, _1: col <= n,
+          InnerBody, [row, constant_op.constant(1.), ta],
           return_same_structure=False)[2]
-      return row + 1., ta, n
+      return row + 1., ta
 
     ta = tensor_array_ops.TensorArray(dtype=dtypes.float32, size=9)
     ta = control_flow_ops.while_loop(
-        lambda row, _, _1: row <= n,
-        Body, [constant_op.constant(1.), ta, n],
+        lambda row, _: row <= n,
+        Body, [constant_op.constant(1.), ta],
         return_same_structure=False)[1]
 
     output = array_ops.reshape(ta.stack(), [3, 3])
diff --git a/tensorflow/python/ops/while_v2.py b/tensorflow/python/ops/while_v2.py
index 0e427d3c6a..584faf11df 100644
--- a/tensorflow/python/ops/while_v2.py
+++ b/tensorflow/python/ops/while_v2.py
@@ -122,40 +122,34 @@ def while_loop(cond,
     cond_graph = func_graph_module.func_graph_from_py_func(
         cond_name,
         wrapped_cond,
-        loop_vars, {},
+        [],  # We provide signature instead of args.
+        {},
         signature=_build_signature(loop_vars, shape_invariants),
         func_graph=util.WhileCondFuncGraph(
             cond_name, collections=ops.get_default_graph()._collections),  # pylint: disable=protected-access
         add_control_dependencies=add_control_dependencies)
 
-    # Add external_captures of cond to the list of loop vars.
-    # Note that external tensors will be treated as loop invariants, i.e.,
-    # the value of that tensor in each iteration is the same as it was at the
-    # beginning of the loop execution.
-    loop_vars = loop_vars + cond_graph.external_captures
-    shape_invariants = shape_invariants + type(shape_invariants)(
-        [t.shape for t in cond_graph.external_captures])
-
     def wrapped_body(loop_counter, *args):
       """Loop body augmented with counter update.
 
       Args:
         loop_counter: Loop counter which needs to be incremented in the body.
         *args: List of args
-          args[:len_orig_loop_vars] - Args for the original loop body.
-          args[len_orig_loop_vars:] - External captures of cond. These get
-            passed through as is.
 
       Returns:
         A list of tensors the same length as args.
       """
+      # Capture the tensors already captured in cond_graph so that they appear
+      # in the same order in body_graph.external_captures.
+      for t in cond_graph.external_captures:
+        ops.get_default_graph().capture(t)
+
       # Convert the flow variables in `args` to TensorArrays. `args` should
       # already have the same structure as `orig_loop_vars` but currently there
       # is no nest.zip so we call `_pack_sequence_as` which flattens both
       # `orig_loop_vars` and `args`, converts flows in `args` to TensorArrays
       # and packs it into the structure of `orig_loop_vars`.
-      outputs = body(
-          *_pack_sequence_as(orig_loop_vars, args[:len_orig_loop_vars]))
+      outputs = body(*_pack_sequence_as(orig_loop_vars, args))
       if not nest.is_sequence(outputs):
         outputs = [outputs]
       # Compare the structure of input and output of body converting the
@@ -164,17 +158,15 @@ def while_loop(cond,
 
       outputs = _tensor_array_to_flow(outputs)
 
-      # Return the external_captures of cond_graph as is, i.e., treat them as
-      # loop invariants.
       # TODO(srbs): Update lowering code to create _Enter nodes with
       # is_constant=True for inputs that are directly passed to outputs.
-      return [loop_counter + 1] + list(outputs) + list(
-          args[len_orig_loop_vars:])
+      return [loop_counter + 1] + list(outputs)
 
     body_graph = func_graph_module.func_graph_from_py_func(
         body_name,
         wrapped_body,
-        loop_vars, {},
+        [],  # We provide signature instead of args.
+        {},
         signature=_build_signature(loop_vars, shape_invariants),
         func_graph=util.WhileBodyFuncGraph(
             body_name, collections=ops.get_default_graph()._collections),  # pylint: disable=protected-access
@@ -188,17 +180,15 @@ def while_loop(cond,
     # is_constant=True for inputs that are directly passed to outputs.
     body_graph.outputs.extend(body_graph.internal_captures)
 
-    # Capture `external_captures` of `body_graph` in `cond_graph` so that it
-    # expects to receive those as arguments.
-    # TODO(b/118457764): Dedup tensors that are captured in both the cond and
-    # body. This logic already exists in cond_v2.
+    # Capture the extra `external_captures` of `body_graph` in `cond_graph` so
+    # that it expects to receive those as arguments.
     with cond_graph.as_default():
-      for external_capture in body_graph.external_captures:
-        assert external_capture not in cond_graph.captures, (
-            "Looks like both cond and body are capturing the same tensor %s. "
-            "This is not supported yet. For now consider passing,"
-            " this as a loop variable." % str(external_capture))
-        cond_graph.capture(external_capture)
+      num_cond_captures = len(cond_graph.external_captures)
+      assert (cond_graph.external_captures ==
+              body_graph.external_captures[:num_cond_captures])
+      for body_capture in body_graph.external_captures[num_cond_captures:]:
+        assert body_capture not in cond_graph.captures
+        cond_graph.capture(body_capture)
 
     # Make sure that the shapes of the loop outputs are compatible with the
     # shape invariants, or the shapes of the loop vars if the invariants are not
-- 
GitLab


From 5c987ba0d9ce21edf2b9a0e4b45878a0636e8166 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Wed, 30 Jan 2019 17:27:24 -0800
Subject: [PATCH 1639/2345] Add tf.gradients and tf.hessians to reorders and
 add tests for conv functions.

PiperOrigin-RevId: 231700622
---
 tensorflow/tools/compatibility/reorders_v2.py |  2 +
 .../tools/compatibility/tf_upgrade_v2.py      |  2 +
 .../tools/compatibility/tf_upgrade_v2_test.py | 52 +++++++++++++++++--
 3 files changed, 53 insertions(+), 3 deletions(-)

diff --git a/tensorflow/tools/compatibility/reorders_v2.py b/tensorflow/tools/compatibility/reorders_v2.py
index 01556b1225..bd5ccf6ca8 100644
--- a/tensorflow/tools/compatibility/reorders_v2.py
+++ b/tensorflow/tools/compatibility/reorders_v2.py
@@ -37,6 +37,8 @@ reorders = {
     'tf.decode_csv': ['records', 'record_defaults', 'field_delim', 'use_quote_delim', 'name', 'na_value', 'select_cols'],
     'tf.depth_to_space': ['input', 'block_size', 'name', 'data_format'],
     'tf.feature_column.categorical_column_with_vocabulary_file': ['key', 'vocabulary_file', 'vocabulary_size', 'num_oov_buckets', 'default_value', 'dtype'],
+    'tf.gradients': ['ys', 'xs', 'grad_ys', 'name', 'colocate_gradients_with_ops', 'gate_gradients', 'aggregation_method', 'stop_gradients', 'unconnected_gradients'],
+    'tf.hessians': ['ys', 'xs', 'name', 'colocate_gradients_with_ops', 'gate_gradients', 'aggregation_method'],
     'tf.image.sample_distorted_bounding_box': ['image_size', 'bounding_boxes', 'seed', 'seed2', 'min_object_covered', 'aspect_ratio_range', 'area_range', 'max_attempts', 'use_image_if_no_bounding_boxes', 'name'],
     'tf.io.decode_csv': ['records', 'record_defaults', 'field_delim', 'use_quote_delim', 'name', 'na_value', 'select_cols'],
     'tf.io.parse_example': ['serialized', 'features', 'name', 'example_names'],
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 74d29730b6..11ac70df0a 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -808,6 +808,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.nn.fractional_avg_pool",
         "tf.nn.fractional_max_pool",
         "tf.image.sample_distorted_bounding_box",
+        "tf.gradients",
+        "tf.hessians",
     }
 
     # Functions that were reordered should be changed to the new keyword args
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 440c1635e2..bbe074e242 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -653,16 +653,22 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
     self.assertEqual(errors, [])
 
   def testColocateGradientsWithOps(self):
-    text = "tf.gradients(a, foo=False)\n"
+    text = "tf.gradients(yx=a, foo=False)\n"
     _, unused_report, errors, new_text = self._upgrade(text)
     self.assertEqual(text, new_text)
     self.assertEqual(errors, [])
 
-    text = "tf.gradients(a, colocate_gradients_with_ops=False)\n"
+    text = "tf.gradients(yx=a, colocate_gradients_with_ops=False)\n"
     _, report, unused_errors, new_text = self._upgrade(text)
-    self.assertEqual("tf.gradients(a)\n", new_text)
+    self.assertEqual("tf.gradients(yx=a)\n", new_text)
     self.assertIn("tf.gradients no longer takes", report)
 
+    text = "tf.gradients(y, x, grad_ys, name, colocate, gate)\n"
+    expected = ("tf.gradients(ys=y, xs=x, grad_ys=grad_ys, name=name, "
+                "gate_gradients=gate)\n")
+    _, unused_report, errors, new_text = self._upgrade(text)
+    self.assertEqual(expected, new_text)
+
   def testColocateGradientsWithOpsMinimize(self):
     text = "optimizer.minimize(a, foo=False)\n"
     _, unused_report, errors, new_text = self._upgrade(text)
@@ -849,6 +855,46 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
     _, unused_report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, expected_text)
 
+  def testConv2D(self):
+    text = (
+        "tf.nn.conv2d(input, filter, strides, padding, use_cudnn_on_gpu, "
+        "data_format)")
+    expected_text = (
+        "tf.nn.conv2d(input=input, filters=filter, strides=strides, "
+        "padding=padding, data_format=data_format)")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
+    text = (
+        "tf.nn.conv2d(input, filter=filter, strides=strides, padding=padding, "
+        "use_cudnn_on_gpu=use_cudnn_on_gpu)")
+    expected_text = ("tf.nn.conv2d(input=input, filters=filter, "
+                     "strides=strides, padding=padding)")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
+  def testConv2DBackpropFilter(self):
+    text = (
+        "tf.nn.conv2d_backprop_filter(input, filter_sizes, out_backprop, "
+        "strides, padding, use_cudnn_on_gpu, data_format)")
+    expected_text = (
+        "tf.nn.conv2d_backprop_filter(input=input, filter_sizes=filter_sizes, "
+        "out_backprop=out_backprop, strides=strides, padding=padding, "
+        "data_format=data_format)")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
+  def testConv2DBackpropInput(self):
+    text = (
+        "tf.nn.conv2d_backprop_input(input_sizes, filter, out_backprop, "
+        "strides, padding, use_cudnn_on_gpu, data_format)")
+    expected_text = (
+        "tf.nn.conv2d_backprop_input(input_sizes=input_sizes, filters=filter, "
+        "out_backprop=out_backprop, strides=strides, padding=padding, "
+        "data_format=data_format)")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
   def testSpacetoBatch(self):
     text = "tf.space_to_batch_nd(input, shape, paddings, name)"
     expected_text = "tf.space_to_batch(input, shape, paddings, name)"
-- 
GitLab


From a8189620ee6eb51979e0a68d50a60da105094b35 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 30 Jan 2019 17:32:25 -0800
Subject: [PATCH 1640/2345] Add support for decoding extensions.

PiperOrigin-RevId: 231701359
---
 .../kernel_tests/decode_proto_op_test_base.py | 18 ++++++---
 .../kernel_tests/encode_proto_op_test_base.py |  9 ++---
 .../python/kernel_tests/proto_op_test_base.py | 38 +++++++++++++------
 .../python/kernel_tests/test_example.proto    |  8 +++-
 .../base_api/api_def_DecodeProtoV2.pbtxt      |  3 +-
 tensorflow/core/kernels/BUILD                 |  1 +
 tensorflow/core/kernels/decode_proto_op.cc    | 30 +++++++++++++++
 7 files changed, 83 insertions(+), 24 deletions(-)

diff --git a/tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test_base.py b/tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test_base.py
index 17b69c7b35..c8524e9871 100644
--- a/tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test_base.py
+++ b/tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test_base.py
@@ -84,7 +84,10 @@ class DecodeProtoOpTestBase(test_base.ProtoOpTestBase, parameterized.TestCase):
       values = field_dict[field.name]
       self.assertEqual(dtypes.as_dtype(values.dtype), field.dtype)
 
-      fd = field.value.DESCRIPTOR.fields_by_name[field.name]
+      if 'ext_value' in field.name:
+        fd = test_example_pb2.PrimitiveValue()
+      else:
+        fd = field.value.DESCRIPTOR.fields_by_name[field.name]
 
       # Values has the same shape as the input plus an extra
       # dimension for repeats.
@@ -92,13 +95,16 @@ class DecodeProtoOpTestBase(test_base.ProtoOpTestBase, parameterized.TestCase):
 
       # Nested messages are represented as TF strings, requiring
       # some special handling.
-      if field.name == 'message_value':
+      if field.name == 'message_value' or 'ext_value' in field.name:
         vs = []
         for buf in values.flat:
           msg = test_example_pb2.PrimitiveValue()
           msg.ParseFromString(buf)
           vs.append(msg)
-        evs = getattr(field.value, field.name)
+        if 'ext_value' in field.name:
+          evs = field.value.Extensions[test_example_pb2.ext_value]
+        else:
+          evs = getattr(field.value, field.name)
         if len(vs) != len(evs):
           self.fail('Field %s decoded %d outputs, expected %d' %
                     (fd.name, len(vs), len(evs)))
@@ -223,7 +229,8 @@ class DecodeProtoOpTestBase(test_base.ProtoOpTestBase, parameterized.TestCase):
         sanitize=False,
         force_disordered=True)
 
-  @parameterized.named_parameters(*test_base.ProtoOpTestBase.named_parameters())
+  @parameterized.named_parameters(
+      *test_base.ProtoOpTestBase.named_parameters(extension=False))
   def testPacked(self, case):
     # Now try with the packed serialization.
     #
@@ -235,8 +242,7 @@ class DecodeProtoOpTestBase(test_base.ProtoOpTestBase, parameterized.TestCase):
         # Note: float_format='.17g' is necessary to ensure preservation of
         # doubles and floats in text format.
         text_format.Parse(
-            text_format.MessageToString(
-                value, float_format='.17g'),
+            text_format.MessageToString(value, float_format='.17g'),
             test_example_pb2.PackedTestValue()).SerializeToString()
         for value in case.values
     ]
diff --git a/tensorflow/contrib/proto/python/kernel_tests/encode_proto_op_test_base.py b/tensorflow/contrib/proto/python/kernel_tests/encode_proto_op_test_base.py
index 01b3ccc7fd..5ec681ff55 100644
--- a/tensorflow/contrib/proto/python/kernel_tests/encode_proto_op_test_base.py
+++ b/tensorflow/contrib/proto/python/kernel_tests/encode_proto_op_test_base.py
@@ -15,9 +15,6 @@
 # =============================================================================
 """Table-driven test for encode_proto op.
 
-This test is run once with each of the *.TestCase.pbtxt files
-in the test directory.
-
 It tests that encode_proto is a lossless inverse of decode_proto
 (for the specified fields).
 """
@@ -145,7 +142,8 @@ class EncodeProtoOpTestBase(test_base.ProtoOpTestBase, parameterized.TestCase):
         # loss of packing in the encoding).
         self.assertEqual(in_buf, out_buf)
 
-  @parameterized.named_parameters(*test_base.ProtoOpTestBase.named_parameters())
+  @parameterized.named_parameters(
+      *test_base.ProtoOpTestBase.named_parameters(extension=False))
   def testRoundtrip(self, case):
     in_bufs = [value.SerializeToString() for value in case.values]
 
@@ -154,7 +152,8 @@ class EncodeProtoOpTestBase(test_base.ProtoOpTestBase, parameterized.TestCase):
     return self._testRoundtrip(
         in_bufs, 'tensorflow.contrib.proto.TestValue', case.fields)
 
-  @parameterized.named_parameters(*test_base.ProtoOpTestBase.named_parameters())
+  @parameterized.named_parameters(
+      *test_base.ProtoOpTestBase.named_parameters(extension=False))
   def testRoundtripPacked(self, case):
     # Now try with the packed serialization.
     # We test the packed representations by loading the same test cases using
diff --git a/tensorflow/contrib/proto/python/kernel_tests/proto_op_test_base.py b/tensorflow/contrib/proto/python/kernel_tests/proto_op_test_base.py
index 2950c7dfdc..1a636486a1 100644
--- a/tensorflow/contrib/proto/python/kernel_tests/proto_op_test_base.py
+++ b/tensorflow/contrib/proto/python/kernel_tests/proto_op_test_base.py
@@ -38,17 +38,18 @@ class ProtoOpTestBase(test.TestCase):
       ct.cdll.LoadLibrary(lib)
 
   @staticmethod
-  def named_parameters():
-    return (
-        ("defaults", ProtoOpTestBase.defaults_test_case()),
-        ("minmax", ProtoOpTestBase.minmax_test_case()),
-        ("nested", ProtoOpTestBase.nested_test_case()),
-        ("optional", ProtoOpTestBase.optional_test_case()),
-        ("promote", ProtoOpTestBase.promote_test_case()),
-        ("ragged", ProtoOpTestBase.ragged_test_case()),
-        ("shaped_batch", ProtoOpTestBase.shaped_batch_test_case()),
-        ("simple", ProtoOpTestBase.simple_test_case()),
-    )
+  def named_parameters(extension=True):
+    parameters = [("defaults", ProtoOpTestBase.defaults_test_case()),
+                  ("minmax", ProtoOpTestBase.minmax_test_case()),
+                  ("nested", ProtoOpTestBase.nested_test_case()),
+                  ("optional", ProtoOpTestBase.optional_test_case()),
+                  ("promote", ProtoOpTestBase.promote_test_case()),
+                  ("ragged", ProtoOpTestBase.ragged_test_case()),
+                  ("shaped_batch", ProtoOpTestBase.shaped_batch_test_case()),
+                  ("simple", ProtoOpTestBase.simple_test_case())]
+    if extension:
+      parameters.append(("extension", ProtoOpTestBase.extension_test_case()))
+    return parameters
 
   @staticmethod
   def defaults_test_case():
@@ -399,6 +400,21 @@ class ProtoOpTestBase(test.TestCase):
     field.value.bool_value.append(True)
     return test_case
 
+  @staticmethod
+  def extension_test_case():
+    test_case = test_example_pb2.TestCase()
+    value = test_case.values.add()
+    message_value = value.Extensions[test_example_pb2.ext_value].add()
+    message_value.double_value = 23.5
+    test_case.shapes.append(1)
+    test_case.sizes.append(1)
+    field = test_case.fields.add()
+    field.name = test_example_pb2.ext_value.full_name
+    field.dtype = types_pb2.DT_STRING
+    message_value = field.value.Extensions[test_example_pb2.ext_value].add()
+    message_value.double_value = 23.5
+    return test_case
+
   @staticmethod
   def simple_test_case():
     test_case = test_example_pb2.TestCase()
diff --git a/tensorflow/contrib/proto/python/kernel_tests/test_example.proto b/tensorflow/contrib/proto/python/kernel_tests/test_example.proto
index 674d881220..b1ce66de4f 100644
--- a/tensorflow/contrib/proto/python/kernel_tests/test_example.proto
+++ b/tensorflow/contrib/proto/python/kernel_tests/test_example.proto
@@ -61,6 +61,8 @@ message TestValue {
   optional sfixed64 sfixed64_value_with_default = 32 [default = 11];
   optional sint32 sint32_value_with_default = 33 [default = 12];
   optional sint64 sint64_value_with_default = 34 [default = 13];
+
+  extensions 100 to 199;
 }
 
 // A PackedTestValue looks exactly the same as a TestValue in the text format,
@@ -68,7 +70,7 @@ message TestValue {
 // by loading the same test cases using this definition instead of TestValue.
 //
 // NOTE: This definition must be kept in sync with TestValue in every way except
-// the packed=true declaration.
+// the packed=true declaration and the lack of extensions.
 message PackedTestValue {
   repeated double double_value = 1 [packed = true];
   repeated float float_value = 2 [packed = true];
@@ -132,6 +134,10 @@ message ExtraFields {
   optional bool bool_value = 1777;
 }
 
+extend TestValue {
+  repeated PrimitiveValue ext_value = 100;
+}
+
 // The messages below are for yet-to-be created tests.
 
 message EnumValue {
diff --git a/tensorflow/core/api_def/base_api/api_def_DecodeProtoV2.pbtxt b/tensorflow/core/api_def/base_api/api_def_DecodeProtoV2.pbtxt
index c8152f53c4..22c3524360 100644
--- a/tensorflow/core/api_def/base_api/api_def_DecodeProtoV2.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_DecodeProtoV2.pbtxt
@@ -31,7 +31,8 @@ END
   attr {
     name: "field_names"
     description: <<END
-List of strings containing proto field names.
+List of strings containing proto field names. An extension field can be decoded
+by using its full name, e.g. EXT_PACKAGE.EXT_FIELD_NAME.
 END
   }
   attr {
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index ffb093b2ef..b6a421b795 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -7064,6 +7064,7 @@ tf_kernel_library(
         "//tensorflow/core/util/proto:descriptors",
         "//tensorflow/core/util/proto:proto_utils",
         "//third_party/eigen3",
+        "@com_google_absl//absl/container:flat_hash_map",
     ],
 )
 
diff --git a/tensorflow/core/kernels/decode_proto_op.cc b/tensorflow/core/kernels/decode_proto_op.cc
index b54e1ea8ac..06dc766794 100644
--- a/tensorflow/core/kernels/decode_proto_op.cc
+++ b/tensorflow/core/kernels/decode_proto_op.cc
@@ -31,6 +31,7 @@ limitations under the License.
 #include <string>
 #include <vector>
 
+#include "absl/container/flat_hash_map.h"
 #include "third_party/eigen3/Eigen/Core"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor_types.h"
@@ -625,8 +626,37 @@ class DecodeProtoOp : public OpKernel {
     // Gather the field descriptors and check that requested output types match.
     int field_index = 0;
     std::vector<const FieldDescriptor*> field_descs;
+    std::vector<const FieldDescriptor*> exts;
+    absl::flat_hash_map<string, const FieldDescriptor*> ext_name_to_field;
+    std::vector<const FieldDescriptor*>::iterator ext_it = exts.begin();
     for (const string& name : field_names) {
       auto fd = message_desc->FindFieldByName(name);
+      if (fd == nullptr) {
+        // If field can't be found in original message, try to find a matching
+        // extension (by its full_name). First check a hashmap for a matching
+        // extension, and if not found, then iterate through available
+        // extensions to find a match (updating the hashmap while iterating.)
+        auto lookup_result = ext_name_to_field.find(name);
+        if (lookup_result != ext_name_to_field.end()) {
+          fd = lookup_result->second;
+        } else {
+          if (ext_it == exts.begin()) {
+            desc_pool->FindAllExtensions(message_desc, &exts);
+            ext_it = exts.begin();
+          }
+          while (ext_it != exts.end()) {
+            auto ext_name = (*ext_it)->full_name();
+            auto ext_field = *ext_it;
+            ++ext_it;
+
+            ext_name_to_field.insert({ext_name, ext_field});
+            if (ext_name == name) {
+              fd = ext_field;
+              break;
+            }
+          }
+        }
+      }
       OP_REQUIRES(context, fd != nullptr,
                   errors::InvalidArgument("Unknown field: ", name,
                                           " in message type ", message_type));
-- 
GitLab


From d602a8f8284924a0a3538e0f4dd4986388ae92c5 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Wed, 30 Jan 2019 17:51:28 -0800
Subject: [PATCH 1641/2345] Fix GrpcServerOptions not correctly passed to
 GrpcServer::Init in a few places.

PiperOrigin-RevId: 231703961
---
 tensorflow/contrib/mpi/mpi_server_lib.cc                   | 5 ++++-
 tensorflow/contrib/verbs/verbs_server_lib.cc               | 5 ++++-
 tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc | 2 +-
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/mpi/mpi_server_lib.cc b/tensorflow/contrib/mpi/mpi_server_lib.cc
index a31fa9ce0b..e44e10af08 100644
--- a/tensorflow/contrib/mpi/mpi_server_lib.cc
+++ b/tensorflow/contrib/mpi/mpi_server_lib.cc
@@ -54,7 +54,10 @@ MPIServer::~MPIServer() {
 
 Status MPIServer::Init(ServiceInitFunction service_func,
                        RendezvousMgrCreationFunction rendezvous_mgr_func) {
-  Status s = GrpcServer::Init(service_func, rendezvous_mgr_func);
+  GrpcServerOptions opts;
+  opts.service_func = service_func;
+  opts.rendezvous_mgr_func = rendezvous_mgr_func;
+  Status s = GrpcServer::Init(opts);
   return s;
 }
 
diff --git a/tensorflow/contrib/verbs/verbs_server_lib.cc b/tensorflow/contrib/verbs/verbs_server_lib.cc
index 19ef109f67..d07fd5ae6e 100644
--- a/tensorflow/contrib/verbs/verbs_server_lib.cc
+++ b/tensorflow/contrib/verbs/verbs_server_lib.cc
@@ -81,7 +81,10 @@ Status VerbsServer::ChannelCacheFactory(const ServerDef& server_def,
 Status VerbsServer::Init(ServiceInitFunction service_func,
                          RendezvousMgrCreationFunction rendezvous_mgr_func) {
   std::call_once(reg_mem_visitors_call, []() { RdmaMgr::RegMemVisitors(); });
-  Status s = GrpcServer::Init(service_func, rendezvous_mgr_func);
+  GrpcServerOptions opts;
+  opts.service_func = service_func;
+  opts.rendezvous_mgr_func = rendezvous_mgr_func;
+  Status s = GrpcServer::Init(opts);
   {
     mutex_lock l(mu_);
     CHECK_EQ(verbs_state_, DISCONNECTED);
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc b/tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc
index 1405c760d5..f087a39f01 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc
@@ -431,7 +431,7 @@ Status GrpcServer::Create(const ServerDef& server_def, Env* env,
   ServiceInitFunction service_func = nullptr;
   GrpcServerOptions options;
   options.rendezvous_mgr_func = NewRpcRendezvousMgr;
-  Status s = ret->Init();
+  Status s = ret->Init(options);
   if (!s.ok()) {
     LOG(ERROR) << s;
     return s;
-- 
GitLab


From 397fdb37bc923f1f56d0224617cfd1b1dfd76d99 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 30 Jan 2019 17:59:29 -0800
Subject: [PATCH 1642/2345] Add a wrapper to load .so for
 tpu_ordinal_selector_op so that it works in open source.

PiperOrigin-RevId: 231704869
---
 tensorflow/contrib/tpu/BUILD                  |  8 ++--
 .../tpu/python/ops/tpu_ordinal_selector_op.py | 38 +++++++++++++++++++
 .../contrib/tpu/python/tpu/tpu_estimator.py   |  4 +-
 3 files changed, 44 insertions(+), 6 deletions(-)
 create mode 100644 tensorflow/contrib/tpu/python/ops/tpu_ordinal_selector_op.py

diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index c1a36fecc2..59119633aa 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -162,14 +162,14 @@ tf_gen_op_wrapper_py(
 )
 
 tf_custom_op_library(
-    name = "python/ops/_tpu_ordinal_selector.so",
+    name = "python/ops/_tpu_ordinal_selector_op.so",
     srcs = ["ops/tpu_ordinal_selector_op.cc"],
 )
 
 tf_custom_op_py_library(
     name = "tpu_ordinal_selector_py",
-    srcs = ["ops/gen_tpu_ordinal_selector_op.py"],
-    dso = [":python/ops/_tpu_ordinal_selector.so"],
+    srcs = ["python/ops/tpu_ordinal_selector_op.py"],
+    dso = [":python/ops/_tpu_ordinal_selector_op.so"],
     kernels = [
         ":tpu_ordinal_selector_op_op_lib",
     ],
@@ -221,7 +221,7 @@ py_library(
 
 tf_custom_op_py_library(
     name = "tpu_py",
-    srcs = glob(["python/ops/*.py"]),
+    srcs = ["python/ops/tpu_ops.py"],
     dso = [":python/ops/_tpu_ops.so"],
     kernels = [
         ":all_ops",
diff --git a/tensorflow/contrib/tpu/python/ops/tpu_ordinal_selector_op.py b/tensorflow/contrib/tpu/python/ops/tpu_ordinal_selector_op.py
new file mode 100644
index 0000000000..5ca38cd1ba
--- /dev/null
+++ b/tensorflow/contrib/tpu/python/ops/tpu_ordinal_selector_op.py
@@ -0,0 +1,38 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =============================================================================
+
+"""Operations to select TPU core to run."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import platform
+
+if platform.system() != "Windows":
+  # pylint: disable=wildcard-import,unused-import,g-import-not-at-top
+  from tensorflow.contrib.tpu.ops.gen_tpu_ordinal_selector_op import *
+
+  from tensorflow.contrib.util import loader
+  from tensorflow.python.platform import resource_loader
+  # pylint: enable=wildcard-import,unused-import,g-import-not-at-top
+
+  _tpu_ordinal_selector_op = loader.load_op_library(
+      resource_loader.get_path_to_datafile("_tpu_ordinal_selector_op.so"))
+
+else:
+  # We have already built the appropriate libraries into the binary via CMake
+  # if we have built contrib, so we don't need this
+  pass
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 10f3451619..1d06009b65 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -31,9 +31,9 @@ import six
 from six.moves import queue as Queue  # pylint: disable=redefined-builtin
 from six.moves import xrange  # pylint: disable=redefined-builtin
 
-from tensorflow.contrib.tpu.ops import gen_tpu_ordinal_selector_op
 from tensorflow.contrib.tpu.proto import compilation_result_pb2 as tpu_compilation_result
 from tensorflow.contrib.tpu.python.ops import tpu_ops
+from tensorflow.contrib.tpu.python.ops import tpu_ordinal_selector_op
 from tensorflow.contrib.tpu.python.tpu import _tpu_estimator_embedding
 from tensorflow.contrib.tpu.python.tpu import error_handling
 from tensorflow.contrib.tpu.python.tpu import functional as tpu_functional
@@ -1370,7 +1370,7 @@ def call_computation(computation,
 
     return tpu_functional.TPUPartitionedCall(
         args=tpu_subgraph.captured_inputs,
-        device_ordinal=gen_tpu_ordinal_selector_op.tpu_ordinal_selector(),
+        device_ordinal=tpu_ordinal_selector_op.tpu_ordinal_selector(),
         Tout=[o.type for o in tpu_subgraph.definition.signature.output_arg],
         f=tpu_subgraph)
   else:
-- 
GitLab


From 4a70bff9a8029495359f279aa66534658e5f122e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 30 Jan 2019 18:00:52 -0800
Subject: [PATCH 1643/2345] open-source traceme_recorder.

PiperOrigin-RevId: 231705045
---
 tensorflow/core/profiler/internal/BUILD       |  29 ++
 .../profiler/internal/traceme_recorder.cc     | 248 ++++++++++++++++++
 .../core/profiler/internal/traceme_recorder.h |  95 +++++++
 .../internal/traceme_recorder_test.cc         | 212 +++++++++++++++
 4 files changed, 584 insertions(+)
 create mode 100644 tensorflow/core/profiler/internal/traceme_recorder.cc
 create mode 100644 tensorflow/core/profiler/internal/traceme_recorder.h
 create mode 100644 tensorflow/core/profiler/internal/traceme_recorder_test.cc

diff --git a/tensorflow/core/profiler/internal/BUILD b/tensorflow/core/profiler/internal/BUILD
index 8dcfde9a2a..b373342085 100644
--- a/tensorflow/core/profiler/internal/BUILD
+++ b/tensorflow/core/profiler/internal/BUILD
@@ -6,6 +6,8 @@ licenses(["notice"])  # Apache 2.0
 
 load("//tensorflow:tensorflow.bzl", "tf_cc_test")
 load("//tensorflow:tensorflow.bzl", "if_not_windows")
+load("//tensorflow:tensorflow.bzl", "tf_cuda_library")
+load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_test")
 
 cc_library(
     name = "tfprof_stats",
@@ -365,3 +367,30 @@ cc_library(
         "//tensorflow/core:regexp_internal",
     ],
 )
+
+tf_cuda_library(
+    name = "traceme_recorder",
+    srcs = ["traceme_recorder.cc"],
+    hdrs = ["traceme_recorder.h"],
+    visibility = [
+        "//perftools/accelerators/xprof/xprofilez:__pkg__",  # alias xprof::TraceMeRecorder
+    ],
+    deps = [
+        "//tensorflow/core:lib",
+        "//tensorflow/stream_executor/lib",
+        "@com_google_absl//absl/base:core_headers",
+        "@com_google_absl//absl/container:flat_hash_map",
+    ],
+)
+
+tf_cuda_cc_test(
+    name = "traceme_recorder_test",
+    srcs = ["traceme_recorder_test.cc"],
+    deps = [
+        ":traceme_recorder",
+        "//tensorflow/core:lib",
+        "@com_google_absl//absl/synchronization",
+        "@com_google_absl//absl/time",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
diff --git a/tensorflow/core/profiler/internal/traceme_recorder.cc b/tensorflow/core/profiler/internal/traceme_recorder.cc
new file mode 100644
index 0000000000..0369e0b96d
--- /dev/null
+++ b/tensorflow/core/profiler/internal/traceme_recorder.cc
@@ -0,0 +1,248 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/profiler/internal/traceme_recorder.h"
+
+// To avoid unneccesary synchronization between threads, each thread has a
+// ThreadLocalRecorder that independently records its events.
+//
+// Events are stored in an EventQueue implemented as a linked-list of blocks,
+// with start and end pointers:
+//  [ events........ | next-]--> [ events......... | next ]
+//  ^start_block  ^start         ^end_block  ^end
+//
+// Record() writes at end, and then advances it, allocating a block if needed.
+// Clear() takes ownership of events in the range [start, end).
+// The end pointer is atomic so these can be concurrent.
+//
+// If a thread dies, the ThreadLocalRecorder's destructor hands its data off to
+// the orphaned_events list.
+
+#include <string>
+#include "absl/container/flat_hash_map.h"
+#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/stream_executor/lib/initialize.h"
+
+namespace tensorflow {
+namespace profiler {
+
+// Default value for g_trace_level when tracing is disabled
+constexpr static int kTracingDisabled = -1;
+
+namespace internal {
+std::atomic<int> g_trace_level = ATOMIC_VAR_INIT(kTracingDisabled);
+}  // namespace internal
+
+namespace {
+
+class ThreadLocalRecorder;
+
+struct Data {
+  // Lock for only rare events - start/stop, thread death.
+  mutex global_lock;
+  // Map of the static container instances (thread_local storage) for each
+  // thread, that store the trace events.
+  absl::flat_hash_map<uint64, ThreadLocalRecorder*> threads
+      GUARDED_BY(global_lock);
+  // Events traced from threads that died during tracing.
+  TraceMeRecorder::Events orphaned_events GUARDED_BY(global_lock);
+}* g_data = nullptr;
+
+// A single-producer single-consumer queue of Events.
+// Only the owner thread can write events, writing is lock-free.
+// Consume is also lock-free in this class.
+//
+// Internally, we have a linked list of blocks containing numbered slots.
+// start is the first occupied slot, end is the first unoccupied slot.
+class EventQueue {
+ public:
+  EventQueue()
+      : start_block_(new Block{0, nullptr}), end_block_(start_block_) {}
+
+  // REQUIRES: Consume() was called since the last Push().
+  // Memory should be deallocated and trace events destroyed on destruction.
+  // This doesn't require global lock as this discards all the stored trace
+  // events and we assume of destruction of this class only after the last
+  // Push() has been called.
+  ~EventQueue() {
+    DCHECK_EQ(start_, end_.load()) << "EventQueue destroyed without Consume()";
+    delete end_block_;
+  }
+
+  // Add a new event to the back of the queue. Fast and lock-free.
+  void Push(TraceMeRecorder::Event&& event) {
+    uint64 end = end_.load(std::memory_order_relaxed);
+    new (&end_block_->events[end++ - end_block_->start].event)
+        TraceMeRecorder::Event(std::move(event));
+    if (ABSL_PREDICT_FALSE(end - end_block_->start == Block::kLength)) {
+      auto* new_block = new Block{end, nullptr};
+      end_block_->next = new_block;
+      end_block_ = new_block;
+    }
+    end_.store(end, std::memory_order_release);  // Write index after contents.
+  }
+
+  // Retrieve and remove all events in the queue.
+  std::vector<TraceMeRecorder::Event> Consume() {
+    // Read index before contents.
+    uint64 end = end_.load(std::memory_order_acquire);
+    std::vector<TraceMeRecorder::Event> result;
+    result.reserve(end - start_);
+    while (start_ != end) {
+      Shift(&result);
+    }
+    return result;
+  }
+
+ private:
+  // Shift one event off the front of the queue into *out.
+  void Shift(std::vector<TraceMeRecorder::Event>* out) {
+    // Move the next event into the output.
+    auto& event = start_block_->events[start_++ - start_block_->start].event;
+    out->push_back(std::move(event));
+    event.~Event();  // Events must be individually destroyed.
+    // If we reach the end of a block, we own it and should delete it.
+    // The next block is present: end always points to something.
+    if (start_ - start_block_->start == Block::kLength) {
+      auto* next_block = start_block_->next;
+      delete start_block_;
+      start_block_ = next_block;
+    }
+  }
+
+  // The number of slots in a block. Chosen so that the block fits in 64k.
+  struct Block {
+    static constexpr size_t kLength =
+        ((1 << 16) - (sizeof(uint64) + sizeof(std::atomic<Block*>))) /
+        sizeof(TraceMeRecorder::Event);
+
+    const uint64 start;  // The number of the first slot.
+    Block* next;
+    // Defer construction of Event until the data is available.
+    // Must also destroy manually, as the block may not fill entirely.
+    union MaybeEvent {
+      MaybeEvent() {}
+      ~MaybeEvent() {}
+      TraceMeRecorder::Event event;
+    } events[kLength];
+  };
+
+  // Head of list for reading. Only accessed by consumer thread.
+  Block* start_block_;
+  uint64 start_ = 0;
+  // Tail of list for writing. Accessed by producer thread.
+  Block* end_block_;
+  std::atomic<uint64> end_ = {0};  // Atomic: also read by consumer thread.
+};
+
+class ThreadLocalRecorder {
+ public:
+  // The recorder is created the first time Record() is called on a thread.
+  ThreadLocalRecorder() {
+    auto* env = Env::Default();
+    info_.tid = env->GetCurrentThreadId();
+    env->GetCurrentThreadName(&info_.name);
+    mutex_lock lock(g_data->global_lock);
+    g_data->threads.emplace(info_.tid, this);
+  }
+
+  // The destructor is called when the thread shuts down early.
+  // We unregister this thread, and move its events to orphaned_events.
+  ~ThreadLocalRecorder() {
+    mutex_lock lock(g_data->global_lock);
+    g_data->threads.erase(info_.tid);
+    g_data->orphaned_events.push_back(Clear());
+  }
+
+  // This is the performance-critical part!
+  void Record(TraceMeRecorder::Event&& event) { queue_.Push(std::move(event)); }
+
+  TraceMeRecorder::ThreadEvents Clear()
+      EXCLUSIVE_LOCKS_REQUIRED(g_data->global_lock) {
+    return {info_, queue_.Consume()};
+  }
+
+ private:
+  TraceMeRecorder::ThreadInfo info_;
+  EventQueue queue_;
+};
+
+// Gather events from all active threads, and clear their buffers. The global
+// lock is held, so no threads can be added/removed for the duration while we
+// consume the collected trace entries. This will block any new thread and also
+// the starting and stopping of TraceMeRecorder, hence, this is performance
+// critical and should be kept fast.
+TraceMeRecorder::Events Clear() EXCLUSIVE_LOCKS_REQUIRED(g_data->global_lock) {
+  TraceMeRecorder::Events result;
+  std::swap(g_data->orphaned_events, result);
+  for (const auto& entry : g_data->threads) {
+    auto* recorder = entry.second;
+    result.push_back(recorder->Clear());
+  }
+  return result;
+}
+
+}  // namespace
+
+bool TraceMeRecorder::Start(int level) {
+  level = std::max(0, level);
+  mutex_lock lock(g_data->global_lock);
+  int expected = kTracingDisabled;
+  if (!internal::g_trace_level.compare_exchange_strong(
+          expected, level, std::memory_order_acq_rel)) {
+    return false;
+  }
+  // We may have old events in buffers because Record() raced with Stop().
+  Clear();
+  return true;
+}
+
+
+void TraceMeRecorder::Record(Event event) {
+  static thread_local ThreadLocalRecorder thread_local_recorder;
+  thread_local_recorder.Record(std::move(event));
+}
+
+// Only one thread is expected to call Stop() as first instance of XprofSession
+// prevents another XprofSession from doing any profiling.
+TraceMeRecorder::Events TraceMeRecorder::Stop() {
+  mutex_lock lock(g_data->global_lock);
+  if (internal::g_trace_level.exchange(
+          kTracingDisabled, std::memory_order_acq_rel) == kTracingDisabled) {
+    return {};
+  }
+  return Clear();
+}
+
+TraceMeRecorder::Events TraceMeRecorder::Collect() {
+  mutex_lock lock(g_data->global_lock);
+  if (internal::g_trace_level.load(std::memory_order_acquire) ==
+      kTracingDisabled) {
+    return {};
+  }
+  return Clear();
+}
+
+}  // namespace profiler
+}  // namespace tensorflow
+
+REGISTER_MODULE_INITIALIZER(traceme_recorder, {
+  tensorflow::profiler::g_data = new tensorflow::profiler::Data();
+
+  // Workaround for b/35097229, the first block-scoped thread_local can
+  // trigger false positives in the heap checker. Currently triggered by
+  // //perftools/accelerators/xprof/xprofilez/integration_tests:xla_hlo_trace_test
+  static thread_local tensorflow::string fix_deadlock ABSL_ATTRIBUTE_UNUSED;
+});
diff --git a/tensorflow/core/profiler/internal/traceme_recorder.h b/tensorflow/core/profiler/internal/traceme_recorder.h
new file mode 100644
index 0000000000..1e66b1e5bb
--- /dev/null
+++ b/tensorflow/core/profiler/internal/traceme_recorder.h
@@ -0,0 +1,95 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_PROFILER_INTERNAL_TRACEME_RECORDER_H_
+#define TENSORFLOW_CORE_PROFILER_INTERNAL_TRACEME_RECORDER_H_
+
+#include <atomic>
+#include <vector>
+#include "absl/base/optimization.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace tensorflow {
+namespace profiler {
+
+namespace internal {
+extern std::atomic<int> g_trace_level;
+}  // namespace internal
+
+// TraceMeRecorder is a singleton repository of TraceMe events.
+// It can be safely and cheaply appended to by multiple threads.
+//
+// Start() and Stop() must be called in pairs, Stop() returns the events added
+// since the previous Start().
+//
+// This is the backend for TraceMe instrumentation.
+// The profiler starts the recorder, the TraceMe constructor records begin
+// events, and the destructor records end events.
+// The profiler then stops the recorder and finds start/end pairs. (Unpaired
+// start/end events are discarded at that point).
+class TraceMeRecorder {
+ public:
+  // An Event is either the start of a TraceMe, the end of a TraceMe, or both.
+  // Times are in ns since the Unix epoch.
+  struct Event {
+    uint64 activity_id;
+    string name;
+    uint64 start_time;  // 0 = missing
+    uint64 end_time;    // 0 = missing
+  };
+  struct ThreadInfo {
+    int64 tid;
+    string name;
+  };
+  struct ThreadEvents {
+    const ThreadInfo thread;
+    std::vector<Event> events;
+  };
+  using Events = std::vector<ThreadEvents>;
+
+  // Starts recording of TraceMe().
+  // Only traces <= level will be recorded.
+  // Level must be >= 0.
+  // If level is 0, no traces will be recorded.
+  static bool Start(int level);
+
+  // Stops recording and returns events recorded since Start().
+  static Events Stop();
+
+  // Returns events recorded till now without stopping the recording. Empty
+  // container is returned if the recorder was already stopped.
+  static Events Collect();
+
+  // Returns whether we're currently recording. Racy, but cheap!
+  static inline bool Active(int level = 1) {
+    return ABSL_PREDICT_FALSE(
+        internal::g_trace_level.load(std::memory_order_acquire) >= level);
+  }
+
+  static void Record(Event);
+
+ private:
+  // No copy and assignment
+  TraceMeRecorder(const TraceMeRecorder&) = delete;
+  TraceMeRecorder& operator=(const TraceMeRecorder&) = delete;
+
+  // Implementation of g_trace_level must be lock-free for faster execution
+  // of the TraceMe() public API. This can be commented (if compilation is
+  // failing) but execution might be slow (even when host tracing is disabled).
+  static_assert(ATOMIC_INT_LOCK_FREE == 2, "Assumed atomic<int> was lock free");
+};
+
+}  // namespace profiler
+}  // namespace tensorflow
+#endif  // TENSORFLOW_CORE_PROFILER_INTERNAL_TRACEME_RECORDER_H_
diff --git a/tensorflow/core/profiler/internal/traceme_recorder_test.cc b/tensorflow/core/profiler/internal/traceme_recorder_test.cc
new file mode 100644
index 0000000000..bea3f7c9bd
--- /dev/null
+++ b/tensorflow/core/profiler/internal/traceme_recorder_test.cc
@@ -0,0 +1,212 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/profiler/internal/traceme_recorder.h"
+
+#include <atomic>
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+#include "absl/synchronization/notification.h"
+#include "absl/time/clock.h"
+#include "absl/time/time.h"
+#include "tensorflow/core/lib/core/threadpool.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace tensorflow {
+namespace profiler {
+namespace {
+
+MATCHER_P(Named, name, "") { return arg.name == name; }
+
+constexpr static uint64 kNanosInSec = 1000000000;
+
+TEST(RecorderTest, SingleThreaded) {
+  uint64 start_time = static_cast<uint64>(absl::GetCurrentTimeNanos());
+  uint64 end_time = start_time + kNanosInSec;
+
+  TraceMeRecorder::Record({1, "before", start_time, end_time});
+  TraceMeRecorder::Start(/*level=*/1);
+  TraceMeRecorder::Record({2, "during1", start_time, end_time});
+  TraceMeRecorder::Record({3, "during2", start_time, end_time});
+  auto results = TraceMeRecorder::Stop();
+  TraceMeRecorder::Record({4, "after", start_time, end_time});
+
+  ASSERT_EQ(results.size(), 1);
+  EXPECT_THAT(results[0].events,
+              ::testing::ElementsAre(Named("during1"), Named("during2")));
+}
+
+TEST(RecorderTest, CollectionBeforeStop) {
+  uint64 start_time = static_cast<uint64>(absl::GetCurrentTimeNanos());
+  uint64 end_time = start_time + kNanosInSec;
+
+  TraceMeRecorder::Record({1, "ignored", start_time, end_time});
+  TraceMeRecorder::Start(/*level=*/1);
+  TraceMeRecorder::Record({2, "during1", start_time, end_time});
+  TraceMeRecorder::Record({3, "during2", start_time, end_time});
+  auto collected_results = TraceMeRecorder::Collect();
+  TraceMeRecorder::Record({4, "after_collect", start_time, end_time});
+  auto stopped_results = TraceMeRecorder::Stop();
+  TraceMeRecorder::Record({5, "after_stop", start_time, end_time});
+  auto results_after_stop = TraceMeRecorder::Collect();
+
+  ASSERT_EQ(collected_results.size(), 1);
+  EXPECT_THAT(collected_results[0].events,
+              ::testing::ElementsAre(Named("during1"), Named("during2")));
+
+  ASSERT_EQ(stopped_results.size(), 1);
+  EXPECT_THAT(stopped_results[0].events,
+              ::testing::ElementsAre(Named("after_collect")));
+
+  ASSERT_EQ(results_after_stop.size(), 0);
+}
+
+void SpinNanos(int nanos) {
+  uint64 deadline = absl::GetCurrentTimeNanos() + nanos;
+  while (absl::GetCurrentTimeNanos() < deadline) {
+  }
+}
+
+// Checks the functional behavior of the recorder, when used from several
+// unsynchronized threads.
+//
+// Each thread records a stream of events.
+//   Thread 0: activity=0, activity=1, activity=2, ...
+//   Thread 1: activity=0, activity=1, activity=2, ...
+//   ...
+//
+// We turn the recorder on and off repeatedly in sessions, expecting to see:
+//   - data from every thread (eventually - maybe not every session)
+//   - unbroken sessions: a consecutive sequence of IDs from each thread
+//   - gaps between sessions: a thread's IDs should be non-consecutive overall
+TEST(RecorderTest, Multithreaded) {
+  constexpr static int kNumThreads = 4;
+
+  // Start several threads writing events.
+  absl::Notification start;
+  absl::Notification stop;
+  thread::ThreadPool pool(Env::Default(), "testpool", kNumThreads);
+  std::atomic<int> thread_count = {0};
+  for (int i = 0; i < kNumThreads; i++) {
+    pool.Schedule([&start, &stop, &thread_count, i] {
+      uint64 j = 0;
+      bool was_active = false;
+      auto record_event = [&j, i]() {
+        uint64 start_time = static_cast<uint64>(absl::GetCurrentTimeNanos());
+        uint64 end_time = start_time + kNanosInSec;
+        TraceMeRecorder::Record({/*activity_id=*/j++,
+                                 /*name=*/strings::StrCat(i), start_time,
+                                 end_time});
+      };
+      thread_count.fetch_add(1, std::memory_order_relaxed);
+      start.WaitForNotification();
+      while (!stop.HasBeenNotified()) {
+        // Mimicking production usage, we guard with a racy check.
+        // In principle this isn't needed, but a feedback loop can form:
+        // 1) many events accumulate while the recorder is off
+        // 2) clearing/analyzing these events is slow
+        // 3) while clearing, more events are accumulating, causing 1
+        if (TraceMeRecorder::Active()) {
+          record_event();
+          was_active = true;
+        }
+        // Record some events after the recorder is no longer active to simulate
+        // point 1 and 3.
+        if (was_active && !TraceMeRecorder::Active()) {
+          record_event();
+          record_event();
+          was_active = false;
+        }
+        // This snowballs into OOM in some configurations, causing flakiness.
+        // Keep this big enough to prevent OOM and small enough such that
+        // each thread records at least one event.
+        SpinNanos(10);
+      }
+    });
+  }
+
+  // For each thread, keep track of which events we've seen.
+  struct {
+    bool split_session = false;
+    bool overlapping_sessions = false;
+    std::set<uint64> events;
+  } thread_state[kNumThreads];
+  // We expect each thread to eventually have multiple events, not all in a
+  // contiguous range.
+  auto done = [&thread_state] {
+    for (const auto& t : thread_state) {
+      if (t.events.size() < 2) return false;
+    }
+    return true;
+  };
+
+  // Wait while all the threads are spun up.
+  while (thread_count.load(std::memory_order_relaxed) < kNumThreads) {
+    LOG(INFO) << "Waiting for all threads to spin up...";
+    absl::SleepFor(absl::Milliseconds(1));
+  }
+
+  // We will probably be done after two iterations (with each thread getting
+  // some events each iteration). No guarantees as all the threads might not get
+  // scheduled in a session, so try for a while.
+  start.Notify();
+  constexpr static int kMaxIters = 100;
+  for (int iters = 0; iters < kMaxIters && !done(); ++iters) {
+    LOG(INFO) << "Looping until convergence, iteration: " << iters;
+    TraceMeRecorder::Start(/*level=*/1);
+    absl::SleepFor(absl::Milliseconds(100));
+    auto results = TraceMeRecorder::Stop();
+    for (const auto& thread : results) {
+      if (thread.events.empty()) continue;
+      std::istringstream ss(thread.events.front().name);
+      int thread_index = 0;
+      ss >> thread_index;
+      auto& state = thread_state[thread_index];
+
+      std::set<uint64> session_events;
+      uint64 current = 0;
+      for (const auto& event : thread.events) {
+        session_events.emplace(event.activity_id);
+        // Session events should be contiguous.
+        if (current != 0 && event.activity_id != current + 1) {
+          state.split_session = true;
+        }
+        current = event.activity_id;
+      }
+
+      for (const auto& event : session_events) {
+        auto result = state.events.emplace(event);
+        if (!result.second) {
+          // Session events should not overlap with those from previous
+          // sessions.
+          state.overlapping_sessions = true;
+        }
+      }
+    }
+    absl::SleepFor(absl::Milliseconds(1));
+  }
+  stop.Notify();
+
+  for (const auto& thread : thread_state) {
+    EXPECT_FALSE(thread.split_session)
+        << "Expected contiguous events in a session";
+    EXPECT_FALSE(thread.overlapping_sessions) << "Expected disjoint sessions";
+    EXPECT_GT(thread.events.size(), 1)
+        << "Expected gaps in thread events between sessions";
+  }
+}
+
+}  // namespace
+}  // namespace profiler
+}  // namespace tensorflow
-- 
GitLab


From 7d8bfd88cddd536cd35e0ccb3082fb9f4bd91b28 Mon Sep 17 00:00:00 2001
From: Smit Hinsu <hinsu@google.com>
Date: Wed, 30 Jan 2019 18:15:12 -0800
Subject: [PATCH 1644/2345] Set kIntsonDeviceAttr for native function in
 TRTEngineOp

This is required to make sure that all TRTEngineOp outputs are always on a GPU
device as expected. Otherwise, some of the tensors of type DT_INT32 would be on
host if the op generating the tensor has host memory tag set.

PiperOrigin-RevId: 231707027
---
 tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc
index 8b902dd90e..31d0b2dfa4 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc
@@ -849,6 +849,12 @@ tensorflow::Status RegisterSegmentFunctionToFunctionLibrary(
   auto native_segment = fdeflib.add_function();
   TF_RETURN_IF_ERROR(tensorflow::GraphToFunctionDef(
       sgraph, StrCat(engine_name, "_native_segment"), native_segment));
+  // Set kIntsonDeviceAttr to true so that all TRTEngineOp outputs are always on
+  // a GPU device as expected. Otherwise, some of the tensors of type DT_INT32
+  // would be on host if the op generating the tensor has host memory tag set.
+  (*native_segment
+        ->mutable_attr())[FunctionLibraryDefinition::kIntsOnDeviceAttr]
+      .set_b(true);
   if (VLOG_IS_ON(7)) {
     VLOG(7) << engine_name << " Function_Def ";
     VLOG(7) << native_segment->DebugString();
-- 
GitLab


From 4e4d38647959597ec39fbc08fb888ae9142e35d8 Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Wed, 30 Jan 2019 18:43:54 -0800
Subject: [PATCH 1645/2345] Do not invoke ROCM toolchain unless -x rocm is
 specified.

---
 .../bin/crosstool_wrapper_driver_rocm.tpl     | 28 +++++++++----------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_rocm.tpl b/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_rocm.tpl
index 824238022b..57c7b635c4 100755
--- a/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_rocm.tpl
+++ b/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_rocm.tpl
@@ -205,25 +205,25 @@ def main():
   args, leftover = parser.parse_known_args(sys.argv[1:])
 
   if args.x and args.x[0] == 'rocm':
+    # XXX use hipcc to link
+    if args.pass_exit_codes:
+      gpu_compiler_flags = [flag for flag in sys.argv[1:]
+                                 if not flag.startswith(('-pass-exit-codes'))]
+
+      # special handling for $ORIGIN
+      # - guard every argument with ''
+      modified_gpu_compiler_flags = []
+      for flag in gpu_compiler_flags:
+        modified_gpu_compiler_flags.append("'" + flag + "'")
+
+      if args.rocm_log: Log('Link with hipcc: %s' % (' '.join([HIPCC_PATH] + modified_gpu_compiler_flags)))
+      return subprocess.call([HIPCC_PATH] + modified_gpu_compiler_flags)
+
     if args.rocm_log: Log('-x rocm')
     leftover = [pipes.quote(s) for s in leftover]
     if args.rocm_log: Log('using hipcc')
     return InvokeHipcc(leftover, log=args.rocm_log)
 
-  # XXX use hipcc to link
-  if args.pass_exit_codes:
-    gpu_compiler_flags = [flag for flag in sys.argv[1:]
-                               if not flag.startswith(('-pass-exit-codes'))]
-
-    # special handling for $ORIGIN
-    # - guard every argument with ''
-    modified_gpu_compiler_flags = []
-    for flag in gpu_compiler_flags:
-      modified_gpu_compiler_flags.append("'" + flag + "'")
-
-    if args.rocm_log: Log('Link with hipcc: %s' % (' '.join([HIPCC_PATH] + modified_gpu_compiler_flags)))
-    return subprocess.call([HIPCC_PATH] + modified_gpu_compiler_flags)
-
   # Strip our flags before passing through to the CPU compiler for files which
   # are not -x rocm. We can't just pass 'leftover' because it also strips -x.
   # We not only want to pass -x to the CPU compiler, but also keep it in its
-- 
GitLab


From 82ede0271e31cd963e6a18f2bf0f10eb660555a7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 30 Jan 2019 18:38:29 -0800
Subject: [PATCH 1646/2345] OpHints supports dynamic_rnn, currently only add
 support for time_major=True case.

PiperOrigin-RevId: 231709454
---
 .../lite/experimental/examples/lstm/BUILD     |   2 +-
 .../experimental/examples/lstm/tflite_lstm.py | 247 +++++++++++++
 .../lstm/unidirectional_sequence_lstm_test.py |  38 +-
 tensorflow/lite/python/op_hint.py             | 333 ++++++++++++++++--
 .../python/framework/graph_util_impl.py       |   9 +-
 ...e.-op-hint.-op-hint-argument-tracker.pbtxt |   2 +-
 .../golden/v1/tensorflow.lite.-op-hint.pbtxt  |  10 +-
 ...e.-op-hint.-op-hint-argument-tracker.pbtxt |   2 +-
 .../golden/v2/tensorflow.lite.-op-hint.pbtxt  |  10 +-
 9 files changed, 580 insertions(+), 73 deletions(-)

diff --git a/tensorflow/lite/experimental/examples/lstm/BUILD b/tensorflow/lite/experimental/examples/lstm/BUILD
index a4950d2dad..ac74d73d4d 100644
--- a/tensorflow/lite/experimental/examples/lstm/BUILD
+++ b/tensorflow/lite/experimental/examples/lstm/BUILD
@@ -44,7 +44,7 @@ py_test(
         "//tensorflow:tensorflow_py",
         "//tensorflow/examples/tutorials/mnist:input_data",
         "//tensorflow/lite/python:lite",
-        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:framework",
         "//tensorflow/python:platform",
         "//tensorflow/python/tools:optimize_for_inference",
         "//third_party/py/numpy",
diff --git a/tensorflow/lite/experimental/examples/lstm/tflite_lstm.py b/tensorflow/lite/experimental/examples/lstm/tflite_lstm.py
index 0038e542a0..e6d329ffd1 100644
--- a/tensorflow/lite/experimental/examples/lstm/tflite_lstm.py
+++ b/tensorflow/lite/experimental/examples/lstm/tflite_lstm.py
@@ -22,17 +22,27 @@ from __future__ import print_function
 import tensorflow as tf
 
 from tensorflow.lite.python import lite
+from tensorflow.python.eager import context
+from tensorflow.python.framework import ops
 from tensorflow.python.keras import activations
 from tensorflow.python.keras import initializers
 from tensorflow.python.layers import base as base_layer
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import clip_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import control_flow_util
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import partitioned_variables
 from tensorflow.python.ops import rnn_cell_impl
+from tensorflow.python.ops import variable_scope as vs
+from tensorflow.python.ops.rnn import _best_effort_input_batch_size
+from tensorflow.python.ops.rnn import _dynamic_rnn_loop
+from tensorflow.python.ops.rnn import _should_cache
+from tensorflow.python.ops.rnn import _transpose_batch_time
 from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.util import nest
 
 
 class TFLiteLSTMCell(rnn_cell_impl.LayerRNNCell):
@@ -394,3 +404,240 @@ class TFLiteLSTMCell(rnn_cell_impl.LayerRNNCell):
     }
     base_config = super(TFLiteLSTMCell, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
+
+
+def dynamic_rnn(cell,
+                inputs,
+                sequence_length=None,
+                initial_state=None,
+                dtype=None,
+                parallel_iterations=None,
+                swap_memory=False,
+                time_major=True,
+                scope=None):
+  """Creates a recurrent neural network specified by RNNCell `cell`.
+
+  Performs fully dynamic unrolling of `inputs`.
+
+  Example:
+
+  ```python
+  # create a BasicRNNCell
+  rnn_cell = tf.nn.rnn_cell.BasicRNNCell(hidden_size)
+
+  # 'outputs' is a tensor of shape [batch_size, max_time, cell_state_size]
+
+  # defining initial state
+  initial_state = rnn_cell.zero_state(batch_size, dtype=tf.float32)
+
+  # 'state' is a tensor of shape [batch_size, cell_state_size]
+  outputs, state = tf.nn.dynamic_rnn(rnn_cell, input_data,
+                                     initial_state=initial_state,
+                                     dtype=tf.float32)
+  ```
+
+  ```python
+  # create 2 LSTMCells
+  rnn_layers = [tf.nn.rnn_cell.LSTMCell(size) for size in [128, 256]]
+
+  # create a RNN cell composed sequentially of a number of RNNCells
+  multi_rnn_cell = tf.nn.rnn_cell.MultiRNNCell(rnn_layers)
+
+  # 'outputs' is a tensor of shape [batch_size, max_time, 256]
+  # 'state' is a N-tuple where N is the number of LSTMCells containing a
+  # tf.contrib.rnn.LSTMStateTuple for each cell
+  outputs, state = tf.nn.dynamic_rnn(cell=multi_rnn_cell,
+                                     inputs=data,
+                                     dtype=tf.float32)
+  ```
+
+
+  Args:
+    cell: An instance of RNNCell.
+    inputs: The RNN inputs.
+      If `time_major == False` (default), this must be a `Tensor` of shape:
+        `[batch_size, max_time, ...]`, or a nested tuple of such elements.
+      If `time_major == True`, this must be a `Tensor` of shape: `[max_time,
+        batch_size, ...]`, or a nested tuple of such elements. This may also be
+        a (possibly nested) tuple of Tensors satisfying this property.  The
+        first two dimensions must match across all the inputs, but otherwise the
+        ranks and other shape components may differ. In this case, input to
+        `cell` at each time-step will replicate the structure of these tuples,
+        except for the time dimension (from which the time is taken). The input
+        to `cell` at each time step will be a `Tensor` or (possibly nested)
+        tuple of Tensors each with dimensions `[batch_size, ...]`.
+    sequence_length: (optional) An int32/int64 vector sized `[batch_size]`. Used
+      to copy-through state and zero-out outputs when past a batch element's
+      sequence length.  So it's more for performance than correctness.
+    initial_state: (optional) An initial state for the RNN. If `cell.state_size`
+      is an integer, this must be a `Tensor` of appropriate type and shape
+      `[batch_size, cell.state_size]`. If `cell.state_size` is a tuple, this
+      should be a tuple of tensors having shapes `[batch_size, s] for s in
+      cell.state_size`.
+    dtype: (optional) The data type for the initial state and expected output.
+      Required if initial_state is not provided or RNN state has a heterogeneous
+      dtype.
+    parallel_iterations: (Default: 32).  The number of iterations to run in
+      parallel.  Those operations which do not have any temporal dependency and
+      can be run in parallel, will be.  This parameter trades off time for
+      space.  Values >> 1 use more memory but take less time, while smaller
+      values use less memory but computations take longer.
+    swap_memory: Transparently swap the tensors produced in forward inference
+      but needed for back prop from GPU to CPU.  This allows training RNNs which
+      would typically not fit on a single GPU, with very minimal (or no)
+      performance penalty.
+    time_major: The shape format of the `inputs` and `outputs` Tensors. If true,
+      these `Tensors` must be shaped `[max_time, batch_size, depth]`. If false,
+      these `Tensors` must be shaped `[batch_size, max_time, depth]`. Using
+      `time_major = True` is a bit more efficient because it avoids transposes
+      at the beginning and end of the RNN calculation.  However, most TensorFlow
+      data is batch-major, so by default this function accepts input and emits
+      output in batch-major form.
+    scope: VariableScope for the created subgraph; defaults to "rnn".
+
+  Returns:
+    A pair (outputs, state) where:
+
+    outputs: The RNN output `Tensor`.
+
+      If time_major == False (default), this will be a `Tensor` shaped:
+        `[batch_size, max_time, cell.output_size]`.
+
+      If time_major == True, this will be a `Tensor` shaped:
+        `[max_time, batch_size, cell.output_size]`.
+
+      Note, if `cell.output_size` is a (possibly nested) tuple of integers
+      or `TensorShape` objects, then `outputs` will be a tuple having the
+      same structure as `cell.output_size`, containing Tensors having shapes
+      corresponding to the shape data in `cell.output_size`.
+
+    state: The final state.  If `cell.state_size` is an int, this
+      will be shaped `[batch_size, cell.state_size]`.  If it is a
+      `TensorShape`, this will be shaped `[batch_size] + cell.state_size`.
+      If it is a (possibly nested) tuple of ints or `TensorShape`, this will
+      be a tuple having the corresponding shapes. If cells are `LSTMCells`
+      `state` will be a tuple containing a `LSTMStateTuple` for each cell.
+
+  Raises:
+    TypeError: If `cell` is not an instance of RNNCell.
+    ValueError: If inputs is None or an empty list.
+    RuntimeError: If not using control flow v2.
+  """
+
+  # Currently only support time_major == True case.
+  assert time_major
+
+  # TODO(b/123051275): We need to check if the cells are TfLiteLSTMCells or
+  # TfLiteRNNCells.
+  rnn_cell_impl.assert_like_rnncell("cell", cell)
+
+  if not control_flow_util.ENABLE_CONTROL_FLOW_V2:
+    raise RuntimeError("OpHint dynamic rnn only supports control flow v2.")
+
+  parent_first_child_input = [{
+      "parent_ophint_input_index": 0,
+      "first_child_ophint_input_index": 0
+  }]
+  parent_last_child_output = [{
+      "parent_output_index": 0,
+      # For LstmCell, the index is 2.
+      # For RnnCell, the index is 1.
+      # So we use -1 meaning it's the last one.
+      "child_output_index": -1
+  }]
+  internal_children_input_output = [{
+      "child_input_index": 0,
+      # For LstmCell, the index is 2.
+      # For RnnCell, the index is 1.
+      # So we use -1 meaning it's the last one.
+      "child_output_index": -1
+  }]
+  inputs_outputs_mappings = {
+      "parent_first_child_input": parent_first_child_input,
+      "parent_last_child_output": parent_last_child_output,
+      "internal_children_input_output": internal_children_input_output
+  }
+  tflite_wrapper = lite.OpHint(
+      "TfLiteDynamicRnn",
+      level=2,
+      children_inputs_mappings=inputs_outputs_mappings)
+  with vs.variable_scope(scope or "rnn") as varscope:
+    # Create a new scope in which the caching device is either
+    # determined by the parent scope, or is set to place the cached
+    # Variable using the same placement as for the rest of the RNN.
+    if _should_cache():
+      if varscope.caching_device is None:
+        varscope.set_caching_device(lambda op: op.device)
+
+    inputs = tflite_wrapper.add_input(inputs, name="input", index_override=0)
+
+    # By default, time_major==False and inputs are batch-major: shaped
+    #   [batch, time, depth]
+    # For internal calculations, we transpose to [time, batch, depth]
+    flat_input = nest.flatten(inputs)
+
+    if not time_major:
+      # (batch, time, depth) => (time, batch, depth)
+      flat_input = [ops.convert_to_tensor(input_) for input_ in flat_input]
+      flat_input = tuple(_transpose_batch_time(input_) for input_ in flat_input)
+
+    parallel_iterations = parallel_iterations or 32
+    if sequence_length is not None:
+      sequence_length = math_ops.to_int32(sequence_length)
+      if sequence_length.get_shape().rank not in (None, 1):
+        raise ValueError(
+            "sequence_length must be a vector of length batch_size, "
+            "but saw shape: %s" % sequence_length.get_shape())
+      sequence_length = array_ops.identity(  # Just to find it in the graph.
+          sequence_length,
+          name="sequence_length")
+
+    batch_size = _best_effort_input_batch_size(flat_input)
+
+    if initial_state is not None:
+      state = initial_state
+    else:
+      if not dtype:
+        raise ValueError("If there is no initial_state, you must give a dtype.")
+      if getattr(cell, "get_initial_state", None) is not None:
+        state = cell.get_initial_state(
+            inputs=None, batch_size=batch_size, dtype=dtype)
+      else:
+        state = cell.zero_state(batch_size, dtype)
+
+    def _assert_has_shape(x, shape):
+      x_shape = array_ops.shape(x)
+      packed_shape = array_ops.stack(shape)
+      return control_flow_ops.Assert(
+          math_ops.reduce_all(math_ops.equal(x_shape, packed_shape)), [
+              "Expected shape for Tensor %s is " % x.name, packed_shape,
+              " but saw shape: ", x_shape
+          ])
+
+    if not context.executing_eagerly() and sequence_length is not None:
+      # Perform some shape validation
+      with ops.control_dependencies(
+          [_assert_has_shape(sequence_length, [batch_size])]):
+        sequence_length = array_ops.identity(
+            sequence_length, name="CheckSeqLen")
+
+    inputs = nest.pack_sequence_as(structure=inputs, flat_sequence=flat_input)
+
+    outputs, final_state = _dynamic_rnn_loop(
+        cell,
+        inputs,
+        state,
+        parallel_iterations=parallel_iterations,
+        swap_memory=swap_memory,
+        sequence_length=sequence_length,
+        dtype=dtype)
+
+    # Outputs of _dynamic_rnn_loop are always shaped [time, batch, depth].
+    # If we are performing batch-major calculations, transpose output back
+    # to shape [batch, time, depth]
+    if not time_major:
+      # (time, batch, depth) => (batch, time, depth)
+      outputs = nest.map_structure(_transpose_batch_time, outputs)
+    outputs = tflite_wrapper.add_output(outputs, name="outputs")
+
+    return outputs, final_state
diff --git a/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_lstm_test.py b/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_lstm_test.py
index eeb48d1231..83ca2b7f94 100644
--- a/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_lstm_test.py
+++ b/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_lstm_test.py
@@ -20,12 +20,14 @@ import numpy as np
 import tensorflow as tf
 
 from tensorflow.examples.tutorials.mnist import input_data
+from tensorflow.lite.experimental.examples.lstm.tflite_lstm import dynamic_rnn
 from tensorflow.lite.experimental.examples.lstm.tflite_lstm import TFLiteLSTMCell
 from tensorflow.lite.python.op_hint import convert_op_hints_to_stubs
 from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 from tensorflow.python.tools import optimize_for_inference_lib
 
+
 # Number of steps to train model.
 TRAIN_STEPS = 1
 
@@ -67,7 +69,7 @@ class UnidirectionalSequenceLstmTest(test_util.TensorFlowTestCase):
         TFLiteLSTMCell(self.num_units, forget_bias=0, name="rnn4")
     ])
 
-  def buildModel(self, lstm_layer, is_dynamic_rnn, is_train):
+  def buildModel(self, lstm_layer, is_dynamic_rnn):
     # Weights and biases for output softmax layer.
     out_weights = tf.Variable(
         tf.random_normal([self.num_units, self.n_classes]))
@@ -77,16 +79,11 @@ class UnidirectionalSequenceLstmTest(test_util.TensorFlowTestCase):
     x = tf.placeholder(
         "float", [None, self.time_steps, self.n_input], name="INPUT_IMAGE")
 
-    # For dynamic_rnn, train with dynamic_rnn and inference with static_rnn.
     # x is shaped [batch_size,time_steps,num_inputs]
     if is_dynamic_rnn:
-      if is_train:
-        lstm_input = x
-        outputs, _ = tf.nn.dynamic_rnn(lstm_layer, lstm_input, dtype="float32")
-        outputs = tf.unstack(outputs, axis=1)
-      else:
-        lstm_input = tf.unstack(x, self.time_steps, 1)
-        outputs, _ = tf.nn.static_rnn(lstm_layer, lstm_input, dtype="float32")
+      lstm_input = tf.transpose(x, perm=[1, 0, 2])
+      outputs, _ = dynamic_rnn(lstm_layer, lstm_input, dtype="float32")
+      outputs = tf.unstack(outputs, axis=0)
     else:
       lstm_input = tf.unstack(x, self.time_steps, 1)
       outputs, _ = tf.nn.static_rnn(lstm_layer, lstm_input, dtype="float32")
@@ -126,8 +123,7 @@ class UnidirectionalSequenceLstmTest(test_util.TensorFlowTestCase):
 
     # Reset the graph.
     tf.reset_default_graph()
-    x, prediction, output_class = self.buildModel(
-        lstm_layer, is_dynamic_rnn, is_train=False)
+    x, prediction, output_class = self.buildModel(lstm_layer, is_dynamic_rnn)
 
     new_sess = tf.Session(config=CONFIG)
     saver = tf.train.Saver()
@@ -159,6 +155,7 @@ class UnidirectionalSequenceLstmTest(test_util.TensorFlowTestCase):
 
     tflite = tf.lite.toco_convert(
         curr, [tflite_input], [outputs], allow_custom_ops=False)
+
     interpreter = tf.lite.Interpreter(model_content=tflite)
 
     try:
@@ -179,7 +176,7 @@ class UnidirectionalSequenceLstmTest(test_util.TensorFlowTestCase):
     sess = tf.Session(config=CONFIG)
 
     x, prediction, output_class = self.buildModel(
-        self.buildLstmLayer(), is_dynamic_rnn=False, is_train=True)
+        self.buildLstmLayer(), is_dynamic_rnn=False)
     self.trainModel(x, prediction, output_class, sess)
 
     saver = tf.train.Saver()
@@ -192,26 +189,15 @@ class UnidirectionalSequenceLstmTest(test_util.TensorFlowTestCase):
     result = self.tfliteInvoke(frozen_graph, test_inputs, output_class)
     self.assertTrue(np.allclose(expected_output, result, rtol=1e-6, atol=1e-2))
 
+  @test_util.enable_control_flow_v2
   def testDynamicRnnMultiRnnCell(self):
     sess = tf.Session(config=CONFIG)
 
     x, prediction, output_class = self.buildModel(
-        self.buildLstmLayer(), is_dynamic_rnn=True, is_train=True)
+        self.buildLstmLayer(), is_dynamic_rnn=True)
     self.trainModel(x, prediction, output_class, sess)
 
-    # Since we don't yet support OpHints for dynamic, we will load the model
-    # back in as a static model. This requires the variables to have the same
-    # names as if they were trained as a static. Thus, we get rid of while/rnn
-    # names.
-    variables_to_save = {}
-    for i in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES):
-      op_name = i.name
-      if op_name.startswith("while/rnn/"):
-        op_name = op_name.split("while/rnn/")[1]
-      if op_name.endswith(":0"):
-        op_name = op_name.split(":0")[0]
-      variables_to_save[op_name] = i
-    saver = tf.train.Saver(variables_to_save)
+    saver = tf.train.Saver()
 
     x, prediction, output_class, new_sess = self.saveAndRestoreModel(
         self.buildLstmLayer(), sess, saver, is_dynamic_rnn=True)
diff --git a/tensorflow/lite/python/op_hint.py b/tensorflow/lite/python/op_hint.py
index 8df37c111b..f107abab90 100644
--- a/tensorflow/lite/python/op_hint.py
+++ b/tensorflow/lite/python/op_hint.py
@@ -71,6 +71,7 @@ from __future__ import print_function
 
 import collections as _collections
 import copy as _copy
+import json as _json
 import uuid as _uuid
 import six as _six
 
@@ -132,6 +133,14 @@ class OpHint(object):
   # "stuff", "foo", "bar", -1 (where -1 is unused). So you would set this
   # attribute to [2, 0, 1, -1].
   TFLITE_INPUT_INDICES = "_tflite_input_indices"
+  # OpHint level.
+  FUNCTION_LEVEL_ATTR = "_tflite_ophint_level"
+  # Ophint internal mapping, this is for high level Ophint only.
+  # This basically contains three kinds of mapping:
+  #   1) How parental ophinted inputs map to the first child ophinted inputs;
+  #   2) How internal children nodes are connected;
+  #   3) How parental ophinted outputs map to the last child ophinted outputs.
+  CHILDREN_INPUTS_MAPPINGS = "_tflite_children_ophint_inputs_mapping"
 
   # Types of aggregations
   #  stack: stacks all ophints with matching tags. i.e. for a static rnn.
@@ -149,10 +158,16 @@ class OpHint(object):
     """Conceptually tracks indices of arguments of "OpHint functions".
 
     The inputs and arguments of these functions both use an instance
-    of the class so they can have independent numbering."""
+    of the class so they can have independent numbering.
+    """
 
-    def __init__(self, function_name, unique_function_id, node_name_prefix,
-                 attr_name):
+    def __init__(self,
+                 function_name,
+                 unique_function_id,
+                 node_name_prefix,
+                 attr_name,
+                 level=1,
+                 children_inputs_mappings=None):
       """Initialize ophint argument.
 
       Args:
@@ -161,6 +176,8 @@ class OpHint(object):
         node_name_prefix: How identities that are created are named.
         attr_name: Name of attribute to use to store the index for this hint.
           i.e. FUNCTION_INPUT_INDEX or FUNCTION_OUTPUT_INDEX
+        level: Hierarchical level of the Ophint node, a number.
+        children_inputs_mappings: Inputs/Outputs mapping for children hints.
       """
 
       # The global index is the argument index of the op. This is in contrast
@@ -176,6 +193,8 @@ class OpHint(object):
       self._tag_to_next_sort_index = {}  # The current index for each tag
       self._node_name_prefix = node_name_prefix
       self._attr_name = attr_name
+      self._level = level
+      self._children_inputs_mappings = children_inputs_mappings
 
     def _get_new_global_index(self, index_override):
       """Return the next unused argument index in order or use an override.
@@ -251,6 +270,7 @@ class OpHint(object):
       uuid = self._unique_function_id
       name = "%s-%s-%s-%r-%r-%s" % (self._node_name_prefix, self._function_name,
                                     uuid, global_index, sort_index, name)
+
       identity_op = _array_ops.identity(arg, name=name)
 
       # pylint: disable=protected-access
@@ -264,6 +284,15 @@ class OpHint(object):
               s=_compat.as_bytes(self._unique_function_id)))
       identity_op.op._set_attr(
           self._attr_name, _attr_value_pb2.AttrValue(i=global_index))
+      identity_op.op._set_attr(OpHint.FUNCTION_LEVEL_ATTR,
+                               _attr_value_pb2.AttrValue(i=self._level))
+      if self._children_inputs_mappings:
+        identity_op.op._set_attr(
+            OpHint.CHILDREN_INPUTS_MAPPINGS,
+            _attr_value_pb2.AttrValue(
+                s=_compat.as_bytes(_json.dumps(
+                    self._children_inputs_mappings))))
+
       if sort_index is not None:
         identity_op.op._set_attr(
             OpHint.FUNCTION_SORT_INDEX_ATTR,
@@ -275,23 +304,74 @@ class OpHint(object):
       # pylint: enable=protected-access
       return identity_op
 
-  def __init__(self, function_name, **kwargs):
+  def __init__(self,
+               function_name,
+               level=1,
+               children_inputs_mappings=None,
+               **kwargs):
     """Create a OpHint.
 
     Args:
       function_name: Name of the function (the custom op name in tflite)
+      level: OpHint level.
+      children_inputs_mappings: Children OpHint inputs/outputs mapping.
+        children_inputs_mappings should like below:
+        "parent_first_child_input":
+            [{"parent_input_index": num, "child_input_index": num}, ...]
+        "parent_last_child_output":
+            [{"parent_output_index": num, "child_output_index": num}, ...]
+        "internal_children_input_output":
+            [{"child_input_index": num, "child_output_index": num}, ...]
       **kwargs: Keyword arguments of any constant attributes for the function.
     """
     self._function_name = function_name
+    self._level = level
+    if self._level == 1:
+      assert children_inputs_mappings is None
+    else:
+      assert isinstance(children_inputs_mappings, dict)
+    self._children_inputs_mappings = children_inputs_mappings
+    if self._children_inputs_mappings is not None:
+      self._validate_children_inputs_mappings(self._children_inputs_mappings)
     self._unique_function_id = _uuid.uuid1().hex  # TODO(aselle): Unique enough?
     self._attrs_to_store_later = kwargs
     self._stored_attrs = False
     self._inputs = OpHint.OpHintArgumentTracker(
         self._function_name, self._unique_function_id, "InputHint",
-        OpHint.FUNCTION_INPUT_INDEX_ATTR)
+        OpHint.FUNCTION_INPUT_INDEX_ATTR, level, self._children_inputs_mappings)
     self._outputs = OpHint.OpHintArgumentTracker(
         self._function_name, self._unique_function_id, "OutputHint",
-        OpHint.FUNCTION_OUTPUT_INDEX_ATTR)
+        OpHint.FUNCTION_OUTPUT_INDEX_ATTR, level,
+        self._children_inputs_mappings)
+
+  def _validate_children_inputs_mappings(self, children_inputs_mappings):
+    """Validate children inputs mappings is in the right format.
+
+    Args:
+      children_inputs_mappings: the Children ophint inputs/outputs mapping.
+    """
+    assert isinstance(children_inputs_mappings, dict)
+    assert "parent_first_child_input" in children_inputs_mappings
+    assert "parent_last_child_output" in children_inputs_mappings
+    assert "internal_children_input_output" in children_inputs_mappings
+
+    # validate parent_first_child_input.
+
+    def assert_dictlist_has_keys(dictlist, keys):
+      for dikt in dictlist:
+        assert isinstance(dikt, dict)
+        for key in keys:
+          assert key in dikt
+
+    assert_dictlist_has_keys(
+        children_inputs_mappings["parent_first_child_input"],
+        ["parent_ophint_input_index", "first_child_ophint_input_index"])
+    assert_dictlist_has_keys(
+        children_inputs_mappings["parent_last_child_output"],
+        ["parent_output_index", "child_output_index"])
+    assert_dictlist_has_keys(
+        children_inputs_mappings["internal_children_input_output"],
+        ["child_input_index", "child_output_index"])
 
   def _setattr(self, dest_op, name, value):
     tensor_value = _ops.convert_to_tensor(value)
@@ -382,7 +462,7 @@ class OpHint(object):
 
 
 class _LiteOperand(object):
-  """Abstract operand for a tflite hint function.
+  """Abstract operand for a tflite hint function._dynamic_rnn_loop.
 
   This is a base class that handles representing arguments to an OpHint.
   It also is able to serialize operands to the stubbed graph_def.
@@ -580,15 +660,18 @@ class _LiteFuncCall(object):
   This is uses to accumulate found hints in the graphdef into a single
   conceptual unit.
 
-  Properties:
-    self.inputs: inputs to the op (hash from index # to argument)
-    self.outputs: outputs to the op (hash from index # to argument)
-    self.function_name: the tflite custom op name to use
-    self.uuid: a unique call id for this particular call  (i.e.
+  Attributes:
+    inputs: inputs to the op (hash from index # to argument)
+    outputs: outputs to the op (hash from index # to argument)
+    function_name: the tflite custom op name to use
+    uuid: a unique call id for this particular call  (i.e.
       multiple function calls would have the same function_name but different
       uuids.
-    self.params: A param name to key value for op constant data. I.e. for
+    params: A param name to key value for op constant data. I.e. for
       axis on a reduction, strides on a convolution, etc.
+    level: Level of the OpHint.
+    children_inputs_mappings: If the Ophint has children, children inputs
+      mappings indicate how their inputs & outputs are mapped.
   """
 
   def __init__(self):
@@ -597,6 +680,8 @@ class _LiteFuncCall(object):
     self.function_name = None
     self.uuid = None
     self.params = {}
+    self.level = -1
+    self.children_inputs_mappings = {}
 
   def flattened_inputs_and_outputs(self):
     """Return a list of inputs and outputs in a flattened format.
@@ -622,22 +707,25 @@ class _LiteFuncCall(object):
     inputs_str = "\tInputs\n" + format_args(self.inputs)
     outputs_str = "\tOutputs\n" + format_args(self.outputs)
 
-    return ("tflite function %s call %s\n\tinputs:\n\t\t%s\n\toutputs:\n\t\t%s"
-            % (self.function_name, self.uuid, inputs_str, outputs_str))
+    return (
+        "tflite function %s call %s level %d "
+        "\n\tinputs:\n\t\t%s\n\toutputs:\n\t\t%s" %
+        (self.function_name, self.uuid, self.level, inputs_str, outputs_str))
 
 
-def _find_all_hints_in_graph_def(graphdef):
-  """Look at the current default graph and return a list of LiteFuncCall objs.
+def _find_all_hints_in_nodes(nodes):
+  """Look at the all the input nodes and return a list of LiteFuncCall objs.
 
   Args:
-    graphdef: A TensorFlow graph_def to look for LiteFuncCalls.
+    nodes: A TensorFlow graph_def to look for LiteFuncCalls.
+
   Returns:
     a list of `LifeFuncCall` objects in the form
 
   """
   func_calls = _collections.defaultdict(_LiteFuncCall)
 
-  for node in graphdef.node:
+  for node in nodes:
     attr = node.attr
     # This is an op hint if it has a FUNCTION_UUID_ATTR, otherwise skip
     uuid = attr[OpHint.FUNCTION_UUID_ATTR].s
@@ -649,6 +737,7 @@ def _find_all_hints_in_graph_def(graphdef):
     call_def = func_calls[uuid]
     call_def.uuid = uuid
     call_def.function_name = attr[OpHint.FUNCTION_NAME_ATTR].s
+    call_def.level = attr[OpHint.FUNCTION_LEVEL_ATTR].i
     # Get sorting and aggregation information
 
     sort = (attr[OpHint.FUNCTION_SORT_INDEX_ATTR].i
@@ -658,6 +747,10 @@ def _find_all_hints_in_graph_def(graphdef):
     if OpHint.FUNCTION_AGGREGATE_ATTR in attr:
       aggregation = _compat.as_text(attr[OpHint.FUNCTION_AGGREGATE_ATTR].s)
 
+    if OpHint.CHILDREN_INPUTS_MAPPINGS in attr:
+      call_def.children_inputs_mappings = _json.loads(
+          _compat.as_text(attr[OpHint.CHILDREN_INPUTS_MAPPINGS].s))
+
     # Add the input or output
     def put_operand(stuff, index, sort, operand, aggregation):
       """Add a given index into the function structure."""
@@ -683,6 +776,98 @@ def _find_all_hints_in_graph_def(graphdef):
   return func_calls
 
 
+def _extract_topology_sequence_mapping(nodes):
+  return dict(
+      (_tensor_name_base(node.name), idx) for idx, node in enumerate(nodes))
+
+
+def _find_children_hints_in_while_loop(function_def, nodes_mapping):
+  """Find children hints and all nodes inside the while loop.
+
+  Args:
+    function_def: Function def of the while loop.
+    nodes_mapping: While loop input_arg : real node name.
+
+  Returns:
+    Ordered children hints and all re-mapped nodes inside the while loop.
+  """
+  new_nodes = []
+
+  # Make nodes inside function def inputs point to the real nodes.
+  for node in function_def.node_def:
+    for i in range(len(node.input)):
+      if node.input[i] in nodes_mapping:
+        node.input[i] = nodes_mapping[node.input[i]]
+    new_nodes.append(_copy.deepcopy(node))
+  name_to_seq_num = _extract_topology_sequence_mapping(function_def.node_def)
+  children_hints = _find_all_hints_in_nodes(new_nodes)
+  children_hints_q = []
+  # Ordered by the outputs.
+  for hint in _six.itervalues(children_hints):
+    _, output_names = hint.flattened_inputs_and_outputs()
+    seq = name_to_seq_num[output_names[0]]
+    for output_name in output_names:
+      seq = min(seq, name_to_seq_num[output_name])
+    children_hints_q.append((seq, hint))
+  children_hints_q.sort(key=lambda tup: tup[0])
+  ordered_children_hints = [x[1] for x in children_hints_q]
+  return ordered_children_hints, new_nodes
+
+
+def _find_children_hints(call, graph_def):
+  """Find all children hints.
+
+  For a given OpHint, we find all children hints inside it, we also copy all the
+  nodes inside function defs (if applicable) to the original graph_def, they are
+  returned in a list as well.
+
+  Args:
+    call: Parent OpHint that contains children ophints.
+    graph_def: Original graph def.
+
+  Returns:
+    Ordered children hints inside the parent ophint; new graph def that contains
+    nodes inside function defs (if applicable); nodes inside function defs.
+  """
+  name_to_input_name, _, _ = _extract_graph_summary(graph_def)
+  input_names, output_names = call.flattened_inputs_and_outputs()
+
+  reachable_by_input = _bfs_for_reachable_nodes(input_names, name_to_input_name)
+  reachable_by_output = _bfs_for_reachable_nodes(output_names,
+                                                 name_to_input_name)
+  output_nodes_set = set(output_names)
+  children_hints = []
+  out = _graph_pb2.GraphDef()
+  out.library.CopyFrom(graph_def.library)
+  out.versions.CopyFrom(graph_def.versions)
+  function_def_nodes = set()
+  for node in graph_def.node:
+    out.node.extend([_copy.deepcopy(node)])
+    n = _tensor_name_base(node.name)
+    if n in reachable_by_output:
+      if n not in reachable_by_input and n not in output_nodes_set:
+        # special handle for while loop function def.
+        if node.op == "While":
+          body_name = node.attr["body"].func.name
+          inputs_outside_loop = node.input
+          for function_def in graph_def.library.function:
+            if function_def.signature.name == body_name:
+              function_inputs = function_def.signature.input_arg
+              assert len(inputs_outside_loop) == len(function_inputs)
+              nodes_mapping = {}
+              for i in range(len(function_inputs)):
+                nodes_mapping[function_inputs[i].name] = inputs_outside_loop[i]
+              # TODO(b/123050804): Consider use grappler.
+              (children_hints_in_loop,
+               new_nodes) = _find_children_hints_in_while_loop(
+                   function_def, nodes_mapping)
+              function_def_nodes.update([x.name for x in new_nodes])
+              children_hints.extend(children_hints_in_loop)
+              out.node.extend(new_nodes)
+
+  return children_hints, out, function_def_nodes
+
+
 def _tensor_name_base(full_tensor_name):
   """Removes the device assignment code from a tensor.
 
@@ -735,12 +920,20 @@ def _check_subgraph_closed(n, reachable_by_input, input_nodes_set,
 
 
 # TODO(aselle): This should be converted to grappler in the future.
-def _convert_single_op_hint_to_stub(call, graph_def):
+def _convert_single_op_hint_to_stub(call,
+                                    graph_def,
+                                    function_def_nodes=None,
+                                    is_last_run=True):
   """Given a graph_def, converts `call` into a stub and returns a new graph_def.
 
   Args:
     call: A single function call to be converted.
     graph_def: A graph_def to use as input (that hass call obviously).
+    function_def_nodes: Nodes inside the function def those are not connected to
+      the graph.
+    is_last_run: Whether it is the last run for a given pass (for OpHint has
+      children).
+
   Returns:
     A new transformed graph-def that has call as a stub (single op).
 
@@ -748,6 +941,8 @@ def _convert_single_op_hint_to_stub(call, graph_def):
       the tensorflow runtime, so all future manipulations are done in graph_def
       level.
   """
+  if function_def_nodes is None:
+    function_def_nodes = set()
   name_to_input_name, name_to_node, name_to_seq_num = _extract_graph_summary(
       graph_def)
   input_names, output_names = call.flattened_inputs_and_outputs()
@@ -755,7 +950,6 @@ def _convert_single_op_hint_to_stub(call, graph_def):
   reachable_by_input = _bfs_for_reachable_nodes(input_names, name_to_input_name)
   reachable_by_output = _bfs_for_reachable_nodes(output_names,
                                                  name_to_input_name)
-  input_nodes_set = set(input_names)
   output_nodes_set = set(output_names)
   nodes_after_fuse = []
   nodes_deleted_by_fuse = set()
@@ -766,19 +960,16 @@ def _convert_single_op_hint_to_stub(call, graph_def):
     n = _tensor_name_base(node.name)
     if n in reachable_by_output:
       if n not in reachable_by_input and n not in output_nodes_set:
-        # n is an internal node. Check to make sure it is really internal.
-        # TODO(aselle): this could be done more efficiently by flooding
-        # the graph first.
-        _check_subgraph_closed(n, reachable_by_input, input_nodes_set,
-                               name_to_input_name)
         nodes_deleted_by_fuse.add(n)
-    elif n not in reachable_by_input:
+    elif n not in reachable_by_input and n not in function_def_nodes:
       # n is a node that after all the fusings, so keep it.
       nodes_after_fuse.append(n)
     else:
-      # n is a node that is randomly in the graph but not connected to
-      # the chain of dependencies.
-      pass
+      # In the last run, n is a node that is randomly in the graph but not
+      # connected to the chain of dependencies, we will delete n, otherwise
+      # we keep them.
+      if not is_last_run:
+        nodes_after_fuse.append(n)
 
   # Make a new graphdef with all the pre-input and input nodes
   out = _graph_pb2.GraphDef()
@@ -800,7 +991,8 @@ def _convert_single_op_hint_to_stub(call, graph_def):
   # non-fused things.
   for input_index in sorted_input_indices:
     inputs = call.inputs[input_index]
-    new_node.input.append(inputs.aggregate_and_return_name_for_input(out))
+    input_name = inputs.aggregate_and_return_name_for_input(out)
+    new_node.input.append(input_name)
   new_node.attr[OpHint.TFLITE_INPUT_INDICES].list.i.extend(sorted_input_indices)
 
   # Ceate the function
@@ -936,6 +1128,18 @@ def _remove_redundant_stack_unstack(graph_def):
   return curr
 
 
+def _get_correct_mapping(original_index, nodes):
+  # Special handle for the index is -1 case.
+  # If it is -1, return the last index.
+  if original_index == -1:
+    node_indices = nodes.keys()
+    node_indices.sort()
+    return node_indices[-1]
+  else:
+    return original_index
+  return original_index
+
+
 @_tf_export("lite.convert_op_hints_to_stubs")
 def _convert_op_hints_to_stubs_helper(
     graph_def, write_callback=lambda sess, graph_def: None):
@@ -948,14 +1152,67 @@ def _convert_op_hints_to_stubs_helper(
   Returns:
     A new stubbed graph_def.
   """
+  hints = _find_all_hints_in_nodes(graph_def.node)
+
+  hints_q = []
+  for hint in _six.itervalues(hints):
+    hints_q.append((hint.level, hint.uuid))
+
+  hints_q.sort(key=lambda tup: tup[0])
+  for i in range(len(hints_q) - 1, -1, -1):
+    level, hint_uuid = hints_q[i]
 
-  hints = _find_all_hints_in_graph_def(graph_def)
   curr_graph_def = graph_def
   del graph_def  # prevent using graph_def again (common source of error)
-  for hint in _six.itervalues(hints):
-    curr_graph_def = _convert_single_op_hint_to_stub(
-        hint, curr_graph_def)
-    write_callback(curr_graph_def, "initial")
+  for i in range(len(hints_q) - 1, -1, -1):
+    level, hint_uuid = hints_q[i]
+    if level >= 2:
+      children_hints, curr_graph_def, function_def_nodes = _find_children_hints(
+          hints[hint_uuid], curr_graph_def)
+      # pylint: disable=superfluous-parens
+      assert (len(children_hints) > 0)  #  pylint: disable=g-explicit-length-test
+      # pylint: enable=superfluous-parens
+
+      # Re-wire the children hints inputs/outputs, so latter child's inputs
+      # connect to previous child node's outputs.
+      children_inputs_mappings = hints[hint_uuid].children_inputs_mappings
+      for j in range(len(children_hints)):
+        child_hint = children_hints[j]
+        if j == 0:
+          for mapping in children_inputs_mappings["parent_first_child_input"]:
+            parent_input_index = _get_correct_mapping(
+                mapping["parent_ophint_input_index"], hints[hint_uuid].inputs)
+            child_input_index = _get_correct_mapping(
+                mapping["first_child_ophint_input_index"], child_hint.inputs)
+            child_hint.inputs[child_input_index] = hints[hint_uuid].inputs[
+                parent_input_index]
+        else:
+          for mapping in children_inputs_mappings[
+              "internal_children_input_output"]:
+            input_index = _get_correct_mapping(mapping["child_input_index"],
+                                               child_hint.inputs)
+            output_index = _get_correct_mapping(mapping["child_output_index"],
+                                                children_hints[j - 1].outputs)
+            child_hint.inputs[input_index] = children_hints[
+                j - 1].outputs[output_index]
+        if j == len(children_hints) - 1:
+          for mapping in children_inputs_mappings["parent_last_child_output"]:
+            parent_output_index = _get_correct_mapping(
+                mapping["parent_output_index"], hints[hint_uuid].outputs)
+            child_output_index = _get_correct_mapping(
+                mapping["child_output_index"], child_hint.outputs)
+            child_hint.outputs[child_output_index] = hints[hint_uuid].outputs[
+                parent_output_index]
+
+      for j in range(len(children_hints)):
+        child_hint = children_hints[j]
+        curr_graph_def = _convert_single_op_hint_to_stub(
+            child_hint, curr_graph_def, function_def_nodes,
+            j == len(children_hints) - 1)
+    else:
+      curr_graph_def = _convert_single_op_hint_to_stub(hints[hint_uuid],
+                                                       curr_graph_def)
+      write_callback(curr_graph_def, "initial")
   # The stubbing process can create stacks/unstacks in the case of LSTMs
   # remove them.
   curr_graph_def = _remove_redundant_stack_unstack(curr_graph_def)
@@ -982,9 +1239,9 @@ def find_all_hinted_output_nodes(session=None, graph_def=None):
     raise ValueError("Provide only one of session and graph_def.")
   hinted_outputs_nodes = []
   if session is not None:
-    hints = _find_all_hints_in_graph_def(session.graph_def)
+    hints = _find_all_hints_in_nodes(session.graph_def.node)
   elif graph_def is not None:
-    hints = _find_all_hints_in_graph_def(graph_def)
+    hints = _find_all_hints_in_nodes(graph_def.node)
   for hint in _six.itervalues(hints):
     _, ouput_nodes = hint.flattened_inputs_and_outputs()
     hinted_outputs_nodes.extend(ouput_nodes)
diff --git a/tensorflow/python/framework/graph_util_impl.py b/tensorflow/python/framework/graph_util_impl.py
index 1b61ac925c..a46fccc513 100644
--- a/tensorflow/python/framework/graph_util_impl.py
+++ b/tensorflow/python/framework/graph_util_impl.py
@@ -143,13 +143,14 @@ def _bfs_for_reachable_nodes(target_nodes, name_to_input_name):
   # Breadth first search to find all the nodes that we should keep.
   next_to_visit = target_nodes[:]
   while next_to_visit:
-    n = next_to_visit[0]
+    node = next_to_visit[0]
     del next_to_visit[0]
-    if n in nodes_to_keep:
+    if node in nodes_to_keep:
       # Already visited this node.
       continue
-    nodes_to_keep.add(n)
-    next_to_visit += name_to_input_name[n]
+    nodes_to_keep.add(node)
+    if node in name_to_input_name:
+      next_to_visit += name_to_input_name[node]
   return nodes_to_keep
 
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.lite.-op-hint.-op-hint-argument-tracker.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.lite.-op-hint.-op-hint-argument-tracker.pbtxt
index 1fe179f6c1..68cb07ea6f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.lite.-op-hint.-op-hint-argument-tracker.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.lite.-op-hint.-op-hint-argument-tracker.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'function_name\', \'unique_function_id\', \'node_name_prefix\', \'attr_name\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'self\', \'function_name\', \'unique_function_id\', \'node_name_prefix\', \'attr_name\', \'level\', \'children_inputs_mappings\'], varargs=None, keywords=None, defaults=[\'1\', \'None\'], "
   }
   member_method {
     name: "add"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.lite.-op-hint.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.lite.-op-hint.pbtxt
index 66e692a5a3..3ac478f762 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.lite.-op-hint.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.lite.-op-hint.pbtxt
@@ -14,6 +14,10 @@ tf_class {
     name: "AGGREGATE_STACK"
     mtype: "<type \'str\'>"
   }
+  member {
+    name: "CHILDREN_INPUTS_MAPPINGS"
+    mtype: "<type \'str\'>"
+  }
   member {
     name: "FUNCTION_AGGREGATE_ATTR"
     mtype: "<type \'str\'>"
@@ -22,6 +26,10 @@ tf_class {
     name: "FUNCTION_INPUT_INDEX_ATTR"
     mtype: "<type \'str\'>"
   }
+  member {
+    name: "FUNCTION_LEVEL_ATTR"
+    mtype: "<type \'str\'>"
+  }
   member {
     name: "FUNCTION_NAME_ATTR"
     mtype: "<type \'str\'>"
@@ -48,7 +56,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'function_name\'], varargs=None, keywords=kwargs, defaults=None"
+    argspec: "args=[\'self\', \'function_name\', \'level\', \'children_inputs_mappings\'], varargs=None, keywords=kwargs, defaults=[\'1\', \'None\'], "
   }
   member_method {
     name: "add_input"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.lite.-op-hint.-op-hint-argument-tracker.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.lite.-op-hint.-op-hint-argument-tracker.pbtxt
index 1fe179f6c1..68cb07ea6f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.lite.-op-hint.-op-hint-argument-tracker.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.lite.-op-hint.-op-hint-argument-tracker.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'function_name\', \'unique_function_id\', \'node_name_prefix\', \'attr_name\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'self\', \'function_name\', \'unique_function_id\', \'node_name_prefix\', \'attr_name\', \'level\', \'children_inputs_mappings\'], varargs=None, keywords=None, defaults=[\'1\', \'None\'], "
   }
   member_method {
     name: "add"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.lite.-op-hint.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.lite.-op-hint.pbtxt
index 66e692a5a3..3ac478f762 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.lite.-op-hint.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.lite.-op-hint.pbtxt
@@ -14,6 +14,10 @@ tf_class {
     name: "AGGREGATE_STACK"
     mtype: "<type \'str\'>"
   }
+  member {
+    name: "CHILDREN_INPUTS_MAPPINGS"
+    mtype: "<type \'str\'>"
+  }
   member {
     name: "FUNCTION_AGGREGATE_ATTR"
     mtype: "<type \'str\'>"
@@ -22,6 +26,10 @@ tf_class {
     name: "FUNCTION_INPUT_INDEX_ATTR"
     mtype: "<type \'str\'>"
   }
+  member {
+    name: "FUNCTION_LEVEL_ATTR"
+    mtype: "<type \'str\'>"
+  }
   member {
     name: "FUNCTION_NAME_ATTR"
     mtype: "<type \'str\'>"
@@ -48,7 +56,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'function_name\'], varargs=None, keywords=kwargs, defaults=None"
+    argspec: "args=[\'self\', \'function_name\', \'level\', \'children_inputs_mappings\'], varargs=None, keywords=kwargs, defaults=[\'1\', \'None\'], "
   }
   member_method {
     name: "add_input"
-- 
GitLab


From 6ec5dfe9a88814aac007e683905f4c648421393e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 30 Jan 2019 18:46:09 -0800
Subject: [PATCH 1647/2345] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 231710232

---
 tensorflow/go/op/wrappers.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 97b6d4bba6..5f823be62b 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -33808,7 +33808,8 @@ func DecodeProtoV2Sanitize(value bool) DecodeProtoV2Attr {
 // Arguments:
 //	bytes: Tensor of serialized protos with shape `batch_shape`.
 //	message_type: Name of the proto message type to decode.
-//	field_names: List of strings containing proto field names.
+//	field_names: List of strings containing proto field names. An extension field can be decoded
+// by using its full name, e.g. EXT_PACKAGE.EXT_FIELD_NAME.
 //	output_types: List of TF types to use for the respective field in field_names.
 //
 // Returns Tensor of int32 with shape `[batch_shape, len(field_names)]`.
-- 
GitLab


From c8d5380fa36ba12f6a30439b8d8401b05beb0d67 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Wed, 30 Jan 2019 19:09:40 -0800
Subject: [PATCH 1648/2345] [tf.data] Disabling a flaky test.

PiperOrigin-RevId: 231712475
---
 tensorflow/python/data/experimental/kernel_tests/sleep_test.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/data/experimental/kernel_tests/sleep_test.py b/tensorflow/python/data/experimental/kernel_tests/sleep_test.py
index a4fe847f04..4733c2a833 100644
--- a/tensorflow/python/data/experimental/kernel_tests/sleep_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/sleep_test.py
@@ -33,6 +33,7 @@ _NUMPY_RANDOM_SEED = 42
 class SleepTest(test_base.DatasetTestBase):
 
   def testSleep(self):
+    self.skipTest("b/123597912")
     sleep_microseconds = 100
     dataset = dataset_ops.Dataset.range(10).apply(
         sleep.sleep(sleep_microseconds))
-- 
GitLab


From 9051c1696acd6f727462adeab67747db0c660ddc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 30 Jan 2019 19:16:30 -0800
Subject: [PATCH 1649/2345] internal cleanup

PiperOrigin-RevId: 231713033
---
 .../examples/lstm/bidirectional_sequence_lstm_test.py        | 4 ++--
 .../examples/lstm/bidirectional_sequence_rnn_test.py         | 4 ++--
 .../examples/lstm/unidirectional_sequence_lstm_test.py       | 5 ++---
 .../examples/lstm/unidirectional_sequence_rnn_test.py        | 4 ++--
 4 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/tensorflow/lite/experimental/examples/lstm/bidirectional_sequence_lstm_test.py b/tensorflow/lite/experimental/examples/lstm/bidirectional_sequence_lstm_test.py
index 99f4beda59..9dc81096bc 100644
--- a/tensorflow/lite/experimental/examples/lstm/bidirectional_sequence_lstm_test.py
+++ b/tensorflow/lite/experimental/examples/lstm/bidirectional_sequence_lstm_test.py
@@ -150,8 +150,8 @@ class BidirectionalSequenceLstmTest(test_util.TensorFlowTestCase):
         curr, ["INPUT_IMAGE_LITE"], ["OUTPUT_CLASS"],
         [tf.float32.as_datatype_enum])
 
-    tflite = tf.lite.toco_convert(
-        curr, [tflite_input], [outputs], allow_custom_ops=False)
+    converter = tf.lite.TFLiteConverter(curr, [tflite_input], [outputs])
+    tflite = converter.convert()
 
     interpreter = tf.lite.Interpreter(model_content=tflite)
 
diff --git a/tensorflow/lite/experimental/examples/lstm/bidirectional_sequence_rnn_test.py b/tensorflow/lite/experimental/examples/lstm/bidirectional_sequence_rnn_test.py
index d049c7834e..7a937ce47f 100644
--- a/tensorflow/lite/experimental/examples/lstm/bidirectional_sequence_rnn_test.py
+++ b/tensorflow/lite/experimental/examples/lstm/bidirectional_sequence_rnn_test.py
@@ -149,8 +149,8 @@ class BidirectionalSequenceRnnTest(test_util.TensorFlowTestCase):
         curr, ["INPUT_IMAGE_LITE"], ["OUTPUT_CLASS"],
         [tf.float32.as_datatype_enum])
 
-    tflite = tf.lite.toco_convert(
-        curr, [tflite_input], [outputs], allow_custom_ops=False)
+    converter = tf.lite.TFLiteConverter(curr, [tflite_input], [outputs])
+    tflite = converter.convert()
 
     interpreter = tf.lite.Interpreter(model_content=tflite)
 
diff --git a/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_lstm_test.py b/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_lstm_test.py
index 83ca2b7f94..ed02d6c906 100644
--- a/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_lstm_test.py
+++ b/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_lstm_test.py
@@ -153,9 +153,8 @@ class UnidirectionalSequenceLstmTest(test_util.TensorFlowTestCase):
         curr, ["INPUT_IMAGE_LITE"], ["OUTPUT_CLASS"],
         [tf.float32.as_datatype_enum])
 
-    tflite = tf.lite.toco_convert(
-        curr, [tflite_input], [outputs], allow_custom_ops=False)
-
+    converter = tf.lite.TFLiteConverter(curr, [tflite_input], [outputs])
+    tflite = converter.convert()
     interpreter = tf.lite.Interpreter(model_content=tflite)
 
     try:
diff --git a/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_rnn_test.py b/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_rnn_test.py
index 6f9e2dd949..af7ac4c4d8 100644
--- a/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_rnn_test.py
+++ b/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_rnn_test.py
@@ -160,8 +160,8 @@ class UnidirectionalSequenceRnnTest(test_util.TensorFlowTestCase):
         curr, ["INPUT_IMAGE_LITE"], ["OUTPUT_CLASS"],
         [tf.float32.as_datatype_enum])
 
-    tflite = tf.lite.toco_convert(
-        curr, [tflite_input], [outputs], allow_custom_ops=False)
+    converter = tf.lite.TFLiteConverter(curr, [tflite_input], [outputs])
+    tflite = converter.convert()
     interpreter = tf.lite.Interpreter(model_content=tflite)
     interpreter.allocate_tensors()
 
-- 
GitLab


From f3afacffc3c6b431677a69754bb69a7791261318 Mon Sep 17 00:00:00 2001
From: yonghuiguo <819123722@qq.com>
Date: Thu, 31 Jan 2019 12:01:57 +0800
Subject: [PATCH 1650/2345] Fix bug in hadoop_file_system.cc when reading big
 variable of hdfs

---
 tensorflow/core/platform/hadoop/hadoop_file_system.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/platform/hadoop/hadoop_file_system.cc b/tensorflow/core/platform/hadoop/hadoop_file_system.cc
index 4d1826c3a1..076fad07af 100644
--- a/tensorflow/core/platform/hadoop/hadoop_file_system.cc
+++ b/tensorflow/core/platform/hadoop/hadoop_file_system.cc
@@ -209,8 +209,9 @@ class HDFSRandomAccessFile : public RandomAccessFile {
       // We lock inside the loop rather than outside so we don't block other
       // concurrent readers.
       mutex_lock lock(mu_);
+      size_t read_n = std::min(n,static_cast<size_t>(std::numeric_limits<int>::max()-2));
       tSize r = hdfs_->hdfsPread(fs_, file_, static_cast<tOffset>(offset), dst,
-                                 static_cast<tSize>(n));
+                                 static_cast<tSize>(read_n));
       if (r > 0) {
         dst += r;
         n -= r;
-- 
GitLab


From 40cf0a02e3b0803788f7da09b90bc83829a047e8 Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Wed, 30 Jan 2019 21:04:43 -0800
Subject: [PATCH 1651/2345] Implement Integer per channel quantization of
 DepthwiseConv.

PiperOrigin-RevId: 231722033
---
 tensorflow/lite/kernels/depthwise_conv.cc     |  72 ++++++++--
 .../lite/kernels/depthwise_conv_test.cc       | 102 +++++++++++++-
 tensorflow/lite/kernels/internal/BUILD        |   1 +
 .../reference/integer_ops/depthwise_conv.h    | 125 ++++++++++++++++++
 4 files changed, 284 insertions(+), 16 deletions(-)
 create mode 100644 tensorflow/lite/kernels/internal/reference/integer_ops/depthwise_conv.h

diff --git a/tensorflow/lite/kernels/depthwise_conv.cc b/tensorflow/lite/kernels/depthwise_conv.cc
index 3f4ae5087b..a349b27905 100644
--- a/tensorflow/lite/kernels/depthwise_conv.cc
+++ b/tensorflow/lite/kernels/depthwise_conv.cc
@@ -26,6 +26,7 @@ limitations under the License.
 #include "tensorflow/lite/kernels/internal/quantization_util.h"
 #include "tensorflow/lite/kernels/internal/reference/depthwiseconv_float.h"
 #include "tensorflow/lite/kernels/internal/reference/depthwiseconv_uint8.h"
+#include "tensorflow/lite/kernels/internal/reference/integer_ops/depthwise_conv.h"
 #include "tensorflow/lite/kernels/internal/tensor.h"
 #include "tensorflow/lite/kernels/kernel_util.h"
 #include "tensorflow/lite/kernels/op_macros.h"
@@ -58,6 +59,10 @@ struct OpData {
   // uint8_t these would be 0 and 255.
   int32_t output_activation_min;
   int32_t output_activation_max;
+
+  // Per channel output multiplier and shift.
+  std::vector<int32_t> per_channel_output_multiplier;
+  std::vector<int> per_channel_output_shift;
 };
 
 void* Init(TfLiteContext* context, const char* buffer, size_t length) {
@@ -99,14 +104,15 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
                     SizeOfDimension(filter, 3));
 
   const TfLiteType data_type = input->type;
-  TF_LITE_ENSURE(context,
-                 data_type == kTfLiteFloat32 || data_type == kTfLiteUInt8);
+  TF_LITE_ENSURE(context, data_type == kTfLiteFloat32 ||
+                              data_type == kTfLiteUInt8 ||
+                              data_type == kTfLiteInt8);
   TF_LITE_ENSURE_EQ(context, output->type, data_type);
   TF_LITE_ENSURE_EQ(context, filter->type, data_type);
 
   if (hasBias) {
     bias = GetInput(context, node, kBiasTensor);
-    if (data_type == kTfLiteUInt8) {
+    if (data_type == kTfLiteUInt8 || data_type == kTfLiteInt8) {
       TF_LITE_ENSURE_EQ(context, bias->type, kTfLiteInt32);
       TF_LITE_ENSURE_EQ(context, bias->params.zero_point, 0);
     } else {
@@ -150,17 +156,25 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
                      filter_width, out_width);
 
   // Note that quantized inference requires that all tensors have their
-  // parameters set. This is usually done during quantized training.
+  // parameters set. This is usually done during quantized training or
+  // calibration.
   if (data_type != kTfLiteFloat32) {
-    double real_multiplier = 0.0;
-    TF_LITE_ENSURE_STATUS(GetQuantizedConvolutionMultipler(
-        context, input, filter, bias, output, &real_multiplier));
-    int exponent;
-    QuantizeMultiplier(real_multiplier, &data->output_multiplier, &exponent);
-    data->output_shift = -exponent;
-    CalculateActivationRangeUint8(params->activation, output,
-                                  &data->output_activation_min,
-                                  &data->output_activation_max);
+    TF_LITE_ENSURE_EQ(context, filter->quantization.type,
+                      kTfLiteAffineQuantization);
+    const auto* affine_quantization =
+        reinterpret_cast<TfLiteAffineQuantization*>(
+            filter->quantization.params);
+    TF_LITE_ENSURE(context, affine_quantization);
+    TF_LITE_ENSURE(context, affine_quantization->scale);
+    const int number_channel = affine_quantization->scale->size;
+    data->per_channel_output_multiplier.resize(number_channel);
+    data->per_channel_output_shift.resize(number_channel);
+    TF_LITE_ENSURE_STATUS(tflite::PopulateConvolutionQuantizationParams(
+        context, input, filter, bias, output, params->activation,
+        &data->output_multiplier, &data->output_shift,
+        &data->output_activation_min, &data->output_activation_max,
+        data->per_channel_output_multiplier.data(),
+        data->per_channel_output_shift.data()));
   }
 
   TfLiteIntArray* outputSize = TfLiteIntArrayCreate(4);
@@ -250,6 +264,33 @@ void EvalQuantized(TfLiteContext* context, TfLiteNode* node,
                  GetTensorData<uint8_t>(output));
 }
 
+void EvalQuantizedPerChannel(TfLiteContext* context, TfLiteNode* node,
+                             TfLiteDepthwiseConvParams* params, OpData* data,
+                             const TfLiteTensor* input,
+                             const TfLiteTensor* filter,
+                             const TfLiteTensor* bias, TfLiteTensor* output) {
+  DepthwiseParams op_params;
+  op_params.padding_type = PaddingType::kSame;
+  op_params.padding_values.width = data->padding.width;
+  op_params.padding_values.height = data->padding.height;
+  op_params.stride_width = params->stride_width;
+  op_params.stride_height = params->stride_height;
+  op_params.dilation_width_factor = params->dilation_width_factor;
+  op_params.dilation_height_factor = params->dilation_height_factor;
+  op_params.depth_multiplier = params->depth_multiplier;
+  op_params.input_offset = input->params.zero_point;
+  op_params.weights_offset = 0;
+  op_params.output_offset = output->params.zero_point;
+
+  reference_integer_ops::DepthwiseConvPerChannel(
+      op_params, data->per_channel_output_multiplier.data(),
+      data->per_channel_output_shift.data(), GetTensorShape(input),
+      GetTensorData<int8>(input), GetTensorShape(filter),
+      GetTensorData<int8>(filter), GetTensorShape(bias),
+      GetTensorData<int32>(bias), GetTensorShape(output),
+      GetTensorData<int8>(output));
+}
+
 template <KernelType kernel_type>
 TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   auto* params =
@@ -273,6 +314,11 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
       EvalQuantized<kernel_type>(context, node, params, data, input, filter,
                                  bias, output);
       break;
+    case kTfLiteInt8: {
+      EvalQuantizedPerChannel(context, node, params, data, input, filter, bias,
+                              output);
+      break;
+    }
     default:
       context->ReportError(context, "Type %d not currently supported.",
                            input->type);
diff --git a/tensorflow/lite/kernels/depthwise_conv_test.cc b/tensorflow/lite/kernels/depthwise_conv_test.cc
index 75aed4cc4a..5dc513262b 100644
--- a/tensorflow/lite/kernels/depthwise_conv_test.cc
+++ b/tensorflow/lite/kernels/depthwise_conv_test.cc
@@ -56,9 +56,35 @@ class BaseDepthwiseConvolutionOpModel : public SingleOpModel {
       // This is a quantized version. The scale of 'bias' depends on the scales
       // of input and filter. Supposedly this is correctly set during quantized
       // training.
-      auto bias_scale = GetScale(input_) * GetScale(filter_);
-      TensorData bias{TensorType_INT32, {bias_size}, 0, 0, bias_scale};
-      bias_ = AddInput(bias);
+      if (filter.per_channel_quantization) {
+        // per channel quantization.
+        std::vector<float> bias_scale(
+            filter.per_channel_quantization_scales.size());
+        std::vector<int64_t> bias_zero_points(
+            filter.per_channel_quantization_scales.size());
+        for (int i = 0; i < filter.per_channel_quantization_scales.size();
+             ++i) {
+          bias_scale[i] =
+              input.scale * filter.per_channel_quantization_scales[i];
+          bias_zero_points[i] = 0;
+        }
+        TensorData bias{TensorType_INT32,
+                        {bias_size},
+                        /*min=*/0,
+                        /*max=*/0,
+                        /*scale=*/0,
+                        /*zero_point=*/0,
+                        true,
+                        /*per_channel_scale=*/bias_scale,
+                        /*per_channel_zero_point=*/bias_zero_points,
+                        /*channel_index==*/0};
+        bias_ = AddInput(bias);
+      } else {
+        // per tensor quantization.
+        auto bias_scale = GetScale(input_) * GetScale(filter_);
+        TensorData bias{TensorType_INT32, {bias_size}, 0, 0, bias_scale};
+        bias_ = AddInput(bias);
+      }
     }
 
     output_ = AddOutput(output);
@@ -437,6 +463,76 @@ TEST_P(QuantizedDepthwiseConvolutionOpTest, SimpleDilatedTestPaddingSame) {
               ElementsAreArray({4, 7, 3, 6, 10, 4, 2, 3, 1}));
 }
 
+class PerChannelQuantizedDepthwiseConvolutionOpModel
+    : public BaseDepthwiseConvolutionOpModel {
+ public:
+  using BaseDepthwiseConvolutionOpModel::BaseDepthwiseConvolutionOpModel;
+
+  void SetInput(std::initializer_list<float> data) {
+    QuantizeAndPopulate<int8_t>(input_, data);
+  }
+
+  void SetFilter(std::initializer_list<float> data) {
+    PerChannelSymmetricQuantizeAndPopulate(filter_, data);
+  }
+
+  void SetBias(std::initializer_list<float> data) {
+    PerChannelQuantizeBias(bias_, data);
+  }
+
+  std::vector<int8_t> GetOutput() { return ExtractVector<int8_t>(output_); }
+  std::vector<float> GetDequantizedOutput() {
+    return Dequantize<int8_t>(ExtractVector<int8_t>(output_), GetScale(output_),
+                              GetZeroPoint(output_));
+  }
+};
+
+TEST_P(QuantizedDepthwiseConvolutionOpTest, SimpleTest) {
+  PerChannelQuantizedDepthwiseConvolutionOpModel m(
+      GetRegistration(), {TensorType_INT8, {1, 2, 3, 2}, -63.5, 64, 0.5, -1},
+      {TensorType_INT8,
+       // [1 * 2 * 2 * 4] as [input_channel, y, x, output_channel]
+       {1, 2, 2, 4},
+       0,
+       0,
+       0,
+       0,
+       /*per_channel=*/true,
+       /*per_channel_scales=*/{1, 2, 3, 4},
+       /*per_channel_zeros=*/{0, 0, 0, 0},
+       /*channel_index=*/3},
+      {TensorType_INT8, {}, -63.5, 64, 0.5, -1}, Padding_VALID);
+  m.SetInput({
+      // [1 * 2 * 3 * 2] as [batch, y, x, input_channel]
+      3, 2,    // batch = 0, y = 0, x = 0
+      1, -1,   // batch = 0, y = 0, x = 1
+      -2, -3,  // batch = 0, y = 0, x = 2
+      4, 3,    // batch = 0, y = 1, x = 0
+      2, -2,   // batch = 0, y = 1, x = 1
+      -3, -4,  // batch = 0, y = 1, x = 2
+  });
+  m.SetFilter(
+      /*filter data*/
+      {
+          // [1 * 2 * 2 * 4] as [input_channel, y, x, output_channel]
+          // depth multiplier = 2
+          1, 2, 3, 4,  // y = 0, x = 0
+          3, 4, 5, 6,  // y = 0, x = 1
+          7, 8, 5, 6,  // y = 1, x = 0
+          3, 4, 1, 2,  // y = 1, x = 1
+      });
+  m.SetBias({3, -2, 4, 6});
+
+  // Invoke and verify output.
+  // output has dimension [1 * 1 * 2 * 4] as [batch, y, x, output_channel]
+  m.Invoke();
+  EXPECT_THAT(
+      m.GetDequantizedOutput(),
+      ElementsAreArray(ArrayFloatNear({40.5, 48, 27, 40, 0.5, -4, -24, -36})));
+  EXPECT_THAT(m.GetOutput(),
+              ElementsAreArray({80, 95, 53, 79, 0, -9, -49, -73}));
+}
+
 INSTANTIATE_TEST_SUITE_P(
     DepthwiseConvolutionOpTest, DepthwiseConvolutionOpTest,
     ::testing::ValuesIn(SingleOpTest::GetKernelTags(*kKernelMap)));
diff --git a/tensorflow/lite/kernels/internal/BUILD b/tensorflow/lite/kernels/internal/BUILD
index 9d982a89ea..cbe4c9d414 100644
--- a/tensorflow/lite/kernels/internal/BUILD
+++ b/tensorflow/lite/kernels/internal/BUILD
@@ -307,6 +307,7 @@ cc_library(
         "reference/depthwiseconv_uint8.h",
         "reference/fully_connected.h",
         "reference/integer_ops/conv.h",
+        "reference/integer_ops/depthwise_conv.h",
         "reference/integer_ops/dequantize.h",
         "reference/integer_ops/pooling.h",
         "reference/integer_ops/softmax.h",
diff --git a/tensorflow/lite/kernels/internal/reference/integer_ops/depthwise_conv.h b/tensorflow/lite/kernels/internal/reference/integer_ops/depthwise_conv.h
new file mode 100644
index 0000000000..90a7d613a9
--- /dev/null
+++ b/tensorflow/lite/kernels/internal/reference/integer_ops/depthwise_conv.h
@@ -0,0 +1,125 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_DEPTHWISE_CONV_H_
+#define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_DEPTHWISE_CONV_H_
+
+#include "tensorflow/lite/kernels/internal/common.h"
+
+namespace tflite {
+namespace reference_integer_ops {
+inline void DepthwiseConvPerChannel(
+    const DepthwiseParams& params, const int32* output_multiplier,
+    const int32* output_shift, const RuntimeShape& input_shape,
+    const int8* input_data, const RuntimeShape& filter_shape,
+    const int8* filter_data, const RuntimeShape& bias_shape,
+    const int32* bias_data, const RuntimeShape& output_shape,
+    int8* output_data) {
+  // Get parameters.
+  const int stride_width = params.stride_width;
+  const int stride_height = params.stride_height;
+  const int dilation_width_factor = params.dilation_width_factor;
+  const int dilation_height_factor = params.dilation_height_factor;
+  const int pad_width = params.padding_values.width;
+  const int pad_height = params.padding_values.height;
+  const int depth_multiplier = params.depth_multiplier;
+  const int32 input_offset = params.input_offset;
+  const int32 output_offset = params.output_offset;
+
+  // Set min and max value of the output.
+  const int32 output_activation_min = std::numeric_limits<int8_t>::min();
+  const int32 output_activation_max = std::numeric_limits<int8_t>::max();
+
+  // Check dimensions of the tensors.
+  TFLITE_DCHECK_EQ(input_shape.DimensionsCount(), 4);
+  TFLITE_DCHECK_EQ(filter_shape.DimensionsCount(), 4);
+  TFLITE_DCHECK_EQ(output_shape.DimensionsCount(), 4);
+
+  TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
+  const int batches = MatchingDim(input_shape, 0, output_shape, 0);
+  const int output_depth = MatchingDim(filter_shape, 3, output_shape, 3);
+  const int input_height = input_shape.Dims(1);
+  const int input_width = input_shape.Dims(2);
+  const int input_depth = input_shape.Dims(3);
+  const int filter_height = filter_shape.Dims(1);
+  const int filter_width = filter_shape.Dims(2);
+  const int output_height = output_shape.Dims(1);
+  const int output_width = output_shape.Dims(2);
+  TFLITE_DCHECK_EQ(output_depth, input_depth * depth_multiplier);
+  TFLITE_DCHECK_EQ(bias_shape.FlatSize(), output_depth);
+
+  for (int batch = 0; batch < batches; ++batch) {
+    for (int out_y = 0; out_y < output_height; ++out_y) {
+      for (int out_x = 0; out_x < output_width; ++out_x) {
+        for (int in_channel = 0; in_channel < input_depth; ++in_channel) {
+          for (int m = 0; m < depth_multiplier; ++m) {
+            const int output_channel = m + in_channel * depth_multiplier;
+            const int in_x_origin = (out_x * stride_width) - pad_width;
+            const int in_y_origin = (out_y * stride_height) - pad_height;
+            int32 acc = 0;
+            for (int filter_y = 0; filter_y < filter_height; ++filter_y) {
+              for (int filter_x = 0; filter_x < filter_width; ++filter_x) {
+                const int in_x = in_x_origin + dilation_width_factor * filter_x;
+                const int in_y =
+                    in_y_origin + dilation_height_factor * filter_y;
+                // Zero padding by omitting the areas outside the image.
+                const bool is_point_inside_image =
+                    (in_x >= 0) && (in_x < input_width) && (in_y >= 0) &&
+                    (in_y < input_height);
+                if (is_point_inside_image) {
+                  int32 input_val = input_data[Offset(input_shape, batch, in_y,
+                                                      in_x, in_channel)];
+                  int32 filter_val = filter_data[Offset(
+                      filter_shape, 0, filter_y, filter_x, output_channel)];
+                  // Accumulate with 32 bits accumulator.
+                  // In the nudging process during model quantization, we force
+                  // real value of 0.0 be represented by a quantized value. This
+                  // guarentees that the input_offset is a int8, even though it
+                  // is represented using int32.
+                  // int32 += int8 * (int8 - int8) so the highest value we can
+                  // get from each accumulation is [-127, 127] * ([-128, 127] -
+                  // [-128, 127]), which is [-32512, 32512]. log2(32512)
+                  // = 14.98, which means we can accumulate at least 2^16
+                  // multiplications without overflow. The accumulator is
+                  // applied to a filter so the accumation logic will hold as
+                  // long as the filter size (filter_y * filter_x * in_channel)
+                  // does not exceed 2^16, which is the case in all the models
+                  // we have seen so far.
+                  // TODO(jianlijianli): Add a check to make sure the
+                  // accumulator depth is smaller than 2^16.
+                  acc += filter_val * (input_val - input_offset);
+                }
+              }
+            }
+            if (bias_data) {
+              acc += bias_data[output_channel];
+            }
+            acc = MultiplyByQuantizedMultiplier(
+                acc, output_multiplier[output_channel],
+                output_shift[output_channel]);
+            acc += output_offset;
+            acc = std::max(acc, output_activation_min);
+            acc = std::min(acc, output_activation_max);
+            output_data[Offset(output_shape, batch, out_y, out_x,
+                               output_channel)] = static_cast<int8_t>(acc);
+          }
+        }
+      }
+    }
+  }
+}
+}  // namespace reference_integer_ops
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_DEPTHWISE_CONV_H_
-- 
GitLab


From ee2929ecab4cab0e2be6de5d03738a6642fc2916 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 30 Jan 2019 22:20:33 -0800
Subject: [PATCH 1652/2345] Introduce a switch to disable depthwise backprop
 filter optimizations.

PiperOrigin-RevId: 231728114
---
 tensorflow/compiler/xla/service/convolution_group_converter.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/convolution_group_converter.cc b/tensorflow/compiler/xla/service/convolution_group_converter.cc
index 2ef723febd..f11f9e5fc2 100644
--- a/tensorflow/compiler/xla/service/convolution_group_converter.cc
+++ b/tensorflow/compiler/xla/service/convolution_group_converter.cc
@@ -223,7 +223,7 @@ Status ConvolutionVisitor::HandleBatchGroupCount(HloInstruction* convolution) {
   // We are not yet supporting batch_group of sizes greater than 1.
   TF_RET_CHECK(input_batch == batch_group_count);
 
-  if (!is_cost_viable_(convolution)) {
+  if (!is_cost_viable_(convolution) || filter_expansion_) {
     // We first obtain the expanded the filter (which is the convolution
     // output). The batch dimension is the expanded one (which originally
     // represents kernel input feature dimension). We mask the filter to zero
-- 
GitLab


From f77d3d94cc48948bc2ce3f28bf1e3fae396b1458 Mon Sep 17 00:00:00 2001
From: Suharsh Sivakumar <suharshs@google.com>
Date: Wed, 30 Jan 2019 23:05:06 -0800
Subject: [PATCH 1653/2345] Update version of int8 Conv and Dequant.

Also fix missing version bump of Softmax.

PiperOrigin-RevId: 231731398
---
 tensorflow/lite/kernels/register.cc     | 12 ++++++++----
 tensorflow/lite/toco/tflite/operator.cc | 18 ++++++++++++++++++
 2 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index fcfe0b25bb..fdc7ef3b9a 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -167,15 +167,17 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_RELU6, Register_RELU6());
   AddBuiltin(BuiltinOperator_TANH, Register_TANH());
   AddBuiltin(BuiltinOperator_LOGISTIC, Register_LOGISTIC());
-  AddBuiltin(BuiltinOperator_AVERAGE_POOL_2D, Register_AVERAGE_POOL_2D());
+  AddBuiltin(BuiltinOperator_AVERAGE_POOL_2D, Register_AVERAGE_POOL_2D(),
+             /* min_version */ 1,
+             /* max_version */ 2);
   AddBuiltin(BuiltinOperator_MAX_POOL_2D, Register_MAX_POOL_2D());
   AddBuiltin(BuiltinOperator_L2_POOL_2D, Register_L2_POOL_2D());
   AddBuiltin(BuiltinOperator_CONV_2D, Register_CONV_2D(),
              /* min_version */ 1,
-             /* max_version */ 2);
+             /* max_version */ 3);
   AddBuiltin(BuiltinOperator_DEPTHWISE_CONV_2D, Register_DEPTHWISE_CONV_2D(),
              /* min_version */ 1,
-             /* max_version */ 2);
+             /* max_version */ 3);
   AddBuiltin(BuiltinOperator_SVDF, Register_SVDF(),
              /* min_version */ 1,
              /* max_version */ 2);
@@ -200,7 +202,9 @@ BuiltinOpResolver::BuiltinOpResolver() {
              /* max_version */ 3);
   AddBuiltin(BuiltinOperator_LSH_PROJECTION, Register_LSH_PROJECTION());
   AddBuiltin(BuiltinOperator_HASHTABLE_LOOKUP, Register_HASHTABLE_LOOKUP());
-  AddBuiltin(BuiltinOperator_SOFTMAX, Register_SOFTMAX());
+  AddBuiltin(BuiltinOperator_SOFTMAX, Register_SOFTMAX(),
+             /* min_version */ 1,
+             /* max_version */ 2);
   AddBuiltin(BuiltinOperator_CONCATENATION, Register_CONCATENATION());
   AddBuiltin(BuiltinOperator_ADD, Register_ADD());
   AddBuiltin(BuiltinOperator_SPACE_TO_BATCH_ND, Register_SPACE_TO_BATCH_ND());
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index f61488ef35..61c0b605dd 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -108,6 +108,12 @@ class Convolution
     const Array& input_array = op_signature.model->GetArray(input_name);
     const Array& filter_array = op_signature.model->GetArray(filter_name);
     const Array& output_array = op_signature.model->GetArray(output_name);
+    // If the op has signed int8 inputs and outputs, its version 3.
+    if (input_array.data_type == ArrayDataType::kInt8 &&
+        filter_array.data_type == ArrayDataType::kInt8 &&
+        output_array.data_type == ArrayDataType::kInt8) {
+      return 3;
+    }
     // If the op is a signed int8 hybrid operation, we need to return
     // version 2.
     if (input_array.data_type == ArrayDataType::kFloat &&
@@ -153,6 +159,18 @@ class DepthwiseConvolution
   int GetVersion(const OperatorSignature& op_signature) const override {
     const auto& conv_op =
         static_cast<const DepthwiseConvOperator&>(*op_signature.op);
+    const string& input_name = op_signature.op->inputs[0];
+    const string& filter_name = op_signature.op->inputs[1];
+    const string& output_name = op_signature.op->outputs[0];
+    const Array& input_array = op_signature.model->GetArray(input_name);
+    const Array& filter_array = op_signature.model->GetArray(filter_name);
+    const Array& output_array = op_signature.model->GetArray(output_name);
+    // If the op has signed int8 inputs and outputs, its version 3.
+    if (input_array.data_type == ArrayDataType::kInt8 &&
+        filter_array.data_type == ArrayDataType::kInt8 &&
+        output_array.data_type == ArrayDataType::kInt8) {
+      return 3;
+    }
     if (conv_op.dilation_width_factor != 1 ||
         conv_op.dilation_height_factor != 1) {
       return 2;
-- 
GitLab


From 0a4a1c803f2456d3ba8df837a746f4d1b7963961 Mon Sep 17 00:00:00 2001
From: Rick Chao <rchao@google.com>
Date: Wed, 30 Jan 2019 23:40:10 -0800
Subject: [PATCH 1654/2345] Doc updates for distribute_coordinator.py

PiperOrigin-RevId: 231733825
---
 .../python/distribute/distribute_coordinator.py      | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/distribute/distribute_coordinator.py b/tensorflow/python/distribute/distribute_coordinator.py
index f1d7d1debd..49dfe8db35 100644
--- a/tensorflow/python/distribute/distribute_coordinator.py
+++ b/tensorflow/python/distribute/distribute_coordinator.py
@@ -648,7 +648,7 @@ def run_distribute_coordinator(worker_fn,
   for a task. The distribute coordinator will make a copy of the `strategy`
   object, call its `configure` method and pass it to `worker_fn` as an argument.
 
-  The `worker_fn` defines the training logic and is called under a its own
+  The `worker_fn` defines the training logic and is called under its own
   worker context which can be accessed to via `get_current_worker_context`. A
   worker context provides access to configurations for each task, e.g. the
   task_type, task_id, master target and so on. Since `worker_fn` will be called
@@ -674,7 +674,7 @@ def run_distribute_coordinator(worker_fn,
   the worker context.
 
   The `cluster_spec` can be either passed by the argument or parsed from the
-  "TF_CONFIG" envrionment variable. Example of a TF_CONFIG:
+  "TF_CONFIG" environment variable. Example of a TF_CONFIG:
   ```
     cluster = {'chief': ['host0:2222'],
                'ps': ['host1:2222', 'host2:2222'],
@@ -689,19 +689,19 @@ def run_distribute_coordinator(worker_fn,
   will be created to call `eval_fn` with its `task_type` set to "evaluator". If
   `eval_fn` is not defined, fall back to `worker_fn`. This implies that
   evaluation will be done on a single machine if there is an "evaluator" task.
-  If "evaluator" doesn't exit in the cluster_spec, it entirely depends on the
+  If "evaluator" doesn't exist in the cluster_spec, it entirely depends on the
   `worker_fn` for how to do evaluation.
 
   Args:
     worker_fn: the function to be called. The function should accept a
       `strategy` object and will be given access to a context object via a
       context manager scope.
-    strategy: a DistributionStrategy object which specifying whether it should
+    strategy: a DistributionStrategy object specifying whether it should
       run between-graph replicated training or not, whether to run init ops,
       etc. This object will also be configured given `session_config`,
       `cluster_spec`, `task_type` and `task_id`.
     eval_fn: optional function for "evaluator" task. If `eval_fn` is not passed
-      in but a "evaluator" task found in the `cluster_spec`, the `worker_fn`
+      in but a "evaluator" task is found in the `cluster_spec`, the `worker_fn`
       will be used for this task.
     eval_strategy: optional DistributionStrategy object for "evaluator" task.
     mode: in which mode this distribute coordinator runs.
@@ -736,7 +736,7 @@ def run_distribute_coordinator(worker_fn,
   rpc_layer = tf_config.get("rpc_layer", rpc_layer)
   environment = tf_config.get("environment", None)
 
-  # Setting the session config is necessary for some strategies such
+  # Setting the session config is necessary for some strategies such as
   # CollectiveAllReduceStrategy.
   session_config = session_config or config_pb2.ConfigProto(
       allow_soft_placement=True)
-- 
GitLab


From 31da2ab1401b933e031a91aac2aa474e4099aa58 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 31 Jan 2019 01:02:22 -0800
Subject: [PATCH 1655/2345] compat: Update forward compatibility horizon to
 2019-01-31

PiperOrigin-RevId: 231742820
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 4f84bcba37..44920a46cb 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 30)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 31)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From bbe8343f3ef8534506cea4b064d127734a9f55fb Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 31 Jan 2019 02:53:37 -0800
Subject: [PATCH 1656/2345] Support parsing short form constants in the HLO
 parser

A short HLO text omits the value for large constants for bravity what
made the HLO text unparseble. This CL adds support for this short form
by creating 0 valued constants what is still useful for most debugging
tools.

PiperOrigin-RevId: 231754684
---
 tensorflow/compiler/xla/service/hlo_lexer.cc       | 11 +++++++++++
 tensorflow/compiler/xla/service/hlo_lexer.h        |  1 +
 tensorflow/compiler/xla/service/hlo_parser.cc      | 11 +++++++++++
 tensorflow/compiler/xla/service/hlo_parser_test.cc | 13 +++++++++++++
 4 files changed, 36 insertions(+)

diff --git a/tensorflow/compiler/xla/service/hlo_lexer.cc b/tensorflow/compiler/xla/service/hlo_lexer.cc
index 798760885d..c1a642dfea 100644
--- a/tensorflow/compiler/xla/service/hlo_lexer.cc
+++ b/tensorflow/compiler/xla/service/hlo_lexer.cc
@@ -212,6 +212,15 @@ TokKind HloLexer::LexToken() {
         // A lone '/' is an error.
         return TokKind::kError;
       }
+      case '.':
+        if (PeekCurrentChar() == '.') {
+          current_ptr_++;
+          if (PeekCurrentChar() == '.') {
+            current_ptr_++;
+            return TokKind::kDots;
+          }
+        }
+        return TokKind::kError;
       case '"':
         return LexString();
     }
@@ -513,6 +522,8 @@ string TokKindToString(TokKind kind) {
       return "kInt";
     case TokKind::kDecimal:
       return "kDecimal";
+    case TokKind::kDots:
+      return "kDots";
   }
 }
 
diff --git a/tensorflow/compiler/xla/service/hlo_lexer.h b/tensorflow/compiler/xla/service/hlo_lexer.h
index 94fac3cd8e..16eed21617 100644
--- a/tensorflow/compiler/xla/service/hlo_lexer.h
+++ b/tensorflow/compiler/xla/service/hlo_lexer.h
@@ -47,6 +47,7 @@ enum class TokKind {
   kRbrace,  // {  }
   kLparen,
   kRparen,  // (  )
+  kDots,    // ...
 
   kArrow,  // ->
   kLeq,    // <=
diff --git a/tensorflow/compiler/xla/service/hlo_parser.cc b/tensorflow/compiler/xla/service/hlo_parser.cc
index d7fa6b4a64..dd8e8ff3a5 100644
--- a/tensorflow/compiler/xla/service/hlo_parser.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser.cc
@@ -28,6 +28,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/primitive_util.h"
 #include "tensorflow/compiler/xla/service/hlo_domain_metadata.h"
 #include "tensorflow/compiler/xla/service/hlo_instructions.h"
+#include "tensorflow/compiler/xla/service/hlo_lexer.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
 #include "tensorflow/compiler/xla/service/hlo_schedule.h"
 #include "tensorflow/compiler/xla/service/hlo_sharding_metadata.h"
@@ -2150,6 +2151,16 @@ bool HloParser::ParseDenseLiteral(Literal* literal, const Shape& shape) {
         }
         break;
       }
+      case TokKind::kDots: {
+        if (nest_level != 1) {
+          return TokenError(absl::StrFormat(
+              "expects `...` at nest level 1, but sees it at nest level %d",
+              nest_level));
+        }
+        elems_seen_per_dim[0] = shape.dimensions(0);
+        lexer_.Lex();
+        break;
+      }
       case TokKind::kComma:
         // Skip.
         lexer_.Lex();
diff --git a/tensorflow/compiler/xla/service/hlo_parser_test.cc b/tensorflow/compiler/xla/service/hlo_parser_test.cc
index 6eee767900..b525f66f9b 100644
--- a/tensorflow/compiler/xla/service/hlo_parser_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser_test.cc
@@ -1719,6 +1719,19 @@ ENTRY %ConstantWithExp.v4 () -> f32[] {
   // printed as "300".
 }
 
+TEST_F(HloParserTest, ShortConstant) {
+  const string original = R"(HloModule ShortCOnstant_module
+
+ENTRY %ShortConstant.v4 () -> f32[67,89] {
+  ROOT %constant.1 = f32[67,89]{1,0} constant({...})
+}
+
+)";
+  auto result = ParseHloString(original);
+  TF_EXPECT_OK(result.status());
+  EXPECT_EQ(result.ValueOrDie()->ToString(HloPrintOptions()), original);
+}
+
 TEST_F(HloParserTest, AttibutesAnyOrder) {
   const string original = R"(HloModule any_order_module
 
-- 
GitLab


From d3d6fed6ae2e01e14ec8cd7e5e61f651c0b4c173 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Thu, 31 Jan 2019 06:18:48 -0800
Subject: [PATCH 1657/2345] Don't use the typed visitor for Sort in the
 Evaluator.

Add support to the untyped visitor for the types which were
previously only supported by the typed visitor.
This allows to get rid of a lot of duplicate code.

PiperOrigin-RevId: 231775060
---
 .../compiler/xla/service/hlo_evaluator.cc     | 239 +++++++++++-------
 .../xla/service/hlo_evaluator_typed_visitor.h |  67 +----
 2 files changed, 142 insertions(+), 164 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index 29e41433f4..803baf3564 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -1410,108 +1410,151 @@ StatusOr<Literal> ExtractFromIndexPositions(const Literal& from,
 }  // namespace
 
 Status HloEvaluator::HandleSort(HloInstruction* sort) {
-  if (!sort->shape().IsTuple()) {
-    return DefaultAction(sort);
-  } else {
-    TF_RET_CHECK(sort->operand_count() >= 2) << "Expected key-value sort";
-    for (int64 i = 1; i < sort->operand_count(); ++i) {
-      TF_RET_CHECK(ShapeUtil::SameDimensions(sort->operand(0)->shape(),
-                                             sort->operand(i)->shape()))
-          << "All Sort operands must have the same dimensions";
-    }
+  TF_RET_CHECK(sort->operand_count() >= 1)
+      << "Expected at least 1 operand for sort";
+  for (int64 i = 1; i < sort->operand_count(); ++i) {
+    TF_RET_CHECK(ShapeUtil::SameDimensions(sort->operand(0)->shape(),
+                                           sort->operand(i)->shape()))
+        << "All Sort operands must have the same dimensions";
+  }
 
-    if (VLOG_IS_ON(3)) {
-      for (int64 i = 0; i < sort->operand_count(); ++i) {
-        VLOG(3) << "HandleSort operand " << i << " literal: "
-                << GetEvaluatedLiteralFor(sort->operand(i)).ToString();
-      }
-    }
-    Shape key_shape = sort->operand(0)->shape();
-    auto rank = key_shape.rank();
-    PrimitiveType keys_type = key_shape.element_type();
-    if (keys_type != F32 && keys_type != U32 && keys_type != S32 &&
-        keys_type != BF16) {
-      return InvalidArgument("Unsupported type for Sort: %s",
-                             PrimitiveType_Name(keys_type));
-    }
-    std::vector<Literal> result_literals;
-    result_literals.reserve(sort->operand_count());
+  if (VLOG_IS_ON(3)) {
     for (int64 i = 0; i < sort->operand_count(); ++i) {
-      result_literals.emplace_back(sort->operand(i)->shape());
+      VLOG(3) << "HandleSort operand " << i << " literal: "
+              << GetEvaluatedLiteralFor(sort->operand(i)).ToString();
     }
-    std::vector<int64> zero_base(rank, 0);
-    std::vector<int64> increment(rank, 1);
-    int64 sort_dim = sort->dimensions(0);
-    int64 sort_dim_elements = key_shape.dimensions(sort_dim);
-    increment[sort_dim] = sort_dim_elements;
-    // Iterate through each dimension except 'sort_dim'.
-    TF_RETURN_IF_ERROR(ShapeUtil::ForEachIndexWithStatus(
-        key_shape, zero_base, AsInt64Slice(key_shape.dimensions()), increment,
-        [&](absl::Span<const int64> indices) -> StatusOr<bool> {
-          // Extract a slice from each operand literal that corresponds to
-          // exactly the row in dimension 'sort_dim'.
-          std::vector<int64> limit_indices(indices.begin(), indices.end());
-          absl::c_for_each(limit_indices, [](int64& index) { ++index; });
-          limit_indices[sort_dim] = sort_dim_elements;
-          std::vector<Literal> literals_to_sort;
-          literals_to_sort.reserve(sort->operand_count());
-          for (int64 i = 0; i < sort->operand_count(); ++i) {
-            TF_ASSIGN_OR_RETURN(auto literal_to_sort,
-                                GetEvaluatedLiteralFor(sort->operand(i))
-                                    .Slice(indices, limit_indices)
-                                    .Reshape({sort_dim_elements}));
-            literals_to_sort.push_back(std::move(literal_to_sort));
-          }
-          std::vector<int64> indices_to_sort(sort_dim_elements);
-          std::iota(indices_to_sort.begin(), indices_to_sort.end(), 0);
-          std::stable_sort(
-              indices_to_sort.begin(), indices_to_sort.end(),
-              [keys_type, &literals_to_sort](int64 a, int64 b) {
-                switch (keys_type) {
-                  case F32: {
-                    auto key_lhs = literals_to_sort[0].Get<float>({a});
-                    auto key_rhs = literals_to_sort[0].Get<float>({b});
-                    return SafeLess(key_lhs, key_rhs);
-                  }
-                  case U32: {
-                    auto key_lhs = literals_to_sort[0].Get<uint32>({a});
-                    auto key_rhs = literals_to_sort[0].Get<uint32>({b});
-                    return SafeLess(key_lhs, key_rhs);
-                  }
-                  case S32: {
-                    auto key_lhs = literals_to_sort[0].Get<int32>({a});
-                    auto key_rhs = literals_to_sort[0].Get<int32>({b});
-                    return SafeLess(key_lhs, key_rhs);
-                  }
-                  case BF16: {
-                    auto key_lhs = literals_to_sort[0].Get<bfloat16>({a});
-                    auto key_rhs = literals_to_sort[0].Get<bfloat16>({b});
-                    return SafeLess(key_lhs, key_rhs);
-                  }
-                  default:
-                    // We should never reach here, because we checked earlier
-                    // that 'key_type' is one of the cases above.
-                    LOG(FATAL) << "Invalid key type in Sort: %s",
-                        PrimitiveType_Name(keys_type);
-                    return false;
+  }
+  Shape key_shape = sort->operand(0)->shape();
+  auto rank = key_shape.rank();
+  PrimitiveType keys_type = key_shape.element_type();
+  if (keys_type != F64 && keys_type != U64 && keys_type != S64 &&
+      keys_type != F32 && keys_type != U32 && keys_type != S32 &&
+      keys_type != BF16 && keys_type != F16 && keys_type != U16 &&
+      keys_type != S16 && keys_type != U8 && keys_type != S8) {
+    return InvalidArgument("Unsupported type for Sort: %s",
+                           PrimitiveType_Name(keys_type));
+  }
+  std::vector<Literal> result_literals;
+  result_literals.reserve(sort->operand_count());
+  for (int64 i = 0; i < sort->operand_count(); ++i) {
+    result_literals.emplace_back(sort->operand(i)->shape());
+  }
+  std::vector<int64> zero_base(rank, 0);
+  std::vector<int64> increment(rank, 1);
+  int64 sort_dim = sort->dimensions(0);
+  int64 sort_dim_elements = key_shape.dimensions(sort_dim);
+  increment[sort_dim] = sort_dim_elements;
+  // Iterate through each dimension except 'sort_dim'.
+  TF_RETURN_IF_ERROR(ShapeUtil::ForEachIndexWithStatus(
+      key_shape, zero_base, AsInt64Slice(key_shape.dimensions()), increment,
+      [&](absl::Span<const int64> indices) -> StatusOr<bool> {
+        // Extract a slice from each operand literal that corresponds to
+        // exactly the row in dimension 'sort_dim'.
+        std::vector<int64> limit_indices(indices.begin(), indices.end());
+        absl::c_for_each(limit_indices, [](int64& index) { ++index; });
+        limit_indices[sort_dim] = sort_dim_elements;
+        std::vector<Literal> literals_to_sort;
+        literals_to_sort.reserve(sort->operand_count());
+        for (int64 i = 0; i < sort->operand_count(); ++i) {
+          TF_ASSIGN_OR_RETURN(auto literal_to_sort,
+                              GetEvaluatedLiteralFor(sort->operand(i))
+                                  .Slice(indices, limit_indices)
+                                  .Reshape({sort_dim_elements}));
+          literals_to_sort.push_back(std::move(literal_to_sort));
+        }
+        std::vector<int64> indices_to_sort(sort_dim_elements);
+        std::iota(indices_to_sort.begin(), indices_to_sort.end(), 0);
+        std::stable_sort(
+            indices_to_sort.begin(), indices_to_sort.end(),
+            [keys_type, &literals_to_sort](int64 a, int64 b) {
+              switch (keys_type) {
+                case F64: {
+                  auto key_lhs = literals_to_sort[0].Get<double>({a});
+                  auto key_rhs = literals_to_sort[0].Get<double>({b});
+                  return SafeLess(key_lhs, key_rhs);
+                }
+                case U64: {
+                  auto key_lhs = literals_to_sort[0].Get<uint64>({a});
+                  auto key_rhs = literals_to_sort[0].Get<uint64>({b});
+                  return SafeLess(key_lhs, key_rhs);
+                }
+                case S64: {
+                  auto key_lhs = literals_to_sort[0].Get<int64>({a});
+                  auto key_rhs = literals_to_sort[0].Get<int64>({b});
+                  return SafeLess(key_lhs, key_rhs);
                 }
-              });
-          std::vector<int64> slice_dimensions(rank, 1);
-          slice_dimensions[sort_dim] = sort_dim_elements;
-          std::vector<int64> start_indices(rank, 0);
-          for (int64 i = 0; i < sort->operand_count(); ++i) {
-            TF_ASSIGN_OR_RETURN(Literal sorted_literal,
-                                ExtractFromIndexPositions(literals_to_sort[i],
-                                                          indices_to_sort));
-            TF_ASSIGN_OR_RETURN(auto sorted_literal_reshaped,
-                                sorted_literal.Reshape(slice_dimensions));
-            TF_RETURN_IF_ERROR(result_literals[i].CopySliceFrom(
-                sorted_literal_reshaped, start_indices, indices,
-                slice_dimensions));
-          }
-          return true;
-        }));
+                case F32: {
+                  auto key_lhs = literals_to_sort[0].Get<float>({a});
+                  auto key_rhs = literals_to_sort[0].Get<float>({b});
+                  return SafeLess(key_lhs, key_rhs);
+                }
+                case U32: {
+                  auto key_lhs = literals_to_sort[0].Get<uint32>({a});
+                  auto key_rhs = literals_to_sort[0].Get<uint32>({b});
+                  return SafeLess(key_lhs, key_rhs);
+                }
+                case S32: {
+                  auto key_lhs = literals_to_sort[0].Get<int32>({a});
+                  auto key_rhs = literals_to_sort[0].Get<int32>({b});
+                  return SafeLess(key_lhs, key_rhs);
+                }
+                case BF16: {
+                  auto key_lhs = literals_to_sort[0].Get<bfloat16>({a});
+                  auto key_rhs = literals_to_sort[0].Get<bfloat16>({b});
+                  return SafeLess(key_lhs, key_rhs);
+                }
+                case F16: {
+                  auto key_lhs = literals_to_sort[0].Get<Eigen::half>({a});
+                  auto key_rhs = literals_to_sort[0].Get<Eigen::half>({b});
+                  return SafeLess(key_lhs, key_rhs);
+                }
+                case U16: {
+                  auto key_lhs = literals_to_sort[0].Get<uint16>({a});
+                  auto key_rhs = literals_to_sort[0].Get<uint16>({b});
+                  return SafeLess(key_lhs, key_rhs);
+                }
+                case S16: {
+                  auto key_lhs = literals_to_sort[0].Get<int16>({a});
+                  auto key_rhs = literals_to_sort[0].Get<int16>({b});
+                  return SafeLess(key_lhs, key_rhs);
+                }
+                case U8: {
+                  auto key_lhs = literals_to_sort[0].Get<uint8>({a});
+                  auto key_rhs = literals_to_sort[0].Get<uint8>({b});
+                  return SafeLess(key_lhs, key_rhs);
+                }
+                case S8: {
+                  auto key_lhs = literals_to_sort[0].Get<int8>({a});
+                  auto key_rhs = literals_to_sort[0].Get<int8>({b});
+                  return SafeLess(key_lhs, key_rhs);
+                }
+                default:
+                  // We should never reach here, because we checked earlier
+                  // that 'key_type' is one of the cases above.
+                  LOG(FATAL) << "Invalid key type in Sort: %s",
+                      PrimitiveType_Name(keys_type);
+                  return false;
+              }
+            });
+        std::vector<int64> slice_dimensions(rank, 1);
+        slice_dimensions[sort_dim] = sort_dim_elements;
+        std::vector<int64> start_indices(rank, 0);
+        for (int64 i = 0; i < sort->operand_count(); ++i) {
+          TF_ASSIGN_OR_RETURN(
+              Literal sorted_literal,
+              ExtractFromIndexPositions(literals_to_sort[i], indices_to_sort));
+          TF_ASSIGN_OR_RETURN(auto sorted_literal_reshaped,
+                              sorted_literal.Reshape(slice_dimensions));
+          TF_RETURN_IF_ERROR(result_literals[i].CopySliceFrom(
+              sorted_literal_reshaped, start_indices, indices,
+              slice_dimensions));
+        }
+        return true;
+      }));
 
+  if (sort->operand_count() == 1) {
+    evaluated_[sort] = std::move(result_literals[0]);
+  } else {
     std::vector<const Literal*> literal_ptrs;
     absl::c_transform(result_literals, std::back_inserter(literal_ptrs),
                       [](const Literal& literal) { return &literal; });
@@ -1520,8 +1563,8 @@ Status HloEvaluator::HandleSort(HloInstruction* sort) {
     VLOG(3) << "HandleSort result_tuple: " << result_tuple.ToString();
 
     evaluated_[sort] = std::move(result_tuple);
-    return Status::OK();
   }
+  return Status::OK();
 }
 
 Status HloEvaluator::HandleReduce(HloInstruction* reduce) {
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
index 648c7d0e67..742a389ed0 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
@@ -1662,73 +1662,8 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
     return Status::OK();
   }
 
-  template <typename NativeT,
-            typename std::enable_if<
-                !is_complex_t<NativeT>::value &&
-                !std::is_same<NativeT, bool>::value>::type* = nullptr>
-  Status HandleSort(HloInstruction* sort) {
-    auto keys = sort->operand(0);
-    TF_RET_CHECK(sort->operand_count() == 1)
-        << "Typed visitor does not support key-value sort";
-
-    const Literal& keys_literal = parent_->GetEvaluatedLiteralFor(keys);
-    int64 sort_dim = sort->dimensions(0);
-    int64 sort_dim_elements = keys->shape().dimensions(sort_dim);
-    int64 rank = keys->shape().rank();
-    if (rank == 0) {
-      // Nothing to sort.
-      parent_->evaluated_[sort] = keys_literal.Clone();
-      return Status::OK();
-    }
-    Literal result_literal(keys_literal.shape());
-    std::vector<int64> zero_base(rank, 0);
-    std::vector<int64> increment(rank, 1);
-    increment[sort_dim] = sort_dim_elements;
-    // Iterate through each dimension except 'sort_dim'.
-    TF_RETURN_IF_ERROR(ShapeUtil::ForEachIndexWithStatus(
-        keys->shape(), zero_base, AsInt64Slice(keys->shape().dimensions()),
-        increment, [&](absl::Span<const int64> indices) -> StatusOr<bool> {
-          // Extract a slice from the literal that corresponds to exactly the
-          // row in dimension 'sort_dim'.
-          std::vector<int64> limit_indices(indices.begin(), indices.end());
-          absl::c_for_each(limit_indices, [](int64& index) { ++index; });
-          limit_indices[sort_dim] = sort_dim_elements;
-          TF_ASSIGN_OR_RETURN(auto row_to_sort,
-                              keys_literal.Slice(indices, limit_indices)
-                                  .Reshape({sort_dim_elements}));
-          const auto& row_data = row_to_sort.data<NativeT>();
-
-          std::vector<NativeT> result_data(row_data.begin(), row_data.end());
-          std::stable_sort(result_data.begin(), result_data.end(),
-                           [](const NativeT& a, const NativeT& b) {
-                             return SafeLess<NativeT>(a, b);
-                           });
-          Literal sorted_row(ShapeUtil::MakeShape(keys->shape().element_type(),
-                                                  {sort_dim_elements}));
-          sorted_row.PopulateR1(absl::Span<const NativeT>(result_data));
-          std::vector<int64> slice_dimensions(rank, 1);
-          slice_dimensions[sort_dim] = sort_dim_elements;
-          TF_ASSIGN_OR_RETURN(auto sorted_row_reshaped,
-                              sorted_row.Reshape(slice_dimensions));
-          std::vector<int64> start_indices(rank, 0);
-          TF_RETURN_IF_ERROR(result_literal.CopySliceFrom(
-              sorted_row_reshaped, start_indices, indices, slice_dimensions));
-          return true;
-        }));
-    parent_->evaluated_[sort] = std::move(result_literal);
-    return Status::OK();
-  }
-
-  template <typename NativeT,
-            typename std::enable_if<is_complex_t<NativeT>::value ||
-                                    std::is_same<NativeT, bool>::value>::type* =
-                nullptr>
-  Status HandleSort(HloInstruction* sort) {
-    return UnsupportedTypeError(sort);
-  }
-
   Status HandleSort(HloInstruction* sort) override {
-    return HandleSort<ReturnT>(sort);
+    return UnsupportedTypeError(sort);
   }
 
   Status HandleReduce(HloInstruction* hlo) override {
-- 
GitLab


From 0740fdd9abb00e4bb39a1e91aab3e39277c11ac5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 31 Jan 2019 08:15:02 -0800
Subject: [PATCH 1658/2345] Add $(location) in build_def, because bazel
 requires it

PiperOrigin-RevId: 231788924
---
 tensorflow/lite/build_def.bzl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index f4c9e4ea04..f35bc40c74 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -157,7 +157,7 @@ def tf_to_tflite(name, src, options, out):
     """
 
     toco_cmdline = " ".join([
-        "//tensorflow/lite/toco:toco",
+        "$(location //tensorflow/lite/toco:toco)",
         "--input_format=TENSORFLOW_GRAPHDEF",
         "--output_format=TFLITE",
         ("--input_file=$(location %s)" % src),
-- 
GitLab


From 451fc0ba9689ea57ba4c1d86ef2183d68433628d Mon Sep 17 00:00:00 2001
From: Jeff Poznanovic <jeffrey.poznanovic@amd.com>
Date: Thu, 31 Jan 2019 09:51:41 -0700
Subject: [PATCH 1659/2345] Make install_deb_packages.sh platform independent

install_deb_packages.sh currently installs platform dependent packages.  These should go into the platform-specific Dockerfiles instead, like this relevant commit to Dockerfile.gpu:
https://github.com/tensorflow/tensorflow/commit/03ec3f7b11572b70ef7f33f2a9af9bebbb6ce473
---
 tensorflow/tools/ci_build/install/install_deb_packages.sh | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/tensorflow/tools/ci_build/install/install_deb_packages.sh b/tensorflow/tools/ci_build/install/install_deb_packages.sh
index 989f2a92eb..bd810016d2 100755
--- a/tensorflow/tools/ci_build/install/install_deb_packages.sh
+++ b/tensorflow/tools/ci_build/install/install_deb_packages.sh
@@ -68,12 +68,6 @@ apt-get install -y --no-install-recommends \
     zip \
     zlib1g-dev
 
-apt-get update && \
-  apt-get install nvinfer-runtime-trt-repo-ubuntu1604-4.0.1-ga-cuda9.0 && \
-  apt-get update && \
-  apt-get install libnvinfer4=4.1.2-1+cuda9.0 && \
-  apt-get install libnvinfer-dev=4.1.2-1+cuda9.0
-
 # populate the database
 updatedb
 
-- 
GitLab


From 9a4c3a2eaaa1ca9a6a7d5bdb97c59814afa3c98b Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Thu, 31 Jan 2019 09:33:55 -0800
Subject: [PATCH 1660/2345] Support returning IndexedSlices in while_v2
 gradients.

PiperOrigin-RevId: 231800617
---
 tensorflow/python/BUILD                       |   1 +
 .../kernel_tests/control_flow_ops_py_test.py  |  79 +++++
 tensorflow/python/ops/while_v2.py             |  80 +++--
 .../ops/while_v2_indexed_slices_rewriter.py   | 279 ++++++++++++++++++
 4 files changed, 417 insertions(+), 22 deletions(-)
 create mode 100644 tensorflow/python/ops/while_v2_indexed_slices_rewriter.py

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 109beace58..d550dca0b1 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -2277,6 +2277,7 @@ py_library(
     name = "while_v2",
     srcs = [
         "ops/while_v2.py",
+        "ops/while_v2_indexed_slices_rewriter.py",
     ],
     srcs_version = "PY2AND3",
     deps = [
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index d8f91034cc..25fc42b60d 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -2518,6 +2518,85 @@ class ControlFlowTest(test.TestCase):
       self.evaluate(variables.global_variables_initializer())
       self.assertAllClose(216.0, g[0])
 
+  @test_util.run_deprecated_v1
+  def testWhileGrad_ResourceVarSparseRead(self):
+    # NOTE(skyewm): this test is interesting because the
+    # ResourceVariable.sparse_read gradient function returns an IndexedSlices.
+    var = resource_variable_ops.ResourceVariable(np.ones(5),
+                                                 dtype=dtypes.float32)
+    r = control_flow_ops.while_loop(
+        lambda i, _: i < 3,
+        lambda i, x: (i + 1, x * math_ops.reduce_sum(var.sparse_read([1, 3]))),
+        [0, constant_op.constant(1.0)])[1]
+    grad = gradients_impl.gradients(r, var)[0]
+
+    self.evaluate(variables.global_variables_initializer())
+    grad_val = self.evaluate(grad)
+    self.assertIsInstance(grad_val, ops.IndexedSlicesValue)
+    arr = gradient_checker_v2._to_numpy(grad_val)
+    self.assertAllEqual(arr, [0., 12., 0., 12., 0.])
+
+  @test_util.run_deprecated_v1
+  def testWhileGrad_MultiResourceVarSparseRead(self):
+    # NOTE(skyewm): this test is interesting because the
+    # ResourceVariable.sparse_read gradient function returns an IndexedSlices.
+    var1 = resource_variable_ops.ResourceVariable(np.ones(5),
+                                                  dtype=dtypes.float32)
+    var2 = resource_variable_ops.ResourceVariable(np.ones(3),
+                                                  dtype=dtypes.float32)
+    x1_init = constant_op.constant([0., 0.])
+    x2_init = constant_op.constant(1.)
+    x3_init = constant_op.constant(1.)
+
+    def body(i, unused_x1, x2, x3):
+      y1 = var1.sparse_read([1, 3])
+      y2 = x2 * 2
+      y3 = x3 * math_ops.reduce_sum(var2.sparse_read([0]))
+      return i + 1, y1, y2, y3
+
+    r = control_flow_ops.while_loop(
+        lambda i, x1, x2, x3: i < 3, body,
+        [0, x1_init, x2_init, x3_init])[1:]
+    var1_grad, var2_grad = gradients_impl.gradients(r, [var1, var2])
+
+    self.evaluate(variables.global_variables_initializer())
+    var1_grad_val = self.evaluate(var1_grad)
+    var2_grad_val = self.evaluate(var2_grad)
+    self.assertIsInstance(var1_grad_val, ops.IndexedSlicesValue)
+    self.assertIsInstance(var2_grad_val, ops.IndexedSlicesValue)
+    self.assertAllEqual(gradient_checker_v2._to_numpy(var1_grad_val),
+                        [0., 1., 0., 1., 0.])
+    self.assertAllEqual(gradient_checker_v2._to_numpy(var2_grad_val),
+                        [3., 0., 0.])
+
+  @test_util.run_deprecated_v1
+  def testWhileGrad_Gather(self):
+    # NOTE(skyewm): this test is interesting because the gather gradient
+    # function returns an IndexedSlices.
+    x = constant_op.constant([1., 1., 1., 1., 1.])
+    y = control_flow_ops.while_loop(
+        lambda i, _: i < 3,
+        lambda i, x: (i + 1, x + array_ops.gather(x, [0])),
+        [0, x[:1]])[1]
+    z = y * 3.0
+    grad = gradients_impl.gradients(z, x)[0]
+    self.assertEqual(self.evaluate(y), 8.)
+    self.assertAllEqual(self.evaluate(grad), [24., 0., 0., 0., 0.])
+
+  @test_util.run_deprecated_v1
+  def testWhileGrad_GatherNoFanOut(self):
+    # NOTE(skyewm): this test is interesting because the gather gradient
+    # function returns an IndexedSlices.
+    x = constant_op.constant([1., 1., 1., 1., 1.])
+    y = control_flow_ops.while_loop(
+        lambda i, _: i < 3,
+        lambda i, x: (i + 1, array_ops.gather(x, [0])),
+        [0, x[:1]])[1]
+    z = y * 3.0
+    grad = gradients_impl.gradients(z, x)[0]
+    self.assertEqual(self.evaluate(y), 1.)
+    self.assertAllEqual(self.evaluate(grad), [3., 0., 0., 0., 0.])
+
   @test_util.run_v1_only("b/120545219")
   def testWhileGradInCond(self):
 
diff --git a/tensorflow/python/ops/while_v2.py b/tensorflow/python/ops/while_v2.py
index 584faf11df..9f34172836 100644
--- a/tensorflow/python/ops/while_v2.py
+++ b/tensorflow/python/ops/while_v2.py
@@ -43,6 +43,7 @@ from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import list_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import tensor_array_ops
+from tensorflow.python.ops import while_v2_indexed_slices_rewriter
 from tensorflow.python.util import nest
 
 # pylint: disable=protected-access
@@ -293,6 +294,10 @@ def _WhileGrad(op, *grads):  # pylint: disable=invalid-name
                                            while_op)
   loop_vars = args + captured_inputs
 
+  # This modifies body_grad_graph.
+  loop_vars = while_v2_indexed_slices_rewriter.rewrite_grad_indexed_slices(
+      grads, body_grad_graph, loop_vars, while_op.inputs)
+
   def grad_cond(counter, max_iters, *unused_args):
     return counter < max_iters
 
@@ -310,26 +315,15 @@ def _WhileGrad(op, *grads):  # pylint: disable=invalid-name
       output_shapes=[t.shape for t in body_grad_graph.outputs],
       parallel_iterations=parallel_iterations,
       name="%s_grad" % while_op.name)
+  grad_op = outputs[0].op
 
   _copy_handle_data(body_grad_graph.outputs, outputs)
-  util.maybe_set_lowering_attr(outputs[0].op)
-  _maybe_set_maximum_iterations_attr(outputs[0].op, maximum_iterations)
+  util.maybe_set_lowering_attr(grad_op)
+  _maybe_set_maximum_iterations_attr(grad_op, maximum_iterations)
 
   # See comment in while_loop.
   outputs = [array_ops.identity(t) for t in outputs]
-
-  # Set None as the output gradient for tensors with None input gradient.
-  # outputs[0] is the loop counter.
-  # outputs[1] is the total number of loop iterations.
-  index = 2
-  none_padded_outputs = []
-  for g in grads:
-    if g is None:
-      none_padded_outputs.append(None)
-    else:
-      none_padded_outputs.append(outputs[index])
-      index += 1
-  return none_padded_outputs
+  return _get_structured_grad_output(outputs, grads, body_grad_graph)
 
 
 def _preprocess_grad(grad, body_graph_output, while_op_output):
@@ -365,6 +359,8 @@ def _preprocess_grad(grad, body_graph_output, while_op_output):
   return grad
 
 
+# TODO(skyewm): make this return constants if op_output's shape is fully
+# defined (this can be done by checking the "shape" attr of resource vars).
 def _zeros_like(op_output):
   """Like array_ops.zeros_like() but also accepts resource var handles."""
   if op_output.dtype == dtypes.resource:
@@ -500,14 +496,15 @@ def _create_grad_func(ys, xs, grads, cond_graph, body_graph, name, while_op,
   # Add the popped accumulators to the list of outputs.
   for internal_capture in grad_func_graph.internal_captures:
     if internal_capture in grad_func_graph.popped_tensor_lists:
-      grad_func_graph.outputs.append(
-          grad_func_graph.popped_tensor_lists[internal_capture])
+      new_output = grad_func_graph.popped_tensor_lists[internal_capture]
     elif internal_capture.dtype == dtypes.resource:
-      grad_func_graph.outputs.append(internal_capture)
+      new_output = internal_capture
     else:
       raise ValueError("Tensor %s is in list of internal_captures but is"
                        " neither a resource nor is in popped_tensor_lists." %
                        str(internal_capture))
+    grad_func_graph.outputs.append(new_output)
+    grad_func_graph.structured_outputs.append(new_output)
 
   return grad_func_graph, args
 
@@ -590,6 +587,45 @@ def _resolve_grad_captures(body_graph, body_grad_graph, while_op):
   return new_capture_inputs
 
 
+def _get_structured_grad_output(outputs, grads, body_grad_graph):
+  """Returns the values that should be returned from the while grad function.
+
+  Args:
+    outputs: the raw Tensor outputs of the grad While op.
+    grads: the input gradients to the gradient function.
+    body_grad_graph: _WhileBodyGradFuncGraph.
+
+  Returns:
+    A list of gradient values. May include Nones.
+  """
+  result = []
+  # outputs[0] is the loop counter.
+  # outputs[1] is the total number of loop iterations.
+  outputs_idx = 2
+  structured_outputs_idx = 2
+  for g in grads:
+    # Set None as the output gradient for tensors with None input gradient.
+    if g is None:
+      result.append(None)
+      continue
+    output = body_grad_graph.structured_outputs[structured_outputs_idx]
+    structured_outputs_idx += 1
+    if isinstance(output, ops.IndexedSlices):
+      # TODO(skyewm): is there a more robust way to determine the order of
+      # flattened IndexedSlices components?
+      result.append(ops.IndexedSlices(
+          values=outputs[outputs_idx],
+          indices=outputs[outputs_idx + 1],
+          dense_shape=outputs[outputs_idx + 2]))
+      outputs_idx += 3
+    else:
+      assert isinstance(output, ops.Tensor)
+      result.append(outputs[outputs_idx])
+      outputs_idx += 1
+
+  return result
+
+
 def _get_accumulator(tensor):
   r"""Returns TensorList if any containing accumulated values of tensor.
 
@@ -731,9 +767,9 @@ class _WhileBodyGradFuncGraph(util.WhileBodyFuncGraph):
     """
     if (not whitelisted and tensor.graph is not self and
         tensor.graph != self._forward_graph):
-      raise ValueError("Attempting to capture tensor", str(tensor),
-                       " which is not in the forward graph but in ",
-                       _graph_name(tensor.graph), ".")
+      raise ValueError("Attempting to capture tensor %s which is not in the "
+                       "forward graph but in %s." %
+                       (str(tensor), _graph_name(tensor.graph)))
     return super(_WhileBodyGradFuncGraph, self).capture(tensor, name)
 
   def _capture_helper(self, tensor, name):
@@ -847,7 +883,7 @@ def _check_num_inputs_outputs(cond_graph, body_graph, num_flattened_loop_vars):
   assert len(cond_graph.outputs) == 1, (
       "cond_graph has %d outputs; Expected: 1" % len(cond_graph.outputs))
   assert len(body_graph.inputs) == num_flattened_loop_vars, (
-      "body_graph takes %d inputs; Expected: %d" % (len(cond_graph.inputs),
+      "body_graph takes %d inputs; Expected: %d" % (len(body_graph.inputs),
                                                     num_flattened_loop_vars))
   assert len(body_graph.outputs) == num_flattened_loop_vars, (
       "body_graph has %d outputs; Expected: %d" % (len(body_graph.outputs),
diff --git a/tensorflow/python/ops/while_v2_indexed_slices_rewriter.py b/tensorflow/python/ops/while_v2_indexed_slices_rewriter.py
new file mode 100644
index 0000000000..30e9709d70
--- /dev/null
+++ b/tensorflow/python/ops/while_v2_indexed_slices_rewriter.py
@@ -0,0 +1,279 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =============================================================================
+"""Methods for rewriting while_v2 grad functions with IndexedSlices output."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import func_graph
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gen_resource_variable_ops
+from tensorflow.python.util import nest
+
+
+def rewrite_grad_indexed_slices(grads, body_grad_graph, loop_vars,
+                                forward_inputs):
+  """Handles special case of IndexedSlices returned from while gradient.
+
+  Some gradient functions return IndexedSlices instead of a Tensor (e.g. the
+  gradient of Gather ops). When this happens in the gradient of a while body,
+  the resulting gradient body function will have mismatched inputs and outputs,
+  since the input is a single Tensor, but the IndexedSlices gets unnested into
+  three output Tensors.
+
+  This function fixes this by rewriting the gradient body to have three inputs
+  to match the three outputs, i.e., it effectively converts the input Tensor
+  into an input IndexedSlices. It also returns new `loop_vars` to reflect the
+  new inputs.
+
+  Args:
+    grads: the input gradient Tensors to the while gradient computation.
+    body_grad_graph: _WhileBodyGradFuncGraph.
+    loop_vars: list of Tensors. The inputs to body_grad_graph.
+    forward_inputs: list of Tensors. The (flat) inputs to the forward-pass
+      While op.
+
+  Returns:
+    The new loop_vars to pass to body_grad_graph.
+  """
+  # Match up body_grad_graph.structured_outputs with the corresponding
+  # forward_inputs.
+  #
+  # Note that we don't expect a gradient computation to have structured output
+  # (e.g. no nested lists), so no need to flatten
+  # body_grad_graph.structured_outputs. However, structured_outputs may still
+  # contain composite tensors such as IndexedSlices, unlike
+  # body_grad_graph.outputs, which contains flattened composite tensors.
+  inputs_with_grads = [t for g, t in zip(grads, forward_inputs)
+                       if g is not None]
+  # Skip loop counter and total number of loop iterations.
+  structured_outputs = body_grad_graph.structured_outputs[2:]
+
+  for forward_input, output in zip(inputs_with_grads, structured_outputs):
+    if not isinstance(output, ops.IndexedSlices): continue
+
+    if forward_input.dtype == dtypes.resource:
+      # TODO(skyewm): In theory we should use this for all captured inputs, not
+      # just resource handles (which can only be captured). We can do this by
+      # checking that forward_input is passed straight through to its output.
+      loop_vars = _rewrite_input_as_indexed_slices(body_grad_graph, output,
+                                                   forward_input, loop_vars)
+    else:
+      _rewrite_output_as_tensor(body_grad_graph, output)
+
+  return loop_vars
+
+
+def _rewrite_output_as_tensor(body_grad_graph, grad_output_slices):
+  """Rewrites grad_output_slices to be a Tensor output.
+
+  Args:
+    body_grad_graph: _WhileBodyGradFuncGraph.
+    grad_output_slices: IndexedSlices output of body_grad_graph.
+  """
+  with body_grad_graph.as_default():
+    new_output = ops.convert_to_tensor_v2(grad_output_slices)
+
+  idx = body_grad_graph.structured_outputs.index(grad_output_slices)
+  body_grad_graph.structured_outputs[idx] = new_output
+  body_grad_graph.outputs = func_graph.flatten(
+      body_grad_graph.structured_outputs)
+
+
+def _rewrite_input_as_indexed_slices(body_grad_graph, grad_output_slices,
+                                     forward_input, loop_vars):
+  """Rewrites grad_output_slices's corresponding input to be an IndexedSlices.
+
+  This rewrite requires that forward_input was captured in the forward loop,
+  i.e. is not a user-specified loop variable. This is important because the
+  rewrite assumes that forward_input is passed through to its corresponding
+  output unchanged. This assumption is used in _rewrite_input_as_indexed_slices,
+  which depends on the exact gradient structure produced by the input's fanout.
+
+  This can yield a more efficient computation than using
+  _rewrite_output_as_tensor, since it preserves the IndexedSlices structure
+  instead of converting the IndexedSlices to a dense Tensor.
+
+  Args:
+    body_grad_graph: _WhileBodyGradFuncGraph.
+    grad_output_slices: IndexedSlices output of body_grad_graph.
+    forward_input: the corresonding Tensor input to the forward loop.
+    loop_vars: list of Tensors. The inputs to body_grad_graph.
+
+  Returns:
+    The new loop_vars to pass to body_grad_graph.
+  """
+  # Create initial IndexedSlices that will be the input to the grad While
+  # op. This will start as zeros, and accumulate the IndexedSlices grad output.
+  # Note that because forward_input is captured and not a loop var, its incoming
+  # gradient should always be zero.
+  init_slices = _create_grad_indexed_slices_init(grad_output_slices,
+                                                 forward_input)
+
+  # Create a new version of grad_output_slices's gradient computation that uses
+  # the new IndexedSlices input instead of the original Tensor input. We'll
+  # return the new computation and leave the old computation as dead code.
+  # TODO(skyewm): considering pruning body_grad_graph to remove the old
+  # computation.
+  with body_grad_graph.as_default():
+    input_slices = ops.IndexedSlices(
+        values=body_grad_graph.capture(init_slices.values, whitelisted=True),
+        indices=body_grad_graph.capture(init_slices.indices, whitelisted=True),
+        dense_shape=body_grad_graph.capture(init_slices.dense_shape,
+                                            whitelisted=True))
+
+    # Remove the captured tensors from the function inputs. We'll add them back
+    # at the correct index in _update_indexed_slices_param.
+    for t in _flatten(init_slices):
+      captured_t = body_grad_graph.captures.pop(t)
+      body_grad_graph.inputs.remove(captured_t)
+
+    new_output_slices = _rewrite_grad_indexed_slices_output(grad_output_slices,
+                                                            input_slices)
+
+  # Update body_grad_graph's inputs and outputs to reflect the new
+  # IndexedSlices computation.
+  return _update_indexed_slices_param(
+      body_grad_graph, loop_vars, init_slices, input_slices, new_output_slices,
+      grad_output_slices)
+
+
+def _create_grad_indexed_slices_init(grad_output_slices, forward_input):
+  """Creates an IndexedSlices to pass as input to the while grad function.
+
+  Args:
+    grad_output_slices: IndexedSlices. The corresponding while grad function
+      output.
+    forward_input: Tensor. The corresonding input to the forward while op.
+
+  Returns:
+    Zeros IndexedSlices, created in current Graph.
+  """
+  assert isinstance(grad_output_slices, ops.IndexedSlices)
+  assert isinstance(forward_input, ops.Tensor)
+  values_out = grad_output_slices.values
+  indices_out = grad_output_slices.indices
+
+  # Create the initial values tensor.
+  if values_out.shape.is_fully_defined():
+    values_shape = tensor_shape.TensorShape([0] +
+                                            values_out.shape.as_list()[1:])
+    values = array_ops.zeros(values_shape, dtype=values_out.dtype,
+                             name="values_init")
+  else:
+    if forward_input.dtype == dtypes.resource:
+      forward_shape = gen_resource_variable_ops.variable_shape(forward_input)
+    else:
+      forward_shape = array_ops.shape(forward_input)
+    values_shape = array_ops.concat([[0], forward_shape[1:]], 0)
+    values = array_ops.zeros(values_shape, dtype=values_out.dtype,
+                             name="values_init")
+
+  # Create the initial indices tensor.
+  indices = constant_op.constant([], indices_out.dtype, name="indices_init")
+
+  # Create the initial dense_shape tensor. We assume is the same shape as
+  # forward_input, since captured tensors don't change shape across loop
+  # iterations.
+  if forward_input.dtype == dtypes.resource:
+    shape = gen_resource_variable_ops.variable_shape(forward_input,
+                                                     name="shape_init")
+  else:
+    shape = array_ops.shape(forward_input, name="shape_init")
+
+  return ops.IndexedSlices(values=values, indices=indices, dense_shape=shape)
+
+
+def _rewrite_grad_indexed_slices_output(old_output_slices, new_input_slices):
+  """Creates a new verson of old_output_slices with new_input_slices as input.
+
+  This method assumes that old_output_slices.{values,indices} are produced by
+  concatenating the incoming gradient Tensor input with the IndexedSlices
+  produced by the gradient computation of the while body. See
+  gradients_impl._AggregateIndexedSlicesGradients for where these concats are
+  constructed. We build new concats that use new_input_slices instead of the
+  original Tensor input.
+
+  Args:
+    old_output_slices: original IndexedSlices output of while gradient.
+    new_input_slices: new IndexedSlices to use as input to while gradient.
+
+  Returns:
+    A new IndexedSlices to replace old_output_slices.
+  """
+
+  def rewrite(old_output, new_input):
+    assert old_output.type == "Identity"
+    concat_op = old_output.inputs[0].op
+    assert concat_op.type == "ConcatV2"
+    # Don't include axis arg
+    old_concat_args = concat_op.inputs[:-1]
+    # We assume that the original gradient input was the first argument to the
+    # concat op.
+    # TODO(skyewm): do this in a more robust way.
+    return array_ops.concat([new_input] + old_concat_args[1:], 0)
+
+  values = rewrite(old_output_slices.values.op, new_input_slices.values)
+  indices = rewrite(old_output_slices.indices.op, new_input_slices.indices)
+  return ops.IndexedSlices(values=values, indices=indices,
+                           dense_shape=new_input_slices.dense_shape)
+
+
+def _update_indexed_slices_param(graph, loop_vars, init_slices, input_slices,
+                                 output_slices, old_output_slices):
+  """Updates graph with new IndexedSlices input/output.
+
+  Updates graph's metadata to output the gradient computation defined by
+  init_slices, input_slices, and output_slices, instead of outputting
+  old_output_slices. Also returns a new version of loop_vars with init_slices
+  replacing the old input.
+
+  Args:
+    graph: _WhileBodyGradFuncGraph.
+    loop_vars: the inputs to graph.
+    init_slices: the new IndexedSlices to use as input to graph.
+    input_slices: the new IndexedSlices in graph that should be fed by
+      init_slices.
+    output_slices: the new IndexedSlices in graph that should be the
+      corresonding output to input_slices.
+    old_output_slices: the IndexedSlices in graph that are currently
+      being output.
+
+  Returns:
+    New loop_vars to pass to graph.
+  """
+  structured_idx = graph.structured_outputs.index(old_output_slices)
+  # We assume that the component tensors of old_output_slices appear
+  # sequentially in graph.outputs. We use the first of these tensors
+  # as the reference index.
+  flat_idx = graph.outputs.index(func_graph.flatten(old_output_slices)[0])
+
+  graph.structured_outputs[structured_idx] = output_slices
+  graph.outputs = func_graph.flatten(
+      graph.structured_outputs)
+
+  graph.inputs = (graph.inputs[:flat_idx] + _flatten(input_slices) +
+                  graph.inputs[flat_idx + 1:])
+
+  return loop_vars[:flat_idx] + _flatten(init_slices) + loop_vars[flat_idx + 1:]
+
+
+def _flatten(arg):
+  return nest.flatten(arg, expand_composites=True)
-- 
GitLab


From 7a8542ea251986626d35c9030bdbbcc3ff085fd9 Mon Sep 17 00:00:00 2001
From: Michael Banfield <micban@google.com>
Date: Thu, 31 Jan 2019 11:01:52 -0800
Subject: [PATCH 1661/2345] Ensure workers are configured to shut down on
 timeout.

PiperOrigin-RevId: 231816885
---
 tensorflow/contrib/tpu/python/tpu/session_support.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/session_support.py b/tensorflow/contrib/tpu/python/tpu/session_support.py
index f5735cecc3..5cb2ca6478 100644
--- a/tensorflow/contrib/tpu/python/tpu/session_support.py
+++ b/tensorflow/contrib/tpu/python/tpu/session_support.py
@@ -172,7 +172,8 @@ class WorkerHeartbeatManager(object):
     """Shutdown all workers after `shutdown_timeout_secs`."""
     logging.info('Shutting down %s.', self)
     req = event_pb2.WorkerHeartbeatRequest(
-        watchdog_config=event_pb2.WatchdogConfig(timeout_ms=timeout_ms))
+        watchdog_config=event_pb2.WatchdogConfig(timeout_ms=timeout_ms),
+        shutdown_mode=event_pb2.WAIT_FOR_COORDINATOR)
     self.configure(req)
 
     # Wait for workers to shutdown.  This isn't strictly required
-- 
GitLab


From 4353f16e1675a021ff9a796decda7ed5ab9e1bb0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 31 Jan 2019 11:07:51 -0800
Subject: [PATCH 1662/2345] Add shared kernel selection across depthwise
 convolution kernels.

PiperOrigin-RevId: 231818297
---
 .../internal/optimized/depthwiseconv_uint8.h  |  1 +
 .../depthwiseconv_uint8_3x3_filter.h          |  1 +
 .../internal/reference/depthwiseconv_uint8.h  | 21 +++++++++++++++++++
 3 files changed, 23 insertions(+)

diff --git a/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8.h b/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8.h
index 0f4226dd15..13629494bd 100644
--- a/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8.h
+++ b/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8.h
@@ -19,6 +19,7 @@ limitations under the License.
 #include "public/gemmlowp.h"
 #include "tensorflow/lite/kernels/internal/common.h"
 #include "tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h"
+#include "tensorflow/lite/kernels/internal/reference/depthwiseconv_uint8.h"
 #include "tensorflow/lite/kernels/internal/types.h"
 
 namespace tflite {
diff --git a/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h b/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h
index 5859bcaed4..6bfe27d96d 100644
--- a/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h
+++ b/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h
@@ -18,6 +18,7 @@ limitations under the License.
 #include "fixedpoint/fixedpoint.h"
 #include "public/gemmlowp.h"
 #include "tensorflow/lite/kernels/internal/common.h"
+#include "tensorflow/lite/kernels/internal/reference/depthwiseconv_uint8.h"
 #include "tensorflow/lite/kernels/internal/types.h"
 
 namespace tflite {
diff --git a/tensorflow/lite/kernels/internal/reference/depthwiseconv_uint8.h b/tensorflow/lite/kernels/internal/reference/depthwiseconv_uint8.h
index 002444b681..d9a5372e6e 100644
--- a/tensorflow/lite/kernels/internal/reference/depthwiseconv_uint8.h
+++ b/tensorflow/lite/kernels/internal/reference/depthwiseconv_uint8.h
@@ -23,6 +23,27 @@ limitations under the License.
 #include "tensorflow/lite/kernels/internal/types.h"
 
 namespace tflite {
+
+// Used in tests and template parameters to control which version of depthwise
+// convolution is called. Primarily for reference code, and specializations
+// forced in tests.
+enum class DepthwiseConvKernelInvocation {
+  // Run all tests against kUseStandardEntry even if also testing another
+  // kernel, since we need to be sure that the main DepthwiseConv() function in
+  // optimized_ops.h dispatches to a correctly-executing kernel.
+  kNone = 0,                 // The "default" option: use the normal
+                             // DepthwiseConv kernel (entry) function.
+  kUseGenericKernel,         // Forced use of generic kernel.
+  kUseNeon3x3,               // 3x3 kernel that uses NEON when available.
+  kUseNeon3x3DotProduct,     // 3x3 kernel that uses dot-product enabled NEON
+                             // when available.
+  kUseCModel3x3DotProduct,   // 3x3 kernel, reference C model that is intended
+                             // to match overall design NEON code.
+  kUseUnwound3x3DotProduct,  // 3x3 kernel, reference C model with unwound loops
+                             // and some arrays.
+  kUseIntrinsics3x3DotProduct,  // 3x3 kernel using NEON intrinsics.
+};
+
 namespace reference_ops {
 
 inline void DepthwiseConv(
-- 
GitLab


From aad38dc01185350c30812dbe4e9c7c8114d2647e Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Thu, 31 Jan 2019 11:14:28 -0800
Subject: [PATCH 1663/2345] Populate
 FunctionLibraryRuntime::Options::step_container

I found this when prototyping a speculative idea which may or may not eventually
land; splitting this out separately since it looks like an obvious omission.

PiperOrigin-RevId: 231819452
---
 tensorflow/core/kernels/functional_ops.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/core/kernels/functional_ops.cc b/tensorflow/core/kernels/functional_ops.cc
index 5ecb203cbc..a3e7b53d55 100644
--- a/tensorflow/core/kernels/functional_ops.cc
+++ b/tensorflow/core/kernels/functional_ops.cc
@@ -120,6 +120,7 @@ void SetRunOptions(OpKernelContext* ctx, FunctionLibraryRuntime::Options* opts,
     opts->stats_collector = ctx->stats_collector();
   }
   opts->runner = ctx->runner();
+  opts->step_container = ctx->step_container();
 }
 
 class IfOp : public AsyncOpKernel {
-- 
GitLab


From 25af654bcb9155b88a5328ff0c65afab8e92daf3 Mon Sep 17 00:00:00 2001
From: Guozhong Zhuang <guozhong.zhuang@intel.com>
Date: Thu, 31 Jan 2019 11:39:39 -0800
Subject: [PATCH 1664/2345] Per code review suggest, enhance with move
 constructor and destructor of Entity structure

---
 tensorflow/core/util/mkl_util.h | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/tensorflow/core/util/mkl_util.h b/tensorflow/core/util/mkl_util.h
index 1a5c6d8e56..b357b3bf75 100644
--- a/tensorflow/core/util/mkl_util.h
+++ b/tensorflow/core/util/mkl_util.h
@@ -2105,13 +2105,13 @@ class LRUCache {
     // Insert an entry to the front of the LRU list
     lru_list_.push_front(key);
     Entry entry(op, lru_list_.begin());
-    cache_.insert(std::make_pair(key, entry));
+    cache_.emplace(std::make_pair(key, std::move(entry)));
   }
 
   void Clear() {
     if (lru_list_.empty()) return;
 
-    // clean up the cache
+    // Clean up the cache
     cache_.clear();
     lru_list_.clear();
   }
@@ -2123,10 +2123,24 @@ class LRUCache {
 
     // A list iterator pointing to the entry's position in the LRU list.
     std::list<string>::iterator lru_iterator;
+
+    // Constructor
     Entry(T* op, std::list<string>::iterator it) {
       this->op = op;
       this->lru_iterator = it;
     }
+
+    // Move construcctor
+    Entry(Entry&& source) noexcept :
+      lru_iterator(std::move(source.lru_iterator)) {
+      op = std::move(source.op);
+      source.op = std::forward<T*>(nullptr);
+    }
+
+    // Destructor
+    ~Entry() {
+      if (op != nullptr) delete op;  
+    }
   };
 
   // Remove the least recently accessed entry from LRU list, which
@@ -2134,10 +2148,8 @@ class LRUCache {
   bool Delete() {
     if (lru_list_.empty()) return false;
     string key = lru_list_.back();
-    auto it = cache_.find(key);
     lru_list_.pop_back();
-    delete it->second.op;  // delete the object
-    cache_.erase(it);
+    cache_.erase(key);
     return true;
   }
 
-- 
GitLab


From 95e43f861779d94180f71493a215a0bd4825d406 Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Thu, 31 Jan 2019 12:05:23 -0800
Subject: [PATCH 1665/2345] DType is no longer convertible to an int

Use dtype.as_datatype_enum instead of int(dtype) to get the same
result.

PiperOrigin-RevId: 231828864
---
 .../pylib/python/ops/remote_fused_graph_ops.py                | 4 ++--
 tensorflow/python/framework/dtypes.py                         | 3 ---
 tensorflow/python/util/example_parser_configuration.py        | 4 ++--
 3 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/remote_fused_graph/pylib/python/ops/remote_fused_graph_ops.py b/tensorflow/contrib/remote_fused_graph/pylib/python/ops/remote_fused_graph_ops.py
index 2054367f0d..7e79785d28 100644
--- a/tensorflow/contrib/remote_fused_graph/pylib/python/ops/remote_fused_graph_ops.py
+++ b/tensorflow/contrib/remote_fused_graph/pylib/python/ops/remote_fused_graph_ops.py
@@ -50,13 +50,13 @@ def remote_fused_graph_execute(inputs,
   if default_graph_input_tensor_type_shapes:
     for type_shape in default_graph_input_tensor_type_shapes:
       type_shape_proto = info_proto.default_graph_input_tensor_shape.add()
-      type_shape_proto.dtype = int(dtypes.as_dtype(type_shape[0]))
+      type_shape_proto.dtype = dtypes.as_dtype(type_shape[0]).as_datatype_enum
       for dim in type_shape[1]:
         type_shape_proto.shape.dim.add().size = dim
   if default_graph_output_tensor_type_shapes:
     for type_shape in default_graph_output_tensor_type_shapes:
       type_shape_proto = info_proto.default_graph_output_tensor_shape.add()
-      type_shape_proto.dtype = int(dtypes.as_dtype(type_shape[0]))
+      type_shape_proto.dtype = dtypes.as_dtype(type_shape[0]).as_datatype_enum
       for dim in type_shape[1]:
         type_shape_proto.shape.dim.add().size = dim
 
diff --git a/tensorflow/python/framework/dtypes.py b/tensorflow/python/framework/dtypes.py
index 9d643e041c..6638be219d 100644
--- a/tensorflow/python/framework/dtypes.py
+++ b/tensorflow/python/framework/dtypes.py
@@ -282,9 +282,6 @@ class DType(object):
     """Returns the string name for this `DType`."""
     return _TYPE_TO_STRING[self._type_enum]
 
-  def __int__(self):
-    return self._type_enum
-
   def __str__(self):
     return "<dtype: %r>" % self.name
 
diff --git a/tensorflow/python/util/example_parser_configuration.py b/tensorflow/python/util/example_parser_configuration.py
index e3fdcf956e..dc8937a319 100644
--- a/tensorflow/python/util/example_parser_configuration.py
+++ b/tensorflow/python/util/example_parser_configuration.py
@@ -101,7 +101,7 @@ def extract_example_parser_configuration(parse_example_op, sess):
     fixed_config.shape.CopyFrom(
         tensor_shape.TensorShape(dense_shapes[i]).as_proto())
 
-    fixed_config.dtype = int(dense_types[i])
+    fixed_config.dtype = dense_types[i].as_datatype_enum
     # Get the output tensor name.
     fixed_config.values_output_tensor_name = parse_example_op.outputs[
         dense_values_start + i].name
@@ -111,7 +111,7 @@ def extract_example_parser_configuration(parse_example_op, sess):
     key = fetched[sparse_keys_start + i]
     feature_config = config.feature_map[key]
     var_len_feature = feature_config.var_len_feature
-    var_len_feature.dtype = int(sparse_types[i])
+    var_len_feature.dtype = sparse_types[i].as_datatype_enum
     var_len_feature.indices_output_tensor_name = parse_example_op.outputs[
         sparse_indices_start + i].name
     var_len_feature.values_output_tensor_name = parse_example_op.outputs[
-- 
GitLab


From 31723c74caf0cf3effb46e12de00eb80ae86a899 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Thu, 31 Jan 2019 12:24:41 -0800
Subject: [PATCH 1666/2345] [Grappler] Prune stateful and dataset ops in TF 2.0
 functions

PiperOrigin-RevId: 231832111
---
 .../common_runtime/eager/kernel_and_device.cc | 29 +++++++------------
 tensorflow/core/grappler/grappler_item.cc     |  9 ++----
 tensorflow/core/grappler/grappler_item.h      |  8 ++++-
 .../grappler/optimizers/meta_optimizer.cc     |  9 ++++--
 .../core/grappler/optimizers/meta_optimizer.h |  3 +-
 tensorflow/core/grappler/utils/functions.cc   |  2 +-
 .../core/grappler/utils/functions_test.cc     |  4 ++-
 .../core/kernels/partitioned_function_ops.cc  | 27 +++++++----------
 8 files changed, 42 insertions(+), 49 deletions(-)

diff --git a/tensorflow/core/common_runtime/eager/kernel_and_device.cc b/tensorflow/core/common_runtime/eager/kernel_and_device.cc
index 09f60a7e91..41b4608c7e 100644
--- a/tensorflow/core/common_runtime/eager/kernel_and_device.cc
+++ b/tensorflow/core/common_runtime/eager/kernel_and_device.cc
@@ -96,31 +96,24 @@ Status KernelAndDeviceFunc::Init(const NodeDef& ndef,
           "Failed to parse config_proto attribute as tensorflow::ConfigProto "
           "proto.");
     }
-    // We are going to execute the graph via function library runtime, and
-    // because function execution semantics is slightly different from the
-    // regular tensorlow graph, we need to make sure that Grappler respects it
-    // when doing it's optimization passes (e.g. do not prune stateful and
-    // dataset ops).
     grappler::GrapplerItem::OptimizationOptions optimization_options;
-    optimization_options.is_function_instantiation = true;
 
-    // Keras graphs expected to be executed with regular graph execution
-    // semantics (it's allowed to prune stateful and dataset ops).
-    if (absl::StrContains(function_def->signature().name(), "keras_graph")) {
-      optimization_options.is_function_instantiation = false;
-    }
+    // Tensorflow 2.0 in eager mode with automatic control dependencies will
+    // prune all nodes that are not in the transitive fanin of the fetch nodes.
+    // However because the function will be executed via FunctionLibraryRuntime,
+    // and current function implementation does not prune stateful and dataset
+    // ops, we rely on Grappler to do the correct graph pruning.
+    optimization_options.allow_pruning_stateful_and_dataset_ops = true;
 
-    // Wrapped function expects execution semantics to be the same as
-    // `session.run`, so we should prune unreachable stateful and dataset ops.
-    if (absl::StrContains(function_def->signature().name(),
-                          "wrapped_function")) {
-      optimization_options.is_function_instantiation = false;
-    }
+    // All the nested function calls will be executed and optimized via
+    // PartitionedCallOp, there is no need to optimize functions now.
+    optimization_options.optimize_function_library = false;
 
     options.optimize_graph_fn = std::bind(
         grappler::OptimizeGraph, std::placeholders::_1, std::placeholders::_2,
         std::placeholders::_3, std::placeholders::_4, config_proto,
-        optimization_options, std::placeholders::_5);
+        function_def->signature().name(), optimization_options,
+        std::placeholders::_5);
   }
 #endif
   options.graph_collector = graph_collector;
diff --git a/tensorflow/core/grappler/grappler_item.cc b/tensorflow/core/grappler/grappler_item.cc
index d4bd2cc763..bc95c9cf72 100644
--- a/tensorflow/core/grappler/grappler_item.cc
+++ b/tensorflow/core/grappler/grappler_item.cc
@@ -117,15 +117,10 @@ std::unordered_set<string> GrapplerItem::NodesToPreserve() const {
 
   // Tensorflow functions do not prune stateful or dataset-output ops from
   // the function body (see PruneFunctionBody in common_runtime/function.cc).
-  //
-  // We also keep placeholders in the functions body, because it's a bug to have
-  // placeholders inside functions, and we want to catch such invalid graphs
-  // early.
-  if (optimization_options_.is_function_instantiation) {
+  if (!optimization_options_.allow_pruning_stateful_and_dataset_ops) {
     FunctionLibraryDefinition fn_library(OpRegistry::Global(), graph.library());
     for (const NodeDef& node : graph.node()) {
-      if (IsStateful(node, &fn_library) || IsDataset(node) ||
-          IsPlaceholder(node)) {
+      if (IsStateful(node, &fn_library) || IsDataset(node)) {
         result.insert(node.name());
       }
     }
diff --git a/tensorflow/core/grappler/grappler_item.h b/tensorflow/core/grappler/grappler_item.h
index 75712e9f92..57949b322d 100644
--- a/tensorflow/core/grappler/grappler_item.h
+++ b/tensorflow/core/grappler/grappler_item.h
@@ -91,7 +91,13 @@ struct GrapplerItem {
     // by running Grappler optimizer passes. One main difference is that
     // functions do not prune ops with side-effects and dataset-output ops (see
     // PruneFunctionBody in common_runtime/function.cc).
-    bool is_function_instantiation = false;
+    bool allow_pruning_stateful_and_dataset_ops = true;
+
+    // If true Grappler will optimize the main graph, and also all functions in
+    // the graph function library (function can't be polymorphic, it can't have
+    // undefined type parameters in the function signature, or placeholder
+    // attributes in the function body).
+    bool optimize_function_library = true;
   };
 
   const std::unordered_set<string>& devices() const;
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
index c9d519af11..33942a591f 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
@@ -553,7 +553,8 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
 
   // Optimize each function only once.
   absl::flat_hash_set<string> optimized_funcs;
-  bool optimize_function_library = true;
+  bool optimize_function_library =
+      item.optimization_options().optimize_function_library;
 
   while (optimize_function_library) {
     optimize_function_library = false;
@@ -605,7 +606,8 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
       // instantiated by the function definition, because we must guarantee
       // function execution semantics wrt side effects (see
       // function_optimizer.cc).
-      func_item.optimization_options().is_function_instantiation = true;
+      func_item.optimization_options().allow_pruning_stateful_and_dataset_ops =
+          false;
 
       // Optimize function body graph.
       GraphDef optimized_func_graph;
@@ -701,7 +703,7 @@ Status RunMetaOptimizer(const GrapplerItem& item, const ConfigProto& cfg,
 Status OptimizeGraph(
     std::vector<string> ret_node_names, FunctionLibraryDefinition* flib,
     const DeviceSet& device_set, Device* cpu_device,
-    const ConfigProto& config_proto,
+    const ConfigProto& config_proto, const string& grappler_item_id,
     const GrapplerItem::OptimizationOptions& optimization_options,
     std::unique_ptr<tensorflow::Graph>* g) {
   if (!tensorflow::grappler::MetaOptimizerEnabled(config_proto)) {
@@ -709,6 +711,7 @@ Status OptimizeGraph(
   }
 
   tensorflow::grappler::GrapplerItem item;
+  item.id = grappler_item_id;
   item.optimization_options() = optimization_options;
 
   // Add all available devices so that inlined function can be placed.
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.h b/tensorflow/core/grappler/optimizers/meta_optimizer.h
index ec78cc5771..751a9e5d60 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer.h
@@ -120,6 +120,7 @@ Status RunMetaOptimizer(const GrapplerItem& item, const ConfigProto& cfg,
 // `device_set`: the set of devices that graph can refer to.
 // `cpu_device`: the CPU device.
 // `config_proto`: Grapper configuration.
+// `grappler_item_id': Grappler item id (e.g. optimized function name).
 // `optimization_options`: Grappler optimization constraints that are known only
 //    at runtime.
 //
@@ -130,7 +131,7 @@ Status RunMetaOptimizer(const GrapplerItem& item, const ConfigProto& cfg,
 Status OptimizeGraph(
     std::vector<string> ret_node_names, FunctionLibraryDefinition* lib,
     const DeviceSet& device_set, Device* cpu_device,
-    const ConfigProto& config_proto,
+    const ConfigProto& config_proto, const string& grappler_item_id,
     const GrapplerItem::OptimizationOptions& optimization_options,
     std::unique_ptr<tensorflow::Graph>* g);
 
diff --git a/tensorflow/core/grappler/utils/functions.cc b/tensorflow/core/grappler/utils/functions.cc
index 357a0b3b47..5f38aeac54 100644
--- a/tensorflow/core/grappler/utils/functions.cc
+++ b/tensorflow/core/grappler/utils/functions.cc
@@ -339,7 +339,7 @@ GrapplerFunctionItem::GrapplerFunctionItem(
 
   // Tensorflow functions execution semantics is different from the main graph,
   // and we need to preserve it when we do graph optimizations.
-  optimization_options().is_function_instantiation = true;
+  optimization_options().allow_pruning_stateful_and_dataset_ops = false;
 }
 
 const string& GrapplerFunctionItem::description() const { return description_; }
diff --git a/tensorflow/core/grappler/utils/functions_test.cc b/tensorflow/core/grappler/utils/functions_test.cc
index 7720888828..30b6195a9b 100644
--- a/tensorflow/core/grappler/utils/functions_test.cc
+++ b/tensorflow/core/grappler/utils/functions_test.cc
@@ -641,7 +641,9 @@ TEST_F(FunctionsTest, FromFunctionDefWithSideEffectfulOps) {
   EXPECT_EQ(3, item.function_body().node_size());
   EXPECT_EQ(1, item.input_size());
   EXPECT_EQ(0, item.output_size());
-  EXPECT_EQ(true, item.optimization_options().is_function_instantiation);
+
+  const auto &opts = item.optimization_options();
+  EXPECT_FALSE(opts.allow_pruning_stateful_and_dataset_ops);
 }
 
 TEST_F(FunctionsTest, MakeFunctionDef) {
diff --git a/tensorflow/core/kernels/partitioned_function_ops.cc b/tensorflow/core/kernels/partitioned_function_ops.cc
index 5d26265aaa..d8c38fe400 100644
--- a/tensorflow/core/kernels/partitioned_function_ops.cc
+++ b/tensorflow/core/kernels/partitioned_function_ops.cc
@@ -157,25 +157,18 @@ class PartitionedCallOp : public AsyncOpKernel {
   Status Instantiate(FunctionLibraryRuntime* lib, OpKernelContext* ctx,
                      std::vector<Tensor>* inputs,
                      FunctionLibraryRuntime::Handle* handle) {
-    // We are going to execute the graph via function library runtime, and
-    // because function execution semantics is slightly different from the
-    // regular tensorlow graph, we need to make sure that Grappler respects it
-    // when doing it's optimization passes (e.g. do not prune stateful and
-    // dataset ops).
     grappler::GrapplerItem::OptimizationOptions optimization_options;
-    optimization_options.is_function_instantiation = true;
 
-    // Keras graphs expected to be executed with regular graph execution
-    // semantics (it's allowed to prune stateful and dataset ops).
-    if (absl::StrContains(func_.name(), "keras_graph")) {
-      optimization_options.is_function_instantiation = false;
-    }
+    // Tensorflow 2.0 in eager mode with automatic control dependencies will
+    // prune all nodes that are not in the transitive fanin of the fetch nodes.
+    // However because the function will be executed via FunctionLibraryRuntime,
+    // and current function implementation does not prune stateful and dataset
+    // ops, we rely on Grappler to do the correct graph pruning.
+    optimization_options.allow_pruning_stateful_and_dataset_ops = true;
 
-    // Wrapped function expects execution semantics to be the same as
-    // `session.run`, so we should prune unreachable stateful and dataset ops.
-    if (absl::StrContains(func_.name(), "wrapped_function")) {
-      optimization_options.is_function_instantiation = false;
-    }
+    // All the nested function calls will be executed and optimized via
+    // PartitionedCallOp, there is no need to optimize functions now.
+    optimization_options.optimize_function_library = false;
 
     FunctionLibraryRuntime::InstantiateOptions opts;
     opts.target = lib->device()->name();
@@ -183,7 +176,7 @@ class PartitionedCallOp : public AsyncOpKernel {
     opts.optimize_graph_fn = std::bind(
         grappler::OptimizeGraph, std::placeholders::_1, std::placeholders::_2,
         std::placeholders::_3, std::placeholders::_4, config_proto_,
-        optimization_options, std::placeholders::_5);
+        func_.name(), optimization_options, std::placeholders::_5);
     opts.graph_collector = ctx->graph_collector();
     opts.executor_type = executor_type_;
 
-- 
GitLab


From d7640706ca636b590e0d2cb272c0ca48a038720c Mon Sep 17 00:00:00 2001
From: Tong Shen <endlessroad@google.com>
Date: Thu, 31 Jan 2019 13:13:44 -0800
Subject: [PATCH 1667/2345] Fix FunctionalizeCond crash when fed with invalid
 control flow graph.

In some cases, we might get invalid control flow graph where Merge node inputs are not in any CondContext. As a result, in JoinCondStatesMerge(), the CondId 'src' will be nullptr. Check this case and return an error.

PiperOrigin-RevId: 231839691
---
 tensorflow/compiler/tf2xla/functionalize_cond.cc      |  5 +++++
 tensorflow/compiler/tf2xla/functionalize_cond_test.cc | 11 +++++++++++
 2 files changed, 16 insertions(+)

diff --git a/tensorflow/compiler/tf2xla/functionalize_cond.cc b/tensorflow/compiler/tf2xla/functionalize_cond.cc
index e780c93a8f..b21752ad57 100644
--- a/tensorflow/compiler/tf2xla/functionalize_cond.cc
+++ b/tensorflow/compiler/tf2xla/functionalize_cond.cc
@@ -34,6 +34,7 @@ limitations under the License.
 #include "tensorflow/core/graph/algorithm.h"
 #include "tensorflow/core/graph/control_flow.h"
 #include "tensorflow/core/graph/node_builder.h"
+#include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 
 using xla::StatusOr;
@@ -967,6 +968,10 @@ StatusOr<StateMap::CondId> FunctionalizeCond::JoinCondStatesMerge(
   VLOG(4) << "Joining (for merge) " << DebugString(src) << " and "
           << DebugString(dst);
   if (state_map_.IsEmpty(dst)) return src;
+  if (state_map_.IsEmpty(src)) {
+    return errors::Internal("Merge node ", merge->name(),
+                            " has input that's not in any CondContext.");
+  }
 
   if (state_map_.IsDead(src)) return src;
   if (state_map_.IsDead(dst)) return dst;
diff --git a/tensorflow/compiler/tf2xla/functionalize_cond_test.cc b/tensorflow/compiler/tf2xla/functionalize_cond_test.cc
index b0aabd63bb..05fa1ee92d 100644
--- a/tensorflow/compiler/tf2xla/functionalize_cond_test.cc
+++ b/tensorflow/compiler/tf2xla/functionalize_cond_test.cc
@@ -101,6 +101,17 @@ TEST_F(FunctionalizeCondTest, JoinCondStates) {
   TF_EXPECT_OK(t.status());
 }
 
+TEST_F(FunctionalizeCondTest, JoinCondStatesMergeWithInputNotInCondContext) {
+  Tensor val_tensor(DT_INT32, TensorShape());
+  val_tensor.flat<int>().setZero();
+  Node* val = test::graph::Constant(graph_.get(), val_tensor, "val");
+  Node* m = test::graph::Merge(graph_.get(), val, val);
+
+  StateMap::CondState cond_state;
+  auto joined_or = JoinCondStatesMerge(m, /*src=*/nullptr, &cond_state);
+  EXPECT_FALSE(joined_or.ok());
+}
+
 }  // namespace
 }  // namespace functionalize_cond
 }  // namespace tensorflow
-- 
GitLab


From 1b1f0959cbf3d02e15f1dba5e4ef4a66131d5cfe Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Thu, 31 Jan 2019 13:13:59 -0800
Subject: [PATCH 1668/2345] Properly account for the locality of symbols in
 comprehension expressions.

PiperOrigin-RevId: 231839777
---
 .../pyct/static_analysis/activity.py          | 65 +++++++++++++++++--
 .../pyct/static_analysis/liveness_test.py     | 58 +++++++++++++++++
 .../reaching_definitions_test.py              | 20 ++++++
 3 files changed, 137 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/autograph/pyct/static_analysis/activity.py b/tensorflow/python/autograph/pyct/static_analysis/activity.py
index 4359e0a268..65e4682215 100644
--- a/tensorflow/python/autograph/pyct/static_analysis/activity.py
+++ b/tensorflow/python/autograph/pyct/static_analysis/activity.py
@@ -25,6 +25,7 @@ import copy
 import weakref
 
 import gast
+import six
 
 from tensorflow.python.autograph.pyct import anno
 from tensorflow.python.autograph.pyct import qual_names
@@ -149,6 +150,14 @@ class _Lambda(object):
     self.args = set()
 
 
+class _Comprehension(object):
+
+  no_root = True
+
+  def __init__(self):
+    self.targets = set()
+
+
 class ActivityAnalyzer(transformer.Base):
   """Annotates nodes with local scope information.
 
@@ -199,12 +208,27 @@ class ActivityAnalyzer(transformer.Base):
       if qn.owner_set & set(l.args):
         return
 
+    # When inside a comprehension, ignore any of the comprehensions's targets.
+    # This includes attributes or slices of those arguments.
+    # This is not true in Python2, which leaks symbols.
+    if six.PY3:
+      for l in self.state[_Comprehension]:
+        if qn in l.targets:
+          return
+        if qn.owner_set & set(l.targets):
+          return
+
     if isinstance(node.ctx, gast.Store):
-      self.scope.mark_modified(qn)
-      if qn.is_composite and composite_writes_alter_parent:
-        self.scope.mark_modified(qn.parent)
-      if self._in_aug_assign:
-        self.scope.mark_read(qn)
+      # In comprehensions, modified symbols are the comprehension targets.
+      if six.PY3 and self.state[_Comprehension].level > 0:
+        # Like a lambda's args, they are tracked separately in Python3.
+        self.state[_Comprehension].targets.add(qn)
+      else:
+        self.scope.mark_modified(qn)
+        if qn.is_composite and composite_writes_alter_parent:
+          self.scope.mark_modified(qn.parent)
+        if self._in_aug_assign:
+          self.scope.mark_read(qn)
     elif isinstance(node.ctx, gast.Load):
       self.scope.mark_read(qn)
     elif isinstance(node.ctx, gast.Param):
@@ -338,12 +362,41 @@ class ActivityAnalyzer(transformer.Base):
     self.state[_Lambda].exit()
     return node
 
+  def _process_iterable_comprehension(self, node):
+    # This handles ListComp, SetComp, GeneratorExp.
+    self.state[_Comprehension].enter()
+    # Note: it's important to visit the generators first to properly account
+    # for the variables local to these generators. Example: `x` is local to the
+    # expression `x for x in y`.
+    node.generators = self.visit_block(node.generators)
+    node.elt = self.visit(node.elt)
+    self.state[_Comprehension].exit()
+    return node
+
+  def visit_DictComp(self, node):
+    # Identical to _process_iterable_comprehension, different node names.
+    self.state[_Comprehension].enter()
+    node.generators = self.visit_block(node.generators)
+    node.key = self.visit(node.key)
+    node.value = self.visit(node.value)
+    self.state[_Comprehension].exit()
+    return node
+
+  def visit_ListComp(self, node):
+    return self._process_iterable_comprehension(node)
+
+  def visit_SetComp(self, node):
+    return self._process_iterable_comprehension(node)
+
+  def visit_GeneratorExp(self, node):
+    return self._process_iterable_comprehension(node)
+
   def visit_arguments(self, node):
     return self._process_statement(node)
 
   def visit_FunctionDef(self, node):
     # The FunctionDef node itself has a Scope object that tracks the creation
-    # of its name, along with the usage of any decorator accompany it.
+    # of its name, along with the usage of any decorator accompanying it.
     self._enter_scope(False)
     node.decorator_list = self.visit_block(node.decorator_list)
     self.scope.mark_modified(qual_names.QN(node.name))
diff --git a/tensorflow/python/autograph/pyct/static_analysis/liveness_test.py b/tensorflow/python/autograph/pyct/static_analysis/liveness_test.py
index 9738c6d0b4..f14b1a3e79 100644
--- a/tensorflow/python/autograph/pyct/static_analysis/liveness_test.py
+++ b/tensorflow/python/autograph/pyct/static_analysis/liveness_test.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import six
+
 from tensorflow.python.autograph.pyct import anno
 from tensorflow.python.autograph.pyct import cfg
 from tensorflow.python.autograph.pyct import parser
@@ -243,6 +245,62 @@ class LivenessTest(test.TestCase):
 
     self.assertHasLiveIn(fn_body[0], ('a', 'x', 'y'))
 
+  def test_live_in_generator_comprehension(self):
+
+    def test_fn(y):
+      if all(x for x in y):
+        return
+
+    node = self._parse_and_analyze(test_fn)
+    fn_body = node.body[0].body
+
+    if six.PY2:
+      self.assertHasLiveIn(fn_body[0], ('all', 'x', 'y'))
+    else:
+      self.assertHasLiveIn(fn_body[0], ('all', 'y'))
+
+  def test_live_in_list_comprehension(self):
+
+    def test_fn(y):
+      if [x for x in y]:
+        return
+
+    node = self._parse_and_analyze(test_fn)
+    fn_body = node.body[0].body
+
+    if six.PY2:
+      self.assertHasLiveIn(fn_body[0], ('x', 'y'))
+    else:
+      self.assertHasLiveIn(fn_body[0], ('y',))
+
+  def test_live_in_set_comprehension(self):
+
+    def test_fn(y):
+      if {x for x in y}:
+        return
+
+    node = self._parse_and_analyze(test_fn)
+    fn_body = node.body[0].body
+
+    if six.PY2:
+      self.assertHasLiveIn(fn_body[0], ('x', 'y'))
+    else:
+      self.assertHasLiveIn(fn_body[0], ('y',))
+
+  def test_live_in_dict_comprehension(self):
+
+    def test_fn(y):
+      if {k: v for k, v in y}:
+        return
+
+    node = self._parse_and_analyze(test_fn)
+    fn_body = node.body[0].body
+
+    if six.PY2:
+      self.assertHasLiveIn(fn_body[0], ('k', 'v', 'y'))
+    else:
+      self.assertHasLiveIn(fn_body[0], ('y',))
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/autograph/pyct/static_analysis/reaching_definitions_test.py b/tensorflow/python/autograph/pyct/static_analysis/reaching_definitions_test.py
index fd91721c74..848c5460e6 100644
--- a/tensorflow/python/autograph/pyct/static_analysis/reaching_definitions_test.py
+++ b/tensorflow/python/autograph/pyct/static_analysis/reaching_definitions_test.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import six
+
 from tensorflow.python.autograph.pyct import anno
 from tensorflow.python.autograph.pyct import cfg
 from tensorflow.python.autograph.pyct import parser
@@ -294,6 +296,24 @@ class DefinitionInfoTest(test.TestCase):
     self.assertNotSameDef(source, target)
     self.assertSameDef(target, retval)
 
+  def test_comprehension_leaking(self):
+
+    def test_fn(a):
+      all(x for x in a)
+      return x  # pylint:disable=undefined-variable
+
+    node = self._parse_and_analyze(test_fn)
+    fn_body = node.body[0].body
+
+    listcomp_target = fn_body[0].value.args[0].generators[0].target
+    retval = fn_body[1].value
+
+    # Python2 leaks comprehension symbols. Python3 doesn't.
+    if six.PY2:
+      self.assertSameDef(retval, listcomp_target)
+    else:
+      self.assertHasDefs(retval, 0)
+
 
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 983a547e85b5cd6e6abf62bacc0e2370d474577b Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Thu, 31 Jan 2019 13:15:04 -0800
Subject: [PATCH 1669/2345] [XLA] Make replay_computation not crash when
 compile fails.

PiperOrigin-RevId: 231840092
---
 .../compiler/xla/tools/replay_computation.cc   | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/tensorflow/compiler/xla/tools/replay_computation.cc b/tensorflow/compiler/xla/tools/replay_computation.cc
index c01a47b510..21217c23f6 100644
--- a/tensorflow/compiler/xla/tools/replay_computation.cc
+++ b/tensorflow/compiler/xla/tools/replay_computation.cc
@@ -90,8 +90,8 @@ struct Options {
   int num_runs = 1;
 };
 
-std::unique_ptr<LocalExecutable> CompileExecutable(const HloSnapshot& module,
-                                                   LocalClient* client) {
+StatusOr<std::unique_ptr<LocalExecutable>> CompileExecutable(
+    const HloSnapshot& module, LocalClient* client) {
   XlaComputation computation(module.hlo().hlo_module());
   std::vector<Shape> argument_layouts;
   argument_layouts.reserve(
@@ -102,9 +102,8 @@ std::unique_ptr<LocalExecutable> CompileExecutable(const HloSnapshot& module,
     argument_layouts.push_back(Shape(param));
     argument_layout_ptrs.push_back(&argument_layouts.back());
   }
-  return client
-      ->Compile(computation, argument_layout_ptrs, ExecutableBuildOptions())
-      .ValueOrDie();
+  return client->Compile(computation, argument_layout_ptrs,
+                         ExecutableBuildOptions());
 }
 
 absl::optional<Shape> GetXfeedShape(bool is_infeed,
@@ -357,7 +356,7 @@ int RealMain(absl::Span<char* const> args, const Options& opts) {
 
   // Compile all the modules in parallel.
   LOG(INFO) << "Compiling " << snapshots.size() << " modules in parallel.";
-  std::vector<std::unique_ptr<LocalExecutable>> executables;
+  std::vector<StatusOr<std::unique_ptr<LocalExecutable>>> executables;
   {
     // ThreadPool CHECK-fails if we give it 0 threads.
     tensorflow::thread::ThreadPool thread_pool(
@@ -374,7 +373,12 @@ int RealMain(absl::Span<char* const> args, const Options& opts) {
   LOG(INFO) << "Done compiling; now running the modules.";
 
   for (int64 i = 0; i < executables.size(); ++i) {
-    LocalExecutable* executable = executables[i].get();
+    if (!executables[i].ok()) {
+      LOG(ERROR) << "Compilation failed: " << executables[i].status();
+      exit_status = EXIT_FAILURE;
+      continue;
+    }
+    LocalExecutable* executable = executables[i].ValueOrDie().get();
     LOG(ERROR) << "Running iteration " << i;
     StatusOr<Literal> result_status =
         ReplayComputation(snapshots[i], executable, client, opts);
-- 
GitLab


From e41cb124cd0b325821af85cdacd9d8a12e206418 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Thu, 31 Jan 2019 13:23:48 -0800
Subject: [PATCH 1670/2345] Prevent null pointer dereference in decode_gif.

PiperOrigin-RevId: 231841542
---
 tensorflow/core/lib/gif/gif_io.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/core/lib/gif/gif_io.cc b/tensorflow/core/lib/gif/gif_io.cc
index ce842e9723..dc5406920a 100644
--- a/tensorflow/core/lib/gif/gif_io.cc
+++ b/tensorflow/core/lib/gif/gif_io.cc
@@ -140,6 +140,10 @@ uint8* Decode(const void* srcdata, int datasize,
     ColorMapObject* color_map = this_image->ImageDesc.ColorMap
                                     ? this_image->ImageDesc.ColorMap
                                     : gif_file->SColorMap;
+    if (color_map == nullptr) {
+      *error_string = strings::StrCat("missing color map for frame ", k);
+      return nullptr;
+    }
 
     for (int i = imgTop; i < imgBottom; ++i) {
       uint8* p_dst = this_dst + i * width * channel;
-- 
GitLab


From 309a00dbb406963e74f9b9dc5afdb7bb391fece6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 31 Jan 2019 13:32:16 -0800
Subject: [PATCH 1671/2345] Add add_n to the op schema.

PiperOrigin-RevId: 231843455
---
 tensorflow/lite/builtin_ops.h                 |   1 +
 .../lite/core/api/flatbuffer_conversions.cc   |   1 +
 tensorflow/lite/nnapi_delegate.cc             |   1 +
 tensorflow/lite/schema/schema.fbs             |   5 +
 tensorflow/lite/schema/schema_generated.h     | 124 +++++++++++++++++-
 5 files changed, 126 insertions(+), 6 deletions(-)

diff --git a/tensorflow/lite/builtin_ops.h b/tensorflow/lite/builtin_ops.h
index b6ffb826f5..fc871d3592 100644
--- a/tensorflow/lite/builtin_ops.h
+++ b/tensorflow/lite/builtin_ops.h
@@ -131,6 +131,7 @@ typedef enum {
   kTfLiteBuiltinUnique = 103,
   kTfLiteBuiltinCeil = 104,
   kTfLiteBuiltinReverseV2 = 105,
+  kTfLiteBuiltinAddN = 106,
 } TfLiteBuiltinOperator;
 
 #ifdef __cplusplus
diff --git a/tensorflow/lite/core/api/flatbuffer_conversions.cc b/tensorflow/lite/core/api/flatbuffer_conversions.cc
index 3a74b1e6c9..175caa1afa 100644
--- a/tensorflow/lite/core/api/flatbuffer_conversions.cc
+++ b/tensorflow/lite/core/api/flatbuffer_conversions.cc
@@ -727,6 +727,7 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type,
     case BuiltinOperator_RANGE:
     case BuiltinOperator_SQUARED_DIFFERENCE:
     case BuiltinOperator_REVERSE_V2:
+    case BuiltinOperator_ADD_N:
       break;
   }
   return kTfLiteOk;
diff --git a/tensorflow/lite/nnapi_delegate.cc b/tensorflow/lite/nnapi_delegate.cc
index f7cb1588e1..7c67468879 100644
--- a/tensorflow/lite/nnapi_delegate.cc
+++ b/tensorflow/lite/nnapi_delegate.cc
@@ -664,6 +664,7 @@ TfLiteStatus AddOpsAndParams(
       case tflite::BuiltinOperator_UNIQUE:
       case tflite::BuiltinOperator_CEIL:
       case tflite::BuiltinOperator_REVERSE_V2:
+      case tflite::BuiltinOperator_ADD_N:
         logError("Op code %d is currently not delegated to NNAPI", builtin);
         return kTfLiteError;
         break;
diff --git a/tensorflow/lite/schema/schema.fbs b/tensorflow/lite/schema/schema.fbs
index fbcb18fbeb..098b7c7193 100644
--- a/tensorflow/lite/schema/schema.fbs
+++ b/tensorflow/lite/schema/schema.fbs
@@ -219,6 +219,7 @@ enum BuiltinOperator : byte {
   UNIQUE = 103,
   CEIL = 104,
   REVERSE_V2 = 105,
+  ADD_N = 106,
 }
 
 // Options for the builtin operators.
@@ -304,6 +305,7 @@ union BuiltinOptions {
   SplitVOptions,
   UniqueOptions,
   ReverseV2Options,
+  AddNOptions,
 }
 
 enum Padding : byte { SAME, VALID }
@@ -724,6 +726,9 @@ table UniqueOptions {
 table ReverseV2Options {
 }
 
+table AddNOptions {
+}
+
 // An OperatorCode can be an enum value (BuiltinOperator) if the operator is a
 // builtin, or a string if the operator is custom.
 table OperatorCode {
diff --git a/tensorflow/lite/schema/schema_generated.h b/tensorflow/lite/schema/schema_generated.h
index 6ad7df050e..f0a6b00f96 100755
--- a/tensorflow/lite/schema/schema_generated.h
+++ b/tensorflow/lite/schema/schema_generated.h
@@ -274,6 +274,9 @@ struct UniqueOptionsT;
 struct ReverseV2Options;
 struct ReverseV2OptionsT;
 
+struct AddNOptions;
+struct AddNOptionsT;
+
 struct OperatorCode;
 struct OperatorCodeT;
 
@@ -529,11 +532,12 @@ enum BuiltinOperator {
   BuiltinOperator_UNIQUE = 103,
   BuiltinOperator_CEIL = 104,
   BuiltinOperator_REVERSE_V2 = 105,
+  BuiltinOperator_ADD_N = 106,
   BuiltinOperator_MIN = BuiltinOperator_ADD,
-  BuiltinOperator_MAX = BuiltinOperator_REVERSE_V2
+  BuiltinOperator_MAX = BuiltinOperator_ADD_N
 };
 
-inline const BuiltinOperator (&EnumValuesBuiltinOperator())[105] {
+inline const BuiltinOperator (&EnumValuesBuiltinOperator())[106] {
   static const BuiltinOperator values[] = {
     BuiltinOperator_ADD,
     BuiltinOperator_AVERAGE_POOL_2D,
@@ -639,7 +643,8 @@ inline const BuiltinOperator (&EnumValuesBuiltinOperator())[105] {
     BuiltinOperator_SPLIT_V,
     BuiltinOperator_UNIQUE,
     BuiltinOperator_CEIL,
-    BuiltinOperator_REVERSE_V2
+    BuiltinOperator_REVERSE_V2,
+    BuiltinOperator_ADD_N
   };
   return values;
 }
@@ -752,6 +757,7 @@ inline const char * const *EnumNamesBuiltinOperator() {
     "UNIQUE",
     "CEIL",
     "REVERSE_V2",
+    "ADD_N",
     nullptr
   };
   return names;
@@ -845,11 +851,12 @@ enum BuiltinOptions {
   BuiltinOptions_SplitVOptions = 79,
   BuiltinOptions_UniqueOptions = 80,
   BuiltinOptions_ReverseV2Options = 81,
+  BuiltinOptions_AddNOptions = 82,
   BuiltinOptions_MIN = BuiltinOptions_NONE,
-  BuiltinOptions_MAX = BuiltinOptions_ReverseV2Options
+  BuiltinOptions_MAX = BuiltinOptions_AddNOptions
 };
 
-inline const BuiltinOptions (&EnumValuesBuiltinOptions())[82] {
+inline const BuiltinOptions (&EnumValuesBuiltinOptions())[83] {
   static const BuiltinOptions values[] = {
     BuiltinOptions_NONE,
     BuiltinOptions_Conv2DOptions,
@@ -932,7 +939,8 @@ inline const BuiltinOptions (&EnumValuesBuiltinOptions())[82] {
     BuiltinOptions_AbsOptions,
     BuiltinOptions_SplitVOptions,
     BuiltinOptions_UniqueOptions,
-    BuiltinOptions_ReverseV2Options
+    BuiltinOptions_ReverseV2Options,
+    BuiltinOptions_AddNOptions
   };
   return values;
 }
@@ -1021,6 +1029,7 @@ inline const char * const *EnumNamesBuiltinOptions() {
     "SplitVOptions",
     "UniqueOptions",
     "ReverseV2Options",
+    "AddNOptions",
     nullptr
   };
   return names;
@@ -1359,6 +1368,10 @@ template<> struct BuiltinOptionsTraits<ReverseV2Options> {
   static const BuiltinOptions enum_value = BuiltinOptions_ReverseV2Options;
 };
 
+template<> struct BuiltinOptionsTraits<AddNOptions> {
+  static const BuiltinOptions enum_value = BuiltinOptions_AddNOptions;
+};
+
 struct BuiltinOptionsUnion {
   BuiltinOptions type;
   void *value;
@@ -2038,6 +2051,14 @@ struct BuiltinOptionsUnion {
     return type == BuiltinOptions_ReverseV2Options ?
       reinterpret_cast<const ReverseV2OptionsT *>(value) : nullptr;
   }
+  AddNOptionsT *AsAddNOptions() {
+    return type == BuiltinOptions_AddNOptions ?
+      reinterpret_cast<AddNOptionsT *>(value) : nullptr;
+  }
+  const AddNOptionsT *AsAddNOptions() const {
+    return type == BuiltinOptions_AddNOptions ?
+      reinterpret_cast<const AddNOptionsT *>(value) : nullptr;
+  }
 };
 
 bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *obj, BuiltinOptions type);
@@ -7174,6 +7195,46 @@ inline flatbuffers::Offset<ReverseV2Options> CreateReverseV2Options(
 
 flatbuffers::Offset<ReverseV2Options> CreateReverseV2Options(flatbuffers::FlatBufferBuilder &_fbb, const ReverseV2OptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
 
+struct AddNOptionsT : public flatbuffers::NativeTable {
+  typedef AddNOptions TableType;
+  AddNOptionsT() {
+  }
+};
+
+struct AddNOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef AddNOptionsT NativeTableType;
+  bool Verify(flatbuffers::Verifier &verifier) const {
+    return VerifyTableStart(verifier) &&
+           verifier.EndTable();
+  }
+  AddNOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const;
+  void UnPackTo(AddNOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const;
+  static flatbuffers::Offset<AddNOptions> Pack(flatbuffers::FlatBufferBuilder &_fbb, const AddNOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
+};
+
+struct AddNOptionsBuilder {
+  flatbuffers::FlatBufferBuilder &fbb_;
+  flatbuffers::uoffset_t start_;
+  explicit AddNOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb)
+        : fbb_(_fbb) {
+    start_ = fbb_.StartTable();
+  }
+  AddNOptionsBuilder &operator=(const AddNOptionsBuilder &);
+  flatbuffers::Offset<AddNOptions> Finish() {
+    const auto end = fbb_.EndTable(start_);
+    auto o = flatbuffers::Offset<AddNOptions>(end);
+    return o;
+  }
+};
+
+inline flatbuffers::Offset<AddNOptions> CreateAddNOptions(
+    flatbuffers::FlatBufferBuilder &_fbb) {
+  AddNOptionsBuilder builder_(_fbb);
+  return builder_.Finish();
+}
+
+flatbuffers::Offset<AddNOptions> CreateAddNOptions(flatbuffers::FlatBufferBuilder &_fbb, const AddNOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
+
 struct OperatorCodeT : public flatbuffers::NativeTable {
   typedef OperatorCode TableType;
   BuiltinOperator builtin_code;
@@ -7550,6 +7611,9 @@ struct Operator FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
   const ReverseV2Options *builtin_options_as_ReverseV2Options() const {
     return builtin_options_type() == BuiltinOptions_ReverseV2Options ? static_cast<const ReverseV2Options *>(builtin_options()) : nullptr;
   }
+  const AddNOptions *builtin_options_as_AddNOptions() const {
+    return builtin_options_type() == BuiltinOptions_AddNOptions ? static_cast<const AddNOptions *>(builtin_options()) : nullptr;
+  }
   const flatbuffers::Vector<uint8_t> *custom_options() const {
     return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_CUSTOM_OPTIONS);
   }
@@ -7905,6 +7969,10 @@ template<> inline const ReverseV2Options *Operator::builtin_options_as<ReverseV2
   return builtin_options_as_ReverseV2Options();
 }
 
+template<> inline const AddNOptions *Operator::builtin_options_as<AddNOptions>() const {
+  return builtin_options_as_AddNOptions();
+}
+
 struct OperatorBuilder {
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
@@ -10575,6 +10643,29 @@ inline flatbuffers::Offset<ReverseV2Options> CreateReverseV2Options(flatbuffers:
       _fbb);
 }
 
+inline AddNOptionsT *AddNOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
+  auto _o = new AddNOptionsT();
+  UnPackTo(_o, _resolver);
+  return _o;
+}
+
+inline void AddNOptions::UnPackTo(AddNOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const {
+  (void)_o;
+  (void)_resolver;
+}
+
+inline flatbuffers::Offset<AddNOptions> AddNOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const AddNOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) {
+  return CreateAddNOptions(_fbb, _o, _rehasher);
+}
+
+inline flatbuffers::Offset<AddNOptions> CreateAddNOptions(flatbuffers::FlatBufferBuilder &_fbb, const AddNOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) {
+  (void)_rehasher;
+  (void)_o;
+  struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const AddNOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va;
+  return tflite::CreateAddNOptions(
+      _fbb);
+}
+
 inline OperatorCodeT *OperatorCode::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
   auto _o = new OperatorCodeT();
   UnPackTo(_o, _resolver);
@@ -11157,6 +11248,10 @@ inline bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *ob
       auto ptr = reinterpret_cast<const ReverseV2Options *>(obj);
       return verifier.VerifyTable(ptr);
     }
+    case BuiltinOptions_AddNOptions: {
+      auto ptr = reinterpret_cast<const AddNOptions *>(obj);
+      return verifier.VerifyTable(ptr);
+    }
     default: return false;
   }
 }
@@ -11499,6 +11594,10 @@ inline void *BuiltinOptionsUnion::UnPack(const void *obj, BuiltinOptions type, c
       auto ptr = reinterpret_cast<const ReverseV2Options *>(obj);
       return ptr->UnPack(resolver);
     }
+    case BuiltinOptions_AddNOptions: {
+      auto ptr = reinterpret_cast<const AddNOptions *>(obj);
+      return ptr->UnPack(resolver);
+    }
     default: return nullptr;
   }
 }
@@ -11829,6 +11928,10 @@ inline flatbuffers::Offset<void> BuiltinOptionsUnion::Pack(flatbuffers::FlatBuff
       auto ptr = reinterpret_cast<const ReverseV2OptionsT *>(value);
       return CreateReverseV2Options(_fbb, ptr, _rehasher).Union();
     }
+    case BuiltinOptions_AddNOptions: {
+      auto ptr = reinterpret_cast<const AddNOptionsT *>(value);
+      return CreateAddNOptions(_fbb, ptr, _rehasher).Union();
+    }
     default: return 0;
   }
 }
@@ -12159,6 +12262,10 @@ inline BuiltinOptionsUnion::BuiltinOptionsUnion(const BuiltinOptionsUnion &u) FL
       value = new ReverseV2OptionsT(*reinterpret_cast<ReverseV2OptionsT *>(u.value));
       break;
     }
+    case BuiltinOptions_AddNOptions: {
+      value = new AddNOptionsT(*reinterpret_cast<AddNOptionsT *>(u.value));
+      break;
+    }
     default:
       break;
   }
@@ -12571,6 +12678,11 @@ inline void BuiltinOptionsUnion::Reset() {
       delete ptr;
       break;
     }
+    case BuiltinOptions_AddNOptions: {
+      auto ptr = reinterpret_cast<AddNOptionsT *>(value);
+      delete ptr;
+      break;
+    }
     default: break;
   }
   value = nullptr;
-- 
GitLab


From 25062cbbfe8c7c049a5014dfc64cdd839b13fad4 Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Thu, 31 Jan 2019 13:35:12 -0800
Subject: [PATCH 1672/2345] [Grappler] Add updating node and converting all
 regular fanins to controlling fanins helpers to MutableGraphView.

PiperOrigin-RevId: 231844047
---
 .../core/grappler/mutable_graph_view.cc       | 260 ++++++++++++++----
 tensorflow/core/grappler/mutable_graph_view.h |  28 ++
 .../core/grappler/mutable_graph_view_test.cc  | 199 ++++++++++++++
 3 files changed, 440 insertions(+), 47 deletions(-)

diff --git a/tensorflow/core/grappler/mutable_graph_view.cc b/tensorflow/core/grappler/mutable_graph_view.cc
index e56431090f..64098e7e28 100644
--- a/tensorflow/core/grappler/mutable_graph_view.cc
+++ b/tensorflow/core/grappler/mutable_graph_view.cc
@@ -94,6 +94,18 @@ bool CanDedupControlWithRegularInput(const MutableGraphView& graph,
   return CanDedupControlWithRegularInput(graph, *control_node);
 }
 
+bool HasRegularFaninNode(const MutableGraphView& graph, const NodeDef& node,
+                         absl::string_view fanin_node_name) {
+  const int num_regular_fanins =
+      graph.NumFanins(node, /*include_controlling_nodes=*/false);
+  for (int i = 0; i < num_regular_fanins; ++i) {
+    if (ParseTensorName(node.input(i)).node() == fanin_node_name) {
+      return true;
+    }
+  }
+  return false;
+}
+
 Status MutationError(absl::string_view function_name, absl::string_view params,
                      absl::string_view msg) {
   return errors::InvalidArgument(absl::Substitute(
@@ -168,6 +180,13 @@ Status CheckPortRange(int port, int min, int max, ErrorHandler handler) {
   return Status::OK();
 }
 
+string GeneratedNameForIdentityConsumingSwitch(
+    const MutableGraphView::OutputPort& fanin) {
+  return AddPrefixToNodeName(
+      absl::StrCat(fanin.node->name(), "_", fanin.port_id),
+      kMutableGraphViewCtrl);
+}
+
 }  // namespace
 
 void MutableGraphView::AddAndDedupFanouts(NodeDef* node) {
@@ -325,6 +344,60 @@ Status MutableGraphView::AddSubgraph(GraphDef&& subgraph) {
   return Status::OK();
 }
 
+Status MutableGraphView::UpdateNode(
+    absl::string_view node_name, absl::string_view op, absl::string_view device,
+    absl::Span<const std::pair<string, AttrValue>> attrs) {
+  auto error_status = [node_name, op, device, attrs](absl::string_view msg) {
+    std::vector<string> attr_strs;
+    attr_strs.reserve(attrs.size());
+    for (const auto& attr : attrs) {
+      string attr_str = absl::Substitute("('$0', $1)", attr.first,
+                                         attr.second.ShortDebugString());
+      attr_strs.push_back(attr_str);
+    }
+    string params =
+        absl::Substitute("node_name='$0', op='$1', device='$2', attrs={$3}",
+                         node_name, op, device, absl::StrJoin(attr_strs, ", "));
+    return MutationError("UpdateNodeOp", params, msg);
+  };
+
+  NodeDef* node = GetNode(node_name);
+  TF_RETURN_IF_ERROR(CheckNodeExists(node_name, node, error_status));
+
+  MutableGraphView::OutputPort control_port(node, Graph::kControlSlot);
+  auto control_fanouts = GetFanout(control_port);
+  if (op == "Switch" && !control_fanouts.empty()) {
+    return error_status(
+        "can't change node op to Switch when node drives a control dependency "
+        "(alternatively, we could add the identity node needed, but it seems "
+        "like an unlikely event and probably a mistake)");
+  }
+
+  if (node->device() != device) {
+    node->set_device(string(device));
+  }
+  node->mutable_attr()->clear();
+  for (const auto& attr : attrs) {
+    (*node->mutable_attr())[attr.first] = attr.second;
+  }
+
+  if (node->op() == op) {
+    return Status::OK();
+  }
+
+  node->set_op(string(op));
+
+  if (CanDedupControlWithRegularInput(*this, *node)) {
+    for (const auto& control_fanout : control_fanouts) {
+      if (HasRegularFaninNode(*this, *control_fanout.node, node->name())) {
+        RemoveControllingFaninInternal(control_fanout.node, node);
+      }
+    }
+  }
+
+  return Status::OK();
+}
+
 Status MutableGraphView::UpdateFanouts(absl::string_view from_node_name,
                                        absl::string_view to_node_name) {
   NodeDef* from_node = GetNode(from_node_name);
@@ -546,29 +619,19 @@ Status MutableGraphView::AddRegularFaninByPort(absl::string_view node_name,
   return Status::OK();
 }
 
-Status MutableGraphView::AddControllingFanin(absl::string_view node_name,
-                                             const TensorId& fanin) {
-  auto error_status = [node_name, fanin](absl::string_view msg) {
-    string params = absl::Substitute("node_name='$0', fanin='$1'", node_name,
-                                     fanin.ToString());
-    return MutationError("AddControllingFanin", params, msg);
-  };
-
-  TF_RETURN_IF_ERROR(CheckFaninIsValid(fanin, error_status));
-  TF_RETURN_IF_ERROR(CheckAddingFaninToSelf(node_name, fanin, error_status));
-  NodeDef* node = GetNode(node_name);
-  TF_RETURN_IF_ERROR(CheckNodeExists(node_name, node, error_status));
-  NodeDef* fanin_node = GetNode(fanin.node());
-  TF_RETURN_IF_ERROR(CheckNodeExists(fanin.node(), fanin_node, error_status));
-
-  if (!IsSwitch(*fanin_node)) {
-    AddFaninInternal(node, {fanin_node, Graph::kControlSlot});
+NodeDef* MutableGraphView::GetControllingFaninToAdd(absl::string_view node_name,
+                                                    const OutputPort& fanin,
+                                                    string* error_msg) {
+  if (!IsSwitch(*fanin.node)) {
+    return fanin.node;
   } else {
-    if (IsTensorIdControlling(fanin)) {
+    TensorId tensor_id(fanin.node->name(), fanin.port_id);
+    if (IsOutputPortControlling(fanin)) {
       // Can't add a Switch node control dependency.
-      return error_status(absl::Substitute(
+      *error_msg = absl::Substitute(
           "can't add fanin '$0' as it will become a Switch control dependency",
-          fanin.ToString()));
+          tensor_id.ToString());
+      return nullptr;
     }
     // We can't anchor control dependencies directly on the switch node: unlike
     // other nodes only one of the outputs of the switch node will be generated
@@ -576,44 +639,76 @@ Status MutableGraphView::AddControllingFanin(absl::string_view node_name,
     // dependency is only triggered when the corresponding output is triggered.
     // We start by looking for an identity node connected to the output of the
     // switch node, and use it to anchor the control dependency.
-    auto fanouts = GetFanouts(*fanin_node, /*include_controlled_nodes=*/false);
+    auto fanouts = GetFanouts(*fanin.node, /*include_controlled_nodes=*/false);
     for (auto fanout : fanouts) {
       if (IsIdentity(*fanout.node) || IsIdentityNSingleInput(*fanout.node)) {
-        if (ParseTensorName(fanout.node->input(0)) == fanin) {
+        if (ParseTensorName(fanout.node->input(0)) == tensor_id) {
           if (fanout.node->name() == node_name) {
-            return error_status(
+            *error_msg =
                 absl::Substitute("can't add found fanin '$0' to self",
-                                 AsControlDependency(fanout.node->name())));
+                                 AsControlDependency(fanout.node->name()));
+            return nullptr;
           }
-          AddFaninInternal(node, {fanout.node, Graph::kControlSlot});
-          return Status::OK();
+          return fanout.node;
         }
       }
     }
-    // We haven't found an existing node where we can anchor the control
-    // dependency: add a new identity node.
-    string ctrl_dep_name = AddPrefixToNodeName(
-        absl::StrCat(fanin.node(), "_", fanin.index()), kMutableGraphViewCtrl);
-    if (node_name == ctrl_dep_name) {
-      return error_status(
-          absl::Substitute("can't add generated fanin '$0' to self",
-                           AsControlDependency(ctrl_dep_name)));
-    }
 
-    // Reuse a previously created node, if possible.
-    NodeDef* ctrl_dep_node = GetNode(ctrl_dep_name);
-    if (ctrl_dep_node == nullptr) {
-      NodeDef new_node;
-      new_node.set_name(ctrl_dep_name);
-      new_node.set_op("Identity");
-      new_node.set_device(fanin_node->device());
-      (*new_node.mutable_attr())["T"].set_type(
-          fanin_node->attr().at("T").type());
-      new_node.add_input(TensorIdToString(fanin));
-      ctrl_dep_node = AddNode(std::move(new_node));
+    // No node found, check if node to be created is itself.
+    if (GeneratedNameForIdentityConsumingSwitch(fanin) == node_name) {
+      *error_msg = absl::Substitute("can't add generated fanin '$0' to self",
+                                    AsControlDependency(string(node_name)));
     }
-    AddFaninInternal(node, {ctrl_dep_node, Graph::kControlSlot});
   }
+  return nullptr;
+}
+
+NodeDef* MutableGraphView::GetOrCreateIdentityConsumingSwitch(
+    const OutputPort& fanin) {
+  // We haven't found an existing node where we can anchor the control
+  // dependency: add a new identity node.
+  string identity_name = GeneratedNameForIdentityConsumingSwitch(fanin);
+  NodeDef* identity_node = GetNode(identity_name);
+  if (identity_node == nullptr) {
+    NodeDef new_node;
+    new_node.set_name(identity_name);
+    new_node.set_op("Identity");
+    new_node.set_device(fanin.node->device());
+    (*new_node.mutable_attr())["T"].set_type(fanin.node->attr().at("T").type());
+    new_node.add_input(TensorIdToString({fanin.node->name(), fanin.port_id}));
+    identity_node = AddNode(std::move(new_node));
+  }
+  return identity_node;
+}
+
+Status MutableGraphView::AddControllingFanin(absl::string_view node_name,
+                                             const TensorId& fanin) {
+  auto error_status = [node_name, fanin](absl::string_view msg) {
+    string params = absl::Substitute("node_name='$0', fanin='$1'", node_name,
+                                     fanin.ToString());
+    return MutationError("AddControllingFanin", params, msg);
+  };
+
+  TF_RETURN_IF_ERROR(CheckFaninIsValid(fanin, error_status));
+  TF_RETURN_IF_ERROR(CheckAddingFaninToSelf(node_name, fanin, error_status));
+  NodeDef* node = GetNode(node_name);
+  TF_RETURN_IF_ERROR(CheckNodeExists(node_name, node, error_status));
+  NodeDef* fanin_node = GetNode(fanin.node());
+  TF_RETURN_IF_ERROR(CheckNodeExists(fanin.node(), fanin_node, error_status));
+
+  OutputPort fanin_port(fanin_node, fanin.index());
+
+  string error_msg = "";
+  NodeDef* control_node = GetControllingFaninToAdd(
+      node_name, {fanin_node, fanin.index()}, &error_msg);
+  if (!error_msg.empty()) {
+    return error_status(error_msg);
+  }
+  if (control_node == nullptr) {
+    control_node = GetOrCreateIdentityConsumingSwitch(fanin_port);
+  }
+  AddFaninInternal(node, {control_node, Graph::kControlSlot});
+
   return Status::OK();
 }
 
@@ -990,6 +1085,77 @@ Status MutableGraphView::SwapRegularFaninsByPorts(absl::string_view node_name,
   return Status::OK();
 }
 
+Status MutableGraphView::UpdateAllRegularFaninsToControlling(
+    absl::string_view node_name) {
+  auto error_status = [node_name](absl::string_view msg) {
+    string params = absl::Substitute("node_name='$0'", node_name);
+    return MutationError("UpdateAllRegularFaninsToControlling", params, msg);
+  };
+
+  NodeDef* node = GetNode(node_name);
+  TF_RETURN_IF_ERROR(CheckNodeExists(node_name, node, error_status));
+
+  const int num_regular_fanins =
+      NumFanins(*node, /*include_controlling_nodes=*/false);
+  std::vector<OutputPort> regular_fanins;
+  regular_fanins.reserve(num_regular_fanins);
+  std::vector<NodeDef*> controlling_fanins;
+  controlling_fanins.reserve(num_regular_fanins);
+
+  // Get all regular fanins and derive controlling fanins.
+  for (int i = 0; i < num_regular_fanins; ++i) {
+    TensorId tensor_id = ParseTensorName(node->input(i));
+    OutputPort fanin_port(nodes()[tensor_id.node()], tensor_id.index());
+
+    string error_msg = "";
+    NodeDef* control_node =
+        GetControllingFaninToAdd(node_name, fanin_port, &error_msg);
+    if (!error_msg.empty()) {
+      return error_status(error_msg);
+    }
+
+    regular_fanins.push_back(fanin_port);
+    controlling_fanins.push_back(control_node);
+  }
+
+  // Replace regular fanins with controlling fanins and dedup.
+  int pos = 0;
+  InputPort input_port(node, Graph::kControlSlot);
+  absl::flat_hash_set<absl::string_view> controls;
+  for (int i = 0; i < num_regular_fanins; ++i) {
+    OutputPort fanin_port = regular_fanins[i];
+    NodeDef* control = controlling_fanins[i];
+    if (control == nullptr) {
+      control = GetOrCreateIdentityConsumingSwitch(fanin_port);
+    }
+    fanouts()[fanin_port].erase({node, i});
+    if (controls.contains(control->name())) {
+      continue;
+    }
+    controls.insert(control->name());
+    node->set_input(pos, AsControlDependency(control->name()));
+    fanouts()[{control, Graph::kControlSlot}].insert(input_port);
+    ++pos;
+  }
+
+  // Shift existing controlling fanins and dedup.
+  for (int i = num_regular_fanins; i < node->input_size(); ++i) {
+    TensorId tensor_id = ParseTensorName(node->input(i));
+    if (controls.contains(tensor_id.node())) {
+      continue;
+    }
+    controls.insert(tensor_id.node());
+    node->mutable_input()->SwapElements(pos, i);
+    ++pos;
+  }
+
+  // Remove duplicate controls and leftover regular fanins.
+  node->mutable_input()->DeleteSubrange(pos, node->input_size() - pos);
+  max_regular_input_port().erase(node);
+
+  return Status::OK();
+}
+
 Status MutableGraphView::CheckNodesCanBeDeleted(
     const absl::flat_hash_set<string>& nodes_to_delete) {
   std::vector<string> missing_nodes;
diff --git a/tensorflow/core/grappler/mutable_graph_view.h b/tensorflow/core/grappler/mutable_graph_view.h
index 16ef832a4e..08a1cd18a1 100644
--- a/tensorflow/core/grappler/mutable_graph_view.h
+++ b/tensorflow/core/grappler/mutable_graph_view.h
@@ -76,6 +76,14 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // underlying graph, which leaves subgraph in valid but undefined state.
   Status AddSubgraph(GraphDef&& subgraph);
 
+  // Updates node `node_name` op, device, and attributes. This will clear any
+  // existing attributes. If it is not possible to update the node or if the
+  // node does not exist, an error will be returned and nothing will be modified
+  // in the graph.
+  Status UpdateNode(absl::string_view node_name, absl::string_view op,
+                    absl::string_view device,
+                    absl::Span<const std::pair<string, AttrValue>> attrs);
+
   // Updates all fanouts (input ports fetching output tensors) from
   // `from_node_name` to the `to_node_name`, including control dependencies.
   //
@@ -196,6 +204,11 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   Status SwapRegularFaninsByPorts(absl::string_view node_name, int from_port,
                                   int to_port);
 
+  // Updates all regular fanins to equivalent controlling fanins. If it is not
+  // possible, an error will be returned and nothing will be modified in the
+  // graph.
+  Status UpdateAllRegularFaninsToControlling(absl::string_view node_name);
+
   // Deletes nodes from the graph. If a node can't be safely removed,
   // specifically if a node still has fanouts, an error will be returned. Nodes
   // that can't be found are ignored.
@@ -241,6 +254,21 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // added after existing non control dependency inputs.
   bool AddFaninInternal(NodeDef* node, const OutputPort& fanin);
 
+  // Finds control dependency node to be used based on fanin. If fanin is not a
+  // Switch node, fanin.node is simply returned. Otherwise this will try to find
+  // a candidate Identity node consuming fanin, as the control dependency. If it
+  // is not possible or will introduce a self loop, an error message will be
+  // set. If nullptr is returned with no error
+  // GetOrCreateIdentityConsumingSwitch should be called to generate the new
+  // Identity node.
+  NodeDef* GetControllingFaninToAdd(absl::string_view node_name,
+                                    const OutputPort& fanin, string* error_msg);
+
+  // Finds a generated Identity node consuming Switch node `fanin.node` at port
+  // `fanin.port_id`. If such a node does not exist, a new Identity node will be
+  // created.
+  NodeDef* GetOrCreateIdentityConsumingSwitch(const OutputPort& fanin);
+
   // Removes all instances of regular fanin `fanin` from node `node`.
   bool RemoveRegularFaninInternal(NodeDef* node, const OutputPort& fanin);
 
diff --git a/tensorflow/core/grappler/mutable_graph_view_test.cc b/tensorflow/core/grappler/mutable_graph_view_test.cc
index f4ee4d89b1..06333d3a34 100644
--- a/tensorflow/core/grappler/mutable_graph_view_test.cc
+++ b/tensorflow/core/grappler/mutable_graph_view_test.cc
@@ -209,6 +209,102 @@ TEST(MutableGraphViewTest, AddSubgraphAndFailIfFunctionDifferent) {
             "different function definition with the same name: XTimesTwo.");
 }
 
+TEST(MutableGraphViewTest, UpdateNodeNoDedupControlDependency) {
+  constexpr char kDevice[] = "/device:foo:0";
+  GraphDef graph_def = test::function::GDef(
+      {NDef("bar_1", "Switch", {}, {}), NDef("bar_2", "Identity", {"bar_1:1"}),
+       NDef("other", "NotImportant", {}, {}),
+       NDef("foo_1", "NotImportant", {"bar_2", "other", "bar_2:1", "^bar_2"}),
+       NDef("foo_2", "NotImportant", {"other:1", "bar_2:2", "^bar_2"})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  AttrValue list_value;
+  list_value.mutable_list()->add_type(DT_FLOAT);
+  TF_EXPECT_OK(
+      graph.UpdateNode("bar_2", "IdentityN", kDevice, {{"T", list_value}}));
+
+  CheckNode(graph, "bar_1", "Switch", "", {}, {}, {"bar_2"});
+  CheckNode(graph, "bar_2", "IdentityN", kDevice, {{"T", list_value}},
+            {"bar_1:1"}, {"foo_1", "foo_1:2", "^foo_1", "foo_2:1", "^foo_2"});
+  CheckNode(graph, "other", "NotImportant", "", {}, {}, {"foo_1:1", "foo_2"});
+  CheckNode(graph, "foo_1", "NotImportant", "", {},
+            {"bar_2", "other", "bar_2:1", "^bar_2"}, {});
+  CheckNode(graph, "foo_2", "NotImportant", "", {},
+            {"other:1", "bar_2:2", "^bar_2"}, {});
+
+  CheckGraph(graph);
+}
+
+TEST(MutableGraphViewTest, UpdateNodeDedupControlDependency) {
+  constexpr char kDevice[] = "/device:foo:0";
+  GraphDef graph_def = test::function::GDef(
+      {NDef("bar_1", "Switch", {}, {}), NDef("bar_2", "Identity", {"bar_1:1"}),
+       NDef("other", "NotImportant", {}, {}),
+       NDef("foo_1", "NotImportant", {"bar_2", "other", "bar_2:1", "^bar_2"}),
+       NDef("foo_2", "NotImportant", {"other:1", "bar_2:2", "^bar_2"})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  TF_EXPECT_OK(graph.UpdateNode("bar_2", "NotImportant", kDevice, {}));
+
+  CheckNode(graph, "bar_1", "Switch", "", {}, {}, {"bar_2"});
+  CheckNode(graph, "bar_2", "NotImportant", kDevice, {}, {"bar_1:1"},
+            {"foo_1", "foo_1:2", "foo_2:1"});
+  CheckNode(graph, "other", "NotImportant", "", {}, {}, {"foo_1:1", "foo_2"});
+  CheckNode(graph, "foo_1", "NotImportant", "", {},
+            {"bar_2", "other", "bar_2:1"}, {});
+  CheckNode(graph, "foo_2", "NotImportant", "", {}, {"other:1", "bar_2:2"}, {});
+
+  CheckGraph(graph);
+}
+
+TEST(MutableGraphViewTest, UpdateNodeSwitchNoControlDependency) {
+  constexpr char kDevice[] = "/device:foo:0";
+  GraphDef graph_def =
+      test::function::GDef({NDef("foo", "NotImportant", {}, {}),
+                            NDef("bar", "NotImportant", {"foo:1"})},
+                           /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  TF_EXPECT_OK(graph.UpdateNode("foo", "Switch", kDevice, {}));
+
+  CheckNode(graph, "foo", "Switch", kDevice, {}, {}, {"bar"});
+  CheckNode(graph, "bar", "NotImportant", "", {}, {"foo:1"}, {});
+
+  CheckGraph(graph);
+}
+
+TEST(MutableGraphViewTest, UpdateNodeSwitchControlDependency) {
+  constexpr char kDevice[] = "/device:foo:0";
+  GraphDef graph_def =
+      test::function::GDef({NDef("foo", "NotImportant", {}, {}),
+                            NDef("bar", "NotImportant", {"^foo"})},
+                           /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  AttrValue attr;
+  attr.set_type(DT_FLOAT);
+  Status s = graph.UpdateNode("foo", "Switch", kDevice, {{"T", attr}});
+  EXPECT_FALSE(s.ok());
+  string expected_msg =
+      "MutableGraphView::UpdateNodeOp(node_name='foo', op='Switch', "
+      "device='/device:foo:0', attrs={('T', type: DT_FLOAT)}) error: can't "
+      "change node op to Switch when node drives a control dependency "
+      "(alternatively, we could add the identity node needed, but it seems "
+      "like an unlikely event and probably a mistake).";
+  EXPECT_EQ(s.error_message(), expected_msg);
+
+  CheckNode(graph, "foo", "NotImportant", "", {}, {}, {"^bar"});
+  CheckNode(graph, "bar", "NotImportant", "", {}, {"^foo"}, {});
+
+  CheckGraph(graph);
+}
+
 TEST(MutableGraphViewTest, AddAndUpdateFanouts) {
   // Actual node.op() is not important in this test.
   GraphDef graph_def = test::function::GDef(
@@ -2216,6 +2312,109 @@ TEST(MutableGraphViewTest, RemoveControllingFaninSelfLoop) {
   CheckGraph(graph);
 }
 
+void TestUpdateAllRegularFaninsToControlling(
+    absl::string_view node_name, bool node_exists, bool success,
+    const string& error_msg, absl::Span<const string> expected_fanins) {
+  constexpr char kDevice[] = "/device:foo:0";
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}),
+       NDef("switch", "Switch", {}, {{"T", DT_FLOAT}}, kDevice),
+       NDef("b", "NotImportant", {"switch:1"}, {}),
+       NDef("ConstantFoldingCtrl/switch_1", "Identity", {"switch:1"},
+            {{"T", DT_FLOAT}}, kDevice),
+       NDef("c", "NotImportant", {"a", "^b"}, {}),
+       NDef("d", "NotImportant", {"b", "c"}, {}),
+       NDef("e", "NotImportant", {"^d"}, {})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  NodeDef* node = graph.GetNode(node_name);
+  if (node_exists) {
+    EXPECT_NE(node, nullptr);
+  } else {
+    EXPECT_EQ(node, nullptr);
+  }
+
+  absl::flat_hash_map<string, std::vector<string>> unmodified_node_inputs =
+      GetNodeInputsFromGraph(graph_def, node_name);
+
+  Status s = graph.UpdateAllRegularFaninsToControlling(node_name);
+  EXPECT_EQ(s.ok(), success);
+  if (!success) {
+    EXPECT_EQ(s.error_message(), error_msg);
+  }
+  if (node_exists) {
+    CompareNodeFanins(graph, node, expected_fanins);
+  }
+
+  CheckUnmodifiedNodeFanins(graph_def, node_name, unmodified_node_inputs);
+
+  CheckGraph(graph);
+}
+
+TEST(MutableGraphViewTest, UpdateAllRegularFaninsToControlling) {
+  string error_msg;
+  // Nodes with some regular fanins and some controls.
+  TestUpdateAllRegularFaninsToControlling("a", /*node_exists=*/true,
+                                          /*success=*/true, error_msg, {});
+  TestUpdateAllRegularFaninsToControlling("c", /*node_exists=*/true,
+                                          /*success=*/true, error_msg,
+                                          {"^a", "^b"});
+  TestUpdateAllRegularFaninsToControlling("d", /*node_exists=*/true,
+                                          /*success=*/true, error_msg,
+                                          {"^b", "^c"});
+  TestUpdateAllRegularFaninsToControlling("e", /*node_exists=*/true,
+                                          /*success=*/true, error_msg, {"^d"});
+
+  // Use existing Identity to pin control dependency of Switch.
+  TestUpdateAllRegularFaninsToControlling("b", /*node_exists=*/true,
+                                          /*success=*/true, error_msg,
+                                          {"^ConstantFoldingCtrl/switch_1"});
+
+  // Missing node.
+  error_msg =
+      "MutableGraphView::UpdateAllRegularFaninsToControlling(node_name='f') "
+      "error: node 'f' was not found.";
+  TestUpdateAllRegularFaninsToControlling("f", /*node_exists=*/false,
+                                          /*success=*/false, error_msg, {});
+
+  // Error in getting controlling fanin.
+  error_msg =
+      "MutableGraphView::UpdateAllRegularFaninsToControlling(node_name='"
+      "ConstantFoldingCtrl/switch_1') error: can't add found fanin "
+      "'^ConstantFoldingCtrl/switch_1' to self.";
+  TestUpdateAllRegularFaninsToControlling("ConstantFoldingCtrl/switch_1",
+                                          /*node_exists=*/true,
+                                          /*success=*/false, error_msg,
+                                          {"switch:1"});
+}
+
+TEST(MutableGraphViewTest, UpdateAllRegularFaninsToControllingConsumingSwitch) {
+  constexpr char kDevice[] = "/device:foo:0";
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}),
+       NDef("switch", "Switch", {}, {{"T", DT_FLOAT}}, kDevice),
+       NDef("b", "NotImportant", {"switch:1"}, {})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  TF_EXPECT_OK(graph.UpdateAllRegularFaninsToControlling("b"));
+
+  EXPECT_EQ(graph.graph()->node_size(), 4);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {});
+  CheckNode(graph, "switch", "Switch", kDevice, {{"T", DT_FLOAT}}, {},
+            {"ConstantFoldingCtrl/switch_1"});
+  CheckNode(graph, "b", "NotImportant", "", {},
+            {"^ConstantFoldingCtrl/switch_1"}, {});
+  CheckNode(graph, "ConstantFoldingCtrl/switch_1", "Identity", kDevice,
+            {{"T", DT_FLOAT}}, {"switch:1"}, {"^b"});
+
+  CheckGraph(graph);
+}
+
 TEST(MutableGraphViewTest, DeleteNodes) {
   // Actual node.op() is not important in this test.
   GraphDef graph_def = test::function::GDef(
-- 
GitLab


From c0e3d0a7c7d680a5d74cf2c9e4a4a04a910da275 Mon Sep 17 00:00:00 2001
From: Tong Shen <endlessroad@google.com>
Date: Thu, 31 Jan 2019 13:35:18 -0800
Subject: [PATCH 1673/2345] Avoid quadratic node explosion in
 FunctionalizeCond.

Before this change, if we have the following tf.cond:

  def computation(p, a, b, c, d):
    def f():
      return (a, a + b, a + b + c, a + b + c + d)
    return tf.cond(p, f, f)

We will create one "If" node for each Merge node, because those Merge nodes have different sets of ancestor Switch nodes. Merge node for first return value will have "Switch for input a", Merge node for second return value will have "Switch for input a & Switch for input b", and so on. As a result, the total number of If nodes is the number of nodes, and total number of nodes in If's branch functions is quadratic.

In this CL, we add predicate of the Switch nodes in ancestor id. For the previous example, now we only generate 1 If node.

PiperOrigin-RevId: 231844072
---
 .../compiler/tf2xla/functionalize_cond.cc     | 50 +++++++++++++++++--
 .../compiler/tf2xla/functionalize_cond.h      | 29 ++++++++++-
 2 files changed, 73 insertions(+), 6 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/functionalize_cond.cc b/tensorflow/compiler/tf2xla/functionalize_cond.cc
index b21752ad57..c8341a2c6b 100644
--- a/tensorflow/compiler/tf2xla/functionalize_cond.cc
+++ b/tensorflow/compiler/tf2xla/functionalize_cond.cc
@@ -35,6 +35,7 @@ limitations under the License.
 #include "tensorflow/core/graph/control_flow.h"
 #include "tensorflow/core/graph/node_builder.h"
 #include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/hash/hash.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 
 using xla::StatusOr;
@@ -42,6 +43,26 @@ using xla::StatusOr;
 namespace tensorflow {
 namespace functionalize_cond {
 
+bool AncestorNode::operator<(const AncestorNode& other) const {
+  return (output_tensor.node->id() < other.output_tensor.node->id()) ||
+         (output_tensor.node->id() == other.output_tensor.node->id() &&
+          output_tensor.index < other.output_tensor.index) ||
+         (output_tensor.node->id() == other.output_tensor.node->id() &&
+          output_tensor.index == other.output_tensor.index &&
+          type < other.type);
+}
+
+bool AncestorNode::operator==(const AncestorNode& other) const {
+  return output_tensor.node->id() == other.output_tensor.node->id() &&
+         output_tensor.index == other.output_tensor.index && type == other.type;
+}
+
+size_t AncestorNode::Hash::operator()(const AncestorNode& ancestor) const {
+  size_t h = std::hash<int>()(ancestor.output_tensor.node->id());
+  h = Hash64Combine(h, std::hash<int>()(ancestor.output_tensor.index));
+  return Hash64Combine(h, std::hash<int>()(static_cast<int>(ancestor.type)));
+}
+
 // TODO(jpienaar): Move to OutputTensor.
 string DebugString(const OutputTensor& tensor) {
   return absl::StrCat(tensor.node->name(), ":", tensor.index);
@@ -146,10 +167,10 @@ size_t StateMap::Hash::operator()(const StateMap::AncestorState& map) const {
   if (map.empty()) return 0;
   // Compute hash of the front element.
   auto it = map.begin();
-  size_t h = hash<Node*>()(*it);
+  size_t h = AncestorNode::Hash()(*it);
   for (++it; it != map.end(); ++it) {
     // Combine the has with the different elements in the map.
-    h = Hash64Combine(h, hash<Node*>()(*it));
+    h = Hash64Combine(h, AncestorNode::Hash()(*it));
   }
   return h;
 }
@@ -230,7 +251,17 @@ string StateMap::CondStateToString(StateMap::CondId id) const {
 }
 
 string StateMap::AncestorStateToString(const Node* node) const {
-  if (auto id = LookupAncestorId(node)) return NodesToString(*id);
+  if (auto id = LookupAncestorId(node)) {
+    return absl::StrCat(
+        "{",
+        absl::StrJoin(*id, ",",
+                      [](string* output, const AncestorNode& ancestor) {
+                        absl::StrAppend(output,
+                                        ancestor.output_tensor.node->name(),
+                                        ":", ancestor.output_tensor.index);
+                      }),
+        "}");
+  }
   return "{}";
 }
 
@@ -1206,8 +1237,17 @@ Status FunctionalizeCond::DetermineAncestorState(Node* dst) {
     if (other_id != id && other_id != nullptr) {
       state.insert(other_id->begin(), other_id->end());
     }
-    if (IsSwitch(src) || IsMerge(src)) {
-      state.insert(src);
+    if (IsMerge(src)) {
+      state.insert({{src, 0}, AncestorNode::AncestorNodeType::kMerge});
+    } else if (IsSwitch(src)) {
+      OutputTensor pred;
+      // For dead switch nodes, GetSwitchPredicate() will fail, and we use
+      // the switch node directly as ancestor.
+      if (GetSwitchPredicate(*src, &pred).ok()) {
+        state.insert({pred, AncestorNode::AncestorNodeType::kPred});
+      } else {
+        state.insert({{src, 0}, AncestorNode::AncestorNodeType::kSwitch});
+      }
     }
     return state_map_.GetAncestorId(state);
   };
diff --git a/tensorflow/compiler/tf2xla/functionalize_cond.h b/tensorflow/compiler/tf2xla/functionalize_cond.h
index 9a610a5cbf..d85800fb8e 100644
--- a/tensorflow/compiler/tf2xla/functionalize_cond.h
+++ b/tensorflow/compiler/tf2xla/functionalize_cond.h
@@ -43,6 +43,33 @@ enum class BranchType {
   kNeither = 3,
 };
 
+// When we keep track of which switch/merge node's feed into a node, we record
+// 1) predicate for non-dead switch node,
+// 2) the switch node itself for dead switch node,
+// 3) the merge node itself for merge node.
+// Case 1) is an optimization. With this optimization, if there are nodes from
+// different switch nodes but those switch nodes have the same predicate, the
+// nodes will still have same AncestorState, and they will be clustered into a
+// single "If".
+struct AncestorNode {
+  enum class AncestorNodeType {
+    kPred = 0,
+    kSwitch = 1,
+    kMerge = 2,
+  };
+
+  OutputTensor output_tensor;
+  AncestorNodeType type;
+
+  // Compare two AncestorNodes by (node id, index, type).
+  bool operator<(const AncestorNode& other) const;
+  bool operator==(const AncestorNode& other) const;
+
+  struct Hash {
+    size_t operator()(const AncestorNode&) const;
+  };
+};
+
 // StateMap is responsible for mapping from each graph Node to
 // * a CondState, where each CondState is a map from predicate to branch (i,e.,
 //   what predicates have to hold or not hold).
@@ -68,7 +95,7 @@ class StateMap {
   using CondId = const CondState*;
 
   // Keep track of which switch/merge node's feed into a node's values.
-  using AncestorState = std::set<Node*>;
+  using AncestorState = std::set<AncestorNode>;
 
   // Every unique ID is mapped to a AncestorState.
   using AncestorId = const AncestorState*;
-- 
GitLab


From d0065b7d68c2df1f75e7689d01360840f16c0fb6 Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Thu, 31 Jan 2019 13:36:33 -0800
Subject: [PATCH 1674/2345] [XLA] Fix AlgebraicSimplifier to handle reshape
 instructions without layouts.

The AlgebraicSimplifier pass is run before layout assignment and may encounter
instructions without layouts. Fix the pass to use a default layout to create
the resulting literal when handling a reshape instruction without a layout.

Add a test case.

PiperOrigin-RevId: 231844361
---
 .../xla/service/algebraic_simplifier.cc       |  8 +++++-
 .../xla/service/algebraic_simplifier_test.cc  | 27 +++++++++++++++++++
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index 549e675d65..cd06cfcdd3 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -2427,8 +2427,14 @@ Status AlgebraicSimplifierVisitor::HandleReshape(HloInstruction* reshape) {
   // Reshape directly to empty constant if the shape contains zero-element
   // dimension.
   if (ShapeUtil::IsZeroElementArray(reshape->shape())) {
+    // If the instruction doesn't have a layout, use a default layout for
+    // the literal result.
+    Shape reshaped_shape = reshape->shape();
+    if (!LayoutUtil::HasLayout(reshaped_shape)) {
+      LayoutUtil::SetToDefaultLayout(&reshaped_shape);
+    }
     auto empty_constant = HloInstruction::CreateConstant(
-        Literal::CreateFromShape(reshape->shape()));
+        Literal::CreateFromShape(reshaped_shape));
 
     return ReplaceWithNewInstruction(reshape, std::move(empty_constant));
   }
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index b5fed239d8..f55a1886b8 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -27,6 +27,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_instructions.h"
+#include "tensorflow/compiler/xla/service/hlo_matchers.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
 #include "tensorflow/compiler/xla/service/hlo_parser.h"
 #include "tensorflow/compiler/xla/service/hlo_pass_fix.h"
@@ -45,6 +46,7 @@ namespace {
 
 using ::testing::ElementsAre;
 namespace m = match;
+namespace op = xla::testing::opcode_matchers;
 
 class AlgebraicSimplifierTest : public HloTestBase {
  protected:
@@ -4765,5 +4767,30 @@ ENTRY entry {
                                         m::Broadcast(m::ConstantScalar(1)))));
 }
 
+TEST_F(AlgebraicSimplifierTest, ZeroSizedReshapeWithoutLayout) {
+  auto builder = HloComputation::Builder(TestName());
+  HloInstruction* param =
+      builder.AddInstruction(HloInstruction::CreateParameter(
+          0, ShapeUtil::MakeShape(F32, {1}), "param"));
+  HloInstruction* broadcast =
+      builder.AddInstruction(HloInstruction::CreateBroadcast(
+          ShapeUtil::MakeShape(F32, {0, 1}), param, {1}));
+
+  // Create a reshape with zero sized result and without layout.
+  Shape reshaped_shape = ShapeUtil::MakeShape(F32, {0});
+  reshaped_shape.clear_layout();
+  builder.AddInstruction(
+      HloInstruction::CreateReshape(reshaped_shape, broadcast));
+
+  std::unique_ptr<VerifiedHloModule> module = CreateNewVerifiedModule();
+  module->AddEntryComputation(builder.Build());
+
+  AlgebraicSimplifierOptions options;
+  AlgebraicSimplifier simplifier(options);
+  EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
+  HloInstruction* root = module->entry_computation()->root_instruction();
+  EXPECT_THAT(root, op::Constant());
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 2cd4c39ff36f9a96c118a07b854b8a082deaa40a Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Thu, 31 Jan 2019 13:36:54 -0800
Subject: [PATCH 1675/2345] Add support for TensorList in WhileOp

Required adding DT_VARIANT support to While, _RetVal, _Arg & Identity

PiperOrigin-RevId: 231844442
---
 tensorflow/compiler/tf2xla/kernels/arg_op.cc    | 10 ++++++++--
 .../compiler/tf2xla/kernels/identity_op.cc      | 10 +++++++---
 tensorflow/compiler/tf2xla/kernels/retval_op.cc | 17 +++++++++++------
 tensorflow/compiler/tf2xla/kernels/while_op.cc  |  9 ++++++---
 tensorflow/compiler/tf2xla/xla_compiler.cc      |  6 +++++-
 tensorflow/compiler/tf2xla/xla_op_registry.cc   | 13 +++++++++++++
 tensorflow/compiler/tf2xla/xla_op_registry.h    | 13 ++++++++++---
 7 files changed, 60 insertions(+), 18 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/arg_op.cc b/tensorflow/compiler/tf2xla/kernels/arg_op.cc
index 795ea09831..5554d7a377 100644
--- a/tensorflow/compiler/tf2xla/kernels/arg_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/arg_op.cc
@@ -53,7 +53,11 @@ class XlaArgOp : public XlaOpKernel {
     const XlaExpression& arg = ctx->xla_context()->args()[index_];
     OP_REQUIRES(ctx, arg.kind() != XlaExpression::Kind::kInvalid,
                 errors::InvalidArgument("Invalid/missing argument expression"));
-    ctx->SetOutputExpression(0, arg);
+    if (ctx->expected_output_dtype(0) == DT_VARIANT) {
+      ctx->SetTensorListOutput(0, arg.handle());
+    } else {
+      ctx->SetOutputExpression(0, arg);
+    }
   }
 
  private:
@@ -63,6 +67,8 @@ class XlaArgOp : public XlaOpKernel {
   TF_DISALLOW_COPY_AND_ASSIGN(XlaArgOp);
 };
 
-REGISTER_XLA_OP(Name("_Arg").AllowResourceTypes().CompilationOnly(), XlaArgOp);
+REGISTER_XLA_OP(
+    Name("_Arg").AllowResourceTypes().AllowVariantTypes().CompilationOnly(),
+    XlaArgOp);
 
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/tf2xla/kernels/identity_op.cc b/tensorflow/compiler/tf2xla/kernels/identity_op.cc
index 19dd38c46e..8b27e8e85a 100644
--- a/tensorflow/compiler/tf2xla/kernels/identity_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/identity_op.cc
@@ -38,9 +38,13 @@ class IdentityOp : public XlaOpKernel {
 
 // XLA_* devices also register a "real" Identity operator so we suppress the
 // dummy operator using CompilationOnly().
-REGISTER_XLA_OP(Name("Identity").AllowResourceTypes().CompilationOnly(),
-                IdentityOp);
-REGISTER_XLA_OP(Name("IdentityN").AllowResourceTypes().CompilationOnly(),
+REGISTER_XLA_OP(
+    Name("Identity").AllowResourceTypes().AllowVariantTypes().CompilationOnly(),
+    IdentityOp);
+REGISTER_XLA_OP(Name("IdentityN")
+                    .AllowResourceTypes()
+                    .AllowVariantTypes()
+                    .CompilationOnly(),
                 IdentityOp);
 REGISTER_XLA_OP(Name("PlaceholderWithDefault"), IdentityOp);
 REGISTER_XLA_OP(Name("PreventGradient"), IdentityOp);
diff --git a/tensorflow/compiler/tf2xla/kernels/retval_op.cc b/tensorflow/compiler/tf2xla/kernels/retval_op.cc
index e4046c7955..1f41703728 100644
--- a/tensorflow/compiler/tf2xla/kernels/retval_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/retval_op.cc
@@ -37,10 +37,14 @@ class RetvalOp : public XlaOpKernel {
   void Compile(XlaOpKernelContext* ctx) override {
     const Tensor& input = ctx->op_kernel_context()->input(0);
 
-    OP_REQUIRES(ctx, input.dtype() == dtype_,
-                errors::InvalidArgument(
-                    "Type mismatch: actual ", DataTypeString(input.dtype()),
-                    " vs. expect ", DataTypeString(dtype_)));
+    // DT_VARIANT types represent Tensor Lists and are wrapped in a DT_UINT8
+    // tensor so we skip the check here.
+    if (dtype_ != DT_VARIANT) {
+      OP_REQUIRES(ctx, input.dtype() == dtype_,
+                  errors::InvalidArgument(
+                      "Type mismatch: actual ", DataTypeString(input.dtype()),
+                      " vs. expect ", DataTypeString(dtype_)));
+    }
     auto frame = ctx->call_frame();
     if (frame) {
       // If 'frame' is non-null, this is an inner function call inside a JIT
@@ -59,8 +63,9 @@ class RetvalOp : public XlaOpKernel {
   TF_DISALLOW_COPY_AND_ASSIGN(RetvalOp);
 };
 
-REGISTER_XLA_OP(Name("_Retval").AllowResourceTypes().CompilationOnly(),
-                RetvalOp);
+REGISTER_XLA_OP(
+    Name("_Retval").AllowResourceTypes().AllowVariantTypes().CompilationOnly(),
+    RetvalOp);
 
 }  // anonymous namespace
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/tf2xla/kernels/while_op.cc b/tensorflow/compiler/tf2xla/kernels/while_op.cc
index fd5ff10ae0..f49da9683b 100644
--- a/tensorflow/compiler/tf2xla/kernels/while_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/while_op.cc
@@ -348,8 +348,11 @@ void XlaWhileOp::Compile(XlaOpKernelContext* ctx) {
   VLOG(1) << "Done building while loop";
 }
 
-REGISTER_XLA_OP(Name("While").AllowResourceTypes(), XlaWhileOp);
-REGISTER_XLA_OP(Name("StatelessWhile").AllowResourceTypes(), XlaWhileOp);
-REGISTER_XLA_OP(Name("XlaWhile").AllowResourceTypes(), XlaWhileOp);
+REGISTER_XLA_OP(Name("While").AllowResourceTypes().AllowVariantTypes(),
+                XlaWhileOp);
+REGISTER_XLA_OP(Name("StatelessWhile").AllowResourceTypes().AllowVariantTypes(),
+                XlaWhileOp);
+REGISTER_XLA_OP(Name("XlaWhile").AllowResourceTypes().AllowVariantTypes(),
+                XlaWhileOp);
 
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/tf2xla/xla_compiler.cc b/tensorflow/compiler/tf2xla/xla_compiler.cc
index 1f9cfcdd24..514b156deb 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler.cc
@@ -58,7 +58,11 @@ Status CheckSignature(const DataTypeVector& types,
                             " elements while function has ", types.size());
   }
   for (int i = 0; i < types.size(); ++i) {
-    if (types[i] != args[i].type && types[i] != DT_RESOURCE) {
+    // Don't perform type checks on resource variables and tensor
+    // lists (DT_VARIANT) as we have to trick the type system in order to
+    // plumb them through. DT_VARIANTS are wrapped in a DT_UINT8 tensor.
+    if (types[i] != args[i].type && types[i] != DT_RESOURCE &&
+        types[i] != DT_VARIANT) {
       return errors::Internal(
           "Argument ", i, " has declared type ", DataTypeString(args[i].type),
           " but function parameter has type ", DataTypeString(types[i]));
diff --git a/tensorflow/compiler/tf2xla/xla_op_registry.cc b/tensorflow/compiler/tf2xla/xla_op_registry.cc
index 14237df690..26314034a1 100644
--- a/tensorflow/compiler/tf2xla/xla_op_registry.cc
+++ b/tensorflow/compiler/tf2xla/xla_op_registry.cc
@@ -73,6 +73,11 @@ XlaOpRegistry::~XlaOpRegistry() = default;
                  << " have incompatible allow_resource_types settings.";
     return false;
   }
+  if (x.allow_variant_types != y.allow_variant_types) {
+    LOG(WARNING) << "Registrations of " << x.name
+                 << " have incompatible allow_variant_types settings.";
+    return false;
+  }
   if (!x.has_device_whitelist && !y.has_device_whitelist) {
     LOG(WARNING) << "Duplicate registrations of " << x.name
                  << "with no device whitelists.";
@@ -289,6 +294,9 @@ void XlaOpRegistry::RegisterCompilationKernels() {
           if (op_registration->allow_resource_types) {
             allowed_values->add_type(DT_RESOURCE);
           }
+          if (op_registration->allow_variant_types) {
+            allowed_values->add_type(DT_VARIANT);
+          }
           // Don't build KernelDefs that have unsatisfiable type constraints.
           if (allowed_values->type().empty()) {
             unsatisfiable_type_constraint = true;
@@ -485,6 +493,11 @@ XlaOpRegistrationBuilder& XlaOpRegistrationBuilder::AllowResourceTypes() {
   return *this;
 }
 
+XlaOpRegistrationBuilder& XlaOpRegistrationBuilder::AllowVariantTypes() {
+  registration_->allow_variant_types = true;
+  return *this;
+}
+
 XlaOpRegistrationBuilder& XlaOpRegistrationBuilder::TypeConstraint(
     absl::string_view attr_name, DataType allowed) {
   std::set<DataType>& types =
diff --git a/tensorflow/compiler/tf2xla/xla_op_registry.h b/tensorflow/compiler/tf2xla/xla_op_registry.h
index ce3b6b298c..c5e078a02d 100644
--- a/tensorflow/compiler/tf2xla/xla_op_registry.h
+++ b/tensorflow/compiler/tf2xla/xla_op_registry.h
@@ -212,6 +212,10 @@ class XlaOpRegistry {
     // allow DT_RESOURCE.
     bool allow_resource_types = false;
 
+    // Should we allow variant types for type attributes? Used by While to
+    // allow TensorList which is of type DT_VARIANT.
+    bool allow_variant_types = false;
+
     // Mapping from attribute name to a list of supported types.
     std::unordered_map<string, std::set<DataType>> type_constraints;
 
@@ -233,9 +237,9 @@ class XlaOpRegistry {
 
   // Returns true if registrations x and y can both be added to the registry.
   // This is always the case if they refer to different ops. If they refer to
-  // the same op name, they must: have the same values for compilation_only and
-  // allow_resource_types; use a device_whitelist; and their
-  // whitelists must not intersect.
+  // the same op name, they must: have the same values for compilation_only,
+  // allow_resource_types and allow_variant_types; use a device_whitelist; and
+  // their whitelists must not intersect.
   static bool IsCompatible(const OpRegistration& x, const OpRegistration& y);
 
   static Status CompileTimeConstantInputs(const NodeDef& node_def,
@@ -293,6 +297,9 @@ class XlaOpRegistrationBuilder {
   // Allow DT_RESOURCE types for type parameters.
   XlaOpRegistrationBuilder& AllowResourceTypes();
 
+  // Allow DT_VARIANT types for type parameters.
+  XlaOpRegistrationBuilder& AllowVariantTypes();
+
   // Mark 'input_name' as an argument whose value must be known at compile-time.
   XlaOpRegistrationBuilder& CompileTimeConstantInput(
       absl::string_view input_name);
-- 
GitLab


From 376351335de1b7a471b212ebf31f897dc8ce8662 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 31 Jan 2019 13:44:12 -0800
Subject: [PATCH 1676/2345] Refactor EagerProfiler: rename it to
 ProfileSession.

PiperOrigin-RevId: 231845855
---
 tensorflow/c/eager/BUILD                      |  4 +-
 tensorflow/c/eager/c_api_internal.h           |  6 +-
 tensorflow/core/profiler/internal/BUILD       | 11 +++
 tensorflow/core/profiler/internal/gpu/BUILD   | 26 ++++++
 .../core/profiler/internal/gpu/tracer.cc      | 59 ++++++++++++
 .../core/profiler/internal/gpu/tracer.h       | 48 ++++++++++
 .../profiler/internal/profiler_interface.h    | 49 ++++++++++
 .../core/profiler/internal/runtime/BUILD      | 24 +++++
 .../internal/runtime/eager_profiler.cc        | 61 ++++++++++++
 .../internal/runtime/eager_profiler.h         | 64 +++++++++++++
 tensorflow/core/profiler/lib/BUILD            |  9 +-
 ...{eager_profiler.cc => profiler_session.cc} | 93 ++++++-------------
 .../{eager_profiler.h => profiler_session.h}  | 33 +++----
 tensorflow/core/profiler/rpc/BUILD            |  2 +-
 .../profiler/rpc/profiler_service_impl.cc     |  6 +-
 .../core/profiler/rpc/profiler_service_impl.h |  1 -
 16 files changed, 400 insertions(+), 96 deletions(-)
 create mode 100644 tensorflow/core/profiler/internal/gpu/BUILD
 create mode 100644 tensorflow/core/profiler/internal/gpu/tracer.cc
 create mode 100644 tensorflow/core/profiler/internal/gpu/tracer.h
 create mode 100644 tensorflow/core/profiler/internal/profiler_interface.h
 create mode 100644 tensorflow/core/profiler/internal/runtime/BUILD
 create mode 100644 tensorflow/core/profiler/internal/runtime/eager_profiler.cc
 create mode 100644 tensorflow/core/profiler/internal/runtime/eager_profiler.h
 rename tensorflow/core/profiler/lib/{eager_profiler.cc => profiler_session.cc} (54%)
 rename tensorflow/core/profiler/lib/{eager_profiler.h => profiler_session.h} (62%)

diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
index 04dfefa6da..257be6379c 100644
--- a/tensorflow/c/eager/BUILD
+++ b/tensorflow/c/eager/BUILD
@@ -70,7 +70,7 @@ tf_cuda_library(
         "//tensorflow/core/distributed_runtime:remote_device",
         "//tensorflow/core/distributed_runtime:server_lib",
         "//tensorflow/core/distributed_runtime:worker_env",
-        "//tensorflow/core/profiler/lib:eager_profiler",
+        "//tensorflow/core/profiler/lib:profiler_session",
         "//tensorflow/core:gpu_runtime",
     ],
 )
@@ -110,7 +110,7 @@ tf_cuda_library(
         "//tensorflow/core/distributed_runtime/rpc:grpc_worker_service",
         "//tensorflow/core/distributed_runtime/rpc:rpc_rendezvous_mgr",
         "//tensorflow/core/distributed_runtime/rpc/eager:grpc_eager_client",
-        "//tensorflow/core/profiler/lib:eager_profiler",
+        "//tensorflow/core/profiler/lib:profiler_session",
     ],
 )
 
diff --git a/tensorflow/c/eager/c_api_internal.h b/tensorflow/c/eager/c_api_internal.h
index 3b9e681194..b70c0f1c11 100644
--- a/tensorflow/c/eager/c_api_internal.h
+++ b/tensorflow/c/eager/c_api_internal.h
@@ -52,7 +52,7 @@ limitations under the License.
 #include "tensorflow/core/lib/gtl/stl_util.h"
 #include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/platform/thread_annotations.h"
-#include "tensorflow/core/profiler/lib/eager_profiler.h"
+#include "tensorflow/core/profiler/lib/profiler_session.h"
 #include "tensorflow/core/public/version.h"
 
 struct TFE_ContextOptions {
@@ -109,9 +109,9 @@ struct TFE_Op {
 
 struct TFE_Profiler {
   TFE_Profiler(TFE_Context* ctx)
-      : profiler(tensorflow::EagerProfiler::Create(&ctx->context)) {}
+      : profiler(tensorflow::ProfilerSession::Create(&ctx->context)) {}
 
-  std::unique_ptr<tensorflow::EagerProfiler> profiler;
+  std::unique_ptr<tensorflow::ProfilerSession> profiler;
 };
 
 namespace tensorflow {
diff --git a/tensorflow/core/profiler/internal/BUILD b/tensorflow/core/profiler/internal/BUILD
index b373342085..2456adadaa 100644
--- a/tensorflow/core/profiler/internal/BUILD
+++ b/tensorflow/core/profiler/internal/BUILD
@@ -394,3 +394,14 @@ tf_cuda_cc_test(
         "@com_google_googletest//:gtest_main",
     ],
 )
+
+tf_cuda_library(
+    name = "profiler_interface",
+    hdrs = [
+        "profiler_interface.h",
+    ],
+    deps = [
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+    ],
+)
diff --git a/tensorflow/core/profiler/internal/gpu/BUILD b/tensorflow/core/profiler/internal/gpu/BUILD
new file mode 100644
index 0000000000..c4b5b5243c
--- /dev/null
+++ b/tensorflow/core/profiler/internal/gpu/BUILD
@@ -0,0 +1,26 @@
+package(
+    default_visibility = ["//tensorflow:internal"],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+load(
+    "//tensorflow:tensorflow.bzl",
+    "tf_cuda_library",
+)
+
+tf_cuda_library(
+    name = "tracer",
+    srcs = [
+        "tracer.cc",
+    ],
+    hdrs = [
+        "tracer.h",
+    ],
+    deps = [
+        "//tensorflow/core:core_cpu_lib",
+        "//tensorflow/core:device_tracer",
+        "//tensorflow/core/profiler/internal:profiler_interface",
+        "//tensorflow/core/profiler/internal/runtime:eager_profiler",
+    ],
+)
diff --git a/tensorflow/core/profiler/internal/gpu/tracer.cc b/tensorflow/core/profiler/internal/gpu/tracer.cc
new file mode 100644
index 0000000000..f1cb54161c
--- /dev/null
+++ b/tensorflow/core/profiler/internal/gpu/tracer.cc
@@ -0,0 +1,59 @@
+/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/profiler/internal/gpu/tracer.h"
+#include "tensorflow/core/common_runtime/step_stats_collector.h"
+
+namespace tensorflow {
+namespace profiler {
+namespace gpu {
+
+/* static */ std::unique_ptr<ProfilerInterface> Tracer::Create() {
+  return absl::WrapUnique(new Tracer());
+}
+
+Status Tracer::Start() {
+  device_tracer_ = CreateDeviceTracer();
+  if (!device_tracer_) {
+    return Status(tensorflow::error::Code::FAILED_PRECONDITION,
+                  "Failed to create device tracer.");
+  }
+  return device_tracer_->Start();
+}
+
+Status Tracer::Stop() {
+  if (!device_tracer_) {
+    return Status(tensorflow::error::Code::FAILED_PRECONDITION,
+                  "No running device tracer.");
+  }
+  return device_tracer_->Stop();
+}
+
+Status Tracer::CollectData(RunMetadata* run_metadata) {
+  if (!device_tracer_) {
+    return Status(tensorflow::error::Code::FAILED_PRECONDITION,
+                  "No running device tracer.");
+  }
+  auto step_stats_collector =
+      absl::make_unique<StepStatsCollector>(run_metadata->mutable_step_stats());
+  Status s = device_tracer_->Collect(step_stats_collector.get());
+  step_stats_collector->Finalize();
+  return s;
+}
+
+Tracer::Tracer() {}
+
+}  // namespace gpu
+}  // namespace profiler
+}  // namespace tensorflow
diff --git a/tensorflow/core/profiler/internal/gpu/tracer.h b/tensorflow/core/profiler/internal/gpu/tracer.h
new file mode 100644
index 0000000000..d7765432de
--- /dev/null
+++ b/tensorflow/core/profiler/internal/gpu/tracer.h
@@ -0,0 +1,48 @@
+/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_PROFILER_INTERNAL_GPU_TRACER_H_
+#define TENSORFLOW_CORE_PROFILER_INTERNAL_GPU_TRACER_H_
+
+#include "tensorflow/core/platform/device_tracer.h"
+#include "tensorflow/core/profiler/internal/profiler_interface.h"
+
+namespace tensorflow {
+namespace profiler {
+namespace gpu {
+
+class Tracer : public ProfilerInterface {
+ public:
+  static std::unique_ptr<ProfilerInterface> Create();
+
+  Status Start() override;
+
+  Status Stop() override;
+
+  Status CollectData(RunMetadata* run_metadata) override;
+
+ private:
+  Tracer();
+
+  // Trace is neither copyable nor movable.
+  Tracer(const Tracer&) = delete;
+  Tracer& operator=(const Tracer&) = delete;
+
+  std::unique_ptr<DeviceTracer> device_tracer_;
+};
+
+}  // namespace gpu
+}  // namespace profiler
+}  // namespace tensorflow
+#endif  // TENSORFLOW_CORE_PROFILER_INTERNAL_GPU_TRACER_H_
diff --git a/tensorflow/core/profiler/internal/profiler_interface.h b/tensorflow/core/profiler/internal/profiler_interface.h
new file mode 100644
index 0000000000..144c4bb44d
--- /dev/null
+++ b/tensorflow/core/profiler/internal/profiler_interface.h
@@ -0,0 +1,49 @@
+/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_PROFILER_INTERNAL_PROFILER_INTERFACE_H_
+#define TENSORFLOW_CORE_PROFILER_INTERNAL_PROFILER_INTERFACE_H_
+
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/protobuf/config.pb.h"
+
+namespace tensorflow {
+namespace profiler {
+
+// Interface for tensorflow profiler plugins.
+//
+// ProfileSession calls each of these methods at most once per instance, and
+// implementations can rely on that guarantee for simplicity.
+//
+// Thread-safety: Implementations are only required to be go/thread-compatible.
+// ProfileSession is go/thread-safe and synchronizes access to ProfilerInterface
+// instances.
+class ProfilerInterface {
+ public:
+  virtual ~ProfilerInterface() = default;
+
+  // Starts profiling.
+  virtual Status Start() = 0;
+
+  // Stops profiling.
+  virtual Status Stop() = 0;
+
+  // Moves collected profile data into run_metadata.
+  virtual Status CollectData(RunMetadata* run_metadata) = 0;
+};
+
+}  // namespace profiler
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_PROFILER_INTERNAL_PROFILER_INTERFACE_H_
diff --git a/tensorflow/core/profiler/internal/runtime/BUILD b/tensorflow/core/profiler/internal/runtime/BUILD
new file mode 100644
index 0000000000..2e383f1716
--- /dev/null
+++ b/tensorflow/core/profiler/internal/runtime/BUILD
@@ -0,0 +1,24 @@
+package(
+    default_visibility = ["//tensorflow:internal"],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+load(
+    "//tensorflow:tensorflow.bzl",
+    "tf_cuda_library",
+)
+
+tf_cuda_library(
+    name = "eager_profiler",
+    srcs = [
+        "eager_profiler.cc",
+    ],
+    hdrs = [
+        "eager_profiler.h",
+    ],
+    deps = [
+        "//tensorflow/core/common_runtime/eager:context",
+        "//tensorflow/core/profiler/internal:profiler_interface",
+    ],
+)
diff --git a/tensorflow/core/profiler/internal/runtime/eager_profiler.cc b/tensorflow/core/profiler/internal/runtime/eager_profiler.cc
new file mode 100644
index 0000000000..aad692b01f
--- /dev/null
+++ b/tensorflow/core/profiler/internal/runtime/eager_profiler.cc
@@ -0,0 +1,61 @@
+/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/profiler/internal/runtime/eager_profiler.h"
+
+namespace tensorflow {
+namespace profiler {
+namespace runtime {
+
+TraceCollector::TraceCollector(EagerContext* const eager_context)
+    : context_(eager_context) {}
+
+void TraceCollector::BeforeClearRunMetadata() {
+  run_metadata_.MergeFrom(*context_->RunMetadataProto());
+}
+
+Status TraceCollector::CollectData(RunMetadata* run_metadata) {
+  run_metadata->MergeFrom(run_metadata_);
+  return Status::OK();
+}
+
+/* static */ std::unique_ptr<ProfilerInterface> EagerProfiler::Create(
+    EagerContext* const eager_context) {
+  return absl::WrapUnique(new EagerProfiler(eager_context));
+}
+
+Status EagerProfiler::Start() {
+  if (context_ == nullptr) {
+    return Status(tensorflow::error::Code::FAILED_PRECONDITION,
+                  "No eager context attached.");
+  }
+  return context_->RegisterRunMetadataListener(&collector_);
+}
+
+Status EagerProfiler::Stop() {
+  collector_.BeforeClearRunMetadata();
+  context_->ClearRunMetadataListener();
+  return Status::OK();
+}
+
+Status EagerProfiler::CollectData(RunMetadata* run_metadata) {
+  return collector_.CollectData(run_metadata);
+}
+
+EagerProfiler::EagerProfiler(EagerContext* const eager_context)
+    : context_(eager_context), collector_(eager_context) {}
+
+}  // namespace runtime
+}  // namespace profiler
+}  // namespace tensorflow
diff --git a/tensorflow/core/profiler/internal/runtime/eager_profiler.h b/tensorflow/core/profiler/internal/runtime/eager_profiler.h
new file mode 100644
index 0000000000..7135355e6f
--- /dev/null
+++ b/tensorflow/core/profiler/internal/runtime/eager_profiler.h
@@ -0,0 +1,64 @@
+/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_PROFILER_INTERNAL_RUNTIME_EAGER_PROFILER_H_
+#define TENSORFLOW_CORE_PROFILER_INTERNAL_RUNTIME_EAGER_PROFILER_H_
+
+#include "tensorflow/core/common_runtime/eager/context.h"
+#include "tensorflow/core/profiler/internal/profiler_interface.h"
+
+namespace tensorflow {
+namespace profiler {
+namespace runtime {
+
+class TraceCollector : public RunMetadataListener {
+ public:
+  TraceCollector(EagerContext* const eager_context);
+
+  void BeforeClearRunMetadata() override;
+
+  Status CollectData(RunMetadata* run_metadata);
+
+ private:
+  RunMetadata run_metadata_;
+  EagerContext* const context_;
+};
+
+class EagerProfiler : public ProfilerInterface {
+ public:
+  static std::unique_ptr<ProfilerInterface> Create(
+      EagerContext* const eager_context);
+
+  Status Start() override;
+
+  Status Stop() override;
+
+  Status CollectData(RunMetadata* run_metadata) override;
+
+ private:
+  EagerProfiler(EagerContext* const eager_context);
+
+  // Trace is neither copyable nor movable.
+  EagerProfiler(const EagerProfiler&) = delete;
+  EagerProfiler& operator=(const EagerProfiler&) = delete;
+
+  EagerContext* const context_;
+  TraceCollector collector_;
+};
+
+}  // namespace runtime
+}  // namespace profiler
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_PROFILER_INTERNAL_RUNTIME_EAGER_PROFILER_H_
diff --git a/tensorflow/core/profiler/lib/BUILD b/tensorflow/core/profiler/lib/BUILD
index 0320ae1a68..10d5ee645e 100644
--- a/tensorflow/core/profiler/lib/BUILD
+++ b/tensorflow/core/profiler/lib/BUILD
@@ -13,17 +13,20 @@ load(
 )
 
 tf_cuda_library(
-    name = "eager_profiler",
+    name = "profiler_session",
     srcs = [
-        "eager_profiler.cc",
+        "profiler_session.cc",
     ],
     hdrs = [
-        "eager_profiler.h",
+        "profiler_session.h",
     ],
     visibility = ["//tensorflow:internal"],
     deps = [
         "//tensorflow/core/common_runtime/eager:context",
         "//tensorflow/contrib/tpu/profiler:trace_events_proto_cc",
+        "//tensorflow/core/profiler/internal/gpu:tracer",
+        "//tensorflow/core/profiler/internal/runtime:eager_profiler",
+        "//tensorflow/core/profiler/internal:profiler_interface",
     ] + select({
         "//tensorflow:android": [
             "//tensorflow/core:android_tensorflow_lib_lite",
diff --git a/tensorflow/core/profiler/lib/eager_profiler.cc b/tensorflow/core/profiler/lib/profiler_session.cc
similarity index 54%
rename from tensorflow/core/profiler/lib/eager_profiler.cc
rename to tensorflow/core/profiler/lib/profiler_session.cc
index 9293e7aba9..31070b76c7 100644
--- a/tensorflow/core/profiler/lib/eager_profiler.cc
+++ b/tensorflow/core/profiler/lib/profiler_session.cc
@@ -13,16 +13,16 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/core/profiler/lib/eager_profiler.h"
+#include "tensorflow/core/profiler/lib/profiler_session.h"
 #include <string>
 #include "tensorflow/contrib/tpu/profiler/trace_events.pb.h"
 #include "tensorflow/core/common_runtime/eager/context.h"
-#include "tensorflow/core/common_runtime/step_stats_collector.h"
 #include "tensorflow/core/framework/graph.pb.h"
-#include "tensorflow/core/platform/device_tracer.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/platform/types.h"
+#include "tensorflow/core/profiler/internal/gpu/tracer.h"
+#include "tensorflow/core/profiler/internal/runtime/eager_profiler.h"
 #include "tensorflow/core/protobuf/config.pb.h"
 
 namespace tensorflow {
@@ -72,90 +72,55 @@ void ConvertRunMetadataToTraceEvent(RunMetadata* run_metadata,
 
 }  // namespace
 
-/*static*/ std::unique_ptr<EagerProfiler> EagerProfiler::Create(
+/*static*/ std::unique_ptr<ProfilerSession> ProfilerSession::Create(
     EagerContext* const context) {
-  return absl::WrapUnique(new EagerProfiler(context));
+  return absl::WrapUnique(new ProfilerSession(context));
 }
 
-void EagerProfiler::BeforeClearRunMetadata() {
-  mutex_lock l(mutex_);
-  run_metadata_.MergeFrom(*context_->RunMetadataProto());
-}
-
-Status EagerProfiler::Status() {
+Status ProfilerSession::Status() {
   mutex_lock l(mutex_);
   return status_;
 }
 
-Status EagerProfiler::SerializeToString(string* content) {
+Status ProfilerSession::SerializeToString(string* content) {
   mutex_lock l(mutex_);
   if (!status_.ok()) return status_;
-  Stop();
-
-  // Get profiling data from device tracer
-  if (device_tracer_ != nullptr) {
-    std::unique_ptr<StepStatsCollector> step_stats_collector(
-        new StepStatsCollector(run_metadata_.mutable_step_stats()));
-    tensorflow::Status s = device_tracer_->Collect(step_stats_collector.get());
-    if (!s.ok()) {
-      device_tracer_.reset(nullptr);
-      LOG(WARNING) << "Failed to collect data from device tracer. "
-                   << s.error_message();
-    }
-    step_stats_collector->Finalize();
+  for (auto& profiler : profilers_) {
+    profiler->Stop().IgnoreError();
+  }
+  RunMetadata run_metadata;
+  for (auto& profiler : profilers_) {
+    profiler->CollectData(&run_metadata).IgnoreError();
   }
 
   tpu::Trace trace;
 
-  ConvertRunMetadataToTraceEvent(&run_metadata_, &trace, start_time_micros_);
+  ConvertRunMetadataToTraceEvent(&run_metadata, &trace, start_time_micros_);
 
   trace.SerializeToString(content);
   return Status::OK();
 }
 
-EagerProfiler::EagerProfiler(EagerContext* const context)
-    : context_(context),
-      start_time_micros_(Env::Default()->NowNanos() / EnvTime::kMicrosToNanos) {
-  LOG(INFO) << "Eager Profiler started.";
-
-  status_ = context_->RegisterRunMetadataListener(this);
-  if (!status_.ok()) {
-    context_ = nullptr;
-    LOG(WARNING)
-        << "Eager Profiler failed to start. Another profiler is running.";
-    return;
-  }
+ProfilerSession::ProfilerSession(EagerContext* const context)
+    : start_time_micros_(Env::Default()->NowNanos() / EnvTime::kMicrosToNanos) {
+  LOG(INFO) << "Profile Session started.";
 
-  // TODO(fishx): Allow user disable device tracer.
-  device_tracer_ = CreateDeviceTracer();
-  if (!device_tracer_) {
-    LOG(WARNING) << "Continue profiling without device tracer. "
-                 << "Failed to create device tracer.";
-    return;
+  if (context != nullptr) {
+    profilers_.push_back(
+        tensorflow::profiler::runtime::EagerProfiler::Create(context));
   }
-  class Status s = device_tracer_->Start();
-  if (!s.ok()) {
-    device_tracer_.reset(nullptr);
-    LOG(WARNING) << "Continue profiling without device tracer. "
-                 << s.error_message();
+  profilers_.push_back(tensorflow::profiler::gpu::Tracer::Create());
+
+  status_ = Status::OK();
+
+  for (auto& profiler : profilers_) {
+    profiler->Start().IgnoreError();
   }
 }
 
-EagerProfiler::~EagerProfiler() { Stop(); }
-
-void EagerProfiler::Stop() {
-  if (context_ != nullptr) {
-    context_->ClearRunMetadataListener();
-    run_metadata_.MergeFrom(*context_->RunMetadataProto());
-    context_ = nullptr;
-    if (device_tracer_ != nullptr) {
-      tensorflow::Status s = device_tracer_->Stop();
-      if (!s.ok()) {
-        device_tracer_.reset(nullptr);
-        LOG(WARNING) << "Failed to stop device tracer. " << s.error_message();
-      }
-    }
-    LOG(INFO) << "Eager Profiler ended with status:" << status_;
+ProfilerSession::~ProfilerSession() {
+  for (auto& profiler : profilers_) {
+    profiler->Stop().IgnoreError();
   }
 }
 
diff --git a/tensorflow/core/profiler/lib/eager_profiler.h b/tensorflow/core/profiler/lib/profiler_session.h
similarity index 62%
rename from tensorflow/core/profiler/lib/eager_profiler.h
rename to tensorflow/core/profiler/lib/profiler_session.h
index 7cdb76ff36..1ab4825055 100644
--- a/tensorflow/core/profiler/lib/eager_profiler.h
+++ b/tensorflow/core/profiler/lib/profiler_session.h
@@ -12,14 +12,13 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_PROFILER_H_
-#define TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_PROFILER_H_
+#ifndef TENSORFLOW_CORE_PROFILER_LIB_PROFILER_SESSION_H_
+#define TENSORFLOW_CORE_PROFILER_LIB_PROFILER_SESSION_H_
 
 #include "tensorflow/core/common_runtime/eager/context.h"
 #include "tensorflow/core/lib/core/status.h"
-#include "tensorflow/core/platform/device_tracer.h"
 #include "tensorflow/core/platform/mutex.h"
-#include "tensorflow/core/protobuf/config.pb.h"
+#include "tensorflow/core/profiler/internal/profiler_interface.h"
 
 namespace tensorflow {
 
@@ -29,38 +28,34 @@ namespace tensorflow {
 // Multiple instances of it can be created, but at most one of them will profile
 // for each EagerContext. Status() will return OK only for the instance that is
 // profiling.
-// Thread-safety: TFE_Profiler is thread-safe.
-class EagerProfiler : RunMetadataListener {
+// Thread-safety: ProfilerSession is thread-safe.
+class ProfilerSession {
  public:
-  // Creates and EagerProfiler and starts profiling.
-  static std::unique_ptr<EagerProfiler> Create(EagerContext* const context);
+  // Creates and ProfilerSession and starts profiling.
+  static std::unique_ptr<ProfilerSession> Create(EagerContext* const context);
 
   // Deletes an exsiting Profiler and enables starting a new one.
-  ~EagerProfiler() override;
+  ~ProfilerSession();
 
-  void BeforeClearRunMetadata() override LOCKS_EXCLUDED(mutex_)
-      EXCLUSIVE_LOCKS_REQUIRED(context_->MetadataMu());
   tensorflow::Status Status() LOCKS_EXCLUDED(mutex_);
 
   tensorflow::Status SerializeToString(string* content) LOCKS_EXCLUDED(mutex_);
 
  private:
   // Constructs an instance of the class and starts profiling
-  explicit EagerProfiler(EagerContext* const context);
+  explicit ProfilerSession(EagerContext* const context);
 
   // Profiler is neither copyable or movable.
-  EagerProfiler(const EagerProfiler&) = delete;
-  EagerProfiler& operator=(const EagerProfiler&) = delete;
+  ProfilerSession(const ProfilerSession&) = delete;
+  ProfilerSession& operator=(const ProfilerSession&) = delete;
 
-  void Stop() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+  std::vector<std::unique_ptr<tensorflow::profiler::ProfilerInterface>>
+      profilers_ GUARDED_BY(mutex_);
 
-  RunMetadata run_metadata_ GUARDED_BY(mutex_);
   tensorflow::Status status_ GUARDED_BY(mutex_);
-  std::unique_ptr<DeviceTracer> device_tracer_ GUARDED_BY(mutex_);
-  EagerContext* context_ GUARDED_BY(mutex_);
   const uint64 start_time_micros_;
   mutex mutex_;
 };
 
 }  // namespace tensorflow
-#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_PROFILER_H_
+#endif  // TENSORFLOW_CORE_PROFILER_LIB_PROFILER_SESSION_H_
diff --git a/tensorflow/core/profiler/rpc/BUILD b/tensorflow/core/profiler/rpc/BUILD
index 83ec75dfb4..6b4576ec9b 100644
--- a/tensorflow/core/profiler/rpc/BUILD
+++ b/tensorflow/core/profiler/rpc/BUILD
@@ -14,7 +14,7 @@ tf_cuda_library(
         "//tensorflow/contrib/tpu/profiler:tpu_profiler_proto_cc",
         "//tensorflow/core:framework",
         "//tensorflow/core/common_runtime/eager:context",
-        "//tensorflow/core/profiler/lib:eager_profiler",
+        "//tensorflow/core/profiler/lib:profiler_session",
     ],
     alwayslink = 1,
 )
diff --git a/tensorflow/core/profiler/rpc/profiler_service_impl.cc b/tensorflow/core/profiler/rpc/profiler_service_impl.cc
index bde2ff2f5f..872ef2ada9 100644
--- a/tensorflow/core/profiler/rpc/profiler_service_impl.cc
+++ b/tensorflow/core/profiler/rpc/profiler_service_impl.cc
@@ -17,7 +17,7 @@ limitations under the License.
 #include "grpcpp/support/status.h"
 #include "tensorflow/contrib/tpu/profiler/tpu_profiler.grpc.pb.h"
 #include "tensorflow/core/common_runtime/eager/context.h"
-#include "tensorflow/core/profiler/lib/eager_profiler.h"
+#include "tensorflow/core/profiler/lib/profiler_session.h"
 #include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
@@ -38,8 +38,8 @@ class ProfilerServiceImpl : public TPUProfiler::Service {
   ::grpc::Status Profile(::grpc::ServerContext* ctx, const ProfileRequest* req,
                          ProfileResponse* response) override {
     LOG(INFO) << "Received a profile request.";
-    std::unique_ptr<EagerProfiler> profiler =
-        EagerProfiler::Create(eager_context_);
+    std::unique_ptr<ProfilerSession> profiler =
+        ProfilerSession::Create(eager_context_);
     if (!profiler->Status().ok()) {
       return ::grpc::Status(::grpc::StatusCode::INTERNAL,
                             profiler->Status().error_message());
diff --git a/tensorflow/core/profiler/rpc/profiler_service_impl.h b/tensorflow/core/profiler/rpc/profiler_service_impl.h
index 79dc767eb1..311a267451 100644
--- a/tensorflow/core/profiler/rpc/profiler_service_impl.h
+++ b/tensorflow/core/profiler/rpc/profiler_service_impl.h
@@ -20,7 +20,6 @@ limitations under the License.
 #include "grpcpp/support/status.h"
 #include "tensorflow/contrib/tpu/profiler/tpu_profiler.grpc.pb.h"
 #include "tensorflow/core/common_runtime/eager/context.h"
-#include "tensorflow/core/profiler/lib/eager_profiler.h"
 
 namespace tensorflow {
 std::unique_ptr<TPUProfiler::Service> CreateProfilerService(
-- 
GitLab


From cddcca20a8d70af9caa52362c8ed7175cba9a197 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Thu, 31 Jan 2019 13:46:00 -0800
Subject: [PATCH 1677/2345] Initialize variables in tf.function without conds
 when possible

Works around the fact that we can't compile conditional initialization with XLA. It still won't work if variable initialization depends on function arguments.

Also fixes an issue where get_concrete_function would sometimes clobber variables which had already been initialized.

PiperOrigin-RevId: 231846163
---
 tensorflow/python/eager/BUILD                 |  14 +++
 tensorflow/python/eager/def_function.py       | 117 +++++++++---------
 tensorflow/python/eager/def_function_test.py  |  42 ++++++-
 .../python/eager/def_function_xla_test.py     |  49 ++++++++
 tensorflow/python/eager/lift_to_graph.py      |   7 +-
 5 files changed, 169 insertions(+), 60 deletions(-)
 create mode 100644 tensorflow/python/eager/def_function_xla_test.py

diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD
index 6f798fcdb0..af7930224b 100644
--- a/tensorflow/python/eager/BUILD
+++ b/tensorflow/python/eager/BUILD
@@ -6,6 +6,7 @@ load(
     "//tensorflow/tools/test:performance.bzl",
     "tf_py_logged_benchmark",
 )
+load("//tensorflow/compiler/tests:build_defs.bzl", "tf_xla_py_test")
 
 cc_library(
     name = "pywrap_tfe_lib",
@@ -523,6 +524,19 @@ tf_py_test(
     ],
 )
 
+tf_xla_py_test(
+    name = "def_function_xla_test",
+    srcs = ["def_function_xla_test.py"],
+    tags = ["no_pip"],
+    deps = [
+        ":def_function",
+        "//tensorflow/compiler/tests:xla_test",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:framework_ops",
+    ],
+)
+
 py_library(
     name = "wrap_function",
     srcs = ["wrap_function.py"],
diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py
index 897a38e7d3..43cb07e79d 100644
--- a/tensorflow/python/eager/def_function.py
+++ b/tensorflow/python/eager/def_function.py
@@ -25,7 +25,6 @@ import weakref
 from tensorflow.python.eager import context
 from tensorflow.python.eager import function as function_lib
 from tensorflow.python.eager import lift_to_graph
-from tensorflow.python.eager import tape
 from tensorflow.python.framework import func_graph as func_graph_module
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import control_flow_ops
@@ -57,8 +56,6 @@ class UnliftedInitializerVariable(resource_variable_ops.ResourceVariable):
                constraint=None,
                add_initializers_to=None,
                lifted_initializer_graph=None,
-               lifted_all_initializers=None,
-               lifted_placeholders=None,
                **unused_kwargs):
     """Creates a variable.
 
@@ -90,13 +87,9 @@ class UnliftedInitializerVariable(resource_variable_ops.ResourceVariable):
         (which must have the same shape). Constraints are not safe to
         use when doing asynchronous distributed training.
       add_initializers_to: if not None and not in legacy graph mode, the
-        initializer tensor will be added to this map instead of adding the
+        initializer tensor will be added to this map in addition to adding the
         assignment to the function.
       lifted_initializer_graph: FuncGraph to try to lift initializers to.
-      lifted_all_initializers: list with one boolean element, which will be
-        set to False if we cannot lift this initializer to the above graph.
-      lifted_placeholders: placeholders for resource handles lifted out of
-        this graph.
 
     Raises:
       ValueError: If the initial value is not specified, or does not have a
@@ -174,7 +167,6 @@ class UnliftedInitializerVariable(resource_variable_ops.ResourceVariable):
             with ops.name_scope("Assign") as n, ops.colocate_with(self._handle):
               self._initializer_op = resource_variable_ops.assign_variable_op(
                   self._handle, lifted_initializer, name=n)
-              assign = self._initializer_op
           with ops.name_scope("Read"), ops.colocate_with(self._handle):
             # Manually assign reads to the handle's device to avoid log
             # messages.
@@ -185,32 +177,21 @@ class UnliftedInitializerVariable(resource_variable_ops.ResourceVariable):
       else:
         if add_initializers_to is not None:
           add_initializers_to[self] = initial_value
-          assign = None
-        else:
-          def assign_fn():
-            with ops.name_scope("Assign") as n, ops.colocate_with(self._handle):
-              resource_variable_ops.assign_variable_op(
-                  self._handle,
-                  initial_value,
-                  name=n)
-              # Returning values to keep tf.cond happy.
-            return ops.convert_to_tensor(1)
-          def not_assign_fn():
-            return ops.convert_to_tensor(0)
-          # Note: this cond is always guaranteed to run because we're inside a
-          # defun which will insert automatic control dependencies.
-          assign = control_flow_ops.cond(
-              resource_variable_ops.var_is_initialized_op(self._handle),
-              not_assign_fn, assign_fn)
-      if lifted_initializer_graph is not None and assign is not None:
-        try:
-          handle_placeholder = ops.convert_to_tensor(self._handle)
-          op_map = lift_to_graph.lift_to_graph(
-              assign, lifted_initializer_graph,
-              sources=[handle_placeholder])
-          lifted_placeholders.append((self._handle, op_map[handle_placeholder]))
-        except ValueError:
-          lifted_all_initializers[0] = False
+        def assign_fn():
+          with ops.name_scope("Assign") as n, ops.colocate_with(self._handle):
+            resource_variable_ops.assign_variable_op(
+                self._handle,
+                initial_value,
+                name=n)
+            # Returning values to keep tf.cond happy.
+          return ops.convert_to_tensor(1)
+        def not_assign_fn():
+          return ops.convert_to_tensor(0)
+        # Note: this cond is always guaranteed to run because we're inside a
+        # defun which will insert automatic control dependencies.
+        control_flow_ops.cond(
+            resource_variable_ops.var_is_initialized_op(self._handle),
+            not_assign_fn, assign_fn)
 
     # After the handle has been created, set up a way to clean it up when
     # executing eagerly. We'll hold the only reference to the deleter, so that
@@ -340,16 +321,12 @@ class Function(object):
 
     created_variables = []
     lifted_initializer_graph = func_graph_module.FuncGraph("initializer")
-    lifted_all_initializers = [True]
-    lifted_placeholders = []
 
     def variable_capturing_scope(unused_next_creator, **kwds):
       """Creates UnliftedInitializerVariables and saves references to them."""
       v = UnliftedInitializerVariable(
           add_initializers_to=add_initializers_to,
-          lifted_initializer_graph=lifted_initializer_graph,
-          lifted_all_initializers=lifted_all_initializers,
-          lifted_placeholders=lifted_placeholders, **kwds)
+          lifted_initializer_graph=lifted_initializer_graph, **kwds)
       created_variables.append(weakref.ref(v))
       return v
 
@@ -359,11 +336,9 @@ class Function(object):
     # Force the definition of the function for these arguments
     self._lifted_initializer_graph = lifted_initializer_graph
     self._graph_deleter = FunctionDeleter(self._lifted_initializer_graph)
-    self._lifted_placeholders = lifted_placeholders
     self._concrete_stateful_fn = (
         self._stateful_fn._get_concrete_function_internal_garbage_collected(  # pylint: disable=protected-access
             *args, **kwds))
-    self._lifted_all_initializers = lifted_all_initializers[0]
 
     def invalid_creator_scope(*unused_args, **unused_kwds):
       """Disables variable creation."""
@@ -390,21 +365,22 @@ class Function(object):
       return results
 
     # This is the first call of __call__, so we have to initialize.
-    self._initialize(args, kwds)
-    if self._lifted_all_initializers and self._lifted_placeholders:
-      with ops.init_scope():
-        handles, placeholders = zip(*self._lifted_placeholders)
-        if context.executing_eagerly():
-          lifted_fn = function_lib._EagerDefinedFunction(  # pylint: disable=protected-access
-              "initializer" + str(ops.uid()),
-              self._lifted_initializer_graph,
-              placeholders, [], {})
-          with tape.stop_recording():
-            lifted_fn.call(context.context(), list(handles))
-      return self._stateless_fn(*args, **kwds)
-    canon_args, canon_kwds = self._canonicalize_function_inputs(args, kwds)
-
-    if not self._created_variables:
+    initializer_map = {}
+    self._initialize(args, kwds, add_initializers_to=initializer_map)
+    if self._created_variables:
+      try:
+        # Attempt to initialize variables eagerly and without conds by lifting
+        # out initialization graphs. This is the only initialization strategy
+        # compatible with XLA at the moment.
+        self._initialize_uninitialized_variables(initializer_map)
+      except lift_to_graph.UnliftableError:
+        pass  # Fall through to cond-based initialization.
+      else:
+        # Lifting succeeded, so variables are initialized and we can run the
+        # stateless function.
+        return self._stateless_fn(*args, **kwds)
+    else:
+      canon_args, canon_kwds = self._canonicalize_function_inputs(args, kwds)
       # If we did not create any variables the trace we have is good enough.
       return self._concrete_stateful_fn._filtered_call(canon_args, canon_kwds)  # pylint: disable=protected-access
 
@@ -459,6 +435,9 @@ class Function(object):
           functools.partial(self._concrete_stateful_fn._filtered_call,  # pylint: disable=protected-access
                             inner_args, inner_kwds))
 
+    # We've created variables and are unable to lift the initialization graphs,
+    # so we fall back to initializing with conds while running the function.
+    canon_args, canon_kwds = self._canonicalize_function_inputs(args, kwds)
     return function_lib.defun(fn_with_cond)(*canon_args, **canon_kwds)
 
   @property
@@ -474,6 +453,23 @@ class Function(object):
   def function_spec(self):
     return self._function_spec
 
+  def _initialize_uninitialized_variables(self, initializer_map):
+    """Make and call a `ConcreteFunction` which initializes variables."""
+
+    # Note: using defun here avoids an infinite recursion.
+    @function_lib.defun
+    def initialize_variables():
+      for v, init in initializer_map.items():
+        with ops.init_scope():
+          if resource_variable_ops.var_is_initialized_op(v.handle):
+            # Ignore variables which are already initialized at trace time.
+            continue
+        v.assign(lift_to_graph.lift_to_graph(
+            init, ops.get_default_graph())[init])
+
+    with ops.init_scope():
+      return initialize_variables.get_concrete_function()()
+
   def get_initialization_function(self, *args, **kwargs):
     """Returns a `ConcreteFunction` which initializes this function's variables.
 
@@ -482,6 +478,9 @@ class Function(object):
     function which does not depend on the concrete values of the inputs to this
     function.
 
+    Note that running this function will overwrite any values currently assigned
+    to variables, for example restores from a checkpoint.
+
     Args:
       *args: arguments to the underlying python callable.
       **kwargs: keyword arguments to the python callable.
@@ -626,7 +625,9 @@ class Function(object):
     """
     assert context.executing_eagerly()
     if self._stateful_fn is None:
-      self.get_initialization_function(*args, **kwargs)()
+      initializer_map = {}
+      self._initialize(args, kwargs, add_initializers_to=initializer_map)
+      self._initialize_uninitialized_variables(initializer_map)
 
     if self._created_variables:
       # In this case we have created variables on the first call, so we run the
diff --git a/tensorflow/python/eager/def_function_test.py b/tensorflow/python/eager/def_function_test.py
index 912198dfcf..4a47f67fd1 100644
--- a/tensorflow/python/eager/def_function_test.py
+++ b/tensorflow/python/eager/def_function_test.py
@@ -22,6 +22,7 @@ import weakref
 
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import def_function
+from tensorflow.python.eager import lift_to_graph
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -208,7 +209,7 @@ class DefFunctionTest(test.TestCase):
           state.append(variables.Variable(2.0 * x))
         return state[0] * x
 
-      with self.assertRaises(ValueError):
+      with self.assertRaises(lift_to_graph.UnliftableError):
         fn(constant_op.constant(3.0))
 
   def testMethod(self):
@@ -343,6 +344,45 @@ class DefFunctionTest(test.TestCase):
       f()
     self.assertEqual(created_variables, captured_variables)
 
+  def testVarAlreadyInitializedNoClobbering(self):
+    v_holder = []
+
+    @def_function.function
+    def add_var(x):
+      if not v_holder:
+        v = variables.Variable([1., 2.])
+        v_holder.append(v)
+        already_initialized = variables.Variable(3.)
+        with ops.init_scope():
+          already_initialized.assign(10.)
+        v_holder.append(already_initialized)
+      return v_holder[0] + v_holder[1] + x
+
+    add_var.get_concrete_function(constant_op.constant(2.))
+    self.assertAllClose([13., 14.], add_var(constant_op.constant(2.)))
+
+  def testInitializationInNestedCall(self):
+    v_holder = []
+
+    @def_function.function
+    def add_var(x):
+      if not v_holder:
+        v = variables.Variable([1., 2.])
+        v_holder.append(v)
+        already_initialized = variables.Variable(3.)
+        with ops.init_scope():
+          already_initialized.assign(10.)
+        v_holder.append(already_initialized)
+      return v_holder[0] + v_holder[1] + x
+
+    @def_function.function
+    def wrapper(x):
+      return add_var(x)
+
+    self.assertAllClose([13., 14.], wrapper(constant_op.constant(2.)))
+    v_holder[1].assign(11.)
+    self.assertAllClose([14., 15.], wrapper(constant_op.constant(2.)))
+
 
 if __name__ == '__main__':
   ops.enable_eager_execution()
diff --git a/tensorflow/python/eager/def_function_xla_test.py b/tensorflow/python/eager/def_function_xla_test.py
new file mode 100644
index 0000000000..9115d8a694
--- /dev/null
+++ b/tensorflow/python/eager/def_function_xla_test.py
@@ -0,0 +1,49 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.compiler.tests import xla_test
+from tensorflow.python.eager import def_function
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+
+
+class DefFunctionTests(xla_test.XLATestCase):
+
+  def testVarInitializedInFunction(self):
+    with self.test_scope():
+      v_holder = []
+
+      @def_function.function
+      def add_var(x):
+        if not v_holder:
+          v = variables.Variable([1., 2.])
+          v_holder.append(v)
+          already_initialized = variables.Variable(3.)
+          with ops.init_scope():
+            already_initialized.assign(10.)
+          v_holder.append(already_initialized)
+        return v_holder[0] + v_holder[1] + x
+
+      self.assertAllClose([13., 14.], add_var(constant_op.constant(2.)))
+
+
+if __name__ == "__main__":
+  ops.enable_eager_execution()
+  test.main()
diff --git a/tensorflow/python/eager/lift_to_graph.py b/tensorflow/python/eager/lift_to_graph.py
index 2e9d24f61e..ad62e6d10a 100644
--- a/tensorflow/python/eager/lift_to_graph.py
+++ b/tensorflow/python/eager/lift_to_graph.py
@@ -35,6 +35,11 @@ def _as_operation(op_or_tensor):
   return op_or_tensor
 
 
+class UnliftableError(Exception):
+  """Raised if a Tensor cannot be lifted from the graph."""
+  pass
+
+
 def lift_to_graph(init_tensor, graph, sources=None):
   """Copies the tensor and all its inputs recursively to the outer graph."""
   # Check that the initializer does not depend on any placeholders.
@@ -52,7 +57,7 @@ def lift_to_graph(init_tensor, graph, sources=None):
     # and placeholders the user might directly use to initialize
     # variables.
     if op.type == "Placeholder":
-      raise ValueError(
+      raise UnliftableError(
           "Unable to lift tensor", init_tensor,
           "because it depends transitively on placeholder ", op)
     for inp in _graph_inputs(op):
-- 
GitLab


From 6aec62eab07b59c433c30a4de03f844d651b5121 Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Thu, 31 Jan 2019 13:49:52 -0800
Subject: [PATCH 1678/2345] Remove unused stream executor convolution
 functions.

PiperOrigin-RevId: 231846823
---
 tensorflow/stream_executor/stream.cc | 365 +--------------------------
 tensorflow/stream_executor/stream.h  | 126 ---------
 2 files changed, 5 insertions(+), 486 deletions(-)

diff --git a/tensorflow/stream_executor/stream.cc b/tensorflow/stream_executor/stream.cc
index fde7730cfe..bd89147a26 100644
--- a/tensorflow/stream_executor/stream.cc
+++ b/tensorflow/stream_executor/stream.cc
@@ -431,172 +431,6 @@ Stream &Stream::ThenBatchNormalizationBackward(
   return *this;
 }
 
-Stream &Stream::ThenFusedConvolveWithScratch(
-    const dnn::BatchDescriptor &conv_input_descriptor,
-    const DeviceMemory<int8> &conv_input_data, float conv_input_scale,
-    const dnn::FilterDescriptor &filter_descriptor,
-    const DeviceMemory<int8> &filter_data,
-    const dnn::ConvolutionDescriptor &convolution_descriptor,
-    const DeviceMemory<int8> &side_input_data, float side_input_scale,
-    const dnn::BatchDescriptor &bias_descriptor,
-    const DeviceMemory<float> &biases, dnn::ActivationMode activation_mode,
-    const dnn::BatchDescriptor &output_descriptor, DeviceMemory<int8> *output,
-    ScratchAllocator *scratch_allocator) {
-  VLOG_CALL(PARAM(conv_input_descriptor), PARAM(conv_input_data),
-            PARAM(conv_input_scale), PARAM(filter_descriptor),
-            PARAM(filter_data), PARAM(convolution_descriptor),
-            PARAM(side_input_data), PARAM(side_input_scale),
-            PARAM(bias_descriptor), PARAM(biases), PARAM(activation_mode),
-            PARAM(output_descriptor), PARAM(output));
-
-  if (ok()) {
-    if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      CheckError(dnn->DoFusedConvolve(
-          this, conv_input_descriptor, conv_input_data, conv_input_scale,
-          filter_descriptor, filter_data, convolution_descriptor,
-          side_input_data, side_input_scale, bias_descriptor, biases,
-          activation_mode, output_descriptor, output, scratch_allocator,
-          dnn::AlgorithmConfig(), /*output_profile_result=*/nullptr));
-    } else {
-      SetErrorAndLogNoDnnSupport();
-    }
-  }
-  return *this;
-}
-
-Stream &Stream::ThenFusedConvolveWithScratch(
-    const dnn::BatchDescriptor &conv_input_descriptor,
-    const DeviceMemory<Eigen::half> &conv_input_data, float conv_input_scale,
-    const dnn::FilterDescriptor &filter_descriptor,
-    const DeviceMemory<Eigen::half> &filter_data,
-    const dnn::ConvolutionDescriptor &convolution_descriptor,
-    const DeviceMemory<Eigen::half> &side_input_data, float side_input_scale,
-    const dnn::BatchDescriptor &bias_descriptor,
-    const DeviceMemory<Eigen::half> &biases,
-    dnn::ActivationMode activation_mode,
-    const dnn::BatchDescriptor &output_descriptor,
-    DeviceMemory<Eigen::half> *output, ScratchAllocator *scratch_allocator) {
-  VLOG_CALL(PARAM(conv_input_descriptor), PARAM(conv_input_data),
-            PARAM(conv_input_scale), PARAM(filter_descriptor),
-            PARAM(filter_data), PARAM(convolution_descriptor),
-            PARAM(side_input_data), PARAM(side_input_scale),
-            PARAM(bias_descriptor), PARAM(biases), PARAM(activation_mode),
-            PARAM(output_descriptor), PARAM(output));
-
-  if (ok()) {
-    if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      CheckError(dnn->DoFusedConvolve(
-          this, conv_input_descriptor, conv_input_data, conv_input_scale,
-          filter_descriptor, filter_data, convolution_descriptor,
-          side_input_data, side_input_scale, bias_descriptor, biases,
-          activation_mode, output_descriptor, output, scratch_allocator,
-          dnn::AlgorithmConfig(), /*output_profile_result=*/nullptr));
-    } else {
-      SetErrorAndLogNoDnnSupport();
-    }
-  }
-  return *this;
-}
-
-Stream &Stream::ThenFusedConvolveWithScratch(
-    const dnn::BatchDescriptor &conv_input_descriptor,
-    const DeviceMemory<float> &conv_input_data, float conv_input_scale,
-    const dnn::FilterDescriptor &filter_descriptor,
-    const DeviceMemory<float> &filter_data,
-    const dnn::ConvolutionDescriptor &convolution_descriptor,
-    const DeviceMemory<float> &side_input_data, float side_input_scale,
-    const dnn::BatchDescriptor &bias_descriptor,
-    const DeviceMemory<float> &biases, dnn::ActivationMode activation_mode,
-    const dnn::BatchDescriptor &output_descriptor, DeviceMemory<float> *output,
-    ScratchAllocator *scratch_allocator) {
-  VLOG_CALL(PARAM(conv_input_descriptor), PARAM(conv_input_data),
-            PARAM(conv_input_scale), PARAM(filter_descriptor),
-            PARAM(filter_data), PARAM(convolution_descriptor),
-            PARAM(side_input_data), PARAM(side_input_scale),
-            PARAM(bias_descriptor), PARAM(biases), PARAM(activation_mode),
-            PARAM(output_descriptor), PARAM(output));
-
-  if (ok()) {
-    if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      CheckError(dnn->DoFusedConvolve(
-          this, conv_input_descriptor, conv_input_data, conv_input_scale,
-          filter_descriptor, filter_data, convolution_descriptor,
-          side_input_data, side_input_scale, bias_descriptor, biases,
-          activation_mode, output_descriptor, output, scratch_allocator,
-          dnn::AlgorithmConfig(), /*output_profile_result=*/nullptr));
-    } else {
-      SetErrorAndLogNoDnnSupport();
-    }
-  }
-  return *this;
-}
-
-Stream &Stream::ThenConvolveWithScratch(
-    const dnn::BatchDescriptor &input_descriptor,
-    const DeviceMemory<Eigen::half> &input_data,
-    const dnn::FilterDescriptor &filter_descriptor,
-    const DeviceMemory<Eigen::half> &filter_data,
-    const dnn::ConvolutionDescriptor &convolution_descriptor,
-    const dnn::BatchDescriptor &output_descriptor,
-    DeviceMemory<Eigen::half> *output, ScratchAllocator *scratch_allocator) {
-  VLOG_CALL(PARAM(input_descriptor), PARAM(input_data),
-            PARAM(filter_descriptor), PARAM(filter_data),
-            PARAM(convolution_descriptor), PARAM(output_descriptor),
-            PARAM(output));
-
-  if (ok()) {
-    if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      DeviceMemory<uint8> scratch_memory;
-      dnn::AlgorithmDesc algorithm_desc;
-      CheckStatus(dnn->PrepareForConvolution(
-          dnn::ConvolutionKind::FORWARD, this, input_descriptor, input_data,
-          filter_descriptor, filter_data, output_descriptor, *output,
-          convolution_descriptor, dnn::AlgorithmConfig(), scratch_allocator,
-          &algorithm_desc, &scratch_memory));
-      CheckError(dnn->DoConvolve(
-          this, input_descriptor, input_data, filter_descriptor, filter_data,
-          convolution_descriptor, output_descriptor, output, algorithm_desc,
-          &scratch_memory, nullptr));
-    } else {
-      SetErrorAndLogNoDnnSupport();
-    }
-  }
-  return *this;
-}
-
-Stream &Stream::ThenConvolveWithScratch(
-    const dnn::BatchDescriptor &input_descriptor,
-    const DeviceMemory<float> &input_data,
-    const dnn::FilterDescriptor &filter_descriptor,
-    const DeviceMemory<float> &filter_data,
-    const dnn::ConvolutionDescriptor &convolution_descriptor,
-    const dnn::BatchDescriptor &output_descriptor, DeviceMemory<float> *output,
-    ScratchAllocator *scratch_allocator) {
-  VLOG_CALL(PARAM(input_descriptor), PARAM(input_data),
-            PARAM(filter_descriptor), PARAM(filter_data),
-            PARAM(convolution_descriptor), PARAM(output_descriptor),
-            PARAM(output));
-
-  if (ok()) {
-    if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      DeviceMemory<uint8> scratch_memory;
-      dnn::AlgorithmDesc algorithm_desc;
-      CheckStatus(dnn->PrepareForConvolution(
-          dnn::ConvolutionKind::FORWARD, this, input_descriptor, input_data,
-          filter_descriptor, filter_data, output_descriptor, *output,
-          convolution_descriptor, dnn::AlgorithmConfig(), scratch_allocator,
-          &algorithm_desc, &scratch_memory));
-      CheckError(dnn->DoConvolve(
-          this, input_descriptor, input_data, filter_descriptor, filter_data,
-          convolution_descriptor, output_descriptor, output, algorithm_desc,
-          &scratch_memory, nullptr));
-    } else {
-      SetErrorAndLogNoDnnSupport();
-    }
-  }
-  return *this;
-}
-
 Stream &Stream::ThenFusedConvolveWithAlgorithm(
     const dnn::BatchDescriptor &conv_input_descriptor,
     const DeviceMemory<double> &conv_input_data, double conv_input_scale,
@@ -876,24 +710,6 @@ Stream &Stream::ThenConvolveWithAlgorithm(
   return *this;
 }
 
-Stream &Stream::ThenFusedConvolve(
-    const dnn::BatchDescriptor &conv_input_descriptor,
-    const DeviceMemory<int8> &conv_input_data, float conv_input_scale,
-    const dnn::FilterDescriptor &filter_descriptor,
-    const DeviceMemory<int8> &filter_data,
-    const dnn::ConvolutionDescriptor &convolution_descriptor,
-    const DeviceMemory<int8> &side_input_data, float side_input_scale,
-    const dnn::BatchDescriptor &bias_descriptor,
-    const DeviceMemory<float> &biases, dnn::ActivationMode activation_mode,
-    const dnn::BatchDescriptor &output_descriptor, DeviceMemory<int8> *output) {
-  return ThenFusedConvolveWithScratch(
-      conv_input_descriptor, conv_input_data, conv_input_scale,
-      filter_descriptor, filter_data, convolution_descriptor, side_input_data,
-      side_input_scale, bias_descriptor, biases, activation_mode,
-      output_descriptor, output,
-      /*scratch_allocator=*/nullptr);
-}
-
 Stream &Stream::ThenConvolve(
     const dnn::BatchDescriptor &input_descriptor,
     const DeviceMemory<float> &input_data,
@@ -902,10 +718,11 @@ Stream &Stream::ThenConvolve(
     const dnn::ConvolutionDescriptor &convolution_descriptor,
     const dnn::BatchDescriptor &output_descriptor,
     DeviceMemory<float> *output) {
-  return ThenConvolveWithScratch(input_descriptor, input_data,
-                                 filter_descriptor, filter_data,
-                                 convolution_descriptor, output_descriptor,
-                                 output, /*scratch_allocator=*/nullptr);
+  return ThenConvolveWithAlgorithm(
+      input_descriptor, input_data, filter_descriptor, filter_data,
+      convolution_descriptor, output_descriptor, output,
+      /*scratch_allocator=*/nullptr, dnn::AlgorithmConfig(),
+      /*output_profile_result=*/nullptr);
 }
 
 Stream &Stream::ThenConvolveQuantized(
@@ -995,42 +812,6 @@ Stream &Stream::ThenSeparableConvolve(
   return *this;
 }
 
-Stream &Stream::ThenConvolveBackwardDataWithScratch(
-    const dnn::FilterDescriptor &filter_descriptor,
-    const DeviceMemory<float> &filter_data,
-    const dnn::BatchDescriptor &output_descriptor,
-    DeviceMemory<float> backward_output_data,
-    const dnn::ConvolutionDescriptor &convolution_descriptor,
-    const dnn::BatchDescriptor &input_descriptor,
-    DeviceMemory<float> *backward_input_data,
-    ScratchAllocator *scratch_allocator) {
-  VLOG_CALL(PARAM(filter_descriptor), PARAM(filter_data),
-            PARAM(output_descriptor), PARAM(backward_output_data),
-            PARAM(convolution_descriptor), PARAM(input_descriptor),
-            PARAM(backward_input_data));
-
-  if (ok()) {
-    if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      DeviceMemory<uint8> scratch_memory;
-      dnn::AlgorithmDesc algorithm_desc;
-      CheckStatus(dnn->PrepareForConvolution(
-          dnn::ConvolutionKind::BACKWARD_DATA, this, input_descriptor,
-          *backward_input_data, filter_descriptor, filter_data,
-          output_descriptor, backward_output_data, convolution_descriptor,
-          dnn::AlgorithmConfig(), scratch_allocator, &algorithm_desc,
-          &scratch_memory));
-      CheckError(dnn->DoConvolveBackwardData(
-          this, filter_descriptor, filter_data, output_descriptor,
-          backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, algorithm_desc, &scratch_memory,
-          /*output_profile_result=*/nullptr));
-    } else {
-      SetErrorAndLogNoDnnSupport();
-    }
-  }
-  return *this;
-}
-
 Stream &Stream::ThenConvolveBackwardDataWithAlgorithm(
     const dnn::FilterDescriptor &filter_descriptor,
     const DeviceMemory<double> &filter_data,
@@ -1166,92 +947,6 @@ Stream &Stream::ThenConvolveBackwardDataWithAlgorithm(
   return *this;
 }
 
-Stream &Stream::ThenConvolveBackwardDataWithScratch(
-    const dnn::FilterDescriptor &filter_descriptor,
-    const DeviceMemory<Eigen::half> &filter_data,
-    const dnn::BatchDescriptor &output_descriptor,
-    DeviceMemory<Eigen::half> backward_output_data,
-    const dnn::ConvolutionDescriptor &convolution_descriptor,
-    const dnn::BatchDescriptor &input_descriptor,
-    DeviceMemory<Eigen::half> *backward_input_data,
-    ScratchAllocator *scratch_allocator) {
-  VLOG_CALL(PARAM(filter_descriptor), PARAM(filter_data),
-            PARAM(output_descriptor), PARAM(backward_output_data),
-            PARAM(convolution_descriptor), PARAM(input_descriptor),
-            PARAM(backward_input_data));
-
-  if (ok()) {
-    if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      DeviceMemory<uint8> scratch_memory;
-      dnn::AlgorithmDesc algorithm_desc;
-      CheckStatus(dnn->PrepareForConvolution(
-          dnn::ConvolutionKind::BACKWARD_DATA, this, input_descriptor,
-          *backward_input_data, filter_descriptor, filter_data,
-          output_descriptor, backward_output_data, convolution_descriptor,
-          dnn::AlgorithmConfig(), scratch_allocator, &algorithm_desc,
-          &scratch_memory));
-      CheckError(dnn->DoConvolveBackwardData(
-          this, filter_descriptor, filter_data, output_descriptor,
-          backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, algorithm_desc, &scratch_memory,
-          /*output_profile_result=*/nullptr));
-    } else {
-      SetErrorAndLogNoDnnSupport();
-    }
-  }
-  return *this;
-}
-
-Stream &Stream::ThenConvolveBackwardData(
-    const dnn::FilterDescriptor &filter_descriptor,
-    const DeviceMemory<float> &filter_data,
-    const dnn::BatchDescriptor &output_descriptor,
-    DeviceMemory<float> backward_output_data,
-    const dnn::ConvolutionDescriptor &convolution_descriptor,
-    const dnn::BatchDescriptor &input_descriptor,
-    DeviceMemory<float> *backward_input_data) {
-  return ThenConvolveBackwardDataWithScratch(
-      filter_descriptor, filter_data, output_descriptor, backward_output_data,
-      convolution_descriptor, input_descriptor, backward_input_data,
-      /*scratch_allocator=*/nullptr);
-}
-
-Stream &Stream::ThenConvolveBackwardFilterWithScratch(
-    const dnn::BatchDescriptor &input_descriptor,
-    const DeviceMemory<float> &input_data,
-    const dnn::BatchDescriptor &output_descriptor,
-    DeviceMemory<float> backward_output_data,
-    const dnn::ConvolutionDescriptor &convolution_descriptor,
-    const dnn::FilterDescriptor &filter_descriptor,
-    DeviceMemory<float> *backward_filter_data,
-    ScratchAllocator *scratch_allocator) {
-  VLOG_CALL(PARAM(input_descriptor), PARAM(input_data),
-            PARAM(output_descriptor), PARAM(backward_output_data),
-            PARAM(convolution_descriptor), PARAM(filter_descriptor),
-            PARAM(backward_filter_data));
-
-  if (ok()) {
-    if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      DeviceMemory<uint8> scratch_memory;
-      dnn::AlgorithmDesc algorithm_desc;
-      CheckStatus(dnn->PrepareForConvolution(
-          dnn::ConvolutionKind::BACKWARD_FILTER, this, input_descriptor,
-          input_data, filter_descriptor, *backward_filter_data,
-          output_descriptor, backward_output_data, convolution_descriptor,
-          dnn::AlgorithmConfig(), scratch_allocator, &algorithm_desc,
-          &scratch_memory));
-      CheckError(dnn->DoConvolveBackwardFilter(
-          this, input_descriptor, input_data, output_descriptor,
-          backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, algorithm_desc, &scratch_memory,
-          /*output_profile_result=*/nullptr));
-    } else {
-      SetErrorAndLogNoDnnSupport();
-    }
-  }
-  return *this;
-}
-
 Stream &Stream::ThenConvolveBackwardFilterWithAlgorithm(
     const dnn::BatchDescriptor &input_descriptor,
     const DeviceMemory<double> &input_data,
@@ -1342,42 +1037,6 @@ Stream &Stream::ThenConvolveBackwardFilterWithAlgorithm(
   return *this;
 }
 
-Stream &Stream::ThenConvolveBackwardFilterWithScratch(
-    const dnn::BatchDescriptor &input_descriptor,
-    const DeviceMemory<Eigen::half> &input_data,
-    const dnn::BatchDescriptor &output_descriptor,
-    DeviceMemory<Eigen::half> backward_output_data,
-    const dnn::ConvolutionDescriptor &convolution_descriptor,
-    const dnn::FilterDescriptor &filter_descriptor,
-    DeviceMemory<Eigen::half> *backward_filter_data,
-    ScratchAllocator *scratch_allocator) {
-  VLOG_CALL(PARAM(input_descriptor), PARAM(input_data),
-            PARAM(output_descriptor), PARAM(backward_output_data),
-            PARAM(convolution_descriptor), PARAM(filter_descriptor),
-            PARAM(backward_filter_data));
-
-  if (ok()) {
-    if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      DeviceMemory<uint8> scratch_memory;
-      dnn::AlgorithmDesc algorithm_desc;
-      CheckStatus(dnn->PrepareForConvolution(
-          dnn::ConvolutionKind::BACKWARD_FILTER, this, input_descriptor,
-          input_data, filter_descriptor, *backward_filter_data,
-          output_descriptor, backward_output_data, convolution_descriptor,
-          dnn::AlgorithmConfig(), scratch_allocator, &algorithm_desc,
-          &scratch_memory));
-      CheckError(dnn->DoConvolveBackwardFilter(
-          this, input_descriptor, input_data, output_descriptor,
-          backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, algorithm_desc, &scratch_memory,
-          /*output_profile_result=*/nullptr));
-    } else {
-      SetErrorAndLogNoDnnSupport();
-    }
-  }
-  return *this;
-}
-
 Stream &Stream::ThenConvolveBackwardFilterWithAlgorithm(
     const dnn::BatchDescriptor &input_descriptor,
     const DeviceMemory<Eigen::half> &input_data,
@@ -1423,20 +1082,6 @@ Stream &Stream::ThenConvolveBackwardFilterWithAlgorithm(
   return *this;
 }
 
-Stream &Stream::ThenConvolveBackwardFilter(
-    const dnn::BatchDescriptor &input_descriptor,
-    const DeviceMemory<float> &input_data,
-    const dnn::BatchDescriptor &output_descriptor,
-    DeviceMemory<float> backward_output_data,
-    const dnn::ConvolutionDescriptor &convolution_descriptor,
-    const dnn::FilterDescriptor &filter_descriptor,
-    DeviceMemory<float> *backward_filter_data) {
-  return ThenConvolveBackwardFilterWithScratch(
-      input_descriptor, input_data, output_descriptor, backward_output_data,
-      convolution_descriptor, filter_descriptor, backward_filter_data,
-      /*scratch_allocator=*/nullptr);
-}
-
 template <typename T>
 Stream &Stream::ThenConvolveBackwardBiasImpl(
     const dnn::BatchDescriptor &input_descriptor,
diff --git a/tensorflow/stream_executor/stream.h b/tensorflow/stream_executor/stream.h
index 65025c2bc8..dd4b58e602 100644
--- a/tensorflow/stream_executor/stream.h
+++ b/tensorflow/stream_executor/stream.h
@@ -262,19 +262,6 @@ class Stream {
       DeviceMemory<float> *scale_backprop,
       DeviceMemory<float> *offset_backprop);
 
-  // TODO(leary) add double-precision version of this interface.
-  Stream &ThenFusedConvolve(
-      const dnn::BatchDescriptor &conv_input_descriptor,
-      const DeviceMemory<int8> &conv_input_data, float conv_input_scale,
-      const dnn::FilterDescriptor &filter_descriptor,
-      const DeviceMemory<int8> &filter_data,
-      const dnn::ConvolutionDescriptor &convolution_descriptor,
-      const DeviceMemory<int8> &side_input_data, float side_input_scale,
-      const dnn::BatchDescriptor &bias_descriptor,
-      const DeviceMemory<float> &biases, dnn::ActivationMode activation_mode,
-      const dnn::BatchDescriptor &output_descriptor,
-      DeviceMemory<int8> *output);
-
   Stream &ThenConvolve(const dnn::BatchDescriptor &input_descriptor,
                        const DeviceMemory<float> &input_data,
                        const dnn::FilterDescriptor &filter_descriptor,
@@ -303,61 +290,6 @@ class Stream {
       const dnn::BatchDescriptor &output_descriptor,
       DeviceMemory<float> *output_data);
 
-  Stream &ThenFusedConvolveWithScratch(
-      const dnn::BatchDescriptor &conv_input_descriptor,
-      const DeviceMemory<int8> &conv_input_data, float conv_input_scale,
-      const dnn::FilterDescriptor &filter_descriptor,
-      const DeviceMemory<int8> &filter_data,
-      const dnn::ConvolutionDescriptor &convolution_descriptor,
-      const DeviceMemory<int8> &side_input_data, float side_input_scale,
-      const dnn::BatchDescriptor &bias_descriptor,
-      const DeviceMemory<float> &biases, dnn::ActivationMode activation_mode,
-      const dnn::BatchDescriptor &output_descriptor, DeviceMemory<int8> *output,
-      ScratchAllocator *scratch_allocator);
-
-  Stream &ThenFusedConvolveWithScratch(
-      const dnn::BatchDescriptor &conv_input_descriptor,
-      const DeviceMemory<Eigen::half> &conv_input_data, float conv_input_scale,
-      const dnn::FilterDescriptor &filter_descriptor,
-      const DeviceMemory<Eigen::half> &filter_data,
-      const dnn::ConvolutionDescriptor &convolution_descriptor,
-      const DeviceMemory<Eigen::half> &side_input_data, float side_input_scale,
-      const dnn::BatchDescriptor &bias_descriptor,
-      const DeviceMemory<Eigen::half> &biases,
-      dnn::ActivationMode activation_mode,
-      const dnn::BatchDescriptor &output_descriptor,
-      DeviceMemory<Eigen::half> *output, ScratchAllocator *scratch_allocator);
-
-  Stream &ThenFusedConvolveWithScratch(
-      const dnn::BatchDescriptor &conv_input_descriptor,
-      const DeviceMemory<float> &conv_input_data, float conv_input_scale,
-      const dnn::FilterDescriptor &filter_descriptor,
-      const DeviceMemory<float> &filter_data,
-      const dnn::ConvolutionDescriptor &convolution_descriptor,
-      const DeviceMemory<float> &side_input_data, float side_input_scale,
-      const dnn::BatchDescriptor &bias_descriptor,
-      const DeviceMemory<float> &biases, dnn::ActivationMode activation_mode,
-      const dnn::BatchDescriptor &output_descriptor,
-      DeviceMemory<float> *output, ScratchAllocator *scratch_allocator);
-
-  Stream &ThenConvolveWithScratch(
-      const dnn::BatchDescriptor &input_descriptor,
-      const DeviceMemory<Eigen::half> &input_data,
-      const dnn::FilterDescriptor &filter_descriptor,
-      const DeviceMemory<Eigen::half> &filter_data,
-      const dnn::ConvolutionDescriptor &convolution_descriptor,
-      const dnn::BatchDescriptor &output_descriptor,
-      DeviceMemory<Eigen::half> *output, ScratchAllocator *scratch_allocator);
-
-  Stream &ThenConvolveWithScratch(
-      const dnn::BatchDescriptor &input_descriptor,
-      const DeviceMemory<float> &input_data,
-      const dnn::FilterDescriptor &filter_descriptor,
-      const DeviceMemory<float> &filter_data,
-      const dnn::ConvolutionDescriptor &convolution_descriptor,
-      const dnn::BatchDescriptor &output_descriptor,
-      DeviceMemory<float> *output, ScratchAllocator *scratch_allocator);
-
   Stream &ThenConvolveWithAlgorithm(
       const dnn::BatchDescriptor &input_descriptor,
       const DeviceMemory<double> &input_data,
@@ -458,35 +390,6 @@ class Stream {
       const dnn::BatchDescriptor &output_descriptor,
       DeviceMemory<float> *output);
 
-  Stream &ThenConvolveBackwardData(
-      const dnn::FilterDescriptor &filter_descriptor,
-      const DeviceMemory<float> &filter_data,
-      const dnn::BatchDescriptor &output_descriptor,
-      DeviceMemory<float> backward_output_data,
-      const dnn::ConvolutionDescriptor &convolution_descriptor,
-      const dnn::BatchDescriptor &input_descriptor,
-      DeviceMemory<float> *backward_input_data);
-
-  Stream &ThenConvolveBackwardDataWithScratch(
-      const dnn::FilterDescriptor &filter_descriptor,
-      const DeviceMemory<float> &filter_data,
-      const dnn::BatchDescriptor &output_descriptor,
-      DeviceMemory<float> backward_output_data,
-      const dnn::ConvolutionDescriptor &convolution_descriptor,
-      const dnn::BatchDescriptor &input_descriptor,
-      DeviceMemory<float> *backward_input_data,
-      ScratchAllocator *scratch_allocator);
-
-  Stream &ThenConvolveBackwardDataWithScratch(
-      const dnn::FilterDescriptor &filter_descriptor,
-      const DeviceMemory<Eigen::half> &filter_data,
-      const dnn::BatchDescriptor &output_descriptor,
-      DeviceMemory<Eigen::half> backward_output_data,
-      const dnn::ConvolutionDescriptor &convolution_descriptor,
-      const dnn::BatchDescriptor &input_descriptor,
-      DeviceMemory<Eigen::half> *backward_input_data,
-      ScratchAllocator *scratch_allocator);
-
   Stream &ThenConvolveBackwardDataWithAlgorithm(
       const dnn::FilterDescriptor &filter_descriptor,
       const DeviceMemory<double> &filter_data,
@@ -523,35 +426,6 @@ class Stream {
       const dnn::AlgorithmConfig &algorithm_config,
       dnn::ProfileResult *output_profile_result);
 
-  Stream &ThenConvolveBackwardFilter(
-      const dnn::BatchDescriptor &input_descriptor,
-      const DeviceMemory<float> &input_data,
-      const dnn::BatchDescriptor &output_descriptor,
-      DeviceMemory<float> backward_output_data,
-      const dnn::ConvolutionDescriptor &convolution_descriptor,
-      const dnn::FilterDescriptor &filter_descriptor,
-      DeviceMemory<float> *backward_filter_data);
-
-  Stream &ThenConvolveBackwardFilterWithScratch(
-      const dnn::BatchDescriptor &input_descriptor,
-      const DeviceMemory<float> &input_data,
-      const dnn::BatchDescriptor &output_descriptor,
-      DeviceMemory<float> backward_output_data,
-      const dnn::ConvolutionDescriptor &convolution_descriptor,
-      const dnn::FilterDescriptor &filter_descriptor,
-      DeviceMemory<float> *backward_filter_data,
-      ScratchAllocator *scratch_allocator);
-
-  Stream &ThenConvolveBackwardFilterWithScratch(
-      const dnn::BatchDescriptor &input_descriptor,
-      const DeviceMemory<Eigen::half> &input_data,
-      const dnn::BatchDescriptor &output_descriptor,
-      DeviceMemory<Eigen::half> backward_output_data,
-      const dnn::ConvolutionDescriptor &convolution_descriptor,
-      const dnn::FilterDescriptor &filter_descriptor,
-      DeviceMemory<Eigen::half> *backward_filter_data,
-      ScratchAllocator *scratch_allocator);
-
   Stream &ThenConvolveBackwardFilterWithAlgorithm(
       const dnn::BatchDescriptor &input_descriptor,
       const DeviceMemory<double> &input_data,
-- 
GitLab


From 4c08b96795a1b3367ebbb676c00201318122ecd0 Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Thu, 31 Jan 2019 14:03:27 -0800
Subject: [PATCH 1679/2345] Split-up gradients_impl into gradients_util

This allows use of many of the underlying utilities without having to
import all the gradients.

PiperOrigin-RevId: 231849392
---
 .../contrib/optimizer_v2/optimizer_v2_test.py |    4 +-
 tensorflow/python/BUILD                       |   31 +-
 .../python/distribute/cross_device_utils.py   |    6 +-
 tensorflow/python/eager/function.py           |   11 +-
 tensorflow/python/ops/cond_v2.py              |    6 +-
 tensorflow/python/ops/control_flow_ops.py     |    4 +-
 tensorflow/python/ops/gradients.py            |    2 +-
 tensorflow/python/ops/gradients_impl.py       | 1064 +----------------
 tensorflow/python/ops/gradients_test.py       |   11 +-
 tensorflow/python/ops/gradients_util.py       | 1062 ++++++++++++++++
 tensorflow/python/ops/ragged/BUILD            |    1 +
 tensorflow/python/ops/while_v2.py             |    8 +-
 tensorflow/python/training/optimizer_test.py  |    4 +-
 .../v1/tensorflow.-aggregation-method.pbtxt   |    2 +-
 .../v2/tensorflow.-aggregation-method.pbtxt   |    2 +-
 15 files changed, 1133 insertions(+), 1085 deletions(-)
 create mode 100644 tensorflow/python/ops/gradients_util.py

diff --git a/tensorflow/contrib/optimizer_v2/optimizer_v2_test.py b/tensorflow/contrib/optimizer_v2/optimizer_v2_test.py
index dd7f2f4405..2fc0b5ea4d 100644
--- a/tensorflow/contrib/optimizer_v2/optimizer_v2_test.py
+++ b/tensorflow/contrib/optimizer_v2/optimizer_v2_test.py
@@ -26,7 +26,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import clip_ops
-from tensorflow.python.ops import gradients_impl
+from tensorflow.python.ops import gradients_util
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variables
@@ -71,7 +71,7 @@ class OptimizerTest(test.TestCase):
         opt_op = sgd_op.minimize(
             cost,
             global_step, [var0, var1],
-            aggregation_method=gradients_impl.AggregationMethod.
+            aggregation_method=gradients_util.AggregationMethod.
             EXPERIMENTAL_ACCUMULATE_N)
 
         variables.global_variables_initializer().run()
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index d550dca0b1..32d5d873cb 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -2264,7 +2264,7 @@ py_library(
         ":function_def_to_graph",
         ":functional_ops_gen",
         ":gradients",
-        ":gradients_impl",
+        ":gradients_util",
         ":graph_to_function_def",
         ":pywrap_tensorflow",
         ":util",
@@ -2290,7 +2290,7 @@ py_library(
         ":framework_ops",
         ":function_def_to_graph",
         ":functional_ops_gen",
-        ":gradients_impl",
+        ":gradients_util",
         ":list_ops",
         ":tensor_array_ops",
         ":tensor_shape",
@@ -2381,6 +2381,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":gradients_impl",
+        ":gradients_util",
         ":unconnected_gradients",
         "//tensorflow/python/eager:function",
         "//tensorflow/python/eager:tape",
@@ -2404,7 +2405,6 @@ py_library(
         ":framework",
         ":framework_for_generated_wrappers",
         ":framework_ops",
-        ":functional_ops",
         ":image_grad",
         ":linalg_grad",
         ":linalg_ops",
@@ -2416,15 +2416,34 @@ py_library(
         ":optional_grad",
         ":platform",
         ":random_grad",
-        ":resource_variable_ops",
         ":tensor_array_ops",
+        ":unconnected_gradients",
+        ":util",
+    ],
+)
+
+py_library(
+    name = "gradients_util",
+    srcs = [
+        "ops/gradients_util.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":array_ops",
+        ":control_flow_ops",
+        ":control_flow_util",
+        ":framework",
+        ":framework_for_generated_wrappers",
+        ":framework_ops",
+        ":functional_ops",
+        ":math_ops",
+        ":platform",
+        ":resource_variable_ops",
         ":tensor_util",
         ":unconnected_gradients",
         ":util",
-        ":variable_scope",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python/eager:context",
-        "//tensorflow/python/eager:tape",
         "//third_party/py/numpy",
         "@six_archive//:six",
     ],
diff --git a/tensorflow/python/distribute/cross_device_utils.py b/tensorflow/python/distribute/cross_device_utils.py
index e8066dd467..cb5417ea15 100644
--- a/tensorflow/python/distribute/cross_device_utils.py
+++ b/tensorflow/python/distribute/cross_device_utils.py
@@ -30,7 +30,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import collective_ops
-from tensorflow.python.ops import gradients_impl
+from tensorflow.python.ops import gradients_util
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nccl_ops
 
@@ -645,14 +645,14 @@ def unpack_small_tensors(replica_grads, packing):
 def aggregate_tensors_or_indexed_slices(values, accumulation_fn=math_ops.add_n):
   """Aggregate tensors using `accumulation_fn` and IndexedSlices via concat."""
   if any(isinstance(v, ops.IndexedSlices) for v in values):
-    return gradients_impl._AggregateIndexedSlicesGradients(values)  # pylint: disable=protected-access
+    return gradients_util._AggregateIndexedSlicesGradients(values)  # pylint: disable=protected-access
   else:
     return accumulation_fn(values)
 
 
 def divide_by_n_tensors_or_indexed_slices(value, n):
   if isinstance(value, ops.IndexedSlices):
-    value = gradients_impl._HandleNestedIndexedSlices(value)  # pylint: disable=protected-access
+    value = gradients_util._HandleNestedIndexedSlices(value)  # pylint: disable=protected-access
     return ops.IndexedSlices(
         value.values / n, value.indices, value.dense_shape)
   else:
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 00932c4c4f..06e88e808a 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -22,7 +22,6 @@ from __future__ import print_function
 import collections
 import functools
 import re
-import sys
 import threading
 import types as types_lib
 import weakref
@@ -48,7 +47,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_spec
 from tensorflow.python.ops import custom_gradient
 from tensorflow.python.ops import functional_ops
-from tensorflow.python.ops import gradients_impl
+from tensorflow.python.ops import gradients_util
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import compat
@@ -58,8 +57,6 @@ from tensorflow.python.util import nest
 from tensorflow.python.util import tf_decorator
 from tensorflow.python.util import tf_inspect
 
-# This is to avoid a circular dependency with gradients_impl
-gradients_impl._function = sys.modules[__name__]  # pylint: disable=protected-access
 
 FORWARD_FUNCTION_ATTRIBUTE_NAME = "forward_function_name"
 BACKWARD_FUNCTION_ATTRIBUTE_NAME = "backward_function_name"
@@ -664,12 +661,12 @@ class ConcreteFunction(object):
         _backward_name(self._func_graph.name))
     forward_function_name = _forward_name(self._func_graph.name)
     outputs = [x for x in self._func_graph.outputs
-               if gradients_impl.IsTrainable(x)]
+               if gradients_util.IsTrainable(x)]
     with backwards_graph.as_default():
       gradients_wrt_outputs = [
           graph_placeholder(x.dtype, x.shape) for x in outputs
       ]
-      gradients_wrt_inputs = gradients_impl._GradientsHelper(  # pylint: disable=protected-access
+      gradients_wrt_inputs = gradients_util._GradientsHelper(  # pylint: disable=protected-access
           outputs,
           self._func_graph.inputs,
           grad_ys=gradients_wrt_outputs,
@@ -738,7 +735,7 @@ class ConcreteFunction(object):
     # the forward graph function so that we can compute its gradient.
     real_outputs = outputs[:self._num_outputs]
     skip_positions = [i for i, t in enumerate(real_outputs)
-                      if not gradients_impl.IsTrainable(t)]
+                      if not gradients_util.IsTrainable(t)]
     side_outputs = outputs[self._num_outputs:]
 
     def backward_function(*args):
diff --git a/tensorflow/python/ops/cond_v2.py b/tensorflow/python/ops/cond_v2.py
index a0a13fbde3..74f5b52f18 100644
--- a/tensorflow/python/ops/cond_v2.py
+++ b/tensorflow/python/ops/cond_v2.py
@@ -35,7 +35,7 @@ from tensorflow.python.ops import control_flow_util_v2 as util
 from tensorflow.python.ops import gen_dataset_ops
 from tensorflow.python.ops import gen_functional_ops
 from tensorflow.python.ops import gen_resource_variable_ops
-from tensorflow.python.ops import gradients_impl
+from tensorflow.python.ops import gradients_util
 from tensorflow.python.ops import math_ops
 from tensorflow.python.util import nest
 
@@ -277,7 +277,7 @@ def _grad_fn(func_graph, grads):
   ys = []
   grad_ys = []
   for y, grad_y in zip(func_graph.outputs, grads):
-    if not gradients_impl.IsTrainable(y):
+    if not gradients_util.IsTrainable(y):
       continue
     ys.append(y)
     grad_ys.append(grad_y)
@@ -286,7 +286,7 @@ def _grad_fn(func_graph, grads):
   # func_graph in the current graph, which requires capturing tensors from
   # func_graph. The captured func_graph tensors are resolved to external tensors
   # in _resolve_grad_inputs.
-  result = gradients_impl._GradientsHelper(
+  result = gradients_util._GradientsHelper(
       ys, func_graph.inputs, grad_ys=grad_ys,
       src_graph=func_graph)
 
diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py
index cfdbe63c2f..e0b83c490c 100644
--- a/tensorflow/python/ops/control_flow_ops.py
+++ b/tensorflow/python/ops/control_flow_ops.py
@@ -59,13 +59,13 @@ from tensorflow.python.util.lazy_loader import LazyLoader
 from tensorflow.python.util.tf_export import tf_export
 
 # This is to avoid a circular dependency:
-# cond_v2 -> gradients_impl -> control_flow_ops
+# cond_v2 -> gradients_util -> control_flow_ops
 cond_v2 = LazyLoader("cond_v2", globals(),
                      "tensorflow.python.ops.cond_v2")
 
 # This is to avoid circular dependencies:
 # while_v2 -> control_flow_ops
-# while_v2 -> gradients_impl -> control_flow_ops
+# while_v2 -> gradients_util -> control_flow_ops
 while_v2 = LazyLoader("while_v2", globals(),
                       "tensorflow.python.ops.while_v2")
 
diff --git a/tensorflow/python/ops/gradients.py b/tensorflow/python/ops/gradients.py
index cd11447e1f..96389abded 100644
--- a/tensorflow/python/ops/gradients.py
+++ b/tensorflow/python/ops/gradients.py
@@ -22,7 +22,7 @@ from __future__ import print_function
 from tensorflow.python.eager import function
 from tensorflow.python.eager.backprop import GradientTape
 from tensorflow.python.ops.custom_gradient import custom_gradient
-from tensorflow.python.ops.gradients_impl import AggregationMethod
+from tensorflow.python.ops.gradients_util import AggregationMethod
 from tensorflow.python.ops.gradients_impl import gradients
 from tensorflow.python.ops.gradients_impl import hessians
 from tensorflow.python.ops.unconnected_gradients import UnconnectedGradients
diff --git a/tensorflow/python/ops/gradients_impl.py b/tensorflow/python/ops/gradients_impl.py
index 0a70d6ee61..c66efad7d3 100644
--- a/tensorflow/python/ops/gradients_impl.py
+++ b/tensorflow/python/ops/gradients_impl.py
@@ -18,30 +18,14 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import collections
-import contextlib
-import warnings
-
-import numpy as np
-import six
-from six.moves import xrange  # pylint: disable=redefined-builtin
-
-from tensorflow.core.framework import attr_value_pb2
-from tensorflow.python.eager import context
-from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import function as framework_function
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_shape
-from tensorflow.python.framework import tensor_util
-from tensorflow.python.framework.func_graph import FuncGraph
 from tensorflow.python.ops import array_grad  # pylint: disable=unused-import
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops  # pylint: disable=unused-import
 from tensorflow.python.ops import control_flow_grad  # pylint: disable=unused-import
 from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.ops import control_flow_util
-from tensorflow.python.ops import functional_ops
+from tensorflow.python.ops import gradients_util
 from tensorflow.python.ops import image_grad  # pylint: disable=unused-import
 from tensorflow.python.ops import linalg_grad  # pylint: disable=unused-import
 from tensorflow.python.ops import linalg_ops  # pylint: disable=unused-import
@@ -51,503 +35,11 @@ from tensorflow.python.ops import math_grad  # pylint: disable=unused-import
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import optional_grad  # pylint: disable=unused-import
 from tensorflow.python.ops import random_grad  # pylint: disable=unused-import
-from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import tensor_array_ops
 from tensorflow.python.ops.unconnected_gradients import UnconnectedGradients
-from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.util import compat
 from tensorflow.python.util.tf_export import tf_export
 
 
-# This is to avoid a circular dependency (eager.function depends on
-# gradients_impl). This is set in eager/function.py.
-_function = None
-
-# Warn the user if we convert a sparse representation to dense with at
-# least this number of elements.
-_LARGE_SPARSE_NUM_ELEMENTS = 100000000
-
-
-def _IndexedSlicesToTensor(value, dtype=None, name=None, as_ref=False):
-  """Converts an IndexedSlices object `value` to a Tensor.
-
-  NOTE(mrry): This function is potentially expensive.
-
-  Args:
-    value: An ops.IndexedSlices object.
-    dtype: The dtype of the Tensor to be returned.
-    name: Optional name to use for the returned Tensor.
-    as_ref: True if a ref is requested.
-
-  Returns:
-    A dense Tensor representing the values in the given IndexedSlices.
-
-  Raises:
-    ValueError: If the IndexedSlices does not have the same dtype.
-  """
-  _ = as_ref
-  if dtype and not dtype.is_compatible_with(value.dtype):
-    raise ValueError(
-        "Tensor conversion requested dtype %s for IndexedSlices with dtype %s" %
-        (dtype.name, value.dtype.name))
-  if value.dense_shape is None:
-    raise ValueError(
-        "Tensor conversion requested for IndexedSlices without dense_shape: %s"
-        % str(value))
-  # TODO(mrry): Consider adding static shape information to
-  # IndexedSlices, to avoid using numpy here.
-  if not context.executing_eagerly():
-    dense_shape_value = tensor_util.constant_value(value.dense_shape)
-    if dense_shape_value is not None:
-      num_elements = np.prod(dense_shape_value)
-      if num_elements >= _LARGE_SPARSE_NUM_ELEMENTS:
-        warnings.warn(
-            "Converting sparse IndexedSlices to a dense Tensor with %d "
-            "elements. This may consume a large amount of memory." %
-            num_elements)
-    else:
-      warnings.warn(
-          "Converting sparse IndexedSlices to a dense Tensor of unknown shape. "
-          "This may consume a large amount of memory.")
-  return math_ops.unsorted_segment_sum(
-      value.values, value.indices, value.dense_shape[0], name=name)
-
-
-ops.register_tensor_conversion_function(ops.IndexedSlices,
-                                        _IndexedSlicesToTensor)
-
-
-def _MarkReachedOps(from_ops, reached_ops, func_graphs):
-  """Mark all ops reached from "from_ops".
-
-  Args:
-    from_ops: list of Operations.
-    reached_ops: set of Operations.
-    func_graphs: list of FuncGraphs. This method will traverse through
-      these functions if they capture from_ops or any reachable ops.
-  """
-  queue = collections.deque()
-  queue.extend(from_ops)
-  while queue:
-    op = queue.popleft()
-    if op not in reached_ops:
-      reached_ops.add(op)
-      for output in op.outputs:
-        if _IsBackpropagatable(output):
-          queue.extend(_Consumers(output, func_graphs))
-
-
-def _PendingCount(to_ops, from_ops, colocate_gradients_with_ops, func_graphs,
-                  xs):
-  """Initialize the pending count for ops between two lists of Operations.
-
-  'pending_count[op]' indicates the number of backprop inputs
-  to this operation.
-
-  Args:
-    to_ops: list of Operations.
-    from_ops: list of Operations.
-    colocate_gradients_with_ops: Python bool.  See docstring of gradients().
-    func_graphs: list of FuncGraphs. This method will traverse through
-      these functions if they capture from_ops or any reachable ops. This is
-      useful if to_ops occur in a function and from_ops are in an outer function
-      or graph.
-    xs: list of Tensors.
-
-  Returns:
-    A tuple containing: (1) the subset of to_ops reachable from from_ops by a
-    path of zero or more backpropagatable tensors, (2) a mapping from operation
-    to the number of backprop inputs to that op, and (3) a ControlFlowState
-    object which is not None if the ops between from_ops and to_ops contain
-    control flow loops.
-  """
-  # Mark reachable ops from from_ops.
-  reached_ops = set()
-  _MarkReachedOps(from_ops, reached_ops, func_graphs)
-  # X in reached_ops iff X is reachable from from_ops by a path of zero or more
-  # backpropagatable tensors.
-
-  reachable_to_ops = set(op for op in to_ops if op in reached_ops)
-
-  # Mark between ops.
-  between_ops = set()
-  between_op_list = []
-  queue = collections.deque()
-  queue.extend(to_ops)
-  while queue:
-    op = queue.popleft()
-    # We are interested in this op.
-    if op in reached_ops:
-      between_ops.add(op)
-      between_op_list.append(op)
-      # Clear the boolean so we won't add the inputs again.
-      reached_ops.remove(op)
-      for inp in _NonEagerInputs(op, xs):
-        queue.append(inp.op)
-  # X in between_ops iff X is on a path of zero or more backpropagatable tensors
-  # between from_ops and to_ops
-
-  # 'loop_state' is None if there are no while loops.
-  loop_state = control_flow_ops.MaybeCreateControlFlowState(
-      between_op_list, between_ops, colocate_gradients_with_ops)
-
-  # Initialize pending count for between ops.
-  pending_count = collections.defaultdict(int)
-  for op in between_op_list:
-    for x in _NonEagerInputs(op, xs):
-      if x.op in between_ops:
-        pending_count[x.op] += 1
-
-  return reachable_to_ops, pending_count, loop_state
-
-
-def _AsList(x):
-  return x if isinstance(x, (list, tuple)) else [x]
-
-
-def _DefaultGradYs(grad_ys,
-                   ys,
-                   colocate_gradients_with_ops,
-                   gradient_uid="__unsupported__"):
-  """Fill in default values for grad_ys.
-
-  Args:
-    grad_ys: List of gradients, can contain None.
-    ys: List of tensors.
-    colocate_gradients_with_ops: If True, try colocating gradients with
-      the corresponding op.
-    gradient_uid: A unique identifier within the graph indicating
-      which invocation of gradients is being executed. Used to cluster
-      ops for compilation.
-
-  Returns:
-    A list of gradients to use, without None.
-
-  Raises:
-    ValueError: If sizes of gradients and inputs don't match
-    TypeError: If type of any gradient is not valid for its input.
-  """
-  if len(grad_ys) != len(ys):
-    raise ValueError("Passed %d grad_ys for %d ys" % (len(grad_ys), len(ys)))
-  grad_ys = ops.convert_n_to_tensor_or_indexed_slices(grad_ys, name="grad_y")
-  new_grad_ys = []
-  for i in xrange(len(grad_ys)):
-    grad_y = grad_ys[i]
-    y = ys[i]
-    with _maybe_colocate_with(y.op, gradient_uid, colocate_gradients_with_ops):
-      if grad_y is None:
-        if y.dtype.is_complex:
-          raise TypeError(
-              "Gradients of complex tensors must set grad_ys (y.dtype = %r)" %
-              y.dtype)
-        new_grad_ys.append(
-            array_ops.fill(
-                array_ops.shape(y),
-                constant_op.constant(1, dtype=y.dtype, name="grad_ys_%d" % i)))
-        continue
-      if y.dtype.is_floating or y.dtype.is_integer:
-        if not grad_y.dtype.is_floating and not grad_y.dtype.is_integer:
-          raise TypeError(
-              "Gradient type %s generated for real or "
-              "integer-valued tensor %s with type %s must be "
-              "real or integer" % (dtypes.as_dtype(grad_y.dtype).name, y,
-                                   dtypes.as_dtype(y.dtype).name))
-      elif y.dtype.is_complex:
-        if not grad_y.dtype.is_complex:
-          raise TypeError(
-              "Gradient type %s generated for complex-valued "
-              "tensor %s with type %s must be real" % (dtypes.as_dtype(
-                  grad_y.dtype).name, y, dtypes.as_dtype(y.dtype).name))
-      elif y.dtype == dtypes.variant:
-        if grad_y.dtype != dtypes.variant:
-          raise TypeError(
-              "Gradient type %s generated for variant "
-              "tensor %s with type %s must be variant" % (dtypes.as_dtype(
-                  grad_y.dtype).name, y, dtypes.as_dtype(y.dtype).name))
-      elif y.dtype == dtypes.resource:
-        # We assume y is the handle of a ResourceVariable. The gradient of a
-        # ResourceVariable should be a numeric value, not another resource.
-        if grad_y.dtype == dtypes.resource:
-          raise TypeError("Input gradient %s for resource tensor %s should not "
-                          "be a resource" % (grad_y, y))
-      else:
-        raise TypeError(
-            "Tensor %s with type %s must be numeric "
-            "to obtain a default gradient" % (y, dtypes.as_dtype(y.dtype).name))
-      # Create a grad_y tensor in the name scope of the gradient.
-      # Required for TensorArrays to identify which gradient call a
-      # grad_y value is coming from.
-      if isinstance(grad_y, ops.IndexedSlices):
-        new_grad_ys.append(
-            ops.IndexedSlices(
-                indices=(array_ops.identity(
-                    grad_y.indices, name="grad_ys_%d_indices" % i)
-                         if isinstance(grad_y.indices, ops.Tensor) else
-                         grad_y.indices),
-                values=(array_ops.identity(
-                    grad_y.values, name="grad_ys_%d_values" % i) if isinstance(
-                        grad_y.values, ops.Tensor) else grad_y.values),
-                dense_shape=(array_ops.identity(
-                    grad_y.dense_shape, name="grad_ys_%d_shape" % i)
-                             if isinstance(grad_y.dense_shape, ops.Tensor) else
-                             grad_y.dense_shape)))
-      else:
-        new_grad_ys.append(array_ops.identity(grad_y, name="grad_ys_%d" % i))
-
-  return new_grad_ys
-
-
-def IsTrainable(tensor_or_dtype):
-  if isinstance(tensor_or_dtype, ops.Tensor):
-    dtype = tensor_or_dtype.dtype
-  else:
-    dtype = tensor_or_dtype
-  dtype = dtypes.as_dtype(dtype)
-  return dtype.base_dtype in (dtypes.float16, dtypes.float32, dtypes.float64,
-                              dtypes.complex64, dtypes.complex128,
-                              dtypes.resource, dtypes.variant)
-
-
-def _IsBackpropagatable(tensor):
-  if IsTrainable(tensor):
-    return True
-  dtype = dtypes.as_dtype(tensor.dtype)
-  return dtype.base_dtype == dtypes.bfloat16
-
-
-def _VerifyGeneratedGradients(grads, op):
-  """Verify that gradients are valid in number and type.
-
-  Args:
-    grads: List of generated gradients.
-    op: Operation for which the gradients where generated.
-
-  Raises:
-    ValueError: if sizes of gradients and inputs don't match.
-    TypeError: if type of any gradient is not valid for its input.
-  """
-  # While ops have inputs added to them during the gradient computation, so we
-  # skip the below check. See while_v2 for details.
-  if op.type == "While": return
-
-  if len(grads) != len(op.inputs):
-    raise ValueError("Num gradients %d generated for op %s do not match num "
-                     "inputs %d" % (len(grads), op.node_def, len(op.inputs)))
-
-
-def _StopOps(from_ops, stop_gradient_ops, pending_count, xs):
-  """The set of ops that terminate the gradient computation.
-
-  This computes the frontier of the forward graph *before* which backprop
-  should stop. Operations in the returned set will not be differentiated.
-  This set is defined as the subset of `from_ops` containing ops that have
-  no predecessor in `from_ops`. `pending_count` is the result of
-  `_PendingCount(xs, from_ops)`. An 'op' has predecessors in `from_ops`
-  iff pending_count[op] > 0.
-
-  In addition, none of `stop_gradient_ops` will be differentiated.
-
-  Args:
-    from_ops: list of Operations.
-    stop_gradient_ops: list of Operations never to backprop through.
-    pending_count: mapping from operation to number of backprop inputs.
-    xs: list of Tensors.
-
-  Returns:
-    The set of operations.
-  """
-  stop_ops = set()
-  for op in from_ops:
-    is_stop_op = True
-    for inp in _NonEagerInputs(op, xs):
-      if pending_count[inp.op] > 0:
-        is_stop_op = False
-        break
-    if is_stop_op:
-      stop_ops.add(op)
-  stop_ops.update(op for op in stop_gradient_ops)
-  return stop_ops
-
-
-@contextlib.contextmanager
-def _maybe_colocate_with(op, gradient_uid, colocate_gradients_with_ops):  # pylint: disable=invalid-name
-  """Context to colocate with `op` if `colocate_gradients_with_ops`."""
-  if colocate_gradients_with_ops:
-    with ops._colocate_with_for_gradient(op, gradient_uid):  # pylint: disable=protected-access
-      yield
-  else:
-    yield
-
-
-def _IsPartitionedCall(op):
-  return op.type == "PartitionedCall" or op.type == "StatefulPartitionedCall"
-
-
-def _SymGrad(op, out_grads):
-  """Backprop through a function call node op given its outputs' gradients."""
-  f_in = [x for x in op.inputs] + out_grads
-  f_types = [x.dtype for x in op.inputs]
-  f = attr_value_pb2.NameAttrList()
-  if _IsPartitionedCall(op):
-    f.name = op.get_attr("f").name
-  else:
-    f.name = op.type
-  for k in op.node_def.attr:
-    f.attr[k].CopyFrom(op.node_def.attr[k])
-  # TODO(apassos) use a better dtype here
-  in_grads = functional_ops.symbolic_gradient(
-      input=f_in,
-      Tout=[x if x != dtypes.resource else dtypes.float32 for x in f_types],
-      f=f)
-  return in_grads
-
-
-def _MaybeCompile(scope, op, func, grad_fn):
-  """Compile the calculation in grad_fn if op was marked as compiled."""
-  scope = scope.rstrip("/").replace("/", "_")
-  if func is not None:
-    xla_compile = func.definition.attr["_XlaCompile"].b
-    xla_separate_compiled_gradients = func.definition.attr[
-        "_XlaSeparateCompiledGradients"].b
-    xla_scope = func.definition.attr["_XlaScope"].s.decode()
-  else:
-    try:
-      xla_compile = op.get_attr("_XlaCompile")
-      xla_separate_compiled_gradients = op.get_attr(
-          "_XlaSeparateCompiledGradients")
-      xla_scope = op.get_attr("_XlaScope").decode()
-    except ValueError:
-      return grad_fn()  # Exit early
-
-  if not xla_compile:
-    return grad_fn()  # Exit early
-
-  # If the gradients are supposed to be compiled separately, we give them a
-  # _XlaScope name that is based on the name_scope of the gradients.  Otherwise
-  # they just inherit the existing _XlaScope name, which lets them be merged
-  # together with the non-gradient computation.
-  if xla_separate_compiled_gradients:
-    xla_grad_scope = "%s_grad_%s" % (xla_scope, scope)
-  else:
-    xla_grad_scope = xla_scope
-
-  attrs = {
-      "_XlaCompile": attr_value_pb2.AttrValue(b=xla_compile),
-      "_XlaScope": attr_value_pb2.AttrValue(s=xla_grad_scope.encode())
-  }
-  with ops.get_default_graph()._attr_scope(attrs):  # pylint: disable=protected-access
-    return grad_fn()
-
-
-def _RaiseNoGradWrtInitialLoopValError(op, from_ops, xs):
-  """Raises an error if we backprop through a loop var."""
-  # Find the nearest 'to_op' reachable from 'op' to provide a more helpful error
-  # message.
-  target_op = None
-  queue = collections.deque([op])
-  visited = set()
-  while queue:
-    curr_op = queue.popleft()
-    if curr_op in visited: continue
-    visited.add(curr_op)
-    if curr_op in from_ops:
-      target_op = curr_op
-      break
-    queue.extend(t.op for t in _NonEagerInputs(curr_op, xs))
-  assert target_op
-  raise ValueError(
-      "Cannot compute gradient inside while loop with respect to op '%s'. "
-      "We do not support taking the gradient wrt or through the initial value "
-      "of a loop variable. Gradients can be computed through loop invariants "
-      "or wrt the input parameters to the loop body."
-      % target_op.name)
-
-
-def _IsFunction(graph):
-  return (isinstance(graph, FuncGraph) or
-          isinstance(graph, framework_function._FuncGraph))  # pylint: disable=protected-access
-
-
-def _Captures(func_graph):
-  if isinstance(func_graph, FuncGraph):
-    return func_graph.captures
-  else:
-    assert isinstance(func_graph, framework_function._FuncGraph)  # pylint: disable=protected-access
-    return func_graph._captured  # pylint: disable=protected-access
-
-
-def _MaybeCaptured(t):
-  """If t is a captured value placeholder, returns the original captured value.
-
-  Args:
-    t: Tensor
-
-  Returns:
-    A tensor, potentially from a different Graph/FuncGraph.
-  """
-  # pylint: disable=protected-access
-  if (not isinstance(t, ops.EagerTensor) and
-      _IsFunction(t.op.graph) and t.op.type == "Placeholder"):
-    for input_t, placeholder_t in _Captures(t.op.graph).items():
-      if t == placeholder_t:
-        return _MaybeCaptured(input_t)
-  # pylint: enable=protected-access
-  return t
-
-
-# TODO(skyewm): plumbing xs through everywhere is ugly, consider making
-# _GradientsHelper a class with xs as a member variable.
-def _NonEagerInputs(op, xs):
-  """Returns the inputs of op, crossing closure boundaries where necessary.
-
-  Does not return any captured EagerTensors, i.e., the number of tensors
-  returned may be less than than the actual number of inputs.
-
-  Args:
-    op: Operation
-    xs: list of Tensors we are differentiating w.r.t.
-
-  Returns:
-    A list of tensors. The tensors may be from multiple Graph/FuncGraphs if op
-    is in a FuncGraph and has captured inputs.
-  """
-  if _IsFunction(op.graph):  # pylint: disable=protected-access
-    inputs = []
-    for t in op.inputs:
-      # If we're differentiating w.r.t. `t`, do not attempt to traverse through
-      # it to a captured value. The algorithm needs to "see" `t` in this case,
-      # even if it's a function input for a captured value, whereas usually we'd
-      # like to traverse through these closures as if the captured value was the
-      # direct input to op.
-      if t not in xs:
-        t = _MaybeCaptured(t)
-        # Skip captured eager inputs.
-        if isinstance(t, ops.EagerTensor): continue
-      inputs.append(t)
-    return inputs
-  else:
-    return op.inputs
-
-
-def _Consumers(t, func_graphs):
-  """Returns the consumers of t, crossing closure boundaries where necessary.
-
-  Args:
-    t: Tensor
-    func_graphs: a list of FuncGraphs that may have captured t.
-
-  Returns:
-    A list of tensors. The tensors will be from the current graph and/or
-    func_graphs.
-  """
-  consumers = t.consumers()
-  for func in func_graphs:
-    for input_t, placeholder in _Captures(func).items():
-      if input_t == t:
-        consumers.extend(_Consumers(placeholder, func_graphs))
-  return consumers
-
-
 @tf_export(v1=["gradients"])
 def gradients(ys,
               xs,
@@ -658,10 +150,13 @@ def gradients(ys,
   # Creating the gradient graph for control flow mutates Operations.
   # _mutation_lock ensures a Session.run call cannot occur between creating and
   # mutating new ops.
-  with ops.get_default_graph()._mutation_lock():  # pylint: disable=protected-access
-    return _GradientsHelper(ys, xs, grad_ys, name, colocate_gradients_with_ops,
-                            gate_gradients, aggregation_method, stop_gradients,
-                            unconnected_gradients)
+  # pylint: disable=protected-access
+  with ops.get_default_graph()._mutation_lock():
+    return gradients_util._GradientsHelper(
+        ys, xs, grad_ys, name, colocate_gradients_with_ops,
+        gate_gradients, aggregation_method, stop_gradients,
+        unconnected_gradients)
+  # pylint: enable=protected-access
 
 
 @tf_export("gradients", v1=[])
@@ -771,540 +266,13 @@ def gradients_v2(ys,  # pylint: disable=invalid-name
   # Creating the gradient graph for control flow mutates Operations.
   # _mutation_lock ensures a Session.run call cannot occur between creating and
   # mutating new ops.
-  with ops.get_default_graph()._mutation_lock():  # pylint: disable=protected-access
-    return _GradientsHelper(ys, xs, grad_ys, name, True, gate_gradients,
-                            aggregation_method, stop_gradients,
-                            unconnected_gradients)
-
-
-def _GradientsHelper(ys,
-                     xs,
-                     grad_ys=None,
-                     name="gradients",
-                     colocate_gradients_with_ops=False,
-                     gate_gradients=False,
-                     aggregation_method=None,
-                     stop_gradients=None,
-                     unconnected_gradients=UnconnectedGradients.NONE,
-                     src_graph=None):
-  """Implementation of gradients()."""
-  if context.executing_eagerly():
-    raise RuntimeError("tf.gradients is not supported when eager execution "
-                       "is enabled. Use tf.GradientTape instead.")
-  if src_graph is None:
-    src_graph = ops.get_default_graph()
-  try:
-    unconnected_gradients = UnconnectedGradients(unconnected_gradients)
-  except ValueError:
-    raise ValueError(
-        "Unknown value for unconnected_gradients: %r" % unconnected_gradients)
-
-  # If src_graph is a _FuncGraph (i.e. a function body), gather it and all
-  # ancestor graphs. This is necessary for correctly handling captured values.
-  func_graphs = []
-  curr_graph = src_graph
-  while _IsFunction(curr_graph):
-    func_graphs.append(curr_graph)
-    if isinstance(curr_graph, FuncGraph):
-      curr_graph = curr_graph.outer_graph
-    else:
-      assert isinstance(curr_graph, framework_function._FuncGraph)  # pylint: disable=protected-access
-      curr_graph = curr_graph._outer_graph  # pylint: disable=protected-access
-
-  ys = _AsList(ys)
-  xs = _AsList(xs)
-  stop_gradients = [] if stop_gradients is None else _AsList(stop_gradients)
-  if grad_ys is None:
-    grad_ys = [None] * len(ys)
-  else:
-    grad_ys = _AsList(grad_ys)
-
-  with ops.name_scope(
-      name, "gradients",
-      list(ys) + list(xs) + list(stop_gradients) + list(grad_ys)) as grad_scope:
-    # Get a uid for this call to gradients that can be used to help
-    # cluster ops for compilation.
-    gradient_uid = ops.get_default_graph().unique_name("uid")
-    ys = ops.convert_n_to_tensor_or_indexed_slices(ys, name="y")
-    xs = [
-        x.handle if resource_variable_ops.is_resource_variable(x) else x
-        for x in xs
-    ]
-    xs = ops.internal_convert_n_to_tensor_or_indexed_slices(
-        xs, name="x", as_ref=True)
-    grad_ys = _DefaultGradYs(grad_ys, ys, colocate_gradients_with_ops,
-                             gradient_uid)
-
-    # The approach we take here is as follows: Create a list of all ops in the
-    # subgraph between the ys and xs.  Visit these ops in reverse order of ids
-    # to ensure that when we visit an op the gradients w.r.t its outputs have
-    # been collected.  Then aggregate these gradients if needed, call the op's
-    # gradient function, and add the generated gradients to the gradients for
-    # its input.
-
-    # Initialize the pending count for ops in the connected subgraph from ys
-    # to the xs.
-    to_ops = [t.op for t in ys]
-    from_ops = [t.op for t in xs]
-    stop_gradient_ops = [t.op for t in stop_gradients]
-    reachable_to_ops, pending_count, loop_state = _PendingCount(
-        to_ops, from_ops, colocate_gradients_with_ops, func_graphs, xs)
-
-    # Iterate over the collected ops.
-    #
-    # grads: op => list of gradients received on each output endpoint of the
-    # op.  The gradients for each endpoint are initially collected as a list.
-    # When it is time to call the op's gradient function, for each endpoint we
-    # aggregate the list of received gradients into a Add() Operation if there
-    # is more than one.
-    grads = {}
-
-    # Add the initial gradients for the ys.
-    for y, grad_y in zip(ys, grad_ys):
-      _SetGrad(grads, y, grad_y)
-
-    # Initialize queue with to_ops.
-    queue = collections.deque()
-    # Add the ops in 'to_ops' into the queue.
-    to_ops_set = set()
-    for op in to_ops:
-      # 'ready' handles the case where one output gradient relies on
-      # another output's gradient.
-      ready = (pending_count[op] == 0)
-      if ready and op not in to_ops_set and op in reachable_to_ops:
-        to_ops_set.add(op)
-        queue.append(op)
-
-    if loop_state:
-      loop_exits = loop_state.ProcessUnusedLoopExits(pending_count, to_ops_set)
-      for y in loop_exits:
-        if IsTrainable(y):
-          _SetGrad(grads, y, loop_state.ZerosLikeForExit(y))
-          queue.append(y.op)
-
-    stop_ops = _StopOps(from_ops, stop_gradient_ops, pending_count, xs)
-    while queue:
-      # generate gradient subgraph for op.
-      op = queue.popleft()
-      with _maybe_colocate_with(op, gradient_uid, colocate_gradients_with_ops):
-        if loop_state:
-          loop_state.EnterGradWhileContext(op, before=True)
-        out_grads = _AggregatedGrads(grads, op, gradient_uid, loop_state,
-                                     aggregation_method)
-        if loop_state:
-          loop_state.ExitGradWhileContext(op, before=True)
-
-        grad_fn = None
-        func_call = None
-        is_partitioned_call = _IsPartitionedCall(op)
-        # pylint: disable=protected-access
-        is_func_call = (
-            src_graph._is_function(op.type) or is_partitioned_call)
-        # pylint: enable=protected-access
-        has_out_grads = any(isinstance(g, ops.Tensor) or g for g in out_grads)
-        if has_out_grads and (op not in stop_ops):
-          try:
-            grad_fn = ops.get_gradient_function(op)
-          except LookupError:
-            if is_func_call:
-              if is_partitioned_call:
-                func_call = src_graph._get_function(  # pylint: disable=protected-access
-                    compat.as_bytes(op.get_attr("f").name))
-              else:
-                func_call = src_graph._get_function(op.type)  # pylint: disable=protected-access
-              # Note that __defun is not set if the graph is
-              # imported. If it's set, we prefer to access the original
-              # defun.
-              func_call = getattr(op, "__defun", func_call)
-              grad_fn = func_call.python_grad_func
-            else:
-              raise LookupError(
-                  "No gradient defined for operation '%s' (op type: %s)" %
-                  (op.name, op.type))
-        if loop_state:
-          loop_state.EnterGradWhileContext(op, before=False)
-
-        # NOTE(skyewm): We don't support computing gradients wrt a loop variable
-        # unless it's within the context of a single iteration (i.e. the
-        # gradient is wrt to the loop parameter in the body function, not wrt or
-        # through the initial value). This means if we're in a while loop
-        # context, we should never see a switch node from this context.
-        # pylint: disable=protected-access
-        if (control_flow_util.IsSwitch(op) and
-            op._control_flow_context is not None and
-            op._control_flow_context.IsWhileContext() and
-            op._control_flow_context ==
-            ops.get_default_graph()._get_control_flow_context()):
-          _RaiseNoGradWrtInitialLoopValError(op, from_ops, xs)
-        # pylint: enable=protected-access
-
-        if (grad_fn or is_func_call) and has_out_grads:
-          # NOTE: If _AggregatedGrads didn't compute a value for the i'th
-          # output, it means that the cost does not depend on output[i],
-          # therefore dC/doutput[i] is 0.
-          for i, out_grad in enumerate(out_grads):
-            if (not isinstance(out_grad, ops.Tensor) and not out_grad) and (
-                (not grad_fn and is_func_call) or IsTrainable(op.outputs[i])):
-              # Only trainable outputs or outputs for a function call that
-              # will use SymbolicGradient get a zero gradient. Gradient
-              # functions should ignore the gradient for other outputs.
-              # TODO(apassos) gradients of resource handles might be an
-              # issue here because of zeros.
-              if loop_state:
-                out_grads[i] = loop_state.ZerosLike(op, i)
-              else:
-                out_grads[i] = control_flow_ops.ZerosLikeOutsideLoop(op, i)
-          with ops.name_scope(op.name + "_grad"):
-            # pylint: disable=protected-access
-            with src_graph._original_op(op):
-              # pylint: enable=protected-access
-              if grad_fn:
-                # If grad_fn was found, do not use SymbolicGradient even for
-                # functions.
-                in_grads = _MaybeCompile(grad_scope, op, func_call,
-                                         lambda: grad_fn(op, *out_grads))
-              else:
-                # For function call ops, we add a 'SymbolicGradient'
-                # node to the graph to compute gradients.
-                in_grads = _MaybeCompile(grad_scope, op, func_call,
-                                         lambda: _SymGrad(op, out_grads))
-              in_grads = _AsList(in_grads)
-              _VerifyGeneratedGradients(in_grads, op)
-              if gate_gradients and len([x for x in in_grads
-                                         if x is not None]) > 1:
-                with ops.device(None):
-                  with ops._colocate_with_for_gradient(  # pylint: disable=protected-access
-                      None,
-                      gradient_uid,
-                      ignore_existing=True):
-                    in_grads = control_flow_ops.tuple(in_grads)
-          _LogOpGradients(op, out_grads, in_grads)
-        else:
-          # If no grad_fn is defined or none of out_grads is available,
-          # just propagate a list of None backwards.
-          in_grads = [None] * len(_NonEagerInputs(op, xs))
-        for i, (t_in, in_grad) in enumerate(zip(_NonEagerInputs(op, xs),
-                                                in_grads)):
-          if in_grad is not None:
-            if (isinstance(in_grad, ops.Tensor) and
-                t_in.dtype != dtypes.resource):
-              try:
-                in_grad.set_shape(t_in.get_shape())
-              except ValueError:
-                raise ValueError(
-                    "Incompatible shapes between op input and calculated "
-                    "input gradient.  Forward operation: %s.  Input index: %d. "
-                    "Original input shape: %s.  "
-                    "Calculated input gradient shape: %s" %
-                    (op.name, i, t_in.shape, in_grad.shape))
-            _SetGrad(grads, t_in, in_grad)
-        if loop_state:
-          loop_state.ExitGradWhileContext(op, before=False)
-
-      # Update pending count for the inputs of op and enqueue ready ops.
-      _UpdatePendingAndEnqueueReady(grads, op, queue, pending_count, loop_state,
-                                    xs)
-
-  if loop_state:
-    loop_state.PostProcessing()
-  return [_GetGrad(grads, x, unconnected_gradients) for x in xs]
-
-
-def _HasAnyNotNoneGrads(grads, op):
-  """Return true iff op has real gradient."""
-  out_grads = _GetGrads(grads, op)
-  for out_grad in out_grads:
-    if isinstance(out_grad, (ops.Tensor, ops.IndexedSlices)):
-      return True
-    if out_grad and isinstance(out_grad, collections.Sequence):
-      if any(g is not None for g in out_grad):
-        return True
-  return False
-
-
-def _UpdatePendingAndEnqueueReady(grads, op, queue, pending_count, loop_state,
-                                  xs):
-  """Update pending count for the inputs of op and enqueue ready ops."""
-  for x in _NonEagerInputs(op, xs):
-    pending_count[x.op] -= 1
-    ready = (pending_count[x.op] == 0)
-    if loop_state and not ready:
-      ready = pending_count[x.op] > 0 and control_flow_util.IsLoopSwitch(x.op)
-    if ready:
-      if control_flow_util.IsLoopExit(x.op):
-        # if x is an exit without real gradient, defer processing them.
-        grad_state = loop_state.GetGradState(x.op, before=False)
-        grad_state.deferred_exits.append(x)
-        grad_state.pending_exits_count -= 1
-        if grad_state.pending_exits_count == 0:
-          # We now have all the exits so process them.
-          has_not_none_grad = False
-          for y in grad_state.deferred_exits:
-            if _HasAnyNotNoneGrads(grads, y.op):
-              has_not_none_grad = True
-              queue.append(y.op)
-            else:
-              grad_state.unused_exits.append(y)
-          if has_not_none_grad:
-            # For an unused exit, if it has trainable outputs, backprop
-            # a zero gradient. Otherwise, just ignore it.
-            for y in grad_state.unused_exits:
-              if IsTrainable(y):
-                _SetGrad(grads, y, loop_state.ZerosLikeForExit(y))
-              queue.append(y.op)
-          else:
-            # All exits are "unused" so use None as gradient.
-            for y in grad_state.unused_exits:
-              queue.append(y.op)
-      else:
-        queue.append(x.op)
-
-
-def _SetGrad(grads, t, grad):
-  """Sets gradient "grad" in "grads" for tensor "t"."""
-  op = t.op
-  op_grads = grads.get(op)
-  if not op_grads:
-    op_grads = [[] for _ in xrange(len(op.outputs))]
-    grads[op] = op_grads
-  t_grads = op_grads[t.value_index]
-  if isinstance(t_grads, list):
-    t_grads.append(grad)
-  else:
-    assert control_flow_util.IsLoopSwitch(op)
-    op_grads[t.value_index] = grad
-
-
-def _GetGrad(grads, t, unconnected_gradients):
-  """Gets gradient for tensor "t"."""
-  op = t.op
-  op_grads = grads.get(op)
-  if not op_grads:
-    if unconnected_gradients == UnconnectedGradients.ZERO:
-      t_dtype = t.dtype if t.dtype != dtypes.resource else dtypes.float32
-      return array_ops.zeros_like(t, dtype=t_dtype)
-    elif unconnected_gradients == UnconnectedGradients.NONE:
-      return None
-    else:
-      raise ValueError(
-          "Unknown value for unconnected_gradients: %r" % unconnected_gradients)
-
-  t_grad = op_grads[t.value_index]
-  assert not isinstance(
-      t_grad, list), ("gradients list should have been aggregated by now.")
-  return t_grad
-
-
-def _GetGrads(grads, op):
-  """Gets all gradients for op."""
-  if op in grads:
-    return grads[op]
-  else:
-    return [[] for _ in xrange(len(op.outputs))]
-
-
-def _HandleNestedIndexedSlices(grad):
-  assert isinstance(grad, ops.IndexedSlices)
-  if isinstance(grad.values, ops.Tensor):
-    return grad
-  else:
-    assert isinstance(grad.values, ops.IndexedSlices)
-    g = _HandleNestedIndexedSlices(grad.values)
-    return ops.IndexedSlices(g.values, array_ops.gather(
-        grad.indices, g.indices), g.dense_shape)
-
-
-def _AccumulatorShape(inputs):
-  shape = tensor_shape.unknown_shape()
-  for i in inputs:
-    if isinstance(i, ops.Tensor):
-      shape = shape.merge_with(i.get_shape())
-  return shape
-
-
-def _LogOpGradients(op, out_grads, in_grads):
-  """Log the in and out grads of an op."""
-  logging.vlog(1, "Gradient for '" + op.name + "'")
-
-  def _FilterGrad(x):
-    if x is None:
-      return False
-    if isinstance(x, (list, tuple)):
-      return bool(x)
-    else:
-      return True
-
-  logging.vlog(1, "  in  --> %s",
-               ", ".join([x.name for x in out_grads if _FilterGrad(x)]))
-  logging.vlog(1, "  out --> %s",
-               ", ".join([x.name for x in in_grads if _FilterGrad(x)]))
-
-
-def _MultiDeviceAddN(tensor_list, gradient_uid):
-  """Adds tensors from potentially multiple devices."""
-  # Basic function structure comes from control_flow_ops.group().
-  # Sort tensors according to their devices.
-  tensors_on_device = collections.defaultdict(lambda: [])
-  for tensor in tensor_list:
-    tensors_on_device[tensor.device].append(tensor)
-
-  # For each device, add the tensors on that device first.
-  # Then gather the partial sums from multiple devices.
-  # TODO(sjhwang): Create hierarchical aggregation tree as pbar's suggestion.
-  # E.g., aggregate per GPU, then per task, and so on.
-  summands = []
-
-  def DeviceKey(dev):
-    return "" if dev is None else dev
-
-  for dev in sorted(six.iterkeys(tensors_on_device), key=DeviceKey):
-    tensors = tensors_on_device[dev]
-    with ops._colocate_with_for_gradient(  # pylint: disable=protected-access
-        tensors[0].op,
-        gradient_uid,
-        ignore_existing=True):
-      summands.append(math_ops.add_n(tensors))
-
-  return math_ops.add_n(summands)
-
-
-@tf_export("AggregationMethod")
-class AggregationMethod(object):
-  """A class listing aggregation methods used to combine gradients.
-
-  Computing partial derivatives can require aggregating gradient
-  contributions. This class lists the various methods that can
-  be used to combine gradients in the graph:
-
-  *  `ADD_N`: All of the gradient terms are summed as part of one
-     operation using the "AddN" op. It has the property that all
-     gradients must be ready before any aggregation is performed.
-  *  `DEFAULT`: The system-chosen default aggregation method.
-  """
-  ADD_N = 0
-  DEFAULT = ADD_N
-  # The following are experimental and may not be supported in future releases.
-  EXPERIMENTAL_TREE = 1
-  EXPERIMENTAL_ACCUMULATE_N = 2
-
-
-def _AggregatedGrads(grads,
-                     op,
-                     gradient_uid,
-                     loop_state,
-                     aggregation_method=None):
-  """Get the aggregated gradients for op.
-
-  Args:
-    grads: The map of memoized gradients.
-    op: The op to get gradients for.
-    gradient_uid: A unique identifier within the graph indicating
-      which invocation of gradients is being executed. Used to cluster
-      ops for compilation.
-    loop_state: An object for maintaining the state of the while loops in the
-                graph. It is of type ControlFlowState. None if the graph
-                contains no while loops.
-    aggregation_method: Specifies the method used to combine gradient terms.
-      Accepted values are constants defined in the class `AggregationMethod`.
-
-  Returns:
-    A list of gradients, one per each output of `op`. If the gradients
-      for a particular output is a list, this function aggregates it
-      before returning.
-
-  Raises:
-    TypeError: if the incoming grads are not Tensors or IndexedSlices.
-    ValueError: if the arguments are invalid.
-
-  """
-  if aggregation_method is None:
-    aggregation_method = AggregationMethod.DEFAULT
-  if aggregation_method not in [
-      AggregationMethod.ADD_N, AggregationMethod.EXPERIMENTAL_TREE,
-      AggregationMethod.EXPERIMENTAL_ACCUMULATE_N
-  ]:
-    raise ValueError(
-        "Invalid aggregation_method specified %s." % aggregation_method)
-  out_grads = _GetGrads(grads, op)
-  for i, out_grad in enumerate(out_grads):
-    if loop_state:
-      if isinstance(out_grad, (ops.Tensor, ops.IndexedSlices)):
-        assert control_flow_util.IsLoopSwitch(op)
-        continue
-    # Grads have to be Tensors or IndexedSlices
-    if (isinstance(out_grad, collections.Sequence) and not all(
-        isinstance(g, (ops.Tensor, ops.IndexedSlices))
-        for g in out_grad
-        if g is not None
-    )):
-      raise TypeError("gradients have to be either all Tensors "
-                      "or all IndexedSlices")
-    # Aggregate multiple gradients, and convert [] to None.
-    if out_grad:
-      if len(out_grad) < 2:
-        used = "nop"
-        out_grads[i] = out_grad[0]
-      elif all(isinstance(g, ops.Tensor) for g in out_grad if g is not None):
-        tensor_shape = _AccumulatorShape(out_grad)
-        if (aggregation_method == AggregationMethod.EXPERIMENTAL_ACCUMULATE_N
-            and len(out_grad) > 2 and tensor_shape.is_fully_defined()):
-          # The benefit of using AccumulateN is that its inputs can be combined
-          # in any order and this can allow the expression to be evaluated with
-          # a smaller memory footprint.  When used with gpu_allocator_retry,
-          # it is possible to compute a sum of terms which are much larger than
-          # total GPU memory.
-          # AccumulateN can currently only be used if we know the shape for
-          # an accumulator variable.  If this is not known, or if we only have
-          # 2 grads then we fall through to the "tree" case below.
-          used = "accumulate_n"
-          out_grads[i] = math_ops.accumulate_n(out_grad)
-        elif aggregation_method in [
-            AggregationMethod.EXPERIMENTAL_TREE,
-            AggregationMethod.EXPERIMENTAL_ACCUMULATE_N
-        ]:
-          # Aggregate all gradients by doing pairwise sums: this may
-          # reduce performance, but it can improve memory because the
-          # gradients can be released earlier.
-          #
-          # TODO(vrv): Consider replacing this with a version of
-          # tf.AddN() that eagerly frees its inputs as soon as they are
-          # ready, so the order of this tree does not become a problem.
-          used = "tree"
-          with ops.name_scope(op.name + "_gradient_sum"):
-            running_sum = out_grad[0]
-            for grad in out_grad[1:]:
-              running_sum = math_ops.add_n([running_sum, grad])
-            out_grads[i] = running_sum
-        else:
-          used = "add_n"
-          out_grads[i] = _MultiDeviceAddN(out_grad, gradient_uid)
-        logging.vlog(2, "  _AggregatedGrads %d x %s using %s", len(out_grad),
-                     tensor_shape, used)
-      else:
-        out_grads[i] = _AggregateIndexedSlicesGradients(out_grad)
-    else:  # not out_grad
-      # out_grads[i] is [], thus its aggregation is simply None.
-      out_grads[i] = None
-  return out_grads
-
-
-def _AggregateIndexedSlicesGradients(grads):
-  """Aggregates gradients of type `IndexedSlices` by concatenation."""
-  if len(grads) < 1:
-    return None
-  elif len(grads) == 1:
-    return grads[0]
-  else:
-    grads = math_ops._as_indexed_slices_list(  # pylint: disable=protected-access
-        [g for g in grads if g is not None])
-    grads = [_HandleNestedIndexedSlices(x) for x in grads]  # pylint: disable=protected-access
-    # Form IndexedSlices out of the concatenated values and indices.
-    concat_grad = ops.IndexedSlices(
-        array_ops.concat([x.values for x in grads], axis=0),
-        array_ops.concat([x.indices for x in grads], axis=0),
-        grads[0].dense_shape)
-
-    return concat_grad
+  # pylint: disable=protected-access
+  with ops.get_default_graph()._mutation_lock():
+    return gradients_util._GradientsHelper(
+        ys, xs, grad_ys, name, True, gate_gradients,
+        aggregation_method, stop_gradients,
+        unconnected_gradients)
+  # pylint: enable=protected-access
 
 
 # TODO(vrv): Make this available when we want to make it public.
@@ -1393,7 +361,7 @@ def hessians(ys,
     LookupError: if one of the operations between `xs` and `ys` does not
       have a registered gradient function.
   """
-  xs = _AsList(xs)
+  xs = gradients_util._AsList(xs)  # pylint: disable=protected-access
   kwargs = {
       "colocate_gradients_with_ops": colocate_gradients_with_ops,
       "gate_gradients": gate_gradients,
diff --git a/tensorflow/python/ops/gradients_test.py b/tensorflow/python/ops/gradients_test.py
index c53afef63b..9caffa3ea8 100644
--- a/tensorflow/python/ops/gradients_test.py
+++ b/tensorflow/python/ops/gradients_test.py
@@ -45,6 +45,7 @@ from tensorflow.python.ops import data_flow_ops  # pylint: disable=unused-import
 from tensorflow.python.ops import functional_ops  # pylint: disable=unused-import
 from tensorflow.python.ops import gradients
 from tensorflow.python.ops import gradients_impl
+from tensorflow.python.ops import gradients_util
 from tensorflow.python.ops import list_ops
 from tensorflow.python.ops import math_grad  # pylint: disable=unused-import
 from tensorflow.python.ops import math_ops
@@ -1040,12 +1041,12 @@ class AggregateIndexedSlicesGradientsTest(test_util.TensorFlowTestCase):
         self.evaluate(ops.convert_to_tensor(right)))
 
   def testNoGradients(self):
-    self.assertIsNone(gradients_impl._AggregateIndexedSlicesGradients([]))
+    self.assertIsNone(gradients_util._AggregateIndexedSlicesGradients([]))
 
   def testOneGradient(self):
     t = math_ops._as_indexed_slices(constant_op.constant(
         [[1., 2.], [0, 0], [3., 4.]]))
-    result = gradients_impl._AggregateIndexedSlicesGradients([t])
+    result = gradients_util._AggregateIndexedSlicesGradients([t])
     self._assert_indexed_slices_equal(t, result)
 
   def testMultipleGradients(self):
@@ -1055,7 +1056,7 @@ class AggregateIndexedSlicesGradientsTest(test_util.TensorFlowTestCase):
         [[0., 0.], [5, 6], [7., 8.]]))
     total = constant_op.constant(
         [[1., 2.], [5, 6], [10., 12.]])
-    result = gradients_impl._AggregateIndexedSlicesGradients([t0, t1])
+    result = gradients_util._AggregateIndexedSlicesGradients([t0, t1])
     self._assert_indexed_slices_equal(total, result)
 
   def testMultipleGradientsWithNones(self):
@@ -1066,7 +1067,7 @@ class AggregateIndexedSlicesGradientsTest(test_util.TensorFlowTestCase):
     t3 = None
     total = constant_op.constant(
         [[1., 2.], [5, 6], [10., 12.]])
-    result = gradients_impl._AggregateIndexedSlicesGradients([t0, t1, t3])
+    result = gradients_util._AggregateIndexedSlicesGradients([t0, t1, t3])
     self._assert_indexed_slices_equal(total, result)
 
   def testMixedTensorAndIndexedSlices(self):
@@ -1076,7 +1077,7 @@ class AggregateIndexedSlicesGradientsTest(test_util.TensorFlowTestCase):
         [[0., 0.], [5, 6], [7., 8.]])
     total = constant_op.constant(
         [[1., 2.], [5, 6], [10., 12.]])
-    result = gradients_impl._AggregateIndexedSlicesGradients([t0, t1])
+    result = gradients_util._AggregateIndexedSlicesGradients([t0, t1])
     self._assert_indexed_slices_equal(total, result)
 
 
diff --git a/tensorflow/python/ops/gradients_util.py b/tensorflow/python/ops/gradients_util.py
new file mode 100644
index 0000000000..4f73423d75
--- /dev/null
+++ b/tensorflow/python/ops/gradients_util.py
@@ -0,0 +1,1062 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Implements the graph generation for computation of gradients."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import collections
+import contextlib
+import warnings
+
+import numpy as np
+import six
+from six.moves import xrange  # pylint: disable=redefined-builtin
+
+from tensorflow.core.framework import attr_value_pb2
+from tensorflow.python.eager import context
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import function as framework_function
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import tensor_util
+from tensorflow.python.framework.func_graph import FuncGraph
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import control_flow_util
+from tensorflow.python.ops import functional_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops.unconnected_gradients import UnconnectedGradients
+from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.util import compat
+from tensorflow.python.util.tf_export import tf_export
+
+
+# Warn the user if we convert a sparse representation to dense with at
+# least this number of elements.
+_LARGE_SPARSE_NUM_ELEMENTS = 100000000
+
+
+def _IndexedSlicesToTensor(value, dtype=None, name=None, as_ref=False):
+  """Converts an IndexedSlices object `value` to a Tensor.
+
+  NOTE(mrry): This function is potentially expensive.
+
+  Args:
+    value: An ops.IndexedSlices object.
+    dtype: The dtype of the Tensor to be returned.
+    name: Optional name to use for the returned Tensor.
+    as_ref: True if a ref is requested.
+
+  Returns:
+    A dense Tensor representing the values in the given IndexedSlices.
+
+  Raises:
+    ValueError: If the IndexedSlices does not have the same dtype.
+  """
+  _ = as_ref
+  if dtype and not dtype.is_compatible_with(value.dtype):
+    raise ValueError(
+        "Tensor conversion requested dtype %s for IndexedSlices with dtype %s" %
+        (dtype.name, value.dtype.name))
+  if value.dense_shape is None:
+    raise ValueError(
+        "Tensor conversion requested for IndexedSlices without dense_shape: %s"
+        % str(value))
+  # TODO(mrry): Consider adding static shape information to
+  # IndexedSlices, to avoid using numpy here.
+  if not context.executing_eagerly():
+    dense_shape_value = tensor_util.constant_value(value.dense_shape)
+    if dense_shape_value is not None:
+      num_elements = np.prod(dense_shape_value)
+      if num_elements >= _LARGE_SPARSE_NUM_ELEMENTS:
+        warnings.warn(
+            "Converting sparse IndexedSlices to a dense Tensor with %d "
+            "elements. This may consume a large amount of memory." %
+            num_elements)
+    else:
+      warnings.warn(
+          "Converting sparse IndexedSlices to a dense Tensor of unknown shape. "
+          "This may consume a large amount of memory.")
+  return math_ops.unsorted_segment_sum(
+      value.values, value.indices, value.dense_shape[0], name=name)
+
+
+ops.register_tensor_conversion_function(ops.IndexedSlices,
+                                        _IndexedSlicesToTensor)
+
+
+def _MarkReachedOps(from_ops, reached_ops, func_graphs):
+  """Mark all ops reached from "from_ops".
+
+  Args:
+    from_ops: list of Operations.
+    reached_ops: set of Operations.
+    func_graphs: list of FuncGraphs. This method will traverse through
+      these functions if they capture from_ops or any reachable ops.
+  """
+  queue = collections.deque()
+  queue.extend(from_ops)
+  while queue:
+    op = queue.popleft()
+    if op not in reached_ops:
+      reached_ops.add(op)
+      for output in op.outputs:
+        if _IsBackpropagatable(output):
+          queue.extend(_Consumers(output, func_graphs))
+
+
+def _PendingCount(to_ops, from_ops, colocate_gradients_with_ops, func_graphs,
+                  xs):
+  """Initialize the pending count for ops between two lists of Operations.
+
+  'pending_count[op]' indicates the number of backprop inputs
+  to this operation.
+
+  Args:
+    to_ops: list of Operations.
+    from_ops: list of Operations.
+    colocate_gradients_with_ops: Python bool.  See docstring of gradients().
+    func_graphs: list of FuncGraphs. This method will traverse through
+      these functions if they capture from_ops or any reachable ops. This is
+      useful if to_ops occur in a function and from_ops are in an outer function
+      or graph.
+    xs: list of Tensors.
+
+  Returns:
+    A tuple containing: (1) the subset of to_ops reachable from from_ops by a
+    path of zero or more backpropagatable tensors, (2) a mapping from operation
+    to the number of backprop inputs to that op, and (3) a ControlFlowState
+    object which is not None if the ops between from_ops and to_ops contain
+    control flow loops.
+  """
+  # Mark reachable ops from from_ops.
+  reached_ops = set()
+  _MarkReachedOps(from_ops, reached_ops, func_graphs)
+  # X in reached_ops iff X is reachable from from_ops by a path of zero or more
+  # backpropagatable tensors.
+
+  reachable_to_ops = set(op for op in to_ops if op in reached_ops)
+
+  # Mark between ops.
+  between_ops = set()
+  between_op_list = []
+  queue = collections.deque()
+  queue.extend(to_ops)
+  while queue:
+    op = queue.popleft()
+    # We are interested in this op.
+    if op in reached_ops:
+      between_ops.add(op)
+      between_op_list.append(op)
+      # Clear the boolean so we won't add the inputs again.
+      reached_ops.remove(op)
+      for inp in _NonEagerInputs(op, xs):
+        queue.append(inp.op)
+  # X in between_ops iff X is on a path of zero or more backpropagatable tensors
+  # between from_ops and to_ops
+
+  # 'loop_state' is None if there are no while loops.
+  loop_state = control_flow_ops.MaybeCreateControlFlowState(
+      between_op_list, between_ops, colocate_gradients_with_ops)
+
+  # Initialize pending count for between ops.
+  pending_count = collections.defaultdict(int)
+  for op in between_op_list:
+    for x in _NonEagerInputs(op, xs):
+      if x.op in between_ops:
+        pending_count[x.op] += 1
+
+  return reachable_to_ops, pending_count, loop_state
+
+
+def _AsList(x):
+  return x if isinstance(x, (list, tuple)) else [x]
+
+
+def _DefaultGradYs(grad_ys,
+                   ys,
+                   colocate_gradients_with_ops,
+                   gradient_uid="__unsupported__"):
+  """Fill in default values for grad_ys.
+
+  Args:
+    grad_ys: List of gradients, can contain None.
+    ys: List of tensors.
+    colocate_gradients_with_ops: If True, try colocating gradients with
+      the corresponding op.
+    gradient_uid: A unique identifier within the graph indicating
+      which invocation of gradients is being executed. Used to cluster
+      ops for compilation.
+
+  Returns:
+    A list of gradients to use, without None.
+
+  Raises:
+    ValueError: If sizes of gradients and inputs don't match
+    TypeError: If type of any gradient is not valid for its input.
+  """
+  if len(grad_ys) != len(ys):
+    raise ValueError("Passed %d grad_ys for %d ys" % (len(grad_ys), len(ys)))
+  grad_ys = ops.convert_n_to_tensor_or_indexed_slices(grad_ys, name="grad_y")
+  new_grad_ys = []
+  for i in xrange(len(grad_ys)):
+    grad_y = grad_ys[i]
+    y = ys[i]
+    with _maybe_colocate_with(y.op, gradient_uid, colocate_gradients_with_ops):
+      if grad_y is None:
+        if y.dtype.is_complex:
+          raise TypeError(
+              "Gradients of complex tensors must set grad_ys (y.dtype = %r)" %
+              y.dtype)
+        new_grad_ys.append(
+            array_ops.fill(
+                array_ops.shape(y),
+                constant_op.constant(1, dtype=y.dtype, name="grad_ys_%d" % i)))
+        continue
+      if y.dtype.is_floating or y.dtype.is_integer:
+        if not grad_y.dtype.is_floating and not grad_y.dtype.is_integer:
+          raise TypeError(
+              "Gradient type %s generated for real or "
+              "integer-valued tensor %s with type %s must be "
+              "real or integer" % (dtypes.as_dtype(grad_y.dtype).name, y,
+                                   dtypes.as_dtype(y.dtype).name))
+      elif y.dtype.is_complex:
+        if not grad_y.dtype.is_complex:
+          raise TypeError(
+              "Gradient type %s generated for complex-valued "
+              "tensor %s with type %s must be real" % (dtypes.as_dtype(
+                  grad_y.dtype).name, y, dtypes.as_dtype(y.dtype).name))
+      elif y.dtype == dtypes.variant:
+        if grad_y.dtype != dtypes.variant:
+          raise TypeError(
+              "Gradient type %s generated for variant "
+              "tensor %s with type %s must be variant" % (dtypes.as_dtype(
+                  grad_y.dtype).name, y, dtypes.as_dtype(y.dtype).name))
+      elif y.dtype == dtypes.resource:
+        # We assume y is the handle of a ResourceVariable. The gradient of a
+        # ResourceVariable should be a numeric value, not another resource.
+        if grad_y.dtype == dtypes.resource:
+          raise TypeError("Input gradient %s for resource tensor %s should not "
+                          "be a resource" % (grad_y, y))
+      else:
+        raise TypeError(
+            "Tensor %s with type %s must be numeric "
+            "to obtain a default gradient" % (y, dtypes.as_dtype(y.dtype).name))
+      # Create a grad_y tensor in the name scope of the gradient.
+      # Required for TensorArrays to identify which gradient call a
+      # grad_y value is coming from.
+      if isinstance(grad_y, ops.IndexedSlices):
+        new_grad_ys.append(
+            ops.IndexedSlices(
+                indices=(array_ops.identity(
+                    grad_y.indices, name="grad_ys_%d_indices" % i)
+                         if isinstance(grad_y.indices, ops.Tensor) else
+                         grad_y.indices),
+                values=(array_ops.identity(
+                    grad_y.values, name="grad_ys_%d_values" % i) if isinstance(
+                        grad_y.values, ops.Tensor) else grad_y.values),
+                dense_shape=(array_ops.identity(
+                    grad_y.dense_shape, name="grad_ys_%d_shape" % i)
+                             if isinstance(grad_y.dense_shape, ops.Tensor) else
+                             grad_y.dense_shape)))
+      else:
+        new_grad_ys.append(array_ops.identity(grad_y, name="grad_ys_%d" % i))
+
+  return new_grad_ys
+
+
+def IsTrainable(tensor_or_dtype):
+  if isinstance(tensor_or_dtype, ops.Tensor):
+    dtype = tensor_or_dtype.dtype
+  else:
+    dtype = tensor_or_dtype
+  dtype = dtypes.as_dtype(dtype)
+  return dtype.base_dtype in (dtypes.float16, dtypes.float32, dtypes.float64,
+                              dtypes.complex64, dtypes.complex128,
+                              dtypes.resource, dtypes.variant)
+
+
+def _IsBackpropagatable(tensor):
+  if IsTrainable(tensor):
+    return True
+  dtype = dtypes.as_dtype(tensor.dtype)
+  return dtype.base_dtype == dtypes.bfloat16
+
+
+def _VerifyGeneratedGradients(grads, op):
+  """Verify that gradients are valid in number and type.
+
+  Args:
+    grads: List of generated gradients.
+    op: Operation for which the gradients where generated.
+
+  Raises:
+    ValueError: if sizes of gradients and inputs don't match.
+    TypeError: if type of any gradient is not valid for its input.
+  """
+  # While ops have inputs added to them during the gradient computation, so we
+  # skip the below check. See while_v2 for details.
+  if op.type == "While": return
+
+  if len(grads) != len(op.inputs):
+    raise ValueError("Num gradients %d generated for op %s do not match num "
+                     "inputs %d" % (len(grads), op.node_def, len(op.inputs)))
+
+
+def _StopOps(from_ops, stop_gradient_ops, pending_count, xs):
+  """The set of ops that terminate the gradient computation.
+
+  This computes the frontier of the forward graph *before* which backprop
+  should stop. Operations in the returned set will not be differentiated.
+  This set is defined as the subset of `from_ops` containing ops that have
+  no predecessor in `from_ops`. `pending_count` is the result of
+  `_PendingCount(xs, from_ops)`. An 'op' has predecessors in `from_ops`
+  iff pending_count[op] > 0.
+
+  In addition, none of `stop_gradient_ops` will be differentiated.
+
+  Args:
+    from_ops: list of Operations.
+    stop_gradient_ops: list of Operations never to backprop through.
+    pending_count: mapping from operation to number of backprop inputs.
+    xs: list of Tensors.
+
+  Returns:
+    The set of operations.
+  """
+  stop_ops = set()
+  for op in from_ops:
+    is_stop_op = True
+    for inp in _NonEagerInputs(op, xs):
+      if pending_count[inp.op] > 0:
+        is_stop_op = False
+        break
+    if is_stop_op:
+      stop_ops.add(op)
+  stop_ops.update(op for op in stop_gradient_ops)
+  return stop_ops
+
+
+@contextlib.contextmanager
+def _maybe_colocate_with(op, gradient_uid, colocate_gradients_with_ops):  # pylint: disable=invalid-name
+  """Context to colocate with `op` if `colocate_gradients_with_ops`."""
+  if colocate_gradients_with_ops:
+    with ops._colocate_with_for_gradient(op, gradient_uid):  # pylint: disable=protected-access
+      yield
+  else:
+    yield
+
+
+def _IsPartitionedCall(op):
+  return op.type == "PartitionedCall" or op.type == "StatefulPartitionedCall"
+
+
+def _SymGrad(op, out_grads):
+  """Backprop through a function call node op given its outputs' gradients."""
+  f_in = [x for x in op.inputs] + out_grads
+  f_types = [x.dtype for x in op.inputs]
+  f = attr_value_pb2.NameAttrList()
+  if _IsPartitionedCall(op):
+    f.name = op.get_attr("f").name
+  else:
+    f.name = op.type
+  for k in op.node_def.attr:
+    f.attr[k].CopyFrom(op.node_def.attr[k])
+  # TODO(apassos) use a better dtype here
+  in_grads = functional_ops.symbolic_gradient(
+      input=f_in,
+      Tout=[x if x != dtypes.resource else dtypes.float32 for x in f_types],
+      f=f)
+  return in_grads
+
+
+def _MaybeCompile(scope, op, func, grad_fn):
+  """Compile the calculation in grad_fn if op was marked as compiled."""
+  scope = scope.rstrip("/").replace("/", "_")
+  if func is not None:
+    xla_compile = func.definition.attr["_XlaCompile"].b
+    xla_separate_compiled_gradients = func.definition.attr[
+        "_XlaSeparateCompiledGradients"].b
+    xla_scope = func.definition.attr["_XlaScope"].s.decode()
+  else:
+    try:
+      xla_compile = op.get_attr("_XlaCompile")
+      xla_separate_compiled_gradients = op.get_attr(
+          "_XlaSeparateCompiledGradients")
+      xla_scope = op.get_attr("_XlaScope").decode()
+    except ValueError:
+      return grad_fn()  # Exit early
+
+  if not xla_compile:
+    return grad_fn()  # Exit early
+
+  # If the gradients are supposed to be compiled separately, we give them a
+  # _XlaScope name that is based on the name_scope of the gradients.  Otherwise
+  # they just inherit the existing _XlaScope name, which lets them be merged
+  # together with the non-gradient computation.
+  if xla_separate_compiled_gradients:
+    xla_grad_scope = "%s_grad_%s" % (xla_scope, scope)
+  else:
+    xla_grad_scope = xla_scope
+
+  attrs = {
+      "_XlaCompile": attr_value_pb2.AttrValue(b=xla_compile),
+      "_XlaScope": attr_value_pb2.AttrValue(s=xla_grad_scope.encode())
+  }
+  with ops.get_default_graph()._attr_scope(attrs):  # pylint: disable=protected-access
+    return grad_fn()
+
+
+def _RaiseNoGradWrtInitialLoopValError(op, from_ops, xs):
+  """Raises an error if we backprop through a loop var."""
+  # Find the nearest 'to_op' reachable from 'op' to provide a more helpful error
+  # message.
+  target_op = None
+  queue = collections.deque([op])
+  visited = set()
+  while queue:
+    curr_op = queue.popleft()
+    if curr_op in visited: continue
+    visited.add(curr_op)
+    if curr_op in from_ops:
+      target_op = curr_op
+      break
+    queue.extend(t.op for t in _NonEagerInputs(curr_op, xs))
+  assert target_op
+  raise ValueError(
+      "Cannot compute gradient inside while loop with respect to op '%s'. "
+      "We do not support taking the gradient wrt or through the initial value "
+      "of a loop variable. Gradients can be computed through loop invariants "
+      "or wrt the input parameters to the loop body."
+      % target_op.name)
+
+
+def _IsFunction(graph):
+  return (isinstance(graph, FuncGraph) or
+          isinstance(graph, framework_function._FuncGraph))  # pylint: disable=protected-access
+
+
+def _Captures(func_graph):
+  if isinstance(func_graph, FuncGraph):
+    return func_graph.captures
+  else:
+    assert isinstance(func_graph, framework_function._FuncGraph)  # pylint: disable=protected-access
+    return func_graph._captured  # pylint: disable=protected-access
+
+
+def _MaybeCaptured(t):
+  """If t is a captured value placeholder, returns the original captured value.
+
+  Args:
+    t: Tensor
+
+  Returns:
+    A tensor, potentially from a different Graph/FuncGraph.
+  """
+  # pylint: disable=protected-access
+  if (not isinstance(t, ops.EagerTensor) and
+      _IsFunction(t.op.graph) and t.op.type == "Placeholder"):
+    for input_t, placeholder_t in _Captures(t.op.graph).items():
+      if t == placeholder_t:
+        return _MaybeCaptured(input_t)
+  # pylint: enable=protected-access
+  return t
+
+
+# TODO(skyewm): plumbing xs through everywhere is ugly, consider making
+# _GradientsHelper a class with xs as a member variable.
+def _NonEagerInputs(op, xs):
+  """Returns the inputs of op, crossing closure boundaries where necessary.
+
+  Does not return any captured EagerTensors, i.e., the number of tensors
+  returned may be less than than the actual number of inputs.
+
+  Args:
+    op: Operation
+    xs: list of Tensors we are differentiating w.r.t.
+
+  Returns:
+    A list of tensors. The tensors may be from multiple Graph/FuncGraphs if op
+    is in a FuncGraph and has captured inputs.
+  """
+  if _IsFunction(op.graph):  # pylint: disable=protected-access
+    inputs = []
+    for t in op.inputs:
+      # If we're differentiating w.r.t. `t`, do not attempt to traverse through
+      # it to a captured value. The algorithm needs to "see" `t` in this case,
+      # even if it's a function input for a captured value, whereas usually we'd
+      # like to traverse through these closures as if the captured value was the
+      # direct input to op.
+      if t not in xs:
+        t = _MaybeCaptured(t)
+        # Skip captured eager inputs.
+        if isinstance(t, ops.EagerTensor): continue
+      inputs.append(t)
+    return inputs
+  else:
+    return op.inputs
+
+
+def _Consumers(t, func_graphs):
+  """Returns the consumers of t, crossing closure boundaries where necessary.
+
+  Args:
+    t: Tensor
+    func_graphs: a list of FuncGraphs that may have captured t.
+
+  Returns:
+    A list of tensors. The tensors will be from the current graph and/or
+    func_graphs.
+  """
+  consumers = t.consumers()
+  for func in func_graphs:
+    for input_t, placeholder in _Captures(func).items():
+      if input_t == t:
+        consumers.extend(_Consumers(placeholder, func_graphs))
+  return consumers
+
+
+def _GradientsHelper(ys,
+                     xs,
+                     grad_ys=None,
+                     name="gradients",
+                     colocate_gradients_with_ops=False,
+                     gate_gradients=False,
+                     aggregation_method=None,
+                     stop_gradients=None,
+                     unconnected_gradients=UnconnectedGradients.NONE,
+                     src_graph=None):
+  """Implementation of gradients()."""
+  if context.executing_eagerly():
+    raise RuntimeError("tf.gradients is not supported when eager execution "
+                       "is enabled. Use tf.GradientTape instead.")
+  if src_graph is None:
+    src_graph = ops.get_default_graph()
+  try:
+    unconnected_gradients = UnconnectedGradients(unconnected_gradients)
+  except ValueError:
+    raise ValueError(
+        "Unknown value for unconnected_gradients: %r" % unconnected_gradients)
+
+  # If src_graph is a _FuncGraph (i.e. a function body), gather it and all
+  # ancestor graphs. This is necessary for correctly handling captured values.
+  func_graphs = []
+  curr_graph = src_graph
+  while _IsFunction(curr_graph):
+    func_graphs.append(curr_graph)
+    if isinstance(curr_graph, FuncGraph):
+      curr_graph = curr_graph.outer_graph
+    else:
+      assert isinstance(curr_graph, framework_function._FuncGraph)  # pylint: disable=protected-access
+      curr_graph = curr_graph._outer_graph  # pylint: disable=protected-access
+
+  ys = _AsList(ys)
+  xs = _AsList(xs)
+  stop_gradients = [] if stop_gradients is None else _AsList(stop_gradients)
+  if grad_ys is None:
+    grad_ys = [None] * len(ys)
+  else:
+    grad_ys = _AsList(grad_ys)
+
+  with ops.name_scope(
+      name, "gradients",
+      list(ys) + list(xs) + list(stop_gradients) + list(grad_ys)) as grad_scope:
+    # Get a uid for this call to gradients that can be used to help
+    # cluster ops for compilation.
+    gradient_uid = ops.get_default_graph().unique_name("uid")
+    ys = ops.convert_n_to_tensor_or_indexed_slices(ys, name="y")
+    xs = [
+        x.handle if resource_variable_ops.is_resource_variable(x) else x
+        for x in xs
+    ]
+    xs = ops.internal_convert_n_to_tensor_or_indexed_slices(
+        xs, name="x", as_ref=True)
+    grad_ys = _DefaultGradYs(grad_ys, ys, colocate_gradients_with_ops,
+                             gradient_uid)
+
+    # The approach we take here is as follows: Create a list of all ops in the
+    # subgraph between the ys and xs.  Visit these ops in reverse order of ids
+    # to ensure that when we visit an op the gradients w.r.t its outputs have
+    # been collected.  Then aggregate these gradients if needed, call the op's
+    # gradient function, and add the generated gradients to the gradients for
+    # its input.
+
+    # Initialize the pending count for ops in the connected subgraph from ys
+    # to the xs.
+    to_ops = [t.op for t in ys]
+    from_ops = [t.op for t in xs]
+    stop_gradient_ops = [t.op for t in stop_gradients]
+    reachable_to_ops, pending_count, loop_state = _PendingCount(
+        to_ops, from_ops, colocate_gradients_with_ops, func_graphs, xs)
+
+    # Iterate over the collected ops.
+    #
+    # grads: op => list of gradients received on each output endpoint of the
+    # op.  The gradients for each endpoint are initially collected as a list.
+    # When it is time to call the op's gradient function, for each endpoint we
+    # aggregate the list of received gradients into a Add() Operation if there
+    # is more than one.
+    grads = {}
+
+    # Add the initial gradients for the ys.
+    for y, grad_y in zip(ys, grad_ys):
+      _SetGrad(grads, y, grad_y)
+
+    # Initialize queue with to_ops.
+    queue = collections.deque()
+    # Add the ops in 'to_ops' into the queue.
+    to_ops_set = set()
+    for op in to_ops:
+      # 'ready' handles the case where one output gradient relies on
+      # another output's gradient.
+      ready = (pending_count[op] == 0)
+      if ready and op not in to_ops_set and op in reachable_to_ops:
+        to_ops_set.add(op)
+        queue.append(op)
+
+    if loop_state:
+      loop_exits = loop_state.ProcessUnusedLoopExits(pending_count, to_ops_set)
+      for y in loop_exits:
+        if IsTrainable(y):
+          _SetGrad(grads, y, loop_state.ZerosLikeForExit(y))
+          queue.append(y.op)
+
+    stop_ops = _StopOps(from_ops, stop_gradient_ops, pending_count, xs)
+    while queue:
+      # generate gradient subgraph for op.
+      op = queue.popleft()
+      with _maybe_colocate_with(op, gradient_uid, colocate_gradients_with_ops):
+        if loop_state:
+          loop_state.EnterGradWhileContext(op, before=True)
+        out_grads = _AggregatedGrads(grads, op, gradient_uid, loop_state,
+                                     aggregation_method)
+        if loop_state:
+          loop_state.ExitGradWhileContext(op, before=True)
+
+        grad_fn = None
+        func_call = None
+        is_partitioned_call = _IsPartitionedCall(op)
+        # pylint: disable=protected-access
+        is_func_call = (
+            src_graph._is_function(op.type) or is_partitioned_call)
+        # pylint: enable=protected-access
+        has_out_grads = any(isinstance(g, ops.Tensor) or g for g in out_grads)
+        if has_out_grads and (op not in stop_ops):
+          try:
+            grad_fn = ops.get_gradient_function(op)
+          except LookupError:
+            if is_func_call:
+              if is_partitioned_call:
+                func_call = src_graph._get_function(  # pylint: disable=protected-access
+                    compat.as_bytes(op.get_attr("f").name))
+              else:
+                func_call = src_graph._get_function(op.type)  # pylint: disable=protected-access
+              # Note that __defun is not set if the graph is
+              # imported. If it's set, we prefer to access the original
+              # defun.
+              func_call = getattr(op, "__defun", func_call)
+              grad_fn = func_call.python_grad_func
+            else:
+              raise LookupError(
+                  "No gradient defined for operation '%s' (op type: %s)" %
+                  (op.name, op.type))
+        if loop_state:
+          loop_state.EnterGradWhileContext(op, before=False)
+
+        # NOTE(skyewm): We don't support computing gradients wrt a loop variable
+        # unless it's within the context of a single iteration (i.e. the
+        # gradient is wrt to the loop parameter in the body function, not wrt or
+        # through the initial value). This means if we're in a while loop
+        # context, we should never see a switch node from this context.
+        # pylint: disable=protected-access
+        if (control_flow_util.IsSwitch(op) and
+            op._control_flow_context is not None and
+            op._control_flow_context.IsWhileContext() and
+            op._control_flow_context ==
+            ops.get_default_graph()._get_control_flow_context()):
+          _RaiseNoGradWrtInitialLoopValError(op, from_ops, xs)
+        # pylint: enable=protected-access
+
+        if (grad_fn or is_func_call) and has_out_grads:
+          # NOTE: If _AggregatedGrads didn't compute a value for the i'th
+          # output, it means that the cost does not depend on output[i],
+          # therefore dC/doutput[i] is 0.
+          for i, out_grad in enumerate(out_grads):
+            if (not isinstance(out_grad, ops.Tensor) and not out_grad) and (
+                (not grad_fn and is_func_call) or IsTrainable(op.outputs[i])):
+              # Only trainable outputs or outputs for a function call that
+              # will use SymbolicGradient get a zero gradient. Gradient
+              # functions should ignore the gradient for other outputs.
+              # TODO(apassos) gradients of resource handles might be an
+              # issue here because of zeros.
+              if loop_state:
+                out_grads[i] = loop_state.ZerosLike(op, i)
+              else:
+                out_grads[i] = control_flow_ops.ZerosLikeOutsideLoop(op, i)
+          with ops.name_scope(op.name + "_grad"):
+            # pylint: disable=protected-access
+            with src_graph._original_op(op):
+              # pylint: enable=protected-access
+              if grad_fn:
+                # If grad_fn was found, do not use SymbolicGradient even for
+                # functions.
+                in_grads = _MaybeCompile(grad_scope, op, func_call,
+                                         lambda: grad_fn(op, *out_grads))
+              else:
+                # For function call ops, we add a 'SymbolicGradient'
+                # node to the graph to compute gradients.
+                in_grads = _MaybeCompile(grad_scope, op, func_call,
+                                         lambda: _SymGrad(op, out_grads))
+              in_grads = _AsList(in_grads)
+              _VerifyGeneratedGradients(in_grads, op)
+              if gate_gradients and len([x for x in in_grads
+                                         if x is not None]) > 1:
+                with ops.device(None):
+                  with ops._colocate_with_for_gradient(  # pylint: disable=protected-access
+                      None,
+                      gradient_uid,
+                      ignore_existing=True):
+                    in_grads = control_flow_ops.tuple(in_grads)
+          _LogOpGradients(op, out_grads, in_grads)
+        else:
+          # If no grad_fn is defined or none of out_grads is available,
+          # just propagate a list of None backwards.
+          in_grads = [None] * len(_NonEagerInputs(op, xs))
+        for i, (t_in, in_grad) in enumerate(zip(_NonEagerInputs(op, xs),
+                                                in_grads)):
+          if in_grad is not None:
+            if (isinstance(in_grad, ops.Tensor) and
+                t_in.dtype != dtypes.resource):
+              try:
+                in_grad.set_shape(t_in.get_shape())
+              except ValueError:
+                raise ValueError(
+                    "Incompatible shapes between op input and calculated "
+                    "input gradient.  Forward operation: %s.  Input index: %d. "
+                    "Original input shape: %s.  "
+                    "Calculated input gradient shape: %s" %
+                    (op.name, i, t_in.shape, in_grad.shape))
+            _SetGrad(grads, t_in, in_grad)
+        if loop_state:
+          loop_state.ExitGradWhileContext(op, before=False)
+
+      # Update pending count for the inputs of op and enqueue ready ops.
+      _UpdatePendingAndEnqueueReady(grads, op, queue, pending_count, loop_state,
+                                    xs)
+
+  if loop_state:
+    loop_state.PostProcessing()
+  return [_GetGrad(grads, x, unconnected_gradients) for x in xs]
+
+
+def _HasAnyNotNoneGrads(grads, op):
+  """Return true iff op has real gradient."""
+  out_grads = _GetGrads(grads, op)
+  for out_grad in out_grads:
+    if isinstance(out_grad, (ops.Tensor, ops.IndexedSlices)):
+      return True
+    if out_grad and isinstance(out_grad, collections.Sequence):
+      if any(g is not None for g in out_grad):
+        return True
+  return False
+
+
+def _UpdatePendingAndEnqueueReady(grads, op, queue, pending_count, loop_state,
+                                  xs):
+  """Update pending count for the inputs of op and enqueue ready ops."""
+  for x in _NonEagerInputs(op, xs):
+    pending_count[x.op] -= 1
+    ready = (pending_count[x.op] == 0)
+    if loop_state and not ready:
+      ready = pending_count[x.op] > 0 and control_flow_util.IsLoopSwitch(x.op)
+    if ready:
+      if control_flow_util.IsLoopExit(x.op):
+        # if x is an exit without real gradient, defer processing them.
+        grad_state = loop_state.GetGradState(x.op, before=False)
+        grad_state.deferred_exits.append(x)
+        grad_state.pending_exits_count -= 1
+        if grad_state.pending_exits_count == 0:
+          # We now have all the exits so process them.
+          has_not_none_grad = False
+          for y in grad_state.deferred_exits:
+            if _HasAnyNotNoneGrads(grads, y.op):
+              has_not_none_grad = True
+              queue.append(y.op)
+            else:
+              grad_state.unused_exits.append(y)
+          if has_not_none_grad:
+            # For an unused exit, if it has trainable outputs, backprop
+            # a zero gradient. Otherwise, just ignore it.
+            for y in grad_state.unused_exits:
+              if IsTrainable(y):
+                _SetGrad(grads, y, loop_state.ZerosLikeForExit(y))
+              queue.append(y.op)
+          else:
+            # All exits are "unused" so use None as gradient.
+            for y in grad_state.unused_exits:
+              queue.append(y.op)
+      else:
+        queue.append(x.op)
+
+
+def _SetGrad(grads, t, grad):
+  """Sets gradient "grad" in "grads" for tensor "t"."""
+  op = t.op
+  op_grads = grads.get(op)
+  if not op_grads:
+    op_grads = [[] for _ in xrange(len(op.outputs))]
+    grads[op] = op_grads
+  t_grads = op_grads[t.value_index]
+  if isinstance(t_grads, list):
+    t_grads.append(grad)
+  else:
+    assert control_flow_util.IsLoopSwitch(op)
+    op_grads[t.value_index] = grad
+
+
+def _GetGrad(grads, t, unconnected_gradients):
+  """Gets gradient for tensor "t"."""
+  op = t.op
+  op_grads = grads.get(op)
+  if not op_grads:
+    if unconnected_gradients == UnconnectedGradients.ZERO:
+      t_dtype = t.dtype if t.dtype != dtypes.resource else dtypes.float32
+      return array_ops.zeros_like(t, dtype=t_dtype)
+    elif unconnected_gradients == UnconnectedGradients.NONE:
+      return None
+    else:
+      raise ValueError(
+          "Unknown value for unconnected_gradients: %r" % unconnected_gradients)
+
+  t_grad = op_grads[t.value_index]
+  assert not isinstance(
+      t_grad, list), ("gradients list should have been aggregated by now.")
+  return t_grad
+
+
+def _GetGrads(grads, op):
+  """Gets all gradients for op."""
+  if op in grads:
+    return grads[op]
+  else:
+    return [[] for _ in xrange(len(op.outputs))]
+
+
+def _HandleNestedIndexedSlices(grad):
+  assert isinstance(grad, ops.IndexedSlices)
+  if isinstance(grad.values, ops.Tensor):
+    return grad
+  else:
+    assert isinstance(grad.values, ops.IndexedSlices)
+    g = _HandleNestedIndexedSlices(grad.values)
+    return ops.IndexedSlices(g.values, array_ops.gather(
+        grad.indices, g.indices), g.dense_shape)
+
+
+def _AccumulatorShape(inputs):
+  shape = tensor_shape.unknown_shape()
+  for i in inputs:
+    if isinstance(i, ops.Tensor):
+      shape = shape.merge_with(i.get_shape())
+  return shape
+
+
+def _LogOpGradients(op, out_grads, in_grads):
+  """Log the in and out grads of an op."""
+  logging.vlog(1, "Gradient for '" + op.name + "'")
+
+  def _FilterGrad(x):
+    if x is None:
+      return False
+    if isinstance(x, (list, tuple)):
+      return bool(x)
+    else:
+      return True
+
+  logging.vlog(1, "  in  --> %s",
+               ", ".join([x.name for x in out_grads if _FilterGrad(x)]))
+  logging.vlog(1, "  out --> %s",
+               ", ".join([x.name for x in in_grads if _FilterGrad(x)]))
+
+
+def _MultiDeviceAddN(tensor_list, gradient_uid):
+  """Adds tensors from potentially multiple devices."""
+  # Basic function structure comes from control_flow_ops.group().
+  # Sort tensors according to their devices.
+  tensors_on_device = collections.defaultdict(lambda: [])
+  for tensor in tensor_list:
+    tensors_on_device[tensor.device].append(tensor)
+
+  # For each device, add the tensors on that device first.
+  # Then gather the partial sums from multiple devices.
+  # TODO(sjhwang): Create hierarchical aggregation tree as pbar's suggestion.
+  # E.g., aggregate per GPU, then per task, and so on.
+  summands = []
+
+  def DeviceKey(dev):
+    return "" if dev is None else dev
+
+  for dev in sorted(six.iterkeys(tensors_on_device), key=DeviceKey):
+    tensors = tensors_on_device[dev]
+    with ops._colocate_with_for_gradient(  # pylint: disable=protected-access
+        tensors[0].op,
+        gradient_uid,
+        ignore_existing=True):
+      summands.append(math_ops.add_n(tensors))
+
+  return math_ops.add_n(summands)
+
+
+@tf_export("AggregationMethod")
+class AggregationMethod(object):
+  """A class listing aggregation methods used to combine gradients.
+
+  Computing partial derivatives can require aggregating gradient
+  contributions. This class lists the various methods that can
+  be used to combine gradients in the graph:
+
+  *  `ADD_N`: All of the gradient terms are summed as part of one
+     operation using the "AddN" op. It has the property that all
+     gradients must be ready before any aggregation is performed.
+  *  `DEFAULT`: The system-chosen default aggregation method.
+  """
+  ADD_N = 0
+  DEFAULT = ADD_N
+  # The following are experimental and may not be supported in future releases.
+  EXPERIMENTAL_TREE = 1
+  EXPERIMENTAL_ACCUMULATE_N = 2
+
+
+def _AggregatedGrads(grads,
+                     op,
+                     gradient_uid,
+                     loop_state,
+                     aggregation_method=None):
+  """Get the aggregated gradients for op.
+
+  Args:
+    grads: The map of memoized gradients.
+    op: The op to get gradients for.
+    gradient_uid: A unique identifier within the graph indicating
+      which invocation of gradients is being executed. Used to cluster
+      ops for compilation.
+    loop_state: An object for maintaining the state of the while loops in the
+                graph. It is of type ControlFlowState. None if the graph
+                contains no while loops.
+    aggregation_method: Specifies the method used to combine gradient terms.
+      Accepted values are constants defined in the class `AggregationMethod`.
+
+  Returns:
+    A list of gradients, one per each output of `op`. If the gradients
+      for a particular output is a list, this function aggregates it
+      before returning.
+
+  Raises:
+    TypeError: if the incoming grads are not Tensors or IndexedSlices.
+    ValueError: if the arguments are invalid.
+
+  """
+  if aggregation_method is None:
+    aggregation_method = AggregationMethod.DEFAULT
+  if aggregation_method not in [
+      AggregationMethod.ADD_N, AggregationMethod.EXPERIMENTAL_TREE,
+      AggregationMethod.EXPERIMENTAL_ACCUMULATE_N
+  ]:
+    raise ValueError(
+        "Invalid aggregation_method specified %s." % aggregation_method)
+  out_grads = _GetGrads(grads, op)
+  for i, out_grad in enumerate(out_grads):
+    if loop_state:
+      if isinstance(out_grad, (ops.Tensor, ops.IndexedSlices)):
+        assert control_flow_util.IsLoopSwitch(op)
+        continue
+    # Grads have to be Tensors or IndexedSlices
+    if (isinstance(out_grad, collections.Sequence) and not all(
+        isinstance(g, (ops.Tensor, ops.IndexedSlices))
+        for g in out_grad
+        if g is not None
+    )):
+      raise TypeError("gradients have to be either all Tensors "
+                      "or all IndexedSlices")
+    # Aggregate multiple gradients, and convert [] to None.
+    if out_grad:
+      if len(out_grad) < 2:
+        used = "nop"
+        out_grads[i] = out_grad[0]
+      elif all(isinstance(g, ops.Tensor) for g in out_grad if g is not None):
+        tensor_shape = _AccumulatorShape(out_grad)
+        if (aggregation_method == AggregationMethod.EXPERIMENTAL_ACCUMULATE_N
+            and len(out_grad) > 2 and tensor_shape.is_fully_defined()):
+          # The benefit of using AccumulateN is that its inputs can be combined
+          # in any order and this can allow the expression to be evaluated with
+          # a smaller memory footprint.  When used with gpu_allocator_retry,
+          # it is possible to compute a sum of terms which are much larger than
+          # total GPU memory.
+          # AccumulateN can currently only be used if we know the shape for
+          # an accumulator variable.  If this is not known, or if we only have
+          # 2 grads then we fall through to the "tree" case below.
+          used = "accumulate_n"
+          out_grads[i] = math_ops.accumulate_n(out_grad)
+        elif aggregation_method in [
+            AggregationMethod.EXPERIMENTAL_TREE,
+            AggregationMethod.EXPERIMENTAL_ACCUMULATE_N
+        ]:
+          # Aggregate all gradients by doing pairwise sums: this may
+          # reduce performance, but it can improve memory because the
+          # gradients can be released earlier.
+          #
+          # TODO(vrv): Consider replacing this with a version of
+          # tf.AddN() that eagerly frees its inputs as soon as they are
+          # ready, so the order of this tree does not become a problem.
+          used = "tree"
+          with ops.name_scope(op.name + "_gradient_sum"):
+            running_sum = out_grad[0]
+            for grad in out_grad[1:]:
+              running_sum = math_ops.add_n([running_sum, grad])
+            out_grads[i] = running_sum
+        else:
+          used = "add_n"
+          out_grads[i] = _MultiDeviceAddN(out_grad, gradient_uid)
+        logging.vlog(2, "  _AggregatedGrads %d x %s using %s", len(out_grad),
+                     tensor_shape, used)
+      else:
+        out_grads[i] = _AggregateIndexedSlicesGradients(out_grad)
+    else:  # not out_grad
+      # out_grads[i] is [], thus its aggregation is simply None.
+      out_grads[i] = None
+  return out_grads
+
+
+def _AggregateIndexedSlicesGradients(grads):
+  """Aggregates gradients of type `IndexedSlices` by concatenation."""
+  if len(grads) < 1:
+    return None
+  elif len(grads) == 1:
+    return grads[0]
+  else:
+    grads = math_ops._as_indexed_slices_list(  # pylint: disable=protected-access
+        [g for g in grads if g is not None])
+    grads = [_HandleNestedIndexedSlices(x) for x in grads]  # pylint: disable=protected-access
+    # Form IndexedSlices out of the concatenated values and indices.
+    concat_grad = ops.IndexedSlices(
+        array_ops.concat([x.values for x in grads], axis=0),
+        array_ops.concat([x.indices for x in grads], axis=0),
+        grads[0].dense_shape)
+
+    return concat_grad
diff --git a/tensorflow/python/ops/ragged/BUILD b/tensorflow/python/ops/ragged/BUILD
index e3bdb74cfe..ef95169e9c 100644
--- a/tensorflow/python/ops/ragged/BUILD
+++ b/tensorflow/python/ops/ragged/BUILD
@@ -370,6 +370,7 @@ py_library(
         ":ragged_tensor_shape",
         ":ragged_util",
         "//tensorflow/python:array_ops",
+        "//tensorflow/python:bitwise_ops",
         "//tensorflow/python:clip_ops",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:math_ops",
diff --git a/tensorflow/python/ops/while_v2.py b/tensorflow/python/ops/while_v2.py
index 9f34172836..40c3be9a1e 100644
--- a/tensorflow/python/ops/while_v2.py
+++ b/tensorflow/python/ops/while_v2.py
@@ -39,7 +39,7 @@ from tensorflow.python.ops import control_flow_util_v2 as util
 from tensorflow.python.ops import custom_gradient
 from tensorflow.python.ops import gen_functional_ops
 from tensorflow.python.ops import gen_resource_variable_ops
-from tensorflow.python.ops import gradients_impl
+from tensorflow.python.ops import gradients_util
 from tensorflow.python.ops import list_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import tensor_array_ops
@@ -371,7 +371,7 @@ def _zeros_like(op_output):
 
 def _is_trainable(tensor):
   """Returns whether the given tensor is trainable."""
-  if not gradients_impl.IsTrainable(tensor):
+  if not gradients_util.IsTrainable(tensor):
     return False
 
   # Special case: untrainable accumulator output. The gradients algorithm
@@ -382,7 +382,7 @@ def _is_trainable(tensor):
   if tensor.op.type == "TensorListPopBack" and tensor.value_index == 0:
     assert tensor.dtype == dtypes.variant
     element_type = tensor.op.get_attr("element_dtype")
-    return gradients_impl.IsTrainable(element_type)
+    return gradients_util.IsTrainable(element_type)
 
   return True
 
@@ -534,7 +534,7 @@ def _grad_fn(ys, xs, args, func_graph):
   # func_graph. The captured func_graph tensors are resolved to external tensors
   # after the forward While op has been rewritten in _resolve_grad_captures.
   # TODO(srbs): Mark GradientsHelper as public?
-  grad_outs = gradients_impl._GradientsHelper(
+  grad_outs = gradients_util._GradientsHelper(
       ys, xs, grad_ys=grad_ys, src_graph=func_graph,
       unconnected_gradients="zero")
 
diff --git a/tensorflow/python/training/optimizer_test.py b/tensorflow/python/training/optimizer_test.py
index e175b5a799..ac831cb642 100644
--- a/tensorflow/python/training/optimizer_test.py
+++ b/tensorflow/python/training/optimizer_test.py
@@ -24,7 +24,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import clip_ops
-from tensorflow.python.ops import gradients_impl
+from tensorflow.python.ops import gradients_util
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variables
@@ -75,7 +75,7 @@ class OptimizerTest(test.TestCase):
         opt_op = sgd_op.minimize(
             cost,
             global_step, [var0, var1],
-            aggregation_method=gradients_impl.AggregationMethod.
+            aggregation_method=gradients_util.AggregationMethod.
             EXPERIMENTAL_ACCUMULATE_N)
 
         variables.global_variables_initializer().run()
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.-aggregation-method.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.-aggregation-method.pbtxt
index f79029d3fe..cc2d5c87d6 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.-aggregation-method.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.-aggregation-method.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.AggregationMethod"
 tf_class {
-  is_instance: "<class \'tensorflow.python.ops.gradients_impl.AggregationMethod\'>"
+  is_instance: "<class \'tensorflow.python.ops.gradients_util.AggregationMethod\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "ADD_N"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-aggregation-method.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-aggregation-method.pbtxt
index f79029d3fe..cc2d5c87d6 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.-aggregation-method.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.-aggregation-method.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.AggregationMethod"
 tf_class {
-  is_instance: "<class \'tensorflow.python.ops.gradients_impl.AggregationMethod\'>"
+  is_instance: "<class \'tensorflow.python.ops.gradients_util.AggregationMethod\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "ADD_N"
-- 
GitLab


From 96526b3d24692426117fbf5b7ae572271172cdff Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Thu, 31 Jan 2019 14:04:34 -0800
Subject: [PATCH 1680/2345] Add validation support for TopK op.

PiperOrigin-RevId: 231849685
---
 tensorflow/compiler/tf2tensorrt/BUILD         |   2 +
 .../tf2tensorrt/convert/convert_nodes.cc      |  39 +-
 .../tf2tensorrt/convert/convert_nodes.h       |  28 +-
 .../tf2tensorrt/convert/convert_nodes_test.cc | 372 ++++++++++++------
 4 files changed, 289 insertions(+), 152 deletions(-)

diff --git a/tensorflow/compiler/tf2tensorrt/BUILD b/tensorflow/compiler/tf2tensorrt/BUILD
index 35d577e2c1..00d3c8cc5f 100644
--- a/tensorflow/compiler/tf2tensorrt/BUILD
+++ b/tensorflow/compiler/tf2tensorrt/BUILD
@@ -323,6 +323,7 @@ tf_cuda_cc_test(
         ":trt_plugins",
         "@com_google_googletest//:gtest",
         "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:span",
         "//tensorflow/cc:cc_ops",
         "//tensorflow/cc:ops",
         "//tensorflow/cc:scope",
@@ -332,6 +333,7 @@ tf_cuda_cc_test(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:tensor_testutil",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
index ff6b6d2cb5..6d4a06ab15 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
@@ -3547,31 +3547,34 @@ tensorflow::Status ConvertSoftmax(OpConverterParams* params) {
 
 tensorflow::Status ConvertTopK(OpConverterParams* params) {
   const auto& inputs = params->inputs;
+  if (inputs.size() != 2 || !inputs.at(0).is_tensor() ||
+      !inputs.at(1).is_weights()) {
+    return errors::InvalidArgument("Input expects tensor and weights, at ",
+                                   params->node_def.name());
+  }
+
   const auto& node_def = params->node_def;
   const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
-
-  int nbDims = tensor->getDimensions().nbDims;
-  if (nbDims == 0) {
-    return tensorflow::errors::InvalidArgument(
-        "TensorRT TopK cannot apply on batch dimension, at" + node_def.name());
+  const int num_dims = tensor->getDimensions().nbDims;
+  if (num_dims == 0) {
+    return errors::InvalidArgument(
+        "TensorRT TopK cannot apply on batch dimension, at", node_def.name());
   }
 
   TRT_ShapedWeights k_w = inputs.at(1).weights();
-  int k = *(static_cast<int*>(const_cast<void*>(k_w.GetValues())));
-
-  nvinfer1::TopKOperation op;
-  uint32_t reducedAxes = 0;
-  if (node_def.op() == "TopKV2") {
-    op = nvinfer1::TopKOperation::kMAX;
-    reducedAxes |= 1 << (nbDims - 1);
-  } else {
-    return tensorflow::errors::Unimplemented(
-        "Operation: ", node_def.op(),
-        " not implemented, at: ", node_def.name());
+  if (k_w.count() != 1) {
+    return errors::InvalidArgument("k value of TopK should be a scalar, at",
+                                   node_def.name());
   }
+  // Note that ITopKLayer always have sorted outputs, so we don't need to handle
+  // the 'sorted' attribute of the node.
+  if (params->validation_only) return Status::OK();
 
+  const nvinfer1::TopKOperation op = nvinfer1::TopKOperation::kMAX;
+  const int k = *(static_cast<int*>(const_cast<void*>(k_w.GetValues())));
+  const uint32_t reduce_axes = 1 << (num_dims - 1);
   nvinfer1::ITopKLayer* layer = params->converter->network()->addTopK(
-      *const_cast<nvinfer1::ITensor*>(tensor), op, k, reducedAxes);
+      *const_cast<nvinfer1::ITensor*>(tensor), op, k, reduce_axes);
   TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name());
 
   nvinfer1::ITensor* output_value_tensor = layer->getOutput(0);
@@ -3598,6 +3601,7 @@ static void RegisterValidatableOpConverters(
   (*registration)["Squeeze"] = ConvertSqueeze;
   (*registration)["StridedSlice"] = ConvertStridedSlice;
   (*registration)["Transpose"] = ConvertTranspose;
+  (*registration)["TopKV2"] = ConvertTopK;
 
   for (auto quantization_op_type :
        {"QuantizeAndDequantizeV2", "QuantizeAndDequantizeV3",
@@ -3644,7 +3648,6 @@ void Converter::RegisterOpConverters() {
   op_registry_["Mean"] = ConvertReduce;
   op_registry_["Softmax"] = ConvertSoftmax;
   op_registry_["BatchMatMul"] = ConvertBatchMatMul;
-  op_registry_["TopKV2"] = ConvertTopK;
 
   plugin_converter_ = ConvertPlugin;
 }
diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
index 0a33d0056a..00373051f5 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
@@ -178,6 +178,8 @@ class TRT_ShapedWeights {
 
   nvinfer1::Weights GetTrtWeights() const;
 
+  // Returns the raw pointer to the underlying buffer which holds the weights
+  // value.
   void* GetValues() const {
     return const_cast<char*>(tensor_.tensor_data().data());
   }
@@ -400,6 +402,19 @@ class TrtNodeValidator {
 // Class to convert TF nodes to TRT network.
 class Converter {
  public:
+  // Used for Converter::RenameAndMarkOutputTensors()
+  struct EngineOutputInfo {
+    // The TRT tensor name which produces the output.
+    string source_tensor_name;
+    // The TensorFlow node name which is receiving the output from the TRT
+    // engine. This should always be the Identity node created in
+    // ConvertSegmentToGraphDef.
+    string dest_node_name;
+    // Output type. TensorRT requires this to be explicitly set for engine
+    // outputs.
+    nvinfer1::DataType trt_dtype;
+  };
+
   Converter(nvinfer1::INetworkDefinition* trt_network, int precision_mode,
             bool use_calibration);
 
@@ -415,19 +430,6 @@ class Converter {
   Status AddInputTensor(const string& name, nvinfer1::DataType dtype,
                         const nvinfer1::Dims& dims, int batch_size);
 
-  // Used for Converter::RenameAndMarkOutputTensors()
-  struct EngineOutputInfo {
-    // The TRT tensor name which produces the output.
-    string source_tensor_name;
-    // The TensorFlow node name which is receiving the output from the TRT
-    // engine. This should always be the Identity node created in
-    // ConvertSegmentToGraphDef.
-    string dest_node_name;
-    // Output type. TensorRT requires this to be explicitly set for engine
-    // outputs.
-    nvinfer1::DataType trt_dtype;
-  };
-
   // Mark the tensors with names specified by source_tensor_name as output of
   // the TRT network, and set their names in the TRT network as dest_node_name.
   Status RenameAndMarkOutputTensors(
diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
index 39f4fb5708..9789475ba9 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
 #include "absl/strings/str_cat.h"
+#include "absl/types/span.h"
 #include "tensorflow/cc/framework/ops.h"
 #include "tensorflow/cc/framework/scope.h"
 #include "tensorflow/cc/ops/standard_ops.h"
@@ -154,7 +155,7 @@ void ExpectTrtDimsEqualsArray(const std::vector<int>& lhs,
 }
 
 template <typename T>
-void ExpectArrayNear(const std::vector<T>& lhs, const std::vector<T>& rhs) {
+void ExpectArrayNear(const std::vector<T>& lhs, absl::Span<const T> rhs) {
   ASSERT_EQ(lhs.size(), rhs.size());
   for (int i = 0; i < lhs.size(); i++) {
     EXPECT_FLOAT_EQ(lhs[i], rhs[i]);
@@ -165,7 +166,7 @@ void ExpectArrayNear(const std::vector<T>& lhs, const std::vector<T>& rhs) {
 // EXPECT_FLOAT_EQ.
 template <>
 void ExpectArrayNear(const std::vector<Eigen::half>& lhs,
-                     const std::vector<Eigen::half>& rhs) {
+                     absl::Span<const Eigen::half> rhs) {
   ASSERT_EQ(lhs.size(), rhs.size());
   for (int i = 0; i < lhs.size(); i++) {
     EXPECT_FLOAT_EQ(Eigen::half_impl::half_to_float(lhs[i]),
@@ -928,6 +929,32 @@ TEST_F(ConverterTest, CreateConstantLayer) {
   }
 }
 
+// Input/output data format for OpConverterTest::BuildAndRun().
+struct InputOutputData {
+  void* Buffer() const {
+    return const_cast<char*>(tensor.tensor_data().data());
+  }
+
+  size_t TotalBytes() const { return tensor.TotalBytes(); }
+
+  const char* name;
+  Tensor tensor;
+};
+
+template <typename T>
+Tensor ConstructTensor(int data_size, const T& value = T()) {
+  std::vector<T> values(data_size, value);
+  return test::AsTensor<T>(values);
+}
+
+using DataVec = std::vector<InputOutputData>;
+
+template <typename T>
+inline absl::Span<const T> GetSpanForData(const InputOutputData& data) {
+  const auto& tensor_map = data.tensor.flat<T>();
+  return absl::Span<const T>(tensor_map.data(), tensor_map.size());
+}
+
 // Class to test various op converters, using both a TrtNodeValidator and
 // Converter.
 class OpConverterTest : public ::testing::Test {
@@ -953,6 +980,7 @@ class OpConverterTest : public ::testing::Test {
     builder_.reset(nvinfer1::createInferBuilder(logger_));
     network_.reset(builder_->createNetwork());
     builder_->setMaxBatchSize(1);
+    builder_->setMaxWorkspaceSize(1 << 26);
 
     // Reset the validator and converter.
     validator_.reset(new TrtNodeValidator);
@@ -966,15 +994,14 @@ class OpConverterTest : public ::testing::Test {
   }
 
   // TODO(laigd): test fp16 and int8 support.
-  template <typename T>
-  void BuildAndRun(
-      const std::vector<std::pair<const char*, const std::vector<T>>>&
-          input_data,
-      const char* output_name, std::vector<T>* output_data) {
+  void BuildAndRun(const DataVec& input_data, DataVec* output_data) {
     // Mark the output tensor as TRT engine output.
-    TF_EXPECT_OK(converter_->RenameAndMarkOutputTensors(
-        {{string(output_name), string(output_name),
-          TfDataTypeToTrt(DataTypeToEnum<T>::v())}}));
+    std::vector<Converter::EngineOutputInfo> output_info;
+    for (const auto& data : *output_data) {
+      output_info.push_back(
+          {data.name, data.name, TfDataTypeToTrt(data.tensor.dtype())});
+    }
+    TF_EXPECT_OK(converter_->RenameAndMarkOutputTensors(output_info));
 
     // Build the TRT engine.
     ASSERT_EQ(nullptr, engine_.get());
@@ -982,31 +1009,44 @@ class OpConverterTest : public ::testing::Test {
     CHECK_NOTNULL(engine_.get());
 
     // Execute the TRT engine.
-    ASSERT_LE(input_data.size() + 1, 3);
-    void* buffers[3];
-    for (const auto name_and_data : input_data) {
-      const int input_size = name_and_data.second.size() * sizeof(T);
-      const int input_index = engine_->getBindingIndex(name_and_data.first);
-      ASSERT_EQ(0, cudaMalloc(&buffers[input_index], input_size));
-      ASSERT_EQ(
-          0, cudaMemcpyAsync(buffers[input_index], name_and_data.second.data(),
-                             input_size, cudaMemcpyHostToDevice, stream_));
+    const int num_bindings = input_data.size() + output_data->size();
+    std::vector<void*> buffers(num_bindings);
+
+    for (const auto& data : input_data) {
+      const int input_index = engine_->getBindingIndex(data.name);
+      ASSERT_EQ(0, cudaMalloc(&buffers[input_index], data.TotalBytes()));
+      ASSERT_EQ(0, cudaMemcpyAsync(buffers[input_index], data.Buffer(),
+                                   data.TotalBytes(), cudaMemcpyHostToDevice,
+                                   stream_));
+    }
+    struct SizeAndIndex {
+      SizeAndIndex(int in_size, int in_index)
+          : size(in_size), index(in_index) {}
+      int size;
+      int index;
+    };
+    std::vector<SizeAndIndex> output_infos;
+    for (const auto& data : *output_data) {
+      const int output_index = engine_->getBindingIndex(data.name);
+      output_infos.emplace_back(data.TotalBytes(), output_index);
+      ASSERT_EQ(0, cudaMalloc(&buffers[output_index], data.TotalBytes()));
     }
 
-    const int output_size = output_data->size() * sizeof(T);
-    const int output_index = engine_->getBindingIndex(output_name);
-    ASSERT_EQ(0, cudaMalloc(&buffers[output_index], output_size));
-
-    ASSERT_EQ(engine_->getNbBindings(), input_data.size() + 1);
-
+    ASSERT_EQ(engine_->getNbBindings(), num_bindings);
     TrtUniquePtrType<nvinfer1::IExecutionContext> execution_context(
         engine_->createExecutionContext());
-    execution_context->enqueue(/*batchSize=*/1, buffers, stream_, nullptr);
-    ASSERT_EQ(0, cudaMemcpyAsync(output_data->data(), buffers[output_index],
-                                 output_size, cudaMemcpyDeviceToHost, stream_));
+    execution_context->enqueue(/*batchSize=*/1, buffers.data(), stream_,
+                               nullptr);
+
+    for (int i = 0; i < output_infos.size(); ++i) {
+      const auto& output_info = output_infos[i];
+      ASSERT_EQ(0, cudaMemcpyAsync(output_data->at(i).Buffer(),
+                                   buffers[output_info.index], output_info.size,
+                                   cudaMemcpyDeviceToHost, stream_));
+    }
     cudaStreamSynchronize(stream_);
 
-    for (int i = 0; i < input_data.size() + 1; ++i) {
+    for (int i = 0; i < num_bindings; ++i) {
       ASSERT_EQ(0, cudaFree(buffers[i]));
     }
   }
@@ -1301,10 +1341,12 @@ TEST_F(OpConverterTest, ConvertTranspose) {
     EXPECT_TRUE(output.is_tensor());
     ExpectTrtDimsEqualsArray({3, 1, 2}, output.tensor()->getDimensions());
 
-    std::vector<float> output_data(6);
-    BuildAndRun<float>({{"input", {1, 2, 3, 4, 5, 6}}}, "my_transpose",
-                       &output_data);
-    EXPECT_THAT(output_data, ElementsAre(1, 4, 2, 5, 3, 6));
+    const DataVec input_data{
+        {"input", test::AsTensor<float>({1, 2, 3, 4, 5, 6})}};
+    DataVec output_data{{"my_transpose", ConstructTensor<float>(6)}};
+    BuildAndRun(input_data, &output_data);
+    EXPECT_THAT(GetSpanForData<float>(output_data[0]),
+                ElementsAre(1, 4, 2, 5, 3, 6));
   }
 }
 
@@ -1391,10 +1433,12 @@ TEST_F(OpConverterTest, ConvertReshape) {
     EXPECT_TRUE(output.is_tensor());
     ExpectTrtDimsEqualsArray({1, 3, 2}, output.tensor()->getDimensions());
 
-    std::vector<float> output_data(6);
-    BuildAndRun<float>({{"input", {1, 2, 3, 4, 5, 6}}}, "my_reshape",
-                       &output_data);
-    EXPECT_THAT(output_data, ElementsAre(1, 2, 3, 4, 5, 6));
+    const DataVec input_data{
+        {"input", test::AsTensor<float>({1, 2, 3, 4, 5, 6})}};
+    DataVec output_data{{"my_reshape", ConstructTensor<float>(6)}};
+    BuildAndRun(input_data, &output_data);
+    EXPECT_THAT(GetSpanForData<float>(output_data[0]),
+                ElementsAre(1, 2, 3, 4, 5, 6));
   }
 }
 
@@ -1454,12 +1498,13 @@ TEST_F(OpConverterTest, ConvertMatMul) {
     EXPECT_TRUE(output.is_tensor());
     ExpectTrtDimsEqualsArray({2}, output.tensor()->getDimensions());
 
-    std::vector<float> output_data(2);
-    BuildAndRun<float>({{"input", {0, 1}}}, "my_matmul", &output_data);
+    const DataVec input_data{{"input", test::AsTensor<float>({0, 1})}};
+    DataVec output_data{{"my_matmul", ConstructTensor<float>(2)}};
+    BuildAndRun(input_data, &output_data);
     if (transpose_b) {
-      EXPECT_THAT(output_data, ElementsAre(1, 3));
+      EXPECT_THAT(GetSpanForData<float>(output_data[0]), ElementsAre(1, 3));
     } else {
-      EXPECT_THAT(output_data, ElementsAre(2, 3));
+      EXPECT_THAT(GetSpanForData<float>(output_data[0]), ElementsAre(2, 3));
     }
   }
 }
@@ -1513,23 +1558,28 @@ void TestConvertBiasAdd(OpConverterTest* test) {
       const int num_input = TrtDimsNumElements(GetTestDims(dims_array));
       ASSERT_EQ(trt_input_rank > 1 ? 6 : (data_format == "NHWC" ? 3 : 2),
                 num_input);
-      std::vector<CType> output_data(num_input);
-      test->BuildAndRun<CType>(
-          {{"input", std::vector<CType>(num_input, CType(0))}}, "my_biasadd",
-          &output_data);
+
+      const DataVec input_data{
+          {"input", ConstructTensor<CType>(num_input, CType(0))}};
+      DataVec output_data{{"my_biasadd", ConstructTensor<CType>(num_input)}};
+      test->BuildAndRun(input_data, &output_data);
       if (trt_input_rank == 1) {
         if (data_format == "NHWC") {
-          EXPECT_THAT(output_data, ElementsAre(CType(1), CType(2), CType(3)));
+          EXPECT_THAT(GetSpanForData<CType>(output_data[0]),
+                      ElementsAre(CType(1), CType(2), CType(3)));
         } else {
-          EXPECT_THAT(output_data, ElementsAre(CType(1), CType(2)));
+          EXPECT_THAT(GetSpanForData<CType>(output_data[0]),
+                      ElementsAre(CType(1), CType(2)));
         }
       } else {
         if (data_format == "NHWC") {
-          EXPECT_THAT(output_data, ElementsAre(CType(1), CType(2), CType(3),
-                                               CType(1), CType(2), CType(3)));
+          EXPECT_THAT(GetSpanForData<CType>(output_data[0]),
+                      ElementsAre(CType(1), CType(2), CType(3), CType(1),
+                                  CType(2), CType(3)));
         } else {
-          EXPECT_THAT(output_data, ElementsAre(CType(1), CType(1), CType(1),
-                                               CType(2), CType(2), CType(2)));
+          EXPECT_THAT(GetSpanForData<CType>(output_data[0]),
+                      ElementsAre(CType(1), CType(1), CType(1), CType(2),
+                                  CType(2), CType(2)));
         }
       }
     }
@@ -1607,21 +1657,25 @@ void TestBinaryTensorOpWeightNoBroadcast(OpConverterTest* test) {
     EXPECT_TRUE(output.is_tensor());
     ExpectTrtDimsEqualsArray({1, 1, 2}, output.tensor()->getDimensions());
 
-    std::vector<CType> output_data(2);
-    test->BuildAndRun<CType>(
-        {{"input",
-          /*input_data=*/swap_inputs ? operand2 : operand1}},
-        "my_binary", &output_data);
+    const DataVec input_data{
+        {"input", test::AsTensor<CType>(swap_inputs ? operand2 : operand1)}};
+    DataVec output_data{{"my_binary", ConstructTensor<CType>(2)}};
+    test->BuildAndRun(input_data, &output_data);
     if (node_def.op() == "Add") {
-      EXPECT_THAT(output_data, ElementsAre(CType(5), CType(10.5)));
+      EXPECT_THAT(GetSpanForData<CType>(output_data[0]),
+                  ElementsAre(CType(5), CType(10.5)));
     } else if (node_def.op() == "Sub") {
-      EXPECT_THAT(output_data, ElementsAre(CType(1), CType(4.5)));
+      EXPECT_THAT(GetSpanForData<CType>(output_data[0]),
+                  ElementsAre(CType(1), CType(4.5)));
     } else if (node_def.op() == "Mul") {
-      EXPECT_THAT(output_data, ElementsAre(CType(6), CType(22.5)));
+      EXPECT_THAT(GetSpanForData<CType>(output_data[0]),
+                  ElementsAre(CType(6), CType(22.5)));
     } else if (node_def.op() == "Div") {
-      EXPECT_THAT(output_data, ElementsAre(CType(1.5), CType(2.5)));
+      EXPECT_THAT(GetSpanForData<CType>(output_data[0]),
+                  ElementsAre(CType(1.5), CType(2.5)));
     } else if (node_def.op() == "RealDiv") {
-      EXPECT_THAT(output_data, ElementsAre(CType(1.5), CType(2.5)));
+      EXPECT_THAT(GetSpanForData<CType>(output_data[0]),
+                  ElementsAre(CType(1.5), CType(2.5)));
     } else {
       ASSERT_TRUE(false);
     }
@@ -1656,13 +1710,14 @@ void TestBinaryTensorOpWeightWithChannelWiseBroadcast(OpConverterTest* test) {
     EXPECT_TRUE(output.is_tensor());
     ExpectTrtDimsEqualsArray({2, 1, 2}, output.tensor()->getDimensions());
 
-    std::vector<CType> output_data(4);
-    test->BuildAndRun<CType>({{"input", input}}, "my_binary", &output_data);
+    const DataVec input_data{{"input", test::AsTensor<CType>(input)}};
+    DataVec output_data{{"my_binary", ConstructTensor<CType>(4)}};
+    test->BuildAndRun(input_data, &output_data);
     if (weights_dims.size() == 1) {
-      EXPECT_THAT(output_data,
+      EXPECT_THAT(GetSpanForData<CType>(output_data[0]),
                   ElementsAre(CType(11), CType(22), CType(13), CType(24)));
     } else {
-      EXPECT_THAT(output_data,
+      EXPECT_THAT(GetSpanForData<CType>(output_data[0]),
                   ElementsAre(CType(11), CType(12), CType(23), CType(24)));
     }
   }
@@ -1690,9 +1745,10 @@ void TestBinaryTensorOpWeightWithUniformlyBroadcast(OpConverterTest* test) {
   EXPECT_TRUE(output.is_tensor());
   ExpectTrtDimsEqualsArray({2, 1, 2}, output.tensor()->getDimensions());
 
-  std::vector<CType> output_data(4);
-  test->BuildAndRun<CType>({{"input", input}}, "my_binary", &output_data);
-  EXPECT_THAT(output_data,
+  const DataVec input_data{{"input", test::AsTensor<CType>(input)}};
+  DataVec output_data{{"my_binary", ConstructTensor<CType>(4)}};
+  test->BuildAndRun(input_data, &output_data);
+  EXPECT_THAT(GetSpanForData<CType>(output_data[0]),
               ElementsAre(CType(11), CType(12), CType(13), CType(14)));
 }
 
@@ -1740,17 +1796,19 @@ void TestBinaryTensorOpWeightFallback(OpConverterTest* test,
   // Check the result of running the engine.
   const int expected_num_outputs =
       TrtDimsNumElements(GetTestDims(expected_output_dims));
-  std::vector<CType> output_data(expected_num_outputs);
-  test->BuildAndRun<CType>(
-      {{"input",
-        /*input_data=*/std::vector<CType>(num_inputs, CType(2))}},
-      "my_binary", &output_data);
+  const DataVec input_data{
+      {"input", ConstructTensor<CType>(num_inputs, CType(2))}};
+  DataVec output_data{
+      {"my_binary", ConstructTensor<CType>(expected_num_outputs)}};
+  test->BuildAndRun(input_data, &output_data);
   if (node_def.op() == "Add") {
-    EXPECT_THAT(output_data, ElementsAreArray(std::vector<CType>(
-                                 expected_num_outputs, CType(3))));
+    EXPECT_THAT(
+        GetSpanForData<CType>(output_data[0]),
+        ElementsAreArray(std::vector<CType>(expected_num_outputs, CType(3))));
   } else if (node_def.op() == "Minimum") {
-    EXPECT_THAT(output_data, ElementsAreArray(std::vector<CType>(
-                                 expected_num_outputs, CType(1))));
+    EXPECT_THAT(
+        GetSpanForData<CType>(output_data[0]),
+        ElementsAreArray(std::vector<CType>(expected_num_outputs, CType(1))));
   } else {
     ASSERT_TRUE(false);
   }
@@ -1777,32 +1835,33 @@ void TestBinaryTensorOpTensor(OpConverterTest* test) {
   EXPECT_TRUE(output.is_tensor());
   ExpectTrtDimsEqualsArray({2, 2}, output.tensor()->getDimensions());
 
-  std::vector<CType> output_data(4);
+  const DataVec input_data{
+      {"input1", test::AsTensor<CType>({CType(3), CType(6)})},
+      {"input2", test::AsTensor<CType>({CType(2), CType(3)})}};
+  DataVec output_data{{"my_binary", ConstructTensor<CType>(4)}};
   // After broadcasting first input becomes {3, 6, 3, 6} and second input
   // becomes {2, 3, 2, 3}.
-  test->BuildAndRun<CType>(
-      {{"input1", {CType(3), CType(6)}}, {"input2", {CType(2), CType(3)}}},
-      "my_binary", &output_data);
+  test->BuildAndRun(input_data, &output_data);
   if (node_def.op() == "Add") {
-    EXPECT_THAT(output_data,
+    EXPECT_THAT(GetSpanForData<CType>(output_data[0]),
                 ElementsAre(CType(5), CType(8), CType(6), CType(9)));
   } else if (node_def.op() == "Sub") {
-    EXPECT_THAT(output_data,
+    EXPECT_THAT(GetSpanForData<CType>(output_data[0]),
                 ElementsAre(CType(1), CType(4), CType(0), CType(3)));
   } else if (node_def.op() == "Mul") {
-    EXPECT_THAT(output_data,
+    EXPECT_THAT(GetSpanForData<CType>(output_data[0]),
                 ElementsAre(CType(6), CType(12), CType(9), CType(18)));
   } else if (node_def.op() == "Div") {
-    EXPECT_THAT(output_data,
+    EXPECT_THAT(GetSpanForData<CType>(output_data[0]),
                 ElementsAre(CType(1.5), CType(3), CType(1), CType(2)));
   } else if (node_def.op() == "RealDiv") {
-    EXPECT_THAT(output_data,
+    EXPECT_THAT(GetSpanForData<CType>(output_data[0]),
                 ElementsAre(CType(1.5), CType(3), CType(1), CType(2)));
   } else if (node_def.op() == "Minimum") {
-    EXPECT_THAT(output_data,
+    EXPECT_THAT(GetSpanForData<CType>(output_data[0]),
                 ElementsAre(CType(2), CType(2), CType(3), CType(3)));
   } else if (node_def.op() == "Maximum") {
-    EXPECT_THAT(output_data,
+    EXPECT_THAT(GetSpanForData<CType>(output_data[0]),
                 ElementsAre(CType(3), CType(6), CType(3), CType(6)));
   } else {
     ASSERT_TRUE(false);
@@ -2042,10 +2101,12 @@ TEST_F(OpConverterTest, ConvertRelu6) {
     auto ranges = quantization_ranges();
     EXPECT_EQ(ranges[output.tensor()], 6.0f);
 
-    std::vector<float> output_data(6);
-    BuildAndRun<float>({{"input", {-100, -1, 0, 3, 5, 9}}}, "my_relu6",
-                       &output_data);
-    EXPECT_THAT(output_data, ElementsAre(0, 0, 0, 3, 5, 6));
+    const DataVec input_data{
+        {"input", test::AsTensor<float>({-100, -1, 0, 3, 5, 9})}};
+    DataVec output_data{{"my_relu6", ConstructTensor<float>(6)}};
+    BuildAndRun(input_data, &output_data);
+    EXPECT_THAT(GetSpanForData<float>(output_data[0]),
+                ElementsAre(0, 0, 0, 3, 5, 6));
   }
 }
 
@@ -2067,16 +2128,17 @@ void TestConvertSquare(OpConverterTest* test) {
   ExpectTrtDimsEqualsArray({1, 20}, output.tensor()->getDimensions());
 
   const int num_inputs = 20;
-  std::vector<CType> input_data(num_inputs);
-  std::vector<CType> expected_output_data(num_inputs);
+  std::vector<CType> inputs(num_inputs);
+  std::vector<CType> expected_outputs(num_inputs);
   for (int i = 0; i < 20; i++) {
     const CType value = CType(i - 9);
-    input_data[i] = value;
-    expected_output_data[i] = value * value;
+    inputs[i] = value;
+    expected_outputs[i] = value * value;
   }
-  std::vector<CType> output_data(num_inputs);
-  test->BuildAndRun<CType>({{"input", input_data}}, "my_square", &output_data);
-  ExpectArrayNear(expected_output_data, output_data);
+  const DataVec input_data{{"input", test::AsTensor<CType>(inputs)}};
+  DataVec output_data{{"my_square", ConstructTensor<CType>(num_inputs)}};
+  test->BuildAndRun(input_data, &output_data);
+  ExpectArrayNear(expected_outputs, GetSpanForData<CType>(output_data[0]));
 }
 
 TEST_F(OpConverterTest, ConvertSquare) {
@@ -2168,12 +2230,14 @@ TEST_F(OpConverterTest, ConvertActivation) {
     EXPECT_TRUE(output.is_tensor());
     ExpectTrtDimsEqualsArray({1, 2, 3}, output.tensor()->getDimensions());
 
-    const std::vector<float> input_data = {-100, -2, -1, 0, 1, 100};
-    std::vector<float> output_data(6);
-    BuildAndRun<float>({{"input", input_data}}, "my_act", &output_data);
-    for (int i = 0; i < input_data.size(); i++) {
-      const float expected_output = get_act_output(op_name, input_data[i]);
-      EXPECT_FLOAT_EQ(output_data[i], expected_output);
+    const std::vector<float> input = {-100, -2, -1, 0, 1, 100};
+    const DataVec input_data{{"input", test::AsTensor<float>(input)}};
+    DataVec output_data{{"my_act", ConstructTensor<float>(6)}};
+    BuildAndRun(input_data, &output_data);
+    for (int i = 0; i < input.size(); i++) {
+      const float expected_output = get_act_output(op_name, input[i]);
+      EXPECT_FLOAT_EQ(GetSpanForData<float>(output_data[0])[i],
+                      expected_output);
     }
   }
 }
@@ -2286,10 +2350,12 @@ TEST_F(OpConverterTest, ConvertExpandDims) {
     ExpectTrtDimsEqualsArray(ok_params[i].expected_output_dims,
                              output.tensor()->getDimensions());
 
-    std::vector<float> output_data(6);
-    BuildAndRun<float>({{"input", {1, 2, 3, 4, 5, 6}}}, "my_expanddims",
-                       &output_data);
-    EXPECT_THAT(output_data, ElementsAre(1, 2, 3, 4, 5, 6));
+    const DataVec input_data{
+        {"input", test::AsTensor<float>({1, 2, 3, 4, 5, 6})}};
+    DataVec output_data{{"my_expanddims", ConstructTensor<float>(6)}};
+    BuildAndRun(input_data, &output_data);
+    EXPECT_THAT(GetSpanForData<float>(output_data[0]),
+                ElementsAre(1, 2, 3, 4, 5, 6));
   }
 }
 
@@ -2406,10 +2472,12 @@ TEST_F(OpConverterTest, ConvertSqueeze) {
     ExpectTrtDimsEqualsArray(ok_params[i].expected_output_dims,
                              output.tensor()->getDimensions());
 
-    std::vector<float> output_data(6);
-    BuildAndRun<float>({{"input", {1, 2, 3, 4, 5, 6}}}, "my_squeeze",
-                       &output_data);
-    EXPECT_THAT(output_data, ElementsAre(1, 2, 3, 4, 5, 6));
+    const DataVec input_data{
+        {"input", test::AsTensor<float>({1, 2, 3, 4, 5, 6})}};
+    DataVec output_data{{"my_squeeze", ConstructTensor<float>(6)}};
+    BuildAndRun(input_data, &output_data);
+    EXPECT_THAT(GetSpanForData<float>(output_data[0]),
+                ElementsAre(1, 2, 3, 4, 5, 6));
   }
 }
 
@@ -2679,10 +2747,15 @@ TEST_F(OpConverterTest, ConvertStridedSlice) {
 
     TRT_TensorOrWeights output;
     TF_EXPECT_OK(GetTensorOrWeights("my_strided_slice", &output));
-    std::vector<float> output_data(ok_params[i].expected_output.size());
-    BuildAndRun<float>({{"input", {1, 2, 3, 4, 5, 6}}}, "my_strided_slice",
-                       &output_data);
-    EXPECT_THAT(output_data, ElementsAreArray(ok_params[i].expected_output));
+
+    const DataVec input_data{
+        {"input", test::AsTensor<float>({1, 2, 3, 4, 5, 6})}};
+    DataVec output_data{
+        {"my_strided_slice",
+         ConstructTensor<float>(ok_params[i].expected_output.size())}};
+    BuildAndRun(input_data, &output_data);
+    EXPECT_THAT(GetSpanForData<float>(output_data[0]),
+                ElementsAreArray(ok_params[i].expected_output));
   }
 }
 
@@ -2913,10 +2986,67 @@ TEST_F(OpConverterTest, ConvertConv2D) {
     EXPECT_TRUE(output.is_tensor());
     ExpectTrtDimsEqualsArray(ok_params[i].expected_output_dims,
                              output.tensor()->getDimensions());
-    std::vector<float> output_data(ok_params[i].expected_output.size());
-    BuildAndRun<float>({{"input", ok_params[i].input}}, "my_conv2d",
-                       &output_data);
-    EXPECT_THAT(output_data, ElementsAreArray(ok_params[i].expected_output));
+
+    const DataVec input_data{
+        {"input", test::AsTensor<float>(ok_params[i].input)}};
+    DataVec output_data{
+        {"my_conv2d",
+         ConstructTensor<float>(ok_params[i].expected_output.size())}};
+    BuildAndRun(input_data, &output_data);
+    EXPECT_THAT(GetSpanForData<float>(output_data[0]),
+                ElementsAreArray(ok_params[i].expected_output));
+  }
+}
+
+TEST_F(OpConverterTest, ConvertTopK) {
+  {
+    // Input list is empty, should fail.
+    NodeDef node_def = MakeNodeDef("my_topk", "TopKV2", {});
+    RunValidationAndConversion(node_def, error::INVALID_ARGUMENT,
+                               "Input expects tensor and weights, at my_topk");
+  }
+
+  for (const auto dtype : {DT_FLOAT, DT_INT32}) {
+    // Get the NodeDef for TopKV2.
+    Scope s = Scope::NewRootScope();
+    auto input = ops::Placeholder(s.WithOpName("input"), dtype);
+    auto weights = ops::Placeholder(s.WithOpName("weights"), DT_INT32);
+    auto topk = ops::TopK(s.WithOpName("my_topk"), input, weights);
+    const NodeDef& node_def = topk.operation.node()->def();
+    {
+      // K is a tensor, should fail.
+      Reset();
+      AddTestTensor("input", {1, 2, 3}, /*batch_size=*/1,
+                    /*trt_dtype=*/TfDataTypeToTrt(dtype));
+      AddTestTensor("weights", {2});
+      RunValidationAndConversion(
+          node_def, error::INVALID_ARGUMENT,
+          "Input expects tensor and weights, at my_topk");
+    }
+    {
+      // Ok.
+      Reset();
+      AddTestTensor("input", {1, 2, 5});
+      AddTestWeights<int32>("weights", {1}, {2});
+      RunValidationAndConversion(node_def);
+      TRT_TensorOrWeights outputs[2];
+      TF_EXPECT_OK(GetTensorOrWeights("my_topk", &outputs[0]));
+      TF_EXPECT_OK(GetTensorOrWeights("my_topk:1", &outputs[1]));
+      for (auto& output : outputs) {
+        EXPECT_TRUE(output.is_tensor());
+        ExpectTrtDimsEqualsArray({1, 2, 2}, output.tensor()->getDimensions());
+      }
+
+      const DataVec input_data{
+          {"input", test::AsTensor<float>({-9, 3, 5, 1, 6, -5, 7, 1, 0, -1})}};
+      DataVec output_data{{"my_topk", ConstructTensor<float>(4)},
+                          {"my_topk:1", ConstructTensor<int32>(4)}};
+      BuildAndRun(input_data, &output_data);
+      EXPECT_THAT(GetSpanForData<float>(output_data[0]),
+                  ElementsAre(6, 5, 7, 1));
+      EXPECT_THAT(GetSpanForData<int32>(output_data[1]),
+                  ElementsAre(4, 2, 1, 2));
+    }
   }
 }
 
-- 
GitLab


From b1489fe141088e1e794f3ee48a19c4b1b9c82f72 Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Thu, 31 Jan 2019 14:13:35 -0800
Subject: [PATCH 1681/2345] Do not enforce the time dimension in
 `TimeDistributed`'s `InputSpec`.

PiperOrigin-RevId: 231851598
---
 tensorflow/python/keras/layers/wrappers.py    |  8 +++++--
 .../python/keras/layers/wrappers_test.py      | 22 +++++++++++++++++++
 2 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/layers/wrappers.py b/tensorflow/python/keras/layers/wrappers.py
index c9424c9f63..a10db509d3 100644
--- a/tensorflow/python/keras/layers/wrappers.py
+++ b/tensorflow/python/keras/layers/wrappers.py
@@ -206,8 +206,12 @@ class TimeDistributed(Wrapper):
 
   def build(self, input_shape):
     input_shape = tensor_shape.TensorShape(input_shape).as_list()
-    assert len(input_shape) >= 3
-    self.input_spec = InputSpec(shape=input_shape)
+    if len(input_shape) < 3:
+      raise ValueError(
+          '`TimeDistributed` Layer should be passed an `input_shape ` '
+          'with at least 3 dimensions, received: ' + str(input_shape))
+    # Don't enforce the batch or time dimension.
+    self.input_spec = InputSpec(shape=[None, None] + input_shape[2:])
     child_input_shape = [input_shape[0]] + input_shape[2:]
     if not self.layer.built:
       # The base layer class calls a conversion function on the input shape to
diff --git a/tensorflow/python/keras/layers/wrappers_test.py b/tensorflow/python/keras/layers/wrappers_test.py
index f3aa5c4d68..8a0b265c0d 100644
--- a/tensorflow/python/keras/layers/wrappers_test.py
+++ b/tensorflow/python/keras/layers/wrappers_test.py
@@ -256,6 +256,28 @@ class TimeDistributedTest(test.TestCase):
       self.assertEqual((mask_outputs_val[1]).all(),
                        model_input.all())
 
+  def test_TimeDistributed_with_different_time_shapes(self):
+    time_dist = keras.layers.TimeDistributed(keras.layers.Dense(5))
+    ph_1 = keras.backend.placeholder(shape=(None, 10, 13))
+    out_1 = time_dist(ph_1)
+    self.assertEqual(out_1.shape.as_list(), [None, 10, 5])
+
+    ph_2 = keras.backend.placeholder(shape=(None, 1, 13))
+    out_2 = time_dist(ph_2)
+    self.assertEqual(out_2.shape.as_list(), [None, 1, 5])
+
+    ph_3 = keras.backend.placeholder(shape=(None, 1, 18))
+    with self.assertRaisesRegexp(ValueError, 'is incompatible with layer'):
+      time_dist(ph_3)
+
+  def test_TimeDistributed_with_invalid_dimensions(self):
+    time_dist = keras.layers.TimeDistributed(keras.layers.Dense(5))
+    ph = keras.backend.placeholder(shape=(None, 10))
+    with self.assertRaisesRegexp(
+        ValueError,
+        '`TimeDistributed` Layer should be passed an `input_shape `'):
+      time_dist(ph)
+
 
 class BidirectionalTest(test.TestCase):
 
-- 
GitLab


From 28ae789bd65c3b512d6ee90a238c2a37088b3dee Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 31 Jan 2019 14:17:46 -0800
Subject: [PATCH 1682/2345] Enhance the tensor tracer with the capability to
 save traced tensor values to memory.

PiperOrigin-RevId: 231852461
---
 .../contrib/tpu/python/tpu/tensor_tracer.py   | 494 +++++++++++++++---
 .../contrib/tpu/python/tpu/tpu_estimator.py   |  15 +-
 2 files changed, 424 insertions(+), 85 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py b/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py
index bf492e78a1..2c5ea65182 100644
--- a/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py
+++ b/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py
@@ -30,11 +30,15 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import control_flow_util
 from tensorflow.python.ops import gen_math_ops
+from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import logging_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import state_ops
+from tensorflow.python.ops import variable_scope
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import tf_logging as logging
 
@@ -67,14 +71,18 @@ _SECTION_NAME_CONFIG = 'configuration'
 _SECTION_NAME_REASON = 'reason'
 _SECTION_NAME_OP_LIST = 'op-list'
 _SECTION_NAME_TENSOR_LIST = 'tensor-list'
+_SECTION_NAME_CACHE_INDEX_MAP = 'cache-index-map'
 _SECTION_NAME_GRAPH = 'graph'
 _FIELD_NAME_VERSION = 'version:'
 _FIELD_NAME_DEVICE = 'device:'
 _FIELD_NAME_TRACE_MODE = 'trace-mode:'
 _FIELD_NAME_SUBMODE = 'submode:'
 _FIELD_NAME_NUM_REPLICAS = 'num-replicas:'
+_FIELD_NAME_NUM_REPLICAS_PER_HOST = 'num-replicas-per-host:'
+_FIELD_NAME_NUM_HOSTS = 'num-hosts:'
 _FIELD_NAME_NUM_OPS = 'number-of-ops:'
 _FIELD_NAME_NUM_TENSORS = 'number-of-tensors:'
+_FIELD_NAME_NUM_CACHE_INDICES = 'number-of-indices:'
 _FIELD_NAME_TOPOLOGICAL_SORT_SUCCEED = 'topological-sort-succeed:'
 _FLAGS_ENV_VAR = 'TENSOR_TRACER_FLAGS'
 _FLAG_SINGLE_QUOTE_PAT = re.compile(r"\s*--([^=]+)='([^']*)'")
@@ -83,14 +91,15 @@ _FLAG_NO_QUOTE_PAT = re.compile(r'\s*--([^=]+)=(\S*)')
 _FLAG_NO_EQUAL_PAT = re.compile(r'\s*--([^=]+)\s*')
 _FLAG_NAME_ENABLE = 'enable'
 _FLAG_NAME_TRACE_MODE = 'trace_mode'
+_FLAG_NAME_USE_COMPACT_TRACE = 'compact_trace'
 _FLAG_NAME_SUBMODE = 'submode'
 _FLAG_NAME_INCLUDE_LESS_INTERESTING_OPS = 'include_less_interesting_ops'
 _FLAG_NAME_EXCLUDED_OPNAMES = 'excluded_opnames'
 _FLAG_NAME_EXCLUDED_OPTYPES = 'excluded_optypes'
 _FLAG_NAME_INCLUDED_OPNAMES = 'included_opnames'
 _FLAG_NAME_INCLUDED_OPTYPES = 'included_optypes'
-_FLAG_NAME_TRACE_FILE = 'trace_file_path'
-_FLAG_NAME_REPORT_FILE = 'report_file_path'
+_FLAG_NAME_TRACE_DIR = 'trace_dir'
+_FLAG_NAME_REPORT_FILE = 'report_file'
 _FLAG_NAME_USE_TEST_UNDECLARED_OUTPUTS_DIR = 'use_test_undeclared_outputs_dir'
 _FLAG_NAME_OP_RANGE = 'op_range'
 _OP_RANGE_PAT = re.compile(r'(\d+):(\d+)')
@@ -98,7 +107,12 @@ _OUTPUT_STREAM_ESCAPE = 'file://'
 _TEST_UNDECLARED_OUTPUTS_DIR_ENV_VAR = 'TEST_UNDECLARED_OUTPUTS_DIR'
 _TENSOR_TRACER_COLLECTION = 'tensor_tracer_variables'
 _TENSOR_TRACER_CHECKPOINT = 'tensor_tracer_checkpoint'
-
+_TRACE_FILE_NAME = 'trace.all'
+_COMPACT_TRACE_FILE_PREFIX = 'compact_trace.'
+_COMPACT_TRACE_ENTRY_INIT_VALUE = -1.0
+_TENSOR_TRACER_STORAGE = 'tensor_tracer_storage'
+_TENSOR_VALUES_CACHE = 'tensor_values_cache'
+_REPLICA_ID_TAG = '#replica-id: '
 
 def tensor_tracepoint(tensor, checkpoint_name):
   """Adds a checkpoint with the given checkpoint name for the given tensor.
@@ -152,6 +166,68 @@ def keras_layer_tracepoint(layer, checkpoint_name):
   return layer
 
 
+def _trace_files_need_precreated(output_dir):
+  """Return True if trace files must be pre-created by users."""
+
+  if not output_dir.startswith('/'):
+    return False
+  if len(output_dir) < 5:
+    return False
+  if output_dir[2] != 'n':
+    return False
+  if output_dir[3] != 's':
+    return False
+  if output_dir[1] != 'c':
+    return False
+  if output_dir[4] != '/':
+    return False
+  return True
+
+
+def _get_tensor_values_cache(graph=None):
+  """Returns the variable that implements tensor-value caching."""
+
+  graph = graph or ops.get_default_graph()
+  collection = graph.get_collection(_TENSOR_TRACER_STORAGE)
+  if len(collection) == 1:
+    return collection[0]
+  elif not collection:
+    raise RuntimeError('%s has not been created'%_TENSOR_VALUES_CACHE)
+  else:
+    raise RuntimeError('Multiple %s created'%_TENSOR_VALUES_CACHE)
+  return None
+
+
+def _create_tensor_values_cache(graph, num_tensors):
+  """Creates a variable as the cache to store intermediate tensor values."""
+
+  graph = graph or ops.get_default_graph()
+  # Create in proper graph and base name_scope.
+  with graph.as_default() as g, g.name_scope(None):
+    return variable_scope.get_variable(
+        _TENSOR_VALUES_CACHE,
+        shape=[num_tensors],
+        dtype=dtypes.float32,
+        initializer=init_ops.constant_initializer(
+            _COMPACT_TRACE_ENTRY_INIT_VALUE),
+        trainable=False,
+        use_resource=True,
+        collections=[_TENSOR_TRACER_STORAGE, ops.GraphKeys.GLOBAL_VARIABLES])
+
+
+def _set_fetches(result_tensor, train_op):
+  """Sets the fetches from the result tensor and training op."""
+
+  fetches = []
+  if result_tensor is not None:
+    fetches.append(result_tensor)
+  if train_op is not None:
+    fetches.append(train_op)
+  if not fetches:
+    return None
+  return fetches
+
+
 class TensorTracer(object):
   """A software construct for tracing tensor values in a TF graph on TPU.
 
@@ -203,12 +279,14 @@ class TensorTracer(object):
   def validate_flag_names():
     """Validates if the TensorTrace flags passed are valid."""
     valid_flag_names = [_FLAG_NAME_ENABLE, _FLAG_NAME_TRACE_MODE,
+                        _FLAG_NAME_USE_COMPACT_TRACE,
                         _FLAG_NAME_SUBMODE,
                         _FLAG_NAME_EXCLUDED_OPNAMES,
                         _FLAG_NAME_EXCLUDED_OPTYPES,
                         _FLAG_NAME_INCLUDED_OPNAMES,
                         _FLAG_NAME_INCLUDED_OPTYPES,
-                        _FLAG_NAME_TRACE_FILE, _FLAG_NAME_REPORT_FILE,
+                        _FLAG_NAME_TRACE_DIR,
+                        _FLAG_NAME_REPORT_FILE,
                         _FLAG_NAME_USE_TEST_UNDECLARED_OUTPUTS_DIR,
                         _FLAG_NAME_INCLUDE_LESS_INTERESTING_OPS,
                         _FLAG_NAME_OP_RANGE]
@@ -338,6 +416,10 @@ class TensorTracer(object):
     return TensorTracer._is_flag_on(
         _FLAG_NAME_USE_TEST_UNDECLARED_OUTPUTS_DIR)
 
+  @staticmethod
+  def use_compact_trace():
+    return TensorTracer._is_flag_on(
+        _FLAG_NAME_USE_COMPACT_TRACE)
 
   @staticmethod
   def check_device_type(device_type):
@@ -535,7 +617,7 @@ class TensorTracer(object):
     TensorTracer.check_submode(self._submode)
     self._part_tensor_size = _TRACE_MODE_PART_TENSOR_SIZE
     self._instrument_records = {}
-    self._set_trace_file_path()
+    self._set_trace_dir()
     self._set_report_file()
     self._set_op_range()
     self._set_excluded_opnames()
@@ -543,17 +625,17 @@ class TensorTracer(object):
     self._set_included_opnames()
     self._set_included_optypes()
     self._num_replicas = None
+    self._num_replicas_per_host = None
+    self._num_hosts = None
     self._replica_id = None
 
-  def _add_replica_id_to_graph(self, num_replicas, result_tensor):
+  def _add_replica_id_to_graph(self, result_tensor):
     """Adds nodes for computing the replica ID to the graph."""
 
-    if not num_replicas:
+    if not self._num_replicas:
       self._replica_id = 'unknown'
       return result_tensor
 
-    self._num_replicas = num_replicas
-
     with ops.control_dependencies(None):
       # Uses None as dependency to run outside of TPU graph rewrites.
       self._replica_id = tpu_ops.tpu_replicated_input(
@@ -565,20 +647,15 @@ class TensorTracer(object):
       # the replica_id to ensure that replica_id will be added to the graph.
       return array_ops.identity(result_tensor)
 
-  def _set_trace_file_path(self):
-    """Sets the path of the output trace file."""
-
-    found, self._trace_file_path = TensorTracer.get_flag_value(
-        _FLAG_NAME_TRACE_FILE)
-    if found and self._trace_file_path \
+  def _set_trace_dir(self):
+    found, self._trace_dir = TensorTracer.get_flag_value(_FLAG_NAME_TRACE_DIR)
+    if found and self._trace_dir \
        and TensorTracer.use_test_undeclared_outputs_dir():
-      if os.path.isabs(self._trace_file_path):
-        raise ValueError('If use_test_undeclared_outputs_dir is set,'
-                         'trace_file_path cannot be an absolute path (%s)'
-                         %self._trace_file_path)
-      outputs_dir = os.environ.get(_TEST_UNDECLARED_OUTPUTS_DIR_ENV_VAR)
-      self._trace_file_path = os.path.join(outputs_dir,
-                                           self._trace_file_path)
+      raise ValueError('Cannot not use --%s and --%s at the same time'
+                       %(_FLAG_NAME_TRACE_DIR,
+                         _FLAG_NAME_USE_TEST_UNDECLARED_OUTPUTS_DIR))
+    if TensorTracer.use_test_undeclared_outputs_dir():
+      self._trace_dir = os.environ.get(_TEST_UNDECLARED_OUTPUTS_DIR_ENV_VAR)
 
   def _set_report_file(self):
     """Sets the path of the output report file."""
@@ -660,6 +737,25 @@ class TensorTracer(object):
         return True
     return False
 
+  def _use_tensor_values_cache(self):
+    """Returns True if immediate tensors should be first saved to a cache."""
+
+    if self._trace_mode not in set([_TRACE_MODE_NAN_INF,
+                                    _TRACE_MODE_NORM, _TRACE_MODE_MAX_ABS]):
+      return False
+    if self._trace_dir and _trace_files_need_precreated(self._trace_dir):
+      return True
+    if TensorTracer.use_compact_trace():
+      return True
+    return False
+
+  def _save_tensor_value_to_cache_op(self, graph, cache_idx, updates):
+    """Returns an Op that will save the given updates to an entry in the cache."""
+
+    cache = _get_tensor_values_cache(graph)
+    indices = constant_op.constant([cache_idx])
+    return state_ops.scatter_update(cache, indices, updates).op
+
   def _write_report(self, content):
     """Writes the given content to the report."""
 
@@ -678,6 +774,9 @@ class TensorTracer(object):
     self._write_report('%s %s\n'%(_FIELD_NAME_TRACE_MODE, self._trace_mode))
     self._write_report('%s %s\n'%(_FIELD_NAME_SUBMODE, self._submode))
     self._write_report('%s %s\n'%(_FIELD_NAME_NUM_REPLICAS, self._num_replicas))
+    self._write_report('%s %s\n'%(_FIELD_NAME_NUM_REPLICAS_PER_HOST,
+                                  self._num_replicas_per_host))
+    self._write_report('%s %s\n'%(_FIELD_NAME_NUM_HOSTS, self._num_hosts))
     self._write_report('%s %s\n'%(_MARKER_SECTION_END, _SECTION_NAME_CONFIG))
 
   def _write_reason_section(self):
@@ -724,6 +823,20 @@ class TensorTracer(object):
     self._write_report('%s %s\n'%(_MARKER_SECTION_END,
                                   _SECTION_NAME_TENSOR_LIST))
 
+  def _write_cache_index_map_section(self):
+    """Writes the mapping from cache index to tensor index to the report."""
+
+    self._write_report('%s %s\n'%(_MARKER_SECTION_BEGIN,
+                                  _SECTION_NAME_CACHE_INDEX_MAP))
+    self._write_report('%s %d\n'%(_FIELD_NAME_NUM_CACHE_INDICES,
+                                  len(self._cache_idx_to_tensor_idx)))
+    for cache_idx in range(0, len(self._cache_idx_to_tensor_idx)):
+      tensor_idx = self._cache_idx_to_tensor_idx[cache_idx]
+      line = '%d %d\n'%(cache_idx, tensor_idx)
+      self._write_report(line)
+    self._write_report('%s %s\n'%(_MARKER_SECTION_END,
+                                  _SECTION_NAME_CACHE_INDEX_MAP))
+
   def _write_graph_section(self, succeed, sorted_or_cycle):
     """Writes the graph section of the report."""
 
@@ -750,11 +863,14 @@ class TensorTracer(object):
       """Trace function for detecting any NaN/Inf in the tensor."""
 
       if tensor.dtype.is_floating:
-        output_tensor = math_ops.reduce_any(
+        mask = math_ops.reduce_any(
             gen_math_ops.logical_or(
                 gen_math_ops.is_nan(tensor), gen_math_ops.is_inf(tensor)))
+        output_tensor = control_flow_ops.cond(mask,
+                                              lambda: constant_op.constant(1.0),
+                                              lambda: constant_op.constant(0.0))
       else:
-        output_tensor = constant_op.constant(False)
+        output_tensor = constant_op.constant(0.0)
       # The shape has to be 1. Set it if it does not have the information.
       output_tensor = array_ops.reshape(output_tensor, [1])
       return output_tensor
@@ -826,8 +942,9 @@ class TensorTracer(object):
       else:
         msg = '"%s"'%tensor_name
 
-      if self._trace_file_path:
-        output_stream = _OUTPUT_STREAM_ESCAPE + self._trace_file_path
+      if self._trace_dir:
+        output_path = os.path.join(self._trace_dir, _TRACE_FILE_NAME)
+        output_stream = _OUTPUT_STREAM_ESCAPE + output_path
       else:
         output_stream = sys.stderr
       print_op = logging_ops.print_v2(msg, array_ops.shape(output_tensor),
@@ -954,6 +1071,7 @@ class TensorTracer(object):
 
   def _filter_execution_path_operations(self, operations, fetches):
     """Returns the set of ops in the execution path to compute given fetches."""
+
     # If no fetch provided, then return all operations.
     if fetches is None:
       return set(operations)
@@ -986,17 +1104,85 @@ class TensorTracer(object):
           traverse_stack.append(input_op)
     return execution_path_operations
 
-  def _pre_tracing(self, graph):
+  def _determine_traced_tensors(self, graph, fetches):
+    """Determines the tensors that will be traced."""
+
+    self._traced_tensorname_to_cache_idx_map = {}
+    self._cache_idx_to_tensor_idx = []
+    operations = graph.get_operations()
+    # Filter out the operations that won't be executed.
+    # if fetches=None, then ops_in_exec_path = set(operations)
+    ops_in_exec_path = self._filter_execution_path_operations(operations,
+                                                              fetches)
+    checkpoint_operations = self._get_checkpoints(graph)
+    for op_id, op in enumerate(operations):
+      if checkpoint_operations and op.name not in checkpoint_operations:
+        continue
+      user_included = self._is_user_included_op(op)
+      user_excluded = self._is_user_excluded_op(op)
+      in_exec_path = op in ops_in_exec_path
+      if self._skip_op(op_id, op, user_included, user_excluded, in_exec_path):
+        continue
+      for i in range(len(op.outputs)):
+        out_tensor = op.outputs[i]
+        if self._skip_tensor(op_id, out_tensor, user_included,
+                             user_excluded):
+          continue
+        tensor_name = out_tensor.name
+        if tensor_name in self._traced_tensorname_to_cache_idx_map:
+          raise ValueError(
+              'Tensor name %s should not be already in '
+              'traced_tensorname_to_cache_idx_map'%tensor_name)
+        if tensor_name not in self._tensorname_idx_map:
+          raise ValueError(
+              'Tensor name %s is not in the tensorname_idx_map'%tensor_name)
+        tensor_idx = self._tensorname_idx_map[tensor_name]
+        cache_idx = len(self._traced_tensorname_to_cache_idx_map)
+        self._traced_tensorname_to_cache_idx_map[tensor_name] = cache_idx
+        self._cache_idx_to_tensor_idx.append(tensor_idx)
+        if len(self._traced_tensorname_to_cache_idx_map) != len(
+            self._cache_idx_to_tensor_idx):
+          raise RuntimeError('len(self._traced_tensorname_to_cache_idx_map) != '
+                             'len(self._cache_idx_to_tensor_idx')
+
+  def _check_trace_files(self):
+    """Checks if any requirements for trace files are satisfied."""
+
+    if not self._trace_dir:
+      # traces will be written to stderr. No need to check trace files.
+      return
+    if _trace_files_need_precreated(self._trace_dir):
+      for replica_id in range(0, self._num_replicas):
+        trace_file_path = os.path.join(
+            self._trace_dir,
+            _COMPACT_TRACE_FILE_PREFIX) + '%d'%replica_id
+        if not gfile.Exists(trace_file_path):
+          raise RuntimeError(
+              '%s must be pre-created with the '
+              'appropriate properties.'%trace_file_path)
+    else:
+      if not gfile.Exists(self._trace_dir):
+        gfile.MkDir(self._trace_dir)
+        if not gfile.Exists(self._trace_dir):
+          raise RuntimeError('Failed to create %s'%self._trace_dir)
+
+  def _pre_tracing(self, graph, fetches):
     """Work needs to be done prior to TPU or CPU tracing."""
 
+    self._check_trace_files()
     operations = graph.get_operations()
     (opname_idx_map, tensor_list, self._tensorname_idx_map) = (
         TensorTracer._make_op_and_tensor_maps(operations))
     self._write_config_section()
     self._write_op_list_section(operations)
     self._write_tensor_list_section(tensor_list, opname_idx_map)
+    self._determine_traced_tensors(graph, fetches)
+    self._write_cache_index_map_section()
     # Does the topological sort before adding any nodes to the graph.
     (succeed, sorted_or_cycle) = TensorTracer.topological_sort(graph)
+    if self._use_tensor_values_cache():
+      _create_tensor_values_cache(graph,
+                                  len(self._cache_idx_to_tensor_idx))
     return (operations, succeed, sorted_or_cycle)
 
   def _post_tracing(self, succeed, sorted_or_cycle):
@@ -1027,15 +1213,118 @@ class TensorTracer(object):
                                   _TENSOR_TRACER_CHECKPOINT))
     return checkpoint_operations
 
-  def trace_tpu(self, graph, result_tensor, num_replicas=None, fetches=None):
+  def _generate_flush_cache_op(self, graph, start_replica, on_tpu):
+    """Generates an Op that will flush the cache to file.
+
+    Args:
+      graph: the graph of Ops
+      start_replica: the ID of the first replica being flushed by this Op.
+      on_tpu: if the graph is executed on TPU.
+
+    Returns:
+      The Op to flush the cache to file.
+    """
+    def _make_flush_fun(replica_id):
+      """Makes a function for flushing the cache for the given replica."""
+
+      def _fun():
+        """A function that flushes the cache to a file."""
+
+        def _flush_fun(cache):
+          """Flushes the cache to a file."""
+
+          if isinstance(replica_id, str):
+            replica_id_str = replica_id
+          else:
+            replica_id_str = '%d'%replica_id
+          output_path = os.path.join(self._trace_dir,
+                                     _COMPACT_TRACE_FILE_PREFIX) \
+                                     + replica_id_str
+          output_stream = _OUTPUT_STREAM_ESCAPE + output_path
+          new_step_line = _REPLICA_ID_TAG + replica_id_str
+          print_op = logging_ops.print_v2(
+              new_step_line, '\n',
+              cache, '\n',
+              summarize=-1,
+              output_stream=output_stream)
+          with ops.control_dependencies([print_op]):
+            return constant_op.constant(0).op
+
+        cache = _get_tensor_values_cache(graph)
+        if on_tpu:
+          flush_op = tpu.outside_compilation(_flush_fun, cache.value())
+        else:
+          flush_op = _flush_fun(cache.value())
+        with ops.control_dependencies([flush_op]):
+          reset_value = constant_op.constant(_COMPACT_TRACE_ENTRY_INIT_VALUE,
+                                             dtype=cache.dtype,
+                                             shape=cache.shape)
+          assign_op = state_ops.assign(cache, reset_value).op
+          with ops.control_dependencies([assign_op]):
+            return flush_op.outputs[0]
+
+      return _fun
+
+    def _f(replica_id):
+      return _make_flush_fun(replica_id)
+    def _eq(x):
+      return math_ops.equal(x, self._replica_id)
+    def _do_nothing():
+      return constant_op.constant(0)
+
+    return control_flow_ops.case({\
+                                  _eq(start_replica): _f(start_replica), \
+                                  _eq(start_replica+1): _f(start_replica+1), \
+                                  _eq(start_replica+2): _f(start_replica+2), \
+                                  _eq(start_replica+3): _f(start_replica+3), \
+                                  _eq(start_replica+4): _f(start_replica+4), \
+                                  _eq(start_replica+5): _f(start_replica+5), \
+                                  _eq(start_replica+6): _f(start_replica+6), \
+                                  _eq(start_replica+7): _f(start_replica+7), \
+    },
+                                 default=_do_nothing,
+                                 exclusive=True).op
+
+  def _flush_tensor_values_cache(self, graph, result_tensor, train_op, on_tpu):
+    """Flushes the intermediate tensor values in the graph to the cache.
+
+    Args:
+      graph: the graph of Ops
+      result_tensor: a result tensor of evaluating the graph.
+      train_op: the training op.
+      on_tpu: if the graph is executed on TPU.
+
+    Returns:
+      An identical copy of result tensor.
+    """
+
+    train_op_list = []
+    if train_op is not None:
+      train_op_list.append(train_op)
+    with ops.control_dependencies(train_op_list):
+      flush_cache_op_list = []
+      for host in range(self._num_hosts):
+        start_replica = host * 8
+        flush_op = self._generate_flush_cache_op(graph, start_replica, on_tpu)
+        flush_cache_op_list.append(flush_op)
+      with ops.control_dependencies(flush_cache_op_list):
+        return array_ops.identity(result_tensor)
+
+  def trace_tpu(self, graph,
+                result_tensor,
+                train_op,
+                num_replicas=None,
+                num_replicas_per_host=None,
+                num_hosts=None):
     """Traces the tensors generated by TPU Ops in a TF graph.
 
     Args:
       graph: the graph of Ops executed on the TPU.
       result_tensor: a result tensor of evaluating the graph.
+      train_op: the training op.
       num_replicas: number of replicas used on the TPU.
-      fetches: the list of fetches given to session.run, used to determine the
-      ops in execution path. If None, the whole graph will be traced.
+      num_replicas_per_host: number of replicas per TPU host.
+      num_hosts: total number of TPU hosts.
 
     Returns:
       A tuple (result_tensor_copy, tracing_ops), where:
@@ -1045,6 +1334,9 @@ class TensorTracer(object):
                      should pose control dependencies upon these
                      Ops so that they will be executed when the
                      graph is evaluated.
+
+    Raises:
+      RuntimeError: If num_replicas_per_host > 8.
     """
 
     def _cast_unsupported_dtypes(tensor):
@@ -1060,88 +1352,130 @@ class TensorTracer(object):
       return tensor
 
     self._device_type = _DEVICE_TYPE_TPU
+    self._num_replicas = num_replicas
+    self._num_replicas_per_host = num_replicas_per_host
+    self._num_hosts = num_hosts
+    if self._num_replicas_per_host > 8:
+      # Checks for the assumption in _generate_flush_cache_op().
+      raise RuntimeError(
+          'num_replicas_per_host (%d) is '
+          'greater than 8'%self._num_replicas_per_host)
+
     TensorTracer.check_device_type(self._device_type)
-    result_tensor_copy = self._add_replica_id_to_graph(num_replicas,
-                                                       result_tensor)
-    (operations, succeed, sorted_or_cycle) = self._pre_tracing(graph)
-    # Filter out the operations that won't be executed.
-    # if fetches=None, then ops_in_exec_path = set(operations)
-    ops_in_exec_path = self._filter_execution_path_operations(operations,
-                                                              fetches)
-    tracing_ops = []
-    checkpoint_operations = self._get_checkpoints(graph)
+    result_tensor_copy = self._add_replica_id_to_graph(result_tensor)
+    fetches = _set_fetches(result_tensor, train_op)
+    (operations, succeed, sorted_or_cycle) = self._pre_tracing(graph, fetches)
 
-    for op_id, op in enumerate(operations):
-      if checkpoint_operations and op.name not in checkpoint_operations:
-        continue
-      user_included = self._is_user_included_op(op)
-      user_excluded = self._is_user_excluded_op(op)
-      in_exec_path = op in ops_in_exec_path
-      if self._skip_op(op_id, op, user_included, user_excluded, in_exec_path):
-        continue
+    tracing_ops = []
+    for op in operations:
       for i in range(len(op.outputs)):
         out_tensor = op.outputs[i]
-        if self._skip_tensor(op_id, out_tensor, user_included,
-                             user_excluded):
+        tensor_name = out_tensor.name
+        if tensor_name not in self._traced_tensorname_to_cache_idx_map:
           continue
         # Create the list of consumers before calling _preprocess_traced_tensor.
         # Otherwise, adding control input below, will introduce a cycle in the
         # graph.
         consumers = out_tensor.consumers()
-        tensor_name = out_tensor.name
+        if not consumers:
+          continue
         processed_out_tensor = self._preprocess_traced_tensor(out_tensor)
         processed_out_tensor = _cast_unsupported_dtypes(processed_out_tensor)
-        trace_op = tpu.outside_compilation(
-            self._make_tensor_trace_fun(tensor_name), processed_out_tensor)
-        if consumers:
-          for consumer_op in consumers:
-            # pylint: disable=protected-access
-            consumer_op._add_control_input(trace_op)
-            # pylint: enable=protected-access
+        if self._use_tensor_values_cache():
+          cache_idx = self._traced_tensorname_to_cache_idx_map[tensor_name]
+          trace_op = self._save_tensor_value_to_cache_op(graph,
+                                                         cache_idx,
+                                                         processed_out_tensor)
         else:
-          # if there is no consumer, we will add the control dependence later
-          # when we add the control dependency to the output operations.
-          tracing_ops.append(trace_op)
+          trace_op = tpu.outside_compilation(
+              self._make_tensor_trace_fun(tensor_name), processed_out_tensor)
+        for consumer_op in consumers:
+          # pylint: disable=protected-access
+          consumer_op._add_control_input(trace_op)
+          # pylint: enable=protected-access
+    if self._use_tensor_values_cache():
+      result_tensor_final = self._flush_tensor_values_cache(graph,
+                                                            result_tensor_copy,
+                                                            train_op,
+                                                            on_tpu=True)
+    else:
+      result_tensor_final = result_tensor_copy
     self._post_tracing(succeed, sorted_or_cycle)
-    return (result_tensor_copy, tracing_ops)
+    return (result_tensor_final, tracing_ops)
+
+  def _generate_cpu_result(self, result_tensor, train_op, graph):
+    """Generates the final CPU result."""
+
+    if self._use_tensor_values_cache():
+      result_tensor_final = self._flush_tensor_values_cache(graph,
+                                                            result_tensor,
+                                                            train_op,
+                                                            on_tpu=False)
+    else:
+      result_tensor_final = array_ops.identity(result_tensor)
+    return result_tensor_final
 
-  def trace_cpu(self, graph):
+  def trace_cpu(self, graph, result_tensor, train_op):
     """Traces the tensors generated by CPU Ops in a TF graph.
 
     Args:
       graph: the graph of Ops executed on the CPU.
+      result_tensor: a result tensor of evaluating the graph.
+      train_op: the training op.
 
     Returns:
-      tracing_calls: a map from keys to trace calls.
+      A pair (final_result_tensor, tracing_calls) where:
+         final_result_tensor: an identical copy of result_tensor.
+         tracing_calls: a map from keys to trace calls.
                      A key is constructed from an Op's name.
                      A trace call consists of a function and a tensor (
                      the function will be invoked with the tensor).
     """
 
+    if result_tensor is None:
+      raise ValueError(
+          'The result_tensor passed to trace_cpu should not be None')
+
     self._device_type = _DEVICE_TYPE_CPU
     TensorTracer.check_device_type(self._device_type)
     self._num_replicas = 1
+    self._num_replicas_per_host = 1
+    self._num_hosts = 1
     self._replica_id = 0
-    (operations, succeed, sorted_or_cycle) = self._pre_tracing(graph)
-    tracing_calls = {}
-    checkpoint_operations = self._get_checkpoints(graph)
+    fetches = _set_fetches(result_tensor, train_op)
+    (operations, succeed, sorted_or_cycle) = self._pre_tracing(graph, fetches)
 
-    for op_id, op in enumerate(operations):
-      if checkpoint_operations and op.name not in checkpoint_operations:
-        continue
-      user_included = self._is_user_included_op(op)
-      user_excluded = self._is_user_excluded_op(op)
-      if self._skip_op(op_id, op, user_included, user_excluded):
-        continue
+    tracing_calls = {}
+    for op in operations:
       for i in range(len(op.outputs)):
         out_tensor = op.outputs[i]
-        if self._skip_tensor(op_id, out_tensor, user_included,
-                             user_excluded):
+        tensor_name = out_tensor.name
+        if tensor_name not in self._traced_tensorname_to_cache_idx_map:
+          continue
+        # Create the list of consumers before calling _preprocess_traced_tensor.
+        # Otherwise, adding control input below, will introduce a cycle in the
+        # graph.
+        consumers = out_tensor.consumers()
+        if not consumers:
           continue
         processed_out_tensor = self._preprocess_traced_tensor(out_tensor)
-        trace_fun = self._make_tensor_trace_fun(out_tensor.name)
-        trace_call = (trace_fun, [processed_out_tensor])
-        trace_call_key = 'tensor_tracing_cpu-%s:%d'%(op.name, i)
-        tracing_calls[trace_call_key] = trace_call
+        if self._use_tensor_values_cache():
+          cache_idx = self._traced_tensorname_to_cache_idx_map[tensor_name]
+          trace_op = self._save_tensor_value_to_cache_op(graph,
+                                                         cache_idx,
+                                                         processed_out_tensor)
+          for consumer_op in consumers:
+            # pylint: disable=protected-access
+            consumer_op._add_control_input(trace_op)
+            # pylint: enable=protected-access
+        else:
+          trace_fun = self._make_tensor_trace_fun(tensor_name)
+          trace_call = (trace_fun, [processed_out_tensor])
+          trace_call_key = 'tensor_tracing_cpu-%s:%d'%(op.name, i)
+          tracing_calls[trace_call_key] = trace_call
+
     self._post_tracing(succeed, sorted_or_cycle)
-    return tracing_calls
+    final_result_tensor = self._generate_cpu_result(result_tensor,
+                                                    train_op,
+                                                    graph)
+    return (final_result_tensor, tracing_calls)
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 1d06009b65..6a3ed9bb79 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -354,9 +354,12 @@ class TPUEstimatorSpec(model_fn_lib._TPUEstimatorSpec):  # pylint: disable=prote
     hooks = None
     if self.host_call is not None:
       hooks = [_OutfeedHostCallHook(host_call_ret['host_call'])]
-    if tensor_tracer.TensorTracer.is_enabled():
+    loss = self.loss
+    if tensor_tracer.TensorTracer.is_enabled() \
+       and self.train_op is not None:
       tt = tensor_tracer.TensorTracer()
-      tracing_calls = tt.trace_cpu(ops.get_default_graph())
+      (loss, tracing_calls) = tt.trace_cpu(ops.get_default_graph(),
+                                           loss, self.train_op)
       tracing_call_ret = _OutfeedHostCall.create_cpu_hostcall(tracing_calls)
       tracing_functions = tracing_call_ret.values()
       if tracing_functions:
@@ -369,7 +372,7 @@ class TPUEstimatorSpec(model_fn_lib._TPUEstimatorSpec):  # pylint: disable=prote
     return model_fn_lib.EstimatorSpec(
         mode=self.mode,
         predictions=self.predictions,
-        loss=self.loss,
+        loss=loss,
         train_op=self.train_op,
         eval_metric_ops=eval_metric_ops,
         export_outputs=self.export_outputs,
@@ -1450,9 +1453,11 @@ class _ModelFnWrapper(object):
       tracing_ops = []
       if tensor_tracer.TensorTracer.is_enabled():
         tt = tensor_tracer.TensorTracer()
-        loss, tracing_ops = tt.trace_tpu(ops.get_default_graph(), loss,
+        loss, tracing_ops = tt.trace_tpu(ops.get_default_graph(),
+                                         loss, train_op,
                                          self._ctx.num_replicas,
-                                         fetches=[loss, train_op])
+                                         self._ctx.num_of_replicas_per_host,
+                                         self._ctx.num_hosts)
 
       if self._ctx.embedding_config is None:
         apply_sparse_grads = []
-- 
GitLab


From 5e370f7a36e943bbae1a61b79b24eecab1c357bf Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Thu, 31 Jan 2019 14:18:13 -0800
Subject: [PATCH 1683/2345] If context is None, make a call to context(), to
 force create it.

enable_eager_execution() guarantees the existence of the context, but in v2, we
will not have that explicit call.

Its faster to access the context directly as opposed to making a function
call, and I don't want to add to the overhead of every single op.

PiperOrigin-RevId: 231852561
---
 tensorflow/python/framework/python_op_gen.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/framework/python_op_gen.cc b/tensorflow/python/framework/python_op_gen.cc
index d460168631..ba622d1203 100644
--- a/tensorflow/python/framework/python_op_gen.cc
+++ b/tensorflow/python/framework/python_op_gen.cc
@@ -649,7 +649,7 @@ bool GenEagerPythonOp::AddEagerFastPathAndGraphCode(
   strings::StrAppend(&result_, "  \"\"\"\n");
 
   strings::StrAppend(&result_,
-                     "  _ctx = _context._context\n"
+                     "  _ctx = _context._context or _context.context()\n"
                      "  if _ctx is not None and _ctx._eager_context.is_eager:",
                      "\n");
   if (eager_not_allowed_error.empty()) {
-- 
GitLab


From 80e9f0c13141c96833d9ed5d58dabf03f523d88d Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Thu, 31 Jan 2019 14:18:16 -0800
Subject: [PATCH 1684/2345] Remove default values for deprecated arguments to
 pacify deprecated_argument_lookup

PiperOrigin-RevId: 231852576
---
 tensorflow/python/ops/nn_impl.py                   | 12 +++++++++---
 tensorflow/python/ops/nn_ops.py                    |  6 +++---
 tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt |  8 ++++----
 3 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/tensorflow/python/ops/nn_impl.py b/tensorflow/python/ops/nn_impl.py
index 1853323cad..dc252c7e7f 100644
--- a/tensorflow/python/ops/nn_impl.py
+++ b/tensorflow/python/ops/nn_impl.py
@@ -757,7 +757,7 @@ def separable_conv2d_v2(
 
 
 @tf_export(v1=["nn.sufficient_statistics"])
-def sufficient_statistics(x, axes, shift=None, keep_dims=False, name=None,
+def sufficient_statistics(x, axes, shift=None, keep_dims=None, name=None,
                           keepdims=None):
   """Calculate the sufficient statistics for the mean and variance of `x`.
 
@@ -786,6 +786,8 @@ def sufficient_statistics(x, axes, shift=None, keep_dims=False, name=None,
   axes = list(set(axes))
   keep_dims = deprecated_argument_lookup(
       "keepdims", keepdims, "keep_dims", keep_dims)
+  if keep_dims is None:
+    keep_dims = False
   with ops.name_scope(name, "sufficient_statistics", [x, shift]):
     x = ops.convert_to_tensor(x, name="x")
     x_shape = x.get_shape()
@@ -877,7 +879,7 @@ def moments(
     axes,
     shift=None,  # pylint: disable=unused-argument
     name=None,
-    keep_dims=False,
+    keep_dims=None,
     keepdims=None):
   """Calculate the mean and variance of `x`.
 
@@ -908,6 +910,8 @@ def moments(
   """
   keep_dims = deprecated_argument_lookup(
       "keepdims", keepdims, "keep_dims", keep_dims)
+  if keep_dims is None:
+    keep_dims = False
   with ops.name_scope(name, "moments", [x, axes]):
     # The dynamic range of fp16 is too limited to support the collection of
     # sufficient statistics. As a workaround we simply perform the operations
@@ -971,7 +975,7 @@ def moments_v2(
 
 
 @tf_export(v1=["nn.weighted_moments"])
-def weighted_moments(x, axes, frequency_weights, name=None, keep_dims=False,
+def weighted_moments(x, axes, frequency_weights, name=None, keep_dims=None,
                      keepdims=None):
   """Returns the frequency-weighted mean and variance of `x`.
 
@@ -990,6 +994,8 @@ def weighted_moments(x, axes, frequency_weights, name=None, keep_dims=False,
   """
   keep_dims = deprecated_argument_lookup(
       "keepdims", keepdims, "keep_dims", keep_dims)
+  if keep_dims is None:
+    keep_dims = False
   with ops.name_scope(name, "weighted_moments", [x, frequency_weights, axes]):
     x = ops.convert_to_tensor(x, name="x")
     frequency_weights = ops.convert_to_tensor(
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index fd1173bc3a..51c1f15bac 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -3211,13 +3211,13 @@ def max_pool_with_argmax_v1(input,  # pylint: disable=missing-docstring,invalid-
                             strides,
                             padding,
                             data_format="NHWC",
-                            Targmax=dtypes.int64,  # pylint: disable=invalid-name
+                            Targmax=None,  # pylint: disable=invalid-name
                             name=None,
                             output_dtype=None):
   Targmax = deprecated_argument_lookup(
       "output_dtype", output_dtype, "Targmax", Targmax)
-  if output_dtype is not None:
-    Targmax = output_dtype
+  if Targmax is None:
+    Targmax = dtypes.int64
   return gen_nn_ops.max_pool_with_argmax(
       input, ksize, strides, padding, data_format, Targmax, name)
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
index 4a55224248..3320a0eecd 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
@@ -242,11 +242,11 @@ tf_module {
   }
   member_method {
     name: "max_pool_with_argmax"
-    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\', \'Targmax\', \'name\', \'output_dtype\'], varargs=None, keywords=None, defaults=[\'NHWC\', \"<dtype: \'int64\'>\", \'None\', \'None\'], "
+    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\', \'Targmax\', \'name\', \'output_dtype\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "moments"
-    argspec: "args=[\'x\', \'axes\', \'shift\', \'name\', \'keep_dims\', \'keepdims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'None\'], "
+    argspec: "args=[\'x\', \'axes\', \'shift\', \'name\', \'keep_dims\', \'keepdims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "nce_loss"
@@ -362,7 +362,7 @@ tf_module {
   }
   member_method {
     name: "sufficient_statistics"
-    argspec: "args=[\'x\', \'axes\', \'shift\', \'keep_dims\', \'name\', \'keepdims\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'None\'], "
+    argspec: "args=[\'x\', \'axes\', \'shift\', \'keep_dims\', \'name\', \'keepdims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "tanh"
@@ -382,7 +382,7 @@ tf_module {
   }
   member_method {
     name: "weighted_moments"
-    argspec: "args=[\'x\', \'axes\', \'frequency_weights\', \'name\', \'keep_dims\', \'keepdims\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'x\', \'axes\', \'frequency_weights\', \'name\', \'keep_dims\', \'keepdims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "with_space_to_batch"
-- 
GitLab


From c870e70e5c6fd166217112c4bf8e0cae46a8ae97 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Thu, 31 Jan 2019 14:20:40 -0800
Subject: [PATCH 1685/2345] Remove a checkpointable dependency on __delattr__

I ran into this when writing an error message for ".signatures" being a reserved attribute in tf.saved_model.save. I want to write "'del obj.signatures' if you don't care", but that didn't actually remove the dependency.

This isn't any more unsafe than overwriting an attribute with __setattr__, which we already support. It's a somewhat weird thing to do in terms of checkpointing, though.

PiperOrigin-RevId: 231853096
---
 .../python/training/checkpointable/tracking.py    | 11 +++++++++++
 .../training/checkpointable/tracking_test.py      | 15 +++++++++++++++
 2 files changed, 26 insertions(+)

diff --git a/tensorflow/python/training/checkpointable/tracking.py b/tensorflow/python/training/checkpointable/tracking.py
index 04fd5547e1..f1ef5cca49 100644
--- a/tensorflow/python/training/checkpointable/tracking.py
+++ b/tensorflow/python/training/checkpointable/tracking.py
@@ -76,6 +76,17 @@ class AutoCheckpointable(base.Checkpointable):
           checkpointable=self, value=value, name=name)
     super(AutoCheckpointable, self).__setattr__(name, value)
 
+  def __delattr__(self, name):
+    self._maybe_initialize_checkpointable()
+    if name in self._unconditional_dependency_names:
+      del self._unconditional_dependency_names[name]
+      for index, (dep_name, _) in enumerate(
+          self._unconditional_checkpoint_dependencies):
+        if dep_name == name:
+          del self._unconditional_checkpoint_dependencies[index]
+          break
+    super(AutoCheckpointable, self).__delattr__(name)
+
   def _no_dependency(self, value):
     """Override to allow CheckpointableBase to disable dependency tracking."""
     return data_structures.NoDependency(value)
diff --git a/tensorflow/python/training/checkpointable/tracking_test.py b/tensorflow/python/training/checkpointable/tracking_test.py
index eb70919b9c..87c6603ca2 100644
--- a/tensorflow/python/training/checkpointable/tracking_test.py
+++ b/tensorflow/python/training/checkpointable/tracking_test.py
@@ -71,6 +71,21 @@ class InterfaceTests(test.TestCase):
     nodeps = NoDependencyModel()
     self.assertEqual([nodeps], util.list_objects(nodeps))
 
+  def testRemoveDependency(self):
+    root = tracking.AutoCheckpointable()
+    root.a = tracking.AutoCheckpointable()
+    self.assertEqual(1, len(root._checkpoint_dependencies))
+    self.assertEqual(1, len(root._unconditional_checkpoint_dependencies))
+    self.assertIs(root.a, root._checkpoint_dependencies[0].ref)
+    del root.a
+    self.assertFalse(hasattr(root, "a"))
+    self.assertEqual(0, len(root._checkpoint_dependencies))
+    self.assertEqual(0, len(root._unconditional_checkpoint_dependencies))
+    root.a = tracking.AutoCheckpointable()
+    self.assertEqual(1, len(root._checkpoint_dependencies))
+    self.assertEqual(1, len(root._unconditional_checkpoint_dependencies))
+    self.assertIs(root.a, root._checkpoint_dependencies[0].ref)
+
   def testListBasic(self):
     a = tracking.AutoCheckpointable()
     b = tracking.AutoCheckpointable()
-- 
GitLab


From 1850f415f70d398c28081053b523a67a636463e2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 31 Jan 2019 14:23:52 -0800
Subject: [PATCH 1686/2345] open-source TraceMe. TraceMe permits user-specified
 (CPU) tracing activities.

PiperOrigin-RevId: 231853700
---
 tensorflow/core/profiler/internal/BUILD |   2 +
 tensorflow/core/profiler/lib/BUILD      |  14 ++
 tensorflow/core/profiler/lib/traceme.cc |  48 ++++++
 tensorflow/core/profiler/lib/traceme.h  | 192 ++++++++++++++++++++++++
 4 files changed, 256 insertions(+)
 create mode 100644 tensorflow/core/profiler/lib/traceme.cc
 create mode 100644 tensorflow/core/profiler/lib/traceme.h

diff --git a/tensorflow/core/profiler/internal/BUILD b/tensorflow/core/profiler/internal/BUILD
index 2456adadaa..bae2148bb0 100644
--- a/tensorflow/core/profiler/internal/BUILD
+++ b/tensorflow/core/profiler/internal/BUILD
@@ -373,7 +373,9 @@ tf_cuda_library(
     srcs = ["traceme_recorder.cc"],
     hdrs = ["traceme_recorder.h"],
     visibility = [
+        "//learning/brain/runtime:__pkg__",  # xprof_bridge
         "//perftools/accelerators/xprof/xprofilez:__pkg__",  # alias xprof::TraceMeRecorder
+        "//tensorflow/core/profiler/lib:__pkg__",  # traceme
     ],
     deps = [
         "//tensorflow/core:lib",
diff --git a/tensorflow/core/profiler/lib/BUILD b/tensorflow/core/profiler/lib/BUILD
index 10d5ee645e..697076c56a 100644
--- a/tensorflow/core/profiler/lib/BUILD
+++ b/tensorflow/core/profiler/lib/BUILD
@@ -43,3 +43,17 @@ tf_cuda_library(
         ],
     }),
 )
+
+tf_cuda_library(
+    name = "traceme",
+    srcs = ["traceme.cc"],
+    hdrs = ["traceme.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tensorflow/core:lib",
+        "//tensorflow/core/profiler/internal:traceme_recorder",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/time",
+        "@com_google_absl//absl/types:optional",
+    ],
+)
diff --git a/tensorflow/core/profiler/lib/traceme.cc b/tensorflow/core/profiler/lib/traceme.cc
new file mode 100644
index 0000000000..ae1c9215e2
--- /dev/null
+++ b/tensorflow/core/profiler/lib/traceme.cc
@@ -0,0 +1,48 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/profiler/lib/traceme.h"
+
+namespace tensorflow {
+namespace profiler {
+
+// Activity IDs: To avoid contention over a counter, the top 32 bits identify
+// the originating thread, the bottom 32 bits name the event within a thread.
+// IDs may be reused after 4 billion events on one thread, or 4 billion threads.
+static std::atomic<uint32> thread_counter(1);  // avoid kUntracedActivity
+uint64 NewActivityId() {
+  const thread_local static uint32 thread_id = thread_counter.fetch_add(1);
+  thread_local static uint32 per_thread_activity_id = 0;
+  return static_cast<uint64>(thread_id) << 32 | per_thread_activity_id++;
+}
+
+/* static */ uint64 TraceMe::ActivityStartImpl(
+    absl::string_view activity_name) {
+  uint64 activity_id = NewActivityId();
+  TraceMeRecorder::Record(
+      {activity_id, string(activity_name),
+       /*start_time=*/static_cast<uint64>(absl::GetCurrentTimeNanos()),
+       /*end_time=*/0});
+  return activity_id;
+}
+
+/* static */ void TraceMe::ActivityEndImpl(uint64 activity_id) {
+  TraceMeRecorder::Record(
+      {activity_id, /*name=*/"", /*start_time=*/0,
+       /*end_time=*/static_cast<uint64>(absl::GetCurrentTimeNanos())});
+}
+
+}  // namespace profiler
+}  // namespace tensorflow
diff --git a/tensorflow/core/profiler/lib/traceme.h b/tensorflow/core/profiler/lib/traceme.h
new file mode 100644
index 0000000000..a6bff3adc5
--- /dev/null
+++ b/tensorflow/core/profiler/lib/traceme.h
@@ -0,0 +1,192 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_PROFILER_LIB_TRACEME_H_
+#define TENSORFLOW_CORE_PROFILER_LIB_TRACEME_H_
+
+#include <string>
+
+#include "absl/strings/string_view.h"
+#include "absl/types/optional.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/types.h"
+#include "tensorflow/core/profiler/internal/traceme_recorder.h"
+
+namespace tensorflow {
+namespace profiler {
+
+// This is specifically used in xprof_bridge for instrumenting Tensorflow ops.
+// Takes input as whether a TF op is expensive or not and returns the TraceMe
+// level to be assigned to trace that particular op. Assigns level 2 for
+// expensive ops (these are high-level details and shown by default in xprof
+// UI). Assigns level 3 for cheap ops (low-level details not shown by default).
+inline int GetTFTraceMeLevel(bool is_expensive) { return is_expensive ? 2 : 3; }
+
+// This class permits user-specified (CPU) tracing activities. A trace activity
+// is started when an object of this class is created and stopped when the
+// object is destroyed.
+//
+// CPU tracing can be useful when trying to understand what parts of GPU
+// computation (e.g., kernels and memcpy) correspond to higher level activities
+// in the overall program. For instance, a collection of kernels maybe
+// performing one "step" of a program that is better visualized together than
+// interspersed with kernels from other "steps". Therefore, a TraceMe object
+// can be created at each "step".
+//
+// Two APIs are provided:
+//   (1) Scoped object: a TraceMe object starts tracing on construction, and
+//       stops tracing when it goes out of scope.
+//          {
+//            TraceMe trace("step");
+//            ... do some work ...
+//          }
+//       TraceMe objects can be members of a class, or allocated on the heap.
+//   (2) Static methods: ActivityStart and ActivityEnd may be called in pairs.
+//          auto id = ActivityStart("step");
+//          ... do some work ...
+//          ActivityEnd(id);
+class TraceMe {
+ public:
+  // Constructor that traces a user-defined activity labeled with activity_name
+  // in the UI. Level defines the trace priority, used for filtering TraceMe
+  // events. By default, traces with TraceMe level <= 2 are recorded. Levels:
+  // - Must be a positive integer.
+  // - Level 1 is the default and used only for user instrumentation.
+  // - Level 2 is used by xprof for instrumenting high level program execution
+  //   details (expensive TF ops, XLA ops, etc).
+  // - Level 3 is also used by xprof to instrument more verbose (low-level)
+  //   program execution details (cheap TF ops, etc).
+  // Users are welcome to use level >= 2 in their code, if they wish to filter
+  // out their host traces based on verbosity.
+  explicit TraceMe(absl::string_view activity_name, int level = 1) {
+    DCHECK_GE(level, 1);
+    if (TraceMeRecorder::Active(level)) {
+      new (&no_init_.name) string(activity_name);
+      start_time_ = absl::GetCurrentTimeNanos();
+    } else {
+      start_time_ = kUntracedActivity;
+    }
+  }
+
+  // string&& constructor to prevent an unnecessary string copy, e.g. when a
+  // TraceMe is constructed based on the result of a StrCat operation.
+  // Note: We can't take the string by value because a) it would make the
+  // overloads ambiguous, and b) we want lvalue strings to use the string_view
+  // constructor so we avoid copying them when tracing is disabled.
+  explicit TraceMe(string &&activity_name, int level = 1) {
+    DCHECK_GE(level, 1);
+    if (TraceMeRecorder::Active(level)) {
+      new (&no_init_.name) string(std::move(activity_name));
+      start_time_ = absl::GetCurrentTimeNanos();
+    } else {
+      start_time_ = kUntracedActivity;
+    }
+  }
+
+  // Do not allow passing strings by reference or value since the caller
+  // may unintentionally maintain ownership of the activity_name.
+  // Explicitly std::move the activity_name or wrap it in a string_view if
+  // you really wish to maintain ownership.
+  explicit TraceMe(const string &activity_name, int level = 1) = delete;
+
+  // This overload is necessary to make TraceMe's with string literals work.
+  // Otherwise, the string&& and the string_view constructor would be equally
+  // good overload candidates.
+  explicit TraceMe(const char *raw, int level = 1)
+      : TraceMe(absl::string_view(raw), level) {}
+
+  // This overload only generates the activity name if tracing is enabled.
+  // Useful for avoiding things like string concatenation when tracing is
+  // disabled. The |name_generator| may be a lambda or functor that returns a
+  // type that the string() constructor can take.
+  // name_generator is templated, rather than a std::function to avoid
+  // allocations std::function might make even if never called.
+  // Usage: xprof::TraceMe([&]{ return StrCat(prefix, ":", postfix); });
+  template <typename NameGeneratorT>
+  explicit TraceMe(NameGeneratorT name_generator, int level = 1) {
+    DCHECK_GE(level, 1);
+    if (TraceMeRecorder::Active(level)) {
+      new (&no_init_.name) string(name_generator());
+      start_time_ = absl::GetCurrentTimeNanos();
+    } else {
+      start_time_ = kUntracedActivity;
+    }
+  }
+
+  ~TraceMe() {
+    // We do not need to check the trace level again here.
+    // - If tracing wasn't active to start with, we have kUntracedActivity.
+    // - If tracing was active and was stopped, we have
+    //   TraceMeRecorder::Active().
+    // - If tracing was active and was restarted at a lower level, we may
+    //   spuriously record the event. This is extremely rare, and acceptable as
+    //   event will be discarded when its start timestamp fall outside of the
+    //   start/stop session timestamp (recorded in XprofResponse).
+    if (start_time_ != kUntracedActivity) {
+      if (TraceMeRecorder::Active()) {
+        TraceMeRecorder::Record(
+            {kCompleteActivity, std::move(no_init_.name), start_time_,
+             static_cast<uint64>(absl::GetCurrentTimeNanos())});
+      }
+      no_init_.name.~string();
+    }
+  }
+
+  // TraceMe is not movable or copyable.
+  TraceMe(const TraceMe &) = delete;
+  TraceMe &operator=(const TraceMe &) = delete;
+
+  // Static API, for use when scoped objects are inconvenient.
+
+  // Record the start time of an activity.
+  // Returns the activity ID, which is used to stop the activity.
+  static uint64 ActivityStart(absl::string_view name, int level = 1) {
+    return TraceMeRecorder::Active(level) ? ActivityStartImpl(name)
+                                          : kUntracedActivity;
+  }
+
+  // Record the end time of an activity started by ActivityStart().
+  static void ActivityEnd(uint64 activity_id) {
+    // We don't check the level again (see ~TraceMe()).
+    if (activity_id != kUntracedActivity) {
+      if (TraceMeRecorder::Active()) {
+        ActivityEndImpl(activity_id);
+      }
+    }
+  }
+
+ private:
+  // Activity ID or start time used when tracing is disabled.
+  constexpr static uint64 kUntracedActivity = 0;
+  // Activity ID used as a placeholder when both start and end are present.
+  constexpr static uint64 kCompleteActivity = 1;
+
+  static uint64 ActivityStartImpl(absl::string_view activity_name);
+  static void ActivityEndImpl(uint64 activity_id);
+
+  // Wrap the name into a union so that we can avoid the cost of string
+  // initialization when tracing is disabled.
+  union NoInit {
+    NoInit() {}
+    ~NoInit() {}
+    string name;
+  } no_init_;
+
+  uint64 start_time_;
+};
+
+}  // namespace profiler
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_PROFILER_LIB_TRACEME_H_
-- 
GitLab


From 42c4f4ab6b53bce8639c203d7839d27eac11bd2f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 31 Jan 2019 14:27:41 -0800
Subject: [PATCH 1687/2345] parallel_for: Add converter for tf.reduce_mean.

PiperOrigin-RevId: 231854413
---
 tensorflow/python/ops/parallel_for/math_test.py | 2 +-
 tensorflow/python/ops/parallel_for/pfor.py      | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/parallel_for/math_test.py b/tensorflow/python/ops/parallel_for/math_test.py
index db88f4fe03..98c24e6caa 100644
--- a/tensorflow/python/ops/parallel_for/math_test.py
+++ b/tensorflow/python/ops/parallel_for/math_test.py
@@ -278,7 +278,7 @@ class MathTest(PForTestCase):
     x = random_ops.random_uniform([2, 3, 4, 5])
     for op in [
         math_ops.reduce_sum, math_ops.reduce_prod, math_ops.reduce_max,
-        math_ops.reduce_min
+        math_ops.reduce_min, math_ops.reduce_mean,
     ]:
       for axis in ([1], None, [0, 2]):
         for keepdims in (True, False):
diff --git a/tensorflow/python/ops/parallel_for/pfor.py b/tensorflow/python/ops/parallel_for/pfor.py
index b0f6a6a443..74cb444aad 100644
--- a/tensorflow/python/ops/parallel_for/pfor.py
+++ b/tensorflow/python/ops/parallel_for/pfor.py
@@ -1876,6 +1876,7 @@ def _convert_batch_mat_mul(pfor_input):
 @RegisterPForWithArgs("Prod", math_ops.reduce_prod)
 @RegisterPForWithArgs("Max", math_ops.reduce_max)
 @RegisterPForWithArgs("Min", math_ops.reduce_min)
+@RegisterPForWithArgs("Mean", math_ops.reduce_mean)
 def _convert_reduction(pfor_input, _, op_func):
   t = pfor_input.stacked_input(0)
   indices = pfor_input.unstacked_input(1)
-- 
GitLab


From a7d9897dea3ce7addc69bde78d3916e9a9ae0930 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 31 Jan 2019 14:29:55 -0800
Subject: [PATCH 1688/2345] Fix large tests size to medium

PiperOrigin-RevId: 231854858
---
 .../boosted_trees/estimator_batch/BUILD        |  3 ++-
 tensorflow/contrib/distributions/BUILD         |  4 ++--
 tensorflow/contrib/factorization/BUILD         |  7 ++++---
 tensorflow/contrib/slim/python/slim/nets/BUILD | 18 ++++++++++--------
 .../contrib/timeseries/python/timeseries/BUILD |  4 ++--
 tensorflow/contrib/training/BUILD              |  4 ++--
 6 files changed, 22 insertions(+), 18 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/BUILD b/tensorflow/contrib/boosted_trees/estimator_batch/BUILD
index d3b23d949e..64e4c4560b 100644
--- a/tensorflow/contrib/boosted_trees/estimator_batch/BUILD
+++ b/tensorflow/contrib/boosted_trees/estimator_batch/BUILD
@@ -193,8 +193,9 @@ py_test(
 
 py_test(
     name = "estimator_test",
-    size = "large",
+    size = "medium",
     srcs = ["estimator_test.py"],
+    shard_count = 4,
     srcs_version = "PY2AND3",
     tags = [
         "no_gpu",
diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD
index 3079175015..c2300286d3 100644
--- a/tensorflow/contrib/distributions/BUILD
+++ b/tensorflow/contrib/distributions/BUILD
@@ -822,7 +822,7 @@ cuda_py_test(
 
 cuda_py_test(
     name = "affine_test",
-    size = "large",
+    size = "medium",
     srcs = ["python/kernel_tests/bijectors/affine_test.py"],
     additional_deps = [
         ":bijectors_py",
@@ -837,7 +837,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform_test",
     ],
-    shard_count = 5,
+    shard_count = 10,
     tags = ["noasan"],  # times out b/63678675
 )
 
diff --git a/tensorflow/contrib/factorization/BUILD b/tensorflow/contrib/factorization/BUILD
index cb86efb8da..48a6ef4dca 100644
--- a/tensorflow/contrib/factorization/BUILD
+++ b/tensorflow/contrib/factorization/BUILD
@@ -109,7 +109,7 @@ tf_gen_op_wrapper_py(
 # Ops tests
 tf_py_test(
     name = "gmm_test",
-    size = "large",
+    size = "medium",
     srcs = [
         "python/ops/gmm_test.py",
     ],
@@ -130,6 +130,7 @@ tf_py_test(
         "//tensorflow/python:random_seed",
         "//tensorflow/python:training",
     ],
+    shard_count = 4,
     tags = [
         "no_pip",  # b/38283730
         "notsan",  # Flaky: b/30756419
@@ -227,7 +228,7 @@ py_test(
 
 tf_py_test(
     name = "wals_test",
-    size = "large",
+    size = "medium",
     srcs = ["python/ops/wals_test.py"],
     additional_deps = [
         ":factorization_py",
@@ -250,8 +251,8 @@ tf_py_test(
         "//tensorflow/python:training",
         "//tensorflow/python:variables",
     ],
+    shard_count = 4,
     tags = [
-        "manual",
         "noasan",  # times out b/63678675
         "nomsan",
     ],
diff --git a/tensorflow/contrib/slim/python/slim/nets/BUILD b/tensorflow/contrib/slim/python/slim/nets/BUILD
index 8bbdf96384..e9595d1b32 100644
--- a/tensorflow/contrib/slim/python/slim/nets/BUILD
+++ b/tensorflow/contrib/slim/python/slim/nets/BUILD
@@ -115,9 +115,9 @@ py_library(
 
 py_test(
     name = "inception_v1_test",
-    size = "large",
+    size = "medium",
     srcs = ["inception_v1_test.py"],
-    shard_count = 3,
+    shard_count = 8,
     srcs_version = "PY2AND3",
     deps = [
         ":inception_v1",
@@ -135,9 +135,9 @@ py_test(
 
 py_test(
     name = "inception_v2_test",
-    size = "large",
+    size = "medium",
     srcs = ["inception_v2_test.py"],
-    shard_count = 3,
+    shard_count = 8,
     srcs_version = "PY2AND3",
     deps = [
         ":inception_v2",
@@ -155,9 +155,9 @@ py_test(
 
 py_test(
     name = "inception_v3_test",
-    size = "large",
+    size = "medium",
     srcs = ["inception_v3_test.py"],
-    shard_count = 3,
+    shard_count = 8,
     srcs_version = "PY2AND3",
     deps = [
         ":inception_v3",
@@ -233,8 +233,9 @@ py_library(
 
 py_test(
     name = "resnet_v1_test",
-    size = "large",
+    size = "medium",
     srcs = ["resnet_v1_test.py"],
+    shard_count = 4,
     srcs_version = "PY2AND3",
     deps = [
         ":resnet_utils",
@@ -268,8 +269,9 @@ py_library(
 
 py_test(
     name = "resnet_v2_test",
-    size = "large",
+    size = "medium",
     srcs = ["resnet_v2_test.py"],
+    shard_count = 4,
     srcs_version = "PY2AND3",
     deps = [
         ":resnet_utils",
diff --git a/tensorflow/contrib/timeseries/python/timeseries/BUILD b/tensorflow/contrib/timeseries/python/timeseries/BUILD
index 2a22295197..d1be31ddc7 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/BUILD
+++ b/tensorflow/contrib/timeseries/python/timeseries/BUILD
@@ -155,11 +155,11 @@ py_library(
 
 py_test(
     name = "head_test",
-    size = "large",
+    size = "medium",
     srcs = [
         "head_test.py",
     ],
-    shard_count = 4,
+    shard_count = 10,
     srcs_version = "PY2AND3",
     tags = ["no_pip_gpu"],  # b/63391119
     deps = [
diff --git a/tensorflow/contrib/training/BUILD b/tensorflow/contrib/training/BUILD
index f6427ae05a..5bc4c3b88e 100644
--- a/tensorflow/contrib/training/BUILD
+++ b/tensorflow/contrib/training/BUILD
@@ -264,9 +264,9 @@ py_test(
 
 py_test(
     name = "training_test",
-    size = "large",
+    size = "medium",
     srcs = ["python/training/training_test.py"],
-    shard_count = 3,
+    shard_count = 8,
     srcs_version = "PY2AND3",
     tags = ["notsan"],
     deps = [
-- 
GitLab


From e61e004a597826024ba2b62d7b6db7631561e105 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 31 Jan 2019 14:52:29 -0800
Subject: [PATCH 1689/2345] Add a wrapper to load .so for TPUPartitionedCallOp
 so that it works in open source.

PiperOrigin-RevId: 231859274
---
 tensorflow/contrib/tpu/BUILD                    | 12 +++++++++++-
 tensorflow/contrib/tpu/python/tpu/functional.py | 14 ++++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index 59119633aa..1859dee9d0 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -187,6 +187,11 @@ tf_gen_op_wrapper_py(
     ],
 )
 
+tf_custom_op_library(
+    name = "python/ops/_functional_ops.so",
+    srcs = ["ops/functional_ops.cc"],
+)
+
 tf_gen_op_wrapper_py(
     name = "gen_functional_ops",
     out = "python/tpu/gen_functional_ops.py",
@@ -196,9 +201,14 @@ tf_gen_op_wrapper_py(
     deps = [":functional_ops_op_lib"],
 )
 
-py_library(
+tf_custom_op_py_library(
     name = "functional",
     srcs = ["python/tpu/functional.py"],
+    dso = [":python/ops/_functional_ops.so"],
+    kernels = [
+        ":functional_ops_op_lib",
+    ],
+    srcs_version = "PY2AND3",
     visibility = [
         "//visibility:public",
     ],
diff --git a/tensorflow/contrib/tpu/python/tpu/functional.py b/tensorflow/contrib/tpu/python/tpu/functional.py
index 1ec9b5b33d..24c85156e5 100644
--- a/tensorflow/contrib/tpu/python/tpu/functional.py
+++ b/tensorflow/contrib/tpu/python/tpu/functional.py
@@ -18,8 +18,22 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import platform
+
 from tensorflow.contrib.tpu.python.tpu import gen_functional_ops
 
 
 TPUPartitionedCall = gen_functional_ops._tpu_partitioned_call  # pylint: disable=invalid-name,protected-access
 
+
+if platform.system() != "Windows":
+  # pylint: disable=wildcard-import,unused-import,g-import-not-at-top
+  from tensorflow.contrib.tpu.ops.gen_tpu_ordinal_selector_op import *
+
+  from tensorflow.contrib.util import loader
+  from tensorflow.python.platform import resource_loader
+  # pylint: enable=wildcard-import,unused-import,g-import-not-at-top
+
+  _tpu_partitioned_call_op = loader.load_op_library(
+      resource_loader.get_path_to_datafile("../ops/_functional_ops.so")
+  )
-- 
GitLab


From 5342cdba063fa5e3dbd64d6310dfcadb5b28cf97 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Thu, 24 Jan 2019 12:51:15 -0800
Subject: [PATCH 1690/2345] Leaky relu + unit tests

---
 .../compiler/tf2tensorrt/convert/convert_graph.cc  |  1 +
 .../compiler/tf2tensorrt/convert/convert_nodes.cc  | 14 +++++++++++++-
 .../tf2tensorrt/convert/convert_nodes_test.cc      | 14 +++++++++++---
 3 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc
index 31d0b2dfa4..75252ed8ae 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc
@@ -105,6 +105,7 @@ Status TrtCandidateSelector::IsTensorRTCandidate(const tensorflow::Node* node) {
       "FusedBatchNorm",
       "FusedBatchNormV2",
       "Identity",
+      "LeakyRelu",
       "Log",
       "MatMul",
       "Max",
diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
index 6d4a06ab15..5186acde81 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
@@ -2443,6 +2443,9 @@ tensorflow::Status ConvertActivation(OpConverterParams* params) {
         node_def.name());
   }
   static const std::unordered_map<string, nvinfer1::ActivationType> ops{
+#if NV_TENSORRT_MAJOR >= 5 && NV_TENSORRT_MINOR >= 1
+      {"LeakyRelu", nvinfer1::ActivationType::kLEAKY_RELU},
+#endif
       {"Relu", nvinfer1::ActivationType::kRELU},
       {"Sigmoid", nvinfer1::ActivationType::kSIGMOID},
       {"Tanh", nvinfer1::ActivationType::kTANH},
@@ -2462,6 +2465,15 @@ tensorflow::Status ConvertActivation(OpConverterParams* params) {
           *const_cast<nvinfer1::ITensor*>(tensor), op_pair->second);
   TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name());
   nvinfer1::ITensor* output_tensor = layer->getOutput(0);
+  // Set alpha parameter for LeakyRelu
+  // TODO(tmorris): Support Elu, Selu, Softplus, Clip, HardSigmoid, ScaledTanh,
+  // ThresholdedRelu.
+#if NV_TENSORRT_MAJOR >= 5 && NV_TENSORRT_MINOR >= 1
+  if (node_def.op() == "LeakyRelu") {
+    TFAttrs attrs(node_def);
+    layer->setAlpha(attrs.get<float>("alpha"));
+  }
+#endif
   // Set quantization range for output of Sigmoid, Tanh.
   if (node_def.op() == "Sigmoid") {
     params->converter->ProvideQuantizationRange(output_tensor, 0.0f, 1.0f);
@@ -3612,7 +3624,7 @@ static void RegisterValidatableOpConverters(
        {"Add", "Mul", "Sub", "Div", "RealDiv", "Maximum", "Minimum"}) {
     (*registration)[binary_op_type] = ConvertBinary;
   }
-  for (auto activation_op_type : {"Relu", "Sigmoid", "Tanh"}) {
+  for (auto activation_op_type : {"LeakyRelu", "Relu", "Sigmoid", "Tanh"}) {
     (*registration)[activation_op_type] = ConvertActivation;
   }
   for (auto pool_op_type : {"AvgPool", "MaxPool"}) {
diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
index 9789475ba9..a367886bed 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
@@ -2189,11 +2189,17 @@ TEST_F(OpConverterTest, ConvertActivation) {
         "Relu is only implemented for tensors, at my_act");
   }
 
+  const float ALPHA = 0.2f;
+
   // Get nodedef for activation layer.
   auto get_act_nodedef = [](string op_name) -> NodeDef {
     Scope s = Scope::NewRootScope();
     auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
-    if (op_name == "Relu") {
+    if (op_name == "LeakyRelu") {
+      auto attrs = ops::LeakyRelu::Attrs().Alpha(ALPHA);
+      auto act = ops::LeakyRelu(s.WithOpName("my_act"), input, attrs);
+      return act.operation.node()->def();
+    } else if (op_name == "Relu") {
       auto act = ops::Relu(s.WithOpName("my_act"), input);
       return act.operation.node()->def();
     } else if (op_name == "Sigmoid") {
@@ -2208,7 +2214,9 @@ TEST_F(OpConverterTest, ConvertActivation) {
   };
   // Get expected output for activation layer.
   auto get_act_output = [](string op_name, float input) -> float {
-    if (op_name == "Relu") {
+    if (op_name == "LeakyRelu") {
+      return (input > 0.0f) ? input : input * ALPHA;
+    } else if (op_name == "Relu") {
       return (input > 0.0f) ? input : 0.0f;
     } else if (op_name == "Sigmoid") {
       return 1.0f / (1.0f + std::exp(-input));
@@ -2220,7 +2228,7 @@ TEST_F(OpConverterTest, ConvertActivation) {
   };
 
   // Ok.
-  for (string op_name : {"Relu", "Sigmoid", "Tanh"}) {
+  for (string op_name : {"LeakyRelu", "Relu", "Sigmoid", "Tanh"}) {
     Reset();
     NodeDef node_def = get_act_nodedef(op_name);
     AddTestTensor("input", {1, 2, 3});
-- 
GitLab


From 278324cbe1392ef165cab8d1b21c3042ace03336 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Thu, 24 Jan 2019 16:27:24 -0800
Subject: [PATCH 1691/2345] Add test for LeakyRelu

---
 .../tf2tensorrt/convert/convert_nodes_test.cc | 22 ++++++++++++++-----
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
index a367886bed..2c8c44bc1a 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
@@ -2192,13 +2192,14 @@ TEST_F(OpConverterTest, ConvertActivation) {
   const float ALPHA = 0.2f;
 
   // Get nodedef for activation layer.
-  auto get_act_nodedef = [](string op_name) -> NodeDef {
+  auto get_act_nodedef = [ALPHA](string op_name) -> NodeDef {
     Scope s = Scope::NewRootScope();
     auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
     if (op_name == "LeakyRelu") {
-      auto attrs = ops::LeakyRelu::Attrs().Alpha(ALPHA);
-      auto act = ops::LeakyRelu(s.WithOpName("my_act"), input, attrs);
-      return act.operation.node()->def();
+      // LeakyRelu does not have a C++ API
+      NodeDef node_def = MakeNodeDef("my_act", "LeakyRelu", {"input"});
+      (*node_def.mutable_attr())["alpha"].set_f(ALPHA);
+      return node_def;
     } else if (op_name == "Relu") {
       auto act = ops::Relu(s.WithOpName("my_act"), input);
       return act.operation.node()->def();
@@ -2213,7 +2214,7 @@ TEST_F(OpConverterTest, ConvertActivation) {
     return NodeDef();
   };
   // Get expected output for activation layer.
-  auto get_act_output = [](string op_name, float input) -> float {
+  auto get_act_output = [ALPHA](string op_name, float input) -> float {
     if (op_name == "LeakyRelu") {
       return (input > 0.0f) ? input : input * ALPHA;
     } else if (op_name == "Relu") {
@@ -2227,8 +2228,17 @@ TEST_F(OpConverterTest, ConvertActivation) {
     return 0;
   };
 
+  const std::vector<string> ops_to_test = {
+#if NV_TENSORRT_MAJOR >= 5 && NV_TENSORRT_MINOR >= 1
+    "LeakyRelu",
+#endif
+    "Relu",
+    "Sigmoid",
+    "Tanh",
+  };
+
   // Ok.
-  for (string op_name : {"LeakyRelu", "Relu", "Sigmoid", "Tanh"}) {
+  for (const string& op_name : ops_to_test) {
     Reset();
     NodeDef node_def = get_act_nodedef(op_name);
     AddTestTensor("input", {1, 2, 3});
-- 
GitLab


From eb5423da7183c52d1993859464d7e58a50b4e892 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 30 Jan 2019 13:45:28 -0800
Subject: [PATCH 1692/2345] Rename ALPHA to kALPHA and make constexpr

---
 .../compiler/tf2tensorrt/convert/convert_nodes_test.cc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
index 2c8c44bc1a..edd161ee1e 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
@@ -2189,16 +2189,16 @@ TEST_F(OpConverterTest, ConvertActivation) {
         "Relu is only implemented for tensors, at my_act");
   }
 
-  const float ALPHA = 0.2f;
+  constexpr float kALPHA = 0.2f;
 
   // Get nodedef for activation layer.
-  auto get_act_nodedef = [ALPHA](string op_name) -> NodeDef {
+  auto get_act_nodedef = [](string op_name) -> NodeDef {
     Scope s = Scope::NewRootScope();
     auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
     if (op_name == "LeakyRelu") {
       // LeakyRelu does not have a C++ API
       NodeDef node_def = MakeNodeDef("my_act", "LeakyRelu", {"input"});
-      (*node_def.mutable_attr())["alpha"].set_f(ALPHA);
+      (*node_def.mutable_attr())["alpha"].set_f(kALPHA);
       return node_def;
     } else if (op_name == "Relu") {
       auto act = ops::Relu(s.WithOpName("my_act"), input);
@@ -2214,9 +2214,9 @@ TEST_F(OpConverterTest, ConvertActivation) {
     return NodeDef();
   };
   // Get expected output for activation layer.
-  auto get_act_output = [ALPHA](string op_name, float input) -> float {
+  auto get_act_output = [](string op_name, float input) -> float {
     if (op_name == "LeakyRelu") {
-      return (input > 0.0f) ? input : input * ALPHA;
+      return (input > 0.0f) ? input : input * kALPHA;
     } else if (op_name == "Relu") {
       return (input > 0.0f) ? input : 0.0f;
     } else if (op_name == "Sigmoid") {
-- 
GitLab


From 409f0806d4cc425e4bfc3ab45b1ecda8d38a1b43 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Thu, 31 Jan 2019 13:52:33 -0800
Subject: [PATCH 1693/2345] s/kALPHA/kAlpha

---
 .../compiler/tf2tensorrt/convert/convert_nodes_test.cc      | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
index edd161ee1e..35caccd4e7 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
@@ -2189,7 +2189,7 @@ TEST_F(OpConverterTest, ConvertActivation) {
         "Relu is only implemented for tensors, at my_act");
   }
 
-  constexpr float kALPHA = 0.2f;
+  constexpr float kAlpha = 0.2f;
 
   // Get nodedef for activation layer.
   auto get_act_nodedef = [](string op_name) -> NodeDef {
@@ -2198,7 +2198,7 @@ TEST_F(OpConverterTest, ConvertActivation) {
     if (op_name == "LeakyRelu") {
       // LeakyRelu does not have a C++ API
       NodeDef node_def = MakeNodeDef("my_act", "LeakyRelu", {"input"});
-      (*node_def.mutable_attr())["alpha"].set_f(kALPHA);
+      (*node_def.mutable_attr())["alpha"].set_f(kAlpha);
       return node_def;
     } else if (op_name == "Relu") {
       auto act = ops::Relu(s.WithOpName("my_act"), input);
@@ -2216,7 +2216,7 @@ TEST_F(OpConverterTest, ConvertActivation) {
   // Get expected output for activation layer.
   auto get_act_output = [](string op_name, float input) -> float {
     if (op_name == "LeakyRelu") {
-      return (input > 0.0f) ? input : input * kALPHA;
+      return (input > 0.0f) ? input : input * kAlpha;
     } else if (op_name == "Relu") {
       return (input > 0.0f) ? input : 0.0f;
     } else if (op_name == "Sigmoid") {
-- 
GitLab


From 61fb3b4a00662ff0f069b527009992edcb6c900f Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Thu, 31 Jan 2019 14:59:04 -0800
Subject: [PATCH 1694/2345] Use elementwise ops to implement LeakyRelu until
 ActivationLayer performance improves

---
 .../tf2tensorrt/convert/convert_nodes.cc      | 87 +++++++++++++++----
 .../tf2tensorrt/convert/convert_nodes_test.cc | 11 +--
 2 files changed, 70 insertions(+), 28 deletions(-)

diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
index 5186acde81..bc193e4bdf 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
@@ -2430,6 +2430,69 @@ tensorflow::Status ConvertPool(OpConverterParams* params) {
   return tensorflow::Status::OK();
 }
 
+// TODO(tmorris): Use ActivationType::kLEAKY_RELU in TRT 5.1+ once perf
+// improves.
+tensorflow::Status ConvertLeakyRelu(OpConverterParams* params) {
+  const auto& inputs = params->inputs;
+  const auto& node_def = params->node_def;
+  if (inputs.size() != 1) {
+    return tensorflow::errors::InvalidArgument(
+        node_def.op(), " expects one input, at ", node_def.name());
+  }
+  if (!inputs.at(0).is_tensor()) {
+    return tensorflow::errors::Unimplemented(
+        node_def.op(), " is only implemented for tensors, at ",
+        node_def.name());
+  }
+  TFAttrs attrs(node_def);
+  const float alpha = attrs.get<float>("alpha");
+  if (alpha < 0.0f || alpha > 1.0f) {
+    return tensorflow::errors::Unimplemented(
+        "Alpha value for LeakyRelu must be between 0 and 1, at ",
+        node_def.name());
+  }
+  if (params->validation_only) return tensorflow::Status::OK();
+
+  // Input Tensor
+  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
+  // Create const for alpha. The constant has to have the same rank as the input
+  // in order for TRT to broadcast.
+  nvinfer1::Dims dims;
+  dims.nbDims = inputs.at(0).GetTrtDims().nbDims;
+  for (int i = 0; i < dims.nbDims; i++) {
+    dims.d[i] = 1;
+  }
+  TRT_ShapedWeights weights = params->weight_store->GetTempWeights(
+      tensorflow::DataType::DT_FLOAT, dims);
+  auto weights_ptr =
+      static_cast<float*>(const_cast<void*>(weights.GetValues()));
+  weights_ptr[0] = alpha;
+  nvinfer1::ITensor* const_alpha_tensor =
+      params->converter->CreateConstantLayer(weights, dims);
+  TFTRT_RETURN_ERROR_IF_NULLPTR(const_alpha_tensor, node_def.name());
+  params->converter->ProvideQuantizationRange(const_alpha_tensor, -alpha,
+                                              alpha);
+  // alpha * x
+  nvinfer1::IElementWiseLayer* mul_layer =
+      params->converter->network()->addElementWise(
+          *const_cast<nvinfer1::ITensor*>(tensor), *const_alpha_tensor,
+          nvinfer1::ElementWiseOperation::kPROD);
+  TFTRT_RETURN_ERROR_IF_NULLPTR(mul_layer, node_def.name());
+  // max(x, alpha * x)
+  nvinfer1::IElementWiseLayer* max_layer =
+      params->converter->network()->addElementWise(
+          *const_cast<nvinfer1::ITensor*>(tensor),
+          *const_cast<nvinfer1::ITensor*>(mul_layer->getOutput(0)),
+          nvinfer1::ElementWiseOperation::kMAX);
+  TFTRT_RETURN_ERROR_IF_NULLPTR(max_layer, node_def.name());
+  nvinfer1::ITensor* output_tensor = max_layer->getOutput(0);
+  params->converter->MarkQuantizationRangesAsInferrable(
+      output_tensor, const_cast<nvinfer1::ITensor*>(mul_layer->getOutput(0)));
+
+  params->outputs->push_back(TRT_TensorOrWeights(output_tensor));
+  return Status::OK();
+}
+
 tensorflow::Status ConvertActivation(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
@@ -2443,18 +2506,15 @@ tensorflow::Status ConvertActivation(OpConverterParams* params) {
         node_def.name());
   }
   static const std::unordered_map<string, nvinfer1::ActivationType> ops{
-#if NV_TENSORRT_MAJOR >= 5 && NV_TENSORRT_MINOR >= 1
-      {"LeakyRelu", nvinfer1::ActivationType::kLEAKY_RELU},
-#endif
       {"Relu", nvinfer1::ActivationType::kRELU},
       {"Sigmoid", nvinfer1::ActivationType::kSIGMOID},
       {"Tanh", nvinfer1::ActivationType::kTANH},
   };
   auto op_pair = ops.find(node_def.op());
   if (op_pair == ops.end()) {
-    return tensorflow::errors::Unimplemented(
-        "Activation op: ", node_def.op(),
-        " not supported at: ", node_def.name());
+    return tensorflow::errors::Unimplemented("Activation op: ", node_def.op(),
+                                             " not supported at: ",
+                                             node_def.name());
   }
   if (params->validation_only) return tensorflow::Status::OK();
 
@@ -2465,15 +2525,6 @@ tensorflow::Status ConvertActivation(OpConverterParams* params) {
           *const_cast<nvinfer1::ITensor*>(tensor), op_pair->second);
   TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name());
   nvinfer1::ITensor* output_tensor = layer->getOutput(0);
-  // Set alpha parameter for LeakyRelu
-  // TODO(tmorris): Support Elu, Selu, Softplus, Clip, HardSigmoid, ScaledTanh,
-  // ThresholdedRelu.
-#if NV_TENSORRT_MAJOR >= 5 && NV_TENSORRT_MINOR >= 1
-  if (node_def.op() == "LeakyRelu") {
-    TFAttrs attrs(node_def);
-    layer->setAlpha(attrs.get<float>("alpha"));
-  }
-#endif
   // Set quantization range for output of Sigmoid, Tanh.
   if (node_def.op() == "Sigmoid") {
     params->converter->ProvideQuantizationRange(output_tensor, 0.0f, 1.0f);
@@ -2551,8 +2602,7 @@ Status ConvertQuantize(OpConverterParams* params) {
   return Status::OK();
 }
 
-// TODO(pdavoodi): we should update relu6 implementation once TensorRT supports
-// Relu6 natively.
+// TODO(tmorris): Use ActivationType::kCLIP in TRT 5.1+ once perf improves.
 tensorflow::Status ConvertRelu6(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
@@ -3605,6 +3655,7 @@ static void RegisterValidatableOpConverters(
   (*registration)["Conv2D"] = ConvertConv2D;
   (*registration)["DepthwiseConv2dNative"] = ConvertConv2DDepthwise;
   (*registration)["ExpandDims"] = ConvertExpandDims;
+  (*registration)["LeakyRelu"] = ConvertLeakyRelu;
   (*registration)["MatMul"] = ConvertMatMul;
   (*registration)["Pad"] = ConvertPad;
   (*registration)["Relu6"] = ConvertRelu6;
@@ -3624,7 +3675,7 @@ static void RegisterValidatableOpConverters(
        {"Add", "Mul", "Sub", "Div", "RealDiv", "Maximum", "Minimum"}) {
     (*registration)[binary_op_type] = ConvertBinary;
   }
-  for (auto activation_op_type : {"LeakyRelu", "Relu", "Sigmoid", "Tanh"}) {
+  for (auto activation_op_type : {"Relu", "Sigmoid", "Tanh"}) {
     (*registration)[activation_op_type] = ConvertActivation;
   }
   for (auto pool_op_type : {"AvgPool", "MaxPool"}) {
diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
index 35caccd4e7..b2ae8234d5 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
@@ -2228,17 +2228,8 @@ TEST_F(OpConverterTest, ConvertActivation) {
     return 0;
   };
 
-  const std::vector<string> ops_to_test = {
-#if NV_TENSORRT_MAJOR >= 5 && NV_TENSORRT_MINOR >= 1
-    "LeakyRelu",
-#endif
-    "Relu",
-    "Sigmoid",
-    "Tanh",
-  };
-
   // Ok.
-  for (const string& op_name : ops_to_test) {
+  for (const string& op_name : {"LeakyRelu", "Relu", "Sigmoid", "Tanh"}) {
     Reset();
     NodeDef node_def = get_act_nodedef(op_name);
     AddTestTensor("input", {1, 2, 3});
-- 
GitLab


From 0780dd3b2a2be351a084068b25cbe072d45c5238 Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Thu, 31 Jan 2019 14:52:56 -0800
Subject: [PATCH 1695/2345] Fix error when quantizing audio models This makes
 the quantization more robust to ops with no shape.

PiperOrigin-RevId: 231859355
---
 .../contrib/quantize/python/quant_ops.py      |   2 +-
 .../contrib/quantize/python/quant_ops_test.py |  10 +-
 tensorflow/examples/speech_commands/BUILD     |  10 ++
 .../examples/speech_commands/train_test.py    | 144 ++++++++++++++++++
 4 files changed, 163 insertions(+), 3 deletions(-)
 create mode 100644 tensorflow/examples/speech_commands/train_test.py

diff --git a/tensorflow/contrib/quantize/python/quant_ops.py b/tensorflow/contrib/quantize/python/quant_ops.py
index 8619708cda..39082cacf9 100644
--- a/tensorflow/contrib/quantize/python/quant_ops.py
+++ b/tensorflow/contrib/quantize/python/quant_ops.py
@@ -224,8 +224,8 @@ def MovingAvgQuantize(inputs,
       None, default_name=name_prefix, values=[inputs], reuse=reuse) as scope:
     scope.set_partitioner(None)
     input_shape = inputs.get_shape()
-    input_dim = len(input_shape)
     if per_channel:
+      input_dim = len(input_shape)
       # Only support quantizing 1-, 2- and 4-dimensional tensors.
       assert input_dim in [1, 2, 4], ('Expected 1D, 2D or 4D input, was: %s in '
                                       ' scope: %s' % (input_shape, name_prefix))
diff --git a/tensorflow/contrib/quantize/python/quant_ops_test.py b/tensorflow/contrib/quantize/python/quant_ops_test.py
index 36d2af94e0..c636c90d23 100644
--- a/tensorflow/contrib/quantize/python/quant_ops_test.py
+++ b/tensorflow/contrib/quantize/python/quant_ops_test.py
@@ -63,6 +63,12 @@ class QuantOpsTest(googletest.TestCase):
     self.assertAlmostEqual(min_value, -0.5, delta=1e-3)
     self.assertAlmostEqual(max_value, 0.5, delta=1e-3)
 
+  def testMovingAvgQuantizeTrainingAssignNoShape(self):
+    min_value, max_value = self._GetMinMaxValues(
+        quant_ops.MovingAvgQuantize, [[-1, 1], [0, 0]], shape=None)
+    self.assertAlmostEqual(min_value, -0.5, delta=1e-3)
+    self.assertAlmostEqual(max_value, 0.5, delta=1e-3)
+
   def testMovingAvgSymmetricQuantizeTrainingAssign(self):
     min_value, max_value = self._GetMinMaxValues(
         quant_ops.MovingAvgQuantize, [[-1, 0.5], [0, 0]], symmetric=True)
@@ -109,10 +115,10 @@ class QuantOpsTest(googletest.TestCase):
             is_training=True,
             vars_collection=_MIN_MAX_VARS)
 
-  def _GetMinMaxValues(self, quantize_fn, input_values, **kwds):
+  def _GetMinMaxValues(self, quantize_fn, input_values, shape=(2), **kwds):
     g = ops.Graph()
     with session.Session(graph=g) as sess:
-      x = array_ops.placeholder(dtypes.float32, shape=[2])
+      x = array_ops.placeholder(dtypes.float32, shape=shape)
       y = quantize_fn(
           x,
           init_min=0.0,
diff --git a/tensorflow/examples/speech_commands/BUILD b/tensorflow/examples/speech_commands/BUILD
index 7f3c764fac..960dd110b6 100644
--- a/tensorflow/examples/speech_commands/BUILD
+++ b/tensorflow/examples/speech_commands/BUILD
@@ -76,6 +76,16 @@ py_binary(
     ],
 )
 
+tf_py_test(
+    name = "train_test",
+    size = "small",
+    srcs = ["train_test.py"],
+    additional_deps = [
+        ":train",
+        "//tensorflow/python:client_testlib",
+    ],
+)
+
 py_binary(
     name = "freeze",
     srcs = [
diff --git a/tensorflow/examples/speech_commands/train_test.py b/tensorflow/examples/speech_commands/train_test.py
new file mode 100644
index 0000000000..db195760e9
--- /dev/null
+++ b/tensorflow/examples/speech_commands/train_test.py
@@ -0,0 +1,144 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for data input for speech commands."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+
+import tensorflow as tf
+
+from tensorflow.contrib.framework.python.ops import audio_ops as contrib_audio
+from tensorflow.examples.speech_commands import train
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import gfile
+from tensorflow.python.platform import test
+
+
+# Used to convert a dictionary into an object, for mocking parsed flags.
+class DictStruct(object):
+
+  def __init__(self, **entries):
+    self.__dict__.update(entries)
+
+
+class TrainTest(test.TestCase):
+
+  def _getWavData(self):
+    with self.cached_session():
+      sample_data = tf.zeros([32000, 2])
+      wav_encoder = contrib_audio.encode_wav(sample_data, 16000)
+      wav_data = self.evaluate(wav_encoder)
+    return wav_data
+
+  def _saveTestWavFile(self, filename, wav_data):
+    with open(filename, 'wb') as f:
+      f.write(wav_data)
+
+  def _saveWavFolders(self, root_dir, labels, how_many):
+    wav_data = self._getWavData()
+    for label in labels:
+      dir_name = os.path.join(root_dir, label)
+      os.mkdir(dir_name)
+      for i in range(how_many):
+        file_path = os.path.join(dir_name, 'some_audio_%d.wav' % i)
+        self._saveTestWavFile(file_path, wav_data)
+
+  def _prepareDummyTrainingData(self):
+    tmp_dir = self.get_temp_dir()
+    wav_dir = os.path.join(tmp_dir, 'wavs')
+    os.mkdir(wav_dir)
+    self._saveWavFolders(wav_dir, ['a', 'b', 'c'], 100)
+    background_dir = os.path.join(wav_dir, '_background_noise_')
+    os.mkdir(background_dir)
+    wav_data = self._getWavData()
+    for i in range(10):
+      file_path = os.path.join(background_dir, 'background_audio_%d.wav' % i)
+      self._saveTestWavFile(file_path, wav_data)
+    return wav_dir
+
+  def _getDefaultFlags(self):
+    flags = {
+        'data_url': '',
+        'data_dir': self._prepareDummyTrainingData(),
+        'wanted_words': 'a,b,c',
+        'sample_rate': 16000,
+        'clip_duration_ms': 1000,
+        'window_size_ms': 30,
+        'window_stride_ms': 20,
+        'feature_bin_count': 40,
+        'preprocess': 'mfcc',
+        'silence_percentage': 25,
+        'unknown_percentage': 25,
+        'validation_percentage': 10,
+        'testing_percentage': 10,
+        'summaries_dir': os.path.join(self.get_temp_dir(), 'summaries'),
+        'train_dir': os.path.join(self.get_temp_dir(), 'train'),
+        'time_shift_ms': 100,
+        'how_many_training_steps': '2',
+        'learning_rate': '0.01',
+        'quantize': False,
+        'model_architecture': 'conv',
+        'check_nans': False,
+        'start_checkpoint': '',
+        'batch_size': 1,
+        'background_volume': 0.25,
+        'background_frequency': 0.8,
+        'eval_step_interval': 1,
+        'save_step_interval': 1,
+    }
+    return DictStruct(**flags)
+
+  @test_util.run_deprecated_v1
+  def testTrain(self):
+    train.FLAGS = self._getDefaultFlags()
+    train.main('')
+    self.assertTrue(
+        gfile.Exists(
+            os.path.join(train.FLAGS.train_dir,
+                         train.FLAGS.model_architecture + '.pbtxt')))
+    self.assertTrue(
+        gfile.Exists(
+            os.path.join(train.FLAGS.train_dir,
+                         train.FLAGS.model_architecture + '_labels.txt')))
+    self.assertTrue(
+        gfile.Exists(
+            os.path.join(train.FLAGS.train_dir,
+                         train.FLAGS.model_architecture + '.ckpt-1.meta')))
+
+  @test_util.run_deprecated_v1
+  def testQuantizedTrain(self):
+    train.FLAGS = self._getDefaultFlags()
+    train.FLAGS.quantize = True
+    train.FLAGS.model_architecture = 'tiny_conv'
+    train.main('')
+    self.assertTrue(
+        gfile.Exists(
+            os.path.join(train.FLAGS.train_dir,
+                         train.FLAGS.model_architecture + '.pbtxt')))
+    self.assertTrue(
+        gfile.Exists(
+            os.path.join(train.FLAGS.train_dir,
+                         train.FLAGS.model_architecture + '_labels.txt')))
+    self.assertTrue(
+        gfile.Exists(
+            os.path.join(train.FLAGS.train_dir,
+                         train.FLAGS.model_architecture + '.ckpt-1.meta')))
+
+
+if __name__ == '__main__':
+  test.main()
-- 
GitLab


From e760628fccb964a96389c59445e0c99fde505fa2 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Thu, 31 Jan 2019 15:03:37 -0800
Subject: [PATCH 1696/2345] Unify Relu6 with the other activation tests

---
 .../tf2tensorrt/convert/convert_nodes_test.cc | 55 ++++---------------
 1 file changed, 12 insertions(+), 43 deletions(-)

diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
index b2ae8234d5..894a240241 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
@@ -2068,48 +2068,6 @@ TEST_F(OpConverterTest, ConvertQuantize) {
   }
 }
 
-TEST_F(OpConverterTest, ConvertRelu6) {
-  {
-    // Input list is empty, should fail.
-    NodeDef node_def = MakeNodeDef("my_relu6", "Relu6", {});
-    RunValidationAndConversion(
-        node_def, error::INVALID_ARGUMENT,
-        "Invalid number of inputs for Relu6, at my_relu6");
-  }
-
-  // Get the NodeDef for Relu6.
-  Scope s = Scope::NewRootScope();
-  auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
-  auto relu6 = ops::Relu6(s.WithOpName("my_relu6"), input);
-  const NodeDef node_def = relu6.operation.node()->def();
-  {
-    // Input is weights, should fail.
-    Reset();
-    AddTestWeights<float>("input", {1}, {1.0f});
-    RunValidationAndConversion(
-        node_def, error::UNIMPLEMENTED,
-        "Relu6 is only implemented for tensors, not weights, at my_relu6");
-  }
-  {
-    // Clip tensor values and set quantization ranges, ok.
-    Reset();
-    AddTestTensor("input", {1, 2, 3});
-    RunValidationAndConversion(node_def);
-    TRT_TensorOrWeights output;
-    TF_EXPECT_OK(GetTensorOrWeights("my_relu6", &output));
-    EXPECT_TRUE(output.is_tensor());
-    auto ranges = quantization_ranges();
-    EXPECT_EQ(ranges[output.tensor()], 6.0f);
-
-    const DataVec input_data{
-        {"input", test::AsTensor<float>({-100, -1, 0, 3, 5, 9})}};
-    DataVec output_data{{"my_relu6", ConstructTensor<float>(6)}};
-    BuildAndRun(input_data, &output_data);
-    EXPECT_THAT(GetSpanForData<float>(output_data[0]),
-                ElementsAre(0, 0, 0, 3, 5, 6));
-  }
-}
-
 template <DataType dtype>
 void TestConvertSquare(OpConverterTest* test) {
   test->Reset();
@@ -2203,6 +2161,9 @@ TEST_F(OpConverterTest, ConvertActivation) {
     } else if (op_name == "Relu") {
       auto act = ops::Relu(s.WithOpName("my_act"), input);
       return act.operation.node()->def();
+    } else if (op_name == "Relu6") {
+      auto act = ops::Relu6(s.WithOpName("my_act"), input);
+      return act.operation.node()->def();
     } else if (op_name == "Sigmoid") {
       auto act = ops::Sigmoid(s.WithOpName("my_act"), input);
       return act.operation.node()->def();
@@ -2219,6 +2180,8 @@ TEST_F(OpConverterTest, ConvertActivation) {
       return (input > 0.0f) ? input : input * kAlpha;
     } else if (op_name == "Relu") {
       return (input > 0.0f) ? input : 0.0f;
+    } else if (op_name == "Relu6") {
+      return std::min(std::max(input, 0.0f), 6.0f);
     } else if (op_name == "Sigmoid") {
       return 1.0f / (1.0f + std::exp(-input));
     } else if (op_name == "Tanh") {
@@ -2229,7 +2192,8 @@ TEST_F(OpConverterTest, ConvertActivation) {
   };
 
   // Ok.
-  for (const string& op_name : {"LeakyRelu", "Relu", "Sigmoid", "Tanh"}) {
+  for (const string& op_name :
+      {"LeakyRelu", "Relu", "Relu6", "Sigmoid", "Tanh"}) {
     Reset();
     NodeDef node_def = get_act_nodedef(op_name);
     AddTestTensor("input", {1, 2, 3});
@@ -2238,6 +2202,11 @@ TEST_F(OpConverterTest, ConvertActivation) {
     TF_EXPECT_OK(GetTensorOrWeights("my_act", &output));
     EXPECT_TRUE(output.is_tensor());
     ExpectTrtDimsEqualsArray({1, 2, 3}, output.tensor()->getDimensions());
+    if (op_name == "Relu6") {
+      // Relu6 should set quantization range automatically.
+      auto ranges = quantization_ranges();
+      EXPECT_EQ(ranges[output.tensor()], 6.0f);
+    }
 
     const std::vector<float> input = {-100, -2, -1, 0, 1, 100};
     const DataVec input_data{{"input", test::AsTensor<float>(input)}};
-- 
GitLab


From c76b061b500234d533d074ff1045e0f3db897506 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Thu, 31 Jan 2019 15:02:26 -0800
Subject: [PATCH 1697/2345] Initial implementation of IF control flow op *
 Properly support dynamic tensor use cases.   The output of IF is dynamic if: 
  (1) Any subgraph is dynamic   (2) 2 subgraphs are both static, but has
 different output shapes * Not optimized yet. Currently copying input/output
 tensors   between subgraphs.

PiperOrigin-RevId: 231861155
---
 tensorflow/lite/kernels/BUILD       |  18 ++
 tensorflow/lite/kernels/if.cc       | 196 ++++++++++++++++++++
 tensorflow/lite/kernels/if_test.cc  | 275 ++++++++++++++++++++++++++++
 tensorflow/lite/kernels/register.cc |   4 +
 4 files changed, 493 insertions(+)
 create mode 100644 tensorflow/lite/kernels/if.cc
 create mode 100644 tensorflow/lite/kernels/if_test.cc

diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index d338bc5499..92f51350c2 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -180,6 +180,7 @@ cc_library(
         "fully_connected.cc",
         "gather.cc",
         "hashtable_lookup.cc",
+        "if.cc",
         "l2norm.cc",
         "layer_norm_lstm.cc",
         "local_response_norm.cc",
@@ -1217,9 +1218,26 @@ tf_cc_test(
     srcs = ["squared_difference_test.cc"],
     deps = [
         ":builtin_ops",
+        "//tensorflow/lite:builtin_op_data",
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite/kernels:test_util",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
+tf_cc_test(
+    name = "if_test",
+    size = "small",
+    srcs = ["if_test.cc"],
+    tags = ["tflite_not_portable_ios"],
+    deps = [
+        ":builtin_ops",
+        ":kernel_util",
+        "//tensorflow/lite:builtin_op_data",
         "//tensorflow/lite:framework",
         "//tensorflow/lite/kernels:test_util",
         "@com_google_googletest//:gtest",
+        "@flatbuffers",
     ],
 )
 
diff --git a/tensorflow/lite/kernels/if.cc b/tensorflow/lite/kernels/if.cc
new file mode 100644
index 0000000000..a814ac997d
--- /dev/null
+++ b/tensorflow/lite/kernels/if.cc
@@ -0,0 +1,196 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "flatbuffers/flexbuffers.h"  // TF:flatbuffers
+#include "tensorflow/lite/c/builtin_op_data.h"
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/core/subgraph.h"
+#include "tensorflow/lite/kernels/kernel_util.h"
+
+namespace tflite {
+namespace ops {
+namespace custom {
+namespace if_kernel {
+
+struct OpData {
+  int then_subgraph_index;
+  int else_subgraph_index;
+};
+
+void* Init(TfLiteContext* context, const char* buffer, size_t length) {
+  auto* op_data = new OpData;
+  const uint8_t* buffer_t = reinterpret_cast<const uint8_t*>(buffer);
+  const flexbuffers::Map& m = flexbuffers::GetRoot(buffer_t, length).AsMap();
+  op_data->then_subgraph_index = m["then_subgraph_index"].AsInt32();
+  op_data->else_subgraph_index = m["else_subgraph_index"].AsInt32();
+  return op_data;
+}
+
+void Free(TfLiteContext* context, void* buffer) {
+  delete reinterpret_cast<OpData*>(buffer);
+}
+
+TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
+  const OpData* op_data = reinterpret_cast<OpData*>(node->user_data);
+
+  TF_LITE_ENSURE(context, node->inputs->size > 0);
+
+  // The first input is the condition.
+  const TfLiteTensor* cond = GetInput(context, node, 0);
+  // Currently only bool is supported.
+  // TODO(ycling): Support other types since TensorFlow also support
+  // non-bool types as condition.
+  TF_LITE_ENSURE_EQ(context, cond->type, kTfLiteBool);
+  TF_LITE_ENSURE_EQ(context, NumElements(cond), 1);
+
+  // The first input of the node is the condition. The rest of inputs are
+  // passed to the branch subgraphs. Therefore, the number of subgraph inputs
+  // will be the number of node inputs - 1.
+  int num_inputs = node->inputs->size - 1;
+  int num_outputs = node->outputs->size;
+
+  Subgraph* this_subgraph = reinterpret_cast<Subgraph*>(context->impl_);
+  auto* subgraphs = this_subgraph->GetSubgraphs();
+  TF_LITE_ENSURE(context, op_data->then_subgraph_index < subgraphs->size());
+  TF_LITE_ENSURE(context, op_data->else_subgraph_index < subgraphs->size());
+
+  Subgraph* then_subgraph = (*subgraphs)[op_data->then_subgraph_index].get();
+  Subgraph* else_subgraph = (*subgraphs)[op_data->else_subgraph_index].get();
+
+  for (auto* subgraph : {then_subgraph, else_subgraph}) {
+    TF_LITE_ENSURE_EQ(context, num_inputs, subgraph->inputs().size());
+    TF_LITE_ENSURE_EQ(context, num_outputs, subgraph->outputs().size());
+  }
+
+  bool has_dynamic_output_tensors = false;
+  for (auto* subgraph : {then_subgraph, else_subgraph}) {
+    for (int i = 0; i < num_inputs; ++i) {
+      // The first input of the node is the condition. The indices of the inputs
+      // passed to the subgraphs are offset by 1.
+      const TfLiteTensor* input = GetInput(context, node, i + 1);
+      std::vector<int> dims(input->dims->data,
+                            input->dims->data + input->dims->size);
+      subgraph->ResizeInputTensor(i, dims);
+      TfLiteTensor* subgraph_input = subgraph->tensor(subgraph->inputs()[i]);
+      TF_LITE_ENSURE_EQ(context, input->type, subgraph_input->type);
+    }
+    // Note: The `Prepare` function is responsible to run `AllocateTensors` on
+    // both subgraphs. It's intentionally not to break out of the loop when
+    // finding a dynamic output tensor.
+    TF_LITE_ENSURE_OK(context, subgraph->AllocateTensors());
+    has_dynamic_output_tensors |= subgraph->HasDynamicTensors();
+  }
+
+  if (!has_dynamic_output_tensors) {
+    for (int i = 0; i < num_outputs; ++i) {
+      TfLiteTensor* then_output =
+          then_subgraph->tensor(then_subgraph->outputs()[i]);
+      TfLiteTensor* else_output =
+          else_subgraph->tensor(else_subgraph->outputs()[i]);
+      // If the 2 subgraphs have static but different output shapes, the output
+      // tensors of the IF op have dynamic sizes.
+      if (!TfLiteIntArrayEqual(then_output->dims, else_output->dims)) {
+        has_dynamic_output_tensors = true;
+        break;
+      }
+    }
+  }
+
+  for (int i = 0; i < num_outputs; ++i) {
+    TfLiteTensor* output = GetOutput(context, node, i);
+    if (has_dynamic_output_tensors) {
+      SetTensorToDynamic(output);
+    } else {
+      // When there's no dynamic output tensors, the 2 subgraph has exactly
+      // the same static sized outputs.
+      TfLiteTensor* then_output =
+          then_subgraph->tensor(then_subgraph->outputs()[i]);
+      TfLiteIntArray* output_size = TfLiteIntArrayCopy(then_output->dims);
+      TF_LITE_ENSURE_OK(context,
+                        context->ResizeTensor(context, output, output_size));
+    }
+  }
+
+  return kTfLiteOk;
+}
+
+TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
+  const OpData* op_data = reinterpret_cast<OpData*>(node->user_data);
+
+  const TfLiteTensor* cond = GetInput(context, node, 0);
+  bool cond_value = cond->data.b[0];
+
+  Subgraph* this_subgraph = reinterpret_cast<Subgraph*>(context->impl_);
+  auto* subgraphs = this_subgraph->GetSubgraphs();
+
+  // Currently we copy the input / output between the subgraphs. This isn't
+  // optimized yet.
+  // TODO(b/120234921): Optimize and avoid copying tensors between subgraphs.
+  int active_branch_subgraph_index =
+      cond_value ? op_data->then_subgraph_index : op_data->else_subgraph_index;
+  Subgraph& active_branch_subgraph =
+      *(*subgraphs)[active_branch_subgraph_index];
+  for (int i = 0; i < active_branch_subgraph.inputs().size(); ++i) {
+    const TfLiteTensor* input = GetInput(context, node, i + 1);
+    TfLiteTensor* subgraph_input =
+        active_branch_subgraph.tensor(active_branch_subgraph.inputs()[i]);
+    TF_LITE_ENSURE_EQ(context, input->bytes, subgraph_input->bytes);
+    memcpy(subgraph_input->data.raw, input->data.raw, input->bytes);
+  }
+
+  // Note: It's guaranteed that the subgraphs' `AllocateTensors` are called
+  // in `Prepare`, so we don't need to do it here again.
+  TF_LITE_ENSURE_OK(context, active_branch_subgraph.Invoke());
+
+  bool has_dynamic_output_tensors = false;
+  for (int i = 0; i < node->outputs->size; ++i) {
+    TfLiteTensor* output = GetOutput(context, node, i);
+    if (IsDynamicTensor(output)) {
+      has_dynamic_output_tensors = true;
+      break;
+    }
+  }
+
+  if (has_dynamic_output_tensors) {
+    for (int i = 0; i < node->outputs->size; ++i) {
+      TfLiteTensor* output = GetOutput(context, node, i);
+      TfLiteTensor* subgraph_output =
+          active_branch_subgraph.tensor(active_branch_subgraph.outputs()[i]);
+      TfLiteIntArray* output_size = TfLiteIntArrayCopy(subgraph_output->dims);
+      TF_LITE_ENSURE_OK(context,
+                        context->ResizeTensor(context, output, output_size));
+    }
+  }
+
+  for (int i = 0; i < active_branch_subgraph.outputs().size(); ++i) {
+    const TfLiteTensor* subgraph_output =
+        active_branch_subgraph.tensor(active_branch_subgraph.outputs()[i]);
+    TfLiteTensor* output = GetOutput(context, node, i);
+    TF_LITE_ENSURE_EQ(context, output->bytes, subgraph_output->bytes);
+    memcpy(output->data.raw, subgraph_output->data.raw, output->bytes);
+  }
+  return kTfLiteOk;
+}
+
+}  // namespace if_kernel
+
+TfLiteRegistration* Register_IF() {
+  static TfLiteRegistration r = {if_kernel::Init, if_kernel::Free,
+                                 if_kernel::Prepare, if_kernel::Eval};
+  return &r;
+}
+
+}  // namespace custom
+}  // namespace ops
+}  // namespace tflite
diff --git a/tensorflow/lite/kernels/if_test.cc b/tensorflow/lite/kernels/if_test.cc
new file mode 100644
index 0000000000..80e4ef133a
--- /dev/null
+++ b/tensorflow/lite/kernels/if_test.cc
@@ -0,0 +1,275 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <gtest/gtest.h>
+#include "flatbuffers/flexbuffers.h"  // TF:flatbuffers
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/kernels/kernel_util.h"
+#include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/kernels/test_util.h"
+#include "tensorflow/lite/model.h"
+
+namespace tflite {
+namespace ops {
+namespace builtin {
+// ADD and MUL are used to test simple branch.
+TfLiteRegistration* Register_ADD();
+TfLiteRegistration* Register_MUL();
+// ADD and MUL are used to test dynamic sized subgraphs.
+TfLiteRegistration* Register_PAD();
+}  // namespace builtin
+namespace custom {
+TfLiteRegistration* Register_IF();
+}  // namespace custom
+}  // namespace ops
+
+namespace {
+
+void SetupTensor(Subgraph* subgraph, int tensor_index,
+                 const std::vector<int>& shape, TfLiteType type) {
+  ASSERT_EQ(subgraph->SetTensorParametersReadWrite(
+                tensor_index, type, "", shape.size(), shape.data(), {}, false),
+            kTfLiteOk);
+}
+
+// TODO(ycling): Consider to move all the test helper functions to another
+// build target (e.g. subgraph_test_util).
+// Build a subgraph with an add op. Helper function for testing.
+void BuildAddSubgraph(Subgraph* subgraph) {
+  int first_new_tensor_index;
+  ASSERT_EQ(subgraph->AddTensors(3, &first_new_tensor_index), kTfLiteOk);
+  ASSERT_EQ(first_new_tensor_index, 0);
+  ASSERT_EQ(subgraph->SetInputs({0, 1}), kTfLiteOk);
+  ASSERT_EQ(subgraph->SetOutputs({2}), kTfLiteOk);
+
+  SetupTensor(subgraph, 0, {2}, kTfLiteInt32);
+  SetupTensor(subgraph, 1, {1, 2}, kTfLiteInt32);
+  // Intentionally set the wrong output size for testing. This should be
+  // overridden by Prepare function.
+  SetupTensor(subgraph, 2, {100}, kTfLiteInt32);
+
+  TfLiteAddParams* params =
+      reinterpret_cast<TfLiteAddParams*>(malloc(sizeof(TfLiteAddParams)));
+  params->activation = kTfLiteActNone;
+  int node_index;
+  subgraph->AddNodeWithParameters({0, 1}, {2}, nullptr, 0, params,
+                                  ::tflite::ops::builtin::Register_ADD(),
+                                  &node_index);
+}
+
+// Build a subgraph with an mul op. Helper function for testing.
+void BuildMulSubgraph(Subgraph* subgraph) {
+  int first_new_tensor_index;
+  ASSERT_EQ(subgraph->AddTensors(3, &first_new_tensor_index), kTfLiteOk);
+  ASSERT_EQ(first_new_tensor_index, 0);
+  ASSERT_EQ(subgraph->SetInputs({0, 1}), kTfLiteOk);
+  ASSERT_EQ(subgraph->SetOutputs({2}), kTfLiteOk);
+
+  SetupTensor(subgraph, 0, {2}, kTfLiteInt32);
+  SetupTensor(subgraph, 1, {1, 2}, kTfLiteInt32);
+  // Intentionally set the wrong output size for testing. This should be
+  // overridden by Prepare function.
+  SetupTensor(subgraph, 2, {100}, kTfLiteInt32);
+
+  TfLiteMulParams* params =
+      reinterpret_cast<TfLiteMulParams*>(malloc(sizeof(TfLiteMulParams)));
+  params->activation = kTfLiteActNone;
+  int node_index;
+  subgraph->AddNodeWithParameters({0, 1}, {2}, nullptr, 0, params,
+                                  ::tflite::ops::builtin::Register_MUL(),
+                                  &node_index);
+}
+
+// Build a subgraph with a pad op. Helper function for testing.
+void BuildPadSubgraph(Subgraph* subgraph) {
+  int first_new_tensor_index;
+  ASSERT_EQ(subgraph->AddTensors(3, &first_new_tensor_index), kTfLiteOk);
+  ASSERT_EQ(first_new_tensor_index, 0);
+  ASSERT_EQ(subgraph->SetInputs({0, 1}), kTfLiteOk);
+  ASSERT_EQ(subgraph->SetOutputs({2}), kTfLiteOk);
+
+  SetupTensor(subgraph, 0, {2}, kTfLiteInt32);
+  SetupTensor(subgraph, 1, {1, 2}, kTfLiteInt32);
+  // Intentionally set the wrong output size for testing. This should be
+  // overridden by Prepare function.
+  SetupTensor(subgraph, 2, {100}, kTfLiteInt32);
+
+  TfLitePadParams* params =
+      reinterpret_cast<TfLitePadParams*>(malloc(sizeof(TfLitePadParams)));
+  int node_index;
+  subgraph->AddNodeWithParameters({0, 1}, {2}, nullptr, 0, params,
+                                  ::tflite::ops::builtin::Register_PAD(),
+                                  &node_index);
+}
+
+void BuildIfSubgraph(Subgraph* subgraph) {
+  int first_new_tensor_index;
+  ASSERT_EQ(subgraph->AddTensors(4, &first_new_tensor_index), kTfLiteOk);
+  ASSERT_EQ(first_new_tensor_index, 0);
+  ASSERT_EQ(subgraph->SetInputs({0, 1, 2}), kTfLiteOk);
+  ASSERT_EQ(subgraph->SetOutputs({3}), kTfLiteOk);
+
+  SetupTensor(subgraph, 0, {1}, kTfLiteBool);
+  SetupTensor(subgraph, 1, {2}, kTfLiteInt32);
+  SetupTensor(subgraph, 2, {1, 2}, kTfLiteInt32);
+  // Intentionally set the wrong output size for testing. This should be
+  // overridden by Prepare function.
+  SetupTensor(subgraph, 3, {100}, kTfLiteInt32);
+
+  flexbuffers::Builder fbb;
+  fbb.Map([&]() {
+    fbb.Int("then_subgraph_index", 1);
+    fbb.Int("else_subgraph_index", 2);
+  });
+  fbb.Finish();
+  const auto& buffer = fbb.GetBuffer();
+
+  int node_index;
+  subgraph->AddNodeWithParameters(
+      {0, 1, 2}, {3}, reinterpret_cast<const char*>(buffer.data()),
+      buffer.size(), nullptr, ::tflite::ops::custom::Register_IF(),
+      &node_index);
+}
+
+void FillIntTensor(TfLiteTensor* tensor, const std::vector<int32_t>& data) {
+  int count = NumElements(tensor);
+  ASSERT_EQ(count, data.size());
+  for (int i = 0; i < count; ++i) {
+    tensor->data.i32[i] = data[i];
+  }
+}
+
+void CheckIntTensor(const TfLiteTensor* tensor, const std::vector<int>& shape,
+                    const std::vector<int32_t>& data) {
+  ASSERT_EQ(tensor->dims->size, shape.size());
+  for (int i = 0; i < tensor->dims->size; ++i) {
+    ASSERT_EQ(tensor->dims->data[i], shape[i]);
+  }
+  ASSERT_EQ(tensor->type, kTfLiteInt32);
+  int count = NumElements(tensor);
+  ASSERT_EQ(count, data.size());
+  for (int i = 0; i < count; ++i) {
+    EXPECT_EQ(tensor->data.i32[i], data[i]);
+  }
+}
+
+// TestHelperfunctionTest tests the helper functions defined in this file.
+TEST(TestHelperfunctionTest, TestBuildAddSubgraph) {
+  std::unique_ptr<Interpreter> interpreter(new Interpreter);
+  BuildAddSubgraph(&interpreter->primary_subgraph());
+  ASSERT_EQ(interpreter->AllocateTensors(), kTfLiteOk);
+  FillIntTensor(interpreter->tensor(interpreter->inputs()[0]), {5, 7});
+  FillIntTensor(interpreter->tensor(interpreter->inputs()[1]), {1, 2});
+  ASSERT_EQ(interpreter->Invoke(), kTfLiteOk);
+  TfLiteTensor* output = interpreter->tensor(interpreter->outputs()[0]);
+  CheckIntTensor(output, {1, 2}, {6, 9});
+}
+
+TEST(TestHelperfunctionTest, TestBuildMulSubgraph) {
+  std::unique_ptr<Interpreter> interpreter(new Interpreter);
+  BuildMulSubgraph(&interpreter->primary_subgraph());
+  ASSERT_EQ(interpreter->AllocateTensors(), kTfLiteOk);
+  FillIntTensor(interpreter->tensor(interpreter->inputs()[0]), {5, 7});
+  FillIntTensor(interpreter->tensor(interpreter->inputs()[1]), {1, 2});
+  ASSERT_EQ(interpreter->Invoke(), kTfLiteOk);
+  TfLiteTensor* output = interpreter->tensor(interpreter->outputs()[0]);
+  CheckIntTensor(output, {1, 2}, {5, 14});
+}
+
+TEST(TestHelperfunctionTest, TestBuildPadSubgraph) {
+  std::unique_ptr<Interpreter> interpreter(new Interpreter);
+  BuildPadSubgraph(&interpreter->primary_subgraph());
+  ASSERT_EQ(interpreter->AllocateTensors(), kTfLiteOk);
+  FillIntTensor(interpreter->tensor(interpreter->inputs()[0]), {5, 7});
+  FillIntTensor(interpreter->tensor(interpreter->inputs()[1]), {1, 2});
+  ASSERT_EQ(interpreter->Invoke(), kTfLiteOk);
+  TfLiteTensor* output = interpreter->tensor(interpreter->outputs()[0]);
+  CheckIntTensor(output, {5}, {0, 5, 7, 0, 0});
+}
+
+// A simple test that performs `ADD` if condition is true, and `MUL` otherwise.
+// The computation is: `cond ? a + b : a * b`.
+class SimpleIfTest : public ::testing::Test {
+ protected:
+  void SetUp() override {
+    interpreter_.reset(new Interpreter);
+    interpreter_->AddSubgraphs(2);
+    BuildAddSubgraph(interpreter_->subgraph(1));
+    BuildMulSubgraph(interpreter_->subgraph(2));
+    BuildIfSubgraph(&interpreter_->primary_subgraph());
+    ASSERT_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
+    FillIntTensor(interpreter_->tensor(interpreter_->inputs()[1]), {5, 7});
+    FillIntTensor(interpreter_->tensor(interpreter_->inputs()[2]), {1, 2});
+  }
+  std::unique_ptr<Interpreter> interpreter_;
+};
+
+TEST_F(SimpleIfTest, TestIfTrue) {
+  interpreter_->typed_input_tensor<bool>(0)[0] = true;
+  ASSERT_EQ(interpreter_->Invoke(), kTfLiteOk);
+  TfLiteTensor* output = interpreter_->tensor(interpreter_->outputs()[0]);
+  CheckIntTensor(output, {1, 2}, {6, 9});
+}
+
+TEST_F(SimpleIfTest, TestIfFalse) {
+  interpreter_->typed_input_tensor<bool>(0)[0] = false;
+  ASSERT_EQ(interpreter_->Invoke(), kTfLiteOk);
+  TfLiteTensor* output = interpreter_->tensor(interpreter_->outputs()[0]);
+  CheckIntTensor(output, {1, 2}, {5, 14});
+}
+
+// Test IF op using subgraphs with dynamically sized outputs.
+// The computation is: `cond ? a + b : pad(a, b)`.
+class DynamicSubgraphIfTest : public ::testing::Test {
+ protected:
+  void SetUp() override {
+    interpreter_.reset(new Interpreter);
+    interpreter_->AddSubgraphs(2);
+    BuildAddSubgraph(interpreter_->subgraph(1));
+    BuildPadSubgraph(interpreter_->subgraph(2));
+    BuildIfSubgraph(&interpreter_->primary_subgraph());
+    ASSERT_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
+    FillIntTensor(interpreter_->tensor(interpreter_->inputs()[1]), {5, 7});
+    FillIntTensor(interpreter_->tensor(interpreter_->inputs()[2]), {1, 2});
+  }
+  std::unique_ptr<Interpreter> interpreter_;
+};
+
+TEST_F(DynamicSubgraphIfTest, TestIfTrue) {
+  interpreter_->typed_input_tensor<bool>(0)[0] = true;
+  ASSERT_EQ(interpreter_->Invoke(), kTfLiteOk);
+  TfLiteTensor* output = interpreter_->tensor(interpreter_->outputs()[0]);
+  // Even if the true branch has a static type output, the output of the
+  // if op is dynamic because the other branch has dynamic output.
+  EXPECT_TRUE(IsDynamicTensor(output));
+  CheckIntTensor(output, {1, 2}, {6, 9});
+}
+
+TEST_F(DynamicSubgraphIfTest, TestIfFalse) {
+  interpreter_->typed_input_tensor<bool>(0)[0] = false;
+  ASSERT_EQ(interpreter_->Invoke(), kTfLiteOk);
+  TfLiteTensor* output = interpreter_->tensor(interpreter_->outputs()[0]);
+  // The false branch has dynamic output.
+  EXPECT_TRUE(IsDynamicTensor(output));
+  CheckIntTensor(output, {5}, {0, 5, 7, 0, 0});
+}
+
+}  // namespace
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index fdc7ef3b9a..11861a8578 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -26,6 +26,7 @@ TfLiteRegistration* Register_LAYER_NORM_LSTM();
 TfLiteRegistration* Register_MFCC();
 TfLiteRegistration* Register_DETECTION_POSTPROCESS();
 TfLiteRegistration* Register_RELU_1();
+TfLiteRegistration* Register_IF();
 
 }  // namespace custom
 
@@ -304,6 +305,9 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddCustom("Relu1", tflite::ops::custom::Register_RELU_1());
   AddCustom("TFLite_Detection_PostProcess",
             tflite::ops::custom::Register_DETECTION_POSTPROCESS());
+
+  // WARNING: Control flow ops are experimental and subject to change.
+  AddCustom("Experimental_If", tflite::ops::custom::Register_IF());
 }
 
 }  // namespace builtin
-- 
GitLab


From db820306defa458e38edc55c715d2c7fcbe71041 Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Thu, 31 Jan 2019 15:08:28 -0800
Subject: [PATCH 1698/2345] Replace uninitialized tensors with zeros in
 TensorListConcat. In TensorListGetItem, infer the shape of the requested
 entry from other list elements if element_shape is not fully defined.

PiperOrigin-RevId: 231862507
---
 .../base_api/api_def_TensorListConcatV2.pbtxt |  18 ++
 .../api_def_TensorListConcatV2.pbtxt          |   4 +
 tensorflow/core/kernels/list_kernels.cc       |   5 +
 tensorflow/core/kernels/list_kernels.cu.cc    |   7 +
 tensorflow/core/kernels/list_kernels.h        | 192 +++++++++++++-----
 tensorflow/core/ops/list_ops.cc               |  84 +++++---
 .../python/kernel_tests/list_ops_test.py      |  64 +++++-
 .../kernel_tests/tensor_array_ops_test.py     |  18 +-
 tensorflow/python/ops/list_ops.py             |  53 +++--
 tensorflow/python/ops/tensor_array_ops.py     |   5 +
 10 files changed, 342 insertions(+), 108 deletions(-)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorListConcatV2.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_TensorListConcatV2.pbtxt

diff --git a/tensorflow/core/api_def/base_api/api_def_TensorListConcatV2.pbtxt b/tensorflow/core/api_def/base_api/api_def_TensorListConcatV2.pbtxt
new file mode 100644
index 0000000000..9b2af2c23b
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_TensorListConcatV2.pbtxt
@@ -0,0 +1,18 @@
+op {
+  graph_op_name: "TensorListConcatV2"
+  summary: "Concats all tensors in the list along the 0th dimension."
+  description: <<END
+Requires that all tensors have the same shape except the first dimension.
+
+input_handle: The input list.
+element_shape: The shape of the uninitialized elements in the list. If the first
+  dimension is not -1, it is assumed that all list elements have the same
+  leading dim.
+leading_dims: The list of leading dims of uninitialized list elements. Used if
+  the leading dim of input_handle.element_shape or the element_shape input arg
+  is not already set.
+tensor: The concated result.
+lengths: Output tensor containing sizes of the 0th dimension of tensors in the list, used for computing the gradient.
+
+END
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_TensorListConcatV2.pbtxt b/tensorflow/core/api_def/python_api/api_def_TensorListConcatV2.pbtxt
new file mode 100644
index 0000000000..237774a388
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_TensorListConcatV2.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "TensorListConcatV2"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/kernels/list_kernels.cc b/tensorflow/core/kernels/list_kernels.cc
index 1e449ddb2c..12863815fb 100644
--- a/tensorflow/core/kernels/list_kernels.cc
+++ b/tensorflow/core/kernels/list_kernels.cc
@@ -148,6 +148,7 @@ Status TensorShapeFromTensor(const Tensor& t, PartialTensorShape* out) {
   if (t.shape() == TensorShape({})) {
     if ((t.dtype() == DT_INT32 && t.scalar<int32>()() == -1) ||
         (t.dtype() == DT_INT64 && t.scalar<int64>()() == -1)) {
+      *out = PartialTensorShape();
       return Status::OK();
     }
     return errors::InvalidArgument(
@@ -636,6 +637,10 @@ REGISTER_KERNEL_BUILDER(Name("TensorListConcatLists").Device(DEVICE_GPU),
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_CPU),                \
                           TensorListConcat<CPUDevice, T>)         \
+  REGISTER_KERNEL_BUILDER(Name("TensorListConcatV2")              \
+                              .TypeConstraint<T>("element_dtype") \
+                              .Device(DEVICE_CPU),                \
+                          TensorListConcat<CPUDevice, T>)         \
   REGISTER_KERNEL_BUILDER(Name("TensorListGetItem")               \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_CPU),                \
diff --git a/tensorflow/core/kernels/list_kernels.cu.cc b/tensorflow/core/kernels/list_kernels.cu.cc
index 5259389712..652ca2b7e2 100644
--- a/tensorflow/core/kernels/list_kernels.cu.cc
+++ b/tensorflow/core/kernels/list_kernels.cu.cc
@@ -64,6 +64,13 @@ typedef Eigen::GpuDevice GPUDevice;
                               .Device(DEVICE_GPU)                 \
                               .HostMemory("lengths"),             \
                           TensorListConcat<GPUDevice, T>)         \
+  REGISTER_KERNEL_BUILDER(Name("TensorListConcatV2")              \
+                              .TypeConstraint<T>("element_dtype") \
+                              .Device(DEVICE_GPU)                 \
+                              .HostMemory("leading_dims")         \
+                              .HostMemory("element_shape")        \
+                              .HostMemory("lengths"),             \
+                          TensorListConcat<GPUDevice, T>)         \
   REGISTER_KERNEL_BUILDER(Name("TensorListPushBackBatch")         \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_GPU),                \
diff --git a/tensorflow/core/kernels/list_kernels.h b/tensorflow/core/kernels/list_kernels.h
index 7b3ff078c8..c25e9cecb4 100644
--- a/tensorflow/core/kernels/list_kernels.h
+++ b/tensorflow/core/kernels/list_kernels.h
@@ -209,11 +209,29 @@ class TensorListGetItem : public OpKernel {
       OP_REQUIRES_OK(
           c, GetElementShapeFromInput(c, *l, 2, &partial_element_shape));
       TensorShape element_shape;
+      // If l->element_shape and the element_shape input are both not fully
+      // defined, try to infer the shape from other list elements. This requires
+      // that all initialized list elements have the same shape.
+      // NOTE(srbs): This might be a performance bottleneck since we are
+      // iterating over the entire list here. This is necessary for feature
+      // parity with TensorArray.read. TensorArray has a mode in which all
+      // elements are required to be of the same shape, TensorList does not.
+      // In that mode TensorArray sets the array's element_shape on the first
+      // write call. We could do something similar here if needed.
+      if (!partial_element_shape.IsFullyDefined()) {
+        for (const Tensor& t : l->tensors) {
+          if (t.dtype() != DT_INVALID) {
+            PartialTensorShape tmp = partial_element_shape;
+            OP_REQUIRES_OK(c, tmp.MergeWith(t.shape(), &partial_element_shape));
+          }
+        }
+      }
       OP_REQUIRES(
           c, partial_element_shape.AsTensorShape(&element_shape),
           errors::InvalidArgument("Trying to read an uninitialized tensor but ",
-                                  "element_shape is not fully defined.",
-                                  partial_element_shape.DebugString()));
+                                  "element_shape is not fully defined: ",
+                                  partial_element_shape.DebugString(),
+                                  " and no list element is set."));
       Tensor* result;
       AllocatorAttributes attr;
       if (element_dtype_ == DT_VARIANT) {
@@ -327,60 +345,83 @@ class TensorListConcat : public OpKernel {
         errors::InvalidArgument(
             "Invalid data types; op elements ", DataTypeString(element_dtype_),
             " but list elements ", DataTypeString(tensor_list->element_dtype)));
-    // If the TensorList is empty, its element_shape must be fully defined
-    // except for the first dimension.
-    if (!element_shape_except_first_dim_.IsFullyDefined()) {
-      if (!tensor_list->element_shape.unknown_rank()) {
-        OP_REQUIRES(c, tensor_list->element_shape.dims() >= 1,
-                    errors::InvalidArgument(
-                        "Concat requires elements to be at least vectors, ",
-                        "found scalars instead."));
-        PartialTensorShape shape_except_first_dim(
-            gtl::ArraySlice<int64>(tensor_list->element_shape.dim_sizes())
-                .subspan(1));
-        PartialTensorShape tmp = element_shape_except_first_dim_;
-        OP_REQUIRES_OK(c, tmp.MergeWith(shape_except_first_dim,
-                                        &element_shape_except_first_dim_));
-      }
+    // The leading dimension of all list elements if they are all the same.
+    // This is used as the leading dim of uninitialized tensors in the list
+    // if leading_dims is not provided.
+    int64 first_dim = -1;
+    if (c->num_inputs() > 1) {
+      // TensorListConcatV2
+      PartialTensorShape element_shape;
+      OP_REQUIRES_OK(
+          c, GetElementShapeFromInput(c, *tensor_list, 1, &element_shape));
+      OP_REQUIRES(c, element_shape.unknown_rank() || element_shape.dims() >= 1,
+                  errors::InvalidArgument(
+                      "Concat requires elements to be at least vectors, ",
+                      "found scalars instead."));
+      // Split `element_shape` into `first_dim` and
+      // `element_shape_except_first_dim_`.
+      first_dim = element_shape.dim_size(0);
+      element_shape_except_first_dim_ = element_shape;
+      element_shape_except_first_dim_.RemoveDim(0);
     }
+    // If the TensorList is empty, element_shape_except_first_dim_ must be fully
+    // defined.
     OP_REQUIRES(c,
                 !tensor_list->tensors.empty() ||
                     element_shape_except_first_dim_.IsFullyDefined(),
                 errors::InvalidArgument(
                     "All except the first dimension must be fully defined ",
                     "when concating an empty tensor list. element_shape: ",
-                    tensor_list->element_shape.DebugString()));
-    // 1. Compute the shape of the output tensor.
-    // If `element_shape_except_first_dim_` is fully-defined we just prepend the
-    // leading dim to it. Otherwise we use the shape of the first element tensor
-    // and check to make sure shapes of all tensors are compatible.
-    TensorShape output_shape;
-    if (!element_shape_except_first_dim_.AsTensorShape(&output_shape)) {
-      const Tensor& element_tensor = tensor_list->tensors[0];
-      OP_REQUIRES(
-          c, TensorShapeUtils::IsVectorOrHigher(element_tensor.shape()),
-          errors::InvalidArgument("Concat saw a scalar shape at index ", 0,
-                                  " but requires at least vectors."));
-      output_shape =
-          TensorShape(gtl::ArraySlice<int64>(element_tensor.shape().dim_sizes())
-                          .subspan(1));
-      for (int i = 1; i < tensor_list->tensors.size(); ++i) {
-        const Tensor& element_tensor = tensor_list->tensors[i];
-        OP_REQUIRES(
-            c, TensorShapeUtils::IsVectorOrHigher(element_tensor.shape()),
-            errors::InvalidArgument("Concat saw a scalar shape at index ", i,
-                                    " but requires at least vectors."));
-        TensorShape actual_shape(
-            gtl::ArraySlice<int64>(element_tensor.shape().dim_sizes())
-                .subspan(1));
-        OP_REQUIRES(c, actual_shape.dim_sizes() == output_shape.dim_sizes(),
-                    errors::InvalidArgument(
-                        "Tried to concat tensors with unequal shapes: ",
-                        output_shape.DebugString(), " vs ",
-                        actual_shape.DebugString()));
+                    element_shape_except_first_dim_.DebugString()));
+    // 1. Check that `element_shape_except_first_dim_` input tensor is
+    //    compatible with the shapes of element tensors.
+    // 2. Check that the elements have the same shape except the first dim.
+    // 3. If `first_dim` is known, check that it is compatible with the leading
+    //    dims of all elements.
+    // 4. If `first_dim` is unknown (-1), check whether all initialized
+    //    elements have the same leading dim and if so set `first_dim` to that
+    //    value.
+    if (!tensor_list->element_shape.IsFullyDefined()) {
+      bool check_dim = (first_dim == -1);
+      int64 inferred_first_dim = first_dim;
+      for (int i = 0; i < tensor_list->tensors.size(); ++i) {
+        const Tensor& t = tensor_list->tensors[i];
+        if (t.dtype() != DT_INVALID) {
+          PartialTensorShape tmp = element_shape_except_first_dim_;
+          OP_REQUIRES(
+              c, TensorShapeUtils::IsVectorOrHigher(t.shape()),
+              errors::InvalidArgument("Concat saw a scalar shape at index ", i,
+                                      " but requires at least vectors."));
+          TensorShape shape_except_first_dim = TensorShape(
+              gtl::ArraySlice<int64>(t.shape().dim_sizes()).subspan(1));
+          OP_REQUIRES_OK(c, tmp.MergeWith(shape_except_first_dim,
+                                          &element_shape_except_first_dim_));
+          OP_REQUIRES(c, first_dim == -1 || first_dim == t.shape().dim_size(0),
+                      errors::InvalidArgument(
+                          "First entry of element_shape input does not match ",
+                          "the first dim of list element at index: ", i,
+                          " Expected: ", first_dim,
+                          " Actual: ", t.shape().dim_size(0)));
+          if (check_dim) {
+            if (inferred_first_dim == -1) {
+              inferred_first_dim = t.shape().dim_size(0);
+            } else if (inferred_first_dim != t.shape().dim_size(0)) {
+              inferred_first_dim = -1;
+              check_dim = false;
+            }
+          }
+        }
       }
+      first_dim = inferred_first_dim;
     }
-    // 2. Build the lengths_tensor and leading dim of the output tensor by
+    TensorShape output_shape;
+    OP_REQUIRES(
+        c, element_shape_except_first_dim_.AsTensorShape(&output_shape),
+        errors::InvalidArgument(
+            "Trying to concat list with only uninitialized tensors ",
+            "but element_shape_except_first_dim_ is not fully defined: ",
+            element_shape_except_first_dim_.DebugString()));
+    // Build the lengths_tensor and leading dim of the output tensor by
     // iterating over all element tensors.
     Tensor* lengths_tensor = nullptr;
     OP_REQUIRES_OK(
@@ -391,13 +432,36 @@ class TensorListConcat : public OpKernel {
     auto lengths_tensor_vec = lengths_tensor->vec<int64>();
     int64 leading_dim = 0;
     for (size_t i = 0; i < tensor_list->tensors.size(); i++) {
-      int64 dim = tensor_list->tensors[i].shape().dim_size(0);
+      int64 dim;
+      if (tensor_list->tensors[i].dtype() != DT_INVALID) {
+        dim = tensor_list->tensors[i].shape().dim_size(0);
+      } else {
+        // If leading_dims is not provided or does not contain an entry for
+        // index i use the inferred `first_dim` if set.
+        if ((c->num_inputs() <= 2 || i >= c->input(2).NumElements()) &&
+            first_dim != -1) {
+          dim = first_dim;
+        } else {
+          OP_REQUIRES(c, c->num_inputs() > 2,
+                      errors::InvalidArgument(
+                          "Concating lists with uninitialized tensors is not ",
+                          "supported in this version of TensorListConcat. ",
+                          "Consider updating your GraphDef to run the newer ",
+                          "version."));
+          OP_REQUIRES(c, i < c->input(2).NumElements(),
+                      errors::InvalidArgument(
+                          "List contains uninitialized tensor at index ", i,
+                          " but leading_dims has only ",
+                          c->input(2).NumElements(), " elements."));
+          dim = c->input(2).vec<int64>()(i);
+        }
+      }
       leading_dim += dim;
       lengths_tensor_vec(i) = dim;
     }
     output_shape.InsertDim(0, leading_dim);
     Tensor* output;
-    // 3. Allocate the output tensor and fill it up with the concated element
+    // Allocate the output tensor and fill it up with the concated element
     // tensors.
     OP_REQUIRES_OK(c, c->allocate_output(0, output_shape, &output));
     if (output->NumElements() == 0) {
@@ -406,9 +470,31 @@ class TensorListConcat : public OpKernel {
 
     ConstMatrixVector inputs_flat;
     inputs_flat.reserve(tensor_list->tensors.size());
-    for (const auto& element_tensor : tensor_list->tensors) {
-      inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
-          element_tensor.shaped<T, 2>({1, element_tensor.NumElements()})));
+    // Store the zeros tensors in a vector to prevent them from being GC'ed till
+    // concat is complete.
+    std::vector<Tensor> zeros_vec;
+    for (int i = 0; i < tensor_list->tensors.size(); i++) {
+      const Tensor& element_tensor = tensor_list->tensors[i];
+      if (element_tensor.dtype() != DT_INVALID) {
+        inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
+            element_tensor.shaped<T, 2>({1, element_tensor.NumElements()})));
+      } else {
+        AllocatorAttributes attr;
+        if (element_dtype_ == DT_VARIANT) {
+          attr.set_on_host(true);
+        }
+        TensorShape element_shape = output_shape;
+        element_shape.set_dim(0, lengths_tensor_vec(i));
+        zeros_vec.emplace_back();
+        Tensor& zeros = zeros_vec.back();
+        OP_REQUIRES_OK(
+            c, c->allocate_temp(element_dtype_, element_shape, &zeros, attr));
+        functor::SetZeroFunctor<Device, T>()(c->eigen_device<Device>(),
+                                             zeros.flat<T>());
+        inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
+            const_cast<const Tensor&>(zeros).shaped<T, 2>(
+                {1, zeros.NumElements()})));
+      }
     }
     auto output_flat = output->shaped<T, 2>({1, output->NumElements()});
 
@@ -522,7 +608,7 @@ class TensorListGather : public OpKernel {
         errors::InvalidArgument(
             "Invalid data types; op elements ", DataTypeString(element_dtype_),
             " but list elements ", DataTypeString(tensor_list->element_dtype)));
-    Tensor indices = c->input(1);
+    const Tensor& indices = c->input(1);
     PartialTensorShape partial_element_shape;
     OP_REQUIRES_OK(c, GetElementShapeFromInput(c, *tensor_list, 2,
                                                &partial_element_shape));
diff --git a/tensorflow/core/ops/list_ops.cc b/tensorflow/core/ops/list_ops.cc
index fdaa5a2129..f8fdb10bae 100644
--- a/tensorflow/core/ops/list_ops.cc
+++ b/tensorflow/core/ops/list_ops.cc
@@ -209,6 +209,42 @@ REGISTER_OP("TensorListStack")
       return Status::OK();
     });
 
+Status TensorListConcatShapeInference(
+    shape_inference::InferenceContext* c,
+    shape_inference::ShapeHandle element_shape) {
+  DataType element_dtype;
+  TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype));
+  auto* handle_data = c->input_handle_shapes_and_types(0);
+  if (handle_data != nullptr && handle_data->size() > 1) {
+    return errors::InvalidArgument(
+        "Trying to read from list with wrong variant data.");
+  }
+  if (handle_data != nullptr && handle_data->size() == 1) {
+    const shape_inference::ShapeAndType& list_shape_type = (*handle_data)[0];
+    if (list_shape_type.dtype != element_dtype) {
+      return errors::InvalidArgument(
+          "Trying to read from list with wrong element dtype. List has "
+          "type ",
+          DataTypeString(list_shape_type.dtype), " but expected type ",
+          DataTypeString(element_dtype));
+    }
+    shape_inference::ShapeHandle merged;
+    TF_RETURN_IF_ERROR(c->Merge(element_shape, list_shape_type.shape, &merged));
+    element_shape = merged;
+  }
+  if (c->RankKnown(element_shape)) {
+    shape_inference::ShapeHandle result;
+    TF_RETURN_IF_ERROR(c->Subshape(element_shape, 1, &result));
+    TF_RETURN_IF_ERROR(
+        c->Concatenate(c->MakeShape({c->UnknownDim()}), result, &result));
+    c->set_output(0, result);
+  } else {
+    c->set_output(0, c->UnknownShape());
+  }
+  c->set_output(1, c->MakeShape({c->UnknownDim()}));
+  return Status::OK();
+}
+
 REGISTER_OP("TensorListConcat")
     .Input("input_handle: variant")
     .Output("tensor: element_dtype")
@@ -216,45 +252,27 @@ REGISTER_OP("TensorListConcat")
     .Attr("element_dtype: type")
     .Attr("element_shape: shape = { unknown_rank: true }")
     .SetShapeFn([](shape_inference::InferenceContext* c) {
-      DataType element_dtype;
-      TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype));
       PartialTensorShape raw_element_shape;
       TF_RETURN_IF_ERROR(c->GetAttr("element_shape", &raw_element_shape));
       shape_inference::ShapeHandle element_shape;
       TF_RETURN_IF_ERROR(c->MakeShapeFromPartialTensorShape(raw_element_shape,
                                                             &element_shape));
+      return TensorListConcatShapeInference(c, element_shape);
+    });
 
-      auto* handle_data = c->input_handle_shapes_and_types(0);
-      if (handle_data != nullptr && handle_data->size() > 1) {
-        return errors::InvalidArgument(
-            "Trying to read from list with wrong variant data.");
-      }
-      if (handle_data != nullptr && handle_data->size() == 1) {
-        const shape_inference::ShapeAndType& list_shape_type =
-            (*handle_data)[0];
-        if (list_shape_type.dtype != element_dtype) {
-          return errors::InvalidArgument(
-              "Trying to read from list with wrong element dtype. List has "
-              "type ",
-              DataTypeString(list_shape_type.dtype), " but expected type ",
-              DataTypeString(element_dtype));
-        }
-        shape_inference::ShapeHandle merged;
-        TF_RETURN_IF_ERROR(
-            c->Merge(element_shape, list_shape_type.shape, &merged));
-        element_shape = merged;
-      }
-      if (c->RankKnown(element_shape)) {
-        shape_inference::ShapeHandle result;
-        TF_RETURN_IF_ERROR(c->Subshape(element_shape, 1, &result));
-        TF_RETURN_IF_ERROR(
-            c->Concatenate(c->MakeShape({c->UnknownDim()}), result, &result));
-        c->set_output(0, result);
-      } else {
-        c->set_output(0, c->UnknownShape());
-      }
-      c->set_output(1, c->MakeShape({c->UnknownDim()}));
-      return Status::OK();
+REGISTER_OP("TensorListConcatV2")
+    .Input("input_handle: variant")
+    .Input("element_shape: shape_type")
+    .Input("leading_dims: int64")
+    .Output("tensor: element_dtype")
+    .Output("lengths: int64")
+    .Attr("element_dtype: type")
+    .Attr("shape_type: {int32, int64}")
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      shape_inference::ShapeHandle element_shape;
+      TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensorTreatScalarAsUnknownShape(
+          1, &element_shape));
+      return TensorListConcatShapeInference(c, element_shape);
     });
 
 REGISTER_OP("TensorListSplit")
diff --git a/tensorflow/python/kernel_tests/list_ops_test.py b/tensorflow/python/kernel_tests/list_ops_test.py
index e203d1b909..7d4c045f79 100644
--- a/tensorflow/python/kernel_tests/list_ops_test.py
+++ b/tensorflow/python/kernel_tests/list_ops_test.py
@@ -1235,9 +1235,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     l = list_ops.tensor_list_push_back(l, [[0., 1.]])
     l = list_ops.tensor_list_push_back(l, [[2.], [4.]])
     with self.assertRaisesRegexp(
-        errors.InvalidArgumentError,
-        r"Tried to concat tensors with unequal shapes: "
-        r"\[2\] vs \[1\]"):
+        errors.InvalidArgumentError, r"Incompatible shapes during merge: "
+        r"\[2\] vs. \[1\]"):
       t = list_ops.tensor_list_concat(l, element_dtype=dtypes.float32)
       self.evaluate(t)
 
@@ -1298,6 +1297,65 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       t = list_ops.tensor_list_concat(l1, element_dtype=dtypes.float32)
       self.evaluate(t)
 
+  def testConcatWithUninitializedTensorsUseListElementShape(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=[2, 3], num_elements=3)
+    t = list_ops.tensor_list_concat(l, element_dtype=dtypes.float32)
+    self.assertAllEqual(np.zeros((6, 3)), t)
+
+  def testConcatWithUninitializedTensorsUseProvidedElementShape(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
+    t = list_ops.tensor_list_concat(
+        l, element_dtype=dtypes.float32, element_shape=(2, 3))
+    self.assertAllEqual(np.zeros((6, 3)), t)
+
+  def testConcatWithUninitializedTensorsUseProvidedElementShapeAndLengths(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
+    t, _ = gen_list_ops.tensor_list_concat_v2(
+        l,
+        element_dtype=dtypes.float32,
+        element_shape=list_ops._build_element_shape((None, 3)),
+        leading_dims=[2, 3, 5])
+    self.assertAllEqual(np.zeros((10, 3)), t)
+    l = list_ops.tensor_list_set_item(l, 1, [[2., 3.], [4., 5.], [6., 7.]])
+    t, _ = gen_list_ops.tensor_list_concat_v2(
+        l,
+        element_dtype=dtypes.float32,
+        element_shape=list_ops._build_element_shape((None, 2)),
+        leading_dims=[2, 3, 4])
+    self.assertAllEqual([[0., 0.], [0., 0.], [2., 3.], [4., 5.], [6., 7.],
+                         [0., 0.], [0., 0.], [0., 0.], [0., 0.]], t)
+
+  def testConcatWithUninitializedTensorsInferShapeFromElements(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
+    l = list_ops.tensor_list_set_item(l, 1, [[2., 3.], [4., 5.], [6., 7.]])
+    t = list_ops.tensor_list_concat(l, element_dtype=dtypes.float32)
+    self.assertAllEqual([[0., 0.], [0., 0.], [0., 0.], [2., 3.], [4., 5.],
+                         [6., 7.], [0., 0.], [0., 0.], [0., 0.]], t)
+
+  def testConcatWithUninitializedTensorsFailsIfNoElementShape(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        r"Trying to concat list with only uninitialized tensors "
+        r"but element_shape_except_first_dim_ is not fully defined"):
+      t = list_ops.tensor_list_concat(l, element_dtype=dtypes.float32)
+      self.evaluate(t)
+
+  def testConcatWithUninitializedTensorsFailsIfNoInputLengths(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=[None, 3], num_elements=3)
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        r"List contains uninitialized tensor at index 0"
+        r" but leading_dims has only 0 elements."):
+      t = list_ops.tensor_list_concat(l, element_dtype=dtypes.float32)
+      self.evaluate(t)
+
   def testEvenSplit(self):
 
     def RunTest(input_tensor, lengths, expected_stacked_output):
diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
index d0efb47a38..3b2f476e0c 100644
--- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py
+++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
@@ -185,7 +185,6 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual([[0.0, 0.0], [4.0, 5.0], [0.0, 0.0]],
                           self.evaluate(ta.write(1, [[4.0, 5.0]]).concat()))
 
-  @test_util.disable_control_flow_v2("b/122324791")
   @test_util.run_v1_only("b/122324791")
   def testTensorArrayReadOrPackNotAllValuesAvailableFillsZeros(self):
     self._testTensorArrayReadOrPackNotAllValuesAvailableFillsZeros()
@@ -202,11 +201,21 @@ class TensorArrayTest(test.TestCase):
     self.assertAllEqual([[0.0, 0.0], [4.0, 5.0], [0.0, 0.0]],
                         self.evaluate(ta.write(1, [[4.0, 5.0]]).concat()))
 
-  @test_util.disable_control_flow_v2("b/122324791")
   @test_util.run_v1_only("b/122324791")
   def testTensorArrayReadOrPackNotAllValuesAvailableInferShapeFillsZeros(self):
     self._testTensorArrayReadOrPackNotAllValuesAvailableInferShapeFillsZeros()
 
+  @test_util.run_v1_only("Uses placeholders")
+  def testSkipEagerTensorArrayReadUninitializedInferShapeFillsZeros(self):
+    with self.cached_session(use_gpu=True) as sess:
+      ta = tensor_array_ops.TensorArray(
+          dtype=dtypes.float32,
+          tensor_array_name="foo",
+          size=3)
+      val = array_ops.placeholder(dtypes.float32)
+      self.assertAllEqual(
+          [[0.0, 0.0]], sess.run(ta.write(1, val).read(0), {val: [[4.0, 5.0]]}))
+
   def _testTensorArrayUnpackRead(self, tf_dtype):
     with self.cached_session(use_gpu=True):
       convert = _make_converter(tf_dtype)
@@ -531,7 +540,10 @@ class TensorArrayTest(test.TestCase):
 
       # The exact error messages differ between eager execution and graph
       # construction as the former bubbles up the error from array_op.concat.
-      with self.assertRaisesOpError("shape"):
+      error_msg = ("Incompatible ranks"
+                   if control_flow_util.ENABLE_CONTROL_FLOW_V2 and
+                   not context.executing_eagerly() else "shape")
+      with self.assertRaisesRegexp(errors.InvalidArgumentError, error_msg):
         self.evaluate(w3.concat())
 
   def testTensorArraySplitIncompatibleShapesFails(self):
diff --git a/tensorflow/python/ops/list_ops.py b/tensorflow/python/ops/list_ops.py
index fcd5b816fe..87409eb2ac 100644
--- a/tensorflow/python/ops/list_ops.py
+++ b/tensorflow/python/ops/list_ops.py
@@ -71,11 +71,12 @@ def tensor_list_from_tensor(tensor, element_shape, name=None):
       name=name)
 
 
-def tensor_list_get_item(input_handle, index, element_dtype, name=None):
+def tensor_list_get_item(input_handle, index, element_dtype, element_shape=None,
+                         name=None):
   return gen_list_ops.tensor_list_get_item(
       input_handle=input_handle,
       index=index,
-      element_shape=-1,
+      element_shape=_build_element_shape(element_shape),
       element_dtype=element_dtype,
       name=name)
 
@@ -119,9 +120,12 @@ def tensor_list_concat(input_handle, element_dtype, element_shape=None,
                        name=None):
   # Ignore the lengths output of TensorListConcat. It is only used during
   # gradient computation.
-  return gen_list_ops.tensor_list_concat(
-      input_handle=input_handle, element_dtype=element_dtype,
-      element_shape=element_shape, name=name)[0]
+  return gen_list_ops.tensor_list_concat_v2(
+      input_handle=input_handle,
+      element_dtype=element_dtype,
+      element_shape=_build_element_shape(element_shape),
+      leading_dims=ops.convert_to_tensor([], dtype=dtypes.int64),
+      name=name)[0]
 
 
 def tensor_list_split(tensor, element_shape, lengths, name=None):
@@ -175,22 +179,30 @@ def _TensorListStackGrad(unused_op, dtensor):
 
 
 @ops.RegisterGradient("TensorListConcat")
+@ops.RegisterGradient("TensorListConcatV2")
 def _TensorListConcatGrad(op, dtensor, unused_dlengths):
-  # TODO(srbs): We lose the element_shape information in tensor_list_concat.
-  # Consider providing that as an output of TensorListConcat?
-  if dtensor.shape.rank is None:
-    element_shape = None
-  else:
-    element_shape = [None] + dtensor.shape.as_list()[1:]
-  return tensor_list_split(
+  """Gradient function for TensorListConcat."""
+  dlist = tensor_list_split(
       dtensor,
-      element_shape=_build_element_shape(element_shape),
+      element_shape=gen_list_ops.tensor_list_element_shape(
+          op.inputs[0], shape_type=dtypes.int32),
       lengths=op.outputs[1])
+  if op.type == "TensorListConcatV2":
+    return dlist, None, None
+  else:
+    return dlist
 
 
 @ops.RegisterGradient("TensorListSplit")
 def _TensorListSplitGrad(op, dlist):
-  return tensor_list_concat(dlist, element_dtype=op.inputs[0].dtype), None, None
+  tensor, _, lengths = op.inputs
+  element_shape = array_ops.slice(array_ops.shape(tensor), [1], [-1])
+  element_shape = array_ops.concat([[-1], element_shape], axis=0)
+  return gen_list_ops.tensor_list_concat_v2(
+      dlist,
+      element_shape=element_shape,
+      leading_dims=lengths,
+      element_dtype=op.inputs[0].dtype)[0], None, None
 
 
 @ops.RegisterGradient("TensorListFromTensor")
@@ -238,7 +250,7 @@ def _TensorListSetItemGrad(op, dlist):
   list_grad = gen_list_ops.tensor_list_set_item(
       dlist, index=index, item=array_ops.zeros_like(item))
   index_grad = None
-  element_grad = gen_list_ops.tensor_list_get_item(
+  element_grad = tensor_list_get_item(
       dlist,
       index,
       element_shape=array_ops.shape(item),
@@ -317,4 +329,13 @@ def _build_element_shape(shape):
   if not shape:
     return ops.convert_to_tensor(shape, dtype=dtypes.int32)
   # Shape is a sequence of dimensions. Convert None dims to -1.
-  return [d if d is not None else -1 for d in shape]
+  def convert(val):
+    if val is None:
+      return -1
+    if isinstance(val, ops.Tensor):
+      return val
+    if isinstance(val, tensor_shape.Dimension):
+      return val.value if val.value is not None else -1
+    return val
+
+  return [convert(d) for d in shape]
diff --git a/tensorflow/python/ops/tensor_array_ops.py b/tensorflow/python/ops/tensor_array_ops.py
index 41a481430f..8e272daf6f 100644
--- a/tensorflow/python/ops/tensor_array_ops.py
+++ b/tensorflow/python/ops/tensor_array_ops.py
@@ -542,10 +542,15 @@ class _GraphTensorArrayV2(object):
 
   def read(self, index, name=None):
     """See TensorArray."""
+    if self._element_shape:
+      element_shape = self._element_shape[0]
+    else:
+      element_shape = tensor_shape.TensorShape(None)
     value = list_ops.tensor_list_get_item(
         input_handle=self._flow,
         index=index,
         element_dtype=self._dtype,
+        element_shape=element_shape,
         name=name)
     if self._element_shape:
       value.set_shape(self._element_shape[0].dims)
-- 
GitLab


From e8811ffb5cdce562281e1b474ea6e49fa6d52e67 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Thu, 31 Jan 2019 15:13:01 -0800
Subject: [PATCH 1699/2345] Support both binary and -1/1 label input in v2
 hinge and squared hinge losses.

PiperOrigin-RevId: 231863307
---
 tensorflow/python/keras/losses.py       |  59 ++++++-
 tensorflow/python/keras/losses_test.py  | 198 +++++++++++++++++-------
 tensorflow/python/keras/metrics.py      |  28 +++-
 tensorflow/python/keras/metrics_test.py |  84 +++++++---
 4 files changed, 277 insertions(+), 92 deletions(-)

diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index e83b805ce4..366469d9dd 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -23,6 +23,7 @@ import abc
 
 import six
 
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import smart_cond
 from tensorflow.python.keras import backend as K
@@ -31,6 +32,7 @@ from tensorflow.python.keras.utils.generic_utils import serialize_keras_object
 from tensorflow.python.keras.utils.losses_utils import compute_weighted_loss
 from tensorflow.python.keras.utils.tf_utils import is_tensor_or_variable
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
 from tensorflow.python.ops.losses import losses_impl
@@ -445,12 +447,18 @@ class SparseCategoricalCrossentropy(LossFunctionWrapper):
 class Hinge(LossFunctionWrapper):
   """Computes the hinge loss between `y_true` and `y_pred`.
 
+  `y_true` values are expected to be -1 or 1. If binary (0 or 1) labels are
+  provided we will convert them to -1 or 1.
+
   Usage:
 
   ```python
   h = tf.losses.Hinge()
-  loss = h([0., 1., 1.], [1., 0., 1.])
-  print('Loss: ', loss.numpy())  # Loss: 0.66
+  loss = h([-1., 1., 1.], [0.6, -0.7, -0.5])
+
+  # loss = max(0, 1 - y_true * y_pred) = [1.6 + 1.7 + 1.5] / 3
+
+  print('Loss: ', loss.numpy())  # Loss: 1.6
   ```
 
   Usage with tf.keras API:
@@ -471,12 +479,18 @@ class Hinge(LossFunctionWrapper):
 class SquaredHinge(LossFunctionWrapper):
   """Computes the squared hinge loss between `y_true` and `y_pred`.
 
+  `y_true` values are expected to be -1 or 1. If binary (0 or 1) labels are
+  provided we will convert them to -1 or 1.
+
   Usage:
 
   ```python
   sh = tf.losses.SquaredHinge()
-  loss = sh([0., 1., 1.], [1., 0., 1.])
-  print('Loss: ', loss.numpy())  # Loss: 0.66
+  loss = sh([-1., 1., 1.], [0.6, -0.7, -0.5])
+
+  # loss = (max(0, 1 - y_true * y_pred))^2 = [1.6^2 + 1.7^2 + 1.5^2] / 3
+
+  print('Loss: ', loss.numpy())  # Loss: 2.566666
   ```
 
   Usage with tf.keras API:
@@ -729,18 +743,55 @@ def mean_squared_logarithmic_error(y_true, y_pred):  # pylint: disable=missing-d
   return K.mean(math_ops.squared_difference(first_log, second_log), axis=-1)
 
 
+def _maybe_convert_labels(y_true):
+  """Converts binary labels into -1/1."""
+  are_zeros = math_ops.equal(y_true, 0)
+  are_ones = math_ops.equal(y_true, 1)
+  is_binary = math_ops.reduce_all(math_ops.logical_or(are_zeros, are_ones))
+
+  def _convert_binary_labels():
+    # Convert the binary labels to -1 or 1.
+    return 2. * y_true - 1.
+
+  updated_y_true = smart_cond.smart_cond(is_binary,
+                                         _convert_binary_labels, lambda: y_true)
+  return updated_y_true
+
+
 @keras_export('keras.metrics.squared_hinge', 'keras.losses.squared_hinge')
 def squared_hinge(y_true, y_pred):
+  """Computes the squared hinge loss between `y_true` and `y_pred`.
+
+  Args:
+    y_true: The ground truth values. `y_true` values are expected to be -1 or 1.
+      If binary (0 or 1) labels are provided we will convert them to -1 or 1.
+    y_pred: The predicted values.
+
+  Returns:
+    Tensor with one scalar loss entry per sample.
+  """
   y_pred = ops.convert_to_tensor(y_pred)
   y_true = math_ops.cast(y_true, y_pred.dtype)
+  y_true = _maybe_convert_labels(y_true)
   return K.mean(
       math_ops.square(math_ops.maximum(1. - y_true * y_pred, 0.)), axis=-1)
 
 
 @keras_export('keras.metrics.hinge', 'keras.losses.hinge')
 def hinge(y_true, y_pred):
+  """Computes the hinge loss between `y_true` and `y_pred`.
+
+  Args:
+    y_true: The ground truth values. `y_true` values are expected to be -1 or 1.
+      If binary (0 or 1) labels are provided we will convert them to -1 or 1.
+    y_pred: The predicted values.
+
+  Returns:
+    Tensor with one scalar loss entry per sample.
+  """
   y_pred = ops.convert_to_tensor(y_pred)
   y_true = math_ops.cast(y_true, y_pred.dtype)
+  y_true = _maybe_convert_labels(y_true)
   return K.mean(math_ops.maximum(1. - y_true * y_pred, 0.), axis=-1)
 
 
diff --git a/tensorflow/python/keras/losses_test.py b/tensorflow/python/keras/losses_test.py
index 04dd71241e..e72209059b 100644
--- a/tensorflow/python/keras/losses_test.py
+++ b/tensorflow/python/keras/losses_test.py
@@ -930,54 +930,93 @@ class HingeTest(test.TestCase):
 
   def test_unweighted(self):
     hinge_obj = keras.losses.Hinge()
-    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
-    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
-                                  shape=(2, 3),
-                                  dtype=dtypes.float32)
+    y_true = constant_op.constant([[0, 1, 0, 1], [0, 0, 1, 1]])
+    y_pred = constant_op.constant([[-0.3, 0.2, -0.1, 1.6],
+                                   [-0.25, -1., 0.5, 0.6]])
+
+    # loss = max(0, 1-y_true * y_pred), where y_true is -1/1
+
+    # y_true = [[-1, 1, -1, 1], [-1, -1, 1, 1]]
+    # y_true * y_pred = [[0.3, 0.2, 0.1, 1.6], [0.25, 1, 0.5, 0.6]]
+    # 1 - y_true * y_pred = [[0.7, 0.8, 0.9, -0.6], [0.75, 0, 0.5, 0.4]]
+    # loss = [(0.7 + 0.8 + 0.9 + 0) / 4, (0.75 + 0 + 0.5 + 0.4) / 4]
+    #      = [0.6, 0.4125]
+    # reduced loss = (0.6 + 0.4125) / 2
+
     loss = hinge_obj(y_true, y_pred)
-    self.assertAlmostEqual(self.evaluate(loss), 7.3333, 3)
+    self.assertAllClose(0.506, self.evaluate(loss), atol=1e-3)
 
   def test_scalar_weighted(self):
     hinge_obj = keras.losses.Hinge()
-    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
-    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
-                                  shape=(2, 3),
-                                  dtype=dtypes.float32)
+    y_true = constant_op.constant([[0, 1, 0, 1], [0, 0, 1, 1]])
+    y_pred = constant_op.constant([[-0.3, 0.2, -0.1, 1.6],
+                                   [-0.25, -1., 0.5, 0.6]])
+
+    # loss = max(0, 1-y_true * y_pred), where y_true is -1/1
+
+    # y_true = [[-1, 1, -1, 1], [-1, -1, 1, 1]]
+    # y_true * y_pred = [[0.3, 0.2, 0.1, 1.6], [0.25, 1, 0.5, 0.6]]
+    # 1 - y_true * y_pred = [[0.7, 0.8, 0.9, -0.6], [0.75, 0, 0.5, 0.4]]
+    # loss = [(0.7 + 0.8 + 0.9 + 0) / 4, (0.75 + 0 + 0.5 + 0.4) / 4]
+    #      = [0.6, 0.4125]
+    # weighted_loss = [0.6 * 2.3, 0.4125 * 2.3]
+    # reduced loss = (0.6 + 0.4125) * 2.3 / 2
+
     loss = hinge_obj(y_true, y_pred, sample_weight=2.3)
-    self.assertAlmostEqual(self.evaluate(loss), 16.8666, 3)
+    self.assertAlmostEqual(self.evaluate(loss), 1.164, 3)
 
     # Verify we get the same output when the same input is given
     loss_2 = hinge_obj(y_true, y_pred, sample_weight=2.3)
-    self.assertAlmostEqual(self.evaluate(loss), self.evaluate(loss_2), 3)
+    self.assertAllClose(self.evaluate(loss), self.evaluate(loss_2), 1e-3)
 
   def test_sample_weighted(self):
     hinge_obj = keras.losses.Hinge()
-    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
-    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
-                                  shape=(2, 3),
-                                  dtype=dtypes.float32)
+    y_true = constant_op.constant([[0, 1, 0, 1], [0, 0, 1, 1]])
+    y_pred = constant_op.constant([[-0.3, 0.2, -0.1, 1.6],
+                                   [-0.25, -1., 0.5, 0.6]])
+
+    # loss = max(0, 1-y_true * y_pred), where y_true is -1/1
+
+    # y_true = [[-1, 1, -1, 1], [-1, -1, 1, 1]]
+    # y_true * y_pred = [[0.3, 0.2, 0.1, 1.6], [0.25, 1, 0.5, 0.6]]
+    # 1 - y_true * y_pred = [[0.7, 0.8, 0.9, -0.6], [0.75, 0, 0.5, 0.4]]
+    # loss = [(0.7 + 0.8 + 0.9 + 0) / 4, (0.75 + 0 + 0.5 + 0.4) / 4]
+    #      = [0.6, 0.4125]
+    # weighted loss = [0.6 * 1.2, 0.4125 * 3.4]
+    # reduced loss = (0.6 * 1.2 + 0.4125 * 3.4) / 2
+
     sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
     loss = hinge_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAlmostEqual(self.evaluate(loss), 24.9333, 3)
+    self.assertAllClose(self.evaluate(loss), 1.061, 1e-3)
 
   def test_timestep_weighted(self):
     hinge_obj = keras.losses.Hinge()
-    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3, 1))
-    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
-                                  shape=(2, 3, 1),
-                                  dtype=dtypes.float32)
-    sample_weight = constant_op.constant([3, 6, 5, 0, 4, 2], shape=(2, 3))
+    y_true = constant_op.constant([[0, 1, 0, 1], [0, 0, 1, 1]], shape=(2, 4, 1))
+    y_pred = constant_op.constant(
+        [[-0.3, 0.2, -0.1, 1.6], [-0.25, -1., 0.5, 0.6]], shape=(2, 4, 1))
+    sample_weight = constant_op.constant([3, 6, 5, 0, 4, 2, 1, 3], shape=(2, 4))
+
+    # loss = max(0, 1-y_true * y_pred), where y_true is -1/1
+
+    # y_true = [[[-1], [1], [-1], [1]], [[-1], [-1], [1], [1]]]
+    # y_true * y_pred = [[[0.3], [0.2], [0.1], [1.6]],
+    #                    [[0.25], [1], [0.5], [0.6]]]
+    # 1 - y_true * y_pred = [[[0.7], [0.8], [0.9], [-0.6]],
+    #                        [[0.75], [0], [0.5], [0.4]]]
+    # loss = [[0.7, 0.8, 0.9, 0], [0.75, 0, 0.5, 0.4]]
+    # weighted loss    = [[2.1, 4.8, 4.5, 0], [3, 0, 0.5, 1.2]]
+    # reduced loss = (2.1 + 4.8 + 4.5 + 0 + 3 + 0 + 0.5 + 1.2) / 8
+
     loss = hinge_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAlmostEqual(self.evaluate(loss), 2.0, 3)
+    self.assertAllClose(self.evaluate(loss), 2.012, 1e-3)
 
   def test_zero_weighted(self):
     hinge_obj = keras.losses.Hinge()
-    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
-    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
-                                  shape=(2, 3),
-                                  dtype=dtypes.float32)
+    y_true = constant_op.constant([[0, 1, 0, 1], [0, 0, 1, 1]])
+    y_pred = constant_op.constant([[-0.3, 0.2, -0.1, 1.6],
+                                   [-0.25, -1., 0.5, 0.6]])
     loss = hinge_obj(y_true, y_pred, sample_weight=0)
-    self.assertAlmostEqual(self.evaluate(loss), 0., 3)
+    self.assertAllClose(self.evaluate(loss), 0., 1e-3)
 
 
 @test_util.run_all_in_graph_and_eager_modes
@@ -991,26 +1030,46 @@ class SquaredHingeTest(test.TestCase):
 
   def test_unweighted(self):
     sq_hinge_obj = keras.losses.SquaredHinge()
-    y_true = constant_op.constant([1, 9, 2, -5], shape=(2, 2))
-    y_pred = constant_op.constant([4, 8, 12, 8],
-                                  shape=(2, 2),
-                                  dtype=dtypes.float32)
+    y_true = constant_op.constant([[0, 1, 0, 1], [0, 0, 1, 1]])
+    y_pred = constant_op.constant([[-0.3, 0.2, -0.1, 1.6],
+                                   [-0.25, -1., 0.5, 0.6]])
+
+    # loss = max(0, 1-y_true * y_pred), where y_true is -1/1
+
+    # y_true = [[-1, 1, -1, 1], [-1, -1, 1, 1]]
+    # y_true * y_pred = [[0.3, 0.2, 0.1, 1.6], [0.25, 1, 0.5, 0.6]]
+    # 1 - y_true * y_pred = [[0.7, 0.8, 0.9, -0.6], [0.75, 0, 0.5, 0.4]]
+    # max(0, 1 - y_true * y_pred) = [[0.7, 0.8, 0.9, 0], [0.75, 0, 0.5, 0.4]]
+    # squared(max(0, 1 - y_true * y_pred)) = [[0.49, 0.64, 0.81, 0],
+    #                                         [0.5625, 0, 0.25, 0.16]]
+    # loss = [(0.49 + 0.64 + 0.81 + 0) / 4, (0.5625 + 0 + 0.25 + 0.16) / 4]
+    #      = [0.485, 0.2431]
+    # reduced loss = (0.485 + 0.2431) / 2
 
-    # Sq hinge = mean(square(max(1. - y_true * y_pred, 0.)), axis=-1)
-    # (1. - y_true * y_pred) = [[1-4, 1-72], [1-24, 1+40]] = [0, 48]
-    # sq(max(above val, 0)) = sq([[0, 0], [0, 41]) = [[0, 0], [0, 1681]]
-    # Mean = [0, 840.5]. Reduced loss = (0 + 840.5)/2 = 420.25
     loss = sq_hinge_obj(y_true, y_pred)
-    self.assertAlmostEqual(self.evaluate(loss), 420.25, 3)
+    self.assertAllClose(self.evaluate(loss), 0.364, 1e-3)
 
   def test_scalar_weighted(self):
     sq_hinge_obj = keras.losses.SquaredHinge()
-    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
-    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
-                                  shape=(2, 3),
-                                  dtype=dtypes.float32)
+    y_true = constant_op.constant([[0, 1, 0, 1], [0, 0, 1, 1]])
+    y_pred = constant_op.constant([[-0.3, 0.2, -0.1, 1.6],
+                                   [-0.25, -1., 0.5, 0.6]])
+
+    # loss = max(0, 1-y_true * y_pred), where y_true is -1/1
+
+    # y_true = [[-1, 1, -1, 1], [-1, -1, 1, 1]]
+    # y_true * y_pred = [[0.3, 0.2, 0.1, 1.6], [0.25, 1, 0.5, 0.6]]
+    # 1 - y_true * y_pred = [[0.7, 0.8, 0.9, -0.6], [0.75, 0, 0.5, 0.4]]
+    # max(0, 1 - y_true * y_pred) = [[0.7, 0.8, 0.9, 0], [0.75, 0, 0.5, 0.4]]
+    # squared(max(0, 1 - y_true * y_pred)) = [[0.49, 0.64, 0.81, 0],
+    #                                         [0.5625, 0, 0.25, 0.16]]
+    # loss = [(0.49 + 0.64 + 0.81 + 0) / 4, (0.5625 + 0 + 0.25 + 0.16) / 4]
+    #      = [0.485, 0.2431]
+    # weighted loss = [0.485 * 2.3, 0.2431 * 2.3]
+    # reduced loss = (0.485 + 0.2431) * 2.3 / 2
+
     loss = sq_hinge_obj(y_true, y_pred, sample_weight=2.3)
-    self.assertAlmostEqual(self.evaluate(loss), 647.833, 3)
+    self.assertAllClose(self.evaluate(loss), 0.837, 1e-3)
 
     # Verify we get the same output when the same input is given
     loss_2 = sq_hinge_obj(y_true, y_pred, sample_weight=2.3)
@@ -1018,32 +1077,55 @@ class SquaredHingeTest(test.TestCase):
 
   def test_sample_weighted(self):
     sq_hinge_obj = keras.losses.SquaredHinge()
-    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
-    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
-                                  shape=(2, 3),
-                                  dtype=dtypes.float32)
-    sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
+    y_true = constant_op.constant([[0, 1, 0, 1], [0, 0, 1, 1]])
+    y_pred = constant_op.constant([[-0.3, 0.2, -0.1, 1.6],
+                                   [-0.25, -1., 0.5, 0.6]])
+
+    # loss = max(0, 1-y_true * y_pred), where y_true is -1/1
+
+    # y_true = [[-1, 1, -1, 1], [-1, -1, 1, 1]]
+    # y_true * y_pred = [[0.3, 0.2, 0.1, 1.6], [0.25, 1, 0.5, 0.6]]
+    # 1 - y_true * y_pred = [[0.7, 0.8, 0.9, -0.6], [0.75, 0, 0.5, 0.4]]
+    # max(0, 1 - y_true * y_pred) = [[0.7, 0.8, 0.9, 0], [0.75, 0, 0.5, 0.4]]
+    # squared(max(0, 1 - y_true * y_pred)) = [[0.49, 0.64, 0.81, 0],
+    #                                         [0.5625, 0, 0.25, 0.16]]
+    # loss = [(0.49 + 0.64 + 0.81 + 0) / 4, (0.5625 + 0 + 0.25 + 0.16) / 4]
+    #      = [0.485, 0.2431]
+    # weighted loss = [0.485 * 1.2, 0.2431 * 3.4]
+    # reduced loss = (0.485 * 1.2 + 0.2431 * 3.4) / 2
+
+    sample_weight = constant_op.constant([1.2, 3.4])
     loss = sq_hinge_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAlmostEqual(self.evaluate(loss), 957.667, 3)
+    self.assertAllClose(self.evaluate(loss), 0.704, 1e-3)
 
   def test_timestep_weighted(self):
     sq_hinge_obj = keras.losses.SquaredHinge()
-    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3, 1))
-    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
-                                  shape=(2, 3, 1),
-                                  dtype=dtypes.float32)
-    sample_weight = constant_op.constant([3, 6, 5, 0, 4, 2], shape=(2, 3))
+    y_true = constant_op.constant([[0, 1, 0, 1], [0, 0, 1, 1]], shape=(2, 4, 1))
+    y_pred = constant_op.constant(
+        [[-0.3, 0.2, -0.1, 1.6], [-0.25, -1., 0.5, 0.6]], shape=(2, 4, 1))
+    sample_weight = constant_op.constant([3, 6, 5, 0, 4, 2, 1, 3], shape=(2, 4))
+
+    # loss = max(0, 1-y_true * y_pred), where y_true is -1/1
+
+    # y_true = [[[-1], [1], [-1], [1]], [[-1], [-1], [1], [1]]]
+    # y_true * y_pred = [[[0.3], [0.2], [0.1], [1.6]],
+    #                    [[0.25], [1], [0.5], [0.6]]]
+    # 1 - y_true * y_pred = [[[0.7], [0.8], [0.9], [-0.6]],
+    #                        [[0.75], [0], [0.5], [0.4]]]
+    # loss = [[0.49, 0.64, 0.81, 0], [0.5625, 0, 0.25, 0.16]]
+    # weighted loss    = [[1.47, 3.84, 4.05, 0], [2.25, 0, 0.25, 0.48]]
+    # reduced loss = (1.47 + 3.84 + 4.05 + 0 + 2.25 + 0 + 0.25 + 0.48) / 8
+
     loss = sq_hinge_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAlmostEqual(self.evaluate(loss), 6.0, 3)
+    self.assertAllClose(self.evaluate(loss), 1.542, 1e-3)
 
   def test_zero_weighted(self):
     sq_hinge_obj = keras.losses.SquaredHinge()
-    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
-    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
-                                  shape=(2, 3),
-                                  dtype=dtypes.float32)
+    y_true = constant_op.constant([[0, 1, 0, 1], [0, 0, 1, 1]])
+    y_pred = constant_op.constant([[-0.3, 0.2, -0.1, 1.6],
+                                   [-0.25, -1., 0.5, 0.6]])
     loss = sq_hinge_obj(y_true, y_pred, sample_weight=0)
-    self.assertAlmostEqual(self.evaluate(loss), 0., 3)
+    self.assertAllClose(self.evaluate(loss), 0., 1e-3)
 
 
 @test_util.run_all_in_graph_and_eager_modes
diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index ca0939e363..5d41a5c6f1 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -1977,15 +1977,21 @@ class MeanSquaredLogarithmicError(MeanMetricWrapper):
 class Hinge(MeanMetricWrapper):
   """Computes the hinge metric between `y_true` and `y_pred`.
 
-  For example, if `y_true` is [0., 1., 1.], and `y_pred` is [1., 0., 1.]
-  the hinge metric value is 0.66.
+  `y_true` values are expected to be -1 or 1. If binary (0 or 1) labels are
+  provided we will convert them to -1 or 1.
+
+  For example, if `y_true` is [-1., 1., 1.], and `y_pred` is [0.6, -0.7, -0.5]
+  the hinge metric value is 1.6.
 
   Usage:
 
   ```python
   m = tf.keras.metrics.Hinge()
-  m.update_state([0., 1., 1.], [1., 0., 1.])
-  print('Final result: ', m.result().numpy())  # Final result: 0.66
+  m.update_state([-1., 1., 1.], [0.6, -0.7, -0.5])
+
+  # result = max(0, 1-y_true * y_pred) = [1.6 + 1.7 + 1.5] / 3
+
+  print('Final result: ', m.result().numpy())  # Final result: 1.6
   ```
 
   Usage with tf.keras API:
@@ -2004,15 +2010,21 @@ class Hinge(MeanMetricWrapper):
 class SquaredHinge(MeanMetricWrapper):
   """Computes the squared hinge metric between `y_true` and `y_pred`.
 
-  For example, if `y_true` is [0., 1., 1.], and `y_pred` is [1., 0., 1.]
-  the squared hinge metric value is 0.66.
+  `y_true` values are expected to be -1 or 1. If binary (0 or 1) labels are
+  provided we will convert them to -1 or 1.
+
+  For example, if `y_true` is [-1., 1., 1.], and `y_pred` is [0.6, -0.7, -0.5]
+  the squared hinge metric value is 2.6.
 
   Usage:
 
   ```python
   m = tf.keras.metrics.SquaredHinge()
-  m.update_state([0., 1., 1.], [1., 0., 1.])
-  print('Final result: ', m.result().numpy())  # Final result: 0.66
+  m.update_state([-1., 1., 1.], [0.6, -0.7, -0.5])
+
+  # result = max(0, 1-y_true * y_pred) = [1.6^2 + 1.7^2 + 1.5^2] / 3
+
+  print('Final result: ', m.result().numpy())  # Final result: 2.6
   ```
 
   Usage with tf.keras API:
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index 7d1f88877c..4143066a1c 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -685,26 +685,43 @@ class HingeTest(test.TestCase):
   def test_unweighted(self):
     hinge_obj = metrics.Hinge()
     self.evaluate(variables.variables_initializer(hinge_obj.variables))
-    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
-                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
-    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
-                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
+    y_true = constant_op.constant([[0, 1, 0, 1], [0, 0, 1, 1]])
+    y_pred = constant_op.constant([[-0.3, 0.2, -0.1, 1.6],
+                                   [-0.25, -1., 0.5, 0.6]])
+
+    # metric = max(0, 1-y_true * y_pred), where y_true is -1/1
+
+    # y_true = [[-1, 1, -1, 1], [-1, -1, 1, 1]]
+    # y_true * y_pred = [[0.3, 0.2, 0.1, 1.6], [0.25, 1, 0.5, 0.6]]
+    # 1 - y_true * y_pred = [[0.7, 0.8, 0.9, -0.6], [0.75, 0, 0.5, 0.4]]
+    # metric = [(0.7 + 0.8 + 0.9 + 0) / 4, (0.75 + 0 + 0.5 + 0.4) / 4]
+    #        = [0.6, 0.4125]
+    # reduced metric = (0.6 + 0.4125) / 2
 
     update_op = hinge_obj.update_state(y_true, y_pred)
     self.evaluate(update_op)
     result = hinge_obj.result()
-    self.assertAllClose(0.65, result, atol=1e-5)
+    self.assertAllClose(0.506, result, atol=1e-3)
 
   def test_weighted(self):
     hinge_obj = metrics.Hinge()
     self.evaluate(variables.variables_initializer(hinge_obj.variables))
-    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
-                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
-    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
-                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
-    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
+    y_true = constant_op.constant([[-1, 1, -1, 1], [-1, -1, 1, 1]])
+    y_pred = constant_op.constant([[-0.3, 0.2, -0.1, 1.6],
+                                   [-0.25, -1., 0.5, 0.6]])
+    sample_weight = constant_op.constant([1.5, 2.])
+
+    # metric = max(0, 1-y_true * y_pred), where y_true is -1/1
+
+    # y_true * y_pred = [[0.3, 0.2, 0.1, 1.6], [0.25, 1, 0.5, 0.6]]
+    # 1 - y_true * y_pred = [[0.7, 0.8, 0.9, -0.6], [0.75, 0, 0.5, 0.4]]
+    # metric = [(0.7 + 0.8 + 0.9 + 0) / 4, (0.75 + 0 + 0.5 + 0.4) / 4]
+    #        = [0.6, 0.4125]
+    # weighted metric = [0.6 * 1.5, 0.4125 * 2]
+    # reduced metric = (0.6 * 1.5 + 0.4125 * 2) / (1.5 + 2)
+
     result = hinge_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAllClose(0.65714, self.evaluate(result), atol=1e-5)
+    self.assertAllClose(0.493, self.evaluate(result), atol=1e-3)
 
 
 @test_util.run_all_in_graph_and_eager_modes
@@ -723,26 +740,49 @@ class SquaredHingeTest(test.TestCase):
   def test_unweighted(self):
     sq_hinge_obj = metrics.SquaredHinge()
     self.evaluate(variables.variables_initializer(sq_hinge_obj.variables))
-    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
-                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
-    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
-                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
+    y_true = constant_op.constant([[0, 1, 0, 1], [0, 0, 1, 1]])
+    y_pred = constant_op.constant([[-0.3, 0.2, -0.1, 1.6],
+                                   [-0.25, -1., 0.5, 0.6]])
+
+    # metric = max(0, 1-y_true * y_pred), where y_true is -1/1
+
+    # y_true = [[-1, 1, -1, 1], [-1, -1, 1, 1]]
+    # y_true * y_pred = [[0.3, 0.2, 0.1, 1.6], [0.25, 1, 0.5, 0.6]]
+    # 1 - y_true * y_pred = [[0.7, 0.8, 0.9, -0.6], [0.75, 0, 0.5, 0.4]]
+    # max(0, 1 - y_true * y_pred) = [[0.7, 0.8, 0.9, 0], [0.75, 0, 0.5, 0.4]]
+    # squared(max(0, 1 - y_true * y_pred)) = [[0.49, 0.64, 0.81, 0],
+    #                                         [0.5625, 0, 0.25, 0.16]]
+    # metric = [(0.49 + 0.64 + 0.81 + 0) / 4, (0.5625 + 0 + 0.25 + 0.16) / 4]
+    #        = [0.485, 0.2431]
+    # reduced metric = (0.485 + 0.2431) / 2
 
     update_op = sq_hinge_obj.update_state(y_true, y_pred)
     self.evaluate(update_op)
     result = sq_hinge_obj.result()
-    self.assertAllClose(0.65, result, atol=1e-5)
+    self.assertAllClose(0.364, result, atol=1e-3)
 
   def test_weighted(self):
     sq_hinge_obj = metrics.SquaredHinge()
     self.evaluate(variables.variables_initializer(sq_hinge_obj.variables))
-    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
-                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
-    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
-                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
-    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
+    y_true = constant_op.constant([[-1, 1, -1, 1], [-1, -1, 1, 1]])
+    y_pred = constant_op.constant([[-0.3, 0.2, -0.1, 1.6],
+                                   [-0.25, -1., 0.5, 0.6]])
+    sample_weight = constant_op.constant([1.5, 2.])
+
+    # metric = max(0, 1-y_true * y_pred), where y_true is -1/1
+
+    # y_true * y_pred = [[0.3, 0.2, 0.1, 1.6], [0.25, 1, 0.5, 0.6]]
+    # 1 - y_true * y_pred = [[0.7, 0.8, 0.9, -0.6], [0.75, 0, 0.5, 0.4]]
+    # max(0, 1 - y_true * y_pred) = [[0.7, 0.8, 0.9, 0], [0.75, 0, 0.5, 0.4]]
+    # squared(max(0, 1 - y_true * y_pred)) = [[0.49, 0.64, 0.81, 0],
+    #                                         [0.5625, 0, 0.25, 0.16]]
+    # metric = [(0.49 + 0.64 + 0.81 + 0) / 4, (0.5625 + 0 + 0.25 + 0.16) / 4]
+    #        = [0.485, 0.2431]
+    # weighted metric = [0.485 * 1.5, 0.2431 * 2]
+    # reduced metric = (0.485 * 1.5 + 0.2431 * 2) / (1.5 + 2)
+
     result = sq_hinge_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAllClose(0.65714, self.evaluate(result), atol=1e-5)
+    self.assertAllClose(0.347, self.evaluate(result), atol=1e-3)
 
 
 @test_util.run_all_in_graph_and_eager_modes
-- 
GitLab


From 6d166402a2e4c566549bb0a0f8ea39f455bf06aa Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Thu, 31 Jan 2019 15:17:03 -0800
Subject: [PATCH 1700/2345] Cleanup: replace precision mode (currently an int)
 with a struct.

PiperOrigin-RevId: 231864132
---
 .../tf2tensorrt/convert/convert_graph.cc      | 20 +++++++++-------
 .../tf2tensorrt/convert/convert_graph.h       | 15 ++++++------
 .../tf2tensorrt/convert/convert_graph_test.cc |  5 ++--
 .../tf2tensorrt/convert/convert_nodes.cc      | 24 +++++++++----------
 .../tf2tensorrt/convert/convert_nodes.h       | 16 ++++++-------
 .../tf2tensorrt/convert/convert_nodes_test.cc |  8 +++----
 .../convert/trt_optimization_pass.cc          |  4 ++--
 .../convert/trt_optimization_pass.h           |  5 ++--
 .../compiler/tf2tensorrt/convert/utils.cc     | 23 +++++++++---------
 .../compiler/tf2tensorrt/convert/utils.h      |  9 +++----
 .../tf2tensorrt/kernels/trt_engine_op.cc      | 17 +++++++------
 11 files changed, 75 insertions(+), 71 deletions(-)

diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc
index 31d0b2dfa4..f93d69839c 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc
@@ -83,7 +83,8 @@ std::vector<int> GetLoadedTensorRTVersion() {
 }
 
 TrtCandidateSelector::TrtCandidateSelector(
-    const grappler::GraphProperties& graph_properties, int precision_mode)
+    const grappler::GraphProperties& graph_properties,
+    TrtPrecisionMode precision_mode)
     : graph_properties_(graph_properties), precision_mode_(precision_mode) {}
 
 Status TrtCandidateSelector::IsTensorRTCandidate(const tensorflow::Node* node) {
@@ -149,7 +150,8 @@ Status TrtCandidateSelector::IsTensorRTCandidate(const tensorflow::Node* node) {
   // In INT8 mode, we will always apply the quantization ranges provided by
   // these ops to the relevant tensors. This happens regardless of the value of
   // use_calibration.
-  if (precision_mode_ == INT8MODE && quantize_ops.count(node->type_string())) {
+  if (precision_mode_ == TrtPrecisionMode::INT8 &&
+      quantize_ops.count(node->type_string())) {
     is_supported_op_type = true;
   }
   // LINT.ThenChange(//tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc)
@@ -239,7 +241,7 @@ tensorflow::Status ConvertGraphDefToTensorRT(
     const tensorflow::GraphDef& graph_def,
     const std::vector<string>& output_names, size_t max_batch_size,
     size_t max_workspace_size_bytes, tensorflow::GraphDef* new_graph_def,
-    int precision_mode, int minimum_segment_size, bool is_dyn_op,
+    TrtPrecisionMode precision_mode, int minimum_segment_size, bool is_dyn_op,
     int max_cached_engines, std::vector<int> cached_engine_batches,
     bool use_calibration) {
   // Create GrapplerItem.
@@ -299,7 +301,7 @@ tensorflow::Status ConvertGraphDefToTensorRT(
   parameters["max_batch_size"].set_i(max_batch_size);
   parameters["is_dynamic_op"].set_b(is_dyn_op);
   parameters["max_workspace_size_bytes"].set_i(max_workspace_size_bytes);
-  TF_RETURN_IF_ERROR(GetPrecisionModeName(
+  TF_RETURN_IF_ERROR(TrtPrecisionModeToName(
       precision_mode, parameters["precision_mode"].mutable_s()));
   parameters["maximum_cached_engines"].set_i(max_cached_engines);
   if (!cached_engine_batches.empty()) {
@@ -638,7 +640,7 @@ tensorflow::Status CreateTRTNode(const std::vector<EngineInfo>& infos, int pos,
   }
 
   const bool calibrate_int8 =
-      (info.precision_mode == INT8MODE && info.use_calibration);
+      (info.precision_mode == TrtPrecisionMode::INT8 && info.use_calibration);
   // Build the engine and get its serialized representation.
   string segment_string;
   if (info.engine_type == EngineInfo::EngineType::TRTStatic || calibrate_int8) {
@@ -651,7 +653,8 @@ tensorflow::Status CreateTRTNode(const std::vector<EngineInfo>& infos, int pos,
     TrtUniquePtrType<nvinfer1::ICudaEngine> engine;
     // TODO(sami): What happens if 1st dim is not batch?
     TF_RETURN_IF_ERROR(ConvertGraphDefToEngine(
-        info.segment_graph_def, calibrate_int8 ? FP32MODE : info.precision_mode,
+        info.segment_graph_def,
+        calibrate_int8 ? TrtPrecisionMode::FP32 : info.precision_mode,
         max_batch_size, info.max_workspace_size_bytes, input_shapes,
         &trt_logger, alloc, /*calibrator=*/nullptr, &engine,
         info.use_calibration,
@@ -668,7 +671,7 @@ tensorflow::Status CreateTRTNode(const std::vector<EngineInfo>& infos, int pos,
   }
 
   string prec_string;
-  TF_RETURN_IF_ERROR(GetPrecisionModeName(info.precision_mode, &prec_string));
+  TF_RETURN_IF_ERROR(TrtPrecisionModeToName(info.precision_mode, &prec_string));
   tensorflow::NodeDefBuilder node_builder(info.engine_name, "TRTEngineOp");
   if (!info.device.empty()) node_builder.Device(info.device);
   if (VLOG_IS_ON(1)) {
@@ -976,7 +979,8 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
       continue;
     }
     curr_engine.precision_mode = params.precision_mode;
-    if (params.use_calibration && params.precision_mode != INT8MODE) {
+    if (params.use_calibration &&
+        params.precision_mode != TrtPrecisionMode::INT8) {
       return errors::InvalidArgument(
           "Calibration with FP32 or FP16 is not supported.");
     }
diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_graph.h b/tensorflow/compiler/tf2tensorrt/convert/convert_graph.h
index fb82a430c6..95cf0227dc 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_graph.h
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_graph.h
@@ -36,7 +36,7 @@ namespace convert {
 class TrtCandidateSelector {
  public:
   TrtCandidateSelector(const grappler::GraphProperties& graph_properties,
-                       int precision_mode);
+                       TrtPrecisionMode precision_mode);
 
   // Returns OK iff 'node' is a TF-TRT conversion candidate, which will be added
   // to TRT subgraph and later converted into TRT engine.
@@ -52,7 +52,7 @@ class TrtCandidateSelector {
   const grappler::GraphProperties& graph_properties_;
 
   // Quantization ops are only converted when using quantized precisions.
-  const int precision_mode_;
+  const TrtPrecisionMode precision_mode_;
 };
 
 struct ConversionParams {
@@ -61,7 +61,7 @@ struct ConversionParams {
         max_batch_size(1),
         max_workspace_size_bytes(1 << 30),
         output_graph_def(nullptr),
-        precision_mode(1),
+        precision_mode(TrtPrecisionMode::FP32),
         minimum_segment_size(3),
         graph_properties(nullptr),
         cluster(nullptr),
@@ -74,7 +74,7 @@ struct ConversionParams {
   size_t max_batch_size;
   size_t max_workspace_size_bytes;
   tensorflow::GraphDef* output_graph_def;
-  int precision_mode;
+  TrtPrecisionMode precision_mode;
   int minimum_segment_size;
   const tensorflow::grappler::GraphProperties* graph_properties;
   const tensorflow::grappler::Cluster* cluster;
@@ -99,9 +99,10 @@ tensorflow::Status ConvertGraphDefToTensorRT(
     const tensorflow::GraphDef& graph_def,
     const std::vector<string>& output_names, size_t max_batch_size,
     size_t max_workspace_size_bytes, tensorflow::GraphDef* new_graph_def,
-    int precision_mode = 1, int minimum_segment_size = 3,
-    bool is_dyn_op = false, int max_cached_engines = 1,
-    std::vector<int> cached_engine_batches = {}, bool use_calibration = true);
+    TrtPrecisionMode precision_mode = TrtPrecisionMode::FP32,
+    int minimum_segment_size = 3, bool is_dyn_op = false,
+    int max_cached_engines = 1, std::vector<int> cached_engine_batches = {},
+    bool use_calibration = true);
 
 // Method to call from optimization pass
 tensorflow::Status ConvertAfterShapes(ConversionParams& params);
diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_graph_test.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_graph_test.cc
index a3c3a8ac65..cabc6ccfa1 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_graph_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_graph_test.cc
@@ -98,7 +98,8 @@ TEST(TrtCandidateSelector, Basics) {
   grappler::GraphProperties graph_properties(item);
   TF_EXPECT_OK(graph_properties.InferStatically(true));
 
-  for (const int precision_mode : {FP32MODE, INT8MODE}) {
+  for (const TrtPrecisionMode precision_mode :
+       {TrtPrecisionMode::FP32, TrtPrecisionMode::INT8}) {
     TrtCandidateSelector selector(graph_properties, precision_mode);
     TF_EXPECT_OK(selector.IsTensorRTCandidate(matmul.operation.node()));
     ExpectStatus(
@@ -113,7 +114,7 @@ TEST(TrtCandidateSelector, Basics) {
             matmul_with_incompatible_input.operation.node()),
         error::INTERNAL,
         "Failed to convert input with index 0 to a TRT_TensorOrWeights");
-    if (precision_mode == INT8MODE) {
+    if (precision_mode == TrtPrecisionMode::INT8) {
       TF_EXPECT_OK(selector.IsTensorRTCandidate(quantize.operation.node()));
     } else {
       ExpectStatus(selector.IsTensorRTCandidate(quantize.operation.node()),
diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
index 6d4a06ab15..e6bbe2dfb8 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
@@ -865,7 +865,7 @@ Status TrtNodeValidator::ConvertConstToWeights(
 }
 
 Converter::Converter(nvinfer1::INetworkDefinition* trt_network,
-                     int precision_mode, bool use_calibration)
+                     TrtPrecisionMode precision_mode, bool use_calibration)
     : trt_network_(trt_network),
       precision_mode_(precision_mode),
       use_calibration_(use_calibration) {
@@ -1128,7 +1128,7 @@ Status Converter::PrepareTensorForShape(const TRT_TensorOrWeights& input,
   } else {
     *tensor = CreateConstantLayer(input.weights(), dims);
     TFTRT_RETURN_ERROR_IF_NULLPTR(*tensor, "TF-TRT Internal Reshape");
-    if (precision_mode() == INT8MODE && !use_calibration()) {
+    if (precision_mode() == TrtPrecisionMode::INT8 && !use_calibration()) {
       // If we are in int8 mode and not calibrating, we need to explicitly set a
       // quantization range for the output tensor of the IConstantLayer. Here we
       // set the range to [min(weights), max(weights)].
@@ -1163,7 +1163,7 @@ void Converter::ProvideQuantizationRange(nvinfer1::ITensor* tensor,
 }
 
 void Converter::MaybeApplyQuantizationRanges() {
-  if (precision_mode() != INT8MODE) return;
+  if (precision_mode() != TrtPrecisionMode::INT8) return;
 
   // Infer ranges across marked ops.
   PropagateQuantizationRanges();
@@ -1478,7 +1478,7 @@ Status BinaryTensorOpWeight(OpConverterParams* params,
         const_cast<nvinfer1::ITensor*>(tensor), permutation, &tensor));
   }
 
-  if (params->converter->precision_mode() == FP16MODE) {
+  if (params->converter->precision_mode() == TrtPrecisionMode::FP16) {
     weights = ConvertFP32ToFP16(params->weight_store, weights);
   }
 
@@ -1521,7 +1521,7 @@ Status BinaryTensorOpWeight(OpConverterParams* params,
       // Because of this issue, fall back to BinaryTensorOpTensor if we are
       // doing INT8 with no calibration. There is most likely no performance
       // penalty by falling back here.
-      if (params->converter->precision_mode() == INT8MODE &&
+      if (params->converter->precision_mode() == TrtPrecisionMode::INT8 &&
           !params->converter->use_calibration()) {
         return errors::Unimplemented(
             "Intermediate quantization range cannot be determined without"
@@ -1643,7 +1643,7 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
   // input's channel dim. For a non-depthwise conv, num_groups will be 1.
   const int num_groups = (group == 0) ? tensor_dim.d[0] : group;
 
-  if (params->converter->precision_mode() == FP16MODE) {
+  if (params->converter->precision_mode() == TrtPrecisionMode::FP16) {
     weights_rsck =
         ConvertFP32ToFP16(params->weight_store, inputs.at(1).weights());
   }
@@ -2675,7 +2675,7 @@ tensorflow::Status ConvertBiasAdd(OpConverterParams* params) {
   }
 
   TRT_ShapedWeights weights = inputs.at(1).weights();
-  if (params->converter->precision_mode() == FP16MODE) {
+  if (params->converter->precision_mode() == TrtPrecisionMode::FP16) {
     weights = ConvertFP32ToFP16(params->weight_store, weights);
   }
   nvinfer1::ScaleMode mode = nvinfer1::ScaleMode::kCHANNEL;
@@ -2908,7 +2908,7 @@ tensorflow::Status ConvertUnary(OpConverterParams* params) {
     //   x -> [Sqrt] -> sqrt(x) -> [Recip] -> 1/sqrt(x)
     //                     ^
     //               need range here
-    if (params->converter->precision_mode() == INT8MODE &&
+    if (params->converter->precision_mode() == TrtPrecisionMode::INT8 &&
         !params->converter->use_calibration()) {
       return errors::Unimplemented(
           "Intermediate quantization range cannot be determined without"
@@ -3653,8 +3653,8 @@ void Converter::RegisterOpConverters() {
 }
 
 tensorflow::Status ConvertGraphDefToEngine(
-    const tensorflow::GraphDef& gdef, int precision_mode, int max_batch_size,
-    size_t max_workspace_size_bytes,
+    const tensorflow::GraphDef& gdef, TrtPrecisionMode precision_mode,
+    int max_batch_size, size_t max_workspace_size_bytes,
     const std::vector<tensorflow::PartialTensorShape>& input_shapes,
     Logger* logger, nvinfer1::IGpuAllocator* allocator,
     TRTInt8Calibrator* calibrator,
@@ -3669,9 +3669,9 @@ tensorflow::Status ConvertGraphDefToEngine(
   builder->setMaxBatchSize(max_batch_size);
   builder->setMaxWorkspaceSize(max_workspace_size_bytes);
   builder->setGpuAllocator(allocator);
-  if (precision_mode == FP16MODE) {
+  if (precision_mode == TrtPrecisionMode::FP16) {
     builder->setHalf2Mode(true);
-  } else if (precision_mode == INT8MODE) {
+  } else if (precision_mode == TrtPrecisionMode::INT8) {
     builder->setInt8Mode(true);
     if (use_calibration) {
       builder->setInt8Calibrator(calibrator);
diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
index 00373051f5..d1e30eb848 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
@@ -92,7 +92,7 @@ struct EngineInfo {
   EngineInfo()
       : engine_type(EngineType::TRTStatic),
         max_workspace_size_bytes(0),
-        precision_mode(FP32MODE),
+        precision_mode(TrtPrecisionMode::FP32),
         use_calibration(true) {}
 
   string engine_name;
@@ -109,7 +109,7 @@ struct EngineInfo {
   int64 max_workspace_size_bytes;
   int maximum_cached_engines;
   std::vector<int> cached_engine_batches;
-  int precision_mode;
+  TrtPrecisionMode precision_mode;
   bool use_calibration;
 };
 
@@ -141,8 +141,8 @@ tensorflow::Status ConvertSegmentToGraphDef(
 //   is successful. This is different than successfully building the engine:
 //   building can still fail afterwards.
 tensorflow::Status ConvertGraphDefToEngine(
-    const tensorflow::GraphDef& gdef, int precision_mode, int max_batch_size,
-    size_t max_workspace_size_bytes,
+    const tensorflow::GraphDef& gdef, TrtPrecisionMode precision_mode,
+    int max_batch_size, size_t max_workspace_size_bytes,
     const std::vector<tensorflow::PartialTensorShape>& input_shapes,
     Logger* logger, nvinfer1::IGpuAllocator* allocator,
     TRTInt8Calibrator* calibrator,
@@ -415,8 +415,8 @@ class Converter {
     nvinfer1::DataType trt_dtype;
   };
 
-  Converter(nvinfer1::INetworkDefinition* trt_network, int precision_mode,
-            bool use_calibration);
+  Converter(nvinfer1::INetworkDefinition* trt_network,
+            TrtPrecisionMode precision_mode, bool use_calibration);
 
   //////////////////////////////////////////////////////////////////////////////
   // Methods used by the TRT engine builder to build a TRT network from a TF
@@ -444,7 +444,7 @@ class Converter {
   nvinfer1::INetworkDefinition* network() { return trt_network_; }
 
   // What precision are we targeting?
-  int precision_mode() const { return precision_mode_; }
+  TrtPrecisionMode precision_mode() const { return precision_mode_; }
 
   // Calibration will be or was previously performed on this network?
   bool use_calibration() const { return use_calibration_; }
@@ -546,7 +546,7 @@ class Converter {
   std::vector<std::pair<nvinfer1::ITensor*, nvinfer1::ITensor*>>
       quantization_infer_;
 
-  const int precision_mode_;
+  const TrtPrecisionMode precision_mode_;
 
   const bool use_calibration_;
 
diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
index 9789475ba9..c3d64e840f 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
@@ -481,8 +481,7 @@ class ConverterTest : public ::testing::Test {
   ConverterTest() {
     builder_.reset(nvinfer1::createInferBuilder(logger_));
     network_.reset(builder_->createNetwork());
-    converter_.reset(new Converter(network_.get(),
-                                   /*precision_mode=*/FP32MODE,
+    converter_.reset(new Converter(network_.get(), TrtPrecisionMode::FP32,
                                    /*use_calibration=*/false));
     weight_store_ = &converter_->weight_store_;
   }
@@ -784,7 +783,7 @@ TEST_F(ConverterTest, MaybeApplyQuantizationRanges) {
   // input -> infer1 -> infer2 -> infer3
   FakeITensor input, infer_1, infer_2, infer_3;
   FakeITensor not_infer;
-  Converter int8_converter(/*trt_network=*/nullptr, INT8MODE,
+  Converter int8_converter(/*trt_network=*/nullptr, TrtPrecisionMode::INT8,
                            /*use_calibration=*/true);
   int8_converter.ProvideQuantizationRange(&input, -5.0f, 5.0f);
   int8_converter.ProvideQuantizationRange(&not_infer, -100.0f, 100.0f);
@@ -984,8 +983,7 @@ class OpConverterTest : public ::testing::Test {
 
     // Reset the validator and converter.
     validator_.reset(new TrtNodeValidator);
-    converter_.reset(new Converter(network_.get(),
-                                   /*precision_mode=*/FP32MODE,
+    converter_.reset(new Converter(network_.get(), TrtPrecisionMode::FP32,
                                    /*use_calibration=*/false));
 
     // Reset other related artifacts.
diff --git a/tensorflow/compiler/tf2tensorrt/convert/trt_optimization_pass.cc b/tensorflow/compiler/tf2tensorrt/convert/trt_optimization_pass.cc
index ebf8df1349..f36aa558ea 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/trt_optimization_pass.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/trt_optimization_pass.cc
@@ -66,7 +66,7 @@ tensorflow::Status TRTOptimizationPass::Init(
     max_workspace_size_bytes_ = params.at("max_workspace_size_bytes").i();
   }
   if (params.count("precision_mode")) {
-    TF_RETURN_IF_ERROR(GetPrecisionMode(
+    TF_RETURN_IF_ERROR(TrtPrecisionModeFromName(
         Uppercase(params.at("precision_mode").s()), &precision_mode_));
   }
   if (params.count("use_calibration")) {
@@ -227,7 +227,7 @@ tensorflow::Status TRTOptimizationPass::Optimize(
   TF_RETURN_IF_ERROR(static_graph_properties.InferStatically(true));
   tensorflow::tensorrt::convert::ConversionParams cp;
 
-  if (use_calibration_ && precision_mode_ != INT8MODE) {
+  if (use_calibration_ && precision_mode_ != TrtPrecisionMode::INT8) {
     VLOG(1) << "Calibration with FP32 or FP16 is not implemented. "
             << "Falling back to use_calibration = False."
             << "Note that the default value of use_calibration is True.";
diff --git a/tensorflow/compiler/tf2tensorrt/convert/trt_optimization_pass.h b/tensorflow/compiler/tf2tensorrt/convert/trt_optimization_pass.h
index bd6c6dbce1..b2aed2a37a 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/trt_optimization_pass.h
+++ b/tensorflow/compiler/tf2tensorrt/convert/trt_optimization_pass.h
@@ -18,6 +18,7 @@ limitations under the License.
 
 #include <string>
 
+#include "tensorflow/compiler/tf2tensorrt/convert/utils.h"
 #include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h"
 #include "tensorflow/core/platform/logging.h"
@@ -34,7 +35,7 @@ class TRTOptimizationPass : public tensorflow::grappler::CustomGraphOptimizer {
   TRTOptimizationPass(const string& name = "TRTOptimizationPass")
       : name_(name),
         minimum_segment_size_(3),
-        precision_mode_(0),
+        precision_mode_(TrtPrecisionMode::FP32),
         maximum_batch_size_(-1),
         is_dynamic_op_(false),
         max_cached_batches_(1),
@@ -62,7 +63,7 @@ class TRTOptimizationPass : public tensorflow::grappler::CustomGraphOptimizer {
  private:
   const string name_;
   int minimum_segment_size_;
-  int precision_mode_;
+  TrtPrecisionMode precision_mode_;
   int maximum_batch_size_;
   bool is_dynamic_op_;
   std::vector<int> batches_;
diff --git a/tensorflow/compiler/tf2tensorrt/convert/utils.cc b/tensorflow/compiler/tf2tensorrt/convert/utils.cc
index 62a0f62ad6..0ca3a5a4a5 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/utils.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/utils.cc
@@ -34,33 +34,32 @@ bool IsGoogleTensorRTEnabled() {
 #endif
 }
 
-Status GetPrecisionModeName(const int precision_mode, string* name) {
-  switch (precision_mode) {
-    case FP32MODE:
+Status TrtPrecisionModeToName(TrtPrecisionMode mode, string* name) {
+  switch (mode) {
+    case TrtPrecisionMode::FP32:
       *name = "FP32";
       break;
-    case FP16MODE:
+    case TrtPrecisionMode::FP16:
       *name = "FP16";
       break;
-    case INT8MODE:
+    case TrtPrecisionMode::INT8:
       *name = "INT8";
       break;
     default:
-      return tensorflow::errors::OutOfRange("Unknown precision mode");
+      return errors::OutOfRange("Unknown precision mode");
   }
   return Status::OK();
 }
 
-Status GetPrecisionMode(const string& name, int* precision_mode) {
+Status TrtPrecisionModeFromName(const string& name, TrtPrecisionMode* mode) {
   if (name == "FP32") {
-    *precision_mode = FP32MODE;
+    *mode = TrtPrecisionMode::FP32;
   } else if (name == "FP16") {
-    *precision_mode = FP16MODE;
+    *mode = TrtPrecisionMode::FP16;
   } else if (name == "INT8") {
-    *precision_mode = INT8MODE;
+    *mode = TrtPrecisionMode::INT8;
   } else {
-    return tensorflow::errors::InvalidArgument("Invalid precision mode name: ",
-                                               name);
+    return errors::InvalidArgument("Invalid precision mode name: ", name);
   }
   return Status::OK();
 }
diff --git a/tensorflow/compiler/tf2tensorrt/convert/utils.h b/tensorflow/compiler/tf2tensorrt/convert/utils.h
index 9f9ee59087..0aa602dda2 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/utils.h
+++ b/tensorflow/compiler/tf2tensorrt/convert/utils.h
@@ -35,14 +35,11 @@ using TrtUniquePtrType = std::unique_ptr<T, TrtDestroyer<T>>;
 
 bool IsGoogleTensorRTEnabled();
 
-// TODO(aaroey): use an enum instead.
-const int FP32MODE = 0;
-const int FP16MODE = 1;
-const int INT8MODE = 2;
+enum class TrtPrecisionMode { FP32, FP16, INT8 };
 
-Status GetPrecisionModeName(const int precision_mode, string* name);
+Status TrtPrecisionModeToName(TrtPrecisionMode mode, string* name);
 
-Status GetPrecisionMode(const string& name, int* precision_mode);
+Status TrtPrecisionModeFromName(const string& name, TrtPrecisionMode* mode);
 
 }  // namespace tensorrt
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.cc b/tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.cc
index ec9dc08c12..bc5335ef5a 100644
--- a/tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.cc
@@ -113,7 +113,7 @@ class TRTEngineOp : public AsyncOpKernel {
   GraphDef segment_graph_;
 
   // Engine Precision mode.
-  int precision_mode_;
+  TrtPrecisionMode precision_mode_;
 
   // Whether engine is constructed during the conversion or needs to be
   // constructed from protobuf segment.
@@ -210,11 +210,13 @@ TRTEngineOp::TRTEngineOp(OpKernelConstruction* context)
                  context->GetAttr("calibration_data", &calibration_data));
   OP_REQUIRES_OK(context,
                  context->GetAttr("segment_funcdef_name", &funcdef_name_));
-  OP_REQUIRES_OK(context, GetPrecisionMode(precision_string, &precision_mode_));
+  OP_REQUIRES_OK(context,
+                 TrtPrecisionModeFromName(precision_string, &precision_mode_));
   OP_REQUIRES_OK(context,
                  context->GetAttr("use_calibration", &use_calibration_));
-  calibration_mode_ = (use_calibration_ && precision_mode_ == INT8MODE &&
-                       calibration_data.size() == 0);
+  calibration_mode_ =
+      (use_calibration_ && precision_mode_ == TrtPrecisionMode::INT8 &&
+       calibration_data.size() == 0);
   if (calibration_data.size()) {
     calibrator_.reset(new TRTInt8Calibrator(calibration_data));
     calibration_data.resize(0);
@@ -712,9 +714,10 @@ tensorflow::Status TRTEngineOp::AllocateCalibrationResources(
     // TODO(aaroey): maybe setting the max batch size using the python
     // calibration wrapper class.
     auto s = convert::ConvertGraphDefToEngine(
-        *segment_graph, INT8MODE, cres->calibrator_->getBatchSize(),
-        workspace_size_bytes, shapes, &cres->logger_, cres->allocator_.get(),
-        cres->calibrator_.get(), &cres->engine_,
+        *segment_graph, TrtPrecisionMode::INT8,
+        cres->calibrator_->getBatchSize(), workspace_size_bytes, shapes,
+        &cres->logger_, cres->allocator_.get(), cres->calibrator_.get(),
+        &cres->engine_,
         /*use_calibration=*/true,
         /*convert_successfully=*/nullptr);
     if (!s.ok()) {
-- 
GitLab


From ecab2c91cb66f12d90be295ac3d54bc1598d4817 Mon Sep 17 00:00:00 2001
From: Bhavani Subramanian <bhavani1.subramanian@intel.com>
Date: Thu, 31 Jan 2019 16:26:55 -0800
Subject: [PATCH 1701/2345] Check if filter is a constant only in the forward
 pass for Depthwise Conv2D.

---
 tensorflow/core/graph/mkl_layout_pass.cc      | 27 ++++++++++++++++++-
 tensorflow/core/kernels/mkl_fused_ops_test.cc |  4 +--
 2 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 1a8ff342ad..e934978e76 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -389,7 +389,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
                       CopyAttrsConv, AlwaysRewrite});
     rinfo_.push_back({csinfo_.depthwise_conv2d,
                       mkl_op_registry::GetMklOpName(csinfo_.depthwise_conv2d),
-                      CopyAttrsConv2DDepthwise, AlwaysRewrite});
+                      CopyAttrsConv2DDepthwiseCheckConstFilter, AlwaysRewrite});
     rinfo_.push_back(
         {csinfo_.depthwise_conv2d_grad_input,
          mkl_op_registry::GetMklOpName(csinfo_.depthwise_conv2d_grad_input),
@@ -1495,6 +1495,8 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
                             bool change_format = false);
   static void CopyAttrsConv2DDepthwise(const Node* orig_node, NodeBuilder* nb,
                                        bool change_format = false);
+  static void CopyAttrsConv2DDepthwiseCheckConstFilter(
+      const Node* orig_node, NodeBuilder* nb, bool change_format = false);
   static void CopyAttrsConvCheckConstFilter(const Node* orig_node,
                                             NodeBuilder* nb,
                                             bool change_format = false);
@@ -2231,6 +2233,29 @@ void MklLayoutRewritePass::CopyAttrsConv2DDepthwise(const Node* orig_node,
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "padding", &padding));
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "data_format", &data_format));
 
+  // Add attributes to new node.
+  nb->Attr("T", T);
+  nb->Attr("strides", strides);
+  nb->Attr("dilations", dilations);
+  nb->Attr("padding", padding);
+  nb->Attr("data_format", data_format);
+}
+
+void MklLayoutRewritePass::CopyAttrsConv2DDepthwiseCheckConstFilter(
+    const Node* orig_node, NodeBuilder* nb, bool change_format) {
+  DataType T;
+  string data_format;
+  string padding;
+  std::vector<int32> strides;
+  std::vector<int32> dilations;
+
+  // Get all attributes from old node.
+  TF_CHECK_OK(GetNodeAttr(orig_node->def(), "T", &T));
+  TF_CHECK_OK(GetNodeAttr(orig_node->def(), "strides", &strides));
+  TF_CHECK_OK(GetNodeAttr(orig_node->def(), "dilations", &dilations));
+  TF_CHECK_OK(GetNodeAttr(orig_node->def(), "padding", &padding));
+  TF_CHECK_OK(GetNodeAttr(orig_node->def(), "data_format", &data_format));
+
   Node* filter_node = nullptr;
   orig_node->input_node(1, &filter_node);
 
diff --git a/tensorflow/core/kernels/mkl_fused_ops_test.cc b/tensorflow/core/kernels/mkl_fused_ops_test.cc
index 756ee907f1..288515de0b 100644
--- a/tensorflow/core/kernels/mkl_fused_ops_test.cc
+++ b/tensorflow/core/kernels/mkl_fused_ops_test.cc
@@ -434,7 +434,7 @@ class FilterCacheTest : public OpsTestBase {
     // Compare outputs to expected results
     const Tensor& output = *GetOutput(0);
     const Tensor& output_layout = *GetOutput(2);
-    ConvMklToTF<T> conv_comp;
+    CommonTestUtilities<T> conv_comp;
     conv_comp.ConvertAndCompare(dtype, output, output_layout, expected);
 
     // TODO(bhavanis): For now, we rely on internal performance tests to
@@ -446,7 +446,7 @@ class FilterCacheTest : public OpsTestBase {
     // Compare output to expected results
     const Tensor& output_new = *GetOutput(0);
     const Tensor& output_layout_new = *GetOutput(2);
-    ConvMklToTF<T> conv_comp_new;
+    CommonTestUtilities<T> conv_comp_new;
     conv_comp_new.ConvertAndCompare(dtype, output_new, output_layout_new,
                                     expected);
   }
-- 
GitLab


From 16ada08c8d054291a63961e06ec76a9a38b4be2d Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Thu, 31 Jan 2019 15:28:02 -0800
Subject: [PATCH 1702/2345] Making the expiration date for old feature columns
 None.

PiperOrigin-RevId: 231866087
---
 tensorflow/python/feature_column/feature_column_v2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/feature_column/feature_column_v2.py b/tensorflow/python/feature_column/feature_column_v2.py
index a94245612b..4b93c2f28d 100644
--- a/tensorflow/python/feature_column/feature_column_v2.py
+++ b/tensorflow/python/feature_column/feature_column_v2.py
@@ -169,7 +169,7 @@ from tensorflow.python.util.tf_export import keras_export
 from tensorflow.python.util.tf_export import tf_export
 
 
-_FEATURE_COLUMN_DEPRECATION_DATE = '2018-11-30'
+_FEATURE_COLUMN_DEPRECATION_DATE = None
 _FEATURE_COLUMN_DEPRECATION = ('The old _FeatureColumn APIs are being '
                                'deprecated. Please use the new FeatureColumn '
                                'APIs instead.')
-- 
GitLab


From aebb09b43cb9e980f5da1675459099755cd5d8eb Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Thu, 31 Jan 2019 15:37:06 -0800
Subject: [PATCH 1703/2345] Fix bug causing inconsistent AST when
 comprehensions are involved in template expansions.

PiperOrigin-RevId: 231867856
---
 tensorflow/python/autograph/pyct/templates.py      |  4 ++++
 tensorflow/python/autograph/pyct/templates_test.py | 10 ++++++++++
 2 files changed, 14 insertions(+)

diff --git a/tensorflow/python/autograph/pyct/templates.py b/tensorflow/python/autograph/pyct/templates.py
index 0f0b8613ac..831eb6dbbf 100644
--- a/tensorflow/python/autograph/pyct/templates.py
+++ b/tensorflow/python/autograph/pyct/templates.py
@@ -91,6 +91,10 @@ class ContextAdjuster(gast.NodeTransformer):
     self._ctx_override = None
     return self.generic_visit(node)
 
+  def visit_comprehension(self, node):
+    self._ctx_override = None
+    return self.generic_visit(node)
+
 
 class ReplaceTransformer(gast.NodeTransformer):
   """Replace AST nodes."""
diff --git a/tensorflow/python/autograph/pyct/templates_test.py b/tensorflow/python/autograph/pyct/templates_test.py
index cdb44b822e..bd6b451eda 100644
--- a/tensorflow/python/autograph/pyct/templates_test.py
+++ b/tensorflow/python/autograph/pyct/templates_test.py
@@ -238,6 +238,16 @@ class TemplatesTest(test.TestCase):
     source = parser.parse_expression('[a(b(1))]')
     templates.replace_as_expression(template, bar=source)
 
+  def test_star_comprehension_in_function_call(self):
+    template = """
+      a = foo(func, args)
+    """
+    source = parser.parse_expression('bar(*[i for i in range(j)])')
+    node = templates.replace(template, func=source.func, args=source.args)
+    arg_node = node[0].value.args[1].value
+    self.assertIsInstance(arg_node.generators[0].target.ctx, gast.Store)
+    self.assertIsInstance(arg_node.elt.ctx, gast.Load)
+
 
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 7b2ded24ace6f771a62c93134bc7b58325e6867e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 31 Jan 2019 15:37:14 -0800
Subject: [PATCH 1704/2345] Exposes some experimental learning rate schedules
 added to TF 2.0 in TF 1.x.

PiperOrigin-RevId: 231867881
---
 .../optimizer_v2/learning_rate_schedule.py     |  9 +++------
 ...s.experimental.-cosine-decay-restarts.pbtxt | 18 ++++++++++++++++++
 ...ras.experimental.-linear-cosine-decay.pbtxt | 18 ++++++++++++++++++
 ...perimental.-noisy-linear-cosine-decay.pbtxt | 18 ++++++++++++++++++
 .../v1/tensorflow.keras.experimental.pbtxt     | 12 ++++++++++++
 .../tools/compatibility/tf_upgrade_v2.py       |  8 ++------
 .../tools/compatibility/tf_upgrade_v2_test.py  |  3 ++-
 7 files changed, 73 insertions(+), 13 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-cosine-decay-restarts.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-linear-cosine-decay.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-noisy-linear-cosine-decay.pbtxt

diff --git a/tensorflow/python/keras/optimizer_v2/learning_rate_schedule.py b/tensorflow/python/keras/optimizer_v2/learning_rate_schedule.py
index a182d74e56..c44263bdcf 100644
--- a/tensorflow/python/keras/optimizer_v2/learning_rate_schedule.py
+++ b/tensorflow/python/keras/optimizer_v2/learning_rate_schedule.py
@@ -631,8 +631,7 @@ class CosineDecay(LearningRateSchedule):
     }
 
 
-@keras_export("keras.experimental.CosineDecayRestarts",
-              v1=[])
+@keras_export("keras.experimental.CosineDecayRestarts")
 class CosineDecayRestarts(LearningRateSchedule):
   """A LearningRateSchedule that uses a cosine decay schedule with restarts."""
 
@@ -761,8 +760,7 @@ class CosineDecayRestarts(LearningRateSchedule):
     }
 
 
-@keras_export("keras.experimental.LinearCosineDecay",
-              v1=[])
+@keras_export("keras.experimental.LinearCosineDecay")
 class LinearCosineDecay(LearningRateSchedule):
   """A LearningRateSchedule that uses a linear cosine decay schedule."""
 
@@ -879,8 +877,7 @@ class LinearCosineDecay(LearningRateSchedule):
     }
 
 
-@keras_export("keras.experimental.NoisyLinearCosineDecay",
-              v1=[])
+@keras_export("keras.experimental.NoisyLinearCosineDecay")
 class NoisyLinearCosineDecay(LearningRateSchedule):
   """A LearningRateSchedule that uses a noisy linear cosine decay schedule."""
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-cosine-decay-restarts.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-cosine-decay-restarts.pbtxt
new file mode 100644
index 0000000000..58bede556d
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-cosine-decay-restarts.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.keras.experimental.CosineDecayRestarts"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.CosineDecayRestarts\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.LearningRateSchedule\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'initial_learning_rate\', \'first_decay_steps\', \'t_mul\', \'m_mul\', \'alpha\', \'name\'], varargs=None, keywords=None, defaults=[\'2.0\', \'1.0\', \'0.0\', \'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-linear-cosine-decay.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-linear-cosine-decay.pbtxt
new file mode 100644
index 0000000000..f083120b52
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-linear-cosine-decay.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.keras.experimental.LinearCosineDecay"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.LinearCosineDecay\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.LearningRateSchedule\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'initial_learning_rate\', \'decay_steps\', \'num_periods\', \'alpha\', \'beta\', \'name\'], varargs=None, keywords=None, defaults=[\'0.5\', \'0.0\', \'0.001\', \'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-noisy-linear-cosine-decay.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-noisy-linear-cosine-decay.pbtxt
new file mode 100644
index 0000000000..8ea3c6beb1
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-noisy-linear-cosine-decay.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.keras.experimental.NoisyLinearCosineDecay"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.NoisyLinearCosineDecay\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.LearningRateSchedule\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'initial_learning_rate\', \'decay_steps\', \'initial_variance\', \'variance_decay\', \'num_periods\', \'alpha\', \'beta\', \'name\'], varargs=None, keywords=None, defaults=[\'1.0\', \'0.55\', \'0.5\', \'0.0\', \'0.001\', \'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.pbtxt
index 24684b9f4d..721c18890a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.pbtxt
@@ -4,6 +4,18 @@ tf_module {
     name: "CosineDecay"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "CosineDecayRestarts"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "LinearCosineDecay"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "NoisyLinearCosineDecay"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "PeepholeLSTMCell"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 11ac70df0a..85ece0313f 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -827,12 +827,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
 
     decay_function_comment = (
         ast_edits.INFO,
-        "<function name> has been changed to return a callable instead "
-        "of a tensor when graph building, but its functionality remains "
-        "unchanged during eager execution (returns a callable like "
-        "before). The converter cannot detect and fix this reliably, so "
-        "this usage has been converted to compat.v1 (even though it may already"
-        " be correct).\n"
+        "To use learning rate decay schedules with TensorFlow 2.0, switch to "
+        "the schedules in `tf.keras.optimizers.schedules`.\n"
     )
 
     assert_return_type_comment = (
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index bbe074e242..11ed634ed4 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -409,7 +409,8 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
 
       text = "%s(a, b)\n" % decay
       _, report, unused_errors, _ = self._upgrade(text)
-      self.assertIn("%s has been changed to return a callable" % decay, report)
+      self.assertIn("switch to the schedules in "
+                    "`tf.keras.optimizers.schedules`", report)
 
   def testMetrics(self):
     metrics = [
-- 
GitLab


From 33cace40dfb97d6e0c2a79f38dd326d75048141d Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Thu, 31 Jan 2019 15:37:21 -0800
Subject: [PATCH 1705/2345] Add more logging to help debug errors caused by
 inconsistent AST.

PiperOrigin-RevId: 231867904
---
 .../python/autograph/pyct/origin_info.py      | 74 ++++++++++++-------
 .../python/autograph/pyct/pretty_printer.py   |  9 ++-
 2 files changed, 53 insertions(+), 30 deletions(-)

diff --git a/tensorflow/python/autograph/pyct/origin_info.py b/tensorflow/python/autograph/pyct/origin_info.py
index 230468eab9..f41b3285d1 100644
--- a/tensorflow/python/autograph/pyct/origin_info.py
+++ b/tensorflow/python/autograph/pyct/origin_info.py
@@ -18,6 +18,7 @@ from __future__ import division
 from __future__ import print_function
 
 import collections
+import difflib
 import os
 import tokenize
 
@@ -27,6 +28,8 @@ import six
 from tensorflow.python.autograph.pyct import anno
 from tensorflow.python.autograph.pyct import ast_util
 from tensorflow.python.autograph.pyct import parser
+from tensorflow.python.autograph.pyct import pretty_printer
+from tensorflow.python.autograph.utils import ag_logging as logging
 from tensorflow.python.util import tf_inspect
 
 
@@ -75,9 +78,11 @@ class OriginInfo(
             self.source_code_line)
 
   def __repr__(self):
-    return '{}:{}:{}'.format(
-        os.path.split(self.loc.filename)[1], self.loc.lineno,
-        self.loc.col_offset)
+    if self.loc.filename:
+      return '{}:{}:{}'.format(
+          os.path.split(self.loc.filename)[1], self.loc.lineno,
+          self.loc.col_offset)
+    return '<no file>:{}:{}'.format(self.loc.lineno, self.loc.col_offset)
 
 
 # TODO(mdan): This source map should be a class - easier to refer to.
@@ -103,32 +108,47 @@ def create_source_map(nodes, code, filename, indices_in_code):
   resolve(reparsed_nodes, code)
   result = {}
 
-  for before, after in ast_util.parallel_walk(nodes, reparsed_nodes):
-    # Note: generated code might not be mapped back to its origin.
-    # TODO(mdan): Generated code should always be mapped to something.
-    origin_info = anno.getanno(before, anno.Basic.ORIGIN, default=None)
-    final_info = anno.getanno(after, anno.Basic.ORIGIN, default=None)
-    if origin_info is None or final_info is None:
-      continue
-
-    line_loc = LineLocation(filename, final_info.loc.lineno)
-
-    existing_origin = result.get(line_loc)
-    if existing_origin is not None:
-      # Overlaps may exist because of child nodes, but almost never to
-      # different line locations. Exception make decorated functions, where
-      # both lines are mapped to the same line in the AST.
-
-      # Line overlaps: keep bottom node.
-      if existing_origin.loc.line_loc == origin_info.loc.line_loc:
-        if existing_origin.loc.lineno >= origin_info.loc.lineno:
-          continue
-
-      # In case of overlaps, keep the leftmost node.
-      if existing_origin.loc.col_offset <= origin_info.loc.col_offset:
+  try:
+    for before, after in ast_util.parallel_walk(nodes, reparsed_nodes):
+      # Note: generated code might not be mapped back to its origin.
+      # TODO(mdan): Generated code should always be mapped to something.
+      origin_info = anno.getanno(before, anno.Basic.ORIGIN, default=None)
+      final_info = anno.getanno(after, anno.Basic.ORIGIN, default=None)
+      if origin_info is None or final_info is None:
         continue
 
-    result[line_loc] = origin_info
+      line_loc = LineLocation(filename, final_info.loc.lineno)
+
+      existing_origin = result.get(line_loc)
+      if existing_origin is not None:
+        # Overlaps may exist because of child nodes, but almost never to
+        # different line locations. Exception make decorated functions, where
+        # both lines are mapped to the same line in the AST.
+
+        # Line overlaps: keep bottom node.
+        if existing_origin.loc.line_loc == origin_info.loc.line_loc:
+          if existing_origin.loc.lineno >= origin_info.loc.lineno:
+            continue
+
+        # In case of overlaps, keep the leftmost node.
+        if existing_origin.loc.col_offset <= origin_info.loc.col_offset:
+          continue
+
+      result[line_loc] = origin_info
+  except ValueError:
+    if logging.has_verbosity(3):
+      for n, rn in zip(nodes, reparsed_nodes):
+        nodes_str = pretty_printer.fmt(n, color=False, noanno=True)
+        reparsed_nodes_str = pretty_printer.fmt(rn, color=False, noanno=True)
+        diff = difflib.context_diff(
+            nodes_str.split('\n'),
+            reparsed_nodes_str.split('\n'),
+            fromfile='Original nodes',
+            tofile='Reparsed nodes',
+            n=7)
+        diff = '\n'.join(diff)
+        logging.log(3, 'AST seems to lack integrity. Diff:\n%s', diff)
+    raise
 
   return result
 
diff --git a/tensorflow/python/autograph/pyct/pretty_printer.py b/tensorflow/python/autograph/pyct/pretty_printer.py
index bacc1e4a77..d963416bed 100644
--- a/tensorflow/python/autograph/pyct/pretty_printer.py
+++ b/tensorflow/python/autograph/pyct/pretty_printer.py
@@ -25,10 +25,11 @@ import termcolor
 class PrettyPrinter(gast.NodeVisitor):
   """Print AST nodes."""
 
-  def __init__(self, color):
+  def __init__(self, color, noanno):
     self.indent_lvl = 0
     self.result = ''
     self.color = color
+    self.noanno = noanno
 
   def _color(self, string, color, attrs=None):
     if self.color:
@@ -68,6 +69,8 @@ class PrettyPrinter(gast.NodeVisitor):
 
     self.indent_lvl += 1
     for f in node._fields:
+      if self.noanno and f.startswith('__'):
+        continue
       if not hasattr(node, f):
         self._print('%s%s' % (self._indent(), self._warning('%s=<unset>' % f)))
         continue
@@ -103,8 +106,8 @@ class PrettyPrinter(gast.NodeVisitor):
     self.indent_lvl -= 1
 
 
-def fmt(node, color=True):
-  printer = PrettyPrinter(color)
+def fmt(node, color=True, noanno=False):
+  printer = PrettyPrinter(color, noanno)
   if isinstance(node, (list, tuple)):
     for n in node:
       printer.visit(n)
-- 
GitLab


From dd84a42ab04d98133bf3108accd2face08e6aa16 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 31 Jan 2019 15:41:24 -0800
Subject: [PATCH 1706/2345] [TF:XLA] Reduce the size of adadelta tests to
 medium.

PiperOrigin-RevId: 231868597
---
 tensorflow/compiler/tests/BUILD            |  2 +-
 tensorflow/compiler/tests/adadelta_test.py | 12 ++++++++++--
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD
index 139c92728d..9b6ca4092c 100644
--- a/tensorflow/compiler/tests/BUILD
+++ b/tensorflow/compiler/tests/BUILD
@@ -72,7 +72,7 @@ py_test(
 
 tf_xla_py_test(
     name = "adadelta_test",
-    size = "large",
+    size = "medium",
     srcs = ["adadelta_test.py"],
     deps = [
         ":xla_test",
diff --git a/tensorflow/compiler/tests/adadelta_test.py b/tensorflow/compiler/tests/adadelta_test.py
index b7b7fda293..6cf16cc07f 100644
--- a/tensorflow/compiler/tests/adadelta_test.py
+++ b/tensorflow/compiler/tests/adadelta_test.py
@@ -32,10 +32,18 @@ class AdadeltaOptimizerTest(xla_test.XLATestCase):
 
   def testBasic(self):
     num_updates = 4  # number of ADADELTA steps to perform
+    if "CPU" in self.device:
+      # To avoid timeout on CPU.
+      all_grad = [0.2, 0.01]
+      all_lr = [1.0, 0.1]
+    else:
+      all_grad = [0.2, 0.1, 0.01]
+      all_lr = [1.0, 0.5, 0.1]
+
     for dtype in self.float_types:
       with self.cached_session(), self.test_scope():
-        for grad in [0.2, 0.1, 0.01]:
-          for lr in [1.0, 0.5, 0.1]:
+        for grad in all_grad:
+          for lr in all_lr:
             var0_init = [1.0, 2.0]
             var1_init = [3.0, 4.0]
             var0 = resource_variable_ops.ResourceVariable(
-- 
GitLab


From d2e6f0132984cd714a4735aeefa9d9cd70be9101 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 31 Jan 2019 15:45:28 -0800
Subject: [PATCH 1707/2345] [XLA:CPU] Use vectorized implementation for all
 calls to log/exp/tanh.

Previously, we used llvm.{log,exp,tanh}.f32 when we failed to vectorize and for
the tail of vectorized loops.  The latter was particularly problematic because
the LLVM intrinsic doesn't have the same numeric properties as our vectorized
implementation, and so we'd get different numerics for different output
elements!

PiperOrigin-RevId: 231869262
---
 .../xla/service/cpu/llvm_ir_runtime.cc        | 236 ++++++++----------
 .../exhaustive_f32_elementwise_op_test.cc     |  11 +-
 2 files changed, 112 insertions(+), 135 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/llvm_ir_runtime.cc b/tensorflow/compiler/xla/service/cpu/llvm_ir_runtime.cc
index a246494c6c..e6183c19f7 100644
--- a/tensorflow/compiler/xla/service/cpu/llvm_ir_runtime.cc
+++ b/tensorflow/compiler/xla/service/cpu/llvm_ir_runtime.cc
@@ -36,57 +36,88 @@ const char* const kLogV4F32SymbolName = "__xla_cpu_runtime_LogV4F32AVX";
 const char* const kLogV8F32SymbolName = "__xla_cpu_runtime_LogV8F32AVX";
 
 namespace {
-llvm::Function* EmitVectorF32TanhIfNeeded(llvm::Module* module,
-                                          llvm::StringRef function_name,
-                                          int vector_width,
-                                          bool enable_fast_math) {
-  llvm::Function* vector_tanh_function = module->getFunction(function_name);
-  if (vector_tanh_function == nullptr) {
+
+// Replaces calls to the function `fn_name` with the code generated by
+// fn_body_generator.
+//
+// We assume that fn_name accepts either a scalar f32 or a vector of
+// vector_width f32s, and that fn_body_generator generates a function body with
+// the same inputs/outputs as fn_name.
+void RewriteCalls(
+    llvm::Module* module, const char* fn_name,
+    std::function<llvm::Value*(llvm::IRBuilder<>* b, llvm::Value* input,
+                               int32 vector_width)>
+        fn_body_generator,
+    int32 vector_width, bool enable_fast_math) {
+  llvm::Function* fn = module->getFunction(fn_name);
+  if (fn == nullptr) {
     // If the function declaration is not present in the module, there can't be
     // any calls to resolve.  Don't emit the function in this case.
-    return nullptr;
+    return;
   }
 
-  llvm::LLVMContext* context = &module->getContext();
+  // Our task is to generate a function body for `fn`, but we can't generate a
+  // function body for an LLVM intrinsic. So if fn is an intrinsic, replace it
+  // with a new function.
+  if (fn->isIntrinsic()) {
+    llvm::Function* new_fn = llvm::Function::Create(
+        fn->getFunctionType(), llvm::GlobalValue::InternalLinkage,
+        llvm::Twine("xla_impl.") + fn_name, module);
+    fn->replaceAllUsesWith(new_fn);
+    fn->eraseFromParent();
+    fn = new_fn;
+  }
 
-  llvm::BasicBlock* vector_tanh_body =
-      llvm::BasicBlock::Create(*context, "body", vector_tanh_function);
+  llvm::LLVMContext* context = &module->getContext();
 
-  llvm::IRBuilder<> b(vector_tanh_body);
+  llvm::BasicBlock* fn_body = llvm::BasicBlock::Create(*context, "body", fn);
+  llvm::IRBuilder<> b(fn_body);
   llvm::FastMathFlags fast_math_flags;
   fast_math_flags.setFast(enable_fast_math);
   b.setFastMathFlags(fast_math_flags);
 
-  llvm::Value* input = &*vector_tanh_function->arg_begin();
-  CHECK_EQ(vector_width, input->getType()->getVectorNumElements());
-  b.CreateRet(llvm_ir::EmitFastTanh(&b, input));
-
-  DCHECK(!llvm::verifyFunction(*vector_tanh_function));
-  return vector_tanh_function;
-}
+  llvm::Value* input = &*fn->arg_begin();
 
-llvm::Function* EmitVectorF32ExpIfNeeded(llvm::Module* module,
-                                         llvm::StringRef function_name,
-                                         int vector_width,
-                                         bool enable_fast_math) {
-  llvm::Function* vector_exp_function = module->getFunction(function_name);
-  if (vector_exp_function == nullptr) {
-    // If the function declaration is not present in the module, there can't be
-    // any calls to resolve.  Don't emit the function in this case.
-    return nullptr;
+  // Upcast to vector type if input is a scalar.
+  if (vector_width == 1) {
+    llvm::Type* v1_type = llvm::VectorType::get(input->getType(), 1);
+    input = b.CreateInsertElement(llvm::UndefValue::get(v1_type), input,
+                                  uint64_t{0});
   }
 
-  llvm::LLVMContext* context = &module->getContext();
+  // Generate the vectorized code.
+  CHECK_EQ(vector_width, input->getType()->getVectorNumElements());
+  llvm::Value* result = fn_body_generator(&b, input, vector_width);
 
-  llvm::BasicBlock* vector_exp_body =
-      llvm::BasicBlock::Create(*context, "body", vector_exp_function);
+  // Downcast result to scalar type if necessary.
+  if (vector_width == 1) {
+    result = b.CreateExtractElement(result, uint64_t{0});
+  }
+  b.CreateRet(result);
+  DCHECK(!llvm::verifyFunction(*fn));
 
-  llvm::IRBuilder<> b(vector_exp_body);
-  llvm::FastMathFlags fast_math_flags;
-  fast_math_flags.setFast(enable_fast_math);
-  b.setFastMathFlags(fast_math_flags);
+  // Force-inline `fn` into all of its callers and then delete `fn`.
+  //
+  // TODO(b/73081976): Should we avoid inlining these in some cases?
+  std::vector<llvm::CallInst*> calls_to_inline;
+  for (auto* user : fn->users()) {
+    calls_to_inline.push_back(llvm::cast<llvm::CallInst>(user));
+  }
+  for (auto* call_to_inline : calls_to_inline) {
+    llvm::InlineFunctionInfo inline_function_info;
+    CHECK(llvm::InlineFunction(call_to_inline, inline_function_info));
+  }
+  fn->eraseFromParent();
+}
 
-  VectorSupportLibrary vsl(F32, vector_width, &b, "exp_f32");
+llvm::Value* GenerateVF32Tanh(llvm::IRBuilder<>* b, llvm::Value* input,
+                              int32 /*vector_width*/) {
+  return llvm_ir::EmitFastTanh(b, input);
+}
+
+llvm::Value* GenerateVF32Exp(llvm::IRBuilder<>* b, llvm::Value* input,
+                             int32 vector_width) {
+  VectorSupportLibrary vsl(F32, vector_width, b, "exp_f32");
 
   // This implements the same polynomial approximation as implemented in Eigen3.
 
@@ -107,7 +138,6 @@ llvm::Function* EmitVectorF32ExpIfNeeded(llvm::Module* module,
   const llvm::APFloat cephes_exp_p4 = GetIeeeF32(1.6666665459E-1);
   const llvm::APFloat cephes_exp_p5 = GetIeeeF32(5.0000001201E-1);
 
-  llvm::Value* input = &*vector_exp_function->arg_begin();
   llvm::Value* input_clamped =
       vsl.Clamp(input, /*low=*/exp_lo, /*high=*/exp_hi);
   llvm::Value* fx = vsl.Floor(vsl.MulAdd(input_clamped, cephes_LOG2EF, half));
@@ -128,49 +158,24 @@ llvm::Function* EmitVectorF32ExpIfNeeded(llvm::Module* module,
   // VectorSupportLibrary (intentionally) can't juggle more than one type at a
   // time so drop down to IRBuilder for this bit.
   llvm::Value* vector_constant_0x7f =
-      b.CreateVectorSplat(vector_width, b.getInt32(0x7f));
+      b->CreateVectorSplat(vector_width, b->getInt32(0x7f));
   llvm::Value* vector_constant_23 =
-      b.CreateVectorSplat(vector_width, b.getInt32(23));
+      b->CreateVectorSplat(vector_width, b->getInt32(23));
   llvm::Type* i32_vector_type =
-      llvm::VectorType::get(b.getInt32Ty(), vector_width);
+      llvm::VectorType::get(b->getInt32Ty(), vector_width);
   // fx is clamped so we don't have to worry about it being out of range for
   // i32.
-  llvm::Value* emm0 = b.CreateFPToSI(fx, i32_vector_type);
-  emm0 = b.CreateAdd(emm0, vector_constant_0x7f);
-  emm0 = b.CreateShl(emm0, vector_constant_23);
-  llvm::Value* emm0_f32 = b.CreateBitCast(emm0, vsl.vector_type());
-
-  llvm::Value* result = vsl.Max(vsl.Mul(y, emm0_f32), input);
+  llvm::Value* emm0 = b->CreateFPToSI(fx, i32_vector_type);
+  emm0 = b->CreateAdd(emm0, vector_constant_0x7f);
+  emm0 = b->CreateShl(emm0, vector_constant_23);
+  llvm::Value* emm0_f32 = b->CreateBitCast(emm0, vsl.vector_type());
 
-  b.CreateRet(result);
-
-  DCHECK(!llvm::verifyFunction(*vector_exp_function));
-  return vector_exp_function;
+  return vsl.Max(vsl.Mul(y, emm0_f32), input);
 }
 
-llvm::Function* EmitVectorF32LogIfNeeded(llvm::Module* module,
-                                         llvm::StringRef function_name,
-                                         int vector_width,
-                                         bool enable_fast_math) {
-  llvm::Function* vector_log_function = module->getFunction(function_name);
-  if (vector_log_function == nullptr) {
-    // If the function declaration is not present in the module, there can't be
-    // any calls to resolve.  Don't emit the function in this case.
-    return nullptr;
-  }
-
-  llvm::LLVMContext* context = &module->getContext();
-
-  llvm::BasicBlock* vector_log_body =
-      llvm::BasicBlock::Create(*context, "body", vector_log_function);
-
-  llvm::IRBuilder<> b(vector_log_body);
-  llvm::FastMathFlags fast_math_flags;
-  fast_math_flags.setFast(enable_fast_math);
-  b.setFastMathFlags(fast_math_flags);
-
-  llvm::Value* input = &*vector_log_function->arg_begin();
-  VectorSupportLibrary vsl(F32, vector_width, &b, "log_f32");
+llvm::Value* GenerateVF32Log(llvm::IRBuilder<>* b, llvm::Value* input,
+                             int32 vector_width) {
+  VectorSupportLibrary vsl(F32, vector_width, b, "log_f32");
 
   const llvm::APFloat half = GetIeeeF32(0.5);
   const llvm::APFloat one = GetIeeeF32(1.0);
@@ -206,21 +211,21 @@ llvm::Function* EmitVectorF32LogIfNeeded(llvm::Module* module,
   // VectorSupportLibrary (intentionally) can't juggle more than one type at a
   // time so drop down to IRBuilder for this bit.
   llvm::Value* vector_constant_0x7f =
-      b.CreateVectorSplat(vector_width, b.getInt32(0x7f));
+      b->CreateVectorSplat(vector_width, b->getInt32(0x7f));
   llvm::Value* vector_constant_23 =
-      b.CreateVectorSplat(vector_width, b.getInt32(23));
+      b->CreateVectorSplat(vector_width, b->getInt32(23));
   llvm::Type* i32_vector_type =
-      llvm::VectorType::get(b.getInt32Ty(), vector_width);
+      llvm::VectorType::get(b->getInt32Ty(), vector_width);
 
-  llvm::Value* emm0 =
-      b.CreateLShr(b.CreateBitCast(input, i32_vector_type), vector_constant_23);
+  llvm::Value* emm0 = b->CreateLShr(b->CreateBitCast(input, i32_vector_type),
+                                    vector_constant_23);
 
   // Keep only the fractional part.
   input = vsl.FloatAnd(input, inv_mant_mask);
   input = vsl.FloatOr(input, half);
 
-  emm0 = b.CreateSub(emm0, vector_constant_0x7f);
-  llvm::Value* e = vsl.Add(one, b.CreateSIToFP(emm0, vsl.vector_type()));
+  emm0 = b->CreateSub(emm0, vector_constant_0x7f);
+  llvm::Value* e = vsl.Add(one, b->CreateSIToFP(emm0, vsl.vector_type()));
 
   // part2:
   //   if( x < SQRTHF ) {
@@ -259,63 +264,30 @@ llvm::Function* EmitVectorF32LogIfNeeded(llvm::Module* module,
   llvm::Value* or_lhs =
       vsl.FloatAndNot(iszero_mask, vsl.FloatOr(input, invalid_mask));
   llvm::Value* or_rhs = vsl.FloatAnd(iszero_mask, minus_inf);
-  llvm::Value* result = vsl.FloatOr(or_lhs, or_rhs);
-
-  b.CreateRet(result);
-
-  DCHECK(!llvm::verifyFunction(*vector_log_function));
-  return vector_log_function;
+  return vsl.FloatOr(or_lhs, or_rhs);
 }
 }  // namespace
 
 void RewriteIRRuntimeFunctions(llvm::Module* module, bool enable_fast_math) {
-  auto* tanh_v4f32 =
-      EmitVectorF32TanhIfNeeded(module, kTanhV4F32SymbolName,
-                                /*vector_width=*/4, enable_fast_math);
-  auto* tanh_v8f32 =
-      EmitVectorF32TanhIfNeeded(module, kTanhV8F32SymbolName,
-                                /*vector_width=*/8, enable_fast_math);
-
-  auto* exp_v4f32 =
-      EmitVectorF32ExpIfNeeded(module, kExpV4F32SymbolName,
-                               /*vector_width=*/4, enable_fast_math);
-  auto* exp_v8f32 =
-      EmitVectorF32ExpIfNeeded(module, kExpV8F32SymbolName,
-                               /*vector_width=*/8, enable_fast_math);
-
-  auto* log_v4f32 =
-      EmitVectorF32LogIfNeeded(module, kLogV4F32SymbolName,
-                               /*vector_width=*/4, enable_fast_math);
-  auto* log_v8f32 =
-      EmitVectorF32LogIfNeeded(module, kLogV8F32SymbolName,
-                               /*vector_width=*/8, enable_fast_math);
-
-  // Gather all the call sites, force inline them and then delete the vector
-  // function bodies.
-  //
-  // TODO(b/73081976): Should we avoid inlining these intrinsics in some cases?
-
-  std::vector<llvm::CallInst*> calls_to_inline;
-  for (auto* function :
-       {tanh_v4f32, tanh_v8f32, exp_v4f32, exp_v8f32, log_v4f32, log_v8f32}) {
-    if (function != nullptr) {
-      for (auto* user : function->users()) {
-        calls_to_inline.push_back(llvm::cast<llvm::CallInst>(user));
-      }
-    }
-  }
-
-  for (auto* call_to_inline : calls_to_inline) {
-    llvm::InlineFunctionInfo inline_function_info;
-    CHECK(llvm::InlineFunction(call_to_inline, inline_function_info));
-  }
-
-  for (auto* function :
-       {tanh_v4f32, tanh_v8f32, exp_v4f32, exp_v8f32, log_v4f32, log_v8f32}) {
-    if (function != nullptr) {
-      function->eraseFromParent();
-    }
-  }
+  // Curry some params to RewriteCalls.
+  auto rewrite_calls =
+      std::bind(RewriteCalls, module, std::placeholders::_1,
+                std::placeholders::_2, std::placeholders::_3, enable_fast_math);
+
+  rewrite_calls("tanhf", GenerateVF32Tanh, /*vector_width=*/1);
+  rewrite_calls("llvm.tanh.f32", GenerateVF32Tanh, /*vector_width=*/1);
+  rewrite_calls(kTanhV4F32SymbolName, GenerateVF32Tanh, /*vector_width=*/4);
+  rewrite_calls(kTanhV8F32SymbolName, GenerateVF32Tanh, /*vector_width=*/8);
+
+  rewrite_calls("expf", GenerateVF32Exp, /*vector_width=*/1);
+  rewrite_calls("llvm.exp.f32", GenerateVF32Exp, /*vector_width=*/1);
+  rewrite_calls(kExpV4F32SymbolName, GenerateVF32Exp, /*vector_width=*/4);
+  rewrite_calls(kExpV8F32SymbolName, GenerateVF32Exp, /*vector_width=*/8);
+
+  rewrite_calls("logf", GenerateVF32Log, /*vector_width=*/1);
+  rewrite_calls("llvm.log.f32", GenerateVF32Log, /*vector_width=*/1);
+  rewrite_calls(kLogV4F32SymbolName, GenerateVF32Log, /*vector_width=*/4);
+  rewrite_calls(kLogV8F32SymbolName, GenerateVF32Log, /*vector_width=*/8);
 }
 
 }  // namespace runtime
diff --git a/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc b/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc
index 87e912fa28..3d32ca7c67 100644
--- a/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc
+++ b/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc
@@ -122,9 +122,14 @@ class ExhaustiveF32ElementwiseOpTest
         constexpr int64 kMaxMismatchesPrinted = 1000;
         mismatches++;
         if (mismatches < kMaxMismatchesPrinted || VLOG_IS_ON(2)) {
-          LOG(ERROR) << "Mismatch on " << input << " (0x"
-                     << absl::StrCat(absl::Hex(input, absl::kZeroPad8))
-                     << "). Expected " << expected << ", but got " << actual;
+          // Use %0.9g because that's guaranteed to print an f32 to full
+          // precision.
+          LOG(ERROR) << absl::StreamFormat(
+              "Mismatch on %0.9g (0x%08x). Expected %0.9g (0x%08x), but got "
+              "%0.9g (0x%08x).",
+              input, absl::bit_cast<uint32>(input),        //
+              expected, absl::bit_cast<uint32>(expected),  //
+              actual, absl::bit_cast<uint32>(actual));
         }
         if (mismatches == kMaxMismatchesPrinted && !VLOG_IS_ON(2)) {
           LOG(ERROR) << "Not printing any more mismatches; pass "
-- 
GitLab


From 3dd58ab5e3ba4e688a0f7842f5f971f09ec82d5d Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Thu, 31 Jan 2019 16:01:33 -0800
Subject: [PATCH 1708/2345] Add constructor to TensorData.

PiperOrigin-RevId: 231872037
---
 tensorflow/lite/kernels/test_util.h | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/tensorflow/lite/kernels/test_util.h b/tensorflow/lite/kernels/test_util.h
index 83f0868595..9bec8ce3ce 100644
--- a/tensorflow/lite/kernels/test_util.h
+++ b/tensorflow/lite/kernels/test_util.h
@@ -85,6 +85,24 @@ inline std::vector<float> Dequantize(const std::vector<T>& data, float scale,
 // the actual data is known. This mimics what happens in practice: quantization
 // parameters are calculated during training or post training..
 struct TensorData {
+  TensorData(TensorType type = TensorType_FLOAT32, std::vector<int> shape = {},
+             float min = 0.0f, float max = 0.0f, float scale = 0.0f,
+             int32_t zero_point = 0, bool per_channel_quantization = false,
+             std::vector<float> per_channel_quantization_scales = {},
+             std::vector<int64_t> per_channel_quantization_offsets = {},
+             int32_t channel_index = 0)
+      : type(type),
+        shape(shape),
+        min(min),
+        max(max),
+        scale(scale),
+        zero_point(zero_point),
+        per_channel_quantization(per_channel_quantization),
+        per_channel_quantization_scales(
+            std::move(per_channel_quantization_scales)),
+        per_channel_quantization_offsets(
+            std::move(per_channel_quantization_offsets)),
+        channel_index(channel_index) {}
   TensorType type;
   std::vector<int> shape;
   float min;
-- 
GitLab


From 10430cf1f67688364806026757288c4d8b8440f1 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 31 Jan 2019 16:04:19 -0800
Subject: [PATCH 1709/2345] [XLA] Handle denormals explicitly in
 exhaustive_f32_elementwise_op_test.

Previously, we expected that fn(x) would return something close to std::fn(x),
even if x was denormal.  Now, we accept a result that's close to either
std::fn(x) or std::fn(0).

As an example of why we need this fix, let x be a negative denormal, and
consider log(x).  log of any negative number (other than -0) should be nan.
But log(+/-0) is -inf.

XLA:CPU (and indeed, all of TF) runs in a mode where if x is denormal, x == 0
is true.  So although XLA:CPU's generated code correctly handles negative
denormal inputs to log, it's thwarted by this hardware mode and returns -inf
for log(x).
PiperOrigin-RevId: 231872731
---
 .../exhaustive_f32_elementwise_op_test.cc     | 76 ++++++++++++-------
 1 file changed, 50 insertions(+), 26 deletions(-)

diff --git a/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc b/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc
index 3d32ca7c67..e442b7b26d 100644
--- a/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc
+++ b/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc
@@ -13,6 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include <cmath>
 #include "absl/base/casts.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/tests/client_library_test_base.h"
@@ -21,12 +22,22 @@ limitations under the License.
 
 namespace xla {
 namespace {
+
 class ExhaustiveF32ElementwiseOpTest
     : public ClientLibraryTestBase,
       public ::testing::WithParamInterface<std::pair<int64, int64>> {
  protected:
   ErrorSpec error_spec_{0.0001, 0.0001};
 
+  bool IsClose(float expected, float actual) {
+    float abs_err = std::abs(expected - actual);
+    float rel_err = abs_err / std::abs(expected);
+    return abs_err < error_spec_.abs || rel_err < error_spec_.rel ||
+           (std::isnan(expected) && std::isnan(actual)) ||
+           (std::isinf(expected) && std::isinf(actual) &&
+            (expected > 0) == (actual > 0));
+  }
+
   template <typename EnqueueOpTy>
   void ExhaustivelyTestF32Op(EnqueueOpTy enqueue_op,
                              float (*evaluate_op)(float),
@@ -104,26 +115,47 @@ class ExhaustiveF32ElementwiseOpTest
     //  b) we can print out better error messages (namely, we can print out
     //     which floating-point value input failed, while LiteralTestUtil::Near
     //     can only print out the input index that failed).
+    //  c) we need special handling of certain inputs.  For example, we say that
+    //     a denormal input has multiple correct outputs (namely, f(x) and f(0))
+    //     and just needs to be close to one of them.
     absl::Span<float> result_arr = result_literal.data<float>();
     ASSERT_EQ(result_arr.size(), input_arr.size());
     int64 mismatches = 0;
+    // Hoisting this out of the loop is a nice speedup on shards that have many
+    // denormals.
+    const float expected_at_zero = evaluate_op(0);
     for (int64 i = 0; i < input_arr.size(); ++i) {
       float input = ith_input_elem(i);
-      float expected = evaluate_op(input);
       float actual = result_arr[i];
-      float abs_err = std::abs(expected - actual);
-      float rel_err = abs_err / std::abs(expected);
-      if (abs_err < error_spec_.abs || rel_err < error_spec_.rel ||
-          (std::isnan(expected) && std::isnan(actual)) ||
-          (std::isinf(expected) && std::isinf(actual) &&
-           (expected > 0) == (actual > 0))) {
-        // Successful match!  Nothing to do.
-      } else {
-        constexpr int64 kMaxMismatchesPrinted = 1000;
-        mismatches++;
+      float expected = evaluate_op(input);
+      if (IsClose(expected, actual)) {
+        continue;
+      }
+
+      constexpr int64 kMaxMismatchesPrinted = 1000;
+      if (std::fpclassify(input) == FP_SUBNORMAL) {
+        // For denormal inputs, we accept answers that are close to either
+        //   - evaluate_op(input) OR
+        //   - evaluate_op(0).
+        if (IsClose(expected_at_zero, actual)) {
+          continue;
+        }
+        ++mismatches;
         if (mismatches < kMaxMismatchesPrinted || VLOG_IS_ON(2)) {
           // Use %0.9g because that's guaranteed to print an f32 to full
           // precision.
+          LOG(ERROR) << absl::StreamFormat(
+              "Mismatch on denormal value %0.9g (0x%08x). Expected either "
+              "%0.9g (0x%08x) (evaluated at true value) or %0.9g (0x%08x) "
+              "(evaluated at zero), but got %0.9g (0x%08x).",
+              input, absl::bit_cast<uint32>(input),        //
+              expected, absl::bit_cast<uint32>(expected),  //
+              expected_at_zero, absl::bit_cast<uint32>(expected_at_zero),
+              actual, absl::bit_cast<uint32>(actual));
+        }
+      } else {
+        mismatches++;
+        if (mismatches < kMaxMismatchesPrinted || VLOG_IS_ON(2)) {
           LOG(ERROR) << absl::StreamFormat(
               "Mismatch on %0.9g (0x%08x). Expected %0.9g (0x%08x), but got "
               "%0.9g (0x%08x).",
@@ -131,11 +163,12 @@ class ExhaustiveF32ElementwiseOpTest
               expected, absl::bit_cast<uint32>(expected),  //
               actual, absl::bit_cast<uint32>(actual));
         }
-        if (mismatches == kMaxMismatchesPrinted && !VLOG_IS_ON(2)) {
-          LOG(ERROR) << "Not printing any more mismatches; pass "
-                        "--vmodule=exhaustive_f32_elementwise_op_test=2 to see "
-                        "all of them.";
-        }
+      }
+
+      if (mismatches == kMaxMismatchesPrinted && !VLOG_IS_ON(2)) {
+        LOG(ERROR) << "Not printing any more mismatches; pass "
+                      "--vmodule=exhaustive_f32_elementwise_op_test=2 to see "
+                      "all of them.";
       }
     }
     EXPECT_EQ(mismatches, 0);
@@ -143,18 +176,9 @@ class ExhaustiveF32ElementwiseOpTest
 };
 
 XLA_TEST_P(ExhaustiveF32ElementwiseOpTest, LogF32) {
-#ifdef XLA_TEST_BACKEND_CPU
-  // TODO(b/73141998): The vectorized Log implementation gives results outside
-  // our error spec in this range (these numbers are bitwise representations of
-  // floats expressed as a zero extended int64).
-  std::pair<int64, int64> known_incorrect_range = {1, 8388608};
-#else
-  std::pair<int64, int64> known_incorrect_range = {0, 0};
-#endif
-
   ExhaustivelyTestF32Op(
       [](XlaBuilder* builder, const XlaOp& input) { Log(input); }, std::log,
-      known_incorrect_range);
+      /*known_incorrect_range=*/{0, 0});
 }
 
 XLA_TEST_P(ExhaustiveF32ElementwiseOpTest, ExpF32) {
-- 
GitLab


From 4045b52a3c91dd761722c38b2777aea6c97508ea Mon Sep 17 00:00:00 2001
From: Duncan Riach <duncan@nvidia.com>
Date: Thu, 31 Jan 2019 17:30:18 -0800
Subject: [PATCH 1710/2345] Improves docstring for
 tf.contrib.nn.conv1d_transpose

---
 tensorflow/python/ops/nn_ops.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 51c1f15bac..c06a0c2614 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -3988,21 +3988,22 @@ def conv1d_transpose(
     filter: A 3-D `Tensor` with the same type as `value` and shape
       `[filter_width, output_channels, in_channels]`.  `filter`'s
       `in_channels` dimension must match that of `value`.
-    output_shape: A 1-D `Tensor` representing the output shape of the
-      deconvolution op.
+    output_shape: A 1-D `Tensor`, containing three elements, representing the
+      output shape of the deconvolution op.
     stride: An `integer`.  The number of entries by which
       the filter is moved right at each step.
     padding: A string, either `'VALID'` or `'SAME'`. The padding algorithm.
       See the "returns" section of `tf.nn.convolution` for details.
-    data_format: A string. 'NHWC' and 'NCHW' are supported.
+    data_format: A string. `'NWC'` and `'NCW'` are supported.
     name: Optional name for the returned tensor.
 
   Returns:
     A `Tensor` with the same type as `value`.
 
   Raises:
-    ValueError: If input/output depth does not match `filter`'s shape, or if
-      padding is other than `'VALID'` or `'SAME'`.
+    ValueError: If input/output depth does not match `filter`'s shape, if
+      `output_shape` is not at 3-element vector, if `padding` is other than
+      `'VALID'` or `'SAME'`, or if `data_format` is invalid.
   """
   with ops.name_scope(name, "conv1d_transpose",
                       [value, filter, output_shape]) as name:
-- 
GitLab


From db9bb4a8c24671b6bd083e7cda110d7e8e6946e6 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Thu, 31 Jan 2019 16:06:01 -0800
Subject: [PATCH 1711/2345] Be able to parse multiple subgraphs from TFLite
 models.

PiperOrigin-RevId: 231873030
---
 tensorflow/lite/core/subgraph.cc |  8 ++-
 tensorflow/lite/core/subgraph.h  | 38 ++++++++++---
 tensorflow/lite/interpreter.cc   |  9 ++-
 tensorflow/lite/model.cc         | 97 ++++++++++++++++++--------------
 tensorflow/lite/model.h          |  4 +-
 tensorflow/lite/model_test.cc    | 23 ++++----
 6 files changed, 112 insertions(+), 67 deletions(-)

diff --git a/tensorflow/lite/core/subgraph.cc b/tensorflow/lite/core/subgraph.cc
index ab83456229..2fdafa356c 100644
--- a/tensorflow/lite/core/subgraph.cc
+++ b/tensorflow/lite/core/subgraph.cc
@@ -397,6 +397,10 @@ void Subgraph::SetCancellationFunction(void* data,
   check_cancelled_func_ = check_cancelled_func;
 }
 
+void Subgraph::ReserveNodes(int count) {
+  nodes_and_registration_.reserve(count);
+}
+
 TfLiteStatus Subgraph::CheckTensorIndices(const char* label, const int* indices,
                                           int length) {
   // Making sure kOptionalTensor is not re-defined to something other than -1.
@@ -410,7 +414,9 @@ TfLiteStatus Subgraph::CheckTensorIndices(const char* label, const int* indices,
       continue;
     }
     if (index < 0 || static_cast<size_t>(index) >= context_->tensors_size) {
-      ReportError("Invalid tensor index %d in %s\n", index, label);
+      ReportError(
+          "Invalid tensor index %d in %s. The subgraph has %d tensors\n", index,
+          label, context_->tensors_size);
       consistent_ = false;
       return kTfLiteError;
     }
diff --git a/tensorflow/lite/core/subgraph.h b/tensorflow/lite/core/subgraph.h
index 5ca2977e0c..5db15a177e 100644
--- a/tensorflow/lite/core/subgraph.h
+++ b/tensorflow/lite/core/subgraph.h
@@ -59,6 +59,11 @@ class Subgraph {
   // interpreter.
   TfLiteStatus SetVariables(std::vector<int> variables);
 
+  // Ensure the internal node storage memory allocates at least `count`
+  // spots for node. NOTE, this doesn't actually add operators. This is an
+  // efficiency optimization that is subject to change.
+  void ReserveNodes(int count);
+
   // Adds a node with the given parameters and returns the index of the new
   // node in `node_index` (optionally). Interpreter will take ownership of
   // `builtin_data` and destroy it with `free`. Ownership of 'init_data'
@@ -68,33 +73,48 @@ class Subgraph {
                                      const char* init_data,
                                      size_t init_data_size, void* builtin_data,
                                      const TfLiteRegistration* registration,
-                                     int* node_index);
+                                     int* node_index = nullptr);
 
   // Adds `tensors_to_add` tensors, preserving pre-existing Tensor entries.
   // The value pointed to by `first_new_tensor_index` will be set to the
   // index of the first new tensor if `first_new_tensor_index` is non-null.
-  TfLiteStatus AddTensors(int tensors_to_add, int* first_new_tensor_index);
+  TfLiteStatus AddTensors(int tensors_to_add,
+                          int* first_new_tensor_index = nullptr);
 
   // Set description of inputs/outputs/data/fptrs for node `node_index`.
   // This variant assumes an external buffer has been allocated of size
   // bytes. The lifetime of buffer must be ensured to be greater or equal
   // to Interpreter.
-  TfLiteStatus SetTensorParametersReadOnly(int tensor_index, TfLiteType type,
-                                           const char* name, const size_t rank,
-                                           const int* dims,
-                                           TfLiteQuantization quantization,
-                                           const char* buffer, size_t bytes,
-                                           const Allocation* allocation);
+  inline TfLiteStatus SetTensorParametersReadOnly(
+      int tensor_index, TfLiteType type, const char* name,
+      const std::vector<int>& dims, TfLiteQuantization quantization,
+      const char* buffer, size_t bytes,
+      const Allocation* allocation = nullptr) {
+    return SetTensorParametersReadOnly(tensor_index, type, name, dims.size(),
+                                       dims.data(), quantization, buffer, bytes,
+                                       allocation);
+  }
+  TfLiteStatus SetTensorParametersReadOnly(
+      int tensor_index, TfLiteType type, const char* name, const size_t rank,
+      const int* dims, TfLiteQuantization quantization, const char* buffer,
+      size_t bytes, const Allocation* allocation = nullptr);
 
   // Set description of inputs/outputs/data/fptrs for node `node_index`.
   // This variant assumes an external buffer has been allocated of size
   // bytes. The lifetime of buffer must be ensured to be greater or equal
   // to Interpreter.
+  inline TfLiteStatus SetTensorParametersReadWrite(
+      int tensor_index, TfLiteType type, const char* name,
+      const std::vector<int>& dims, TfLiteQuantization quantization,
+      bool is_variable = false) {
+    return SetTensorParametersReadWrite(tensor_index, type, name, dims.size(),
+                                        dims.data(), quantization, is_variable);
+  }
   TfLiteStatus SetTensorParametersReadWrite(int tensor_index, TfLiteType type,
                                             const char* name, const size_t rank,
                                             const int* dims,
                                             TfLiteQuantization quantization,
-                                            bool is_variable);
+                                            bool is_variable = false);
 
   // WARNING: Experimental interface, subject to change
   // Overrides execution plan. This bounds checks indices sent in.
diff --git a/tensorflow/lite/interpreter.cc b/tensorflow/lite/interpreter.cc
index 7a6a074ded..c840c9a759 100644
--- a/tensorflow/lite/interpreter.cc
+++ b/tensorflow/lite/interpreter.cc
@@ -91,7 +91,7 @@ TfLiteStatus Interpreter::AllocateTensors() {
 }
 
 void Interpreter::ReserveNodes(int count) {
-  primary_subgraph().nodes_and_registration().reserve(count);
+  primary_subgraph().ReserveNodes(count);
 }
 
 void Interpreter::AddSubgraphs(int subgraphs_to_add,
@@ -223,7 +223,12 @@ void Interpreter::SetCancellationFunction(void* data,
 }
 
 TfLiteStatus Interpreter::ModifyGraphWithDelegate(TfLiteDelegate* delegate) {
-  return primary_subgraph().ModifyGraphWithDelegate(delegate);
+  // TODO(ycling): It seems Flex delegate doesn't work on non-primary subgraphs.
+  // Need to investigate.
+  for (auto& subgraph : subgraphs_) {
+    TF_LITE_ENSURE_OK(context_, subgraph->ModifyGraphWithDelegate(delegate));
+  }
+  return kTfLiteOk;
 }
 
 TfLiteStatus Interpreter::SetBufferHandle(int tensor_index,
diff --git a/tensorflow/lite/model.cc b/tensorflow/lite/model.cc
index c736685a98..e333138fb7 100644
--- a/tensorflow/lite/model.cc
+++ b/tensorflow/lite/model.cc
@@ -246,11 +246,11 @@ class MallocDataAllocator : public BuiltinDataAllocator {
 
 TfLiteStatus InterpreterBuilder::ParseNodes(
     const flatbuffers::Vector<flatbuffers::Offset<Operator>>* operators,
-    Interpreter* interpreter) {
+    Subgraph* subgraph) {
   TfLiteStatus status = kTfLiteOk;
 
   // Reduce the number of redundant allocations
-  interpreter->ReserveNodes(operators->Length());
+  subgraph->ReserveNodes(operators->Length());
 
   for (int i = 0; i < operators->Length(); ++i) {
     const auto* op = operators->Get(i);
@@ -280,7 +280,7 @@ TfLiteStatus InterpreterBuilder::ParseNodes(
     }
 
     if (op->custom_options()) {
-      interpreter->AddNodeWithParameters(
+      subgraph->AddNodeWithParameters(
           FlatBufferIntArrayToVector(op->inputs()),
           FlatBufferIntArrayToVector(op->outputs()),
           reinterpret_cast<const char*>(op->custom_options()->data()),
@@ -290,10 +290,9 @@ TfLiteStatus InterpreterBuilder::ParseNodes(
       MallocDataAllocator malloc_allocator;
       TF_LITE_ENSURE_STATUS(ParseOpData(op, op_type, error_reporter_,
                                         &malloc_allocator, &builtin_data));
-      interpreter->AddNodeWithParameters(
-          FlatBufferIntArrayToVector(op->inputs()),
-          FlatBufferIntArrayToVector(op->outputs()), nullptr, 0, builtin_data,
-          registration);
+      subgraph->AddNodeWithParameters(FlatBufferIntArrayToVector(op->inputs()),
+                                      FlatBufferIntArrayToVector(op->outputs()),
+                                      nullptr, 0, builtin_data, registration);
     }
   }
 
@@ -353,11 +352,11 @@ TfLiteStatus InterpreterBuilder::ParseQuantization(
 TfLiteStatus InterpreterBuilder::ParseTensors(
     const flatbuffers::Vector<flatbuffers::Offset<Buffer>>* buffers,
     const flatbuffers::Vector<flatbuffers::Offset<Tensor>>* tensors,
-    Interpreter* interpreter) {
+    Subgraph* subgraph) {
   TfLiteStatus status = kTfLiteOk;
 
   // A little helper to get the names of inputs and outputs. Note that they
-  // must outlive the interpreter.
+  // must outlive the subgraph.
   auto get_name = [](const tflite::Tensor* t) -> const char* {
     auto name = t->name();
     if (name) return name->c_str();
@@ -418,7 +417,7 @@ TfLiteStatus InterpreterBuilder::ParseTensors(
         status = kTfLiteError;
       }
 
-      if (interpreter->SetTensorParametersReadOnly(
+      if (subgraph->SetTensorParametersReadOnly(
               i, type, get_name(tensor), dims, quantization, buffer_ptr,
               buffer_size, allocation_) != kTfLiteOk) {
         error_reporter_->Report("Tensor %d is invalidly specified in schema.\n",
@@ -426,9 +425,9 @@ TfLiteStatus InterpreterBuilder::ParseTensors(
         status = kTfLiteError;
       }
     } else {
-      if (interpreter->SetTensorParametersReadWrite(i, type, get_name(tensor),
-                                                    dims, quantization,
-                                                    is_variable) != kTfLiteOk) {
+      if (subgraph->SetTensorParametersReadWrite(i, type, get_name(tensor),
+                                                 dims, quantization,
+                                                 is_variable) != kTfLiteOk) {
         error_reporter_->Report("Tensor %d is invalidly specified in schema.\n",
                                 i);
         status = kTfLiteError;
@@ -510,42 +509,56 @@ TfLiteStatus InterpreterBuilder::operator()(
   // Construct interpreter with correct number of tensors and operators.
   auto* subgraphs = model_->subgraphs();
   auto* buffers = model_->buffers();
-  if (subgraphs->size() != 1) {
-    error_reporter_->Report("Only 1 subgraph is currently supported.\n");
-    return cleanup_and_error();
-  }
-  const tflite::SubGraph* subgraph = (*subgraphs)[0];
-  auto operators = subgraph->operators();
-  auto tensors = subgraph->tensors();
-  if (!operators || !tensors || !buffers) {
-    error_reporter_->Report(
-        "Did not get operators, tensors, or buffers in input flat buffer.\n");
+
+  if (subgraphs->size() == 0) {
+    error_reporter_->Report("No subgraph in the model.\n");
     return cleanup_and_error();
   }
+
   interpreter->reset(new Interpreter(error_reporter_));
-  if ((**interpreter).AddTensors(tensors->Length()) != kTfLiteOk) {
-    return cleanup_and_error();
+  (*interpreter)->SetNumThreads(num_threads);
+  if (subgraphs->Length() > 1) {
+    (*interpreter)->AddSubgraphs(subgraphs->Length() - 1);
   }
-  // Set num threads
-  (**interpreter).SetNumThreads(num_threads);
-  // Parse inputs/outputs
-  (**interpreter).SetInputs(FlatBufferIntArrayToVector(subgraph->inputs()));
-  (**interpreter).SetOutputs(FlatBufferIntArrayToVector(subgraph->outputs()));
-
-  // Finally setup nodes and tensors
-  if (ParseNodes(operators, interpreter->get()) != kTfLiteOk)
-    return cleanup_and_error();
-  if (ParseTensors(buffers, tensors, interpreter->get()) != kTfLiteOk)
-    return cleanup_and_error();
 
-  std::vector<int> variables;
-  for (int i = 0; i < (*interpreter)->tensors_size(); ++i) {
-    auto* tensor = (*interpreter)->tensor(i);
-    if (tensor->is_variable) {
-      variables.push_back(i);
+  for (int subgraph_index = 0; subgraph_index < subgraphs->Length();
+       ++subgraph_index) {
+    const tflite::SubGraph* subgraph = (*subgraphs)[subgraph_index];
+    tflite::Subgraph* modified_subgraph =
+        (*interpreter)->subgraph(subgraph_index);
+    auto operators = subgraph->operators();
+    auto tensors = subgraph->tensors();
+    if (!operators || !tensors || !buffers) {
+      error_reporter_->Report(
+          "Did not get operators, tensors, or buffers in subgraph %d.\n",
+          subgraph_index);
+      return cleanup_and_error();
+    }
+    if (modified_subgraph->AddTensors(tensors->Length()) != kTfLiteOk) {
+      return cleanup_and_error();
+    }
+    // Set num threads
+    // Parse inputs/outputs
+    modified_subgraph->SetInputs(
+        FlatBufferIntArrayToVector(subgraph->inputs()));
+    modified_subgraph->SetOutputs(
+        FlatBufferIntArrayToVector(subgraph->outputs()));
+
+    // Finally setup nodes and tensors
+    if (ParseNodes(operators, modified_subgraph) != kTfLiteOk)
+      return cleanup_and_error();
+    if (ParseTensors(buffers, tensors, modified_subgraph) != kTfLiteOk)
+      return cleanup_and_error();
+
+    std::vector<int> variables;
+    for (int i = 0; i < modified_subgraph->tensors_size(); ++i) {
+      auto* tensor = modified_subgraph->tensor(i);
+      if (tensor->is_variable) {
+        variables.push_back(i);
+      }
     }
+    modified_subgraph->SetVariables(std::move(variables));
   }
-  (**interpreter).SetVariables(std::move(variables));
 
   if (ApplyDelegates(interpreter->get()) != kTfLiteOk)
     return cleanup_and_error();
diff --git a/tensorflow/lite/model.h b/tensorflow/lite/model.h
index a9bd4c9c09..bae4229cba 100644
--- a/tensorflow/lite/model.h
+++ b/tensorflow/lite/model.h
@@ -198,11 +198,11 @@ class InterpreterBuilder {
   TfLiteStatus BuildLocalIndexToRegistrationMapping();
   TfLiteStatus ParseNodes(
       const flatbuffers::Vector<flatbuffers::Offset<Operator>>* operators,
-      Interpreter* interpreter);
+      Subgraph* subgraph);
   TfLiteStatus ParseTensors(
       const flatbuffers::Vector<flatbuffers::Offset<Buffer>>* buffers,
       const flatbuffers::Vector<flatbuffers::Offset<Tensor>>* tensors,
-      Interpreter* interpreter);
+      Subgraph* subgraph);
   TfLiteStatus ApplyDelegates(Interpreter* interpreter);
   TfLiteStatus ParseQuantization(const QuantizationParameters* src_quantization,
                                  TfLiteQuantization* quantization);
diff --git a/tensorflow/lite/model_test.cc b/tensorflow/lite/model_test.cc
index e677ea94a7..67d2380be0 100644
--- a/tensorflow/lite/model_test.cc
+++ b/tensorflow/lite/model_test.cc
@@ -87,20 +87,21 @@ TEST(BasicFlatBufferModel, TestEmptyModelsAndNullDestination) {
 
 // Make sure currently unsupported # of subgraphs are checked
 // TODO(aselle): Replace this test when multiple subgraphs are supported.
-TEST(BasicFlatBufferModel, TestZeroAndMultipleSubgraphs) {
-  auto m1 = FlatBufferModel::BuildFromFile(
+TEST(BasicFlatBufferModel, TestZeroSubgraphs) {
+  auto m = FlatBufferModel::BuildFromFile(
       "tensorflow/lite/testdata/0_subgraphs.bin");
-  ASSERT_TRUE(m1);
-  std::unique_ptr<Interpreter> interpreter1;
-  ASSERT_NE(InterpreterBuilder(*m1, TrivialResolver())(&interpreter1),
-            kTfLiteOk);
+  ASSERT_TRUE(m);
+  std::unique_ptr<Interpreter> interpreter;
+  ASSERT_NE(InterpreterBuilder(*m, TrivialResolver())(&interpreter), kTfLiteOk);
+}
 
-  auto m2 = FlatBufferModel::BuildFromFile(
+TEST(BasicFlatBufferModel, TestMultipleSubgraphs) {
+  auto m = FlatBufferModel::BuildFromFile(
       "tensorflow/lite/testdata/2_subgraphs.bin");
-  ASSERT_TRUE(m2);
-  std::unique_ptr<Interpreter> interpreter2;
-  ASSERT_NE(InterpreterBuilder(*m2, TrivialResolver())(&interpreter2),
-            kTfLiteOk);
+  ASSERT_TRUE(m);
+  std::unique_ptr<Interpreter> interpreter;
+  ASSERT_EQ(InterpreterBuilder(*m, TrivialResolver())(&interpreter), kTfLiteOk);
+  EXPECT_EQ(interpreter->subgraphs_size(), 2);
 }
 
 // Test what happens if we cannot bind any of the ops.
-- 
GitLab


From b4150f00e3a94e98d82da3ba150e682b57b1554f Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Thu, 31 Jan 2019 16:12:00 -0800
Subject: [PATCH 1712/2345] Automated rollback of commit
 7157690e660676f8b01aec55d42d1e76101e6fa8

PiperOrigin-RevId: 231874141
---
 tensorflow/python/BUILD                       |    9 +
 tensorflow/python/framework/python_op_gen.cc  |   67 +
 tensorflow/python/ops/raw_ops_test.py         |   42 +
 .../tools/api/generator/api_init_files.bzl    |    1 +
 .../tools/api/generator/api_init_files_v1.bzl |    1 +
 tensorflow/python/util/tf_export.py           |   10 +
 tensorflow/python/util/tf_inspect.py          |   26 +-
 .../tools/api/golden/v1/tensorflow.pbtxt      |    4 +
 .../api/golden/v1/tensorflow.raw_ops.pbtxt    | 4027 +++++++++++++++++
 .../tools/api/golden/v2/tensorflow.pbtxt      |    4 +
 .../api/golden/v2/tensorflow.raw_ops.pbtxt    | 4027 +++++++++++++++++
 11 files changed, 8207 insertions(+), 11 deletions(-)
 create mode 100644 tensorflow/python/ops/raw_ops_test.py
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 32d5d873cb..35acc6f59b 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -650,6 +650,7 @@ py_library(
         "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:core",
         "//tensorflow/python/eager:execute",
+        "//tensorflow/tools/docs:doc_controls",
     ],
 )
 
@@ -6090,3 +6091,11 @@ py_library(
     srcs = ["tf2.py"],
     srcs_version = "PY2AND3",
 )
+
+cuda_py_test(
+    name = "raw_ops_test",
+    srcs = ["ops/raw_ops_test.py"],
+    additional_deps = [
+        "//tensorflow/python:client_testlib",
+    ],
+)
diff --git a/tensorflow/python/framework/python_op_gen.cc b/tensorflow/python/framework/python_op_gen.cc
index ba622d1203..b0b9ce7055 100644
--- a/tensorflow/python/framework/python_op_gen.cc
+++ b/tensorflow/python/framework/python_op_gen.cc
@@ -144,6 +144,8 @@ class GenEagerPythonOp : public python_op_gen_internal::GenPythonOp {
                        const string& num_outputs_expr);
   void AddDispatch(const string& prefix);
 
+  void AddRawOpExport();
+
   void AddAttrForArg(const string& attr, int arg_index) {
     gtl::InsertIfNotPresent(&inferred_attrs_, attr,
                             op_def_.input_arg(arg_index).name());
@@ -668,6 +670,7 @@ bool GenEagerPythonOp::AddEagerFastPathAndGraphCode(
   AddEagerFunctionTeardown("  ", output_sizes,
                            true /* execute_record_gradient */);
 
+  AddRawOpExport();
   strings::StrAppend(&result_, "\n\n");
   return true;
 }
@@ -921,6 +924,68 @@ void GenEagerPythonOp::AddDispatch(const string& prefix) {
   strings::StrAppend(&result_, prefix, "  raise\n");
 }
 
+void GenEagerPythonOp::AddRawOpExport() {
+  // Create function for python op.
+  string raw_parameters;
+  string function_call_parameters;
+  string inputs;
+  string attrs;
+
+  std::map<string, string> renames;
+
+  for (const auto& param_names : param_names_) {
+    renames.insert({param_names.GetName(), param_names.GetRenameTo()});
+  }
+
+  for (const auto& input_arg : op_def_.input_arg()) {
+    const string input_arg_name =
+        python_op_gen_internal::AvoidPythonReserved(input_arg.name());
+    if (!raw_parameters.empty()) strings::StrAppend(&raw_parameters, ", ");
+    strings::StrAppend(&raw_parameters, input_arg_name);
+
+    if (!inputs.empty()) strings::StrAppend(&inputs, ", ");
+    strings::StrAppend(&inputs, input_arg_name);
+
+    if (!function_call_parameters.empty()) {
+      strings::StrAppend(&function_call_parameters, ", ");
+    }
+    strings::StrAppend(&function_call_parameters, renames[input_arg.name()],
+                       "=", input_arg_name);
+  }
+  for (const auto& attr : op_def_.attr()) {
+    if (inferred_attrs_.find(attr.name()) != inferred_attrs_.end()) continue;
+
+    const string attr_name =
+        python_op_gen_internal::AvoidPythonReserved(attr.name());
+
+    if (!raw_parameters.empty()) strings::StrAppend(&raw_parameters, ", ");
+    strings::StrAppend(&raw_parameters, attr_name);
+
+    if (!attrs.empty()) strings::StrAppend(&attrs, ", ");
+    strings::StrAppend(&attrs, "\"", attr_name, "\", ", attr_name);
+
+    if (!function_call_parameters.empty()) {
+      strings::StrAppend(&function_call_parameters, ", ");
+    }
+    strings::StrAppend(&function_call_parameters, renames[attr.name()], "=",
+                       attr_name);
+  }
+
+  const string raw_function_name =
+      python_op_gen_internal::AvoidPythonReserved(op_def_.name());
+
+  strings::StrAppend(&result_,
+                     "@_doc_controls.do_not_generate_docs\n@_kwarg_only\ndef ",
+                     raw_function_name, "(", raw_parameters, "):\n");
+
+  // Function body.
+  strings::StrAppend(&result_, "  return ", function_name_, "(",
+                     function_call_parameters, ")\n");
+
+  strings::StrAppend(&result_, "tf_export(\"raw_ops.", raw_function_name,
+                     "\")(", raw_function_name, ")\n");
+}
+
 string GetPythonOps(const OpList& ops, const ApiDefMap& api_defs,
                     const std::vector<string>& hidden_ops, bool require_shapes,
                     const string& source_file_name = "") {
@@ -962,6 +1027,8 @@ from tensorflow.python.framework import op_def_library as _op_def_library
 from tensorflow.python.util.deprecation import deprecated_endpoints
 from tensorflow.python.util import dispatch as _dispatch
 from tensorflow.python.util.tf_export import tf_export
+from tensorflow.python.util.tf_export import kwarg_only as _kwarg_only
+from tensorflow.tools.docs import doc_controls as _doc_controls
 
 )");
 
diff --git a/tensorflow/python/ops/raw_ops_test.py b/tensorflow/python/ops/raw_ops_test.py
new file mode 100644
index 0000000000..82ea137354
--- /dev/null
+++ b/tensorflow/python/ops/raw_ops_test.py
@@ -0,0 +1,42 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Raw ops tests."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import gen_math_ops
+from tensorflow.python.platform import test
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class RawOpsTest(test.TestCase):
+
+  def testSimple(self):
+
+    with self.assertRaises(TypeError):
+      _ = gen_math_ops.Add(1., 1.)
+
+    x = constant_op.constant(1)
+    self.assertEqual([2], self.evaluate(gen_math_ops.Add(x=x, y=x)))
+
+
+if __name__ == '__main__':
+  ops.enable_eager_execution()
+  test.main()
diff --git a/tensorflow/python/tools/api/generator/api_init_files.bzl b/tensorflow/python/tools/api/generator/api_init_files.bzl
index 245380287a..5095f9fdbe 100644
--- a/tensorflow/python/tools/api/generator/api_init_files.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files.bzl
@@ -35,6 +35,7 @@ TENSORFLOW_API_INIT_FILES = [
     "quantization/__init__.py",
     "ragged/__init__.py",
     "random/__init__.py",
+    "raw_ops/__init__.py",
     "rnn/__init__.py",
     "saved_model/__init__.py",
     "sets/__init__.py",
diff --git a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
index c8c7c51b40..01bf12ebe7 100644
--- a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
@@ -46,6 +46,7 @@ TENSORFLOW_API_INIT_FILES_V1 = [
     "quantization/__init__.py",
     "ragged/__init__.py",
     "random/__init__.py",
+    "raw_ops/__init__.py",
     "resource_loader/__init__.py",
     "strings/__init__.py",
     "saved_model/__init__.py",
diff --git a/tensorflow/python/util/tf_export.py b/tensorflow/python/util/tf_export.py
index ce1a1a5c30..7b44d72ab9 100644
--- a/tensorflow/python/util/tf_export.py
+++ b/tensorflow/python/util/tf_export.py
@@ -46,6 +46,7 @@ import functools
 import sys
 
 from tensorflow.python.util import tf_decorator
+from tensorflow.python.util import tf_inspect
 
 ESTIMATOR_API_NAME = 'estimator'
 KERAS_API_NAME = 'keras'
@@ -379,6 +380,15 @@ class api_export(object):  # pylint: disable=invalid-name
         (self._names_v1, name))
 
 
+def kwarg_only(f):
+  """A wrapper that throws away all non-kwarg arguments."""
+  def wrapper(**kwargs):
+    return f(**kwargs)
+
+  return tf_decorator.make_decorator(
+      f, wrapper, decorator_argspec=tf_inspect.getargspec(f))
+
+
 tf_export = functools.partial(api_export, api_name=TENSORFLOW_API_NAME)
 estimator_export = functools.partial(api_export, api_name=ESTIMATOR_API_NAME)
 keras_export = functools.partial(api_export, api_name=KERAS_API_NAME)
diff --git a/tensorflow/python/util/tf_inspect.py b/tensorflow/python/util/tf_inspect.py
index 5171cdfdbc..3a1c4a6e12 100644
--- a/tensorflow/python/util/tf_inspect.py
+++ b/tensorflow/python/util/tf_inspect.py
@@ -36,6 +36,19 @@ else:
       'annotations'
   ])
 
+
+def _convert_maybe_argspec_to_fullargspec(argspec):
+  if isinstance(argspec, FullArgSpec):
+    return argspec
+  return FullArgSpec(
+      args=argspec.args,
+      varargs=argspec.varargs,
+      varkw=argspec.keywords,
+      defaults=argspec.defaults,
+      kwonlyargs=[],
+      kwonlydefaults=None,
+      annotations={})
+
 if hasattr(_inspect, 'getfullargspec'):
   _getfullargspec = _inspect.getfullargspec  # pylint: disable=invalid-name
 
@@ -74,16 +87,7 @@ else:
     Returns:
       A FullArgSpec with empty kwonlyargs, kwonlydefaults and annotations.
     """
-    argspecs = getargspec(target)
-    fullargspecs = FullArgSpec(
-        args=argspecs.args,
-        varargs=argspecs.varargs,
-        varkw=argspecs.keywords,
-        defaults=argspecs.defaults,
-        kwonlyargs=[],
-        kwonlydefaults=None,
-        annotations={})
-    return fullargspecs
+    return _convert_maybe_argspec_to_fullargspec(getargspec(target))
 
 
 def currentframe():
@@ -238,7 +242,7 @@ def getfullargspec(obj):
     directly on the callable.
   """
   decorators, target = tf_decorator.unwrap(obj)
-  return next((d.decorator_argspec
+  return next((_convert_maybe_argspec_to_fullargspec(d.decorator_argspec)
                for d in decorators
                if d.decorator_argspec is not None), _getfullargspec(target))
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
index c8ec8da259..56d06053f1 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
@@ -552,6 +552,10 @@ tf_module {
     name: "random_uniform_initializer"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "raw_ops"
+    mtype: "<type \'module\'>"
+  }
   member {
     name: "resource"
     mtype: "<class \'tensorflow.python.framework.dtypes.DType\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt
new file mode 100644
index 0000000000..e7375af182
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt
@@ -0,0 +1,4027 @@
+path: "tensorflow.raw_ops"
+tf_module {
+  member_method {
+    name: "Abort"
+    argspec: "args=[\'error_msg\', \'exit_without_error\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Abs"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AccumulateNV2"
+    argspec: "args=[\'inputs\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AccumulatorApplyGradient"
+    argspec: "args=[\'handle\', \'local_step\', \'gradient\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AccumulatorNumAccumulated"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AccumulatorSetGlobalStep"
+    argspec: "args=[\'handle\', \'new_global_step\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AccumulatorTakeGradient"
+    argspec: "args=[\'handle\', \'num_required\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Acos"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Acosh"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Add"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AddManySparseToTensorsMap"
+    argspec: "args=[\'sparse_indices\', \'sparse_values\', \'sparse_shape\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AddN"
+    argspec: "args=[\'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AddSparseToTensorsMap"
+    argspec: "args=[\'sparse_indices\', \'sparse_values\', \'sparse_shape\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AddV2"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AdjustContrast"
+    argspec: "args=[\'images\', \'contrast_factor\', \'min_value\', \'max_value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AdjustContrastv2"
+    argspec: "args=[\'images\', \'contrast_factor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AdjustHue"
+    argspec: "args=[\'images\', \'delta\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AdjustSaturation"
+    argspec: "args=[\'images\', \'scale\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "All"
+    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AllCandidateSampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Angle"
+    argspec: "args=[\'input\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AnonymousIterator"
+    argspec: "args=[\'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Any"
+    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyAdaMax"
+    argspec: "args=[\'var\', \'m\', \'v\', \'beta1_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyAdadelta"
+    argspec: "args=[\'var\', \'accum\', \'accum_update\', \'lr\', \'rho\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'use_locking\', \'update_slots\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyAdagradDA"
+    argspec: "args=[\'var\', \'gradient_accumulator\', \'gradient_squared_accumulator\', \'grad\', \'lr\', \'l1\', \'l2\', \'global_step\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyAdam"
+    argspec: "args=[\'var\', \'m\', \'v\', \'beta1_power\', \'beta2_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyAddSign"
+    argspec: "args=[\'var\', \'m\', \'lr\', \'alpha\', \'sign_decay\', \'beta\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyCenteredRMSProp"
+    argspec: "args=[\'var\', \'mg\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyFtrl"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'lr\', \'l1\', \'l2\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyFtrlV2"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'lr\', \'l1\', \'l2\', \'l2_shrinkage\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyGradientDescent"
+    argspec: "args=[\'var\', \'alpha\', \'delta\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyMomentum"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyPowerSign"
+    argspec: "args=[\'var\', \'m\', \'lr\', \'logbase\', \'sign_decay\', \'beta\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyProximalAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'l1\', \'l2\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyProximalGradientDescent"
+    argspec: "args=[\'var\', \'alpha\', \'l1\', \'l2\', \'delta\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyRMSProp"
+    argspec: "args=[\'var\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApproximateEqual"
+    argspec: "args=[\'x\', \'y\', \'tolerance\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ArgMax"
+    argspec: "args=[\'input\', \'dimension\', \'output_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ArgMin"
+    argspec: "args=[\'input\', \'dimension\', \'output_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AsString"
+    argspec: "args=[\'input\', \'precision\', \'scientific\', \'shortest\', \'width\', \'fill\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Asin"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Asinh"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Assert"
+    argspec: "args=[\'condition\', \'data\', \'summarize\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Assign"
+    argspec: "args=[\'ref\', \'value\', \'validate_shape\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AssignAdd"
+    argspec: "args=[\'ref\', \'value\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AssignAddVariableOp"
+    argspec: "args=[\'resource\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AssignSub"
+    argspec: "args=[\'ref\', \'value\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AssignSubVariableOp"
+    argspec: "args=[\'resource\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AssignVariableOp"
+    argspec: "args=[\'resource\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Atan"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Atan2"
+    argspec: "args=[\'y\', \'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Atanh"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AudioSpectrogram"
+    argspec: "args=[\'input\', \'window_size\', \'stride\', \'magnitude_squared\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AudioSummary"
+    argspec: "args=[\'tag\', \'tensor\', \'sample_rate\', \'max_outputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AudioSummaryV2"
+    argspec: "args=[\'tag\', \'tensor\', \'sample_rate\', \'max_outputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AvgPool"
+    argspec: "args=[\'value\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AvgPool3D"
+    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AvgPool3DGrad"
+    argspec: "args=[\'orig_input_shape\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AvgPoolGrad"
+    argspec: "args=[\'orig_input_shape\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Barrier"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BarrierClose"
+    argspec: "args=[\'handle\', \'cancel_pending_enqueues\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BarrierIncompleteSize"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BarrierInsertMany"
+    argspec: "args=[\'handle\', \'keys\', \'values\', \'component_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BarrierReadySize"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BarrierTakeMany"
+    argspec: "args=[\'handle\', \'num_elements\', \'component_types\', \'allow_small_batch\', \'wait_for_incomplete\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchCholesky"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchCholeskyGrad"
+    argspec: "args=[\'l\', \'grad\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchDataset"
+    argspec: "args=[\'input_dataset\', \'batch_size\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchDatasetV2"
+    argspec: "args=[\'input_dataset\', \'batch_size\', \'drop_remainder\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchFFT"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchFFT2D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchFFT3D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchIFFT"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchIFFT2D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchIFFT3D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatMul"
+    argspec: "args=[\'x\', \'y\', \'adj_x\', \'adj_y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixBandPart"
+    argspec: "args=[\'input\', \'num_lower\', \'num_upper\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixDeterminant"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixDiag"
+    argspec: "args=[\'diagonal\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixDiagPart"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixInverse"
+    argspec: "args=[\'input\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixSetDiag"
+    argspec: "args=[\'input\', \'diagonal\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixSolve"
+    argspec: "args=[\'matrix\', \'rhs\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixSolveLs"
+    argspec: "args=[\'matrix\', \'rhs\', \'l2_regularizer\', \'fast\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixTriangularSolve"
+    argspec: "args=[\'matrix\', \'rhs\', \'lower\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchNormWithGlobalNormalization"
+    argspec: "args=[\'t\', \'m\', \'v\', \'beta\', \'gamma\', \'variance_epsilon\', \'scale_after_normalization\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchNormWithGlobalNormalizationGrad"
+    argspec: "args=[\'t\', \'m\', \'v\', \'gamma\', \'backprop\', \'variance_epsilon\', \'scale_after_normalization\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchSelfAdjointEig"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchSelfAdjointEigV2"
+    argspec: "args=[\'input\', \'compute_v\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchSvd"
+    argspec: "args=[\'input\', \'compute_uv\', \'full_matrices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchToSpace"
+    argspec: "args=[\'input\', \'crops\', \'block_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchToSpaceND"
+    argspec: "args=[\'input\', \'block_shape\', \'crops\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BesselI0e"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BesselI1e"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Betainc"
+    argspec: "args=[\'a\', \'b\', \'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BiasAdd"
+    argspec: "args=[\'value\', \'bias\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BiasAddGrad"
+    argspec: "args=[\'out_backprop\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BiasAddV1"
+    argspec: "args=[\'value\', \'bias\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Bincount"
+    argspec: "args=[\'arr\', \'size\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Bitcast"
+    argspec: "args=[\'input\', \'type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BitwiseAnd"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BitwiseOr"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BitwiseXor"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesBucketize"
+    argspec: "args=[\'float_values\', \'bucket_boundaries\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesCalculateBestGainsPerFeature"
+    argspec: "args=[\'node_id_range\', \'stats_summary_list\', \'l1\', \'l2\', \'tree_complexity\', \'min_node_weight\', \'max_splits\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesCenterBias"
+    argspec: "args=[\'tree_ensemble_handle\', \'mean_gradients\', \'mean_hessians\', \'l1\', \'l2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesCreateEnsemble"
+    argspec: "args=[\'tree_ensemble_handle\', \'stamp_token\', \'tree_ensemble_serialized\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesCreateQuantileStreamResource"
+    argspec: "args=[\'quantile_stream_resource_handle\', \'epsilon\', \'num_streams\', \'max_elements\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesDeserializeEnsemble"
+    argspec: "args=[\'tree_ensemble_handle\', \'stamp_token\', \'tree_ensemble_serialized\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesEnsembleResourceHandleOp"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesExampleDebugOutputs"
+    argspec: "args=[\'tree_ensemble_handle\', \'bucketized_features\', \'logits_dimension\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesGetEnsembleStates"
+    argspec: "args=[\'tree_ensemble_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesMakeQuantileSummaries"
+    argspec: "args=[\'float_values\', \'example_weights\', \'epsilon\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesMakeStatsSummary"
+    argspec: "args=[\'node_ids\', \'gradients\', \'hessians\', \'bucketized_features_list\', \'max_splits\', \'num_buckets\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesPredict"
+    argspec: "args=[\'tree_ensemble_handle\', \'bucketized_features\', \'logits_dimension\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesQuantileStreamResourceAddSummaries"
+    argspec: "args=[\'quantile_stream_resource_handle\', \'summaries\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesQuantileStreamResourceDeserialize"
+    argspec: "args=[\'quantile_stream_resource_handle\', \'bucket_boundaries\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesQuantileStreamResourceFlush"
+    argspec: "args=[\'quantile_stream_resource_handle\', \'num_buckets\', \'generate_quantiles\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesQuantileStreamResourceGetBucketBoundaries"
+    argspec: "args=[\'quantile_stream_resource_handle\', \'num_features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesQuantileStreamResourceHandleOp"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesSerializeEnsemble"
+    argspec: "args=[\'tree_ensemble_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesTrainingPredict"
+    argspec: "args=[\'tree_ensemble_handle\', \'cached_tree_ids\', \'cached_node_ids\', \'bucketized_features\', \'logits_dimension\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesUpdateEnsemble"
+    argspec: "args=[\'tree_ensemble_handle\', \'feature_ids\', \'node_ids\', \'gains\', \'thresholds\', \'left_node_contribs\', \'right_node_contribs\', \'max_depth\', \'learning_rate\', \'pruning_mode\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BroadcastArgs"
+    argspec: "args=[\'s0\', \'s1\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BroadcastGradientArgs"
+    argspec: "args=[\'s0\', \'s1\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BroadcastTo"
+    argspec: "args=[\'input\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Bucketize"
+    argspec: "args=[\'input\', \'boundaries\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CTCBeamSearchDecoder"
+    argspec: "args=[\'inputs\', \'sequence_length\', \'beam_width\', \'top_paths\', \'merge_repeated\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CTCGreedyDecoder"
+    argspec: "args=[\'inputs\', \'sequence_length\', \'merge_repeated\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CTCLoss"
+    argspec: "args=[\'inputs\', \'labels_indices\', \'labels_values\', \'sequence_length\', \'preprocess_collapse_repeated\', \'ctc_merge_repeated\', \'ignore_longer_outputs_than_inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CacheDataset"
+    argspec: "args=[\'input_dataset\', \'filename\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Cast"
+    argspec: "args=[\'x\', \'DstT\', \'Truncate\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Ceil"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CheckNumerics"
+    argspec: "args=[\'tensor\', \'message\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Cholesky"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CholeskyGrad"
+    argspec: "args=[\'l\', \'grad\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ClipByValue"
+    argspec: "args=[\'t\', \'clip_value_min\', \'clip_value_max\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CloseSummaryWriter"
+    argspec: "args=[\'writer\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CollectiveBcastRecv"
+    argspec: "args=[\'T\', \'group_size\', \'group_key\', \'instance_key\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CollectiveBcastSend"
+    argspec: "args=[\'input\', \'group_size\', \'group_key\', \'instance_key\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CollectiveReduce"
+    argspec: "args=[\'input\', \'group_size\', \'group_key\', \'instance_key\', \'merge_op\', \'final_op\', \'subdiv_offsets\', \'wait_for\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CombinedNonMaxSuppression"
+    argspec: "args=[\'boxes\', \'scores\', \'max_output_size_per_class\', \'max_total_size\', \'iou_threshold\', \'score_threshold\', \'pad_per_class\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CompareAndBitpack"
+    argspec: "args=[\'input\', \'threshold\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Complex"
+    argspec: "args=[\'real\', \'imag\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ComplexAbs"
+    argspec: "args=[\'x\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ComputeAccidentalHits"
+    argspec: "args=[\'true_classes\', \'sampled_candidates\', \'num_true\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Concat"
+    argspec: "args=[\'concat_dim\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ConcatOffset"
+    argspec: "args=[\'concat_dim\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ConcatV2"
+    argspec: "args=[\'values\', \'axis\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ConcatenateDataset"
+    argspec: "args=[\'input_dataset\', \'another_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ConditionalAccumulator"
+    argspec: "args=[\'dtype\', \'shape\', \'container\', \'shared_name\', \'reduction_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conj"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ConjugateTranspose"
+    argspec: "args=[\'x\', \'perm\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Const"
+    argspec: "args=[\'value\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ConsumeMutexLock"
+    argspec: "args=[\'mutex_lock\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ControlTrigger"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv2D"
+    argspec: "args=[\'input\', \'filter\', \'strides\', \'use_cudnn_on_gpu\', \'padding\', \'explicit_paddings\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv2DBackpropFilter"
+    argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'use_cudnn_on_gpu\', \'padding\', \'explicit_paddings\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv2DBackpropInput"
+    argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'use_cudnn_on_gpu\', \'padding\', \'explicit_paddings\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv3D"
+    argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv3DBackpropFilter"
+    argspec: "args=[\'input\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv3DBackpropFilterV2"
+    argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv3DBackpropInput"
+    argspec: "args=[\'input\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv3DBackpropInputV2"
+    argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Cos"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Cosh"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CountUpTo"
+    argspec: "args=[\'ref\', \'limit\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CreateSummaryDbWriter"
+    argspec: "args=[\'writer\', \'db_uri\', \'experiment_name\', \'run_name\', \'user_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CreateSummaryFileWriter"
+    argspec: "args=[\'writer\', \'logdir\', \'max_queue\', \'flush_millis\', \'filename_suffix\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CropAndResize"
+    argspec: "args=[\'image\', \'boxes\', \'box_ind\', \'crop_size\', \'method\', \'extrapolation_value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CropAndResizeGradBoxes"
+    argspec: "args=[\'grads\', \'image\', \'boxes\', \'box_ind\', \'method\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CropAndResizeGradImage"
+    argspec: "args=[\'grads\', \'boxes\', \'box_ind\', \'image_size\', \'T\', \'method\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Cross"
+    argspec: "args=[\'a\', \'b\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNN"
+    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\', \'is_training\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNBackprop"
+    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'output\', \'output_h\', \'output_c\', \'output_backprop\', \'output_h_backprop\', \'output_c_backprop\', \'reserve_space\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNBackpropV2"
+    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'output\', \'output_h\', \'output_c\', \'output_backprop\', \'output_h_backprop\', \'output_c_backprop\', \'reserve_space\', \'host_reserved\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNBackpropV3"
+    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'sequence_lengths\', \'output\', \'output_h\', \'output_c\', \'output_backprop\', \'output_h_backprop\', \'output_c_backprop\', \'reserve_space\', \'host_reserved\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNCanonicalToParams"
+    argspec: "args=[\'num_layers\', \'num_units\', \'input_size\', \'weights\', \'biases\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNParamsSize"
+    argspec: "args=[\'num_layers\', \'num_units\', \'input_size\', \'T\', \'S\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNParamsToCanonical"
+    argspec: "args=[\'num_layers\', \'num_units\', \'input_size\', \'params\', \'num_params\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNV2"
+    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\', \'is_training\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNV3"
+    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'sequence_lengths\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\', \'is_training\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Cumprod"
+    argspec: "args=[\'x\', \'axis\', \'exclusive\', \'reverse\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Cumsum"
+    argspec: "args=[\'x\', \'axis\', \'exclusive\', \'reverse\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DataFormatDimMap"
+    argspec: "args=[\'x\', \'src_format\', \'dst_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DataFormatVecPermute"
+    argspec: "args=[\'x\', \'src_format\', \'dst_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DatasetToGraph"
+    argspec: "args=[\'input_dataset\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DatasetToSingleElement"
+    argspec: "args=[\'dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DebugGradientIdentity"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DebugGradientRefIdentity"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeAndCropJpeg"
+    argspec: "args=[\'contents\', \'crop_window\', \'channels\', \'ratio\', \'fancy_upscaling\', \'try_recover_truncated\', \'acceptable_fraction\', \'dct_method\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeBase64"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeBmp"
+    argspec: "args=[\'contents\', \'channels\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeCSV"
+    argspec: "args=[\'records\', \'record_defaults\', \'field_delim\', \'use_quote_delim\', \'na_value\', \'select_cols\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeCompressed"
+    argspec: "args=[\'bytes\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeGif"
+    argspec: "args=[\'contents\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeJSONExample"
+    argspec: "args=[\'json_examples\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeJpeg"
+    argspec: "args=[\'contents\', \'channels\', \'ratio\', \'fancy_upscaling\', \'try_recover_truncated\', \'acceptable_fraction\', \'dct_method\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodePng"
+    argspec: "args=[\'contents\', \'channels\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeRaw"
+    argspec: "args=[\'bytes\', \'out_type\', \'little_endian\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeWav"
+    argspec: "args=[\'contents\', \'desired_channels\', \'desired_samples\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DeepCopy"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DeleteSessionTensor"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DenseToDenseSetOperation"
+    argspec: "args=[\'set1\', \'set2\', \'set_operation\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DenseToSparseSetOperation"
+    argspec: "args=[\'set1\', \'set2_indices\', \'set2_values\', \'set2_shape\', \'set_operation\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DepthToSpace"
+    argspec: "args=[\'input\', \'block_size\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DepthwiseConv2dNative"
+    argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DepthwiseConv2dNativeBackpropFilter"
+    argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DepthwiseConv2dNativeBackpropInput"
+    argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Dequantize"
+    argspec: "args=[\'input\', \'min_range\', \'max_range\', \'mode\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DeserializeIterator"
+    argspec: "args=[\'resource_handle\', \'serialized\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DeserializeManySparse"
+    argspec: "args=[\'serialized_sparse\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DeserializeSparse"
+    argspec: "args=[\'serialized_sparse\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DestroyResourceOp"
+    argspec: "args=[\'resource\', \'ignore_lookup_error\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DestroyTemporaryVariable"
+    argspec: "args=[\'ref\', \'var_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Diag"
+    argspec: "args=[\'diagonal\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DiagPart"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Digamma"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Dilation2D"
+    argspec: "args=[\'input\', \'filter\', \'strides\', \'rates\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Dilation2DBackpropFilter"
+    argspec: "args=[\'input\', \'filter\', \'out_backprop\', \'strides\', \'rates\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Dilation2DBackpropInput"
+    argspec: "args=[\'input\', \'filter\', \'out_backprop\', \'strides\', \'rates\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Div"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DivNoNan"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DrawBoundingBoxes"
+    argspec: "args=[\'images\', \'boxes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DynamicPartition"
+    argspec: "args=[\'data\', \'partitions\', \'num_partitions\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DynamicStitch"
+    argspec: "args=[\'indices\', \'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EagerPyFunc"
+    argspec: "args=[\'input\', \'token\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EditDistance"
+    argspec: "args=[\'hypothesis_indices\', \'hypothesis_values\', \'hypothesis_shape\', \'truth_indices\', \'truth_values\', \'truth_shape\', \'normalize\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Elu"
+    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EluGrad"
+    argspec: "args=[\'gradients\', \'outputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Empty"
+    argspec: "args=[\'shape\', \'dtype\', \'init\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EmptyTensorList"
+    argspec: "args=[\'element_shape\', \'max_num_elements\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EncodeBase64"
+    argspec: "args=[\'input\', \'pad\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EncodeJpeg"
+    argspec: "args=[\'image\', \'format\', \'quality\', \'progressive\', \'optimize_size\', \'chroma_downsampling\', \'density_unit\', \'x_density\', \'y_density\', \'xmp_metadata\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EncodePng"
+    argspec: "args=[\'image\', \'compression\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EncodeWav"
+    argspec: "args=[\'audio\', \'sample_rate\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EnsureShape"
+    argspec: "args=[\'input\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Enter"
+    argspec: "args=[\'data\', \'frame_name\', \'is_constant\', \'parallel_iterations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Equal"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Erf"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Erfc"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Exit"
+    argspec: "args=[\'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Exp"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExpandDims"
+    argspec: "args=[\'input\', \'dim\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalAssertNextDataset"
+    argspec: "args=[\'input_dataset\', \'transformations\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalBytesProducedStatsDataset"
+    argspec: "args=[\'input_dataset\', \'tag\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalCSVDataset"
+    argspec: "args=[\'filenames\', \'compression_type\', \'buffer_size\', \'header\', \'field_delim\', \'use_quote_delim\', \'na_value\', \'select_cols\', \'record_defaults\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalChooseFastestDataset"
+    argspec: "args=[\'input_datasets\', \'num_experiments\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalDatasetCardinality"
+    argspec: "args=[\'input_dataset\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalDatasetToTFRecord"
+    argspec: "args=[\'input_dataset\', \'filename\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalDenseToSparseBatchDataset"
+    argspec: "args=[\'input_dataset\', \'batch_size\', \'row_shape\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalDirectedInterleaveDataset"
+    argspec: "args=[\'selector_input_dataset\', \'data_input_datasets\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalGroupByReducerDataset"
+    argspec: "args=[\'input_dataset\', \'key_func_other_arguments\', \'init_func_other_arguments\', \'reduce_func_other_arguments\', \'finalize_func_other_arguments\', \'key_func\', \'init_func\', \'reduce_func\', \'finalize_func\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalGroupByWindowDataset"
+    argspec: "args=[\'input_dataset\', \'key_func_other_arguments\', \'reduce_func_other_arguments\', \'window_size_func_other_arguments\', \'key_func\', \'reduce_func\', \'window_size_func\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalIdentityIndexedDataset"
+    argspec: "args=[\'size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalIgnoreErrorsDataset"
+    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalIndexedDatasetGet"
+    argspec: "args=[\'materialized\', \'index\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalIndexedDatasetMaterialize"
+    argspec: "args=[\'dataset\', \'materialized\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalIteratorGetDevice"
+    argspec: "args=[\'resource\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalLMDBDataset"
+    argspec: "args=[\'filenames\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalLatencyStatsDataset"
+    argspec: "args=[\'input_dataset\', \'tag\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalMapAndBatchDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'batch_size\', \'num_parallel_calls\', \'drop_remainder\', \'f\', \'output_types\', \'output_shapes\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalMapDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\', \'use_inter_op_parallelism\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalMatchingFilesDataset"
+    argspec: "args=[\'patterns\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalMaterializedIndexDatasetHandle"
+    argspec: "args=[\'container\', \'shared_name\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalMaxIntraOpParallelismDataset"
+    argspec: "args=[\'input_dataset\', \'max_intra_op_parallelism\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalNonSerializableDataset"
+    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalNumaMapAndBatchDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'batch_size\', \'num_parallel_calls\', \'drop_remainder\', \'f\', \'output_types\', \'output_shapes\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalParallelInterleaveDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'cycle_length\', \'block_length\', \'sloppy\', \'buffer_output_elements\', \'prefetch_input_elements\', \'f\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalParseExampleDataset"
+    argspec: "args=[\'input_dataset\', \'num_parallel_calls\', \'dense_defaults\', \'sparse_keys\', \'dense_keys\', \'sparse_types\', \'dense_shapes\', \'output_types\', \'output_shapes\', \'sloppy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalPrivateThreadPoolDataset"
+    argspec: "args=[\'input_dataset\', \'num_threads\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalRandomDataset"
+    argspec: "args=[\'seed\', \'seed2\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalScanDataset"
+    argspec: "args=[\'input_dataset\', \'initial_state\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalSetStatsAggregatorDataset"
+    argspec: "args=[\'input_dataset\', \'stats_aggregator\', \'tag\', \'counter_prefix\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalSleepDataset"
+    argspec: "args=[\'input_dataset\', \'sleep_microseconds\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalSlidingWindowDataset"
+    argspec: "args=[\'input_dataset\', \'window_size\', \'window_shift\', \'window_stride\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalSqlDataset"
+    argspec: "args=[\'driver_name\', \'data_source_name\', \'query\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalStatsAggregatorHandle"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalStatsAggregatorSummary"
+    argspec: "args=[\'iterator\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalTakeWhileDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'predicate\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalThreadPoolDataset"
+    argspec: "args=[\'input_dataset\', \'thread_pool\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalThreadPoolHandle"
+    argspec: "args=[\'num_threads\', \'max_intra_op_parallelism\', \'display_name\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalUnbatchDataset"
+    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalUniqueDataset"
+    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Expm1"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExtractGlimpse"
+    argspec: "args=[\'input\', \'size\', \'offsets\', \'centered\', \'normalized\', \'uniform_noise\', \'noise\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExtractImagePatches"
+    argspec: "args=[\'images\', \'ksizes\', \'strides\', \'rates\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExtractJpegShape"
+    argspec: "args=[\'contents\', \'output_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExtractVolumePatches"
+    argspec: "args=[\'input\', \'ksizes\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FFT"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FFT2D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FFT3D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FIFOQueue"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FIFOQueueV2"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Fact"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeParam"
+    argspec: "args=[\'dtype\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeQuantWithMinMaxArgs"
+    argspec: "args=[\'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeQuantWithMinMaxArgsGradient"
+    argspec: "args=[\'gradients\', \'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeQuantWithMinMaxVars"
+    argspec: "args=[\'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeQuantWithMinMaxVarsGradient"
+    argspec: "args=[\'gradients\', \'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeQuantWithMinMaxVarsPerChannel"
+    argspec: "args=[\'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeQuantWithMinMaxVarsPerChannelGradient"
+    argspec: "args=[\'gradients\', \'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeQueue"
+    argspec: "args=[\'resource\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Fill"
+    argspec: "args=[\'dims\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FilterByLastComponentDataset"
+    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FilterDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'predicate\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FixedLengthRecordDataset"
+    argspec: "args=[\'filenames\', \'header_bytes\', \'record_bytes\', \'footer_bytes\', \'buffer_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FixedLengthRecordDatasetV2"
+    argspec: "args=[\'filenames\', \'header_bytes\', \'record_bytes\', \'footer_bytes\', \'buffer_size\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FixedLengthRecordReader"
+    argspec: "args=[\'header_bytes\', \'record_bytes\', \'footer_bytes\', \'hop_bytes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FixedLengthRecordReaderV2"
+    argspec: "args=[\'header_bytes\', \'record_bytes\', \'footer_bytes\', \'hop_bytes\', \'container\', \'shared_name\', \'encoding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FixedUnigramCandidateSampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'vocab_file\', \'distortion\', \'num_reserved_ids\', \'num_shards\', \'shard\', \'unigrams\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FlatMapDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Floor"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FloorDiv"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FloorMod"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FlushSummaryWriter"
+    argspec: "args=[\'writer\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "For"
+    argspec: "args=[\'start\', \'limit\', \'delta\', \'input\', \'body\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FractionalAvgPool"
+    argspec: "args=[\'value\', \'pooling_ratio\', \'pseudo_random\', \'overlapping\', \'deterministic\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FractionalAvgPoolGrad"
+    argspec: "args=[\'orig_input_tensor_shape\', \'out_backprop\', \'row_pooling_sequence\', \'col_pooling_sequence\', \'overlapping\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FractionalMaxPool"
+    argspec: "args=[\'value\', \'pooling_ratio\', \'pseudo_random\', \'overlapping\', \'deterministic\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FractionalMaxPoolGrad"
+    argspec: "args=[\'orig_input\', \'orig_output\', \'out_backprop\', \'row_pooling_sequence\', \'col_pooling_sequence\', \'overlapping\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FusedBatchNorm"
+    argspec: "args=[\'x\', \'scale\', \'offset\', \'mean\', \'variance\', \'epsilon\', \'data_format\', \'is_training\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FusedBatchNormGrad"
+    argspec: "args=[\'y_backprop\', \'x\', \'scale\', \'reserve_space_1\', \'reserve_space_2\', \'epsilon\', \'data_format\', \'is_training\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FusedBatchNormGradV2"
+    argspec: "args=[\'y_backprop\', \'x\', \'scale\', \'reserve_space_1\', \'reserve_space_2\', \'epsilon\', \'data_format\', \'is_training\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FusedBatchNormV2"
+    argspec: "args=[\'x\', \'scale\', \'offset\', \'mean\', \'variance\', \'epsilon\', \'data_format\', \'is_training\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FusedPadConv2D"
+    argspec: "args=[\'input\', \'paddings\', \'filter\', \'mode\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FusedResizeAndPadConv2D"
+    argspec: "args=[\'input\', \'size\', \'paddings\', \'filter\', \'resize_align_corners\', \'mode\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Gather"
+    argspec: "args=[\'params\', \'indices\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GatherNd"
+    argspec: "args=[\'params\', \'indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GatherV2"
+    argspec: "args=[\'params\', \'indices\', \'axis\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GenerateVocabRemapping"
+    argspec: "args=[\'new_vocab_file\', \'old_vocab_file\', \'new_vocab_offset\', \'num_new_vocab\', \'old_vocab_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GeneratorDataset"
+    argspec: "args=[\'init_func_other_args\', \'next_func_other_args\', \'finalize_func_other_args\', \'init_func\', \'next_func\', \'finalize_func\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GetSessionHandle"
+    argspec: "args=[\'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GetSessionHandleV2"
+    argspec: "args=[\'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GetSessionTensor"
+    argspec: "args=[\'handle\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Greater"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GreaterEqual"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GuaranteeConst"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "HSVToRGB"
+    argspec: "args=[\'images\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "HashTable"
+    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "HashTableV2"
+    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "HistogramFixedWidth"
+    argspec: "args=[\'values\', \'value_range\', \'nbins\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "HistogramSummary"
+    argspec: "args=[\'tag\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IFFT"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IFFT2D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IFFT3D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IRFFT"
+    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IRFFT2D"
+    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IRFFT3D"
+    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Identity"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IdentityN"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IdentityReader"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IdentityReaderV2"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "If"
+    argspec: "args=[\'cond\', \'input\', \'Tout\', \'then_branch\', \'else_branch\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Igamma"
+    argspec: "args=[\'a\', \'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IgammaGradA"
+    argspec: "args=[\'a\', \'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Igammac"
+    argspec: "args=[\'a\', \'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Imag"
+    argspec: "args=[\'input\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ImageSummary"
+    argspec: "args=[\'tag\', \'tensor\', \'max_images\', \'bad_color\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ImmutableConst"
+    argspec: "args=[\'dtype\', \'shape\', \'memory_region_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ImportEvent"
+    argspec: "args=[\'writer\', \'event\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InTopK"
+    argspec: "args=[\'predictions\', \'targets\', \'k\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InTopKV2"
+    argspec: "args=[\'predictions\', \'targets\', \'k\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InitializeTable"
+    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InitializeTableFromTextFile"
+    argspec: "args=[\'table_handle\', \'filename\', \'key_index\', \'value_index\', \'vocab_size\', \'delimiter\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InitializeTableFromTextFileV2"
+    argspec: "args=[\'table_handle\', \'filename\', \'key_index\', \'value_index\', \'vocab_size\', \'delimiter\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InitializeTableV2"
+    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InplaceAdd"
+    argspec: "args=[\'x\', \'i\', \'v\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InplaceSub"
+    argspec: "args=[\'x\', \'i\', \'v\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InplaceUpdate"
+    argspec: "args=[\'x\', \'i\', \'v\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InterleaveDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'cycle_length\', \'block_length\', \'f\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Inv"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InvGrad"
+    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Invert"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InvertPermutation"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IsBoostedTreesEnsembleInitialized"
+    argspec: "args=[\'tree_ensemble_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IsBoostedTreesQuantileStreamResourceInitialized"
+    argspec: "args=[\'quantile_stream_resource_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IsFinite"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IsInf"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IsNan"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IsVariableInitialized"
+    argspec: "args=[\'ref\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Iterator"
+    argspec: "args=[\'shared_name\', \'container\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IteratorFromStringHandle"
+    argspec: "args=[\'string_handle\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IteratorFromStringHandleV2"
+    argspec: "args=[\'string_handle\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IteratorGetNext"
+    argspec: "args=[\'iterator\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IteratorGetNextAsOptional"
+    argspec: "args=[\'iterator\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IteratorGetNextSync"
+    argspec: "args=[\'iterator\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IteratorToStringHandle"
+    argspec: "args=[\'resource_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IteratorV2"
+    argspec: "args=[\'shared_name\', \'container\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "L2Loss"
+    argspec: "args=[\'t\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LMDBReader"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LRN"
+    argspec: "args=[\'input\', \'depth_radius\', \'bias\', \'alpha\', \'beta\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LRNGrad"
+    argspec: "args=[\'input_grads\', \'input_image\', \'output_image\', \'depth_radius\', \'bias\', \'alpha\', \'beta\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LeakyRelu"
+    argspec: "args=[\'features\', \'alpha\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LeakyReluGrad"
+    argspec: "args=[\'gradients\', \'features\', \'alpha\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LearnedUnigramCandidateSampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LeftShift"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Less"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LessEqual"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Lgamma"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LinSpace"
+    argspec: "args=[\'start\', \'stop\', \'num\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ListDiff"
+    argspec: "args=[\'x\', \'y\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LoadAndRemapMatrix"
+    argspec: "args=[\'ckpt_path\', \'old_tensor_name\', \'row_remapping\', \'col_remapping\', \'initializing_values\', \'num_rows\', \'num_cols\', \'max_rows_in_memory\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Log"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Log1p"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LogMatrixDeterminant"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LogSoftmax"
+    argspec: "args=[\'logits\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LogUniformCandidateSampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LogicalAnd"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LogicalNot"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LogicalOr"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableExport"
+    argspec: "args=[\'table_handle\', \'Tkeys\', \'Tvalues\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableExportV2"
+    argspec: "args=[\'table_handle\', \'Tkeys\', \'Tvalues\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableFind"
+    argspec: "args=[\'table_handle\', \'keys\', \'default_value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableFindV2"
+    argspec: "args=[\'table_handle\', \'keys\', \'default_value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableImport"
+    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableImportV2"
+    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableInsert"
+    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableInsertV2"
+    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableRemoveV2"
+    argspec: "args=[\'table_handle\', \'keys\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableSize"
+    argspec: "args=[\'table_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableSizeV2"
+    argspec: "args=[\'table_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LoopCond"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LowerBound"
+    argspec: "args=[\'sorted_inputs\', \'values\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Lu"
+    argspec: "args=[\'input\', \'output_idx_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MakeIterator"
+    argspec: "args=[\'dataset\', \'iterator\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapClear"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\', \'use_inter_op_parallelism\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapDefun"
+    argspec: "args=[\'arguments\', \'captured_inputs\', \'output_types\', \'output_shapes\', \'f\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapIncompleteSize"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapPeek"
+    argspec: "args=[\'key\', \'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapSize"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapStage"
+    argspec: "args=[\'key\', \'indices\', \'values\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapUnstage"
+    argspec: "args=[\'key\', \'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapUnstageNoKey"
+    argspec: "args=[\'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatMul"
+    argspec: "args=[\'a\', \'b\', \'transpose_a\', \'transpose_b\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatchingFiles"
+    argspec: "args=[\'pattern\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixBandPart"
+    argspec: "args=[\'input\', \'num_lower\', \'num_upper\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixDeterminant"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixDiag"
+    argspec: "args=[\'diagonal\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixDiagPart"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixExponential"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixInverse"
+    argspec: "args=[\'input\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixLogarithm"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixSetDiag"
+    argspec: "args=[\'input\', \'diagonal\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixSolve"
+    argspec: "args=[\'matrix\', \'rhs\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixSolveLs"
+    argspec: "args=[\'matrix\', \'rhs\', \'l2_regularizer\', \'fast\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixSquareRoot"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixTriangularSolve"
+    argspec: "args=[\'matrix\', \'rhs\', \'lower\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Max"
+    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPool"
+    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPool3D"
+    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPool3DGrad"
+    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPool3DGradGrad"
+    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolGrad"
+    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolGradGrad"
+    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolGradGradV2"
+    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolGradGradWithArgmax"
+    argspec: "args=[\'input\', \'grad\', \'argmax\', \'ksize\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolGradV2"
+    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolGradWithArgmax"
+    argspec: "args=[\'input\', \'grad\', \'argmax\', \'ksize\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolV2"
+    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolWithArgmax"
+    argspec: "args=[\'input\', \'ksize\', \'strides\', \'Targmax\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Maximum"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Mean"
+    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Merge"
+    argspec: "args=[\'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MergeSummary"
+    argspec: "args=[\'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MergeV2Checkpoints"
+    argspec: "args=[\'checkpoint_prefixes\', \'destination_prefix\', \'delete_old_dirs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Mfcc"
+    argspec: "args=[\'spectrogram\', \'sample_rate\', \'upper_frequency_limit\', \'lower_frequency_limit\', \'filterbank_channel_count\', \'dct_coefficient_count\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Min"
+    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Minimum"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MirrorPad"
+    argspec: "args=[\'input\', \'paddings\', \'mode\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MirrorPadGrad"
+    argspec: "args=[\'input\', \'paddings\', \'mode\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Mod"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ModelDataset"
+    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Mul"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MultiDeviceIterator"
+    argspec: "args=[\'devices\', \'shared_name\', \'container\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MultiDeviceIteratorFromStringHandle"
+    argspec: "args=[\'string_handle\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MultiDeviceIteratorGetNextFromShard"
+    argspec: "args=[\'multi_device_iterator\', \'shard_num\', \'incarnation_id\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MultiDeviceIteratorInit"
+    argspec: "args=[\'dataset\', \'multi_device_iterator\', \'max_buffer_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MultiDeviceIteratorToStringHandle"
+    argspec: "args=[\'multi_device_iterator\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Multinomial"
+    argspec: "args=[\'logits\', \'num_samples\', \'seed\', \'seed2\', \'output_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutableDenseHashTable"
+    argspec: "args=[\'empty_key\', \'container\', \'shared_name\', \'use_node_name_sharing\', \'value_dtype\', \'value_shape\', \'initial_num_buckets\', \'max_load_factor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutableDenseHashTableV2"
+    argspec: "args=[\'empty_key\', \'deleted_key\', \'container\', \'shared_name\', \'use_node_name_sharing\', \'value_dtype\', \'value_shape\', \'initial_num_buckets\', \'max_load_factor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutableHashTable"
+    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutableHashTableOfTensors"
+    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\', \'value_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutableHashTableOfTensorsV2"
+    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\', \'value_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutableHashTableV2"
+    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutexLock"
+    argspec: "args=[\'mutex\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutexV2"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NcclAllReduce"
+    argspec: "args=[\'input\', \'reduction\', \'num_devices\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NcclBroadcast"
+    argspec: "args=[\'input\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NcclReduce"
+    argspec: "args=[\'input\', \'reduction\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Neg"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NextAfter"
+    argspec: "args=[\'x1\', \'x2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NextIteration"
+    argspec: "args=[\'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NoOp"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NonMaxSuppression"
+    argspec: "args=[\'boxes\', \'scores\', \'max_output_size\', \'iou_threshold\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NonMaxSuppressionV2"
+    argspec: "args=[\'boxes\', \'scores\', \'max_output_size\', \'iou_threshold\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NonMaxSuppressionV3"
+    argspec: "args=[\'boxes\', \'scores\', \'max_output_size\', \'iou_threshold\', \'score_threshold\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NonMaxSuppressionV4"
+    argspec: "args=[\'boxes\', \'scores\', \'max_output_size\', \'iou_threshold\', \'score_threshold\', \'pad_to_max_output_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NonMaxSuppressionWithOverlaps"
+    argspec: "args=[\'overlaps\', \'scores\', \'max_output_size\', \'overlap_threshold\', \'score_threshold\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NotEqual"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NthElement"
+    argspec: "args=[\'input\', \'n\', \'reverse\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OneHot"
+    argspec: "args=[\'indices\', \'depth\', \'on_value\', \'off_value\', \'axis\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OneShotIterator"
+    argspec: "args=[\'dataset_factory\', \'output_types\', \'output_shapes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OnesLike"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OptimizeDataset"
+    argspec: "args=[\'input_dataset\', \'optimizations\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OptionalFromValue"
+    argspec: "args=[\'components\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OptionalGetValue"
+    argspec: "args=[\'optional\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OptionalHasValue"
+    argspec: "args=[\'optional\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OptionalNone"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OrderedMapClear"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OrderedMapIncompleteSize"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OrderedMapPeek"
+    argspec: "args=[\'key\', \'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OrderedMapSize"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OrderedMapStage"
+    argspec: "args=[\'key\', \'indices\', \'values\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OrderedMapUnstage"
+    argspec: "args=[\'key\', \'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OrderedMapUnstageNoKey"
+    argspec: "args=[\'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Pack"
+    argspec: "args=[\'values\', \'axis\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Pad"
+    argspec: "args=[\'input\', \'paddings\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PadV2"
+    argspec: "args=[\'input\', \'paddings\', \'constant_values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PaddedBatchDataset"
+    argspec: "args=[\'input_dataset\', \'batch_size\', \'padded_shapes\', \'padding_values\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PaddedBatchDatasetV2"
+    argspec: "args=[\'input_dataset\', \'batch_size\', \'padded_shapes\', \'padding_values\', \'drop_remainder\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PaddingFIFOQueue"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PaddingFIFOQueueV2"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParallelConcat"
+    argspec: "args=[\'values\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParallelDynamicStitch"
+    argspec: "args=[\'indices\', \'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParallelInterleaveDatasetV2"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'cycle_length\', \'block_length\', \'num_parallel_calls\', \'f\', \'output_types\', \'output_shapes\', \'sloppy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParallelMapDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'num_parallel_calls\', \'f\', \'output_types\', \'output_shapes\', \'use_inter_op_parallelism\', \'sloppy\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParameterizedTruncatedNormal"
+    argspec: "args=[\'shape\', \'means\', \'stdevs\', \'minvals\', \'maxvals\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParseExample"
+    argspec: "args=[\'serialized\', \'names\', \'sparse_keys\', \'dense_keys\', \'dense_defaults\', \'sparse_types\', \'dense_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParseSequenceExample"
+    argspec: "args=[\'serialized\', \'debug_name\', \'context_dense_defaults\', \'feature_list_dense_missing_assumed_empty\', \'context_sparse_keys\', \'context_dense_keys\', \'feature_list_sparse_keys\', \'feature_list_dense_keys\', \'Ncontext_sparse\', \'Ncontext_dense\', \'Nfeature_list_sparse\', \'Nfeature_list_dense\', \'context_sparse_types\', \'feature_list_dense_types\', \'context_dense_shapes\', \'feature_list_sparse_types\', \'feature_list_dense_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParseSingleExample"
+    argspec: "args=[\'serialized\', \'dense_defaults\', \'num_sparse\', \'sparse_keys\', \'dense_keys\', \'sparse_types\', \'dense_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParseSingleSequenceExample"
+    argspec: "args=[\'serialized\', \'feature_list_dense_missing_assumed_empty\', \'context_sparse_keys\', \'context_dense_keys\', \'feature_list_sparse_keys\', \'feature_list_dense_keys\', \'context_dense_defaults\', \'debug_name\', \'context_sparse_types\', \'feature_list_dense_types\', \'context_dense_shapes\', \'feature_list_sparse_types\', \'feature_list_dense_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParseTensor"
+    argspec: "args=[\'serialized\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PartitionedCall"
+    argspec: "args=[\'args\', \'Tout\', \'f\', \'config\', \'config_proto\', \'executor_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Placeholder"
+    argspec: "args=[\'dtype\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PlaceholderV2"
+    argspec: "args=[\'dtype\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PlaceholderWithDefault"
+    argspec: "args=[\'input\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Polygamma"
+    argspec: "args=[\'a\', \'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PopulationCount"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Pow"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PrefetchDataset"
+    argspec: "args=[\'input_dataset\', \'buffer_size\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PreventGradient"
+    argspec: "args=[\'input\', \'message\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Print"
+    argspec: "args=[\'input\', \'data\', \'message\', \'first_n\', \'summarize\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PrintV2"
+    argspec: "args=[\'input\', \'output_stream\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PriorityQueue"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PriorityQueueV2"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Prod"
+    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PyFunc"
+    argspec: "args=[\'input\', \'token\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PyFuncStateless"
+    argspec: "args=[\'input\', \'token\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Qr"
+    argspec: "args=[\'input\', \'full_matrices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizeAndDequantize"
+    argspec: "args=[\'input\', \'signed_input\', \'num_bits\', \'range_given\', \'input_min\', \'input_max\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizeAndDequantizeV2"
+    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'signed_input\', \'num_bits\', \'range_given\', \'round_mode\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizeAndDequantizeV3"
+    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'num_bits\', \'signed_input\', \'range_given\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizeDownAndShrinkRange"
+    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizeV2"
+    argspec: "args=[\'input\', \'min_range\', \'max_range\', \'T\', \'mode\', \'round_mode\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedAdd"
+    argspec: "args=[\'x\', \'y\', \'min_x\', \'max_x\', \'min_y\', \'max_y\', \'Toutput\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedAvgPool"
+    argspec: "args=[\'input\', \'min_input\', \'max_input\', \'ksize\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedBatchNormWithGlobalNormalization"
+    argspec: "args=[\'t\', \'t_min\', \'t_max\', \'m\', \'m_min\', \'m_max\', \'v\', \'v_min\', \'v_max\', \'beta\', \'beta_min\', \'beta_max\', \'gamma\', \'gamma_min\', \'gamma_max\', \'out_type\', \'variance_epsilon\', \'scale_after_normalization\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedBiasAdd"
+    argspec: "args=[\'input\', \'bias\', \'min_input\', \'max_input\', \'min_bias\', \'max_bias\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConcat"
+    argspec: "args=[\'concat_dim\', \'values\', \'input_mins\', \'input_maxes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2D"
+    argspec: "args=[\'input\', \'filter\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DAndRelu"
+    argspec: "args=[\'input\', \'filter\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DAndReluAndRequantize"
+    argspec: "args=[\'input\', \'filter\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DAndRequantize"
+    argspec: "args=[\'input\', \'filter\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DWithBias"
+    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DWithBiasAndRelu"
+    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DWithBiasAndReluAndRequantize"
+    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DWithBiasAndRequantize"
+    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DWithBiasSignedSumAndReluAndRequantize"
+    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'summand\', \'min_summand\', \'max_summand\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DWithBiasSumAndRelu"
+    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'summand\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DWithBiasSumAndReluAndRequantize"
+    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'summand\', \'min_summand\', \'max_summand\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedInstanceNorm"
+    argspec: "args=[\'x\', \'x_min\', \'x_max\', \'output_range_given\', \'given_y_min\', \'given_y_max\', \'variance_epsilon\', \'min_separation\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedMatMul"
+    argspec: "args=[\'a\', \'b\', \'min_a\', \'max_a\', \'min_b\', \'max_b\', \'Toutput\', \'transpose_a\', \'transpose_b\', \'Tactivation\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedMaxPool"
+    argspec: "args=[\'input\', \'min_input\', \'max_input\', \'ksize\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedMul"
+    argspec: "args=[\'x\', \'y\', \'min_x\', \'max_x\', \'min_y\', \'max_y\', \'Toutput\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedRelu"
+    argspec: "args=[\'features\', \'min_features\', \'max_features\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedRelu6"
+    argspec: "args=[\'features\', \'min_features\', \'max_features\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedReluX"
+    argspec: "args=[\'features\', \'max_value\', \'min_features\', \'max_features\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedReshape"
+    argspec: "args=[\'tensor\', \'shape\', \'input_min\', \'input_max\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedResizeBilinear"
+    argspec: "args=[\'images\', \'size\', \'min\', \'max\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueClose"
+    argspec: "args=[\'handle\', \'cancel_pending_enqueues\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueCloseV2"
+    argspec: "args=[\'handle\', \'cancel_pending_enqueues\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueDequeue"
+    argspec: "args=[\'handle\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueDequeueMany"
+    argspec: "args=[\'handle\', \'n\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueDequeueManyV2"
+    argspec: "args=[\'handle\', \'n\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueDequeueUpTo"
+    argspec: "args=[\'handle\', \'n\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueDequeueUpToV2"
+    argspec: "args=[\'handle\', \'n\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueDequeueV2"
+    argspec: "args=[\'handle\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueEnqueue"
+    argspec: "args=[\'handle\', \'components\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueEnqueueMany"
+    argspec: "args=[\'handle\', \'components\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueEnqueueManyV2"
+    argspec: "args=[\'handle\', \'components\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueEnqueueV2"
+    argspec: "args=[\'handle\', \'components\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueIsClosed"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueIsClosedV2"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueSize"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueSizeV2"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RFFT"
+    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RFFT2D"
+    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RFFT3D"
+    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RGBToHSV"
+    argspec: "args=[\'images\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RaggedGather"
+    argspec: "args=[\'params_nested_splits\', \'params_dense_values\', \'indices\', \'OUTPUT_RAGGED_RANK\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RaggedRange"
+    argspec: "args=[\'starts\', \'limits\', \'deltas\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RaggedTensorToSparse"
+    argspec: "args=[\'rt_nested_splits\', \'rt_dense_values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomCrop"
+    argspec: "args=[\'image\', \'size\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomGamma"
+    argspec: "args=[\'shape\', \'alpha\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomGammaGrad"
+    argspec: "args=[\'alpha\', \'sample\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomPoisson"
+    argspec: "args=[\'shape\', \'rate\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomPoissonV2"
+    argspec: "args=[\'shape\', \'rate\', \'seed\', \'seed2\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomShuffle"
+    argspec: "args=[\'value\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomShuffleQueue"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'min_after_dequeue\', \'seed\', \'seed2\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomShuffleQueueV2"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'min_after_dequeue\', \'seed\', \'seed2\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomStandardNormal"
+    argspec: "args=[\'shape\', \'seed\', \'seed2\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomUniform"
+    argspec: "args=[\'shape\', \'seed\', \'seed2\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomUniformInt"
+    argspec: "args=[\'shape\', \'minval\', \'maxval\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Range"
+    argspec: "args=[\'start\', \'limit\', \'delta\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RangeDataset"
+    argspec: "args=[\'start\', \'stop\', \'step\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Rank"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReadFile"
+    argspec: "args=[\'filename\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReadVariableOp"
+    argspec: "args=[\'resource\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderNumRecordsProduced"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderNumRecordsProducedV2"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderNumWorkUnitsCompleted"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderNumWorkUnitsCompletedV2"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderRead"
+    argspec: "args=[\'reader_handle\', \'queue_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderReadUpTo"
+    argspec: "args=[\'reader_handle\', \'queue_handle\', \'num_records\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderReadUpToV2"
+    argspec: "args=[\'reader_handle\', \'queue_handle\', \'num_records\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderReadV2"
+    argspec: "args=[\'reader_handle\', \'queue_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderReset"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderResetV2"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderRestoreState"
+    argspec: "args=[\'reader_handle\', \'state\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderRestoreStateV2"
+    argspec: "args=[\'reader_handle\', \'state\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderSerializeState"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderSerializeStateV2"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Real"
+    argspec: "args=[\'input\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RealDiv"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Reciprocal"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReciprocalGrad"
+    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RecordInput"
+    argspec: "args=[\'file_pattern\', \'file_random_seed\', \'file_shuffle_shift_ratio\', \'file_buffer_size\', \'file_parallelism\', \'batch_size\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReduceDataset"
+    argspec: "args=[\'input_dataset\', \'initial_state\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\', \'use_inter_op_parallelism\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReduceJoin"
+    argspec: "args=[\'inputs\', \'reduction_indices\', \'keep_dims\', \'separator\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RefEnter"
+    argspec: "args=[\'data\', \'frame_name\', \'is_constant\', \'parallel_iterations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RefExit"
+    argspec: "args=[\'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RefIdentity"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RefMerge"
+    argspec: "args=[\'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RefNextIteration"
+    argspec: "args=[\'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RefSelect"
+    argspec: "args=[\'index\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RefSwitch"
+    argspec: "args=[\'data\', \'pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RegexFullMatch"
+    argspec: "args=[\'input\', \'pattern\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RegexReplace"
+    argspec: "args=[\'input\', \'pattern\', \'rewrite\', \'replace_global\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Relu"
+    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Relu6"
+    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Relu6Grad"
+    argspec: "args=[\'gradients\', \'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReluGrad"
+    argspec: "args=[\'gradients\', \'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RemoteCall"
+    argspec: "args=[\'target\', \'args\', \'Tout\', \'f\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RepeatDataset"
+    argspec: "args=[\'input_dataset\', \'count\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RequantizationRange"
+    argspec: "args=[\'input\', \'input_min\', \'input_max\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Requantize"
+    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'requested_output_min\', \'requested_output_max\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Reshape"
+    argspec: "args=[\'tensor\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResizeArea"
+    argspec: "args=[\'images\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResizeBicubic"
+    argspec: "args=[\'images\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResizeBicubicGrad"
+    argspec: "args=[\'grads\', \'original_image\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResizeBilinear"
+    argspec: "args=[\'images\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResizeBilinearGrad"
+    argspec: "args=[\'grads\', \'original_image\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResizeNearestNeighbor"
+    argspec: "args=[\'images\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResizeNearestNeighborGrad"
+    argspec: "args=[\'grads\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyAdaMax"
+    argspec: "args=[\'var\', \'m\', \'v\', \'beta1_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyAdadelta"
+    argspec: "args=[\'var\', \'accum\', \'accum_update\', \'lr\', \'rho\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'use_locking\', \'update_slots\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyAdagradDA"
+    argspec: "args=[\'var\', \'gradient_accumulator\', \'gradient_squared_accumulator\', \'grad\', \'lr\', \'l1\', \'l2\', \'global_step\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyAdam"
+    argspec: "args=[\'var\', \'m\', \'v\', \'beta1_power\', \'beta2_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyAdamWithAmsgrad"
+    argspec: "args=[\'var\', \'m\', \'v\', \'vhat\', \'beta1_power\', \'beta2_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyAddSign"
+    argspec: "args=[\'var\', \'m\', \'lr\', \'alpha\', \'sign_decay\', \'beta\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyCenteredRMSProp"
+    argspec: "args=[\'var\', \'mg\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyFtrl"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'lr\', \'l1\', \'l2\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyFtrlV2"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'lr\', \'l1\', \'l2\', \'l2_shrinkage\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyGradientDescent"
+    argspec: "args=[\'var\', \'alpha\', \'delta\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyKerasMomentum"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyMomentum"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyPowerSign"
+    argspec: "args=[\'var\', \'m\', \'lr\', \'logbase\', \'sign_decay\', \'beta\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyProximalAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'l1\', \'l2\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyProximalGradientDescent"
+    argspec: "args=[\'var\', \'alpha\', \'l1\', \'l2\', \'delta\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyRMSProp"
+    argspec: "args=[\'var\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceCountUpTo"
+    argspec: "args=[\'resource\', \'limit\', \'T\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceGather"
+    argspec: "args=[\'resource\', \'indices\', \'validate_indices\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterAdd"
+    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterDiv"
+    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterMax"
+    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterMin"
+    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterMul"
+    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterNdAdd"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterNdSub"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterNdUpdate"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterSub"
+    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterUpdate"
+    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyAdadelta"
+    argspec: "args=[\'var\', \'accum\', \'accum_update\', \'lr\', \'rho\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'use_locking\', \'update_slots\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyAdagradDA"
+    argspec: "args=[\'var\', \'gradient_accumulator\', \'gradient_squared_accumulator\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'global_step\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyCenteredRMSProp"
+    argspec: "args=[\'var\', \'mg\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyFtrl"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyFtrlV2"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'l2_shrinkage\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyKerasMomentum"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyMomentum"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyProximalAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'l1\', \'l2\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyProximalGradientDescent"
+    argspec: "args=[\'var\', \'alpha\', \'l1\', \'l2\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyRMSProp"
+    argspec: "args=[\'var\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceStridedSliceAssign"
+    argspec: "args=[\'ref\', \'begin\', \'end\', \'strides\', \'value\', \'begin_mask\', \'end_mask\', \'ellipsis_mask\', \'new_axis_mask\', \'shrink_axis_mask\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Restore"
+    argspec: "args=[\'file_pattern\', \'tensor_name\', \'dt\', \'preferred_shard\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RestoreSlice"
+    argspec: "args=[\'file_pattern\', \'tensor_name\', \'shape_and_slice\', \'dt\', \'preferred_shard\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RestoreV2"
+    argspec: "args=[\'prefix\', \'tensor_names\', \'shape_and_slices\', \'dtypes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Reverse"
+    argspec: "args=[\'tensor\', \'dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReverseSequence"
+    argspec: "args=[\'input\', \'seq_lengths\', \'seq_dim\', \'batch_dim\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReverseV2"
+    argspec: "args=[\'tensor\', \'axis\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RightShift"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Rint"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Roll"
+    argspec: "args=[\'input\', \'shift\', \'axis\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Round"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Rsqrt"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RsqrtGrad"
+    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SampleDistortedBoundingBox"
+    argspec: "args=[\'image_size\', \'bounding_boxes\', \'seed\', \'seed2\', \'min_object_covered\', \'aspect_ratio_range\', \'area_range\', \'max_attempts\', \'use_image_if_no_bounding_boxes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SampleDistortedBoundingBoxV2"
+    argspec: "args=[\'image_size\', \'bounding_boxes\', \'min_object_covered\', \'seed\', \'seed2\', \'aspect_ratio_range\', \'area_range\', \'max_attempts\', \'use_image_if_no_bounding_boxes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Save"
+    argspec: "args=[\'filename\', \'tensor_names\', \'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SaveSlices"
+    argspec: "args=[\'filename\', \'tensor_names\', \'shapes_and_slices\', \'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SaveV2"
+    argspec: "args=[\'prefix\', \'tensor_names\', \'shape_and_slices\', \'tensors\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScalarSummary"
+    argspec: "args=[\'tags\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScaleAndTranslate"
+    argspec: "args=[\'images\', \'size\', \'scale\', \'translation\', \'kernel_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScaleAndTranslateGrad"
+    argspec: "args=[\'grads\', \'original_image\', \'scale\', \'translation\', \'kernel_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterAdd"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterDiv"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterMax"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterMin"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterMul"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterNd"
+    argspec: "args=[\'indices\', \'updates\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterNdAdd"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterNdNonAliasingAdd"
+    argspec: "args=[\'input\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterNdSub"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterNdUpdate"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterSub"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterUpdate"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SdcaFprint"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SdcaOptimizer"
+    argspec: "args=[\'sparse_example_indices\', \'sparse_feature_indices\', \'sparse_feature_values\', \'dense_features\', \'example_weights\', \'example_labels\', \'sparse_indices\', \'sparse_weights\', \'dense_weights\', \'example_state_data\', \'loss_type\', \'adaptative\', \'l1\', \'l2\', \'num_loss_partitions\', \'num_inner_iterations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SdcaOptimizerV2"
+    argspec: "args=[\'sparse_example_indices\', \'sparse_feature_indices\', \'sparse_feature_values\', \'dense_features\', \'example_weights\', \'example_labels\', \'sparse_indices\', \'sparse_weights\', \'dense_weights\', \'example_state_data\', \'loss_type\', \'adaptive\', \'l1\', \'l2\', \'num_loss_partitions\', \'num_inner_iterations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SdcaShrinkL1"
+    argspec: "args=[\'weights\', \'l1\', \'l2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SegmentMax"
+    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SegmentMean"
+    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SegmentMin"
+    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SegmentProd"
+    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SegmentSum"
+    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Select"
+    argspec: "args=[\'condition\', \'t\', \'e\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SelfAdjointEig"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SelfAdjointEigV2"
+    argspec: "args=[\'input\', \'compute_v\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Selu"
+    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SeluGrad"
+    argspec: "args=[\'gradients\', \'outputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SerializeIterator"
+    argspec: "args=[\'resource_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SerializeManySparse"
+    argspec: "args=[\'sparse_indices\', \'sparse_values\', \'sparse_shape\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SerializeSparse"
+    argspec: "args=[\'sparse_indices\', \'sparse_values\', \'sparse_shape\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SerializeTensor"
+    argspec: "args=[\'tensor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SetSize"
+    argspec: "args=[\'set_indices\', \'set_values\', \'set_shape\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Shape"
+    argspec: "args=[\'input\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ShapeN"
+    argspec: "args=[\'input\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ShardedFilename"
+    argspec: "args=[\'basename\', \'shard\', \'num_shards\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ShardedFilespec"
+    argspec: "args=[\'basename\', \'num_shards\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ShuffleAndRepeatDataset"
+    argspec: "args=[\'input_dataset\', \'buffer_size\', \'seed\', \'seed2\', \'count\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ShuffleDataset"
+    argspec: "args=[\'input_dataset\', \'buffer_size\', \'seed\', \'seed2\', \'reshuffle_each_iteration\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Sigmoid"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SigmoidGrad"
+    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Sign"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Sin"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Sinh"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Size"
+    argspec: "args=[\'input\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SkipDataset"
+    argspec: "args=[\'input_dataset\', \'count\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Slice"
+    argspec: "args=[\'input\', \'begin\', \'size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Snapshot"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Softmax"
+    argspec: "args=[\'logits\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SoftmaxCrossEntropyWithLogits"
+    argspec: "args=[\'features\', \'labels\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Softplus"
+    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SoftplusGrad"
+    argspec: "args=[\'gradients\', \'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Softsign"
+    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SoftsignGrad"
+    argspec: "args=[\'gradients\', \'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SpaceToBatch"
+    argspec: "args=[\'input\', \'paddings\', \'block_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SpaceToBatchND"
+    argspec: "args=[\'input\', \'block_shape\', \'paddings\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SpaceToDepth"
+    argspec: "args=[\'input\', \'block_size\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseAccumulatorApplyGradient"
+    argspec: "args=[\'handle\', \'local_step\', \'gradient_indices\', \'gradient_values\', \'gradient_shape\', \'has_known_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseAccumulatorTakeGradient"
+    argspec: "args=[\'handle\', \'num_required\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseAdd"
+    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b_indices\', \'b_values\', \'b_shape\', \'thresh\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseAddGrad"
+    argspec: "args=[\'backprop_val_grad\', \'a_indices\', \'b_indices\', \'sum_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyAdadelta"
+    argspec: "args=[\'var\', \'accum\', \'accum_update\', \'lr\', \'rho\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'use_locking\', \'update_slots\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyAdagradDA"
+    argspec: "args=[\'var\', \'gradient_accumulator\', \'gradient_squared_accumulator\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'global_step\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyCenteredRMSProp"
+    argspec: "args=[\'var\', \'mg\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyFtrl"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyFtrlV2"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'l2_shrinkage\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyMomentum"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyProximalAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'l1\', \'l2\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyProximalGradientDescent"
+    argspec: "args=[\'var\', \'alpha\', \'l1\', \'l2\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyRMSProp"
+    argspec: "args=[\'var\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseConcat"
+    argspec: "args=[\'indices\', \'values\', \'shapes\', \'concat_dim\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseConditionalAccumulator"
+    argspec: "args=[\'dtype\', \'shape\', \'container\', \'shared_name\', \'reduction_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseCross"
+    argspec: "args=[\'indices\', \'values\', \'shapes\', \'dense_inputs\', \'hashed_output\', \'num_buckets\', \'hash_key\', \'out_type\', \'internal_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseDenseCwiseAdd"
+    argspec: "args=[\'sp_indices\', \'sp_values\', \'sp_shape\', \'dense\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseDenseCwiseDiv"
+    argspec: "args=[\'sp_indices\', \'sp_values\', \'sp_shape\', \'dense\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseDenseCwiseMul"
+    argspec: "args=[\'sp_indices\', \'sp_values\', \'sp_shape\', \'dense\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseFillEmptyRows"
+    argspec: "args=[\'indices\', \'values\', \'dense_shape\', \'default_value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseFillEmptyRowsGrad"
+    argspec: "args=[\'reverse_index_map\', \'grad_values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseMatMul"
+    argspec: "args=[\'a\', \'b\', \'transpose_a\', \'transpose_b\', \'a_is_sparse\', \'b_is_sparse\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseReduceMax"
+    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseReduceMaxSparse"
+    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseReduceSum"
+    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseReduceSumSparse"
+    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseReorder"
+    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseReshape"
+    argspec: "args=[\'input_indices\', \'input_shape\', \'new_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentMean"
+    argspec: "args=[\'data\', \'indices\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentMeanGrad"
+    argspec: "args=[\'grad\', \'indices\', \'segment_ids\', \'output_dim0\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentMeanWithNumSegments"
+    argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentSqrtN"
+    argspec: "args=[\'data\', \'indices\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentSqrtNGrad"
+    argspec: "args=[\'grad\', \'indices\', \'segment_ids\', \'output_dim0\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentSqrtNWithNumSegments"
+    argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentSum"
+    argspec: "args=[\'data\', \'indices\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentSumWithNumSegments"
+    argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSlice"
+    argspec: "args=[\'indices\', \'values\', \'shape\', \'start\', \'size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSliceGrad"
+    argspec: "args=[\'backprop_val_grad\', \'input_indices\', \'input_start\', \'output_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSoftmax"
+    argspec: "args=[\'sp_indices\', \'sp_values\', \'sp_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSoftmaxCrossEntropyWithLogits"
+    argspec: "args=[\'features\', \'labels\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSparseMaximum"
+    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b_indices\', \'b_values\', \'b_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSparseMinimum"
+    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b_indices\', \'b_values\', \'b_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSplit"
+    argspec: "args=[\'split_dim\', \'indices\', \'values\', \'shape\', \'num_split\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseTensorDenseAdd"
+    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseTensorDenseMatMul"
+    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b\', \'adjoint_a\', \'adjoint_b\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseTensorSliceDataset"
+    argspec: "args=[\'indices\', \'values\', \'dense_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseToDense"
+    argspec: "args=[\'sparse_indices\', \'output_shape\', \'sparse_values\', \'default_value\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseToSparseSetOperation"
+    argspec: "args=[\'set1_indices\', \'set1_values\', \'set1_shape\', \'set2_indices\', \'set2_values\', \'set2_shape\', \'set_operation\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Split"
+    argspec: "args=[\'split_dim\', \'value\', \'num_split\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SplitV"
+    argspec: "args=[\'value\', \'size_splits\', \'split_dim\', \'num_split\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Sqrt"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SqrtGrad"
+    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Square"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SquaredDifference"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Squeeze"
+    argspec: "args=[\'input\', \'squeeze_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Stack"
+    argspec: "args=[\'elem_type\', \'stack_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StackClose"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StackCloseV2"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StackPop"
+    argspec: "args=[\'handle\', \'elem_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StackPopV2"
+    argspec: "args=[\'handle\', \'elem_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StackPush"
+    argspec: "args=[\'handle\', \'elem\', \'swap_memory\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StackPushV2"
+    argspec: "args=[\'handle\', \'elem\', \'swap_memory\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StackV2"
+    argspec: "args=[\'max_size\', \'elem_type\', \'stack_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Stage"
+    argspec: "args=[\'values\', \'capacity\', \'memory_limit\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StageClear"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StagePeek"
+    argspec: "args=[\'index\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StageSize"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatefulPartitionedCall"
+    argspec: "args=[\'args\', \'Tout\', \'f\', \'config\', \'config_proto\', \'executor_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatelessIf"
+    argspec: "args=[\'cond\', \'input\', \'Tout\', \'then_branch\', \'else_branch\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatelessMultinomial"
+    argspec: "args=[\'logits\', \'num_samples\', \'seed\', \'output_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatelessRandomNormal"
+    argspec: "args=[\'shape\', \'seed\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatelessRandomUniform"
+    argspec: "args=[\'shape\', \'seed\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatelessRandomUniformInt"
+    argspec: "args=[\'shape\', \'seed\', \'minval\', \'maxval\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatelessTruncatedNormal"
+    argspec: "args=[\'shape\', \'seed\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatelessWhile"
+    argspec: "args=[\'input\', \'cond\', \'body\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StaticRegexFullMatch"
+    argspec: "args=[\'input\', \'pattern\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StaticRegexReplace"
+    argspec: "args=[\'input\', \'pattern\', \'rewrite\', \'replace_global\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StopGradient"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StridedSlice"
+    argspec: "args=[\'input\', \'begin\', \'end\', \'strides\', \'begin_mask\', \'end_mask\', \'ellipsis_mask\', \'new_axis_mask\', \'shrink_axis_mask\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StridedSliceAssign"
+    argspec: "args=[\'ref\', \'begin\', \'end\', \'strides\', \'value\', \'begin_mask\', \'end_mask\', \'ellipsis_mask\', \'new_axis_mask\', \'shrink_axis_mask\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StridedSliceGrad"
+    argspec: "args=[\'shape\', \'begin\', \'end\', \'strides\', \'dy\', \'begin_mask\', \'end_mask\', \'ellipsis_mask\', \'new_axis_mask\', \'shrink_axis_mask\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringFormat"
+    argspec: "args=[\'inputs\', \'template\', \'placeholder\', \'summarize\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringJoin"
+    argspec: "args=[\'inputs\', \'separator\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringLength"
+    argspec: "args=[\'input\', \'unit\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringSplit"
+    argspec: "args=[\'input\', \'delimiter\', \'skip_empty\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringSplitV2"
+    argspec: "args=[\'input\', \'sep\', \'maxsplit\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringStrip"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringToHashBucket"
+    argspec: "args=[\'string_tensor\', \'num_buckets\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringToHashBucketFast"
+    argspec: "args=[\'input\', \'num_buckets\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringToHashBucketStrong"
+    argspec: "args=[\'input\', \'num_buckets\', \'key\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringToNumber"
+    argspec: "args=[\'string_tensor\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Sub"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Substr"
+    argspec: "args=[\'input\', \'pos\', \'len\', \'unit\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Sum"
+    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SummaryWriter"
+    argspec: "args=[\'shared_name\', \'container\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Svd"
+    argspec: "args=[\'input\', \'compute_uv\', \'full_matrices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Switch"
+    argspec: "args=[\'data\', \'pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SymbolicGradient"
+    argspec: "args=[\'input\', \'Tout\', \'f\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TFRecordDataset"
+    argspec: "args=[\'filenames\', \'compression_type\', \'buffer_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TFRecordReader"
+    argspec: "args=[\'container\', \'shared_name\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TFRecordReaderV2"
+    argspec: "args=[\'container\', \'shared_name\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TakeDataset"
+    argspec: "args=[\'input_dataset\', \'count\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TakeManySparseFromTensorsMap"
+    argspec: "args=[\'sparse_handles\', \'dtype\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Tan"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Tanh"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TanhGrad"
+    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TemporaryVariable"
+    argspec: "args=[\'shape\', \'dtype\', \'var_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArray"
+    argspec: "args=[\'size\', \'dtype\', \'dynamic_size\', \'clear_after_read\', \'tensor_array_name\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayClose"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayCloseV2"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayCloseV3"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayConcat"
+    argspec: "args=[\'handle\', \'flow_in\', \'dtype\', \'element_shape_except0\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayConcatV2"
+    argspec: "args=[\'handle\', \'flow_in\', \'dtype\', \'element_shape_except0\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayConcatV3"
+    argspec: "args=[\'handle\', \'flow_in\', \'dtype\', \'element_shape_except0\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayGather"
+    argspec: "args=[\'handle\', \'indices\', \'flow_in\', \'dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayGatherV2"
+    argspec: "args=[\'handle\', \'indices\', \'flow_in\', \'dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayGatherV3"
+    argspec: "args=[\'handle\', \'indices\', \'flow_in\', \'dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayGrad"
+    argspec: "args=[\'handle\', \'flow_in\', \'source\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayGradV2"
+    argspec: "args=[\'handle\', \'flow_in\', \'source\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayGradV3"
+    argspec: "args=[\'handle\', \'flow_in\', \'source\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayGradWithShape"
+    argspec: "args=[\'handle\', \'flow_in\', \'shape_to_prepend\', \'source\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayPack"
+    argspec: "args=[\'handle\', \'flow_in\', \'dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayRead"
+    argspec: "args=[\'handle\', \'index\', \'flow_in\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayReadV2"
+    argspec: "args=[\'handle\', \'index\', \'flow_in\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayReadV3"
+    argspec: "args=[\'handle\', \'index\', \'flow_in\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayScatter"
+    argspec: "args=[\'handle\', \'indices\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayScatterV2"
+    argspec: "args=[\'handle\', \'indices\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayScatterV3"
+    argspec: "args=[\'handle\', \'indices\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArraySize"
+    argspec: "args=[\'handle\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArraySizeV2"
+    argspec: "args=[\'handle\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArraySizeV3"
+    argspec: "args=[\'handle\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArraySplit"
+    argspec: "args=[\'handle\', \'value\', \'lengths\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArraySplitV2"
+    argspec: "args=[\'handle\', \'value\', \'lengths\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArraySplitV3"
+    argspec: "args=[\'handle\', \'value\', \'lengths\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayUnpack"
+    argspec: "args=[\'handle\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayV2"
+    argspec: "args=[\'size\', \'dtype\', \'element_shape\', \'dynamic_size\', \'clear_after_read\', \'tensor_array_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayV3"
+    argspec: "args=[\'size\', \'dtype\', \'element_shape\', \'dynamic_size\', \'clear_after_read\', \'identical_element_shapes\', \'tensor_array_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayWrite"
+    argspec: "args=[\'handle\', \'index\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayWriteV2"
+    argspec: "args=[\'handle\', \'index\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayWriteV3"
+    argspec: "args=[\'handle\', \'index\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorDataset"
+    argspec: "args=[\'components\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListConcat"
+    argspec: "args=[\'input_handle\', \'element_dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListConcatLists"
+    argspec: "args=[\'input_a\', \'input_b\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListConcatV2"
+    argspec: "args=[\'input_handle\', \'element_shape\', \'leading_dims\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListElementShape"
+    argspec: "args=[\'input_handle\', \'shape_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListFromTensor"
+    argspec: "args=[\'tensor\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListGather"
+    argspec: "args=[\'input_handle\', \'indices\', \'element_shape\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListGetItem"
+    argspec: "args=[\'input_handle\', \'index\', \'element_shape\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListLength"
+    argspec: "args=[\'input_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListPopBack"
+    argspec: "args=[\'input_handle\', \'element_shape\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListPushBack"
+    argspec: "args=[\'input_handle\', \'tensor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListPushBackBatch"
+    argspec: "args=[\'input_handles\', \'tensor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListReserve"
+    argspec: "args=[\'element_shape\', \'num_elements\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListResize"
+    argspec: "args=[\'input_handle\', \'size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListScatter"
+    argspec: "args=[\'tensor\', \'indices\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListScatterV2"
+    argspec: "args=[\'tensor\', \'indices\', \'element_shape\', \'num_elements\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListSetItem"
+    argspec: "args=[\'input_handle\', \'index\', \'item\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListSplit"
+    argspec: "args=[\'tensor\', \'element_shape\', \'lengths\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListStack"
+    argspec: "args=[\'input_handle\', \'element_shape\', \'element_dtype\', \'num_elements\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorScatterAdd"
+    argspec: "args=[\'tensor\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorScatterSub"
+    argspec: "args=[\'tensor\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorScatterUpdate"
+    argspec: "args=[\'tensor\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorSliceDataset"
+    argspec: "args=[\'components\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorSummary"
+    argspec: "args=[\'tensor\', \'description\', \'labels\', \'display_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorSummaryV2"
+    argspec: "args=[\'tag\', \'tensor\', \'serialized_summary_metadata\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TextLineDataset"
+    argspec: "args=[\'filenames\', \'compression_type\', \'buffer_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TextLineReader"
+    argspec: "args=[\'skip_header_lines\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TextLineReaderV2"
+    argspec: "args=[\'skip_header_lines\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ThreadUnsafeUnigramCandidateSampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Tile"
+    argspec: "args=[\'input\', \'multiples\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TileGrad"
+    argspec: "args=[\'input\', \'multiples\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Timestamp"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TopK"
+    argspec: "args=[\'input\', \'k\', \'sorted\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TopKV2"
+    argspec: "args=[\'input\', \'k\', \'sorted\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Transpose"
+    argspec: "args=[\'x\', \'perm\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TruncateDiv"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TruncateMod"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TruncatedNormal"
+    argspec: "args=[\'shape\', \'seed\', \'seed2\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnicodeDecode"
+    argspec: "args=[\'input\', \'input_encoding\', \'errors\', \'replacement_char\', \'replace_control_characters\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnicodeDecodeWithOffsets"
+    argspec: "args=[\'input\', \'input_encoding\', \'errors\', \'replacement_char\', \'replace_control_characters\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnicodeEncode"
+    argspec: "args=[\'input_values\', \'input_splits\', \'errors\', \'output_encoding\', \'replacement_char\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnicodeScript"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnicodeTranscode"
+    argspec: "args=[\'input\', \'input_encoding\', \'output_encoding\', \'errors\', \'replacement_char\', \'replace_control_characters\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UniformCandidateSampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Unique"
+    argspec: "args=[\'x\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UniqueV2"
+    argspec: "args=[\'x\', \'axis\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UniqueWithCounts"
+    argspec: "args=[\'x\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UniqueWithCountsV2"
+    argspec: "args=[\'x\', \'axis\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Unpack"
+    argspec: "args=[\'value\', \'num\', \'axis\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnravelIndex"
+    argspec: "args=[\'indices\', \'dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnsortedSegmentMax"
+    argspec: "args=[\'data\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnsortedSegmentMin"
+    argspec: "args=[\'data\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnsortedSegmentProd"
+    argspec: "args=[\'data\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnsortedSegmentSum"
+    argspec: "args=[\'data\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Unstage"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnwrapDatasetVariant"
+    argspec: "args=[\'input_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UpperBound"
+    argspec: "args=[\'sorted_inputs\', \'values\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "VarHandleOp"
+    argspec: "args=[\'container\', \'shared_name\', \'dtype\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "VarIsInitializedOp"
+    argspec: "args=[\'resource\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Variable"
+    argspec: "args=[\'shape\', \'dtype\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "VariableShape"
+    argspec: "args=[\'input\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "VariableV2"
+    argspec: "args=[\'shape\', \'dtype\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Where"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "While"
+    argspec: "args=[\'input\', \'cond\', \'body\', \'output_shapes\', \'parallel_iterations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WholeFileReader"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WholeFileReaderV2"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WindowDataset"
+    argspec: "args=[\'input_dataset\', \'size\', \'shift\', \'stride\', \'drop_remainder\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WrapDatasetVariant"
+    argspec: "args=[\'input_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WriteAudioSummary"
+    argspec: "args=[\'writer\', \'step\', \'tag\', \'tensor\', \'sample_rate\', \'max_outputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WriteFile"
+    argspec: "args=[\'filename\', \'contents\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WriteGraphSummary"
+    argspec: "args=[\'writer\', \'step\', \'tensor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WriteHistogramSummary"
+    argspec: "args=[\'writer\', \'step\', \'tag\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WriteImageSummary"
+    argspec: "args=[\'writer\', \'step\', \'tag\', \'tensor\', \'bad_color\', \'max_images\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WriteScalarSummary"
+    argspec: "args=[\'writer\', \'step\', \'tag\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WriteSummary"
+    argspec: "args=[\'writer\', \'step\', \'tensor\', \'tag\', \'summary_metadata\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Xdivy"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Xlogy"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ZerosLike"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Zeta"
+    argspec: "args=[\'x\', \'q\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ZipDataset"
+    argspec: "args=[\'input_datasets\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index e1eef058f4..bb7ea2e13e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -272,6 +272,10 @@ tf_module {
     name: "random_uniform_initializer"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "raw_ops"
+    mtype: "<type \'module\'>"
+  }
   member {
     name: "resource"
     mtype: "<class \'tensorflow.python.framework.dtypes.DType\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt
new file mode 100644
index 0000000000..e7375af182
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt
@@ -0,0 +1,4027 @@
+path: "tensorflow.raw_ops"
+tf_module {
+  member_method {
+    name: "Abort"
+    argspec: "args=[\'error_msg\', \'exit_without_error\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Abs"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AccumulateNV2"
+    argspec: "args=[\'inputs\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AccumulatorApplyGradient"
+    argspec: "args=[\'handle\', \'local_step\', \'gradient\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AccumulatorNumAccumulated"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AccumulatorSetGlobalStep"
+    argspec: "args=[\'handle\', \'new_global_step\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AccumulatorTakeGradient"
+    argspec: "args=[\'handle\', \'num_required\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Acos"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Acosh"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Add"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AddManySparseToTensorsMap"
+    argspec: "args=[\'sparse_indices\', \'sparse_values\', \'sparse_shape\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AddN"
+    argspec: "args=[\'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AddSparseToTensorsMap"
+    argspec: "args=[\'sparse_indices\', \'sparse_values\', \'sparse_shape\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AddV2"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AdjustContrast"
+    argspec: "args=[\'images\', \'contrast_factor\', \'min_value\', \'max_value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AdjustContrastv2"
+    argspec: "args=[\'images\', \'contrast_factor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AdjustHue"
+    argspec: "args=[\'images\', \'delta\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AdjustSaturation"
+    argspec: "args=[\'images\', \'scale\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "All"
+    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AllCandidateSampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Angle"
+    argspec: "args=[\'input\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AnonymousIterator"
+    argspec: "args=[\'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Any"
+    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyAdaMax"
+    argspec: "args=[\'var\', \'m\', \'v\', \'beta1_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyAdadelta"
+    argspec: "args=[\'var\', \'accum\', \'accum_update\', \'lr\', \'rho\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'use_locking\', \'update_slots\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyAdagradDA"
+    argspec: "args=[\'var\', \'gradient_accumulator\', \'gradient_squared_accumulator\', \'grad\', \'lr\', \'l1\', \'l2\', \'global_step\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyAdam"
+    argspec: "args=[\'var\', \'m\', \'v\', \'beta1_power\', \'beta2_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyAddSign"
+    argspec: "args=[\'var\', \'m\', \'lr\', \'alpha\', \'sign_decay\', \'beta\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyCenteredRMSProp"
+    argspec: "args=[\'var\', \'mg\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyFtrl"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'lr\', \'l1\', \'l2\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyFtrlV2"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'lr\', \'l1\', \'l2\', \'l2_shrinkage\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyGradientDescent"
+    argspec: "args=[\'var\', \'alpha\', \'delta\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyMomentum"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyPowerSign"
+    argspec: "args=[\'var\', \'m\', \'lr\', \'logbase\', \'sign_decay\', \'beta\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyProximalAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'l1\', \'l2\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyProximalGradientDescent"
+    argspec: "args=[\'var\', \'alpha\', \'l1\', \'l2\', \'delta\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApplyRMSProp"
+    argspec: "args=[\'var\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ApproximateEqual"
+    argspec: "args=[\'x\', \'y\', \'tolerance\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ArgMax"
+    argspec: "args=[\'input\', \'dimension\', \'output_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ArgMin"
+    argspec: "args=[\'input\', \'dimension\', \'output_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AsString"
+    argspec: "args=[\'input\', \'precision\', \'scientific\', \'shortest\', \'width\', \'fill\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Asin"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Asinh"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Assert"
+    argspec: "args=[\'condition\', \'data\', \'summarize\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Assign"
+    argspec: "args=[\'ref\', \'value\', \'validate_shape\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AssignAdd"
+    argspec: "args=[\'ref\', \'value\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AssignAddVariableOp"
+    argspec: "args=[\'resource\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AssignSub"
+    argspec: "args=[\'ref\', \'value\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AssignSubVariableOp"
+    argspec: "args=[\'resource\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AssignVariableOp"
+    argspec: "args=[\'resource\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Atan"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Atan2"
+    argspec: "args=[\'y\', \'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Atanh"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AudioSpectrogram"
+    argspec: "args=[\'input\', \'window_size\', \'stride\', \'magnitude_squared\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AudioSummary"
+    argspec: "args=[\'tag\', \'tensor\', \'sample_rate\', \'max_outputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AudioSummaryV2"
+    argspec: "args=[\'tag\', \'tensor\', \'sample_rate\', \'max_outputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AvgPool"
+    argspec: "args=[\'value\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AvgPool3D"
+    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AvgPool3DGrad"
+    argspec: "args=[\'orig_input_shape\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "AvgPoolGrad"
+    argspec: "args=[\'orig_input_shape\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Barrier"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BarrierClose"
+    argspec: "args=[\'handle\', \'cancel_pending_enqueues\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BarrierIncompleteSize"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BarrierInsertMany"
+    argspec: "args=[\'handle\', \'keys\', \'values\', \'component_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BarrierReadySize"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BarrierTakeMany"
+    argspec: "args=[\'handle\', \'num_elements\', \'component_types\', \'allow_small_batch\', \'wait_for_incomplete\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchCholesky"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchCholeskyGrad"
+    argspec: "args=[\'l\', \'grad\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchDataset"
+    argspec: "args=[\'input_dataset\', \'batch_size\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchDatasetV2"
+    argspec: "args=[\'input_dataset\', \'batch_size\', \'drop_remainder\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchFFT"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchFFT2D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchFFT3D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchIFFT"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchIFFT2D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchIFFT3D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatMul"
+    argspec: "args=[\'x\', \'y\', \'adj_x\', \'adj_y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixBandPart"
+    argspec: "args=[\'input\', \'num_lower\', \'num_upper\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixDeterminant"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixDiag"
+    argspec: "args=[\'diagonal\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixDiagPart"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixInverse"
+    argspec: "args=[\'input\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixSetDiag"
+    argspec: "args=[\'input\', \'diagonal\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixSolve"
+    argspec: "args=[\'matrix\', \'rhs\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixSolveLs"
+    argspec: "args=[\'matrix\', \'rhs\', \'l2_regularizer\', \'fast\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchMatrixTriangularSolve"
+    argspec: "args=[\'matrix\', \'rhs\', \'lower\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchNormWithGlobalNormalization"
+    argspec: "args=[\'t\', \'m\', \'v\', \'beta\', \'gamma\', \'variance_epsilon\', \'scale_after_normalization\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchNormWithGlobalNormalizationGrad"
+    argspec: "args=[\'t\', \'m\', \'v\', \'gamma\', \'backprop\', \'variance_epsilon\', \'scale_after_normalization\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchSelfAdjointEig"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchSelfAdjointEigV2"
+    argspec: "args=[\'input\', \'compute_v\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchSvd"
+    argspec: "args=[\'input\', \'compute_uv\', \'full_matrices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchToSpace"
+    argspec: "args=[\'input\', \'crops\', \'block_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BatchToSpaceND"
+    argspec: "args=[\'input\', \'block_shape\', \'crops\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BesselI0e"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BesselI1e"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Betainc"
+    argspec: "args=[\'a\', \'b\', \'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BiasAdd"
+    argspec: "args=[\'value\', \'bias\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BiasAddGrad"
+    argspec: "args=[\'out_backprop\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BiasAddV1"
+    argspec: "args=[\'value\', \'bias\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Bincount"
+    argspec: "args=[\'arr\', \'size\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Bitcast"
+    argspec: "args=[\'input\', \'type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BitwiseAnd"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BitwiseOr"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BitwiseXor"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesBucketize"
+    argspec: "args=[\'float_values\', \'bucket_boundaries\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesCalculateBestGainsPerFeature"
+    argspec: "args=[\'node_id_range\', \'stats_summary_list\', \'l1\', \'l2\', \'tree_complexity\', \'min_node_weight\', \'max_splits\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesCenterBias"
+    argspec: "args=[\'tree_ensemble_handle\', \'mean_gradients\', \'mean_hessians\', \'l1\', \'l2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesCreateEnsemble"
+    argspec: "args=[\'tree_ensemble_handle\', \'stamp_token\', \'tree_ensemble_serialized\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesCreateQuantileStreamResource"
+    argspec: "args=[\'quantile_stream_resource_handle\', \'epsilon\', \'num_streams\', \'max_elements\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesDeserializeEnsemble"
+    argspec: "args=[\'tree_ensemble_handle\', \'stamp_token\', \'tree_ensemble_serialized\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesEnsembleResourceHandleOp"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesExampleDebugOutputs"
+    argspec: "args=[\'tree_ensemble_handle\', \'bucketized_features\', \'logits_dimension\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesGetEnsembleStates"
+    argspec: "args=[\'tree_ensemble_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesMakeQuantileSummaries"
+    argspec: "args=[\'float_values\', \'example_weights\', \'epsilon\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesMakeStatsSummary"
+    argspec: "args=[\'node_ids\', \'gradients\', \'hessians\', \'bucketized_features_list\', \'max_splits\', \'num_buckets\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesPredict"
+    argspec: "args=[\'tree_ensemble_handle\', \'bucketized_features\', \'logits_dimension\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesQuantileStreamResourceAddSummaries"
+    argspec: "args=[\'quantile_stream_resource_handle\', \'summaries\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesQuantileStreamResourceDeserialize"
+    argspec: "args=[\'quantile_stream_resource_handle\', \'bucket_boundaries\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesQuantileStreamResourceFlush"
+    argspec: "args=[\'quantile_stream_resource_handle\', \'num_buckets\', \'generate_quantiles\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesQuantileStreamResourceGetBucketBoundaries"
+    argspec: "args=[\'quantile_stream_resource_handle\', \'num_features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesQuantileStreamResourceHandleOp"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesSerializeEnsemble"
+    argspec: "args=[\'tree_ensemble_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesTrainingPredict"
+    argspec: "args=[\'tree_ensemble_handle\', \'cached_tree_ids\', \'cached_node_ids\', \'bucketized_features\', \'logits_dimension\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BoostedTreesUpdateEnsemble"
+    argspec: "args=[\'tree_ensemble_handle\', \'feature_ids\', \'node_ids\', \'gains\', \'thresholds\', \'left_node_contribs\', \'right_node_contribs\', \'max_depth\', \'learning_rate\', \'pruning_mode\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BroadcastArgs"
+    argspec: "args=[\'s0\', \'s1\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BroadcastGradientArgs"
+    argspec: "args=[\'s0\', \'s1\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "BroadcastTo"
+    argspec: "args=[\'input\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Bucketize"
+    argspec: "args=[\'input\', \'boundaries\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CTCBeamSearchDecoder"
+    argspec: "args=[\'inputs\', \'sequence_length\', \'beam_width\', \'top_paths\', \'merge_repeated\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CTCGreedyDecoder"
+    argspec: "args=[\'inputs\', \'sequence_length\', \'merge_repeated\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CTCLoss"
+    argspec: "args=[\'inputs\', \'labels_indices\', \'labels_values\', \'sequence_length\', \'preprocess_collapse_repeated\', \'ctc_merge_repeated\', \'ignore_longer_outputs_than_inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CacheDataset"
+    argspec: "args=[\'input_dataset\', \'filename\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Cast"
+    argspec: "args=[\'x\', \'DstT\', \'Truncate\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Ceil"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CheckNumerics"
+    argspec: "args=[\'tensor\', \'message\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Cholesky"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CholeskyGrad"
+    argspec: "args=[\'l\', \'grad\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ClipByValue"
+    argspec: "args=[\'t\', \'clip_value_min\', \'clip_value_max\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CloseSummaryWriter"
+    argspec: "args=[\'writer\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CollectiveBcastRecv"
+    argspec: "args=[\'T\', \'group_size\', \'group_key\', \'instance_key\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CollectiveBcastSend"
+    argspec: "args=[\'input\', \'group_size\', \'group_key\', \'instance_key\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CollectiveReduce"
+    argspec: "args=[\'input\', \'group_size\', \'group_key\', \'instance_key\', \'merge_op\', \'final_op\', \'subdiv_offsets\', \'wait_for\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CombinedNonMaxSuppression"
+    argspec: "args=[\'boxes\', \'scores\', \'max_output_size_per_class\', \'max_total_size\', \'iou_threshold\', \'score_threshold\', \'pad_per_class\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CompareAndBitpack"
+    argspec: "args=[\'input\', \'threshold\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Complex"
+    argspec: "args=[\'real\', \'imag\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ComplexAbs"
+    argspec: "args=[\'x\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ComputeAccidentalHits"
+    argspec: "args=[\'true_classes\', \'sampled_candidates\', \'num_true\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Concat"
+    argspec: "args=[\'concat_dim\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ConcatOffset"
+    argspec: "args=[\'concat_dim\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ConcatV2"
+    argspec: "args=[\'values\', \'axis\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ConcatenateDataset"
+    argspec: "args=[\'input_dataset\', \'another_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ConditionalAccumulator"
+    argspec: "args=[\'dtype\', \'shape\', \'container\', \'shared_name\', \'reduction_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conj"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ConjugateTranspose"
+    argspec: "args=[\'x\', \'perm\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Const"
+    argspec: "args=[\'value\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ConsumeMutexLock"
+    argspec: "args=[\'mutex_lock\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ControlTrigger"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv2D"
+    argspec: "args=[\'input\', \'filter\', \'strides\', \'use_cudnn_on_gpu\', \'padding\', \'explicit_paddings\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv2DBackpropFilter"
+    argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'use_cudnn_on_gpu\', \'padding\', \'explicit_paddings\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv2DBackpropInput"
+    argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'use_cudnn_on_gpu\', \'padding\', \'explicit_paddings\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv3D"
+    argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv3DBackpropFilter"
+    argspec: "args=[\'input\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv3DBackpropFilterV2"
+    argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv3DBackpropInput"
+    argspec: "args=[\'input\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Conv3DBackpropInputV2"
+    argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Cos"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Cosh"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CountUpTo"
+    argspec: "args=[\'ref\', \'limit\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CreateSummaryDbWriter"
+    argspec: "args=[\'writer\', \'db_uri\', \'experiment_name\', \'run_name\', \'user_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CreateSummaryFileWriter"
+    argspec: "args=[\'writer\', \'logdir\', \'max_queue\', \'flush_millis\', \'filename_suffix\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CropAndResize"
+    argspec: "args=[\'image\', \'boxes\', \'box_ind\', \'crop_size\', \'method\', \'extrapolation_value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CropAndResizeGradBoxes"
+    argspec: "args=[\'grads\', \'image\', \'boxes\', \'box_ind\', \'method\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CropAndResizeGradImage"
+    argspec: "args=[\'grads\', \'boxes\', \'box_ind\', \'image_size\', \'T\', \'method\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Cross"
+    argspec: "args=[\'a\', \'b\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNN"
+    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\', \'is_training\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNBackprop"
+    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'output\', \'output_h\', \'output_c\', \'output_backprop\', \'output_h_backprop\', \'output_c_backprop\', \'reserve_space\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNBackpropV2"
+    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'output\', \'output_h\', \'output_c\', \'output_backprop\', \'output_h_backprop\', \'output_c_backprop\', \'reserve_space\', \'host_reserved\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNBackpropV3"
+    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'sequence_lengths\', \'output\', \'output_h\', \'output_c\', \'output_backprop\', \'output_h_backprop\', \'output_c_backprop\', \'reserve_space\', \'host_reserved\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNCanonicalToParams"
+    argspec: "args=[\'num_layers\', \'num_units\', \'input_size\', \'weights\', \'biases\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNParamsSize"
+    argspec: "args=[\'num_layers\', \'num_units\', \'input_size\', \'T\', \'S\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNParamsToCanonical"
+    argspec: "args=[\'num_layers\', \'num_units\', \'input_size\', \'params\', \'num_params\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNV2"
+    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\', \'is_training\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "CudnnRNNV3"
+    argspec: "args=[\'input\', \'input_h\', \'input_c\', \'params\', \'sequence_lengths\', \'rnn_mode\', \'input_mode\', \'direction\', \'dropout\', \'seed\', \'seed2\', \'is_training\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Cumprod"
+    argspec: "args=[\'x\', \'axis\', \'exclusive\', \'reverse\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Cumsum"
+    argspec: "args=[\'x\', \'axis\', \'exclusive\', \'reverse\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DataFormatDimMap"
+    argspec: "args=[\'x\', \'src_format\', \'dst_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DataFormatVecPermute"
+    argspec: "args=[\'x\', \'src_format\', \'dst_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DatasetToGraph"
+    argspec: "args=[\'input_dataset\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DatasetToSingleElement"
+    argspec: "args=[\'dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DebugGradientIdentity"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DebugGradientRefIdentity"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeAndCropJpeg"
+    argspec: "args=[\'contents\', \'crop_window\', \'channels\', \'ratio\', \'fancy_upscaling\', \'try_recover_truncated\', \'acceptable_fraction\', \'dct_method\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeBase64"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeBmp"
+    argspec: "args=[\'contents\', \'channels\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeCSV"
+    argspec: "args=[\'records\', \'record_defaults\', \'field_delim\', \'use_quote_delim\', \'na_value\', \'select_cols\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeCompressed"
+    argspec: "args=[\'bytes\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeGif"
+    argspec: "args=[\'contents\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeJSONExample"
+    argspec: "args=[\'json_examples\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeJpeg"
+    argspec: "args=[\'contents\', \'channels\', \'ratio\', \'fancy_upscaling\', \'try_recover_truncated\', \'acceptable_fraction\', \'dct_method\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodePng"
+    argspec: "args=[\'contents\', \'channels\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeRaw"
+    argspec: "args=[\'bytes\', \'out_type\', \'little_endian\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DecodeWav"
+    argspec: "args=[\'contents\', \'desired_channels\', \'desired_samples\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DeepCopy"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DeleteSessionTensor"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DenseToDenseSetOperation"
+    argspec: "args=[\'set1\', \'set2\', \'set_operation\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DenseToSparseSetOperation"
+    argspec: "args=[\'set1\', \'set2_indices\', \'set2_values\', \'set2_shape\', \'set_operation\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DepthToSpace"
+    argspec: "args=[\'input\', \'block_size\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DepthwiseConv2dNative"
+    argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DepthwiseConv2dNativeBackpropFilter"
+    argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DepthwiseConv2dNativeBackpropInput"
+    argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Dequantize"
+    argspec: "args=[\'input\', \'min_range\', \'max_range\', \'mode\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DeserializeIterator"
+    argspec: "args=[\'resource_handle\', \'serialized\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DeserializeManySparse"
+    argspec: "args=[\'serialized_sparse\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DeserializeSparse"
+    argspec: "args=[\'serialized_sparse\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DestroyResourceOp"
+    argspec: "args=[\'resource\', \'ignore_lookup_error\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DestroyTemporaryVariable"
+    argspec: "args=[\'ref\', \'var_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Diag"
+    argspec: "args=[\'diagonal\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DiagPart"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Digamma"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Dilation2D"
+    argspec: "args=[\'input\', \'filter\', \'strides\', \'rates\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Dilation2DBackpropFilter"
+    argspec: "args=[\'input\', \'filter\', \'out_backprop\', \'strides\', \'rates\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Dilation2DBackpropInput"
+    argspec: "args=[\'input\', \'filter\', \'out_backprop\', \'strides\', \'rates\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Div"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DivNoNan"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DrawBoundingBoxes"
+    argspec: "args=[\'images\', \'boxes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DynamicPartition"
+    argspec: "args=[\'data\', \'partitions\', \'num_partitions\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "DynamicStitch"
+    argspec: "args=[\'indices\', \'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EagerPyFunc"
+    argspec: "args=[\'input\', \'token\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EditDistance"
+    argspec: "args=[\'hypothesis_indices\', \'hypothesis_values\', \'hypothesis_shape\', \'truth_indices\', \'truth_values\', \'truth_shape\', \'normalize\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Elu"
+    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EluGrad"
+    argspec: "args=[\'gradients\', \'outputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Empty"
+    argspec: "args=[\'shape\', \'dtype\', \'init\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EmptyTensorList"
+    argspec: "args=[\'element_shape\', \'max_num_elements\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EncodeBase64"
+    argspec: "args=[\'input\', \'pad\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EncodeJpeg"
+    argspec: "args=[\'image\', \'format\', \'quality\', \'progressive\', \'optimize_size\', \'chroma_downsampling\', \'density_unit\', \'x_density\', \'y_density\', \'xmp_metadata\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EncodePng"
+    argspec: "args=[\'image\', \'compression\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EncodeWav"
+    argspec: "args=[\'audio\', \'sample_rate\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "EnsureShape"
+    argspec: "args=[\'input\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Enter"
+    argspec: "args=[\'data\', \'frame_name\', \'is_constant\', \'parallel_iterations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Equal"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Erf"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Erfc"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Exit"
+    argspec: "args=[\'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Exp"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExpandDims"
+    argspec: "args=[\'input\', \'dim\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalAssertNextDataset"
+    argspec: "args=[\'input_dataset\', \'transformations\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalBytesProducedStatsDataset"
+    argspec: "args=[\'input_dataset\', \'tag\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalCSVDataset"
+    argspec: "args=[\'filenames\', \'compression_type\', \'buffer_size\', \'header\', \'field_delim\', \'use_quote_delim\', \'na_value\', \'select_cols\', \'record_defaults\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalChooseFastestDataset"
+    argspec: "args=[\'input_datasets\', \'num_experiments\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalDatasetCardinality"
+    argspec: "args=[\'input_dataset\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalDatasetToTFRecord"
+    argspec: "args=[\'input_dataset\', \'filename\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalDenseToSparseBatchDataset"
+    argspec: "args=[\'input_dataset\', \'batch_size\', \'row_shape\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalDirectedInterleaveDataset"
+    argspec: "args=[\'selector_input_dataset\', \'data_input_datasets\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalGroupByReducerDataset"
+    argspec: "args=[\'input_dataset\', \'key_func_other_arguments\', \'init_func_other_arguments\', \'reduce_func_other_arguments\', \'finalize_func_other_arguments\', \'key_func\', \'init_func\', \'reduce_func\', \'finalize_func\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalGroupByWindowDataset"
+    argspec: "args=[\'input_dataset\', \'key_func_other_arguments\', \'reduce_func_other_arguments\', \'window_size_func_other_arguments\', \'key_func\', \'reduce_func\', \'window_size_func\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalIdentityIndexedDataset"
+    argspec: "args=[\'size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalIgnoreErrorsDataset"
+    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalIndexedDatasetGet"
+    argspec: "args=[\'materialized\', \'index\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalIndexedDatasetMaterialize"
+    argspec: "args=[\'dataset\', \'materialized\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalIteratorGetDevice"
+    argspec: "args=[\'resource\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalLMDBDataset"
+    argspec: "args=[\'filenames\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalLatencyStatsDataset"
+    argspec: "args=[\'input_dataset\', \'tag\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalMapAndBatchDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'batch_size\', \'num_parallel_calls\', \'drop_remainder\', \'f\', \'output_types\', \'output_shapes\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalMapDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\', \'use_inter_op_parallelism\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalMatchingFilesDataset"
+    argspec: "args=[\'patterns\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalMaterializedIndexDatasetHandle"
+    argspec: "args=[\'container\', \'shared_name\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalMaxIntraOpParallelismDataset"
+    argspec: "args=[\'input_dataset\', \'max_intra_op_parallelism\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalNonSerializableDataset"
+    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalNumaMapAndBatchDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'batch_size\', \'num_parallel_calls\', \'drop_remainder\', \'f\', \'output_types\', \'output_shapes\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalParallelInterleaveDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'cycle_length\', \'block_length\', \'sloppy\', \'buffer_output_elements\', \'prefetch_input_elements\', \'f\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalParseExampleDataset"
+    argspec: "args=[\'input_dataset\', \'num_parallel_calls\', \'dense_defaults\', \'sparse_keys\', \'dense_keys\', \'sparse_types\', \'dense_shapes\', \'output_types\', \'output_shapes\', \'sloppy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalPrivateThreadPoolDataset"
+    argspec: "args=[\'input_dataset\', \'num_threads\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalRandomDataset"
+    argspec: "args=[\'seed\', \'seed2\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalScanDataset"
+    argspec: "args=[\'input_dataset\', \'initial_state\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalSetStatsAggregatorDataset"
+    argspec: "args=[\'input_dataset\', \'stats_aggregator\', \'tag\', \'counter_prefix\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalSleepDataset"
+    argspec: "args=[\'input_dataset\', \'sleep_microseconds\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalSlidingWindowDataset"
+    argspec: "args=[\'input_dataset\', \'window_size\', \'window_shift\', \'window_stride\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalSqlDataset"
+    argspec: "args=[\'driver_name\', \'data_source_name\', \'query\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalStatsAggregatorHandle"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalStatsAggregatorSummary"
+    argspec: "args=[\'iterator\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalTakeWhileDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'predicate\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalThreadPoolDataset"
+    argspec: "args=[\'input_dataset\', \'thread_pool\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalThreadPoolHandle"
+    argspec: "args=[\'num_threads\', \'max_intra_op_parallelism\', \'display_name\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalUnbatchDataset"
+    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExperimentalUniqueDataset"
+    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Expm1"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExtractGlimpse"
+    argspec: "args=[\'input\', \'size\', \'offsets\', \'centered\', \'normalized\', \'uniform_noise\', \'noise\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExtractImagePatches"
+    argspec: "args=[\'images\', \'ksizes\', \'strides\', \'rates\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExtractJpegShape"
+    argspec: "args=[\'contents\', \'output_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ExtractVolumePatches"
+    argspec: "args=[\'input\', \'ksizes\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FFT"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FFT2D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FFT3D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FIFOQueue"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FIFOQueueV2"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Fact"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeParam"
+    argspec: "args=[\'dtype\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeQuantWithMinMaxArgs"
+    argspec: "args=[\'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeQuantWithMinMaxArgsGradient"
+    argspec: "args=[\'gradients\', \'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeQuantWithMinMaxVars"
+    argspec: "args=[\'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeQuantWithMinMaxVarsGradient"
+    argspec: "args=[\'gradients\', \'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeQuantWithMinMaxVarsPerChannel"
+    argspec: "args=[\'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeQuantWithMinMaxVarsPerChannelGradient"
+    argspec: "args=[\'gradients\', \'inputs\', \'min\', \'max\', \'num_bits\', \'narrow_range\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FakeQueue"
+    argspec: "args=[\'resource\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Fill"
+    argspec: "args=[\'dims\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FilterByLastComponentDataset"
+    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FilterDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'predicate\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FixedLengthRecordDataset"
+    argspec: "args=[\'filenames\', \'header_bytes\', \'record_bytes\', \'footer_bytes\', \'buffer_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FixedLengthRecordDatasetV2"
+    argspec: "args=[\'filenames\', \'header_bytes\', \'record_bytes\', \'footer_bytes\', \'buffer_size\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FixedLengthRecordReader"
+    argspec: "args=[\'header_bytes\', \'record_bytes\', \'footer_bytes\', \'hop_bytes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FixedLengthRecordReaderV2"
+    argspec: "args=[\'header_bytes\', \'record_bytes\', \'footer_bytes\', \'hop_bytes\', \'container\', \'shared_name\', \'encoding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FixedUnigramCandidateSampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'vocab_file\', \'distortion\', \'num_reserved_ids\', \'num_shards\', \'shard\', \'unigrams\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FlatMapDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Floor"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FloorDiv"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FloorMod"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FlushSummaryWriter"
+    argspec: "args=[\'writer\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "For"
+    argspec: "args=[\'start\', \'limit\', \'delta\', \'input\', \'body\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FractionalAvgPool"
+    argspec: "args=[\'value\', \'pooling_ratio\', \'pseudo_random\', \'overlapping\', \'deterministic\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FractionalAvgPoolGrad"
+    argspec: "args=[\'orig_input_tensor_shape\', \'out_backprop\', \'row_pooling_sequence\', \'col_pooling_sequence\', \'overlapping\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FractionalMaxPool"
+    argspec: "args=[\'value\', \'pooling_ratio\', \'pseudo_random\', \'overlapping\', \'deterministic\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FractionalMaxPoolGrad"
+    argspec: "args=[\'orig_input\', \'orig_output\', \'out_backprop\', \'row_pooling_sequence\', \'col_pooling_sequence\', \'overlapping\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FusedBatchNorm"
+    argspec: "args=[\'x\', \'scale\', \'offset\', \'mean\', \'variance\', \'epsilon\', \'data_format\', \'is_training\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FusedBatchNormGrad"
+    argspec: "args=[\'y_backprop\', \'x\', \'scale\', \'reserve_space_1\', \'reserve_space_2\', \'epsilon\', \'data_format\', \'is_training\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FusedBatchNormGradV2"
+    argspec: "args=[\'y_backprop\', \'x\', \'scale\', \'reserve_space_1\', \'reserve_space_2\', \'epsilon\', \'data_format\', \'is_training\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FusedBatchNormV2"
+    argspec: "args=[\'x\', \'scale\', \'offset\', \'mean\', \'variance\', \'epsilon\', \'data_format\', \'is_training\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FusedPadConv2D"
+    argspec: "args=[\'input\', \'paddings\', \'filter\', \'mode\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "FusedResizeAndPadConv2D"
+    argspec: "args=[\'input\', \'size\', \'paddings\', \'filter\', \'resize_align_corners\', \'mode\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Gather"
+    argspec: "args=[\'params\', \'indices\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GatherNd"
+    argspec: "args=[\'params\', \'indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GatherV2"
+    argspec: "args=[\'params\', \'indices\', \'axis\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GenerateVocabRemapping"
+    argspec: "args=[\'new_vocab_file\', \'old_vocab_file\', \'new_vocab_offset\', \'num_new_vocab\', \'old_vocab_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GeneratorDataset"
+    argspec: "args=[\'init_func_other_args\', \'next_func_other_args\', \'finalize_func_other_args\', \'init_func\', \'next_func\', \'finalize_func\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GetSessionHandle"
+    argspec: "args=[\'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GetSessionHandleV2"
+    argspec: "args=[\'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GetSessionTensor"
+    argspec: "args=[\'handle\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Greater"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GreaterEqual"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "GuaranteeConst"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "HSVToRGB"
+    argspec: "args=[\'images\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "HashTable"
+    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "HashTableV2"
+    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "HistogramFixedWidth"
+    argspec: "args=[\'values\', \'value_range\', \'nbins\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "HistogramSummary"
+    argspec: "args=[\'tag\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IFFT"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IFFT2D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IFFT3D"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IRFFT"
+    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IRFFT2D"
+    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IRFFT3D"
+    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Identity"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IdentityN"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IdentityReader"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IdentityReaderV2"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "If"
+    argspec: "args=[\'cond\', \'input\', \'Tout\', \'then_branch\', \'else_branch\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Igamma"
+    argspec: "args=[\'a\', \'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IgammaGradA"
+    argspec: "args=[\'a\', \'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Igammac"
+    argspec: "args=[\'a\', \'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Imag"
+    argspec: "args=[\'input\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ImageSummary"
+    argspec: "args=[\'tag\', \'tensor\', \'max_images\', \'bad_color\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ImmutableConst"
+    argspec: "args=[\'dtype\', \'shape\', \'memory_region_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ImportEvent"
+    argspec: "args=[\'writer\', \'event\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InTopK"
+    argspec: "args=[\'predictions\', \'targets\', \'k\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InTopKV2"
+    argspec: "args=[\'predictions\', \'targets\', \'k\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InitializeTable"
+    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InitializeTableFromTextFile"
+    argspec: "args=[\'table_handle\', \'filename\', \'key_index\', \'value_index\', \'vocab_size\', \'delimiter\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InitializeTableFromTextFileV2"
+    argspec: "args=[\'table_handle\', \'filename\', \'key_index\', \'value_index\', \'vocab_size\', \'delimiter\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InitializeTableV2"
+    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InplaceAdd"
+    argspec: "args=[\'x\', \'i\', \'v\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InplaceSub"
+    argspec: "args=[\'x\', \'i\', \'v\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InplaceUpdate"
+    argspec: "args=[\'x\', \'i\', \'v\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InterleaveDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'cycle_length\', \'block_length\', \'f\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Inv"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InvGrad"
+    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Invert"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "InvertPermutation"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IsBoostedTreesEnsembleInitialized"
+    argspec: "args=[\'tree_ensemble_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IsBoostedTreesQuantileStreamResourceInitialized"
+    argspec: "args=[\'quantile_stream_resource_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IsFinite"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IsInf"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IsNan"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IsVariableInitialized"
+    argspec: "args=[\'ref\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Iterator"
+    argspec: "args=[\'shared_name\', \'container\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IteratorFromStringHandle"
+    argspec: "args=[\'string_handle\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IteratorFromStringHandleV2"
+    argspec: "args=[\'string_handle\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IteratorGetNext"
+    argspec: "args=[\'iterator\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IteratorGetNextAsOptional"
+    argspec: "args=[\'iterator\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IteratorGetNextSync"
+    argspec: "args=[\'iterator\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IteratorToStringHandle"
+    argspec: "args=[\'resource_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "IteratorV2"
+    argspec: "args=[\'shared_name\', \'container\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "L2Loss"
+    argspec: "args=[\'t\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LMDBReader"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LRN"
+    argspec: "args=[\'input\', \'depth_radius\', \'bias\', \'alpha\', \'beta\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LRNGrad"
+    argspec: "args=[\'input_grads\', \'input_image\', \'output_image\', \'depth_radius\', \'bias\', \'alpha\', \'beta\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LeakyRelu"
+    argspec: "args=[\'features\', \'alpha\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LeakyReluGrad"
+    argspec: "args=[\'gradients\', \'features\', \'alpha\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LearnedUnigramCandidateSampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LeftShift"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Less"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LessEqual"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Lgamma"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LinSpace"
+    argspec: "args=[\'start\', \'stop\', \'num\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ListDiff"
+    argspec: "args=[\'x\', \'y\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LoadAndRemapMatrix"
+    argspec: "args=[\'ckpt_path\', \'old_tensor_name\', \'row_remapping\', \'col_remapping\', \'initializing_values\', \'num_rows\', \'num_cols\', \'max_rows_in_memory\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Log"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Log1p"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LogMatrixDeterminant"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LogSoftmax"
+    argspec: "args=[\'logits\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LogUniformCandidateSampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LogicalAnd"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LogicalNot"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LogicalOr"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableExport"
+    argspec: "args=[\'table_handle\', \'Tkeys\', \'Tvalues\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableExportV2"
+    argspec: "args=[\'table_handle\', \'Tkeys\', \'Tvalues\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableFind"
+    argspec: "args=[\'table_handle\', \'keys\', \'default_value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableFindV2"
+    argspec: "args=[\'table_handle\', \'keys\', \'default_value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableImport"
+    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableImportV2"
+    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableInsert"
+    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableInsertV2"
+    argspec: "args=[\'table_handle\', \'keys\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableRemoveV2"
+    argspec: "args=[\'table_handle\', \'keys\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableSize"
+    argspec: "args=[\'table_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LookupTableSizeV2"
+    argspec: "args=[\'table_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LoopCond"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "LowerBound"
+    argspec: "args=[\'sorted_inputs\', \'values\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Lu"
+    argspec: "args=[\'input\', \'output_idx_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MakeIterator"
+    argspec: "args=[\'dataset\', \'iterator\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapClear"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\', \'use_inter_op_parallelism\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapDefun"
+    argspec: "args=[\'arguments\', \'captured_inputs\', \'output_types\', \'output_shapes\', \'f\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapIncompleteSize"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapPeek"
+    argspec: "args=[\'key\', \'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapSize"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapStage"
+    argspec: "args=[\'key\', \'indices\', \'values\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapUnstage"
+    argspec: "args=[\'key\', \'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MapUnstageNoKey"
+    argspec: "args=[\'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatMul"
+    argspec: "args=[\'a\', \'b\', \'transpose_a\', \'transpose_b\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatchingFiles"
+    argspec: "args=[\'pattern\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixBandPart"
+    argspec: "args=[\'input\', \'num_lower\', \'num_upper\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixDeterminant"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixDiag"
+    argspec: "args=[\'diagonal\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixDiagPart"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixExponential"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixInverse"
+    argspec: "args=[\'input\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixLogarithm"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixSetDiag"
+    argspec: "args=[\'input\', \'diagonal\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixSolve"
+    argspec: "args=[\'matrix\', \'rhs\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixSolveLs"
+    argspec: "args=[\'matrix\', \'rhs\', \'l2_regularizer\', \'fast\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixSquareRoot"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MatrixTriangularSolve"
+    argspec: "args=[\'matrix\', \'rhs\', \'lower\', \'adjoint\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Max"
+    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPool"
+    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPool3D"
+    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPool3DGrad"
+    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPool3DGradGrad"
+    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolGrad"
+    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolGradGrad"
+    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolGradGradV2"
+    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolGradGradWithArgmax"
+    argspec: "args=[\'input\', \'grad\', \'argmax\', \'ksize\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolGradV2"
+    argspec: "args=[\'orig_input\', \'orig_output\', \'grad\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolGradWithArgmax"
+    argspec: "args=[\'input\', \'grad\', \'argmax\', \'ksize\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolV2"
+    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MaxPoolWithArgmax"
+    argspec: "args=[\'input\', \'ksize\', \'strides\', \'Targmax\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Maximum"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Mean"
+    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Merge"
+    argspec: "args=[\'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MergeSummary"
+    argspec: "args=[\'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MergeV2Checkpoints"
+    argspec: "args=[\'checkpoint_prefixes\', \'destination_prefix\', \'delete_old_dirs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Mfcc"
+    argspec: "args=[\'spectrogram\', \'sample_rate\', \'upper_frequency_limit\', \'lower_frequency_limit\', \'filterbank_channel_count\', \'dct_coefficient_count\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Min"
+    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Minimum"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MirrorPad"
+    argspec: "args=[\'input\', \'paddings\', \'mode\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MirrorPadGrad"
+    argspec: "args=[\'input\', \'paddings\', \'mode\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Mod"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ModelDataset"
+    argspec: "args=[\'input_dataset\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Mul"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MultiDeviceIterator"
+    argspec: "args=[\'devices\', \'shared_name\', \'container\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MultiDeviceIteratorFromStringHandle"
+    argspec: "args=[\'string_handle\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MultiDeviceIteratorGetNextFromShard"
+    argspec: "args=[\'multi_device_iterator\', \'shard_num\', \'incarnation_id\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MultiDeviceIteratorInit"
+    argspec: "args=[\'dataset\', \'multi_device_iterator\', \'max_buffer_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MultiDeviceIteratorToStringHandle"
+    argspec: "args=[\'multi_device_iterator\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Multinomial"
+    argspec: "args=[\'logits\', \'num_samples\', \'seed\', \'seed2\', \'output_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutableDenseHashTable"
+    argspec: "args=[\'empty_key\', \'container\', \'shared_name\', \'use_node_name_sharing\', \'value_dtype\', \'value_shape\', \'initial_num_buckets\', \'max_load_factor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutableDenseHashTableV2"
+    argspec: "args=[\'empty_key\', \'deleted_key\', \'container\', \'shared_name\', \'use_node_name_sharing\', \'value_dtype\', \'value_shape\', \'initial_num_buckets\', \'max_load_factor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutableHashTable"
+    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutableHashTableOfTensors"
+    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\', \'value_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutableHashTableOfTensorsV2"
+    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\', \'value_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutableHashTableV2"
+    argspec: "args=[\'container\', \'shared_name\', \'use_node_name_sharing\', \'key_dtype\', \'value_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutexLock"
+    argspec: "args=[\'mutex\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MutexV2"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NcclAllReduce"
+    argspec: "args=[\'input\', \'reduction\', \'num_devices\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NcclBroadcast"
+    argspec: "args=[\'input\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NcclReduce"
+    argspec: "args=[\'input\', \'reduction\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Neg"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NextAfter"
+    argspec: "args=[\'x1\', \'x2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NextIteration"
+    argspec: "args=[\'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NoOp"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NonMaxSuppression"
+    argspec: "args=[\'boxes\', \'scores\', \'max_output_size\', \'iou_threshold\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NonMaxSuppressionV2"
+    argspec: "args=[\'boxes\', \'scores\', \'max_output_size\', \'iou_threshold\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NonMaxSuppressionV3"
+    argspec: "args=[\'boxes\', \'scores\', \'max_output_size\', \'iou_threshold\', \'score_threshold\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NonMaxSuppressionV4"
+    argspec: "args=[\'boxes\', \'scores\', \'max_output_size\', \'iou_threshold\', \'score_threshold\', \'pad_to_max_output_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NonMaxSuppressionWithOverlaps"
+    argspec: "args=[\'overlaps\', \'scores\', \'max_output_size\', \'overlap_threshold\', \'score_threshold\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NotEqual"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "NthElement"
+    argspec: "args=[\'input\', \'n\', \'reverse\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OneHot"
+    argspec: "args=[\'indices\', \'depth\', \'on_value\', \'off_value\', \'axis\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OneShotIterator"
+    argspec: "args=[\'dataset_factory\', \'output_types\', \'output_shapes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OnesLike"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OptimizeDataset"
+    argspec: "args=[\'input_dataset\', \'optimizations\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OptionalFromValue"
+    argspec: "args=[\'components\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OptionalGetValue"
+    argspec: "args=[\'optional\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OptionalHasValue"
+    argspec: "args=[\'optional\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OptionalNone"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OrderedMapClear"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OrderedMapIncompleteSize"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OrderedMapPeek"
+    argspec: "args=[\'key\', \'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OrderedMapSize"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OrderedMapStage"
+    argspec: "args=[\'key\', \'indices\', \'values\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OrderedMapUnstage"
+    argspec: "args=[\'key\', \'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "OrderedMapUnstageNoKey"
+    argspec: "args=[\'indices\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Pack"
+    argspec: "args=[\'values\', \'axis\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Pad"
+    argspec: "args=[\'input\', \'paddings\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PadV2"
+    argspec: "args=[\'input\', \'paddings\', \'constant_values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PaddedBatchDataset"
+    argspec: "args=[\'input_dataset\', \'batch_size\', \'padded_shapes\', \'padding_values\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PaddedBatchDatasetV2"
+    argspec: "args=[\'input_dataset\', \'batch_size\', \'padded_shapes\', \'padding_values\', \'drop_remainder\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PaddingFIFOQueue"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PaddingFIFOQueueV2"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParallelConcat"
+    argspec: "args=[\'values\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParallelDynamicStitch"
+    argspec: "args=[\'indices\', \'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParallelInterleaveDatasetV2"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'cycle_length\', \'block_length\', \'num_parallel_calls\', \'f\', \'output_types\', \'output_shapes\', \'sloppy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParallelMapDataset"
+    argspec: "args=[\'input_dataset\', \'other_arguments\', \'num_parallel_calls\', \'f\', \'output_types\', \'output_shapes\', \'use_inter_op_parallelism\', \'sloppy\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParameterizedTruncatedNormal"
+    argspec: "args=[\'shape\', \'means\', \'stdevs\', \'minvals\', \'maxvals\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParseExample"
+    argspec: "args=[\'serialized\', \'names\', \'sparse_keys\', \'dense_keys\', \'dense_defaults\', \'sparse_types\', \'dense_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParseSequenceExample"
+    argspec: "args=[\'serialized\', \'debug_name\', \'context_dense_defaults\', \'feature_list_dense_missing_assumed_empty\', \'context_sparse_keys\', \'context_dense_keys\', \'feature_list_sparse_keys\', \'feature_list_dense_keys\', \'Ncontext_sparse\', \'Ncontext_dense\', \'Nfeature_list_sparse\', \'Nfeature_list_dense\', \'context_sparse_types\', \'feature_list_dense_types\', \'context_dense_shapes\', \'feature_list_sparse_types\', \'feature_list_dense_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParseSingleExample"
+    argspec: "args=[\'serialized\', \'dense_defaults\', \'num_sparse\', \'sparse_keys\', \'dense_keys\', \'sparse_types\', \'dense_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParseSingleSequenceExample"
+    argspec: "args=[\'serialized\', \'feature_list_dense_missing_assumed_empty\', \'context_sparse_keys\', \'context_dense_keys\', \'feature_list_sparse_keys\', \'feature_list_dense_keys\', \'context_dense_defaults\', \'debug_name\', \'context_sparse_types\', \'feature_list_dense_types\', \'context_dense_shapes\', \'feature_list_sparse_types\', \'feature_list_dense_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ParseTensor"
+    argspec: "args=[\'serialized\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PartitionedCall"
+    argspec: "args=[\'args\', \'Tout\', \'f\', \'config\', \'config_proto\', \'executor_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Placeholder"
+    argspec: "args=[\'dtype\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PlaceholderV2"
+    argspec: "args=[\'dtype\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PlaceholderWithDefault"
+    argspec: "args=[\'input\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Polygamma"
+    argspec: "args=[\'a\', \'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PopulationCount"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Pow"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PrefetchDataset"
+    argspec: "args=[\'input_dataset\', \'buffer_size\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PreventGradient"
+    argspec: "args=[\'input\', \'message\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Print"
+    argspec: "args=[\'input\', \'data\', \'message\', \'first_n\', \'summarize\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PrintV2"
+    argspec: "args=[\'input\', \'output_stream\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PriorityQueue"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PriorityQueueV2"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Prod"
+    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PyFunc"
+    argspec: "args=[\'input\', \'token\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "PyFuncStateless"
+    argspec: "args=[\'input\', \'token\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Qr"
+    argspec: "args=[\'input\', \'full_matrices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizeAndDequantize"
+    argspec: "args=[\'input\', \'signed_input\', \'num_bits\', \'range_given\', \'input_min\', \'input_max\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizeAndDequantizeV2"
+    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'signed_input\', \'num_bits\', \'range_given\', \'round_mode\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizeAndDequantizeV3"
+    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'num_bits\', \'signed_input\', \'range_given\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizeDownAndShrinkRange"
+    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizeV2"
+    argspec: "args=[\'input\', \'min_range\', \'max_range\', \'T\', \'mode\', \'round_mode\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedAdd"
+    argspec: "args=[\'x\', \'y\', \'min_x\', \'max_x\', \'min_y\', \'max_y\', \'Toutput\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedAvgPool"
+    argspec: "args=[\'input\', \'min_input\', \'max_input\', \'ksize\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedBatchNormWithGlobalNormalization"
+    argspec: "args=[\'t\', \'t_min\', \'t_max\', \'m\', \'m_min\', \'m_max\', \'v\', \'v_min\', \'v_max\', \'beta\', \'beta_min\', \'beta_max\', \'gamma\', \'gamma_min\', \'gamma_max\', \'out_type\', \'variance_epsilon\', \'scale_after_normalization\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedBiasAdd"
+    argspec: "args=[\'input\', \'bias\', \'min_input\', \'max_input\', \'min_bias\', \'max_bias\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConcat"
+    argspec: "args=[\'concat_dim\', \'values\', \'input_mins\', \'input_maxes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2D"
+    argspec: "args=[\'input\', \'filter\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DAndRelu"
+    argspec: "args=[\'input\', \'filter\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DAndReluAndRequantize"
+    argspec: "args=[\'input\', \'filter\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DAndRequantize"
+    argspec: "args=[\'input\', \'filter\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DWithBias"
+    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DWithBiasAndRelu"
+    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DWithBiasAndReluAndRequantize"
+    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DWithBiasAndRequantize"
+    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DWithBiasSignedSumAndReluAndRequantize"
+    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'summand\', \'min_summand\', \'max_summand\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DWithBiasSumAndRelu"
+    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'summand\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedConv2DWithBiasSumAndReluAndRequantize"
+    argspec: "args=[\'input\', \'filter\', \'bias\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'min_freezed_output\', \'max_freezed_output\', \'summand\', \'min_summand\', \'max_summand\', \'out_type\', \'strides\', \'padding\', \'dilations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedInstanceNorm"
+    argspec: "args=[\'x\', \'x_min\', \'x_max\', \'output_range_given\', \'given_y_min\', \'given_y_max\', \'variance_epsilon\', \'min_separation\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedMatMul"
+    argspec: "args=[\'a\', \'b\', \'min_a\', \'max_a\', \'min_b\', \'max_b\', \'Toutput\', \'transpose_a\', \'transpose_b\', \'Tactivation\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedMaxPool"
+    argspec: "args=[\'input\', \'min_input\', \'max_input\', \'ksize\', \'strides\', \'padding\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedMul"
+    argspec: "args=[\'x\', \'y\', \'min_x\', \'max_x\', \'min_y\', \'max_y\', \'Toutput\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedRelu"
+    argspec: "args=[\'features\', \'min_features\', \'max_features\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedRelu6"
+    argspec: "args=[\'features\', \'min_features\', \'max_features\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedReluX"
+    argspec: "args=[\'features\', \'max_value\', \'min_features\', \'max_features\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedReshape"
+    argspec: "args=[\'tensor\', \'shape\', \'input_min\', \'input_max\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QuantizedResizeBilinear"
+    argspec: "args=[\'images\', \'size\', \'min\', \'max\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueClose"
+    argspec: "args=[\'handle\', \'cancel_pending_enqueues\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueCloseV2"
+    argspec: "args=[\'handle\', \'cancel_pending_enqueues\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueDequeue"
+    argspec: "args=[\'handle\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueDequeueMany"
+    argspec: "args=[\'handle\', \'n\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueDequeueManyV2"
+    argspec: "args=[\'handle\', \'n\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueDequeueUpTo"
+    argspec: "args=[\'handle\', \'n\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueDequeueUpToV2"
+    argspec: "args=[\'handle\', \'n\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueDequeueV2"
+    argspec: "args=[\'handle\', \'component_types\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueEnqueue"
+    argspec: "args=[\'handle\', \'components\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueEnqueueMany"
+    argspec: "args=[\'handle\', \'components\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueEnqueueManyV2"
+    argspec: "args=[\'handle\', \'components\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueEnqueueV2"
+    argspec: "args=[\'handle\', \'components\', \'timeout_ms\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueIsClosed"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueIsClosedV2"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueSize"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "QueueSizeV2"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RFFT"
+    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RFFT2D"
+    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RFFT3D"
+    argspec: "args=[\'input\', \'fft_length\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RGBToHSV"
+    argspec: "args=[\'images\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RaggedGather"
+    argspec: "args=[\'params_nested_splits\', \'params_dense_values\', \'indices\', \'OUTPUT_RAGGED_RANK\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RaggedRange"
+    argspec: "args=[\'starts\', \'limits\', \'deltas\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RaggedTensorToSparse"
+    argspec: "args=[\'rt_nested_splits\', \'rt_dense_values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomCrop"
+    argspec: "args=[\'image\', \'size\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomGamma"
+    argspec: "args=[\'shape\', \'alpha\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomGammaGrad"
+    argspec: "args=[\'alpha\', \'sample\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomPoisson"
+    argspec: "args=[\'shape\', \'rate\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomPoissonV2"
+    argspec: "args=[\'shape\', \'rate\', \'seed\', \'seed2\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomShuffle"
+    argspec: "args=[\'value\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomShuffleQueue"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'min_after_dequeue\', \'seed\', \'seed2\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomShuffleQueueV2"
+    argspec: "args=[\'component_types\', \'shapes\', \'capacity\', \'min_after_dequeue\', \'seed\', \'seed2\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomStandardNormal"
+    argspec: "args=[\'shape\', \'seed\', \'seed2\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomUniform"
+    argspec: "args=[\'shape\', \'seed\', \'seed2\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RandomUniformInt"
+    argspec: "args=[\'shape\', \'minval\', \'maxval\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Range"
+    argspec: "args=[\'start\', \'limit\', \'delta\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RangeDataset"
+    argspec: "args=[\'start\', \'stop\', \'step\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Rank"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReadFile"
+    argspec: "args=[\'filename\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReadVariableOp"
+    argspec: "args=[\'resource\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderNumRecordsProduced"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderNumRecordsProducedV2"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderNumWorkUnitsCompleted"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderNumWorkUnitsCompletedV2"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderRead"
+    argspec: "args=[\'reader_handle\', \'queue_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderReadUpTo"
+    argspec: "args=[\'reader_handle\', \'queue_handle\', \'num_records\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderReadUpToV2"
+    argspec: "args=[\'reader_handle\', \'queue_handle\', \'num_records\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderReadV2"
+    argspec: "args=[\'reader_handle\', \'queue_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderReset"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderResetV2"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderRestoreState"
+    argspec: "args=[\'reader_handle\', \'state\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderRestoreStateV2"
+    argspec: "args=[\'reader_handle\', \'state\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderSerializeState"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReaderSerializeStateV2"
+    argspec: "args=[\'reader_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Real"
+    argspec: "args=[\'input\', \'Tout\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RealDiv"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Reciprocal"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReciprocalGrad"
+    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RecordInput"
+    argspec: "args=[\'file_pattern\', \'file_random_seed\', \'file_shuffle_shift_ratio\', \'file_buffer_size\', \'file_parallelism\', \'batch_size\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReduceDataset"
+    argspec: "args=[\'input_dataset\', \'initial_state\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\', \'use_inter_op_parallelism\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReduceJoin"
+    argspec: "args=[\'inputs\', \'reduction_indices\', \'keep_dims\', \'separator\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RefEnter"
+    argspec: "args=[\'data\', \'frame_name\', \'is_constant\', \'parallel_iterations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RefExit"
+    argspec: "args=[\'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RefIdentity"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RefMerge"
+    argspec: "args=[\'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RefNextIteration"
+    argspec: "args=[\'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RefSelect"
+    argspec: "args=[\'index\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RefSwitch"
+    argspec: "args=[\'data\', \'pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RegexFullMatch"
+    argspec: "args=[\'input\', \'pattern\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RegexReplace"
+    argspec: "args=[\'input\', \'pattern\', \'rewrite\', \'replace_global\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Relu"
+    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Relu6"
+    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Relu6Grad"
+    argspec: "args=[\'gradients\', \'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReluGrad"
+    argspec: "args=[\'gradients\', \'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RemoteCall"
+    argspec: "args=[\'target\', \'args\', \'Tout\', \'f\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RepeatDataset"
+    argspec: "args=[\'input_dataset\', \'count\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RequantizationRange"
+    argspec: "args=[\'input\', \'input_min\', \'input_max\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Requantize"
+    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'requested_output_min\', \'requested_output_max\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Reshape"
+    argspec: "args=[\'tensor\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResizeArea"
+    argspec: "args=[\'images\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResizeBicubic"
+    argspec: "args=[\'images\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResizeBicubicGrad"
+    argspec: "args=[\'grads\', \'original_image\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResizeBilinear"
+    argspec: "args=[\'images\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResizeBilinearGrad"
+    argspec: "args=[\'grads\', \'original_image\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResizeNearestNeighbor"
+    argspec: "args=[\'images\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResizeNearestNeighborGrad"
+    argspec: "args=[\'grads\', \'size\', \'align_corners\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyAdaMax"
+    argspec: "args=[\'var\', \'m\', \'v\', \'beta1_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyAdadelta"
+    argspec: "args=[\'var\', \'accum\', \'accum_update\', \'lr\', \'rho\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'use_locking\', \'update_slots\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyAdagradDA"
+    argspec: "args=[\'var\', \'gradient_accumulator\', \'gradient_squared_accumulator\', \'grad\', \'lr\', \'l1\', \'l2\', \'global_step\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyAdam"
+    argspec: "args=[\'var\', \'m\', \'v\', \'beta1_power\', \'beta2_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyAdamWithAmsgrad"
+    argspec: "args=[\'var\', \'m\', \'v\', \'vhat\', \'beta1_power\', \'beta2_power\', \'lr\', \'beta1\', \'beta2\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyAddSign"
+    argspec: "args=[\'var\', \'m\', \'lr\', \'alpha\', \'sign_decay\', \'beta\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyCenteredRMSProp"
+    argspec: "args=[\'var\', \'mg\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyFtrl"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'lr\', \'l1\', \'l2\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyFtrlV2"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'lr\', \'l1\', \'l2\', \'l2_shrinkage\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyGradientDescent"
+    argspec: "args=[\'var\', \'alpha\', \'delta\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyKerasMomentum"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyMomentum"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyPowerSign"
+    argspec: "args=[\'var\', \'m\', \'lr\', \'logbase\', \'sign_decay\', \'beta\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyProximalAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'l1\', \'l2\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyProximalGradientDescent"
+    argspec: "args=[\'var\', \'alpha\', \'l1\', \'l2\', \'delta\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceApplyRMSProp"
+    argspec: "args=[\'var\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceCountUpTo"
+    argspec: "args=[\'resource\', \'limit\', \'T\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceGather"
+    argspec: "args=[\'resource\', \'indices\', \'validate_indices\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterAdd"
+    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterDiv"
+    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterMax"
+    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterMin"
+    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterMul"
+    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterNdAdd"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterNdSub"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterNdUpdate"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterSub"
+    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceScatterUpdate"
+    argspec: "args=[\'resource\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyAdadelta"
+    argspec: "args=[\'var\', \'accum\', \'accum_update\', \'lr\', \'rho\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'use_locking\', \'update_slots\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyAdagradDA"
+    argspec: "args=[\'var\', \'gradient_accumulator\', \'gradient_squared_accumulator\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'global_step\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyCenteredRMSProp"
+    argspec: "args=[\'var\', \'mg\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyFtrl"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyFtrlV2"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'l2_shrinkage\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyKerasMomentum"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyMomentum"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyProximalAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'l1\', \'l2\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyProximalGradientDescent"
+    argspec: "args=[\'var\', \'alpha\', \'l1\', \'l2\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceSparseApplyRMSProp"
+    argspec: "args=[\'var\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ResourceStridedSliceAssign"
+    argspec: "args=[\'ref\', \'begin\', \'end\', \'strides\', \'value\', \'begin_mask\', \'end_mask\', \'ellipsis_mask\', \'new_axis_mask\', \'shrink_axis_mask\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Restore"
+    argspec: "args=[\'file_pattern\', \'tensor_name\', \'dt\', \'preferred_shard\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RestoreSlice"
+    argspec: "args=[\'file_pattern\', \'tensor_name\', \'shape_and_slice\', \'dt\', \'preferred_shard\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RestoreV2"
+    argspec: "args=[\'prefix\', \'tensor_names\', \'shape_and_slices\', \'dtypes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Reverse"
+    argspec: "args=[\'tensor\', \'dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReverseSequence"
+    argspec: "args=[\'input\', \'seq_lengths\', \'seq_dim\', \'batch_dim\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ReverseV2"
+    argspec: "args=[\'tensor\', \'axis\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RightShift"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Rint"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Roll"
+    argspec: "args=[\'input\', \'shift\', \'axis\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Round"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Rsqrt"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "RsqrtGrad"
+    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SampleDistortedBoundingBox"
+    argspec: "args=[\'image_size\', \'bounding_boxes\', \'seed\', \'seed2\', \'min_object_covered\', \'aspect_ratio_range\', \'area_range\', \'max_attempts\', \'use_image_if_no_bounding_boxes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SampleDistortedBoundingBoxV2"
+    argspec: "args=[\'image_size\', \'bounding_boxes\', \'min_object_covered\', \'seed\', \'seed2\', \'aspect_ratio_range\', \'area_range\', \'max_attempts\', \'use_image_if_no_bounding_boxes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Save"
+    argspec: "args=[\'filename\', \'tensor_names\', \'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SaveSlices"
+    argspec: "args=[\'filename\', \'tensor_names\', \'shapes_and_slices\', \'data\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SaveV2"
+    argspec: "args=[\'prefix\', \'tensor_names\', \'shape_and_slices\', \'tensors\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScalarSummary"
+    argspec: "args=[\'tags\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScaleAndTranslate"
+    argspec: "args=[\'images\', \'size\', \'scale\', \'translation\', \'kernel_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScaleAndTranslateGrad"
+    argspec: "args=[\'grads\', \'original_image\', \'scale\', \'translation\', \'kernel_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterAdd"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterDiv"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterMax"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterMin"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterMul"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterNd"
+    argspec: "args=[\'indices\', \'updates\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterNdAdd"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterNdNonAliasingAdd"
+    argspec: "args=[\'input\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterNdSub"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterNdUpdate"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterSub"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ScatterUpdate"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SdcaFprint"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SdcaOptimizer"
+    argspec: "args=[\'sparse_example_indices\', \'sparse_feature_indices\', \'sparse_feature_values\', \'dense_features\', \'example_weights\', \'example_labels\', \'sparse_indices\', \'sparse_weights\', \'dense_weights\', \'example_state_data\', \'loss_type\', \'adaptative\', \'l1\', \'l2\', \'num_loss_partitions\', \'num_inner_iterations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SdcaOptimizerV2"
+    argspec: "args=[\'sparse_example_indices\', \'sparse_feature_indices\', \'sparse_feature_values\', \'dense_features\', \'example_weights\', \'example_labels\', \'sparse_indices\', \'sparse_weights\', \'dense_weights\', \'example_state_data\', \'loss_type\', \'adaptive\', \'l1\', \'l2\', \'num_loss_partitions\', \'num_inner_iterations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SdcaShrinkL1"
+    argspec: "args=[\'weights\', \'l1\', \'l2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SegmentMax"
+    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SegmentMean"
+    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SegmentMin"
+    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SegmentProd"
+    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SegmentSum"
+    argspec: "args=[\'data\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Select"
+    argspec: "args=[\'condition\', \'t\', \'e\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SelfAdjointEig"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SelfAdjointEigV2"
+    argspec: "args=[\'input\', \'compute_v\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Selu"
+    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SeluGrad"
+    argspec: "args=[\'gradients\', \'outputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SerializeIterator"
+    argspec: "args=[\'resource_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SerializeManySparse"
+    argspec: "args=[\'sparse_indices\', \'sparse_values\', \'sparse_shape\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SerializeSparse"
+    argspec: "args=[\'sparse_indices\', \'sparse_values\', \'sparse_shape\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SerializeTensor"
+    argspec: "args=[\'tensor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SetSize"
+    argspec: "args=[\'set_indices\', \'set_values\', \'set_shape\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Shape"
+    argspec: "args=[\'input\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ShapeN"
+    argspec: "args=[\'input\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ShardedFilename"
+    argspec: "args=[\'basename\', \'shard\', \'num_shards\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ShardedFilespec"
+    argspec: "args=[\'basename\', \'num_shards\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ShuffleAndRepeatDataset"
+    argspec: "args=[\'input_dataset\', \'buffer_size\', \'seed\', \'seed2\', \'count\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ShuffleDataset"
+    argspec: "args=[\'input_dataset\', \'buffer_size\', \'seed\', \'seed2\', \'reshuffle_each_iteration\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Sigmoid"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SigmoidGrad"
+    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Sign"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Sin"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Sinh"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Size"
+    argspec: "args=[\'input\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SkipDataset"
+    argspec: "args=[\'input_dataset\', \'count\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Slice"
+    argspec: "args=[\'input\', \'begin\', \'size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Snapshot"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Softmax"
+    argspec: "args=[\'logits\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SoftmaxCrossEntropyWithLogits"
+    argspec: "args=[\'features\', \'labels\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Softplus"
+    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SoftplusGrad"
+    argspec: "args=[\'gradients\', \'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Softsign"
+    argspec: "args=[\'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SoftsignGrad"
+    argspec: "args=[\'gradients\', \'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SpaceToBatch"
+    argspec: "args=[\'input\', \'paddings\', \'block_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SpaceToBatchND"
+    argspec: "args=[\'input\', \'block_shape\', \'paddings\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SpaceToDepth"
+    argspec: "args=[\'input\', \'block_size\', \'data_format\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseAccumulatorApplyGradient"
+    argspec: "args=[\'handle\', \'local_step\', \'gradient_indices\', \'gradient_values\', \'gradient_shape\', \'has_known_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseAccumulatorTakeGradient"
+    argspec: "args=[\'handle\', \'num_required\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseAdd"
+    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b_indices\', \'b_values\', \'b_shape\', \'thresh\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseAddGrad"
+    argspec: "args=[\'backprop_val_grad\', \'a_indices\', \'b_indices\', \'sum_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyAdadelta"
+    argspec: "args=[\'var\', \'accum\', \'accum_update\', \'lr\', \'rho\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'use_locking\', \'update_slots\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyAdagradDA"
+    argspec: "args=[\'var\', \'gradient_accumulator\', \'gradient_squared_accumulator\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'global_step\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyCenteredRMSProp"
+    argspec: "args=[\'var\', \'mg\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyFtrl"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyFtrlV2"
+    argspec: "args=[\'var\', \'accum\', \'linear\', \'grad\', \'indices\', \'lr\', \'l1\', \'l2\', \'l2_shrinkage\', \'lr_power\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyMomentum"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'grad\', \'indices\', \'momentum\', \'use_locking\', \'use_nesterov\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyProximalAdagrad"
+    argspec: "args=[\'var\', \'accum\', \'lr\', \'l1\', \'l2\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyProximalGradientDescent"
+    argspec: "args=[\'var\', \'alpha\', \'l1\', \'l2\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseApplyRMSProp"
+    argspec: "args=[\'var\', \'ms\', \'mom\', \'lr\', \'rho\', \'momentum\', \'epsilon\', \'grad\', \'indices\', \'use_locking\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseConcat"
+    argspec: "args=[\'indices\', \'values\', \'shapes\', \'concat_dim\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseConditionalAccumulator"
+    argspec: "args=[\'dtype\', \'shape\', \'container\', \'shared_name\', \'reduction_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseCross"
+    argspec: "args=[\'indices\', \'values\', \'shapes\', \'dense_inputs\', \'hashed_output\', \'num_buckets\', \'hash_key\', \'out_type\', \'internal_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseDenseCwiseAdd"
+    argspec: "args=[\'sp_indices\', \'sp_values\', \'sp_shape\', \'dense\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseDenseCwiseDiv"
+    argspec: "args=[\'sp_indices\', \'sp_values\', \'sp_shape\', \'dense\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseDenseCwiseMul"
+    argspec: "args=[\'sp_indices\', \'sp_values\', \'sp_shape\', \'dense\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseFillEmptyRows"
+    argspec: "args=[\'indices\', \'values\', \'dense_shape\', \'default_value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseFillEmptyRowsGrad"
+    argspec: "args=[\'reverse_index_map\', \'grad_values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseMatMul"
+    argspec: "args=[\'a\', \'b\', \'transpose_a\', \'transpose_b\', \'a_is_sparse\', \'b_is_sparse\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseReduceMax"
+    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseReduceMaxSparse"
+    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseReduceSum"
+    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseReduceSumSparse"
+    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseReorder"
+    argspec: "args=[\'input_indices\', \'input_values\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseReshape"
+    argspec: "args=[\'input_indices\', \'input_shape\', \'new_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentMean"
+    argspec: "args=[\'data\', \'indices\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentMeanGrad"
+    argspec: "args=[\'grad\', \'indices\', \'segment_ids\', \'output_dim0\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentMeanWithNumSegments"
+    argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentSqrtN"
+    argspec: "args=[\'data\', \'indices\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentSqrtNGrad"
+    argspec: "args=[\'grad\', \'indices\', \'segment_ids\', \'output_dim0\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentSqrtNWithNumSegments"
+    argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentSum"
+    argspec: "args=[\'data\', \'indices\', \'segment_ids\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSegmentSumWithNumSegments"
+    argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSlice"
+    argspec: "args=[\'indices\', \'values\', \'shape\', \'start\', \'size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSliceGrad"
+    argspec: "args=[\'backprop_val_grad\', \'input_indices\', \'input_start\', \'output_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSoftmax"
+    argspec: "args=[\'sp_indices\', \'sp_values\', \'sp_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSoftmaxCrossEntropyWithLogits"
+    argspec: "args=[\'features\', \'labels\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSparseMaximum"
+    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b_indices\', \'b_values\', \'b_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSparseMinimum"
+    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b_indices\', \'b_values\', \'b_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseSplit"
+    argspec: "args=[\'split_dim\', \'indices\', \'values\', \'shape\', \'num_split\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseTensorDenseAdd"
+    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseTensorDenseMatMul"
+    argspec: "args=[\'a_indices\', \'a_values\', \'a_shape\', \'b\', \'adjoint_a\', \'adjoint_b\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseTensorSliceDataset"
+    argspec: "args=[\'indices\', \'values\', \'dense_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseToDense"
+    argspec: "args=[\'sparse_indices\', \'output_shape\', \'sparse_values\', \'default_value\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SparseToSparseSetOperation"
+    argspec: "args=[\'set1_indices\', \'set1_values\', \'set1_shape\', \'set2_indices\', \'set2_values\', \'set2_shape\', \'set_operation\', \'validate_indices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Split"
+    argspec: "args=[\'split_dim\', \'value\', \'num_split\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SplitV"
+    argspec: "args=[\'value\', \'size_splits\', \'split_dim\', \'num_split\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Sqrt"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SqrtGrad"
+    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Square"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SquaredDifference"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Squeeze"
+    argspec: "args=[\'input\', \'squeeze_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Stack"
+    argspec: "args=[\'elem_type\', \'stack_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StackClose"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StackCloseV2"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StackPop"
+    argspec: "args=[\'handle\', \'elem_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StackPopV2"
+    argspec: "args=[\'handle\', \'elem_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StackPush"
+    argspec: "args=[\'handle\', \'elem\', \'swap_memory\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StackPushV2"
+    argspec: "args=[\'handle\', \'elem\', \'swap_memory\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StackV2"
+    argspec: "args=[\'max_size\', \'elem_type\', \'stack_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Stage"
+    argspec: "args=[\'values\', \'capacity\', \'memory_limit\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StageClear"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StagePeek"
+    argspec: "args=[\'index\', \'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StageSize"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatefulPartitionedCall"
+    argspec: "args=[\'args\', \'Tout\', \'f\', \'config\', \'config_proto\', \'executor_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatelessIf"
+    argspec: "args=[\'cond\', \'input\', \'Tout\', \'then_branch\', \'else_branch\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatelessMultinomial"
+    argspec: "args=[\'logits\', \'num_samples\', \'seed\', \'output_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatelessRandomNormal"
+    argspec: "args=[\'shape\', \'seed\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatelessRandomUniform"
+    argspec: "args=[\'shape\', \'seed\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatelessRandomUniformInt"
+    argspec: "args=[\'shape\', \'seed\', \'minval\', \'maxval\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatelessTruncatedNormal"
+    argspec: "args=[\'shape\', \'seed\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StatelessWhile"
+    argspec: "args=[\'input\', \'cond\', \'body\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StaticRegexFullMatch"
+    argspec: "args=[\'input\', \'pattern\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StaticRegexReplace"
+    argspec: "args=[\'input\', \'pattern\', \'rewrite\', \'replace_global\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StopGradient"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StridedSlice"
+    argspec: "args=[\'input\', \'begin\', \'end\', \'strides\', \'begin_mask\', \'end_mask\', \'ellipsis_mask\', \'new_axis_mask\', \'shrink_axis_mask\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StridedSliceAssign"
+    argspec: "args=[\'ref\', \'begin\', \'end\', \'strides\', \'value\', \'begin_mask\', \'end_mask\', \'ellipsis_mask\', \'new_axis_mask\', \'shrink_axis_mask\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StridedSliceGrad"
+    argspec: "args=[\'shape\', \'begin\', \'end\', \'strides\', \'dy\', \'begin_mask\', \'end_mask\', \'ellipsis_mask\', \'new_axis_mask\', \'shrink_axis_mask\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringFormat"
+    argspec: "args=[\'inputs\', \'template\', \'placeholder\', \'summarize\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringJoin"
+    argspec: "args=[\'inputs\', \'separator\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringLength"
+    argspec: "args=[\'input\', \'unit\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringSplit"
+    argspec: "args=[\'input\', \'delimiter\', \'skip_empty\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringSplitV2"
+    argspec: "args=[\'input\', \'sep\', \'maxsplit\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringStrip"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringToHashBucket"
+    argspec: "args=[\'string_tensor\', \'num_buckets\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringToHashBucketFast"
+    argspec: "args=[\'input\', \'num_buckets\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringToHashBucketStrong"
+    argspec: "args=[\'input\', \'num_buckets\', \'key\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "StringToNumber"
+    argspec: "args=[\'string_tensor\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Sub"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Substr"
+    argspec: "args=[\'input\', \'pos\', \'len\', \'unit\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Sum"
+    argspec: "args=[\'input\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SummaryWriter"
+    argspec: "args=[\'shared_name\', \'container\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Svd"
+    argspec: "args=[\'input\', \'compute_uv\', \'full_matrices\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Switch"
+    argspec: "args=[\'data\', \'pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "SymbolicGradient"
+    argspec: "args=[\'input\', \'Tout\', \'f\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TFRecordDataset"
+    argspec: "args=[\'filenames\', \'compression_type\', \'buffer_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TFRecordReader"
+    argspec: "args=[\'container\', \'shared_name\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TFRecordReaderV2"
+    argspec: "args=[\'container\', \'shared_name\', \'compression_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TakeDataset"
+    argspec: "args=[\'input_dataset\', \'count\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TakeManySparseFromTensorsMap"
+    argspec: "args=[\'sparse_handles\', \'dtype\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Tan"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Tanh"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TanhGrad"
+    argspec: "args=[\'y\', \'dy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TemporaryVariable"
+    argspec: "args=[\'shape\', \'dtype\', \'var_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArray"
+    argspec: "args=[\'size\', \'dtype\', \'dynamic_size\', \'clear_after_read\', \'tensor_array_name\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayClose"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayCloseV2"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayCloseV3"
+    argspec: "args=[\'handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayConcat"
+    argspec: "args=[\'handle\', \'flow_in\', \'dtype\', \'element_shape_except0\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayConcatV2"
+    argspec: "args=[\'handle\', \'flow_in\', \'dtype\', \'element_shape_except0\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayConcatV3"
+    argspec: "args=[\'handle\', \'flow_in\', \'dtype\', \'element_shape_except0\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayGather"
+    argspec: "args=[\'handle\', \'indices\', \'flow_in\', \'dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayGatherV2"
+    argspec: "args=[\'handle\', \'indices\', \'flow_in\', \'dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayGatherV3"
+    argspec: "args=[\'handle\', \'indices\', \'flow_in\', \'dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayGrad"
+    argspec: "args=[\'handle\', \'flow_in\', \'source\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayGradV2"
+    argspec: "args=[\'handle\', \'flow_in\', \'source\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayGradV3"
+    argspec: "args=[\'handle\', \'flow_in\', \'source\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayGradWithShape"
+    argspec: "args=[\'handle\', \'flow_in\', \'shape_to_prepend\', \'source\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayPack"
+    argspec: "args=[\'handle\', \'flow_in\', \'dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayRead"
+    argspec: "args=[\'handle\', \'index\', \'flow_in\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayReadV2"
+    argspec: "args=[\'handle\', \'index\', \'flow_in\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayReadV3"
+    argspec: "args=[\'handle\', \'index\', \'flow_in\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayScatter"
+    argspec: "args=[\'handle\', \'indices\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayScatterV2"
+    argspec: "args=[\'handle\', \'indices\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayScatterV3"
+    argspec: "args=[\'handle\', \'indices\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArraySize"
+    argspec: "args=[\'handle\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArraySizeV2"
+    argspec: "args=[\'handle\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArraySizeV3"
+    argspec: "args=[\'handle\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArraySplit"
+    argspec: "args=[\'handle\', \'value\', \'lengths\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArraySplitV2"
+    argspec: "args=[\'handle\', \'value\', \'lengths\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArraySplitV3"
+    argspec: "args=[\'handle\', \'value\', \'lengths\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayUnpack"
+    argspec: "args=[\'handle\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayV2"
+    argspec: "args=[\'size\', \'dtype\', \'element_shape\', \'dynamic_size\', \'clear_after_read\', \'tensor_array_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayV3"
+    argspec: "args=[\'size\', \'dtype\', \'element_shape\', \'dynamic_size\', \'clear_after_read\', \'identical_element_shapes\', \'tensor_array_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayWrite"
+    argspec: "args=[\'handle\', \'index\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayWriteV2"
+    argspec: "args=[\'handle\', \'index\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorArrayWriteV3"
+    argspec: "args=[\'handle\', \'index\', \'value\', \'flow_in\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorDataset"
+    argspec: "args=[\'components\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListConcat"
+    argspec: "args=[\'input_handle\', \'element_dtype\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListConcatLists"
+    argspec: "args=[\'input_a\', \'input_b\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListConcatV2"
+    argspec: "args=[\'input_handle\', \'element_shape\', \'leading_dims\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListElementShape"
+    argspec: "args=[\'input_handle\', \'shape_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListFromTensor"
+    argspec: "args=[\'tensor\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListGather"
+    argspec: "args=[\'input_handle\', \'indices\', \'element_shape\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListGetItem"
+    argspec: "args=[\'input_handle\', \'index\', \'element_shape\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListLength"
+    argspec: "args=[\'input_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListPopBack"
+    argspec: "args=[\'input_handle\', \'element_shape\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListPushBack"
+    argspec: "args=[\'input_handle\', \'tensor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListPushBackBatch"
+    argspec: "args=[\'input_handles\', \'tensor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListReserve"
+    argspec: "args=[\'element_shape\', \'num_elements\', \'element_dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListResize"
+    argspec: "args=[\'input_handle\', \'size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListScatter"
+    argspec: "args=[\'tensor\', \'indices\', \'element_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListScatterV2"
+    argspec: "args=[\'tensor\', \'indices\', \'element_shape\', \'num_elements\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListSetItem"
+    argspec: "args=[\'input_handle\', \'index\', \'item\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListSplit"
+    argspec: "args=[\'tensor\', \'element_shape\', \'lengths\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorListStack"
+    argspec: "args=[\'input_handle\', \'element_shape\', \'element_dtype\', \'num_elements\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorScatterAdd"
+    argspec: "args=[\'tensor\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorScatterSub"
+    argspec: "args=[\'tensor\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorScatterUpdate"
+    argspec: "args=[\'tensor\', \'indices\', \'updates\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorSliceDataset"
+    argspec: "args=[\'components\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorSummary"
+    argspec: "args=[\'tensor\', \'description\', \'labels\', \'display_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TensorSummaryV2"
+    argspec: "args=[\'tag\', \'tensor\', \'serialized_summary_metadata\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TextLineDataset"
+    argspec: "args=[\'filenames\', \'compression_type\', \'buffer_size\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TextLineReader"
+    argspec: "args=[\'skip_header_lines\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TextLineReaderV2"
+    argspec: "args=[\'skip_header_lines\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ThreadUnsafeUnigramCandidateSampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Tile"
+    argspec: "args=[\'input\', \'multiples\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TileGrad"
+    argspec: "args=[\'input\', \'multiples\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Timestamp"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TopK"
+    argspec: "args=[\'input\', \'k\', \'sorted\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TopKV2"
+    argspec: "args=[\'input\', \'k\', \'sorted\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Transpose"
+    argspec: "args=[\'x\', \'perm\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TruncateDiv"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TruncateMod"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "TruncatedNormal"
+    argspec: "args=[\'shape\', \'seed\', \'seed2\', \'dtype\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnicodeDecode"
+    argspec: "args=[\'input\', \'input_encoding\', \'errors\', \'replacement_char\', \'replace_control_characters\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnicodeDecodeWithOffsets"
+    argspec: "args=[\'input\', \'input_encoding\', \'errors\', \'replacement_char\', \'replace_control_characters\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnicodeEncode"
+    argspec: "args=[\'input_values\', \'input_splits\', \'errors\', \'output_encoding\', \'replacement_char\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnicodeScript"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnicodeTranscode"
+    argspec: "args=[\'input\', \'input_encoding\', \'output_encoding\', \'errors\', \'replacement_char\', \'replace_control_characters\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UniformCandidateSampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'seed2\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Unique"
+    argspec: "args=[\'x\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UniqueV2"
+    argspec: "args=[\'x\', \'axis\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UniqueWithCounts"
+    argspec: "args=[\'x\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UniqueWithCountsV2"
+    argspec: "args=[\'x\', \'axis\', \'out_idx\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Unpack"
+    argspec: "args=[\'value\', \'num\', \'axis\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnravelIndex"
+    argspec: "args=[\'indices\', \'dims\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnsortedSegmentMax"
+    argspec: "args=[\'data\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnsortedSegmentMin"
+    argspec: "args=[\'data\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnsortedSegmentProd"
+    argspec: "args=[\'data\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnsortedSegmentSum"
+    argspec: "args=[\'data\', \'segment_ids\', \'num_segments\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Unstage"
+    argspec: "args=[\'capacity\', \'memory_limit\', \'dtypes\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UnwrapDatasetVariant"
+    argspec: "args=[\'input_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "UpperBound"
+    argspec: "args=[\'sorted_inputs\', \'values\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "VarHandleOp"
+    argspec: "args=[\'container\', \'shared_name\', \'dtype\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "VarIsInitializedOp"
+    argspec: "args=[\'resource\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Variable"
+    argspec: "args=[\'shape\', \'dtype\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "VariableShape"
+    argspec: "args=[\'input\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "VariableV2"
+    argspec: "args=[\'shape\', \'dtype\', \'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Where"
+    argspec: "args=[\'input\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "While"
+    argspec: "args=[\'input\', \'cond\', \'body\', \'output_shapes\', \'parallel_iterations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WholeFileReader"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WholeFileReaderV2"
+    argspec: "args=[\'container\', \'shared_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WindowDataset"
+    argspec: "args=[\'input_dataset\', \'size\', \'shift\', \'stride\', \'drop_remainder\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WrapDatasetVariant"
+    argspec: "args=[\'input_handle\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WriteAudioSummary"
+    argspec: "args=[\'writer\', \'step\', \'tag\', \'tensor\', \'sample_rate\', \'max_outputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WriteFile"
+    argspec: "args=[\'filename\', \'contents\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WriteGraphSummary"
+    argspec: "args=[\'writer\', \'step\', \'tensor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WriteHistogramSummary"
+    argspec: "args=[\'writer\', \'step\', \'tag\', \'values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WriteImageSummary"
+    argspec: "args=[\'writer\', \'step\', \'tag\', \'tensor\', \'bad_color\', \'max_images\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WriteScalarSummary"
+    argspec: "args=[\'writer\', \'step\', \'tag\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "WriteSummary"
+    argspec: "args=[\'writer\', \'step\', \'tensor\', \'tag\', \'summary_metadata\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Xdivy"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Xlogy"
+    argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ZerosLike"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "Zeta"
+    argspec: "args=[\'x\', \'q\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "ZipDataset"
+    argspec: "args=[\'input_datasets\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
+}
-- 
GitLab


From cae2b48ea50cc768c38f5037c99bd09ebf5539cd Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Thu, 31 Jan 2019 17:42:32 -0800
Subject: [PATCH 1713/2345] Add the missing dependencies

---
 tensorflow/core/kernels/data/BUILD | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD
index 184c61d4b4..c1df9b5707 100644
--- a/tensorflow/core/kernels/data/BUILD
+++ b/tensorflow/core/kernels/data/BUILD
@@ -27,6 +27,7 @@ cc_library(
     srcs = ["dataset_test_base.cc"],
     hdrs = ["dataset_test_base.h"],
     deps = [
+        ":dataset_utils",
         ":iterator_ops",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:core_cpu_internal",
@@ -210,6 +211,7 @@ tf_cc_test(
     srcs = ["map_dataset_op_test.cc"],
     deps = [
         ":dataset_test_base",
+        ":dataset_utils",
         ":iterator_ops",
         ":map_dataset_op",
         ":range_dataset_op",
@@ -217,6 +219,7 @@ tf_cc_test(
         "//tensorflow/core:dataset_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
         "//tensorflow/core/kernels:cwise_op",
@@ -396,8 +399,12 @@ tf_cc_test(
     srcs = ["range_dataset_op_test.cc"],
     deps = [
         ":dataset_test_base",
+        ":dataset_utils",
         ":iterator_ops",
         ":range_dataset_op",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:ptr_util",
+        "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
     ],
-- 
GitLab


From f4f588f49b02a2d3e1545b0136d4331d72f6456a Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Thu, 31 Jan 2019 16:16:47 -0800
Subject: [PATCH 1714/2345] Set element_shape for TensorArray in tf.scan

XLA needs the element_shape to be know to support TensorArray.

PiperOrigin-RevId: 231875046
---
 tensorflow/compiler/tests/eager_test.py | 16 ++++++++++++++++
 tensorflow/python/ops/functional_ops.py |  6 ++++--
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/tests/eager_test.py b/tensorflow/compiler/tests/eager_test.py
index c9fce39f6c..632eccbb09 100644
--- a/tensorflow/compiler/tests/eager_test.py
+++ b/tensorflow/compiler/tests/eager_test.py
@@ -34,6 +34,7 @@ from tensorflow.python.layers import pooling
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import embedding_ops
+from tensorflow.python.ops import functional_ops
 from tensorflow.python.ops import gen_random_ops
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import math_ops
@@ -607,6 +608,21 @@ class EagerFunctionTest(xla_test.XLATestCase):
     self.assertEqual(11.0, plus_one.numpy())
     self.assertEqual(9.0, minus_one.numpy())
 
+  def testScanInDefun(self):
+    with self.test_scope():
+      elems = constant_op.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], name='data')
+      v = constant_op.constant(2.0, name='v')
+
+      @def_function.function
+      def f(y):
+        # pylint: disable=unnecessary-lambda
+        return functional_ops.scan(
+            lambda a, x: math_ops.multiply(a, x), y, initializer=v)
+        # pylint: enable=unnecessary-lambda
+
+      r = f(elems)
+      self.assertAllEqual([2., 4., 12., 48., 240., 1440.], self.evaluate(r))
+
 
 class ExcessivePaddingTest(xla_test.XLATestCase):
   """Test that eager execution works with TPU flattened tensors.
diff --git a/tensorflow/python/ops/functional_ops.py b/tensorflow/python/ops/functional_ops.py
index 324f55a89f..448e45cae3 100644
--- a/tensorflow/python/ops/functional_ops.py
+++ b/tensorflow/python/ops/functional_ops.py
@@ -397,13 +397,15 @@ def scan(fn, elems, initializer=None, parallel_iterations=10, back_prop=True,
         ops.convert_to_tensor(elem, name="elem") for elem in elems_flat]
 
     # Convert elems to tensor array. n may be known statically.
-    n = (tensor_shape.dimension_value(elems_flat[0].shape[0])
-         or array_ops.shape(elems_flat[0])[0])
+    n = tensor_shape.dimension_value(elems_flat[0].shape[0])
+    if n is None:
+      n = array_ops.shape(elems_flat[0])[0]
 
     # TensorArrays are always flat
     elems_ta = [
         tensor_array_ops.TensorArray(dtype=elem.dtype, size=n,
                                      dynamic_size=False,
+                                     element_shape=elem.shape[1:],
                                      infer_shape=True)
         for elem in elems_flat]
     # Unpack elements
-- 
GitLab


From e8b2f12a84e0fbedc94d797174eb87d2d72a5866 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 31 Jan 2019 16:16:58 -0800
Subject: [PATCH 1715/2345] Allows the gradient multiplier tensors to be
 non-constant.

PiperOrigin-RevId: 231875076
---
 tensorflow/contrib/training/python/training/training.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/training/python/training/training.py b/tensorflow/contrib/training/python/training/training.py
index fc6e38ab4a..093765dc20 100644
--- a/tensorflow/contrib/training/python/training/training.py
+++ b/tensorflow/contrib/training/python/training/training.py
@@ -244,7 +244,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import clip_ops
@@ -354,11 +353,11 @@ def multiply_gradients(grads_and_vars, gradient_multipliers):
         raise ValueError('Requested multiple of `None` gradient.')
 
       if isinstance(grad, ops.IndexedSlices):
-        tmp = grad.values * constant_op.constant(
+        tmp = grad.values * ops.convert_to_tensor(
             gradient_multipliers[key], dtype=grad.dtype)
         grad = ops.IndexedSlices(tmp, grad.indices, grad.dense_shape)
       else:
-        grad *= constant_op.constant(
+        grad *= ops.convert_to_tensor(
             gradient_multipliers[key], dtype=grad.dtype)
     multiplied_grads_and_vars.append((grad, var))
   return multiplied_grads_and_vars
-- 
GitLab


From 74e496ee2aa558e8ee274086dfef870bfe7462ad Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 31 Jan 2019 16:22:35 -0800
Subject: [PATCH 1716/2345] parallel_for: Handle case where add_bias bias input
 is not loop invariant.

PiperOrigin-RevId: 231876031
---
 .../python/ops/parallel_for/math_test.py      | 54 +++++++++++++------
 tensorflow/python/ops/parallel_for/pfor.py    | 32 +++++++----
 2 files changed, 60 insertions(+), 26 deletions(-)

diff --git a/tensorflow/python/ops/parallel_for/math_test.py b/tensorflow/python/ops/parallel_for/math_test.py
index 98c24e6caa..7a5bef7229 100644
--- a/tensorflow/python/ops/parallel_for/math_test.py
+++ b/tensorflow/python/ops/parallel_for/math_test.py
@@ -325,26 +325,46 @@ class MathTest(PForTestCase):
           self._test_loop_fn(loop_fn, 2)
 
   def test_bias_add(self):
-    x_shape = [2, 3, 4, 5, 6]
-    x = random_ops.random_uniform(x_shape)
     for data_format in ("NCHW", "NHWC"):
-      with backprop.GradientTape(persistent=True) as g:
-        bias_dim = 2 if data_format == "NCHW" else -1
-        bias_shape = x_shape[bias_dim]
-        bias = random_ops.random_uniform([bias_shape])
-        g.watch(bias)
+      for stacked_value in (True, False):
+        x_shape = [3, 4, 5, 6]
+        if stacked_value:
+          x_shape = [2] + x_shape
+        x = random_ops.random_uniform(x_shape)
+        for stacked_bias in (True, False):
+          if not (stacked_value or stacked_bias):
+            continue
+          with backprop.GradientTape(persistent=True) as g:
+            bias_dim = -1
+            if data_format == "NCHW":
+              bias_dim = 2 if stacked_value else 1
+            bias_shape = [x_shape[bias_dim]]
+            if stacked_bias:
+              bias_shape = [2] + bias_shape
+            bias = random_ops.random_uniform(bias_shape)
+            g.watch(bias)
 
-      # pylint: disable=cell-var-from-loop
-      def loop_fn(i):
-        with g:
-          a = array_ops.gather(x, i)
-          y = nn.bias_add(a, bias, data_format=data_format)
-          loss = math_ops.reduce_sum(y * y)
-        return y, g.gradient(loss, bias)
-      # pylint: enable=cell-var-from-loop
+          # pylint: disable=cell-var-from-loop
+          def loop_fn(i):
+            with g:
+              a = array_ops.gather(x, i) if stacked_value else x
+              b = array_ops.gather(bias, i) if stacked_bias else bias
+              y = nn.bias_add(a, b, data_format=data_format)
+              loss = math_ops.reduce_sum(y * y)
+            grad = g.gradient(loss, bias)
+            if stacked_bias:
+              # If we gather over bias in loop_fn, the gradient will be an
+              # instance of `IndexedSlices` with attrs `values` and `indices`.
+              return y, grad.values, grad.indices
+            else:
+              return y, grad
+          # pylint: enable=cell-var-from-loop
 
-      self._test_loop_fn(
-          loop_fn, 2, loop_fn_dtypes=[dtypes.float32, dtypes.float32])
+          out_dtypes = [dtypes.float32, dtypes.float32]
+          if stacked_bias:
+            out_dtypes = out_dtypes + [dtypes.int32]
+          self._test_loop_fn(
+              loop_fn, 2, loop_fn_dtypes=out_dtypes)
 
   def test_unsorted_segment_sum(self):
     t = random_ops.random_uniform([3, 3, 2])
diff --git a/tensorflow/python/ops/parallel_for/pfor.py b/tensorflow/python/ops/parallel_for/pfor.py
index 74cb444aad..019d4f2224 100644
--- a/tensorflow/python/ops/parallel_for/pfor.py
+++ b/tensorflow/python/ops/parallel_for/pfor.py
@@ -42,6 +42,7 @@ from tensorflow.python.ops import sparse_ops
 from tensorflow.python.ops import tensor_array_ops
 from tensorflow.python.platform import flags
 from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.util import compat
 from tensorflow.python.util import nest
 
 flags.DEFINE_bool(
@@ -1900,17 +1901,30 @@ def _convert_cumfoo(pfor_input, _, op_func):
 
 @RegisterPFor("BiasAdd")
 def _convert_biasadd(pfor_input):
-  t = pfor_input.stacked_input(0)
-  bias = pfor_input.unstacked_input(1)
+  t, t_stacked, _ = pfor_input.input(0)
+  bias, bias_stacked, _ = pfor_input.input(1)
   data_format = pfor_input.get_attr("data_format")
-  if data_format != b"NCHW":
+  if bias_stacked:
+    # BiasAdd only supports 1-D biases, so cast bias to match value and use Add.
+    pfor_input.expanddim_inputs_for_broadcast()
+    t, _, _ = pfor_input.input(0)
+    bias = math_ops.cast(pfor_input.stacked_input(1), t.dtype)
+    if compat.as_bytes(data_format) == b"NCHW":
+      b_shape = array_ops.shape(bias)
+      new_b_shape = array_ops.concat(
+          [b_shape[:-3], b_shape[-1:], b_shape[-3:-1]], axis=0)
+      bias = array_ops.reshape(bias, new_b_shape)
+    return wrap(math_ops.add(t, bias), True)
+  else:
+    assert t_stacked, "At least one input to BiasAdd should be loop variant."
+    if compat.as_bytes(data_format) == b"NCHW":
+      shape = array_ops.shape(t)
+      flattened_shape = array_ops.concat([[-1], shape[2:]], axis=0)
+      t = array_ops.reshape(t, flattened_shape)
+      t = nn_ops.bias_add(t, bias, data_format=b"NCHW")
+      t = array_ops.reshape(t, shape)
+      return wrap(t, True)
     return wrap(nn_ops.bias_add(t, bias, data_format=data_format), True)
-  shape = array_ops.shape(t)
-  flattened_shape = array_ops.concat([[-1], shape[2:]], axis=0)
-  t = array_ops.reshape(t, flattened_shape)
-  t = nn_ops.bias_add(t, bias, data_format=b"NCHW")
-  t = array_ops.reshape(t, shape)
-  return wrap(t, True)
 
 
 @RegisterPFor("UnsortedSegmentSum")
-- 
GitLab


From f76a3e3461d2460da8b91a7b195489d28de43201 Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Thu, 31 Jan 2019 16:49:03 -0800
Subject: [PATCH 1717/2345] [XLA] Add unoptimized_hlo_proto dump to
 Service::BuildExecutables.

With a recent change to Client::Execute, flag xla_dump_unoptimized_hlo_proto_to
stops working. Fix this by handling the flag in Service::BuildExecutables.

PiperOrigin-RevId: 231880458
---
 tensorflow/compiler/xla/service/service.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc
index 83434528a2..9bda6fba3a 100644
--- a/tensorflow/compiler/xla/service/service.cc
+++ b/tensorflow/compiler/xla/service/service.cc
@@ -368,6 +368,7 @@ StatusOr<std::vector<std::unique_ptr<Executable>>> Service::BuildExecutables(
     const HloModuleProto* proto = module_protos[i];
     const HloModuleConfig& config = *module_configs[i];
     TF_ASSIGN_OR_RETURN(auto module, CreateModuleFromProto(*proto, config));
+    TF_RETURN_IF_ERROR(MaybeDumpUnoptimizedHloModule(*module));
     module_group->push_back(std::move(module));
   }
 
-- 
GitLab


From 593ebaa0baffc0b07aed26e39d496351a178ca8e Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 31 Jan 2019 17:05:37 -0800
Subject: [PATCH 1718/2345] [XLA:CPU] Check for +inf in implementation of log.

Previously we log(+inf) == log(MAX_FLOAT).  Now log(+inf) == +inf.

PiperOrigin-RevId: 231883322
---
 .../xla/service/cpu/llvm_ir_runtime.cc        | 23 ++++++++++++++-----
 .../xla/service/cpu/vector_support_library.h  |  3 +++
 2 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/llvm_ir_runtime.cc b/tensorflow/compiler/xla/service/cpu/llvm_ir_runtime.cc
index e6183c19f7..be39fb8602 100644
--- a/tensorflow/compiler/xla/service/cpu/llvm_ir_runtime.cc
+++ b/tensorflow/compiler/xla/service/cpu/llvm_ir_runtime.cc
@@ -198,12 +198,14 @@ llvm::Value* GenerateVF32Log(llvm::IRBuilder<>* b, llvm::Value* input,
   // The smallest non denormalized float number.
   const llvm::APFloat min_norm_pos = GetIeeeF32FromBitwiseRep(0x00800000);
   const llvm::APFloat minus_inf = GetIeeeF32FromBitwiseRep(0xff800000);
+  const llvm::APFloat pos_inf = GetIeeeF32FromBitwiseRep(0x7f800000);
   const llvm::APFloat inv_mant_mask = GetIeeeF32FromBitwiseRep(~0x7f800000);
 
   // invalid_mask is set if x is negative or NaN (and therefore output
   // must be NaN).
   llvm::Value* invalid_mask = vsl.FCmpULEMask(input, vsl.GetZeroVector());
-  llvm::Value* iszero_mask = vsl.FCmpEQMask(input, vsl.GetZeroVector());
+  llvm::Value* is_zero_mask = vsl.FCmpEQMask(input, vsl.GetZeroVector());
+  llvm::Value* is_pos_inf_mask = vsl.FCmpEQMask(input, pos_inf);
 
   // Cut off denormalized stuff.
   input = vsl.Max(min_norm_pos, input);
@@ -260,11 +262,20 @@ llvm::Value* GenerateVF32Log(llvm::IRBuilder<>* b, llvm::Value* input,
   input = vsl.Add(input, y);
   input = vsl.Add(input, y2);
 
-  // Negative arg will be NAN, 0 will be -INF.
-  llvm::Value* or_lhs =
-      vsl.FloatAndNot(iszero_mask, vsl.FloatOr(input, invalid_mask));
-  llvm::Value* or_rhs = vsl.FloatAnd(iszero_mask, minus_inf);
-  return vsl.FloatOr(or_lhs, or_rhs);
+  // Contains +/-inf where +/-inf is the correct answer, otherwise 0.
+  llvm::Value* result_inf = vsl.FloatOr(vsl.FloatAnd(is_zero_mask, minus_inf),
+                                        vsl.FloatAnd(is_pos_inf_mask, pos_inf));
+
+  // Contains a finite result or nan.  This is the correct answer only if both
+  // result_minus_inf and result_pos_inf are both 0.
+  //
+  // (This implementation works because 0xffffffff is a nan.)
+  llvm::Value* result_finite_or_nan = vsl.FloatOr(input, invalid_mask);
+
+  // Combine the above into a final result.
+  return vsl.FloatOr(result_inf,
+                     vsl.FloatAndNot(vsl.FloatOr(is_zero_mask, is_pos_inf_mask),
+                                     result_finite_or_nan));
 }
 }  // namespace
 
diff --git a/tensorflow/compiler/xla/service/cpu/vector_support_library.h b/tensorflow/compiler/xla/service/cpu/vector_support_library.h
index 5690d2be2f..c444fd7d4a 100644
--- a/tensorflow/compiler/xla/service/cpu/vector_support_library.h
+++ b/tensorflow/compiler/xla/service/cpu/vector_support_library.h
@@ -114,6 +114,9 @@ class VectorSupportLibrary {
   // raison d'etre) less cluttered.
 
   llvm::Value* FCmpEQMask(llvm::Value* lhs, llvm::Value* rhs);
+  llvm::Value* FCmpEQMask(llvm::Value* lhs, const llvm::APFloat& rhs) {
+    return FCmpEQMask(lhs, GetConstantFloat(lhs->getType(), rhs));
+  }
   llvm::Value* FCmpULEMask(llvm::Value* lhs, llvm::Value* rhs);
   llvm::Value* FCmpOLTMask(llvm::Value* lhs, llvm::Value* rhs);
   llvm::Value* FCmpOLTMask(llvm::Value* lhs, const llvm::APFloat& rhs) {
-- 
GitLab


From dd1f4c5d0483a30001d4a0f3f8fceadc3230e37e Mon Sep 17 00:00:00 2001
From: Brian Lee <brianklee@google.com>
Date: Thu, 31 Jan 2019 17:20:57 -0800
Subject: [PATCH 1719/2345] Detect and skip converting wrapt-decorated
 functions.

PiperOrigin-RevId: 231885544
---
 tensorflow/python/autograph/impl/api.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tensorflow/python/autograph/impl/api.py b/tensorflow/python/autograph/impl/api.py
index 3ad69689f7..7e0b5e79b8 100644
--- a/tensorflow/python/autograph/impl/api.py
+++ b/tensorflow/python/autograph/impl/api.py
@@ -180,6 +180,12 @@ def converted_call(f, owner, options, *args, **kwargs):
   if inspect_utils.isbuiltin(f):
     return py_builtins.overload_of(f)(*args, **kwargs)
 
+  # Don't convert wrapt-decorated functions/methods.
+  # TODO(b/122265385): Fully support wrapt
+  if ('wrapt' in sys.modules and
+      isinstance(f, sys.modules['wrapt'].FunctionWrapper)):
+    return f(*args, **kwargs)
+
   # TODO(mdan): This needs cleanup.
   # In particular, we may want to avoid renaming functions altogether.
   if not options.force_conversion and conversion.is_whitelisted_for_graph(f):
-- 
GitLab


From db832c0797b79a6e33e1c54b043c58ab2d602a3c Mon Sep 17 00:00:00 2001
From: Shivani Agrawal <shivaniagrawal@google.com>
Date: Thu, 31 Jan 2019 17:20:59 -0800
Subject: [PATCH 1720/2345] [tf.data] Adds test coverage for MapDataset,
 function having variables placed at different devices.

PiperOrigin-RevId: 231885550
---
 .../python/data/kernel_tests/map_test.py      | 66 +++++++++++++++++++
 1 file changed, 66 insertions(+)

diff --git a/tensorflow/python/data/kernel_tests/map_test.py b/tensorflow/python/data/kernel_tests/map_test.py
index 8d98d650e1..bc9a67e5db 100644
--- a/tensorflow/python/data/kernel_tests/map_test.py
+++ b/tensorflow/python/data/kernel_tests/map_test.py
@@ -25,6 +25,7 @@ from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.core.framework import attr_value_pb2
+from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.data.experimental.ops import threading_options
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
@@ -391,6 +392,71 @@ class MapTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.assertRaises(errors.OutOfRangeError):
       self.evaluate(get_next())
 
+  # TODO(b/121264236): add eager mode coverage when we have mutli-device setup.
+  @test_util.run_v1_only("b/121264236")
+  def testSkipEagerCaptureConstantsWithConflictingDevices(self):
+    config = config_pb2.ConfigProto(
+        device_count={"CPU": 3}, log_device_placement=True)
+    with self.cached_session(config=config):
+      with ops.device("/device:CPU:0"):
+        a = constant_op.constant(3.0)
+      with ops.device("/device:CPU:1"):
+        b = constant_op.constant(5.0)
+
+      def func(_):
+        return math_ops.add(a, b)
+
+      dataset = dataset_ops.Dataset.from_tensors(0).repeat(10).map(func)
+      expected_output = [8.0] * 10
+      self.assertDatasetProduces(dataset, expected_output=expected_output)
+
+  # TODO(b/121264236): add eager mode coverage when we have mutli-device setup.
+  @test_util.run_v1_only(
+      "defun will convert RefVariables to ResourceVariables.")
+  def testSkipEagerRefVariablesWithConflictingDevices(self):
+    config = config_pb2.ConfigProto(
+        device_count={"CPU": 3}, log_device_placement=True)
+    with self.cached_session(config=config):
+
+      def func(_):
+        with ops.device("/device:CPU:0"):
+          a = variables.VariableV1(3.0)
+        with ops.device("/device:CPU:1"):
+          b = variables.VariableV1(5.0)
+        return math_ops.add(a, b)
+
+      dataset = dataset_ops.Dataset.from_tensors(0).repeat(10).map(func)
+      self.evaluate(variables.global_variables_initializer())
+      expected_output = [8.0] * 10
+      self.assertDatasetProduces(
+          dataset,
+          expected_output=expected_output,
+          requires_initialization=True)
+
+  # TODO(b/121264236): add eager mode coverage when we have mutli-device setup.
+  @test_util.run_v1_only("b/121264236")
+  def testSkipEagerResourceVariablesWithConflictingDevices(self):
+    config = config_pb2.ConfigProto(
+        device_count={"CPU": 3}, log_device_placement=True)
+    with self.cached_session(config=config):
+
+      def func(_):
+        with ops.device("/device:CPU:0"):
+          a = variables.Variable(3.0)
+        with ops.device("/device:CPU:1"):
+          b = variables.Variable(5.0)
+        return math_ops.add(a, b)
+
+      # The MapDataset node ends up with two ResourceVariable inputs, one on
+      # device CPU:0 and the other on device CPU:1. The placer cannot resolve
+      # this as it cannot place the MapDatasetOp on both devices.
+      dataset = dataset_ops.Dataset.from_tensors(0).repeat(10).map(func)
+      expected_error = (
+          errors.InvalidArgumentError,
+          "Could not colocate node with its resource and reference inputs")
+      self.assertDatasetProduces(
+          dataset, expected_error=expected_error, requires_initialization=True)
+
   def testCaptureVariable(self):
     counter_var = variable_scope.get_variable(
         "counter", (), dtypes.int32, use_resource=True)
-- 
GitLab


From 7cd7e58594bd7bda2100a147376dd4364821b9a0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 31 Jan 2019 17:31:02 -0800
Subject: [PATCH 1721/2345] Fixed arguments passed to max_pool_with_arhmax.

PiperOrigin-RevId: 231886927
---
 tensorflow/python/ops/nn_ops.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 51c1f15bac..146b891943 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -3214,12 +3214,15 @@ def max_pool_with_argmax_v1(input,  # pylint: disable=missing-docstring,invalid-
                             Targmax=None,  # pylint: disable=invalid-name
                             name=None,
                             output_dtype=None):
+  if data_format != "NHWC":
+    raise ValueError("Data formats other than 'NHWC' are not yet supported")
+
   Targmax = deprecated_argument_lookup(
       "output_dtype", output_dtype, "Targmax", Targmax)
   if Targmax is None:
     Targmax = dtypes.int64
   return gen_nn_ops.max_pool_with_argmax(
-      input, ksize, strides, padding, data_format, Targmax, name)
+      input, ksize, strides, padding, Targmax, name)
 
 max_pool_with_argmax_v1.__doc__ = gen_nn_ops.max_pool_with_argmax.__doc__
 # pylint: enable=redefined-builtin
-- 
GitLab


From 1fbbdfbac75be1cde62296b7af20f42b090f1d16 Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Thu, 31 Jan 2019 17:40:30 -0800
Subject: [PATCH 1722/2345] Add Multi-Worker IndependentWorker to Keras `fit`
 and `evaluate` for synchronous case.

PiperOrigin-RevId: 231888383
---
 .../distribute/distribute_coordinator.py      |  48 ++++++--
 .../distribute/distribute_coordinator_test.py |   3 +
 tensorflow/python/keras/BUILD                 |   4 +
 tensorflow/python/keras/backend.py            |  67 ++++++++++-
 .../engine/distributed_training_utils.py      |  29 -----
 tensorflow/python/keras/engine/training.py    | 108 +++++++++++++-----
 6 files changed, 187 insertions(+), 72 deletions(-)

diff --git a/tensorflow/python/distribute/distribute_coordinator.py b/tensorflow/python/distribute/distribute_coordinator.py
index 49dfe8db35..eb3fd1d82e 100644
--- a/tensorflow/python/distribute/distribute_coordinator.py
+++ b/tensorflow/python/distribute/distribute_coordinator.py
@@ -34,6 +34,9 @@ from tensorflow.python.training import monitored_session
 from tensorflow.python.training import server_lib
 
 
+_thread_local = threading.local()
+
+
 class _TaskType(object):
   PS = "ps"
   WORKER = "worker"
@@ -383,6 +386,27 @@ def _run_std_server(cluster_spec=None,
                     rpc_layer=None,
                     environment=None):
   """Runs a standard server."""
+  # Check if the Server is already running. If so, assert that no configuration
+  # options have changed, and return the existing Server. This allows us to
+  # call `run_distribute_coordinator` multiple times.
+  if getattr(_thread_local, "server", None) is not None:
+    assert _thread_local.cluster_spec == cluster_spec
+    assert _thread_local.task_type == task_type
+    assert _thread_local.task_id == task_id
+    assert _thread_local.session_config_str == repr(session_config)
+    assert _thread_local.rpc_layer == rpc_layer
+    assert _thread_local.environment == environment
+    return _thread_local.server
+  else:
+    # This method is not thread-safe.
+    _thread_local.server_started = True
+    _thread_local.cluster_spec = cluster_spec
+    _thread_local.task_type = task_type
+    _thread_local.task_id = task_id
+    _thread_local.session_config_str = repr(session_config)
+    _thread_local.rpc_layer = rpc_layer
+    _thread_local.environment = environment
+
   assert cluster_spec
   target = cluster_spec.task_address(task_type, task_id)
   if rpc_layer:
@@ -404,8 +428,6 @@ def _run_std_server(cluster_spec=None,
 
   if environment == "google":
     server = _FakeServer()
-    server.start()
-    return server
   else:
     if session_config:
       logging.info(
@@ -420,8 +442,10 @@ def _run_std_server(cluster_spec=None,
         task_index=task_id,
         config=session_config,
         protocol=rpc_layer)
-    server.start()
-    return server
+
+  server.start()
+  _thread_local.server = server
+  return server
 
 
 def _run_between_graph_client(worker_fn, strategy, eval_fn, eval_strategy,
@@ -719,7 +743,8 @@ def run_distribute_coordinator(worker_fn,
 
   Returns:
     In the client job, return the value returned by `worker_fn` if
-    it is in-graph replication; return None otherwise.
+    it is in-graph replication or INDEPENDENT_WORKER mode; return None
+    otherwise.
   """
   tf_config = json.loads(os.environ.get("TF_CONFIG", "{}"))
   if not cluster_spec:
@@ -813,23 +838,22 @@ def run_distribute_coordinator(worker_fn,
         session_config=session_config,
         rpc_layer=rpc_layer,
         environment=environment)
-
     if task_type in [_TaskType.CHIEF, _TaskType.WORKER]:
       if strategy.extended.experimental_between_graph:
         # All jobs run `worker_fn` if between-graph.
-        _run_single_worker(worker_fn, strategy, cluster_spec, task_type,
-                           task_id, session_config, rpc_layer)
+        return _run_single_worker(worker_fn, strategy, cluster_spec, task_type,
+                                  task_id, session_config, rpc_layer)
       else:
         # Only one node runs `worker_fn` if in-graph.
         context = _WorkerContext(strategy, cluster_spec, task_type, task_id)
         if context.is_chief:
-          _run_single_worker(worker_fn, strategy, cluster_spec, None, None,
-                             session_config, rpc_layer)
+          return _run_single_worker(worker_fn, strategy, cluster_spec, None,
+                                    None, session_config, rpc_layer)
         else:
           server.join()
     elif task_type == _TaskType.EVALUATOR:
-      _run_single_worker(eval_fn, eval_strategy, cluster_spec, task_type,
-                         task_id, session_config, rpc_layer)
+      return _run_single_worker(eval_fn, eval_strategy, cluster_spec, task_type,
+                                task_id, session_config, rpc_layer)
     else:
       if task_type != _TaskType.PS:
         raise ValueError("Unexpected task_type: %r" % task_type)
diff --git a/tensorflow/python/distribute/distribute_coordinator_test.py b/tensorflow/python/distribute/distribute_coordinator_test.py
index ceb4483ebb..22997169fc 100644
--- a/tensorflow/python/distribute/distribute_coordinator_test.py
+++ b/tensorflow/python/distribute/distribute_coordinator_test.py
@@ -864,6 +864,9 @@ class StrategyConfigureTest(test.TestCase):
     cluster_spec = {"worker": ["localhost:0"]}
     tf_config = {"cluster": cluster_spec}
 
+    # Reset the saved Server state.
+    distribute_coordinator._thread_local = threading.local()  # pylint: disable=protected-access
+
     with test.mock.patch.dict("os.environ",
                               {"TF_CONFIG": json.dumps(tf_config)}):
       distribute_coordinator.run_distribute_coordinator(
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index ad895d21dc..17e4c084b4 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -91,6 +91,7 @@ py_library(
         "//tensorflow/python:constant_op",
         "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:ctc_ops",
+        "//tensorflow/python:distribute",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:framework",
         "//tensorflow/python:framework_ops",
@@ -114,8 +115,10 @@ py_library(
         "//tensorflow/python:tensor_array_grad",
         "//tensorflow/python:tensor_array_ops",
         "//tensorflow/python:tensor_shape",
+        "//tensorflow/python:training",
         "//tensorflow/python:util",
         "//tensorflow/python:variables",
+        "//tensorflow/python/distribute:distribute_coordinator",
     ],
 )
 
@@ -161,6 +164,7 @@ py_library(
         ":regularizers",
         ":saving",
         "//tensorflow/python/data",
+        "//tensorflow/python/distribute:distribute_coordinator",
         "//tensorflow/python/distribute:distribute_lib",
         "//tensorflow/python/distribute:input_lib",
         "//tensorflow/python/distribute:reduce_util",
diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index 837fd2643a..9c2b46ee24 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -32,6 +32,9 @@ import numpy as np
 
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.client import session as session_module
+from tensorflow.python.distribute import distribute_coordinator as dc
+from tensorflow.python.distribute import distribute_coordinator_context as dc_context
+from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.eager import context
 from tensorflow.python.eager import function as eager_function
 from tensorflow.python.framework import constant_op
@@ -61,7 +64,7 @@ from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import tensor_array_grad  # pylint: disable=unused-import
 from tensorflow.python.ops import tensor_array_ops
 from tensorflow.python.ops import variables as variables_module
-
+from tensorflow.python.training import server_lib
 from tensorflow.python.util import nest
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util import tf_inspect
@@ -394,8 +397,14 @@ def _get_session():
     session = default_session
   else:
     if getattr(_SESSION, 'session', None) is None:
-      _SESSION.session = session_module.Session(
-          config=get_default_session_config())
+      # We are creating the Session inside a Distribution
+      # Strategy scope.
+      if distribution_strategy_context.has_strategy():
+        configure_and_create_distributed_session(
+            distribution_strategy_context.get_strategy())
+      else:
+        _SESSION.session = session_module.Session(
+            config=get_default_session_config())
     session = _SESSION.session
   return session
 
@@ -5201,3 +5210,55 @@ if not os.path.exists(_config_path):
   except IOError:
     # Except permission denied.
     pass
+
+
+def in_multi_worker_mode():
+  """Whether we are operating in a Multi-Worker setting."""
+  tf_config = json.loads(os.environ.get('TF_CONFIG', '{}'))
+  cluster_spec = server_lib.ClusterSpec(tf_config.get('cluster', {}))
+  return tf_config and 'master' not in cluster_spec.jobs
+
+
+def configure_and_create_distributed_session(distribution_strategy):
+  """Configure session config and create a session with it."""
+
+  # TODO(priyag): Throw error if a session already exists.
+  def _create_session(distribution_strategy):
+    """Create the Distributed Strategy session."""
+    session_config = get_default_session_config()
+
+    if is_tpu_strategy(distribution_strategy):
+      # TODO(priyag, yuefengz): Remove this workaround when Distribute
+      # Coordinator is integrated with keras and we can create a session from
+      # there.
+      distribution_strategy.configure(session_config)
+      master = distribution_strategy.extended._tpu_cluster_resolver.master()  # pylint: disable=protected-access
+      session = session_module.Session(config=session_config, target=master)
+    else:
+      worker_context = dc_context.get_current_worker_context()
+      if worker_context:
+        dc_session_config = worker_context.session_config
+        # Merge the default session config to the one from distribute
+        # coordinator, which is fine for now since they don't have
+        # conflicting configurations.
+        dc_session_config.MergeFrom(session_config)
+        session = session_module.Session(
+            config=dc_session_config, target=worker_context.master_target)
+      else:
+        distribution_strategy.configure(session_config)
+        session = session_module.Session(config=session_config)
+
+    set_session(session)
+
+  if in_multi_worker_mode():
+    dc.run_distribute_coordinator(
+        _create_session,
+        distribution_strategy,
+        mode=dc.CoordinatorMode.INDEPENDENT_WORKER)
+  else:
+    _create_session(distribution_strategy)
+
+
+def is_tpu_strategy(strategy):
+  """We're executing TPU Strategy."""
+  return strategy is not None and strategy.__class__.__name__ == 'TPUStrategy'
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index 70de386245..4d3da9d413 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -20,7 +20,6 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.python.client import session as session_module
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.distribute import distribute_coordinator_context as dc_context
@@ -351,34 +350,6 @@ def init_restore_or_wait_for_variables():
     _wait_for_variable_initialization(session)
 
 
-def configure_and_create_session(distribution_strategy):
-  """Configure session config and create a session with it."""
-  # TODO(priyag): Throw error if a session already exists.
-  session_config = K.get_default_session_config()
-
-  if is_tpu_strategy(distribution_strategy):
-    # TODO(priyag, yuefengz): Remove this workaround when Distribute
-    # Coordinator is integrated with keras and we can create a session from
-    # there.
-    distribution_strategy.configure(session_config)
-    master = distribution_strategy.extended._tpu_cluster_resolver.master()  # pylint: disable=protected-access
-    session = session_module.Session(config=session_config, target=master)
-  else:
-    worker_context = dc_context.get_current_worker_context()
-    if worker_context:
-      dc_session_config = worker_context.session_config
-      # Merge the default session config to the one from distribute coordinator,
-      # which is fine for now since they don't have conflicting configurations.
-      dc_session_config.MergeFrom(session_config)
-      session = session_module.Session(
-          config=dc_session_config, target=worker_context.master_target)
-    else:
-      distribution_strategy.configure(session_config)
-      session = session_module.Session(config=session_config)
-
-  K.set_session(session)
-
-
 def validate_inputs(x, y, distribution_strategy, allow_partial_batch=False):
   """Validate inputs when using DistributionStrategy.
 
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 53c6a8896a..63bc4b62ae 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -24,6 +24,7 @@ import numpy as np
 from tensorflow.python import tf2
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
+from tensorflow.python.distribute import distribute_coordinator as dc
 from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
@@ -275,8 +276,9 @@ class Model(Network):
     for mode in [ModeKeys.TRAIN, ModeKeys.TEST, ModeKeys.PREDICT]:
       distributed_training_utils.set_distributed_model(self, mode, None)
     if self._distribution_strategy is not None:
-      distributed_training_utils.configure_and_create_session(
-          self._distribution_strategy)
+      # Ensures a Session is created and configured correctly for Distribution
+      # Strategy.
+      K.configure_and_create_distributed_session(self._distribution_strategy)
     # Initialize model metric attributes.
     self._init_metric_attributes()
     if not self.built or not self.inputs or not self.outputs:
@@ -786,23 +788,52 @@ class Model(Network):
 
     # Case 1: distribution strategy.
     if self._distribution_strategy:
-      return training_distributed.fit_distributed(
-          self,
-          x=x,
-          y=y,
-          batch_size=batch_size,
-          epochs=epochs,
-          verbose=verbose,
-          callbacks=callbacks,
-          validation_split=validation_split,
-          validation_data=validation_data,
-          shuffle=shuffle,
-          class_weight=class_weight,
-          sample_weight=sample_weight,
-          initial_epoch=initial_epoch,
-          steps_per_epoch=steps_per_epoch,
-          validation_steps=validation_steps,
-          validation_freq=validation_freq)
+      if K.in_multi_worker_mode():
+        # Multi-Worker mode runs the Keras training loop on multiple
+        # servers via the Distribute Coordinator.
+        def _worker_fn(_):
+          """Run training inside the distributed coordinator."""
+          return training_distributed.fit_distributed(
+              self,
+              x=x,
+              y=y,
+              batch_size=batch_size,
+              epochs=epochs,
+              verbose=verbose,
+              callbacks=callbacks,
+              validation_split=validation_split,
+              validation_data=validation_data,
+              shuffle=shuffle,
+              class_weight=class_weight,
+              sample_weight=sample_weight,
+              initial_epoch=initial_epoch,
+              steps_per_epoch=steps_per_epoch,
+              validation_steps=validation_steps,
+              validation_freq=validation_freq)
+
+        # Independent worker only for now.
+        return dc.run_distribute_coordinator(
+            _worker_fn,
+            self._distribution_strategy,
+            mode=dc.CoordinatorMode.INDEPENDENT_WORKER)
+      else:
+        return training_distributed.fit_distributed(
+            self,
+            x=x,
+            y=y,
+            batch_size=batch_size,
+            epochs=epochs,
+            verbose=verbose,
+            callbacks=callbacks,
+            validation_split=validation_split,
+            validation_data=validation_data,
+            shuffle=shuffle,
+            class_weight=class_weight,
+            sample_weight=sample_weight,
+            initial_epoch=initial_epoch,
+            steps_per_epoch=steps_per_epoch,
+            validation_steps=validation_steps,
+            validation_freq=validation_freq)
 
     batch_size = self._validate_or_infer_batch_size(
         batch_size, steps_per_epoch, x)
@@ -1017,15 +1048,36 @@ class Model(Network):
     """
     # Case 1: distribution strategy.
     if self._distribution_strategy:
-      return training_distributed.evaluate_distributed(
-          self,
-          x=x,
-          y=y,
-          batch_size=batch_size,
-          verbose=verbose,
-          sample_weight=sample_weight,
-          steps=steps,
-          callbacks=callbacks)
+      if K.in_multi_worker_mode():
+        # Multi-Worker mode runs the Keras evaluation loop on multiple
+        # servers via the Distribute Coordinator.
+        def _worker_fn(_):
+          """Run training inside the distributed coordinator."""
+          return training_distributed.evaluate_distributed(
+              self,
+              x=x,
+              y=y,
+              batch_size=batch_size,
+              verbose=verbose,
+              sample_weight=sample_weight,
+              steps=steps,
+              callbacks=callbacks)
+
+        # Independent worker only for now.
+        return dc.run_distribute_coordinator(
+            _worker_fn,
+            self._distribution_strategy,
+            mode=dc.CoordinatorMode.INDEPENDENT_WORKER)
+      else:
+        return training_distributed.evaluate_distributed(
+            self,
+            x=x,
+            y=y,
+            batch_size=batch_size,
+            verbose=verbose,
+            sample_weight=sample_weight,
+            steps=steps,
+            callbacks=callbacks)
 
     batch_size = self._validate_or_infer_batch_size(batch_size, steps, x)
 
-- 
GitLab


From 5458340eced84db0827af1464d216dea9257b8dd Mon Sep 17 00:00:00 2001
From: Mani Varadarajan <maniv@google.com>
Date: Thu, 31 Jan 2019 17:47:10 -0800
Subject: [PATCH 1723/2345] Rename thread to "profiler_server" from "profiler
 server" (replace space with "_").

PiperOrigin-RevId: 231889226
---
 tensorflow/core/profiler/rpc/profiler_server.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/profiler/rpc/profiler_server.cc b/tensorflow/core/profiler/rpc/profiler_server.cc
index 08affffb2d..835aa1e3f5 100644
--- a/tensorflow/core/profiler/rpc/profiler_server.cc
+++ b/tensorflow/core/profiler/rpc/profiler_server.cc
@@ -26,7 +26,7 @@ namespace tensorflow {
 std::unique_ptr<Thread> StartProfilerServer(EagerContext* const eager_context,
                                             int32 port) {
   return WrapUnique(eager_context->TFEnv()->StartThread(
-      {}, "profiler server", [eager_context, port]() {
+      {}, "profiler_server", [eager_context, port]() {
         string server_address = strings::StrCat("0.0.0.0:", port);
         std::unique_ptr<TPUProfiler::Service> service =
             CreateProfilerService(eager_context);
-- 
GitLab


From 5d643de7a541888223cbce315b8273b367adc8b7 Mon Sep 17 00:00:00 2001
From: Shivani Agrawal <shivaniagrawal@google.com>
Date: Thu, 31 Jan 2019 17:52:54 -0800
Subject: [PATCH 1724/2345] [data-stats] Fixes the document.

RELNOTES: n/a
PiperOrigin-RevId: 231889859
---
 tensorflow/python/data/experimental/ops/stats_aggregator.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/data/experimental/ops/stats_aggregator.py b/tensorflow/python/data/experimental/ops/stats_aggregator.py
index 3e4c66be27..0c6e686481 100644
--- a/tensorflow/python/data/experimental/ops/stats_aggregator.py
+++ b/tensorflow/python/data/experimental/ops/stats_aggregator.py
@@ -44,7 +44,7 @@ class StatsAggregator(object):
   dataset = ...
 
   # Apply `StatsOptions` to associate `dataset` with `aggregator`.
-  options = dataset_ops.Options()
+  options = tf.data.Options()
   options.experimental_stats.aggregator = aggregator
   dataset = dataset.with_options(options)
   ```
-- 
GitLab


From 610db7b37196ba8a75d5133b124848beeedd803f Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Thu, 31 Jan 2019 17:55:34 -0800
Subject: [PATCH 1725/2345] Correctly handle return statements in with
 statements.

PiperOrigin-RevId: 231890185
---
 .../python/autograph/converters/return_statements.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tensorflow/python/autograph/converters/return_statements.py b/tensorflow/python/autograph/converters/return_statements.py
index c6c6c3ba5a..6cf6f26649 100644
--- a/tensorflow/python/autograph/converters/return_statements.py
+++ b/tensorflow/python/autograph/converters/return_statements.py
@@ -108,6 +108,13 @@ class ConditionalReturnRewriter(converter.Base):
     node.orelse, _ = self._visit_statement_block(node, node.orelse)
     return node
 
+  def visit_With(self, node):
+    node.items = self.visit_block(node.items)
+    node.body, definitely_returns = self._visit_statement_block(node, node.body)
+    if definitely_returns:
+      anno.setanno(node, STMT_DEFINITELY_RETURNS, True)
+    return node
+
   def visit_If(self, node):
     node.test = self.visit(node.test)
 
@@ -305,6 +312,11 @@ class ReturnStatementsTransformer(converter.Base):
     node.orelse = self._visit_statement_block(node, node.orelse)
     return node
 
+  def visit_With(self, node):
+    node.items = self.visit_block(node.items)
+    node.body = self._visit_statement_block(node, node.body)
+    return node
+
   def visit_If(self, node):
     node.test = self.visit(node.test)
     node.body = self._visit_statement_block(node, node.body)
-- 
GitLab


From 17e4b9a0c63cb8a278d28d3459f03b83a2224b21 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 31 Jan 2019 17:59:12 -0800
Subject: [PATCH 1726/2345] [XLA] Add real_hardware_only parameter to xla_test.

PiperOrigin-RevId: 231890605
---
 tensorflow/compiler/xla/tests/build_defs.bzl | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tensorflow/compiler/xla/tests/build_defs.bzl b/tensorflow/compiler/xla/tests/build_defs.bzl
index 05d4d04034..c14d279ac5 100644
--- a/tensorflow/compiler/xla/tests/build_defs.bzl
+++ b/tensorflow/compiler/xla/tests/build_defs.bzl
@@ -34,6 +34,7 @@ def xla_test(
         xla_test_library_deps = [],
         backends = [],
         blacklisted_backends = [],
+        real_hardware_only = False,
         args = [],
         tags = [],
         copts = [],
@@ -108,6 +109,10 @@ def xla_test(
         use for that target.
       **kwargs: Additional keyword arguments to pass to native.cc_test.
     """
+
+    # All of the backends in all_backends are real hardware.
+    _ignore = [real_hardware_only]
+
     test_names = []
     if not backends:
         backends = all_backends
-- 
GitLab


From 1d63c46b34803685a93b3b817a058cf0b576c547 Mon Sep 17 00:00:00 2001
From: Mani Varadarajan <maniv@google.com>
Date: Thu, 31 Jan 2019 18:01:34 -0800
Subject: [PATCH 1727/2345] Fix small typo and tweak wording in TensorBoard
 callback doc.

PiperOrigin-RevId: 231890935
---
 tensorflow/python/keras/callbacks.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/keras/callbacks.py b/tensorflow/python/keras/callbacks.py
index 3223c89812..41fedbb31f 100644
--- a/tensorflow/python/keras/callbacks.py
+++ b/tensorflow/python/keras/callbacks.py
@@ -1118,9 +1118,9 @@ class TensorBoard(Callback):
       ValueError: If histogram_freq is set and no validation data is provided.
 
   @compatibility(eager)
-  Using `Tensorboard` callback will work while eager execution is enabled,
-  however outputting histogram summaries of weights and gradients is not
-  supported, and thus `histogram_freq` will be ignored.
+  Using the `TensorBoard` callback will work when eager execution is enabled,
+  with the restriction that outputting histogram summaries of weights and
+  gradients is not supported. Consequently, `histogram_freq` will be ignored.
   @end_compatibility
   """
 
-- 
GitLab


From fd7b85df67292982b6da1e0c89300830a1da6066 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 31 Jan 2019 18:01:52 -0800
Subject: [PATCH 1728/2345] Update ops-related pbtxt files.

PiperOrigin-RevId: 231890974
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 37 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 37 +++++++++++++++++++
 2 files changed, 74 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index f67895755a..941a9c07b7 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -79118,6 +79118,43 @@ op {
     type: "type"
   }
 }
+op {
+  name: "TensorListConcatV2"
+  input_arg {
+    name: "input_handle"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "element_shape"
+    type_attr: "shape_type"
+  }
+  input_arg {
+    name: "leading_dims"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "tensor"
+    type_attr: "element_dtype"
+  }
+  output_arg {
+    name: "lengths"
+    type: DT_INT64
+  }
+  attr {
+    name: "element_dtype"
+    type: "type"
+  }
+  attr {
+    name: "shape_type"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
 op {
   name: "TensorListElementShape"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 7fd8a36795..f77fd63c4c 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -38346,6 +38346,43 @@ op {
     type: "type"
   }
 }
+op {
+  name: "TensorListConcatV2"
+  input_arg {
+    name: "input_handle"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "element_shape"
+    type_attr: "shape_type"
+  }
+  input_arg {
+    name: "leading_dims"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "tensor"
+    type_attr: "element_dtype"
+  }
+  output_arg {
+    name: "lengths"
+    type: DT_INT64
+  }
+  attr {
+    name: "element_dtype"
+    type: "type"
+  }
+  attr {
+    name: "shape_type"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
 op {
   name: "TensorListElementShape"
   input_arg {
-- 
GitLab


From 1cd954876fd2b19868a1ce8154b0adac51a37fc1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 31 Jan 2019 18:02:39 -0800
Subject: [PATCH 1729/2345] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 231891161

---
 tensorflow/go/op/wrappers.go | 224 ++++++++++++++++++++---------------
 1 file changed, 127 insertions(+), 97 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 5f823be62b..e281191ab5 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -23987,6 +23987,36 @@ func SdcaOptimizer(scope *Scope, sparse_example_indices []tf.Output, sparse_feat
 	return out_example_state_data, out_delta_sparse_weights, out_delta_dense_weights
 }
 
+// Concats all tensors in the list along the 0th dimension.
+//
+// Requires that all tensors have the same shape except the first dimension.
+//
+// input_handle: The input list.
+// element_shape: The shape of the uninitialized elements in the list. If the first
+//   dimension is not -1, it is assumed that all list elements have the same
+//   leading dim.
+// leading_dims: The list of leading dims of uninitialized list elements. Used if
+//   the leading dim of input_handle.element_shape or the element_shape input arg
+//   is not already set.
+// tensor: The concated result.
+// lengths: Output tensor containing sizes of the 0th dimension of tensors in the list, used for computing the gradient.
+//
+func TensorListConcatV2(scope *Scope, input_handle tf.Output, element_shape tf.Output, leading_dims tf.Output, element_dtype tf.DataType) (tensor tf.Output, lengths tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"element_dtype": element_dtype}
+	opspec := tf.OpSpec{
+		Type: "TensorListConcatV2",
+		Input: []tf.Input{
+			input_handle, element_shape, leading_dims,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
 // MatrixTriangularSolveAttr is an optional argument to MatrixTriangularSolve.
 type MatrixTriangularSolveAttr func(optionalAttr)
 
@@ -29125,6 +29155,103 @@ func TensorListScatter(scope *Scope, tensor tf.Output, indices tf.Output, elemen
 	return op.Output(0)
 }
 
+// Deprecated. Use TensorArrayScatterV3
+//
+// DEPRECATED at GraphDef version 26: Use TensorArrayScatterV3
+func TensorArrayScatterV2(scope *Scope, handle tf.Output, indices tf.Output, value tf.Output, flow_in tf.Output) (flow_out tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorArrayScatterV2",
+		Input: []tf.Input{
+			handle, indices, value, flow_in,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// AsStringAttr is an optional argument to AsString.
+type AsStringAttr func(optionalAttr)
+
+// AsStringPrecision sets the optional precision attribute to value.
+//
+// value: The post-decimal precision to use for floating point numbers.
+// Only used if precision > -1.
+// If not specified, defaults to -1
+func AsStringPrecision(value int64) AsStringAttr {
+	return func(m optionalAttr) {
+		m["precision"] = value
+	}
+}
+
+// AsStringScientific sets the optional scientific attribute to value.
+//
+// value: Use scientific notation for floating point numbers.
+// If not specified, defaults to false
+func AsStringScientific(value bool) AsStringAttr {
+	return func(m optionalAttr) {
+		m["scientific"] = value
+	}
+}
+
+// AsStringShortest sets the optional shortest attribute to value.
+//
+// value: Use shortest representation (either scientific or standard) for
+// floating point numbers.
+// If not specified, defaults to false
+func AsStringShortest(value bool) AsStringAttr {
+	return func(m optionalAttr) {
+		m["shortest"] = value
+	}
+}
+
+// AsStringWidth sets the optional width attribute to value.
+//
+// value: Pad pre-decimal numbers to this width.
+// Applies to both floating point and integer numbers.
+// Only used if width > -1.
+// If not specified, defaults to -1
+func AsStringWidth(value int64) AsStringAttr {
+	return func(m optionalAttr) {
+		m["width"] = value
+	}
+}
+
+// AsStringFill sets the optional fill attribute to value.
+//
+// value: The value to pad if width > -1.  If empty, pads with spaces.
+// Another typical value is '0'.  String cannot be longer than 1 character.
+// If not specified, defaults to ""
+func AsStringFill(value string) AsStringAttr {
+	return func(m optionalAttr) {
+		m["fill"] = value
+	}
+}
+
+// Converts each entry in the given tensor to strings.  Supports many numeric
+//
+// types and boolean.
+func AsString(scope *Scope, input tf.Output, optional ...AsStringAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "AsString",
+		Input: []tf.Input{
+			input,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Returns a `RaggedTensor` containing the specified sequences of numbers.
 //
 //
@@ -35584,100 +35711,3 @@ func TensorArrayGradV2(scope *Scope, handle tf.Output, flow_in tf.Output, source
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
-
-// AsStringAttr is an optional argument to AsString.
-type AsStringAttr func(optionalAttr)
-
-// AsStringPrecision sets the optional precision attribute to value.
-//
-// value: The post-decimal precision to use for floating point numbers.
-// Only used if precision > -1.
-// If not specified, defaults to -1
-func AsStringPrecision(value int64) AsStringAttr {
-	return func(m optionalAttr) {
-		m["precision"] = value
-	}
-}
-
-// AsStringScientific sets the optional scientific attribute to value.
-//
-// value: Use scientific notation for floating point numbers.
-// If not specified, defaults to false
-func AsStringScientific(value bool) AsStringAttr {
-	return func(m optionalAttr) {
-		m["scientific"] = value
-	}
-}
-
-// AsStringShortest sets the optional shortest attribute to value.
-//
-// value: Use shortest representation (either scientific or standard) for
-// floating point numbers.
-// If not specified, defaults to false
-func AsStringShortest(value bool) AsStringAttr {
-	return func(m optionalAttr) {
-		m["shortest"] = value
-	}
-}
-
-// AsStringWidth sets the optional width attribute to value.
-//
-// value: Pad pre-decimal numbers to this width.
-// Applies to both floating point and integer numbers.
-// Only used if width > -1.
-// If not specified, defaults to -1
-func AsStringWidth(value int64) AsStringAttr {
-	return func(m optionalAttr) {
-		m["width"] = value
-	}
-}
-
-// AsStringFill sets the optional fill attribute to value.
-//
-// value: The value to pad if width > -1.  If empty, pads with spaces.
-// Another typical value is '0'.  String cannot be longer than 1 character.
-// If not specified, defaults to ""
-func AsStringFill(value string) AsStringAttr {
-	return func(m optionalAttr) {
-		m["fill"] = value
-	}
-}
-
-// Converts each entry in the given tensor to strings.  Supports many numeric
-//
-// types and boolean.
-func AsString(scope *Scope, input tf.Output, optional ...AsStringAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "AsString",
-		Input: []tf.Input{
-			input,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Deprecated. Use TensorArrayScatterV3
-//
-// DEPRECATED at GraphDef version 26: Use TensorArrayScatterV3
-func TensorArrayScatterV2(scope *Scope, handle tf.Output, indices tf.Output, value tf.Output, flow_in tf.Output) (flow_out tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "TensorArrayScatterV2",
-		Input: []tf.Input{
-			handle, indices, value, flow_in,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-- 
GitLab


From 3051d59f35baa2268d8a7c5ad324b0ecdfe85ecb Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 31 Jan 2019 18:04:42 -0800
Subject: [PATCH 1730/2345] [XLA] Tweak error spec for
 exhaustive_f32_elementwise_op_test.

PiperOrigin-RevId: 231891574
---
 .../compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc   | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc b/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc
index e442b7b26d..6200f0e34f 100644
--- a/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc
+++ b/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc
@@ -176,6 +176,9 @@ class ExhaustiveF32ElementwiseOpTest
 };
 
 XLA_TEST_P(ExhaustiveF32ElementwiseOpTest, LogF32) {
+#if !defined(XLA_TEST_BACKEND_CPU) && !defined(XLA_TEST_BACKEND_GPU)
+  error_spec_ = ErrorSpec{0.001, 0.001};
+#endif
   ExhaustivelyTestF32Op(
       [](XlaBuilder* builder, const XlaOp& input) { Log(input); }, std::log,
       /*known_incorrect_range=*/{0, 0});
-- 
GitLab


From 65d8d1c41bf1677230f0256eee1580ebf2f0d1d4 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Thu, 31 Jan 2019 18:05:38 -0800
Subject: [PATCH 1731/2345] Do not use "-lrt" flag when building on macos.

https://stackoverflow.com/questions/47703239/ld-library-not-found-for-lrt
Also, the flag breaks the build on macos.

PiperOrigin-RevId: 231891701
---
 tensorflow/lite/nnapi/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/lite/nnapi/BUILD b/tensorflow/lite/nnapi/BUILD
index 7af2b099e7..8ee9f3c221 100644
--- a/tensorflow/lite/nnapi/BUILD
+++ b/tensorflow/lite/nnapi/BUILD
@@ -31,6 +31,7 @@ cc_library(
     ],
     linkopts = ["-ldl"] + select({
         "//tensorflow:android": [],
+        "//tensorflow:darwin": [],
         "//tensorflow:ios": [],
         "//tensorflow:windows": [],
         "//conditions:default": ["-lrt"],
-- 
GitLab


From 238b6b789689219a8c90dd7b3bc9fae2123a0fa2 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 31 Jan 2019 18:10:51 -0800
Subject: [PATCH 1732/2345] [XLA] Enable exhaustive_f32_elementwise_op_test.

PiperOrigin-RevId: 231892537
---
 tensorflow/compiler/xla/tests/BUILD | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 8fb6742550..383ad5d35d 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -670,17 +670,13 @@ xla_test(
 
 xla_test(
     name = "exhaustive_f32_elementwise_op_test",
-    size = "enormous",
     srcs = ["exhaustive_f32_elementwise_op_test.cc"],
-    backends = [
-        "cpu",
-        "gpu",
-    ],
+    real_hardware_only = True,  # Very slow on the interpreter.
     shard_count = 48,
     tags = [
-        "broken",
-        "manual",
-        "notap",
+        "optonly",
+        # This is a big test that we skip for capacity reasons in OSS testing.
+        "nooss",
     ],
     deps = [
         ":client_library_test_base",
-- 
GitLab


From ca2a1cb4cad2dba0d7c4bde662f36de592f5668f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 31 Jan 2019 18:24:04 -0800
Subject: [PATCH 1733/2345] PR #25205: TFTRT: Support conv2d_transpose
 (Conv2DBackpropInput)

Please approve this CL. It will be submitted automatically, and its GitHub pull request will be marked as merged.

Imported from GitHub PR #25205

Support conv2d_transpose in TFTRT, with unit tests.

Copybara import of the project:

  - 0c6d54d404caca7912e7195c7bd9a692fd12e376 Initial support for conv2d_transpose by Trevor Morris <tmorris@nvidia.com>
  - e8f91bee122df05edc59b5f1fc4d728ed4bc6ae1 Add tests. Fix formatting. Properly set num outputs and a... by Trevor Morris <tmorris@nvidia.com>
  - 9b5491c4530c070bd300cd141fe10230a6a3f747 Revert change to logger by Trevor Morris <tmorris@nvidia.com>
  - 6d39208c54d970637a1f360e4e4046929eb45401 Remove Conv2DHelper wrapper. Improve variable names by Trevor Morris <tmorris@nvidia.com>
  - 6faf0034c76b5e2dbbac64f64ede349c0e3039d1 Fix line too long by Trevor Morris <tmorris@nvidia.com>
  - 00f78b1ec65c87214620a01e821ee4dae7a333ef Rename is_deconv -> is_conv2d_backprop_input by Trevor Morris <tmorris@nvidia.com>
  - 603dedb4ff53197b18ec88a3b81dfabcc8d96221 Small improvement to comments and error messages by Trevor Morris <tmorris@nvidia.com>
  - 8eb98a19f506821394d71b9bc02c93107e101859 Update error message in test by Trevor Morris <tmorris@nvidia.com>
  - 2a91f9811266f1e680af2b03f8bc8dc939d90523 Remove spurious comma by Trevor Morris <tmorris@nvidia.com>
  - 95c1766626c09dab916cfe7cddfba34334271315 Merge 2a91f9811266f1e680af2b03f8bc8dc939d90523 into a7d98... by Trevor Morris <trevoraidanmorris@gmail.com>

COPYBARA_INTEGRATE_REVIEW=https://github.com/tensorflow/tensorflow/pull/25205 from trevor-m:tmorris_tftrt_conv2d_transpose 2a91f9811266f1e680af2b03f8bc8dc939d90523
PiperOrigin-RevId: 231894127
---
 .../tf2tensorrt/convert/convert_graph.cc      |   1 +
 .../tf2tensorrt/convert/convert_nodes.cc      | 151 ++++++++++++------
 .../tf2tensorrt/convert/convert_nodes_test.cc |  74 +++++++--
 .../contrib/tensorrt/test/conv2d_test.py      |  42 +++++
 4 files changed, 210 insertions(+), 58 deletions(-)

diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc
index f93d69839c..22394f7cdd 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc
@@ -99,6 +99,7 @@ Status TrtCandidateSelector::IsTensorRTCandidate(const tensorflow::Node* node) {
       "ConcatV2",
       "Const",
       "Conv2D",
+      "Conv2DBackpropInput",
       "DepthwiseConv2dNative",
       "Div",
       "Exp",
diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
index e6bbe2dfb8..7594f3d44d 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
@@ -1571,20 +1571,48 @@ Status BinaryTensorOpWeight(OpConverterParams* params,
   return tensorflow::Status::OK();
 }
 
-enum class ConvolutionType { DEFAULT, DEPTHWISE_CONV };
-
-tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
+tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group,
+                                       bool is_conv2d_backprop_input) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
-  if (inputs.size() != 2) {
-    return tensorflow::errors::InvalidArgument("Two inputs are expected for ",
-                                               node_def.op(), ", at ",
-                                               node_def.name());
-  }
-  if (inputs.at(0).is_weights()) {
-    return tensorflow::errors::Unimplemented(
-        node_def.op(), " is only implemented for tensors, not weights, at ",
-        node_def.name());
+  TRT_TensorOrWeights backprop_output_size;
+  const nvinfer1::ITensor* tensor = nullptr;
+  // TODO(tmorris): Add helper functions for validating inputs to avoid code
+  // duplication.
+  if (is_conv2d_backprop_input) {
+    // Conv2DBackpropInput has 3 inputs: input_sizes, filter, and out_backprop.
+    // In the case of conv2d_transpose, these correspond to: output size,
+    // filter, and input.
+    if (inputs.size() != 3) {
+      return tensorflow::errors::InvalidArgument(
+          "Three inputs are expected for ", node_def.op(), ", at ",
+          node_def.name());
+    }
+    if (inputs.at(0).is_tensor()) {
+      return tensorflow::errors::Unimplemented(
+          "input_sizes for ", node_def.op(), " must be constant weights, at ",
+          node_def.name());
+    }
+    if (inputs.at(2).is_weights()) {
+      return tensorflow::errors::Unimplemented(
+          node_def.op(), " is only implemented for tensors, not weights, at ",
+          node_def.name());
+    }
+    backprop_output_size = inputs.at(0);
+    tensor = inputs.at(2).tensor();
+  } else {
+    // Regular convolution has 2 inputs: input, filter.
+    if (inputs.size() != 2) {
+      return tensorflow::errors::InvalidArgument("Two inputs are expected for ",
+                                                 node_def.op(), ", at ",
+                                                 node_def.name());
+    }
+    if (inputs.at(0).is_weights()) {
+      return tensorflow::errors::Unimplemented(
+          node_def.op(), " is only implemented for tensors, not weights, at ",
+          node_def.name());
+    }
+    tensor = inputs.at(0).tensor();
   }
   if (inputs.at(1).is_tensor()) {
     return tensorflow::errors::Unimplemented("Kernel for ", node_def.op(),
@@ -1613,6 +1641,11 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
         node_def.name());
   }
   const nvinfer1::DimsHW dilation(tf_dilations[h_index], tf_dilations[w_index]);
+  if (is_conv2d_backprop_input && (dilation.d[0] != 1 || dilation.d[1] != 1)) {
+    return tensorflow::errors::Unimplemented(
+        "Dilation with Conv2DBackpropInput (conv2d_transpose) is not supported",
+        ", at ", node_def.name());
+  }
 
   const auto tf_stride = attrs.get<std::vector<int>>("strides");
   if (tf_stride.size() != 4) {
@@ -1628,8 +1661,6 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
   const nvinfer1::DimsHW stride(tf_stride[h_index], tf_stride[w_index]);
   if (params->validation_only) return tensorflow::Status::OK();
 
-  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
-
   // Transpose to NCHW (NCHW is required for IConvLayer).
   const bool need_transpose = (data_format == "NHWC");
   if (need_transpose) {
@@ -1639,19 +1670,23 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
   // Dimensions of transposed tensor.
   const auto tensor_dim = tensor->getDimensions();
 
-  // For depthwise convolution, group will be 0 so set num_groups to size of
-  // input's channel dim. For a non-depthwise conv, num_groups will be 1.
+  // group == 0 signifies that this is a depthwise convolution, so set
+  // num_groups to size of input's channel dim. For a non-depthwise conv,
+  // num_groups will be 1.
   const int num_groups = (group == 0) ? tensor_dim.d[0] : group;
 
   if (params->converter->precision_mode() == TrtPrecisionMode::FP16) {
-    weights_rsck =
-        ConvertFP32ToFP16(params->weight_store, inputs.at(1).weights());
+    weights_rsck = ConvertFP32ToFP16(params->weight_store, weights_rsck);
   }
+  // For conv, TF weights are RSCK, and TRT expects KCRS.
+  // For backprop, TF weights are RSKC, and TRT expects CKRS.
+  // Therefore, this reorder will work for both cases.
   TRT_ShapedWeights weights =
       params->weight_store->GetTempWeights(weights_rsck);
   ReorderRSCKToKCRS(weights_rsck, &weights, num_groups);
   TRT_ShapedWeights biases(weights.type_);
-  const int noutput = weights.shape_.d[0] * num_groups;
+  const int output_axis = is_conv2d_backprop_input ? 1 : 0;
+  const int noutput = weights.shape_.d[output_axis] * num_groups;
   nvinfer1::DimsHW kernel_size;
   kernel_size.h() = weights.shape_.d[2];
   kernel_size.w() = weights.shape_.d[3];
@@ -1662,9 +1697,23 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
     nvinfer1::DimsHW effective_kernel_size = kernel_size;
     effective_kernel_size.h() += (kernel_size.h() - 1) * (dilation.h() - 1);
     effective_kernel_size.w() += (kernel_size.w() - 1) * (dilation.w() - 1);
-    padding = CreateSamePadding(
-        stride, effective_kernel_size,
-        {static_cast<int>(tensor_dim.d[1]), static_cast<int>(tensor_dim.d[2])});
+    std::vector<int64_t> input_dims;
+    if (is_conv2d_backprop_input) {
+      // For backprop, calculate padding based on "input_sizes" input, which
+      // actually corresponds to output size. ("input_sizes" makes sense in the
+      // context of Conv2DBackpropInput).
+      // We use h_index and w_index instead of 1 and 2 because we havent
+      // transposed backprop_output_size along with the input.
+      auto output_size_weights = static_cast<int*>(
+          const_cast<void*>(backprop_output_size.weights().GetValues()));
+      input_dims = {output_size_weights[h_index], output_size_weights[w_index]};
+    } else {
+      // Use 1 and 2 because tensor_dim has the dimensions of the transposed
+      // input.
+      input_dims = {static_cast<int>(tensor_dim.d[1]),
+                    static_cast<int>(tensor_dim.d[2])};
+    }
+    padding = CreateSamePadding(stride, effective_kernel_size, input_dims);
   } else {
     padding = {{0, 0}, {0, 0}};
   }
@@ -1683,17 +1732,32 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
   }
 
   // Add convolution.
-  nvinfer1::IConvolutionLayer* layer =
-      params->converter->network()->addConvolution(
-          *const_cast<nvinfer1::ITensor*>(tensor), noutput, kernel_size,
-          weights.GetTrtWeights(), biases.GetTrtWeights());
-  TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name());
-  layer->setStride(stride);
-  layer->setPadding({padding[0].first, padding[1].first});
-  layer->setName(node_def.name().c_str());
-  layer->setNbGroups(num_groups);
-  layer->setDilation(dilation);
-  const nvinfer1::ITensor* output_tensor = layer->getOutput(0);
+  nvinfer1::ILayer* conv_layer = nullptr;
+  if (is_conv2d_backprop_input) {
+    nvinfer1::IDeconvolutionLayer* layer =
+        params->converter->network()->addDeconvolution(
+            *const_cast<nvinfer1::ITensor*>(tensor), noutput, kernel_size,
+            weights.GetTrtWeights(), biases.GetTrtWeights());
+    TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name());
+    layer->setStride(stride);
+    layer->setPadding({padding[0].first, padding[1].first});
+    layer->setName(node_def.name().c_str());
+    layer->setNbGroups(num_groups);
+    conv_layer = layer;
+  } else {
+    nvinfer1::IConvolutionLayer* layer =
+        params->converter->network()->addConvolution(
+            *const_cast<nvinfer1::ITensor*>(tensor), noutput, kernel_size,
+            weights.GetTrtWeights(), biases.GetTrtWeights());
+    TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name());
+    layer->setStride(stride);
+    layer->setPadding({padding[0].first, padding[1].first});
+    layer->setName(node_def.name().c_str());
+    layer->setNbGroups(num_groups);
+    layer->setDilation(dilation);
+    conv_layer = layer;
+  }
+  const nvinfer1::ITensor* output_tensor = conv_layer->getOutput(0);
 
   // Restore transpose.
   if (need_transpose) {
@@ -1706,18 +1770,6 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
   return tensorflow::Status::OK();
 }
 
-tensorflow::Status ConvertConv2DHelper(OpConverterParams* params,
-                                       ConvolutionType type) {
-  switch (type) {
-    case ConvolutionType::DEFAULT:
-      return ConvertConv2DHelper(params, 1);
-    case ConvolutionType::DEPTHWISE_CONV:
-      return ConvertConv2DHelper(params, 0);
-  }
-  return tensorflow::errors::Unimplemented("Unsupported convolution type, at ",
-                                           params->node_def.name());
-}
-
 Status BinaryTensorOpTensor(OpConverterParams* params,
                             const TRT_TensorOrWeights& operand_l,
                             const TRT_TensorOrWeights& operand_r) {
@@ -2329,11 +2381,15 @@ tensorflow::Status ConvertStridedSlice(OpConverterParams* params) {
 }
 
 tensorflow::Status ConvertConv2D(OpConverterParams* params) {
-  return ConvertConv2DHelper(params, ConvolutionType::DEFAULT);
+  return ConvertConv2DHelper(params, 1, /*is_conv2d_backprop_input=*/false);
 }
 
 tensorflow::Status ConvertConv2DDepthwise(OpConverterParams* params) {
-  return ConvertConv2DHelper(params, ConvolutionType::DEPTHWISE_CONV);
+  return ConvertConv2DHelper(params, 0, /*is_conv2d_backprop_input=*/false);
+}
+
+tensorflow::Status ConvertConv2DBackpropInput(OpConverterParams* params) {
+  return ConvertConv2DHelper(params, 1, /*is_conv2d_backprop_input=*/true);
 }
 
 tensorflow::Status ConvertPool(OpConverterParams* params) {
@@ -3591,6 +3647,7 @@ static void RegisterValidatableOpConverters(
   (*registration)["ConcatV2"] = ConvertConcat;
   (*registration)["Const"] = ConvertConst;
   (*registration)["Conv2D"] = ConvertConv2D;
+  (*registration)["Conv2DBackpropInput"] = ConvertConv2DBackpropInput;
   (*registration)["DepthwiseConv2dNative"] = ConvertConv2DDepthwise;
   (*registration)["ExpandDims"] = ConvertExpandDims;
   (*registration)["MatMul"] = ConvertMatMul;
diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
index c3d64e840f..a297bb7990 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
@@ -2769,16 +2769,28 @@ TEST_F(OpConverterTest, ConvertConv2D) {
   // Get nodedef for Conv2D layer.
   auto get_conv2d_nodedef =
       [](std::vector<int> strides = {1, 1, 1, 1}, string padding = "SAME",
-         string data_format = "NCHW",
-         std::vector<int> dilations = {1, 1, 1, 1}) -> NodeDef {
+         string data_format = "NCHW", std::vector<int> dilations = {1, 1, 1, 1},
+         bool is_conv2d_backprop_input = false) -> NodeDef {
     Scope s = Scope::NewRootScope();
     auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
     auto filter = ops::Placeholder(s.WithOpName("weights"), DT_FLOAT);
-    ops::Conv2D::Attrs attrs =
-        ops::Conv2D::Attrs().DataFormat(data_format).Dilations(dilations);
-    auto conv2d = ops::Conv2D(s.WithOpName("my_conv2d"), input, filter, strides,
-                              padding, attrs);
-    return conv2d.operation.node()->def();
+    if (is_conv2d_backprop_input) {
+      auto input_sizes =
+          ops::Placeholder(s.WithOpName("input_sizes"), DT_INT32);
+      ops::Conv2DBackpropInput::Attrs attrs = ops::Conv2DBackpropInput::Attrs()
+                                                  .DataFormat(data_format)
+                                                  .Dilations(dilations);
+      auto conv2d =
+          ops::Conv2DBackpropInput(s.WithOpName("my_conv2d"), input_sizes,
+                                   filter, input, strides, padding, attrs);
+      return conv2d.operation.node()->def();
+    } else {
+      ops::Conv2D::Attrs attrs =
+          ops::Conv2D::Attrs().DataFormat(data_format).Dilations(dilations);
+      auto conv2d = ops::Conv2D(s.WithOpName("my_conv2d"), input, filter,
+                                strides, padding, attrs);
+      return conv2d.operation.node()->def();
+    }
   };
 
   {
@@ -2844,6 +2856,19 @@ TEST_F(OpConverterTest, ConvertConv2D) {
                                "Dilation rate must be 1 for batch and channel "
                                "dimensions, at my_conv2d");
   }
+  {
+    // Dilation + Conv2DBackpropInput, should fail.
+    Reset();
+    NodeDef node_def =
+        get_conv2d_nodedef({1, 1, 1, 1}, "SAME", "NHWC", {1, 1, 2, 1}, true);
+    AddTestTensor("input", {2, 3, 1});
+    AddTestWeights<float>("weights", {3, 3, 1, 1}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
+    AddTestWeights<int>("input_sizes", {4}, {1, 2, 3, 1});
+    RunValidationAndConversion(node_def, error::UNIMPLEMENTED,
+                               "Dilation with Conv2DBackpropInput "
+                               "(conv2d_transpose) is not supported, "
+                               "at my_conv2d");
+  }
   {
     // Strides is not 4D, should fail.
     Reset();
@@ -2874,6 +2899,7 @@ TEST_F(OpConverterTest, ConvertConv2D) {
                const std::vector<float>& filter,
                const std::vector<int>& strides, const string& padding,
                const string& data_format, const std::vector<int>& dilations,
+               bool is_conv2d_backprop_input,
                const std::vector<int>& expected_output_dims,
                const std::vector<float>& expected_output)
         : input_dims(input_dims),
@@ -2884,6 +2910,7 @@ TEST_F(OpConverterTest, ConvertConv2D) {
           padding(padding),
           data_format(data_format),
           dilations(dilations),
+          is_conv2d_backprop_input(is_conv2d_backprop_input),
           expected_output_dims(expected_output_dims),
           expected_output(expected_output) {}
 
@@ -2895,12 +2922,13 @@ TEST_F(OpConverterTest, ConvertConv2D) {
     string padding;
     string data_format;
     std::vector<int> dilations;
+    bool is_conv2d_backprop_input;
     std::vector<int> expected_output_dims;
     std::vector<float> expected_output;
   };
 
   // Ok.
-  const int kConv2DOKCases = 6;
+  const int kConv2DOKCases = 7;
   TestParams ok_params[kConv2DOKCases] = {
       // Basic
       TestParams{/*input_dims=*/{1, 2, 3},
@@ -2911,6 +2939,7 @@ TEST_F(OpConverterTest, ConvertConv2D) {
                  /*padding=*/"VALID",
                  /*data_format=*/"NCHW",
                  /*dilations=*/{1, 1, 1, 1},
+                 /*is_conv2d_backprop_input=*/false,
                  /*expected_output_dims=*/{1, 2, 2},
                  /*expected_output=*/{1, 1, 0, 1}},
       // SAME padding (Asymmetric)
@@ -2922,6 +2951,7 @@ TEST_F(OpConverterTest, ConvertConv2D) {
                  /*padding=*/"SAME",
                  /*data_format=*/"NCHW",
                  /*dilations=*/{1, 1, 1, 1},
+                 /*is_conv2d_backprop_input=*/false,
                  /*expected_output_dims=*/{1, 2, 3},
                  /*expected_output=*/{1, 1, -2, 0, 1, -4}},
       // SAME padding (Symmetric)
@@ -2933,6 +2963,7 @@ TEST_F(OpConverterTest, ConvertConv2D) {
                  /*padding=*/"SAME",
                  /*data_format=*/"NCHW",
                  /*dilations=*/{1, 1, 1, 1},
+                 /*is_conv2d_backprop_input=*/false,
                  /*expected_output_dims=*/{1, 2, 3},
                  /*expected_output=*/{1, 2, -1, 3, 1, -3}},
       // NHWC
@@ -2944,6 +2975,7 @@ TEST_F(OpConverterTest, ConvertConv2D) {
                  /*padding=*/"VALID",
                  /*data_format=*/"NHWC",
                  /*dilations=*/{1, 1, 1, 1},
+                 /*is_conv2d_backprop_input=*/false,
                  /*expected_output_dims=*/{2, 2, 1},
                  /*expected_output=*/{1, 1, 0, 1}},
       // Dilated
@@ -2955,6 +2987,7 @@ TEST_F(OpConverterTest, ConvertConv2D) {
                  /*padding=*/"VALID",
                  /*data_format=*/"NCHW",
                  /*dilations=*/{1, 1, 1, 2},
+                 /*is_conv2d_backprop_input=*/false,
                  /*expected_output_dims=*/{1, 2, 1},
                  /*expected_output=*/{2, 1}},
       // Strided
@@ -2966,18 +2999,37 @@ TEST_F(OpConverterTest, ConvertConv2D) {
                  /*padding=*/"VALID",
                  /*data_format=*/"NCHW",
                  /*dilations=*/{1, 1, 1, 1},
+                 /*is_conv2d_backprop_input=*/false,
                  /*expected_output_dims=*/{1, 2, 2},
                  /*expected_output=*/{1, 0, 1, 3}},
+      // Transpose Strided
+      TestParams{/*input_dims=*/{1, 2, 2},
+                 /*input=*/{0, 1, 2, 3},
+                 /*filter_dims=*/{1, 2, 1, 1},
+                 /*filter=*/{-1, 1},
+                 /*strides=*/{1, 1, 1, 2},
+                 /*padding=*/"SAME",
+                 /*data_format=*/"NCHW",
+                 /*dilations=*/{1, 1, 1, 1},
+                 /*is_conv2d_backprop_input=*/true,
+                 /*expected_output_dims=*/{1, 2, 4},
+                 /*expected_output=*/{0, 0, -1, 1, -2, 2, -3, 3}},
   };
 
   for (int i = 0; i < kConv2DOKCases; i++) {
     Reset();
-    NodeDef node_def =
-        get_conv2d_nodedef(ok_params[i].strides, ok_params[i].padding,
-                           ok_params[i].data_format, ok_params[i].dilations);
+    NodeDef node_def = get_conv2d_nodedef(
+        ok_params[i].strides, ok_params[i].padding, ok_params[i].data_format,
+        ok_params[i].dilations, ok_params[i].is_conv2d_backprop_input);
     AddTestTensor("input", ok_params[i].input_dims);
     AddTestWeights<float>("weights", ok_params[i].filter_dims,
                           ok_params[i].filter);
+    if (ok_params[i].is_conv2d_backprop_input) {
+      AddTestWeights<float>(
+          "input_sizes",
+          {static_cast<int>(ok_params[i].expected_output.size())},
+          ok_params[i].expected_output);
+    }
     RunValidationAndConversion(node_def);
     TRT_TensorOrWeights output;
     TF_EXPECT_OK(GetTensorOrWeights("my_conv2d", &output));
diff --git a/tensorflow/contrib/tensorrt/test/conv2d_test.py b/tensorflow/contrib/tensorrt/test/conv2d_test.py
index e7993b4620..c32c24fe0c 100644
--- a/tensorflow/contrib/tensorrt/test/conv2d_test.py
+++ b/tensorflow/contrib/tensorrt/test/conv2d_test.py
@@ -26,6 +26,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_nn_ops
+from tensorflow.python.ops import nn_ops
 from tensorflow.python.platform import test
 
 
@@ -187,5 +188,46 @@ class Conv2DStridedNCHWTest(trt_test.TfTrtIntegrationTestBase):
     return ["TRTEngineOp_0"]
 
 
+class Conv2DTranposeTest(trt_test.TfTrtIntegrationTestBase):
+
+  def GetParams(self):
+    """Testing conversion of conv2d_transpose (AKA Conv2DBackpropInput)"""
+    np.random.seed(1234)
+    dtype = dtypes.float32
+    input_name = "input"
+    n, c, h, w = 13, 3, 7, 11
+    num_filters = 8
+    input_dims = [n, c, h, w]
+    output_name = "output"
+    g = ops.Graph()
+    with g.as_default():
+      inp = array_ops.placeholder(
+          dtype=dtype, shape=[None] + input_dims[1:], name=input_name)
+      with g.device("/GPU:0"):
+        weights_shape = [2, 2, num_filters, c]
+        weights = constant_op.constant(
+            np.random.randn(*weights_shape), dtype=dtype)
+        output_shape = constant_op.constant([n, num_filters, h * 2, w * 2],
+                                            dtype=dtypes.int32)
+        output = nn_ops.conv2d_transpose(
+            inp,
+            weights,
+            output_shape,
+            strides=[1, 1, 2, 2],
+            padding="SAME",
+            data_format="NCHW")
+        output = array_ops.identity(output, name=output_name)
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=[input_name],
+        input_dims=[[input_dims]],
+        output_names=[output_name],
+        expected_output_dims=[[[n, num_filters, h * 2, w * 2]]])
+
+  def ExpectedEnginesToBuild(self, run_params):
+    """Return the expected engines to build."""
+    return ["TRTEngineOp_0"]
+
+
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 7b37024e464c400dd8065373f1c3d8792ed5f42b Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 31 Jan 2019 18:45:52 -0800
Subject: [PATCH 1734/2345] [XLA] Fix erf for f16 inputs, and improve precision
 in f32.

Switch to using the f32 implementations.  Previously we were using the f64
implementation with all constants truncated down to f32.

Part of the problem is that it was computing

  erf(x) = x * polynomial1(x^2) / polynomial2(x^2)

and assuming that we had enough precision that neither of the polynomials would
overflow.  On f16, polynomial2 would overflow to inf even for quite reasonable
values of x (e.g. 0.79), thus causing the result to be 0 incorrectly.

This may reduce precision for f64, but that's a trade-off we're willing to make
(especially since we were using f32 coefficients *anyway*).

erf and erfc also now call each other when their inputs are out of range.

Relevant to #25052.

PiperOrigin-RevId: 231896451
---
 .../compiler/tf2xla/kernels/unary_ops.cc      |  30 +---
 tensorflow/compiler/xla/client/lib/math.cc    | 148 ++++++++----------
 .../xla/client/lib/math_exhaustive_test.cc    |   9 +-
 3 files changed, 68 insertions(+), 119 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc
index a7cc8c102a..4544e03491 100644
--- a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc
@@ -113,37 +113,11 @@ XLAJIT_MAKE_UNARY(Tanh, xla::Tanh(x));
 
 XLAJIT_MAKE_UNARY(Real, xla::Real(x));
 XLAJIT_MAKE_UNARY(Imag, xla::Imag(x));
+XLAJIT_MAKE_UNARY(Erf, xla::Erf(x));
+XLAJIT_MAKE_UNARY(Erfc, xla::Erfc(x));
 
 #undef XLAJIT_MAKE_UNARY
 
-// Erf/Erfc.  For x in (-1, 1), the erf approximation is used; erfc polynomial
-// is used outside of this range.
-class ErfOp : public XlaOpKernel {
- public:
-  explicit ErfOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {}
-  void Compile(XlaOpKernelContext* ctx) override {
-    xla::XlaOp x = ctx->Input(0);
-    xla::XlaOp one = xla::ScalarLike(x, 1.0);
-    auto y =
-        xla::Select(xla::Gt(xla::Abs(x), one), one - xla::Erfc(x), xla::Erf(x));
-    ctx->SetOutput(0, y);
-  }
-};
-REGISTER_XLA_OP(Name("Erf"), ErfOp);
-
-class ErfcOp : public XlaOpKernel {
- public:
-  explicit ErfcOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {}
-  void Compile(XlaOpKernelContext* ctx) override {
-    xla::XlaOp x = ctx->Input(0);
-    xla::XlaOp one = xla::ScalarLike(x, 1.0);
-    auto y =
-        xla::Select(xla::Lt(xla::Abs(x), one), one - xla::Erf(x), xla::Erfc(x));
-    ctx->SetOutput(0, y);
-  }
-};
-REGISTER_XLA_OP(Name("Erfc"), ErfcOp);
-
 class LgammaOp : public XlaOpKernel {
  public:
   explicit LgammaOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {}
diff --git a/tensorflow/compiler/xla/client/lib/math.cc b/tensorflow/compiler/xla/client/lib/math.cc
index 675884050e..1489120685 100644
--- a/tensorflow/compiler/xla/client/lib/math.cc
+++ b/tensorflow/compiler/xla/client/lib/math.cc
@@ -87,44 +87,6 @@ XlaOp Square(XlaOp operand) { return operand * operand; }
 
 XlaOp Reciprocal(XlaOp operand) { return ScalarLike(operand, 1.0) / operand; }
 
-namespace {
-
-// Polynomials for computing erf/erfc.  Originally from cephes.
-// Note we use float for compatibility across devices, at the cost of some
-// precision for 64 bit computations.
-//
-// Coefficients are in descending order.
-std::array<float, 9> kErfcPCoefficient = {
-    2.46196981473530512524E-10, 5.64189564831068821977E-1,
-    7.46321056442269912687E0,   4.86371970985681366614E1,
-    1.96520832956077098242E2,   5.26445194995477358631E2,
-    9.34528527171957607540E2,   1.02755188689515710272E3,
-    5.57535335369399327526E2};
-std::array<float, 9> kErfcQCoefficient = {
-    1.00000000000000000000E0, 1.32281951154744992508E1,
-    8.67072140885989742329E1, 3.54937778887819891062E2,
-    9.75708501743205489753E2, 1.82390916687909736289E3,
-    2.24633760818710981792E3, 1.65666309194161350182E3,
-    5.57535340817727675546E2};
-std::array<float, 6> kErfcRCoefficient = {
-    5.64189583547755073984E-1, 1.27536670759978104416E0,
-    5.01905042251180477414E0,  6.16021097993053585195E0,
-    7.40974269950448939160E0,  2.97886665372100240670E0};
-std::array<float, 7> kErfcSCoefficient = {
-    1.00000000000000000000E0, 2.26052863220117276590E0,
-    9.39603524938001434673E0, 1.20489539808096656605E1,
-    1.70814450747565897222E1, 9.60896809063285878198E0,
-    3.36907645100081516050E0};
-std::array<float, 5> kErfTCoefficient = {
-    9.60497373987051638749E0, 9.00260197203842689217E1,
-    2.23200534594684319226E3, 7.00332514112805075473E3,
-    5.55923013010394962768E4};
-std::array<float, 6> kErfUCoefficient = {
-    1.00000000000000000000E0, 3.35617141647503099647E1,
-    5.21357949780152679795E2, 4.59432382970980127987E3,
-    2.26290000613890934246E4, 4.92673942608635921086E4};
-}  // namespace
-
 // Evaluate the polynomial given coefficients and `x`.
 // N.B. Coefficients should be supplied in decreasing order.
 XlaOp EvaluatePolynomial(XlaOp x, absl::Span<const float> coefficients) {
@@ -135,73 +97,85 @@ XlaOp EvaluatePolynomial(XlaOp x, absl::Span<const float> coefficients) {
   return poly;
 }
 
-// Compute an approximation of the error function complement (1 - erf(x)).
+// Computes an approximation of the error function complement (1 - erf(x)).
 //
-// TODO(jlebar): This is not particularly efficient.  The implementation in
-// Cephes that this follows was written for double precision, but our
-// coefficients are specified only to single-precision!  Cephes has a different,
-// simpler implementation for single-precision.
+// Precondition: abs(x) >= 1.  Otherwise, use ErfImpl.
 //
-// Furthermore, we could simplify this further for f16 -- for example, because
-// exp(-4.2 * 4.2) = 0 (f16), the computations in service of the x < 8.0 branch
-// below are unnecessary.
+// This follows Cephes's f32 implementation of erfc, and so it may have errors
+// for double precision.
 //
 // See also these alternate implementations of erf and erfc:
 //
 //   https://stackoverflow.com/questions/35148198
 //   https://stackoverflow.com/questions/35966695
 //
+static XlaOp ErfcImpl(XlaOp x) {
+  // Coefficients for erfc(f32), from Cephes.
+  //
+  // erfc(x) = exp(-x^2) P(1/x), 1 < x < 2
+  static std::array<float, 9> kErfcPCoefficient{
+      +2.326819970068386E-2, -1.387039388740657E-1, +3.687424674597105E-1,
+      -5.824733027278666E-1, +6.210004621745983E-1, -4.944515323274145E-1,
+      +3.404879937665872E-1, -2.741127028184656E-1, +5.638259427386472E-1,
+  };
+  // erfc(x) = exp(-x^2) 1/x P(1/x^2), 2 < x < 14
+  static std::array<float, 8> kErfcRCoefficient{
+      -1.047766399936249E+1, +1.297719955372516E+1, -7.495518717768503E+0,
+      +2.921019019210786E+0, -1.015265279202700E+0, +4.218463358204948E-1,
+      -2.820767439740514E-1, +5.641895067754075E-1,
+  };
+
+  XlaOp abs_x = Abs(x);
+  XlaOp z = Exp(-x * x);
+  XlaOp q = ScalarLike(x, 1) / abs_x;
+  XlaOp y = q * q;
+  XlaOp p = Select(Lt(abs_x, ScalarLike(x, 2.0)),
+                   EvaluatePolynomial(y, kErfcPCoefficient),
+                   EvaluatePolynomial(y, kErfcRCoefficient));
+  y = z * q * p;
+  return Select(Lt(x, ScalarLike(x, 0)), ScalarLike(x, 2.0) - y, y);
+}
+
+// Compute a polynomial approximation of the error function.
+//
+// Precondition: abs(x) <= 1.  Otherwise, use ErfcImpl.
+//
+// This follows Cephes's f32 implementation of erf, so it may have errors for
+// double precision.
+static XlaOp ErfImpl(XlaOp x) {
+  // Coefficients for by erf(f32), from Cephes.
+  //
+  // erf(x) = x P(x^2), 0 < x < 1
+  static std::array<float, 7> kErfTCoefficient{
+      +7.853861353153693E-5, -8.010193625184903E-4, +5.188327685732524E-3,
+      -2.685381193529856E-2, +1.128358514861418E-1, -3.761262582423300E-1,
+      +1.128379165726710E+0,
+  };
+
+  return x * EvaluatePolynomial(x * x, kErfTCoefficient);
+}
+
 XlaOp Erfc(XlaOp x) {
   auto& b = *x.builder();
   return b.ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
     TF_RETURN_IF_ERROR(EnsureOperandIsRealFp("Erfc", x));
-
-    XlaOp abs_x = Abs(x);
-    XlaOp z = Exp(-x * x);
-
-    XlaOp pp = EvaluatePolynomial(abs_x, kErfcPCoefficient);
-    XlaOp pq = EvaluatePolynomial(abs_x, kErfcQCoefficient);
-    XlaOp pr = EvaluatePolynomial(abs_x, kErfcRCoefficient);
-    XlaOp ps = EvaluatePolynomial(abs_x, kErfcSCoefficient);
-
-    XlaOp abs_x_small = Lt(abs_x, ScalarLike(x, 8.0));
-    XlaOp y = Select(abs_x_small, z * pp / pq, z * pr / ps);
-    XlaOp result_no_underflow =
-        Select(Lt(x, ScalarLike(x, 0.0)), ScalarLike(x, 2.0) - y, y);
-
-    // Check for edge cases, namely, exp(-x^2) is exactly 0, or the appropriate
-    // denominator (ps or pq) is inf.  (The check for exp(-x^2) == 0 is
-    // necessary only for x == +/- inf, where this check lets us avoid
-    // multiplying 0 by inf and getting nan.)
-    auto is_pos_inf = [](XlaOp op) {
-      return And(Not(IsFinite(op)), Gt(op, ScalarLike(op, 0)));
-    };
-    XlaOp underflow =
-        Or(Eq(z, ScalarLike(z, 0)), Or(And(is_pos_inf(pq), abs_x_small),
-                                       And(is_pos_inf(ps), Not(abs_x_small))));
-    XlaOp result_underflow =
-        Select(Lt(x, ScalarLike(x, 0)), FullLike(x, 2), FullLike(x, 0));
-
-    return Select(underflow, result_underflow, result_no_underflow);
+    // erfc(x) =
+    //   erfc_impl(x)           if x > 1
+    //   1 - erf_impl(x)        otherwise
+    return Select(Gt(Abs(x), ScalarLike(x, 1)), ErfcImpl(x),
+                  ScalarLike(x, 1) - ErfImpl(x));
   });
 }
 
-// Compute a polynomial approximation of the error function.
 XlaOp Erf(XlaOp x) {
   auto& b = *x.builder();
   return b.ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
-    // Reject non-real non-fp inputs.  (We could extend erf to accept complex
-    // types, but it doesn't seem necessary at this point.)
-    TF_ASSIGN_OR_RETURN(auto shape, b.GetShape(x));
-    if (!ShapeUtil::ElementIsFloating(shape)) {
-      return InvalidArgument(
-          "erf only accepts real floating-point arrays or scalars, but got %s",
-          shape.ToString());
-    }
-    XlaOp z = x * x;
-    XlaOp pt = EvaluatePolynomial(z, kErfTCoefficient);
-    XlaOp pu = EvaluatePolynomial(z, kErfUCoefficient);
-    return x * pt / pu;
+    TF_RETURN_IF_ERROR(EnsureOperandIsRealFp("Erf", x));
+    // erf(x) =
+    //   erf_impl(x)            if x < 1
+    //   1 - erfc_impl(x)       otherwise
+    return Select(Lt(Abs(x), ScalarLike(x, 1)), ErfImpl(x),
+                  ScalarLike(x, 1) - ErfcImpl(x));
   });
 }
 
diff --git a/tensorflow/compiler/xla/client/lib/math_exhaustive_test.cc b/tensorflow/compiler/xla/client/lib/math_exhaustive_test.cc
index 0fb13a73b4..f4ed0db56f 100644
--- a/tensorflow/compiler/xla/client/lib/math_exhaustive_test.cc
+++ b/tensorflow/compiler/xla/client/lib/math_exhaustive_test.cc
@@ -71,7 +71,7 @@ struct Testcase {
   XlaOp (*op)(XlaOp);
   float (*host_op)(float);
 
-  ErrorSpec error{0.01};
+  ErrorSpec error{0.01, 0.01};
 
   // If true, don't test +/-infinity or negative 0.
   bool skip_pos_inf = false;
@@ -145,7 +145,6 @@ XLA_TEST_P(MathExhaustiveTest, DISABLED_ON_INTERPRETER(F16)) {
 //   Testcase{"asinh", Asinh, std::asinh},
 //   Testcase{"sinh", Sinh, std::sinh},
 //   Testcase{"cosh", Cosh, std::cosh}.set_fewer_infs_ok(),
-//   Testcase{"erf", Erf, std::erf},
 //   Testcase{"round_to_even", RoundToEven,
 //            [](float x) { return std::nearbyint(x / 2) * 2; }},
 //
@@ -161,7 +160,8 @@ XLA_TEST_P(MathExhaustiveTest, DISABLED_ON_INTERPRETER(F16)) {
 //
 // TODO(b/123355973): Test math functions not from math.cc (e.g. log).
 // TODO(b/123355973): Test bf16 and f32.
-//
+// TODO(b/123355973): Get rid of skip_infs / skip_neg_zero below if possible.
+// TODO(b/123355973): Reduce lgamma error if possible; it is very high.
 INSTANTIATE_TEST_CASE_P(
     MathExhaustiveTest_Instantiation, MathExhaustiveTest,
     ::testing::ValuesIn(std::vector<Testcase>{
@@ -172,7 +172,8 @@ INSTANTIATE_TEST_CASE_P(
             .set_skip_neg_zero(),
         Testcase{"square", Square, [](float x) { return x * x; }},
         Testcase{"reciprocal", Reciprocal, [](float x) { return 1 / x; }},
-        Testcase{"erfc", Erfc, std::erfc},
+        Testcase{"erf", Erf, std::erf}.set_tolerance(0.001, 0.0001),
+        Testcase{"erfc", Erfc, std::erfc}.set_tolerance(0.001, 0.0001),
         Testcase{"lgamma", Lgamma, std::lgamma}
             .set_tolerance(0.1, 0.15)
             .set_fewer_infs_ok(),
-- 
GitLab


From 6c7d78e1acc046cc9dd17609ceff835109537cca Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 31 Jan 2019 19:01:00 -0800
Subject: [PATCH 1735/2345] [XLA] Add erf and erfc to
 exhaustive_f32_elementwise_op_test.

Relevant to #25052.

PiperOrigin-RevId: 231897859
---
 tensorflow/compiler/xla/tests/BUILD                 |  1 +
 .../xla/tests/exhaustive_f32_elementwise_op_test.cc | 13 +++++++++++++
 2 files changed, 14 insertions(+)

diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 383ad5d35d..e8e779fb2a 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -682,6 +682,7 @@ xla_test(
         ":client_library_test_base",
         ":literal_test_util",
         "//tensorflow/compiler/xla/client:xla_builder",
+        "//tensorflow/compiler/xla/client/lib:math",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
         "//tensorflow/core:lib",
         "@com_google_absl//absl/base",
diff --git a/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc b/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc
index 6200f0e34f..b961e61026 100644
--- a/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc
+++ b/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include <cmath>
 #include "absl/base/casts.h"
+#include "tensorflow/compiler/xla/client/lib/math.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/tests/client_library_test_base.h"
 #include "tensorflow/compiler/xla/tests/literal_test_util.h"
@@ -206,6 +207,18 @@ XLA_TEST_P(ExhaustiveF32ElementwiseOpTest, TanhF32) {
       /*known_incorrect_range=*/{0, 0});
 }
 
+XLA_TEST_P(ExhaustiveF32ElementwiseOpTest, ErfF32) {
+  ExhaustivelyTestF32Op(
+      [](XlaBuilder* builder, const XlaOp& input) { Erf(input); }, std::erf,
+      /*known_incorrect_range=*/{0, 0});
+}
+
+XLA_TEST_P(ExhaustiveF32ElementwiseOpTest, ErfcF32) {
+  ExhaustivelyTestF32Op(
+      [](XlaBuilder* builder, const XlaOp& input) { Erfc(input); }, std::erfc,
+      /*known_incorrect_range=*/{0, 0});
+}
+
 std::vector<std::pair<int64, int64>> CreateExhaustiveParameters() {
   // We break up the 2^32-element space into small'ish chunks to keep peak
   // memory usage low.
-- 
GitLab


From 8053d6eefda70dc7645519a620fa96383295e567 Mon Sep 17 00:00:00 2001
From: Dayananda-V <dayanandasiet@gmail.com>
Date: Thu, 31 Jan 2019 14:25:41 +0530
Subject: [PATCH 1736/2345] TF Framework registry_test missing test case add
 1-list api test case 2-register method empty name parameter test case

---
 tensorflow/python/BUILD                      |  1 +
 tensorflow/python/framework/registry_test.py | 33 ++++++++++----------
 2 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index c89027582c..c1a91c68fb 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -1175,6 +1175,7 @@ tf_py_test(
     srcs = ["framework/registry_test.py"],
     additional_deps = [
         ":framework_for_generated_wrappers",
+        "@absl_py//absl/testing:parameterized",
         "//tensorflow/python:client_testlib",
     ],
     main = "framework/registry_test.py",
diff --git a/tensorflow/python/framework/registry_test.py b/tensorflow/python/framework/registry_test.py
index 1a0d3f200d..924628928c 100644
--- a/tensorflow/python/framework/registry_test.py
+++ b/tensorflow/python/framework/registry_test.py
@@ -19,28 +19,31 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from absl.testing import parameterized
+
 from tensorflow.python.framework import registry
 from tensorflow.python.platform import test
 
 
-class RegistryTest(test.TestCase):
+def bar():
+  pass
+
+class RegistryTest(test.TestCase, parameterized.TestCase):
 
   class Foo(object):
     pass
 
-  def testRegisterClass(self):
-    myreg = registry.Registry('testfoo')
-    with self.assertRaises(LookupError):
-      myreg.lookup('Foo')
-    myreg.register(RegistryTest.Foo, 'Foo')
-    assert myreg.lookup('Foo') == RegistryTest.Foo
-
-  def testRegisterFunction(self):
-    myreg = registry.Registry('testbar')
+  @parameterized.parameters([Foo, bar])
+  def testRegistryBasics(self, candidate):
+    myreg = registry.Registry('testRegistry')
     with self.assertRaises(LookupError):
-      myreg.lookup('Bar')
-    myreg.register(bar, 'Bar')
-    assert myreg.lookup('Bar') == bar
+      myreg.lookup('testKey')
+    myreg.register(candidate)
+    self.assertEqual(myreg.lookup(candidate.__name__), candidate)
+    myreg.register(candidate, "testKey")
+    self.assertEqual(myreg.lookup('testKey'), candidate)
+    self.assertEqual(sorted(myreg.list()),
+                     sorted(["testKey", candidate.__name__]))
 
   def testDuplicate(self):
     myreg = registry.Registry('testbar')
@@ -51,9 +54,5 @@ class RegistryTest(test.TestCase):
       myreg.register(bar, 'Bar')
 
 
-def bar():
-  pass
-
-
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From ac295dbb7abe6307295465dcf4293b0e43a22b98 Mon Sep 17 00:00:00 2001
From: Vijay Vasudevan <vrv@google.com>
Date: Thu, 31 Jan 2019 20:44:35 -0800
Subject: [PATCH 1737/2345] Small documentation addendum.

---
 tensorflow/python/framework/registry_test.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/framework/registry_test.py b/tensorflow/python/framework/registry_test.py
index 924628928c..3b1cf61f56 100644
--- a/tensorflow/python/framework/registry_test.py
+++ b/tensorflow/python/framework/registry_test.py
@@ -33,6 +33,7 @@ class RegistryTest(test.TestCase, parameterized.TestCase):
   class Foo(object):
     pass
 
+  # Test the registry basics on both classes (Foo) and functions (bar).
   @parameterized.parameters([Foo, bar])
   def testRegistryBasics(self, candidate):
     myreg = registry.Registry('testRegistry')
-- 
GitLab


From 037894c5e792ce546bdd012e265f27d85295cb25 Mon Sep 17 00:00:00 2001
From: Abolfazl Shahbazi <abolfazl.shahbazi@intel.com>
Date: Thu, 31 Jan 2019 20:47:02 -0800
Subject: [PATCH 1738/2345] Optimize wording in README.md for APIs and their
 backwards compatibility

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 4e37b239b1..7357298b1a 100644
--- a/README.md
+++ b/README.md
@@ -24,7 +24,7 @@ organization for the purposes of conducting machine learning and deep neural
 networks research.  The system is general enough to be applicable in a wide
 variety of other domains, as well.
 
-TensorFlow provides stable Python API and C APIs as well as without API backwards compatibility guarantee like C++, Go, Java, JavaScript and Swift.
+TensorFlow provides stable Python and C APIs as well as non-guaranteed backwards compatible API's for C++, Go, Java, JavaScript and Swift.
 
 Keep up to date with release announcements and security updates by
 subscribing to
-- 
GitLab


From 67f67690b29257d8ee039bcb74d19e0b5fbbe4e7 Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Thu, 31 Jan 2019 21:25:32 -0800
Subject: [PATCH 1739/2345] TF/gRPC: add RPC caching and retries for TF worker
 service.

Adds RPC state tracking and caching to the TF grpc worker service and enables
client side retrying of worker operations for recoverable errors.

PiperOrigin-RevId: 231910518
---
 tensorflow/core/distributed_runtime/rpc/BUILD |  14 ++
 .../rpc/grpc_remote_worker.cc                 |  12 +-
 .../rpc/grpc_response_cache.cc                | 184 ++++++++++++++++++
 .../rpc/grpc_response_cache.h                 |  92 +++++++++
 .../rpc/grpc_rpc_factory.cc                   |   2 +-
 .../core/distributed_runtime/rpc/grpc_state.h |  85 ++++++--
 .../rpc/grpc_worker_service.cc                |  95 ++++++---
 .../rpc/grpc_worker_service.h                 |   7 +-
 8 files changed, 438 insertions(+), 53 deletions(-)
 create mode 100644 tensorflow/core/distributed_runtime/rpc/grpc_response_cache.cc
 create mode 100644 tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h

diff --git a/tensorflow/core/distributed_runtime/rpc/BUILD b/tensorflow/core/distributed_runtime/rpc/BUILD
index a081ec7b67..7e4070c875 100644
--- a/tensorflow/core/distributed_runtime/rpc/BUILD
+++ b/tensorflow/core/distributed_runtime/rpc/BUILD
@@ -159,6 +159,17 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "grpc_response_cache",
+    srcs = ["grpc_response_cache.cc"],
+    hdrs = ["grpc_response_cache.h"],
+    deps = [
+        ":grpc_util",
+        "//tensorflow/core:lib",
+        "@com_google_absl//absl/time",
+    ],
+)
+
 tf_cuda_library(
     name = "grpc_worker_service",
     srcs = ["grpc_worker_service.cc"],
@@ -166,6 +177,7 @@ tf_cuda_library(
     deps = [
         ":async_service_interface",
         ":grpc_call",
+        ":grpc_response_cache",
         ":grpc_tensor_coding",
         ":grpc_util",
         ":grpc_worker_service_impl",
@@ -183,6 +195,8 @@ tf_cuda_library(
         "//tensorflow/core/distributed_runtime:worker_cache",
         "//tensorflow/core/distributed_runtime:worker_env",
         "//tensorflow/core/distributed_runtime:worker_session",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/time",
     ],
 )
 
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.cc b/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.cc
index 2daefcb399..2479e7368b 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.cc
@@ -39,6 +39,8 @@ limitations under the License.
 
 namespace tensorflow {
 
+const int kMaxWorkerRpcRetries = 10;
+
 class GrpcRemoteWorker : public WorkerInterface {
  public:
   explicit GrpcRemoteWorker(SharedGrpcChannelPtr channel,
@@ -259,17 +261,19 @@ class GrpcRemoteWorker : public WorkerInterface {
   // given callback, `done`, will be called when the RPC completes.
   void IssueRequest(const protobuf::Message* request,
                     protobuf::Message* response, const ::grpc::string& method,
-                    StatusCallback done, CallOptions* call_opts = nullptr) {
+                    StatusCallback done, CallOptions* call_opts = nullptr,
+                    int max_retries = kMaxWorkerRpcRetries) {
     new RPCState<protobuf::Message>(&stub_, cq_, method, *request, response,
                                     std::move(done), call_opts,
-                                    callback_threadpool_);
+                                    callback_threadpool_, max_retries);
   }
   void IssueRequest(const protobuf::Message* request, TensorResponse* response,
                     const ::grpc::string& method, StatusCallback done,
-                    CallOptions* call_opts = nullptr) {
+                    CallOptions* call_opts = nullptr,
+                    int max_retries = kMaxWorkerRpcRetries) {
     new RPCState<TensorResponse>(&stub_, cq_, method, *request, response,
                                  std::move(done), call_opts,
-                                 callback_threadpool_);
+                                 callback_threadpool_, max_retries);
   }
 
   // Helper function for initializing the RpcMethod objects below.
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.cc b/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.cc
new file mode 100644
index 0000000000..9e8f7eafa5
--- /dev/null
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.cc
@@ -0,0 +1,184 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h"
+
+#include "absl/time/clock.h"
+
+namespace tensorflow {
+
+struct WorkerCacheEntry {
+  enum class State {
+    PENDING = 0,
+    ACTIVE = 1,
+    FINISHED = 2,
+  };
+
+  State state = State::PENDING;
+  absl::Time expires;
+
+  ::grpc::ByteBuffer response_buf;
+  Status response_status;
+
+  // Additional retries may arrive while a request is still executing.  The
+  // callbacks for these calls are queued in `callbacks` and evaluated after
+  // the original request is completed.
+  std::vector<std::pair<RPCResponse, StatusCallback>> callbacks;
+};
+
+void RPCResponse::Encode(::grpc::ByteBuffer* tgt) const {
+  if (buf_ != nullptr) {
+    *tgt = *buf_;
+  } else {
+    CHECK(msg_ != nullptr);
+    ::grpc::Slice slice(msg_->ByteSizeLong());
+    msg_->SerializeWithCachedSizesToArray(
+        const_cast<uint8*>(reinterpret_cast<const uint8*>(slice.begin())));
+    ::grpc::ByteBuffer tmp(&slice, 1);
+    tgt->Swap(&tmp);
+  }
+}
+
+void RPCResponse::CopyFrom(const ::grpc::ByteBuffer& src) {
+  if (buf_ != nullptr) {
+    *buf_ = src;
+    return;
+  }
+
+  CHECK(msg_ != nullptr);
+  // We create a single slice when encoding protocol messages.
+  std::vector<::grpc::Slice> slices;
+  if (src.Dump(&slices).ok()) {
+    msg_->ParseFromArray(slices[0].begin(), slices[0].size());
+  } else {
+    LOG(ERROR) << "Failed to decode cached buffer.";
+  }
+}
+
+void GrpcResponseCache::LookupOrCompute(const string& key, RPCResponse response,
+                                        ComputeFunc compute_func,
+                                        StatusCallback done_cb) {
+  VLOG(1) << "Lookup " << key;
+  std::shared_ptr<WorkerCacheEntry> req;
+  MaybeCleanup();
+  {
+    mutex_lock m(mu_);
+
+    if (requests_.find(key) != requests_.end()) {
+      req = requests_[key];
+    } else {
+      req.reset(new WorkerCacheEntry);
+      requests_[key] = req;
+    }
+
+    if (req->state == WorkerCacheEntry::State::FINISHED) {
+      if (req->expires > absl::Now()) {
+        VLOG(1) << "Reuse cached response for " << key;
+        response.CopyFrom(req->response_buf);
+        done_cb(req->response_status);
+        return;
+      }
+      VLOG(1) << "Found expired cache entry for " << key;
+      req->state = WorkerCacheEntry::State::PENDING;
+      req->response_buf.Clear();
+    }
+
+    req->callbacks.push_back(std::make_pair(response, done_cb));
+
+    if (req->state == WorkerCacheEntry::State::ACTIVE) {
+      VLOG(1) << "Found active request for " << key
+              << ".  Adding entry to response queue.";
+      return;
+    }
+
+    VLOG(2) << "No cache entry for " << key << ", running user computation.";
+    req->state = WorkerCacheEntry::State::ACTIVE;
+    req->expires = absl::Now() + expire_duration_;
+  }
+
+  compute_func([this, key, req, response](Status status) {
+    mutex_lock m(mu_);
+    response.Encode(&req->response_buf);
+    current_bytes_ += req->response_buf.Length();
+
+    req->response_status = status;
+    req->state = WorkerCacheEntry::State::FINISHED;
+
+    VLOG(1) << "Operation for " << key << " finished. "
+            << "Status: " << status << ", " << req->response_buf.Length()
+            << " response bytes, " << req->callbacks.size()
+            << " pending callbacks.";
+    for (auto& cb : req->callbacks) {
+      cb.first.CopyFrom(req->response_buf);
+      cb.second(req->response_status);
+    }
+    req->callbacks.clear();
+  });
+}
+
+// Remove all stale or expired cache entries if the cache is full.
+void GrpcResponseCache::MaybeCleanup() {
+  mutex_lock m(mu_);
+  if (current_bytes_ < max_bytes_) {
+    return;
+  }
+
+  VLOG(1) << "Cleanup: " << current_bytes_ << " -> " << max_bytes_;
+  std::vector<std::pair<string, std::shared_ptr<WorkerCacheEntry>>>
+      ordered_entries;
+  ordered_entries.reserve(requests_.size());
+  for (const auto& p : requests_) {
+    ordered_entries.push_back(std::make_pair(p.first, p.second));
+  }
+
+  std::sort(ordered_entries.begin(), ordered_entries.end(),
+            [](const std::pair<string, std::shared_ptr<WorkerCacheEntry>>& a,
+               const std::pair<string, std::shared_ptr<WorkerCacheEntry>>& b) {
+              return a.second->expires > b.second->expires;
+            });
+
+  std::unordered_map<string, std::shared_ptr<WorkerCacheEntry>> kept;
+  absl::Time now = absl::Now();
+  int64 bytes_used = 0;
+
+  // Always keep active requests.
+  for (auto& pair : ordered_entries) {
+    if (pair.second->state != WorkerCacheEntry::State::FINISHED) {
+      kept.insert(pair);
+    }
+  }
+
+  // Keep unexpired, finished requests up to half of max_bytes_.  This reduces
+  // chances of overfilling the cache when active requests complete and
+  // amortizes cache cleanup cost.
+  for (auto& pair : ordered_entries) {
+    if (pair.second->expires < now || bytes_used >= max_bytes_ / 2) {
+      break;
+    }
+
+    if (pair.second->state == WorkerCacheEntry::State::FINISHED) {
+      kept.insert(pair);
+      bytes_used += pair.second->response_buf.Length();
+    }
+  }
+
+  VLOG(1) << "Cleaned cache.  Bytes used: " << current_bytes_ << " -> "
+          << bytes_used << ". Cache size: " << requests_.size() << " -> "
+          << kept.size();
+  current_bytes_ = bytes_used;
+  std::swap(requests_, kept);
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h b/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h
new file mode 100644
index 0000000000..a5ea79c319
--- /dev/null
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h
@@ -0,0 +1,92 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_RESPONSE_CACHE_H_
+#define TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_RESPONSE_CACHE_H_
+
+#include <memory>
+#include <unordered_map>
+#include <vector>
+
+#include "absl/time/time.h"
+#include "tensorflow/core/distributed_runtime/rpc/grpc_util.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/platform/protobuf.h"
+
+// gRPC response caching.  Most WorkerService methods cannot be retried directly
+// as they will fail or deadlock.  To enable retrying, we can instead cache
+// responses for a short period of time and reply to duplicate requests from the
+// cache.
+namespace tensorflow {
+
+// Union type to aid caching of either raw buffers (for RecvTensor RPCs) and
+// protocol buffer messages (for all other RPCs).
+class RPCResponse {
+ public:
+  explicit RPCResponse() : buf_(nullptr), msg_(nullptr) {}
+  explicit RPCResponse(::grpc::ByteBuffer* b) : buf_(b), msg_(nullptr) {}
+  explicit RPCResponse(protobuf::Message* m) : buf_(nullptr), msg_(m) {}
+
+  // Encode this response into the target buffer.
+  void Encode(::grpc::ByteBuffer* tgt) const;
+
+  // Copy from `src`: if this is a buffer, make a shallow copy.
+  // For protocol messages, parse the response from `src`.
+  void CopyFrom(const ::grpc::ByteBuffer& src);
+
+ private:
+  ::grpc::ByteBuffer* buf_;
+  protobuf::Message* msg_;
+};
+
+typedef std::function<void(StatusCallback)> ComputeFunc;
+struct WorkerCacheEntry;
+
+// Track and cache the state of worker service RPCs.  An RPC can be in 3 states:
+//
+// * PENDING: this is the first call of the RPC, and it will transition to
+// * ACTIVE: another thread is active processing this RPC
+// * FINISHED: the worker has finished processing the method
+//
+// The response from completed RPCs are LRU cached until either `max_bytes`
+// bytes are in use by the cache or they expire (according to `expire_time`).
+class GrpcResponseCache {
+ public:
+  GrpcResponseCache(int64 max_bytes, absl::Duration expire_duration)
+      : max_bytes_(max_bytes), expire_duration_(expire_duration) {}
+
+  // Lookup the result for key.
+  // If it is finished, invoke `done_cb` immediately after filling `response`.
+  // If active, done_db will be invoked when the current call completes.
+  // Otherwise, invoke `compute_func` to fill the cache and invoke done_cb.
+  void LookupOrCompute(const string& key, RPCResponse response,
+                       ComputeFunc compute_func, StatusCallback done_cb);
+
+  // Remove all stale or expired cache entries if the cache is full.
+  void MaybeCleanup();
+
+ private:
+  int64 current_bytes_ GUARDED_BY(mu_) = 0;
+  const int64 max_bytes_;
+  const absl::Duration expire_duration_;
+
+  std::unordered_map<string, std::shared_ptr<WorkerCacheEntry>> requests_
+      GUARDED_BY(mu_);
+  mutex mu_;
+};
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_RESPONSE_CACHE_H_
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_rpc_factory.cc b/tensorflow/core/distributed_runtime/rpc/grpc_rpc_factory.cc
index 7f63cc9344..3635caf3d1 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_rpc_factory.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_rpc_factory.cc
@@ -210,7 +210,7 @@ void GrpcRPCFactory::StartCall(const Tensor& address_t, const Tensor& method_t,
       get_stub(index), &completion_queue_, *get_method_ptr(index),
       call->request(), call->response(),
       /*done=*/[call](const Status& s) { call->Done(s); }, call->call_opts(),
-      nullptr /*threadpool*/, fail_fast_, timeout_in_ms_);
+      nullptr /*threadpool*/, fail_fast_, timeout_in_ms_, 0 /* max_retries */);
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_state.h b/tensorflow/core/distributed_runtime/rpc/grpc_state.h
index d73638651f..0ca64dc159 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_state.h
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_state.h
@@ -32,6 +32,9 @@ limitations under the License.
 namespace tensorflow {
 
 // Object allocated per active RPC.
+// Manage the state of a single asynchronous RPC request.  If `max_retries`
+// is greater than 0, the request will be retried for any transient failures
+// as long as the overall deadline has not elapsed.
 template <class Response>
 class RPCState : public GrpcClientCQTag {
  public:
@@ -39,34 +42,55 @@ class RPCState : public GrpcClientCQTag {
   RPCState(::grpc::GenericStub* stub, ::grpc::CompletionQueue* cq,
            const ::grpc::string& method, const protobuf::Message& request,
            Response* response, StatusCallback done, CallOptions* call_opts,
-           thread::ThreadPool* threadpool)
+           thread::ThreadPool* threadpool, int32 max_retries = 0)
       : RPCState(stub, cq, method, request, response, std::move(done),
                  call_opts, threadpool, /*fail_fast=*/false,
-                 /*timeout_in_ms=*/0) {}
+                 /*timeout_in_ms=*/0, max_retries) {}
 
   template <typename Request>
   RPCState(::grpc::GenericStub* stub, ::grpc::CompletionQueue* cq,
            const ::grpc::string& method, const Request& request,
            Response* response, StatusCallback done, CallOptions* call_opts,
-           thread::ThreadPool* threadpool, bool fail_fast, int64 timeout_in_ms)
-      : call_opts_(call_opts), threadpool_(threadpool), done_(std::move(done)) {
-    context_.set_fail_fast(fail_fast);
-    if (timeout_in_ms > 0) {
-      context_.set_deadline(gpr_time_from_millis(timeout_in_ms, GPR_TIMESPAN));
-    }
-
-    if (call_opts) {
-      call_opts->SetCancelCallback([this]() { context_.TryCancel(); });
-    }
-
+           thread::ThreadPool* threadpool, bool fail_fast, int64 timeout_in_ms,
+           int32 max_retries)
+      : call_opts_(call_opts),
+        threadpool_(threadpool),
+        done_(std::move(done)),
+        cq_(cq),
+        stub_(stub),
+        method_(method),
+        max_retries_(max_retries),
+        timeout_in_ms_(timeout_in_ms),
+        fail_fast_(fail_fast) {
     response_ = response;
     ::grpc::Status s = GrpcMaybeUnparseProto(request, &request_buf_);
     if (!s.ok()) {
       LOG(ERROR) << "GrpcMaybeUnparseProto returned with non-ok status: "
                  << s.error_message();
+      // Skip retry logic if we fail to parse our request.
+      done_(FromGrpcStatus(s));
+      delete this;
+      return;
+    }
+    StartCall();
+  }
+
+  void StartCall() {
+    context_.reset(new ::grpc::ClientContext());
+    context_->set_fail_fast(fail_fast_);
+
+    if (timeout_in_ms_ > 0) {
+      context_->set_deadline(
+          gpr_time_from_millis(timeout_in_ms_, GPR_TIMESPAN));
     }
-    call_ =
-        std::move(stub->PrepareUnaryCall(&context_, method, request_buf_, cq));
+    if (call_opts_) {
+      call_opts_->SetCancelCallback([this]() { context_->TryCancel(); });
+    }
+
+    VLOG(2) << "Starting call: " << method_;
+
+    call_ = std::move(
+        stub_->PrepareUnaryCall(context_.get(), method_, request_buf_, cq_));
     call_->StartCall();
     call_->Finish(&response_buf_, &status_, this);
   }
@@ -89,16 +113,26 @@ class RPCState : public GrpcClientCQTag {
         threadpool_->Schedule([this]() { ParseAndCallDone(); });
       } else {
         ParseAndCallDone();
-        return;
       }
-    } else {
-      VLOG(2) << "Call returned with non-ok status: " << s;
+      return;
+    }
 
-      // Attach additional GRPC error information if any
+    VLOG(1) << method_ << " returned with non-ok status: " << s
+            << " Retries: " << num_retries_ << " Max: " << max_retries_ << "\n"
+            << context_->debug_error_string();
+    // Retry if we have any attempts left
+    if (++num_retries_ <= max_retries_ &&
+        (errors::IsUnavailable(s) || errors::IsUnknown(s))) {
+      response_buf_.Clear();
+      VLOG(1) << "Retrying call for " << method_ << "Retry: " << num_retries_
+              << " of " << max_retries_;
+      StartCall();
+    } else {
+      // Attach additional GRPC error information if any to the final status
       s = Status(s.code(),
                  strings::StrCat(s.error_message(),
                                  "\nAdditional GRPC error information:\n",
-                                 context_.debug_error_string()));
+                                 context_->debug_error_string()));
       done_(s);
       delete this;
     }
@@ -115,7 +149,7 @@ class RPCState : public GrpcClientCQTag {
 
  private:
   CallOptions* call_opts_;
-  ::grpc::ClientContext context_;
+  std::unique_ptr<::grpc::ClientContext> context_;
   thread::ThreadPool* threadpool_;
   std::unique_ptr<::grpc::GenericClientAsyncResponseReader> call_;
   Response* response_;
@@ -123,6 +157,15 @@ class RPCState : public GrpcClientCQTag {
   ::grpc::ByteBuffer response_buf_;
   ::grpc::Status status_;
   StatusCallback done_;
+  int64 timeout_in_ms_;
+
+  size_t num_retries_ = 0;
+  size_t max_retries_;
+
+  ::grpc::CompletionQueue* cq_;
+  ::grpc::GenericStub* stub_;
+  ::grpc::string method_;
+  bool fail_fast_;
 };
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc
index 34bf629c8a..fdd36ccecd 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc
@@ -16,11 +16,14 @@ limitations under the License.
 #include "tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h"
 
 #include <deque>
+#include <memory>
 #include <unordered_map>
+#include <vector>
 
 #include "grpcpp/alarm.h"
 #include "grpcpp/server_builder.h"
 
+#include "absl/container/flat_hash_map.h"
 #include "tensorflow/core/common_runtime/buf_rendezvous.h"
 #include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/common_runtime/device_mgr.h"
@@ -32,6 +35,7 @@ limitations under the License.
 #include "tensorflow/core/distributed_runtime/rendezvous_mgr_interface.h"
 #include "tensorflow/core/distributed_runtime/rpc/async_service_interface.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_call.h"
+#include "tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_tensor_coding.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_util.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h"
@@ -42,8 +46,12 @@ limitations under the License.
 #include "tensorflow/core/framework/collective.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/gtl/map_util.h"
+#include "tensorflow/core/lib/strings/strcat.h"
+#include "tensorflow/core/lib/strings/stringprintf.h"
 #include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/platform/tracing.h"
 #include "tensorflow/core/protobuf/transport_options.pb.h"
 #include "tensorflow/core/protobuf/worker.pb.h"
@@ -91,10 +99,11 @@ class GrpcWorkerServiceThread {
  public:
   explicit GrpcWorkerServiceThread(
       GrpcWorker* worker, ::grpc::ServerBuilder* builder,
-      std::unordered_map<int, int> queue_depth,
+      std::unordered_map<int, int> queue_depth, GrpcResponseCache* cache,
       grpc::WorkerService::AsyncService* worker_service)
       : worker_(worker),
         queue_depth_(queue_depth),
+        cache_(cache),
         worker_service_(worker_service),
         is_shutdown_(false) {
     cq_ = builder->AddCompletionQueue();
@@ -220,18 +229,32 @@ class GrpcWorkerServiceThread {
       NonOwnedProtoRunGraphResponse* wrapped_response =
           new NonOwnedProtoRunGraphResponse(&call->response);
       call->SetCancelCallback([call_opts]() { call_opts->StartCancel(); });
-      worker_->RunGraphAsync(call_opts, wrapped_request, wrapped_response,
-                             [call, call_opts, wrapped_request,
-                              wrapped_response](const Status& s) {
-                               if (!s.ok()) {
-                                 VLOG(1) << "Bad response from RunGraph:" << s;
-                               }
-                               call->ClearCancelCallback();
-                               delete call_opts;
-                               delete wrapped_request;
-                               delete wrapped_response;
-                               call->SendResponse(ToGrpcStatus(s));
-                             });
+      auto done_cb = [call, call_opts, wrapped_request,
+                      wrapped_response](const Status& s) {
+        VLOG(1) << "RunGraph::Done";
+        if (!s.ok()) {
+          VLOG(1) << "Bad response from RunGraph:" << s;
+        }
+        call->ClearCancelCallback();
+        delete call_opts;
+        delete wrapped_request;
+        delete wrapped_response;
+        call->SendResponse(ToGrpcStatus(s));
+      };
+
+      auto compute_fn = [this, call_opts, wrapped_request,
+                         wrapped_response](StatusCallback done) {
+        worker_->RunGraphAsync(call_opts, wrapped_request, wrapped_response,
+                               done);
+      };
+
+      if (cache_) {
+        string request_key = call->request.ShortDebugString();
+        cache_->LookupOrCompute(request_key, RPCResponse(&call->response),
+                                compute_fn, done_cb);
+      } else {
+        compute_fn(done_cb);
+      }
     });
     ENQUEUE_REQUEST(RunGraph, true);
   }
@@ -241,16 +264,28 @@ class GrpcWorkerServiceThread {
     Schedule([this, call]() {
       CallOptions* call_opts = new CallOptions;
       call->SetCancelCallback([call_opts]() { call_opts->StartCancel(); });
-      worker_->GrpcRecvTensorAsync(
-          call_opts, &call->request, &call->response,
-          [call, call_opts](const Status& s) {
-            call->ClearCancelCallback();
-            delete call_opts;
-            if (!s.ok()) {
-              VLOG(1) << "Bad response from RecvTensor:" << s;
-            }
-            call->SendResponse(ToGrpcStatus(s));
-          });
+
+      auto done_cb = [call, call_opts](const Status& s) {
+        call->ClearCancelCallback();
+        delete call_opts;
+        if (!s.ok()) {
+          VLOG(1) << "Bad response from RecvTensor:" << s;
+        }
+        call->SendResponse(ToGrpcStatus(s));
+      };
+
+      auto compute_fn = [this, &call_opts, &call](StatusCallback done) {
+        worker_->GrpcRecvTensorAsync(call_opts, &call->request, &call->response,
+                                     done);
+      };
+
+      if (cache_) {
+        string request_key = call->request.ShortDebugString();
+        cache_->LookupOrCompute(request_key, RPCResponse(&call->response),
+                                compute_fn, done_cb);
+      } else {
+        compute_fn(done_cb);
+      }
     });
     EnqueueRecvTensorRequestRaw();
   }
@@ -328,6 +363,7 @@ class GrpcWorkerServiceThread {
   std::unique_ptr<::grpc::ServerCompletionQueue> cq_;
   std::unique_ptr<Thread> thread_;
   std::unordered_map<int, int> queue_depth_;
+  GrpcResponseCache* cache_;
   grpc::WorkerService::AsyncService* const worker_service_;
 
   mutex shutdown_mu_;
@@ -341,9 +377,15 @@ class GrpcWorkerService : public AsyncServiceInterface {
                     GrpcWorkerServiceOptions options)
       : is_shutdown_(false) {
     builder->RegisterService(&worker_service_);
-    for (int i = 0; i < options.num_worker_threads; i++) {
-      threads_.emplace_back(new GrpcWorkerServiceThread(
-          worker, builder, options.queue_depth, &worker_service_));
+    if (options.response_cache_bytes > 0) {
+      cache_.reset(new GrpcResponseCache(options.response_cache_bytes,
+                                         options.response_cache_duration));
+    }
+
+    for (int i = 0; i < options.num_serving_threads; i++) {
+      threads_.emplace_back(
+          new GrpcWorkerServiceThread(worker, builder, options.queue_depth,
+                                      cache_.get(), &worker_service_));
     }
   }
 
@@ -378,6 +420,7 @@ class GrpcWorkerService : public AsyncServiceInterface {
   grpc::WorkerService::AsyncService worker_service_;
   std::vector<std::unique_ptr<GrpcWorkerServiceThread>> threads_;
 
+  std::unique_ptr<GrpcResponseCache> cache_;
   mutex service_shutdown_mu_;
   bool is_shutdown_ GUARDED_BY(service_shutdown_mu_);
 
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h
index 88beb6c216..467927a8e8 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h
@@ -16,8 +16,11 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_WORKER_SERVICE_H_
 #define TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_WORKER_SERVICE_H_
 
+#include <memory>
 #include <unordered_map>
+#include "absl/time/time.h"
 #include "tensorflow/core/distributed_runtime/recent_request_ids.h"
+#include "tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h"
 #include "tensorflow/core/distributed_runtime/worker.h"
 
@@ -63,7 +66,9 @@ struct GrpcWorkerServiceOptions {
   // Map from GrpcWorkerMethod id to queue depth.  If set this overrides the
   // default queue depth for a method.
   std::unordered_map<int, int> queue_depth;
-  int num_worker_threads = 8;
+  int num_serving_threads = 8;
+  int64 response_cache_bytes = 0;
+  absl::Duration response_cache_duration = absl::ZeroDuration();
 };
 
 // Returns an implementation of WorkerService rpc service.
-- 
GitLab


From 6e84333e15bc1609b49351dfa82fe8ac37e83a72 Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Thu, 31 Jan 2019 21:30:00 -0800
Subject: [PATCH 1740/2345] Add a shuffle layer for output tensors which are
 also input tensors.

PiperOrigin-RevId: 231910868
---
 .../tf2tensorrt/convert/convert_nodes.cc      | 25 ++++++---
 .../tf2tensorrt/convert/convert_nodes_test.cc | 55 +++++++++++++++++++
 2 files changed, 72 insertions(+), 8 deletions(-)

diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
index 7594f3d44d..d1d8ab9d3c 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
@@ -24,7 +24,9 @@ limitations under the License.
 #include <utility>
 #include <vector>
 
+#include "absl/strings/match.h"
 #include "absl/strings/str_cat.h"
+#include "absl/strings/string_view.h"
 #include "tensorflow/compiler/tf2tensorrt/convert/utils.h"
 #include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.h"
 #include "tensorflow/compiler/tf2tensorrt/utils/trt_logger.h"
@@ -82,6 +84,13 @@ namespace tensorrt {
 const char* const kInputPHName = "TensorRTInputPH_";
 const char* const kOutputPHName = "TensorRTOutputPH_";
 
+bool IsEngineInput(absl::string_view name) {
+  return absl::StartsWith(name, kInputPHName);
+}
+bool IsEngineOutput(absl::string_view name) {
+  return absl::StartsWith(name, kOutputPHName);
+}
+
 namespace convert {
 using absl::StrAppend;
 using absl::StrCat;
@@ -900,7 +909,7 @@ Status Converter::ConvertNode(const NodeDef& node_def) {
     // in ConvertIdentity.
     if (output.is_tensor()) {
       const char* tensor_name = output.tensor()->getName();
-      if (!tensorflow::str_util::StartsWith(tensor_name, kInputPHName)) {
+      if (!IsEngineInput(tensor_name)) {
         // TRT initializes tensor names as "(Unnamed ITensor* N)". We rename
         // them to match their corresponding TensorFlow name.
         // Note: ITensors that we create internally within TF-TRT which are
@@ -960,15 +969,17 @@ Status Converter::RenameAndMarkOutputTensors(
       return errors::NotFound("Output tensor not found: ",
                               output.source_tensor_name);
     }
-    // Check if this tensor has already been marked as an output.
+    // Check if this tensor has already been marked as an input or output.
+    //
     // ConvertIdentity can cause the same tensor to be repeated in
     // output_tensors, which can cause us to overwrite the name of the output
     // tensor binding. For example, if we rename OutputPH_0 to OutputPH_1 then
     // we won't be able to locate OutputPH_0 during runtime. To fix this,
     // duplicate the tensor using no-op shuffle.
+    //
     // TODO(tmorris): Remove this work-around once we use TRT's IIdentityLayer
     // in ConvertIdentity.
-    if (tensorflow::str_util::StartsWith(tensor->getName(), kOutputPHName)) {
+    if (IsEngineInput(tensor->getName()) || IsEngineOutput(tensor->getName())) {
       // Using shuffle layer for identity by not setting reshape or transpose.
       nvinfer1::IShuffleLayer* layer = network()->addShuffle(*tensor);
       TFTRT_RETURN_ERROR_IF_NULLPTR(
@@ -3753,8 +3764,7 @@ tensorflow::Status ConvertGraphDefToEngine(
   for (const auto& node_def : gdef.node()) {
     string node_name = node_def.name();
     VLOG(2) << "Converting op name=" << node_name << ", op=" << node_def.op();
-    if (tensorflow::str_util::StartsWith(node_name, kInputPHName) &&
-        (node_def.op() == "Placeholder")) {
+    if (IsEngineInput(node_name) && (node_def.op() == "Placeholder")) {
       int32 slot_number = -1;
       if (!tensorflow::strings::safe_strto32(  // non-absl ok
               node_name.c_str() + strlen(kInputPHName), &slot_number)) {
@@ -3782,8 +3792,7 @@ tensorflow::Status ConvertGraphDefToEngine(
       // engines offline, by calling sess.run() and cache/serialize the engines.
       TF_RETURN_IF_ERROR(
           converter.AddInputTensor(node_name, trt_dtype, trt_dims, batch_size));
-    } else if (tensorflow::str_util::StartsWith(node_name, kOutputPHName) &&
-               (node_def.op() == "Identity")) {
+    } else if (IsEngineOutput(node_name) && (node_def.op() == "Identity")) {
       int32 slot_number = -1;
       if (!tensorflow::strings::safe_strto32(  // non-absl ok
               node_name.c_str() + strlen(kOutputPHName), &slot_number)) {
@@ -3937,7 +3946,7 @@ tensorflow::Status ConvertSegmentToGraphDef(
       TensorId input = ParseTensorName(snode->input(input_idx));
       if (!subgraph_node_names.count(
               string(input.first.data(), input.first.size())) &&
-          !str_util::StartsWith(input.first, kInputPHName)) {
+          !IsEngineInput(input.first)) {
         if (input.second == Graph::kControlSlot) {
           VLOG(1) << "... removing control inputs " << input.first
                   << " from subgraph.";
diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
index a297bb7990..37ac63c605 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
@@ -21,7 +21,10 @@ limitations under the License.
 
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
+#include "absl/strings/match.h"
+#include "absl/strings/numbers.h"
 #include "absl/strings/str_cat.h"
+#include "absl/strings/string_view.h"
 #include "absl/types/span.h"
 #include "tensorflow/cc/framework/ops.h"
 #include "tensorflow/cc/framework/scope.h"
@@ -37,6 +40,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/costs/graph_properties.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/protobuf.h"
 #include "tensorflow/core/platform/test.h"
@@ -928,6 +932,57 @@ TEST_F(ConverterTest, CreateConstantLayer) {
   }
 }
 
+class ConvertGraphDefToEngineTest : public ::testing::Test {
+ public:
+  Status RunConvertGraphDefToEngine(Scope* s) {
+    GraphDef gdef;
+    TF_EXPECT_OK(s->ToGraphDef(&gdef));
+    std::vector<tensorflow::PartialTensorShape> input_shapes;
+    int batch_size = -1;
+    for (const NodeDef& node : gdef.node()) {
+      absl::string_view node_name(node.name());
+      if (str_util::ConsumePrefix(&node_name, kInputPHName)) {
+        int port = -1;
+        EXPECT_TRUE(absl::SimpleAtoi(node_name, &port)) << node.name();
+        if (input_shapes.size() < port + 1) input_shapes.resize(port + 1);
+        input_shapes[port] =
+            PartialTensorShape(node.attr().at("shape").shape());
+        if (batch_size == -1) {
+          batch_size = input_shapes[port].dim_size(0);
+        } else {
+          EXPECT_EQ(batch_size, input_shapes[port].dim_size(0));
+        }
+      }
+    }
+    // TODO(laigd): execute the engine and get outputs.
+    return ConvertGraphDefToEngine(
+        gdef, TrtPrecisionMode::FP32, /*max_batch_size=*/1,
+        /*max_workspace_size_bytes=*/64 << 20, input_shapes, &logger_,
+        /*allocator=*/nullptr, /*calibrator=*/nullptr, &engine_,
+        /*use_calibration=*/false, /*convert_successfully=*/nullptr);
+  }
+
+ protected:
+  TrtUniquePtrType<nvinfer1::ICudaEngine> engine_;
+
+ private:
+  Logger logger_;
+};
+
+TEST_F(ConvertGraphDefToEngineTest, IdentityGraph) {
+  Scope s = Scope::NewRootScope();
+  auto input = ops::Placeholder(s.WithOpName(StrCat(kInputPHName, 0)), DT_FLOAT,
+                                ops::Placeholder::Shape({1, 1}));
+  auto output = ops::Identity(s.WithOpName("identity1"), input);
+  output = ops::Identity(s.WithOpName("identity2"), output);
+  output = ops::Identity(s.WithOpName(StrCat(kOutputPHName, 0)), output);
+  // If the converter marks the input tensor as output tensor, the conversion
+  // below will fail with:
+  // > TensorRTOutputPH_0 cannot be both input and output
+  // > Network must have at least one output
+  TF_EXPECT_OK(RunConvertGraphDefToEngine(&s));
+}
+
 // Input/output data format for OpConverterTest::BuildAndRun().
 struct InputOutputData {
   void* Buffer() const {
-- 
GitLab


From eb32155adfa674a81b010e3c1874a4b51a14f1a5 Mon Sep 17 00:00:00 2001
From: yonghuiguo <41990276+guoguo11191@users.noreply.github.com>
Date: Fri, 1 Feb 2019 13:51:49 +0800
Subject: [PATCH 1741/2345] Update hadoop_file_system.cc

Add comment for  bug fix of hdfs max read length
---
 tensorflow/core/platform/hadoop/hadoop_file_system.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/core/platform/hadoop/hadoop_file_system.cc b/tensorflow/core/platform/hadoop/hadoop_file_system.cc
index 076fad07af..2f34e7c660 100644
--- a/tensorflow/core/platform/hadoop/hadoop_file_system.cc
+++ b/tensorflow/core/platform/hadoop/hadoop_file_system.cc
@@ -209,6 +209,8 @@ class HDFSRandomAccessFile : public RandomAccessFile {
       // We lock inside the loop rather than outside so we don't block other
       // concurrent readers.
       mutex_lock lock(mu_);
+      //Max read length is INT_MAX-2, for hdfsPread function take a parameter of int32. 
+      //-2 offset can avoid JVM OutOfMemoryError. 
       size_t read_n = std::min(n,static_cast<size_t>(std::numeric_limits<int>::max()-2));
       tSize r = hdfs_->hdfsPread(fs_, file_, static_cast<tOffset>(offset), dst,
                                  static_cast<tSize>(read_n));
-- 
GitLab


From 9fc9fd672afd6f2d01cb1f1667a80da6d78638be Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Fri, 1 Feb 2019 00:38:25 -0800
Subject: [PATCH 1742/2345] Add missing support for types to
 ExtractFromIndexPositions.

Also allow to extract a scalar literal. This will be needed when we support
comparison computations.

PiperOrigin-RevId: 231928350
---
 .../compiler/xla/service/hlo_evaluator.cc     | 93 +++++++++++++------
 1 file changed, 64 insertions(+), 29 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index 803baf3564..56a1b6f439 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -1361,46 +1361,81 @@ Status HloEvaluator::HandleWhile(HloInstruction* while_hlo) {
 }
 
 namespace {
+template <typename NativeT>
+Literal ExtractLiteralFromIndexPositions(const Literal& from,
+                                         absl::Span<int64 const> indices,
+                                         bool extract_as_scalar) {
+  if (extract_as_scalar) {
+    return LiteralUtil::CreateR0<NativeT>(from.Get<NativeT>({indices[0]}));
+  }
+  // We use a InlinedVector here because we need to convert it to an
+  // absl::Span later, and this would not work with std::vector<bool>.
+  absl::InlinedVector<NativeT, 10> values;
+  for (int64 index : indices) {
+    values.push_back(from.Get<NativeT>({index}));
+  }
+  return LiteralUtil::CreateR1<NativeT>(values);
+}
+
 StatusOr<Literal> ExtractFromIndexPositions(const Literal& from,
-                                            absl::Span<int64 const> indices) {
+                                            absl::Span<int64 const> indices,
+                                            bool extract_as_scalar = false) {
+  if (extract_as_scalar) {
+    CHECK_EQ(indices.size(), 1);
+  }
   PrimitiveType type = from.shape().element_type();
   switch (type) {
     case PRED: {
-      // We use a InlinedVector here because we need to convert it to an
-      // absl::Span later, and this would not work with std::vector<bool>.
-      absl::InlinedVector<bool, 10> values;
-      for (int64 index : indices) {
-        values.push_back(from.Get<bool>({index}));
-      }
-      return LiteralUtil::CreateR1<bool>(values);
+      return ExtractLiteralFromIndexPositions<bool>(from, indices,
+                                                    extract_as_scalar);
+    }
+    case U8: {
+      return ExtractLiteralFromIndexPositions<uint8>(from, indices,
+                                                     extract_as_scalar);
+    }
+    case S8: {
+      return ExtractLiteralFromIndexPositions<int8>(from, indices,
+                                                    extract_as_scalar);
+    }
+    case BF16: {
+      return ExtractLiteralFromIndexPositions<bfloat16>(from, indices,
+                                                        extract_as_scalar);
+    }
+    case F16: {
+      return ExtractLiteralFromIndexPositions<Eigen::half>(from, indices,
+                                                           extract_as_scalar);
+    }
+    case U16: {
+      return ExtractLiteralFromIndexPositions<uint16>(from, indices,
+                                                      extract_as_scalar);
+    }
+    case S16: {
+      return ExtractLiteralFromIndexPositions<int16>(from, indices,
+                                                     extract_as_scalar);
     }
     case F32: {
-      std::vector<float> values;
-      for (int64 index : indices) {
-        values.push_back(from.Get<float>({index}));
-      }
-      return LiteralUtil::CreateR1<float>(values);
+      return ExtractLiteralFromIndexPositions<float>(from, indices,
+                                                     extract_as_scalar);
     }
     case U32: {
-      std::vector<uint32> values;
-      for (int64 index : indices) {
-        values.push_back(from.Get<uint32>({index}));
-      }
-      return LiteralUtil::CreateR1<uint32>(values);
+      return ExtractLiteralFromIndexPositions<uint32>(from, indices,
+                                                      extract_as_scalar);
     }
     case S32: {
-      std::vector<int32> values;
-      for (int64 index : indices) {
-        values.push_back(from.Get<int32>({index}));
-      }
-      return LiteralUtil::CreateR1<int32>(values);
+      return ExtractLiteralFromIndexPositions<int32>(from, indices,
+                                                     extract_as_scalar);
     }
-    case BF16: {
-      std::vector<bfloat16> values;
-      for (int64 index : indices) {
-        values.push_back(from.Get<bfloat16>({index}));
-      }
-      return LiteralUtil::CreateR1<bfloat16>(values);
+    case F64: {
+      return ExtractLiteralFromIndexPositions<double>(from, indices,
+                                                      extract_as_scalar);
+    }
+    case U64: {
+      return ExtractLiteralFromIndexPositions<uint64>(from, indices,
+                                                      extract_as_scalar);
+    }
+    case S64: {
+      return ExtractLiteralFromIndexPositions<int64>(from, indices,
+                                                     extract_as_scalar);
     }
     default:
       return InvalidArgument("Unsupported type for Sort: %s",
-- 
GitLab


From 3aa6c5c20264fb3ae3564a59298f6ce4e49a3a44 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 1 Feb 2019 01:02:24 -0800
Subject: [PATCH 1743/2345] compat: Update forward compatibility horizon to
 2019-02-01

PiperOrigin-RevId: 231930395
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 44920a46cb..bef953b28f 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 31)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 2, 1)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From fe8ee8b0b70b8b14b4a42cc72e140f21902897cd Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Fri, 1 Feb 2019 01:58:21 -0800
Subject: [PATCH 1744/2345] Create a seperate method to perform max_pool1d

PiperOrigin-RevId: 231936281
---
 tensorflow/python/ops/nn_ops.py               | 48 ++++++++++++++++++-
 .../tools/api/golden/v1/tensorflow.nn.pbtxt   |  4 ++
 .../tools/api/golden/v2/tensorflow.nn.pbtxt   |  4 ++
 3 files changed, 54 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 146b891943..b6f97033b9 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -3098,6 +3098,52 @@ def max_pool(value, ksize, strides, padding, data_format="NHWC", name=None):
         name=name)
 
 
+# pylint: disable=redefined-builtin
+@tf_export("nn.max_pool1d")
+def max_pool1d(input, ksize, strides, padding, data_format="NWC", name=None):
+  """Performs the max pooling on the input.
+
+  Note internally this op reshapes and uses the underlying 2d operation.
+
+  Args:
+    input: A 3-D `Tensor` of the format specified by `data_format`.
+    ksize: An int or list of `ints` that has length `1` or `3`. The size of the
+      window for each dimension of the input tensor.
+    strides: An int or list of `ints` that has length `1` or `3`. The stride of
+      the sliding window for each dimension of the input tensor.
+    padding: A string, either `'VALID'` or `'SAME'`. The padding algorithm. See
+      the "returns" section of `tf.nn.convolution` for details.
+    data_format: An optional string from: "NWC", "NCW". Defaults to "NWC".
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor` of format specified by `data_format`.
+    The max pooled output tensor.
+  """
+  with ops.name_scope(name, "MaxPool1d", [input]) as name:
+    if data_format is None:
+      data_format = "NWC"
+    channel_index = 1 if data_format.startswith("NC") else 2
+    ksize = [1] + _get_sequence(ksize, 1, channel_index, "ksize")
+    strides = [1] + _get_sequence(strides, 1, channel_index, "strides")
+
+    data_format = "NHWC" if data_format == "NWC" else "NCHW"
+    expanding_dim = 1 if data_format == "NWC" else 2
+
+    input = array_ops.expand_dims_v2(input, expanding_dim)
+    result = gen_nn_ops.max_pool(
+        input,
+        ksize=ksize,
+        strides=strides,
+        padding=padding,
+        data_format=data_format,
+        name=name)
+    return array_ops.squeeze(result, expanding_dim)
+
+
+# pylint: enable=redefined-builtin
+
+
 # pylint: disable=redefined-builtin
 @tf_export("nn.max_pool3d")
 def max_pool3d(input, ksize, strides, padding, data_format="NDHWC", name=None):
@@ -3138,8 +3184,6 @@ def max_pool3d(input, ksize, strides, padding, data_format="NDHWC", name=None):
         padding=padding,
         data_format=data_format,
         name=name)
-
-
 # pylint: enable=redefined-builtin
 
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
index 3320a0eecd..cf935dcf85 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
@@ -236,6 +236,10 @@ tf_module {
     name: "max_pool"
     argspec: "args=[\'value\', \'ksize\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'None\'], "
   }
+  member_method {
+    name: "max_pool1d"
+    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'NWC\', \'None\'], "
+  }
   member_method {
     name: "max_pool3d"
     argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'NDHWC\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index c75c75f2ef..ac3fd16224 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -196,6 +196,10 @@ tf_module {
     name: "max_pool"
     argspec: "args=[\'value\', \'ksize\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'None\'], "
   }
+  member_method {
+    name: "max_pool1d"
+    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'NWC\', \'None\'], "
+  }
   member_method {
     name: "max_pool3d"
     argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'NDHWC\', \'None\'], "
-- 
GitLab


From aa9a1127a894f66eaffe3ee60191a39ffa6cc66d Mon Sep 17 00:00:00 2001
From: Andr? Susano Pinto <andresp@google.com>
Date: Fri, 1 Feb 2019 02:42:19 -0800
Subject: [PATCH 1745/2345] Add support to call `tf.saved_model.load` while in
 graph mode.

This works by wiring the restore ops of variables as their initializers.

For now there is only a small test added. Will follow up with changes in
load_test so most of the tests also exercise that code path.

PiperOrigin-RevId: 231940750
---
 tensorflow/python/saved_model/load.py      | 38 +++++++++++++++---
 tensorflow/python/saved_model/load_test.py | 46 ++++++++++++++++++++++
 2 files changed, 79 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
index 4b9ba02656..6a1937d88f 100644
--- a/tensorflow/python/saved_model/load.py
+++ b/tensorflow/python/saved_model/load.py
@@ -22,6 +22,7 @@ import functools
 import os
 
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import init_ops
@@ -34,6 +35,7 @@ from tensorflow.python.saved_model import nested_structure_coder
 from tensorflow.python.saved_model import revived_types
 from tensorflow.python.saved_model import saved_object_graph_pb2
 from tensorflow.python.saved_model import utils_impl as saved_model_utils
+from tensorflow.python.training.checkpointable import base
 from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.training.checkpointable import util
 from tensorflow.python.util import compat
@@ -127,9 +129,34 @@ class _Loader(object):
           setattr(type(obj), "__call__", _call_attribute)
 
   def _restore_checkpoint(self):
+    """Load state from checkpoint into the deserialized objects."""
     variables_path = saved_model_utils.get_variables_path(self._export_dir)
+    # TODO(andresp): Clean use of private methods of CheckpointableSaver.
+    # pylint: disable=protected-access
     saver = util.CheckpointableSaver(self.get(0))
-    saver.restore(variables_path).assert_consumed()
+    saver._file_prefix_placeholder = constant_op.constant(variables_path)
+    load_status = saver.restore(variables_path)
+    load_status.assert_existing_objects_matched()
+    checkpoint = load_status._checkpoint
+
+    # When running in eager mode, the `restore` call above has already run and
+    # restored the state of checkpointables, call `position.restore_ops()` will
+    # return an empty list as there is nothing left to do. In graph mode, that
+    # will return the list of ops that must run to restore the object on that
+    # position. We have to wire them in the initializers of the objects so that
+    # they get initialized properly when using common practices (e.g. the ones
+    # used by ManagedSession) without further user action.
+    for object_id, obj in dict(checkpoint.object_by_proto_id).items():
+      position = base._CheckpointPosition(checkpoint=checkpoint,
+                                          proto_id=object_id)
+      restore_ops = position.restore_ops()
+      if restore_ops:
+        if resource_variable_ops.is_resource_variable(obj):
+          obj._initializer_op = restore_ops
+        else:
+          raise NotImplementedError(
+              ("Missing functionality to restore state of object "
+               "%r from the checkpoint." % obj))
 
   def get(self, node_id):
     return self._nodes[node_id]
@@ -210,10 +237,11 @@ def load(export_dir):
       compat.as_bytes("object_graph.pb"))
   if file_io.file_exists(object_graph_filename):
     object_graph_proto = _load_saved_object_graph_proto(object_graph_filename)
-    loader = _Loader(object_graph_proto,
-                     saved_model_proto,
-                     export_dir)
-    root = loader.get(0)
+    with ops.init_scope():
+      loader = _Loader(object_graph_proto,
+                       saved_model_proto,
+                       export_dir)
+      root = loader.get(0)
   else:
     raise NotImplementedError(
         "Currently only SavedModels exported with `tf.saved_model.save` may be "
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index 299b6aefa0..3edecd5be2 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -29,6 +29,7 @@ from tensorflow.python.eager import def_function
 from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_spec
 from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import array_ops
@@ -37,6 +38,7 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.saved_model import load
 from tensorflow.python.saved_model import save
+from tensorflow.python.training import monitored_session
 from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.training.checkpointable import util
 
@@ -536,6 +538,50 @@ class LoadTest(test.TestCase, parameterized.TestCase):
     x = constant_op.constant(1.0)
     self.assertAllEqual(imported(x).numpy(), 3.0)
 
+  def test_load_in_graph_mode(self, cycles):
+    root = tracking.AutoCheckpointable()
+    root.v1 = variables.Variable(1.)
+    root.v2 = variables.Variable(2.)
+    root.f = def_function.function(
+        lambda x: root.v2 * x,
+        input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
+
+    if cycles > 1:
+      root = self.cycle(root, cycles - 1)
+    path = tempfile.mkdtemp(prefix=self.get_temp_dir())
+    save.save(root, path)
+
+    with ops.Graph().as_default():
+      imported = load.load(path)
+      var_v1 = imported.v1
+      output = imported.f(constant_op.constant(2.))
+      with monitored_session.MonitoredSession() as sess:
+        self.assertEqual(1.0, sess.run(var_v1))
+        self.assertEqual(4.0, sess.run(output))
+
+  def test_load_in_func_graph(self, cycles):
+    root = tracking.AutoCheckpointable()
+    root.v1 = variables.Variable(1.)
+    root.v2 = variables.Variable(2.)
+    root.f = def_function.function(
+        lambda x: root.v2 * x,
+        input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
+
+    if cycles > 1:
+      root = self.cycle(root, cycles - 1)
+    path = tempfile.mkdtemp(prefix=self.get_temp_dir())
+    save.save(root, path)
+
+    closure = tracking.AutoCheckpointable()
+    @def_function.function
+    def func(x):
+      if not hasattr(closure, "model"):
+        closure.model = load.load(path)
+      return closure.model.f(x)
+
+    inputs = constant_op.constant(2.)
+    self.assertEqual(4.0, func(inputs).numpy())
+
   def test_soft_matching(self, cycles):
 
     @def_function.function(
-- 
GitLab


From 1e511799cf0396eecbf1eac5db7d133a6d0b1e7c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 1 Feb 2019 06:47:24 -0800
Subject: [PATCH 1746/2345] [TF:XLA] Remove Floating Point equality check in
 qr_op_test.

Instead check if the values are close. This test exists to check concurrency exists without error, so the equality is not necessary and fails with XLA due to to autotuning.

PiperOrigin-RevId: 231963469
---
 tensorflow/python/kernel_tests/qr_op_test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/kernel_tests/qr_op_test.py b/tensorflow/python/kernel_tests/qr_op_test.py
index 5adb95c7d6..f9b221a365 100644
--- a/tensorflow/python/kernel_tests/qr_op_test.py
+++ b/tensorflow/python/kernel_tests/qr_op_test.py
@@ -67,8 +67,8 @@ class QrOpTest(test.TestCase):
       val = self.evaluate(all_ops)
       for i in range(8):
         q = 4 * i
-        self.assertAllEqual(val[q], val[q + 2])  # q1 == q2
-        self.assertAllEqual(val[q + 1], val[q + 3])  # r1 == r2
+        self.assertAllClose(val[q], val[q + 2])  # q1 == q2
+        self.assertAllClose(val[q + 1], val[q + 3])  # r1 == r2
 
 
 def _GetQrOpTest(dtype_, shape_, full_matrices_, use_static_shape_):
-- 
GitLab


From 499ea79789748820e19b24e0fe95e651aa245fc7 Mon Sep 17 00:00:00 2001
From: Lukas Geiger <lukas.geiger94@gmail.com>
Date: Fri, 1 Feb 2019 16:33:46 +0100
Subject: [PATCH 1747/2345] Add missing elementwise monotonic functions

This PR adds `Acos`, `Acosh`, `Asin`, `Atan`, `Softsign` and `Softplus` to the collection of elementwise monotonic operations. This allows the arithmetic optimizer to optimize more cases.

Continuation of #25330
---
 tensorflow/core/grappler/op_types.cc | 34 +++++++++++++++++++++++-----
 1 file changed, 28 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc
index 27fba4fd11..628b61e723 100644
--- a/tensorflow/core/grappler/op_types.cc
+++ b/tensorflow/core/grappler/op_types.cc
@@ -164,17 +164,39 @@ bool IsDiv(const NodeDef& node) { return node.op() == "Div"; }
 bool IsElementWiseMonotonic(const NodeDef& node, bool* is_non_decreasing) {
   static const gtl::FlatSet<string>* const kMonotonicNonDecreasingOps =
       CHECK_NOTNULL((new gtl::FlatSet<string>{
-          "Asinh", "Atanh",   "Ceil",  "Elu",  "Erf",  "Exp",   "Expm1",
-          "Floor", "Log",     "Log1p", "Relu", "Relu", "Relu6", "Rint",
-          "Selu",  "Sigmoid", "Sign",  "Sinh", "Sqrt", "Tanh",
+          "Acosh",
+          "Asin",
+          "Asinh",
+          "Atan",
+          "Atanh",
+          "Ceil",
+          "Elu",
+          "Erf",
+          "Exp",
+          "Expm1",
+          "Floor",
+          "Log",
+          "Log1p",
+          "Relu",
+          "Relu6",
+          "Rint",
+          "Selu",
+          "Sigmoid",
+          "Sign",
+          "Sinh",
+          "Softsign",
+          "Softplus",
+          "Sqrt",
+          "Tanh",
       }));
   static const gtl::FlatSet<string>* const kMonotonicNonIncreasingOps =
       CHECK_NOTNULL((new gtl::FlatSet<string>{
-          "Inv",
-          "Reciprocal",
+          "Acos",
           "Erfc",
-          "Rsqrt",
+          "Inv",
           "Neg",
+          "Reciprocal",
+          "Rsqrt"
       }));
   if (kMonotonicNonDecreasingOps->count(node.op()) > 0) {
     if (is_non_decreasing) {
-- 
GitLab


From b2f47e6a9eb317c9fb881522265565c805c6c2db Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Fri, 1 Feb 2019 07:30:10 -0800
Subject: [PATCH 1748/2345] Implement callbacks for training distributed eval
 and predict.

PiperOrigin-RevId: 231968116
---
 .../contrib/distribute/python/keras_test.py   | 71 ++++++++++---------
 .../keras/engine/training_distributed.py      | 59 +++++++++------
 2 files changed, 72 insertions(+), 58 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 19e7d848ad..dd975c6c36 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -1156,16 +1156,7 @@ class Counter(keras.callbacks.Callback):
 class TestDistributionStrategyWithCallbacks(test.TestCase,
                                             parameterized.TestCase):
 
-  def _check_counts(self, counter, expected_counts):
-    """Checks that the counts registered by `counter` are those expected."""
-    for method_name, expected_count in expected_counts.items():
-      self.assertEqual(
-          counter.method_counts[method_name],
-          expected_count,
-          msg='For method {}: expected {}, got: {}'.format(
-              method_name, expected_count, counter.method_counts[method_name]))
-
-  @combinations.generate(strategy_minus_tpu_combinations())
+  @combinations.generate(all_strategy_combinations())
   def test_callbacks_in_fit(self, distribution):
     with distribution.scope():
       model = get_model()
@@ -1174,36 +1165,46 @@ class TestDistributionStrategyWithCallbacks(test.TestCase,
     dataset = get_dataset(distribution)
     counter = Counter()
 
+    epochs = 2
+    steps_per_epoch = 5
+    validation_steps = 3
+
     model.fit(
         dataset,
-        epochs=2,
-        steps_per_epoch=5,
+        epochs=epochs,
+        steps_per_epoch=steps_per_epoch,
         verbose=0,
         validation_data=dataset,
-        validation_steps=2,
+        validation_steps=validation_steps,
         callbacks=[counter])
 
-    self._check_counts(
-        counter, {
-            'on_batch_begin': 10,
-            'on_batch_end': 10,
-            'on_epoch_begin': 2,
-            'on_epoch_end': 2,
-            'on_predict_batch_begin': 0,
-            'on_predict_batch_end': 0,
-            'on_predict_begin': 0,
-            'on_predict_end': 0,
-            'on_test_batch_begin': 4,
-            'on_test_batch_end': 4,
-            'on_test_begin': 2,
-            'on_test_end': 2,
-            'on_train_batch_begin': 10,
-            'on_train_batch_end': 10,
+    if isinstance(distribution, tpu_strategy.TPUStrategy):
+      # TPU Strategy can have multi step training, from extended.steps_per_run
+      # if steps_per_run = 1, then num_batch_call_per_epoch = steps_per_epoch
+      steps_per_run = distribution.extended.steps_per_run
+      num_batch_call_per_epoch = steps_per_epoch // steps_per_run
+      if steps_per_epoch % steps_per_run:
+        num_batch_call_per_epoch += 1
+    else:
+      num_batch_call_per_epoch = steps_per_epoch
+
+    self.assertDictEqual(
+        counter.method_counts, {
+            'on_batch_begin': epochs * num_batch_call_per_epoch,
+            'on_batch_end': epochs * num_batch_call_per_epoch,
+            'on_epoch_begin': epochs,
+            'on_epoch_end': epochs,
+            'on_test_batch_begin': epochs * validation_steps,
+            'on_test_batch_end': epochs * validation_steps,
+            'on_test_begin': epochs,
+            'on_test_end': epochs,
+            'on_train_batch_begin': epochs * num_batch_call_per_epoch,
+            'on_train_batch_end': epochs * num_batch_call_per_epoch,
             'on_train_begin': 1,
             'on_train_end': 1
         })
 
-  @combinations.generate(strategy_minus_tpu_combinations())
+  @combinations.generate(all_strategy_combinations())
   def test_callbacks_in_eval(self, distribution):
     with distribution.scope():
       model = get_model()
@@ -1214,15 +1215,15 @@ class TestDistributionStrategyWithCallbacks(test.TestCase,
 
     model.evaluate(dataset, steps=5, callbacks=[counter])
 
-    self._check_counts(
-        counter, {
+    self.assertDictEqual(
+        counter.method_counts, {
             'on_test_batch_begin': 5,
             'on_test_batch_end': 5,
             'on_test_begin': 1,
             'on_test_end': 1
         })
 
-  @combinations.generate(strategy_minus_tpu_combinations())
+  @combinations.generate(all_strategy_combinations())
   def test_callbacks_in_predict(self, distribution):
     with distribution.scope():
       model = get_model()
@@ -1233,8 +1234,8 @@ class TestDistributionStrategyWithCallbacks(test.TestCase,
 
     model.predict(get_predict_dataset(dataset), steps=5, callbacks=[counter])
 
-    self._check_counts(
-        counter, {
+    self.assertDictEqual(
+        counter.method_counts, {
             'on_predict_batch_begin': 5,
             'on_predict_batch_end': 5,
             'on_predict_begin': 1,
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 735bd38438..c0ac70f436 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -268,6 +268,7 @@ def experimental_tpu_fit_loop(model,
   Raises:
       ValueError: in case of invalid arguments.
   """
+  mode = ModeKeys.TRAIN
   # TODO(fchollet): add support for `steps_per_epoch=None` in TPU loops.
   current_strategy = model._distribution_strategy
   iterator = distributed_training_utils.get_iterator(dataset, current_strategy)
@@ -288,11 +289,10 @@ def experimental_tpu_fit_loop(model,
     inputs, targets = inputs
     if model._compile_distribution:
       distributed_training_utils.clone_model_on_replicas(
-          model, current_strategy, ModeKeys.TRAIN, inputs=inputs,
-          targets=targets)
+          model, current_strategy, mode, inputs=inputs, targets=targets)
     else:
       distributed_training_utils._build_distributed_network(
-          model, current_strategy, ModeKeys.TRAIN, inputs, targets)
+          model, current_strategy, mode, inputs, targets)
 
     (grouped_inputs, grouped_outputs, grouped_updates,
      grouped_session_args) = current_strategy.extended.call_for_each_replica(
@@ -347,8 +347,7 @@ def experimental_tpu_fit_loop(model,
   do_validation = bool(validation_steps)
 
   if model._compile_distribution:
-    distributed_training_utils._copy_weights_to_distributed_model(
-        model, ModeKeys.TRAIN)
+    distributed_training_utils._copy_weights_to_distributed_model(model, mode)
 
   callbacks = cbks.configure_callbacks(
       callbacks,
@@ -358,7 +357,7 @@ def experimental_tpu_fit_loop(model,
       steps_per_epoch=steps_per_epoch,
       verbose=verbose,
       count_mode='steps',
-      mode=ModeKeys.TRAIN)
+      mode=mode)
 
   # Calculate the steps each time on the device.
   steps_to_run = [current_strategy.extended.steps_per_run] * (
@@ -367,7 +366,7 @@ def experimental_tpu_fit_loop(model,
     steps_to_run.append(
         steps_per_epoch % current_strategy.extended.steps_per_run)
 
-  callbacks.on_train_begin()
+  callbacks._call_begin_hook(mode)
   for epoch in range(initial_epoch, epochs):
     distributed_training_utils._reset_metrics(model)
     callbacks.on_epoch_begin(epoch)
@@ -376,7 +375,7 @@ def experimental_tpu_fit_loop(model,
     prev_step_count = None
     for step_count in steps_to_run:
       batch_logs = {'batch': step_index, 'size': 1, 'num_steps': step_count}
-      callbacks.on_batch_begin(step_index, batch_logs)
+      callbacks._call_batch_hook(mode, 'begin', step_index, batch_logs)
       if prev_step_count is None or step_count != prev_step_count:
         steps_per_run.load(step_count, K.get_session())
         prev_step_count = step_count
@@ -391,7 +390,7 @@ def experimental_tpu_fit_loop(model,
         break
 
       batch_logs.update(outputs)
-      callbacks.on_batch_end(step_index, batch_logs)
+      callbacks._call_batch_hook(mode, 'end', step_index, batch_logs)
       step_index = step_index + step_count
       if callbacks.model.stop_training:
         break
@@ -410,7 +409,8 @@ def experimental_tpu_fit_loop(model,
           model,
           val_dataset,
           steps=validation_steps,
-          verbose=verbose)
+          verbose=verbose,
+          callbacks=callbacks)
       if not isinstance(val_outs, list):
         val_outs = [val_outs]
       # Same labels assumed.
@@ -420,7 +420,7 @@ def experimental_tpu_fit_loop(model,
     callbacks.on_epoch_end(epoch, epoch_logs)
     if callbacks.model.stop_training:
       break
-  callbacks.on_train_end()
+  callbacks._call_end_hook(mode)
 
   if model._compile_distribution:
     # Copy the weights back from the replicated model to the original model.
@@ -452,6 +452,7 @@ def experimental_tpu_test_loop(model,
       and/or metrics). The attribute `model.metrics_names` will give you
       the display labels for the outputs.
   """
+  mode = ModeKeys.TEST
   current_strategy = model._distribution_strategy
   iterator = distributed_training_utils.get_iterator(dataset, current_strategy)
   scope = distributed_training_utils.distributed_scope(
@@ -468,12 +469,11 @@ def experimental_tpu_test_loop(model,
     """Clones the model and calls make_eval_function."""
     inputs, targets = inputs
     if model._compile_distribution:
-      distributed_training_utils. clone_model_on_replicas(
-          model, current_strategy, mode=ModeKeys.TEST, inputs=inputs,
-          targets=targets)
+      distributed_training_utils.clone_model_on_replicas(
+          model, current_strategy, mode=mode, inputs=inputs, targets=targets)
     else:
       distributed_training_utils._build_distributed_network(
-          model, current_strategy, ModeKeys.TEST, inputs, targets)
+          model, current_strategy, mode, inputs, targets)
 
     (grouped_inputs, grouped_outputs, grouped_updates,
      grouped_session_args) = current_strategy.extended.call_for_each_replica(
@@ -521,8 +521,7 @@ def experimental_tpu_test_loop(model,
     progbar = Progbar(target=steps)
 
   if model._compile_distribution:
-    distributed_training_utils._copy_weights_to_distributed_model(
-        model, ModeKeys.TEST)
+    distributed_training_utils._copy_weights_to_distributed_model(model, mode)
 
   distributed_training_utils._reset_metrics(model)
 
@@ -535,10 +534,13 @@ def experimental_tpu_test_loop(model,
       verbose=verbose,
       count_mode='steps',
       mode=ModeKeys.TEST)
+  callbacks._call_begin_hook(mode)
 
   assert steps is not None
   outs = [0.] * len(model.metrics_names)
   for step in range(steps):
+    batch_logs = {'batch': step, 'size': 1}
+    callbacks._call_batch_hook(mode, 'begin', step, batch_logs)
     _, batch_outs = K.get_session().run([test_op, output_tensors])
     for i, label in enumerate(model.metrics_names):
       if i == 0:
@@ -548,9 +550,13 @@ def experimental_tpu_test_loop(model,
         # For all stateful metrics, the aggregation is handled by mirrored vars.
         outs[i] = batch_outs[label]
 
+    batch_logs = cbks.make_logs(model, batch_logs, outs, mode)
+    callbacks._call_batch_hook(mode, 'end', step, batch_logs)
     if verbose >= 1:
       progbar.update(step + 1)
 
+  callbacks._call_end_hook(mode)
+
   scope.__exit__(None, None, None)
   if len(outs) >= 0:
     outs[0] /= (steps)
@@ -581,6 +587,7 @@ def experimental_tpu_predict_loop(model,
       or list of arrays of predictions
       (if the model has multiple outputs).
   """
+  mode = ModeKeys.PREDICT
   dataset_fully_shaped = (distributed_training_utils.
                           is_dataset_shape_fully_defined(dataset))
   padding_handler = None
@@ -624,11 +631,11 @@ def experimental_tpu_predict_loop(model,
   def step_fn(ctx, inputs):
     """Clones the model and calls make_predict_function."""
     if model._compile_distribution:
-      distributed_training_utils. clone_model_on_replicas(
-          model, current_strategy, ModeKeys.PREDICT, inputs=inputs)
+      distributed_training_utils.clone_model_on_replicas(
+          model, current_strategy, mode, inputs=inputs)
     else:
       distributed_training_utils._build_distributed_network(
-          model, current_strategy, ModeKeys.PREDICT, inputs)
+          model, current_strategy, mode, inputs)
 
     (grouped_inputs, grouped_outputs, grouped_updates,
      grouped_session_args) = current_strategy.extended.call_for_each_replica(
@@ -672,8 +679,7 @@ def experimental_tpu_predict_loop(model,
     progbar = Progbar(target=steps)
 
   if model._compile_distribution:
-    distributed_training_utils._copy_weights_to_distributed_model(
-        model, ModeKeys.PREDICT)
+    distributed_training_utils._copy_weights_to_distributed_model(model, mode)
 
   distributed_training_utils._reset_metrics(model)
 
@@ -685,7 +691,8 @@ def experimental_tpu_predict_loop(model,
       steps_per_epoch=steps,
       verbose=verbose,
       count_mode='steps',
-      mode=ModeKeys.PREDICT)
+      mode=mode)
+  callbacks._call_begin_hook(mode)
 
   assert steps is not None
   # Since we do not know how many samples we will see, we cannot pre-allocate
@@ -693,13 +700,19 @@ def experimental_tpu_predict_loop(model,
   # and concatenate them upon returning.
   unconcatenated_outs = [[] for _ in model.outputs]
   for step in range(steps):
+    batch_logs = {'batch': step, 'size': 1}
+    callbacks._call_batch_hook(mode, 'begin', step, batch_logs)
     _, batch_outs = K.get_session().run([predict_op, output_tensors])
     # TODO(priyag): maybe need to unwrap the outputs first for MirroredStrategy.
     for i, label in enumerate(model.output_names):
       unconcatenated_outs[i].extend(batch_outs[label])
+    batch_logs = cbks.make_logs(model, batch_logs, batch_outs, mode)
+    callbacks._call_batch_hook(mode, 'end', step, batch_logs)
     if verbose >= 1:
       progbar.update(step + 1)
 
+  callbacks._call_end_hook(mode)
+
   scope.__exit__(None, None, None)
 
   if len(unconcatenated_outs) == 1:
-- 
GitLab


From f08bc174d86b5756c01e8f4ac16d8804e01d4b6d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 1 Feb 2019 07:38:49 -0800
Subject: [PATCH 1749/2345] Update ResidualWrapper to LayerRNNCell.

Replicates the logic of DropoutWrapper.

PiperOrigin-RevId: 231969205
---
 .../python/kernel_tests/core_rnn_cell_test.py | 325 ++++++++++--------
 tensorflow/python/ops/rnn_cell_impl.py        | 279 +++++++++++----
 ...sorflow.nn.rnn_cell.-dropout-wrapper.pbtxt |   1 +
 ...orflow.nn.rnn_cell.-residual-wrapper.pbtxt |   1 +
 .../golden/v2/tensorflow.nn.rnn_cell.pbtxt    |   4 -
 .../v2/tensorflow.rnn.-dropout-wrapper.pbtxt  |   2 +
 ...=> tensorflow.rnn.-residual-wrapper.pbtxt} |  10 +-
 .../tools/api/golden/v2/tensorflow.rnn.pbtxt  |   4 +
 tensorflow/tools/compatibility/renames_v2.py  |   1 +
 9 files changed, 398 insertions(+), 229 deletions(-)
 rename tensorflow/tools/api/golden/v2/{tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt => tensorflow.rnn.-residual-wrapper.pbtxt} (91%)

diff --git a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py
index 7bad4a60a1..a70e806211 100644
--- a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py
+++ b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py
@@ -48,7 +48,7 @@ from tensorflow.python.training.checkpointable import util as checkpointable_uti
 Linear = core_rnn_cell._Linear  # pylint: disable=invalid-name
 
 
-class RNNCellTest(test.TestCase):
+class RNNCellTest(test.TestCase, parameterized.TestCase):
 
   def testLinear(self):
     with self.cached_session() as sess:
@@ -642,58 +642,54 @@ class RNNCellTest(test.TestCase):
         # The numbers in results were not calculated, this is just a smoke test.
         self.assertAllClose(res[0], [[0.154605, 0.154605, 0.154605]])
 
-  def testResidualWrapper(self):
-    with self.cached_session() as sess:
-      with variable_scope.variable_scope(
-          "root", initializer=init_ops.constant_initializer(0.5)):
-        x = array_ops.zeros([1, 3])
-        m = array_ops.zeros([1, 3])
-        base_cell = rnn_cell_impl.GRUCell(3)
-        g, m_new = base_cell(x, m)
-        variable_scope.get_variable_scope().reuse_variables()
-        wrapper_object = rnn_cell_impl.ResidualWrapper(base_cell)
-        (name, dep), = wrapper_object._checkpoint_dependencies
-        wrapper_object.get_config()  # Should not throw an error
-        self.assertIs(dep, base_cell)
-        self.assertEqual("cell", name)
-
-        g_res, m_new_res = wrapper_object(x, m)
-        sess.run([variables_lib.global_variables_initializer()])
-        res = sess.run([g, g_res, m_new, m_new_res], {
-            x: np.array([[1., 1., 1.]]),
-            m: np.array([[0.1, 0.1, 0.1]])
-        })
-        # Residual connections
-        self.assertAllClose(res[1], res[0] + [1., 1., 1.])
-        # States are left untouched
-        self.assertAllClose(res[2], res[3])
-
-  def testResidualWrapperWithSlice(self):
-    with self.cached_session() as sess:
-      with variable_scope.variable_scope(
-          "root", initializer=init_ops.constant_initializer(0.5)):
-        x = array_ops.zeros([1, 5])
-        m = array_ops.zeros([1, 3])
-        base_cell = rnn_cell_impl.GRUCell(3)
-        g, m_new = base_cell(x, m)
-        variable_scope.get_variable_scope().reuse_variables()
-
-        def residual_with_slice_fn(inp, out):
-          inp_sliced = array_ops.slice(inp, [0, 0], [-1, 3])
-          return inp_sliced + out
+  @parameterized.parameters(
+      [rnn_cell_impl.ResidualWrapper, rnn_cell_impl.ResidualWrapperV2])
+  @test_util.run_in_graph_and_eager_modes
+  def testResidualWrapper(self, wrapper_type):
+    x = ops.convert_to_tensor(np.array([[1., 1., 1.]]))
+    m = ops.convert_to_tensor(np.array([[0.1, 0.1, 0.1]]))
+    base_cell = rnn_cell_impl.GRUCell(
+        3, kernel_initializer=init_ops.constant_initializer(0.5),
+        bias_initializer=init_ops.constant_initializer(0.5))
+    g, m_new = base_cell(x, m)
+    wrapper_object = wrapper_type(base_cell)
+    (name, dep), = wrapper_object._checkpoint_dependencies
+    wrapper_object.get_config()  # Should not throw an error
+    self.assertIs(dep, base_cell)
+    self.assertEqual("cell", name)
+
+    g_res, m_new_res = wrapper_object(x, m)
+    self.evaluate([variables_lib.global_variables_initializer()])
+    res = self.evaluate([g, g_res, m_new, m_new_res])
+    # Residual connections
+    self.assertAllClose(res[1], res[0] + [1., 1., 1.])
+    # States are left untouched
+    self.assertAllClose(res[2], res[3])
 
-        g_res, m_new_res = rnn_cell_impl.ResidualWrapper(
-            base_cell, residual_with_slice_fn)(x, m)
-        sess.run([variables_lib.global_variables_initializer()])
-        res_g, res_g_res, res_m_new, res_m_new_res = sess.run(
-            [g, g_res, m_new, m_new_res], {
-                x: np.array([[1., 1., 1., 1., 1.]]),
-                m: np.array([[0.1, 0.1, 0.1]])
-            })
-        # Residual connections
-        self.assertAllClose(res_g_res, res_g + [1., 1., 1.])
-        # States are left untouched
-        self.assertAllClose(res_m_new, res_m_new_res)
+  @parameterized.parameters(
+      [rnn_cell_impl.ResidualWrapper, rnn_cell_impl.ResidualWrapperV2])
+  @test_util.run_in_graph_and_eager_modes
+  def testResidualWrapperWithSlice(self, wrapper_type):
+    x = ops.convert_to_tensor(np.array([[1., 1., 1., 1., 1.]]))
+    m = ops.convert_to_tensor(np.array([[0.1, 0.1, 0.1]]))
+    base_cell = rnn_cell_impl.GRUCell(
+        3, kernel_initializer=init_ops.constant_initializer(0.5),
+        bias_initializer=init_ops.constant_initializer(0.5))
+    g, m_new = base_cell(x, m)
+
+    def residual_with_slice_fn(inp, out):
+      inp_sliced = array_ops.slice(inp, [0, 0], [-1, 3])
+      return inp_sliced + out
+
+    g_res, m_new_res = wrapper_type(
+        base_cell, residual_with_slice_fn)(x, m)
+    self.evaluate([variables_lib.global_variables_initializer()])
+    res_g, res_g_res, res_m_new, res_m_new_res = self.evaluate(
+        [g, g_res, m_new, m_new_res])
+    # Residual connections
+    self.assertAllClose(res_g_res, res_g + [1., 1., 1.])
+    # States are left untouched
+    self.assertAllClose(res_m_new, res_m_new_res)
 
   def testDeviceWrapper(self):
     with variable_scope.variable_scope(
@@ -836,7 +832,98 @@ class RNNCellTest(test.TestCase):
         self.assertAllClose(res[0], [[0.175991, 0.175991]])
         self.assertAllClose(res[1], [[0.13248, 0.13248]])
 
+  @parameterized.parameters(
+      [[rnn_cell_impl.DropoutWrapper, rnn_cell_impl.DropoutWrapperV2],
+       [rnn_cell_impl.ResidualWrapper, rnn_cell_impl.ResidualWrapperV2]])
+  @test_util.run_in_graph_and_eager_modes
+  def testWrapperKerasStyle(self, wrapper, wrapper_v2):
+    """Tests if wrapper cell is instantiated in keras style scope."""
+    wrapped_cell_v2 = wrapper_v2(rnn_cell_impl.BasicRNNCell(1))
+    self.assertTrue(wrapped_cell_v2._keras_style)
 
+    wrapped_cell = wrapper(rnn_cell_impl.BasicRNNCell(1))
+    self.assertFalse(wrapped_cell._keras_style)
+
+  @parameterized.parameters(
+      [rnn_cell_impl.DropoutWrapperV2, rnn_cell_impl.ResidualWrapperV2])
+  @test_util.run_in_graph_and_eager_modes
+  def testWrapperV2VariableNames(self, wrapper):
+    """Tests that variables names do not depend on wrapper in RNN layer."""
+
+    def _rnn_input(apply_wrapper, name):
+      """Creates a RNN layer with/without wrapper and returns built rnn cell."""
+      with base_layer.keras_style_scope():
+        base_cell = rnn_cell_impl.MultiRNNCell(
+            [rnn_cell_impl.BasicRNNCell(1, name="basic_rnn_cell")
+             for _ in range(2)])
+      if apply_wrapper:
+        rnn_cell = wrapper(base_cell)
+      else:
+        rnn_cell = base_cell
+      rnn_layer = keras_layers.RNN(rnn_cell, name=name)
+      inputs = ops.convert_to_tensor([[[1]]], dtype=dtypes.float32)
+      _ = rnn_layer(inputs)
+      return base_cell._cells[0]
+
+    rnn_1 = _rnn_input(True, name="rnn_0")
+    rnn_2 = _rnn_input(False, name="rnn_1")
+
+    for i, cell in enumerate([rnn_1, rnn_2]):
+      var_prefix = "rnn_{}/cell_0/basic_rnn_cell/".format(i)
+      self.assertCountEqual([v.name for v in cell.weights],
+                            (var_prefix + "kernel:0", var_prefix + "bias:0"))
+
+  @parameterized.parameters(
+      [rnn_cell_impl.DropoutWrapperV2, rnn_cell_impl.ResidualWrapperV2])
+  @test_util.run_in_graph_and_eager_modes
+  def testWrapperWeights(self, wrapper):
+    """Tests that wrapper weights contain wrapped cells weights."""
+
+    with base_layer.keras_style_scope():
+      base_cell = rnn_cell_impl.BasicRNNCell(1, name="basic_rnn_cell")
+    rnn_cell = wrapper(base_cell)
+    rnn_layer = keras_layers.RNN(rnn_cell)
+    inputs = ops.convert_to_tensor([[[1]]], dtype=dtypes.float32)
+    rnn_layer(inputs)
+
+    expected_weights = ["rnn/" + var for var in ("kernel:0", "bias:0")]
+    self.assertEqual(len(rnn_cell.weights), 2)
+    self.assertCountEqual([v.name for v in rnn_cell.weights], expected_weights)
+    self.assertCountEqual([v.name for v in rnn_cell.trainable_variables],
+                          expected_weights)
+    self.assertCountEqual([v.name for v in rnn_cell.non_trainable_variables],
+                          [])
+    self.assertCountEqual([v.name for v in rnn_cell._cell.weights],
+                          expected_weights)
+
+  @parameterized.parameters(
+      [rnn_cell_impl.DropoutWrapperV2, rnn_cell_impl.ResidualWrapperV2])
+  @test_util.run_in_graph_and_eager_modes
+  def testWrapperV2Caller(self, wrapper):
+    """Tests that wrapper V2 is using the LayerRNNCell's caller."""
+
+    with base_layer.keras_style_scope():
+      base_cell = rnn_cell_impl.MultiRNNCell(
+          [rnn_cell_impl.BasicRNNCell(1) for _ in range(2)])
+    rnn_cell = wrapper(base_cell)
+    inputs = ops.convert_to_tensor([[1]], dtype=dtypes.float32)
+    state = ops.convert_to_tensor([[1]], dtype=dtypes.float32)
+    _ = rnn_cell(inputs, [state, state])
+    weights = base_cell._cells[0].weights
+    self.assertLen(weights, expected_len=2)
+    self.assertTrue(all(["_wrapper" in v.name for v in weights]))
+
+  @parameterized.parameters(
+      [rnn_cell_impl.DropoutWrapperV2, rnn_cell_impl.ResidualWrapperV2])
+  @test_util.run_in_graph_and_eager_modes
+  def testWrapperV2Build(self, wrapper):
+    cell = rnn_cell_impl.LSTMCell(10)
+    wrapper = wrapper(cell)
+    wrapper.build((1,))
+    self.assertTrue(cell.built)
+
+
+@test_util.run_all_in_graph_and_eager_modes
 class DropoutWrapperTest(test.TestCase, parameterized.TestCase):
 
   def _testDropoutWrapper(self,
@@ -844,39 +931,38 @@ class DropoutWrapperTest(test.TestCase, parameterized.TestCase):
                           time_steps=None,
                           parallel_iterations=None,
                           wrapper_type=None,
+                          scope="root",
                           **kwargs):
-    with self.cached_session() as sess:
-      with variable_scope.variable_scope(
-          "root", initializer=init_ops.constant_initializer(0.5)):
-        if batch_size is None and time_steps is None:
-          # 2 time steps, batch size 1, depth 3
-          batch_size = 1
-          time_steps = 2
-          x = constant_op.constant(
-              [[[2., 2., 2.]], [[1., 1., 1.]]], dtype=dtypes.float32)
-          m = rnn_cell_impl.LSTMStateTuple(
-              *[constant_op.constant([[0.1, 0.1, 0.1]], dtype=dtypes.float32
-                                    )] * 2)
-        else:
-          x = constant_op.constant(
-              np.random.randn(time_steps, batch_size, 3).astype(np.float32))
-          m = rnn_cell_impl.LSTMStateTuple(*[
-              constant_op.
-              constant([[0.1, 0.1, 0.1]] * batch_size, dtype=dtypes.float32)
-          ] * 2)
-        outputs, final_state = rnn.dynamic_rnn(
-            cell=wrapper_type(
-                rnn_cell_impl.LSTMCell(3), dtype=x.dtype, **kwargs),
-            time_major=True,
-            parallel_iterations=parallel_iterations,
-            inputs=x,
-            initial_state=m)
-        sess.run([variables_lib.global_variables_initializer()])
-        res = sess.run([outputs, final_state])
-        self.assertEqual(res[0].shape, (time_steps, batch_size, 3))
-        self.assertEqual(res[1].c.shape, (batch_size, 3))
-        self.assertEqual(res[1].h.shape, (batch_size, 3))
-        return res
+    if batch_size is None and time_steps is None:
+      # 2 time steps, batch size 1, depth 3
+      batch_size = 1
+      time_steps = 2
+      x = constant_op.constant(
+          [[[2., 2., 2.]], [[1., 1., 1.]]], dtype=dtypes.float32)
+      m = rnn_cell_impl.LSTMStateTuple(
+          *[constant_op.constant([[0.1, 0.1, 0.1]], dtype=dtypes.float32)] * 2)
+    else:
+      x = constant_op.constant(
+          np.random.randn(time_steps, batch_size, 3).astype(np.float32))
+      m = rnn_cell_impl.LSTMStateTuple(*[
+          constant_op.
+          constant([[0.1, 0.1, 0.1]] * batch_size, dtype=dtypes.float32)] * 2)
+    outputs, final_state = rnn.dynamic_rnn(
+        cell=wrapper_type(
+            rnn_cell_impl.LSTMCell(
+                3, initializer=init_ops.constant_initializer(0.5)),
+            dtype=x.dtype, **kwargs),
+        time_major=True,
+        parallel_iterations=parallel_iterations,
+        inputs=x,
+        initial_state=m,
+        scope=scope)
+    self.evaluate([variables_lib.global_variables_initializer()])
+    res = self.evaluate([outputs, final_state])
+    self.assertEqual(res[0].shape, (time_steps, batch_size, 3))
+    self.assertEqual(res[1].c.shape, (batch_size, 3))
+    self.assertEqual(res[1].h.shape, (batch_size, 3))
+    return res
 
   @parameterized.parameters(
       [rnn_cell_impl.DropoutWrapper, rnn_cell_impl.DropoutWrapperV2])
@@ -946,10 +1032,8 @@ class DropoutWrapperTest(test.TestCase, parameterized.TestCase):
         state_keep_prob=keep_some,
         seed=10,
         parallel_iterations=1,
-        wrapper_type=wrapper_type)
-    # Clear away the graph and the test session (which keeps variables around)
-    ops.reset_default_graph()
-    self._ClearCachedSession()
+        wrapper_type=wrapper_type,
+        scope="root_1")
     random_seed.set_random_seed(2)
     res_standard_2 = self._testDropoutWrapper(
         input_keep_prob=keep_some,
@@ -957,7 +1041,8 @@ class DropoutWrapperTest(test.TestCase, parameterized.TestCase):
         state_keep_prob=keep_some,
         seed=10,
         parallel_iterations=1,
-        wrapper_type=wrapper_type)
+        wrapper_type=wrapper_type,
+        scope="root_2")
     self.assertAllClose(res_standard_1[0], res_standard_2[0])
     self.assertAllClose(res_standard_1[1].c, res_standard_2[1].c)
     self.assertAllClose(res_standard_1[1].h, res_standard_2[1].h)
@@ -1091,9 +1176,8 @@ class DropoutWrapperTest(test.TestCase, parameterized.TestCase):
         input_size=3,
         batch_size=5,
         time_steps=7,
-        seed=-234987)
-    ops.reset_default_graph()
-    self._ClearCachedSession()
+        seed=-234987,
+        scope="root_0")
     random_seed.set_random_seed(2347)
     np.random.seed(23487)
     res1 = self._testDropoutWrapper(
@@ -1105,7 +1189,8 @@ class DropoutWrapperTest(test.TestCase, parameterized.TestCase):
         input_size=3,
         batch_size=5,
         time_steps=7,
-        seed=-234987)
+        seed=-234987,
+        scope="root_1")
 
     output_mask = np.abs(res0[0]) > 1e-6
     for time_step in output_mask:
@@ -1128,60 +1213,6 @@ class DropoutWrapperTest(test.TestCase, parameterized.TestCase):
     self.assertAllClose(res0[1].c, res1[1].c)
     self.assertAllClose(res0[1].h, res1[1].h)
 
-  def testDropoutWrapperKerasStyle(self):
-    """Tests if DropoutWrapperV2 cell is instantiated in keras style scope."""
-    wrapped_cell_v2 = rnn_cell_impl.DropoutWrapperV2(
-        rnn_cell_impl.BasicRNNCell(1))
-    self.assertTrue(wrapped_cell_v2._keras_style)
-
-    wrapped_cell = rnn_cell_impl.DropoutWrapper(rnn_cell_impl.BasicRNNCell(1))
-    self.assertFalse(wrapped_cell._keras_style)
-
-  def testDropoutWrapperV2VariableNames(self):
-    """Tests that variables names do not depend on wrapper in RNN layer."""
-
-    def _rnn_input(apply_wrapper):
-      """Creates a RNN layer with/without wrapper and returns built rnn cell."""
-      with base_layer.keras_style_scope():
-        base_cell = rnn_cell_impl.MultiRNNCell(
-            [rnn_cell_impl.BasicRNNCell(1) for _ in range(2)])
-      if apply_wrapper:
-        rnn_cell = rnn_cell_impl.DropoutWrapperV2(base_cell)
-      else:
-        rnn_cell = base_cell
-      rnn_layer = keras_layers.RNN(rnn_cell)
-      inputs = ops.convert_to_tensor([[[1]]], dtype=dtypes.float32)
-      _ = rnn_layer(inputs)
-      return base_cell._cells[0]
-
-    rnn_1 = _rnn_input(True)
-    ops.reset_default_graph()
-    rnn_2 = _rnn_input(False)
-
-    self.assertLen(rnn_1.weights, expected_len=2)
-    self.assertCountEqual([v.name for v in rnn_1.weights],
-                          [v.name for v in rnn_2.weights])
-
-  def testDropoutWrapperV2Caller(self):
-    """Tests that DropoutWrapperV2 is using the LayerRNNCell's caller."""
-
-    with base_layer.keras_style_scope():
-      base_cell = rnn_cell_impl.MultiRNNCell(
-          [rnn_cell_impl.BasicRNNCell(1) for _ in range(2)])
-    rnn_cell = rnn_cell_impl.DropoutWrapperV2(base_cell)
-    inputs = ops.convert_to_tensor([[1]], dtype=dtypes.float32)
-    state = ops.convert_to_tensor([[1]], dtype=dtypes.float32)
-    _ = rnn_cell(inputs, [state, state])
-    weights = base_cell._cells[0].weights
-    self.assertLen(weights, expected_len=2)
-    self.assertTrue(all(["dropout_wrapper" in v.name for v in weights]))
-
-  def testDropoutWrapperV2Build(self):
-    cell = rnn_cell_impl.LSTMCell(10)
-    wrapper = rnn_cell_impl.DropoutWrapperV2(cell)
-    wrapper.build((1,))
-    self.assertTrue(cell.built)
-
 
 def basic_rnn_cell(inputs, state, num_units, scope=None):
   if state is None:
diff --git a/tensorflow/python/ops/rnn_cell_impl.py b/tensorflow/python/ops/rnn_cell_impl.py
index 40c3771f4e..603baea281 100644
--- a/tensorflow/python/ops/rnn_cell_impl.py
+++ b/tensorflow/python/ops/rnn_cell_impl.py
@@ -1085,8 +1085,107 @@ def _default_dropout_state_filter_visitor(substate):
   return True
 
 
+class _RNNCellWrapperV1(RNNCell):
+  """Base class for cells wrappers V1 compatibility.
+
+  This class along with `_RNNCellWrapperV2` allows to define cells wrappers that
+  are compatible with V1 and V2, and defines helper methods for this purpose.
+  """
+
+  def __init__(self, cell):
+    super(_RNNCellWrapperV1, self).__init__()
+    self._cell = cell
+    if isinstance(cell, checkpointable.Checkpointable):
+      self._track_checkpointable(self._cell, name="cell")
+
+  def _call_wrapped_cell(self, inputs, state, cell_call_fn, **kwargs):
+    """Calls the wrapped cell and performs the wrapping logic.
+
+    This method is called from the wrapper's `call` or `__call__` methods.
+
+    Args:
+      inputs: A tensor with wrapped cell's input.
+      state: A tensor or tuple of tensors with wrapped cell's state.
+      cell_call_fn: Wrapped cell's method to use for step computation (cell's
+        `__call__` or 'call' method).
+      **kwargs: Additional arguments.
+
+    Returns:
+      A pair containing:
+      - Output: A tensor with cell's output.
+      - New state: A tensor or tuple of tensors with new wrapped cell's state.
+    """
+    raise NotImplementedError
+
+  def __call__(self, inputs, state, scope=None):
+    """Runs the RNN cell step computation.
+
+    We assume that the wrapped RNNCell is being built within its `__call__`
+    method. We directly use the wrapped cell's `__call__` in the overridden
+    wrapper `__call__` method.
+
+    This allows to use the wrapped cell and the non-wrapped cell equivalently
+    when using `__call__`.
+
+    Args:
+      inputs: A tensor with wrapped cell's input.
+      state: A tensor or tuple of tensors with wrapped cell's state.
+      scope: VariableScope for the subgraph created in the wrapped cells'
+        `__call__`.
+
+    Returns:
+      A pair containing:
+
+      - Output: A tensor with cell's output.
+      - New state: A tensor or tuple of tensors with new wrapped cell's state.
+    """
+    return self._call_wrapped_cell(
+        inputs, state, cell_call_fn=self._cell.__call__, scope=scope)
+
+
+class _RNNCellWrapperV2(LayerRNNCell, _RNNCellWrapperV1):
+  """Base class for cells wrappers V2 compatibility.
+
+  This class along with `_RNNCellWrapperV1` allows to define cells wrappers that
+  are compatible with V1 and V2, and defines helper methods for this purpose.
+  """
+
+  def __init__(self, *args, **kwargs):
+    super(_RNNCellWrapperV2, self).__init__(*args, **kwargs)
+    self._layers = [self._cell]
+
+  def call(self, inputs, state, **kwargs):
+    """Runs the RNN cell step computation.
+
+    When `call` is being used, we assume that the wrapper object has been built,
+    and therefore the wrapped cells has been built via its `build` method and
+    its `call` method can be used directly.
+
+    This allows to use the wrapped cell and the non-wrapped cell equivalently
+    when using `call` and `build`.
+
+    Args:
+      inputs: A tensor with wrapped cell's input.
+      state: A tensor or tuple of tensors with wrapped cell's state.
+      **kwargs: Additional arguments passed to the wrapped cell's `call`.
+
+    Returns:
+      A pair containing:
+
+      - Output: A tensor with cell's output.
+      - New state: A tensor or tuple of tensors with new wrapped cell's state.
+    """
+    return self._call_wrapped_cell(
+        inputs, state, cell_call_fn=self._cell.call, **kwargs)
+
+  def build(self, inputs_shape):
+    """Builds the wrapped cell."""
+    self._cell.build(inputs_shape)
+    self.built = True
+
+
 @tf_export(v1=["nn.rnn_cell.DropoutWrapper"])
-class DropoutWrapper(RNNCell):
+class DropoutWrapper(_RNNCellWrapperV1):
   """Operator adding dropout to inputs and outputs of the given cell."""
 
   def __init__(self, cell, input_keep_prob=1.0, output_keep_prob=1.0,
@@ -1156,7 +1255,7 @@ class DropoutWrapper(RNNCell):
         but not `callable`.
       ValueError: if any of the keep_probs are not between 0 and 1.
     """
-    super(DropoutWrapper, self).__init__()
+    super(DropoutWrapper, self).__init__(cell)
     assert_like_rnncell("cell", cell)
 
     if (dropout_state_filter_visitor is not None
@@ -1181,10 +1280,7 @@ class DropoutWrapper(RNNCell):
         else:
           setattr(self, "_%s" % attr, tensor_prob)
 
-    # Set cell, variational_recurrent, seed before running the code below
-    self._cell = cell
-    if isinstance(cell, checkpointable.Checkpointable):
-      self._track_checkpointable(self._cell, name="cell")
+    # Set variational_recurrent, seed before running the code below
     self._variational_recurrent = variational_recurrent
     self._seed = seed
 
@@ -1291,16 +1387,13 @@ class DropoutWrapper(RNNCell):
           shallow_filtered_substructure, dropout,
           *[shallow_filtered_substructure, values, recurrent_noise])
 
-  def _call(self, inputs, state, call_fn, **kwargs):
-    """Defines a helper method that runs the wrapped cell and applies dropout.
-
-    This helper is called from the DropoutWrapper's `call` or `__call__`
-    methods.
+  def _call_wrapped_cell(self, inputs, state, cell_call_fn, **kwargs):
+    """Runs the wrapped cell and applies dropout.
 
     Args:
       inputs: A tensor with wrapped cell's input.
       state: A tensor or tuple of tensors with wrapped cell's state.
-      call_fn: Wrapped cell's method to use for step computation (cell's
+      cell_call_fn: Wrapped cell's method to use for step computation (cell's
         `__call__` or 'call' method).
       **kwargs: Additional arguments.
 
@@ -1317,7 +1410,7 @@ class DropoutWrapper(RNNCell):
       inputs = self._dropout(inputs, "input",
                              self._recurrent_input_noise,
                              self._input_keep_prob)
-    output, new_state = call_fn(inputs, state, **kwargs)
+    output, new_state = cell_call_fn(inputs, state, **kwargs)
     if _should_dropout(self._state_keep_prob):
       # Identify which subsets of the state to perform dropout on and
       # which ones to keep.
@@ -1333,40 +1426,81 @@ class DropoutWrapper(RNNCell):
                              self._output_keep_prob)
     return output, new_state
 
-  def __call__(self, inputs, state, scope=None):
-    """Runs the cell with the declared dropouts.
-
-    We assume that the wrapped RNNCell is being built within its `__call__`
-    method. We directly use the wrapped cell's `__call__` in the overridden
-    DropoutWrapper `__call__` method.
-
-    This should allow to use the wrapped cell and the non-wrapped cell
-    equivalently when using `__call__`.
-
-    Args:
-      inputs: A tensor with wrapped cell's input.
-      state: A tensor or tuple of tensors with wrapped cell's state.
-      scope: VariableScope for the subgraph created in the wrapped cells'
-        `__call__`.
-
-    Returns:
-      A pair containing:
-
-      - Output: A tensor with cell's output.
-      - New state: A tensor or tuple of tensors with new wrapped cell's state.
-    """
-    return self._call(inputs, state, call_fn=self._cell.__call__, scope=scope)
-
 
 @tf_export("rnn.DropoutWrapper", v1=[])
-class DropoutWrapperV2(LayerRNNCell, DropoutWrapper):
+class DropoutWrapperV2(_RNNCellWrapperV2, DropoutWrapper):
   """Operator adding dropout to inputs and outputs of the given cell."""
 
   def __init__(self, cell, input_keep_prob=1.0, output_keep_prob=1.0,
                state_keep_prob=1.0, variational_recurrent=False,
                input_size=None, dtype=None, seed=None,
                dropout_state_filter_visitor=None):
-    """Runs init in Keras style scope to use Keras-style variable management."""
+    """Create a cell with added input, state, and/or output dropout.
+
+    If `variational_recurrent` is set to `True` (**NOT** the default behavior),
+    then the same dropout mask is applied at every step, as described in:
+
+    Y. Gal, Z Ghahramani.  "A Theoretically Grounded Application of Dropout in
+    Recurrent Neural Networks".  https://arxiv.org/abs/1512.05287
+
+    Otherwise a different dropout mask is applied at every time step.
+
+    Note, by default (unless a custom `dropout_state_filter` is provided),
+    the memory state (`c` component of any `LSTMStateTuple`) passing through
+    a `DropoutWrapper` is never modified.  This behavior is described in the
+    above article.
+
+    Runs initialization in Keras style scope to use Keras-style variable
+    management.
+
+    Args:
+      cell: a LayerRNNCell, a projection to output_size is added to it.
+      input_keep_prob: unit Tensor or float between 0 and 1, input keep
+        probability; if it is constant and 1, no input dropout will be added.
+      output_keep_prob: unit Tensor or float between 0 and 1, output keep
+        probability; if it is constant and 1, no output dropout will be added.
+      state_keep_prob: unit Tensor or float between 0 and 1, output keep
+        probability; if it is constant and 1, no output dropout will be added.
+        State dropout is performed on the outgoing states of the cell.
+        **Note** the state components to which dropout is applied when
+        `state_keep_prob` is in `(0, 1)` are also determined by
+        the argument `dropout_state_filter_visitor` (e.g. by default dropout
+        is never applied to the `c` component of an `LSTMStateTuple`).
+      variational_recurrent: Python bool.  If `True`, then the same
+        dropout pattern is applied across all time steps per run call.
+        If this parameter is set, `input_size` **must** be provided.
+      input_size: (optional) (possibly nested tuple of) `TensorShape` objects
+        containing the depth(s) of the input tensors expected to be passed in to
+        the `DropoutWrapper`.  Required and used **iff**
+         `variational_recurrent = True` and `input_keep_prob < 1`.
+      dtype: (optional) The `dtype` of the input, state, and output tensors.
+        Required and used **iff** `variational_recurrent = True`.
+      seed: (optional) integer, the randomness seed.
+      dropout_state_filter_visitor: (optional), default: (see below).  Function
+        that takes any hierarchical level of the state and returns
+        a scalar or depth=1 structure of Python booleans describing
+        which terms in the state should be dropped out.  In addition, if the
+        function returns `True`, dropout is applied across this sublevel.  If
+        the function returns `False`, dropout is not applied across this entire
+        sublevel.
+        Default behavior: perform dropout on all terms except the memory (`c`)
+        state of `LSTMCellState` objects, and don't try to apply dropout to
+        `TensorArray` objects:
+        ```
+        def dropout_state_filter_visitor(s):
+          if isinstance(s, LSTMCellState):
+            # Never perform dropout on the c state.
+            return LSTMCellState(c=False, h=True)
+          elif isinstance(s, TensorArray):
+            return False
+          return True
+        ```
+
+    Raises:
+      TypeError: if `cell` is not an `RNNCell`, or `keep_state_fn` is provided
+        but not `callable`.
+      ValueError: if any of the keep_probs are not between 0 and 1.
+    """
 
     with base_layer.keras_style_scope():
       super(DropoutWrapperV2, self).__init__(
@@ -1380,36 +1514,9 @@ class DropoutWrapperV2(LayerRNNCell, DropoutWrapper):
           seed=seed,
           dropout_state_filter_visitor=dropout_state_filter_visitor)
 
-  def build(self, inputs_shape):
-    self._cell.build(inputs_shape)
-    self.built = True
-
-  def call(self, inputs, state, **kwargs):
-    """Runs the cell with the declared dropouts.
-
-    When `call` is being used, we assume that the DropoutWrapper object has
-    been built and therefore the wrapped cells has been built via its `build`
-    method and its `call` method can be used directly.
-
-    This should allow to use the wrapped cell and the non-wrapped cell
-    equivalently when using `call` and `build`.
-
-    Args:
-      inputs: A tensor with wrapped cell's input.
-      state: A tensor or tuple of tensors with wrapped cell's state.
-      **kwargs: Additional arguments passed to the wrapped cell's `call`.
-
-    Returns:
-      A pair containing:
-
-      - Output: A tensor with cell's output.
-      - New state: A tensor or tuple of tensors with new wrapped cell's state.
-    """
-    return self._call(inputs, state, call_fn=self._cell.call, **kwargs)
-
 
-@tf_export("nn.rnn_cell.ResidualWrapper")
-class ResidualWrapper(RNNCell):
+@tf_export(v1=["nn.rnn_cell.ResidualWrapper"])
+class ResidualWrapper(_RNNCellWrapperV1):
   """RNNCell wrapper that ensures cell inputs are added to the outputs."""
 
   def __init__(self, cell, residual_fn=None):
@@ -1422,10 +1529,7 @@ class ResidualWrapper(RNNCell):
         Defaults to calling nest.map_structure on (lambda i, o: i + o), inputs
         and outputs.
     """
-    super(ResidualWrapper, self).__init__()
-    self._cell = cell
-    if isinstance(cell, checkpointable.Checkpointable):
-      self._track_checkpointable(self._cell, name="cell")
+    super(ResidualWrapper, self).__init__(cell)
     self._residual_fn = residual_fn
 
   @property
@@ -1440,13 +1544,15 @@ class ResidualWrapper(RNNCell):
     with ops.name_scope(type(self).__name__ + "ZeroState", values=[batch_size]):
       return self._cell.zero_state(batch_size, dtype)
 
-  def __call__(self, inputs, state, scope=None):
+  def _call_wrapped_cell(self, inputs, state, cell_call_fn, **kwargs):
     """Run the cell and then apply the residual_fn on its inputs to its outputs.
 
     Args:
       inputs: cell inputs.
       state: cell state.
-      scope: optional cell scope.
+      cell_call_fn: Wrapped cell's method to use for step computation (cell's
+        `__call__` or 'call' method).
+      **kwargs: Additional arguments passed to the wrapped cell's `call`.
 
     Returns:
       Tuple of cell outputs and new state.
@@ -1455,7 +1561,7 @@ class ResidualWrapper(RNNCell):
       TypeError: If cell inputs and outputs have different structure (type).
       ValueError: If cell inputs and outputs have different structure (value).
     """
-    outputs, new_state = self._cell(inputs, state, scope=scope)
+    outputs, new_state = cell_call_fn(inputs, state, **kwargs)
     # Ensure shapes match
     def assert_shape_match(inp, out):
       inp.get_shape().assert_is_compatible_with(out.get_shape())
@@ -1467,6 +1573,29 @@ class ResidualWrapper(RNNCell):
     return (res_outputs, new_state)
 
 
+@tf_export("rnn.ResidualWrapper", v1=[])
+class ResidualWrapperV2(_RNNCellWrapperV2, ResidualWrapper):
+  """RNNCell wrapper that ensures cell inputs are added to the outputs."""
+
+  def __init__(self, cell, residual_fn=None):
+    """Constructs a `ResidualWrapperV2` for `cell`.
+
+    Runs initialization in Keras style scope to use Keras-style variable
+    management.
+
+    Args:
+      cell: An instance of `LayerRNNCell`.
+      residual_fn: (Optional) The function to map raw cell inputs and raw cell
+        outputs to the actual cell outputs of the residual network.
+        Defaults to calling nest.map_structure on (lambda i, o: i + o), inputs
+        and outputs.
+    """
+
+    with base_layer.keras_style_scope():
+      super(ResidualWrapperV2, self).__init__(
+          cell=cell, residual_fn=residual_fn)
+
+
 @tf_export("nn.rnn_cell.DeviceWrapper")
 class DeviceWrapper(RNNCell):
   """Operator that ensures an RNNCell runs on a particular device."""
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt
index 14cf5ce456..f273e11a08 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.nn.rnn_cell.DropoutWrapper"
 tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.DropoutWrapper\'>"
+  is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl._RNNCellWrapperV1\'>"
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt
index 9de73076b1..4003e87b89 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.nn.rnn_cell.ResidualWrapper"
 tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.ResidualWrapper\'>"
+  is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl._RNNCellWrapperV1\'>"
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.pbtxt
index e2496dff63..953722467b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.pbtxt
@@ -12,8 +12,4 @@ tf_module {
     name: "RNNCell"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "ResidualWrapper"
-    mtype: "<type \'type\'>"
-  }
 }
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.rnn.-dropout-wrapper.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.rnn.-dropout-wrapper.pbtxt
index 7781337c82..9f6ce04225 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.rnn.-dropout-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.rnn.-dropout-wrapper.pbtxt
@@ -1,8 +1,10 @@
 path: "tensorflow.rnn.DropoutWrapper"
 tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.DropoutWrapperV2\'>"
+  is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl._RNNCellWrapperV2\'>"
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.LayerRNNCell\'>"
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.DropoutWrapper\'>"
+  is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl._RNNCellWrapperV1\'>"
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.rnn.-residual-wrapper.pbtxt
similarity index 91%
rename from tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt
rename to tensorflow/tools/api/golden/v2/tensorflow.rnn.-residual-wrapper.pbtxt
index 9de73076b1..51dc8c1517 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.rnn.-residual-wrapper.pbtxt
@@ -1,6 +1,10 @@
-path: "tensorflow.nn.rnn_cell.ResidualWrapper"
+path: "tensorflow.rnn.ResidualWrapper"
 tf_class {
+  is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.ResidualWrapperV2\'>"
+  is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl._RNNCellWrapperV2\'>"
+  is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.LayerRNNCell\'>"
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.ResidualWrapper\'>"
+  is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl._RNNCellWrapperV1\'>"
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
@@ -132,11 +136,11 @@ tf_class {
   }
   member_method {
     name: "build"
-    argspec: "args=[\'self\', \'_\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'self\', \'inputs_shape\'], varargs=None, keywords=None, defaults=None"
   }
   member_method {
     name: "call"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+    argspec: "args=[\'self\', \'inputs\', \'state\'], varargs=None, keywords=kwargs, defaults=None"
   }
   member_method {
     name: "compute_mask"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.rnn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.rnn.pbtxt
index 42b13533dd..32be6c7018 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.rnn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.rnn.pbtxt
@@ -4,4 +4,8 @@ tf_module {
     name: "DropoutWrapper"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "ResidualWrapper"
+    mtype: "<type \'type\'>"
+  }
 }
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 011be5172d..3e147d5478 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -444,6 +444,7 @@ renames = {
     'tf.nn.rnn_cell.GRUCell': 'tf.compat.v1.nn.rnn_cell.GRUCell',
     'tf.nn.rnn_cell.LSTMCell': 'tf.compat.v1.nn.rnn_cell.LSTMCell',
     'tf.nn.rnn_cell.MultiRNNCell': 'tf.compat.v1.nn.rnn_cell.MultiRNNCell',
+    'tf.nn.rnn_cell.ResidualWrapper': 'tf.compat.v1.nn.rnn_cell.ResidualWrapper',
     'tf.nn.static_bidirectional_rnn': 'tf.compat.v1.nn.static_bidirectional_rnn',
     'tf.nn.static_rnn': 'tf.compat.v1.nn.static_rnn',
     'tf.nn.uniform_candidate_sampler': 'tf.random.uniform_candidate_sampler',
-- 
GitLab


From 0db554ab7e31311bb6176cbcb7cb046c6a89fdcb Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Fri, 1 Feb 2019 08:42:57 -0800
Subject: [PATCH 1750/2345] [Grappler] Update DependencyOptimizer to use
 MutableGraphView in place of NodeMap.

PiperOrigin-RevId: 231977395
---
 tensorflow/core/grappler/optimizers/BUILD     |   9 +-
 .../optimizers/dependency_optimizer.cc        | 515 ++++++++----------
 .../optimizers/dependency_optimizer.h         |  30 +-
 .../optimizers/dependency_optimizer_test.cc   | 269 ++++-----
 4 files changed, 378 insertions(+), 445 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index cb5d7d6746..4c8b4f69d6 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -1,9 +1,7 @@
 licenses(["notice"])  # Apache 2.0
 
-load("//tensorflow:tensorflow.bzl", "tf_cc_test")
+load("//tensorflow:tensorflow.bzl", "tf_cc_test", "tf_kernel_library")
 load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_test")
-load("//tensorflow:tensorflow.bzl", "tf_kernel_library")
-load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda")
 
 # Platform specific build config
 load(
@@ -306,14 +304,19 @@ cc_library(
         ":constant_folding",
         ":graph_optimizer",
         "//tensorflow/core:framework",
+        "//tensorflow/core:graph",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/grappler:grappler_item",
+        "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/costs:graph_properties",
         "//tensorflow/core/grappler/utils:topological_sort",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/container:flat_hash_set",
+        "@com_google_absl//absl/strings",
     ],
 )
 
diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer.cc b/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
index 8b81cb2430..9712ad63b0 100644
--- a/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
@@ -15,13 +15,13 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/dependency_optimizer.h"
 
-#include <unordered_map>
-#include <unordered_set>
-
+#include "absl/container/flat_hash_map.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/graph/tensor_id.h"
 #include "tensorflow/core/grappler/costs/graph_properties.h"
 #include "tensorflow/core/grappler/grappler_item.h"
+#include "tensorflow/core/grappler/mutable_graph_view.h"
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/grappler/optimizers/constant_folding.h"
 #include "tensorflow/core/grappler/utils.h"
@@ -38,20 +38,15 @@ namespace grappler {
 
 namespace {
 
-bool RemoveInput(NodeDef* node, const string& input, NodeMap* node_map) {
-  bool removed_input = false;
-  int pos = 0;
-  while (pos < node->input_size()) {
-    if (node->input(pos) == input) {
-      node->mutable_input()->SwapElements(pos, node->input_size() - 1);
-      node->mutable_input()->RemoveLast();
-      node_map->RemoveOutput(NodeName(input), node->name());
-      removed_input = true;
-    } else {
-      ++pos;
-    }
+// Builds a map from the &graph->node(i) to i.
+absl::flat_hash_map<const NodeDef*, int> BuildNodeToIdx(const GraphDef& graph) {
+  // Set up &node -> index map.
+  absl::flat_hash_map<const NodeDef*, int> node_to_idx;
+  for (int i = 0; i < graph.node_size(); ++i) {
+    const NodeDef& node = graph.node(i);
+    node_to_idx[&node] = i;
   }
-  return removed_input;
+  return node_to_idx;
 }
 
 }  // namespace
@@ -68,7 +63,9 @@ bool DependencyOptimizer::SafeToRemoveIdentity(const NodeDef& node) const {
     // The output values of this node may be needed.
     return false;
   }
-  const NodeDef* input = node_map_->GetNode(NodeName(node.input(0)));
+  MutableGraphView::OutputPort port = graph_view_->GetRegularFanin(
+      MutableGraphView::InputPort(const_cast<NodeDef*>(&node), 0));
+  NodeDef* input = port.node;
   CHECK(input != nullptr) << "node = " << node.name()
                           << " input = " << node.input(0);
   // Don't remove Identity nodes corresponding to Variable reads or following
@@ -79,20 +76,23 @@ bool DependencyOptimizer::SafeToRemoveIdentity(const NodeDef& node) const {
     // Don't turn Identity nodes following Switch into NoOp or remove them
     // if it requires anchoring a control dependencies the Switch node, which
     // is not valid.
-    if (str_util::StartsWith(node.name(), kConstantFoldingCtrl)) {
-      // TODO(rmlarsen): Try to remove this artificial contraint.
+    MutableGraphView::OutputPort control_port(const_cast<NodeDef*>(&node),
+                                              Graph::kControlSlot);
+    if (!graph_view_->GetFanout(control_port).empty()) {
       return false;
     }
   }
-  for (auto consumer : node_map_->GetOutputs(node.name())) {
-    if (node.input_size() > 1 && IsMerge(*consumer)) {
+  bool node_has_multiple_inputs =
+      graph_view_->NumFanins(node, /*include_controlling_nodes=*/true) > 1;
+  for (auto consumer :
+       graph_view_->GetFanouts(node, /*include_controlled_nodes=*/true)) {
+    if (node_has_multiple_inputs && IsMerge(*consumer.node)) {
       return false;
     }
     if (IsSwitch(*input)) {
-      for (const string& consumer_input : consumer->input()) {
-        if (consumer_input == AsControlDependency(node.name())) {
-          return false;
-        }
+      if (graph_view_->HasFanin(*consumer.node,
+                                {node.name(), Graph::kControlSlot})) {
+        return false;
       }
     }
   }
@@ -116,7 +116,7 @@ bool DependencyOptimizer::SafeToConvertToNoOp(const NodeDef& node) const {
   if (!status.ok() || op_def->output_arg_size() == 0) {
     return false;
   }
-  const std::unordered_set<string> do_not_rewrite_ops{
+  const absl::flat_hash_set<string> do_not_rewrite_ops{
       "Assert",     "CheckNumerics",         "_Retval",
       "_Arg",       "_ParallelConcatUpdate", "TPUExecute",
       "TPUCompile", "ControlTrigger"};
@@ -126,7 +126,7 @@ bool DependencyOptimizer::SafeToConvertToNoOp(const NodeDef& node) const {
   if (!SafeToRemoveIdentity(node)) {
     return false;
   }
-  if (NumNonControlOutputs(node, *node_map_) > 0) {
+  if (graph_view_->NumFanouts(node, /*include_controlled_nodes=*/false) > 0) {
     // The output values of this node may be needed.
     return false;
   }
@@ -134,61 +134,58 @@ bool DependencyOptimizer::SafeToConvertToNoOp(const NodeDef& node) const {
 }
 
 int DependencyOptimizer::NumEdgesIfBypassed(
-    const NodeDef& node, const std::vector<NodeDef*>& output_nodes) const {
+    const NodeDef& node,
+    const absl::flat_hash_set<MutableGraphView::Edge>& fanout_edges) const {
   const bool is_multi_input_identity_n =
       IsIdentityN(node) && !IsIdentityNSingleInput(node);
-  const int num_outputs = output_nodes.size();
-  const int num_inputs = node.input_size();
+  const int num_fanins = node.input_size();
+  const int num_fanouts = fanout_edges.size();
 
   if (is_multi_input_identity_n) {
     // multi-input identity_n with input/output control dependencies will likely
     // increase number of edges after optimization.
     int num_edges_if_bypassed(0);
-    for (string input_node_name : node.input()) {
-      if (IsControlInput(input_node_name)) {
-        num_edges_if_bypassed += num_outputs;
+    int num_non_controlling_fanins =
+        graph_view_->NumFanins(node, /*include_controlling_nodes=*/false);
+    num_edges_if_bypassed += num_non_controlling_fanins;
+    num_edges_if_bypassed +=
+        (num_fanins - num_non_controlling_fanins) * num_fanouts;
+
+    for (const auto& fanout : fanout_edges) {
+      if (fanout.dst.port_id == Graph::kControlSlot) {
+        num_edges_if_bypassed += num_fanins;
       } else {
         ++num_edges_if_bypassed;
       }
     }
-
-    for (auto consumer : output_nodes) {
-      for (int j = 0; j < consumer->input_size(); ++j) {
-        const TensorId consumer_input = ParseTensorName(consumer->input(j));
-        if (consumer_input.node() == node.name()) {
-          if (IsControlInput(consumer_input)) {
-            num_edges_if_bypassed += num_inputs;
-          } else {
-            ++num_edges_if_bypassed;
-          }
-        }
-      }
-    }
     return num_edges_if_bypassed;
   } else {
-    return num_inputs * num_outputs;
+    return num_fanins * num_fanouts;
   }
 }
 
 bool DependencyOptimizer::BypassingNodeIsBeneficial(
-    const NodeDef& node, const std::vector<NodeDef*>& input_nodes,
-    const std::vector<NodeDef*>& output_nodes) const {
+    const NodeDef& node,
+    const absl::flat_hash_set<MutableGraphView::Edge>& fanout_edges) const {
   const bool is_identity = IsIdentity(node) || IsIdentityNSingleInput(node);
   const bool is_multi_input_identity_n =
       IsIdentityN(node) && !IsIdentityNSingleInput(node);
-  const int num_outputs = output_nodes.size();
-  const int num_inputs = node.input_size();
+  const int num_fanins = node.input_size();
+  const int num_fanouts = fanout_edges.size();
 
-  if (NumEdgesIfBypassed(node, output_nodes) > num_inputs + num_outputs) {
+  if (NumEdgesIfBypassed(node, fanout_edges) > num_fanins + num_fanouts) {
     return false;
   }
 
+  auto fanin_edges =
+      graph_view_->GetFaninEdges(node, /*include_controlling_edges=*/true);
+
   // Make sure that we don't increase the number of edges that cross
   // device boundaries.
-  if ((num_inputs == 1 && num_outputs > 1 &&
-       input_nodes[0]->device() != node.device()) ||
-      (num_inputs > 1 && num_outputs == 1 &&
-       output_nodes[0]->device() != node.device())) {
+  if ((num_fanins == 1 && num_fanouts > 1 &&
+       fanin_edges.begin()->src.node->device() != node.device()) ||
+      (num_fanins > 1 && num_fanouts == 1 &&
+       fanout_edges.begin()->dst.node->device() != node.device())) {
     return false;
   }
 
@@ -197,21 +194,21 @@ bool DependencyOptimizer::BypassingNodeIsBeneficial(
   // cost before and after.
   const string& node_dev = node.device();
   int num_cross_in = 0;
-  for (NodeDef* input_node : input_nodes) {
-    num_cross_in += static_cast<int>(input_node->device() != node_dev);
+  for (const auto& fanin : fanin_edges) {
+    num_cross_in += static_cast<int>(fanin.src.node->device() != node_dev);
   }
   int num_cross_out = 0;
-  for (NodeDef* output_node : output_nodes) {
-    num_cross_out += static_cast<int>(output_node->device() != node_dev);
+  for (const auto& fanout : fanout_edges) {
+    num_cross_out += static_cast<int>(fanout.dst.node->device() != node_dev);
   }
 
   // Make sure we do not increase the number of device crossings.
   const int num_cross_before = num_cross_in + num_cross_out;
   int num_cross_after = 0;
-  for (NodeDef* input_node : input_nodes) {
-    for (NodeDef* output_node : output_nodes) {
-      num_cross_after +=
-          static_cast<int>(input_node->device() != output_node->device());
+  for (const auto& fanin : fanin_edges) {
+    for (const auto& fanout : fanout_edges) {
+      num_cross_after += static_cast<int>(fanin.src.node->device() !=
+                                          fanout.dst.node->device());
     }
   }
   if (num_cross_after > num_cross_before) {
@@ -229,47 +226,34 @@ bool DependencyOptimizer::BypassingNodeIsBeneficial(
   return true;
 }
 
-void DependencyOptimizer::OptimizeNode(int node_idx,
-                                       SetVector<int>* nodes_to_simplify,
-                                       std::set<int>* nodes_to_delete) {
-  NodeDef* node = optimized_graph_->mutable_node(node_idx);
+Status DependencyOptimizer::OptimizeNode(
+    const string& node_name, SetVector<string>* nodes_to_simplify,
+    absl::flat_hash_set<string>* nodes_to_delete) {
+  NodeDef* node = graph_view_->GetNode(node_name);
   const bool is_noop = IsNoOp(*node);
   const bool is_identity = IsIdentity(*node) || IsIdentityNSingleInput(*node);
   const bool is_multi_input_identity =
       IsIdentityN(*node) && !IsIdentityNSingleInput(*node);
-  const string node_name = node->name();
-  // Constant nodes with no input control dependency are always executed early,
-  // so we can prune all their output control dependencies.
-  if (IsConstant(*node) && node->input_size() == 0) {
-    const std::set<NodeDef*> output_nodes = node_map_->GetOutputs(node_name);
-    for (NodeDef* fanout : output_nodes) {
-      bool optimize_fanout = false;
-      bool data_connection = false;
-      for (int i = fanout->input_size() - 1; i >= 0; --i) {
-        const TensorId input_tensor = ParseTensorName(fanout->input(i));
-        if (input_tensor.node() == node_name) {
-          if (input_tensor.index() < 0) {
-            fanout->mutable_input()->SwapElements(i, fanout->input_size() - 1);
-            fanout->mutable_input()->RemoveLast();
-            optimize_fanout = true;
-          } else {
-            data_connection = true;
-          }
-        }
-      }
-      if (optimize_fanout) {
-        nodes_to_simplify->PushBack(node_to_idx_[fanout]);
-        if (!data_connection) {
-          node_map_->RemoveOutput(node_name, fanout->name());
-        }
-      }
+  // WARNING: This is a strong assumption based on the executor behavior that
+  // constant nodes with no input control dependency are always executed early.
+  // In this case we then can prune all their output control dependencies.
+  if (IsConstant(*node) &&
+      graph_view_->NumFanins(*node, /*include_controlling_nodes=*/true) == 0) {
+    for (const auto& fanout :
+         graph_view_->GetFanouts(*node, /*include_controlled_nodes=*/true)) {
+      TF_RETURN_IF_ERROR(
+          graph_view_->RemoveControllingFanin(fanout.node->name(), node_name));
+      nodes_to_simplify->PushBack(fanout.node->name());
     }
-    if (node_map_->GetOutputs(node_name).empty() && fetch_nodes_known_ &&
+
+    if (graph_view_->NumFanouts(*node, /*include_controlled_nodes=*/true) ==
+            0 &&
+        fetch_nodes_known_ &&
         nodes_to_preserve_.find(node_name) == nodes_to_preserve_.end()) {
       // Mark the node for deletion.
-      nodes_to_delete->insert(node_to_idx_[node]);
+      nodes_to_delete->insert(node->name());
     }
-    return;
+    return Status::OK();
   }
 
   // Change ops that only have control dependencies as outputs to NoOps.
@@ -278,34 +262,21 @@ void DependencyOptimizer::OptimizeNode(int node_idx,
             << ") with NoOp.";
     // The outputs of this node are not consumed. Replace its inputs with
     // control dependencies and replace the op itself with the NoOp op.
-    std::unordered_set<string> ctrl_inputs;
-    int pos = 0;
-    while (pos < node->input_size()) {
-      const string old_input = node->input(pos);
-      if (IsControlInput(old_input)) {
-        if (!ctrl_inputs.insert(old_input).second) {
-          // We found a duplicate control input. Remove it.
-          node->mutable_input()->SwapElements(pos, node->input_size() - 1);
-          node->mutable_input()->RemoveLast();
-        } else {
-          ++pos;
-        }
-        continue;
-      }
-      // Replace a normal input with a control input.
-      const string ctrl_input = ConstantFolding::AddControlDependency(
-          old_input, optimized_graph_, node_map_.get());
-      ctrl_inputs.insert(ctrl_input);
-      node->set_input(pos, ctrl_input);
-      node_map_->UpdateInput(node_name, old_input, ctrl_input);
-      const NodeDef* old_input_node = node_map_->GetNode(old_input);
-      nodes_to_simplify->PushBack(node_to_idx_[old_input_node]);
-      ++pos;
+    const int num_regular_fanins =
+        graph_view_->NumFanins(*node, /*include_controlling_nodes=*/false);
+    absl::flat_hash_set<string> regular_fanin_names;
+    for (int i = 0; i < num_regular_fanins; ++i) {
+      regular_fanin_names.emplace(ParseTensorName(node->input(i)).node());
     }
-    node->set_op("NoOp");
-    node->clear_attr();
-    nodes_to_simplify->PushBack(node_to_idx_[node]);
-    return;
+    TF_RETURN_IF_ERROR(
+        graph_view_->UpdateAllRegularFaninsToControlling(node_name));
+    TF_RETURN_IF_ERROR(
+        graph_view_->UpdateNode(node->name(), "NoOp", node->device(), {}));
+    nodes_to_simplify->PushBack(node->name());
+    for (const string& regular_fanin_name : regular_fanin_names) {
+      nodes_to_simplify->PushBack(regular_fanin_name);
+    }
+    return Status::OK();
   }
 
   // Remove NoOp nodes if the product of their fan-in and fan-out is less than
@@ -358,128 +329,114 @@ void DependencyOptimizer::OptimizeNode(int node_idx,
 
   if (is_noop || ((is_identity || is_multi_input_identity) &&
                   SafeToRemoveIdentity(*node))) {
-    const auto& output_node_set = node_map_->GetOutputs(node_name);
-    const std::vector<NodeDef*> output_nodes(output_node_set.begin(),
-                                             output_node_set.end());
-    const int num_inputs = node->input_size();
-    std::vector<NodeDef*> input_nodes;
-    for (int i = 0; i < num_inputs; ++i) {
-      NodeDef* input_node = node_map_->GetNode(node->input(i));
-      if (input_node == nullptr) {
-        LOG(ERROR) << "Invalid input " << node->input(i);
-        return;
-      }
-      input_nodes.push_back(input_node);
+    auto fanout_edges =
+        graph_view_->GetFanoutEdges(*node, /*include_controlled_edges=*/true);
+    if (!BypassingNodeIsBeneficial(*node, fanout_edges)) {
+      return Status::OK();
     }
 
-    if (!BypassingNodeIsBeneficial(*node, input_nodes, output_nodes)) {
-      return;
-    }
+    if (!fanout_edges.empty()) {
+      const int num_regular_fanins =
+          graph_view_->NumFanins(*node, /*include_controlling_nodes=*/false);
+      std::vector<TensorId> regular_fanin_ids;
+      regular_fanin_ids.reserve(num_regular_fanins);
+      for (int i = 0; i < num_regular_fanins; ++i) {
+        regular_fanin_ids.push_back(ParseTensorName(node->input(i)));
+      }
 
-    VLOG(1) << "***** Rerouting input around\n" << node->DebugString();
-    // Now remove the node and re-wire its inputs to its outputs.
-    for (auto consumer : output_nodes) {
-      bool updated_consumer = false;
-      VLOG(1) << "consumer before:\n" << consumer->DebugString();
-      for (int i = 0; i < num_inputs; ++i) {
-        const NodeDef* input = input_nodes[i];
-        // Forward dependency from input to consumer if it doesn't already
-        // depend on it.
-        if ((is_identity && i == 0) ||
-            (is_multi_input_identity && !IsControlInput(node->input(i)))) {
-          // Replace regular input from Identity node.
-          string new_input;
-          const string& input_to_forward = node->input(i);
-          CHECK(!IsControlInput(input_to_forward));
-          for (int j = 0; j < consumer->input_size(); ++j) {
-            const TensorId old_input = ParseTensorName(consumer->input(j));
-            if (old_input.node() == node_name) {
-              if (old_input.index() == i) {
-                // Regular input
-                new_input = input_to_forward;
-                node_map_->UpdateInput(consumer->name(), old_input.ToString(),
-                                       new_input);
-                consumer->set_input(j, new_input);
-              } else if (old_input.index() == -1) {
-                // Control dependency
-                new_input = AsControlDependency(NodeName(input_to_forward));
-                node_map_->UpdateInput(consumer->name(), old_input.ToString(),
-                                       new_input);
-                consumer->set_input(j, new_input);
-              }
-            }
+      absl::flat_hash_set<NodeDef*> processed_consumers;
+      VLOG(1) << "***** Rerouting input around\n" << node->DebugString();
+      // Now remove the node and re-wire its inputs to its outputs.
+      for (auto fanout_edge : fanout_edges) {
+        NodeDef* consumer = fanout_edge.dst.node;
+        VLOG(1) << "consumer before:\n" << consumer->DebugString();
+        if (fanout_edge.src.port_id == Graph::kControlSlot) {
+          TF_RETURN_IF_ERROR(
+              graph_view_->RemoveControllingFanin(consumer->name(), node_name));
+          // Add all non controlling fanins of node as controlling fanins to
+          // consumer.
+          for (const TensorId& regular_fanin_id : regular_fanin_ids) {
+            TF_RETURN_IF_ERROR(graph_view_->AddControllingFanin(
+                consumer->name(), regular_fanin_id));
+            nodes_to_simplify->PushBack(string(regular_fanin_id.node()));
           }
-          updated_consumer = true;
         } else {
-          // Forward dependency from input to consumer if it doesn't already
-          // depend on it.
-          if (node_map_->GetOutputs(input->name()).count(consumer) == 0) {
-            consumer->add_input(AsControlDependency(input->name()));
-            node_map_->AddOutput(input->name(), consumer->name());
-            nodes_to_simplify->PushBack(node_to_idx_[input]);
-            updated_consumer = true;
-          }
+          TF_RETURN_IF_ERROR(graph_view_->UpdateRegularFaninByPort(
+              consumer->name(), fanout_edge.dst.port_id,
+              ParseTensorName(node->input(fanout_edge.src.port_id))));
+        }
+        processed_consumers.insert(consumer);
+        VLOG(1) << "consumer after:\n" << consumer->DebugString();
+      }
+
+      std::vector<TensorId> controlling_fanin_ids;
+      controlling_fanin_ids.reserve(node->input_size() - num_regular_fanins);
+      for (int i = num_regular_fanins; i < node->input_size(); ++i) {
+        controlling_fanin_ids.push_back(ParseTensorName(node->input(i)));
+      }
+
+      for (NodeDef* consumer : processed_consumers) {
+        // Forward all controlling fanins of node to consumer.
+        VLOG(1) << "consumer before:\n" << consumer->DebugString();
+        for (const TensorId& controlling_fanin_id : controlling_fanin_ids) {
+          TF_RETURN_IF_ERROR(graph_view_->AddControllingFanin(
+              consumer->name(), controlling_fanin_id));
         }
+        nodes_to_simplify->PushBack(consumer->name());
+        VLOG(1) << "consumer after:\n" << consumer->DebugString();
       }
-      // Remove dependency on node from consumer.
-      updated_consumer |= RemoveInput(consumer, AsControlDependency(node_name),
-                                      node_map_.get());
-      if (updated_consumer) {
-        nodes_to_simplify->PushBack(node_to_idx_[consumer]);
+      for (const TensorId& controlling_fanin_id : controlling_fanin_ids) {
+        nodes_to_simplify->PushBack(string(controlling_fanin_id.node()));
       }
-      VLOG(1) << "consumer after:\n" << consumer->DebugString();
     }
-    node_map_->RemoveOutputs(node_name);
+
     if (fetch_nodes_known_ &&
         nodes_to_preserve_.find(node_name) == nodes_to_preserve_.end()) {
       // Mark the node for deletion.
-      nodes_to_delete->insert(node_idx);
+      nodes_to_delete->insert(node_name);
 
       // Disconnect the node from its inputs to enable further optimizations.
-      node_map_->RemoveInputs(node_name);
-      node->clear_input();
+      TF_RETURN_IF_ERROR(
+          graph_view_->RemoveAllFanins(node_name,
+                                       /*keep_controlling_fanins=*/false));
     }
   }
-}
-
-void DependencyOptimizer::CleanControlInputs() {
-  for (int i = 0; i < optimized_graph_->node_size(); ++i) {
-    DedupControlInputs(optimized_graph_->mutable_node(i));
-  }
+  return Status::OK();
 }
 
 Status DependencyOptimizer::OptimizeDependencies() {
-  SetVector<int> nodes_to_simplify;
-  std::set<int> nodes_to_delete;
-  for (int i = 0; i < optimized_graph_->node_size(); ++i) {
-    const NodeDef& node = optimized_graph_->node(i);
+  SetVector<string> nodes_to_simplify;
+  absl::flat_hash_set<string> nodes_to_delete;
+  for (int i = 0; i < graph_view_->graph()->node_size(); ++i) {
+    const NodeDef& node = graph_view_->graph()->node(i);
     if (IsNoOp(node) || IsIdentity(node) || IsIdentityN(node) ||
         IsConstant(node) || SafeToConvertToNoOp(node)) {
-      nodes_to_simplify.PushBack(i);
+      nodes_to_simplify.PushBack(node.name());
     }
   }
   while (!nodes_to_simplify.Empty()) {
-    int node_to_simplify = nodes_to_simplify.PopBack();
+    string node_to_simplify = nodes_to_simplify.PopBack();
     // Discard nodes that were marked for deletion already.
     while (nodes_to_delete.find(node_to_simplify) != nodes_to_delete.end()) {
       node_to_simplify = nodes_to_simplify.PopBack();
     }
-    OptimizeNode(node_to_simplify, &nodes_to_simplify, &nodes_to_delete);
+    TF_RETURN_IF_ERROR(
+        OptimizeNode(node_to_simplify, &nodes_to_simplify, &nodes_to_delete));
   }
 
   if (fetch_nodes_known_) {
     VLOG(1) << "Deleted " << nodes_to_delete.size() << " out of "
-            << optimized_graph_->node_size() << " nodes.";
-    EraseNodesFromGraph(nodes_to_delete, optimized_graph_);
-    node_map_.reset(new NodeMap(optimized_graph_));
-    BuildNodeToIdx();
+            << graph_view_->graph()->node_size() << " nodes.";
+    TF_RETURN_IF_ERROR(graph_view_->DeleteNodes(nodes_to_delete));
   }
   return Status::OK();
 }
 
 Status DependencyOptimizer::TransitiveReduction() {
   // PRECONDITION: optimized_graph_ must be sorted topologically.
-  const int num_nodes = optimized_graph_->node_size();
+  GraphDef* graph = graph_view_->graph();
+  auto node_to_idx = BuildNodeToIdx(*graph);
+  const int num_nodes = graph->node_size();
   // Set up a compressed version of the graph to save a constant factor in the
   // expensive algorithm below. Also cache the set of control outputs and the
   // highest index of a target of any control output from each node.
@@ -488,20 +445,20 @@ Status DependencyOptimizer::TransitiveReduction() {
   std::vector<gtl::InlinedVector<std::pair<int, int>, 2>> control_outputs(
       num_nodes);
   for (int node_idx = 0; node_idx < num_nodes; ++node_idx) {
-    const NodeDef& node = optimized_graph_->node(node_idx);
+    const NodeDef& node = graph->node(node_idx);
     if (ModifiesFrameInfo(node) || !HasOpDef(node)) {
       // Ignore function nodes and nodes that modify frame info.
       continue;
     }
     for (int input_slot = 0; input_slot < node.input_size(); ++input_slot) {
       const string& input = node.input(input_slot);
-      const NodeDef* input_node = node_map_->GetNode(input);
+      const NodeDef* input_node = graph_view_->GetNode(NodeName(input));
       if (ModifiesFrameInfo(*input_node) || IsMerge(*input_node)) {
         // Ignore edges from nodes that modify frame info and from Merge nodes,
         // because we cannot know which of it's input paths executes.
         continue;
       }
-      const int input_node_idx = node_to_idx_[input_node];
+      const int input_node_idx = node_to_idx[input_node];
       inputs[node_idx].push_back(input_node_idx);
       if (IsControlInput(input)) {
         ++num_controls;
@@ -520,7 +477,7 @@ Status DependencyOptimizer::TransitiveReduction() {
   // such that when we swap them out so we don't clobber the
   // node(target).input() repeated field.
   typedef std::pair<int, int> InputSlotAndSource;
-  std::unordered_map<
+  absl::flat_hash_map<
       int, std::set<InputSlotAndSource, std::greater<InputSlotAndSource>>>
       control_edges_to_remove;
   for (int source = 0; source < num_nodes; ++source) {
@@ -567,16 +524,13 @@ Status DependencyOptimizer::TransitiveReduction() {
 
   for (const auto& it : control_edges_to_remove) {
     const int target = it.first;
-    NodeDef* target_node = optimized_graph_->mutable_node(target);
+    const NodeDef& target_node = graph->node(target);
+    const string target_node_name = target_node.name();
     for (const InputSlotAndSource& slot_and_source : it.second) {
       const int input_slot = slot_and_source.first;
-      const int source = slot_and_source.second;
-      const NodeDef& source_node = optimized_graph_->node(source);
-      CHECK_LT(input_slot, target_node->input_size());
-      target_node->mutable_input()->SwapElements(input_slot,
-                                                 target_node->input_size() - 1);
-      node_map_->RemoveOutput(source_node.name(), target_node->name());
-      target_node->mutable_input()->RemoveLast();
+      const TensorId tensor_id = ParseTensorName(target_node.input(input_slot));
+      TF_RETURN_IF_ERROR(graph_view_->RemoveControllingFanin(target_node_name,
+                                                             tensor_id.node()));
       ++num_controls_removed;
     }
   }
@@ -585,26 +539,17 @@ Status DependencyOptimizer::TransitiveReduction() {
   return Status::OK();
 }
 
-void DependencyOptimizer::BuildNodeToIdx() {
-  // Set up &node -> index map.
-  node_to_idx_.clear();
-  for (int i = 0; i < optimized_graph_->node_size(); ++i) {
-    const NodeDef& node = optimized_graph_->node(i);
-    node_to_idx_[&node] = i;
-  }
-}
-
 // Suppose there are cross-device control inputs to node C from multiple nodes
 // that are located on another device, e.g., we have control edges:
 // A->C, B->C
 // where A and B are on device X and C is on device Y.
 // We can reduce cross-device communication by introducing an intermediate
 // NoOp node C' on device X and rewriting the control edges to:
-// A->C', B->C', C' -> C
-void DependencyOptimizer::GroupCrossDeviceControlEdges() {
-  const int num_nodes = optimized_graph_->node_size();
+// A->C', B->C', C'->C
+Status DependencyOptimizer::GroupCrossDeviceControlEdges() {
+  const int num_nodes = graph_view_->graph()->node_size();
   for (int i = 0; i < num_nodes; ++i) {
-    NodeDef* node = optimized_graph_->mutable_node(i);
+    NodeDef* node = graph_view_->graph()->mutable_node(i);
     if (node->device().empty()) continue;
 
     // Creates new noop nodes for devices on which multiple control inputs are
@@ -615,88 +560,71 @@ void DependencyOptimizer::GroupCrossDeviceControlEdges() {
     // that device.
     std::map<string, NodeDef*> noops;
     int num_noops = 0;
-    for (int j = 0; j < node->input_size(); ++j) {
-      if (IsControlInput(node->input(j))) {
-        const NodeDef* input = node_map_->GetNode(node->input(j));
-        if (input != nullptr && !input->device().empty() &&
-            input->device() != node->device()) {
-          auto emplace_result = noops.emplace(input->device(), nullptr);
-          if (!emplace_result.second &&
-              emplace_result.first->second == nullptr) {
-            // This is the second cross-device control input from the same
-            // device. Creates an intermediate noop node on that device.
-            string group_name;
-            NodeDef* noop;
-            // Creates a fresh node name; there may be conflicting names from
-            // a previous iteration of the optimizer.
-            do {
-              group_name = AddPrefixToNodeName(
-                  node->name(),
-                  strings::StrCat("GroupCrossDeviceControlEdges_", num_noops));
-              noop = node_map_->GetNode(group_name);
-              ++num_noops;
-            } while (noop != nullptr);
-            noop = optimized_graph_->add_node();
-            noop->set_name(group_name);
-            noop->set_device(input->device());
-            noop->set_op("NoOp");
-            node_map_->AddNode(noop->name(), noop);
-            emplace_result.first->second = noop;
-          }
+    auto controlling_fanins = graph_view_->GetFanin(
+        MutableGraphView::InputPort(node, Graph::kControlSlot));
+    for (const auto& controlling_fanin : controlling_fanins) {
+      const NodeDef* fanin_node = controlling_fanin.node;
+      if (!fanin_node->device().empty() &&
+          fanin_node->device() != node->device()) {
+        auto emplace_result = noops.emplace(fanin_node->device(), nullptr);
+        if (!emplace_result.second && emplace_result.first->second == nullptr) {
+          // This is the second cross-device control input from the same
+          // device. Creates an intermediate noop node on that device.
+          string group_name;
+          NodeDef* noop;
+          // Creates a fresh node name; there may be conflicting names from
+          // a previous iteration of the optimizer.
+          do {
+            group_name = AddPrefixToNodeName(
+                node->name(),
+                strings::StrCat("GroupCrossDeviceControlEdges_", num_noops));
+            noop = graph_view_->GetNode(group_name);
+            ++num_noops;
+          } while (noop != nullptr);
+          NodeDef new_node;
+          new_node.set_name(group_name);
+          new_node.set_device(fanin_node->device());
+          new_node.set_op("NoOp");
+          emplace_result.first->second =
+              graph_view_->AddNode(std::move(new_node));
         }
       }
     }
 
     // Reroute existing control edges to go via the newly introduced NoOp nodes.
-    int pos = 0;
-    while (pos < node->input_size()) {
-      const string& input_name = node->input(pos);
-      if (IsControlInput(input_name)) {
-        NodeDef* input = node_map_->GetNode(input_name);
-        if (input == nullptr) {
-          ++pos;
-        } else {
-          auto it = noops.find(input->device());
-          if (it == noops.end() || it->second == nullptr) {
-            ++pos;
-          } else {
-            node->mutable_input()->SwapElements(pos, node->input_size() - 1);
-            node->mutable_input()->RemoveLast();
-            it->second->add_input(AsControlDependency(*input));
-            node_map_->UpdateOutput(input_name, node->name(),
-                                    it->second->name());
-          }
-        }
-      } else {
-        ++pos;
+    for (const auto& controlling_fanin : controlling_fanins) {
+      auto it = noops.find(controlling_fanin.node->device());
+      if (it != noops.end() && it->second != nullptr) {
+        TF_RETURN_IF_ERROR(graph_view_->RemoveControllingFanin(
+            node->name(), controlling_fanin.node->name()));
+        TF_RETURN_IF_ERROR(graph_view_->AddControllingFanin(
+            it->second->name(),
+            {controlling_fanin.node->name(), Graph::kControlSlot}));
       }
     }
     for (const auto& entry : noops) {
       if (entry.second) {
-        node->add_input(AsControlDependency(*entry.second));
-        node_map_->AddOutput(entry.second->name(), node->name());
+        TF_RETURN_IF_ERROR(graph_view_->AddControllingFanin(
+            node->name(), {entry.second->name(), Graph::kControlSlot}));
       }
     }
   }
+  return Status::OK();
 }
 
 Status DependencyOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
                                      GraphDef* optimized_graph) {
-  optimized_graph_ = optimized_graph;
-  *optimized_graph_ = item.graph;
+  *optimized_graph = item.graph;
   nodes_to_preserve_ = item.NodesToPreserve();
   fetch_nodes_known_ = !item.fetch.empty();
-  CleanControlInputs();
+  graph_view_.reset(new MutableGraphView(optimized_graph));
 
   const int num_iterations = 2;
   for (int iteration = 0; iteration < num_iterations; ++iteration) {
     GRAPPLER_RETURN_IF_DEADLINE_EXCEEDED();
     Status topo_sort_status;
     // Perform topological sort to prepare the graph for transitive reduction.
-    topo_sort_status = TopologicalSort(optimized_graph_);
-    // Set up index-based graph datastructures to speed up analysis steps below.
-    node_map_.reset(new NodeMap(optimized_graph_));
-    BuildNodeToIdx();
+    topo_sort_status = TopologicalSort(optimized_graph);
 
     if (topo_sort_status.ok()) {
       // Remove redundant control dependencies.
@@ -710,10 +638,7 @@ Status DependencyOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
     // nodes.
     TF_RETURN_IF_ERROR(OptimizeDependencies());
 
-    // Dedup control inputs.
-    CleanControlInputs();
-
-    GroupCrossDeviceControlEdges();
+    TF_RETURN_IF_ERROR(GroupCrossDeviceControlEdges());
   }
 
   return Status::OK();
diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer.h b/tensorflow/core/grappler/optimizers/dependency_optimizer.h
index 99021b955f..98b69b4b7a 100644
--- a/tensorflow/core/grappler/optimizers/dependency_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/dependency_optimizer.h
@@ -17,6 +17,10 @@ limitations under the License.
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DEPENDENCY_OPTIMIZER_H_
 
 #include <unordered_set>
+
+#include "absl/container/flat_hash_set.h"
+#include "absl/strings/string_view.h"
+#include "tensorflow/core/grappler/mutable_graph_view.h"
 #include "tensorflow/core/grappler/optimizers/graph_optimizer.h"
 #include "tensorflow/core/grappler/utils.h"
 #include "tensorflow/core/protobuf/rewriter_config.pb.h"
@@ -45,24 +49,22 @@ class DependencyOptimizer : public GraphOptimizer {
   // Returns true if bypassing node does not increase the number of edges or
   // number of edges crossing a device boundary.
   bool BypassingNodeIsBeneficial(
-      const NodeDef& node, const std::vector<NodeDef*>& input_nodes,
-      const std::vector<NodeDef*>& output_nodes) const;
-  int NumEdgesIfBypassed(const NodeDef& node,
-                         const std::vector<NodeDef*>& output_nodes) const;
+      const NodeDef& node,
+      const absl::flat_hash_set<MutableGraphView::Edge>& fanout_edges) const;
+  int NumEdgesIfBypassed(
+      const NodeDef& node,
+      const absl::flat_hash_set<MutableGraphView::Edge>& fanout_edges) const;
   // Returns true if node is not an Identity node or if it is an Identity
   // that is safe to remove.
   bool SafeToRemoveIdentity(const NodeDef& node) const;
   // Returns true if it is safe to convert node to NoOp.
   bool SafeToConvertToNoOp(const NodeDef& node) const;
-  // Removes all duplicate control dependencies.
-  void CleanControlInputs();
-  // Builds a map from the &optimized_graph_->node(i) to i.
-  void BuildNodeToIdx();
-  // Tries to optimize the node with the given index, possibly additional
+  // Tries to optimize the node with the given node name, possibly additional
   // optimizations by inserting nodes in nodes_to_simplify, and pruning nodes by
   // inserting them in nodes_to_delete.
-  void OptimizeNode(int node_idx, SetVector<int>* nodes_to_simplify,
-                    std::set<int>* nodes_to_delete);
+  Status OptimizeNode(const string& node_name,
+                      SetVector<string>* nodes_to_simplify,
+                      absl::flat_hash_set<string>* nodes_to_delete);
   // Eliminates redundant control dependencies by computing the transitive
   // reduction of the graph.
   Status TransitiveReduction();
@@ -70,13 +72,11 @@ class DependencyOptimizer : public GraphOptimizer {
   Status OptimizeDependencies();
   // Replaces multiple cross-device control edges from the same device with a
   // single control edge.
-  void GroupCrossDeviceControlEdges();
+  Status GroupCrossDeviceControlEdges();
 
   bool fetch_nodes_known_;
   std::unordered_set<string> nodes_to_preserve_;
-  std::unique_ptr<NodeMap> node_map_;
-  std::unordered_map<const NodeDef*, int> node_to_idx_;
-  GraphDef* optimized_graph_;  // Not owned.
+  std::unique_ptr<MutableGraphView> graph_view_;
 };
 
 }  // end namespace grappler
diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc b/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
index 5883fcb926..59512d8b5d 100644
--- a/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
@@ -41,7 +41,7 @@ void VerifyGraphsEqual(const GraphDef& original_graph,
     const NodeDef& optimized = optimized_graph.node(i);
     EXPECT_EQ(original.name(), optimized.name()) << func;
     EXPECT_EQ(original.op(), optimized.op()) << func;
-    EXPECT_EQ(original.input_size(), optimized.input_size()) << func;
+    ASSERT_EQ(original.input_size(), optimized.input_size()) << func;
     for (int j = 0; j < original.input_size(); ++j) {
       EXPECT_EQ(original.input(j), optimized.input(j)) << func;
     }
@@ -89,12 +89,12 @@ TEST_F(DependencyOptimizerTest, DependenciesDrivenByConstants) {
   TF_EXPECT_OK(status);
 
   // The 'z' node should have been optimized away leaving only 5 nodes.
-  EXPECT_EQ(5, output.node_size());
+  EXPECT_EQ(output.node_size(), 5);
 
-  for (const NodeDef& node : item.graph.node()) {
+  for (const NodeDef& node : output.node()) {
     if (node.name() == "id1" || node.name() == "id2") {
-      EXPECT_EQ(1, node.input_size());
-      EXPECT_EQ("add", node.input(0));
+      ASSERT_EQ(node.input_size(), 1);
+      EXPECT_EQ(node.input(0), "add");
     }
   }
 }
@@ -123,30 +123,30 @@ TEST_F(DependencyOptimizerTest, ChangeToNoop) {
   status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
-  EXPECT_EQ(item.graph.node_size(), output.node_size());
+  EXPECT_EQ(output.node_size(), item.graph.node_size());
   int found = 0;
-  for (int i = 0; i < item.graph.node_size(); ++i) {
-    const NodeDef& node = item.graph.node(i);
+  for (int i = 0; i < output.node_size(); ++i) {
+    const NodeDef& node = output.node(i);
     // "add" should get turned into a NoOp and removed.
-    EXPECT_NE("add", node.name());
+    EXPECT_NE(node.name(), "add");
     if (node.name() == "id1") {
-      EXPECT_EQ("Identity", node.op());
-      EXPECT_EQ(2, node.input_size());
-      EXPECT_EQ("x", node.input(0));
-      EXPECT_EQ("^y", node.input(1));
+      EXPECT_EQ(node.op(), "Identity");
+      ASSERT_EQ(node.input_size(), 2);
+      EXPECT_EQ(node.input(0), "x");
+      EXPECT_EQ(node.input(1), "^y");
       ++found;
     } else if (node.name() == "id2") {
-      EXPECT_EQ("Identity", node.op());
-      EXPECT_EQ(2, node.input_size());
-      EXPECT_EQ("y", node.input(0));
-      EXPECT_EQ("^x", node.input(1));
+      EXPECT_EQ(node.op(), "Identity");
+      ASSERT_EQ(node.input_size(), 2);
+      EXPECT_EQ(node.input(0), "y");
+      EXPECT_EQ(node.input(1), "^x");
       ++found;
     }
   }
-  EXPECT_EQ(2, found);
+  EXPECT_EQ(found, 2);
 }
 
-TEST_F(DependencyOptimizerTest, ChangeToNoop_RepeatedInput) {
+TEST_F(DependencyOptimizerTest, ChangeToNoopRepeatedInput) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output x = ops::RandomUniform(s.WithOpName("x"), {1, 2}, DT_FLOAT);
   Output add = ops::Add(s.WithOpName("add"), x, x);
@@ -164,25 +164,24 @@ TEST_F(DependencyOptimizerTest, ChangeToNoop_RepeatedInput) {
   item.graph.Swap(&output);
   status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
-  LOG(INFO) << output.DebugString();
 
-  EXPECT_EQ(item.graph.node_size(), output.node_size());
+  EXPECT_EQ(output.node_size(), item.graph.node_size());
   int found = 0;
   for (int i = 0; i < item.graph.node_size(); ++i) {
     const NodeDef& node = item.graph.node(i);
     // "add" should get turned into a NoOp and removed.
-    EXPECT_NE("add", node.name());
+    EXPECT_NE(node.name(), "add");
     if (node.name() == "id1") {
-      EXPECT_EQ("Identity", node.op());
-      EXPECT_EQ(1, node.input_size());
-      EXPECT_EQ("x", node.input(0));
+      EXPECT_EQ(node.op(), "Identity");
+      EXPECT_EQ(node.input_size(), 1);
+      EXPECT_EQ(node.input(0), "x");
       ++found;
     }
   }
-  EXPECT_EQ(1, found);
+  EXPECT_EQ(found, 1);
 }
 
-TEST_F(DependencyOptimizerTest, ChangeToNoop_SwitchIdentity) {
+TEST_F(DependencyOptimizerTest, ChangeToNoopSwitchIdentity) {
   // This tests that we don't try to repeatedly add Identity nodes
   // with names like "ConstantFoldingCtrl/foo/bar/switch_$port" when
   // multiple nodes reading the same output of a Switch node get
@@ -220,23 +219,23 @@ TEST_F(DependencyOptimizerTest, ChangeToNoop_SwitchIdentity) {
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
-  EXPECT_EQ(item.graph.node_size() - 1, output.node_size());
+  EXPECT_EQ(output.node_size(), item.graph.node_size() - 1);
   for (int i = 0; i < output.node_size(); ++i) {
     const NodeDef& node = output.node(i);
     // "neg" should be eliminated.
-    EXPECT_NE("neg", node.name());
+    EXPECT_NE(node.name(), "neg");
     // A control dep from "^ConstantFoldingCtrl/switch_1"
     // should be attached to "c1".
     if (node.name() == "c1") {
-      EXPECT_EQ("Const", node.op());
-      EXPECT_EQ(1, node.input_size());
-      EXPECT_EQ("^ConstantFoldingCtrl/switch_1", node.input(0));
+      EXPECT_EQ(node.op(), "Const");
+      ASSERT_EQ(node.input_size(), 1);
+      EXPECT_EQ(node.input(0), "^ConstantFoldingCtrl/switch_1");
     }
   }
 }
 
 // TODO(rmlarsen): Add test to make sure we skip Switch and Merge.
-TEST_F(DependencyOptimizerTest, ChangeToNoop_NoFetch) {
+TEST_F(DependencyOptimizerTest, ChangeToNoopNoFetch) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output x = ops::RandomUniform(s.WithOpName("x"), {1, 2}, DT_FLOAT);
   Output y = ops::RandomUniform(s.WithOpName("y"), {1, 2}, DT_FLOAT);
@@ -258,7 +257,7 @@ TEST_F(DependencyOptimizerTest, ChangeToNoop_NoFetch) {
   VerifyGraphsEqual(item.graph, output, __FUNCTION__);
 }
 
-TEST_F(DependencyOptimizerTest, RemoveNoOps_EmptyInputOrOutput) {
+TEST_F(DependencyOptimizerTest, RemoveNoOpsEmptyInputOrOutput) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output x = ops::RandomUniform(s, {1, 2}, DT_FLOAT);
   auto noop1 = ops::NoOp(s);
@@ -278,18 +277,18 @@ TEST_F(DependencyOptimizerTest, RemoveNoOps_EmptyInputOrOutput) {
   status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
-  EXPECT_EQ(item.graph.node_size(), output.node_size());
+  EXPECT_EQ(output.node_size(), item.graph.node_size());
   for (const NodeDef& node : output.node()) {
     if (node.name() == "NoOp" || node.name() == "NoOp_1") {
       EXPECT_EQ(0, node.input_size());
     } else if (node.name() == "Identity") {
-      EXPECT_EQ(1, node.input_size());
-      EXPECT_EQ("RandomUniform", node.input(0));
+      ASSERT_EQ(node.input_size(), 1);
+      EXPECT_EQ(node.input(0), "RandomUniform");
     }
   }
 }
 
-TEST_F(DependencyOptimizerTest, RemoveNoOps_DeviceBoundaries) {
+TEST_F(DependencyOptimizerTest, RemoveNoOpsDeviceBoundaries) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output x = ops::RandomUniform(s.WithOpName("x").WithDevice("/CPU:0"), {1, 2},
                                 DT_FLOAT);
@@ -324,7 +323,7 @@ TEST_F(DependencyOptimizerTest, RemoveNoOps_DeviceBoundaries) {
   VerifyGraphsEqual(item.graph, output, __FUNCTION__);
 }
 
-TEST_F(DependencyOptimizerTest, RemoveIdentityOps_DeviceBoundaries) {
+TEST_F(DependencyOptimizerTest, RemoveIdentityOpsDeviceBoundaries) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output x = ops::RandomUniform(s.WithOpName("x").WithDevice("/CPU:0"), {1, 2},
                                 DT_FLOAT);
@@ -356,7 +355,7 @@ TEST_F(DependencyOptimizerTest, RemoveIdentityOps_DeviceBoundaries) {
   VerifyGraphsEqual(item.graph, output, __FUNCTION__);
 }
 
-TEST_F(DependencyOptimizerTest, RemoveIdentityOps_IdenticalDevices) {
+TEST_F(DependencyOptimizerTest, RemoveIdentityOpsIdenticalDevices) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output x = ops::RandomUniform(s.WithOpName("x").WithDevice("/CPU:0"), {1, 2},
                                 DT_FLOAT);
@@ -373,16 +372,17 @@ TEST_F(DependencyOptimizerTest, RemoveIdentityOps_IdenticalDevices) {
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
-  EXPECT_EQ(item.graph.node_size() - 1, output.node_size());
+  EXPECT_EQ(output.node_size(), item.graph.node_size() - 1);
   for (const NodeDef& node : output.node()) {
     EXPECT_NE(node.name(), "id_a");
     if (node.name() == "Identity") {
+      ASSERT_EQ(node.input_size(), 1);
       EXPECT_EQ(node.input(0), "x");
     }
   }
 }
 
-TEST_F(DependencyOptimizerTest, RemoveNoOps_SingleInputOrOutput) {
+TEST_F(DependencyOptimizerTest, RemoveNoOpsSingleInputOrOutput) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output x = ops::RandomUniform(s.WithOpName("x"), {1, 2}, DT_FLOAT);
   Output y = ops::RandomUniform(s.WithOpName("y"), {1, 2}, DT_FLOAT);
@@ -409,15 +409,17 @@ TEST_F(DependencyOptimizerTest, RemoveNoOps_SingleInputOrOutput) {
   status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
-  EXPECT_EQ(item.graph.node_size(), output.node_size());
+  EXPECT_EQ(output.node_size(), item.graph.node_size());
   for (const NodeDef& node : output.node()) {
     if (node.name() == "NoOp" || node.name() == "NoOp_1") {
-      EXPECT_EQ(0, node.input_size());
+      EXPECT_EQ(node.input_size(), 0);
     } else if (node.name() == "Identity") {
-      EXPECT_EQ("x", node.input(0));
+      ASSERT_EQ(node.input_size(), 1);
+      EXPECT_EQ(node.input(0), "x");
     } else if (node.name() == "Identity_1") {
-      EXPECT_EQ("y", node.input(0));
-      EXPECT_EQ("^x", node.input(1));
+      ASSERT_EQ(node.input_size(), 2);
+      EXPECT_EQ(node.input(0), "y");
+      EXPECT_EQ(node.input(1), "^x");
     }
   }
 }
@@ -462,48 +464,48 @@ TEST_F(DependencyOptimizerTest, RemoveIdentity) {
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
-  EXPECT_EQ(item.graph.node_size() - 3, output.node_size());
+  EXPECT_EQ(output.node_size(), item.graph.node_size() - 3);
   int found = 0;
   for (const NodeDef& node : output.node()) {
-    EXPECT_NE("id_a", node.name());
-    EXPECT_NE("id_b", node.name());
-    EXPECT_NE("id_c", node.name());
+    EXPECT_NE(node.name(), "id_a");
+    EXPECT_NE(node.name(), "id_b");
+    EXPECT_NE(node.name(), "id_c");
     if (node.name() == "a_a" || node.name() == "a_b") {
-      EXPECT_EQ(1, node.input_size());
-      EXPECT_EQ("x", node.input(0));
+      ASSERT_EQ(node.input_size(), 1);
+      EXPECT_EQ(node.input(0), "x");
       ++found;
     }
     if (node.name() == "a_c" || node.name() == "a_d") {
-      EXPECT_EQ(2, node.input_size());
-      EXPECT_EQ("z", node.input(0));
-      EXPECT_EQ("^x", node.input(1));
+      ASSERT_EQ(node.input_size(), 2);
+      EXPECT_EQ(node.input(0), "z");
+      EXPECT_EQ(node.input(1), "^x");
       ++found;
     }
     if (node.name() == "b_a") {
-      EXPECT_EQ(3, node.input_size());
-      EXPECT_EQ("x", node.input(0));
-      EXPECT_EQ("^y", node.input(1));
-      EXPECT_EQ("^z", node.input(2));
+      ASSERT_EQ(node.input_size(), 3);
+      EXPECT_EQ(node.input(0), "x");
+      EXPECT_EQ(node.input(1), "^y");
+      EXPECT_EQ(node.input(2), "^z");
       ++found;
     }
     if (node.name() == "c_a") {
-      EXPECT_EQ(2, node.input_size());
-      EXPECT_EQ("x", node.input(0));
-      EXPECT_EQ("^y", node.input(1));
+      ASSERT_EQ(node.input_size(), 2);
+      EXPECT_EQ(node.input(0), "x");
+      EXPECT_EQ(node.input(1), "^y");
       ++found;
     }
     if (node.name() == "c_b") {
-      EXPECT_EQ(3, node.input_size());
-      EXPECT_EQ("z", node.input(0));
-      EXPECT_EQ("^x", node.input(1));
-      EXPECT_EQ("^y", node.input(2));
+      ASSERT_EQ(node.input_size(), 3);
+      EXPECT_EQ(node.input(0), "z");
+      EXPECT_EQ(node.input(1), "^x");
+      EXPECT_EQ(node.input(2), "^y");
       ++found;
     }
   }
   EXPECT_EQ(found, 7);
 }
 
-TEST_F(DependencyOptimizerTest, RemoveIdentity_RepeatedInputs) {
+TEST_F(DependencyOptimizerTest, RemoveIdentityRepeatedInputs) {
   // Corner cases with repeated inputs.
   tensorflow::Scope scope = tensorflow::Scope::NewRootScope();
   ops::Variable x(scope.WithOpName("x"), {}, DT_BOOL);
@@ -529,35 +531,35 @@ TEST_F(DependencyOptimizerTest, RemoveIdentity_RepeatedInputs) {
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
-  EXPECT_EQ(item.graph.node_size() - 1, output.node_size());
+  EXPECT_EQ(output.node_size(), item.graph.node_size() - 1);
   int found = 0;
   for (const NodeDef& node : output.node()) {
-    EXPECT_NE("id0", node.name());
+    EXPECT_NE(node.name(), "id0");
     if (node.name() == "or0") {
-      EXPECT_EQ(2, node.input_size());
-      EXPECT_EQ("switch:1", node.input(0));
-      EXPECT_EQ("switch:1", node.input(1));
+      ASSERT_EQ(node.input_size(), 2);
+      EXPECT_EQ(node.input(0), "switch:1");
+      EXPECT_EQ(node.input(1), "switch:1");
       ++found;
     }
     if (node.name() == "or1") {
-      EXPECT_EQ(2, node.input_size());
-      EXPECT_EQ("switch:1", node.input(0));
-      EXPECT_EQ("y", node.input(1));
+      ASSERT_EQ(node.input_size(), 2);
+      EXPECT_EQ(node.input(0), "switch:1");
+      EXPECT_EQ(node.input(1), "y");
       ++found;
     }
     if (node.name() == "or2") {
       // or1 should be unchanged.
-      EXPECT_EQ(3, node.input_size());
-      EXPECT_EQ("y", node.input(0));
-      EXPECT_EQ("y", node.input(1));
-      EXPECT_EQ("^id1", node.input(2));
+      ASSERT_EQ(node.input_size(), 3);
+      EXPECT_EQ(node.input(0), "y");
+      EXPECT_EQ(node.input(1), "y");
+      EXPECT_EQ(node.input(2), "^id1");
       ++found;
     }
   }
   EXPECT_EQ(found, 3);
 }
 
-TEST_F(DependencyOptimizerTest, Transitive_Reduction_Simple) {
+TEST_F(DependencyOptimizerTest, TransitiveReductionSimple) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output c = ops::Const(s.WithOpName("c"), {1.0f, 2.0f}, {1, 2});
   Output x = ops::Square(s.WithOpName("x"), c);
@@ -572,13 +574,13 @@ TEST_F(DependencyOptimizerTest, Transitive_Reduction_Simple) {
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
-  EXPECT_EQ(4, output.node_size());
-  EXPECT_EQ("neg2", output.node(3).name());
-  EXPECT_EQ(1, output.node(3).input_size());
-  EXPECT_EQ("neg1", output.node(3).input(0));
+  ASSERT_EQ(output.node_size(), 4);
+  EXPECT_EQ(output.node(3).name(), "neg2");
+  ASSERT_EQ(output.node(3).input_size(), 1);
+  EXPECT_EQ(output.node(3).input(0), "neg1");
 }
 
-TEST_F(DependencyOptimizerTest, ChangeToNoop_Identity) {
+TEST_F(DependencyOptimizerTest, ChangeToNoopIdentity) {
   tensorflow::Scope scope = tensorflow::Scope::NewRootScope();
   ops::Variable v_in(scope.WithOpName("v_in"), {3}, DT_FLOAT);
   Output id_after_var = ops::Identity(scope.WithOpName("id_after_var"), v_in);
@@ -609,18 +611,18 @@ TEST_F(DependencyOptimizerTest, ChangeToNoop_Identity) {
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
-  EXPECT_EQ(item.graph.node_size() - 2, output.node_size());
+  EXPECT_EQ(output.node_size(), item.graph.node_size() - 2);
   bool found = false;
   for (int i = 0; i < output.node_size(); ++i) {
     const NodeDef& node = output.node(i);
     // "id0" and "id1" but neither "ConstantFoldingCtrl/switch_1",
     // "id_after_var, nor "id2"" should be eliminated.
-    EXPECT_NE("id0", node.name());
-    EXPECT_NE("id1", node.name());
+    EXPECT_NE(node.name(), "id0");
+    EXPECT_NE(node.name(), "id1");
     if (node.name() == "c1") {
-      EXPECT_EQ("Const", node.op());
-      EXPECT_EQ(1, node.input_size());
-      EXPECT_EQ("^ConstantFoldingCtrl/switch_1", node.input(0));
+      EXPECT_EQ(node.op(), "Const");
+      ASSERT_EQ(node.input_size(), 1);
+      EXPECT_EQ(node.input(0), "^ConstantFoldingCtrl/switch_1");
       found = true;
     }
   }
@@ -650,17 +652,17 @@ TEST_F(DependencyOptimizerTest, IdentityInputs) {
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
-  EXPECT_EQ(6, output.node_size());
-  EXPECT_EQ("out1", output.node(4).name());
-  EXPECT_EQ(1, output.node(4).input_size());
-  EXPECT_EQ("s", output.node(4).input(0));
+  ASSERT_EQ(output.node_size(), 6);
+  EXPECT_EQ(output.node(4).name(), "out2");
+  ASSERT_EQ(output.node(4).input_size(), 1);
+  EXPECT_EQ(output.node(4).input(0), "s:1");
 
-  EXPECT_EQ("out2", output.node(5).name());
-  EXPECT_EQ(1, output.node(5).input_size());
-  EXPECT_EQ("s:1", output.node(5).input(0));
+  EXPECT_EQ(output.node(5).name(), "out1");
+  ASSERT_EQ(output.node(5).input_size(), 1);
+  EXPECT_EQ(output.node(5).input(0), "s");
 }
 
-TEST_F(DependencyOptimizerTest, RemoveIdentityN_SwitchInput) {
+TEST_F(DependencyOptimizerTest, RemoveIdentityNSwitchInput) {
   tensorflow::Scope scope = tensorflow::Scope::NewRootScope();
   Output b = ops::Placeholder(scope.WithOpName("b"), DT_BOOL);
   Output x = ops::RandomUniform(scope.WithOpName("x"), {1, 2}, DT_FLOAT);
@@ -687,27 +689,27 @@ TEST_F(DependencyOptimizerTest, RemoveIdentityN_SwitchInput) {
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
-  EXPECT_EQ(8, output.node_size());
+  ASSERT_EQ(output.node_size(), 8);
 
   auto out1_node = output.node(7);
-  EXPECT_EQ("out1", out1_node.name());
-  EXPECT_EQ(1, out1_node.input_size());
-  EXPECT_EQ("s", out1_node.input(0));
+  EXPECT_EQ(out1_node.name(), "out1");
+  ASSERT_EQ(out1_node.input_size(), 1);
+  EXPECT_EQ(out1_node.input(0), "s");
 
-  auto out2_node = output.node(4);
-  EXPECT_EQ("out2", out2_node.name());
-  EXPECT_EQ(1, out2_node.input_size());
-  EXPECT_EQ("s:1", out2_node.input(0));
+  auto out2_node = output.node(6);
+  EXPECT_EQ(out2_node.name(), "out2");
+  ASSERT_EQ(out2_node.input_size(), 1);
+  EXPECT_EQ(out2_node.input(0), "s:1");
 
   auto out3_node = output.node(5);
-  EXPECT_EQ("out3", out3_node.name());
-  EXPECT_EQ(1, out3_node.input_size());
-  EXPECT_EQ("s", out3_node.input(0));
-
-  auto out4_node = output.node(6);
-  EXPECT_EQ("out4", out4_node.name());
-  EXPECT_EQ(1, out4_node.input_size());
-  EXPECT_EQ("s:1", out4_node.input(0));
+  EXPECT_EQ(out3_node.name(), "out3");
+  ASSERT_EQ(out3_node.input_size(), 1);
+  EXPECT_EQ(out3_node.input(0), "s");
+
+  auto out4_node = output.node(4);
+  EXPECT_EQ(out4_node.name(), "out4");
+  ASSERT_EQ(out4_node.input_size(), 1);
+  EXPECT_EQ(out4_node.input(0), "s:1");
 }
 
 TEST_F(DependencyOptimizerTest, DoNotRemoveIdentityNWithControlDependency) {
@@ -730,11 +732,11 @@ TEST_F(DependencyOptimizerTest, DoNotRemoveIdentityNWithControlDependency) {
   Status status = optimizer.Optimize(nullptr, item, &optimized_graph_def);
   TF_EXPECT_OK(status);
 
-  EXPECT_EQ(6, optimized_graph_def.node_size());
+  EXPECT_EQ(optimized_graph_def.node_size(), 6);
 }
 
 TEST_F(DependencyOptimizerTest,
-       Identity_DeviceCrossing_ConsumerOnDifferentDevice) {
+       IdentityDeviceCrossingConsumerOnDifferentDevice) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output x_on_1 =
       ops::Const(s.WithOpName("x_on_1").WithDevice("/gpu:1"), {1.0f}, {});
@@ -756,7 +758,7 @@ TEST_F(DependencyOptimizerTest,
   VerifyGraphsEqual(item.graph, output, __FUNCTION__);
 }
 
-TEST_F(DependencyOptimizerTest, Identity_DeviceCrossing_ConsumerOnSameDevice) {
+TEST_F(DependencyOptimizerTest, IdentityDeviceCrossingConsumerOnSameDevice) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output x_on_1 =
       ops::Const(s.WithOpName("x_on_1").WithDevice("/gpu:1"), {1.0f}, {});
@@ -774,12 +776,15 @@ TEST_F(DependencyOptimizerTest, Identity_DeviceCrossing_ConsumerOnSameDevice) {
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
-  LOG(INFO) << output.DebugString();
-  EXPECT_EQ(3, output.node_size());
+
+  EXPECT_EQ(output.node_size(), 3);
   for (const auto& node : output.node()) {
-    EXPECT_NE("x_on_2", node.name());
+    EXPECT_NE(node.name(), "x_on_2");
     if (node.name() == "result") {
-      EXPECT_EQ("x_on_1", node.input(0));
+      LOG(INFO) << node.DebugString();
+      ASSERT_EQ(node.input_size(), 2);
+      EXPECT_EQ(node.input(0), "x_on_1");
+      EXPECT_EQ(node.input(1), "one_on_2");
     }
   }
 }
@@ -807,25 +812,25 @@ TEST_F(DependencyOptimizerTest, RemoveGreaterEqualWithNoOp) {
   for (const NodeDef& node : output.node()) {
     if (node.name() == "x") {
       count++;
-      EXPECT_EQ("Placeholder", node.op());
-      EXPECT_EQ(0, node.input_size());
+      EXPECT_EQ(node.op(), "Placeholder");
+      EXPECT_EQ(node.input_size(), 0);
     } else if (node.name() == "y") {
       count++;
-      EXPECT_EQ("Placeholder", node.op());
-      EXPECT_EQ(0, node.input_size());
+      EXPECT_EQ(node.op(), "Placeholder");
+      EXPECT_EQ(node.input_size(), 0);
     } else if (node.name() == "GreaterEqual") {
       count++;
     } else if (node.name() == "NoOp") {
       count++;
     } else if (node.name() == "z") {
       count++;
-      EXPECT_EQ("Add", node.op());
-      EXPECT_EQ(2, node.input_size());
-      EXPECT_EQ("x", node.input(0));
-      EXPECT_EQ("y", node.input(1));
+      EXPECT_EQ(node.op(), "Add");
+      ASSERT_EQ(node.input_size(), 2);
+      EXPECT_EQ(node.input(0), "x");
+      EXPECT_EQ(node.input(1), "y");
     }
   }
-  EXPECT_EQ(3, count);
+  EXPECT_EQ(count, 3);
 }
 
 TEST_F(DependencyOptimizerTest, GroupCrossDeviceControlDeps) {
-- 
GitLab


From b09f3fbed2a74785e2a5eda624dfaf8de932102b Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Tue, 29 Jan 2019 18:05:23 -0800
Subject: [PATCH 1751/2345] Add CheckInputsWeights method to validate number of
 inputs and whether they must be weights.

---
 .../tf2tensorrt/convert/convert_nodes.cc      | 267 +++++++-----------
 .../tf2tensorrt/convert/convert_nodes.h       |   3 +
 2 files changed, 100 insertions(+), 170 deletions(-)

diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
index d1d8ab9d3c..868fdd4480 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
@@ -1297,6 +1297,39 @@ Status Converter::GetInputs(const tensorflow::NodeDef& node_def,
   return tensorflow::Status::OK();
 }
 
+// Checks that the number of inputs match, and enforces that the inputs marked
+// as true are constant weights. true means that the input must be a weight,
+// while false means the input must be a tensor. In the future, false will mean
+// the input can be a tensor or weight.
+tensorflow::Status CheckInputsWeights(
+    OpConverterParams* params,
+    const std::vector<std::pair<string, bool>> inputs_is_weight) {
+  const auto& inputs = params->inputs;
+  const auto& node_def = params->node_def;
+  if (inputs.size() != inputs_is_weight.size()) {
+    return tensorflow::errors::InvalidArgument(
+        node_def.op(), " got ", inputs.size(), " inputs but expected ",
+        inputs_is_weight.size(), ", at ", node_def.name());
+  }
+  for (int i = 0; i < inputs.size(); i++) {
+    if (inputs_is_weight[i].second && inputs.at(i).is_tensor()) {
+      return tensorflow::errors::Unimplemented(
+          "The input \",", inputs_is_weight[i].first, "\" for ", node_def.op(),
+          " must be a constant, at ", node_def.name());
+    }
+    // TODO(tmorris): Remove this check and provide a method to automatically
+    // retrive an input as a tensor, converting via CreateConstantLayer if it
+    // was originally a weight. We will want a caching mechanism to prevent many
+    // duplicate constants from being created.
+    if (!inputs_is_weight[i].second && inputs.at(i).is_weights()) {
+      return tensorflow::errors::Unimplemented(
+          "The input \",", inputs_is_weight[i].first, "\" for ", node_def.op(),
+          " must be a tensor, at ", node_def.name());
+    }
+  }
+  return tensorflow::Status::OK();
+}
+
 TRT_ShapedWeights ConvertFP32ToFP16(TrtWeightStore* store,
                                     const TRT_ShapedWeights& weights_src) {
   auto dtype_new = tensorflow::DataType::DT_HALF;
@@ -1588,48 +1621,17 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group,
   const auto& node_def = params->node_def;
   TRT_TensorOrWeights backprop_output_size;
   const nvinfer1::ITensor* tensor = nullptr;
-  // TODO(tmorris): Add helper functions for validating inputs to avoid code
-  // duplication.
   if (is_conv2d_backprop_input) {
-    // Conv2DBackpropInput has 3 inputs: input_sizes, filter, and out_backprop.
-    // In the case of conv2d_transpose, these correspond to: output size,
-    // filter, and input.
-    if (inputs.size() != 3) {
-      return tensorflow::errors::InvalidArgument(
-          "Three inputs are expected for ", node_def.op(), ", at ",
-          node_def.name());
-    }
-    if (inputs.at(0).is_tensor()) {
-      return tensorflow::errors::Unimplemented(
-          "input_sizes for ", node_def.op(), " must be constant weights, at ",
-          node_def.name());
-    }
-    if (inputs.at(2).is_weights()) {
-      return tensorflow::errors::Unimplemented(
-          node_def.op(), " is only implemented for tensors, not weights, at ",
-          node_def.name());
-    }
+    TF_RETURN_IF_ERROR(CheckInputsWeights(
+        params,
+        {{"input_sizes", true}, {"filter", true},  {"out_backprop", false}}));
     backprop_output_size = inputs.at(0);
     tensor = inputs.at(2).tensor();
   } else {
-    // Regular convolution has 2 inputs: input, filter.
-    if (inputs.size() != 2) {
-      return tensorflow::errors::InvalidArgument("Two inputs are expected for ",
-                                                 node_def.op(), ", at ",
-                                                 node_def.name());
-    }
-    if (inputs.at(0).is_weights()) {
-      return tensorflow::errors::Unimplemented(
-          node_def.op(), " is only implemented for tensors, not weights, at ",
-          node_def.name());
-    }
+    TF_RETURN_IF_ERROR(
+        CheckInputsWeights(params, {{"input", false}, {"filter", true}}));
     tensor = inputs.at(0).tensor();
   }
-  if (inputs.at(1).is_tensor()) {
-    return tensorflow::errors::Unimplemented("Kernel for ", node_def.op(),
-                                             " must be constant weights, at ",
-                                             node_def.name());
-  }
   TRT_ShapedWeights weights_rsck = inputs.at(1).weights();
   if (weights_rsck.shape_.nbDims != 4) {
     return tensorflow::errors::InvalidArgument(
@@ -1890,12 +1892,8 @@ tensorflow::Status ConvertPlugin(OpConverterParams* params) {
 
 tensorflow::Status ConvertTranspose(OpConverterParams* params) {
   const auto& inputs = params->inputs;
-  if (inputs.size() != 2 || !inputs.at(0).is_tensor() ||
-      !inputs.at(1).is_weights()) {
-    return tensorflow::errors::InvalidArgument(
-        "Input expects tensor and weights, at ", params->node_def.name());
-  }
-
+  TF_RETURN_IF_ERROR(
+      CheckInputsWeights(params, {{"input", false}, {"perm", true}}));
   // Get the permutation from weights.
   TRT_ShapedWeights weights = inputs.at(1).weights();
   const int* weights_ptr =
@@ -1928,11 +1926,8 @@ tensorflow::Status ConvertTranspose(OpConverterParams* params) {
 tensorflow::Status ConvertReshape(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
-  if (inputs.size() != 2 || !inputs.at(1).is_weights()) {
-    return tensorflow::errors::InvalidArgument(
-        "Input expects weights for shape, at ", node_def.name());
-  }
-
+  TF_RETURN_IF_ERROR(
+      CheckInputsWeights(params, {{"input", false}, {"shape", true}}));
   TRT_TensorOrWeights input_tensor = inputs.at(0);
   TRT_ShapedWeights weights = inputs.at(1).weights();
   if (weights.count() == 0) {
@@ -2028,18 +2023,8 @@ tensorflow::Status ConvertReshape(OpConverterParams* params) {
 tensorflow::Status ConvertExpandDims(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
-  if (inputs.size() != 2) {
-    return tensorflow::errors::InvalidArgument(
-        "Two inputs expected for ExpandDims, at ", node_def.name());
-  }
-  if (inputs.at(0).is_weights()) {
-    return tensorflow::errors::Unimplemented(
-        "ExpandDims expects tensor for input, at ", node_def.name());
-  }
-  if (!inputs.at(1).is_weights()) {
-    return tensorflow::errors::InvalidArgument(
-        "ExpandDims expects weights for axis, at ", node_def.name());
-  }
+  TF_RETURN_IF_ERROR(
+      CheckInputsWeights(params, {{"input", false}, {"axis", true}}));
   // Get input shape as vector.
   TRT_TensorOrWeights input_tensor = inputs.at(0);
   const nvinfer1::Dims dims = input_tensor.GetTrtDims();
@@ -2089,14 +2074,7 @@ tensorflow::Status ConvertExpandDims(OpConverterParams* params) {
 tensorflow::Status ConvertSqueeze(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
-  if (inputs.size() != 1) {
-    return tensorflow::errors::InvalidArgument(
-        "One input expected for Squeeze, at ", node_def.name());
-  }
-  if (inputs.at(0).is_weights()) {
-    return tensorflow::errors::Unimplemented(
-        "Squeeze expects tensor for input, at ", node_def.name());
-  }
+  TF_RETURN_IF_ERROR(CheckInputsWeights(params, {{"input", false}}));
   // Get input shape.
   TRT_TensorOrWeights input_tensor = inputs.at(0);
   const nvinfer1::Dims dims = input_tensor.GetTrtDims();
@@ -2198,20 +2176,9 @@ tensorflow::Status GetStridedSliceBound(const std::vector<int>& input_dims,
 tensorflow::Status ConvertStridedSlice(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
-  if (inputs.size() != 4) {
-    return tensorflow::errors::InvalidArgument(
-        "StridedSlice expects 4 inputs, at ", node_def.name());
-  }
-  if (!inputs.at(1).is_weights() || !inputs.at(2).is_weights() ||
-      !inputs.at(3).is_weights()) {
-    return tensorflow::errors::InvalidArgument(
-        "StridedSlice expects weights for begin, end, and strides, at ",
-        node_def.name());
-  }
-  if (!inputs.at(0).is_tensor()) {
-    return tensorflow::errors::Unimplemented(
-        "StridedSlice is only implemented for tensors, at ", node_def.name());
-  }
+  TF_RETURN_IF_ERROR(CheckInputsWeights(
+      params,
+      {{"input", false}, {"begin", true}, {"end", true}, {"strides", true}}));
   // Get input dims.
   nvinfer1::Dims dims = inputs.at(0).GetTrtDims();
   std::vector<int> input_dims(dims.d, dims.d + dims.nbDims);
@@ -2406,11 +2373,7 @@ tensorflow::Status ConvertConv2DBackpropInput(OpConverterParams* params) {
 tensorflow::Status ConvertPool(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
-  if (inputs.at(0).is_weights()) {
-    return tensorflow::errors::Unimplemented(
-        node_def.op(), " is only implemented for tensors, not weights, at ",
-        node_def.name());
-  }
+  TF_RETURN_IF_ERROR(CheckInputsWeights(params, {{"input", false}}));
   nvinfer1::PoolingType type;
   if (node_def.op() == "MaxPool") {
     type = nvinfer1::PoolingType::kMAX;
@@ -2500,15 +2463,7 @@ tensorflow::Status ConvertPool(OpConverterParams* params) {
 tensorflow::Status ConvertActivation(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
-  if (inputs.size() != 1) {
-    return tensorflow::errors::InvalidArgument(
-        node_def.op(), " expects one input, at ", node_def.name());
-  }
-  if (!inputs.at(0).is_tensor()) {
-    return tensorflow::errors::Unimplemented(
-        node_def.op(), " is only implemented for tensors, at ",
-        node_def.name());
-  }
+  TF_RETURN_IF_ERROR(CheckInputsWeights(params, {{"input", false}}));
   static const std::unordered_map<string, nvinfer1::ActivationType> ops{
       {"Relu", nvinfer1::ActivationType::kRELU},
       {"Sigmoid", nvinfer1::ActivationType::kSIGMOID},
@@ -2542,20 +2497,23 @@ tensorflow::Status ConvertActivation(OpConverterParams* params) {
 Status ConvertQuantize(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
-  if ((inputs.size() == 0) ||
-      (node_def.op() == "FakeQuantWithMinMaxArgs" && inputs.size() != 1) ||
-      (node_def.op() == "FakeQuantWithMinMaxVars" && inputs.size() != 3) ||
-      (node_def.op() == "QuantizeAndDequantizeV2" && inputs.size() != 3) ||
-      (node_def.op() == "QuantizeAndDequantizeV3" && inputs.size() != 4)) {
-    return errors::InvalidArgument("Invalid number of inputs for ",
-                                   node_def.op(), ", at ", node_def.name());
-  }
   if (inputs.at(0).is_weights()) {
-    // TensorRT will automatically quantize weights, so we will ignore ranges
-    // for weights.
+    // This shouldn't happen unless constant folding failed. TRT will
+    // automatically quantize weights so it should be okay.
     params->outputs->push_back(inputs.at(0));
     return Status::OK();
   }
+  if (node_def.op() == "FakeQuantWithMinMaxArgs") {
+    TF_RETURN_IF_ERROR(CheckInputsWeights(params, {{"input", false}}));
+  } else if (node_def.op() == "FakeQuantWithMinMaxVars" ||
+             node_def.op() == "QuantizeAndDequantizeV2") {
+    TF_RETURN_IF_ERROR(CheckInputsWeights(
+        params, {{"input", false}, {"min", true}, {"max", true}}));
+  } else if (node_def.op() == "QuantizeAndDequantizeV3") {
+    TF_RETURN_IF_ERROR(CheckInputsWeights(
+        params,
+        {{"input", false}, {"min", true}, {"max", true}, {"num_bits", true}}));
+  }
   float min_range = 0.0f;
   float max_range = 0.0f;
   if (node_def.op() == "FakeQuantWithMinMaxArgs") {
@@ -2571,11 +2529,6 @@ Status ConvertQuantize(OpConverterParams* params) {
              node_def.op() == "QuantizeAndDequantizeV2" ||
              node_def.op() == "QuantizeAndDequantizeV3") {
     // Get ranges via inputs.
-    if (!inputs.at(1).is_weights() || !inputs.at(2).is_weights()) {
-      return errors::InvalidArgument("Min and max inputs for ", node_def.op(),
-                                     " must be weights not tensors, at ",
-                                     node_def.name());
-    }
     auto get_weights_value = [&inputs](int index) {
       auto raw_weights = static_cast<float*>(
           const_cast<void*>(inputs.at(index).weights().GetValues()));
@@ -2611,15 +2564,7 @@ Status ConvertQuantize(OpConverterParams* params) {
 tensorflow::Status ConvertRelu6(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
-  if (inputs.size() != 1) {
-    return tensorflow::errors::InvalidArgument(
-        "Invalid number of inputs for Relu6, at ", node_def.name());
-  }
-  if (inputs.at(0).is_weights()) {
-    return tensorflow::errors::Unimplemented(
-        "Relu6 is only implemented for tensors, not weights, at ",
-        node_def.name());
-  }
+  TF_RETURN_IF_ERROR(CheckInputsWeights(params, {{"input", false}}));
   if (params->validation_only) return Status::OK();
   // ***************************************************************************
   // TensorRT does not implement Relu6 natively. This function converts Relu6 op
@@ -2680,11 +2625,8 @@ tensorflow::Status ConvertRelu6(OpConverterParams* params) {
 tensorflow::Status ConvertBiasAdd(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
-  if (inputs.size() != 2 || !inputs.at(0).is_tensor() ||
-      !inputs.at(1).is_weights()) {
-    return errors::InvalidArgument("Input expects tensor and weights, at ",
-                                   node_def.name());
-  }
+  TF_RETURN_IF_ERROR(
+      CheckInputsWeights(params, {{"input", false}, {"bias", true}}));
   TFAttrs attrs(node_def);
   tensorflow::DataType tf_dtype = attrs.get<tensorflow::DataType>("T");
   if (tf_dtype != DataType::DT_FLOAT && tf_dtype != DataType::DT_HALF) {
@@ -2904,9 +2846,13 @@ tensorflow::Status ConvertIdentity(OpConverterParams* params) {
 Status ConvertBinary(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
+  // TODO(tmorris): Enable once false is updated to mean either tensor or weight
+  // TF_RETURN_IF_ERROR(CheckInputsWeights(params, {{"x", false}, {"y",
+  // false}}));
   if (inputs.size() != 2) {
-    return errors::InvalidArgument("Binary ops require two inputs, at ",
-                                   node_def.name());
+    return tensorflow::errors::InvalidArgument(
+        node_def.op(), " got ", inputs.size(), " inputs but expected 2, at ",
+        node_def.name());
   }
 
   // Constant folding should have been done by TensorFlow
@@ -2956,11 +2902,7 @@ tensorflow::Status ConvertUnary(OpConverterParams* params) {
       {"Abs", nvinfer1::UnaryOperation::kABS},
       {"Reciprocal", nvinfer1::UnaryOperation::kRECIP},
   };
-
-  if (inputs.size() != 1) {
-    return tensorflow::errors::FailedPrecondition(
-        "Unary ops require single tensor input, at ", node_def.name());
-  }
+  TF_RETURN_IF_ERROR(CheckInputsWeights(params, {{"input", false}}));
 
   // TODO(jie): check type
   const nvinfer1::ITensor* tensor = nullptr;
@@ -3009,14 +2951,7 @@ tensorflow::Status ConvertUnary(OpConverterParams* params) {
 tensorflow::Status ConvertSquare(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
-  if (inputs.size() != 1) {
-    return tensorflow::errors::InvalidArgument("Square expects one input, at ",
-                                               node_def.name());
-  }
-  if (inputs.at(0).is_weights()) {
-    return tensorflow::errors::Unimplemented(
-        "Square is only implemented for tensors, at ", node_def.name());
-  }
+  TF_RETURN_IF_ERROR(CheckInputsWeights(params, {{"input", false}}));
   if (params->validation_only) return Status::OK();
 
   // Constant 2 with same rank as input
@@ -3048,11 +2983,8 @@ tensorflow::Status ConvertSquare(OpConverterParams* params) {
 tensorflow::Status ConvertReduce(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
-  if (inputs.size() != 2 || !inputs.at(0).is_tensor() ||
-      !inputs.at(1).is_weights()) {
-    return tensorflow::errors::InvalidArgument(
-        "Input expects tensor and weights, at", node_def.name());
-  }
+  TF_RETURN_IF_ERROR(
+      CheckInputsWeights(params, {{"input", false}, {"axis", true}}));
 
   const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
   TRT_ShapedWeights index_list = inputs.at(1).weights();
@@ -3113,12 +3045,8 @@ tensorflow::Status ConvertReduce(OpConverterParams* params) {
 tensorflow::Status ConvertPad(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
-  // TODO(aaroey): make a routine for this check and reuse it.
-  if (inputs.size() != 2 || !inputs.at(0).is_tensor() ||
-      !inputs.at(1).is_weights()) {
-    return tensorflow::errors::InvalidArgument(
-        "Input expects tensor and weights, at", node_def.name());
-  }
+  TF_RETURN_IF_ERROR(
+      CheckInputsWeights(params, {{"input", false}, {"paddings", true}}));
 
   // Implement tensor binaryOp weight [channel wise] for now;
   const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
@@ -3299,6 +3227,11 @@ tensorflow::Status ConvertConcat(OpConverterParams* params) {
 tensorflow::Status ConvertFusedBatchNorm(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
+  TF_RETURN_IF_ERROR(CheckInputsWeights(params, {{"input", false},
+                                                 {"scale", true},
+                                                 {"offset", true},
+                                                 {"mean", true},
+                                                 {"variance", true}}));
   TFAttrs attrs(node_def);
   float epsilon = attrs.get<float>("epsilon");
   auto data_format = attrs.get<string>("data_format");
@@ -3319,21 +3252,6 @@ tensorflow::Status ConvertFusedBatchNorm(OpConverterParams* params) {
         node_def.op(), " only supports is_training=false, at ",
         node_def.name());
   }
-  if (inputs.at(0).is_weights()) {
-    return tensorflow::errors::Unimplemented(
-        node_def.op(),
-        " is only implemented for tensor inputs, not weights, at ",
-        node_def.name());
-  }
-  for (int i = 1; i < 5; i++) {
-    if (inputs.at(i).is_tensor()) {
-      return tensorflow::errors::Unimplemented(
-          node_def.op(),
-          " must have constant inputs for scale, offset, mean and variance, "
-          "at ",
-          node_def.name());
-    }
-  }
   nvinfer1::ITensor const* tensor = inputs.at(0).tensor();
 
   //  Check parameter types
@@ -3490,11 +3408,7 @@ tensorflow::Status ConvertMatMulHelper(OpConverterParams* params,
 tensorflow::Status ConvertMatMul(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
-  if (inputs.size() != 2 || !inputs.at(0).is_tensor() ||
-      !inputs.at(1).is_weights()) {
-    return errors::InvalidArgument("Input expects tensor and weights, at ",
-                                   node_def.name());
-  }
+  TF_RETURN_IF_ERROR(CheckInputsWeights(params, {{"a", false}, {"b", true}}));
 
   TFAttrs attrs(node_def);
   tensorflow::DataType tf_dtype = attrs.get<tensorflow::DataType>("T");
@@ -3520,6 +3434,14 @@ tensorflow::Status ConvertMatMul(OpConverterParams* params) {
 tensorflow::Status ConvertBatchMatMul(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
+  // TODO(tmorris): Enable once false is updated to mean either tensor or weight
+  // TF_RETURN_IF_ERROR(CheckInputsWeights(params, {{"x", false}, {"y",
+  // false}}));
+  if (inputs.size() != 2) {
+    return tensorflow::errors::InvalidArgument(
+        node_def.op(), " got ", inputs.size(), " inputs but expected 2, at ",
+        node_def.name());
+  }
   TFAttrs attrs(node_def);
 
   tensorflow::DataType tf_dtype = attrs.get<tensorflow::DataType>("T");
@@ -3591,6 +3513,7 @@ tensorflow::Status ConvertBatchMatMul(OpConverterParams* params) {
 tensorflow::Status ConvertSoftmax(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
+  TF_RETURN_IF_ERROR(CheckInputsWeights(params, {{"input", false}}));
   const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
 
   int nbDims = tensor->getDimensions().nbDims;
@@ -3599,6 +3522,8 @@ tensorflow::Status ConvertSoftmax(OpConverterParams* params) {
         "TensorRT Softmax cannot apply on batch dimension, at" +
         node_def.name());
   }
+  if (params->validation_only) return Status::OK();
+
   nvinfer1::ISoftMaxLayer* layer = params->converter->network()->addSoftMax(
       *const_cast<nvinfer1::ITensor*>(tensor));
   TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name());
@@ -3621,6 +3546,8 @@ tensorflow::Status ConvertTopK(OpConverterParams* params) {
   }
 
   const auto& node_def = params->node_def;
+  TF_RETURN_IF_ERROR(
+      CheckInputsWeights(params, {{"input", false}, {"k", true}}));
   const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
   const int num_dims = tensor->getDimensions().nbDims;
   if (num_dims == 0) {
diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
index d1e30eb848..9b5ad51aa1 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
@@ -521,6 +521,9 @@ class Converter {
   // Tensors/weights added during construction of trt_network_.
   std::unordered_map<string, TRT_TensorOrWeights> trt_tensors_;
 
+  // Input constants which have been converted to tensors via IConstLayer
+  std::unordered_map<TRT_ShapedWeights*, nvinfer1::ITensor*> trt_tensors_from_constants_;
+
   // Special op converter for custom plugins.
   OpConverter plugin_converter_;
 
-- 
GitLab


From dc8163414538f4bd14cd909ad20b959340fc0093 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 30 Jan 2019 09:55:06 -0800
Subject: [PATCH 1752/2345] Remove accidental change

---
 tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
index 9b5ad51aa1..d1e30eb848 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
@@ -521,9 +521,6 @@ class Converter {
   // Tensors/weights added during construction of trt_network_.
   std::unordered_map<string, TRT_TensorOrWeights> trt_tensors_;
 
-  // Input constants which have been converted to tensors via IConstLayer
-  std::unordered_map<TRT_ShapedWeights*, nvinfer1::ITensor*> trt_tensors_from_constants_;
-
   // Special op converter for custom plugins.
   OpConverter plugin_converter_;
 
-- 
GitLab


From f65eefd10bcc6a07ded87bb5ab44990ac7279829 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Thu, 31 Jan 2019 11:03:18 -0800
Subject: [PATCH 1753/2345] Update tests for new error messages

---
 .../tf2tensorrt/convert/convert_nodes.cc      | 10 +-
 .../tf2tensorrt/convert/convert_nodes_test.cc | 94 ++++++++++---------
 2 files changed, 53 insertions(+), 51 deletions(-)

diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
index 868fdd4480..e2ca0db737 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
@@ -1314,7 +1314,7 @@ tensorflow::Status CheckInputsWeights(
   for (int i = 0; i < inputs.size(); i++) {
     if (inputs_is_weight[i].second && inputs.at(i).is_tensor()) {
       return tensorflow::errors::Unimplemented(
-          "The input \",", inputs_is_weight[i].first, "\" for ", node_def.op(),
+          "The input \"", inputs_is_weight[i].first, "\" for ", node_def.op(),
           " must be a constant, at ", node_def.name());
     }
     // TODO(tmorris): Remove this check and provide a method to automatically
@@ -1323,7 +1323,7 @@ tensorflow::Status CheckInputsWeights(
     // duplicate constants from being created.
     if (!inputs_is_weight[i].second && inputs.at(i).is_weights()) {
       return tensorflow::errors::Unimplemented(
-          "The input \",", inputs_is_weight[i].first, "\" for ", node_def.op(),
+          "The input \"", inputs_is_weight[i].first, "\" for ", node_def.op(),
           " must be a tensor, at ", node_def.name());
     }
   }
@@ -2497,12 +2497,6 @@ tensorflow::Status ConvertActivation(OpConverterParams* params) {
 Status ConvertQuantize(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
-  if (inputs.at(0).is_weights()) {
-    // This shouldn't happen unless constant folding failed. TRT will
-    // automatically quantize weights so it should be okay.
-    params->outputs->push_back(inputs.at(0));
-    return Status::OK();
-  }
   if (node_def.op() == "FakeQuantWithMinMaxArgs") {
     TF_RETURN_IF_ERROR(CheckInputsWeights(params, {{"input", false}}));
   } else if (node_def.op() == "FakeQuantWithMinMaxVars" ||
diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
index 37ac63c605..1632132d67 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
@@ -1347,7 +1347,7 @@ TEST_F(OpConverterTest, ConvertTranspose) {
     NodeDef node_def = MakeNodeDef("my_transpose", "Transpose", {});
     RunValidationAndConversion(
         node_def, error::INVALID_ARGUMENT,
-        "Input expects tensor and weights, at my_transpose");
+        "Transpose got 0 inputs but expected 2, at my_transpose");
   }
 
   // Get the NodeDef for Transpose.
@@ -1363,8 +1363,8 @@ TEST_F(OpConverterTest, ConvertTranspose) {
     AddTestTensor("input", {1, 2, 3});
     AddTestTensor("weights", {3});
     RunValidationAndConversion(
-        node_def, error::INVALID_ARGUMENT,
-        "Input expects tensor and weights, at my_transpose");
+        node_def, error::UNIMPLEMENTED,
+        "The input \"perm\" for Transpose must be a constant, at my_transpose");
   }
   {
     // Transpose at batch dimension, should fail.
@@ -1409,7 +1409,7 @@ TEST_F(OpConverterTest, ConvertReshape) {
     NodeDef node_def = MakeNodeDef("my_reshape", "Reshape", {});
     RunValidationAndConversion(
         node_def, error::INVALID_ARGUMENT,
-        "Input expects weights for shape, at my_reshape");
+        "Reshape got 0 inputs but expected 2, at my_reshape");
   }
 
   // Get the NodeDef for Reshape.
@@ -1425,8 +1425,8 @@ TEST_F(OpConverterTest, ConvertReshape) {
     AddTestTensor("input", {1, 2, 3});
     AddTestTensor("weights", {3});
     RunValidationAndConversion(
-        node_def, error::INVALID_ARGUMENT,
-        "Input expects weights for shape, at my_reshape");
+        node_def, error::UNIMPLEMENTED,
+        "The input \"shape\" for Reshape must be a constant, at my_reshape");
   }
   {
     // Reshape to scalar, should fail.
@@ -1501,7 +1501,7 @@ TEST_F(OpConverterTest, ConvertMatMul) {
     NodeDef node_def = MakeNodeDef("my_matmul", "MatMul", {});
     RunValidationAndConversion(
         node_def, error::INVALID_ARGUMENT,
-        "Input expects tensor and weights, at my_matmul");
+        "MatMul got 0 inputs but expected 2, at my_matmul");
   }
 
   // Get the NodeDef for MatMul.
@@ -1645,7 +1645,7 @@ TEST_F(OpConverterTest, ConvertBiasAdd) {
     NodeDef node_def = MakeNodeDef("my_biasadd", "BiasAdd", {});
     RunValidationAndConversion(
         node_def, error::INVALID_ARGUMENT,
-        "Input expects tensor and weights, at my_biasadd");
+        "BiasAdd got 0 inputs but expected 2, at my_biasadd");
   }
 
   // OK. Note that kINT32 is not supported by IScaleLayer, so we don't test
@@ -1928,7 +1928,9 @@ TEST_F(OpConverterTest, ConvertBinary) {
     NodeDef node_def = MakeNodeDef("my_add", "Add", {num_inputs, "input"});
     AddTestTensor("input", {1}, /*batch_size=*/1, nvinfer1::DataType::kFLOAT);
     RunValidationAndConversion(node_def, error::INVALID_ARGUMENT,
-                               "Binary ops require two inputs, at my_add");
+                               StrCat("Add got ", std::to_string(num_inputs),
+                                      "inputs but expected 2, at my_quantize")
+                                   .c_str());
   }
   {
     // Both inputs are weights.
@@ -1998,14 +2000,18 @@ TEST_F(OpConverterTest, ConvertBinary) {
 }
 
 TEST_F(OpConverterTest, ConvertQuantize) {
-  for (const string& op :
-       {"FakeQuantWithMinMaxArgs", "FakeQuantWithMinMaxVars",
-        "QuantizeAndDequantizeV2", "QuantizeAndDequantizeV3"}) {
+  const std::pair<string, int> op_with_num_inputs[4] = {
+      {"FakeQuantWithMinMaxArgs", 1},
+      {"FakeQuantWithMinMaxVars", 3},
+      {"QuantizeAndDequantizeV2", 3},
+      {"QuantizeAndDequantizeV3", 4}};
+  for (const auto& pair : op_with_num_inputs) {
     // Input list is empty, should fail.
-    NodeDef node_def = MakeNodeDef("my_quantize", op, {});
+    NodeDef node_def = MakeNodeDef("my_quantize", pair.first, {});
     RunValidationAndConversion(
         node_def, error::INVALID_ARGUMENT,
-        StrCat("Invalid number of inputs for ", op, ", at my_quantize")
+        StrCat(pair.first, " got 0 inputs but expected ",
+               std::to_string(pair.second), ", at my_quantize")
             .c_str());
   }
   {
@@ -2092,9 +2098,9 @@ TEST_F(OpConverterTest, ConvertQuantize) {
     AddTestTensor("weights_min", {1});
     AddTestTensor("weights_max", {1});
     RunValidationAndConversion(
-        node_def, error::INVALID_ARGUMENT,
-        "Min and max inputs for QuantizeAndDequantizeV2 must be weights not "
-        "tensors, at my_quantize");
+        node_def, error::UNIMPLEMENTED,
+        "The input \"min\" for QuantizeAndDequantizeV2 must be a constant"
+        ", at my_quantize");
   }
   {
     // QuantizeAndDequantizeV3 ranges set via inputs, ok.
@@ -2127,7 +2133,7 @@ TEST_F(OpConverterTest, ConvertRelu6) {
     NodeDef node_def = MakeNodeDef("my_relu6", "Relu6", {});
     RunValidationAndConversion(
         node_def, error::INVALID_ARGUMENT,
-        "Invalid number of inputs for Relu6, at my_relu6");
+        "Relu6 got 0 inputs but expected 1, at my_relu6");
   }
 
   // Get the NodeDef for Relu6.
@@ -2141,7 +2147,7 @@ TEST_F(OpConverterTest, ConvertRelu6) {
     AddTestWeights<float>("input", {1}, {1.0f});
     RunValidationAndConversion(
         node_def, error::UNIMPLEMENTED,
-        "Relu6 is only implemented for tensors, not weights, at my_relu6");
+        "The input \"input\" for Relu6 must be a tensor, at my_relu6");
   }
   {
     // Clip tensor values and set quantization ranges, ok.
@@ -2198,8 +2204,9 @@ TEST_F(OpConverterTest, ConvertSquare) {
   {
     // Input list is empty, should fail.
     NodeDef node_def = MakeNodeDef("my_square", "Square", {});
-    RunValidationAndConversion(node_def, error::INVALID_ARGUMENT,
-                               "Square expects one input, at my_square");
+    RunValidationAndConversion(
+        node_def, error::INVALID_ARGUMENT,
+        "Square got 0 inputs but expected 1, at my_square");
   }
   {
     // Input is weights, should fail.
@@ -2211,7 +2218,7 @@ TEST_F(OpConverterTest, ConvertSquare) {
     AddTestWeights<float>("input", {1, 2, 3}, {1, 2, 3, 4, -5, 6});
     RunValidationAndConversion(
         node_def, error::UNIMPLEMENTED,
-        "Square is only implemented for tensors, at my_square");
+        "The input \"input\" for Square must be a tensor, at my_square");
   }
 
   // OK. Note that kINT32 is not supported by IElementWiseLayer, so we don't
@@ -2227,7 +2234,7 @@ TEST_F(OpConverterTest, ConvertActivation) {
     // Input list is empty, should fail.
     NodeDef node_def = MakeNodeDef("my_act", "Relu", {});
     RunValidationAndConversion(node_def, error::INVALID_ARGUMENT,
-                               "Relu expects one input, at my_act");
+                               "Relu got 0 inputs but expected 1, at my_act");
   }
   {
     // Input is weights, should fail.
@@ -2239,7 +2246,7 @@ TEST_F(OpConverterTest, ConvertActivation) {
     AddTestWeights<int32>("input", {1, 2, 3}, {-3, -2, -1, 0, 1, 2});
     RunValidationAndConversion(
         node_def, error::UNIMPLEMENTED,
-        "Relu is only implemented for tensors, at my_act");
+        "The input \"input\" for Relu must be a tensor, at my_act");
   }
 
   // Get nodedef for activation layer.
@@ -2301,7 +2308,7 @@ TEST_F(OpConverterTest, ConvertExpandDims) {
     NodeDef node_def = MakeNodeDef("my_expanddims", "ExpandDims", {});
     RunValidationAndConversion(
         node_def, error::INVALID_ARGUMENT,
-        "Two inputs expected for ExpandDims, at my_expanddims");
+        "ExpandDims got 0 inputs but expected 2, at my_expanddims");
   }
 
   // Get the NodeDef for ExpandDims.
@@ -2316,18 +2323,18 @@ TEST_F(OpConverterTest, ConvertExpandDims) {
     Reset();
     AddTestWeights<int32>("input", {1, 2, 3}, {1, 2, 3, 4, 5, 6});
     AddTestWeights<int32>("weights", {1}, {1});
-    RunValidationAndConversion(
-        node_def, error::UNIMPLEMENTED,
-        "ExpandDims expects tensor for input, at my_expanddims");
+    RunValidationAndConversion(node_def, error::UNIMPLEMENTED,
+                               "The input \"input\" for ExpandDims must be a "
+                               "tensor, at my_expanddims");
   }
   {
     // Axis is a tensor, should fail.
     Reset();
     AddTestTensor("input", {1, 2, 3});
     AddTestTensor("weights", {3});
-    RunValidationAndConversion(
-        node_def, error::INVALID_ARGUMENT,
-        "ExpandDims expects weights for axis, at my_expanddims");
+    RunValidationAndConversion(node_def, error::UNIMPLEMENTED,
+                               "The input \"axis\" for ExpandDims must be a "
+                               "constant, at my_expanddims");
   }
   {
     // Add dim at batch dimension, should fail.
@@ -2416,8 +2423,9 @@ TEST_F(OpConverterTest, ConvertSqueeze) {
   {
     // Input list is empty, should fail.
     NodeDef node_def = MakeNodeDef("my_squeeze", "Squeeze", {});
-    RunValidationAndConversion(node_def, error::INVALID_ARGUMENT,
-                               "One input expected for Squeeze, at my_squeeze");
+    RunValidationAndConversion(
+        node_def, error::INVALID_ARGUMENT,
+        "Squeeze got 0 inputs but expected 1, at my_squeeze");
   }
   {
     // No attrs, should fail.
@@ -2450,7 +2458,7 @@ TEST_F(OpConverterTest, ConvertSqueeze) {
     AddTestWeights<float>("input", {1, 2, 3}, {1, 2, 3, 4, 5, 6});
     RunValidationAndConversion(
         node_def, error::UNIMPLEMENTED,
-        "Squeeze expects tensor for input, at my_squeeze");
+        "The input \"input\" for Squeeze must be a tensor, at my_squeeze");
   }
   {
     // Squeeze batch dim, should fail.
@@ -2540,7 +2548,7 @@ TEST_F(OpConverterTest, ConvertStridedSlice) {
     NodeDef node_def = MakeNodeDef("my_strided_slice", "StridedSlice", {});
     RunValidationAndConversion(
         node_def, error::INVALID_ARGUMENT,
-        "StridedSlice expects 4 inputs, at my_strided_slice");
+        "StridedSlice got 0 inputs but expected 4, at my_strided_slice");
   }
 
   // Get nodedef for StridedSlice layer.
@@ -2571,9 +2579,9 @@ TEST_F(OpConverterTest, ConvertStridedSlice) {
     AddTestWeights<int32>("begin", {4}, {0, 0, 0, 0});
     AddTestWeights<int32>("end", {4}, {1, 1, 2, 3});
     AddTestWeights<int32>("strides", {4}, {1, 1, 1, 1});
-    RunValidationAndConversion(
-        node_def, error::UNIMPLEMENTED,
-        "StridedSlice is only implemented for tensors, at my_strided_slice");
+    RunValidationAndConversion(node_def, error::UNIMPLEMENTED,
+                               "The input \"input\" for StridedSlice must be a "
+                               "tensor, at my_strided_slice");
   }
   {
     // Begin, end, strides are tensors, should fail.
@@ -2584,8 +2592,8 @@ TEST_F(OpConverterTest, ConvertStridedSlice) {
     AddTestTensor("end", {4});
     AddTestTensor("strides", {4});
     RunValidationAndConversion(
-        node_def, error::INVALID_ARGUMENT,
-        "StridedSlice expects weights for begin, end, and strides, at "
+        node_def, error::UNIMPLEMENTED,
+        "The input \"begin\" for StridedSlice must be a constant, at "
         "my_strided_slice");
   }
   {
@@ -2818,7 +2826,7 @@ TEST_F(OpConverterTest, ConvertConv2D) {
     NodeDef node_def = MakeNodeDef("my_conv2d", "Conv2D", {});
     RunValidationAndConversion(
         node_def, error::INVALID_ARGUMENT,
-        "Two inputs are expected for Conv2D, at my_conv2d");
+        "Conv2D got 0 inputs but expected 2, at my_conv2d");
   }
 
   // Get nodedef for Conv2D layer.
@@ -2856,7 +2864,7 @@ TEST_F(OpConverterTest, ConvertConv2D) {
     AddTestWeights<float>("weights", {3, 3, 1, 1}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
     RunValidationAndConversion(
         node_def, error::UNIMPLEMENTED,
-        "Conv2D is only implemented for tensors, not weights, at my_conv2d");
+        "The input \"input\" for Conv2D must be a tensor, at my_conv2d");
   }
   {
     // Filter is tensor, should fail.
@@ -2866,7 +2874,7 @@ TEST_F(OpConverterTest, ConvertConv2D) {
     AddTestTensor("weights", {3, 3, 1, 1});
     RunValidationAndConversion(
         node_def, error::UNIMPLEMENTED,
-        "Kernel for Conv2D must be constant weights, at my_conv2d");
+        "The input \"filter\" for Conv2D must be a constant, at my_conv2d");
   }
   {
     // Filter is not 4D, should fail.
-- 
GitLab


From ef8910afc1fdeefc2f56387550e83ebb7763022e Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Thu, 31 Jan 2019 11:04:09 -0800
Subject: [PATCH 1754/2345] Fix failing test

---
 tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
index 1632132d67..d638dc2f3a 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
@@ -1927,7 +1927,7 @@ TEST_F(OpConverterTest, ConvertBinary) {
     Reset();
     NodeDef node_def = MakeNodeDef("my_add", "Add", {num_inputs, "input"});
     AddTestTensor("input", {1}, /*batch_size=*/1, nvinfer1::DataType::kFLOAT);
-    RunValidationAndConversion(node_def, error::INVALID_ARGUMENT,
+    RunValidationAndConversion(node_def, error::UNIMPLEMENTED,
                                StrCat("Add got ", std::to_string(num_inputs),
                                       "inputs but expected 2, at my_quantize")
                                    .c_str());
-- 
GitLab


From fa567f57ecc84e7f3bb703c332f3646b45697f24 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Thu, 31 Jan 2019 12:51:51 -0800
Subject: [PATCH 1755/2345] Fix failing test

---
 tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
index d638dc2f3a..ba4405dc69 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
@@ -1927,9 +1927,9 @@ TEST_F(OpConverterTest, ConvertBinary) {
     Reset();
     NodeDef node_def = MakeNodeDef("my_add", "Add", {num_inputs, "input"});
     AddTestTensor("input", {1}, /*batch_size=*/1, nvinfer1::DataType::kFLOAT);
-    RunValidationAndConversion(node_def, error::UNIMPLEMENTED,
+    RunValidationAndConversion(node_def, error::INVALID_ARGUMENT,
                                StrCat("Add got ", std::to_string(num_inputs),
-                                      "inputs but expected 2, at my_quantize")
+                                      " inputs but expected 2, at my_add")
                                    .c_str());
   }
   {
-- 
GitLab


From d377c96998351d42991bf7a52fcf4c92883808b3 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 1 Feb 2019 09:19:01 -0800
Subject: [PATCH 1756/2345] Make args for CheckInputsWeights const references.
 Update input names to match c++ API

---
 .../tf2tensorrt/convert/convert_nodes.cc      | 77 ++++++++++---------
 .../tf2tensorrt/convert/convert_nodes_test.cc |  4 +-
 2 files changed, 42 insertions(+), 39 deletions(-)

diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
index e2ca0db737..cfec175817 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
@@ -1302,10 +1302,10 @@ Status Converter::GetInputs(const tensorflow::NodeDef& node_def,
 // while false means the input must be a tensor. In the future, false will mean
 // the input can be a tensor or weight.
 tensorflow::Status CheckInputsWeights(
-    OpConverterParams* params,
-    const std::vector<std::pair<string, bool>> inputs_is_weight) {
-  const auto& inputs = params->inputs;
-  const auto& node_def = params->node_def;
+    const OpConverterParams& params,
+    const std::vector<std::pair<string, bool>>& inputs_is_weight) {
+  const auto& inputs = params.inputs;
+  const auto& node_def = params.node_def;
   if (inputs.size() != inputs_is_weight.size()) {
     return tensorflow::errors::InvalidArgument(
         node_def.op(), " got ", inputs.size(), " inputs but expected ",
@@ -1623,13 +1623,13 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group,
   const nvinfer1::ITensor* tensor = nullptr;
   if (is_conv2d_backprop_input) {
     TF_RETURN_IF_ERROR(CheckInputsWeights(
-        params,
-        {{"input_sizes", true}, {"filter", true},  {"out_backprop", false}}));
+        *params,
+        {{"input_sizes", true}, {"filter", true}, {"out_backprop", false}}));
     backprop_output_size = inputs.at(0);
     tensor = inputs.at(2).tensor();
   } else {
     TF_RETURN_IF_ERROR(
-        CheckInputsWeights(params, {{"input", false}, {"filter", true}}));
+        CheckInputsWeights(*params, {{"input", false}, {"filter", true}}));
     tensor = inputs.at(0).tensor();
   }
   TRT_ShapedWeights weights_rsck = inputs.at(1).weights();
@@ -1893,7 +1893,7 @@ tensorflow::Status ConvertPlugin(OpConverterParams* params) {
 tensorflow::Status ConvertTranspose(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   TF_RETURN_IF_ERROR(
-      CheckInputsWeights(params, {{"input", false}, {"perm", true}}));
+      CheckInputsWeights(*params, {{"x", false}, {"perm", true}}));
   // Get the permutation from weights.
   TRT_ShapedWeights weights = inputs.at(1).weights();
   const int* weights_ptr =
@@ -1927,7 +1927,7 @@ tensorflow::Status ConvertReshape(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
   TF_RETURN_IF_ERROR(
-      CheckInputsWeights(params, {{"input", false}, {"shape", true}}));
+      CheckInputsWeights(*params, {{"tensor", false}, {"shape", true}}));
   TRT_TensorOrWeights input_tensor = inputs.at(0);
   TRT_ShapedWeights weights = inputs.at(1).weights();
   if (weights.count() == 0) {
@@ -2024,7 +2024,7 @@ tensorflow::Status ConvertExpandDims(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
   TF_RETURN_IF_ERROR(
-      CheckInputsWeights(params, {{"input", false}, {"axis", true}}));
+      CheckInputsWeights(*params, {{"input", false}, {"axis", true}}));
   // Get input shape as vector.
   TRT_TensorOrWeights input_tensor = inputs.at(0);
   const nvinfer1::Dims dims = input_tensor.GetTrtDims();
@@ -2074,7 +2074,7 @@ tensorflow::Status ConvertExpandDims(OpConverterParams* params) {
 tensorflow::Status ConvertSqueeze(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
-  TF_RETURN_IF_ERROR(CheckInputsWeights(params, {{"input", false}}));
+  TF_RETURN_IF_ERROR(CheckInputsWeights(*params, {{"input", false}}));
   // Get input shape.
   TRT_TensorOrWeights input_tensor = inputs.at(0);
   const nvinfer1::Dims dims = input_tensor.GetTrtDims();
@@ -2177,7 +2177,7 @@ tensorflow::Status ConvertStridedSlice(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
   TF_RETURN_IF_ERROR(CheckInputsWeights(
-      params,
+      *params,
       {{"input", false}, {"begin", true}, {"end", true}, {"strides", true}}));
   // Get input dims.
   nvinfer1::Dims dims = inputs.at(0).GetTrtDims();
@@ -2373,7 +2373,7 @@ tensorflow::Status ConvertConv2DBackpropInput(OpConverterParams* params) {
 tensorflow::Status ConvertPool(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
-  TF_RETURN_IF_ERROR(CheckInputsWeights(params, {{"input", false}}));
+  TF_RETURN_IF_ERROR(CheckInputsWeights(*params, {{"input", false}}));
   nvinfer1::PoolingType type;
   if (node_def.op() == "MaxPool") {
     type = nvinfer1::PoolingType::kMAX;
@@ -2463,7 +2463,7 @@ tensorflow::Status ConvertPool(OpConverterParams* params) {
 tensorflow::Status ConvertActivation(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
-  TF_RETURN_IF_ERROR(CheckInputsWeights(params, {{"input", false}}));
+  TF_RETURN_IF_ERROR(CheckInputsWeights(*params, {{"input", false}}));
   static const std::unordered_map<string, nvinfer1::ActivationType> ops{
       {"Relu", nvinfer1::ActivationType::kRELU},
       {"Sigmoid", nvinfer1::ActivationType::kSIGMOID},
@@ -2498,15 +2498,18 @@ Status ConvertQuantize(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
   if (node_def.op() == "FakeQuantWithMinMaxArgs") {
-    TF_RETURN_IF_ERROR(CheckInputsWeights(params, {{"input", false}}));
-  } else if (node_def.op() == "FakeQuantWithMinMaxVars" ||
-             node_def.op() == "QuantizeAndDequantizeV2") {
+    TF_RETURN_IF_ERROR(CheckInputsWeights(*params, {{"input", false}}));
+  } else if (node_def.op() == "FakeQuantWithMinMaxVars") {
     TF_RETURN_IF_ERROR(CheckInputsWeights(
-        params, {{"input", false}, {"min", true}, {"max", true}}));
-  } else if (node_def.op() == "QuantizeAndDequantizeV3") {
+        *params, {{"input", false}, {"min", true}, {"max", true}}));
+  } else if (node_def.op() == "QuantizeAndDequantizeV2") {
     TF_RETURN_IF_ERROR(CheckInputsWeights(
-        params,
-        {{"input", false}, {"min", true}, {"max", true}, {"num_bits", true}}));
+        *params, {{"input", false}, {"input_min", true}, {"input_max", true}}));
+  } else if (node_def.op() == "QuantizeAndDequantizeV3") {
+    TF_RETURN_IF_ERROR(CheckInputsWeights(*params, {{"input", false},
+                                                    {"input_min", true},
+                                                    {"input_max", true},
+                                                    {"num_bits", true}}));
   }
   float min_range = 0.0f;
   float max_range = 0.0f;
@@ -2558,7 +2561,7 @@ Status ConvertQuantize(OpConverterParams* params) {
 tensorflow::Status ConvertRelu6(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
-  TF_RETURN_IF_ERROR(CheckInputsWeights(params, {{"input", false}}));
+  TF_RETURN_IF_ERROR(CheckInputsWeights(*params, {{"input", false}}));
   if (params->validation_only) return Status::OK();
   // ***************************************************************************
   // TensorRT does not implement Relu6 natively. This function converts Relu6 op
@@ -2620,7 +2623,7 @@ tensorflow::Status ConvertBiasAdd(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
   TF_RETURN_IF_ERROR(
-      CheckInputsWeights(params, {{"input", false}, {"bias", true}}));
+      CheckInputsWeights(*params, {{"value", false}, {"bias", true}}));
   TFAttrs attrs(node_def);
   tensorflow::DataType tf_dtype = attrs.get<tensorflow::DataType>("T");
   if (tf_dtype != DataType::DT_FLOAT && tf_dtype != DataType::DT_HALF) {
@@ -2841,7 +2844,7 @@ Status ConvertBinary(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
   // TODO(tmorris): Enable once false is updated to mean either tensor or weight
-  // TF_RETURN_IF_ERROR(CheckInputsWeights(params, {{"x", false}, {"y",
+  // TF_RETURN_IF_ERROR(CheckInputsWeights(*params, {{"x", false}, {"y",
   // false}}));
   if (inputs.size() != 2) {
     return tensorflow::errors::InvalidArgument(
@@ -2896,7 +2899,7 @@ tensorflow::Status ConvertUnary(OpConverterParams* params) {
       {"Abs", nvinfer1::UnaryOperation::kABS},
       {"Reciprocal", nvinfer1::UnaryOperation::kRECIP},
   };
-  TF_RETURN_IF_ERROR(CheckInputsWeights(params, {{"input", false}}));
+  TF_RETURN_IF_ERROR(CheckInputsWeights(*params, {{"x", false}}));
 
   // TODO(jie): check type
   const nvinfer1::ITensor* tensor = nullptr;
@@ -2945,7 +2948,7 @@ tensorflow::Status ConvertUnary(OpConverterParams* params) {
 tensorflow::Status ConvertSquare(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
-  TF_RETURN_IF_ERROR(CheckInputsWeights(params, {{"input", false}}));
+  TF_RETURN_IF_ERROR(CheckInputsWeights(*params, {{"x", false}}));
   if (params->validation_only) return Status::OK();
 
   // Constant 2 with same rank as input
@@ -2978,7 +2981,7 @@ tensorflow::Status ConvertReduce(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
   TF_RETURN_IF_ERROR(
-      CheckInputsWeights(params, {{"input", false}, {"axis", true}}));
+      CheckInputsWeights(*params, {{"input", false}, {"axis", true}}));
 
   const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
   TRT_ShapedWeights index_list = inputs.at(1).weights();
@@ -3040,7 +3043,7 @@ tensorflow::Status ConvertPad(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
   TF_RETURN_IF_ERROR(
-      CheckInputsWeights(params, {{"input", false}, {"paddings", true}}));
+      CheckInputsWeights(*params, {{"tensor", false}, {"paddings", true}}));
 
   // Implement tensor binaryOp weight [channel wise] for now;
   const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
@@ -3221,11 +3224,11 @@ tensorflow::Status ConvertConcat(OpConverterParams* params) {
 tensorflow::Status ConvertFusedBatchNorm(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
-  TF_RETURN_IF_ERROR(CheckInputsWeights(params, {{"input", false},
-                                                 {"scale", true},
-                                                 {"offset", true},
-                                                 {"mean", true},
-                                                 {"variance", true}}));
+  TF_RETURN_IF_ERROR(CheckInputsWeights(*params, {{"x", false},
+                                                  {"scale", true},
+                                                  {"offset", true},
+                                                  {"mean", true},
+                                                  {"variance", true}}));
   TFAttrs attrs(node_def);
   float epsilon = attrs.get<float>("epsilon");
   auto data_format = attrs.get<string>("data_format");
@@ -3402,7 +3405,7 @@ tensorflow::Status ConvertMatMulHelper(OpConverterParams* params,
 tensorflow::Status ConvertMatMul(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
-  TF_RETURN_IF_ERROR(CheckInputsWeights(params, {{"a", false}, {"b", true}}));
+  TF_RETURN_IF_ERROR(CheckInputsWeights(*params, {{"a", false}, {"b", true}}));
 
   TFAttrs attrs(node_def);
   tensorflow::DataType tf_dtype = attrs.get<tensorflow::DataType>("T");
@@ -3429,7 +3432,7 @@ tensorflow::Status ConvertBatchMatMul(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
   // TODO(tmorris): Enable once false is updated to mean either tensor or weight
-  // TF_RETURN_IF_ERROR(CheckInputsWeights(params, {{"x", false}, {"y",
+  // TF_RETURN_IF_ERROR(CheckInputsWeights(*params, {{"x", false}, {"y",
   // false}}));
   if (inputs.size() != 2) {
     return tensorflow::errors::InvalidArgument(
@@ -3507,7 +3510,7 @@ tensorflow::Status ConvertBatchMatMul(OpConverterParams* params) {
 tensorflow::Status ConvertSoftmax(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
-  TF_RETURN_IF_ERROR(CheckInputsWeights(params, {{"input", false}}));
+  TF_RETURN_IF_ERROR(CheckInputsWeights(*params, {{"logits", false}}));
   const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
 
   int nbDims = tensor->getDimensions().nbDims;
@@ -3541,7 +3544,7 @@ tensorflow::Status ConvertTopK(OpConverterParams* params) {
 
   const auto& node_def = params->node_def;
   TF_RETURN_IF_ERROR(
-      CheckInputsWeights(params, {{"input", false}, {"k", true}}));
+      CheckInputsWeights(*params, {{"input", false}, {"k", true}}));
   const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
   const int num_dims = tensor->getDimensions().nbDims;
   if (num_dims == 0) {
diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
index ba4405dc69..77221f6d9a 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
@@ -2099,7 +2099,7 @@ TEST_F(OpConverterTest, ConvertQuantize) {
     AddTestTensor("weights_max", {1});
     RunValidationAndConversion(
         node_def, error::UNIMPLEMENTED,
-        "The input \"min\" for QuantizeAndDequantizeV2 must be a constant"
+        "The input \"input_min\" for QuantizeAndDequantizeV2 must be a constant"
         ", at my_quantize");
   }
   {
@@ -2218,7 +2218,7 @@ TEST_F(OpConverterTest, ConvertSquare) {
     AddTestWeights<float>("input", {1, 2, 3}, {1, 2, 3, 4, -5, 6});
     RunValidationAndConversion(
         node_def, error::UNIMPLEMENTED,
-        "The input \"input\" for Square must be a tensor, at my_square");
+        "The input \"x\" for Square must be a tensor, at my_square");
   }
 
   // OK. Note that kINT32 is not supported by IElementWiseLayer, so we don't
-- 
GitLab


From 90a8b12b2860eae2d3f893fd7abdde19089c41b6 Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Fri, 1 Feb 2019 09:21:19 -0800
Subject: [PATCH 1757/2345] Fixed the regression bug about prefetch_buffer.

PiperOrigin-RevId: 231982986
---
 tensorflow/python/distribute/input_lib.py     | 52 ++++++++++++-------
 .../keras/engine/training_distributed.py      |  4 +-
 2 files changed, 34 insertions(+), 22 deletions(-)

diff --git a/tensorflow/python/distribute/input_lib.py b/tensorflow/python/distribute/input_lib.py
index be6b713f04..6b13db39c7 100644
--- a/tensorflow/python/distribute/input_lib.py
+++ b/tensorflow/python/distribute/input_lib.py
@@ -345,7 +345,7 @@ class _SingleWorkerCallableIterator(object):
 
 # TODO(sourabhbajaj): Remove this in lieu of distributed datasets
 def _get_batched_dataset(d):
-  """Get the underlying batch dataset from the dataset object."""
+  """Get the batched dataset from `d`."""
   # pylint: disable=protected-access
   if isinstance(d, dataset_ops.DatasetV1Adapter):
     d = d._dataset
@@ -361,24 +361,17 @@ def _get_batched_dataset(d):
       "The batch operations can be followed by a prefetch.")
 
 
-def _get_batched_dataset_attributes(dataset):
-  """Get `batch_size`, `drop_remainder`, and `prefetch_buffer` of dataset."""
+def _get_batched_dataset_attributes(d):
+  """Get `batch_size`, `drop_remainder` of dataset."""
   # pylint: disable=protected-access
-  assert isinstance(dataset,
+  assert isinstance(d,
                     (dataset_ops.BatchDataset, batching._MapAndBatchDataset))
-  if isinstance(dataset, dataset_ops.BatchDataset):
-    batch_size = dataset._batch_size
-    drop_remainder = dataset._drop_remainder
-  elif isinstance(dataset, batching._MapAndBatchDataset):
-    batch_size = dataset._batch_size_t
-    drop_remainder = dataset._drop_remainder_t
-
-  prefetch_buffer = None
-  if isinstance(dataset, dataset_ops.PrefetchDataset):
-    prefetch_buffer = dataset._buffer_size
-  elif (isinstance(dataset, dataset_ops.DatasetV1Adapter)
-        and isinstance(dataset._dataset, dataset_ops.PrefetchDataset)):
-    prefetch_buffer = dataset._dataset._buffer_size
+  if isinstance(d, dataset_ops.BatchDataset):
+    batch_size = d._batch_size
+    drop_remainder = d._drop_remainder
+  elif isinstance(d, batching._MapAndBatchDataset):
+    batch_size = d._batch_size_t
+    drop_remainder = d._drop_remainder_t
   # pylint: enable=protected-access
 
   if tensor_util.is_tensor(batch_size):
@@ -387,14 +380,35 @@ def _get_batched_dataset_attributes(dataset):
   if tensor_util.is_tensor(drop_remainder):
     drop_remainder = tensor_util.constant_value(drop_remainder)
 
+  return batch_size, drop_remainder
+
+
+# TODO(sourabhbajaj): Remove this in lieu of distributed datasets
+def _get_dataset_attributes(dataset):
+  """Get the underlying attributes from the dataset object."""
+  # pylint: disable=protected-access
+
+  # First, get batch_size and drop_remainder from the dataset. We need
+  # to walk back the dataset creation process and find the batched version in
+  # order to get the attributes.
+  batched_dataset = _get_batched_dataset(dataset)
+  batch_size, drop_remainder = _get_batched_dataset_attributes(batched_dataset)
+
+  # Second, prefetch buffer should be get from the original dataset.
+  prefetch_buffer = None
+  if isinstance(dataset, dataset_ops.PrefetchDataset):
+    prefetch_buffer = dataset._buffer_size
+  elif (isinstance(dataset, dataset_ops.DatasetV1Adapter)
+        and isinstance(dataset._dataset, dataset_ops.PrefetchDataset)):
+    prefetch_buffer = dataset._dataset._buffer_size
+
   return batch_size, drop_remainder, prefetch_buffer
 
 
 def _split_dataset_batch(dataset, split_batch_by):
   """Divide a batch-ed dataset's batches into smaller batches."""
-  batched_dataset = _get_batched_dataset(dataset)
   batch_size, drop_remainder, prefetch_buffer = (
-      _get_batched_dataset_attributes(batched_dataset))
+      _get_dataset_attributes(dataset))
 
   if batch_size % split_batch_by:
     raise ValueError(
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index c0ac70f436..3d72a7b6f1 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -597,9 +597,7 @@ def experimental_tpu_predict_loop(model,
     # during graph optimization.
     padding_handler = padding_util.PartialBatchPaddingHandler(
         model._feed_output_shapes)
-    batched_dataset = input_lib._get_batched_dataset(dataset)
-    batch_size, _, prefetch_buffer = input_lib._get_batched_dataset_attributes(
-        batched_dataset)
+    batch_size, _, prefetch_buffer = input_lib._get_dataset_attributes(dataset)
     padding_handler.padded_batch_size = batch_size
     padding_handler.padding_mask = dataset.reduce(padding_handler.padding_mask,
                                                   padding_handler.update_mask)
-- 
GitLab


From de87e628e6d89382783ea948c21fa66b182d69d3 Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Fri, 1 Feb 2019 09:25:37 -0800
Subject: [PATCH 1758/2345] Adding a RebatchDatasetOp that uses Grappler to
 modify the dataset graph to mutate the batch size of the Dataset.

Right now this basically does all the wiring through Grappler etc. and the logic is very minimal and basic. Subsequent CL's will make the logic expand to different scenarios.

PiperOrigin-RevId: 231983575
---
 .../api_def_ExperimentalRebatchDataset.pbtxt  |  23 ++
 .../core/grappler/optimizers/data/BUILD       |  15 +
 .../grappler/optimizers/data/graph_utils.cc   |   7 +
 .../grappler/optimizers/data/graph_utils.h    |   4 +
 .../optimizers/data/graph_utils_test.cc       |  15 +
 .../core/grappler/optimizers/data/rebatch.cc  | 115 ++++++++
 .../core/grappler/optimizers/data/rebatch.h   |  52 ++++
 tensorflow/core/kernels/data/BUILD            |  24 +-
 .../core/kernels/data/experimental/BUILD      |  16 ++
 .../data/experimental/rebatch_dataset_op.cc   |  92 ++++++
 .../kernels/data/graph_rewrite_dataset.cc     | 239 ++++++++++++++++
 .../core/kernels/data/graph_rewrite_dataset.h |  92 ++++++
 .../core/kernels/data/optimize_dataset_op.cc  | 264 +-----------------
 .../core/ops/experimental_dataset_ops.cc      |   8 +
 .../data/experimental/kernel_tests/BUILD      |  14 +
 .../kernel_tests/rebatch_dataset_test.py      |  60 ++++
 .../kernel_tests/serialization/BUILD          |  18 ++
 .../rebatch_dataset_serialization_test.py     |  41 +++
 .../python/data/experimental/ops/batching.py  |  31 ++
 .../api/golden/v1/tensorflow.raw_ops.pbtxt    |   4 +
 .../api/golden/v2/tensorflow.raw_ops.pbtxt    |   4 +
 21 files changed, 874 insertions(+), 264 deletions(-)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_ExperimentalRebatchDataset.pbtxt
 create mode 100644 tensorflow/core/grappler/optimizers/data/rebatch.cc
 create mode 100644 tensorflow/core/grappler/optimizers/data/rebatch.h
 create mode 100644 tensorflow/core/kernels/data/experimental/rebatch_dataset_op.cc
 create mode 100644 tensorflow/core/kernels/data/graph_rewrite_dataset.cc
 create mode 100644 tensorflow/core/kernels/data/graph_rewrite_dataset.h
 create mode 100644 tensorflow/python/data/experimental/kernel_tests/rebatch_dataset_test.py
 create mode 100644 tensorflow/python/data/experimental/kernel_tests/serialization/rebatch_dataset_serialization_test.py

diff --git a/tensorflow/core/api_def/base_api/api_def_ExperimentalRebatchDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalRebatchDataset.pbtxt
new file mode 100644
index 0000000000..b8455308e5
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_ExperimentalRebatchDataset.pbtxt
@@ -0,0 +1,23 @@
+op {
+  graph_op_name: "ExperimentalRebatchDataset"
+  visibility: HIDDEN
+  in_arg {
+    name: "input_dataset"
+    description: <<END
+A variant tensor representing the input dataset.
+END
+  }
+  in_arg {
+  name: "num_workers"
+  description: <<END
+A scalar representing the number of workers to distribute this batch across. As
+a result of this transformation the current batch size would end up being
+divided  by this parameter.
+END
+  }
+  summary: "Creates a dataset that changes the batch size."
+  description: <<END
+Creates a dataset that changes the batch size of the dataset to current batch
+size // num_workers.
+END
+}
diff --git a/tensorflow/core/grappler/optimizers/data/BUILD b/tensorflow/core/grappler/optimizers/data/BUILD
index 682e7cd6c4..ef02962975 100644
--- a/tensorflow/core/grappler/optimizers/data/BUILD
+++ b/tensorflow/core/grappler/optimizers/data/BUILD
@@ -6,6 +6,7 @@ load("//tensorflow/core:platform/default/build_config.bzl", "tf_protos_all")
 package(default_visibility = [
     "//tensorflow/core/grappler/optimizers/data:__subpackages__",
     "//tensorflow/core/kernels/data:__pkg__",
+    "//tensorflow/core/kernels/data/experimental:__pkg__",
 ])
 
 cc_library(
@@ -540,6 +541,20 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "rebatch",
+    srcs = ["rebatch.cc"],
+    hdrs = ["rebatch.h"],
+    deps = [
+        ":graph_utils",
+        "//tensorflow/core/grappler:grappler_item",
+        "//tensorflow/core/grappler:mutable_graph_view",
+        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
+        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
+    ],
+    alwayslink = 1,
+)
+
 cc_library(
     name = "noop_elimination",
     srcs = ["noop_elimination.cc"],
diff --git a/tensorflow/core/grappler/optimizers/data/graph_utils.cc b/tensorflow/core/grappler/optimizers/data/graph_utils.cc
index cbafb9dc82..7bcc12c3a4 100644
--- a/tensorflow/core/grappler/optimizers/data/graph_utils.cc
+++ b/tensorflow/core/grappler/optimizers/data/graph_utils.cc
@@ -232,6 +232,13 @@ NodeDef* GetInputNode(const NodeDef& node, const MutableGraphView& graph) {
   return graph.GetRegularFanin(input_port).node;
 }
 
+NodeDef* GetInputNode(const NodeDef& node, const MutableGraphView& graph,
+                      int64 i) {
+  if (node.input_size() <= i) return nullptr;
+  MutableGraphView::InputPort input_port = graph.GetInputPort(node.name(), i);
+  return graph.GetRegularFanin(input_port).node;
+}
+
 void SetUniqueGraphNodeName(StringPiece prefix, GraphDef* graph,
                             NodeDef* node) {
   string name = string(prefix);
diff --git a/tensorflow/core/grappler/optimizers/data/graph_utils.h b/tensorflow/core/grappler/optimizers/data/graph_utils.h
index 8f2872c146..22298cc311 100644
--- a/tensorflow/core/grappler/optimizers/data/graph_utils.h
+++ b/tensorflow/core/grappler/optimizers/data/graph_utils.h
@@ -108,6 +108,10 @@ int FindGraphNodeWithOp(StringPiece op, const GraphDef& graph);
 // Gets the 0th input to a node in the graph.
 NodeDef* GetInputNode(const NodeDef& node, const MutableGraphView& graph);
 
+// Gets the ith input to a node in the graph.
+NodeDef* GetInputNode(const NodeDef& node, const MutableGraphView& graph,
+                      int64 i);
+
 // Returns the list of indices of all nodes with the given op or empty list if
 // no such node exists.
 std::vector<int> FindAllGraphNodesWithOp(const string& op,
diff --git a/tensorflow/core/grappler/optimizers/data/graph_utils_test.cc b/tensorflow/core/grappler/optimizers/data/graph_utils_test.cc
index 3b6d223fd3..879cecd13d 100644
--- a/tensorflow/core/grappler/optimizers/data/graph_utils_test.cc
+++ b/tensorflow/core/grappler/optimizers/data/graph_utils_test.cc
@@ -228,6 +228,21 @@ TEST(GraphUtilsTest, GetInputNode) {
   EXPECT_EQ(GetInputNode(*node1, graph), nullptr);
 }
 
+TEST(GraphUtilsTest, GetIthInputNode) {
+  GraphDef graph_def;
+  MutableGraphView graph(&graph_def);
+
+  NodeDef* node1 = AddNode("", "A", {}, {}, &graph);
+  NodeDef* node2 = AddNode("", "A", {}, {}, &graph);
+  NodeDef* node3 = AddNode("", "A", {node1->name(), node2->name()}, {}, &graph);
+
+  EXPECT_EQ(GetInputNode(*node3, graph), node1);
+  EXPECT_EQ(GetInputNode(*node3, graph, 1), node2);
+  EXPECT_EQ(GetInputNode(*node3, graph, 0), node1);
+  EXPECT_EQ(GetInputNode(*node3, graph, 2), nullptr);
+  EXPECT_EQ(GetInputNode(*node1, graph), nullptr);
+}
+
 TEST(GraphUtilsTest, EnsureNodeNamesUnique) {
   Graph g(OpRegistry::Global());
 
diff --git a/tensorflow/core/grappler/optimizers/data/rebatch.cc b/tensorflow/core/grappler/optimizers/data/rebatch.cc
new file mode 100644
index 0000000000..187e1a62af
--- /dev/null
+++ b/tensorflow/core/grappler/optimizers/data/rebatch.cc
@@ -0,0 +1,115 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/grappler/optimizers/data/rebatch.h"
+
+#include "tensorflow/core/grappler/grappler_item.h"
+#include "tensorflow/core/grappler/mutable_graph_view.h"
+#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.h"
+#include "tensorflow/core/grappler/optimizers/data/graph_utils.h"
+
+namespace tensorflow {
+namespace grappler {
+
+Status RebatchOptimizer::Init(
+    const tensorflow::RewriterConfig_CustomGraphOptimizer* config) {
+  if (!config) return Status::OK();
+
+  num_workers_ = config->parameter_map().at("num_workers").i();
+  return Status::OK();
+}
+
+namespace {
+
+constexpr char kCastOp[] = "Cast";
+constexpr char kRealDivOp[] = "RealDiv";
+constexpr char kBatchDatasetOp[] = "BatchDatasetV2";
+
+NodeDef* AddCastNode(const string& input, DataType src_t, DataType dst_t,
+                     MutableGraphView* graph) {
+  NodeDef cast_node;
+  cast_node.set_op(kCastOp);
+  cast_node.add_input(input);
+  graph_utils::SetUniqueGraphNodeName(cast_node.op(), graph->graph(),
+                                      &cast_node);
+  AddNodeAttr("SrcT", src_t, &cast_node);
+  AddNodeAttr("DstT", dst_t, &cast_node);
+
+  return graph->AddNode(std::move(cast_node));
+}
+
+NodeDef* AddBinaryNode(const string& input_x, const string& input_y,
+                       const string& op, DataType type,
+                       MutableGraphView* graph) {
+  NodeDef node;
+  node.set_op(op);
+  node.add_input(input_x);
+  node.add_input(input_y);
+  graph_utils::SetUniqueGraphNodeName(op, graph->graph(), &node);
+  AddNodeAttr("T", type, &node);
+
+  return graph->AddNode(std::move(node));
+}
+
+NodeDef* AddFloatDivNode(const string& input_x, const string& input_y,
+                         MutableGraphView* graph) {
+  return AddBinaryNode(input_x, input_y, kRealDivOp, DT_FLOAT, graph);
+}
+
+}  // anonymous namespace
+
+Status RebatchOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
+                                  GraphDef* output) {
+  *output = item.graph;
+  MutableGraphView graph(output);
+
+  absl::flat_hash_set<string> nodes_to_delete;
+  for (const NodeDef& node : item.graph.node()) {
+    if (node.op() == kBatchDatasetOp) {
+      NodeDef* batch_size_node = graph_utils::GetInputNode(node, graph, 1);
+      NodeDef tmp_node;
+      tmp_node = *batch_size_node;
+      graph_utils::SetUniqueGraphNodeName(tmp_node.op(), graph.graph(),
+                                          &tmp_node);
+      NodeDef* copy_batch_size_node = graph.AddNode(std::move(tmp_node));
+      NodeDef* float_copy_batch_size_node =
+          AddCastNode(copy_batch_size_node->name(), DT_INT64, DT_FLOAT, &graph);
+      NodeDef* num_worker_node =
+          graph_utils::AddScalarConstNode<int64>(num_workers_, &graph);
+      NodeDef* float_num_worker_node =
+          AddCastNode(num_worker_node->name(), DT_INT64, DT_FLOAT, &graph);
+      NodeDef* divided_batch_size_node =
+          AddFloatDivNode(float_copy_batch_size_node->name(),
+                          float_num_worker_node->name(), &graph);
+      NodeDef* cast_new_batch_size_node = AddCastNode(
+          divided_batch_size_node->name(), DT_FLOAT, DT_INT64, &graph);
+      TF_RETURN_IF_ERROR(graph.UpdateFanouts(batch_size_node->name(),
+                                             cast_new_batch_size_node->name()));
+      nodes_to_delete.insert(batch_size_node->name());
+      break;
+    }
+  }
+  TF_RETURN_IF_ERROR(graph.DeleteNodes(nodes_to_delete));
+  return Status::OK();
+}
+
+void RebatchOptimizer::Feedback(Cluster* cluster, const GrapplerItem& item,
+                                const GraphDef& optimize_output,
+                                double result) {}
+
+REGISTER_GRAPH_OPTIMIZER_AS(RebatchOptimizer, "tf_data_rebatcher");
+
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/rebatch.h b/tensorflow/core/grappler/optimizers/data/rebatch.h
new file mode 100644
index 0000000000..f7aa69f3ff
--- /dev/null
+++ b/tensorflow/core/grappler/optimizers/data/rebatch.h
@@ -0,0 +1,52 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_REBATCH_H_
+#define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_REBATCH_H_
+
+#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h"
+
+namespace tensorflow {
+namespace grappler {
+
+// This optimizer changes the batch size of the output dataset by dividing the
+// current batch size by parameter `num_workers`. Currently, this works only
+// for very simple pipelines with a single BatchDatasetV2 transformation.
+//
+// TODO(rohanj): Extend this logic to correctly handle any input pipeline that
+// uses core tf.data APIs + MapAndBatch.
+class RebatchOptimizer : public CustomGraphOptimizer {
+ public:
+  RebatchOptimizer() = default;
+  ~RebatchOptimizer() override = default;
+
+  string name() const override { return "tf_data_rebatcher"; }
+
+  Status Init(
+      const tensorflow::RewriterConfig_CustomGraphOptimizer* config) override;
+
+  Status Optimize(Cluster* cluster, const GrapplerItem& item,
+                  GraphDef* output) override;
+
+  void Feedback(Cluster* cluster, const GrapplerItem& item,
+                const GraphDef& optimize_output, double result) override;
+
+ private:
+  int64 num_workers_;
+};
+
+}  // namespace grappler
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_REBATCH_H_
diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD
index c1df9b5707..cd6803fb3f 100644
--- a/tensorflow/core/kernels/data/BUILD
+++ b/tensorflow/core/kernels/data/BUILD
@@ -548,17 +548,14 @@ tf_kernel_library(
     ],
 )
 
-tf_kernel_library(
-    name = "optimize_dataset_op",
-    srcs = ["optimize_dataset_op.cc"],
+cc_library(
+    name = "graph_rewrite_dataset",
+    srcs = ["graph_rewrite_dataset.cc"],
+    hdrs = ["graph_rewrite_dataset.h"],
     deps = [
         "//tensorflow/core:core_cpu_internal",
-        "//tensorflow/core:dataset_ops_op_lib",
         "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
-        "//tensorflow/core/grappler:graph_view",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:grappler_item_builder",
         "//tensorflow/core/grappler/clusters:virtual_cluster",
@@ -569,6 +566,19 @@ tf_kernel_library(
     ],
 )
 
+tf_kernel_library(
+    name = "optimize_dataset_op",
+    srcs = ["optimize_dataset_op.cc"],
+    deps = [
+        ":graph_rewrite_dataset",
+        "//tensorflow/core:core_cpu_internal",
+        "//tensorflow/core:dataset_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core:protos_all_cc",
+    ],
+)
+
 tf_kernel_library(
     name = "model_dataset_op",
     srcs = ["model_dataset_op.cc"],
diff --git a/tensorflow/core/kernels/data/experimental/BUILD b/tensorflow/core/kernels/data/experimental/BUILD
index 4f7c8f156c..9171b91a62 100644
--- a/tensorflow/core/kernels/data/experimental/BUILD
+++ b/tensorflow/core/kernels/data/experimental/BUILD
@@ -232,6 +232,21 @@ tf_kernel_library(
     ],
 )
 
+tf_kernel_library(
+    name = "rebatch_dataset_op",
+    srcs = ["rebatch_dataset_op.cc"],
+    deps = [
+        "//tensorflow/core:core_cpu_internal",
+        "//tensorflow/core:dataset_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core/grappler/optimizers/data:rebatch",
+        "//tensorflow/core/kernels/data:graph_rewrite_dataset",
+    ],
+)
+
 tf_kernel_library(
     name = "scan_dataset_op",
     srcs = ["scan_dataset_op.cc"],
@@ -391,6 +406,7 @@ tf_kernel_library(
         ":parse_example_dataset_op",
         ":prefetching_kernels",
         ":random_dataset_op",
+        ":rebatch_dataset_op",
         ":scan_dataset_op",
         ":set_stats_aggregator_dataset_op",
         ":sleep_dataset_op",
diff --git a/tensorflow/core/kernels/data/experimental/rebatch_dataset_op.cc b/tensorflow/core/kernels/data/experimental/rebatch_dataset_op.cc
new file mode 100644
index 0000000000..a95773afd1
--- /dev/null
+++ b/tensorflow/core/kernels/data/experimental/rebatch_dataset_op.cc
@@ -0,0 +1,92 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/framework/dataset.h"
+#include "tensorflow/core/kernels/data/graph_rewrite_dataset.h"
+
+namespace tensorflow {
+namespace data {
+namespace {
+
+constexpr char kOptimizerName[] = "tf_data_rebatcher";
+
+class RebatchDatasetOp : public UnaryDatasetOpKernel {
+ public:
+  explicit RebatchDatasetOp(OpKernelConstruction* ctx)
+      : UnaryDatasetOpKernel(ctx),
+        graph_def_version_(ctx->graph_def_version()) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("output_types", &output_types_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("output_shapes", &output_shapes_));
+  }
+
+ protected:
+  void MakeDataset(OpKernelContext* ctx, DatasetBase* input,
+                   DatasetBase** output) override {
+    int64 num_workers;
+    OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "num_workers", &num_workers));
+    OP_REQUIRES(ctx, num_workers > 0,
+                errors::InvalidArgument(
+                    "num_parallel_calls must be greater than zero."));
+
+    Dataset* dataset =
+        new Dataset(ctx, input, num_workers, output_types_, output_shapes_);
+    Status s = dataset->Optimize(ctx);
+    if (s.ok()) {
+      *output = dataset;
+    } else {
+      dataset->Unref();
+      OP_REQUIRES_OK(ctx, s);
+    }
+  }
+
+ private:
+  class Dataset : public GraphRewriteDataset {
+   public:
+    Dataset(OpKernelContext* ctx, const DatasetBase* input,
+            const int64 num_workers, const DataTypeVector& output_types,
+            const std::vector<PartialTensorShape>& output_shapes)
+        : GraphRewriteDataset(ctx, input, output_types, output_shapes),
+          num_workers_(num_workers) {}
+
+    string DebugString() const override { return "RebatchDatasetOp::Dataset"; }
+
+   private:
+    RewriterConfig CreateGrapplerRewriteConfig() override {
+      RewriterConfig rewriter_config;
+      rewriter_config.add_optimizers(kOptimizerName);
+      rewriter_config.set_meta_optimizer_iterations(
+          RewriterConfig_NumIterationsType_ONE);
+      auto custom_optimizer = rewriter_config.add_custom_optimizers();
+      custom_optimizer->set_name(kOptimizerName);
+      AttrValue num_workers_attr;
+      num_workers_attr.set_i(num_workers_);
+      (*custom_optimizer->mutable_parameter_map())["num_workers"] =
+          num_workers_attr;
+      return rewriter_config;
+    }
+
+    const int64 num_workers_;
+  };
+
+  const int graph_def_version_;
+  DataTypeVector output_types_;
+  std::vector<PartialTensorShape> output_shapes_;
+};
+
+REGISTER_KERNEL_BUILDER(Name("ExperimentalRebatchDataset").Device(DEVICE_CPU),
+                        RebatchDatasetOp);
+
+}  // anonymous namespace
+}  // namespace data
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/data/graph_rewrite_dataset.cc b/tensorflow/core/kernels/data/graph_rewrite_dataset.cc
new file mode 100644
index 0000000000..bc4bb46010
--- /dev/null
+++ b/tensorflow/core/kernels/data/graph_rewrite_dataset.cc
@@ -0,0 +1,239 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/kernels/data/graph_rewrite_dataset.h"
+
+#include "tensorflow/core/protobuf/meta_graph.pb.h"
+#include "tensorflow/core/protobuf/rewriter_config.pb.h"
+
+namespace tensorflow {
+namespace data {
+
+GraphRewriteDataset::~GraphRewriteDataset() {
+  input_->Unref();
+  if (optimized_input_) {
+    optimized_input_->Unref();
+  }
+}
+
+Status GraphRewriteDataset::Optimize(OpKernelContext* ctx) {
+  GraphDefBuilder b;
+  DatasetGraphDefBuilder db(&b);
+  Node* input_node = nullptr;
+  SerializationContext::Params params;
+  std::vector<std::pair<string, Tensor>> input_list;
+  params.flib_def = ctx->function_library()->GetFunctionLibraryDefinition();
+  params.input_list = &input_list;
+  params.optimization_only = true;
+  SerializationContext serialization_ctx(params);
+  TF_RETURN_IF_ERROR(
+      db.AddInputDataset(&serialization_ctx, input_, &input_node));
+  string output_node = input_node->name();
+
+  GraphDef graph_def;
+  TF_RETURN_IF_ERROR(b.ToGraphDef(&graph_def));
+  VLOG(3) << "Before optimization: " << graph_def.DebugString();
+
+  TF_RETURN_IF_ERROR(ApplyOptimizations(ctx, &graph_def, &output_node));
+  VLOG(3) << "After optimization: " << graph_def.DebugString();
+
+  // Instantiate the optimized input pipeline by running the optimized graph
+  // using the optimized function library.
+  TF_RETURN_IF_ERROR(ctx->function_library()->Clone(&flib_def_, &pflr_, &lib_));
+
+  // Create a FunctionHandleCache.
+  function_handle_cache_ = absl::make_unique<FunctionHandleCache>(lib_);
+
+  // Some functions may have been modified without having their names
+  // changed (for example, nested dataset graphs from FlatMap or
+  // Interleave). To avoid name conflicts, we remove these functions from
+  // flib_def_ before adding the optimized function library.
+  for (const FunctionDef& fd : graph_def.library().function()) {
+    if (flib_def_->Find(fd.signature().name()) != nullptr) {
+      TF_RETURN_IF_ERROR(flib_def_->RemoveFunction(fd.signature().name()));
+    }
+  }
+  TF_RETURN_IF_ERROR(flib_def_->AddLibrary(graph_def.library()));
+
+  Graph graph(OpRegistry::Global());
+  TF_RETURN_IF_ERROR(ImportGraphDef({}, graph_def, &graph, nullptr));
+  std::vector<Tensor> outputs;
+  GraphRunner graph_runner(ctx->function_library()->device());
+
+  TF_RETURN_IF_ERROR(
+      graph_runner.Run(&graph, lib_, input_list, {output_node}, &outputs));
+  TF_RETURN_IF_ERROR(
+      GetDatasetFromVariantTensor(outputs[0], &optimized_input_));
+  optimized_input_->Ref();
+  return Status::OK();
+}
+
+Status GraphRewriteDataset::AsGraphDefInternal(SerializationContext* ctx,
+                                               DatasetGraphDefBuilder* b,
+                                               Node** output) const {
+  // We only serialize the optimized dataset to avoid re-running
+  // optimizations when the input pipeline is restored from a checkpoint.
+  TF_RETURN_IF_ERROR(b->AddInputDataset(ctx, optimized_input_, output));
+  return Status::OK();
+}
+
+namespace {
+void AddFakeSinks(FunctionDef* function_def) {
+  int counter = 0;
+  for (const auto& output : function_def->signature().output_arg()) {
+    NodeDef* node = function_def->add_node_def();
+    tensorflow::grappler::function_utils::SetUniqueFunctionNodeName(
+        strings::StrCat("FakeSink", counter++), function_def, node);
+    node->set_op("Identity");
+    node->add_input(function_def->ret().at(output.name()));
+    (*node->mutable_attr())["T"].set_type(output.type());
+
+    (*function_def->mutable_ret())[output.name()] =
+        strings::StrCat(node->name(), ":output:0");
+  }
+}
+
+void RemoveFakeSinks(FunctionDef* function_def) {
+  // Map from identity node names to their input tensor strings
+  std::map<string, string> identity_map;
+  for (const auto& node : function_def->node_def()) {
+    if (node.op() == "Identity" && node.input_size() == 1) {
+      identity_map[node.name()] = node.input(0);
+    }
+  }
+  for (const auto& output_arg : function_def->signature().output_arg()) {
+    const string& tensor = function_def->ret().at(output_arg.name());
+    const string& output_node = tensor.substr(0, tensor.find(':'));
+    if (identity_map.find(output_node) != identity_map.end()) {
+      (*function_def->mutable_ret())[output_arg.name()] =
+          identity_map.at(output_node);
+    }
+  }
+}
+}  // anonymous namespace
+
+Status GraphRewriteDataset::ApplyOptimizations(OpKernelContext* ctx,
+                                               GraphDef* graph_def,
+                                               string* output_node) {
+  // Add an identity node as the fetch node, otherwise we might get
+  // 'placeholder is both fed and fetched' errors in some cases when using
+  // input list with placeholder dataset nodes.
+  NodeDef* node = graph_def->mutable_node()->Add();
+  tensorflow::grappler::graph_utils::SetUniqueGraphNodeName("Sink", graph_def,
+                                                            node);
+  node->set_op("Identity");
+  node->add_input(*output_node);
+  (*node->mutable_attr())["T"].set_type(DT_VARIANT);
+  *output_node = node->name();
+
+  // Add fake sink node to graph and functions to allow rewriting the actual
+  // sink nodes.
+  // TODO(b/118820916): When MetaOptimizer adds provisions for function
+  // retvals to be optimizable, we will no longer need this.
+  for (auto& function_def : *graph_def->mutable_library()->mutable_function()) {
+    AddFakeSinks(&function_def);
+  }
+
+  // Create metagraph.
+  MetaGraphDef meta_graph_def;
+  (*meta_graph_def.mutable_graph_def()) = *graph_def;
+
+  // Grappler determines fetch ops from collection 'train_op'.
+  CollectionDef collection_def;
+  auto node_list = collection_def.mutable_node_list();
+  node_list->add_value(*output_node);
+  (*meta_graph_def.mutable_collection_def())["train_op"] = collection_def;
+
+  // Create Grappler item.
+  tensorflow::grappler::ItemConfig item_config;
+  item_config.apply_optimizations = true;
+  std::unique_ptr<tensorflow::grappler::GrapplerItem> grappler_item =
+      tensorflow::grappler::GrapplerItemFromMetaGraphDef(
+          "graph", meta_graph_def, item_config);
+  std::unordered_map<string, tensorflow::DeviceProperties> device_map;
+  tensorflow::grappler::VirtualCluster cluster(device_map);
+
+  // Run data optimizer using grappler's meta optimizer.
+  tensorflow::ConfigProto config;
+  *config.mutable_graph_options()->mutable_rewrite_options() =
+      CreateGrapplerRewriteConfig();
+  TF_RETURN_IF_ERROR(tensorflow::grappler::RunMetaOptimizer(
+      *grappler_item, config, ctx->device(), &cluster, graph_def));
+
+  // Remove fake sinks after optimizations are done.
+  // TODO(b/118820916): When MetaOptimizer adds provisions for function
+  // retvals to be optimizable, we will no longer need this.
+  for (auto& function_def : *graph_def->mutable_library()->mutable_function()) {
+    RemoveFakeSinks(&function_def);
+  }
+
+  return Status::OK();
+}
+
+class GraphRewriteDataset::Iterator
+    : public DatasetIterator<GraphRewriteDataset> {
+ public:
+  explicit Iterator(const Params& params)
+      : DatasetIterator<GraphRewriteDataset>(params) {}
+
+  Status Initialize(IteratorContext* ctx) override {
+    IteratorContext::Params params(ctx);
+    params.lib = dataset()->lib_;
+    params.function_handle_cache = dataset()->function_handle_cache_.get();
+    return dataset()->optimized_input_->MakeIterator(
+        IteratorContext(std::move(params)), prefix(), &input_impl_);
+  }
+
+  Status GetNextInternal(IteratorContext* ctx, std::vector<Tensor>* out_tensors,
+                         bool* end_of_sequence) override {
+    IteratorContext::Params params(ctx);
+    params.lib = dataset()->lib_;
+    params.function_handle_cache = dataset()->function_handle_cache_.get();
+    return input_impl_->GetNext(IteratorContext(std::move(params)), out_tensors,
+                                end_of_sequence);
+  }
+
+ protected:
+  std::shared_ptr<model::Node> CreateNode(
+      IteratorContext* ctx, model::Node::Args args) const override {
+    return model::MakeKnownRatioNode(std::move(args),
+                                     /*ratio=*/1);
+  }
+
+  Status SaveInternal(IteratorStateWriter* writer) override {
+    TF_RETURN_IF_ERROR(SaveInput(writer, input_impl_));
+    return Status::OK();
+  }
+
+  Status RestoreInternal(IteratorContext* ctx,
+                         IteratorStateReader* reader) override {
+    TF_RETURN_IF_ERROR(RestoreInput(ctx, reader, input_impl_));
+    return Status::OK();
+  }
+
+ private:
+  std::unique_ptr<IteratorBase> input_impl_;
+};
+
+std::unique_ptr<IteratorBase> GraphRewriteDataset::MakeIteratorInternal(
+    const string& prefix) const {
+  // We do not add a token for this dataset to the prefix. The
+  // prefix is used to identify checkpoint elements and since this
+  // dataset is excluded from the checkpoint, adding a token
+  // here would result in invalid checkpoint identifiers.
+  return absl::make_unique<Iterator>(Iterator::Params{this, prefix});
+}
+
+}  // namespace data
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/data/graph_rewrite_dataset.h b/tensorflow/core/kernels/data/graph_rewrite_dataset.h
new file mode 100644
index 0000000000..dedbdce71f
--- /dev/null
+++ b/tensorflow/core/kernels/data/graph_rewrite_dataset.h
@@ -0,0 +1,92 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_KERNELS_DATA_GRAPH_REWRITE_DATASET_H_
+#define TENSORFLOW_CORE_KERNELS_DATA_GRAPH_REWRITE_DATASET_H_
+
+#include "tensorflow/core/common_runtime/graph_runner.h"
+#include "tensorflow/core/common_runtime/process_function_library_runtime.h"
+#include "tensorflow/core/framework/dataset.h"
+#include "tensorflow/core/framework/function_handle_cache.h"
+#include "tensorflow/core/graph/graph_constructor.h"
+#include "tensorflow/core/graph/graph_def_builder.h"
+#include "tensorflow/core/grappler/clusters/virtual_cluster.h"
+#include "tensorflow/core/grappler/grappler_item.h"
+#include "tensorflow/core/grappler/grappler_item_builder.h"
+#include "tensorflow/core/grappler/optimizers/data/function_utils.h"
+#include "tensorflow/core/grappler/optimizers/data/graph_utils.h"
+#include "tensorflow/core/grappler/optimizers/meta_optimizer.h"
+
+namespace tensorflow {
+namespace data {
+
+class GraphRewriteDataset : public DatasetBase {
+ public:
+  GraphRewriteDataset(OpKernelContext* ctx, const DatasetBase* input,
+                      const DataTypeVector& output_types,
+                      const std::vector<PartialTensorShape>& output_shapes)
+      : DatasetBase(DatasetContext(ctx)),
+        optimized_input_(nullptr),
+        input_(input),
+        output_types_(output_types),
+        output_shapes_(output_shapes) {
+    input_->Ref();
+  }
+
+  ~GraphRewriteDataset() override;
+
+  // Runs Grappler to transform the input dataset into optimized_input_
+  // dataset.
+  Status Optimize(OpKernelContext* ctx);
+
+  std::unique_ptr<IteratorBase> MakeIteratorInternal(
+      const string& prefix) const override;
+
+  const DataTypeVector& output_dtypes() const override { return output_types_; }
+
+  const std::vector<PartialTensorShape>& output_shapes() const override {
+    return output_shapes_;
+  }
+
+  int64 Cardinality() const override { return input_->Cardinality(); }
+
+ protected:
+  Status AsGraphDefInternal(SerializationContext* ctx,
+                            DatasetGraphDefBuilder* b,
+                            Node** output) const override;
+
+ private:
+  class Iterator;
+
+  // Create a Grappler RewriteConfig proto that defines the list of
+  // optimizations to be run by the Grappler Meta Optimizer.
+  virtual RewriterConfig CreateGrapplerRewriteConfig() = 0;
+
+  Status ApplyOptimizations(OpKernelContext* ctx, GraphDef* graph_def,
+                            string* output_node);
+
+  DatasetBase* optimized_input_;
+  FunctionLibraryRuntime* lib_ = nullptr;
+  std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_ = nullptr;
+  std::unique_ptr<FunctionLibraryDefinition> flib_def_ = nullptr;
+  std::unique_ptr<FunctionHandleCache> function_handle_cache_ = nullptr;
+  const DatasetBase* input_;
+  const DataTypeVector output_types_;
+  const std::vector<PartialTensorShape> output_shapes_;
+};
+
+}  // namespace data
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_KERNELS_DATA_GRAPH_REWRITE_DATASET_H_
diff --git a/tensorflow/core/kernels/data/optimize_dataset_op.cc b/tensorflow/core/kernels/data/optimize_dataset_op.cc
index 6047dc5f3f..17094e3001 100644
--- a/tensorflow/core/kernels/data/optimize_dataset_op.cc
+++ b/tensorflow/core/kernels/data/optimize_dataset_op.cc
@@ -14,26 +14,11 @@ limitations under the License.
 ==============================================================================*/
 #include <map>
 
-#include "tensorflow/core/common_runtime/device_mgr.h"
-#include "tensorflow/core/common_runtime/graph_runner.h"
-#include "tensorflow/core/common_runtime/process_function_library_runtime.h"
 #include "tensorflow/core/framework/dataset.h"
-#include "tensorflow/core/framework/device_base.h"
-#include "tensorflow/core/framework/function_handle_cache.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/graph/graph_constructor.h"
-#include "tensorflow/core/graph/graph_def_builder.h"
-#include "tensorflow/core/grappler/clusters/virtual_cluster.h"
-#include "tensorflow/core/grappler/graph_view.h"
-#include "tensorflow/core/grappler/grappler_item.h"
-#include "tensorflow/core/grappler/grappler_item_builder.h"
-#include "tensorflow/core/grappler/optimizers/data/function_utils.h"
-#include "tensorflow/core/grappler/optimizers/data/graph_utils.h"
-#include "tensorflow/core/grappler/optimizers/meta_optimizer.h"
-#include "tensorflow/core/lib/core/refcount.h"
+#include "tensorflow/core/kernels/data/graph_rewrite_dataset.h"
 #include "tensorflow/core/lib/random/random.h"
-#include "tensorflow/core/protobuf/meta_graph.pb.h"
 #include "tensorflow/core/protobuf/rewriter_config.pb.h"
 
 namespace tensorflow {
@@ -71,235 +56,20 @@ class OptimizeDatasetOp : public UnaryDatasetOpKernel {
   }
 
  private:
-  class Dataset : public DatasetBase {
+  class Dataset : public GraphRewriteDataset {
    public:
     Dataset(OpKernelContext* ctx, const DatasetBase* input,
             const std::vector<string>& optimizations,
             const DataTypeVector& output_types,
             const std::vector<PartialTensorShape>& output_shapes)
-        : DatasetBase(DatasetContext(ctx)),
-          optimized_input_(nullptr),
-          input_(input),
-          optimizations_(optimizations),
-          output_types_(output_types),
-          output_shapes_(output_shapes) {
-      input_->Ref();
-    }
-
-    ~Dataset() override {
-      input_->Unref();
-      if (optimized_input_) {
-        optimized_input_->Unref();
-      }
-    }
-
-    std::unique_ptr<IteratorBase> MakeIteratorInternal(
-        const string& prefix) const override {
-      // We do not add a token for the optimization dataset to the prefix. The
-      // prefix is used to identify checkpoint elements and since the
-      // optimization dataset is excluded from the checkpoint, adding a token
-      // here would result in invalid checkpoint identifiers.
-      return absl::make_unique<Iterator>(Iterator::Params{this, prefix});
-    }
-
-    Status Optimize(OpKernelContext* ctx) {
-      GraphDefBuilder b;
-      DatasetGraphDefBuilder db(&b);
-      Node* input_node = nullptr;
-      SerializationContext::Params params;
-      std::vector<std::pair<string, Tensor>> input_list;
-      params.flib_def = ctx->function_library()->GetFunctionLibraryDefinition();
-      params.input_list = &input_list;
-      params.optimization_only = true;
-      SerializationContext serialization_ctx(params);
-      TF_RETURN_IF_ERROR(
-          db.AddInputDataset(&serialization_ctx, input_, &input_node));
-      string output_node = input_node->name();
-
-      GraphDef graph_def;
-      TF_RETURN_IF_ERROR(b.ToGraphDef(&graph_def));
-      VLOG(3) << "Before optimization: " << graph_def.DebugString();
-
-      TF_RETURN_IF_ERROR(ApplyOptimizations(ctx, &graph_def, &output_node));
-      VLOG(3) << "After optimization: " << graph_def.DebugString();
-
-      // Instantiate the optimized input pipeline by running the optimized graph
-      // using the optimized function library.
-      TF_RETURN_IF_ERROR(
-          ctx->function_library()->Clone(&flib_def_, &pflr_, &lib_));
-
-      // Create a FunctionHandleCache.
-      function_handle_cache_ = absl::make_unique<FunctionHandleCache>(lib_);
-
-      // Some functions may have been modified without having their names
-      // changed (for example, nested dataset graphs from FlatMap or
-      // Interleave). To avoid name conflicts, we remove these functions from
-      // flib_def_ before adding the optimized function library.
-      for (const FunctionDef& fd : graph_def.library().function()) {
-        if (flib_def_->Find(fd.signature().name()) != nullptr) {
-          TF_RETURN_IF_ERROR(flib_def_->RemoveFunction(fd.signature().name()));
-        }
-      }
-      TF_RETURN_IF_ERROR(flib_def_->AddLibrary(graph_def.library()));
-
-      Graph graph(OpRegistry::Global());
-      TF_RETURN_IF_ERROR(ImportGraphDef({}, graph_def, &graph, nullptr));
-      std::vector<Tensor> outputs;
-      GraphRunner graph_runner(ctx->function_library()->device());
-
-      TF_RETURN_IF_ERROR(
-          graph_runner.Run(&graph, lib_, input_list, {output_node}, &outputs));
-      TF_RETURN_IF_ERROR(
-          GetDatasetFromVariantTensor(outputs[0], &optimized_input_));
-      optimized_input_->Ref();
-      return Status::OK();
-    }
-
-    const DataTypeVector& output_dtypes() const override {
-      return output_types_;
-    }
-    const std::vector<PartialTensorShape>& output_shapes() const override {
-      return output_shapes_;
-    }
+        : GraphRewriteDataset(ctx, input, output_types, output_shapes),
+          optimizations_(optimizations) {}
 
     string DebugString() const override { return "OptimizeDatasetOp::Dataset"; }
 
-    int64 Cardinality() const override { return input_->Cardinality(); }
-
-   protected:
-    Status AsGraphDefInternal(SerializationContext* ctx,
-                              DatasetGraphDefBuilder* b,
-                              Node** output) const override {
-      // We only serialize the optimized dataset to avoid re-running
-      // optimizations when the input pipeline is restored from a checkpoint.
-      TF_RETURN_IF_ERROR(b->AddInputDataset(ctx, optimized_input_, output));
-      return Status::OK();
-    }
-
    private:
-    class Iterator : public DatasetIterator<Dataset> {
-     public:
-      explicit Iterator(const Params& params)
-          : DatasetIterator<Dataset>(params) {}
-
-      Status Initialize(IteratorContext* ctx) override {
-        IteratorContext::Params params(ctx);
-        params.lib = dataset()->lib_;
-        params.function_handle_cache = dataset()->function_handle_cache_.get();
-        return dataset()->optimized_input_->MakeIterator(
-            IteratorContext(std::move(params)), prefix(), &input_impl_);
-      }
-
-      Status GetNextInternal(IteratorContext* ctx,
-                             std::vector<Tensor>* out_tensors,
-                             bool* end_of_sequence) override {
-        IteratorContext::Params params(ctx);
-        params.lib = dataset()->lib_;
-        params.function_handle_cache = dataset()->function_handle_cache_.get();
-        return input_impl_->GetNext(IteratorContext(std::move(params)),
-                                    out_tensors, end_of_sequence);
-      }
-
-     protected:
-      std::shared_ptr<model::Node> CreateNode(
-          IteratorContext* ctx, model::Node::Args args) const override {
-        return model::MakeKnownRatioNode(std::move(args),
-                                         /*ratio=*/1);
-      }
-
-      Status SaveInternal(IteratorStateWriter* writer) override {
-        TF_RETURN_IF_ERROR(SaveInput(writer, input_impl_));
-        return Status::OK();
-      }
-
-      Status RestoreInternal(IteratorContext* ctx,
-                             IteratorStateReader* reader) override {
-        TF_RETURN_IF_ERROR(RestoreInput(ctx, reader, input_impl_));
-        return Status::OK();
-      }
-
-     private:
-      std::unique_ptr<IteratorBase> input_impl_;
-    };
-
-    void AddFakeSinks(FunctionDef* function_def) {
-      int counter = 0;
-      for (const auto& output : function_def->signature().output_arg()) {
-        NodeDef* node = function_def->add_node_def();
-        tensorflow::grappler::function_utils::SetUniqueFunctionNodeName(
-            strings::StrCat("FakeSink", counter++), function_def, node);
-        node->set_op("Identity");
-        node->add_input(function_def->ret().at(output.name()));
-        (*node->mutable_attr())["T"].set_type(output.type());
-
-        (*function_def->mutable_ret())[output.name()] =
-            strings::StrCat(node->name(), ":output:0");
-      }
-    }
-
-    void RemoveFakeSinks(FunctionDef* function_def) {
-      // Map from identity node names to their input tensor strings
-      std::map<string, string> identity_map;
-      for (const auto& node : function_def->node_def()) {
-        if (node.op() == "Identity" && node.input_size() == 1) {
-          identity_map[node.name()] = node.input(0);
-        }
-      }
-      for (const auto& output_arg : function_def->signature().output_arg()) {
-        const string& tensor = function_def->ret().at(output_arg.name());
-        const string& output_node = tensor.substr(0, tensor.find(':'));
-        if (identity_map.find(output_node) != identity_map.end()) {
-          (*function_def->mutable_ret())[output_arg.name()] =
-              identity_map.at(output_node);
-        }
-      }
-    }
-
-    Status ApplyOptimizations(OpKernelContext* ctx, GraphDef* graph_def,
-                              string* output_node) {
-      // Add an identity node as the fetch node, otherwise we might get
-      // 'placeholder is both fed and fetched' errors in some cases when using
-      // input list with placeholder dataset nodes.
-      NodeDef* node = graph_def->mutable_node()->Add();
-      tensorflow::grappler::graph_utils::SetUniqueGraphNodeName(
-          "Sink", graph_def, node);
-      node->set_op("Identity");
-      node->add_input(*output_node);
-      (*node->mutable_attr())["T"].set_type(DT_VARIANT);
-      *output_node = node->name();
-
-      // Add fake sink node to graph and functions to allow rewriting the actual
-      // sink nodes.
-      // TODO(b/118820916): When MetaOptimizer adds provisions for function
-      // retvals to be optimizable, we will no longer need this.
-      for (auto& function_def :
-           *graph_def->mutable_library()->mutable_function()) {
-        AddFakeSinks(&function_def);
-      }
-
-      // Create metagraph.
-      MetaGraphDef meta_graph_def;
-      (*meta_graph_def.mutable_graph_def()) = *graph_def;
-
-      // Grappler determines fetch ops from collection 'train_op'.
-      CollectionDef collection_def;
-      auto node_list = collection_def.mutable_node_list();
-      node_list->add_value(*output_node);
-      (*meta_graph_def.mutable_collection_def())["train_op"] = collection_def;
-
-      // Create Grappler item.
-      tensorflow::grappler::ItemConfig item_config;
-      item_config.apply_optimizations = true;
-      std::unique_ptr<tensorflow::grappler::GrapplerItem> grappler_item =
-          tensorflow::grappler::GrapplerItemFromMetaGraphDef(
-              "graph", meta_graph_def, item_config);
-      std::unordered_map<string, tensorflow::DeviceProperties> device_map;
-      tensorflow::grappler::VirtualCluster cluster(device_map);
-
-      // Run data optimizer using grappler's meta optimizer.
-      tensorflow::ConfigProto config;
-      RewriterConfig& rewriter_config =
-          *config.mutable_graph_options()->mutable_rewrite_options();
+    RewriterConfig CreateGrapplerRewriteConfig() override {
+      RewriterConfig rewriter_config;
       rewriter_config.add_optimizers(kOptimizerName);
       rewriter_config.set_meta_optimizer_iterations(
           RewriterConfig_NumIterationsType_ONE);
@@ -311,30 +81,10 @@ class OptimizeDatasetOp : public UnaryDatasetOpKernel {
       for (const auto& opt : optimizations_) {
         custom_optimizations_list->add_s(opt);
       }
-
-      TF_RETURN_IF_ERROR(tensorflow::grappler::RunMetaOptimizer(
-          *grappler_item, config, ctx->device(), &cluster, graph_def));
-
-      // Remove fake sinks after optimizations are done.
-      // TODO(b/118820916): When MetaOptimizer adds provisions for function
-      // retvals to be optimizable, we will no longer need this.
-      for (auto& function_def :
-           *graph_def->mutable_library()->mutable_function()) {
-        RemoveFakeSinks(&function_def);
-      }
-
-      return Status::OK();
+      return rewriter_config;
     }
 
-    DatasetBase* optimized_input_;
-    FunctionLibraryRuntime* lib_ = nullptr;
-    std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_ = nullptr;
-    std::unique_ptr<FunctionLibraryDefinition> flib_def_ = nullptr;
-    std::unique_ptr<FunctionHandleCache> function_handle_cache_ = nullptr;
-    const DatasetBase* input_;
     const std::vector<string> optimizations_;
-    const DataTypeVector output_types_;
-    const std::vector<PartialTensorShape> output_shapes_;
   };
 
   const int graph_def_version_;
diff --git a/tensorflow/core/ops/experimental_dataset_ops.cc b/tensorflow/core/ops/experimental_dataset_ops.cc
index 316e405188..41348972d3 100644
--- a/tensorflow/core/ops/experimental_dataset_ops.cc
+++ b/tensorflow/core/ops/experimental_dataset_ops.cc
@@ -190,6 +190,14 @@ REGISTER_OP("ExperimentalMapAndBatchDataset")
       return shape_inference::ScalarShape(c);
     });
 
+REGISTER_OP("ExperimentalRebatchDataset")
+    .Input("input_dataset: variant")
+    .Input("num_workers: int64")
+    .Output("handle: variant")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")
+    .SetShapeFn(shape_inference::ScalarShape);
+
 REGISTER_OP("ExperimentalMapDataset")
     .Input("input_dataset: variant")
     .Input("other_arguments: Targuments")
diff --git a/tensorflow/python/data/experimental/kernel_tests/BUILD b/tensorflow/python/data/experimental/kernel_tests/BUILD
index 0481913064..d0e5abcd5b 100644
--- a/tensorflow/python/data/experimental/kernel_tests/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/BUILD
@@ -470,6 +470,20 @@ py_library(
     ],
 )
 
+py_test(
+    name = "rebatch_dataset_test",
+    size = "small",
+    srcs = ["rebatch_dataset_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python/data/experimental/ops:batching",
+        "//tensorflow/python/data/kernel_tests:test_base",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/data/util:nest",
+    ],
+)
+
 py_test(
     name = "rejection_resample_test",
     size = "medium",
diff --git a/tensorflow/python/data/experimental/kernel_tests/rebatch_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/rebatch_dataset_test.py
new file mode 100644
index 0000000000..0dcbd563fa
--- /dev/null
+++ b/tensorflow/python/data/experimental/kernel_tests/rebatch_dataset_test.py
@@ -0,0 +1,60 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for the private `_RebatchDataset` transformation."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.data.experimental.ops import batching
+from tensorflow.python.data.kernel_tests import test_base
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.util import nest
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import test
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class RebatchDatasetTest(test_base.DatasetTestBase):
+
+  def testBasic(self):
+    dataset = dataset_ops.Dataset.range(1024).batch(32, drop_remainder=True)
+    rebatched_dataset = batching._RebatchDataset(dataset, num_workers=4)
+    self.assertEqual(
+        [[32]], [ts.as_list() for ts in nest.flatten(dataset.output_shapes)])
+    self.assertEqual(
+        [[8]],
+        [ts.as_list() for ts in nest.flatten(rebatched_dataset.output_shapes)])
+
+    expected_output = [[k for k in range(i, i + 8)] for i in range(0, 1024, 8)]  # pylint: disable=g-complex-comprehension
+    self.assertDatasetProduces(rebatched_dataset, expected_output)
+
+  def testScalarInputError(self):
+    dataset = dataset_ops.Dataset.range(1024)
+    with self.assertRaisesRegexp(ValueError, "at least one dimension"):
+      batching._RebatchDataset(dataset, num_workers=4)
+
+  def testUnknownBatchSizeError(self):
+    dataset = dataset_ops.Dataset.range(1024).batch(32)
+    with self.assertRaisesRegexp(ValueError, "unknown batch size datasets"):
+      batching._RebatchDataset(dataset, num_workers=4)
+
+  def testNotDivisibleError(self):
+    dataset = dataset_ops.Dataset.range(1024).batch(32, drop_remainder=True)
+    with self.assertRaisesRegexp(ValueError, "not divisible by"):
+      batching._RebatchDataset(dataset, num_workers=5)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD b/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
index 4fd2a2ec4b..dc168d867a 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
@@ -408,6 +408,24 @@ py_test(
     ],
 )
 
+py_test(
+    name = "rebatch_dataset_serialization_test",
+    size = "small",
+    srcs = ["rebatch_dataset_serialization_test.py"],
+    srcs_version = "PY2AND3",
+    tags = [
+        "no_oss",
+        "no_pip",
+        "no_windows",
+    ],
+    deps = [
+        ":dataset_serialization_test_base",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python/data/experimental/ops:batching",
+        "//tensorflow/python/data/ops:dataset_ops",
+    ],
+)
+
 py_test(
     name = "padded_batch_dataset_serialization_test",
     size = "medium",
diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/rebatch_dataset_serialization_test.py b/tensorflow/python/data/experimental/kernel_tests/serialization/rebatch_dataset_serialization_test.py
new file mode 100644
index 0000000000..b30db58906
--- /dev/null
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/rebatch_dataset_serialization_test.py
@@ -0,0 +1,41 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for the _RebatchDataset serialization."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.data.experimental.kernel_tests.serialization import dataset_serialization_test_base
+from tensorflow.python.data.experimental.ops import batching
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.platform import test
+
+
+class RebatchDatasetSerializationTest(
+    dataset_serialization_test_base.DatasetSerializationTestBase):
+
+  def testCore(self):
+
+    def build_dataset(num_elements, batch_size):
+      return batching._RebatchDataset(
+          dataset_ops.Dataset.range(num_elements).batch(
+              4 * batch_size, drop_remainder=True),
+          num_workers=4)
+
+    self.run_core_tests(lambda: build_dataset(200, 10), None, 20)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/experimental/ops/batching.py b/tensorflow/python/data/experimental/ops/batching.py
index f0cf7f0a99..39cb0a68f8 100644
--- a/tensorflow/python/data/experimental/ops/batching.py
+++ b/tensorflow/python/data/experimental/ops/batching.py
@@ -645,3 +645,34 @@ def map_and_batch(map_func,
                                num_parallel_calls, drop_remainder)
 
   return _apply_fn
+
+
+class _RebatchDataset(dataset_ops.UnaryDataset):
+  """A `Dataset` that divides the batch size by `num_workers`."""
+
+  def __init__(self, input_dataset, num_workers):
+    self._input_dataset = input_dataset
+    output_shapes = input_dataset.output_shapes
+    if len(output_shapes) < 1:
+      raise ValueError("Input shape should have at least one dimension.")
+    if not output_shapes.dims[0].value:
+      raise ValueError("Cannot rebatch unknown batch size datasets.")
+    if output_shapes.dims[0].value % num_workers != 0:
+      raise ValueError(
+          "First dim of input shape: %d is not divisible by num_workers: %d" %
+          (output_shapes[0], num_workers))
+    output_dims = [d for d in output_shapes.dims]
+    output_dims[0] = output_dims[0] // num_workers
+    output_shapes = tensor_shape.TensorShapeV1(output_dims)
+    self._structure = structure.convert_legacy_structure(
+        self._input_dataset.output_types, output_shapes,
+        self._input_dataset.output_classes)
+    variant_tensor = ged_ops.experimental_rebatch_dataset(
+        self._input_dataset._variant_tensor,  # pylint: disable=protected-access
+        num_workers=num_workers,
+        **dataset_ops.flat_structure(self))
+    super(_RebatchDataset, self).__init__(input_dataset, variant_tensor)
+
+  @property
+  def _element_structure(self):
+    return self._structure
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt
index e7375af182..0b2a15f521 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt
@@ -1096,6 +1096,10 @@ tf_module {
     name: "ExperimentalRandomDataset"
     argspec: "args=[\'seed\', \'seed2\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "ExperimentalRebatchDataset"
+    argspec: "args=[\'input_dataset\', \'num_workers\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "ExperimentalScanDataset"
     argspec: "args=[\'input_dataset\', \'initial_state\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt
index e7375af182..0b2a15f521 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt
@@ -1096,6 +1096,10 @@ tf_module {
     name: "ExperimentalRandomDataset"
     argspec: "args=[\'seed\', \'seed2\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "ExperimentalRebatchDataset"
+    argspec: "args=[\'input_dataset\', \'num_workers\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "ExperimentalScanDataset"
     argspec: "args=[\'input_dataset\', \'initial_state\', \'other_arguments\', \'f\', \'output_types\', \'output_shapes\', \'preserve_cardinality\'], varargs=None, keywords=None, defaults=None"
-- 
GitLab


From 1501b31d25885b64ffc99df44783f549c7cde2ef Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Fri, 1 Feb 2019 09:27:02 -0800
Subject: [PATCH 1759/2345] Don't convert batch_gather to gather with
 batch_dims=-1 as they're not equivalent.

PiperOrigin-RevId: 231983764
---
 tensorflow/python/ops/array_ops.py            |  2 +-
 tensorflow/tools/compatibility/reorders_v2.py | 37 +++++++++----------
 .../tools/compatibility/tf_upgrade_v2.py      | 20 ----------
 .../tools/compatibility/tf_upgrade_v2_test.py | 13 -------
 4 files changed, 19 insertions(+), 53 deletions(-)

diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 89d6de6dff..230ce62db7 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -3375,7 +3375,7 @@ gather.__doc__ = gather_v2.__doc__ = gen_array_ops.gather_v2.__doc__
 @dispatch.add_dispatch_support
 @deprecation.deprecated(
     "2017-10-25", "`tf.batch_gather` is deprecated, please use `tf.gather` "
-    "with `batch_dims=-1` instead.")  # pylint: disable=missing-docstring
+    "with `batch_dims` instead.")  # pylint: disable=missing-docstring
 def batch_gather(params, indices, name=None):
   """Gather slices from params according to indices with leading batch dims."""
   with ops.name_scope(name, "BatchGather", [params, indices]):
diff --git a/tensorflow/tools/compatibility/reorders_v2.py b/tensorflow/tools/compatibility/reorders_v2.py
index bd5ccf6ca8..ac890e2474 100644
--- a/tensorflow/tools/compatibility/reorders_v2.py
+++ b/tensorflow/tools/compatibility/reorders_v2.py
@@ -28,12 +28,11 @@ from __future__ import print_function
 reorders = {
     'tf.argmax': ['input', 'axis', 'name', 'dimension', 'output_type'],
     'tf.argmin': ['input', 'axis', 'name', 'dimension', 'output_type'],
-    'tf.batch_gather': ['params', 'indices', 'name'],
-    'tf.batch_to_space': ['input', 'crops', 'block_size', 'name'],
+    'tf.batch_to_space': ['input', 'crops', 'block_size', 'name', 'block_shape'],
     'tf.boolean_mask': ['tensor', 'mask', 'name', 'axis'],
     'tf.cond': ['pred', 'true_fn', 'false_fn', 'strict', 'name', 'fn1', 'fn2'],
     'tf.confusion_matrix': ['labels', 'predictions', 'num_classes', 'dtype', 'name', 'weights'],
-    'tf.convert_to_tensor': ['value', 'dtype', 'name', 'preferred_dtype'],
+    'tf.convert_to_tensor': ['value', 'dtype', 'name', 'preferred_dtype', 'dtype_hint'],
     'tf.decode_csv': ['records', 'record_defaults', 'field_delim', 'use_quote_delim', 'name', 'na_value', 'select_cols'],
     'tf.depth_to_space': ['input', 'block_size', 'name', 'data_format'],
     'tf.feature_column.categorical_column_with_vocabulary_file': ['key', 'vocabulary_file', 'vocabulary_size', 'num_oov_buckets', 'default_value', 'dtype'],
@@ -59,27 +58,27 @@ reorders = {
     'tf.math.reduce_prod': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
     'tf.math.reduce_sum': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
     'tf.multinomial': ['logits', 'num_samples', 'seed', 'name', 'output_dtype'],
-    'tf.nn.conv1d': ['value', 'filters', 'stride', 'padding', 'use_cudnn_on_gpu', 'data_format', 'name'],
-    'tf.nn.conv2d': ['input', 'filter', 'strides', 'padding', 'use_cudnn_on_gpu', 'data_format', 'dilations', 'name'],
+    'tf.nn.conv1d': ['value', 'filters', 'stride', 'padding', 'use_cudnn_on_gpu', 'data_format', 'name', 'input'],
+    'tf.nn.conv2d': ['input', 'filter', 'strides', 'padding', 'use_cudnn_on_gpu', 'data_format', 'dilations', 'name', 'filters'],
     'tf.nn.conv2d_backprop_filter': ['input', 'filter_sizes', 'out_backprop', 'strides', 'padding', 'use_cudnn_on_gpu', 'data_format', 'dilations', 'name'],
-    'tf.nn.conv2d_backprop_input': ['input_sizes', 'filter', 'out_backprop', 'strides', 'padding', 'use_cudnn_on_gpu', 'data_format', 'dilations', 'name'],
-    'tf.nn.convolution': ['input', 'filter', 'padding', 'strides', 'dilation_rate', 'name', 'data_format'],
+    'tf.nn.conv2d_backprop_input': ['input_sizes', 'filter', 'out_backprop', 'strides', 'padding', 'use_cudnn_on_gpu', 'data_format', 'dilations', 'name', 'filters'],
+    'tf.nn.convolution': ['input', 'filter', 'padding', 'strides', 'dilation_rate', 'name', 'data_format', 'filters', 'dilations'],
     'tf.nn.crelu': ['features', 'name', 'axis'],
     'tf.nn.ctc_beam_search_decoder': ['inputs', 'sequence_length', 'beam_width', 'top_paths', 'merge_repeated'],
     'tf.nn.depth_to_space': ['input', 'block_size', 'name', 'data_format'],
-    'tf.nn.depthwise_conv2d': ['input', 'filter', 'strides', 'padding', 'rate', 'name', 'data_format'],
+    'tf.nn.depthwise_conv2d': ['input', 'filter', 'strides', 'padding', 'rate', 'name', 'data_format', 'dilations'],
     'tf.nn.embedding_lookup': ['params', 'ids', 'partition_strategy', 'name', 'validate_indices', 'max_norm'],
     'tf.nn.embedding_lookup_sparse': ['params', 'sp_ids', 'sp_weights', 'partition_strategy', 'name', 'combiner', 'max_norm'],
     'tf.nn.fractional_avg_pool': ['value', 'pooling_ratio', 'pseudo_random', 'overlapping', 'deterministic', 'seed', 'seed2', 'name'],
     'tf.nn.fractional_max_pool': ['value', 'pooling_ratio', 'pseudo_random', 'overlapping', 'deterministic', 'seed', 'seed2', 'name'],
     'tf.nn.in_top_k': ['predictions', 'targets', 'k', 'name'],
-    'tf.nn.moments': ['x', 'axes', 'shift', 'name', 'keep_dims'],
-    'tf.nn.pool': ['input', 'window_shape', 'pooling_type', 'padding', 'dilation_rate', 'strides', 'name', 'data_format'],
-    'tf.nn.separable_conv2d': ['input', 'depthwise_filter', 'pointwise_filter', 'strides', 'padding', 'rate', 'name', 'data_format'],
-    'tf.nn.softmax_cross_entropy_with_logits': ['_sentinel', 'labels', 'logits', 'dim', 'name'],
-    'tf.nn.space_to_batch': ['input', 'paddings', 'block_size', 'name'],
+    'tf.nn.moments': ['x', 'axes', 'shift', 'name', 'keep_dims', 'keepdims'],
+    'tf.nn.pool': ['input', 'window_shape', 'pooling_type', 'padding', 'dilation_rate', 'strides', 'name', 'data_format', 'dilations'],
+    'tf.nn.separable_conv2d': ['input', 'depthwise_filter', 'pointwise_filter', 'strides', 'padding', 'rate', 'name', 'data_format', 'dilations'],
+    'tf.nn.softmax_cross_entropy_with_logits': ['_sentinel', 'labels', 'logits', 'dim', 'name', 'axis'],
+    'tf.nn.space_to_batch': ['input', 'paddings', 'block_size', 'name', 'block_shape'],
     'tf.nn.space_to_depth': ['input', 'block_size', 'name', 'data_format'],
-    'tf.nn.weighted_moments': ['x', 'axes', 'frequency_weights', 'name', 'keep_dims'],
+    'tf.nn.weighted_moments': ['x', 'axes', 'frequency_weights', 'name', 'keep_dims', 'keepdims'],
     'tf.norm': ['tensor', 'ord', 'axis', 'keepdims', 'name', 'keep_dims'],
     'tf.pad': ['tensor', 'paddings', 'mode', 'name', 'constant_values'],
     'tf.parse_example': ['serialized', 'features', 'name', 'example_names'],
@@ -90,7 +89,7 @@ reorders = {
     'tf.random_poisson': ['lam', 'shape', 'dtype', 'seed', 'name'],
     'tf.reduce_all': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
     'tf.reduce_any': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
-    'tf.reduce_join': ['inputs', 'axis', 'keep_dims', 'separator', 'name', 'reduction_indices'],
+    'tf.reduce_join': ['inputs', 'axis', 'keep_dims', 'separator', 'name', 'reduction_indices', 'keepdims'],
     'tf.reduce_logsumexp': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
     'tf.reduce_max': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
     'tf.reduce_mean': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
@@ -102,17 +101,17 @@ reorders = {
     'tf.serialize_sparse': ['sp_input', 'name', 'out_type'],
     'tf.shape': ['input', 'name', 'out_type'],
     'tf.size': ['input', 'name', 'out_type'],
-    'tf.space_to_batch': ['input', 'paddings', 'block_size', 'name'],
+    'tf.space_to_batch': ['input', 'paddings', 'block_size', 'name', 'block_shape'],
     'tf.space_to_depth': ['input', 'block_size', 'name', 'data_format'],
     'tf.sparse.add': ['a', 'b', 'threshold', 'thresh'],
-    'tf.sparse.concat': ['axis', 'sp_inputs', 'name', 'expand_nonconcat_dim', 'concat_dim'],
+    'tf.sparse.concat': ['axis', 'sp_inputs', 'name', 'expand_nonconcat_dim', 'concat_dim', 'expand_nonconcat_dims'],
     'tf.sparse.reduce_max': ['sp_input', 'axis', 'keepdims', 'reduction_axes', 'keep_dims'],
     'tf.sparse.segment_mean': ['data', 'indices', 'segment_ids', 'name', 'num_segments'],
     'tf.sparse.segment_sqrt_n': ['data', 'indices', 'segment_ids', 'name', 'num_segments'],
     'tf.sparse.segment_sum': ['data', 'indices', 'segment_ids', 'name', 'num_segments'],
     'tf.sparse.split': ['keyword_required', 'sp_input', 'num_split', 'axis', 'name', 'split_dim'],
     'tf.sparse_add': ['a', 'b', 'threshold', 'thresh'],
-    'tf.sparse_concat': ['axis', 'sp_inputs', 'name', 'expand_nonconcat_dim', 'concat_dim'],
+    'tf.sparse_concat': ['axis', 'sp_inputs', 'name', 'expand_nonconcat_dim', 'concat_dim', 'expand_nonconcat_dims'],
     'tf.sparse_matmul': ['a', 'b', 'transpose_a', 'transpose_b', 'a_is_sparse', 'b_is_sparse', 'name'],
     'tf.sparse_reduce_max': ['sp_input', 'axis', 'keepdims', 'reduction_axes', 'keep_dims'],
     'tf.sparse_segment_mean': ['data', 'indices', 'segment_ids', 'name', 'num_segments'],
@@ -120,7 +119,7 @@ reorders = {
     'tf.sparse_segment_sum': ['data', 'indices', 'segment_ids', 'name', 'num_segments'],
     'tf.sparse_split': ['keyword_required', 'sp_input', 'num_split', 'axis', 'name', 'split_dim'],
     'tf.strings.length': ['input', 'name', 'unit'],
-    'tf.strings.reduce_join': ['inputs', 'axis', 'keep_dims', 'separator', 'name', 'reduction_indices'],
+    'tf.strings.reduce_join': ['inputs', 'axis', 'keep_dims', 'separator', 'name', 'reduction_indices', 'keepdims'],
     'tf.strings.substr': ['input', 'pos', 'len', 'name', 'unit'],
     'tf.substr': ['input', 'pos', 'len', 'name', 'unit'],
     'tf.test.assert_equal_graph_def': ['actual', 'expected', 'checkpoint_v2'],
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 85ece0313f..aca4cc3d53 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -723,7 +723,6 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.io.serialize_many_sparse",
         "tf.argmax",
         "tf.argmin",
-        "tf.batch_gather",
         "tf.batch_to_space",
         "tf.cond",
         "tf.nn.space_to_batch",
@@ -1264,7 +1263,6 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "*.make_initializable_iterator": _iterator_transformer,
         "*.make_one_shot_iterator": _iterator_transformer,
         "tf.nn.dropout": _dropout_transformer,
-        "tf.batch_gather": _batch_gather_transformer,
         "tf.to_bfloat16": _cast_transformer,
         "tf.to_complex128": _cast_transformer,
         "tf.to_complex64": _cast_transformer,
@@ -1574,24 +1572,6 @@ def _softmax_cross_entropy_with_logits_transformer(
   return node
 
 
-def _batch_gather_transformer(parent, node, full_name, name, logs):
-  """Add batch_dims argument for gather calls."""
-  # Check if the call already has a batch_dims argument
-  if any([kw.arg == "batch_dims" for kw in node.keywords]):
-    logs.append((ast_edits.INFO, node.lineno, node.col_offset,
-                 "tf.batch_gather already has batch_dims argument. Neat."))
-    return None
-
-  minus_one = ast.Num(n=-1)
-  minus_one.lineno = 0
-  minus_one.col_offset = 0
-  new_arg = ast.keyword("batch_dims", minus_one)
-  node.keywords.append(new_arg)
-  logs.append((ast_edits.INFO, node.lineno, node.col_offset,
-               "Added keyword argument batch_dims=-1 to tf.batch_gather."))
-  return node
-
-
 def _image_resize_transformer(parent, node, full_name, name, logs):
   """Transforms image.resize_* to image.resize(..., method=*, ...)."""
   resize_method = name[7:].upper()
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 11ed634ed4..c5459feb5c 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -994,19 +994,6 @@ tf.print('abc')
     _, unused_report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, expected_text)
 
-  def testBatchGather(self):
-    text = "tf.batch_gather(foo, bar)"
-    expected_text1 = "tf.gather(params=foo, indices=bar, batch_dims=-1)"
-    expected_text2 = "tf.gather(batch_dims=-1, params=foo, indices=bar)"
-    _, unused_report, unused_errors, new_text = self._upgrade(text)
-    self.assertIn(new_text, [expected_text1, expected_text2])
-
-    text = "tf.batch_gather(params=foo, indices=bar)"
-    expected_text1 = "tf.gather(params=foo, indices=bar, batch_dims=-1)"
-    expected_text2 = "tf.gather(batch_dims=-1, params=foo, indices=bar)"
-    _, unused_report, unused_errors, new_text = self._upgrade(text)
-    self.assertIn(new_text, [expected_text1, expected_text2])
-
   def testIterators(self):
     for (text, expected) in [
         ("(expr + yielding(data)).make_one_shot_iterator()",
-- 
GitLab


From d22fe4022cfd0e142e6f014899dbb8a7856897f4 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 1 Feb 2019 09:44:21 -0800
Subject: [PATCH 1760/2345] Add back comment about Conv2D transpose args

---
 tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
index cfec175817..b2a937e21c 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
@@ -1622,6 +1622,8 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group,
   TRT_TensorOrWeights backprop_output_size;
   const nvinfer1::ITensor* tensor = nullptr;
   if (is_conv2d_backprop_input) {
+    // In the case when Conv2dBackpropInput is used for conv2d_transpose, these
+    // inputs correspond to: output size, filter, and input.
     TF_RETURN_IF_ERROR(CheckInputsWeights(
         *params,
         {{"input_sizes", true}, {"filter", true}, {"out_backprop", false}}));
-- 
GitLab


From 0dc200d8da26a9a42281df160168fe905c6126d0 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 1 Feb 2019 09:29:50 -0800
Subject: [PATCH 1761/2345] Remove call to std::random_shuffle.

std::random_shuffle is removed in C++17, and is causing TF to be not buildable in such configurations.

Helps with Issue #23561

PiperOrigin-RevId: 231984187
---
 tensorflow/core/kernels/sdca_internal.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/sdca_internal.cc b/tensorflow/core/kernels/sdca_internal.cc
index 2bb2c0d91e..cbc754af0e 100644
--- a/tensorflow/core/kernels/sdca_internal.cc
+++ b/tensorflow/core/kernels/sdca_internal.cc
@@ -310,7 +310,10 @@ Status Examples::SampleAdaptiveProbabilities(
 
 void Examples::RandomShuffle() {
   std::iota(sampled_index_.begin(), sampled_index_.end(), 0);
-  std::random_shuffle(sampled_index_.begin(), sampled_index_.end());
+
+  std::random_device rd;
+  std::mt19937 rng(rd());
+  std::shuffle(sampled_index_.begin(), sampled_index_.end(), rng);
 }
 
 // TODO(sibyl-Aix6ihai): Refactor/shorten this function.
-- 
GitLab


From ebd96894465ddf4162284e05add8fa1d72d4d263 Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Fri, 1 Feb 2019 10:05:09 -0800
Subject: [PATCH 1762/2345] [TF StreamExecutor] Better checks of device vs.
 host pointer location.

Check whether source/dest pointers really reside on the device being requested
during memcpy.

PiperOrigin-RevId: 231990786
---
 .../python/debug/cli/stepper_cli_test.py      |   6 +
 tensorflow/python/debug/lib/stepper_test.py   |  16 +++
 .../kernel_tests/control_flow_ops_py_test.py  |  23 ++--
 tensorflow/stream_executor/cuda/BUILD         |  29 +++--
 .../stream_executor/cuda/cuda_driver.cc       | 122 +++++++++++++-----
 5 files changed, 142 insertions(+), 54 deletions(-)

diff --git a/tensorflow/python/debug/cli/stepper_cli_test.py b/tensorflow/python/debug/cli/stepper_cli_test.py
index 5cf69d0168..c728373ae2 100644
--- a/tensorflow/python/debug/cli/stepper_cli_test.py
+++ b/tensorflow/python/debug/cli/stepper_cli_test.py
@@ -235,6 +235,9 @@ class NodeStepperSimpleGraphTest(test_util.TensorFlowTestCase):
       ], output.lines)
 
   def testContToValidNodeShouldUpdateStatus(self):
+    if test_util.is_gpu_available():
+      self.skipTest("b/123446705 this causes a segfault on GPU")
+
     with stepper.NodeStepper(self.sess, self.e) as node_stepper:
       cli = stepper_cli.NodeStepperCLI(node_stepper)
 
@@ -275,6 +278,9 @@ class NodeStepperSimpleGraphTest(test_util.TensorFlowTestCase):
       self.assertIn(stepper_cli.NodeStepperCLI.STATE_CONT, stat_labels[index_d])
 
   def testSteppingOneStepAtATimeShouldUpdateStatus(self):
+    if test_util.is_gpu_available():
+      self.skipTest("b/123446705 this causes a segfault on GPU")
+
     with stepper.NodeStepper(self.sess, self.e) as node_stepper:
       cli = stepper_cli.NodeStepperCLI(node_stepper)
 
diff --git a/tensorflow/python/debug/lib/stepper_test.py b/tensorflow/python/debug/lib/stepper_test.py
index 9e78e207b8..bec858a1ba 100644
--- a/tensorflow/python/debug/lib/stepper_test.py
+++ b/tensorflow/python/debug/lib/stepper_test.py
@@ -94,6 +94,9 @@ class StepperTest(test_util.TensorFlowTestCase):
       self.assertAllClose(6.0, stepper.cont("c"))
 
   def testUsingNamesNotUsingIntermediateTensors(self):
+    if test_util.is_gpu_available():
+      self.skipTest("b/123446705 this causes a segfault on GPU")
+
     with NodeStepper(self.sess, "e:0") as stepper:
       # The first cont() call should have used no feeds.
       result = stepper.cont("c:0")
@@ -119,6 +122,9 @@ class StepperTest(test_util.TensorFlowTestCase):
       }, stepper.last_feed_types())
 
   def testUsingNodesNotUsingIntermediateTensors(self):
+    if test_util.is_gpu_available():
+      self.skipTest("b/123446705 this causes a segfault on GPU")
+
     with NodeStepper(self.sess, self.e) as stepper:
       # There should be no handles before any cont() calls.
       self.assertEqual([], stepper.handle_names())
@@ -493,6 +499,9 @@ class StepperTestWithPlaceHolders(test_util.TensorFlowTestCase):
       self.assertSetEqual({"ph0", "ph1"}, set(stepper.placeholders()))
 
   def testContWithPlaceholders(self):
+    if test_util.is_gpu_available():
+      self.skipTest("b/123446705 this causes a segfault on GPU")
+
     with NodeStepper(
         self.sess,
         self.y,
@@ -739,6 +748,9 @@ class StepperBackwardRunTest(test_util.TensorFlowTestCase):
     ops.reset_default_graph()
 
   def testContToUpdateA(self):
+    if test_util.is_gpu_available():
+      self.skipTest("b/123446705 this causes a segfault on GPU")
+
     with NodeStepper(self.sess, "optim") as stepper:
       result = stepper.cont("a:0")
       self.assertAllClose(1.0, result)
@@ -887,6 +899,8 @@ class StepperBackwardRunTest(test_util.TensorFlowTestCase):
 
     "clean" means no Variables have been updated by preceding cont() calls.
     """
+    if test_util.is_gpu_available():
+      self.skipTest("b/123446705 this causes a segfault on GPU")
 
     with NodeStepper(self.sess, "optim") as stepper:
       # First, call cont() on the two tensors on the intermediate level: e and
@@ -979,6 +993,8 @@ class StepperBackwardRunTest(test_util.TensorFlowTestCase):
 
   def testOverrideThenContToUpdateThenRemoveOverrideThenUpdateAgain(self):
     """Test cont() to update nodes after overriding tensor values."""
+    if test_util.is_gpu_available():
+      self.skipTest("b/123446705 this causes a segfault on GPU")
 
     with NodeStepper(self.sess, "optim") as stepper:
       result = stepper.cont("d:0")
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 25fc42b60d..a52b1f09d6 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -1055,7 +1055,8 @@ class ControlFlowTest(test.TestCase):
 
       with self.captureWritesToStream(sys.stderr) as printed:
         self.assertEqual(self.evaluate(cond()), 10)
-      self.assertEqual(printed.contents(), "A\nB\nC\n")
+      self.assertTrue(printed.contents().endswith("A\nB\nC\n"),
+                      printed.contents())
 
       @eager_function.defun
       def nested_cond():
@@ -1063,7 +1064,8 @@ class ControlFlowTest(test.TestCase):
 
       with self.captureWritesToStream(sys.stderr) as printed:
         self.assertEqual(self.evaluate(nested_cond()), 10)
-      self.assertEqual(printed.contents(), "A\nB\nC\n")
+      self.assertTrue(printed.contents().endswith("A\nB\nC\n"),
+                      printed.contents())
 
     # wrap_function should prune.
     def pruned_cond():
@@ -1112,11 +1114,13 @@ class ControlFlowTest(test.TestCase):
       with self.cached_session():
         with self.captureWritesToStream(sys.stderr) as printed:
           self.assertEqual(self.evaluate(build_while()[0]), 2)
-        self.assertEqual(printed.contents(), "D\nD\n")
+        self.assertTrue(printed.contents().endswith("D\nD\n"),
+                        printed.contents())
 
         with self.captureWritesToStream(sys.stderr) as printed:
           self.assertEqual(self.evaluate(build_nested_while()[0]), 2)
-        self.assertEqual(printed.contents(), "D\nD\n")
+        self.assertTrue(printed.contents().endswith("D\nD\n"),
+                        printed.contents())
 
     # In defuns, all prints should execute in program order.
     @eager_function.defun
@@ -1125,7 +1129,8 @@ class ControlFlowTest(test.TestCase):
 
     with self.captureWritesToStream(sys.stderr) as printed:
       self.assertEqual(self.evaluate(while_loop()), 2)
-    self.assertEqual(printed.contents(), "A\nB\nC\nD\nA\nB\nC\nD\nA\n")
+    self.assertTrue(printed.contents().endswith("A\nB\nC\nD\nA\nB\nC\nD\nA\n"),
+                    printed.contents())
 
     @eager_function.defun
     def nested_while_loop():
@@ -1135,7 +1140,9 @@ class ControlFlowTest(test.TestCase):
     if not context.executing_eagerly():
       with self.captureWritesToStream(sys.stderr) as printed:
         self.assertEqual(self.evaluate(nested_while_loop()), 2)
-      self.assertEqual(printed.contents(), "A\nB\nC\nD\nA\nB\nC\nD\nA\n")
+      self.assertTrue(
+          printed.contents().endswith("A\nB\nC\nD\nA\nB\nC\nD\nA\n"),
+          printed.contents())
 
     # wrap_function should prune.
     def pruned_while():
@@ -1144,7 +1151,7 @@ class ControlFlowTest(test.TestCase):
 
     with self.captureWritesToStream(sys.stderr) as printed:
       self.assertEqual(self.evaluate(pruned_while()), 2)
-    self.assertEqual(printed.contents(), "D\nD\n")
+    self.assertTrue(printed.contents().endswith("D\nD\n"), printed.contents())
 
     def pruned_nested_while():
       return build_nested_while()[0]
@@ -1154,7 +1161,7 @@ class ControlFlowTest(test.TestCase):
     if not context.executing_eagerly():
       with self.captureWritesToStream(sys.stderr) as printed:
         self.assertEqual(self.evaluate(pruned_nested_while()), 2)
-      self.assertEqual(printed.contents(), "D\nD\n")
+      self.assertTrue(printed.contents().endswith("D\nD\n"), printed.contents())
 
   # Microbenchmark: 256,000 iterations/s.
   def testWhile_1(self):
diff --git a/tensorflow/stream_executor/cuda/BUILD b/tensorflow/stream_executor/cuda/BUILD
index 9dc3be45e7..009462085b 100644
--- a/tensorflow/stream_executor/cuda/BUILD
+++ b/tensorflow/stream_executor/cuda/BUILD
@@ -45,17 +45,19 @@ cc_library(
     srcs = if_cuda_is_configured(["cuda_platform.cc"]),
     hdrs = if_cuda_is_configured(["cuda_platform.h"]),
     visibility = ["//visibility:public"],
-    deps = if_cuda_is_configured([
-        ":cuda_driver",
-        ":cuda_gpu_executor",
-        ":cuda_platform_id",
-        "//tensorflow/stream_executor",  # buildcleaner: keep
-        "//tensorflow/stream_executor:executor_cache",
-        "//tensorflow/stream_executor:multi_platform_manager",
-        "//tensorflow/stream_executor:stream_executor_pimpl_header",
-        "//tensorflow/stream_executor/lib",
-        "//tensorflow/stream_executor/platform",
-    ] + tf_additional_cuda_platform_deps()),
+    deps = if_cuda_is_configured(
+        [
+            ":cuda_driver",
+            ":cuda_gpu_executor",
+            ":cuda_platform_id",
+            "//tensorflow/stream_executor",  # buildcleaner: keep
+            "//tensorflow/stream_executor:executor_cache",
+            "//tensorflow/stream_executor:multi_platform_manager",
+            "//tensorflow/stream_executor:stream_executor_pimpl_header",
+            "//tensorflow/stream_executor/lib",
+            "//tensorflow/stream_executor/platform",
+        ],
+    ) + tf_additional_cuda_platform_deps(),
     alwayslink = True,  # Registers itself with the MultiPlatformManager.
 )
 
@@ -85,6 +87,7 @@ cc_library(
         "@com_google_absl//absl/container:inlined_vector",
         "@com_google_absl//absl/strings",
         "@local_config_cuda//cuda:cuda_headers",
+        "//tensorflow/core:cuda",
         "//tensorflow/stream_executor:device_options",
         "//tensorflow/stream_executor/gpu:gpu_driver_header",
         "//tensorflow/stream_executor/lib",
@@ -221,7 +224,9 @@ cc_library(
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
         "//tensorflow/stream_executor/platform:dso_loader",
-    ] + tf_additional_cudnn_plugin_deps() + if_static(["@local_config_cuda//cuda:cudnn"])),
+    ]) + tf_additional_cudnn_plugin_deps() + if_cuda_is_configured(if_static([
+        "@local_config_cuda//cuda:cudnn",
+    ])),
     alwayslink = True,
 )
 
diff --git a/tensorflow/stream_executor/cuda/cuda_driver.cc b/tensorflow/stream_executor/cuda/cuda_driver.cc
index 34ba7c5b15..080c26fdfe 100644
--- a/tensorflow/stream_executor/cuda/cuda_driver.cc
+++ b/tensorflow/stream_executor/cuda/cuda_driver.cc
@@ -24,6 +24,7 @@ limitations under the License.
 #include "absl/base/casts.h"
 #include "absl/container/inlined_vector.h"
 #include "absl/strings/str_cat.h"
+#include "cuda/include/cuda_runtime_api.h"
 #include "tensorflow/stream_executor/cuda/cuda_diagnostics.h"
 #include "tensorflow/stream_executor/cuda/cuda_driver_wrapper.h"
 #include "tensorflow/stream_executor/lib/env.h"
@@ -107,11 +108,11 @@ class CreatedContexts {
 
 // Formats CUresult to output prettified values into a log stream.
 string ToString(CUresult result) {
-  const char *error_name;
+  const char* error_name;
   if (tensorflow::wrap::cuGetErrorName(result, &error_name)) {
     return absl::StrCat("UNKNOWN ERROR (", static_cast<int>(result), ")");
   }
-  const char *error_string;
+  const char* error_string;
   if (tensorflow::wrap::cuGetErrorString(result, &error_string)) {
     return error_name;
   }
@@ -139,14 +140,14 @@ CUcontext CurrentContext() {
 // thread::ThreadPool on some platforms), we run certain routines in this pool
 // and wait for completion.
 static mutex driver_executor_threadpool_mu(LINKER_INITIALIZED);
-static port::ThreadPool *InitializeDriverExecutor() {
+static port::ThreadPool* InitializeDriverExecutor() {
   return new port::ThreadPool(port::Env::Default(), port::ThreadOptions(),
                               "cuda_driver", 1);
 }
 
-port::ThreadPool *GetDriverExecutor() {
+port::ThreadPool* GetDriverExecutor() {
   mutex_lock lock(driver_executor_threadpool_mu);
-  static port::ThreadPool *thread_pool = InitializeDriverExecutor();
+  static port::ThreadPool* thread_pool = InitializeDriverExecutor();
   return thread_pool;
 }
 
@@ -165,12 +166,30 @@ string MemorySpaceString(MemorySpace memory_space) {
 
 namespace {
 
+template <typename PtrT>
+bool PointerIsValid(const PtrT ptr) {
+  // Checks that the pointer is to a location on the device it purports to be.
+  // PtrT is one of CUdeviceptr or void*.  If it's a CUdeviceptr, then
+  // cudaPointerGetAttributes should not fail, and return a memoryType of
+  // cudaMemoryTypeDevice.
+
+  bool is_host_ptr = !std::is_same<PtrT, CUdeviceptr>::value;
+  cudaPointerAttributes attributes;
+  cudaError_t err =
+      cudaPointerGetAttributes(&attributes, reinterpret_cast<const void*>(ptr));
+  // If we failed, reset cuda error status to avoid poisoning cuda streams.
+  if (err != cudaSuccess) cudaGetLastError();
+  bool points_to_host_memory = (err == cudaErrorInvalidValue ||
+                                attributes.memoryType != cudaMemoryTypeDevice);
+  return (is_host_ptr == points_to_host_memory);
+}
+
 // Call cuCtxtSynchronize and crash if it doesn't succeed.
 void SynchronizeOrDie() {
   auto res = tensorflow::wrap::cuCtxSynchronize();
   if (res != CUDA_SUCCESS) {
-    LOG(FATAL) << "Synchronize found "
-               << ToString(res) << " :: " << port::CurrentStackTrace();
+    LOG(FATAL) << "Synchronize found " << ToString(res)
+               << " :: " << port::CurrentStackTrace();
   }
 }
 
@@ -284,7 +303,6 @@ string CUDAPointersToCanAccessString(CUdeviceptr from, CUdeviceptr to) {
              : "false";
 }
 
-
 // Actually performs the work of CUDA initialization. Wrapped up in one-time
 // execution guard.
 static port::Status InternalInit() {
@@ -312,7 +330,7 @@ static port::Status InternalInit() {
   // called once, but GpuDriver::Init may be called many times.
   static port::Status init_retval;
   static bool set = false;
-  static mutex *init_mu = new mutex;
+  static mutex* init_mu = new mutex;
 
   mutex_lock lock(*init_mu);
   if (!set) {
@@ -351,8 +369,8 @@ static port::Status InternalInit() {
   return true;
 }
 
-bool DeviceOptionsToContextFlags(const DeviceOptions &device_options,
-                                 int *flags) {
+bool DeviceOptionsToContextFlags(const DeviceOptions& device_options,
+                                 int* flags) {
   static_assert(DeviceOptions::kMask == 0xf,
                 "needs update for new device options");
 
@@ -580,7 +598,7 @@ GpuDriver::ContextGetSharedMemConfig(GpuContext* context) {
   GetDriverExecutor()->Schedule([context, ptx_contents, module, &ret,
                                  &notification]() {
     ScopedActivateContext activation(context);
-    void *ptx_data = const_cast<char *>(ptx_contents);
+    void* ptx_data = const_cast<char*>(ptx_contents);
     static const unsigned int kLogBufferBytesLimit = 1024;
     unsigned int error_log_buffer_bytes = kLogBufferBytesLimit;
     unsigned int info_log_buffer_bytes = kLogBufferBytesLimit;
@@ -593,12 +611,12 @@ GpuDriver::ContextGetSharedMemConfig(GpuContext* context) {
                               CU_JIT_INFO_LOG_BUFFER, CU_JIT_LOG_VERBOSE};
     // Note that the driver API wants the contents of this values to be stored
     // in an array of void*s, so we coerce them accordingly.
-    void *option_values[] = {
-        absl::bit_cast<void *>(uintptr_t(error_log_buffer_bytes)),
-        absl::bit_cast<void *>(error_log_buffer.data()),
-        absl::bit_cast<void *>(uintptr_t(info_log_buffer_bytes)),
-        absl::bit_cast<void *>(info_log_buffer.data()),
-        absl::bit_cast<void *>(uintptr_t(log_verbose))};
+    void* option_values[] = {
+        absl::bit_cast<void*>(uintptr_t(error_log_buffer_bytes)),
+        absl::bit_cast<void*>(error_log_buffer.data()),
+        absl::bit_cast<void*>(uintptr_t(info_log_buffer_bytes)),
+        absl::bit_cast<void*>(info_log_buffer.data()),
+        absl::bit_cast<void*>(uintptr_t(log_verbose))};
     CHECK(TF_ARRAYSIZE(options) == TF_ARRAYSIZE(option_values));
 
     CUresult res;
@@ -622,8 +640,8 @@ GpuDriver::ContextGetSharedMemConfig(GpuContext* context) {
       LOG(ERROR) << "failed to load PTX text as a module: " << ToString(res);
       // As a precaution for null termination of the API-provided value, ensure
       // that at least the last byte is null.
-      error_log_buffer[error_log_buffer_bytes ?
-                       error_log_buffer_bytes - 1 : 0] = '\0';
+      error_log_buffer[error_log_buffer_bytes ? error_log_buffer_bytes - 1
+                                              : 0] = '\0';
       LOG(ERROR) << "error log buffer (" << error_log_buffer_bytes
                  << " bytes): " << error_log_buffer.data();
       ret = false;
@@ -828,7 +846,7 @@ GpuDriver::ContextGetSharedMemConfig(GpuContext* context) {
                << " bytes) from device: " << ToString(res);
     return nullptr;
   }
-  void *ptr = reinterpret_cast<void *>(result);
+  void* ptr = reinterpret_cast<void*>(result);
   VLOG(2) << "allocated " << ptr << " for context " << context->context()
           << " of " << bytes << " bytes";
   return ptr;
@@ -860,7 +878,7 @@ GpuDriver::ContextGetSharedMemConfig(GpuContext* context) {
                << " bytes unified memory; result: " << ToString(res);
     return nullptr;
   }
-  void *ptr = reinterpret_cast<void *>(result);
+  void* ptr = reinterpret_cast<void*>(result);
   VLOG(2) << "allocated " << ptr << " for context " << context->context()
           << " of " << bytes << " bytes in unified memory";
   return ptr;
@@ -882,7 +900,7 @@ GpuDriver::ContextGetSharedMemConfig(GpuContext* context) {
 
 /* static */ void* GpuDriver::HostAllocate(GpuContext* context, uint64 bytes) {
   ScopedActivateContext activation(context);
-  void *host_mem = nullptr;
+  void* host_mem = nullptr;
   // "Portable" memory is visible to all CUDA contexts. Safe for our use model.
   CUresult res = tensorflow::wrap::cuMemHostAlloc(&host_mem, bytes,
                                                   CU_MEMHOSTALLOC_PORTABLE);
@@ -1074,13 +1092,19 @@ GpuDriver::ContextGetSharedMemConfig(GpuContext* context) {
                                                           CUdeviceptr gpu_src,
                                                           uint64 size) {
   ScopedActivateContext activation(context);
+  if (size > 0) {
+    CHECK(PointerIsValid(gpu_src))
+        << "Source pointer is not actually on GPU: " << gpu_src;
+    CHECK(PointerIsValid(host_dst))
+        << "Destination pointer is not actually on CPU: " << host_dst;
+  }
   CUresult res = tensorflow::wrap::cuMemcpyDtoH(host_dst, gpu_src, size);
   if (res != CUDA_SUCCESS) {
     return port::InternalError(
         port::Printf("failed to synchronous memcpy from device to host: %s; "
                      "host dst: %p; GPU src: %p; size: %llu=0x%llx",
                      ToString(res).c_str(), host_dst,
-                     absl::bit_cast<void *>(gpu_src), size, size));
+                     absl::bit_cast<void*>(gpu_src), size, size));
   }
   VLOG(2) << "successfully sync memcpy'd d2h of " << size << " bytes to "
           << host_dst;
@@ -1092,12 +1116,18 @@ GpuDriver::ContextGetSharedMemConfig(GpuContext* context) {
                                                           const void* host_src,
                                                           uint64 size) {
   ScopedActivateContext activation(context);
+  if (size > 0) {
+    CHECK(PointerIsValid(host_src))
+        << "Source pointer is not actually on CPU: " << host_src;
+    CHECK(PointerIsValid(gpu_dst))
+        << "Destination pointer is not actually on GPU: " << gpu_dst;
+  }
   CUresult res = tensorflow::wrap::cuMemcpyHtoD(gpu_dst, host_src, size);
   if (res != CUDA_SUCCESS) {
     return port::InternalError(port::Printf(
         "failed to synchronous memcpy from host to device: %s; GPU dst: %p;"
         " host src: %p; size: %llu=0x%llx",
-        ToString(res).c_str(), absl::bit_cast<void *>(gpu_dst), host_src, size,
+        ToString(res).c_str(), absl::bit_cast<void*>(gpu_dst), host_src, size,
         size));
   }
   VLOG(2) << "successfully enqueued sync memcpy h2d of " << size << " bytes";
@@ -1109,13 +1139,19 @@ GpuDriver::ContextGetSharedMemConfig(GpuContext* context) {
                                                           CUdeviceptr gpu_src,
                                                           uint64 size) {
   ScopedActivateContext activation(context);
+  if (size > 0) {
+    CHECK(PointerIsValid(gpu_src))
+        << "Source pointer is not actually on GPU: " << gpu_src;
+    CHECK(PointerIsValid(gpu_dst))
+        << "Destination pointer is not actually on GPU: " << gpu_dst;
+  }
   CUresult res = tensorflow::wrap::cuMemcpyDtoD(gpu_dst, gpu_src, size);
   if (res != CUDA_SUCCESS) {
     return port::InternalError(port::Printf(
         "failed to synchronous memcpy from host to device: %s; GPU dst: %p; "
         "GPU src: %p; size: %llu=0x%llx",
-        ToString(res).c_str(), absl::bit_cast<void *>(gpu_dst),
-        absl::bit_cast<void *>(gpu_src), size, size));
+        ToString(res).c_str(), absl::bit_cast<void*>(gpu_dst),
+        absl::bit_cast<void*>(gpu_src), size, size));
   }
   VLOG(2) << "successfully sync memcpy'd d2d of " << size << " bytes";
   return port::Status::OK();
@@ -1127,18 +1163,24 @@ GpuDriver::ContextGetSharedMemConfig(GpuContext* context) {
                                                    uint64 size,
                                                    CUstream stream) {
   ScopedActivateContext activation(context);
+  if (size > 0) {
+    CHECK(PointerIsValid(gpu_src))
+        << "Source pointer is not actually on GPU: " << gpu_src;
+    CHECK(PointerIsValid(host_dst))
+        << "Destination pointer is not actually on CPU: " << host_dst;
+  }
   CUresult res =
       tensorflow::wrap::cuMemcpyDtoHAsync(host_dst, gpu_src, size, stream);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << port::Printf(
         "failed to enqueue async memcpy from device to host: %s; host dst: %p; "
         "GPU src: %p; size: %llu=0x%llx",
-        ToString(res).c_str(), host_dst, absl::bit_cast<void *>(gpu_src), size,
+        ToString(res).c_str(), host_dst, absl::bit_cast<void*>(gpu_src), size,
         size);
     return false;
   }
   VLOG(2) << "successfully enqueued async memcpy d2h of " << size
-          << " bytes from " << absl::bit_cast<void *>(gpu_src) << " to "
+          << " bytes from " << absl::bit_cast<void*>(gpu_src) << " to "
           << host_dst << " on stream " << stream;
   return true;
 }
@@ -1149,13 +1191,19 @@ GpuDriver::ContextGetSharedMemConfig(GpuContext* context) {
                                                    uint64 size,
                                                    CUstream stream) {
   ScopedActivateContext activation(context);
+  if (size > 0) {
+    CHECK(PointerIsValid(host_src))
+        << "Source pointer is not actually on CPU: " << host_src;
+    CHECK(PointerIsValid(gpu_dst))
+        << "Destination pointer is not actually on GPU: " << gpu_dst;
+  }
   CUresult res =
       tensorflow::wrap::cuMemcpyHtoDAsync(gpu_dst, host_src, size, stream);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << port::Printf(
         "failed to enqueue async memcpy from host to device: %s; GPU dst: %p; "
         "host src: %p; size: %llu=0x%llx",
-        ToString(res).c_str(), absl::bit_cast<void *>(gpu_dst), host_src, size,
+        ToString(res).c_str(), absl::bit_cast<void*>(gpu_dst), host_src, size,
         size);
     return false;
   }
@@ -1170,6 +1218,12 @@ GpuDriver::ContextGetSharedMemConfig(GpuContext* context) {
                                                    uint64 size,
                                                    CUstream stream) {
   ScopedActivateContext activation(context);
+  if (size > 0) {
+    CHECK(PointerIsValid(gpu_src))
+        << "Source pointer is not actually on GPU: " << gpu_src;
+    CHECK(PointerIsValid(gpu_dst))
+        << "Destination pointer is not actually on GPU: " << gpu_dst;
+  }
   CUresult result =
       tensorflow::wrap::cuMemcpyDtoDAsync(gpu_dst, gpu_src, size, stream);
   if (result != CUDA_SUCCESS) {
@@ -1178,10 +1232,10 @@ GpuDriver::ContextGetSharedMemConfig(GpuContext* context) {
         "; GPU dst: %p on %s %s"
         "; GPU src: %p on %s %s"
         "; can access? %s; size: %llu=0x%llx",
-        ToString(result).c_str(), absl::bit_cast<void *>(gpu_dst),
+        ToString(result).c_str(), absl::bit_cast<void*>(gpu_dst),
         CUDAPointerToMemorySpaceString(gpu_dst).c_str(),
         CUDAPointerToDeviceString(gpu_dst).c_str(),
-        absl::bit_cast<void *>(gpu_src),
+        absl::bit_cast<void*>(gpu_src),
         CUDAPointerToMemorySpaceString(gpu_src).c_str(),
         CUDAPointerToDeviceString(gpu_src).c_str(),
         CUDAPointersToCanAccessString(gpu_src, gpu_dst).c_str(), size, size);
@@ -1289,13 +1343,13 @@ GpuDriver::ContextGetSharedMemConfig(GpuContext* context) {
     return port::Status(
         port::error::NOT_FOUND,
         port::Printf("not a device pointer %p; %s",
-                     reinterpret_cast<void *>(dptr), ToString(result).c_str()));
+                     reinterpret_cast<void*>(dptr), ToString(result).c_str()));
   }
 
   return port::Status(
       port::error::INTERNAL,
       port::Printf("failed to get pointer into for device pointer %p; %s",
-                   reinterpret_cast<void *>(dptr), ToString(result).c_str()));
+                   reinterpret_cast<void*>(dptr), ToString(result).c_str()));
 }
 
 /* static */ port::StatusOr<CUdevice> GpuDriver::GetPointerDevice(
-- 
GitLab


From 1ea023a26f24c9de246ae77c3b65278df13bf654 Mon Sep 17 00:00:00 2001
From: Eric Liu <ioeric@google.com>
Date: Fri, 1 Feb 2019 10:10:04 -0800
Subject: [PATCH 1763/2345] Adapt llvm interface changes in r352827

PiperOrigin-RevId: 231991808
---
 .../xla/service/cpu/dot_op_emitter.cc         | 12 ++--
 .../xla/service/cpu/elemental_ir_emitter.cc   | 17 ++---
 .../compiler/xla/service/cpu/ir_emitter.cc    | 63 ++++++++++++-------
 .../compiler/xla/service/cpu/ir_function.cc   |  8 ++-
 .../xla/service/cpu/tests/cpu_noalias_test.cc |  5 +-
 .../xla/service/gpu/elemental_ir_emitter.cc   |  8 ++-
 6 files changed, 71 insertions(+), 42 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
index 48510181bd..0fecbaf391 100644
--- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
@@ -643,11 +643,13 @@ Status DotOpEmitter::EmitCallToRuntime() {
   llvm::Function* function = b_->GetInsertBlock()->getParent();
   llvm::Module* module = function->getParent();
 
-  llvm::Function* matmul_func = llvm::cast<llvm::Function>(
-      module->getOrInsertFunction(fn_name, matmul_type));
-  matmul_func->setCallingConv(llvm::CallingConv::C);
-  matmul_func->setDoesNotThrow();
-  matmul_func->setOnlyAccessesArgMemory();
+  llvm::FunctionCallee matmul_func =
+      module->getOrInsertFunction(fn_name, matmul_type);
+  if (auto* fn = llvm::dyn_cast<llvm::Function>(matmul_func.getCallee())) {
+    fn->setCallingConv(llvm::CallingConv::C);
+    fn->setDoesNotThrow();
+    fn->setOnlyAccessesArgMemory();
+  }
 
   // The Eigen runtime function expects column-major layout. If the matrices are
   // row major, then use the following identity to compute the product:
diff --git a/tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.cc
index c8312d80bd..0028fbaed8 100644
--- a/tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.cc
@@ -51,10 +51,11 @@ StatusOr<llvm::Value*> CpuElementalIrEmitter::EmitAtan2(PrimitiveType prim_type,
       return Unimplemented("atan2");
   }
   // Create a function declaration.
-  llvm::Function* function =
-      llvm::cast<llvm::Function>(module_->getOrInsertFunction(
-          llvm_ir::AsStringRef(function_name), lhs->getType(), lhs->getType(),
-          rhs->getType()));
+  llvm::Function* function = llvm::dyn_cast<llvm::Function>(
+      module_
+          ->getOrInsertFunction(llvm_ir::AsStringRef(function_name),
+                                lhs->getType(), lhs->getType(), rhs->getType())
+          .getCallee());
   function->setCallingConv(llvm::CallingConv::C);
   function->setDoesNotThrow();
   function->setDoesNotAccessMemory();
@@ -85,9 +86,11 @@ StatusOr<llvm::Value*> CpuElementalIrEmitter::EmitTanh(PrimitiveType prim_type,
       return Unimplemented("tanh");
   }
   // Create a function declaration.
-  llvm::Function* function = llvm::cast<llvm::Function>(
-      module_->getOrInsertFunction(llvm_ir::AsStringRef(function_name),
-                                   value->getType(), value->getType()));
+  llvm::Function* function = llvm::dyn_cast<llvm::Function>(
+      module_
+          ->getOrInsertFunction(llvm_ir::AsStringRef(function_name),
+                                value->getType(), value->getType())
+          .getCallee());
   function->setCallingConv(llvm::CallingConv::C);
   function->setDoesNotThrow();
   function->setDoesNotAccessMemory();
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index b26bfffc6a..efdda8599a 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -412,11 +412,18 @@ Status IrEmitter::EmitXfeedTransfer(XfeedKind kind, const Shape& shape,
 
   llvm::Function* acquire_func;
   if (kind == XfeedKind::kInfeed) {
-    acquire_func = llvm::cast<llvm::Function>(module_->getOrInsertFunction(
-        runtime::kAcquireInfeedBufferForDequeueSymbolName, acquire_type));
+    acquire_func = llvm::dyn_cast<llvm::Function>(
+        module_
+            ->getOrInsertFunction(
+                runtime::kAcquireInfeedBufferForDequeueSymbolName, acquire_type)
+            .getCallee());
   } else {
-    acquire_func = llvm::cast<llvm::Function>(module_->getOrInsertFunction(
-        runtime::kAcquireOutfeedBufferForPopulationSymbolName, acquire_type));
+    acquire_func = llvm::dyn_cast<llvm::Function>(
+        module_
+            ->getOrInsertFunction(
+                runtime::kAcquireOutfeedBufferForPopulationSymbolName,
+                acquire_type)
+            .getCallee());
   }
   acquire_func->setCallingConv(llvm::CallingConv::C);
 
@@ -429,11 +436,19 @@ Status IrEmitter::EmitXfeedTransfer(XfeedKind kind, const Shape& shape,
 
   llvm::Function* release_func;
   if (kind == XfeedKind::kInfeed) {
-    release_func = llvm::cast<llvm::Function>(module_->getOrInsertFunction(
-        runtime::kReleaseInfeedBufferAfterDequeueSymbolName, release_type));
+    release_func = llvm::dyn_cast<llvm::Function>(
+        module_
+            ->getOrInsertFunction(
+                runtime::kReleaseInfeedBufferAfterDequeueSymbolName,
+                release_type)
+            .getCallee());
   } else {
-    release_func = llvm::cast<llvm::Function>(module_->getOrInsertFunction(
-        runtime::kReleaseOutfeedBufferAfterPopulationSymbolName, release_type));
+    release_func = llvm::dyn_cast<llvm::Function>(
+        module_
+            ->getOrInsertFunction(
+                runtime::kReleaseOutfeedBufferAfterPopulationSymbolName,
+                release_type)
+            .getCallee());
   }
   release_func->setCallingConv(llvm::CallingConv::C);
 
@@ -629,9 +644,11 @@ Status IrEmitter::HandleSort(HloInstruction* hlo) {
        b_.getInt8PtrTy()->getPointerTo(), b_.getInt32Ty(),
        b_.getInt32Ty()->getPointerTo(), less_than_function->getType()},
       /*isVarArg=*/false);
-  auto* key_value_sort_func =
-      llvm::cast<llvm::Function>(module_->getOrInsertFunction(
-          runtime::kKeyValueSortSymbolName, key_value_sort_type));
+  auto* key_value_sort_func = llvm::dyn_cast<llvm::Function>(
+      module_
+          ->getOrInsertFunction(runtime::kKeyValueSortSymbolName,
+                                key_value_sort_type)
+          .getCallee());
   key_value_sort_func->setCallingConv(llvm::CallingConv::C);
   key_value_sort_func->setDoesNotThrow();
   llvm::Value* values = llvm_ir::EmitAllocaAtFunctionEntryWithCount(
@@ -1240,8 +1257,8 @@ Status IrEmitter::HandleConvolution(HloInstruction* convolution) {
         LOG(WARNING) << "Using Eigen instead of MKL-DNN for single-threaded "
                         "conv2d function.";
       }
-      llvm::Function* conv_func = llvm::cast<llvm::Function>(
-          module_->getOrInsertFunction(fn_name, conv_type));
+      llvm::Function* conv_func = llvm::dyn_cast<llvm::Function>(
+          module_->getOrInsertFunction(fn_name, conv_type).getCallee());
       conv_func->setCallingConv(llvm::CallingConv::C);
       conv_func->setDoesNotThrow();
       conv_func->setOnlyAccessesArgMemory();
@@ -1324,8 +1341,8 @@ Status IrEmitter::HandleFft(HloInstruction* fft) {
                             ? runtime::kEigenFftSymbolName
                             : runtime::kEigenSingleThreadedFftSymbolName;
 
-  llvm::Function* fft_func = llvm::cast<llvm::Function>(
-      module_->getOrInsertFunction(fn_name, fft_type));
+  llvm::Function* fft_func = llvm::dyn_cast<llvm::Function>(
+      module_->getOrInsertFunction(fn_name, fft_type).getCallee());
   fft_func->setCallingConv(llvm::CallingConv::C);
   fft_func->setDoesNotThrow();
   fft_func->setOnlyAccessesInaccessibleMemOrArgMem();
@@ -2264,13 +2281,15 @@ Status IrEmitter::HandleCustomCall(HloInstruction* custom_call) {
         InBoundsGEP(operands_alloca, {b_.getInt64(i)});
     Store(operand_as_i8ptr, slot_in_operands_alloca);
   }
-  auto* custom_call_ir_function =
-      llvm::cast<llvm::Function>(module_->getOrInsertFunction(
-          AsStringRef(custom_call_target),
-          llvm::FunctionType::get(
-              /*Result=*/b_.getVoidTy(),
-              /*Params=*/{i8_ptr_type, operands_alloca->getType()},
-              /*isVarArg=*/false)));
+  auto* custom_call_ir_function = llvm::dyn_cast<llvm::Function>(
+      module_
+          ->getOrInsertFunction(
+              AsStringRef(custom_call_target),
+              llvm::FunctionType::get(
+                  /*Result=*/b_.getVoidTy(),
+                  /*Params=*/{i8_ptr_type, operands_alloca->getType()},
+                  /*isVarArg=*/false))
+          .getCallee());
 
   TF_RETURN_IF_ERROR(EmitTargetAddressForOp(custom_call));
   // Write the tuple table if the output is a tuple.
diff --git a/tensorflow/compiler/xla/service/cpu/ir_function.cc b/tensorflow/compiler/xla/service/cpu/ir_function.cc
index adfb8392bf..84a5b058cf 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_function.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_function.cc
@@ -266,9 +266,11 @@ Status EmitCallToParallelForkJoin(
       /*Params=*/compute_function_params,
       /*isVarArg=*/false);
 
-  llvm::Function* fork_join_func =
-      llvm::cast<llvm::Function>(module->getOrInsertFunction(
-          runtime::kParallelForkJoinSymbolName, fork_join_type));
+  llvm::Function* fork_join_func = llvm::dyn_cast<llvm::Function>(
+      module
+          ->getOrInsertFunction(runtime::kParallelForkJoinSymbolName,
+                                fork_join_type)
+          .getCallee());
   fork_join_func->setCallingConv(llvm::CallingConv::C);
   fork_join_func->setDoesNotThrow();
 
diff --git a/tensorflow/compiler/xla/service/cpu/tests/cpu_noalias_test.cc b/tensorflow/compiler/xla/service/cpu/tests/cpu_noalias_test.cc
index a7702c2aee..030bd41c2f 100644
--- a/tensorflow/compiler/xla/service/cpu/tests/cpu_noalias_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/tests/cpu_noalias_test.cc
@@ -75,8 +75,9 @@ TEST_F(CpuNoAliasTest, Concat) {
   // the buffers in the HLO module.  We'll inspect these loads to ensure that
   // they have the expected alias information.
   llvm::Module ir_module("test", context);
-  llvm::Function* func = llvm::cast<llvm::Function>(
-      ir_module.getOrInsertFunction("test_fn", llvm::Type::getVoidTy(context)));
+  llvm::Function* func = llvm::dyn_cast<llvm::Function>(
+      ir_module.getOrInsertFunction("test_fn", llvm::Type::getVoidTy(context))
+          .getCallee());
   llvm::BasicBlock* bb = llvm::BasicBlock::Create(context, "body", func);
   llvm::IRBuilder<> b(bb);
   auto* zero = llvm::ConstantInt::get(llvm::Type::getInt32Ty(context), 0);
diff --git a/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc
index 2ab754a471..ffd4214958 100644
--- a/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc
@@ -308,9 +308,11 @@ llvm::Value* GpuElementalIrEmitter::EmitDeviceFunctionCall(
       false);  // No variadic arguments.
 
   // Declares the callee if it is not declared already.
-  llvm::Function* callee = llvm::cast<llvm::Function>(
-      b_->GetInsertBlock()->getModule()->getOrInsertFunction(
-          llvm_ir::AsStringRef(callee_name), callee_type));
+  llvm::Function* callee = llvm::dyn_cast<llvm::Function>(
+      b_->GetInsertBlock()
+          ->getModule()
+          ->getOrInsertFunction(llvm_ir::AsStringRef(callee_name), callee_type)
+          .getCallee());
 
   for (auto attribute : attributes) {
     callee->addFnAttr(attribute);
-- 
GitLab


From 0b9429168e73df0fbec61cc9176913920d183ac2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 1 Feb 2019 10:18:49 -0800
Subject: [PATCH 1764/2345] Update ops-related pbtxt files.

PiperOrigin-RevId: 231993421
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 27 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 27 +++++++++++++++++++
 2 files changed, 54 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 941a9c07b7..c73ae0cc16 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -23561,6 +23561,33 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ExperimentalRebatchDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "num_workers"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "ExperimentalScanDataset"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index f77fd63c4c..2b75389be9 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -11326,6 +11326,33 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ExperimentalRebatchDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "num_workers"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "ExperimentalScanDataset"
   input_arg {
-- 
GitLab


From 9d0f11f5c1cdb591e30455c4495d310e73f2a559 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 1 Feb 2019 10:27:27 -0800
Subject: [PATCH 1765/2345] Add helper function for creating a constant scalar
 which is broadcastable

---
 .../tf2tensorrt/convert/convert_nodes.cc      | 62 ++++++++-----------
 .../tf2tensorrt/convert/convert_nodes.h       |  7 +++
 2 files changed, 34 insertions(+), 35 deletions(-)

diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
index bc193e4bdf..84a36a85cb 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
@@ -351,6 +351,25 @@ nvinfer1::ITensor* Converter::CreateConstantLayer(
   return trt_tensor;
 }
 
+tensorflow::Status Converter::CreateBroadcastableScalarConstant(
+    OpConverterParams* params, float value, const nvinfer1::Dims& dims,
+    const nvinfer1::ITensor** tensor) {
+  // In order to be broadcastable, the number of dims has to match.
+  nvinfer1::Dims broadcastable_dims(dims);
+  for (int i = 0; i < broadcastable_dims.nbDims; i++) {
+    broadcastable_dims.d[i] = 1;
+  }
+  TRT_ShapedWeights weights = params->weight_store->GetTempWeights(
+      tensorflow::DataType::DT_FLOAT, broadcastable_dims);
+  auto weights_ptr =
+      static_cast<float*>(const_cast<void*>(weights.GetValues()));
+  weights_ptr[0] = value;
+  *tensor = params->converter->CreateConstantLayer(weights, broadcastable_dims);
+  TFTRT_RETURN_ERROR_IF_NULLPTR(*tensor, params->node_def.name());
+  params->converter->ProvideQuantizationRange(*tensor, value, value);
+  return Status::OK();
+}
+
 inline bool DimsEqual(const nvinfer1::Dims& dim_l,
                       const nvinfer1::Dims& dim_r) {
   if (dim_l.nbDims != dim_r.nbDims) {
@@ -2455,23 +2474,10 @@ tensorflow::Status ConvertLeakyRelu(OpConverterParams* params) {
 
   // Input Tensor
   const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
-  // Create const for alpha. The constant has to have the same rank as the input
-  // in order for TRT to broadcast.
-  nvinfer1::Dims dims;
-  dims.nbDims = inputs.at(0).GetTrtDims().nbDims;
-  for (int i = 0; i < dims.nbDims; i++) {
-    dims.d[i] = 1;
-  }
-  TRT_ShapedWeights weights = params->weight_store->GetTempWeights(
-      tensorflow::DataType::DT_FLOAT, dims);
-  auto weights_ptr =
-      static_cast<float*>(const_cast<void*>(weights.GetValues()));
-  weights_ptr[0] = alpha;
-  nvinfer1::ITensor* const_alpha_tensor =
-      params->converter->CreateConstantLayer(weights, dims);
-  TFTRT_RETURN_ERROR_IF_NULLPTR(const_alpha_tensor, node_def.name());
-  params->converter->ProvideQuantizationRange(const_alpha_tensor, -alpha,
-                                              alpha);
+  // Create const for alpha.
+  const nvinfer1::ITensor* const_alpha_tensor = nullptr;
+  TF_RETURN_IF_ERROR(CreateBroadcastableScalarConstant(
+      params, alpha, tensor->getDimensions(), &const_alpha_tensor));
   // alpha * x
   nvinfer1::IElementWiseLayer* mul_layer =
       params->converter->network()->addElementWise(
@@ -2638,24 +2644,10 @@ tensorflow::Status ConvertRelu6(OpConverterParams* params) {
   params->converter->ProvideQuantizationRange(relu_layer->getOutput(0), 0.0f,
                                               6.0f);
 
-  // Create a constant layer to store the floating point weight i.e. 6.0f This
-  // tensor will be broadcasted uniformly during elementwise `min` operation.
-  // The constant has to have the same rank as the input in order for TRT to
-  // broadcast
-  nvinfer1::Dims dims;
-  dims.nbDims = relu_layer->getOutput(0)->getDimensions().nbDims;
-  for (int i = 0; i < dims.nbDims; i++) {
-    dims.d[i] = 1;
-  }
-  TRT_ShapedWeights weights = params->weight_store->GetTempWeights(
-      tensorflow::DataType::DT_FLOAT, dims);
-  auto weights_ptr =
-      static_cast<float*>(const_cast<void*>(weights.GetValues()));
-  weights_ptr[0] = 6.0f;
-  nvinfer1::ITensor* const6_tensor =
-      params->converter->CreateConstantLayer(weights, dims);
-  TFTRT_RETURN_ERROR_IF_NULLPTR(const6_tensor, node_def.name());
-  params->converter->ProvideQuantizationRange(const6_tensor, 0.0f, 6.0f);
+  // Create a constant layer to store the floating point weight i.e. 6.0f
+  const nvinfer1::ITensor* const6_tensor = nullptr;
+  TF_RETURN_IF_ERROR(CreateBroadcastableScalarConstant(
+      params, 6.0f, relu_layer->getOutput(0)->getDimensions(), &const6_tensor));
 
   // ElementWise Min Operation
   // Min op is a nop for INT8 execution path, as the input tensor
diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
index 00373051f5..89b9a0f645 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
@@ -492,6 +492,13 @@ class Converter {
   nvinfer1::ITensor* CreateConstantLayer(const TRT_ShapedWeights& weights,
                                          const nvinfer1::Dims& dims);
 
+  // Creates an IConstantLayer scalar which can be broadcasted against dims by
+  // IElementWise layer.
+  Status CreateBroadcastableScalarConstant(OpConverterParams* params,
+                                           float value,
+                                           const nvinfer1::Dims& dims,
+                                           const nvinfer1::ITensor** tensor);
+
  private:
   // Verify the provided batch_size is consistent with batch_size_ and update it
   // if necessary.
-- 
GitLab


From 8217d1d4eecc311e78b6d785f62a171e83d5b8fc Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 1 Feb 2019 10:42:33 -0800
Subject: [PATCH 1766/2345] Fix compilation issues

---
 .../compiler/tf2tensorrt/convert/convert_nodes.cc      | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
index 84a36a85cb..1f5b961af4 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
@@ -2476,12 +2476,13 @@ tensorflow::Status ConvertLeakyRelu(OpConverterParams* params) {
   const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
   // Create const for alpha.
   const nvinfer1::ITensor* const_alpha_tensor = nullptr;
-  TF_RETURN_IF_ERROR(CreateBroadcastableScalarConstant(
+  TF_RETURN_IF_ERROR(params->converter->CreateBroadcastableScalarConstant(
       params, alpha, tensor->getDimensions(), &const_alpha_tensor));
   // alpha * x
   nvinfer1::IElementWiseLayer* mul_layer =
       params->converter->network()->addElementWise(
-          *const_cast<nvinfer1::ITensor*>(tensor), *const_alpha_tensor,
+          *const_cast<nvinfer1::ITensor*>(tensor),
+          *const_cast<nvinfer1::ITensor*>(const_alpha_tensor),
           nvinfer1::ElementWiseOperation::kPROD);
   TFTRT_RETURN_ERROR_IF_NULLPTR(mul_layer, node_def.name());
   // max(x, alpha * x)
@@ -2646,7 +2647,7 @@ tensorflow::Status ConvertRelu6(OpConverterParams* params) {
 
   // Create a constant layer to store the floating point weight i.e. 6.0f
   const nvinfer1::ITensor* const6_tensor = nullptr;
-  TF_RETURN_IF_ERROR(CreateBroadcastableScalarConstant(
+  TF_RETURN_IF_ERROR(params->converter->CreateBroadcastableScalarConstant(
       params, 6.0f, relu_layer->getOutput(0)->getDimensions(), &const6_tensor));
 
   // ElementWise Min Operation
@@ -2655,7 +2656,8 @@ tensorflow::Status ConvertRelu6(OpConverterParams* params) {
   nvinfer1::IElementWiseLayer* relu6_layer =
       params->converter->network()->addElementWise(
           *const_cast<nvinfer1::ITensor*>(relu_layer->getOutput(0)),
-          *const6_tensor, nvinfer1::ElementWiseOperation::kMIN);
+          *const_cast<nvinfer1::ITensor*>(const6_tensor),
+          nvinfer1::ElementWiseOperation::kMIN);
   TFTRT_RETURN_ERROR_IF_NULLPTR(relu6_layer, node_def.name());
   nvinfer1::ITensor* output_tensor = relu6_layer->getOutput(0);
   params->converter->ProvideQuantizationRange(output_tensor, 0.0f, 6.0f);
-- 
GitLab


From 98e0bbd0d2f4fe504c0f0834d8fab53788308aed Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 1 Feb 2019 10:45:36 -0800
Subject: [PATCH 1767/2345] Fix compilation issues

---
 tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
index 1f5b961af4..c36260ccf8 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
@@ -366,7 +366,8 @@ tensorflow::Status Converter::CreateBroadcastableScalarConstant(
   weights_ptr[0] = value;
   *tensor = params->converter->CreateConstantLayer(weights, broadcastable_dims);
   TFTRT_RETURN_ERROR_IF_NULLPTR(*tensor, params->node_def.name());
-  params->converter->ProvideQuantizationRange(*tensor, value, value);
+  params->converter->ProvideQuantizationRange(
+      const_cast<nvinfer1::ITensor*>(*tensor), value, value);
   return Status::OK();
 }
 
-- 
GitLab


From c78d300508f31a7ca8680fdcee90f5484019ca64 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 1 Feb 2019 10:50:05 -0800
Subject: [PATCH 1768/2345] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 231999432

---
 tensorflow/go/op/wrappers.go | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index e281191ab5..05b8e6fd19 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -22887,6 +22887,34 @@ func SparseSliceGrad(scope *Scope, backprop_val_grad tf.Output, input_indices tf
 	return op.Output(0)
 }
 
+// Creates a dataset that changes the batch size.
+//
+// Creates a dataset that changes the batch size of the dataset to current batch
+// size // num_workers.
+//
+// Arguments:
+//	input_dataset: A variant tensor representing the input dataset.
+//	num_workers: A scalar representing the number of workers to distribute this batch across. As
+// a result of this transformation the current batch size would end up being
+// divided  by this parameter.
+//
+//
+func ExperimentalRebatchDataset(scope *Scope, input_dataset tf.Output, num_workers tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalRebatchDataset",
+		Input: []tf.Input{
+			input_dataset, num_workers,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Computes the gradient of the sigmoid of `x` wrt its input.
 //
 // Specifically, `grad = dy * y * (1 - y)`, where `y = sigmoid(x)`, and
-- 
GitLab


From 09e3397fdd158dfdcc7c1d8cb5c3e607253b81ac Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Fri, 1 Feb 2019 11:31:39 -0800
Subject: [PATCH 1769/2345] Internal change.

PiperOrigin-RevId: 232007932
---
 tensorflow/BUILD | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 5a8b97ede9..6bc8403d12 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -392,17 +392,7 @@ config_setting(
 
 package_group(
     name = "internal",
-    packages = [
-        "-//third_party/tensorflow/python/estimator",
-        "//learning/deepmind/...",
-        "//learning/meta_rank/...",
-        "//platforms/performance/autograppler/...",
-        "//tensorflow/...",
-        "//tensorflow_estimator/contrib/...",
-        "//tensorflow_fold/llgtm/...",
-        "//tensorflow_text/...",
-        "//third_party/py/tensor2tensor/...",
-    ],
+    packages = ["//tensorflow/..."],
 )
 
 load(
-- 
GitLab


From de3b376d24e6397e83588162dc4c21310332a57a Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Fri, 1 Feb 2019 11:46:10 -0800
Subject: [PATCH 1770/2345] [TF:XLA] Bump open source llvm revision to r352865

PiperOrigin-RevId: 232010757
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 1165a1a151..93e73df883 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -514,11 +514,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "48699c52e64e428388b0c7d2daa5197864f1f2425608a89fc15d494bf4bab0e0",
-        strip_prefix = "llvm-66b8f683ab6b3be52fe606a0db0f377f9f66e170",
+        sha256 = "ec524f6c8e4d7514b1753131e844c73df4cef1a22d8773b0e43ccd9f813bbd48",
+        strip_prefix = "llvm-ba3936b0ea89935ab8ea943c1b372d69923691d9",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/66b8f683ab6b3be52fe606a0db0f377f9f66e170.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/66b8f683ab6b3be52fe606a0db0f377f9f66e170.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/ba3936b0ea89935ab8ea943c1b372d69923691d9.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/ba3936b0ea89935ab8ea943c1b372d69923691d9.tar.gz",
         ],
     )
 
-- 
GitLab


From 97866b03d5c7180af1d843ec20be6ae8a3802550 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Fri, 1 Feb 2019 11:58:43 -0800
Subject: [PATCH 1771/2345] Fix bug in AutomaticControlDependencies.

Prior to this change, AutomaticControlDependencies didn't correctly
handle an op that receives the same resource tensor twice as input.

PiperOrigin-RevId: 232012900
---
 .../python/framework/auto_control_deps.py     | 39 +++++++++++--------
 .../framework/auto_control_deps_test.py       | 15 +++++++
 2 files changed, 37 insertions(+), 17 deletions(-)

diff --git a/tensorflow/python/framework/auto_control_deps.py b/tensorflow/python/framework/auto_control_deps.py
index da76a84e55..6210010cff 100644
--- a/tensorflow/python/framework/auto_control_deps.py
+++ b/tensorflow/python/framework/auto_control_deps.py
@@ -249,23 +249,28 @@ class AutomaticControlDependencies(object):
         ops_which_must_run = set([op])
         continue
       found_resource = False
-      for inp in op.inputs:
-        if inp.dtype == dtypes_module.resource:
-          found_resource = True
-          # Deal with switches, finally.
-          if inp.op.type == "Switch":
-            self._process_switch(inp.op, ops_which_must_run,
-                                 last_op_using_resource_tensor,
-                                 merge_for_resource)
-          # Ensure uses of resources are serialized
-          if inp in last_op_using_resource_tensor:
-            if (last_op_using_resource_tensor[inp]._control_flow_context  # pylint: disable=protected-access
-                is op._control_flow_context):  # pylint: disable=protected-access
-              control_inputs.add(last_op_using_resource_tensor[inp])
-          # Ensure merges happen after the closing of a cond block
-          if inp in merge_for_resource:
-            merge_for_resource[inp]._add_control_input(op)  # pylint: disable=protected-access
-          last_op_using_resource_tensor[inp] = op
+      # Check for any resource inputs. If we find any, we update control_inputs
+      # and last_op_using_resource_tensor. Note that we dedup op.inputs in case
+      # op receives the same resource tensor twice as input, which would result
+      # in op getting a control dependency on itself.
+      for inp in set(op.inputs):
+        if inp.dtype != dtypes_module.resource:
+          continue
+        found_resource = True
+        # Deal with switches, finally.
+        if inp.op.type == "Switch":
+          self._process_switch(inp.op, ops_which_must_run,
+                               last_op_using_resource_tensor,
+                               merge_for_resource)
+        # Ensure uses of resources are serialized
+        if inp in last_op_using_resource_tensor:
+          if (last_op_using_resource_tensor[inp]._control_flow_context  # pylint: disable=protected-access
+              is op._control_flow_context):  # pylint: disable=protected-access
+            control_inputs.add(last_op_using_resource_tensor[inp])
+        # Ensure merges happen after the closing of a cond block
+        if inp in merge_for_resource:
+          merge_for_resource[inp]._add_control_input(op)  # pylint: disable=protected-access
+        last_op_using_resource_tensor[inp] = op
       if (op.op_def.is_stateful and op.type not in ASYNC_STATEFUL_OPS
           and not found_resource and op._control_flow_context is None):  # pylint: disable=protected-access
         if None in last_op_using_resource_tensor:
diff --git a/tensorflow/python/framework/auto_control_deps_test.py b/tensorflow/python/framework/auto_control_deps_test.py
index d81adef26a..2c25ab133b 100644
--- a/tensorflow/python/framework/auto_control_deps_test.py
+++ b/tensorflow/python/framework/auto_control_deps_test.py
@@ -19,6 +19,7 @@ from __future__ import print_function
 
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
+from tensorflow.python.eager import def_function
 from tensorflow.python.eager import function
 from tensorflow.python.framework import auto_control_deps as acd
 from tensorflow.python.framework import constant_op
@@ -281,6 +282,20 @@ class AutomaticControlDependenciesTest(test.TestCase):
     train()
     self.assertEqual(v.numpy(), -1.0)
 
+  def testRepeatedResourceInput(self):
+    var = resource_variable_ops.ResourceVariable(1.0)
+
+    @def_function.function
+    def inner(var1, var2):
+      return (resource_variable_ops.read_variable_op(var1, dtypes.float32) +
+              resource_variable_ops.read_variable_op(var2, dtypes.float32))
+
+    @def_function.function
+    def outer():
+      return inner(var.handle, var.handle)
+
+    self.assertEqual(self.evaluate(outer()), 2.0)
+
 
 if __name__ == '__main__':
   ops.enable_eager_execution()
-- 
GitLab


From f291fa19e7d9a3e73c7895a073a6acbf2dd7e15d Mon Sep 17 00:00:00 2001
From: Nick Felt <nickfelt@google.com>
Date: Fri, 1 Feb 2019 12:00:35 -0800
Subject: [PATCH 1772/2345] Make converter handle tf.flags as a module
 deprecation warning

PiperOrigin-RevId: 232013230
---
 tensorflow/tools/compatibility/ast_edits.py          |  4 ++--
 tensorflow/tools/compatibility/tf_upgrade_v2.py      | 10 ++++++----
 tensorflow/tools/compatibility/tf_upgrade_v2_test.py |  8 ++++++++
 3 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/tensorflow/tools/compatibility/ast_edits.py b/tensorflow/tools/compatibility/ast_edits.py
index 7c11cc17b2..ed388b83eb 100644
--- a/tensorflow/tools/compatibility/ast_edits.py
+++ b/tensorflow/tools/compatibility/ast_edits.py
@@ -116,8 +116,8 @@ class APIChangeSpec(object):
   * `function_warnings`: maps full names of functions to warnings that will be
     printed out if the function is used. (e.g. tf.nn.convolution())
   * `function_transformers`: maps function names to custom handlers
-  * `leftover_warnings`: These warnings are printed if a matching Attribute
-    still exists after all other transformations have run.
+  * `module_deprecations`: maps module names to warnings that will be printed
+    if the module is still used after all other transformations have run
 
   For an example, see `TFAPIChangeSpec`.
   """
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index aca4cc3d53..b774acd6d2 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -824,6 +824,11 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "the required code."
     )
 
+    flags_warning = (
+        ast_edits.ERROR,
+        "tf.flags has been removed, please use the argparse or absl"
+        " modules if you need command line parsing.")
+
     decay_function_comment = (
         ast_edits.INFO,
         "To use learning rate decay schedules with TensorFlow 2.0, switch to "
@@ -973,10 +978,6 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             assert_rank_comment,
         "tf.debugging.assert_rank_in":
             assert_rank_comment,
-        "tf.flags": (
-            ast_edits.ERROR,
-            "tf.flags has been removed, please use the argparse or absl"
-            " modules if you need command line parsing."),
         "tf.train.exponential_decay":
             decay_function_comment,
         "tf.train.piecewise_constant_decay":
@@ -1321,6 +1322,7 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
 
     self.module_deprecations = {
         "tf.contrib": contrib_warning,
+        "tf.flags": flags_warning,
     }
 
 
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index c5459feb5c..7668ba2260 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -1159,6 +1159,14 @@ def _log_prob(self, x):
     _, _, _, new_text = self._upgrade(text)
     self.assertEqual(expected, new_text)
 
+  def test_flags_bare(self):
+    _, _, errors, _ = self._upgrade("tf.flags")
+    self.assertIn("tf.flags has been removed", errors[0])
+
+  def test_flags_flags(self):
+    _, _, errors, _ = self._upgrade("tf.flags.FLAGS")
+    self.assertIn("tf.flags has been removed", errors[0])
+
 
 class TestUpgradeFiles(test_util.TensorFlowTestCase):
 
-- 
GitLab


From 28958d61baef4c828aec6dbc02ce878075701a4b Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Fri, 1 Feb 2019 12:17:51 -0800
Subject: [PATCH 1773/2345] Move TF-TRT python APIs to
 python/compiler/tensorrt.

PiperOrigin-RevId: 232016402
---
 tensorflow/contrib/tensorrt/BUILD             | 152 +------
 tensorflow/contrib/tensorrt/__init__.py       |  18 +-
 .../contrib/tensorrt/python/__init__.py       |   6 -
 .../contrib/tensorrt/python/trt_convert.py    | 409 +----------------
 .../contrib/tensorrt/test/test_tftrt.py       | 287 ------------
 tensorflow/python/compiler/tensorrt/BUILD     | 176 +++++++
 .../compiler}/tensorrt/README.md              |   0
 .../python/compiler/tensorrt/__init__.py      |  42 ++
 .../compiler}/tensorrt/test/base_test.py      |   4 +-
 .../tensorrt/test/batch_matmul_test.py        |   2 +-
 .../tensorrt/test/biasadd_matmul_test.py      |   2 +-
 .../binary_tensor_weight_broadcast_test.py    |   2 +-
 .../tensorrt/test/concatenation_test.py       |   2 +-
 .../tensorrt/test/const_broadcast_test.py     |   2 +-
 .../compiler}/tensorrt/test/conv2d_test.py    |   2 +-
 .../test/dynamic_input_shapes_test.py         |   2 +-
 .../tensorrt/test/identity_output_test.py     |   2 +-
 .../compiler}/tensorrt/test/int32_test.py     |   2 +-
 .../compiler}/tensorrt/test/lru_cache_test.py |   2 +-
 .../tensorrt/test/memory_alignment_test.py    |   2 +-
 .../multi_connection_neighbor_engine_test.py  |   2 +-
 .../tensorrt/test/neighboring_engine_test.py  |   2 +-
 .../tensorrt/test/quantization_mnist_test.py  |   4 +-
 .../tensorrt/test/quantization_test.py        |   4 +-
 .../compiler}/tensorrt/test/rank_two_test.py  |   2 +-
 .../tensorrt/test/reshape_transpose_test.py   |   2 +-
 .../tensorrt/test/testdata/checkpoint         |   0
 .../model.ckpt-46900.data-00000-of-00001      | Bin
 .../test/testdata/model.ckpt-46900.index      | Bin
 .../test/tf_trt_integration_test_base.py      |   2 +-
 .../compiler}/tensorrt/test/topk_test.py      |   2 +-
 .../compiler}/tensorrt/test/unary_test.py     |   2 +-
 .../tensorrt/test/vgg_block_nchw_test.py      |   2 +-
 .../compiler}/tensorrt/test/vgg_block_test.py |   2 +-
 .../compiler}/tensorrt/trt_conversion.i       |   0
 .../python/compiler/tensorrt/trt_convert.py   | 428 ++++++++++++++++++
 .../compiler/tensorrt}/trt_convert_test.py    |   2 +-
 37 files changed, 697 insertions(+), 875 deletions(-)
 delete mode 100644 tensorflow/contrib/tensorrt/test/test_tftrt.py
 create mode 100644 tensorflow/python/compiler/tensorrt/BUILD
 rename tensorflow/{contrib => python/compiler}/tensorrt/README.md (100%)
 create mode 100644 tensorflow/python/compiler/tensorrt/__init__.py
 rename tensorflow/{contrib => python/compiler}/tensorrt/test/base_test.py (98%)
 rename tensorflow/{contrib => python/compiler}/tensorrt/test/batch_matmul_test.py (97%)
 rename tensorflow/{contrib => python/compiler}/tensorrt/test/biasadd_matmul_test.py (98%)
 rename tensorflow/{contrib => python/compiler}/tensorrt/test/binary_tensor_weight_broadcast_test.py (96%)
 rename tensorflow/{contrib => python/compiler}/tensorrt/test/concatenation_test.py (97%)
 rename tensorflow/{contrib => python/compiler}/tensorrt/test/const_broadcast_test.py (96%)
 rename tensorflow/{contrib => python/compiler}/tensorrt/test/conv2d_test.py (98%)
 rename tensorflow/{contrib => python/compiler}/tensorrt/test/dynamic_input_shapes_test.py (97%)
 rename tensorflow/{contrib => python/compiler}/tensorrt/test/identity_output_test.py (96%)
 rename tensorflow/{contrib => python/compiler}/tensorrt/test/int32_test.py (97%)
 rename tensorflow/{contrib => python/compiler}/tensorrt/test/lru_cache_test.py (96%)
 rename tensorflow/{contrib => python/compiler}/tensorrt/test/memory_alignment_test.py (96%)
 rename tensorflow/{contrib => python/compiler}/tensorrt/test/multi_connection_neighbor_engine_test.py (96%)
 rename tensorflow/{contrib => python/compiler}/tensorrt/test/neighboring_engine_test.py (96%)
 rename tensorflow/{contrib => python/compiler}/tensorrt/test/quantization_mnist_test.py (98%)
 rename tensorflow/{contrib => python/compiler}/tensorrt/test/quantization_test.py (97%)
 rename tensorflow/{contrib => python/compiler}/tensorrt/test/rank_two_test.py (97%)
 rename tensorflow/{contrib => python/compiler}/tensorrt/test/reshape_transpose_test.py (98%)
 rename tensorflow/{contrib => python/compiler}/tensorrt/test/testdata/checkpoint (100%)
 rename tensorflow/{contrib => python/compiler}/tensorrt/test/testdata/model.ckpt-46900.data-00000-of-00001 (100%)
 rename tensorflow/{contrib => python/compiler}/tensorrt/test/testdata/model.ckpt-46900.index (100%)
 rename tensorflow/{contrib => python/compiler}/tensorrt/test/tf_trt_integration_test_base.py (99%)
 rename tensorflow/{contrib => python/compiler}/tensorrt/test/topk_test.py (97%)
 rename tensorflow/{contrib => python/compiler}/tensorrt/test/unary_test.py (97%)
 rename tensorflow/{contrib => python/compiler}/tensorrt/test/vgg_block_nchw_test.py (96%)
 rename tensorflow/{contrib => python/compiler}/tensorrt/test/vgg_block_test.py (96%)
 rename tensorflow/{contrib => python/compiler}/tensorrt/trt_conversion.i (100%)
 create mode 100644 tensorflow/python/compiler/tensorrt/trt_convert.py
 rename tensorflow/{contrib/tensorrt/python => python/compiler/tensorrt}/trt_convert_test.py (99%)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index e13edd1b5e..91b6d2614a 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -11,22 +11,14 @@ exports_files(["LICENSE"])
 
 load(
     "//tensorflow:tensorflow.bzl",
-    "tf_copts",
     "tf_cuda_library",
     "tf_custom_op_library_additional_deps",
 )
-load("//tensorflow:tensorflow.bzl", "cuda_py_test")
-load("//tensorflow:tensorflow.bzl", "cuda_py_tests")
-load("//tensorflow:tensorflow.bzl", "tf_py_wrap_cc")
 load(
     "@local_config_tensorrt//:build_defs.bzl",
     "if_tensorrt",
 )
 
-exports_files(glob([
-    "test/testdata/*",
-]))
-
 tf_cuda_library(
     name = "trt_shape_function",
     srcs = ["shape_fn/trt_shfn.cc"],
@@ -45,151 +37,11 @@ py_library(
     srcs = [
         "__init__.py",
         "python/__init__.py",
+        "python/trt_convert.py",
     ],
     srcs_version = "PY2AND3",
     deps = [
-        ":tf_trt_integration_test_base",
-        ":trt_convert_py",
-        ":trt_ops_py",
-        "//tensorflow/python:errors",
-    ],
-)
-
-py_library(
-    name = "trt_ops_py",
-    srcs_version = "PY2AND3",
-    deps = [
-        "//tensorflow/compiler/tf2tensorrt:trt_ops",
-        "//tensorflow/compiler/tf2tensorrt:trt_ops_loader",
-    ],
-)
-
-py_library(
-    name = "trt_convert_py",
-    srcs = ["python/trt_convert.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":wrap_conversion",
-        "//tensorflow/python:graph_util",
-        "//tensorflow/python:session",
-        "//tensorflow/python:tf_optimizer",
-        "//tensorflow/python/saved_model:builder",
-        "//tensorflow/python/saved_model:loader",
-        "//tensorflow/python/saved_model:tag_constants",
-    ],
-)
-
-# TODO(aaroey): this wrapper has been causing troubles of double linking, so
-# either get rid of it, or split to make it contain minimum dependencies.
-tf_py_wrap_cc(
-    name = "wrap_conversion",
-    srcs = ["trt_conversion.i"],
-    copts = tf_copts(),
-    swig_includes = [
-        "//tensorflow/python:platform/base.i",
-    ],
-    deps = [
-        "//tensorflow/compiler/tf2tensorrt:test_utils",
-        "//tensorflow/compiler/tf2tensorrt:trt_conversion",
-        "//tensorflow/compiler/tf2tensorrt:trt_op_kernels",
-        "//third_party/python_runtime:headers",
-    ],
-)
-
-py_library(
-    name = "tf_trt_integration_test_base",
-    srcs = ["test/tf_trt_integration_test_base.py"],
-    deps = [
-        ":trt_convert_py",
-        ":trt_ops_py",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_test_lib",
-    ],
-)
-
-cuda_py_test(
-    name = "trt_convert_test",
-    srcs = ["python/trt_convert_test.py"],
-    additional_deps = [
-        ":trt_convert_py",
-        ":trt_ops_py",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:graph_util",
-        "//tensorflow/python/saved_model:builder",
-        "//tensorflow/python/saved_model:loader",
-        "//tensorflow/python/saved_model:signature_constants",
-        "//tensorflow/python/saved_model:signature_def_utils",
-        "//tensorflow/python/saved_model:tag_constants",
-        "//tensorflow/python/saved_model:utils",
-        "//tensorflow/python/tools:freeze_graph_lib",
-        "//tensorflow/python/tools:saved_model_utils",
-    ],
-    tags = [
-        "no_cuda_on_cpu_tap",
-        "no_windows",
-        "nomac",
-    ],
-)
-
-cuda_py_tests(
-    name = "tf_trt_integration_test",
-    srcs = [
-        "test/base_test.py",
-        "test/batch_matmul_test.py",
-        "test/biasadd_matmul_test.py",
-        "test/binary_tensor_weight_broadcast_test.py",
-        "test/concatenation_test.py",
-        "test/const_broadcast_test.py",
-        "test/conv2d_test.py",
-        "test/dynamic_input_shapes_test.py",
-        "test/identity_output_test.py",
-        "test/int32_test.py",
-        "test/lru_cache_test.py",
-        "test/memory_alignment_test.py",
-        "test/multi_connection_neighbor_engine_test.py",
-        "test/neighboring_engine_test.py",
-        "test/quantization_test.py",
-        "test/rank_two_test.py",
-        "test/reshape_transpose_test.py",
-        "test/topk_test.py",
-        "test/unary_test.py",
-        "test/vgg_block_nchw_test.py",
-        "test/vgg_block_test.py",
-    ],
-    additional_deps = [
-        ":tf_trt_integration_test_base",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_test_lib",
-    ],
-    tags = [
-        "no_cuda_on_cpu_tap",
-        "no_windows",
-        "nomac",
-    ],
-)
-
-cuda_py_test(
-    name = "quantization_mnist_test",
-    srcs = ["test/quantization_mnist_test.py"],
-    additional_deps = [
-        ":tf_trt_integration_test_base",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python/keras:keras",
-        "//tensorflow/python/estimator:estimator",
-    ],
-    data = [
-        "test/testdata/checkpoint",
-        "test/testdata/model.ckpt-46900.data-00000-of-00001",
-        "test/testdata/model.ckpt-46900.index",
-    ],
-    tags = [
-        "no_cuda_on_cpu_tap",
-        "no_pip",
-        "no_tap",  # It is not able to download the mnist data.
-        "no_windows",
-        "nomac",
+        "//tensorflow/python/compiler/tensorrt:init_py",
     ],
 )
 
diff --git a/tensorflow/contrib/tensorrt/__init__.py b/tensorflow/contrib/tensorrt/__init__.py
index 140ad48282..fd551d70b4 100644
--- a/tensorflow/contrib/tensorrt/__init__.py
+++ b/tensorflow/contrib/tensorrt/__init__.py
@@ -18,18 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.python.framework import errors
-
-# pylint: disable=unused-import,wildcard-import,g-import-not-at-top
-try:
-  from tensorflow.contrib.tensorrt.python import *
-except errors.NotFoundError as e:
-  no_trt_message = (
-      '**** Failed to initialize TensorRT. This is either because the TensorRT'
-      ' installation path is not in LD_LIBRARY_PATH, or because you do not have'
-      ' it installed. If not installed, please go to'
-      ' https://developer.nvidia.com/tensorrt to download and install'
-      ' TensorRT ****')
-  print(no_trt_message)
-  raise e
-# pylint: enable=unused-import,wildcard-import,g-import-not-at-top
+# pylint: disable=unused-import,wildcard-import
+from tensorflow.contrib.tensorrt.python import *
+# pylint: enable=unused-import,wildcard-import
diff --git a/tensorflow/contrib/tensorrt/python/__init__.py b/tensorflow/contrib/tensorrt/python/__init__.py
index 75490aecfb..0cae401023 100644
--- a/tensorflow/contrib/tensorrt/python/__init__.py
+++ b/tensorflow/contrib/tensorrt/python/__init__.py
@@ -19,12 +19,6 @@ from __future__ import division
 from __future__ import print_function
 
 # pylint: disable=unused-import,line-too-long
-from tensorflow.compiler.tf2tensorrt.python.ops import trt_ops
-from tensorflow.contrib.tensorrt.python.trt_convert import add_test_value
 from tensorflow.contrib.tensorrt.python.trt_convert import calib_graph_to_infer_graph
-from tensorflow.contrib.tensorrt.python.trt_convert import clear_test_values
 from tensorflow.contrib.tensorrt.python.trt_convert import create_inference_graph
-from tensorflow.contrib.tensorrt.python.trt_convert import enable_test_value
-from tensorflow.contrib.tensorrt.python.trt_convert import get_test_value
-from tensorflow.contrib.tensorrt.python.trt_convert import is_tensorrt_enabled
 # pylint: enable=unused-import,line-too-long
diff --git a/tensorflow/contrib/tensorrt/python/trt_convert.py b/tensorflow/contrib/tensorrt/python/trt_convert.py
index 49d72232aa..42bf1eda51 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert.py
@@ -18,155 +18,14 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import six as _six
-# pylint: disable=unused-import,line-too-long
-from tensorflow.contrib.tensorrt.wrap_conversion import add_test_value
-from tensorflow.contrib.tensorrt.wrap_conversion import calib_convert
-from tensorflow.contrib.tensorrt.wrap_conversion import clear_test_values
-from tensorflow.contrib.tensorrt.wrap_conversion import enable_test_value
-from tensorflow.contrib.tensorrt.wrap_conversion import get_linked_tensorrt_version
-from tensorflow.contrib.tensorrt.wrap_conversion import get_loaded_tensorrt_version
-from tensorflow.contrib.tensorrt.wrap_conversion import get_test_value
-from tensorflow.contrib.tensorrt.wrap_conversion import is_tensorrt_enabled
-# pylint: enable=unused-import,line-too-long
-from tensorflow.core.framework import graph_pb2
-from tensorflow.core.protobuf import config_pb2
-from tensorflow.core.protobuf import meta_graph_pb2
-from tensorflow.core.protobuf import rewriter_config_pb2
-from tensorflow.python.client import session
-from tensorflow.python.framework import errors_impl as _impl
-from tensorflow.python.framework import graph_util
-from tensorflow.python.framework import importer
-from tensorflow.python.framework import ops
-from tensorflow.python.grappler import tf_optimizer
-from tensorflow.python.platform import tf_logging
-from tensorflow.python.saved_model import builder
-from tensorflow.python.saved_model import loader_impl
-from tensorflow.python.saved_model import tag_constants
-from tensorflow.python.training import saver
-
-
-def _to_bytes(s):
-  """Encode s if it is a sequence of chars."""
-  if isinstance(s, _six.text_type):
-    return s.encode("utf-8", errors="surrogateescape")
-  return s
-
-
-def _to_string(s):
-  """Decode s if it is a sequence of bytes."""
-  if isinstance(s, _six.binary_type):
-    return s.decode("utf-8")
-  return s
-
-
-class TrtPrecisionMode(object):
-  FP32 = "FP32"
-  FP16 = "FP16"
-  INT8 = "INT8"
-
-  @staticmethod
-  def supported_precision_modes():
-    return [TrtPrecisionMode.FP32, TrtPrecisionMode.FP16, TrtPrecisionMode.INT8]
-
-
-def get_tensorrt_rewriter_config(rewriter_config=None,
-                                 max_batch_size=1,
-                                 max_workspace_size_bytes=2 << 20,
-                                 precision_mode=TrtPrecisionMode.FP32,
-                                 minimum_segment_size=3,
-                                 is_dynamic_op=False,
-                                 maximum_cached_engines=1,
-                                 cached_engine_batches=None,
-                                 use_calibration=True):
-  """Returns a RewriterConfig proto for TRT transformation.
-
-  Args:
-    rewriter_config: a template RewriterConfig proto used to create a
-      TRT-enabled RewriterConfig. If None, it will use a default one.
-    max_batch_size: max size for the input batch
-    max_workspace_size_bytes: the maximum GPU temporary memory which the TRT
-      engine can use at execution time. This corresponds to the 'workspaceSize'
-      parameter of nvinfer1::IBuilder::setMaxWorkspaceSize().
-    precision_mode: one of TrtPrecisionMode.supported_precision_modes().
-    minimum_segment_size: the minimum number of nodes required for a subgraph to
-      be replaced by TRTEngineOp.
-    is_dynamic_op: whether to generate dynamic TRT ops which will build the TRT
-      network and engine at run time.
-    maximum_cached_engines: max number of cached TRT engines in dynamic TRT ops.
-      If the number of cached engines is already at max but none of them can
-      serve the input, the TRTEngineOp will fall back to run the TF function
-      based on which the TRTEngineOp is created.
-    cached_engine_batches: a list of batch sizes used to create cached
-      engines, only used when is_dynamic_op is True. The length of the list
-      should be <= maximum_cached_engines, and the dynamic TRT op will
-      use this list to determine the batch sizes of the cached engines, instead
-      of making the decision on the fly. This is useful when we know the most
-      common batch size(s) the application is going to generate.
-    use_calibration: this argument is ignored if precision_mode is not INT8. If
-      set to True, a calibration graph will be created to calibrate the missing
-      ranges. The calibration graph must be converted to an inference graph
-      using calib_graph_to_infer_graph() after running calibration. if set to
-      False, quantization nodes will be expected for every tensor in the graph
-      (exlcuding those which will be fused). If a range is missing, an error
-      will occur. Please note that accuracy may be negatively affected if there
-      is a mismatch between which tensors TRT quantizes and which tensors were
-      trained with fake quantization.
-
-  Returns:
-    A RewriterConfig proto which sets a TensorRTOptimizer to run Grappler.
-
-  Raises:
-    TypeError: if any of the parameters are of unexpected type.
-    ValueError: if any of the parameters are of unexpected value.
-  """
-  if rewriter_config is not None and not isinstance(
-      rewriter_config, rewriter_config_pb2.RewriterConfig):
-    raise TypeError("rewriter_config should be a RewriterConfig proto.")
-
-  rewriter_config_with_trt = rewriter_config_pb2.RewriterConfig()
-  if rewriter_config is None:
-    # Layout optimizer may add Const nodes followed by Reshape nodes, thus we
-    # need to run constant folding again.
-    rewriter_config_with_trt.optimizers.extend(
-        ["constfold", "layout", "constfold"])
-    rewriter_config_with_trt.meta_optimizer_iterations = (
-        rewriter_config_pb2.RewriterConfig.ONE)
-  else:
-    rewriter_config_with_trt.CopyFrom(rewriter_config)
-
-  if precision_mode.upper() not in TrtPrecisionMode.supported_precision_modes():
-    raise ValueError(("precision mode '{}' is not supported."
-                      "It should be one of {}").format(
-                          precision_mode,
-                          TrtPrecisionMode.supported_precision_modes))
-
-  optimizer = rewriter_config_with_trt.custom_optimizers.add()
-  optimizer.name = "TensorRTOptimizer"
-  optimizer.parameter_map["minimum_segment_size"].i = minimum_segment_size
-  optimizer.parameter_map["max_batch_size"].i = max_batch_size
-  optimizer.parameter_map["is_dynamic_op"].b = is_dynamic_op
-  optimizer.parameter_map[
-      "max_workspace_size_bytes"].i = max_workspace_size_bytes
-  optimizer.parameter_map["precision_mode"].s = _to_bytes(precision_mode)
-  optimizer.parameter_map["maximum_cached_engines"].i = maximum_cached_engines
-  if cached_engine_batches:
-    if not isinstance(cached_engine_batches, list):
-      raise TypeError("cached_engine_batches should be a list.")
-    if len(cached_engine_batches) > maximum_cached_engines:
-      raise ValueError("cached_engine_batches should not contain more than "
-                       "maximum_cached_engines items.")
-    optimizer.parameter_map["cached_engine_batches"].list.i.extend(
-        cached_engine_batches)
-  optimizer.parameter_map["use_calibration"].b = use_calibration
-  return rewriter_config_with_trt
+from tensorflow.python.compiler.tensorrt import trt_convert
 
 
 def create_inference_graph(input_graph_def,
                            outputs,
                            max_batch_size=1,
                            max_workspace_size_bytes=2 << 20,
-                           precision_mode=TrtPrecisionMode.FP32,
+                           precision_mode=trt_convert.TrtPrecisionMode.FP32,
                            minimum_segment_size=3,
                            is_dynamic_op=False,
                            maximum_cached_engines=1,
@@ -176,253 +35,23 @@ def create_inference_graph(input_graph_def,
                            input_saved_model_tags=None,
                            output_saved_model_dir=None,
                            session_config=None):
-  """Python wrapper for the TRT transformation.
-
-  Args:
-    input_graph_def: a GraphDef object containing a model to be transformed. If
-      set to None, the graph will be read from the SavedModel loaded from
-      input_saved_model_dir.
-    outputs: list of tensors or node names for the model outputs. Only used when
-      input_graph_def is not None.
-    max_batch_size: max size for the input batch.
-    max_workspace_size_bytes: the maximum GPU temporary memory which the TRT
-      engine can use at execution time. This corresponds to the 'workspaceSize'
-      parameter of nvinfer1::IBuilder::setMaxWorkspaceSize().
-    precision_mode: one of TrtPrecisionMode.supported_precision_modes().
-    minimum_segment_size: the minimum number of nodes required for a subgraph to
-      be replaced by TRTEngineOp.
-    is_dynamic_op: whether to generate dynamic TRT ops which will build the TRT
-      network and engine at run time.
-    maximum_cached_engines: max number of cached TRT engines in dynamic TRT ops.
-      If the number of cached engines is already at max but none of them can
-      serve the input, the TRTEngineOp will fall back to run the TF function
-      based on which the TRTEngineOp is created.
-    cached_engine_batches: a list of batch sizes used to create cached
-      engines, only used when is_dynamic_op is True. The length of the list
-      should be <= maximum_cached_engines, and the dynamic TRT op will
-      use this list to determine the batch sizes of the cached engines, instead
-      of making the decision on the fly. This is useful when we know the most
-      common batch size(s) the application is going to generate.
-    use_calibration: this argument is ignored if precision_mode is not INT8. If
-      set to True, a calibration graph will be created to calibrate the missing
-      ranges. The calibration graph must be converted to an inference graph
-      using calib_graph_to_infer_graph() after running calibration. if set to
-      False, quantization nodes will be expected for every tensor in the graph
-      (exlcuding those which will be fused). If a range is missing, an error
-      will occur. Please note that accuracy may be negatively affected if there
-      is a mismatch between which tensors TRT quantizes and which tensors were
-      trained with fake quantization.
-    input_saved_model_dir: the directory to load the SavedModel which contains
-      the input graph to transforms. Used only when input_graph_def is None.
-    input_saved_model_tags: list of tags to load the SavedModel.
-    output_saved_model_dir: if not None, construct a SavedModel using the
-      returned GraphDef and save it to the specified directory. This option only
-      works when the input graph is loaded from a SavedModel, i.e. when
-      input_saved_model_dir is specified and input_graph_def is None.
-    session_config: the ConfigProto used to create a Session. It's also used as
-      a template to create a TRT-enabled ConfigProto for conversion. If not
-      specified, a default ConfigProto will be used.
-
-  Returns:
-    A GraphDef transformed from input_graph_def (or the SavedModel graph def
-    loaded from input_saved_model_dir, if input_graph_def is not present), where
-    all TRT compatible subgraphs are replaced with TRTEngineOps, and a TF
-    function is added for each of the subgraphs.
-
-    If is_dynamic_op is True, each TRTEngineOp will contain a serialized
-    subgraph GraphDef, which will be converted to a TRT engine at execution time
-    and the TRT engine will be cached for future usage. A new TRT engine will be
-    created each time when none of the cached engines match the input shapes. If
-    it fails to execute the TRT engine or the number of cached engines reaches
-    maximum_cached_engines, the op will fall back to call the corresponding TF
-    function.
-
-    If is_dynamic_op is False, each TRTEngineOp will contain a serialized TRT
-    engine created from the corresponding subgraph. No more engines will be
-    created on the fly, and the op will fall back to call the corresponding TF
-    function when it fails to execute the engine.
-
-  Raises:
-    ValueError: if the combination of the parameters is invalid.
-    RuntimeError: if the TensorRT library version is incompatible.
-  """
-  compiled_version = get_linked_tensorrt_version()
-  loaded_version = get_loaded_tensorrt_version()
-  version_mismatch = False
-  if loaded_version[0] < compiled_version[0]:
-    tf_logging.error(
-        "TensorRT version mismatch. Tensorflow was compiled against " +
-        "TensorRT %s but library loaded from environment is TensorRT %s" %
-        (".".join([str(x) for x in compiled_version]),
-         ".".join([str(x) for x in loaded_version])) +
-        ". Please make sure that correct version of TensorRT " +
-        "is available in the system and added to ldconfig or LD_LIBRARY_PATH")
-    raise RuntimeError("Incompatible TensorRT library version")
-  for i in zip(loaded_version, compiled_version):
-    if i[0] != i[1]:
-      tf_logging.warn("TensorRT mismatch. Compiled against version " +
-                      "%s, but loaded %s. Things may not work" %
-                      (".".join([str(x) for x in compiled_version]),
-                       ".".join([str(x) for x in loaded_version])))
-      version_mismatch = True
-      break
-  if not version_mismatch:
-    tf_logging.info("Running against TensorRT version %s" % ".".join(
-        [str(x) for x in loaded_version]))
-
-  if session_config is None:
-    session_config = config_pb2.ConfigProto()
-
-  if input_saved_model_tags is None:
-    input_saved_model_tags = [tag_constants.SERVING]
-  saved_model_loader = None
-  grappler_meta_graph_def = None
-
-  if input_graph_def is None:
-    # Read from SavedModel and freeze the graph if necessary.
-    if input_saved_model_dir is None:
-      raise ValueError("input_graph_def and input_saved_model_dir cannot be "
-                       "both None")
-    with ops.Graph().as_default():
-      with session.Session(config=session_config) as sess:
-        saved_model_loader = loader_impl.SavedModelLoader(input_saved_model_dir)
-        input_meta_graph_def = saved_model_loader.load(sess,
-                                                       input_saved_model_tags)
-        output_node_names = set()
-
-        def _gather_names(tensor_info):
-          """Get the node names from a TensorInfo."""
-          return set(
-              [tensor_info[key].name.split(":")[0] for key in tensor_info])
-
-        # Get input and outputs from all SignatureDef.
-        for key in input_meta_graph_def.signature_def:
-          signature_def = input_meta_graph_def.signature_def[key]
-          output_node_names.update(_gather_names(signature_def.inputs))
-          output_node_names.update(_gather_names(signature_def.outputs))
-
-        # Freeze the variables in the SavedModel graph and copy the frozen
-        # graph over.
-        frozen_graph_def = graph_util.convert_variables_to_constants(
-            sess, sess.graph.as_graph_def(add_shapes=True),
-            list(output_node_names))
-        grappler_meta_graph_def = meta_graph_pb2.MetaGraphDef()
-        grappler_meta_graph_def.graph_def.CopyFrom(frozen_graph_def)
-
-        # Copy the collections that are not variables.
-        for key in input_meta_graph_def.collection_def:
-          # TODO(laigd): currently we use the collection key to filter out
-          # collections that depend on variable ops, but this may miss some
-          # other user-defined collections. A better way would be to use
-          # CollectionDef::NodeList for the filtering.
-          if key not in [
-              "variables", "local_variables", "model_variables",
-              "trainable_variables", "train_op", "table_initializer"
-          ]:
-            grappler_meta_graph_def.collection_def[key].CopyFrom(
-                input_meta_graph_def.collection_def[key])
-
-        # Copy other information.
-        grappler_meta_graph_def.meta_info_def.CopyFrom(
-            input_meta_graph_def.meta_info_def)
-        for key in input_meta_graph_def.signature_def:
-          grappler_meta_graph_def.signature_def[key].CopyFrom(
-              input_meta_graph_def.signature_def[key])
-        # TODO(laigd): maybe add back AssetFileDef.
-  else:
-    if output_saved_model_dir is not None:
-      raise ValueError("output_saved_model_dir cannot be set when "
-                       "input_graph_def is set")
-    # Create MetaGraphDef from input graph.
-    graph = ops.Graph()
-    with graph.as_default():
-      importer.import_graph_def(input_graph_def, name="")
-    grappler_meta_graph_def = saver.export_meta_graph(
-        graph_def=graph.as_graph_def(add_shapes=True), graph=graph)
-    if outputs:
-      output_collection = meta_graph_pb2.CollectionDef()
-      output_list = output_collection.node_list.value
-      for i in outputs:
-        if isinstance(i, ops.Tensor):
-          output_list.append(_to_bytes(i.name))
-        else:
-          output_list.append(_to_bytes(i))
-      # TODO(laigd): use another key as the outputs are really not train_op.
-      grappler_meta_graph_def.collection_def["train_op"].CopyFrom(
-          output_collection)
-
-  # Create TRT-enabled ConfigProto.
-  session_config_with_trt = config_pb2.ConfigProto()
-  session_config_with_trt.CopyFrom(session_config)
-  rewriter_config = None
-  if (session_config_with_trt.HasField("graph_options") and
-      session_config_with_trt.graph_options.HasField("rewrite_options")):
-    rewriter_config = session_config_with_trt.graph_options.rewrite_options
-  rewriter_config_with_trt = get_tensorrt_rewriter_config(
-      rewriter_config, max_batch_size, max_workspace_size_bytes, precision_mode,
-      minimum_segment_size, is_dynamic_op, maximum_cached_engines,
-      cached_engine_batches, use_calibration)
-  session_config_with_trt.graph_options.rewrite_options.CopyFrom(
-      rewriter_config_with_trt)
-
-  # Run Grappler.
-  transformed_graph_def = tf_optimizer.OptimizeGraph(
-      session_config_with_trt, grappler_meta_graph_def, graph_id=b"tf_graph")
-
-  # Optionally write the transformed graphdef as SavedModel.
-  if output_saved_model_dir is not None:
-    saved_model_builder = builder.SavedModelBuilder(output_saved_model_dir)
-    with ops.Graph().as_default():
-      importer.import_graph_def(transformed_graph_def, name="")
-      # We don't use TRT here.
-      with session.Session(config=session_config) as sess:
-        saved_model_builder.add_meta_graph_and_variables(
-            sess,
-            input_saved_model_tags,
-            signature_def_map=grappler_meta_graph_def.signature_def)
-    # Ignore other meta graphs from the input SavedModel.
-    saved_model_builder.save()
-
-  return transformed_graph_def
+  return trt_convert.create_inference_graph(
+      input_graph_def=input_graph_def,
+      outputs=outputs,
+      max_batch_size=max_batch_size,
+      max_workspace_size_bytes=max_workspace_size_bytes,
+      precision_mode=precision_mode,
+      minimum_segment_size=minimum_segment_size,
+      is_dynamic_op=is_dynamic_op,
+      maximum_cached_engines=maximum_cached_engines,
+      cached_engine_batches=cached_engine_batches,
+      use_calibration=use_calibration,
+      input_saved_model_dir=input_saved_model_dir,
+      input_saved_model_tags=input_saved_model_tags,
+      output_saved_model_dir=output_saved_model_dir,
+      session_config=session_config)
 
 
 def calib_graph_to_infer_graph(calibration_graph_def, is_dynamic_op=False):
-  """Convert an existing calibration graph to inference graph.
-
-  Args:
-    calibration_graph_def: the calibration GraphDef object with calibration data
-    is_dynamic_op: whether to create dynamic static engines from calibration
-
-  Returns:
-    New GraphDef with TRTEngineOps placed in graph replacing calibration nodes.
-  Raises:
-    RuntimeError: if the returned status message is malformed.
-  """
-
-  is_calib_graph = False
-  for n in calibration_graph_def.node:
-    if n.op == "TRTEngineOp":
-      is_calib_graph = is_calib_graph or not n.attr["calibration_data"].s
-  if not is_calib_graph:
-    tf_logging.error(
-        "Not a calib graph. Doesn't seem to contain any calibration nodes.")
-    return None
-  graph_str = calibration_graph_def.SerializeToString()
-  out = calib_convert(graph_str, is_dynamic_op)
-  status = _to_string(out[0])
-  output_graph_def_string = out[1]
-  del graph_str  # Save some memory
-  if len(status) < 2:
-    raise _impl.UnknownError(None, None, status)
-  if status[:2] != "OK":
-    msg = status.split(";")
-    if len(msg) == 1:
-      raise RuntimeError("Status message is malformed {}".format(status))
-    # pylint: disable=protected-access
-    raise _impl._make_specific_exception(None, None, ";".join(msg[1:]),
-                                         int(msg[0]))
-    # pylint: enable=protected-access
-  output_graph_def = graph_pb2.GraphDef()
-  output_graph_def.ParseFromString(output_graph_def_string)
-  del output_graph_def_string  # Save some memory
-  return output_graph_def
+  return trt_convert.calib_graph_to_infer_graph(
+      calibration_graph_def=calibration_graph_def, is_dynamic_op=is_dynamic_op)
diff --git a/tensorflow/contrib/tensorrt/test/test_tftrt.py b/tensorflow/contrib/tensorrt/test/test_tftrt.py
deleted file mode 100644
index 090aa8bdb0..0000000000
--- a/tensorflow/contrib/tensorrt/test/test_tftrt.py
+++ /dev/null
@@ -1,287 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Script to test TF-TensorRT integration."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import argparse
-import numpy as np
-import six as _six
-
-# normally we should do import tensorflow as tf and then
-# tf.placeholder, tf.constant, tf.nn.conv2d etc but
-# it looks like internal builds don't like it so
-# importing every module individually
-
-from tensorflow.contrib import tensorrt as trt
-from tensorflow.core.protobuf import config_pb2 as cpb2
-from tensorflow.core.protobuf import rewriter_config_pb2 as rwpb2
-from tensorflow.python.client import session as csess
-from tensorflow.python.framework import constant_op as cop
-from tensorflow.python.framework import dtypes as dtypes
-from tensorflow.python.framework import importer as importer
-from tensorflow.python.framework import ops as ops
-from tensorflow.python.ops import array_ops as aops
-from tensorflow.python.ops import math_ops as mops
-from tensorflow.python.ops import nn as nn
-from tensorflow.python.ops import nn_ops as nn_ops
-
-
-def py2bytes(inp):
-  return inp
-
-
-def py3bytes(inp):
-  return inp.encode("utf-8", errors="surrogateescape")
-
-
-def py2string(inp):
-  return inp
-
-
-def py3string(inp):
-  return inp.decode("utf-8")
-
-
-if _six.PY2:
-  to_bytes = py2bytes
-  to_string = py2string
-else:
-  to_bytes = py3bytes
-  to_string = py3string
-
-
-def get_multi_engine_graph_def(mode="FP32"):
-  """Create a simple graph and return its graph_def."""
-  dtype = dtypes.float32
-  if mode.upper() == "FP16":
-    dtype = dtypes.float16
-  else:
-    pass
-
-  g = ops.Graph()
-  with g.as_default():
-    x = aops.placeholder(shape=[None, 3, 7, 5], name="input", dtype=dtype)
-    with g.name_scope("Global_scope"):
-      with g.name_scope("first_scope"):
-        e = cop.constant(
-            np.random.randn(3, 2, 3, 4), name="weights", dtype=dtype)
-        conv = nn.conv2d(
-            input=x,
-            filter=e,
-            data_format="NCHW",
-            strides=[1, 1, 1, 1],
-            padding="VALID",
-            name="conv")
-        b = cop.constant(np.random.randn(1, 4, 1, 1), name="bias1", dtype=dtype)
-        t = conv * b
-
-        b = cop.constant(np.random.randn(1, 4, 1, 1), name="bias2", dtype=dtype)
-        q = conv / b
-      edge = mops.sin(q)
-      edge1 = mops.cos(conv)
-      with g.name_scope("test_scope"):
-        de = edge + edge1
-        t -= edge1
-        q *= edge
-        t += q
-        t -= de
-    k = aops.squeeze(t, name="output")
-  print(k.dtype)
-  return g.as_graph_def()
-
-
-def get_simple_graph_def():
-  """Create a simple graph and return its graph_def."""
-  g = ops.Graph()
-  with g.as_default():
-    a = aops.placeholder(
-        dtype=dtypes.float32, shape=(None, 24, 24, 2), name="input")
-    e = cop.constant(
-        [[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]],
-        name="weights",
-        dtype=dtypes.float32)
-    conv = nn.conv2d(
-        input=a, filter=e, strides=[1, 2, 2, 1], padding="SAME", name="conv")
-    b = cop.constant(
-        [4., 1.5, 2., 3., 5., 7.], name="bias", dtype=dtypes.float32)
-    t = nn.bias_add(conv, b, name="biasAdd")
-    relu = nn.relu(t, "relu")
-    idty = aops.identity(relu, "ID")
-    v = nn_ops.max_pool(
-        idty, [1, 2, 2, 1], [1, 2, 2, 1], "VALID", name="max_pool")
-    aops.squeeze(v, name="output")
-  return g.as_graph_def()
-
-
-def execute_graph(gdef, dumm_inp):
-  """Run given graphdef once."""
-  print("executing")
-  gpu_options = None
-  if trt.trt_convert.get_linked_tensorrt_version()[0] == 3:
-    gpu_options = cpb2.GPUOptions(per_process_gpu_memory_fraction=0.50)
-  sessconfig = cpb2.ConfigProto(gpu_options=gpu_options)
-  ops.reset_default_graph()
-  g = ops.Graph()
-  with g.as_default():
-    inp, out = importer.import_graph_def(
-        graph_def=gdef, return_elements=["input", "output"])
-    inp = inp.outputs[0]
-    out = out.outputs[0]
-  with csess.Session(config=sessconfig, graph=g) as sess:
-    val = sess.run(out, {inp: dumm_inp})
-  return val
-
-
-# Use real data that is representative of the inference dataset
-# for calibration. For this test script it is random data.
-def execute_calibration(gdef, dumm_inp):
-  """Run given calibration graph multiple times."""
-  gpu_options = None
-  if trt.trt_convert.get_linked_tensorrt_version()[0] == 3:
-    gpu_options = cpb2.GPUOptions(per_process_gpu_memory_fraction=0.50)
-  ops.reset_default_graph()
-  g = ops.Graph()
-  with g.as_default():
-    inp, out = importer.import_graph_def(
-        graph_def=gdef, return_elements=["input", "output"])
-    inp = inp.outputs[0]
-    out = out.outputs[0]
-  with csess.Session(
-      config=cpb2.ConfigProto(gpu_options=gpu_options), graph=g) as sess:
-    # run over real calibration data here, we are mimicking a calibration set of
-    # 30 different batches. Use as much calibration data as you want
-    for _ in range(30):
-      val = sess.run(out, {inp: dumm_inp})
-  return val
-
-
-def user(multi_engine,
-         run_graph=execute_graph,
-         run_calibration=execute_calibration):
-  """Example function that converts a graph to TFTRT graph."""
-  if multi_engine:
-    inp_dims = (2, 3, 7, 5)
-    orig_graph = get_multi_engine_graph_def()
-  else:
-    inp_dims = (100, 24, 24, 2)
-    orig_graph = get_simple_graph_def()  # use a frozen graph for inference
-  dummy_input = np.random.random_sample(inp_dims)
-  # Get optimized graph
-  trt_graph = trt.create_inference_graph(
-      input_graph_def=orig_graph,
-      outputs=["output"],
-      max_batch_size=inp_dims[0],
-      max_workspace_size_bytes=1 << 25,
-      precision_mode="FP32",  # TRT Engine precision "FP32","FP16" or "INT8"
-      minimum_segment_size=2,  # minimum number of nodes in an engine
-      is_dynamic_op=False,
-      maximum_cached_engines=1,
-      cached_engine_batches=[])
-  o1 = run_graph(orig_graph, dummy_input)
-  o2 = run_graph(trt_graph, dummy_input)
-  o3 = run_graph(trt_graph, dummy_input)
-  assert np.array_equal(o1, o2)
-  assert np.array_equal(o3, o2)  # sanity check
-  fp16_graph = trt.create_inference_graph(
-      input_graph_def=orig_graph,
-      outputs=["output"],
-      max_batch_size=inp_dims[0],
-      max_workspace_size_bytes=1 << 25,
-      precision_mode="FP16",  # TRT Engine precision "FP32","FP16" or "INT8"
-      minimum_segment_size=2,  # minimum number of nodes in an engine
-      is_dynamic_op=False,
-      maximum_cached_engines=1,
-      cached_engine_batches=[])
-  int8_calib_gdef = trt.create_inference_graph(
-      input_graph_def=orig_graph,
-      outputs=["output"],
-      max_batch_size=inp_dims[0],
-      max_workspace_size_bytes=1 << 25,
-      precision_mode="INT8",  # TRT Engine precision "FP32","FP16" or "INT8"
-      minimum_segment_size=2,  # minimum number of nodes in an engine
-      is_dynamic_op=False,
-      maximum_cached_engines=1,
-      cached_engine_batches=[])
-  o4 = run_graph(fp16_graph, dummy_input)
-  _ = run_calibration(int8_calib_gdef, dummy_input)
-  int8_graph = trt.calib_graph_to_infer_graph(int8_calib_gdef)
-  o5 = run_graph(int8_graph, dummy_input)
-  print("Is FP32 == FP16? %s (False is possible)" % np.allclose(o1, o4))
-  print("Is FP32 == INT8? %s (False is possible)" % np.allclose(o1, o5))
-  print("Pass")
-
-
-def auto(multi_engine):
-  """Run the conversion as an optimization pass."""
-  if multi_engine:
-    inp_dims = (2, 3, 7, 5)
-    orig_graph = get_multi_engine_graph_def()
-  else:
-    inp_dims = (100, 24, 24, 2)
-    orig_graph = get_simple_graph_def()  # use a frozen graph for inference
-  dummy_input = np.random.random_sample(inp_dims)
-  opt_config = rwpb2.RewriterConfig()
-  opt_config.meta_optimizer_iterations = opt_config.ONE
-  opt_config.optimizers.extend(["constfold", "layout"])
-  custom_op = opt_config.custom_optimizers.add()
-  custom_op.name = "TensorRTOptimizer"
-  custom_op.parameter_map["minimum_segment_size"].i = 3
-  custom_op.parameter_map["precision_mode"].s = to_bytes("FP32")
-  custom_op.parameter_map["max_batch_size"].i = inp_dims[0]
-  custom_op.parameter_map["max_workspace_size_bytes"].i = 1 << 25
-  print(custom_op)
-  gpu_options = None
-  if trt.trt_convert.get_linked_tensorrt_version()[0] == 3:
-    gpu_options = cpb2.GPUOptions(per_process_gpu_memory_fraction=0.50)
-  graph_options = cpb2.GraphOptions(rewrite_options=opt_config)
-  sessconfig = cpb2.ConfigProto(
-      gpu_options=gpu_options, graph_options=graph_options)
-  print(sessconfig)
-  g = ops.Graph()
-  ops.reset_default_graph()
-  with g.as_default():
-    inp, out = importer.import_graph_def(
-        graph_def=orig_graph, return_elements=["input", "output"], name="")
-    inp = inp.outputs[0]
-    out = out.outputs[0]
-    with csess.Session(config=sessconfig, graph=g) as sess:
-      val = sess.run(out, {inp: dummy_input})
-  print(val.shape)
-
-
-if "__main__" in __name__:
-  P = argparse.ArgumentParser(
-      prog="tftrt_test",
-      description="Example utilization of TensorFlow-TensorRT integration")
-  P.add_argument(
-      "--automatic",
-      "-a",
-      action="store_true",
-      help="Do TRT conversion automatically",
-      default=False)
-  P.add_argument(
-      "--multi-engine",
-      "-m",
-      action="store_true",
-      help="Use a graph that will result in 2 engines",
-      default=False)
-  flags, unparsed = P.parse_known_args()
-  if flags.automatic:
-    auto(flags.multi_engine)
-  else:
-    user(flags.multi_engine)
diff --git a/tensorflow/python/compiler/tensorrt/BUILD b/tensorflow/python/compiler/tensorrt/BUILD
new file mode 100644
index 0000000000..a382579f5a
--- /dev/null
+++ b/tensorflow/python/compiler/tensorrt/BUILD
@@ -0,0 +1,176 @@
+# Description:
+#   Wrap NVIDIA TensorRT (http://developer.nvidia.com/tensorrt) with tensorflow
+#   and provide TensorRT operators and converter package.
+#   APIs are meant to change over time.
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])  # Apache 2.0
+
+exports_files(["LICENSE"])
+
+load(
+    "//tensorflow:tensorflow.bzl",
+    "tf_copts",
+)
+load("//tensorflow:tensorflow.bzl", "cuda_py_test")
+load("//tensorflow:tensorflow.bzl", "cuda_py_tests")
+load("//tensorflow:tensorflow.bzl", "tf_py_wrap_cc")
+load(
+    "@local_config_tensorrt//:build_defs.bzl",
+    "if_tensorrt",
+)
+
+exports_files(glob([
+    "test/testdata/*",
+]))
+
+py_library(
+    name = "init_py",
+    srcs = ["__init__.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":tf_trt_integration_test_base",
+        ":trt_convert_py",
+        ":trt_ops_py",
+        "//tensorflow/python:errors",
+    ],
+)
+
+py_library(
+    name = "trt_ops_py",
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/compiler/tf2tensorrt:trt_ops",
+        "//tensorflow/compiler/tf2tensorrt:trt_ops_loader",
+    ],
+)
+
+py_library(
+    name = "trt_convert_py",
+    srcs = ["trt_convert.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":wrap_conversion",
+        "//tensorflow/python:graph_util",
+        "//tensorflow/python:session",
+        "//tensorflow/python:tf_optimizer",
+        "//tensorflow/python/saved_model:builder",
+        "//tensorflow/python/saved_model:loader",
+        "//tensorflow/python/saved_model:tag_constants",
+    ],
+)
+
+# TODO(aaroey): this wrapper has been causing troubles of double linking, so
+# either get rid of it, or split to make it contain minimum dependencies.
+tf_py_wrap_cc(
+    name = "wrap_conversion",
+    srcs = ["trt_conversion.i"],
+    copts = tf_copts(),
+    swig_includes = [
+        "//tensorflow/python:platform/base.i",
+    ],
+    deps = [
+        "//tensorflow/compiler/tf2tensorrt:test_utils",
+        "//tensorflow/compiler/tf2tensorrt:trt_conversion",
+        "//tensorflow/compiler/tf2tensorrt:trt_op_kernels",
+        "//third_party/python_runtime:headers",
+    ],
+)
+
+py_library(
+    name = "tf_trt_integration_test_base",
+    srcs = ["test/tf_trt_integration_test_base.py"],
+    deps = [
+        ":trt_convert_py",
+        ":trt_ops_py",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_test_lib",
+    ],
+)
+
+cuda_py_test(
+    name = "trt_convert_test",
+    srcs = ["trt_convert_test.py"],
+    additional_deps = [
+        ":trt_convert_py",
+        ":trt_ops_py",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:graph_util",
+        "//tensorflow/python/saved_model:builder",
+        "//tensorflow/python/saved_model:loader",
+        "//tensorflow/python/saved_model:signature_constants",
+        "//tensorflow/python/saved_model:signature_def_utils",
+        "//tensorflow/python/saved_model:tag_constants",
+        "//tensorflow/python/saved_model:utils",
+        "//tensorflow/python/tools:freeze_graph_lib",
+        "//tensorflow/python/tools:saved_model_utils",
+    ],
+    tags = [
+        "no_cuda_on_cpu_tap",
+        "no_windows",
+        "nomac",
+    ],
+)
+
+cuda_py_tests(
+    name = "tf_trt_integration_test",
+    srcs = [
+        "test/base_test.py",
+        "test/batch_matmul_test.py",
+        "test/biasadd_matmul_test.py",
+        "test/binary_tensor_weight_broadcast_test.py",
+        "test/concatenation_test.py",
+        "test/const_broadcast_test.py",
+        "test/conv2d_test.py",
+        "test/dynamic_input_shapes_test.py",
+        "test/identity_output_test.py",
+        "test/int32_test.py",
+        "test/lru_cache_test.py",
+        "test/memory_alignment_test.py",
+        "test/multi_connection_neighbor_engine_test.py",
+        "test/neighboring_engine_test.py",
+        "test/quantization_test.py",
+        "test/rank_two_test.py",
+        "test/reshape_transpose_test.py",
+        "test/topk_test.py",
+        "test/unary_test.py",
+        "test/vgg_block_nchw_test.py",
+        "test/vgg_block_test.py",
+    ],
+    additional_deps = [
+        ":tf_trt_integration_test_base",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_test_lib",
+    ],
+    tags = [
+        "no_cuda_on_cpu_tap",
+        "no_windows",
+        "nomac",
+    ],
+)
+
+cuda_py_test(
+    name = "quantization_mnist_test",
+    srcs = ["test/quantization_mnist_test.py"],
+    additional_deps = [
+        ":tf_trt_integration_test_base",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python/keras:keras",
+        "//tensorflow/python/estimator:estimator",
+    ],
+    data = [
+        "test/testdata/checkpoint",
+        "test/testdata/model.ckpt-46900.data-00000-of-00001",
+        "test/testdata/model.ckpt-46900.index",
+    ],
+    tags = [
+        "no_cuda_on_cpu_tap",
+        "no_pip",
+        "no_tap",  # It is not able to download the mnist data.
+        "no_windows",
+        "nomac",
+    ],
+)
diff --git a/tensorflow/contrib/tensorrt/README.md b/tensorflow/python/compiler/tensorrt/README.md
similarity index 100%
rename from tensorflow/contrib/tensorrt/README.md
rename to tensorflow/python/compiler/tensorrt/README.md
diff --git a/tensorflow/python/compiler/tensorrt/__init__.py b/tensorflow/python/compiler/tensorrt/__init__.py
new file mode 100644
index 0000000000..88fb69101d
--- /dev/null
+++ b/tensorflow/python/compiler/tensorrt/__init__.py
@@ -0,0 +1,42 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =============================================================================
+"""Exposes the python wrapper for TensorRT graph transforms."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import errors
+
+# pylint: disable=unused-import,g-import-not-at-top,line-too-long
+try:
+  from tensorflow.compiler.tf2tensorrt.python.ops import trt_ops
+  from tensorflow.python.compiler.tensorrt.trt_convert import add_test_value
+  from tensorflow.python.compiler.tensorrt.trt_convert import calib_graph_to_infer_graph
+  from tensorflow.python.compiler.tensorrt.trt_convert import clear_test_values
+  from tensorflow.python.compiler.tensorrt.trt_convert import create_inference_graph
+  from tensorflow.python.compiler.tensorrt.trt_convert import enable_test_value
+  from tensorflow.python.compiler.tensorrt.trt_convert import get_test_value
+  from tensorflow.python.compiler.tensorrt.trt_convert import is_tensorrt_enabled
+except errors.NotFoundError as e:
+  no_trt_message = (
+      '**** Failed to initialize TensorRT. This is either because the TensorRT'
+      ' installation path is not in LD_LIBRARY_PATH, or because you do not have'
+      ' it installed. If not installed, please go to'
+      ' https://developer.nvidia.com/tensorrt to download and install'
+      ' TensorRT ****')
+  print(no_trt_message)
+  raise e
+# pylint: enable=unused-import,g-import-not-at-top,line-too-long
diff --git a/tensorflow/contrib/tensorrt/test/base_test.py b/tensorflow/python/compiler/tensorrt/test/base_test.py
similarity index 98%
rename from tensorflow/contrib/tensorrt/test/base_test.py
rename to tensorflow/python/compiler/tensorrt/test/base_test.py
index 17e0b6f4d2..cc3109968e 100644
--- a/tensorflow/contrib/tensorrt/test/base_test.py
+++ b/tensorflow/python/compiler/tensorrt/test/base_test.py
@@ -20,8 +20,8 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib.tensorrt.python import trt_convert
-from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.compiler.tensorrt import trt_convert
+from tensorflow.python.compiler.tensorrt.test import tf_trt_integration_test_base as trt_test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
diff --git a/tensorflow/contrib/tensorrt/test/batch_matmul_test.py b/tensorflow/python/compiler/tensorrt/test/batch_matmul_test.py
similarity index 97%
rename from tensorflow/contrib/tensorrt/test/batch_matmul_test.py
rename to tensorflow/python/compiler/tensorrt/test/batch_matmul_test.py
index 46e3407d96..49ad09aea3 100644
--- a/tensorflow/contrib/tensorrt/test/batch_matmul_test.py
+++ b/tensorflow/python/compiler/tensorrt/test/batch_matmul_test.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.compiler.tensorrt.test import tf_trt_integration_test_base as trt_test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
diff --git a/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py b/tensorflow/python/compiler/tensorrt/test/biasadd_matmul_test.py
similarity index 98%
rename from tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
rename to tensorflow/python/compiler/tensorrt/test/biasadd_matmul_test.py
index ca23629efa..2b7bbbc960 100644
--- a/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
+++ b/tensorflow/python/compiler/tensorrt/test/biasadd_matmul_test.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.compiler.tensorrt.test import tf_trt_integration_test_base as trt_test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
diff --git a/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py b/tensorflow/python/compiler/tensorrt/test/binary_tensor_weight_broadcast_test.py
similarity index 96%
rename from tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
rename to tensorflow/python/compiler/tensorrt/test/binary_tensor_weight_broadcast_test.py
index 846fd009db..7e1d3afdd9 100644
--- a/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
+++ b/tensorflow/python/compiler/tensorrt/test/binary_tensor_weight_broadcast_test.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.compiler.tensorrt.test import tf_trt_integration_test_base as trt_test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
diff --git a/tensorflow/contrib/tensorrt/test/concatenation_test.py b/tensorflow/python/compiler/tensorrt/test/concatenation_test.py
similarity index 97%
rename from tensorflow/contrib/tensorrt/test/concatenation_test.py
rename to tensorflow/python/compiler/tensorrt/test/concatenation_test.py
index 5d8742ae35..f30324e7db 100644
--- a/tensorflow/contrib/tensorrt/test/concatenation_test.py
+++ b/tensorflow/python/compiler/tensorrt/test/concatenation_test.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.compiler.tensorrt.test import tf_trt_integration_test_base as trt_test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
diff --git a/tensorflow/contrib/tensorrt/test/const_broadcast_test.py b/tensorflow/python/compiler/tensorrt/test/const_broadcast_test.py
similarity index 96%
rename from tensorflow/contrib/tensorrt/test/const_broadcast_test.py
rename to tensorflow/python/compiler/tensorrt/test/const_broadcast_test.py
index 9137d0072f..2d764665be 100644
--- a/tensorflow/contrib/tensorrt/test/const_broadcast_test.py
+++ b/tensorflow/python/compiler/tensorrt/test/const_broadcast_test.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.compiler.tensorrt.test import tf_trt_integration_test_base as trt_test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
diff --git a/tensorflow/contrib/tensorrt/test/conv2d_test.py b/tensorflow/python/compiler/tensorrt/test/conv2d_test.py
similarity index 98%
rename from tensorflow/contrib/tensorrt/test/conv2d_test.py
rename to tensorflow/python/compiler/tensorrt/test/conv2d_test.py
index c32c24fe0c..326cad5297 100644
--- a/tensorflow/contrib/tensorrt/test/conv2d_test.py
+++ b/tensorflow/python/compiler/tensorrt/test/conv2d_test.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.compiler.tensorrt.test import tf_trt_integration_test_base as trt_test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
diff --git a/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py b/tensorflow/python/compiler/tensorrt/test/dynamic_input_shapes_test.py
similarity index 97%
rename from tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py
rename to tensorflow/python/compiler/tensorrt/test/dynamic_input_shapes_test.py
index cc28cd6087..cb358d4f9b 100644
--- a/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py
+++ b/tensorflow/python/compiler/tensorrt/test/dynamic_input_shapes_test.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.compiler.tensorrt.test import tf_trt_integration_test_base as trt_test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
diff --git a/tensorflow/contrib/tensorrt/test/identity_output_test.py b/tensorflow/python/compiler/tensorrt/test/identity_output_test.py
similarity index 96%
rename from tensorflow/contrib/tensorrt/test/identity_output_test.py
rename to tensorflow/python/compiler/tensorrt/test/identity_output_test.py
index b568eeda94..23a72c5b0b 100644
--- a/tensorflow/contrib/tensorrt/test/identity_output_test.py
+++ b/tensorflow/python/compiler/tensorrt/test/identity_output_test.py
@@ -25,7 +25,7 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.compiler.tensorrt.test import tf_trt_integration_test_base as trt_test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
diff --git a/tensorflow/contrib/tensorrt/test/int32_test.py b/tensorflow/python/compiler/tensorrt/test/int32_test.py
similarity index 97%
rename from tensorflow/contrib/tensorrt/test/int32_test.py
rename to tensorflow/python/compiler/tensorrt/test/int32_test.py
index 8cf5387038..6d4446940a 100644
--- a/tensorflow/contrib/tensorrt/test/int32_test.py
+++ b/tensorflow/python/compiler/tensorrt/test/int32_test.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.compiler.tensorrt.test import tf_trt_integration_test_base as trt_test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
diff --git a/tensorflow/contrib/tensorrt/test/lru_cache_test.py b/tensorflow/python/compiler/tensorrt/test/lru_cache_test.py
similarity index 96%
rename from tensorflow/contrib/tensorrt/test/lru_cache_test.py
rename to tensorflow/python/compiler/tensorrt/test/lru_cache_test.py
index 7702413e6c..18e6d32dfe 100644
--- a/tensorflow/contrib/tensorrt/test/lru_cache_test.py
+++ b/tensorflow/python/compiler/tensorrt/test/lru_cache_test.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.compiler.tensorrt.test import tf_trt_integration_test_base as trt_test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
diff --git a/tensorflow/contrib/tensorrt/test/memory_alignment_test.py b/tensorflow/python/compiler/tensorrt/test/memory_alignment_test.py
similarity index 96%
rename from tensorflow/contrib/tensorrt/test/memory_alignment_test.py
rename to tensorflow/python/compiler/tensorrt/test/memory_alignment_test.py
index cc64329bbd..89625aa629 100644
--- a/tensorflow/contrib/tensorrt/test/memory_alignment_test.py
+++ b/tensorflow/python/compiler/tensorrt/test/memory_alignment_test.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.compiler.tensorrt.test import tf_trt_integration_test_base as trt_test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
diff --git a/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py b/tensorflow/python/compiler/tensorrt/test/multi_connection_neighbor_engine_test.py
similarity index 96%
rename from tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
rename to tensorflow/python/compiler/tensorrt/test/multi_connection_neighbor_engine_test.py
index a14bb0396e..d04c6958fb 100644
--- a/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
+++ b/tensorflow/python/compiler/tensorrt/test/multi_connection_neighbor_engine_test.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.compiler.tensorrt.test import tf_trt_integration_test_base as trt_test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
diff --git a/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py b/tensorflow/python/compiler/tensorrt/test/neighboring_engine_test.py
similarity index 96%
rename from tensorflow/contrib/tensorrt/test/neighboring_engine_test.py
rename to tensorflow/python/compiler/tensorrt/test/neighboring_engine_test.py
index 06a86bbb8d..1f7189f0eb 100644
--- a/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py
+++ b/tensorflow/python/compiler/tensorrt/test/neighboring_engine_test.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.compiler.tensorrt.test import tf_trt_integration_test_base as trt_test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
diff --git a/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py b/tensorflow/python/compiler/tensorrt/test/quantization_mnist_test.py
similarity index 98%
rename from tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
rename to tensorflow/python/compiler/tensorrt/test/quantization_mnist_test.py
index d68211a7ee..1d7792c01a 100644
--- a/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
+++ b/tensorflow/python/compiler/tensorrt/test/quantization_mnist_test.py
@@ -21,10 +21,10 @@ from __future__ import print_function
 # pylint: disable=unused-import
 from tensorflow.compiler.tf2tensorrt.python.ops import trt_ops
 # pylint: enable=unused-import
-from tensorflow.contrib.tensorrt.python import trt_convert
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python import data
 from tensorflow.python import keras
+from tensorflow.python.compiler.tensorrt import trt_convert
 from tensorflow.python.estimator.estimator import Estimator
 from tensorflow.python.estimator.model_fn import EstimatorSpec
 from tensorflow.python.estimator.model_fn import ModeKeys
@@ -265,7 +265,7 @@ class QuantizationAwareTrainingMNISTTest(test_util.TensorFlowTestCase):
   def testEval(self):
     if not trt_convert.is_tensorrt_enabled():
       return
-    model_dir = test.test_src_dir_path('contrib/tensorrt/test/testdata')
+    model_dir = test.test_src_dir_path('python/compiler/tensorrt/test/testdata')
 
     accuracy_tf_native = self._Run(
         is_training=False,
diff --git a/tensorflow/contrib/tensorrt/test/quantization_test.py b/tensorflow/python/compiler/tensorrt/test/quantization_test.py
similarity index 97%
rename from tensorflow/contrib/tensorrt/test/quantization_test.py
rename to tensorflow/python/compiler/tensorrt/test/quantization_test.py
index ce1b25ebf3..086e070f1b 100644
--- a/tensorflow/contrib/tensorrt/test/quantization_test.py
+++ b/tensorflow/python/compiler/tensorrt/test/quantization_test.py
@@ -20,8 +20,8 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib.tensorrt.python import trt_convert
-from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.compiler.tensorrt import trt_convert
+from tensorflow.python.compiler.tensorrt.test import tf_trt_integration_test_base as trt_test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
diff --git a/tensorflow/contrib/tensorrt/test/rank_two_test.py b/tensorflow/python/compiler/tensorrt/test/rank_two_test.py
similarity index 97%
rename from tensorflow/contrib/tensorrt/test/rank_two_test.py
rename to tensorflow/python/compiler/tensorrt/test/rank_two_test.py
index 97159bb008..a951638b50 100644
--- a/tensorflow/contrib/tensorrt/test/rank_two_test.py
+++ b/tensorflow/python/compiler/tensorrt/test/rank_two_test.py
@@ -18,7 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.compiler.tensorrt.test import tf_trt_integration_test_base as trt_test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
diff --git a/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py b/tensorflow/python/compiler/tensorrt/test/reshape_transpose_test.py
similarity index 98%
rename from tensorflow/contrib/tensorrt/test/reshape_transpose_test.py
rename to tensorflow/python/compiler/tensorrt/test/reshape_transpose_test.py
index 7fb2cbde07..423d70f2e4 100644
--- a/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py
+++ b/tensorflow/python/compiler/tensorrt/test/reshape_transpose_test.py
@@ -18,7 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.compiler.tensorrt.test import tf_trt_integration_test_base as trt_test
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
diff --git a/tensorflow/contrib/tensorrt/test/testdata/checkpoint b/tensorflow/python/compiler/tensorrt/test/testdata/checkpoint
similarity index 100%
rename from tensorflow/contrib/tensorrt/test/testdata/checkpoint
rename to tensorflow/python/compiler/tensorrt/test/testdata/checkpoint
diff --git a/tensorflow/contrib/tensorrt/test/testdata/model.ckpt-46900.data-00000-of-00001 b/tensorflow/python/compiler/tensorrt/test/testdata/model.ckpt-46900.data-00000-of-00001
similarity index 100%
rename from tensorflow/contrib/tensorrt/test/testdata/model.ckpt-46900.data-00000-of-00001
rename to tensorflow/python/compiler/tensorrt/test/testdata/model.ckpt-46900.data-00000-of-00001
diff --git a/tensorflow/contrib/tensorrt/test/testdata/model.ckpt-46900.index b/tensorflow/python/compiler/tensorrt/test/testdata/model.ckpt-46900.index
similarity index 100%
rename from tensorflow/contrib/tensorrt/test/testdata/model.ckpt-46900.index
rename to tensorflow/python/compiler/tensorrt/test/testdata/model.ckpt-46900.index
diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/python/compiler/tensorrt/test/tf_trt_integration_test_base.py
similarity index 99%
rename from tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
rename to tensorflow/python/compiler/tensorrt/test/tf_trt_integration_test_base.py
index 9a00cdb11a..bba661b879 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/python/compiler/tensorrt/test/tf_trt_integration_test_base.py
@@ -28,9 +28,9 @@ import six
 # pylint: disable=unused-import
 from tensorflow.compiler.tf2tensorrt.python.ops import trt_ops
 # pylint: enable=unused-import
-from tensorflow.contrib.tensorrt.python import trt_convert
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import rewriter_config_pb2
+from tensorflow.python.compiler.tensorrt import trt_convert
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import graph_io
 from tensorflow.python.framework import importer
diff --git a/tensorflow/contrib/tensorrt/test/topk_test.py b/tensorflow/python/compiler/tensorrt/test/topk_test.py
similarity index 97%
rename from tensorflow/contrib/tensorrt/test/topk_test.py
rename to tensorflow/python/compiler/tensorrt/test/topk_test.py
index 5524fcd3d9..1e2bf3b65c 100644
--- a/tensorflow/contrib/tensorrt/test/topk_test.py
+++ b/tensorflow/python/compiler/tensorrt/test/topk_test.py
@@ -18,7 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.compiler.tensorrt.test import tf_trt_integration_test_base as trt_test
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import constant_op
diff --git a/tensorflow/contrib/tensorrt/test/unary_test.py b/tensorflow/python/compiler/tensorrt/test/unary_test.py
similarity index 97%
rename from tensorflow/contrib/tensorrt/test/unary_test.py
rename to tensorflow/python/compiler/tensorrt/test/unary_test.py
index 497ea2848a..83569bcfbf 100644
--- a/tensorflow/contrib/tensorrt/test/unary_test.py
+++ b/tensorflow/python/compiler/tensorrt/test/unary_test.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.compiler.tensorrt.test import tf_trt_integration_test_base as trt_test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
diff --git a/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py b/tensorflow/python/compiler/tensorrt/test/vgg_block_nchw_test.py
similarity index 96%
rename from tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py
rename to tensorflow/python/compiler/tensorrt/test/vgg_block_nchw_test.py
index b5fed73d2d..97ee11747e 100644
--- a/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py
+++ b/tensorflow/python/compiler/tensorrt/test/vgg_block_nchw_test.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.compiler.tensorrt.test import tf_trt_integration_test_base as trt_test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
diff --git a/tensorflow/contrib/tensorrt/test/vgg_block_test.py b/tensorflow/python/compiler/tensorrt/test/vgg_block_test.py
similarity index 96%
rename from tensorflow/contrib/tensorrt/test/vgg_block_test.py
rename to tensorflow/python/compiler/tensorrt/test/vgg_block_test.py
index 307128f1a8..a4fa1d6705 100644
--- a/tensorflow/contrib/tensorrt/test/vgg_block_test.py
+++ b/tensorflow/python/compiler/tensorrt/test/vgg_block_test.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.compiler.tensorrt.test import tf_trt_integration_test_base as trt_test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
diff --git a/tensorflow/contrib/tensorrt/trt_conversion.i b/tensorflow/python/compiler/tensorrt/trt_conversion.i
similarity index 100%
rename from tensorflow/contrib/tensorrt/trt_conversion.i
rename to tensorflow/python/compiler/tensorrt/trt_conversion.i
diff --git a/tensorflow/python/compiler/tensorrt/trt_convert.py b/tensorflow/python/compiler/tensorrt/trt_convert.py
new file mode 100644
index 0000000000..ca7872b478
--- /dev/null
+++ b/tensorflow/python/compiler/tensorrt/trt_convert.py
@@ -0,0 +1,428 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =============================================================================
+"""Exposes the Python wrapper conversion to trt_graph."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import six as _six
+# pylint: disable=unused-import,line-too-long
+from tensorflow.python.compiler.tensorrt.wrap_conversion import add_test_value
+from tensorflow.python.compiler.tensorrt.wrap_conversion import calib_convert
+from tensorflow.python.compiler.tensorrt.wrap_conversion import clear_test_values
+from tensorflow.python.compiler.tensorrt.wrap_conversion import enable_test_value
+from tensorflow.python.compiler.tensorrt.wrap_conversion import get_linked_tensorrt_version
+from tensorflow.python.compiler.tensorrt.wrap_conversion import get_loaded_tensorrt_version
+from tensorflow.python.compiler.tensorrt.wrap_conversion import get_test_value
+from tensorflow.python.compiler.tensorrt.wrap_conversion import is_tensorrt_enabled
+# pylint: enable=unused-import,line-too-long
+from tensorflow.core.framework import graph_pb2
+from tensorflow.core.protobuf import config_pb2
+from tensorflow.core.protobuf import meta_graph_pb2
+from tensorflow.core.protobuf import rewriter_config_pb2
+from tensorflow.python.client import session
+from tensorflow.python.framework import errors_impl as _impl
+from tensorflow.python.framework import graph_util
+from tensorflow.python.framework import importer
+from tensorflow.python.framework import ops
+from tensorflow.python.grappler import tf_optimizer
+from tensorflow.python.platform import tf_logging
+from tensorflow.python.saved_model import builder
+from tensorflow.python.saved_model import loader_impl
+from tensorflow.python.saved_model import tag_constants
+from tensorflow.python.training import saver
+
+
+def _to_bytes(s):
+  """Encode s if it is a sequence of chars."""
+  if isinstance(s, _six.text_type):
+    return s.encode("utf-8", errors="surrogateescape")
+  return s
+
+
+def _to_string(s):
+  """Decode s if it is a sequence of bytes."""
+  if isinstance(s, _six.binary_type):
+    return s.decode("utf-8")
+  return s
+
+
+class TrtPrecisionMode(object):
+  FP32 = "FP32"
+  FP16 = "FP16"
+  INT8 = "INT8"
+
+  @staticmethod
+  def supported_precision_modes():
+    return [TrtPrecisionMode.FP32, TrtPrecisionMode.FP16, TrtPrecisionMode.INT8]
+
+
+def get_tensorrt_rewriter_config(rewriter_config=None,
+                                 max_batch_size=1,
+                                 max_workspace_size_bytes=2 << 20,
+                                 precision_mode=TrtPrecisionMode.FP32,
+                                 minimum_segment_size=3,
+                                 is_dynamic_op=False,
+                                 maximum_cached_engines=1,
+                                 cached_engine_batches=None,
+                                 use_calibration=True):
+  """Returns a RewriterConfig proto for TRT transformation.
+
+  Args:
+    rewriter_config: a template RewriterConfig proto used to create a
+      TRT-enabled RewriterConfig. If None, it will use a default one.
+    max_batch_size: max size for the input batch
+    max_workspace_size_bytes: the maximum GPU temporary memory which the TRT
+      engine can use at execution time. This corresponds to the 'workspaceSize'
+      parameter of nvinfer1::IBuilder::setMaxWorkspaceSize().
+    precision_mode: one of TrtPrecisionMode.supported_precision_modes().
+    minimum_segment_size: the minimum number of nodes required for a subgraph to
+      be replaced by TRTEngineOp.
+    is_dynamic_op: whether to generate dynamic TRT ops which will build the TRT
+      network and engine at run time.
+    maximum_cached_engines: max number of cached TRT engines in dynamic TRT ops.
+      If the number of cached engines is already at max but none of them can
+      serve the input, the TRTEngineOp will fall back to run the TF function
+      based on which the TRTEngineOp is created.
+    cached_engine_batches: a list of batch sizes used to create cached
+      engines, only used when is_dynamic_op is True. The length of the list
+      should be <= maximum_cached_engines, and the dynamic TRT op will
+      use this list to determine the batch sizes of the cached engines, instead
+      of making the decision on the fly. This is useful when we know the most
+      common batch size(s) the application is going to generate.
+    use_calibration: this argument is ignored if precision_mode is not INT8. If
+      set to True, a calibration graph will be created to calibrate the missing
+      ranges. The calibration graph must be converted to an inference graph
+      using calib_graph_to_infer_graph() after running calibration. if set to
+      False, quantization nodes will be expected for every tensor in the graph
+      (exlcuding those which will be fused). If a range is missing, an error
+      will occur. Please note that accuracy may be negatively affected if there
+      is a mismatch between which tensors TRT quantizes and which tensors were
+      trained with fake quantization.
+
+  Returns:
+    A RewriterConfig proto which sets a TensorRTOptimizer to run Grappler.
+
+  Raises:
+    TypeError: if any of the parameters are of unexpected type.
+    ValueError: if any of the parameters are of unexpected value.
+  """
+  if rewriter_config is not None and not isinstance(
+      rewriter_config, rewriter_config_pb2.RewriterConfig):
+    raise TypeError("rewriter_config should be a RewriterConfig proto.")
+
+  rewriter_config_with_trt = rewriter_config_pb2.RewriterConfig()
+  if rewriter_config is None:
+    # Layout optimizer may add Const nodes followed by Reshape nodes, thus we
+    # need to run constant folding again.
+    rewriter_config_with_trt.optimizers.extend(
+        ["constfold", "layout", "constfold"])
+    rewriter_config_with_trt.meta_optimizer_iterations = (
+        rewriter_config_pb2.RewriterConfig.ONE)
+  else:
+    rewriter_config_with_trt.CopyFrom(rewriter_config)
+
+  if precision_mode.upper() not in TrtPrecisionMode.supported_precision_modes():
+    raise ValueError(("precision mode '{}' is not supported."
+                      "It should be one of {}").format(
+                          precision_mode,
+                          TrtPrecisionMode.supported_precision_modes))
+
+  optimizer = rewriter_config_with_trt.custom_optimizers.add()
+  optimizer.name = "TensorRTOptimizer"
+  optimizer.parameter_map["minimum_segment_size"].i = minimum_segment_size
+  optimizer.parameter_map["max_batch_size"].i = max_batch_size
+  optimizer.parameter_map["is_dynamic_op"].b = is_dynamic_op
+  optimizer.parameter_map[
+      "max_workspace_size_bytes"].i = max_workspace_size_bytes
+  optimizer.parameter_map["precision_mode"].s = _to_bytes(precision_mode)
+  optimizer.parameter_map["maximum_cached_engines"].i = maximum_cached_engines
+  if cached_engine_batches:
+    if not isinstance(cached_engine_batches, list):
+      raise TypeError("cached_engine_batches should be a list.")
+    if len(cached_engine_batches) > maximum_cached_engines:
+      raise ValueError("cached_engine_batches should not contain more than "
+                       "maximum_cached_engines items.")
+    optimizer.parameter_map["cached_engine_batches"].list.i.extend(
+        cached_engine_batches)
+  optimizer.parameter_map["use_calibration"].b = use_calibration
+  return rewriter_config_with_trt
+
+
+def create_inference_graph(input_graph_def,
+                           outputs,
+                           max_batch_size=1,
+                           max_workspace_size_bytes=2 << 20,
+                           precision_mode=TrtPrecisionMode.FP32,
+                           minimum_segment_size=3,
+                           is_dynamic_op=False,
+                           maximum_cached_engines=1,
+                           cached_engine_batches=None,
+                           use_calibration=True,
+                           input_saved_model_dir=None,
+                           input_saved_model_tags=None,
+                           output_saved_model_dir=None,
+                           session_config=None):
+  """Python wrapper for the TRT transformation.
+
+  Args:
+    input_graph_def: a GraphDef object containing a model to be transformed. If
+      set to None, the graph will be read from the SavedModel loaded from
+      input_saved_model_dir.
+    outputs: list of tensors or node names for the model outputs. Only used when
+      input_graph_def is not None.
+    max_batch_size: max size for the input batch.
+    max_workspace_size_bytes: the maximum GPU temporary memory which the TRT
+      engine can use at execution time. This corresponds to the 'workspaceSize'
+      parameter of nvinfer1::IBuilder::setMaxWorkspaceSize().
+    precision_mode: one of TrtPrecisionMode.supported_precision_modes().
+    minimum_segment_size: the minimum number of nodes required for a subgraph to
+      be replaced by TRTEngineOp.
+    is_dynamic_op: whether to generate dynamic TRT ops which will build the TRT
+      network and engine at run time.
+    maximum_cached_engines: max number of cached TRT engines in dynamic TRT ops.
+      If the number of cached engines is already at max but none of them can
+      serve the input, the TRTEngineOp will fall back to run the TF function
+      based on which the TRTEngineOp is created.
+    cached_engine_batches: a list of batch sizes used to create cached
+      engines, only used when is_dynamic_op is True. The length of the list
+      should be <= maximum_cached_engines, and the dynamic TRT op will
+      use this list to determine the batch sizes of the cached engines, instead
+      of making the decision on the fly. This is useful when we know the most
+      common batch size(s) the application is going to generate.
+    use_calibration: this argument is ignored if precision_mode is not INT8. If
+      set to True, a calibration graph will be created to calibrate the missing
+      ranges. The calibration graph must be converted to an inference graph
+      using calib_graph_to_infer_graph() after running calibration. if set to
+      False, quantization nodes will be expected for every tensor in the graph
+      (exlcuding those which will be fused). If a range is missing, an error
+      will occur. Please note that accuracy may be negatively affected if there
+      is a mismatch between which tensors TRT quantizes and which tensors were
+      trained with fake quantization.
+    input_saved_model_dir: the directory to load the SavedModel which contains
+      the input graph to transforms. Used only when input_graph_def is None.
+    input_saved_model_tags: list of tags to load the SavedModel.
+    output_saved_model_dir: if not None, construct a SavedModel using the
+      returned GraphDef and save it to the specified directory. This option only
+      works when the input graph is loaded from a SavedModel, i.e. when
+      input_saved_model_dir is specified and input_graph_def is None.
+    session_config: the ConfigProto used to create a Session. It's also used as
+      a template to create a TRT-enabled ConfigProto for conversion. If not
+      specified, a default ConfigProto will be used.
+
+  Returns:
+    A GraphDef transformed from input_graph_def (or the SavedModel graph def
+    loaded from input_saved_model_dir, if input_graph_def is not present), where
+    all TRT compatible subgraphs are replaced with TRTEngineOps, and a TF
+    function is added for each of the subgraphs.
+
+    If is_dynamic_op is True, each TRTEngineOp will contain a serialized
+    subgraph GraphDef, which will be converted to a TRT engine at execution time
+    and the TRT engine will be cached for future usage. A new TRT engine will be
+    created each time when none of the cached engines match the input shapes. If
+    it fails to execute the TRT engine or the number of cached engines reaches
+    maximum_cached_engines, the op will fall back to call the corresponding TF
+    function.
+
+    If is_dynamic_op is False, each TRTEngineOp will contain a serialized TRT
+    engine created from the corresponding subgraph. No more engines will be
+    created on the fly, and the op will fall back to call the corresponding TF
+    function when it fails to execute the engine.
+
+  Raises:
+    ValueError: if the combination of the parameters is invalid.
+    RuntimeError: if the TensorRT library version is incompatible.
+  """
+  compiled_version = get_linked_tensorrt_version()
+  loaded_version = get_loaded_tensorrt_version()
+  version_mismatch = False
+  if loaded_version[0] < compiled_version[0]:
+    tf_logging.error(
+        "TensorRT version mismatch. Tensorflow was compiled against " +
+        "TensorRT %s but library loaded from environment is TensorRT %s" %
+        (".".join([str(x) for x in compiled_version]),
+         ".".join([str(x) for x in loaded_version])) +
+        ". Please make sure that correct version of TensorRT " +
+        "is available in the system and added to ldconfig or LD_LIBRARY_PATH")
+    raise RuntimeError("Incompatible TensorRT library version")
+  for i in zip(loaded_version, compiled_version):
+    if i[0] != i[1]:
+      tf_logging.warn("TensorRT mismatch. Compiled against version " +
+                      "%s, but loaded %s. Things may not work" %
+                      (".".join([str(x) for x in compiled_version]),
+                       ".".join([str(x) for x in loaded_version])))
+      version_mismatch = True
+      break
+  if not version_mismatch:
+    tf_logging.info("Running against TensorRT version %s" % ".".join(
+        [str(x) for x in loaded_version]))
+
+  if session_config is None:
+    session_config = config_pb2.ConfigProto()
+
+  if input_saved_model_tags is None:
+    input_saved_model_tags = [tag_constants.SERVING]
+  saved_model_loader = None
+  grappler_meta_graph_def = None
+
+  if input_graph_def is None:
+    # Read from SavedModel and freeze the graph if necessary.
+    if input_saved_model_dir is None:
+      raise ValueError("input_graph_def and input_saved_model_dir cannot be "
+                       "both None")
+    with ops.Graph().as_default():
+      with session.Session(config=session_config) as sess:
+        saved_model_loader = loader_impl.SavedModelLoader(input_saved_model_dir)
+        input_meta_graph_def = saved_model_loader.load(sess,
+                                                       input_saved_model_tags)
+        output_node_names = set()
+
+        def _gather_names(tensor_info):
+          """Get the node names from a TensorInfo."""
+          return set(
+              [tensor_info[key].name.split(":")[0] for key in tensor_info])
+
+        # Get input and outputs from all SignatureDef.
+        for key in input_meta_graph_def.signature_def:
+          signature_def = input_meta_graph_def.signature_def[key]
+          output_node_names.update(_gather_names(signature_def.inputs))
+          output_node_names.update(_gather_names(signature_def.outputs))
+
+        # Freeze the variables in the SavedModel graph and copy the frozen
+        # graph over.
+        frozen_graph_def = graph_util.convert_variables_to_constants(
+            sess, sess.graph.as_graph_def(add_shapes=True),
+            list(output_node_names))
+        grappler_meta_graph_def = meta_graph_pb2.MetaGraphDef()
+        grappler_meta_graph_def.graph_def.CopyFrom(frozen_graph_def)
+
+        # Copy the collections that are not variables.
+        for key in input_meta_graph_def.collection_def:
+          # TODO(laigd): currently we use the collection key to filter out
+          # collections that depend on variable ops, but this may miss some
+          # other user-defined collections. A better way would be to use
+          # CollectionDef::NodeList for the filtering.
+          if key not in [
+              "variables", "local_variables", "model_variables",
+              "trainable_variables", "train_op", "table_initializer"
+          ]:
+            grappler_meta_graph_def.collection_def[key].CopyFrom(
+                input_meta_graph_def.collection_def[key])
+
+        # Copy other information.
+        grappler_meta_graph_def.meta_info_def.CopyFrom(
+            input_meta_graph_def.meta_info_def)
+        for key in input_meta_graph_def.signature_def:
+          grappler_meta_graph_def.signature_def[key].CopyFrom(
+              input_meta_graph_def.signature_def[key])
+        # TODO(laigd): maybe add back AssetFileDef.
+  else:
+    if output_saved_model_dir is not None:
+      raise ValueError("output_saved_model_dir cannot be set when "
+                       "input_graph_def is set")
+    # Create MetaGraphDef from input graph.
+    graph = ops.Graph()
+    with graph.as_default():
+      importer.import_graph_def(input_graph_def, name="")
+    grappler_meta_graph_def = saver.export_meta_graph(
+        graph_def=graph.as_graph_def(add_shapes=True), graph=graph)
+    if outputs:
+      output_collection = meta_graph_pb2.CollectionDef()
+      output_list = output_collection.node_list.value
+      for i in outputs:
+        if isinstance(i, ops.Tensor):
+          output_list.append(_to_bytes(i.name))
+        else:
+          output_list.append(_to_bytes(i))
+      # TODO(laigd): use another key as the outputs are really not train_op.
+      grappler_meta_graph_def.collection_def["train_op"].CopyFrom(
+          output_collection)
+
+  # Create TRT-enabled ConfigProto.
+  session_config_with_trt = config_pb2.ConfigProto()
+  session_config_with_trt.CopyFrom(session_config)
+  rewriter_config = None
+  if (session_config_with_trt.HasField("graph_options") and
+      session_config_with_trt.graph_options.HasField("rewrite_options")):
+    rewriter_config = session_config_with_trt.graph_options.rewrite_options
+  rewriter_config_with_trt = get_tensorrt_rewriter_config(
+      rewriter_config, max_batch_size, max_workspace_size_bytes, precision_mode,
+      minimum_segment_size, is_dynamic_op, maximum_cached_engines,
+      cached_engine_batches, use_calibration)
+  session_config_with_trt.graph_options.rewrite_options.CopyFrom(
+      rewriter_config_with_trt)
+
+  # Run Grappler.
+  transformed_graph_def = tf_optimizer.OptimizeGraph(
+      session_config_with_trt, grappler_meta_graph_def, graph_id=b"tf_graph")
+
+  # Optionally write the transformed graphdef as SavedModel.
+  if output_saved_model_dir is not None:
+    saved_model_builder = builder.SavedModelBuilder(output_saved_model_dir)
+    with ops.Graph().as_default():
+      importer.import_graph_def(transformed_graph_def, name="")
+      # We don't use TRT here.
+      with session.Session(config=session_config) as sess:
+        saved_model_builder.add_meta_graph_and_variables(
+            sess,
+            input_saved_model_tags,
+            signature_def_map=grappler_meta_graph_def.signature_def)
+    # Ignore other meta graphs from the input SavedModel.
+    saved_model_builder.save()
+
+  return transformed_graph_def
+
+
+def calib_graph_to_infer_graph(calibration_graph_def, is_dynamic_op=False):
+  """Convert an existing calibration graph to inference graph.
+
+  Args:
+    calibration_graph_def: the calibration GraphDef object with calibration data
+    is_dynamic_op: whether to create dynamic static engines from calibration
+
+  Returns:
+    New GraphDef with TRTEngineOps placed in graph replacing calibration nodes.
+  Raises:
+    RuntimeError: if the returned status message is malformed.
+  """
+
+  is_calib_graph = False
+  for n in calibration_graph_def.node:
+    if n.op == "TRTEngineOp":
+      is_calib_graph = is_calib_graph or not n.attr["calibration_data"].s
+  if not is_calib_graph:
+    tf_logging.error(
+        "Not a calib graph. Doesn't seem to contain any calibration nodes.")
+    return None
+  graph_str = calibration_graph_def.SerializeToString()
+  out = calib_convert(graph_str, is_dynamic_op)
+  status = _to_string(out[0])
+  output_graph_def_string = out[1]
+  del graph_str  # Save some memory
+  if len(status) < 2:
+    raise _impl.UnknownError(None, None, status)
+  if status[:2] != "OK":
+    msg = status.split(";")
+    if len(msg) == 1:
+      raise RuntimeError("Status message is malformed {}".format(status))
+    # pylint: disable=protected-access
+    raise _impl._make_specific_exception(None, None, ";".join(msg[1:]),
+                                         int(msg[0]))
+    # pylint: enable=protected-access
+  output_graph_def = graph_pb2.GraphDef()
+  output_graph_def.ParseFromString(output_graph_def_string)
+  del output_graph_def_string  # Save some memory
+  return output_graph_def
diff --git a/tensorflow/contrib/tensorrt/python/trt_convert_test.py b/tensorflow/python/compiler/tensorrt/trt_convert_test.py
similarity index 99%
rename from tensorflow/contrib/tensorrt/python/trt_convert_test.py
rename to tensorflow/python/compiler/tensorrt/trt_convert_test.py
index abd822c7b7..7f839117cd 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert_test.py
+++ b/tensorflow/python/compiler/tensorrt/trt_convert_test.py
@@ -23,10 +23,10 @@ import os
 # pylint: disable=unused-import
 from tensorflow.compiler.tf2tensorrt.python.ops import trt_ops
 # pylint: enable=unused-import
-from tensorflow.contrib.tensorrt.python import trt_convert
 from tensorflow.core.framework import graph_pb2
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import rewriter_config_pb2
+from tensorflow.python.compiler.tensorrt import trt_convert
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import graph_util
 from tensorflow.python.framework import importer
-- 
GitLab


From d25e50f6cc86f76f5f0cdb8e8ae58c9f4e29abcd Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Fri, 1 Feb 2019 12:48:13 -0800
Subject: [PATCH 1774/2345] Enable lite_test and interpreter_test in oss

PiperOrigin-RevId: 232021084
---
 tensorflow/lite/build_def.bzl            |  3 +-
 tensorflow/lite/python/BUILD             | 19 +++++++-
 tensorflow/lite/python/lite_flex_test.py | 58 ++++++++++++++++++++++++
 tensorflow/lite/python/lite_test.py      | 23 ----------
 4 files changed, 78 insertions(+), 25 deletions(-)
 create mode 100644 tensorflow/lite/python/lite_flex_test.py

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index f35bc40c74..181590e5a9 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -2,6 +2,7 @@
 
 load(
     "//tensorflow:tensorflow.bzl",
+    "tf_binary_additional_srcs",
     "tf_cc_shared_object",
     "tf_cc_test",
 )
@@ -168,7 +169,7 @@ def tf_to_tflite(name, src, options, out):
         srcs = [src],
         outs = [out],
         cmd = toco_cmdline,
-        tools = ["//tensorflow/lite/toco:toco"],
+        tools = ["//tensorflow/lite/toco:toco"] + tf_binary_additional_srcs(),
     )
 
 def tflite_to_json(name, src, out):
diff --git a/tensorflow/lite/python/BUILD b/tensorflow/lite/python/BUILD
index 00366623da..a8c8c9f03b 100644
--- a/tensorflow/lite/python/BUILD
+++ b/tensorflow/lite/python/BUILD
@@ -23,7 +23,9 @@ py_test(
     srcs = ["interpreter_test.py"],
     data = ["//tensorflow/lite/python/testdata:interpreter_test_data"],
     srcs_version = "PY2AND3",
-    tags = ["no_oss"],
+    tags = [
+        "no_windows",
+    ],
     deps = [
         ":interpreter",
         "//tensorflow/python:client_testlib",
@@ -69,6 +71,21 @@ py_test(
     shard_count = 4,
     srcs_version = "PY2AND3",
     tags = [
+        "no_windows",
+    ],
+    deps = [
+        ":lite",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_test_lib",
+    ],
+)
+
+py_test(
+    name = "lite_flex_test",
+    srcs = ["lite_flex_test.py"],
+    srcs_version = "PY2AND3",
+    tags = [
+        # TODO(b/111881877): Enable in oss after resolving op registry issues.
         "no_oss",
         "no_windows",
     ],
diff --git a/tensorflow/lite/python/lite_flex_test.py b/tensorflow/lite/python/lite_flex_test.py
new file mode 100644
index 0000000000..a5ae629413
--- /dev/null
+++ b/tensorflow/lite/python/lite_flex_test.py
@@ -0,0 +1,58 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for lite.py functionality related to select TF op usage."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.lite.python import lite
+from tensorflow.lite.python.interpreter import Interpreter
+from tensorflow.python.client import session
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.platform import test
+
+
+@test_util.run_v1_only('b/120545219')
+class FromSessionTest(test_util.TensorFlowTestCase):
+
+  def testFlexMode(self):
+    in_tensor = array_ops.placeholder(
+        shape=[1, 16, 16, 3], dtype=dtypes.float32)
+    out_tensor = in_tensor + in_tensor
+    sess = session.Session()
+
+    # Convert model and ensure model is not None.
+    converter = lite.TFLiteConverter.from_session(sess, [in_tensor],
+                                                  [out_tensor])
+    converter.target_ops = set([lite.OpsSet.SELECT_TF_OPS])
+    tflite_model = converter.convert()
+    self.assertTrue(tflite_model)
+
+    # Ensures the model contains TensorFlow ops.
+    # TODO(nupurgarg): Check values once there is a Python delegate interface.
+    interpreter = Interpreter(model_content=tflite_model)
+    with self.assertRaises(RuntimeError) as error:
+      interpreter.allocate_tensors()
+    self.assertIn(
+        'Regular TensorFlow ops are not supported by this interpreter. Make '
+        'sure you invoke the Flex delegate before inference.',
+        str(error.exception))
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/lite/python/lite_test.py b/tensorflow/lite/python/lite_test.py
index 83fd56bf1d..ff91d5e297 100644
--- a/tensorflow/lite/python/lite_test.py
+++ b/tensorflow/lite/python/lite_test.py
@@ -511,29 +511,6 @@ class FromSessionTest(test_util.TensorFlowTestCase):
     # Ensure that the quantized weights tflite model is smaller.
     self.assertTrue(len(quantized_tflite) < len(float_tflite))
 
-  def testFlexMode(self):
-    in_tensor = array_ops.placeholder(
-        shape=[1, 16, 16, 3], dtype=dtypes.float32)
-    out_tensor = in_tensor + in_tensor
-    sess = session.Session()
-
-    # Convert model and ensure model is not None.
-    converter = lite.TFLiteConverter.from_session(sess, [in_tensor],
-                                                  [out_tensor])
-    converter.target_ops = set([lite.OpsSet.SELECT_TF_OPS])
-    tflite_model = converter.convert()
-    self.assertTrue(tflite_model)
-
-    # Ensures the model contains TensorFlow ops.
-    # TODO(nupurgarg): Check values once there is a Python delegate interface.
-    interpreter = Interpreter(model_content=tflite_model)
-    with self.assertRaises(RuntimeError) as error:
-      interpreter.allocate_tensors()
-    self.assertIn(
-        'Regular TensorFlow ops are not supported by this interpreter. Make '
-        'sure you invoke the Flex delegate before inference.',
-        str(error.exception))
-
   def testFloatTocoConverter(self):
     """Tests deprecated test TocoConverter."""
     in_tensor = array_ops.placeholder(
-- 
GitLab


From 1b78e28439f2f185785b20c3779cf587bc74b8dc Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 1 Feb 2019 13:18:14 -0800
Subject: [PATCH 1775/2345] Make CreateBroadcastableScalarConstant a free
 helper method

---
 tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc | 6 +++---
 tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h  | 7 -------
 2 files changed, 3 insertions(+), 10 deletions(-)

diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
index c36260ccf8..ce04b1d8fb 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
@@ -351,7 +351,7 @@ nvinfer1::ITensor* Converter::CreateConstantLayer(
   return trt_tensor;
 }
 
-tensorflow::Status Converter::CreateBroadcastableScalarConstant(
+tensorflow::Status CreateBroadcastableScalarConstant(
     OpConverterParams* params, float value, const nvinfer1::Dims& dims,
     const nvinfer1::ITensor** tensor) {
   // In order to be broadcastable, the number of dims has to match.
@@ -2477,7 +2477,7 @@ tensorflow::Status ConvertLeakyRelu(OpConverterParams* params) {
   const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
   // Create const for alpha.
   const nvinfer1::ITensor* const_alpha_tensor = nullptr;
-  TF_RETURN_IF_ERROR(params->converter->CreateBroadcastableScalarConstant(
+  TF_RETURN_IF_ERROR(CreateBroadcastableScalarConstant(
       params, alpha, tensor->getDimensions(), &const_alpha_tensor));
   // alpha * x
   nvinfer1::IElementWiseLayer* mul_layer =
@@ -2648,7 +2648,7 @@ tensorflow::Status ConvertRelu6(OpConverterParams* params) {
 
   // Create a constant layer to store the floating point weight i.e. 6.0f
   const nvinfer1::ITensor* const6_tensor = nullptr;
-  TF_RETURN_IF_ERROR(params->converter->CreateBroadcastableScalarConstant(
+  TF_RETURN_IF_ERROR(CreateBroadcastableScalarConstant(
       params, 6.0f, relu_layer->getOutput(0)->getDimensions(), &const6_tensor));
 
   // ElementWise Min Operation
diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
index 89b9a0f645..00373051f5 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
@@ -492,13 +492,6 @@ class Converter {
   nvinfer1::ITensor* CreateConstantLayer(const TRT_ShapedWeights& weights,
                                          const nvinfer1::Dims& dims);
 
-  // Creates an IConstantLayer scalar which can be broadcasted against dims by
-  // IElementWise layer.
-  Status CreateBroadcastableScalarConstant(OpConverterParams* params,
-                                           float value,
-                                           const nvinfer1::Dims& dims,
-                                           const nvinfer1::ITensor** tensor);
-
  private:
   // Verify the provided batch_size is consistent with batch_size_ and update it
   // if necessary.
-- 
GitLab


From fdef22593395be26f81bed4a58b5c30d948cf03d Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Fri, 1 Feb 2019 13:17:45 -0800
Subject: [PATCH 1776/2345] Internal change

PiperOrigin-RevId: 232025890
---
 tensorflow/lite/BUILD | 31 +++++++++++++++++++++++++------
 1 file changed, 25 insertions(+), 6 deletions(-)

diff --git a/tensorflow/lite/BUILD b/tensorflow/lite/BUILD
index 2cc661f2ba..7d71ca32d5 100644
--- a/tensorflow/lite/BUILD
+++ b/tensorflow/lite/BUILD
@@ -4,9 +4,9 @@ package(
 
 licenses(["notice"])  # Apache 2.0
 
-load("//tensorflow:tensorflow.bzl", "tf_cc_test")
+load("//tensorflow:tensorflow.bzl", "tf_cc_test", "if_not_windows")
 load("//tensorflow/lite:build_def.bzl", "tflite_copts")
-load("//tensorflow:tensorflow.bzl", "if_not_windows")
+load("//tensorflow/lite:special_rules.bzl", "tflite_portable_test_suite")
 
 exports_files(glob([
     "testdata/*.bin",
@@ -191,8 +191,6 @@ cc_library(
         ":util",
         "//tensorflow/lite/c:c_api_internal",
         "//tensorflow/lite/core/api:api",
-        "//tensorflow/lite/kernels:eigen_support",
-        "//tensorflow/lite/kernels:gemm_support",
         "//tensorflow/lite/nnapi:nnapi_implementation",
         "//tensorflow/lite/profiling:profiler",
         "//tensorflow/lite/schema:schema_fbs",
@@ -219,6 +217,9 @@ cc_test(
     name = "string_util_test",
     size = "small",
     srcs = ["string_util_test.cc"],
+    tags = [
+        "tflite_not_portable_ios",  # TODO(b/117786830)
+    ],
     deps = [
         ":framework",
         ":string_util",
@@ -233,10 +234,12 @@ cc_test(
     name = "interpreter_test",
     size = "small",
     srcs = ["interpreter_test.cc"],
+    tags = [
+        "tflite_not_portable_ios",  # TODO(b/117786830)
+    ],
     deps = [
         ":framework",
         ":string_util",
-        "//tensorflow/lite/c:c_api_internal",
         "//tensorflow/lite/core/api",
         "//tensorflow/lite/kernels:builtin_ops",
         "//tensorflow/lite/kernels:kernel_util",
@@ -252,6 +255,9 @@ cc_test(
     name = "graph_info_test",
     size = "small",
     srcs = ["graph_info_test.cc"],
+    tags = [
+        "tflite_not_portable_ios",  # TODO(b/117786830)
+    ],
     deps = [
         ":framework",
         "//tensorflow/lite/testing:util",
@@ -264,6 +270,9 @@ cc_test(
     name = "simple_memory_arena_test",
     size = "small",
     srcs = ["simple_memory_arena_test.cc"],
+    tags = [
+        "tflite_not_portable_ios",  # TODO(b/117786830)
+    ],
     deps = [
         ":simple_memory_arena",
         "//tensorflow/lite/testing:util",
@@ -284,9 +293,11 @@ cc_test(
         "testdata/test_model.bin",
         "testdata/test_model_broken.bin",
     ],
+    tags = [
+        "tflite_not_portable",
+    ],
     deps = [
         ":framework",
-        "//tensorflow/lite/c:c_api_internal",
         "//tensorflow/lite/core/api",
         "//tensorflow/lite/kernels:builtin_ops",
         "//tensorflow/lite/testing:util",
@@ -323,6 +334,9 @@ cc_test(
     name = "mutable_op_resolver_test",
     size = "small",
     srcs = ["mutable_op_resolver_test.cc"],
+    tags = [
+        "tflite_not_portable_ios",  # TODO(b/117786830)
+    ],
     deps = [
         ":framework",
         "//tensorflow/lite/testing:util",
@@ -344,9 +358,14 @@ cc_test(
     name = "util_test",
     size = "small",
     srcs = ["util_test.cc"],
+    tags = [
+        "tflite_not_portable_ios",  # TODO(b/117786830)
+    ],
     deps = [
         ":util",
         "//tensorflow/lite/c:c_api_internal",
         "@com_google_googletest//:gtest",
     ],
 )
+
+tflite_portable_test_suite()
-- 
GitLab


From 216263a0eb555f7d0674b54285ead62bd6c74d82 Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Fri, 1 Feb 2019 13:19:24 -0800
Subject: [PATCH 1777/2345] Fix two links in Quantization-aware training page.

PiperOrigin-RevId: 232026119
---
 tensorflow/contrib/quantize/README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/quantize/README.md b/tensorflow/contrib/quantize/README.md
index 5b8da92491..b335e1af69 100644
--- a/tensorflow/contrib/quantize/README.md
+++ b/tensorflow/contrib/quantize/README.md
@@ -8,9 +8,9 @@ for both training and inference. There are two aspects to this:
 
 For efficient inference, TensorFlow combines batch normalization with the preceding
 convolutional and fully-connected layers prior to quantization by
-[folding batch norm layers](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/quantize/python/fold_batch_norms.py){:.external}. 
+[folding batch norm layers](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/quantize/python/fold_batch_norms.py){:.external}.
 
-The quantization error is modeled using [fake quantization](../api_guides/python/array_ops.md#Fake_quantization)
+The quantization error is modeled using [fake quantization](../../api_guides/python/array_ops.md#Fake_quantization)
 nodes to simulate the effect of quantization in the forward and backward passes. The
 forward-pass models quantization, while the backward-pass models quantization as a
 straight-through estimator. Both the forward- and backward-pass simulate the quantization
@@ -105,7 +105,7 @@ toco \
   --std_value=127.5 --mean_value=127.5
 ```
 
-See the documentation for `tf.contrib.quantize` and [TensorFlow Lite](../lite/).
+See the documentation for `tf.contrib.quantize` and [TensorFlow Lite](../../lite/).
 
 
 ## Quantized accuracy results
-- 
GitLab


From 3bc5132922c0185b88043992688c76eeec2817b7 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 1 Feb 2019 13:27:15 -0800
Subject: [PATCH 1778/2345] Add a new bazel config option for C++ 17

PiperOrigin-RevId: 232027497
---
 .bazelrc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.bazelrc b/.bazelrc
index c70c571361..17285afdb3 100644
--- a/.bazelrc
+++ b/.bazelrc
@@ -90,6 +90,12 @@ build --define=grpc_no_ares=true
 build:dynamic_kernels --define=dynamic_loaded_kernels=true
 build:dynamic_kernels --copt=-DAUTOLOAD_DYNAMIC_KERNELS
 
+# Build TF with C++ 17 features.
+build:c++17 --cxxopt=-std=c++1z
+build:c++17 --cxxopt=-stdlib=libc++
+build:c++1z --cxxopt=-std=c++1z
+build:c++1z --cxxopt=-stdlib=libc++
+
 # Default paths for TF_SYSTEM_LIBS
 build --define=PREFIX=/usr
 build --define=LIBDIR=$(PREFIX)/lib
-- 
GitLab


From 4aae38176f1d7551332630003d8e4de07d8652bb Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 1 Feb 2019 13:37:55 -0800
Subject: [PATCH 1779/2345] Automated rollback of commit
 354b9c8f52b209ad031f5187f4188899b80507d1. Revert #25273.

PiperOrigin-RevId: 232029310
---
 tensorflow/core/api_def/python_api/api_def_DecodeWav.pbtxt | 4 +---
 tensorflow/tools/api/golden/v1/tensorflow.audio.pbtxt      | 4 ----
 2 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/tensorflow/core/api_def/python_api/api_def_DecodeWav.pbtxt b/tensorflow/core/api_def/python_api/api_def_DecodeWav.pbtxt
index 28f4514bd8..d6fd4691f7 100644
--- a/tensorflow/core/api_def/python_api/api_def_DecodeWav.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_DecodeWav.pbtxt
@@ -1,6 +1,4 @@
 op {
   graph_op_name: "DecodeWav"
-  endpoint {
-    name: "audio.decode_wav"
-  }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.audio.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.audio.pbtxt
index db1ff06efb..ce29615f72 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.audio.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.audio.pbtxt
@@ -4,8 +4,4 @@ tf_module {
     name: "encode_wav"
     argspec: "args=[\'audio\', \'sample_rate\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "decode_wav"
-    argspec: "args=[\'contents\', \'audio\', \'sample_rate\', \'desired_channels\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
 }
-- 
GitLab


From 1bf52f4cef02dde5eaf91ace9ec87ac5afb1b943 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 1 Feb 2019 13:44:16 -0800
Subject: [PATCH 1780/2345] A few micro optimizations to common sub-expression
 elimination in ArithmeticOptimizer:

* Memoize node signatures.
* Don't discard candidate information after each outer iteration.
* Switch to using absl::flat_hash_map.
* Avoid copying regular inputs for non-commutative op types.

Add missing support for PrintV2.

PiperOrigin-RevId: 232030525
---
 tensorflow/core/grappler/op_types.cc          |  4 +-
 .../optimizers/arithmetic_optimizer.cc        | 51 ++++++++++---------
 2 files changed, 30 insertions(+), 25 deletions(-)

diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc
index 2236f57f74..4eb04df416 100644
--- a/tensorflow/core/grappler/op_types.cc
+++ b/tensorflow/core/grappler/op_types.cc
@@ -355,7 +355,9 @@ bool IsPolygamma(const NodeDef& node) { return node.op() == "Polygamma"; }
 
 bool IsPow(const NodeDef& node) { return node.op() == "Pow"; }
 
-bool IsPrint(const NodeDef& node) { return node.op() == "Print"; }
+bool IsPrint(const NodeDef& node) {
+  return node.op() == "Print" || node.op() == "PrintV2";
+}
 
 bool IsProd(const NodeDef& node) { return node.op() == "Prod"; }
 
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index 6c53ba9e77..e2c8b7cdbc 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -3257,13 +3257,17 @@ class UniqueNodes {
   }
 
  private:
-  uint64 ComputeSignature(const NodeDef& node) const;
+  uint64 ComputeSignature(const NodeDef& node);
   bool SameNode(const NodeDef& node1, const NodeDef& node2) const;
 
-  std::unordered_map<uint64, std::vector<NodeDef*>> rep_;
+  absl::flat_hash_map<uint64, std::vector<NodeDef*>> rep_;
+  absl::flat_hash_map<const NodeDef*, uint64> memoized_signatures_;
 };
 
-uint64 UniqueNodes::ComputeSignature(const NodeDef& node) const {
+uint64 UniqueNodes::ComputeSignature(const NodeDef& node) {
+  auto it = memoized_signatures_.find(&node);
+  if (it != memoized_signatures_.end()) return it->second;
+
   uint64 h = Hash64(node.op());
   h = Hash64Combine(Hash64(node.device()), h);
 
@@ -3277,6 +3281,7 @@ uint64 UniqueNodes::ComputeSignature(const NodeDef& node) const {
     h = Hash64CombineUnordered(Hash64(attr.first), h);
     h = Hash64CombineUnordered(FastAttrValueHash(attr.second), h);
   }
+  memoized_signatures_.emplace(&node, h);
   return h;
 }
 
@@ -3297,31 +3302,29 @@ bool UniqueNodes::SameNode(const NodeDef& node1, const NodeDef& node2) const {
   // Compare inputs.
   if (IsCommutative(node1)) {
     std::vector<string> inputs1(node1.input().begin(), node1.input().end());
-    std::vector<string> inputs2(node2.input().begin(), node2.input().end());
     std::sort(inputs1.begin(), inputs1.end());
+    std::vector<string> inputs2(node2.input().begin(), node2.input().end());
     std::sort(inputs2.begin(), inputs2.end());
     return inputs1 == inputs2;
   } else {
-    std::vector<string> regular_inputs1;
-    std::vector<string> regular_inputs2;
-    std::vector<string> ctrl_inputs1;
-    std::vector<string> ctrl_inputs2;
-    for (int index = 0; index < node1.input_size(); ++index) {
+    // The order or ordinary inputs matters.
+    int index = 0;
+    for (; index < node1.input_size(); ++index) {
       if (IsControlInput(node1.input(index))) {
-        ctrl_inputs1.push_back(node1.input(index));
-        ctrl_inputs2.push_back(node2.input(index));
-      } else {
-        regular_inputs1.push_back(node1.input(index));
-        regular_inputs2.push_back(node2.input(index));
+        break;
+      } else if (node1.input(index) != node2.input(index)) {
+        return false;
       }
     }
-    if (regular_inputs1 != regular_inputs2) {
-      return false;
-    }
-    std::sort(ctrl_inputs1.begin(), ctrl_inputs1.end());
-    std::sort(ctrl_inputs2.begin(), ctrl_inputs2.end());
-    if (ctrl_inputs1 != ctrl_inputs2) {
-      return false;
+    // The order of control inputs does not matter.
+    if (index < node1.input_size()) {
+      std::vector<string> ctrl_inputs1(node1.input().begin() + index,
+                                       node1.input().end());
+      std::sort(ctrl_inputs1.begin(), ctrl_inputs1.end());
+      std::vector<string> ctrl_inputs2(node2.input().begin() + index,
+                                       node2.input().end());
+      std::sort(ctrl_inputs2.begin(), ctrl_inputs2.end());
+      return ctrl_inputs1 != ctrl_inputs2;
     }
   }
 
@@ -3348,8 +3351,8 @@ bool ArithmeticOptimizer::CanDedup(const NodeDef& node) const {
   if (node.device().find("SPU") != string::npos) {
     return false;
   }
-  // Workaround for Assert mistakenly being labeled as stateful.
-  if (IsAssert(node)) {
+  // Workaround for Assert and Print mistakenly being labeled as stateful.
+  if (IsAssert(node) || IsPrint(node)) {
     return true;
   }
   return IsFreeOfSideEffect(node);
@@ -3387,9 +3390,9 @@ void ArithmeticOptimizer::DedupComputations() {
 
   bool stop = true;
   std::set<int> duplicates;
+  UniqueNodes nodes;
   do {
     stop = true;
-    UniqueNodes nodes;
     for (int i = 0; i < optimized_graph_->node_size(); ++i) {
       if (duplicates.find(i) != duplicates.end()) {
         continue;
-- 
GitLab


From c7fd42e2d5738f491b0cd88eab8d307c04a19b62 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 1 Feb 2019 14:00:59 -0800
Subject: [PATCH 1781/2345] Fixes for hwloc build files.

PiperOrigin-RevId: 232033559
---
 third_party/hwloc/BUILD               |   8 +-
 third_party/hwloc/BUILD.bazel         | 308 ++++++++++++++------------
 third_party/hwloc/static-components.h |  26 +++
 3 files changed, 196 insertions(+), 146 deletions(-)
 create mode 100644 third_party/hwloc/static-components.h

diff --git a/third_party/hwloc/BUILD b/third_party/hwloc/BUILD
index 2f5d02becb..2469c95668 100644
--- a/third_party/hwloc/BUILD
+++ b/third_party/hwloc/BUILD
@@ -1 +1,7 @@
-# Dummy BUILD file to make this directory a package.
+# BUILD file to make this directory a package.
+
+licenses(["notice"])
+
+exports_files(
+    ["static-components.h"],
+)
diff --git a/third_party/hwloc/BUILD.bazel b/third_party/hwloc/BUILD.bazel
index 1f29301645..5d636cabd3 100644
--- a/third_party/hwloc/BUILD.bazel
+++ b/third_party/hwloc/BUILD.bazel
@@ -31,11 +31,11 @@ template_rule(
     src = "include/hwloc/autogen/config.h.in",
     out = "include/hwloc/autogen/config.h",
     substitutions = {
-        "#undef HWLOC_VERSION": "#define HWLOC_VERSION \"2.0.3\"",
         "#undef HWLOC_VERSION_MAJOR": "#define HWLOC_VERSION_MAJOR 2",
         "#undef HWLOC_VERSION_MINOR": "#define HWLOC_VERSION_MINOR 0",
         "#undef HWLOC_VERSION_RELEASE": "#define HWLOC_VERSION_RELEASE 3",
         "#undef HWLOC_VERSION_GREEK": "#define HWLOC_VERSION_GREEK \"\"",
+        "#undef HWLOC_VERSION": "#define HWLOC_VERSION \"2.0.3\"",
         "#undef HWLOC_LINUX_SYS": "#define HWLOC_LINUX_SYS 1",
         "#undef hwloc_pid_t": "#define hwloc_pid_t pid_t",
         "#undef hwloc_thread_t": "#define hwloc_thread_t pthread_t",
@@ -46,154 +46,168 @@ template_rule(
     },
 )
 
+_INCLUDE_PRIVATE_HWLOC_AUTOIGEN_CONFIG_H_COMMON_SUBS = {
+    "#undef HAVE_CLOCK_GETTIME": "#define HAVE_CLOCK_GETTIME 1",
+    "#undef HAVE_CTYPE_H": "#define HAVE_CTYPE_H 1",
+    "#undef HAVE_DECL_CTL_HW": "#define HAVE_DECL_CTL_HW 0",
+    "#undef HAVE_DECL_FABSF": "#define HAVE_DECL_FABSF 1",
+    "#undef HAVE_DECL_GETEXECNAME": "#define HAVE_DECL_GETEXECNAME 0",
+    "#undef HAVE_DECL_GETMODULEFILENAME": "#define HAVE_DECL_GETMODULEFILENAME 0",
+    "#undef HAVE_DECL_GETPROGNAME": "#define HAVE_DECL_GETPROGNAME 0",
+    "#undef HAVE_DECL_HW_NCPU": "#define HAVE_DECL_HW_NCPU 0",
+    "#undef HAVE_DECL_MODFF": "#define HAVE_DECL_MODFF 1",
+    "#undef HAVE_DECL_PTHREAD_GETAFFINITY_NP": "#define HAVE_DECL_PTHREAD_GETAFFINITY_NP 1",
+    "#undef HAVE_DECL_PTHREAD_SETAFFINITY_NP": "#define HAVE_DECL_PTHREAD_SETAFFINITY_NP 1",
+    "#undef HAVE_DECL_RUNNING_ON_VALGRIND": "#define HAVE_DECL_RUNNING_ON_VALGRIND 0",
+    "#undef HAVE_DECL_SCHED_GETCPU": "#define HAVE_DECL_SCHED_GETCPU 1",
+    "#undef HAVE_DECL_SNPRINTF": "#define HAVE_DECL_SNPRINTF 1",
+    "#undef HAVE_DECL_STRTOULL": "#define HAVE_DECL_STRTOULL 1",
+    "#undef HAVE_DECL__PUTENV": "#define HAVE_DECL__PUTENV 0",
+    "#undef HAVE_DECL__SC_LARGE_PAGESIZE": "#define HAVE_DECL__SC_LARGE_PAGESIZE 0",
+    "#undef HAVE_DECL__SC_NPROCESSORS_CONF": "#define HAVE_DECL__SC_NPROCESSORS_CONF 1",
+    "#undef HAVE_DECL__SC_NPROCESSORS_ONLN": "#define HAVE_DECL__SC_NPROCESSORS_ONLN 1",
+    "#undef HAVE_DECL__SC_NPROC_CONF": "#define HAVE_DECL__SC_NPROC_CONF 0",
+    "#undef HAVE_DECL__SC_NPROC_ONLN": "#define HAVE_DECL__SC_NPROC_ONLN 0",
+    "#undef HAVE_DECL__SC_PAGESIZE": "#define HAVE_DECL__SC_PAGESIZE 1",
+    "#undef HAVE_DECL__SC_PAGE_SIZE": "#define HAVE_DECL__SC_PAGE_SIZE 1",
+    "#undef HAVE_DECL__STRDUP": "#define HAVE_DECL__STRDUP 0",
+    "#undef HAVE_DIRENT_H": "#define HAVE_DIRENT_H 1",
+    "#undef HAVE_DLFCN_H": "#define HAVE_DLFCN_H 1",
+    "#undef HAVE_FFS": "#define HAVE_FFS 1",
+    "#undef HAVE_FFSL": "#define HAVE_FFSL 1",
+    "#undef HAVE_GETPAGESIZE": "#define HAVE_GETPAGESIZE 1",
+    "#undef HAVE_INTTYPES_H": "#define HAVE_INTTYPES_H 1",
+    "#undef HAVE_LANGINFO_H": "#define HAVE_LANGINFO_H 1",
+    "#undef HAVE_LOCALE_H": "#define HAVE_LOCALE_H 1",
+    "#undef HAVE_MALLOC_H": "#define HAVE_MALLOC_H 1",
+    "#undef HAVE_MEMALIGN": "#define HAVE_MEMALIGN 1",
+    "#undef HAVE_MEMORY_H": "#define HAVE_MEMORY_H 1",
+    "#undef HAVE_MKSTEMP": "#define HAVE_MKSTEMP 1",
+    "#undef HAVE_NL_LANGINFO": "#define HAVE_NL_LANGINFO 1",
+    "#undef HAVE_OPENAT": "#define HAVE_OPENAT 1",
+    "#undef HAVE_POSIX_MEMALIGN": "#define HAVE_POSIX_MEMALIGN 1",
+    "#undef HAVE_PROGRAM_INVOCATION_NAME": "#define HAVE_PROGRAM_INVOCATION_NAME 1",
+    "#undef HAVE_PTHREAD_T": "#define HAVE_PTHREAD_T 1",
+    "#undef HAVE_PUTWC": "#define HAVE_PUTWC 1",
+    "#undef HAVE_SETLOCALE": "#define HAVE_SETLOCALE 1",
+    "#undef HAVE_SSIZE_T": "#define HAVE_SSIZE_T 1",
+    "#undef HAVE_STDINT_H": "#define HAVE_STDINT_H 1",
+    "#undef HAVE_STDLIB_H": "#define HAVE_STDLIB_H 1",
+    "#undef HAVE_STRCASECMP": "#define HAVE_STRCASECMP 1",
+    "#undef HAVE_STRFTIME": "#define HAVE_STRFTIME 1",
+    "#undef HAVE_STRINGS_H": "#define HAVE_STRINGS_H 1",
+    "#undef HAVE_STRING_H": "#define HAVE_STRING_H 1",
+    "#undef HAVE_STRNCASECMP": "#define HAVE_STRNCASECMP 1",
+    "#undef HAVE_SYS_MMAN_H": "#define HAVE_SYS_MMAN_H 1",
+    "#undef HAVE_SYS_PARAM_H": "#define HAVE_SYS_PARAM_H 1",
+    "#undef HAVE_SYS_STAT_H": "#define HAVE_SYS_STAT_H 1",
+    "#undef HAVE_SYS_SYSCTL_H": "#define HAVE_SYS_SYSCTL_H 1",
+    "#undef HAVE_SYS_TYPES_H": "#define HAVE_SYS_TYPES_H 1",
+    "#undef HAVE_SYS_UTSNAME_H": "#define HAVE_SYS_UTSNAME_H 1",
+    "#undef HAVE_TIME_H": "#define HAVE_TIME_H 1",
+    "#undef HAVE_UNAME": "#define HAVE_UNAME 1",
+    "#undef HAVE_UNISTD_H": "#define HAVE_UNISTD_H 1",
+    "#undef HAVE_USELOCALE": "#define HAVE_USELOCALE 1",
+    "#undef HAVE_WCHAR_T": "#define HAVE_WCHAR_T 1",
+    "#undef HAVE_X11_KEYSYM_H": "#define HAVE_X11_KEYSYM_H 1",
+    "#undef HAVE_X11_XLIB_H": "#define HAVE_X11_XLIB_H 1",
+    "#undef HAVE_X11_XUTIL_H": "#define HAVE_X11_XUTIL_H 1",
+    "#undef HAVE_XLOCALE_H": "#define HAVE_XLOCALE_H 1",
+    "#undef HAVE___PROGNAME": "#define HAVE___PROGNAME 1",
+    "#undef HWLOC_C_HAVE_VISIBILITY": "#define HWLOC_C_HAVE_VISIBILITY 1",
+    "#undef HWLOC_HAVE_ATTRIBUTE": "#define HWLOC_HAVE_ATTRIBUTE 1",
+    "#undef HWLOC_HAVE_ATTRIBUTE_ALIGNED": "#define HWLOC_HAVE_ATTRIBUTE_ALIGNED 1",
+    "#undef HWLOC_HAVE_ATTRIBUTE_ALWAYS_INLINE": "#define HWLOC_HAVE_ATTRIBUTE_ALWAYS_INLINE 1",
+    "#undef HWLOC_HAVE_ATTRIBUTE_COLD": "#define HWLOC_HAVE_ATTRIBUTE_COLD 1",
+    "#undef HWLOC_HAVE_ATTRIBUTE_CONST": "#define HWLOC_HAVE_ATTRIBUTE_CONST 1",
+    "#undef HWLOC_HAVE_ATTRIBUTE_DEPRECATED": "#define HWLOC_HAVE_ATTRIBUTE_DEPRECATED 1",
+    "#undef HWLOC_HAVE_ATTRIBUTE_FORMAT": "#define HWLOC_HAVE_ATTRIBUTE_FORMAT 1",
+    "#undef HWLOC_HAVE_ATTRIBUTE_HOT": "#define HWLOC_HAVE_ATTRIBUTE_HOT 1",
+    "#undef HWLOC_HAVE_ATTRIBUTE_MALLOC": "#define HWLOC_HAVE_ATTRIBUTE_MALLOC 1",
+    "#undef HWLOC_HAVE_ATTRIBUTE_MAY_ALIAS": "#define HWLOC_HAVE_ATTRIBUTE_MAY_ALIAS 1",
+    "#undef HWLOC_HAVE_ATTRIBUTE_NONNULL": "#define HWLOC_HAVE_ATTRIBUTE_NONNULL 1",
+    "#undef HWLOC_HAVE_ATTRIBUTE_NORETURN": "#define HWLOC_HAVE_ATTRIBUTE_NORETURN 1",
+    "#undef HWLOC_HAVE_ATTRIBUTE_NO_INSTRUMENT_FUNCTION": "#define HWLOC_HAVE_ATTRIBUTE_NO_INSTRUMENT_FUNCTION 1",
+    "#undef HWLOC_HAVE_ATTRIBUTE_PACKED": "#define HWLOC_HAVE_ATTRIBUTE_PACKED 1",
+    "#undef HWLOC_HAVE_ATTRIBUTE_PURE": "#define HWLOC_HAVE_ATTRIBUTE_PURE 1",
+    "#undef HWLOC_HAVE_ATTRIBUTE_SENTINEL": "#define HWLOC_HAVE_ATTRIBUTE_SENTINEL 1",
+    "#undef HWLOC_HAVE_ATTRIBUTE_UNUSED": "#define HWLOC_HAVE_ATTRIBUTE_UNUSED 1",
+    "#undef HWLOC_HAVE_ATTRIBUTE_WARN_UNUSED_RESULT": "#define HWLOC_HAVE_ATTRIBUTE_WARN_UNUSED_RESULT 1",
+    "#undef HWLOC_HAVE_ATTRIBUTE_WEAK_ALIAS": "#define HWLOC_HAVE_ATTRIBUTE_WEAK_ALIAS 1",
+    "#undef HWLOC_HAVE_CPU_SET": "#define HWLOC_HAVE_CPU_SET 1",
+    "#undef HWLOC_HAVE_CPU_SET_S": "#define HWLOC_HAVE_CPU_SET_S 1",
+    "#undef HWLOC_HAVE_DECL_FFS": "#define HWLOC_HAVE_DECL_FFS 1",
+    "#undef HWLOC_HAVE_DECL_FFSL": "#define HWLOC_HAVE_DECL_FFSL 1",
+    "#undef HWLOC_HAVE_DECL_STRCASECMP": "#define HWLOC_HAVE_DECL_STRCASECMP 1",
+    "#undef HWLOC_HAVE_DECL_STRNCASECMP": "#define HWLOC_HAVE_DECL_STRNCASECMP 1",
+    "#undef HWLOC_HAVE_FFS": "#define HWLOC_HAVE_FFS 1",
+    "#undef HWLOC_HAVE_FFSL": "#define HWLOC_HAVE_FFSL 1",
+    "#undef HWLOC_HAVE_LIBTERMCAP": "#define HWLOC_HAVE_LIBTERMCAP 1",
+    "#undef HWLOC_HAVE_LINUXIO": "#define HWLOC_HAVE_LINUXIO 1",
+    "#undef HWLOC_HAVE_PTHREAD_MUTEX": "#define HWLOC_HAVE_PTHREAD_MUTEX 1",
+    "#undef HWLOC_HAVE_SCHED_SETAFFINITY": "#define HWLOC_HAVE_SCHED_SETAFFINITY 1",
+    "#undef HWLOC_HAVE_STDINT_H": "#define HWLOC_HAVE_STDINT_H 1",
+    "#undef HWLOC_HAVE_SYSCALL": "#define HWLOC_HAVE_SYSCALL 1",
+    "#undef HWLOC_HAVE_X11_KEYSYM": "#define HWLOC_HAVE_X11_KEYSYM 1",
+    "#undef HWLOC_HAVE_X86_CPUID": "#define HWLOC_HAVE_X86_CPUID 1",
+    "#undef HWLOC_LINUX_SYS": "#define HWLOC_LINUX_SYS 1",
+    "#undef HWLOC_SIZEOF_UNSIGNED_INT": "#define HWLOC_SIZEOF_UNSIGNED_INT 4",
+    "#undef HWLOC_SIZEOF_UNSIGNED_LONG": "#define HWLOC_SIZEOF_UNSIGNED_LONG 8",
+    "#undef HWLOC_SYM_PREFIX": "#define HWLOC_SYM_PREFIX hwloc_",
+    "#undef HWLOC_SYM_PREFIX_CAPS": "#define HWLOC_SYM_PREFIX_CAPS HWLOC_",
+    "#undef HWLOC_SYM_TRANSFORM": "#define HWLOC_SYM_TRANSFORM 0",
+    "#undef HWLOC_USE_NCURSES": "#define HWLOC_USE_NCURSES 1",
+    "#undef HWLOC_VERSION": "#define HWLOC_VERSION \"2.0.3\"",
+    "#undef HWLOC_VERSION_GREEK": "#define HWLOC_VERSION_GREEK \"\"",
+    "#undef HWLOC_VERSION_MAJOR": "#define HWLOC_VERSION_MAJOR 2",
+    "#undef HWLOC_VERSION_MINOR": "#define HWLOC_VERSION_MINOR 0",
+    "#undef HWLOC_VERSION_RELEASE": "#define HWLOC_VERSION_RELEASE 3",
+    "#undef HWLOC_X86_64_ARCH": "#define HWLOC_X86_64_ARCH 1",
+    "#undef LT_OBJDIR": "#define LT_OBJDIR \".libs/\"",
+    "#undef PACKAGE": "#define PACKAGE \"hwloc\"",
+    "#undef PACKAGE_BUGREPORT": "#define PACKAGE_BUGREPORT \"http://github.com/open-mpi/hwloc/i",
+    "#undef PACKAGE_NAME": "#define PACKAGE_NAME \"hwloc\"",
+    "#undef PACKAGE_STRING": "#define PACKAGE_STRING \"hwloc 2.0.3\"",
+    "#undef PACKAGE_TARNAME": "#define PACKAGE_TARNAME \"hwloc\"",
+    "#undef PACKAGE_URL": "#define PACKAGE_URL \"\"",
+    "#undef PACKAGE_VERSION": "#define PACKAGE_VERSION \"2.0.3\"",
+    "#undef SIZEOF_UNSIGNED_INT": "#define SIZEOF_UNSIGNED_INT 4",
+    "#undef SIZEOF_UNSIGNED_LONG": "#define SIZEOF_UNSIGNED_LONG 8",
+    "#undef SIZEOF_VOID_P": "#define SIZEOF_VOID_P 8",
+    "#undef STDC_HEADERS": "#define STDC_HEADERS 1",
+    "# undef _HPUX_SOURCE": "# define _HPUX_SOURCE 1",
+    "# undef _ALL_SOURCE": "# define _ALL_SOURCE 1",
+    "# undef _GNU_SOURCE": "# define _GNU_SOURCE 1",
+    "# undef _POSIX_PTHREAD_SEMANTICS": "# define _POSIX_PTHREAD_SEMANTICS 1",
+    "# undef _TANDEM_SOURCE": "# define _TANDEM_SOURCE 1",
+    "# undef __EXTENSIONS__": "# define __EXTENSIONS__ 1",
+    "#undef VERSION": "#define VERSION \"2.0.3\"",
+    "#undef _HPUX_SOURCE": "#define _HPUX_SOURCE 1",
+    "#undef hwloc_pid_t": "#define hwloc_pid_t pid_t",
+    "#undef hwloc_thread_t": "#define hwloc_thread_t pthread_t",
+}
+
+_INCLUDE_PRIVATE_HWLOC_AUTOIGEN_CONFIG_H_CUDA_SUBS = {
+    "#undef HAVE_CUDA": "#undef HAVE_CUDA 1",
+    "#undef HAVE_CUDA_H": "#undef HAVE_CUDA_H 1",
+    "#undef HAVE_CUDA_RUNTIME_API_H": "#undef HAVE_CUDA_RUNTIME_API_H 1",
+} + _INCLUDE_PRIVATE_HWLOC_AUTOIGEN_CONFIG_H_COMMON_SUBS
+
 template_rule(
     name = "include_private_hwloc_autogen__config_h",
     src = "include/private/autogen/config.h.in",
     out = "include/private/autogen/config.h",
-    substitutions = {
-        "#undef HAVE_CLOCK_GETTIME": "#define HAVE_CLOCK_GETTIME 1",
-        "#undef HAVE_CTYPE_H": "#define HAVE_CTYPE_H 1",
-        "#undef HAVE_DECL_CTL_HW": "#define HAVE_DECL_CTL_HW 0",
-        "#undef HAVE_DECL_FABSF": "#define HAVE_DECL_FABSF 1",
-        "#undef HAVE_DECL_GETEXECNAME": "#define HAVE_DECL_GETEXECNAME 0",
-        "#undef HAVE_DECL_GETMODULEFILENAME": "#define HAVE_DECL_GETMODULEFILENAME 0",
-        "#undef HAVE_DECL_GETPROGNAME": "#define HAVE_DECL_GETPROGNAME 0",
-        "#undef HAVE_DECL_HW_NCPU": "#define HAVE_DECL_HW_NCPU 0",
-        "#undef HAVE_DECL_MODFF": "#define HAVE_DECL_MODFF 1",
-        "#undef HAVE_DECL_PTHREAD_GETAFFINITY_NP": "#define HAVE_DECL_PTHREAD_GETAFFINITY_NP 1",
-        "#undef HAVE_DECL_PTHREAD_SETAFFINITY_NP": "#define HAVE_DECL_PTHREAD_SETAFFINITY_NP 1",
-        "#undef HAVE_DECL_RUNNING_ON_VALGRIND": "#define HAVE_DECL_RUNNING_ON_VALGRIND 0",
-        "#undef HAVE_DECL_SCHED_GETCPU": "#define HAVE_DECL_SCHED_GETCPU 1",
-        "#undef HAVE_DECL_SNPRINTF": "#define HAVE_DECL_SNPRINTF 1",
-        "#undef HAVE_DECL_STRTOULL": "#define HAVE_DECL_STRTOULL 1",
-        "#undef HAVE_DECL__PUTENV": "#define HAVE_DECL__PUTENV 0",
-        "#undef HAVE_DECL__SC_LARGE_PAGESIZE": "#define HAVE_DECL__SC_LARGE_PAGESIZE 0",
-        "#undef HAVE_DECL__SC_NPROCESSORS_CONF": "#define HAVE_DECL__SC_NPROCESSORS_CONF 1",
-        "#undef HAVE_DECL__SC_NPROCESSORS_ONLN": "#define HAVE_DECL__SC_NPROCESSORS_ONLN 1",
-        "#undef HAVE_DECL__SC_NPROC_CONF": "#define HAVE_DECL__SC_NPROC_CONF 0",
-        "#undef HAVE_DECL__SC_NPROC_ONLN": "#define HAVE_DECL__SC_NPROC_ONLN 0",
-        "#undef HAVE_DECL__SC_PAGESIZE": "#define HAVE_DECL__SC_PAGESIZE 1",
-        "#undef HAVE_DECL__SC_PAGE_SIZE": "#define HAVE_DECL__SC_PAGE_SIZE 1",
-        "#undef HAVE_DECL__STRDUP": "#define HAVE_DECL__STRDUP 0",
-        "#undef HAVE_DIRENT_H": "#define HAVE_DIRENT_H 1",
-        "#undef HAVE_DLFCN_H": "#define HAVE_DLFCN_H 1",
-        "#undef HAVE_FFS": "#define HAVE_FFS 1",
-        "#undef HAVE_FFSL": "#define HAVE_FFSL 1",
-        "#undef HAVE_GETPAGESIZE": "#define HAVE_GETPAGESIZE 1",
-        "#undef HAVE_INTTYPES_H": "#define HAVE_INTTYPES_H 1",
-        "#undef HAVE_LANGINFO_H": "#define HAVE_LANGINFO_H 1",
-        "#undef HAVE_LOCALE_H": "#define HAVE_LOCALE_H 1",
-        "#undef HAVE_MALLOC_H": "#define HAVE_MALLOC_H 1",
-        "#undef HAVE_MEMALIGN": "#define HAVE_MEMALIGN 1",
-        "#undef HAVE_MEMORY_H": "#define HAVE_MEMORY_H 1",
-        "#undef HAVE_MKSTEMP": "#define HAVE_MKSTEMP 1",
-        "#undef HAVE_NL_LANGINFO": "#define HAVE_NL_LANGINFO 1",
-        "#undef HAVE_OPENAT": "#define HAVE_OPENAT 1",
-        "#undef HAVE_POSIX_MEMALIGN": "#define HAVE_POSIX_MEMALIGN 1",
-        "#undef HAVE_PROGRAM_INVOCATION_NAME": "#define HAVE_PROGRAM_INVOCATION_NAME 1",
-        "#undef HAVE_PTHREAD_T": "#define HAVE_PTHREAD_T 1",
-        "#undef HAVE_PUTWC": "#define HAVE_PUTWC 1",
-        "#undef HAVE_SETLOCALE": "#define HAVE_SETLOCALE 1",
-        "#undef HAVE_SSIZE_T": "#define HAVE_SSIZE_T 1",
-        "#undef HAVE_STDINT_H": "#define HAVE_STDINT_H 1",
-        "#undef HAVE_STDLIB_H": "#define HAVE_STDLIB_H 1",
-        "#undef HAVE_STRCASECMP": "#define HAVE_STRCASECMP 1",
-        "#undef HAVE_STRFTIME": "#define HAVE_STRFTIME 1",
-        "#undef HAVE_STRINGS_H": "#define HAVE_STRINGS_H 1",
-        "#undef HAVE_STRING_H": "#define HAVE_STRING_H 1",
-        "#undef HAVE_STRNCASECMP": "#define HAVE_STRNCASECMP 1",
-        "#undef HAVE_SYS_MMAN_H": "#define HAVE_SYS_MMAN_H 1",
-        "#undef HAVE_SYS_PARAM_H": "#define HAVE_SYS_PARAM_H 1",
-        "#undef HAVE_SYS_STAT_H": "#define HAVE_SYS_STAT_H 1",
-        "#undef HAVE_SYS_SYSCTL_H": "#define HAVE_SYS_SYSCTL_H 1",
-        "#undef HAVE_SYS_TYPES_H": "#define HAVE_SYS_TYPES_H 1",
-        "#undef HAVE_SYS_UTSNAME_H": "#define HAVE_SYS_UTSNAME_H 1",
-        "#undef HAVE_TIME_H": "#define HAVE_TIME_H 1",
-        "#undef HAVE_UNAME": "#define HAVE_UNAME 1",
-        "#undef HAVE_UNISTD_H": "#define HAVE_UNISTD_H 1",
-        "#undef HAVE_USELOCALE": "#define HAVE_USELOCALE 1",
-        "#undef HAVE_WCHAR_T": "#define HAVE_WCHAR_T 1",
-        "#undef HAVE_X11_KEYSYM_H": "#define HAVE_X11_KEYSYM_H 1",
-        "#undef HAVE_X11_XLIB_H": "#define HAVE_X11_XLIB_H 1",
-        "#undef HAVE_X11_XUTIL_H": "#define HAVE_X11_XUTIL_H 1",
-        "#undef HAVE_XLOCALE_H": "#define HAVE_XLOCALE_H 1",
-        "#undef HAVE___PROGNAME": "#define HAVE___PROGNAME 1",
-        "#undef HWLOC_C_HAVE_VISIBILITY": "#define HWLOC_C_HAVE_VISIBILITY 1",
-        "#undef HWLOC_HAVE_ATTRIBUTE": "#define HWLOC_HAVE_ATTRIBUTE 1",
-        "#undef HWLOC_HAVE_ATTRIBUTE_ALIGNED": "#define HWLOC_HAVE_ATTRIBUTE_ALIGNED 1",
-        "#undef HWLOC_HAVE_ATTRIBUTE_ALWAYS_INLINE": "#define HWLOC_HAVE_ATTRIBUTE_ALWAYS_INLINE 1",
-        "#undef HWLOC_HAVE_ATTRIBUTE_COLD": "#define HWLOC_HAVE_ATTRIBUTE_COLD 1",
-        "#undef HWLOC_HAVE_ATTRIBUTE_CONST": "#define HWLOC_HAVE_ATTRIBUTE_CONST 1",
-        "#undef HWLOC_HAVE_ATTRIBUTE_DEPRECATED": "#define HWLOC_HAVE_ATTRIBUTE_DEPRECATED 1",
-        "#undef HWLOC_HAVE_ATTRIBUTE_FORMAT": "#define HWLOC_HAVE_ATTRIBUTE_FORMAT 1",
-        "#undef HWLOC_HAVE_ATTRIBUTE_HOT": "#define HWLOC_HAVE_ATTRIBUTE_HOT 1",
-        "#undef HWLOC_HAVE_ATTRIBUTE_MALLOC": "#define HWLOC_HAVE_ATTRIBUTE_MALLOC 1",
-        "#undef HWLOC_HAVE_ATTRIBUTE_MAY_ALIAS": "#define HWLOC_HAVE_ATTRIBUTE_MAY_ALIAS 1",
-        "#undef HWLOC_HAVE_ATTRIBUTE_NONNULL": "#define HWLOC_HAVE_ATTRIBUTE_NONNULL 1",
-        "#undef HWLOC_HAVE_ATTRIBUTE_NORETURN": "#define HWLOC_HAVE_ATTRIBUTE_NORETURN 1",
-        "#undef HWLOC_HAVE_ATTRIBUTE_NO_INSTRUMENT_FUNCTION": "#define HWLOC_HAVE_ATTRIBUTE_NO_INSTRUMENT_FUNCTION 1",
-        "#undef HWLOC_HAVE_ATTRIBUTE_PACKED": "#define HWLOC_HAVE_ATTRIBUTE_PACKED 1",
-        "#undef HWLOC_HAVE_ATTRIBUTE_PURE": "#define HWLOC_HAVE_ATTRIBUTE_PURE 1",
-        "#undef HWLOC_HAVE_ATTRIBUTE_SENTINEL": "#define HWLOC_HAVE_ATTRIBUTE_SENTINEL 1",
-        "#undef HWLOC_HAVE_ATTRIBUTE_UNUSED": "#define HWLOC_HAVE_ATTRIBUTE_UNUSED 1",
-        "#undef HWLOC_HAVE_ATTRIBUTE_WARN_UNUSED_RESULT": "#define HWLOC_HAVE_ATTRIBUTE_WARN_UNUSED_RESULT 1",
-        "#undef HWLOC_HAVE_ATTRIBUTE_WEAK_ALIAS": "#define HWLOC_HAVE_ATTRIBUTE_WEAK_ALIAS 1",
-        "#undef HWLOC_HAVE_CPU_SET": "#define HWLOC_HAVE_CPU_SET 1",
-        "#undef HWLOC_HAVE_CPU_SET_S": "#define HWLOC_HAVE_CPU_SET_S 1",
-        "#undef HWLOC_HAVE_DECL_FFS": "#define HWLOC_HAVE_DECL_FFS 1",
-        "#undef HWLOC_HAVE_DECL_FFSL": "#define HWLOC_HAVE_DECL_FFSL 1",
-        "#undef HWLOC_HAVE_DECL_STRCASECMP": "#define HWLOC_HAVE_DECL_STRCASECMP 1",
-        "#undef HWLOC_HAVE_DECL_STRNCASECMP": "#define HWLOC_HAVE_DECL_STRNCASECMP 1",
-        "#undef HWLOC_HAVE_FFS": "#define HWLOC_HAVE_FFS 1",
-        "#undef HWLOC_HAVE_FFSL": "#define HWLOC_HAVE_FFSL 1",
-        "#undef HWLOC_HAVE_LIBTERMCAP": "#define HWLOC_HAVE_LIBTERMCAP 1",
-        "#undef HWLOC_HAVE_LINUXIO": "#define HWLOC_HAVE_LINUXIO 1",
-        "#undef HWLOC_HAVE_PTHREAD_MUTEX": "#define HWLOC_HAVE_PTHREAD_MUTEX 1",
-        "#undef HWLOC_HAVE_SCHED_SETAFFINITY": "#define HWLOC_HAVE_SCHED_SETAFFINITY 1",
-        "#undef HWLOC_HAVE_STDINT_H": "#define HWLOC_HAVE_STDINT_H 1",
-        "#undef HWLOC_HAVE_SYSCALL": "#define HWLOC_HAVE_SYSCALL 1",
-        "#undef HWLOC_HAVE_X11_KEYSYM": "#define HWLOC_HAVE_X11_KEYSYM 1",
-        "#undef HWLOC_HAVE_X86_CPUID": "#define HWLOC_HAVE_X86_CPUID 1",
-        "#undef HWLOC_LINUX_SYS": "#define HWLOC_LINUX_SYS 1",
-        "#undef HWLOC_SIZEOF_UNSIGNED_INT": "#define HWLOC_SIZEOF_UNSIGNED_INT 4",
-        "#undef HWLOC_SIZEOF_UNSIGNED_LONG": "#define HWLOC_SIZEOF_UNSIGNED_LONG 8",
-        "#undef HWLOC_SYM_PREFIX": "#define HWLOC_SYM_PREFIX hwloc_",
-        "#undef HWLOC_SYM_PREFIX_CAPS": "#define HWLOC_SYM_PREFIX_CAPS HWLOC_",
-        "#undef HWLOC_SYM_TRANSFORM": "#define HWLOC_SYM_TRANSFORM 0",
-        "#undef HWLOC_USE_NCURSES": "#define HWLOC_USE_NCURSES 1",
-        "#undef HWLOC_VERSION": "#define HWLOC_VERSION \"2.0.3\"",
-        "#undef HWLOC_VERSION_GREEK": "#define HWLOC_VERSION_GREEK \"\"",
-        "#undef HWLOC_VERSION_MAJOR": "#define HWLOC_VERSION_MAJOR 2",
-        "#undef HWLOC_VERSION_MINOR": "#define HWLOC_VERSION_MINOR 0",
-        "#undef HWLOC_VERSION_RELEASE": "#define HWLOC_VERSION_RELEASE 3",
-        "#undef HWLOC_X86_64_ARCH": "#define HWLOC_X86_64_ARCH 1",
-        "#undef LT_OBJDIR": "#define LT_OBJDIR \".libs/\"",
-        "#undef PACKAGE": "#define PACKAGE \"hwloc\"",
-        "#undef PACKAGE_BUGREPORT": "#define PACKAGE_BUGREPORT \"http://github.com/open-mpi/hwloc/i",
-        "#undef PACKAGE_NAME": "#define PACKAGE_NAME \"hwloc\"",
-        "#undef PACKAGE_STRING": "#define PACKAGE_STRING \"hwloc 2.0.3\"",
-        "#undef PACKAGE_TARNAME": "#define PACKAGE_TARNAME \"hwloc\"",
-        "#undef PACKAGE_URL": "#define PACKAGE_URL \"\"",
-        "#undef PACKAGE_VERSION": "#define PACKAGE_VERSION \"2.0.3\"",
-        "#undef SIZEOF_UNSIGNED_INT": "#define SIZEOF_UNSIGNED_INT 4",
-        "#undef SIZEOF_UNSIGNED_LONG": "#define SIZEOF_UNSIGNED_LONG 8",
-        "#undef SIZEOF_VOID_P": "#define SIZEOF_VOID_P 8",
-        "#undef STDC_HEADERS": "#define STDC_HEADERS 1",
-        "# undef _HPUX_SOURCE": "# define _HPUX_SOURCE 1",
-        "# undef _ALL_SOURCE": "# define _ALL_SOURCE 1",
-        "# undef _GNU_SOURCE": "# define _GNU_SOURCE 1",
-        "# undef _POSIX_PTHREAD_SEMANTICS": "# define _POSIX_PTHREAD_SEMANTICS 1",
-        "# undef _TANDEM_SOURCE": "# define _TANDEM_SOURCE 1",
-        "# undef __EXTENSIONS__": "# define __EXTENSIONS__ 1",
-        "#undef VERSION": "#define VERSION \"2.0.3\"",
-        "#undef _HPUX_SOURCE": "#define _HPUX_SOURCE 1",
-        "#undef hwloc_pid_t": "#define hwloc_pid_t pid_t",
-        "#undef hwloc_thread_t": "#define hwloc_thread_t pthread_t",
-    } + if_cuda({
-        "#undef HAVE_CUDA": "#undef HAVE_CUDA 1",
-        "#undef HAVE_CUDA_H": "#undef HAVE_CUDA_H 1",
-        "#undef HAVE_CUDA_RUNTIME_API_H": "#undef HAVE_CUDA_RUNTIME_API_H 1",
-    }),
+    substitutions = if_cuda(
+        _INCLUDE_PRIVATE_HWLOC_AUTOIGEN_CONFIG_H_CUDA_SUBS,
+        if_false = _INCLUDE_PRIVATE_HWLOC_AUTOIGEN_CONFIG_H_COMMON_SUBS,
+    ),
+)
+
+template_rule(
+    name = "move_static_components_h",
+    src = "@org_tensorflow//third_party/hwloc:static-components.h",
+    out = "hwloc/static-components.h",
+    substitutions = {},
 )
 
 cc_library(
@@ -247,6 +261,10 @@ cc_library(
         "-parse_headers",
         "-layering_check",
     ],
+    includes = [
+        "hwloc",
+        "include",
+    ],
     deps = [],
 )
 
diff --git a/third_party/hwloc/static-components.h b/third_party/hwloc/static-components.h
new file mode 100644
index 0000000000..8cae42a9c1
--- /dev/null
+++ b/third_party/hwloc/static-components.h
@@ -0,0 +1,26 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef THIRD_PARTY_HWLOC_STATIC_COMPONENTS_H_
+#define THIRD_PARTY_HWLOC_STATIC_COMPONENTS_H_
+
+#include <private/internal-components.h>
+static const struct hwloc_component* hwloc_static_components[] = {
+    &hwloc_noos_component,      &hwloc_xml_component,
+    &hwloc_synthetic_component, &hwloc_xml_nolibxml_component,
+    &hwloc_linux_component,     &hwloc_linuxio_component,
+    &hwloc_x86_component,       NULL};
+
+#endif  // THIRD_PARTY_HWLOC_STATIC_COMPONENTS_H_
-- 
GitLab


From 5679c6f03a5f0053e8946c61da99eb3480e133aa Mon Sep 17 00:00:00 2001
From: Karl Weinmeister <11586922+kweinmeister@users.noreply.github.com>
Date: Fri, 1 Feb 2019 16:11:47 -0600
Subject: [PATCH 1782/2345] Add example code to tf.data.Dataset.filter()
 documentation

---
 tensorflow/python/data/ops/dataset_ops.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index 766c6d5395..6f90b52e69 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -1100,6 +1100,21 @@ class DatasetV2(object):
   def filter(self, predicate):
     """Filters this dataset according to `predicate`.
 
+    ```python
+    # NOTE: The following examples use `{ ... }` to represent the
+    # contents of a dataset.
+    a = { 1, 2, 3 }
+    b = { 4, 5, 6, 7 }
+
+    a.filter(lambda x: x < 3) == { 1, 2 }
+
+    # `tf.math.equal(x, y)` is required for equality comparison
+    def filter_fn(x):
+      return tf.math.equal(x, 4)
+
+    b.filter(filter_fn) == { 4 }
+    ```
+
     Args:
       predicate: A function mapping a nested structure of tensors (having shapes
         and types defined by `self.output_shapes` and `self.output_types`) to a
-- 
GitLab


From e1d20b3b4f25047679cf34107f8d87329cc9070f Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 1 Feb 2019 14:01:52 -0800
Subject: [PATCH 1783/2345] Add direct dependencies to op libraries.

PiperOrigin-RevId: 232033752
---
 tensorflow/c/BUILD                            | 3 +++
 tensorflow/compiler/jit/BUILD                 | 4 ++++
 tensorflow/compiler/jit/kernels/BUILD         | 1 +
 tensorflow/compiler/tf2xla/kernels/BUILD      | 5 +++++
 tensorflow/core/common_runtime/eager/BUILD    | 2 ++
 tensorflow/core/debug/BUILD                   | 2 ++
 tensorflow/core/distributed_runtime/BUILD     | 3 +++
 tensorflow/core/distributed_runtime/rpc/BUILD | 6 ++++++
 tensorflow/tools/graph_transforms/BUILD       | 9 +++++++++
 9 files changed, 35 insertions(+)

diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index 6e50a09bfc..ef52a28460 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -300,9 +300,12 @@ tf_cuda_cc_test(
         "//tensorflow/core:lib",
         "//tensorflow/core:math_ops_op_lib",
         "//tensorflow/core:nn_ops_op_lib",
+        "//tensorflow/core:no_op_op_lib",
         "//tensorflow/core:proto_text",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:sendrecv_ops_op_lib",
         "//tensorflow/core:spectral_ops_op_lib",
+        "//tensorflow/core:state_ops_op_lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core/kernels:array",
diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index 55e2e6d11f..3cae081ce7 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -179,14 +179,18 @@ cc_library(
         "//tensorflow/core:control_flow_ops_op_lib",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:core_cpu_internal",
+        "//tensorflow/core:dataset_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:math_ops_op_lib",
         "//tensorflow/core:nn_ops_op_lib",
+        "//tensorflow/core:no_op_op_lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:resource_variable_ops_op_lib",
+        "//tensorflow/core:sendrecv_ops_op_lib",
+        "//tensorflow/core:state_ops_op_lib",
         "//tensorflow/core:stream_executor_no_cuda",
         "//tensorflow/core/kernels:cast_op",
         "//tensorflow/core/kernels:constant_op",
diff --git a/tensorflow/compiler/jit/kernels/BUILD b/tensorflow/compiler/jit/kernels/BUILD
index 0583774714..bab824c15f 100644
--- a/tensorflow/compiler/jit/kernels/BUILD
+++ b/tensorflow/compiler/jit/kernels/BUILD
@@ -25,6 +25,7 @@ cc_library(
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:state_ops_op_lib",
         "//tensorflow/core:stream_executor_no_cuda",
         "//tensorflow/core/kernels:variable_ops",
         "@com_google_absl//absl/container:flat_hash_map",
diff --git a/tensorflow/compiler/tf2xla/kernels/BUILD b/tensorflow/compiler/tf2xla/kernels/BUILD
index bfe668f793..69353fe87d 100644
--- a/tensorflow/compiler/tf2xla/kernels/BUILD
+++ b/tensorflow/compiler/tf2xla/kernels/BUILD
@@ -147,6 +147,7 @@ tf_kernel_library(
         "//tensorflow/compiler/xla/client/lib:triangular_solve",
         "//tensorflow/core:bitwise_ops_op_lib",
         "//tensorflow/core:control_flow_ops_op_lib",
+        "//tensorflow/core:data_flow_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:image_ops_op_lib",
@@ -155,10 +156,14 @@ tf_kernel_library(
         "//tensorflow/core:list_ops_op_lib",
         "//tensorflow/core:math_ops_op_lib",
         "//tensorflow/core:nn_ops_op_lib",
+        "//tensorflow/core:no_op_op_lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:random_ops_op_lib",
         "//tensorflow/core:resource_variable_ops_op_lib",
+        "//tensorflow/core:sendrecv_ops_op_lib",
+        "//tensorflow/core:sparse_ops_op_lib",
         "//tensorflow/core:spectral_ops_op_lib",
+        "//tensorflow/core:state_ops_op_lib",
         "//tensorflow/core:stateless_random_ops_op_lib",
         "//tensorflow/core:training_ops_op_lib",
         "//tensorflow/core/kernels:bounds_check",
diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD
index 6f2680bf8f..aef64da794 100644
--- a/tensorflow/core/common_runtime/eager/BUILD
+++ b/tensorflow/core/common_runtime/eager/BUILD
@@ -191,7 +191,9 @@ tf_cc_test(
         "//tensorflow/core:lib",
         "//tensorflow/core:math_ops_op_lib",
         "//tensorflow/core:nn_ops_op_lib",
+        "//tensorflow/core:no_op_op_lib",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:sendrecv_ops_op_lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core/kernels:constant_op",
diff --git a/tensorflow/core/debug/BUILD b/tensorflow/core/debug/BUILD
index e1961b822b..f8c07dde46 100644
--- a/tensorflow/core/debug/BUILD
+++ b/tensorflow/core/debug/BUILD
@@ -229,7 +229,9 @@ tf_cc_test(
         "//tensorflow/core:master_proto_cc",
         "//tensorflow/core:math_ops_op_lib",
         "//tensorflow/core:nn_ops_op_lib",
+        "//tensorflow/core:no_op_op_lib",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:sendrecv_ops_op_lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
diff --git a/tensorflow/core/distributed_runtime/BUILD b/tensorflow/core/distributed_runtime/BUILD
index 351dbebf79..da88f9cb93 100644
--- a/tensorflow/core/distributed_runtime/BUILD
+++ b/tensorflow/core/distributed_runtime/BUILD
@@ -591,7 +591,9 @@ tf_cc_test(
         "//tensorflow/core:core_cpu_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:no_op_op_lib",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:sendrecv_ops_op_lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
@@ -658,6 +660,7 @@ tf_cuda_cc_test(
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:master_proto_cc",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:state_ops_op_lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
diff --git a/tensorflow/core/distributed_runtime/rpc/BUILD b/tensorflow/core/distributed_runtime/rpc/BUILD
index 7e4070c875..607963484d 100644
--- a/tensorflow/core/distributed_runtime/rpc/BUILD
+++ b/tensorflow/core/distributed_runtime/rpc/BUILD
@@ -327,11 +327,15 @@ tf_cc_binary(
         ":grpc_server_lib",
         "//tensorflow:grpc++",
         "//tensorflow/core:core_cpu",
+        "//tensorflow/core:data_flow_ops_op_lib",
         "//tensorflow/core:framework_internal",
         "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:lib",
+        "//tensorflow/core:lookup_ops_op_lib",
         "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:no_op_op_lib",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:sendrecv_ops_op_lib",
         "//tensorflow/core/distributed_runtime:server_lib",
         "//tensorflow/core/kernels:data_flow",
     ],
@@ -353,6 +357,7 @@ tf_cc_binary(
         "//tensorflow/core:lib",
         "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:state_ops_op_lib",
         "//tensorflow/core:testlib",
         "//tensorflow/core/distributed_runtime:server_lib",
         "//tensorflow/core/kernels:constant_op",
@@ -492,6 +497,7 @@ tf_cuda_cc_test(
         "//tensorflow/core:lib",
         "//tensorflow/core:master_proto_cc",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:state_ops_op_lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
diff --git a/tensorflow/tools/graph_transforms/BUILD b/tensorflow/tools/graph_transforms/BUILD
index 41ed31a5c1..2145b3b0d5 100644
--- a/tensorflow/tools/graph_transforms/BUILD
+++ b/tensorflow/tools/graph_transforms/BUILD
@@ -136,6 +136,15 @@ cc_library(
     ] + if_not_windows([
         "//tensorflow/core/kernels:remote_fused_graph_rewriter_transform",
         "//tensorflow/core/kernels/hexagon:hexagon_rewriter_transform",
+        "//tensorflow/core:sparse_ops_op_lib",
+        "//tensorflow/core:parsing_ops_op_lib",
+        "//tensorflow/core:sendrecv_ops_op_lib",
+        "//tensorflow/core:io_ops_op_lib",
+        "//tensorflow/core:logging_ops_op_lib",
+        "//tensorflow/core:lookup_ops_op_lib",
+        "//tensorflow/core:data_flow_ops_op_lib",
+        "//tensorflow/core:no_op_op_lib",
+        "//tensorflow/core:state_ops_op_lib",
         "//tensorflow/core:user_ops_op_lib",
         "//tensorflow/core:training_ops_op_lib",
         "//tensorflow/core:string_ops_op_lib",
-- 
GitLab


From 8e38fb87fc50aaf0f086a21e1ca1f39361604234 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 1 Feb 2019 14:18:24 -0800
Subject: [PATCH 1784/2345] Automated rollback of commit
 67f67690b29257d8ee039bcb74d19e0b5fbbe4e7

PiperOrigin-RevId: 232037027
---
 tensorflow/core/distributed_runtime/rpc/BUILD |  14 --
 .../rpc/grpc_remote_worker.cc                 |  12 +-
 .../rpc/grpc_response_cache.cc                | 184 ------------------
 .../rpc/grpc_response_cache.h                 |  92 ---------
 .../rpc/grpc_rpc_factory.cc                   |   2 +-
 .../core/distributed_runtime/rpc/grpc_state.h |  85 ++------
 .../rpc/grpc_worker_service.cc                |  95 +++------
 .../rpc/grpc_worker_service.h                 |   7 +-
 8 files changed, 53 insertions(+), 438 deletions(-)
 delete mode 100644 tensorflow/core/distributed_runtime/rpc/grpc_response_cache.cc
 delete mode 100644 tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h

diff --git a/tensorflow/core/distributed_runtime/rpc/BUILD b/tensorflow/core/distributed_runtime/rpc/BUILD
index 607963484d..16078f9688 100644
--- a/tensorflow/core/distributed_runtime/rpc/BUILD
+++ b/tensorflow/core/distributed_runtime/rpc/BUILD
@@ -159,17 +159,6 @@ cc_library(
     ],
 )
 
-cc_library(
-    name = "grpc_response_cache",
-    srcs = ["grpc_response_cache.cc"],
-    hdrs = ["grpc_response_cache.h"],
-    deps = [
-        ":grpc_util",
-        "//tensorflow/core:lib",
-        "@com_google_absl//absl/time",
-    ],
-)
-
 tf_cuda_library(
     name = "grpc_worker_service",
     srcs = ["grpc_worker_service.cc"],
@@ -177,7 +166,6 @@ tf_cuda_library(
     deps = [
         ":async_service_interface",
         ":grpc_call",
-        ":grpc_response_cache",
         ":grpc_tensor_coding",
         ":grpc_util",
         ":grpc_worker_service_impl",
@@ -195,8 +183,6 @@ tf_cuda_library(
         "//tensorflow/core/distributed_runtime:worker_cache",
         "//tensorflow/core/distributed_runtime:worker_env",
         "//tensorflow/core/distributed_runtime:worker_session",
-        "@com_google_absl//absl/container:flat_hash_map",
-        "@com_google_absl//absl/time",
     ],
 )
 
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.cc b/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.cc
index 2479e7368b..2daefcb399 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.cc
@@ -39,8 +39,6 @@ limitations under the License.
 
 namespace tensorflow {
 
-const int kMaxWorkerRpcRetries = 10;
-
 class GrpcRemoteWorker : public WorkerInterface {
  public:
   explicit GrpcRemoteWorker(SharedGrpcChannelPtr channel,
@@ -261,19 +259,17 @@ class GrpcRemoteWorker : public WorkerInterface {
   // given callback, `done`, will be called when the RPC completes.
   void IssueRequest(const protobuf::Message* request,
                     protobuf::Message* response, const ::grpc::string& method,
-                    StatusCallback done, CallOptions* call_opts = nullptr,
-                    int max_retries = kMaxWorkerRpcRetries) {
+                    StatusCallback done, CallOptions* call_opts = nullptr) {
     new RPCState<protobuf::Message>(&stub_, cq_, method, *request, response,
                                     std::move(done), call_opts,
-                                    callback_threadpool_, max_retries);
+                                    callback_threadpool_);
   }
   void IssueRequest(const protobuf::Message* request, TensorResponse* response,
                     const ::grpc::string& method, StatusCallback done,
-                    CallOptions* call_opts = nullptr,
-                    int max_retries = kMaxWorkerRpcRetries) {
+                    CallOptions* call_opts = nullptr) {
     new RPCState<TensorResponse>(&stub_, cq_, method, *request, response,
                                  std::move(done), call_opts,
-                                 callback_threadpool_, max_retries);
+                                 callback_threadpool_);
   }
 
   // Helper function for initializing the RpcMethod objects below.
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.cc b/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.cc
deleted file mode 100644
index 9e8f7eafa5..0000000000
--- a/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.cc
+++ /dev/null
@@ -1,184 +0,0 @@
-/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h"
-
-#include "absl/time/clock.h"
-
-namespace tensorflow {
-
-struct WorkerCacheEntry {
-  enum class State {
-    PENDING = 0,
-    ACTIVE = 1,
-    FINISHED = 2,
-  };
-
-  State state = State::PENDING;
-  absl::Time expires;
-
-  ::grpc::ByteBuffer response_buf;
-  Status response_status;
-
-  // Additional retries may arrive while a request is still executing.  The
-  // callbacks for these calls are queued in `callbacks` and evaluated after
-  // the original request is completed.
-  std::vector<std::pair<RPCResponse, StatusCallback>> callbacks;
-};
-
-void RPCResponse::Encode(::grpc::ByteBuffer* tgt) const {
-  if (buf_ != nullptr) {
-    *tgt = *buf_;
-  } else {
-    CHECK(msg_ != nullptr);
-    ::grpc::Slice slice(msg_->ByteSizeLong());
-    msg_->SerializeWithCachedSizesToArray(
-        const_cast<uint8*>(reinterpret_cast<const uint8*>(slice.begin())));
-    ::grpc::ByteBuffer tmp(&slice, 1);
-    tgt->Swap(&tmp);
-  }
-}
-
-void RPCResponse::CopyFrom(const ::grpc::ByteBuffer& src) {
-  if (buf_ != nullptr) {
-    *buf_ = src;
-    return;
-  }
-
-  CHECK(msg_ != nullptr);
-  // We create a single slice when encoding protocol messages.
-  std::vector<::grpc::Slice> slices;
-  if (src.Dump(&slices).ok()) {
-    msg_->ParseFromArray(slices[0].begin(), slices[0].size());
-  } else {
-    LOG(ERROR) << "Failed to decode cached buffer.";
-  }
-}
-
-void GrpcResponseCache::LookupOrCompute(const string& key, RPCResponse response,
-                                        ComputeFunc compute_func,
-                                        StatusCallback done_cb) {
-  VLOG(1) << "Lookup " << key;
-  std::shared_ptr<WorkerCacheEntry> req;
-  MaybeCleanup();
-  {
-    mutex_lock m(mu_);
-
-    if (requests_.find(key) != requests_.end()) {
-      req = requests_[key];
-    } else {
-      req.reset(new WorkerCacheEntry);
-      requests_[key] = req;
-    }
-
-    if (req->state == WorkerCacheEntry::State::FINISHED) {
-      if (req->expires > absl::Now()) {
-        VLOG(1) << "Reuse cached response for " << key;
-        response.CopyFrom(req->response_buf);
-        done_cb(req->response_status);
-        return;
-      }
-      VLOG(1) << "Found expired cache entry for " << key;
-      req->state = WorkerCacheEntry::State::PENDING;
-      req->response_buf.Clear();
-    }
-
-    req->callbacks.push_back(std::make_pair(response, done_cb));
-
-    if (req->state == WorkerCacheEntry::State::ACTIVE) {
-      VLOG(1) << "Found active request for " << key
-              << ".  Adding entry to response queue.";
-      return;
-    }
-
-    VLOG(2) << "No cache entry for " << key << ", running user computation.";
-    req->state = WorkerCacheEntry::State::ACTIVE;
-    req->expires = absl::Now() + expire_duration_;
-  }
-
-  compute_func([this, key, req, response](Status status) {
-    mutex_lock m(mu_);
-    response.Encode(&req->response_buf);
-    current_bytes_ += req->response_buf.Length();
-
-    req->response_status = status;
-    req->state = WorkerCacheEntry::State::FINISHED;
-
-    VLOG(1) << "Operation for " << key << " finished. "
-            << "Status: " << status << ", " << req->response_buf.Length()
-            << " response bytes, " << req->callbacks.size()
-            << " pending callbacks.";
-    for (auto& cb : req->callbacks) {
-      cb.first.CopyFrom(req->response_buf);
-      cb.second(req->response_status);
-    }
-    req->callbacks.clear();
-  });
-}
-
-// Remove all stale or expired cache entries if the cache is full.
-void GrpcResponseCache::MaybeCleanup() {
-  mutex_lock m(mu_);
-  if (current_bytes_ < max_bytes_) {
-    return;
-  }
-
-  VLOG(1) << "Cleanup: " << current_bytes_ << " -> " << max_bytes_;
-  std::vector<std::pair<string, std::shared_ptr<WorkerCacheEntry>>>
-      ordered_entries;
-  ordered_entries.reserve(requests_.size());
-  for (const auto& p : requests_) {
-    ordered_entries.push_back(std::make_pair(p.first, p.second));
-  }
-
-  std::sort(ordered_entries.begin(), ordered_entries.end(),
-            [](const std::pair<string, std::shared_ptr<WorkerCacheEntry>>& a,
-               const std::pair<string, std::shared_ptr<WorkerCacheEntry>>& b) {
-              return a.second->expires > b.second->expires;
-            });
-
-  std::unordered_map<string, std::shared_ptr<WorkerCacheEntry>> kept;
-  absl::Time now = absl::Now();
-  int64 bytes_used = 0;
-
-  // Always keep active requests.
-  for (auto& pair : ordered_entries) {
-    if (pair.second->state != WorkerCacheEntry::State::FINISHED) {
-      kept.insert(pair);
-    }
-  }
-
-  // Keep unexpired, finished requests up to half of max_bytes_.  This reduces
-  // chances of overfilling the cache when active requests complete and
-  // amortizes cache cleanup cost.
-  for (auto& pair : ordered_entries) {
-    if (pair.second->expires < now || bytes_used >= max_bytes_ / 2) {
-      break;
-    }
-
-    if (pair.second->state == WorkerCacheEntry::State::FINISHED) {
-      kept.insert(pair);
-      bytes_used += pair.second->response_buf.Length();
-    }
-  }
-
-  VLOG(1) << "Cleaned cache.  Bytes used: " << current_bytes_ << " -> "
-          << bytes_used << ". Cache size: " << requests_.size() << " -> "
-          << kept.size();
-  current_bytes_ = bytes_used;
-  std::swap(requests_, kept);
-}
-
-}  // namespace tensorflow
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h b/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h
deleted file mode 100644
index a5ea79c319..0000000000
--- a/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h
+++ /dev/null
@@ -1,92 +0,0 @@
-/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#ifndef TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_RESPONSE_CACHE_H_
-#define TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_RESPONSE_CACHE_H_
-
-#include <memory>
-#include <unordered_map>
-#include <vector>
-
-#include "absl/time/time.h"
-#include "tensorflow/core/distributed_runtime/rpc/grpc_util.h"
-#include "tensorflow/core/lib/core/errors.h"
-#include "tensorflow/core/lib/core/status.h"
-#include "tensorflow/core/platform/protobuf.h"
-
-// gRPC response caching.  Most WorkerService methods cannot be retried directly
-// as they will fail or deadlock.  To enable retrying, we can instead cache
-// responses for a short period of time and reply to duplicate requests from the
-// cache.
-namespace tensorflow {
-
-// Union type to aid caching of either raw buffers (for RecvTensor RPCs) and
-// protocol buffer messages (for all other RPCs).
-class RPCResponse {
- public:
-  explicit RPCResponse() : buf_(nullptr), msg_(nullptr) {}
-  explicit RPCResponse(::grpc::ByteBuffer* b) : buf_(b), msg_(nullptr) {}
-  explicit RPCResponse(protobuf::Message* m) : buf_(nullptr), msg_(m) {}
-
-  // Encode this response into the target buffer.
-  void Encode(::grpc::ByteBuffer* tgt) const;
-
-  // Copy from `src`: if this is a buffer, make a shallow copy.
-  // For protocol messages, parse the response from `src`.
-  void CopyFrom(const ::grpc::ByteBuffer& src);
-
- private:
-  ::grpc::ByteBuffer* buf_;
-  protobuf::Message* msg_;
-};
-
-typedef std::function<void(StatusCallback)> ComputeFunc;
-struct WorkerCacheEntry;
-
-// Track and cache the state of worker service RPCs.  An RPC can be in 3 states:
-//
-// * PENDING: this is the first call of the RPC, and it will transition to
-// * ACTIVE: another thread is active processing this RPC
-// * FINISHED: the worker has finished processing the method
-//
-// The response from completed RPCs are LRU cached until either `max_bytes`
-// bytes are in use by the cache or they expire (according to `expire_time`).
-class GrpcResponseCache {
- public:
-  GrpcResponseCache(int64 max_bytes, absl::Duration expire_duration)
-      : max_bytes_(max_bytes), expire_duration_(expire_duration) {}
-
-  // Lookup the result for key.
-  // If it is finished, invoke `done_cb` immediately after filling `response`.
-  // If active, done_db will be invoked when the current call completes.
-  // Otherwise, invoke `compute_func` to fill the cache and invoke done_cb.
-  void LookupOrCompute(const string& key, RPCResponse response,
-                       ComputeFunc compute_func, StatusCallback done_cb);
-
-  // Remove all stale or expired cache entries if the cache is full.
-  void MaybeCleanup();
-
- private:
-  int64 current_bytes_ GUARDED_BY(mu_) = 0;
-  const int64 max_bytes_;
-  const absl::Duration expire_duration_;
-
-  std::unordered_map<string, std::shared_ptr<WorkerCacheEntry>> requests_
-      GUARDED_BY(mu_);
-  mutex mu_;
-};
-
-}  // namespace tensorflow
-
-#endif  // TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_RESPONSE_CACHE_H_
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_rpc_factory.cc b/tensorflow/core/distributed_runtime/rpc/grpc_rpc_factory.cc
index 3635caf3d1..7f63cc9344 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_rpc_factory.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_rpc_factory.cc
@@ -210,7 +210,7 @@ void GrpcRPCFactory::StartCall(const Tensor& address_t, const Tensor& method_t,
       get_stub(index), &completion_queue_, *get_method_ptr(index),
       call->request(), call->response(),
       /*done=*/[call](const Status& s) { call->Done(s); }, call->call_opts(),
-      nullptr /*threadpool*/, fail_fast_, timeout_in_ms_, 0 /* max_retries */);
+      nullptr /*threadpool*/, fail_fast_, timeout_in_ms_);
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_state.h b/tensorflow/core/distributed_runtime/rpc/grpc_state.h
index 0ca64dc159..d73638651f 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_state.h
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_state.h
@@ -32,9 +32,6 @@ limitations under the License.
 namespace tensorflow {
 
 // Object allocated per active RPC.
-// Manage the state of a single asynchronous RPC request.  If `max_retries`
-// is greater than 0, the request will be retried for any transient failures
-// as long as the overall deadline has not elapsed.
 template <class Response>
 class RPCState : public GrpcClientCQTag {
  public:
@@ -42,55 +39,34 @@ class RPCState : public GrpcClientCQTag {
   RPCState(::grpc::GenericStub* stub, ::grpc::CompletionQueue* cq,
            const ::grpc::string& method, const protobuf::Message& request,
            Response* response, StatusCallback done, CallOptions* call_opts,
-           thread::ThreadPool* threadpool, int32 max_retries = 0)
+           thread::ThreadPool* threadpool)
       : RPCState(stub, cq, method, request, response, std::move(done),
                  call_opts, threadpool, /*fail_fast=*/false,
-                 /*timeout_in_ms=*/0, max_retries) {}
+                 /*timeout_in_ms=*/0) {}
 
   template <typename Request>
   RPCState(::grpc::GenericStub* stub, ::grpc::CompletionQueue* cq,
            const ::grpc::string& method, const Request& request,
            Response* response, StatusCallback done, CallOptions* call_opts,
-           thread::ThreadPool* threadpool, bool fail_fast, int64 timeout_in_ms,
-           int32 max_retries)
-      : call_opts_(call_opts),
-        threadpool_(threadpool),
-        done_(std::move(done)),
-        cq_(cq),
-        stub_(stub),
-        method_(method),
-        max_retries_(max_retries),
-        timeout_in_ms_(timeout_in_ms),
-        fail_fast_(fail_fast) {
+           thread::ThreadPool* threadpool, bool fail_fast, int64 timeout_in_ms)
+      : call_opts_(call_opts), threadpool_(threadpool), done_(std::move(done)) {
+    context_.set_fail_fast(fail_fast);
+    if (timeout_in_ms > 0) {
+      context_.set_deadline(gpr_time_from_millis(timeout_in_ms, GPR_TIMESPAN));
+    }
+
+    if (call_opts) {
+      call_opts->SetCancelCallback([this]() { context_.TryCancel(); });
+    }
+
     response_ = response;
     ::grpc::Status s = GrpcMaybeUnparseProto(request, &request_buf_);
     if (!s.ok()) {
       LOG(ERROR) << "GrpcMaybeUnparseProto returned with non-ok status: "
                  << s.error_message();
-      // Skip retry logic if we fail to parse our request.
-      done_(FromGrpcStatus(s));
-      delete this;
-      return;
-    }
-    StartCall();
-  }
-
-  void StartCall() {
-    context_.reset(new ::grpc::ClientContext());
-    context_->set_fail_fast(fail_fast_);
-
-    if (timeout_in_ms_ > 0) {
-      context_->set_deadline(
-          gpr_time_from_millis(timeout_in_ms_, GPR_TIMESPAN));
     }
-    if (call_opts_) {
-      call_opts_->SetCancelCallback([this]() { context_->TryCancel(); });
-    }
-
-    VLOG(2) << "Starting call: " << method_;
-
-    call_ = std::move(
-        stub_->PrepareUnaryCall(context_.get(), method_, request_buf_, cq_));
+    call_ =
+        std::move(stub->PrepareUnaryCall(&context_, method, request_buf_, cq));
     call_->StartCall();
     call_->Finish(&response_buf_, &status_, this);
   }
@@ -113,26 +89,16 @@ class RPCState : public GrpcClientCQTag {
         threadpool_->Schedule([this]() { ParseAndCallDone(); });
       } else {
         ParseAndCallDone();
+        return;
       }
-      return;
-    }
-
-    VLOG(1) << method_ << " returned with non-ok status: " << s
-            << " Retries: " << num_retries_ << " Max: " << max_retries_ << "\n"
-            << context_->debug_error_string();
-    // Retry if we have any attempts left
-    if (++num_retries_ <= max_retries_ &&
-        (errors::IsUnavailable(s) || errors::IsUnknown(s))) {
-      response_buf_.Clear();
-      VLOG(1) << "Retrying call for " << method_ << "Retry: " << num_retries_
-              << " of " << max_retries_;
-      StartCall();
     } else {
-      // Attach additional GRPC error information if any to the final status
+      VLOG(2) << "Call returned with non-ok status: " << s;
+
+      // Attach additional GRPC error information if any
       s = Status(s.code(),
                  strings::StrCat(s.error_message(),
                                  "\nAdditional GRPC error information:\n",
-                                 context_->debug_error_string()));
+                                 context_.debug_error_string()));
       done_(s);
       delete this;
     }
@@ -149,7 +115,7 @@ class RPCState : public GrpcClientCQTag {
 
  private:
   CallOptions* call_opts_;
-  std::unique_ptr<::grpc::ClientContext> context_;
+  ::grpc::ClientContext context_;
   thread::ThreadPool* threadpool_;
   std::unique_ptr<::grpc::GenericClientAsyncResponseReader> call_;
   Response* response_;
@@ -157,15 +123,6 @@ class RPCState : public GrpcClientCQTag {
   ::grpc::ByteBuffer response_buf_;
   ::grpc::Status status_;
   StatusCallback done_;
-  int64 timeout_in_ms_;
-
-  size_t num_retries_ = 0;
-  size_t max_retries_;
-
-  ::grpc::CompletionQueue* cq_;
-  ::grpc::GenericStub* stub_;
-  ::grpc::string method_;
-  bool fail_fast_;
 };
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc
index fdd36ccecd..34bf629c8a 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc
@@ -16,14 +16,11 @@ limitations under the License.
 #include "tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h"
 
 #include <deque>
-#include <memory>
 #include <unordered_map>
-#include <vector>
 
 #include "grpcpp/alarm.h"
 #include "grpcpp/server_builder.h"
 
-#include "absl/container/flat_hash_map.h"
 #include "tensorflow/core/common_runtime/buf_rendezvous.h"
 #include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/common_runtime/device_mgr.h"
@@ -35,7 +32,6 @@ limitations under the License.
 #include "tensorflow/core/distributed_runtime/rendezvous_mgr_interface.h"
 #include "tensorflow/core/distributed_runtime/rpc/async_service_interface.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_call.h"
-#include "tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_tensor_coding.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_util.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h"
@@ -46,12 +42,8 @@ limitations under the License.
 #include "tensorflow/core/framework/collective.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/lib/core/errors.h"
-#include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/gtl/map_util.h"
-#include "tensorflow/core/lib/strings/strcat.h"
-#include "tensorflow/core/lib/strings/stringprintf.h"
 #include "tensorflow/core/platform/logging.h"
-#include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/platform/tracing.h"
 #include "tensorflow/core/protobuf/transport_options.pb.h"
 #include "tensorflow/core/protobuf/worker.pb.h"
@@ -99,11 +91,10 @@ class GrpcWorkerServiceThread {
  public:
   explicit GrpcWorkerServiceThread(
       GrpcWorker* worker, ::grpc::ServerBuilder* builder,
-      std::unordered_map<int, int> queue_depth, GrpcResponseCache* cache,
+      std::unordered_map<int, int> queue_depth,
       grpc::WorkerService::AsyncService* worker_service)
       : worker_(worker),
         queue_depth_(queue_depth),
-        cache_(cache),
         worker_service_(worker_service),
         is_shutdown_(false) {
     cq_ = builder->AddCompletionQueue();
@@ -229,32 +220,18 @@ class GrpcWorkerServiceThread {
       NonOwnedProtoRunGraphResponse* wrapped_response =
           new NonOwnedProtoRunGraphResponse(&call->response);
       call->SetCancelCallback([call_opts]() { call_opts->StartCancel(); });
-      auto done_cb = [call, call_opts, wrapped_request,
-                      wrapped_response](const Status& s) {
-        VLOG(1) << "RunGraph::Done";
-        if (!s.ok()) {
-          VLOG(1) << "Bad response from RunGraph:" << s;
-        }
-        call->ClearCancelCallback();
-        delete call_opts;
-        delete wrapped_request;
-        delete wrapped_response;
-        call->SendResponse(ToGrpcStatus(s));
-      };
-
-      auto compute_fn = [this, call_opts, wrapped_request,
-                         wrapped_response](StatusCallback done) {
-        worker_->RunGraphAsync(call_opts, wrapped_request, wrapped_response,
-                               done);
-      };
-
-      if (cache_) {
-        string request_key = call->request.ShortDebugString();
-        cache_->LookupOrCompute(request_key, RPCResponse(&call->response),
-                                compute_fn, done_cb);
-      } else {
-        compute_fn(done_cb);
-      }
+      worker_->RunGraphAsync(call_opts, wrapped_request, wrapped_response,
+                             [call, call_opts, wrapped_request,
+                              wrapped_response](const Status& s) {
+                               if (!s.ok()) {
+                                 VLOG(1) << "Bad response from RunGraph:" << s;
+                               }
+                               call->ClearCancelCallback();
+                               delete call_opts;
+                               delete wrapped_request;
+                               delete wrapped_response;
+                               call->SendResponse(ToGrpcStatus(s));
+                             });
     });
     ENQUEUE_REQUEST(RunGraph, true);
   }
@@ -264,28 +241,16 @@ class GrpcWorkerServiceThread {
     Schedule([this, call]() {
       CallOptions* call_opts = new CallOptions;
       call->SetCancelCallback([call_opts]() { call_opts->StartCancel(); });
-
-      auto done_cb = [call, call_opts](const Status& s) {
-        call->ClearCancelCallback();
-        delete call_opts;
-        if (!s.ok()) {
-          VLOG(1) << "Bad response from RecvTensor:" << s;
-        }
-        call->SendResponse(ToGrpcStatus(s));
-      };
-
-      auto compute_fn = [this, &call_opts, &call](StatusCallback done) {
-        worker_->GrpcRecvTensorAsync(call_opts, &call->request, &call->response,
-                                     done);
-      };
-
-      if (cache_) {
-        string request_key = call->request.ShortDebugString();
-        cache_->LookupOrCompute(request_key, RPCResponse(&call->response),
-                                compute_fn, done_cb);
-      } else {
-        compute_fn(done_cb);
-      }
+      worker_->GrpcRecvTensorAsync(
+          call_opts, &call->request, &call->response,
+          [call, call_opts](const Status& s) {
+            call->ClearCancelCallback();
+            delete call_opts;
+            if (!s.ok()) {
+              VLOG(1) << "Bad response from RecvTensor:" << s;
+            }
+            call->SendResponse(ToGrpcStatus(s));
+          });
     });
     EnqueueRecvTensorRequestRaw();
   }
@@ -363,7 +328,6 @@ class GrpcWorkerServiceThread {
   std::unique_ptr<::grpc::ServerCompletionQueue> cq_;
   std::unique_ptr<Thread> thread_;
   std::unordered_map<int, int> queue_depth_;
-  GrpcResponseCache* cache_;
   grpc::WorkerService::AsyncService* const worker_service_;
 
   mutex shutdown_mu_;
@@ -377,15 +341,9 @@ class GrpcWorkerService : public AsyncServiceInterface {
                     GrpcWorkerServiceOptions options)
       : is_shutdown_(false) {
     builder->RegisterService(&worker_service_);
-    if (options.response_cache_bytes > 0) {
-      cache_.reset(new GrpcResponseCache(options.response_cache_bytes,
-                                         options.response_cache_duration));
-    }
-
-    for (int i = 0; i < options.num_serving_threads; i++) {
-      threads_.emplace_back(
-          new GrpcWorkerServiceThread(worker, builder, options.queue_depth,
-                                      cache_.get(), &worker_service_));
+    for (int i = 0; i < options.num_worker_threads; i++) {
+      threads_.emplace_back(new GrpcWorkerServiceThread(
+          worker, builder, options.queue_depth, &worker_service_));
     }
   }
 
@@ -420,7 +378,6 @@ class GrpcWorkerService : public AsyncServiceInterface {
   grpc::WorkerService::AsyncService worker_service_;
   std::vector<std::unique_ptr<GrpcWorkerServiceThread>> threads_;
 
-  std::unique_ptr<GrpcResponseCache> cache_;
   mutex service_shutdown_mu_;
   bool is_shutdown_ GUARDED_BY(service_shutdown_mu_);
 
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h
index 467927a8e8..88beb6c216 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h
@@ -16,11 +16,8 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_WORKER_SERVICE_H_
 #define TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_WORKER_SERVICE_H_
 
-#include <memory>
 #include <unordered_map>
-#include "absl/time/time.h"
 #include "tensorflow/core/distributed_runtime/recent_request_ids.h"
-#include "tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h"
 #include "tensorflow/core/distributed_runtime/worker.h"
 
@@ -66,9 +63,7 @@ struct GrpcWorkerServiceOptions {
   // Map from GrpcWorkerMethod id to queue depth.  If set this overrides the
   // default queue depth for a method.
   std::unordered_map<int, int> queue_depth;
-  int num_serving_threads = 8;
-  int64 response_cache_bytes = 0;
-  absl::Duration response_cache_duration = absl::ZeroDuration();
+  int num_worker_threads = 8;
 };
 
 // Returns an implementation of WorkerService rpc service.
-- 
GitLab


From 2e93ffc56a9ff655fc83906133c3da1d6a08a213 Mon Sep 17 00:00:00 2001
From: Alan Chiao <alanchiao@google.com>
Date: Fri, 1 Feb 2019 14:40:08 -0800
Subject: [PATCH 1785/2345] Internalize custom op.

PiperOrigin-RevId: 232040167
---
 tensorflow/lite/kernels/BUILD                 |   14 -
 tensorflow/lite/kernels/layer_norm_lstm.cc    | 1324 -----------------
 .../lite/kernels/layer_norm_lstm_test.cc      |  885 -----------
 tensorflow/lite/kernels/register.cc           |    2 -
 tensorflow/lite/kernels/register_ref.cc       |    2 -
 5 files changed, 2227 deletions(-)
 delete mode 100644 tensorflow/lite/kernels/layer_norm_lstm.cc
 delete mode 100644 tensorflow/lite/kernels/layer_norm_lstm_test.cc

diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index 92f51350c2..5029f10cd6 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -182,7 +182,6 @@ cc_library(
         "hashtable_lookup.cc",
         "if.cc",
         "l2norm.cc",
-        "layer_norm_lstm.cc",
         "local_response_norm.cc",
         "logical.cc",
         "lsh_projection.cc",
@@ -879,19 +878,6 @@ tf_cc_test(
     ],
 )
 
-tf_cc_test(
-    name = "layer_norm_lstm_test",
-    size = "small",
-    srcs = ["layer_norm_lstm_test.cc"],
-    deps = [
-        ":builtin_ops",
-        "//tensorflow/lite:framework",
-        "//tensorflow/lite/kernels:test_util",
-        "@com_google_googletest//:gtest",
-        "@flatbuffers",
-    ],
-)
-
 tf_cc_test(
     name = "lstm_test",
     size = "small",
diff --git a/tensorflow/lite/kernels/layer_norm_lstm.cc b/tensorflow/lite/kernels/layer_norm_lstm.cc
deleted file mode 100644
index ce0c21dfcb..0000000000
--- a/tensorflow/lite/kernels/layer_norm_lstm.cc
+++ /dev/null
@@ -1,1324 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-// DEPRECATED: Tensorflow Lite has implemented layer norm lstm as builtin Op and
-// the implementation of layer norm lstm as custom Op in this file is
-// deprecated. It is only kept for backward compatibility.
-//
-// Layer Normalization LSTM op that applies normalization by mean and standard
-// deviation to the activation of the LSTM layers. Please see
-// https://arxiv.org/abs/1607.06450 for details.
-#include "flatbuffers/flexbuffers.h"  // TF:flatbuffers
-#include "tensorflow/lite/context.h"
-#include "tensorflow/lite/kernels/internal/tensor_utils.h"
-#include "tensorflow/lite/kernels/kernel_util.h"
-
-namespace tflite {
-namespace ops {
-namespace custom {
-namespace layer_norm_lstm {
-
-// Struct to hold Layer Norm LSTM option data.
-struct OpData {
-  TfLiteFusedActivation activation;
-  float cell_clip;
-  float proj_clip;
-  int scratch_tensor_index;
-};
-
-// Input Tensors of size {n_batch, n_input}
-constexpr int kInputTensor = 0;
-
-// Input weight tensors of size: {n_cell, n_input}
-constexpr int kInputToInputWeightsTensor = 1;  // Optional
-constexpr int kInputToForgetWeightsTensor = 2;
-constexpr int kInputToCellWeightsTensor = 3;
-constexpr int kInputToOutputWeightsTensor = 4;
-
-// Recurrent weight tensors of size {n_cell, n_output}
-constexpr int kRecurrentToInputWeightsTensor = 5;  // Optional
-constexpr int kRecurrentToForgetWeightsTensor = 6;
-constexpr int kRecurrentToCellWeightsTensor = 7;
-constexpr int kRecurrentToOutputWeightsTensor = 8;
-
-// Peephole weights tensors of size {n_cell}, representing a diagonal matrix.
-constexpr int kCellToInputWeightsTensor = 9;    // Optional
-constexpr int kCellToForgetWeightsTensor = 10;  // Optional
-constexpr int kCellToOutputWeightsTensor = 11;  // Optional
-
-// Layer norm weights tensors of size {n_cell}, representing a diagonal matrix.
-constexpr int kInputLayerNormWeightsTensor = 12;  // Optional
-constexpr int kForgetLayerNormWeightsTensor = 13;
-constexpr int kCellLayerNormWeightsTensor = 14;
-constexpr int kOutputLayerNormWeightsTensor = 15;
-
-// Gates bias tensors of size {n_cell}
-constexpr int kInputGateBiasTensor = 16;  // Optional
-constexpr int kForgetGateBiasTensor = 17;
-constexpr int kCellGateBiasTensor = 18;
-constexpr int kOutputGateBiasTensor = 19;
-
-// Projection weight tensor of size {n_output, n_cell}
-constexpr int kProjectionWeightsTensor = 20;  // Optional
-// Projection bias tensor of size {n_output}
-constexpr int kProjectionBiasTensor = 21;  // Optional
-
-// State tensors.
-constexpr int kInputActivationStateTensor = 22;
-constexpr int kInputCellStateTensor = 23;
-
-// Output tensor.
-constexpr int kOutputTensor = 0;
-
-// Total number of scratch tensors for hybrid Op.
-constexpr int kTensorsToAdd = 7;
-
-// Small float to avoid divergence during calculation of deviation.
-const float kLayerNormEpsilon = 1e-8;
-
-void* Init(TfLiteContext* context, const char* buffer, size_t length) {
-  auto* data = new OpData;
-
-  // Turn custom option data into flexbuffer map format.
-  const uint8_t* buffer_t = reinterpret_cast<const uint8_t*>(buffer);
-  const flexbuffers::Map& m = flexbuffers::GetRoot(buffer_t, length).AsMap();
-
-  // Get activation function, cell_clip and proj_clip from the flexbuffer.
-  // TODO(b/113824099): make activation more generic.
-  assert(m["fused_activation_function"].ToString() == "TANH");
-  data->activation = kTfLiteActTanh;
-  data->cell_clip = m["cell_clip"].AsFloat();
-  data->proj_clip = m["proj_clip"].AsFloat();
-
-  // Populate scratch_tensor_index.
-  context->AddTensors(context, /*tensors_to_add=*/kTensorsToAdd,
-                      &data->scratch_tensor_index);
-  return data;
-}
-
-// Check that input tensor dimensions matches with each other.
-TfLiteStatus CheckInputTensorDimensions(TfLiteContext* context,
-                                        TfLiteNode* node, int n_input,
-                                        int n_output, int n_cell) {
-  const OpData* op_data = reinterpret_cast<OpData*>(node->user_data);
-
-  // Making sure clipping parameters have valid values.
-  // == 0 means no clipping
-  //  > 0 means clipping
-  TF_LITE_ENSURE(context, op_data->cell_clip >= 0);
-  TF_LITE_ENSURE(context, op_data->proj_clip >= 0);
-
-  const TfLiteTensor* input_to_input_weights =
-      GetOptionalInputTensor(context, node, kInputToInputWeightsTensor);
-  const bool use_cifg = (input_to_input_weights == nullptr);
-  if (!use_cifg) {
-    TF_LITE_ENSURE_EQ(context, input_to_input_weights->dims->size, 2);
-    TF_LITE_ENSURE_EQ(context, input_to_input_weights->dims->data[0], n_cell);
-    TF_LITE_ENSURE_EQ(context, input_to_input_weights->dims->data[1], n_input);
-  }
-
-  const TfLiteTensor* input_to_forget_weights =
-      GetInput(context, node, kInputToForgetWeightsTensor);
-  TF_LITE_ENSURE_EQ(context, input_to_forget_weights->dims->size, 2);
-  TF_LITE_ENSURE_EQ(context, input_to_forget_weights->dims->data[0], n_cell);
-  TF_LITE_ENSURE_EQ(context, input_to_forget_weights->dims->data[1], n_input);
-
-  const TfLiteTensor* input_to_cell_weights =
-      GetInput(context, node, kInputToCellWeightsTensor);
-  TF_LITE_ENSURE_EQ(context, input_to_cell_weights->dims->size, 2);
-  TF_LITE_ENSURE_EQ(context, input_to_cell_weights->dims->data[0], n_cell);
-  TF_LITE_ENSURE_EQ(context, input_to_cell_weights->dims->data[1], n_input);
-
-  const TfLiteTensor* recurrent_to_input_weights =
-      GetOptionalInputTensor(context, node, kRecurrentToInputWeightsTensor);
-  if (use_cifg) {
-    TF_LITE_ENSURE_EQ(context, recurrent_to_input_weights, nullptr);
-  } else {
-    TF_LITE_ENSURE_EQ(context, recurrent_to_input_weights->dims->size, 2);
-    TF_LITE_ENSURE_EQ(context, recurrent_to_input_weights->dims->data[0],
-                      n_cell);
-    TF_LITE_ENSURE_EQ(context, recurrent_to_input_weights->dims->data[1],
-                      n_output);
-  }
-
-  const TfLiteTensor* recurrent_to_forget_weights =
-      GetInput(context, node, kRecurrentToForgetWeightsTensor);
-  TF_LITE_ENSURE_EQ(context, recurrent_to_forget_weights->dims->size, 2);
-  TF_LITE_ENSURE_EQ(context, recurrent_to_forget_weights->dims->data[0],
-                    n_cell);
-  TF_LITE_ENSURE_EQ(context, recurrent_to_forget_weights->dims->data[1],
-                    n_output);
-
-  const TfLiteTensor* recurrent_to_cell_weights =
-      GetInput(context, node, kRecurrentToCellWeightsTensor);
-  TF_LITE_ENSURE_EQ(context, recurrent_to_cell_weights->dims->size, 2);
-  TF_LITE_ENSURE_EQ(context, recurrent_to_cell_weights->dims->data[0], n_cell);
-  TF_LITE_ENSURE_EQ(context, recurrent_to_cell_weights->dims->data[1],
-                    n_output);
-
-  const TfLiteTensor* cell_to_input_weights =
-      GetOptionalInputTensor(context, node, kCellToInputWeightsTensor);
-  if (cell_to_input_weights) {
-    TF_LITE_ENSURE_EQ(context, cell_to_input_weights->dims->size, 1);
-    TF_LITE_ENSURE_EQ(context, cell_to_input_weights->dims->data[0], n_cell);
-  }
-
-  const TfLiteTensor* cell_to_forget_weights =
-      GetOptionalInputTensor(context, node, kCellToForgetWeightsTensor);
-  if (cell_to_forget_weights) {
-    TF_LITE_ENSURE_EQ(context, cell_to_forget_weights->dims->size, 1);
-    TF_LITE_ENSURE_EQ(context, cell_to_forget_weights->dims->data[0], n_cell);
-  }
-
-  const TfLiteTensor* cell_to_output_weights =
-      GetOptionalInputTensor(context, node, kCellToOutputWeightsTensor);
-  if (cell_to_output_weights) {
-    TF_LITE_ENSURE_EQ(context, cell_to_output_weights->dims->size, 1);
-    TF_LITE_ENSURE_EQ(context, cell_to_output_weights->dims->data[0], n_cell);
-  }
-
-  // Making sure the peephole weights are there all or none.
-  const bool peephole_weights_all_or_none =
-      ((cell_to_input_weights != nullptr || use_cifg) &&
-       (cell_to_forget_weights != nullptr) &&
-       (cell_to_output_weights != nullptr)) ||
-      ((cell_to_input_weights == nullptr) &&
-       (cell_to_forget_weights == nullptr) &&
-       (cell_to_output_weights == nullptr));
-  TF_LITE_ENSURE(context, peephole_weights_all_or_none == true);
-
-  // Making sure layer norm weights are not null and have the right dimension.
-  const TfLiteTensor* input_layer_norm_weights =
-      GetOptionalInputTensor(context, node, kInputLayerNormWeightsTensor);
-  if (use_cifg) {
-    TF_LITE_ENSURE_EQ(context, input_layer_norm_weights, nullptr);
-  } else {
-    TF_LITE_ENSURE(context, input_layer_norm_weights != nullptr);
-    TF_LITE_ENSURE_EQ(context, input_layer_norm_weights->dims->size, 1);
-    TF_LITE_ENSURE_EQ(context, input_layer_norm_weights->dims->data[0], n_cell);
-  }
-
-  const TfLiteTensor* forget_layer_norm_weights =
-      GetInput(context, node, kForgetLayerNormWeightsTensor);
-  TF_LITE_ENSURE(context, forget_layer_norm_weights != nullptr);
-  TF_LITE_ENSURE_EQ(context, forget_layer_norm_weights->dims->size, 1);
-  TF_LITE_ENSURE_EQ(context, forget_layer_norm_weights->dims->data[0], n_cell);
-
-  const TfLiteTensor* cell_layer_norm_weights =
-      GetInput(context, node, kCellLayerNormWeightsTensor);
-  TF_LITE_ENSURE(context, cell_layer_norm_weights != nullptr);
-  TF_LITE_ENSURE_EQ(context, cell_layer_norm_weights->dims->size, 1);
-  TF_LITE_ENSURE_EQ(context, cell_layer_norm_weights->dims->data[0], n_cell);
-
-  const TfLiteTensor* output_layer_norm_weights =
-      GetInput(context, node, kOutputLayerNormWeightsTensor);
-  TF_LITE_ENSURE(context, output_layer_norm_weights != nullptr);
-  TF_LITE_ENSURE_EQ(context, output_layer_norm_weights->dims->size, 1);
-  TF_LITE_ENSURE_EQ(context, output_layer_norm_weights->dims->data[0], n_cell);
-
-  // Make sure the input gate bias is present only when not a CIFG-LSTM.
-  const TfLiteTensor* input_gate_bias =
-      GetOptionalInputTensor(context, node, kInputGateBiasTensor);
-  if (use_cifg) {
-    TF_LITE_ENSURE_EQ(context, input_gate_bias, nullptr);
-  } else {
-    TF_LITE_ENSURE_EQ(context, input_gate_bias->dims->size, 1);
-    TF_LITE_ENSURE_EQ(context, input_gate_bias->dims->data[0], n_cell);
-  }
-
-  const TfLiteTensor* forget_gate_bias =
-      GetInput(context, node, kForgetGateBiasTensor);
-  TF_LITE_ENSURE_EQ(context, forget_gate_bias->dims->size, 1);
-  TF_LITE_ENSURE_EQ(context, forget_gate_bias->dims->data[0], n_cell);
-
-  const TfLiteTensor* cell_bias = GetInput(context, node, kCellGateBiasTensor);
-  TF_LITE_ENSURE_EQ(context, cell_bias->dims->size, 1);
-  TF_LITE_ENSURE_EQ(context, cell_bias->dims->data[0], n_cell);
-
-  const TfLiteTensor* output_gate_bias =
-      GetInput(context, node, kOutputGateBiasTensor);
-  TF_LITE_ENSURE_EQ(context, output_gate_bias->dims->size, 1);
-  TF_LITE_ENSURE_EQ(context, output_gate_bias->dims->data[0], n_cell);
-
-  const TfLiteTensor* projection_weights =
-      GetOptionalInputTensor(context, node, kProjectionWeightsTensor);
-  if (projection_weights != nullptr) {
-    TF_LITE_ENSURE_EQ(context, projection_weights->dims->size, 2);
-    TF_LITE_ENSURE_EQ(context, projection_weights->dims->data[0], n_output);
-    TF_LITE_ENSURE_EQ(context, projection_weights->dims->data[1], n_cell);
-  }
-
-  const TfLiteTensor* projection_bias =
-      GetOptionalInputTensor(context, node, kProjectionBiasTensor);
-  if (projection_bias != nullptr) {
-    TF_LITE_ENSURE_EQ(context, projection_bias->dims->size, 1);
-    TF_LITE_ENSURE_EQ(context, projection_bias->dims->data[0], n_output);
-  }
-
-  // Making sure the projection tensors are consistent:
-  // 1) If projection weight is not present, then projection bias should not be
-  // present.
-  // 2) If projection weight is present, then projection bias is optional.
-  const bool projection_tensors_consistent =
-      ((projection_weights != nullptr) || (projection_bias == nullptr));
-  TF_LITE_ENSURE(context, projection_tensors_consistent == true);
-
-  return kTfLiteOk;
-}
-
-// Resize the output, state tensors based on the sizes of the input tensors.
-// Allocate a temporary scratch tensor. Also check that the sizes of the input
-// tensors match each other.
-TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
-  OpData* op_data = reinterpret_cast<OpData*>(node->user_data);
-  TF_LITE_ENSURE_EQ(context, node->inputs->size, 24);
-  TF_LITE_ENSURE_EQ(context, node->outputs->size, 1);
-
-  // Inferring batch size, number of outputs and number of cells from the
-  // input tensors.
-  const TfLiteTensor* input = GetInput(context, node, kInputTensor);
-  TF_LITE_ENSURE_EQ(context, input->type, kTfLiteFloat32);
-  TF_LITE_ENSURE(context, input->dims->size > 1);
-  const int n_batch = input->dims->data[0];
-  const int n_input = input->dims->data[1];
-
-  const TfLiteTensor* input_to_output_weights =
-      GetInput(context, node, kInputToOutputWeightsTensor);
-  const int n_cell = input_to_output_weights->dims->data[0];
-  TF_LITE_ENSURE_EQ(context, input_to_output_weights->dims->size, 2);
-  TF_LITE_ENSURE_EQ(context, input_to_output_weights->dims->data[1], n_input);
-
-  const TfLiteTensor* recurrent_to_output_weights =
-      GetInput(context, node, kRecurrentToOutputWeightsTensor);
-  TF_LITE_ENSURE_EQ(context, recurrent_to_output_weights->dims->size, 2);
-  TF_LITE_ENSURE_EQ(context, recurrent_to_output_weights->dims->data[0],
-                    n_cell);
-  const int n_output = recurrent_to_output_weights->dims->data[1];
-
-  // Check that input tensor dimensions matches with each other.
-  TF_LITE_ENSURE_OK(context, CheckInputTensorDimensions(context, node, n_input,
-                                                        n_output, n_cell));
-
-  // Get the pointer to output, activation_state and cell_state tensors.
-  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
-
-  const TfLiteTensor* activation_state =
-      GetInput(context, node, kInputActivationStateTensor);
-  const TfLiteTensor* cell_state =
-      GetInput(context, node, kInputCellStateTensor);
-
-  // Check the shape of input state tensors.
-  // These tensor may be 1D or 2D. It's fine as long as the total size is
-  // correct.
-  TF_LITE_ENSURE_EQ(context, NumElements(activation_state), n_batch * n_output);
-  TF_LITE_ENSURE_EQ(context, NumElements(cell_state), n_batch * n_cell);
-  // Resize the output tensors.
-  TfLiteIntArray* output_size = TfLiteIntArrayCreate(2);
-  output_size->data[0] = n_batch;
-  output_size->data[1] = n_output;
-  TF_LITE_ENSURE_OK(context,
-                    context->ResizeTensor(context, output, output_size));
-
-  // The weights are of consistent type, so it suffices to check one.
-  const bool is_hybrid_op = (input_to_output_weights->type == kTfLiteUInt8 &&
-                             input->type == kTfLiteFloat32);
-
-  TfLiteIntArrayFree(node->temporaries);
-  if (is_hybrid_op) {
-    node->temporaries = TfLiteIntArrayCreate(7);
-  } else {
-    node->temporaries = TfLiteIntArrayCreate(1);
-  }
-  node->temporaries->data[0] = op_data->scratch_tensor_index;
-
-  // Create a scratch buffer tensor.
-  TfLiteTensor* scratch_buffer = GetTemporary(context, node, /*index=*/0);
-  scratch_buffer->type = input->type;
-  scratch_buffer->allocation_type = kTfLiteArenaRw;
-
-  const TfLiteTensor* input_to_input_weights =
-      GetOptionalInputTensor(context, node, kInputToInputWeightsTensor);
-  const bool use_cifg = (input_to_input_weights == nullptr);
-  TfLiteIntArray* scratch_buffer_size = TfLiteIntArrayCreate(2);
-  scratch_buffer_size->data[0] = n_batch;
-  if (use_cifg) {
-    // Reserving space for Cell, Forget, Output gates
-    scratch_buffer_size->data[1] = n_cell * 3;
-  } else {
-    // Reserving space for Input, Cell, Forget, Output gates
-    scratch_buffer_size->data[1] = n_cell * 4;
-  }
-  TF_LITE_ENSURE_OK(context, context->ResizeTensor(context, scratch_buffer,
-                                                   scratch_buffer_size));
-
-  if (is_hybrid_op) {
-    // Allocate temporary tensors to store quantized values of input,
-    // activation_state and cell_state tensors.
-    node->temporaries->data[1] = op_data->scratch_tensor_index + 1;
-    TfLiteTensor* input_quantized = GetTemporary(context, node, /*index=*/1);
-    input_quantized->type = kTfLiteUInt8;
-    input_quantized->allocation_type = kTfLiteArenaRw;
-    if (!TfLiteIntArrayEqual(input_quantized->dims, input->dims)) {
-      TfLiteIntArray* input_quantized_size = TfLiteIntArrayCopy(input->dims);
-      TF_LITE_ENSURE_OK(context, context->ResizeTensor(context, input_quantized,
-                                                       input_quantized_size));
-    }
-    node->temporaries->data[2] = op_data->scratch_tensor_index + 2;
-    TfLiteTensor* activation_state_quantized =
-        GetTemporary(context, node, /*index=*/2);
-    activation_state_quantized->type = kTfLiteUInt8;
-    activation_state_quantized->allocation_type = kTfLiteArenaRw;
-    if (!TfLiteIntArrayEqual(activation_state_quantized->dims,
-                             activation_state->dims)) {
-      TfLiteIntArray* activation_state_quantized_size =
-          TfLiteIntArrayCopy(activation_state->dims);
-      TF_LITE_ENSURE_OK(
-          context, context->ResizeTensor(context, activation_state_quantized,
-                                         activation_state_quantized_size));
-    }
-    node->temporaries->data[3] = op_data->scratch_tensor_index + 3;
-    TfLiteTensor* cell_state_quantized =
-        GetTemporary(context, node, /*index=*/3);
-    cell_state_quantized->type = kTfLiteUInt8;
-    cell_state_quantized->allocation_type = kTfLiteArenaRw;
-    if (!TfLiteIntArrayEqual(cell_state_quantized->dims, cell_state->dims)) {
-      TfLiteIntArray* cell_state_quantized_size =
-          TfLiteIntArrayCopy(cell_state->dims);
-      TF_LITE_ENSURE_OK(context,
-                        context->ResizeTensor(context, cell_state_quantized,
-                                              cell_state_quantized_size));
-    }
-
-    // Allocate temporary tensors to store scaling factors and product scaling
-    // factors. The latter is a convenience storage which allows to quantize
-    // a vector once (which produces the scaling factors) and multiply it with
-    // different matrices (which requires multiplying the scaling factors with
-    // the scaling factor of the matrix).
-    node->temporaries->data[4] = op_data->scratch_tensor_index + 4;
-    TfLiteTensor* scaling_factors = GetTemporary(context, node, /*index=*/4);
-    scaling_factors->type = kTfLiteFloat32;
-    scaling_factors->allocation_type = kTfLiteArenaRw;
-    int scaling_dims[1] = {n_batch};
-    if (!TfLiteIntArrayEqualsArray(scaling_factors->dims, 1, scaling_dims)) {
-      TfLiteIntArray* scaling_factors_size = TfLiteIntArrayCreate(1);
-      scaling_factors_size->data[0] = n_batch;
-      TF_LITE_ENSURE_OK(context, context->ResizeTensor(context, scaling_factors,
-                                                       scaling_factors_size));
-    }
-    node->temporaries->data[5] = op_data->scratch_tensor_index + 5;
-    TfLiteTensor* prod_scaling_factors =
-        GetTemporary(context, node, /*index=*/5);
-    prod_scaling_factors->type = kTfLiteFloat32;
-    prod_scaling_factors->allocation_type = kTfLiteArenaRw;
-    if (!TfLiteIntArrayEqualsArray(prod_scaling_factors->dims, 1,
-                                   scaling_dims)) {
-      TfLiteIntArray* prod_scaling_factors_size = TfLiteIntArrayCreate(1);
-      prod_scaling_factors_size->data[0] = n_batch;
-      TF_LITE_ENSURE_OK(context,
-                        context->ResizeTensor(context, prod_scaling_factors,
-                                              prod_scaling_factors_size));
-    }
-
-    // Allocate a temporary tensor to store the recovered weights. Since
-    // this is used for diagonal matrices, only need to store n_cell values.
-    node->temporaries->data[6] = op_data->scratch_tensor_index + 6;
-    TfLiteTensor* recovered_weights = GetTemporary(context, node, /*index=*/6);
-    recovered_weights->type = kTfLiteFloat32;
-    recovered_weights->allocation_type = kTfLiteArenaRw;
-    int recovered_dims[1] = {n_cell};
-    if (!TfLiteIntArrayEqualsArray(recovered_weights->dims, 1,
-                                   recovered_dims)) {
-      TfLiteIntArray* recovered_weights_size = TfLiteIntArrayCreate(1);
-      recovered_weights_size->data[0] = n_cell;
-      TF_LITE_ENSURE_OK(context,
-                        context->ResizeTensor(context, recovered_weights,
-                                              recovered_weights_size));
-    }
-  }
-  return kTfLiteOk;
-}
-
-void LayerNormLstmStep(
-    const float* input_ptr_batch, const float* input_to_input_weights_ptr,
-    const float* input_to_forget_weights_ptr,
-    const float* input_to_cell_weights_ptr,
-    const float* input_to_output_weights_ptr,
-    const float* recurrent_to_input_weights_ptr,
-    const float* recurrent_to_forget_weights_ptr,
-    const float* recurrent_to_cell_weights_ptr,
-    const float* recurrent_to_output_weights_ptr,
-    const float* cell_to_input_weights_ptr,
-    const float* cell_to_forget_weights_ptr,
-    const float* cell_to_output_weights_ptr,
-    const float* input_layer_norm_weight_ptr,
-    const float* forget_layer_norm_weight_ptr,
-    const float* cell_layer_norm_weight_ptr,
-    const float* output_layer_norm_weight_ptr, const float* input_gate_bias_ptr,
-    const float* forget_gate_bias_ptr, const float* cell_bias_ptr,
-    const float* output_gate_bias_ptr, const float* projection_weights_ptr,
-    const float* projection_bias_ptr, float cell_clip, float proj_clip,
-    const TfLiteFusedActivation& activation, int n_batch, int n_cell,
-    int n_input, int n_output, float* output_state_ptr, float* cell_state_ptr,
-    float* input_gate_scratch, float* forget_gate_scratch, float* cell_scratch,
-    float* output_gate_scratch, float* output_ptr_batch) {
-  // Since we have already checked that weights are all there or none, we can
-  // check the existense of only one to the get the condition.
-  const bool use_cifg = (input_to_input_weights_ptr == nullptr);
-  const bool use_peephole = (cell_to_output_weights_ptr != nullptr);
-
-  // Initialize scratch buffers with 0.
-  if (!use_cifg) {
-    tensor_utils::ZeroVector(input_gate_scratch, n_cell * n_batch);
-  }
-  tensor_utils::ZeroVector(forget_gate_scratch, n_cell * n_batch);
-  tensor_utils::ZeroVector(cell_scratch, n_cell * n_batch);
-  tensor_utils::ZeroVector(output_gate_scratch, n_cell * n_batch);
-
-  // For each batch and cell: compute input_weight * input.
-  if (!use_cifg) {
-    tensor_utils::MatrixBatchVectorMultiplyAccumulate(
-        input_to_input_weights_ptr, n_cell, n_input, input_ptr_batch, n_batch,
-        input_gate_scratch, /*result_stride=*/1);
-  }
-
-  tensor_utils::MatrixBatchVectorMultiplyAccumulate(
-      input_to_forget_weights_ptr, n_cell, n_input, input_ptr_batch, n_batch,
-      forget_gate_scratch, /*result_stride=*/1);
-  tensor_utils::MatrixBatchVectorMultiplyAccumulate(
-      input_to_cell_weights_ptr, n_cell, n_input, input_ptr_batch, n_batch,
-      cell_scratch, /*result_stride=*/1);
-  tensor_utils::MatrixBatchVectorMultiplyAccumulate(
-      input_to_output_weights_ptr, n_cell, n_input, input_ptr_batch, n_batch,
-      output_gate_scratch, /*result_stride=*/1);
-
-  // For each batch and cell: compute recurrent_weight * output_state.
-  if (!use_cifg) {
-    tensor_utils::MatrixBatchVectorMultiplyAccumulate(
-        recurrent_to_input_weights_ptr, n_cell, n_output, output_state_ptr,
-        n_batch, input_gate_scratch, /*result_stride=*/1);
-  }
-  tensor_utils::MatrixBatchVectorMultiplyAccumulate(
-      recurrent_to_forget_weights_ptr, n_cell, n_output, output_state_ptr,
-      n_batch, forget_gate_scratch,
-      /*result_stride=*/1);
-  tensor_utils::MatrixBatchVectorMultiplyAccumulate(
-      recurrent_to_cell_weights_ptr, n_cell, n_output, output_state_ptr,
-      n_batch, cell_scratch, /*result_stride=*/1);
-  tensor_utils::MatrixBatchVectorMultiplyAccumulate(
-      recurrent_to_output_weights_ptr, n_cell, n_output, output_state_ptr,
-      n_batch, output_gate_scratch,
-      /*result_stride=*/1);
-
-  // For each batch and cell: update input gate.
-  if (!use_cifg) {
-    if (use_peephole) {
-      tensor_utils::VectorBatchVectorCwiseProductAccumulate(
-          cell_to_input_weights_ptr, n_cell, cell_state_ptr, n_batch,
-          input_gate_scratch);
-    }
-    tensor_utils::MeanStddevNormalization(input_gate_scratch,
-                                          input_gate_scratch, n_cell, n_batch,
-                                          kLayerNormEpsilon);
-    tensor_utils::VectorBatchVectorCwiseProduct(input_layer_norm_weight_ptr,
-                                                n_cell, input_gate_scratch,
-                                                n_batch, input_gate_scratch);
-    tensor_utils::VectorBatchVectorAdd(input_gate_bias_ptr, n_cell, n_batch,
-                                       input_gate_scratch);
-    tensor_utils::ApplySigmoidToVector(input_gate_scratch, n_cell * n_batch,
-                                       input_gate_scratch);
-  }
-
-  // For each batch and cell: update forget gate.
-  if (use_peephole) {
-    tensor_utils::VectorBatchVectorCwiseProductAccumulate(
-        cell_to_forget_weights_ptr, n_cell, cell_state_ptr, n_batch,
-        forget_gate_scratch);
-  }
-  tensor_utils::MeanStddevNormalization(forget_gate_scratch,
-                                        forget_gate_scratch, n_cell, n_batch,
-                                        kLayerNormEpsilon);
-  tensor_utils::VectorBatchVectorCwiseProduct(forget_layer_norm_weight_ptr,
-                                              n_cell, forget_gate_scratch,
-                                              n_batch, forget_gate_scratch);
-  tensor_utils::VectorBatchVectorAdd(forget_gate_bias_ptr, n_cell, n_batch,
-                                     forget_gate_scratch);
-  tensor_utils::ApplySigmoidToVector(forget_gate_scratch, n_cell * n_batch,
-                                     forget_gate_scratch);
-
-  // For each batch and cell: update the cell.
-  tensor_utils::MeanStddevNormalization(cell_scratch, cell_scratch, n_cell,
-                                        n_batch, kLayerNormEpsilon);
-  tensor_utils::VectorBatchVectorCwiseProduct(
-      cell_layer_norm_weight_ptr, n_cell, cell_scratch, n_batch, cell_scratch);
-  tensor_utils::VectorBatchVectorAdd(cell_bias_ptr, n_cell, n_batch,
-                                     cell_scratch);
-  tensor_utils::VectorVectorCwiseProduct(forget_gate_scratch, cell_state_ptr,
-                                         n_batch * n_cell, cell_state_ptr);
-  tensor_utils::ApplyActivationToVector(cell_scratch, n_batch * n_cell,
-                                        activation, cell_scratch);
-  if (use_cifg) {
-    tensor_utils::Sub1Vector(forget_gate_scratch, n_batch * n_cell,
-                             forget_gate_scratch);
-    tensor_utils::VectorVectorCwiseProductAccumulate(
-        cell_scratch, forget_gate_scratch, n_batch * n_cell, cell_state_ptr);
-  } else {
-    tensor_utils::VectorVectorCwiseProductAccumulate(
-        cell_scratch, input_gate_scratch, n_batch * n_cell, cell_state_ptr);
-  }
-  if (cell_clip > 0.0) {
-    tensor_utils::ClipVector(cell_state_ptr, n_batch * n_cell, cell_clip,
-                             cell_state_ptr);
-  }
-
-  // For each batch and cell: update the output gate.
-  if (use_peephole) {
-    tensor_utils::VectorBatchVectorCwiseProductAccumulate(
-        cell_to_output_weights_ptr, n_cell, cell_state_ptr, n_batch,
-        output_gate_scratch);
-  }
-  tensor_utils::MeanStddevNormalization(output_gate_scratch,
-                                        output_gate_scratch, n_cell, n_batch,
-                                        kLayerNormEpsilon);
-  tensor_utils::VectorBatchVectorCwiseProduct(output_layer_norm_weight_ptr,
-                                              n_cell, output_gate_scratch,
-                                              n_batch, output_gate_scratch);
-  tensor_utils::VectorBatchVectorAdd(output_gate_bias_ptr, n_cell, n_batch,
-                                     output_gate_scratch);
-  tensor_utils::ApplySigmoidToVector(output_gate_scratch, n_batch * n_cell,
-                                     output_gate_scratch);
-  tensor_utils::ApplyActivationToVector(cell_state_ptr, n_batch * n_cell,
-                                        activation, cell_scratch);
-  tensor_utils::VectorVectorCwiseProduct(output_gate_scratch, cell_scratch,
-                                         n_batch * n_cell, output_gate_scratch);
-
-  // For each batch: update the projection and output_state.
-  const bool use_projection_weight = (projection_weights_ptr != nullptr);
-  const bool use_projection_bias = (projection_bias_ptr != nullptr);
-  if (use_projection_weight) {
-    if (use_projection_bias) {
-      tensor_utils::VectorBatchVectorAssign(projection_bias_ptr, n_output,
-                                            n_batch, output_ptr_batch);
-    } else {
-      tensor_utils::ZeroVector(output_ptr_batch, n_batch * n_output);
-    }
-    tensor_utils::MatrixBatchVectorMultiplyAccumulate(
-        projection_weights_ptr, n_output, n_cell, output_gate_scratch, n_batch,
-        output_ptr_batch, /*result_stride=*/1);
-    if (proj_clip > 0.0) {
-      tensor_utils::ClipVector(output_ptr_batch, n_batch * n_output, proj_clip,
-                               output_ptr_batch);
-    }
-  } else {
-    tensor_utils::CopyVector(output_gate_scratch, n_batch * n_output,
-                             output_ptr_batch);
-  }
-  tensor_utils::CopyVector(output_ptr_batch, n_batch * n_output,
-                           output_state_ptr);
-}
-
-void LayerNormLstmStep(
-    const float* input_ptr_batch, const int8_t* input_to_input_weights_ptr,
-    float input_to_input_weights_scale,
-    const int8_t* input_to_forget_weights_ptr,
-    float input_to_forget_weights_scale,
-    const int8_t* input_to_cell_weights_ptr, float input_to_cell_weights_scale,
-    const int8_t* input_to_output_weights_ptr,
-    float input_to_output_weights_scale,
-    const int8_t* recurrent_to_input_weights_ptr,
-    float recurrent_to_input_weights_scale,
-    const int8_t* recurrent_to_forget_weights_ptr,
-    float recurrent_to_forget_weights_scale,
-    const int8_t* recurrent_to_cell_weights_ptr,
-    float recurrent_to_cell_weights_scale,
-    const int8_t* recurrent_to_output_weights_ptr,
-    float recurrent_to_output_weights_scale,
-    const int8_t* cell_to_input_weights_ptr, float cell_to_input_weights_scale,
-    const int8_t* cell_to_forget_weights_ptr,
-    float cell_to_forget_weights_scale,
-    const int8_t* cell_to_output_weights_ptr,
-    float cell_to_output_weights_scale,
-    const float* input_layer_norm_weight_ptr,
-    const float* forget_layer_norm_weight_ptr,
-    const float* cell_layer_norm_weight_ptr,
-    const float* output_layer_norm_weight_ptr, const float* input_gate_bias_ptr,
-    const float* forget_gate_bias_ptr, const float* cell_bias_ptr,
-    const float* output_gate_bias_ptr, const int8_t* projection_weights_ptr,
-    float projection_weights_scale, const float* projection_bias_ptr,
-    float cell_clip, float proj_clip, const TfLiteFusedActivation& activation,
-    int n_batch, int n_cell, int n_input, int n_output,
-    float* input_gate_scratch, float* forget_gate_scratch, float* cell_scratch,
-    float* output_gate_scratch, float* scaling_factors,
-    float* product_scaling_factors, float* recovered_weights,
-    int8_t* quantized_input_ptr_batch, int8_t* quantized_output_state_ptr,
-    int8_t* quantized_cell_state_ptr, float* output_state_ptr,
-    float* cell_state_ptr, float* output_ptr_batch) {
-  // Since we have already checked that weights are all there or none, we can
-  // check the existense of only one to the get the condition.
-  const bool use_cifg = (input_to_input_weights_ptr == nullptr);
-  const bool use_peephole = (cell_to_output_weights_ptr != nullptr);
-
-  // Initialize scratch buffers with 0.
-  if (!use_cifg) {
-    tensor_utils::ZeroVector(input_gate_scratch, n_cell * n_batch);
-  }
-  tensor_utils::ZeroVector(forget_gate_scratch, n_cell * n_batch);
-  tensor_utils::ZeroVector(cell_scratch, n_cell * n_batch);
-  tensor_utils::ZeroVector(output_gate_scratch, n_cell * n_batch);
-
-  if (!tensor_utils::IsZeroVector(input_ptr_batch, n_batch * n_input)) {
-    // Save quantization and matmul computation for all zero input.
-    float unused_min, unused_max;
-    for (int b = 0; b < n_batch; ++b) {
-      const int offset = b * n_input;
-      tensor_utils::SymmetricQuantizeFloats(
-          input_ptr_batch + offset, n_input, quantized_input_ptr_batch + offset,
-          &unused_min, &unused_max, &scaling_factors[b]);
-    }
-    // For each batch and cell: compute input_weight * input.
-    if (!use_cifg) {
-      for (int b = 0; b < n_batch; ++b) {
-        product_scaling_factors[b] =
-            scaling_factors[b] * input_to_input_weights_scale;
-      }
-      tensor_utils::MatrixBatchVectorMultiplyAccumulate(
-          input_to_input_weights_ptr, n_cell, n_input,
-          quantized_input_ptr_batch, product_scaling_factors, n_batch,
-          input_gate_scratch, /*result_stride=*/1);
-    }
-
-    for (int b = 0; b < n_batch; ++b) {
-      product_scaling_factors[b] =
-          scaling_factors[b] * input_to_forget_weights_scale;
-    }
-    tensor_utils::MatrixBatchVectorMultiplyAccumulate(
-        input_to_forget_weights_ptr, n_cell, n_input, quantized_input_ptr_batch,
-        product_scaling_factors, n_batch, forget_gate_scratch,
-        /*result_stride=*/1);
-
-    for (int b = 0; b < n_batch; ++b) {
-      product_scaling_factors[b] =
-          scaling_factors[b] * input_to_cell_weights_scale;
-    }
-    tensor_utils::MatrixBatchVectorMultiplyAccumulate(
-        input_to_cell_weights_ptr, n_cell, n_input, quantized_input_ptr_batch,
-        product_scaling_factors, n_batch, cell_scratch, /*result_stride=*/1);
-
-    for (int b = 0; b < n_batch; ++b) {
-      product_scaling_factors[b] =
-          scaling_factors[b] * input_to_output_weights_scale;
-    }
-    tensor_utils::MatrixBatchVectorMultiplyAccumulate(
-        input_to_output_weights_ptr, n_cell, n_input, quantized_input_ptr_batch,
-        product_scaling_factors, n_batch, output_gate_scratch,
-        /*result_stride=*/1);
-  }
-
-  if (!tensor_utils::IsZeroVector(output_state_ptr, n_batch * n_output)) {
-    // Save quantization and matmul computation for all zero input.
-    float unused_min, unused_max;
-    for (int b = 0; b < n_batch; ++b) {
-      const int offset = b * n_output;
-      tensor_utils::SymmetricQuantizeFloats(output_state_ptr + offset, n_output,
-                                            quantized_output_state_ptr + offset,
-                                            &unused_min, &unused_max,
-                                            &scaling_factors[b]);
-    }
-    // For each batch and cell: compute recurrent_weight * output_state.
-    if (!use_cifg) {
-      for (int b = 0; b < n_batch; ++b) {
-        product_scaling_factors[b] =
-            scaling_factors[b] * recurrent_to_input_weights_scale;
-      }
-      tensor_utils::MatrixBatchVectorMultiplyAccumulate(
-          recurrent_to_input_weights_ptr, n_cell, n_output,
-          quantized_output_state_ptr, product_scaling_factors, n_batch,
-          input_gate_scratch, /*result_stride=*/1);
-    }
-
-    for (int b = 0; b < n_batch; ++b) {
-      product_scaling_factors[b] =
-          scaling_factors[b] * recurrent_to_forget_weights_scale;
-    }
-    tensor_utils::MatrixBatchVectorMultiplyAccumulate(
-        recurrent_to_forget_weights_ptr, n_cell, n_output,
-        quantized_output_state_ptr, product_scaling_factors, n_batch,
-        forget_gate_scratch, /*result_stride=*/1);
-
-    for (int b = 0; b < n_batch; ++b) {
-      product_scaling_factors[b] =
-          scaling_factors[b] * recurrent_to_cell_weights_scale;
-    }
-    tensor_utils::MatrixBatchVectorMultiplyAccumulate(
-        recurrent_to_cell_weights_ptr, n_cell, n_output,
-        quantized_output_state_ptr, product_scaling_factors, n_batch,
-        cell_scratch, /*result_stride=*/1);
-
-    for (int b = 0; b < n_batch; ++b) {
-      product_scaling_factors[b] =
-          scaling_factors[b] * recurrent_to_output_weights_scale;
-    }
-    tensor_utils::MatrixBatchVectorMultiplyAccumulate(
-        recurrent_to_output_weights_ptr, n_cell, n_output,
-        quantized_output_state_ptr, product_scaling_factors, n_batch,
-        output_gate_scratch, /*result_stride=*/1);
-  }
-
-  // Save quantization and matmul computation for all zero input.
-  bool is_cell_state_all_zeros =
-      tensor_utils::IsZeroVector(cell_state_ptr, n_batch * n_cell);
-
-  // For each batch and cell: update input gate.
-  if (!use_cifg) {
-    if (use_peephole && !is_cell_state_all_zeros) {
-      tensor_utils::VectorScalarMultiply(cell_to_input_weights_ptr, n_cell,
-                                         cell_to_input_weights_scale,
-                                         recovered_weights);
-      tensor_utils::VectorBatchVectorCwiseProductAccumulate(
-          recovered_weights, n_cell, cell_state_ptr, n_batch,
-          input_gate_scratch);
-    }
-    tensor_utils::MeanStddevNormalization(input_gate_scratch,
-                                          input_gate_scratch, n_cell, n_batch,
-                                          kLayerNormEpsilon);
-    tensor_utils::VectorBatchVectorCwiseProduct(input_layer_norm_weight_ptr,
-                                                n_cell, input_gate_scratch,
-                                                n_batch, input_gate_scratch);
-    tensor_utils::VectorBatchVectorAdd(input_gate_bias_ptr, n_cell, n_batch,
-                                       input_gate_scratch);
-    tensor_utils::ApplySigmoidToVector(input_gate_scratch, n_cell * n_batch,
-                                       input_gate_scratch);
-  }
-
-  // For each batch and cell: update forget gate.
-  if (use_peephole && !is_cell_state_all_zeros) {
-    tensor_utils::VectorScalarMultiply(cell_to_forget_weights_ptr, n_cell,
-                                       cell_to_forget_weights_scale,
-                                       recovered_weights);
-    tensor_utils::VectorBatchVectorCwiseProductAccumulate(
-        recovered_weights, n_cell, cell_state_ptr, n_batch,
-        forget_gate_scratch);
-  }
-  tensor_utils::MeanStddevNormalization(forget_gate_scratch,
-                                        forget_gate_scratch, n_cell, n_batch,
-                                        kLayerNormEpsilon);
-  tensor_utils::VectorBatchVectorCwiseProduct(forget_layer_norm_weight_ptr,
-                                              n_cell, forget_gate_scratch,
-                                              n_batch, forget_gate_scratch);
-  tensor_utils::VectorBatchVectorAdd(forget_gate_bias_ptr, n_cell, n_batch,
-                                     forget_gate_scratch);
-  tensor_utils::ApplySigmoidToVector(forget_gate_scratch, n_cell * n_batch,
-                                     forget_gate_scratch);
-
-  // For each batch and cell: update the cell.
-  tensor_utils::MeanStddevNormalization(cell_scratch, cell_scratch, n_cell,
-                                        n_batch, kLayerNormEpsilon);
-  tensor_utils::VectorBatchVectorCwiseProduct(
-      cell_layer_norm_weight_ptr, n_cell, cell_scratch, n_batch, cell_scratch);
-  tensor_utils::VectorBatchVectorAdd(cell_bias_ptr, n_cell, n_batch,
-                                     cell_scratch);
-  tensor_utils::VectorVectorCwiseProduct(forget_gate_scratch, cell_state_ptr,
-                                         n_batch * n_cell, cell_state_ptr);
-  tensor_utils::ApplyActivationToVector(cell_scratch, n_batch * n_cell,
-                                        activation, cell_scratch);
-  if (use_cifg) {
-    tensor_utils::Sub1Vector(forget_gate_scratch, n_batch * n_cell,
-                             forget_gate_scratch);
-    tensor_utils::VectorVectorCwiseProductAccumulate(
-        cell_scratch, forget_gate_scratch, n_batch * n_cell, cell_state_ptr);
-  } else {
-    tensor_utils::VectorVectorCwiseProductAccumulate(
-        cell_scratch, input_gate_scratch, n_batch * n_cell, cell_state_ptr);
-  }
-  if (cell_clip > 0.0) {
-    tensor_utils::ClipVector(cell_state_ptr, n_batch * n_cell, cell_clip,
-                             cell_state_ptr);
-  }
-
-  is_cell_state_all_zeros =
-      tensor_utils::IsZeroVector(cell_state_ptr, n_batch * n_cell);
-  // For each batch and cell: update the output gate.
-  if (use_peephole && !is_cell_state_all_zeros) {
-    tensor_utils::VectorScalarMultiply(cell_to_output_weights_ptr, n_cell,
-                                       cell_to_output_weights_scale,
-                                       recovered_weights);
-    tensor_utils::VectorBatchVectorCwiseProductAccumulate(
-        recovered_weights, n_cell, cell_state_ptr, n_batch,
-        output_gate_scratch);
-  }
-  tensor_utils::MeanStddevNormalization(output_gate_scratch,
-                                        output_gate_scratch, n_cell, n_batch,
-                                        kLayerNormEpsilon);
-  tensor_utils::VectorBatchVectorCwiseProduct(output_layer_norm_weight_ptr,
-                                              n_cell, output_gate_scratch,
-                                              n_batch, output_gate_scratch);
-  tensor_utils::VectorBatchVectorAdd(output_gate_bias_ptr, n_cell, n_batch,
-                                     output_gate_scratch);
-  tensor_utils::ApplySigmoidToVector(output_gate_scratch, n_batch * n_cell,
-                                     output_gate_scratch);
-  tensor_utils::ApplyActivationToVector(cell_state_ptr, n_batch * n_cell,
-                                        activation, cell_scratch);
-  tensor_utils::VectorVectorCwiseProduct(output_gate_scratch, cell_scratch,
-                                         n_batch * n_cell, output_gate_scratch);
-
-  // For each batch: update the projection and output_state.
-  const bool use_projection_weight = (projection_weights_ptr != nullptr);
-  const bool use_projection_bias = (projection_bias_ptr != nullptr);
-  if (use_projection_weight) {
-    if (use_projection_bias) {
-      tensor_utils::VectorBatchVectorAssign(projection_bias_ptr, n_output,
-                                            n_batch, output_ptr_batch);
-    } else {
-      tensor_utils::ZeroVector(output_ptr_batch, n_batch * n_output);
-    }
-    if (!tensor_utils::IsZeroVector(output_gate_scratch, n_batch * n_cell)) {
-      // Save quantization and matmul computation for all zero input.
-      float unused_min, unused_max;
-      for (int b = 0; b < n_batch; ++b) {
-        const int offset = b * n_cell;
-        tensor_utils::SymmetricQuantizeFloats(
-            output_gate_scratch + offset, n_cell,
-            quantized_cell_state_ptr + offset, &unused_min, &unused_max,
-            &scaling_factors[b]);
-      }
-      for (int b = 0; b < n_batch; ++b) {
-        product_scaling_factors[b] =
-            scaling_factors[b] * projection_weights_scale;
-      }
-      tensor_utils::MatrixBatchVectorMultiplyAccumulate(
-          projection_weights_ptr, n_output, n_cell, quantized_cell_state_ptr,
-          product_scaling_factors, n_batch, output_ptr_batch,
-          /*result_stride=*/1);
-    }
-    if (proj_clip > 0.0) {
-      tensor_utils::ClipVector(output_ptr_batch, n_batch * n_output, proj_clip,
-                               output_ptr_batch);
-    }
-  } else {
-    tensor_utils::CopyVector(output_gate_scratch, n_batch * n_output,
-                             output_ptr_batch);
-  }
-  tensor_utils::CopyVector(output_ptr_batch, n_batch * n_output,
-                           output_state_ptr);
-}
-
-// The LayerNormLSTM Op engine.
-TfLiteStatus EvalFloat(
-    const TfLiteTensor* input, const TfLiteTensor* input_to_input_weights,
-    const TfLiteTensor* input_to_forget_weights,
-    const TfLiteTensor* input_to_cell_weights,
-    const TfLiteTensor* input_to_output_weights,
-    const TfLiteTensor* recurrent_to_input_weights,
-    const TfLiteTensor* recurrent_to_forget_weights,
-    const TfLiteTensor* recurrent_to_cell_weights,
-    const TfLiteTensor* recurrent_to_output_weights,
-    const TfLiteTensor* cell_to_input_weights,
-    const TfLiteTensor* cell_to_forget_weights,
-    const TfLiteTensor* cell_to_output_weights,
-    const TfLiteTensor* input_layer_norm_weights,
-    const TfLiteTensor* forget_layer_norm_weights,
-    const TfLiteTensor* cell_layer_norm_weights,
-    const TfLiteTensor* output_layer_norm_weights,
-    const TfLiteTensor* input_gate_bias, const TfLiteTensor* forget_gate_bias,
-    const TfLiteTensor* cell_bias, const TfLiteTensor* output_gate_bias,
-    const TfLiteTensor* projection_weights, const TfLiteTensor* projection_bias,
-    float cell_clip, float proj_clip, const TfLiteFusedActivation& activation,
-    TfLiteTensor* scratch_buffer, TfLiteTensor* activation_state,
-    TfLiteTensor* cell_state, TfLiteTensor* output) {
-  const int n_batch = input->dims->data[0];
-  const int n_input = input->dims->data[1];
-  // n_cell and n_output will be the same size when there is no projection.
-  const int n_cell = input_to_output_weights->dims->data[0];
-  const int n_output = recurrent_to_output_weights->dims->data[1];
-
-  // Since we have already checked that weights are all there or none, we can
-  // check the existence of only one to get the condition.
-  const bool use_cifg = (input_to_input_weights == nullptr);
-  const bool use_peephole = (cell_to_output_weights != nullptr);
-
-  float* input_gate_scratch = nullptr;
-  float* cell_scratch = nullptr;
-  float* forget_gate_scratch = nullptr;
-  float* output_gate_scratch = nullptr;
-  if (use_cifg) {
-    cell_scratch = scratch_buffer->data.f;
-    forget_gate_scratch = scratch_buffer->data.f + n_cell * n_batch;
-    output_gate_scratch = scratch_buffer->data.f + 2 * n_cell * n_batch;
-  } else {
-    input_gate_scratch = scratch_buffer->data.f;
-    cell_scratch = scratch_buffer->data.f + n_cell * n_batch;
-    forget_gate_scratch = scratch_buffer->data.f + 2 * n_cell * n_batch;
-    output_gate_scratch = scratch_buffer->data.f + 3 * n_cell * n_batch;
-  }
-
-  // Check optional tensors, the respective pointers can be null.
-  const float* input_to_input_weights_ptr =
-      (use_cifg) ? nullptr : input_to_input_weights->data.f;
-  const float* recurrent_to_input_weights_ptr =
-      (use_cifg) ? nullptr : recurrent_to_input_weights->data.f;
-  const float* input_gate_bias_ptr =
-      (use_cifg) ? nullptr : input_gate_bias->data.f;
-  const float* cell_to_input_weights_ptr =
-      (use_peephole && !use_cifg) ? cell_to_input_weights->data.f : nullptr;
-  const float* cell_to_forget_weights_ptr =
-      (use_peephole) ? cell_to_forget_weights->data.f : nullptr;
-  const float* cell_to_output_weights_ptr =
-      (use_peephole) ? cell_to_output_weights->data.f : nullptr;
-  const float* projection_weights_ptr =
-      (projection_weights == nullptr) ? nullptr : projection_weights->data.f;
-  const float* projection_bias_ptr =
-      (projection_bias == nullptr) ? nullptr : projection_bias->data.f;
-  const float* input_layer_norm_weight_ptr =
-      (input_layer_norm_weights == nullptr) ? nullptr
-                                            : input_layer_norm_weights->data.f;
-
-  // Required tensors, pointers are non-null.
-  const float* input_ptr_batch = input->data.f;
-  const float* input_to_forget_weights_ptr = input_to_forget_weights->data.f;
-  const float* input_to_cell_weights_ptr = input_to_cell_weights->data.f;
-  const float* input_to_output_weights_ptr = input_to_output_weights->data.f;
-  const float* recurrent_to_forget_weights_ptr =
-      recurrent_to_forget_weights->data.f;
-  const float* recurrent_to_cell_weights_ptr =
-      recurrent_to_cell_weights->data.f;
-  const float* recurrent_to_output_weights_ptr =
-      recurrent_to_output_weights->data.f;
-  const float* forget_layer_norm_weight_ptr = forget_layer_norm_weights->data.f;
-  const float* cell_layer_norm_weight_ptr = cell_layer_norm_weights->data.f;
-  const float* output_layer_norm_weight_ptr = output_layer_norm_weights->data.f;
-  const float* forget_gate_bias_ptr = forget_gate_bias->data.f;
-  const float* cell_bias_ptr = cell_bias->data.f;
-  const float* output_gate_bias_ptr = output_gate_bias->data.f;
-
-  float* activation_state_ptr = activation_state->data.f;
-  float* cell_state_ptr = cell_state->data.f;
-  float* output_ptr_batch = output->data.f;
-
-  LayerNormLstmStep(
-      input_ptr_batch, input_to_input_weights_ptr, input_to_forget_weights_ptr,
-      input_to_cell_weights_ptr, input_to_output_weights_ptr,
-      recurrent_to_input_weights_ptr, recurrent_to_forget_weights_ptr,
-      recurrent_to_cell_weights_ptr, recurrent_to_output_weights_ptr,
-      cell_to_input_weights_ptr, cell_to_forget_weights_ptr,
-      cell_to_output_weights_ptr, input_layer_norm_weight_ptr,
-      forget_layer_norm_weight_ptr, cell_layer_norm_weight_ptr,
-      output_layer_norm_weight_ptr, input_gate_bias_ptr, forget_gate_bias_ptr,
-      cell_bias_ptr, output_gate_bias_ptr, projection_weights_ptr,
-      projection_bias_ptr, cell_clip, proj_clip, activation, n_batch, n_cell,
-      n_input, n_output, activation_state_ptr, cell_state_ptr,
-      input_gate_scratch, forget_gate_scratch, cell_scratch,
-      output_gate_scratch, output_ptr_batch);
-
-  return kTfLiteOk;
-}
-
-TfLiteStatus EvalHybrid(
-    const TfLiteTensor* input, const TfLiteTensor* input_to_input_weights,
-    const TfLiteTensor* input_to_forget_weights,
-    const TfLiteTensor* input_to_cell_weights,
-    const TfLiteTensor* input_to_output_weights,
-    const TfLiteTensor* recurrent_to_input_weights,
-    const TfLiteTensor* recurrent_to_forget_weights,
-    const TfLiteTensor* recurrent_to_cell_weights,
-    const TfLiteTensor* recurrent_to_output_weights,
-    const TfLiteTensor* cell_to_input_weights,
-    const TfLiteTensor* cell_to_forget_weights,
-    const TfLiteTensor* cell_to_output_weights,
-    const TfLiteTensor* input_layer_norm_weights,
-    const TfLiteTensor* forget_layer_norm_weights,
-    const TfLiteTensor* cell_layer_norm_weights,
-    const TfLiteTensor* output_layer_norm_weights,
-    const TfLiteTensor* input_gate_bias, const TfLiteTensor* forget_gate_bias,
-    const TfLiteTensor* cell_bias, const TfLiteTensor* output_gate_bias,
-    const TfLiteTensor* projection_weights, const TfLiteTensor* projection_bias,
-    float cell_clip, float proj_clip, const TfLiteFusedActivation& activation,
-    TfLiteTensor* scratch_buffer, TfLiteTensor* scaling_factors,
-    TfLiteTensor* prod_scaling_factors, TfLiteTensor* recovered_weights,
-    TfLiteTensor* input_quantized, TfLiteTensor* activation_state_quantized,
-    TfLiteTensor* cell_state_quantized, TfLiteTensor* activation_state,
-    TfLiteTensor* cell_state, TfLiteTensor* output) {
-  const int n_batch = input->dims->data[0];
-  const int n_input = input->dims->data[1];
-  // n_cell and n_output will be the same size when there is no projection.
-  const int n_cell = input_to_output_weights->dims->data[0];
-  const int n_output = recurrent_to_output_weights->dims->data[1];
-
-  // Since we have already checked that weights are all there or none, we can
-  // check the existence of only one to get the condition.
-  const bool use_cifg = (input_to_input_weights == nullptr);
-  const bool use_peephole = (cell_to_output_weights != nullptr);
-
-  float* input_gate_scratch = nullptr;
-  float* cell_scratch = nullptr;
-  float* forget_gate_scratch = nullptr;
-  float* output_gate_scratch = nullptr;
-  if (use_cifg) {
-    cell_scratch = scratch_buffer->data.f;
-    forget_gate_scratch = scratch_buffer->data.f + n_cell * n_batch;
-    output_gate_scratch = scratch_buffer->data.f + 2 * n_cell * n_batch;
-  } else {
-    input_gate_scratch = scratch_buffer->data.f;
-    cell_scratch = scratch_buffer->data.f + n_cell * n_batch;
-    forget_gate_scratch = scratch_buffer->data.f + 2 * n_cell * n_batch;
-    output_gate_scratch = scratch_buffer->data.f + 3 * n_cell * n_batch;
-  }
-
-  // Check optional tensors, the respective pointers can be null.
-  int8_t* input_to_input_weights_ptr = nullptr;
-  float input_to_input_weights_scale = 1.0f;
-  int8_t* recurrent_to_input_weights_ptr = nullptr;
-  float recurrent_to_input_weights_scale = 1.0f;
-  float* input_gate_bias_ptr = nullptr;
-  if (!use_cifg) {
-    input_to_input_weights_ptr =
-        reinterpret_cast<int8_t*>(input_to_input_weights->data.uint8);
-    recurrent_to_input_weights_ptr =
-        reinterpret_cast<int8_t*>(recurrent_to_input_weights->data.uint8);
-    input_gate_bias_ptr = input_gate_bias->data.f;
-    input_to_input_weights_scale = input_to_input_weights->params.scale;
-    recurrent_to_input_weights_scale = recurrent_to_input_weights->params.scale;
-  }
-
-  int8_t* cell_to_input_weights_ptr = nullptr;
-  int8_t* cell_to_forget_weights_ptr = nullptr;
-  int8_t* cell_to_output_weights_ptr = nullptr;
-  float cell_to_input_weights_scale = 1.0f;
-  float cell_to_forget_weights_scale = 1.0f;
-  float cell_to_output_weights_scale = 1.0f;
-  if (use_peephole) {
-    if (!use_cifg) {
-      cell_to_input_weights_ptr =
-          reinterpret_cast<int8_t*>(cell_to_input_weights->data.uint8);
-      cell_to_input_weights_scale = cell_to_input_weights->params.scale;
-    }
-    cell_to_forget_weights_ptr =
-        reinterpret_cast<int8_t*>(cell_to_forget_weights->data.uint8);
-    cell_to_output_weights_ptr =
-        reinterpret_cast<int8_t*>(cell_to_output_weights->data.uint8);
-    cell_to_forget_weights_scale = cell_to_forget_weights->params.scale;
-    cell_to_output_weights_scale = cell_to_output_weights->params.scale;
-  }
-
-  const int8_t* projection_weights_ptr =
-      (projection_weights == nullptr)
-          ? nullptr
-          : reinterpret_cast<int8_t*>(projection_weights->data.uint8);
-  const float projection_weights_scale =
-      (projection_weights == nullptr) ? 1.0f : projection_weights->params.scale;
-  const float* projection_bias_ptr =
-      (projection_bias == nullptr) ? nullptr : projection_bias->data.f;
-  const float* input_layer_norm_weight_ptr =
-      (input_layer_norm_weights == nullptr) ? nullptr
-                                            : input_layer_norm_weights->data.f;
-
-  // Required tensors, pointers are non-null.
-  const float* input_ptr_batch = input->data.f;
-  const int8_t* input_to_forget_weights_ptr =
-      reinterpret_cast<int8_t*>(input_to_forget_weights->data.uint8);
-  const float input_to_forget_weights_scale =
-      input_to_forget_weights->params.scale;
-  const int8_t* input_to_cell_weights_ptr =
-      reinterpret_cast<int8_t*>(input_to_cell_weights->data.uint8);
-  const float input_to_cell_weights_scale = input_to_cell_weights->params.scale;
-  const int8_t* input_to_output_weights_ptr =
-      reinterpret_cast<int8_t*>(input_to_output_weights->data.uint8);
-  const float input_to_output_weights_scale =
-      input_to_output_weights->params.scale;
-  const int8_t* recurrent_to_forget_weights_ptr =
-      reinterpret_cast<int8_t*>(recurrent_to_forget_weights->data.uint8);
-  const float recurrent_to_forget_weights_scale =
-      recurrent_to_forget_weights->params.scale;
-  const int8_t* recurrent_to_cell_weights_ptr =
-      reinterpret_cast<int8_t*>(recurrent_to_cell_weights->data.uint8);
-  const float recurrent_to_cell_weights_scale =
-      recurrent_to_cell_weights->params.scale;
-  const int8_t* recurrent_to_output_weights_ptr =
-      reinterpret_cast<int8_t*>(recurrent_to_output_weights->data.uint8);
-  const float recurrent_to_output_weights_scale =
-      recurrent_to_output_weights->params.scale;
-  const float* forget_layer_norm_weight_ptr = forget_layer_norm_weights->data.f;
-  const float* cell_layer_norm_weight_ptr = cell_layer_norm_weights->data.f;
-  const float* output_layer_norm_weight_ptr = output_layer_norm_weights->data.f;
-  const float* forget_gate_bias_ptr = forget_gate_bias->data.f;
-  const float* cell_bias_ptr = cell_bias->data.f;
-  const float* output_gate_bias_ptr = output_gate_bias->data.f;
-
-  float* activation_state_ptr = activation_state->data.f;
-  float* cell_state_ptr = cell_state->data.f;
-  float* output_ptr_batch = output->data.f;
-
-  // Temporary storage for quantized values and scaling factors.
-  int8_t* quantized_input_ptr =
-      reinterpret_cast<int8_t*>(input_quantized->data.uint8);
-  int8_t* quantized_activation_state_ptr =
-      reinterpret_cast<int8_t*>(activation_state_quantized->data.uint8);
-  int8_t* quantized_cell_state_ptr =
-      reinterpret_cast<int8_t*>(cell_state_quantized->data.uint8);
-  float* scaling_factors_ptr = scaling_factors->data.f;
-  float* prod_scaling_factors_ptr = prod_scaling_factors->data.f;
-  float* recovered_weights_ptr = recovered_weights->data.f;
-
-  LayerNormLstmStep(
-      input_ptr_batch, input_to_input_weights_ptr, input_to_input_weights_scale,
-      input_to_forget_weights_ptr, input_to_forget_weights_scale,
-      input_to_cell_weights_ptr, input_to_cell_weights_scale,
-      input_to_output_weights_ptr, input_to_output_weights_scale,
-      recurrent_to_input_weights_ptr, recurrent_to_input_weights_scale,
-      recurrent_to_forget_weights_ptr, recurrent_to_forget_weights_scale,
-      recurrent_to_cell_weights_ptr, recurrent_to_cell_weights_scale,
-      recurrent_to_output_weights_ptr, recurrent_to_output_weights_scale,
-      cell_to_input_weights_ptr, cell_to_input_weights_scale,
-      cell_to_forget_weights_ptr, cell_to_forget_weights_scale,
-      cell_to_output_weights_ptr, cell_to_output_weights_scale,
-      input_layer_norm_weight_ptr, forget_layer_norm_weight_ptr,
-      cell_layer_norm_weight_ptr, output_layer_norm_weight_ptr,
-      input_gate_bias_ptr, forget_gate_bias_ptr, cell_bias_ptr,
-      output_gate_bias_ptr, projection_weights_ptr, projection_weights_scale,
-      projection_bias_ptr, cell_clip, proj_clip, activation, n_batch, n_cell,
-      n_input, n_output, input_gate_scratch, forget_gate_scratch, cell_scratch,
-      output_gate_scratch, scaling_factors_ptr, prod_scaling_factors_ptr,
-      recovered_weights_ptr, quantized_input_ptr,
-      quantized_activation_state_ptr, quantized_cell_state_ptr,
-      activation_state_ptr, cell_state_ptr, output_ptr_batch);
-
-  return kTfLiteOk;
-}
-
-TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
-  const OpData* op_data = reinterpret_cast<OpData*>(node->user_data);
-
-  const TfLiteTensor* input = GetInput(context, node, kInputTensor);
-
-  const TfLiteTensor* input_to_input_weights =
-      GetOptionalInputTensor(context, node, kInputToInputWeightsTensor);
-  const TfLiteTensor* input_to_forget_weights =
-      GetInput(context, node, kInputToForgetWeightsTensor);
-  const TfLiteTensor* input_to_cell_weights =
-      GetInput(context, node, kInputToCellWeightsTensor);
-  const TfLiteTensor* input_to_output_weights =
-      GetInput(context, node, kInputToOutputWeightsTensor);
-
-  const TfLiteTensor* recurrent_to_input_weights =
-      GetOptionalInputTensor(context, node, kRecurrentToInputWeightsTensor);
-  const TfLiteTensor* recurrent_to_forget_weights =
-      GetInput(context, node, kRecurrentToForgetWeightsTensor);
-  const TfLiteTensor* recurrent_to_cell_weights =
-      GetInput(context, node, kRecurrentToCellWeightsTensor);
-  const TfLiteTensor* recurrent_to_output_weights =
-      GetInput(context, node, kRecurrentToOutputWeightsTensor);
-
-  const TfLiteTensor* cell_to_input_weights =
-      GetOptionalInputTensor(context, node, kCellToInputWeightsTensor);
-  const TfLiteTensor* cell_to_forget_weights =
-      GetOptionalInputTensor(context, node, kCellToForgetWeightsTensor);
-  const TfLiteTensor* cell_to_output_weights =
-      GetOptionalInputTensor(context, node, kCellToOutputWeightsTensor);
-
-  const TfLiteTensor* input_layer_norm_weights =
-      GetOptionalInputTensor(context, node, kInputLayerNormWeightsTensor);
-  const TfLiteTensor* forget_layer_norm_weights =
-      GetInput(context, node, kForgetLayerNormWeightsTensor);
-  const TfLiteTensor* cell_layer_norm_weights =
-      GetInput(context, node, kCellLayerNormWeightsTensor);
-  const TfLiteTensor* output_layer_norm_weights =
-      GetInput(context, node, kOutputLayerNormWeightsTensor);
-
-  const TfLiteTensor* input_gate_bias =
-      GetOptionalInputTensor(context, node, kInputGateBiasTensor);
-  const TfLiteTensor* forget_gate_bias =
-      GetInput(context, node, kForgetGateBiasTensor);
-  const TfLiteTensor* cell_bias = GetInput(context, node, kCellGateBiasTensor);
-  const TfLiteTensor* output_gate_bias =
-      GetInput(context, node, kOutputGateBiasTensor);
-
-  const TfLiteTensor* projection_weights =
-      GetOptionalInputTensor(context, node, kProjectionWeightsTensor);
-  const TfLiteTensor* projection_bias =
-      GetOptionalInputTensor(context, node, kProjectionBiasTensor);
-
-  // Index the scratch buffers pointers to the global scratch buffer.
-  TfLiteTensor* scratch_buffer = GetTemporary(context, node, /*index=*/0);
-
-  TfLiteTensor* activation_state =
-      &context->tensors[node->inputs->data[kInputActivationStateTensor]];
-  TfLiteTensor* cell_state =
-      &context->tensors[node->inputs->data[kInputCellStateTensor]];
-
-  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
-
-  switch (input_to_output_weights->type) {
-    case kTfLiteFloat32: {
-      return EvalFloat(input, input_to_input_weights, input_to_forget_weights,
-                       input_to_cell_weights, input_to_output_weights,
-                       recurrent_to_input_weights, recurrent_to_forget_weights,
-                       recurrent_to_cell_weights, recurrent_to_output_weights,
-                       cell_to_input_weights, cell_to_forget_weights,
-                       cell_to_output_weights, input_layer_norm_weights,
-                       forget_layer_norm_weights, cell_layer_norm_weights,
-                       output_layer_norm_weights, input_gate_bias,
-                       forget_gate_bias, cell_bias, output_gate_bias,
-                       projection_weights, projection_bias, op_data->cell_clip,
-                       op_data->proj_clip, op_data->activation, scratch_buffer,
-                       activation_state, cell_state, output);
-    }
-    case kTfLiteUInt8: {
-      TfLiteTensor* input_quantized = GetTemporary(context, node, /*index=*/1);
-      TfLiteTensor* activation_state_quantized =
-          GetTemporary(context, node, /*index=*/2);
-      TfLiteTensor* cell_state_quantized =
-          GetTemporary(context, node, /*index=*/3);
-      TfLiteTensor* scaling_factors = GetTemporary(context, node, /*index=*/4);
-      TfLiteTensor* prod_scaling_factors =
-          GetTemporary(context, node, /*index=*/5);
-      TfLiteTensor* recovered_weights =
-          GetTemporary(context, node, /*index=*/6);
-      return EvalHybrid(
-          input, input_to_input_weights, input_to_forget_weights,
-          input_to_cell_weights, input_to_output_weights,
-          recurrent_to_input_weights, recurrent_to_forget_weights,
-          recurrent_to_cell_weights, recurrent_to_output_weights,
-          cell_to_input_weights, cell_to_forget_weights, cell_to_output_weights,
-          input_layer_norm_weights, forget_layer_norm_weights,
-          cell_layer_norm_weights, output_layer_norm_weights, input_gate_bias,
-          forget_gate_bias, cell_bias, output_gate_bias, projection_weights,
-          projection_bias, op_data->cell_clip, op_data->proj_clip,
-          op_data->activation, scratch_buffer, scaling_factors,
-          prod_scaling_factors, recovered_weights, input_quantized,
-          activation_state_quantized, cell_state_quantized, activation_state,
-          cell_state, output);
-    }
-    default:
-      context->ReportError(context, "Type %d is not currently supported.",
-                           input_to_output_weights->type);
-      return kTfLiteError;
-  }
-  return kTfLiteOk;
-}
-
-void Free(TfLiteContext* context, void* buffer) {
-  delete reinterpret_cast<OpData*>(buffer);
-}
-
-}  // namespace layer_norm_lstm
-
-TfLiteRegistration* Register_LAYER_NORM_LSTM() {
-  static TfLiteRegistration r = {layer_norm_lstm::Init, layer_norm_lstm::Free,
-                                 layer_norm_lstm::Prepare,
-                                 layer_norm_lstm::Eval};
-  return &r;
-}
-
-}  // namespace custom
-}  // namespace ops
-}  // namespace tflite
diff --git a/tensorflow/lite/kernels/layer_norm_lstm_test.cc b/tensorflow/lite/kernels/layer_norm_lstm_test.cc
deleted file mode 100644
index 5aed818f24..0000000000
--- a/tensorflow/lite/kernels/layer_norm_lstm_test.cc
+++ /dev/null
@@ -1,885 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-// Unit test for TFLite Layer Norm LSTM op.
-
-#include <memory>
-#include <vector>
-
-#include <gmock/gmock.h>
-#include <gtest/gtest.h>
-#include "flatbuffers/flexbuffers.h"  // TF:flatbuffers
-#include "tensorflow/lite/interpreter.h"
-#include "tensorflow/lite/kernels/register.h"
-#include "tensorflow/lite/kernels/test_util.h"
-#include "tensorflow/lite/model.h"
-
-namespace tflite {
-namespace ops {
-namespace custom {
-
-TfLiteRegistration* Register_LAYER_NORM_LSTM();
-
-namespace {
-
-using ::testing::ElementsAreArray;
-
-class LayerNormLSTMOpModel : public SingleOpModel {
- public:
-  LayerNormLSTMOpModel(int n_batch, int n_input, int n_cell, int n_output,
-                       bool use_cifg, bool use_peephole,
-                       bool use_projection_weights, bool use_projection_bias,
-                       float cell_clip, float proj_clip,
-                       const std::vector<std::vector<int>>& input_shapes,
-                       const TensorType& weight_type = TensorType_FLOAT32)
-      : n_batch_(n_batch),
-        n_input_(n_input),
-        n_cell_(n_cell),
-        n_output_(n_output) {
-    input_ = AddInput(TensorType_FLOAT32);
-
-    if (use_cifg) {
-      input_to_input_weights_ = AddNullInput();
-    } else {
-      input_to_input_weights_ = AddInput(weight_type);
-    }
-
-    input_to_forget_weights_ = AddInput(weight_type);
-    input_to_cell_weights_ = AddInput(weight_type);
-    input_to_output_weights_ = AddInput(weight_type);
-
-    if (use_cifg) {
-      recurrent_to_input_weights_ = AddNullInput();
-    } else {
-      recurrent_to_input_weights_ = AddInput(weight_type);
-    }
-
-    recurrent_to_forget_weights_ = AddInput(weight_type);
-    recurrent_to_cell_weights_ = AddInput(weight_type);
-    recurrent_to_output_weights_ = AddInput(weight_type);
-
-    if (use_peephole) {
-      if (use_cifg) {
-        cell_to_input_weights_ = AddNullInput();
-      } else {
-        cell_to_input_weights_ = AddInput(weight_type);
-      }
-      cell_to_forget_weights_ = AddInput(weight_type);
-      cell_to_output_weights_ = AddInput(weight_type);
-    } else {
-      cell_to_input_weights_ = AddNullInput();
-      cell_to_forget_weights_ = AddNullInput();
-      cell_to_output_weights_ = AddNullInput();
-    }
-
-    if (use_cifg) {
-      input_layer_norm_weights_ = AddNullInput();
-    } else {
-      input_layer_norm_weights_ = AddInput(TensorType_FLOAT32);
-    }
-    forget_layer_norm_weights_ = AddInput(TensorType_FLOAT32);
-    cell_layer_norm_weights_ = AddInput(TensorType_FLOAT32);
-    output_layer_norm_weights_ = AddInput(TensorType_FLOAT32);
-
-    if (use_cifg) {
-      input_gate_bias_ = AddNullInput();
-    } else {
-      input_gate_bias_ = AddInput(TensorType_FLOAT32);
-    }
-    forget_gate_bias_ = AddInput(TensorType_FLOAT32);
-    cell_bias_ = AddInput(TensorType_FLOAT32);
-    output_gate_bias_ = AddInput(TensorType_FLOAT32);
-
-    if (use_projection_weights) {
-      projection_weights_ = AddInput(weight_type);
-      if (use_projection_bias) {
-        projection_bias_ = AddInput(TensorType_FLOAT32);
-      } else {
-        projection_bias_ = AddNullInput();
-      }
-    } else {
-      projection_weights_ = AddNullInput();
-      projection_bias_ = AddNullInput();
-    }
-
-    // Adding the 2 state tensors.
-    output_state_ =
-        AddInput(TensorData{TensorType_FLOAT32, {n_output_ * n_batch_}}, true);
-    cell_state_ =
-        AddInput(TensorData{TensorType_FLOAT32, {n_cell_ * n_batch_}}, true);
-
-    output_ = AddOutput(TensorType_FLOAT32);
-
-    // Set up and pass in custom options using flexbuffer.
-    flexbuffers::Builder fbb;
-    fbb.Map([&]() {
-      fbb.Int("cell_clip", cell_clip);
-      fbb.Int("proj_clip", proj_clip);
-      fbb.String("fused_activation_function", "TANH");
-    });
-    fbb.Finish();
-    SetCustomOp("LAYER_NORM_LSTM", fbb.GetBuffer(), Register_LAYER_NORM_LSTM);
-    BuildInterpreter(input_shapes);
-  }
-
-  void SetInputToInputWeights(const std::vector<float>& f) {
-    PopulateTensor(input_to_input_weights_, f);
-  }
-
-  void SetInputToForgetWeights(const std::vector<float>& f) {
-    PopulateTensor(input_to_forget_weights_, f);
-  }
-
-  void SetInputToCellWeights(const std::vector<float>& f) {
-    PopulateTensor(input_to_cell_weights_, f);
-  }
-
-  void SetInputToOutputWeights(const std::vector<float>& f) {
-    PopulateTensor(input_to_output_weights_, f);
-  }
-
-  void SetRecurrentToInputWeights(const std::vector<float>& f) {
-    PopulateTensor(recurrent_to_input_weights_, f);
-  }
-
-  void SetRecurrentToForgetWeights(const std::vector<float>& f) {
-    PopulateTensor(recurrent_to_forget_weights_, f);
-  }
-
-  void SetRecurrentToCellWeights(const std::vector<float>& f) {
-    PopulateTensor(recurrent_to_cell_weights_, f);
-  }
-
-  void SetRecurrentToOutputWeights(const std::vector<float>& f) {
-    PopulateTensor(recurrent_to_output_weights_, f);
-  }
-
-  void SetCellToInputWeights(const std::vector<float>& f) {
-    PopulateTensor(cell_to_input_weights_, f);
-  }
-
-  void SetCellToForgetWeights(const std::vector<float>& f) {
-    PopulateTensor(cell_to_forget_weights_, f);
-  }
-
-  void SetCellToOutputWeights(const std::vector<float>& f) {
-    PopulateTensor(cell_to_output_weights_, f);
-  }
-
-  void SetInputLayerNormWeights(const std::vector<float>& f) {
-    PopulateTensor(input_layer_norm_weights_, f);
-  }
-
-  void SetForgetLayerNormWeights(const std::vector<float>& f) {
-    PopulateTensor(forget_layer_norm_weights_, f);
-  }
-
-  void SetCellLayerNormWeights(const std::vector<float>& f) {
-    PopulateTensor(cell_layer_norm_weights_, f);
-  }
-
-  void SetOutputLayerNormWeights(const std::vector<float>& f) {
-    PopulateTensor(output_layer_norm_weights_, f);
-  }
-
-  void SetInputGateBias(const std::vector<float>& f) {
-    PopulateTensor(input_gate_bias_, f);
-  }
-
-  void SetForgetGateBias(const std::vector<float>& f) {
-    PopulateTensor(forget_gate_bias_, f);
-  }
-
-  void SetCellBias(const std::vector<float>& f) {
-    PopulateTensor(cell_bias_, f);
-  }
-
-  void SetOutputGateBias(const std::vector<float>& f) {
-    PopulateTensor(output_gate_bias_, f);
-  }
-
-  void SetProjectionWeights(const std::vector<float>& f) {
-    PopulateTensor(projection_weights_, f);
-  }
-
-  void SetProjectionBias(const std::vector<float>& f) {
-    PopulateTensor(projection_bias_, f);
-  }
-
-  void SetInput(int offset, const float* begin, const float* end) {
-    PopulateTensor(input_, offset, const_cast<float*>(begin),
-                   const_cast<float*>(end));
-  }
-
-  std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
-
-  int num_inputs() { return n_input_; }
-  int num_outputs() { return n_output_; }
-  int num_cells() { return n_cell_; }
-  int num_batches() { return n_batch_; }
-
- protected:
-  int input_;
-  int input_to_input_weights_;
-  int input_to_forget_weights_;
-  int input_to_cell_weights_;
-  int input_to_output_weights_;
-
-  int recurrent_to_input_weights_;
-  int recurrent_to_forget_weights_;
-  int recurrent_to_cell_weights_;
-  int recurrent_to_output_weights_;
-
-  int cell_to_input_weights_;
-  int cell_to_forget_weights_;
-  int cell_to_output_weights_;
-
-  int input_layer_norm_weights_;
-  int forget_layer_norm_weights_;
-  int cell_layer_norm_weights_;
-  int output_layer_norm_weights_;
-
-  int input_gate_bias_;
-  int forget_gate_bias_;
-  int cell_bias_;
-  int output_gate_bias_;
-
-  int projection_weights_;
-  int projection_bias_;
-
-  int output_state_;
-  int cell_state_;
-
-  int output_;
-
-  int n_batch_;
-  int n_input_;
-  int n_cell_;
-  int n_output_;
-};
-
-class HybridLayerNormLSTMOpModel : public LayerNormLSTMOpModel {
- public:
-  HybridLayerNormLSTMOpModel(int n_batch, int n_input, int n_cell, int n_output,
-                             bool use_cifg, bool use_peephole,
-                             bool use_projection_weights,
-                             bool use_projection_bias, float cell_clip,
-                             float proj_clip,
-                             const std::vector<std::vector<int>>& input_shapes)
-      : LayerNormLSTMOpModel(n_batch, n_input, n_cell, n_output, use_cifg,
-                             use_peephole, use_projection_weights,
-                             use_projection_bias, cell_clip, proj_clip,
-                             input_shapes, TensorType_UINT8) {}
-
-  void SetInputToInputWeights(const std::vector<float>& f) {
-    SymmetricQuantizeAndPopulate(input_to_input_weights_, f);
-  }
-
-  void SetInputToForgetWeights(const std::vector<float>& f) {
-    SymmetricQuantizeAndPopulate(input_to_forget_weights_, f);
-  }
-
-  void SetInputToCellWeights(const std::vector<float>& f) {
-    SymmetricQuantizeAndPopulate(input_to_cell_weights_, f);
-  }
-
-  void SetInputToOutputWeights(const std::vector<float>& f) {
-    SymmetricQuantizeAndPopulate(input_to_output_weights_, f);
-  }
-
-  void SetRecurrentToInputWeights(const std::vector<float>& f) {
-    SymmetricQuantizeAndPopulate(recurrent_to_input_weights_, f);
-  }
-
-  void SetRecurrentToForgetWeights(const std::vector<float>& f) {
-    SymmetricQuantizeAndPopulate(recurrent_to_forget_weights_, f);
-  }
-
-  void SetRecurrentToCellWeights(const std::vector<float>& f) {
-    SymmetricQuantizeAndPopulate(recurrent_to_cell_weights_, f);
-  }
-
-  void SetRecurrentToOutputWeights(const std::vector<float>& f) {
-    SymmetricQuantizeAndPopulate(recurrent_to_output_weights_, f);
-  }
-
-  void SetCellToInputWeights(const std::vector<float>& f) {
-    SymmetricQuantizeAndPopulate(cell_to_input_weights_, f);
-  }
-
-  void SetCellToForgetWeights(const std::vector<float>& f) {
-    SymmetricQuantizeAndPopulate(cell_to_forget_weights_, f);
-  }
-
-  void SetCellToOutputWeights(const std::vector<float>& f) {
-    SymmetricQuantizeAndPopulate(cell_to_output_weights_, f);
-  }
-
-  void SetInputLayerNormWeights(const std::vector<float>& f) {
-    PopulateTensor(input_layer_norm_weights_, f);
-  }
-
-  void SetForgetLayerNormWeights(const std::vector<float>& f) {
-    PopulateTensor(forget_layer_norm_weights_, f);
-  }
-
-  void SetCellLayerNormWeights(const std::vector<float>& f) {
-    PopulateTensor(cell_layer_norm_weights_, f);
-  }
-
-  void SetOutputLayerNormWeights(const std::vector<float>& f) {
-    PopulateTensor(output_layer_norm_weights_, f);
-  }
-
-  void SetProjectionWeights(const std::vector<float>& f) {
-    SymmetricQuantizeAndPopulate(projection_weights_, f);
-  }
-};
-
-class BaseLayerNormLstmTest : public ::testing::Test {
- protected:
-  // Weights of the Layer Norm LSTM model. Some are optional.
-  std::vector<float> input_to_input_weights_;
-  std::vector<float> input_to_cell_weights_;
-  std::vector<float> input_to_forget_weights_;
-  std::vector<float> input_to_output_weights_;
-  std::vector<float> input_gate_bias_;
-  std::vector<float> cell_gate_bias_;
-  std::vector<float> forget_gate_bias_;
-  std::vector<float> output_gate_bias_;
-  std::vector<float> recurrent_to_input_weights_;
-  std::vector<float> recurrent_to_cell_weights_;
-  std::vector<float> recurrent_to_forget_weights_;
-  std::vector<float> recurrent_to_output_weights_;
-  std::vector<float> cell_to_input_weights_;
-  std::vector<float> cell_to_forget_weights_;
-  std::vector<float> cell_to_output_weights_;
-  std::vector<float> input_layer_norm_weights_;
-  std::vector<float> forget_layer_norm_weights_;
-  std::vector<float> cell_layer_norm_weights_;
-  std::vector<float> output_layer_norm_weights_;
-  std::vector<float> projection_weights_;
-
-  // Layer Norm LSTM input is stored as num_batch x num_inputs vector.
-  std::vector<std::vector<float>> layer_norm_lstm_input_;
-
-  // Compares output up to tolerance to the result of the layer_norm_lstm given
-  // the input.
-  void VerifyGoldens(const std::vector<std::vector<float>>& input,
-                     const std::vector<std::vector<float>>& output,
-                     LayerNormLSTMOpModel* layer_norm_lstm,
-                     float tolerance = 1e-5) {
-    const int num_batches = input.size();
-    EXPECT_GT(num_batches, 0);
-    const int num_inputs = layer_norm_lstm->num_inputs();
-    EXPECT_GT(num_inputs, 0);
-    const int input_sequence_size = input[0].size() / num_inputs;
-    EXPECT_GT(input_sequence_size, 0);
-    for (int i = 0; i < input_sequence_size; ++i) {
-      for (int b = 0; b < num_batches; ++b) {
-        const float* batch_start = input[b].data() + i * num_inputs;
-        const float* batch_end = batch_start + num_inputs;
-
-        layer_norm_lstm->SetInput(b * layer_norm_lstm->num_inputs(),
-                                  batch_start, batch_end);
-      }
-
-      layer_norm_lstm->Invoke();
-
-      const int num_outputs = layer_norm_lstm->num_outputs();
-      std::vector<float> expected;
-      for (int b = 0; b < num_batches; ++b) {
-        const float* golden_start_batch = output[b].data() + i * num_outputs;
-        const float* golden_end_batch = golden_start_batch + num_outputs;
-        expected.insert(expected.end(), golden_start_batch, golden_end_batch);
-      }
-      EXPECT_THAT(layer_norm_lstm->GetOutput(),
-                  ElementsAreArray(ArrayFloatNear(expected, tolerance)));
-    }
-  }
-};
-
-class NoCifgPeepholeProjectionNoClippingLayerNormLstmTest
-    : public BaseLayerNormLstmTest {
-  void SetUp() override {
-    input_to_input_weights_ = {0.5,  0.6,  0.7,  -0.8, -0.9, 0.1,  0.2,
-                               0.3,  -0.4, 0.5,  -0.8, 0.7,  -0.6, 0.5,
-                               -0.4, -0.5, -0.4, -0.3, -0.2, -0.1};
-
-    input_to_forget_weights_ = {-0.6, -0.1, 0.3,  0.2,  0.9,  -0.5, -0.2,
-                                -0.4, 0.3,  -0.8, -0.4, 0.3,  -0.5, -0.4,
-                                -0.6, 0.3,  -0.4, -0.6, -0.5, -0.5};
-
-    input_to_cell_weights_ = {-0.4, -0.3, -0.2, -0.1, -0.5, 0.5,  -0.2,
-                              -0.3, -0.2, -0.6, 0.6,  -0.1, -0.4, -0.3,
-                              -0.7, 0.7,  -0.9, -0.5, 0.8,  0.6};
-
-    input_to_output_weights_ = {-0.8, -0.4, -0.2, -0.9, -0.1, -0.7, 0.3,
-                                -0.3, -0.8, -0.2, 0.6,  -0.2, 0.4,  -0.7,
-                                -0.3, -0.5, 0.1,  0.5,  -0.6, -0.4};
-
-    input_gate_bias_ = {0.03, 0.15, 0.22, 0.38};
-
-    forget_gate_bias_ = {0.1, -0.3, -0.2, 0.1};
-
-    cell_gate_bias_ = {-0.05, 0.72, 0.25, 0.08};
-
-    output_gate_bias_ = {0.05, -0.01, 0.2, 0.1};
-
-    recurrent_to_input_weights_ = {-0.2, -0.3, 0.4,  0.1,  -0.5, 0.9,
-                                   -0.2, -0.3, -0.7, 0.05, -0.2, -0.6};
-
-    recurrent_to_cell_weights_ = {-0.3, 0.2, 0.1, -0.3, 0.8,  -0.08,
-                                  -0.2, 0.3, 0.8, -0.6, -0.1, 0.2};
-
-    recurrent_to_forget_weights_ = {-0.5, -0.3, -0.5, -0.2, 0.6, 0.4,
-                                    0.9,  0.3,  -0.1, 0.2,  0.5, 0.2};
-
-    recurrent_to_output_weights_ = {0.3,  -0.1, 0.1,  -0.2, -0.5, -0.7,
-                                    -0.2, -0.6, -0.1, -0.4, -0.7, -0.2};
-
-    cell_to_input_weights_ = {0.05, 0.1, 0.25, 0.15};
-
-    cell_to_forget_weights_ = {-0.02, -0.15, -0.25, -0.03};
-
-    cell_to_output_weights_ = {0.1, -0.1, -0.5, 0.05};
-
-    input_layer_norm_weights_ = {0.1, 0.2, 0.3, 0.5};
-    forget_layer_norm_weights_ = {0.2, 0.2, 0.4, 0.3};
-    cell_layer_norm_weights_ = {0.7, 0.2, 0.3, 0.8};
-    output_layer_norm_weights_ = {0.6, 0.2, 0.2, 0.5};
-
-    projection_weights_ = {-0.1, 0.2,  0.01, -0.2, 0.1,  0.5,
-                           0.3,  0.08, 0.07, 0.2,  -0.4, 0.2};
-
-    layer_norm_lstm_input_ = {
-        {// Batch0: 3 (input_sequence_size) * 5 (n_input)
-         0.7, 0.8, 0.1, 0.2, 0.3,   // seq 0
-         0.8, 0.1, 0.2, 0.4, 0.5,   // seq 1
-         0.2, 0.7, 0.7, 0.1, 0.7},  // seq 2
-
-        {// Batch1: 3 (input_sequence_size) * 5 (n_input)
-         0.3, 0.2, 0.9, 0.8, 0.1,   // seq 0
-         0.1, 0.5, 0.2, 0.4, 0.2,   // seq 1
-         0.6, 0.9, 0.2, 0.5, 0.7},  // seq 2
-    };
-  }
-};
-
-TEST_F(NoCifgPeepholeProjectionNoClippingLayerNormLstmTest,
-       LayerNormLstmBlackBoxTest) {
-  const int n_batch = 2;
-  const int n_input = 5;
-  const int n_cell = 4;
-  const int n_output = 3;
-  const float ceil_clip = 0.0;
-  const float proj_clip = 0.0;
-
-  LayerNormLSTMOpModel layer_norm_lstm(
-      n_batch, n_input, n_cell, n_output,
-      /*use_cifg=*/false, /*use_peephole=*/true,
-      /*use_projection_weights=*/true,
-      /*use_projection_bias=*/false, ceil_clip, proj_clip,
-      {
-          {n_batch, n_input},  // input tensor
-
-          {n_cell, n_input},  // input_to_input_weight tensor
-          {n_cell, n_input},  // input_to_forget_weight tensor
-          {n_cell, n_input},  // input_to_cell_weight tensor
-          {n_cell, n_input},  // input_to_output_weight tensor
-
-          {n_cell, n_output},  // recurrent_to_input_weight tensor
-          {n_cell, n_output},  // recurrent_to_forget_weight tensor
-          {n_cell, n_output},  // recurrent_to_cell_weight tensor
-          {n_cell, n_output},  // recurrent_to_output_weight tensor
-
-          {n_cell},  // cell_to_input_weight tensor
-          {n_cell},  // cell_to_forget_weight tensor
-          {n_cell},  // cell_to_output_weight tensor
-
-          {n_cell},  // input_layer_norm_weight tensor
-          {n_cell},  // forget_layer_norm_weight tensor
-          {n_cell},  // cell_layer_norm_weight tensor
-          {n_cell},  // output_layer_norm_weight tensor
-
-          {n_cell},  // input_gate_bias tensor
-          {n_cell},  // forget_gate_bias tensor
-          {n_cell},  // cell_bias tensor
-          {n_cell},  // output_gate_bias tensor
-
-          {n_output, n_cell},  // projection_weight tensor
-          {0},                 // projection_bias tensor
-      });
-
-  layer_norm_lstm.SetInputToInputWeights(input_to_input_weights_);
-  layer_norm_lstm.SetInputToCellWeights(input_to_cell_weights_);
-  layer_norm_lstm.SetInputToForgetWeights(input_to_forget_weights_);
-  layer_norm_lstm.SetInputToOutputWeights(input_to_output_weights_);
-
-  layer_norm_lstm.SetInputGateBias(input_gate_bias_);
-  layer_norm_lstm.SetCellBias(cell_gate_bias_);
-  layer_norm_lstm.SetForgetGateBias(forget_gate_bias_);
-  layer_norm_lstm.SetOutputGateBias(output_gate_bias_);
-
-  layer_norm_lstm.SetRecurrentToInputWeights(recurrent_to_input_weights_);
-  layer_norm_lstm.SetRecurrentToCellWeights(recurrent_to_cell_weights_);
-  layer_norm_lstm.SetRecurrentToForgetWeights(recurrent_to_forget_weights_);
-  layer_norm_lstm.SetRecurrentToOutputWeights(recurrent_to_output_weights_);
-
-  layer_norm_lstm.SetCellToInputWeights(cell_to_input_weights_);
-  layer_norm_lstm.SetCellToForgetWeights(cell_to_forget_weights_);
-  layer_norm_lstm.SetCellToOutputWeights(cell_to_output_weights_);
-
-  layer_norm_lstm.SetInputLayerNormWeights(input_layer_norm_weights_);
-  layer_norm_lstm.SetForgetLayerNormWeights(forget_layer_norm_weights_);
-  layer_norm_lstm.SetCellLayerNormWeights(cell_layer_norm_weights_);
-  layer_norm_lstm.SetOutputLayerNormWeights(output_layer_norm_weights_);
-
-  layer_norm_lstm.SetProjectionWeights(projection_weights_);
-
-  // Verify the final output.
-  const std::vector<std::vector<float>> layer_norm_lstm_golden_output = {
-      {
-          // Batch0: 3 (input_sequence_size) * 3 (n_output)
-          0.0244077, 0.128027, -0.00170918,  // seq 0
-          0.0137642, 0.140751, 0.0395835,    // seq 1
-          -0.00459231, 0.155278, 0.0837377,  // seq 2
-      },
-      {
-          // Batch1: 3 (input_sequence_size) * 3 (n_output)
-          -0.00692428, 0.0848741, 0.063445,  // seq 0
-          -0.00403912, 0.139963, 0.072681,   // seq 1
-          0.00752706, 0.161903, 0.0561371,   // seq 2
-      }};
-
-  VerifyGoldens(layer_norm_lstm_input_, layer_norm_lstm_golden_output,
-                &layer_norm_lstm);
-}
-
-TEST_F(NoCifgPeepholeProjectionNoClippingLayerNormLstmTest,
-       HybridLayerNormLstmBlackBoxTest) {
-  const int n_batch = 2;
-  const int n_input = 5;
-  const int n_cell = 4;
-  const int n_output = 3;
-  const float ceil_clip = 0.0;
-  const float proj_clip = 0.0;
-
-  HybridLayerNormLSTMOpModel layer_norm_lstm(
-      n_batch, n_input, n_cell, n_output,
-      /*use_cifg=*/false, /*use_peephole=*/true,
-      /*use_projection_weights=*/true,
-      /*use_projection_bias=*/false, ceil_clip, proj_clip,
-      {
-          {n_batch, n_input},  // input tensor
-
-          {n_cell, n_input},  // input_to_input_weight tensor
-          {n_cell, n_input},  // input_to_forget_weight tensor
-          {n_cell, n_input},  // input_to_cell_weight tensor
-          {n_cell, n_input},  // input_to_output_weight tensor
-
-          {n_cell, n_output},  // recurrent_to_input_weight tensor
-          {n_cell, n_output},  // recurrent_to_forget_weight tensor
-          {n_cell, n_output},  // recurrent_to_cell_weight tensor
-          {n_cell, n_output},  // recurrent_to_output_weight tensor
-
-          {n_cell},  // cell_to_input_weight tensor
-          {n_cell},  // cell_to_forget_weight tensor
-          {n_cell},  // cell_to_output_weight tensor
-
-          {n_cell},  // input_layer_norm_weight tensor
-          {n_cell},  // forget_layer_norm_weight tensor
-          {n_cell},  // cell_layer_norm_weight tensor
-          {n_cell},  // output_layer_norm_weight tensor
-
-          {n_cell},  // input_gate_bias tensor
-          {n_cell},  // forget_gate_bias tensor
-          {n_cell},  // cell_bias tensor
-          {n_cell},  // output_gate_bias tensor
-
-          {n_output, n_cell},  // projection_weight tensor
-          {0},                 // projection_bias tensor
-      });
-
-  layer_norm_lstm.SetInputToInputWeights(input_to_input_weights_);
-  layer_norm_lstm.SetInputToCellWeights(input_to_cell_weights_);
-  layer_norm_lstm.SetInputToForgetWeights(input_to_forget_weights_);
-  layer_norm_lstm.SetInputToOutputWeights(input_to_output_weights_);
-
-  layer_norm_lstm.SetInputGateBias(input_gate_bias_);
-  layer_norm_lstm.SetCellBias(cell_gate_bias_);
-  layer_norm_lstm.SetForgetGateBias(forget_gate_bias_);
-  layer_norm_lstm.SetOutputGateBias(output_gate_bias_);
-
-  layer_norm_lstm.SetRecurrentToInputWeights(recurrent_to_input_weights_);
-  layer_norm_lstm.SetRecurrentToCellWeights(recurrent_to_cell_weights_);
-  layer_norm_lstm.SetRecurrentToForgetWeights(recurrent_to_forget_weights_);
-  layer_norm_lstm.SetRecurrentToOutputWeights(recurrent_to_output_weights_);
-
-  layer_norm_lstm.SetCellToInputWeights(cell_to_input_weights_);
-  layer_norm_lstm.SetCellToForgetWeights(cell_to_forget_weights_);
-  layer_norm_lstm.SetCellToOutputWeights(cell_to_output_weights_);
-
-  layer_norm_lstm.SetInputLayerNormWeights(input_layer_norm_weights_);
-  layer_norm_lstm.SetForgetLayerNormWeights(forget_layer_norm_weights_);
-  layer_norm_lstm.SetCellLayerNormWeights(cell_layer_norm_weights_);
-  layer_norm_lstm.SetOutputLayerNormWeights(output_layer_norm_weights_);
-
-  layer_norm_lstm.SetProjectionWeights(projection_weights_);
-
-  const std::vector<std::vector<float>> layer_norm_lstm_golden_output = {
-      {
-          // Batch0: 3 (input_sequence_size) * 3 (n_output)
-          0.0244576, 0.127847, -0.00181765,  // seq 0
-          0.0137518, 0.140892, 0.0402234,    // seq 1
-          -0.0048839, 0.155096, 0.0840309,   // seq 2
-      },
-      {
-          // Batch1: 3 (input_sequence_size) * 3 (n_output)
-          -0.00728636, 0.0843957, 0.0634786,  // seq 0
-          -0.00448382, 0.139278, 0.0737372,   // seq 1
-          0.00734616, 0.161793, 0.0560238,    // seq 2
-      }};
-
-  VerifyGoldens(layer_norm_lstm_input_, layer_norm_lstm_golden_output,
-                &layer_norm_lstm);
-}
-
-class CifgPeepholeProjectionNoClippingLayerNormLstmTest
-    : public BaseLayerNormLstmTest {
-  void SetUp() override {
-    input_to_forget_weights_ = {-0.6, -0.1, 0.3,  0.2,  0.9,  -0.5, -0.2,
-                                -0.4, 0.3,  -0.8, -0.4, 0.3,  -0.5, -0.4,
-                                -0.6, 0.3,  -0.4, -0.6, -0.5, -0.5};
-    input_to_cell_weights_ = {-0.4, -0.3, -0.2, -0.1, -0.5, 0.5,  -0.2,
-                              -0.3, -0.2, -0.6, 0.6,  -0.1, -0.4, -0.3,
-                              -0.7, 0.7,  -0.9, -0.5, 0.8,  0.6};
-    input_to_output_weights_ = {-0.8, -0.4, -0.2, -0.9, -0.1, -0.7, 0.3,
-                                -0.3, -0.8, -0.2, 0.6,  -0.2, 0.4,  -0.7,
-                                -0.3, -0.5, 0.1,  0.5,  -0.6, -0.4};
-
-    forget_gate_bias_ = {0.1, -0.3, -0.2, 0.1};
-    cell_gate_bias_ = {-0.05, 0.72, 0.25, 0.08};
-    output_gate_bias_ = {0.05, -0.01, 0.2, 0.1};
-
-    recurrent_to_cell_weights_ = {-0.3, 0.2, 0.1, -0.3, 0.8,  -0.08,
-                                  -0.2, 0.3, 0.8, -0.6, -0.1, 0.2};
-    recurrent_to_forget_weights_ = {-0.5, -0.3, -0.5, -0.2, 0.6, 0.4,
-                                    0.9,  0.3,  -0.1, 0.2,  0.5, 0.2};
-    recurrent_to_output_weights_ = {0.3,  -0.1, 0.1,  -0.2, -0.5, -0.7,
-                                    -0.2, -0.6, -0.1, -0.4, -0.7, -0.2};
-
-    cell_to_forget_weights_ = {-0.02, -0.15, -0.25, -0.03};
-    cell_to_output_weights_ = {0.1, -0.1, -0.5, 0.05};
-
-    forget_layer_norm_weights_ = {0.2, 0.2, 0.4, 0.3};
-    cell_layer_norm_weights_ = {0.7, 0.2, 0.3, 0.8};
-    output_layer_norm_weights_ = {0.6, 0.2, 0.2, 0.5};
-    projection_weights_ = {-0.1, 0.2,  0.01, -0.2, 0.1,  0.5,
-                           0.3,  0.08, 0.07, 0.2,  -0.4, 0.2};
-
-    layer_norm_lstm_input_ = {
-        {// Batch0: 3 (input_sequence_size) * 5 (n_input)
-         0.7, 0.8, 0.1, 0.2, 0.3,   // seq 0
-         0.8, 0.1, 0.2, 0.4, 0.5,   // seq 1
-         0.2, 0.7, 0.7, 0.1, 0.7},  // seq 2
-
-        {// Batch1: 3 (input_sequence_size) * 5 (n_input)
-         0.3, 0.2, 0.9, 0.8, 0.1,   // seq 0
-         0.1, 0.5, 0.2, 0.4, 0.2,   // seq 1
-         0.6, 0.9, 0.2, 0.5, 0.7},  // seq 2
-    };
-  }
-};
-
-TEST_F(CifgPeepholeProjectionNoClippingLayerNormLstmTest,
-       LayerNormLstmBlackBoxTest) {
-  const int n_batch = 2;
-  const int n_input = 5;
-  const int n_cell = 4;
-  const int n_output = 3;
-  const float ceil_clip = 0.0;
-  const float proj_clip = 0.0;
-
-  LayerNormLSTMOpModel layer_norm_lstm(
-      n_batch, n_input, n_cell, n_output,
-      /*use_cifg=*/true, /*use_peephole=*/true,
-      /*use_projection_weights=*/true,
-      /*use_projection_bias=*/false, ceil_clip, proj_clip,
-      {
-          {n_batch, n_input},  // input tensor
-
-          {0, 0},             // input_to_input_weight tensor
-          {n_cell, n_input},  // input_to_forget_weight tensor
-          {n_cell, n_input},  // input_to_cell_weight tensor
-          {n_cell, n_input},  // input_to_output_weight tensor
-
-          {0, 0},              // recurrent_to_input_weight tensor
-          {n_cell, n_output},  // recurrent_to_forget_weight tensor
-          {n_cell, n_output},  // recurrent_to_cell_weight tensor
-          {n_cell, n_output},  // recurrent_to_output_weight tensor
-
-          {0},       // cell_to_input_weight tensor
-          {n_cell},  // cell_to_forget_weight tensor
-          {n_cell},  // cell_to_output_weight tensor
-
-          {0},       // input_layer_norm_weight tensor
-          {n_cell},  // forget_layer_norm_weight tensor
-          {n_cell},  // cell_layer_norm_weight tensor
-          {n_cell},  // output_layer_norm_weight tensor
-
-          {0},       // input_gate_bias tensor
-          {n_cell},  // forget_gate_bias tensor
-          {n_cell},  // cell_bias tensor
-          {n_cell},  // output_gate_bias tensor
-
-          {n_output, n_cell},  // projection_weight tensor
-          {0},                 // projection_bias tensor
-      });
-
-  layer_norm_lstm.SetInputToCellWeights(input_to_cell_weights_);
-  layer_norm_lstm.SetInputToForgetWeights(input_to_forget_weights_);
-  layer_norm_lstm.SetInputToOutputWeights(input_to_output_weights_);
-
-  layer_norm_lstm.SetCellBias(cell_gate_bias_);
-  layer_norm_lstm.SetForgetGateBias(forget_gate_bias_);
-  layer_norm_lstm.SetOutputGateBias(output_gate_bias_);
-
-  layer_norm_lstm.SetRecurrentToCellWeights(recurrent_to_cell_weights_);
-  layer_norm_lstm.SetRecurrentToForgetWeights(recurrent_to_forget_weights_);
-  layer_norm_lstm.SetRecurrentToOutputWeights(recurrent_to_output_weights_);
-
-  layer_norm_lstm.SetCellToForgetWeights(cell_to_forget_weights_);
-  layer_norm_lstm.SetCellToOutputWeights(cell_to_output_weights_);
-
-  layer_norm_lstm.SetForgetLayerNormWeights(forget_layer_norm_weights_);
-  layer_norm_lstm.SetCellLayerNormWeights(cell_layer_norm_weights_);
-  layer_norm_lstm.SetOutputLayerNormWeights(output_layer_norm_weights_);
-
-  layer_norm_lstm.SetProjectionWeights(projection_weights_);
-
-  // Verify the final output.
-  const std::vector<std::vector<float>> layer_norm_lstm_golden_output = {
-      {
-          // Batch0: 3 (input_sequence_size) * 3 (n_output)
-          0.02129706, 0.140816242, 0.0112733059,     // seq 0
-          0.0132302344, 0.152308047, 0.0346313119,   // seq 1
-          -0.0123688057, 0.165790111, 0.0893077999,  // seq 2
-      },
-      {
-          // Batch1: 3 (input_sequence_size) * 3 (n_output)
-          -0.0226350538, 0.0916948169, 0.0769175813,  // seq 0
-          -0.0269966982, 0.149707705, 0.094149217,    // seq 1
-          -0.0103429332, 0.173016444, 0.0720508844,   // seq 2
-      }};
-
-  VerifyGoldens(layer_norm_lstm_input_, layer_norm_lstm_golden_output,
-                &layer_norm_lstm);
-}
-
-TEST_F(CifgPeepholeProjectionNoClippingLayerNormLstmTest,
-       HybridLayerNormLstmBlackBoxTest) {
-  const int n_batch = 2;
-  const int n_input = 5;
-  const int n_cell = 4;
-  const int n_output = 3;
-  const float ceil_clip = 0.0;
-  const float proj_clip = 0.0;
-
-  HybridLayerNormLSTMOpModel layer_norm_lstm(
-      n_batch, n_input, n_cell, n_output,
-      /*use_cifg=*/true, /*use_peephole=*/true,
-      /*use_projection_weights=*/true,
-      /*use_projection_bias=*/false, ceil_clip, proj_clip,
-      {
-          {n_batch, n_input},  // input tensor
-
-          {0, 0},             // input_to_input_weight tensor
-          {n_cell, n_input},  // input_to_forget_weight tensor
-          {n_cell, n_input},  // input_to_cell_weight tensor
-          {n_cell, n_input},  // input_to_output_weight tensor
-
-          {0, 0},              // recurrent_to_input_weight tensor
-          {n_cell, n_output},  // recurrent_to_forget_weight tensor
-          {n_cell, n_output},  // recurrent_to_cell_weight tensor
-          {n_cell, n_output},  // recurrent_to_output_weight tensor
-
-          {0},       // cell_to_input_weight tensor
-          {n_cell},  // cell_to_forget_weight tensor
-          {n_cell},  // cell_to_output_weight tensor
-
-          {0},       // input_layer_norm_weight tensor
-          {n_cell},  // forget_layer_norm_weight tensor
-          {n_cell},  // cell_layer_norm_weight tensor
-          {n_cell},  // output_layer_norm_weight tensor
-
-          {0},       // input_gate_bias tensor
-          {n_cell},  // forget_gate_bias tensor
-          {n_cell},  // cell_bias tensor
-          {n_cell},  // output_gate_bias tensor
-
-          {n_output, n_cell},  // projection_weight tensor
-          {0},                 // projection_bias tensor
-      });
-
-  layer_norm_lstm.SetInputToCellWeights(input_to_cell_weights_);
-  layer_norm_lstm.SetInputToForgetWeights(input_to_forget_weights_);
-  layer_norm_lstm.SetInputToOutputWeights(input_to_output_weights_);
-
-  layer_norm_lstm.SetCellBias(cell_gate_bias_);
-  layer_norm_lstm.SetForgetGateBias(forget_gate_bias_);
-  layer_norm_lstm.SetOutputGateBias(output_gate_bias_);
-
-  layer_norm_lstm.SetRecurrentToCellWeights(recurrent_to_cell_weights_);
-  layer_norm_lstm.SetRecurrentToForgetWeights(recurrent_to_forget_weights_);
-  layer_norm_lstm.SetRecurrentToOutputWeights(recurrent_to_output_weights_);
-
-  layer_norm_lstm.SetCellToForgetWeights(cell_to_forget_weights_);
-  layer_norm_lstm.SetCellToOutputWeights(cell_to_output_weights_);
-
-  layer_norm_lstm.SetForgetLayerNormWeights(forget_layer_norm_weights_);
-  layer_norm_lstm.SetCellLayerNormWeights(cell_layer_norm_weights_);
-  layer_norm_lstm.SetOutputLayerNormWeights(output_layer_norm_weights_);
-
-  layer_norm_lstm.SetProjectionWeights(projection_weights_);
-
-  // Verify the final output.
-  const std::vector<std::vector<float>> layer_norm_lstm_golden_output = {
-      {
-          // Batch0: 3 (input_sequence_size) * 3 (n_output)
-          0.0212250091, 0.140474007, 0.0115012666,   // seq 0
-          0.0130806509, 0.152660668, 0.0347516984,   // seq 1
-          -0.0124010444, 0.166042402, 0.0898982584,  // seq 2
-      },
-      {
-          // Batch1: 3 (input_sequence_size) * 3 (n_output)
-          -0.0228835996, 0.0917588323, 0.0778886303,  // seq 0
-          -0.0275101066, 0.148769245, 0.0938384682,   // seq 1
-          -0.0103605557, 0.172605693, 0.0728750974,   // seq 2
-      }};
-
-  VerifyGoldens(layer_norm_lstm_input_, layer_norm_lstm_golden_output,
-                &layer_norm_lstm);
-}
-
-}  // namespace
-}  // namespace custom
-}  // namespace ops
-}  // namespace tflite
-
-int main(int argc, char** argv) {
-  ::tflite::LogToStderr();
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}
diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index 11861a8578..62e17160a5 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -22,7 +22,6 @@ namespace ops {
 namespace custom {
 
 TfLiteRegistration* Register_AUDIO_SPECTROGRAM();
-TfLiteRegistration* Register_LAYER_NORM_LSTM();
 TfLiteRegistration* Register_MFCC();
 TfLiteRegistration* Register_DETECTION_POSTPROCESS();
 TfLiteRegistration* Register_RELU_1();
@@ -301,7 +300,6 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddCustom("Mfcc", tflite::ops::custom::Register_MFCC());
   AddCustom("AudioSpectrogram",
             tflite::ops::custom::Register_AUDIO_SPECTROGRAM());
-  AddCustom("LayerNormLstm", tflite::ops::custom::Register_LAYER_NORM_LSTM());
   AddCustom("Relu1", tflite::ops::custom::Register_RELU_1());
   AddCustom("TFLite_Detection_PostProcess",
             tflite::ops::custom::Register_DETECTION_POSTPROCESS());
diff --git a/tensorflow/lite/kernels/register_ref.cc b/tensorflow/lite/kernels/register_ref.cc
index 6840ea39bf..f833a6a9d1 100644
--- a/tensorflow/lite/kernels/register_ref.cc
+++ b/tensorflow/lite/kernels/register_ref.cc
@@ -22,7 +22,6 @@ namespace ops {
 namespace custom {
 
 TfLiteRegistration* Register_AUDIO_SPECTROGRAM();
-TfLiteRegistration* Register_LAYER_NORM_LSTM();
 TfLiteRegistration* Register_MFCC();
 TfLiteRegistration* Register_DETECTION_POSTPROCESS();
 TfLiteRegistration* Register_RELU_1();
@@ -286,7 +285,6 @@ BuiltinRefOpResolver::BuiltinRefOpResolver() {
   AddCustom("Mfcc", tflite::ops::custom::Register_MFCC());
   AddCustom("AudioSpectrogram",
             tflite::ops::custom::Register_AUDIO_SPECTROGRAM());
-  AddCustom("LayerNormLstm", tflite::ops::custom::Register_LAYER_NORM_LSTM());
   AddCustom("Relu1", tflite::ops::custom::Register_RELU_1());
   AddCustom("TFLite_Detection_PostProcess",
             tflite::ops::custom::Register_DETECTION_POSTPROCESS());
-- 
GitLab


From 522e0f7a5513ef8ea236742a94055f74e27d6453 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 1 Feb 2019 14:58:58 -0800
Subject: [PATCH 1786/2345] n/a

PiperOrigin-RevId: 232043298
---
 tensorflow/examples/speech_commands/BUILD     | 28 ++++++++++
 .../lite/experimental/examples/lstm/BUILD     |  2 +-
 tensorflow/lite/python/BUILD                  |  8 +++
 tensorflow/python/debug/BUILD                 | 54 ++++++++++++++++---
 tensorflow/python/tools/BUILD                 | 52 +++++++++++++-----
 tensorflow/tools/compatibility/BUILD          | 11 +++-
 tensorflow/tools/dist_test/server/BUILD       | 10 +++-
 7 files changed, 142 insertions(+), 23 deletions(-)

diff --git a/tensorflow/examples/speech_commands/BUILD b/tensorflow/examples/speech_commands/BUILD
index 960dd110b6..ca044e57ed 100644
--- a/tensorflow/examples/speech_commands/BUILD
+++ b/tensorflow/examples/speech_commands/BUILD
@@ -88,6 +88,13 @@ tf_py_test(
 
 py_binary(
     name = "freeze",
+    srcs = ["freeze.py"],
+    srcs_version = "PY2AND3",
+    deps = [":freeze_lib"],
+)
+
+py_library(
+    name = "freeze_lib",
     srcs = [
         "freeze.py",
     ],
@@ -113,6 +120,13 @@ tf_py_test(
 
 py_binary(
     name = "wav_to_features",
+    srcs = ["wav_to_features.py"],
+    srcs_version = "PY2AND3",
+    deps = [":wav_to_features_lib"],
+)
+
+py_library(
+    name = "wav_to_features_lib",
     srcs = [
         "wav_to_features.py",
     ],
@@ -138,6 +152,13 @@ tf_py_test(
 
 py_binary(
     name = "generate_streaming_test_wav",
+    srcs = ["generate_streaming_test_wav.py"],
+    srcs_version = "PY2AND3",
+    deps = [":generate_streaming_test_wav_lib"],
+)
+
+py_library(
+    name = "generate_streaming_test_wav_lib",
     srcs = [
         "generate_streaming_test_wav.py",
     ],
@@ -178,6 +199,13 @@ tf_cc_binary(
 
 py_binary(
     name = "label_wav",
+    srcs = ["label_wav.py"],
+    srcs_version = "PY2AND3",
+    deps = [":label_wav_lib"],
+)
+
+py_library(
+    name = "label_wav_lib",
     srcs = [
         "label_wav.py",
     ],
diff --git a/tensorflow/lite/experimental/examples/lstm/BUILD b/tensorflow/lite/experimental/examples/lstm/BUILD
index ac74d73d4d..827b10449b 100644
--- a/tensorflow/lite/experimental/examples/lstm/BUILD
+++ b/tensorflow/lite/experimental/examples/lstm/BUILD
@@ -46,7 +46,7 @@ py_test(
         "//tensorflow/lite/python:lite",
         "//tensorflow/python:framework",
         "//tensorflow/python:platform",
-        "//tensorflow/python/tools:optimize_for_inference",
+        "//tensorflow/python/tools:optimize_for_inference_lib",
         "//third_party/py/numpy",
         "@six_archive//:six",
     ],
diff --git a/tensorflow/lite/python/BUILD b/tensorflow/lite/python/BUILD
index a8c8c9f03b..aa0ffe90e1 100644
--- a/tensorflow/lite/python/BUILD
+++ b/tensorflow/lite/python/BUILD
@@ -40,6 +40,14 @@ py_binary(
     srcs = ["tflite_convert.py"],
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
+    deps = [":tflite_convert_lib"],
+)
+
+py_library(
+    name = "tflite_convert_lib",
+    srcs = ["tflite_convert.py"],
+    srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
     deps = [
         ":lite",
     ],
diff --git a/tensorflow/python/debug/BUILD b/tensorflow/python/debug/BUILD
index 27a700f813..4634f505f6 100644
--- a/tensorflow/python/debug/BUILD
+++ b/tensorflow/python/debug/BUILD
@@ -47,7 +47,7 @@ py_library(
         ":cli_test_utils",
         ":debug_py",
         ":grpc_debug_test_server",
-        ":offline_analyzer",
+        ":offline_analyzer_lib",
         ":session_debug_testlib",
         ":source_remote",
     ] + if_not_windows([
@@ -393,6 +393,13 @@ py_binary(
     name = "offline_analyzer",
     srcs = ["cli/offline_analyzer.py"],
     srcs_version = "PY2AND3",
+    deps = [":offline_analyzer_lib"],
+)
+
+py_library(
+    name = "offline_analyzer_lib",
+    srcs = ["cli/offline_analyzer.py"],
+    srcs_version = "PY2AND3",
     deps = [
         ":analyzer_cli",
         ":debug_data",
@@ -404,12 +411,12 @@ py_binary(
 py_library(
     name = "debug_examples",
     deps = [
-        ":debug_errors",
-        ":debug_fibonacci",
-        ":debug_keras",
+        ":debug_errors_lib",
+        ":debug_fibonacci_lib",
+        ":debug_keras_lib",
     ] + if_not_v2([
-        ":debug_mnist",
-        ":debug_tflearn_iris",
+        ":debug_mnist_lib",
+        ":debug_tflearn_iris_lib",
     ]),
 )
 
@@ -417,6 +424,13 @@ py_binary(
     name = "debug_fibonacci",
     srcs = ["examples/debug_fibonacci.py"],
     srcs_version = "PY2AND3",
+    deps = [":debug_fibonacci_lib"],
+)
+
+py_library(
+    name = "debug_fibonacci_lib",
+    srcs = ["examples/debug_fibonacci.py"],
+    srcs_version = "PY2AND3",
     deps = [
         ":debug_py",
         "//tensorflow:tensorflow_py",
@@ -429,6 +443,13 @@ py_binary(
     name = "debug_errors",
     srcs = ["examples/debug_errors.py"],
     srcs_version = "PY2AND3",
+    deps = [":debug_errors_lib"],
+)
+
+py_library(
+    name = "debug_errors_lib",
+    srcs = ["examples/debug_errors.py"],
+    srcs_version = "PY2AND3",
     deps = [
         ":debug_py",
         "//tensorflow:tensorflow_py",
@@ -440,6 +461,13 @@ py_binary(
     name = "debug_mnist",
     srcs = ["examples/debug_mnist.py"],
     srcs_version = "PY2AND3",
+    deps = [":debug_mnist_lib"],
+)
+
+py_library(
+    name = "debug_mnist_lib",
+    srcs = ["examples/debug_mnist.py"],
+    srcs_version = "PY2AND3",
     deps = [
         ":debug_py",
         "//tensorflow:tensorflow_py",
@@ -451,6 +479,13 @@ py_binary(
     name = "debug_tflearn_iris",
     srcs = ["examples/debug_tflearn_iris.py"],
     srcs_version = "PY2AND3",
+    deps = [":debug_tflearn_iris_lib"],
+)
+
+py_library(
+    name = "debug_tflearn_iris_lib",
+    srcs = ["examples/debug_tflearn_iris.py"],
+    srcs_version = "PY2AND3",
     deps = [
         ":debug_py",
         "//tensorflow:tensorflow_py",
@@ -462,6 +497,13 @@ py_binary(
     name = "debug_keras",
     srcs = ["examples/debug_keras.py"],
     srcs_version = "PY2AND3",
+    deps = [":debug_keras_lib"],
+)
+
+py_library(
+    name = "debug_keras_lib",
+    srcs = ["examples/debug_keras.py"],
+    srcs_version = "PY2AND3",
     deps = [
         ":debug_py",
         "//tensorflow:tensorflow_py",
diff --git a/tensorflow/python/tools/BUILD b/tensorflow/python/tools/BUILD
index f1a911eb48..b8ad31e88b 100644
--- a/tensorflow/python/tools/BUILD
+++ b/tensorflow/python/tools/BUILD
@@ -14,23 +14,20 @@ load("//tensorflow:tensorflow.bzl", "py_binary")
 py_library(
     name = "tools_pip",
     deps = [
-        ":freeze_graph",
-        ":import_pb_to_tensorboard",
-        ":inspect_checkpoint",
-        ":optimize_for_inference",
-        ":print_selective_registration_header",
-        ":saved_model_cli",
+        ":freeze_graph_lib",
+        ":import_pb_to_tensorboard_lib",
+        ":inspect_checkpoint_lib",
+        ":optimize_for_inference_lib",
+        ":print_selective_registration_header_lib",
+        ":saved_model_cli_lib",
         ":saved_model_utils",
-        ":strip_unused",
+        ":strip_unused_lib",
         # The following py_library are needed because
         # py_binary may not depend on them when --define=no_tensorflow_py_deps=true
         # is specified. See https://github.com/tensorflow/tensorflow/issues/22390
-        ":freeze_graph_lib",
-        ":optimize_for_inference_lib",
         ":selective_registration_header_lib",
-        ":strip_unused_lib",
         # Include the TF upgrade script to users can run it directly after install TF
-        "//tensorflow/tools/compatibility:tf_upgrade_v2",
+        "//tensorflow/tools/compatibility:tf_upgrade_v2_lib",
     ],
 )
 
@@ -86,6 +83,13 @@ py_binary(
     name = "import_pb_to_tensorboard",
     srcs = ["import_pb_to_tensorboard.py"],
     srcs_version = "PY2AND3",
+    deps = [":import_pb_to_tensorboard_lib"],
+)
+
+py_library(
+    name = "import_pb_to_tensorboard_lib",
+    srcs = ["import_pb_to_tensorboard.py"],
+    srcs_version = "PY2AND3",
     deps = [
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python",
@@ -103,7 +107,7 @@ py_test(
     srcs = ["freeze_graph_test.py"],
     srcs_version = "PY2AND3",
     deps = [
-        ":freeze_graph",
+        ":freeze_graph_lib",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:client",
         "//tensorflow/python:client_testlib",
@@ -120,6 +124,13 @@ py_binary(
     name = "inspect_checkpoint",
     srcs = ["inspect_checkpoint.py"],
     srcs_version = "PY2AND3",
+    deps = [":inspect_checkpoint_lib"],
+)
+
+py_library(
+    name = "inspect_checkpoint_lib",
+    srcs = ["inspect_checkpoint.py"],
+    srcs_version = "PY2AND3",
     deps = [
         "//tensorflow/python",  # TODO(b/34059704): remove when fixed
         "//tensorflow/python:platform",
@@ -240,6 +251,14 @@ py_binary(
     srcs = ["print_selective_registration_header.py"],
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
+    deps = [":print_selective_registration_header_lib"],
+)
+
+py_library(
+    name = "print_selective_registration_header_lib",
+    srcs = ["print_selective_registration_header.py"],
+    srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
     deps = [
         ":selective_registration_header_lib",
         "//tensorflow/python:platform",
@@ -261,6 +280,13 @@ py_binary(
     name = "saved_model_cli",
     srcs = ["saved_model_cli.py"],
     srcs_version = "PY2AND3",
+    deps = [":saved_model_cli_lib"],
+)
+
+py_library(
+    name = "saved_model_cli_lib",
+    srcs = ["saved_model_cli.py"],
+    srcs_version = "PY2AND3",
     deps = [
         ":saved_model_utils",
         "//tensorflow/python",
@@ -280,7 +306,7 @@ py_test(
         "no-internal-py3",
     ],
     deps = [
-        ":saved_model_cli",
+        ":saved_model_cli_lib",
         "//tensorflow/core:protos_all_py",
     ],
 )
diff --git a/tensorflow/tools/compatibility/BUILD b/tensorflow/tools/compatibility/BUILD
index 31dbc02963..74ef9ec98b 100644
--- a/tensorflow/tools/compatibility/BUILD
+++ b/tensorflow/tools/compatibility/BUILD
@@ -34,6 +34,13 @@ py_binary(
     name = "tf_upgrade",
     srcs = ["tf_upgrade.py"],
     srcs_version = "PY2AND3",
+    deps = [":tf_upgrade_lib"],
+)
+
+py_library(
+    name = "tf_upgrade_lib",
+    srcs = ["tf_upgrade.py"],
+    srcs_version = "PY2AND3",
     deps = [":ast_edits"],
 )
 
@@ -42,7 +49,7 @@ py_test(
     srcs = ["tf_upgrade_test.py"],
     srcs_version = "PY2AND3",
     deps = [
-        ":tf_upgrade",
+        ":tf_upgrade_lib",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_test_lib",
         "@six_archive//:six",
@@ -89,7 +96,7 @@ py_test(
     srcs = ["tf_upgrade_v2_test.py"],
     srcs_version = "PY2AND3",
     deps = [
-        ":tf_upgrade_v2",
+        ":tf_upgrade_v2_lib",
         "//tensorflow:tensorflow_py",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_test_lib",
diff --git a/tensorflow/tools/dist_test/server/BUILD b/tensorflow/tools/dist_test/server/BUILD
index 3aa53a5615..56810ae157 100644
--- a/tensorflow/tools/dist_test/server/BUILD
+++ b/tensorflow/tools/dist_test/server/BUILD
@@ -12,6 +12,14 @@ load("//tensorflow:tensorflow.bzl", "py_binary")
 
 py_binary(
     name = "grpc_tensorflow_server",
+    srcs = ["grpc_tensorflow_server.py"],
+    srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
+    deps = [":grpc_tensorflow_server_lib"],
+)
+
+py_library(
+    name = "grpc_tensorflow_server_lib",
     srcs = [
         "grpc_tensorflow_server.py",
     ],
@@ -33,7 +41,7 @@ py_test(
     main = "parse_cluster_spec_test.py",
     srcs_version = "PY2AND3",
     deps = [
-        ":grpc_tensorflow_server",
+        ":grpc_tensorflow_server_lib",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:client_testlib",
     ],
-- 
GitLab


From 53c5fa9a4b4b8a87185a8f23e9d9b09546868c53 Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Fri, 1 Feb 2019 15:08:21 -0800
Subject: [PATCH 1787/2345] Add == and =! operators to Shape class

This enables EXPECT_EQ of two shapes in tests, which is much easier to write debug than EXPECT_TRUE(ShapeUtil::Equal(...))

PiperOrigin-RevId: 232045107
---
 tensorflow/compiler/xla/shape.h       |  4 ++++
 tensorflow/compiler/xla/shape_test.cc | 18 ++++++++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/tensorflow/compiler/xla/shape.h b/tensorflow/compiler/xla/shape.h
index e6b4e872f6..1d594904e0 100644
--- a/tensorflow/compiler/xla/shape.h
+++ b/tensorflow/compiler/xla/shape.h
@@ -181,6 +181,10 @@ class Shape {
     bool ignore_dynamic_dimension_ = false;
   };
 
+  // Test that all fields of the shape are the same, equivalent to Equal().
+  bool operator==(const Shape& other) const { return Equal()(*this, other); }
+  bool operator!=(const Shape& other) const { return !(*this == other); }
+
  private:
   // The element type of this shape (tuple, array, etc).
   PrimitiveType element_type_ = PRIMITIVE_TYPE_INVALID;
diff --git a/tensorflow/compiler/xla/shape_test.cc b/tensorflow/compiler/xla/shape_test.cc
index 55ce5fe884..526abafea5 100644
--- a/tensorflow/compiler/xla/shape_test.cc
+++ b/tensorflow/compiler/xla/shape_test.cc
@@ -85,6 +85,24 @@ TEST_F(ShapeTest, DynamicShapeToString) {
   EXPECT_EQ("f32[<=23,44,55]", array_shape.ToString());
 }
 
+TEST_F(ShapeTest, EqualityTest) {
+  // Different layouts.
+  EXPECT_NE(ShapeUtil::MakeShapeWithLayout(F32, {23, 44}, {1, 0}),
+            ShapeUtil::MakeShapeWithLayout(F32, {23, 44}, {0, 1}));
+
+  // Different dims.
+  EXPECT_NE(ShapeUtil::MakeShapeWithLayout(F32, {44, 23}, {1, 0}),
+            ShapeUtil::MakeShapeWithLayout(F32, {23, 44}, {1, 0}));
+
+  // Different elements.
+  EXPECT_NE(ShapeUtil::MakeShapeWithLayout(S32, {44, 23}, {1, 0}),
+            ShapeUtil::MakeShapeWithLayout(F32, {23, 44}, {1, 0}));
+
+  // Equal shapes.
+  EXPECT_EQ(ShapeUtil::MakeShapeWithLayout(F32, {23, 44}, {1, 0}),
+            ShapeUtil::MakeShapeWithLayout(F32, {23, 44}, {1, 0}));
+}
+
 TEST_F(ShapeTest, IsStatic) {
   EXPECT_TRUE(opaque_.is_static());
   EXPECT_TRUE(token_.is_static());
-- 
GitLab


From ae3bfa56a3f682f9e644182272be5faf9befedae Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 1 Feb 2019 15:16:12 -0800
Subject: [PATCH 1788/2345] Update testMutableHashTableOfTensors to sort both
 expected and calculated values with the same function.

PiperOrigin-RevId: 232046440
---
 tensorflow/python/kernel_tests/lookup_ops_test.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/lookup_ops_test.py b/tensorflow/python/kernel_tests/lookup_ops_test.py
index bbadeab8e0..652260559f 100644
--- a/tensorflow/python/kernel_tests/lookup_ops_test.py
+++ b/tensorflow/python/kernel_tests/lookup_ops_test.py
@@ -1790,7 +1790,8 @@ class MutableHashTableOpTest(test.TestCase):
       sorted_keys = np.sort(self.evaluate(exported_keys))
       sorted_values = np.sort(self.evaluate(exported_values))
       self.assertAllEqual([b"brain", b"salad", b"surgery"], sorted_keys)
-      self.assertAllEqual([[4, 5], [2, 3], [0, 1]], sorted_values)
+      sorted_expected_values = np.sort([[4, 5], [2, 3], [0, 1]])
+      self.assertAllEqual(sorted_expected_values, sorted_values)
 
   def testMutableHashTableExportInsert(self):
     with self.cached_session():
-- 
GitLab


From a6c18c2f9c5aff3f685d90e05c19744d2f73600a Mon Sep 17 00:00:00 2001
From: Nick Felt <nickfelt@google.com>
Date: Fri, 1 Feb 2019 15:25:00 -0800
Subject: [PATCH 1789/2345] PR #25273 (updated): Add decode_wav api endpoint

Re-do of merged GitHub PR #25273, but with corrected API goldens.

PiperOrigin-RevId: 232047982
---
 tensorflow/core/api_def/python_api/api_def_DecodeWav.pbtxt | 4 +++-
 tensorflow/tools/api/golden/v1/tensorflow.audio.pbtxt      | 4 ++++
 tensorflow/tools/api/golden/v2/tensorflow.audio.pbtxt      | 4 ++++
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/api_def/python_api/api_def_DecodeWav.pbtxt b/tensorflow/core/api_def/python_api/api_def_DecodeWav.pbtxt
index d6fd4691f7..28f4514bd8 100644
--- a/tensorflow/core/api_def/python_api/api_def_DecodeWav.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_DecodeWav.pbtxt
@@ -1,4 +1,6 @@
 op {
   graph_op_name: "DecodeWav"
-  visibility: HIDDEN
+  endpoint {
+    name: "audio.decode_wav"
+  }
 }
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.audio.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.audio.pbtxt
index ce29615f72..6c57240783 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.audio.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.audio.pbtxt
@@ -1,5 +1,9 @@
 path: "tensorflow.audio"
 tf_module {
+  member_method {
+    name: "decode_wav"
+    argspec: "args=[\'contents\', \'desired_channels\', \'desired_samples\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'-1\', \'None\'], "
+  }
   member_method {
     name: "encode_wav"
     argspec: "args=[\'audio\', \'sample_rate\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.audio.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.audio.pbtxt
index ce29615f72..6c57240783 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.audio.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.audio.pbtxt
@@ -1,5 +1,9 @@
 path: "tensorflow.audio"
 tf_module {
+  member_method {
+    name: "decode_wav"
+    argspec: "args=[\'contents\', \'desired_channels\', \'desired_samples\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'-1\', \'None\'], "
+  }
   member_method {
     name: "encode_wav"
     argspec: "args=[\'audio\', \'sample_rate\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-- 
GitLab


From 25a1ab5dd01c34926048c931ecdf9f2ec63946eb Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Fri, 1 Feb 2019 15:32:13 -0800
Subject: [PATCH 1790/2345] Fix issue where tf.function(variable, variable)
 crashes.

PiperOrigin-RevId: 232049223
---
 tensorflow/python/eager/def_function_test.py | 10 ++++++++++
 tensorflow/python/eager/function.py          |  6 ++++++
 tensorflow/python/framework/func_graph.py    |  4 +++-
 3 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/eager/def_function_test.py b/tensorflow/python/eager/def_function_test.py
index 4a47f67fd1..76e47fce85 100644
--- a/tensorflow/python/eager/def_function_test.py
+++ b/tensorflow/python/eager/def_function_test.py
@@ -361,6 +361,16 @@ class DefFunctionTest(test.TestCase):
     add_var.get_concrete_function(constant_op.constant(2.))
     self.assertAllClose([13., 14.], add_var(constant_op.constant(2.)))
 
+  def testSameVariableTwice(self):
+
+    v = variables.Variable(1.0)
+
+    @def_function.function
+    def add(a, b):
+      return a + b
+
+    self.assertAllEqual(add(v, v), 2.0)
+
   def testInitializationInNestedCall(self):
     v_holder = []
 
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 06e88e808a..04f1999a84 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -475,11 +475,17 @@ class ConcreteFunction(object):
     tape.variables_accessed(self._func_graph.variables)
 
     tensor_inputs = []
+    variables_used = set([])
     for i, arg in enumerate(args):
       if isinstance(arg, resource_variable_ops.ResourceVariable):
+        # We can pass a variable more than once, and in this case we need to
+        # pass its handle only once.
+        if arg.handle in variables_used:
+          continue
         if arg.trainable:
           tape.variable_accessed(arg)
         tensor_inputs.append(arg.handle)
+        variables_used.add(arg.handle)
       elif isinstance(arg, ops.Tensor):
         tensor_inputs.append(arg)
       elif (self._signature is not None and
diff --git a/tensorflow/python/framework/func_graph.py b/tensorflow/python/framework/func_graph.py
index fe968480b2..3f81466cd4 100644
--- a/tensorflow/python/framework/func_graph.py
+++ b/tensorflow/python/framework/func_graph.py
@@ -556,7 +556,9 @@ def func_graph_from_py_func(name,
         # Even if an argument variable was not used in the function, we've
         # already manually captured the resource Tensor when creating argument
         # placeholders.
-        resource_placeholder = func_graph.captures.pop(arg.handle)
+        resource_placeholder = func_graph.captures.pop(arg.handle, None)
+        if resource_placeholder is None:
+          continue
         arg_variables.add(arg)
         inputs.append(resource_placeholder)
       elif isinstance(arg, ops.Tensor):
-- 
GitLab


From 0cad868457a497e2d3a5bdd89c8f4ea0f8cd5902 Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Fri, 1 Feb 2019 15:33:12 -0800
Subject: [PATCH 1791/2345] [ShapeInference] Propagate dynamic dimensions on
 select.

PiperOrigin-RevId: 232049402
---
 .../compiler/xla/client/xla_builder_test.cc   | 23 +++++++++++++++++++
 .../compiler/xla/service/shape_inference.cc   | 17 ++++++++++++--
 2 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_builder_test.cc b/tensorflow/compiler/xla/client/xla_builder_test.cc
index 368a9452ca..c9fa738a19 100644
--- a/tensorflow/compiler/xla/client/xla_builder_test.cc
+++ b/tensorflow/compiler/xla/client/xla_builder_test.cc
@@ -611,6 +611,29 @@ TEST_F(XlaBuilderTest, DynamicBinaryHasDegenerateBroadcast) {
       << result_shape;
 }
 
+TEST_F(XlaBuilderTest, DynamicSelectOnlyPredDynamic) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(PRED, {10}), ShapeUtil::MakeShape(F32, {10}),
+       ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{1},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/0));
+  auto gte0 = GetTupleElement(p0, 0);
+  auto gte1 = GetTupleElement(p0, 1);
+
+  Select(gte0, gte1, gte1);
+
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  const Shape& result_shape =
+      module->entry_computation()->root_instruction()->shape();
+  EXPECT_TRUE(ContainersEqual(result_shape.dynamic_dimensions(), {true}))
+      << result_shape;
+}
+
 TEST_F(XlaBuilderTest, DynamicPad) {
   XlaBuilder b(TestName());
   Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index 4680c924cc..3ddcaae193 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -2716,13 +2716,26 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
         "Select's pred operand must have PRED element type; got %s.",
         ShapeUtil::HumanString(pred));
   }
-  if (ShapeUtil::CompatibleIgnoringElementType(pred, on_true) ||
+  if (Shape::Equal()
+          .IgnoreElementType()
+          .IgnoreLayout()
+          .IgnoreDynamicDimension()(pred, on_true) ||
       ShapeUtil::IsScalar(pred)) {
     // By this stage we know that pred's element type is PRED. Therefore, this
     // check restricts pred to be a PRED scalar, or a PRED array with the same
     // dimensions as on_true and on_false.
-    return ShapeUtil::ChangeElementType(
+    Shape inferred_shape = ShapeUtil::ChangeElementType(
         on_true, ShapeUtil::HigherPrecisionElementType(on_true, on_false));
+
+    // Propagate dynamic dimensions if pred is not a scalar.
+    if (!ShapeUtil::IsScalar(pred)) {
+      for (int i = 0; i < inferred_shape.rank(); i++) {
+        if (pred.is_dynamic_dimension(i)) {
+          inferred_shape.set_dynamic_dimension(i, true);
+        }
+      }
+    }
+    return inferred_shape;
   }
   return InvalidArgument(
       "Select operation with non-scalar predicate with dimensionality "
-- 
GitLab


From 9c3ec576ab4e0f440fd4d4cd1891f2dfb9bde9b9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 1 Feb 2019 15:38:55 -0800
Subject: [PATCH 1792/2345] Don't use delegate LSTM & SVDF to NNAPI on Android
 8.1 (OMR1)

PiperOrigin-RevId: 232050432
---
 tensorflow/lite/delegates/nnapi/nnapi_delegate.cc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
index d5d319432f..8345eb7497 100644
--- a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
+++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
@@ -720,7 +720,9 @@ class NNAPIDelegateKernel {
         break;
       case kTfLiteBuiltinSvdf:
         // NNAPI only support float32 weights.
+        // Only delegate to NNAPI 1.1, as SVDF does not support rank > 1 on 1.0.
         if (version == 1 && node->inputs->size == 5 &&
+            android_sdk_version >= kMinSdkVersionForNNAPI11 &&
             context->tensors[node->inputs->data[/*kWeightsFeatureTensor*/ 1]]
                     .type == kTfLiteFloat32) {
           return [](const NNAPIOpMappingArgs& mapping_args)
@@ -746,8 +748,11 @@ class NNAPIDelegateKernel {
         break;
       case kTfLiteBuiltinLstm:
         // NNAPI only support float32 weights.
+        // Only delegate to NNAPI 1.1,  as 1.0 has a bug for optional tensors
+        // which would affect LSTM.
         // TODO(miaowang): add loggings to indicate why the op is rejected.
         if (version == 1 && node->inputs->size == 20 &&
+            android_sdk_version >= kMinSdkVersionForNNAPI11 &&
             context->tensors[node->inputs
                                  ->data[/*kInputToOutputWeightsTensor*/ 4]]
                     .type == kTfLiteFloat32) {
-- 
GitLab


From 2e6cd2f99508d9ebd7f0f340852d209f8e1dfd99 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Fri, 1 Feb 2019 15:39:16 -0800
Subject: [PATCH 1793/2345] Rename `input` to `tmp`; NFC

`input` is quite misleading since we're updating it in place as we construct the
IR.

PiperOrigin-RevId: 232050505
---
 .../xla/service/cpu/llvm_ir_runtime.cc        | 42 +++++++++----------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/llvm_ir_runtime.cc b/tensorflow/compiler/xla/service/cpu/llvm_ir_runtime.cc
index be39fb8602..93ef51754d 100644
--- a/tensorflow/compiler/xla/service/cpu/llvm_ir_runtime.cc
+++ b/tensorflow/compiler/xla/service/cpu/llvm_ir_runtime.cc
@@ -208,7 +208,7 @@ llvm::Value* GenerateVF32Log(llvm::IRBuilder<>* b, llvm::Value* input,
   llvm::Value* is_pos_inf_mask = vsl.FCmpEQMask(input, pos_inf);
 
   // Cut off denormalized stuff.
-  input = vsl.Max(min_norm_pos, input);
+  llvm::Value* tmp0 = vsl.Max(min_norm_pos, input);
 
   // VectorSupportLibrary (intentionally) can't juggle more than one type at a
   // time so drop down to IRBuilder for this bit.
@@ -219,12 +219,12 @@ llvm::Value* GenerateVF32Log(llvm::IRBuilder<>* b, llvm::Value* input,
   llvm::Type* i32_vector_type =
       llvm::VectorType::get(b->getInt32Ty(), vector_width);
 
-  llvm::Value* emm0 = b->CreateLShr(b->CreateBitCast(input, i32_vector_type),
+  llvm::Value* emm0 = b->CreateLShr(b->CreateBitCast(tmp0, i32_vector_type),
                                     vector_constant_23);
 
   // Keep only the fractional part.
-  input = vsl.FloatAnd(input, inv_mant_mask);
-  input = vsl.FloatOr(input, half);
+  tmp0 = vsl.FloatAnd(tmp0, inv_mant_mask);
+  tmp0 = vsl.FloatOr(tmp0, half);
 
   emm0 = b->CreateSub(emm0, vector_constant_0x7f);
   llvm::Value* e = vsl.Add(one, b->CreateSIToFP(emm0, vsl.vector_type()));
@@ -234,33 +234,33 @@ llvm::Value* GenerateVF32Log(llvm::IRBuilder<>* b, llvm::Value* input,
   //     e -= 1;
   //     x = x + x - 1.0;
   //   } else { x = x - 1.0; }
-  llvm::Value* mask = vsl.FCmpOLTMask(input, cephes_SQRTHF);
-  llvm::Value* tmp = vsl.FloatAnd(input, mask);
-  input = vsl.Sub(input, one);
+  llvm::Value* mask = vsl.FCmpOLTMask(tmp0, cephes_SQRTHF);
+  llvm::Value* tmp1 = vsl.FloatAnd(tmp0, mask);
+  tmp0 = vsl.Sub(tmp0, one);
   e = vsl.Sub(e, vsl.FloatAnd(mask, one));
-  input = vsl.Add(input, tmp);
+  tmp0 = vsl.Add(tmp0, tmp1);
 
-  llvm::Value* x2 = vsl.Mul(input, input);
-  llvm::Value* x3 = vsl.Mul(x2, input);
+  llvm::Value* x2 = vsl.Mul(tmp0, tmp0);
+  llvm::Value* x3 = vsl.Mul(x2, tmp0);
 
   llvm::Value *y, *y1, *y2;
-  y = vsl.MulAdd(input, cephes_log_p0, cephes_log_p1);
-  y1 = vsl.MulAdd(input, cephes_log_p3, cephes_log_p4);
-  y2 = vsl.MulAdd(input, cephes_log_p6, cephes_log_p7);
-  y = vsl.MulAdd(y, input, cephes_log_p2);
-  y1 = vsl.MulAdd(y1, input, cephes_log_p5);
-  y2 = vsl.MulAdd(y2, input, cephes_log_p8);
+  y = vsl.MulAdd(tmp0, cephes_log_p0, cephes_log_p1);
+  y1 = vsl.MulAdd(tmp0, cephes_log_p3, cephes_log_p4);
+  y2 = vsl.MulAdd(tmp0, cephes_log_p6, cephes_log_p7);
+  y = vsl.MulAdd(y, tmp0, cephes_log_p2);
+  y1 = vsl.MulAdd(y1, tmp0, cephes_log_p5);
+  y2 = vsl.MulAdd(y2, tmp0, cephes_log_p8);
   y = vsl.MulAdd(y, x3, y1);
   y = vsl.MulAdd(y, x3, y2);
   y = vsl.Mul(y, x3);
 
   y1 = vsl.Mul(cephes_log_q1, e);
-  tmp = vsl.Mul(half, x2);
+  llvm::Value* tmp2 = vsl.Mul(half, x2);
   y = vsl.Add(y, y1);
-  input = vsl.Sub(input, tmp);
+  tmp0 = vsl.Sub(tmp0, tmp2);
   y2 = vsl.Mul(cephes_log_q2, e);
-  input = vsl.Add(input, y);
-  input = vsl.Add(input, y2);
+  tmp0 = vsl.Add(tmp0, y);
+  tmp0 = vsl.Add(tmp0, y2);
 
   // Contains +/-inf where +/-inf is the correct answer, otherwise 0.
   llvm::Value* result_inf = vsl.FloatOr(vsl.FloatAnd(is_zero_mask, minus_inf),
@@ -270,7 +270,7 @@ llvm::Value* GenerateVF32Log(llvm::IRBuilder<>* b, llvm::Value* input,
   // result_minus_inf and result_pos_inf are both 0.
   //
   // (This implementation works because 0xffffffff is a nan.)
-  llvm::Value* result_finite_or_nan = vsl.FloatOr(input, invalid_mask);
+  llvm::Value* result_finite_or_nan = vsl.FloatOr(tmp0, invalid_mask);
 
   // Combine the above into a final result.
   return vsl.FloatOr(result_inf,
-- 
GitLab


From 4b2be43f63b3b11e6d8d5a6648330e42bf5600cb Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Fri, 1 Feb 2019 15:39:18 -0800
Subject: [PATCH 1794/2345] Fix TF/XLA lowering for FusedBatchnorm training on
 GPU

The FusedBatchnorm TensorFlow operation has two outputs, reserve_space_{1|2},
that contain unspecified results; in practice on CPUs the they contain the mean
and the variance and on GPUs they contain the mean the *inverse sqrt of the
variance* (what cudnn returns).  This is unfortunate for XLA because without a
spec describing what these outputs are, we can't soundly replace these outputs
with XLA computed values.

For now we rely on the in-practice values returned in reserve_space_{1|2} and
add a test case checking that these values are indeed correct.  In the future we
may consider some of the cleaner fixes outlined in the tracking bug.

PiperOrigin-RevId: 232050515
---
 .../compiler/tests/fused_batchnorm_test.py    |  32 ++++-
 tensorflow/compiler/tf2xla/BUILD              |  24 +++-
 .../fused_batchnorm_reserve_space_test.cc     | 130 ++++++++++++++++++
 .../compiler/tf2xla/kernels/batch_norm_op.cc  |  34 ++++-
 4 files changed, 213 insertions(+), 7 deletions(-)
 create mode 100644 tensorflow/compiler/tf2xla/fused_batchnorm_reserve_space_test.cc

diff --git a/tensorflow/compiler/tests/fused_batchnorm_test.py b/tensorflow/compiler/tests/fused_batchnorm_test.py
index 374942a0b3..56a8e1b166 100644
--- a/tensorflow/compiler/tests/fused_batchnorm_test.py
+++ b/tensorflow/compiler/tests/fused_batchnorm_test.py
@@ -191,6 +191,20 @@ class FusedBatchNormTest(xla_test.XLATestCase, parameterized.TestCase):
     mean_val = np.random.random_sample(scale_shape).astype(np.float32)
     var_val = np.random.random_sample(scale_shape).astype(np.float32)
     epsilon = 0.001
+
+    # The TensorFlow FusedBatchNormGrad training operation takes two inputs with
+    # implementation defined values.  In theory the only correct value these
+    # inputs are the corresponding reserve_space_{1|2} outputs from the
+    # FusedBatchNorm training operation.  However, in practice, we rely on the
+    # first one being mean on {C|G}PU, and the second one being variance on CPU
+    # and inverse(sqrt(variance + epsilon)) on GPU (we test this assumption
+    # separately).
+    reserve_space_1_val = mean_val
+    if self.device == "XLA_GPU":
+      reserve_space_2_val = np.reciprocal(np.sqrt(var_val + epsilon))
+    else:
+      reserve_space_2_val = var_val
+
     data_format_src = "NHWC"
     grad_x_ref, grad_scale_ref, grad_offset_ref = self._reference_grad(
         x_val, grad_val, scale_val, mean_val, var_val, epsilon, data_format_src)
@@ -207,18 +221,26 @@ class FusedBatchNormTest(xla_test.XLATestCase, parameterized.TestCase):
           np.float32, shape=x_val_converted.shape, name="grad")
       x = array_ops.placeholder(
           np.float32, shape=x_val_converted.shape, name="x")
-      mean = array_ops.placeholder(np.float32, shape=scale_shape, name="mean")
-      var = array_ops.placeholder(np.float32, shape=scale_shape, name="var")
+      reserve_space_1 = array_ops.placeholder(
+          np.float32, shape=scale_shape, name="reserve_space_1")
+      reserve_space_2 = array_ops.placeholder(
+          np.float32, shape=scale_shape, name="reserve_space_2")
       scale = array_ops.placeholder(np.float32, shape=scale_shape, name="scale")
       grad_x, grad_scale, grad_offset, _, _ = gen_nn_ops.fused_batch_norm_grad(
-          grad, x, scale, mean, var, data_format=data_format, is_training=True)
+          grad,
+          x,
+          scale,
+          reserve_space_1,
+          reserve_space_2,
+          data_format=data_format,
+          is_training=True)
 
       grad_x_val, grad_scale_val, grad_offset_val = sess.run(
           [grad_x, grad_scale, grad_offset], {
               grad: grad_val_converted,
               x: x_val_converted,
-              mean: mean_val,
-              var: var_val,
+              reserve_space_1: reserve_space_1_val,
+              reserve_space_2: reserve_space_2_val,
               scale: scale_val
           })
 
diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD
index 14bd9d4b13..02de95141d 100644
--- a/tensorflow/compiler/tf2xla/BUILD
+++ b/tensorflow/compiler/tf2xla/BUILD
@@ -1,6 +1,6 @@
 licenses(["notice"])  # Apache 2.0
 
-load("//tensorflow:tensorflow.bzl", "tf_cc_binary", "tf_cc_test")
+load("//tensorflow:tensorflow.bzl", "tf_cc_binary", "tf_cc_test", "tf_cuda_cc_test")
 
 package_group(
     name = "internal",
@@ -679,3 +679,25 @@ cc_library(
         "@com_google_absl//absl/strings",
     ],
 )
+
+tf_cuda_cc_test(
+    name = "fused_batchnorm_reserve_space_test",
+    size = "medium",
+    srcs = ["fused_batchnorm_reserve_space_test.cc"],
+    deps = [
+        "//tensorflow/cc:cc_ops",
+        "//tensorflow/compiler/jit",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:framework_internal",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:tensorflow",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+        "//tensorflow/core/kernels:ops_testutil",
+        "//tensorflow/core/kernels:ops_util",
+        "@com_google_absl//absl/algorithm:container",
+    ],
+)
diff --git a/tensorflow/compiler/tf2xla/fused_batchnorm_reserve_space_test.cc b/tensorflow/compiler/tf2xla/fused_batchnorm_reserve_space_test.cc
new file mode 100644
index 0000000000..4535ece374
--- /dev/null
+++ b/tensorflow/compiler/tf2xla/fused_batchnorm_reserve_space_test.cc
@@ -0,0 +1,130 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <string>
+#include <vector>
+
+#include "absl/algorithm/container.h"
+#include "tensorflow/cc/ops/array_ops.h"
+#include "tensorflow/cc/ops/const_op.h"
+#include "tensorflow/cc/ops/nn_ops.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/types.pb.h"
+#include "tensorflow/core/kernels/ops_testutil.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/public/session.h"
+
+namespace tensorflow {
+namespace {
+Status GetTestDevice(Session* session, string* test_device) {
+  std::vector<DeviceAttributes> devices;
+  TF_RETURN_IF_ERROR(session->ListDevices(&devices));
+
+  bool found_cpu = absl::c_any_of(devices, [&](const DeviceAttributes& device) {
+    return device.device_type() == "CPU";
+  });
+
+  bool found_gpu = absl::c_any_of(devices, [&](const DeviceAttributes& device) {
+    return device.device_type() == "GPU";
+  });
+
+  if (!found_gpu && !found_cpu) {
+    return errors::Internal("Expected at least one CPU or GPU!");
+  }
+
+  *test_device = found_gpu ? "GPU" : "CPU";
+  VLOG(2) << "Using test device " << *test_device;
+  return Status::OK();
+}
+
+void FillZeros(Tensor* tensor) {
+  auto flat = tensor->flat<float>();
+  for (int i = 0; i < flat.size(); i++) {
+    flat.data()[i] = 0.0f;
+  }
+}
+
+// This tests check that the implementation outputs from FusedBatchnorm
+// training, reserve_space_{1|2}, are what we assume them to be in the TF/XLA
+// lowering.
+//
+// If this test starts failing then it doesn't indicate that TF/cudnn have
+// violated their contract, but it indicates that we need to update the TF/XLA
+// lowering for FusedBatchnorm training to match the new implementation defined
+// behavior.
+TEST(FusedBatchnormReserveSpaceTest, Test) {
+  using ::tensorflow::ops::Const;
+  using ::tensorflow::ops::FusedBatchNorm;
+
+  std::unique_ptr<tensorflow::Session> session(
+      tensorflow::NewSession(tensorflow::SessionOptions{}));
+
+  string test_device;
+  TF_ASSERT_OK(GetTestDevice(session.get(), &test_device));
+
+  Scope root = tensorflow::Scope::NewRootScope();
+  Output input = ops::Placeholder(root.WithOpName("input"), DT_FLOAT);
+
+  Tensor scale_data(DT_FLOAT, TensorShape({10}));
+  FillZeros(&scale_data);
+  Output scale =
+      Const(root.WithOpName("scale"), Input::Initializer(scale_data));
+
+  Tensor offset_data(DT_FLOAT, TensorShape({10}));
+  FillZeros(&offset_data);
+  Output offset =
+      Const(root.WithOpName("offset"), Input::Initializer(offset_data));
+
+  Tensor mean_data(DT_FLOAT, TensorShape({0}));
+  Output mean = Const(root.WithOpName("offset"), Input::Initializer(mean_data));
+
+  Tensor variance_data(DT_FLOAT, TensorShape({0}));
+  Output variance =
+      Const(root.WithOpName("variance"), Input::Initializer(variance_data));
+
+  string tf_device = absl::StrCat("/device:", test_device, ":0");
+  string xla_device = absl::StrCat("/device:XLA_", test_device, ":0");
+
+  FusedBatchNorm fused_batch_norm_tf(
+      root.WithOpName("fused_batch_norm_tf").WithDevice(tf_device), input,
+      scale, offset, mean, variance, FusedBatchNorm::Attrs{}.IsTraining(true));
+  FusedBatchNorm fused_batch_norm_xla(
+      root.WithOpName("fused_batch_norm_xla").WithDevice(xla_device), input,
+      scale, offset, mean, variance, FusedBatchNorm::Attrs{}.IsTraining(true));
+
+  tensorflow::GraphDef graph;
+  TF_ASSERT_OK(root.ToGraphDef(&graph));
+
+  TF_ASSERT_OK(session->Create(graph));
+
+  Tensor input_data(DT_FLOAT, TensorShape({10, 10, 10, 10}));
+  auto flat_input = input_data.flat<float>();
+  for (int i = 0; i < flat_input.size(); i++) {
+    flat_input.data()[i] = (i - 5) / 1000.0f;
+  }
+
+  std::vector<Tensor> results;
+  TF_ASSERT_OK(session->Run({{"input", input_data}},
+                            {fused_batch_norm_tf.reserve_space_1.name(),
+                             fused_batch_norm_xla.reserve_space_1.name(),
+                             fused_batch_norm_tf.reserve_space_2.name(),
+                             fused_batch_norm_xla.reserve_space_2.name()},
+                            {}, &results));
+
+  test::ExpectClose(results[0], results[1], /*atol=*/1e-4);
+  test::ExpectClose(results[2], results[3], /*atol=*/1e-4);
+}
+}  // namespace
+}  // namespace tensorflow
diff --git a/tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc b/tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc
index 83c6a65dc1..f1d78c8752 100644
--- a/tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_helpers.h"
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
+#include "tensorflow/compiler/xla/client/lib/constants.h"
 #include "tensorflow/compiler/xla/client/lib/math.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/core/util/tensor_format.h"
@@ -35,6 +36,7 @@ class FusedBatchNormOp : public XlaOpKernel {
     OP_REQUIRES(
         ctx, FormatFromString(data_format_str, &data_format_),
         errors::InvalidArgument("Invalid data format: ", data_format_str));
+    is_on_gpu_ = ctx->device_type().type_string() == DEVICE_GPU_XLA_JIT;
   }
 
   void Compile(XlaOpKernelContext* ctx) override {
@@ -72,7 +74,18 @@ class FusedBatchNormOp : public XlaOpKernel {
       // variance to the gradient. Here we maintain the same behavior by setting
       // them to the mean and variance calculated by BatchNormTraining.
       ctx->SetOutput(3, xla::GetTupleElement(output, 1));
-      ctx->SetOutput(4, xla::GetTupleElement(output, 2));
+      if (is_on_gpu_) {
+        // The last two outputs from the FusedBatchNorm training TensorFlow GPU
+        // op are implementation defined.  For now we rely on the in-practice
+        // behavior of the op:
+        //   output 3 is the mean
+        //   output 4 is rsqrt(variance + epsilon)
+        xla::XlaOp variance = xla::GetTupleElement(output, 2);
+        ctx->SetOutput(4, xla::Rsqrt(xla::Add(
+                              variance, xla::ScalarLike(variance, epsilon_))));
+      } else {
+        ctx->SetOutput(4, xla::GetTupleElement(output, 2));
+      }
     } else {
       xla::XlaOp output = xla::BatchNormInference(
           input, ctx->Input(1), ctx->Input(2), ctx->Input(3), ctx->Input(4),
@@ -90,6 +103,7 @@ class FusedBatchNormOp : public XlaOpKernel {
   float epsilon_;
   TensorFormat data_format_;
   bool is_training_;
+  bool is_on_gpu_;
 };
 
 REGISTER_XLA_OP(Name("FusedBatchNorm"), FusedBatchNormOp);
@@ -105,6 +119,7 @@ class FusedBatchNormGradOp : public XlaOpKernel {
     OP_REQUIRES(
         ctx, FormatFromString(data_format_str, &data_format_),
         errors::InvalidArgument("Invalid data format: ", data_format_str));
+    is_on_gpu_ = ctx->device_type().type_string() == DEVICE_GPU_XLA_JIT;
   }
 
   void Compile(XlaOpKernelContext* ctx) override {
@@ -131,6 +146,22 @@ class FusedBatchNormGradOp : public XlaOpKernel {
     xla::XlaOp scale_backprop;
     xla::XlaOp offset_backprop;
     if (is_training_) {
+      if (is_on_gpu_) {
+        // The last two inputs to the FusedBatchNormGrad training TensorFlow GPU
+        // op are implementation defined.  For now we rely on the in-practice
+        // behavior of the op: input 3 is the mean input 4 is rsqrt(variance +
+        // epsilon)
+        //
+        // The XLA op expects:
+        //   input 3 is the mean
+        //   input 4 is the variance
+        //
+        // so we adjust input 4 here.
+        xla::XlaOp one = xla::ScalarLike(var, 1.0f);
+        xla::XlaOp epsilon = xla::ScalarLike(var, epsilon_);
+        var = xla::Sub(one / (var * var), epsilon);
+      }
+
       xla::XlaOp output =
           xla::BatchNormGrad(activations, scale, mean, var, grad_backprop,
                              epsilon_, feature_index);
@@ -187,6 +218,7 @@ class FusedBatchNormGradOp : public XlaOpKernel {
   TensorFormat data_format_;
   float epsilon_;
   bool is_training_;
+  bool is_on_gpu_;
 };
 
 REGISTER_XLA_OP(Name("FusedBatchNormGrad"), FusedBatchNormGradOp);
-- 
GitLab


From f3de78482f7dc95383ee40e548aac0e729fabbe6 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Fri, 1 Feb 2019 15:44:47 -0800
Subject: [PATCH 1795/2345] Refactor some code to allow for distributed model
 caching in the future.

PiperOrigin-RevId: 232051357
---
 .../python/keras_backward_compat_test.py      |  5 ++++-
 .../engine/distributed_training_utils.py      | 21 ++++++++-----------
 tensorflow/python/keras/engine/training.py    |  4 ++--
 .../keras/engine/training_distributed.py      | 12 ++++++++---
 4 files changed, 24 insertions(+), 18 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_backward_compat_test.py b/tensorflow/contrib/distribute/python/keras_backward_compat_test.py
index 3a7488025b..9a581e7141 100644
--- a/tensorflow/contrib/distribute/python/keras_backward_compat_test.py
+++ b/tensorflow/contrib/distribute/python/keras_backward_compat_test.py
@@ -34,6 +34,7 @@ from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_de
 from tensorflow.python.ops.parsing_ops import gen_parsing_ops
 from tensorflow.python.training import gradient_descent
 from tensorflow.python.training import rmsprop
+from tensorflow.python.training.mode_keys import ModeKeys
 
 _RANDOM_SEED = 1337
 _TRAIN_SIZE = 200
@@ -745,7 +746,9 @@ class TestDistributionStrategyWithDatasets(test.TestCase,
 
       model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0,
                 callbacks=[keras.callbacks.LearningRateScheduler(schedule)])
-      grouped_models = distribution.unwrap(model._distributed_model_train)
+      grouped_models = distribution.unwrap(
+          distributed_training_utils.get_distributed_model(
+              model, ModeKeys.TRAIN))
       with distribution.scope():
         for m in grouped_models:
           self.assertAllClose(0.001, keras.backend.get_value(
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index 4d3da9d413..91df7705d1 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -851,21 +851,18 @@ def _reset_metrics(model):
 
 
 def get_distributed_model(model, mode):
-  if mode is ModeKeys.TRAIN:
-    return model._distributed_model_train
-  elif mode is ModeKeys.TEST:
-    return model._distributed_model_test
-  elif mode is ModeKeys.PREDICT:
-    return model._distributed_model_predict
+  key = _generate_cache_key(mode)
+  return model._distributed_model_cache.get(key, None)
 
 
 def set_distributed_model(model, mode, distributed_model):
-  if mode is ModeKeys.TRAIN:
-    model._distributed_model_train = distributed_model
-  elif mode is ModeKeys.TEST:
-    model._distributed_model_test = distributed_model
-  elif mode is ModeKeys.PREDICT:
-    model._distributed_model_predict = distributed_model
+  key = _generate_cache_key(mode)
+  model._distributed_model_cache[key] = distributed_model
+
+
+def _generate_cache_key(mode):
+  key = hash(mode)
+  return key
 
 
 @tf_contextlib.contextmanager
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 63bc4b62ae..6b4dadf2f4 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -273,8 +273,8 @@ class Model(Network):
     self.target_tensors = target_tensors
 
     # Set DistributionStrategy specific parameters.
-    for mode in [ModeKeys.TRAIN, ModeKeys.TEST, ModeKeys.PREDICT]:
-      distributed_training_utils.set_distributed_model(self, mode, None)
+    self._distributed_model_cache = {}
+
     if self._distribution_strategy is not None:
       # Ensures a Session is created and configured correctly for Distribution
       # Strategy.
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 3d72a7b6f1..6412adc5a6 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -296,7 +296,9 @@ def experimental_tpu_fit_loop(model,
 
     (grouped_inputs, grouped_outputs, grouped_updates,
      grouped_session_args) = current_strategy.extended.call_for_each_replica(
-         _per_device_fit_function, args=(model._distributed_model_train,))
+         _per_device_fit_function,
+         args=(distributed_training_utils.get_distributed_model(
+             model, ModeKeys.TRAIN),))
     (all_inputs, all_outputs, all_updates,
      all_session_args) = distributed_training_utils.unwrap_values(
          current_strategy, grouped_inputs, grouped_outputs,
@@ -477,7 +479,9 @@ def experimental_tpu_test_loop(model,
 
     (grouped_inputs, grouped_outputs, grouped_updates,
      grouped_session_args) = current_strategy.extended.call_for_each_replica(
-         _per_device_eval_function, args=(model._distributed_model_test,))
+         _per_device_eval_function,
+         args=(distributed_training_utils.get_distributed_model(
+             model, ModeKeys.TEST),))
 
     (all_inputs, all_outputs, all_updates,
      all_session_args) = distributed_training_utils.unwrap_values(
@@ -637,7 +641,9 @@ def experimental_tpu_predict_loop(model,
 
     (grouped_inputs, grouped_outputs, grouped_updates,
      grouped_session_args) = current_strategy.extended.call_for_each_replica(
-         _per_device_predict_function, args=(model._distributed_model_predict,))
+         _per_device_predict_function,
+         args=(distributed_training_utils.get_distributed_model(
+             model, ModeKeys.PREDICT),))
 
     (all_inputs, all_outputs, all_updates,
      all_session_args) = distributed_training_utils.unwrap_values(
-- 
GitLab


From ab7a61cfa35a7bf7bef089540df4252d0826961f Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Fri, 1 Feb 2019 15:45:56 -0800
Subject: [PATCH 1796/2345] Automated rollback of commit
 8e38fb87fc50aaf0f086a21e1ca1f39361604234

PiperOrigin-RevId: 232051524
---
 tensorflow/core/distributed_runtime/rpc/BUILD |  14 ++
 .../rpc/grpc_remote_worker.cc                 |  12 +-
 .../rpc/grpc_response_cache.cc                | 183 ++++++++++++++++++
 .../rpc/grpc_response_cache.h                 |  91 +++++++++
 .../rpc/grpc_rpc_factory.cc                   |   2 +-
 .../core/distributed_runtime/rpc/grpc_state.h |  85 ++++++--
 .../rpc/grpc_worker_service.cc                |  96 ++++++---
 .../rpc/grpc_worker_service.h                 |   6 +-
 8 files changed, 436 insertions(+), 53 deletions(-)
 create mode 100644 tensorflow/core/distributed_runtime/rpc/grpc_response_cache.cc
 create mode 100644 tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h

diff --git a/tensorflow/core/distributed_runtime/rpc/BUILD b/tensorflow/core/distributed_runtime/rpc/BUILD
index 16078f9688..607963484d 100644
--- a/tensorflow/core/distributed_runtime/rpc/BUILD
+++ b/tensorflow/core/distributed_runtime/rpc/BUILD
@@ -159,6 +159,17 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "grpc_response_cache",
+    srcs = ["grpc_response_cache.cc"],
+    hdrs = ["grpc_response_cache.h"],
+    deps = [
+        ":grpc_util",
+        "//tensorflow/core:lib",
+        "@com_google_absl//absl/time",
+    ],
+)
+
 tf_cuda_library(
     name = "grpc_worker_service",
     srcs = ["grpc_worker_service.cc"],
@@ -166,6 +177,7 @@ tf_cuda_library(
     deps = [
         ":async_service_interface",
         ":grpc_call",
+        ":grpc_response_cache",
         ":grpc_tensor_coding",
         ":grpc_util",
         ":grpc_worker_service_impl",
@@ -183,6 +195,8 @@ tf_cuda_library(
         "//tensorflow/core/distributed_runtime:worker_cache",
         "//tensorflow/core/distributed_runtime:worker_env",
         "//tensorflow/core/distributed_runtime:worker_session",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/time",
     ],
 )
 
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.cc b/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.cc
index 2daefcb399..2479e7368b 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.cc
@@ -39,6 +39,8 @@ limitations under the License.
 
 namespace tensorflow {
 
+const int kMaxWorkerRpcRetries = 10;
+
 class GrpcRemoteWorker : public WorkerInterface {
  public:
   explicit GrpcRemoteWorker(SharedGrpcChannelPtr channel,
@@ -259,17 +261,19 @@ class GrpcRemoteWorker : public WorkerInterface {
   // given callback, `done`, will be called when the RPC completes.
   void IssueRequest(const protobuf::Message* request,
                     protobuf::Message* response, const ::grpc::string& method,
-                    StatusCallback done, CallOptions* call_opts = nullptr) {
+                    StatusCallback done, CallOptions* call_opts = nullptr,
+                    int max_retries = kMaxWorkerRpcRetries) {
     new RPCState<protobuf::Message>(&stub_, cq_, method, *request, response,
                                     std::move(done), call_opts,
-                                    callback_threadpool_);
+                                    callback_threadpool_, max_retries);
   }
   void IssueRequest(const protobuf::Message* request, TensorResponse* response,
                     const ::grpc::string& method, StatusCallback done,
-                    CallOptions* call_opts = nullptr) {
+                    CallOptions* call_opts = nullptr,
+                    int max_retries = kMaxWorkerRpcRetries) {
     new RPCState<TensorResponse>(&stub_, cq_, method, *request, response,
                                  std::move(done), call_opts,
-                                 callback_threadpool_);
+                                 callback_threadpool_, max_retries);
   }
 
   // Helper function for initializing the RpcMethod objects below.
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.cc b/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.cc
new file mode 100644
index 0000000000..613c290905
--- /dev/null
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.cc
@@ -0,0 +1,183 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h"
+#include "tensorflow/core/platform/env.h"
+
+namespace tensorflow {
+
+struct WorkerCacheEntry {
+  enum class State {
+    PENDING = 0,
+    ACTIVE = 1,
+    FINISHED = 2,
+  };
+
+  State state = State::PENDING;
+  int64 expires_seconds;
+
+  ::grpc::ByteBuffer response_buf;
+  Status response_status;
+
+  // Additional retries may arrive while a request is still executing.  The
+  // callbacks for these calls are queued in `callbacks` and evaluated after
+  // the original request is completed.
+  std::vector<std::pair<RPCResponse, StatusCallback>> callbacks;
+};
+
+void RPCResponse::Encode(::grpc::ByteBuffer* tgt) const {
+  if (buf_ != nullptr) {
+    *tgt = *buf_;
+  } else {
+    CHECK(msg_ != nullptr);
+    ::grpc::Slice slice(msg_->ByteSizeLong());
+    msg_->SerializeWithCachedSizesToArray(
+        const_cast<uint8*>(reinterpret_cast<const uint8*>(slice.begin())));
+    ::grpc::ByteBuffer tmp(&slice, 1);
+    tgt->Swap(&tmp);
+  }
+}
+
+void RPCResponse::CopyFrom(const ::grpc::ByteBuffer& src) {
+  if (buf_ != nullptr) {
+    *buf_ = src;
+    return;
+  }
+
+  CHECK(msg_ != nullptr);
+  // We create a single slice when encoding protocol messages.
+  std::vector<::grpc::Slice> slices;
+  if (src.Dump(&slices).ok()) {
+    msg_->ParseFromArray(slices[0].begin(), slices[0].size());
+  } else {
+    LOG(ERROR) << "Failed to decode cached buffer.";
+  }
+}
+
+void GrpcResponseCache::LookupOrCompute(const string& key, RPCResponse response,
+                                        ComputeFunc compute_func,
+                                        StatusCallback done_cb) {
+  VLOG(1) << "Lookup " << key;
+  std::shared_ptr<WorkerCacheEntry> req;
+  MaybeCleanup();
+  {
+    mutex_lock m(mu_);
+
+    if (requests_.find(key) != requests_.end()) {
+      req = requests_[key];
+    } else {
+      req.reset(new WorkerCacheEntry);
+      requests_[key] = req;
+    }
+
+    if (req->state == WorkerCacheEntry::State::FINISHED) {
+      if (req->expires_seconds > Env::Default()->NowSeconds()) {
+        VLOG(1) << "Reuse cached response for " << key;
+        response.CopyFrom(req->response_buf);
+        done_cb(req->response_status);
+        return;
+      }
+      VLOG(1) << "Found expired cache entry for " << key;
+      req->state = WorkerCacheEntry::State::PENDING;
+      req->response_buf.Clear();
+    }
+
+    req->callbacks.push_back(std::make_pair(response, done_cb));
+
+    if (req->state == WorkerCacheEntry::State::ACTIVE) {
+      VLOG(1) << "Found active request for " << key
+              << ".  Adding entry to response queue.";
+      return;
+    }
+
+    VLOG(2) << "No cache entry for " << key << ", running user computation.";
+    req->state = WorkerCacheEntry::State::ACTIVE;
+    req->expires_seconds = Env::Default()->NowSeconds() + expire_time_seconds_;
+  }
+
+  compute_func([this, key, req, response](Status status) {
+    mutex_lock m(mu_);
+    response.Encode(&req->response_buf);
+    current_bytes_ += req->response_buf.Length();
+
+    req->response_status = status;
+    req->state = WorkerCacheEntry::State::FINISHED;
+
+    VLOG(1) << "Operation for " << key << " finished. "
+            << "Status: " << status << ", " << req->response_buf.Length()
+            << " response bytes, " << req->callbacks.size()
+            << " pending callbacks.";
+    for (auto& cb : req->callbacks) {
+      cb.first.CopyFrom(req->response_buf);
+      cb.second(req->response_status);
+    }
+    req->callbacks.clear();
+  });
+}
+
+// Remove all stale or expired cache entries if the cache is full.
+void GrpcResponseCache::MaybeCleanup() {
+  mutex_lock m(mu_);
+  if (current_bytes_ < max_bytes_) {
+    return;
+  }
+
+  VLOG(1) << "Cleanup: " << current_bytes_ << " -> " << max_bytes_;
+  std::vector<std::pair<string, std::shared_ptr<WorkerCacheEntry>>>
+      ordered_entries;
+  ordered_entries.reserve(requests_.size());
+  for (const auto& p : requests_) {
+    ordered_entries.push_back(std::make_pair(p.first, p.second));
+  }
+
+  std::sort(ordered_entries.begin(), ordered_entries.end(),
+            [](const std::pair<string, std::shared_ptr<WorkerCacheEntry>>& a,
+               const std::pair<string, std::shared_ptr<WorkerCacheEntry>>& b) {
+              return a.second->expires_seconds > b.second->expires_seconds;
+            });
+
+  std::unordered_map<string, std::shared_ptr<WorkerCacheEntry>> kept;
+  int64 now = Env::Default()->NowSeconds();
+  int64 bytes_used = 0;
+
+  // Always keep active requests.
+  for (auto& pair : ordered_entries) {
+    if (pair.second->state != WorkerCacheEntry::State::FINISHED) {
+      kept.insert(pair);
+    }
+  }
+
+  // Keep unexpired, finished requests up to half of max_bytes_.  This reduces
+  // chances of overfilling the cache when active requests complete and
+  // amortizes cache cleanup cost.
+  for (auto& pair : ordered_entries) {
+    if (pair.second->expires_seconds < now || bytes_used >= max_bytes_ / 2) {
+      break;
+    }
+
+    if (pair.second->state == WorkerCacheEntry::State::FINISHED) {
+      kept.insert(pair);
+      bytes_used += pair.second->response_buf.Length();
+    }
+  }
+
+  VLOG(1) << "Cleaned cache.  Bytes used: " << current_bytes_ << " -> "
+          << bytes_used << ". Cache size: " << requests_.size() << " -> "
+          << kept.size();
+  current_bytes_ = bytes_used;
+  std::swap(requests_, kept);
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h b/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h
new file mode 100644
index 0000000000..0892d9f788
--- /dev/null
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h
@@ -0,0 +1,91 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_RESPONSE_CACHE_H_
+#define TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_RESPONSE_CACHE_H_
+
+#include <memory>
+#include <unordered_map>
+#include <vector>
+
+#include "tensorflow/core/distributed_runtime/rpc/grpc_util.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/platform/protobuf.h"
+
+// gRPC response caching.  Most WorkerService methods cannot be retried directly
+// as they will fail or deadlock.  To enable retrying, we can instead cache
+// responses for a short period of time and reply to duplicate requests from the
+// cache.
+namespace tensorflow {
+
+// Union type to aid caching of either raw buffers (for RecvTensor RPCs) and
+// protocol buffer messages (for all other RPCs).
+class RPCResponse {
+ public:
+  explicit RPCResponse() : buf_(nullptr), msg_(nullptr) {}
+  explicit RPCResponse(::grpc::ByteBuffer* b) : buf_(b), msg_(nullptr) {}
+  explicit RPCResponse(protobuf::Message* m) : buf_(nullptr), msg_(m) {}
+
+  // Encode this response into the target buffer.
+  void Encode(::grpc::ByteBuffer* tgt) const;
+
+  // Copy from `src`: if this is a buffer, make a shallow copy.
+  // For protocol messages, parse the response from `src`.
+  void CopyFrom(const ::grpc::ByteBuffer& src);
+
+ private:
+  ::grpc::ByteBuffer* buf_;
+  protobuf::Message* msg_;
+};
+
+typedef std::function<void(StatusCallback)> ComputeFunc;
+struct WorkerCacheEntry;
+
+// Track and cache the state of worker service RPCs.  An RPC can be in 3 states:
+//
+// * PENDING: this is the first call of the RPC, and it will transition to
+// * ACTIVE: another thread is active processing this RPC
+// * FINISHED: the worker has finished processing the method
+//
+// The response from completed RPCs are LRU cached until either `max_bytes`
+// bytes are in use by the cache or they expire (according to `expire_time`).
+class GrpcResponseCache {
+ public:
+  GrpcResponseCache(int64 max_bytes, int64 expire_time_seconds)
+      : max_bytes_(max_bytes), expire_time_seconds_(expire_time_seconds) {}
+
+  // Lookup the result for key.
+  // If it is finished, invoke `done_cb` immediately after filling `response`.
+  // If active, done_db will be invoked when the current call completes.
+  // Otherwise, invoke `compute_func` to fill the cache and invoke done_cb.
+  void LookupOrCompute(const string& key, RPCResponse response,
+                       ComputeFunc compute_func, StatusCallback done_cb);
+
+  // Remove all stale or expired cache entries if the cache is full.
+  void MaybeCleanup();
+
+ private:
+  int64 current_bytes_ GUARDED_BY(mu_) = 0;
+  const int64 max_bytes_;
+  const int64 expire_time_seconds_;
+
+  std::unordered_map<string, std::shared_ptr<WorkerCacheEntry>> requests_
+      GUARDED_BY(mu_);
+  mutex mu_;
+};
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_RESPONSE_CACHE_H_
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_rpc_factory.cc b/tensorflow/core/distributed_runtime/rpc/grpc_rpc_factory.cc
index 7f63cc9344..3635caf3d1 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_rpc_factory.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_rpc_factory.cc
@@ -210,7 +210,7 @@ void GrpcRPCFactory::StartCall(const Tensor& address_t, const Tensor& method_t,
       get_stub(index), &completion_queue_, *get_method_ptr(index),
       call->request(), call->response(),
       /*done=*/[call](const Status& s) { call->Done(s); }, call->call_opts(),
-      nullptr /*threadpool*/, fail_fast_, timeout_in_ms_);
+      nullptr /*threadpool*/, fail_fast_, timeout_in_ms_, 0 /* max_retries */);
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_state.h b/tensorflow/core/distributed_runtime/rpc/grpc_state.h
index d73638651f..0ca64dc159 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_state.h
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_state.h
@@ -32,6 +32,9 @@ limitations under the License.
 namespace tensorflow {
 
 // Object allocated per active RPC.
+// Manage the state of a single asynchronous RPC request.  If `max_retries`
+// is greater than 0, the request will be retried for any transient failures
+// as long as the overall deadline has not elapsed.
 template <class Response>
 class RPCState : public GrpcClientCQTag {
  public:
@@ -39,34 +42,55 @@ class RPCState : public GrpcClientCQTag {
   RPCState(::grpc::GenericStub* stub, ::grpc::CompletionQueue* cq,
            const ::grpc::string& method, const protobuf::Message& request,
            Response* response, StatusCallback done, CallOptions* call_opts,
-           thread::ThreadPool* threadpool)
+           thread::ThreadPool* threadpool, int32 max_retries = 0)
       : RPCState(stub, cq, method, request, response, std::move(done),
                  call_opts, threadpool, /*fail_fast=*/false,
-                 /*timeout_in_ms=*/0) {}
+                 /*timeout_in_ms=*/0, max_retries) {}
 
   template <typename Request>
   RPCState(::grpc::GenericStub* stub, ::grpc::CompletionQueue* cq,
            const ::grpc::string& method, const Request& request,
            Response* response, StatusCallback done, CallOptions* call_opts,
-           thread::ThreadPool* threadpool, bool fail_fast, int64 timeout_in_ms)
-      : call_opts_(call_opts), threadpool_(threadpool), done_(std::move(done)) {
-    context_.set_fail_fast(fail_fast);
-    if (timeout_in_ms > 0) {
-      context_.set_deadline(gpr_time_from_millis(timeout_in_ms, GPR_TIMESPAN));
-    }
-
-    if (call_opts) {
-      call_opts->SetCancelCallback([this]() { context_.TryCancel(); });
-    }
-
+           thread::ThreadPool* threadpool, bool fail_fast, int64 timeout_in_ms,
+           int32 max_retries)
+      : call_opts_(call_opts),
+        threadpool_(threadpool),
+        done_(std::move(done)),
+        cq_(cq),
+        stub_(stub),
+        method_(method),
+        max_retries_(max_retries),
+        timeout_in_ms_(timeout_in_ms),
+        fail_fast_(fail_fast) {
     response_ = response;
     ::grpc::Status s = GrpcMaybeUnparseProto(request, &request_buf_);
     if (!s.ok()) {
       LOG(ERROR) << "GrpcMaybeUnparseProto returned with non-ok status: "
                  << s.error_message();
+      // Skip retry logic if we fail to parse our request.
+      done_(FromGrpcStatus(s));
+      delete this;
+      return;
+    }
+    StartCall();
+  }
+
+  void StartCall() {
+    context_.reset(new ::grpc::ClientContext());
+    context_->set_fail_fast(fail_fast_);
+
+    if (timeout_in_ms_ > 0) {
+      context_->set_deadline(
+          gpr_time_from_millis(timeout_in_ms_, GPR_TIMESPAN));
     }
-    call_ =
-        std::move(stub->PrepareUnaryCall(&context_, method, request_buf_, cq));
+    if (call_opts_) {
+      call_opts_->SetCancelCallback([this]() { context_->TryCancel(); });
+    }
+
+    VLOG(2) << "Starting call: " << method_;
+
+    call_ = std::move(
+        stub_->PrepareUnaryCall(context_.get(), method_, request_buf_, cq_));
     call_->StartCall();
     call_->Finish(&response_buf_, &status_, this);
   }
@@ -89,16 +113,26 @@ class RPCState : public GrpcClientCQTag {
         threadpool_->Schedule([this]() { ParseAndCallDone(); });
       } else {
         ParseAndCallDone();
-        return;
       }
-    } else {
-      VLOG(2) << "Call returned with non-ok status: " << s;
+      return;
+    }
 
-      // Attach additional GRPC error information if any
+    VLOG(1) << method_ << " returned with non-ok status: " << s
+            << " Retries: " << num_retries_ << " Max: " << max_retries_ << "\n"
+            << context_->debug_error_string();
+    // Retry if we have any attempts left
+    if (++num_retries_ <= max_retries_ &&
+        (errors::IsUnavailable(s) || errors::IsUnknown(s))) {
+      response_buf_.Clear();
+      VLOG(1) << "Retrying call for " << method_ << "Retry: " << num_retries_
+              << " of " << max_retries_;
+      StartCall();
+    } else {
+      // Attach additional GRPC error information if any to the final status
       s = Status(s.code(),
                  strings::StrCat(s.error_message(),
                                  "\nAdditional GRPC error information:\n",
-                                 context_.debug_error_string()));
+                                 context_->debug_error_string()));
       done_(s);
       delete this;
     }
@@ -115,7 +149,7 @@ class RPCState : public GrpcClientCQTag {
 
  private:
   CallOptions* call_opts_;
-  ::grpc::ClientContext context_;
+  std::unique_ptr<::grpc::ClientContext> context_;
   thread::ThreadPool* threadpool_;
   std::unique_ptr<::grpc::GenericClientAsyncResponseReader> call_;
   Response* response_;
@@ -123,6 +157,15 @@ class RPCState : public GrpcClientCQTag {
   ::grpc::ByteBuffer response_buf_;
   ::grpc::Status status_;
   StatusCallback done_;
+  int64 timeout_in_ms_;
+
+  size_t num_retries_ = 0;
+  size_t max_retries_;
+
+  ::grpc::CompletionQueue* cq_;
+  ::grpc::GenericStub* stub_;
+  ::grpc::string method_;
+  bool fail_fast_;
 };
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc
index 34bf629c8a..1abf24d071 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc
@@ -16,11 +16,14 @@ limitations under the License.
 #include "tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h"
 
 #include <deque>
+#include <memory>
 #include <unordered_map>
+#include <vector>
 
 #include "grpcpp/alarm.h"
 #include "grpcpp/server_builder.h"
 
+#include "absl/container/flat_hash_map.h"
 #include "tensorflow/core/common_runtime/buf_rendezvous.h"
 #include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/common_runtime/device_mgr.h"
@@ -32,6 +35,7 @@ limitations under the License.
 #include "tensorflow/core/distributed_runtime/rendezvous_mgr_interface.h"
 #include "tensorflow/core/distributed_runtime/rpc/async_service_interface.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_call.h"
+#include "tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_tensor_coding.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_util.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h"
@@ -42,8 +46,12 @@ limitations under the License.
 #include "tensorflow/core/framework/collective.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/gtl/map_util.h"
+#include "tensorflow/core/lib/strings/strcat.h"
+#include "tensorflow/core/lib/strings/stringprintf.h"
 #include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/platform/tracing.h"
 #include "tensorflow/core/protobuf/transport_options.pb.h"
 #include "tensorflow/core/protobuf/worker.pb.h"
@@ -91,10 +99,11 @@ class GrpcWorkerServiceThread {
  public:
   explicit GrpcWorkerServiceThread(
       GrpcWorker* worker, ::grpc::ServerBuilder* builder,
-      std::unordered_map<int, int> queue_depth,
+      std::unordered_map<int, int> queue_depth, GrpcResponseCache* cache,
       grpc::WorkerService::AsyncService* worker_service)
       : worker_(worker),
         queue_depth_(queue_depth),
+        cache_(cache),
         worker_service_(worker_service),
         is_shutdown_(false) {
     cq_ = builder->AddCompletionQueue();
@@ -220,18 +229,32 @@ class GrpcWorkerServiceThread {
       NonOwnedProtoRunGraphResponse* wrapped_response =
           new NonOwnedProtoRunGraphResponse(&call->response);
       call->SetCancelCallback([call_opts]() { call_opts->StartCancel(); });
-      worker_->RunGraphAsync(call_opts, wrapped_request, wrapped_response,
-                             [call, call_opts, wrapped_request,
-                              wrapped_response](const Status& s) {
-                               if (!s.ok()) {
-                                 VLOG(1) << "Bad response from RunGraph:" << s;
-                               }
-                               call->ClearCancelCallback();
-                               delete call_opts;
-                               delete wrapped_request;
-                               delete wrapped_response;
-                               call->SendResponse(ToGrpcStatus(s));
-                             });
+      auto done_cb = [call, call_opts, wrapped_request,
+                      wrapped_response](const Status& s) {
+        VLOG(1) << "RunGraph::Done";
+        if (!s.ok()) {
+          VLOG(1) << "Bad response from RunGraph:" << s;
+        }
+        call->ClearCancelCallback();
+        delete call_opts;
+        delete wrapped_request;
+        delete wrapped_response;
+        call->SendResponse(ToGrpcStatus(s));
+      };
+
+      auto compute_fn = [this, call_opts, wrapped_request,
+                         wrapped_response](StatusCallback done) {
+        worker_->RunGraphAsync(call_opts, wrapped_request, wrapped_response,
+                               done);
+      };
+
+      if (cache_) {
+        string request_key = call->request.ShortDebugString();
+        cache_->LookupOrCompute(request_key, RPCResponse(&call->response),
+                                compute_fn, done_cb);
+      } else {
+        compute_fn(done_cb);
+      }
     });
     ENQUEUE_REQUEST(RunGraph, true);
   }
@@ -241,16 +264,28 @@ class GrpcWorkerServiceThread {
     Schedule([this, call]() {
       CallOptions* call_opts = new CallOptions;
       call->SetCancelCallback([call_opts]() { call_opts->StartCancel(); });
-      worker_->GrpcRecvTensorAsync(
-          call_opts, &call->request, &call->response,
-          [call, call_opts](const Status& s) {
-            call->ClearCancelCallback();
-            delete call_opts;
-            if (!s.ok()) {
-              VLOG(1) << "Bad response from RecvTensor:" << s;
-            }
-            call->SendResponse(ToGrpcStatus(s));
-          });
+
+      auto done_cb = [call, call_opts](const Status& s) {
+        call->ClearCancelCallback();
+        delete call_opts;
+        if (!s.ok()) {
+          VLOG(1) << "Bad response from RecvTensor:" << s;
+        }
+        call->SendResponse(ToGrpcStatus(s));
+      };
+
+      auto compute_fn = [this, &call_opts, &call](StatusCallback done) {
+        worker_->GrpcRecvTensorAsync(call_opts, &call->request, &call->response,
+                                     done);
+      };
+
+      if (cache_) {
+        string request_key = call->request.ShortDebugString();
+        cache_->LookupOrCompute(request_key, RPCResponse(&call->response),
+                                compute_fn, done_cb);
+      } else {
+        compute_fn(done_cb);
+      }
     });
     EnqueueRecvTensorRequestRaw();
   }
@@ -328,6 +363,7 @@ class GrpcWorkerServiceThread {
   std::unique_ptr<::grpc::ServerCompletionQueue> cq_;
   std::unique_ptr<Thread> thread_;
   std::unordered_map<int, int> queue_depth_;
+  GrpcResponseCache* cache_;
   grpc::WorkerService::AsyncService* const worker_service_;
 
   mutex shutdown_mu_;
@@ -341,9 +377,16 @@ class GrpcWorkerService : public AsyncServiceInterface {
                     GrpcWorkerServiceOptions options)
       : is_shutdown_(false) {
     builder->RegisterService(&worker_service_);
-    for (int i = 0; i < options.num_worker_threads; i++) {
-      threads_.emplace_back(new GrpcWorkerServiceThread(
-          worker, builder, options.queue_depth, &worker_service_));
+    if (options.response_cache_bytes > 0) {
+      cache_.reset(
+          new GrpcResponseCache(options.response_cache_bytes,
+                                options.response_cache_expires_seconds));
+    }
+
+    for (int i = 0; i < options.num_serving_threads; i++) {
+      threads_.emplace_back(
+          new GrpcWorkerServiceThread(worker, builder, options.queue_depth,
+                                      cache_.get(), &worker_service_));
     }
   }
 
@@ -378,6 +421,7 @@ class GrpcWorkerService : public AsyncServiceInterface {
   grpc::WorkerService::AsyncService worker_service_;
   std::vector<std::unique_ptr<GrpcWorkerServiceThread>> threads_;
 
+  std::unique_ptr<GrpcResponseCache> cache_;
   mutex service_shutdown_mu_;
   bool is_shutdown_ GUARDED_BY(service_shutdown_mu_);
 
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h
index 88beb6c216..8f2830c899 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h
@@ -16,8 +16,10 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_WORKER_SERVICE_H_
 #define TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_WORKER_SERVICE_H_
 
+#include <memory>
 #include <unordered_map>
 #include "tensorflow/core/distributed_runtime/recent_request_ids.h"
+#include "tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h"
 #include "tensorflow/core/distributed_runtime/worker.h"
 
@@ -63,7 +65,9 @@ struct GrpcWorkerServiceOptions {
   // Map from GrpcWorkerMethod id to queue depth.  If set this overrides the
   // default queue depth for a method.
   std::unordered_map<int, int> queue_depth;
-  int num_worker_threads = 8;
+  int num_serving_threads = 8;
+  int64 response_cache_bytes = 0;
+  int64 response_cache_expires_seconds = 0;
 };
 
 // Returns an implementation of WorkerService rpc service.
-- 
GitLab


From 1edc3920d63eee6fe4c72e27c1d3e73deb6254e3 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Fri, 1 Feb 2019 15:47:53 -0800
Subject: [PATCH 1797/2345] [TF:XLA] Bump open source abseil revision to
 d78310fe5a82f2e0e6e16509ef8079c8d7e4674e

PiperOrigin-RevId: 232051832
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 93e73df883..f6689ba4a3 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -129,11 +129,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "com_google_absl",
         build_file = clean_dep("//third_party:com_google_absl.BUILD"),
-        sha256 = "583e5801372a0bb12eb561858532e3bb9a3528f15f65cfc87b2c0f4c1ab1a0ca",
-        strip_prefix = "abseil-cpp-111ca7060a6ff50115ca85b59f6b5d8c8c5e9105",
+        sha256 = "4c2e4194bbddcb5162933e45fe574d2c4e77a2ef00818b8dac0392459707bfff",
+        strip_prefix = "abseil-cpp-d78310fe5a82f2e0e6e16509ef8079c8d7e4674e",
         urls = [
-            "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/111ca7060a6ff50115ca85b59f6b5d8c8c5e9105.tar.gz",
-            "https://github.com/abseil/abseil-cpp/archive/111ca7060a6ff50115ca85b59f6b5d8c8c5e9105.tar.gz",
+            "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/d78310fe5a82f2e0e6e16509ef8079c8d7e4674e.tar.gz",
+            "https://github.com/abseil/abseil-cpp/archive/d78310fe5a82f2e0e6e16509ef8079c8d7e4674e.tar.gz",
         ],
     )
 
-- 
GitLab


From 722510d6964715cba51c86e29cde8be396d1bf72 Mon Sep 17 00:00:00 2001
From: Yanhui Liang <yhliang@google.com>
Date: Fri, 1 Feb 2019 15:52:07 -0800
Subject: [PATCH 1798/2345] Fix save and load of tf.nn.softmax in TF 2.0.

PiperOrigin-RevId: 232052501
---
 tensorflow/python/keras/BUILD               |  2 +
 tensorflow/python/keras/activations.py      | 15 +++++++
 tensorflow/python/keras/activations_test.py |  9 +++++
 tensorflow/python/keras/integration_test.py | 43 +++++++++++++++++++++
 4 files changed, 69 insertions(+)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 17e4c084b4..7aea97f115 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -364,6 +364,7 @@ tf_py_test(
         "@absl_py//absl/testing:parameterized",
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
+        "//tensorflow/python:nn_ops",
     ],
     shard_count = 12,
     tags = ["notsan"],
@@ -378,6 +379,7 @@ tf_py_test(
         "@absl_py//absl/testing:parameterized",
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
+        "//tensorflow/python:nn_ops",
     ],
 )
 
diff --git a/tensorflow/python/keras/activations.py b/tensorflow/python/keras/activations.py
index 8f10aca020..a10629a5fc 100644
--- a/tensorflow/python/keras/activations.py
+++ b/tensorflow/python/keras/activations.py
@@ -26,6 +26,19 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
 from tensorflow.python.util.tf_export import keras_export
 
+# b/123041942
+# In TF 2.x, if the `tf.nn.softmax` is used as an activation function in Keras
+# layers, it gets serialized as 'softmax_v2' instead of 'softmax' as the
+# internal method name is returned in serialization. This results in errors in
+# model exporting and loading as Keras can't find any activation function with
+# the name of `softmax_v2`.
+
+# This dict maps the activation function name from its v2 version to its
+# canonical name.
+_TF_ACTIVATIONS_V2 = {
+    'softmax_v2': 'softmax',
+}
+
 
 @keras_export('keras.activations.softmax')
 def softmax(x, axis=-1):
@@ -190,6 +203,8 @@ def linear(x):
 
 @keras_export('keras.activations.serialize')
 def serialize(activation):
+  if activation.__name__ in _TF_ACTIVATIONS_V2:
+    return _TF_ACTIVATIONS_V2[activation.__name__]
   return activation.__name__
 
 
diff --git a/tensorflow/python/keras/activations_test.py b/tensorflow/python/keras/activations_test.py
index 33001f419e..9d2195480d 100644
--- a/tensorflow/python/keras/activations_test.py
+++ b/tensorflow/python/keras/activations_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python import keras
 from tensorflow.python.framework import test_util
+from tensorflow.python.ops import nn_ops as nn
 from tensorflow.python.platform import test
 
 
@@ -46,6 +47,14 @@ class KerasActivationsTest(test.TestCase):
       fn = keras.activations.deserialize(config)
       assert fn == ref_fn
 
+  def test_serialization_v2(self):
+    activation_map = {nn.softmax_v2: 'softmax'}
+    for fn_v2_key in activation_map:
+      fn_v2 = keras.activations.get(fn_v2_key)
+      config = keras.activations.serialize(fn_v2)
+      fn = keras.activations.deserialize(config)
+      assert fn.__name__ == activation_map[fn_v2_key]
+
   def test_softmax(self):
     x = keras.backend.placeholder(ndim=2)
     f = keras.backend.function([x], [keras.activations.softmax(x)])
diff --git a/tensorflow/python/keras/integration_test.py b/tensorflow/python/keras/integration_test.py
index dc8d1deddb..bb889f48b1 100644
--- a/tensorflow/python/keras/integration_test.py
+++ b/tensorflow/python/keras/integration_test.py
@@ -24,6 +24,7 @@ from tensorflow.python import keras
 from tensorflow.python.framework import dtypes
 from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import testing_utils
+from tensorflow.python.ops import nn_ops as nn
 from tensorflow.python.ops import rnn_cell
 from tensorflow.python.platform import test
 
@@ -197,5 +198,47 @@ class ImageClassificationIntegrationTest(keras_parameterized.TestCase):
     self.assertEqual(predictions.shape, (x_train.shape[0], 2))
 
 
+@keras_parameterized.run_all_keras_modes
+class ActivationV2IntegrationTest(keras_parameterized.TestCase):
+  """Tests activation function V2 in model exporting and loading.
+
+  This test is to verify in TF 2.x, when 'tf.nn.softmax' is used as an
+  activition function, its model exporting and loading work as expected.
+  Check b/123041942 for details.
+  """
+
+  def test_serialization_v2_model(self):
+    np.random.seed(1337)
+    (x_train, y_train), _ = testing_utils.get_test_data(
+        train_samples=100,
+        test_samples=0,
+        input_shape=(10,),
+        num_classes=2)
+    y_train = keras.utils.to_categorical(y_train)
+
+    model = keras.Sequential([
+        keras.layers.Flatten(input_shape=x_train.shape[1:]),
+        keras.layers.Dense(10, activation=nn.relu),
+        # To mimic 'tf.nn.softmax' used in TF 2.x.
+        keras.layers.Dense(y_train.shape[-1], activation=nn.softmax_v2),
+    ])
+
+    # Check if 'softmax' is in model.get_config().
+    last_layer_activation = model.get_layer(index=2).get_config()['activation']
+    self.assertEqual(last_layer_activation, 'softmax')
+
+    model.compile(loss='categorical_crossentropy',
+                  optimizer=keras.optimizer_v2.adam.Adam(0.005),
+                  metrics=['accuracy'],
+                  run_eagerly=testing_utils.should_run_eagerly())
+    model.fit(x_train, y_train, epochs=2, batch_size=10,
+              validation_data=(x_train, y_train),
+              verbose=2)
+
+    output_path = keras.saving.saved_model.export(
+        model, '/tmp/tf_keras_saved_model')
+    loaded_model = keras.saving.saved_model.load_from_saved_model(output_path)
+    self.assertEqual(model.summary(), loaded_model.summary())
+
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 9ad0f778676331ad76978498859d88a8d50acd4c Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Fri, 1 Feb 2019 15:54:13 -0800
Subject: [PATCH 1799/2345] while_v2: handle resource tensors captured from
 function call output.

We could still use more test coverage, such as calling a function and
then taking the gradient in a while loop, then taking the gradient of
the outer while loop. I ran into other errors writing this test case
though.

PiperOrigin-RevId: 232052854
---
 .../kernel_tests/control_flow_ops_py_test.py  |  93 ++++++++++++++++
 tensorflow/python/ops/while_v2.py             | 102 +++++++++++++++---
 2 files changed, 179 insertions(+), 16 deletions(-)

diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index a52b1f09d6..81a530a263 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -2525,6 +2525,99 @@ class ControlFlowTest(test.TestCase):
       self.evaluate(variables.global_variables_initializer())
       self.assertAllClose(216.0, g[0])
 
+  def testWhileGrad_ResourceVarInFunctionCall(self):
+
+    @def_function.function
+    def foo(x, var):
+      return x + math_ops.reduce_sum(var.sparse_read([1, 3]))
+
+    @def_function.function
+    def bar(var):
+      r = control_flow_ops.while_loop(
+          lambda i, _: i < 2,
+          lambda i, x: (i + 1, foo(x, var)),
+          [0, 0.0])[1]
+      return gradients_impl.gradients(r, var)[0]
+
+    var = resource_variable_ops.ResourceVariable([1., 2., 3., 4.])
+    self.evaluate(variables.global_variables_initializer())
+    grad = self.evaluate(bar(var))
+    self.assertIsInstance(grad, ops.IndexedSlicesValue)
+    self.assertAllEqual(gradient_checker_v2._to_numpy(grad), [0., 2., 0., 2.])
+
+  def testWhileGrad_ResourceVarInNestedFunctionCall(self):
+
+    @def_function.function
+    def foo(x, var):
+      return x + math_ops.reduce_sum(var.sparse_read([1, 3]))
+
+    @def_function.function
+    def foo2(x, var):
+      return foo(x, var)
+
+    @def_function.function
+    def bar(var):
+      r = control_flow_ops.while_loop(
+          lambda i, _: i < 2,
+          lambda i, x: (i + 1, foo2(x, var)),
+          [0, 0.0])[1]
+      return gradients_impl.gradients(r, var)[0]
+
+    var = resource_variable_ops.ResourceVariable([1., 1., 1., 1.])
+    self.evaluate(variables.global_variables_initializer())
+    grad = self.evaluate(bar(var))
+    self.assertIsInstance(grad, ops.IndexedSlicesValue)
+    self.assertAllEqual(gradient_checker_v2._to_numpy(grad), [0., 2., 0., 2.])
+
+  def testWhileGrad_ResourceVarInLoopInFunctionCall(self):
+
+    @def_function.function
+    def foo(x, var):
+      return control_flow_ops.while_loop(
+          lambda j, _: j < 3,
+          lambda j, y: (j + 1,
+                        y + math_ops.reduce_sum(var.sparse_read([1, 2]))),
+          [0, x])[1]
+
+    @def_function.function
+    def bar(var):
+      r = control_flow_ops.while_loop(
+          lambda i, _: i < 2,
+          lambda i, x: (i + 1, foo(x, var)),
+          [0, 0.0])[1]
+      return gradients_impl.gradients(r, var)[0]
+
+    var = resource_variable_ops.ResourceVariable([1., 1., 1., 1.])
+    self.evaluate(variables.global_variables_initializer())
+    grad = self.evaluate(bar(var))
+    self.assertIsInstance(grad, ops.IndexedSlicesValue)
+    self.assertAllEqual(gradient_checker_v2._to_numpy(grad), [0., 6., 6., 0.])
+
+  def testWhileCondGrad_ResourceVarInFunctionCall(self):
+
+    @def_function.function
+    def foo(x, var):
+      return x + var.sparse_read([1])[0]
+
+    def body(i, x):
+      return (i + 1, control_flow_ops.cond(
+          math_ops.equal(i % 2, 0),
+          lambda: foo(x, var1),
+          lambda: foo(x, var2)))
+
+    @def_function.function
+    def bar(var1, var2):
+      r = control_flow_ops.while_loop(
+          lambda i, _: i < 4, body, [0, 0.0])
+      return gradients_impl.gradients(r, [var1, var2])
+
+    var1 = resource_variable_ops.ResourceVariable([1., 2., 3.])
+    var2 = resource_variable_ops.ResourceVariable([4., 5.])
+    self.evaluate(variables.global_variables_initializer())
+    grads = self.evaluate(bar(var1, var2))
+    self.assertAllEqual(gradient_checker_v2._to_numpy(grads[0]), [0., 2., 0.])
+    self.assertAllEqual(gradient_checker_v2._to_numpy(grads[1]), [0., 2.])
+
   @test_util.run_deprecated_v1
   def testWhileGrad_ResourceVarSparseRead(self):
     # NOTE(skyewm): this test is interesting because the
diff --git a/tensorflow/python/ops/while_v2.py b/tensorflow/python/ops/while_v2.py
index 40c3be9a1e..68f1cbae0a 100644
--- a/tensorflow/python/ops/while_v2.py
+++ b/tensorflow/python/ops/while_v2.py
@@ -842,27 +842,97 @@ class _WhileBodyGradFuncGraph(util.WhileBodyFuncGraph):
       Tensor in this graph.
     """
     assert tensor.dtype == dtypes.resource
-    if tensor in self._forward_graph.inputs:
-      index = self._forward_graph.inputs.index(tensor)
-    elif tensor.op.type == "While":
-      # Captured resources occur at the same index in the lists of inputs and
-      # outputs of a while op. So we lookup the input of `tensor.op` at the
-      # same index as the index of `tensor` in the `tensor.op.outputs`.
-      index = self._forward_graph.inputs.index(
-          tensor.op.inputs[tensor.value_index])
-    else:
-      raise ValueError(
-          "Taking gradient of a while loop which creates "
-          "a resource in its body is not supported: %s" % tensor)
-    # This must be a loop invariant.
-    assert self._forward_graph.inputs[index] == self._forward_graph.outputs[
-        index], ("Resource tensors must be loop invariants %s." %
-                 self._forward_graph._while.inputs[index])
+
+    index = self._resource_input_index(
+        tensor.name,
+        [t.name for t in self._forward_graph.inputs],
+        {op.name: op.node_def for op in self._forward_graph.get_operations()},
+        self._forward_graph._functions)
+
+    input_placeholder = self._forward_graph.inputs[index]
     tensor_in_outer_graph = self._forward_graph._while.inputs[index]
+
+    assert input_placeholder.dtype == dtypes.resource
+    assert tensor_in_outer_graph.dtype == dtypes.resource
+    # This must be a loop invariant.
+    assert input_placeholder == self._forward_graph.outputs[index], (
+        "Resource tensors must be loop invariants %s." %
+        tensor_in_outer_graph)
+
     self._indirect_captures[tensor] = self.capture(
         tensor_in_outer_graph, whitelisted=True)
     return self._indirect_captures[tensor]
 
+  def _resource_input_index(self, tensor_name, input_names, node_defs,
+                            functions):
+    """Returns the index of the input corresponding to `tensor_name`.
+
+    This method is used to find the corresponding index of an arbitrary resource
+    tensor in a function (the function could be a loop body). We assume that
+    resource handles are never created in functions, so that every resource
+    tensor can be traced back to a function input.
+
+    The awkward signature of this method is to make it work with both FuncGraphs
+    and FunctionDefs. This is so we can recurse on function call ops without
+    building the corresponding FuncGraph (note that even if a FuncGraph for a
+    FunctionDef already exists, the input/output/node names may have been
+    changed when the FuncGraph was serialized to the FunctionDef, which makes it
+    unusable with this algorithm).
+
+    Args:
+      tensor_name: the name of the resource tensor to be resolved to an input.
+      input_names: a list of the names of all inputs to the function.
+      node_defs: a dict mapping op name -> NodeDef for every op in the function.
+      functions: a dict mapping function name -> _EagerDefinedFunction.
+
+    Returns:
+      The index into input_names corresponding to `tensor_name`.
+    """
+    while tensor_name not in input_names:
+      # FunctionDefs and graphs use different tensor naming conventions.
+      parts = tensor_name.split(":")
+      if len(parts) == 3:
+        op_name, _, output_idx = parts
+      elif len(parts) == 2:
+        op_name, output_idx = parts
+      else:
+        assert len(parts) == 1
+        op_name = parts[0]
+        output_idx = 0
+      output_idx = int(output_idx)
+      node_def = node_defs[op_name]
+
+      if node_def.op == "While":
+        # Captured resources occur at the same index in the lists of inputs and
+        # outputs of a while op. So we lookup the input of `tensor.op` at the
+        # same index as the index of `tensor` in the `tensor.op.outputs`.
+        tensor_name = node_def.input[output_idx]
+      elif node_def.op in ("PartitionedCall", "StatefulPartitionedCall"):
+        # Functions output any captured resource tensors used by their
+        # gradients.  `tensor_name` is one of these outputs from a nested
+        # function call, so recursively find the corresponding input in the
+        # nested FunctionDef.
+        func_name = node_def.attr["f"].func.name
+        fdef = functions[func_name].definition
+        output_arg_name = fdef.signature.output_arg[output_idx].name
+        output_tensor_name = fdef.ret[output_arg_name]
+        input_index = self._resource_input_index(
+            output_tensor_name,
+            [arg.name for arg in fdef.signature.input_arg],
+            {ndef.name: ndef for ndef in fdef.node_def},
+            functions)
+        tensor_name = node_def.input[input_index]
+      else:
+        # We assume there are no other ops types that will "forward" resource
+        # handles like this, so all other handles must have been created by the
+        # op. (Note that cond_v2 wraps resource handle outputs in optionals,
+        # which we'll end up accumulating).
+        raise ValueError(
+            "Taking gradient of a while loop which creates "
+            "a resource in its body is not supported: %s" % op_name)
+
+    return input_names.index(tensor_name)
+
 
 def _check_shapes_compat(output_tensors, shape_invariants, input_tensors):
   for (t, shape, input_t) in zip(output_tensors, shape_invariants,
-- 
GitLab


From e717d92de4ed2c029a1fa5384a356ce64f0b044a Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Fri, 1 Feb 2019 16:49:13 -0800
Subject: [PATCH 1800/2345] TF/gRPC: Don't abort the step when an RPC is
 cancelled.

gRPC can trigger a cancellation on the server in the event of a failed channel.
If we abort the step immediately, this prevents us from retrying the operation.
As a failed client RPC will eventually trigger a cancellation and teardown, we
can skip aborting the step on the server side.

PiperOrigin-RevId: 232061494
---
 .../distributed_runtime/rpc/grpc_worker_service.cc  | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc
index 1abf24d071..904862100e 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc
@@ -466,11 +466,14 @@ void GrpcWorker::GrpcRecvTensorAsync(CallOptions* opts,
     return;
   }
 
-  // Request the tensor associated with the rendezvous key. Any time
-  // while waiting for the tensor to be produced, up until the start
-  // of execution of the callback lambda body below, an RPC
-  // cancellation should abort the rendezvous.
-  opts->SetCancelCallback([this, step_id]() { AbortStep(step_id); });
+  // Request the tensor associated with the rendezvous key.
+  // Note that we log the cancellation here but do not abort the current step.
+  // gRPC can generate cancellations in response to transient network failures,
+  // and aborting the step eliminates the opportunity for client side retries.
+  // Repeated client failures will eventually cause the step to be aborted by
+  // the client.
+  opts->SetCancelCallback(
+      [step_id]() { LOG(WARNING) << "RecvTensor cancelled for " << step_id; });
   env_->rendezvous_mgr->RecvLocalAsync(
       step_id, parsed,
       [opts, response, done, src_dev, request](
-- 
GitLab


From 006c90ac70ff5ef8e2b3d619dc5c3858555f3e28 Mon Sep 17 00:00:00 2001
From: Alan Chiao <alanchiao@google.com>
Date: Fri, 1 Feb 2019 16:49:24 -0800
Subject: [PATCH 1801/2345] Internalize custom op.

PiperOrigin-RevId: 232061526
---
 tensorflow/lite/kernels/BUILD           | 14 -----
 tensorflow/lite/kernels/register.cc     |  2 -
 tensorflow/lite/kernels/register_ref.cc |  2 -
 tensorflow/lite/kernels/relu1.cc        | 59 ------------------
 tensorflow/lite/kernels/relu1_test.cc   | 79 -------------------------
 5 files changed, 156 deletions(-)
 delete mode 100644 tensorflow/lite/kernels/relu1.cc
 delete mode 100644 tensorflow/lite/kernels/relu1_test.cc

diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index 5029f10cd6..826bb9a258 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -198,7 +198,6 @@ cc_library(
         "pow.cc",
         "range.cc",
         "reduce.cc",
-        "relu1.cc",
         "reshape.cc",
         "resize_bilinear.cc",
         "resize_nearest_neighbor.cc",
@@ -332,19 +331,6 @@ tf_cc_test(
     ],
 )
 
-tf_cc_test(
-    name = "relu1_test",
-    size = "small",
-    srcs = ["relu1_test.cc"],
-    deps = [
-        ":builtin_ops",
-        "//tensorflow/lite:framework",
-        "//tensorflow/lite/kernels:test_util",
-        "@com_google_googletest//:gtest",
-        "@flatbuffers",
-    ],
-)
-
 tf_cc_test(
     name = "activations_test",
     size = "small",
diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index 62e17160a5..bc5ec4a1d6 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -24,7 +24,6 @@ namespace custom {
 TfLiteRegistration* Register_AUDIO_SPECTROGRAM();
 TfLiteRegistration* Register_MFCC();
 TfLiteRegistration* Register_DETECTION_POSTPROCESS();
-TfLiteRegistration* Register_RELU_1();
 TfLiteRegistration* Register_IF();
 
 }  // namespace custom
@@ -300,7 +299,6 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddCustom("Mfcc", tflite::ops::custom::Register_MFCC());
   AddCustom("AudioSpectrogram",
             tflite::ops::custom::Register_AUDIO_SPECTROGRAM());
-  AddCustom("Relu1", tflite::ops::custom::Register_RELU_1());
   AddCustom("TFLite_Detection_PostProcess",
             tflite::ops::custom::Register_DETECTION_POSTPROCESS());
 
diff --git a/tensorflow/lite/kernels/register_ref.cc b/tensorflow/lite/kernels/register_ref.cc
index f833a6a9d1..faa864b0e2 100644
--- a/tensorflow/lite/kernels/register_ref.cc
+++ b/tensorflow/lite/kernels/register_ref.cc
@@ -24,7 +24,6 @@ namespace custom {
 TfLiteRegistration* Register_AUDIO_SPECTROGRAM();
 TfLiteRegistration* Register_MFCC();
 TfLiteRegistration* Register_DETECTION_POSTPROCESS();
-TfLiteRegistration* Register_RELU_1();
 
 }  // namespace custom
 
@@ -285,7 +284,6 @@ BuiltinRefOpResolver::BuiltinRefOpResolver() {
   AddCustom("Mfcc", tflite::ops::custom::Register_MFCC());
   AddCustom("AudioSpectrogram",
             tflite::ops::custom::Register_AUDIO_SPECTROGRAM());
-  AddCustom("Relu1", tflite::ops::custom::Register_RELU_1());
   AddCustom("TFLite_Detection_PostProcess",
             tflite::ops::custom::Register_DETECTION_POSTPROCESS());
 }
diff --git a/tensorflow/lite/kernels/relu1.cc b/tensorflow/lite/kernels/relu1.cc
deleted file mode 100644
index 5a55631405..0000000000
--- a/tensorflow/lite/kernels/relu1.cc
+++ /dev/null
@@ -1,59 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include "tensorflow/lite/context.h"
-#include "tensorflow/lite/kernels/internal/tensor.h"
-#include "tensorflow/lite/kernels/kernel_util.h"
-
-namespace tflite {
-namespace ops {
-namespace custom {
-namespace relu1 {
-
-TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
-  TF_LITE_ENSURE_EQ(context, NumInputs(node), 1);
-  TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);
-  const TfLiteTensor* input = GetInput(context, node, 0);
-  TF_LITE_ENSURE_EQ(context, input->type, kTfLiteFloat32);
-  TfLiteTensor* output = GetOutput(context, node, 0);
-  output->type = input->type;
-  return context->ResizeTensor(context, output,
-                               TfLiteIntArrayCopy(input->dims));
-}
-
-// This is derived from lite/kernels/activations.cc.
-TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
-  const TfLiteTensor* input = GetInput(context, node, 0);
-  TfLiteTensor* output = GetOutput(context, node, 0);
-  const int elements = NumElements(input);
-  const float* in = input->data.f;
-  const float* in_end = in + elements;
-  float* out = output->data.f;
-  for (; in < in_end; ++in, ++out) {
-    *out = std::min(std::max(0.f, *in), 1.f);
-  }
-  return kTfLiteOk;
-}
-
-}  // namespace relu1
-
-TfLiteRegistration* Register_RELU_1() {
-  static TfLiteRegistration r = {/*init=*/nullptr, /*free=*/nullptr,
-                                 relu1::Prepare, relu1::Eval};
-  return &r;
-}
-
-}  // namespace custom
-}  // namespace ops
-}  // namespace tflite
diff --git a/tensorflow/lite/kernels/relu1_test.cc b/tensorflow/lite/kernels/relu1_test.cc
deleted file mode 100644
index f52d10b0b7..0000000000
--- a/tensorflow/lite/kernels/relu1_test.cc
+++ /dev/null
@@ -1,79 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include <gtest/gtest.h>
-#include "flatbuffers/flexbuffers.h"  // TF:flatbuffers
-#include "tensorflow/lite/kernels/register.h"
-#include "tensorflow/lite/kernels/test_util.h"
-
-namespace tflite {
-namespace ops {
-namespace custom {
-
-TfLiteRegistration* Register_RELU_1();
-
-namespace {
-
-using ::testing::ElementsAreArray;
-
-class BaseActivationsOpModel : public SingleOpModel {
- public:
-  explicit BaseActivationsOpModel(const TensorData& input) {
-    input_ = AddInput(input);
-    output_ = AddOutput({input.type, {}});
-    flexbuffers::Builder fbb;
-    fbb.Map([&]() {});
-    fbb.Finish();
-    SetCustomOp("RELU_1", fbb.GetBuffer(), Register_RELU_1);
-    BuildInterpreter({GetShape(input_)});
-  }
-
- protected:
-  int input_;
-  int output_;
-};
-
-class FloatActivationsOpModel : public BaseActivationsOpModel {
- public:
-  using BaseActivationsOpModel::BaseActivationsOpModel;
-
-  void SetInput(std::initializer_list<float> data) {
-    PopulateTensor(input_, data);
-  }
-  std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
-};
-
-TEST(FloatActivationsOpTest, Relu1) {
-  FloatActivationsOpModel m(/*input=*/{TensorType_FLOAT32, {1, 2, 4, 1}});
-  m.SetInput({
-      0.0, -0.6, 0.2, -0.4,  //
-      0.3, -2.0, 1.1, -0.1,  //
-  });
-  m.Invoke();
-  EXPECT_THAT(m.GetOutput(), ElementsAreArray({
-                                 0.0, 0.0, 0.2, 0.0,  //
-                                 0.3, 0.0, 1.0, 0.0,  //
-                             }));
-}
-
-}  // namespace
-}  // namespace custom
-}  // namespace ops
-}  // namespace tflite
-
-int main(int argc, char** argv) {
-  ::tflite::LogToStderr();
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}
-- 
GitLab


From c92fb7b693108cd69c0edc68fa788e77cec93ee0 Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Fri, 1 Feb 2019 17:01:06 -0800
Subject: [PATCH 1802/2345] Add name_scope to TensorArray.read.

PiperOrigin-RevId: 232063039
---
 tensorflow/python/ops/tensor_array_ops.py | 27 ++++++++++++-----------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/tensorflow/python/ops/tensor_array_ops.py b/tensorflow/python/ops/tensor_array_ops.py
index 8e272daf6f..3c0f98466b 100644
--- a/tensorflow/python/ops/tensor_array_ops.py
+++ b/tensorflow/python/ops/tensor_array_ops.py
@@ -542,19 +542,20 @@ class _GraphTensorArrayV2(object):
 
   def read(self, index, name=None):
     """See TensorArray."""
-    if self._element_shape:
-      element_shape = self._element_shape[0]
-    else:
-      element_shape = tensor_shape.TensorShape(None)
-    value = list_ops.tensor_list_get_item(
-        input_handle=self._flow,
-        index=index,
-        element_dtype=self._dtype,
-        element_shape=element_shape,
-        name=name)
-    if self._element_shape:
-      value.set_shape(self._element_shape[0].dims)
-    return value
+    with ops.name_scope(name, "TensorArrayV2Read", [self._flow, index]):
+      if self._element_shape:
+        element_shape = self._element_shape[0]
+      else:
+        element_shape = tensor_shape.TensorShape(None)
+      value = list_ops.tensor_list_get_item(
+          input_handle=self._flow,
+          index=index,
+          element_dtype=self._dtype,
+          element_shape=element_shape,
+          name=name)
+      if self._element_shape:
+        value.set_shape(self._element_shape[0].dims)
+      return value
 
   @tf_should_use.should_use_result
   def write(self, index, value, name=None):
-- 
GitLab


From 7c5aed39f0148f93cebef850d939558a938b41e7 Mon Sep 17 00:00:00 2001
From: Yifei Feng <yifeif@google.com>
Date: Fri, 1 Feb 2019 17:05:28 -0800
Subject: [PATCH 1803/2345] Remove tensorflow core runtime's explicit
 dependency on cuda runtime. - Add stub implementation which is wrapped in
 dynamic loader - Replace the runtime dependency with stub in tf_cuda_library

Adopted csigg@'s method in cl/230858143 to work around the issue where the cuda runtime calls in Eigen::initializeDeviceProp() cannot be wrapped with the macro method.

PiperOrigin-RevId: 232063778
---
 .../gpu/gpu_managed_allocator.cc              |   7 +-
 .../core/platform/default/build_config.bzl    |   1 -
 tensorflow/stream_executor/cuda/BUILD         |  24 ++++
 .../stream_executor/cuda/cudart_stub.cc       | 114 ++++++++++++++++++
 .../platform/default/dso_loader.cc            |  13 ++
 .../platform/default/dso_loader.h             |   2 +
 tensorflow/tensorflow.bzl                     |   2 +-
 7 files changed, 160 insertions(+), 3 deletions(-)
 create mode 100644 tensorflow/stream_executor/cuda/cudart_stub.cc

diff --git a/tensorflow/core/common_runtime/gpu/gpu_managed_allocator.cc b/tensorflow/core/common_runtime/gpu/gpu_managed_allocator.cc
index 613633eb91..2375896606 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_managed_allocator.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_managed_allocator.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 
 #ifdef GOOGLE_CUDA
+#include "tensorflow/stream_executor/cuda/cuda_driver_wrapper.h"
 #define EIGEN_USE_GPU
 #endif
 
@@ -24,7 +25,11 @@ namespace tensorflow {
 void* GpuManagedAllocator::AllocateRaw(size_t alignment, size_t num_bytes) {
   void* ptr = nullptr;
 #ifdef GOOGLE_CUDA
-  CHECK_EQ(cudaMallocManaged(&ptr, num_bytes), cudaSuccess);
+  CUdeviceptr result = 0;
+  CHECK_EQ(tensorflow::wrap::cuMemAllocManaged(&result, num_bytes,
+                                               CU_MEM_ATTACH_GLOBAL),
+           CUDA_SUCCESS);
+  ptr = reinterpret_cast<void*>(result);
 #endif
   CHECK(!(reinterpret_cast<uintptr_t>(ptr) & (alignment - 1)));
   return ptr;
diff --git a/tensorflow/core/platform/default/build_config.bzl b/tensorflow/core/platform/default/build_config.bzl
index e949008403..e17ce4ebf3 100644
--- a/tensorflow/core/platform/default/build_config.bzl
+++ b/tensorflow/core/platform/default/build_config.bzl
@@ -734,7 +734,6 @@ def tf_additional_binary_deps():
     return ["@nsync//:nsync_cpp"] + if_cuda(
         [
             "//tensorflow/stream_executor:cuda_platform",
-            "//tensorflow/core/platform/default/build_config:cuda",
         ],
     ) + if_rocm(
         [
diff --git a/tensorflow/stream_executor/cuda/BUILD b/tensorflow/stream_executor/cuda/BUILD
index 009462085b..0e23cc3eb4 100644
--- a/tensorflow/stream_executor/cuda/BUILD
+++ b/tensorflow/stream_executor/cuda/BUILD
@@ -96,6 +96,30 @@ cc_library(
     ] + tf_additional_cuda_driver_deps()),
 )
 
+cc_library(
+    name = "cudart_stub",
+    srcs = select({
+        # include dynamic loading implementation only when if_cuda_is_configured and build dynamically
+        "//tensorflow:using_cuda_nvcc_with_dynamic_build": ["cudart_stub.cc"],
+        "//tensorflow:using_cuda_clang_with_dynamic_build": ["cudart_stub.cc"],
+        "//conditions:default": [],
+    }),
+    visibility = ["//visibility:public"],
+    deps = select({
+        "//tensorflow:using_cuda_nvcc_with_dynamic_build": [
+            "@local_config_cuda//cuda:cuda_headers",
+            "//tensorflow/stream_executor/lib",
+            "//tensorflow/stream_executor/platform:dso_loader",
+        ],
+        "//tensorflow:using_cuda_clang_with_dynamic_build": [
+            "@local_config_cuda//cuda:cuda_headers",
+            "//tensorflow/stream_executor/lib",
+            "//tensorflow/stream_executor/platform:dso_loader",
+        ],
+        "//conditions:default": [],
+    }),
+)
+
 # The activation library is tightly coupled to the executor library.
 # TODO(leary) split up cuda_gpu_executor.cc so that this can stand alone.
 cc_library(
diff --git a/tensorflow/stream_executor/cuda/cudart_stub.cc b/tensorflow/stream_executor/cuda/cudart_stub.cc
new file mode 100644
index 0000000000..cb35719183
--- /dev/null
+++ b/tensorflow/stream_executor/cuda/cudart_stub.cc
@@ -0,0 +1,114 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// This file wraps cuda runtime calls with dso loader so that we don't need to
+// have explicit linking to libcuda.
+
+#include "cuda/include/cuda_runtime_api.h"
+#include "tensorflow/stream_executor/lib/env.h"
+#include "tensorflow/stream_executor/platform/dso_loader.h"
+
+namespace {
+void *GetDsoHandle() {
+  static auto handle = [] {
+    void *result = nullptr;
+    using DsoLoader = stream_executor::internal::DsoLoader;
+    DsoLoader::GetLibcudartDsoHandle(&result).IgnoreError();
+    return result;
+  }();
+  return handle;
+}
+
+template <typename T>
+T LoadSymbol(const char *symbol_name) {
+  void *symbol = nullptr;
+  auto env = stream_executor::port::Env::Default();
+  env->GetSymbolFromLibrary(GetDsoHandle(), symbol_name, &symbol).IgnoreError();
+  return reinterpret_cast<T>(symbol);
+}
+cudaError_t GetSymbolNotFoundError() {
+  return cudaErrorSharedObjectSymbolNotFound;
+}
+const char *GetSymbolNotFoundStrError() {
+  return "cudaErrorSharedObjectSymbolNotFound";
+}
+}  // namespace
+
+// Code below is auto-generated.
+extern "C" {
+cudaError_t CUDART_CB cudaFree(void *devPtr) {
+  using FuncPtr = cudaError_t (*)(void *devPtr);
+  static auto func_ptr = LoadSymbol<FuncPtr>("cudaFree");
+  if (!func_ptr) return GetSymbolNotFoundError();
+  return func_ptr(devPtr);
+}
+
+cudaError_t CUDART_CB cudaGetDevice(int *device) {
+  using FuncPtr = cudaError_t (*)(int *device);
+  static auto func_ptr = LoadSymbol<FuncPtr>("cudaGetDevice");
+  if (!func_ptr) return GetSymbolNotFoundError();
+  return func_ptr(device);
+}
+
+cudaError_t CUDART_CB cudaGetDeviceProperties(cudaDeviceProp *prop,
+                                              int device) {
+  using FuncPtr = cudaError_t (*)(cudaDeviceProp * prop, int device);
+  static auto func_ptr = LoadSymbol<FuncPtr>("cudaGetDeviceProperties");
+  if (!func_ptr) return GetSymbolNotFoundError();
+  return func_ptr(prop, device);
+}
+
+const char *CUDART_CB cudaGetErrorString(cudaError_t error) {
+  using FuncPtr = const char *(*)(cudaError_t error);
+  static auto func_ptr = LoadSymbol<FuncPtr>("cudaGetErrorString");
+  if (!func_ptr) return GetSymbolNotFoundStrError();
+  return func_ptr(error);
+}
+
+cudaError_t CUDART_CB cudaSetDevice(int device) {
+  using FuncPtr = cudaError_t (*)(int device);
+  static auto func_ptr = LoadSymbol<FuncPtr>("cudaSetDevice");
+  if (!func_ptr) return GetSymbolNotFoundError();
+  return func_ptr(device);
+}
+
+cudaError_t CUDART_CB cudaStreamAddCallback(cudaStream_t stream,
+                                            cudaStreamCallback_t callback,
+                                            void *userData,
+                                            unsigned int flags) {
+  using FuncPtr =
+      cudaError_t (*)(cudaStream_t stream, cudaStreamCallback_t callback,
+                      void *userData, unsigned int flags);
+  static auto func_ptr = LoadSymbol<FuncPtr>("cudaStreamAddCallback");
+  if (!func_ptr) return GetSymbolNotFoundError();
+  return func_ptr(stream, callback, userData, flags);
+}
+
+cudaError_t CUDART_CB cudaGetDeviceCount(int *count) {
+  using FuncPtr = cudaError_t (*)(int *count);
+  static auto func_ptr = LoadSymbol<FuncPtr>("cudaGetDeviceCount");
+  if (!func_ptr) return GetSymbolNotFoundError();
+  return func_ptr(count);
+}
+
+cudaError_t CUDART_CB cudaPointerGetAttributes(
+    struct cudaPointerAttributes *attributes, const void *ptr) {
+  using FuncPtr = cudaError_t (*)(struct cudaPointerAttributes * attributes,
+                                  const void *ptr);
+  static auto func_ptr = LoadSymbol<FuncPtr>("cudaPointerGetAttributes");
+  if (!func_ptr) return GetSymbolNotFoundError();
+  return func_ptr(attributes, ptr);
+}
+}  // extern "C"
diff --git a/tensorflow/stream_executor/platform/default/dso_loader.cc b/tensorflow/stream_executor/platform/default/dso_loader.cc
index 668eeee3f3..8592455860 100644
--- a/tensorflow/stream_executor/platform/default/dso_loader.cc
+++ b/tensorflow/stream_executor/platform/default/dso_loader.cc
@@ -117,6 +117,13 @@ string GetCudnnVersion() { return TF_CUDNN_VERSION; }
 #endif
 }
 
+/* static */ port::Status DsoLoader::GetLibcudartDsoHandle(void** dso_handle) {
+  return GetDsoHandle(FindDsoPath(port::Env::Default()->FormatLibraryFileName(
+                                      "cudart", GetCudaVersion()),
+                                  GetCudaLibraryDirPath()),
+                      dso_handle);
+}
+
 static mutex& GetRpathMutex() {
   static mutex* mu = new mutex;
   return *mu;
@@ -282,6 +289,12 @@ static std::vector<string>* CreatePrimordialRpaths() {
   return result;
 }
 
+/* static */ port::StatusOr<void*> CachedDsoLoader::GetLibcudartDsoHandle() {
+  static port::StatusOr<void*> result =
+      FetchHandleResult(DsoLoader::GetLibcudartDsoHandle);
+  return result;
+}
+
 /* static */ port::StatusOr<void*> CachedDsoLoader::FetchHandleResult(
     std::function<port::Status(void**)> load_dso) {
   void* handle;
diff --git a/tensorflow/stream_executor/platform/default/dso_loader.h b/tensorflow/stream_executor/platform/default/dso_loader.h
index 806f65b24c..92c0db7037 100644
--- a/tensorflow/stream_executor/platform/default/dso_loader.h
+++ b/tensorflow/stream_executor/platform/default/dso_loader.h
@@ -46,6 +46,7 @@ class DsoLoader {
   static port::Status GetCurandDsoHandle(void** dso_handle);
   static port::Status GetLibcudaDsoHandle(void** dso_handle);
   static port::Status GetLibcuptiDsoHandle(void** dso_handle);
+  static port::Status GetLibcudartDsoHandle(void** dso_handle);
 
   // Registers a new binary-relative path to use as a dlopen search path.
   static void RegisterRpath(absl::string_view path);
@@ -101,6 +102,7 @@ class CachedDsoLoader {
   static port::StatusOr<void*> GetCurandDsoHandle();
   static port::StatusOr<void*> GetLibcudaDsoHandle();
   static port::StatusOr<void*> GetLibcuptiDsoHandle();
+  static port::StatusOr<void*> GetLibcudartDsoHandle();
 
  private:
   // Fetches a DSO handle via "load_dso" and returns the StatusOr form of the
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index 874d9e872c..fab1584cbb 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -1132,7 +1132,7 @@ def tf_cuda_library(deps = None, cuda_deps = None, copts = tf_copts(), **kwargs)
     kwargs["features"] = kwargs.get("features", []) + ["-use_header_modules"]
     native.cc_library(
         deps = deps + if_cuda_is_configured_compat(cuda_deps + [
-            clean_dep("//tensorflow/core:cuda"),
+            clean_dep("//tensorflow/stream_executor/cuda:cudart_stub"),
             "@local_config_cuda//cuda:cuda_headers",
         ]) + if_rocm_is_configured(cuda_deps + [
             # rocm_header placeholder
-- 
GitLab


From 6f650f54e4c16be4fe94bc9e2172077481b224ac Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Fri, 1 Feb 2019 17:39:03 -0800
Subject: [PATCH 1804/2345] Move CollectiveAllReduceStrategy to core.

PiperOrigin-RevId: 232067817
---
 tensorflow/contrib/distribute/python/BUILD    |  16 +-
 .../python/collective_all_reduce_strategy.py  | 340 +--------------
 .../collective_all_reduce_strategy_test.py    | 297 ++++++++++----
 tensorflow/python/distribute/BUILD            |  22 +
 .../collective_all_reduce_strategy.py         | 387 ++++++++++++++++++
 5 files changed, 650 insertions(+), 412 deletions(-)
 create mode 100644 tensorflow/python/distribute/collective_all_reduce_strategy.py

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index c67af0e46c..509eb78128 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -136,19 +136,9 @@ py_library(
     srcs = ["collective_all_reduce_strategy.py"],
     visibility = ["//tensorflow:internal"],
     deps = [
-        ":mirrored_strategy",
-        "//tensorflow/core:protos_all_py",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:collective_ops",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:training",
-        "//tensorflow/python/distribute:cross_device_ops",
-        "//tensorflow/python/distribute:cross_device_utils",
-        "//tensorflow/python/distribute:input_lib",
-        "//tensorflow/python/distribute:multi_worker_util",
-        "//tensorflow/python/distribute:numpy_dataset",
-        "//tensorflow/python/distribute:values",
-        "//tensorflow/python/eager:context",
+        "//tensorflow/python/distribute:collective_all_reduce_strategy",
+        "//tensorflow/python/distribute:distribute_lib",
+        "//tensorflow/python/distribute/cluster_resolver:cluster_resolver_lib",
     ],
 )
 
diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
index aa4d82b4d0..1974162798 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
@@ -18,30 +18,18 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import copy
-
-from tensorflow.contrib.distribute.python import mirrored_strategy
-from tensorflow.core.protobuf import rewriter_config_pb2
-from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib
-from tensorflow.python.distribute import cross_device_utils
-from tensorflow.python.distribute import device_util
+from tensorflow.python.distribute import collective_all_reduce_strategy
 from tensorflow.python.distribute import distribute_lib
-from tensorflow.python.distribute import input_lib
-from tensorflow.python.distribute import multi_worker_util
-from tensorflow.python.distribute import numpy_dataset
-from tensorflow.python.distribute import values
-from tensorflow.python.eager import context
-from tensorflow.python.eager import tape
-from tensorflow.python.framework import ops
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import collective_ops
-from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.distribute.cluster_resolver import SimpleClusterResolver
+from tensorflow.python.distribute.cluster_resolver import TFConfigClusterResolver
 
 
 # TODO(yuefengz): support in-graph replication.
 class CollectiveAllReduceStrategy(distribute_lib.DistributionStrategy):
   """Distribution strategy that uses collective ops for all-reduce.
 
+  *** contrib version ***
+
   It is similar to the MirroredStrategy but it uses collective ops for
   reduction.
 
@@ -64,311 +52,19 @@ class CollectiveAllReduceStrategy(distribute_lib.DistributionStrategy):
         CollectiveAllReduceExtended(self, num_gpus_per_worker))
 
 
-class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
+class CollectiveAllReduceExtended(
+    collective_all_reduce_strategy.CollectiveAllReduceExtended):
   """Implementation of CollectiveAllReduceStrategy."""
 
   def __init__(self, container_strategy, num_gpus_per_worker):
-    distribute_lib.DistributionStrategyExtended.__init__(
-        self, container_strategy)
-    self._cross_device_ops = None
-    self._num_gpus_per_worker = num_gpus_per_worker
-    self._initialize_local_worker(num_gpus_per_worker)
-    assert isinstance(self._get_cross_device_ops(),
-                      cross_device_ops_lib.CollectiveAllReduce)
-
-  def _initialize_local_worker(self, num_gpus_per_worker):
-    """Initializes the object for local training."""
-    self._is_chief = True
-    self._num_workers = 1
-
-    if num_gpus_per_worker:
-      local_devices = tuple(
-          "/device:GPU:%d" % i for i in range(num_gpus_per_worker)
-      )
-    else:
-      local_devices = ("/device:CPU:0",)
-    self._worker_device = device_util.canonicalize("/device:CPU:0")
-    self._host_input_device = numpy_dataset.SingleDevice(self._worker_device)
-
-    self._collective_keys = cross_device_utils.CollectiveKeys()
-    self._initialize_local(local_devices)
-    # TODO(yuefengz): remove num_gpus_per_worker from CollectiveAllReduce.
-    self._cross_device_ops = cross_device_ops_lib.CollectiveAllReduce(
-        num_workers=self._num_workers,
-        num_gpus_per_worker=num_gpus_per_worker,
-        collective_keys=self._collective_keys)
-
-    self._cluster_spec = None
-    self._task_type = None
-    self._task_id = None
-
-    logging.info("CollectiveAllReduceStrategy with local_devices = %r",
-                 local_devices)
-
-  def _initialize_multi_worker(self, num_gpus_per_worker, cluster_spec,
-                               task_type, task_id):
-    """Initializes the object for multi-worker training."""
-    if task_type is None or task_id is None:
-      raise ValueError("When `cluster_spec` is given, you must also specify "
-                       "`task_type` and `task_id`")
-    if task_type not in ("chief", "worker"):
-      raise ValueError(
-          "Unrecognized task_type: %r, valid task types are: \"chief\", "
-          "\"worker\"." % task_type)
-    cluster_spec = multi_worker_util.normalize_cluster_spec(cluster_spec)
-    self._num_workers = multi_worker_util.worker_count(cluster_spec, task_type)
-    if not self._num_workers:
-      raise ValueError("No `worker` or `chief` tasks can be found in "
-                       "`cluster_spec`.")
-
-    self._is_chief = multi_worker_util.is_chief(cluster_spec, task_type,
-                                                task_id)
-
-    self._worker_device = "/job:%s/task:%d" % (task_type, task_id)
-    self._host_input_device = numpy_dataset.SingleDevice(self._worker_device)
-    if num_gpus_per_worker:
-      local_devices = tuple(
-          "%s/device:GPU:%d" % (self._worker_device, i)
-          for i in range(num_gpus_per_worker)
-      )
-    else:
-      local_devices = (self._worker_device,)
-
-    self._collective_keys = cross_device_utils.CollectiveKeys()
-    self._initialize_local(local_devices)
-    self._input_workers = input_lib.InputWorkers(
-        self._device_map, [(self._worker_device, self.worker_devices)])
-    self._cross_device_ops = cross_device_ops_lib.CollectiveAllReduce(
-        num_workers=self._num_workers,
-        num_gpus_per_worker=num_gpus_per_worker,
-        collective_keys=self._collective_keys)
-
-    # Add a default device so that ops without specified devices will not end up
-    # on other workers.
-    self._default_device = "/job:%s/task:%d" % (task_type, task_id)
-
-    self._cluster_spec = multi_worker_util.normalize_cluster_spec(cluster_spec)
-    self._task_type = task_type
-    self._task_id = task_id
-
-    logging.info(
-        "Multi-worker CollectiveAllReduceStrategy with "
-        "cluster_spec = %r, task_type = %r, task_id = %r, "
-        "num_workers = %r, local_devices = %r", cluster_spec.as_dict(),
-        task_type, task_id, self._num_workers, local_devices)
-
-  def _create_variable(self, next_creator, *args, **kwargs):
-    colocate_with = kwargs.pop("colocate_with", None)
-    if colocate_with is None:
-      device_map = self._device_map
-      logical_device = 0  # TODO(josh11b): Get logical device from scope here.
-    elif isinstance(colocate_with, numpy_dataset.SingleDevice):
-      with ops.device(colocate_with.device):
-        return next_creator(*args, **kwargs)
-    else:
-      device_map = colocate_with.device_map
-      logical_device = colocate_with.logical_device
-
-    def _real_mirrored_creator(devices, *args, **kwargs):
-      """Creates one MirroredVariable on the current worker."""
-      unique_var_name = ops.get_default_graph().unique_name(
-          kwargs["name"], mark_as_used=False).rstrip("/")
-      # pylint: disable=protected-access
-      collective_instance_key = self._collective_keys.get_instance_key(
-          key_id=unique_var_name)
-      # Only the first device participles in the broadcast of initial values.
-      group_key = self._collective_keys.get_group_key([devices[0]])
-      group_size = self._num_workers
-      if "initial_value" not in kwargs:
-        raise ValueError("Initial value must be specified.")
-      initial_value = kwargs["initial_value"]
-      if callable(initial_value):
-        initial_value_fn = initial_value
-      else:
-        initial_value_fn = lambda: initial_value
-
-      value_list = []
-      for i, d in enumerate(devices):
-        with ops.init_scope(), ops.device(d):
-          if i == 0:
-            # The initial value fn makes sure variables all initialized to
-            # same values. The first device of the chief worker will send their
-            # variable values to other workers.
-            def _overridden_initial_value_fn(device=d, index=i):  # pylint: disable=g-missing-docstring
-              with ops.device(device):
-                initial_value = initial_value_fn()
-                assert not callable(initial_value)
-                initial_value = ops.convert_to_tensor(initial_value)
-
-                assert index == 0, index
-                if self._num_workers > 1:
-                  if self._is_chief:
-                    bcast_send = collective_ops.broadcast_send(
-                        initial_value, initial_value.shape, initial_value.dtype,
-                        group_size, group_key, collective_instance_key)
-                    with ops.control_dependencies([bcast_send]):
-                      return array_ops.identity(initial_value)
-                  else:
-                    return collective_ops.broadcast_recv(
-                        initial_value.shape, initial_value.dtype, group_size,
-                        group_key, collective_instance_key)
-                return initial_value
-          else:
-            # Give replicas meaningful distinct names:
-            var0name = value_list[0].name.split(":")[0]
-            # We append a / to variable names created on replicas with id > 0 to
-            # ensure that we ignore the name scope and instead use the given
-            # name as the absolute name of the variable.
-            kwargs["name"] = "%s/replica_%d/" % (var0name, i)
-
-            # Variables on non-first replica get initial values from the
-            # variables created on the first device of each worker.
-            def _overridden_initial_value_fn(device=d, index=i):
-              assert index > 0
-              with ops.device(device):
-                if context.executing_eagerly():
-                  return array_ops.identity(value_list[0].value())
-                else:
-                  return array_ops.identity(value_list[0].initial_value)
-
-          kwargs["initial_value"] = _overridden_initial_value_fn
-          with context.context().device_policy(context.DEVICE_PLACEMENT_SILENT):
-            # Don't record operations (e.g. other variable reads) during
-            # variable creation.
-            with tape.stop_recording():
-              v = next_creator(*args, **kwargs)
-
-          if i == 0:
-            actual_var_name = v.name.split(":")[0]
-            assert unique_var_name == actual_var_name, "%r vs %r" % (
-                unique_var_name, actual_var_name)
-          assert not isinstance(v, values.DistributedVariable)
-          value_list.append(v)
-      return value_list
-
-    # pylint: disable=protected-access
-    return mirrored_strategy._create_mirrored_variable(
-        self._container_strategy(), device_map, logical_device,
-        _real_mirrored_creator, *args, **kwargs)
-
-  def _make_dataset_iterator(self, dataset):
-    return input_lib.DatasetIterator(dataset, self._input_workers,
-                                     self._num_replicas_in_sync)
-
-  def _make_input_fn_iterator(
-      self,
-      input_fn,
-      replication_mode=distribute_lib.InputReplicationMode.PER_WORKER):
-    """Distributes the dataset to each local GPU."""
-    if self._cluster_spec is None:
-      input_pipeline_id = 0
-    else:
-      input_pipeline_id = multi_worker_util.id_in_cluster(
-          self._cluster_spec, self._task_type, self._task_id)
-    input_context = distribute_lib.InputContext(
-        num_input_pipelines=self._num_workers,
-        input_pipeline_id=input_pipeline_id,
-        num_replicas_in_sync=self._num_replicas_in_sync)
-
-    return input_lib.InputFunctionIterator(
-        input_fn, self._input_workers, [input_context])
-
-  def _configure(self,
-                 session_config=None,
-                 cluster_spec=None,
-                 task_type=None,
-                 task_id=None):
-    """Configures the object.
-
-    Args:
-      session_config: a `tf.ConfigProto`
-      cluster_spec: a dict, ClusterDef or ClusterSpec object specifying the
-        cluster configurations.
-      task_type: the current task type, such as "worker".
-      task_id: the current task id.
-
-    Raises:
-      ValueError: if `task_type` is not in the `cluster_spec`.
-    """
-    if not self._cluster_spec and cluster_spec:
-      # If a `cluster_spec` is already passed in, do nothing here.
-      # TODO(yuefengz): check `cluster_spec` is the same if this object has
-      # already been initialized with a `cluster_spec`.
-      self._initialize_multi_worker(self._num_gpus_per_worker, cluster_spec,
-                                    task_type, task_id)
-      assert isinstance(self._get_cross_device_ops(),
-                        cross_device_ops_lib.CollectiveAllReduce)
-
-    if session_config:
-      session_config.CopyFrom(self._update_config_proto(session_config))
-
-  def _update_config_proto(self, config_proto):
-    updated_config = copy.deepcopy(config_proto)
-    # Enable the scoped allocator optimization for CollectiveOps.  This
-    # optimization converts many small all-reduces into fewer larger
-    # all-reduces.
-    rewrite_options = updated_config.graph_options.rewrite_options
-    rewrite_options.scoped_allocator_optimization = (
-        rewriter_config_pb2.RewriterConfig.ON)
-    # We turn on ScopedAllocator only for CollectiveReduce op, i.e. enable_op =
-    # ["CollectiveReduce"].  Since we can't assign to a repeated proto field, we
-    # clear and then append.
-    del rewrite_options.scoped_allocator_opts.enable_op[:]
-    rewrite_options.scoped_allocator_opts.enable_op.append("CollectiveReduce")
-
-    if not self._cluster_spec:
-      return updated_config
-
-    assert self._task_type
-    assert self._task_id is not None
-
-    # Collective group leader is needed for collective ops to coordinate
-    # workers.
-    if "chief" in self._cluster_spec.jobs:
-      updated_config.experimental.collective_group_leader = (
-          "/job:chief/replica:0/task:0")
-    else:
-      if "worker" not in self._cluster_spec.jobs:
-        raise ValueError(
-            "You must have `chief` or `worker` jobs in the `cluster_spec`.")
-      updated_config.experimental.collective_group_leader = (
-          "/job:worker/replica:0/task:0")
-
-    # The device filters prevent communication between workers.
-    del updated_config.device_filters[:]
-    updated_config.device_filters.append(
-        "/job:%s/task:%d" % (self._task_type, self._task_id))
-
-    return updated_config
-
-  @property
-  def experimental_between_graph(self):
-    return True
-
-  @property
-  def experimental_should_init(self):
-    return True
-
-  @property
-  def should_checkpoint(self):
-    return self._is_chief
-
-  @property
-  def should_save_summary(self):
-    return self._is_chief
-
-  @property
-  def _num_replicas_in_sync(self):
-    return len(self.worker_devices) * self._num_workers
-
-  # TODO(priyag): Delete this once all strategies use global batch size.
-  @property
-  def _global_batch_size(self):
-    """`make_dataset_iterator` and `make_numpy_iterator` use global batch size.
-
-    `make_input_fn_iterator` assumes per-replica batching.
-
-    Returns:
-      Boolean.
-    """
-    return True
+    # Use TFConfigClusterResolver to parse TF_CONFIG. We don't want to change
+    # the constructor's interface to allow customized cluster resolver. Use
+    # SimpleClusterResolver to override num_accelerators.
+    tfconfig = TFConfigClusterResolver()
+    cluster_resolver = SimpleClusterResolver(
+        cluster_spec=tfconfig.cluster_spec(),
+        task_type=tfconfig.task_type,
+        task_id=tfconfig.task_id,
+        num_accelerators=num_gpus_per_worker)
+    super(CollectiveAllReduceExtended, self).__init__(
+        container_strategy, cluster_resolver=cluster_resolver)
diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
index bba0bce9f2..acbe4677b4 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
@@ -29,9 +29,13 @@ from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python import keras
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.distribute import collective_all_reduce_strategy as core_collective_all_reduce_strategy
 from tensorflow.python.distribute import cross_device_utils
+from tensorflow.python.distribute import distribute_lib
+from tensorflow.python.distribute import multi_worker_util
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.distribute import values
+from tensorflow.python.distribute.cluster_resolver import SimpleClusterResolver
 from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -49,6 +53,55 @@ from tensorflow.python.ops.losses import losses
 from tensorflow.python.platform import test
 from tensorflow.python.training import adam
 from tensorflow.python.training import training_util
+from tensorflow.python.training.server_lib import ClusterSpec
+
+
+class MockCollectiveAllReduceStrategy(distribute_lib.DistributionStrategy):
+  """Mock the strategy to allow cluster resolver as an argument."""
+
+  def __init__(self, cluster_resolver):
+    super(MockCollectiveAllReduceStrategy, self).__init__(
+        core_collective_all_reduce_strategy.CollectiveAllReduceExtended(
+            self, cluster_resolver=cluster_resolver))
+
+
+def create_test_objects(cluster_spec=None,
+                        task_type=None,
+                        task_id=None,
+                        num_gpus=None,
+                        use_core_strategy=False):
+  sess_config = config_pb2.ConfigProto()
+  if num_gpus is None:
+    num_gpus = context.num_gpus()
+  if use_core_strategy:
+    if cluster_spec and task_type and task_id is not None:
+      cluster_resolver = SimpleClusterResolver(
+          cluster_spec=multi_worker_util.normalize_cluster_spec(cluster_spec),
+          task_type=task_type,
+          task_id=task_id,
+          num_accelerators=num_gpus)
+      target = 'grpc://' + cluster_spec[task_type][task_id]
+    else:
+      cluster_resolver = SimpleClusterResolver(
+          ClusterSpec({}), num_accelerators=num_gpus)
+      target = ''
+
+    strategy = MockCollectiveAllReduceStrategy(cluster_resolver)
+    sess_config = strategy.update_config_proto(sess_config)
+  else:
+    strategy = collective_all_reduce_strategy.CollectiveAllReduceStrategy(
+        num_gpus_per_worker=num_gpus)
+    if task_type and task_id is not None:
+      strategy.configure(
+          session_config=sess_config,
+          cluster_spec=cluster_spec,
+          task_type=task_type,
+          task_id=task_id)
+      target = 'grpc://' + cluster_spec[task_type][task_id]
+    else:
+      target = ''
+
+  return strategy, target, sess_config
 
 
 class CollectiveAllReduceStrategyTestBase(
@@ -64,16 +117,18 @@ class CollectiveAllReduceStrategyTestBase(
     CollectiveAllReduceStrategyTestBase.collective_key_base += 100000
     super(CollectiveAllReduceStrategyTestBase, self).setUp()
 
-  def _get_test_object(self, task_type, task_id, num_gpus=0):
-    distribution = collective_all_reduce_strategy.CollectiveAllReduceStrategy(
-        num_gpus_per_worker=num_gpus)
-    session_config = config_pb2.ConfigProto()
-    if task_type and task_id is not None:
-      distribution.configure(
-          session_config=session_config,
-          cluster_spec=self._cluster_spec,
-          task_type=task_type,
-          task_id=task_id)
+  def _get_test_object(self,
+                       task_type,
+                       task_id,
+                       num_gpus=0,
+                       use_core_strategy=False):
+    strategy, target, session_config = create_test_objects(
+        cluster_spec=self._cluster_spec,
+        task_type=task_type,
+        task_id=task_id,
+        num_gpus=num_gpus,
+        use_core_strategy=use_core_strategy)
+
     collective_keys = cross_device_utils.CollectiveKeys(
         group_key_start=10 * num_gpus +
         CollectiveAllReduceStrategyTestBase.collective_key_base,
@@ -81,16 +136,16 @@ class CollectiveAllReduceStrategyTestBase(
         CollectiveAllReduceStrategyTestBase.collective_key_base,
         instance_key_with_id_start=num_gpus * 10000 +
         CollectiveAllReduceStrategyTestBase.collective_key_base)
-    distribution.extended._collective_keys = collective_keys
-    distribution.extended._cross_device_ops._collective_keys = (
-        collective_keys)
-    if task_type and task_id is not None:
-      return distribution, 'grpc://' + self._cluster_spec[task_type][
-          task_id], session_config
-    else:
-      return distribution, '', session_config
+    strategy.extended._collective_keys = collective_keys
+    strategy.extended._cross_device_ops._collective_keys = (collective_keys)
 
-  def _test_minimize_loss_graph(self, task_type, task_id, num_gpus):
+    return strategy, target, session_config
+
+  def _test_minimize_loss_graph(self,
+                                task_type,
+                                task_id,
+                                num_gpus,
+                                use_core_strategy=False):
     d, master_target, config = self._get_test_object(task_type, task_id,
                                                      num_gpus)
     with ops.Graph().as_default(), \
@@ -158,7 +213,11 @@ class CollectiveAllReduceStrategyTestBase(
       self.assertLess(error_after, error_before)
       return error_after < error_before
 
-  def _test_complex_model(self, task_type, task_id, num_gpus):
+  def _test_complex_model(self,
+                          task_type,
+                          task_id,
+                          num_gpus,
+                          use_core_strategy=False):
     d, master_target, config = self._get_test_object(task_type, task_id,
                                                      num_gpus)
 
@@ -210,7 +269,11 @@ class CollectiveAllReduceStrategyTestBase(
       sess.run(train_op)
       return True
 
-  def _test_variable_initialization(self, task_type, task_id, num_gpus):
+  def _test_variable_initialization(self,
+                                    task_type,
+                                    task_id,
+                                    num_gpus,
+                                    use_core_strategy=False):
     distribution, master_target, config = self._get_test_object(
         task_type, task_id, num_gpus)
     with ops.Graph().as_default(), \
@@ -239,8 +302,14 @@ class CollectiveAllReduceStrategyTestBase(
                                                        reduced_x_value)))
     return np.allclose(x_value, reduced_x_value, atol=1e-5)
 
-  def _test_input_fn_iterator(self, task_type, task_id, num_gpus, input_fn,
-                              expected_values, test_reinitialize=True):
+  def _test_input_fn_iterator(self,
+                              task_type,
+                              task_id,
+                              num_gpus,
+                              input_fn,
+                              expected_values,
+                              test_reinitialize=True,
+                              use_core_strategy=False):
     distribution, master_target, config = self._get_test_object(
         task_type, task_id, num_gpus)
     devices = distribution.extended.worker_devices
@@ -284,45 +353,72 @@ class DistributedCollectiveAllReduceStrategyTest(
     cls._cluster_spec = multi_worker_test_base.create_in_process_cluster(
         num_workers=3, num_ps=0)
 
-  def test_num_replicas_in_sync(self):
-    distribution = collective_all_reduce_strategy.CollectiveAllReduceStrategy(
-        num_gpus_per_worker=2)
-    distribution.configure(cluster_spec=self._cluster_spec, task_type='worker',
-                           task_id=0)
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def test_num_replicas_in_sync(self, use_core_strategy):
+    distribution, _, _ = create_test_objects(
+        cluster_spec=self._cluster_spec,
+        task_type='worker',
+        task_id=0,
+        num_gpus=2,
+        use_core_strategy=use_core_strategy)
     num_workers = len(self._cluster_spec.get('chief', []) +
                       self._cluster_spec.get('worker', []))
     self.assertEqual(2 * num_workers,
                      distribution.num_replicas_in_sync)
 
   @combinations.generate(
-      combinations.combine(mode=['graph'], num_gpus=[0, 1, 2], required_gpus=1))
-  def testMinimizeLossGraph(self, num_gpus):
-    self._run_between_graph_clients(self._test_minimize_loss_graph,
-                                    self._cluster_spec, num_gpus)
+      combinations.combine(
+          mode=['graph'],
+          num_gpus=[0, 1, 2],
+          required_gpus=1,
+          use_core_strategy=[True, False]))
+  def testMinimizeLossGraph(self, num_gpus, use_core_strategy):
+    self._run_between_graph_clients(
+        self._test_minimize_loss_graph,
+        self._cluster_spec,
+        num_gpus,
+        use_core_strategy=use_core_strategy)
 
   @combinations.generate(
-      combinations.combine(mode=['graph'], num_gpus=[0, 1, 2], required_gpus=1))
-  def testVariableInitialization(self, num_gpus):
+      combinations.combine(
+          mode=['graph'],
+          num_gpus=[0, 1, 2],
+          required_gpus=1,
+          use_core_strategy=[True, False]))
+  def testVariableInitialization(self, num_gpus, use_core_strategy):
     if context.num_gpus() < num_gpus:
       self.skipTest('Not enough GPUs')
     self._run_between_graph_clients(
         self._test_variable_initialization,
         self._cluster_spec,
-        num_gpus=num_gpus)
+        num_gpus=num_gpus,
+        use_core_strategy=use_core_strategy)
 
   @combinations.generate(
-      combinations.combine(mode=['graph'], num_gpus=[0, 1, 2], required_gpus=1))
-  def testComplexModel(self, num_gpus):
+      combinations.combine(
+          mode=['graph'],
+          num_gpus=[0, 1, 2],
+          required_gpus=1,
+          use_core_strategy=[True, False]))
+  def testComplexModel(self, num_gpus, use_core_strategy):
     if context.num_gpus() < num_gpus:
       self.skipTest('Not enough GPUs')
     self._run_between_graph_clients(
-        self._test_complex_model, self._cluster_spec, num_gpus=num_gpus)
+        self._test_complex_model,
+        self._cluster_spec,
+        num_gpus=num_gpus,
+        use_core_strategy=use_core_strategy)
 
   # TODO(yuefengz): Update how we use num_gpus and required_gpus
   @combinations.generate(
-      combinations.combine(mode=['graph'], num_gpus=[0, 1, 2], required_gpus=1,
-                           use_dataset=[True, False]))
-  def testMakeInputFnIterator(self, num_gpus, use_dataset):
+      combinations.combine(
+          mode=['graph'],
+          num_gpus=[0, 1, 2],
+          required_gpus=1,
+          use_dataset=[True, False],
+          use_core_strategy=[True, False]))
+  def testMakeInputFnIterator(self, num_gpus, use_dataset, use_core_strategy):
     if context.num_gpus() < num_gpus:
       self.skipTest('Not enough GPUs')
     if use_dataset:
@@ -342,21 +438,29 @@ class DistributedCollectiveAllReduceStrategyTest(
         expected_num_replicas_in_sync=3*devices_per_worker,
         expected_num_input_pipelines=3,
         expected_input_pipeline_id=1)  # because task_id = 1
-    self._test_input_fn_iterator('worker', 1, num_gpus,
-                                 input_fn, expected_values,
-                                 test_reinitialize=use_dataset)
+    self._test_input_fn_iterator(
+        'worker',
+        1,
+        num_gpus,
+        input_fn,
+        expected_values,
+        test_reinitialize=use_dataset,
+        use_core_strategy=use_core_strategy)
 
-  def testUpdateConfigProto(self):
-    distribution = collective_all_reduce_strategy.CollectiveAllReduceStrategy(
-        num_gpus_per_worker=2)
-    distribution.configure(
-        cluster_spec=self._cluster_spec, task_type='worker', task_id=1)
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testUpdateConfigProto(self, use_core_strategy):
+    strategy, _, _ = self._get_test_object(
+        task_type='worker',
+        task_id=1,
+        num_gpus=2,
+        use_core_strategy=use_core_strategy)
 
     config_proto = config_pb2.ConfigProto(device_filters=['to_be_overridden'])
     rewrite_options = config_proto.graph_options.rewrite_options
     rewrite_options.scoped_allocator_opts.enable_op.append('to_be_removed')
 
-    new_config = distribution.update_config_proto(config_proto)
+    new_config = strategy.update_config_proto(config_proto)
 
     # Verify group leader
     self.assertEqual('/job:worker/replica:0/task:0',
@@ -415,28 +519,41 @@ class LocalCollectiveAllReduceStrategy(
 
   @combinations.generate(
       combinations.combine(
-          mode=['graph', 'eager'], num_gpus=[2, 4], required_gpus=2))
-  def testMinimizeLoss(self, num_gpus):
+          mode=['graph', 'eager'],
+          num_gpus=[2, 4],
+          required_gpus=2,
+          use_core_strategy=[True, False]))
+  def testMinimizeLoss(self, num_gpus, use_core_strategy):
     # Collective ops doesn't support strategy with one device.
     if context.num_gpus() < num_gpus:
       self.skipTest('Not enough GPUs')
     if context.executing_eagerly():
-      strategy, _, _ = self._get_test_object(None, None, num_gpus)
+      strategy, _, _ = self._get_test_object(
+          None, None, num_gpus, use_core_strategy=use_core_strategy)
       self._test_minimize_loss_eager(strategy)
     else:
-      self._test_minimize_loss_graph(None, None, num_gpus)
+      self._test_minimize_loss_graph(
+          None, None, num_gpus, use_core_strategy=use_core_strategy)
 
   @combinations.generate(
-      combinations.combine(mode=['graph'], num_gpus=[2, 4], required_gpus=2))
-  def testComplexModel(self, num_gpus):
+      combinations.combine(
+          mode=['graph'],
+          num_gpus=[2, 4],
+          required_gpus=2,
+          use_core_strategy=[True, False]))
+  def testComplexModel(self, num_gpus, use_core_strategy):
     if context.num_gpus() < num_gpus:
       self.skipTest('Not enough GPUs')
-    self._test_complex_model(None, None, num_gpus)
+    self._test_complex_model(
+        None, None, num_gpus, use_core_strategy=use_core_strategy)
 
   @combinations.generate(
-      combinations.combine(mode=['graph', 'eager'], required_gpus=2,
-                           use_dataset=[True, False]))
-  def testMakeInputFnIterator(self, use_dataset):
+      combinations.combine(
+          mode=['graph', 'eager'],
+          required_gpus=2,
+          use_dataset=[True, False],
+          use_core_strategy=[True, False]))
+  def testMakeInputFnIterator(self, use_dataset, use_core_strategy):
     num_gpus = 2
     if use_dataset:
       fn = lambda: dataset_ops.Dataset.range(5 * num_gpus)
@@ -452,51 +569,77 @@ class LocalCollectiveAllReduceStrategy(
         expected_num_replicas_in_sync=num_gpus,
         expected_num_input_pipelines=1,
         expected_input_pipeline_id=0)
-    self._test_input_fn_iterator(None, None, num_gpus,
-                                 input_fn, expected_values,
-                                 test_reinitialize=use_dataset)
+    self._test_input_fn_iterator(
+        None,
+        None,
+        num_gpus,
+        input_fn,
+        expected_values,
+        test_reinitialize=use_dataset,
+        use_core_strategy=use_core_strategy)
 
-  def testAllReduceSum(self):
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testAllReduceSum(self, use_core_strategy):
     if context.num_gpus() < 2: self.skipTest('Not enough GPUs')
-    distribution, target, config = self._get_test_object(None, None, num_gpus=2)
+    distribution, target, config = self._get_test_object(
+        None, None, num_gpus=2, use_core_strategy=use_core_strategy)
     with self.cached_session(config=config, target=target):
       self._test_all_reduce_sum(distribution)
 
-  def testAllReduceSumGradients(self):
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testAllReduceSumGradients(self, use_core_strategy):
     if context.num_gpus() < 2: self.skipTest('Not enough GPUs')
-    distribution, target, config = self._get_test_object(None, None, num_gpus=2)
+    distribution, target, config = self._get_test_object(
+        None, None, num_gpus=2, use_core_strategy=use_core_strategy)
     with self.cached_session(config=config, target=target):
       self._test_all_reduce_sum_gradients(distribution)
 
-  def testAllReduceSumGradientTape(self):
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testAllReduceSumGradientTape(self, use_core_strategy):
     if context.num_gpus() < 2: self.skipTest('Not enough GPUs')
-    distribution, target, config = self._get_test_object(None, None, num_gpus=2)
+    distribution, target, config = self._get_test_object(
+        None, None, num_gpus=2, use_core_strategy=use_core_strategy)
     with self.cached_session(config=config, target=target):
       self._test_all_reduce_sum_gradient_tape(distribution)
 
-  def testAllReduceMean(self):
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testAllReduceMean(self, use_core_strategy):
     if context.num_gpus() < 2: self.skipTest('Not enough GPUs')
-    distribution, target, config = self._get_test_object(None, None, num_gpus=2)
+    distribution, target, config = self._get_test_object(
+        None, None, num_gpus=2, use_core_strategy=use_core_strategy)
     with self.cached_session(config=config, target=target):
       self._test_all_reduce_mean(distribution)
 
-  def testAllReduceMeanGradients(self):
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testAllReduceMeanGradients(self, use_core_strategy):
     if context.num_gpus() < 2: self.skipTest('Not enough GPUs')
-    distribution, target, config = self._get_test_object(None, None, num_gpus=2)
+    distribution, target, config = self._get_test_object(
+        None, None, num_gpus=2, use_core_strategy=use_core_strategy)
     with self.cached_session(config=config, target=target):
       self._test_all_reduce_mean_gradients(distribution)
 
-  def testAllReduceMeanGradientTape(self):
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testAllReduceMeanGradientTape(self, use_core_strategy):
     if context.num_gpus() < 2: self.skipTest('Not enough GPUs')
-    distribution, target, config = self._get_test_object(None, None, num_gpus=2)
+    distribution, target, config = self._get_test_object(
+        None, None, num_gpus=2, use_core_strategy=use_core_strategy)
     with self.cached_session(config=config, target=target):
       self._test_all_reduce_mean_gradient_tape(distribution)
 
-  def testNumpyIterator(self):
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testNumpyIterator(self, use_core_strategy):
     num_gpus = 2
     if context.num_gpus() < num_gpus:
       self.skipTest('Not enough GPUs')
-    strategy, _, _ = self._get_test_object(None, None, num_gpus)
+    strategy, _, _ = self._get_test_object(
+        None, None, num_gpus=num_gpus, use_core_strategy=use_core_strategy)
     self._test_numpy_iterator(strategy)
 
 
diff --git a/tensorflow/python/distribute/BUILD b/tensorflow/python/distribute/BUILD
index 77ec4a5be3..bc465e1fde 100644
--- a/tensorflow/python/distribute/BUILD
+++ b/tensorflow/python/distribute/BUILD
@@ -283,6 +283,28 @@ py_library(
     ],
 )
 
+py_library(
+    name = "collective_all_reduce_strategy",
+    srcs = ["collective_all_reduce_strategy.py"],
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        ":mirrored_strategy",
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:collective_ops",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:training",
+        "//tensorflow/python/distribute:cross_device_ops",
+        "//tensorflow/python/distribute:cross_device_utils",
+        "//tensorflow/python/distribute:input_lib",
+        "//tensorflow/python/distribute:multi_worker_util",
+        "//tensorflow/python/distribute:numpy_dataset",
+        "//tensorflow/python/distribute:values",
+        "//tensorflow/python/distribute/cluster_resolver:cluster_resolver_lib",
+        "//tensorflow/python/eager:context",
+    ],
+)
+
 py_library(
     name = "multi_worker_util",
     srcs = [
diff --git a/tensorflow/python/distribute/collective_all_reduce_strategy.py b/tensorflow/python/distribute/collective_all_reduce_strategy.py
new file mode 100644
index 0000000000..74d3030270
--- /dev/null
+++ b/tensorflow/python/distribute/collective_all_reduce_strategy.py
@@ -0,0 +1,387 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Class CollectiveAllReduceStrategy implementing DistributionStrategy."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import copy
+
+from tensorflow.core.protobuf import rewriter_config_pb2
+from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib
+from tensorflow.python.distribute import cross_device_utils
+from tensorflow.python.distribute import device_util
+from tensorflow.python.distribute import distribute_lib
+from tensorflow.python.distribute import input_lib
+from tensorflow.python.distribute import mirrored_strategy
+from tensorflow.python.distribute import multi_worker_util
+from tensorflow.python.distribute import numpy_dataset
+from tensorflow.python.distribute import values
+from tensorflow.python.distribute.cluster_resolver import SimpleClusterResolver
+from tensorflow.python.distribute.cluster_resolver import TFConfigClusterResolver
+from tensorflow.python.eager import context
+from tensorflow.python.eager import tape
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import collective_ops
+from tensorflow.python.platform import tf_logging as logging
+
+
+# TODO(yuefengz): support in-graph replication.
+class CollectiveAllReduceStrategy(distribute_lib.DistributionStrategy):
+  """Distribution strategy that uses collective ops for all-reduce.
+
+  It is similar to MirroredStrategy but it uses collective ops for reduction.
+
+  By default it uses all local GPUs or CPU for single-worker training.
+
+  When 'TF_CONFIG' environment variable is given, it parses cluster_spec,
+  task_type and task_id from 'TF_CONFIG' and turns into a multi-worker strategy
+  which mirrores models on GPUs of all machines in a cluster. In the current
+  implementation, it uses all GPUs in a cluster and it assumes all workers have
+  the same number of GPUs.
+  """
+
+  def __init__(self):
+    """Initializes the object."""
+    super(CollectiveAllReduceStrategy, self).__init__(
+        CollectiveAllReduceExtended(self))
+
+
+class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
+  """Implementation of CollectiveAllReduceStrategy."""
+
+  def __init__(self,
+               container_strategy,
+               cluster_resolver=TFConfigClusterResolver()):
+    distribute_lib.DistributionStrategyExtended.__init__(
+        self, container_strategy)
+    self._cross_device_ops = None
+    self._initialize_strategy(cluster_resolver)
+    assert isinstance(self._get_cross_device_ops(),
+                      cross_device_ops_lib.CollectiveAllReduce)
+
+  def _initialize_strategy(self, cluster_resolver):
+    if cluster_resolver.cluster_spec().as_dict():
+      self._initialize_multi_worker(cluster_resolver)
+    else:
+      self._initialize_local(cluster_resolver)
+    # Save the num_gpus_per_worker for configure method.
+    self._num_gpus_per_worker = cluster_resolver.num_accelerators()
+
+  def _initialize_local(self, cluster_resolver):
+    """Initializes the object for local training."""
+    self._is_chief = True
+    self._num_workers = 1
+
+    num_gpus = cluster_resolver.num_accelerators()
+    if num_gpus:
+      local_devices = tuple("/device:GPU:%d" % i for i in range(num_gpus))
+    else:
+      local_devices = ("/device:CPU:0",)
+    self._worker_device = device_util.canonicalize("/device:CPU:0")
+    self._host_input_device = numpy_dataset.SingleDevice(self._worker_device)
+
+    self._collective_keys = cross_device_utils.CollectiveKeys()
+    super(CollectiveAllReduceExtended, self)._initialize_local(local_devices)
+    # TODO(yuefengz): remove num_gpus_per_worker from CollectiveAllReduce.
+    self._cross_device_ops = cross_device_ops_lib.CollectiveAllReduce(
+        num_workers=self._num_workers,
+        num_gpus_per_worker=num_gpus,
+        collective_keys=self._collective_keys)
+
+    self._cluster_spec = None
+    self._task_type = None
+    self._task_id = None
+
+    logging.info("CollectiveAllReduceStrategy with local_devices = %r",
+                 local_devices)
+
+  def _initialize_multi_worker(self, cluster_resolver):
+    """Initializes the object for multi-worker training."""
+    # TODO(yuefengz): The `num_gpus` is only for this particular task. It
+    # assumes all workers have the same number of GPUs. We should remove this
+    # assumption by querying all tasks for their numbers of GPUs.
+    num_gpus = cluster_resolver.num_accelerators()
+    cluster_spec = multi_worker_util.normalize_cluster_spec(
+        cluster_resolver.cluster_spec())
+    task_type = cluster_resolver.task_type
+    task_id = cluster_resolver.task_id
+    if task_type is None or task_id is None:
+      raise ValueError("When `cluster_spec` is given, you must also specify "
+                       "`task_type` and `task_id` in the `cluster_resolver`.")
+    if task_type not in ("chief", "worker"):
+      raise ValueError(
+          "Unrecognized task_type: %r, valid task types are: \"chief\", "
+          "\"worker\"." % task_type)
+
+    self._num_workers = multi_worker_util.worker_count(cluster_spec, task_type)
+    if not self._num_workers:
+      raise ValueError("No `worker` or `chief` tasks can be found in "
+                       "`cluster_spec`.")
+
+    self._is_chief = multi_worker_util.is_chief(cluster_spec, task_type,
+                                                task_id)
+
+    self._worker_device = "/job:%s/task:%d" % (task_type, task_id)
+    self._host_input_device = numpy_dataset.SingleDevice(self._worker_device)
+    if num_gpus:
+      local_devices = tuple("%s/device:GPU:%d" % (self._worker_device, i)
+                            for i in range(num_gpus))
+    else:
+      local_devices = (self._worker_device,)
+
+    self._collective_keys = cross_device_utils.CollectiveKeys()
+    super(CollectiveAllReduceExtended, self)._initialize_local(local_devices)
+    self._input_workers = input_lib.InputWorkers(
+        self._device_map, [(self._worker_device, self.worker_devices)])
+    self._cross_device_ops = cross_device_ops_lib.CollectiveAllReduce(
+        num_workers=self._num_workers,
+        num_gpus_per_worker=num_gpus,
+        collective_keys=self._collective_keys)
+
+    # Add a default device so that ops without specified devices will not end up
+    # on other workers.
+    self._default_device = "/job:%s/task:%d" % (task_type, task_id)
+
+    self._cluster_spec = cluster_spec
+    self._task_type = task_type
+    self._task_id = task_id
+
+    logging.info(
+        "Multi-worker CollectiveAllReduceStrategy with "
+        "cluster_spec = %r, task_type = %r, task_id = %r, "
+        "num_workers = %r, local_devices = %r", cluster_spec.as_dict(),
+        task_type, task_id, self._num_workers, local_devices)
+
+  def _create_variable(self, next_creator, *args, **kwargs):
+    colocate_with = kwargs.pop("colocate_with", None)
+    if colocate_with is None:
+      device_map = self._device_map
+      logical_device = 0  # TODO(josh11b): Get logical device from scope here.
+    elif isinstance(colocate_with, numpy_dataset.SingleDevice):
+      with ops.device(colocate_with.device):
+        return next_creator(*args, **kwargs)
+    else:
+      device_map = colocate_with.device_map
+      logical_device = colocate_with.logical_device
+
+    def _real_mirrored_creator(devices, *args, **kwargs):
+      """Creates one MirroredVariable on the current worker."""
+      unique_var_name = ops.get_default_graph().unique_name(
+          kwargs["name"], mark_as_used=False).rstrip("/")
+      # pylint: disable=protected-access
+      collective_instance_key = self._collective_keys.get_instance_key(
+          key_id=unique_var_name)
+      # Only the first device participles in the broadcast of initial values.
+      group_key = self._collective_keys.get_group_key([devices[0]])
+      group_size = self._num_workers
+      if "initial_value" not in kwargs:
+        raise ValueError("Initial value must be specified.")
+      initial_value = kwargs["initial_value"]
+      if callable(initial_value):
+        initial_value_fn = initial_value
+      else:
+        initial_value_fn = lambda: initial_value
+
+      value_list = []
+      for i, d in enumerate(devices):
+        with ops.init_scope(), ops.device(d):
+          if i == 0:
+            # The initial value fn makes sure variables all initialized to
+            # same values. The first device of the chief worker will send their
+            # variable values to other workers.
+            def _overridden_initial_value_fn(device=d, index=i):  # pylint: disable=g-missing-docstring
+              with ops.device(device):
+                initial_value = initial_value_fn()
+                assert not callable(initial_value)
+                initial_value = ops.convert_to_tensor(initial_value)
+
+                assert index == 0, index
+                if self._num_workers > 1:
+                  if self._is_chief:
+                    bcast_send = collective_ops.broadcast_send(
+                        initial_value, initial_value.shape, initial_value.dtype,
+                        group_size, group_key, collective_instance_key)
+                    with ops.control_dependencies([bcast_send]):
+                      return array_ops.identity(initial_value)
+                  else:
+                    return collective_ops.broadcast_recv(
+                        initial_value.shape, initial_value.dtype, group_size,
+                        group_key, collective_instance_key)
+                return initial_value
+          else:
+            # Give replicas meaningful distinct names:
+            var0name = value_list[0].name.split(":")[0]
+            # We append a / to variable names created on replicas with id > 0 to
+            # ensure that we ignore the name scope and instead use the given
+            # name as the absolute name of the variable.
+            kwargs["name"] = "%s/replica_%d/" % (var0name, i)
+
+            # Variables on non-first replica get initial values from the
+            # variables created on the first device of each worker.
+            def _overridden_initial_value_fn(device=d, index=i):
+              assert index > 0
+              with ops.device(device):
+                if context.executing_eagerly():
+                  return array_ops.identity(value_list[0].value())
+                else:
+                  return array_ops.identity(value_list[0].initial_value)
+
+          kwargs["initial_value"] = _overridden_initial_value_fn
+          with context.context().device_policy(context.DEVICE_PLACEMENT_SILENT):
+            # Don't record operations (e.g. other variable reads) during
+            # variable creation.
+            with tape.stop_recording():
+              v = next_creator(*args, **kwargs)
+
+          if i == 0:
+            actual_var_name = v.name.split(":")[0]
+            assert unique_var_name == actual_var_name, "%r vs %r" % (
+                unique_var_name, actual_var_name)
+          assert not isinstance(v, values.DistributedVariable)
+          value_list.append(v)
+      return value_list
+
+    # pylint: disable=protected-access
+    return mirrored_strategy._create_mirrored_variable(
+        self._container_strategy(), device_map, logical_device,
+        _real_mirrored_creator, *args, **kwargs)
+
+  def _make_dataset_iterator(self, dataset):
+    return input_lib.DatasetIterator(dataset, self._input_workers,
+                                     self._num_replicas_in_sync)
+
+  def _make_input_fn_iterator(
+      self,
+      input_fn,
+      replication_mode=distribute_lib.InputReplicationMode.PER_WORKER):
+    """Distributes the dataset to each local GPU."""
+    if self._cluster_spec is None:
+      input_pipeline_id = 0
+    else:
+      input_pipeline_id = multi_worker_util.id_in_cluster(
+          self._cluster_spec, self._task_type, self._task_id)
+    input_context = distribute_lib.InputContext(
+        num_input_pipelines=self._num_workers,
+        input_pipeline_id=input_pipeline_id,
+        num_replicas_in_sync=self._num_replicas_in_sync)
+
+    return input_lib.InputFunctionIterator(
+        input_fn, self._input_workers, [input_context])
+
+  def _configure(self,
+                 session_config=None,
+                 cluster_spec=None,
+                 task_type=None,
+                 task_id=None):
+    """Configures the object.
+
+    Args:
+      session_config: a `tf.ConfigProto`
+      cluster_spec: a dict, ClusterDef or ClusterSpec object specifying the
+        cluster configurations.
+      task_type: the current task type, such as "worker".
+      task_id: the current task id.
+
+    Raises:
+      ValueError: if `task_type` is not in the `cluster_spec`.
+    """
+    if cluster_spec:
+      # Use the num_gpus_per_worker recorded in constructor since _configure
+      # doesn't take num_gpus.
+      cluster_resolver = SimpleClusterResolver(
+          cluster_spec=multi_worker_util.normalize_cluster_spec(cluster_spec),
+          task_type=task_type,
+          task_id=task_id,
+          num_accelerators=self._num_gpus_per_worker)
+      self._initialize_multi_worker(cluster_resolver)
+      assert isinstance(self._get_cross_device_ops(),
+                        cross_device_ops_lib.CollectiveAllReduce)
+
+    if session_config:
+      session_config.CopyFrom(self._update_config_proto(session_config))
+
+  def _update_config_proto(self, config_proto):
+    updated_config = copy.deepcopy(config_proto)
+    # Enable the scoped allocator optimization for CollectiveOps.  This
+    # optimization converts many small all-reduces into fewer larger
+    # all-reduces.
+    rewrite_options = updated_config.graph_options.rewrite_options
+    rewrite_options.scoped_allocator_optimization = (
+        rewriter_config_pb2.RewriterConfig.ON)
+    # We turn on ScopedAllocator only for CollectiveReduce op, i.e. enable_op =
+    # ["CollectiveReduce"].  Since we can't assign to a repeated proto field, we
+    # clear and then append.
+    del rewrite_options.scoped_allocator_opts.enable_op[:]
+    rewrite_options.scoped_allocator_opts.enable_op.append("CollectiveReduce")
+
+    if not self._cluster_spec:
+      return updated_config
+
+    assert self._task_type
+    assert self._task_id is not None
+
+    # Collective group leader is needed for collective ops to coordinate
+    # workers.
+    if "chief" in self._cluster_spec.jobs:
+      updated_config.experimental.collective_group_leader = (
+          "/job:chief/replica:0/task:0")
+    else:
+      if "worker" not in self._cluster_spec.jobs:
+        raise ValueError(
+            "You must have `chief` or `worker` jobs in the `cluster_spec`.")
+      updated_config.experimental.collective_group_leader = (
+          "/job:worker/replica:0/task:0")
+
+    # The device filters prevent communication between workers.
+    del updated_config.device_filters[:]
+    updated_config.device_filters.append(
+        "/job:%s/task:%d" % (self._task_type, self._task_id))
+
+    return updated_config
+
+  @property
+  def experimental_between_graph(self):
+    return True
+
+  @property
+  def experimental_should_init(self):
+    return True
+
+  @property
+  def should_checkpoint(self):
+    return self._is_chief
+
+  @property
+  def should_save_summary(self):
+    return self._is_chief
+
+  @property
+  def _num_replicas_in_sync(self):
+    return len(self.worker_devices) * self._num_workers
+
+  # TODO(priyag): Delete this once all strategies use global batch size.
+  @property
+  def _global_batch_size(self):
+    """`make_dataset_iterator` and `make_numpy_iterator` use global batch size.
+
+    `make_input_fn_iterator` assumes per-replica batching.
+
+    Returns:
+      Boolean.
+    """
+    return True
-- 
GitLab


From 131270d780a85ccb97e1f7c6e1c92dcee176bc28 Mon Sep 17 00:00:00 2001
From: Karl Weinmeister <11586922+kweinmeister@users.noreply.github.com>
Date: Fri, 1 Feb 2019 19:57:33 -0600
Subject: [PATCH 1805/2345] Changed tensor pseudo-code so that the example is
 runnable

---
 tensorflow/python/data/ops/dataset_ops.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index 6f90b52e69..338cfb331e 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -1101,18 +1101,15 @@ class DatasetV2(object):
     """Filters this dataset according to `predicate`.
 
     ```python
-    # NOTE: The following examples use `{ ... }` to represent the
-    # contents of a dataset.
-    a = { 1, 2, 3 }
-    b = { 4, 5, 6, 7 }
-
-    a.filter(lambda x: x < 3) == { 1, 2 }
+    d = tf.data.Dataset.from_tensor_slices([1, 2, 3])
+    
+    d = d.filter(lambda x: x < 3) # [1, 2]
 
     # `tf.math.equal(x, y)` is required for equality comparison
     def filter_fn(x):
-      return tf.math.equal(x, 4)
+      return tf.math.equal(x, 1)
 
-    b.filter(filter_fn) == { 4 }
+    d = d.filter(filter_fn) # [1]
     ```
 
     Args:
-- 
GitLab


From 16d10052a2e2d36ab457c3d3a2f5212f6688efe9 Mon Sep 17 00:00:00 2001
From: Shivani Agrawal <shivaniagrawal@google.com>
Date: Fri, 1 Feb 2019 17:46:46 -0800
Subject: [PATCH 1806/2345] [data-stats] Removes the code which was added to
 handle nodes generated in first iteration, this has become redundant after
 the number of iteration of the `tf_data_meta_optimizer` has been set to 1.

PiperOrigin-RevId: 232068630
---
 .../optimizers/data/latency_all_edges.cc         | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc b/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc
index 4529b89bd4..9bff0685ba 100644
--- a/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc
+++ b/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc
@@ -37,8 +37,8 @@ constexpr char kInsertOpName[] = "ExperimentalLatencyStatsDataset";
 NodeDef MakeLatencyNode(const NodeDef& node, MutableGraphView* graph) {
   NodeDef new_node;
   new_node.set_op(kInsertOpName);
-  graph_utils::SetUniqueGraphNodeName(
-      strings::StrCat(kInsertOpName, "_generated"), graph->graph(), &new_node);
+  graph_utils::SetUniqueGraphNodeName(strings::StrCat(kInsertOpName),
+                                      graph->graph(), &new_node);
   // Set the input of LatencyDataset node as `node`
   new_node.add_input(node.name());
 
@@ -75,8 +75,7 @@ Status LatencyAllEdges::OptimizeAndCollectStats(Cluster* cluster,
   // TODO(shivaniagrawal): Add Op to return Latency for the particular Op than
   // for the edge (e2 - e1?).
   for (const NodeDef& node : item.graph.node()) {
-    if (!str_util::EndsWith(node.op(), "Dataset") || node.attr().empty() ||
-        str_util::EndsWith(node.name(), "_generated")) {
+    if (!str_util::EndsWith(node.op(), "Dataset") || node.attr().empty()) {
       // TODO(b/111805951): Replace this with non-approximate way to check if
       // node corresponds to a `Dataset` op.
       continue;
@@ -87,15 +86,8 @@ Status LatencyAllEdges::OptimizeAndCollectStats(Cluster* cluster,
     if (fanout.size() > 1) {
       LOG(WARNING) << node.name() << " has fanout size " << fanout.size();
       continue;
-    } else {  // fanout will have size 0 for last dataset node in the pipeline.
-      if (fanout.size() == 1) {
-        NodeDef* output_node = (*(fanout.begin())).node;
-        if (str_util::EndsWith(output_node->name(), "_generated")) {
-          continue;
-        }
-      }
     }
-
+    // fanout will have size 0 for last dataset node in the pipeline.
     NodeDef* latency_node = graph.AddNode(MakeLatencyNode(node, &graph));
     TF_RETURN_IF_ERROR(graph.UpdateFanouts(node.name(), latency_node->name()));
     stats->num_changes++;
-- 
GitLab


From 882191e4041c1ab5595f6c70c5aae7ec63206204 Mon Sep 17 00:00:00 2001
From: Wenhao Jia <jiawenhao@google.com>
Date: Fri, 1 Feb 2019 17:50:16 -0800
Subject: [PATCH 1807/2345] Add fine-grained status synchronization methods,
 RefreshStatus, to Device and Stream, so the executor can use these methods
 instead of the coarse-grained Sync and BlockHostUntilDone methods to update
 execution status at session completion on supported devices.

PiperOrigin-RevId: 232068945
---
 tensorflow/compiler/jit/xla_device.cc              |  5 +++--
 tensorflow/compiler/jit/xla_device.h               |  3 +--
 tensorflow/core/common_runtime/device.h            | 14 ++++++++++----
 tensorflow/core/common_runtime/executor.cc         |  8 ++++----
 tensorflow/stream_executor/stream.cc               |  6 ++++++
 tensorflow/stream_executor/stream.h                | 11 +++++++++++
 .../stream_executor/stream_executor_internal.h     |  4 ++++
 .../stream_executor/stream_executor_pimpl.cc       |  4 ++++
 tensorflow/stream_executor/stream_executor_pimpl.h |  3 +++
 9 files changed, 46 insertions(+), 12 deletions(-)

diff --git a/tensorflow/compiler/jit/xla_device.cc b/tensorflow/compiler/jit/xla_device.cc
index e2397f6fcb..c67b4f11b0 100644
--- a/tensorflow/compiler/jit/xla_device.cc
+++ b/tensorflow/compiler/jit/xla_device.cc
@@ -479,7 +479,7 @@ bool XlaDevice::AllowsSyncOnCompletion() const {
   return sync_on_completion_;
 }
 
-Status XlaDevice::CurrentStatus() {
+Status XlaDevice::RefreshStatus() {
   std::shared_ptr<se::Stream> stream;
   {
     mutex_lock lock(mu_);
@@ -488,7 +488,8 @@ Status XlaDevice::CurrentStatus() {
   if (!stream) {
     return Status::OK();
   }
-  return stream->ok() ? Status::OK() : errors::Internal("XlaDevice is not OK.");
+  // Stream status is XlaDevice status, no extra operations needed.
+  return stream->RefreshStatus();
 }
 
 XlaDeviceOpRegistrations* RegisterXlaDeviceKernels(const char* device,
diff --git a/tensorflow/compiler/jit/xla_device.h b/tensorflow/compiler/jit/xla_device.h
index e35a1c7d29..5fe1290fa0 100644
--- a/tensorflow/compiler/jit/xla_device.h
+++ b/tensorflow/compiler/jit/xla_device.h
@@ -169,10 +169,9 @@ class XlaDevice : public LocalDevice {
   // Instructs this XlaDevice to return 'sync_on_completion' for
   // AllowsSyncOnCompletion().
   void SetAllowsSyncOnCompletion(bool sync_on_completion) LOCKS_EXCLUDED(mu_);
-
   bool AllowsSyncOnCompletion() const override LOCKS_EXCLUDED(mu_);
 
-  Status CurrentStatus() override LOCKS_EXCLUDED(mu_);
+  Status RefreshStatus() override LOCKS_EXCLUDED(mu_);
 
  private:
   xla::LocalClient* client() const;
diff --git a/tensorflow/core/common_runtime/device.h b/tensorflow/core/common_runtime/device.h
index 5a0ef28ff2..64119e85c7 100644
--- a/tensorflow/core/common_runtime/device.h
+++ b/tensorflow/core/common_runtime/device.h
@@ -127,16 +127,22 @@ class Device : public DeviceBase {
   // flag settings. Override this to return false for devices that don't allow
   // such calls. Instead, these devices must use other mechanisms (such as
   // num_deferred_ops) to ensure the device has finished processing necessary
-  // work at session completion.
+  // work at session completion. In addition, for these devices, RefreshStatus
+  // must be called at session completion to retrieve execution result status.
   //
-  // Devices that override this function must also implement CurrentStatus.
+  // Devices that override this function must also implement RefreshStatus.
   virtual bool AllowsSyncOnCompletion() const { return true; }
 
   // This is used in conjunction with AllowsSyncOnCompletion to allow the
   // executor to get execution result status at session completion.
-  virtual Status CurrentStatus() {
+  //
+  // For supported devices, this call returns the underlying device stream's
+  // current status in a non-blocking way, without using blocking calls such as
+  // Stream::BlockHostUntilDone or Device::Sync. When applicable, the device
+  // status is also updated with the retrieved stream status.
+  virtual Status RefreshStatus() {
     return errors::Unimplemented(
-        "CurrentStatus is not supported on this device.");
+        "RefreshStatus is not supported on this device.");
   }
 
   // Optionally modify the device's GraphDef before execution.
diff --git a/tensorflow/core/common_runtime/executor.cc b/tensorflow/core/common_runtime/executor.cc
index d068bbf1e4..05f3e851f7 100644
--- a/tensorflow/core/common_runtime/executor.cc
+++ b/tensorflow/core/common_runtime/executor.cc
@@ -2432,11 +2432,11 @@ void ExecutorState::Finish() {
 
   // There are several potential race conditions below. To name a few:
   // 1. Even if the device's status is OK at the precise moment when
-  // num_deferred_ops_ reaches 0, it could go bad before device->CurrentStatus()
+  // num_deferred_ops_ reaches 0, it could go bad before device->RefreshStatus()
   // is called below, caused by work enqueued onto the same device by other
   // concurrent ExecutorState objects.
-  // 2. Some implementations of Device::CurrentStatus, such as
-  // XlaDevice::CurrentStatus, may be inherently racy because it releases the
+  // 2. Some implementations of Device::RefreshStatus, such as
+  // XlaDevice::RefreshStatus, may be inherently racy because it releases the
   // device mutex after a stream pointer is acquired and before the stream is
   // queried for status.
   // 3. It's the same for some implementations of Device::Sync, such as
@@ -2462,7 +2462,7 @@ void ExecutorState::Finish() {
   // these devices should have used num_deferred_ops correctly to ensure the
   // device has finished all relevant work at this point.
   if (!device->AllowsSyncOnCompletion()) {
-    status.Update(device->CurrentStatus());
+    status.Update(device->RefreshStatus());
     delete this;
     runner([=]() { done_cb(status); });
     return;
diff --git a/tensorflow/stream_executor/stream.cc b/tensorflow/stream_executor/stream.cc
index bd89147a26..2577d3825f 100644
--- a/tensorflow/stream_executor/stream.cc
+++ b/tensorflow/stream_executor/stream.cc
@@ -281,6 +281,12 @@ Stream::~Stream() {
   }
 }
 
+port::Status Stream::RefreshStatus() {
+  port::Status status = parent_->GetStatus(this);
+  CheckStatus(status);
+  return status;
+}
+
 Stream &Stream::Init() {
   VLOG_CALL();
 
diff --git a/tensorflow/stream_executor/stream.h b/tensorflow/stream_executor/stream.h
index dd4b58e602..3e67d55922 100644
--- a/tensorflow/stream_executor/stream.h
+++ b/tensorflow/stream_executor/stream.h
@@ -109,6 +109,17 @@ class Stream {
   // stream.
   bool ok() const { return !InErrorState(); }
 
+  // Retrieves execution status back into the stream from the underlying
+  // implementation without blocking the stream.
+  //
+  // Normally, Stream::BlockHostUntilDone is used to get execution status.
+  // However, some devices use out-of-band mechnanisms to ensure their streams
+  // have finished on-device work, without needing to block the streams. (These
+  // devices should also override AllowsSyncOnCompletion to return false.) For
+  // these devices, this method can be used after work is finished to retrieve
+  // execution status.
+  port::Status RefreshStatus() LOCKS_EXCLUDED(mu_);
+
   // Initialize the stream. This must be performed before entraining any other
   // operations.
   Stream &Init() LOCKS_EXCLUDED(mu_);
diff --git a/tensorflow/stream_executor/stream_executor_internal.h b/tensorflow/stream_executor/stream_executor_internal.h
index 6138554e9e..e234e5d7e1 100644
--- a/tensorflow/stream_executor/stream_executor_internal.h
+++ b/tensorflow/stream_executor/stream_executor_internal.h
@@ -253,6 +253,10 @@ class StreamExecutorInterface {
   virtual bool StartTimer(Stream *stream, Timer *timer) = 0;
   virtual bool StopTimer(Stream *stream, Timer *timer) = 0;
   virtual port::Status BlockHostUntilDone(Stream *stream) = 0;
+  virtual port::Status GetStatus(Stream *stream) {
+    return port::Status(port::error::UNIMPLEMENTED,
+                        "GetStatus is not supported on this executor.");
+  }
   virtual int PlatformDeviceCount() = 0;
   virtual port::Status EnablePeerAccessTo(StreamExecutorInterface *other) = 0;
   virtual bool CanEnablePeerAccessTo(StreamExecutorInterface *other) = 0;
diff --git a/tensorflow/stream_executor/stream_executor_pimpl.cc b/tensorflow/stream_executor/stream_executor_pimpl.cc
index 6f0ba51ec6..c680a02e9c 100644
--- a/tensorflow/stream_executor/stream_executor_pimpl.cc
+++ b/tensorflow/stream_executor/stream_executor_pimpl.cc
@@ -492,6 +492,10 @@ port::Status StreamExecutor::BlockHostUntilDone(Stream *stream) {
   return result;
 }
 
+port::Status StreamExecutor::GetStatus(Stream *stream) {
+  return implementation_->GetStatus(stream);
+}
+
 void *StreamExecutor::Allocate(uint64 size) {
   if (memory_limit_bytes_ > 0 &&
       mem_alloc_bytes_ + size > memory_limit_bytes_) {
diff --git a/tensorflow/stream_executor/stream_executor_pimpl.h b/tensorflow/stream_executor/stream_executor_pimpl.h
index ad2bc3c733..508273e112 100644
--- a/tensorflow/stream_executor/stream_executor_pimpl.h
+++ b/tensorflow/stream_executor/stream_executor_pimpl.h
@@ -524,6 +524,9 @@ class StreamExecutor {
   // operations enqueued on the stream before this program point.
   port::Status BlockHostUntilDone(Stream *stream);
 
+  // Without blocking the device, retrieve the current stream status.
+  port::Status GetStatus(Stream *stream);
+
   // Synchronously allocates size bytes on the underlying platform and returns
   // an opaque void* representing that allocation. In the case of failure,
   // nullptr is returned.
-- 
GitLab


From 6477b46119ddc8e4b6a0fb1b399a96cd98f3f904 Mon Sep 17 00:00:00 2001
From: Mahmoud Abuzaina <mahmoud.abuzaina@intel.com>
Date: Fri, 1 Feb 2019 18:11:53 -0800
Subject: [PATCH 1808/2345] Addressed review comments

---
 tensorflow/core/kernels/mkl_concat_op.cc      | 31 +++++++++----------
 .../kernels/mkl_quantized_concat_op_test.cc   | 26 ++++++++--------
 tensorflow/core/ops/mkl_array_ops.cc          |  8 ++---
 3 files changed, 31 insertions(+), 34 deletions(-)

diff --git a/tensorflow/core/kernels/mkl_concat_op.cc b/tensorflow/core/kernels/mkl_concat_op.cc
index a955a90990..ba6c0f2389 100644
--- a/tensorflow/core/kernels/mkl_concat_op.cc
+++ b/tensorflow/core/kernels/mkl_concat_op.cc
@@ -228,9 +228,9 @@ class MklConcatOp : public OpKernel {
 
       OpInputList input_mins, input_maxes;
       if (std::is_same<T, qint8>::value || std::is_same<T, quint8>::value) {
-        // MKL DNN concat does not support input tensors that have different
-        // ranges, check if the ranges of the all input tensors are the same
-        // if not, forward it to Eigen implementation.
+        // MKL-DNN concat does not support input tensors that have different
+        // ranges. Check if the ranges of the all input tensors are the same.
+        // If not, forward it to Eigen implementation.
 
         OP_REQUIRES_OK(context, context->input_list("input_mins", &input_mins));
         OP_REQUIRES(context, (input_mins.size() == N),
@@ -247,7 +247,7 @@ class MklConcatOp : public OpKernel {
         float input_min = input_mins[0].flat<float>()(0);
         float input_max = input_maxes[0].flat<float>()(0);
         const float eps = 1.0e-6;
-        for (int i = 1; i < N; i++) {
+        for (int i = 1; i < N; ++i) {
           float min = input_mins[i].flat<float>()(0);
           float max = input_maxes[i].flat<float>()(0);
 
@@ -260,19 +260,16 @@ class MklConcatOp : public OpKernel {
 
       // Call Eigen library
       if (invoke_eigen) {
-        if (std::is_same<T, qint8>::value || std::is_same<T, quint8>::value) {
-          // MKL DNN quantized concat does not support input tensors with
-          // different ranges.
-          // TODO (mabuzain): Add quantized version of CallEigen() to support
-          // this case.
-          OP_REQUIRES(context, false,
-                      errors::Unimplemented("MKL DNN quantized concat does not "
-                                            "support input tensors that have "
-                                            "different ranges"));
-        } else {
-          CallEigenVersion(context, input_tensors, mkl_input_shapes);
-        }
-
+        // MKL-DNN quantized concat does not support input tensors with
+        // different ranges.
+        // TODO (mabuzain): Add quantized version of CallEigen() to support
+        // this case.
+        OP_REQUIRES(context, (!std::is_same<T, qint8>::value &&
+                              !std::is_same<T, quint8>::value),
+                    errors::Unimplemented("MKL DNN quantized concat does not "
+                                          "support input tensors that have "
+                                          "different ranges"));
+        CallEigenVersion(context, input_tensors, mkl_input_shapes);
         return;
       }
 
diff --git a/tensorflow/core/kernels/mkl_quantized_concat_op_test.cc b/tensorflow/core/kernels/mkl_quantized_concat_op_test.cc
index f9098f960a..d483b85d48 100644
--- a/tensorflow/core/kernels/mkl_quantized_concat_op_test.cc
+++ b/tensorflow/core/kernels/mkl_quantized_concat_op_test.cc
@@ -1,4 +1,4 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -41,12 +41,12 @@ namespace tensorflow {
 
 using test::graph::Constant;
 
-// Helper class for converting MKL tesnors to TF tensors and comparing to
-// expected values
-
 static const uint8 dummy_tensor[] = {0, 0, 0, 0, 0, 0, 0, 0};
 static const TensorShape dummy_shape({8});
 
+// Helper class for converting MKL tensors to TF tensors and comparing to
+// expected values
+
 class ConvMklToTF : public OpsTestBase {
  public:
   template <typename T>
@@ -55,7 +55,7 @@ class ConvMklToTF : public OpsTestBase {
     // Create an MKL to TF conversion node and execute it
     TF_EXPECT_OK(NodeDefBuilder("mkl_to_tf_op", "_MklToTf")
                      .Input(FakeInput(dtype))     // Input
-                     .Input(FakeInput(DT_UINT8))  // Mkl second tensor
+                     .Input(FakeInput(DT_UINT8))  // MKL second tensor
                      .Attr("T", dtype)
                      .Attr("_kernel", "MklOp")
                      .Finalize(node_def()));
@@ -91,10 +91,10 @@ void QuantizedConcatTest::TestSmall8Bit(float first_min, float first_max,
                    .Input(FakeInput(DT_INT32))
                    .Input(FakeInput(2, DT_FLOAT))
                    .Input(FakeInput(2, DT_FLOAT))
-                   .Input(FakeInput(2, DT_UINT8))  // MKl second tensor
-                   .Input(FakeInput(DT_UINT8))     // MKl second tensor
-                   .Input(FakeInput(2, DT_UINT8))  // MKl second tensor
-                   .Input(FakeInput(2, DT_UINT8))  // MKl second tensor
+                   .Input(FakeInput(2, DT_UINT8))  // MKL second tensor
+                   .Input(FakeInput(DT_UINT8))     // MKL second tensor
+                   .Input(FakeInput(2, DT_UINT8))  // MKL second tensor
+                   .Input(FakeInput(2, DT_UINT8))  // MKL second tensor
                    .Attr("N", 2)
                    .Attr("T", DataTypeToEnum<quint8>::v())
                    .Attr("Tidx", DT_INT32)
@@ -164,10 +164,10 @@ void QuantizedConcatTest::TestSecondDim8Bit(float first_min, float first_max,
                    .Input(FakeInput(DT_INT32))
                    .Input(FakeInput(2, DT_FLOAT))
                    .Input(FakeInput(2, DT_FLOAT))
-                   .Input(FakeInput(2, DT_UINT8))  // MKl second tensor
-                   .Input(FakeInput(DT_UINT8))     // MKl second tensor
-                   .Input(FakeInput(2, DT_UINT8))  // MKl second tensor
-                   .Input(FakeInput(2, DT_UINT8))  // MKl second tensor
+                   .Input(FakeInput(2, DT_UINT8))  // MKL second tensor
+                   .Input(FakeInput(DT_UINT8))     // MKL second tensor
+                   .Input(FakeInput(2, DT_UINT8))  // MKL second tensor
+                   .Input(FakeInput(2, DT_UINT8))  // MKL second tensor
                    .Attr("N", 2)
                    .Attr("T", DataTypeToEnum<quint8>::v())
                    .Attr("Tidx", DT_INT32)
diff --git a/tensorflow/core/ops/mkl_array_ops.cc b/tensorflow/core/ops/mkl_array_ops.cc
index ad15d0bf88..0279401289 100644
--- a/tensorflow/core/ops/mkl_array_ops.cc
+++ b/tensorflow/core/ops/mkl_array_ops.cc
@@ -1,4 +1,4 @@
-/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -33,8 +33,8 @@ using shape_inference::InferenceContext;
 using shape_inference::ShapeHandle;
 using shape_inference::UnchangedShape;
 
-// Adding QuantizedConcatV2 which is similar to the existing QuantizedConcat
-// op to be able to replace it by _MklQuantizedConcatV2 in the graph rewrite.
+// Adding QuantizedConcatV2 op to be able to replace it by
+// _MklQuantizedConcatV2 in the graph rewrite.
 REGISTER_OP("QuantizedConcatV2")
     .Input("values: N * T")
     .Input("axis: Tidx")
@@ -89,4 +89,4 @@ REGISTER_OP("_MklQuantizedConcatV2")
     });
 }
 
-#endif
\ No newline at end of file
+#endif
-- 
GitLab


From 9018715ac3e7652e247deec9bf72d632143fe0d7 Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Fri, 1 Feb 2019 18:40:52 -0800
Subject: [PATCH 1809/2345] MNIST eager dist strat example: Add an option to
 run the step in a tf.function. Also changed the example to run with TF 2.0
 from compat.

PiperOrigin-RevId: 232073692
---
 .../python/examples/mnist_eager_multigpu.py   | 56 ++++++++++++-------
 1 file changed, 37 insertions(+), 19 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/examples/mnist_eager_multigpu.py b/tensorflow/contrib/distribute/python/examples/mnist_eager_multigpu.py
index 11a3b5e91a..c045a5586b 100644
--- a/tensorflow/contrib/distribute/python/examples/mnist_eager_multigpu.py
+++ b/tensorflow/contrib/distribute/python/examples/mnist_eager_multigpu.py
@@ -25,20 +25,23 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from absl import app
+from absl import flags
 import numpy as np
-import tensorflow as tf
-
-tf.flags.DEFINE_integer("num_gpus", None, "How many GPUs should we run on?"
-                        "Defaults to all available GPUs, otherwise CPU.")
-tf.flags.DEFINE_integer("batch_size", 64,
-                        "What should be the size of each batch?")
-tf.flags.DEFINE_integer("num_epochs", 10, "How many epochs to run?")
-tf.flags.DEFINE_float("learning_rate", 0.01, "Learning Rate")
-tf.flags.DEFINE_float("momentum", 0.5, "SGD momentum")
-
-FLAGS = tf.flags.FLAGS
+import tensorflow.compat.v2 as tf
+
+flags.DEFINE_integer("num_gpus", None, "How many GPUs should we run on?"
+                     "Defaults to all available GPUs, otherwise CPU.")
+flags.DEFINE_integer("batch_size", 64,
+                     "What should be the size of each batch?")
+flags.DEFINE_integer("num_epochs", 10, "How many epochs to run?")
+flags.DEFINE_float("learning_rate", 0.01, "Learning Rate")
+flags.DEFINE_float("momentum", 0.5, "SGD momentum")
+flags.DEFINE_boolean("use_function", False,
+                     "Should we wrap the step in a tf.function.")
+
+FLAGS = flags.FLAGS
 NUM_TRAIN_IMAGES = 60000
-NUM_TEST_IMAGES = 10000
 
 
 def create_model():
@@ -82,7 +85,7 @@ def mnist_datasets():
 def main(unused_argv):
   """Run a CNN model on MNIST data to demonstrate DistributedStrategies."""
 
-  tf.enable_eager_execution()
+  tf.enable_v2_behavior()
 
   num_gpus = FLAGS.num_gpus
   if num_gpus is None:
@@ -99,7 +102,7 @@ def main(unused_argv):
     test_ds = test_ds.batch(FLAGS.batch_size)
 
     model = create_model()
-    optimizer = tf.train.MomentumOptimizer(FLAGS.learning_rate, FLAGS.momentum)
+    optimizer = tf.keras.optimizers.SGD(FLAGS.learning_rate, FLAGS.momentum)
     training_loss = tf.keras.metrics.Mean("training_loss", dtype=tf.float32)
     training_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(
         "training_accuracy", dtype=tf.float32)
@@ -126,12 +129,24 @@ def main(unused_argv):
 
     train_iterator = strategy.make_dataset_iterator(train_ds)
     test_iterator = strategy.make_dataset_iterator(test_ds)
+
     for epoch in range(0, FLAGS.num_epochs):
+      # TODO(b/123315763): Create the tf.function outside this loop once we are
+      # able to initialize iterator in eager mode.
+      dist_train = lambda it: strategy.experimental_run(train_step, it)
+      dist_test = lambda it: strategy.experimental_run(test_step, it)
+      if FLAGS.use_function:
+        dist_train = tf.function(dist_train)
+        dist_test = tf.function(dist_test)
+
       # Train
       print("Starting epoch {}".format(epoch))
       train_iterator.initialize()
-      for _ in range(NUM_TRAIN_IMAGES // FLAGS.batch_size):
-        strategy.experimental_run(train_step, train_iterator)
+      while True:
+        try:
+          dist_train(train_iterator)
+        except tf.errors.OutOfRangeError:
+          break
       print("Training loss: {:0.4f}, accuracy: {:0.2f}%".format(
           training_loss.result(), training_accuracy.result() * 100))
       training_loss.reset_states()
@@ -139,8 +154,11 @@ def main(unused_argv):
 
       # Test
       test_iterator.initialize()
-      for _ in range(NUM_TEST_IMAGES // FLAGS.batch_size):
-        strategy.experimental_run(test_step, test_iterator)
+      while True:
+        try:
+          dist_test(test_iterator)
+        except tf.errors.OutOfRangeError:
+          break
       print("Test loss: {:0.4f}, accuracy: {:0.2f}%".format(
           test_loss.result(), test_accuracy.result() * 100))
       test_loss.reset_states()
@@ -148,4 +166,4 @@ def main(unused_argv):
 
 
 if __name__ == "__main__":
-  tf.app.run()
+  app.run(main)
-- 
GitLab


From ee5f47416e26064dfba40620a54e0f590ea80331 Mon Sep 17 00:00:00 2001
From: Peter Ma <pcma@google.com>
Date: Fri, 1 Feb 2019 18:58:02 -0800
Subject: [PATCH 1810/2345] Slight change to add some debugging loggings.

PiperOrigin-RevId: 232074849
---
 tensorflow/core/grappler/costs/virtual_placer.cc | 6 ++++--
 tensorflow/core/grappler/costs/virtual_placer.h  | 1 -
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/grappler/costs/virtual_placer.cc b/tensorflow/core/grappler/costs/virtual_placer.cc
index 8f5f16e490..146eecf5bc 100644
--- a/tensorflow/core/grappler/costs/virtual_placer.cc
+++ b/tensorflow/core/grappler/costs/virtual_placer.cc
@@ -87,6 +87,7 @@ VirtualPlacer::VirtualPlacer(const Cluster* cluster) {
       default_device_name_ = devices_.begin()->first;  // Any device.
     }
   }
+  VLOG(3) << "default device name: " << default_device_name_;
 
   // Scan the device names from the cluster, and if there is one job name used,
   // use it for canonical device name.
@@ -102,14 +103,15 @@ VirtualPlacer::VirtualPlacer(const Cluster* cluster) {
       }
     }
   }
-  // If there is only  type of job name in all the devices in the cluster, use
-  // that one as default job name; otherwise, use localhost.
+  // If there is only one type of job name in all the devices in the cluster,
+  // use that one as default job name; otherwise, use localhost.
   // TODO(dyoon): this should be improved, especially when the cluster is
   // composed of multiple worker, PS, and other types of jobs.
   if (job_names_from_cluster.size() == 1) {
     auto it = job_names_from_cluster.begin();
     default_job_name_lowercase_ = *it;
   }
+  VLOG(3) << "default job name: " << default_job_name_lowercase_;
 }
 
 const DeviceProperties& VirtualPlacer::get_device(const NodeDef& node) const {
diff --git a/tensorflow/core/grappler/costs/virtual_placer.h b/tensorflow/core/grappler/costs/virtual_placer.h
index fee5ce0f51..e17ece7c1a 100644
--- a/tensorflow/core/grappler/costs/virtual_placer.h
+++ b/tensorflow/core/grappler/costs/virtual_placer.h
@@ -16,7 +16,6 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_GRAPPLER_COSTS_VIRTUAL_PLACER_H_
 #define TENSORFLOW_CORE_GRAPPLER_COSTS_VIRTUAL_PLACER_H_
 
-#include <unordered_map>
 #include "tensorflow/core/platform/types.h"
 #include "tensorflow/core/protobuf/device_properties.pb.h"
 
-- 
GitLab


From 3c2dabf53dd085c21e38a28b467e52c566c0dfaf Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 1 Feb 2019 19:40:02 -0800
Subject: [PATCH 1811/2345] Add a batch_gather_with_default op

PiperOrigin-RevId: 232077748
---
 tensorflow/python/ops/ragged/BUILD            |  26 ++
 tensorflow/python/ops/ragged/__init__.py      |   1 +
 .../ops/ragged/ragged_batch_gather_op_test.py | 320 ++++++++++++++++++
 .../ragged_batch_gather_with_default_op.py    | 186 ++++++++++
 4 files changed, 533 insertions(+)
 create mode 100644 tensorflow/python/ops/ragged/ragged_batch_gather_with_default_op.py

diff --git a/tensorflow/python/ops/ragged/BUILD b/tensorflow/python/ops/ragged/BUILD
index ef95169e9c..4f29fcc41d 100644
--- a/tensorflow/python/ops/ragged/BUILD
+++ b/tensorflow/python/ops/ragged/BUILD
@@ -25,6 +25,7 @@ py_library(
     deps = [
         ":ragged_array_ops",
         ":ragged_batch_gather_ops",
+        ":ragged_batch_gather_with_default_op",
         ":ragged_concat_ops",
         ":ragged_conversion_ops",
         ":ragged_dispatch",
@@ -90,6 +91,29 @@ py_library(
     ],
 )
 
+py_library(
+    name = "ragged_batch_gather_with_default_op",
+    srcs = [
+        "ragged_batch_gather_with_default_op.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":ragged_array_ops",
+        ":ragged_batch_gather_ops",
+        ":ragged_concat_ops",
+        ":ragged_dispatch",
+        ":ragged_operators",
+        ":ragged_tensor",
+        ":ragged_tensor_shape",
+        ":ragged_where_op",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:check_ops",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:math_ops",
+    ],
+)
+
 py_library(
     name = "ragged_concat_ops",
     srcs = ["ragged_concat_ops.py"],
@@ -365,6 +389,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":ragged_array_ops",
+        ":ragged_batch_gather_ops",
         ":ragged_math_ops",
         ":ragged_tensor",
         ":ragged_tensor_shape",
@@ -510,6 +535,7 @@ py_test(
     deps = [
         ":ragged_array_ops",
         ":ragged_batch_gather_ops",
+        ":ragged_batch_gather_with_default_op",
         ":ragged_factory_ops",
         ":ragged_tensor",
         ":ragged_test_util",
diff --git a/tensorflow/python/ops/ragged/__init__.py b/tensorflow/python/ops/ragged/__init__.py
index a5ffd8a950..e9232a1c64 100644
--- a/tensorflow/python/ops/ragged/__init__.py
+++ b/tensorflow/python/ops/ragged/__init__.py
@@ -30,6 +30,7 @@ from __future__ import print_function
 
 from tensorflow.python.ops.ragged import ragged_array_ops
 from tensorflow.python.ops.ragged import ragged_batch_gather_ops
+from tensorflow.python.ops.ragged import ragged_batch_gather_with_default_op
 from tensorflow.python.ops.ragged import ragged_concat_ops
 from tensorflow.python.ops.ragged import ragged_conversion_ops
 from tensorflow.python.ops.ragged import ragged_dispatch
diff --git a/tensorflow/python/ops/ragged/ragged_batch_gather_op_test.py b/tensorflow/python/ops/ragged/ragged_batch_gather_op_test.py
index 72692fcef0..17c55eb810 100644
--- a/tensorflow/python/ops/ragged/ragged_batch_gather_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_batch_gather_op_test.py
@@ -21,11 +21,13 @@ from __future__ import print_function
 from absl.testing import parameterized
 
 from tensorflow.python.eager import context
+from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops.ragged import ragged_batch_gather_ops
+from tensorflow.python.ops.ragged import ragged_batch_gather_with_default_op
 from tensorflow.python.ops.ragged import ragged_factory_ops
 from tensorflow.python.ops.ragged import ragged_tensor
 from tensorflow.python.ops.ragged import ragged_test_util
@@ -149,6 +151,324 @@ class RaggedBatchGatherOpTest(ragged_test_util.RaggedTensorTestCase,
     result = ragged_batch_gather_ops.batch_gather(params, indices)
     self.assertRaggedEqual(result, expected)
 
+  @parameterized.parameters([
+      # Docstring example:
+      dict(
+          descr='Docstring example',
+          params=[['a', 'b', 'c'], ['d'], [], ['e']],
+          indices=[[1, 2, -1], [], [], [0, 10]],
+          expected=[['b', 'c', 'FOO'], [], [], ['e', 'FOO']],
+          default_value='FOO',
+      ),
+      # Dimensions:
+      # indices: [4]
+      # params: [2, (d1), (d2)]
+      dict(
+          descr='params: [2, (d1), (d2), indices: [4]',
+          indices=[1, 100, 0, -1],
+          params=[[['The', 'deal', 'came', 'about', '18', 'months', 'after',
+                    'Yahoo', '!', 'rejected', 'a', '47.5', '-', 'billion', '-',
+                    'dollar', 'takeover', 'offer', 'from', 'Microsoft', '.'],
+                   ['Trumpty', 'Dumpty', 'sat', 'on', 'a', 'wall']],
+                  [["It's", 'always', 'darkest', 'before', 'the', 'dawn']]],
+          expected=[[["It's", 'always', 'darkest', 'before', 'the', 'dawn']],
+                    [['$NONE^']],
+                    [['The', 'deal', 'came', 'about', '18', 'months', 'after',
+                      'Yahoo', '!', 'rejected', 'a', '47.5', '-', 'billion',
+                      '-', 'dollar', 'takeover', 'offer', 'from', 'Microsoft',
+                      '.'],
+                     ['Trumpty', 'Dumpty', 'sat', 'on', 'a', 'wall']],
+                    [['$NONE^']]],
+      ),
+      # Dimensions:
+      # params: [1, (d1)]
+      # indices: [3]
+      dict(
+          descr='params: rank 2, indices: rank 1',
+          params=[
+              ['Bruce', 'Wayne'],
+          ],
+          indices=[-1, 0, 1000],
+          expected=[['$NONE^'], ['Bruce', 'Wayne'], ['$NONE^']]
+      ),
+      # Dimensions:
+      # params: [1, (d1)]
+      # indices: [1, (d2)]
+      dict(
+          descr='Test underbound indices of shape [1, (d2)]',
+          params=[
+              ['The', 'deal', 'came', 'about', '18', 'months', 'after', 'Yahoo',
+               '!', 'rejected', 'a', '47.5', '-', 'billion', '-', 'dollar',
+               'takeover', 'offer', 'from', 'Microsoft', '.'],
+          ],
+          indices=[[8, -1]],
+          expected=[['!', '$NONE^']],
+      ),
+      dict(
+          descr='Test underbound indices of shape [2, (d2)]',
+          params=[
+              ['The', 'deal', 'came', 'about', '18', 'months', 'after', 'Yahoo',
+               '!', 'rejected', 'a', '47.5', '-', 'billion', '-', 'dollar',
+               'takeover', 'offer', 'from', 'Microsoft', '.'],
+              ['Who', 'let', 'the', 'dogs', 'out', '?'],
+          ],
+          indices=[[8, -1], [1, 100]],
+          expected=[['!', '$NONE^'], ['let', '$NONE^']],
+      ),
+      # Dimensions:
+      # params: [2, (d1)]
+      # indices: [2, (d2)]
+      dict(
+          descr='Test underbound indices of rank 2',
+          params=[
+              ['The', 'deal', 'came', 'about', '18', 'months', 'after', 'Yahoo',
+               '!', 'rejected', 'a', '47.5', '-', 'billion', '-', 'dollar',
+               'takeover', 'offer', 'from', 'Microsoft', '.'],
+              ['He', 'left', 'us', '.', 'Little', 'boys', 'crowded', 'together',
+               'on', 'long', 'wooden', 'benches', ',', 'and', 'in', 'the',
+               'center', 'of', 'the', 'room', 'sat', 'the', 'teacher', '.',
+               'His', 'black', 'beard', 'dripped', 'down', 'over', 'the',
+               'front', 'of', 'his', 'coat', '.', 'One', 'white', 'hand',
+               'poised', 'a', 'stick', 'above', 'his', 'desk', '.', 'He',
+               'turned', 'his', 'surly', ',', 'half', '-', 'closed', 'eyes',
+               'toward', 'us', ',', 'stared', 'for', 'a', 'second', ',', 'then',
+               'shouted', 'in', 'Yiddish', ',', '``', 'One', ',', 'two', ',',
+               'three', "''", '!', '!', 'Rapping', 'the', 'stick', 'against',
+               'the', 'desk', '.', 'The', 'little', 'boys', 'shrilled', 'out',
+               'a', 'Yiddish', 'translation', 'or', 'interpretation', 'of',
+               'the', 'Five', 'Books', 'of', 'Moses', ',', 'which', 'they',
+               'had', 'previously', 'chanted', 'in', 'Hebrew', '.']],
+          indices=[[8, -1], [3, 23, 35, 45, 75, 83, -121]],
+          expected=[['!', '$NONE^'], ['.', '.', '.', '.', '!', '.', '$NONE^']],
+      ),
+      dict(
+          descr='Test overbound indices of rank 2',
+          params=[
+              ['The', 'deal', 'came', 'about', '18', 'months', 'after', 'Yahoo',
+               '!', 'rejected', 'a', '47.5', '-', 'billion', '-', 'dollar',
+               'takeover', 'offer', 'from', 'Microsoft', '.'],
+              ['He', 'left', 'us', '.', 'Little', 'boys', 'crowded', 'together',
+               'on', 'long', 'wooden', 'benches', ',', 'and', 'in', 'the',
+               'center', 'of', 'the', 'room', 'sat', 'the', 'teacher', '.',
+               'His', 'black', 'beard', 'dripped', 'down', 'over', 'the',
+               'front', 'of', 'his', 'coat', '.', 'One', 'white', 'hand',
+               'poised', 'a', 'stick', 'above', 'his', 'desk', '.', 'He',
+               'turned', 'his', 'surly', ',', 'half', '-', 'closed', 'eyes',
+               'toward', 'us', ',', 'stared', 'for', 'a', 'second', ',', 'then',
+               'shouted', 'in', 'Yiddish', ',', '``', 'One', ',', 'two', ',',
+               'three', "''", '!', '!', 'Rapping', 'the', 'stick', 'against',
+               'the', 'desk', '.', 'The', 'little', 'boys', 'shrilled', 'out',
+               'a', 'Yiddish', 'translation', 'or', 'interpretation', 'of',
+               'the', 'Five', 'Books', 'of', 'Moses', ',', 'which', 'they',
+               'had', 'previously', 'chanted', 'in', 'Hebrew', '.']],
+          indices=[[8, 8823], [3, 23, 35, 45, 75, 83, 1234]],
+          expected=[['!', '$NONE^'], ['.', '.', '.', '.', '!', '.', '$NONE^']],
+      ),
+      # Dimensions:
+      # params: [2, (d1), 2]
+      # indices: [2, (d2)]
+      dict(
+          descr='params: rank 3, indices: rank 2',
+          params=[
+              [['The', 'deal'], ['takeover', 'offer'], ['from', 'Microsoft']],
+              [['Who', 'let'], ['the', 'dogs'], ['out', '?']],
+          ],
+          ragged_rank=1,
+          indices=[[1, -1, 2, 30], [1, 100]],
+          indices_ragged_rank=1,
+          expected=[[['takeover', 'offer'],
+                     ['$NONE^', '$NONE^'],
+                     ['from', 'Microsoft'],
+                     ['$NONE^', '$NONE^']],
+                    [['the', 'dogs'],
+                     ['$NONE^', '$NONE^']]],
+          expected_ragged_rank=1,
+          default_value=['$NONE^', '$NONE^'],
+      ),
+      # Dimensions:
+      # params: [2, (d1), (d2)]
+      # indices: [2, (d3)]
+      dict(
+          descr='params: [2, (d1), (d2)], indices: [2, (d3)]',
+          params=[
+              [['The', 'deal', 'came', 'about', '18', 'months', 'after',
+                'Yahoo', '!', 'rejected', 'a', '47.5', '-', 'billion', '-',
+                'dollar', 'takeover', 'offer', 'from', 'Microsoft', '.'],
+               ['Trumpty', 'Dumpty', 'sat', 'on', 'a', 'wall'],
+              ],
+              [['It\'s', 'always', 'darkest', 'before', 'the', 'dawn']]
+          ],
+          indices=[[1, 100], [0, -1]],
+          expected=[[['Trumpty', 'Dumpty', 'sat', 'on', 'a', 'wall'],
+                     ['$NONE^']],
+                    [["It's", 'always', 'darkest', 'before', 'the', 'dawn'],
+                     ['$NONE^']]]
+      ),
+      # Dimensions:
+      # params: [2, (d1), (d2)]
+      # indices: [2, (d1), (d3)]
+      dict(
+          descr='Test overbound indices of rank 3',
+          params=[
+              [['The', 'deal', 'came', 'about', '18', 'months', 'after',
+                'Yahoo', '!', 'rejected', 'a', '47.5', '-', 'billion', '-',
+                'dollar', 'takeover', 'offer', 'from', 'Microsoft', '.'],
+               ['Foo', 'bar', 'mar']],
+              [['He', 'left', 'us', '.', 'Little', 'boys', 'crowded',
+                'together', 'on', 'long', 'wooden', 'benches', ',', 'and', 'in',
+                'the', 'center', 'of', 'the', 'room', 'sat', 'the', 'teacher',
+                '.', 'His', 'black', 'beard', 'dripped', 'down', 'over', 'the',
+                'front', 'of', 'his', 'coat', '.', 'One', 'white', 'hand',
+                'poised', 'a', 'stick', 'above', 'his', 'desk', '.', 'He',
+                'turned', 'his', 'surly', ',', 'half', '-', 'closed', 'eyes',
+                'toward', 'us', ',', 'stared', 'for', 'a', 'second', ',',
+                'then', 'shouted', 'in', 'Yiddish', ',', '``', 'One', ',',
+                'two', ',',
+                'three', "''", '!', '!', 'Rapping', 'the', 'stick', 'against',
+                'the', 'desk', '.', 'The', 'little', 'boys', 'shrilled', 'out',
+                'a', 'Yiddish', 'translation', 'or', 'interpretation', 'of',
+                'the', 'Five', 'Books', 'of', 'Moses', ',', 'which', 'they',
+                'had', 'previously', 'chanted', 'in', 'Hebrew', '.'],
+               ['I', 'too', 'was', 'hustled', 'scammed', 'bamboozled', 'hood',
+                'winked', 'lead', 'astray']]
+          ],
+          indices=[[[8, 8823], [0, 100]], [[3, 23, 35, 45, 75, 83, 1234], [5]]],
+          expected=[[['!', '$NONE^'], ['Foo', '$NONE^']],
+                    [['.', '.', '.', '.', '!', '.', '$NONE^'],
+                     ['bamboozled']]],
+      ),
+      # params.shape = [2, (d1), 8]
+      # indices.shape = [2, (d1), 3]
+      dict(
+          descr='params = [2, (2, 1), 8], indices = [2, (2, 1), 3]',
+          params=[[['h'] * 8, ['w'] * 8], [['b'] * 8]],
+          ragged_rank=1,
+          indices=[[[0, 100, 1], [0, 1, 0]], [[1, 0, 0]]],
+          indices_ragged_rank=1,
+          expected=[[['h', '$NONE^', 'h'], ['w', 'w', 'w']], [['b', 'b', 'b']]],
+          expected_ragged_rank=1,
+      ),
+  ])
+  def testRaggedBatchGatherWithDefault(
+      self, descr, params, indices, expected, indices_ragged_rank=None,
+      expected_ragged_rank=None, ragged_rank=None, default_value='$NONE^'):
+    params = ragged_factory_ops.constant(params, ragged_rank=ragged_rank)
+    indices = ragged_factory_ops.constant(
+        indices, ragged_rank=indices_ragged_rank or ragged_rank)
+    expected = ragged_factory_ops.constant(
+        expected, ragged_rank=expected_ragged_rank or ragged_rank)
+    result = ragged_batch_gather_with_default_op.batch_gather_with_default(
+        params, indices, default_value)
+    self.assertRaggedEqual(result, expected)
+
+  @parameterized.parameters([
+      # Dimensions:
+      #  params: dims [2, 5], indices: [2, 2]
+      dict(
+          descr='params: dims [2, 5], indices: [2, 2]',
+          params=[
+              ['The', 'deal', 'came', 'about', '18'],
+              ['He', 'left', 'us', '.', 'Little']],
+          indices=[[0, -1], [3, 121]],
+          expected=[['The', '$NONE^'], ['.', '$NONE^']],
+          default_value='$NONE^',
+      ),
+      # Dimensions:
+      #  params: dims [2, 2, 5], indices: [2, 2]
+      dict(
+          descr='params: dims [2, 2, 5], indices: [2, 2]',
+          params=[
+              [['The', 'deal', 'came', 'about', '18'],
+               ['The', 'deal', 'came', 'about', '19'],
+              ],
+              [['He', 'left', 'us', '.', 'Little'],
+               ['The', 'deal', 'came', 'about', '20'],
+              ]
+          ],
+          indices=[[0, -1], [0, 121]],
+          expected=[[['The', 'deal', 'came', 'about', '18'],
+                     ['$NONE^', '$NONE^', '$NONE^', '$NONE^', '$NONE^']],
+                    [['He', 'left', 'us', '.', 'Little'],
+                     ['$NONE^', '$NONE^', '$NONE^', '$NONE^', '$NONE^']]],
+          default_value='$NONE^',
+      ),
+      # Test default_value with shape [5]
+      dict(
+          descr='params: dims [2, 2, 5], indices: [2, 2]',
+          params=[
+              [['The', 'deal', 'came', 'about', '18'],
+               ['The', 'deal', 'came', 'about', '19'],
+              ],
+              [['He', 'left', 'us', '.', 'Little'],
+               ['The', 'deal', 'came', 'about', '20'],
+              ]
+          ],
+          indices=[[0, -1], [0, 121]],
+          expected=[[['The', 'deal', 'came', 'about', '18'],
+                     [':FOO:', ':FOO:', ':FOO:', ':FOO:', ':FOO:']],
+                    [['He', 'left', 'us', '.', 'Little'],
+                     [':FOO:', ':FOO:', ':FOO:', ':FOO:', ':FOO:']]],
+          default_value=[':FOO:', ':FOO:', ':FOO:', ':FOO:', ':FOO:'],
+      ),
+  ])
+  def testRaggedBatchGatherWithDefaultOnTensors(
+      self, descr, params, indices, expected, default_value):
+    params = constant_op.constant(params)
+    indices = constant_op.constant(indices)
+    expected = constant_op.constant(expected)
+    result = ragged_batch_gather_with_default_op.batch_gather_with_default(
+        params, indices, default_value)
+    self.assertAllEqual(expected, result)
+
+  @parameterized.parameters([
+      dict(
+          params=[['The', 'deal', 'came', 'about', '18', 'months', 'after',
+                   'Yahoo', '!', 'rejected', 'a', '47.5', '-', 'billion', '-',
+                   'dollar', 'takeover', 'offer', 'from', 'Microsoft', '.']],
+          indices=[[[8, -1]]],
+          # Exception here because different errors are thrown in eager vs
+          # graph mode.
+          error=Exception,
+          default_value='$NONE^',
+      ),
+  ])
+  def testRankMismatch(
+      self, params, indices, default_value, error):
+    params = ragged_factory_ops.constant(params)
+    indices = ragged_factory_ops.constant(indices)
+    with self.assertRaises(error):
+      _ = ragged_batch_gather_with_default_op.batch_gather_with_default(
+          params, indices, default_value)
+
+  @parameterized.parameters([
+      # Dimensions:
+      # params: [2, (d1), 2]
+      # indices: [2, (d2)]
+      # default_value: []
+      dict(
+          descr='params: rank 3, indices: rank 2, default: rank = [], but'
+          ' should be [2]',
+          params=[
+              [['The', 'deal'], ['takeover', 'offer'], ['from', 'Microsoft']],
+              [['Who', 'let'], ['the', 'dogs'], ['out', '?']],
+          ],
+          ragged_rank=1,
+          indices=[[1, -1, 2, 30], [1, 100]],
+          indices_ragged_rank=1,
+          default_value='$NONE^',
+          error=Exception,
+      )
+  ])
+  def testInvalidDefaultValueRank(
+      self, descr, params, indices, default_value, error, ragged_rank=None,
+      indices_ragged_rank=None):
+    params = ragged_factory_ops.constant(params, ragged_rank=ragged_rank)
+    indices = ragged_factory_ops.constant(
+        indices, ragged_rank=indices_ragged_rank)
+    with self.assertRaises(error):
+      _ = ragged_batch_gather_with_default_op.batch_gather_with_default(
+          params, indices, default_value)
+
   def testRaggedBatchGatherUnknownRankError(self):
     if context.executing_eagerly():
       return
diff --git a/tensorflow/python/ops/ragged/ragged_batch_gather_with_default_op.py b/tensorflow/python/ops/ragged/ragged_batch_gather_with_default_op.py
new file mode 100644
index 0000000000..0d99540e32
--- /dev/null
+++ b/tensorflow/python/ops/ragged/ragged_batch_gather_with_default_op.py
@@ -0,0 +1,186 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Array operations for RaggedTensors."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import check_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.ragged import ragged_array_ops
+from tensorflow.python.ops.ragged import ragged_dispatch  # pylint: disable=unused-import
+from tensorflow.python.ops.ragged import ragged_operators  # pylint: disable=unused-import
+from tensorflow.python.ops.ragged import ragged_tensor
+from tensorflow.python.ops.ragged import ragged_tensor_shape
+from tensorflow.python.ops.ragged import ragged_where_op
+
+
+#===============================================================================
+# ragged.batch_gather_with_default
+#===============================================================================
+def batch_gather_with_default(params,
+                              indices,
+                              default_value='',
+                              name=None):
+  """Same as `batch_gather` but inserts `default_value` for invalid indices.
+
+  This operation is similar to `batch_gather` except that it will substitute
+  the value for invalid indices with `default_value` as the contents.
+  See `batch_gather` for more details.
+
+
+  Args:
+    params: A potentially ragged tensor with shape `[B1...BN, P1...PM]` (`N>=0`,
+      `M>0`).
+    indices: A potentially ragged tensor with shape `[B1...BN, I]` (`N>=0`).
+    default_value: A value to be inserted in places where `indices` are out of
+      bounds. Must be the same dtype as params and either a scalar or rank 1.
+    name: A name for the operation (optional).
+
+  Returns:
+    A potentially ragged tensor with shape `[B1...BN, I, P2...PM]`.
+    `result.ragged_rank = max(indices.ragged_rank, params.ragged_rank)`.
+
+  #### Example:
+    ```python
+    >>> params = tf.ragged.constant([
+          ['a', 'b', 'c'],
+          ['d'],
+          [],
+          ['e']])
+    >>> indices = tf.ragged.constant([[1, 2, -1], [], [], [0, 10]])
+    >>> batch_gather_with_default(params, indices, 'FOO')
+    [['b', 'c', 'FOO'], [], [], ['e', 'FOO']]
+  ```
+  """
+  with ops.name_scope(name, 'RaggedBatchGatherWithDefault'):
+    params = ragged_tensor.convert_to_tensor_or_ragged_tensor(
+        params, name='params',
+    )
+    indices = ragged_tensor.convert_to_tensor_or_ragged_tensor(
+        indices, name='indices',
+    )
+    default_value = ragged_tensor.convert_to_tensor_or_ragged_tensor(
+        default_value, name='default_value',
+    )
+    # TODO(hterry): lift this restriction and support default_values of
+    #               of rank > 1
+    if (default_value.shape.ndims is not 0
+        and default_value.shape.ndims is not 1):
+      raise ValueError('"default_value" must be a scalar or vector')
+    upper_bounds = None
+    if indices.shape.ndims is None:
+      raise ValueError('Indices must have a known rank.')
+    if params.shape.ndims is None:
+      raise ValueError('Params must have a known rank.')
+
+    num_batch_dimensions = indices.shape.ndims - 1
+    pad = None
+    # The logic for this works as follows:
+    # - create a padded params, where:
+    #    padded_params[b1...bn, 0] = default_value
+    #    padded_params[b1...bn, i] = params[b1...bn, i-1] (i>0)
+    # - create an `upper_bounds` Tensor that contains the number of elements
+    #   in each innermost rank. Broadcast `upper_bounds` to be the same shape
+    #   as `indices`.
+    # - check to see which index in `indices` are out of bounds and substitute
+    #   it with the index containing `default_value` (the first).
+    # - call batch_gather with the indices adjusted.
+    with ops.control_dependencies([
+        check_ops.assert_greater_equal(array_ops.rank(params),
+                                       array_ops.rank(indices))]):
+      if ragged_tensor.is_ragged(params):
+        row_lengths = ragged_array_ops.expand_dims(
+            params.row_lengths(axis=num_batch_dimensions),
+            axis=-1)
+        upper_bounds = math_ops.cast(row_lengths, indices.dtype)
+
+        pad_shape = _get_pad_shape(params, indices)
+
+        pad = ragged_tensor_shape.broadcast_to(
+            default_value, pad_shape)
+      else:
+        params_shape = array_ops.shape(params)
+        pad_shape = array_ops.concat([
+            params_shape[:num_batch_dimensions],
+            [1],
+            params_shape[num_batch_dimensions + 1:params.shape.ndims]
+        ], 0)
+        upper_bounds = params_shape[num_batch_dimensions]
+        pad = array_ops.broadcast_to(default_value, pad_shape)
+
+      # Add `default_value` as the first value in the innermost (ragged) rank.
+      pad = math_ops.cast(pad, params.dtype)
+      padded_params = array_ops.concat(
+          [pad, params], axis=num_batch_dimensions)
+
+      # Adjust the indices by substituting out-of-bound indices to the
+      # default-value index (which is the first element)
+      shifted_indices = indices + 1
+      is_out_of_bounds = (indices < 0) | (indices > upper_bounds)
+      adjusted_indices = ragged_where_op.where(
+          is_out_of_bounds,
+          x=array_ops.zeros_like(indices), y=shifted_indices,
+      )
+      return array_ops.batch_gather(
+          params=padded_params, indices=adjusted_indices, name=name)
+
+
+def _get_pad_shape(params, indices):
+  """Gets the RaggedTensorDynamicShape for the pad tensor."""
+  num_batch_dimensions = indices.shape.ndims - 1
+  params_shape = ragged_tensor_shape.RaggedTensorDynamicShape.from_tensor(
+      params)
+
+  # We want to create a pad tensor that can be concatenated with the params.
+  if params.shape.ndims == indices.shape.ndims:
+    # When params and indices are the same rank, the shape of the pad tensor is
+    # almost identical to params, except the last dimension which has size = 1.
+    if params_shape.num_inner_dimensions is 0:
+      pad_dims = params_shape.partitioned_dim_sizes[:-1] + (
+          array_ops.ones_like(params_shape.partitioned_dim_sizes[-1]),)
+      return ragged_tensor_shape.RaggedTensorDynamicShape(
+          pad_dims, [])
+    else:
+      return ragged_tensor_shape.RaggedTensorDynamicShape(
+          params_shape.partitioned_dim_sizes,
+          array_ops.concat([params_shape.inner_dim_sizes[:-1], [1]], axis=0))
+  else:
+    # When the rank of indices < params, the pad has the same dimension as
+    # params up to the 'num_batch_dimensions' rank. Every dimension after that
+    # has size 1.
+    pad_dims = None
+    if num_batch_dimensions == 0:
+      pad_dims = (constant_op.constant(1, dtype=dtypes.int64),) + (
+          constant_op.constant([1], dtype=dtypes.int64),) * (
+              params_shape.num_partitioned_dimensions -
+              num_batch_dimensions - 1)
+    else:
+      batch_dimensions = params_shape.partitioned_dim_sizes[
+          :num_batch_dimensions]
+      gather_dimension = params_shape.partitioned_dim_sizes[
+          num_batch_dimensions]
+      pad_dims = batch_dimensions + (
+          array_ops.ones_like(gather_dimension),) * (
+              params_shape.num_partitioned_dimensions - num_batch_dimensions)
+
+    return ragged_tensor_shape.RaggedTensorDynamicShape(
+        pad_dims, params_shape.inner_dim_sizes)
-- 
GitLab


From d08271d73c9f7d399d16917b47fcfda74806dd02 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 1 Feb 2019 21:27:52 -0800
Subject: [PATCH 1812/2345] Automated rollback of commit
 8e38fb87fc50aaf0f086a21e1ca1f39361604234

PiperOrigin-RevId: 232084279
---
 tensorflow/core/distributed_runtime/rpc/BUILD |  14 --
 .../rpc/grpc_remote_worker.cc                 |  12 +-
 .../rpc/grpc_response_cache.cc                | 183 ------------------
 .../rpc/grpc_response_cache.h                 |  91 ---------
 .../rpc/grpc_rpc_factory.cc                   |   2 +-
 .../core/distributed_runtime/rpc/grpc_state.h |  85 ++------
 .../rpc/grpc_worker_service.cc                |  96 +++------
 .../rpc/grpc_worker_service.h                 |   6 +-
 8 files changed, 53 insertions(+), 436 deletions(-)
 delete mode 100644 tensorflow/core/distributed_runtime/rpc/grpc_response_cache.cc
 delete mode 100644 tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h

diff --git a/tensorflow/core/distributed_runtime/rpc/BUILD b/tensorflow/core/distributed_runtime/rpc/BUILD
index 607963484d..16078f9688 100644
--- a/tensorflow/core/distributed_runtime/rpc/BUILD
+++ b/tensorflow/core/distributed_runtime/rpc/BUILD
@@ -159,17 +159,6 @@ cc_library(
     ],
 )
 
-cc_library(
-    name = "grpc_response_cache",
-    srcs = ["grpc_response_cache.cc"],
-    hdrs = ["grpc_response_cache.h"],
-    deps = [
-        ":grpc_util",
-        "//tensorflow/core:lib",
-        "@com_google_absl//absl/time",
-    ],
-)
-
 tf_cuda_library(
     name = "grpc_worker_service",
     srcs = ["grpc_worker_service.cc"],
@@ -177,7 +166,6 @@ tf_cuda_library(
     deps = [
         ":async_service_interface",
         ":grpc_call",
-        ":grpc_response_cache",
         ":grpc_tensor_coding",
         ":grpc_util",
         ":grpc_worker_service_impl",
@@ -195,8 +183,6 @@ tf_cuda_library(
         "//tensorflow/core/distributed_runtime:worker_cache",
         "//tensorflow/core/distributed_runtime:worker_env",
         "//tensorflow/core/distributed_runtime:worker_session",
-        "@com_google_absl//absl/container:flat_hash_map",
-        "@com_google_absl//absl/time",
     ],
 )
 
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.cc b/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.cc
index 2479e7368b..2daefcb399 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.cc
@@ -39,8 +39,6 @@ limitations under the License.
 
 namespace tensorflow {
 
-const int kMaxWorkerRpcRetries = 10;
-
 class GrpcRemoteWorker : public WorkerInterface {
  public:
   explicit GrpcRemoteWorker(SharedGrpcChannelPtr channel,
@@ -261,19 +259,17 @@ class GrpcRemoteWorker : public WorkerInterface {
   // given callback, `done`, will be called when the RPC completes.
   void IssueRequest(const protobuf::Message* request,
                     protobuf::Message* response, const ::grpc::string& method,
-                    StatusCallback done, CallOptions* call_opts = nullptr,
-                    int max_retries = kMaxWorkerRpcRetries) {
+                    StatusCallback done, CallOptions* call_opts = nullptr) {
     new RPCState<protobuf::Message>(&stub_, cq_, method, *request, response,
                                     std::move(done), call_opts,
-                                    callback_threadpool_, max_retries);
+                                    callback_threadpool_);
   }
   void IssueRequest(const protobuf::Message* request, TensorResponse* response,
                     const ::grpc::string& method, StatusCallback done,
-                    CallOptions* call_opts = nullptr,
-                    int max_retries = kMaxWorkerRpcRetries) {
+                    CallOptions* call_opts = nullptr) {
     new RPCState<TensorResponse>(&stub_, cq_, method, *request, response,
                                  std::move(done), call_opts,
-                                 callback_threadpool_, max_retries);
+                                 callback_threadpool_);
   }
 
   // Helper function for initializing the RpcMethod objects below.
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.cc b/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.cc
deleted file mode 100644
index 613c290905..0000000000
--- a/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.cc
+++ /dev/null
@@ -1,183 +0,0 @@
-/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h"
-#include "tensorflow/core/platform/env.h"
-
-namespace tensorflow {
-
-struct WorkerCacheEntry {
-  enum class State {
-    PENDING = 0,
-    ACTIVE = 1,
-    FINISHED = 2,
-  };
-
-  State state = State::PENDING;
-  int64 expires_seconds;
-
-  ::grpc::ByteBuffer response_buf;
-  Status response_status;
-
-  // Additional retries may arrive while a request is still executing.  The
-  // callbacks for these calls are queued in `callbacks` and evaluated after
-  // the original request is completed.
-  std::vector<std::pair<RPCResponse, StatusCallback>> callbacks;
-};
-
-void RPCResponse::Encode(::grpc::ByteBuffer* tgt) const {
-  if (buf_ != nullptr) {
-    *tgt = *buf_;
-  } else {
-    CHECK(msg_ != nullptr);
-    ::grpc::Slice slice(msg_->ByteSizeLong());
-    msg_->SerializeWithCachedSizesToArray(
-        const_cast<uint8*>(reinterpret_cast<const uint8*>(slice.begin())));
-    ::grpc::ByteBuffer tmp(&slice, 1);
-    tgt->Swap(&tmp);
-  }
-}
-
-void RPCResponse::CopyFrom(const ::grpc::ByteBuffer& src) {
-  if (buf_ != nullptr) {
-    *buf_ = src;
-    return;
-  }
-
-  CHECK(msg_ != nullptr);
-  // We create a single slice when encoding protocol messages.
-  std::vector<::grpc::Slice> slices;
-  if (src.Dump(&slices).ok()) {
-    msg_->ParseFromArray(slices[0].begin(), slices[0].size());
-  } else {
-    LOG(ERROR) << "Failed to decode cached buffer.";
-  }
-}
-
-void GrpcResponseCache::LookupOrCompute(const string& key, RPCResponse response,
-                                        ComputeFunc compute_func,
-                                        StatusCallback done_cb) {
-  VLOG(1) << "Lookup " << key;
-  std::shared_ptr<WorkerCacheEntry> req;
-  MaybeCleanup();
-  {
-    mutex_lock m(mu_);
-
-    if (requests_.find(key) != requests_.end()) {
-      req = requests_[key];
-    } else {
-      req.reset(new WorkerCacheEntry);
-      requests_[key] = req;
-    }
-
-    if (req->state == WorkerCacheEntry::State::FINISHED) {
-      if (req->expires_seconds > Env::Default()->NowSeconds()) {
-        VLOG(1) << "Reuse cached response for " << key;
-        response.CopyFrom(req->response_buf);
-        done_cb(req->response_status);
-        return;
-      }
-      VLOG(1) << "Found expired cache entry for " << key;
-      req->state = WorkerCacheEntry::State::PENDING;
-      req->response_buf.Clear();
-    }
-
-    req->callbacks.push_back(std::make_pair(response, done_cb));
-
-    if (req->state == WorkerCacheEntry::State::ACTIVE) {
-      VLOG(1) << "Found active request for " << key
-              << ".  Adding entry to response queue.";
-      return;
-    }
-
-    VLOG(2) << "No cache entry for " << key << ", running user computation.";
-    req->state = WorkerCacheEntry::State::ACTIVE;
-    req->expires_seconds = Env::Default()->NowSeconds() + expire_time_seconds_;
-  }
-
-  compute_func([this, key, req, response](Status status) {
-    mutex_lock m(mu_);
-    response.Encode(&req->response_buf);
-    current_bytes_ += req->response_buf.Length();
-
-    req->response_status = status;
-    req->state = WorkerCacheEntry::State::FINISHED;
-
-    VLOG(1) << "Operation for " << key << " finished. "
-            << "Status: " << status << ", " << req->response_buf.Length()
-            << " response bytes, " << req->callbacks.size()
-            << " pending callbacks.";
-    for (auto& cb : req->callbacks) {
-      cb.first.CopyFrom(req->response_buf);
-      cb.second(req->response_status);
-    }
-    req->callbacks.clear();
-  });
-}
-
-// Remove all stale or expired cache entries if the cache is full.
-void GrpcResponseCache::MaybeCleanup() {
-  mutex_lock m(mu_);
-  if (current_bytes_ < max_bytes_) {
-    return;
-  }
-
-  VLOG(1) << "Cleanup: " << current_bytes_ << " -> " << max_bytes_;
-  std::vector<std::pair<string, std::shared_ptr<WorkerCacheEntry>>>
-      ordered_entries;
-  ordered_entries.reserve(requests_.size());
-  for (const auto& p : requests_) {
-    ordered_entries.push_back(std::make_pair(p.first, p.second));
-  }
-
-  std::sort(ordered_entries.begin(), ordered_entries.end(),
-            [](const std::pair<string, std::shared_ptr<WorkerCacheEntry>>& a,
-               const std::pair<string, std::shared_ptr<WorkerCacheEntry>>& b) {
-              return a.second->expires_seconds > b.second->expires_seconds;
-            });
-
-  std::unordered_map<string, std::shared_ptr<WorkerCacheEntry>> kept;
-  int64 now = Env::Default()->NowSeconds();
-  int64 bytes_used = 0;
-
-  // Always keep active requests.
-  for (auto& pair : ordered_entries) {
-    if (pair.second->state != WorkerCacheEntry::State::FINISHED) {
-      kept.insert(pair);
-    }
-  }
-
-  // Keep unexpired, finished requests up to half of max_bytes_.  This reduces
-  // chances of overfilling the cache when active requests complete and
-  // amortizes cache cleanup cost.
-  for (auto& pair : ordered_entries) {
-    if (pair.second->expires_seconds < now || bytes_used >= max_bytes_ / 2) {
-      break;
-    }
-
-    if (pair.second->state == WorkerCacheEntry::State::FINISHED) {
-      kept.insert(pair);
-      bytes_used += pair.second->response_buf.Length();
-    }
-  }
-
-  VLOG(1) << "Cleaned cache.  Bytes used: " << current_bytes_ << " -> "
-          << bytes_used << ". Cache size: " << requests_.size() << " -> "
-          << kept.size();
-  current_bytes_ = bytes_used;
-  std::swap(requests_, kept);
-}
-
-}  // namespace tensorflow
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h b/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h
deleted file mode 100644
index 0892d9f788..0000000000
--- a/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h
+++ /dev/null
@@ -1,91 +0,0 @@
-/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#ifndef TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_RESPONSE_CACHE_H_
-#define TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_RESPONSE_CACHE_H_
-
-#include <memory>
-#include <unordered_map>
-#include <vector>
-
-#include "tensorflow/core/distributed_runtime/rpc/grpc_util.h"
-#include "tensorflow/core/lib/core/errors.h"
-#include "tensorflow/core/lib/core/status.h"
-#include "tensorflow/core/platform/protobuf.h"
-
-// gRPC response caching.  Most WorkerService methods cannot be retried directly
-// as they will fail or deadlock.  To enable retrying, we can instead cache
-// responses for a short period of time and reply to duplicate requests from the
-// cache.
-namespace tensorflow {
-
-// Union type to aid caching of either raw buffers (for RecvTensor RPCs) and
-// protocol buffer messages (for all other RPCs).
-class RPCResponse {
- public:
-  explicit RPCResponse() : buf_(nullptr), msg_(nullptr) {}
-  explicit RPCResponse(::grpc::ByteBuffer* b) : buf_(b), msg_(nullptr) {}
-  explicit RPCResponse(protobuf::Message* m) : buf_(nullptr), msg_(m) {}
-
-  // Encode this response into the target buffer.
-  void Encode(::grpc::ByteBuffer* tgt) const;
-
-  // Copy from `src`: if this is a buffer, make a shallow copy.
-  // For protocol messages, parse the response from `src`.
-  void CopyFrom(const ::grpc::ByteBuffer& src);
-
- private:
-  ::grpc::ByteBuffer* buf_;
-  protobuf::Message* msg_;
-};
-
-typedef std::function<void(StatusCallback)> ComputeFunc;
-struct WorkerCacheEntry;
-
-// Track and cache the state of worker service RPCs.  An RPC can be in 3 states:
-//
-// * PENDING: this is the first call of the RPC, and it will transition to
-// * ACTIVE: another thread is active processing this RPC
-// * FINISHED: the worker has finished processing the method
-//
-// The response from completed RPCs are LRU cached until either `max_bytes`
-// bytes are in use by the cache or they expire (according to `expire_time`).
-class GrpcResponseCache {
- public:
-  GrpcResponseCache(int64 max_bytes, int64 expire_time_seconds)
-      : max_bytes_(max_bytes), expire_time_seconds_(expire_time_seconds) {}
-
-  // Lookup the result for key.
-  // If it is finished, invoke `done_cb` immediately after filling `response`.
-  // If active, done_db will be invoked when the current call completes.
-  // Otherwise, invoke `compute_func` to fill the cache and invoke done_cb.
-  void LookupOrCompute(const string& key, RPCResponse response,
-                       ComputeFunc compute_func, StatusCallback done_cb);
-
-  // Remove all stale or expired cache entries if the cache is full.
-  void MaybeCleanup();
-
- private:
-  int64 current_bytes_ GUARDED_BY(mu_) = 0;
-  const int64 max_bytes_;
-  const int64 expire_time_seconds_;
-
-  std::unordered_map<string, std::shared_ptr<WorkerCacheEntry>> requests_
-      GUARDED_BY(mu_);
-  mutex mu_;
-};
-
-}  // namespace tensorflow
-
-#endif  // TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_RESPONSE_CACHE_H_
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_rpc_factory.cc b/tensorflow/core/distributed_runtime/rpc/grpc_rpc_factory.cc
index 3635caf3d1..7f63cc9344 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_rpc_factory.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_rpc_factory.cc
@@ -210,7 +210,7 @@ void GrpcRPCFactory::StartCall(const Tensor& address_t, const Tensor& method_t,
       get_stub(index), &completion_queue_, *get_method_ptr(index),
       call->request(), call->response(),
       /*done=*/[call](const Status& s) { call->Done(s); }, call->call_opts(),
-      nullptr /*threadpool*/, fail_fast_, timeout_in_ms_, 0 /* max_retries */);
+      nullptr /*threadpool*/, fail_fast_, timeout_in_ms_);
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_state.h b/tensorflow/core/distributed_runtime/rpc/grpc_state.h
index 0ca64dc159..d73638651f 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_state.h
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_state.h
@@ -32,9 +32,6 @@ limitations under the License.
 namespace tensorflow {
 
 // Object allocated per active RPC.
-// Manage the state of a single asynchronous RPC request.  If `max_retries`
-// is greater than 0, the request will be retried for any transient failures
-// as long as the overall deadline has not elapsed.
 template <class Response>
 class RPCState : public GrpcClientCQTag {
  public:
@@ -42,55 +39,34 @@ class RPCState : public GrpcClientCQTag {
   RPCState(::grpc::GenericStub* stub, ::grpc::CompletionQueue* cq,
            const ::grpc::string& method, const protobuf::Message& request,
            Response* response, StatusCallback done, CallOptions* call_opts,
-           thread::ThreadPool* threadpool, int32 max_retries = 0)
+           thread::ThreadPool* threadpool)
       : RPCState(stub, cq, method, request, response, std::move(done),
                  call_opts, threadpool, /*fail_fast=*/false,
-                 /*timeout_in_ms=*/0, max_retries) {}
+                 /*timeout_in_ms=*/0) {}
 
   template <typename Request>
   RPCState(::grpc::GenericStub* stub, ::grpc::CompletionQueue* cq,
            const ::grpc::string& method, const Request& request,
            Response* response, StatusCallback done, CallOptions* call_opts,
-           thread::ThreadPool* threadpool, bool fail_fast, int64 timeout_in_ms,
-           int32 max_retries)
-      : call_opts_(call_opts),
-        threadpool_(threadpool),
-        done_(std::move(done)),
-        cq_(cq),
-        stub_(stub),
-        method_(method),
-        max_retries_(max_retries),
-        timeout_in_ms_(timeout_in_ms),
-        fail_fast_(fail_fast) {
+           thread::ThreadPool* threadpool, bool fail_fast, int64 timeout_in_ms)
+      : call_opts_(call_opts), threadpool_(threadpool), done_(std::move(done)) {
+    context_.set_fail_fast(fail_fast);
+    if (timeout_in_ms > 0) {
+      context_.set_deadline(gpr_time_from_millis(timeout_in_ms, GPR_TIMESPAN));
+    }
+
+    if (call_opts) {
+      call_opts->SetCancelCallback([this]() { context_.TryCancel(); });
+    }
+
     response_ = response;
     ::grpc::Status s = GrpcMaybeUnparseProto(request, &request_buf_);
     if (!s.ok()) {
       LOG(ERROR) << "GrpcMaybeUnparseProto returned with non-ok status: "
                  << s.error_message();
-      // Skip retry logic if we fail to parse our request.
-      done_(FromGrpcStatus(s));
-      delete this;
-      return;
-    }
-    StartCall();
-  }
-
-  void StartCall() {
-    context_.reset(new ::grpc::ClientContext());
-    context_->set_fail_fast(fail_fast_);
-
-    if (timeout_in_ms_ > 0) {
-      context_->set_deadline(
-          gpr_time_from_millis(timeout_in_ms_, GPR_TIMESPAN));
     }
-    if (call_opts_) {
-      call_opts_->SetCancelCallback([this]() { context_->TryCancel(); });
-    }
-
-    VLOG(2) << "Starting call: " << method_;
-
-    call_ = std::move(
-        stub_->PrepareUnaryCall(context_.get(), method_, request_buf_, cq_));
+    call_ =
+        std::move(stub->PrepareUnaryCall(&context_, method, request_buf_, cq));
     call_->StartCall();
     call_->Finish(&response_buf_, &status_, this);
   }
@@ -113,26 +89,16 @@ class RPCState : public GrpcClientCQTag {
         threadpool_->Schedule([this]() { ParseAndCallDone(); });
       } else {
         ParseAndCallDone();
+        return;
       }
-      return;
-    }
-
-    VLOG(1) << method_ << " returned with non-ok status: " << s
-            << " Retries: " << num_retries_ << " Max: " << max_retries_ << "\n"
-            << context_->debug_error_string();
-    // Retry if we have any attempts left
-    if (++num_retries_ <= max_retries_ &&
-        (errors::IsUnavailable(s) || errors::IsUnknown(s))) {
-      response_buf_.Clear();
-      VLOG(1) << "Retrying call for " << method_ << "Retry: " << num_retries_
-              << " of " << max_retries_;
-      StartCall();
     } else {
-      // Attach additional GRPC error information if any to the final status
+      VLOG(2) << "Call returned with non-ok status: " << s;
+
+      // Attach additional GRPC error information if any
       s = Status(s.code(),
                  strings::StrCat(s.error_message(),
                                  "\nAdditional GRPC error information:\n",
-                                 context_->debug_error_string()));
+                                 context_.debug_error_string()));
       done_(s);
       delete this;
     }
@@ -149,7 +115,7 @@ class RPCState : public GrpcClientCQTag {
 
  private:
   CallOptions* call_opts_;
-  std::unique_ptr<::grpc::ClientContext> context_;
+  ::grpc::ClientContext context_;
   thread::ThreadPool* threadpool_;
   std::unique_ptr<::grpc::GenericClientAsyncResponseReader> call_;
   Response* response_;
@@ -157,15 +123,6 @@ class RPCState : public GrpcClientCQTag {
   ::grpc::ByteBuffer response_buf_;
   ::grpc::Status status_;
   StatusCallback done_;
-  int64 timeout_in_ms_;
-
-  size_t num_retries_ = 0;
-  size_t max_retries_;
-
-  ::grpc::CompletionQueue* cq_;
-  ::grpc::GenericStub* stub_;
-  ::grpc::string method_;
-  bool fail_fast_;
 };
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc
index 904862100e..f45b9bf97a 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc
@@ -16,14 +16,11 @@ limitations under the License.
 #include "tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h"
 
 #include <deque>
-#include <memory>
 #include <unordered_map>
-#include <vector>
 
 #include "grpcpp/alarm.h"
 #include "grpcpp/server_builder.h"
 
-#include "absl/container/flat_hash_map.h"
 #include "tensorflow/core/common_runtime/buf_rendezvous.h"
 #include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/common_runtime/device_mgr.h"
@@ -35,7 +32,6 @@ limitations under the License.
 #include "tensorflow/core/distributed_runtime/rendezvous_mgr_interface.h"
 #include "tensorflow/core/distributed_runtime/rpc/async_service_interface.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_call.h"
-#include "tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_tensor_coding.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_util.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h"
@@ -46,12 +42,8 @@ limitations under the License.
 #include "tensorflow/core/framework/collective.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/lib/core/errors.h"
-#include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/gtl/map_util.h"
-#include "tensorflow/core/lib/strings/strcat.h"
-#include "tensorflow/core/lib/strings/stringprintf.h"
 #include "tensorflow/core/platform/logging.h"
-#include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/platform/tracing.h"
 #include "tensorflow/core/protobuf/transport_options.pb.h"
 #include "tensorflow/core/protobuf/worker.pb.h"
@@ -99,11 +91,10 @@ class GrpcWorkerServiceThread {
  public:
   explicit GrpcWorkerServiceThread(
       GrpcWorker* worker, ::grpc::ServerBuilder* builder,
-      std::unordered_map<int, int> queue_depth, GrpcResponseCache* cache,
+      std::unordered_map<int, int> queue_depth,
       grpc::WorkerService::AsyncService* worker_service)
       : worker_(worker),
         queue_depth_(queue_depth),
-        cache_(cache),
         worker_service_(worker_service),
         is_shutdown_(false) {
     cq_ = builder->AddCompletionQueue();
@@ -229,32 +220,18 @@ class GrpcWorkerServiceThread {
       NonOwnedProtoRunGraphResponse* wrapped_response =
           new NonOwnedProtoRunGraphResponse(&call->response);
       call->SetCancelCallback([call_opts]() { call_opts->StartCancel(); });
-      auto done_cb = [call, call_opts, wrapped_request,
-                      wrapped_response](const Status& s) {
-        VLOG(1) << "RunGraph::Done";
-        if (!s.ok()) {
-          VLOG(1) << "Bad response from RunGraph:" << s;
-        }
-        call->ClearCancelCallback();
-        delete call_opts;
-        delete wrapped_request;
-        delete wrapped_response;
-        call->SendResponse(ToGrpcStatus(s));
-      };
-
-      auto compute_fn = [this, call_opts, wrapped_request,
-                         wrapped_response](StatusCallback done) {
-        worker_->RunGraphAsync(call_opts, wrapped_request, wrapped_response,
-                               done);
-      };
-
-      if (cache_) {
-        string request_key = call->request.ShortDebugString();
-        cache_->LookupOrCompute(request_key, RPCResponse(&call->response),
-                                compute_fn, done_cb);
-      } else {
-        compute_fn(done_cb);
-      }
+      worker_->RunGraphAsync(call_opts, wrapped_request, wrapped_response,
+                             [call, call_opts, wrapped_request,
+                              wrapped_response](const Status& s) {
+                               if (!s.ok()) {
+                                 VLOG(1) << "Bad response from RunGraph:" << s;
+                               }
+                               call->ClearCancelCallback();
+                               delete call_opts;
+                               delete wrapped_request;
+                               delete wrapped_response;
+                               call->SendResponse(ToGrpcStatus(s));
+                             });
     });
     ENQUEUE_REQUEST(RunGraph, true);
   }
@@ -264,28 +241,16 @@ class GrpcWorkerServiceThread {
     Schedule([this, call]() {
       CallOptions* call_opts = new CallOptions;
       call->SetCancelCallback([call_opts]() { call_opts->StartCancel(); });
-
-      auto done_cb = [call, call_opts](const Status& s) {
-        call->ClearCancelCallback();
-        delete call_opts;
-        if (!s.ok()) {
-          VLOG(1) << "Bad response from RecvTensor:" << s;
-        }
-        call->SendResponse(ToGrpcStatus(s));
-      };
-
-      auto compute_fn = [this, &call_opts, &call](StatusCallback done) {
-        worker_->GrpcRecvTensorAsync(call_opts, &call->request, &call->response,
-                                     done);
-      };
-
-      if (cache_) {
-        string request_key = call->request.ShortDebugString();
-        cache_->LookupOrCompute(request_key, RPCResponse(&call->response),
-                                compute_fn, done_cb);
-      } else {
-        compute_fn(done_cb);
-      }
+      worker_->GrpcRecvTensorAsync(
+          call_opts, &call->request, &call->response,
+          [call, call_opts](const Status& s) {
+            call->ClearCancelCallback();
+            delete call_opts;
+            if (!s.ok()) {
+              VLOG(1) << "Bad response from RecvTensor:" << s;
+            }
+            call->SendResponse(ToGrpcStatus(s));
+          });
     });
     EnqueueRecvTensorRequestRaw();
   }
@@ -363,7 +328,6 @@ class GrpcWorkerServiceThread {
   std::unique_ptr<::grpc::ServerCompletionQueue> cq_;
   std::unique_ptr<Thread> thread_;
   std::unordered_map<int, int> queue_depth_;
-  GrpcResponseCache* cache_;
   grpc::WorkerService::AsyncService* const worker_service_;
 
   mutex shutdown_mu_;
@@ -377,16 +341,9 @@ class GrpcWorkerService : public AsyncServiceInterface {
                     GrpcWorkerServiceOptions options)
       : is_shutdown_(false) {
     builder->RegisterService(&worker_service_);
-    if (options.response_cache_bytes > 0) {
-      cache_.reset(
-          new GrpcResponseCache(options.response_cache_bytes,
-                                options.response_cache_expires_seconds));
-    }
-
-    for (int i = 0; i < options.num_serving_threads; i++) {
-      threads_.emplace_back(
-          new GrpcWorkerServiceThread(worker, builder, options.queue_depth,
-                                      cache_.get(), &worker_service_));
+    for (int i = 0; i < options.num_worker_threads; i++) {
+      threads_.emplace_back(new GrpcWorkerServiceThread(
+          worker, builder, options.queue_depth, &worker_service_));
     }
   }
 
@@ -421,7 +378,6 @@ class GrpcWorkerService : public AsyncServiceInterface {
   grpc::WorkerService::AsyncService worker_service_;
   std::vector<std::unique_ptr<GrpcWorkerServiceThread>> threads_;
 
-  std::unique_ptr<GrpcResponseCache> cache_;
   mutex service_shutdown_mu_;
   bool is_shutdown_ GUARDED_BY(service_shutdown_mu_);
 
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h
index 8f2830c899..88beb6c216 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h
@@ -16,10 +16,8 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_WORKER_SERVICE_H_
 #define TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_WORKER_SERVICE_H_
 
-#include <memory>
 #include <unordered_map>
 #include "tensorflow/core/distributed_runtime/recent_request_ids.h"
-#include "tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h"
 #include "tensorflow/core/distributed_runtime/worker.h"
 
@@ -65,9 +63,7 @@ struct GrpcWorkerServiceOptions {
   // Map from GrpcWorkerMethod id to queue depth.  If set this overrides the
   // default queue depth for a method.
   std::unordered_map<int, int> queue_depth;
-  int num_serving_threads = 8;
-  int64 response_cache_bytes = 0;
-  int64 response_cache_expires_seconds = 0;
+  int num_worker_threads = 8;
 };
 
 // Returns an implementation of WorkerService rpc service.
-- 
GitLab


From 2df4ae7033d57d6e61a30a36d1b3ea12ff223bb0 Mon Sep 17 00:00:00 2001
From: nammbash <niroopshankar.ammbashankar@intel.com>
Date: Fri, 1 Feb 2019 21:41:49 -0800
Subject: [PATCH 1813/2345] review comments 2a

---
 ...pi_def_RequantizationRangePerChannel.pbtxt | 44 +++++++++++++
 .../api_def_RequantizePerChannel.pbtxt        | 62 +++++++++++++++++++
 ...mkl_requantization_range_per_channel_op.cc | 31 +++++++---
 .../core/kernels/mkl_requantize_ops_test.cc   | 46 +++++++-------
 .../kernels/mkl_requantize_per_channel_op.cc  |  2 +-
 5 files changed, 152 insertions(+), 33 deletions(-)

diff --git a/tensorflow/core/api_def/base_api/api_def_RequantizationRangePerChannel.pbtxt b/tensorflow/core/api_def/base_api/api_def_RequantizationRangePerChannel.pbtxt
index 2226027a42..bf80a35691 100644
--- a/tensorflow/core/api_def/base_api/api_def_RequantizationRangePerChannel.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_RequantizationRangePerChannel.pbtxt
@@ -1,4 +1,48 @@
 op {
 graph_op_name: "RequantizationRangePerChannel"
 visibility : HIDDEN
+  in_arg {
+    name: "input"
+    description: <<END
+The original input tensor.
+END
+  }
+  in_arg {
+    name: "input_min"
+    description: <<END
+The minimum value of the input tensor
+END
+  }
+  in_arg {
+    name: "input_max"
+    description: <<END
+The maximum value of the input tensor.
+END
+  }
+  out_arg {
+    name: "output_min"
+    description: <<END
+The minimum value of the final output tensor
+END
+  }
+  out_arg {
+    name: "output_max"
+    description: <<END
+The maximum value of the final output tensor.
+END
+  }
+  attr {
+    name: "T"
+    description: <<END
+The quantized type of input tensor that needs to be converted. 
+END
+  }
+  attr {
+    name: "clip_value_max"
+    description: <<END
+The maximum value of the output that needs to to be clipped.
+Example: set this to 6 for Relu6. 
+END
+  }
+  summary: "Computes requantization range per channel."
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_RequantizePerChannel.pbtxt b/tensorflow/core/api_def/base_api/api_def_RequantizePerChannel.pbtxt
index c7bfc0a051..e46bc77e7a 100644
--- a/tensorflow/core/api_def/base_api/api_def_RequantizePerChannel.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_RequantizePerChannel.pbtxt
@@ -1,4 +1,66 @@
 op {
 graph_op_name: "RequantizePerChannel"
 visibility : HIDDEN
+in_arg {
+    name: "input"
+    description: <<END
+The original input tensor.
+END
+  }
+  in_arg {
+    name: "input_min"
+    description: <<END
+The minimum value of the input tensor
+END
+  }
+  in_arg {
+    name: "input_max"
+    description: <<END
+The maximum value of the input tensor.
+END
+  }
+  in_arg {
+    name: "requested_output_min"
+    description: <<END
+The minimum value of the output tensor requested.
+END
+  }
+  in_arg {
+    name: "requested_output_max"
+    description: <<END
+The maximum value of the output tensor requested.
+END
+  }  
+  out_arg {
+    name: "output"
+    description: <<END
+Output tensor.
+END
+  }
+  out_arg {
+    name: "output_min"
+    description: <<END
+The minimum value of the final output tensor
+END
+  }
+  out_arg {
+    name: "output_max"
+    description: <<END
+The maximum value of the final output tensor.
+END
+  }
+  attr {
+    name: "T"
+    description: <<END
+The quantized type of input tensor that needs to be converted. 
+END
+  }
+  attr {
+    name: "out_type"
+    description: <<END
+The quantized type of output tensor that needs to be converted. 
+input tensor.
+END
+  }
+  summary: "Requantizes input with min and max values known per channel."
 }
diff --git a/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc b/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc
index 78ef2fc4a5..1de60c67ae 100644
--- a/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc
+++ b/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc
@@ -46,7 +46,7 @@ class MklRequantizationRangePerChannelOp : public OpKernel {
     const Tensor& input_min = ctx->input(kInputMinIndex);
     const Tensor& input_max = ctx->input(kInputMaxIndex);
 
-    size_t depth = input_max.NumElements();
+    const size_t depth = input_max.NumElements();
     OP_REQUIRES(ctx, input_min.dim_size(0) == depth,
                 errors::InvalidArgument("input_min has incorrect size, expected ",
                                         depth, " was ", input_min.dim_size(0)));
@@ -60,27 +60,40 @@ class MklRequantizationRangePerChannelOp : public OpKernel {
     bool is_non_negative = true;
     Eigen::array<int, 2> shuffling({1, 0});
     auto input_matrix = input.flat_inner_dims<qint32>();
+
+    // TODO: verify performance of not transposing and finding the min max directly
+    // from input_matrix vs the one presentd below of transposing and using the
+    // transposed matrix as the transpose operation might be more costly.
+    // Note that this operation is a calibration step for quantization and will cease 
+    // to exist in the final inference graph as it will exist as a const node. 
     auto transposed_input = input_matrix.shuffle(shuffling);
 
     // Find the ranges of each channel in parallel.
-#pragma omp parallel for
+    float out_min_max = std::numeric_limits<float>::min();
+    #pragma omp parallel for //reduction(min:out_min_max)
     for (size_t i = 0; i < depth; ++i) {
       Eigen::Tensor<qint32, 0, Eigen::RowMajor> min =
           transposed_input.chip<0>(i).minimum();
       Eigen::Tensor<qint32, 0, Eigen::RowMajor> max =
           transposed_input.chip<0>(i).maximum();
-      int32_t min_per_channel = min();
-      int32_t max_per_channel = max();
-      int32_t abs_max =
+      const int32_t min_per_channel = min();
+      const int32_t max_per_channel = max();
+      const int32_t abs_max =
           std::max(std::abs(min_per_channel), std::abs(max_per_channel));
       float scale =
           std::max(std::abs(input_min_data[i]), std::abs(input_max_data[i]));
       ranges[i] = scale * static_cast<float>(abs_max) / static_cast<float>(1L << 31);
       if (min_per_channel < 0) is_non_negative = false;
-    }
 
-    // Obtain ranges of out_min_max after all parallel_for openMP threads have joined. 
-    float out_min_max = std::numeric_limits<float>::min();
+        // thread-local out_min_max. 
+        //if (out_min_max < ranges[i]) out_min_max = ranges[i];
+      } 
+      // All local out_min_max gets min-reduced into one global out_min_max at 
+      // the end of the loop by specifying reduction(min:out_min_max) along with 
+      // omp parallel for.
+    
+
+    // Obtain ranges of out_min_max after all parallel_for openMP threads have joined.     
     for (size_t i = 0; i < depth; ++i) {
       if (out_min_max < ranges[i]) out_min_max = ranges[i];
     }
@@ -99,7 +112,7 @@ class MklRequantizationRangePerChannelOp : public OpKernel {
   float clip_value_max_ = std::numeric_limits<float>::infinity();
   const int kInputTensorIndex = 0;
   const int kInputMinIndex = 1;
-  const int kInputMaxIndexIndex = 2;
+  const int kInputMaxIndex = 2;
   const int kOutputMinIndex = 0;
   const int kOutputMaxIndex = 1;
 };
diff --git a/tensorflow/core/kernels/mkl_requantize_ops_test.cc b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
index ebc7687271..524daba6e5 100644
--- a/tensorflow/core/kernels/mkl_requantize_ops_test.cc
+++ b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
@@ -69,18 +69,19 @@ void MklRequantizatedOpsTestHelper::Setup(Tensor &input_tensor_qint32,
   // Step 3: Assumption of quantizedconv on quantized input&weights(per channel)
   // ---------------------------------------------------------------------------
   // The input T and weights W1 (or W2) will be convolved.
-  // The output Tensor T is in int32 whose range is [-(2^31-1), 2^31-1)
-  // The Range of the convolved T*W1 is ((2^31)-1) * 5.0/255 * 2.0/127 = 663110.59
+  // The output Tensor T is in int32 whose range is [-2^31, 2^31).
+  // For simplicity, we truncate the range to (-2^31, 2^31) to make it symmetric.
+  // The range of the convolved T*W1 is ((2^31)-1) * 5.0/255 * 2.0/127 = 663110.59
   // So the range of convolved T*W1 in int32[-(2^31-1), 22^31-1] that maps to
-  // orig T Range in FP32[0,5.0] * [-2.0, 2.0] is [-663110.59, 663110.59]
+  // orig T range in FP32[0, 5.0] * [-2.0, 2.0] is [-663110.59, 663110.59]
 
-  // The Range of the convolved T*W2 is 2^31-1 * 5.0/255 * 3.0/127 = 994665.88
-  // So Range of convolved T*W2 in int32[-(2^31-1), 22^31-1] that maps to
-  // orig T Range in FP32 [0, 5.0] * [-3.0, 3.0]  is [-994665.88, 994665.88]
+  // The range of the convolved T*W2 is (2^31-1) * 5.0/255 * 3.0/127 = 994665.88
+  // So the range of convolved T*W1 in int32(-2^31, 2^31) that maps to
+  // orig T range in FP32 [0, 5.0] * [-3.0, 3.0]  is [-994665.88, 994665.88]
 
   // Step 4: Assumption output above is fed to Requantization_range_perchannel
   // --------------------------------------------------------------------------
-  // Here we recalculate the new Range for convolved T*W so that we
+  // Here we recalculate the new range for convolved T*W so that we
   // make good use in int8 quantization from int32 to int8.
 
   // We assume the above operations are performed and use these values above
@@ -88,7 +89,7 @@ void MklRequantizatedOpsTestHelper::Setup(Tensor &input_tensor_qint32,
   range_weights_ch1 = 663110.59;  // For W1 channel
   range_weights_ch2 = 994665.88;  // For W2 Channel
 
-  // We Fill the inputs Tensor T qint32 with arbitrary int32 values
+  // We Fill the input Tensor T qint32 with arbitrary int32 values
   test::FillValues<qint32>(
       &input_tensor_qint32,
       {-1000, -2000,  2000,   4000,   -3000,  -6000,  4000,   8000,
@@ -106,7 +107,7 @@ void MklRequantizatedOpsTestHelper::Setup(Tensor &input_tensor_qint32,
 // of the weights is calculated per channel.
 TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_Basic) {
   // Let us set up the tensor and inputs before we run this op.
-  float clip_max_value = pow(2, 31);
+  float clip_value_max = static_cast<float>((1L << 31) - 1);
   float range_weights_ch1 = 0.0;
   float range_weights_ch2 = 0.0;
 
@@ -125,14 +126,14 @@ TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_Basic) {
 
   // Step 5: Define and run requantization_range_perchannel
   // -------------------------------------------------------
-  // Define, Create and initalize the OP in question.
+  // Define, create and initialize the OP in question.
   TF_ASSERT_OK(NodeDefBuilder("requantization_range_per_channel",
                               "RequantizationRangePerChannel")
                    .Input(FakeInput(DT_QINT32))
                    .Input(FakeInput(DT_FLOAT))
                    .Input(FakeInput(DT_FLOAT))
                    .Attr("T", DataTypeToEnum<qint32>::v())
-                   .Attr("clip_value_max", clip_max_value)
+                   .Attr("clip_value_max", clip_value_max)
                    .Finalize(node_def()));
   TF_ASSERT_OK(InitOp());
 
@@ -141,9 +142,9 @@ TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_Basic) {
                             input_tensor_qint32.flat<qint32>());
 
   // Calculate the Min and max from the ranges
-  float ch1_min = -1.0 * range_weights_ch1;
+  float ch1_min = -range_weights_ch1;
   float ch1_max = range_weights_ch1;
-  float ch2_min = -1.0 * range_weights_ch2;
+  float ch2_min = -range_weights_ch2;
   float ch2_max = range_weights_ch2;
 
   // Add the Perchannel range Nodes to the Op.
@@ -159,15 +160,15 @@ TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_Basic) {
   // Verify the Expected Outputs
   const float output_min = GetOutput(0)->flat<float>()(0);
   const float output_max = GetOutput(1)->flat<float>()(0);
-  EXPECT_NEAR(-14.8217, output_min, 0.002);
-  EXPECT_NEAR(14.8217, output_max, 0.002);
+  EXPECT_NEAR(-14.8217, output_min, 0.02);
+  EXPECT_NEAR(14.8217, output_max, 0.02);
 
   // output range is made use in RequantizePerChannelTest_Basic
 }
 
 TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_ClipMax) {
   // Let us setup the tensor and inputs before we run this op.
-  float clip_max_value = 6;  // Can be used as 6 for Relu 6 activations.
+  float clip_value_max = 6;  // Can be used as 6 for Relu 6 activations.
   float range_weights_ch1 = 0.0;
   float range_weights_ch2 = 0.0;
 
@@ -193,7 +194,7 @@ TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_ClipMax) {
                    .Input(FakeInput(DT_FLOAT))
                    .Input(FakeInput(DT_FLOAT))
                    .Attr("T", DataTypeToEnum<qint32>::v())
-                   .Attr("clip_value_max", clip_max_value)
+                   .Attr("clip_value_max", clip_value_max)
                    .Finalize(node_def()));
   TF_ASSERT_OK(InitOp());
 
@@ -220,13 +221,12 @@ TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_ClipMax) {
   // Verify the Expected Outputs
   const float output_min = GetOutput(0)->flat<float>()(0);
   const float output_max = GetOutput(1)->flat<float>()(0);
-  EXPECT_NEAR(-6.0, output_min, 0.002);  // Values are Max as with clip_value
-  EXPECT_NEAR(6.0, output_max, 0.002);   // Values are Max as with clip_value
+  EXPECT_NEAR(-6.0, output_min, 0.02);  // Values are Max as with clip_value
+  EXPECT_NEAR(6.0, output_max, 0.02);   // Values are Max as with clip_value
 }
 
 TEST_F(MklRequantizatedOpsTest, RequantizePerChannelTest_Basic) {
   // Let us setup the tensor and inputs before we run this op.
-  float clip_max_value = pow(2, 31);
   float range_weights_ch1 = 0.0;
   float range_weights_ch2 = 0.0;
 
@@ -245,7 +245,7 @@ TEST_F(MklRequantizatedOpsTest, RequantizePerChannelTest_Basic) {
 
   // Step 7: Define and run requantize_perchannel
   // --------------------------------------------
-  // The Output of Requantization_range_op_per_channel which calculated the
+  // The output of Requantization_range_op_per_channel which calculated the
   // new ranges of int8 is fed to the requantize per channel op.
   // Here the Values of Convolved T*W is converted from int32 to int8.
 
@@ -290,8 +290,8 @@ TEST_F(MklRequantizatedOpsTest, RequantizePerChannelTest_Basic) {
   Tensor output = *GetOutput(0);
   const float output_min = GetOutput(1)->flat<float>()(0);
   const float output_max = GetOutput(2)->flat<float>()(0);
-  EXPECT_NEAR(range_op_output_min, output_min, 0.002);
-  EXPECT_NEAR(range_op_output_max, output_max, 0.002);
+  EXPECT_NEAR(range_op_output_min, output_min, 0.02);
+  EXPECT_NEAR(range_op_output_max, output_max, 0.02);
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc b/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc
index e531ec8931..16940ff9bb 100644
--- a/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc
+++ b/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc
@@ -86,7 +86,7 @@ class MklRequantizePerChannelOp : public OpKernel {
         float min_max_from_vec = std::max(std::abs(input_min_vec_data[i]),
                                           std::abs(input_max_vec_data[i]));
         scales[i] =
-            factor * (min_max_from_vec / requested_min_max / (float)(1L << 31));
+            factor * (min_max_from_vec / requested_min_max / static_cast<float>(1L << 31));
       }
 
       mkldnn::primitive_attr reorder_attr;
-- 
GitLab


From 481fa24613a5e5f784663b6e6c0ddd664483b1a5 Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <38085909+penpornk@users.noreply.github.com>
Date: Fri, 1 Feb 2019 21:56:15 -0800
Subject: [PATCH 1814/2345] Update
 tensorflow/core/kernels/mkl_requantize_ops_test.cc

Co-Authored-By: nammbash <38869685+nammbash@users.noreply.github.com>
---
 tensorflow/core/kernels/mkl_requantize_ops_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/mkl_requantize_ops_test.cc b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
index 524daba6e5..03f544249a 100644
--- a/tensorflow/core/kernels/mkl_requantize_ops_test.cc
+++ b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
@@ -72,7 +72,7 @@ void MklRequantizatedOpsTestHelper::Setup(Tensor &input_tensor_qint32,
   // The output Tensor T is in int32 whose range is [-2^31, 2^31).
   // For simplicity, we truncate the range to (-2^31, 2^31) to make it symmetric.
   // The range of the convolved T*W1 is ((2^31)-1) * 5.0/255 * 2.0/127 = 663110.59
-  // So the range of convolved T*W1 in int32[-(2^31-1), 22^31-1] that maps to
+  // So the range of convolved T*W1 in int32(-2^31, 2^31) that maps to
   // orig T range in FP32[0, 5.0] * [-2.0, 2.0] is [-663110.59, 663110.59]
 
   // The range of the convolved T*W2 is (2^31-1) * 5.0/255 * 3.0/127 = 994665.88
-- 
GitLab


From 74147d54076b5dc372c1593a1d381bae03eca6dd Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <38085909+penpornk@users.noreply.github.com>
Date: Fri, 1 Feb 2019 21:56:39 -0800
Subject: [PATCH 1815/2345] Update
 tensorflow/core/kernels/mkl_requantize_ops_test.cc

Co-Authored-By: nammbash <38869685+nammbash@users.noreply.github.com>
---
 tensorflow/core/kernels/mkl_requantize_ops_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/mkl_requantize_ops_test.cc b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
index 03f544249a..a730f94fd2 100644
--- a/tensorflow/core/kernels/mkl_requantize_ops_test.cc
+++ b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
@@ -267,7 +267,7 @@ TEST_F(MklRequantizatedOpsTest, RequantizePerChannelTest_Basic) {
   // Calculate the Min and max from the ranges
   float ch1_min = -1.0 * range_weights_ch1;
   float ch1_max = range_weights_ch1;
-  float ch2_min = -1.0 * range_weights_ch2;
+  float ch2_min = -range_weights_ch2;
   float ch2_max = range_weights_ch2;
 
   // Add the Perchannel range Nodes to the Op.
-- 
GitLab


From 33a704aad006ee80c6165a79297f6c3aba19576f Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <38085909+penpornk@users.noreply.github.com>
Date: Fri, 1 Feb 2019 21:56:54 -0800
Subject: [PATCH 1816/2345] Update
 tensorflow/core/kernels/mkl_requantize_ops_test.cc

Co-Authored-By: nammbash <38869685+nammbash@users.noreply.github.com>
---
 tensorflow/core/kernels/mkl_requantize_ops_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/mkl_requantize_ops_test.cc b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
index a730f94fd2..111d575bcc 100644
--- a/tensorflow/core/kernels/mkl_requantize_ops_test.cc
+++ b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
@@ -265,7 +265,7 @@ TEST_F(MklRequantizatedOpsTest, RequantizePerChannelTest_Basic) {
                             input_tensor_qint32.flat<qint32>());
 
   // Calculate the Min and max from the ranges
-  float ch1_min = -1.0 * range_weights_ch1;
+  float ch1_min = -range_weights_ch1;
   float ch1_max = range_weights_ch1;
   float ch2_min = -range_weights_ch2;
   float ch2_max = range_weights_ch2;
-- 
GitLab


From bf34b0fdc61401917a0c238f1141e00d2f682014 Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <38085909+penpornk@users.noreply.github.com>
Date: Fri, 1 Feb 2019 21:57:17 -0800
Subject: [PATCH 1817/2345] Update
 tensorflow/core/kernels/mkl_requantize_ops_test.cc

Co-Authored-By: nammbash <38869685+nammbash@users.noreply.github.com>
---
 tensorflow/core/kernels/mkl_requantize_ops_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/mkl_requantize_ops_test.cc b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
index 111d575bcc..9ec379c08e 100644
--- a/tensorflow/core/kernels/mkl_requantize_ops_test.cc
+++ b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
@@ -187,7 +187,7 @@ TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_ClipMax) {
 
   // Step 5: Define and run requantization_range_perchannel
   // -------------------------------------------------------
-  // Define, Create and initalize the OP in question.
+  // Define, create and initalize the OP in question.
   TF_ASSERT_OK(NodeDefBuilder("requantization_range_per_channel",
                               "RequantizationRangePerChannel")
                    .Input(FakeInput(DT_QINT32))
-- 
GitLab


From eeaca34ce368eae8cf54c053678f464fd05bcaa6 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Sat, 2 Feb 2019 00:37:46 -0800
Subject: [PATCH 1818/2345] Automated rollback of commit
 1edc3920d63eee6fe4c72e27c1d3e73deb6254e3

PiperOrigin-RevId: 232096800
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index f6689ba4a3..93e73df883 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -129,11 +129,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "com_google_absl",
         build_file = clean_dep("//third_party:com_google_absl.BUILD"),
-        sha256 = "4c2e4194bbddcb5162933e45fe574d2c4e77a2ef00818b8dac0392459707bfff",
-        strip_prefix = "abseil-cpp-d78310fe5a82f2e0e6e16509ef8079c8d7e4674e",
+        sha256 = "583e5801372a0bb12eb561858532e3bb9a3528f15f65cfc87b2c0f4c1ab1a0ca",
+        strip_prefix = "abseil-cpp-111ca7060a6ff50115ca85b59f6b5d8c8c5e9105",
         urls = [
-            "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/d78310fe5a82f2e0e6e16509ef8079c8d7e4674e.tar.gz",
-            "https://github.com/abseil/abseil-cpp/archive/d78310fe5a82f2e0e6e16509ef8079c8d7e4674e.tar.gz",
+            "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/111ca7060a6ff50115ca85b59f6b5d8c8c5e9105.tar.gz",
+            "https://github.com/abseil/abseil-cpp/archive/111ca7060a6ff50115ca85b59f6b5d8c8c5e9105.tar.gz",
         ],
     )
 
-- 
GitLab


From 6d594251b7bb55b88dafca53f28959ab92bf079b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 2 Feb 2019 01:02:19 -0800
Subject: [PATCH 1819/2345] compat: Update forward compatibility horizon to
 2019-02-02

PiperOrigin-RevId: 232098298
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index bef953b28f..27d4abafbc 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 2, 1)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 2, 2)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 0b93b8851d335043075d0b046eef2b7c6581b756 Mon Sep 17 00:00:00 2001
From: Grzegorz Pawelczak <grzegorzp@graphcore.ai>
Date: Sat, 2 Feb 2019 12:40:58 +0000
Subject: [PATCH 1820/2345] Check shape has layout

---
 tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc b/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc
index c1f69db74e..334b0d90e1 100644
--- a/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc
+++ b/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc
@@ -162,7 +162,7 @@ void HloTfGraphBuilder::SetNodeAttrs(const HloInstruction* instruction,
       // For tuples, emit the full shape because the layout of a tuple is not
       // represented in a single Layout field.
       layout_string = ShapeUtil::HumanStringWithLayout(instruction->shape());
-    } else {
+    } else if (instruction->shape().has_layout()) {
       layout_string = StrCat(
           "{",
           absl::StrJoin(LayoutUtil::MinorToMajor(instruction->shape()), ","),
-- 
GitLab


From f22fc30109a637c7be65657709104ee899da9f5e Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Sat, 2 Feb 2019 11:50:50 -0800
Subject: [PATCH 1821/2345] Internal change.

PiperOrigin-RevId: 232131870
---
 tensorflow/stream_executor/gpu/BUILD | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/stream_executor/gpu/BUILD b/tensorflow/stream_executor/gpu/BUILD
index 78fe478679..e6812389e3 100644
--- a/tensorflow/stream_executor/gpu/BUILD
+++ b/tensorflow/stream_executor/gpu/BUILD
@@ -25,7 +25,7 @@ filegroup(
 
 cc_library(
     name = "gpu_activation_header",
-    hdrs = if_gpu_is_configured(["gpu_activation.h"]),
+    hdrs = ["gpu_activation.h"],
     deps = ["//tensorflow/stream_executor/platform"],
 )
 
@@ -33,13 +33,13 @@ cc_library(
     name = "gpu_activation",
     srcs = if_gpu_is_configured(["gpu_activation.cc"]),
     hdrs = if_gpu_is_configured(["gpu_activation.h"]),
-    deps = [
+    deps = if_gpu_is_configured([
         ":gpu_activation_header",
         ":gpu_driver_header",
         "//tensorflow/stream_executor",
         "//tensorflow/stream_executor:stream_executor_internal",
         "//tensorflow/stream_executor/platform",
-    ],
+    ]),
 )
 
 cc_library(
-- 
GitLab


From 14c78f9675a29190460d6cd5e10ce38e3be84fad Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Sat, 2 Feb 2019 13:05:11 -0800
Subject: [PATCH 1822/2345] Always convert function calls dynamically. This
 greatly simplifies the implementation at the cost of less readability of the
 output. Also included is a cleanup of the tests that had to be updated due to
 this change. Addresses #25281, #24759. Lastly, the CL enables the automatic
 fallback on compilation error.

PiperOrigin-RevId: 232135777
---
 .../autograph/integration_tests/keras_test.py |  22 +-
 .../python/autograph/converters/call_trees.py | 293 ++----------------
 .../autograph/converters/call_trees_test.py   | 146 ++-------
 tensorflow/python/autograph/core/config.py    |  19 +-
 tensorflow/python/autograph/impl/api.py       | 221 ++++++++-----
 .../python/autograph/impl/conversion.py       |  47 ++-
 .../python/autograph/impl/conversion_test.py  |   4 +-
 .../python/autograph/pyct/inspect_utils.py    |   9 +-
 tensorflow/python/autograph/pyct/parser.py    |   7 +-
 .../python/autograph/utils/ag_logging.py      |   6 +-
 10 files changed, 250 insertions(+), 524 deletions(-)

diff --git a/tensorflow/examples/autograph/integration_tests/keras_test.py b/tensorflow/examples/autograph/integration_tests/keras_test.py
index 3fe33df920..72b62f1ad4 100644
--- a/tensorflow/examples/autograph/integration_tests/keras_test.py
+++ b/tensorflow/examples/autograph/integration_tests/keras_test.py
@@ -87,18 +87,16 @@ class KerasTest(tf.test.TestCase):
 
   @test_util.run_deprecated_v1
   def test_recursive_true(self):
-    with self.assertRaisesRegexp(NotImplementedError,
-                                 'Object conversion is not yet supported.'):
-      with tf.Graph().as_default():
-        model = CompoundModel()
-        model.build(tf.TensorShape((None, 10, 10, 1)))
-        init = tf.global_variables_initializer()
-
-        with tf.Session() as sess:
-          self.evaluate(init)
-          sample_input = tf.random_uniform((1, 10, 10, 1))
-          output = model(sample_input)  # pylint: disable=not-callable
-          self.assertEqual(self.evaluate(output).shape, (1, 3))
+    with tf.Graph().as_default():
+      model = CompoundModel()
+      model.build(tf.TensorShape((None, 10, 10, 1)))
+      init = tf.global_variables_initializer()
+
+      with tf.Session() as sess:
+        self.evaluate(init)
+        sample_input = tf.random_uniform((1, 10, 10, 1))
+        output = model(sample_input)  # pylint: disable=not-callable
+        self.assertEqual(self.evaluate(output).shape, (1, 3))
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/autograph/converters/call_trees.py b/tensorflow/python/autograph/converters/call_trees.py
index 7026a162a2..04439ba516 100644
--- a/tensorflow/python/autograph/converters/call_trees.py
+++ b/tensorflow/python/autograph/converters/call_trees.py
@@ -22,231 +22,46 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import collections
-
 import gast
 
 from tensorflow.python.autograph.core import converter
 from tensorflow.python.autograph.pyct import anno
-from tensorflow.python.autograph.pyct import ast_util
-from tensorflow.python.autograph.pyct import inspect_utils
 from tensorflow.python.autograph.pyct import parser
 from tensorflow.python.autograph.pyct import templates
-from tensorflow.python.util import tf_inspect
-
-
-class FunctionInfo(collections.namedtuple('FunctionInfo', ('dtype',))):
-  pass
-
-
-# TODO(mdan): Move this to a separate transformer.
-KNOWN_NUMPY_FUNCTIONS = {
-    ('numpy', 'random', 'binomial'): FunctionInfo(dtype='tf.int64'),
-}
 
 
-# TODO(mdan): Get rid of these interfaces. Can now depend directly on Namer.
-
-
-class FunctionNamer(object):
-  """Describes the interface for CallTreeTransformer's namer."""
-
-  def compiled_function_name(self,
-                             original_fqn,
-                             live_entity=None,
-                             owner_type=None):
-    """Generate the name corresponding to the compiled version of a function.
-
-    Args:
-      original_fqn: string or tuple(string)
-      live_entity: Callable, the actual target function, if known.
-      owner_type: Optional object. If present, it indicates that the function is
-          a member of the given type.
-    Returns:
-      string, bool
-    """
-    raise NotImplementedError()
-
-  def compiled_class_name(self, original_fqn, live_entity=None):
-    """Generate the name corresponding to the compiled version of a class.
-
-    Args:
-      original_fqn: string or tuple(string)
-      live_entity: The actual target class, if known.
-    Returns:
-      string
-    """
-    raise NotImplementedError()
-
-
-# TODO(mdan): Rename to CallsTransformer.
+# TODO(mdan): Rename to FunctionCallsTransformer.
 
 
 class CallTreeTransformer(converter.Base):
   """Transforms the call tree by renaming transformed symbols."""
 
-  def _resolve_decorator_name(self, node):
-    """Used to resolve decorator info."""
-    if isinstance(node, gast.Call):
-      return self._resolve_decorator_name(node.func)
-    if isinstance(node, gast.Name):
-      # TODO(mdan): Add test coverage for this branch.
-      return self.ctx.info.namespace.get(node.id)
-    if isinstance(node, gast.Attribute):
-      parent = self._resolve_decorator_name(node.value)
-      if parent is not None:
-        return getattr(parent, node.attr)
-      return None
-    raise ValueError(node)
-
-  def _try_resolve_target(self, node):
-    """Works for methods of objects of known type."""
-    if anno.hasanno(node, 'live_val'):
-      return anno.getanno(node, 'live_val')
-    if isinstance(node, gast.Attribute) and anno.hasanno(node, 'type'):
-      owner_type = anno.getanno(node, 'type')
-      if hasattr(owner_type, node.attr):
-        return getattr(owner_type, node.attr)
-      else:
-        # TODO(mdan): We should probably return None here rather than an error.
-        raise ValueError('Type "%s" has no attribute "%s". Is it dynamic?' %
-                         (owner_type, node.attr))
-    return None
-
-  def _function_is_compilable(self, target_entity):
-    """Determines whether an entity can be compiled at all."""
-    # TODO(mdan): Expand.
-
-    if target_entity.__module__ is None:
-      # Functions like builtins and NumPy don't expose a module.
-      # Those in general should not be compiled.
-      return False
-
-    if inspect_utils.isbuiltin(target_entity):
-      return False
-
-    if inspect_utils.isnamedtuple(target_entity):
-      # namedtuple doesn't expose its source code, making it uncompilable.
-      return False
-
-    return True
-
-  def _should_compile(self, node, fqn):
-    """Determines whether an entity should be compiled in the context."""
-    # TODO(mdan): Needs cleanup. We should remove the use of fqn altogether.
-    module_name = fqn[0]
-    for mod in self.ctx.program.uncompiled_modules:
-      if module_name.startswith(mod[0] + '.'):
-        return False
-
-    for i in range(1, len(fqn)):
-      if fqn[:i] in self.ctx.program.uncompiled_modules:
-        return False
-
-    target_entity = self._try_resolve_target(node.func)
-
-    if target_entity is not None:
-
-      # Currently, lambdas are always converted.
-      # TODO(mdan): Allow markers of the kind f = ag.do_not_convert(lambda: ...)
-      if inspect_utils.islambda(target_entity):
-        return True
-
-      # This may be reached when "calling" a callable attribute of an object.
-      # For example:
-      #
-      #   self.fc = tf.keras.layers.Dense()
-      #   self.fc()
-      #
-      for mod in self.ctx.program.uncompiled_modules:
-        if target_entity.__module__.startswith(mod[0] + '.'):
-          return False
-
-      # Inspect the target function decorators. If any include a @convert
-      # or @do_not_convert annotation, then they must be called as they are.
-      # TODO(mdan): This may be quite heavy. Perhaps always dynamically convert?
-      # To parse and re-analyze each function for every call site could be quite
-      # wasteful. Maybe we could cache the parsed AST?
-      try:
-        target_node, _ = parser.parse_entity(target_entity)
-        target_node = target_node.body[0]
-      except TypeError:
-        # Functions whose source we cannot access are compilable (e.g. wrapped
-        # to py_func).
-        return True
-
-      # This attribute is set when the decorator was applied before the
-      # function was parsed. See api.py.
-      if hasattr(target_entity, '__ag_compiled'):
-        return False
-
-      for dec in target_node.decorator_list:
-        decorator_fn = self._resolve_decorator_name(dec)
-        if (decorator_fn is not None and
-            self.ctx.program.options.should_strip(decorator_fn)):
-          return False
-
-    return True
-
-  def _rename_compilable_function(self, node):
-    assert anno.hasanno(node.func, 'live_val')
-    assert anno.hasanno(node.func, 'fqn')
-    target_entity = anno.getanno(node.func, 'live_val')
-    target_fqn = anno.getanno(node.func, 'fqn')
-
-    if anno.hasanno(node, 'is_constructor'):
-      new_name = self.ctx.namer.compiled_class_name(
-          target_fqn, live_entity=target_entity)
-      do_rename = True
-    else:
-      if anno.hasanno(node.func, 'parent_type'):
-        owner_type = anno.getanno(node.func, 'parent_type')
-      else:
-        # Fallback - not reliable.
-        owner_type = inspect_utils.getmethodclass(target_entity)
-      new_name, do_rename = self.ctx.namer.compiled_function_name(
-          target_fqn, live_entity=target_entity, owner_type=owner_type)
+  def visit_FunctionDef(self, node):
+    node.args = self.visit(node.args)
+    node.body = self.visit_block(node.body)
+    # TODO(mdan): Is this correct for local functions?
+    node.decorator_list = []
+    if node.returns:
+      node.returns = self.visit(node.returns)
+    return node
 
-    if do_rename:
-      if target_entity is not None:
-        if tf_inspect.ismethod(target_entity):
-          # The renaming process will transform it into a regular function.
-          # TODO(mdan): Is this complete? How does it work with nested members?
-          node.args = [node.func.value] + node.args
-      node.func = templates.replace_as_expression(
-          'func_name', func_name=new_name)
+  def visit_With(self, node):
+    # Context manager calls (in node.items) are not converted.
+    node.body = self.visit_block(node.body)
     return node
 
-  def _wrap_to_py_func_single_return(self, node, dtype):
-    # TODO(mdan): Properly handle varargs, etc.
-    template = """
-      ag__.utils.wrap_py_func(func, dtype, (args,), kwargs, False)
-    """
-    return templates.replace_as_expression(
-        template,
-        func=node.func,
-        dtype=parser.parse_expression(dtype),
-        args=node.args,
-        kwargs=ast_util.keywords_to_dict(node.keywords))
+  def visit_Call(self, node):
+    # TODO(mdan): Refactor converted_call as a 'Call' operator.
+
+    # Calls to the internal 'ag__' module are never converted (though their
+    # arguments might be).
+    full_name = str(anno.getanno(node.func, anno.Basic.QN, default=''))
+    if full_name.startswith('ag__.'):
+      return self.generic_visit(node)
+    if (full_name == 'print' and
+        not self.ctx.program.options.uses(converter.Feature.BUILTIN_FUNCTIONS)):
+      return self.generic_visit(node)
 
-  def _insert_dynamic_conversion(self, node):
-    """Inlines a dynamic conversion for a dynamic function."""
-    # TODO(mdan): Pass information on the statically compiled functions.
-    # Having access to the statically compiled functions can help avoid
-    # unnecessary compilation.
-    # For example, this would lead to function `a` being compiled twice:
-    #
-    #   def a():
-    #     v = b
-    #     b()
-    #   def b():
-    #     a()
-    #
-    # This is really a problem with recursive calls, which currently can
-    # only be gated by a static condition, and should be rare.
-    # TODO(mdan): It probably makes sense to use dynamic conversion every time.
-    # Before we could convert all the time though, we'd need a reasonable
-    # caching mechanism.
     template = """
       ag__.converted_call(func, owner, options, args)
     """
@@ -256,6 +71,7 @@ class CallTreeTransformer(converter.Base):
     else:
       func = node.func
       owner = parser.parse_expression('None')
+
     new_call = templates.replace_as_expression(
         template,
         func=func,
@@ -266,67 +82,8 @@ class CallTreeTransformer(converter.Base):
         args=node.args)
     # TODO(mdan): Improve the template mechanism to better support this.
     new_call.keywords = node.keywords
-    return new_call
 
-  def visit_FunctionDef(self, node):
-    node.args = self.visit(node.args)
-    node.body = self.visit_block(node.body)
-    node.decorator_list = []
-    node.returns = self.visit_block(node.returns)
-    return node
-
-  def visit_Call(self, node):
-    if anno.hasanno(node.func, 'live_val'):
-      target_entity = anno.getanno(node.func, 'live_val')
-
-      if anno.hasanno(node.func, 'fqn'):
-        target_fqn = anno.getanno(node.func, 'fqn')
-      else:
-        target_fqn = None
-
-      if self._function_is_compilable(target_entity):
-        if self._should_compile(node, target_fqn):
-          node = self._rename_compilable_function(node)
-        else:
-          node = self.generic_visit(node)
-          return node
-
-      elif target_fqn and target_fqn in KNOWN_NUMPY_FUNCTIONS:
-        # TODO(mdan): Should we replace these with equivalent TF ops instead?
-        node = self._wrap_to_py_func_single_return(
-            node, KNOWN_NUMPY_FUNCTIONS[target_fqn].dtype)
-
-      elif inspect_utils.isbuiltin(target_entity):
-        # Note: Any builtin that passed the builtins converter is assumed to be
-        # safe for graph mode.
-        return node
-
-      elif inspect_utils.isnamedtuple(target_entity):
-        # Although not compilable, we assume they are safe for graph mode.
-        node = self.generic_visit(node)
-        return node
-
-      else:
-        # TODO(mdan): Instert dynamic conversion here instead.
-        raise NotImplementedError(
-            'py_func with return values (unknown function)')
-    else:
-      # Special cases
-      # TODO(mdan): These need a systematic review - there may be more.
-
-      # 1. super() calls - these are preserved. The class conversion mechanism
-      # will ensure that they return the correct value.
-      if ast_util.matches(node, parser.parse_expression('super(_)')):
-        return node
-
-      # 2. super().method calls - these are preserved as well, when the
-      # conversion processes the entire class.
-      if (ast_util.matches(node, parser.parse_expression('super(_)._(_)')) and
-          self.ctx.info.owner_type is not None):
-        return node
-
-      node = self._insert_dynamic_conversion(node)
-    return node
+    return new_call
 
 
 def transform(node, ctx):
diff --git a/tensorflow/python/autograph/converters/call_trees_test.py b/tensorflow/python/autograph/converters/call_trees_test.py
index 454d75d755..6ee56bf6be 100644
--- a/tensorflow/python/autograph/converters/call_trees_test.py
+++ b/tensorflow/python/autograph/converters/call_trees_test.py
@@ -18,147 +18,49 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import collections
-
-import numpy as np
-
 from tensorflow.python.autograph.converters import call_trees
 from tensorflow.python.autograph.core import converter_testing
-from tensorflow.python.framework import constant_op
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops
-from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
 
 class CallTreesTest(converter_testing.TestCase):
 
-  def test_basic(self):
-
-    def test_fn_1(_):
-      raise ValueError('This should not be called in the compiled version.')
-
-    def other_test_fn_1(a):
-      return a + 1
-
-    def test_fn_2(a):
-      return test_fn_1(a) + 1
-
-    ns = {'test_fn_1': test_fn_1}
-    node, ctx = self.prepare(test_fn_2, ns)
-    node = call_trees.transform(node, ctx)
-
-    with self.compiled(node, ns) as result:
-      new_name, _ = ctx.namer.compiled_function_name(('test_fn_1',))
-      setattr(result, new_name, other_test_fn_1)
-      self.assertEquals(result.test_fn_2(1), 3)
-
-  def test_dynamic_function(self):
+  def test_normal_function(self):
 
-    def test_fn_1():
-      raise ValueError('This should be masked by the mock in self.compiled.')
-
-    def test_fn_2(f):
+    def test_fn(f):
       return f() + 3
 
-    with self.converted(test_fn_2, call_trees, {}) as result:
-      # 10 = 7 (from the mock) + 3 (from test_fn_2)
-      self.assertEquals(10, result.test_fn_2(test_fn_1))
+    with self.converted(test_fn, call_trees, {}) as result:
+      self.assertEquals(
+          result.test_fn(None),
+          converter_testing.RESULT_OF_MOCK_CONVERTED_CALL + 3)
+      self.assertListEqual(self.dynamic_calls, [()])
 
-  def test_basic_method(self):
+  def test_class_method(self):
 
     class TestClass(object):
 
-      def test_fn_1(self, a):
-        return a + 1
-
-      def test_fn_2(self, a):
-        return self.test_fn_1(a) + 1
-
-    ns = {'TestClass': TestClass}
-    node, ctx = self.prepare(
-        TestClass.test_fn_2,
-        ns,
-        namer=converter_testing.FakeNoRenameNamer(),
-        arg_types={'self': (TestClass.__name__, TestClass)})
-    node = call_trees.transform(node, ctx)
-
-    with self.compiled(node, ns) as result:
-      tc = TestClass()
-      self.assertEquals(3, result.test_fn_2(tc, 1))
-
-  def test_known_called_lambda(self):
-
-    l = lambda x: x
-
-    def test_fn(a):
-      return l(a)
+      def test_method(self, a):
+        return self.other_method(a) + 1
 
-    ns = {'l': l}
-    node, ctx = self.prepare(test_fn, ns)
-    node = call_trees.transform(node, ctx)
+    tc = TestClass()
+    with self.converted(TestClass.test_method, call_trees, {}) as result:
+      self.assertEquals(converter_testing.RESULT_OF_MOCK_CONVERTED_CALL + 1,
+                        result.test_method(tc, 1))
+      self.assertListEqual(self.dynamic_calls, [(1,)])
 
-    with self.compiled(node, ns) as result:
-      self.assertEquals(1, result.test_fn(1))
+  def test_object_method(self):
 
-  def test_known_called_namedtuple(self):
-
-    nt = collections.namedtuple('TestNamedTuple', ['a'])
-
-    def test_fn(a):
-      return nt(a)
-
-    ns = {'nt': nt}
-    node, ctx = self.prepare(test_fn, ns)
-    node = call_trees.transform(node, ctx)
-
-    with self.compiled(node, ns) as result:
-      self.assertEquals(nt(1), result.test_fn(1))
-
-  def test_py_func_known_function(self):
-
-    def test_fn():
-      return np.random.binomial(2, 0.5)
-
-    with self.converted(test_fn, call_trees, {'np': np},
-                        dtypes.int64) as result:
-      with self.cached_session() as sess:
-        self.assertTrue(isinstance(result.test_fn(), ops.Tensor))
-        self.assertIn(self.evaluate(result.test_fn()), (0, 1, 2))
-
-  def test_uncompiled_modules(self):
-
-    def test_fn(a):
-      a = math_ops.multiply(a, constant_op.constant(2))
-      a = math_ops.add(a, constant_op.constant(1))
-      return a
-
-    ns = {'math_ops': math_ops, 'constant_op': constant_op}
-    node, ctx = self.prepare(
-        test_fn,
-        ns,
-        arg_types=set(((math_ops.__name__,), (constant_op.__name__,))))
-    node = call_trees.transform(node, ctx)
-
-    with self.compiled(node, ns) as result:
-      with self.cached_session() as sess:
-        result_tensor = result.test_fn(constant_op.constant(1))
-        self.assertEquals(self.evaluate(result_tensor), 3)
-
-  def test_call_to_decorated_function(self):
-
-    def decorator(f):
-      return f
-
-    @decorator
-    def called_fn(a):
-      return a
+    class TestClass(object):
 
-    def test_fn(a):
-      return called_fn(a)
+      def test_method(self, a):
+        return self.other_method(a) + 1
 
-    node, ctx = self.prepare(test_fn, {'called_fn': called_fn})
-    node = call_trees.transform(node, ctx)
+    tc = TestClass()
+    with self.converted(tc.test_method, call_trees, {}) as result:
+      self.assertEquals(converter_testing.RESULT_OF_MOCK_CONVERTED_CALL + 1,
+                        result.test_method(tc, 1))
+      self.assertListEqual(self.dynamic_calls, [(1,)])
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/autograph/core/config.py b/tensorflow/python/autograph/core/config.py
index 574f819504..5dce3e6dee 100644
--- a/tensorflow/python/autograph/core/config.py
+++ b/tensorflow/python/autograph/core/config.py
@@ -28,21 +28,16 @@ PYTHON_LITERALS = {
     'float': float,
 }
 
-DEFAULT_UNCOMPILED_MODULES = set((
-    ('tensorflow',),
-    (utils.__name__,),
 
-    # All of tensorflow's subpackages. Unlike the root tf module, they don't
-    # have well-known names. Not referring to the module directly to avoid
-    # circular imports.
-    (
-        utils.__name__[:-len('.python.autograph.utils')],),
-))
+def internal_module_name(name):
+  full_name = utils.__name__
+  name_start = full_name.find(name)
+  name_end = name_start + len(name) + 1
+  return full_name[:name_end]
 
-NO_SIDE_EFFECT_CONSTRUCTORS = set(('tensorflow',))
 
-# TODO(mdan): Also allow controlling the generated names.
-# TODO(mdan); Consolidate all internal imports into a single __ag module.
+DEFAULT_UNCOMPILED_MODULES = set(((internal_module_name('tensorflow'),),))
+
 COMPILED_IMPORT_STATEMENTS = (
     'from __future__ import print_function',
 )
diff --git a/tensorflow/python/autograph/impl/api.py b/tensorflow/python/autograph/impl/api.py
index 7e0b5e79b8..3ce6896e93 100644
--- a/tensorflow/python/autograph/impl/api.py
+++ b/tensorflow/python/autograph/impl/api.py
@@ -18,7 +18,10 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import collections
+import copy
 import functools
+import pdb
 import sys
 
 from enum import Enum
@@ -35,9 +38,9 @@ from tensorflow.python.autograph.operators import py_builtins
 from tensorflow.python.autograph.pyct import compiler
 from tensorflow.python.autograph.pyct import errors
 from tensorflow.python.autograph.pyct import inspect_utils
+from tensorflow.python.autograph.utils import ag_logging as logging
 from tensorflow.python.autograph.utils import py_func
 from tensorflow.python.framework import tensor_util
-from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import nest
 from tensorflow.python.util import tf_decorator
 from tensorflow.python.util import tf_inspect
@@ -161,7 +164,9 @@ def do_not_convert(run_as=RunMode.GRAPH, return_dtypes=None):
 
 def converted_call(f, owner, options, *args, **kwargs):
   """Compiles a function call inline. For internal use only."""
-  logging.vlog(logging.DEBUG, 'Converted call: %s; owner: %s', f, owner)
+  logging.log(1,
+              'Converted call: %s; owner: %s\n    args: %s\n    kwargs: %s\n',
+              f, owner, args, kwargs)
 
   if owner is not None:
     if not isinstance(f, str):
@@ -180,10 +185,20 @@ def converted_call(f, owner, options, *args, **kwargs):
   if inspect_utils.isbuiltin(f):
     return py_builtins.overload_of(f)(*args, **kwargs)
 
-  # Don't convert wrapt-decorated functions/methods.
-  # TODO(b/122265385): Fully support wrapt
+  # TODO(b/122265385): Remove this bypass.
   if ('wrapt' in sys.modules and
+      hasattr(sys.modules['wrapt'], 'FunctionWrapper') and
       isinstance(f, sys.modules['wrapt'].FunctionWrapper)):
+    logging.warn(
+        'Entity {} appears to be decorated by wrapt, which is not yet supported'
+        ' by AutoGraph. The function will be called without transformation.'
+        ' You may however apply AutoGraph before the decorator.'.format(f), 1)
+    return f(*args, **kwargs)
+
+  # Other built-in modules are permanently whitelisted.
+  # TODO(mdan): Figure out how to do this consistently for all stdlib modules.
+  if (f in collections.__dict__.values() or f in pdb.__dict__.values() or
+      f in copy.__dict__.values()):
     return f(*args, **kwargs)
 
   # TODO(mdan): This needs cleanup.
@@ -213,91 +228,118 @@ def converted_call(f, owner, options, *args, **kwargs):
   if not options.internal_convert_user_code:
     return f(*args, **kwargs)
 
-  # Unwrap functools.partial objects
-  # TODO(mdan): Consider sharing unwrapping logic with tf_inspect.
-  while isinstance(f, functools.partial):
-    args = f.args + args
-    new_kwargs = {}
-    if f.keywords is not None:
-      new_kwargs.update(f.keywords)
-    new_kwargs.update(kwargs)
-    kwargs = new_kwargs
-    f = f.func
-
-  if tf_inspect.isfunction(f) or tf_inspect.ismethod(f):
-    # Regular functions
-    target_entity = f
-    arg_map_target = f
-    f_self = inspect_utils.getmethodself(f)
-
-    # TODO(b/119246461): This may be more elegantly handled using __get__?
-    if f_self is not None:
-      # If this is a method call, it may or may not include self.
-      #
-      # Example when self is included:
-      #   converted_call(to_graph(foo.bar), foo)
-      #
-      # Example when self is not included:
-      #   super(...).foo(args)
-      #
-      if owner is not None and (not args or args[0] is not owner):
-        effective_args = (owner,) + args
-      else:
-        # When the owner is not specified, use the result of
-        # inspect_utils.getmethodclass.
-        # TODO(b/119246461): Make sure an owner is always specified.
-        if not args or args[0] is not f_self:
-          effective_args = (f_self,) + args
+  # TODO(mdan): Move this entire block inside to_graph.
+  try:  # Begin of transformation error guards
+
+    # Unwrap functools.partial objects
+    # TODO(mdan): Consider sharing unwrapping logic with tf_inspect.
+    while isinstance(f, functools.partial):
+      args = f.args + args
+      new_kwargs = {}
+      if f.keywords is not None:
+        new_kwargs.update(f.keywords)
+      new_kwargs.update(kwargs)
+      kwargs = new_kwargs
+      f = f.func
+
+    if tf_inspect.isfunction(f) or tf_inspect.ismethod(f):
+      # Regular functions
+      target_entity = f
+      arg_map_target = f
+      f_self = inspect_utils.getmethodself(f)
+
+      # TODO(b/119246461): This may be more elegantly handled using __get__?
+      if f_self is not None:
+        # If this is a method call, it may or may not include self.
+        #
+        # Example when self is included:
+        #   converted_call(to_graph(foo.bar), foo)
+        #
+        # Example when self is not included:
+        #   super(...).foo(args)
+        #
+        if owner is not None and (not args or args[0] is not owner):
+          effective_args = (owner,) + args
         else:
-          effective_args = (f_self,) + args[1:]
-      partial_types = (f_self,)
-    else:
+          # When the owner is not specified, use the result of
+          # inspect_utils.getmethodclass.
+          # TODO(b/119246461): Make sure an owner is always specified.
+          if not args or args[0] is not f_self:
+            effective_args = (f_self,) + args
+          else:
+            effective_args = (f_self,) + args[1:]
+        partial_types = (f_self,)
+      else:
+        effective_args = args
+        partial_types = ()
+
+    elif tf_inspect.isclass(f):
+      # Constructors
+      target_entity = f
+      arg_map_target = f.__init__
       effective_args = args
       partial_types = ()
 
-  elif tf_inspect.isclass(f):
-    # Constructors
-    target_entity = f
-    arg_map_target = f.__init__
-    effective_args = args
-    partial_types = ()
-
-  elif hasattr(f, '__call__') and hasattr(f, '__class__'):
-    # Callable objects
-    target_entity = f.__call__
-    arg_map_target = f.__call__
-    effective_args = (f,) + args
-    partial_types = (f.__class__,)
-
-  else:
-    raise NotImplementedError('unknown callable type "%s"' % type(f))
-
-  arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs)
-  arg_types = {}
-  for name, arg in arg_values.items():
-    arg_class = arg.__class__
-    arg_types[name] = (arg_class.__name__, arg_class)
-
-  # When called from within a decorator, this is the only indication that
-  # the function is a method - it appears that the decorator is applied
-  # before the method is bound.
-  if not partial_types:
-    if 'self' in arg_values:
-      if tf_inspect.isclass(arg_values['self'].__class__):
-        partial_types = (arg_values['self'].__class__,)
-    elif 'cls' in arg_values:
-      if tf_inspect.isclass(arg_values['cls']):
-        partial_types = (arg_values['cls'],)
-
-  converted_f = to_graph(
-      target_entity,
-      recursive=options.recursive,
-      arg_values=arg_values,
-      arg_types=arg_types,
-      experimental_optional_features=options.optional_features,
-      experimental_strip_decorators=options.strip_decorators,
-      experimental_verbose=options.verbose,
-      experimental_partial_types=partial_types)
+    elif hasattr(f, '__call__') and hasattr(f, '__class__'):
+      # Callable objects
+      target_entity = f.__call__
+      arg_map_target = f.__call__
+      effective_args = (f,) + args
+      partial_types = (f.__class__,)
+
+    else:
+      raise NotImplementedError('unknown callable type "%s"' % type(f))
+
+    arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs)
+    arg_types = {}
+    for name, arg in arg_values.items():
+      arg_class = arg.__class__
+      arg_types[name] = (arg_class.__name__, arg_class)
+
+    # When called from within a decorator, this is the only indication that
+    # the function is a method - it appears that the decorator is applied
+    # before the method is bound.
+    if not partial_types:
+      if 'self' in arg_values:
+        if tf_inspect.isclass(arg_values['self'].__class__):
+          partial_types = (arg_values['self'].__class__,)
+      elif 'cls' in arg_values:
+        if tf_inspect.isclass(arg_values['cls']):
+          partial_types = (arg_values['cls'],)
+
+    logging.log(3, 'Partial types in conversion of %s: %s', target_entity,
+                partial_types)
+
+    converted_f = to_graph(
+        target_entity,
+        recursive=options.recursive,
+        arg_values=arg_values,
+        arg_types=arg_types,
+        experimental_optional_features=options.optional_features,
+        experimental_strip_decorators=options.strip_decorators,
+        experimental_verbose=options.verbose,
+        experimental_partial_types=partial_types)
+
+    if logging.has_verbosity(2):
+      logging.log(2, 'Defaults of %s : %s', converted_f,
+                  converted_f.__defaults__)
+      callargs = tf_inspect.getcallargs(converted_f, *effective_args, **kwargs)
+      formatted_callargs = '\n'.join(
+          '    {}: {}'.format(k, v) for k, v in callargs.items())
+      logging.log(2, 'Calling %s with\n%s\n', converted_f, formatted_callargs)
+
+  # TODO(mdan): Reduce this list.
+  except (errors.AutoGraphError, AssertionError, AttributeError, IndexError,
+          KeyError, NameError, NotImplementedError, SyntaxError, TypeError,
+          ValueError, IOError) as e:
+    logging.log(1, 'Error transforming entity %s', target_entity, exc_info=True)
+    logging.warn(
+        'Entity %s could not be transformed and will be staged without change.'
+        ' Error details can be found in the logs when running with the env'
+        ' variable AUTOGRAPH_VERBOSITY=5. Please report this to the AutoGraph'
+        ' team. Cause: %s', e)
+
+    return f(*args, **kwargs)
 
   result = converted_f(*effective_args, **kwargs)
 
@@ -442,8 +484,15 @@ def to_graph(entity,
         compiled_module.__dict__[key] = val
     compiled = getattr(compiled_module, name)
 
-    if tf_inspect.isfunction(entity):
+    if hasattr(entity, '__defaults__'):
+      logging.log(3, 'Default args mapping: %s has: %s', entity,
+                  entity.__defaults__)
       compiled.__defaults__ = entity.__defaults__
+    else:
+      logging.log(3, 'Default args mapping: %s has no __defaults__', entity)
+
+    logging.log(3, 'Namespace of %s includes: %s', compiled,
+                compiled_module.__dict__.keys())
 
     if hasattr(compiled, '__globals__'):
       # Remove self to avoid circular references. This will probably only work
diff --git a/tensorflow/python/autograph/impl/conversion.py b/tensorflow/python/autograph/impl/conversion.py
index 6912106165..32a7d89621 100644
--- a/tensorflow/python/autograph/impl/conversion.py
+++ b/tensorflow/python/autograph/impl/conversion.py
@@ -20,6 +20,8 @@ from __future__ import print_function
 
 import functools
 import imp
+# import types
+import unittest
 
 import gast
 
@@ -80,28 +82,31 @@ def is_whitelisted_for_graph(o):
     m = functools
   else:
     m = tf_inspect.getmodule(o)
-  if not hasattr(m, '__name__'):
-    # Note: typically it's builtins that fall in this category. Builtins will
-    # be handled by specific code that follows this screening layer.
-    logging.log(2, '%s is NOT whitelisted: unknown module name', o)
-    return False
-
-  for prefix, in config.DEFAULT_UNCOMPILED_MODULES:
-    if m.__name__.startswith(prefix):
-      logging.log(2, '%s is whitelisted: name starts with "%s"', o, prefix)
+
+  if hasattr(m, '__name__'):
+    # Builtins typically have unnamed modules.
+    for prefix, in config.DEFAULT_UNCOMPILED_MODULES:
+      if m.__name__.startswith(prefix):
+        logging.log(2, '%s is whitelisted: name starts with "%s"', o, prefix)
+        return True
+
+    # Temporary -- whitelist tensorboard modules.
+    # TODO(b/122731813): Remove.
+    if m.__name__ == 'tensorboard' or '.tensorboard' in m.__name__:
+      logging.log(2, '%s is whitelisted: name contains "tensorboard"', o)
       return True
 
   if hasattr(o, 'autograph_info__') or hasattr(o, '__ag_compiled'):
     logging.log(2, '%s is whitelisted: already converted', o)
     return True
 
-  if (not inspect_utils.isweakrefself(o) and not tf_inspect.isclass(o) and
-      hasattr(o, '__call__') and hasattr(o, '__class__')):
+  if hasattr(o, '__call__'):
     # Callable objects: whitelisted if their __call__ method is.
-    call_whitelisted = is_whitelisted_for_graph(o.__call__)
-    if call_whitelisted:
+    # The type check avoids infinite recursion around the __call__ method
+    # of function objects.
+    if (type(o) != type(o.__call__)) and is_whitelisted_for_graph(o.__call__):  # pylint: disable=unidiomatic-typecheck
       logging.log(2, '%s is whitelisted: object __call__ whitelisted', o)
-      return call_whitelisted
+      return True
 
   if tf_inspect.ismethod(o):
     # Methods of whitelisted classes are also whitelisted, even if they are
@@ -121,6 +126,10 @@ def is_whitelisted_for_graph(o):
 
     owner_class = inspect_utils.getmethodclass(o)
     if owner_class is not None:
+      if issubclass(owner_class, unittest.TestCase):
+        logging.log(2, '%s is whitelisted: method of TestCase subclass', o)
+        return True
+
       owner_class = inspect_utils.getdefiningclass(o, owner_class)
       if is_whitelisted_for_graph(owner_class):
         logging.log(2, '%s is whitelisted: owner is whitelisted %s', o,
@@ -132,9 +141,10 @@ def is_whitelisted_for_graph(o):
     # because they don't expose source code. But we assume they are safe for
     # graph mode since they are just containers.
     if tf_inspect.isclass(o) and len(o.__bases__) > 1:
-      logging.warn_first_n(
-          'Entity {} looks like a namedtuple subclass. If it has any custom'
-          ' methods, they will not be converted by AutoGraph.'.format(o), 1)
+      logging.warn(
+          'Entity {} looks like a namedtuple subclass. Its constructor will'
+          ' not be converted by AutoGraph, but if it has any custom methods,'
+          ' those will be.'.format(o), 1)
     logging.log(2, '%s is whitelisted: named tuple', o)
     return True
 
@@ -207,6 +217,9 @@ def entity_to_graph(o, program_ctx, arg_values, arg_types):
   if logging.has_verbosity(2):
     logging.log(2, 'Compiled output of %s:\n\n%s\n', o,
                 compiler.ast_to_source(node))
+  if logging.has_verbosity(4):
+    for n in node:
+      logging.log(4, 'Compiled AST of %s:\n\n%s\n', o, gast.dump(n))
 
   if program_ctx.options.recursive:
     while True:
diff --git a/tensorflow/python/autograph/impl/conversion_test.py b/tensorflow/python/autograph/impl/conversion_test.py
index cd893e3ff1..ddda4089fd 100644
--- a/tensorflow/python/autograph/impl/conversion_test.py
+++ b/tensorflow/python/autograph/impl/conversion_test.py
@@ -92,11 +92,9 @@ class ConversionTest(test.TestCase):
     conversion.entity_to_graph(f, program_ctx, None, None)
 
     self.assertTrue(f in program_ctx.dependency_cache)
-    self.assertTrue(g in program_ctx.dependency_cache)
+    self.assertFalse(g in program_ctx.dependency_cache)
     f_node = program_ctx.dependency_cache[f][0]
-    g_node = program_ctx.dependency_cache[g][0]
     self.assertEqual('tf__f', f_node.name)
-    self.assertEqual('tf__g', g_node.name)
 
   def test_entity_to_graph_class_hierarchy(self):
 
diff --git a/tensorflow/python/autograph/pyct/inspect_utils.py b/tensorflow/python/autograph/pyct/inspect_utils.py
index 6d9bc43d34..eab01ee9cd 100644
--- a/tensorflow/python/autograph/pyct/inspect_utils.py
+++ b/tensorflow/python/autograph/pyct/inspect_utils.py
@@ -31,7 +31,7 @@ from tensorflow.python.util import tf_inspect
 
 # These functions test negative for isinstance(*, types.BuiltinFunctionType)
 # and inspect.isbuiltin, and are generally not visible in globals().
-# TODO(mdan): Find a more generic way to test this - just enumerate __builtin__?
+# TODO(mdan): Remove this.
 SPECIAL_BUILTINS = {
     'dict': dict,
     'enumerate': enumerate,
@@ -42,6 +42,7 @@ SPECIAL_BUILTINS = {
     'print': print,
     'range': range,
     'tuple': tuple,
+    'type': type,
     'zip': zip
 }
 
@@ -73,7 +74,7 @@ def isnamedtuple(f):
 
 def isbuiltin(f):
   """Returns True if the argument is a built-in function."""
-  if f in SPECIAL_BUILTINS.values():
+  if f in six.moves.builtins.__dict__.values():
     return True
   if isinstance(f, types.BuiltinFunctionType):
     return True
@@ -125,6 +126,10 @@ def getqualifiedname(namespace, object_, max_depth=5, visited=None):
   if visited is None:
     visited = set()
 
+  # Copy the dict to avoid "changed size error" during concurrent invocations.
+  # TODO(mdan): This is on the hot path. Can we avoid the copy?
+  namespace = dict(namespace)
+
   for name in namespace:
     # The value may be referenced by more than one symbol, case in which
     # any symbol will be fine. If the program contains symbol aliases that
diff --git a/tensorflow/python/autograph/pyct/parser.py b/tensorflow/python/autograph/pyct/parser.py
index 011d80dd62..67bfb55fc7 100644
--- a/tensorflow/python/autograph/pyct/parser.py
+++ b/tensorflow/python/autograph/pyct/parser.py
@@ -23,6 +23,7 @@ from __future__ import print_function
 
 import re
 import textwrap
+import threading
 
 import gast
 import six
@@ -30,10 +31,14 @@ import six
 from tensorflow.python.util import tf_inspect
 
 
+_parse_lock = threading.Lock()  # Prevents linecache concurrency errors.
+
+
 def parse_entity(entity):
   """Returns the AST of given entity."""
   try:
-    source = tf_inspect.getsource(entity)
+    with _parse_lock:
+      source = tf_inspect.getsource(entity)
   except (IOError, OSError) as e:
     raise ValueError(
         'Unable to locate the source code of {}. Note that functions defined'
diff --git a/tensorflow/python/autograph/utils/ag_logging.py b/tensorflow/python/autograph/utils/ag_logging.py
index 847000aa25..cd737a8290 100644
--- a/tensorflow/python/autograph/utils/ag_logging.py
+++ b/tensorflow/python/autograph/utils/ag_logging.py
@@ -110,7 +110,7 @@ def get_verbosity():
   global verbosity_level
   if verbosity_level is not None:
     return verbosity_level
-  return os.getenv(VERBOSITY_VAR_NAME, DEFAULT_VERBOSITY)
+  return int(os.getenv(VERBOSITY_VAR_NAME, DEFAULT_VERBOSITY))
 
 
 def has_verbosity(level):
@@ -131,5 +131,9 @@ def log(level, msg, *args, **kwargs):
       print(msg % args)
 
 
+def warn(msg, *args, **kwargs):
+  logging.warn(msg, *args, **kwargs)
+
+
 def warn_first_n(msg, *args, **kwargs):
   logging.log_first_n(logging.WARN, msg, *args, **kwargs)
-- 
GitLab


From 2cfa7dc2da11731545b189d08628685b009a889d Mon Sep 17 00:00:00 2001
From: Suharsh Sivakumar <suharshs@google.com>
Date: Sat, 2 Feb 2019 20:26:43 -0800
Subject: [PATCH 1823/2345] Gather int8 support.

PiperOrigin-RevId: 232157127
---
 tensorflow/lite/kernels/gather.cc       |  5 +++++
 tensorflow/lite/kernels/gather_test.cc  | 18 ++++++++++++++++++
 tensorflow/lite/kernels/register.cc     |  4 +++-
 tensorflow/lite/toco/tflite/operator.cc |  6 ++++++
 4 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/kernels/gather.cc b/tensorflow/lite/kernels/gather.cc
index f205daae13..54d05adbcf 100644
--- a/tensorflow/lite/kernels/gather.cc
+++ b/tensorflow/lite/kernels/gather.cc
@@ -57,6 +57,7 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   switch (input->type) {
     case kTfLiteFloat32:
     case kTfLiteUInt8:
+    case kTfLiteInt8:
     case kTfLiteInt64:
     case kTfLiteInt32:
       break;
@@ -135,6 +136,8 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
         return Gather<float, int32_t>(*params, input, positions, output);
       case kTfLiteUInt8:
         return Gather<uint8_t, int32_t>(*params, input, positions, output);
+      case kTfLiteInt8:
+        return Gather<int8_t, int32_t>(*params, input, positions, output);
       case kTfLiteInt32:
         return Gather<int32_t, int32_t>(*params, input, positions, output);
       case kTfLiteInt64:
@@ -153,6 +156,8 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
         return Gather<float, int64_t>(*params, input, positions, output);
       case kTfLiteUInt8:
         return Gather<uint8_t, int64_t>(*params, input, positions, output);
+      case kTfLiteInt8:
+        return Gather<int8_t, int64_t>(*params, input, positions, output);
       case kTfLiteInt32:
         return Gather<int32_t, int64_t>(*params, input, positions, output);
       case kTfLiteInt64:
diff --git a/tensorflow/lite/kernels/gather_test.cc b/tensorflow/lite/kernels/gather_test.cc
index 7b5f843489..b5461c204f 100644
--- a/tensorflow/lite/kernels/gather_test.cc
+++ b/tensorflow/lite/kernels/gather_test.cc
@@ -205,6 +205,24 @@ TEST(TypesGatherOpTest, Uint8Int64) {
   EXPECT_THAT(m.GetOutput<uint8_t>(), ElementsAreArray({14, 15, 133, 134}));
 }
 
+TEST(TypesGatherOpTest, Int8Int32) {
+  GatherOpModel m({TensorType_INT8, {2, 2}}, {TensorType_INT32, {2}});
+  m.SetInput<int8_t>({-13, -120, 14, 15});
+  m.SetPositions<int32_t>({1, 0});
+  m.Invoke();
+
+  EXPECT_THAT(m.GetOutput<int8_t>(), ElementsAreArray({14, 15, -13, -120}));
+}
+
+TEST(TypesGatherOpTest, Int8Int64) {
+  GatherOpModel m({TensorType_INT8, {2, 2}}, {TensorType_INT64, {2}});
+  m.SetInput<int8_t>({-13, -120, 14, 15});
+  m.SetPositions<int64_t>({1LL, 0LL});
+  m.Invoke();
+
+  EXPECT_THAT(m.GetOutput<int8_t>(), ElementsAreArray({14, 15, -13, -120}));
+}
+
 TEST(TypesGatherOpTest, Int64Int32) {
   GatherOpModel m({TensorType_INT64, {2, 2}}, {TensorType_INT32, {2}});
   m.SetInput<int64_t>({-(1LL << 34), 134LL, 14LL, 15LL});
diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index bc5ec4a1d6..12a53f93b8 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -228,7 +228,9 @@ BuiltinOpResolver::BuiltinOpResolver() {
              Register_RESIZE_NEAREST_NEIGHBOR());
   AddBuiltin(BuiltinOperator_SKIP_GRAM, Register_SKIP_GRAM());
   AddBuiltin(BuiltinOperator_SPACE_TO_DEPTH, Register_SPACE_TO_DEPTH());
-  AddBuiltin(BuiltinOperator_GATHER, Register_GATHER());
+  AddBuiltin(BuiltinOperator_GATHER, Register_GATHER(),
+             /* min_version */ 1,
+             /* max_version */ 2);
   AddBuiltin(BuiltinOperator_TRANSPOSE, Register_TRANSPOSE());
   AddBuiltin(BuiltinOperator_MEAN, Register_MEAN());
   AddBuiltin(BuiltinOperator_DIV, Register_DIV());
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index 61c0b605dd..81203dfa04 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -467,6 +467,12 @@ class Gather : public BuiltinOperator<GatherOperator, ::tflite::GatherOptions,
   }
 
   int GetVersion(const OperatorSignature& op_signature) const override {
+    const string& input_name = op_signature.op->inputs[0];
+    const Array& input_array = op_signature.model->GetArray(input_name);
+    // If the op take int8 input, it is version 2.
+    if (input_array.data_type == ArrayDataType::kInt8) {
+      return 2;
+    }
     return 1;
   }
 };
-- 
GitLab


From edbd8a15b91ef0d5ac3fdf3f573ce2b99cdd8a63 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Sat, 2 Feb 2019 22:07:46 -0800
Subject: [PATCH 1824/2345] Automated rollback of commit
 8e38fb87fc50aaf0f086a21e1ca1f39361604234

PiperOrigin-RevId: 232162141
---
 tensorflow/core/distributed_runtime/rpc/BUILD |  14 ++
 .../rpc/grpc_remote_worker.cc                 |  12 +-
 .../rpc/grpc_response_cache.cc                | 183 ++++++++++++++++++
 .../rpc/grpc_response_cache.h                 |  91 +++++++++
 .../rpc/grpc_rpc_factory.cc                   |   2 +-
 .../core/distributed_runtime/rpc/grpc_state.h |  85 ++++++--
 .../rpc/grpc_worker_service.cc                |  96 ++++++---
 .../rpc/grpc_worker_service.h                 |   6 +-
 8 files changed, 436 insertions(+), 53 deletions(-)
 create mode 100644 tensorflow/core/distributed_runtime/rpc/grpc_response_cache.cc
 create mode 100644 tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h

diff --git a/tensorflow/core/distributed_runtime/rpc/BUILD b/tensorflow/core/distributed_runtime/rpc/BUILD
index 16078f9688..607963484d 100644
--- a/tensorflow/core/distributed_runtime/rpc/BUILD
+++ b/tensorflow/core/distributed_runtime/rpc/BUILD
@@ -159,6 +159,17 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "grpc_response_cache",
+    srcs = ["grpc_response_cache.cc"],
+    hdrs = ["grpc_response_cache.h"],
+    deps = [
+        ":grpc_util",
+        "//tensorflow/core:lib",
+        "@com_google_absl//absl/time",
+    ],
+)
+
 tf_cuda_library(
     name = "grpc_worker_service",
     srcs = ["grpc_worker_service.cc"],
@@ -166,6 +177,7 @@ tf_cuda_library(
     deps = [
         ":async_service_interface",
         ":grpc_call",
+        ":grpc_response_cache",
         ":grpc_tensor_coding",
         ":grpc_util",
         ":grpc_worker_service_impl",
@@ -183,6 +195,8 @@ tf_cuda_library(
         "//tensorflow/core/distributed_runtime:worker_cache",
         "//tensorflow/core/distributed_runtime:worker_env",
         "//tensorflow/core/distributed_runtime:worker_session",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/time",
     ],
 )
 
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.cc b/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.cc
index 2daefcb399..2479e7368b 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.cc
@@ -39,6 +39,8 @@ limitations under the License.
 
 namespace tensorflow {
 
+const int kMaxWorkerRpcRetries = 10;
+
 class GrpcRemoteWorker : public WorkerInterface {
  public:
   explicit GrpcRemoteWorker(SharedGrpcChannelPtr channel,
@@ -259,17 +261,19 @@ class GrpcRemoteWorker : public WorkerInterface {
   // given callback, `done`, will be called when the RPC completes.
   void IssueRequest(const protobuf::Message* request,
                     protobuf::Message* response, const ::grpc::string& method,
-                    StatusCallback done, CallOptions* call_opts = nullptr) {
+                    StatusCallback done, CallOptions* call_opts = nullptr,
+                    int max_retries = kMaxWorkerRpcRetries) {
     new RPCState<protobuf::Message>(&stub_, cq_, method, *request, response,
                                     std::move(done), call_opts,
-                                    callback_threadpool_);
+                                    callback_threadpool_, max_retries);
   }
   void IssueRequest(const protobuf::Message* request, TensorResponse* response,
                     const ::grpc::string& method, StatusCallback done,
-                    CallOptions* call_opts = nullptr) {
+                    CallOptions* call_opts = nullptr,
+                    int max_retries = kMaxWorkerRpcRetries) {
     new RPCState<TensorResponse>(&stub_, cq_, method, *request, response,
                                  std::move(done), call_opts,
-                                 callback_threadpool_);
+                                 callback_threadpool_, max_retries);
   }
 
   // Helper function for initializing the RpcMethod objects below.
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.cc b/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.cc
new file mode 100644
index 0000000000..613c290905
--- /dev/null
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.cc
@@ -0,0 +1,183 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h"
+#include "tensorflow/core/platform/env.h"
+
+namespace tensorflow {
+
+struct WorkerCacheEntry {
+  enum class State {
+    PENDING = 0,
+    ACTIVE = 1,
+    FINISHED = 2,
+  };
+
+  State state = State::PENDING;
+  int64 expires_seconds;
+
+  ::grpc::ByteBuffer response_buf;
+  Status response_status;
+
+  // Additional retries may arrive while a request is still executing.  The
+  // callbacks for these calls are queued in `callbacks` and evaluated after
+  // the original request is completed.
+  std::vector<std::pair<RPCResponse, StatusCallback>> callbacks;
+};
+
+void RPCResponse::Encode(::grpc::ByteBuffer* tgt) const {
+  if (buf_ != nullptr) {
+    *tgt = *buf_;
+  } else {
+    CHECK(msg_ != nullptr);
+    ::grpc::Slice slice(msg_->ByteSizeLong());
+    msg_->SerializeWithCachedSizesToArray(
+        const_cast<uint8*>(reinterpret_cast<const uint8*>(slice.begin())));
+    ::grpc::ByteBuffer tmp(&slice, 1);
+    tgt->Swap(&tmp);
+  }
+}
+
+void RPCResponse::CopyFrom(const ::grpc::ByteBuffer& src) {
+  if (buf_ != nullptr) {
+    *buf_ = src;
+    return;
+  }
+
+  CHECK(msg_ != nullptr);
+  // We create a single slice when encoding protocol messages.
+  std::vector<::grpc::Slice> slices;
+  if (src.Dump(&slices).ok()) {
+    msg_->ParseFromArray(slices[0].begin(), slices[0].size());
+  } else {
+    LOG(ERROR) << "Failed to decode cached buffer.";
+  }
+}
+
+void GrpcResponseCache::LookupOrCompute(const string& key, RPCResponse response,
+                                        ComputeFunc compute_func,
+                                        StatusCallback done_cb) {
+  VLOG(1) << "Lookup " << key;
+  std::shared_ptr<WorkerCacheEntry> req;
+  MaybeCleanup();
+  {
+    mutex_lock m(mu_);
+
+    if (requests_.find(key) != requests_.end()) {
+      req = requests_[key];
+    } else {
+      req.reset(new WorkerCacheEntry);
+      requests_[key] = req;
+    }
+
+    if (req->state == WorkerCacheEntry::State::FINISHED) {
+      if (req->expires_seconds > Env::Default()->NowSeconds()) {
+        VLOG(1) << "Reuse cached response for " << key;
+        response.CopyFrom(req->response_buf);
+        done_cb(req->response_status);
+        return;
+      }
+      VLOG(1) << "Found expired cache entry for " << key;
+      req->state = WorkerCacheEntry::State::PENDING;
+      req->response_buf.Clear();
+    }
+
+    req->callbacks.push_back(std::make_pair(response, done_cb));
+
+    if (req->state == WorkerCacheEntry::State::ACTIVE) {
+      VLOG(1) << "Found active request for " << key
+              << ".  Adding entry to response queue.";
+      return;
+    }
+
+    VLOG(2) << "No cache entry for " << key << ", running user computation.";
+    req->state = WorkerCacheEntry::State::ACTIVE;
+    req->expires_seconds = Env::Default()->NowSeconds() + expire_time_seconds_;
+  }
+
+  compute_func([this, key, req, response](Status status) {
+    mutex_lock m(mu_);
+    response.Encode(&req->response_buf);
+    current_bytes_ += req->response_buf.Length();
+
+    req->response_status = status;
+    req->state = WorkerCacheEntry::State::FINISHED;
+
+    VLOG(1) << "Operation for " << key << " finished. "
+            << "Status: " << status << ", " << req->response_buf.Length()
+            << " response bytes, " << req->callbacks.size()
+            << " pending callbacks.";
+    for (auto& cb : req->callbacks) {
+      cb.first.CopyFrom(req->response_buf);
+      cb.second(req->response_status);
+    }
+    req->callbacks.clear();
+  });
+}
+
+// Remove all stale or expired cache entries if the cache is full.
+void GrpcResponseCache::MaybeCleanup() {
+  mutex_lock m(mu_);
+  if (current_bytes_ < max_bytes_) {
+    return;
+  }
+
+  VLOG(1) << "Cleanup: " << current_bytes_ << " -> " << max_bytes_;
+  std::vector<std::pair<string, std::shared_ptr<WorkerCacheEntry>>>
+      ordered_entries;
+  ordered_entries.reserve(requests_.size());
+  for (const auto& p : requests_) {
+    ordered_entries.push_back(std::make_pair(p.first, p.second));
+  }
+
+  std::sort(ordered_entries.begin(), ordered_entries.end(),
+            [](const std::pair<string, std::shared_ptr<WorkerCacheEntry>>& a,
+               const std::pair<string, std::shared_ptr<WorkerCacheEntry>>& b) {
+              return a.second->expires_seconds > b.second->expires_seconds;
+            });
+
+  std::unordered_map<string, std::shared_ptr<WorkerCacheEntry>> kept;
+  int64 now = Env::Default()->NowSeconds();
+  int64 bytes_used = 0;
+
+  // Always keep active requests.
+  for (auto& pair : ordered_entries) {
+    if (pair.second->state != WorkerCacheEntry::State::FINISHED) {
+      kept.insert(pair);
+    }
+  }
+
+  // Keep unexpired, finished requests up to half of max_bytes_.  This reduces
+  // chances of overfilling the cache when active requests complete and
+  // amortizes cache cleanup cost.
+  for (auto& pair : ordered_entries) {
+    if (pair.second->expires_seconds < now || bytes_used >= max_bytes_ / 2) {
+      break;
+    }
+
+    if (pair.second->state == WorkerCacheEntry::State::FINISHED) {
+      kept.insert(pair);
+      bytes_used += pair.second->response_buf.Length();
+    }
+  }
+
+  VLOG(1) << "Cleaned cache.  Bytes used: " << current_bytes_ << " -> "
+          << bytes_used << ". Cache size: " << requests_.size() << " -> "
+          << kept.size();
+  current_bytes_ = bytes_used;
+  std::swap(requests_, kept);
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h b/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h
new file mode 100644
index 0000000000..0892d9f788
--- /dev/null
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h
@@ -0,0 +1,91 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_RESPONSE_CACHE_H_
+#define TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_RESPONSE_CACHE_H_
+
+#include <memory>
+#include <unordered_map>
+#include <vector>
+
+#include "tensorflow/core/distributed_runtime/rpc/grpc_util.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/platform/protobuf.h"
+
+// gRPC response caching.  Most WorkerService methods cannot be retried directly
+// as they will fail or deadlock.  To enable retrying, we can instead cache
+// responses for a short period of time and reply to duplicate requests from the
+// cache.
+namespace tensorflow {
+
+// Union type to aid caching of either raw buffers (for RecvTensor RPCs) and
+// protocol buffer messages (for all other RPCs).
+class RPCResponse {
+ public:
+  explicit RPCResponse() : buf_(nullptr), msg_(nullptr) {}
+  explicit RPCResponse(::grpc::ByteBuffer* b) : buf_(b), msg_(nullptr) {}
+  explicit RPCResponse(protobuf::Message* m) : buf_(nullptr), msg_(m) {}
+
+  // Encode this response into the target buffer.
+  void Encode(::grpc::ByteBuffer* tgt) const;
+
+  // Copy from `src`: if this is a buffer, make a shallow copy.
+  // For protocol messages, parse the response from `src`.
+  void CopyFrom(const ::grpc::ByteBuffer& src);
+
+ private:
+  ::grpc::ByteBuffer* buf_;
+  protobuf::Message* msg_;
+};
+
+typedef std::function<void(StatusCallback)> ComputeFunc;
+struct WorkerCacheEntry;
+
+// Track and cache the state of worker service RPCs.  An RPC can be in 3 states:
+//
+// * PENDING: this is the first call of the RPC, and it will transition to
+// * ACTIVE: another thread is active processing this RPC
+// * FINISHED: the worker has finished processing the method
+//
+// The response from completed RPCs are LRU cached until either `max_bytes`
+// bytes are in use by the cache or they expire (according to `expire_time`).
+class GrpcResponseCache {
+ public:
+  GrpcResponseCache(int64 max_bytes, int64 expire_time_seconds)
+      : max_bytes_(max_bytes), expire_time_seconds_(expire_time_seconds) {}
+
+  // Lookup the result for key.
+  // If it is finished, invoke `done_cb` immediately after filling `response`.
+  // If active, done_db will be invoked when the current call completes.
+  // Otherwise, invoke `compute_func` to fill the cache and invoke done_cb.
+  void LookupOrCompute(const string& key, RPCResponse response,
+                       ComputeFunc compute_func, StatusCallback done_cb);
+
+  // Remove all stale or expired cache entries if the cache is full.
+  void MaybeCleanup();
+
+ private:
+  int64 current_bytes_ GUARDED_BY(mu_) = 0;
+  const int64 max_bytes_;
+  const int64 expire_time_seconds_;
+
+  std::unordered_map<string, std::shared_ptr<WorkerCacheEntry>> requests_
+      GUARDED_BY(mu_);
+  mutex mu_;
+};
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_RESPONSE_CACHE_H_
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_rpc_factory.cc b/tensorflow/core/distributed_runtime/rpc/grpc_rpc_factory.cc
index 7f63cc9344..3635caf3d1 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_rpc_factory.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_rpc_factory.cc
@@ -210,7 +210,7 @@ void GrpcRPCFactory::StartCall(const Tensor& address_t, const Tensor& method_t,
       get_stub(index), &completion_queue_, *get_method_ptr(index),
       call->request(), call->response(),
       /*done=*/[call](const Status& s) { call->Done(s); }, call->call_opts(),
-      nullptr /*threadpool*/, fail_fast_, timeout_in_ms_);
+      nullptr /*threadpool*/, fail_fast_, timeout_in_ms_, 0 /* max_retries */);
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_state.h b/tensorflow/core/distributed_runtime/rpc/grpc_state.h
index d73638651f..0ca64dc159 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_state.h
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_state.h
@@ -32,6 +32,9 @@ limitations under the License.
 namespace tensorflow {
 
 // Object allocated per active RPC.
+// Manage the state of a single asynchronous RPC request.  If `max_retries`
+// is greater than 0, the request will be retried for any transient failures
+// as long as the overall deadline has not elapsed.
 template <class Response>
 class RPCState : public GrpcClientCQTag {
  public:
@@ -39,34 +42,55 @@ class RPCState : public GrpcClientCQTag {
   RPCState(::grpc::GenericStub* stub, ::grpc::CompletionQueue* cq,
            const ::grpc::string& method, const protobuf::Message& request,
            Response* response, StatusCallback done, CallOptions* call_opts,
-           thread::ThreadPool* threadpool)
+           thread::ThreadPool* threadpool, int32 max_retries = 0)
       : RPCState(stub, cq, method, request, response, std::move(done),
                  call_opts, threadpool, /*fail_fast=*/false,
-                 /*timeout_in_ms=*/0) {}
+                 /*timeout_in_ms=*/0, max_retries) {}
 
   template <typename Request>
   RPCState(::grpc::GenericStub* stub, ::grpc::CompletionQueue* cq,
            const ::grpc::string& method, const Request& request,
            Response* response, StatusCallback done, CallOptions* call_opts,
-           thread::ThreadPool* threadpool, bool fail_fast, int64 timeout_in_ms)
-      : call_opts_(call_opts), threadpool_(threadpool), done_(std::move(done)) {
-    context_.set_fail_fast(fail_fast);
-    if (timeout_in_ms > 0) {
-      context_.set_deadline(gpr_time_from_millis(timeout_in_ms, GPR_TIMESPAN));
-    }
-
-    if (call_opts) {
-      call_opts->SetCancelCallback([this]() { context_.TryCancel(); });
-    }
-
+           thread::ThreadPool* threadpool, bool fail_fast, int64 timeout_in_ms,
+           int32 max_retries)
+      : call_opts_(call_opts),
+        threadpool_(threadpool),
+        done_(std::move(done)),
+        cq_(cq),
+        stub_(stub),
+        method_(method),
+        max_retries_(max_retries),
+        timeout_in_ms_(timeout_in_ms),
+        fail_fast_(fail_fast) {
     response_ = response;
     ::grpc::Status s = GrpcMaybeUnparseProto(request, &request_buf_);
     if (!s.ok()) {
       LOG(ERROR) << "GrpcMaybeUnparseProto returned with non-ok status: "
                  << s.error_message();
+      // Skip retry logic if we fail to parse our request.
+      done_(FromGrpcStatus(s));
+      delete this;
+      return;
+    }
+    StartCall();
+  }
+
+  void StartCall() {
+    context_.reset(new ::grpc::ClientContext());
+    context_->set_fail_fast(fail_fast_);
+
+    if (timeout_in_ms_ > 0) {
+      context_->set_deadline(
+          gpr_time_from_millis(timeout_in_ms_, GPR_TIMESPAN));
     }
-    call_ =
-        std::move(stub->PrepareUnaryCall(&context_, method, request_buf_, cq));
+    if (call_opts_) {
+      call_opts_->SetCancelCallback([this]() { context_->TryCancel(); });
+    }
+
+    VLOG(2) << "Starting call: " << method_;
+
+    call_ = std::move(
+        stub_->PrepareUnaryCall(context_.get(), method_, request_buf_, cq_));
     call_->StartCall();
     call_->Finish(&response_buf_, &status_, this);
   }
@@ -89,16 +113,26 @@ class RPCState : public GrpcClientCQTag {
         threadpool_->Schedule([this]() { ParseAndCallDone(); });
       } else {
         ParseAndCallDone();
-        return;
       }
-    } else {
-      VLOG(2) << "Call returned with non-ok status: " << s;
+      return;
+    }
 
-      // Attach additional GRPC error information if any
+    VLOG(1) << method_ << " returned with non-ok status: " << s
+            << " Retries: " << num_retries_ << " Max: " << max_retries_ << "\n"
+            << context_->debug_error_string();
+    // Retry if we have any attempts left
+    if (++num_retries_ <= max_retries_ &&
+        (errors::IsUnavailable(s) || errors::IsUnknown(s))) {
+      response_buf_.Clear();
+      VLOG(1) << "Retrying call for " << method_ << "Retry: " << num_retries_
+              << " of " << max_retries_;
+      StartCall();
+    } else {
+      // Attach additional GRPC error information if any to the final status
       s = Status(s.code(),
                  strings::StrCat(s.error_message(),
                                  "\nAdditional GRPC error information:\n",
-                                 context_.debug_error_string()));
+                                 context_->debug_error_string()));
       done_(s);
       delete this;
     }
@@ -115,7 +149,7 @@ class RPCState : public GrpcClientCQTag {
 
  private:
   CallOptions* call_opts_;
-  ::grpc::ClientContext context_;
+  std::unique_ptr<::grpc::ClientContext> context_;
   thread::ThreadPool* threadpool_;
   std::unique_ptr<::grpc::GenericClientAsyncResponseReader> call_;
   Response* response_;
@@ -123,6 +157,15 @@ class RPCState : public GrpcClientCQTag {
   ::grpc::ByteBuffer response_buf_;
   ::grpc::Status status_;
   StatusCallback done_;
+  int64 timeout_in_ms_;
+
+  size_t num_retries_ = 0;
+  size_t max_retries_;
+
+  ::grpc::CompletionQueue* cq_;
+  ::grpc::GenericStub* stub_;
+  ::grpc::string method_;
+  bool fail_fast_;
 };
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc
index f45b9bf97a..904862100e 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.cc
@@ -16,11 +16,14 @@ limitations under the License.
 #include "tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h"
 
 #include <deque>
+#include <memory>
 #include <unordered_map>
+#include <vector>
 
 #include "grpcpp/alarm.h"
 #include "grpcpp/server_builder.h"
 
+#include "absl/container/flat_hash_map.h"
 #include "tensorflow/core/common_runtime/buf_rendezvous.h"
 #include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/common_runtime/device_mgr.h"
@@ -32,6 +35,7 @@ limitations under the License.
 #include "tensorflow/core/distributed_runtime/rendezvous_mgr_interface.h"
 #include "tensorflow/core/distributed_runtime/rpc/async_service_interface.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_call.h"
+#include "tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_tensor_coding.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_util.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h"
@@ -42,8 +46,12 @@ limitations under the License.
 #include "tensorflow/core/framework/collective.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/gtl/map_util.h"
+#include "tensorflow/core/lib/strings/strcat.h"
+#include "tensorflow/core/lib/strings/stringprintf.h"
 #include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/platform/tracing.h"
 #include "tensorflow/core/protobuf/transport_options.pb.h"
 #include "tensorflow/core/protobuf/worker.pb.h"
@@ -91,10 +99,11 @@ class GrpcWorkerServiceThread {
  public:
   explicit GrpcWorkerServiceThread(
       GrpcWorker* worker, ::grpc::ServerBuilder* builder,
-      std::unordered_map<int, int> queue_depth,
+      std::unordered_map<int, int> queue_depth, GrpcResponseCache* cache,
       grpc::WorkerService::AsyncService* worker_service)
       : worker_(worker),
         queue_depth_(queue_depth),
+        cache_(cache),
         worker_service_(worker_service),
         is_shutdown_(false) {
     cq_ = builder->AddCompletionQueue();
@@ -220,18 +229,32 @@ class GrpcWorkerServiceThread {
       NonOwnedProtoRunGraphResponse* wrapped_response =
           new NonOwnedProtoRunGraphResponse(&call->response);
       call->SetCancelCallback([call_opts]() { call_opts->StartCancel(); });
-      worker_->RunGraphAsync(call_opts, wrapped_request, wrapped_response,
-                             [call, call_opts, wrapped_request,
-                              wrapped_response](const Status& s) {
-                               if (!s.ok()) {
-                                 VLOG(1) << "Bad response from RunGraph:" << s;
-                               }
-                               call->ClearCancelCallback();
-                               delete call_opts;
-                               delete wrapped_request;
-                               delete wrapped_response;
-                               call->SendResponse(ToGrpcStatus(s));
-                             });
+      auto done_cb = [call, call_opts, wrapped_request,
+                      wrapped_response](const Status& s) {
+        VLOG(1) << "RunGraph::Done";
+        if (!s.ok()) {
+          VLOG(1) << "Bad response from RunGraph:" << s;
+        }
+        call->ClearCancelCallback();
+        delete call_opts;
+        delete wrapped_request;
+        delete wrapped_response;
+        call->SendResponse(ToGrpcStatus(s));
+      };
+
+      auto compute_fn = [this, call_opts, wrapped_request,
+                         wrapped_response](StatusCallback done) {
+        worker_->RunGraphAsync(call_opts, wrapped_request, wrapped_response,
+                               done);
+      };
+
+      if (cache_) {
+        string request_key = call->request.ShortDebugString();
+        cache_->LookupOrCompute(request_key, RPCResponse(&call->response),
+                                compute_fn, done_cb);
+      } else {
+        compute_fn(done_cb);
+      }
     });
     ENQUEUE_REQUEST(RunGraph, true);
   }
@@ -241,16 +264,28 @@ class GrpcWorkerServiceThread {
     Schedule([this, call]() {
       CallOptions* call_opts = new CallOptions;
       call->SetCancelCallback([call_opts]() { call_opts->StartCancel(); });
-      worker_->GrpcRecvTensorAsync(
-          call_opts, &call->request, &call->response,
-          [call, call_opts](const Status& s) {
-            call->ClearCancelCallback();
-            delete call_opts;
-            if (!s.ok()) {
-              VLOG(1) << "Bad response from RecvTensor:" << s;
-            }
-            call->SendResponse(ToGrpcStatus(s));
-          });
+
+      auto done_cb = [call, call_opts](const Status& s) {
+        call->ClearCancelCallback();
+        delete call_opts;
+        if (!s.ok()) {
+          VLOG(1) << "Bad response from RecvTensor:" << s;
+        }
+        call->SendResponse(ToGrpcStatus(s));
+      };
+
+      auto compute_fn = [this, &call_opts, &call](StatusCallback done) {
+        worker_->GrpcRecvTensorAsync(call_opts, &call->request, &call->response,
+                                     done);
+      };
+
+      if (cache_) {
+        string request_key = call->request.ShortDebugString();
+        cache_->LookupOrCompute(request_key, RPCResponse(&call->response),
+                                compute_fn, done_cb);
+      } else {
+        compute_fn(done_cb);
+      }
     });
     EnqueueRecvTensorRequestRaw();
   }
@@ -328,6 +363,7 @@ class GrpcWorkerServiceThread {
   std::unique_ptr<::grpc::ServerCompletionQueue> cq_;
   std::unique_ptr<Thread> thread_;
   std::unordered_map<int, int> queue_depth_;
+  GrpcResponseCache* cache_;
   grpc::WorkerService::AsyncService* const worker_service_;
 
   mutex shutdown_mu_;
@@ -341,9 +377,16 @@ class GrpcWorkerService : public AsyncServiceInterface {
                     GrpcWorkerServiceOptions options)
       : is_shutdown_(false) {
     builder->RegisterService(&worker_service_);
-    for (int i = 0; i < options.num_worker_threads; i++) {
-      threads_.emplace_back(new GrpcWorkerServiceThread(
-          worker, builder, options.queue_depth, &worker_service_));
+    if (options.response_cache_bytes > 0) {
+      cache_.reset(
+          new GrpcResponseCache(options.response_cache_bytes,
+                                options.response_cache_expires_seconds));
+    }
+
+    for (int i = 0; i < options.num_serving_threads; i++) {
+      threads_.emplace_back(
+          new GrpcWorkerServiceThread(worker, builder, options.queue_depth,
+                                      cache_.get(), &worker_service_));
     }
   }
 
@@ -378,6 +421,7 @@ class GrpcWorkerService : public AsyncServiceInterface {
   grpc::WorkerService::AsyncService worker_service_;
   std::vector<std::unique_ptr<GrpcWorkerServiceThread>> threads_;
 
+  std::unique_ptr<GrpcResponseCache> cache_;
   mutex service_shutdown_mu_;
   bool is_shutdown_ GUARDED_BY(service_shutdown_mu_);
 
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h
index 88beb6c216..8f2830c899 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service.h
@@ -16,8 +16,10 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_WORKER_SERVICE_H_
 #define TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_WORKER_SERVICE_H_
 
+#include <memory>
 #include <unordered_map>
 #include "tensorflow/core/distributed_runtime/recent_request_ids.h"
+#include "tensorflow/core/distributed_runtime/rpc/grpc_response_cache.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h"
 #include "tensorflow/core/distributed_runtime/worker.h"
 
@@ -63,7 +65,9 @@ struct GrpcWorkerServiceOptions {
   // Map from GrpcWorkerMethod id to queue depth.  If set this overrides the
   // default queue depth for a method.
   std::unordered_map<int, int> queue_depth;
-  int num_worker_threads = 8;
+  int num_serving_threads = 8;
+  int64 response_cache_bytes = 0;
+  int64 response_cache_expires_seconds = 0;
 };
 
 // Returns an implementation of WorkerService rpc service.
-- 
GitLab


From 226398bd70d6369dd97fa9cf4bee94af9a05f1d0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 3 Feb 2019 01:02:16 -0800
Subject: [PATCH 1825/2345] compat: Update forward compatibility horizon to
 2019-02-03

PiperOrigin-RevId: 232171049
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 27d4abafbc..104dc39734 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 2, 2)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 2, 3)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From f97802de4bb8ecf49941fd7499e35d02603622e8 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Sun, 3 Feb 2019 13:52:17 -0800
Subject: [PATCH 1826/2345] ODR-define bfloat16::NAN_VALUE and ZERO_VALUE.

These were missing proper definitions, so if they were ever ODR-used, you'd get
a link error.

PiperOrigin-RevId: 232210530
---
 tensorflow/core/lib/bfloat16/bfloat16.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/core/lib/bfloat16/bfloat16.cc b/tensorflow/core/lib/bfloat16/bfloat16.cc
index a591717fd1..e6e24bc078 100644
--- a/tensorflow/core/lib/bfloat16/bfloat16.cc
+++ b/tensorflow/core/lib/bfloat16/bfloat16.cc
@@ -19,6 +19,9 @@ limitations under the License.
 
 namespace tensorflow {
 
+const uint16_t bfloat16::NAN_VALUE;
+const uint16_t bfloat16::ZERO_VALUE;
+
 B16_DEVICE_FUNC bfloat16::operator Eigen::half() const {
   return static_cast<Eigen::half>(float(*this));
 }
-- 
GitLab


From 954224174fb756b136a11f9f871c100006ea48e4 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Sun, 3 Feb 2019 18:03:53 -0800
Subject: [PATCH 1827/2345] [XLA] Add NanValue XLA public function.

Like MaxValue and friends.

PiperOrigin-RevId: 232224733
---
 .../compiler/xla/client/lib/constants.cc      | 24 +++++++++++++++++++
 .../compiler/xla/client/lib/constants.h       |  3 +++
 .../compiler/xla/client/lib/constants_test.cc |  7 ++++++
 3 files changed, 34 insertions(+)

diff --git a/tensorflow/compiler/xla/client/lib/constants.cc b/tensorflow/compiler/xla/client/lib/constants.cc
index 1ada7b4a96..d0efd79bbe 100644
--- a/tensorflow/compiler/xla/client/lib/constants.cc
+++ b/tensorflow/compiler/xla/client/lib/constants.cc
@@ -100,4 +100,28 @@ XlaOp MaxFiniteValue(XlaBuilder* builder, PrimitiveType type) {
   }
 }
 
+XlaOp NanValue(XlaBuilder* builder, PrimitiveType type) {
+  return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    switch (type) {
+      case F16:
+        return ConstantR0<Eigen::half>(
+            builder, Eigen::NumTraits<Eigen::half>::quiet_NaN());
+      case BF16:
+        return ConstantR0<bfloat16>(
+            builder, bfloat16(std::numeric_limits<float>::quiet_NaN()));
+      case F32:
+        return ConstantR0<float>(builder,
+                                 std::numeric_limits<float>::quiet_NaN());
+      case F64:
+        return ConstantR0<double>(builder,
+                                  std::numeric_limits<double>::quiet_NaN());
+      default:
+        return InvalidArgument(
+            "Operand to NanValue was %s, but must be a real-valued "
+            "floating-point type.",
+            PrimitiveType_Name(type));
+    }
+  });
+}
+
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/client/lib/constants.h b/tensorflow/compiler/xla/client/lib/constants.h
index 4e5310a380..77e7ca60c3 100644
--- a/tensorflow/compiler/xla/client/lib/constants.h
+++ b/tensorflow/compiler/xla/client/lib/constants.h
@@ -142,6 +142,9 @@ XlaOp MaxValue(XlaBuilder* builder, PrimitiveType type);
 // Returns the maximum representable finite value for 'type'.
 XlaOp MaxFiniteValue(XlaBuilder* builder, PrimitiveType type);
 
+// Returns a nan for the given type.  Only valid for real-valued fp types.
+XlaOp NanValue(XlaBuilder* builder, PrimitiveType type);
+
 }  // namespace xla
 
 #endif  // TENSORFLOW_COMPILER_XLA_CLIENT_LIB_CONSTANTS_H_
diff --git a/tensorflow/compiler/xla/client/lib/constants_test.cc b/tensorflow/compiler/xla/client/lib/constants_test.cc
index f4320f65c1..180175b749 100644
--- a/tensorflow/compiler/xla/client/lib/constants_test.cc
+++ b/tensorflow/compiler/xla/client/lib/constants_test.cc
@@ -155,5 +155,12 @@ XLA_TEST_F(ConstantsTest, MaxValueF32) {
                              {});
 }
 
+XLA_TEST_F(ConstantsTest, NanValueF32) {
+  XlaBuilder builder(TestName());
+  NanValue(&builder, F32);
+  ComputeAndCompareR0<float>(&builder, std::numeric_limits<float>::quiet_NaN(),
+                             {});
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 656c8f1da9656e21e7d7502b367b87e037341cbb Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Sun, 3 Feb 2019 18:29:11 -0800
Subject: [PATCH 1828/2345] [XLA:Python] Raise an exception if a platform has
 no visible devices during initialization.

PiperOrigin-RevId: 232226157
---
 .../compiler/xla/python/local_computation_builder.cc       | 7 ++++++-
 tensorflow/compiler/xla/python/xla_client.py               | 2 ++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/python/local_computation_builder.cc b/tensorflow/compiler/xla/python/local_computation_builder.cc
index ce4bd6f681..c0daf3c13e 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.cc
+++ b/tensorflow/compiler/xla/python/local_computation_builder.cc
@@ -92,7 +92,12 @@ Status InitializePlatformName(const string& platform_name) {
         "previously created with a platform name of %s.",
         platform_name, *g_platform_name);
   }
-  TF_RETURN_IF_ERROR(PlatformUtil::GetPlatform(platform_name).status());
+  TF_ASSIGN_OR_RETURN(se::Platform * platform,
+                      PlatformUtil::GetPlatform(platform_name));
+  if (platform->VisibleDeviceCount() <= 0) {
+    return InvalidArgument("Platform %s has no visible devices.",
+                           platform_name);
+  }
   *g_platform_name = platform_name;
   return Status::OK();
 }
diff --git a/tensorflow/compiler/xla/python/xla_client.py b/tensorflow/compiler/xla/python/xla_client.py
index 37cae0e3b6..4433bcb1b3 100644
--- a/tensorflow/compiler/xla/python/xla_client.py
+++ b/tensorflow/compiler/xla/python/xla_client.py
@@ -1602,6 +1602,8 @@ def initialize_platform_name(platform_name):
 
   Raises:
     A runtime exception if the XLA service has already been initialized.
+    A runtime exception if the platform does not exist, or there are no devices
+    with that platform.
   """
   platform_name = _maybe_encode_string(platform_name)
   c_api.InitializePlatformName(platform_name)
-- 
GitLab


From 68052b3303153c4e2ada0865e93bb3f9e729ba13 Mon Sep 17 00:00:00 2001
From: Youlong Cheng <ylc@google.com>
Date: Sun, 3 Feb 2019 20:38:06 -0800
Subject: [PATCH 1829/2345] Don't initialize tpu system twice for
 mesh-tensorflow.

PiperOrigin-RevId: 232234083
---
 tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 6a3ed9bb79..14dca03568 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -467,7 +467,11 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook):
 
     self._feed_error = None
     self._finished = False
-    self._should_initialize_tpu = True
+    # When using model parallelism, the TPU is pre-initialized at startup to
+    # fetch mesh information.  We skip re-initializing it here to avoid
+    # suspected issues due to the mesh layout changing on the second
+    # initialization.
+    self._should_initialize_tpu = not ctx.model_parallelism_enabled
     self._tpu_compile_op = tpu_compile_op
 
   def begin(self):
-- 
GitLab


From 5076adf64a87d2370f84dec371178568f3a5daf9 Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Sun, 3 Feb 2019 23:48:23 -0800
Subject: [PATCH 1830/2345] Expose Module._flatten to allow subclasses to
 implement gathering.

I've made this "private" since this exposes the implementation of modules, which
in typical Python code does not represent part of the classes public API. As
such I think having the leading _ makes it clear (at least to the linter) that
you're using an "internal" method. As always this is a "sign" not a cop.

As a concrete example, I strongly encourage the following usage:

    >>> class FooModule(...):
    ...   @property
    ...   def hash_tables(self):
    ...     return tuple(self._flatten(predicate=IS_HASH_TABLE))

But in general we should discourage usage outside of modules:

    >>> m = FooModule()
    >>> hash_tables = tuple(m._flatten(predicate=IS_HASH_TABLE))  # Naughty.

c.f. tensorflow/community#56

PiperOrigin-RevId: 232246092
---
 tensorflow/python/module/module.py      | 159 +++++++++++++-----------
 tensorflow/python/module/module_test.py |  34 ++++-
 2 files changed, 118 insertions(+), 75 deletions(-)

diff --git a/tensorflow/python/module/module.py b/tensorflow/python/module/module.py
index 62d1bd46c4..da2d39a7f0 100644
--- a/tensorflow/python/module/module.py
+++ b/tensorflow/python/module/module.py
@@ -131,7 +131,7 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
 
   >>> class Dense(tf.Module):
   ...   def __init__(self, in_features, output_features):
-  ...     super(Linear, self).__init__()
+  ...     super(Dense, self).__init__()
   ...     self.w = tf.Variable(
   ...         tf.random_normal([input_features, output_features]), name='w')
   ...     self.b = tf.Variable(tf.zeros([output_features]), name='b')
@@ -150,7 +150,7 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
   By subclassing `tf.Module` instead of `object` any variables created inside
   the module are automatically created within the modules name scope:
 
-  >> d.w.name
+  >>> d.w.name
   "dense/w:0"
 
   In eager mode this is useful for debugging, and when used with `@tf.function`
@@ -209,7 +209,7 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
       A collection of variables for the current module (sorted by attribute
       name) followed by variables from all submodules recursively (depth first).
     """
-    return tuple(walk(self, recurse_if=_IS_MODULE, predicate=_IS_VARIABLE))
+    return tuple(self._flatten(predicate=_IS_VARIABLE))
 
   @property
   def owned_variables(self):
@@ -223,7 +223,7 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
       yield variables inside nested structures (lists etc) but not in other
       modules.
     """
-    return tuple(walk(self, predicate=_IS_VARIABLE))
+    return tuple(self._flatten(recursive=False, predicate=_IS_VARIABLE))
 
   @property
   def trainable_variables(self):
@@ -237,8 +237,7 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
       A collection of variables for the current module (sorted by attribute
       name) followed by variables from all submodules recursively (depth first).
     """
-    return tuple(
-        walk(self, recurse_if=_IS_MODULE, predicate=_IS_TRAINABLE_VARIABLE))
+    return tuple(self._flatten(predicate=_IS_TRAINABLE_VARIABLE))
 
   @property
   def owned_trainable_variables(self):
@@ -252,7 +251,8 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
       yield variables inside nested structures (lists etc) but not in other
       modules.
     """
-    return tuple(walk(self, predicate=_IS_TRAINABLE_VARIABLE))
+    return tuple(
+        self._flatten(recursive=False, predicate=_IS_TRAINABLE_VARIABLE))
 
   @property
   def owned_submodules(self):
@@ -273,7 +273,7 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
     Returns:
       A collection of all child modules.
     """
-    return tuple(walk(self, predicate=_IS_MODULE))
+    return tuple(self._flatten(recursive=False, predicate=_IS_MODULE))
 
   @property
   def submodules(self):
@@ -294,7 +294,60 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
     Returns:
       A collection of all submodules.
     """
-    return tuple(walk(self, recurse_if=_IS_MODULE, predicate=_IS_MODULE))
+    return tuple(self._flatten(predicate=_IS_MODULE))
+
+  def _flatten(self,
+               recursive=True,
+               predicate=None,
+               attribute_traversal_key=None):
+    """Flattened attribute values in sorted order by attribute name.
+
+    Modules are flattened by first walking their attributes in name order.
+    Each attribute value is then flattened to find leaf values. If flatten is
+    to be applied `recursive`ly then if the leaf is a `Module` it will also be
+    flattened to find leaves. Finally every leaf value is optionally tested
+    against the given `predicate` and finally yielded.
+
+    >>> class Foo(tf.experimental.Module):
+    ...   def __init__(self):
+    ...     super(Foo, self).__init__()
+    ...     self.x = [tf.constant('a'), tf.constant('b')]
+    ...     self.y = {'i': tf.constant('c'), 'j': tf.constant('d')}
+    ...     self.z = tf.constant('e')
+    ...
+    ...   @property
+    ...   def tensors(self):
+    ...     return tuple(self._flatten(predicate=is_tensor))
+
+    >>> foo = Foo()
+    >>> foo.tensors
+    (<tf.Tensor...'a'>, <tf.Tensor...'b'>, ...'c'>, ...'d'>, ...'e'>)
+
+    `attribute_traversal_key` controls the order object properties are visited.
+    If not set objects are visited in ascending order by name.
+
+    Args:
+      recursive: Whether to recurse into child modules or not.
+      predicate: (Optional) If set then only values matching predicate are
+        yielded. A value of `None` (the default) means no items will be
+        filtered.
+      attribute_traversal_key: (Optional) Method to rekey object attributes
+        before they are sorted. Contract is the same as `key` argument to
+        builtin `sorted` and only applies to object properties.
+
+    Returns:
+      Flat generator for leaves of the current module and optionally all
+      submodules.
+    """
+    if predicate is None:
+      predicate = lambda _: True
+
+    return _flatten_module(
+        self,
+        recursive=recursive,
+        predicate=predicate,
+        attribute_traversal_key=attribute_traversal_key,
+        seen=set())
 
   @classmethod
   def no_name_scope(cls, method):
@@ -334,77 +387,37 @@ def camel_to_snake(value):
   return _CAMEL_TO_SNAKE_R.sub(r"_\1", value).lower()
 
 
-def walk(o, recurse_if=None, predicate=None):
-  """Flattened attributes of `o` in sorted order by attribute name.
-
-  >>> class Foo(object):
-  ...   def __init__(self, prefix=''):
-  ...     self.z = prefix + 'c'
-  ...     self.a = [prefix + 'a', prefix + 'b']
-
-  >>> tuple(walk(Foo()))
-  ('a', 'b', 'c')
-
-  If `predicate` is not None, then only values matching predicate are returned:
-
-  >>> tuple(walk(Foo(), predicate=lambda v: v != 'a'))
-  ('b', 'c')
-
-  If `recurse_if` is not None then it should be a callable which tests if the
-  given leaf should be expanded:
-
-  >>> is_string = lambda v: isinstance(v, str)
-  >>> is_foo = lambda l: isinstance(l, Foo)
-  >>> o = Foo(prefix='root_')
-  >>> o.b = Foo(prefix='child_')
-  >>> tuple(walk(o, predicate=is_string))
-  ('root_a', 'root_b', 'root_c')
-  >>> tuple(walk(o, recurse_if=is_foo, predicate=is_string))
-  ('root_a', 'root_b', 'root_c', 'child_a', 'child_b', 'child_c')
-
-  Args:
-    o: An object who's attributes are walked.
-    recurse_if: (Optional) Visited items of this type will be walked to extract
-      more leaves. If `None`, it will not recurse into leaves.
-    predicate: (Optional) If set then only values matching predicate are
-      yielded.
-
-  Returns:
-    Attributes of `o` in name order. If `recurse_if` is not `None` then
-    attributes for which `recurse_if(attribute) == True` will be walked
-    recursively. If `predicate` is not `None` then only attributes for which
-    `predicate(attribute) == True` will be yielded.
-  """
-  if predicate is None:
-    predicate = lambda _: True
-  return _walk_internal(
-      o, recurse_if=recurse_if, predicate=predicate, seen=set())
-
-
-def _walk_internal(o, recurse_if, predicate, seen):
-  """Implementation of `walk`."""
+def _flatten_module(module, recursive, predicate, attribute_traversal_key,
+                    seen):
+  """Implementation of `flatten`."""
   if seen is None:
-    seen = set([id(o)])
+    seen = set([id(module)])
 
-  o_dict = vars(o)
-  to_walk = []
+  module_dict = vars(module)
+  submodules = []
 
-  for key in sorted(o_dict):
-    values = nest.flatten(o_dict[key])
-    for value in values:
-      value_id = id(value)
-      if value_id in seen:
+  for key in sorted(module_dict, key=attribute_traversal_key):
+    for leaf in nest.flatten(module_dict[key]):
+      leaf_id = id(leaf)
+      if leaf_id in seen:
         continue
 
-      seen.add(value_id)
-      if predicate(value):
-        yield value
+      seen.add(leaf_id)
+      if predicate(leaf):
+        yield leaf
 
-      if recurse_if is not None and recurse_if(value):
+      if recursive and isinstance(leaf, Module):
         # Walk direct properties first then recurse.
-        to_walk.append(value)
+        submodules.append(leaf)
+
+  for submodule in submodules:
+    subvalues = _flatten_module(
+        submodule,
+        recursive=recursive,
+        predicate=predicate,
+        attribute_traversal_key=attribute_traversal_key,
+        seen=seen)
 
-  for value in to_walk:
-    for subvalue in _walk_internal(value, recurse_if, predicate, seen):
+    for subvalue in subvalues:
       # Predicate is already tested for these values.
       yield subvalue
diff --git a/tensorflow/python/module/module_test.py b/tensorflow/python/module/module_test.py
index 1e4e1959c3..a487660435 100644
--- a/tensorflow/python/module/module_test.py
+++ b/tensorflow/python/module/module_test.py
@@ -332,13 +332,43 @@ class WalkTest(parameterized.TestCase, test.TestCase):
     child = parent.c
 
     self.assertEqual(
-        list(module.walk(parent, predicate=IS_MEMBER)),
+        list(parent._flatten(recursive=False, predicate=IS_MEMBER)),
         [parent.a[0], parent.a[1], parent.z])
 
     self.assertEqual(
-        list(module.walk(parent, recurse_if=IS_MODULE, predicate=IS_MEMBER)),
+        list(parent._flatten(predicate=IS_MEMBER)),
         [parent.a[0], parent.a[1], parent.z, child.a[0], child.a[1], child.z])
 
+  def test_attribute_traversal_key(self):
+    mod = LayerModule()
+    self.assertEqual(
+        mod.variables,
+        mod._trainable_variables + mod._non_trainable_variables + [mod._bonus])
+
+
+class LayerModule(module.Module):
+
+  def __init__(self):
+    super(LayerModule, self).__init__()
+    self._trainable_variables = [
+        variables.Variable(1., name="a"),
+        variables.Variable(2., name="b"),
+    ]
+    self._non_trainable_variables = [
+        variables.Variable(3., name="c"),
+        variables.Variable(4., name="d"),
+    ]
+    self._bonus = variables.Variable(5., name="e")
+
+  @property
+  def variables(self):
+    def key_function(name):
+      indexes = {"_trainable_variables": 0, "_non_trainable_variables": 1}
+      return indexes.get(name, 2), name
+
+    return list(self._flatten(predicate=module._IS_VARIABLE,
+                              attribute_traversal_key=key_function))
+
 
 class MemberType(object):
   """A simple type to search for."""
-- 
GitLab


From acf703d39cf29339475b2309f998ae1c6d314a87 Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Mon, 4 Feb 2019 00:48:48 -0800
Subject: [PATCH 1831/2345] Fix embedding_lookup_sparse_v2 calling into itself.

PiperOrigin-RevId: 232251764
---
 tensorflow/python/ops/embedding_ops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/embedding_ops.py b/tensorflow/python/ops/embedding_ops.py
index d0291e2095..881466cb23 100644
--- a/tensorflow/python/ops/embedding_ops.py
+++ b/tensorflow/python/ops/embedding_ops.py
@@ -558,7 +558,7 @@ def embedding_lookup_sparse_v2(params,
                                combiner=None,
                                max_norm=None,
                                name=None):
-  return embedding_lookup_sparse_v2(
+  return embedding_lookup_sparse(
       params, sp_ids, sp_weights, partition_strategy, name, combiner, max_norm)
 
 
-- 
GitLab


From cb011e1dd8d79757fea01be39e19cb1155681e7e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 4 Feb 2019 01:02:26 -0800
Subject: [PATCH 1832/2345] compat: Update forward compatibility horizon to
 2019-02-04

PiperOrigin-RevId: 232253125
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 104dc39734..a278a01410 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 2, 3)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 2, 4)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From be0d9622611e74077c6efcfff7d0d1c80e120fde Mon Sep 17 00:00:00 2001
From: Grzegorz Pawelczak <grzegorzp@graphcore.ai>
Date: Mon, 4 Feb 2019 09:16:42 +0000
Subject: [PATCH 1833/2345] Add clarifying comment and a test case

---
 .../compiler/xla/service/hlo_tfgraph_builder.cc |  3 +++
 .../xla/service/hlo_tfgraph_builder_test.cc     | 17 +++++++++++++++++
 2 files changed, 20 insertions(+)

diff --git a/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc b/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc
index 334b0d90e1..b7a686a9f4 100644
--- a/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc
+++ b/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc
@@ -163,6 +163,9 @@ void HloTfGraphBuilder::SetNodeAttrs(const HloInstruction* instruction,
       // represented in a single Layout field.
       layout_string = ShapeUtil::HumanStringWithLayout(instruction->shape());
     } else if (instruction->shape().has_layout()) {
+      // For non-tuples, only emit the layout when the shape has a Layout.
+      // This extra check is required because LayoutUtil::HasLayout ignores
+      // token, opaque types etc.
       layout_string = StrCat(
           "{",
           absl::StrJoin(LayoutUtil::MinorToMajor(instruction->shape()), ","),
diff --git a/tensorflow/compiler/xla/service/hlo_tfgraph_builder_test.cc b/tensorflow/compiler/xla/service/hlo_tfgraph_builder_test.cc
index 1e2b31a1f2..e5e9517c90 100644
--- a/tensorflow/compiler/xla/service/hlo_tfgraph_builder_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_tfgraph_builder_test.cc
@@ -178,6 +178,23 @@ TEST_F(HloTfGraphBuilderTest, EmbeddedComputationsDiamond) {
   EXPECT_GT(generator_.GetGraphDef().node_size(), 0);
 }
 
+TEST_F(HloTfGraphBuilderTest, TokenHasNoLayout) {
+  auto builder = HloComputation::Builder("Token");
+  auto token = builder.AddInstruction(HloInstruction::CreateToken());
+  OpMetadata metadata;
+  metadata.set_op_name("x");
+  metadata.set_op_type("y");
+  token->set_metadata(metadata);
+  TF_CHECK_OK(generator_.AddComputation(*builder.Build()));
+  GraphDef graph_def = generator_.GetGraphDef();
+  EXPECT_EQ(graph_def.node_size(), 1);
+  const auto &node = graph_def.node(0);
+  EXPECT_EQ(GetNodeAttr(node, "type").s(), "TOKEN");
+  EXPECT_EQ(GetNodeAttr(node, "layout").s(), "");
+  EXPECT_EQ(GetNodeAttr(node, "tf_op_name").s(), "x");
+  EXPECT_EQ(GetNodeAttr(node, "tf_op_type").s(), "y");
+}
+
 }  // namespace
 }  // namespace hlo_graph_dumper
 }  // namespace xla
-- 
GitLab


From 4a9c117ee7477adfdc39a48520622d34296c877e Mon Sep 17 00:00:00 2001
From: "K. Hodges" <khodges42@gmail.com>
Date: Mon, 4 Feb 2019 04:30:58 -0500
Subject: [PATCH 1834/2345] Fixed a misspelled var name in comment.

---
 tensorflow/compiler/xla/service/hlo_cost_analysis.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
index 1ee958114e..bcbf26ed35 100644
--- a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
+++ b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
@@ -245,7 +245,7 @@ Status HloCostAnalysis::HandleDot(const HloInstruction* dot) {
   for (auto dim : dnums.lhs_contracting_dimensions()) {
     reduction_width *= lhs_shape.dimensions(dim);
   }
-  // Each output elment requires reduction_widht FMA operations.
+  // Each output elment requires reduction_width FMA operations.
   current_properties_[kFlopsKey] =
       kFmaFlops * ShapeUtil::ElementsIn(dot_shape) * reduction_width;
   return Status::OK();
-- 
GitLab


From e907e94bc3586fba29603c73cd93b75b35794a93 Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Mon, 4 Feb 2019 01:42:31 -0800
Subject: [PATCH 1835/2345] Allow get_concrete_function on RestoredFunction.

PiperOrigin-RevId: 232258125
---
 .../saved_model/function_deserialization.py   | 11 +++----
 tensorflow/python/saved_model/load_test.py    | 32 ++++++++++++++++++-
 2 files changed, 35 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py
index e82e64260c..992b62ac70 100644
--- a/tensorflow/python/saved_model/function_deserialization.py
+++ b/tensorflow/python/saved_model/function_deserialization.py
@@ -105,18 +105,15 @@ class RestoredFunction(def_function.Function):
     super(RestoredFunction, self).__init__(
         python_function, name, autograph=False)
     self._concrete_functions = concrete_functions
-    # TODO(vbardiovsky): This does not propagate to stateful and stateless
-    # functions of the RestoredFunction, which will have seen only defunned
-    # restored_function_body(*args, **kwargs). Therefore get_concrete_function()
-    # called on RestoredFunction will not work properly.
+    # This does not propagate to stateful and stateless functions of the
+    # RestoredFunction, which will have seen only defunned
+    # restored_function_body(*args, **kwargs). That's why we have to
+    # canonicalize inputs inside restored_function_body.
     self._function_spec = function_spec
 
   def _list_all_concrete_functions_for_serialization(self):
     return self._concrete_functions
 
-  def get_concrete_function(self, *args, **kwargs):
-    raise NotImplementedError()
-
 
 def recreate_function(saved_function, concrete_functions):
   """Creates a `Function` from a `SavedFunction`.
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index 3edecd5be2..d61fa864ad 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -312,7 +312,7 @@ class LoadTest(test.TestCase, parameterized.TestCase):
     imported = self.cycle(root, cycles)
 
     with self.assertRaisesRegexp(AssertionError,
-                                 "Could not find matching function to call.*"):
+                                 "Could not find matching function to call"):
       imported.f(input2)
 
     self.assertEqual(31, imported.f(input1).numpy())
@@ -612,6 +612,36 @@ class LoadTest(test.TestCase, parameterized.TestCase):
     self.assertAllEqual([2, 4, 6],
                         imported.f(constant_op.constant([1, 2, 3])).numpy())
 
+  def test_get_concrete_function(self, cycles):
+
+    @def_function.function
+    def func(x, training=False):
+      if training:
+        return 2 * x
+      else:
+        return 3 * x
+
+    func.get_concrete_function(
+        tensor_spec.TensorSpec([None], dtypes.int32), True)
+    func.get_concrete_function(tensor_spec.TensorSpec([None], dtypes.float32))
+
+    root = tracking.AutoCheckpointable()
+    root.f = func
+
+    imported = self.cycle(root, cycles)
+
+    concrete = imported.f.get_concrete_function(
+        training=True, x=tensor_spec.TensorSpec([None], dtypes.int32))
+
+    self.assertAllEqual([2, 4, 6, 8],
+                        concrete(x=constant_op.constant([1, 2, 3, 4])).numpy())
+    with self.assertRaisesRegexp(AssertionError,
+                                 "Could not find matching function to call"):
+      imported.f.get_concrete_function(
+          tensor_spec.TensorSpec([None], dtypes.int32))
+    imported.f.get_concrete_function(
+        tensor_spec.TensorSpec([None], dtypes.int32), True)
+
   def test_concrete_function(self, cycles):
 
     @def_function.function(
-- 
GitLab


From 2737cdc70846adfe796d8868d0337750c6ab72c4 Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Mon, 4 Feb 2019 05:56:28 -0800
Subject: [PATCH 1836/2345] Add descriptive text to the top of the `tf.math`

This is a simplified replacement for the deleted guide.

Fix links to math#Segmentation

PiperOrigin-RevId: 232282469
---
 .../api_def/base_api/api_def_SegmentMax.pbtxt | 12 +++-
 .../base_api/api_def_SegmentMean.pbtxt        | 12 +++-
 .../api_def/base_api/api_def_SegmentMin.pbtxt | 11 +++-
 .../base_api/api_def_SegmentProd.pbtxt        | 12 +++-
 .../api_def/base_api/api_def_SegmentSum.pbtxt | 12 +++-
 .../base_api/api_def_SparseSegmentMean.pbtxt  |  4 +-
 ...def_SparseSegmentMeanWithNumSegments.pbtxt |  2 +-
 .../base_api/api_def_SparseSegmentSqrtN.pbtxt |  5 +-
 ...ef_SparseSegmentSqrtNWithNumSegments.pbtxt |  2 +-
 .../base_api/api_def_SparseSegmentSum.pbtxt   |  2 +-
 ..._def_SparseSegmentSumWithNumSegments.pbtxt |  2 +-
 .../base_api/api_def_UnsortedSegmentMax.pbtxt | 12 +++-
 .../base_api/api_def_UnsortedSegmentMin.pbtxt | 11 +++-
 .../api_def_UnsortedSegmentProd.pbtxt         | 11 +++-
 .../base_api/api_def_UnsortedSegmentSum.pbtxt | 10 ++-
 tensorflow/python/ops/math_ops.py             | 64 +++++++++++++++++--
 .../python/ops/ragged/segment_id_ops.py       |  4 +-
 17 files changed, 160 insertions(+), 28 deletions(-)

diff --git a/tensorflow/core/api_def/base_api/api_def_SegmentMax.pbtxt b/tensorflow/core/api_def/base_api/api_def_SegmentMax.pbtxt
index d33a36ce06..d5643c8a79 100644
--- a/tensorflow/core/api_def/base_api/api_def_SegmentMax.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_SegmentMax.pbtxt
@@ -17,7 +17,7 @@ END
   summary: "Computes the maximum along segments of a tensor."
   description: <<END
 Read
-[the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+[the section on segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
 for an explanation of segments.
 
 Computes a tensor such that
@@ -29,5 +29,15 @@ If the max is empty for a given segment ID `i`, `output[i] = 0`.
 <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
 <img style="width:100%" src="https://www.tensorflow.org/images/SegmentMax.png" alt>
 </div>
+
+For example:
+
+```
+c = tf.constant([[1,2,3,4], [4, 3, 2, 1], [5,6,7,8]])
+tf.segment_max(c, tf.constant([0, 0, 1]))
+# ==> [[4, 3, 3, 4],
+#      [5, 6, 7, 8]]
+```
+
 END
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_SegmentMean.pbtxt b/tensorflow/core/api_def/base_api/api_def_SegmentMean.pbtxt
index afdc39da96..b03649ab07 100644
--- a/tensorflow/core/api_def/base_api/api_def_SegmentMean.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_SegmentMean.pbtxt
@@ -17,7 +17,7 @@ END
   summary: "Computes the mean along segments of a tensor."
   description: <<END
 Read
-[the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+[the section on segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
 for an explanation of segments.
 
 Computes a tensor such that
@@ -30,5 +30,15 @@ If the mean is empty for a given segment ID `i`, `output[i] = 0`.
 <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
 <img style="width:100%" src="https://www.tensorflow.org/images/SegmentMean.png" alt>
 </div>
+
+For example:
+
+```
+c = tf.constant([[1.0,2,3,4], [4, 3, 2, 1], [5,6,7,8]])
+tf.segment_mean(c, tf.constant([0, 0, 1]))
+# ==> [[2.5, 2.5, 2.5, 2.5],
+#      [5, 6, 7, 8]]
+```
+
 END
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_SegmentMin.pbtxt b/tensorflow/core/api_def/base_api/api_def_SegmentMin.pbtxt
index 026b5b3991..6796678555 100644
--- a/tensorflow/core/api_def/base_api/api_def_SegmentMin.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_SegmentMin.pbtxt
@@ -17,7 +17,7 @@ END
   summary: "Computes the minimum along segments of a tensor."
   description: <<END
 Read
-[the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+[the section on segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
 for an explanation of segments.
 
 Computes a tensor such that
@@ -29,5 +29,14 @@ If the min is empty for a given segment ID `i`, `output[i] = 0`.
 <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
 <img style="width:100%" src="https://www.tensorflow.org/images/SegmentMin.png" alt>
 </div>
+
+For example:
+
+```
+c = tf.constant([[1,2,3,4], [4, 3, 2, 1], [5,6,7,8]])
+tf.segment_min(c, tf.constant([0, 0, 1]))
+# ==> [[1, 2, 2, 1],
+#      [5, 6, 7, 8]]
+```
 END
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_SegmentProd.pbtxt b/tensorflow/core/api_def/base_api/api_def_SegmentProd.pbtxt
index a168eed87f..10b368fcca 100644
--- a/tensorflow/core/api_def/base_api/api_def_SegmentProd.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_SegmentProd.pbtxt
@@ -17,7 +17,7 @@ END
   summary: "Computes the product along segments of a tensor."
   description: <<END
 Read
-[the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+[the section on segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
 for an explanation of segments.
 
 Computes a tensor such that
@@ -29,5 +29,15 @@ If the product is empty for a given segment ID `i`, `output[i] = 1`.
 <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
 <img style="width:100%" src="https://www.tensorflow.org/images/SegmentProd.png" alt>
 </div>
+
+For example:
+
+```
+c = tf.constant([[1,2,3,4], [4, 3, 2, 1], [5,6,7,8]])
+tf.segment_prod(c, tf.constant([0, 0, 1]))
+# ==> [[4, 6, 6, 4],
+#      [5, 6, 7, 8]]
+```
+
 END
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_SegmentSum.pbtxt b/tensorflow/core/api_def/base_api/api_def_SegmentSum.pbtxt
index 876b860824..487a6d1074 100644
--- a/tensorflow/core/api_def/base_api/api_def_SegmentSum.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_SegmentSum.pbtxt
@@ -17,7 +17,7 @@ END
   summary: "Computes the sum along segments of a tensor."
   description: <<END
 Read
-[the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+[the section on segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
 for an explanation of segments.
 
 Computes a tensor such that
@@ -29,5 +29,15 @@ If the sum is empty for a given segment ID `i`, `output[i] = 0`.
 <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
 <img style="width:100%" src="https://www.tensorflow.org/images/SegmentSum.png" alt>
 </div>
+
+For example:
+
+```
+c = tf.constant([[1,2,3,4], [4, 3, 2, 1], [5,6,7,8]])
+tf.segment_sum(c, tf.constant([0, 0, 1]))
+# ==> [[5, 5, 5, 5],
+#      [5, 6, 7, 8]]
+```
+
 END
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_SparseSegmentMean.pbtxt b/tensorflow/core/api_def/base_api/api_def_SparseSegmentMean.pbtxt
index 138a6366c8..0bbc0780df 100644
--- a/tensorflow/core/api_def/base_api/api_def_SparseSegmentMean.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_SparseSegmentMean.pbtxt
@@ -21,9 +21,7 @@ END
   }
   summary: "Computes the mean along sparse segments of a tensor."
   description: <<END
-Read
-[the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
-for an explanation of segments.
+See `tf.sparse.segment_sum` for usage examples.
 
 Like `SegmentMean`, but `segment_ids` can have rank less than `data`'s first
 dimension, selecting a subset of dimension 0, specified by `indices`.
diff --git a/tensorflow/core/api_def/base_api/api_def_SparseSegmentMeanWithNumSegments.pbtxt b/tensorflow/core/api_def/base_api/api_def_SparseSegmentMeanWithNumSegments.pbtxt
index b8073d88ac..65b2358830 100644
--- a/tensorflow/core/api_def/base_api/api_def_SparseSegmentMeanWithNumSegments.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_SparseSegmentMeanWithNumSegments.pbtxt
@@ -31,7 +31,7 @@ Like `SparseSegmentMean`, but allows missing ids in `segment_ids`. If an id is
 misisng, the `output` tensor at that position will be zeroed.
 
 Read
-[the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+[the section on segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
 for an explanation of segments.
 END
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_SparseSegmentSqrtN.pbtxt b/tensorflow/core/api_def/base_api/api_def_SparseSegmentSqrtN.pbtxt
index 945bbdcf62..a28bd1a646 100644
--- a/tensorflow/core/api_def/base_api/api_def_SparseSegmentSqrtN.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_SparseSegmentSqrtN.pbtxt
@@ -23,8 +23,7 @@ END
   description: <<END
 N is the size of the segment being reduced.
 
-Read
-[the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
-for an explanation of segments.
+See `tf.sparse.segment_sum` for usage examples.
+
 END
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_SparseSegmentSqrtNWithNumSegments.pbtxt b/tensorflow/core/api_def/base_api/api_def_SparseSegmentSqrtNWithNumSegments.pbtxt
index ff328c8a61..8a5d2bb02c 100644
--- a/tensorflow/core/api_def/base_api/api_def_SparseSegmentSqrtNWithNumSegments.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_SparseSegmentSqrtNWithNumSegments.pbtxt
@@ -33,7 +33,7 @@ Like `SparseSegmentSqrtN`, but allows missing ids in `segment_ids`. If an id is
 misisng, the `output` tensor at that position will be zeroed.
 
 Read
-[the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+[the section on segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
 for an explanation of segments.
 END
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_SparseSegmentSum.pbtxt b/tensorflow/core/api_def/base_api/api_def_SparseSegmentSum.pbtxt
index a68e14607f..d7494dc8de 100644
--- a/tensorflow/core/api_def/base_api/api_def_SparseSegmentSum.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_SparseSegmentSum.pbtxt
@@ -22,7 +22,7 @@ END
   summary: "Computes the sum along sparse segments of a tensor."
   description: <<END
 Read
-[the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+[the section on segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
 for an explanation of segments.
 
 Like `SegmentSum`, but `segment_ids` can have rank less than `data`'s first
diff --git a/tensorflow/core/api_def/base_api/api_def_SparseSegmentSumWithNumSegments.pbtxt b/tensorflow/core/api_def/base_api/api_def_SparseSegmentSumWithNumSegments.pbtxt
index aa5c1fc8d0..039ca9a23b 100644
--- a/tensorflow/core/api_def/base_api/api_def_SparseSegmentSumWithNumSegments.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_SparseSegmentSumWithNumSegments.pbtxt
@@ -31,7 +31,7 @@ Like `SparseSegmentSum`, but allows missing ids in `segment_ids`. If an id is
 misisng, the `output` tensor at that position will be zeroed.
 
 Read
-[the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+[the section on segmentation](https://tensorflow.org/api_docs/python/tf/sparse#Segmentation)
 for an explanation of segments.
 
 For example:
diff --git a/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentMax.pbtxt b/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentMax.pbtxt
index ed4a2bd558..f282b9fab5 100644
--- a/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentMax.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentMax.pbtxt
@@ -17,7 +17,7 @@ END
   summary: "Computes the maximum along segments of a tensor."
   description: <<END
 Read
-[the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+[the section on segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
 for an explanation of segments.
 
 This operator is similar to the unsorted segment sum operator found
@@ -37,5 +37,15 @@ dropped, and will not be included in the result.
 <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
 <img style="width:100%" src="https://www.tensorflow.org/images/UnsortedSegmentMax.png" alt>
 </div>
+
+For example:
+
+``` python
+c = tf.constant([[1,2,3,4], [5,6,7,8], [4,3,2,1]])
+tf.unsorted_segment_max(c, tf.constant([0, 1, 0]), num_segments=2)
+# ==> [[ 4,  3, 3, 4],
+#       [5,  6, 7, 8]]
+```
+
 END
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentMin.pbtxt b/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentMin.pbtxt
index 7e139ddf4d..0360cc09d0 100644
--- a/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentMin.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentMin.pbtxt
@@ -17,7 +17,7 @@ END
   summary: "Computes the minimum along segments of a tensor."
   description: <<END
 Read
-[the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#segmentation)
+[the section on segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
 for an explanation of segments.
 
 This operator is similar to the unsorted segment sum operator found
@@ -31,6 +31,15 @@ If the minimum is empty for a given segment ID `i`, it outputs the largest
 possible value for the specific numeric type,
 `output[i] = numeric_limits<T>::max()`.
 
+For example:
+
+``` python
+c = tf.constant([[1,2,3,4], [5,6,7,8], [4,3,2,1]])
+tf.unsorted_segment_min(c, tf.constant([0, 1, 0]), num_segments=2)
+# ==> [[ 1,  2, 2, 1],
+#       [5,  6, 7, 8]]
+```
+
 If the given segment ID `i` is negative, then the corresponding value is
 dropped, and will not be included in the result.
 END
diff --git a/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentProd.pbtxt b/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentProd.pbtxt
index 9c8ea3b620..67de4734bd 100644
--- a/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentProd.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentProd.pbtxt
@@ -17,7 +17,7 @@ END
   summary: "Computes the product along segments of a tensor."
   description: <<END
 Read
-[the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#segmentation)
+[the section on segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
 for an explanation of segments.
 
 This operator is similar to the unsorted segment sum operator found
@@ -28,6 +28,15 @@ entries belonging to a segment such that:
 \\(output_i = \prod_{j...} data[j...]\\) where the product is over tuples
 `j...` such that `segment_ids[j...] == i`.
 
+For example:
+
+``` python
+c = tf.constant([[1,2,3,4], [5,6,7,8], [4,3,2,1]])
+tf.unsorted_segment_prod(c, tf.constant([0, 1, 0]), num_segments=2)
+# ==> [[ 4,  6, 6, 4],
+#       [5,  6, 7, 8]]
+```
+
 If there is no entry for a given segment ID `i`, it outputs 1.
 
 If the given segment ID `i` is negative, then the corresponding value is
diff --git a/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentSum.pbtxt b/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentSum.pbtxt
index 7e5d9265c2..08139235f4 100644
--- a/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentSum.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentSum.pbtxt
@@ -17,7 +17,7 @@ END
   summary: "Computes the sum along segments of a tensor."
   description: <<END
 Read
-[the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+[the section on segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
 for an explanation of segments.
 
 Computes a tensor such that
@@ -35,5 +35,13 @@ added to the sum of the segment.
 <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
 <img style="width:100%" src="https://www.tensorflow.org/images/UnsortedSegmentSum.png" alt>
 </div>
+
+``` python
+c = tf.constant([[1,2,3,4], [5,6,7,8], [4,3,2,1]])
+tf.unsorted_segment_sum(c, tf.constant([0, 1, 0]), num_segments=2)
+# ==> [[ 5,  5, 5, 5],
+#       [5,  6, 7, 8]]
+```
+
 END
 }
diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index c620712b1d..7306a45e43 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -12,9 +12,59 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Basic arithmetic operators.
+"""Math Operations.
+
+Note: Functions taking `Tensor` arguments can also take anything accepted by
+`tf.convert_to_tensor`.
+
+Note: Elementwise binary operations in TensorFlow follow [numpy-style
+broadcasting](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html).
+
+TensorFlow provides a variety of math functions including:
+
+* Basic arithmetic operators and trigonometric functions.
+* Special math functions (like: `tf.math.igamma` and `tf.math.zeta`)
+* Complex number functions (like: `tf.math.imag` and `tf.math.angle`)
+* Reductions and scans (like: `tf.math.reduce_mean` and `tf.math.cumsum`)
+* Segment functions (like: `tf.math.segment_sum`)
+
+See: `tf.linalg` for matrix and tensor functions.
+
+<a id=Segmentation></a>
+
+## About Segmentation
+
+TensorFlow provides several operations that you can use to perform common
+math computations on tensor segments.
+Here a segmentation is a partitioning of a tensor along
+the first dimension, i.e. it  defines a mapping from the first dimension onto
+`segment_ids`. The `segment_ids` tensor should be the size of
+the first dimension, `d0`, with consecutive IDs in the range `0` to `k`,
+where `k<d0`.
+In particular, a segmentation of a matrix tensor is a mapping of rows to
+segments.
+
+For example:
+
+```python
+c = tf.constant([[1,2,3,4], [-1,-2,-3,-4], [5,6,7,8]])
+tf.segment_sum(c, tf.constant([0, 0, 1]))
+#  ==>  [[0 0 0 0]
+#        [5 6 7 8]]
+```
+
+The standard `segment_*` functions assert that the segment indices are sorted.
+If you have unsorted indices use the equivalent `unsorted_segment_` function.
+Thses functions take an additional argument `num_segments` so that the output
+tensor can be efficiently allocated.
+
+``` python
+c = tf.constant([[1,2,3,4], [-1,-2,-3,-4], [5,6,7,8]])
+tf.unsorted_segment_sum(c, tf.constant([0, 1, 0]), num_segments=2)
+# ==> [[ 6,  8, 10, 12],
+#       [-1, -2, -3, -4]]
+```
 
-See the [python/math_ops](python/math_ops) guide.
 """
 from __future__ import absolute_import
 from __future__ import division
@@ -3112,7 +3162,7 @@ def unsorted_segment_mean(data, segment_ids, num_segments, name=None):
   r"""Computes the mean along segments of a tensor.
 
   Read [the section on
-  segmentation](https://tensorflow.org/api_guides/python/math_ops#segmentation)
+  segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
   for an explanation of segments.
 
   This operator is similar to the unsorted segment sum operator found
@@ -3158,7 +3208,7 @@ def unsorted_segment_sqrt_n(data, segment_ids, num_segments, name=None):
   r"""Computes the sum along segments of a tensor divided by the sqrt(N).
 
   Read [the section on
-  segmentation](https://tensorflow.org/api_guides/python/math_ops#segmentation)
+  segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
   for an explanation of segments.
 
   This operator is similar to the unsorted segment sum operator found
@@ -3205,7 +3255,7 @@ def sparse_segment_sum(data, indices, segment_ids, name=None,
   r"""Computes the sum along sparse segments of a tensor.
 
   Read [the section on
-  segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+  segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
   for an explanation of segments.
 
   Like `SegmentSum`, but `segment_ids` can have rank less than `data`'s first
@@ -3292,7 +3342,7 @@ def sparse_segment_mean(data,
   r"""Computes the mean along sparse segments of a tensor.
 
   Read [the section on
-  segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+  segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
   for an explanation of segments.
 
   Like `SegmentMean`, but `segment_ids` can have rank less than `data`'s first
@@ -3337,7 +3387,7 @@ def sparse_segment_mean_v2(data,
   r"""Computes the mean along sparse segments of a tensor.
 
   Read [the section on
-  segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+  segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
   for an explanation of segments.
 
   Like `SegmentMean`, but `segment_ids` can have rank less than `data`'s first
diff --git a/tensorflow/python/ops/ragged/segment_id_ops.py b/tensorflow/python/ops/ragged/segment_id_ops.py
index 42dc13223b..31e26e7c9d 100644
--- a/tensorflow/python/ops/ragged/segment_id_ops.py
+++ b/tensorflow/python/ops/ragged/segment_id_ops.py
@@ -29,7 +29,7 @@ from tensorflow.python.util.tf_export import tf_export
 
 
 # For background on "segments" and "segment ids", see:
-# https://www.tensorflow.org/api_guides/python/math_ops#Segmentation
+# https://www.tensorflow.org/api_docs/python/tf/math#Segmentation
 @tf_export("ragged.row_splits_to_segment_ids")
 def row_splits_to_segment_ids(splits, name=None):
   """Generates the segmentation corresponding to a RaggedTensor `row_splits`.
@@ -64,7 +64,7 @@ def row_splits_to_segment_ids(splits, name=None):
 
 
 # For background on "segments" and "segment ids", see:
-# https://www.tensorflow.org/api_guides/python/math_ops#Segmentation
+# https://www.tensorflow.org/api_docs/python/tf/math#Segmentation
 @tf_export("ragged.segment_ids_to_row_splits")
 def segment_ids_to_row_splits(segment_ids, num_segments=None, name=None):
   """Generates the RaggedTensor `row_splits` corresponding to a segmentation.
-- 
GitLab


From a02737b89b732c799783f5471c4e939a17d8ac32 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Mon, 4 Feb 2019 06:02:17 -0800
Subject: [PATCH 1837/2345] Fix shape inference for RFFT on empty shapes.

When the input is a shape with 0 dimensions, the output shape should be the
same. Otherwise we don't detect that nothing needs to be computed.

PiperOrigin-RevId: 232283150
---
 tensorflow/compiler/xla/service/shape_inference.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index 3ddcaae193..bfb18f40b6 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -1858,6 +1858,9 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
               fft_length[i]);
         }
       }
+      if (ShapeUtil::IsZeroElementArray(in)) {
+        return in;
+      }
       Shape result = ShapeUtil::ChangeElementType(in, C64);
       result.set_dimensions(result.dimensions_size() - 1,
                             fft_length[fft_rank - 1] / 2 + 1);
-- 
GitLab


From 1f21340b1ef115f643d1c41f952e655274f934e9 Mon Sep 17 00:00:00 2001
From: Rick Chao <rchao@google.com>
Date: Mon, 4 Feb 2019 06:11:17 -0800
Subject: [PATCH 1838/2345] Doc update: training->evaluation in evaluate() for
 distribution strategy.

PiperOrigin-RevId: 232284526
---
 tensorflow/python/keras/engine/training.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 6b4dadf2f4..c63eee5c44 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -1052,7 +1052,7 @@ class Model(Network):
         # Multi-Worker mode runs the Keras evaluation loop on multiple
         # servers via the Distribute Coordinator.
         def _worker_fn(_):
-          """Run training inside the distributed coordinator."""
+          """Run evaluation inside the distributed coordinator."""
           return training_distributed.evaluate_distributed(
               self,
               x=x,
-- 
GitLab


From 62eec96637f0d7c4cf3493a676fada4a2b903821 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 4 Feb 2019 06:45:58 -0800
Subject: [PATCH 1839/2345] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 232287999

---
 tensorflow/go/op/wrappers.go | 120 +++++++++++++++++++++++++++++------
 1 file changed, 101 insertions(+), 19 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 05b8e6fd19..8d8d6474c2 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -7411,7 +7411,7 @@ func SparseToDense(scope *Scope, sparse_indices tf.Output, output_shape tf.Outpu
 // misisng, the `output` tensor at that position will be zeroed.
 //
 // Read
-// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+// [the section on segmentation](https://tensorflow.org/api_docs/python/tf/sparse#Segmentation)
 // for an explanation of segments.
 //
 // For example:
@@ -7741,7 +7741,7 @@ func Sinh(scope *Scope, x tf.Output) (y tf.Output) {
 // Computes the sum along sparse segments of a tensor.
 //
 // Read
-// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+// [the section on segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
 // for an explanation of segments.
 //
 // Like `SegmentSum`, but `segment_ids` can have rank less than `data`'s first
@@ -9215,7 +9215,7 @@ func SdcaOptimizerV2(scope *Scope, sparse_example_indices []tf.Output, sparse_fe
 // Computes the minimum along segments of a tensor.
 //
 // Read
-// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#segmentation)
+// [the section on segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
 // for an explanation of segments.
 //
 // This operator is similar to the unsorted segment sum operator found
@@ -9229,6 +9229,15 @@ func SdcaOptimizerV2(scope *Scope, sparse_example_indices []tf.Output, sparse_fe
 // possible value for the specific numeric type,
 // `output[i] = numeric_limits<T>::max()`.
 //
+// For example:
+//
+// ``` python
+// c = tf.constant([[1,2,3,4], [5,6,7,8], [4,3,2,1]])
+// tf.unsorted_segment_min(c, tf.constant([0, 1, 0]), num_segments=2)
+// # ==> [[ 1,  2, 2, 1],
+// #       [5,  6, 7, 8]]
+// ```
+//
 // If the given segment ID `i` is negative, then the corresponding value is
 // dropped, and will not be included in the result.
 //
@@ -12685,7 +12694,7 @@ func NthElement(scope *Scope, input tf.Output, n tf.Output, optional ...NthEleme
 // Computes the maximum along segments of a tensor.
 //
 // Read
-// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+// [the section on segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
 // for an explanation of segments.
 //
 // This operator is similar to the unsorted segment sum operator found
@@ -12706,6 +12715,16 @@ func NthElement(scope *Scope, input tf.Output, n tf.Output, optional ...NthEleme
 // <img style="width:100%" src="https://www.tensorflow.org/images/UnsortedSegmentMax.png" alt>
 // </div>
 //
+// For example:
+//
+// ``` python
+// c = tf.constant([[1,2,3,4], [5,6,7,8], [4,3,2,1]])
+// tf.unsorted_segment_max(c, tf.constant([0, 1, 0]), num_segments=2)
+// # ==> [[ 4,  3, 3, 4],
+// #       [5,  6, 7, 8]]
+// ```
+//
+//
 // Arguments:
 //
 //	segment_ids: A tensor whose shape is a prefix of `data.shape`.
@@ -25923,7 +25942,7 @@ func TextLineDataset(scope *Scope, filenames tf.Output, compression_type tf.Outp
 // Computes the sum along segments of a tensor.
 //
 // Read
-// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+// [the section on segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
 // for an explanation of segments.
 //
 // Computes a tensor such that
@@ -25936,6 +25955,16 @@ func TextLineDataset(scope *Scope, filenames tf.Output, compression_type tf.Outp
 // <img style="width:100%" src="https://www.tensorflow.org/images/SegmentSum.png" alt>
 // </div>
 //
+// For example:
+//
+// ```
+// c = tf.constant([[1,2,3,4], [4, 3, 2, 1], [5,6,7,8]])
+// tf.segment_sum(c, tf.constant([0, 0, 1]))
+// # ==> [[5, 5, 5, 5],
+// #      [5, 6, 7, 8]]
+// ```
+//
+//
 // Arguments:
 //
 //	segment_ids: A 1-D tensor whose size is equal to the size of `data`'s
@@ -25960,7 +25989,7 @@ func SegmentSum(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.
 // Computes the mean along segments of a tensor.
 //
 // Read
-// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+// [the section on segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
 // for an explanation of segments.
 //
 // Computes a tensor such that
@@ -25974,6 +26003,16 @@ func SegmentSum(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.
 // <img style="width:100%" src="https://www.tensorflow.org/images/SegmentMean.png" alt>
 // </div>
 //
+// For example:
+//
+// ```
+// c = tf.constant([[1.0,2,3,4], [4, 3, 2, 1], [5,6,7,8]])
+// tf.segment_mean(c, tf.constant([0, 0, 1]))
+// # ==> [[2.5, 2.5, 2.5, 2.5],
+// #      [5, 6, 7, 8]]
+// ```
+//
+//
 // Arguments:
 //
 //	segment_ids: A 1-D tensor whose size is equal to the size of `data`'s
@@ -25998,7 +26037,7 @@ func SegmentMean(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf
 // Computes the minimum along segments of a tensor.
 //
 // Read
-// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+// [the section on segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
 // for an explanation of segments.
 //
 // Computes a tensor such that
@@ -26011,6 +26050,15 @@ func SegmentMean(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf
 // <img style="width:100%" src="https://www.tensorflow.org/images/SegmentMin.png" alt>
 // </div>
 //
+// For example:
+//
+// ```
+// c = tf.constant([[1,2,3,4], [4, 3, 2, 1], [5,6,7,8]])
+// tf.segment_min(c, tf.constant([0, 0, 1]))
+// # ==> [[1, 2, 2, 1],
+// #      [5, 6, 7, 8]]
+// ```
+//
 // Arguments:
 //
 //	segment_ids: A 1-D tensor whose size is equal to the size of `data`'s
@@ -26035,7 +26083,7 @@ func SegmentMin(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.
 // Computes the sum along segments of a tensor.
 //
 // Read
-// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+// [the section on segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
 // for an explanation of segments.
 //
 // Computes a tensor such that
@@ -26054,6 +26102,14 @@ func SegmentMin(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.
 // <img style="width:100%" src="https://www.tensorflow.org/images/UnsortedSegmentSum.png" alt>
 // </div>
 //
+// ``` python
+// c = tf.constant([[1,2,3,4], [5,6,7,8], [4,3,2,1]])
+// tf.unsorted_segment_sum(c, tf.constant([0, 1, 0]), num_segments=2)
+// # ==> [[ 5,  5, 5, 5],
+// #       [5,  6, 7, 8]]
+// ```
+//
+//
 // Arguments:
 //
 //	segment_ids: A tensor whose shape is a prefix of `data.shape`.
@@ -26079,7 +26135,7 @@ func UnsortedSegmentSum(scope *Scope, data tf.Output, segment_ids tf.Output, num
 // Computes the product along segments of a tensor.
 //
 // Read
-// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#segmentation)
+// [the section on segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
 // for an explanation of segments.
 //
 // This operator is similar to the unsorted segment sum operator found
@@ -26090,6 +26146,15 @@ func UnsortedSegmentSum(scope *Scope, data tf.Output, segment_ids tf.Output, num
 // \\(output_i = \prod_{j...} data[j...]\\) where the product is over tuples
 // `j...` such that `segment_ids[j...] == i`.
 //
+// For example:
+//
+// ``` python
+// c = tf.constant([[1,2,3,4], [5,6,7,8], [4,3,2,1]])
+// tf.unsorted_segment_prod(c, tf.constant([0, 1, 0]), num_segments=2)
+// # ==> [[ 4,  6, 6, 4],
+// #       [5,  6, 7, 8]]
+// ```
+//
 // If there is no entry for a given segment ID `i`, it outputs 1.
 //
 // If the given segment ID `i` is negative, then the corresponding value is
@@ -26119,9 +26184,7 @@ func UnsortedSegmentProd(scope *Scope, data tf.Output, segment_ids tf.Output, nu
 
 // Computes the mean along sparse segments of a tensor.
 //
-// Read
-// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
-// for an explanation of segments.
+// See `tf.sparse.segment_sum` for usage examples.
 //
 // Like `SegmentMean`, but `segment_ids` can have rank less than `data`'s first
 // dimension, selecting a subset of dimension 0, specified by `indices`.
@@ -26332,7 +26395,7 @@ func Cosh(scope *Scope, x tf.Output) (y tf.Output) {
 // misisng, the `output` tensor at that position will be zeroed.
 //
 // Read
-// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+// [the section on segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
 // for an explanation of segments.
 //
 // Arguments:
@@ -26478,9 +26541,8 @@ func SparseSegmentMeanGrad(scope *Scope, grad tf.Output, indices tf.Output, segm
 //
 // N is the size of the segment being reduced.
 //
-// Read
-// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
-// for an explanation of segments.
+// See `tf.sparse.segment_sum` for usage examples.
+//
 //
 // Arguments:
 //
@@ -26539,7 +26601,7 @@ func Igammac(scope *Scope, a tf.Output, x tf.Output) (z tf.Output) {
 // misisng, the `output` tensor at that position will be zeroed.
 //
 // Read
-// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+// [the section on segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
 // for an explanation of segments.
 //
 // Arguments:
@@ -26883,7 +26945,7 @@ func Tanh(scope *Scope, x tf.Output) (y tf.Output) {
 // Computes the maximum along segments of a tensor.
 //
 // Read
-// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+// [the section on segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
 // for an explanation of segments.
 //
 // Computes a tensor such that
@@ -26896,6 +26958,16 @@ func Tanh(scope *Scope, x tf.Output) (y tf.Output) {
 // <img style="width:100%" src="https://www.tensorflow.org/images/SegmentMax.png" alt>
 // </div>
 //
+// For example:
+//
+// ```
+// c = tf.constant([[1,2,3,4], [4, 3, 2, 1], [5,6,7,8]])
+// tf.segment_max(c, tf.constant([0, 0, 1]))
+// # ==> [[4, 3, 3, 4],
+// #      [5, 6, 7, 8]]
+// ```
+//
+//
 // Arguments:
 //
 //	segment_ids: A 1-D tensor whose size is equal to the size of `data`'s
@@ -30035,7 +30107,7 @@ func Svd(scope *Scope, input tf.Output, optional ...SvdAttr) (s tf.Output, u tf.
 // Computes the product along segments of a tensor.
 //
 // Read
-// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+// [the section on segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
 // for an explanation of segments.
 //
 // Computes a tensor such that
@@ -30048,6 +30120,16 @@ func Svd(scope *Scope, input tf.Output, optional ...SvdAttr) (s tf.Output, u tf.
 // <img style="width:100%" src="https://www.tensorflow.org/images/SegmentProd.png" alt>
 // </div>
 //
+// For example:
+//
+// ```
+// c = tf.constant([[1,2,3,4], [4, 3, 2, 1], [5,6,7,8]])
+// tf.segment_prod(c, tf.constant([0, 0, 1]))
+// # ==> [[4, 6, 6, 4],
+// #      [5, 6, 7, 8]]
+// ```
+//
+//
 // Arguments:
 //
 //	segment_ids: A 1-D tensor whose size is equal to the size of `data`'s
-- 
GitLab


From b762fc43b88f374b94ebe157639be3e97edb7c2a Mon Sep 17 00:00:00 2001
From: Brian Lee <brianklee@google.com>
Date: Mon, 4 Feb 2019 08:44:48 -0800
Subject: [PATCH 1840/2345] Don't unwrap decorators when autographing.

PiperOrigin-RevId: 232303889
---
 tensorflow/python/autograph/pyct/parser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/autograph/pyct/parser.py b/tensorflow/python/autograph/pyct/parser.py
index 67bfb55fc7..8b73440280 100644
--- a/tensorflow/python/autograph/pyct/parser.py
+++ b/tensorflow/python/autograph/pyct/parser.py
@@ -38,7 +38,7 @@ def parse_entity(entity):
   """Returns the AST of given entity."""
   try:
     with _parse_lock:
-      source = tf_inspect.getsource(entity)
+      source = tf_inspect.getsource_no_unwrap(entity)
   except (IOError, OSError) as e:
     raise ValueError(
         'Unable to locate the source code of {}. Note that functions defined'
-- 
GitLab


From 84db8371a3a52d68a7c59fe87a80a1c5d1d93b75 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 4 Feb 2019 08:47:06 -0800
Subject: [PATCH 1841/2345] Fix error in error message.

PiperOrigin-RevId: 232304217
---
 tensorflow/python/autograph/impl/api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/autograph/impl/api.py b/tensorflow/python/autograph/impl/api.py
index 3ce6896e93..5b6d15f301 100644
--- a/tensorflow/python/autograph/impl/api.py
+++ b/tensorflow/python/autograph/impl/api.py
@@ -337,7 +337,7 @@ def converted_call(f, owner, options, *args, **kwargs):
         'Entity %s could not be transformed and will be staged without change.'
         ' Error details can be found in the logs when running with the env'
         ' variable AUTOGRAPH_VERBOSITY=5. Please report this to the AutoGraph'
-        ' team. Cause: %s', e)
+        ' team. Cause: %s', target_entity, e)
 
     return f(*args, **kwargs)
 
-- 
GitLab


From a127a0b5ceb9531456a9965c673327bc0f64ec14 Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Mon, 4 Feb 2019 09:12:24 -0800
Subject: [PATCH 1842/2345] [tf.data] Fix RangeDataset::output_shapes. Also fix
 C++ tests to actually check shapes.

PiperOrigin-RevId: 232308287
---
 tensorflow/core/kernels/data/map_dataset_op_test.cc   | 9 +++++----
 tensorflow/core/kernels/data/range_dataset_op.cc      | 2 +-
 tensorflow/core/kernels/data/range_dataset_op_test.cc | 9 +++++----
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/tensorflow/core/kernels/data/map_dataset_op_test.cc b/tensorflow/core/kernels/data/map_dataset_op_test.cc
index 813b435a72..f9c1cf4936 100644
--- a/tensorflow/core/kernels/data/map_dataset_op_test.cc
+++ b/tensorflow/core/kernels/data/map_dataset_op_test.cc
@@ -244,10 +244,11 @@ TEST_F(MapDatasetOpTest, DatasetOutputShapes) {
       CreateDataset(map_kernel.get(), map_context.get(), &map_dataset));
   core::ScopedUnref scored_unref_map_dataset(map_dataset);
 
-  std::vector<PartialTensorShape> expected_shapes({{}});
+  std::vector<PartialTensorShape> expected_shapes({PartialTensorShape({})});
   EXPECT_EQ(map_dataset->output_shapes().size(), expected_shapes.size());
   for (int i = 0; i < map_dataset->output_shapes().size(); ++i) {
-    map_dataset->output_shapes()[i].IsIdenticalTo(expected_shapes[i]);
+    EXPECT_TRUE(
+        map_dataset->output_shapes()[i].IsIdenticalTo(expected_shapes[i]));
   }
 }
 
@@ -398,10 +399,10 @@ TEST_F(MapDatasetOpTest, IteratorOutputShapes) {
   TF_ASSERT_OK(
       map_dataset->MakeIterator(iterator_context.get(), "Iterator", &iterator));
 
-  std::vector<PartialTensorShape> expected_shapes({{}});
+  std::vector<PartialTensorShape> expected_shapes({PartialTensorShape({})});
   EXPECT_EQ(iterator->output_shapes().size(), expected_shapes.size());
   for (int i = 0; i < map_dataset->output_shapes().size(); ++i) {
-    iterator->output_shapes()[i].IsIdenticalTo(expected_shapes[i]);
+    EXPECT_TRUE(iterator->output_shapes()[i].IsIdenticalTo(expected_shapes[i]));
   }
 }
 
diff --git a/tensorflow/core/kernels/data/range_dataset_op.cc b/tensorflow/core/kernels/data/range_dataset_op.cc
index aa14d27d5c..87390ad512 100644
--- a/tensorflow/core/kernels/data/range_dataset_op.cc
+++ b/tensorflow/core/kernels/data/range_dataset_op.cc
@@ -64,7 +64,7 @@ class RangeDatasetOp : public DatasetOpKernel {
 
     const std::vector<PartialTensorShape>& output_shapes() const override {
       static std::vector<PartialTensorShape>* shapes =
-          new std::vector<PartialTensorShape>({{}});
+          new std::vector<PartialTensorShape>({PartialTensorShape({})});
       return *shapes;
     }
 
diff --git a/tensorflow/core/kernels/data/range_dataset_op_test.cc b/tensorflow/core/kernels/data/range_dataset_op_test.cc
index 28abea8f97..0bbc09a212 100644
--- a/tensorflow/core/kernels/data/range_dataset_op_test.cc
+++ b/tensorflow/core/kernels/data/range_dataset_op_test.cc
@@ -177,10 +177,11 @@ TEST_F(RangeDatasetOpTest, DatasetOutputShapes) {
       CreateDataset(range_kernel.get(), range_context.get(), &range_dataset));
   core::ScopedUnref scored_unref(range_dataset);
 
-  std::vector<PartialTensorShape> expected_shapes({{}});
+  std::vector<PartialTensorShape> expected_shapes({PartialTensorShape({})});
   EXPECT_EQ(range_dataset->output_shapes().size(), expected_shapes.size());
   for (int i = 0; i < range_dataset->output_shapes().size(); ++i) {
-    range_dataset->output_shapes()[i].IsIdenticalTo(expected_shapes[i]);
+    EXPECT_TRUE(
+        range_dataset->output_shapes()[i].IsIdenticalTo(expected_shapes[i]));
   }
 }
 
@@ -304,10 +305,10 @@ TEST_F(RangeDatasetOpTest, IteratorOutputShapes) {
   TF_ASSERT_OK(range_dataset->MakeIterator(iterator_context.get(), "Iterator",
                                            &iterator));
 
-  std::vector<PartialTensorShape> expected_shapes({{}});
+  std::vector<PartialTensorShape> expected_shapes({PartialTensorShape({})});
   EXPECT_EQ(iterator->output_shapes().size(), expected_shapes.size());
   for (int i = 0; i < range_dataset->output_shapes().size(); ++i) {
-    iterator->output_shapes()[i].IsIdenticalTo(expected_shapes[i]);
+    EXPECT_TRUE(iterator->output_shapes()[i].IsIdenticalTo(expected_shapes[i]));
   }
 }
 
-- 
GitLab


From 1bf1de5b85c7569cc2da12e94ab740f2f39b8ecb Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Mon, 4 Feb 2019 10:08:08 -0800
Subject: [PATCH 1843/2345] Fix #if checking for TRT 5.1+

---
 tensorflow/compiler/tf2tensorrt/utils/trt_logger.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/tf2tensorrt/utils/trt_logger.cc b/tensorflow/compiler/tf2tensorrt/utils/trt_logger.cc
index f454f55f2c..6bc842ed5c 100644
--- a/tensorflow/compiler/tf2tensorrt/utils/trt_logger.cc
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_logger.cc
@@ -26,7 +26,7 @@ namespace tensorrt {
 void Logger::log(Severity severity, const char* msg) {
   // Suppress info-level messages
   switch (severity) {
-#if NV_TENSORRT_MAJOR >= 5 && NV_TENSORRT_MINOR >= 1
+#if NV_TENSORRT_MAJOR > 5 || (NV_TENSORRT_MAJOR == 5 && NV_TENSORRT_MINOR >= 1)
     case Severity::kVERBOSE:
 #endif
     case Severity::kINFO: {  // Mark TRT info messages as debug!
-- 
GitLab


From 63a13795c262071853677cb9fb1f67e79c428216 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Mon, 4 Feb 2019 10:05:44 -0800
Subject: [PATCH 1844/2345] Do not use unix only paths in
 keras/integration_test.py

All python code needs to be able to work on all operating systems, including windows.

PiperOrigin-RevId: 232317348
---
 tensorflow/python/keras/integration_test.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/integration_test.py b/tensorflow/python/keras/integration_test.py
index bb889f48b1..03d7e891ef 100644
--- a/tensorflow/python/keras/integration_test.py
+++ b/tensorflow/python/keras/integration_test.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import os
+
 import numpy as np
 
 from tensorflow.python import keras
@@ -236,7 +238,7 @@ class ActivationV2IntegrationTest(keras_parameterized.TestCase):
               verbose=2)
 
     output_path = keras.saving.saved_model.export(
-        model, '/tmp/tf_keras_saved_model')
+        model, os.path.join(self.get_temp_dir(), 'tf_keras_saved_model'))
     loaded_model = keras.saving.saved_model.load_from_saved_model(output_path)
     self.assertEqual(model.summary(), loaded_model.summary())
 
-- 
GitLab


From de82ac8a808fe35964f0e6e8352018c70a498120 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Mon, 4 Feb 2019 10:13:13 -0800
Subject: [PATCH 1845/2345] Update TFLite iOS demo app documentation

PiperOrigin-RevId: 232318736
---
 tensorflow/lite/g3doc/demo_ios.md             | 211 +++++++++++++++---
 .../g3doc/images/ios/build_and_execute.png    | Bin 0 -> 81148 bytes
 .../g3doc/images/ios/device_selection.png     | Bin 0 -> 119832 bytes
 3 files changed, 182 insertions(+), 29 deletions(-)
 create mode 100644 tensorflow/lite/g3doc/images/ios/build_and_execute.png
 create mode 100644 tensorflow/lite/g3doc/images/ios/device_selection.png

diff --git a/tensorflow/lite/g3doc/demo_ios.md b/tensorflow/lite/g3doc/demo_ios.md
index f4b481dc61..33e74f1a40 100644
--- a/tensorflow/lite/g3doc/demo_ios.md
+++ b/tensorflow/lite/g3doc/demo_ios.md
@@ -1,9 +1,10 @@
-
 # iOS Demo App
 
-The TensorFlow Lite demo is a camera app that continuously classifies whatever
-it sees from your device's back camera, using a quantized MobileNet model. These
-instructions walk you through building and running the demo on an iOS device.
+This tutorial provides a simple iOS mobile application to classify images using
+the iOS device camera. In this tutorial, you will download the demo application
+from the Tensorflow repository, build it on your computer, and install it on
+your iOS Device. You will also learn how to customize the application to suit
+your requirements.
 
 ## Prerequisites
 
@@ -30,47 +31,199 @@ instructions walk you through building and running the demo on an iOS device.
     If this is a new install, you will need to run the Xcode application once to
     agree to the license before continuing.
 
-## Building the iOS Demo App
-
-1.  Install CocoaPods if you don't have it:
+*   Install CocoaPods if you don't have it:
 
         sudo gem install cocoapods
 
-2.  Download the model files used by the demo app (this is done from inside the
-    cloned directory):
+### Step 1. Clone the TensorFlow source code
+
+First, we clone the GitHub repository on the computer in a folder to get the
+demo application.
+
+```
+git clone https://github.com/tensorflow/tensorflow
+```
+
+### Step 2. Download required dependencies
+
+Execute the shell script to download the model files used by the demo app (this
+is done from inside the cloned directory):
+
+```
+    tensorflow/lite/examples/ios/download_models.sh
+```
 
-        sh tensorflow/lite/examples/ios/download_models.sh
+Run the following command to install TensorFlow Lite pod:
 
-3.  Install the pod to generate the workspace file:
+```
+    cd tensorflow/lite/examples/ios/camera
+    pod install
+```
 
-        cd tensorflow/lite/examples/ios/camera
-        pod install
+If you have installed this pod before and that command doesn't work, try
 
-    If you have installed this pod before and that command doesn't work, try
+```
+    pod repo update
+```
 
-        pod repo update
+### Step 3. Build the XCode project
 
-    At the end of this step you should have a file called
-    `tflite_camera_example.xcworkspace`.
+Open the `tflite_camera_example.xcworkspace` project file generated in the last
+step:
 
-4.  Open the project in Xcode by typing this on the command line:
+```
+    open tflite_camera_example.xcworkspace
+```
 
-        open tflite_camera_example.xcworkspace
+Under `Project navigator -> tflite_camera_example -> Targets ->
+tflite_camera_example -> General` change the bundle identifier by pre-pending
+your name:
 
-    This launches Xcode if it isn't open already and opens the
-    `tflite_camera_example` project.
+![pre-pend your name to the bundle identifier](images/ios/bundle_identifier.png)
 
-5.  Under `Project navigator -> tflite_camera_example -> Targets ->
-    tflite_camera_example -> General` change the bundle identifier by
-    pre-pending your name:
+Plug in your iOS device. Note the app must be executed with a real device with
+camera. Select the iOS device from the drop-down menu.
 
-    ![pre-pend your name to the bundle identifier](images/ios/bundle_identifier.png)
+![Device selection](images/ios/device_selection.png)
 
-6.  Build and run the app in Xcode.
+Click the "Run" button to build and run the app
 
-    Note that as mentioned earlier, you must already have a device set up and
-    linked to your Apple Developer account in order to deploy the app on a
-    device.
+![Build and execute](images/ios/build_and_execute.png)
+
+Note that as mentioned earlier, you must already have a device set up and linked
+to your Apple Developer account in order to deploy the app on a device.
 
 You'll have to grant permissions for the app to use the device's camera. Point
 the camera at various objects and enjoy seeing how the model classifies things!
+
+## Understanding iOS App Code
+
+### Get camera input
+
+The main logic of this app is in the Objective C++ source file
+`tensorflow/lite/examples/ios/camera/CameraExampleViewController.mm`.
+
+The `setupAVCapture` method constructs a `AVCaptureSession` and set itself as a
+delegate. The `captureOutput:didOutputSampleBuffer:fromConnection:` method is
+called for every captured frame. It calls `runModelOnFrame` to run the model for
+every frame.
+
+### Create an interpreter
+
+To create the interpreter, we need to load the model file. The following code
+will load a model and create an interpreter.
+
+```
+model = tflite::FlatBufferModel::BuildFromFile([graph_path UTF8String]);
+```
+
+Behind the scenes, the model is loaded as a memory-mapped file. It offers faster
+load times and reduce the dirty pages in memory.
+
+Construct a `BuiltinOpResolver` to use the TensorFliw Lite buildin ops. Then,
+create the interpreter object using `InterpreterBuilder` that takes the model
+file as argument as shown below.
+
+```
+tflite::ops::builtin::BuiltinOpResolver resolver;
+tflite::InterpreterBuilder(*model, resolver)(&interpreter);
+```
+
+### Obtain the input buffer
+
+By default, the app uses quantized model since it's smaller and faster. The
+buffer is a raw pointer to an array of 8 bit unsigned integers (`uint8_t`). The
+following code obtains the input buffer from the interpreter:
+
+```
+// Get the index of first input tensor.
+int input_tensor_index = interpreter->inputs()[0];
+// Get the pointer to the input buffer.
+uint8_t* buffer = interpreter->typed_tensor<uint8_t>(input_tensor_index);
+```
+
+Throughout this document, it's assumed a quantized model is used.
+
+### Pre-process of bitmap image
+
+The MobileNet model we're using takes 224x224x3 inputs, where the dimensions are
+width, height, and colors (RGB). The images returned from `AVCaptureSession` is
+bigger, and has 4 color channels (RGBA).
+
+Many image classification models (like MobileNet) take fixe-sized inputs. It's
+required to scale or crop the image before feeding it into the model, and change
+the channels from RGBA to RGB.
+
+The code to pre-process the images is in `ProcessInputWithQuantizedModel`
+function in
+`tensorflow/lite/examples/ios/camera/CameraExampleViewController.mm`. It's a
+simple implementation for nearest neighbor color sampling, and it only copies
+the first 3 bytes for each pixel.
+
+```
+void ProcessInputWithQuantizedModel(
+    uint8_t* input, uint8_t* output, int image_width, int image_height, int image_channels) {
+  for (int y = 0; y < wanted_input_height; ++y) {
+    uint8_t* out_row = output + (y * wanted_input_width * wanted_input_channels);
+    for (int x = 0; x < wanted_input_width; ++x) {
+      const int in_x = (y * image_width) / wanted_input_width;
+      const int in_y = (x * image_height) / wanted_input_height;
+      uint8_t* in_pixel = input + (in_y * image_width * image_channels) + (in_x * image_channels);
+      uint8_t* out_pixel = out_row + (x * wanted_input_channels);
+      for (int c = 0; c < wanted_input_channels; ++c) {
+        out_pixel[c] = in_pixel[c];
+      }
+    }
+  }
+}
+```
+
+Note the code is preprocessing and preparing the model input from the camera
+data. Therefore the first parameter `input` should be the camera buffer. The
+second parameter `output` should be the buffer of model input.
+
+### Run inference and obtain output buffer
+
+After preprocessing and filling the data into the input buffer of the
+interpreter, it's really easy to run the interpreter:
+
+```
+if (interpreter->Invoke() != kTfLiteOk) {
+  NSLog("Failed to invoke!");
+}
+```
+
+The result is stored in the output tensor buffer of the interpreter. The
+following code obtains the pointer to the buffer:
+
+```
+// Get the index of first output tensor.
+const int output_tensor_index = interpreter->outputs()[0];
+// Get the pointer to the output buffer.
+uint8_t* buffer = interpreter->typed_tensor<uint8_t>(output_tensor_index);
+```
+
+### Post-process values
+
+The output buffer contains an array of `uint8_t`, and the value range is 0-255.
+We need to convert the value to float to get the probabilities with value range
+0.0-1.0. The formula of the quantization value mapping is:
+
+    float_value = (quantized_value - zero_point) * scale
+
+The following code converts quantized values back to float values, using the
+quantizaiton parameters in tensors:
+
+```
+uint8_t* quantized_output = interpreter->typed_output_tensor<uint8_t>(0);
+int32_t zero_point = input_tensor->params.zero_point;
+float scale = input_tensor->params.scale;
+float output[output_size];
+for (int i = 0; i < output_size; ++i) {
+  output[i] = (quantized_output[i] - zero_point) * scale;
+}
+```
+
+Finally, we find the best set of classifications by storing them in a priority
+queue based on their confidence scores. See the `GetTopN` function in
+`tensorflow/lite/examples/ios/camera/CameraExampleViewController.mm`.
diff --git a/tensorflow/lite/g3doc/images/ios/build_and_execute.png b/tensorflow/lite/g3doc/images/ios/build_and_execute.png
new file mode 100644
index 0000000000000000000000000000000000000000..a305350cb7b413b70d2095cfd8fea6d396c63695
GIT binary patch
literal 81148
zcmeAS@N?(olHy`uVBq!ia0y~yVBF2Xz_43@iGhJZ`Azjc1_mC7OlRkS%;aPS29M6E
z;p`E?C&lJP2YF^Ra(Hm=?F?Gt&AHH3V1m{S4o`)JM`j4Pyg9LnYhA3%fwpyB8oPEe
zZ3?@xRj+HWy+~l-wWCeEQ_VFOysmq{_x;@Wv#bBlv3zc?TKs-4qr>tx^{LMm?P7T1
z`6>0IpyC?Nm6<w0>JAK|8I0uyO~<Eu-z+S=$Z+bx-}j%sS#4=6I_i4z>-nGmwW^pF
z&f#YeVlQlSICsAJM>xX=p?x9K_!%y;zgnCA#CvLm&=EGH845>&RL=!H7E(PI@Yrjr
zZ;}oBqgaLsnKtuuxEbE2tw<OBuvwx1n{P*~D$_<523eycjWvS)423F3ls|o1rfIl^
znNRZmhm^m)#>pL%kDr!0^eQUEiBI(O(w%!|{!h35xBbaeEw<zv2OWh^Dm|WRdU)U6
zGc(1O9x>^tR*^K^DDc|(Pn=G`P3N8<*RB&PTWzyWO)pM3&mf!>&Zuv~l@#1@Saql2
zMX6f1WfpU|4Pz8D7Fx`oz2al6%9M8142$fP%a4-T3zzjAzV^fHL&~ZhUcV=I%~9Fb
z_J5hj3E3z0!75%pfzNvaxsRMQ6v%cyGv&F8%gS7X*-C=yX&2m<b%a05XWlqJKuB=W
zv*XKBP91t|`fZcV&xE}<lqSxQUVJe0=b|GXyyx}o{HN|0JaX2F(>im4yT*J0+lX!l
z38`i4WOu7fQOSID)N093#g9_=!!O;j=(Y6|edea|HB4akDQ+Q?eH96IF%z=tY}{ta
zPhW9P`22}yy}iNDo5h<l4lpikU==AyX~<NN?zmkwtLlONS09T9jFKmqoE|K?$Wygs
zMViWoSqxnV<_mB()|}9HG1X>a4ftQ_v3lRG2A``rji;2^ZQ3mOT^eJ4L@)d_iO1EO
zt$%^rpZ*s$GqUw$PVKCJbt6D{?L`f?%1Eo`X-5o_Gz=$6_>~;-{ors%zQ65yg2kOe
z`8(wXx7UAbIo?us$oY_|6Z^w!A%z>??5>JPka^J{c*xW=y(Zc?k3G`%qks*=C+EC5
zowFIlYfO5M9BWR<@n7*Y!zeuLzwP(?SwEJ&ma^{BW?jm&m?c<d-}ftJm%hm)1%CT-
zn{Q?DS&3KL=^4*{|Cp0+rQ7g*#e3b2t~cu5n3Ty_cgr4{#lUUdWtuF$;jov@mM*!K
zVHXp&RD{}DPH_LzUt#0xldAr7-Pd(@*Ui6D{PU30)8B#x%RJLh9f@32BPPA?XRFS-
z7W*QW(|50)6=!%_?(%n5)!C3*d4Xp;*Y9eb#=vq$Mnb~kOQ?7$XT#ri#s&BOt=jy)
zX?=4s!<@*|;U~DRSjAK-9(X^QuVztCqvZxRz72Ba4_N;=8gMWtJL-6_tW4l~vx#F}
zy5tPTv}V=?B2${5Ch)Fcl0GQ5f%y&FZwFxumcHiu2P+FWW0=byyn4xW-ci4T`<Eb#
zqGCx)qlB_phwh`E9m;kd3PNoz${#(nCiSH#3b{0%aT8N|dctc7*Ug?Qiu)DVCq6zA
zxkPqntBu31iQy;QMVvENFlM#gTDUkseoC*~!p;{dCfsxT#1^W)2-+gF*0p?L?~B4O
zHdVZD9p*2tl{m1)KrO9vXM&vsd-u^x8+?4)&K^CrfoqLe_JODe|7%!ZAE?^EZy}ie
zK<5tYx(6OMoX;O${<!mp&>pV)jpmQkKWhJA-Xs3sX@Rg0=Q$R8rs)Qa6I^FFYN&<?
zY!l=Y@H@DCW%q|^9x6ti-(2#4m@E_8+_|~KyW@3db;t3Jw@3Vv<U3uA)GQ}6d2CdX
z)0`jT73F@*)yiv^&nu<9>e&-@PrQ3lP$U)Uq^A}>p?p&R1pUeLC;ZmjB@*2wuR2X>
z!I>pGi*uH43D67LrSUh!c9pGZ!z_=p%k&n61xc@1zM_2P`mI~Gtl3ht^;mZ2)@RwF
zS#PgeE&l58n%Q^$GndB~)Gmr_S-R!zmgFtbTdtQ_-R+a}@n8J>^7KouU#P$M{__1+
z$y>@>civ)pTY2m1t=?PHx5mE+D9I_kVHI4`Riah$?RCtRmlv15xb?EAq;gkr>FQGc
zSNATwy`21F=gXt7p1#z*czb1e*!z`sp7;Iiec~6yFWT?=zokx{IXS}Eg2ygdF<JYu
z$l}ut8439ZOi!3;nBT~~kai(^MZU!?gPR*JY`FDdLn6Cy@-4%iXJVciJ+pgO`s{dO
z+Tk0ATMmoyXG@e--mqT$(Cwg`(_*8=wr<yZ?4&mL9PTOZ370UJyl;HacaG1o*=G!H
zCKR1olD^|$!T*PU9;P1t&bvRiYnIi_W3$g0-A$@GxaItrJ35DavbLRj(-PTldD-TM
z&Fz}4KLtPMaksfHXuXlN^=M#sx{kBh?5U?$rOZwF8Z|4*ElNwrJbGK?x9FSEs*$46
zZ#V8r_ucw!<GC$yxp%YfW+x{apG({3yZ3Bvbl2McwI|BfzRkQ{UiPg_v&{aM=}p_M
z=9`}162E!<_WjZY)-0Aj<ts{eyz(gdQh25?=6%IU&eBgMLYAS{u_bPE_s%(b?)BW;
z?gDN<`X=<OkS#pi`s`4dX2HtxN`pz3MpY>vPb{0L?{>4tOL}Y4m6w6>F}AVAvD<gu
zE4*AT{W|Q;wMXAh-90(?ecao+tIh8O-x;1?eDm?sjOP~*U6gja|0SiO@AIFFo3DP}
zJbnB0Gpi4-zJ2?c_1)daez)~E`p-EpG~ak1%YA|SmHXOj?0;1LeER$C=ijIAAGJST
zzx@AY!*9tg$=mu~@@(LGz?#GNf+K}>8G9br8;(spk65p=+45}T6JyEet~*qglD@fo
zV|>ewwi8Z#EqfmSD127X*P66YaAE4g+{9mro{6D}-w&@nwE58M!_~?*%4x1L-6}3_
zuIoDX95j9WX=mx3dmHa1Kku=XbnW))Sk+Y}@m55)J5RE`Pue=~>W<xp+YBeoSQa60
zx;Iv|TK}9~Ue%ra$DNNK9}DlVmw7K{Z~d=?NjWj=(7BfFor;~Oh2E=Js%TD%np`?b
zUG=)^es50S$eEW66R)&b3ZGj(Ye`q|?8_dRCo^wf`lZ(98)LjNd(!hImprel-}cv=
zQ&ZM;a>^ae_T{g9p8DRM@8{v~d4J}=PVv*LpMFg(FwQcn`YH1->R++rhJ>vPq9qh!
zG;1bFPS|PmIWx@R+Nw2qffH6ZT~U74vHFPqX64Q1Y5Pxip1nEGc*5*SyVd*WOrKXC
zut@*p#5r@)?A8`U?Y<HjrS*2kJM-GJU*aOC<jj_t=s(T;x%2_v->j|8kJyhkHyvnt
z)%4wXQu-VHE80^;;%2-wDgJmy-SwfX{rN-bAJey<+csx!!nbD2H<q)XD-|0*+jegA
zyl|uS*>y+$$a1gXE)x3__$F%4rXObyO=;CtO;i2sZB^g#W8uD#-63IBkKO;vo9i7C
z?-bvv`z>Nm-pA=ijyHD}UH{be$=~~j_qo4Mz6C$^XZALpzD@UY)V4&+!ksBUzlu6)
zm#w)H9l2#w@zouny{jWvpAOp}X<T~tX4geg>FYt)wXT(gxP_ao-yFHU=I(N>z^hwd
zO?YK%y3DjY``%jdHT(A{RyZGa+uZw9+CEPxf7edSq~heYvAIXZ9!+b~?cc7JZ~J%i
z`SYc>eYQ=_|Gg)6)81tFKYf!+@0FHX`<Cmic7HAU_3m!9x4X7jmy{Pbdmnygd&c%@
z^})EK_tN(-`Cn8Ue`VjbzjnW+>$d*g`i$YRL!|O0i_IIidtSM6an7wfbp>w@UK9I2
zjd_3XpWNSyr=731^YZIkJJ|*mr98;EcyV9yrDWUVoBPVQ`Q`2{PCC@NTK)L^9Ba|i
zS7+Wo+&|yqtX=IFp93?U)Bm26vw2>;?&Y5AKaX+G6X*N$@aOSHv-R7Kb>9=8t^YGF
zD^vb*-IJmVPalUnt^ZRuwfgIytuwOUUZ1?~{vN}ZZpVEOo?X7ZD1Yys*n-{v&;8lm
zvOD#A$@|bd=6mCR>mRRw`|`spi$4lq4z{*)^1m<nWIf-0lKrnMp`T{_+wrLL=;U3?
z-}&>GwwK3OnSG4BUi|X&zj>GJzy4cO`S8P|>-(Gj@48^UaQo|}Z=N1lt-8K9KEK}h
zmv44_%>BRHEAxNfpSst)ZvW%v^X-@AgYE2I&i~l?W#y;MkDpJ|4~&1u_k@?nqta+z
zmUhFz#<Vo14_Vd>UqTop3iuu8uCRN)k|l7yZz#tf^)+_ftB!CtuvG--UAH{WaD%)3
zZp{ha_ovoYB^fx_+O)N~J((%@E=y0!=A*zpex5yM%netV4y4~a;PC%fW8?Yg#7)~7
zEX-nS_l0l?&f_)PsdX{6`|pG5pXLiLH(ah@+ttLtz@U)q5#-CjP^HGe(9pub@QZ<g
zq2UDsL#Y7+!>a@a2CEqi4B`cIb_LonFfgzsdAqy(2LsNC?{^p&7&r?&B8wRq#8g3;
z(KATp0|NsCdx@v7EBj+6Ms{ffksDH03=DjsRUr{2L5bxG1x5L3nK`KnC6xuK3YnlT
zOhoJ2nVFNG?BH$v|5?P2Nxa}-;SRSS|HMDAdqz%;j5Yq|VzK(BS73~T4nM!w_xF}}
z_y172`e;vW-#MXG>-YJ{AHEtIb~7~VOyH^`FN#mks{XR?-H~MJE18FXyQI%q*D@(<
zRh-keo2S2=fByXM{aU`=yVAeD-W(JYKk4t`{@>|`oi&1rd{ysm|M^n?qUgqVo8GZI
zuUl>M@W8%fRrjY~dB<sZwd>iR)%Nz2*b7&D4to4|!M?I-LHB>puK8NubN$=C2?;Z!
zuUvXBJL`B?hPIvd`;U`OTyOgGbz;W)<@S}~X3MW_TL1fTJipmt>F&G2eS5@aNn}j@
z{Hk8;%f_9b!iBExpSI<X(#qX(SM23$BA)-g%<}W>`>vfE>g<2CKl}gs{Qr8z5R2@8
zl~W?->2Ka5we)Gl{kG_niCc|MO`P~Rdtvm<ME^Assi!N|?g$^wj0oFjUh!-3`<w|k
z6?XT_Ce08#G)b=g^TTNxYQc2^6F+{rFYT_GV{@}&*LhBfZ6Z^Z`fcY`^oe$^>UgHl
zzyEi%@olejFSlM$Ugfvv=L7ra>Ghf~G(3OY*^&SGzj^BZvKdW>+|R`e&r@Cy@=dJv
z;pOaw%A&dtcYO`Md9pmovv}g)FZZvvoL4jRh_~7$?)5Q>!}I%m?xVAc&+uP35`Ae(
z)yY)p7j<(Kp4Q6lteN?LE3;J8*R#{*WS6G72ls8W|GhPQ<6_U>ih;-f%#hT!5q@{W
zdvUB#>+~i0LWx`N>^u8ESN6$~_&DKr-ck3&PTiNb5z5Wa(ERv2<NP`GsF?cn{`97b
z%M)jQxfodU>o5x^m)nm|Z+<JiJeT@Od6~<$w?0i(mQU6OH*7Bnxw!fNm5CuL#+f`9
zTy$of==ix$?4S1WfO7|`-r2g}{ajb}@`b7Ajim~+DpxVaZe+do{E)r*$*-pEtvu`B
z9utY&t@!Kmai?co-}~~rk8bD9?cq$|tGuS)e7}5K>-{Br)$C@A*wjv+?xJQVJ8ju|
z#-G*Zlixi5Xe7MdCi%<PJAc3a<ki)GdhYkHSwDVTe7C%?=%18{y|}Tay=vby*85_m
z*ISypManc(oMNQCwX@4UYu`$^vuA&I{V%1p7r*d-{(tA8l!M04Jxt*<t{svOjEtMb
z?`eOFb6@Y1y6gRGq`Tfo$p4!vfBcU9+FE^;C3B<i)Nrc1ysT}BE>To@vm#9~;EkQw
zuPG((7ON+{mC~`-{j%f0{a@GH4e!kV_xbI&`*Hu19iJ{L(Xt7Q3{EcoUwU13_kGR(
zN@q_e>Yuzb-|zdvJ3qW1{Cl+GV#1U6B_D39e9rT}Usm6G?dzSr>wj$&Xy6Z1pP#mr
zbyksm=2xlN#|(9r-~To3chv{Yo7D-|;+ekdt~g!cx#WNI%)`$E_ixac$R$|4+WX&`
z@10jQ;;Se6yw+vk6J|8eNpSX4kIGfdpB9OC<aieDwosla9@<i3IAQ&xr*Dfdv3^@0
z#Qe+t_xVEcAKOc&yJUviUQa)BUu;UDIES)r{I6e8ADW$y?RtM?$8*;$X_vV^p8C-E
zY3HNs=F<7eyf<{pSxxVi2O8hFoU6L}aFC((ePi9<T$6jOBkNAT(vR8sXszvx+D5Ci
zbsBpM&m3dQjc~Xa;SlK%xxjd(xwSBRO5aYteNWR1+#)i4HYGF1)%8kuznM`TQ{kTJ
zF1_ORKdsz_pIm}(DQ*`2t7z1a%wnL{;8-AdYl+xm{*Px9KIllDcj#!ivT!4Z5PK@m
zrM5nog~k;dE<f<nU~y7pSf5yUH^%(>;g-uEPr7X06`sMZ7ns3*b<tvuORi1)uN)pL
z?mq4%A88)6{OGoY*1V4-Gj)t+oM{s*bqNtX>sGzt!pVCP{)>{#jvLQkw5|1>;Dn0m
zLte*r3*QuTbHB}E^8J);_wsXpU%cykH?_y)V#gC_txs(iUwVBmdbnk&N`cfdw~1WY
zjO!x<q%yf=J)dVZtuSIN5fd&+%2P^zoOL?=Md2|C%hDM4+H<`<v5v`7dXpw<mESs$
zCD~N=Tt_Nx-@hhR&qtTIjJ;<?tUu80CHvzZi@}4YzjrFy6Hl>aDgJU}sXeN$wL3g0
z&~*)0NyJ;PuMQtR^SFi|Sfj@kZMJrKU?AV0?-3iUvJO3xK9%#sGy9_Xk=(hz586(a
zTC@IPkCaYyijjwUu}uEeNkPt8kE_+czkYD=Nn7j`PFt4ymSOIj=JmFfEo!ohS&+Q2
zPwUsLee#=)>pyf(yz+Bmv;2Z3Q#UssOpf?b;QEo7+i%8O6;<Du#Ah!x`5Rl`rmwGy
zn(=VQ=LL&X>nmOeac_MmUKPm`Q}Jik$@NFezicdQ@BN;;#=Y}>xvjz*1sU!fLD{!^
zJH5ZQf0p0#hk=)Uz1@pfJkF0EpP#o(@y^bOL&tsQ>{r?(6y5wUbnaOmmrq~yPcXWN
zbL6&NxOnY~K-uF0Q-N?rFTRZ)3MEc&G>>@RUa-Z#@4$2J-2so88MgHKpXU)saS}H3
zdk|>NrfKUTz2P&fMb0-C87H0#bzV~*F<CNikx=XxwV3KUdv6xUl*J3DNP1Z>?CDtW
zWv-mog#c9+t@Z$ElSS_v@^=RK^YY3lUSKv2$XMHc%|E02z0V$ncI8Y-N6VFbG3C0-
zZ3~afS<QMWDqDJ&@ji!at5Su6Ys*1Hc5Xfu$<^%63NZ?2&TQ>7xOje!vCo&?3~N<&
zwya8=!R8|?*8Mhs>84dq$)?al#}<COR<Plir}Y8(!wdiS2%mfRu_=gCPpHyBEh(8v
zszG)c6GJphn6f|@hqaNm+YNQqte)QWe~;`Cvs~_8pxqRzr#5v)TF(W?1D{+bx}LON
z{6#Rew0qqQ$vND;XXdVa@maBXv%yW<Tf6&=R&}QyE@AdHx#l%xqeJZEH`|gH3+3zx
zc+Ihi<r^<^dZ37WXgG^6ha5{wkK3uYH%b|fvkP<<E@9Himf<;~yvXskk5u~2!c7li
zM9<7Ad^maav>i7pa!Xo@9#4}L<l$LAaYCnzfX4fe86EA5=WH&Fd$qyoL+PS&-bW1n
zVrSaBm}c~9CK)|U^3;6A@SXM2<{+jy!M2=X(}dK7Q$)4nOVuNFAO6;CKEE?C?fi=%
zJMEj=vw{Na?yt=8{U)^}>e7XouU4&O)u=Kvvk0Ee%QQt_T2uPvg9q%3u0}2k_mr7>
zrbtNLZ}Ed^dzWvNW?ReYHC4@hmi5xs#?EOi^VI~@0+*?HiA^`t<Wf0krzp!N6Z80u
zN19L3RG+_0@*&;gVoJOG_mn8}&*ZxJ_=Lt5E0IHCqDRkkpYGadpE>tbmUPC1Q!DN<
zKHXz>b?4ryCfRR_0iKT5L9Y+I-J^UnCDW+IVH%r_kj@9El3Sq(MmLsvr7N>2ZuH-I
z@8*UN4q1=<4OaQAadT{C_MW~}BH8!mgofB{54g(Li4?J&Kex$+KjHcW#<B@=D-=YT
z*k>Lnk<>r2QRw;ts}0i(l{>eli%fgFwtY+URQAAYMeAfrgme#NDQ=#&PV&<1z_!y<
z1C)B7w{dUVRWw^!TVt{7LF>~Mrsd1N`dM4@Z!w#)Ez<ZR!^WKxG8Esf(p|q&GG)=P
zIhFlW8*7XzPN*~#U40>WKFM?w<B!vm`FfNhTowl|*Z!aq(rql+thznMbkp7+x4XXU
zT(WcA_x9~)t(FAE6Wp15LyopfIwy#4+N_;>ev!_STitS>ryRS;FSBsk!E=j?ZP_ig
zZT>pMN`K%jmnf(!I;FSxjpjO;kh%mecAxgXi7y`NO5c9)-m$n<IU!-uPGt@rqlFxg
z`1z;)Ui$ZlSIC5KZu=I=ScmH>R9mhM^9Ydb{NuDpW49Sw_M6be54#+7HAN3Q6)kCc
zUn;OALD6RGMV-|F>ja#y{&>yCWZuYTaUfyIJl@lr#C`TkF3otdLhKoni;ai4Y?!#g
zLbF9v{_K=mcP~ZwjnApt1ARWb1&;=p+?@F(K*%fGm9hD*$c3=b>tEiu#cZ>yR=#Rp
za^u*{Trq*>eXJ$A&(;**(o36t*H0<;)wR_(n9FVImK?GEcYVqi#*A{+>CqcBBAA{W
zSSjV`KIch|iT~b10-v>byaJakV&8hb{TAD#hoKv+uNoINEvtQaCE@Q2ch6?8nJnU~
zcYQF}G-1;BlJC9Sb@;4SoZZ!&cqm^yGgNEkR)+T`x>NMGcqs>S1-i}<Zdm*LTwH*t
z$c2xRdveN`6zzR_<MtH314b^2#+%D0*-0+tKa`WBy7;_ZVw3j+r6-?!-`!YJ`#D}^
zllehDso4u-YaN{3B2;t~ANx<RTljmQrDEYKmm&}4?R@sm?^ZD<9DaGrLCPumZbo-e
z+m=;6Y)jqcXHGSXKXd-j46B|GJ*-<>yO<s)NFCK$(|4jV#7R}tQkG@KA(;a|YED!w
zVQEm&&0@^3FR<Pm@L-lXN9xxl4>o*X^-MnVtl6^`z0K3DB)n#8BsYl__ZUTH7ESOx
zHtT_8=VBiR#z#@l9qU%je9FE>*Y_L~uWx#w?oQS_`k#2KLY^>mq^#d_A?)00oq#!C
zFC5Nl-Lm83Sti9}?`C9wHNIf`<a&C0jqE>G?rTn5^SMuL=KJ!XL`(Y8Quz|@r{`Mk
zr(C)r#WB<D##T?|=yNJuM<?FCw(b7W1KqvM6NDBrCLh}P;cu%g<JEnPz6Vkl<S!^a
zT>X91_kAI)H%jF{=FR7vcuqvbdOkxu$MXZJ0iG9_Rb&+#CzLy6Zzz8oe9G!*58K6@
zo=gkbkmG{PUm0|lJ3n<_%U&&{!OQz9^2mv4h2Lf+JlRw(awYbgcS1$=+zD~(P2OCe
z`P%m|<L-6t+VSr_J>Go}===V0^26`yd+yELAwT#1_5UmXf4%&;|Niw$s*&||TPpT$
zFpo=^|8Y-8yZJr-Yqj&^g;IB_v@KZn<@DFb%oCd$mVUo((Rk_gPf?}$WqaQ?X5L?X
z{Oh)7I?1))#60v}nWfIZSA2a*v5a$lyr+HtH;Ki+^Bt#6d}IIQ*XPIonCETEJe~Th
zTb_ZTfx*+oF{I+wo7i#(k>lUzTR)I`$`>$8=_pf#V`o$Bo4wM}Z(@aeb+_FRT6{5L
zzgustu=kt2$Ko6oH;D!~Z)DUexT1aF?)(1jv&tpY?mj;7Wcr7H7SnFt)XYyyxBGpo
z<3!x1{x|ANL%fjDQI|*icQYc3Ed7$Noc-^%m$m!UPb>CJ+cNLZ@kOBuJ2hoh>=&(H
zeS6hQmg^JvH0#U1|IcmN8GA%1=4rFa>#v%#*8URwzh#?fzVtuNw?DTF&2wE^lNP9%
z?zM7?*US69N>PD)8#b*BPkz2^@~yJ0+k0~FF8Jj5x5@3s{Vn#n^@k>%;tI=N_l$3g
zlzRS(8{f*y_e=rlTnQt)d=%6gPCK4sUe0f4x9Md^kKn3L2j4ZHx^lYglFQTAKe;z~
zPiBwuZh5k0iOj>Tf5dMo@5wFkQ+)ctKD0>l=vDn%-r8CFHeFUtouPCw=G=erXa1Sh
zUblm9&Aaq%%C_hGug&X^?BhK0CT4B%(Q~0<Z2Esn63s)kLl<bB5K3b_Xr{kYzi^Vx
z#wU@n$8B!^>Aker%<|e{o&3;?9@cK#qwJlxxmo#riY@#qsIh)+-8Q?6&z7X@JK}Aj
z&h50QW<7hvT$32Rh~NuhcWz|yEZH2iV|hT;N1<5#)nyOgZd~&8%jRo`wc9GMnDTAC
z|04ELQAw4LchWzOnU*@g-tUQbT|DVQziRDI{SU{5w4VBF)_eW@?Ejdb-O#b_=ilet
z)wTb#m%o}DRq{&XN#g0q_!x!|4rPPLO@`d-kKVb*Ip6J9>aOTp3abxm*Uym4`>c^z
z@x^dbj;yw!*joMnMMbB-U+!+OZ#&~SciEJS`75XWIq9;=^@;X5y<2}a&hbk6XK>o{
za`>g?-g8v^Rr}IxLYI8+uJiKnI{DH4#O$~&R}W13dVG?csD8Zx`-5APLO1UC(LDce
z^>hBYmp|=2T=Xx(Prm-&jMLj6&9bl8ulM4c7iCw^;Oe2~)tox<**(#??oRie`J3Lm
z+@EN{y8p?ayUikVgc*eW7OszcdU*2%-d)KbPTTJGDfd1-W!t5<joFX+Hchs;@bhZ=
zmB3dXe-1rZUQpezb3zhl{fC02g3j9wpB;L4UNyaT>uk*Gs%bM{3QkSPd3NRXi~6_U
zQv+sqFfRW2+2-fgzZ0I-&6wP_T*c{*Sn<_6KbzCJZR6yN|IIkF$fQ=a)@t2cRz2@`
ze;3`)s4ED#^ygr-R6(1ZX4P$N%YQR=%hk>K%6`rH4EqerHFZ<J{L0Blx#L;&Pxu-C
z_X7#Za}M(9`R>@_Q|>M8^8I7-Ir)E6`j5zW8Grfr)$logcf!$QsnbmAe|)~6sd`p^
zPkQ1HkL#@EXP@%x$#+an+GSdh{r1|kg8PS!`Obd`sa#f}wo$70qW1so%RIvy1wHay
zHc$Q9+Y#*LSJGpioFHr_dVpK+$-$q6+-INi`z?RHG<e4s?|gQZuN&s?joklZdQ;X-
zPRaXfIdAF{Zb|;kuAK14a&`Zt&`Sa@bN-6&;<#J$YgYDoFhkM5@ppLG-8I)D-$s~T
zsx07KBl550;g?^%g>(LH+hn(0TmJXa=n}<T|KnFQV~_FdPOx5`xYUgMLq=Uf-h%g*
z|MIQHQW6<C6z?x^T4GVM*JQ^L%>bsHCwF38{|f2lFZi95*R{eX(y^|2SId>tzp9o!
z)wrO1G4i6|M@7Y_hhvkylhvKYXYspEcAKHyn|w9M<GsZBM}|-Hn|I!li|EdouFa=0
zC!_As&D3>fZFRF>T#YF6s0-aFo}cqbd(plW{d=ZfP4+v)!u%$GZJfyd*=xs{8TIBm
zPu4zPyZ6bn{t4#X_m`%72tMb3FMZ@rapm;_g=@DRmv44!G1Z=-yWBu%_3~-nZjn2h
z=KpY?dGsp3gHKp+1are@X62rHO8y_i)h2g_A7YH=TOzT>)%$4r!^msqUC%zx$-Y^d
z5y5s{XUDNq!5oi296R%4({uJ&fBqPjUvfBi=ju%1pzMj!*-~}IxBqp#*<)gr{ZHWJ
zm497@eA;u~Kl&3PmtfQsYr<Pxedy3xuZ5qTPbca$eE#(RrrCzwy~R?ty!WLaeF*Mi
z|5S5RU`EXkZ;omCyXNjteU$I2eL^IO@o7!<*Ox12p7Z8gd-Fz5+l-yti@rZq<7oW)
zLv+z|{%mf|4~DxY%>Qt$!`|M(%4z?!HSb-k4j+EcePU4$v#aN$zmIAjt2le>tZk1x
z^^0ZEZ;rhuej06$&XYPTe^kYl)m8u3->0vVa!#t{^Co{z-m7m?zvhwa`ja02j?SIO
zul7D+*1k!0u~+)PsLtF{(#2|0_&aibtBJ~@;}i3wdGkK-Bsy|QiEVhs|9?66UZ+cQ
zp3Cb?$#DG4Ii~q6YU$^CDN*tImh&l(xs4jP9C*3Llx3ZxylZiD+ou`NO}-p||M>V5
z{(JXZdvw-#m`rIg7rdsY___W6Wai};_8v|?*eeoswJ`RNgYbHL9mAFn{bFZAceG8r
zoXuKiR-XB8^Bvnmg?$sY3O-+cdWrj~CwifF!bzLj#LPSPCrk8o%#zf5T0GITSNw?Y
zn&VwUJ?DAvU)EW?R7z&|T2RbpF1v8wN^Q1=vzh1qoRa#F+KEO7richT`#0zu`svJX
zpe?&DQB#o5&{8U=$N0v<A~t>Xw!(t5Q~x{6)0b`%{WyamK#WPLS=3hK-i7jqsgFu(
zVm&xl)a+=})Vq~dVqWt6ndnp|6_%t=+h_jQ`)hRhr{Tx@ON$Sg>X>X;Fyn(k#6{<a
zz4k>PVs#Rk6j-Ab;;vXc<-Tg&XV1fTfyKX3km=u+1CP(e^Zmc``1ul5)y?yk{yd=7
z8_M*TQ(eX0_L`H0^Yl!kdaa29hduuZF<d*~-YW2qyRz#j<F5EW`7-sHs%Pc@Nd}!b
zW#$>^c#&J7X@Yt8QTczjGXFiEqjPT>OZ^K|0gi9n0&X*Ympx9`t>52LeE81x=o7z^
z-_7g(e1S>xMDjXaHD39jsWbm1St!nL3OK~>w0^UiaIC-*=kFq?pD1pcoT8+7B;^p}
zjwPo}9N%4AwDi&DU450?ezpi)3^HV_<oT1dfOA9W-<`LtZnp@2FcNTTy3fersqMMo
z$i<emE~fvT<F9s^9QA88-)^Y9pp#Km_qxe+`yYDTlV+`W(`olMwuXUW`lY{BE5Gj3
zaq4*@a5nnJ3K5rwt6zlL-LPHbV|&+eHG{hS`s8iDU8IYjrRDkjJ@_r6b-s=HvomLT
z{O=~rejxBFX1<Ja%N76Ivo9ZxxW01vTf2+C@7DD@uP9Erb#D$sa_q~)O`e&DJvU@c
zSTyf#Onbodhb<GPwLCCd$eZb1R3-7+idEdlt9r%iw5?Y%nWuOPaVV;&TNvAXeHj0W
z*PuwZ=6>6>MYAo7jo${D@7XbR%h$KOtbtzwW;l2<O#f1`kt1Q|bHnrVZPGt(zj^J}
z!-d6eeg)4KgsfX$aQ*X)w|0TG4|-l3UcXrPe%{^L!U<org<}jf-};~1qrOY8Z+o$<
z)7<vQ*-j^%Cw;U_UG}-)QjfqTt_r3<?G1u+o-AJ5v}bb7fiqVo+dqF^{qtb;P3a}4
zkDcgja;Px6W36%jb3|*v`reY+FQ*+2ZkfQ~wBx3!hOtd4^Av~gfAwu^&ljHwtq~7;
zFl)8oS>e9$j2@q;j<j$^UH7LKY!B6Fq~z_9HWjNkTlVk3Q^(xj3IArEJ6x=n+P1t%
zo~PvZ$K9Rt|DOKc@%+5n^M_xma}V{N;>(}YzMS`rr|I6)O6%sF+LrHPUfg8NE@~FH
z;T8iw_sq{f)|$)KHvajO@keuu?T)i*f3uzmyt=b$f|PK;?F8GD56|^$^h#_GE<X10
zJ}98rEZypqq<zyeJ3qGX{@v=l;&Z`u{g_Vey|a2Z+J@CR?CdpJ|HAy@6W{x891rJh
z{quLx&Vz5QEB-dtsor=eeD_GXVS2=C{%l^&lDoHO{A$iyP@MI%y7f)-iWG^DIg2_^
zT`!iO!{1ZbrntbTeNv`E$47o{*9{%#`cEG(-^e0+-EX?W39;agE4`_~98JM~MK{kB
zX#RJvIr~a$uKcB!BGZnt?eCb!Jn>+cTx+Apt)|`xKi?C+LE8oX8C=mb;oo^X+GXFP
zH#+mL%*}tgZnpW~eKo7r?RJ@dzj<=fy>0&wedM2f3EUuB7r&#gOjdix+Ir)Q>ot7O
zKex00{U_RSX`{&J=e5tTwJfc>w)p3a&w9J9=1Pb4*9(hYYI~$qwbOjN>PFcl_hWp_
zm268kOgX0z-zYL+Q$=KW#fzDVd9&^L>VH@UZ&}_ImUgk~uVMPd`&V!EUp_u*#nbtB
z?$<N@ExvYO|Mu4ZE!$@ZuW1mGarBv1D7*jK6#Jt>d)%LkY?eCyGw$kz<uA1#JAI55
zW-z%Sk-mXhQ`S3&kHtW+`s=1o53;4^J+j#mdLv7CH6NqbA=5PW^at-%Dzyq(R;pkB
z{)lP11Je)YXgk}4Egtu?CY|*SoSMUb@#CG@3P*J!W`$q>A@Mv>lPTd;(oBKw2V2GO
zxz3p%x^F^Nq`q;TSpW17Vper)s~+A6KH(I$g>`Ah-e>v!0(ll@1=8`ALSbRE_8dK3
zvVF_@eMtvXlE5`U>b}t2kUt0Fjk5VT9`;Yz`0RY7iGqiH>*?i7%hwdW*f-l+J0d8e
z&ThtSt+2X(*Dm?boGov!TC?WmK9OAC1lv%dKMPe~{88N;6=d@7f~Mx`pli$iocMow
z)!vGqhgY@EH~w;W(cYJ?)%(xnuHCs+WG(NXt<BR;K4I%BI?!L=5}n0mc$JxHO;fp8
zSZ1q(&$NiCW;(503_3{;%}euio?2WFh+B{uWUVECswIAwro#0WksBqMv%MwQ&Q-l%
z^P=O20Na#lEpyYG<~c69_vrJY=a1GO*%wj78PmF+al^qy>^#vQ9(1yPDVx2}@!a(_
z-yHtb&S47vw%1`^`p$<T4@0c9S@nh2JngJs{eXStk4WkA-NkyYJoEdq1^4krF!e>=
z(Tmb8iQQ%u==Ljg9m^@l_MaOSO|@jDKTX$3dD(8Zf8AG;_ikp-Ruwq7tzI3m^}-*)
zL!ti)6B{`no=SRH`=tKyhutSn1$Q(>9}v@!(c7FL|35>Fcb#K=OOIWBuj<+Pi?#}^
zp7is0><ONhFTRURv+FJHex03ne3$5h``h^5PP+B{(Uhyq(_8Cqch^tdHphNVc}r-T
z;xofNf42Pjo3yoSs@@)!hsj=(cfOX8m~qgHZwBKAVf%jv&mH&KtD)GUd;LhIff1v1
z;4R-%EZZBV%?O+Fc=L(OoM@AUt2sTKrWoj6d_1%FwBhvYdiEPXZRjZ5wjevjQr^OD
zd(_&viSHr}{7e~VdtY99agz^^Qe&ru?D7P!@-r55%Yv#RP0m*|yjY`tDt!~nIg{s^
z%7uR&Y8&^hc7D}%?X%U*mRnmU==jyzWSlCgzbdly;z1Rb{<lKUd$@$VzV&vzj9FfM
z{haI>$w{xL++J82B3$}+?V8>V0kaS7pHQ%squz3XLRv3VjI{q<&OLh=MV_`DKl~@n
zYWKO!-3yM}{gd*|ag(gvk-akT<qd&mn+h@J604&rJsH&s_i`Ji3hy}hf8L|*47cOC
z<i%&%Pbdo8d-~b)O<(m*){EXc^fXU(c2(J$Yg{wcr)gA1N-r0X6U&vGWnCYU(kXQR
zPI}}d%i`>x-J#hVS6k~oT>kXW#yQ4<<#pyRQyZr|bDw3Nac^tJ+xmBZb21)%ofx?I
zTmQRx1%`@h4^@|BAKLQ!%Sns$Hk0C#TQ6s&X8UmN`tf@EZteGLw2uUw68aZm$KLY!
z&YhnJ?)#nNnzr5T+AY;3?njS??l}FnJmF1ClJABO{9<|>2lHDk#FP&%D)?%p*RFZ=
zNjeLkB$w~OtdbXUZl@x}FDqT*D!Tf-Z|<GU?e#AY<m(+?|NgG?-m-Tgb8hS`3%1|E
z?Q?LGzsJI-a|5_Lni$n)RV=Vq6@0Yqlhv;k;rgm`@6G%DcAM|w<=dyr71sRx^Q`<d
zuj%~SO^>6OK0YmNKkel14?dMobS6I2zpTUHz`4{hK4YcI0dc8o4{R7T>bdK0ep`|A
z_tB9xOKroev)_u_MLnOqEi<$|{@nh|y=A95AAg+x{`!*pr#<{<vwwXwsgTRV!7_hG
zf-8UPcD<<Ue#;+M-1d}S#=YU!5(S(4M-H-nN{%@Ef9{vY_3IRO`t6i0Q-8N>OZMYS
zEuzz}o9s6If2}c^{Y7tAU2pfdQy;w#`9$r$y5qLMvA;r2_bR8)wwVCxk{>dhv}V13
zjPia}`G{k4xc4mCW%r`)$J#I4h8rf{c`yC4=6&9i^Mdj7=JRQO=nj6MxudP9rP!_G
z(DZ6;`4gX0ow*Fd9LsD>9;Qy2Cbel=xW~tLN}L-m#^#s3%lnbPUOdgl$mQ}{Q6`qc
znSY9#^O#>pUdwqRpZaj?<<gB)zW%wzaLt`#?&NyA52s%mK0iCv&PjKNPt-Hrn7_T=
z55li=RqE>)x<8yB=yYoN@xI_fsZ;s-^)}~p^CIu~pVzmE|L})rb(x|}aQL~535Kzo
zVrHqwYX+abt}$&_c!~YXUHo-VpQqZY<i(s*_&WWS!yfL>t2<v#6gA5FV!B$USoDa_
zyvv^*cifqnw_JdkC(2rVkN%ggQ>Qdj*k;*m01Zf_HQHE7h^aYv-B50A>sR=8NHsa;
z2-DUryZJAR3EML$Oez!CP-)@X#jUg9-t?chKbKVBZ~k8Xbo$Rnet$|QOLTQ5{(M=v
zu5AD6^~>%SSc-4`9#+{f`&z!-u6r@3o>nTHpU!*G?;nFhW2Zvxk+6&d9B-FLng`1>
zPx<me{OZ5ATOJogPJJ#sRa<nbv1tMC{(|t89+?{|cV2yXrqrM5;8%8LJ07K%*>j9*
z3!jPC%iOn)jjccZ?1SO8`1rU9o4m{HEn8-5^gemH(_fMGU0&t9xWuJ$^A62Pbd1jX
zA$~GFO}{bwgSP&UyINOfZ8k~Obo-fm<-=)b|MItdXI;-$y>)tP^7hP%;6QHW*LgGd
zEd4+2;_SY>?wguz>$QI!U63eHFWusx@kf+fo=uH0)xl!_*%x^)&P~#Id|hsPq{nTp
zlg297HMcvjGWi<!YIVAC@VUL52Q)4C|2z`8_F}^UF>bAgHg}?Tf<t-lPMt*u+b8s@
zEc^fR&*8X}`wC~z-*W2rv%;Jo2iNG!m)qEPvwnS%t@z4Vs_SpvuU8vROTT}=*Z$4F
z!X4c6Z6~llQ~4+GX!{`%zOO$THS3yNG!Fh0-L!)x@0{_x)V<b!Y7d_Oq<#G7{p^}v
z^ZZ|W`+Jph%?|&1(75}@<_V_tXF@%h8?OkY2^P##Wli5IzwU3}&%aN$3;LbwKGX3^
z>}n5d(ho(}6RTCMI_JOH_Gn_#lryK+cE8q|p1Ml?RQ9RXJMRmQFYH^rx4iP~=UQz!
zoqGzWjtg?Nu8}i{j#%u_&yl_E(w`^uZ?F03_E~2C^^(~K`KFrAe;BzZdgG3>_x*}L
zU#Jx{Vt--T);Uc-X@du|T}^N4wg-!w9x8KtJzD<5QMYcJK*veeLf7EQ+YUW(dYHhf
z*ED-h>9bd7I9F~}=}PuL*TXEJ`A%n_>Pw#r&Z1Lm{Z91XXSu0<UCZkOx8{v*F`4qB
zWvOaQ{FqLx*pn~&t7dwJ_QgowrueYE4CY6#2LH1tG5ubb_V>!e8@3&<m6x_0RbW$@
zUhVm-$vN6^%@cKoxzWd#^xm7tuUfEou~(ba!|T5S=SA#{*uCL__2a-3%t1af-&arA
zwZgS=>%)r*Jd<;F<xI?RoRmLFMN>b?VTWGK{;cz>=7-DPy<B|$mhGI4mVZTF|L%_q
zd%@at?yzp6)$7hrGr!z9u;)-|BzVA5`3O_zvwqI?OMad<;r6Mx<bN;3j;XGYi&xdK
z@gQ?|c**UEfX=wK7w7H!Cg|$hr!3s*xLsY(YS;U>nd`%r@At@J)4b^)UoJYm^3$&q
zEy>dHHkEPTeujiu&Ui8Hz&6Hbl5_Uk%#!|)xY;Jn_rtC|6|-ABIoB>a+CTZx+=rXl
z(;lSupU&6cza!wQ(_-$k4<a?5Z+hW1U;FqAsX{K>@W@Mi(^3_=(>8jXm=<iZ-JmX@
zJGjNeq}AH(g60-c3*M@TBDGzu1=Ig;o*?I2muX&S_Gfa(1L=wBr}D4-Uy}HB`uzp0
z@!>x=ojR(uO;deRx@!56UEI6kqAu~jT(g*|SUaQbMbtt4B@){t+N9T}2B&Oa8nmmX
z?x*~Vh<%gm{-@2<kIQ}VmcjdiulKgH`!gLrH^%LKS=#)MLy?JNI`jLlov!MS{f{|j
zTF3wWb-3~GVOjUPe{JfkdL{kSZurf$+WYfO<OS}E*NV^AKmQ(b^Rc|(uj+$qpMK$a
zcwe}6`35^9!;7!^wbj?2I{t$D^~w*1N+u5u9}vv;+Bl(4-s@2qXW&HU!a_sS@=|k?
z1vA2&*o!>hZH^JHwYk1Ss*9_ue4W;Nv#;s(P6zI!@0@)y=+cBYisAWkT@!B34Sw&d
zt)C)wF-273=eJ|)TTIk*f6rp`eR^(cW>UHJ_tSIoZqBx>KJ{wn-6C^utCiWYyVu4f
z%l_K4u3~Db)=yb4V>_(^<FtuK!(O&9+%e%(y419Eha1DJSrT98X0PQeJn>@7l(tE0
zrrfT)IeT5H@5gDibN4t@$nSi4)nneg-(SAXdLw#0Z5v1Eyf+y^-O0N)Z4HhV^5OW%
zR$6e)mG`9X1mEqm6kpA7|J1Q>@zqG_&KFnXk3IbxvQ0Hbey3;s+DY?TMAB@X^`}eN
z6lz{h?M{jDzihrl-+O-biie696+hmcvs<owbxSGNZn3frZW$Yrjq9hsJhkc(LtR39
zZg#{c-8<*LclUkX{jcaim4M<C8~g2%+$`(3n{)1%t&5wxb@QHOW$kJK!YS9it!`@R
z-gm3<HtJo+y=4E4>XK9JYi2#OdQ+^o<f)pFyVJLcv-7u2oY>m>{#d?Yq}Q60^93pw
zWM*gnR>|{TaO=R%A0gF0Ba=3rUgc%aUS1V?d7G={?qj)DS0|{)^u?K$T;851xnEUg
zW%RyJ3AZ=0On3e+KAm4PKYeBUQ@?wKsp+rhYVS@dnRy{K_vg-c4^z#S<c8*-StmAC
zy3+R2`kRN-LT7*FKAC%GNA2bGC!9MEpLJGJYMF5D^6j#`>5G2aq=%Kwox?t}Iq3Dl
zYemNDe?ENm+Shn*+dbc<#oT=wTF%}JuAe_`6F=cKtC!OCo%K;0KHg5AS+fUh2ix^|
zoT6rxSC<zvooER47JPbT$sgAT3|A&ipK|xlSBHWt%ytR)>pNPwu0QzVc3N{`O~jXj
zeEjEAWCBG)(w=Hp@B0|TQ>h@!UeFfxK_W5l!rsHWEpZNefBJuZF1@|+>ZU0ZJX4t0
z*v}Dpm6*2U<r^>A89!Zivz;ou_bedNjO~7F)PkBAW%I0JWA&wc{+?cuzxDb#874_h
z?c?p;cFQ~8=NgyIBgsC={i!m(-zM=)@h^OpdHdzvTO9HRKT@P(>}>sa%rv-OyzF_w
z{ipk`*A%}y_5R-O->cr-`((RH<dE~V-TuODQ>#|mg+;rc=t^fYU-|uFe)3e&P`CGa
zJGC>y=l*r)x3@1WI`H+plf5*b{N#t(hqT-IXB(y+dnH+#^W3VW_#NAwhMP>AucVaD
zoo%UkH6^7``o+x?Us=ywU1j+<<R8bs2@LZ}R^40ay1eg|LFp8$OW$T3R(am***xi$
zntAc;mvYbTa#Jn~S}KR=AJIRzFF@`_?Sk)(fwxbVZJOJo9z6Aa#fyE{!mjPRq@K0z
z`8S`)shiA}D)%U#(VKPq&MbqcH!r-&viKt(r=F;G*yr&xn~CgE-aGz^yxs62?(p)a
z%g$F-tn>a|eyzAy`cdHH?VFFhHD|1kU$uKL^Ykx_h7s>hG+xxddY9`nPogVVQft<8
zg=OFwK6j@RiAK!sSD&ZId3S!@y;gg>ci!PU^C~Vsy0lt(`Yfx+)t9ujjD#mWl9S|<
zZu)B6azZ~dC`C=>XzBUf6&tVRwS_V6XxS~W#Y1q-i?tuNS!&MIFH}*P{$$z7^)KEe
zMR$jv3_9sD?a8F(*)0Nw+s}5dic;-<!&u_PDY@ME{W8BG7V9qs_X1u_cw?Mea%buv
z+kWdwvDaDzj&8oA*5$TSRAhS5dXD${Qzl4HtF|iVteu|iT$<!-Hu>rk{SORHVv)U@
z|D6_TxvtRr<coo6U)n+6n(Y~Fw?B6M2>;k`ZhHSqmN2JQ`SRa$m(8^*k-C{F!M8i0
zBlP~AGoCBeau<9SS@rMsuj$6d8lJ{?ogN;08S{2U?u5^0-~Ev}%M@w<(|BjFo#OTz
zcNQ$Pbhz|Jc9y)!YfpWr4s%|;%m=R9h3jn`>ipDhwVKqFi!48Bo&CzhY~q<Ib~-br
zlr5QXo@b6zqkTr=r!6=BO<dlz{r7oWdySQk<hN><@^^97Tuk2fWM;}Hl{0(0%lBp9
zSDz|p5j#)8v*XvgeePFxHmbZ}_Dh(-r_H}qUP}D3wwl?o($!a|UC>>3%WYPxRPCqP
zPbO;~PZrs@Z{zkW?pLl~F`wEyg?sA#AJ;yVel4B8cJkUOUzHZ06i+EXa^<z<^|sWC
zls#uF1XcLFxU$NB)U$tk``t^=BXfdLapYvtf2ZsJrX8OnXPn+R=kfCH*Hd3#J;^<(
zX7a4l*G`6=s`|eAPJZ(K<_C)o%Ih*J-1XOd{kG0(_8t3|lT+R$FFt)$FMGGkwij3b
z#hlO0-?q1)E=Ks1icSB2Q$=sH*l)c#j?-7i&MKF@u{vG<QnpUclW4X4m-|!qWi8Fx
zo|XQm|Kt4Q`kyBLs`wG|vG4o6XM3MCJ#L*QZq<D9)Qw%q*?n{G{qgvG;o0H;m%0}$
zd3MBuL($oqcdwa9m7z`5zqR|eFUy^|Yw6Evku4LR8LVkJ6K7ie_RY=da&b!2IchW}
z$)9|lGSS=hx~klB_w~|Oiz<9yI)DFAelF2Oqvhr&_ZPNT5BEk(zrFSIbxp%>y@Tui
z+W39le_4O!cazM9l9?g9Joo+lvn}}V`!By2mhW%a8G7g>+XS<#+#D-qF73FtYFGB_
ze>?X6`X+b#x0_t&9%)OaDyb^#FP<6whQ;n}{0|R2b-M5BJ#EhRvw>?S^mUpa>wY9}
z%~w9>g@t*6OTz5~3s#5*sxMZWTr>HW+EdfBS*tS4s$NWS{QW!6;@^jmE0(oKPD+0?
z-#6|26w@iaeQ}bu3cHqXUw%Fy|M$do%5Hr*oU3xHjy`bvdA`!fDJ=M(OWtkO#om|o
z{#s4!TcVs;+rki<ANV)!YdCwh+-K|QBITC1zJ4>=WWK<CuIrX1U2&5eGc-Dbw|H+U
zEwS3`mA-i95{Y#Ro32_^aqfM0@Ks~D(2MH#&b5nvKgsRAEwn2&?dng)+0155bqXb=
zFD74@@!;mcyG^?r+a4Zx$2^(ujQ@Ybzn{PC><gAvw>tNB@4D|x?j4uv&HCcI*uO4s
zS9r12#jNEIkI#v|`~90J?_-Bc%U;>9y*uY!`rH4#|F>58mFi!+TL1s^qviMY)h-_K
z`qFvJxHj~cfAX)MlTyZKJ6R?_$gB8!|MKS_x4c>IW&e1iQtfh0<#6rl+alX}>yH09
z^~>^?p#39v5A~vHYkq%x*3Vogvubi`Nqya!wY~Qi+?2Veen@*!dC1-so`LbIKh-=Z
z34MIS^v~ganaxeZ9C!1#8gFCY${l~$CoKECUhCD}o!^%ovz30E?egu-&CTED{QGs_
zgR1%Y?&qD~CG%8m`tMJfAZlQjP<`NDptzf^MCq~3GoSmM4@>B-Px#(Ajs4m3`k(*y
z{#g6}Po4djGQN4zcXTT(bTf}-8>xTY<2vX5wI$~@9`XILahBT0uVNA?x<<)FW6Ad;
z-{a(`-m^aOPQkjf`EAirMpp6pEl)dqj_+N_AMs4&sYjRRhl#!tn#FZDIbsf()@BGD
zko|b#O?#xf!44A{GnUu<!UtKrJ~v&RwR$J>N5zo$+dZ@2rpcTXN-=)K&K7vgXIWHD
zR^#Fdmep>5rtI6ExaX?wtwNUHGnnsP-+E^Ab+_cFJ9a)0(x~qF=$clq!P~4-7hkH-
zF{RV8`TeKWVUe#gbz~WDO0O=v!S>?Ihr)+b4{J9`$zG96_~3W@>4(*7nOB)}yF2eL
z{P8&T(B>cBBFi6#UHo(K<+aMTguhBZSpPl9j!|B6*z|;T=8MkiuI-F#q83f{{SkT6
ztaIv<1t)g|Y!rzo+f~w_74vxYg7&-T<xjt!_V>Zh=LNZU9pBWg*ybzRnP0Z<&BkS3
zZd1N2Sod-@=cQ$=*E}b$iMyWl_~D|sn#0eI-AsS<ck#B+_6cs5?D6dOZ<((v{7<-f
zerw*fUCFCO{@ebm`^UFe`fJ>c-?P76y~ScS-C@^vo%MZjW)`uJ7j<5#cqG>-qIFO`
zG2ze@24~Uo`B(Jz#fq2YN=%WcP%Jp;ab57Ha<tRAx4hf7-@nJsaD2|@Vx{MI@`CrC
z70)gEU-9|)H+w$T7jDrD9l0#b961A|rd)WMC7vvPcqRMtu(_}18mor>%yIp->51q1
z6h_B?oQ#az&Ej>tHb_6YS)2O(%iZ|H)f27VH-{@%zWA0A;xjv{JZtY7QDej8MW<b^
zpSU=YS?Ra^n=cwh*UEMs{}O#!`+!*)IJvnosu${u^~HYK|9-jS&)Ku`ckS1dtYVC5
z`oyfxvW=JL*P`p<^J^~(rp!E9sJThx=7nPCY@X8QxRhIIWuEUjR(VFv><D&nVqV4_
zU^FGb?{(qf(#QH+%YE+cJ*;|iX`eTfU!$qwa<_LZcV|_de-*c8w_I4!ONUI|oBS8P
zOD3n~%-bj!CRDJT+0H&sV)nNeyH;(|3JH~db?MH9zDsrW3vYqS^ZKmYyNloZ8aVsy
zwV7I7WXupeL$v%z`KQG>hYuZ|bJ!?Rho_i_{pi68nWg+GiAIwP@@E|Gw$csLiM*fY
z`})+{;scyQ+9{dL3?A%~5zXeM<<~fRYXo;H7ukL|8?*b#omD$uZ((K7V+?cRT5-bn
zTFY-C^^?L?y+_|$|9$6tMqozr=KV8lQ@%y`%Jxow7Wn?;e81PCyle&r>pw&s{`zU>
z>HpjJn4a;h{XOU3O1HH?Hac%HUZWlme(C$k?nz8nyRSaKdWSRDFnzK4(HFJ9y6;OY
zP5#vrY4A*F&*m@dTJC4Bo4mK@=MwHb`z!xG+&Mh2KW}B3C0}6-+XeNT-j_npJ!GHp
zv-uh?pZuJnniHL^%heh-^1h4BSbSmC!oLg3>w5pxfBBzxw`;5KHM{mt3snvHck{hn
z^kUch(>vqq{yohU*wk=W_sNkxzv_N2x-Wk1P(I`7zkQ}|UlM0XG&@@TFTAhj*3Xg5
zFt0KD&3&icybncsL^CgBCUwueZ@s+gTIC+ghihWgQcfSO@6y}x>B4MD2H!v46(_Ui
z>{*y3&085M*8a`S(<7~P4a@)Fw3BLc?w@pf_Vv}$l<gI%jt1&ykFGaw(l_7{=sNMO
z>sVl5XN1E7z2(`0OtTrTeU47tByw<qd&g1E?hTjb%>R(dvgIo$@23}L$(CH(j+#ur
zy+w*?|FJ)Mjvszb-0S|`xly>S=f|w#3AYyRToK7{xlo43!BAt#_S~q>w!3^U4tVY;
zSIev1BeqLtb+Cs0;}4(Z-|bSjt@-ixXlFf3d~U^pLwh$*uUfbFnZfCozt?#2TgBa)
zxLxOK-m#l+LTAeHpPH{Zf0KR9g@x9BUwMDtv<r-ly|qiGX5Yi6*9)`C&&|F0|8SFd
zexCgQx9!g_md}%0KiBrrP5$raH|so4;GBGY^4iI@PuCQ$d-;a3mOawq#A*fEPp9Kg
zH$OSQb@7(RTU#R*ILSp?u8g=9b<4#pN~^hup{zlMyMgb($|L^5)hgev9?R$q`{nc}
z=|J~4ozorx+YT+el<mHJqO5h3%!=#x;ucr;&Rsq;y53KRh3~WF)B1H;4cXS)^Y8XK
z>2BPbwC&iwrYMCehEv{D-<V!uo_%QrXQora;;$2byr|__D0eW>wCd`M$mHbSZw&nj
zkK0pazdt%N<MO}dP7B3fHNMJx^>e-Tp0Y~YOKN96ME*0oUu37gcUt2i!6@!izkbbi
zp1QD4|Ng$Ns;y5;F3p@^yzrIX%h{LJS4FS9zOsDdJ#LT6b#?E4-LZXm`^e(X$_etn
zW|(MHdVczlepu-8BRz%*Z-ZBtU%kg2|9_&<*U;|fZvJ`J72D_ge&67i%UE|%k*RT~
zqc4-C0Nb@K2bNy%H{#x8scm)D?5fh$H);K|Vl&Gas|0T>y!P&Zy4_AYIlItR?^n*f
zUQo9;R&lj+`iVcOF3S$jj(>3OfV1OG#jW;&?E6~Nt=GN(P<;EJK_3Hm(f-Tq5m#IO
zEZD@(mo1jDKI{6txQb67IK$5*&QPylJ^Z?}Q%dA0^TPWsRbL<5sjZ9<V9<+vEzR!i
zsOx<)Eb_f|jI;i(XQgM~&tiO4@#6*4;r=JFn)XXu8X^{cTzxE6YC5~zp`IqAM7tN;
zUcSpxUw!(vebJX?;T09~A2ZsjYHn@JFP|`ptFZXU(%6i`g`Zki88<)w5?#TiW2C&V
zdxmV>zWbl{-TkdqD3dxv{X!bU<qya9u9_(_;fmn*b#JGbO}~AmWzOBQ@|m|(#V#!@
z513W|vq6xN*C8wV?9Q^1!yEJV@31e7W?_)pFMd*VQk=?l%d4-p{5n(fCvJ)9NsIZf
zTkl?v__m|9$H>WL!ON#VIu#Vkn43g$CU$x+zPs$DROU}rnMlv~`WMXjlA0FXsoMUQ
z`L}`8+gTPt2@{Tg2z+{O-cOgS7gox#P293;PJAPu>VZcdEfdaFEviX#>-|%3-mdAB
zu+zPjCmu>Qq(#j$ySP~9Tr+z!UlhyNTWbZUmj7_A*lV)$+r?`I-3zisirReNefyKo
zb#KOr6%)L<c;jDq`ba-`d~z-4mHX$U6_$v95IU80O8)fIg{z(CxBO$g-(72KH}}q3
zYd%*0--XHd9d7B&eH!a+{cA?SG1oW$s^t6@GFx(TPHf)ka+-VRKII=AT!puIIW8%x
zmwit>sq;{8Zr&!A#Vh8>1S<ae`IzTIQBuslyCti=9y#wSyZLcv;?y^<dp2BL-nK7q
zcDb>--S)z`8O6G(-n;d=UWx6|*wOddeC?a!Bk8`!qMnL;&#s(km4Ci++u^mhpMP!I
zUBS0O=f;b&BC}}j;N4O07sjr%aJ6*Z;WHup;(|Nh_9eBJ^ccn7bh>%5bmncj?TyWC
z{81JeVk~nHKE2>=f9B_$lRxu$&KAy8ux|Xh?QU{)|Jz!}M@x6EeY@c8%9s^Z>#EF)
zHDU$A7hhRXl9N#!_ed>YBPwIF^FOisQ}0hxKehVNl&<4zW=o_fGzQNN><#+b7#AY(
ztn|Qc)-4n2xTgHLzxHR|iSolq*2gN2^@Nud?|F8GTg741{WyPLspI>9rrtSKa8{r(
zesjsfH#6Qe9M(80cJjf2S&eaxRTD%l-v!78hO`x2FbKLed)KX79v68dcbCtObUBcE
z@uQb^%=Gy&b_@%&zAz;yJyfs1$n@y-v=#O%ZuW%p|6-1~6Mg5fMv^FV_LQVHm0b_y
z{Mcu32l>oU*W38+vhA9WHi@jy4=v`rV7a0uVE+Qa8$r{`#23$XvIzY#-Aejw#e~R9
zle{~oaK=41V2<1+c<nde*2969{J4&vJ+(NYgQ?t6gf(XA)962s*!FyrTK@d&<tc`e
z)8Cn$`}DEE*E`O)tScgO%Ng$JZ1WR8tTfQb3AnVtNa1ulKj)m_d;W4-j~7U+QU9Qs
zy*qN^GK*ucSk3cSmQ2|5=-$cQCug1T4U4MKpTaKdzOp9q*@0@tpt9xOrwm+`PS^4_
z*WJ+n;C1e1vHgVSr`r+=CA=;9b})Hs7h0{GZl~R6x@=1w4@=XX8AX$YG*4x2Ijwce
zWl^WO!Q8JdKhMOl=a=9A`^A|-yEP)+iBn+4-@gyv)xVTDuF$f;w0FtPy5$>BNjs>p
ztBOZ2mA>{kzG3Qw`yX8Q8UJMQYKXhge8lTxZpz)&rvj50H+n5-muFCAiDlnk6wvr^
z;nBCpU7XFM)_I=JaXH@W*jDFM#rR6}N14p$geR93f1l#hnIH9Z-UN;m9r^T})h^mQ
zd@EdQy>C8y)>l&J{XC`7?AoSmrK2({^e>dNZr~`-RF|8OamlNALdK!Z3YzJyRS)+s
z+yBxqVJV~PGoH66{XR?f^QkYtv*N5zNeN%MWoGdA%xiDIRtjCreLeT;rcL5vHl`&9
z_ghMT>c9QwW9ho^+xmNg{`>v5-@7_8Ue74S+3D|<cP+&aCbVlY9c}(NXURi5kCzI|
z7M#qP`L4#`K8x3-uA*J#23898-utKR`E)XfK{%nE{entLVeM@;>DAjf3-~2ix{`II
z7wkQ^Q)pAI+m9Am`}#*8I%OApVvE+>V9>pK+K!h~-JN!`tY`VZpvduVb5*_7hUwd;
zhlQRlT=vSY>{h^;$=|2E_KjL6oVp}l^t^Dn>77N}QxtAI)6V%)^eK1RzPk%|B?lbN
zTKn!>B$onP6=R?EjMY0@?Aea}`E-C=LFB|br$hD|x&8%J`v<!&zPPO*^3A?kPiq(!
zhdqsZdUk4zquA4DuUC~`HQReD$$j?rr>81ds(9G^vrhbfz0G!$tx>zG*WAk;+h*q0
z<!T=-WDxpw)ZBjtx3uMo)NG%1n>GZgW=Q85AE>Gc=*Zi(t$CZYhV#PKf>~0z$AmvC
zRn>km?eP6w^X~KJ^ZQ#jGIMH#N0q*<)HN4-aE<wsR>0Zjo}E#R_1610FED7`I{jb3
z@z0j)s;7KiD*up~;mPGc2dDakS#G)Q6yr0w{J;0y-ZYy(MSmC#c7c~0ua1x8Q+2Aj
zA*;Si_v?j2Utiu$=SlZFJf}U|qdfPC)Y~lf`RpI9g^V7nHz_h|EOylOo1XL8<Im<q
zS--@{t*gGLE?H&SogS!K)R6Ph?%~`Qvdq&i_xARdnZ9b6`{c!3R*r6)2a~tH;hDZZ
zc4z6!KL@&t^^ZCH(3*TNx6Cr`8n2y5;y%%&Gau&K9ayFIfzfWlgS`zi7tdP775w)4
zzAKU?%U@nxUallzdP9H5)<=yt@n$Ry3f?sV97PMJJdoiJ{3A2%Ptb|jymt%vpE=#B
z?MO%}?OFV+^s&=EvGX0fYES35%P+Xzt<}52>_V%6Q{AKWr=R>db$QipUcaT9K|;x=
z8`~}FcPJ?^%u-;yTJv~9`2M|D1>ItK+9yfzuK9lDYps=edH$Kj=Eb76@!_gXoKxPE
zuCoYTx9fi4D(CFw+tzr@nepk+=eo=OTh)z}Vq-2G``0jcZ>P?axgwwCESusP>PijX
zx-?DgY1yRT`t-K#x!Cu4ms1iZFZE4p*IBnd&tT`FFQ4X~a{YAJBIs<F#nq>CG8foB
zh-v!s#^PUqpTUCzOV%$7-#e>0wd+jfqiOG#i%;}<l>TaO-`pslIXMB+A7a;S$!Zbc
zXXxHk_i0)EL(Znz5t;wu)_Fd;eBiG8{o4h%Uro#PZ|_|;LGZWu<+%A3GL8Im>h%^m
zF<BZ~IA6~W3Qs?Oa;w>ncjw;bS3ItbyOw(U>Y3Ec)o*`S#N<!1`jK?>>byA9wv^*p
zde8co*HoVta+-9aV!HFL=vxbZdn=XS-}V0M=Ib98EZ$bUYS;CDAvYr}BLgkl6>i5r
zN#(gT_k!EPQ0LnXR-u+LI|D4ab@UI49y5!1xaNdPEQeyOW8k_?#W~AgIdZIYDQgz-
z6<vO5Wmd5uSAOWtsHbg@<&Mvn?O!LguWOCcX7h7#R@L7gtZX)7TOYYS=kEH~b=`Xm
z)33bw7`1d+mz%JVu#x?j)|~Bs^WRw)m&|;)GCcB}fAe;pTP7!xC)a%bnLB&`ug9xb
z#{XTocV+JSxi`&Lyxp{oai3h=WZPZaw`CUZKHI(a@wS${l@f0y_G<0g{%T&b`*H4l
z^FXa&K_{un>ldV44Vmg;d3@bon+sK9OB!Na<Q{yuv{HJd?>+gA43~U&n^Z0+ovrw}
zJw{gI_vOu{@$0LNSO1!Rn(d5g%O15Qk#p11XUlHO%v3M(3}(}goq9Q>tnY2e)<*9;
zw@g)6lou5L-^92od(!(q`)_}_Yw+FT`@eUKWjmD%RaSAX+O}$eNpjzb8EK#99op@A
z!mjAD=D}$dSKq(As=n*IdiyomH)RH&FRXBW%>P#L?8(|mcW$pfzrD84?S8e4Y`w`h
zuQNAhTx@>N^DlA3;^n7|j}||#F{;1x=cl>Wot&y-m(6R>-ZHfnc2sIQJiqqq(=GnX
z?w9^P`d5AXV!P=zOto9)%vs5**m7W2bGi|O)TOR9%DZ!_EvA&_&D-lGJk{vT-K(+h
z?c|@g=>C7vkvxCypQm5tHG|E!7Sz1maqFYk{XKU3zeI8;uX?jGXW^CAQ7fH7x90Yg
zEskCGeeSL0`SUZQ)$%{ydi4Kko4#v&SLdWXYMP4o+gsYNX&STsS(*F2^Pc3r1wVNV
zp8o4N{%X0u^Sd^SwwgTa>xpZMt0Vqqoc7+s#L#|U`&8+B2G#%b4zcRqQd*^d?%d8V
z;i?vIH>7SdIy`I7RDq!0XT}mUd2G$E7;HJy_eJ(_S&n7t`zJppSD#qkGGSZ7>DQL`
zHaDiPKISdmpR-4OPRry1qrca_#4nd`xOC^$j-9)0C&ZU<l)svGJ<LAZ{_@5-$$gK_
zispP#o&B;@BHNPdSJ&}dY`gPIS3flC6!<jd&E%h_`%kJBFTUUWMtylf_RhHXXBFL;
z=CfZ~yy>|S=gz(~z9|1UpXV^2(wC{qdokVlu37D_m;aV+TYloCzh4vUhu!y7LnNMt
z#y?&7D{!kvc;s!z-Dg*}=c-@yeV~5j(F=vi&-Q*@^H%cNO|B(B+T~<oN;jB)k5ifC
zos#^1W$mF~Z??~UE$|?x`S`4(jJ2lCudd3RG)oG-QPp=|^U1Xu#$Wz<rq2Es8Kz7P
z=9np?uygOJZ&PolCmo&eG{xR};qAry1fB<Z)CKfKyI!%E&eC}yx?%aYH_Q&x90fkz
zEvuOJcj76Dj7yDO4VTO_<g4{g-%OZ)^`&PWf4kT`wc`et8ieK)ElJPal)`9{erlq*
zO6u8zE01MGODEj+5^F5llWJyp_w4bl!I6)w+$!FyT$BE;Yd=5wVpT`Z@z*c+TKj0c
z-h1M8Xsb=li4@C6tJm4}e+x*Cb^d9xr+LQo2eTNA*!)EIX{#8fm`6Ohz%KOZcGaa6
zrg90dM4h87KLxyVd6&$rslA!6d)jp#*Uf#`CRnbxJjJN&7@O>@t!XD&KZe`A<zZRd
zRB-=d^`h4<>JJvUN_Zvx_U1A9SgPy0c16_0gx80Eb)|ZWW!iB+WD_{Ud%r!-**5ND
zdq>lk=fAu^hq>JRR>1f7Z}IBfpL)FKv}Ku@kKec;RJd#Ff$hh)=GB&OT(|1g>SF)@
z>RG|ot3DRIFs(OGIG33)^W8KvkCLzwml}>kuM7N=^XDp^K5rEG#T?W_V|{7yg<n>V
zwZ`BrzZLtG@6AFB^e)_PaN$~_@W8z$<-)}Lei5PP2hZR7zjnWDM2X``i!U25D_k|$
z-CpsG;mQH_iboHuCkSts$ui_T^||$R#`TR4(paV~u*l&1qIC4fv;HS@H&{LN;4AsL
zBUgr9Z?EixE6#@J5?epVZkWUN&w0)X^GwgDaa)@&aVfbhUSo0obz1c7t=UoXZ(IuH
z9S?DpJ`8XynO*q)z*DB<&zpB!cnF_053_owZz0jR`NRvk#ch37z1auZbq}36(m1Uk
z`(@>!O>>&fVzyfTKf5ve!?nZdLi@wo?Gyx;G|W%?I`8WCTbqr~+dc|akYM6f+1K_*
zG^%pnr$G6+)hWLhW<9umhHG2vPWSygAFN?Bo*8HS&^Kg%=<(H`LsCQRw4`_Gotv6l
z`mE(j)GD>qZM%!xzdtfey6RNS+`i^e8e0k1YfYQWFBl@!^0l|f9A!Q3|FC>=sMK4b
zcK$;T<i)GD3hp~3u78n<`=S5LBS%Bnw1cO3C0&fXR=?Dwzd`yf`%?MfCsQ|;&-_;M
z_i%1Om;XDrZ1>qB>ogC4y8JmN-kkr9<ex`h7kMeKGG%{iF?0RKYhoMY8agvq=dR<+
zTW>kJ_<2=!<-BuxWiu=GI8IS)w`qHow77_I#nH{yQ+f0L>|WA-#4D&HXzD+y1qW3p
zysFo#PTgF`ck5HK+THlc_8-$^T9lab3r{eI&0Zk3zA${-@^$O~`c@~!pI{5)@ccYI
zQ*4R8m~LR~=abX76n+aADC({Ge5+Yt6W1YE!6e(je|pxxqL!wXez|V;x9HI+#+Wv@
z>AN46nSO9u{x7!u8sD0wD{kE_`x~EYwNY2_?Zy*Uo28=;S*bH?7vI;m3t9h+<B_!L
z(IcOj&#=p6UUHe%`Cv!N>?4szK4w1mLiUNV>oT!Vk(qrbdZ+T7(sQPw#~pq$T@sI8
zv|{Cl6N)=kzQ-CY?p5jPNo!?4^md`@EE5H@X6cRyd5InCuFDpwRwl18nfYC6dDvw$
z>zk!Bz9b%8V!}DC=|Z%~vlPqx`3v{ePTt>9SEYVCZfb$vvp3!9U7Nc6W#zThHMN-M
zzbt$>?Ip|gi*a8sFDjZPp}xfRsetmwH!pXd7rvDyRhqA-n9}?<=Jw1*vs-RV`u^N#
zu~BLD+G$d+Z`EqwpJ6s<LX&6aoH-1yHc4ibuRCS?ayzT^efEhjCxG@>G)xMR*miT$
zmdSG_O}QSrxvT8fgIkwN=f7AMJIm%yRda5W^NQlXTe2)|dxU)$N=%>ka7o+r-+U*0
z^?2N|9drGPW1W4sdumAb&ni`YeREOcrS?yr*TN0YD{fk>U8uV5`iTms6K+*HMbE@q
z_TT=q;Yv&OH}3F9UL1<Xp4y-d7+UE8X%(;9xVU5Ho_4!^cPiW7ex}{cbCipBmgaX|
zdhx;f$(K3NiIcB?nDgMwhb!$Wy%W9{8ReCA8m;kt)AsVkya#npB-YHCVs_^n_ZF!o
zqAbg1_AGuYzHPCq`9g7pi{=kSAE}$iaX;zny0`B4ouIqlrtE+F^H_1AU%|5lS9n=Y
zrta9aDpGLyE`iI_1)S#WJh4yU>jk|Cxrk-+)^0uQ{JckIl4sD)bZ*X!tBThwx21Kt
ze_HcIyw|k%w25B6_4ULppWXg^yXe@RVJl=*vUB3Zx$Dlom{Hg}=bXjTEyW?4J9jC_
zuKOCl>qf?$6D-Md6qRyM#)tFgwa$ON;r!3oSbpQfzTs@=tPDWynDdG=AFkwo=^s!b
zJolN_VNsuCxp%pHr%0~5;c&@A!LaqZf?@2VJzEa3-QKxvVJmC@VtdELFU8N&*k<j!
zKEbK^bh+^b#VaLKKK^!H|Jjd!>SPgVoBqnSs=Ot!X^Q-H)AV>>HGOQY<apiuZtc||
z^$UR_MFkB0GB1?Ye3%w|@kq^wDJy4g=vE7Qoaz6jbFV8elO$(C?(D!>(?70_>EJF@
zd@{pZT>R4lXVvLhXXozUWS?;S;c*@(4#hCur@W%K_kb49R_;9B#iwUE3A9>xgOqyy
ziWw`_i!LQae{-mHe7r7<o%_(|#XD^73pm}1?wxENer2LSQmD>iM-D|6ot9U(CtN=3
zIk&a)8!w+wj{3F<Hp<Q()<tu94hD#R?mg=75y-RQ->p?TH+|~VRm!*7nVGibsI&+i
z&Hb$}^Z&%!$y>E*85L9Zh#Ds;=RS1bFEmH!TjIebTOKmZtE&E5`0Db)c`Md!Tz9Z^
zZg%nY4O{kF+V))Duu0mzMEgn0m#nZOEA>uapY-%<-&J$9jasWG`W|UMYA)=xt#7IX
zr{Y{6@3f;`$&&mIr%$u*o^15>l>5%rr*lkKE;SPrc3RXiD@XUqECDBvZ^mbj{`q=G
z{IkRAkmc1|F0GpIvgh$^CJFE&mylagy*Wy9a~fF=3yDuYxjBTz?vLv{wQFCC|Eg}N
z|1d=$`%)LntZ4@h79Nv%tipbpL(#F&_ig_)%Z00^-LNm(^?mxicmHmEI&M?{$;8xS
z!M0saXG8^5ma3lp!ttnxVaH}OMVsRZc0Xi4-+sPbQ0YTq<t_oIT|uA1^H>Z6p7}Ul
zs9C(Qxl@qasN%rh!-ctq43GXxEdG6NYG+nrYwa>ee~+nZp|jSig?jKQwoGW)pVRJL
zWYi<fp{da7`8QIi>W|;Wn&iC;7koLA&nu97!Rh;I_qgSrxozqES6rUHF22Myz4YPC
zhSSf#e@K1lkkoEF??c7f_i6V{?kA_RPm7OBO}uq)j@+^D8y=sxJU{8h;nH`yy<5Mv
zUEbbzcM5-px}|I1!&?*oOlqE?#~dTRyV>`lN7b6fb{+Z8PM?ysQ%th9Eab~Q(|n)%
z8Kb<$H-mec1_zp=BA)sDh)rIbX|iDXefM8~7hU(+^JwEUp8X-S^AvM!+Im%e!hQ%T
zJ6f-td4AT7q*wkgZofzua@saKUSr?0d?odTHf~a4Wz(N7Q;>1rAD-wxWwwW;W$?}1
zNo`#-cl-^z^k?e(Z7ETIbyZk}=j#25JY%kAUwTdWsd2FX(w%$cGPj;B(0QKiv_|?w
z$Dge>aqG^moouSRmr-x>Qo9Fv$}^m_ZFgRu{p3r*t$@2Lt3JPCe!J*N<F)AA=T<#s
za;L3cPvd)aT6Y;wx6d45=TNB|4@F*x%gp9XTpMBBDByHLJfh1}zwd|XGIP-?%Vna!
zTI&k`S+ZYP#1?;Bq5olpQ|pFhOQ%YNJ!<U?E?l%(^~ZhRIqV|lA98!e3-1W+ZvW(&
zdH8j@$-np+i<eAQ2@q-#by%Y+dfre&v06!S(}@Gd2ZNoIHor>aU3Kl;#>Uz8PZ@UY
zuTa~N&~-Ac=UDgFr57hf?b&)kJ>lY|+N#3|j8Zzni_%5dDz~he^^SWl<K^fO#pV8q
z?-K8_a;^0*p3QdCq&{Osl!yKbf2L~@7Y_FCkPFP0iRI2+>h7HK>u%$21NF#!b8dml
za}GXrwk(NPn6W1|w{pSOmJjt&i??^i@jC`uXPkN9*c_X6ZpGFuX6f=X#h3YCi(wUC
z_dRS|%Y-WXtMj5)%f58mr(^O;mo?Y-PJ_UXg9YucO@CcDx_NHh_sqQkio3+GxSz50
z`@r~KP_Fz6_b*Kw?F5&M+PMP8Dly#8IOplE@lUF<U1;g~TY7_?u7drV4;$`Jy!+Ff
z+v12)%G~8EGfwn|GA-FK&vfHan>*h>^c*(c&@Z>z;^9^26N-FcTxvYpk*W`DWJ-4H
z%u2U$*c?}(FQAdQ>d;wL4Pm~<Bg-c^|NIl&GL>oKRU51KS=)5%R!=tb5uZ@E?fUx1
zehKmRCpf<uZn-M;MDWr6Gn=j)R6Ss-eN55Y^4uqZ^vJkJlMcS(%V~&6o)v1ox-`uC
z^|9~K_fKd~FUfgutaRo4uNijF_MBel_q^@u<M)EA^>-})V;Na#Am8#~+WjbzBBqF%
zxBq^9%g(RUwD_i)e$(&cZ8px0(NCxQY)<1oZPT2(=OEX-^P995+cQKIa-ON!ExAdu
zYDy$Sx9O=rl|P;<)^n}S{qr_m(&>VHxYIMU>~kA#9XNHk%DMH7{4|@O6K9_5x7Ys@
zXF6JOU$*YC(l3X=2;Kq}4=bhi340rkGCthaeCYFs=%UM;y-%D=&8vPfG3l+{pO3aL
z{`}dSAn84$CWQOOd!5GF1-W{0Nmu(WTXwA6QXHb$|L*P2>?3iaJAQMtXZ-OE@eP!c
zl`)K9nLcGrccx$_t3v&umpT!eD*UVGTCgUtw_PmhI{9RQ!}6V)Gd{%DiCSjwI<WPc
zV)GL9ou>aQ-+T`i`K(%Z?%&62%>K28u2KrNic>EoFv(a)7%5(Rq*`OWFkWTaK~wuR
zBAd20?6f_gY4du){$t@vQy*HNi9UB%{pF1Vy|3eTmESPSneuyY+3o5*OMQRuG+Y!{
zdd=zZEB(DkH~l!U|LNyq=6;14jgPm9Zo1&{d3vUp!-a}D!q$H`nAtzP&|be<fGP6F
z!sR<J#GjaU`28<8?=^*1*-LbOei54IenLju#qB7!e#@6d8S~woWPU^_Yu9WQ{CD!8
z{r1SpAE%c`pRStS;;ixSJcC$x>mKekw<7cBPp~QBZml`9p-lCk#bT!g(*N^X?`JE1
zj{jk&>XM^a)3;Zpa20!TME3520?j9y(>$8&TSKicFOI2TFW_oWuWj2_cTa^Ow}iz`
zG4;~Z3ul-zGy+epyRGxLA<Npy-d$dicjB4JEh1~<pDp8fzkWhK$1kZRGu9cK{Wv3;
zbwi_8>wL}Us)nmu_A(e|#yQqKQ2F+;W%mzjiS~I54_y$jYW=VI@aM~E7c38o^B<eb
zb~&c!<j<embDDDZPkOuM*}2{Wa!)yrrTOjc>)O<O+}x7;4#Ro7nr9Ps)+BtLI6wKv
zx(WL(KTXJU;QHe6BZu9h@4k`MgiRmzaa4L)tnV`W{^Rml+2fCL`!(FJEy<DKo20))
zuQ*%qT=5;bhhHzLHndk<tT?CT`y)c+J;$f@-m0<tc5a?k-}B;Q>RI71>4?Xt&kO&X
zbT*bZB6o93_)*jPAGZ^J{$ide6V$dR@4uR4kijyeKDEy$!=GedG-?*gc2Qhjp!1RO
z5A(CuJ+l&LNvApZTw8ES$gwU`p}yH5(ZJ>Y((Hw*9bcAt+iNdTl`!eH>B%w_Hxw~T
zo8x4ElxgChpRHTkrzJ5R@!@(Mm;EN=_HMuK(_5C_Zm3LJv)p##>}fw2=M~63Kaf)#
zvMK7*xlPL#*UhoxU+&!X!RE)hxBS;qk3N4OHNQ5g*kjk-WwmkQcN_LN9ZI(BnVPrJ
z^Qdl1(St9VJc}Q)9++gQbF?K+`fc9YJy$M4&XSn!n5)Frb8dgk;}Y9-`adpQ+4K3)
z{I%>w#a~<Y_5Nec6Me?;jCpRK=arc}%3OOL^)IJyDB{v|7s{S7y`HOr!&1>_d&*Q3
z7b|H7)h8?+no-_qli#<VH?Me~ke3l9q<v`a6P9F_y35BdKGA%*$f4ufB(D~0xiAl_
zyLAs`7CfIlA-#18_fD}(a(ApBh(2R@(r)Q9pW#TB;%hxy?JsqYEU)FZeCFLgEsxXw
z{s*4}e=mh^iptvRQ#Vay!rKqe4q5xH>6y96_=oO;TME-MRJ|T2ZkqUH64S)yjiP5v
ze%(F(bjB8zsR!)KoUX0juWfIuUBDId+*j0o`@3UjH6BQqPHsG05MJk%lI_DL_vq4@
z+WkKFB9C?XPWSBJs``0O{oMtxg)X^UssEp2p2@#ei0k3<ouApOcz??-`Mh||qQu#Y
zo*wz(&T-2yaLW9B6&X(!GG+F7Exps!-LS@KfzaLDX)(QZr(C4ER19@abS$0v=T*K&
z?`wmtUpGZCe{j8i!@~Va@7JjcDHCSYF5k!B;?LSJgHO5aZrsV0?vdMOthu!_U!m(u
z{0pU$&zXKqQ#7Oe4bOl4AvVd6x$26aSk59#w%oSICywO5V~PL6sD0|FZIiK7z49KP
zx_yktKiC|vd|}Bm=gj_;3Z-)fZ_ayPU326#d-wS@Wt)x(dK<QW;r`5P!W^@=$6vF!
zQnlqgLviEeACWfdXKNov{7L*cZQb<!wiUSx=WE?7yOeuO=CMBedFL6)7yNd3f1VcF
z_u;GX{iWB_PahY&rJoXO^!@Hb`TLJ__h-~3yzTk(USI6~hF`AdColSX@y;Ko3h(au
z3&fq$nm&BHuyo@w?pw~+PoL6n3IB7*y6Q-`_~e8`)<(VcEjHXgea~sMzn8PoR22UI
ziLs|mB!SU{+rLC?<Ei7P<<m5#oNo;0irK{0P+y(=mGAU1uFS-VY(o1O9!!^y`E)(#
zL9oGAg;W1J-t5^T(CvSQy}0w)64eKj)UI7R)+@rAai+Dw;QIn`h6i1%pXx07$(-93
zZ)5*ZT&(c5+M`o!sy5s<8(YrHEO5#1-T7JgQ*pj8xajk~bN9v3kdB)c1r2rAVs;!;
zQY_}|mtAf9=K;gKUltE1ZrsUH?`Olmu)M-AYLo6qd++*J(VMIiweFR2PJU{$aY@a;
zi$??mPBGfeII&UUz5d>9ms5iHYIGENPu}<5@wY<$;q|$jI-VJS5xJ+ihi}G<vm5wL
zli4o4%6Yh&@eB8K2WH=Ee;(TLbY(t0_>_Bk`K6z0BO~n<n3=WLHCAn%=h6A4WlGA#
zHD$~0`@f&4|8qizXnl8a@N;e<$7h?fjt6tiyTtyYbKmN}X1_0gQ}arklO&&Hf8LH|
zd%NYCDLD<#c=qp-dA@w<&JR=8rOxw`)>V4a9HcbOgoAIwl-B6KYDy|SSCsB-X%_r>
zfW;|3>Wi6zVvv$*WPHj*olo=fThFm82H7fb?3m!c=3LbO$lX_W-%`Ake(-(R!fP|v
ze2RRW6lATl{>ZZ}Z<uTKZ;7S8yKY&qe!}`7w_O4!8#_&sw#<96|Iw0v*-JMsy+3*D
zla`m|_Sc2HjCw6k)=m^Ync1meci_&spK-U%|8F^S&BATJ@TJeqYtKdXFS~vI|2<JA
zZB0#`{WcBqc8_*DHK?t&mAT*1;=!dUdH(#GU6F4$-MW~sq`UONy~r3lroIa8)V+~2
zj7?ZHkL=(`JL*u>WZ>SSdQ;SIhV%xz{r5MVte&YI<@)n}UfBK(E@3~GcYOHx^m$F;
z)bmolibtYO?#$l8eCYPsyML9sQ$(32eb`*_=EJPRSr6lnomjPD&Yx_lyK=&BxfcI*
zy{uxOcq=2b<BN8bP^Y3Y+kz5n?wR%;S6ZK5koaHlx9eQjiA#>06~1YEPygUO^f>gy
z9Fc?v;U|7Cof5lUam!V^y-j;x|Bes0@$2g3pO@sT@WNobz?`ioC$T1Is_AZCvVYow
z&I>U;J3fXaILWI>9^K@#%dO|j$JI{H+owIqolzZHP`$#exblGcoT_P8<@|~|+l0N+
zK1#+n2VMOY=6L$gR*U6ji3iJ%E&kwrMx{>C#+IwZKe9I6(Y&F5&Bvt&RehbNRm2!+
zocjFk#jVWSd-?a~iE+L+Rg3Uiu>6k}|FuJnPbZf2{?LpOzw7ra)}V6vgR^(lHcpax
zGM~RPly`oo`@?Ao1uM@AuATU0>F;xo)b-lK91feVn>%&>{`aoZ?qNT5&LtfBJlQmy
zJMzfNqq4WyG<TG*G4H;nywPoPYV+yy?20nWE@$#R_}O;dx$bxMzq*656_-01k`AW+
z`>4C~)6eU|ziJK3Z4=Kjt_fUt-?~aCV$Q<*g1-(=ou6#7<4OJdAGi6B{9nd#b7w@g
z{9dEO9iPw3&k_DSv;J-B+S_W@rp&Gvwtjw|c(w2J^W!0NIs~e^?!9wr65CnxY|~L!
zC-aG^-3M-JEp%FI{d~TOPE&T>3kQAAL%D(sd5`oHtoYxV{TH9nsUjMi{89g|kd^)3
zm`yrq1=T;Mr%m{O&@B%<e;ez!cFW3qf2(bGKEHfb)@A?6Jp1?Fp4ZvdJLSy;%I0Lv
zS#)pWzKAU@SNo?r33~`H&8xh1MKp7I$yt@$+`n%Qz5SaYQ)sQ24q4M9z<6ZU+|a%W
z*SJph->u!?ZmFZ8qd8@Q)X&cQSzHRUzUGIyf7VWz%zsB~TCBp74RdxyG_H=&T=SiE
z9b-(AhG+caxX0i3R{T7vC45@+M=@6>|LU4)zH0?4IajOBc)sROK=s8p)-OLkou0D!
zR)Zr$y3>IR=RZt8T-CMp*<-WK2R5ocKAk1)JvpH0>FjLR<G0>N2+a8}o9=ln^0$<n
z<FB^~Z#%!3t}d@Q_xb%xoqLbIPL9uRd~S9vZ)Wy_qf+sIo^86-bG%PZzbd6QV^_--
z;pu0p_W#heYhA<KS}{etd8($DvhAE7hGAB1+&^ZWJ3p(&#F;yBa!}dbG|gkzotBx_
z|Bkf$>3Kq2y8h%E@&5VSAF#(d7B6|jCO`YLzSRFhlkj&|w?B9JuJGkPne?)5@0F_f
zZS5~A^u!y_yqmgl@1k|<K!<uTUwLhDwCCTu>)va>f1KX@`NQ-B@!VVP-}#iOwPM=b
z2_M(Ksei?opPkqd8S^;f-I;~y`T`za8NUOiPKo3$t4&Celu4*%NZ85O($QqG_0#oF
z3O|b#bD!1w|D2!HJ7wqlSv#Lz?lXF}&g{_r^5W+X7q3})r)?F;`}}>~j%S;9u3dWk
z<9prvcg$v7IU3>LVw3H8Kl0_8ou__DXsnaW=CuELKl`CgV&KJR%>uV?F6xcFSQWTs
zm(P+JuYawN+^;9@xA@r1qP8biPs=vkUE>ifv-tAES*6$SloWd>KV7)KbxMw>?jOtU
z>CC&%Z;D<i|76R~J$2_VYh~uMEm^5MU+F(<SNy#PZodycS|+zm^ftGo>FM+xA1n{V
zum3PJ?c|x|#g9skr(E?54cRq&*Ltf9OII~dy*quqe5zQA{_&TGZtE{`Uo&m?>D+(I
zTl$whHU2Q)M}G5ir=}kc5lUeP^f_)VExhQGWutgx#ss66HYGO{=CC&vOpW<Df&GZp
z&SH~lQ=u2($JX*(=MkLC=YKF(%>Ti<0=wu-a>bV&Ee>&Q{`&DY|7Z4jM;CT<W!DtG
zRW+<va1d>o`Iza<!-52{{qfKG^!cin{|LJlRd~jwYv-P0pBA2GPwt4^((8C)S?bSY
zneVpc=K0;J+a+F-xyI(t?c9qN3<Achlgfh1L|=Y;bNuBc^PP`cx4nKLbV`foa<SXP
z-6dSw_A&eStgm&=^3>AWcC+u<dCr$5C;ymoo%Fn(nx|V3{{Pa)^8V?dIsZ?BP7V_P
z=Lj0@d-hDwF8t?h9%GkZo9{}7mIkRbMH$qzlqfb#+Wqg~t~Ebo4g`dTU#+>C`PK1j
zPU-9aOfkmO6E+^3t=;|N#b%Sjjaq+?6tB5`!Ll&+Poa6MK;H?E_`_^s&&9Uuyj$*f
z)q3lTD}wV^9j*7`{qNZN<)x0(sVyr0o>TgNAF8hso_nZX=lSu>^Z&np`#k?`%FiF4
z<k_Q&o^r4Bw%|YeR9)}5OV`e?Po^Ee{i}Ws?}Cg^`zNgn_c&MI7t+jjaN2~j_GgRI
zY!9q|wCnVLk-6?3b*Ep>Hz<>z?3Hxv663LlvqT@wkQa({`g8DqVYZ`K>-xe4$vh__
z8u;gJ<utCDw!gP@3IhxO^>wD-1b-$)GO<WJcbcetQ^L&W(1!jz&eu}&jy`2*EMe^T
zRt%bBEw$85b-nKS39A*=mpGhO?4R(bfcIs|yc6ORt(C5S|N5h(X#2Z0laF|V&z^L5
z4Vm)4IN{p<POqI>EoYuidMYov)J3>KH)@yX{4STtPZvzy&!XrsnbjxLfa}^)ou{(w
zhte|rpO-OobUswm)jzT}i7`^D<P)=Rm|s9k;)>%zr56n*p7Uq+wX$cMUcei4MqOCA
zM030S?i<1PZmkwcniiz_xBP_47b|Vyy)Qql=Jw{CTyTz~-LXDjKjOOla^7c74vNos
z|5&ybFRLitG-ZbT{Qm*m4thL?8j?SBA5`7{YpQ(AC;QX<dJA)Z2zcw{9z8IzG)3ke
zE3Y2oghnn2uZG<>xV)~Mz2RLVx{fjJq4K|o1p?FgUoyP>p*p?9^2e;^suQ<*)TZ$W
z1uZx`;SkgG2~OU6AGcohJKS%%eM8o6(2<(|xK%v5ej2{~KV4;I#@fFYMO**J&-wHH
z;{N{s3{!PBE?>f^#J0vlJ(tN}dhO;M-pobW3s0x4T;BSH=`_>EpEjq>0(YngO+P<>
z{xpS4YX7$w$$E5zO!_La|B}o)Z_%eqL%ja`3UQzJI_<49{r(JX&qwP&KKB06we2ET
z{VAb_Ad{IHA3p_nWFC6n_Ox`W%kM~*KilujeKg;BPWxik;z@1II!CRxl`NUxv|A@r
z)sOMt<^!rRmnTdtYh_%i!s}dE#}N9ZgL%i1wM?9<G0jsZ#&hx=(SO`>OHuZ)^#r>j
z1Cgt&<%T{_rh6s*;BayFw{q=Z4%VC+r1>}cBu~f5py|yn2k)f+yxck^*rfk`y_D$M
z+@oKo2ncg{dw6838k(PZDx|&q=^cid@^kqdTH>GjKG%%<RehgZ?8mapz6<1*{#fRH
zz{oo%@#K4n7H>9(w)adGNtNk`P1(Qtb>{S*jHv7U-26%QRAcs=FNc|CG4L9$5fKV%
z;bUa(i`;Q`OVOOR&Mh?}&zA=ma6b6Ma((Hi)AibSLu_ql8@+V+9BupP_~trA{%@vw
zOaC*~Dr5+D=aqZpu8f*|OX2mz!x@TKe_Sh-S?0I?N{prc<(Kc@7C;VlI09aMzVy8p
zyI1zfx(m6N?_Tv23z~I(ecbOoWl>2{3G1vxt%V<y$UN?vdMMt>Q@FtHu;JtFJ&Rw3
zy()bbyUKV~;3~UQC8^x0?us@#aaT9*^ivPmUA6Joo?W}w?fSMS?EUOF))&j49DlNK
zCyR}LpZCP21@7B+|J;=oyEX6k9sS+aAFC6m%P`Jh-qEZ&(cwmt(5DRz3cQAq^B!C9
zdvl&EVG#1(BBUCwy!4rv($Zy1=SfU>Xg)<ggQ<h1FVQ-1+8g0d>fb%G=1lR}Up`BA
zRza@pBmF3cnx~~H$5X0Jw{Ml*Vq17xQTATIeW`ux_PzU|=Y6U3;EJ3r8Pg5*vg%8!
z-t1eXc`7sD5LeA%u8Gage6>D{nh*Vn=5$E8<oa=u(SPCPqAD-mIqf=l@>Fz`_D;83
z?`Qs-dG^xPSITQY@7=#~-jyRa^Xq5w{bxNlWs>QiM&naE6AWj2O8q<_#`oaUk@JQ2
zRq_wduk4z(-fUa-tKZ)?o)Or-;Y{jP_H&(jnFd@BJ}rF{x_3(A(=Br5G3JkUA3B(N
zX~7fKlZQDKgHNzMe|xTfuD;k>TPKFg-Bq5^lcM*0^N#wVHD5;I{v;Wr27yakGE8qh
zuX?}o;})YrtF&FlT>SzkmBd)~7(eivYV8|;Dz0{#(V3OgFGr+u|16W7IB`k*q}S@(
zJH(D3IUZAR|JAj_xz2CjZk4~DR&+_I!1K0X(RG)MnUF&bdfv$g8y@R>qaxbr@_2de
z8ei@|wi=7BrJJPa$8|>+KQwJPy?g8XX}e$Rzt)OXus;8FQQ4<6PYf<Ko2VJ)x!(=n
zT|M(OyEliM;!5|R$4h2!%uhadWZxvv+8CevI~ZFgc(qK>YVpWE-FNccOSX*F8N2uX
z|Fe2gZddW4^m*0A-^1UPZrr|O-{QJE)&-f1)a)i+RC~;t`<k=Jy)$CbRDnw~c^<zy
z!QiQ~UrJV(ZKVL8=M}$Cn^P9FWpuC|PU2g2s?)~Vui&K4f=p)>7pIH@&ICWnH9mo}
zA`%asxVN_U`^mefSDji_x@sSTp5=m@{-Ud1Ee0*W{(AI*&|%+(g`As(uBrYw&GcmI
z(_Y(mJKcB7Kiz)u$tlL8wccw(-!Z(jiJbAK<DK{FrGC;moHyO=7=AF!d0X@KH9z||
zXD`S2j}f;M+}R@w;(E`N?sVHLtoVf8>DRfxB}L|kp4PZ>+~j|GC)oFBa<1@NH;G-(
z@BErEZNc_u_pJ(EA836gE*1O2FyU&$R_3}R7ah8%H-uk&blmgfxp`XWMb0mow7T1M
z(z8un+d5lba4PawEz#fb>)rY4pDhATA}s={ffJ@%{+iR%^X8t^?Hb#e-Y4proBmJl
zY@YETaNXPWk?SRMyhTh!(@s2DwZ1gw=QH2d*M0rPZI8aBf4#i=MqZVHRc*{o;jBf9
zc~3r`Z<P<@GYFAgAgOM3vF}Ho&x?>wjrcUKRkA1U@Jbw5I)Cf`=z439#`MTZ_A%Vw
z&xaYc)~GbvH+YNbh&gSrJ^oip-@B)<=wjd1wlKA?&ac9cRO+xhm;5?h%wKJ{Yw5hD
zam&kA)T@`o-!*vt^_=24xt0mXpD5k*bMiPU{&mtemHrx?KEC>r&*vJf7naZZ#Ozyq
z``FTQmB%Lx8G>B7t3~!I9J8)+THw1NJV19slG+ay6Q2)Z>`VMVh}{%&o%M+MPQt_q
za|CwT<ZrH<b8ks#2`7V|vQVB)Z$!++g-$!U#Q)~5)?YpGXPt|ityO*bwY#O|_ZiRD
zzWaRF*K1o#%L$jQ`lW1lGcWZC1Tio77C+T@lJCW`Z0n5Oj$J3en%#|ko1$377A5pE
zF70O99_9rt0+-qZjQr-e%NA<vSjn=3^TOpLGus3{o$F-3&KJ;sM%lL0uDma;Z&`2B
z_C3$712~>uzIXFap2HlU=>nhh{%I_gd2Vz3>DH%?pPr}4z4ScMyGk%sX;bX}(%g4t
z`~HQ04R24I@M(#>q@&zK?I%p-cV{Ne&-|4Y^!(TN>&IK?`+x9{m@whN#hbA&-<{~H
zKgl%r!@ddAmT-j46xeY0-J+O0-HV}{r5<fDSzlGweLbK5n$5FvJ<Z?q)U@LL;}*`z
z{N#|WC#7-6)~RoT^0V+5m0y3qTuq#Ao;P<-_Ve||d;KfatB&8;aeHp@O*an7zlsU5
zCXW<u)f6fGJJ5Vz?n=!AM~+BuFM%tYJcNQagd1L+ka6Ntzj$@$+pXHXoozaj<-C=j
z{Jogo{!5>(wq4W}dcx)LR=1_e&X8j^PTy`|G?zILVRxf8U}jZ7_|3+r!K?3_=JnbA
zWl`q2={xThUH$_)(I@>|U6j|MlP+DI`5*ERPij-K+pF_)ra(T^gH5Y_RsVK>RuE<2
zJb3rdm1AXhwQ8pAGdw<Vzu@^-%f)Zx1@&K>uqn0GWLDnnos5?f7b{txur&JaR&r!=
z$LV$YcZ2R;S~Ycb)Y2?P(?c&sF8)o{I`$^xm-pp^zU2>hy9V2e^NO|q5h~JG5o%7@
zRVJlzyJYgYO*5VCRNYV7Y&C3N(Q1CV<;>JYNB>Iwk4*CUYrRnab=8KDhfni&@R$Ae
z)th=}b7kpXw~D>9v-d2C`yF)njH$f<UrpwYWA=KS_f=d}a;BG0Hl0{Ko%?&iH;FmM
zD&{)dor5wy`Fmbsx&I@O^YZH4b=$UQZJ1Cd=9}i(Y__Xqd-3G#Js0jvR6aj&_vk&g
z_pj_W|IJ-(6>fTwt>XO$%jdS&UzsawRtQe4e4Ke?g24N#Ugu?2!s$s>%fyeX9^@)f
zd?56$O8#ryTiZjwv-9KgEf&1jEU&guT%x9!a`m^!tZ70{y9As#Do>sKTq0>|Z1Y9^
zi>dF_$7h0bt<ri<8+=q^-;m*~a%9P0wF2XlYA*|eN<KXliR|(*kG<`2+s8C!Y53Q`
z@8Q=&-aon~tN_|5!u=q^G-j*w)#K(juYaBE$U0?O;)w~V3>?pNOs9UHV*5X-KJ(T5
ze<72cZii_-*R0kmJaX!}#J55t??rLT+rGX!w0f2&qo-{0=j2mIQi@-Fx_yi9*1Ro?
zWO%eJo>~MgD`uGDxMfky$F+O5K9W#OVON~PwOd{B(2_8Poh${M4qPe=EQ`4htyK-P
z=CyxvCuo1r^uX!raxDT(4=%ZAnN6Q?g|}tG{d<~cTTcXZyD98E@}THU^F;slH=E~X
z@AiL^QTg+aNWzLo?<W5`{?%oDxVHS2dIz~0XFIM7PtHiF)cdj*bgpO2u-h@SMDu=e
zR<nwejVJS?MQ<XDPjBvM-}>wP+UXAsy?3|>@9_1`HJ80BwmDeFP$5Qqo>7QR$$<;s
zo`~J_>RT_n_1&AcFRa_2y`5qAC+Fzt`Z?=A=UiBwAHRK;)3OPgD%+gjoVbx#AXV|<
z&4avycfG3i{Ifn%^8Mj`rmx|h{g0)0^xd7YWTLL!jPD^qc5lucYAF)!DUdBUu3fyU
zW4+jZ<t9}>6$b^@!aJ*~mhO5a`+o1)YEQ5Jl;<h=0iMhYYJPmW_}D!YveQrXAb2Bl
z!@hs@KhiI&er`Vfs8rQZPT;MfSxa`qq#b1qvvo9ezC`VdHFGcvspxf8-tfQp)l-u&
zNi7x~C%NVRbF3_7j%;f_^iy+=UBM5zr$N&MW(&+Mxb%L-|Kk5s8Rk9|QJS97?>9AV
z$r-Q6=`w6DowCFqO>>bybHYF1ckjgVXLsjDU0moinM-^%BwH?13)r+jxAeJG-~{0*
zF9Vnwv`!?iD^-(a;JANu-|Yn1isbwwt2)>OQ<e60tYF#y@wV_Cp?#iK)1OQ%3HJEa
zx=LIsHiPZGPHwB`b-&y3PP0GiYp>y#xN*RzK_tMucv84dXs_WpmBIqf*K8794qUp=
z4>z9OcvoFx_see%@f-F`Zky+@;MuVc;?H!$rWacZUT6`iIV8sG5ZP4}wTRR4<-u9O
zKVpRZrrYZ6-}{HjC$F#6D(dYF=DEicrzCZ!7}WlWEfn<7+wODkQvK-+hUv_4yf@4f
zo*xfkbdD80Ua4Bu)4r2Y=<JkG&FtX1EqiMIRehd%%tL$m8r`4XJmN2HbfYSAC;yQN
zyIH)od2-^KCrganCVS@H(rdpj-tSVRe!E>eF8%p&j+GyePJW&`-=8(_{)&RlFV6}x
z6fayZ$b6{u;jc9Q*>iKtcKn?9jCWQ!>w_7Yvu~>IP2M(HO)LKXLyzO_vPRYZ|4O3P
zXJ~w0_b2bc-39Sk?`Q0(X75fu|0q7EUNWU~w~)<gEk?eLTU7X;IbXch@cvR|oc@`L
zln%3Ux5&r!Ew_`BCgo4Nx~WL*`pM)Vo&&5;7O!GBe%CAcUfSJ;NnacoG``)-OqPrk
zy7{s!Wy#vL5gVPXj#{4nx9RK36*pK4n^Y9;{onI{(xRQ^Gqe+rmNObP{Z-XCl`q5b
z|JyOQWx8sogFZ9-oc5&Q?~31H+coo~b@~fW+~=O~=tO7F2@b}sI~XR#Pdqzi`Kxsq
zo!g{euk4r1G+G#6^SNc`D!Eh2t6ZO6vSG4)m%JmOI`ZxDt@_U@kNm00F8<W{<g(_y
zb7$v%pY{I7(N}`|w}^7wj;WH~liKR37y0Dp&itocPq}|}&Wg@GvZ`3;qK;nEOg7sl
zk)u(bJ9>9H==yDa^u{Ii>5?SP=e7M)pOsi0uy{9F(<|tB+U`R6oj!X`a5~-Kb9Q><
zThy*srt;WB(VM>}g>5STE}@df=E(jhUOSWD$($9Rm8~*eK&VUMvFVEMeow8R@TE_D
zlm7Poll<q7cNX96Z!(&@x^~{C<w5+HeE;h1_4%*Idf)Z>0<my5?dB=x3%Pw0?#D0f
zFFkPfbosn|`}^guKAm&%yvNDp6Z+)h6O9;=M{Gv|rd-ziz&PjOw2eBCH564Ql$e)4
zW6-J4@Bd@`R8+CWr0(htpU0o3|Bl~R_jCJ!@Pqx%r@$vM>#;g;T3$BZDmD9ei-6@v
zt$UGYr^Q(Hl>J-bm8HN?b5Lc5#pK`B@A-Gv|2cS`pKbb*S27O#ekbQn4vn?(H$4BZ
zG12d7@+phgoby6M^cE?emtP$(J$q`b{-*6KKQsw=+D*S-*#9;{y#4y`Er&1vS1|TC
z&40C`=;X>{%ip=m^;~j0>mxO_L^pHYwuxyA-R9RQ{hU1|oU`l50+;LDoApZeT)6pl
zLiCpCdF+Mi@Aq7BI9IFpVbZH#3#NpfeD$QSH{Mk4=8iw_qIcKdtyS2mpsKL+LG{^3
zS>5_I#+80QcuwAqe3iI!@^$^6nGbg{d)}TbrndA{?9<Lqk$0_1t#9YbIz;@t)>Knc
z-Te0L_V1nLJ@umJOLu#p*9(lTd$V=>^%rr;j9b>8iZo5Qy0hZ=kKL_L8!z73RI&QH
zn0}@CP3fKaUrxPXylCqj-fqYFm)TD0@Nbt2*Dqn;J@3!?clE}u<o^D<EV@f>Kd`t{
zxae`r9d3>K;1AC}-}Za>O@D@U{_zqf8EcWEjK>S5Z&lCIlG=D%{oUj{d-l~B$Jy3R
zPj_ndjr$rYV_<ogeP7|(xhV@Y@=fpleri1N!~JiJb=;G!4wz115SytWpL&wjTki~4
z&BtbuwQ&v}PbVDP`p5p|tsQm@M(2KSn$aPzGINFd?iXxNWXofpZ9AT;pmB;(ZC|<B
zFNKGHE{GTO3&=g*kg?H&kt>5OV|%vw%+vA}i7n5!tNc>4SM3n`t$t(|kJfk3qifeZ
zh~b;Jb<L|a%+}W*_1=#1u<=;MGU3N5t|i<l&o@31TA+SF>X-PtLodR!<8t$_CS<NL
ziF$UlYm36156ijvcC{Q-4T##@!x|}VneDCeliAzX<2i%sRK54={8R59`(!DsF6|ZD
zw{%9q^lx10CoLX2i)?2}(D0CaS$yE3RD)Vf^6eyp(~PaAr+2!0FeqO;>8y59JGsZ$
zQ--a>Ku>_<FSx0Ee_x%2jk1kw(2g&MA4)!aS@`{N(4$EP<{Nq@yOwaMWQeZVbJq5*
zyP>mCY{B~z&C2^_ExGt?f1j7xSNuq0iGv*YW*3fA?48f_o6Tb5?+Sf-Jm;u4gP!BN
zujQV4KUd0p3=N;(t~yIq(12sl*RPecjxAvbVu(4k^*Yy8(Lz~<=_l<om+Z{8O}E-A
zw06?^$uB?4d!Lf1o2=6`XJ=PTwf52pQn&pwi+B3^LJk~(9Q<%;U)1aa&nEec*ZnHh
z=l6L3_Udxa`H6FPmGHdCSmb?jBIuN}gN4FNek82CxjXQE>)!pGUOamQc05@9e%UXd
zT()A-152Y9{acXs`Ht}C<@J{reVu8x{q~+4S86W*^*!nIq>|yygQ{~kEhiS|Ip5aH
z-EH#nQq6=Dc6R@FoO;Y~VVT~mEfE`2YNxM`?TO{tz5dO-eZRByqMWv@o9idORYA+>
z72AW!kfS_4I(2hT)eot?@iH)F&D(JA<<ETfaxM4_*&M_=?`Up>UQ}L8-hyhzr}t%Q
zj1~5D?vMT2yOnv_ao^cSsyy?U`gpmdr<knZ`QlXWxjSsREo3`;9bbXK<ZVZFG8Z|j
z?D+k`KGid!Y)y+$T=%a|tC(8$ZCm<J%KJdJT&}yWTM}j)oxk|H{=GT|v4eWTW(I$_
zUrvxOI;6iuWPPvlYrPvqFZW#wyT;b2C47IWJC_5$Lt@?5SrcB&n<mf48Q^0Vd@aCi
z*C(U=mD#c~JTk&JCN+67MC^RHTOut|uU<5c>3NXyk98}AEFHEbD1^qe98~QSdR5|R
zdAUwvZ3f#+)wjQzQ)h0leXx83yZKtZ<3T<<x)&;z-8^WSc=SQYRl{R5Rrp>V{rOO8
zH#_5-<W~RVk2!u8Zt-SFw>bJlk7@e*r8DX|16#5iL%BZ-6!*tnHjsE4KBf32&vUss
z^B_lZ{4{;(_c;7e=v9U*QQz}a^R7zGsbGkBAo7FtWX$zy*`(RKn_Yx|v0k6R{%#t}
zkJH?j)w9<fk_+6U8ttc=E^)wr>uIa<=`u`DE&R%*HLq*BSzq1s+GUeNW?Z`4&iu9e
z&-~Z?zGKJE51&+j%jiv&ely{5!HJI=Gj<(3(bN(Y=fBPG+f?sRcJ1BzmUb*2{vJ!g
zWmE53zbO;zLVr1ksZBnx;^5{D>bK@h-*Hmf$?|~h1QE4r&t0e5=Xlt}{tK#(czBih
z>g6k+=dYf>+pau)tMG%^Jxc#V|2$q-8T)y6T0^!V!~LV(;`3(z&;5C7UcQa^b<hTe
zU+gbD{Pk@rf3#@Vyvk1A;-dXD<=cV78Q0_T*k`+&2426<WwtnU-7m|~wer*C%l4ei
z(~z&faA@;|?m1FBg-fy@Eat9S7`tNbvbmkrj0(RuciTQ)yZ6Carhiwg1wQScApY^b
z+AX{6Lo=8CQ1hJhX~Ra|ZkD-$eVpx=jviTA!YXj@-^=AiVo&F94d>Elyv+W_ePZjP
z&PqGQ7J*RR<1<(Ne$|!b>9##vdfofI*JSHVxwnLG&3>pI>GA$d?&h795w=2;_?D|j
zU*dkqqQ4e=!p-lE7P~Glo_!-~;<tu<b_O*k&wl<98Fq47^rKJ0Y`eF<yHXYDJ)w2N
zb)Wee_a|=1d02SlQc}&Q9Up>EM6#S`nsPnrWm?wZ1NHLn4yAMc{TaAE{e-yU^*x98
zuCFR)U^-wjg<+}aTNkA_@0!<Ie$-!*+~Ccu@UNtz_~m!;MXrY?PMW}^Rdh$ZCdr~s
z?8(H|IpKZxlzC1tI5o}@TBoSq78dcVqgVS7_-viR*c)QP*ZI~SEttkUT|wE#b=D-E
zIl|l9P2B{FR@_{i7CtMMLreaOeum56a|=%exYzLRpD<Z_{-!G;e!slJyw^^9DQx<3
zr){e}>+R1g*KFQiFU>If(Z=cNb$gX|-uSS+Zuuz(u9#*Xwmqj4KR4&{uTy*u*?}y+
zo`>J;$3c6Zf2kj$zE+gxm)?IXce^_0{cnE`-M+o664}Ba2b8I*CvLUV>N^#}7~#Wz
zGRJF;kopQ~$-Kkw@5Y@!CYgRH|JL0%%17kpHO;xEEc%xBdBN$QubF0WJlHp3#rfx|
zd7s)MWD;b2pG{t$V*0fIX-v+BJMX_RhM6YjKYArQ>GhP`jyC>xt`>AZoAgEcrf}qv
z%l+RrUVFEIS*EnG<j0Hj2RSL0->-%IUwP@7ob|ptJL}c2i?2C;x$olIYv&uX0}2kC
z&d}(n?yi=YStxY{eErl8Hr*A_qg(VH%DSr;?p!Fl^sn4>2Cr{BuNAL4SIIgt*MZ@Q
z-1Bwi;#=Hj=pH@uC(zvD{7gUbcmBTGO}_8^Hs{TpdN7`8ZHe*p4S$&DR40KCdSH*%
zp1QuM|8dFp^>^p(-p;i~@`dP%dO5l2e>X@z5ecfhxFlnZjRNbQLWMuKr7agsU}b2T
zI<MrP#&2u)_vf1DFw`pja=+5UA@pv>q5I1JN)#GQKNvB0*vMDeeci*n<<G7B>&ol?
zGnTb%`!1@zB)Me$*410XKTL}eTKh=-M^lJo$huVmQg05Oekt_Y?(X3ueK$AGDw@=M
zrn38Q>g(VwI>P<>yoVwzf8OR4a@??vS%as~{m-3dsfUpT8O=KxRUPUh*Di9DFq2y_
z|Ip8u>5hSY!9P^jJA8O{$+3IGS=AHzhFhN69AC*Xea>N-Q=Bg&7<sQV#5~AlGFSe=
zT59}3W?kdz0`WBKDa`Mgu5cU>e4wz7yE*)h^*?9X%&#Ufhs&5AbH8N_wFql!J~c0F
z%H!;U)|N{hGnf1byS8hEd^LlQqY&pL(Sj=soT<-t-8uM6cSC2?Ta!OCin%X6<ZacM
z@j*x2XeA2=v+stf@w+OXel?X;eVcMviAV3@V%7txp6*Y5EE8l>?R9o1|KoWa@nfre
zj;-sRFHw)#`P^=<Eh}8leeU4<tA?B#f0i@m9V<PRHRFHtuh2AI!R3~MOideivhG!8
z{&Ga~mwt($m2j$_L_)$%<2Bn){++C)?#~>+x|OAe|AyQKahViH+4uE|3yKd!1;{-9
z5MkmNUsF_TQ?p8Di)BV_0Oy`vI=e1CKeh4wlKcB3pK+I}Zt?!$^?~2x>-=~3C7<bD
zdwlYm$8pbni%&o26X8%`J$64bm%&)*1?!RqmMznkg@vs3u;^>tFaAJ>_4<d*sQmB~
zuP#Wwl)f~5>3LBjXC14>doomF?wwq<cZZ+S!#}OHLTqY2jNTjqVa=sdG0i3%@0|Q9
zx~45o(2AQb@}sKZE$=>8hgiODsa$*Zp9lK{LY^>uTF|oS=IzL$Gs-#ZzHV>zTPe*p
zr==vc;gsi7jt6XSybr9>yk(h|cqUQNA-H07%b7Tx{a^CmFaI$4a-~V*uiKT(5vJ4D
zKaJiv%X~_}rQ^m)6Syu2Sj_xV`)Sb<hq`x;5^4>65;32ypRm%7si;$!&v@_Ty?Aro
z%WFa}>utz6$g}d6+5y`Hfh{b{onOez>3g3Papsfu@mD!_%w)KQZ#zD^=EQa6!c#lP
zi9L;B=E*;`&y=m%#bnX()?#5zhICu!g!XS2#rRAdg*YE`=eCLJ95Yih-e%h1w}E%5
zXk0`Q=X*5`)xzFG&$V)9KD(9Fq;7F!Y74Kl7sLN}owY~GN*~BE{(Km@LiWvvmg{<R
z7&5-?zx=-a-rt~mYV~vXu&B(E7tUzfu0Jj2*V;9gc6dEH-r4tZgOT}xwrXyn__Qif
z9^0?(SNvH8*S}u6WP)Lvp5?qnogZcw*%(HayiR<d{;_S}Eb+?u6Afbe1T@tgR#)jd
zO>4PgUHRn6&dKKIORdV^-f`TaT{^LMvgx6GLzm+rzM-8{=CrQJTBYT)^?vF6=vC27
zOMd9Cy|mQJ;si(d+sLn<S1n$kAzzSt;3MOm_NP}?UY>n^-qshdTiwNG9rV{=oO0zb
z<Y?;F*PkYzW<PL>Vb$Bby}ReW-KJL)Q}Oge*L?GzMejqT1$O`NsQF^Y79;hAZ~2t-
z8rqxQw|=*NVcn6#VCN*wpz9$0Z0j6xlm5@T2RGcaJ+yY^Y;`^PlsWfq+9}r@nx;8@
zZ``y^f36grJtd|m8Y^D?N9{;<leI%o=JORna<Lb4e(sve&KUC1p6y`ggA)0$7o{I~
zk3Z~<D8J91DIdQrXZQJ<<7y>L<_ckN|J=Lqr!2qX^-|~cpTA2rv+1l9lGECgVOF{0
zY=S@Mb%o_DoqT!B=lEwEuHEoBzxbov)eJpH>xZ}b9MpS4Jh!(`es$HOer?_*g|&`{
ze|`TLJHg(u#>FG?ppn5whW8a#eVLx0#eUB?Y}9qPA*o1i$vyr*%Aap8P4{))^E#Ak
zX4c7%%MWof+i}>7L~Ez5e)-aF%ipB}i~PSIbeL=>Hffr|BbzgmOINIz|K;R^34Y5B
zWXl7dcnjB;Ii0X(wc@t>G+*VM7vs&8FXv`-8Op@8@n=puyW%O!U)IdYA3m3Ucxm?~
znBlGi=ef!$)iHI|KUwDrPrtl3HU8p;EtBkggzhvHFf9IV_e}K_8_QRw?ao>CN_;iH
z4Q2bycx*RY+Zx5DK7G33{i9pQ4}bdml<%6Ql4IYzt7YLI1mA!86KP_ko%&;0*k0M@
zmh0Tcj#HGggUooEwr}Qp%fiw3*}>`@XJrrQgynO|8GJt#E38kjKGZv*kofN5t_RKR
zWf32)w+U?sRptF_xqUY1Rxjzf+bu(;^&Zq^*=k{DBD!6&hyBG&|K%6<Htb&W>Eq@H
z@$t{1ue!0S$27}1zR;NP=2CJ>(XReAb4vD_T)*>@^}+{6PxVya{(v7d1@gZo|9E%4
zMZnSVdZR_CvgWQBUIE*SlcZCW)%)MLpJ3;oa7Mi)<+0Ei+w9{)35h~mIs}|NE-iTS
z@8rgJM{>KQjsLf<5<k(}yJS^muAUx4h>1n(L;VAMw~M;8x|HNJel-Ty2UwlRc~rH;
zJ#Eg8#e0h%`#p?V<+;PvV4}*TBThz3{?02?*_^%AIJEgr@fWtgE_*!}XnQ-kaB{Zu
zO}O}eMg5fjUIzb9te9;Vc=qPbthFV^Tlp04ebs86qrq8llIuX9=*?T(cda<&_x<6X
z6BmEl9&%Kl!tdDq!E{ng+JoR41p@(_)9aO<ACSK&(A(LnI$ijXH0$Z$CoJrui@Xm6
zh<Y^#S3Y3yWRWk9d@yOLKzm7D0Vm6;y!!_CP3|YltkLlg>X`Un)0@NYjDwtK--Pxf
zbDVE;D8AojcsFkK()ZgOn!=cF=$vH!Bwcw>{@#f@2j06X>lJ*j`NSQ`U6hp3tXs&T
zJ5$Q?fQnj+MSV>{=Sjmu50x}Cq=H%_n*UZYH~Mc?T7BaD%a)p~yk3)l<gc>x<-RPm
zIkjpXW68|AC+qzW-ue9Z?Dl2rSB8~YSNiBZI=`))o&CdE#w*i3jHVxO6nWqI+hkGN
z`}u#1^p>1;syZ0|FaFm53zp4iR2RO##b*0y#{0She;aZ)B`sPsVM3o;(3!RB+imV$
zi$5Xu<4epjnRXM7b?5ZgvItFL=a?TDXVP$}{+ii|3LXA)Oe|)V4}#BRd0xEbZ+&FL
zEuJ759`PTaKX2cep{ry0>qqIs31Y6=>{r+iEnYFp#c8WWoTl0gPJtU74N|tz|MU*I
zYd$J)dLfeU_sw(uDc|_VMwv@3r^PohNnP(YXSQ-vi<NqEr1HQ`j!Q3XN_2my6zYT?
zEjL^r;V^lr)$-FDSuCRXZ2!Lyxcw+G?zH_b9?1<M9WsImm(Tlt&+=1mnyXsDn7rDn
zVQs=(&-rP6-p!NGDn38n60g&FYk#MO<;Tw*E`<d_Zh7LfMUC`-dsJ3VTAjCaA4BCP
z`>otz?~MP=`{uLvlAYhDs3S9@Q+~dR?df%yZxI^Mt3OL*^OS!-k2v~lXUkiA_m<B3
zjWsVm9h{bL@5C2!q|$=ZSN*u?`)LVZCw}3SRPr$Vo^WiE@|`rr<4-?cKF;X>XZbpD
zea`oS|I-iHFQ0p8Wk!2P-I=%uyJYXCAJMMN)?yFmn0#s3&Y&R6V)kge(ox5z7Z2X5
zG4@}Z<aK1KJd3lR)SgSvXLE1gma_8P&xZ=!C0vYaT$NvBzWE?gdnT(m^lC(<`+e5k
zJ{r8M?G)$My}Hl)^<P}m<otWjmQ}61ueSd3_l?hNw%&E~EPHytQDjG5>5W$|H?8F6
za^Kp!v-k4czcR-yQmP`uD_%rC`W>QK>ti%AY2EAnX@7U!=~sT(pewmx$%oue@3#v6
z`G0lQqOJe`hsXbmTalG#{c^G9!Pcg)i~(CVANsGi=%GR`yQjR7=lAYVKJjTCO|vB~
z&AiPf#k48MJhR)g?zXSglBBQZ)0Q~B)!E>&{DbxpZ$6zUC;qiben+;wnRq=jWA@#p
z0qduDz1;7dRxu@6tF3quc;M{6db9V#+CS9?eJ}b=No+LsKKENQQ$8!~boccCsu#I;
zwN5ysAFFpO@?pLB$J0{22UI<3A{S0P#54bC{$a;H1@-jA&f-PEl{+@LXz+&}xtzOT
zt(GDyv%6lsbkW2ckDH|}uI`+$SjKPSG8Jz3eNPzT8&4d((Iwi&A?#h086?&GW9s`E
zXTA!E`|U_6wmk)2y!PLEW=q_0b*m4PpR)7M`aMNeY^G~=^hKuoQ)EPYji<3l|FTG(
zB|T-|lV@_V|L$8q`hUNZRVlNWX@Xsm<4g7D`wdN=PcPq;GFhSU2*X8}*uJd9tuyVi
zgzmE*XgR3*R<ZB=n}g>sC3d!~KJ+sCR{Pv{-1V^*=^wqU6n%JNKix0A=L23h6&upT
z?jgp@6;b4}<U>&xTboYi?5C6VPhu`O`ZeSthxUu^)$fn0UQe0&+3@G*_YFJ$7}?j%
zv+!H~P+|Wg=_QB0oV>e3?9Gd2?>3Pbqb1FARaZZakm`CUGNsvB`QdI0g`FQ8GWs_z
zROPdk$eWejn#q}{ak6(4cwsKz^)sw#MxRUO2Wc%0`c*!mb5YE0=i<Q4`<ATUa-{Cv
z{G9yb{8PQ0W@+pRm4B`&ChD6O6QeqN_T`v=5kW;K3!lAL{S_l$H<`i8zO7%v<<bM8
z=j}FIT2-gHUt$n>e*F4^vgHRgCV-a_^=&)E!f?E+iL*h->%`opwsUVYK3G;z5NYCC
z=^2rdq`|r5{Do`hXCKLOUMZTUIrUO~?~L>*)91W-GCyZ$M&-$)(t)3ni<hU|3NBie
z{_u{TW#tmpO}VFg4gY0+X6aySefI2meV4sGFIQyQA~iw1K(6p8Q0#N*xHBp-JG5l`
zdMCNEJy_1wIji$_gM(Rvcc6K){>9K8YzMfmOSD~Ld3vGUXVZcW1zMKfJ{*=`{4Mr_
z*YVk@&8(PG|NrNqxcmR3Hu#j@E0)|jb*Fu<y3F6c^L*2$PVQQ|_Mug7&b0VCp(l->
z^&YO!JJDZ!D7JI@oxqLZa;htnH_J}fjMe>j=7Rs03u*0tbY(86XKP1It9LR#_hEAd
z`;=z>cZXFSmYCEmQ{Te1Xr<$>!m2oXUr7!5?d<kPz0$u5E_e1@;aYvk;^HpZvl#)R
z7vDC<N!;#MI%*9$(XOQ4(DDAuZ6BYD--{3P(~!T;XPNl9!R&InkMsT)-yU3gAoa`d
zyrIqX52knHH5ju}t~RZYjCuN;Gh6P{cCIvwGrTsZcJ%+y{B`%>)@-R4zq<A<f39My
z{X*!)p9;21dlsdO{4M)0%<<;LM!7co*8ft||Ey}=w(qR)@`K0T8|iOYbM@n9Cm#EM
znMZzcELrUHq26P&lDfm1NDD(5zS1LQGYhsT&so*v@6)#6X+atHoOmY9Kg+c~Pw(_j
zoBH|V?c}_@OMiBo|F_Sz`?LC1v*I&_=Z%-y<-fnwuS=c+84dej?j5EOt6;r;>!q_-
z+BkPY&QY9WciwU3t);iu@LryK^ygCNl;=xz3$2zMFr6zeJm+99|F-AqyYwC|Js9h&
z{FbS=SaI(Uw{I?DoY(u9jpUcOb3IhKt97kZ(f_E|;?y`R``89E!FcBFg>NsI-MxP6
z)GdWwESua@eEBYxN0xHt8)x77`8!pQ^SOk;S$W0JEuluwboC-Vysy}v;9q6DZfnZi
zt*JlL`E0KlWS#l*f$gV&|HY&X>Hl}0rRCdH=FRDvzv%VU@(KJCEgu&ZoctlRU(m?|
zG96s(vpI4$*H)2SyNqNL*)8@Dn6<jyer!>E=<oQWR_1_wkcqX)!&_Eoj{kAx?$GZ9
zO}kpKo|}8<dE4f7-8bAtZpg4O-l&?ikNf41#eAQf{bG#xrs@4*I0?<lnn!Xh3SKSP
z`6+Lc+NZPA#MeeH*t;s0`}(Juo7y&A+%R?3w}~fTg|$AhU$A$Hq|>6#sV{_o=}ifU
zx!bnwV(O(_u}@Pb?4Gve)THG<s=F??Uh#W=a^*y8$L+_y?MrGcIm3Rd(xKYZc*%~m
z>-LR058HOCPLG*wuU7j&xn01C<6nbdh}zeihnaT&(2IF&GePP1+pkxa6u!JIyK296
zDvLtC;b%tqNu2CAj5t2cI1t>h@PZlJ5ta<@<N1%~9KGG`^sDuNMy;H`<`JExLBG~d
z?A+a-*!OUOg~VYUv5S>2L~a}lsbBHs6!WW(ukO9tw~lYqty!uHkq7tBYA)MbTdT^g
zZJX_)(yn#X{7-Dq^;)0U{GhF8Uaj(eAYSBTnQt4D%fQgU;OXKRqIUk%wrQo?1x~Os
zWGq{=FvldDxvfMomFwZc37=Z09Of7QaZuHMNt$iz<<`TRQZch`2ZyabQ`=Q7W|yf`
zG3{B~j_o_zFCLLQH)-a9`RosJJg1!Tm*6YXoH8e<<H;ozevb=|UR?R#%{QrU^7z9o
z>1fWhKjxR*L{rO_;w<nS(Z%a?{v|e)?Jg6(%hoU(bh38Qr;sW4bK(L&Ol+=|Q2REy
z{Z!uP=?O}|dqA--5#wZZj(MwtoYnGzLq^_FPMyID>TFY%hIlOv@lq2AwgHd8#cyMG
zdH(SJu9(et+;;5q+~^-SKWN&&sWr>9-e37Lnc4Sw*{cU%JwKdqJHCg{tnyY(!qMAn
z-+x^d@iV7edG6+5@t?m->veLY=3VJ)FnSoN$k4s@Z1RLa4u&L$35!0qpVMSn_366m
zlK4Nl!k5IY(iP<;*tbkKarSv7%J4wv{mKd7oVo7G)m3W!cNcjbnYZs@rD}rO2Vre>
z`yJa^YCa3=rY>1J$)dHnM<JVKtM#PROV=YD^Mm|Ab6l)#=Es&Vk(Sh1vXw!2p$x-B
znSz`pu?q^1c~vqjFj->~-o+{8{!X4n)v`b`d5-6!2YV_!4xakL|An<NkS{=K2WNRE
ztJQa%Nl9Govm;{rQYUYDC38Edd8vM`YCI@}{+s?f@X~JD>Z=SKk3<?~f9SqEx&D$z
z@pZcw6Zai>W&NX8<Xc(AzvTsU|2ynF{`fV+>tmm1^MAYbVsfluvmWF2nfw1bP1)uY
zvg_F=cIUX$Pd)$qP4c|;!!pbyQte7ckiLC`Fpoo|QMjW2Ly@rRh=5Bmn)+w!|3$Sp
zzgpyZygOXf&s%S2iPx!)%MYdA1Vp~d7cTHzdSQ|uLsLv9=w$QL>~1OGv|!WfJsCVj
z|7a3p$*1YcE#VSIOjC|rpHox=I*FEH34_YgTwC8IvpH5>Dvu19v{YvK<&8Fz{!29|
zmwjWB=weiTE5>Q*9r^^cOx*Mr!;vNCtCqa=j8|oCIwTeJW2tXM-ASfahdkw{R!1|O
z7b$w1&HLAC^HW{5vFty0gwUtV?wzNc^~AO7Q`b%D^L9_*2y%EUvDc`|q2zpp>z~y>
zju(i>JKtAq-CyJL>=1WY*90lU9fEpGg_*QlzlV3&%y_(iW*UR*<eg%dl;bp$FJ?V?
zZkgU~6L&ZI?)2G#uCup`opx!R+__m+-yy}*aHAu4xZ1QxrB(~Q<45!+Ysp9ZTb<gJ
zbm=+AOYUirPMkNaCI@#bU&!34BYfNO()BGOlfzbCKCn?BT=AA|M^#wI86oW*8*?V5
zhI$@XZLpcp=UrAZf%W;-hD%<}VxT4I*H64LKJ_^6*mmDJ6)V0iT-9#FIN{-LolIf2
zSrZnmTQM!AWb&>P$4*K)a(Nj3%x3R=wry{}p7j2{?n0@^VvUHbO(EM$yq>%`!8q~n
z)t&W!X65!W9(=le`tNw2oii#dzdj23>hRU%>z|hAA7=|~(p$TC%g0M<cU9j;%{5C*
zT~zH9Hzhn|`ihAG$_;)RlY>uih0XdrS=C@m#?#A3ia3pS>xtdq1&uIvU#r_CF2CW_
zQ`4>o&l?K%eXP%Ot#UQ(p2hjuy+U4h`*B?<LuCnt6DO43xaU85e4c6Xlr59A+ZE+m
zKeW#+n$fc7v0~%2j3kXnlhfWj0+!)&3=>~SX<yj+XUlVo!+OHjzb;gAc?v3=Hp#S4
zO+RVX&wfB@T38Z;SNpG1G1ILh_P0uBGp0{j{`Tz4#5Je?|E@|rrtG|B!ki!1?-=SX
z4boCQeR9@+GhU-FLA7hP`F+~@<nqboe3K79G5EOqk@>}Z{}bOPq$$2!^}J73*8S0+
zh`LOri+>L1w!CI!m^C3-eC=xO)!te{lj>!+FLdQzpF3&3XX>V?M<xGGelDDuIm@T$
za+zhOhUzw#eG6`BKiBX4-rKRItH?=h_0HuP5{w3hPIbnnNrCT9Nv+$}z4h;@*j1&0
zbLZAM)v7CMeRRBI{604BbItu<|0O0o*7tljKg6<fTiTSo9d2$M8`Uo}N&RwXx$<e<
z<EKymcG#$&=IE@Hel5LQ*-%AMHF5WO+4)!P!b7w}cZXVAzLUKWcZsL5YN>C^;-yAm
zw<81?Q&v|d{z^F;`f1h|wc3+RMtvrS@;oj2&sKKM-lLcv+^w?p!3ow>C%-##8HzE?
zFVd&&k2Z4L*Cw#((O;g03vBMjPyRIfgqwG2#Kv>8@_$HeXWTCENuo^pTW8NUV^J5g
zwmi{wt;OPtk1-W<?l@kUV?I+sX%_1(x%XKI?<`Jz*k;+*{K(YOw5T(W*QUGjsM4Dq
z%ugRjx}5KQ^eK9|r$>kCWC1^()m~O5BHAB>PaprIKgmxiIwvoF!X3lV$BXZ-Go9Fw
zWT{%~JM{yjQDE`@;H+{3-{Y5D-gx)rnwVOf82jF>D!8EJ#G$yIdtzRc%oYxBjh7o-
ze&{Vp2|52m_wbRmkX2a)QEAuK{`vhW`_t|BkK_;E<(c#`+RZ;R?%<Ox@0O@5wyr$=
zJ-qYAyxX~l53as>E!^(LgU%&}JN-F2#V_rCxLxYo+59Vi3poO1G^FaiR!;F+nUTU|
zc%ONqXv^(gEgtSk(noI{S*Gf~w{AksmHm6}A8t5SntkQ2xK;+Y&)f-@k9j@3H)YTC
zOE1J;&He0usPwB;YW?)N-_|TD66CG?7xDMjjWd<YCf?Amo#@r^O#MXg#no^6&*jGT
zm)wba80{N&Y3UND2G)j#?5@|2g2G08OSFybcDujR?^JnI`)JxR=}hiR=ThFBI^h?d
zQZwyB`<YH_sbaHRH(l2&t#Z|xe9bMsm$P%<>aUkhl~(T&uPFVJC1<QvmbgVnS?zDh
zv6aGi{oY%BTqB!lyEQf*eCx$U)}v3;#ra-kXRmtYp0VzgSoX2Dj8|6eXFk{q7+kr3
zB&omclY;!-2St@{T4ze8Us-G{=2+!h*!xG}_ATzLc;1-R)oWwd?k!sM_GPEK%uX9!
zPqCK)#@pHqS7b5=&2QoH04MNUOSev+qo?W>^R!vzwc_$mbDH~uv%hz}Q;+?wdd+Xe
ztPQf~R_142uc*EJnNcO7`aoR3OSh-)vHiDZ2U;(7Zd-oLtK9jr;*!L>jEaAoud(Iw
z|9V>cQ(C`ple}m2Plc1e*6p0@d3{}Z(tC;bW_#01?X@>)&QVYeJW;FidUkI9pU}+?
z-x=TC-|e2rcllJ(gems}?iTF&D5`j*Q^4s&<)jNP7gHi8FHB+lsCrg^?cOIPnZc7U
z`o@_`z6!cny772^yQuqo*%BU$nA9+b>IdH{kN)JGuC{08-(6Xs_$0WuPu9!7!?tbJ
z#a(r~O)IB=*;`u6?`!v@L?h5L*wVGqI60~<nC-aXclUMtw(HdV{0h8otlPP|dXZ$b
z<3r1X?sw;H>COo@&T<S%xzhWCwe#+y_>-SiuRpDxe0!hX>^;1v_;1R63iYtM8?109
zvBE3PJIeoEWOm6~#;q27m;E1_ce9#zr@cR|zjMj^qzsSUKN1#np8nq*W9)PKn}ho$
z_hL>3WpUn;-B}G;EB0*aIdSi#=Hrvak3)Wz-SpYXxnN0Sn&O=Yy;C};K8UDW8Tli;
z)wJk>;l|BNuHKhf^JGr-lZCB2KW)%4Y89B4^lDZ1x&4#lwN7guJ=JIBu6uDG_ucrv
z>)G`jy80%3PU#NbwW)Paa$B-nm7Bli#0lk-&Nc0Ov~k<PCra_smwsjLpVnI+sra|f
zvD#zhrmNNK4HeaO4$rqSDgRPjv}O8<{#OiVEiQ6u%siu*)^Z5cxlb23@n-YmE!sTS
zR)v*ihW(vWb?cq*-9Ni;t4`sXm?Wl~l;l>n%WCoC-WeTl1yz=%zL}iyd)Z+r{)}73
zAM!3Na5(pG(zX*NJQMkip9p+XKcaKt)g4uH(a&$>C7xAm4agO~Ug>w`$cdE~Lof1`
zr<TSq+4r;MPfE~Vp4*y}_O?vmYCXjPIzrz3`aQWTtFASqmz|eCBEK?n+p#GNx*Ls|
zo3qY_?)S@Iy6xc0e|3yX|95i!`#4)D?NsO`ucG+b@+z4{`fn4~#$0#Vt5_TS%VK-H
z)3hs}Kl*=oEn=W~(Na-$q3`DFz0zjZy17c(>QiH9R3@Hcbr(ug2-~H1Emp3yQf_^J
zdEb0N#r-p-KF;-*ZhvxWUy<uk`)H4&VJrolnzdYZTi+Y&#>wk<ir@dbqx$E&{;Tgo
zYM=bKIce@WY0C3|C%!H@ezlBs?ecBWA1=JTxMGIc_j`BZg>G*7)_5y8oA;vK^}I`I
zjaPPV4SgNC_V(#NsV>(ZB|Un2QGEI&dk#h0S!c40)|?CocyMn<+2VV=@!vnb+k5x7
z+uY8sZm&x%ncrVd-hJogqu<lxdcMDWYwY`a_0){d3;tC7Q)G53WBd9gg!Q`8@(yXm
znX8T78m%&)d+o-~mnu^|rY^RswR-yH+GXWOaYxPl);#mOGkX_*!dblu7I8j%=2R6<
ztZ$5OZdkM6$N}y~ZoN4>1<DmU9o{b2uWJALP4D%qq6=%|e>Bei`OsH}UHi9n`s-j5
z^)JdbDt7%f^O~nb*@WE*o^t=!?`vzL=7&|UoO^Y$-nP9ydV00brp^#6KWKYy?havw
zo2k>iKi|3HyUSEkOk?+!Z<nrl&rXUva<60G?JKK3US3w1nR<D7O3a?}Xa6RhSoZ2;
zmVIe_^X!L<R-E0;{IPbPZtmSJrAMzloh52=*?e|c#;t`_?KbasxXA6ft)J8QIP`dS
zi-42VW4olKmOVvZ-em-CDRnt8Uxux7*-ppBUE!tY<g?~Yb+Y$9>^n{73S<_&UgrER
zP2tB2z8vb@yy3_OA&vu2MQ{34YUSp7HhW8zTzVj+ed^)5&aJ9}>x17bUH*SYQ&vND
zle|K#*S{@SOfvt^fA!w}ZdJ#&<!S9|G2R9jjvVZ4`P|O8v2W?rIk~D#Qwq+`HM$yC
zZ=KKJ;8~M2^VyQRq&Vx;DVtg+1y5-%F)ogM_h63hw%4-Xtn0Ox<j#NES}HtU+QjZh
z%Zt_5yNzExKP~!siT;u23EiSWhmLi0d~VxqKl$^Kgm*_Ymj?Zs&6)7@{~G@ipWDAm
zE_!$VV^mDhR@@|!>1*`tvLdgQ_WW%(V}55xI|lvPKi#KUEHLnI!<oD59O9PtUKCm@
z!miF#z<H|hMuLN8W~`gM-dg_iWxK*^ySD0U9KZZ7O<wWa1io_)aoQo_M@nzaU%kiY
zqglz!>fqLMbsN5l6kcAVzbV+|;hGI^9r(PbSgF7JV1Cs2XWC22hIwmBZkF6sy2;ca
zd%<tAUD;&QiTTrBsMNh*vjB2Nx1%PHkmjX8AFfFUnfzld7dsv+VR+;z{pI-l^48@q
z9P9(ak4b)g(X-oeqiH(pR)z_TMr-HUfAM<#m{s##;k&AG=?vy@?+L%WOBJTD={5#7
zT=?}*m$~hi_a58YwyN8QB^pJHO?KVk-M;#s@7}+APdnCM2H#72`W5RG{t4pi7XH~4
z{mQL9E=W_WE^9h}^27NLp9db*+o`bB@l=d~R>Pl!yo33#IT@rZrbU~U&)vpw>%h}J
zt9LJy<Sx3yrvCbOtInn68QWgDt9<dCBOJbL-mZI>s|pXOO2og<j9aax(_*Hg<Mro}
zq`6q;&X>+cR!LWP=CASRU7ueAURO}Dv}$)1|B4HzK2PnxWqmun#Xn-}lU1+(_%nVl
zh}XJN@%Gcj-_!Xe1TQeWYBjU5{(MI{j@7HorvA#jm+qJAnfS7&z5brL$b!#3b<5c|
zeU?1wthIj<URvA;P~5JWpV!bPurX-MK?%J_&p2b5bnh-%`M`Sh`ZsA6_e0;RU;g1g
zCrR-1s!K<v%LM84)oXUlXK;3OdI{Rxxlvlc@A<9z=-Z9VQ%Ze89)`Rq-EnSh@{hxM
z9Z&iFmOn6Xp1~lJv|RqzhJYFwJqJ$<K5^!-9kb`H+?!e~e@f)tyT>mQu0~&Zb!ys+
z=vy-<FMqew&t&5CN7AqMGyUwH5PzJby?e=mCvKakc&&Vr@$zo-sqL@6RL{KpDf(>9
zjqI?QzqzYw%}%7c{*A1g{IvVJrvDd@;=g<E3vWDkWZC>}A<N!B%C3`6-63smlj`zM
z_*(3aW!Xp0i#>X$Y~#SA#vrM1{8iEN(~I@aGYVY{n<3C6Yd`y5`1{rIv;GIaD=hqM
z|75*Ur&eLpu89xJsyxzylS;Q=-&}ZX=7c6k^*gQc|F${CTXefQaVQ$qOHO?GKx#R+
zx?^JbzA0ZOus;%c(0?lM>AFX4XP5U`^qM^R1lm)Se{5Q}iDS0q)%x47Hws>!S7EbH
z=CJ0ZFY&K(Odc23-3i{6(y-icvP%7q>BlWoU#DoN?v2>{LI3@AO@-6CSI*vj%eZ<j
z<VMf=Ql)7>G$k31IQ>vo*rjzw`>66!4N3D;s~+|_%Z4}ak#u%eVCef$$)MRUeQcTS
zF225)TdKFLtmfu)z1cRU`R=3IM;ed6$GqLT<m(grM;94|_{H)Cp4K0j{(sM!_9MK`
zo($hz7!vucYaUKmTXUe$;!<tLZ>L`o9>vY;#C1J>vp4YuzO=|wwKEfA5Ai>F<=#Yp
z!;Kf0+^ywMd~b97tZmbzZA(%#_eD;dZgN2+Y2rywy+8-0Tg7wuY>vEPo#y^YPN8$Y
z(Aw+$_0gVdgSD^BwUm2xg{w4O=A_X5rF)Mu1|+e|Dlkkem|l9%_H8ns=7Yd<S2YgY
zWxrf~wURfm+V7_BoT#U3gtwY1*K1C_WZxjSr@|~gUh&#r*JbZc2Xfz5v*cYDx9?ES
zN#pyx>t3(b=D&Ys)%N5I>Bpo0ynE9)e^<%(#>LMLUb#FwOillH)`f43C*u#8vUNzG
zpKmoc&Gy~q`DQBfXV*TNeod3RJwCqn^U)7S7k|IK{MzxQxe`@r<yB9Xw9OWtG{yM)
zpFLl{wydB2X4Q{x+B`lBSu}XFYD8y68S6_n=wIPGKg-GI$wTd!tViWX_0OMQd3INr
z^sS89zByK#GELvCX^^e@(JYo)erA564EO#CMak|l^P94_-#WTi&*FP@!j#>KZ%`L1
zF`qlH^n!cGAqTD#8yTt|eQ}z%R&}lV|A}nrYSX5Gt`Sc@s3qvoziDdL+D+3^cAEI_
z6H2MRw|S3M)lG*w9dq3$F_{{U&o^9KS$byIj2SbwFEC=;U;5!``>uOS_Pm?5hxy8p
z1ukWh4|d(StD<6Re}R#6&i|m=qf5jsmjBsi6Bsvj^Sl_nvy<1id@9se;9p(y*6FRu
zTOWp+(A}&u-zWc^{;~V>YxX@tM(Z8ixUa8LKb3u|^#Yr@_Vw&_#S5Pnh<lr7m#m&C
z{iFEDgN{G5=Q2hmp3Ghlo=~=S(*^E~*$dkh1HOp%=x_10_Og!K&7RW!VM=0m;*N_)
zmbAQMjZoSDPdW9~YB7Vja_4tpyQ2AYj515j&M?kUpTM;D^N(YDyRtcsKdjnRx@mpJ
zxf#+WdsSqbGmI>E+J#B!erD=aE5F@d(IwV=YWC!$W1s=~R+l3^Q{7JQ`LM8Wf*#AO
zi|?%ey!@ee{Iim*xaVQ79M8_h$)AMx|K{@x^bFm(>afX$mXbO0395e%#Ri`~ZGTX;
z>;mJ4j-^w#JP}%b!jiRW!N0P53?7Y9Thx1n*WSLkVe1sHl~Xdhc&nezm5)(7eemEf
z_WpwB)n<7U_!0yeR@|@bXL%hhy1SxevP%<7VeI2z%a`$2jeo^Yyji!u^g!A6(9oPy
zKfRafU6%Xx<gGD7>-kOh^-lk0^O~*k`cw5xuK87_(hGM;?KF;CUfl92f7QHszB5be
z?(b!s71<nT{`+vDrH8-cyj}mVY?aG8=kW37;Vs5h^5VU()5~t}pLcilw$oA#htAje
z-;MgPEcELH-ATOmc`|oT{_VZ#*xhhPYRA^H?8`f&o_Eb%z2)WovdG(U3nG>j7qNRN
zE#TbbGi{2=$JAxv2QOsz%3DaJ-nd}BIyGyK$1L#jC+>G#nd<9U&7vmoH(HrKbN%uw
zW5WdH4L6fL<peBh7~U|?ly}hE@jCk4^Cj1f8j>Ee=RIq`&OCkPg}l<W&iM>1ckX?9
zu0Gqk!G?bxQ!`7N&vEd)!op~$HLvB^L|^Hxj^A9S%f#&waCg_=NB@rG2CnUh^3UGO
z5~|~H{)l+zzE19qQsG;Dw|baL2W>dxUleTRQOB}CdRh{byh8fqi@t59l3aTm!0Fef
zHJgR0fn&aWf_dewid)}GviY_tF*9F#Y5id#6JHR+^#!~N{Vmg&ulSYtd3fI7TbpaO
zG;?xV_}9|XXw$h4b({@CuWUp9f0)$lmcf4}(V+K$Je$Hn)))M5j~IpgP&jyD!fgAp
zKCP#L8j`m-Up96&wln#@X-oXqV99buSR!@S;V;L}-xoc=!Qt$7%~ark?}SvQloFW<
zKNW0+nmu?tEtkuPgvp-Pe<<$0>4~~n$`Oajs*iPM?V7{A+RxFG$Ch>LoTi>u8Ij89
z+m5$|6_k0cUMkBd$~yb&+-(s#=P%>BHN{h7D|q8{g&2csgU1PG1<@dfy?u=<!3Vz=
zJFTC;Q8v{k^IgQ0AkDwl9J{~HF}}ll`czz2>5mi9`z7a=-SOYY(V+kEx4HiP9G}_q
zck1qU+oh7dBk|3{<>v2FGd9<J)jQ_K!18(iv-9S8#cN(}IPuHpv6Z8y*=<L)TZ>}0
zYAnyturKdhw`bwcyYC8DPWl&ctG9IH&z;H(;?{;-Kc}&Gb^rNGOIMyxu@-%^;qT#R
z8@@ij8+Ij5XD7o_M@vWFnsr|LwnrteP&og~wY;vO%12S|DQM&>N-%pdQ>6D@-O|fW
zdvo4vZ}UE$A(p`zyx#kTfa{&D-BFVqEnc7GYMBr^!M=LZuSFir*S9^NYPdv7QN`Wm
ziFr&cr((+y0l%l;pItVew<uuxT~UsQD$CwqouYm3s$ctl&WkM}+&g7L1Ty3+TyF)L
zg=kG&Kh<<%-%FdNk2k&JKJxiUEN?8QqAv%d#PxS6ueZM4vV>F8*CTiTtwmAdo_gVR
z!kZ+UOlqYr?0#_heyRG_?9$wYEc5g1d~3{h-`s6pSiP9f$9~~E79;zWZehOHIHsI1
zJar?ye{-kxPS<x0uf(qGSmp54v;A-5;btjzwbHY1eRc`OHeQQ6u2~x*9I9-|`7!Oo
z)NebVY`&I0%WcWYi*w3$tciK{#d4?5r4!G$eHY!mG<gYoivN;sJJr3jYkskL$L*GT
zQMK~2fYXTprzdwkgiEzE<L7(d`}R)#?)6>Ur>>v!c3S>7MY|)*xvqD=`6B$v`V-U6
zy){x_obLHv3)ptEy74^6{H>y9Rc}9=CS^)895Lg*6_>j^Y;V}!h`Wp_MGVhMPTnfr
z6?|PRPG0n)oBq|~S<>0ZvtGYAc5>Rux=BSwfemd5c@LF8zI=Q5=gZ<;4Si|FY==5U
z)?@c~$GrNq;&gH7x>L@fu9u^4J6ErJ&cFBD*N;jq7J^B$SAUq$SGec#j}sqvK5HxM
z3RBUauy(R-z5IjB7rPn!-yYlk?(RB$iyT3xSrat!j&Oc_?xWmc5mghqW!vW|eZP0<
zZ`iv&LeW?KG>0O~8&l~yYyN9IvD_m3WJ-Yb7mjE161x_5p3vB@-#w*BrflXTtAojo
zaWno+T&AmdWS-CFn@XM2%$F?f|HAWsM$6UvYhF7&P<A-R^qaZwUdxNLtWNNr6OZJj
zcT<{6<FlE!#H{h+3v23q7VcJlF?q*nE9Z{+5s94-^ezW&_E5Z*ssHJ_SMeu3g^4@6
zit6SUHuSu@{fh6^ygQ~RH04(ux)-k;;W^!3Zi0c*!MaBU)5FS&KY1)Ne&JUf_oRN+
zInKH+=9Qf-6Ba)R%$wxrq3|;ASWETtH}71%R2MA$GJgTrd<*_fTC<CN@^fP(u4|_T
z{Nq}q^nTBVdu2~oEzSHr?T_Hn%w@aN=dY|#;4HtReOGP&{F;k~C*oB81x7l5y|O{6
zp!cwhYgn_x{#l=%UCUjY-hJxT_t#U{wqBKWX5Fp&<l}tPQ%OrqJ5Rc=yqweCc<sCF
z>hKv0v(~)H)Co}Es-)<zdgOWe@tb8vPAA?kncyL%p#AsUkJqQn!{=sO3#`_hnq#}}
z#tD}c{hRLJnt9R!4mSp~pB0r`J7Iq0gvryAlA?XieO=xZ?XozuvCsE*S&ZtDZ9>8`
zO5K;n+f;txe!2O{1nqc{wR7b!&uy}P>MF0_DV{C7W$M;Aqx_q~lG|0AV<r@z5|mNd
zlVO{5w0}#Bx|1JslMQFt$JdX~Hy`f(vFSyZ;k64lZg@Ol|29#eRYW>gq}QveW@o~`
zsm4+%KI{UTHEgr0i-Jy0oX|1Dq~^#=1{p=x)B1&V6Uq<8)|{Q{{BsHOmnVH?Pk0kl
zj-(!sdF`VnzhZ%FYeZa91n2Wf%HF<d+>&O~3w&<}&uZ{iIl83y)AQq9MLMUK$hH-9
zlm>J(pApi&nb~CYyQ;=&XLR{B!%5Q5V!s&0L@l+vy*lBv_@`;rYb=`O=NQ%aU3Mv)
z%zyUjA#dS-Ihs>1-FLllNigVulJJygzr$@-C39H+e{gBel;6VECFjcZ-iO|O+IY6*
zW49TLnm^y_dnxm>d(X+PDiPyl3E#QJZKlxO>vwOP+*mHP)8?T^O4jBBymrMR{=4k?
zBo|mPls2Ayx}@pc+*3!g_FXz_x-GWs&4$uf-GS3@->|BSf4tGZ_@=;w8C6oJZ6=ni
zzJ2Zex<zkpBxd)kOl38-YDfsU$$xuy+Bxpb|BPp|Z=Pe;Tq&78h5Noyp&!FUlQl;c
z2y0~J1-+Y{xl(_AxW`+D{gHX=e=mQozU`#t$26{(%lpsovbj4W@5<NeTFd`DcheM`
zPZi~Bt#khQqG&^V#fy^{);o(IF#W>@UTQ!6Tc7x&1efm~6QYHbMfMc%&8X($yVE+)
zY)0Jv%o7X`n9FOeQ=DEni|r7(q^x+scZV>~YPS`vdrlbc5IN^8^C-^4+pBMepy9{D
zzA0N({g%ttHZXW?I-r`{eRA$TW0$hHTa&tTcScGtvp&sK|EJw{lgLxo;@1yvbuulV
z{ekPCyOo4BcT7^MId{yV*t&v~OjEY#2ro~t_713dToU!RFLllXSH1^A57d?%-uc9W
z^-<ESYPX%2`EHuuntJH-^dQZ@_38nA&1;twHvBdYN_X69dw<_r#tD1}Qm<F`rhA{R
z*|d0*<FCo5OHxA%84ft^;4b)lW9j|ym7C`*nP)0v<}~wL;iT(}ocK&N{(H|X;Y|O^
zm)iF4#diOD!f&h@8=AWIbcODG9>kYwS-$CippD>kX5CD~jla^rO)poQ_i~;z_j#^K
zNuPx6mo%?!itplNyrC;8(Xn3h&f0@95?trxKW^vsd!l6*KfCnI;twUKPu`itXfSn4
z&6XQg>I=3Wy}Ihrs&0G#ja#Q#{p~RQ-yNpxz0T8He&-*Lw3DX}GtHW_Md+Yi)U?cX
z<(6t<7gClQR4O^tb@un^=smTFagI<rp4;{CCx2#vXLNMY9p{%3$2x-_%nF;BtS%QB
zeR0ztVFvAuON8xNEhB}_LT+)N;$`T)a!QM^#iNF`DkqF07hZSN;+!`9+SE1a-E5EM
z6ilz1W-qIw|8m0nHOm_u7I9uz3-YNsqwqiT^p+_ds%sY>RMp*I5|>!LN@4q>=TG}?
z{C!k&>s!UYrJuH)yJ^O(C>#G+<d3)emz1qLGoI)Dzo__sQbf0E_Zr^WM|uD8%6VUD
zDUS0v*}7$d)%T8bhrOP+`>vU?$kVw|-D!cr+2SZCO_wyyJ$l}sILz+}1aWi6Jo+cb
zdoF2z{+zz44<;}rOccGxVa+T#e^OqGL9E&V_i4{}yxD#|c=9n(-WaZhN^&BcnmPrX
zELug3a!=PNy!sJU7#(A1E##oM<nr|o-A|{NIFzn?dS&5627%1ya(Q?BrPQOgAO4wX
zYNdPeNWzwkpfg41W!_8oPxmtMe!1UsK^t3r?!V&N2YMdDFXyaD|9N>~*~6ubj?xdV
zyL4Gr-8}Jlv9^O+R1t&S!k3?ArmTupJ#le;&$~s(`!=na@x}6GRCWFF$_=Ftzg@`H
zR@oH?UPbhF`vxgL@QNX?Zyjq+Dsm{Db$!41yv{;q_r(_V8+b3D<N6|bHYW4S!K!%4
zyRQA*n)8EPHZ>(&e&t=Y**ws9Q&Xm}6URZZ(it0`a4r`)RR6$6;As82>vl(ix6H7U
zZsUH@dyMs6(dWe7&ZX{?A9Hb6o{Di=`&3@DYK`MI-hIz6Oq#O$*5OU&>%fa`TIX~)
z_xdRsvwZK6`s=jLo2R1ib7ET|zo-7rPpSVtC2uNUUhz4~bCt`frK?J|8YWM=C8)hy
zQTBlRT%|p#5l$yJEq76WC&1Cw$Jh8K^QU9qA<p?{o#$)uT|f41k;g~AIeo!9#N0ez
z-N<=9-L5Wy<@d?ir`$8jGxV~qU4B_Gw@9fntjN7M*U;L#$xmqd_4%o9PUpwAT&`7<
zjQW>T{IB6*>PnH#Q@mcjR}${nx$gh0udj>dmYy@6YYRDjv%zQEY|+fFWQ(IkO{$#w
z{uQ2&)D_<vZa@3y=h?Lr&*gj0R+L@Wx;@}T@KM)iMg9AiyJoL_x;dnRbJ}@_YcC?~
z_`6rTyH<ytFg?py%y}xXH1urv+;gv2$S*&?_<dB+zMU$9-dluHJ*Vh^k5DY^de-N*
zGH~OikHKr^$akuH-7KBG^cPR=-#wla^_9w9)`{~ODzZMA?(|AXep78;OxlY{k2N2?
zIs8HL=~4lRFVv1~vDAvx-MDaie(}j?6H0dbR?k^vH_1-XdC99MJDf6?e12Yfk@tk~
zmNio@9ap(6zvgt;`KLbT>XS46m2Y1<<)7DwX`Gei1%Af!tU$&Hu2r4dvHD-jzTSVW
zOXfbAvSd%8Nu^iNi%_O_PJUsWuF;7t&6eO3*SoAHC$?<<d`<d`QEi)1#LPt|Ne*n5
zqMx7GKArQ(;&E78b{j|0rXYihANZDw^FHa^ecR&nap`Z*mu9hOcD&8MHGiA^M99Sz
zVO+<Zm%iC`HmFZs^471)$pwCj*Cp(ncCAaDx7RGH>YAFj*aG<#`WK4XpS)UB`nF`(
zj^Blvg`T=qTkWqGOM4&Pw%_Z>w>x>#P3zxn+2`c!=AH3lZCmYU=|=IjPnS*&(wsV}
zU{l)N1l_-L{w$dMm}}|8dvjj+ax<t-)3_nzI;&}dU(1AV1>Zxtb8Ba-TP{B7oyiy|
zq_ocEdiL&VQ!Yn_zI$DAMs1sl#K(`S^3$&WHoUFF5;HBPx@&t1gZZoGS2tf>`;%H1
z!WSJd{R6-8oXb7S|Hbg$b3A5nIQ{4CBCCI;VGQ|SWxqc5aa8$pOJ?etX-g&u*K1yG
zVK~OPWbx(V=NaF!ie7F@e8D+kqMqc_K;}CwA`H_5l($4)iZt?JIuJ6~+I@4l@<Gi2
zsVD718?Fevdz4+5bTHuIgP=`bQc82@esvW)ne^Oo!G!%LQj?OH96UJ`pS0{-H|<HG
zrK_CV)(SSodbT-}b0$Z4^1XWXs-y4p;nukKo*s!oJ`D|LcU`kc`l5K-gW;$AORqOA
zU;aBk`EsMIVTni#=paqrBc6s2mrQ<YI*mWpPGZRnSM9!gYKAY4lq<?AJe&MCN`Lo^
zF2#SbS$h7j*3ZrQBQ(#{BC^suTlKofZ`;fEn<IBc)gHNS|8{xx?3x(cvYTiBKhOVD
zFP`)F|D|*P^(TC}@B8eOQRIaQ(_%hNcbLvrz<WTxKDI!7ougl8>a;C?CaLLMJR;=9
zFh!yy>7?pwy|&InqBp1Y%Qkrb72m~ix5gvUQ*Y;=DSw3QS+D>3<Q-O*+3=-h%Y^2X
zR&P*!<n{8t`y6-6IXmZUb2T}<P*Phv=k}II(!$$zED9}(u;hCnv`+Tlhlt0Yf1cL=
zbj@QM+ETB~RNmL3TR+|QjJKQ~n6vtxy~&-&^QyP}4y@DO_-K<|%0ZEbW?9p&#BGV2
z@%zegme*^ly01q~`P~D~jE$3aT6Qk*)MOO(oOV+1nL4jtX5Quls_MU8Cb{nDekdDL
z%vJ97x7YhI`*eo@?gMsFd0YWPubONM#1E+6S$a^_##{ST;ohU!lAQ-t1-Lniej0E2
z`dKZJ)x53WN#yCF+iXE+9=*y0uMe+RnY_UD)6`w>R?b_qCLw121%rKQN&O+2ul8OE
z|Do65p<}z~>Qf8W2Wo%P!fH&_{?+Fp)^^GLe-s5dR_ytgJD)=mc#p4L6~0Ql#Jl0g
zLPZ<a{nIR_Z+K^C^L@kf#nB}vRT=;NSU7uUO6BbCYpWjR8VelY)0jDXvdEgB7BO0x
z<*WOhbxv?xS2!y2Nr!EgL|&4nT1@hd_VtSVdXXKyY6q+<#Nv}HRSpC<O!~lW6v(4t
zc;Uz-@2EgOXa9#>C5kOeCpm1X5nO(`JaXG?rv^1YK_-o}c8m9bM&$m@7iKqn3Lf*e
z3Jdltny_)xAv1y7hR2N4{=C!Pz5M;ZU*{UEL*JP^eOGpo_k{I1Caw~zC+sgW%MYxY
zeD~~=`6id{I3+O_6%>Px{b5@&+q?R1L2Y!|eE(0$%elVSpZ@fHUFXwfYu^5OdOJLD
z{Wcw`eHot?ZSB5QKYxow`O%|OkB5n^30Nbs+vaP{i=S6M^G!1rNqC)ibiL#5SmAx1
z9UcapOhGZ9oOEUi^EqdBI4qxiVqw2mn0L+wmkFVpCJ5?B$-gYwrRwx*(R`+8pW^KF
zs~LI?|CTrU>3lqTUBaSiGas+2rKHNFBqupOi$&RgM3y+roX|A&`V!uq9&ZXWe`ua)
zUTgC(@(=d}zT2rxdk%l{-my%!p`eao3aI0=C1;ZA=_k_*H*v(&zbLu2Z4z74oTX`Z
z*IbKyTO#<&@)w84@yZN=&SI{7(G^D-^O8Pvo#)kB8l<)KKi^D^Q!j6%>3HdRWKJ;p
zp0afETfc3#R*!C^tJilHcSnnT4;GBRcyr(SoP8=w64ocJ%3FBP^1sTyn~Peno1HxW
zfx93We1y{ZEDhU`hv|==PL0V^vEbQQbCt2mX+`4KwlBUfAMe`ocGAt4tM7?ceSeWF
zeM8s4BIC}%hUQX!Gv=4dZL1>goy@X3ck+JV>7~mZB8=2+?Dw8+PMO6PT~a5e&@5zO
z7u@wy=^T5Ak>2F}syY*nRGlhXx$3EM24}s&M{j|jxBH)VRNQI&|G=2<=jFRsYOgk0
zuy)J~m&m(xVRHOlJ|ovR+s~b1E@xDD6hDW3$s~!AM@0+`O*TTyH$R!ElVA8Ns^jU>
zf~A&ke=#2rT$d?o)>eK=;G~Tw#}me*R%~<C>K<vx+8>=1f2&UQcIMq#soy(4J0Cfl
zqwX8BZ{E3?#^%O0rObYfo1H(zTD2b4p7VMB{J!6Z&a->}aDJ|-`}}?&-`^|t%l{VS
zZrjc8@mo)4_m9g*+b!;%J@s|g>llmDU68Y6wk(zPo;LZ4{`=zXZ!IUjD`zsl^Y^c-
zdh_C>d4HJ?&pCB`O5rtOsd*pzww|g1FKc>}w?6%GkFhXlQ)|J&nc1~L|2i!0nrW;o
zyHfC7*6hExT*KbGg?kyArf-k__xO|WWikKz_Ita(ht98#`^_+;Nym7>!g?2nwQ9_>
zb#je<2OM@@<ow8;ebMdXSqt^<N6eRsyU!D4Zc$UfHN~@-`G>%B@p}7@uN7}P*GWGK
zSQ~RyD|Bw(eEY&5NBTG_j~wnewllQ&dM5XRE%$n>^xypD?Q{KQI5*_SWA0D=vdb2+
zE->7_e@#aB551V=liKyGf874yU+_&Rh)>S_-vy@~COi)!_8f{S^$B|$;(E|jb{)s`
zV@+)Wzr5b7&r80`7PKKmh}r#*$dtn;rD9DcO8JLvQ1Q{ZsiMa7i6MhY<>_?COAV9s
zc!hOeZc&*bSg)bPA<bgsQD?4D%q)5__dsLU0yl<b6WF&uTo`($^TxERTz_u=oSXWU
z<2m~V?xl(4N)sj}U03pW_?$oXpS`;M7s2*QCdef}cm6CtWis7<i?Nt`zMk&$|CN?U
zf6Ix6|1RV_vtve!qfSRBXF}NJqO;S?qnQ}GrwW^Gm~nlUmE7Akv#U;JrP;om@No8i
zrgzu$Uf+o=I<azEtoo#>W#+$6SRcJP*L>ygudzMSTP8kI3%wIN^^pRP;mflUOD1hx
zyy*V1vO@yFrlF$CZt<)8J5;^DVAow0eOPch13P=Utcr9Aliw$XujO}jOBXxqU%fA8
zwKVTW`FioPA942|q}1-8;Myq6beoMWc1793r>*CfyUmrX^ZNGpId^sKe{GqgX4~)I
zSy${{-utd<^J~`k74LKJc^_KeV$XG7a0SOKsgjOolb$lL9_Eu-a6HDsqnh#Z!#X>@
zIgh@(i`^4feAwmH*L6($-*Sd&jZ*}JUIaXsP`LD<|Lev^N3A6Zl?L_t5eXCR?d?65
z)*jf{tM-lSdB}(59+wX#{Y+rpCGcHAY{!%7*^kmARj0Gud9puSZ%2x!XXXOW`>j(t
zF5KtN(|r%w9Jc1d>n|&hD{v=1O~1SJ@>$*IkDni}ZQs2sE6jVYS;N;=J7pHU7wet8
z^y!~^o#T_fuh?;`|FSnrQq}oSuU@1IZ?X-M()#%Cmi3WphW}wDyPdPVuSzw{_}093
zdB^Oe2+r$k(uKiiTRe}iS+_U+{d=YTe_ye97-lZDbU3qf`g65&wXe6Ix_|J!M4@5Y
zM7_+%m-$m_UKQTh+xGgA<6(AP8I$bSZEG)YuY0Zh@#1fZU0>goZM$uCW7jqx(5ZAU
zDr)zr#NBzbJ<WfX?X9VMf7kENn^X2j`}<0+2hth`83XDnAD!M9zpXg;`M0&V11C(%
z`knc7m#t%Gqd1c|gUVOmZ*yN*Pl|AGdQu=^HKV0(QNt{Ooof4!bpMasC3Bi_{>~yh
z?<ZT2ex0)O$Ro=Wtnv1zbWX(e^@=ehTwEaB9LO-&r04OKn~81bKW%?HSNwmDt-y5W
z(_V$%KOQd^GrUt}eNxl2@@IDCgg=&jah&D$3hcI^qex_8WRGuN@>)n~xrum2)H~O{
z!yC9KCVe`6<A>@j3jv{e1LZ$Dchh2&S#-1F*ZL+DckWc5Iz32p>ZSkjMjtiAi&~Ul
z`=neIero#kP@3@UzMQDF+r#Ip)wVUJ`2Dyx8!{!o27Cw~qs1Z9<udaZ<fZKBO`SQx
zZ+f$A{{yM&P1@p`{hIkre^YNS&-LG8Yx(5bgR16xeY-5IYQ7jt#T_nlWp8?>ezGWY
zUd_^7Q>*4yiEl}{(l*i7{Vn@H1`pPqnq@P!q(1JRbRlf2#eM;8pEp0bRUa=eusZ!{
z@>R2(|KWRN5}qCuy71`Jk^dH#0>r`@nI4Hq$WE|kk~#9pTfu|zfyeC)PF5>6zbuMa
z*v$}=^ugWRa3iz8InU^zM<p4|b?l4}E<Ko3_KK;3-Ew0$e}i96BB$n`uo-{8P2b^G
zle<&ga@)}-{Jnkt_YZ2C>{!w)wo#^`fNNixr+Ma5%K*(C(48voI|a(+A-jlnOq)A(
z-?VouGh?o=jGbM+R#Z&xnas4zb)|FH^{kVNoO{=>zO8T#_<)OYp~G8`@yqmVihlbj
zPJE8F#nU;*EIz)P(Q@O@ZU*!CZj<TUH463D!XuA!r)`$0eNmO|%kQ;%@6vDIZuMu!
zuerA8*1c5+cbrw&&vRec@aM57Gpr5!llHM$pYvG1A}!GP;kxVKgUY40Nwi6?UC+-G
zU-a4LguXub2&+Hln(KF~y`P{rMRu}m!<S%wzqM9Zr9+<Vz5O-v!gJw<g`97N_6Ysy
zE)O_c%KV7`KJx@tW^Ntb!v=!QW?>QXf7VRcnP_QV@LtDK`Sz!8GoqQa?c16a+*|e?
z@L&{Z$&R|l@j=krP?4Y4dE<Vw=UaPEG2Upmw`8zL;xF{k*}-`|A+gg@<=>XY$tj<X
zKWN$gp*vBPFNEb(;hR~9*Xup!UvZ9K)%($u0;vyAw`;t}_{$Tt;g*!j0-f!0eYdA9
z=0D&P)y4VpN2FBSBcTn^LZ>U2f*X@-QyvBC{V@z^Z59y-IXO+crt!d3_5)8}S7~ad
z@GQAKzx12wt4H6C{L_-+(Q|~H2=PZp<GA6PBaHW!@~0#kb!P2&9B};VgNGj1`Oa^%
zyZo$dws>A_O!!Wn--cypyGqwS6wmQ_sCUBpa9;c0^)I(}d+WvRx**`B(AaWG`|9nh
z)eU?4-pt6+Khbve<tvrar(3^X5B<IJ=PvcvervKBSN)VS&z8>)Xo~-)(<$q%b&@N&
zWqqWODUW=n#>uK_SDiEt7A3gf=9{&8#nPV+lP7r0eAf2yIX|P@_olW^6Om>cy-?XT
zZ}zdwIB`<P|HbVW@eVUfEOnX;o4z?@>ckiQ3rqa%J*Vlp;#=3g%b7dX{v>g#zMfMU
z_Mk+p;0b?7$d+W4$d;!Q*UXq+;OlLrXussKh~lfFY2J)?g~C=_xQjhm6(l7N8hY68
zcI}y6z)^vMC0Tc)!n3cg6n(s=xb98O#B;$WuT6F@zIMN}?##V~@02e&be-l658u=#
zFzK_OyYBvnue?0W0^@7eZ@u^Sfpf>-@8908eI_hkG<(j0tiqazrCF=8npPcdk$rsh
z@%7Dbd@WV21D9x?&@i&9DQTaw^5o`8{>2|k=4wSxZTfxHZcXg-vcFDCzV=<Mp89Zx
z;%SRlx6bL_$u-&OQKzuj)7t4o#tHil9zOXyj(g+YuG%-r&fxpfQd9Zx{@AkfIp*3+
zx~!ZkPUJJ5d}KDg-s9oRZA(7<=&PLl^)Xkc-ls+>-yB=(NrgvvPJ7%DdZ#y0pi`kz
z-F_nvXV>Z2&aZ2BZc0&1z0{Jct$I>O>EV_8GWjbDYt$bq=X(^-UOdU^^pe^$FL$y}
z;M>jkG5X!5<<kzEZtH*IW&QDau$pb_fj1ZG7O7VWmT@w9`+BthiBQXOR(TQ|Z|Kls
zQ>1i$LNTAitaGyx<(XQY))@TsK47_VuIj#J{-WZKb~G0kZPeYbAedsef4;>uYp1$h
zXJdXUJ~1&2?7JK`PsjD=eb+S#?mFR$@1n9otjxAt?KHZy{?qCoAA|GLSGGU()BC(O
zYw;uYhi!~ELYY$bG8`|IZrLK!-dgncQ0bP}5mx-M%LJ}33esA-XriQJ_Y2`;t@5_V
z<+r@}esKNdSY@q{wUU9)E5Hk@RQE9V3G#`19^CBy-6?PT|CH|TXs;9Ohh7Kh{#T!q
zuFDeesr}8ymNTWh1>bEssI_y6?e5m<W#ajJ4@C!UY4FwQ&u^F-vn7Nv&;00B{X24s
z|Nd>6%@=D_dyn6)SjO48Zn=Dnn(o&lFSnEgdu%9CTxWmO`_I3}Yitgke|d1BXTi)v
zf6qoP>`t2HD&Hsf_v78#ps=NDe18^J^-B7u-N<^MxhlitR_m?ITQfJk+~m56^{kwk
z_3bwW->22O9=>#dx%}ConeJ)xy5l5m6?UmV`MLkwKAUsLKK=ba^Xr_*8IsMCzlGWa
zg0G!0)19Wn>6ywI8FX@8tHu(YtLolMJDh?zT{pR${IqbPLGvbKw!=DS<`k#>{`;}+
zSIzp*yPxgaecsZzqxf%Ood5S?`|sam=6;X4{<SV%V!2n^)J>sTU+&%ez5Cwjq-)1&
z1U56gZI`#RdGVp)d)S`sPxhL2#LxW8yxL;bldrQczTaH;S;{Bl!dA!d%vSAP>t}3F
z;=8o^URCz3Whe9hT-3Jv^!;!C){5;H>!*GC=X*cDYRk*`#k+g;Os$v_OzKXa6F2Xl
zzq#74-STg4r`h4Ojn}qFg;`he*&4h!zdW3&%1_Rq^2w8vo}rVOCU4qkJ%`QB?er0O
z^<(~byuZINU0<1`$8zfB`Gs~no}BlZ7V_*(cciD}Me%uiCN2Kox%@{)QlR6uPtx3a
zOHD-%$fhx7pMQS!r0<s{$IKm6_-A>)o%g=!UF+!$ubWRFI<@D@%JQYf+b&s{tiB|Z
z8*=XG9*bYszIa>DDn9!`|LJbeSN#_&Hq49nW>ueiBHj3p_s=|`+G%V4S*(@g<JZ{l
z-Td;!$wbw;-b~93X03mzRlH}zDc3`bIX&F7UwMSqyomiJ!rmd|Gv%GvbpucF%Q-9G
zns08=m>DNBZI@zcd{#|T|B=I;F9n&E-CXvt$v(PN?y@(uO!?A$vvoi8-kr8<$UffO
zKi|5J<$T%CrQUVZ_uMGj*njL>OB2@um8yAXEJJ^n-1uSpc)!Gd(*sYFi>pejA90v?
zGCWqZv48lI^E^|Vm$uE%8z=MBt7UuaOe`*jblH9_`n0TjYrq{AgN@!--Iv*1lF??b
zXRz(Ra@+Xpa<}#GW*C$ndTDh0l*K8Qgfg42XD5W#hU?CZe%_olzl3|QOvrtE0nNC#
zR$IHX)6<T`o!j%NB+7kD!xHI#2^D`o+<5)7Z%<kFY)K9FyT=WB_<QTF7gzlWn96Pb
z>HnSVdWD@Q<5R!R|N8U7$3siAex7I97k$aRCgjA1J--xpH~n@MH{#emZ`rrEJJnUz
z$xUuQ`jf-2KC0%C#Iql}`#+vnf3xIN=7u?biZbt4^?#Nx60<MlDVCTad)?}By!^b~
z=Q0BxtG{H?JGt9=_e}Yff6j?asB*Mf@F#ltvReU7f(!qN2q>Am8}l5_(Ej0Q%xulh
z_d_~k);=GjIw{WX*#;YvZ*4F!xaP|>F;Svz;u+Q6LsBhA+vYAStv#gHTFPI%;;T;S
zCWECpe&Ks(huCmD{PaKjq>*DZ<IeuxH~crbSKLhA^U7FT{miD@vlgx}ySL=Vk|b7v
z+Bv^6I66PaMu@X4Wjz}G?di4sHWyyxNti2%bT>)a?t5dn<oIUY``MPemKMJHaPs5v
zxW8M^_f6X!z1`+oQORk$ZJFB&w}l5fu-JagdDU^napl(SXMU~x>aQQo_iSI0WrpiY
z`Tw6kpZt7xa=O~|y1n0@Ojir<h?d>1!1w9p`7qh2uaj4C&-;I#TYi5*TJ+7_Yq{x3
zJrc@pF|*%$-LL(5d41$3c9)oqc7j*5R?JG2$nLGo{d~JG|9AG?_}6+XcHjE(>3@CJ
zQQ_705u4Ti9N+LZv5&J`-Dma(uPf74EbPulq}WWYGAVU`m4995=T)Uy<*zPn3_H%S
zxclSOjdqGAg+58<k4qP~30`0QCi(8}U+pEDaoR=)40@v4xYzKkuk_8`H_v8G+MZYa
z!L?zj-dAi(_pLTtUb6Vc!SZWb_j#`W+3Xc^Gj-wQLf$}?nCbUJm&JcMw&;d;;>>eD
zvIOPz43;=+JWTeQdaBUZw)@U*wT*{d=WlG&5v&hOTDiMw+ofMGBk$gbdYJWp>94=5
z!u?)azsh?5s<6anpHTJONfl?xz5Xn|v)I$}?v_s$HnY++J&f#@ex32pdqu1h_v9y?
zmjZTh`hQ>Iy7cD({w}*o$0v%nJSi_+8$3Pceuc02toJMQq$+cFtqt${seh$q-kMpK
zJcnLR`F_^D`QxOiCA@d)PrkG>H#@JdtCX93ODbl*pRH`^E*_iKr#Y$RuDT8L`OAaj
zJ>Jf|m3{t+ZT$P6Rl&=BZ=2UPU#L7QYMPuS=05*Sx?WxV4ZfB8)XL4jtgY`n7plG9
zZKjr{!cn<5B^v(<I?i!Tl;?~U4{4ZvKk?tCpZ>}<{r+p%Bd4nS$^ZZ1;?^6V<aMV~
z|NI^ogTJc3Zv-j#O_h@`Tb{pGy7W`r?zv|*1J*6>_Vm5ayoBHFs(t1>?vmZ+o1Q1^
z(%6&qD>+Eu@Iozd#YK--N88+=EA{GSj}Ggd(rfGaZ<U<6D)>e6*QHpkE<MA8OUq8l
zUtd{t?)L&#?T1@W{N%WB>&ZSJ;Xh9;*YufeVb*yxtBd#J8ZW5_%S@{i7?#;QpR_H(
z_LbI?7};Q6D^>m#RwaJT4-7cHR*7ow^w@3HZ=9ZeFYvxe+}t_mJRDWxn>|I1cn(YE
z9b%Y$W$V%ITi4dcZO^>>F7ExeS@*N${&WX$WY(_udVcNp3-czdo^4hB>T)o9kB@X{
z;TcYrDeeDkmi(OXh~bpxU!$leTXefhU;XF0oD+CbB|`qormZ_mIo7SOXKKElCtv(2
z>D7y*UFq{$l(|oze`3kA%kH&4*8=8<lbp`$o`v1ZX^VX!+<&EZU3$TttF@l>J%I~W
z>I)rt|7v>K6t`!mru>Oo{zLt1{G+JY9}W8>`AibCj0AXR6+gZ(!8z#GIrj;_f)^ih
zHjvF%5;s^B<bLNx@Q1v%o~?Rc>phN7Kbd;Fx4u&`T2N$up?vRb!@Lt)*e=V>zBctz
zA;YbxDLEe_SI^bwt$jIpy4V7X4e!n7&;QJS`LEMMi>*~ro40)A+~+>2&;8QQb8kGO
zN;0o^n)|J@oXM-N`M}g>v4Y~taOoP2iVau!TNY|QUS2XwsQvtotA%IJ6zpm2xfIOo
z7iYJvwV$*8x%>H=FE8A4cPeFG6S=Lwt#Ze=F3-xfFMN2W+`qw~wDr*0RUxOF#lQY4
zb(;LkF5$1>1*g49SL%=bo%L(=Rl~W0D*bb-cq>kYzV-Jn3yfI4aI^m0&l8uNEA{!$
z&%@cE^x^Dzu?bUt8Cm|GogMdR)-TPMrhjJbQFmK>MgEr4@#j;|zBK*iJ*RM&^~vdx
z5nVH6c3l*cstRMalU3Uh#XY&-$5{VmjNc~rw!}YguP@o~^;p&E3Eh?-4mZa0#`Ldo
zIW@a&3jey=>R)B2zcYSeIkDL`K|lBX)?M#knTPG~wbMPfRpaI&8>Xip%9$=bEWX4%
zIZ6Gla0Y+R#&@+D1-IX9``&oDcV0r-;^)(TZx6X&l>Ss)OEpyQL44AmnLo}ywqK&m
zyyTGi&5wuMr*EnLbY!;re0$TnU*7q$_1WC-n8KRarfe|z*mop=PwDft=WD;5+<s5A
zL`CC;+g`!-clp$u&dhGt3yzSSUlMQbwc^CyNu5C&H?Im$_~Z6}!OBdDwispiQ{_({
zy?yuePvAStc9)p2SgFgQ->ljjmZ?l|IQ&R?h4cSc`Aqh_FKaSN<^SGYc|7F(>6fwn
zr*_T|R}ZawzG|jOiPwESr^|-xeN)fcv|pXrR(6hSxzA@NreFISw;eFBaee*w!tPg_
zCvmyHno#}ap;~(D;VX%S=l(7Gq9D6-5tG9c=9@=zI4?=5ICwfg+3{^=x{txK*}CQ@
zqT3pFSTXe4cx*U#TBl1+HrFYU!B^y8uf6pD&_ADy-TkLq*|L2oxwLNjuHI_dedlL(
zr<cvJEIIW=^YHQ+zhyU8969ii``xaP?KyXuPfTSgm*y#PaNyvv4gA~5>Yu}0U~-qy
z{qVMS-)OUvn=4aSN=VMX&r<XFfA7!OR`=o)>zn!cZcV9Bdw96zZSt(1+WIqlx%Wq=
zL|@r=@sJy*@Ake9`F#Zyp>lfd`KBv2mhLNEV;=Bs>mp<I*fW*+=i-g(+gC@`W^J53
zn|D>!ChNa~8tIY07{A5Lo!IxF^IX!!m%*#PzkVf^*Oq_f>%`psl|E|=UY^`Oy=}Sb
z4o^*&pc&Ckhe9HSbwlDOf4{!cX<hu>JH?B&rn0u2o@uaKxO>i>r*-aD%XW$F+!wcO
z3-=ex^6;RwOZv`kzP;yHfw4w_*fq_U5=~baf?nO!_wlx#GX22QYnfu2^6ce;GVb0#
z*;0Q^%h7r2sI-FjRAB1|u9KpZEF#<4_h~A2_V@e~$aLzCx;^RL3GMA54~Z|i&V4%X
z-~6kG?k-lD_;}uwUC-@*?peM%Yod&a_uH(9mHXLVYQ8*Edat`W|5|ZI=#~5brN6#E
z?P)akN#3HlpU?gb&iYuks<l%r?(&>-Cz$VBu}OH%FYQ;4+hO$P4m;<$9sk|-n_7yh
zE;zZg+hRw{b|KTVg}qVFPn>(Q&yzFk(mcPOPj-DV$CWv?h2D8uIHor2d1ku)E?=6{
znGF{@K5Q`kJb%vbhkwp2YMU`>XE^&)x4#QsonOt|-<20vnzmg1vB<kDTl<Z(<^Sy5
zb#6AJPSvIM$t@R}7<PTp`5rcl@!h4!kCCDl9!u{9MD<Nt7kp~-691LOdYRnoxUWz5
z3^iCPc5=yq?ZQ)3oL-pNPipqt5<FwY<F!+IoX)(Czk578?(?l))BX~}(lf=8^A|1t
zy<@vyc=%GWWrd}@^FsD5dpocH@x7fddz9zSsJ=O8UJdiru1T*hR)i{kwOU%Nap!W_
z`k3V(l<%{?yJX|h+30t0=@j#myb~sBClxUXurK52F8<LW_HJ#+=Yy&mCsli2=)Bvj
zX}Tlib&}WO!1!fVHLrGlX|c@n->`lA?bxY@KhL_pr!0Gh*pV+R(?qxOAG3XR=4<@&
zzLlDq*W3TU)9RkHex`7(jePs}i4`}6eBEnT34ORU{owRpFa9?!+#Gd&|I$}eSG2ZV
z@t>LG?8C&V5X9AV*~a_J^N0C<?;Pj<SzjCW#N2%06Pv}e1dp2PUz~r~F=q9{1-qj9
zzf3Wio-*nGq;G-UEIY#|#@I@Q?`sJREI+5M-;f!1`qSQ{K_x{Mai3>h&r7@9G{c;0
znrLhLo2ps=_E`PuyYB8h#c1x`)&CYT@3(N6a%^H&ckqj2OCMjo+Vf-OE~&WhOV&Ky
z@7+CR-L&<0UY_ri{I|5mJaE0r*VLj<y(!DuH`PA)b>NeucvDidAG1!99+&y%-8ZV!
z>)Us4Q5RKl@;^CS<K)`QyC<dW645BXx<t!PBqL~r`|GKfmtT2s>CDcQO|LF(mbvms
zugK)$Ax?)oE5t9Kj`=q0dY-wl;<Nu*Q+DUNE}XaG-2N-_KYz8{`uhB-@(b@*E7Gq^
zy4U}mw4qPEbW(==)&hRs7oRui#5PMQyb`%L#qHalUXRy1D@)Rs79TLRjrR09X|p}h
zQvCCyD?;xSF0Uvx^yPml@Yr8@=C)$ifOo&A>{)na2Y-<I?McNaOKleaI%Ydvq-xPC
zeo=3C>+9Q=*v`H2X#Mm4AG_=L=Y@uIPEybjHGXu+E$hO|sXxFj{k)<<W$PjVp6`2>
z@4I^=XhMT*R{zAlmlcomLU^VBRyj>#()gPC>TvnoL$j|;eiiq&=<B|cFQ)l&pO-`l
zeNCD*Ywi*~%i3w@oaQfT%(vUeB2#HFcl!$8RhN~2>i^to(w8C<Hf!E#gV*aBjV<pi
z39Y{}bK-*;k7sqwe|@2C$yQ6vq<<0<64$1BZ55W=b<QGgQUCQlWxH?M8=P9NzWnPm
z5#vuWzMp^1XPfS<rDM0-WV(velCa%VzO6W)p{L{OAFew8&h6>Hc=Frw_f<EA1<C8F
z9C4p^q-TYvOtF;1a^=szY=eH>-kbeZQmi#-*{(wt?-$Rn^z6+}FS@7{&bwLu_v469
z8&&id-^q^KefN!gk>UD!-oo(D;<hh?MM{<)vNBw^VC6ggi8&byZeIQ6@ay%H$FHv#
zURs~4{I&4>V~yTr#(h=s72K{L=PnG};aFyR`)+8NWlxiLjQ57A-xeybsGT5U7gVp#
zI$a_-Rv_rOyynxGOA6Hw4|!Gm+o6AXk968!(*(8Fo*+^ATk_Y1I@`nFEmT#tz2B{_
zTyZu#?)Udu!beK<1LAfVT=`tE({A>UXD2USIlo~_)vF0PR}_=^WMigke~P(u$M!8#
zc(=1x@-A~e)AQO3Uq?Kw%P)0UV)>ruf6%_!Uw$86AwKz>NA!fsmo66+Q&pC7J)2Yg
z?ugQU0TJnvS?gSwCJLp;9AB*IDZ7))r1HX?>q^H>c|(;y3)!mf;(xpTTdJ3nYu5Xf
zRsnrmD_`2heZKWJ(@$XYh9#;hA0F;J|Hkd4%F7@BuT+0&QCb|FJ-gaAcI{l<gUY3A
z-yX6GeK+-rxc#H6%_SYyuVy`4cWeK2^>e-<60%c^4fXr_6k1PM?wNdSiJf26#5vc@
zp9i_r{GAbK|34_Hd-YU(?(J#)eSz{S-@}yLCl{Tml=M8oV_9i?a>gB%Mg1k>0%20U
zU1E3nO6K2vSpRQ^&8OEpZ148%SE&70R`CB?vuNHIuf5N<zD|_8s?mPeW@gIvw#vAr
z&#rAdboNk4^usWf{TrA4i>oXO-`^jSK4)@EiQ9x#|14Th*D~ieCjQ!b^lFfc)hD-g
zOaT>2nhTdRNC_-_<54+tomSGNLz8xR@13z{SAWRLnA{cbiftYoIU&q{Z-vy;>zA+j
z*&2LvnttZNnWVZz&A&CXp7bv&Kfl#ycSgO=-mg=yT)C=SzkZYE4}1Q*;mIi<t?vB^
zS$}ucvm+Yi`5m*LKM%9N{vmBz&qcwlf$zJ2SA4m7=iS%xt2UC6>@i~3PL&36cSSu}
zvEhvE#nQUzUMsz8mz=US`Q!WkP`an>bOX;4*~gbJzL;SmetAwfr_9yF!mle|g=mHM
zI~-K)VLNrTra&)BCHC^ahW;IFDN6!`Y%4F^nY_oR!Z1I#!1L`RkN@);z4=OZ8y(Hp
z`nxAa>;2&eRhDmcpDTGUOSw7w$D93WC;7WG>K0#rZMt4`s;K9JKW;PoY<hRiz17>q
zWVOibRc+yp2XVcU;kGM(ZMZRcj_cg==l6cB+gB*-HF?XwLtfmPI*%_oUD+``@YIPV
zjrp%4kHpTJ?z!Iars30@Cu6QeYqB29tW}89?wIALx6JB!(DCmlkBj?H4NSDS=sqcN
z`G-JWy-I&B4b@m5EAiA>-q)?l-fsOOtGPW;FYVsRJv^_uzf8V&Hc~vo{)_z%yFVWu
zI<~WmT>GiAf4Yr+<+*t$tM}deY`V_SC~cwfqj$Y_>g)en_tbZnoMC<WRr>YM-TyYn
z*T2sR{Bih0^V>=>9rIVm59z!TUL3jP)how!UW<3T+bnv$Kk#9GY3sW~QT%JA`{q8H
z{7YHZaG&ts+m|K>{?1_Zd})#Ke5vx!%`#p=ag`3U_be~6EB)j+?OfOSRTC@?U!Pv%
zuq8lj=Sdm;xuWw!8{<A#Ufm*hZ(aP(c4yDI%Pnf2{PVu`Zk5Qykj?GeJ*LcM`ZIm~
zuQR8NUmL#+|8BV7c)PEK#RkuJ(+?(B?B+C|&l@8?jbs0_C1-cf^PO$8{nz`i#m8#P
zQo;}IG}o)ITpL}dJU7y;^wyTz%jP0#Oe{XDe?7aMpC4zD^>8wG`y6mj{!BUh_4l`*
zMBFX^D!ux?+kAbmy1>$-JJ#Cgn{J!6Skd`PD8s|8w~xQr?(u&m6Vp@1)jHkGRv*^2
zdcVK0KU+(`tgFIZe`37U@qbsYvIq0;*lsqr(1!h>;Pb=N+umQ64{m47Jr%EgUv~a<
z=HPuouQc}v&Ap|)bzjxfJDW8s8NKgE-qHW!YS-si|NYu||4kbn?3<-?^fO;({akzg
zlAe9Cd$OyNEZDXF^=BWjSTnh8!`aYYP>-R#c=2QJ0~0r$;MUObjMBKVwC(E(M@#c7
z!iUsZS6(UEGvki`%T5Ms)jJlU>gtzXXZ=@Q`>1(2gYZ%x-%e3JBZ1VppO<IE7J0o3
zEIyei!gBvsoX@V^J*}4c&;O>(f7Wp(vfY2n{rdW){>;;_{9OIz)0}EX^UZ9xnf_*{
zm~3COYR8i^it`r;Dc1A(NR>Ra`x?_QHD3S!ug3N4?^o^Eef3?s!S9*2p51zHtxO+0
zd72#k>))S>^~aZ)%($>1yLsKu$D8M$(3%MuZesXVEA}P!&UZPDsZ3Y${-00pw##!&
z)VM!aJ%2)<La+1tq7}VjnJim8>LPz#Y0b=R|0EPMcW;5e!~WP?i%Y_-R5QhH#n&Fn
z6wH*qeC<K>k#hC5r{@1--`CNW_@?+h$J}}aMy^1230*nXdi#yv|5@v?3;4KSlRUmj
zrNp^HHaz{hyu1Amy)LoZq%Edg9AAo+S}pzDlV?0MGUQfZ-oS8O=EVZlMYTUJ6&y~q
zxMk&&(UiR}UhlJ<q5g)b1jfHtgFZa7?BUuck+W@vEocn$#BU{^^zYa7mx{fZuN%L7
z<;0aA`(A8sT6g)+=K~!3WMUY2p1$upY<)TSx9_tSpXHMcdk&~J*7`j7arXayxzEw&
z%I{as@$Gxx+wI1f?j5~B=;eF+Tl2ZrCU1>sUc5MY#^w!zw$91(0?$_Z*c{!v@J@}D
zG;>b^m$2`S%oNGU#+$OThh;Jy-2$Q}aJn_CvzGD~RlGYIBCX35>G1pWG5fap6AyYa
z%;D6Q?agu9=JCDp;<Ww7j%(cvH|7bf_0GFy_1{q8aFq91)%KYizVa*I^k~Wm>_}W=
zl6SDj$6@=KawGGlf11Qj|2QD>zig(k@v9$xtM+ZXVt8L(>k5bd+s|7JPwWu5bIZ==
zO~k!>%*R|G=QBucRC;pr($VvOXIxeI+x2nH#|E40;*a-N)E?dW<3a7l-*J9YbGF#N
z+^T(kX{q=7o%WV?Iq#*P`>U9H%k6xzFEx7Kj7P>de?@+<nQzZDsaa35T>5FxBnHO&
z`}VGSvr2et{pz{p>CBJKAG;?1*>#lNS|!h{r`c^$;>@(z?60*&cP<;ZoBmQw`20G1
z`+tM9+B?6(*VUAyUYmM7Dg0Pi_ufmtW8P2QJKfDz-c;q4e1y#E@U@;1llR&r-Taqo
znbBFd^UOx0*~k23H|O^#e<=#6{eNI*sojcip4)$P{P_DZZGOc38<iXFd1`Yn^eqo(
zva0`Zxjoe0$CT%>^kEBo)s<O6GGRjd6l)58Jhqm*ADNl%fBSu;>87Gz<*)Yd{cm2l
z`thuzwneXg%siT&Y`@_Q_x$Lu6(2qpAMbbJ>5&k;xGH=3I{UJ@{dN%#{qO(XQgmAS
z+uCiFWzjw>XSiqEeYmmZc--5-vcj83?}~jdin@L3M%KpgHaUqe&o66V-B;PLt-to!
zx!E6`olYOo7e4!cziI8VlRav=)l+M8Law_{pZ7cI*!B3-w_9y<UjO@Z@A)jd^k2)(
z&F{|HwDT4B<zq5cwhu$TT|9DMNUSgN$CE?hlUAv1^**InyryTdkHh|xf9+nI^VfSM
z%xRJAELZuLus@-D^WQmJPn+E^`@8V|tt)vSkG`MgEZ#puT|hSK*=(b`aq^}gCr+2M
z`uy+Y_j6Z&CO!FNdwO--@0Ro2{}t_&{;mEor=w9&St)Mu<Mvq2-S%r12R{8@#y<c1
zqqJ^oojRM9Cx0$}?%X65?H+af(q>(S06j<Zwe@LR)@@;byh6A1>(qy=q7yG~U$i)5
zhq~X9za4(RQ~%jBNXhDYt`qY5J4;3{;JoBlKTn(ZTl3Qjy#8K|oOfmRio=D^s>@6N
z{yB0-yYS@omGk+%?(aKi^>g>(bsOV%Wv}V_7uZ<#?)sAj_SKu;3;nmd^e&;H=aS`R
z23zsu<#9eQV;3K7S$N<2mesj}`c=Qy$W-#JR$kjMwcLzN;^3BV{VaM{cg}eAX_6IN
zy~?_s3wAv@y+nD1^Fx#8ldfxAU+L`^U6QcW<j`A%s3W=+K^vJXg|+tz|J(fDc6tAj
zxD)xcmH)3^bu;wVJ)L(u@4$ta*?WB!aPF498L8V7^L0l5lvkJ51-IXr{B`@4`IpM;
z_RIdAWPPndWovlM`o4bi*QVCD^s}n3WH;}ZzLWhyoq@gL)TdovSD4B<*#DgR<mdC-
zOCo~m{4K9PdDph@tNzh*-LW#iBPVb2jnB3E^mFg|uhr}RZ)|Cat3IwPwP(Z2LmEF^
z)c;u+);+qn`C4)BWcD?78>$|^GJRF5Brl@;<^GlZdH*wd1R!IvZacnsy=piU*wDGd
zS}E7%rC^|S+uIfU9`~`_SN1%v{PC=h&EyY>hrWf)-}>r$mgKLTo%P;db;}`R;U{j)
zOkUUfb7|W7s7}-Qv7gV}T7IpTt>XLJFXh|M?73WYV^YFeliTTK|K99rH`{Xe_O~;~
z3+4ZR{8)Ke|Ht2-;pfVI{un-d?BafXUTpmvEBObP3{USnF19*u_AeIbJRIkGwJjbe
z4s*4a%iCrB=+WJ}-Dbm5W4DaLb*?AHo|b>Rmw5jn!;RMZU1$5uCgxspeq)yzUI!f*
ziuk!_`ZM3z->aU!*Pg#-&W<l@Q>~}#9@Q)0`^LTPd#=6x{l`C-&wC%Cz4Y0SS#zIn
zjrwrw(R=CZSwVX}x!3%6dGV3IKmXX9ANFVZ%B<t=e>5ul754k(dO!QP`ij40Uz4BT
zGk-t5^5vxWVh@C_U;Z~Es`iy>)wU1WR;;I*{Y34f7ir5L`g2j#>G_qCUlo_;M+Ju8
z{N@sJXYbaB?z8@-tO`}stO=~HzxrUR)Zd58lULQBC}sM1M@7Qjk?ro{Bk|8(Uy0mr
zRk7B;D5yEGv{r4_(G!U?{k%Bed;UN0I6(Sfp<ZL(&B`+6d5Z$(1vP1;wjT~&e!%?a
z6Nky`+L?SEJnO>$Tc@o%VtzqP!tJ2a0+qLmAFsV4XfrA5$kx5-nMpGpB%g|WyU1X5
zwra@)k!8+*Co*Wv57*uDZdIbm_pYixyLs#nhpga8EUaM)TeR0@?*%_Mw=WA6sy_(a
zdAnurxs3m-Eag+`o=ZR9f2=&dYKN6V^rHM94HudjXEQHup7r>?^^IRk=XdRkdRF>n
zWz~kD|C^q_2>uqHdo_3Coa_AdF%@4wEPGf!z0&B{^%uPv_AjRY`1_DO2Q(jj^J7rW
z=D&8gPv!02^46zJ`uNW^@eCPP(whz~6crEA3fyX16J?m4Y}XNKkosqeU+|*`hgUy|
z|CrFaWtMmr-&)JG(<|cTZcFcddQSIyZf39GiUW~9mOspI^Y6HQ-RSE**U;>%QB~(>
zo64`ax<Bo~-uY&-cUSPb<(YlB_wMYzLtN{7{Wz_^o|_h0Ut64X_i0|Zorg4cK(&SX
z#*{}_uh^}89HbsR<3>uw<d44f$E+UCU0bl|(ebJ8)>iF%)v>C1!}s{*|E+S4$L%va
zS+?$WY0Y)9+X=V1w~0z5hF_n1{;f{_>@?~0YRi8<f1ZDoPkI_#_j1yG)%Rx`^ZTlb
z<E~s-(RuN9^875L-Dgeq%$>n%dT*s6OLCGp!>`KIpTmAViPxRDRsXo)C-)E5(-xMN
z-`~6KrbExoUjeV@N9Sw)u9_(o=sQbg%Zc|N(%0^hN?o<&LGOjH?)km%<nJyoo+`Kg
z`~28_Rju8k_g;l8zx;Qu)iS%fr<1jnAGj|2%~a!4VZN~3>H8YZZSgsuU))S8XE&4E
z@!H{LeZrLD`Xww@v0rsweP3?BXdXjw?QXlQ?ix2P(<^ocd~GW0=dr|FI6VA0<yDR2
zzscv1FIm0lZ|quEm9NV`1lIq}y0P}$gDL0YZTZfBopNm%^Uar=gY5M7cK<!3?fG}s
zm)eU{KbR)S%$l%(|HxbqSDp;7#esd6>ohr*UAJ;8He9MSA>-o0-!D>9UR|y|6MX4g
zaM8}vb6n3S2>zUs6~({*nRejj6Pef4crLwuwt0I=sk}quiPoRiwP_4G|3pjl?sC{$
z+I-!&XK(ndCEJbuT-db#_{zke`t$!QoStOKe|{0`bxYaLuf+XB+vAr`K2!DX*0<H)
zBaTPk-{Qyk#$or{HMZy972VymChc3C-Q~5WYum2MT{BIp?v&%0y6w&7qO{YTvA6Ai
zIV?|oB{*U4^V^rVNALR)&A<Qdo4K3i=iWV=pLuUicC}V*U}oylrF+z!w(nTxF!Mhn
z^M|>S>Zk92<MdlRXYHGHD_A4!kIDbJJVDmkFO*~bgcwCme^y(WFI%pj)b4EfI_GY&
z)tQ(5(>6@^+M;jh_n+tC`lF`|t?jDsT`!J0fBoTpm-8OgPhuyg7Rvt0esFXS{|5Iz
z8x8Y6{E@fcwdrPkar;_ZLna2*116Jr8NQxcHcc-$XHn(9>Pw3*)p*?({2F$%_e0yI
z_oDwxPxEig-MeGvwg+D_xc?m7vh>WN7Z)$eCx#mSGu6M!bM)P*=TqZj{#h?~*!g9<
z+V?3R6e=!WO)YDZ%4<E5Zzk$9ckk}1c~@2+f2j8T!^P~m4a#qo8Fl!4SG1|VeV@fz
zw$lFlzgKJGWqY1kF8i`>pT6(h*PdrXrv_(9%siZJw_Zf!xx&<WqD%jI*tx!(t-yLj
zRZ2isGR|jtjqj&xJDQfIi0~;{_}yl-&Fr=c=$jYgKKcC8$ETmnDU2yTaQtugiCa%>
zHFqreHJxD@N3r)KmrGnb<Cd|QWL{I-dfQk0bJp=W(?88Wn!YjhTT<QS^4hvLAMd=i
zKWlBIu>9G)Z^!kX<lWl)W4$?(JkyIk4<>Kb-Olyvbx;2k<^$cy7mugO{%*L}%HO#y
z;&fz<?z8HJF4O*R)c=?M!I$Zp$&E1mt;g;%NNz38-q!z9@QcEiE1oUgy7IqW-WYS7
z4ia3tJ$zs7$G6%E8p<BMkq<0aM{UmvxoY%b-Y3I_DqW(_e9!zow!P7$`bYD_uZNRY
zr-W%nFJ+60x_A9jM&^~%tG2I?vw6m9aX8j5<>1XW_m=6L_FY>a%6!?+Iki9h-<C}$
z_wOsuy3Fb8|48_O^xG2|E(Lq;E7z{A-GBe{oA+1y!dMTu&R0ImA8+?1d-fc&`&(oB
zwmU48`!|PS&eF}nKh6F`Dpejozjs!NN~O98+l6*Drz!aX%kobiduCzyEo$3ji|tYR
zSE6EGdv8e&{je>myyBE*fZod_ubjLeBCeg1Ph-P*7ynFuZd$PC<ol|ic^eDPPX4>=
zxWcqgOfT1|axhJ9I-Mo2&Z>0jRB6kp*tI<8j`rKc@84Is=!K@E+sV~k7t9{SKCC-j
z*f!~qDr@nSz;{ztuDo(|$>R%R=j4|kES&rC>-UaHlNg#LdJlUw_{lCfX*>7M@t0X*
zYT_Z!C-{8|ezhVY=&!e)pUOcmJ`L^@CrZ5A{+1<7S^jro)Y)>2C<dARGv^7+s(t13
z_(@8##w}wx56frCJ1st3s_%_!m{%fW)s*tVRWUm6?yJ(-*WR+pFs@({Y-v)r=ojE(
zw>fI++v^<g^-%oD<a2lVKk+uLTFbxf-^Q=oHy=uioiWwMAnC)u=SAD<c9lN4cJf2u
z%Ilpz7s}?|I=VHxMX=3B)~N2pYK~7#uMSRTx4+;0Q~%eV7u|mjX*Gt*gnoLF@$h|{
zK7YK|-og*Qm;V1|3^{nXUHHY`z2D3CPET~}v47d<xpDG4$9e|+iU%4pErI&V^Zveh
zcivt1zl{2t&ublw4#<Ah=Fz=bedc>|ynHn4eDzgxUKA&p+!JSdabm5^kB0}=v#om{
z`SRnmetrix%?HW1ca&|uy?5iEC!5xu7VD8Tw#z%D-B4vA{6^&cmWKS(@(15Xv_0DU
z_>R|_9tjb%x$~ksA9VO1y7uQ*`tjSl&F!y$I;wx1MIl;5a!W;5UH1j2^J4dT>$asa
zo<yFE@_Du4Z+m@+m#L%d+JlQOEC2Z}_A7YyO7{~MS6WkrqG#S)dZ!@#z8(LhOaB_|
zu6*~rttlTFwRx82`ndgD*!<5;erD>m_@mvc%ghfRFLzm}_4e>fmrKzNg`armhWlQ4
zc+dA%u+m?T`@C@llbFxebIe+A9wjNcBjmScDVNhOt-bF)ZC5?>EBk;?XFlI$#_1v=
zL3}%`xP!A6D(~2(EHc3-Vq$^+A<iF`dAX%$&a3C!&ffMSC+ghF3yVKQvWb4*|LpU|
zbDMkTuh?~FeWXj)yOUq?nC<H}q`Ry+7yN;nDdj<q;;G=Onj3TXJ^S7Cd{Pq+L()IJ
z3w#mIHy_PVEp>ji?@Gm!p9^IZ-wFH6^jm+7x*+W6yS6%5-e>OoT|7qw9ZbHM7l-~{
zBER))@1HFdrLO~)FVLMiy(xLd7X7b!KYq-(yGiZAtp5zVtbZNnm*@R#wQK2io81PF
zK5w{s@ao0BtN*UlJQjZKz3lw?H_szF%Bq<9?Jk^7Kal$4{Ziv8Y0r0C%q(8@?WMu%
zn9~~$to$;i<lagXqdlhs0$Dvd4ot6G{34j4&qQgVU9@_~&WyW<l;_*6{J(jX7{6B5
z?v0zPlSCXm1YDY`gnmrWDqo#CNkv0#CJXnEUzz^339IV!-p$=36?NBU8QY^Tvkz;(
zxbU8#*1F=lfZ6^64cqUpa^`MdA1oNR+-j#(jMvuFFE@Yiy2LbTOFxGR=aD7HRJ$&$
zi$A+(L(esh<=0;on5{R~3pKdjxX?9yyY!JiEB?1V4S&?z)tG$v()uOO_0Hd4@c)f$
zwo~6NVZHm)<ud9+<&*AdFmGUpb^A4)d)=?Kzm759c|841P2uN?I`RMh&FVkW>>BTT
z|0{ggZ)JZbUVg*BbXiHY@Uty5E}mT^-<4;*``w#=n}5HoHq2jhcTU}@MGcM;!imAJ
z_FtLl&7>_-v%pn8Z&l)Imb@?S(?sOCxutgf>f?Gnd)fQksn5PXk;zzb_o}__m3^M;
zN>A;bSMoRJ>ZHTIAuj^^8^jvCw*)NU^ytdg+I22*>CJfOyt%Was)9D0m|xnz`|SKj
z6Q&$;pOF7TN+sfC^GloS!izm`PMegN;<2s-v{K7`x+&woShkLtQ>NWieK7ZfUY+!&
z=-b<@=bg}2`1|Hn@<QH2VQqDVrT$Uow}KKo&oAhCGFv{W>TQ_OJ4HPm?FGB~pSS%o
zf46b}S7X6$y*=+dcRyb}X~snx3%#cwF3z#%3$^^(o@VmuNwe4IPYr91%%0}+YW+FY
zrP*QGtzNIo*LO`ix4kL&sfNv>>sFStUKLL`_Hg%v7e-l5{O6~-$WPwr_pYdmf9~{c
z!BPCb4k$A<bj;iH%;I}w(1K3?Q}XWTRPDZQ)+*hkyY$k;>v4YPcm7%VrSHmvNz9I?
z<d@f~6m(|)@2$%&(y|imQZbP|d08{CDTa0Wvs0W)UtT+Rw8i#^<Js%$w^daHO^sDI
zcbC(y>tA2^e?wOG`T575&lOE-dp2#|<M@p?mIlRox$ZC9VVIveF(&ggYyRDW^|9gF
zTm_ukEc(mp4n)~oPY!4MbR;h)Y{hf!@au;H(|>sW^X9X){hqYyckuhUzaqn%|Cw)o
z`9pT+o{3XRwR-&z-_HoI*Dg3)Y<2gxEc==mKkvEIHCazxeakRWChyV&s}jS~Z;SMl
zJ%5XYJySI|oL{s&c~P=Df2+4*5ub+YF{_^6c8nUDwkP(+_^eUfSbAtXr*l=!&18>$
zr{WaQ+}G}(YX$bq_~qmzWIQ+QuU^+FhgNPq?r^5g{$sA!m+F|mdiv<Ghn)V_I+d?X
zLF<?Q+jHsXYEk*ptcjm1udXRL_u$)-SKI9t&zz^&{^?S+ixLaN{J!mz_W4%q$eb&;
z(*A*h|AYrM1$AwIST<On+VouLoz)Wg1CH<3E_onSZrB&~Wkz4y9B(}vmSAr|&qc1k
zrk;v5?ez9zRGOYXeb0<%3qS+&+>7`5*oo&ZzvmUFF-bdq6|Xjn_}8)zNmXaF;}$=A
zee!USM$Dx2=_+@e<$G&3U5WS_v&CNaX7PdWlCF7H#~2&;>F~Ac+pO?<X&Q4?=u8Qp
zCW9bDdBu%cht<9;DZeYsd}*Td^p;aQXY7fTpTk%pn^^o(T6M3u>_v@z?t1T>6BkW+
z?sZVx?ZU)mk2X10biMEBcW8caX};ADz7^c<Ub|*&>3_2J$>Rlc*^gJ=n9Fwj>Qnj3
z3o%?x;#=c|_jw+$lgbGDysy&wfh@x!DJe09jptl$-93Hu$S+k(oAs4vv$t=nWHDrv
z{CR$1pZcUKlhQv6N_=kpf5tz1m7~G`o6~MQit(DeZAErd@jb()RWd;V>m}sZHt4ur
z3SRzh=jzw(g$jP*hxRV{6LNKG@0I88<E-m{m-J`ZtH!JIE|~p(qwl?7OTU|SHtLhk
z9o@Tg$1@kndA28SzvTGASKzPywc9(+hmUKiec+h`<{ztyF7>ah_hfon!XnF|aaCrz
z&8%O_-p@j&%Pev|f4lsadhmbMtEZPN5LRQow$ferLcph(qQ{3kMPgq4Hd-nE`P#a7
zk4@KC_RbF0d$z~?x&F-kiwhj3&#rap_7nJXU&G^xr`XDBAt%4jTt8>6JNI<wiV0hU
zO2ZzVouD5r;!~{oIW*pI%ZoCv)E*6=B{ydVsi_`&SY9c%QYBoa(a&?L@O2TpAbIs%
zM(MfVuk1AYY;7&g+>=mRwrn*|Nx1LwZ~alSlfO+rur%rbN5E9BwY#+{TqK{a&|9nD
z$uv!S$+p1DtgP>L<SCi2+R%}1*jHU!vo11vT9)&Bv1>mr?c-Ygx9Z)}7r!dbyT2;_
zd%geT*O!iRp6i3PW3<kn^?$>_vF}&tsdVw>(bHP;IMxS8ELX98D#y_J^02t)`Ad7h
z7G%yobTusGb?2qszl2^`t(yPtYxsU{&qW^X3Qu2^Hym>8@n?NrSQTS%VzOkV^}3zM
zCo;Rw&F^4owd{DZ;KIb`8>}w9*mK=oeDb8yCD~u5xb&5n=$AUZ67dR}B)~ddChygZ
z>7by!d+u4_f;qF!`70b)S{L!>mrP^z%{dJxEE)GzGaX-99CY`#`1yGy=XZBzN1dMS
z{mSF(?xlY`G=H3{zLaCX{MLQrKi)s{imd#ZWS8u5j$1nG+3ZhSJ(av_&c2HAf4TPM
zm+3)FO9FP7%qmg)W35~}KXb>NG{Z}u!^NjPQR|c`TxJ=sn>O*!M46IHUoZSK(VzUz
z^;1~U43CBCduIGHJ=N=E#HFEmA*6fhYDe|l!1I2Mi)S8uX}IIdtm}K8nW}em_RKsq
zw;^1M{lXLt$E*Y4J4(+d&N~oUdTr%u<=_s5r>|!2(__21gHg|-vB9j;|7p!Dmai;M
z{Ofksp9|IMDPanUbGHd~@_CrRXPgu&bU2px)XQ0xg6;1`-da^I-4e^RQif|aw_*`P
z*Y=a?)6dQMrD?fX=K8allk=yuC}q|!4;6Ik-<6?q{m>HS2YRh-m*yLWERp@2c_Nx=
zmPp>EY^^izvU-b_hjP9R`Z4={t0&hi5sRRRlTX_n?o7;@>#Y#_DRoy?oZHUpd&;b5
z3m++&`d$5?OV$*jmJLRGW(Zj<j+{NmwNKFIq)5s88C^~*IzB(zDYMI>$>I3JqMHnQ
zlPY#cP3V|)YyR@d{T}73_@=Lp)|ea>^nHo0P;JY)#wxL;vW(LT&nvxSIuIDI{9pNC
zjOD5`#+5auj4K#Eem=X;`_Jze9(R)Cq*#{!-1JAY$49k&Nob(Ksqhm&9|tjd3Wxn-
zyTx>P0y`&z?^fN5>)7UIu<tnHwPcRBX2Uu$-AjqfE|=9M$%K8{Z4+?rrJamdlUB<=
zhK_%_wX2`coVCD@=}n97IZvlorI+lio~2HyKAXLL^GpHb74w!BUXTgjJ7dzhr6~gH
zLH4eW>gf}cS-#iIv^sDlblS$gLXL9}P1(zYn4?~GX3I)SeT^-BCKEq9>iXpKK@H}8
zU!K1<U#ZS=$|L)k!Rfs$%;&BwTl`V1=REUPArICI&(D-|-tiWQTy^0vul5r4^XF&o
zV|{6T_@UZ?6<^XDR6<<VYu32<E%_<=*!trBwULZ3qLUjZcKV+aUfxq7Wiow%O6$Ta
z_cc4kV&&Z|zvnG#?Ot81>vQGD!qCbG<xdv5GEC2SSm!&d`{Y7({^_p|_`W(?^1cSN
z7VU5K+0J*Hp{+^&qbu~=gY~X#kqiHMU1Y_={pxFq&%bo}<x^=j{oJhBRI%N=#OHp@
z)SJ8O7q@!i3zpok*Wb%s_4Hl2ute-l-ywZ3ze%f>uGX}8SNLbC@aH{2mht@e=bqc{
z;k#p^QB`F%qlsovGkd@!@%c;g+n?txQa^NOi|lm~uS)?I&z!iQPpS@Dyr<!F;y%y6
zyI;+Ew&x!IEk^dck0*IQ%ZgJ=onn4I@AIp}XT3`JBVTTgu6OhAX?vzPGsSX-^`}n`
z_na!<RdAIry}F41`Mp;ko=(2+J#Tu>w2q!CyDX0oMiGru%gU|lH)-v%?qof4y?OZo
zzOeNNv*SMBiAkGMyrR5TtgBKpwBu0l74}Q%W#a2zHt$(9?b$>x{axGHVuH)I*<JbU
z(&U~lp0#56J>{B3C)k%(O>RAX_u0?BR}N-hk5z6Gva<dzS$xh^m0=-&{Bjn(tND?W
z<b${+X1;!z!(-6I$hfj!__4>sB2OmHaz@pwP3{>VT^5}veyLPjwsV%N-|x$1D_ak_
zue^J8?}MAd+<r^uc9e&1IZ+&avG(X5(c)?U_tgCP6n-&DHox|=)lsvvX6L>cMMb4<
z;mWgp@-zGV{OUZe2iI2{tJNBRZT}oRX_eYh-#>*(e~#>h46!!+YlrMZdZ~P$t8(?u
zx8K)DMMge5QO+vM$+c=~P;t%KbI$xczdpa5TC%)a?fuWH^VjF!{rhvp)A{n5bw55W
zjK3JCx}j^aSKNL{&=l1AgDq^a(j`ZKykA?9f3iAz(keApHOs%<+TeMr12Yb63_W(!
zPWRu-%iVFlbEXMQ5;*j|&-UBg`}ywspZLXEU3(?5VddpzZ)fkDiL`~u@%%OEX!+dQ
zj`cS|F*AqhDP+#t^Y8_m6RYF(gxriZV|jBgr)T=9^P9ZB!med?xt+<%R`tM=I<DF!
z0=nk&<TJx|Kf5`hMaAV{qliX(<)QlPH(RGjdwq9x-9NYDi%!Ve;IHlL!(<P!?KwOn
zBw@~c3D)T*D}3*F{M#>U)#TY_HOE{teXq;OMd}Zxq&Z9yu88#bH*Kj`;Fon$F_+RA
zFP_;7UYWnLL$yKqYrGQ&=LEsGlb%m)JAE;!Ygc`?j)BqH?au2;_spKhezr2=&dc5I
zDd#OOTzNik_WM@Bo-1z+!i|j*ugmWXSsQJ-Dr@Gv?_cL!`}}p;dvn`cU)DRvtCv{h
ztghL5$~g3v&9}dcms_1*EgSy(w6*kt`lk02mpq<m-CQPj=O&Zw_NjNvGNT#)%X`0>
zSYcsk`)jk2`{Mg=K6y|7qgR<)=C-|S-3OG7N6zObTQ1L<&j(%PKS!$I)2EHP{XLZ`
z?<Q9Luz6=8S8=EO&}ZR!^FHW&%{vx;>w4Aw(+pE$SJ@doo%rl6Xt&b!7YliFzi+b*
z{?z+{^`P;ryY=~#tKT1Z1KE4kW_o;Dcdf}b^<T$zR($m9Rcu(h{{NR_;lbgFXD04-
zuQM;wJD_^)&)tJxf16pZdw%=lt+xqvZv2beKhC&6wLxV6O<DJns4pH-8Q+thJHOv}
zjOFc?S#yn=jHBXvIeYj|a9K2+-}7|m*H3*+e=hE>VY+iP<HwQBuTlf&FMYdVZRcHg
zDfwxl6U=^nGXJ-q{a>nXZ7tt97RghgqSrVUraJB8JbA`;R<y3Uto`|y-&0LaWjM}X
zTKi)29(%h?|67v;r(85!Y|GHKbz14_uj}{d*Pi=fw)pSasYlpTSY;l5_R@CQ{a$DP
z*Sg3VuIYdO9JylrYwZW!=eyNYicZ{CJkGc(r*qr1ZCM|bU(M;S(yL4Qlbck2w3hky
zkMk$*O_)A2{%TI9NW}g6U50n{kJ`8Q&971`jmRpFUa@wa?yT)UdOxmxJ$ps5-IY(%
zo|>IHdpLNT<;~P$uS<6mZ7o#4EIH3~?B=i3>nB&b+_%_&?B&ANI{V7*Dcv(|Z(ck-
zG3%PJ{rvay<>&3aQn+6Ea@{$(EOXl_|1#^Q|2_5VjiHysLO18fn+kP{)6!=96rX+3
z>Hq5Pnf3Gkd)Kv|Sv&Kqv2n(NJvT01@LIU@h4cEgscWyVVY6f2#&(78#M8#F4MvRL
z7^LkCt#g-Ow|dXPGB>8$uXPV^M&Xp=km3;Y{Q9S#mz<yU!1JIyd%x@Mm%%>gwW{B4
znps?V?bGz9D}pb2^f=60T7L2VA3l>)6&j28##{gY^iTJ%_>bJL(GC8uHkW)Ud}1ZH
z(MEiSZh=UwW!0-UhOfW-WIx;4Ev+xNXUEo03%M*W{$`Gj)MB~6`16|QVcSw~Mx|z7
zeHXYp@vYmmsng_jUYnWS`SR=7YJa_|qBoQC?QZRg{wY@;TDvFf=&EHMU!IHG>;J4Q
z$(pRaczwy<&-=^Wu2gB%W&FAP@vpqx)x6Vx-zPrR^bWqZY|X#22isrXcPm}}KVx#<
z>;>O%rUV~8{nh;Mjm*be&aQtl?Tqd;>8pwhoXgs?<N4w)Cb_=uRX2Zox8}`LLH*}@
zOE%rUl(%@7@6P3qZTs(jPB{~8=$$MkE$OyGFzV%MFK=z#V_pVz=k@xR+HBhzemTy$
z`mEXS>phyHkK3mvJiT#uXZrb4tMdB?cQmi&JK_^<YnuM*pjiHvirq)6Rja4IiP&`c
z-_zgTZ_eMf$auU;-Z_eOo6Nnen~(3x^}qO;^H;RAe*MqLoQk*ZS@K%Zn;xC=;$58O
zUfQ*)u=ZZx^*narQp1PJQ@mVOxpurTy?*G)uAV)gEWSs6x|vz8>GE#X{Vfcix(-Y)
zpZe;E@BMVQT34aPGmBU6%;@XWQ(NzuJ1Ios<B2aD7n<>0e(>ap=oW7|ugt%D<jm9S
zi{(|)4oU?$MnA9lar>dvv_H3*B#v-LHE}s_meX83zwO-1?{k6|tJLlKXYMW}H>sxD
zis?s5O(}QH7NM8NIYor(6@LYs*AsVoBzh+EpK8s_FM3SxYTQeBx7=l!zCv`#L5=I*
z<7c%hUC#NQ*8io}M8W7qE7Lt$P32CpxxZ|uygS@!J;^if<R8UjDaktT7r#FmQtg~C
z=iE_qe1q)O@;i+6>5$pz|226<o1a{NaQBJ-+?qEj+b8c8U}+Z@_^VwX|1;&O%NNVQ
zr!Mn_7cbuJvNyacZ1QgDU)T1xe6!#8C;5PUn#j9%N6Y1R{$Bop{qXDd<Ci{$o5nxg
zeCqq#srm6WzfW~sn3(KV_WQ-me!0IZbmKdpHf-&^y5{Sn8L9V^i>25<ht(|&I_|}I
zx_N&`tlZVtcjG=+pKacLwZ8I1V^Z#`vh}+ctXskJ*mV8QbXJyA7n|O=X`VUTxY+K=
zqg_2auJIpy?Y-l2=*pv0e^gyrt+mT0q&7n5rK6Wm7T@v;|9>K^4s&Mn+;E9ra&xJ@
zs7k8)R+;`;^}btOcZN(hS;nHeRE2TY@ssY07fjf^U-fJtS9js`F5aErdlyM95~=w-
zZ_>9VQZB1{|Lr`~sK#10<&<-x>eIH_|IR5Mo9||HK)!Xh$yNuU*ynF&%w3(fXs&bU
zdC8|+m@Ixg*ip7B-0#3l7U9YQnFsw$0k;k{u2k80?z8^V*Y!7E`bI9E^M8NF^Zfs3
zZ-)ypT)cEV=)6_)Kl#klbEDtgzBH@9S-|+vonU>R>WW>=2J61>yr*5Qztk2yC)byK
z;AU&SE?2aL$oYM{k`G<sTwWisPcgqa_?gLBN#3fPx9(WE+?u$zFL(1<p5pXtCqHek
zeYEBN^)B;X84Jkbgu4DOTfhC?_eC}^f9l;IcQ4=fkGB8w^Z%X?AGy=R)<;d=aO|vA
zT-@hbDQZW~+{(W{^=r<Bm5<WBOIRzK;wStvvaFuvWc%*x5(OV#F4cHV(|0?sFm5g3
zttmT@@s;Cxd~>jG!9<(sSsaVbZF}PO`M*k-errHOWY9H*+=sOls&ZewEM~Szt^MVE
zzIQ&eee_9huioDos(-y>tPd6@mhs(jbX_DD@43k1l=)IO&geh058v03jeK?fzs^Jl
z^}p8&7o7Y(J8A}x&L#iNRu_%!#6P|ET>YrX_0r3Sv!11X`nTs*|H8x<nm;vnTDI<C
zEwenoZAqNZyyKh}9;Jqs3`b4%6;A%%ku^hb*NiD=Ym<Bbd;eVfJ>Rr0dunC+vzq94
zzc#-%zFxa2<HMn4+-H4~D$IA5{89eAOXt<6D^I?w)f-j%T|FH7Dy&M@a^jY{lBbWS
z+OPd%_O*@aL+@*=h5viLp6GNr|H1HK?xFpQ9xHd3lx)Aa)Xm*T(thdXDLWeF`Tk@*
zsLG6g=nmSF^!V+GPd0O_tX`E!_Ix;Zcys^RA6ds%Y@hA_zGu1F@BDWgwm$EzGmhJR
z^-L?%q=sz2Ej(<tJ9j>R*%J6(bKcTNZ`b8bb>6;6nX^)_{mAX53?@FSE;wIzPOVy!
zWs-5+-{RRKI|I+ScD93--80uq##zNZn7DraYhSPD4y?yiE6YySpG;ij_t8hM!sUC@
zycdg~WnVUaE;=KjB%XDP{JGgvHqTq%a?azuSAE)w!!K&eOnIJ8t+Tk`;?MWyf%Ow1
zSHEY|=fB&0Fd~18kx_;*`zeNwJN2HH&tg7wIC;i>Z9k{c==WtwotMOSFRz+{mMcp)
zSnOS`XQ6qqRz-i=lhY-pF@DeYTHZX>5Fh>Seo*b{xw5ym_2*Y#KXjCPxo%DB$p@~l
z?Pl@KmuFD2nfv9YSA0lRWTxhf2Mey}<bQwtaPH%b^J34VrsM?58SpKUH`x9E#>VHT
zrX4xoUuXB=v)IMDz}t)Cm`&UJ_WytP_vyFmF7-A4zPHD-<wO_!Je_^_lk~^EOTzrK
zr<u*$E`Ouq!8+#DhihGK$R7NgTrcBoH?3v)shg+XbzM5O{_CvkU(RX<_+=Y>(DCH@
zsxsB<*1E~vCw<S_E?;orbHLZHBC(fNwy+qBnn-dy_~7wLno;hfmeCb$wWnT}B3C^3
zS+&pidRz9nm}S|=VmY$C>vY~JKWVJcWWQp&v-IzsfPH>kH<#Lph^@Y_^Jn@V{_ifA
z(kJpLOf5cQdNwI!^XZqrL)%Zhobl(&i&#Frmoa-9RB{(Saxn?;Yuv5k>LU5{)uBm$
z?(8@}DLrMEdf*baiD}kPx;#AQN`3Ds+NfwHzV`Lv`EyVHoh9^T%A2&)M_Fbp{(tY`
zZD)nukhMb#g*hGWtn^#{+q(2fFuS?zo4O2s*><l}i=WSp&8`(szI&&n_|&sUAF7-_
zaNP6qP&*rAv-$9ug%8S?$9ejCt^Jp?IDLQhw`8r}mXF*0e}7#*xBH^}{LRA8&1TG>
zJGaF2os;~o^(($z%y(ITFxY0f@Xi?@b~5=WJJ^43uXMM+aW^Azam%vk%7Xi#K8#DO
zkDid<LqT08P0g@Ht?`#%KAid_Mb^q%Hs1Z!o)>SbR^?YMTHsl#`cR+ct4n<DM7G7M
zzV4Z2ToyliV$KJxE$|GLu9V*zxA@$fNgFI%w|!Az>|46|>c#y_Y_I0Gtaoug5nIdj
zs`#>bZL7EDk8`(wR-L)jw{-KP%~}hVHZ1s4wRq}8$Im(|d3J|5OxkqRkf(myw}*ds
zR<7$f>HD@bB)GnUyCL&L#Ot8{?J6tFR$A>;e>LNG^SA5w<!>)6yZx)M(frzU&iSgM
z(#wOMOyaN<jncQZd*gRM$|S?DoaN+~KLWR}b<h5pQSs``yd|#F%N8!JX*R0$HhY%t
z#M-k$kwu?#)#tAs>Jw~!Iz8w9{K0IgX~o&>d9zPnt+lwYWJAA`|34A-RuMa|5&^q?
zSClw6@2=9Aaoyf}edA-_AHGXPKb3k_hs@_cBfPWdZ*8TX?z@w3)?ItK(EsIOi}2SB
z52K{+hi2A(<$5X4u;-k>`W?z!6HLNWKdgLX^RB9HUXc7sv7IaqmG$qYE_wW5*5@Od
z#&hq5rUwOOoKU@1DSvNr|4M!<&Eqkz79Um(_uh2)$~?dCtNwYkT@;wmx}*PjGH59M
z_x2eQ8q<9~g?-xNBW2|-ugRhJG~?4BZ__(RwyHmv)aTs%DJ#utx%o%o%O|5tLuV}0
zShZ!hxskZUq_pWXuDmi`ee>xY*L87<x$Jv_*>7GA(VAzVz-Tn9So5m+^YfQVSZ4*V
z{-5jUFMQzW55F(@-19Z;ELoPE_mZ4e^4{vH($xo(Hl27GGv~gqqxxC~mA7-5-EW<H
z@Ab9%ifw6H`$Y!xgr${XEHOn|{lETtJrH{Ayejt8(J5|=bzh~gygvEVq0bjBVvm6~
z`K`^>t2Vqk&y{O)MQ+}`_~Y+2SWo@5nDjk_>-FyNUfH+53eHZBdR4+ZY5U2~3zn;|
ze)KR+v!!eK5{7kE{qrguxUXlQK0f8!y11Q_StL%HEKN1OI`!ID=jTG~@uw<TD>;77
zy(ArS*lX6k1>cfB*_l}A86F9|9P=sc(3P1}{!HmQ)V(tQSLlh#4bjJUuiP(W$G!J;
zTyN2O@ypq9yKT?1x>@@0eo_fkG+iI_w*1LIdn^5#IW<1(9(lfhb(rs@_o}+CU)h%b
zW9R>J4%c?n?%Wl4MeXJ6AdSO`y?s}9p5uJ+Ba(5U%E^*j=W6Fnx<0wuWa-}3-<K|~
zU%AXV@c!~!8jEGlKblqLyfbBzimuAZwU_4!ee9?<T=_{g^wg5KQPpRPBV;otoe#<i
zbas>dx1-9fcJivO%ey~#HO)UK`7~x)hxoK7PVX+gVVJ&v{h@HVYT3>{Pn!(zEX68I
z$1_*92bnLoTlIN{$dY5G>I^fP^llkFn=@1IWnR!<KP#rUhclh$XzDZCS#9c57kV7<
zK$|f}i^=u1WAJ6c9eNDw1od{FyM5&%`&Dy3+oe-Ib_cWHNwf-j!r;HfO7K8$RKYwa
zZP{fzx0HWfcd&y!w8Q9cY0Z|3X@??T{&wy?WT&_ENcqBV{XzMv?60@*Ppx?_`lQe|
zQ8l%%IBw@jjr4!sKTI_>rZByDwa3%>@Rh44zlIvDytrJ$;mN@gb%tM6|2XcO@Tpq%
zM>8$>q8C5)>G1}hY~32Mg76C$mppS(-<4TnxPE`-!>!lz{EJT=-7;<Sk0`mI>g!7y
zO#ixA>+5K;9@o6Gc%D69XyT*=ld4vmSe!e!s7^up@ruSwzlQho4is7i`KIldEF!(6
z{I0fihrsfE%dZLd?TPv_``yIt-n%cKymh59=}NM@_D7F<UOz;>?rqMR;r+fSNYJMB
z)pza6ppUr=pZovqdsXgeJL^jn(_A49lW-@SiI;jlOuh7B*AbcJ2~*gPs9x$%NvMAu
z<$ie5w}=0{>(+kjkE^z{eg0|rE770_92Lj68vAON@@bz8-f}|4L2K<z<<|YomrWQL
z6wN$c978Vt)cF&Y)#$mVcx$EQY4gp0Z##b2U;EcJ*Zft@nV2&#(k}A5Zl4=lUiSay
z-$na1cI#IyVA)xrr)H|D;P<y<%ktGpZ%Sk`eKeIP_bo7<{@ilPy#BBqGR5bMg}bX*
zPL-EUowMY;)pg0<Z2=Wpz3as|AF--yb{?N>6Xhx4k+ra<>}AW>-z$DTxU_w`o%i0U
zSud6N^j01aXs`W{bm+KdqwB4BUcIHC!`Uy#p1pYG;nd>EMTUiEOAU91PujNnv-Q{T
z)V@8J4KHpq-Uyz<oVBMk-(xfD!fK_k>AWX?8mAZiTmJ9rAN3z!9{OM0y+=JNqyBGU
z$*<34=Z{~PjI<J-yZO@n=~cP&);zzzuP*%h;?IBgOue`MUsZ;`dWU89%jnt5`}BhM
zUwWa{zoPlNL1~*Rdkx2lpk>=knvO(Y7c_gM9p2658CD`>|7(h}?9<bWmTlg#$NK55
zD<WcAj~1^lTbVpbZh6$2_H{=lPhObbB=*Ysi_E6gEA8hLExBi{{%D0}%%#qYWh~zh
zrcbYUYOwWu&@S$J^|vd7-vqV`EooamPq@}%&6UQUJ<I*(GnE(~UbCO=%$c3`+ZX@W
z^n3SEEwDpHO5j#~>`v>C{d)J`*w4IUqrSQ?IrJWX+x0zR9%3a58@kv3-21g|_l~<a
zJ_|}d6`A(Z!E=?;%6UO&HKzZnP})`Zxnuk587!$+5}N)^*!_xiipsvyiaYNL_uAU?
zW{PLsa5?XLmbbjL=B3Gs_-V$+cRruml)&}%!xXzc(|+7C*rmm1tJ<+%Ozu&->Oo)G
z8QPgCml$@b-`6&+kJ)wlc;VLleV$Wf|IXXd)*|5aK}4h|UX8ua?`o|_{mZCZ+-~>e
zww8ao_+M{jiN?x@jXlS<an=8RpV;@kNbQL_o85c+o6FVY9c#mne{nC!*PFXMr##Jm
z#{P3v8VSw*yB9?`C(gb4)kgU5@|fKH8`GWMi@ht~{o-46$$cxqq+PQnYTlii(y20E
zW4c&SwRvo7=Q~dRmpw0c%Bi2(`(Q_u)$#)0<FijJoLVhs{dez**IyQ}-kf$K^y<v+
z{sr%+KUH3FzVY(x7kl1unq7=lZk{YWJLAE<z2#4|?o2r+S$c9O*Rp*tcBNP~7M0d-
z5uf&7=$^~G`jut@=^AgE-%tMK`1R<GrP&SMDV0_$qx`miJ@wP^_~nz`{&_P$W&dWD
zeg5so+5Hcf)!$oweb2M4(MN0M-~L#0Se)H<-tUNw$B%!!9bzUkU#Mou-<{tla!Y?q
zFWqAH@$vGuRYK8og8I13clc<!><P0rH+A(Cc_$cgSSD(x);r~1&-!zgSD(o{{b?z3
zk-WTaoy#iGbCJ(Q;%2|jpZq<fFZiFZ>ZuQZy_jsLn5hJAQd!0OHNfk+gX`m+A~tp_
zYj&O688}H;xA5eBwOc=Ly|{gR>y<-)r+k?eTw_r@hdEv=;mGQxyR)+Fm-5{`9$K)n
zZG%WmklxNCpUhrQ6)xguox)LE%&5#K^V4)L_g_Zdo!_T4dG{P>y_RSa^UnDEj07L^
zbjvs4`nGFs=)KOgwYsaDp?_=l@*f8mS1z9Z$aH-y|L3!Lk1ghZ?cOzcN!d!XQdPMu
zp*u?+2pKu-SoSuiUsyLrfAXQ!xqo-DpL_8pNM3925_==vU(ZXznBt}FPs*6Qc+-9Q
zn#Y^Wxm>nZ%arFVS@*?xZ))16<16NLR~A~xue84}dtdOonpZk&y`pU6x9NVJP7yuI
z8!uXBgh}t2^v9pcO8oD?fC_hG_luM3<SV+ZIQ-m~-3!+9KGNwmMauQ8aHJ9^)2h|K
zJxuF<%=l&J$7QZsXZ~H2sbmqqo5O)9*_7o9leAO4_LS~R>rvp`>}`Fv<K*3Vn_l))
z`Rt#4Ps)~>zWc|u?f&$AkxASBZnWQfFJ}E+v1iuy+5cYtWS{=CcH+i2zEKX7v;!_3
z|B~H$c*z1wdG}9+N~b6PR@+%Gq8rlTzPv(NT=CK0(h`%8A}-xr-$OkYhH5&TWG#&O
zsur|DrBYb+!m(cOdjat)RpzK<rhF=UBYvRc+|p&d3tQhgc`+;~=$+qhc|GmE>c3hI
z6{q@Fr=L_Wud{n~v1FC5pTp*JyBB|$xGq3_{-kp!?n~7#pX6cPQ*}S{pwCJ%%dgo<
zxet0~pDzP#E%`F}{0ilbdNchzVzqQu+H;HRPO6&qY6eR{&`zb!M~8X;Ue%uH<+!S~
zcZDaD+rjGE_dBHOt$*u(R{ERPlIN8*b^q$3`_=r<_HDJ}lFV^6S3VX#|L5C3ZhOjV
zLEYe>Z%bIjt}?j=DSFN7KB;>7XUv`y)!gSxR68H8lWU*XFpE=<Q)Oq=jv%ACZyahD
z`#e9i^vw>hWfR>FY_)0+vXYmwIv!h<m-R?QzeQtW!jjm_>6ST9S3g>PRowNGrEt%)
zod+iSJzKgfW66|>o@YDnIR9NN<ud<}mdMJ(O8UEMd}|l+=NE2mSW@jDI9cLpANa_M
z+4}!?JwKrD{p#(0^)&vPneXPF+<!gm|Fl`Vo+ZtkUn4a?BIKj?@eRR&tFKMDW4h$(
zQz5(gp=%>{6yJ^A`Ro0`rAgJH8za``{C@hNdS>v8yKfjRul<pK^?t>zL$`iDI2bJb
zX4~T1=gw{HFD*A)e^)OoyQo!F?EWOlqusyH|M>Z~dG7k&3zerlt77j?eU=1DfVx+%
zbFf+M^{%&wWu0E)HgC7*`%5vN4c_kzpI$A`keL3i`+1eu!;0@skMFE#_dNCL@+wY~
z!~3VaSMz$mK%k}nL6*au_V=-Och4;6w!Z_;9+%#{vI%6+tjlAlpR~f-R(_uEiEt->
zo^nm$IrG`R=6$;A#aK9ZmfNKJ6F1-8s5Q&mYyQ-t`bve_alsku`1MwPv8pQCsq{C-
zr&Ml<$b)5Zp5flUpXR-5^nMlmB%uHLo5q&`l`m$<Y!jQX+q26pXz@HhtG*}ux2WH^
zef#aR`Xy{@bn{BT`sDF;-wm|>l(oK(dA0e+o!Q}kPxpOT9BUPqW%FJ=C4HBb)Smvo
z@l_uRTI#=wJ)O7J$GtB7O1@zEkM#XD8;)zM{?GVdbok=YGxhOvrA;}xc;&BscxbFT
z&&~VyN8VQ=+TR}BxN+jb<PYU`AO75Jc3S>$^9tKFlU|<bw%fA&%A(J2dX4t6-CK0-
zW@&!^rPy0~Uo(sS_by)8Jilt4@7xasul|S4-@|FR`P=RCR9kQ6rv~qKK5-S)beWv>
z=fSK)UhDc+%C|lHIX$T2?p0AQW3Q-rA~HMF)!tW%DVzRPY7zSAD&@H7M~7&}lzVo6
z*96S12{8RIX@PG0Jgeu*XFWO{GZr3^eQEN;B514nuBgmVoAycTdc(dfO!uyQXL6Qd
z!#w7&QlX|R=MJj2_H1cTxy!B2yfZzkh<}}GRDWOirOB_JG)z_a8&K-wb>Hy5GH2gq
zr-^@}c&#o9Cj_24y2R*>&=eg9*~yH--nys1i*>6Ws}I|E+Tqo;4eHOQe2)I+uzJ#3
z`JAh%Plaw5pWSbJZ)HqH|C-?WlBMUCU9Ho4y*U2-Zmz0R{{vpBpFX-z^!<I&kA?|J
zOV4}#7cHp2+@E!hYhHAj`;+&K7vptq|I7UK`1yT5$@KYCuilc&dBb0&STOCql!o%Y
zzfZ1j){Fh4_jLbtBcbe;>ATY;Z>g_8yP!jqpWVbJK8GR7K>PqpgN_|zO6fJv=C2iE
za$$B%zf5L^u8uxY`=VLQj_F!hb?Q=)*W0IxyZ7?sI7eSl-&FX*Vg9s|nk|d@?S8(A
z5Uo0t?EXF}v?TY}`|iGmi7&YflUtuHwd3E>{BC>R$x~O??!UbAh2*DLkL8a3{BQsK
zUDRsl{d>j!`4Ky}Xw`qc=<!6t(?;V6$G$)1MIqnRzumj(r!X}|L+pV0^t9tMBzWDT
z&n(!MX!uXA{o9rJg(0dd8eTn5Vfkux^_izer0klI^GwDIrwZ0s-e{H2w_9KRct+&H
zWA^W-tcw@QF}E|kC8hVf?dm7zlK440lpEz<O+EAPh~b&W{L@h{bs5^?y44Q8&JR*w
za;@Ls(Db(Vi@CNg`+l)tOKFGoqDvmfl-m+TLM|Tvv0uS%>6;7XjZ>{v1Gik~UG{lm
zCigdK1C<Yd4=}j9AK$=q^fvF|AJ&siSC@p<gs~nrxFz*Wy3k)X_*3L7mg8S5U#zQR
zPJGZDe2Jg2_5TMQhWU#p{#C55uG0N#K2>Kni(k>YN6Gh6i(DrKExSBp%iphm1oveg
z>p!;S|6O0u2{mu07rXs<BK39Cd(Z8XZ5#LRsCNIl-tfnKISDb#(*LvW*_UR_laYx}
z_kX@Oz20xH#om|f+rH~&s$NrjntoO{{_njxt8Z$XsfY4ehwXm(^Lvq6$i+*+Z{!N&
zKR#;q?&sHjetY}#En;pn1S{A4(77-E>g<Y7g%?<o+Y=?+_ABQ;I>a9H({^v-InBq4
z%BL)3|4dUo``vz~f9l^U#&rST%l5n4RGGaDIy)y<;&Y!~syx@3%p0~TOH<2FJ?YtO
zX)3ON&%|i0@B1w~#1i|>a)XN9JASe!dZqoEzOz@2H~!n3)_JcKtgqSc`}p|$blx57
z!`B|WmT}nNo20Rx&D1b!^`~<$uHNSxSydPQs%_4ji_3TQX;|uhO86N5=hpN%p=)RV
zw@+r*yK=HF{mEtd`gtYpy{W11H}=TJEkFJ8MBK5(ZHZqr@4l|DUAH8{NbPCPii=C`
zWS(yO!0xR7+xFwk+TxT~CoPZNI;nZAc5!ySv{Ckt>{vhdZ=CZ|EI*g0q~)HzD{Rzu
z?&XHEqEmP7O@H@a<I}}?%cFgGqN}!czMZ@0leLzLT}AGPO=8jWeb<{5-gx}id-`7G
z#W~jx{J6p!?){{0-lUoP6v~fjw@=?Q`OY(sS2qQI$Gr4qypdQgS~caBbb=wzVS{Nh
zk%eVT{~q|g`1-W{Hh(|9)0Nk$Gy3GR(fHrW^<RFJBm|w7yT9#3k4NU-C+q$#ELH!z
z@6B0m<wsfJH5Wd;J5kkdq9<AVaOIZpqJJ8;dTVC?=QsXd|M~6Ga`Ugf`>ktVb^qS|
zN3wFkpC^-kGVVO+tCL=1-Ws-*@B7)GhSm4;CHt>Mhd(U|kpJ_u@ZH5JyEpyr$=>+l
z=98Bf7ax}YBL92R`Q&IX*HphP)6aJ+pZ^y5<<hak;eVdH*5}9QJ^0HMwy$Qb``bzN
zEZ;YL`C7~VR{L+g<(y~x7k%FLKzkx9%c-5^6ZqG*waKjWRQ_Q1Bl7R+x&M!|-wXEs
z@A25fa`B`+f;aB}sylngh|l7S^T|8Gs*;-|N~^N2R$iZX{p!T8{wMbeUpAaO>t)h>
zNzegCFMmf?8pM7weO0~8_wVVxzQn$qSN%_PO8*__<nInHni2ADQnks|R&C2KOAZ#k
zNV>G^xky-&&lUM__Ok~~o(lw=nz7_rR&UYDteyKjub#~0Uj8*eetFcojyUFXt6c9^
zTC-0N>Mi1PczD$0|6?)UDcA1@rahb*x>x(&-!C7ve$+h6di+Vft?c)%o9_1Y(gsQ!
z3U3+h=bLG{{MPq>t1>r!>FU?dtoU-^?$^u7@x8U08zP@|ZOJ;nB;9uNhfZ~?`=(zq
z*n12T+!h<F-#U6Ebe+(SIp<hk?6k<aylu;oWai29Pg?GJapl;t@L%WizI?qazJHc|
zUF^)$+<`mtv;K(->#P0$aW7v!Mpk~y(Mg$C7C*jX5woqSw0&#jgXjCsGjP3$E8h|K
z>?mYo0>@+J_|J_EzcP<Xt^e{rH8N`c^YXe2CNF(1l$_4Cd(F^uMg4NB$L0L){nhdN
zqj=W@r@i~H_9p1nW$*iUOTYb@$vM4d(_g9Q*Z<s<3VIXrL1DjS`aWOF2RA;S%U=C!
zxz600jGy1QpTBz%v@JWq=Fg_E-A}9KtrtB19{lfjncL6l|9+Y1ShChKtlF@xl(YJt
zul%iLp?hl;g&$m%o^o}aQ>SWV^4s-Kw>!(fS^E5?*wsT)%uJU)|5Ql6d*t-=0;UX0
zq2hO2)@HsxX>2`brrqgns~#>5p89Hvoq&PN^TpG&-MgGxzk8SRZ`fm3c{E?VEvd!r
zZPGXP>8oOQKWY|xeQW0bwMB1F?_PcXW5$K=`tRQTieBmdaQ?lZKi?;;u4g#n?k2MO
zrMb}is+irbqLb|ZzI_oNpStz^v+HY|ZY+AxupsPTy3*UDYQO9~%6}z)+qa=7_HCAF
z#ljykb8fd>oo_qEhF87wX5qGmXS3^X#s9zcM6vYQ7Q63;^WILo{O98a|DE|l#(fn#
ziv3b5&sZpZSu8$zUexWP?}sICZ~XVFKXixCoad|m^Lui+T^5V$lrHG?=D3({d4JEM
zr?;-nuvU>@#rLtJ?e5Yyo?l!9=B#&Q+R-X~tL%J?!dZ>Y60x$uuNKS2r$3RdUL}*@
z$;oi%bx*d3*Tp?<zeK{fK53DDWm@9T9OT>nXy>+){x8*suJdd780%QpEcsi>{oeDT
z>AO{@T`$dhR=>rx&Zx`tH><?aty7{w>(}Rb%YJwz_hO2NnTbf=KHp{Cp4DCs!bZ+n
zno7HbP8NHnUcIOkd!H{%=U>#_$<lJ!a;BNj`R}?-k=aq3#C^CZ;bG@7<JWg*-unFA
z&E9-VDbE@1nNc+?@Azg!rrmDb@>^afT>Gv6@ox|066>GU)YiQ1OYPNvw)V~s&0h~b
zpZn0Tz5LA7XRZ3WPrcld-ks_GYxL{6{-($`Z$ImX_ussC@^gE+zyy)TqD+|+9!o>c
zRam7fMLn3g`}H}kwx<TK0;X#R8(sV_dBE!j6JyX-#m8BSsr7st4$1G#y4QE^r9Okf
z-b;@!f1muS=EC3I;z4S2{{Fh8+*h1FS?}tZpBqY#R=Itc&vEVZMLnCT+l-zs^fmon
zU$OS+zP$RowYk6ISD&kW+ret^>+z+|#oKn**_YheaGc=_i_VAbKW$hfyb=pvf7!jR
z_1e)H4Us#Ry*|R(Al16{;mu`Nv$yeE^%`^TIezI-`BV{?V<)d{4}B?WBY*Emd$d`N
z-5!a%vge*}7j}PtYR`^6hql>YF0(aUn7VVrzJ~sWe=j8Wf7x+YPVk||w~6m%*03Cq
z`mJSWvt#XjoiBGT>J^5j1<x_|dmM9l%I;0)*T4OC|C?l_^?Zvjr>>T72tE*I(`xTM
z!J=?c`6r*_&r;*_?oNo~)IL6+v7~&4V7jzqOijamjf!j8IeOde3xv)sKXOVcxz@h?
z)X_8Y^Z&Ypd0GjuR$7UOdGf`cN?o$-z4ep@>Z0mFr?p<%Xs_L@v1(d%-j9#duBf{H
z+g;|TVru%K=zQ0&kgFkzPwaw%`ug_ReB};WS<ENAQ(dS<Zh1_c+ulzp`$OK$-4?8`
z`E~ZC-COMUDVi*vQmtKC|8DJ*2_l<kFD!PGUUcd8o_qbCT0$Oyla@@p^?OCyEbsHL
z8p3NIAIzR*+bLkC{`Z2=#C1XSJL;#T$9b(O(`r%qX(Xtze`~Q}_BjpnlXd#*g1;^Q
zTk&=8jI5{SEIB@zRSRRk<!!nD(eGt!-Ta?#ziGcu|JJic{F}pz&r|!Z&x!iD^>qBy
zbppF3--a)L`fuiS)i)6?hfjCwA5{(iwp#ftcZFjkQ$i}E?M(5>(N8*)=Kt_tq^lWv
z`TMh(516krM1Npfcjj)L)!Oxar!=pfJg8xlcz4n)TfuKkf8u7UzqNmU)%w!29NXhA
zXS~bRXXfsH+-Wn}x#=H+?UwWZ{QJJ2pVjmAur)`&g2mTob-I$1i_9)8e<bwUR(KO{
z_TqFW-{miq4t#vKTln$1zI$=6KP*}!$8EO$|HDoH_2wV>opYV5Li}5j@L%~Orb`PL
zHzbKST>QZzvTXUmM*X!RdXf%17v@&DyBD1g+hEUO;ay%{_2!_<@l<vFG{~BRr3N3T
z*e-ecaDRsUy2O=i8bSe&7YDvqUVmY+-26>PRT;YE^Gi9sVw$Iw)E;`;nhK8YtNuHW
ze=W<OE5jnb?*FB$7d4h#_GF%5;`mX=)_-e~hUlf`1!2eQSFS5tEXKTOpEK9*Iah31
z6SC&b5C~eXwKj-h?ze}l)|J?D%(@)Op!aaLiDOpJdzb&)LK<WboC7brzr$Q<b@iC4
z;kQKwo2T(JPM`C|dHvnV@{7LKS-e;x=X-CRoMdCs?}u{)?RL7Bd%5Oj#m7Ecu~}Gm
zzgmyQzqeDlUvlTXR99w7yZ>?D$G`I>tL@M5*_7n}nrB`7PpWQV;k(ISJ|Eeu*)LVR
z*824Qz<0{->wg`7ANOc_l<B&gUoJbZkJ0{k?c9mQ+^ftMZhkzOS?Y&)cAs|q^-tTj
zI8_)jE@XJ~A~vP(P&em&6{f}M*O%RyESvo8=DP}|_s$*)iEe!VG(uUg|CT*&{AT*Z
z|LryIDeuol8@)+t-PF8o`)tvVPER&I`u>O4nPI1axJ}}Ri&k%!_9chT`}tp~A-qcb
z@5Eq^2dw_;-&|GaY~bcPSHSdT&PDd}?Q?GBr-Uso*mh&%0d6~c?>L`^{2wA`KRRFT
zy>n;DVe##$<$F%YrS_O|zWsjdmBhhB{`}l6+aiorv=ZVi41aJJw*6jne0mz+vfU2;
ze7v8ZJ%4Y~=l%Q`?{eP%8k5)yc>62dj-KH!nyXO#%X;SPVt17-_IJ<B<CeO)vUK{Y
zCwm%qm*;4it83iap876je%yZnjeeP;kNsTwAIeIfEvlQF7su+JRI_@s)62~v4Y9LY
zzb}6qqxt!n$@iKG>!Kd^op&q<6#0B5fBCZDgC3uZRz<$GX%1d>QsrsYrKt=0uLRpN
z@jeaC*Zy7=#BpNgvlq?_Ri<h`IuN+(uXFms58vxu=Knu$@n`8huLh?J4G(@_VehaB
zST`;1x$dhjN5A(IJA^-$bw1UtG%UShsQO=H{dZ^gX@{N)?U=Hp&qqJ4;@jbuW{=}D
zvf}TacfKrrW&ZEC?}~lj#YU~YVSAp%!q3cq&jev3&D{Gx8@Z?J*_AH+yl?v9Z&P2D
z%&7O%D*Lin-MXgw|4%E~GWq%SNjG1rU!V4Cv*XVz=2h?JUr#!>Z%6ju)pheWy?*8X
zGx}qvq~Ix+h8-_o&YJ8rtF$9WBIr!ZJdOp$=cfg0*gU(F86C_u<3VfFfBp|i@t2ct
z=iFCn=$^CZ`z)SYfy-<j+?x1ib7$XnjUQ~@Qt7N6Z5!UCe>=N3xSh#quXWyoeHq42
zx&m|-NdKuh9RK=LVTwM>Cc%5A*Yh`SKfnDzvb(?BKF<SQuNOA(+P!6Xe8E^%EqBM`
z2TSfxudP|A{ww+K4ezwNe|wLvTEOqG!ccJJmOxnD$>$qAmoEBoYeVoUPj~AmGoOC{
zol%@SmrF3tf5E)|bF#kqYngA)`?xxV*XO$X>3&k17}z!MgyGwmwI5>>ubE%}YAbt{
z{eh73<?^oV`v>iOd$hioym<HFI9vPsip>^FtNg=%T4XR81@o6#OJ=_A$>-jbX1sZG
z-6>7`{YQUghaYQlGh2E*xYm0=^Q4<ke_vvLyMA8fizMx8&G{Qj1*H%FYI<k98@%Vz
z^_KGcE}hew)~|d{9k4HUf7W;4Jb%sn@-?Sl#y_w0|Cd-S>ho^-6UjwctiGO7vTB{%
z<NrVS@pa~Pk@@cf7X`nJkgfZ5c4xPA>ZNx(n3kTic;U}_@8UXrYyQc5W(fT-wp|b2
zg|kR<X35_WH@WklK4vod%O;lHoBvC5>7^-C0`u0b+H81$W7$5R5avtib6;;beNpeH
zsjcAwFRgHv<{m-brq$Z7S2#aWn?A=wuyC7&{N&)8inwLB7I*Hxz4)7--LLzqQ~6X)
zZ3<qCPGFJAysuq#Y?Y#Z*qbf-cIn?%Uj3T<cYl0UUC8uGtHzLsyX;Q;`|W?7>akK3
zf98MMrgF{2+TH)Fe^pfYI+#27*hzWlHh5V?Z((R##NTRnvONB$wmO3@1J{?{2az$C
z1uyXBOCHdE*H`uaV|VENbp@M#G_2Y8qVlb{yXyRinDsvYQXN*w?ri(eyHGaB>xN(%
z`+Y5&vK`%GyPO)#c13;Fd-3aI&X*S_%ypL~SUf%SIPb~B4dqPP#cdXMt*^C}^|)pF
zdzdW<_}85F+AXG1JpG(ML-w??#xk4Lah<ClUl0q}#3q_;zS6~WLd%8reW_Jm75XJ6
zRaF@a@4L<ar~fAF>fi6${JJ|%7q0ppoO|wj3{S_&P9uZTvvRj4hhKZ>zw)+a#1x0+
ze$Q0p+nbp=58eAC=j(r9&d+5nS?1>bm2rk1={|q0tX@sna%Fy>&)!axe&OpPmv)9<
zKfbr}qsZD!mtYQyFCouUXJuP}7SY=(tl7mR5LHns&*jFgpi&ANLg$$HI8xI)$o*%~
z{&oFdrc~c|<IJ=An)tA2%j%sEgC@-V_C#Tl(KXcyuEpmsWvh2ot`7TNdGVg@(f9wR
zpRV1$mQ~8^w+esl0dZI5ukDVqpW{wzpNo4eUAxNX=Zn3YkG*+y&%o@-`3d4q{2l_q
z$9J$yiW2dZV!ZxE(}SsTYR|L@Io{RxA7nFrd;i&}{(W`L;#`pj9EU!BF!fUY{E_>5
zNty5O2U3pdKUgpJ?NzW`_-kHKIJ5kM>f@$19Sc<No~aLe$Xj+g`=95><qn5~>MvhO
zKk@!o$xhIj5Ai?W*-a5oVLa0FZ_%55FB*5OU3Nl&fp@F--!rbV^H*32tk;V<xlmo-
zt8?BSpQmT}Rj-C!i938qcE1M0qSecog)>(%us`~u_dUi-d&!2AlbPmyuv5~Wc<2a|
zh@oT;TP$1T!kDj<|2=FhGoA8e>T%10DNC+bE|qWaes}#zoR1CbdDV3<H)s7++{GPU
zYWS6L>6OWc>|RF7{#Sk5nf>?BB&!L5&#PB_OI`Q=$K`eRz3#4x^KtO$SRo%HWb)x&
z#{KX=+L2K=?auS_&0NepgF)}}y%Uz1|2ZEokP={eaox*G?nCYat%SXH#}+PM{orc0
zd2^wEdg4v{CrPVT{|MMAe`}qX*`$cudt{cH?5=7R(ec`Pv9co8*ZbG@8T+`FRX>Ys
zEj;h8y-UzSvwgkHhwfE=i)01Z%VW=8d~)^DRm&-p0&h*w{OK2VLex;xS?0i$Q|f_#
zy<gpBsd+NVO~mEX<R`zT+lWuDa{j3Ew7*zVnNKg%nsxe@$aCz+x9*DSUoGc5chw8O
z`?hR)OJh7ghk$2^mn@O#<oA28<g#vYlV;&b)rRgeFHxu6At|5kPwjr`#5(=vG=1e_
zVJ*?>#ijlY-Ul3IXRtc38FZyH2JK&0_Tw<y*9RZ>S{_vG30rwEt|C=-{oOxhzovKl
zr~H5V)8ft?g|sM_TgSGTPOuHPKUUlOM{nIqB{lXbdt#fG%QSpiz9cj-!t?k;(KFqT
zI$lcM+5P&Ej+A8euVPQFPOESEXZBB-aknAf<V5#m#y^WT^hk?dTbNVA^&q33^Vuh<
z7Zv(4VP3xvnez8to)_|8=(>ni@s2Z$=AtT#l8er(-WSh(805~-_hZTDOQwrknS8A_
z?K-#PmjB9c4TqeTiG0^r6qCJbsHf%fKImZy-@hFh7q`CL7H4UH;bMjSuQgk=w$_IP
zoi$mX$#}AU>X%)s?}afqHn{#>&ft9ZgZ8|+xj&7J%s(%@@b7B+i~izd|IT-h#J&XG
zXA6snOJBaFUHt9c;6MN5)iN2PuWk5T%D}*&TH+c}l9E`GYL#4+npl#`U}RuuscT@Q
zYhV~+Xli9-Ze?PjYhYnzU@$@K<6RUDx%nxXX_dG&JoNTVWME)m@O1TaS?83{1ORic
BF+u<U

literal 0
HcmV?d00001

diff --git a/tensorflow/lite/g3doc/images/ios/device_selection.png b/tensorflow/lite/g3doc/images/ios/device_selection.png
new file mode 100644
index 0000000000000000000000000000000000000000..1565fa0f2b65d48f4308ba50caacf745125c7431
GIT binary patch
literal 119832
zcmeAS@N?(olHy`uVBq!ia0y~yV6tLhV7Mf}#K6EX$4F6#fq};%)7d#7GdY=o!J~6(
zID16!NwIm+L7o|n93Gr|JA;;Zb1rlhn4ooo!&715kr@ImZ%%CDS{Lhbplw~3#;#pV
zo5HSa)$7`8FA^Ad?PwG4RCCP*uj}6LeLwg8?CSq>ET7w}7Qdg%=&-y^ed@DCyBMB$
zeoFl)sJMo6Wu{J$x&wn~24lHF)A8xvHwz0dGMsww_x-1DR$JPNj=G-wdj98sttzI4
zbNCsA*bCbn&Yf@m5zg>IXkW-Qeuj(euhynN@t#^CbcD@lhQg5`)pJ3Qg;dW4JocLE
zn`FcOD3)PDrp-JZZicsME7C<jY*y(1=Gzgg%CwP%LDuL<V~wCcL!rtM<xiiMX&P=}
z=99etA?0tcadOAx<ENz#y^0EP;uAf+bmyL#|I@AiZGZAqi!J%aK}X?}N{^?S9^QBN
z%uKPRM@%}ZRU{2J3cPmy6Q>h!)43<ewd;h+R@<yo(~A?%GYBVzGwPdgB?WgJR^4fM
zQL5H$nZ+D#!x+Vkg%<N?ulN|NGNoNL!y-H7@}p$-!eu>&ul+Flkg{rr*YC+)b5ypq
z{a@yBLiS00u!@&Y;Pak9?jt7+1+tybOnI*2vNG3TwvwQF+6A{|9pTUNnK#Z45E5MU
z?D(>jQ->a#e%oa8Ghy!yrHM177at7$x#)-o?|D5t|Ec>0kDPVlw9cI1t}$Q0Hlo`>
zLTcGM+1)BrR5D*3wOaC1@uSrJ@Jn|rdTsqgpSfv#4HKAsid)EJUqyml%!I5u8@E~V
z(^s4mK7Zm_Z*TDPX7Q$s1B?qBSVam_8Zs56J8oCas(PUR)yLuiqvQ!Drw5BJ@>DHZ
zk*4xt7DLy8`2w7cH7E34Oto291O8Wftlqb)!RKmD<0)l!n>Gu6m&TYM(F;FK;&Jt6
z>tEpZr~gIGjBGubQ#<Ql-3Sm~dr^a}GSaGf+7W{!4Z}$iekDhIKRDcx?{B-FU~#8V
z{!Y2U?e*VUj<=K@az140#QyMFNa4meyQ?A+WL`7~9x^peuZcF!V~@1`C}6|z$vJOM
z=WGV?8k3$Q$C?vz{8v29FbWU*Z~Ogz){kYcrL4QOS(oxGW(k(r_x(!QrEfAxf#1H|
z=37~OR^pX*dd9QgKj!3H={9^{@m_bM>y5fMCS@|#-Li*fF>qUVnI?;GIP7J!rAuyQ
z*u{h`6`^*P6WsswSJ?Raq^dt%_jTRfb@Q(j|2*XM^tWKaGSBo=M<N&1h)M7J*{ZXy
z#lDE;^xdmx#TlNKyZoJ1bvC3{Uf|iz^}AZ9F|eGGk&v+X5-MKG+3>fWalyTRt2V!H
zTHjpEFembK_zA8nRxy=|2i{NSt69|3Xt{xnZ-ZR<1J*x|1{}=EjyfJJD-*chY~om#
zE;)lSt(kR!$du-%3A`(qqz{U1V1C2)+d<fZrLXz^!O8;880PW^uU;~pchs-o{w2tw
zs94g{D4{IYq5G(3hq9fAf>4`_@<$J?Nqs4bLN1MG+{Bcgp72`2b+hM+;(i78iH}c2
zE|J~YYU8kLV)zMn5$DVmj9G2B7A_8upVI5Lu=7QV3HRJSv4yHHg0={)buC}m`=aoR
zO%?B3hxv<ZB@S#cP)qCFnP4Zu-hK4a1|Ofcvqw*D;94V=eIV+={~FfU2dXykTL`8<
z(7D69?tzC5=kv#xKkob?w1?|{qxmECkJ>+&_lW;@S|IGhd5*=NX}Uq<1lJjk8mb`z
z+XVRp{0=T(+5KUfhl)|>H<$b$Cd-62cW&<R?s(l<-Eq9*?Ge8u`A!!jHOt9N9vfBU
zH0OtSMY-Q{wes5K^Ga#2diF%!6YriB6iG!o>8XWJD4*0nL4Wf63BNUWi9~nFt4>o|
zaAt|l;+&;h0`!7*Y5WbbU1e+9Fw5iYGQ9<1LDDOhuP9%+e(RPkYqr#EJ(k_M^;vdk
z*4wLAi@!R&X7-)`%;oU~wTmKKmTq~wC3#Eqmg^-}cl+dg{1-pJJpIz^7wRv*zkI({
z@|N<}owu0YR^GaLtM}IQt?@4cN^(kXSOu4Km1vcGdmVG-<;A5hZoMojsoYguy1JDA
z)x8UEFDJj)`SR$ir!RFc-d<TA_I{<E=Y2nWpZEpwi}t(zZ>dvfPL43P;IT_qOxAuZ
zviNjEMne7p(-USI<~MRLq+Q5fk#BL!;O2%48*Y8rkjO5ae9Lg>nV4rr&+ML+K0BV6
zcKF8OmcwHF*%D=yH>?*wbUWzgwAg5|t=sh;JE_e*hkJ^9!X?Zl?;9WVo#S(C_8Ehl
z2}P%tr0+Ob@c-eThpC6Z^X|{>nq@Wf*z9vgcay3PZaIJEj?N*UtZnDsv_$q>UbeYm
zbGv5iPr=W5+-<H4T5lw6JsQ}ZuH!5=d+O;`DRWc4M$L+Hi_+3DkKPvfE&67(YNTlN
z+l{-@eYbwwcy3Ev?%k}r*~y8<=hC+M?me3u-L<xV?TNCrZ!>R~mwhYKEVI95dee5R
z`KIT$#BW}|eZO>pHH)QB`HIpVuRKb=6rL%Jd0%mov-DGmkY%WKY>C_4y>pJ9dp-BI
zyMWt|z6m`mWD5_sK08#VS+KIa(qNLMQB}&v6U!#*yWQ;ZlHQth<z--ejBRXj?Dk#v
z3NM#SzYcqI?a{YWcTdiJANRKIYV$k6cZTN|-+cTu<N3uy7p2|qe@Ut6`~2tP=BuAK
zPv1WM%<6-yZ{I#<eRub<-);Sk{&UU?%{Si1a$n$n<-YbB`yZ7*pZ<RP`S<DjN9~W-
zFaLkp@LO_A^0vO0JR7(ku;%c+;7DOz#-7LZhGP@YBi5^IwmjST#8~pV>kgHrq;D?Y
z7~is^?SvCw%bv$S3ZE79wI(eTT$s8rH}O}ZXJTmL_rq%sZ9eq+aJ90Ha+<45w~C9K
z>$;9T2TdP;+F5$%-o|^$&wFeoUAw(HR&`ZLycN;y&Xa8KleW&gx?{KDHp59XmPH7h
z?u`|#)<0*LS9K@<ap&X5$HM#TW!{U~TmLIzQclb|bgpH4r()-6q4z45Dw>m`CYMf9
zSG}&f-<#7na^@w&#49b9!snLHTGACf`?5#o$;{iAeyO$j#u#tRp7ealCC}^XxBd0z
z)Rc9doN`CAefcY&r@nXR`+4|#-k<reQ~dPmr(aVGjI)fYe#-od`d93@Az|x+XbFWF
z&6-J)6LuPX&J1(7wrWjY;Di-USCpT1tUjW@S$VU0+Wym>XK&6ko-ljTZuR~-)8~~3
zEYd$Yan77HyR`*TyRSq>X}z8C&b;>Qm$=9&IkROZ`cE@|E`5OaH*0J2Ble@sO$VA@
zHGMapl>SEliuM$dxEU`^ia(xFcYWw;fBsPV$Mmh|w$0g_@U7YMjpgj;O2x*{ww>EN
zFWhK-cHNObvfL}Ui^To}zKPni>BreaQ(ARZ(^NluTh({`Shz1_cSu;(WB32^=6Z+3
zJH@x^ev8<X_i?(B<ISB#*FSZA^7sDXeeUm*Z^2LfnZ1ptZ`1u8wJp)IaA(TTucD6H
zWoxcPM{e0ve04`?@9N0ar^EI~8ke5E*>zD=`g+iHt!rf=ZsBI@H%D%-xw~8|@aooA
z6JFVxE;H@UzPDC<&Hg=#70!p<HupZ2w$BsF-?h^+sW^FUZ0=F9N7I^g`?ss*+y32r
z{(R|epKVj~fA5Liv^UxPPv7Lyd!?n;zU6wW-Cv7-y}MiO?XE4>CFRA<-iM#rp0Rye
zeK79mz4ZM{{ukB8U)gu<uibCyx~+e=K4W<75UG60V)MrBo>#71oOA0=UBR1!*Tnu$
zW8UBUC-=AFY3Hl$y!`ssPPRcsDGxF(Ufh>_DcSb;=DzZ6ez|*#lMZ#RRzE&J$6B=X
z)tUDX_s_RDYghZl=fF(o^uOoiY@Qddd%5TO&tu&4#QFX_{CT|5Z2h)l-S@<2>;H_)
z%9Ouc_oV2;)5qaX>;KeEt^WFF>x}HT*C(&LzsK;U+i~B6XP0j;%HO*uwqW=FbANWX
z>`wh&@;>yA`QG^7`p4_vzWngY;*Y|YgRSkH{O?OXS<knhWdG|*=%*R~c0B4lI(gUf
zcmDjP?d9=RW*;N37r*@cZ{8*Qum9FmKK$_L`u?W>yDnHS-2Qs$o2Lg>tFG^j&#yQB
z<(nNJbN}!5%KYE=r|vbc+yA)veEVhjU^}~)^FMZeS@~)6<LA@#1LNQEJ>li?s5F|F
zrQL9_F)fYhLzXqemk<Vt0)EH2E9{=HWC@(_8_MxVeT^OWsw3PDY!$(I*Da4T+~97%
zTXTZ<{i(H8Nd^wKHf?QgPiD%!%hHpw`6zIYpJ$I5bHf#;1L-#pIQ;+B*myoVanp7N
z3$xhTeIZ<e^LWj6YF$k2{`;W%r}=`*4VNp}b~Q0DFeoH@1o<*BRH-pAG_)`<{9<5W
zXn4WEP-?)y@G60U!D<ErgLuK5U4b?X3=C{Z-tI2{!GJU3`yB=b2F?PH$YKTtF;x&|
z^bAt@z`(%3UgGKN%Kn&%kzLwA<c5?L0|Q@ZRY*ihP-3}4K~a8MW=^U?No7H*LMEsS
z6VbYMX6B?PJ9u0Fe-^Q05-&JdxWnzoKk*Ojo{>`{V~xMLSggM36&T~7!_V*a{k`Sg
z{XbN$KH5{;cTQ;4`h7m~hp&c)-3-k-6S(Tgi{jI>s=w@ecO+T*O6K9;F6ndDwM@!d
z73cKr=IJlzpFjV5zm{+JuJo_3HwVSUPx^bf|9ARfXN{mDU)8(Yf4<beD7x|8rg!Yl
z>sFgQJh1Or)&1#L-f<dU?RxfSwY~i$_QDmPgC74~u&-=d(EZ=DYrfX^T>rLjLc+}G
zE0^BO&N|+ep>3!A{^O(**PH%)otUwHxqW50+45_f*8hGS&u?~Ey8Eth-yX495*brJ
zzp5AevT^69aG|UFr)~M8v~suH6?^%bi08jAv-~{!zH8@(I{P2(&;GwY|G!=_#3K7&
z<&=nd`kS{%Eqz*Xzb*P?;#Q+m6DK~-UKl+y(SMCZ>gh_gJHm%EBf|EXSNvN1K4-#B
zh28zKNi)O_O_FQ>{BW9vT5z4f#E)O@OS^04*xanxb)HjVo5)n9e%pB!eWIPKI-cqC
z@BbZbeB0~X%dHobSNZMv`M~~pdcEcg4bLBUcI1ElZ=SlpY(~=|_jB>W^OP5ad=smE
zcsYBavZ(IEU0;K5o-9xDES~uH%l+#u=he(S;;nXxdwq=J@ccfX`{=CVGyE5hL|>Xx
zbuv}@Mco{Qr?s*>Yi9o6$}AQ2_3U&x*`;aj!F}88e{T)nxY+ZzV&L&VGbFWbgx}rp
zUK}gbI(<pLP~z4*`_BH)m3?v~K2G?Zcho(xQ}?B9gmUvUG(Y~%IDbw(DyBZYKfS5q
z^2C{6E(X^8I?Te!<@V##o8L+=&!v7+Ugomxtxr>x<&(9+4ckjXE^hvRWnzemaVF0N
z7o8a=I)3gG`=@<8;M{?#ced_#Ki8GLd|~Q&W2wTd%2kZ98(FVCKV)xy@~dfkE6@73
z$3!A`EB?BC-02zD_rCn@quY6NdpHyLDzE7`-!I?RdVk4YHM`j&Hnr2IyQta8PFuE~
z@n^O9<TuYh8VPT=N&fQn&fl*;d3E)lp8Nf4){oy7-z_gJ`X^;#FK(=9ui7__^}bl?
z^_He?kunVxrx<B(?d-D8+P4z!?AhO4|4V7@#V`Dy|KE8i<)HC%4^#MzYlq|mBjaZA
zd)nXP+}Hc0?t1?k>8>{t^8cpFAHQS2wpL$d$=v8WHJs`$FKb((OB7Y!tVmM~cw;B_
zYf8zx#p+3KrF86dzw9`0|JU_)!#nf;eSZ7xe%$|L$ES-*v}^(+gOiK@mtL3MeP8px
z(%I9A`X}$q_xrx^&JXVg{~oQlnDFF%$%or2pYy!$m({mk`+8^Z`d=Fb8u-K1=cg@Y
zomFI?`BiH6F+-i@_kT_MUG+ipW_7}~c&6{VD^6E<F8SX)^YHV){TnnUatT(i_WpO~
zd*@Y+`09y1uXWk?gc;3q5}f_iqjDAVr$yo&Ii7{PEtF@9hqja$PFVlw>D%H<tl!oL
zG5@mveZElq$M%xxE}5aW*VE737n@Qj&Y^4@|La%Ohi2zvyWSt!@!WMw+GVbfr#>`(
z+WF|ZxpaOq?+u-DR?~asfyOs3=c=wg9As#H-&prI*W@1S$hy<7^ka5DT5CI_w$UnW
zoyOk6Gsl>6BOES9I7B)`E-+qcZY|87(zlau-_!I0w}?!iP07r0b-mKvZ)Q}-RJdol
zORu>7Pb+ufCzs$`ikpT1DjGE;vlyr~I2H)rS|YZX|Kr(&4?0rk9XcAWEZoQ;#GcA?
zsjbgtp>f5A%MZLXSez6Y)+ZL;jWNG|xaIQ4lP;Tgg=cW<1!k~cU9_0vl4}$HD~HF5
zyN`RxN16vMKe}z9HSZ(IOdX>cXW9fyT|xxUx>awuaPnS+|Dq(b<HqwBZEJleIH98Y
zkk_%@!Z*d-+;6j(d_QH|z5LwY7w`JsP3<wc*zv?!>r>msmtLQX9&TBxQXqBAZ6a4T
z<NC+|sZ1_e&*vFUD~uRR#Dq(d@|4maXPr)eQFu(kvNXoM_FQjItYfm2-lU0I<+l!G
zNj8-|*O5xw_peFS^U)<PWA9lJ>ko8$$^N*<V(_5p@12VF#8Yfpioe`gYLBXG?G6tL
zbX~($67kmStHX!SJg(sf*649Xo2^|Q7|8eMd&CB-tV55aPv!jZ%)V%TBzNxbgSL~U
z)~tWnBc&6aV&tJ-ER%nAQjl}j<7)NquOA$I(iS^~)0XAFWtjV>dA)6Ai<<0W79=n1
z)A}`QpZsRy`VXBGul$_YEWcpM)XmKYlOui<xPD~j_M7omMb$SZ@!3mF{>Ij~>FevF
zW<1>SdBNh;`id7q+*{v?S4Hx~RQ#EBa{bZrFB=Qnd%x$daqoOzZmaM{L54d=Q1<QK
zPVcYnpXK-bVc=z7Z};LAkMpC)=jSa`yt6an&~cwR`;|5cMK}KooqLwY<<nRF6O8WR
z9Jy^5E?&DLQ1-aMR3Kc@i*KWcLW$EG%_E+-7i{tGJMf%)cfez2hAn;m=XnHDoP^E%
z9t2vmY1(>7Z}`k=k@Jm3#);=bo!68{OqR@BBozBaEvCB8-kZfSW%0r(l3vyedpZ_;
znJcGtAwZQyt35#4WYPPE{G9>*yu31s7nn^0GS;?V^Uvsh@3TjtT{%<I(Q+kUOu4Rd
z+rr~=R<mA;%9h?`yw4%qs#Kxi+H%m4otsZZay7fNLX5(hGh6!%E}oxb?DJ(e!&+6H
zEvph|u=&V}b-xW@x@nbDvMKb?v4!8R6>K=>X?;Nc@WTH+!sp(7YzpGk6RI>&OG;*v
zYLH#V#1PFArYz9KVQr-Cc0*k?tEYGU-y=K3ESI|%Xg7uGsZE`c)^ow}z$cf9t|zS*
ze-TVA?Or!Sat?RznYk-pd{%7UY;e=|*6u!|Ro$tFOPGC4u6a$_=ny;k&9<b)LODAE
zUUO_>`Nqqf9w;Io8qVU&A;;3v<96!pjZ%i=>;j#IOPF-BWq6J#FLJ!?Bb9!$aMObr
z(KB-jA5LC9ZO4s@+>(}}$I~PQd3e@OoX{yFpz;1=Mo0VNIhzaPUTtvtP`ap`_Ys4?
z*qQb&rWw7ONk$KoJT+f2d}qD1If!Xauq|iUG$A$N6jAN?QuRpPhrczO&+iOOJO3ic
zPWz_ztf0WU`zv#Nzez2Lx^!XYt5qvmHLA?aEP`kAGELE!)|7tv-~s!htC7pXJ!Phz
zDH2lmTl`?!-sKym+17G;O;vNBWxceuv2$9>d^G{Jz-204V$;nuxl|6?Dax|R#5{iE
zk>*o0)#ophd`P#rn9?r)JtfNgGr2B4KB2M2O5{+O=+QIXr@J=VXU;v9C7m(h)QWqI
zPxqKz-MP1_N%os!fTyE%(CY(l_bA^?$uw$ln8s!!r1Qb4<W^{c(T$~E>B=mM8~u0Q
zySd?mL)IgIgH=9j+#Fk(y{9jgNcO!sp&@qL1FrIQB1LTH&uwzyPq;pTv223e3I$On
z_L&DtB=t{h6uQ2^YQr=`<<70?BGcZkZQs&7l|ArU(K?wDA>9L6ikqjcle{!Lu<i8J
z0HxmNZQR>-70p)G)>!O%(E4<RY5B6Re%6-!Tg;|xi!{E-uyN;v48?b=bl0zxOj-16
zPG$ep#u}rF6DkcwS6@h;Pcq%a_~Z0sz8<9rm&Jk0wLhqYbQ?=Ht8R}m-L&_|?XK@S
zm+Tz(y?y&xt0h741b61%kfZIA&I#h1Hf!geU!=3-R=3>eDaS7I%PgFB@Z92JTXsur
zo4*dR(jR!sB?{__PU$Uvqq$Bdq%MJr-KV{8;){p6(zhSHcPwsIPDog^Q<;OuXd%ZV
ze*USym;OEC6*A$Q+rC9I*5SGe)s}0+JOX4p|2Qqu*los^{U$W=!!Ac%P0_<nMN69A
zmkMl2P_)^4QD=3)IsxaaKVGvjnK!ao97tF)kN5N@ai6`COEaFV5PQbtV&fq$8zyeB
z&}`9^KRcz?-AfUE<8!L^K%dWU!J`2tH)nnc5c0})Wo*7Hav?1A`j<CuG285_m9LtY
z+&DHfS4^OJA8U#3vo*!H^wMVE^;61yb#3(x=5m|5B}c6PU7zxWF{4~{dh`a32&N|o
zR!TX#&v{Z~;=lKhz-KKUufS!C*tcG9zr{A`Vdw_ytHy;*%W5B9N%;H1-Lu(iCX4v$
zT^|fKO_=n(<a_US9X_iSXLmIx9?BQb4Aok>mEpaK?iBqkUdq8-fvz)z8`eHQ7Z)HZ
za^a)oo}BU}MSGv#xIKmMfRT%$@#gYLc9Ki^59Q>jE<SIU*yQ~{>B%SGcQ=;QevVh!
zWPXrOYWBj|S_fyh2o)X0$Nm%S7XIF6saUwmrN~2hJD<JtyH(5yhhN@ukaCK?o6%j=
zwq=zM+fsM=nN!W;&zwIr!>Z>)59`*}E~du`Qb)Dc^qpu7aZ=T^lx10QNanzgniEw^
zSQ=Dxvluh%3#>N>JeXz9k@|JXgALzTJ(JHoYxb-~Z}W6339s22$xUL#Jw}n4MH4)a
z&3Yi&x!A{n@ln)s$GTNBpR#Y!^*zVL>zf{^yOZ^f{wLn5kS7csDeL!K2s^i0Ct%Lk
z3x~5>x9s?MmPzs0yBXPEjW5_fxt`u$Bm0k)`<fHieC|`5`Mx|T(UQKjRKA4!>A9Bs
zDVJ_Yam+NkvDH&K`kV^a(TTUOZM%Q;KzA?m1fhkD$%i(6_}gmBcy%A6?}5|>`3p)9
zSAXC1eP2lHjZ*oKdGk3Zo)ZzVp3e}^@%%t)fae8f6<Ni`3FQvi8_M4XpRzjI!*(&J
zC(}YU<hUU7R|ehX&QIOfvR4af@bbQjJaS@M;kQ`{Pd1f{T#5bWolsFdcS78HlQ-9A
zzV<!LxO<(ucKmx!k9Xe#`o4dh{P4T_o_jNQ$j^O${r}4UUoSuIzkmIbYGi%gmWsU_
z%;OT~f85j2ZhnvdTJ8LJq12r!Z3~utIsNr9^TeixrQfexG+uiBQ&eew+1|H}nfDhT
z|GMp&PIB!xF%NxLW~uY<6<=RcEaO}s?`hxvO=9uye8*`M-`GF-_4)BX=6RbkPpAIs
zmS<pSVDNNt45_&FrnbB$#P$2XoLleI6mMRb)G*<K<AgZZr7J`xu2S;V>}u%Z63g)j
z7UWilaHw(ASrEY>s;Q;mCA{*Ildpq^(`hx8o0C#L|9-sl*Sz2J<|dkP?%Q|&{yh6n
zJ7-##KRYvXXZd;K<vp$;VpCoAa0E?g5?P|as_Eg#1z}EYa{70hhXo-3mb>(ym8)qr
zvO18k7W2>){9m~#b*goI;<ftO-SzKw{9AuY-Q&N>rjApKCUyMJ-z*3XkV##;uCBYI
zqGeRHU~B%74S$x-dwP%m-)$L?*SRLuTSY7Ed3gK#(z(`8_PpDF)_cOgJa}+i`p;1N
zDVRI@(TuWq6_ulJf8~D`0-LIJt18_sD(rK5{f-Gjfmf|B-afbc)&8@A6Hd*$q<rc8
zvizq{9ScOkN*#a6FUZf&kA6POB1S)_EUn<)fo%Eg52arZUGtu`B{%-~r-DPj15LYr
zxoq|Lqwvq6aQo)!Ju}}ePuhM_Y?IY(Yxmw`Ti5i>y5MxD>iVxem0OnliTHk4FYwOx
z9rN?!B73KQ5B;^gAl$a>bkXT>#l`w(x9&|9uUYctW=F;C)i-mErK7HeWJVd;U6hK_
zn8}lA&HCA9>*HCA_LvkqsxH{<7Z87Uv7hYCh!-6tw<ZeC)%TEAk<L8%<UqLJf<G57
zF7%CCH%)YU#E*>E8`s$G(U%Q6-TqxDJpKA6=e>c)-|ox4@%Qzdn}YdAc{gs|a&}v(
zW97x8fBZkM<6bLZsqm`&-r}#t8!!Ld@?7uYp^sg&%uB6bS*_EZXLyhI+v~OO-AXRK
z3|zjxp6_LkNBp@dMPkq2oQbS5{J-e)qrI7;mz^$s%X_n7<F#X9*<oucOMH{BaKe)A
zrT+{Ei&|g%E{`sK_1Sy*6uqfcccMRr+~I%vF2Mfv?=1-}>19{mCb~PvFMGN!<MjE&
z=LO~|4nLe;C;!om`JDcFpYEFAFFL38YA&>IJ+Eo;YUlmAjb%>dZ9jD8uky~&*LYa-
zX<DFP^kbPzg{{{&?#uVq$a=`FDc7)ENAPG^yyL^tqpZ`aE8ee=4_v>(`Ji%1PloZE
za_@?d?iFI!kM>9W5{uWn^&nd8lR}mI&hHK1<=(sMh2Cni-Tvd)p;ICArbOS&xg^Gu
zS;F=5eYN`@Jv(0AejyLGwHH4I&3vrqwW8*TaFWAC)hW@>MD({$IlTE;Y}|a|D1|=(
zhc-*J=JXtm>sD8pGcCcz&^bBk2+xwNjla#mC%-GLdOtzH$>FFww}zwj7M9KZTbEzv
z&K3T;utR(f&rPvkZ0{Da#yqcz;jetu`_<X))bB;-SGhZS9y9*+=<!q4Ei%^wRw~v<
zwcE8?t_sa=eLr`3&ieK{dDTz8bScj4x0N(kes^()<gel{`~9xjUK70LJa5^&)_H9I
zbL0N)D?A%xy87Xp$Mfs<Z@D&Uck=@+NHO%q-eE(G>HCnaaZmZwx7&z(y>(pl{`y^C
zXDO~b9-FwXW%b3ye|f7H+A8O-tq!~wzONvD`-S<3!Xj6R)`x|?75}DTm%CH?ddAJ`
zM>ie#&iA%^`iWZ$%PUV_U9e$q>wm3?$PMa$_^WeIUwGu%%A0E1EgrqBMb<&qBQ*V5
zifLe9cGdL0DBJQasWzeOic=Q7^_2|YWVNa2oT8WB`t$zf_FcbraPv7|6Yb}zoN#So
z;pz37^D}f;t<J3ov5%~sTNWT!mwiI+{?iS4*W3atk5yc+`1aiOqLBTjODmLDpZ|F=
zU-v@Y=~t0nKTS8kvz;kA`NzWid+Kc}Z<X_E*M<Jzf7*6z%f@}uhN_Ru&r7^l$v=JR
z3)7;tFSq>hXSTWK@;djL?7Wim<@3T`?cTBPi{rQ0>3+$%tLN)RzkGRKD(vFsT=g{H
zm$EG3(Bk|*W7E@BueZN__kLUL|Nini<1OWH?(GoQ{@1X3qIk~v*6<6N>iZY{UfR^X
zXWyC_w-~`Cx+kt&o3wRJarpA%4m*RcKNsTOvae2d>lSbQy^~I=d(^8>ja$0!R!G6=
zr$;Sszg^dQfBzO{)#v#;wqL!x{Htzws;Qf#ia{e6+x7zS>t7-l*Q`q2y>!N>W7P|D
zXRR|`Fy-QUwmXyd{?7XMHF0fJ)i<q)xr=1Z-8+{Tx%KDf*UQ(&?@bIg5Uqw(N-yi%
zHiiF8+O)4NSfDQErN7QMzK@r^=PwFlz4E*?!}^t&!yT)IZJ}FOZ|z%?8r7G*_0{>)
zKQ#i}&2~AwF77P+uvpo@D%^~_ro?DMnA8IijW<_TN~%b<_VsmayT3GkKaT=?;G@5H
zR9F2Jy*2U578}8fN?VNAJrtAJYkWrVmjA7Da<4gFD+-6Nc5Uj=d-&p8gnI<P<ffoQ
zT8sHwLm8Zs_e|cS`*X9K<>dV$VGH#y*==?Fxbr})a89ymCU@b(jk6c~7jT~V@w{k(
z@r#I;5yIs?+YhHpK5Ym+c4wm1j=X8N-*{XOMU?gK!EY|yXitvssny*RwL<WIlS7Jw
zX4GZtB^9YhG}o1_)|J~cx1(Qo@7}A0Yni9Lh?P9B;OUm~S3Jw|g3mJVzqb9{!qmLx
zM|X<urIcoHwjVUgJn<{^aA2*vbD-Oo^BeP6xdN{&;WzNQUw6?af8Fciy6M*@_WJQo
zPD#;t!O}A0UC@dR2eP)8P2YC-O8IXSL(}aI({{}NQ2Tn`<Mv{%#u=06o#|8Ao3^%4
z@|UD+Lt~3@<nKMHx}KNfI2;}obe~(p<MCYgROQzo_u$%-6`CoN4!Xt!ON(r;m3(V#
z>5z11qd?dxg&oiq#^1%7)rH@pZ=O%Lk1W`g!SJnl;(bm-p*!ymhUS*aN@X|B+QcDH
z`)Hd^q-=~bpSzf^in8R2$&qtX!mRUBCpo&5cKV23%WUj-O26{Uf2HTK^Y-&CTpp?f
zH~E%siCw-!Y4v40kF-F>84IVePdV)tZlpbJ;SOVu$L;?QJzCzq{H3$+b(fD$TQnIt
z_k~=UmU`jZqa{hfDbr6T9bV&Kr{i~K<@x@zS>RN=U2`(STFqi9lc>CO&&?9ifjhV*
zobUcGxXv)=u4~(grF-KO%TyL{=Vx5`Fk|_t{3$mtyzq!k|GfRV<=-_j@7{T=OFh?8
z_vApe`dZfHb<k$kf994MtNSY7UMoLgV_tEh{IH(eypG=+-0s{z-+61|`kck{G?w0<
zljoT)_BZCu%^kBdgjoOHWA4@Fo|5r9b*jpP`c?*=(go-K=c{xd=3KxU$#VvrN~>q>
ztxwtUa1HCyIab$8#U+)duYSC#L)ml>%Y=qUeX~mLedL>V``f*Sljj%j(_F~<J}}KX
z$jejeddszKV(OyLPcqE-qIsx!{%!^jeUa^&+fFPHe;%M#Y5!GqgV!^Sj0;&46&77x
z{8-31&EVu#o&5*+&xE;649u3;TE1}3G!|W0X;W`<P<O(#3zI$n`OS8|*0RQYvzkm)
z!HaFq-!JdqomBSq*4ICs{eJIHL@dZ#n6iP(ulc3*rRrthHDX>zKHK#1T1gb2NZ88M
z)qT;we<z0Cv^v4+70w*?GQ#hVMVRLskGea%cJO*V`XXv~+|0bHbysawIGePOmVV*h
z;9{jiI_o3U9tm~ngjTK$%655wAeFmb^`f28?EWi)g^gG9-sWjMb@|Dctu#+(7DL`d
zd*P=+r$r|UJ)P`*RYJ3yZ_>%jJJP19%67QE^{!hWwdB;Jtrd3!pLd;#5i4H0`Orb;
zQtmIw=E<NS2;<%;U&QXT?^r=^`p4;Zu4_{?_~Gpf`&kdfG<L6MmRQHeu-b6hv9qRJ
zuWIXZ?^m4_j#jOkyyMj4b91L`{d?n-^JjjW(`%y{Jbs*xe*83}QSJS@&5Qp1$=3gT
zCV5NhR&J%B(%0(A0jIyciag-u_Frc8#C_J1;ZHT6OxS4<`025mt4M0Fuh_fucW3&n
z%WzTU-MO-&+|hP1-)7DNjiq7xmTI!xxWVh^Ruwbvs>X9O_N}M#R$QNyA<vhyp^NA1
zEVnpOG5c7X-hS`7E(;%2$6uaMR1&x2lpeE9z@wEiJEHU_Z^(ROCD4812+N-nJ1X{X
z>6^b>$a{)aWa}=qJ4`Pnd@sy8sJlkY-bgX%X(`Xi;ty_5d@e@?WXqj8yg^xFr^2Vz
zCqk7Vb-BordX=O5#N<SOt*i*(xTtv{eLWiw$Lp-O`Q`4V+f;A;a_w{x+<M#WVA<xi
zr%zV<t+SGhOYHg0`#oUy2K%h}6Q@lm%a8inU8`rMFO?N`#;cnD_2!RHCskcN)vWp|
zW&ULQc|7@gv+D2O$@uRSIL-RE=J}^@*6YU!yq;Wjwd%`_d6Sv_|A&?5u3Z?~v$^NO
zrtPuYo(Hv`{@A+iQLGb(;=A?RzunsRb=Kl@H>HjD%v@_MHs@{WvhE0lY2CkdC$BZ1
zaytO5_nxEdmxygE*D;q))?R%t_3yvA%XwaYU!*MGbHezT^~tCGlj~RSY`e4M<Qmm=
zF|Q<qUjDR~QTfrXSNABr!)ed0j%`XGI*XHvcHdH**7fn&=A1VLqMNy<#-0w@y!)%!
zYrc!Ie^NiQZeMqCo#oF%lmF$fo6vAudCwv39jkN~@6U;Jew)j4GDkHa?Bxu1Z=YAy
z!Rgy|?2S)67vIo$;{KHO3AdK5s8O(untba1jx?Ks3C!8cp9_A!amQ}W+%Tc%&dm=(
za%A)Np4FPV_RQb&sxhZOwf58pbA2tFX5uZ9+i3l7rrX`yH+8+I$`(#Kz9Ty_D)Hr%
zrOUNl!U{ruZ20uxl($y(&TPi-bFO_{x~%=?%G0T96U%wsH9nOwJ)JOZ$}<I<n~Jqo
z=ck3vopjbC^OpY2J=1;cc69%h`>NRa%B=L7|4V)io-HxD^EcTVe9StNpLuN2XTG?%
z-?`sLck8Y_V*AQ`g~@&Sxq*K^U-w^7yYh8#<)?jnUdLMAdn-8it)XCUL2uFb{CCIi
zyxzmU=X-^I*!w>}Kbd#;+&kY}mHjEaBjv%`*u5{rdkmkhYd#%ZW_mvJU+HJ@O4%J8
zho-eo`KW&1_UGf11rfV)TDKj~?dA`+Ej%p0_0)c;SldS>KUJru9r<p1>-mQj%}*29
zn|Tr$x#dN7TK;;xq-P2LWWR-}TbLsog>M;^`mU+;-x|iL{BTD{{LilL$JU{9Y7;Y_
zwQ={yp4tCmPsREikzXNFnXheDuM7UQ^O<fzfaB{I-#f~<zIR=BO8@XH<l*Jc|Aq%G
zITapml;3kC!&m60Q*zgiS1bF@FVy|LQRL&S6p=5tR3zUr$MK$X-~H@K`bu|~pXR;Y
zM>!bf4rHmHUhb~>{dD(9@vcY9Y7=S~%Wh&-dZzX3YnOd!*Tw$Z4+|Y*)HYq7_G0s<
znUAC;H%(RU^;No<SE8eqZkZ<d=#NBL*pU;0KVMt$E>f%Szw&zW_2q`uH9d6}>8ty;
zr=9$x_xaDx*(()RY&g}mF7EXI?vkxfdtNWta=Po^zpKCZf8F=z^UlA|i+}jb<u6#f
z^0o8P#;5~VWu>}&=bu_Xv-JJRH8YOz*M7EhR5|=o)Y(YBI_Q6fpXtdfsd48HimrE7
z3|M#k>jd?Ge-8E3rRy`^Tfo=gd&d2nTv+_ud%SL@H9sAmD)08M6P8VyEBZ-5L7*zr
zVAH{J-h0P`8$2~Ezn|Rc_*{wW+4Jq^Wt+r)>=007%Shkq>ff2&{7v6>YW1n;wH2Gx
zpB1S~?r4gNbSRtqKl(_c?+vfhzWbK%_|^9-WbPr28ur@rBDIV7+e^-I1Ub*0<Y;%x
zFY`^l(G9iDF)y!#uYUIG%OfM!>!s(en%C>d>OY!Z;?|z6Y4S9TwNuMyYIX2F>$-bW
zp1gM1>2-ff-S3;yT6cf`JHCF-Zky}IpNbacNPE~%ZJqydlj#DsS0zsiJkprvKWM+H
zt0?^2=C|_1LdG2S4Iu)X(lrgP)yHet%gFpHG*U5o-r&V{#aFv=4P$0MsD){EG-jc3
z$Bb1kWzJs|y7ENzm}#BYNq*h^+MmmpsJi(Y{rwXe^Ef|l*6)mib@yhynKdnjx9-Iv
z!zVVkOx5kZ+u3jRRq18?{*`=XW_ILXho{0Xeoef3U0;9m&!o4X`5j+g=YJcRUJ{;K
zcXCI`z1qvSb!VndZi#HVa_M|H!+m)pt6x(!ZvMI`fBpW>x<4mkjFvZ^TOU{l3Xe58
zaTfE{$|oA$^Sn3Bt#*_Cxxc?(ZrnL-ndQcHn^`?swYiri+s}|YdvbBupCcRhfIBXc
zmLU=Qq}PV^2|jjt&2Z+#D|eHg9~z68{xMaB-k9`YUDAz5t9aILIr)+MK+r4O<+J8-
zvwwM{9&_+ldzqg6cir9}o*4l#duOywyqwf_Y)gmaj5(1@^N&gk%?kOs=#^DuE!XND
zcQUG&XFmDq>G-SsrRh|Ibt|LZt7{qu-ZJ3IoBF8s@9nSe&QD>hdsMG-G2qI#ru|#o
zH4<%3ahR@D$+Rx<y);czebWu;cMDWc^BosG;_$e*=iq^72Of1~%r^0C5WL*<QAw1g
zF?3nsUa)ug415wAR=!*|anf69tDu^j8B9Ej?HHp<Vy2i|iRl&BEI!q{RZnAjUyGQ@
z&L%~}6)(?BQ=RS0w|a-o%Nv=qO-^}y+jsedVbpxLH8U$7%-OzqN#vqGJ*_=W#{&ZY
zwX$7o5t|~vhCNt_J8k;n)kpnn#csP@Z`&9Zks}(*zr~$7CeW01N7!EflON?x6@n(D
zy$SHT!g@>mX`?@X)~BThMB}Gic;J1Qd-4{Z(B`@@p7SATFP8YvjPow=`xft;EctfU
z<oT`HTUA!xxxjZm*Xj1duc?*QK98?v%$HuAS$p{;54b67w7ff7cfP~*{!9^<z-4<R
zvpd(Y<juGiX}|2JQnl!oTC40Yu9Ln>uc-0ypICbO{=Q3R^z+ZEv{!y}S9*Nq?P)XL
z7acz8egT}HC$jCRd!a6r`$;tQQTwS6t&hx(ip998-h9<!K54D|zkQ#-CO65y)ybTu
zag!$~Qqrh>4(FSN@1tHmbzbUt-FWV0hG)|cs8px^{&ZsJmiWthnosS|eCkS75^?-@
z_{{lg!(T~yt(8v?*LC02RdErGTzRx1Mf&mQ6Q?aNIGt8}-R0h9(;;-ki!q5YZ0WK2
zv2_o{8<Zlho+{V>AGkB+N_%(0qyEmLox6*{BSv1g-T%(1j#+J^=+Pr+dfaf=p7|Og
z@jKQ>zwpoCy{7(!Z|A|80vjVTE*ssP6`bdMF!21}cRW)ba6Eg{`%dlArV9!YT^XDX
z-0ZO;LMwDFU+u2(vkYJMr_@GV()xD$xg7`98x8AX{&98B*PG{7xxh^9ob<b*aN9M1
zPOtv_zQjH`s#@7r?|fFC^pXjO11ddFhiuEZ*^!et^O4WuWve!*++_`SDD>UWyhZlD
z@w9gj(|g)aio4~O8ZzoOgzlN0I4xX9SkXVv_2K^bP0EEY%a1+i?Y-WA)^U;hLv3Gg
zpWBrgGf&w3xi-Oe-{}qOX7dTQT|2Vx;U4#yTIU;;{`MN&Y*i7=Q<<i7`h2j=zlGv#
zP6wE;`}=TyJXs!R6~q+VGi_tni+DGag{_K{S5A97E$NrH%h$?3k3R2QCbb~5G_i!|
z((GF<NkU5Le0jPn-_QT^EroksQ$a{pP=Lp|YO9-b0w<q3H-+I{VAiYdwAbovzUz8U
z=_LQToDg(B_4lSpu@S4TNxOnmL9nInLp4_=Tj#BlPP7!A=w0=P^}&S>=a|Uf6O2v8
zK7UKy`=OY5$)yQ97*<~Auib0ezUk(Z$B)i=7~RO9mHjlv<Ef@(cVCS~_@-$)Q%as)
z|FUvwj_QZ<m}~pa$egba*m=GB<<rH+?>D{g?<=>czn_}mR(~(`zT4bijQis!eLds8
zd|tf$YnNxY#1}ldx_he0;@G%vOT)zD*H74A^8XCy4gR+1grjeMJYO}(K=aF)g|4sS
zmhvn%e`Vk<UQqgM*_G}7Jq!X)mJFi3{fpktse6-?c+AIw{nD>cb5;A0;)RR%Dc`xs
zGe=^jzRJY_2EPif2g2&{sb6+nQ8X8F;MP?RWh>$ddhT#bYEAl;twCbC$&)&d$ac7i
zy*zT#Vdk<=)29~S`?F<c#@@5Lm|xkoYhO8fvH!DBG-sXHl)T1G=CiKr+?;8-<w(K&
z>_p8^PrJlj)D5S)n-qr5whHh}Z;6<c+E-RLXJv=B$p5?x*HZc-A}l2YStmqr-Z=Fz
z_Y}jVbJN348JBV_S$Q%tr~gc@j=pmGpQY2bWqyvEX~isXdgfn;#mnEYsU!%5$m$<?
zyoN`?@$Tmn)dx$B-#uC}ZJy4)KD%tM=7V)-Q~e7&3(tDZdNe<u-C%Fjvo~5-4`-f!
zvqxv4^Q7QC7uKtOb$E4qrTNJzQJ0q(sY+bW+UoCayXftSDW$j6{!X0s&U{6v@X}q*
zjdL<}G9ni#{g~;oN#Tdy*33?g=_`FY_Z(Uicyh|F+F;)sPfzsm%@^NwB4zV}&2>Ip
zG97;yR@BeAa=??<VBH2jQ-}3YJExpHo-yAeVr>??)qbO$E5n)c<F84nnpPNiMnu~g
zS<Zeo_aKkvgb8nyKjn+hEnla!;(F(ree+8febjrgKYr8K_w6Rnwx>wj{t7rHu~Nl#
z2TzP-nZz`YsJV*{+|YMVdKz@(%#>#q4=xEmHwdUpSyyhfz)7*oPe-_Y_ogd+o0k7M
zqujmt!Nx?Ht{;upe$?Ipwe@AfB;R^Jm{w)`I>+qfyhWvdR_xdL%gicQ60_-xkQ%3k
zaMXz}N|AH=t9oqKwl0+Q6_q~1t>$<jiX&RA+WBQgYII->-<?U1)dib)c6pfiM4m2)
zDn7Dr`~OgJo$z;^i|5xRG98$@CROvm?Wj2+bDnE+Xv%+Vf7t7K+1_JE&};iSCEMD*
z9$YocIr9A#Ht)aZcpmNAa%`IgyZ5FElCm-j*ExG%sP4F;+GEih@L4jqN$#lY?S*>v
zTb6lRD)1?8Ir7CZvD9LNVbAg-6SShmw#4i$S&;IhGj-CN6Tb?Tl-60)Xcjmdmui|6
zZrW<G(!#&JUUs>csYLL0e{tSpuDy@d(hZXQzADSxz0VQ+ywi+lUyAIhJ2QELO$)Pv
z*7MrGj^t20zhTMZTfYh;uT1OrJjBvfTyFZy?$vtvc@sj?_8P8@xMnw%q4a^ty7N1K
zJ?jtMuUz+pdGqcdJ2gw;=W&<!2!Cg|yh=^`)R*2z@2xi;K5<X*Rc38#h3+Fih3e>^
zbBgLpo;X*VtQGk7HEg}SeX+}Z>udVu>z3U)CE{yuA0l;tL5thy@xqq*#>q0TH9IFP
z^pY<9{>adH3roM0>U%ez&L6r~`q#q#xt)?;t67|8d{kC2;>6>c0^ROlqa5)iGM6UH
zo;j5~$H46tpDI_l!SXvwM;<q&eDqdJw+Wjc7o2))#e7iaJzP+itZ;CHyXPbG7c*T$
z&bf*3&Mu!K@=k}3xAU3M;`eI(nG4Hbu93`V?+lq@tNMGA#4L%euS7r{^t1l5a<xYq
zCR-`Fvi+|x5eZ%MvfSaC_)~!>f$jcGb;X*2<{tk-KMManl(b?;_7=~~w8BU$cGcNk
z9>3Mv9kvw}oZbG==Z?@PW-|ljD~$n3{fkOdR5!)T&*e4Zn!WPg>dX4uC!KWjN}tf|
z@Gsgwede`?`-6>S9ACe_V)*j#39%VM^DZf+io_R0{r-5WHZr-<%g%qP;^D{hYgheP
z*EXyEpVCR6S3!O%-!$U-Vh>Dsd2}`Rs#$UTpDtdEzh<=EWNz)>{rh8%3p&l!3$8a?
z`0eHHtPi>klRnmLef{E%+{$bz?c!N8W<@`kr4iTh&wify`ZedaT%360s<r*|_tS4*
zRm+;qB(Xr%<ElgO#%6(qM}jxH-;XUUS$fiD>gG4M`nN>-&UE#Dt95LBYUiY@VhmZC
zhL*k)3+n&w$x;ZOeJb$9ihxZUU#OUvpA4S#Zcohj6Z$!qf4<+8<FX}Y+QWdfyNjY&
zwt6jeN#E^zy0!o8lwGFxcKgYis;g=TuIHV+oI!P)WvbT9tebh7VY`hkeRY<emX<l$
z;)-HwpvR8)lfFyM)Nj_bmWkvyvYqVq=;M)htlMMa=4_T-d85}-DX+2QidC54skLWr
z20hobH&!iFc&=L@Ju!ds(#Q%fcBf56rc>W&*5;jl6Df6W@tVvA?N?$Q(wENsI+c7b
zWAPT*qC3$y1!M21_{`kLQ|ITcA^CsOh4Yt$ABZ`<x~gYxbX;*}f98|eiD%!2H$UCq
z`N{tL{inMQadMXI+0^gXdQCKL<p+BnMFsW=RlF<iGVHkiyS_Tb@z?+E55C8~&z$yQ
zmJvrw$eVB5{zutu7reu?S&i?r&AzK)yAL^kT>m+Pm#a6S)j@U_qq}Iyx^;yPPPYZu
z9&s?7toFO@(Yi8;#}n=U$S8Wr>?$yw%WS!CrzZ0p3Bg+nji!9`5%xH7hjr%{jTEOK
z_Q;Z6`R{wcRg#s)s{_oDmhBx@%fmljJQ>3hIC<I0iMe{G&6+BGP45Kl{vbKEqc0`u
z@DaxIBG+BEN%So5Fji+(Tqg2LSI2c<%;LWnr~TWv(q@@dipJ%K`@HXEr_7#hHH-g#
z(y}b8ewok0{VEq3UcOlJjPXP}<2LSVXR`lgJUXsxpqQ5S^Zu1(95ah|O!KznRt?x)
zaC!?XU+I+t&Sl|JPo)!l=hkqk#m>F?_lxC4`MJCNPMFVo(!JIBMSAS6>Vwl3@E8_f
z-oVK*waw&~;V)HF-{rx=Gg2PSpYr?R|5bu(BC=+lE}9;A=&qJv=(*6J^V>redYYsY
z3#UsJ|6hMdON-rV`KCR^SrM+ix4!7rckNnza>pc9=j)m6(_g<gE>!pY``W>CR)mei
zKjx>$`*R`>eZTZ&QE;$X_A4`Azs^l9U!TX+em*`cK{Urj=eSf(q{R}6=Uz|C&)G}u
z>-yHsa@;K`k4tXG(US}+;6~&Zzsu_vo35KwWZkcMS++TJi)_s7O;-iCTfNHh-ye17
zY)_NM7r(j7UR3Pto^nuqqLxU;ibFe=MSco6G}nw#ROA1Z|4d(Zx^ft~b6&|SaQ$t>
zAXm)%c$+EL+bjRLUlr9^=qSifHJH!(sx0G<sukN?R@0sToR2g8Q2cFbzO&HlQo-k0
zbA;D$otU;SUgVn8yUR!Jp8K(MMNYh&<@Jc_L{L-pqsX(?XErXJtyvS)&kDNpM={(n
zI=~;f{l~W8_H_FVlUwFr_;_i(@HMUWIHTRu&K&LD$KAGOgQ3f_x#?Gz&5>Al8a$`y
zV&=Z(&iQXER1PbqJUza2j;hYYgJ$-LtWRb|Pxw+FoRPh$q|i$DL<!Hi%_p;cm{zC!
zeRXH*+}%stYOWpYyLR=Glg~-*lR_qzM?Hm}eR55RGASuc2%ILB@v+u)@sbTcUOvh!
z>2LfkzuIy?)BjYjO^d&$1YGNHX6f|*y`OP~#l^yl=NbQUtjg{GKfPmJ`epvjDN{eq
zwXC~0@tofzDYc6pX8#1QXg;_35qM4YrR|sX7yCXY`zRmz5ihjX{VK;z|JI)R(y*Y&
z_jP=13x0B{euyyZw<=!#EYjBfi(Y@Fae^qPkNGb3{j7JE_AX2L5YbiM(Vvp<^6~Y(
zxvsA(eMI^*nz}N(Rw$}|ncZI$<y2p|VpFS-M)4QDb(vW?!7L|s3kECIEq>i~ZAtdO
zC(|Zts*5R{mVcnSPX9>SzLITf>sCH(IWBVY0Gk5K_SFHZ^Nw?0ymI);)VcFS+Ls61
zFLC1A8ZgmprOL~QirT-Ado=B*?3ir7RbR3&p5@GijR6U5Li_xFyce7=7!$m}sb6Ek
zf{yslLbukhX*uM|HK)S&E1MpxgoxbbBEC<(<r9{Ec=pNg(fWw2(#wekCP!^-XLlUE
zePQE@_n)$Z{}`JkiAB7<_U_K5yd2So->3ZLDvoe<KU@8L|7kaUm30x{RKF+Bex+yR
zvd-@TLoWN@Lq$g|7e4)>cPcf(S?cK89`<Qd*>6QHn7zMlQsBow9H-u#{StXu@z5Qw
zge55p)Mqke9K6YY-Q$GcIXk1HniGY2eAIg<%#V2U)Fb$j5r4(Lh~Dd)rb=j}NgZBX
zdEmn<qx!;N%g6W6Uu-#XO5c;!`}GRrr5`S<r*L=wQ@Q*xNPWVhov!{ON4x)@KT*=f
z6uI$#_`+qThj*zQvN>(BqG3(kzG;8f=xs83$>03Oxp}`q@%;JKR>p6Y)+*o5yu7Jv
zL$UoIjn_|@-Ou-3-p=&1(Ye4y@%MsdQ`DcFJ*2m0elxGcp9hPB58KGj^Zj~q`ehLr
z|LuFrI+iBwIk#?MPko&Gw<$gK^IDD_v1bjL6udk`M9em@ONdu&`F`0S*>C1PzODl5
z952!_m%Y2$owN7b#75zmV%FBUVhM-(<tw(f+}adiRD5+!+vfi7Rd3E6Z|8acJM4b@
zufF%!b{s$P@A@ODr{%}*-~IDWK5%lLWI(r!f7E03gKUg@xA1MXmV7DyVeU*Vvkq|$
z6@x{Q5`{kl9GvPeYTQ`xYE5FP=l7cDnXJ*0fA20)==%2bd5!m~Z9<`)x0`nKIX)Dd
ztGsW~ibeiE7kVCh*|qfirH#eY{j%OKon&stp|X7A#S){EJw^(O#TrkAmgP0t{N&nx
z^1^+E+7-^Z@mGpey>*tbdB6Ia$z&OluX0U`%ggt^_8N^z5p(JmAIk5o(y!STD7wFV
z!$nUAi|c!RYbTZnZgELFI%%!{u5FpJ%tCiU_Uv9D_g{A5(>-?6dDvt3=vS@%w{=pJ
z;iN}Rw`VbJjXAlk;p+KQ4bvFcy--XwncOyiZqct>6XttfK6!+{>Q-D%*OVzc=f}<{
zuiJF8UfjOjX0!0@oaO?arEm0}E?=3xaJl0N*7Z+=gq1hn0Z)@gKm4)dN9X%LTbCRc
z`Sj~YcdXmwAa8$(33DV59Z%X6CKy?G{OE;^3)i=&bp>6%nD4j7YFB>t&$a$JZ)1L4
zJ2mZ(44+5e#^2R<ESQ&WV%=LZZRH|0YmLd%e*8=^Xno98HnsCn>x&PQ_y4_8dh<@t
zQb}JUy9sYUo)DVo{k_HGSN^;Bb33oH8-#nz@LQvOM|f`f$;Ut0em;-;-`--s)bD{G
z|J$Y37mjJDeDn@)*ugU6<9uN&H@?Gv{%o0WEoJwEx;2uMD%uXo9bK+l{(6u2ovEi&
z*c0}CkgdHXwLQw=*%h%Pl@Iz-=E%?e;<3S9H~(Mmsqd`M*ZqBbWaHt;%AM7F|JoYs
z-7l--SQj|YOjlR)ZRM3C`g~`XpUPExG1p1$M$>`?m#x}nt7f*J>SAJ^_o~5teW1HO
z*UuGCFGNTmVLsY%I}<b==djl8OZm*>@gHAECBJ6TWa&72!KA;SdU37bcgvc&JFFM+
zL|xuf*;cJ``iREqvnRs!b(f#AovPly)i`ZJqwp!qeG;z(KXQ29toGe+zv<7q#<<f;
zt$z;KRDH4hT@`H4DmlaCFIP)s+(sW0)$JUc_ax2qSiiC+<$hVjIz989F;73uXXD}P
z>S&&Sk;!X@w$ysh(>vcw$gVf(H?hueIsPjpYO<FF%LUcndl#!VBwGEGog=^4ZmWzJ
zkMyhi8C&;iU)bHok}^&BRhY}(US8MvJ0B{#ccnl6UK*yWY4!Wz6#p=H`|VFIAMF;h
zs{ioRy7TDod-JWnuTg*hTRd!QiTv&#WzU3{6`!41%uz0}p!9|5A$I5YCt@~!+xp_v
z^12zD^=m@1en<XII=A$n#oF{2cY?3$+qCR%sEJXX`Zr}3lLn`TQ-N;Xr8*D$%TH&o
z>aIVx<KJ4b{P^i{f1gI?uP?DT`s+UBfgaz~!<+x4aN4_XGZfxjS2yEQz^PdF>C4|#
z{r+%U`To9#Z?o^!J$PG~zBgTa{@?Tlv1#QmxcKHTRa2SFaAiq7e~<NsF9(?Beu`KB
zt5uilF*#_8=*%nbCm2uqt8NQh*{r=a$+KqG$A`vs+kS(VqCNQY=WKa>r5jsD>8H0p
zD!cx&{MtC*_`|W^Ps?APU(TZwxbl>d&dItHsf(|%2WD{#y`9eFG;d+|{~N2zANWjD
zJ+@iiT_oxC=gm<Ov#Q_492eF&y+^d*%i0`4$3n+5qUX=3-EMm<#w=N2<Qgp)EzHWZ
z>FfEqH4A24*0=YqQwWX_-rE@}bURX>duGrKamZBUj&0!Oeqyg$uIUv$KdQJ{X)43^
zq=n*hrX|Nr{=9O>+^%(2Tzh>!rECn65PU0Jx<Xool|gNV+nP|ftd_lxwqB47DO~>2
zMB~B2)v?PQ7V||t3ll$M^z`zLJsz9C_w!6wy|mz5=CQ2A<?1ijID0M2m3>s@?(N0q
zx56jGXQH2yz~frW=dWhlPdoPJ<Hi;Kj8E4Vez&ii(>Za?B84S2N96y+PR^|etoxwt
zd6q3=zS|<%ZLUX!Kg>v47g53ee){nk$@pY3N!Qc8?VYkdX_Av8^GlTGep$Tm&k>7E
z_g76jj1||P%uSlZ{xwkP%L}R6jumBJpDzxpF8&Z7tsJco9-68dP^qWppL0yO{oC!d
zH5zfB-EKth>@{MKFz-<P7V>q=+y5WW<+pwF+h?3vbcyLQZ(W7VX|<NRMB(n+(?h>9
zSHF3m_|{LKccySzpoD$GH2xO#FCQNz9*X+H=Cw&|M}NrdNRw^9rU~+H5Y}hb{lj2)
zDq))K;zti}buYRjJ-u#1`C?n`|1*jfDAg_ay{jd+oOkQBcORYSXg_qSO_uj7D>9qD
z-)g-`P-I8Y<?Y+@-~a#C?C)Ru>&bHUfJqxH3rnVE`j(qk?maoPuJ?E3uh5^r=f2Z8
z@c8r5^{2h0UX+KQSr@l^-gcjD&!*fCSQmD}A)o!u-kLkxO&%?OVd(mCXMcR3eu?P&
zuAe9F9e(9p`7y=+%u_#&m3Ma^T+yaKT{qmS$Ux=TwHfnE7O5QP``;jTPCHKQlT+Yd
z`FX#W)p;FNp3T3;{_9K4`xpDnUG9XxKIqu#Jo`&V{@y0v<_Y`u8@Vz541X_Qzpv<T
z*Zc6hRc9w~f1jWE{rK@6$NcOkm0XVvHed5D>R$K!JwcNWuk^j>`({D$*G6{L$qiwC
zDi*&#E;e4swSgf!a=+D{muD`D7U?lIob2mN7c=8q_e1f}JA?1i?-sg>f3`ono3EXt
zeR=D5zWQUS@`bO9*B+|ZFgID&YmzRtx##CerPDeFD^5l))+_!J;eV4+ZW*(4<QAdx
zJ{=XSs)OEbyko8S{r4SqeQ}rjQh)t+7QFYpmo`sNaJEue^!0@@{;j^7Wwpznd_J=@
z+y2kCV_!{QU6t{4=RG?8kdxi@OHUYEcJO2#vy3U8EqYcVeT~?rw%)}~Z;$$mo>`vz
zNUG?>*Bx@7j;QaRENi-LQEg7fBEA^6hb~^fZU5f5d8lXR#jxMEulZ;R&y&35f9Wkl
zztQvDOZq(<olZ#j)^03cSkdygqV?jvfb^-G_oi-ex&Kf3@tkeWuRm=3ee;^)1m|g+
z=Io2##NnftYqx9C)E_^MjOBdpd;Stt-Scr;rmwJs@<I)ToMqv=K2LT}^VR=h_r9^|
zcu&Y1PDA6<|I>c`P5q}_w{P9z*y2x{dzx!h_^MTZ&kL?z?`LN#>%PO`uh*5l&%Z8S
ziCJai6n%E#8kb|7xxUx@IQ+vL7e`i<NpY=a$Y)r+=-r9C(=UgyhgAw+3jQ9JU-DVp
zR(yR6xBRKzH=aLs&IJt-nAJ`2u=M)0>gk&uTR1mp<t<tJ?&ZgYXAhriyT{_tx<6y`
zv?=`CRp;6=M~i(5xL8}_teR^5`ifM+%jP$`_2yn{4_$C(>9+gEw;pM3aj4J8)RT=2
zbDI_Jwfy)b!2=BIa^K(j^yc)O-<$58Shd|SXK`xs^Un7^H~yvwGi>PIA6N4$iSzS@
zsXZ@dnLKkXiVbtIzt9kB`F%%NrIU@abF}+5=9*1wxZnQ?Kk;?NiL)2w7w>UOVRdii
z5B#6yahQM2q@B_iZ4Dl6%)OPZSe^EVd197#>Aalj7kz&mGLw4hAL_aC*WGi+!|Jye
zaV*;4^{w;x?rs0(eSQDu$3e+xw_ozVdp@l_>Up&Cbhh66&3m3dvKG4@DN~pgnYOgJ
z{j$)dcfPCbmS6kh$=xO?`gq&j-B<U$TK>K+s+9Rw`)Q?HORR6Q<;Mgj*l($kpHlDe
z_r|VBHEp}HD;G~zxx~uyxU$UqRWdDJVD6C+<z!JwLI2vsn90h9JrljkSgzW8)Y-hu
z{Bld;y-L2f^F$>>p<|UsCfX0U7fqgVDpeG`2s1TG=;BQ|;frn8Tb_EQg&Q6_cj#SU
z-Yv^@LO0b;Ip*3O<1<%^^XPGbOcxutxiH3_I<xQNCB-#9^ZqatI=1~XI{(ijWsas?
zT7PNFSAp}9$4{y|X+@?pCUMGEIhF_oAJlCxfUH}*Y_0iTZLe{MlWalvcfR_SJdAZa
zmNpgnDYEz0EcSn>*D52D@Ob6iq~kAhxBva{`j*^kd)AVl@1|ajyze`2<LjqK!hfHi
zZK?hEZ1uA6Ise{ji;KnXU-5d;HIILd(>`8#_u_8p_b(2!jobrvY4N!3oU3|Ph}F_T
z?aF?)X{#^H2!52&E~@IiU$OYnR-KZoyDa*T$3L89Wm0r{_N)z;4D?RzKbbk_=0;v6
zzrsyBl&zG4f3Zu2d#?LA{aVGe5096Ne|tN1*SCqgm&M&MJZO@wlrSkr=h%(NYYe4H
zY*MnXo`0Vnv2;oN-K$EDE|-s38l8MmfBxmF2WwZZHQaRVORn{@i=I{C{TKPSBnMqB
z%%1Ym)7$5%&Z-zS?JJxQ)zb>tZS0?dhACcsxLrQ~@7L%9S_^#zB|A_0`bp|cyDhRW
zchyO!)^DBtrsYdF>lMiTPBlspZ4FxRsM(eC==9XRk2SYA+<rbWWKT`_^*e9Q=6+2S
zRkdjq`>`YC8@s(<b9iq;>BPHd-@V9s_Hj)`+R<k1_?h1y>@oh__HK8%`}_Fqv9@&@
z7e$qde=qz!YpHro%}QIY>vHAqmaKUzV)yV@(m&VdN2>$bucvA=zKhFfnttZ!lxq`f
z#jf5vT(G0mf4_l+t>vqn(}KCPpWEKBd2-B8H?4gC+V98Lg`dA>U3`0~<o-Fa*PFEO
zt1;W&-eAyhVXtJY@xB9HxBk7J))KGn^lW<8vyk&vTT8EBJHCC|>mR|#AIp}TKo+TR
zKLaglcqI3@ujs40b?mG%yUP!^E?jw7oqw<Jd8WLDz6@rQg(2yCigf4BmvL!{A1@!h
z>~rrGi;BdD1)$CdWA4oNJ&W$VXkGSQl~3!6>+uyc-~BvynQQN27Q@vw<(DU1V!v4)
zYt=hzS#yBrIlV7a_i6t9w%I(PBxI?<PEO6o`!YDsZe&|}tv}fOQS54$ZJk9QraUS+
zdda3jga36}uE^B(#Vv`On{G#6yVHI3y!$S>GTrie&UEP%?j;wEtn&P>NS>d<@$lA!
zmkXPD&oPQD{35(s|69Li&8+wz?_P7ys`<HoeVN{UhyBtfdL8eCr9bqvd@UBW`5k+`
z;=W}JbAYH{$@P6P*R3z!K9{Ys_`Lkg`}+IabAxYKU7TRoeV^k4=ed^^jIp=2?6X_^
zwY6vcrRV8-%oh&L$*ZqB{e5=b;p+0cYku8*9oL?|u`a=v$z|bz&f5+vWD0y2uPvxu
zS@q3&x?7Mo`_>IxUsfG{Ja^wK`^*Oo|NI~4J-aT+Y_xxM$-Y+`o_PJ0=BX_@vZZ>x
z?2K#wSFU+Jr#?kL*6Pvancn`3Q?Dz1PYi$i#`Wgsx$7mjttm>My=1n7>Go2IKNH@o
zuVI<;`OV66bJj?2Ov*eYy+(%1#wXgfCq<z3{?hlRr5Q_1Q-40HxT2AE(?3VF`~O1C
z{hIlIG*}N;PU`LGZ@O;6=rL`kuk3HH>(U9!-yFz`nw&pjX4UcSeZQCl!X5IqPQLlv
zaao48-Dxid=hrVH@>3qT=O$mwx4-b~*WaR(C(Pt`eknP&s4mP*|I)_EUsq0>`th8U
zd}`;RQ>O!&x`P9~UVT0u-I=whJwwaOWm|vWbJ?BmKIxtn+wQlii)*E*b8p;=-1hXJ
zzi$f9y>xZc*=L-#&ZjlkXx$2pSehZcF7f&!U5o1#-|k=VD-m}3aq{2s-?kr%A8t~%
zJ$h-wB;If9#b!O<Exxt>oYk3MJGai7UpF<dK=s|PE|%XtZ)2n9{S7TWJK5I1&-<>a
z`x$ZWHq+hDuK$p|yL->=z2(y%mR4PR6S(~2)yDyQ1qH5NHM3bYzeB)jN+Yuz#|O=4
z%D){FEm)ZBp7wp&@@=o|=ZSyAUr*@%9uxCbea)h4S`B)y)j$igFGlTA@|vn3$2ZgO
z;hHzkFR(5w<C{{ebA|2WhB<%E8=c-BRTkTxzb!X!OZKs!v$K0vRu-@R+SkZgE$sBh
zGBjva-S+%TVH*pBb~fi~Z>v~zYK>I$tauSmrw+HIc`Jn9oqsH;nRI#X-Az$z>$1<D
zyqeB$U-#qc;rXfjr}|3d3idxbS@hE;`Hr@&ZSdjCN&auYlzh1IiD%uF&8ME9<X+Y5
z^<&we<BvA%2)ol@X8!g@&9CcMZ+35bZY^%=pHY|pd2e>`wU8+3N&8fjGt<`o?7jat
z*KzX4q&XF5JaYDCTxI7Kx1VrPZSS|6`YpkKbdEH<VvdvkQ1iUsYWvR08H^h*xy=55
z!Zb>Clkb$~Z4<s_zWO)$_gcT#=dT@>eO{V%rLZXeVzuA=+r~l0vNPvjwo2Mj5!12m
z>n_{-SIs?mM4H<VoN~Nc_+j##AZ5)+l`{VwQz|N7uUory`?SwJpLIT6k)LJ$<;8`<
zx%RLAJUDy!PmdFa;*o~kY<H{{sJ)W$wAl2-BYg}1>CKA^?>sJ?ef+zG`OA&dHZbJK
z>`nZCOnF}6Jkfc-Jj>PAuQaUqu`E&3S^Pk6*LuBOW@RNew`Ba?w|@7!S^IW*J~NIy
zJ9oKoo>xx9(;u?mzeZo$zS%?IJ6Geqz@W;4r>ax;zF#-%dcnVEeazNZc4Z1KFv{Vp
zw!6EzzCUsQs_8-YEAJ^dq%y3E?2G)Gw*O&<%=d~IuUpM4@A|!~-*Mrl@>1ukPi1d~
z&y2Ou$_f5)dY<7ur~J0s)F-o6XzvI-ct~Js%4*>}jlMIU>-*)OFEUYNDO@Oc=<40A
zyA-&$&E1>-?&STwpD*V2-;$F}^xdKsSsl!9#&K@@oa;}lX79IsFEx3Swc{SmmeR~0
z|0aBHU9@6b$KnPqmWbBu!_%f}mi=unzM_0CFS7PU<F+fQt6!aJF}>blclr08%fDZW
zo?lZ@uylp#s%w*Gg_`O8efeYA!?&}xS1-Ps%p2>w_w4Vy>Z0_2@2;Db#D(WyZ9i8V
z`E`@tk+QA++ugp3)n2O7&)KnH<=&s~o{ERhiOstqy73UBhr;|N+@AZEsLd#RV#f2|
z@Ke$o-g`5r8(n(8(>U?M-e&EVb$vTNufOt>drevPyI*Htf6v*${J!+QaBcXPziu)5
zySNI^cAd(V)m=Zk^z93_y*=+gJA~_=XTRaIbC2Er&fkH*3!=k+Uz=PU+`mkBu4bC)
z9Nlvk=VPaTy~Ar7eLbs6KB{?&nauq^r`9`9_y3=B_W13idWtUf$5-uWE7aV}wcGX1
zuk6!X%Kqj{Y^=KVYWKAqt2_3oUlvw6b$oB}=(Yb@`057Rb@Lau9^7c~WzLhIATU|P
zeD$TeE!+0JzJA&M{^Y%<r@k<ZlC~&5aB#QrBzBeClX%m^?Xz|rdw6S3PdaO{gj>rc
zHSf(IL|)6i=E!D@J?zG!aH>^6>6Yq=y880#bN=Ym%>Tb>Qtk=!7_r3Y#dA(4P5vQa
zdBo_{&d|mk6$>52MEQ-3%**;k%nq)NxO4Yni4Etwt`q<MKNL78!|}JL+s@m4;>Y}Z
zr{3z8Z<}*o_vf@%f4E*pbbWb!qT<5(^?8kx(&vA^e%Z6^;fr|PUc009kw3Ot6*#EP
zbYNUkRbusbm2PR9QijDm&u<DlbzdJj_Qi90^z_x!?kcZdslgh!TAX*n*U6#!7prEM
zvfugn;DP(&l^Sa{#YoRe)O4S})PY-M)BW_5b>{?*gU1`cyRP3DKlOLQM|n$UFXOl!
z9G?tVHD12DEwYh;XGN-m?~2a3es3Ne%KY%E@Wkb3aeMb=ykXY~7qK{4=c*gNW81Ao
z%a3ejys+y_$UDvat(%&TygDLr$MaG_&D7q$8m|Y+@A9e(&;DzkcI3|tVfT;bUs^c+
z>u{E?QtSFTvsCE#@i}Mm`*(|qT`M-fCwTAurGv%0{r&%bx^#WJwXM4O`!_9RAH+HD
z`2M#2{HEXK*u(VozhCCXm#+`1FZz4$>-%@Ho4@PUTHX6r@_ol`HRcu9<-_t5_CD6w
z|M09?xv6)<)2=?I-<MymHZEH}eYN#UzqMBJjK?0_3`|U(C$Ur9bmR8k*PgeR%gruH
z-@fy1(e8e?7Db0{&$W(fs-K*9zgWh9FZvHdp5Ho~oxh&gY}v53;`O(;`b=SSYra3}
zS3jKH<GXJ1mm8;<(@d&%3jT<nz3JJ}t%@61l0>TS-w7`L-rJE@^I-Wy_I9;=`Q2;p
zed|?!zD&+{_TPx}A9LmGubC`;`ES|%yo~MJ?#_w~;MU-;5xJq?epTeHdGv*xxXbSj
zoAXvDyr@2QtKz)voz2A#Um4f)uILO*+2TLHyzJ4j^X7RsDkhwo_Fg!7uFi*=KOZjX
zXUVPIzt{1(=H+E2vo_hiQ$23Fk(qJDB%?`oK31u#b5Hlq=$K@B`ud%H%S036nkVw^
z%l=@u=$VWp-)`l)ocR|^<qP=#7QMK*@ZOYgzWXzS<TFbzB;RV=GIQnBLfZ%ZjQspN
z7AG}l8Bba)aMIy%?uyG(&R8)@PoKDUUHQjl6T+`<)Mk&jzw`5h!W!OPpZA*PXWzIs
zRoXgd&DM=q*Ea8J|GtL*1A|@a+Jiz#n=%qh1-Cm}I22V+>3N{nk$q3u{=&4hGtB2X
z{|dPu{XYL@ZQ9bA-;?`3e0#V(bLaET%kA&=eYIKmbMs4o-PP9HKJx0``}^U+r6sBB
zrTpf--`QSWc4(7J<>Ba=A9J2P;z`ywdfR;^Pu;%rp4df~5a|$$H#Z-aZj4%ytav3X
zKJQ`wyQD1+vY+ofk15|hJ;7ze@};3Z+@80&fA~&bkf2juTmStYqc%JL?$vJZEobQG
z<Q|yTdhwy`!56PYuFt)mVC<G2v05zkeEGh%sW+b%98b4ad7OLn+oOf7?RnNE^?p&;
zc6?qOzHf`oyVWgfzR@u`-(;hj@7T$HOYL1Beet4Xe8Mz4#ei;s(C<52+mAiJ5SnN)
zgYkgC)qrV-q(y$Nz41KDW9QVH%Ssk3&RjA{XOoTgrk6+5H~oBiT<o~a{>oJ<_8YhU
z(X3m$CFbJB1H1b4LwCwvTi-Qz(^=ozZh`-Hr(eB@o_r<u)so@~UptyhQg{AKQEhBe
z`dV?dA>er33Ck%rSMI1`>f=0keBC0|U(G8j8|_Y?{9Y82_mC^*w4X9#pvI#+bL+KU
z$wvAz=+3(1<m-`l@V8Np@{;?LELnoOe%%xmmn`^aEBC+5lw$(J#jq`N4(m?&Fl)i4
zv;FRlO;Q0XAG$1XNw!b8>>R~?Bqp%)=`WkdHTjEGtd3Y|KT%}RJrX}v)V^_<P-FGf
zU3_-z8v>s8-LTJFd_v<^a{PUbzwh>LKmR&L?_y!?!LQcob@saUSFT(Cp7UGi)r0&$
zjgRuuzt53NJ1M%~(%4M2_V<I2)5~(I@;(0jZol_yr_HUMTdd#yS*dIz%2GGyQQ8q(
zDRbqr#kJe7zN$F;G4GAl<LB~X9*QjjTVw)rYeZZ>JKKEY{jqYhXvFrL*3LJbtQk%Q
z^as@6opDwBx^CBtf+r7i`>lfK*jt@u6uH<?{fn`6wo=I9xYoOJJHN(n_)zlx=HJhG
zQ<vEpZ@oRmsfU#-`^|?Nfqm*1xxej+o%k+LudvejU;n-KgU)|8{pkECYwiE$&fd$u
z9+j7#TwInu>u=2e@4xn6`TOFQ`m6Qp_b)m5PwkqTTMw_y^8?NwWxh}Nb1dV2L4B2;
z^}WddZ+zXa$rxTs30l77<H_e6{Q2_~op)5vjhyuF$<JL3AsgZ?h3Dv{tIgNkb#}@t
z%Z-*_FP`}~^RkioBD)4*nT-dem;6oGYnaKe$CwixSnc|+Yi_}#qAPuKC4TEpns{K@
z$KN{+ukQb|Z~Mi#^Y=>68K1j9_wyWotK%usi_1bvW1dU!R)^1B@~U~&+p;(1K0R-H
zcURnB*(??-DwA`ec-o5H(|MBCR@7I{)z9{?PpkU;ayft3;rFsH*nV(Xd*<A|5iJ|s
z&Lh3y)w!=W-LmB?7o7{Y*~U_yxW4#)$PHJS1!7t;mxU&zCpap{tKPirD?9(iNA(G!
zrx+yy_ItbapRWD4;+3Dro@F05PHlfCUm^B1!d2tFOx@>=(md*(45Bg7c0Bbumrhr1
z_}cq#!J>(2$E;4@I$pKobLY8_bB(2JE+u^3I{);er+W%r&x_`7;NO0AyZmC~-EY^u
z-Jd(VHDhDB_4(E2^`GBO-0l88vRBk8F=F-WTQ0HtKkV|m{_ny9aRpnQEjOxmn7sJY
za^~r#ix02!F7DP4->Ld6Vej(S^?~mTyf5m#Y!mzu{dl(2=1H@et>*oHa%VxYVC3F&
z*KKYex%T+TC)@tqnpev?Z_oRaT<~o+tMujSc~Lbs=C5bId0uQk%~kKC&=L2SE><6%
zrBWjOeQkvVjt2kx(9^o<i|8gH$)Akvr*zgEEqBc7T$H?g`TW>}e}kS~Ic~%#<<Pg2
z`_qfC$ei;}!{*hAweuwy6t0-N*>Cy2TluZqm795gT?{k2v2wm>fNfmlhGwh7i3=2h
zkInAUuG-$h_+7lKxczCb-p$Pl{_2J&rp_t%zu`Z1lHja0|JSS!^eBAey`?33wLmv(
zyTd!N6VV?ID?X}uG4q{jliQpXza$!tNNj1S3JgndOWVoRvFNEt*w*ZO?|RFoA73Ls
z*J#z>x9@J>-(MSXGfhP2$Gz(&OXm3b%326&Fa|gpHAHNPy!b?3<Kg*BR`vFt$`=F}
zUHMFwnCqAA-|_T7(~{Jux`NF5%dNM!<QpCf+Ifh3V!HD>{<We}a;jG^XYrS~Zi&0J
zB=@(K`?2HiI;6v=Jo~%m+sio*=c#kAa_Q%hmfdvUPT^SM>tp`i?%V-K1W&tHI#@c0
z>$aa|&*s;@|EnQ);#Sue8uPDS?Od+L9yo!gn`>TC-rC<^&(8gw^Wt~L{3BKK<8N<|
zE4G*1?jh8%Vcx{oGRySk{EdE}URJqvnR&EvneUgZ<j0yDtDQC<IsL8X_8<HE9HlAB
z2NZQ5e%g4x@J#WSV;Uc}`){?I`s;_5-@3BePi0y0OS2!`FlS%(XJ78Su1fcNo9C`8
z&N}hYv_edQk>RAjnC7Aj65DQOUVFXPHnH4^`{e1#-&L-+_VQg@a{Be>%Xj-{r7<4Z
z0UDlbwUA$;)mX~3meFhHVcXo;*QH?=N$XzJ1pXG?p6ckk{OP7yJKkM<xQ)-Yj`7Ew
z@(**_pDjORH@{ZpclC4bhHkfO))Ab~ZpwG|EOyWO<=N=lx7Pgcna$>cM;Jv_Ed(Cj
zdLm&4+WXP1HLK&s!|FefMaPN>{#(xr?bEzJYhBQ<v(Hx<-wn~)w>_b(uI1oW@CuP)
z`C|9#<(;XlUqAEYi|mth-}&f+M7Gc!rag=bTXU~2^SnKaA!LK#LB(llYCL!4FReIn
zqO#jm^YNT?g(Wq=;<Wy3DiBJLo4Ps2t!SId-)l>@*G4`ueyVl-kJZM|)>F0bUse0w
zf3Bo|V;=W<2i2Fwk#W58s?1s*KNhm-?(kObd$DS=YQ+Pwh1w~CQ+9lQIlbVYZz<E3
z10NnMY4{k;NthgS`k&3p{_O`&_wL&ftzhXr;d#>6;HI-)Z*zS!i<CW$*tW*D#+{D3
z*2r$P@n++$s~^R7m|U?rk#(wy)8zcZU+b#gY_bX}Q1zB;u-$(|UbtJhJiaIA(z>Ti
zfn1-idDm^c##g&l!n9cK?8`9A<KI0JS?f0jWgQA$C%ty(*K4o7XX|d;BJ|zlpKVQz
z_u9p}sYjwUdiS#anJzkGVk2YT@<aRXT0Gw|<+z-@UG|R`f}69>t}WemO+WPji*n1h
z_}*z(pKKI)FCfntaIr#5OZ6R>M{s1!-ke5-zf%_%pC~%Fvpe|n>IYUSr9p*-&C1^8
ze`eGwD+Ir5I#jvOjlpl{T^Fx2efHnx{5RCwb!oQgW99p6KD^3xZ?~QCdi5uV&na6P
zezmDo{>pq3spQ3fr<#ZL3+NDp=4LK|Fl7bvm`N{Uf2&p3Mzh}VNqBqEnrSOfsg=p=
zEsLBVOXbHrUVG(Zkkhvbe|!U0`OMsLqMSckF?8+J+QRAk`J#=V|NfnD(1D4;WzwA3
znHS&Zy#1=XxqeG(*zBm+>)52L_7&gmj=d)B@UG```EthX3FnsxT-2EVSZ<PjZ}=9`
zTXNfVk7+D;b+C0)Uh|Vw&$tter7NZw=uJ_0yeNCi>N&4@-iW^4bgS;hxz<@3leJ!`
zwWe*!_Nx<9SW!Fg@20(4Yl`QdD_?UZGw#sokIuqeni{_z{@a=yHos%P^*iz7r}}ov
z>dLNf>{j@8;&RyXvwpQqTz?O=G9Tk#!=^2^gr(x;tw)lN?-sOvy|40S--ml5Qi0VX
zQ%Zxs9JLI;|7W9(sb%Qvm0uU1{G--9Yn}Z5sy{zJzGn`|mEOPh>9*Sp%83UZ&N<}B
zGn^?nk#Zxp)3y8ECgq<kXJozGANXYKSigK{zu24O((b|iA42}E(6nB&Gw*Pe^J<CA
zMGS3o+vl>LyX#@o{Hg8vE3?%Xg|%fmRUbI*%)az5PV?o;$>(z{!cR|N_FkXAXRXW0
zW$8Ymrk~#OukAGD%wxRwAcI|_b^iv7g{-qC3+SFunf52}tL;?o4WBZ#=C18qdq2mx
z?(u_75kES<_tqSqeT*eud?)|ztjn`*EIIJ%ZOr>)g=@E~&0BNDEN{W<wTDC(eph}k
z!!Y$@sN%)~k&Hk)OPz%ck}l^p7EAk-Xir;b^;Pl2C6%u|!hfym9<ABo$~U|AmYKk+
z6Q}x0o<CQe_38K|otrW`ADgXyGz!<9`uGOa(PXG(%G~A<70;XCm3DWLYF72EGbbl(
zUq87$z{BJCr^S;m`?I~hl6<}V9=8eiqJ2*dRQ^6I@vbtCGyU4V<z%MvdV!lkPj~&9
zCEp<w`0((>-;eEee{74KdTqP3s!;00>A$0{+`POw`&(&i^lIKEPdCIDYKw6gMpdkO
z@vdWT>9ZBpjH#k2+t06+G_`rQ@{_c*FUQs`Rl?819_npRJ-H@Y!<~Ce4BLlO*G~Gj
zrz9|RDa5Q5V6)9Tw4?Cxbdxz(3la`btp9&9ynerrM#mymlX?ZOgqMpL*5njR1RXjx
zQB!Zq6~=>$>hElswfybXEZ1$@w@cK&K5|0o#u_30?Osw}L(cl=?=>jcx!9}w<@YN+
z_pWCY?0>yxWoiEIR7S5;?_c~`KT(%4{8Vb&P2ID1Eh{fxZHv0U<J(s2_3o!_Y7PEM
zI-g-yQksx=@WD>d2w7V!hken8I)_wthUO&q?^9o!um4~Amw#i@L_Wq(`ycPC)BROe
zWcx_wZOl)ndsAkvzqg^_-lA&z3}Mr&QgTwQ*Z#Z_5C<&==lyek_n!JQbE5@Me|)oC
z_?!K1`EBAFHKA4NHK9)<o~K*Ad$Hs7i}UW^Wqs%G`n}XX?yJnccOQ7&-!)E_o8{6X
z=jMLD_QqlcKHuW&7Vkdp5Gm*gob&JDoto$J{+9E9{aop|!9`McPon;o^aZtZ0@{T%
z<Qv?+>YM!db8GptJwMj(o4;pI!TCLZ%I_Q9%e&FnA<TEyoa3szg7C+{4d;AkS!5nw
za<1b?*YArDdv<f53n{p^tN8r_=Von%=h9EuOlA8LA5?zYd2+d$yW&))7}efHdqp;p
z*rr2WO)I=xb&GxW2TRDZde?nga`DJ7o`eOmyQZICD6Sxuba<ZM{0U#L#1=d=u~ayi
z#=5}OQsts@z_Lote~Y$mSa<DTnRyZOskm!zj^4~qweR15MR&p{hlj%Z*?Jb9oVzjV
z(Cdi%+a=>C&3I~<tgYvob5Fnjea41#hM@WRv-UpS8-M@EdWI-wm*c?&3uAU?T%9oW
zx|;vGTPr^m?)WNov9R~CT)#J`q`<xlza(BshfP2A?ding%m9ZgdDVF?M%R`w&NG%N
zbFaBEr{lQt`$e1Gq7B|8fI8@Mx&b>CjRfM;9E^JS*n9UF{B1Rze5maIrrPt@-qo%x
z-*0rp!jgA3$CtMb{Tn=vFPZ*dvC!m{qVN_2(|Wg8YqSdFOSg305-8hjsJDhO?e@aX
zm#GuOjk?A9`4-*gWqXjmT<hbOXEvU;Pu+7D1|5$qG_v*fQw_2%3_O>)^Q3gs6PqXg
zGoz+0nfH8UPmlDG?t?6cC2|^s6_VCXxNyj%Z0YCOxAykNv1c^~Hd~!rxJLh`M#-Gl
zCBG)czTTns>Vss4tgMG-bL+G#-~H>VH?gt@Lf5vtn{cV_{I#^#wnD+|-}3T0JGsLA
zZ|dp4SNQKf_1kDy;Tf~<Eq2rL`kntSs&$NfvaBX<UcB+^dy^xl7VF>teKxbM!$4g|
zY0mGko%8eFH`e{_alK@{fNzT8n;`xAeLuGI%U$X`#I)ws??p?WZrGmP<5zS&qHC_}
ztz~P<P3>>&_ur%W@wV3fDRNH~WLqazX0BZkx32z9dEmr_($QP8Z|&Mx`{BxN`#r^P
z?{hBgiqpx8O0m86>B+e**E2V4^6}cCr=9s=N8Ay)fE%1YTpq7n5!)?i-XnD6;go*@
z4;Pttt<#s9KWo;kS--U7`RA;dvP6B+gvGn~-`ttGD=+u)`|P?M#kY>G2+ox*?W*7T
z^^J9P=gOq!ibnfXE51*2R_Fh_!94Z*nR{2)JkreX6<!#qoxQAlhwZPv`se>m<nC(6
zanDVBvu#rHLB__EcbkqSKJ}RQZ-@4osSPh=|6Of5${85ck>F?|lFP`j;L)QWexm8q
z?L~hl{A6`kliT_FL}Tp4-9=@lju&iljq9A6loP+NP2Y95boN4~OIIGc{Z`qndCaML
z|DhKV_q<Q7&8eFxrnJTBNyCc!7rAqkZMK@vt<_^JXnlWjm+kA-|GubfWmnq9b?RyF
z-01XeyLP`?HLGyzD@DOQuRnLcPknm({i+WRF)FgMvg>o)4Ej65J0+zqD;?Y7%cdOz
z*%^0SBG<jRZ|VK)&9(my{*Ha2cxc-#QMs=AyBmT%%Pzkx+;!~s!j13yidHNaVG4{m
zv`{J}`rVUbvg<T#oM*Q`lX+G5Vco>1Zu@>7=zl)x{ND*Tj~d@L*VH*BTfXzs(ZX$O
zik@qqIb=F-mDxtOy1QK3((7*gdRfW9W`AzG4^zpKFB}O~Z}gAlb#kmvFP5#TjXxXP
zW-2GA{e>gp7SlcMGqQi(E*>tc)Q?Gu3ClbgDKA#qnVS?jsrb;11&OO{ZcY5UsH}zg
zdBn^wbM$7;mf1S#W88*B-pIhbmpv;k+pk#vbJx_Z>w_1?$A8;5rLF!~-;3pO0oR{|
zS*+b*${Vp_^5XLz^DqB9t0;VrX@}MSqYo?}&s+JT^Z(;ndta3?oqeyKeRkuyjmHGv
zu6!Q(;_DQ*NlFe}Uqo~!fL04D**%X*d;8dI$-y7hzjXgrTut%h^v|4V)Ft*bykvh!
zfneOB9d4Bsmz%O<j%%<uS=uc=#=i9V#p(Qa<R(t~_2*-l$@B$`E8eXX4gLQ8Oa3dH
z+aIE9s;-;~{K05nw=Sjqt?r-jgI>`&b#JcudTj7BW>nh7<<~I1+`T6Bd!2Xu?MW?c
zUoM|_<a_JQmjA)>p8et#6~`AJKUeu$UaqFD=t=DJb-gxI?<;;fRwV4CcXUtK=VKfG
zmF~|BG%g93t$xzEgFpL7;LT6>u6oVrnfznt$E}a6TkYg_sg^9$U=1v|y0ms`d(L&!
zn9}dci+5jn6S4jJR_8Z+4=>iKU){;|K+DwlYfP55?1tV+n~DNhb}Vwd-SM;WZ%f*`
z;OPpnj*<=9O1x7}I9Sx1`#$J$Q07`UC%xS6^Tkf%mq{UiE<WMrHQ#bx+g|6}=TBEP
zb~Jf0W*lkc4ST<GY4rU2H?%F6r|p;0OtgF&9XC1KLFdxqk_k^b+#X*jV6xe9noBir
zx<OL=g9q*nb6&n<T7CDH?c<1J+LFEe@2&2<-Z202NiEwq5#paYpH1AweJLk@8GBea
zi{RPAM^d|_YNzzgIuvgIF=2`9t$p$eyy`wlhYeHjWTks<c4@pFYq-1WZPh*7gP(hU
zGvyvIJACVM*v4Zsq|C2tI8^c6%ruU<_~b&MZ^_p~suB~{yh!@Dyz+YJiTGXh&sZ8(
zA3ED#XY)AGDsJnq6XD#Yrxs1%+|VAysUcvb69PVb=YnmHR`jYD#k&p5&p&?f>V?1i
zZy7_QJIDBXYb~ywaO0RQx(>3&aqX|Hv+k?Stb=5BepJkQAg*y`ish=Gn&~T7r*1y5
zR5#k&#7!?C&+^5hwHGTNp8qt}=d`86)#=d-gqCGHvn#yZ&uUc^%&hqG_iDyK3(0($
zlLZ-D!WOM5U;5mai!b%{oPwq`8~Sd1|F-{CQ?$aqknj)kn!%F}>ZXgQn=cofF41sj
zi~rVLQh(U1zGrPe|Nhj0g{k%5?|*xr!FA$DXnowH<(rSkCaz;)>0?$oAk?}<++@AN
zo+m4VI%XaFI`4!1o2?h8_px?PbKdBjBEmE`@LjFZ9r3ERXS~jqq0U#Ym-PJ6`{7u_
zH-+I!ySz=3T)<>+A)CuqMmI0jEiAZ@(Y0yQRT+u$gqMx1C+4vAfBV1gwETl3oobo-
zb8k4N2bje-+`Rr}o$cCl-b>qOc?b(#ygDoC?~&ctg3lXETq;<rb3)tfPD0mSR?R%?
zZ{^B<tv;Vr7vG7#Yp$xYHRqwm4@ah?xuIT-R^g{sdUa)VolUg2-}Z5{*rL#_+wM<&
zEmB#0B)a1Cj!owh`A=O`PLfzxsl5N@$M0IbS^sa!h6+!5JojVy$IWk-I;SpZ`lkAr
z{raXe`Nx7;e2QLAklJ%`-=7ncBi8kZSb!s2!vCZ5B~|rpCtrCSKRSPDQQD`6wx6c`
zJg?5$WwUjAkAGzUAI+KB+I)h?z8-yToRjXHUAyg7XT7eT`L$gOVydnkNt}0ldze(T
zuHDKepPAD%J6>7UP4RoQ$7TKBEr%EAY;8GlO>~~`Uz4kIXMHSrae2Gl+?~0{w|(Cp
zt9WgTXWitNzTINAe-mG~iv3zvyv{34ed0;4Wi_WA1lc`|e#_}jzbVh;`)4gbYido=
z0@c}$dXjSlT0@@wny}U6KwEQq>1->9!1!}k$vO{TFojg|o<7gFF{J1E^vVSrCVcNM
zl@{N-;d6I!!IXlE?e4GrWI~N*_v~`7UE!=a?I0)>rN6tn;njl{?VA<r<y3xG-@2}T
z`rVPe+28JaPOW>j#C<&@zm9Xax~SQdmE|ITPBci)(~8TPDOA>Q>$s0(X^P?o6=_i&
zZ`I;;!SBC3p4BWr^OM(czHnQvy&Cx@OzzJ-7QdSwKV5Xkujc|W4{Wa5=2xaGRW|gQ
zD46d!{wIDCTSt@jlna5uKLZZ<UzZ5qxANB6@MW9^1-Hec60|lI_isKi^M&n$y~nFJ
z{&V)+sx9+9v`0g4|MXjZJ4`#BWG|cgt~mVa^pA;+hFrZXw9d>om0IT4)8b>gJZf8X
zS76^0mJK-<9?SUnpNuid{5|FT=AXAe&#61++Se7<oX(}WRH1jy68G~y#wX|B*pc$~
z!o5_57`H#O{SS(6jbV&=x=q?Tq09f7%Ch&jmh77rIWwi;T*d4e%!*zQww%1n<hWzE
zN!7_qOQ#2~QJ*$f>RI(6YpwdFZ(eY#$h?=hxw5Q%?p3|;o3kyZMV~KG5DPzUxnjZE
z!ge>wDUZq|?mON$cAfuWUrN|MheT1|9>>ZB??v94Ke+{3!Sm8*Dq9@es{nUSgEc>L
zzfZS3YE;Lxwc+9{qm`%bxo3Bky0lE)HSK3*`qsJcTG*z(Nr>4h)f;H>cuwRwb|3ZT
zRLe&)?=KeB{=52^X^Te0=Vuc$KMPhqJyW;fVsP2a=!BEUmg@UUE-+GA%(#(BHKNFd
zbDrb9GnYR9_iyl2T`bU``s*+ow@`ocq?_w~&ey%qmI~^-w!D0$)|o`n|Anr;zpifx
zT(c=e`(Mwu6FV23TBP%M&KbM3j<$&QAHgCV3uRvEto2;#IN#~C#==vEZ^PNcB>!a0
zfBA2})0KBFZyOyKX-j4|tQTvj)G~Lz%C-i)c#rq!bQxm?b%(>E69l&YoDsuyGFIb4
zN3)mQgdA_b71nFtJuj91CA8+U_LoIpGtJB<f70Ep);{C$@iPJ6CGG1TuxIj~|J2iZ
z=bZCN5oxt%_6-4R#W<Cuv;@!gb=)nytoUY=hnA62fnIB<PP2CC%B!DKPt}*2)mP8E
zoFUnem0cKAD90Yc_MD~UV*1OinQV+3y$Wxbzxlu8-0y80{4Q?Z)3<LjzXHozf93QH
zHFK{J+3A)UfdP)$mu@<8+6r>}1r*pFiS@|2vF4$2+t>O3PHi#Oe{z0>#p3tc+xNYy
z_<AgS`U97((=WyC{CsuakK500@4NfEhB?BOfAZ3plMWpdUo|XgV)$=VE%GYhg2z(D
z&0&S#&8o!B9ID=lO<kC??&>PtsKv@g`=oP%nN2fIWNV!>`J8876)fl|o?5><BgfvM
za@yQn*Z)ysyRu?U=E|OO_j-L<a>-^lcin37l6z7N{B<`B^WDSTpa0SRYg4f$hIj7|
zjU7CJd>yPa78+aKYO2y{2rb|~qIl@l#gjs2D~exS%<gfzrslB8&#328@XN%jS0BH9
zuM&JlwDiMgKFv7E!mhB;<1s7ezGGOVWVh|hw=ni64Z`9LGTYl`{@QDr(*NFQE62+(
zec4>GE1c)<6ZO7pw3X+Bhwj`m4gM`I?yJ7!y-e;+(AV1Q9Q)E!EPlx}t);6I&;EXY
z>iLfwk3;+JJ9w9*H0`Zt$y(gPq&4B>%-)GF9de%q9AEF-;GWqz$+5NJ?irIcSw@!%
zt3qzix{|4`F;i)-%HyZ!y4rj>Pu6$7UD5Jsp~}XLnYR@8&Q`kLv0LH&^z-`jxxa1`
z?A_*6`%cW`d-GS>-Y;`!rDSPr=UX_(YBQtD%s2c;W=<$RsvG;NXu_G!KKJO8mx`tw
z`?TP_kB9e;r5<K(ky}Df#+<&A*JZUt{;z8G!W1!;sLAeYUQRe?v+AdgHdn%))F~N_
z(d(`}wK~7by>YvK)$2R{>XM!txPIAwe5iffyZDY1gOJ{37SaDQo9wb<B@(~xc_XQ~
zeetUsQQk+L8^kP6&s+X|%S~g?SWdl~1~(hVEmlk|+ZL}^V!zeL7rS@4Mb2bLO<4u?
z;)=!BbZ@BzWhYD*PvCzke5IZHw3w1ohk&ok>C7(^*Z6X!mGqfC`<nSqO>X-#VdeI0
zF0CD_JNf1t{X7w8Q2SHltZvmJjfQZMzKJ`YWjj27yWm}@NJiiyy){<rw#_No$C>K=
z|L!;M%fIa%jlyTM-{RIk&grels+iqn_2W?OU8j02@F{^OZ%u4mz2dUr+Ee#KHZAz$
zaF^+CkKf9)!W#~k=R4f|6x_b@q72{4$5~8g|G4d3nrU-I`O%!$?izuaFBts(u8@lf
zez`JH^sTFW!&IFt4l&)Vg<K{@p7qn#EBs{G{A6nG{<>pzqDEI5zs!|RJgT)fi*LKb
z?VeWY^!?SQN$XNvbi*ZBWq)5SX8QAg=9Q>LPY?9}UbFky>l>{M8xvBsw4Z6@Ot$<H
zc!}L(&oUmSm)a{c9H+%Rd_UuiedqG63##{8b{CgD({XJ|j&X+^h@_MlVE^jdhQ$l@
z#ldIox<O6|u9+{^^Wn@o5lQ8m>baY3c3ul(U2JoaHSP-k{QR@L{Rd2SkMC5iF#GTG
zkuN$VXzq=$pV4zdZQr|>=~{q~B;J?OXZzst-`T$nzgtbQm}TW8`*P`Y{w0mmUP@@r
zfBw1H#Y6a%;#ARfhqHg&60_~wJ%NATqs0wduI^cwwhnYum)xHdPnu_mq<p%T@?o|&
zbLZwt{TY!~%a<yCjrm{P@vQDoe9r&<-ZdpV?ao`=zkN~hTBIWXOAm&xYq`7X&fY)u
z{*&={n_QdxDmJs3pALWQ*9+ffp%=1i-;4H*?(@Dlq)v34;*cCz;hXt}*Skw$v+fCn
zmOfU`<$dQiZxSlC?BD!ogBN>CfriH8v;O;b6tCO&GdVMg<K_)-W`ixSye}oJ_<1oh
zbIzCRo2y(@y)P~Jue0@SLLY<d`gw1)8U9V>)ZG!fYRwzDD@Cl^53J^0mvX>!+B92_
zIj6MzdL~W0uqa1y@+>dr30{|P-9KMHp=k+o-9NuYfzkh2>i%7S`TzWS#`|{~Rk<#1
zD9L7CE_blZYklSHlIm8WZI*k~H``86yS^{~Zn=DK>(j0#?X6KO!$Q~Z3i2zvb~Ufg
zD&}O=g^ddzMm>)-tqJOP@>T=yJD9-G#=Py)_Vw?7gojBom8Ln??_=_7)qmKsJa@r2
z-mSqgCoS}A|Nbv+daCth+RoB=<0m~=tn1UY%4D@9XIv>f|65&{HE`3$3q^XWE4Ow{
zZT~ZO1J{!E9WSo5+|u%n4BXZdmm$>s-<##@i~VPh{+G3L{BN>Vn(O2L`5U;j*$wBU
zUVV1||6k4Mf8NKMHNCYOLp2V*S`@%_Ej-5kJx_7VH}2P2Z}ZD9Ps`QK-JBa+(Z8Yh
z@vW~LvUmRWeq6PvLh;vmfr6!L8P?@(@s)evH~(qnpDPzlmt~i|n+Z9~VBOBOZpTE`
zXKrn`KOMbHQ{&3{B0Ec$h2@!_*h)V!U-ozop1*o~V%^Wp8VBPe-)}mhaZgHK*6WIy
z|I9mH{Hw3LeKGyAkb}+_ZU<ITfu~9iTq0_JKPQT+92Rz(vcdFI?62RQ{Z3|j_8%TZ
zzCL81eJT5L@T(Q--$V}d>{_-)-;X0ov$tkNu9ulu)VZZ9&5QTgxJ(gLbZq_TynFkH
zcvkI|hZZgtSj?B}7<>7J>YeMlUzT@V=W^;=^~vhjpVQxhE&mpaT;3G=GDly0?PtDU
z)<5+neRfIJI8Cz3UY~O0dQZ-~7u6-EKPK*-8lL}VN``(tzsIu2tPBCW<zLs=e-OVf
z|2SWAZ~v>Zim;%+>8HCkq%v?ms9j?8aL$FV{hQe<TST|V9Gn_{thp9+&IQASfK7Aj
z;@poPOZ}qP7-2Gr>#2>rY)+&R$HAWq-(Ik)i%NTHbiw0wfx)rcOIBLHh%N22Zjjf~
z{<UUJeB8OEM~vSTvisQuzxq5kW2e-lm)9p2u8WV44K~YG;9BaUZ5VK7iSC1S0hdIJ
z{%WRbm}p%1mU{E^i`ZZ_Tc$ZXLUz`!H@VaFA-2g`L3iTE^Pm5{yfQ^_4s)h&oRYHe
zii(@Yhh8s?c5U3oaOmPLtJ;;%MXsBOH^~3o(5owXQepnC@~tuzJ53g{@>;lcN`*XF
zF5<<uKl=}ZIp=QuJ?8tp%grw~?TY^;d@j35_<-*B%?o3drY+z*FL~nU@!%gnr}s|Z
zwc~{k|I?Z8-d_s*{6e$z505N&>Hfga3*HG{&*;7Wu-N7JqP+HN6SuB+RxQq49Q5c<
z!F9D=>#l5?dNNp7c+0Z5c8%rl7o1`Bj<3p+JT;~0%0>B=IcHCV?bsE6QzPT_reAx?
zzy5NpjR;AJ7yUJbQE+z4UGaO;*MH<)*!fIm)&0CvoUx^^)*F`Zn!MI<?$pKevj3f3
zxHn+tg|Yy7v(<ORTF)s>ek#9#YnnFKMvWas;6?vCU%i{EEFR)`MwCy`?OW8}%Udj7
z+H@<k6o?7y9yC4WefY5b%rD1}E)5B5nf7p1d(5ox_4<+fPps?Q{QkoFD&7aPu1)vs
zeE!X4%B-A{O>^cvU-x~<jbF#jHtL+TRGjTN_wg0q^OJUGo%{Rc==)DOE0*-V(RJoj
z;M&f;^7H*!=cJD;6#3G?)%#|XRr{XH!R&<{6Yp;Q@Oyfhg5vL=`tRgFm84osIKAo9
ztc<N50v8wUJ|VF4kJ|Zn(w`PAe|vP<zVbD<wcZ%cdwL}5aPai#%3qN(frpdlY3(<>
zuw6Mn@Z9R@R-e){EvpRYYzH4T8JJhey=>zY&4sB^k6vf`t}W1cBJKalRoVTW=&z-p
zou2xndY@W4P5TdnG#6u|&kxO`{OWtn>SDqpSBciMX|RSesD5FeYS$|C$AkGsz|v{j
zDQ!$F_ab~(@X3S~o=^{dd0_dia}KdF8M0c!QTI1=O_&?CLLug&`LY&`M5Ah{yOGPi
zB;L-o>lV`JE15p`-M-7$cCcuy33~o@k)Pug&fZnCt(J#N?rAJeTKdH3x&O!AbL8#z
zONh6J)|ge!HadA(?q$zu6^C%6T6W)La1-I<RJCYbnFk-u0{v8bgm#5#p1-qAxjAdo
z*B--DKLVpCUES;%u9Mq)?fErhW<QU8_9Y^0&;8qeJDp$9QfL0-*YlvKo1Kq)$3F9k
zw_#hc!sJ5V%j^U9#2;5{{Im;5Xj`vQ%l=d6^Up;axZYm)SQPOzJ*~5Nf$etC5&a&&
z4rRw_oRghqF7?Xq-E{HE`XY{{%|*)D@>)`_J|r{r6yGi|Fua-nZS$^GRr9U(v#2lp
z_;_b!|Dr2P^4#}}$4zcxnOt+jGu33Ry<2bk&Zsr-T+i}{%3S1*TIOyYVBnlRE$-Zu
z9zjviY>@8yV^6I&SzR!EG)vAXZ_bH!!B)^7=OqR_GOPGik4s!#VEClzdeKVGf0x9B
z_HDajKU>NAjuRV?labB&m_M`TMP$sXWm{cgEunA!iBaa$_DSa~p0;f3xBBvT(T1*T
zjhA>%9846Q%gv=}sHf$eEHWwdN&3`FlmDJrDDs2tQt`S6Cvw+hN&Z-<(im!F<j7>>
z{ojk@(-cSLy0hLX6MCjKZ-*Sja%ksL(~Cw7{r_%Q7S7Au|3&{vvev^Rjv}oQcU{{~
zm>x7!Dn9-1-Tt@tFHZUYrm(bY+J*VWayxv=ZGZZ$+GSU2${61GO*=R`wS1+P^Y5p+
zCz>|@tf<^?#LW;s>59=4<)<+#Q$4=-zudaU*NV$Z?`z%`pQI~?I42x<KIQ!M9Me_T
zB`TVQ*2syk<&QY${MnK}rYdCa(#4NgJMGQ8YS7c~y=_s&=}qPw3Czjt&#$x{jt*F^
zF=1_`l~P>$so=SP8oY)6TrTxUZ`sdX!kV<?OVqb_A1bF$J+aR|%UfZ-_02h__VK(D
zJ*V0n_<4K7I=#R7EB)TjNm%RT=l4Y4)zZso|GDbr3fCT=n>%Ie-kWb<o7e2&;M5gR
z%VSs=bfnZHcH0BDqnmnX%CLI68Qp5yb^MZgj>hV;FVP>ARXxqNHN={za@tocbJ2Tv
zNk#3V`n=xW??*SEzgSrKuk>7G(CM`~Kc|0n_5Raicwpx~t<OPv*&$)iL^oMXvx?nq
zcd<o`0d_o?-;>FOOYOuWYxAc1SvQ_?ng3@;P<-p;t*2I<Wq5LU)1Fs0-o_?T`&Rw(
zJuyqmqw}?-<SnaBvX@>eP1MtO%!-{=5k9SG`n8EypO+rse78#Y->F*@KOTs7=3HkO
zq{n`1)~xxHPPX)g9}`{MUvy77xA(LE?I*P-m#j4|+j=DB$nVh4YT1l`7qt{03UAGs
zyP|SJuixCdDEDoz>N0(IoqH)2(C@8Vzw6ivp0C#%eC+<jPN?;~@^-~j|M#b^E%fUu
z_@}CQbJetoRkEknmEO7-dDSXE;N@|T-<Ld(pT8f!%PgSi=Z^f$BmPGx+Zs-{zWF=#
zz0drZ`@4;wA79edyJoId{#kvg+n*jDSv~Q=46W#2=?`A4$O!*E!@E|i_gA#hPZlj<
zmi6v3+&iy+pHt9T^U-YY>CfdKKV1^G*F51b)ewBGUHO=G?m6RgOn*z6I-G*NrnD97
z+A-@qFWvFSu*|>oeDuAUOCQ<<aC%JkKhhQ5(yV@E+Q;?hnWt}9Ggrp{*^9@Vtl9T$
zjB@LC?tOZ0+UKYLjnpb{t^IYu&t7?c9n(jKtXEGiTwbv_@BV~&ZoZLo9exI05Bwaw
zNi-=kJooPG*Kw10{nv`Dx~g3Jbkenpfmg0*?eN>L8thgo;&L<M$^}1mhgbc6^;fJu
z=mdN??6}ykbC&$qU@Prl_75kXIQu`o_|X1B;mKDgt}W*aSv<ixzUr0f`PVOUH2&xd
z*|PS{<5O3zx+R^<$k5!(JLN%C?77>8uF-XT*6h<QtBj=QElS_CP-dcc;{NDE-S&%0
zFYxK=OFG&eueZ9YvpH-3VO!~vza4Uyd{@3Wx@ei}wF}RkzqeF%-P72$&tuiJnM}vI
zLC0QB`{jD;$(D&%@0?~kzmH$fykqAolWkEoL9D-~ya`CSC8hT>Z@c_zku0w_{1(4Y
zRDf4ase%urzi4)`J5ab(<KU~%h(9GCO%|*1&q}xPc6YA~X1nhlZ2e)M%5&?LPm`qt
zr%v+uB=zu9N1uUAd&kxZw<VSuDbD%vP#1hM%1X7~GY9V&YA-#LmyqSS*iJ`oAIs}6
zTzi*j-Z~jmuyWQl#UCo3Ixi~Sqs)XiC26*oSU2}yKYzx{uVrzOnVZVS)>mInhl|TE
z{+PE7bZV;tuS3jXL!m;)InkeVPHEcj5nmIOwUev%_sWi&=8OFnJlD)T$UJFY<F+;L
zxMz!d2nKFz=sGM~YOV2c-@Dm@k9sPUQku>$yzjlhX8MK}26Zj-;Qc%*@r*HzOsz_D
z3IZ~(otV=(FEXa~$laQWS4~&fFX@Pu-p*tCSnP<;=00c3DQ5TlPX5reoys#Wu_U!L
zA^T=g$FB*NPYo_Sc1$fzc{{yMM8xdE*TeG{y;cwK^V0u0$#cP%zyE|g4)aG^dTG_<
z%0HCqJ3Q+!Xi8l5fkW1ni8tK>f^9(a@QZ|wZM?WqHn-Z<@)n=%ubJPR_$MtZ)!6>U
zAfsUReartDKeQUeqT1zN_166~ye|=$(RzY0aR29DHcy%T{|jVr_U1h*t=J!O4*8NE
zrtQb&t?WJ=IqtlSH`8#3SLE86f6jj{|2wl>Jnr|lOwc)#hwI{>?F;5_w+-5zbl9@~
z@+aP)2~8!U8c%NSd9lH6-}J>r+~I$=S894|%_)9!sL{PkTE#Bfyk*{r<;jeTmn1(A
ziN17Y+o?-)9;;-#XFb32qMO-C(k0~jl-SF&*z%Y3&7YL>vAJN?&*{B^+%bV#Gntk%
z>n`}QYO3F@zCfc>!m$r@d!trv{_XcrzO`~r^f}I~M^U`osRnMjwig<$tQJghj;Zo>
z_d2dO*=DVCdarYaUZq~HUC_C;J<fXCi~C%Xew}Mss;D)&{!m(DPf5%ki$DRT&9_(F
zW|8Syla=}O+~t?8_gN#fz*hh*Xv^BdGJ$PD)2%fDaV3WTFYkG+QL|{dLE)n^MZOyv
z9*ZaVOq(@l`^q+-O+uy%CQUv3idCSah-)32jpc_E2Y3?=7cx7uO)q;o^WfCY+c!st
z<$mwz-#71X$JM!(KdwxFnE7z;;rW^#S`m>6bAnqYUHZOS?MuY3eWzB<U47O$@~UR^
zJi+qv%5~{ap1R!9-`g#8<)G=Q%vm{pCZdgR^lnZ0xZwV*a}wT%b6@`CpCTVwD>>il
z&+F@)CF`!fbx>RS@WY(%R@ra2g;tmPN!@5^(oPhg_((BpdwBD@O9#{=f2&q^6(7C5
zu+V)|;ENtt-uVyY8}D<NPFZwb>aW(p*|#1o=kNU6u=>*O?;j-C-dvL|SN+g=TmH(0
zr|(OgCmodIlPH+Z#C&c!UyqLVq=QbTC4rwcna#4YRkpW1oOSIvL+9x*_q;5NkZqrq
zH}v+0eA6rV`TBGGtf~Lc7v2lred%OYk%G!*ro!!3e5*B5J}qncCt~NGc74&GfbTXp
zUp(06e!In#r=a!onYYOzE+=!sel760#<zBHTjCBjj-G9|?r)Q_x6!bSdiddcd(^b4
zPgiB+T@-d*Hs8Eq_2PB4<?sJ#<>j;nPyH15?ycl32_wr#vlo@zPl>TU7P;8gZJp%}
zp+ec6cANYh`J<d<bxvo69-b;9AY#jODfwIAx%p0K-$=dd&zYMkdu!I3^*1JOtG@I4
z=jN<?UdC<()f=a0FRYyyRkf**)r;YCh5t*_=?!aICw=w1{rvIN!lnAowI@Hg=>Ph9
z&_2wrmU|uBf2+rnu6FJUxnZ<CGVbSw`$=W1e=p59RP8zRYv+xZHMfeVXFRN6RX!BF
zWm%82)nkpodgl~nCu>pNi}EgKyZ7yW^ulYh^7>B8h<~B4{M5X&7R`Uta&3B}YRv27
zS7I4IWH1}dayV+TPVZdaxtLc=P6@C3u~Eu%!NiN3+Rl2(H~d_=e#^y^>vUJ|i~e{!
zJacLIwvrq>x6V!LFGX{n3afo$W-iNjwM_T#j_H^Bqj<Src3k0WH0qwiV|UT&iqYcF
zB5!45UfwSIa#!EmOuTWO|1O~~$N&A8etAIaN+Xl(iaQ!>E^VA0xZT03`O2x5Sq+j5
zu2YqgTk`d%7-Tg&`DXD>KB)E1>crz^JMMC-Xg@D6)d{yc*Er?;rc&-DX4+or+voUL
zZdAx~yR*2^xM#`9VBOogNB1myFX{dLgYmxRed%3xj$O|`SZ>>SQQPSa_~Z)CWj{Cn
z5mBx>G%0y$vhN+k2HPs%%b}Oko!9Zsl(Ulfd+Brc2c5p%q2E3HeIChN`c^o7na*-$
z({tHEPJbl-J6lda+4$*Xwf=S6Pu#alwlH&E7UPmD>_7YV@Nw6t4(^VxFGo6C=FU|6
zH?3E&W#5mEA3RdJ!kMR^&7HsS$oc(;IwrXGo|-xT+EG=1|7(vl|Gn&3@h4NN@YMeE
zsg<TRBJ%}jeVDsGMOfW!@qfFQ-RBnP%)M-B>X{EZgJ8l%wa!47NhLSmd??K*I1%*l
zvZ?cR6|W1`5BEGuRn$)q`4hWf&gE-Y{(RW#e&00aGK;~UWoMrD+zWM9t6h*fWrK*q
zFEOpHv*%RB7)4K$uk(8HFT6_p;G~0ER>vCdGI_f#?YGd<zP`rW|LMF>VcP$*O1^en
zw)$E*r7y}d?D*9`e*+DzY^U+`z3O*WpFQ);yk*ZK*#r&a8<*|dAvH;V@7z~e7p}_f
zzLmH>WnG&Q(~`zohhv`ar0>1f;a())|6^~3N^6#lU6-A9ciW<M4HeUtO<}mmT%gGi
z9KAZgz9hZl{3lJNSr+zpG6U?zypJ;->%6w`V%QGu6_s^~Zu>V+&nv(G@88?QUiQmn
zViYgfD4SlM__NaKZOd}uZ&Py@J+JqZ+pqBVeb9o{D^ga!+rKsJ>*L~oKku;3if~HT
zS@z9S&UIe!#gg!8KLa=fJE9X7^wnGCRqTnmtGwOo<gcl{A-d^n4W}K_-SB0a_N~QV
z1Ricu(&)>INL^8-V%9nBaa4R~`1OSwy1t#LHm{I)+Nt+Js;>h)F8hkhWB#L4CeOQe
zx&O0WqHNy7Q6u&H%jeIxpYsKt4mK0h^NjR;_r~3F;o3q!p6TA37Oi+<n03oeb(fmA
ze&Wn2FJ=WEmXLiBU1hiP%c*U?UPckD0!o_$3OkYtx=lnUP1>^frnS=g=NC9sAE>PK
z%US(Lx8S9te!=Yp>)IOhzUbd#WH6o-cV_#uHg|=l?mmv|8@vJ+G|kG2XwAOD!oys`
zm|?hR&$fzV73LQ%uGICG(FxQLg<LL_BBu3g`gzu;z2~-m_Iy4gN7LrAdWzte%lT7J
zm<969nWfqsYT`Y0<(xljPMw(YR8wisytArIDs#R&d4IWm@SpkC`3ycu^P|_+U0mMt
zani3%=|6*2mFJdyd3qpqg&gbp#qpCr<!_3~+~VNwvYxqBz-sruS019WT`#hHH`Xa-
zPmX`>;V{2`-M-7;1NVld?paxRt;=H44LzT?b2iz@7|!c=y|(+`9)&{teED?^$Lz}l
zHRTrly>##EL-zIO9hY6?W)wXZucXbKqv6O}{%!JCZi!=-(;e+(BTg(7)SrH?TGEo`
z($7h|Smk~jRmYq;Dr~)3%wh>=Q1`U%Rr8p4toxjFCwJX4U$M{%gZzCujE3rYxf3GW
zzdTMk*JJzp;>_D~7M}Ay$8x^<-NSivtxm_fef+Mvre}RrY2MW8Rdwo*BiRp}&@m3b
zopms{p+HVUW|Ae>t2ulv&32cw=iHyR^-8tn|6g;mPu~>{z5UJQXSudq=YP9r%GLa<
z`OmL<TW9jFujuq$5zBU~Yk^;*nwo{zW$%s6&h*tySe(qgORBgQa+jgx`82(A9L}DQ
zb<xlZS=LlBu6!H#%m3mGB?h5+bNNjx_XKDDSon3K0rNJUtjI-QuNuahtZEcKc4#yA
zs}#N|cU?JqtC(24?oYgDy!ZH0L-AK>_qwdsNpn}VFS%LI7yr`RsWaX8W4oa8n!7sR
ztHdv5-#IEBmF&|yJ6j^YdaG-<o9TilPj;)<+eqH~yY*e)J7K5eZTsY;4%8WJ50-rA
z{G=gaN{a7^D&uUY-q+bzR$E>=(r+|(o#xAehJ_D}Zdqr&pJZF7RHMbeZqJfU<{y_G
z$zT1)>2xY*jpOpub8{!ziL+ZJmlQM{S$JlTsn`B|i;B|=*H6BCvQH}dp^BRHX-V)I
zGkQE{?-V>VOxFDg+MlLgA5!c8wCU6GBzc|v4eo6}X4YP^t6Fci+biPGk00TWS09;w
z%`@W6)A|oz56|W2Z+fg(pRJX4+&=s%i`(+UQ`gM-eopoM^3;=YJN$yu=JF&jOP{Ll
zzA{{2pMA}zbziT1=9%W{dHR>i!&NPMSw^>=8fJbe@i@-w6BV3PJ4tORLqt($&Btw3
zX&0IJmL9m?ziher(kzX|!e`bc6bDD;uUO;tSZ#U!d#(dPBHDR^e8TNg$qP$Z%MXH@
zqO)EZ2c+LgsAOH5*Z6v#Z|}N!A7g(Dzx339t@EgA>uLVtwChoCB4*n%FVNny-Xb~o
z#CqZ2B%^cpZ(a$vyFFdCS;i*J-Cq6Px5DV!+tN$cpSv+PP7kuVrA<L@gYKqn6AtRu
zP2YAQEo}D$v-XNbOp)8n?erqg2;FMwniFQItCG83!!-0&WNN{urmXp@HNpF~>b&xF
z-}dTGj|kg;yLTU6Det!i?M_=(Iw5!7)_Jwda_8KADfNEd(;|!f(y|gURek?^lVi*?
z)7D;L)AbVDeB|^+>rI+Lzi04#Srz-0VKv`(f63YmIj;l7hi_iDyH${Q@|n}z<$=c~
zgUqZWjBl6a+~57S<#F|KH}{M)$%4u*SAJfz6r2}Z8Wmeze|h~zHn07QzHlCRd}Y%a
zujhQ9zw&IT{rP!M^1JI_Z~N?;@WeZdX~&7;*B5<dBDuKc72Pd7R#pEm>YdMtM&91*
zi^Pm)En`wODp(<(c=V?`r%t%w*Dfm&8zy#FuGQb(T3I}Mv0aq!xU2O)o5uEoTKBzM
z<Hh_IJ)PqBzvkKZ9`)I8@{@8ewL0lsxZ1t*SH+{_*7laK`Mz%{UHiQJqv^X(dEf8F
zwyl@mT=jI9tH%54Men=z-B)<lc<s9Vyh~p~8eTE2(LNsMceC)pM9AGiidxF*y4iPr
zUVLt6Y5pRH<&<T6R?}ZDfu?5d4+Rfh!u-xvTygvx620nkiuD2M$o>X_<!pZ@POS4j
zTwFT)>I=`s?w5ovz4KiiS6EoHyleC7UBW`^GuJ%potD41_WX{+w$d$X@?G{KFDfiO
zwD0fz<NkC~^0ebUYj=nUIyPLcxWE2&$y<Kjs+XVDl<OLdo{4-6T&6tfrewgei^rBb
zsr{U^u<PWx&Y=G3)1_aB6<x2Mn_t`!EdOOuOOrMqYpKkf5{D+gqO7kU_eJR?3WZz!
zjVa68?l4(NPjazT%B){X4&8?gX39=t()xO4+A-GvHoq^)lf9m)E#K-i!)HsC=HeOJ
zk~5~9iWic0J2>Nt?u4kN40G~s#~2&Fjr=A4bj9SNH<#G^A{sVsT|fWF+IL<zzwTUK
z+_la!#p1caa?MvMs@8i#pLMu~7Bnn+_VaA=toN-&*S0ur{ynR%N9Yf0${|ycj4ONc
z!v4N1UEiFfJDLB-M9s4gSiJ1LDr8?-ahQq}_)TfLaOh|Lxw=VKlZu?y+G-d+n)ocb
z_^`6Ze6{H|kE+~Ly=Cw9$#}h;FUA-Vy59Y7>DO7?>p%YTd|TfXQh4`E_p;@)j-}Mi
zHB|V#d&|e;lleV!J8RNXE!i{!wUw513B6mOQGC_-z}r__Zm#sd5+9Zy*L?8@_q@XJ
zXBMl<<}KNLX8xr@sV6LT-^^~kzPX$4wM@~zbxEhQ-tO;yKlMZApWNSacfKCE_Hjkz
zEYYR@R@1WhR&Q8&F#FW0g{LmhiS;N<^q9J1HRCh|CGMxk%%87#URTbxMo(j5{YRY#
zEKv>0GVdNq)fb9SIP6jDxFGywf@tZn#ia*~r|SM{J!-Vg`^uk=thtBt?aG|L@Y=`k
zU;VXF47_md;rp+5qwKH$@r(2OA6B#N)8|mRW!v<<WWr9)Vf)IN)uO-sgM0bMT>tX^
z-MOC@urY7vmuuQT7rZk^uAVDibk&jBb*tjP^?az(ShGkk+UU8w*PX4+amC-Y3YeDK
zKn}{kr<N0Q<s;Aa27e7t$W=*2zn-1hn^2Ui`zEmFR+0HS<N2p-W=Fm~(zW<6Up~9Y
z&&L^2_d1h}PV34(Uh4Q<Jw<Ol<f5&__(QQD__J>>y*mBf&N*H`9rYg+-w*#?Z&#uJ
zRFdWP`ka_8rg46%Ju6PEShwwA;9rKxTDu?L_nE;od+Ng~uM`f4(u{Kp-?6`E+H=Ho
zZS$nf-&5b%h`axWtb}V6J8*la@ymUeqgT%P_-10BJ6C_nQJzI=2TrQ`m#y9-Z>SW-
zzRmGw%!}LWU%WaRn^*qyT|rDJ>l&v{p0Jbmn7Gt^wlhko*h=LFRXf?9cUpfOymavD
z5&J^p#b*!CbAB!Hwjf*X<<WkX<fFQW9=h8JJvS82nGnHeST>V;(KKfD2C-9bm4a+q
zmhZ4OQPF1>-6Y4n^}uTfL0Rv`E9ab9!&mcY6BmPq*4m(7kAvpd9KF6--csP_0>2sO
zSPsaUR+K%ybH!)Z_3AaQWnX8^+*)QOmJ<Ik=j&tVxv|OW3=IoeIJNh(w+OWDO7Plj
zcS@OS&&1D<uN~9X7MS<Kes;L}a<d#Am0j(t7M`m;XLyr4yR+W9D)6YR_4>@!m(!N3
znb_W|ddd56-uF9)ZJ$>9dSyksd|7cK@?zboUJ*W@rOO&7Y+qT{-=ei`<8QZ(9~X$(
zGx!}6j!lqi-RH}>iS^HcdF8FbqVHCIP<(E%Gq88xNqY{(Y2DG^zTCZBa(-gV^v82K
zzZYFMExpvee-49d<9GiEX-V0)M>YQK+HHA6V*1B5AM!1iu&8dm^x5$5nzPw+e|+aU
zSz%WkSGD&G>+{<wx@!$qIc?{vyS6hbDPr<N#d(pe{2!N>%!%c7cYoNaXArr1(scFr
zp8I;Eum01j4DPwHEqV8ivL(5z&3td&t2UKk)eO8UU#oh}<B!HhrrvLkv)HR%O_`x9
zCt7X!VZ%Q4q&fYU&QxECJTWDzcFOa*^0tZB9$lWZKkJ#!IYG<KtpZ9*JUckV>!YR}
zco=_eVfDYB*-fGvZ&PlyZP|N`J!krwy|(XnnhR)`=NI-`Xqs_6{>$`dGkC}Pb)l_g
z|8}M&q;^S4eO@OMR(ShLe0lnDo-JFqaL#_Pc45mhzqVZU9KR!vCze-U-(6P8f57(r
znMq&N&tHF9ru9m|BbG5x_*~1Q^o`#0x@6y8c)j0YC8rpRQ^3cK<=m==1Fg%F-f%m0
z++8%ae6RNL-LV{sM;!mHxh1Lg==+3kcJ0gO2^>0H|L4Wlb8!_1+2ZxR_cyuUF)jKn
zvgXYE`PNUs=Y{`cIjLRr>#ybhzSpwNkK8^P3RReO-Yae2dB1?abpI>4hkk2*c^t}0
zd{V$2ow_pY&&<{PKsz6{v}~!q%*ViC`O5l~<-Yw_Y=6)77Jm55-DuCuJ52virCsyS
zVgK{_%4eM~v*z%~ewBULm-bw3o^(lGS=`zEi{|B>h@A9$_iDeb#i4S(FC^6;%{uzk
zCwk5{<J^}s&q*5AziIA$y`Gu5>%UC$trc5TGri{;&*}d+^=&R_g{H*kkQg5y>4#!Y
zCeN}W-)!H;=E}NgyG6>j?NeUM{cQC;a^%>OBRVI9B6T;lI9s^f@nG;?bGYRer@wm5
zd!AA;i_4e!mhgJ5&a+K_oBdV}+$q{MZ|k}jcl4Y%0xMM(gg;C^+B&WJkW^Gx&Hdl2
zqQ!pP6MXI3C&A^HpuL%M_J+>vS1PvQpILG;URGbyeR)<W-%BL)$WpiI@Ahm^eExI}
zr=@q|(j%KQoiDrGN)D9V#=qE)@yI99qf`ANmfC!^(agL2D%5(@sVKMXL)tAHR?pp9
zaz{$MUgIVMLj!}Si(|;jV42d2$y&cY$^QLjw_LvBu*{Pg)-h_^IVS|B6j{!w@DOPJ
zlh&gAHrOt+-))LtXUtujuQuHO56=vEnc{KUS)Tu{kd^WO&wpaqG58(4WPc~j=z3M;
z*BPH@b6NyiOFU*Yi_7i`{n5L2l1ul=M|Z9mCR=H0r<DKvC9?66nEf`*<)INr<Fnfe
zvp2<Dw5_oC^ib-;i^<2oxvC#O94k=YZ#Vzz+*<yvg)23`ygLzTX<K-<e21@I7LVVG
z=w~l^9RfaWn9jeOMQIbCP1%RD%*r3cRd;mH`nXkVO21KJN$DnDuYg~bn^<P{HnG3{
ze4*wZ-*etmlK0<F|9JR$?5s=9MnMch7K~B37lR+&3gFvmySzyG_?{b|3KAYyKiPCI
zvx|G)zNTv|4Ns<-m+FfsO`9K-n!HJHbNl|2H&2>QUVLfV+zrdO1V%Te1O+jx&k}3h
z?r{82!d1~ZqJ`q78+U#vkJ`wS=b!g_@7r0u^Yia)_^7aMV(_~Ut&`8KF)wp{ck1R<
zmU>3c4<fJDi%J)NyYu_@_1o|7>|%Usvs~WxEmx*)g6-khm|zb!35RG0*%cW^uMHP)
z*KSri<(k)N$&t|fp3Tki{0SzFMJtUgjs)G_#3Oz8>$P%at3U%c{tv9-3x7LHiOzM{
z#C|#@+ixeY(Rcd;S`8XkmXu%QTX*%Pp+@5DGaqL?zI&t8V!l`XT!uF@D=KVCyjNtN
zmw3&%t?)$T+(V9Rb9(mY^iRIU{$g`L5N~KGha$_B00y?2Xocb>ZSB!_A*;nJXR*Xw
zej#qPJTZ3d-?w>s=@IKDY?IcObl_MyM^LqO76Zc{_Ib<OpJ@sDuHgMtx>4uD-r&@V
zrM`Xf264<XTAP!aHcVm+WO&O{eiOWbPIvX4!VKXJ_K=Mfi~+rs?w`J-_E{CCE2i&Y
zYSMPsKVIoBb%&|;;l8|u_761+x_`|GzH!0e8(Y6gMD1F}szlBsrnk;SzmA*oU3A~W
z>DQkx-!1=CyX!m?YhcaMOG12IDOr2#cqF#TRqU?|x%X<`_C3bTTc;ItB;83c=#*o9
z{an}Y1J9?lf2@vwSKZnYze?<L3R}<ur}tX1hqk=YI+fFSh3VMq_?q8ImyT|HwJ&Aw
zYm2)tBMzi|c+B<MIQoymoK0U9tivMg&CZ;^k?Q_;)B8Pso$^Wr6Z6l{%!xS@l*JU@
z*mf(JVVSe*{HOihKiy*6-3%ADScje5#C?jROKr}zZKik2-mdqzT0Gb9dEuYC#n0!x
zsxB%JH{I!^)4lCcZQya^DFt>bFZqZXX_Q5S4?Oh?^>>+gQRwE~EBjXZ2KTS{`S6Ib
zOwc4p5sm2`pC4);FWDr<IIZL8qj!m$?yZboyvD9HZ_3)ln-ZEXF1IF~sZafK&Zc?4
z2BX3fPvt8?pC;c9V4D9T>v_?!2%~qF(K8GitQeD~zFJd%<;XO~hRBl&VywDzGNcOq
z7}WQ~Jo~}!R3i4i^?2|?mUo^vHZe|FlzC0F-DFZ%{zr#HkUiN~Kd$SWYZZ6<OMBYe
zSuqYLAsfS_`^pNm-qkEBu1SCV_o(%Z?eFf`rq4H<!}(gYRHMS>c#&)6;tQus_dZv@
zKHX*^>r=^mB^{fSGeduc?Qwke@A|UWy#3|Bxb&J<c;0Rl?VI*u$N!6mr!f1bGHzgh
zz&MpDYue|CZ(H8n4rDhfPqBE@SFs~Ec)P>oMJGSSNPe2QV&{s?qUlPy3uL%7O7&T`
zR~%2=%V@u2-3HZzu}e<`JPNnEc0|_L;bMK%I-mC<LRvp6%fmoxQQETCq&P73w7T&g
zEIKMG5~Xggb-VUux|Z;jB~J?qzZeG#xqB73`nxgd9Mo!EtQ@A_(B(68@%{jjCMFTd
z^nIn<j=O9zTzg~Mwwv4ZUI^Cnp1$XGeAbTS7lI8RoYt-pzFB%ObmzH=o!iVU`E9bi
z&9Ztp;uF4bw;#K;SGp(bhmnJ9e&e!V6B=3Sre9wurn_~LZ}>i*Nf)R2KKvo%cjeUH
zYl>^nJY!gGT7LYH>AP1wxA-|8`o&+VIyOo8g~s2gz20j=gJuh@biN-s_vh5q)tA|m
z)iRcx5!1*He#W(etzu5}&8#W6PW^Zlq^22Y&A2N0jE_Rszf%Wh9XAu)614HK?uOH+
zPwreU^yB`!_DR7;`Er{KEx%mItK0s5->u!}bzHcuY70|u^%v=H3|@KF_X)p`#mo=6
zPfvCPX2!04xl8Tp0;|XCW|_}AbUUEKb5X&(X|mU27CvWS@M0FRW>ES2L;KCkOJAfj
zdW#P&P1z)nrNV9Kdc-y%Iwg0a@$KCKPhNiDbn4=M^Y^o5=<zEbf9+uV|H|b1CJSv3
zX)&#RbpcJgSFB|A`|&4fE5pCS1xugJIb{FJXx6U(Pd;6VPMIJ*&HM_>kxk&^UV@!n
zBv&f0thmy?y!W|pKO<WL|HA;UxlzXo0v~_&)7e|HufR-X-NJL1cIXs%++}*XjP1Sb
zb(s^6d~Lp&-CLXg@~wz0iL2XLxxDzi_J*SePq8$pY)(4)KxtjK*^g<hHXYf@6L($x
zx9#c=#*PzQwI?ri9t!`mJNS*o-nV9d?rwT^eR^4KqwZ|k+1uaVtYTlM?_;-D^=d$<
z^P<Cf+ad)RT`dD|uW^`Ad@^X`;`{I8BIka+`{b(R>itol&xl7)`@7-EnP(g3%isPX
z$xs%O^HWxoX9n+UrL7z@L-aQ7U-4A({GIR@=a&SwTV*_CF9i+e3*0L#Xe!~@@yviB
zn_p9P)4kJtCv%xzO#NAU&ziMuvW-r7jNOKqK!eUQ21|j@&Fh!G)CgU+CSkAQO}3=2
zfV?AxwIx;V$Cx7S@yzSAZQs`U^}zd}-I6<wYWePc9&)+(VT1e%&1SKR>4~2Ud&|zA
zJl|xKIQ5XUuWo6{?nXHcpSR%|_GS5Zx<%_7)^hswJP6ok?WddX(aY;6(8zXuS<zgj
z{JskA{hArPtC{>IFD1OPJ|A^5k^i}S7@v2a6vK+&PZhf2lA|4$vBV#|<&YB`e&SNu
zsb2+0Dkf_y1edL}o#U)*|B`oylmE|!dpB4ui%d3*T_$Ft(3rSTN#HftG@p4+&P*yP
zx+hMh)|{I2nx`|6bHkp>tP@9cEjNYgMsGKo`FX0s<W4K4?)Cn2?@X$D;=IH~Hu&_L
zIjf7KUCKhr<K$ihF3dU_mBz?$@`2btF@4^Z>3>g@gfpq9DD+0z|5)i|wA^$a>%`Ce
z-;b_al<aevpK(IQpU~N>-cF01?-^Y9K)>wT#^v1X25**Kej&B>tb3aNqvC_^T~qF#
zV5#`^u4mr#y3q4F37zSk)mzk6Kc+0*R#+D!e{tWLUrXlI+KEkWoe*LDQM^lPfvHZ2
zQMc;W_^CRJm1V_SD|ak&*lV&ps_=x@%1i4F<FY?b+Unz(yPPR<d$jyky`_>C%(fy6
zKM1~;{y9JRx9F4*t}Y(WOF4C)*Y12MXrkAxm3T!-rFb9f{D6sq$6Jdcr%ZYH@Ay86
zFAo?G82r+C-oEU@0#%czR`v=vGUi(_2Bq6eC9<v-IeU8g@4Sq(Efc-$*3Z<wHJfY2
z;nYRj;-|hb$av#c?*HQ6`r9QcTVB{~EdK6i9{Yd&>_>T-SN454c2oBKF;kg`6O1RQ
zZF#Wpn@!9;kvsLE@gU(>i~Xz^|1IubP%|YVSK)lzt>-Hj_wDb=J|lZ}XLz0GQpu0C
z3-8RgC`>wIc=)4`U-^!<)-ETV1IAf9opShZS(ZI`^3>$;+FgH2pU%1+Cy)rb2P$<H
z&!nE_?U(!K>Yka=$-=Y4O{?sKPxJ&wyHldKo|GAX$ak4`ZsXbBIK|+P7yFXunBLm_
z_|wxl?RsmiEcMTp6+O5)Rr2pSDFLa@XAgNgg;aJmsB&F85aDBU+jU8WMqbsM?u*Vm
zzY~MqS18Y2HM>M-)3n~4u7!o$ZiG)1-LOdgUw?LwN1%sLrf$#1H6?30t<#niB&b^#
zR!H8{eDeCy`hsw_#*}yUS>96&#l61USA~l`4K?yUA$p0^;{nT>-M>6Jmeg+4SmU4@
z`|G*&$(T18cckWu*H)fN>s=R7dPkF8rB281%G3T^3;r3uJE176T)yOqUkRfX^XnVo
zz2CNO{(thoq%(UbY(EpuUD~$9KXYl4d!9qYBCoPA{`|GUK~L4#=kJhepPBSr?s|#4
zWcAMirj|uLMF*Ib-D4hSi(fHLVtA0(o;|N??xZ(5nlY=-mn@oe<fe7|>o@DRAN>)S
zd2{ty@wZkh{Xb3Yo&H(PyEAas=jj65-7ZI|)P$(&JPf{Nw?O>U^@DXzb62K(x?*{|
z>-zPFGnZL5T5CKI_I;`!ZLhhSuiW!ys>%lr16P)(lh1Sezw4iPNH;(%cj@HC{KYJ%
zqi=`h&f2o}<nt9@Tr3y5a|d*|=C1PC#US8%`lHAi!!LSM%~(TTolrGBzr`-8njzrf
z5BFgA3uZ5aJ2oB+WU17z;OhEx>zL`ws|$3~BHPWQ`s50mrs>RNS$g;4?$_^LGG?%(
z+iPr9;cRi-0;>C6n|n`7hkuFr$Y$ZdX7&3)O2X4dZ+4T*g?vZizJ~war}giZj!s%b
z**bl$wqI?Rjdw6?xuz3e{CifIhND(l-KtCTUuyrme3kwBpXc-0@2~b>Jjv4fqRYKy
zzfwiwMKY|jma?dJ=sUTH-I-8gRQ|G<)tY}B8^ha-Pi$wV=<K!d&s!H}m3Z8lvvj4F
z<nu3j{<rj^GhdY^S<K=S)zvY6_(j9I(Mz{p&-u#3ps%0xCZwkd_}$&DGt>I%L#}hx
zI+{%LmY&Sv*|Bu)v;`^YCMQ!)D@r<SnqT;F3%gI1)$0lC<-gt*f3YT3k&8P_JZq|w
z>Mu^?7cw*14&M?~U8KF_wZ_p~245FFH1W93w*CD@t2#cxwn<y|eB788>RwQwD!nJ_
z;L-=HUJIK>%e~}HcZ$q@rCeU9d-}@psmWXn3w(mr9o#1_-lNRyWxObO!Jbu)-0J6M
zpObX|Ge`TCfXCfG3u2tkLGB1Bd3D-v>!i6CmQLHBb8q4V*;m%A>l6-zPg!0+`B&zv
zaMS*4_wOy(?6Yc9iQ1;G2OdtFWw5y*pru)3FSo!`Yw=BI>}Jjv-um;WZm?Oo!xM?;
z@w~x5cYNVr$Y-*UHKYE|L(}7j&b!XdEnDSuk|FV=M_FF<n^#@3tKtg42V5BMJ$+EL
zmaBaA_kR2MopxSKTc^EnvAuYNa|7FBy;2btmn8)guN@cU>)7R3drNr^v*B-%fP>#!
zC3LHFH-`l-U{!a>5pH1rz_!}Rh>_{v+*{M*{>GSR?QGKJKJX^#@0ytBXVl`&XJ54G
zI-azWGx-Rc*Z=b0WiKYK{w#9fs=9tn*?aNW>Njtz-aUJKWheK!gc%dJnciBL^P630
z(SzP>qx~lrEIX$0M<dav^NFc%n8?j{YCj*EXht-usa<Puvr@df=|g#`^|2i{f`crg
zlv4YIe0TD-J8(5$oitH%YxoNP6aH3=vXxm&7$-3Cq}B+^v$ltBy1e4bYNz@btLrsA
zCAlk}E@ovtc{1V48Urqii%j-T;Z<Ujm;7}&pAppG&Fm2z|0_F)b7@Ho<5g|r3maeT
z`C7N>zh^_s!LCmx#y2ZcOH)3XY+dZP@Z5YZk&KOjpLfelc)VS?W%W6uqpiF51#Fkx
z+AVulW5r}P=VjMc-#xaXidVJyS(Epxd0!$vz1|S$V0bis$^$*y>mITN=Q}^Ec^`8)
z-jU5Uc}28>*NR=7Ge7(a*pXSg>eOcmGt+#lzzDY`H^CQ3I8WNpWl<!{=dOEzKW6cs
zf1AI`t)0&j@%8sV<;}t6K|i!6^Vp_#AJYk(dHR#El&(O#!x<fsB$uF%zZyanr`G<B
zo%}PlhpFh<$M7#HW_~8V-BJJMFmx$+Hbmq|o_O{6-F^Pm8rut^m+GzAY87%{wtiXN
zQK`Ni#}lWvYUeB!<cts1%<tLgc6(~>e}~#Xu9o|}tu}O}KUkt-y+~Yu@3!`a*$?vr
zmGA$%Fzd(Z=)>jA$t-J}vUhN&t2iiIuqVCly0);duOW1Y-nS?l#fJ~X8o25;|2O#0
zxPRl}#>F!JQp-<5&KE8C^tmTD{_ij6Z;RgiJa=8=IYU(7?zcMyPnyeQ6t23M+jvMz
zDpF*p#-GKHwll@5f3rR;-mcRm9opKiwIp-~w>IOOgbIJg)mpU^e*S)F|KrT3&&52?
z7X>R$t~se;D6{dzdBt|mKknC0_$@0be`x>b&f5u>M1>>-+GS=r9WI!%=$$Q#*@am#
zD@{xN{WK;;K2N!5@nv0(S)hGl!l8RDi5tQ`zEF$IVqDyL^ii9=sQ8%$KDv(=o1L9l
z_NJ?Ewdmeu-;{RA`KYg&Xi)wx!~OicN2;%qn=H@25~z=?y?MiuKg&Ybx7XoBM(+(b
zo*Ql3bJi}Omu|oyy24<2#>(>^znVne+Vso&>L%D6Q)+rqHQB26<g>Z+vsIWyLG$oN
zUtM4Q_MGQ<>SwS~xbxg2q9r05{+@bOu-)NhkaA&$TDRD2Ry(E1$!6WYpEg&`ysfA|
zz0aeJ#h`nkX>a$Q<$v`3zaD)5tFr%u)s-UdBZ&u1G+7T!l2W^9^<|E>taU^thnDr-
zgRYh_3k!5N7dgbb-ErFW{mQ4s+mq^}-imp2-qO4B%{(iAp#d|O>biw;52d(dwx3f}
z;=X?Se9qhbs`IYt_o{e5*3<BP<9Doj(Yc3a?bpm-Tq!BJtUfvInpf@xr?PXR<vXV^
z+UGegz0`ImVe_kqwU-~uZZG;RA>?`^Eu#9C)~uL|91*R5xh@E{2TxzyJ$KfM=;Wt+
zw05tKUt_-Y*Or#uhFR}eev}{5I<?Tsj+d|E-szv$pT~Fy8yyf8xc)_V+2RZ3+nIY7
zGO#@@IIuY2tccQo&>AhN<v9npcJAf~dUS31BC&lF9x?9d7R#7sKWj?U?aLc7Ta){G
zzpelN=gF^0$MPDTqu5y@igwhWdg1)}!K;|pZ=<}k`4%poypwNQTY~fMyS2Z6Z|A?f
z(*G{gZ_{__FT-0`IeI8=*z@d@-RbH_)jNKi`x^YsPxhDVb5Gy61$k-VHzrQq$trYP
zq<nGE?PVFAXJ##2cm8|T%YU|8D-@*KttLf^b<aKecxUp|GY`9p7PU#<S?hglb6;)q
zyN|)^UIZ_j<=^7-!LoqCWcu=3lKtPivm;+|Y~Aa3ztU6P%<b;(GiUvmo8FwY{L<qx
zgVNfW!VC8OmX8lqOLtvbdopFSQhDmrlBe+pCoxQ7FtA^;CFFwD9Ot>&bB}OZz21-#
zw%%d>C*}05dp#Znc~mpzxUCY}D)Ym7-Q!8mm(Q6yjjgDxu&nsmC6kYh`|j3V`?GR_
z=e`u(f~8w!wtBu5i;!xSV>v3jjrB(Hm4jcdoNS*KygOh4U&5S<9^Tx>Pd##1ZC1Q~
zy}P$j&vt6{spz$TR$siT?Dx87QOd-kWzLJ9gRbCc@({ffx|<`&;i#(r&PlSTyFVTN
z&DZA>$#He^DcyiYMS7AuCV2+MA2MO%(M!DX^7!4=iEAevEmyp0c=qUW=5=2<%~T^Y
zHEI@33_qrL=t)O$0k?%p^{EXl;H9;PcrHz@>G@o#vhvbT)7BRZP9jW&8Xk<&t?3Pk
zC+a<NBSn%H^08*NJz0N#!(7ujMJt;hT~)sEl&{q0%iV|O?LN|%k_trUF4x%Mq{SU!
zx#9V%%vX^wG8qb5L&RPmTNAf4Z=(Emhkd{Ge$Sl#^S*Jo$-8gPme<Q87plH0;d}K%
z;GLOTb5pp);uiCn+Tu^c*ch$|+8IUuWGw1F%Ij1S|NZQb)qf&iAL3Xop;fmt=m_id
z{XUG37GLm}TTo`ob7gMO{lxS9Kc+L7ef@Q%(Z@<X{^Wy=CAr>@q&M0L&i8!C{@2c<
zak5a?3+n}0A6NciV&US*w9H)yUKAdtIfGL`q|qX0!aBF7<&U`px*O&=^nYBM{9I_7
ztekE%r&Wm5^?<cYlub%``*$+QA}!S6Vk&(#S@70M$SR$!?>pvB4R-v0(_yOLoZt{6
zJ4c4tjuTC<7yR4Y=%l~Z=oIs>t6#Oht}(KxWb7B`3YlbO`qllEav#%HmQQ70lGhoC
zS%{_e{Z&>roo4OzQaEPADsw-Z(8^vPyQTNlE-0>HnUyg2cICflmxbI4j0|k7nJbz<
zFUY8~FSX*j=k(33T)5D3+v42y8te~T`uWWg&cA+i`NF3c-OYNl7WW_0ik#>0H=C=)
zCoqB~Kr*|{Q7v!tB9-pzLFc>ceEqJxa$TQjs4cl;(Zqx~yG-8JEIXWi-{!~N&#!xo
zcl@$9`|zy#-C|A^3zY|fp;<o?w=8D#xT5%Pj{MB1HDOjuWu`d^PS?Dgwe>ewg3qFy
zmBP#)%6fztR=KylwO6uLu{qr^->)Tt!B7iSC|@cxol<z>_LcdXTd#V{zhbxAzlU{h
zFI&{+AA*JlH(i*r#W`vE1V_HDXWQL({s=tvavA4ssRxULqxYs{S~6{okzCMpBX>!r
zkje!oTW+W3B`0-Lj=Zfq6=CuBUTT?c@EzHw0i4g*O*rCozbaB`k@k`Pr(LVB+<CR=
zzGP_Y&ele@FRMlK=9#X`xf)hm8qZ!d#Vtu;mDIGW5~UvT+Zxvi&%ATR)2pOvxj^@`
zn~&WsS3h^E=lyf}*`xVxR+c=P7q99uhi}llxI;P1@bP=@z1rE6rw5-FT{MruDMhbi
z3+so5%_Rra4+zPM&%5fGba1JQ@>ZGjHQ%4FZ}s)zJ@8MCOC+Xn(W<GhbN+p^S)Sfg
zK54t^rJBcY6jY8*Q{A$l@zW+*ubILMQI~J3Ph0uvm27QRt&z;1XC0D9m3__&o%8){
z2Rcagv>oWKCmAmv!AK+RX+mr)R#vqdhhz`D@OgPdsQ1_TC~>!lhZ-4EJl15z?3}XG
z;?$0*j&_$#H^;2J&EwF<v*+Zo6VK;vQ=817&tFoPy7$BGlHA~er|Q;9*P6p~1e;U8
zIq>azvAOQSt|MAWMM0|hR`RDs-;^$4ZQhgAEg|!(;>ENJGub2*UblQc|F+b~+d972
z=;UfocAMWb<G*g}EAn<VD_66gdL(x8c7@H^Gmjl=5SyOPc~I2<<w1*yj3Fzd6Cx+x
zO6|6)J-F(_{-s{0uEhV%ss7KR)w}ureg9XXEE&u7e{NzjSX%!1ce;~iZMcKv#o~=C
zWIfrEe#YLL&Jz~sEy%b1={19ywp`bZUJLI%J>Tf;VwQl5yEpirT(miWk9)Sc>X+po
zEbp8Ok$OGZ?&6ZSn>=(G4o#jLeDtJQx!+FZ&*#*=E-!v*5x7ij&c@Uw|2Ou}Q)0bT
zSbwQ7;qrMcwbye$FW8oR;>z)_-`CVXKhD1*pUFn^m6=D}v?putg0JI~eqQSt6Z|4?
zua%m+*y$zPrUkgh&z%~`w#7&5bW&lnz=p_~EQ+1oj{~l$F!r8F&U!t$?#;~kp(;DW
zyr(+rl<sfTy4mREd|%D&{V4`fw_XV+2mR7BLMQsO{MOpK#4g*$C%NQe;kU`RJUu#j
zidL-6y>stS<>KTkjf{=!a_!44btXovNcmH6^778zhi**etCMLey0PY|%n6-FtJ^HU
zvhVe;X%TRma!~Qmt7QL7Ev9vT7xxr=d2shq@glpP9cw)1=JFQ=w$FFy?|XXu@vAFS
zCfcpx3RxANES7Q3_Lkk*ri>{YmPpN!kL2og*f}Zfw8}Qqy6NARwuLUWjM-tLve~Y1
zMRHI0q!TO&%mKaAJ`4Su>JV=u)pbbgNDSkj1ugUaU2i|x(7sEDY1abZ1B!fG7_>4@
zcWhvCSd?j;@%u|ag2Q62_}lI~+Ja|GpE|f-^I2EKx|Ais?(<(nMi?0xNuJyAZA++5
zZNE{3>v^L|*VOHEYVz*|d|Y{(?Exd7L~8ZMx=n94)>WRfnj9_CC=<X@ApVNu)XE^I
z(8<p%&+gCrr(|m-8F=vZtNfau%dg}g-uGrpiV1^+lJ<eMjLM!*By^W&*?MJcVw_j*
zmwVn&r$}YF{M)&!tW6d#_Bq$F&8RO^&2nGQPUQd=Eo<F{#z&{hS-CH}llMj-i|b*9
zwUKlaPs``SO)Jk>ZqJHv4HrNEC?jQC%Fm}2mQPt6I=ma!7yPsP)z}9r<%}AH70*mp
z6i!+jy1Z$@6Ly`6Q`^3Wvt_PPQI@g4FmrO|5+1)Fmggdv!x&X0-rHF+U-+SBV0q!!
z7yaiG_HQ=wSl=dW6yTfgHErwpCog^^1zuLtE`R>{u(9o;xmGMmZ$L-V_+)K%vdk*F
z;gjBeUVOivV!@;u_LPb9qo%FC^k!x7blt4z2k)Xc{e81(>uQ!;QJHt1Trn-3!?u2&
z>Eo;|HKr+|EAITL`W?UV#8lD0wq+i*r>F8=6uJE;BSH4G=&6Y;P0!h$ye(L9%<`d`
zYN?9o(-v#?porF@nO|SN?_a`I@vT#oTN$!gv%PMcgXBYnKo#Eb=yeOWh~|9wy6H>s
zZl+bq%uB-5p1o7@mRm6SGw=19`O-(O8s&*wY4GhYUvRtY*Xi<g8E@^L^?mc(e0AGX
zHy@X%xwq%NlPkad<j2pC-;YUqR_%D|l@{oj=a&5UJZP{@@Vmnr{VM&?DWxh3oBsv=
zWS=#Gxl*K8a4l#t<T1;rO~PBg%>5N)9DIK1wj%I)fTusz3r`r$3Ecc$jK6E;%qddk
z`7f+yDpf-lwzVr~?|e5g{>HM^qVbofIfAb(<NDrp-6=g}jik76veiK!H`UEaCy%Ym
z`^mLUC#iC><vyu+{%Oue4|1wcshI3p)^~We&$c~$vD<AF+24kz`Ex7ibz2_wyl?&1
zMZ892Q%uJ((~Z1RVWwU=a;c>&OqK*q{or6}VzF>ipnJEW-xig__0~*NvO4;57k<&Q
zXbo+PIN9l7@{dckWPY6bF3<Us9j5x#o^Go<edpMs`_++xXK#gX%Y8I!z4pyHrz#vQ
z>%DZ%g=4~(&DtRq^jCd?-5z~keFhaH>(B)(7A}s0k2(rs|8y+P-?-3-=|+GB%W3{x
zc7=8)*?WF(Ja5~+``UV4R6OF{L5&MW2RIX^YHmCtc3$E=+q*{@Cu;VYn%gjaW&Dtl
zBeB-=@yZL_TTKfamgMi$;$5>yJ#6Lc4=rLHKE>}&ybk%1{4$*BME}!Qph`_$w{g|k
z$(}P}9w&$@P7|Ix-!p3Rv;b@V=NF#Ds9XL%x<N2CV!F!_=aUOoNnByg`|9NR>|^)z
zPk-L*czmH1a$KEVamcgwnNKyIznFMi-Uz%5EBE3K*Uw^ovv+2N9lm*7J-lM$`yT7G
zE3@q0AAM~W^8e?uwP#beS!7i@pWY%K<##pS{`mW#)lB=pb?GJ<h?H=B4l|9|{qDG|
zg#Oc(rckT>$8wBzU%19zSN-<8)Rzsy7xI&`Z|QSC@2lQ?Ncv%Z$y~d?Pgg|FjW4pj
z&a<iM&WhIbuidMo*UkGMbzai*a(p9C;oe_suYTS4x!7;w#(C#={(9E`H{s5rR(YM-
zaZjhtGQL;!_r;y^=igoSch|_vM10Zzl3JzAkTAu|{%`giVOOt&wYMU7<iC7qTk__Q
z=GWT??*~^cG2LeK=Z57;&hV0JD;DYOJ~lh)#zBcSi}EGQ10P@0$}4@6zwYF>qx*EH
zXxlq@EPD4i(oV0T{uism&P=x(;jO1Wa7`@w5}CBxe6R8uFSnBq+ULk0O?`BH;lxHs
z``7Ki_%?`o<kqS+9AE0_zj%?*0x?}RcDrD!X|`MAnKfL*^i{lkue3QNiaLn&_8yF{
zyRg<zKb3oL$vmx@z6(rV<=pIZlHc)aLSp`sT&IgEue;VU%2rAhRy9U#SGmwFr>ax>
z?9c7k_y3~L)%^;3Dsc43ewC+tcePAg{P&OXQR|$*g2R(jr<(LBtM`>DR9zE(DrOjN
z;%Rk@vEz_rn49qWZhz;~i&L*0-~1&-i!1iQKlj`u6LF=romQJpXdiH@)hPYCCTNrK
z`kdBn$8)<gW&gP9h^^yYH%;`)`PmQC&l@>7W$cjuaQWP%w2#G4wsRT<mdnbx{I2$>
zp5JW6;$+#}b#n3MD_Pp>GEMkGy)+6Itt(Wko6+j~ZL{w~nJI6sZ11!9Hog6<*29!T
z+o!6V|6TNQ<ubz`-NElC?2-G>u<Px@+E)v<zWC)DE56mv()?@2tv3&6x7iwe*?Ura
ze!Sho)5~_x+H>#8ne@kY=l(3v&vBV@c#p~Vi%oyiL3dQIJv(*7pCfZJrynxd^+(W>
zKeotP-CEy*|1{sy)~)P|A3r#lSv;>$?7^%<J&K3f*6b>_y`%P0&tlz0wSS*>E-rb!
zX103Sj%U-p@7{NNuiB1ZZ{NHNuD1XG0enTxHSu1#?8;9cx1P88oO5r*vhQnmd}ryq
z|Kq`p<E!nny9DRzm2dO>bw~94nu>y_+Oq6rD@rdf;yf((|JlT|JK}b~KK^lb+ybK|
zb41?h@bPw@yEXG~$u*4=H69iJ(|_H$z4&mhY}KElKX+EL&HT(*WmuL~>C7FHcjsXL
zA)er4N4L(ans?BA&&_A+(l~4Vzr<erE<Gh!qv_|<ON$pfymxyf8P>i#?9@75_Z^`9
zw!aEjEqOgHYSqrIU0Q#y{8OG?yYHjo;|u%>iSCz~L}of<Zu{YA@$#2RQj5d}lQo9@
z?RlJYJ?}F`wMSR;?3UczGxOMKN0F_!HU<XN`8c$0IozNBY?{M4=2C?z-&~^>?^j;-
zR=rA3%=W>IuSf5*26q}oc6UY`k&QCmw_0Y##C=XHs?u+8bsRd9T)o5AahXDps8~bS
zE~X%B(JOhTD<1ECp|MPrnSW`cjOym>!j-RVR?lNK&A2GZeW_-(dwZoxj#tXx6Is)I
zR#|OR`_eJXDD%SBwfcr$7Q#NZ;=h*kow$@Azqihd`C&ro#r%W)(d&8M^?h;?ueey3
z66&p8F6Xz>=&a!GWrhh2L07C;ZVRsKb$R<_byK#KD(iFGhzS`zV(V7=_ShXgzxL6)
zCXL3^GV`1)6=Z&x$O`)`m@;vKGv}6+ur-@{cQ!p1(0{%$qVKxQ`(x`K$G!jM^7rfC
zBNZlJb-tK9@c$U0JZ1KyWhc%X<R}~~-0LFP^6FS}V8(Bag<%SOEgBt07U`C2GPsSn
zXMD`DJ@2%hJxq#W;jX=_!`J<}wV<8-n#7mD{%3pr?tBbbE+^YwUS+bf`cm@A^^ceA
zK7G8_Te;5Y%H^Do%?7Ex&AaM1JwI8va<^M+Z)MVxdsV@&y4fa%Dy^Ecx`>C}>if)l
z&*$Fr4_;R{_vdTf{oCp{zg}}aK9+6i|EIq$yYKySsI%97pIt_c@qVdaU$@`C`!Dy7
zHotA@^)<bl`OnPyTJ-p&x!<dfmfs@#g0lIejNdpV8~3Ny@vc6%$MtjGtn$KZJ&~K!
z_?%=5_SN0`byeC-Ho7w7Ut{{hMQ>%dmY<w=?!oh`lY{5Y{vrFu{y`*LbGgpVYn8=&
zp|j+t>T4?gez&i^k>j|z(s%cF>pPnrulGMFyRpUpa+L3C-WZF7KL@hZ?d+2*cgSz(
z+Q-_tR%pLx%+xt^=FGXX>-Xoo^{?$-UHmQo>W9PIuA3jvhgv4C@!ZsPC@P|>X6KH+
z`PIHoMP-L0dD+cmZvNVNEjr3v*19<4*oTrkN8ddD^_`bJ-}1}XE%VcOW>+fDZJlht
z^r*~!zISsq8hM5N4{{1ja*^G_>A8rHL3HXj?!?J*^HUsS<};R`PVGH)DyAqSbGE;o
z+uWB|({-v%`97b%?>qC5SML5hUd>j^l$V^e?l$*kyKZ@7f^2Pe*AeDGb(4K34=vze
zmA(?#yS#9DL(8WGht{fjt#Ow(Upe(EDsn>jn?;Kk2=qnoTJg8NuKtC?)jvNv*Dm@n
z^_>0>M-!3l%^QEs5S;n7(=Ga={_=zUZNc+7CA^c?`JB|fvX4jTz_awZ#tTv-qY__E
zS-SkLu+9A)`U)D6WpnNKJditmcw+Y<d#$?eX^R!7Tv%57+-cfP_4bf&U(0=`ZFH>;
zyziQQOhuRL|B4^F_tvj5E!Cgme{S=y$a~yNmRx3<u5~I^_r$T)#`>z#;`~9+_r9KT
z`r6b!o7A@}o`>IE?334cZinA{)z$jhjpvl-%kG<2vD1fdi|@WJi;q@&WMyCeoqW>h
z=B(g6=bcfC%zI^v-W9I>B=NF7$!^p44Zi|{^o-BsW!72lyAZmE`})rJZE0@%9p^84
z3OSZxVQS1`Q;j1PTzelx-g&CJ<5*_e4VHtZ2?kMzH$Hf!D_yt6T(|wRS^1--vo$sB
zx9acxwKMN-b=m(t(M!43YVQeBjG6n}+s%Y?_sTzo;WM-k?mQm(#3$#2NAuO|b$Kc(
z9Rk<8{h4e(O?qZBdG1N);<+!67#B=ejJz|^FDPLPoAj0Q#@lMYzUJ3>Uz|Mu_OFsv
z@tgmh`*r^EuYhHYmQh`0p5{Fd0`&so_ie4M{=Z#5C+64XkG_YOi}_V1dl|Y_otySE
z<Cw~o^WY_Gp&{Y7Kdh+JXf(Q8b@%IJYnOxX7_}uI++k)ucE|49&6&@ie~qf>Kk<3X
z$!@uer7ynx%U^z{w4m~o-i<QI%~OZga7N~*`Br@XyWM<Wc}~R-+ox=AH1;pK)bnFs
zyv_fc8<s{h3Kp*T!hJbD!dR0hfc@LjPrpys2X4GvJ%8Wtw|Ax&dQ{C8U%&tBvS|gW
zEq|B4oqhfNu7wY${WeSeyWxNT#OHbED!*)G=8tx(KN$L<zyAN5$)ev=wpq+AsJS~q
z@vvK;1f$2bZK1Q(a<iEf0y(W8gU;@JaOnE#4Y!rI|66_Um0JIHn_vI-{d{}hlQ|;y
zL(tzfzaqanz0zN6zW8p>Hr-p_3S;_a%Zr~6=Zl(Mdil+%&5x|^RDAAw$87Dt=i|C<
z+&wqfzpFfLE!-fi$~<q+?wfZfGo;-*oA*0cx_tNh+gCHLf0&&g{&&x}!`%Awt3SSp
zeyr9bZC=NDK-8^K!lTC|_s2$4_O5{Ex4&lVukW|}fA*$-kUQUe)0!_GkuSJ6->koM
zfbsAAV|Nng_PjTIcOZEy!^r@nw7)N=1isz)a}(qazyq@1zh+<D9{l>_+WL&Wkz(sr
zPk&tDY%|xY_VhY|xl1oL>0D&puYJ1QeZS?g!%`dFLhN2yXg61=2)z!B4K`Ea<L>>Q
zn)BlI!kJgBj$bZ4QN;YK@Y3Z@fo{?59PJn1-&v&I^(ggv-H-HSbHS&*(~Iq{N{UPs
zDf{tsiaGys$^UzG?uk^+Q+;=Fp2$Uyik1z>Zuv#Lu9VtoJ1;h=y(0b#*ZgFD?d_^N
zmu-G_an0V<Yv#2SJiZzFX#M2KHO)Dn*L<~Eedm33RV4dOzlhIswp&?Sb<5nhHnVc_
z{ktDz@-oGR<y$^)dh46LcfZ#?@x49Mq<6Yk_xw#L$gsJ6;_b&bMpJbP`@bz&ARDQl
z5y^4U_06x<(}lw(%NSUF$;*=bb77YCzq_~M?^N0B&fu<lWiI6O$8wR?wrSVYZN2X-
zwtXfxV@|K7(_goD7w45O`>FKlU`)Zf6>I+VbZ>LE_+C7H+u>(Zrg<!NJaI&E>ds?-
ze$6c8w4bM%*MEG%9!1rCpI&CAZ1xY=d_VV;*Ip^#_p;fST{bRSS`oyd=ptA9YJJ>2
zk@|Hk#rx&_^<SUfR(Z{2|C~LUd6SEtz6~p}`11WxHFw%`cMBt#GWR{%d#b0M4%}P#
zO*iv5e^{x=)z07DyZMhSm~iKja9pspVZUpSK%nL3z1*(z^lyDCI5Km^%4;7kXlzNI
zpDCSQzO44yZISP;m+mT>r{10Uy=nP^6BXAA>nrwcU-$WM(}@mcjf0zaEYs~+?U>Km
zt`Yxk-HOHA*4<0*7892iTiD~Av{B$uq+*LlzI4#}UrSafwnWVOxJ>`(X8uyWJkH$A
zmrtUCKBRcesxA8A6+cN}(X0x~Cnq<|&b@p$&i{7L*NlTZ&s!b4cT@IV<h!W(bsjq6
zhr$x>roGphCz9M!@J`uzvFZ*nzlpuo^3vCTw+P5{E2?o!{2H)Kz^zmFNZstx&fCv_
zoijeMqfX=R?qd1d<~Nt$$V$8SMC!%A8~>*5-Iu#{$2rZS7J);GSzX&*#0`I}7gB8b
zD))Nvwd%S4%la+9Z+t9weD}uhOlj6}#b0`2!xN)gziab2`*N*LQJu3!KWetyk?OFj
zYZd*Hn`_QpHLqV28qq2`%|s>f|D7B6msb97$#}o{(tPLlrTfk*t~{j8#<@7znKw$i
z<%e--NlX6brSV7ROpx3s7?vnnf76`ff>A{<+u#33539_t^mvngBY<_&TKf}@QfnC7
ze=pxtqY<hj<aDKtD=yiox6o9&gJ)y#jB`Ibqhm9@lCpjWf1LezzMxZ%*vEUX>=b<j
zHQTH{J*juH>iM!eJS@HT`^Sf``R*4yk$b*w&5<?#a^o|nUw!toEHc+V=lQ9O`S<?3
zdHFf{{Jx1*m4CN<T|L*o%Qb1=i<#4k)IP;*ZF0YpT>RT5KK_qGT(jEm=*Zb6D?f*w
zl`F05lN1czoOJ5gF?-qjQzn@zzCOF=ds+XD_tv{kKL5~n>VDyYq77#opXbhATkTg;
zrd!lscGg>Zo|(1i%0KnJcQfwiZdHm;nww{OfAvj^|DXR@t#h05xj6k&?egu<d`_-@
zZF%iy((`w7_w~$|j<?<yKK0e#`%)&Cw;sG|ZXXyTcKZJ;O_yzZ0!wNIr7ZGS=lnlo
z#_|8a(;yDTBTDv;pIbIm25wA@aq?;XeRRcHezmu=xBj2nEZiSrXOMcnTKf6FlA6mO
zdLJ!+Jp0kH?-J$THfDV+ek|YjUbRKw(cArV_HA-ue0lMY*q@!3qW(utnmFs_i9a@_
z4`=c|PrI^XhN0f#=f5&vyXRJX0<Vqmaj?F^vZiTv#&gcO<#%=Xy8kY&TYWLr33N%O
z`#sT{clI5zHq2lCnPcCU4XJZurrt8hHDd^xJI}QGN@mR0&n>and9uIy-nPDNADox^
zMC$gJy~f|p-OA3;jFyewe{<Uz*>{<*rREmr-{UwRd|ckz?!%GAYvnTUE;`HhTIO{J
zhqBwb4RY130%c!+MrHkaee=J~A}dFpU&Xo6#%sKJ%vtT*8>0Oqr@N@kzUTOCM|M;F
zB%4K-G}hERuk^7G482m@$H0GB=<7}ar>bSb+mBt?IQvv<=#S=(s0)T+cOLO)2>z3G
zE$+_K6l$B(`FgQb%Y_}9JyQywKA9aX<0ZN0tZH&gIMcqRSvNdAnY{!#XY6}eugWD<
zU;A9uL7Qj8al1QV_y0Zbzp3;+F+6vQMEUd^|Ni6(MgHDY9%mQwVcu5HCi&+Y&4J7G
znlDJ{FW=dp5x+<F9RI6^^upB^I$7NhleQRm1x&uSP?>j1eX#YC)W2m{zW$W(j*nL}
zv{Mc-$~w2+fV<^_nr`^{^;`50N$WbE)_eHwh0)X**BCUf$3GA9-1K#EoTwV7cbIbR
zwC{S4=WLgJaqDtqY*Ob8-Jbi}Z?d~oleV0=GdHt({k4rdq89DT2>mc`^@pN&@%}wx
zY8F-dO8r{Wr$;EcaVxfHeBrqM<I(mtp%-_nAAI|%W161-orl}ph1aZW@yUL-KQZxK
z&U%4L#hp$StN$H6oZ-)vAQ!cPd-jIovn@oNI2;!$Fi7m>vwwQJEL3-eoKUuX^RJt)
zKe<)<A1~{iC7<|fQ|v#R_cMN%Djq4=czJ(^&2=g9_noi*uln5Z*79}RA@@DUKb6Oc
z#VNW#uF2uL8-L|QX_MVsKG5dc@aYVBD^K%!s3{)Ms(ZWZWa*9CiSe^n)c<(6g0o{)
z(cLFPdCAqse=aHZDHDGvoZM1i|58X!aaregS5AIGvlsV2vbp7S?%h;d`6MRBJ!JcG
z_cy|MiZ02)|F#E*2~K>o^80oESM@7`uc-<iJHPhjgv;xa4(*qnutr(QXI0O+n_f1B
zJJlUI5`%7K)O2KS-!U(Jp?$WrkfKbT_R_}-D>Xdd%)6lTvFJDJ+UMr^4$(h^EhM>~
z#xFP*pyL<m>SAQ?wi$BkL&V1Xqs`i{E$0`Ooz`0Ormx;O@sxsL!S4qL&&i$3i`*(6
zyFcUFqG#LQ9qY1E3cGo4>(iUlkMi})K8@fP4`?swh?uA{$>q}g)rIp*XS{V6RGQYM
zcj8xs$+hY;Uw6LnT=xCp20PdH-RG2CO*+B%E1lkT_t(18*v-Da=4R3nPhF<+h3;Cs
z?5W`G;}d_k2>dV1ld3wof4#z)Pg?~Zt(hVEIq{0%>(z_e1r~`Vg)cae`1C^Xk_%0N
z=}8Cg?Ah$pG5<*Ri8t4G&H1Z%JvM#X68p-1c^6MEYIcpkP^TYyzV78Eg9GPM(x*=8
zGLJcNu1fDYixJB`sf!!4ezVP0zqe!mTGi6ptgNix<=@WFGkg8%TJ(0l8^5yUFD@^e
zAII`Qa?j1I+itFV1Z0!uihk-ANIE^g_TV~J_W+lIouET0u3P?ceaFS$t=pqpW|Mj5
zP&S{Swp4ev>)hK%-5)s`@?6Z2p8tF0!)1@9gUt4SdL2+c!}^x>+Oqd-i>0m!seEY4
z3$gs3x_4rzep-f)*BJ*xZ;p-kCPbHiY@OlvRN!ColcP7JW!_ug>)q?mVQ8{#!}J|%
zH5NMd-jQ<8TJvb-jhCHSP95LuDr1D7#TI>&3jNOe)Gq8!(iQ)&DiuL@mWAA%Sng`J
zy;|ahkcbQCHql4wx7EWh-{g#SE$*JD@vB(mKO29W>ZxCk_FpZT^j`1slpTNWziU;~
z7I=6k^F$+qqLm&O+n*=CsT0HhXhu2BGrDT7D!8!N@!7@x=m*b}CTMQ!O%pe15MKA$
z{`~!?lN_ygrOh#sWu2*f@OZ{7U(O}_4|ztGwEo!{Tk6VwRA8OXB_T<Xg})zVEi%*2
z_j`K!x}Mgr1!-@k_k6KhQuOu9vg-@@!Y^3nZ?eidm-;Gm!4l^4OsbQ2$t3*#Wu$qt
zqa?+Zk>T&@)86{Le<Q=Dzg|;3ciPnjv1VnF@1qz@9V+8^yZ&wb(YS|AB<$gAh2KUW
zilQb}9SYp<H|I?xvx_0O=mTK}(SM6r>KW%Jm>-tY-8w1O;!TxSiF9u7;Z+Wt$M;Ct
z?0+uz?k1;0=(}|*Zf#TNlK;4G;{TnySh*&=Fl`Ohn0`>Vc2oYQ>)S&wXf`_eym#8z
z%-Zu$xA#kr%9{x)zdttrJvF7`u<EDQ{T{ADiLLUnvve<X+~09-yXBG22Nm{~?^*=X
zOucJ6HuR(mO!1cDcPn>$+5h+1#Qn$KTa{S6|M24Q;Y{w*VrjpuUkv?!4N_TE)Lmxo
z>IXI3CthB^E8oi_>(GIgy_|=IeyR3#z3<;5R=`oBx^vo7@i+4-7d*cBZ9|L6>D;m<
zFSS%`%T4;KZS+__^L@RW)jdxrJ!Qs`!ej4_&EvmgzL<r9&GV$D^>q`URbk1xYl6Pc
zzW#sLzxK6kTVwCr9o=>J*NUqhnJ%wBKeBwM__NpWaf$o=&Bvc}>=g7n_kj0lhl0}^
z0fto%Ux__GQ)HJAm6@>?T-g0Ea@Zz%tGrv{Vch>;-?;X#kNkV*Rd%?!TM(~HldfN!
zN##dz(|s!qwRc)xn!dc`{_kV?r4I~tZvAsjeOsBEuf_Vfe`{Y{zmT`>o#A2aSyjh(
z9j!XH&FD_m$2Sk{`TA||FhsHh%3fOcso2To&-xNEtNJTH56*56+@5G9ZhB1Zx@T1m
z<mT>s6~Tw!ef{UKGm!JP>Xv`zrfI6?@s>rQcT2Mson6*k^w71)ZR`6K89VoY=s&61
zIxQ)hdRO8ZB7I9mc<*jo;Bctm@Z^-$k1M(V&WU$X;IeP*P5WUocd={IpNtUEt@kcS
z82^f!t+am1M88WJO7j{Ton@u$kFC|y5$hHzU$tbV{)=rQQ41#Kx3MH0)jgExb>*OL
zf=yEIhh>vDHF=*2nf#n{N{ZB#&QFuB8ZP4aa?tI|whpyLw~h*1zO}JmG5=7k!?N!5
zQ}^$mxcYMM?ZfXYUf(*JTO<1_{`&7{YPHKBS}jXjy=w35=QWY3p8Lfac2!t!afx4g
zEx+-);Tok$hL?(DEl=FypECbazs7~-B}>-wXO=u!`G3l;a><zM4;SX7uDBe&VA7TR
z%_%Mm`5ttw;VXUr<lUw}Qs);ffvofYzoC{>@!ZzVjKHV07V>TO3H;jPz7Hq!1O%QI
zeIOjewb7m1b=A_>HFvIUTiO^T;_^_co^eG1Z$#>U%bKP7oZIw&P3h5yc=Sz!Z-vGM
zt`d<eeyk6c=p8h55V@jx+O6=Akyx2Q7O%GL$NBzmz9@Suu071Z&Q!MLl#%sC=GMyn
zCIR!7-&tyT;fLZRn`njS2N|j}Ca<t-*O!}D@MCR!c79#eW0PNE*C7WMycOSaF6Upm
z5kJ$k6<&#4AJZ?2ypXMzy)E0?mo2x(T<xtRGmrAF%wIFZx1H~}5s-iPXQ;f=LRJNq
zH&Go%PJ4JC9J%GXx^2y#X<FYa*aiJNgqGxXRG%#LT{BVl^DaNoeazvTrEi(!ZkO6<
z<&nMd=Reu~@BWGh9SXUZwDx&i*uSDf$Bwj>98=W(^-PJAea_8qCij0Ixwne#N@dRN
zx3_Z{miFnm&v|Ui+$HSrE<<#y;+tnXA`7h_@ua^jyKgo(^#5$@Szjy)KA%~Cct7{w
ze*#KtCQOixfABE6ZN2P2?KR6fa+ymaLq1%5>L0!EyK18Ro;@!-Cf{f~TDn)a-{kk&
zLeVeYj!f5#Rhnm*d|{f&;Gop4JSppe{#;g-IX|p&&dz!NezB3ezD0$U>Q|fIxXf*n
z<!1G4Ouo$im3@o!hsO=UKV7H0$L|qa{Nq!6{M(p)j1622YpR{T*WF#_-ZpJ^_(O$P
z5AvZaM=yMxtk<;n`kknwuRj;fWNqu48#lWzYMHdmntk)dI+XNoXDvMWF7>{iz<CXi
zyGtsq|7?goHMPgz{(?v3BD;#~A&dM9{i2o$&a;`nT(P%4TI8|j9iydN8*Zy;KeS}y
zP4aY4dEIhN+$(7LrVqjGTQVzm9lm_t?ytU}!pid3J5LAyFy5NKv(WeTVp;9)bG{^>
zzM^(+`O#{{>u#!x_dV|aaB!#Mxs~&t>!{CKv)1JPr;3<&okjLsKX_k%UmfrMP2@_U
zZu!d94!`D1&*@Q>u70^QX!nzs-<Ots){czd!8c{{r4y0YuGg2po!XO9nDJ$u-yNw3
z!t0!iy6^wG%o#9W;H>NXr|TLvN$9>)(va0>4Adx`Uwrp%oY3FQH>OXgx4+}h3@K6i
zeRPfK&HnJJ6_5Om>s{|<-Rpd*!gAixSO$aqn2W_nrBgn1PM`ADCP-sxaqbhgoX3@y
zjVCZ|P`uo>MIy#(yMtoO{wv})UY@T~`~CAy|LQ;53tgY;)IC|Zx4dG?(R2L#Q)R`@
z^#7f@^Z!CMRs*%~hun{_oOXIA;5z^5Pv(o>Vc$QSURv*5#G&&0+n*>ADXG|%E0UVI
zj!!6)`OtRjPRh+oSJLl(C|aUZ{;gc@?5d_a<tOsgt&dE*rhnbi=;AlEipL)_+43c?
zsS18PXLioux%@Yi>i;JSR$V$M7qsKpW_$fr;C(0Gf>NbA?|u}SEBL{o|3#}$QCM31
z^Ep*NyXT}AJ)64QIIZeT#!Zck+AYmRv-}uV9-DaMu=u;zV!wj7o(}z{*7r1<v$%2N
zoVqtTiO0@lZC>4?+QY68$o}r~l#f$>Fdmsx++*^M)!paL`9RNYRu6kKR@I*22zvfl
z&#nDU@dL?iULG^{`M#WeDc`=j`7h|!VCKaS+L=CEy_$Hb)H}V`iuaSbU)uYi?~C%D
zbGa*e-@5LZ=6s$1ZR_;cU!*notyFARnE$p{zWKxPd7a<KAiGp6#vT`+nS4^u*abdo
zxaY-BT5{^aP9@(tpO<@_UbezOvE?U=@nYGs3mm>*58nR5yl&IH;(rr!C0_JetYDDc
zwIhqWAXNKR+wA=BZzC(V1tu<<-!A___KY{{j#qV7KZUDTe|>yCFYapE{pQ{CZdmR6
z(!llD>$lR@+b8{#ULU*nY6h3G^`72`Y7eW9Sl-%cw0rUDna7)jYG=>zS*X8PD1Mez
zc*l*48vIixU%Gt%vUlc=N1K>(JpQYFy?kZ;hPnqCj`LY-CLPUtdHB$!$vRUx);DL{
zzl`28e^ZUbzlbH~Nxm6>7rox`x9i#w)0{|;hJ&}4D{x)+sA~KrVo?~8dPaPu^W^jk
z94xw$Lf#>eb><9#ua+xqa*E$nvS>pWlR;FltpC!!&5z2|qw-F;Gkb}ai<ovlSNg8J
zCe*}`tz3D}<K@<i-f2AKw^v=afFb5-U0LG2pC_jt3+whb`t@|?!7D2{G^WNyE_@ph
zI(zlbwqGX{u0i)IT%I;}!*U6|vWok=+sghwGrPDZee#oI@vEwrzgq00dUa*6k!E3n
zX7Ga*cO9l}?Dh7nFW=en*(1p;;K1dN(PAAtk6LFo<>)!Dv2qZ<wa1T(J=oIkms0J|
zz#W~ut16aAWM?KW(NzDb``2UA`Z=%mX)3NP@oridy)W{BJFn)!z1vb&ul`aj^`fn*
z%0>0@g(=ScwXG===PaGb|Fp5Ll8G&uCt=0ptQidgz7~#*JRa(Y<xbx{{x;U;{+jtV
z?UQOc)i^)YtnCrDH|V!>y=C~+V@gUS&%<>t)%WiNmwxY!{~o`y+_fa@&&mGa(>LyV
zc)f3pk)FC|`|F>9NAvd-{qOVr$;{W&Z*k>^wFTF!9ak5o`0ust{dT8rWsKYLFU(v1
zZL5pUzqKp<Y?o>Oy39GyYt%z$$K5G8?VtYOZ_O&NW36X6cW--dwdu9YJSXe+eb!(8
zEWBkj-C*YIIete1BO@P{+-eV6{Veis_{lqeGCI#i{aSud>zwFx(FX_jJvuBMl(ez#
zg~!U!u4iYi^y^)oxN}y?8lNpKyMA62ikci>Y%OZUyGTY%>z?(gOeUTyj5Ui)8J8CD
zcDNREGU*%?a`nE@H-E+2--kc%EoE*Eos!Vc<a5Y$FSiEki`M-zVGpIgT1EsN)`~nc
zRmZf|B$aiix1z9vh)#&jDy|FM{ugIRX8zXZ{?rv4!g%y6Yk1v4i-cd*O4{n_e^&Ar
zPm7F>iJM!${P)b_wRdaRYD)I%@t?bY=C;YTbkMZq!Usjm?W%V5ZM0LhzONRp6y$W<
z#8Q6K=S@B7ou|B$Z8Xk!_^sc#?OD#RWy&XiZ1ukqA79TYEB<!kwBn`H`kj{jxMX%!
zFM5x!^TuQ?i%`=SmdR}DLZ$Pq{AWFi_A=?;WPGF|`;R~<mw4|*!{v5*&MT6RN&T>0
zviJPK{-w*xEcex=<oABjD=b{{vE^8n%DTu2>ao=t?y_8~U*vph?p(R|iiDuf_A84{
zA5e+kzNh89j{m}Q_r0vj><cZV3MUzLPg@gz&(2nE`n<<3{O-$+|9*8qYpeFGH>Veu
z6;7$OU7;=8YVs=ZvWGF3!p3zx?smWUF1_iv;&$ZRY$rq8q!~@N^P&z<X`FUY>#<&=
z?VX<!icZU#Mm}>`EPHciT6h8X9f`dgD;~?=F3YK|ziq9t^76uSUAxRp^Dpno`+Umk
zn~Yy8=k=J&pCW3S(_Y#Z_xql!Te@Ic{-oEpCrnceezW~|-R_;0Uza{Sp{n-4<b_n|
zj?<S|RG*6NvO8aSSZII#<4buQi>H5Ox_xE&ef_mtpS)vZm);>gC)ap;uGj272ERQv
zk5_;BIrH?jly#NqrI#7E^V!(FFy7J}5cON;xQ+Y!-o0jJceWUB{e5zc@Y8_ysLu<Y
zU20i%Wq*v@9kxYzjn^lBY$&Sg{k17Q^=+`+3N8KB!BW}Fcd?$5t_pGH{_1g&d%Do8
z3%ggGe;cyt8zaxPy1i*vZprJr#qIff>EG?NFBOY_AKvoxj^Dk|qvme;9WN4pMsD-_
zH#5xI@6`$|zWE9LOa)9+oaRi`QQ~&sFbt^>Q|P+2rl5C<Xu+%I)-JUg_v|Yl{llJf
zJx&b_{UUXs)9Odza|f;!J%&c7r{2=tkiyq^C(8BoRWt3Zg^LWszsMLzY<Zy3QCylb
zr=~P3fAt0BZne7qq1Osudrwt<<R%_7`Lk1E>HUE3pQraWFSNb?J0@xQ*J}d14UR4s
z4LRF?E?Z`|MNL83t=q@t{_d*Fajj07k&&_F;DVF76U6(L?tY+GziE=g_tQU3U(I;K
z(0%mWqmRpCq7~NPP-0W=WxO=yzSZkzowDmD?%X2D?>T3#*S7lex36q$xL8!>C9!qp
ziF4-{{yu(Z(Neo>!OLn^Uk!_tT@lb~>FQnEcGms*nuU`*x2EW>INATiQ>yT$ovm(r
zk*Df{;#JX+H*36%zfJlW5Vp5@$K04_QtUz2?9<Y=zQ{3O(SPdc^T?0cvy7f_U6Y=a
zG0&Ru)33TH<vl+OHIM92dAiJU(pH~a&($aW+~_&!`-$@}<9g)nA8<{RkErmm;pwfl
z_~zO$S*_p9_VSx_NuCE?)|;5jeyuv&ytn?>(yJ$DbH^XPtLAx?t<~;AVfxlE)>ifN
zPbv<lzs;>*wQy-}RNlF_t?e0eP44@hzrNS0eXhkf@0K$&c^xv|@3g&KcAjCoFZZmy
z_qI&`%>LZla{h%+WjEix+VYz9xdYd_2PQ9U9{;&(slPJ#(vK&_J7({-_`UYcb4H_u
zCWjjfa<B86=j1$?`y%&w>{NTB{qc4EeUq*2Zf?vgO6yfT8NT7b0kvb%4CmjLZI`|M
zE;{OX<*A>uf?f1F{%942-c9`KcV%wB)rM7XBX_vJG5q%aV$_$l7vE3eUOyrJ;q6bo
ze6iIh%1(5&9=LIPOUCbl*^3L3e?ObP)qkDwP8HLa_fwoE|NJ^{-GA4;hZ`2$UUPfV
zHZ#}Lf|r%HxqW#Nw*L2v+jh-!CvSVd;$6ih#a-*(e(^r;%;EJcdHHLWKmVh?UN75!
z?ojNO``0d;t-mWb<-o2RpWD{0FJ0LwviG(4AB$JVcW8yKF7vKg{V(_H?>K9TX(x*l
zpFc~x$Ka{9e|F5)W5&7rv$n5{4c@X@E%oc3_wV{{h-|#QOYi&dJM8-63)UYo?Yt~>
zep#;HZ@t{@AK$!^Dm2wEUw=V&arT>QX}6!{?fv^^rg8Qqc{l6l;OmcD`$eRM8N$ow
z>`o5dpEu(m@3*so(f*G$&nT$eF<JWZ-?IB%^#(5))MuXclCN4Q)6lNA{lNqu*@>(k
zY}ul%aVBe7`}Z(?SyZ}CL6~h_0H1^F<Mjc2FH#FU7(&5!lo#D}(llLpmx*!Cgk?>~
z!ag3}5y*Z`YnHZ;Y%yzr>Ekoo&+4*kpWkd(|I&-0jX{@d-Ln$eN{fH9XWZMk-#U)N
zM2lHuMU+~%#6tz$J;5igAB}&g$)L6F!7PPo;W>hCTZHBu>@i)&`{B#u(2UwC1>s7v
zl0_0~#hylQ6pa#AI;W^du>JoM>MxlS&>T8<$sg6|WYyV%k?&s!YG2^L!z3u~`!Xg&
zCcpRfrG}rA0y7&*7Gyh^l^xa%-c);dYLWBT^xF<~DM#lxr9R$uD2=Q6@=<43?$bV5
z-tLtVJ}(YTI4JUG1!J2@o0m|z%G4$QEJJj{ypHkbJBo2Hpa19(3m@lmqwUdETGQv}
zG&YFU{feFQH|*Qz4H*YBCs?0e`S8e7YwrJ{b8^?q{p<T-m=$<N^M22Z;6UdtWyaNd
zX1aUURa`Qy40WoB4P>vcwBEGkO75iD*Yf|cw}$%4XI5ojKD<Tdfxw2w?L8ZvE!%qe
zzJAP65C6TV%x#|1iqip?t94)BUE;6jaQfIu>G<0fPn@>SnQtSVTEy@_gHg4CH{;<6
zc1_d%o~EFAKO?X4p1argv+(`&{X9R|Bets7d>5|Xx9r-yEt#2~ndi-a>^<1Uwl`{i
z=H9JecPzG-eOsIoT($j6=gZJ{c2~B?&Uc;~tbJc-f8P5|dyT*Komw%m@b}J>4Y`Sz
zbB>>qJ*ctZOMdU&s*=~&8790le5Vm!zh(XvyO-ORq&0+=xHL?8!OFnHR}?TS_1%;Q
z@<Pt1A20^7&T)>3d3xwn^5go(BMi^?m{-oO+{F-f@RG(P-JjFa6+J&pwo?0g=HyGs
zx0j#Koy(yu?B<xtm&eGbXnw{bXX=9s#aI4rNSAvOsl6^N>g;xdpj|>gbRI`~NT|+m
z)JlA}M=SdI(vuDv%B58$r7cr+l)CL)=Q3p0PGPpVXjO2@**8Uwd5x3r<zF*FQ@KUv
z_6M}KH1F;>8sw}NvBisR^@AVBCx-?+`WVs>VYT7Z^ynXpcT3clKAUn#y^i<9k-Ps3
z&Kmzas5oJIa0DyEvct1iJO4PdVd0K<pGtmzoBB#`*7t-YpRd>Tcq^@cRsLD3Dqkvn
zajnMtzZVi^yUfnMs=hy^=<xhiH&fH^>9qWIhujS!|6O!<hY0`45aF2O@0K%vORhcd
zztw1o<D6oF`lkImvWwV~z1A|Wh*65)llz20g==r1`(fEY-O5#6f!9rEzMeAiXFzhg
z>?)hk>pa_cb$qBYn?A9nxo6g`l|Fux&X}GEnY=UPrOK)4JMVogTzR?vbCK-KVs$G$
z&)_-t-YdSX<%_&$GxuI^sP7v8OS6tRWjx~YYt|{2Ro&&4S`@aMY1LdC%Zt0SR<3&e
zVh!WXUxv1-mMl%~>@$V>HZNb2-E-th;q!aiHrgA5E-im?y;FV7bk(8<e2rW9o=$%L
z-S~;;y^NXG^Bq+-Yx|ofK59QG$0f*YcVgZ_r(1Rwj@&=K2If1rbOcSVOYzvLf9T!j
zBinv_npKsU{eUau=hb9h&f|Bvst?s|_c?e!djHCD%Pp!|k2L4XC0=ewJuokScD{Mm
zzfzfp|C8VEf4q3rw(o^CX+dgxFLZ}E9$@Q;2w2&*KqG-sbSbA<$eY-WiS910N4sJ-
zE>h}X-O#|r(V;N+g0P2&Vs21cUA@ioy14T{*GAs9oyoHE`|r5=C-Zinx67Vh-1Ph8
z=4+d6<ow+}_aqywOX7dAe(kyYb`LB6hxVOMH;y|c(R09HgTRaC$N%OXy}8r!JyVld
z_>;FcE=PYpDYvWV?W31PEEW78Bp4(zGCz4=UOmh9$`<#@Y`yLYBG3F|;*YNQl6_e|
zuhj8%Wv*L;d3MyrEs>WeA3S+Lx`A<qB*UaRv%jf}s;mEzu)bd`@`pJtNwlHt&3sAk
zKP>yF8|yq@IH#2%<lOE1VfRcXyb93>x+h+A&mipPqD_ZS&p#|PXZLOHd4JBAbARKE
z5q@A>p?ik$J?rik%M{adRr$AnevsK=_@Q!v@;8~)4h`DhSQhYY*nBy1uH3sVoY_aD
z%M)fFfBe(GK<A#|Jk}{kt5X(mAKo0nx<H{`y)nIMa?<39kJCTT7OmZDlsR#l=(a$k
zCw31ONJu@@vruJFk6{kj*ul!qVwKu)@_=1ogZu)K&rvbAx$cN;-*BY$X3AHswg?U;
zpZ)5^4!=ZVyfuzGHa}PsAuUjwq9$6mXt}=Xw)m#&Y1^LVXfMy3$`G~5K0=3uoyqQG
zmf4*==8dcV>tsX}hCR(pP3}6rR9<XW{KtF!zs=^!2$yH?Emsq<|MR;#+U+=x<A(U(
z!qVrs|FR0SGVFBQuU;ee;=+O1M#eXP9*%e2{b2jPlQj~i+)AfqB%c2bij@lAv~NP{
z_KT9uhBZa&xNYxOa!Dlo*&nq3|9b`1gUxJRTmA@G7SCHd?X^<)kq49OKfU0y3D{J^
zu<OF30!^j}uZ1-yAI>O`e%Bnf(#I({@1dyYjI+iUzXdQQ^oABZ+jg?t?L!`CVYb+l
zWp5eI&n(WV`2V!r+coxZ!+ZnBg^#L@zfZJ`diF-aSi|Uh|CKA{j;<^IJI=GdIQhv;
zg~LpdEwNvf56yn-?0;1>_4obm{5Kn3gmf_Jev;-m>>~M7M)ynLA$5=C7PTzbk9^%N
z5pa?rR(<p9c{8ROuG7)Km#%l=&7=nR1<F6m*PSh0$MU1(UdST#lg5n)Ro0(o3^zM&
zB;(`%s_xQ`$?F&y4kTXLmvsK$dT#fMdxdpjQ)J%PO2k}ScD~&IYuOS1EvNs<WiwYi
z5IeW1&~AsfP~4|-2KDBDchocUF5UO!HmKhqmhrr1w!!P%sT+>hGyJ|>QSt1&<mR7w
zOV%*``d@W<{qg@*ViA1ldG+f0{&qL-@vp!B(IMedee?UqRR*(iZfV^J$Ph}L+~79x
z9M|vu_gWjnA3bY%eW5SF&A{W+UC-$MH+FGfD=)dRE%&J}^U8#c4<f(0b9(0;l-^#i
zKk4(t&nX`KdVJS^`0e^)|LWW~>F9TPmlk|{e5rnkb)@(*&EGd!7&mR%)#CO?WI>-=
z;lU))Ns&notUp*5sH*Czn5#zWU%w;bu#}a>s9tq%=Bfss6)MG{o2A0cB*UWW*+SOo
zKRj|^%YmaWPM&I<XcT^K_Fv79y;H&qW9LMED*JFa_PyAfuKjCj;!>S5!*9lmHKsmh
zo}j$fVS)EQ-&pIK)5=>{E2uFVd=7Lwr2g%E?U`2nO4m)Xayv~llzy7ud{=e-k;Ga1
zoBLZT`TW27E^tm@>E8S~{?y#SDMBlRs%Dl)H($F|uJlk_{@v59cl>$(-1t&hJ2{Fo
z$B6ge%>4qEyTa!CZtBoG=V-IK)^Z|~oN>Yf`=%N9?Y=)3d+U86!f5UOQ(cewm6-B&
z@_h_mRULBwi1yT{s#|Iu81wjqu9bz}W<2|(L+AN|v%y9)&y??!Pzii;{mo6oRmDe*
zQ#KqaZje8q!d7wTketc3Q;p7Pv5^N;cka2EztNlL@va%mlZp<tu>O+&^!>y9!|y(r
zSGPEQ)qD5$ZLLIl^vw`~RlnAK;lCfR>L0fE)yV_?cm6)zbX9rvay_rRJGQLSuTxUy
zo*8~eUT#(T52nul1srS@4E733rRL<;uK%;@z^@i@VK1R{PyVIM-qusU9W<SxmsUOJ
z+ce4T|I*JZ_A~Oa>(*?Tw!5X-!e))^-m;_j&o_T}n4#*VH}U0`^7M1rH|>?mIeNJI
zp3T$fIhY=Gy2{^V9#;nA|7W2a918`n%f5eavE%WF<h`=LOEgNZWgGf`C^{E@9bClE
zaDE)w`#$&1>}sZzN4ttMpS$l4QO*$feXMZ}2jg3dU5Ag#nl<lyWP5x&<H{0F=`(C?
zT)$>=torkN#{Tv{e_8J-@4x?PUg%%j>?h?-S3l|{w#Pd0sVrQ`8tTS;;LK{X?j^US
zyF(4WFbN-R=sNwcX!n^tN&guF&%}62v^{VL6K~o0XIEOx$u7pJEujX_)-N#^+rq%}
zc>W#ZFZ<Rv|Ihuc;2^zOaNPm_l%SLI@332jGlVcCH^fc4K55#d`86~1ca)j`ymn*K
zsyM$irt|v)AEkTp-O%2zy{vxE;)Tm)m9{=G%-d+G;H7JK^N{Uaea1gL{od{Z;VXEU
z9!9avTVD}nZI*ew<je8K=5F!bhce%FMoJ1Bv`_GK<7+Q{zWz<;JeBh-an|yM1!t{F
z`<@&Bef{^n)t=+#jeL!Qem7>b_=Wtsx%7)Gd+p|Vv)O*`=$Crp)}Fj`yWHZLQ>C{q
zfAT%^+xmm{pT(2-wiF+bs0gt+?VfVU;WFP`v(MY(7w1g(*_5qYk^SZJnn~*EU-^@s
zUfTR=ZbtF3M(O?6&n%iT)5lExey2t5b;0-;>-Qn2OOoEXa#nLQ@A{zcu<Ln=e_P10
z$*JA32itG{`~FD!w9&y?3|6c8tTrp%J89}K5+HWyb&!f|_3Hz=6ZzGiKl`Pb?%}_Y
zPsO-`=T(elxL)APYUXa{c%>CN2PQCHX8Xg*$u6dDApc(F&ME7SAN0@n?`VrSnr^Ra
zy(p{v-h@}|{5|?_UrN>~usmIHck{NsmAiy%uie>TyDe+x@mmEAZVSYV(hg)A%=qa4
zmH*@JO>2Mr;AXyYK<c3GgMf2CjXAuX7u;I&*zS%^_Nv1VbF}lNJWN09-jM&s;BfS&
z`-vY;b$mtb7uYRB5}*C?y0qN0xFUulsz82?tZm_%_ktFRuXot7_KU?#`mb?CL{U~T
zT=B0mi=gqrTT2bvO#T%yT%4HMrem~5@R3-<Rvx43dzU}&jyo%;)Dfe}F@ZrKsci;x
z*ugWR3}#QX55ySeB;4a^n<o^Pls7~D8G|Eto9fqhKC`E75wBLg|EYbhUG%3)^?QjS
z`zFP6+=|>4`}^-E^WO*TeT(jXbrt-5L{1`L;~c999kvjj0|)FHH#?mZRS=t(6mVoh
z=R%PPod&%Pp$xINHcy<DvPIz2^1`rlzm=Y88R&lr`xEi!$lXo-+U~5!83KN?SGb4m
z=~Hl!?l_uQ*JnP_|ETR#Q=ha@%h#bS*JKOJF7ZrpdV9&@xmVQ5$)P7VcF);=OY1nF
z$gZc;JQ6);>@KM=`Yig)MrJd6ad(-E*?IZ@GoLT8KAyhq|K1<R`aEnIRbIwFJFgJa
z>l`m|=CEqU*-CD|E}K@ijej;8TeWh0O7wUZKBN5nF@yAzA0BuI@8`8UqFwvu=@n(u
zx4)<42UK6<F<3M+ddY*!&&;pbC0tDvdo%xLj!VD~>qIk8BkxO_XNq2Qm^<zC63z()
zyPwD{^-7<yGC)4}nas5l|8E_5vY+$W{Zz}mCHcoi(>ynHJy&)%+`6+hQ?cTuNvDso
zuLtuznXL|C`H$ZSZkAh6ru+MSonB$#`Se})Zan``9QG$}XKLEjWt*z5x>v6AxFf&$
z*k_~o)4djo>)yV&QFdGZm-5Hco&U-=U9Ph^m*(Qz7jvrS@w%N4)^Ahyd%Ns4_d2Z^
zz0DTylv48Up85KIy?xB%f&=B2btjtD6wWbyEGo?jiii=;eO`9v;KvU+$Gjz*O$|$S
z-WK@KaKeInZZV%|J0H^-29bp~^wv)-51PF>X6_dS*Q;l%mG0~O+Q61@|Ks1ZFQ=cE
zC+Z#HxxuiE+ka}!v)vYkdk^+)iTii2ec6*;wHp&Q7cCIk{$P%VSTy^TL#FK_Oconw
z$=|xk|Ak>^hr#J6hNP>v&S^YsdbV7}nYDmb<EU<@_>sLYtCfFM-V)(H7R$%P;(xU6
zhn}<MHGY;0OgzTk8~2wce6)J+d_JWvHo)v{+n>!wLh6fTb;I}8yG_1uMqBUARja}Y
zVJ;TmKm5^cb>CufAt&_mYm=+`y2f7HvUSZ|Ezde;m}zCrGMD~uW;;``D<Y|G_y1cz
zW*4k^vR{67@uDp?X1e+wr{y0f?RX~JT{8colC$d8zx<22-K+9$?M_QJb$z$~rLb46
z{cO>jdxC$)o%yXc|LJ}1x4PN)i<fM^bnK{f;g>HjZauT!a$mLX{U22ihg4(swf$!6
zRzs5HB=JW@tZN#M+olGxTt9PkKYx+`mA_9cJ6_j+&<s!c_<lyr`oB5Kvf;JXPDOL)
zPl_x{TykUS)ivMv=2@8?myZ7NeNQB({k+d-7gV2nXIOA^-K0x%Zb($ypPf8eDm=9Q
z>xVf<i~IA-Vk7(I^Cuj?*<9~o+RrU&aY9rfgV%<qdk=HSm9I9rd<P@A#dob;dr|NO
z$4>4y4v|4Wr>4rC=xq?a^5oWk-O2f454f&39QhWn@LtzAaO>{#rMYIT9b0vD+CtL`
z%X}LJbT@6A%EOv6v(!Q7<~h^#ht^z)IDbkmU?XEp<IYcaWvlDnFrNu%-1%sl$NfIb
z^Ig?hpI+}c{eUZhQ-MoY=jghVn{GF(ep#<OL#p^p%i`a6BKAtX<yof}U;k>tkI!@S
z9T>w`)=r+K=JR>~BmXb_Yog@3K3WFs`L(p`(Wd^SbzT0Rdjk6M_DSD98!fAomVc)&
z`_5Eb#XXHl$G_R_yj#BH{lyuAQ?E~t54TZCE<3j8jPHlGZ%@KF-#+fslZY`;*dKdm
zGw1oGHF7c1dp7@yy;Hic{uMKS>e^FjQ^TsJRo(l3XUW<4b1m1ilMfob7cLC{R8t|U
zdi#`l*uSS=AX?stK6@GeQvP+hNx1NKRX*>cM>;X9Z$z7Ixx)J6X<e4u^+T)6j^$4I
z5qYWj)7_Kit=Vh$Z=1Zw{@eZLy_t`_@;_;BozFk->2}Ti<`r)KdrU1?ANb)AnV`2n
z@}B%Zvk!BVMdYUM*ZY~f;pQIApSOGKRd3io>#5IuApBNlozC^s*^TCvz4hHG{@?oV
zC;cgF`T16jyM%L(+>0f5OTVQ=KiaUwVxRW>pnBo*>HorlCfv|@dE`lt!N2J(QLSD-
z^s1-bJM(Yr?sHlz`ZxdovZiuMeZ=%f?$z9B`FdL&sx!HY>+7-}%l<3#ofE#b>}ek_
zdl$F)Gtp<_xBedqeipdp=an@_kEvWfdhAvGiJ-^X2edX{D9b%Nr+<&$!CJvCv6bo4
z3ky8vec)SQCcY{sar!H#lso(Os|laiWeD**7u|9o?vU2b&!*Lz*@J&E#ZOc#T<E#+
zvDCua2Io8TCABtd><Lzzv1tRtYKsH@T8vqrMAP)VoBcP<{k-XhfJVpsGm(lG6K-l9
z(Ap5%5UO)7&&;@B)5P@?<SaNQIErj9sK0Ym<I_5mCozt?7t(sq?RsJVWMlCjT_-<5
zj>;(qO`Z9k_g-l9ly2pZ{v-TWN~AH<y|}VrvT^#4BeffpWK;`x9QkvxY4<*N=VI;I
zk8f^TeQKWW+2@O+Ke*>#x416vSfO^Sbjh*fNxA+h@-lN*$8nwgbhmx}4CD18x_kbc
zDFnz(Kl|QkpY<NWGh*!7-x~h?SajySl<uFj!kBCGw%*MBXJqL7J6(0nrpn(nU);M}
ze!aQ*^V_LC{zqapub(jB(fwolS@G(1<1LFW{roO}{RgMRA-OGcSzI2<PR{qL3;gr;
zVeP`!{Ve;Q&HEEK$6kI9C}$kWseJoS{IPsTyju0AD3cc_*lQUiVs`U=XYgRKVVS0)
zQM7jYg6S<yJeD{1f6ndRZ6p6DCI8m<W93g@Y+BCM&e1comvJH6`HCa;4eOsAv@Sf&
z``kAs#ZJNPJfCPx?52&2ygy1eUYuu8Kb^NW)O6Dq@AnICY@c-QqRo|h!<SoMF|)TX
zyyT^r{jg)}>3Od2&#J{3Un<Ogc6=XOl!<kMaq+z7-v1f*n(i<f7qcof?rwKluwZ7V
z6F2*(nZLa)OY5fn{wy=+RQOHFPRZV#8V2ufwKI9?X<sUEoE_lcpc%%L5^+>A%z$Cl
ztw(E8Hq7_s_Q_)2<hjl3b*G)UO+no`*6kU!_i{IXQ~!VR{J!rut*>s+J|o)BYPI{i
z;)g2N12SSS_Gl(&%=djUS4?barE&ah+Yhlj!`E?H3QDzK{kSV7t-I1SN<iUxeBthp
zq%E8eE;_L&^R{f}=h!lH_BY-gMw?FAES>i<Dl@!z!V=3Jfqm~PCOpjYnwh*)EPQAB
zQl52Z@3{%jIeDM+{Ix$@{~vz8$<;HyxG?T>&doEQxBQ>}P~M4m>jke#eH-}Y%4{T>
zTdvk~9_?5s8{EWv)z*;hS!?9hN5T78O+$iyZ24Ip8oT>M++^dF|5*_WEVpyjo?&p#
zewcpoQ3iwB?|N^&zMWgvq`$L#W~X_-eVxWjk(Uc^K3j0U_J=^ux%XWqSIzeG%)0qL
z%Vcw}h}EtBjK>G%?p$*X$T+(|_Wk}ns|&-vStq}XFaL0FPJOEVy6<auNKevoQr4cY
z+3U{jFkRVNu|X_7WnuceU7Kzv)%E_h{PuH3)eFuUEMnXjPU~#QGvoU+SyyVh=<l3A
zXFb;$7*AV&V5`WA2RzzaPM-=lQ<K!<oUe4F@%|s-zwe9rrdsL*RCSm<=Zd)|{k`sY
z=D%m$^WO(f+Lv?Ww0hs#Q_mkOw{Q0>c=~4fM~S!hHhnn$sKdPbkIO9P#Z!&>CTA*t
z**ZZhHZ<gK-G|s|7Ud#8SIk&%d=1>k5c;MNE6w2P#eHCQ@W(W-(*9SoE!NKe%yno<
z$>IJG-zBAq^Jdh~nB}42?8DXa?b$8QDt8+z`FD}$bsyzB{FK|Gx7BRg-<pI#4*P_2
zKkmikyF{-)YIoyD`Mt$c3Jj*@u6=0daZ=!8^;%Em9anr$?*Fy!>3-=oTpH;sJnVL8
zId(d2&v=-9&-~hs;<w8M@(-voC#>A_o&P;+ZN%ycU4Ks1+r|`c%Rj#6NL=Rbl>PIc
z{o8Z?&l=^7kGCdCxA)d%MkXEpGr8V(e#Dj9*O^9N=WLcudB*%tt#`Ra3448Mo`rGx
z*Ijct_<Vj`wzqot`@pG$w)b0pR2s^z?oU+RcrBOvTJD+qS9G5K+;&corM80aMe|L$
z_-*>dCp~Zf_xZo>>cv}P8I3*@zpaezHo5)e!FhwK<h+A*Z%sev+}r3;9=-YhvF!<F
znwkg8Q+HpR+-p4d-`lT$|3{lt1{-_`<Y{5(HM!Tx{QuCK{edd6k7r%EDrjPK|EdYY
zV)Nf~R-Z1PSM|K{#%H!td!3EG?=~!|Zc@Iu{NS1`?X~%#{u;JQ_d{<R)W=`l@a1Q#
zZrQ#)MVAa~`~IxEywfb;6r0`5rK}9bOU^VrJ1Dmy(s5JC`Iqx}u04BX6?K7g3cus}
z^MWO2zs21X-v*qhK6KPU*4Js)?9UEM+7lMLvNoShV`O_(P?G)s;(5hc_gf9vUq>0~
z*}QGgUfdM(wk}*RxK4Zrx3S!x<?PEfodjP0SvS-6!hVK7zn=<CNSJ^9sEz34$4R9>
z;u#iSu{b=h{>Syhhx}*Ei`u=-UMk?qv8{%@Dc9DCR_*&NQt^EI4Ex-FPxf5e)3j&e
z*U7KauhgHenftBm(anDfHX^ZMX6g?<-+Qm`6+5A7$NNWJaT)d-EPvGfQQ8v1=T)=p
zVEWU`UQ*(g_4(_*yLg@cA$xoC<o@XY*?Zl7S8t7<_vo|#SKoyHjUNrwRp!cQ=+B$-
zf405i)5bHxf7gfnkK4tfoMd<O;DO$QFJBxsKBAGaa!z2C*T<z5r;hbsYk#hG`)qaV
zznb1a1Eu%s-*|s3@BO%I$5ET~FB5mKdTada^v=xc6R#ir{9rxx_0-=BzCQeUJ*WQ1
zZ<BiajEDCH<5tG(-X>ibuJ(7*C%$I~78hHu=O{1iE&RFt<h>7OmCt`}XIT7x|JsUb
zSMG25dB;s{4(z`p|K*8Y#`B*W<i8xx+syv$jP1{;dt2*Qr9IoN@vihg??Ha8<81RD
zPCN4GLS|_1g`jDV{%V%xzsbsvdGWT(RCIdilcl|0ucwt2UYzZEx?4t~dD^Qzm-xN<
z9RG{dKPsvB`?=|p^r^W@dpM7)+f072O00GFrr24v{?*;@XU`C;Pddt<>MFWF^ZJXO
z@A^xhuamZXaz`<pXJ@eR&a{HoX=>`rtF@N1xKF)b{6O!skJjwz%cjW8{+D*A<52GT
zd6sptkM%FJEnk0bN55h(&#vZ~?~CM5KJ>Vl`z@I*?VMbce0px*{xcRcoxkp2E%3P4
zS}32_sAHYY_9^(>M{mtm9p}z1wcBn={+4~bcwq~R)Pk$`N?8J2R=0^Lx@W!n$zQz*
zJfeHOBV%ISE!O>CeXqT<a<5sVaqsfs&v_F3Tzl3_-j}#~Tl|ffvtX}dYO<NAF5mJ~
zaw3u?x;bZ#NpP%GJ<2+R=WwR6_>w=7E?!HsbNnTKWNp^toYNxzbN9<xop<D!w%bJa
zY6eA`rnV(A%)aFQ`+LjtlW#K$4@~{beo)N(aJ-Cod}XS_(oIgUgLi3fF;TQ!xpRAU
zC8vD(2c2IY{FgSLjYyk6@z=z~o6|ST7uV0W^sv8l=C|-mtu4E6MLdhld~F&(`|p)^
z|1$1*+5Sv_#_#MsN&WNeGm#VPI8CdsU-}Vo{(6HdPn42rZkXBT+!<4zajv@+9`)p7
z=``at|6L#5*kpLyOia)B)x?YU_v_7I*zoX2{`-2<g`Q839oX?HCuXtjKIhW)r(S#S
zTa^`GEB?J?@7dnQ!lySSSU0pR<o>1p?ZbYF`^$3*AH3>f{Ul|3x{5_?YHiS71&h<=
zynkkVUu6+srg-7Toz2@CeBaFcm)Q9<=B(!-iJR&RwH`ieF>n66cRfoy$1zs+BRYp-
zZKVIFEM32=GW7Y4(~sF^ySKC7|8t?@zG>9_BbuF|oi|g>ryreWnZMj-TaDMx6>IhL
z{?=vBnYnY;?Zn$xepu?S{a+OOz;H{&dfqY(YqKwV@@pT>7fNE?$2+ZZ+VArpRI7G*
z7F>O>;*44*<MSCSWVYtAte32sDXF#jW{;k)@Plh<OJ%3-_^^UydyR!b{NXdUsuGWx
zmG~C#kXD<shVR>i-yazJD$V8@_c^AWmzgE;%;Ee77mlsF7j(1EUU<;;j3b->sWQ%W
zhkjgtmUFL?HNm4mb?N8qZ)R_K?-{VapFQvC!r6yPHYMzOGH-G0XHJKuhnhJQuh|#q
zi{<ps>1Q$66#MVosxzC7)^S-+wJmQHP~|>n=MeSy-IF?_FG<_Hnr<5ebTz4~PF}e&
z&MzXt<iV+91u?g`q@KUbp7zxK<;+PYZr^TJ2jyiR?BiQ*y2Je6?kS!NUwhl-ek$91
zLrwmJ;`u-2FJAP|{+4Dh_I=Gw#oK?b|DAQ#=JV7o?SITq%}i)7TwBN_b;F^1o3Yu0
zRcYE?^X5({eh|xa;Zf+~`hPzbRWk=~ymTwcgO7FfO?lHBCiB>@XYd<$G3~n0l>Myf
ziK)YsY_07TdaLfeuRQhnV13b+Ib!|CAAe+sIjZxWb+3HNn@|1Pf7J>WE{{HzJSnp1
z*Yl}$0YyGOf0uMm3im#6Oic5_qz|6k4g^g*o_1tT_?@+@7W@ik&|;9j^Hhw%AmerD
zT-jR>W#6s-a(4B*qjneP|7DJ`%s<PqG3NDFw(uMoKQ`@`@h`ty6)DJw?fC!dz|KNH
zv6z>>H{Y-Ke_v9&e5ZSzitx>={|&e;*`?DC@KonWZ00aJ_(E@;=?3+g+MFwX=rQwD
zZYeU%ITCCrV%Th0xr4L!Q^{vL-Nepq%u-u6?3`Bpf@`Z1TiZ`_UZ-f*>8Iw{8ZR+i
zcZHFS#qR9Sv|GWSZa*l0m9*%|`)<LKlN(Gb&kAr)@SnZ#tkl+jJZcx2EZC2oWQ)m6
zFzw;AILL5JYeLUbWtP(_w{Px$ZSKuzDYf9zF(+php4n#g*}vHjJo{L2h&9ej&hwCB
z@qz_Uwr<&L`h4xqG}i+?(J2>n&Tl(+P_%E=>wf*XX*<7uc;<e$xFYr4oK4EjKX+6o
z6u2(FaD2zPoype&E4P>ax?)@uJpEU{+VkV@A0=0PVK3Zlm>sye`O`s`ii<NOc?^sX
z3mm$)|BmNug>@NWB4^LtKI3>u_gy}-KL5X6ug<ANw=!5ho11X5Im!H(&7rq<lpLr0
zk-hF9^6PW#^WCZ%tY&MHYR@d%k-mC2Ywg1B1FJ+$`1i~zQClDwq4}k}LEbRoU%$<1
z?sq4wx4-#%#GK*q4^6&TtM4Y*n4JtR<Yrc3i)u(`@W^;Rk7xUZ<87Ohn%S?+KDFPr
zKI-3Fv&2TTW5<)i=dQlHbpKQlksPg>6$Js8=lSl>`nGxXoQ#%w{p%w4D%6B5TxZ=}
z>S+30HuC@9xL+5We#JaUoczA$!QC73#R3XGM&U+F0(L)hT3R}N{^GZtd!}s&V0&wE
ze>Q8k=*yZV9Zn$!>ja;DvUDp>Q0tu;$F@j|Ij2y-y!~G0gkyECLOq5jT(}QRJ==KC
zrplzi$(T>Jq_$ehV>9QO%_n8M)b?u}`lE5Sna%!!st@06u_fwd&Q1#7RvuIj@8&QK
zNsMky-<Zx)H+^RP<xQUqcAYN{&`DeR{XFyfr&HONFSFV5>P&5T+p-7Gr+qrKa9-~3
zeC>GYF3p|xPdk1IGYMZ_msK6H*^syX%#WG2E?lb%Zd^KaY{t??MvoivUzXO}u{@4=
zBO`V+!tAr*>@$3uUcNsw(<rW@<XOs-t~bvc%#Z%GEDE3d^MsC6@{IXuw`{h?J}o;F
z+;`ggnL$W6TacFafuutJ=ci{r)bHzNKQ8g|k8*7I+gX3-F-sY0w`Oy$7O?6lb5Ph~
zy@6}LxHR+QQ=28jPHvQWaOK|rEGOZZ#rA)gpD;YAoX=bP%=hcD_CE)`92Dk>HV8OL
z7K-1y&^LdE`a340#?wvg&sJWLH{jpU^?BJM!v>`o?+DgsT-T+FTjvX@G4ff?dtMRU
zV&}r3_`Q&cyK{e%vx7qQg9iuNuBV;xuQr#8_{uN#y7o-9{(7f>y!UQ8TPd7+yfTMT
zB#~cA;_=HBHWEySS=1XAPhI>nTsfju?98;AoNb<Gf*HdM=QI~CoA!i1Hh+$daZDnA
zAGl4J&>MJAtBGZn-Rzpvx9XDY)|skRO|lWbaOm@vheA)y7wlDL5PiP<6DyO!gXKIw
zC)>p`Ry=83AN%=8zWdDOSBz!P8`U`Zr?Z1QkV3OQNip&lykIfzG+&n*_wm&h*=_&-
zUJ7AhToH2aRK_jo<AU+lJK8PZ%|6Wc^82B!&;R7K1f066ez$m+x-yT#MW)Xi^bW`=
z7OwmGzkTf!K~|N?^|ub>Tx^<K`0wt7JA!8!|4YqKbxT-x_gKV}$h3qajf5F@bT_0s
zX;0l-SpTo?P{Zoj>u>9-uSe=|CM4eqls<lT$=poo*)^|C_P((;+G+Y<$70Q=yHga_
z)wW;dH>uk8Q8`!7W>q_z-<b<c#^?9tl!tzE{n06Aci^P*rWUtL#x@6T@3Oz%7ysYt
z&*^!eEY>wj9^Wi@Wuba?hsd`NH5ZuV+RB*~7~gxhoje(Mqq)ZYndGV6HtcfO<;qSm
z@2WXnwEgM153el#a~(5&GEb=LDv!pk=p_v?Uz7iAW@Z2C$#*Jp<%xc$bE4~1KNK+O
z-2NT!oAHQ+B~H>~<(dQj?F+p={u9|feg2`kM|ug9=IDOfJ>zAPhN`Pt!kwn<HXj+8
z69?OQwwow+dE9T+&Urua)lbps26wbTvtWO!8=}v+Gx9BLeK_s$a&Vhs=O4XahwUG5
zF|U1MEPPC5zWVcH0bFttc^m)ud`S!5f7CK<?#uZ1_18t0xU+}xDNQ=i_B*4!^w08H
zzZ3F)GJoHCvzX_+@Oh?7Q{L%V75&fMVRk+5(D^&MKV4nT3%Sov`1|<5=>YZos<_pA
zUVjMoib+u0(Oq^x*XH!`j4$cuxa=<qU*FVkvm{CPw=maxoyc>><$1zCp6ot%zU*Lr
zt)C3<pU<&Bc|AC}o?YHBIis*Ku_102=V6AI@zSTC_U=FFyU)0({Ij|8escykU4?){
zy6;ZP{fIhec_u33>bbzF47zFWHXOgwXWqwSm0N$~K=4N?)jLMN1H%_`cxRNKee8AE
z@y;K2_OSO?qGTpNZ2fY3?LTdGiRA{iy$W7Z&P+e1NJhTWI&;k649~F^wS}KqyOygQ
zel}~r63Zi-r5~d7KvmD37|Hi+&K8`{Wqzm}m^Y_*#)bp(xBgA|>^ymY-pvixuUFSe
zSN*Ga#O-cd{N%{uLp8I^61^>$*6rN&XZ;q-m4B_~)x6p^b^Uw0U8`39Vp_+sPWR-h
zck%y%zwX*17uw6c@B7^uyK~>_9-n_Jr`Snd<7j3-li0b4FP9{Z_f052Z&QAt$M|<?
z-G=CcCl7CFaGw3ZxnXsty07;9Z@*Gw-DRpT{(gJ^&4%wMPVewPn_|qj@1&1b8_Tl`
z`)09z{b+u8v9(?Hl*wwv^A~Zh*gWyVhLdb-UMHN`wRu+U%sJ+&%uJFuKE(2VD5(Au
z_%Z+Z_4QV2Ausp(^mQ*da%%0q_~c~<CnkE;@@%}({H8NoGxcG^>S!DB(D(i^Z{AO>
zu)FnI&mtgs#hk+A)61vyKT>dLP;H+rBqVNhepMP<MCpU}OSVsi1><)7JG6EF+W&{`
zo_iUKsZD!oTE67nyFJe>uk3W{j#9fCaVeZzc}t)y%bHuYQDv^@INk4E`q<w0exbU<
z(R01Gy|(2%$`_TI)V?);>YZ(|&D&;XW}fT1V6UyXI$^bUqwxLc*>bs7TV(a0bFN_6
zFy*UbSY_BYnblui*9u3Od*)05jZNP$WGP5Hm%C%@%dC^$8Cp#;2Fuu3x|~1k+EcoH
zef7QnQAvfTa&z;tP5*8P+S#t7bMXDs19vWQpL*{jJL5~?p~YsOzVDc3nYl+VPM`gA
zhUvaDb7q`*X>`8D-SE2kWuu8_9GjoLQ7d(Ix*hO}{Zn;$>+HPmzFga$ow52R>=t!5
zN!Lt!YGPNKto`u=8*Z|Pb~W$QesOW$$<BbRJ6XC(R}+m6zA0X+Bx<<5(n~O9x?^XM
zm6Ff0nTvRT%<ui6^R0(rxr+0@Gh2V8l<Mfl81z(ho_Jhzws*ne*ZI9Q8_v~cmS0|@
zC;#um!4LfL|DJ61pI*N$;d9Gb@%%zr_P6}&^`3b}?0+1!Prf{K`o^>Wy6!UnTeZkH
zr|8XxJGPuhdE@LR-22sY`4nG;b{gCK$CbNSB+`C#KYnz&IW?YL@1*xB=`H{D9acW9
z)&EuSLsZ?a{L(Ms>iysE?vCGQpX+>T7JJyfGjcQTPV!;pak_fu+0<OnM6d0Kz>~{9
zEw8vg!`ygt{jG>ek2l>+{QbcB*+GV!3N?$M=J|;?4qyMX#;m@#wB%safo<B%no~vj
zCJ4*St*l>Sp0}NIUI??x)Sss-z0!itxo5A}{kF2~Y~1X?XTFUunF^Awbu;ojkC~Y$
zYI9^O=e&?w<921GYOd=iRvj{R;A_^<&Gu%}Y73PyIguk;wB0E0%W{@!pXMy)<V(r(
zYGXa+qm{OTVcLOm%OEk(pt<}bsSRC{27Et@TW?76+A^*vi2daIQ(ATYqZ8+BQzx|D
zEbw?bpYQf%qYpJlFQ-<YZkra@KA*Ljo9)T=#|oaj3WW+X@x=lQYPfWDm=1n^GEZ`|
z@w3!CrZtCRWm5mm+PBd4TD)fY^7rT7bb7_S?aP{c>s9m?1{XtTR*ANJp1;x4x1Y<{
z%rZf4frs4jjfXXs^_2A-O8$74ZMO04W%(b#^FmL`kE{_ttd(~>dIDqPnJUYgxf3&=
z*d<(gQoiZ=aogsy`CYaT%MV(gv6nO|v_3z-oX4Kqi2D=wx(T1BpJ_W5Y-t&FWP_o<
z>i*o`osYy~n%_Q5u+O!>$iAVA+cUfNNyNE7jC@OVtXa7hG2U-G#y!*d$CGtguP?pJ
z`g-R~z>!c{y{;T9_G1AYGv73q@om|h*vPQw;~L&IMQdj<7fRX$7|c6+&ZR2clyjFz
zpg`M{Klv3iEX6?sPCNdwILm#QYE=$ibC$w3-R!&pljxTAKkFpto%~?`Q)hit>N>Gy
zIy|~}PQH~qYutBo$Ewr6^y*80uKFB5_wU@_i@W-^-Hu7ldN0jcUuL)TQNNqJoL$A$
zZL;q}_xIS%jo!LL<yqIg)6pgC*WB8*dWBNnt5w0<<e%DVez!coQFO8QF_Eob?)KT8
zOn*CXkJ7rkKVJrSUzs$k-ZaiUKln~}?!4X4XD;Rc_u)Y~zYXtIN15sKUl{%T_Io|c
z1*?_6JLmdm{r)+3e!TJTjiuk$&)$2EbH>Yx`r`*>+xfrp*T1<WEG-|mKd$K9AzAKa
zIyOH)KeyZezx)2h<zMsH9^3k6TiKUeqV@Lm&h4+a$8VN8b9Ihuy#B1>SBHwlzx^w;
z{Jv+!Q&<1ED`LilWo{Rjt)AI7Z`ot<zrmCK8+u#csMs)*^L<0*yrA9Z>y}Eji@nq^
z4esw*<{v9v9FQ^5CuGKQa28#7Nd4*KrT)1w&(}u2U^!Lb@@$GptfNevp?2?bCZ9)L
zp+(!a9u3R9)i7z{q#&^%=?kXLim8Fd$v;c^SY5bRvu@kD%Xh!oR31Ug69N->w9nSq
z6em4<EO7Y2#t4^|hbBDr>t6jXyPy55*R6Z+toc*ox9=>wC%9GQte(xX2M6zbnR9aY
z?WXu!wUysK?VVXU{oQ`s_r8%2B>u0A+nezAWqtj{M>Eql`sOj&IC~%4Df!!I+t%o7
z%c56D@32=$b*pnrJ9c^x^N!u6AFq9lj&tOG_F&J%y{XRLvVQt6H*DGZb>0o@)YJX1
zy&v!MmYrIrbnNf%oBP?<?G4!X^h@YWe)AveHWdG}-MM`6(nj;tv`^l=>$5NJ2(7JP
z%V93=czkr(7TxtZr`OCAm5KhVaLdrVednwzuX+z3OG!;lP5m3UVbg+L2ShhAU3K3O
zdgei6{o%X0Yd_y#^^Sk4N8PrU-+i5Z!%gbe74AOv{Movb=k-1r8#Ff_JUpjcUEJ*S
z|B2zZ-fUI7EA_OvR5{Ed<;5c*zlx)uV={Jr{$4aO_8b4Aw=a~7=I8v7Ir%m78-G-d
zj+nl?NRJrzqAOcHIxAgcs!!dyKaaKOjM5IjwPK;GRWnj|7UbrgIk4MBStOF%ce{bq
zMy{AP&DJwpFQ)jfTc3J`|B8jub3I0t$WIa5vR5_x)hoYVdi5fga<gNt%(JeWY`ON_
zWuj~JDl$)AQB~J}`Rm{}{=B~lr)NoD^NU{GmHFs~E@MQ}=^f%W_tp1L+-CcuVzvGy
z>*{$dYYyG*`StRBs~`7gGxc!Y&vvW+E|oX--=px(_rCPGoP6CKyEfP}IGiiB-n{wJ
zX|buYasOhDZI|RWPO6Ljb$eC(_BFG2{`>p2y}vHujPl~J_`<_Go?PT**fMR--^bVT
zqZ@0P)>v&Uo!)o+-JZlJy!o;Bmw)_lX1;Nncztxu*+#FP*8flHt<t}l7q5T!*Y$m?
z?e4<!wz=Q_&U*i+SBAah<X(evmCaR8yq4xzt-gBmPuU3>^~_Hl`P0fS*nF5{+Sabv
zyT)YcSz%koZxKl@x8-w7%@$`gv_G7AHqYh$@^!vW3MXAGt51J;rg@&-@N0p>v`gYL
zdL3eHx;xzDSNzGGaEx_z-p81bz4Ct*m-ks_Z0P!_k+xA`b6;5uv(ZV*1#V*NB*Pdp
zT<51hJ+(oIM>KqWS8V9lsHZt?(Pz9oBbJ^Fxgn7rttc`fy5gLy+=~}4r(F3rXIuJq
zdolajN6l~F&JnI#wDfVJ?EKe1%%l2dRb4si-pg)t;Cx!}?U<sNxZB%u<8!Wm>uB%$
z@uFj~)%h1Sp3~){Qny#FzFO8`-g8t)a25~4_bZ3?&QP3Ko8tY|Tl8qE{Kj)C3t0Eb
zpIrV>s$9~CDeUe3l>2Wj=db;f_H^FV=Xq9juRrDbGv-)q@RBKG6qFV*+qe6s*cnCz
zUZ3!Ungu!r+yAxvy8X(&z4U2Ym;3Y>(c;h=uSLu&1l%|p7=JXD9at6i+4}9PGW!MG
z^K@mOSeh03Ui}o%qu;97WjXba{rtY=SB!1HEc@Pg!&HLX{^RauOXZMF|E#~tt+(ge
z%@XmnrzrT_@7I@&ccmOjdE;pPY}@0uyP-GziyvtG6KUH0qSI2mCh)?B1)_(HD)+97
zRn<7uSQb?$^8880&6snsk9MdmIoQ}g|I!R?-V+CzzhB>>^ZbU^6YVEjF`I2K@4xqd
z-I;BZj0@{#Pmxhy+b=VH{#6yjGjF<V7eAfhx^IGr?e&PNm+_yXp3S^{!0Fr{!_6k^
z{>=(#FZ`F0Xvp((it<kdZ@=!nsnepL#hi={b^guKW@+tkN&M5=lefbBIOH3ZX4b1-
zf0PovQQlBHJYvDLYx=qc`3Z#&PPc#-Y5sqhz<W4x*O#rIcJt@>{I&R2lwY;opYOGR
zHUGkf%GgzXoW76#HAL^Iez$6|e7`1W{XoStQCsa@Y`^yZygWNNJ?_hqKk2`GyenQl
zoW8FjGye6QYty&<TJktnR;?^&>%Wbq;%nY>oVc<y`1<MHdx}%~led@b`*Y-8Y;>7i
z=kD85IiGW*S?AYIo3k_JK=0cAJmW22%hkWXsSSUBI_R87rK0INrOm(IO%OIJx$=r{
z{g&7dw@dez{m6Z|KDN&G)A#7z^Qw}jPkvrjvEiw+Z`)@^TOU=8Q`wW81Z7&D6<Bxu
zle5=;Z<Q>lpTalaSo?>eh++HZk8>*DYK1&IU^#2~9o~1Hve&+~Ki7<Y5>?S=W^n1u
z1KC**>xFa{KH0u4`HZ{eYbA+e*F~QnG5LB||2UI}GvCxH6Ax^Cay%ku?nB9YroWnB
zJKUJp658N%=iJs+0eep+uD*6$r>lBe|2c!@_D>@Zs;^u8eEkBJ6NiK*?R;dX!F5Jw
z8rvm-6YW8ViemVFJ}#4V=&+k4pWN-+z4M0op*=w>HBK0D{+M}3Yhmr3izlR<<XN^c
z`h41Kap82+Dk*l=-qdTU6|-5A8MLATGd0~A|MTsxSXLjAWEpiJY^&_0f0z5zmFDmp
zFvy&1Gu~W1{p+O2^bed4O$JT}gzm6DVVHfypHZO5``O}`{L5pwB-j?TTAFzHbbXa!
z?c=uHyMDUtyzi4@G!x_+RQEZrk<I?n$K|`qimO5KYtt6@%j=k$pM3Cm^j%xi{NcMD
zzkbLb`NxnVc80&?%T((}zhD1Ne*AZa^bOs#S0C1My?dnH_4e6XKC#8>6}k=nYh-I9
z@)+wr=+4|Pc`jyBrPrPI)$J$W2eV0j+H9oUJwH{id%wTt1IEb>0jnH8X;s8V<xOR4
zS+etnJ!9%k>HAYRT)QbBc<u%(Lys8aVpX2qXa5HsG|;O{QdeD&bZtf3!tzr!1!vi7
z)oc7VILr}G*>9hbxRWFO>Jh$;x8{90`IhJN56x%SVt&;zg(Y4M+IpVzXYdwA(`je6
z>U~mpo)9;eN4Ce|{>qPQj(s<KcX#)W?c2X6I?Ks^I`qKWWFfbi*z$Y-OCNHaxBjv7
z?VUrR^P;<>r|o&H>#FepvcUdMd|=U@?}=aBt$!_Mb!_{jao+vF5BbO8Kb|MgV}8!J
zBlkhjh9Hjf$!ce-YY$m_)_=<Jw)-4A=LN&g9d8dF+GKEiV$1Qr&s9u3X3hRmK8bm?
zz$f!plLMvQLlf@g%BS<*_@(D?_m%YZvv)UGU1HwFW54+#149FYr;B6AnNA~SMc=!9
z?;myWD;#uiP4+f9*IxT5s>s?*X4W0yx0-E1JlhMxCtsX?@5#RpXP(L(Qcq8;Tyyqo
zt=yzfqDF64h5r8XZfW;U*L}5hTs%U|a&=uZ<@Mxy_8UGnXxSZgP56H4A(!pH>Uw7$
zDqUyN$d$*Qaw&wx@&Nzani4npYiukRm{cAYzEzyI_<?%><I+aU<sH>lH^ZECCpd~U
z8?r_DiHXe<tqcnakv;aX<_^0+_d$tcUe(qgbLtI#<zEtwTO_eVB)z1ybCRx%(hAP?
z*$bQ3N$v{?jQJ+GdCEDyVD8_?7c$QAKVxwJwc+=LtUv7=JQ<=mG>>e5ep-9>rtdPj
z>>qSm<OQnQC)hCPvS=()R|w8*{&aF`<++^j>c>v`k7x1UnXz!Ou*ZU<m8)iNw~9EU
zTO;#aU-G{Cz6nh&O|h&~qwVfj+}m~kLZ6N8JZ9;-W{CyM?#S%7GKu?Ea<Zah=RpS9
zgJFygP35fu9NHR2x0}t+B`EM5cRE<QZhgmsDVHZ3JD;eR;0Sr9`PC;~;i1q?b+xmJ
z0V|!v*GHYIIH7-&F-<^6amMpg+k`$(JXU|j!T%cPlX*NzTV-@hxAz{)=@(m~zEFJK
zTSvZc?8TR-yf)D@?>Utu!qETxurANHe~Qn!s-rhIIE2(3l9^yW^=-ZWzKv@04*phl
z^qyfknJZ~V(!NKLD|23Q{Y=*Le!)^y|6Khwi%_t7Gh3)V%Z)V$LZ5fWmX-d^Jn?0r
zZTphB8(vO1UUfhF+_p<=E<DfO|9{ye_0@4-Q$qT5e;)k(RDb$fxk%g6o6XVI;j-H`
zN*`ZZweNG<`W0(-$vb4{@7?q^^!#Fbn<dYVs8#h(KAv64@}{8d_)pnB>vdmtY<V1}
zsNVJb&a0+tpKtX<nC7N_7G3TbB5QBXA6b3W>v`VhC1xvSr7Jc$c`kGpaO-e5*!o`o
z)}O~S_HL}6f7;U2&OU9^_Vd%^x5du+{JClJe#^>xSHhU1?_T(|^5I*(s;VD(JzKx-
zT+@4d-o~5q&&+=;IIqx>ILZ5$Q_HKu73$m}7LFxM7B|D67TFn1DofyBr(<-|)KGs(
z%d1QyiAkQ5Yh>SZHtg+vV*7ql%Z@XzZ@oXiP~d_1%8f^MKH0j2@rg&MLyWNl)0CtO
zM<sn<tX%nXW9Fjg4EdA%CtToY+dr3C=8$?+O2+&<>@RhLnDtc_iuia%?ohckwJMCk
zFKV7BL&MDSqA+*KFq`O@#j(%gAD)jhyRagSO^o|gi)Vc0KOJi?Vbf`oL}k`3Yvu5I
zv2<fx>uRUzOSV20`=Iq>p5UvtJvwK&bQzc*_%Os49C6oB$bM?LTDh6$qH%Tbx8$40
z4;eG?{AzGddckyp<(<Zvb$or^-_(wkuC!T`Bb*@de%qd7S_b|N!W(jno-;ldn9+V;
zp~a}FamuOrH$wk>ORQ&*-6(b<HsS4w0%z-IZGktW8Qjj@xU=nNd6V|xbLnSJWz63>
z+r6CWvkRNZ^Y~UqiJ0WxN_EDiTe$pYU0YOhs>@keSLocwgBLB|UDn!iG4+OGl*F5j
z8XYW4-$<U>Hl;tkjmv)K#Ufw7uu!8<1_7om=Kq43z8#;rcIz6;^D~)GN+j$ss@zb0
zpT&yB?Z$?zbN{yU*gZI1^r|zH&7#pN9K5GRWv4Q4)-4C?>p4gNZFtgL>2faq(Z3g(
z4_7>%$$H`GZoe-om1+rB|FrXjSH?HF&3eIlK9VVI=4IB&LgMSR;`qbMSAJ`@<XJ5C
zV!z5m)q3_1Apy6oi_hGA&lA_ZV26jcPNCt=8^^Wt*MI-D?a?B~(E9(#E3yyVJ$1P2
z-Ri8!lAM3-q5SLTZn58;<$v?rscf6S1?Bg<@7b=@{Fr;`@7Fwoda<wGuj@nKXMFIE
zntxCHyVaiO(h<r(RR82$U4GSf?e|A|KUOr}Wy_QNV3DFFm;I~anXr87{7ui7-u=GK
zYU$O;wYz==CH+>dn0&9J+WJe)x%QvS#dzB0r{Aked$O5(R@^0%l_zIS4x6{-TZ_7>
zpXa-&S6&ytydHQ%`ni<le52k&H)DACV{SFDh*r2r-#dBql|abk4FTI`ALcmwb!Ey{
zy>FT4mKbIKI=R40w;}j*+Gpv+`tW^=n=|LwK3y}{Xbr=(8MRHBuj<c)oosOU*LGIG
zdzvU;?0Z=$s|BeMnL+O_NVE7PX>zaptsbfKP1tvys5omIV;G~8#Xr|`qRo2ePBRru
zTGfzqCMGkhQOM%tsc@l&tJ5CZPy6@wU-sizg|#{5FY{Gd?>t)NP`a#s@Bh07Kc%Pm
zA3fkHyfI;Zg_qNsD*a#kSG+l<aGx{$_p;Qhh8tyWwdZIYkZ$>)I9Y44&EiV8m`j&^
z_1*<^{Cm;4{KH<yr{{l09oaQ!&gzq`XFt{7*^>6UF*p4H^K<cq>I%X)%vLsWG~Z#3
zFm+FhH3)AoJR$7dA$vfP+47CSzQmTFj0z7m*YIr+@K1EHxviyLoYM2Tn&teCV^Ioi
zjiyc0ovoYw#VurHroZt`apSf#EdEd(F|pX-(WUc`-Wum{EEe{#SLG<&p%WR{<-X8v
z-5WcupOYoSqPlq$j%d`dq%7;R)U!;NEYrw#$aHN{)tGv|O|xF-jNOk5k*$*+A6d|p
ze!{WyR>YaQf_;<C*zNjGH#ity*z{;CcrM8=^7+SkulL+F@S34uyHsUkCEvL}2ZJ}Z
zSNQXv->uiMu;ol^;B8?8w|l8}r#|Y`mCHCBtv_+VbYn!~3~fuLM!nwGlUKFvvHGVa
z(POE2Jb?Sj{tJtvL=JG?n3tht^e|&b!tHyZ?siY(t0Vfi**7lt0&h9k^fNO5@}eV$
z-n`ox+jiZ)^!|=D)&KLVfARK8<<vQMMSeE43W&4O{BJIu$Gci@&W!Rq^DHwCu3EZD
zUPW7B(!`G`$GTsht7rSesmY%*ufy5WEj_=pGjp%hx@lW?p4mF@%+;koBKL^583r9Z
z|LpEb+mx?Pxz8pwKl+*Ro<ldN(tnlOryWJ2X4&g+&datwJV{Y+Az#uKzOx_fB@%?X
zKA&nhu*>X#*2%Dw^VVo<td4b@7CTKM?W;X+(!%A-tTY;@ttoDuHsk5O1>c)JerkuY
zPDu>e!FI>VPV}%=q>)6OiK+uz_?f(oo_Foj?b6%JD%zPJFou1M{V8(HY3}OA$?WT7
ze=I+<YLoeM{d%SEoX7wE)#Z-AVZX1ka`Liv+0#3Nq7>#wa!z<1c{7{m(B?I*OV8e%
zYPl@r>7!qM&*s_4UlT34=ePLC`APF8*IqajAgrsinQwCEZ?k1@n`Of7jxv@S9p^EK
z-u5}Ca^AUHnl|S(n{@86I{U7T*t}D0T9j_rKCwvoo`*7NCl>DzxtifJZ@I7h+uze`
zV?J+RX`<@GY$Ic`;_LL(DNE&xGZt+zkcn-%cj(LnhPL3xuYJ#dnjq>ezp_e^@7;`@
zXE<HDL-eQk^evNlk>&Y)&byNZJB$Kf`pt?o^j~-D=QGjReFnE`XC<)JfBv%ju;}xc
zGgFl|uj|vR&{`>`@!e7SVr6hp*WH|na!MEfpO;MC_sn<J-g}Q8Cf|CirFpiqccc8q
zdyy~SSLvO#uQ~ls^t)VjYEnUX*;~29RKdiVrHj9s$bZj$uwu&3ZS9-BUAi~P_fpGI
z(*~8>OVbVVgI3JF-VtDH$$oZ!i}M+mnb-3^yPc_C@z%|xEaQ;DG+*~_wqrZge<}Pv
z5wr1>&90f*hdMOG&Ma-WY~b<{IV6><Am(#zf77=GXO<s|%@AVK`%-ecVvXKCCT0W9
z)hQ?Dhq*{x@MsRrX>hv}oV+x}I%+`!yXLoDY~if0xwN>$6nOaG);O3qFP+6`P?;Ot
zc4KRP$&zVx6{0(){kwVQc<S@WM}G_MJWrmS!e23O(*lF<&+Tm1{hWJC=C!hETT0Bk
zYd@o=Y&ubMeQNm<&&l1k&&$GRCMaev<eck$k$dfmjaJiKGn2!uWjii5HSgyB!_ir0
zxyW1SE>p(IdAGDw`cvdw`#REpHk_UyeP$brR!&X0lbwU6vCpI3Qcn!*Tg3|UVpJT?
zO<39-^R(PF_|)%HLc4MjBh-BR;y)xOmN}erOuqV}x~x%S#*agVx{YFoRhIY9=ns1^
zLtS)b#_rGiBw}Xv*M3@Ov&YJr%kJUs^l#7P?9~2!yAnNXO@Zax3lDx=V4nAN=jlC@
zZQNE@{5cSOGgh^jwR?-zgqi9S?NlGEjAT`xz0*@+q2mINoi4L~UdbuRHQ!R}X3eb2
znfF#Z_+;fQN%h&WU0=?#PYmPfuDg>lZ<Bf5b)klXDWW=Sm#6VgzGQPS6tZNjQKamo
zOJh9Ko<=i;d9EFThqj#Ax;E+8k2co$2U-yy7JQ6~^8Uj1jO%G_<T9pm$)v{9lWNag
zUCppD$7szBhAk@5sqZ)ore0UJPAvC1&;6U%R=H`vq4(pULwEFE>ts6S8L(&Mevg-~
z7dDnSQ~F_R_wfbFyLY;Y^}lEma5{VD>zlIsOA@k+t$#m!Jl&>i=5OT<>OKo2`j3B}
zujO__Rq}({{7Zjs*sP8`TQh6^ru=fQmtC_$cy7e7dljo{Sn+vrs8`>5!+G9?$<lJ#
zW}S<N?%Vu+Xy003eLn18Ue@XjGyCsI{*+#IcEa^TI;#EWYG?eMvj2C^9<i6~%Qjls
zO#7M5zj?Lk>M5>=go?IJs^7N$Lh8ANM&TVszqfo}=4in=FLpxzRGz7kt%)a(1Yg>`
z=EBmmlhY*(i;pf7ojFzar}*=Htru|@7pN>vaL(MbJoQt~@4!iIN6voNIrn1T9))f4
zCM(}OjQ^tDb;bDm6r=kSyJRvHx(;?esa(R|>%MB{i>703HyO-Me*NgPNONH}+Z;2Q
zDJ@M>96cp@?VDV=TDB}Vsq;J}Va$@m|4DP&^anNkk_8#UVjW^9BR2-@KAY!qE8cjv
z(Ln~Y31R6Mj{ceYw<B#HS89FgIZ>Wl*$F2F#4J|STv*-g=DwMEVw1Kc!?EKWy-O1(
zw;Nx-sC($s5t+2dKkmPqEt1o_-|g}G(>!%s%08P`Kij&(HnW#!#hm3j2|asOzg=i@
zGu}?sedGVME$M3;#Qx5Wb`lS-50tvmW_UTFtukM-;)9IJkxA{PoPjF$H5|p;J3r+w
zViSpF>^+q);B&|z{Kc$IZujbW<vcdOKl$G2w(#cp25UN6=kTw;dsIG3+<n^pwm)oV
zinj%R3-%VuejPM#>&MJzJ5++7oS9#tp8nspVqO$q1oyYi8RZ9cm)2}peQv6Wrj72>
z^QrH6<tNHk_Jpsh*N?rlEr0cQ>G`)l-cXKzX|OH%_%eI--?x1%*DOkWpS(=&U&Z78
zC;t1YE7nibmU>^bX@Ad$V{(5o%Fix(a;g8ysl$SE`ajL`w-nC1`0mrXO+Q$)A8&jo
z&z-PA#9E<6BxJ&@x?5YlB@QzcwFIiETy>8NKkK3pb8t>q&aR)_6M3Yc9dlAPd%A6_
z-M333-;OR=Y_1t|@K^%dAEB&7QU8?|$9{8tZ<LQ*%Ek6SDpxz^F!KZ6(`$^2Ssjk%
zgq}?hozBdr(G+97D%^7h%a4}q|L*givX&Qgv#pNMx_jr)B+(#Ufel<WA`-d>w6=IZ
zb<oUM=o!PHaI)-x?hlqDNpl^(G_EO&t(k1`>+}VSXO9Y6jvi!k=amtdD<jRg`o{;A
z{s%(mJU8adv{u>5K6k>rxBEN~Ub{2@;bh~y(+`;U>4=6MRFBP`cX3^#VP(g43Hx5b
zB2F`n@|*m>G8lU5i|TnZJna)teUhrp`nxlh`|V|~cbgvGe*B&{e*GV&vkr$=wr&M4
zy$$!(nq}N9v*q*!-{t>qOnD`D)!cKL$GP~^-H-3<d}}V;DXYt~eTR_9|D7%0W%e)s
zqjT-V?jt=D?xeqK-hJy_`^ApMKfPbPo)jwZdE#%Gm0ntZ6t<lI$}7S8$aH6Aabxl{
z$MlG&lN8iXG}=A0_s;y_>@|7!H?PJlDSkCa@js`v;?7ldHeY(S_$=e=(BBhYEnr){
zqsCw#yJ(DOsdZ#zNA;(|fV8_7p8vfJBQw7>9J!ZrGxBAmwZQ4a>(2atsI+a(p5xU@
zwjb{7aQ$Vz`t!$KJNV_lJe)k0Inp3@pY_|=&mCr8cc;kzEWKGTU8G+5ea@n*PLnSz
z-Qn?m*8TNrKIfHB{XMw(*7x(Dw>%HwU)Hmm=T&jqZSWGSXUisBoO0#P?Y(<iw39sl
zab4$`zl4SLupDRO5A~X+$Rx>BjjtP-7KYhLICK=RPAO0j7b&}^)4AEWMk_<E!t|w&
z^t>+BE2-QLQ<$f<Xeos6aH<z~4t+U0{4Tfu#&`42EKHhwv`JFoW2~OcnKPnM5|dm{
zO}#16mt^$0E4=G}!sF#Ot3EkOvb+D>9`_~v8|VMdr`dK9*2$dzPAxU}NI0p=b-ux!
zjh8E+W$R}RKfOqBija|#m}lAk*y()vJcEa==OVZQE;@A7C_gK3?A0`C|8sKQw=+{E
z?K1zy8b4I<Z{gU)zEJ5zfm8E>O-H9Lb$k0mZL8faQ+w6@1!pGwk2tVNb<%uCZsVCe
z2214SraNl(pZk~i;n*5(w<&Y9imh+SNCbEuYkRKjIGshV$oxsojsBR-C2Fk^2^O2&
zqgcDUU!7t)Ui0?PY=;0x;k6rQHZvEsDC}On=Y8?j?IGsVs+X?h-gIg{&#uQx=Zfri
z_TLG$oTjba->`bMhJm)&Canc28$G7U-tRdVBgx++mEM|dr*XSRCaP8SrjYLm`RT^j
zu4zk3h$}pF-g$yGiQ^yxo9-tIJxQUwiCg!ioRE3XFaK_5|7y3%E3T+$TN~=6WGuY8
ze3Rye({uWL8<x(LEppIV>vYI<XZxM@M(zFg4Suzs3O~roW}x(b=NYGv@@MgrBHJ0a
zWW)%6`BlKnaO}`)hNwGDb{wxJ-%Gin$gF-qqT<1!BRp3b&M&yjzK!ecrca{II&1p8
z**<UeP0aW)-9~i#joB>TQSuDS6JiP+&L~Zfg3#%C8g4!{eD_QDzW%xJ<G$>OG+Bq%
z=mP-@?H#7C$!g8sx4JN?G;vRmdrPcLVk(<p52M8Yz48B^&ig;_EQ5b&uX1wT$(Ndr
zeXUgz%$9P3*J}h)6rO)c>_~l5785`Bkj}>%i6qO{OT~&k79PvU{Jin}k4-zX?;G>X
zcFDh~7VZ_7{cy^&1drc$58s}6M{x6wPY(U}MclVV3og9)Lo%0Nb-^T_g)D(K@-N$s
z)g_-4a4fnb`EveY)57VJT9&)IzwfY2D?D@goP4dwTQBL$T8@Re+HAV(+E-o^ey@1C
ze)huF&$-PH!L8s4TX)X+?mg+Fke%iG)(V4lzmCSw3e)#K+q_6@-Lj+i=Xra~jf#+q
zHA)crbhE`SOR@f3uHvC{4%$5;*7Lg5ejm*#eJ=U!{i=;0Uu-&(YtqDSYFsQf)3RUV
zSB<Xau3q-pe>7(EXzT9pF21^@JUveQynw^&{Tov@n5<-3;i&&sGe$Y5Us1q^``+V?
z4C_|y2Ca-0sNtBlW=sCdV>>JxA{q*&M%V5?<})#|an9eTPXfMj=xF_B$(g#1t1e7T
z`Pvz^+g+?$JMKPQ=~2^Uyhfp1G4*D0c<A0R%iTMoLmj^^pVQ4ce@FN73Z2=fLN@U~
zI~LIO+hgm2Tc>t1wI4X(_AF-M+=YV1u7AYyB(07f*O^&-Xr}S3Zwo%BE!|*k7{2K!
zV~B;E<4yk-?p5&<8Imo!7P_pQ{9X0ajyc&(bJ9AFIw^cJ5`DHt{pPxvdsnpGOkctz
zZM6E-)A)|Kj7w5cYLg<TI8}=q9rt`@J#&BA$_@=?*4%Y(pPFA#GTyqS#$(^<37+R%
z-fcJ|Fu8d0@3t~sZJ)_UCItAVO?0(lpKhMKt^aeq%QV5+#<#!z+xSt>vrv6U)T>XL
z%UX|p+^=;f_+0S1Lu+oSyQ*z5&06%lLd>|tQEj3B3ePRK*L$1`?mDr%Xx)>K;mIC3
zKHM88C*OQN?Z)>t))A?3;_deqwjMrwCeriFoDW`St@RJzPLY>%VNcd||1EprRdA;D
z;@-S8LDOfG!?V)PJjpx#$Nc9T`;;%5Khw^<`McXE<jb)?lX%xOZQQ_OW3Xn@&ESmV
zrbas$X6`s7`p0Ca_TN=Min(u3*+2J!cmS8_vJ3KxO+p(A-%UFlCb-6l`w;8Xs0Z@b
zA6%Z0xODB_G$pYZyBXfhySrGO*F1W2f$x2R`2DMv-#+a)cqVe^DX#mXm+k-AS}L)B
zD*Pc|q0U=nxW;JxiR%6PHB6J_`gJeuVLf*usgU~sv*d&4v(|DjJy|ZXs<1*X{&e#W
z=C=o=0y`8G1lnH}PFQ%jWTO9cmr~cKzdjwA^lxh2`)Pk054}^ppKqbLd)1oHe^PJn
zUs?6@_Umo-K{Jht@3ombHBalmb9vdDI`Qt-1&*!9C%4_boFFmVa<x|*$0LC=$GtYQ
zpH<&}V&fLk%z2e+F-$Y=Gd+`8wqpa+#*_nc->)yx+0oC!^G5RR8~0|tH|~eDW+tj0
zK5P=Pw9#ST>Bi!;h^K$_CNG<B>FmSy{^Jg*l82Fn1`9=QZ&>+O?fa2G`P28Dt_-N@
zn4Ve|%Qmm2LOLyE#}>o)JJ~{1wkMzSdSrfrb=J&<lMXh{Q2e)c+UwG=wHtEv^1NO!
z7Ww~KeOhNu%0>we&j%uEA_Ccp^K^OEEHXNBCgx!Aj13Nz52i8t%@SZ*XF20k&5JGn
ztLt+OnTyn<awglR%1ybm-9WK;Q;=BYv`4~wf~FleINxsE7qdEm>(i;7tv4ANi%bO8
zwVpqj@0qsv<kiglmghRmnm;u;f1df7b8E+)Lpxi~RR?~$^W;pn_ku-w2WI<b$h1FK
z6q&%x#JF?^s{!ZrIkrdb7+Vf_o%|IgF>(GOjR{HV_9r@xxT6~45BaYVztF9nx=_7Y
zl|gr1)=?t~DVxb^pVu6)Hq>|&;LsM~H}C4?srz0%V~FzGp}_js@XMtq*LTi6{NEx*
zK$jtE(!xo)JC40r+5TH^-HXuZ`zbSWRzF=R5#N;^-dbbk^C{fI>(kafr_YGS8y6p4
zSZ)+!;Il9>Y1$8qM|W5YZiVgJ_-TEwy-06=?$#$KV|O!~KX`t8dcI8bKgEX2YqHJn
zL{$CmDK^_9Q~W*qe2&%i?azO&dV6`*`-A6opSC*N-rjah^!vv$uFaE`T2?A7ObB;;
zdg^?5Ufj;+a^|-`B6n?mapLdxlQ!Dvd);1gaMoQY{+W4NT7SR6^Ida%E#q%6sC;?y
z^RZRAZ(Z3$`OiADVk>+OHb<~lPdHxt=g6v?%d?+7;yty;y852ay6)e_@5Jm?AI@Yu
z_Qd?WrK`?=g_#<BjgK87+ng)n_k?S`R+?crZ|#%bkEL(d-ME`kZtd>T(DWrGF}rT|
zpWAOw%@o`td0r;ILuBsseBU%}&%cK=e1xYxu$!gRl2()#<u^TfO7YRno367>+5LoT
z*Wxo8>Rb95<rZ)_igW&-vs>x27uyfZw=!H2FMgg2Uhs7DMtRN^Kh|mA?wT*u+8k>c
z{zSdWD*V!q8RukJw_llCQhGMa_Uh3cW?v8HwCuSU+%|LO>Y|yaC1<zZ%JrEYdba6>
zlBmh`dz+^0dET=r<A(R=j&Gl*hwkLtdg!1*21Ao<(lO2iPwrihFE~bKNK|XMDqpcx
zKgXpztH=7VVuL(KY=N8DuM?XNRh(M3W5Z`Q=2T&(-nY&y%BEs**H5t*KbWumxr^nd
zCC3EE=~HH1c%o<WAZ6d_nNbsx*0(AhI<~<<PiYM+%O)37<ESVOnco++;y2FZ{;?}<
zQODZHdOMcc>))`yvpMa5?$c~%!>RlBaDTraQ&fLS@N4XUFZKKJdld7l-py-y#HP>w
zY+5<*u74tr<Y#O+6n@cC{TtJHSLf}rpQBS>Px-d8If(!H-3R{|B;>#UP<y@H_q$yE
z;rrq1e*6Y6TKcf}#?1ab8x{YxEStXN^U;{7YsTN_oXLCHpL*^^ZPI-2s_$=%mYjYV
z8+D*rGQ;y>{+qYm%5f*^7k>QsyYJtg$J^C)-h96i`SvC=v+<dNhxy$1qJKPou`>Bs
zpWL(?8Xq?=dHz80_tC%Q;&*$}XUD|m7u}k|x3BMgdBvXl?dg8jw*{>ue_xrO`(I+_
zKCRCW+3s(h30@1u&||x`_SmlN_4^V&&RpxCef{n7Jj=UwMf<F*4t$(#%07LT^UFE!
z!vvkD-(SMhXncQab)H<{KK(!Qm|PaJ=1YF~I=4J+-=|-%|9so}Jv%tG=fCfPjz4vi
z`Mjf}-*d)ku6Gq{zV<5nr0j&hTQ~8}dd|_)b=!&SC?`kADzie%WIe7ktyi}0s+sj+
zm*os!uEp7DVnN$K$}Hm(z24FN{mX|vtGTv%mz~%#Ke6d=kJDSNDVtUtPZcYEZn<NI
z_Q?Rxu*UV4SGbP*-aJ&dW&i!z|JUV<nQi!ZAno7n{WEt@c%1e7*{uDCUU?)JvgS?y
zojI?6X5psz-IZa2Q*JK(d4omC>m+Y(_*1@&_fFOCH{G>aWL7UJ@iOTrpJxQC^UJuI
z@<(-fiY&HsEo{zkY>>b8@tEnhuJw%PCnTyW3OI;&$34(%5MF0lGtpIT!OP<cT#R*6
zQ<#|B<`;>`*#<ps+Rxy<Bh8V!E%eWnme6z24HsBj`4-J>oP33C%C7oz`!p|!q(vMx
zUOzYgqrKWp?ewcA&f&e9g?CaWELL6c`J)pX*P>W$F`c8j7Ozh}404$u_uRL#UwdEm
z|7*Rr#Zn>llTD4^>-qnUU0;3c`s~OHn=C|L?fX2f>Fqql7o0n$S}1?paY1WK$a@{V
z4?d@E{S3Yz`F?Hv)7!J{e^lJOG52HkvnSuQq8}CawH4OZ#JAWzwLMt6{I&I2+fT|2
zJO&as)8D@jof8rz86WvSXbb<|v=iSJ)-Seq5a+v-!5H5%pFvluz*Ro){LaO1nEd%S
zGqCmDZSawNDtXrU(YCGU>R0djKaamy?%d7li)TJ>pSN4YzVM6vx)(e2`>JO=|8|pm
z{`Nm>F0Yf7{`V(4@Ska;e6k*Y5C0U_XJPZE)L(ly^~KKjEC0%i{hexTwmtXX+*`9#
zH96YFZ$*|o`FDHH{0V#SfmtbRf;Ojj|M5+l?zNfg{weM!zUQ;2md|C;dU5z(#r8t+
z^N-YjZD{|~`1{ryOE&An4~u80mr2_juHJI+<!a-1hwnYgekf?jRg=Z^E=G+pp3gg5
zrt<X0=)P+1zyEiYyxh6JZ~CMcG7&2dGN@;4j{OjQKjmbSVY#|?b|%k^XZIJ+cr3Z0
zYl~{7&QFoUT9)j0u8RD(m=d^3HfvSwxf+)IWtr-?4ruinPQK1}!qM0AjYO)(U*_QU
z00*{pE9#GO)rq*hNZPw2)?>x)dy6+(Y<jbJV_Z$R=#~p-=IKA$e=WOy=k*&69{*qE
zuXj|xo)NcOZC%1QHSUXU!80r_%{6~7lpgIGED`P%e4{(%M7gfFkbCg6h<(a(*WI2t
zC&#~T(%<_3?eB6)gUv@+B@7H)&$wh-A3HN6<>Zd#J8KM9vwAH%eKYZ3dFIqL44-E(
zuGW3l^;08{(c}?VxX#gYavyZq8I7hSCGz}^^z1nL_QaJ-EV>tj8<v-KbKEuvm)US!
zE~iC6<-TZIx=Y2RQ&R<tmtI|4bhco3e$a-mld8-1ic0)Fxw+ndMcuChyH*9)P8a20
zed}TFJR|LGt{LJ#R<QY=4lY{H@Z!+sm9ok8eEasA*Z%%>=Z5I(dDZ`#bVI*S+iqRJ
z@~%wo+Rtz2jF;^-%ssW`x71YgdDosWPSJQeH!&{biE?uB@%sH`f8N{w7n$?@guH)_
z467dZ>cfn`{{7<Tul@5!dw0ygpC@!bd_MTX|5nY#zXyJ*+E1yQ`pdKE{u0Lqs|QU7
zyc;B$XJlIK`DoiK%_ue3VE^&IdncXGFMl_edEJB!Tn8RpOzYoT`{!r({Cyd3KX5+~
zR!s3fKeOtJ@$8FzGfn>P%UtoY>)3hIyz4(=Dk>fx%G_^vx9Vo}YWc|UX;yYSV=rB8
zelItxKJJ_6%Hw~_N^aS{w=a6WS@-teds}L61^+zF{r<%MO>cfRt>#MmZu|L8zTO+7
z1*_h-1=}AGS5-aw+|GXc;p>g6=6|eSY-f4@Z^x!1$8V=@tK9r`<xz3bcX#YXczAx*
zPy7D!-Fd$|d&;Yq&yfs`)t;<%?q!YB^Vhr9+0~z}bY|Pn6ZUqt<o)`;$tUg~4Hx*6
z`Caq&rfp^)V`u&H`YX6+a>Uo)n(z1i|Ht@7?wD-BNrnFt=2z`6NP4>QZN1$0-0hW%
zpWoe^a?$Jge%W^Ote@qNwcD@BU&?urFK_<0<e#{B{+GNRuigjGk(IvpW8SgeyoxW!
zs`Di_ewcOt(Z4%Cjx~zM&AVj!^23(pui2~g-d){%{@p%Vzug~OR3*MYdgQIQ&qCC?
zXXCzKKaMZmf1dm5Y9F25viB}5kF43e_pF(&)|~RYbL^*^{>eVE%h=#i)&FUcx82J*
zw#Z&9QY%{tS-<%G+$;Y3fB$TnlV?3EXI}kTcVYisn%<vUCK*q0+_SFUyrr-)w)f(t
zE-7!eIIoxp!&n}k6DbXnw*q5s{9g0?<AsPwkMMKu*LF@)l1WriJ+fn!n%q0C&W?JG
zZKtohEt{Aw)b?J<l1sdPX~?Xiv+b68hUwCd+{Wuu+fou)bQ7}FCe=EKKbOj!mCv<$
z@!zZQv!|Xd47T1gi+`)mz1+`J))cCIp0cE5_t|sVK680*tCinAv;3w(W1@;ui~mda
zYqOIWlwX&(Rw{d4o0DRZTFLNv_v4GE+a^4mxqRB0xJRm05l017d+L+_i5}K^$?hY_
zFry`O&P3K5i%+gRZ57^-yW{i*ucGWA8)kb+1;#MLJ4r|F-<?c8IOF;zLpCQNDa)EG
z%2z*Y>m{~2++od_n84G}rF#A1<oT(OyF%~uEoYd&MZsaSpF*9S&oi%Y2NS)!nm=#-
zbKCge<F!wd4q8}DIBLA3z#u9{GIEiRL3P@Wt7Y+PmLE8IpLxxyUArvIuPjP%Jjiw4
zljDE!jy0^T+BWKz_s`5(byAYsYya)Pw>HhcAoM%n(6TGMANh~!ciLAM|9ch|y>6?Z
zf@48!LGFihugWhr1sq^5_SV<fFI;@j<oA2Y>^b+#TW&NymMU>P|4yA}hyKfN*UaC!
zCCDw1GFIeWc1(ZWz1qt+&DZUG_@R%#Y+m7?hBd#-b#GK#oOe9*U=9DKQ=2zApPv>V
zRh0vs8J}1E>+Q#PFL!BgxITUUuYbPsGCv<4XUGpeSM%kf`y@v3i^A-GpZ#+?c!pWI
zY@YP}dYAc!{dYcQdEPJLStwdqd2avw>m@s?ZkxU@J!6;ms?&Gho>!}9#^>su&^e*M
zqv^g~-(!y1b8eTv`&Bvj{Qf_C@|=IiR6gEu`nVle1nY-}<<su$dD5eL_nP_rJGsZ-
zafi*FZM=Z<z@@I&^?Lu``5(~w^5^lRsdvNHx=eU-_Wqs9=dr3AwI6vs-v4L!_xthd
zXSWqH=3n@K?(KH|-Mag~@3YhWz3%6)HQ(L$ZQHA0_J7l#-(@-aHY-1!tmQ2~d*)X8
zyK3fN-+P(YFa?W$p8o#DE8Xk9vgNmL#54pkSiXtq|9N-+JEPUF6JyKD<}!XLzPGE`
zYVOZ%|BwD)e>9!(T;lVK|KFzD$$kE5;9$Kc@(l0iBXi1ardsb>!1#^jj(CFQp1+0P
zUmblt$DVC#%yk9Lo&N7c+4FSlo9f+voEPKSxaE$4ndYoRHvMeJVtRTPX`J}H@zJIi
zx2Cpx&JmUW8~UwBLd7`viO@!Y%A_-PD<5<1<j~!H@=kmDyC(A^F((USKfO>oe@fCE
zbg+cpX%CNtW7V&PW(r&qi8}bmAfT(l?3`q<shs?}cR_E<TUm=TCWo!Mn!Lq;D`(cc
z=-!~u*~?5M_MQwdEZ(r_*u{@p5eklNlV@hvSSCGkx*qs$(zee$pP%?e70xha+7i{6
z>D$%(yffQUk9py&TG3CsH?-qUeV6XNx^>C!HReyFSijAAqLDDkH!0Ni`AzUy4lz0s
z_ZSYFC@eYK^4_Usu6v6)TSZi#SRIqd#`YO@`x5_i8cZ%-eL;RgKjYLjD@9ip{*bS8
zy2F$sy6&Rh=k+)EjYArugX-p1d;Mq;aO{1V^vvt@W+(pxN4IPjKL3q<@l8o-V{t(@
zYxUQkO?|RBcd>MfCT~#RX=J83<J>&Ss-w2EKg67Ro@u*EO1y?YV(KlsZN1OW^w>$p
z+U{5Sz;7e^i#bV^N2|`DKK-F>bl%_3h4a;S=s);lzU$4SS9XQ(O?Gd-`EObIvMEc{
zuUPZWnHpIEJuL3t&%)Zr({tlC+v+}_c|EW4@8X%#@=@P>mP^LokNRD=I?{hm-Al>o
zU*_L~Y;h>x`*KC=+j_^0`~0u#XZ`wp(n5xL)!ThG)j#j~$Sf}0!1kE2;&R=Ai>a|e
zhu=t?-H=eXVDWkx8NX)NPIdO!JqJH3_wQVttmJS=S9krJikmBA=i8b-{E~cdN#oa3
zGv*rV7Y94-Zh7>hI+)FL&ebQ1|JWW#7BgSIlpemm)3)AUPO4`A8iW7m4pn#EPrT|P
zbNqi@;^oWMLbu{7Twa`5Yz<j8e@k?$?0qfyx=Z)G?mehyN=U0MxO?<<KL5VrgqnN1
zE4b{rtEZRpvJ~{r+_H1uSI@QUEPl;>7eD6<S6b1PsqXjcLSD}~zL|T|?k3}u8)oL^
z_c-rQS+_g?>eSQEWFx}<R-b8D)4%=h{@j{JTQ@qdihpaeGr8{gpUL0n|G9F~zCrwu
zUCG~V-h0<=o?iL?)lRk4qmtU+*Gu%IGfrCiY#wv$EY-O)<{W(c;(gDj(CI5>_aB*k
zl<z^Wwc>)qkFKfS=UdVC!Tw2h<72KV;{K<~{|QX~lXEQe>$9+L+H2!~7*}L!K3k~C
zoAtCgaZ=$U`yB<bI+1U?-e;9=KQ!gY)W~J2^McNo$ZV1o{8^n8c8ukK`^gOs?*9B=
zCmx9WcsJ*N^~ZI;0@sM0*nRMW!?nU9M$uKpPk9!f@HUw7@PRX*(G;6Wo0E^2X1<>4
z)okNDuX$(x>t2qDoRR;%&MurZb%({0fSdB1YOxQx_;(v^cd`DmnX%=>9v!<Svnr*f
zcNWj<f4*RbjQ8V-TS^NzNWOQF=;T<KC!QwH;=V+atL>utk~@l#4B^%LpE;eES}{$k
zVb+;l-^{k^*yW1M-kR|<aNXZ6C#Gz!eE)1Y-?EeCOEyO6=rB&c8GNVRcJVHePwNj@
zFrLdiSobNaD?Mb6@Z~Gz_V(R&2{UIbbh0k_!!YeY0HcUuN7IeAX`V+FyqH7Gbf#V5
zEScA<J3q*-OHue8(~1RQQ+c>knBN~sIGl4P{^X?tMiv4eJdMK7O?Q6S5%IY3?7{Dk
z#c!TDa!f^I(y{ut&$8PDjBb0iF`X*fdrqe1igo|df^PTes~V&vyz{1BuzQ!0x3aFP
ze)9#V1FM*&S8uD@ck^RX-lqJjs~;=v-mhxf{@wo7&V<X~t=%dvZT7qS>)p3h(eGh%
zVhq_68&$VY>(M-!Sr&FWwEa1g(aF07Z{zl!I<o0@VQ=~M9JgDl|IC(~*q1#M{I2-<
z)vNdG<SMQjay)Ea^^p5@e9qTHs@j+S*(E0b{vrN|zx(%^gQr&p3!5GA56U}qxA1HC
z?~pT`bB`^~uV0-pZGZH2<5R!$^3>B08J~IaK3?4Cz@3WvD89VUIln!p$5~bVxv|k=
zjk5Ua%cgJhvI6-Gm~A~}E`JnRFzcuGp*i67)-&c+Io?{F8Gh17nqkI;6Q|O>V*l59
z{coIlkmr8pNB66<ubJpBuKv_?qwa_0lZ%S_Jo}h8UrlS8x1(>TmYwRyLsuRr`>mKa
z{gdOb#IJUzD~ui={qpmKGqZAe#P6_^xwog<#(%9mV_|btN%*vl_qM-(e)UXU&s(`V
zw{Cw^&!2vwwq<8>KJeD4`S_gkynTML`M$lLZj(j*=3TnCY2niAiY$WVr=N@Mo%q}6
zzut+*MXwGl=@*uJDmK$TY{Ql%{#U>6a8u-!JF3~K?WOqnRmxUvy{0o>ojR)L|Ia(H
zyrrNfyeMW>hIOOHJN^3C7td|rI$pBngX#UL3$yhaODA;{ILS!LaXe&ymK<x)%=Dqc
zCH=qVaZj#I*>yQv#3WZ3WR&D}p9)NzVbWx>sZWeqOZ>tm1+la7T3oA&-m-j{q0Oy)
zl<#r{=UahJW-$rTX{_z?kyAZZ2dCIwaf_P1vTW(<p8@Mmx@P?0eE#8tkKD;82H|Q4
z8%}-h5b*L<nq$0p#`cRT-%SIKES~*5*w}z)s#DzV?)advv)NZ_R&lkseH1m*eRJW@
znw`hO@6L=_e=KRHGy6>$%P!V~k^e2TluHaCCnSAZ|3LTm#6#1y_A%T^+E{9%x={Qc
zQyPZ?i$%IH7st%f6o17n{gIk0xS1Ll=M`?7I%Vqw-3=a*IxTNK3%MDz?yv?l`min#
zcM#NOv9f>M#GE8H(|8R-yM?$nThGZ~;UD<pyxCa7&T{_=opi>5`!oMtc@JxW!oB)u
zO*pwCUMig5*qOb<wMl-Twln+F*q>9_x>SxIUHmz|$@ItZYacG(=zn~^ydik|r*r$f
z8CMsGzkQm?n#}cH>uvbf!?WXl1-yH>dsp@T`_q3scl)9#6;-!2JvOH~!SChS*5e^Q
zeIKjURp!ipHZ}Pp--CeH-aL21ER~<WdUfomxwHC*6+h+$pYKaN$M{#!VaDH=^CT@d
z`^_wSQ~Flz8`m7OW$nz#f7FkJgo`r^ZOOlq<T?2(2OkS_@QtMBr?u73?y9^ziCH`8
z8e6~q`Og|{GFmOHN)r5UZ>Q()t$WP;J-_11w)9sU=UBNh%C{c2En#KeAAbDEk~ftp
zr!Qxx?@-yv8Dw=o2)wBNpx9LNusJb*gIo)5x*ORqDcw+d``Yxp=>6K6`Q5Cu>nq=U
z4?Lm$``xwtPy6)_<osR#L-;RqblCQ&>&GfjPq=x&zkzFw-@2%*Uq^lXZ%wGo+#DaZ
zHE(k7gTwpszwcdk|HZPqv+Hh_@HsKsy4}8KxP40sW5m|Y?<8haUi;fn_Ta*!eOkK}
z%QdQ)!Ws<^$1yHr+LUze2BSd;M_$VTwr}!lraYN=rb*yw|Fgaum!%SyR%Gi1vj3`m
z7^8Wv<dK7;X7Iy5V%@*OH)(7>JHO~j_`@ju*AFhfpOs>{fQuuqt5Rzp+YzT{qJ8f-
ztrK~_Ve)0wgn8XR74^8^RLz)RCivmmEyloR?K9Hq28&xoc0G~j_+T)h_?N`NdE8Gl
zvKoUfIDPU?d&o6AM?1X!WX#R~rm?H}mOMRU@ZwQf-p2JMDU<6}wevEwZ`MbD{MIFQ
zD^}OwyxMCY?yHx%xMt6kv0TR+;3&UVz1mpZ`kIXg+om+tg|Y9wYdDSCdv^vDJ1qXT
zEaH*VKGw|>+I9xbJbQcN#I%h!4`=XvaLZ1-aCB!-jqX*Zo6PSPv6ctt^~K9d@$U%T
zXqOgvXS386W`{$&lTWlHxc*?da<E3F_-JKVg9AgzgOv<>KTO`db^&k4zezh+MLH;K
z4vo3Z)xff_p}}*<>s1k|E;nN5>9%MywC_-#&U4J4E%wLB4P94%&iTju$6bB8fk!gy
zp<70u3)Nq0pP!%VZg}Eiky*+Mh3Anc?TU0b&&1qn2=bo5^#6&^4)rLbc%PZmwjABM
zOTXKX^C#nKu6G*uURmpw*T(p?eSSN2@80+OSbLfJ<t?)BFXdTXP=5U8!s)kcWoE_w
z^9(!@qs`eK_55^=p_+Os*QO;Kwr{F$UTidb39Ep_w{L~>?Y|wK<~rTx_K$tOJGd{F
zh0Szcpis1E-~Mmy`Qm%8{@e2H__q961_rr4%<Ahs!z!Itq&(Uzsh`Tjrr*+RYVuma
zh4JelPgnl<&o_i6SSx-1$SEp(d~Dnu|F2f+fdB(@0)yL;Ldl%azk$5dO`>|rINvau
zD)xN*d~W{rzk80J)ULKH%{s*9VEskr$K#jfbEh?yoydQno+8a)H&@o$GP&y5dj`=T
zQkEOcD(hktu83LBtWI|ApON<csNb&2N#fb=@(rFJVi`Pa($h{Fu%FraaNo1V(dKJp
zru^_}K0jw3xAoksg-Q*=N8Wvpm$fj@vs@rsv`%5e#ie&YKCX)CnzLQ~eC3|2i~Ac_
z9=3Vd`_o(^Y<ivlvxYC4x>3viOO<QiQmkT(>+}9)l&)I5^vT4zGqu?gHr?5vt?n+k
zZbFVkzpAA1KjEG?^2_cR$nIlYl+v-lq&QYHy+lxLIj3Jsrl3Jj`e~Uy9=k2`H)n@&
zh>J8F5B=Y@?&+OFS5`f`88k0cf^UMWQN>cHz<{O|^%GK;&n>ySGimav{IsP;#ky*n
z{rP4lzdOC=?Xt6qLAD2G-Q5~E|M&l8nek2E{+`InoEUcSQM&0~$Jy^A&)s>Zd%NbR
znb=C91-|v`JeO|un7pcC&9n4({Qo)lj>K-+T({s%e^cZ2Q;>tDE;Lmdtw?5ma8ZOw
zGH>HMc?Qw*EsA3Eo+cf9aJGPVANK*BOW#j>HJtmSIscSkTs(MZ>$yEi{AVJ6YSn&T
zRlV?>=!|t9rA%|Lb~ZSyb-h=5FMsW|ok@?GTf&}mmFHghZ!-D(ti3n;MeDoYe@k#X
z8hr5h|9C6w)5q^W+HE_-F#k|h{mbj?bAGL>JhS|$@ZQPJ{7Zef@4nr;{mbP~j-kG>
z`SpR1P8wd%k^B+(VOHMnZ)<LRemR}<=gQx&tm|L++@4&Q^sMLexxHpzKPql7k&e6h
zSHeLy_h0I6?|N-bkF2?NoA$+T%XodX^x%Ug#-Dx-@<r7O%?FjOQag^bpJFOL_=>^f
z!MZ5toXPKd9^4IBdibHVRgsNwN||Wcx&7+jOIo<EeO~78T^DPlqT|))<(2VpmDio%
zZHEu+|7-Jcdw*Dp{J-$iiN0)2=Oc@bO<L=HZB0d8>U+`2@1x3}_w1H`Qu<Kyp6Wg4
zmdHg)lHqBWZ*9zSaJ-j(R=R9)#pA~Hd^zj(?EACpks`x7nVgfGU5|z2nCF*nkoet@
zzSRE<%jxz~LxH4>Q}s#@%WCY6&G#CAKdh&`PHIp2gB~~Y?Qd@?a+asM1xqZoX3~wR
z^}hUW@~OHsRfo6cxBkVOh5g|^y!ycGz@2{I0?V5l8yEdpuJuWCR&`Xap16+Suj)(2
z7UnbmKApkw=kd*=>KS4W<2M^DY)D!v7!&;=yieqJ(|U2=xDP7wo=-oD^hG>!I%Rgk
zX=9XpujB;B=B?b}CV^jPRTf0`)a7*jbc?^Yy{vWT+uB>x{mht`XM`oznFTTE+z<b<
z`=DK_Vp{IDi~Emz_)hzH)kj-q%|Wxg;~E?X9>?fxzG1cKbL#FXdDn}!r!Xe5XMk68
zv^)&{GHsKMnewweUD1!540kV?Bqz@8)~zG?X3dlXLhlY4w`g13@E1-@|H$cJG}F0t
zcht^ANtuq_vqY+6n^)gC{CT^G+{J|AwA){JC4WlJl+>U7R8_^mVYe+$x5zfvTmSph
z&GzlezH&S{Z2Ho<+aI^Cx-nzXqkXgW^{uS)!h;Gw&3`fZXUHeN`}fkeZ`-jj#o^&H
z*5tT~*W0(SO2*sn+Wz(4`=Vds`yzkTJhVNWFTL;EB5t+(>Nid5ZDtv>XHNUJr81qb
zFZ1c_vwy$0bE!}3iQP~cIC;L`tMZcb6V-bi+on#r-tOsHZn-Y<#^FCY3$CA$zF$%M
zOowAf`2NX1lAHCl>+XNPx9q`#?6xb?HXEyJ&i`8;zvkx_$)k0#pVr;+{2Z5QonL2N
zWEb@?Id}U-dB>@$*Twg2sZRhe0#07DMo8^+eRk>;)5*K)h2ETW-g)NDBLgQ*VgAX7
zKI&PPo@qbO9HW1z^C{!qm=DwE|2^;hN#|QnLCH@JuR?j_GY@X=a?+T3M`ik?c@Zm(
zy8_MbvOUOj2z=)@<Bayg-{QP$HcdM6p~5C{rZ#8MmgJ4LAsZg@=B!dFJnzx}_}jm;
z*$-nkt$Y@;`BcKhbIa4}l<s;;PwRKw!7JaoT;{K{YwAU{DSESJ#ZIbS6)C!D*N?>x
zcO#b+^d$;KsEH`JwM~6I*+J@;$TrP->xKH8ea@6*>WJOYY6~@pe0Yjy#f5XP>NQkl
zc3ilu``GXft9)yMYuc8In3H*~S(BDe5?yz|-SfrKx~#DCr*fXzX!Eu$wEh3>Pv+0<
z&%d7VyCK`WIOxmghx@&(e(&F&lku|jceL2P4fQpjKdJfhTZe_s?P}Y<$-H_(nz8%`
zhT?M(Hy2doy!`fR%MUpYk(`U$?j2gCENwS$o8h|Pz2bl4L&}`x_?-TIli&Nl=?<%O
z`<mTFxv%Eeui0DmI9z?3<=18R7(23+BQB~ktUaR=bj@#;q@63*x+lRi`1_7Z1+2VT
zZPF}%Q}}q{ITq&nsg~OxUR$>Mt?l(UQgv^tH{E33?9n)XPxj-;ssqv27W6&YbVf$x
z`36rn@7MX4h0~ZT9-aTP{=2m+--Ps3Hp_FnrkwH?x^gpQ(<PtO^DHYGa`;a#ULDnR
zBxxnn(`!>#s?6YI-Ze$=@CKiz^vxgDoVm`1u1%S5&?MIPNdBpNrjd}D(t|lO4<EPK
za$=Xi(PFn}#|q~4TmE*n`^sP7k#a7h(q&WJ<k#XoWj>P&&AZp$ezPs@Tb&Z?if5K5
z?&$vfoALXMlE$aCGdJ%vY2=cgymebe%-Z@HWvlu`U&{Q`4Xzh&5wVV%G1cjtNP_5h
zb_+%x<}Xb9k5mTBXsqa5zL@o4>xs5UKMNX^5@xi7?nv9}u*G0a5+e_Ty4>7UgGG&@
z39D1?i6n?Vk7wMqAz=5KGS-NrHv@9!gg)(YvTFRKcY0IyhOXj`OLi$di;vD~6biFk
z?J%hpylhWwi@|f#gC>TYzsvF*Ii|6Nn7mlp+j?eZcT)A9D+~T!ox1$FnZ~Y+JD$pu
zq)O)B$SXP4c5Sz`uiw001A`|I6?|5wE(oohUHI3|ruf#)!}<TeJUhMqzt9FQJI2*H
z4hyDnrq#Iz%?_N*W7sUeQ~guK`JY9%F5YQ7KEd|soijaReVYY-9`NT}eS?2quUyze
zt4s6JE@a$e|GX}1^<l^RY+sCok1i77=~7vfwAA|BG06Y}o7R~Yrz(_IchwrLnWQ9R
z6ztvh<YUd8s0i>@KZdZv-D1zuw;BBOUSzQG$>oR@bB|AItm)&7EDm2cCE(lTd6&Y}
zTd$=4U1zYM!>!QKXwsSs38K#(cP7k8SbFG5SfTt(QMY9u!|gczWL9=cE?q7Xzbf-c
zW@+P-SYMafigP0E&swgTT+jFAk+XsJV^5D`3hu$TlXI=seOK+jvPVzy7l-|w$;VB#
z-`|W_QgknECd+T5BWaKSPQTxO_SVzs3CvMDUQPeID)MV!ur<%;kd~ORCDjq-O=1ta
zmrQBxO$D7^S}JnJqJsaPg80F=4T)zm4swZ4EB_|r;6HtaHKQ(j8{}xF1c?S2?vM@s
zY*Blx4yc52n}lDRwCPQnL+Z37cHyTj99<6_rUe|2Wlw6H8U=2KykpK^Bc{RpbB?Tq
zaiY(s%9wVpg$)@K9|`NECjPfQu{<kW*WgTD+*4Vx8i4~si@2lwD^s>7=A8Mvll4Q}
z`)Oa+f2rO+qt$Rm^yCfgPamj0PPr#3`ENO^^R;O!i@v%2bNQ^?|MHcWuG8H&?$gtz
zF-*K(q{A6yqIbe(SK0Mr((jkgS$kII%<q+zX%~{}xUNS@#-C;0C)O6NlXUPy52I-N
zy^|lD7}BP#PvGP|$?B=cT_oc7Rris}ftZIDeFBc|Suq=|q^=*iWRWPLyY8`zLRV_t
z0c*?GRXcWhZ;cO|KFPH@_MhjDCRgQ@ePYhC|1Qrsv+a@PW4-fT*;$?y%C6@RGZ?VG
zYS|ff<Xz?Ytz6-={I835&(Z!P&zCi+G53>Rdc;BI@8-{?Zaz7DMP~=!vBo$X|IO@j
zt;~lP&Gft*8T%yW>a?5fzSq{J-wfuyI(h5<oF=(KS>J**lf;?-r!CAAZC0I`sOqY3
zak;aZlX2P>0lQ~A81hwivnCw23_oLZ@`9b%q6dPOF%#Ucx9BZ0l>EW|g4L2sY}W*<
z#yi4tIZJN5J?nPlklTf%fS(J(*?ara)c%-zuTu@Zwkjtq^Xb1ihc18nKX;?u;vF0P
zo91NnFmfIcnxRyA{ZCHr+2d*R?mW^8jai(dJv;P@4#(ufulUaCecX{|cWPPey<WBz
zH`tZjgjqE9p0qQ~-rjiO*#wW9?ri@KZc#e)Z03z8HAeoam-RL;vDnU4yC=VDTKhlc
zHpdSPwKI-#2|qht8643nB%sl1d*=1kLpNd`U&_fX+7k0wL`UHJpLQ{ZHx>T;=XXzJ
z`|<J4o6QTi9PqC8NRyj>_BjLRL4Jethgs8ivR;pwev;Mc#R-j_teUKS&1#$D^wwG~
zW?Gn&{<4_QV#@hXYi6hxKW*-u6|%$Zx{J|ffr&*TK^Ff4?54~<*dNvuIcraL{OdDk
zsuI8d@MlgtvBdNC>eT5|w+s75EbF~l94vD#z0RvSlp!|b`V-X%OMwks*U$VpF-N;2
zZbO!Qv!UE+$))Z8kN&U=>YecP`px|<p)XEWAKGwgS;~f9#xO=ZM^S-<&f?V<9-TV(
zN~XH<PQsSv&6^rSg*Z;`=9#nO?#e9JH#6UyJmz$1Yl6}hmBMJy-6-kxLeASm&mGu$
z><oiKA@`N@DZ5#CCP`k7P?#+?`=6QUMiYa@8{^E<_+(9r8_rl4%D>I1Ge`_QdxmY=
zjr*qQR++!WGIuQad&cxY^8Iyd10R1;iB?V5IQ`YAyEWgf?a#-pF}s7gW=81j)9&Ug
zmamv>{P3so6~>f9{&p$FYqh3rouHO_YUbui8@gWmZoI;>{@hLWWK(s?WiM8lyZWE(
z3i@%f{?8=Sm4{aFonmm}$nWr3Zt$1$pXWLqpJy{To=o2K$wpB6g=+MZ*nMT3Q66?$
zKCa88IMcn)Px7CjaQxYe%)=9IoZvZ8y72CwHEO35?LTK+;Wpizm|()lx%uSPlc86x
z?OHVP!S;uKxjM79dc6B`pXu;Lj_?m#r(~pmEZMwg_cY<9oKYvrr1|+CtC%pT+CST)
zJSk)P+Y?_O$>>TOzh+#-`hcsonQ@MU*GBUej!idq^)6ujWgy{f%^<|Zt{gLAedH#k
zAjfZ_{_9`v{WxXwdyC|-_2#yLGo}@@F8uH}w>#=te)|H}TR%_mJvegVL|gVGo2e$V
z?PewFJn6hyZ877~r>)tgXFs{>rxq5}c>Uke7qWUo*-?WhH9zu}C{9n4{Ni!ca*3S6
z^{GayS&RJlX@@eVnoW;9;xxJZ`9GmQEpC;&D(9CS)Cwt6`mgt5X^LLvnhq1+3x{+L
zy(_u&%CU2%$4~Zj5nu0sU*Rd&RGm|cr(fs$W*4MbYFVFW_}wJ__YBDitqqmMOF0<7
z8>Y|jIG-rN@aV!ByVQSED`S33dcD)T)U;t1Lv!|gSBF{8?$5irQ9tQ5OIei6m!~fh
zul!;0J~=boY;{4Uvfjl|zKDXJX9~8ngHD*auCNq7pZ2fl$cxXf7a6MW_WUXPP)Bnc
zc*ac7JK@BUscSAAGF^FNi3#grm3_|&>~z&n$%Qq>Kl(aLQRcALJWivn5@9!u!+BPl
zXmU%;(7ySOVbzUKnok#}@M;^cd35PORGgSN9~(pQKK7>2Ia^bO#X9C_e|9`_N<md7
z?b5yDd%jyqraA>R9qd0-J1Ix>{#2W(DLE+}3(RUyI^H~>_1ng9w(gplKCL(3gy+hg
zj&VP6Oi8qC`t^ywder()zq!v-^k$~^?2F1%r7v1&aBk1~7JdFh=WSK_ql-_>(T;h%
zE@ttR$_Yx6X@cB$wg&RCL=<Jo*uJi`F%kPS_3-B{JT~zkO{&X&=p`ka#JQgKpIr1|
z&dw)m`W!m6e*|2aaj8N&oi8R&V$H;aV>|j|x?isov{<Pm$>mY3u=K<UuF2-9@ez?X
z8F}CBYFK*wv7v}y+T~MA{zczZz7_w-bl$-;_8w~O*M)c=D=wUCx#Q!5H*sH9cYj%F
z;5hGp(7M34&rV;J{r)3==9;pju8lJ{Ptuu|(^#lCjW_-1`xA0u52cb7Z}>k;J!oIC
z_J{uJ?=0`z|Li)mYLa8~v)v~;nIe*&b=^)?6P=eD>Tp1s@7QA*-ERx_I)(|R@H*@}
zt>DGYF-2EuI;-Y4$N}>)UrN{gkZ63&;c_FeNzCcS+GpjruD-3~U03iixL19m&y-`n
zo((DQGAD0Vc^DQWe_wo||3ZUhhJI<5r7`YWyz!gGet(s#S)I+N_3P)AHQ&O&2)#8G
zuIayGfBxnky=ya{7#%TPopx-G*stu$m`UuDb*3Gg;Th}r?aitEN(D!6hTWO0dh26#
zOFg?qx%MTcozHyK*KSyEd4)HzE0N_R>-SVKlj;7MTlX9~u~^$@$~NgY_uH~3S-<?0
zdAPPSvUOroyC>hlGi*1eT%P*&jL?nMs|)L<XP-MFwBgD2nYIa<1v|y~1h0q)KD}%3
zGnQ$}u5Zs3<jz*=6z^F4%(2Mpnc(3A4>W#Q9cO;K$MSQp@9FE8rm#ht2dy&+3R{&X
z)pK#iG|#15Pb(ctSjRBa;&7Lvh(d2msKK-y2^vRZ1P<^w$RzOJQgk^O)g;k9KdIwr
z@<O+1I`TaQ2fK=sX8n4{z9#7$*RcbhMPWiEhU`brh;A=d6HO5M93`-#;GNPR|BlVZ
zYmyj_>AvWnZ)D$fmFb{`VTx170+HzHYD;fSEDnu++8mo~>h(wGrKN+;7LRqEJ2ekz
zg?)Iu+u}^-2CnOO<mGnkXM1%fX67^Au$>AO>%QA?W-=Upko&v4QPtCZjpY1r{fcLw
zCRX2?a@T(Q>s$9WTBn`6{lP8c!lLy>Za>qqt254B@7i+oPF=y7xN`@et<0#J<ttos
z{m=EDc%%5)w(eZLIYukD{M?XXee0uWref=zVCm`nfA6gOnEB^82XC0=)>TsiZ$$75
zBy>76N`@IOZ!EK6jXn4SypN7&b;Q~8rb=tOKX>G)+zp-YcA|0T#wdmA%L$^df6o;C
zd1%Vts|ObDnD*1}%#54fk$;>c7ChK@=t&aii>D3Rl{HV!XzbK{))n+gU}b3QqvTyS
zKeE~b#a0I#RawWgBqZ!ci07k!&&oKxx9r{;y7AS0qv`_-t~M8y`3qMY98#3tYp1#L
zkim_#*jeUr;^I6PEd1jF-!=tVdM#PTD3_O>aB_h+8}GKP`S<pg7;ay{?<^%@>>AVF
z_5Rdyi4zO^&HTl3rweo&uVgr1aP~r<@g|9&TMnAi%r%e1!o{wE*W);Lv-q9c_tn>@
z+y3HAQ3i>PL7VK^e-!3w=eFtG>YI0BPm+gPH^=GSJJzkW3O6VSyTjDcx@j#3cM8Md
zg6@NpT7r1CKZ%X`ygB2zY5xhi9j$*q{8`Grr6K+Hx2${43xgSFiq-A<yrk>r{Fu34
zccedC176K;UAOPnm-BwA|H|jbl!m{Y<Uj9kwbR$j_tT&9>DLvUU9l(ZUeDT%t$SC;
zPu%<{^s!WGw|U*BPm`9eHn7_M*}r-w@9gqjuQ$oQ&&{7^Ul2dr_IO6^4~yO3R@&eG
zXE-VS#OE_R_0ogSi7&P8?z>Yz`<c7k-|)J9MW0W}KC`?o`#m){_LNSQU}nUEu>W4d
za~{~;Sp9e5p-$CgQ%>IBSy4Li6CtZSe+XST)L0qw-|JMM$K@y2pT=%!S;_9&Bl-Q|
zZlPT#c6zGX9#Bu)lAzPZXz*+m<I)Dk=XwY2!<0){v^2J#db$4U@<kix{FH9ws!ooq
z{5;3>Bd2s?r@_}{CaF%0i?+<nU=m$b!0Ko*y<Tyu;=`HD>m<4gxUcV-6jlEzVJ&M<
z=1=u!Z<HeyyWTI_FS%mFxx^*2>NYV+oOgMA^Gy32k8Js?YhDE|vGkkxtGv>{`}S!W
z-I+Vp_vz(MH`!PhGh37+wR>jy)|Km@GI=jrB)&d;-3hjRk0Prz@)iqgC<=>9Bp5KS
zV9scnzvip@X)fzMmjgkEET>QF?B-a^-@W#6WSbk$v<t_j<v+1&uNG%X|G6%4Mf2Ap
zUTqsz4b@VnN*Blm)(5j%ub-VWbxkYl<N&W(%!@qK1+L2|br?9M+iSWAoX@fhIhQA(
zebk`yG`n{u*SAcubrVcC{m6TJZRaJU;%{GWKfZfe^v)EvnGzX=I}>M|Ial;-{Q;G4
zhv%u*Oq$T+TNK7!mby3k>tE>Jht-*{e@_puvCDYPK5za0^|$QS{+btWfA{j;&$11d
zkIl2>UA13x_oi3;`OleOuk!mE+cv*)<xyUzT&CjFyV)Kmvy0q&v$yN`q}NS<O>3RI
z*X-Z-H{s$}dtSbEy!RsizT>agm-+lS^t?$;<=NSv=h-~}7}jss_FCM1&xHR@kB|PH
zK5uJL#Fsx;-+c1FT_^M3{g(Xme}(<m*x6$atZLn_Tao{;KjZjCnYaFPKb0&_H(EaD
z=ZnStD)x|-?EfmS=(BcjzQNyc@EZHJ-kU$!=CP{$PWYmpyYEHcgP_kcO21-w)0kIm
zv2=<07T%YwY`*@j=(qacZ`NMVuW_(D&7gKk_V&)g=#$fY-!I?u{oil)Ps^TfJ+-^<
zZ}!4ksV$Eu9b|Ai^VVQR^7$X)Zr`SyYGg2$*`c8*!_N50>!RLOrqsf?b?={S;kN5a
zKfQIK$;9%!i82!(U!IY^^sD2$X^V3!d_IWBHy-<QLPv+QPDNwt;@QP3R7|e^nIWl_
z8x!6gROo%kI6Ok(vv<s*@Z^m#Zw~fc3hvTp?U~5X+&`n{%ajjibk@G>yb$5!oNTrw
zS#SGiubJkP{bxEKHL<Mt{O9fO+i#|J=dZk4;KCd6sk+^wCFReKM>9os^FIzgu`XlL
zt01|<$8r?5Z!6=xT~Ye&z2T+pi$8{J+LJ${lykzFH6ojR8hp-tb1>de;dRvTUigK{
ziK6{|cXS&hBYV9*1#0;BFxbCq26Z>)HVXfKqTt|BSm2<L9>?h7d5-H>W6}2=UQ@3z
z3AC2EC;Xj#V9h2LKE9S3*2B7A63?B=6i;kCqr6du^_Ha6%+0HwY<;8u_sZ7Hg41zp
zm}EE(ON*taOQ<ar|6|ZH+251ZDQ&e%dh2UjOI7CDmDc&1H{MAGT>!0f@5^>rc0_XH
z@z$_~>3^=RnxxgfT1PHy>*3pbKD0iaUh~)I+PV3^&Q;~#yfOD<_Omy?wf|3hBz-nA
z!rk8@$^G@>>iIXGPr7G&8GL!t`jqI8Esy-QAmg$p&bKX%-SDL8X!<t0wX(nJMSc{#
zt8R@v`|Vz=k%aDp?gKUt6!h+Me+&DjDH~il^Ly0i-N!cWJNmT!c$v=1=f3l&{juu%
zx4!1*wQKxEyYyfG`@zY7@6Xw}<@Z7_C1m}1^y;pC%+4M6E_oX$J^Kk=llc1mtv|Wf
zcK*E1QrENd&djUQjThzg&j&o-XEOQBylu=G`W_lLLIYO5<#%)U-{P^2>+R0$Pn)~{
zoO!%Z{quZ3jhE*ZDSy9zETHADcSqA*wlA#yt?QZMJ)mp(-@P|zPcFXw?0%<U<++ST
z`9DX_RNhg}w=6qdnE&wCdhhA;e!s5xcD~YAD8q2W#U~~+*po9$+ngU9i(J!Ge4xVR
z`EJ#h+<A#7xzgF!eZTXrcJA$+u3k&6jU-eG&uJU2{_ofra4KrU#wWX&w{V<MnD3RG
ze};jrT+k#Vt>w^%z)dsO9}>J7b6UW0cA@G8qyD8Ei=)J*eZTxH{?{|p<c(Z1zjv5T
znUXqPNPMD~MaDAQROQA6oUQU)to1id8U=o*pAC2zpy3-`UVNd<A-lgsEy~nvpY&hx
zQjMCKvshYh#+<ptQ(qWkvuVkL<B5}l_SheN$0~Aj+s1;-pnkRTrSYQm>w2H;+H<-_
z@W8^I42Jfn-)=J4$fMj-7tT2C(fl=ELd~Dg(JmHDTz+0AzxzVFj<w)f9~FTl$#Ut4
zrJWHUG#HAiPi+rN(S2Q+xPM`KPjAV*9E*q(8yqAo+NU+&wz*v=BrbT(MJ?sUf+stc
zDC~}WcgRMpW|k;ZVZ-m_n0E~q-g11nU7S<7dWQ85|5Iyjh%JnTta#_0*Y$qd+#SES
z?tQ;L;`^P?;yIeP_Wj>BM|yQ#r~Tf4yO+z}&AlxsZ};oz<FE5(UHJ1>Z2FcvXO)wd
zp5gATNy&Nl<9)c@pXV!AC;!ul-S+PKz3~0jzRyf8Uvwsa*(-my{Mg@vyLbD^6n)LT
z=`Jr3`0MoZm7mN1T4?@1H|N5-D`(uLt7pqw{QY;2xk~n0a^3g!yU#TJ@2{?z{Pps(
zJ>q5eHf*hLcetIXoBqx&HTv2$HTAMLxAPu6D`w-d`nRUGD4n4<?)J|!tJwGN{qeH<
z8sGi}xBc%H?W#3-zA?XDS8D6KeWBrcv)i5Re=T2ss%FQ*@~QWKU%&GHUP9oblLwdI
z-}U+{+xjoRzaF|j?XQ*T-;OWFMsj=Jv~k~)Fk1b%_xSmjAMGl3E;}rJ|MHH&cfF#v
zNwX5$AWP}X?)&Y%|KjJQZ~LA>md796*D9}L0o_f-cx&N#@Ls60e+m|#kD15Kx6D4@
zZpr`r&C!UA{m&M6s^6WWT50))r-tcT($#wXyQR*B|JVCh{+}bc`oi(@%vai9-M{SL
z`Ct9-`Ipu2GsWk9Id%T=zPD!iU-r2Ft4MmRG4K4+>3wqcWpAwCpQ(BJ^}6BY3!9%#
zpC)(v&GfLfw^Ke>*Z*B=?sm?`u(YOV_mb7s_bxo{5>}hHG5bJkx@~(pyZW5%Pn)=E
zyzle>PkS_nUwq&GjL-eaUdve8)$K06`@Zb$<$1ey{Fv0O?{{|2?SkyfR}&mRE;E?i
zTU|b9&fSHzi^aTWtvRlDb$&p7^xx<ABL6<zJwJZh<y`m4XFva3J^9ePNs^Nf2?e#>
zUCF3ww6a1(GegmLvPFC0wyDPs%wWEqqPxam)5~Lflznd<{3#yV5pkmQ=Jab%cfYy6
zELQqxWx3mgWox}>t9KW;L<QV<6F<B2%j3sZHaia1XHWmbU>x#HX@Ar{vDrM=F7?fh
z%Ilxmez9n8R(RBbW76LzR_EoL@%Wro=vDrgm=cn6=<TAJ&S%RS_Iu8rnPH>F^WOD$
z)zj~hoAX}=F#Hh;YfLq27dL8fTp{;|Vb=pG0gl8L4TqfglU_|q4JI5%Ht%4qzQDwN
z)40MQ@8Y!+J=3P_Hjr)*X8*^S*Vue0wtej`?Pc|Usx#6>9nYOhx~y{7wAq36vp0wM
zln<Rtb#f9!f3v8I)t;NG{i3x(%Of)5VY-pXgqcd`7EZb|C1>MFg-OXDV<rDB?~K0k
zU+>wZ2VDP~PObMY4~vbRBb{~7o?A?0&4)QG5v|!9LMzOkPd@uzkH@mQv~X|t?K>gw
zA5HnUvHJgxtiK;`zFn2hdOXZtBfa%?)6d4feMJu3TNuI~Rz<)5vwhX?=BE*5N4g&W
zFX4MVRr_(7+BWT~^%bR&)iRIrGc7;f%?^KDmHO+^@46qdyV-9ve7n78yG;G%)3Wk!
zY##q$y#4KYY+Ovzz3BM3n^X2a7wOfpR4TZg_%>Q~dv5!BruMI2|LJbt*3Xu6_`LXT
z-><eei#}&+cZtU_ZQu>Km=JyXi*h+*n5^ji;#W5fKg8czp5*t!BqsjfBiHvlXPG|C
zw*9&JjQ{Km9WjN!=Jhr8m;1l-Mil;Q=(d-rJh;O8xapDDGhEm2|Nr_pdjG!0y?e8+
zT%GQ7f<fd?$)S0@uG<;H7Czs!J<NX7Iv(rdqi==X@0;*hx7|HlD7=B|Z+^CM<|pU$
zulKGmPrYyYc~_m$`@h`>INsifs|b2J`S+x4fA%h0<DJ;g{(Bz7gOUf27g=}5W$oGH
z`t9Lg!^z&O-0qt`<2b`|ux|OjN-wwmYsdR-o~`TaywTq1%QnyaL10fus0d`*{!Gom
zllSM?<i9$==-e#Mzw3MZ?DxAbX+G&?u#b5@{mGqup99};UbFi*XJW2i7Ei&C<9An!
zuP^?WFvW8Dy*EDcZiOO_oF582<UWTO9q?yTX^ZCB$e@xUnI!4ck^WQjrkq=Z(}$2h
zqJ`blB#$m_xBPJaXof22I;1&SQAr}YtG$x9<lK-eJ~7|2C*bpew|bF+*Caw_m`L9^
ztR!PrGv!#u%)=b1iAmw-rY_sie)~>ScG{L5{QpbpDvu<Fr)kf0<@oVqwQO1Lrlgdo
zs^`94oOg9oYR<ESb00dFRR!Ih`I$GuyYPg-4ub&8S9A0>TgFcbnt66lknU=)(@$65
zZMa&0I^~?Gv*xrZ4&2XUJo7m|7k^7nPKc>Adv1Dg$rRRgruRIzExspmPrO3?+$5)f
zMQke+G8*SexvN`hr&L;Ii#WYCJJwjwGmT;U4%R<Y4{JK+#GZVR)X6Q)Jf$dXVaw0P
z@=Yt}E@fqO>azW16)v;r@I&(nn>2ExzFhcsW6N!$c>mqvGo5yvR;&@qxH-GUA?VAY
zWnVV*erRUtT3Wq&|0laQd$xYtm}`^%Nc}tGi$kR!PTF42wNx)Ce0tIGc}IL{e&udv
zca!*=PuBk0-27EIKkv)ZO1n^Zk?$w>OIUX*|C{7j|0R0vuWPgQ{~N4*^Kwn*<oUaQ
zt<|@yNi19aIREv>g8A3Z#${hx8~88r{<iu}pN~Mcxc<4`+1>xU()Z`a#+gt5Zi>HC
zUw-kdbx?Wvz0Wc;y5eGg&%3`}UZP)Eecq?I^|x=$zxMCOH($f5KUw!R5As;9p1b$k
zH{Ch&zlD8b`!AevP|^Bn^OwV~YPaprJ=k-|zfI=F{XNf_w{O|A>2d1vZ;<_3pT2>Y
zREf1HEaUU}W%2Oh6vKZr=LLk;+?s#(-MN~B_c#ACt?-T%sNeQoUFO$q$l~-h-0#?q
zfmWtGUBghpV882~<-(7e2O)ElwK<nQ%KAsm_~&zMRnM=`)6RzbrpT_(T^BXIhryfW
zmhSui-{Y-+g{{|$4xI4&JNN8)_a3%znSVFuN|UpwbH1K>-0Zc%f>@v1uNhhj#OEj~
za6LG_-MsGqg++52W~$Erspse8{!Hjs`K&vJ`;+ZvII{$$Ju#8I{OY;i^)0(sy8c|h
zU&lAfuw&<@zpMHG1pZUt+j`^nvF+w3Gd3~J5&9JZ*_io&&0KzOZP>S%4RI1nKNk1i
zHnkJC=1e=o87U*SUGHbvo6zZsv3&0@F*!C=n1K#Sy#KR&{mD61-9cW4TxI8fnzh_$
zi@es<%)GS4YWwa70*oqa<EBm9sAd{Cv)PIJ`i4V-GIvC-FFZ7V!i>$2WY!uetUVL_
zYUaB?I$vs@YbG4)sr_{4haS7qu4nm-x>;f_&u4YoHmgkee#Wa|Q@@$>&%ZMwH`PDb
zG^O&<t*OOFjYKWHn5KCw<PLlIc>fpU7n2G+jLgz9bM72{c-4#B@8tQ5ef!gvTFY!Q
zy;;8b<Q)0p`OV2&PhX$O>aoSnbh9(#jVn6;&(`U(9|vVPzP7kI=BmZ(3V0oh*+Qb!
zrtZ1+K;Zk8SCJ3XOSdo3FA?0d++@YEscTfCTkSZwpZMrz&YIGicE*TN=cJ8xO!H%>
z9SbH;{Oa-bPnY)+|K5v9lP|qG7yWP9pWaFSf(0j%UT=Cf^;zTXr6ShrUORQ&m4wXP
z#Qg62Vk>3bWIRiPDN{*|A$HULDLWGObuXNfvMc3=;`fR3cG@4h@u$4{c*vUPH|1_b
zY`yxKx9*wTjXkGkM|}A9&)ru1!SiSQ=DMeC)m~p&`#vvT?DD!@dsK{<<^OE8{QJqf
zD?F6vbw&8@matIX{&O{2sjuv({9PM9GooyFac#ZATZj2=D}Cd$N}sN5pL^@$m2J!I
zE7P7uO7h<;cYn5(SKqp<;?jeQ?9Zmf1=Oi8xUT<r+P{6TOl1~7Ugy!{TiSQj@Nmug
z<9FI$I|LnK=3OoG|MF>jO|GSXr0(Cmyy(%auDMg4pT^%w)HFZ7p3hR!;^*&GZA;G|
z7qr}2du!7f@!K)?e=P|M-(tPWH*0l1bU%5+<S$o0mWQ(|lrizClsTVj@_8OF>YH~k
z(QI9pzh}r%-2{gDfA0O%`?O<K{3;E-;-bk6DpIw@CC3?kXZfDnVQjYX`J*`7-*4XS
zTYcO0_`C+5Ajs~@+<i3$QIdBjiy!&MyvFwVQpd&YTNyV$pBZ{{qe{<wQ&IbS7R#%S
zox0w(=c?Gk@(MG}bxL~Y*H%^sK0No=dF#c4F(=D*u0FX|=+<%Bv!5r{rW}*~#yBCO
z%hSLoso_X#+t2$lykVZpy8m}5M_&j&{V!?4u?d|E1)C+)mRf|LVtutjz3veIVfB~n
z1}CT9H@#Opce`}|yFYwE@?uTf8MVG$|Ffoc#X{e*X_EhLGViZHaW^PRiFM8nzToFw
zjH1T&L4TyK-<jO~!D?aik^deW**BYPn=)mBkJNdao_b}CU+t6jdG)UM;6Fa;>Ez^_
zKRW*f#2ITA{485!zWP;!o9f9I?ia&S_6NOvy|sRoo=;KmH(SQNAEtV(EJ^>D+oT?P
zzbAdJ4&-LrD1(JFcTRdcEA#I`>&%_en<j}eYq?7#@6><sm*J1ZFAq)b-#t1f^-mr;
z7_+yln{V;bN7r^FoY9*W-yaK}(m$DzDH*UM?OE6B_=T?AtSN0=#_977-X$~tR5I1A
zX1sb}kFd}=|Kb!*i2%bn2_KX{FECl=-L*FRXLykaTbdMqTXs9+><I;Rv$o$mw?ojg
zeo^iBcYCi_=s#ax_D)msd2B?1`^T{RnM?n!Dyz8c-G4T3^V-0dg^|~Fe(uc3_<40+
z>5tBTkTvAtpsT$9%j!-LVsMjtXY9AqUhDH-yY~4-$2s$x=4MD{gx*V&dG~I=RYBgJ
zPiulVOQo%4Vz5rm+FxBQdrZ#g@uhmto0lGaaSKnA`F8r~=dik2jAk#-bH0wdQS&^0
zd(MryUq!1Ht(!h4`fvFw+qn8Ged?d>SM9oPy|AL_T82y(qc>N+#*3H<+l$`(x^{cB
zz#(=!_VZ55>?5AlD!udiP*iR9c1Kclo7gl??fX;AZJ$qjpeq_a?VMhW=q?pqpT3ya
zbJCop=X4z~dm^If!{;QGuyjIU%fAP+`u41USoqYvq{sf#k>o4(T*u2khp~63TC1oY
z_`hK3Z&mfb1*d+&7LULGpw8wH^OEslQ^M7Azn(tQU;m~jonL45KigxBT|7cXa_99<
zywhqvbs$C-x(w3u@U6%|fy&eeK1TD-R2be3jF|dJ#xwuOUfC+S>($fVS*D-A>AP;7
zxn(X_l!TFCz3M~W^oT-<g=gkXa%?u*FyXJkg(Q2UHv-rHq%-**Idn9JQ?1u{nMq~&
z^hFAio8y8k1DLc<#$>MHOLs2xf9t(1@a<;yv}a|X&zK(M+^o@RyJd5rvP-xB<4OrX
z1~o<(!#x+d-pty(&E)WoH9;rJm-_d&W}cbxQ{xZ6={^z7%Cja{9V@Tj>p!b?#`wxH
zTML<qygHq7EjL;$ZhfAUW?j<6%s4^i%Ebj!zC4(;R56T8&SG`fZv*CK8$*Q@4yreO
zk2nx}cD9w)zqdbr33&F$zU}4S%R3|Fn(~GfZ-jea7Ac5*HEdSZJd|fPEuj4B=2c!h
zEFb!|h2CkJAGD)s;q>5`AM0lP`}p4S&9$2gW$!&?h!y(gky*O$^asuZC*3YEMTSO2
zBrG}DWhVAQpknW#sf&1@v0P7Z+HpY7S?8qt$+F7uP0?rKKgPanpZ_S`G@qy1$j@(1
zKux$~=)LRn*6xY>5VDj%q)sgB{99MU-_4balciaTz4E91-uC#-1@E_V=gxQS+<8wu
zW1_(7SMQg2_TEd(I}SQZsnqe+iW^%O9<H2!Gb%N;uV?0c>wj0otIjyL38`HC{9h-g
z?!o)EtGrfM?~ClYY;BMi{{G6s<Nd2{-7fIIb}n2_r`9GrYxDcOziR(qH(j&ee((CO
z;y2edoi88$lilxd@Cx@OIp5u@-WHzyeJ#F0!0`_2nb+0lZw9yDu`R!szVG|~DxE~H
z%jwa7ZX{pq=bpuTzxw5_i`k6%6DqI%Y&mzmG<NTY%lX!4Bf0q4161B!KeK*z>BB<E
z)lW%>FKwIK=jHSMP3?LO9-i#mzdMTi_k3BiUcc`3r)l2j{uRF7zSd&B&etC^gjdOa
zWaBVfAiL?k{@$ODsxNaNkZoPI{LbAfwW_pva|~@Cmp}hrQkLqug2&RsMz84|lXkP@
zzRU0Ke|V95{z|oe*)!$*9EJslydKAQ*z26*e04auL6U>Rg`366;JI7(>;Ds9O<$}q
zL-^6}gohi#6V@$|<oL(y)iZ&Ov8RPy`kGX{l*7?q4|a85zke=XlUZ?dhfQ&0P52>$
zl^Q1+>y2~*eqD6p`l-q0v1x(9Vcp#H3+($@*T}!<Ss|kEOiZ&-v6)TQ?~UmB;)PFn
z4$lc>*--fH<}ATU_j0&?U(;c^c(0l5<SV9IJFhT_*X+&s)TEw!fWL3y2@#{|l~+qX
zM6?G;UyO6*O<pK8y|+uDfA!hRX(gNE<k?NG@yOkH!FX#%$jQ~#to#4x)*kvVwtD8T
z_}CwY*$jv5nZ8-{O!%!lnM<+$#G582MF!Css=6ftauLh{S<Hv{ol4F)Xb0&!e(=^g
zsuwMh^wG39SjItvamKRW>HD&kBNO#A4jygg?~8h{uH}ub^?#q=I&XKey;q*narlFZ
zbE}O&flFGgy|3|wpGnzElvdn*`r2Rp-@MS6ugA96fBCYLW%t3bqx!xbyniDv?%i{$
zI_FrjW^R*xF6Oduf8Fyo7YyDXds4=0pMU$M(fPBJ=gHLkUY~sJOZ^?4Rg7ia)jR&K
zT@?8*ZzgYYy6Ct5ne(<Nehw<ulXTI4^ozCcRP?O6c(<r${u$3}k9Dy2ZoYq`OzYQ!
zb;3$F>`DV(J&d|{>rLtBz1-TL=bhe__;$%d>EnA05@nw6N&IqY_NM#s^XJXESmSeD
zSJ!<RFSky*1m}5CL)Nop{CZaR&9`}4{YqH6_Sn_^*MA%LbI7?xTbuAG>N6@#{8BQx
z+gfB1E5qYuZQ-?s-*Uq?xLi!pkW>mVF7EpO<JIF2?aU&F-0Ec#i%)y4wpq6z_}<UE
z)6@0u&*uEV8Q1-OlR2B>kAK|qIvS_!d+&>!pIpvx{X@T)Tz557L+Y7xGp6t4yu{I<
z^vC+>Px}d}$9H^SFm-yeN3ogPRx-?U+DX+qhokjdrDCG=rinSMusyo?^NvE$sV_10
z_a^PC=jmN?W0%MR{>@wmBR$VE<O>-!_V(C=?pl%$b>11bSZjghf%A8g-rHYV;BtL`
z+O<_}6SsM6p19kg+q-M7B4aZPgO<3z>!;U39tJZQ|1UnMYdmA(zn!jWO_{e9A9j5?
zyg*wjAvpTNi-67l^<F65KXh=8eShhTdVZaG>k1?PRG&&{OPs;B!o;&=-4s>ENx@7z
z`tu_`u$~W|$Dj0Sp?!MO$>j$uTDA5t#>;GBw0zF>M<V=q(&O}OwIs#fM9~DH85*2h
z1b(o$b+O*Gj0uo#U7c`JM(dsC=amQA49!&)#U4yvGv#Qx`I?)5p1spdzh8K4P24M+
zvue(h=4|{njaPRsUlOx~_UA8Z>D$V`+&|-R;!aaGZ@@_ztzvo3IdgV7pNg&c@pxwM
z`gc-aKhC(Cn)y8UudTY>H}w`Kvlju8u?G9IeA0GrIg%#1F}`(Qj;+e$^?Q}SS56Pz
z|MN<ICb!YPs*>Z+Z!Ukr%eq6V?)HbR;;|nxJ_$=Nui=qMR_0oNP<r<6SvMPkHb4B(
zJHehs{*>~+gC=auIve_47ytTZz3ciaVX<%3Z`OQEv)Ab;DoE2~{;Z_yb7q0-x9jg_
z{a%n$`0eKJYEgB^e{puF)pja>TlO?2bAiX_lk?6l=(hc^S!dNb8_{Clgx;891_!S%
zDwS#{%FeBw)bw7jtoxr%s;W%?)+5zb6>E&v#}sUF6MAc=;PpCRd0}4Ix7D#fE2K|6
zf1E6TI>bYhTjH?9TK>aY=Wac+<H?WtxATj^%2%EXo<6DWcoqF=@}wjQ9;utm6$bAz
z_5NsW-d~}k`&fIH&FWcC1zm3i-&-3SA+)lhcFF3=?+t(b%(l+Ix7t}U>|n|a`JTfZ
zKg(GX*veBS<G(Q|sN6Qa$jy0P;Va);yE{*FG!7`%ag|?`FMizMuw{nxKb19?mfXly
zxfMB!X+wMI&l#JGmfQwklg)C0iKqArb9m_9FiUCwlb;UG@mD>~n=SN8qTAB2cV|+B
zYV9F~u7J(@6B0vjzPohai;Ig*oaaP-`}~H`hNj=o-|c_nAQmBBS+G5E!j9JG3cneq
zo#@fYn^+kp!=;fZ`fihm`);$|WvcRZKj!tkKef(!uWS2y{xiD@E6>Y@?vK0vqNzRj
znr(Q&z0`%>@1*K3%CDTfUb-qRs@asEZ>RO=XN~8l?wxG>`^)oX`-1-WuAD5lYk$=x
z^>EqBv!}zirM>ewq$T#b;NF^sZXv5vN+*i$?>PEL&*X6+!&0^_*6BYcSRYVx{pC9I
z`I@sa#%I_S>sRb!{rBqA;k*|$_a)6XbnTz!ab0L-<^<0}T6&vLzm@8^RFiejg#QA+
zjKKBs=PMIMGM38E>{Pk>CuM_tLIX?Go5`oyA4hR3KS`II;rWsyvQ&%by^6&~gFQ0?
zq*DHi9%rflDC)66eAx?wv%(v?xRqveKej5r9UdS3*8F;pk+|SC_CPby6=!tQW>%ba
zUXpFt-(B|pkJ#tsXXl@(&rbJHuv98OFuOKj&NuPewGxKTru|!IK2Hw*aYOZ}VB*8;
zw^DWaYWZ5;?(cqS^`p$-JG+BN^3pSkH3l_5@1;A>uM9R_$Spnplex#YBRQ!W3jNvZ
zwNLzIT+MKx;4F_s6_b(c;h2pMt=A5{<>{JnJ!u*DpGm%viE9}QZ>4^l@?^)3PgBJr
zPRzTwZ*}^$FS^N#Ik}J2e+s<o`Yfh#|EYUZ{ziP;*ejn|{4mh=g5FQn8OMw~uiBk7
z^5jojb~=skS+?XpjYxfC{$-6ax#yzfKW~omHjz_$x^+?IwPWc^%KzW?WfDDUnpwV7
zK4$hzy*$JA!=9F0(>6``Gxcn&wAg8#m*?IFU7PcHO~|*C5*-=O)};KHe{6ZH;CX?Z
zXqKA5-><S)|233Op7+21N5xr*<uB%Mnx3?Nx7g11#2MOu_z!EX<mH%bBgEGcv-cER
z)k7<v6Vn8DS?mm5G;7*T6`x7vs^Q0UXSpppmh7?AF_&G*ls9bZJ1$*ck99N8@7mzZ
z9(Hxzp~Sbh4&IUH4Lg_<^83S6=B^W7VV@%T{uf_^9Gz?s_^qn#!pG$QX&M>pHXWTA
zwZm+>_qv&dGZQ#s7r0*Aki;SKKV|9qTbhF0Zl@nE)&lJSJIHLIeKKZaVT3|dL{V&W
z=#I4Wo9=8dTlC~qR@%AyF`71QyjRy1^zM<><XbpXWZi_?8OtXrwCq&dlCi;S^1%x&
z7caU$JK!OZ8mB4$*YqWyvBo6h>uK4{Cru{HndIBl=ZgFm%-tvOcI7Rm{EHPG_atr@
zI4@;oIkaVh{mYq>>UoCTu^FaomdGjGPo36zka_!wtA^dG>8h0mD{W7_wb7ooo?-bD
z8;Ae1JR?{)9LZ`w_sS)1n#h(1W$C&f=gG}0-#OPfeCeCr+4gG_r+$-r+x&FL{$BIn
zc4mz2>$b1_P%5|h=9fmxGot1jZ!kxhT<cia#JhblXOu@?*WZkv#<S!(KbX8bt+PLI
zn@6P1yvnz7YqGCDUwEwl>bqAa`KHld7cwg^&CHo9vfAKtRD<@3_(?}wO~tz37qy9=
z4q2eyfBL{lPU|<1rNwF(dnA8l+zd-A<i2_6?Tm%T!W5Sn6mHxf{^;k$&rt^MoD(*!
zGP%fI$91q#AZvT##v2TC7-pWa3<>?rR6kW^`qA*~{gTI%-mSA$+|e4kW`Duk^w<9-
zLhN2#bzHporsRw%TOD>h__xhf_w)AjHRm<*-XuGg&5)ERkT>>E{g!!{<K~&;GRsyJ
z21iYs{%wkx|7X+lDwnq2E{I7xKkMe{N3$81*ovpi?M-{jemtjP^0oNn)djiFcHCHg
zvw3H$@8rb>=OdF>{D`#VT9>--n*D~jBc>bKmq%=y<bU+)x|m6?rA{AN;87`NepYAZ
z<KD!$-Wv-u<`tDKcAF%6XVK4#zcTLc_mxOolFb?X+-user(L0Uf;UCS^m@&fc^+@*
z@^JCmoBY$iPo0;_C$Z<`8SS>viA7#gYm7Fl>J%xsn>;U{S-b0*SmH9V+Y6ajZ@HSO
zRxanzG}HF%>?x_+Z*J_-znr#AzH)_Od)fvSQ-+I^Z>l^iX6X7LQooVGa?(#n?-y0?
zFI-^!){uQbtLMb8>IT~e!;F>xT_4@pWSFpP!OP>exlW9U&vu)A-O8}KVD84#_Upb#
z7uU17K3G3v?IRm)HV=*ik57ePDh_>q<Md;;*@>c`Js3h7d>T9%TxDcVY&gZqD)YeJ
zYG+IF;e<{0pOXb5xBYc1E#}ipofp*qy4*72<!bT$!K+suz3G^dXjqar@n*mdX-$))
z$!n%a=4GA{IzP=<OnKT{8)4~Zr)tjn$exevTfH$ZDr|O*`?Edzxl7+1ytAk<_13?P
z{K#g{iyta3&Svv?lPLQAy28vziMw-(Q_m-TjJOjo9CUq;;r*kLp||=n=ZJnc*&HXs
z(lNgwl&2ylJ26AWXw8ky+HVB>7hd>q*04KyUfHJQA9RZ9mW6y@v9I%{?z@+3CjQW?
zUU%~0`Ye$j&npa0O`lz<qC4NKIq_TVjs1I8e>Leow0L><&LlSu-8Em2PktPu!)BQ#
z=3x}e{yhJ`S!DE?Bv-8^a+>c2_r_G`MlSiS|NFY$<y8CmyIw@@E`B!G@`{C?@%tmo
z47Ug!+IUKy*(}B1ajU;k!k;xE+(%-c#5}y!w$sVKxLYJ5UDvVq<t3j_ewR#smVP#!
zmzr5RGxX>hk*E}f`88f#u?2Br+fUnQZ~Jrm^YW+n=Rt1Ai1pt1-sA4w%Jaovt4+DL
zzRhM9TEebt@3pPCZ(7Q(H)nYsZuw{W_()dw&Q0=n?ZmY&{aNQxd^pE#O=Md5l5*v*
zH3b%WynHvG2i*Me@L!a_?^*v%&Sq9Ecl17Yn;h%>ojo&llIrAtj3EMC8jMpdUwiI8
z{h6otd(88<$0}n!o%MRg6`<9#qgBK6%z-nkMf1K+sphf}5qNN_c(3}w$%*H~EPt=r
z621j=kr$V6UF705YYnV}l|!9Bzn^I9{yb?N!?c1I>1LeQ@0{~zoKWC@&ic_Cot5vG
zclw-A^;A2^dbP(ZZ{1doT6>A}_LrOsjyy~;m0GH&bIH=+^Rl(JvwYNlUpjVCB;L-V
z`o*Mw+u0gR&HdA6N`z%rx;MC;+pbkKlVQF}N#c&yYzNJ>0|nvF0uo(fm;(&gJX&rN
z!P?OBz?L`c;kq@^M^CkzDaq$-S|xga>e7~*J7f(w&xne1Kl^lW(#}Uk2R5zQ@YzgL
zy!)nZoBt+;jlSnZuSJ(c-kj6mFBhM6I^d4jZ-%W7y3D8MTwUxr`LuxJxr>t)c%Pe4
z>(%RcbN=VuTS9Nk?^Z2OU;HVZf8(3461h`;RUAK)IN9vZl$iBr*Uc(F+2+4`=FQEy
z7kiSPrEkf;d7sT<VQi7hrP)^=T|ToW=!CM@j-+|a_StQ526|aKe|PWDoT$w@xn_FG
ziX{u{X9Vqka9l!&<J#%@Z|n8#eBvf>duaxoZREOMEo39PPMT}YB)Oa^-wR~--!<2~
zF>C&&^(jw3>3?&Ho*v0L;rOJ*x7+y+#XR6**OUxve{8hfyR7@S_k^_i15X<q*qn;O
zZ1N;d{s3>P(y}z4R#+az!Q!Uhn84N;HUE~>>TPG<+~iSvyD-gUVLyBEUiRa!l6qy{
z96!*;C@S)E$qAOjDv^^<RfbJees0IJXb1lSZf1#sm{|#IP0VK*8})vEJ=(ZhTg>3X
zp;E{kLQT{DNsr!4Gzwo=StmRrblQPE!ViQMCO=uWL)M_P`M%6n2gS$ls)GOSUC$EF
zAYgQQ8ME5P9j#$WGmh$-v^pG*UYZ|Qs=wuS%ik+!0^W&PGgszGemX92LhX#tAr<3v
zrsSWcx!Zz{#Yz<?Maj>e6>GCda>9(}*H8L9H!RtD`@|!6tqqFNISq?=4@^FAyG?Sl
z_(OyBuCw=>9iBDQH(~)xLeLkdYOf~wlm89;Rh`*)#-0i{STa$oiRV(v5BcvY4_dN&
z*EV^2mjBfGJ>`-(AJ5_o)|~ealzj9t`Nx&m$kVc;X_|3*YjaS1&BEu_{*9k=TAr5Y
zPWiG$z|MHd#56wTlCO>{mV8kQTh+Vhpt|m!3u*E(iH%V|=1ft3o};a)lc+C}C>H15
zZL=o3N$kzRmzg{PQ*Jc~eo*w<F(t?Tii#57zM0LBURBle%?&tKfBnqPh-dTC)Y@H*
z&MbbQ7oof|XvSZUhu)&`TfobA3lnF5+AjR-k>2x|kDJXty|%G>=M=*H?_uPo2!m;x
z4BktGFb1yndYvr%@3`)S^$zo<R+jkI&Gltc7u!BTsyVUDnNzT({g%*{3x}8v#)$H0
z-<WY<CppaR_nEs<tSY(()~7kvZhd|8PW-)Ay`+2B@7*b3H+ZgR|H0wp1!m_}(#OA?
zm>XP{V(|aKgD$s){kKDw>i0<A|GV^<_Mf*(qDvnzc+_zv`mL!8+gxBLz0CV{{O(7>
z&q8nfw)}DF?bg@EmnPVMzUxtatLJp>4%f*s=Ic+(d^%ox=F$78fcf2w!k0vBii<T8
zBnr8Gr`_eK*rDFeajRm*Opjfg)sG2S^WJmOD$KL@%5D9B_00b7(8XB2cEx%VdZ#HI
zp6y{gcgELC4<<i5XxHy8x88Ka64@h?53a6Zn||}vyA%KF6ZC_+(~iGN+P?JB{GD^p
z$|cR*yOC|dp3{}S4;E`}2VH;~WMF^cVmRNAE$g?Oc>G#tb<{@ry!Iok^ERE|`RM12
z>!(9EEUL7Y>Ud-q^U%vlC*AHyx#j!3x{N*VrOxfY^h{xOR3?w;I@1o-;*BatoX%an
zCceFHzv()@t4!ag@#^l?`~KmNZmYY(b1kX=W{jDp+-EnP|JU?%p6rZ=vR$v&>T<<t
z)nCd~f3W5Mwoh9ZN^E(5Qd9Cp@f2fb@VeB$%)((mwEC(x|JxRDaKrQd8MD6ZsLX>b
z@T>TK@pf3w7kTTwk4@X{x$e5MM85;i02IA%Hstj5yHT*Z@ay}p|8mmK=R|QyrSISP
zHYCQpe)myT-SXQFdI!A2+K$}KUH{Bh#A@r3gzM7Ux3<l?pBcXW_fs#^`QNIVCvx3%
zU$0TI;_4Nn^puljzGwZJjdz$mx5&KJP;qUJc`934RIB!Ht%y14OAqSWr1CHQ@SJDe
z;-i+~X$?mreY7-;(xR{WNME=)lOu6brO52vcid+;{|(-H^zOl$2Guj-j0OcW=7hy|
zJ=IIus{Q5Q#-Lw`0Y@GiZQ%|+6Q1I;@ZVI+e(zZ}Z%#dMTdUr$RLXaA%O5@I)skyA
zac2K{-x~VtR#K^Xo{6>E(*KL+_p?=8$Xj<;B~f&rzoeMPLOu)m70N{gvhz}}x-f{I
z+sTx15`3~$LPLuk$Mea>kChHJa@*cDlDmBEXk)z$SB7xkiK**TIu_&@&StJV_l4hE
zhuwi`ee3R+fQaVMIl@mvb{lO9ih8PjZtA3+5j+!iOfa-Ry|67xOe}25x3Wh!|1J2d
zH`BLGe9<}4=P@UjpGmARs|+wJzj;PxqUDqK+a)8n%b)o7<LKA<V&zp^U(TQF_2+(g
z?)<RY^Xg}>*#(<cU;ME)oTWa-SZn=M^TiJi9=Ll?H`4g?yH8ind+X*2WHrw}dU#b?
z#q7tk7-#&<;EVWrckTTr^|2Ffug>Xwzf9D({`}9Q<}Y8i|Eqd3{rdko|6WgC?ti*|
z(^qEky_HX1PoBrSt}><c^VI2^pU?7KRR8PB^X~NcAICdF-T#EHd~5kV{>P0w&*ab7
zR^9i0zOQa(&Ewr^xsjVz-8?>be};{|b$P@^+k+FMJeE9*zwER{^VFJJ^@&YNvjxsJ
z=RfmNH;!L&b?u`9Yrg-NnV4?Q=HYl!;K+C8v4bO@PRdc6ymtSYy2ozD$ZR|QU+?4u
zr)w5ZZ669ah6^kZSd?&{XVH~#*Tj5_JG&1&DK+#z`oyBKwdBim)mw-6=x6gfY-Ts!
zoyw<aa^Ag9<FMdT_R8F_|CiHJ564$R$C6dETape=3O<y)srB~H$jFNDUlUDNs<1C@
zmAba(-jpr3J_*|f7rtnj`8V3Cde*}q^T5~eJh^{&=8Nk=*VQCfy(stpl~Triq5G$X
z*j|Hf=LP-$mCt`_vR1f$?qU4}GZ%k7mK;7-mjL<yZr63%S8eMNWlhysu-S@ni-ZMZ
z)Pg$(?c81mly9nUHfepb>B;6>yBKbpArIW;S7n`yu)DGBT<F;|arPUw{m`q3+o1jW
z?yC3pr*BqGdmnp1<>uypjZ<RYhvhL&IaK=YvbS693xn8Pd;7HWhRJ!a?!LeIxGvv7
z`)%U==v!+Ju)ejg`p>$P`v7Cz_cu*lth!>gKMzN9r^iT@S0?-ks?Yqnq;{iy%)Wvb
z{+iKkxwH2r-#ya1{jRCz!iN`U`Y-iqWoIyc|JL(u@MftqALAu|`55`#dEVW9FT!kX
zxtrf&^{BGFZ*DyFY<9Q4-_MYCdzt9@FQxOsJ{Fv*eOz90tKj0NXXbX*&mXkkdvCb^
z_x0GcjrD659c@foJu5DJ)7qNkr}5|XYTusqwO{`)av@vm{8LP4O|@Hd-X$N7&#dD!
z-R3pR*I|L>6aQ`5I+Me`g?~!;qw{^b&g5-+x}Uu^Ma#sb|DPhIr{T<g+GgpzmuGw%
z*E+RLRm(nj??Uk79cKd8rd?faVt?*jbh^gXH=1izgH}swr*1Y*d=<ke)hPH#EGFBb
z+txr{)m8J1llYvQM`fhWyMHs!pU+#K61?#Au`?S^9s9p<flj*p<mQ+E9~@rg>|UfC
zG+9TW>7&U`4vEciJJQ^|f2Hv8Zs79I+`x5D;z@$ti{M+^OOo%%*k~t;itud>aOmMa
z$MyEa)Js3KH@lt++gPYD-`fweYCLUAihP0lq)xAjB~^7tRsAF$#v1;8eE;!oA)|$?
zrK_LtNa}b-v@SRCSaxvpg};qkm;*(_(k5p`J+lfwC$nsNeUOMz^|p(gQ{H~aIb5E(
ziT%uM#oZ^jD;3t&7&iU7;eYk&(VYv$cJ$qk3Jv<IxH&ZAyrsQ%di>tC{O9xj<$PRq
z{obvg2Rig`-TVFc>G}JYQvS$3zt_WH!}}(8#^kxNUmm=98{hLg%k+JO+w>{okM>(U
zdZXrh>0S2T4as@ecb|W0WmB6oZS$_|;<UV+4gc1xd1)=b<NfE8KZBq7W&b~-%s*wn
z*`t%&-%C%wS{>UgdahBTcHX&JW<PhXp1=3w%Rc{kAFnRB?rv|H^g4RC-<j<BV)JLt
zu{*TpO?b`MYgfJJPE&UOZvS%9?dkjfX5XGatEhg>qpeqVy8r#fl5q9=Rqu0=_k<oN
zh0Wh@_Iu$?^X#u%w^-g-Et_?JDsOe#<^LB9rAr<6X){i3iJr7G>4A#zf**R_o;xg*
zf<9mFV%q#sqru-ssrp0ychL2M_NunKop|Lzhp#@rxl2MTFl<Jm5VJ~*_3G*!yS%r~
zSJjh9GL(2+Z(n#ev1JeQw+BaL!j>D_e@m=0O+I#JlBk*P#5V3(55>Nh91uM$=b(I3
zsruBSy|d)QA0~7sYcQM1xpLpT%wsc&w_s9Zhr#d1TX(9TKT`Ma*-baA)wj63lg~d3
zmFLo`xp-qC+f0+mj<uH<>MXpy?2aaIMg}!l@$8cLa6n?$hBd5*wHh>@1Y}-d-MZm{
zuk_J1!W~y{wlK}QZTVR%Xu(m2hR@4Xid7ZGRx??(wrPWgwL<SmiTB)Wo%bw9o6YZq
zCZo^}wR77Z*~RF4|C*vR?TUKOy_5%DMXOuB6}l($o1XL9cG}^f2Q$;$OM8``A3w$%
zzk5Lx_<ory=aY+8i7V%?dhzGb%z5S?H<#pp4ZqiZvG{oR&XfCIHbnoJX#_g>Jltvj
z9_i`#p5K)FUUo`8bHC}03y-|M{I7np?P=@VqFqy`ADg2X-Z=khym-y7^*l8q-l+$;
zz|%dv3$jmt&o0-`pH&~`nOXWCvX-W-H0FNl)$4oDy|mu+ed_itbAG(rx>`ES{6$Fp
zW%ax1+y7L#e>}z<FMDIplQ}yd+gYrtyS+vA-{phNvSBwb-hS&=5D=O!S|4U{Fvjjj
z`=8dvqq_a4t$+P2yU)9C`TeW5{%i)1*XZ6~bx78_sJVK(dF)PMU02bcs;%$S=FYlm
zkRAHIwl?ZpY;n|O#qj<+r>qqir$&~{OJBNmcI(~TRi!sM_R0ToaP+mjuy3B|I!T?A
z%Oxyk9%lVH>+c4o`66>B+>H4nY+ES4X3CSrbDZ`baas6RUMu6S@Ma^m1C9+lrYSa^
z%i&bpX|#hyCHY(DLXoOR>-LnTB?*2=dsbk=dtlZ^PqW0jOUt<@ef%d8Dqy<KN83u~
z<(egn7W6;t;&nb6&@QQ^+0!S+7Nysr`f0typ9cY_&wqLyGkXmKgN2)?i(|-E-v5g^
z^D`;~Y}ngajJ^aKe|Eexb7!F1k+dluz3yx7=4e-#t>5*|=y<}jKT4bbZfCYP+iW%e
zr{+!fX%cA{j;7X}ofo@nt~B%GQ=2C`i(Xf&2>N=~Pgb|dc?a7SNBL`<EGG=ZcO=D~
zbGvY8-o-AFB*pVx*>RbwZ3n{`y)p|<&nbTT!`^z!Ve6P@VJ9!xom+Ux=JW%HpO=4{
z?bEDZ{N(bjQ#%hi9^8M=-t6Ab)7llBk<U*ZlUC^EUA%N=?~ZN0Pqobq!g;HX_51!<
zcW~Z?*rWZQ?r!nF<X-K@=yi18j7tZ0H$N29`EX&|>@Z%9o!ctn1I_dD_iZ{E{Wbl1
zwNh=iTccX@ynEJ(Q`N6pYkXU~r1;0pUi;iX#gFG*ciZ>#j<x&#KL@+xZNHzG9$stu
zdG*@AY~T34=G=Pt`kFA`>?@TY440P`rTu<=<mmCgd({4)KlGISpI$^{jqlN&3~96N
z@BaPPxa6Mvx6FI_e+$jG$Y1|+w$4z!{$}oP!+(#f=KWE6zwnOdiE5qsJHMS=qqfld
zk=PPG&Df3Kn{xTz@QcM2JYe{=wP4QC{Hl{v9(*{WK4*Kt!<%tNJ6yXTmw)xNFMOo=
z{_C!_SC34&cresv5wCkn+O8$$n|bz{b=5i~oINFf`-{o!+Rnt!!IMlcy*o9*-i*y}
z$F!!GMhXE{m;Y@3xF*6ixNFbnh(se-O@#}M^(<^LawesYAq9&rINnLk)9SI<oe&rh
zcuKAK<e9vgs#i?oPRtAL3c0WEvsXWrFDh+%WZGRD?awteLc1QYHHb~;PjhIs`y=xE
z;tXxhQk9xVou4)f{iy6Y=Gs{KOp00U_hQ>iIy!9mJVIycauWQPBpg^)AIsb`tI_Qq
z?`NLxXIJQS%#PU{>QVS@*X{QD(<(0}6-@c<5y3i{!H3J~LoB;aAY)QW3gp7eF17fB
z0qi<rpbPE;j&I=F!2W?>blsL79WxEdQ-!J>hh9uv#o;tfwxMgMPJ`aGO%AHsF^|>T
zSr@5$`)05`IGEshdg`Q`kwzQ=-zM6}-ank0TFhkUuPbRB{$S0HbD>M_Wlfu#I_-GE
z<t10o>bU%k`xx7O_{nm{Fvft!(^@M|{l2xi6tdbX{`b4K$$Qu5+}@G&;!w0~{&lhU
z+ONX)Zuv9ikoK&7_lvx4&tU5o^5?VDecr%a_3+TAbMu%pGIwb0Fu8v7JX^QaG_#zt
zSBcBZY_`7Tob6Y+=KJ31yn8=Q?esmj`{|xDinfda^DBQG4Zo)ozxDC=$zt<krQT=G
zwAuaZ5u0f4riy<IPBvEG<Qa6ny`Hl0<>gAdn!TUS98Io^2#XE6mGHkhy4dVn^yhn*
zj_yzYUzh*u)sNk)nYEL`;<j(f&Wp^PdVK5c2lr3et&<Uzxt!qp^yE(F#p`#c^2uKL
zHtlxq487UAvL<@PT*;fTuqD*!jDPTl$xh-?ebXdAc<kinJei`QAnhHQu}ShyJagb3
zVLQem751aLGKa*M*<atkC^q$(R~w&^`O(PGa~(~M7o*N>U6NDsZO2hHoq%R5v(%cc
zJ8v?}%?>(RB>zO}sn3(x`E0Q>ln=$WITtaW$tWyPcxSZcl7(uUNzIMa4`P{~Ni&rD
z9qrF1s7Xa~PwQXLy=+C<Z+GEuVdAG+61@zh(>rw5ylDQuXny+HZ_gh{Fl%iN*xeGE
zmN&t1dP?(0ogW$*e;?LnKCE0PtLDb-&~Z?Roww=kC6=3p&K{`+)-1*c$_*EE#dXcS
zsg+l_X{|;wmqM4Mu7f5Ycgn@GI;}I&W^K<4|A{iI_|A!Cj><VxTlxOG_)@(OS7tWu
zJS3MTx-RQ_R+#u{xk-#gKi+@qTNx4m?4<mT_s90eme)QkO}WOn;=|?4pT7>c&-?8C
z@7|ABX6yl#Rv~NkPM6hH*~s;$-(NoUyZZmNy)Pzh{jn-5<>mC>R=;<dTU?wTx^c@E
z$sKDBKbOD1`un8d`!=eJ-LpUVz2ML6YRWFQzx6lZxtRK&Tgm4#nSc4%{WWfw30Zml
z^!q>4ue-O~uB*KNi#yHsulMBr^-c=%Mw_nH&-oqxp}c2lp`-QVYpv7#ZI>OLSsoYj
ze%UqQ_xs)*)xH&T`{yzCbrwZ;+83AS{7k;iUs?M}?CrO_pN-d9A9C%yG|SK<Dr1FY
z#!1<0&ytx}Y)Q{OwkrFwg|=A(yRNl2H|v#8YnIHgPB-7kr1(3Xp?QM6rSrS<mf?Du
zvMrxG57_@mxmlwA<=wm)Qw<rPt@*b($A40K)9ZBtDdnE=0oF1eg4}uRM~c$2C%$hH
zlP<N(o0xxj&4zUTnI`9-RnB_8c<l)rZRwv}vzt!XowPm~;P6iQbwFlgy--r2|4Ge{
zTMBMWJ@2M$q3HbY)uA<AYXALNmRxOJQXTum;wtOoaO0i-GtQ_n1aUK^IH>;q@LuG8
zs;Wa|z>HSyIkUpQJY{~eVaD!bc{8<x3%Q+j#I{dJZM_+NrYQcxZ^i|&5**vFe$PH5
zqI>w1oyY`N9foNs+%B3C*O+u)Fz0y*A5M7Yb$Vw~hKe(b<$tcX&who!iR2b^v3@qK
zYRCIWU2)suKif;TvmQDS^YE6Pj)BzbNP~rvlRRHstw}xAz~<Vx>Om>*dUx-uA1m$F
z3!h<gXq_*azBKN|&m~DIrHe~BAFP~nw_^FLRb9UeV*f?uOkOs>Ky7MDyU@8)7V8s}
zp4m#Or?xP7Pl{gZ<8QhD?1^XluiozvJGs$8eZza^y~S1Y{@i?X-fMpi*P^#4+MJ$=
z{+hGD<ioekQ8GJhU$=d|?r&LE_4ReRqVnq7bDo{9x!b(<@ox3E(-N2scobXH`>oXW
z*V%^s%q~rQE!lqm>bEW1PU`ND*_c(Ea9iU^gJTQxYhT&BFTNkPJ2hw4PLtx<cPeMU
z3N^cKwzN0*hB~j#&C|QHC1XkxqvlQB`9MECv{1hH=%tjKsu7bmJ$p0r=*_J<nmgTE
z{ulle?b*CDrA>7Ejw5DMf)3dTho9A6skGt+qm{KoRf7%Jq5o$Zj{ht_Ji|8iT$mC2
z`<XM9in~wuPcTi|JhR_+!KH8RuNR-aBPw=+!R?v2X9G*}vhMY&M=tFtOx3+p8p5Kv
zrqA|sVc0X@lgx${%<nokzS;UiWkUC|Jkie*|8<Qx|G$0q<l<Gv$K@Z+p0PN%<%~94
z+9v*HrE`Zaxv{t$j6FB?kzLGTuYda#Tnr$mNl$3-Dhkk$*GWHQTiMHVc*lN`lN_NH
z%=;|z9%hAq(-Av#b;f2VZtqBi00G^Gh>S#mqhFUDh&kBrdrtIB;$-lAvf_gPqrfwT
zjTbK|OiMXgVE&Hh{;37RzgROXT3AmdyS$aU7Fz!A=9^~D#heO0`%JejSrvZi`$k=-
z){ox8NzV0R{$Kp~Y(pTc#4at_vG88+kIsLMq4n{9e<nVi$u9NmE=R=E{Rh80gnIgX
zZ;8JYZ<l(lFn_!C>5pqRIG>m&we?$@_WP3ur6w*cdB>jIH-AS!wCNt+zsWyNot`)E
z&-K{(zBPGoHVXUg`ntP)Uv0)6=Jbb=Um7nxIP#-<m3+<9XZGqxdTLpc?@p^f-D^I-
z^w-zgN59`pC_a56(`eb%^A)SUF6cWiZ{O<9eqq<of6|ZgD%d4sB3N(eavVHtoTibk
z7oEkn@y&jD>8VbZt7ox({Tp4?EM<Q5K<9s7gYXMYhCLTQZQZiy@Eb?&Z<|*xJGw~x
zQ-^-4aY|;ww22e^kAD4FEBGs9+hGYK%UTVN3Ds5izv%GaeKLjLEn!J(!Oq9QoFY@s
zyTw;~aa)^guefBpyxJ(|+M=I^&7S>RU1xDV+ar5r=b`@>9YmbfJQijDnHt93?Kman
z<eBrI9{Nv{OS|TMZU3Zu7n;(QpNdH=x%wtD^7`xg!&%!;O@8+F!Ieo?Q&wkYKI53U
zyrRDA?rKM!8EZmW?{UtQn2_DRZb8ko;yEb{3#UC+xv4Tw_p@J-YeNB_v73M&lR?LZ
zVuOWyx_fjS^k0Z6{pPmY>3?^x=Cd^dN+%K<qZ|`wD;)~>xp`$4>wKP#CSu+5ovL5>
zzGvQ;J*i0eU0-p6)`OeJ6ko54nQaBTdEMUPl*d+wb;VP^Mcyjqy=QN6u(34VLg4?$
zGsnWqd#fHVS?iv@>`isa>orSyt&=t%XZiPy*U!%AipArNQ_K#|pOt!~PW0ENuM>Lq
zf9t#RJT~m_v*7(pbx-_wby0SH`us`eA`aOXvY*eb4ea-PZ+EZOt@_H&O`JuK#bRa~
zH=U6)(MXb?r~X{v?7ds(7p4_-r)O_o{mp9f#9!0vlw|MT+oE(Rt+VW%X0}6S>AEE@
zyLs0nO*p*x_5rQ45l?LEw!WW|(B^&F|HQuCDK}kz%zv^t#Z;@VYjw`e>$8ozO+vs^
zg-Od!$C<@SpZ<5NGG5Fw!OW{|lltsPp2caaFMpe<+Y_bm+_x~Ck9$g=%5hamT{hQC
z{c}2bHYR%{r=9!}se8fjl7g-7<Wp@IS4D2SQQv*C)OB*<lDC%*-rT!m{i(bYvQgI;
zyW|^9KILX?z*D@{&|sa7bb`<ZE}s4;l?>r3YaZ=hH+P9t*onMDK4%mzaQ;z_ENk<b
zC1TyIG==|JjqV**9hN!C5vsmyEZ5e25<MWK&2A+mzWo?$a#>-Ag1zYORj;<5y4m0(
z6y|BhsaS7zb;TLRI-S$&ekMkQ-da#owDh(23f*U3ZLDEyKiNorwy4<ir#AHPrRR?~
zybxJ!Yi)2i`>VBdZ&AWqslGh{-?HtlZ7%%(Vjp*3Mb`fhtMAz__RM(3yqvdg!K<6U
z`{iXG*Zw)Trr7?=p6a_7j4w;AVch<?=-%q^y;cdoj=YRsR$uY;kNd7O5*B$MriNbs
z^61mSiHFpyJ(9x$OqeCs9!)DT-k#hhZ1%8h;-=7Vt<AaL&hE+j%y+{h>Cffpt)`Oy
z)O}AkPfBbQ>e|4rtM9$)m7%v<qTKU<gC+}(HVI3;Kk`@7?xWYzZHo5X0!LHpcNT1@
z@j9f%sj_52D^Jwp-3_6eWUrg;s!w~c@a*L?GcuQb3@qCAY>u{j*b$4ovrTNN-G^fg
zxl^7^ym08RYV(qfL234%B6aII6Wq=PJhnR1)q3&yF_*&`va`wxEEfIWTz`6o_@tO=
zCcdr#+QK|>9hJ*p<vGT`P@1`W>C;HV=+t+bJ2k&d>}oQ4;-8ka@g_SvlO3bcOwqRz
zNtUM?%a4BPukSZrDHF+kB4NUg)+bw+7;3X_ICALqi{xpWCfKfi#`;govriVZpn;{q
z%W2Km^4x!BukS!kG^?Mnu=nI)fgJZ&@6J`vTYZ0Z<ci$;FWfKk1n?eAD>dhmR*)|d
zjQsWUzL2iLY+n78^Hnqd&i;EMc2nZjGyXR}7nwfZnsqGWrn7~#{Y1O$^S(E2R$o0j
zsa(?Qq}`c@n_DvOh}UHOnR=}Gpo&NFAr4_%&v~3hq8|?Pgw7B=+cS+dKl$+0jPj-I
z(#MvSMJ||Cxgm0~@A-_f8P#oq=dT_)m6|cXCE`+ZOQ_EKbC1M=*kZ5kPx|lIH!Yzh
zWv@=!9F>IbxD+Nk$!&$A>zyO_InBOZ&%MTK%J*&Ce2hHIRGXKYu-<=Kqs3-3E2nKu
zE@;7l#radAJr&?>mA6tm6t*{{9;#Xzr4Y8{tiT4YKN1boT0%3XZ{YIxod3--{KULR
zXS7ec7j6@^ZxwKCP}&g05W8-sM55?CRsp8VM|XzK2|c=Iy4n@J_hEI?Zyptgym_=M
zL@sPwXZ0f067VLgJufTQep)xV{`*rgkIc_o|8V9y{JC?@>i^nT|90!2kK1QnoW6c<
zE%)>1eA>IzEiOIVd*uwD+L|WC%bb#t6F+%Oy1C)y%DcTGb_eZZ&fdA)c68H3)9L=1
zu61v-gRb)2bGa3qJ#n{)EpuAF)Q^i*r*=<zdFd4I=isyt%8nwpA|np13DsaV-Xc3;
zX0nZ#6U)xlqnB0JIQgfoo|CS*xqI_kmBp&wPp$kVTg7(t_fFXLb9F}uL#$ou`XCM4
z6aUPTBxb#7*?V(}#It$#EaU6<y}6v5Hm}P!E$(BiX8%D3pJy&NpZI!)bs4W$iP)4L
zdt?7hZDZ-r$sZad|K)Fp_@Cas&rmq^vb=-pGVrqBpJyNEy~)atd6By%V_}1YxMGbE
zUtyf^bC-kmDOwf+g00P#850zheu<m#@0nGi7PVuQXq3dNlQ!BcELl54Grk{d|8vl5
zrMyJ1i}iwutG9*Lzt=t}u3(|KQ&aQMx|5p@Gt4v*b(A(Pjy(6TL#!Z-A>7_KerNq%
zUU|E%i_`CWGu>x9ZF9Q#yV%?Q`ei>mdY*~qFFtC%YfoaQvQJP{Ku}~^Opseq%p>_7
z&wd`c`_NYB0f&8vo!G6H<xiAug)U9c)7qVT{zkQb{&~fM|LLZiXL*{+%(<xibjO+U
zx6UleQ|<bv>^JIHa_{-Ka+iU$9oJha1LO3do8Qx4C>@-!N86c^ae~6CbJwHFrj_Ns
z>)bu3xK*qzV$$+S*6$CjHMc$OY<~0k|0_MFT~jt+I(BsH-kuibUT`gcW>)*Nsn>Ga
zzU@}D-5@VtY(1O(__sf=-P=@SL?m>B%;x?+QKNgnAjWvI@@X+|OI}@nPi-q6kLHW1
zlJS|B_FubywC$pnOk3ACf!bG0VSnozms(C)tM~1`Tf*I(v{d_>!q*%0q}CjYWzhQb
zB)yQ=qNOnP!@S4!tKVOrp_%c}?!)Bryqm8(^BpWVfUka<6MoP}qK+x-V84!-jx#&s
z<cM{16GhiO)pL!1&UI|js|L3iJBbZl?{__OING|ibt}tG`@>ocy3aQ4Sk4>0^lA3S
z!h(dSr*_QkeCuVu{>}TBfiw1;pKo3Dep0L}>+h!7{72r&ZhU-D){^z}g-Nxk_nw_n
z|FHSPx=Wo4`M<?%en0CH?~bxMyUGX62j1QJ<?3H)RP1+pc3|19f0n<Wi(EN;UN=8Y
z{Mde>zqcRC=$<=dHA6CTYw4e@4`-QdPM?!ie(u@DTm|9k4d>@=I_W%Fch;F*z8wo}
z9UlFAR(kb~ty}&X-^`<}Pq)rIo#fUrQzLD@$xe=y>>S0RbAzTZ-`w#{#98Ou{!6F0
zb~gxrOxsiT?Xb`*8}_!KnzgJR?U6^pvvLJ<wbR3{sW$(-6*ss4tWNW~N)Bc}j+#$v
z-mH#wH4wYzQL|vCxpDf$!rkR{-9IlsYghE&8npkJ*h+yFPg66?-|kfI{XNO}p%~}Y
zoFCfvcyoI%PLcZ;zsVzV%Ky!`jC75=eD}#d?`Lf?oOXZn+1XPiV~WD=w6C}H=wEv8
z;}nsN6H;4KPsqnN?`L9eI~6`*pGVA__fsqE+&bqiXIk^<jP|+BlWJAukLpZQ&UO@f
zVz%jxl+PrqMuXY6!X|AlI&k$R`#qgsZ)Y1%liQzn&AK|Qy>IXS7vDa;-6%dYU80NA
zZ>5aTGq<#_Cq->ji@#2sx-!+>{Dhy)@)!Ta>-YVA_&DA<RP~PSkI10^v0>MFLStXB
z-}-TUYi-)}+JB<k-6yxEK91kE`DXtj+fecRJ9hb}?!;eap5mYB+LBtJaOO;y>nh0;
zGUBJ#6nDSTuQk#>s#w2i+tfMD$1-L{%Kqk2nk^<h-|Vev@Wag)g|6I7FezRaDK_o=
zVu^H(lXH|#T4zc%7|fi0^nBKvFo`~o6hqJRXWSpodam}9{pS5W(mdxYo?VRi!gFk4
z^8eKvUR_A%V@yct6PvB0apwM>$?sL_xEcIZHn(Kg?45PO`mmOs?U_3w@2pbmJ&*Sn
zuXy0Qj^i;C-$Kxtt(^b3HgM@SgfhflJgoIxmm}h6`Nbx?O+Fi1LeJgZ@t|Rb>zNqG
zz<}AzhZWClOG-GF)p#yX!>Une)y4&^DK<)XRr9yLf41ZN$IF>P2~&G#CWgj7&yf9<
zw*EnQ^uOFqGq%jU(elJqt8iiH-$`;gMfb9nX=L2L|K|9!yEofSubnwl8T@U}G)rTz
zVD2gYhP#iJZ>@?f(z)rL#n5#9k=R<F4bPSrIev&)|G8cFfQtAO$i9H(3HQQn&bLlG
zy<0EL-R5Y|wBMptV%2ULPn(yfzudg5_UqQFiSg<4+yu*}n`d`qoHLp1?>S$pXHBHQ
z;%7Hm*X0~JqT0c9A!)|K)<1><mskvL>^BqZ-m^J^b-Mbqtp=Ny9JD{X#kP=f+X~Co
z|3e)<?N5=6Uv}2V{?q+=t;avl_lmg{dom{Tkyw!8g!!ox)KahQbmcne%~^W<|F0RY
zh38LMdhfLppR@aDap2cFx#<su6@-6J-_Vuboz2JH!p^xmOKpGO2m5L36V7p6Z}4aE
zhypEXZrJSfK>qrkreg~#)p+JVaMu>l`MhR_>*@VnCF%bTeoCHp<o}Day{{+6oKEfd
z|9#WuH;#cz9zB>?wPI#M^UoLg%2kQ;_V4}DYWc^%$mQ9$S(@y#oRw1Lja}BnD&$U!
znQq|8c66cZsZH4jOAR=eTC{te+EFa9fXAZc<7S(#Yv=Av{~0N{L*uErMaG2ZZP{xj
z48K3TJFWce`@qQ?rrV}w%wK(!DZtbIT!&y+?hMt7p_<&?4GzX0#{GHc-|svi+RXCy
z{THe5WU&JiA|qsyjvp*FI>3MIV8x#`(|)tek7}4avo*otXx)zU|67^<)t}80XJA&i
z+OsjIA~nWs>1Y2v%Vbt;^ehscC9soi%Mwq&&Zk%AX8%zOQcU1`KKc5agTHfL80Tvh
zE#0!gVO^POee_mSFXnF1m(q!CT~nDYKcCW+_n9Orq_EKbesBGzOW}8A95}A`ZRdEZ
z9pNgVt6_Df*2Ljx=!PB3*A$EHZIueq(<}U|F3;Gw)w-+q=Jj)3$8!4i%;2okd%NlV
z&L3-9=fAUAne^|DSfui)Y5TVPJ#Xy&E~+Zj@A?6?&yIKMT()H#I)8?%C2GFM!Tvqj
zle?F`Z97vt>FuT2mKFZr`r9HmMJ6w?Go36m(f5apeYe<2M^C-hgOk|ir-+H2x#NEA
z;-sH}HPaVzTwgoM;N+Xh*HhQksJ{2QplX{v!6z;2<h*qk-x_Z+J7yy<Tu_zQQ)w5I
zHpTMxsWn*{>t?AQH~Q>t_N&D#Ele|{o8xi4$<E@nvezf2KC;_XHth?;<rLj5GV0$B
zzn!%9l8e!YDO0MR|Iywof3v9I%o)9Z^Z#y_==$chBDpQv=(Xo+|5QEgGis5VoURfX
zCwEU;nDmUR_}F{dTZ}V5sBS)C9ltx}{%scjRr_W-PHBl)*L0}+?_Qzv&;JDl3Y2dD
zBVRd9{LaDO2W)0s;koi{!cNWW`vN-@W@t=XUl6g<`0rt%(~&Qj7|y(`5BK^1<NF2S
z>zg_ZW>48P;rgWIOg=`;T9NwQ&CHYAj-36j^UWn=-5KrI8yvvPmb6xEW_Nuy@0(=I
z#RD61FL$N?vP$2o<+W~~6hr2{7q;Aw-3$wUp)RG7pMQ1dgTD28_l>lET%40{&c~B`
zZ*s8Fh3UIYw{0l@s{J9&H0gaQZ`cmLvk|S|H*6}bJS%Z7vo444Bah9s`BP16pEG~R
zWYaZkERHxTe?F(C^twjn^}7ET<D*y3jNHWN_bkA{ci$nmqPuJ=*Imy|J<~ZUNA&yi
z-LpPeHEVoH+}5=|_1RQ2vFd4?#O^3bYQ58p-g0-s1=rJiE}arSUB=7jIoY*+>+YP1
z*S2w8-dy{2>wnGD+V6|H_ugr3iIZBKI_=$rds_-ziX#@V@ukX#M=W?!@Z(2bTH1`G
zlN4Nv0(g2&4s!DSOn$saU(Qu{i^28%m$c$Df3;jv;??iDVj}RhKIZ2Cpt@Nm#`bFO
zy`Fi!>$g>K_vhTc#PCPsqxf&it@@!gjbf4}9hFMzTz`b>PiEbCs5`HF)pzGPu}vEL
z*d8xw)YCKJ`Ogr@8q9q3-@83ZZP90vuIh+$`-mJ6lF)s&Y0j>Q;yt=f@|#|?POf*|
z&{dl?XX={bqY7SGJckOz4LqNPdG7O;Vo<!oq<nY&tjHJpUVRFh>b{lTb*pMapt1dZ
znVDOU*z+BD*=&66JnxhGOF3_iLpM+NIT00kWn!an-Mc+Y6LzO_)ID1&X3nv-Q9dSL
zQ(xxQ%{DRP12;Y%U!`b&zE1bu-Q#xexdJABy}B!V`dyoqf37_=k1O$C?(SFh?P{hd
z|M5HK|5iJ$@2@LrX-PCw>;Jkx#wP#ia!Kv;F*eWIQ=iS-@P1xwz{Ukfr$o!!=Y0zO
z{qNNNr9a%>MaFFU`p&-Z|DQ|c_v)WkaQ2>gz*5c)UMhEGrv5&OidV^N%ANMV=Fa<7
zaBrEu{hvREJ13hhk`k)A$>E*h*mIHN$n875S5!O_^17n4o&^{A23?zT`16JPHjAdX
zOy?2(e*K54Z)3n^rz!tT8&r&Ur^p*0+0+;}iJkkhJLiq!XA2m#PR4IEe|eCNsj%B}
z^{F#?nTt=C%?><c;D1PWeoEaA3w^7P4}Dgrs76~n=|BAQ!eqxOZ<O5=a~;l=Jo-1M
zSk*lI@vI!9vno?oSx-1z|8hO|fA-_A_?Tp~erDeC$}Lm)6+7w9lsoeOrd7Cbp4p*(
zppA!PN{m2O=+Zkf`>jmkK84%(pYi?VF-uF~S#won+B2>M&$Q@`{$h#cDK|aU1=t!Z
zVvM^}^JSDcejLdWUBK~alZA`Ggih7#b8WPRQl6jDmJ^H->tZdo-frNN^g^`i@^6PX
zQa9H<jrU|=YwJD!Rd!y~)fch;OQy+hR{7^XyD#n8tNllf)T->x*IHz!-HV;~tHB4d
zw#~R8>2!_Bx2`M3J5|5GR9$RWBV^yb@5ar-w<{MO_coWy$d%iEv~aGyed&j#h11!O
z>F1gnrj`_3U8Y+-&(1*m^Xn%|X0%QZ@7s}lg6q$n>ZF?OY3uDIKOb7Q{dTJ9?dtNA
z2^A~m1ys5Gv%aIW7INPa+Z#rcQ?^=pp3OTUs|98qRsV2hk$!ze_QM|N9P`}toIRyi
zP1qzfPv)x0Z_kPOym5bz*P8mU6^;*=+gMfTYOXo|c%M!Axhe5MpSFq2mHF`wGUl4{
zWar`al6!K`%~~Hn-&opa_TCk{EW=O9Sgoj>{C3YYkHqLNYB#0V7Hl}<JL%8VCks?O
zCf7>r$`BGyYvkhdnxZ4q{_SLeoo-A3lS-shY@5iVb;rsJxexu>lHerircu0W$MV&&
z6W1FQ8!?2KP2!i@d_*~`{OOA*jf5q8Cg)s_dUEH?9dUb6H{A)LUt>~h(tmV6eq@^F
zsm18!aCCw9>9v<^PAfRGxymTL*x<kD+|)Jos^QDeIyPIC3N8sZ{t$5eVbjDz4bHqw
zSLbGG-!)&nNg=Q3?7XRIVjCaWJ(##9MrZZ}CkY15>pLf1UAM!&i_iNd2j~0*o(Tss
z3a!7l_^>u{p0y00)zRv`Wv_GT`ctn{1rsI3GV;&37j`H(h^&+4J&@0|`IOLsq{7{k
z(_IbD1oG8@*G)A<=wuu`*vR+Y^!RP1FNz2FnGJMZCRl3kTivL!)@|Au8Ru5*&wVR{
z-W*E0c1+ge&*`skYOg+B`gQWYPk$oILYCyey0CrGXYTZ>dr!m;to>uXI?nC`Z&cB_
z^QUfx=f4(>|Nr4v|LVBv?B~qN`!cHE_fNO4e|<XqZvDlK2FbJaaT)$s`}7`6^g1r)
zd*l5d#RHH92T}TOX59YzZt**-wD9TI*GW`-JG)%uao)+Tmd5603{Q5{A6~rOHu(L&
z&dKj~zF$&(e#iH#p7F-JTkZVQeP{o<b2Xvm*`F^5pG^0&EU3Q5`+B{9_jw;b`NXu4
zuJ!kJRjp*^^qW!eXU5}Sb=K*n`*Zp7d(t;{`A605D%$pWIeXf!IO&g1Zh!Bs{`90=
zz1B3qNvFc%^Nze;>r%-rX+NGWo&`M^Y1`kvoNKxMA8zj856+*k*{D0>hVxv|;*+A!
z?E585+CQ8u+bq$>Hgm}x*4aEFQxt0^ot(I;Ehx?T5dYcbJWnnwOq<qv^!z4&-Pzu4
zKQlKQr59(WP3}xlc2|skWBPr|&l?`T836&wCsTGy?)l4nB_u+B{*{l#amz2fHs;>s
z-N-dh=6j2|?s`unSDgmM)XS+I3eIZtbiYsk)i|r5_2wCW*)xAuc+6bq@jR?NVdc8R
zfBvmCa7qcidhJZ)kwZ306)K!_BFYYx+8&OV5nu0Vx1qwSu88{=Lx!TUS90%`MSBfc
z{+(PN!R9Q+_bhwOiHUjd1tUVfb?RvMh)t`SUcPieYj2vu^RR;l;u&3bN@zXWy5x^l
z_h#k|2br6gBLf4t`AW7hcK%LkIHK4zCscKPs+(AU)3+I`9|a5P_eeggwtZ&zd%aXx
zbkG|ap|S!MjjYqR=1q!wTPt&&Aw+7;q_>=ZKjp=|dQ*8TpVw^54x#Jy{dczQ*>LyY
zZLZrQDH%e7|1N#<{GGIY-`3Yh_Iv|R4Hx{=m~c_`Y3mPthBF8Ltl6Lad3)sZM|ldG
z4;$M;qvyxoO_;g=x@E2Vw|$@fW%Aqnd9Z$E|Mt3Nhc`plZ{GN?`uvRS_eSQbgZ-te
zEXsVo>snjo?#QStwv<14>Cc4y1{-#yPVPzG{`JR;Dc{*kEY~v};AyOR5x*<xJ?m|`
z8-=dtE;R{%-uLa<viaembN+vS_ei(?%`e6E{?NrxuO|ue^URuk$qsamfKb<$8TbFb
z*4x~-d*R<5&(7Uj_bc(KvbTL+W#wYWm6~7o`X)!Od17lW&aq8XDd%uW=FJ$hs~g`e
z)mhY-_{>M0GrVAJYF6gYkUx{FPdPY#JJ#4VsrHdwl%80;w<h<^%X{KDcbUlbyq|UW
z&yCHN@!pqJrWhw~emKiy#bQT&-R9G7zwX$`7fUQWHtFU8^;|x)CYc@IG9L*WX?ITj
zuA<XuJ=3{0`$OzA1}DjX$1nKtz6$=dc+2u76}zU@KmB3eB(T%ICY*cipY3PtX1_~_
ziY<=$_V4Vh3ikUI&n~_f(*C-0-Fnbr>Km-&Zx!kuXSH}Z&#<=UaFhOx*8*M{EeGs>
zu$VacOKwY#lYMqf`%#Yev*Q(Ncg~#&T{3mOgw)9xO$qOf)|TNa83$`bt{-~zz~Oj*
z;58>H<0u)W6Goyu_a2@5u$D*ez}E7RjG(E-F=;EpAQ!1FEVn(YC3f??i)jmM6g&Gg
zzE3i<ceH(;`!BkS<FUY-y|ycBv&5n%Ngd_ky8HRR_WN6l7q99sFpK=u;=p3E$YzVu
ziUqE6VjCuAxULBb*$||iwR)SK#<zgC%xhf_u3GRlP*h%_scY&ljiwb}8za7SFft0d
z9At88X5`^)QBm1_^WVq2^FF_wdB$S*bdKjG|85kYe{<*O-@5X{mfs)G{@G@qeErHB
zpOQRV^IMjJX9Sp!G}Yf<a`tgJ?``XCX~*B~`{8n?aP#hOxw{{3c)IiFM{|?8wg*&x
z>|Bud^6YG{SKaA3d-Jd6=AW_px6fT&-t70&=W_F|f9=`auHwb_e&dGQrSAV;t|;7F
z{_Xbkohuiv*>-t<Ty)9h+0W&7ZAq;BIkh3{V(RWq=5IBs|6Sho>Bospg7N=07%w>8
zka2%vZQh%?#YHC%WNL6<`NmeU?a#@r!RDg!bzawg?3&NW6=oLNe`|C4P2c(dzTBTJ
z7=LB)hn7E{HC?yf^ysmE-sNZe@=Bet|IWG(b1v6O@6;<#Npx9vNG@i*#h=4n-;Dx0
zDgqy_c+XcaUGH7T*lA$&`k|=o)6S@MF#(@KqRvlc(Yh5oE2ibmo=G{PhvoXF2)IVO
zeoMVMant+m<Ruxq<&6q$(kEAM`2OtY)uumpA16Pyj9NL{%AU(M{M6neu4(C$L}$l0
z9Wu7K@$JUy(|3PGZe5@=K~m0FJz!s-F}KQ0jW+Y3wAiA^OYd)rzCW9Gz$CKl_MeYs
zOYHf-t(;oAap|d(E$ktQ5ejcz=1pEEHS=C<kyD8G?B|tPzu84{-1a$fEH3VOv-44t
zGjBtoUxV06mX4e=tcDqmHr#0iHZ!y}OBgSEF+Mdd4)grg*j%(Mb^4+`0)}Un8%{rJ
z6S_)CEauAbO)uZLFHE?!A)zU{<KUmwDw5?Q|JHV$&6p$3BWSkt#?EjqInMc84%t^M
zI$TsG`i3<<m3zJS1MyQV^{Pv6Tr9pS<dt#1@1d&1`uDk2ZqrR{LwDQmQ!v{3YUL?w
zBR9omADd%ef1a;Z|L(QtMDx{2@%m?Obe#Y6>Fss5bvH_09ZTQmGpY5&3&H5v@1X~0
z9ZovF&UxbOZF&#9k{sV$y7lux4a2m)v$^I=_bvLed7<q2onO8z_l}qSbYsVykLP>h
z%MbF3i_8`OqvUUX?nLFD|6OaQ?vJ|3{%eBoTIJ-;|Cz6?UX)*(du?V1OUvOdo}vJ;
z6{P~Th9x&2m9wSwW$(#2Un{MBZT`lzbKHNQmqzYjHC(-|=EvUe=O;ctq^VRa<^BE7
z;={9!HCmtl*0J;O=4FXr^QX=KpS1qx=l?q&H2Jsj%U9)XFRr$`@m}ut_RnXwP1+=8
zcA)v@(Yms>55{Mg++6?Z%|`XejhU_uLOq^*UYkOBx>DXx(>Xu+`nT;-->0PX?pwIq
zvmoA}|IDs$uRTtd&YU^l*?6~-C0G9x*9VU0MelnTbSDSRb$&WuZ>i|-pU(p-D-8Ja
zzws8YOR=1PH??J+YMb69%}G@a+v5F>T>5sbdP32wP1-zvlQ%_N-1cS1vJX6>^Y*yA
z%--2#@cP%o_G9xaj0ys0eyI+7?qPrC^Yi~@J@co!cg0$pE=ag^-{Je5mphO0BtOyG
z^n6wJ_icZ(?@zC{-!tt^KvEp{cO~h=8y?(#XL-4}yYSVUpC5C5g)2+v9Xa|&@=t?R
zz=y{xBhMbt5?V3gXx*OTBP>0VbGca?c~%@cv**z|tq6m|r{?6co)EFtta4o~cyL3)
zyF;@kn3`1c9q2h0vvGdwmFCU&CySr{?bXMBZ0nnCPVQlEw${#nqnqG*aLNAX$Ie`T
zw0x=h(R)3H#i^Sjc0UjOzHV2p{hA7gwFLohHmuq%6}LWaZbanQ-?MsW-f!Rk@x`I@
zugvn_f1W>GIoHCj?$bBc)BnPvH<aEw#q0TRd2+b<olTFA9_wVC-}HZD{ML(!tml@m
zj4u2C=aZSf$eelgdKUA8(|PmES4dZ=KXKfdaP`LAdj0t&LC+cGd8(G?|6Q4C&k?7*
zD(qU!?GKBcmu<OwW8JF%D;|HcN?o`-JNDx_zpb06-}`^>X!r64N#^3C)(>m-ZthCC
zJE?wt-RlWmlNzV5{rgovf8BxSy&tptB{wHtf3k{ymC}osOcmLGH_uTwZmL_gCva_A
z>bW)5J8KL3r*^g-JOAqQ8_9qd!n3R1AD;Z2=S6YY{E4+u@>j2<KAhORrO&kf<otBG
z>D#o=KJMJ|Fu3Z!<igH~428z`n|3O2&Ri1jw|mo5?Khw7Ig3(G&fhalz|T%KZR3+p
zldF!oXEwHkN_A{fy>43Aou<^;{OI+KcRx1!9^4<6JEcItzu@$<H5RjVZ@o)g`e*W1
zo7N7qKc72NHn2W8B_=jKtMSg}BL+Q<y<tixjy1Hm-pyU=$S<vSmDN^As_fyM<=bC5
zOugM2X>Rl9L;8p5+iy1L`pldo@3YNi|Fpkrf~F>=Pd-|AtMX>E+}a<DmhJri*d*aW
zWx?5kRz;Z`o0lj>J6NZk&pRO@yFO*BL)x?eff+1~VD6FBHRX#8joL*2sH{m^#~Cw|
zB_v_BgqP>Xa*m=6>b8a4Vp|w498y2^=YuY@R@&r*gJ)JRoOpb;T$-EqiJx9eiXT}z
zNLgO(`n@J4?T4XVgtVBBqS)q$IqE;GD|hFWAHEc_^oC-_#hx{+hT$I%esx~GKKI$Z
z=tp0-zTGT$>i?b>lk($UKG>!lwoUF@V)<Lijk*)JZQX9RZ^{ck`-ITTd-i54i-vN4
z|6`l1VppCXYF1lYxo1sX!B_ns!sS=bcZkW$O}Ub_{#JwPgINcDZt+ZspZ9Zh-$%aV
zTkjwE{we)<oNM{^ec!)Cw}(vOsoDH*X=yRrnR#Dt-2Q(3?TK4IUtT}H;4_z@^E_?a
zM?L2QG=5wc&Wo-tJ~zubOnl9$mximH*Pmewy1;b(#d0PO>wQ0_toFL?bW-4tT#8xW
zyxU)QowMD}bGzc|ua^(*XQjQ{ZKK>IF@;qrU1fJ@{~fEf4Ez^fPwiX1{^q{K_ne{^
z&sVRjs6EQ9o%PQ0<H8@p!U|U!ogC`}=iA!dc((hA@$>tqjX!rOzm2!wV>tUr-CB--
zU#4}QVvMIU!#92KZ!WlW{(*aw%)QNL)V}inj5?$qxg=fOGi!}FkHV>ED{XGRX*#UX
z_G3jx;x&IG=FCNHdcND!@+KLUvH4#Y`~7t5G4af&O|#s8nO-}6uT!m7akHzKZcNSZ
zMJpSV=Dmo26lBGzo3ebGjNzJE?#=I;)Ef#+`nw;e#tYjdY<co(Q+?0-r&syg)-Czs
zC24!}iN&oC*Muw&OKeuU|L@hU%O0iS^D1=N9<a+yFME5_cs-NF5ksDe1)&@kM?SrX
zs%y(KlK$?(_~zq*B(akhp6FHl=h?Mme%E?tww!=-Tiyg2r2W=A^Wye9KmY%t3X?^)
zPdFwd&NKgj=1tZ5y2-8ev*MdWXYF8((23dpEAR2dQt3GR&1>%O+UcJkJ=db}(YyOs
z>m_A#-@lc<!Zh{K0lxUBy!UMjZ%w-I`_7zs{ilcXFEtwN;o4vEw|#b2lJWf?D^xZM
zo(bJ;9W8(Li+<Y18(QIeZ+(4rOZIkHPIY13yLD#Px7hs^R@c9)&Xt#*{BgFuuWX^~
z@9yJszgDi2&3Lferd&~I)B6`PuXn6kne+V8k!yQizCCRzq$B)m>y|$}astc|Q{`+9
z2t3*2{^{6L@rByw<GAZex{v-6H7$9~!T<g;^XdoR3nFiwd&{)nU}p5Ki0<>3H@9T3
z6w{c>y!FgC)5C20!i}6}*|dp0*1Vcr%;^!}_x9Gl)9<&JKe7`181jbGh<iWpmNzrQ
ze}C7zegBN7Z{DfN>I~s;HpM@U>^{%tY9kWyYiCbDS5|?F!hE*33Yvm%{#?9u&~&=T
z3sJU3H@^QezCBSc;*jBQ@shvxw+}LK96qr8!}14b7-AoEKVXbt+TB{gmL?>1a*pTG
zuDiv{o{Jd^IPGBx`EXjIL1s-~@Z#hy$BAZdxx_X^@U7&T!rCU7{``_%Mbjd?r0o)u
zK58VgCCoV+CmthqI^+QVjbn2}|I|gODu&GtY%Ldwi~ag|)z{rtx_ZACe0r?WtnIUV
zQ|is!XI*a{8nvT~6m0A_e4BH&|DBZC=Z^mF&C}mxMQ^k;N`1X;Z@AmNlX|Cn)$GpI
zYu~aAyt^{C#4)MvcQ}9KohiZpPDqFQ#q55wjz4_r+}-bQo=P|Wx3ntkYx7m}|0>02
zzS`Gk)%?HhU-Mw;mloSUMcgZ&O{w!|oPCVxCR<}&Tj<I%jm+#yhbe!;PS4SL@#*mL
zzuenZ9(+80!@;|I3d0%x)h1;e7o3Xnq(mlKUu2lg`(evDZT=3gtNd}@+^n`AbRJLa
znQScgsL1Z5S7u#*E4y)>WKRg=)MF(#ceR$uotwtlp~U?sXL3~irWbzN$C95qXztWI
z&i**+TXW%0c{@|L)BJTy?!UY8>HPjp-8@T0Or_&Oe_s8Wa6xg3YH`_4*+;g9-*!DT
z-cY$~-V#R^ril-|*$=YbGz|K6$FBAGk`-1C+~Oy4<Kknd*>^my{4`aEX-YtEa=}@n
z*E_f`bUf8Fd2#fPV1lT3*bV2FW{;-OhOAjPs%9u2vay@7m636d0COSt0o?=YY_SG%
z+Gj4W-ZD=+jLGMg=u~d6o!k;_e784nO}xP0616}^N#@T{tAx<1pf|tEJ2u?Czi4mP
z@A&U;uitUz?%KA+c1!;HvddY&ruMz5482>pU&3~U(c^WqkInyj{D(DvZQ-|XUFm8u
zCchS&ZNFdg@Pjm0wf%+9=NRjA=G30PZ&&&4o%Q;i%fB29OFsTj(J=E&>fd8>^^e^O
z?EZ;oOIO@zdT?p+(=F+SH<y<G`^zrXb8~IglV4fOkJPE?eB@{H3k|uer|{+f-zC4k
zPg|8ap>zGyKN>ULe{J=c!S1CP#P@%z<J>ncm%2E&Y6eM7*>qEPt<8C{T?e*?37oJ#
z^V29j=Uc_hgJGH}Cyoe-ob?Evpq|OU=)accekJaSQw|y?pPDw&gFoz~oUVWS47R0D
zv>1!O9_`3Dss6r9^WN{k{`U{L{U6S$xiD$1lTE{99quWt?Nars-M>_LCTAD!nH#w-
z$)258M^?kk(@6T@Y6H>W%1<78j}j72+C0r#IqiR@CF7h!{wuw{ihS#|`F<*Czsj^u
z@0z3QbMBum=xg=d^2k9?;YhvA1HF@8nU`1C+-kVB^$b7%^lzDKM1CJO2xssMy5@9l
z57Pp!5Ek_m-m~)-Ph~x2<aE2DO0+mq=ft^*b#C3^jNM<q^}e6K{n7b<n;aqz#LV2l
zHAmEqQ(RVP?~X@5-P`<j>`w7&-`P6ZIDPrA-AmG6?)~!nrykF))>ZTV-}^FS>EEL}
zZ_fO?dP!AW%x~7SfBnxe&#d?w@_1={+T}lWb!R5uG_!v!qPBd$%Z0-MC;w^OOjIp@
zs`+zs_0k(l65OiZ%v}ECLcB@F=9EcOFU^y1zZEkrOk;+~;|Cp0JZj6Iygb1bZ}sB&
zz8|*}IzE&?J9_VJ=ph|}eQpb7e@%aoc$0I+oY12TZAY)oWSzuZpUcMcagXr7Fg}a6
zkC9JQwD*Pg38i_c&F5JyQlv6Z;&=1(sE6)sb-BJ%nIDR~bLxHwdfgs+aK88bWoM3L
z-0SCE9bk5>=$_nW|14wUPtpF4bH8LesM@T#^ei>eb^oSCd%{om%;D)>(yprgA^f2A
zL5_rTn%0}1tee0%|3%LszQ}z{HWGgt%w<jraL74GJ#o1nyy|>>litaylXV7}QfoG`
zD;-u@b4lmH8o#c%i&-qj;X2uj^RzT{RQl)6Ox2aJV9a<ZmB2Yiv|6Igv*YOMH|>_;
z&sww(Xo;n?FL15$UTXf|z1s8Z^l3NOe=Sm)T+g5A_x9wo#MN(tmw%70XgPg&My;6S
zOZKjnb+hHlXT8dQzWqzW)U%P9s`XLJh1V@T(yF%SoE)3bEP>WLo5LEGbB1^<EYr+B
zI8jPq(wV8t)xD&Jb&NBvCma?!ZOrCAZ^HSOQl{o=)#FPGB8~RV%$F@*_h#e%OZm)Z
z|G#M6eq$M)<7pdicKS%#=kPh~awqw2vVVBblJ>nkvo4om<D0vWcK>jBbN$HEr<#0I
za@yX^&$OK87Ug}~d&%|kDRKLQy-Tt(m4s*6pSir@s9Axoq}I)eOWy`JIdJ~j@ngb!
zr@DL2tBThJ{5T#PaU$(|k|#6A6U_*@%SW^#LSlbCjb$*?Q2X*dhV^`})p~|GJ8pOF
zcinR~Aajmrdi0i0e;KpyB|lp=^N{^fo4Mhep09enY?1fof10k`3A)G6SDe|rtNkX&
z;i;{Ox1*YbKhM#=nx3@TbLFuL7iF8ANioyU?q~l$dHefY`(q<!2`%W^`1HDMr3s%`
z^13&Fc(x?W;%Hx6_M+0OK~LkVcj%NuX0yA!{pFb19FB4|zS;OwqxW>ugf^R^sUMyR
z?%g1{Dx~`Gn<S@-f83?h?>PU|mkeTHU{Eb_jVMV;EJ?LWE=o--No6oHFtpS)Fw!+J
z3^6pdGBUR^vCuWJure^1AolStiiX_$l+3hB+!`KwdnPh4Ffe$!`njxgN@xNA4j)Qf

literal 0
HcmV?d00001

-- 
GitLab


From 63dce346513e751b1cfebcf4a2e9d847ec4b0b6a Mon Sep 17 00:00:00 2001
From: Ruoxin Sang <rxsang@google.com>
Date: Mon, 4 Feb 2019 10:16:00 -0800
Subject: [PATCH 1846/2345] Remove `_disable_training_loop_on_host` flag in
 TPUStrategy and all its usages.

PiperOrigin-RevId: 232319252
---
 .../contrib/distribute/python/tpu_strategy.py | 134 ++----------------
 1 file changed, 11 insertions(+), 123 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 69ce1141d8..818cc40f6b 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -155,8 +155,7 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
   def __init__(self,
                tpu_cluster_resolver=None,
                steps_per_run=None,
-               device_assignment=None,
-               **kwargs):
+               device_assignment=None):
     """Initializes the TPUStrategy object.
 
     Args:
@@ -170,18 +169,9 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
       device_assignment: Optional `tf.contrib.tpu.DeviceAssignment` to specify
           the placement of replicas on the TPU cluster. Currently only supports
           the usecase of using a single core within a TPU cluster.
-      **kwargs: Additional experimental flags. Will be removed in future.
     """
-    if len(kwargs) > 1:
-      raise ValueError("TPUStrategy constructor only takes one experimental "
-                       "flag now")
-    elif len(kwargs) == 1 and "_disable_training_loop_on_host" not in kwargs:
-      raise ValueError("TPUStrategy constructor does not support arguments: "
-                       "{}".format(kwargs))
-
     super(TPUStrategy, self).__init__(TPUExtended(
-        self, tpu_cluster_resolver, steps_per_run, device_assignment,
-        kwargs.get("_disable_training_loop_on_host", False)))
+        self, tpu_cluster_resolver, steps_per_run, device_assignment))
 
   @property
   def steps_per_run(self):
@@ -196,11 +186,6 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
     if context.executing_eagerly():
       raise NotImplementedError("Eager mode not supported in TPUStrategy.")
 
-    if self.extended._disable_training_loop_on_host:  # pylint: disable=protected-access
-      raise NotImplementedError(
-          "`experimental_run` is not compatible with "
-          "`_disable_training_loop_on_host=True`")
-
     if input_iterator is None:
       inputs = []
     else:
@@ -241,8 +226,7 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
                container_strategy,
                tpu_cluster_resolver=None,
                steps_per_run=None,
-               device_assignment=None,
-               disable_training_loop_on_host=False):
+               device_assignment=None):
     super(TPUExtended, self).__init__(container_strategy)
 
     if tpu_cluster_resolver is None:
@@ -256,7 +240,6 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
     self._tpu_cluster_resolver = tpu_cluster_resolver
     self._tpu_metadata = get_tpu_system_metadata(self._tpu_cluster_resolver)
     self._device_assignment = device_assignment
-    self._disable_training_loop_on_host = disable_training_loop_on_host
 
     # Device assignment is currently only supported for 1 core case.
     if self._device_assignment:
@@ -284,25 +267,14 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
     self._tpu_devices = self._tpu_devices[:self._num_replicas_in_sync]
     self._device_map = values.ReplicaDeviceMap(self._tpu_devices)
 
-    # If the training loop is on the device, we must use the infeed, with input
-    # on the host. Otherwise, we preload the data onto the TPUs.
-    if disable_training_loop_on_host:
-      input_device_map = values.ReplicaDeviceMap(tuple(
-          self.get_host_cpu_device(hid) for hid in range(self.num_hosts)))
-      worker_devices = [
-          (self.get_host(hid), [self.get_host_cpu_device(hid)])
-          for hid in range(self.num_hosts)
-      ]
-      self._input_workers = input_lib.InputWorkers(
-          input_device_map, worker_devices)
-    else:
-      input_worker_devices = collections.OrderedDict()
-      for tpu_device in self._tpu_devices:
-        host_device = _get_host_for_device(tpu_device)
-        input_worker_devices.setdefault(host_device, [])
-        input_worker_devices[host_device].append(tpu_device)
-      self._input_workers = input_lib.InputWorkers(
-          self._device_map, tuple(input_worker_devices.items()))
+    # Preload the data onto the TPUs.
+    input_worker_devices = collections.OrderedDict()
+    for tpu_device in self._tpu_devices:
+      host_device = _get_host_for_device(tpu_device)
+      input_worker_devices.setdefault(host_device, [])
+      input_worker_devices[host_device].append(tpu_device)
+    self._input_workers = input_lib.InputWorkers(
+        self._device_map, tuple(input_worker_devices.items()))
 
     # TODO(sourabhbajaj): Remove this once performance of running one step
     # at a time is comparable to multiple steps.
@@ -402,17 +374,6 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
   # a mechanism to infer the outputs of `fn`. Pending b/110550782.
   def _experimental_run_steps_on_iterator(
       self, fn, multi_worker_iterator, iterations, initial_loop_values=None):
-    if self._disable_training_loop_on_host:
-      impl = self._run_steps_on_iterator_with_device_loop
-    else:
-      impl = self._run_steps_on_iterator_with_host_loop
-
-    return impl(
-        fn=fn, multi_worker_iterator=multi_worker_iterator,
-        iterations=iterations, initial_loop_values=initial_loop_values)
-
-  def _run_steps_on_iterator_with_host_loop(
-      self, fn, multi_worker_iterator, iterations, initial_loop_values=None):
     output_shapes = multi_worker_iterator.output_shapes
     shapes = nest.flatten(output_shapes)
     if any(not s.is_fully_defined() for s in shapes):
@@ -507,79 +468,6 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
     _set_last_step_outputs(ctx, last_step_tensor_outputs)
     return ctx
 
-  def _run_steps_on_iterator_with_device_loop(
-      self, fn, multi_worker_iterator, iterations, initial_loop_values=None):
-    output_shapes = multi_worker_iterator.output_shapes
-    shapes = nest.flatten(output_shapes)
-    if any(not s.is_fully_defined() for s in shapes):
-      raise ValueError(
-          "TPU currently requires fully defined shapes. Either use "
-          "set_shape() on the input tensors or use "
-          "dataset.batch(..., drop_remainder=True).")
-    types = nest.flatten(multi_worker_iterator.output_types)
-
-    enqueue_ops = [
-        self._get_enqueue_op_per_host(host_id, multi_worker_iterator, shapes,
-                                      iterations)
-        for host_id in range(self.num_hosts)]
-
-    def dequeue_fn():
-      dequeued = tpu_ops.infeed_dequeue_tuple(dtypes=types, shapes=shapes)
-      return nest.pack_sequence_as(output_shapes, dequeued)
-
-    # Wrap `fn` for repeat.
-    if initial_loop_values is None:
-      initial_loop_values = {}
-    initial_loop_values = nest.flatten(initial_loop_values)
-    ctx = input_lib.MultiStepContext()
-
-    def run_fn(*args, **kwargs):
-      """Single step on the TPU device."""
-      del args, kwargs
-      fn_result = fn(ctx, dequeue_fn())
-      flat_last_step_outputs = nest.flatten(ctx.last_step_outputs)
-      if flat_last_step_outputs:
-        with ops.control_dependencies([fn_result]):
-          return [array_ops.identity(f) for f in flat_last_step_outputs]
-      else:
-        return fn_result
-
-    def iterate_on_tpu():
-      return training_loop.repeat(iterations, run_fn, initial_loop_values)
-
-    # We capture the control_flow_context at this point, before we run `fn`
-    # inside a while_loop and TPU replicate context. This is useful in cases
-    # where we might need to exit these contexts and get back to the outer
-    # context to do some things, for e.g. create an op which should be
-    # evaluated only once at the end of the loop on the host. One such usage
-    # is in creating metrics' value op.
-    self._outer_control_flow_context = (
-        ops.get_default_graph()._get_control_flow_context())  # pylint: disable=protected-access
-
-    replicate_inputs = [[]] * self._num_replicas_in_sync
-    replicate_outputs = tpu.replicate(iterate_on_tpu, replicate_inputs)
-
-    del self._outer_control_flow_context
-    ctx.run_op = control_flow_ops.group(replicate_outputs, enqueue_ops)
-
-    # Filter out any ops from the outputs, typically this would be the case
-    # when there were no tensor outputs.
-    last_step_tensor_outputs = [x for x in replicate_outputs
-                                if not isinstance(x, ops.Operation)]
-
-    # Outputs are currently of the structure (grouped by device)
-    # [[output0_device0, output1_device0, output2_device0],
-    #  [output0_device1, output1_device1, output2_device1]]
-    # Convert this to the following structure instead: (grouped by output)
-    # [[output0_device0, output0_device1],
-    #  [output1_device0, output1_device1],
-    #  [output2_device0, output2_device1]]
-    last_step_tensor_outputs = [list(x) for x in
-                                zip(*last_step_tensor_outputs)]
-
-    _set_last_step_outputs(ctx, last_step_tensor_outputs)
-    return ctx
-
   def _call_for_each_replica(self, fn, args, kwargs):
     # TODO(jhseu): Consider making it so call_for_each_replica implies that
     # we're in a tpu.rewrite(), and update TPUMirroredVariable accordingly.
-- 
GitLab


From 3813019bbf73d889c67f2df9ade623bb87b8c767 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 4 Feb 2019 10:28:21 -0800
Subject: [PATCH 1847/2345] Fix "error: 'GetCurrentTimeNanos' is not a member
 of 'absl'" in traceme.h

PiperOrigin-RevId: 232321499
---
 tensorflow/core/profiler/lib/traceme.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/core/profiler/lib/traceme.h b/tensorflow/core/profiler/lib/traceme.h
index a6bff3adc5..098c531092 100644
--- a/tensorflow/core/profiler/lib/traceme.h
+++ b/tensorflow/core/profiler/lib/traceme.h
@@ -18,6 +18,7 @@ limitations under the License.
 #include <string>
 
 #include "absl/strings/string_view.h"
+#include "absl/time/clock.h"
 #include "absl/types/optional.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/types.h"
-- 
GitLab


From 10c0ca0a25a4326c365920f671268b079244b0d7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 4 Feb 2019 10:47:57 -0800
Subject: [PATCH 1848/2345] [TF:XLA] Implement eigen value decomposition for
 symmetric matrix in XLA.

PiperOrigin-RevId: 232325402
---
 tensorflow/compiler/xla/client/lib/BUILD      |  47 +++
 .../xla/client/lib/self_adjoint_eigen.cc      | 398 ++++++++++++++++++
 .../xla/client/lib/self_adjoint_eigen.h       |  42 ++
 .../xla/client/lib/self_adjoint_eigen_test.cc | 300 +++++++++++++
 4 files changed, 787 insertions(+)
 create mode 100644 tensorflow/compiler/xla/client/lib/self_adjoint_eigen.cc
 create mode 100644 tensorflow/compiler/xla/client/lib/self_adjoint_eigen.h
 create mode 100644 tensorflow/compiler/xla/client/lib/self_adjoint_eigen_test.cc

diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD
index b30ab84240..6be8154221 100644
--- a/tensorflow/compiler/xla/client/lib/BUILD
+++ b/tensorflow/compiler/xla/client/lib/BUILD
@@ -494,3 +494,50 @@ xla_test(
         "//tensorflow/core:test",
     ],
 )
+
+cc_library(
+    name = "self_adjoint_eigen",
+    srcs = ["self_adjoint_eigen.cc"],
+    hdrs = ["self_adjoint_eigen.h"],
+    deps = [
+        ":arithmetic",
+        ":constants",
+        ":loops",
+        ":math",
+        ":matrix",
+        ":slicing",
+        "//tensorflow/compiler/xla:literal_util",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:status_macros",
+        "//tensorflow/compiler/xla:statusor",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/client:xla_builder",
+        "//tensorflow/core:lib",
+    ],
+)
+
+xla_test(
+    name = "self_adjoint_eigen_test",
+    size = "medium",
+    srcs = ["self_adjoint_eigen_test.cc"],
+    real_hardware_only = True,
+    shard_count = 10,
+    tags = ["optonly"],
+    deps = [
+        ":arithmetic",
+        ":constants",
+        ":matrix",
+        ":self_adjoint_eigen",
+        "//tensorflow/compiler/xla:array2d",
+        "//tensorflow/compiler/xla:array3d",
+        "//tensorflow/compiler/xla:literal",
+        "//tensorflow/compiler/xla:statusor",
+        "//tensorflow/compiler/xla:test",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/client:xla_builder",
+        "//tensorflow/compiler/xla/tests:client_library_test_base",
+        "//tensorflow/compiler/xla/tests:literal_test_util",
+        "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+        "//tensorflow/core:test",
+    ],
+)
diff --git a/tensorflow/compiler/xla/client/lib/self_adjoint_eigen.cc b/tensorflow/compiler/xla/client/lib/self_adjoint_eigen.cc
new file mode 100644
index 0000000000..c2c8caee6e
--- /dev/null
+++ b/tensorflow/compiler/xla/client/lib/self_adjoint_eigen.cc
@@ -0,0 +1,398 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/client/lib/self_adjoint_eigen.h"
+
+#include <memory>
+#include <vector>
+
+#include "tensorflow/compiler/xla/client/lib/arithmetic.h"
+#include "tensorflow/compiler/xla/client/lib/constants.h"
+#include "tensorflow/compiler/xla/client/lib/loops.h"
+#include "tensorflow/compiler/xla/client/lib/math.h"
+#include "tensorflow/compiler/xla/client/lib/matrix.h"
+#include "tensorflow/compiler/xla/client/lib/slicing.h"
+#include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/status_macros.h"
+#include "tensorflow/compiler/xla/statusor.h"
+#include "tensorflow/core/lib/core/errors.h"
+
+namespace xla {
+
+namespace {
+
+// Jacobi rotation (also known as Givens rotation):
+// G = [[ c, s],
+//      [-s, c]]
+// matmul(G_T, G) = I
+struct SymmetricSchurDecomposition {
+  XlaOp c;          // cosine.
+  XlaOp s;          // sine.
+  XlaOp reduction;  // Reduction in the off diagonal after applying G.
+};
+
+// JacobiUpdate holds the intermediate orthogonal matrix, Jacobi-rotated matrix
+// and the off-diagonal norm of the rotated matrix. After each Jacobi iteration,
+// off-diagonal norm is reduced.
+struct JacobiUpdate {
+  XlaOp v;
+  XlaOp w;
+  XlaOp off_diagonal_norm;
+};
+
+// Given an n-by-n symmetric A and integers p and q that satisfy 0 <= p < q < n,
+// it computes a rotation matrix G = [[c, s], [-s, c]], such that
+//                        G_T * A[[p, q], [p, q]] * G
+// is diagonalized.
+//
+//  def sym_schur2x2(A, p, q):
+//      if np.abs(A[p, q]) > 1e-6:
+//          tau = (A[q, q] - A[p, p]) / (2 * A[p, q])
+//          if tau >= 0:
+//              t = 1.0 / (tau + np.sqrt(1 + tau ** 2))
+//          else:
+//              t = -1.0 / (-tau + np.sqrt(1 + tau ** 2))
+//          c = 1.0 / np.sqrt(1.0 + t ** 2)
+//          s = t * c
+//      else:
+//          c = 1.0
+//          s = 0.0
+//      return c, s
+StatusOr<SymmetricSchurDecomposition> SymmetricShurDecomposition2x2(XlaOp a,
+                                                                    XlaOp p,
+                                                                    XlaOp q,
+                                                                    XlaOp tol) {
+  XlaBuilder* builder = a.builder();
+  TF_ASSIGN_OR_RETURN(Shape a_shape, builder->GetShape(a));
+
+  PrimitiveType type = a_shape.element_type();
+
+  const int64 num_dims = a_shape.rank();
+
+  auto zero = ScalarLike(a, 0.0);
+  auto one = ScalarLike(a, 1.0);
+  auto two = ScalarLike(a, 2.0);
+
+  auto pqs = DynamicSliceInMinorDims(a, {p, q}, {1, 1});
+
+  auto ps = DynamicSliceInMinorDims(a, {p, p}, {1, 1});
+  auto qs = DynamicSliceInMinorDims(a, {q, q}, {1, 1});
+
+  auto tau = (qs - ps) / (pqs * two);
+  auto t_pos = one / (tau + Sqrt(one + Square(tau)));
+  auto t_neg = -one / (-tau + Sqrt(one + Square(tau)));
+  auto t = Select(Ge(tau, zero), t_pos, t_neg);
+
+  auto c_temp = Rsqrt(one + Square(t));
+  auto s_temp = t * c_temp;
+
+  auto c = Select(Ge(Abs(pqs), tol), c_temp, ZerosLike(c_temp) + one);
+  auto s = Select(Ge(Abs(pqs), tol), s_temp, ZerosLike(s_temp));
+  // Renormalize c and s to compensate for low precision arithmetic, this step
+  // is redundant if high precision float is used, like float64.
+  auto rnorm = Rsqrt(Square(c) + Square(s));
+
+  SymmetricSchurDecomposition schur;
+
+  schur.c = c * rnorm;
+  schur.s = s * rnorm;
+  schur.reduction =
+      Reduce(two * Square(pqs), zero, CreateScalarAddComputation(type, builder),
+             {num_dims - 2, num_dims - 1});
+  return schur;
+}
+
+StatusOr<JacobiUpdate> Update(JacobiUpdate jacobi_update, XlaOp p, XlaOp q,
+                              XlaOp tol, int64 n) {
+  XlaBuilder* builder = jacobi_update.w.builder();
+  TF_ASSIGN_OR_RETURN(
+      SymmetricSchurDecomposition schur,
+      SymmetricShurDecomposition2x2(jacobi_update.w, p, q, tol));
+
+  TF_ASSIGN_OR_RETURN(Shape w_shape, builder->GetShape(jacobi_update.w));
+  const std::vector<int64> batch_dims(w_shape.dimensions().begin(),
+                                      w_shape.dimensions().end() - 2);
+  const int64 num_dims = w_shape.rank();
+
+  auto zero = ScalarLike(p, 0);
+
+  XlaOp c = schur.c;
+  XlaOp s = schur.s;
+
+  auto slice_p = DynamicSliceInMinorDims(jacobi_update.w, {p, zero}, {1, n});
+  auto slice_q = DynamicSliceInMinorDims(jacobi_update.w, {q, zero}, {1, n});
+
+  auto slice_p_new = c * slice_p - s * slice_q;
+  auto slice_q_new = s * slice_p + c * slice_q;
+
+  jacobi_update.w =
+      DynamicUpdateSliceInMinorDims(jacobi_update.w, slice_p_new, {p, zero});
+  jacobi_update.w =
+      DynamicUpdateSliceInMinorDims(jacobi_update.w, slice_q_new, {q, zero});
+
+  slice_p = DynamicSliceInMinorDims(jacobi_update.w, {zero, p}, {n, 1});
+  slice_q = DynamicSliceInMinorDims(jacobi_update.w, {zero, q}, {n, 1});
+
+  slice_p_new = c * slice_p - s * slice_q;
+  slice_q_new = s * slice_p + c * slice_q;
+
+  jacobi_update.w =
+      DynamicUpdateSliceInMinorDims(jacobi_update.w, slice_p_new, {zero, p});
+  jacobi_update.w =
+      DynamicUpdateSliceInMinorDims(jacobi_update.w, slice_q_new, {zero, q});
+
+  // Zero out a_{pq} explicitly.
+  std::vector<int64> pq_dims(batch_dims.begin(), batch_dims.end());
+  pq_dims.push_back(1);
+  pq_dims.push_back(1);
+  auto pq_zero = ScalarLike(jacobi_update.w, 0.0);
+  auto pq_zeros = Broadcast(pq_zero, pq_dims);
+  jacobi_update.w =
+      DynamicUpdateSliceInMinorDims(jacobi_update.w, pq_zeros, {p, q});
+  jacobi_update.w =
+      DynamicUpdateSliceInMinorDims(jacobi_update.w, pq_zeros, {q, p});
+
+  slice_p = DynamicSliceInMinorDims(jacobi_update.v, {zero, p}, {n, 1});
+  slice_q = DynamicSliceInMinorDims(jacobi_update.v, {zero, q}, {n, 1});
+
+  std::vector<int64> broadcast_dims(batch_dims.size());
+  std::iota(broadcast_dims.begin(), broadcast_dims.end(), 0);
+  broadcast_dims.push_back(num_dims - 1);
+
+  // Renormalize the p-th and q-th columns. This step is redundant if high
+  // precision floats are used, like 64-bit float. But for 32-bit float, it
+  // becomes necessary. This step will not increase the overall complexity.
+  slice_p_new = c * slice_p - s * slice_q;
+  slice_p_new = Mul(
+      slice_p_new,
+      Rsqrt(Reduce(Square(slice_p_new), pq_zero,
+                   CreateScalarAddComputation(w_shape.element_type(), builder),
+                   {num_dims - 2})),
+      broadcast_dims);
+  slice_q_new = s * slice_p + c * slice_q;
+  slice_q_new = Mul(
+      slice_q_new,
+      Rsqrt(Reduce(Square(slice_q_new), pq_zero,
+                   CreateScalarAddComputation(w_shape.element_type(), builder),
+                   {num_dims - 2})),
+      broadcast_dims);
+
+  jacobi_update.v =
+      DynamicUpdateSliceInMinorDims(jacobi_update.v, slice_p_new, {zero, p});
+  jacobi_update.v =
+      DynamicUpdateSliceInMinorDims(jacobi_update.v, slice_q_new, {zero, q});
+
+  jacobi_update.off_diagonal_norm = Sqrt(
+      Max(Square(jacobi_update.off_diagonal_norm) - schur.reduction, pq_zero));
+
+  return jacobi_update;
+}
+
+StatusOr<std::vector<XlaOp>> WhileLoopFn(
+    absl::Span<const XlaOp> initial_values,  //
+    int matrix_dimension,                    //
+    int max_sweep_updates,                   //
+    PrimitiveType index_type,                //
+    absl::string_view name,                  //
+    XlaBuilder* builder) {
+  auto while_cond_fn = [&](absl::Span<const XlaOp> values,
+                           XlaBuilder* cond_builder) -> StatusOr<XlaOp> {
+    auto k = values[0];
+    auto off_diagonal_norm = values[5];
+    // tol = frobenius_norm * epsilon.
+    auto tol = values[6] * values[7];
+
+    auto max_sweeps = ScalarLike(k, max_sweep_updates);
+
+    auto sweep_update_cond = Gt(max_sweeps, k);
+
+    auto tol_cond = ReduceAll(Lt(tol, off_diagonal_norm),
+                              xla::ConstantR0<bool>(cond_builder, false),
+                              CreateScalarOrComputation(PRED, cond_builder));
+    return And(tol_cond, sweep_update_cond);
+  };
+
+  auto while_body_fn =
+      [&](absl::Span<const XlaOp> values,
+          XlaBuilder* body_builder) -> StatusOr<std::vector<XlaOp>> {
+    auto zero = Zero(body_builder, index_type);
+    auto one = One(body_builder, index_type);
+    auto end_index = ScalarLike(one, matrix_dimension);
+
+    // Indexes.
+    XlaOp k = values[0];
+    XlaOp p = values[1];
+    XlaOp q = values[2];
+
+    JacobiUpdate jacobi_update;
+    jacobi_update.v = values[3];
+    jacobi_update.w = values[4];
+    jacobi_update.off_diagonal_norm = values[5];
+
+    XlaOp frobenius_norm = values[6];
+    XlaOp tol = values[7];
+
+    TF_ASSIGN_OR_RETURN(jacobi_update,
+                        Update(jacobi_update, p, q, tol, matrix_dimension));
+
+    std::vector<XlaOp> updated_values;
+    updated_values.reserve(values.size());
+
+    q = q + one;
+    p = Select(Eq(q, end_index), p + one, p);
+    k = Select(Eq(p, end_index - one), k + one, k);
+    p = Select(Eq(p, end_index - one), zero, p);
+    q = Select(Eq(q, end_index), p + one, q);
+
+    updated_values.push_back(k);
+    updated_values.push_back(p);
+    updated_values.push_back(q);
+
+    updated_values.push_back(jacobi_update.v);
+    updated_values.push_back(jacobi_update.w);
+    updated_values.push_back(jacobi_update.off_diagonal_norm);
+
+    updated_values.push_back(frobenius_norm);
+    updated_values.push_back(tol);
+
+    return updated_values;
+  };
+  std::vector<XlaOp> values;
+  TF_ASSIGN_OR_RETURN(values, WhileLoopHelper(while_cond_fn, while_body_fn,
+                                              initial_values, name, builder));
+
+  return values;
+}
+
+}  // namespace
+
+// This is the cyclic Jacobi iteration. Please note that the eigenvalues are
+// possibly not ordered.
+//
+//  def jacobi(A):
+//      n, _ = A.shape
+//      V = np.eye(n)
+//      nfrob = np.sum(A ** 2)
+//      ndiag = np.sum(np.diag(A) ** 2)
+//      off = nfrob - ndiag
+//      while off > 1e-6 * nfrob:
+//          for p in range(n - 1):
+//              for q in range(p + 1, n):
+//                  if off > 1e-6 * nfrob:
+//                      c, s = sym_schur2x2(A, p, q)
+//                      off = off - 2 * A[p, q] ** 2
+//                      A[[p, q], :] = np.matmul(np.array([[c, -s], [s, c]]),
+//                                               A[[p, q], :])
+//                      A[:, [p, q]] = np.matmul(A[:, [p, q]],
+//                                               np.array([[c, s], [-s, c]]))
+//                      V[:, [p, q]] = np.matmul(V[:, [p, q]],
+//                                               np.array([[c, s], [-s, c]]))
+//
+//      return A, V
+//
+// TODO(kuny): Implement parallel order Jacobi.
+//
+SelfAdjointEigenResult SelfAdjointEigen(XlaOp a, bool lower, int64 max_iter,
+                                        float epsilon) {
+  XlaBuilder* builder = a.builder();
+  auto return_error = [&](const Status& status) {
+    SelfAdjointEigenResult result;
+    result.v = builder->ReportError(status);
+    result.w = builder->ReportError(status);
+    return result;
+  };
+  auto shape_with_status = builder->GetShape(a);
+  if (!shape_with_status.status().ok()) {
+    return return_error(shape_with_status.status());
+  }
+  Shape a_shape = shape_with_status.ValueOrDie();
+  const int64 num_dims = a_shape.rank();
+  if (num_dims < 2) {
+    return return_error(InvalidArgument(
+        "Arguments to Eigen decomposition must have rank >= 2: got shape %s.",
+        a_shape.ToString()));
+  }
+  PrimitiveType type = a_shape.element_type();
+  if (!primitive_util::IsFloatingPointType(type)) {
+    return return_error(InvalidArgument(
+        "Type of the input matrix must be float: got %s.", a_shape.ToString()));
+  }
+
+  const int64 m = ShapeUtil::GetDimension(a_shape, -2);
+  const int64 n = ShapeUtil::GetDimension(a_shape, -1);
+
+  if (m != n) {
+    return return_error(InvalidArgument(
+        "Arguments to Eigen decomposition must be square matrices: got shape "
+        "(%d, %d).",
+        m, n));
+  }
+
+  const int64 num_batch_dims = num_dims - 2;
+  std::vector<int64> batch_dims(num_batch_dims);
+  for (int i = 0; i < num_batch_dims; ++i) {
+    batch_dims[i] = ShapeUtil::GetDimension(a_shape, i);
+  }
+
+  auto zero = ScalarLike(a, 0.0);
+  auto tol = ScalarLike(a, epsilon);
+
+  auto v_init = Broadcast(IdentityMatrix(builder, type, m, m), batch_dims);
+  auto w_init = Triangle(a, lower);
+  w_init = w_init + TransposeInMinorDims(w_init) - w_init * v_init;
+
+  auto frobenius_norm = Sqrt(Reduce(Square(w_init), zero,
+                                    CreateScalarAddComputation(type, builder),
+                                    {num_dims - 2, num_dims - 1}));
+  auto diag = GetMatrixDiagonal(w_init);
+  auto diag_square =
+      Reduce(Square(diag), zero, CreateScalarAddComputation(type, builder),
+             {num_dims - 2});
+
+  auto off_diagonal_init =
+      Sqrt(Max(Square(frobenius_norm) - diag_square, zero));
+
+  auto output_with_status = WhileLoopFn(
+      {
+          Zero(builder, S32),  // k
+          Zero(builder, S32),  // p
+          One(builder, S32),   // q
+          v_init,              //
+          w_init,              //
+          off_diagonal_init,   //
+          frobenius_norm,      //
+          tol,                 //
+      },                       //
+      n,                       //
+      max_iter,                //
+      S32,                     //
+      "CyclicJacobi",          //
+      builder);
+  if (!output_with_status.status().ok()) {
+    return return_error(output_with_status.status());
+  }
+
+  auto output = output_with_status.ValueOrDie();
+
+  SelfAdjointEigenResult result;
+  result.v = output[3];
+  result.w = GetMatrixDiagonal(output[4]);
+
+  return result;
+}
+
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/client/lib/self_adjoint_eigen.h b/tensorflow/compiler/xla/client/lib/self_adjoint_eigen.h
new file mode 100644
index 0000000000..49fc17aa27
--- /dev/null
+++ b/tensorflow/compiler/xla/client/lib/self_adjoint_eigen.h
@@ -0,0 +1,42 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_CLIENT_LIB_SELF_ADJOINT_EIGEN_H_
+#define TENSORFLOW_COMPILER_XLA_CLIENT_LIB_SELF_ADJOINT_EIGEN_H_
+
+#include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+
+namespace xla {
+
+// The eigenvalue decomposition of a symmetric matrix, the original matrix is
+// recovered by v * w * v_t.
+struct SelfAdjointEigenResult {
+  // The i-th column is the normalized eigenvector corresponding to the
+  // eigenvalue w[i]. Will return a matrix object if a is a matrix object.
+  XlaOp v;
+  // TODO(kuny): Sort the eigenvalues.
+  // The eigenvalues in ascending order, each repeated according to its
+  // multiplicity.
+  XlaOp w;
+};
+
+SelfAdjointEigenResult SelfAdjointEigen(XlaOp a, bool lower = true,
+                                        int64 max_iter = 100,
+                                        float epsilon = 1e-6);
+
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_CLIENT_LIB_SELF_ADJOINT_EIGEN_H_
diff --git a/tensorflow/compiler/xla/client/lib/self_adjoint_eigen_test.cc b/tensorflow/compiler/xla/client/lib/self_adjoint_eigen_test.cc
new file mode 100644
index 0000000000..720c49b771
--- /dev/null
+++ b/tensorflow/compiler/xla/client/lib/self_adjoint_eigen_test.cc
@@ -0,0 +1,300 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/client/lib/self_adjoint_eigen.h"
+
+#include "tensorflow/compiler/xla/array2d.h"
+#include "tensorflow/compiler/xla/array3d.h"
+#include "tensorflow/compiler/xla/client/lib/arithmetic.h"
+#include "tensorflow/compiler/xla/client/lib/constants.h"
+#include "tensorflow/compiler/xla/client/lib/matrix.h"
+#include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/compiler/xla/literal.h"
+#include "tensorflow/compiler/xla/statusor.h"
+#include "tensorflow/compiler/xla/test.h"
+#include "tensorflow/compiler/xla/tests/client_library_test_base.h"
+#include "tensorflow/compiler/xla/tests/literal_test_util.h"
+#include "tensorflow/compiler/xla/tests/test_macros.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+
+namespace xla {
+
+class SelfAdjointEigenTest : public ClientLibraryTestBase {
+ protected:
+  void SetUp() override {
+    ClientLibraryTestBase::SetUp();
+    batch_3d_4x4_ = Array3D<float>{
+        {
+            {4, 6, 8, 10},
+            {6, 45, 54, 63},
+            {8, 54, 146, 166},
+            {10, 63, 166, 310},
+        },
+        {
+            {16, 24, 8, 12},
+            {24, 61, 82, 48},
+            {8, 82, 100, 6},
+            {12, 48, 6, 62},
+        },
+    };
+    matrix2d_8x8_ = Array2D<float>{
+        {14., 123., 49., 112., 115., 173., 182., 125.},
+        {123., 14., 60., 118., 150., 130., 91., 72.},
+        {49., 60., 138., 111., 106., 101., 115., 142.},
+        {112., 118., 111., 142., 91., 130., 25., 61.},
+        {115., 150., 106., 91., 116., 121., 128., 85.},
+        {173., 130., 101., 130., 121., 70., 151., 132.},
+        {182., 91., 115., 25., 128., 151., 66., 92.},
+        {125., 72., 142., 61., 85., 132., 92., 156.},
+    };
+    low_rank_4x4_ = Array2D<float>{
+        // x = [[1, 2, 3, 4], [1, -1, 1, -1]]
+        // matmul(x.T, x)
+        {2, 1, 4, 3},
+        {1, 5, 5, 9},
+        {4, 5, 10, 11},
+        {3, 9, 11, 17},
+    };
+  }
+  void TearDown() override { ClientLibraryTestBase::TearDown(); }
+
+  Array3D<float> get_unit_matrix_3d(const Array3D<float>& matrix) {
+    Array3D<float> result(matrix.n1(), matrix.n2(), matrix.n3(), 0.0);
+    for (int i = 0; i < matrix.n1(); ++i) {
+      for (int j = 0; j < matrix.n2(); ++j) {
+        result({i, j, j}) = 1.0;
+      }
+    }
+    return result;
+  }
+
+  Array3D<float> ExtractTriangularMatrix(const Array3D<float>& matrix,
+                                         bool lower) {
+    Array3D<float> result(matrix);
+    for (int i = 0; i < result.n1(); ++i) {
+      for (int j = 0; j < result.n2(); ++j) {
+        if (lower) {
+          for (int k = j + 1; k < result.n3(); ++k) {
+            result({i, j, k}) = 0.0;
+          }
+        } else {
+          for (int k = 0; k < j; ++k) {
+            result({i, j, k}) = 0.0;
+          }
+        }
+      }
+    }
+    return result;
+  }
+
+  XlaOp ComputeMatmulVWVt(SelfAdjointEigenResult result, XlaBuilder* builder) {
+    Shape shape = builder->GetShape(result.v).ValueOrDie();
+    std::vector<int64> out_dims = shape.dimensions();
+    std::vector<int64> broadcast_dims(shape.rank() - 1);
+    std::iota(broadcast_dims.begin(), broadcast_dims.end(), 0);
+
+    broadcast_dims[shape.rank() - 2] = shape.rank() - 1;
+    auto vw = Mul(result.v, BroadcastInDim(result.w, out_dims, broadcast_dims));
+    return BatchDot(vw, TransposeInMinorDims(result.v),
+                    PrecisionConfig::HIGHEST);
+  }
+
+  XlaOp GetAverageAbsoluteError(XlaOp m1, XlaOp m2, XlaBuilder* builder) {
+    Shape shape = builder->GetShape(m1).ValueOrDie();
+    int64 size = 1;
+    for (auto d : shape.dimensions()) {
+      size *= d;
+    }
+    return ReduceAll(Abs(m1 - m2), ConstantR0WithType(builder, F32, 0),
+                     CreateScalarAddComputation(F32, builder)) /
+           ConstantR0WithType(builder, F32, size);
+  }
+
+  Array2D<float> GenerateRandomSymmetricMatrix(int size) {
+    Array2D<float> result{size, size, 0.0};
+    result.FillRandom(10 /* stddev */, 2 /* mean */);
+    for (int i = 0; i < size; ++i) {
+      for (int j = 0; j < i; ++j) {
+        result({j, i}) = result({i, j});
+      }
+    }
+    return result;
+  }
+
+  Array3D<float> batch_3d_4x4_;
+  Array2D<float> matrix2d_8x8_;
+  Array2D<float> low_rank_4x4_;
+  Array2D<int> wrong_type_4x4_;
+};
+
+XLA_TEST_F(SelfAdjointEigenTest, Test_VWVt_EQ_A_2x4x4) {
+  XlaBuilder builder(TestName());
+
+  XlaOp a;
+  auto a_data = CreateR3Parameter<float>(batch_3d_4x4_, 0, "a", &builder, &a);
+  auto result = SelfAdjointEigen(a);
+  ComputeMatmulVWVt(result, &builder);
+
+  ComputeAndCompareR3<float>(&builder, batch_3d_4x4_, {a_data.get()},
+                             ErrorSpec(1e-3, 1e-3));
+}
+
+XLA_TEST_F(SelfAdjointEigenTest, Test_VWVt_EQ_A_Lower_2x4x4) {
+  XlaBuilder builder(TestName());
+
+  XlaOp a;
+  auto a_data = CreateR3Parameter<float>(
+      ExtractTriangularMatrix(batch_3d_4x4_, true), 0, "a", &builder, &a);
+  auto result = SelfAdjointEigen(a);
+  ComputeMatmulVWVt(result, &builder);
+
+  ComputeAndCompareR3<float>(&builder, batch_3d_4x4_, {a_data.get()},
+                             ErrorSpec(1e-3, 1e-3));
+}
+
+XLA_TEST_F(SelfAdjointEigenTest, Test_VWVt_EQ_A_Upper_2x4x4) {
+  XlaBuilder builder(TestName());
+
+  XlaOp a;
+  auto a_data = CreateR3Parameter<float>(
+      ExtractTriangularMatrix(batch_3d_4x4_, false), 0, "a", &builder, &a);
+  auto result = SelfAdjointEigen(a, false);
+  ComputeMatmulVWVt(result, &builder);
+
+  ComputeAndCompareR3<float>(&builder, batch_3d_4x4_, {a_data.get()},
+                             ErrorSpec(1e-3, 1e-3));
+}
+
+XLA_TEST_F(SelfAdjointEigenTest, Test_Orthogonality_2x4x4) {
+  XlaBuilder builder(TestName());
+
+  XlaOp a;
+  auto a_data = CreateR3Parameter<float>(batch_3d_4x4_, 0, "a", &builder, &a);
+  auto result = SelfAdjointEigen(a);
+  BatchDot(result.v, TransposeInMinorDims(result.v), PrecisionConfig::HIGHEST);
+
+  ComputeAndCompareR3<float>(&builder, get_unit_matrix_3d(batch_3d_4x4_),
+                             {a_data.get()}, ErrorSpec(1e-3, 1e-3));
+}
+
+XLA_TEST_F(SelfAdjointEigenTest, Test_VtWV_EQ_A_Rank_Deficient_4x4) {
+  XlaBuilder builder(TestName());
+
+  XlaOp a;
+  auto a_data = CreateR2Parameter<float>(low_rank_4x4_, 0, "a", &builder, &a);
+  auto result = SelfAdjointEigen(a);
+  ComputeMatmulVWVt(result, &builder);
+
+  ComputeAndCompareR2<float>(&builder, low_rank_4x4_, {a_data.get()},
+                             ErrorSpec(1e-3, 1e-3));
+}
+
+XLA_TEST_F(SelfAdjointEigenTest, Test_Eigen_8x8) {
+  XlaBuilder builder(TestName());
+
+  // This is computed by numpy.linalg.eigh with float32.
+  std::vector<float> expected{-182.69205, -116.86245, -105.74489, -9.545369,
+                              37.81711,   104.732285, 120.29153,  868.00385};
+
+  XlaOp a;
+  auto a_data = CreateR2Parameter<float>(matrix2d_8x8_, 0, "a", &builder, &a);
+  auto result = SelfAdjointEigen(a);
+  Sort(result.w);
+
+  ComputeAndCompareR1<float>(&builder, expected, {a_data.get()},
+                             ErrorSpec(1e-3, 1e-3));
+}
+
+XLA_TEST_F(SelfAdjointEigenTest, Test_Orthogonality_8x8) {
+  XlaBuilder builder(TestName());
+
+  float expected_vals = 1e-3;
+
+  XlaOp a;
+  auto a_data = CreateR2Parameter<float>(matrix2d_8x8_, 0, "a", &builder, &a);
+  auto result = SelfAdjointEigen(a);
+  // np.sum(norm(eye(n) - matmul(conj(T(v)), v)) / n**2
+  GetAverageAbsoluteError(IdentityMatrix(&builder, F32, 8, 8),
+                          BatchDot(TransposeInMinorDims(result.v), result.v),
+                          &builder);
+
+  ComputeAndCompareR0<float>(&builder, expected_vals, {a_data.get()},
+                             ErrorSpec(1e-3, 1e-3));
+}
+
+XLA_TEST_F(SelfAdjointEigenTest, Wrong_Type_Int) {
+  XlaBuilder builder(TestName());
+
+  XlaOp a;
+  auto a_data = CreateR2Parameter<int>(wrong_type_4x4_, 0, "a", &builder, &a);
+  auto result = SelfAdjointEigen(a);
+  EXPECT_FALSE(result.v.valid());
+  EXPECT_FALSE(result.w.valid());
+}
+
+XLA_TEST_F(SelfAdjointEigenTest, Various_Size_Random_Matrix_8x8) {
+  XlaBuilder builder(TestName());
+  int size = 8;
+  Array2D<float> a_val = GenerateRandomSymmetricMatrix(size);
+  XlaOp a;
+  auto a_data = CreateR2Parameter<float>(a_val, 0, "a", &builder, &a);
+  auto result = SelfAdjointEigen(a);
+  GetAverageAbsoluteError(ComputeMatmulVWVt(result, &builder), a, &builder);
+
+  ComputeAndCompareR0<float>(&builder, 1e-3, {a_data.get()},
+                             ErrorSpec(1e-2, 1e-2));
+}
+
+XLA_TEST_F(SelfAdjointEigenTest, Various_Size_Random_Matrix_16x16) {
+  XlaBuilder builder(TestName());
+  int size = 16;
+  Array2D<float> a_val = GenerateRandomSymmetricMatrix(size);
+  XlaOp a;
+  auto a_data = CreateR2Parameter<float>(a_val, 0, "a", &builder, &a);
+  auto result = SelfAdjointEigen(a);
+  GetAverageAbsoluteError(ComputeMatmulVWVt(result, &builder), a, &builder);
+
+  ComputeAndCompareR0<float>(&builder, 1e-3, {a_data.get()},
+                             ErrorSpec(1e-2, 1e-2));
+}
+
+XLA_TEST_F(SelfAdjointEigenTest, Various_Size_Random_Matrix_32x32) {
+  XlaBuilder builder(TestName());
+  int size = 32;
+  Array2D<float> a_val = GenerateRandomSymmetricMatrix(size);
+  XlaOp a;
+  auto a_data = CreateR2Parameter<float>(a_val, 0, "a", &builder, &a);
+  auto result = SelfAdjointEigen(a);
+  GetAverageAbsoluteError(ComputeMatmulVWVt(result, &builder), a, &builder);
+
+  ComputeAndCompareR0<float>(&builder, 1e-3, {a_data.get()},
+                             ErrorSpec(1e-2, 1e-2));
+}
+
+XLA_TEST_F(SelfAdjointEigenTest, Various_Size_Random_Matrix_64x64) {
+  XlaBuilder builder(TestName());
+  int size = 64;
+  Array2D<float> a_val = GenerateRandomSymmetricMatrix(size);
+  XlaOp a;
+  auto a_data = CreateR2Parameter<float>(a_val, 0, "a", &builder, &a);
+  auto result = SelfAdjointEigen(a);
+  GetAverageAbsoluteError(ComputeMatmulVWVt(result, &builder), a, &builder);
+
+  ComputeAndCompareR0<float>(&builder, 1e-3, {a_data.get()},
+                             ErrorSpec(1e-2, 1e-2));
+}
+
+}  // namespace xla
-- 
GitLab


From 9e11765dad33212225a92c866c39a248645bc00c Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Mon, 4 Feb 2019 10:50:03 -0800
Subject: [PATCH 1849/2345] Add missing `override` in KernelAndDeviceOp

PiperOrigin-RevId: 232325785
---
 tensorflow/core/common_runtime/eager/kernel_and_device.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/common_runtime/eager/kernel_and_device.h b/tensorflow/core/common_runtime/eager/kernel_and_device.h
index c4ea99f53e..027168dd6e 100644
--- a/tensorflow/core/common_runtime/eager/kernel_and_device.h
+++ b/tensorflow/core/common_runtime/eager/kernel_and_device.h
@@ -151,7 +151,7 @@ class KernelAndDeviceOp final : public KernelAndDevice {
   Device* OutputResourceDevice(int idx) const override;
 
   DataType input_type(int i) const override;
-  const DataTypeVector& output_dtypes() const {
+  const DataTypeVector& output_dtypes() const override {
     return kernel_->output_types();
   }
   int num_inputs() const override { return kernel_->num_inputs(); }
-- 
GitLab


From 3111198adef9a3b56bd380588d38168e63a2e3f0 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Mon, 4 Feb 2019 11:04:48 -0800
Subject: [PATCH 1850/2345] Allow fields to contain collections non-Nodes (i.e.
 strings).

PiperOrigin-RevId: 232328994
---
 tensorflow/python/autograph/pyct/pretty_printer.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tensorflow/python/autograph/pyct/pretty_printer.py b/tensorflow/python/autograph/pyct/pretty_printer.py
index d963416bed..a92017f414 100644
--- a/tensorflow/python/autograph/pyct/pretty_printer.py
+++ b/tensorflow/python/autograph/pyct/pretty_printer.py
@@ -56,6 +56,15 @@ class PrettyPrinter(gast.NodeVisitor):
     self.result += '\n'
 
   def generic_visit(self, node, name=None):
+    # In very rare instances, a list can contain something other than a Node.
+    # e.g. Global contains a list of strings.
+    if isinstance(node, str):
+      if name:
+        self._print('%s%s="%s"' % (self._indent(), name, node))
+      else:
+        self._print('%s"%s"' % (self._indent(), node))
+      return
+
     if node._fields:
       cont = ':'
     else:
-- 
GitLab


From d3d72266051c85963dc5bf4ee75052697dea538e Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Mon, 4 Feb 2019 11:05:53 -0800
Subject: [PATCH 1851/2345] Refactor checkpointable utilities to allow object
 graph customization

We want to add "virtual" objects like signatures before saving a SavedModel. Before this CL the saving utilities were a bit too literal for that to work.

Adds an ObjectGraphView, and gives one of those to the SavedModel _SaveableView. This will let the _SaveableView (or the thing that constructs it) add new edges that weren't in the saving program while making it clear that the same set of objects is going into the SavedModel and being saved in the checkpoint.

This CL isn't changing functionality, just shuffling it around a bit. I removed tf.contrib.eager.CheckpointableSaver as a symbol because its signature is changing and nobody is using it (that I can see); people can just use tf.train.Checkpoint. I've switched unit tests which were using it to using Checkpoint instead.

PiperOrigin-RevId: 232329203
---
 tensorflow/contrib/eager/python/tfe.py        |   2 -
 .../optimizer_v2/checkpointable_utils_test.py |  40 +-
 tensorflow/python/keras/engine/network.py     |   5 +-
 tensorflow/python/keras/saving/saved_model.py |   7 +-
 tensorflow/python/saved_model/BUILD           |  16 +-
 tensorflow/python/saved_model/load.py         |   7 +-
 tensorflow/python/saved_model/save.py         |  33 +-
 .../python/training/checkpointable/BUILD      |  35 +-
 .../python/training/checkpointable/base.py    |  27 +-
 .../training/checkpointable/graph_view.py     | 432 ++++++++++++
 .../checkpointable/object_identity.py         | 156 +++++
 .../python/training/checkpointable/util.py    | 649 ++----------------
 .../training/checkpointable/util_test.py      | 146 ++--
 .../util_with_v1_optimizers_test.py           |  41 +-
 tensorflow/python/training/saver_test.py      |   6 +-
 15 files changed, 839 insertions(+), 763 deletions(-)
 create mode 100644 tensorflow/python/training/checkpointable/graph_view.py
 create mode 100644 tensorflow/python/training/checkpointable/object_identity.py

diff --git a/tensorflow/contrib/eager/python/tfe.py b/tensorflow/contrib/eager/python/tfe.py
index b82e1bb71b..12bbdc08cf 100644
--- a/tensorflow/contrib/eager/python/tfe.py
+++ b/tensorflow/contrib/eager/python/tfe.py
@@ -62,7 +62,6 @@ To use, at program startup, call `tf.enable_eager_execution()`.
 
 @@Checkpoint
 @@Checkpointable
-@@CheckpointableSaver
 
 @@executing_eagerly
 @@in_eager_mode
@@ -139,7 +138,6 @@ from tensorflow.python.ops.variable_scope import EagerVariableStore
 from tensorflow.python.ops import script_ops
 from tensorflow.python.ops import template
 from tensorflow.python.training.checkpointable.tracking import AutoCheckpointable as Checkpointable
-from tensorflow.python.training.checkpointable.util import CheckpointableSaver
 from tensorflow.python.training.checkpointable.util import Checkpoint
 from tensorflow.python.util.all_util import remove_undocumented
 
diff --git a/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py b/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py
index 0243927ce4..b5de726a4c 100644
--- a/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py
+++ b/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py
@@ -44,6 +44,7 @@ from tensorflow.python.ops import variable_scope
 from tensorflow.python.training import checkpoint_management
 from tensorflow.python.training import saver as core_saver
 from tensorflow.python.training import training_util
+from tensorflow.python.training.checkpointable import graph_view
 from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.training.checkpointable import util
 
@@ -118,9 +119,8 @@ class CheckpointingTests(test.TestCase):
       self.evaluate(util.gather_initializers(
           root_checkpointable))
       self.evaluate(train_op)
-    named_variables, serialized_graph, _ = (
-        util._serialize_object_graph(
-            root_checkpointable, saveables_cache=None))
+    named_variables, serialized_graph, _ = graph_view.ObjectGraphView(
+        root_checkpointable).serialize_object_graph()
     expected_checkpoint_names = (
         # Created in the root node, so no prefix.
         "optimizer_step",
@@ -440,7 +440,7 @@ class CheckpointingTests(test.TestCase):
   def testDeferredSlotRestoration(self):
     checkpoint_directory = self.get_temp_dir()
 
-    root = tracking.AutoCheckpointable()
+    root = util.Checkpoint()
     root.var = util.add_variable(
         root, name="var", initializer=0.)
     optimizer = adam.AdamOptimizer(0.1)
@@ -455,21 +455,17 @@ class CheckpointingTests(test.TestCase):
           util.Checkpoint(root=root, optimizer=optimizer)))
       self.evaluate(train_op)
     self.evaluate(state_ops.assign(root.var, 12.))
-    no_slots_path = util.CheckpointableSaver(root).save(
-        os.path.join(checkpoint_directory, "no_slots"))
+    no_slots_path = root.save(os.path.join(checkpoint_directory, "no_slots"))
     root.optimizer = optimizer
     self.evaluate(state_ops.assign(root.var, 13.))
     self.evaluate(state_ops.assign(optimizer.get_slot(name="m", var=root.var),
                                    14.))
-    slots_path = util.CheckpointableSaver(root).save(
-        os.path.join(checkpoint_directory, "with_slots"))
-    new_root = tracking.AutoCheckpointable()
+    slots_path = root.save(os.path.join(checkpoint_directory, "with_slots"))
+    new_root = util.Checkpoint()
     # Load the slot-containing checkpoint (deferred), then immediately overwrite
     # the non-slot variable (also deferred).
-    slot_status = util.CheckpointableSaver(
-        new_root).restore(slots_path)
-    no_slot_status = util.CheckpointableSaver(
-        new_root).restore(no_slots_path)
+    slot_status = new_root.restore(slots_path)
+    no_slot_status = new_root.restore(no_slots_path)
     with self.assertRaises(AssertionError):
       no_slot_status.assert_consumed()
     new_root.var = util.add_variable(
@@ -508,15 +504,14 @@ class CheckpointingTests(test.TestCase):
       with graph.as_default(), self.session(graph):
         checkpoint_directory = self.get_temp_dir()
         checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-        obj = tracking.AutoCheckpointable()
+        obj = util.Checkpoint()
         obj.var = variable_scope.get_variable(name="v", initializer=0.)
         obj.opt = adam.AdamOptimizer(0.1)
         obj.opt.minimize(obj.var.read_value())
         self.evaluate(util.gather_initializers(obj))
-        saver = util.CheckpointableSaver(obj)
-        saver.save(checkpoint_prefix)
+        obj.save(checkpoint_prefix)
         before_ops = graph.get_operations()
-        saver.save(checkpoint_prefix)
+        obj.save(checkpoint_prefix)
         self.assertEqual(before_ops, graph.get_operations())
 
   def testManyRestoresGraph(self):
@@ -526,16 +521,15 @@ class CheckpointingTests(test.TestCase):
       with graph.as_default(), self.session(graph):
         checkpoint_directory = self.get_temp_dir()
         checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-        obj = tracking.AutoCheckpointable()
+        obj = util.Checkpoint()
         obj.var = variable_scope.get_variable(name="v", initializer=0.)
         obj.opt = adam.AdamOptimizer(0.1)
         obj.opt.minimize(obj.var.read_value())
         self.evaluate(util.gather_initializers(obj))
-        saver = util.CheckpointableSaver(obj)
-        save_path = saver.save(checkpoint_prefix)
-        saver.restore(save_path)
+        save_path = obj.save(checkpoint_prefix)
+        obj.restore(save_path)
         before_ops = graph.get_operations()
-        saver.restore(save_path)
+        obj.restore(save_path)
         self.assertEqual(before_ops, graph.get_operations())
 
   def testMultipleGraphsNonSlotVariables(self):
@@ -704,7 +698,7 @@ class CheckpointCompatibilityTests(test.TestCase):
       self._set_sentinels(root)
       with self.assertRaises(AssertionError):
         self._check_sentinels(root)
-      object_saver = util.CheckpointableSaver(root)
+      object_saver = util.CheckpointableSaver(graph_view.ObjectGraphView(root))
       self._set_sentinels(root)
       status = object_saver.restore(save_path)
       if context.executing_eagerly():
diff --git a/tensorflow/python/keras/engine/network.py b/tensorflow/python/keras/engine/network.py
index 0835e7c083..559df56357 100644
--- a/tensorflow/python/keras/engine/network.py
+++ b/tensorflow/python/keras/engine/network.py
@@ -22,7 +22,6 @@ from __future__ import print_function
 import copy
 import json
 import os
-import weakref
 
 import numpy as np
 from six.moves import zip  # pylint: disable=redefined-builtin
@@ -207,8 +206,8 @@ class Network(base_layer.Layer):
     self._outbound_nodes = []
     self._inbound_nodes = []
 
-    self._checkpointable_saver = checkpointable_utils.CheckpointableSaver(
-        weakref.ref(self))
+    self._checkpointable_saver = (
+        checkpointable_utils.saver_with_op_caching(self))
 
   @checkpointable.no_automatic_dependency_tracking
   def _init_graph_network(self, inputs, outputs, name=None):
diff --git a/tensorflow/python/keras/saving/saved_model.py b/tensorflow/python/keras/saving/saved_model.py
index fbf0bf68ef..6e2f3ddacb 100644
--- a/tensorflow/python/keras/saving/saved_model.py
+++ b/tensorflow/python/keras/saving/saved_model.py
@@ -37,7 +37,7 @@ from tensorflow.python.saved_model import save as save_lib
 from tensorflow.python.saved_model import utils_impl as saved_model_utils
 from tensorflow.python.training import mode_keys
 from tensorflow.python.training import saver as saver_lib
-from tensorflow.python.training.checkpointable import util as checkpointable_utils
+from tensorflow.python.training.checkpointable import graph_view
 from tensorflow.python.util import compat
 from tensorflow.python.util import nest
 from tensorflow.python.util.tf_export import keras_export
@@ -220,7 +220,8 @@ def _save_v1_format(model, path, custom_objects, as_text, input_signature):
 
 def _get_var_list(model):
   """Returns list of all checkpointed saveable objects in the model."""
-  return checkpointable_utils.named_saveables(model)
+  var_list, _, _ = graph_view.ObjectGraphView(model).serialize_object_graph()
+  return var_list
 
 
 def create_placeholder(spec):
@@ -287,7 +288,7 @@ def _export_mode(
       clone._make_predict_function()
     g.get_collection_ref(ops.GraphKeys.UPDATE_OPS).extend(clone.state_updates)
 
-    clone_var_list = checkpointable_utils.named_saveables(clone)
+    clone_var_list = _get_var_list(clone)
 
     with session.Session().as_default():
       if has_saved_vars:
diff --git a/tensorflow/python/saved_model/BUILD b/tensorflow/python/saved_model/BUILD
index 5d08a40d67..b605aa746c 100644
--- a/tensorflow/python/saved_model/BUILD
+++ b/tensorflow/python/saved_model/BUILD
@@ -271,6 +271,7 @@ py_library(
         ":builder",
         ":constants",
         ":function_serialization",
+        ":nested_structure_coder",
         ":revived_types",
         ":saved_object_graph_py",
         ":signature_constants",
@@ -286,14 +287,16 @@ py_library(
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:lib",
         "//tensorflow/python:resource_variable_ops",
-        "//tensorflow/python:tensor_spec",
         "//tensorflow/python:util",
         "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:def_function",
         "//tensorflow/python/eager:function",
         "//tensorflow/python/training/checkpointable:base",
+        "//tensorflow/python/training/checkpointable:graph_view",
+        "//tensorflow/python/training/checkpointable:object_identity",
         "//tensorflow/python/training/checkpointable:tracking",
         "//tensorflow/python/training/checkpointable:util",
+        "//tensorflow/python/training/saving:functional_saver",
     ],
 )
 
@@ -321,13 +324,22 @@ py_library(
         ":constants",
         ":function_deserialization",
         ":loader",
+        ":nested_structure_coder",
         ":revived_types",
         ":saved_object_graph_py",
         ":utils",
-        "//tensorflow/python:function",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:init_ops",
         "//tensorflow/python:lib",
+        "//tensorflow/python:resource_variable_ops",
+        "//tensorflow/python:tensor_util",
         "//tensorflow/python:util",
+        "//tensorflow/python:variables",
+        "//tensorflow/python/training/checkpointable:base",
+        "//tensorflow/python/training/checkpointable:graph_view",
         "//tensorflow/python/training/checkpointable:tracking",
+        "//tensorflow/python/training/checkpointable:util",
     ],
 )
 
diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
index 6a1937d88f..e8474f2312 100644
--- a/tensorflow/python/saved_model/load.py
+++ b/tensorflow/python/saved_model/load.py
@@ -36,6 +36,7 @@ from tensorflow.python.saved_model import revived_types
 from tensorflow.python.saved_model import saved_object_graph_pb2
 from tensorflow.python.saved_model import utils_impl as saved_model_utils
 from tensorflow.python.training.checkpointable import base
+from tensorflow.python.training.checkpointable import graph_view
 from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.training.checkpointable import util
 from tensorflow.python.util import compat
@@ -133,7 +134,7 @@ class _Loader(object):
     variables_path = saved_model_utils.get_variables_path(self._export_dir)
     # TODO(andresp): Clean use of private methods of CheckpointableSaver.
     # pylint: disable=protected-access
-    saver = util.CheckpointableSaver(self.get(0))
+    saver = util.CheckpointableSaver(graph_view.ObjectGraphView(self.get(0)))
     saver._file_prefix_placeholder = constant_op.constant(variables_path)
     load_status = saver.restore(variables_path)
     load_status.assert_existing_objects_matched()
@@ -147,8 +148,8 @@ class _Loader(object):
     # they get initialized properly when using common practices (e.g. the ones
     # used by ManagedSession) without further user action.
     for object_id, obj in dict(checkpoint.object_by_proto_id).items():
-      position = base._CheckpointPosition(checkpoint=checkpoint,
-                                          proto_id=object_id)
+      position = base.CheckpointPosition(checkpoint=checkpoint,
+                                         proto_id=object_id)
       restore_ops = position.restore_ops()
       if restore_ops:
         if resource_variable_ops.is_resource_variable(obj):
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index 7eeb14608f..1f16748909 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -47,8 +47,11 @@ from tensorflow.python.saved_model import signature_def_utils
 from tensorflow.python.saved_model import tag_constants
 from tensorflow.python.saved_model import utils_impl
 from tensorflow.python.training.checkpointable import base
+from tensorflow.python.training.checkpointable import graph_view
+from tensorflow.python.training.checkpointable import object_identity
 from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.training.checkpointable import util
+from tensorflow.python.training.saving import functional_saver
 from tensorflow.python.util import compat
 from tensorflow.python.util import nest
 from tensorflow.python.util.tf_export import tf_export
@@ -73,13 +76,15 @@ class _SaveableView(object):
   different objects if invoked more than once.
   """
 
-  def __init__(self, root):
-    checkpointable_objects, node_ids, slot_variables = util.find_objects(root)
+  def __init__(self, checkpoint_view):
+    self.checkpoint_view = checkpoint_view
+    checkpointable_objects, node_ids, slot_variables = (
+        self.checkpoint_view.objects_ids_and_slot_variables())
     self.nodes = checkpointable_objects
     self.node_ids = node_ids
-    self.captured_tensor_node_ids = util.ObjectIdentityDictionary()
+    self.captured_tensor_node_ids = object_identity.ObjectIdentityDictionary()
     self.slot_variables = slot_variables
-    self.functions = util.ObjectIdentityDictionary()
+    self.functions = object_identity.ObjectIdentityDictionary()
     self.concrete_functions = []
 
     # Also add `Function`s as nodes.
@@ -171,7 +176,7 @@ class _SaveableView(object):
     assert not context.executing_eagerly()
     # TODO(allenl): Handle MirroredVariables and other types of variables which
     # may need special casing.
-    object_map = util.ObjectIdentityDictionary()
+    object_map = object_identity.ObjectIdentityDictionary()
     resource_map = {}
     asset_info = _AssetInfo(
         asset_defs=[],
@@ -541,8 +546,7 @@ def _process_asset(trackable_asset, asset_info, resource_map):
   resource_map[original_variable.handle] = asset_variable.handle
 
 
-def _fill_meta_graph_def(meta_graph_def, saveable_view, signature_functions,
-                         object_saver):
+def _fill_meta_graph_def(meta_graph_def, saveable_view, signature_functions):
   """Generates a MetaGraph which calls `signature_functions`.
 
   Args:
@@ -550,7 +554,6 @@ def _fill_meta_graph_def(meta_graph_def, saveable_view, signature_functions,
     saveable_view: The _SaveableView being exported.
     signature_functions: A dictionary mapping signature keys to concrete
       functions containing signatures to add to the MetaGraph.
-    object_saver: A CheckpointableSaver to add to the MetaGraph.
 
   Returns:
     An _AssetInfo, which contains information to help creating the SavedModel.
@@ -590,7 +593,9 @@ def _fill_meta_graph_def(meta_graph_def, saveable_view, signature_functions,
   # gathering from the eager context so Optimizers save the right set of
   # variables, but want any operations associated with the save/restore to be in
   # the exported graph (thus the `to_graph` argument).
-  saver = object_saver.freeze(object_map=object_map, to_graph=exported_graph)
+  saver = functional_saver.Saver(
+      saveable_view.checkpoint_view.frozen_saveable_objects(
+          object_map=object_map, to_graph=exported_graph))
 
   with exported_graph.as_default():
     signatures = _generate_signatures(signature_functions, resource_map)
@@ -839,8 +844,9 @@ def save(obj, export_dir, signatures=None):
   # Use _SaveableView to provide a stable listing of properties and functions.
   # Note we run this twice since, while constructing the view the first time
   # there can be side effects of creating variables.
-  _ = _SaveableView(obj)
-  saveable_view = _SaveableView(obj)
+  checkpoint_graph_view = graph_view.ObjectGraphView(obj)
+  _ = _SaveableView(checkpoint_graph_view)
+  saveable_view = _SaveableView(checkpoint_graph_view)
 
   if signatures is None:
     signatures = _find_function_to_export(saveable_view)
@@ -852,10 +858,9 @@ def save(obj, export_dir, signatures=None):
   # making a SavedModel proto and writing it directly.
   saved_model = saved_model_pb2.SavedModel()
   meta_graph_def = saved_model.meta_graphs.add()
-  # TODO(andresp): Should this be using saveable_view?
-  object_saver = util.CheckpointableSaver(obj)
+  object_saver = util.CheckpointableSaver(checkpoint_graph_view)
   asset_info, exported_graph = _fill_meta_graph_def(
-      meta_graph_def, saveable_view, signatures, object_saver)
+      meta_graph_def, saveable_view, signatures)
   saved_model.saved_model_schema_version = (
       constants.SAVED_MODEL_SCHEMA_VERSION)
   # So far we've just been generating protocol buffers with no I/O. Now we write
diff --git a/tensorflow/python/training/checkpointable/BUILD b/tensorflow/python/training/checkpointable/BUILD
index a39462732f..e1f58a9e4b 100644
--- a/tensorflow/python/training/checkpointable/BUILD
+++ b/tensorflow/python/training/checkpointable/BUILD
@@ -29,6 +29,7 @@ py_library(
         "//tensorflow/python:util",
         "//tensorflow/python/eager:context",
         "//tensorflow/python/training/saving:saveable_object",
+        "@six_archive//:six",
     ],
 )
 
@@ -94,6 +95,29 @@ tf_py_test(
     ],
 )
 
+py_library(
+    name = "object_identity",
+    srcs = ["object_identity.py"],
+    srcs_version = "PY2AND3",
+)
+
+py_library(
+    name = "graph_view",
+    srcs = ["graph_view.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":base",
+        ":object_identity",
+        ":tracking",
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python/training/saving:saveable_object",
+        "//tensorflow/python/training/saving:saveable_object_util",
+    ],
+)
+
 py_library(
     name = "util",
     srcs = ["util.py"],
@@ -101,17 +125,19 @@ py_library(
     deps = [
         ":base",
         ":data_structures",
+        ":graph_view",
+        ":object_identity",
         ":tracking",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:checkpoint_management",
         "//tensorflow/python:constant_op",
-        "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:init_ops",
         "//tensorflow/python:io_ops_gen",
+        "//tensorflow/python:lib",
         "//tensorflow/python:pywrap_tensorflow",
         "//tensorflow/python:saver",
         "//tensorflow/python:session",
@@ -122,7 +148,6 @@ py_library(
         "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:def_function",
         "//tensorflow/python/training/saving:functional_saver",
-        "//tensorflow/python/training/saving:saveable_object",
         "//tensorflow/python/training/saving:saveable_object_util",
     ],
 )
@@ -132,10 +157,12 @@ tf_py_test(
     srcs = ["util_test.py"],
     additional_deps = [
         ":base",
+        ":graph_view",
         ":tracking",
         ":util",
         "@absl_py//absl/testing:parameterized",
         "@six_archive//:six",
+        "//tensorflow/python/keras/optimizer_v2",
         "//tensorflow/python:checkpoint_management",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:control_flow_ops",
@@ -149,8 +176,8 @@ tf_py_test(
         "//tensorflow/python:session",
         "//tensorflow/python:state_ops",
         "//tensorflow/python:template",
-        "//tensorflow/python:training",
         "//tensorflow/python:training_util",
+        "//tensorflow/python:training",
         "//tensorflow/python:variable_scope",
         "//tensorflow/python/eager:backprop",
         "//tensorflow/python/eager:context",
@@ -158,6 +185,7 @@ tf_py_test(
         "//tensorflow/python/eager:test",
         "//tensorflow/python/keras:engine",
         "//tensorflow/python/keras:layers",
+        "//tensorflow/python:variables",
     ],
     tags = ["notsan"],  # b/74395663
 )
@@ -190,6 +218,7 @@ tf_py_test(
     srcs = ["util_with_v1_optimizers_test.py"],
     additional_deps = [
         ":base",
+        ":graph_view",
         ":tracking",
         ":util",
         "@absl_py//absl/testing:parameterized",
diff --git a/tensorflow/python/training/checkpointable/base.py b/tensorflow/python/training/checkpointable/base.py
index 9fb251e455..a2c93cd110 100644
--- a/tensorflow/python/training/checkpointable/base.py
+++ b/tensorflow/python/training/checkpointable/base.py
@@ -187,14 +187,14 @@ class PythonStringStateSaveable(PythonStateSaveable):
     return control_flow_ops.no_op()
 
 
-class _CheckpointPosition(object):
-  """Indicates a position within a `_Checkpoint`."""
+class CheckpointPosition(object):
+  """Indicates a position within a `_CheckpointRestoreCoordinator`."""
 
   def __init__(self, checkpoint, proto_id):
     """Specify an object within a checkpoint.
 
     Args:
-      checkpoint: A _Checkpoint object.
+      checkpoint: A _CheckpointRestoreCoordinator object.
       proto_id: The index of this object in CheckpointableObjectGraph.nodes.
     """
     self._checkpoint = checkpoint
@@ -229,7 +229,7 @@ class _CheckpointPosition(object):
       for deferred_slot_restoration in (
           checkpoint.deferred_slot_restorations.pop(self._proto_id, ())):
         checkpointable._create_or_restore_slot_variable(  # pylint: disable=protected-access
-            slot_variable_position=_CheckpointPosition(
+            slot_variable_position=CheckpointPosition(
                 checkpoint=checkpoint,
                 proto_id=deferred_slot_restoration.slot_variable_id),
             variable=deferred_slot_restoration.original_variable,
@@ -249,7 +249,7 @@ class _CheckpointPosition(object):
                       slot_name=slot_restoration.slot_name))
         else:
           optimizer_object._create_or_restore_slot_variable(  # pylint: disable=protected-access
-              slot_variable_position=_CheckpointPosition(
+              slot_variable_position=CheckpointPosition(
                   checkpoint=checkpoint,
                   proto_id=slot_restoration.slot_variable_id),
               variable=checkpointable,
@@ -325,14 +325,15 @@ class _CheckpointPosition(object):
       # the SaveableObject itself has been cached. If not, we'll make it, and
       # either way we'll extract new ops from it (or if it has Python state to
       # restore, we'll run that).
-      if self._checkpoint.saveable_object_cache is None:
+      saveables_cache = self._checkpoint.graph_view.saveables_cache
+      if saveables_cache is None:
         # No SaveableObject caching when executing eagerly.
         saveable = None
       else:
         # If we've already created and cached a SaveableObject for this
         # attribute, we can re-use it to avoid re-creating some ops when graph
         # building.
-        saveable_list = self._checkpoint.saveable_object_cache.get(
+        saveable_list = saveables_cache.get(
             self.checkpointable, {}).get(serialized_tensor.name, (None,))
         if len(saveable_list) == 1:
           # Almost every attribute will have exactly one SaveableObject.
@@ -347,7 +348,7 @@ class _CheckpointPosition(object):
         # the SaveableObject.
         if serialized_tensor.checkpoint_key not in saveable.name:
           saveable = None
-          del self._checkpoint.saveable_object_cache[self.checkpointable]
+          del saveables_cache[self.checkpointable]
           break
       if saveable is None:
         # If there was no cached SaveableObject, we should check if the Python
@@ -366,8 +367,8 @@ class _CheckpointPosition(object):
           saveable = saveable_factory(name=serialized_tensor.checkpoint_key)
         else:
           saveable = saveable_factory
-        if self._checkpoint.saveable_object_cache is not None:
-          self._checkpoint.saveable_object_cache.setdefault(
+        if saveables_cache is not None:
+          saveables_cache.setdefault(
               self.checkpointable, {})[serialized_tensor.name] = [saveable]
       if isinstance(saveable, PythonStateSaveable):
         python_saveables.append(saveable)
@@ -491,7 +492,7 @@ class Checkpointable(object):
     # Maps names -> Checkpointable objects
     self._unconditional_dependency_names = {}
     # Restorations for other Checkpointable objects on which this object may
-    # eventually depend. Maps local name -> _CheckpointPosition list. Optimizers
+    # eventually depend. Maps local name -> CheckpointPosition list. Optimizers
     # tack on conditional dependencies, and so need separate management of
     # deferred dependencies too.
     self._unconditional_deferred_dependencies = {}
@@ -545,7 +546,7 @@ class Checkpointable(object):
     management of deferred dependencies too).
 
     Returns:
-      A dictionary mapping from local name to a list of _CheckpointPosition
+      A dictionary mapping from local name to a list of CheckpointPosition
       objects.
     """
     return self._unconditional_deferred_dependencies
@@ -791,7 +792,7 @@ class Checkpointable(object):
     else:
       restore_ops = ()
     for child in checkpoint_position.object_proto.children:
-      child_position = _CheckpointPosition(
+      child_position = CheckpointPosition(
           checkpoint=checkpoint,
           proto_id=child.node_id)
       local_object = self._lookup_dependency(child.local_name)
diff --git a/tensorflow/python/training/checkpointable/graph_view.py b/tensorflow/python/training/checkpointable/graph_view.py
new file mode 100644
index 0000000000..1f5f867fb2
--- /dev/null
+++ b/tensorflow/python/training/checkpointable/graph_view.py
@@ -0,0 +1,432 @@
+"""Manages a graph of Checkpointable objects."""
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import collections
+import weakref
+
+from tensorflow.core.protobuf import checkpointable_object_graph_pb2
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.training import optimizer as optimizer_v1
+from tensorflow.python.training.checkpointable import base
+from tensorflow.python.training.checkpointable import object_identity
+from tensorflow.python.training.checkpointable import tracking
+from tensorflow.python.training.saving import saveable_object as saveable_object_lib
+from tensorflow.python.training.saving import saveable_object_util
+
+
+_ESCAPE_CHAR = "."  # For avoiding conflicts with user-specified names.
+
+# Keyword for identifying that the next bit of a checkpoint variable name is a
+# slot name. Checkpoint names for slot variables look like:
+#
+#   <path to variable>/<_OPTIMIZER_SLOTS_NAME>/<path to optimizer>/<slot name>
+#
+# Where <path to variable> is a full path from the checkpoint root to the
+# variable being slotted for.
+_OPTIMIZER_SLOTS_NAME = _ESCAPE_CHAR + "OPTIMIZER_SLOT"
+# Keyword for separating the path to an object from the name of an
+# attribute in checkpoint names. Used like:
+#   <path to variable>/<_OBJECT_ATTRIBUTES_NAME>/<name of attribute>
+_OBJECT_ATTRIBUTES_NAME = _ESCAPE_CHAR + "ATTRIBUTES"
+
+
+def _escape_local_name(name):
+  # We need to support slashes in local names for compatibility, since this
+  # naming scheme is being patched in to things like Layer.add_variable where
+  # slashes were previously accepted. We also want to use slashes to indicate
+  # edges traversed to reach the variable, so we escape forward slashes in
+  # names.
+  return (name.replace(_ESCAPE_CHAR, _ESCAPE_CHAR + _ESCAPE_CHAR)
+          .replace(r"/", _ESCAPE_CHAR + "S"))
+
+
+def _object_prefix_from_path(path_to_root):
+  return "/".join(
+      (_escape_local_name(checkpointable.name)
+       for checkpointable in path_to_root))
+
+
+def _slot_variable_naming_for_optimizer(optimizer_path):
+  """Make a function for naming slot variables in an optimizer."""
+  # Name slot variables:
+  #
+  #   <variable name>/<_OPTIMIZER_SLOTS_NAME>/<optimizer path>/<slot name>
+  #
+  # where <variable name> is exactly the checkpoint name used for the original
+  # variable, including the path from the checkpoint root and the local name in
+  # the object which owns it. Note that we only save slot variables if the
+  # variable it's slotting for is also being saved.
+
+  optimizer_identifier = "/%s/%s/" % (_OPTIMIZER_SLOTS_NAME, optimizer_path)
+
+  def _name_slot_variable(variable_path, slot_name):
+    """With an optimizer specified, name a slot variable."""
+    return (variable_path
+            + optimizer_identifier
+            + _escape_local_name(slot_name))
+
+  return _name_slot_variable
+
+
+def _serialize_slot_variables(checkpointable_objects, node_ids, object_names):
+  """Gather and name slot variables."""
+  non_slot_objects = list(checkpointable_objects)
+  slot_variables = object_identity.ObjectIdentityDictionary()
+  for checkpointable in non_slot_objects:
+    if (isinstance(checkpointable, optimizer_v1.Optimizer)
+        # TODO(b/110718070): Fix Keras imports.
+        or hasattr(checkpointable, "_create_or_restore_slot_variable")):
+      naming_scheme = _slot_variable_naming_for_optimizer(
+          optimizer_path=object_names[checkpointable])
+      slot_names = checkpointable.get_slot_names()
+      for slot_name in slot_names:
+        for original_variable_node_id, original_variable in enumerate(
+            non_slot_objects):
+          try:
+            slot_variable = checkpointable.get_slot(
+                original_variable, slot_name)
+          except (AttributeError, KeyError):
+            slot_variable = None
+          if slot_variable is None:
+            continue
+          slot_variable._maybe_initialize_checkpointable()  # pylint: disable=protected-access
+          if slot_variable._checkpoint_dependencies:  # pylint: disable=protected-access
+            # TODO(allenl): Gather dependencies of slot variables.
+            raise NotImplementedError(
+                "Currently only variables with no dependencies can be saved as "
+                "slot variables. File a feature request if this limitation "
+                "bothers you.")
+          if slot_variable in node_ids:
+            raise NotImplementedError(
+                "A slot variable was re-used as a dependency of a "
+                "Checkpointable object. This is not currently allowed. File a "
+                "feature request if this limitation bothers you.")
+          checkpoint_name = naming_scheme(
+              variable_path=object_names[original_variable],
+              slot_name=slot_name)
+          object_names[slot_variable] = checkpoint_name
+          slot_variable_node_id = len(checkpointable_objects)
+          node_ids[slot_variable] = slot_variable_node_id
+          checkpointable_objects.append(slot_variable)
+          slot_variable_proto = (
+              checkpointable_object_graph_pb2.CheckpointableObjectGraph
+              .CheckpointableObject.SlotVariableReference(
+                  slot_name=slot_name,
+                  original_variable_node_id=original_variable_node_id,
+                  slot_variable_node_id=slot_variable_node_id))
+          slot_variables.setdefault(checkpointable, []).append(
+              slot_variable_proto)
+  return slot_variables
+
+
+def _fill_object_graph_proto(checkpointable_objects,
+                             node_ids,
+                             slot_variables,
+                             object_graph_proto=None):
+  """Name non-slot `Checkpointable`s and add them to `object_graph_proto`."""
+  if object_graph_proto is None:
+    object_graph_proto = (
+        checkpointable_object_graph_pb2.CheckpointableObjectGraph())
+  for checkpoint_id, checkpointable in enumerate(checkpointable_objects):
+    assert node_ids[checkpointable] == checkpoint_id
+    object_proto = object_graph_proto.nodes.add()
+    object_proto.slot_variables.extend(slot_variables.get(checkpointable, ()))
+    for child in checkpointable._checkpoint_dependencies:  # pylint: disable=protected-access
+      child_proto = object_proto.children.add()
+      child_proto.node_id = node_ids[child.ref]
+      child_proto.local_name = child.name
+  return object_graph_proto
+
+
+class ObjectGraphView(object):
+  """Gathers and serializes an object graph."""
+
+  def __init__(self, root, saveables_cache=None):
+    """Configure the graph view.
+
+    Args:
+      root: A `Checkpointable` object whose variables (including the variables
+        of dependencies, recursively) should be saved. May be a weak reference.
+      saveables_cache: A dictionary mapping `Checkpointable` objects ->
+        attribute names -> SaveableObjects, used to avoid re-creating
+        SaveableObjects when graph building.
+    """
+    self._root_ref = root
+    self._saveables_cache = saveables_cache
+
+  def _list_dependencies(self, obj):
+    # pylint: disable=protected-access
+    obj._maybe_initialize_checkpointable()
+    return obj._checkpoint_dependencies
+    # pylint: enable=protected-access
+
+  @property
+  def saveables_cache(self):
+    """Maps Checkpointable objects -> attribute names -> list(SaveableObjects).
+
+    Used to avoid re-creating SaveableObjects when graph building. None when
+    executing eagerly.
+
+    Returns:
+      The cache (an object-identity dictionary), or None if caching is disabled.
+    """
+    return self._saveables_cache
+
+  @property
+  def root(self):
+    if isinstance(self._root_ref, weakref.ref):
+      derefed = self._root_ref()
+      assert derefed is not None
+      return derefed
+    else:
+      return self._root_ref
+
+  def _breadth_first_traversal(self):
+    """Find shortest paths to all dependencies of self.root."""
+    bfs_sorted = []
+    to_visit = collections.deque([self.root])
+    path_to_root = object_identity.ObjectIdentityDictionary()
+    path_to_root[self.root] = ()
+    while to_visit:
+      current_checkpointable = to_visit.popleft()
+      if isinstance(current_checkpointable, tracking.NotCheckpointable):
+        raise NotImplementedError(
+            ("The object %s does not support object-based saving. File a "
+             "feature request if this limitation bothers you. In the meantime, "
+             "you can remove the dependency on this object and save everything "
+             "else.")
+            % (current_checkpointable,))
+      bfs_sorted.append(current_checkpointable)
+      for name, dependency in self._list_dependencies(current_checkpointable):
+        if dependency not in path_to_root:
+          path_to_root[dependency] = (
+              path_to_root[current_checkpointable] + (
+                  base.CheckpointableReference(name, dependency),))
+          to_visit.append(dependency)
+    return bfs_sorted, path_to_root
+
+  def _add_attributes_to_object_graph(
+      self, checkpointable_objects, object_graph_proto, node_ids, object_names,
+      object_map):
+    """Create SaveableObjects and corresponding SerializedTensor protos."""
+    named_saveable_objects = []
+    if self._saveables_cache is None:
+      # No SaveableObject caching. Either we're executing eagerly, or building a
+      # static save which is specialized to the current Python state.
+      feed_additions = None
+    else:
+      # If we are caching SaveableObjects, we need to build up a feed_dict with
+      # functions computing volatile Python state to be saved with the
+      # checkpoint.
+      feed_additions = {}
+    for checkpoint_id, (checkpointable, object_proto) in enumerate(
+        zip(checkpointable_objects, object_graph_proto.nodes)):
+      assert node_ids[checkpointable] == checkpoint_id
+      object_name = object_names[checkpointable]
+      if object_map is None:
+        object_to_save = checkpointable
+      else:
+        object_to_save = object_map.get(checkpointable, checkpointable)
+      if self._saveables_cache is not None:
+        cached_attributes = self._saveables_cache.setdefault(object_to_save, {})
+      else:
+        cached_attributes = None
+
+      for name, saveable_factory in (
+          object_to_save._gather_saveables_for_checkpoint().items()):  # pylint: disable=protected-access
+        attribute = object_proto.attributes.add()
+        attribute.name = name
+        attribute.checkpoint_key = "%s/%s/%s" % (
+            object_name, _OBJECT_ATTRIBUTES_NAME, _escape_local_name(name))
+        if cached_attributes is None:
+          saveables = None
+        else:
+          saveables = cached_attributes.get(name, None)
+          if saveables is not None:
+            for saveable in saveables:
+              if attribute.checkpoint_key not in saveable.name:
+                # The checkpoint key for this SaveableObject is different. We
+                # need to re-create it.
+                saveables = None
+                del cached_attributes[name]
+                break
+        if saveables is None:
+          if callable(saveable_factory):
+            maybe_saveable = saveable_factory(name=attribute.checkpoint_key)
+          else:
+            maybe_saveable = saveable_factory
+          if isinstance(maybe_saveable, saveable_object_lib.SaveableObject):
+            saveables = (maybe_saveable,)
+          else:
+            # Figure out the name-based Saver's name for this variable. If it's
+            # already a SaveableObject we'd just get the checkpoint key back, so
+            # we leave full_name blank.
+            saver_dict = saveable_object_util.op_list_to_dict(
+                [maybe_saveable], convert_variable_to_tensor=False)
+            full_name, = saver_dict.keys()
+            saveables = tuple(saveable_object_util.saveable_objects_for_op(
+                op=maybe_saveable, name=attribute.checkpoint_key))
+            for saveable in saveables:
+              saveable.full_name = full_name
+          for saveable in saveables:
+            if attribute.checkpoint_key not in saveable.name:
+              raise AssertionError(
+                  ("The object %s produced a SaveableObject with name '%s' for "
+                   "attribute '%s'. Expected a name containing '%s'.")
+                  % (checkpointable, name, saveable.name,
+                     attribute.checkpoint_key))
+          if cached_attributes is not None:
+            cached_attributes[name] = saveables
+
+        optional_restore = None
+        for saveable in saveables:
+          if optional_restore is None:
+            optional_restore = saveable.optional_restore
+          else:
+            optional_restore = optional_restore and saveable.optional_restore
+
+          if hasattr(saveable, "full_name"):
+            attribute.full_name = saveable.full_name
+          if isinstance(saveable, base.PythonStateSaveable):
+            if feed_additions is None:
+              assert self._saveables_cache is None
+              # If we're not caching saveables, then we're either executing
+              # eagerly or building a static save/restore (e.g. for a
+              # SavedModel). In either case, we should embed the current Python
+              # state in the graph rather than relying on a feed dict.
+              saveable = saveable.freeze()
+            else:
+              saveable_feed_dict = saveable.feed_dict_additions()
+              for new_feed_key in saveable_feed_dict.keys():
+                if new_feed_key in feed_additions:
+                  raise AssertionError(
+                      ("The object %s tried to feed a value for the Tensor %s "
+                       "when saving, but another object is already feeding a "
+                       "value.")
+                      % (checkpointable, new_feed_key))
+              feed_additions.update(saveable_feed_dict)
+          named_saveable_objects.append(saveable)
+        if optional_restore is None:
+          optional_restore = False
+        attribute.optional_restore = optional_restore
+
+    return named_saveable_objects, feed_additions
+
+  def _serialize_gathered_objects(self, checkpointable_objects, path_to_root,
+                                  object_map=None):
+    """Create SaveableObjects and protos for gathered objects."""
+    object_names = object_identity.ObjectIdentityDictionary()
+    for obj, path in path_to_root.items():
+      object_names[obj] = _object_prefix_from_path(path)
+    node_ids = object_identity.ObjectIdentityDictionary()
+    for node_id, node in enumerate(checkpointable_objects):
+      node_ids[node] = node_id
+    slot_variables = _serialize_slot_variables(
+        checkpointable_objects=checkpointable_objects,
+        node_ids=node_ids,
+        object_names=object_names)
+    object_graph_proto = _fill_object_graph_proto(
+        checkpointable_objects=checkpointable_objects,
+        node_ids=node_ids,
+        slot_variables=slot_variables)
+    named_saveable_objects, feed_additions = (
+        self._add_attributes_to_object_graph(
+            checkpointable_objects=checkpointable_objects,
+            object_graph_proto=object_graph_proto,
+            node_ids=node_ids,
+            object_names=object_names,
+            object_map=object_map))
+    return named_saveable_objects, object_graph_proto, feed_additions
+
+  def serialize_object_graph(self):
+    """Determine checkpoint keys for variables and build a serialized graph.
+
+    Non-slot variables are keyed based on a shortest path from the root saveable
+    to the object which owns the variable (i.e. the one which called
+    `Checkpointable._add_variable` to create it).
+
+    Slot variables are keyed based on a shortest path to the variable being
+    slotted for, a shortest path to their optimizer, and the slot name.
+
+    Returns:
+      A tuple of (named_variables, object_graph_proto, feed_additions):
+        named_variables: A dictionary mapping names to variable objects.
+        object_graph_proto: A CheckpointableObjectGraph protocol buffer
+          containing the serialized object graph and variable references.
+        feed_additions: A dictionary mapping from Tensors to values which should
+          be fed when saving.
+
+    Raises:
+      ValueError: If there are invalid characters in an optimizer's slot names.
+    """
+    checkpointable_objects, path_to_root = self._breadth_first_traversal()
+    return self._serialize_gathered_objects(
+        checkpointable_objects, path_to_root)
+
+  def frozen_saveable_objects(self, object_map=None, to_graph=None):
+    """Creates SaveableObjects with the current object graph frozen."""
+    checkpointable_objects, path_to_root = self._breadth_first_traversal()
+    if to_graph:
+      target_context = to_graph.as_default
+    else:
+      target_context = ops.NullContextmanager
+    with target_context():
+      named_saveable_objects, graph_proto, _ = self._serialize_gathered_objects(
+          checkpointable_objects,
+          path_to_root,
+          object_map)
+      with ops.device("/cpu:0"):
+        object_graph_tensor = constant_op.constant(
+            graph_proto.SerializeToString(), dtype=dtypes.string)
+      named_saveable_objects.append(
+          base.NoRestoreSaveable(
+              tensor=object_graph_tensor,
+              name=base.OBJECT_GRAPH_PROTO_KEY))
+    return named_saveable_objects
+
+  def objects_ids_and_slot_variables(self):
+    """Traverse the object graph and list all accessible objects.
+
+    Looks for `Checkpointable` objects which are dependencies of
+    `root_checkpointable`. Includes slot variables only if the variable they are
+    slotting for and the optimizer are dependencies of `root_checkpointable`
+    (i.e. if they would be saved with a checkpoint).
+
+    Returns:
+      A tuple of (checkpointable objects, object -> node id, slot variables)
+    """
+    checkpointable_objects, path_to_root = self._breadth_first_traversal()
+    object_names = object_identity.ObjectIdentityDictionary()
+    for obj, path in path_to_root.items():
+      object_names[obj] = _object_prefix_from_path(path)
+    node_ids = object_identity.ObjectIdentityDictionary()
+    for node_id, node in enumerate(checkpointable_objects):
+      node_ids[node] = node_id
+    slot_variables = _serialize_slot_variables(
+        checkpointable_objects=checkpointable_objects,
+        node_ids=node_ids,
+        object_names=object_names)
+    return checkpointable_objects, node_ids, slot_variables
+
+  def list_objects(self):
+    """Traverse the object graph and list all accessible objects."""
+    checkpointable_objects, _, _ = self.objects_ids_and_slot_variables()
+    return checkpointable_objects
diff --git a/tensorflow/python/training/checkpointable/object_identity.py b/tensorflow/python/training/checkpointable/object_identity.py
new file mode 100644
index 0000000000..2d3056b0a8
--- /dev/null
+++ b/tensorflow/python/training/checkpointable/object_identity.py
@@ -0,0 +1,156 @@
+"""Utilities for collecting objects based on "is" comparison."""
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import collections
+import weakref
+
+
+class _ObjectIdentityWrapper(object):
+  """Wraps an object, mapping __eq__ on wrapper to "is" on wrapped.
+
+  Since __eq__ is based on object identity, it's safe to also define __hash__
+  based on object ids. This lets us add unhashable types like checkpointable
+  _ListWrapper objects to object-identity collections.
+  """
+
+  def __init__(self, wrapped):
+    self._wrapped = wrapped
+
+  @property
+  def unwrapped(self):
+    return self._wrapped
+
+  def __eq__(self, other):
+    if isinstance(other, _ObjectIdentityWrapper):
+      return self._wrapped is other._wrapped  # pylint: disable=protected-access
+    return self._wrapped is other
+
+  def __hash__(self):
+    # Wrapper id() is also fine for weakrefs. In fact, we rely on
+    # id(weakref.ref(a)) == id(weakref.ref(a)) and weakref.ref(a) is
+    # weakref.ref(a) in _WeakObjectIdentityWrapper.
+    return id(self._wrapped)
+
+
+class _WeakObjectIdentityWrapper(_ObjectIdentityWrapper):
+
+  def __init__(self, wrapped):
+    super(_WeakObjectIdentityWrapper, self).__init__(weakref.ref(wrapped))
+
+  @property
+  def unwrapped(self):
+    return self._wrapped()
+
+
+class ObjectIdentityDictionary(collections.MutableMapping):
+  """A mutable mapping data structure which compares using "is".
+
+  This is necessary because we have checkpointable objects (_ListWrapper) which
+  have behavior identical to built-in Python lists (including being unhashable
+  and comparing based on the equality of their contents by default).
+  """
+
+  def __init__(self):
+    self._storage = {}
+
+  def _wrap_key(self, key):
+    return _ObjectIdentityWrapper(key)
+
+  def __getitem__(self, key):
+    return self._storage[self._wrap_key(key)]
+
+  def __setitem__(self, key, value):
+    self._storage[self._wrap_key(key)] = value
+
+  def __delitem__(self, key):
+    del self._storage[self._wrap_key(key)]
+
+  def __len__(self):
+    return len(self._storage)
+
+  def __iter__(self):
+    for key in self._storage:
+      yield key.unwrapped
+
+
+class ObjectIdentityWeakKeyDictionary(ObjectIdentityDictionary):
+  """Like weakref.WeakKeyDictionary, but compares objects with "is"."""
+
+  def _wrap_key(self, key):
+    return _WeakObjectIdentityWrapper(key)
+
+  def __len__(self):
+    # Iterate, discarding old weak refs
+    return len(list(self._storage))
+
+  def __iter__(self):
+    keys = self._storage.keys()
+    for key in keys:
+      unwrapped = key.unwrapped
+      if unwrapped is None:
+        del self[key]
+      else:
+        yield unwrapped
+
+
+class ObjectIdentitySet(collections.MutableSet):
+  """Like the built-in set, but compares objects with "is"."""
+
+  def __init__(self, *args):
+    self._storage = set([self._wrap_key(obj) for obj in list(*args)])
+
+  def _wrap_key(self, key):
+    return _ObjectIdentityWrapper(key)
+
+  def __contains__(self, key):
+    return self._wrap_key(key) in self._storage
+
+  def discard(self, key):
+    self._storage.discard(self._wrap_key(key))
+
+  def add(self, key):
+    self._storage.add(self._wrap_key(key))
+
+  def __len__(self):
+    return len(self._storage)
+
+  def __iter__(self):
+    keys = list(self._storage)
+    for key in keys:
+      yield key.unwrapped
+
+
+class ObjectIdentityWeakSet(ObjectIdentitySet):
+  """Like weakref.WeakSet, but compares objects with "is"."""
+
+  def _wrap_key(self, key):
+    return _WeakObjectIdentityWrapper(key)
+
+  def __len__(self):
+    # Iterate, discarding old weak refs
+    return len([_ for _ in self])
+
+  def __iter__(self):
+    keys = list(self._storage)
+    for key in keys:
+      unwrapped = key.unwrapped
+      if unwrapped is None:
+        self.discard(key)
+      else:
+        yield unwrapped
diff --git a/tensorflow/python/training/checkpointable/util.py b/tensorflow/python/training/checkpointable/util.py
index 129ad55f96..1ba9720fb5 100644
--- a/tensorflow/python/training/checkpointable/util.py
+++ b/tensorflow/python/training/checkpointable/util.py
@@ -18,7 +18,6 @@ from __future__ import division
 from __future__ import print_function
 
 import abc
-import collections
 import os
 import weakref
 
@@ -39,13 +38,13 @@ from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.training import checkpoint_management
-from tensorflow.python.training import optimizer as optimizer_v1
 from tensorflow.python.training import saver as v1_saver_lib
 from tensorflow.python.training.checkpointable import base
 from tensorflow.python.training.checkpointable import data_structures
+from tensorflow.python.training.checkpointable import graph_view as graph_view_lib
+from tensorflow.python.training.checkpointable import object_identity
 from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.training.saving import functional_saver
-from tensorflow.python.training.saving import saveable_object as saveable_object_lib
 from tensorflow.python.training.saving import saveable_object_util
 from tensorflow.python.util import compat
 from tensorflow.python.util import deprecation
@@ -53,27 +52,11 @@ from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
 
-_ESCAPE_CHAR = "."  # For avoiding conflicts with user-specified names.
-
-# Keyword for identifying that the next bit of a checkpoint variable name is a
-# slot name. Checkpoint names for slot variables look like:
-#
-#   <path to variable>/<_OPTIMIZER_SLOTS_NAME>/<path to optimizer>/<slot name>
-#
-# Where <path to variable> is a full path from the checkpoint root to the
-# variable being slotted for.
-_OPTIMIZER_SLOTS_NAME = _ESCAPE_CHAR + "OPTIMIZER_SLOT"
-# Keyword for separating the path to an object from the name of an
-# attribute in checkpoint names. Used like:
-#   <path to variable>/<_OBJECT_ATTRIBUTES_NAME>/<name of attribute>
-_OBJECT_ATTRIBUTES_NAME = _ESCAPE_CHAR + "ATTRIBUTES"
-
-
 class _CheckpointRestoreCoordinator(object):
   """Holds the status of an object-based checkpoint load."""
 
   def __init__(self, object_graph_proto, save_path, save_path_tensor,
-               restore_op_cache, saveable_object_cache):
+               restore_op_cache, graph_view):
     """Specify the checkpoint being loaded.
 
     Args:
@@ -87,10 +70,8 @@ class _CheckpointRestoreCoordinator(object):
         `_CheckpointRestoreCoordinator`s for the same Python objects, used to
         look up restore ops by name to avoid re-creating them across multiple
         `restore()` calls.
-      saveable_object_cache: A mapping of checkpointable objects -> attribute
-        names -> list(`SaveableObject`s), used when `SaveableObjects` must be
-        referenced every restore (e.g. for Python state); otherwise they would
-        create their own ops every restore.
+      graph_view: A graph_view_lib.ObjectGraphView object for the restored
+        objects.
     """
     self.object_graph_proto = object_graph_proto
     self.restore_uid = ops.uid()
@@ -108,7 +89,7 @@ class _CheckpointRestoreCoordinator(object):
     # use them (for example because of inconsistent references when
     # loading). Used to make status assertions fail when loading checkpoints
     # that don't quite match.
-    self.all_python_objects = _ObjectIdentityWeakSet()
+    self.all_python_objects = object_identity.ObjectIdentityWeakSet()
     self.save_path_tensor = save_path_tensor
     self.save_path_string = save_path
     self.dtype_map = pywrap_tensorflow.NewCheckpointReader(
@@ -119,7 +100,7 @@ class _CheckpointRestoreCoordinator(object):
     # this checkpoint.
     self.restore_ops = []
     self.restore_ops_by_name = restore_op_cache
-    self.saveable_object_cache = saveable_object_cache
+    self.graph_view = graph_view
     self.new_restore_ops_callback = None
     # A mapping from optimizer proto ids to lists of slot variables to be
     # restored when the optimizer is tracked. Only includes slot variables whose
@@ -345,464 +326,6 @@ def object_metadata(save_path):
   return object_graph_proto
 
 
-class _ObjectIdentityWrapper(object):
-  """Wraps an object, mapping __eq__ on wrapper to "is" on wrapped.
-
-  Since __eq__ is based on object identity, it's safe to also define __hash__
-  based on object ids. This lets us add unhashable types like checkpointable
-  _ListWrapper objects to object-identity collections.
-  """
-
-  def __init__(self, wrapped):
-    self._wrapped = wrapped
-
-  @property
-  def unwrapped(self):
-    return self._wrapped
-
-  def __eq__(self, other):
-    if isinstance(other, _ObjectIdentityWrapper):
-      return self._wrapped is other._wrapped  # pylint: disable=protected-access
-    return self._wrapped is other
-
-  def __hash__(self):
-    # Wrapper id() is also fine for weakrefs. In fact, we rely on
-    # id(weakref.ref(a)) == id(weakref.ref(a)) and weakref.ref(a) is
-    # weakref.ref(a) in _WeakObjectIdentityWrapper.
-    return id(self._wrapped)
-
-
-class _WeakObjectIdentityWrapper(_ObjectIdentityWrapper):
-
-  def __init__(self, wrapped):
-    super(_WeakObjectIdentityWrapper, self).__init__(weakref.ref(wrapped))
-
-  @property
-  def unwrapped(self):
-    return self._wrapped()
-
-
-class ObjectIdentityDictionary(collections.MutableMapping):
-  """A mutable mapping data structure which compares using "is".
-
-  This is necessary because we have checkpointable objects (_ListWrapper) which
-  have behavior identical to built-in Python lists (including being unhashable
-  and comparing based on the equality of their contents by default).
-  """
-
-  def __init__(self):
-    self._storage = {}
-
-  def _wrap_key(self, key):
-    return _ObjectIdentityWrapper(key)
-
-  def __getitem__(self, key):
-    return self._storage[self._wrap_key(key)]
-
-  def __setitem__(self, key, value):
-    self._storage[self._wrap_key(key)] = value
-
-  def __delitem__(self, key):
-    del self._storage[self._wrap_key(key)]
-
-  def __len__(self):
-    return len(self._storage)
-
-  def __iter__(self):
-    for key in self._storage:
-      yield key.unwrapped
-
-
-class _ObjectIdentityWeakKeyDictionary(ObjectIdentityDictionary):
-  """Like weakref.WeakKeyDictionary, but compares objects with "is"."""
-
-  def _wrap_key(self, key):
-    return _WeakObjectIdentityWrapper(key)
-
-  def __len__(self):
-    # Iterate, discarding old weak refs
-    return len(list(self._storage))
-
-  def __iter__(self):
-    keys = self._storage.keys()
-    for key in keys:
-      unwrapped = key.unwrapped
-      if unwrapped is None:
-        del self[key]
-      else:
-        yield unwrapped
-
-
-class _ObjectIdentitySet(collections.MutableSet):
-  """Like the built-in set, but compares objects with "is"."""
-
-  def __init__(self, *args):
-    self._storage = set([self._wrap_key(obj) for obj in list(*args)])
-
-  def _wrap_key(self, key):
-    return _ObjectIdentityWrapper(key)
-
-  def __contains__(self, key):
-    return self._wrap_key(key) in self._storage
-
-  def discard(self, key):
-    self._storage.discard(self._wrap_key(key))
-
-  def add(self, key):
-    self._storage.add(self._wrap_key(key))
-
-  def __len__(self):
-    return len(self._storage)
-
-  def __iter__(self):
-    keys = list(self._storage)
-    for key in keys:
-      yield key.unwrapped
-
-
-class _ObjectIdentityWeakSet(_ObjectIdentitySet):
-  """Like weakref.WeakSet, but compares objects with "is"."""
-
-  def _wrap_key(self, key):
-    return _WeakObjectIdentityWrapper(key)
-
-  def __len__(self):
-    # Iterate, discarding old weak refs
-    return len([_ for _ in self])
-
-  def __iter__(self):
-    keys = list(self._storage)
-    for key in keys:
-      unwrapped = key.unwrapped
-      if unwrapped is None:
-        self.discard(key)
-      else:
-        yield unwrapped
-
-
-def _breadth_first_checkpointable_traversal(root_checkpointable):
-  """Find shortest paths to all variables owned by dependencies of root."""
-  bfs_sorted = []
-  to_visit = collections.deque([root_checkpointable])
-  path_to_root = ObjectIdentityDictionary()
-  path_to_root[root_checkpointable] = ()
-  while to_visit:
-    current_checkpointable = to_visit.popleft()
-    if isinstance(current_checkpointable, tracking.NotCheckpointable):
-      raise NotImplementedError(
-          ("The object %s does not support object-based saving. File a feature "
-           "request if this limitation bothers you. In the meantime, you can "
-           "remove the dependency on this object and save everything else.")
-          % (current_checkpointable,))
-    current_checkpointable._maybe_initialize_checkpointable()  # pylint: disable=protected-access
-    bfs_sorted.append(current_checkpointable)
-    for child_checkpointable in (
-        current_checkpointable._checkpoint_dependencies):  # pylint: disable=protected-access
-      if child_checkpointable.ref not in path_to_root:
-        path_to_root[child_checkpointable.ref] = (
-            path_to_root[current_checkpointable] + (child_checkpointable,))
-        to_visit.append(child_checkpointable.ref)
-  return bfs_sorted, path_to_root
-
-
-def _escape_local_name(name):
-  # We need to support slashes in local names for compatibility, since this
-  # naming scheme is being patched in to things like Layer.add_variable where
-  # slashes were previously accepted. We also want to use slashes to indicate
-  # edges traversed to reach the variable, so we escape forward slashes in
-  # names.
-  return (name.replace(_ESCAPE_CHAR, _ESCAPE_CHAR + _ESCAPE_CHAR)
-          .replace(r"/", _ESCAPE_CHAR + "S"))
-
-
-def _object_prefix_from_path(path_to_root):
-  return "/".join(
-      (_escape_local_name(checkpointable.name)
-       for checkpointable in path_to_root))
-
-
-def _slot_variable_naming_for_optimizer(optimizer_path):
-  """Make a function for naming slot variables in an optimizer."""
-  # Name slot variables:
-  #
-  #   <variable name>/<_OPTIMIZER_SLOTS_NAME>/<optimizer path>/<slot name>
-  #
-  # where <variable name> is exactly the checkpoint name used for the original
-  # variable, including the path from the checkpoint root and the local name in
-  # the object which owns it. Note that we only save slot variables if the
-  # variable it's slotting for is also being saved.
-
-  optimizer_identifier = "/%s/%s/" % (_OPTIMIZER_SLOTS_NAME, optimizer_path)
-
-  def _name_slot_variable(variable_path, slot_name):
-    """With an optimizer specified, name a slot variable."""
-    return (variable_path
-            + optimizer_identifier
-            + _escape_local_name(slot_name))
-
-  return _name_slot_variable
-
-
-def _serialize_slot_variables(checkpointable_objects, node_ids, object_names):
-  """Gather and name slot variables."""
-  non_slot_objects = list(checkpointable_objects)
-  slot_variables = ObjectIdentityDictionary()
-  for checkpointable in non_slot_objects:
-    if (isinstance(checkpointable, optimizer_v1.Optimizer)
-        # TODO(b/110718070): Fix Keras imports.
-        or hasattr(checkpointable, "_create_or_restore_slot_variable")):
-      naming_scheme = _slot_variable_naming_for_optimizer(
-          optimizer_path=object_names[checkpointable])
-      slot_names = checkpointable.get_slot_names()
-      for slot_name in slot_names:
-        for original_variable_node_id, original_variable in enumerate(
-            non_slot_objects):
-          try:
-            slot_variable = checkpointable.get_slot(
-                original_variable, slot_name)
-          except (AttributeError, KeyError):
-            slot_variable = None
-          if slot_variable is None:
-            continue
-          slot_variable._maybe_initialize_checkpointable()  # pylint: disable=protected-access
-          if slot_variable._checkpoint_dependencies:  # pylint: disable=protected-access
-            # TODO(allenl): Gather dependencies of slot variables.
-            raise NotImplementedError(
-                "Currently only variables with no dependencies can be saved as "
-                "slot variables. File a feature request if this limitation "
-                "bothers you.")
-          if slot_variable in node_ids:
-            raise NotImplementedError(
-                "A slot variable was re-used as a dependency of a "
-                "Checkpointable object. This is not currently allowed. File a "
-                "feature request if this limitation bothers you.")
-          checkpoint_name = naming_scheme(
-              variable_path=object_names[original_variable],
-              slot_name=slot_name)
-          object_names[slot_variable] = checkpoint_name
-          slot_variable_node_id = len(checkpointable_objects)
-          node_ids[slot_variable] = slot_variable_node_id
-          checkpointable_objects.append(slot_variable)
-          slot_variable_proto = (
-              checkpointable_object_graph_pb2.CheckpointableObjectGraph
-              .CheckpointableObject.SlotVariableReference(
-                  slot_name=slot_name,
-                  original_variable_node_id=original_variable_node_id,
-                  slot_variable_node_id=slot_variable_node_id))
-          slot_variables.setdefault(checkpointable, []).append(
-              slot_variable_proto)
-  return slot_variables
-
-
-def _add_attributes_to_object_graph(
-    checkpointable_objects, object_graph_proto, node_ids, object_names,
-    saveables_cache, object_map):
-  """Create SaveableObjects and corresponding SerializedTensor protos."""
-  named_saveable_objects = []
-  if saveables_cache is None:
-    # No SaveableObject caching. Either we're executing eagerly, or building a
-    # static save which is specialized to the current Python state.
-    feed_additions = None
-  else:
-    # If we are caching SaveableObjects, we need to build up a feed_dict with
-    # functions computing volatile Python state to be saved with the checkpoint.
-    feed_additions = {}
-  for checkpoint_id, (checkpointable, object_proto) in enumerate(
-      zip(checkpointable_objects, object_graph_proto.nodes)):
-    assert node_ids[checkpointable] == checkpoint_id
-    object_name = object_names[checkpointable]
-    if object_map:
-      object_to_save = object_map.get(checkpointable, checkpointable)
-    else:
-      object_to_save = checkpointable
-    if saveables_cache is not None:
-      cached_attributes = saveables_cache.setdefault(object_to_save, {})
-    else:
-      cached_attributes = None
-
-    for name, saveable_factory in (
-        object_to_save._gather_saveables_for_checkpoint().items()):  # pylint: disable=protected-access
-      attribute = object_proto.attributes.add()
-      attribute.name = name
-      attribute.checkpoint_key = "%s/%s/%s" % (
-          object_name, _OBJECT_ATTRIBUTES_NAME, _escape_local_name(name))
-      if cached_attributes is None:
-        saveables = None
-      else:
-        saveables = cached_attributes.get(name, None)
-        if saveables is not None:
-          for saveable in saveables:
-            if attribute.checkpoint_key not in saveable.name:
-              # The checkpoint key for this SaveableObject is different. We need
-              # to re-create it.
-              saveables = None
-              del cached_attributes[name]
-              break
-      if saveables is None:
-        if callable(saveable_factory):
-          maybe_saveable = saveable_factory(name=attribute.checkpoint_key)
-        else:
-          maybe_saveable = saveable_factory
-        if isinstance(maybe_saveable, saveable_object_lib.SaveableObject):
-          saveables = (maybe_saveable,)
-        else:
-          # Figure out the name-based Saver's name for this variable. If it's
-          # already a SaveableObject we'd just get the checkpoint key back, so
-          # we leave full_name blank.
-          saver_dict = saveable_object_util.op_list_to_dict(
-              [maybe_saveable], convert_variable_to_tensor=False)
-          full_name, = saver_dict.keys()
-          saveables = tuple(saveable_object_util.saveable_objects_for_op(
-              op=maybe_saveable, name=attribute.checkpoint_key))
-          for saveable in saveables:
-            saveable.full_name = full_name
-        for saveable in saveables:
-          if attribute.checkpoint_key not in saveable.name:
-            raise AssertionError(
-                ("The object %s produced a SaveableObject with name '%s' for "
-                 "attribute '%s'. Expected a name containing '%s'.")
-                % (checkpointable, name, saveable.name,
-                   attribute.checkpoint_key))
-        if cached_attributes is not None:
-          cached_attributes[name] = saveables
-
-      optional_restore = None
-      for saveable in saveables:
-        if optional_restore is None:
-          optional_restore = saveable.optional_restore
-        else:
-          optional_restore = optional_restore and saveable.optional_restore
-
-        if hasattr(saveable, "full_name"):
-          attribute.full_name = saveable.full_name
-        if isinstance(saveable, base.PythonStateSaveable):
-          if feed_additions is None:
-            assert saveables_cache is None
-            # If we're not caching saveables, then we're either executing
-            # eagerly or building a static save/restore (e.g. for a
-            # SavedModel). In either case, we should embed the current Python
-            # state in the graph rather than relying on a feed dict.
-            saveable = saveable.freeze()
-          else:
-            saveable_feed_dict = saveable.feed_dict_additions()
-            for new_feed_key in saveable_feed_dict.keys():
-              if new_feed_key in feed_additions:
-                raise AssertionError(
-                    ("The object %s tried to feed a value for the Tensor %s "
-                     "when saving, but another object is already feeding a "
-                     "value.")
-                    % (checkpointable, new_feed_key))
-            feed_additions.update(saveable_feed_dict)
-        named_saveable_objects.append(saveable)
-      if optional_restore is None:
-        optional_restore = False
-      attribute.optional_restore = optional_restore
-
-  return named_saveable_objects, feed_additions
-
-
-def fill_object_graph_proto(checkpointable_objects,
-                            node_ids,
-                            slot_variables,
-                            object_graph_proto=None):
-  """Name non-slot `Checkpointable`s and add them to `object_graph_proto`."""
-  if object_graph_proto is None:
-    object_graph_proto = (
-        checkpointable_object_graph_pb2.CheckpointableObjectGraph())
-  for checkpoint_id, checkpointable in enumerate(checkpointable_objects):
-    assert node_ids[checkpointable] == checkpoint_id
-    object_proto = object_graph_proto.nodes.add()
-    object_proto.slot_variables.extend(slot_variables.get(checkpointable, ()))
-    for child in checkpointable._checkpoint_dependencies:  # pylint: disable=protected-access
-      child_proto = object_proto.children.add()
-      child_proto.node_id = node_ids[child.ref]
-      child_proto.local_name = child.name
-  return object_graph_proto
-
-
-def _serialize_gathered_objects(
-    checkpointable_objects, path_to_root, saveables_cache, object_map):
-  """Create SaveableObjects and protos for gathered objects."""
-  object_names = ObjectIdentityDictionary()
-  for obj, path in path_to_root.items():
-    object_names[obj] = _object_prefix_from_path(path)
-  node_ids = ObjectIdentityDictionary()
-  for node_id, node in enumerate(checkpointable_objects):
-    node_ids[node] = node_id
-  slot_variables = _serialize_slot_variables(
-      checkpointable_objects=checkpointable_objects,
-      node_ids=node_ids,
-      object_names=object_names)
-  object_graph_proto = fill_object_graph_proto(
-      checkpointable_objects=checkpointable_objects,
-      node_ids=node_ids,
-      slot_variables=slot_variables)
-  named_saveable_objects, feed_additions = _add_attributes_to_object_graph(
-      checkpointable_objects=checkpointable_objects,
-      object_graph_proto=object_graph_proto,
-      node_ids=node_ids,
-      object_names=object_names,
-      saveables_cache=saveables_cache,
-      object_map=object_map)
-  return named_saveable_objects, object_graph_proto, feed_additions
-
-
-def _serialize_object_graph(root_checkpointable, saveables_cache):
-  """Determine checkpoint keys for variables and build a serialized graph.
-
-  Non-slot variables are keyed based on a shortest path from the root saveable
-  to the object which owns the variable (i.e. the one which called
-  `Checkpointable._add_variable` to create it).
-
-  Slot variables are keyed based on a shortest path to the variable being
-  slotted for, a shortest path to their optimizer, and the slot name.
-
-  Args:
-    root_checkpointable: A `Checkpointable` object whose variables (including
-      the variables of dependencies, recursively) should be saved.
-    saveables_cache: A dictionary mapping `Checkpointable` objects -> attribute
-      names -> SaveableObjects, used to avoid re-creating SaveableObjects when
-      graph building.
-
-  Returns:
-    A tuple of (named_variables, object_graph_proto, feed_additions):
-      named_variables: A dictionary mapping names to variable objects.
-      object_graph_proto: A CheckpointableObjectGraph protocol buffer containing
-        the serialized object graph and variable references.
-      feed_additions: A dictionary mapping from Tensors to values which should
-        be fed when saving.
-
-  Raises:
-    ValueError: If there are invalid characters in an optimizer's slot names.
-  """
-  checkpointable_objects, path_to_root = (
-      _breadth_first_checkpointable_traversal(root_checkpointable))
-  return _serialize_gathered_objects(
-      checkpointable_objects, path_to_root, saveables_cache, object_map=None)
-
-
-def named_saveables(root_checkpointable):
-  """Gather list of all SaveableObjects in the Checkpointable object."""
-  return _serialize_object_graph(root_checkpointable, None)[0]
-
-
-def find_objects(root_checkpointable):
-  """Find and number objects which are dependencies of `root_checkpointable`."""
-  checkpointable_objects, path_to_root = (
-      _breadth_first_checkpointable_traversal(root_checkpointable))
-  object_names = ObjectIdentityDictionary()
-  for obj, path in path_to_root.items():
-    object_names[obj] = _object_prefix_from_path(path)
-  node_ids = ObjectIdentityDictionary()
-  for node_id, node in enumerate(checkpointable_objects):
-    node_ids[node] = node_id
-  slot_variables = _serialize_slot_variables(
-      checkpointable_objects=checkpointable_objects,
-      node_ids=node_ids,
-      object_names=object_names)
-  return checkpointable_objects, node_ids, slot_variables
-
-
 def list_objects(root_checkpointable):
   """Traverse the object graph and list all accessible objects.
 
@@ -817,8 +340,7 @@ def list_objects(root_checkpointable):
   Returns:
     A flat list of objects.
   """
-  checkpointable_objects, _, _ = find_objects(root_checkpointable)
-  return checkpointable_objects
+  return graph_view_lib.ObjectGraphView(root_checkpointable).list_objects()
 
 
 def gather_initializers(root_checkpointable):
@@ -999,10 +521,10 @@ class CheckpointLoadStatus(_LoadStatus):
   See `Saver.restore` for usage examples.
   """
 
-  def __init__(self, checkpoint, feed_dict, root_checkpointable):
+  def __init__(self, checkpoint, feed_dict, graph_view):
     self._checkpoint = checkpoint
     self._feed_dict = feed_dict
-    self._root_checkpointable = root_checkpointable
+    self._graph_view = graph_view
 
   def assert_consumed(self):
     """Asserts that all objects in the checkpoint have been created/matched.
@@ -1055,7 +577,7 @@ class CheckpointLoadStatus(_LoadStatus):
           and checkpointable._update_uid < self._checkpoint.restore_uid):  # pylint: disable=protected-access
         raise AssertionError(
             "Object not assigned a value from checkpoint: %s" % (node,))
-    for checkpointable_object in list_objects(self._root_checkpointable):
+    for checkpointable_object in self._graph_view.list_objects():
       # Remove data structures that do not contain any variables from
       # restoration checks.
       if (isinstance(checkpointable_object,
@@ -1064,8 +586,9 @@ class CheckpointLoadStatus(_LoadStatus):
         continue
       self._checkpoint.all_python_objects.add(checkpointable_object)
     unused_python_objects = (
-        _ObjectIdentitySet(self._checkpoint.all_python_objects)
-        - _ObjectIdentitySet(self._checkpoint.object_by_proto_id.values()))
+        object_identity.ObjectIdentitySet(self._checkpoint.all_python_objects)
+        - object_identity.ObjectIdentitySet(
+            self._checkpoint.object_by_proto_id.values()))
     if unused_python_objects:
       raise AssertionError(
           ("Some Python objects were not bound to checkpointed values, likely "
@@ -1075,12 +598,14 @@ class CheckpointLoadStatus(_LoadStatus):
 
   def assert_nontrivial_match(self):
     """Raises an exception if only the root object matched."""
-    for checkpointable_object in list_objects(self._root_checkpointable):
+    for checkpointable_object in self._graph_view.list_objects():
       self._checkpoint.all_python_objects.add(checkpointable_object)
     if len(self._checkpoint.object_by_proto_id) <= 1:
       unused_python_objects = (
-          _ObjectIdentitySet(self._checkpoint.all_python_objects)
-          - _ObjectIdentitySet(self._checkpoint.object_by_proto_id.values()))
+          object_identity.ObjectIdentitySet(
+              self._checkpoint.all_python_objects)
+          - object_identity.ObjectIdentitySet(
+              self._checkpoint.object_by_proto_id.values()))
       if unused_python_objects:
         raise AssertionError(
             ("Nothing except the root object matched a checkpointed value. "
@@ -1090,7 +615,7 @@ class CheckpointLoadStatus(_LoadStatus):
       else:
         raise AssertionError(
             "Nothing to load. No dependencies have been added to %s yet." % (
-                self._root_checkpointable,))
+                self._graph_view.root,))
     return self
 
   def run_restore_ops(self, session=None):
@@ -1120,8 +645,8 @@ class CheckpointLoadStatus(_LoadStatus):
       return  # Initialization and restoration ops are run eagerly
     if session is None:
       session = ops.get_default_session()
-    all_objects = list_objects(self._root_checkpointable)
-    already_initialized_objects = _ObjectIdentitySet(
+    all_objects = self._graph_view.list_objects()
+    already_initialized_objects = object_identity.ObjectIdentitySet(
         self._checkpoint.object_by_proto_id.values())
     initializers_for_non_restored_variables = [
         c.initializer for c in all_objects
@@ -1143,9 +668,9 @@ class InitializationOnlyStatus(_LoadStatus):
   otherwise.
   """
 
-  def __init__(self, root_checkpointable, restore_uid):
+  def __init__(self, graph_view, restore_uid):
     self._restore_uid = restore_uid
-    self._root_checkpointable = root_checkpointable
+    self._graph_view = graph_view
 
   def assert_consumed(self):
     """Assertion for consistency with `CheckpointLoadStatus`. Always fails."""
@@ -1193,7 +718,7 @@ class InitializationOnlyStatus(_LoadStatus):
       return  # run eagerly
     if session is None:
       session = ops.get_default_session()
-    checkpointable_objects = list_objects(self._root_checkpointable)
+    checkpointable_objects = self._graph_view.list_objects()
     initializers = [
         c.initializer for c in checkpointable_objects
         if hasattr(c, "initializer") and c.initializer is not None
@@ -1218,9 +743,9 @@ class NameBasedSaverStatus(_LoadStatus):
   # interferes with isinstance checks.
   @deprecation.deprecated(
       date=None, instructions=_DEPRECATED_RESTORE_INSTRUCTIONS)
-  def __init__(self, checkpoint, root_checkpointable):
+  def __init__(self, checkpoint, graph_view):
     self._checkpoint = checkpoint
-    self._root_checkpointable = root_checkpointable
+    self._graph_view = graph_view
 
   def assert_consumed(self):
     """Raises an exception if any variables/objects are unmatched."""
@@ -1229,7 +754,7 @@ class NameBasedSaverStatus(_LoadStatus):
       raise AssertionError(
           "Some objects had attributes which were not restored: %s"
           % (unused_attributes,))
-    for checkpointable in list_objects(self._root_checkpointable):
+    for checkpointable in self._graph_view.list_objects():
       # pylint: disable=protected-access
       checkpointable._maybe_initialize_checkpointable()
       if checkpointable._update_uid < self._checkpoint.restore_uid:
@@ -1255,7 +780,7 @@ class NameBasedSaverStatus(_LoadStatus):
 
   def _gather_saveable_objects(self):
     """Walk the object graph, using global names for SaveableObjects."""
-    objects = list_objects(self._root_checkpointable)
+    objects = self._graph_view.list_objects()
     saveable_objects = []
     for checkpointable in objects:
       # pylint: disable=protected-access
@@ -1322,17 +847,13 @@ class CheckpointableSaver(object):
   so allow additional program transformations.
   """
 
-  def __init__(self, root_checkpointable):
+  def __init__(self, graph_view):
     """Configure saving.
 
     Args:
-      root_checkpointable: The root of the object graph to save/restore. This
-        object and all of its dependencies are saved in the checkpoint. When
-        restoring, objects are matched and restored starting from this root.
+      graph_view: A `GraphView` object containing a description of the object
+        graph to save.
     """
-    # Allow passing in a weak reference to avoid reference cycles when
-    # `Checkpointable` objects save themselves.
-    self._root_checkpointable_ref = root_checkpointable
     # The file prefix placeholder is created lazily when graph building (and not
     # at all when executing eagerly) to avoid creating ops in the constructor
     # (when they may never be necessary).
@@ -1346,34 +867,13 @@ class CheckpointableSaver(object):
 
     # Op caching for restore, shared between _CheckpointRestoreCoordinators
     self._restore_op_cache = {}
-
-    if context.executing_eagerly():
-      # SaveableObjects are always recreated when executing eagerly.
-      self._saveable_object_cache = None
-    else:
-      # Maps Checkpointable objects -> attribute names -> list(SaveableObjects),
-      # to avoid re-creating SaveableObjects when graph building.
-      self._saveable_object_cache = _ObjectIdentityWeakKeyDictionary()
-
-  @property
-  def _root_checkpointable(self):
-    if isinstance(self._root_checkpointable_ref, weakref.ref):
-      derefed = self._root_checkpointable_ref()
-      assert derefed is not None
-      return derefed
-    else:
-      return self._root_checkpointable_ref
+    self._graph_view = graph_view
 
   def _gather_saveables(
-      self, object_graph_tensor=None, saveable_object_cache=None):
+      self, object_graph_tensor=None):
     """Wraps _serialize_object_graph to include the object graph proto."""
-    assert ((object_graph_tensor is None and saveable_object_cache is None)
-            or (object_graph_tensor is not None
-                and saveable_object_cache is not None))
     (named_saveable_objects, graph_proto,
-     feed_additions) = _serialize_object_graph(
-         self._root_checkpointable,
-         saveables_cache=saveable_object_cache)
+     feed_additions) = self._graph_view.serialize_object_graph()
     if object_graph_tensor is None:
       with ops.device("/cpu:0"):
         object_graph_tensor = constant_op.constant(
@@ -1388,52 +888,16 @@ class CheckpointableSaver(object):
             name=base.OBJECT_GRAPH_PROTO_KEY))
     return named_saveable_objects, graph_proto, feed_additions
 
-  def gather_objects(self, object_map=None, to_graph=None):
-    """Creates SaveableObjects with the current object graph frozen."""
-    checkpointable_objects, path_to_root = (
-        _breadth_first_checkpointable_traversal(self._root_checkpointable))
-    if to_graph:
-      target_context = to_graph.as_default
-    else:
-      target_context = ops.NullContextmanager
-    with target_context():
-      named_saveable_objects, graph_proto, _ = _serialize_gathered_objects(
-          checkpointable_objects,
-          path_to_root,
-          saveables_cache=None,
-          object_map=object_map)
-      with ops.device("/cpu:0"):
-        object_graph_tensor = constant_op.constant(
-            graph_proto.SerializeToString(), dtype=dtypes.string)
-      named_saveable_objects.append(
-          base.NoRestoreSaveable(
-              tensor=object_graph_tensor,
-              name=base.OBJECT_GRAPH_PROTO_KEY))
-    return named_saveable_objects
-
-  def freeze(self, object_map=None, to_graph=None):
-    named_saveable_objects = self.gather_objects(
-        object_map=object_map, to_graph=to_graph)
-    return functional_saver.Saver(named_saveable_objects)
-
   def _save_cached_when_graph_building(
       self,
       file_prefix,
-      object_graph_tensor=None,
-      saveable_object_cache=None):
+      object_graph_tensor=None):
     """Create or retrieve save ops.
 
-    When graph building, `saveable_object_cache` will typically be non-`None`,
-    meaning that existing `SaveableObject`s are re-used across calls to
-    `_prepare_save` even if the object graph has grown. This avoids
-    unnecessarily re-creating save ops.
-
     Args:
       file_prefix: The prefix for saved checkpoint files.
       object_graph_tensor: A `Tensor` to which the current object graph will be
         fed.
-      saveable_object_cache: A dictionary; if specified, used to cache
-        `SaveableObject`s.
 
     Returns:
       A two-element tuple with a filename tensor and a feed_dict of tensors to
@@ -1443,8 +907,7 @@ class CheckpointableSaver(object):
     """
     (named_saveable_objects, graph_proto,
      feed_additions) = self._gather_saveables(
-         object_graph_tensor=object_graph_tensor,
-         saveable_object_cache=saveable_object_cache)
+         object_graph_tensor=object_graph_tensor)
     if (self._last_save_object_graph != graph_proto
         # When executing eagerly, we need to re-create SaveableObjects each time
         # save() is called so they pick up new Tensors passed to their
@@ -1502,8 +965,7 @@ class CheckpointableSaver(object):
     file_io.recursive_create_dir(os.path.dirname(file_prefix))
     save_path, new_feed_additions = self._save_cached_when_graph_building(
         file_prefix=file_prefix_tensor,
-        object_graph_tensor=object_graph_tensor,
-        saveable_object_cache=self._saveable_object_cache)
+        object_graph_tensor=object_graph_tensor)
     if new_feed_additions:
       feed_dict.update(new_feed_additions)
     if not graph_building:
@@ -1576,7 +1038,7 @@ class CheckpointableSaver(object):
       object is returned which runs restore ops from a name-based saver.
     """
     if save_path is None:
-      return InitializationOnlyStatus(self._root_checkpointable, ops.uid())
+      return InitializationOnlyStatus(self._graph_view, ops.uid())
     reader = pywrap_tensorflow.NewCheckpointReader(save_path)
     graph_building = not context.executing_eagerly()
     if graph_building:
@@ -1592,7 +1054,7 @@ class CheckpointableSaver(object):
       restore_coordinator = _NameBasedRestoreCoordinator(
           save_path=save_path, dtype_map=dtype_map)
       if not graph_building:
-        for existing_checkpointable in list_objects(self._root_checkpointable):
+        for existing_checkpointable in self._graph_view.list_objects():
           # pylint: disable=protected-access
           existing_checkpointable._maybe_initialize_checkpointable()
           existing_checkpointable._name_based_restores.add(restore_coordinator)
@@ -1600,7 +1062,7 @@ class CheckpointableSaver(object):
               restore_coordinator)
           # pylint: enable=protected-access
       return NameBasedSaverStatus(
-          restore_coordinator, root_checkpointable=self._root_checkpointable)
+          restore_coordinator, graph_view=self._graph_view)
 
     if graph_building:
       if self._file_prefix_placeholder is None:
@@ -1620,12 +1082,12 @@ class CheckpointableSaver(object):
         save_path=save_path,
         save_path_tensor=file_prefix_tensor,
         restore_op_cache=self._restore_op_cache,
-        saveable_object_cache=self._saveable_object_cache)
-    base._CheckpointPosition(  # pylint: disable=protected-access
-        checkpoint=checkpoint, proto_id=0).restore(self._root_checkpointable)
+        graph_view=self._graph_view)
+    base.CheckpointPosition(checkpoint=checkpoint, proto_id=0).restore(
+        self._graph_view.root)
     load_status = CheckpointLoadStatus(
         checkpoint,
-        root_checkpointable=self._root_checkpointable,
+        graph_view=self._graph_view,
         feed_dict=file_prefix_feed_dict)
     return load_status
 
@@ -1648,10 +1110,23 @@ def frozen_saver(root_checkpointable):
     root_checkpointable: A checkpointable object to save.
 
   Returns:
-    A `tf.train.Saver` which saves object-based checkpoints for the object graph
-    frozen at the time `frozen_saver` was called.
+    A saver which saves object-based checkpoints for the object graph frozen at
+    the time `frozen_saver` was called.
   """
-  return CheckpointableSaver(root_checkpointable).freeze()
+  named_saveable_objects = graph_view_lib.ObjectGraphView(
+      root_checkpointable).frozen_saveable_objects()
+  return functional_saver.Saver(named_saveable_objects)
+
+
+def saver_with_op_caching(obj):
+  """A CheckpointableSaver with a SaveableObject cache when graph building."""
+  if context.executing_eagerly():
+    saveables_cache = None
+  else:
+    saveables_cache = object_identity.ObjectIdentityWeakKeyDictionary()
+  return CheckpointableSaver(graph_view_lib.ObjectGraphView(
+      weakref.ref(obj),
+      saveables_cache=saveables_cache))
 
 
 @tf_export("train.Checkpoint")
@@ -1767,7 +1242,7 @@ class Checkpoint(tracking.AutoCheckpointable):
       setattr(self, k, v)
     self._save_counter = None  # Created lazily for restore-on-create.
     self._save_assign_op = None
-    self._saver = CheckpointableSaver(weakref.ref(self))
+    self._saver = saver_with_op_caching(self)
 
   def _maybe_create_save_counter(self):
     """Create a save counter if it does not yet exist."""
diff --git a/tensorflow/python/training/checkpointable/util_test.py b/tensorflow/python/training/checkpointable/util_test.py
index cef1075e93..4f9abccfe4 100644
--- a/tensorflow/python/training/checkpointable/util_test.py
+++ b/tensorflow/python/training/checkpointable/util_test.py
@@ -47,6 +47,7 @@ from tensorflow.python.training import checkpoint_management
 from tensorflow.python.training import saver as saver_lib
 from tensorflow.python.training import training_util
 from tensorflow.python.training.checkpointable import base
+from tensorflow.python.training.checkpointable import graph_view
 from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.training.checkpointable import util as checkpointable_utils
 
@@ -125,8 +126,8 @@ class InterfaceTests(test.TestCase):
       # The .name attribute may be globally influenced, but the checkpoint name
       # won't be (tested below).
       self.assertEqual("duplicate_1:0", duplicate.name)
-    named_variables, _, _ = checkpointable_utils._serialize_object_graph(
-        obj, saveables_cache=None)
+    named_variables, _, _ = (
+        graph_view.ObjectGraphView(obj).serialize_object_graph())
     expected_checkpoint_names = (
         "a_variable/.ATTRIBUTES/VARIABLE_VALUE",
         "bare_initializer/.ATTRIBUTES/VARIABLE_VALUE",
@@ -274,9 +275,8 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
     self.evaluate(checkpointable_utils.gather_initializers(
         root_checkpointable))
     self.evaluate(train_op)
-    named_variables, serialized_graph, _ = (
-        checkpointable_utils._serialize_object_graph(
-            root_checkpointable, saveables_cache=None))
+    named_variables, serialized_graph, _ = graph_view.ObjectGraphView(
+        root_checkpointable).serialize_object_graph()
     expected_slot_keys = (
         "model/_second/kernel/.OPTIMIZER_SLOT/optimizer/m",
         "model/_second/kernel/.OPTIMIZER_SLOT/optimizer/v",
@@ -656,8 +656,8 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
     root = tracking.AutoCheckpointable()
     checkpointable_utils.add_variable(
         root, name=name, shape=[1, 2], dtype=dtypes.float64)
-    (named_variable,), _, _ = checkpointable_utils._serialize_object_graph(
-        root, saveables_cache=None)
+    (named_variable,), _, _ = graph_view.ObjectGraphView(
+        root).serialize_object_graph()
     with ops.name_scope("root/" + named_variable.name):
       pass  # Make sure we can use this as an op name if we prefix it.
     return named_variable.name
@@ -678,8 +678,8 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
     leaf = tracking.AutoCheckpointable()
     root.leaf = leaf
     checkpointable_utils.add_variable(leaf, name="v", shape=[])
-    (named_variable,), _, _ = checkpointable_utils._serialize_object_graph(
-        root, saveables_cache=None)
+    (named_variable,), _, _ = graph_view.ObjectGraphView(
+        root).serialize_object_graph()
     self.assertEqual(r"leaf/v/.ATTRIBUTES/VARIABLE_VALUE", named_variable.name)
 
   @test_util.run_in_graph_and_eager_modes
@@ -689,8 +689,8 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
     # Dots are escaped, which avoids conflicts with reserved names.
     root._track_checkpointable(leaf, name=".ATTRIBUTES")
     checkpointable_utils.add_variable(checkpointable=leaf, name="a", shape=[])
-    (named_variable,), _, _ = checkpointable_utils._serialize_object_graph(
-        root, saveables_cache=None)
+    (named_variable,), _, _ = graph_view.ObjectGraphView(
+        root).serialize_object_graph()
     self.assertEqual("..ATTRIBUTES/a/.ATTRIBUTES/VARIABLE_VALUE",
                      named_variable.name)
 
@@ -732,7 +732,7 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
         self.var = checkpointable_utils.add_variable(
             self, "var", initializer=0.)
 
-    class LateDependencies(tracking.AutoCheckpointable):
+    class LateDependencies(checkpointable_utils.Checkpoint):
 
       def add_dep(self):
         self.dep = Dependency()
@@ -743,11 +743,9 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
     self.evaluate(state_ops.assign(original.dep.var, 123.))
     checkpoint_directory = self.get_temp_dir()
     checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-    save_path = checkpointable_utils.CheckpointableSaver(
-        original).save(checkpoint_prefix)
+    save_path = original.save(checkpoint_prefix)
     load_into = LateDependencies()
-    status = checkpointable_utils.CheckpointableSaver(
-        load_into).restore(save_path)
+    status = load_into.restore(save_path)
     status.assert_existing_objects_matched()
     with self.assertRaises(AssertionError):
       status.assert_consumed()
@@ -765,7 +763,7 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
         self.var = checkpointable_utils.add_variable(
             self, "var", initializer=0.)
 
-    class DepAfterVar(tracking.AutoCheckpointable):
+    class DepAfterVar(checkpointable_utils.Checkpoint):
 
       def add_dep(self):
         dep = Dependency()
@@ -777,12 +775,10 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
     self.evaluate(state_ops.assign(dep_after_var.dep.var, -14.))
     checkpoint_directory = self.get_temp_dir()
     checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-    save_path = checkpointable_utils.CheckpointableSaver(dep_after_var).save(
-        checkpoint_prefix)
+    save_path = dep_after_var.save(checkpoint_prefix)
 
     loaded_dep_after_var = DepAfterVar()
-    status = checkpointable_utils.CheckpointableSaver(
-        loaded_dep_after_var).restore(save_path)
+    status = loaded_dep_after_var.restore(save_path)
     loaded_dep_after_var.add_dep()
     status.assert_consumed()
     status.run_restore_ops()
@@ -792,7 +788,7 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
   def testDeferredSlotRestoration(self):
     checkpoint_directory = self.get_temp_dir()
 
-    root = tracking.AutoCheckpointable()
+    root = checkpointable_utils.Checkpoint()
     root.var = checkpointable_utils.add_variable(
         root, name="var", initializer=0.)
     optimizer = adam.Adam(0.1)
@@ -806,22 +802,18 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
         checkpointable_utils.Checkpoint(root=root, optimizer=optimizer)))
     self.evaluate(train_op)
     self.evaluate(state_ops.assign(root.var, 12.))
-    no_slots_path = checkpointable_utils.CheckpointableSaver(root).save(
-        os.path.join(checkpoint_directory, "no_slots"))
+    no_slots_path = root.save(os.path.join(checkpoint_directory, "no_slots"))
     root.optimizer = optimizer
     self.evaluate(state_ops.assign(root.var, 13.))
     self.evaluate(state_ops.assign(
         optimizer.get_slot(slot_name="m", var=root.var),
         14.))
-    slots_path = checkpointable_utils.CheckpointableSaver(root).save(
-        os.path.join(checkpoint_directory, "with_slots"))
-    new_root = tracking.AutoCheckpointable()
+    slots_path = root.save(os.path.join(checkpoint_directory, "with_slots"))
+    new_root = checkpointable_utils.Checkpoint()
     # Load the slot-containing checkpoint (deferred), then immediately overwrite
     # the non-slot variable (also deferred).
-    slot_status = checkpointable_utils.CheckpointableSaver(
-        new_root).restore(slots_path)
-    no_slot_status = checkpointable_utils.CheckpointableSaver(
-        new_root).restore(no_slots_path)
+    slot_status = new_root.restore(slots_path)
+    no_slot_status = new_root.restore(no_slots_path)
     with self.assertRaises(AssertionError):
       no_slot_status.assert_consumed()
     new_root.var = checkpointable_utils.add_variable(
@@ -861,22 +853,19 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
   @test_util.run_in_graph_and_eager_modes
   def testOverlappingRestores(self):
     checkpoint_directory = self.get_temp_dir()
-    save_root = tracking.AutoCheckpointable()
+    save_root = checkpointable_utils.Checkpoint()
     save_root.dep = tracking.AutoCheckpointable()
     save_root.dep.var = checkpointable_utils.add_variable(
         save_root.dep, name="var", initializer=0.)
     self.evaluate(state_ops.assign(save_root.dep.var, 12.))
-    saver = checkpointable_utils.CheckpointableSaver(save_root)
-    first_path = saver.save(os.path.join(checkpoint_directory, "first"))
+    first_path = save_root.save(os.path.join(checkpoint_directory, "first"))
     self.evaluate(state_ops.assign(save_root.dep.var, 13.))
-    second_path = saver.save(os.path.join(checkpoint_directory, "second"))
-
-    first_root = tracking.AutoCheckpointable()
-    second_root = tracking.AutoCheckpointable()
-    first_status = checkpointable_utils.CheckpointableSaver(
-        first_root).restore(first_path)
-    second_status = checkpointable_utils.CheckpointableSaver(
-        second_root).restore(second_path)
+    second_path = save_root.save(os.path.join(checkpoint_directory, "second"))
+
+    first_root = checkpointable_utils.Checkpoint()
+    second_root = checkpointable_utils.Checkpoint()
+    first_status = first_root.restore(first_path)
+    second_status = second_root.restore(second_path)
     load_dep = tracking.AutoCheckpointable()
     load_dep.var = checkpointable_utils.add_variable(
         load_dep, name="var", shape=[])
@@ -891,12 +880,10 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
 
     # Try again with the order of the restore() reversed. The last restore
     # determines the final value.
-    first_root = tracking.AutoCheckpointable()
-    second_root = tracking.AutoCheckpointable()
-    second_status = checkpointable_utils.CheckpointableSaver(
-        second_root).restore(second_path)
-    first_status = checkpointable_utils.CheckpointableSaver(
-        first_root).restore(first_path)
+    first_root = checkpointable_utils.Checkpoint()
+    second_root = checkpointable_utils.Checkpoint()
+    second_status = second_root.restore(second_path)
+    first_status = first_root.restore(first_path)
     load_dep = tracking.AutoCheckpointable()
     load_dep.var = checkpointable_utils.add_variable(
         load_dep, name="var", shape=[])
@@ -913,7 +900,7 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
   def testAmbiguousLoad(self):
     # Not OK to split one checkpoint object into two
     checkpoint_directory = self.get_temp_dir()
-    save_root = tracking.AutoCheckpointable()
+    save_root = checkpointable_utils.Checkpoint()
     save_root.dep_one = tracking.AutoCheckpointable()
     save_root.dep_two = tracking.AutoCheckpointable()
     dep_three = tracking.AutoCheckpointable()
@@ -921,11 +908,9 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
     save_root.dep_two.dep_three = dep_three
     checkpointable_utils.add_variable(dep_three, name="var", initializer=0.)
     self.evaluate(checkpointable_utils.gather_initializers(save_root))
-    save_path = checkpointable_utils.CheckpointableSaver(save_root).save(
-        os.path.join(checkpoint_directory, "ckpt"))
-    load_root = tracking.AutoCheckpointable()
-    status = checkpointable_utils.CheckpointableSaver(load_root).restore(
-        save_path)
+    save_path = save_root.save(os.path.join(checkpoint_directory, "ckpt"))
+    load_root = checkpointable_utils.Checkpoint()
+    status = load_root.restore(save_path)
     load_root.dep_one = tracking.AutoCheckpointable()
     load_root.dep_two = tracking.AutoCheckpointable()
     load_root.dep_one.dep_three = tracking.AutoCheckpointable()
@@ -941,7 +926,7 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
   def testObjectsCombined(self):
     # Currently fine to load two checkpoint objects into one Python object
     checkpoint_directory = self.get_temp_dir()
-    save_root = tracking.AutoCheckpointable()
+    save_root = checkpointable_utils.Checkpoint()
     save_root.dep_one = tracking.AutoCheckpointable()
     save_root.dep_two = tracking.AutoCheckpointable()
     checkpointable_utils.add_variable(
@@ -949,16 +934,15 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
     checkpointable_utils.add_variable(
         save_root.dep_two, name="var2", initializer=64., dtype=dtypes.float64)
     self.evaluate(checkpointable_utils.gather_initializers(save_root))
-    save_path = checkpointable_utils.CheckpointableSaver(save_root).save(
-        os.path.join(checkpoint_directory, "ckpt"))
-    load_root = tracking.AutoCheckpointable()
+    save_path = save_root.save(os.path.join(checkpoint_directory, "ckpt"))
+    load_root = checkpointable_utils.Checkpoint()
     load_root.dep_one = tracking.AutoCheckpointable()
     load_root.dep_two = load_root.dep_one
     v1 = checkpointable_utils.add_variable(
         load_root.dep_one, name="var1", shape=[], dtype=dtypes.float64)
     v2 = checkpointable_utils.add_variable(
         load_root.dep_one, name="var2", shape=[], dtype=dtypes.float64)
-    status = checkpointable_utils.CheckpointableSaver(load_root).restore(
+    status = load_root.restore(
         save_path).assert_consumed().assert_existing_objects_matched()
     status.run_restore_ops()
     self.assertEqual(32., self.evaluate(v1))
@@ -968,8 +952,8 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
   def testDependencyLoop(self):
     # Note: this test creates garbage during eager execution because it
     # purposefully creates a reference cycle.
-    first = tracking.AutoCheckpointable()
-    second = tracking.AutoCheckpointable()
+    first = checkpointable_utils.Checkpoint()
+    second = checkpointable_utils.Checkpoint()
     first.second = second
     second.first = first
     first.v = checkpointable_utils.add_variable(
@@ -978,13 +962,11 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
         second, "v2", initializer=[1., 1., 2., 3.])
     self.evaluate(checkpointable_utils.gather_initializers(first))
     checkpoint_directory = self.get_temp_dir()
-    save_path = checkpointable_utils.CheckpointableSaver(first).save(
-        os.path.join(checkpoint_directory, "ckpt"))
+    save_path = first.save(os.path.join(checkpoint_directory, "ckpt"))
 
     # Test deferred loading
-    first_load = tracking.AutoCheckpointable()
-    status = checkpointable_utils.CheckpointableSaver(
-        first_load).restore(save_path)
+    first_load = checkpointable_utils.Checkpoint()
+    status = first_load.restore(save_path)
     second_load = tracking.AutoCheckpointable()
     first_load.second = second_load
     second_load.first = first_load
@@ -1004,8 +986,7 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
     self.assertAllEqual([2., 7., 1.], self.evaluate(first_load.v))
     self.evaluate(second_load.v.assign([2., 7., 1., 8.]))
     self.assertAllEqual([2., 7., 1., 8.], self.evaluate(second_load.v))
-    status = checkpointable_utils.CheckpointableSaver(first_load).restore(
-        save_path).assert_consumed()
+    status = first_load.restore(save_path).assert_consumed()
     status.run_restore_ops()
     self.assertAllEqual([3., 1., 4.], self.evaluate(first_load.v))
     self.assertAllEqual([1., 1., 2., 3.], self.evaluate(second_load.v))
@@ -1014,18 +995,16 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
   def testRestoreOnAssign(self):
     checkpoint_directory = self.get_temp_dir()
     checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-    first = tracking.AutoCheckpointable()
+    first = checkpointable_utils.Checkpoint()
     first.var1 = variables_lib.Variable(0., name="outside_var")
     first.var2 = variables_lib.Variable(0., name="blah")
     self.evaluate(first.var1.assign(4.))
     self.evaluate(first.var2.assign(8.))
-    save_path = checkpointable_utils.CheckpointableSaver(first).save(
-        checkpoint_prefix)
+    save_path = first.save(checkpoint_prefix)
 
-    second = tracking.AutoCheckpointable()
+    second = checkpointable_utils.Checkpoint()
     second.var2 = variables_lib.Variable(0., name="blah")
-    status = checkpointable_utils.CheckpointableSaver(
-        second).restore(save_path)
+    status = second.restore(save_path)
     recreated_var1 = variables_lib.Variable(0., name="outside_var")
     status.run_restore_ops()
     self.assertEqual(8., self.evaluate(second.var2))
@@ -1042,17 +1021,16 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
       with graph.as_default(), self.session(graph):
         checkpoint_directory = self.get_temp_dir()
         checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-        obj = tracking.AutoCheckpointable()
+        obj = checkpointable_utils.Checkpoint()
         obj.var = variables_lib.Variable(0., name="v")
         obj.opt = adam.Adam(0.1)
         variables = [obj.var]
         gradients = [1.]
         obj.opt.apply_gradients(zip(gradients, variables))
         self.evaluate(checkpointable_utils.gather_initializers(obj))
-        saver = checkpointable_utils.CheckpointableSaver(obj)
-        saver.save(checkpoint_prefix)
+        obj.save(checkpoint_prefix)
         graph.finalize()
-        saver.save(checkpoint_prefix)
+        obj.save(checkpoint_prefix)
 
   @test_util.run_in_graph_and_eager_modes
   def testCheckpointState(self):
@@ -1132,18 +1110,17 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
       with graph.as_default(), self.session(graph):
         checkpoint_directory = self.get_temp_dir()
         checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-        obj = tracking.AutoCheckpointable()
+        obj = checkpointable_utils.Checkpoint()
         obj.var = variables_lib.Variable(0., name="v")
         obj.opt = adam.Adam(0.1)
         variables = [obj.var]
         gradients = [1.]
         obj.opt.apply_gradients(zip(gradients, variables))
         self.evaluate(checkpointable_utils.gather_initializers(obj))
-        saver = checkpointable_utils.CheckpointableSaver(obj)
-        save_path = saver.save(checkpoint_prefix)
-        saver.restore(save_path)
+        save_path = obj.save(checkpoint_prefix)
+        obj.restore(save_path)
         graph.finalize()
-        saver.restore(save_path)
+        obj.restore(save_path)
 
   @test_util.run_in_graph_and_eager_modes
   def test_sequential(self):
@@ -1461,7 +1438,8 @@ class CheckpointCompatibilityTests(test.TestCase):
       self._set_sentinels(root)
       with self.assertRaises(AssertionError):
         self._check_sentinels(root)
-      object_saver = checkpointable_utils.CheckpointableSaver(root)
+      object_saver = checkpointable_utils.CheckpointableSaver(
+          graph_view.ObjectGraphView(root))
       self._set_sentinels(root)
       status = object_saver.restore(save_path)
       if context.executing_eagerly():
diff --git a/tensorflow/python/training/checkpointable/util_with_v1_optimizers_test.py b/tensorflow/python/training/checkpointable/util_with_v1_optimizers_test.py
index bd80fa60f0..d7158c0ddb 100644
--- a/tensorflow/python/training/checkpointable/util_with_v1_optimizers_test.py
+++ b/tensorflow/python/training/checkpointable/util_with_v1_optimizers_test.py
@@ -43,6 +43,7 @@ from tensorflow.python.training import adam
 from tensorflow.python.training import checkpoint_management
 from tensorflow.python.training import saver as saver_lib
 from tensorflow.python.training import training_util
+from tensorflow.python.training.checkpointable import graph_view
 from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.training.checkpointable import util as checkpointable_utils
 
@@ -100,9 +101,8 @@ class CheckpointingTests(test.TestCase):
       self.evaluate(checkpointable_utils.gather_initializers(
           root_checkpointable))
       self.evaluate(train_op)
-    named_variables, serialized_graph, _ = (
-        checkpointable_utils._serialize_object_graph(
-            root_checkpointable, saveables_cache=None))
+    named_variables, serialized_graph, _ = graph_view.ObjectGraphView(
+        root_checkpointable).serialize_object_graph()
     expected_checkpoint_names = (
         # Created in the root node, so no prefix.
         "optimizer_step",
@@ -503,7 +503,7 @@ class CheckpointingTests(test.TestCase):
   def testDeferredSlotRestoration(self):
     checkpoint_directory = self.get_temp_dir()
 
-    root = tracking.AutoCheckpointable()
+    root = checkpointable_utils.Checkpoint()
     root.var = checkpointable_utils.add_variable(
         root, name="var", initializer=0.)
     optimizer = adam.AdamOptimizer(0.1)
@@ -518,21 +518,17 @@ class CheckpointingTests(test.TestCase):
           checkpointable_utils.Checkpoint(root=root, optimizer=optimizer)))
       self.evaluate(train_op)
     self.evaluate(state_ops.assign(root.var, 12.))
-    no_slots_path = checkpointable_utils.CheckpointableSaver(root).save(
-        os.path.join(checkpoint_directory, "no_slots"))
+    no_slots_path = root.save(os.path.join(checkpoint_directory, "no_slots"))
     root.optimizer = optimizer
     self.evaluate(state_ops.assign(root.var, 13.))
     self.evaluate(state_ops.assign(optimizer.get_slot(name="m", var=root.var),
                                    14.))
-    slots_path = checkpointable_utils.CheckpointableSaver(root).save(
-        os.path.join(checkpoint_directory, "with_slots"))
-    new_root = tracking.AutoCheckpointable()
+    slots_path = root.save(os.path.join(checkpoint_directory, "with_slots"))
+    new_root = checkpointable_utils.Checkpoint()
     # Load the slot-containing checkpoint (deferred), then immediately overwrite
     # the non-slot variable (also deferred).
-    slot_status = checkpointable_utils.CheckpointableSaver(
-        new_root).restore(slots_path)
-    no_slot_status = checkpointable_utils.CheckpointableSaver(
-        new_root).restore(no_slots_path)
+    slot_status = new_root.restore(slots_path)
+    no_slot_status = new_root.restore(no_slots_path)
     with self.assertRaises(AssertionError):
       no_slot_status.assert_consumed()
     new_root.var = checkpointable_utils.add_variable(
@@ -572,15 +568,14 @@ class CheckpointingTests(test.TestCase):
       with graph.as_default(), self.session(graph):
         checkpoint_directory = self.get_temp_dir()
         checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-        obj = tracking.AutoCheckpointable()
+        obj = checkpointable_utils.Checkpoint()
         obj.var = variable_scope.get_variable(name="v", initializer=0.)
         obj.opt = adam.AdamOptimizer(0.1)
         obj.opt.minimize(obj.var.read_value())
         self.evaluate(checkpointable_utils.gather_initializers(obj))
-        saver = checkpointable_utils.CheckpointableSaver(obj)
-        saver.save(checkpoint_prefix)
+        obj.save(checkpoint_prefix)
         before_ops = graph.get_operations()
-        saver.save(checkpoint_prefix)
+        obj.save(checkpoint_prefix)
         self.assertEqual(before_ops, graph.get_operations())
 
   def testManyRestoresGraph(self):
@@ -590,16 +585,15 @@ class CheckpointingTests(test.TestCase):
       with graph.as_default(), self.session(graph):
         checkpoint_directory = self.get_temp_dir()
         checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-        obj = tracking.AutoCheckpointable()
+        obj = checkpointable_utils.Checkpoint()
         obj.var = variable_scope.get_variable(name="v", initializer=0.)
         obj.opt = adam.AdamOptimizer(0.1)
         obj.opt.minimize(obj.var.read_value())
         self.evaluate(checkpointable_utils.gather_initializers(obj))
-        saver = checkpointable_utils.CheckpointableSaver(obj)
-        save_path = saver.save(checkpoint_prefix)
-        saver.restore(save_path)
+        save_path = obj.save(checkpoint_prefix)
+        obj.restore(save_path)
         before_ops = graph.get_operations()
-        saver.restore(save_path)
+        obj.restore(save_path)
         self.assertEqual(before_ops, graph.get_operations())
 
   def testMultipleGraphsNonSlotVariables(self):
@@ -869,7 +863,8 @@ class CheckpointCompatibilityTests(test.TestCase):
       self._set_sentinels(root)
       with self.assertRaises(AssertionError):
         self._check_sentinels(root)
-      object_saver = checkpointable_utils.CheckpointableSaver(root)
+      object_saver = checkpointable_utils.CheckpointableSaver(
+          graph_view.ObjectGraphView(root))
       self._set_sentinels(root)
       status = object_saver.restore(save_path)
       if context.executing_eagerly():
diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py
index dec23c50e8..dfc43ee78f 100644
--- a/tensorflow/python/training/saver_test.py
+++ b/tensorflow/python/training/saver_test.py
@@ -3012,13 +3012,13 @@ class CheckpointableCompatibilityTests(test.TestCase):
     save_graph = ops_lib.Graph()
     with save_graph.as_default(), self.session(graph=save_graph) as sess:
       root = self._initialized_model()
-      object_saver = checkpointable_utils.CheckpointableSaver(root)
+      object_saver = checkpointable_utils.Checkpoint(root=root)
       save_path = object_saver.save(file_prefix=checkpoint_prefix)
 
       # An incompatible object-based checkpoint to check error messages
       var = resource_variable_ops.ResourceVariable(1., name="a")
       self.evaluate(var.initializer)
-      second_saver = checkpointable_utils.CheckpointableSaver(var)
+      second_saver = checkpointable_utils.Checkpoint(v=var)
       second_path = second_saver.save(file_prefix=os.path.join(
           checkpoint_directory, "second"))
 
@@ -3046,7 +3046,7 @@ class CheckpointableCompatibilityTests(test.TestCase):
     save_graph = ops_lib.Graph()
     with save_graph.as_default(), self.session(graph=save_graph):
       root = self._initialized_model()
-      object_saver = checkpointable_utils.CheckpointableSaver(root)
+      object_saver = checkpointable_utils.Checkpoint(root=root)
       save_path = object_saver.save(file_prefix=checkpoint_prefix)
 
     with context.eager_mode():
-- 
GitLab


From b869edccadcf4ecdabd1659320679e8be8b6b4d2 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Mon, 4 Feb 2019 11:08:36 -0800
Subject: [PATCH 1852/2345] [XLA:Python] Refactor xla_client, adding a Backend
 abstraction that is implemented by both Local XLA and XRT.

Add support for serializing Python Shape objects to protocol buffers.

PiperOrigin-RevId: 232329732
---
 tensorflow/compiler/xla/python/xla_client.py | 199 ++++++++++++++-----
 1 file changed, 144 insertions(+), 55 deletions(-)

diff --git a/tensorflow/compiler/xla/python/xla_client.py b/tensorflow/compiler/xla/python/xla_client.py
index 4433bcb1b3..34b5079722 100644
--- a/tensorflow/compiler/xla/python/xla_client.py
+++ b/tensorflow/compiler/xla/python/xla_client.py
@@ -12,12 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""An in-process, local XLA client in Python, supporting AOT compilation."""
+"""An XLA client in Python, supporting AOT compilation."""
 
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import abc
 import collections
 import enum  # pylint: disable=g-bad-import-order
 import inspect
@@ -49,13 +50,123 @@ _OP_METADATA_FIELDS = [
 OpMetadata = collections.namedtuple('OpMetadata', _OP_METADATA_FIELDS)
 
 
+@six.add_metaclass(abc.ABCMeta)
+class Backend(object):
+  """Abstract base class for XLA backends."""
+
+  @abc.abstractmethod
+  def buffer_from_pyval(self, pyval, device=0):
+    """Allocates a fresh buffer and populates it with `pyval`."""
+
+  @abc.abstractmethod
+  def delete_buffer(self, c_buffer):
+    """Deletes buffer `c_buffer`."""
+
+  @abc.abstractmethod
+  def destructure_tuple(self, c_buffer):
+    """Destructures a tuple buffer into a sequence of buffers."""
+
+  @abc.abstractmethod
+  def compile(self, computation, argument_shapes, compile_options):
+    """Compiles a computation. Returns an executable."""
+
+  @abc.abstractmethod
+  def delete_executable(self, executable):
+    """Deletes an executable."""
+
+  @abc.abstractmethod
+  def execute(self, executable, args):
+    """Runs an executable without replication."""
+
+  @abc.abstractmethod
+  def execute_replicated(self, executable, per_replica_args):
+    """Runs an executable in a replicated manner."""
+
+
+class XlaLocalBackend(Backend):
+  """XLA backend implemented using the in-process xla::LocalClient API."""
+
+  def buffer_from_pyval(self, pyval, device=0):
+    return c_api.LocalShapedBuffer.FromLiteral(pyval, None, device)
+
+  def delete_buffer(self, c_buffer):
+    c_api.DeleteLocalShapedBuffer(c_buffer)
+
+  def destructure_tuple(self, c_buffer):
+    result = c_api.DestructureLocalShapedBufferTuple(c_buffer)
+    return [result.Release(i) for i in xrange(result.size())]
+
+  def compile(self, c_computation, argument_shapes, compile_options):
+    return c_computation.Compile(argument_shapes, compile_options)
+
+  def delete_executable(self, executable):
+    assert isinstance(executable, c_api.CompiledLocalComputation)
+    c_api.DeleteCompiledLocalComputation(executable)
+
+  def execute(self, executable, args):
+    return executable.Execute(args)
+
+  def execute_replicated(self, executable, per_replica_args):
+    output_buffer_tup = executable.ExecutePerReplica(per_replica_args)
+    size = output_buffer_tup.size()
+    return [output_buffer_tup.Release(i) for i in xrange(size)]
+
+
+class XrtBackend(Backend):
+  """XLA backend implemented using XRT."""
+
+  def __init__(self, target):
+    self.target = target
+
+  def buffer_from_pyval(self, pyval, device=0):
+    if device != 0:
+      raise NotImplementedError(
+          'Multi-replica execution is not yet supported via the XRT backend.')
+    return c_api.XrtAllocation.FromLiteral(pyval,
+                                           _maybe_encode_string(self.target))
+
+  def delete_buffer(self, c_buffer):
+    c_api.DeleteXrtAllocation(c_buffer)
+
+  def destructure_tuple(self, c_buffer):
+    result = c_api.DestructureXrtAllocationTuple(
+        c_buffer, _maybe_encode_string(self.target))
+    return [result.Release(i) for i in xrange(result.size())]
+
+  def compile(self, c_computation, argument_shapes, compile_options):
+    return c_computation.CompileForXrt(argument_shapes,
+                                       _maybe_encode_string(self.target))
+
+  def delete_executable(self, executable):
+    assert isinstance(executable, c_api.CompiledXrtComputation)
+    c_api.DeleteCompiledXrtComputation(executable)
+
+  def execute(self, executable, args):
+    return executable.Execute(args)
+
+  def execute_replicated(self, executable, per_replica_args):
+    if len(per_replica_args) != 1:
+      raise NotImplementedError(
+          'Multi-replica execution is not yet supported via the XRT backend.')
+    return [executable.Execute(per_replica_args[0])]
+
+
+XLA_LOCAL_BACKEND = XlaLocalBackend()
+
+
 class BackendType(enum.Enum):
   XLA_LOCAL = 1
   XRT = 2
 
 
-BackendSpec = collections.namedtuple('Backend', ('backend_type', 'target'))
-XLA_LOCAL_BACKEND = BackendSpec(BackendType.XLA_LOCAL, 'local')
+def BackendSpec(backend, target):
+  """Compatibility wrapper to support older clients. Do not use in new code."""
+  if backend == BackendType.XLA_LOCAL:
+    return XLA_LOCAL_BACKEND
+  elif backend == BackendType.XRT:
+    return XrtBackend(target)
+  else:
+    raise ValueError('Unknown backend {}'.format(backend))
 
 
 def OpMetadataToProto(pyobj):
@@ -227,10 +338,6 @@ class LocalBuffer(object):
     self.c_buffer = c_buffer
     self._backend = backend
     self._replica = replica
-    if backend.backend_type == BackendType.XRT:
-      self._delete = c_api.DeleteXrtAllocation
-    else:
-      self._delete = c_api.DeleteLocalShapedBuffer
 
   @staticmethod
   def from_pyval(pyval, replica=0, backend=XLA_LOCAL_BACKEND):
@@ -241,14 +348,7 @@ class LocalBuffer(object):
       raise ValueError(
           'Attempt to place buffer on replica {} when the replica count is {}'
           .format(replica, num_replicas))
-    if backend.backend_type == BackendType.XRT:
-      if replica != 0:
-        raise NotImplementedError(
-            'Multi-replica execution is not yet supported via the XRT backend.')
-      cbuf = c_api.XrtAllocation.FromLiteral(
-          pyval, _maybe_encode_string(backend.target))
-    else:
-      cbuf = c_api.LocalShapedBuffer.FromLiteral(pyval, None, replica)
+    cbuf = backend.buffer_from_pyval(pyval, replica)
     return LocalBuffer(cbuf, backend, replica)
 
   def to_py(self):
@@ -262,24 +362,17 @@ class LocalBuffer(object):
 
   def delete(self):
     if self.c_buffer is not None:
-      self._delete(self.c_buffer)
+      self._backend.delete_buffer(self.c_buffer)
       self.c_buffer = None
 
   def destructure(self):
     """Assuming a tuple buffer, unpack it into constituent tuple elements."""
     assert self.c_buffer is not None
-    if self._backend.backend_type == BackendType.XRT:
-      result = c_api.DestructureXrtAllocationTuple(
-          self.c_buffer, _maybe_encode_string(self._backend.target))
-    else:
-      result = c_api.DestructureLocalShapedBufferTuple(self.c_buffer)
+    result = self._backend.destructure_tuple(self.c_buffer)
     self.delete()
-    size = result.size()
-    destructured = tuple(
-        LocalBuffer(
-            result.Release(i), replica=self._replica, backend=self._backend)
-        for i in xrange(size))
-    return destructured
+    return tuple(
+        LocalBuffer(sub_buffer, replica=self._replica, backend=self._backend)
+        for sub_buffer in result)
 
   def is_deleted(self):
     return self.c_buffer is None
@@ -428,6 +521,20 @@ class Shape(object):
     updated._check_minor_to_major()  # pylint: disable=protected-access
     return updated
 
+  def serialize(self, proto):
+    """Serializes 'shape' into proto."""
+    if self.is_tuple():
+      proto.element_type = xla_data_pb2.TUPLE
+      for shape in self.tuple_shapes():
+        shape.serialize(proto.tuple_shapes.add())
+    else:
+      proto.element_type = dtype_to_etype(self.element_type())
+      proto.dimensions.extend(self.dimensions())
+      proto.is_dynamic_dimension.extend([False for _ in self.dimensions()])
+      if self.minor_to_major():
+        proto.layout.format = xla_data_pb2.DENSE
+        proto.layout.minor_to_major.extend(self.minor_to_major())
+
 
 def _wrap_shape(shape_info):
   dtype, dims = shape_info
@@ -509,18 +616,6 @@ class LocalComputation(object):
     self._backend = backend
     self._is_compiled = is_compiled
 
-    # Ensure a reference to C-based destructor for use in __del__.
-    if is_compiled:
-      if backend.backend_type == BackendType.XRT:
-        assert isinstance(c_computation, c_api.CompiledXrtComputation)
-        self._delete = c_api.DeleteCompiledXrtComputation
-      else:
-        assert isinstance(c_computation, c_api.CompiledLocalComputation)
-        self._delete = c_api.DeleteCompiledLocalComputation
-    else:
-      assert isinstance(c_computation, c_api.LocalComputation)
-      self._delete = c_api.DeleteLocalComputation
-
   @property
   def computation(self):
     if self._is_compiled:
@@ -574,11 +669,8 @@ class LocalComputation(object):
 
     compile_options = compile_options or CompileOptions()
     compile_options.result_shape = result_shape
-    if self._backend.backend_type == BackendType.XRT:
-      c = self.computation.CompileForXrt(
-          argument_shapes, _maybe_encode_string(self._backend.target))
-    else:
-      c = self.computation.Compile(argument_shapes, compile_options)
+    c = self._backend.compile(self.computation, argument_shapes,
+                              compile_options)
     return LocalComputation(c, is_compiled=True, backend=self._backend)
 
   def CompileWithExampleArguments(self,
@@ -598,7 +690,7 @@ class LocalComputation(object):
     if check_for_deleted_args and any(arg.is_deleted() for arg in arguments):
       raise ValueError('Executing with deleted local buffer argument')
     raw_args = [arg.c_buffer for arg in arguments]
-    output_buffer = self._c_computation.Execute(raw_args)
+    output_buffer = self._backend.execute(self._c_computation, raw_args)
     return LocalBuffer(output_buffer, backend=self._backend, replica=0)
 
   def ExecutePerReplica(self, arguments=None):
@@ -636,15 +728,7 @@ class LocalComputation(object):
     ]
 
     # Execute
-    if self._backend.backend_type == BackendType.XRT:
-      if len(stripped_args) > 1:
-        raise NotImplementedError(
-            'Multi-replica execution is not yet supported via the XRT backend.')
-      output_buffers = [self._c_computation.Execute(stripped_args[0])]
-    else:
-      output_buffer_tup = self._c_computation.ExecutePerReplica(stripped_args)
-      size = output_buffer_tup.size()
-      output_buffers = [output_buffer_tup.Release(i) for i in xrange(size)]
+    output_buffers = self._backend.execute_replicated(stripped_args)
 
     # Wrap output handles in LocalBuffer instances
     return tuple(
@@ -672,7 +756,12 @@ class LocalComputation(object):
     return [out.to_py() for out in self.ExecutePerReplica(arguments)]
 
   def __del__(self):
-    self._delete(self._c_computation)
+    # Ensure a reference to C-based destructor for use in __del__.
+    if self._is_compiled:
+      self._backend.delete_executable(self._c_computation)
+    else:
+      assert isinstance(self._c_computation, c_api.LocalComputation)
+      c_api.DeleteLocalComputation(self._c_computation)
 
 
 def _make_replica_group_proto(replica_group):
-- 
GitLab


From 69b6fdd6da4fbfd64219d548cbdd951bb33cfc8d Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Mon, 4 Feb 2019 11:08:51 -0800
Subject: [PATCH 1853/2345] Also define the sorting axis in
 testMutableHashTableOfTensors.

without that, numpy.sort does not seem to have a deterministic order.

PiperOrigin-RevId: 232329777
---
 tensorflow/python/kernel_tests/lookup_ops_test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/kernel_tests/lookup_ops_test.py b/tensorflow/python/kernel_tests/lookup_ops_test.py
index 652260559f..0b78115eeb 100644
--- a/tensorflow/python/kernel_tests/lookup_ops_test.py
+++ b/tensorflow/python/kernel_tests/lookup_ops_test.py
@@ -1788,9 +1788,9 @@ class MutableHashTableOpTest(test.TestCase):
       exported_keys, exported_values = table.export()
       # exported data is in the order of the internal map, i.e. undefined
       sorted_keys = np.sort(self.evaluate(exported_keys))
-      sorted_values = np.sort(self.evaluate(exported_values))
+      sorted_values = np.sort(self.evaluate(exported_values), axis=0)
       self.assertAllEqual([b"brain", b"salad", b"surgery"], sorted_keys)
-      sorted_expected_values = np.sort([[4, 5], [2, 3], [0, 1]])
+      sorted_expected_values = np.sort([[4, 5], [2, 3], [0, 1]], axis=0)
       self.assertAllEqual(sorted_expected_values, sorted_values)
 
   def testMutableHashTableExportInsert(self):
-- 
GitLab


From 187f876271debd58b20c4eb489c100b7f8d571ed Mon Sep 17 00:00:00 2001
From: Ayush Dubey <ayushd@google.com>
Date: Mon, 4 Feb 2019 11:12:32 -0800
Subject: [PATCH 1854/2345] Add a flag to switch CollectiveReduce
 implementation to use NCCL.

This flag enables an experimental feature that needs more testing.

PiperOrigin-RevId: 232330486
---
 .../core/common_runtime/collective_executor_mgr_test.cc    | 2 +-
 .../core/common_runtime/collective_param_resolver_local.cc | 7 +++----
 .../core/common_runtime/collective_param_resolver_local.h  | 4 +++-
 .../common_runtime/collective_param_resolver_local_test.cc | 4 ++--
 .../core/common_runtime/collective_rma_local_test.cc       | 4 ++--
 tensorflow/core/common_runtime/direct_session.cc           | 5 ++++-
 tensorflow/core/common_runtime/eager/context.cc            | 3 ++-
 .../collective_param_resolver_distributed.cc               | 2 +-
 tensorflow/core/distributed_runtime/master_session.cc      | 2 ++
 tensorflow/core/protobuf/config.proto                      | 4 ++++
 .../golden/v1/tensorflow.-config-proto.-experimental.pbtxt | 6 ++++++
 .../tools/api/golden/v1/tensorflow.-config-proto.pbtxt     | 6 ++++++
 12 files changed, 36 insertions(+), 13 deletions(-)

diff --git a/tensorflow/core/common_runtime/collective_executor_mgr_test.cc b/tensorflow/core/common_runtime/collective_executor_mgr_test.cc
index f3d86aa633..80d4166aa6 100644
--- a/tensorflow/core/common_runtime/collective_executor_mgr_test.cc
+++ b/tensorflow/core/common_runtime/collective_executor_mgr_test.cc
@@ -44,7 +44,7 @@ class CollectiveExecutorMgrTest : public ::testing::Test {
     std::unique_ptr<DeviceResolverInterface> drl(
         new DeviceResolverLocal(device_mgr_.get()));
     std::unique_ptr<ParamResolverInterface> prl(
-        new CollectiveParamResolverLocal(device_mgr_.get(), drl.get(),
+        new CollectiveParamResolverLocal(cp, device_mgr_.get(), drl.get(),
                                          task_name));
     cme_.reset(new CollectiveExecutorMgr(cp, device_mgr_.get(), std::move(drl),
                                          std::move(prl)));
diff --git a/tensorflow/core/common_runtime/collective_param_resolver_local.cc b/tensorflow/core/common_runtime/collective_param_resolver_local.cc
index 8907f6d56a..e286ba2a37 100644
--- a/tensorflow/core/common_runtime/collective_param_resolver_local.cc
+++ b/tensorflow/core/common_runtime/collective_param_resolver_local.cc
@@ -38,9 +38,9 @@ void CollectiveParamResolverLocal::InstanceRec::WaitForOutMu(mutex_lock& lock) {
 }
 
 CollectiveParamResolverLocal::CollectiveParamResolverLocal(
-    const DeviceMgr* dev_mgr, DeviceResolverInterface* dev_resolver,
-    const string& task_name)
-    : nccl_(false),  // (b/111897089): turn on NCCL collectives.
+    const ConfigProto& config, const DeviceMgr* dev_mgr,
+    DeviceResolverInterface* dev_resolver, const string& task_name)
+    : nccl_(config.experimental().collective_nccl()),
       dev_mgr_(dev_mgr),
       dev_resolver_(dev_resolver),
       task_name_(task_name) {}
@@ -144,7 +144,6 @@ void CollectiveParamResolverLocal::CompleteGroupLocal(
 }
 
 namespace {
-
 struct DevRec {
   string task;
   string device;
diff --git a/tensorflow/core/common_runtime/collective_param_resolver_local.h b/tensorflow/core/common_runtime/collective_param_resolver_local.h
index fd408e4ef3..08e2f338f3 100644
--- a/tensorflow/core/common_runtime/collective_param_resolver_local.h
+++ b/tensorflow/core/common_runtime/collective_param_resolver_local.h
@@ -23,6 +23,7 @@ limitations under the License.
 
 #include "tensorflow/core/framework/collective.h"
 #include "tensorflow/core/lib/gtl/flatmap.h"
+#include "tensorflow/core/protobuf/config.pb.h"
 
 namespace tensorflow {
 class CompleteGroupRequest;
@@ -36,7 +37,8 @@ class DeviceMgr;
 // group leader for param resolution in a multi-task context.
 class CollectiveParamResolverLocal : public ParamResolverInterface {
  public:
-  CollectiveParamResolverLocal(const DeviceMgr* dev_mgr,
+  CollectiveParamResolverLocal(const ConfigProto& config,
+                               const DeviceMgr* dev_mgr,
                                DeviceResolverInterface* dev_resolver,
                                const string& task_name);
 
diff --git a/tensorflow/core/common_runtime/collective_param_resolver_local_test.cc b/tensorflow/core/common_runtime/collective_param_resolver_local_test.cc
index 94d889c40d..ab91eaba2f 100644
--- a/tensorflow/core/common_runtime/collective_param_resolver_local_test.cc
+++ b/tensorflow/core/common_runtime/collective_param_resolver_local_test.cc
@@ -41,8 +41,8 @@ class CollectiveParamResolverLocalTest : public ::testing::Test {
     TF_CHECK_OK(DeviceFactory::AddDevices(options, task_name, &devices));
     device_mgr_.reset(new DeviceMgr(std::move(devices)));
     drl_.reset(new DeviceResolverLocal(device_mgr_.get()));
-    prl_.reset(new CollectiveParamResolverLocal(device_mgr_.get(), drl_.get(),
-                                                task_name));
+    prl_.reset(new CollectiveParamResolverLocal(cp, device_mgr_.get(),
+                                                drl_.get(), task_name));
   }
 
   void RunCompleteDefaultRanking(
diff --git a/tensorflow/core/common_runtime/collective_rma_local_test.cc b/tensorflow/core/common_runtime/collective_rma_local_test.cc
index 4263f3a4ad..6979f4e63b 100644
--- a/tensorflow/core/common_runtime/collective_rma_local_test.cc
+++ b/tensorflow/core/common_runtime/collective_rma_local_test.cc
@@ -46,8 +46,8 @@ class CollectiveRemoteAccessLocalTest : public ::testing::Test {
     TF_CHECK_OK(DeviceFactory::AddDevices(options, kTaskName, &devices));
     device_mgr_.reset(new DeviceMgr(std::move(devices)));
     drl_.reset(new DeviceResolverLocal(device_mgr_.get()));
-    prl_.reset(new CollectiveParamResolverLocal(device_mgr_.get(), drl_.get(),
-                                                kTaskName));
+    prl_.reset(new CollectiveParamResolverLocal(cp, device_mgr_.get(),
+                                                drl_.get(), kTaskName));
     rma_.reset(new CollectiveRemoteAccessLocal(device_mgr_.get(), drl_.get(),
                                                kStepId));
   }
diff --git a/tensorflow/core/common_runtime/direct_session.cc b/tensorflow/core/common_runtime/direct_session.cc
index 80b62f273c..40a1ffc42d 100644
--- a/tensorflow/core/common_runtime/direct_session.cc
+++ b/tensorflow/core/common_runtime/direct_session.cc
@@ -501,7 +501,8 @@ Status DirectSession::RunInternal(int64 step_id, const RunOptions& run_options,
       std::unique_ptr<DeviceResolverInterface> drl(
           new DeviceResolverLocal(device_mgr_.get()));
       std::unique_ptr<ParamResolverInterface> cprl(
-          new CollectiveParamResolverLocal(device_mgr_.get(), drl.get(),
+          new CollectiveParamResolverLocal(options_.config, device_mgr_.get(),
+                                           drl.get(),
                                            "/job:localhost/replica:0/task:0"));
       collective_executor_mgr_.reset(new CollectiveExecutorMgr(
           options_.config, device_mgr_.get(), std::move(drl), std::move(cprl)));
@@ -1194,6 +1195,8 @@ Status DirectSession::CreateExecutors(
   if (options_.config.experimental()
           .collective_deterministic_sequential_execution()) {
     options.collective_order = GraphCollectiveOrder::kEdges;
+  } else if (options_.config.experimental().collective_nccl()) {
+    options.collective_order = GraphCollectiveOrder::kAttrs;
   }
 
   std::unique_ptr<FunctionInfo> func_info(new FunctionInfo);
diff --git a/tensorflow/core/common_runtime/eager/context.cc b/tensorflow/core/common_runtime/eager/context.cc
index 8068905da9..3a9d1e02be 100644
--- a/tensorflow/core/common_runtime/eager/context.cc
+++ b/tensorflow/core/common_runtime/eager/context.cc
@@ -81,7 +81,8 @@ EagerContext::EagerContext(const SessionOptions& opts,
   std::unique_ptr<DeviceResolverInterface> drl(
       new DeviceResolverLocal(local_device_mgr()));
   std::unique_ptr<ParamResolverInterface> cprl(new CollectiveParamResolverLocal(
-      local_device_mgr(), drl.get(), "/job:localhost/replica:0/task:0"));
+      opts.config, local_device_mgr(), drl.get(),
+      "/job:localhost/replica:0/task:0"));
   collective_executor_mgr_.reset(new CollectiveExecutorMgr(
       opts.config, local_device_mgr(), std::move(drl), std::move(cprl)));
 }
diff --git a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc
index 9f94a24fcd..df735b1c39 100644
--- a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc
+++ b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc
@@ -82,7 +82,7 @@ CollectiveParamResolverDistributed::CollectiveParamResolverDistributed(
     const ConfigProto& config, const DeviceMgr* dev_mgr,
     DeviceResolverDistributed* dev_resolver, WorkerCacheInterface* worker_cache,
     const string& task_name)
-    : CollectiveParamResolverLocal(dev_mgr, dev_resolver, task_name),
+    : CollectiveParamResolverLocal(config, dev_mgr, dev_resolver, task_name),
       worker_cache_(worker_cache),
       group_leader_(task_name == config.experimental().collective_group_leader()
                         ? ""
diff --git a/tensorflow/core/distributed_runtime/master_session.cc b/tensorflow/core/distributed_runtime/master_session.cc
index 48b72fb948..2f14967656 100644
--- a/tensorflow/core/distributed_runtime/master_session.cc
+++ b/tensorflow/core/distributed_runtime/master_session.cc
@@ -1103,6 +1103,8 @@ void BuildBuildGraphOptions(const RunStepRequestWrapper& req,
       req.options().experimental().collective_graph_key();
   if (config.experimental().collective_deterministic_sequential_execution()) {
     opts->collective_order = GraphCollectiveOrder::kEdges;
+  } else if (config.experimental().collective_nccl()) {
+    opts->collective_order = GraphCollectiveOrder::kAttrs;
   }
 }
 
diff --git a/tensorflow/core/protobuf/config.proto b/tensorflow/core/protobuf/config.proto
index a2cc1bc935..2f3dbad661 100644
--- a/tensorflow/core/protobuf/config.proto
+++ b/tensorflow/core/protobuf/config.proto
@@ -429,6 +429,10 @@ message ConfigProto {
     // If true, make collective op execution order sequential and deterministic
     // for potentially concurrent collective instances.
     bool collective_deterministic_sequential_execution = 6;
+
+    // If true, use NCCL for CollectiveOps.  This feature is highly
+    // experimental.
+    bool collective_nccl = 7;
   };
 
   Experimental experimental = 16;
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.-config-proto.-experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.-config-proto.-experimental.pbtxt
index 078f1028fd..2e8ece122c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.-config-proto.-experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.-config-proto.-experimental.pbtxt
@@ -32,6 +32,12 @@ tf_proto {
       label: LABEL_OPTIONAL
       type: TYPE_BOOL
     }
+    field {
+      name: "collective_nccl"
+      number: 7
+      label: LABEL_OPTIONAL
+      type: TYPE_BOOL
+    }
     reserved_range {
       start: 2
       end: 3
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.-config-proto.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.-config-proto.pbtxt
index d2ee0c4db6..9c7de2c571 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.-config-proto.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.-config-proto.pbtxt
@@ -155,6 +155,12 @@ tf_proto {
         label: LABEL_OPTIONAL
         type: TYPE_BOOL
       }
+      field {
+        name: "collective_nccl"
+        number: 7
+        label: LABEL_OPTIONAL
+        type: TYPE_BOOL
+      }
       reserved_range {
         start: 2
         end: 3
-- 
GitLab


From da5513d02ff833f85e9f466d38caa24e511c4600 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 4 Feb 2019 11:43:16 -0800
Subject: [PATCH 1855/2345] Support compiling Switch/Merge by constant folding
 Switch predicates

This CL adds limited support for clustering Switch and Merge nodes.  We compiled
Switch nodes by versioning the computation on the value of the Switch condition.
This lets us statically track the deadness through the cluster and allows us to
resolve deadness entirely in the TF->XLA graph compiler without touching XLA.

This CL is organized as follows:

 * jit/mark_for_compilation_pass is taught to not auto-cluster Switch and Merge
   even though these operations have XlaOpKernels.  We also make a minor fix
   without which we would always reject Merge nodes, even when it was assigned
   to an XLA_* device.

 * jit/partially_decluster_pass gets a bugfix without which we would decluster
   nodes that were assigned to XLA_* devices to avoid Device->Host copies.

 * tf2xla/graph_compiler and tf2xla/kernels/control_flow_ops contains the main
   part of the change where we introduce XlaOpKernels for Merge and Switch, and
   teach the TF->XLA graph compiler to track deadness.

PiperOrigin-RevId: 232336750
---
 .../compiler/jit/mark_for_compilation_pass.cc | 20 ++++-
 .../compiler/jit/partially_decluster_pass.cc  | 56 +++++++-------
 .../jit/partially_decluster_pass_test.cc      | 43 ++++++++++-
 tensorflow/compiler/tests/BUILD               | 16 ++++
 .../compiler/tests/control_flow_ops_test.py   | 56 ++++++++++++++
 tensorflow/compiler/tf2xla/graph_compiler.cc  | 68 +++++++++++------
 tensorflow/compiler/tf2xla/kernels/BUILD      |  1 +
 .../tf2xla/kernels/control_flow_ops.cc        | 74 +++++++++++++++++++
 8 files changed, 285 insertions(+), 49 deletions(-)
 create mode 100644 tensorflow/compiler/tests/control_flow_ops_test.py
 create mode 100644 tensorflow/compiler/tf2xla/kernels/control_flow_ops.cc

diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass.cc b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
index 20c2cd7e05..0ad85087cd 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
@@ -74,6 +74,13 @@ struct OperationFilter {
   // have dummy XLA implementations.
   bool allow_dummy_ops;
 
+  // Whether Switch/Merge ops are allowed.  We avoid auto-clustering these ops
+  // for now since they can result in mismatching deadness in the output
+  // produced by an XLA cluster, something the TF executor does not allow today.
+  // However, we plan to auto-cluster these in some limited cases in the near
+  // future (b/123096195).
+  bool allow_switch_merge_ops;
+
   // Whether ops that produce or consume DT_VARIANT values are allowed.  We
   // don't auto-cluster these ops because we don't yet support live-in or
   // live-out DT_VARIANT values.
@@ -255,6 +262,10 @@ bool IsCompilableCall(const NodeDef& call_def,
     if (!op_filter.allow_control_trigger && node->IsControlTrigger()) {
       return false;
     }
+    if (!op_filter.allow_switch_merge_ops &&
+        (node->IsSwitch() || node->IsMerge())) {
+      return false;
+    }
     if (!op_filter.allow_dummy_ops && IsDummyImplOp(node->type_string())) {
       return false;
     }
@@ -493,6 +504,7 @@ Status FindCompilationCandidates(
     op_filter.allow_resource_ops = registration->compile_resource_ops;
     op_filter.allow_stateful_rng_ops = always_auto_cluster;
     op_filter.allow_control_trigger = always_auto_cluster;
+    op_filter.allow_switch_merge_ops = always_auto_cluster;
     op_filter.allow_dummy_ops = always_auto_cluster;
     op_filter.allow_ops_producing_or_consuming_variant = always_auto_cluster;
 
@@ -513,6 +525,11 @@ Status FindCompilationCandidates(
       VLOG(2) << "Rejecting " << node->name() << ": is a control trigger op";
       continue;
     }
+    if (!op_filter.allow_switch_merge_ops &&
+        (node->IsSwitch() || node->IsMerge())) {
+      VLOG(2) << "Rejecting " << node->name() << ": is a switch or a merge op";
+      continue;
+    }
     if (!op_filter.allow_dummy_ops && IsDummyImplOp(node->type_string())) {
       VLOG(2) << "Rejecting " << node->name() << ": dummy op ("
               << node->type_string() << ")";
@@ -659,6 +676,7 @@ bool IsCompilable(FunctionLibraryRuntime* flr, const NodeDef& ndef) {
   op_filter.allow_resource_ops = true;
   op_filter.allow_stateful_rng_ops = true;
   op_filter.allow_control_trigger = true;
+  op_filter.allow_switch_merge_ops = true;
   op_filter.allow_dummy_ops = true;
   op_filter.allow_ops_producing_or_consuming_variant = true;
 
@@ -733,7 +751,7 @@ Status MarkForCompilationPass::Run(
     // deadness semantics of these nodes correctly and auto-clustering these
     // nodes can cause deadness to propagate to nodes that should be live.
     if (!deadness_analysis_disabled) {
-      if (node->IsMerge() ||
+      if (!node->IsMerge() &&
           deadness->HasInputsWithMismatchingDeadness(*node)) {
         VLOG(2) << "Rejecting " << node->name() << ": mismatching deadness.";
         return false;
diff --git a/tensorflow/compiler/jit/partially_decluster_pass.cc b/tensorflow/compiler/jit/partially_decluster_pass.cc
index e1fd2aaee2..dd3a122940 100644
--- a/tensorflow/compiler/jit/partially_decluster_pass.cc
+++ b/tensorflow/compiler/jit/partially_decluster_pass.cc
@@ -29,6 +29,31 @@ namespace {
 
 bool NotBackedge(const Edge& edge) { return !edge.src()->IsNextIteration(); }
 
+bool IsMustCompileDevice(const DeviceType& device_type) {
+  const XlaOpRegistry::DeviceRegistration* registration;
+  if (XlaOpRegistry::GetCompilationDevice(device_type.type(), &registration)) {
+    return registration->autoclustering_policy ==
+           XlaOpRegistry::AutoclusteringPolicy::kAlways;
+  }
+
+  return false;
+}
+
+Status MustCompileNode(const Node* n, bool* must_compile) {
+  DeviceType device_type("");
+  TF_RETURN_IF_ERROR(
+      DeviceToDeviceType(n->assigned_device_name(), &device_type));
+
+  if (IsMustCompileDevice(device_type)) {
+    *must_compile = true;
+    return Status::OK();
+  }
+
+  // We must compile `n` if it does not have a TensorFlow kernel.
+  *must_compile = !FindKernelDef(device_type, n->def(), nullptr, nullptr).ok();
+  return Status::OK();
+}
+
 namespace reduce_device_to_host_copies {
 Status FindNodesToDecluster(const Graph& graph,
                             absl::flat_hash_set<Node*>* result,
@@ -51,6 +76,12 @@ Status FindNodesToDecluster(const Graph& graph,
       continue;
     }
 
+    bool must_compile;
+    TF_RETURN_IF_ERROR(MustCompileNode(n, &must_compile));
+    if (must_compile) {
+      continue;
+    }
+
     DeviceType device_type("");
     TF_RETURN_IF_ERROR(
         DeviceToDeviceType(n->assigned_device_name(), &device_type));
@@ -220,31 +251,6 @@ bool IsIntraClusterEdge(const Edge& edge) {
   return src_cluster_name.has_value() && src_cluster_name == dst_cluster_name;
 }
 
-bool IsMustCompileDevice(const DeviceType& device_type) {
-  const XlaOpRegistry::DeviceRegistration* registration;
-  if (XlaOpRegistry::GetCompilationDevice(device_type.type(), &registration)) {
-    return registration->autoclustering_policy ==
-           XlaOpRegistry::AutoclusteringPolicy::kAlways;
-  }
-
-  return false;
-}
-
-Status MustCompileNode(const Node* n, bool* must_compile) {
-  DeviceType device_type("");
-  TF_RETURN_IF_ERROR(
-      DeviceToDeviceType(n->assigned_device_name(), &device_type));
-
-  if (IsMustCompileDevice(device_type)) {
-    *must_compile = true;
-    return Status::OK();
-  }
-
-  // We must compile `n` if it does not have a TensorFlow kernel.
-  *must_compile = !FindKernelDef(device_type, n->def(), nullptr, nullptr).ok();
-  return Status::OK();
-}
-
 // Declusters nodes to reduce the number of times we think we need to recompile
 // a TensorFlow graph.
 //
diff --git a/tensorflow/compiler/jit/partially_decluster_pass_test.cc b/tensorflow/compiler/jit/partially_decluster_pass_test.cc
index 1d81a8f4fc..a6035da8bd 100644
--- a/tensorflow/compiler/jit/partially_decluster_pass_test.cc
+++ b/tensorflow/compiler/jit/partially_decluster_pass_test.cc
@@ -74,6 +74,12 @@ REGISTER_KERNEL_BUILDER(Name("FakeBinary")
                             .HostMemory("host_out"),
                         FakeBinaryOp);
 
+REGISTER_KERNEL_BUILDER(Name("FakeBinary")
+                            .Device(DEVICE_XLA_CPU)
+                            .HostMemory("host_in")
+                            .HostMemory("host_out"),
+                        FakeBinaryOp);
+
 REGISTER_KERNEL_BUILDER(
     Name("FakeResourceUpdate").Device(DEVICE_CPU).HostMemory("something_else"),
     FakeResourceUpdateOp);
@@ -367,7 +373,7 @@ TEST(PartiallyDeclusterPassTest, EdgeAcrossDifferentClusters) {
   EXPECT_EQ(GetXlaClusterForNode(*n), "cluster_1");
 }
 
-TEST(PartiallyDeclusterPassTest, DontDeclusterXlaDeviceOps) {
+TEST(PartiallyDeclusterPassTest, DontDeclusterXlaDeviceOps_A) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output shape_a = ops::Placeholder(s.WithOpName("shape_a"), DT_INT32,
                                     ops::Placeholder::Attrs{});
@@ -401,6 +407,41 @@ TEST(PartiallyDeclusterPassTest, DontDeclusterXlaDeviceOps) {
   EXPECT_EQ(GetXlaClusterForNode(*n), "cluster_0");
 }
 
+TEST(PartiallyDeclusterPassTest, DontDeclusterXlaDeviceOps_B) {
+  std::unique_ptr<Graph> graph(new Graph(OpRegistry::Global()));
+  {
+    GraphDefBuilder builder(GraphDefBuilder::kFailImmediately);
+    Node* input =
+        ops::SourceOp("FakeNullary", builder.opts().WithName("Input"));
+    Node* clustered_producer =
+        ops::BinaryOp("FakeBinary", input, input,
+                      builder.opts().WithName("ClusteredProducer"));
+    ops::BinaryOp("FakeBinary", clustered_producer, input,
+                  builder.opts().WithName("UnclusteredConsumer"));
+    Node* clustered_consumer =
+        ops::BinaryOp("FakeBinary", {clustered_producer, 1}, input,
+                      builder.opts().WithName("ClusteredConsumer"));
+    clustered_producer->AddAttr(kXlaClusterAttr, "cluster_0");
+    clustered_consumer->AddAttr(kXlaClusterAttr, "cluster_0");
+    TF_EXPECT_OK(GraphDefBuilderToGraph(builder, graph.get()));
+  }
+
+  // Scope::ToGraph loses the assigned device name since it goes through
+  // GraphDef/NodeDef which does not have a field for the assigned device name.
+  Node* n = FindNodeByName(*graph, "ClusteredProducer");
+  ASSERT_NE(n, nullptr);
+  n->set_assigned_device_name(
+      "/job:localhost/replica:0/task:0/device:XLA_CPU:0");
+
+  TF_ASSERT_OK(PartiallyDecluster(&graph));
+  std::vector<Node*> unclustered_consumer_inputs;
+  ASSERT_TRUE(GetInputsForNode(*graph, "UnclusteredConsumer",
+                               &unclustered_consumer_inputs));
+  ASSERT_EQ(unclustered_consumer_inputs.size(), 2);
+  EXPECT_EQ(unclustered_consumer_inputs[0]->name(), "ClusteredProducer");
+  EXPECT_EQ(GetXlaClusterForNode(*unclustered_consumer_inputs[0]), "cluster_0");
+}
+
 TEST(PartiallyDeclusterPassTest, DontDeclusterNonTensorFlowOps) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output dynamic_slice_operand =
diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD
index 9b6ca4092c..b6d4218f1d 100644
--- a/tensorflow/compiler/tests/BUILD
+++ b/tensorflow/compiler/tests/BUILD
@@ -1223,3 +1223,19 @@ tf_xla_py_test(
         "@absl_py//absl/testing:parameterized",
     ],
 )
+
+tf_xla_py_test(
+    name = "control_flow_ops_test",
+    size = "small",
+    srcs = ["control_flow_ops_test.py"],
+    deps = [
+        ":xla_test",
+        "//tensorflow/compiler/tf2xla/python:xla",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:control_flow_ops",
+        "//tensorflow/python:errors",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:platform_test",
+    ],
+)
diff --git a/tensorflow/compiler/tests/control_flow_ops_test.py b/tensorflow/compiler/tests/control_flow_ops_test.py
new file mode 100644
index 0000000000..c050290a48
--- /dev/null
+++ b/tensorflow/compiler/tests/control_flow_ops_test.py
@@ -0,0 +1,56 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test cases for Switch and Merge nodes."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np  # pylint: disable=unused-import
+
+from tensorflow.compiler.tests import xla_test
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.platform import googletest
+
+
+class SwitchMergeTest(xla_test.XLATestCase):
+
+  def testSwitchMerge(self):
+    with self.cached_session() as sess:
+      predicate = array_ops.placeholder(dtypes.bool)
+      with self.test_scope():
+        false_output, true_output = control_flow_ops.switch(
+            data=constant_op.constant(42.0), pred=predicate)
+        with ops.control_dependencies([array_ops.identity(false_output)]):
+          five = constant_op.constant(5.0)
+        with ops.control_dependencies([array_ops.identity(true_output)]):
+          ten = constant_op.constant(10.0)
+        result = control_flow_ops.merge([five, ten])
+
+    with_true = sess.run(result, {predicate: True})
+    self.assertEquals(with_true.output, 10.0)
+    self.assertEquals(with_true.value_index, 1)
+
+    with_false = sess.run(result, {predicate: False})
+    self.assertEquals(with_false.output, 5.0)
+    self.assertEquals(with_false.value_index, 0)
+
+
+if __name__ == "__main__":
+  googletest.main()
diff --git a/tensorflow/compiler/tf2xla/graph_compiler.cc b/tensorflow/compiler/tf2xla/graph_compiler.cc
index 5e4699bbb6..19413f6c72 100644
--- a/tensorflow/compiler/tf2xla/graph_compiler.cc
+++ b/tensorflow/compiler/tf2xla/graph_compiler.cc
@@ -119,6 +119,9 @@ Status GraphCompiler::Compile() {
   GetReversePostOrder(*graph_, &topo_sorted_nodes,
                       /*stable_comparator=*/NodeComparatorName());
 
+  std::vector<bool> control_output_dead;
+  control_output_dead.resize(graph_->num_node_ids());
+
   OpKernelContext::Params params;
   PartiallySetupParams(&params);
 
@@ -134,8 +137,8 @@ Status GraphCompiler::Compile() {
       return s;
     }
 
-    TF_RET_CHECK(!n->IsRecv() && !n->IsSend() && !n->IsSwitch())
-        << "Not supported node: " << n->DebugString();
+    TF_RET_CHECK(!n->IsRecv() && !n->IsSend())
+        << "Unsupported node: " << n->DebugString();
     params.op_kernel = op_kernel.get();
     absl::InlinedVector<AllocatorAttributes, 4> output_attr(n->num_outputs());
     params.output_attr_array = output_attr.data();
@@ -145,38 +148,59 @@ Status GraphCompiler::Compile() {
     tensor_inputs_.clear();
     tensor_inputs_.resize(n->num_inputs());
 
+    bool any_data_input_dead = false;
+    bool any_data_input_alive = false;
+    bool any_control_input_dead = false;
+
     // Set up inputs from outputs of previous nodes.
     for (auto* e : n->in_edges()) {
-      if (e->IsControlEdge()) continue;
+      if (e->IsControlEdge()) {
+        any_control_input_dead |= control_output_dead[e->src()->id()];
+        continue;
+      }
+
       const Node* src = e->src();
       TF_RET_CHECK(src->id() < output_registry.size());
       const NodeOutputs& src_outputs = output_registry[src->id()];
 
-      tensor_inputs_.at(e->dst_input()) = src_outputs.at(e->src_output());
-    }
+      const TensorValue& src_output = src_outputs.at(e->src_output());
+      tensor_inputs_.at(e->dst_input()) = src_output;
 
-    OpKernelContext op_context(&params, n->num_outputs());
-    VLOG(3) << "Translating " << params.op_kernel->name();
-    if (IsFunctional(n)) {
-      TF_RETURN_IF_ERROR(CompileFunctionalNode(n, &op_context));
-    } else {
-      device_->Compute(CHECK_NOTNULL(params.op_kernel), &op_context);
-      Status s = op_context.status();
-      if (!s.ok()) {
-        return AttachDef(s, n->def());
-      }
+      any_data_input_alive |= src_output.tensor != nullptr;
+      any_data_input_dead |= src_output.tensor == nullptr;
     }
 
-    // Set up outputs. Also check if outputs from the previous computation is
-    // valid.
+    bool node_is_dead = n->IsMerge()
+                            ? !any_data_input_alive
+                            : (any_data_input_dead || any_control_input_dead);
+
     NodeOutputs& outputs = output_registry[n->id()];
     outputs.resize(n->num_outputs());
-    for (int o = 0; o < n->num_outputs(); ++o) {
-      outputs[o] = op_context.release_output(o);
-      if (outputs[o].tensor == nullptr) {
-        return errors::Internal("Missing xla_context ", o, "-th output from ",
-                                FormatNodeForError(*n));
+
+    if (!node_is_dead) {
+      OpKernelContext op_context(&params, n->num_outputs());
+      VLOG(3) << "Translating " << params.op_kernel->name();
+      if (IsFunctional(n)) {
+        TF_RETURN_IF_ERROR(CompileFunctionalNode(n, &op_context));
+      } else {
+        device_->Compute(CHECK_NOTNULL(params.op_kernel), &op_context);
+        Status s = op_context.status();
+        if (!s.ok()) {
+          return AttachDef(s, n->def());
+        }
       }
+
+      // Set up outputs. Also check if outputs from the previous computation is
+      // valid.
+      for (int o = 0; o < n->num_outputs(); ++o) {
+        outputs[o] = op_context.release_output(o);
+        if (outputs[o].tensor == nullptr && !n->IsSwitch()) {
+          return errors::Internal("Missing xla_context ", o, "-th output from ",
+                                  FormatNodeForError(*n));
+        }
+      }
+    } else {
+      control_output_dead[n->id()] = true;
     }
   }
   return Status::OK();
diff --git a/tensorflow/compiler/tf2xla/kernels/BUILD b/tensorflow/compiler/tf2xla/kernels/BUILD
index 69353fe87d..e3c1ba48fd 100644
--- a/tensorflow/compiler/tf2xla/kernels/BUILD
+++ b/tensorflow/compiler/tf2xla/kernels/BUILD
@@ -25,6 +25,7 @@ tf_kernel_library(
         "clip_by_value_op.cc",
         "concat_op.cc",
         "const_op.cc",
+        "control_flow_ops.cc",
         "conv_ops.cc",
         "cross_op.cc",
         "cwise_ops.cc",
diff --git a/tensorflow/compiler/tf2xla/kernels/control_flow_ops.cc b/tensorflow/compiler/tf2xla/kernels/control_flow_ops.cc
new file mode 100644
index 0000000000..225016b392
--- /dev/null
+++ b/tensorflow/compiler/tf2xla/kernels/control_flow_ops.cc
@@ -0,0 +1,74 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/compiler/tf2xla/lib/util.h"
+
+#include "tensorflow/compiler/tf2xla/type_util.h"
+#include "tensorflow/compiler/tf2xla/xla_helpers.h"
+#include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
+#include "tensorflow/compiler/tf2xla/xla_op_registry.h"
+#include "tensorflow/compiler/xla/client/lib/constants.h"
+#include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/compiler/xla/primitive_util.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+#include "tensorflow/core/framework/kernel_def_builder.h"
+
+namespace tensorflow {
+namespace {
+
+class SwitchOp : public XlaOpKernel {
+ public:
+  explicit SwitchOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {}
+
+  void Compile(XlaOpKernelContext* ctx) override {
+    OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(ctx->InputShape("pred")),
+                errors::InvalidArgument(
+                    "The second input must be a scalar, but it has shape ",
+                    ctx->InputShape("pred").DebugString()));
+
+    xla::Literal pred_literal;
+    OP_REQUIRES_OK(ctx, ctx->ConstantInput("pred", &pred_literal));
+    bool pred = pred_literal.data<bool>()[0];
+    ctx->SetOutput(pred ? 1 : 0, ctx->Input("data"));
+  }
+};
+
+REGISTER_XLA_OP(
+    Name("Switch").CompileTimeConstantInput("pred").CompilationOnly(),
+    SwitchOp);
+
+class MergeOp : public XlaOpKernel {
+ public:
+  explicit MergeOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {}
+
+  void Compile(XlaOpKernelContext* ctx) override {
+    xla::PrimitiveType index_type;
+    OP_REQUIRES_OK(ctx, DataTypeToPrimitiveType(ctx->expected_output_dtype(1),
+                                                &index_type));
+
+    for (int i = 0; i < ctx->num_inputs(); i++) {
+      if (ctx->op_kernel_context()->has_input(i)) {
+        ctx->SetOutput(0, ctx->Input(i));
+        ctx->SetOutput(
+            1, xla::ConstantR0WithType(ctx->Input(i).builder(), index_type, i));
+        return;
+      }
+    }
+  }
+};
+
+REGISTER_XLA_OP(Name("Merge").CompilationOnly(), MergeOp);
+
+}  // anonymous namespace
+}  // namespace tensorflow
-- 
GitLab


From 72553b845b20c2f0a2c7b1ac948f92ff988a6502 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Mon, 4 Feb 2019 11:59:36 -0800
Subject: [PATCH 1856/2345] Adding more comprehensive metric correctness tests
 for different model types, modes etc.

PiperOrigin-RevId: 232339745
---
 tensorflow/python/keras/BUILD                 |  13 +
 tensorflow/python/keras/engine/training.py    |  17 +-
 .../python/keras/engine/training_test.py      |  63 ----
 .../python/keras/engine/training_utils.py     |  16 +-
 tensorflow/python/keras/metrics.py            |  57 +++
 .../python/keras/metrics_correctness_test.py  | 326 ++++++++++++++++++
 6 files changed, 416 insertions(+), 76 deletions(-)
 create mode 100644 tensorflow/python/keras/metrics_correctness_test.py

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 7aea97f115..5ddf901f20 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -483,6 +483,19 @@ tf_py_test(
     shard_count = 4,
 )
 
+tf_py_test(
+    name = "metrics_correctness_test",
+    size = "medium",
+    srcs = ["metrics_correctness_test.py"],
+    additional_deps = [
+        ":keras",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+    ],
+    shard_count = 4,
+)
+
 tf_py_test(
     name = "applications_test",
     size = "enormous",
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index c63eee5c44..8e1a962610 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -47,6 +47,7 @@ from tensorflow.python.keras.utils import data_utils
 from tensorflow.python.keras.utils.generic_utils import slice_arrays
 from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.losses import losses_impl
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import optimizer as tf_optimizer_module
 from tensorflow.python.training.checkpointable import base as checkpointable
@@ -313,8 +314,10 @@ class Model(Network):
                          ' outputs, but you passed loss=' + str(loss))
       loss_functions = [training_utils.get_loss_function(l) for l in loss]
     else:
-      loss_function = training_utils.get_loss_function(loss)
-      loss_functions = [loss_function for _ in range(len(self.outputs))]
+      loss_functions = [
+          training_utils.get_loss_function(loss)
+          for _ in range(len(self.outputs))
+      ]
     self.loss_functions = loss_functions
 
     skip_target_indices = []
@@ -485,13 +488,15 @@ class Model(Network):
                                           '_loss'] = output_loss
 
             # Keep track of stateful result tensor and function for the loss.
-            mean_wrapped_loss = metrics_module.MeanMetricWrapper(
+            # Reset reduction here as metric wrapper will take care of that.
+            loss_fn.reduction = losses_impl.ReductionV2.NONE
+            output_loss_metric = metrics_module.SumOverBatchSizeMetricWrapper(
                 loss_fn, name=loss_fn.name)
-            result_tensor = self._call_metric_fn(mean_wrapped_loss, y_true,
+            result_tensor = self._call_metric_fn(output_loss_metric, y_true,
                                                  y_pred, sample_weight, mask)
             self._compile_stateful_metrics_tensors[self.output_names[i] +
                                                    '_loss'] = result_tensor
-            self._compile_stateful_metric_functions.append(mean_wrapped_loss)
+            self._compile_stateful_metric_functions.append(output_loss_metric)
 
             self._compile_metrics_names.append(self.output_names[i] + '_loss')
           if total_loss is None:
@@ -1814,7 +1819,7 @@ class Model(Network):
     self._per_output_weighted_metrics = \
         training_utils.collect_per_output_metric_info(
             weighted_metrics, self.output_names, output_shapes,
-            self.loss_functions, self.sample_weights)
+            self.loss_functions, is_weighted=True)
 
   def _add_unique_metric_name(self, metric_name, output_index):
     """Makes the metric name unique and adds it to the model's metric name list.
diff --git a/tensorflow/python/keras/engine/training_test.py b/tensorflow/python/keras/engine/training_test.py
index 6be4da70f6..72c4a29118 100644
--- a/tensorflow/python/keras/engine/training_test.py
+++ b/tensorflow/python/keras/engine/training_test.py
@@ -2114,69 +2114,6 @@ class TestTrainingWithMetrics(keras_parameterized.TestCase):
               batch_size=5)
     self.assertEqual(reference_metric_names, model.metrics_names)
 
-  @keras_parameterized.run_all_keras_modes
-  def test_metrics_correctness(self):
-    model = keras.Sequential()
-    model.add(
-        keras.layers.Dense(
-            3, activation='relu', input_dim=4, kernel_initializer='ones'))
-    model.add(
-        keras.layers.Dense(
-            1, activation='sigmoid', kernel_initializer='ones'))
-    model.compile(
-        loss='mae',
-        metrics=['accuracy', metrics_module.BinaryAccuracy()],
-        optimizer=RMSPropOptimizer(learning_rate=0.001),
-        run_eagerly=testing_utils.should_run_eagerly())
-
-    # verify correctness of stateful and stateless metrics.
-    x = np.ones((100, 4))
-    y = np.ones((100, 1))
-    outs = model.evaluate(x, y)
-    self.assertEqual(outs[1], 1.)
-    self.assertEqual(outs[2], 1.)
-
-    y = np.zeros((100, 1))
-    outs = model.evaluate(x, y)
-    self.assertEqual(outs[1], 0.)
-    self.assertEqual(outs[2], 0.)
-
-  @keras_parameterized.run_all_keras_modes
-  def test_metrics_correctness_with_weighted_metrics(self):
-    np.random.seed(1337)
-    x = np.array([[[1.], [1.]], [[0.], [0.]]])
-    model = keras.models.Sequential()
-    model.add(
-        keras.layers.TimeDistributed(
-            keras.layers.Dense(1, kernel_initializer='ones'),
-            input_shape=(2, 1)))
-    model.compile(
-        RMSPropOptimizer(learning_rate=0.001),
-        loss='mse',
-        sample_weight_mode='temporal',
-        weighted_metrics=['accuracy', 'mse'],
-        run_eagerly=testing_utils.should_run_eagerly())
-    y = np.array([[[1.], [1.]], [[1.], [1.]]])
-
-    outs = model.evaluate(x, y)
-    self.assertEqual(outs, [0.5, 0.5, 0.5])
-
-    w = np.array([[0., 0.], [0., 0.]])
-    outs = model.evaluate(x, y, sample_weight=w)
-    self.assertEqual(outs, [0., 0., 0.])
-
-    w = np.array([[3., 4.], [1., 2.]])
-    outs = model.evaluate(x, y, sample_weight=w)
-    self.assertArrayNear(outs, [0.75, 0.7, 0.3], .001)
-
-    # Verify that metric value is same with arbitrary weights and batch size.
-    x = np.random.random((50, 2, 1))
-    y = np.random.random((50, 2, 1))
-    w = np.random.random((50, 2))
-    mse1 = model.evaluate(x, y, sample_weight=w, batch_size=5)[2]
-    mse2 = model.evaluate(x, y, sample_weight=w, batch_size=10)[2]
-    self.assertNear(mse1, mse2, err=1e-7)
-
   @keras_parameterized.run_all_keras_modes
   def test_metric_state_reset_between_fit_and_evaluate(self):
     model = keras.Sequential()
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index 895a592bd9..22f976a705 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -21,7 +21,6 @@ from __future__ import print_function
 import abc
 import collections
 from collections import OrderedDict
-import copy
 
 import numpy as np
 import six
@@ -521,7 +520,7 @@ def collect_per_output_metric_info(metrics,
                                    output_names,
                                    output_shapes,
                                    loss_fns,
-                                   sample_weights=None):
+                                   is_weighted=False):
   """Maps metric names and functions to model outputs.
 
   Arguments:
@@ -529,7 +528,7 @@ def collect_per_output_metric_info(metrics,
       output_names: a list of the names (strings) of model outputs.
       output_shapes: a list of the shapes (strings) of model outputs.
       loss_fns: a list of the loss functions corresponding to the model outputs.
-      sample_weights: a list of weights to be applied on the model outputs.
+      is_weighted: Boolean indicating whether the given metrics are weighted.
 
   Returns:
       A list (one entry per model output) of dicts.
@@ -553,7 +552,12 @@ def collect_per_output_metric_info(metrics,
     return [{} for _ in output_names]
   if isinstance(metrics, list):
     # we then apply all metrics to all outputs.
-    nested_metrics = [copy.copy(metrics) for _ in output_names]
+    if len(output_names) > 1:
+      nested_metrics = []
+      for _ in output_names:
+        nested_metrics.append([metrics_module.clone_metric(m) for m in metrics])
+    else:
+      nested_metrics = [metrics]
   elif isinstance(metrics, dict):
     nested_metrics = []
     for name in output_names:
@@ -569,9 +573,7 @@ def collect_per_output_metric_info(metrics,
   for i, metrics in enumerate(nested_metrics):
     metrics_dict = OrderedDict()
     for metric in metrics:
-      weighted = False if (sample_weights is None) else (
-          sample_weights[i] is not None)
-      metric_name = get_metric_name(metric, weighted)
+      metric_name = get_metric_name(metric, is_weighted)
       metric_fn = get_metric_function(
           metric, output_shape=output_shapes[i], loss_fn=loss_fns[i])
 
diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 5d41a5c6f1..63ff5e63e8 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -2628,6 +2628,63 @@ class SparseCategoricalCrossentropy(MeanMetricWrapper):
         axis=axis)
 
 
+class SumOverBatchSize(Reduce):
+  """Computes the weighted sum over batch size of the given values.
+
+  For example, if values is [1, 3, 5, 7] then the metric value is 4.
+  If the weights were specified as [1, 1, 0, 0] then the value would be 1.
+
+  This metric creates two variables, `total` and `count` that are used to
+  compute the average of `values`. This average is ultimately returned as sum
+  over batch size which is an idempotent operation that simply divides `total`
+  by `count`.
+
+  If `sample_weight` is `None`, weights default to 1.  Use `sample_weight` of 0
+  to mask values.
+  """
+
+  def __init__(self, name='sum_over_batch_size', dtype=None):
+    super(SumOverBatchSize, self).__init__(
+        reduction=metrics_utils.Reduction.SUM_OVER_BATCH_SIZE,
+        name=name,
+        dtype=dtype)
+
+
+class SumOverBatchSizeMetricWrapper(SumOverBatchSize):
+  """Wraps a function with the `SumOverBatchSizeMetricWrapper` metric."""
+
+  def __init__(self, fn, name=None, dtype=None, **kwargs):
+    """Creates a `SumOverBatchSizeMetricWrapper` instance.
+
+    Args:
+      fn: The metric function to wrap, with signature `fn(y_true, y_pred,
+        **kwargs)`.
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+      **kwargs: The keyword arguments that are passed on to `fn`.
+    """
+    super(SumOverBatchSizeMetricWrapper, self).__init__(name=name, dtype=dtype)
+    self._fn = fn
+    self._fn_kwargs = kwargs
+
+  def update_state(self, y_true, y_pred, sample_weight=None):
+    y_true = math_ops.cast(y_true, self._dtype)
+    y_pred = math_ops.cast(y_pred, self._dtype)
+    y_pred, y_true, sample_weight = squeeze_or_expand_dimensions(
+        y_pred, y_true, sample_weight)
+
+    matches = self._fn(y_true, y_pred, **self._fn_kwargs)
+    return super(SumOverBatchSizeMetricWrapper, self).update_state(
+        matches, sample_weight=sample_weight)
+
+  def get_config(self):
+    config = {}
+    for k, v in six.iteritems(self._fn_kwargs):
+      config[k] = K.eval(v) if is_tensor_or_variable(v) else v
+    base_config = super(SumOverBatchSizeMetricWrapper, self).get_config()
+    return dict(list(base_config.items()) + list(config.items()))
+
+
 def accuracy(y_true, y_pred):
   y_pred.get_shape().assert_is_compatible_with(y_true.get_shape())
   if y_true.dtype != y_pred.dtype:
diff --git a/tensorflow/python/keras/metrics_correctness_test.py b/tensorflow/python/keras/metrics_correctness_test.py
new file mode 100644
index 0000000000..b2385aa611
--- /dev/null
+++ b/tensorflow/python/keras/metrics_correctness_test.py
@@ -0,0 +1,326 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests metrics correctness using Keras model."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.keras import keras_parameterized
+from tensorflow.python.keras import layers
+from tensorflow.python.keras import metrics
+from tensorflow.python.keras import testing_utils
+from tensorflow.python.platform import test
+
+
+@keras_parameterized.run_with_all_model_types(exclude_models=['sequential'])
+@keras_parameterized.run_all_keras_modes
+class TestMetricsCorrectnessMultiIO(keras_parameterized.TestCase):
+  # TODO(psv): Remove the run_eagerly checks here when b/123082095 is fixed.
+
+  def _get_multi_io_model(self):
+    inp_1 = layers.Input(shape=(1,), name='input_1')
+    inp_2 = layers.Input(shape=(1,), name='input_2')
+    x = layers.Dense(3, kernel_initializer='ones', trainable=False)
+    out_1 = layers.Dense(
+        1, kernel_initializer='ones', name='output_1', trainable=False)
+    out_2 = layers.Dense(
+        1, kernel_initializer='ones', name='output_2', trainable=False)
+
+    branch_a = [inp_1, x, out_1]
+    branch_b = [inp_2, x, out_2]
+    model = testing_utils.get_multi_io_model(branch_a, branch_b)
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        metrics=[metrics.MeanSquaredError()],
+        weighted_metrics=[metrics.MeanSquaredError()],
+        run_eagerly=testing_utils.should_run_eagerly())
+    return model
+
+  def _custom_generator(self):
+    batch_size = 2
+    num_samples = 4
+    inputs = np.asarray([[1.], [2.], [3.], [4.]])
+    targets = np.asarray([[2.], [4.], [6.], [8.]])
+    w1 = np.asarray([2., 3., 4., 5.])
+    w2 = np.asarray([3.5, 2.5, 1.5, 0.5])
+    i = 0
+    while True:
+      batch_index = i * batch_size % num_samples
+      i += 1
+      start = batch_index
+      end = start + batch_size
+      x = [inputs[start:end], inputs[start:end]]
+      y = [targets[start:end], targets[start:end]]
+      w = [w1[start:end], w2[start:end]]
+      yield x, y, w
+
+  def setUp(self):
+    super(TestMetricsCorrectnessMultiIO, self).setUp()
+    self.x = np.asarray([[1.], [2.], [3.], [4.]])
+    self.y = np.asarray([[2.], [4.], [6.], [8.]])
+    self.weights_1 = np.asarray([2., 3., 4., 5.])
+    self.weights_2 = np.asarray([3.5, 2.5, 1.5, 0.5])
+
+    # y_true = [[2.], [4.], [6.], [8.]], y_pred = [[3.], [6.], [9.], [12.]]
+
+    # Metric `output_1`, `output_2`:
+    #   Total = ((3 - 2)^2 + (6 - 4)^2) + ((9 - 6)^2 + (12 - 8)^2) = 30,
+    #   Count = 2 + 2
+    #   Result = 7.5
+
+    # Weighted metric `output_1`:
+    #   Total = ((3 - 2)^2 * 2  + (6 - 4)^2 * 3) +
+    #           ((9 - 6)^2 * 4 + (12 - 8)^2 * 5)
+    #         = 130
+    #   Count = (2 + 3) + (4 + 5)
+    #   Result = 9.2857141
+
+    # Weighted metric `output_2`:
+    #   Total = ((3 - 2)^2 * 3.5 + (6 - 4)^2 * 2.5) +
+    #           ((9 - 6)^2 * 1.5 + (12 - 8)^2 * 0.5)
+    #         = 35
+    #   Count = (3.5 + 2.5) + (1.5 + 0.5)
+    #   Result = 4.375
+
+    # Loss `output_1`:
+    #   Total = ((3 - 2)^2 * 2  + (6 - 4)^2 * 3) +
+    #           ((9 - 6)^2 * 4 + (12 - 8)^2 * 5)
+    #         = 130
+    #   Count = 2 + 2
+    #   Result = 32.5
+
+    # Loss `output_2`:
+    #   Total = ((3 - 2)^2 * 3.5 + (6 - 4)^2 * 2.5) +
+    #           ((9 - 6)^2 * 1.5 + (12 - 8)^2 * 0.5)
+    #         = 35
+    #   Count = 2 + 2
+    #   Result = 8.75
+
+    # Total loss = 32.5 + 8.75 = 41.25
+
+    self.expected_fit_result = {
+        'output_1_mean_squared_error': [7.5, 7.5],
+        'output_2_mean_squared_error': [7.5, 7.5],
+        'output_1_weighted_mean_squared_error': [9.286, 9.286],
+        'output_2_weighted_mean_squared_error': [4.375, 4.375],
+        'loss': [41.25, 41.25]
+    }
+
+    # In the order: 'loss', 'output_1_loss', 'output_2_loss',
+    # 'output_1_mean_squared_error', 'output_1_weighted_mean_squared_error',
+    # 'output_2_mean_squared_error', 'output_2_weighted_mean_squared_error'
+    self.expected_batch_result = [41.25, 32.5, 8.75, 7.5, 9.286, 7.5, 4.375]
+
+  def test_fit(self):
+    model = self._get_multi_io_model()
+    history = model.fit([self.x, self.x], [self.y, self.y],
+                        sample_weight={
+                            'output_1': self.weights_1,
+                            'output_2': self.weights_2,
+                        },
+                        batch_size=2,
+                        epochs=2,
+                        shuffle=False)
+
+    if not model.run_eagerly:
+      self.expected_fit_result['output_1_loss'] = [32.5, 32.5]
+      self.expected_fit_result['output_2_loss'] = [8.75, 8.75]
+
+    for key, value in self.expected_fit_result.items():
+      self.assertAllClose(history.history[key], value, 1e-3)
+
+  def test_eval(self):
+    model = self._get_multi_io_model()
+    eval_result = model.evaluate([self.x, self.x], [self.y, self.y],
+                                 batch_size=2,
+                                 sample_weight={
+                                     'output_1': self.weights_1,
+                                     'output_2': self.weights_2,
+                                 })
+
+    if model.run_eagerly:
+      self.expected_batch_result = [41.25, 58, 10.75, 7.5, 9.286, 7.5, 4.375]
+    self.assertAllClose(eval_result, self.expected_batch_result, 1e-3)
+
+    if model.run_eagerly:
+      return
+    # Verify that metric value is same with arbitrary weights and batch size.
+    x = np.random.random((50, 1))
+    y = np.random.random((50, 1))
+    w = np.random.random((50,))
+    mse1 = model.evaluate([x, x], [y, y], sample_weight=[w, w], batch_size=5)[3]
+    mse2 = model.evaluate([x, x], [y, y], sample_weight=[w, w],
+                          batch_size=10)[3]
+    self.assertAllClose(mse1, mse2, 1e-3)
+
+  def test_train_on_batch(self):
+    model = self._get_multi_io_model()
+    result = model.train_on_batch([self.x, self.x], [self.y, self.y],
+                                  sample_weight={
+                                      'output_1': self.weights_1,
+                                      'output_2': self.weights_2,
+                                  })
+    self.assertAllClose(result, self.expected_batch_result, 1e-3)
+
+  def test_test_on_batch(self):
+    model = self._get_multi_io_model()
+    result = model.test_on_batch([self.x, self.x], [self.y, self.y],
+                                 sample_weight={
+                                     'output_1': self.weights_1,
+                                     'output_2': self.weights_2,
+                                 })
+    self.assertAllClose(result, self.expected_batch_result, 1e-3)
+
+  def test_fit_generator(self):
+    model = self._get_multi_io_model()
+    history = model.fit_generator(
+        self._custom_generator(), steps_per_epoch=2, epochs=2)
+
+    if not model.run_eagerly:
+      self.expected_fit_result['output_1_loss'] = [32.5, 32.5]
+      self.expected_fit_result['output_2_loss'] = [8.75, 8.75]
+    for key, value in self.expected_fit_result.items():
+      self.assertAllClose(history.history[key], value, 1e-3)
+
+  def test_eval_generator(self):
+    model = self._get_multi_io_model()
+    eval_result = model.evaluate_generator(self._custom_generator(), steps=2)
+    if model.run_eagerly:
+      self.expected_batch_result = [41.25, 58, 10.75, 7.5, 9.286, 7.5, 4.375]
+    self.assertAllClose(eval_result, self.expected_batch_result, 1e-3)
+
+
+@keras_parameterized.run_with_all_model_types
+@keras_parameterized.run_all_keras_modes
+class TestMetricsCorrectnessSingleIO(keras_parameterized.TestCase):
+
+  def _get_model(self):
+    x = layers.Dense(3, kernel_initializer='ones', trainable=False)
+    out = layers.Dense(
+        1, kernel_initializer='ones', name='output', trainable=False)
+    model = testing_utils.get_model_from_layers([x, out], input_shape=(1,))
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        metrics=[metrics.MeanSquaredError()],
+        weighted_metrics=[metrics.MeanSquaredError()],
+        run_eagerly=testing_utils.should_run_eagerly())
+    return model
+
+  def _custom_generator(self):
+    batch_size = 2
+    num_samples = 4
+    x = np.asarray([[1.], [2.], [3.], [4.]])
+    y = np.asarray([[2.], [4.], [6.], [8.]])
+    w = np.asarray([2., 3., 4., 5.])
+    i = 0
+    while True:
+      batch_index = i * batch_size % num_samples
+      i += 1
+      start = batch_index
+      end = start + batch_size
+      yield x[start:end], y[start:end], w[start:end]
+
+  def setUp(self):
+    super(TestMetricsCorrectnessSingleIO, self).setUp()
+    self.x = np.asarray([[1.], [2.], [3.], [4.]])
+    self.y = np.asarray([[2.], [4.], [6.], [8.]])
+    self.weights = np.asarray([2., 3., 4., 5.])
+
+    # y_true = [[2.], [4.], [6.], [8.]], y_pred = [[3.], [6.], [9.], [12.]]
+
+    # Metric:
+    #   Total = ((3 - 2)^2 + (6 - 4)^2) + ((9 - 6)^2 + (12 - 8)^2) = 30,
+    #   Count = 2 + 2
+    #   Result = 7.5
+
+    # Weighted metric:
+    #   Total = ((3 - 2)^2 * 2  + (6 - 4)^2 * 3) +
+    #           ((9 - 6)^2 * 4 + (12 - 8)^2 * 5)
+    #         = 130
+    #   Count = (2 + 3) + (4 + 5)
+    #   Result = 9.2857141
+
+    # Total loss:
+    #   Total = ((3 - 2)^2 * 2  + (6 - 4)^2 * 3) +
+    #           ((9 - 6)^2 * 4 + (12 - 8)^2 * 5)
+    #         = 130,
+    #   Count = 2 + 2
+    #   Result = 32.5
+
+    self.expected_fit_result = {
+        'mean_squared_error': [7.5, 7.5],
+        'weighted_mean_squared_error': [9.286, 9.286],
+        'loss': [32.5, 32.5]
+    }
+
+    # In the order: 'loss', 'mean_squared_error', 'weighted_mean_squared_error'
+    self.expected_batch_result = [32.5, 7.5, 9.286]
+
+  def test_fit(self):
+    model = self._get_model()
+    history = model.fit(
+        self.x,
+        self.y,
+        sample_weight=self.weights,
+        batch_size=2,
+        epochs=2,
+        shuffle=False)
+    for key, value in self.expected_fit_result.items():
+      self.assertAllClose(history.history[key], value, 1e-3)
+
+  def test_eval(self):
+    model = self._get_model()
+    eval_result = model.evaluate(
+        self.x, self.y, batch_size=2, sample_weight=self.weights)
+    self.assertAllClose(eval_result, self.expected_batch_result, 1e-3)
+
+    # Verify that metric value is same with arbitrary weights and batch size.
+    x = np.random.random((50, 1))
+    y = np.random.random((50, 1))
+    w = np.random.random((50,))
+    mse1 = model.evaluate(x, y, sample_weight=w, batch_size=5)[1]
+    mse2 = model.evaluate(x, y, sample_weight=w, batch_size=10)[1]
+    self.assertAllClose(mse1, mse2, 1e-3)
+
+  def test_train_on_batch(self):
+    model = self._get_model()
+    result = model.train_on_batch(self.x, self.y, sample_weight=self.weights)
+    self.assertAllClose(result, self.expected_batch_result, 1e-3)
+
+  def test_test_on_batch(self):
+    model = self._get_model()
+    result = model.test_on_batch(self.x, self.y, sample_weight=self.weights)
+    self.assertAllClose(result, self.expected_batch_result, 1e-3)
+
+  def test_fit_generator(self):
+    model = self._get_model()
+    history = model.fit_generator(
+        self._custom_generator(), steps_per_epoch=2, epochs=2)
+    for key, value in self.expected_fit_result.items():
+      self.assertAllClose(history.history[key], value, 1e-3)
+
+  def test_eval_generator(self):
+    model = self._get_model()
+    eval_result = model.evaluate_generator(self._custom_generator(), steps=2)
+    self.assertAllClose(eval_result, self.expected_batch_result, 1e-3)
+
+
+if __name__ == '__main__':
+  test.main()
-- 
GitLab


From 15fb21f9ed96ed216b0d66898856fd00be75cdaa Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Mon, 4 Feb 2019 12:03:41 -0800
Subject: [PATCH 1857/2345] Optimize Mirror pad op.

PiperOrigin-RevId: 232340687
---
 tensorflow/lite/kernels/mirror_pad.cc | 65 ++++++++++++---------------
 1 file changed, 29 insertions(+), 36 deletions(-)

diff --git a/tensorflow/lite/kernels/mirror_pad.cc b/tensorflow/lite/kernels/mirror_pad.cc
index 70f7e33928..c0924da2a1 100644
--- a/tensorflow/lite/kernels/mirror_pad.cc
+++ b/tensorflow/lite/kernels/mirror_pad.cc
@@ -71,25 +71,32 @@ struct OpData {
 };
 
 // Util method to initialize the memory of the padded tensor.
-// Returns the index of the current item processed in 'padded_tensor_buffer'
-int InitializeTensorMemory(const TfLiteIntArray* const dims, int dim_index,
-                           int dims_size,
-                           std::vector<PaddedTensor>* padded_tensor_buffer,
-                           int element_index, int num_elements) {
-  if (dim_index >= dims_size) {
-    return element_index;
-  }
-  PaddedTensor* padded_tensor = &(*padded_tensor_buffer)[element_index];
-  padded_tensor->values.clear();
-  padded_tensor->values.reserve(dims->data[dim_index]);
-  for (int i = 0; i < dims->data[dim_index]; ++i) {
-    ++element_index;
-    padded_tensor->values.emplace_back(&(*padded_tensor_buffer)[element_index]);
-    element_index = InitializeTensorMemory(dims, dim_index + 1, dims_size,
-                                           padded_tensor_buffer, element_index,
-                                           num_elements);
+void InitializeTensorMemory(const TfLiteIntArray* const dims, int dims_size,
+                            std::vector<PaddedTensor>* padded_tensor_buffer) {
+  int dimension_index = 0;
+  int element_index = 0;
+  // We hold 2 vectors with values for nodes in current level, and
+  // nodes in the next level, and swap while moving on dimensions of the tensor.
+  std::vector<PaddedTensor*> current_nodes, next_level;
+  current_nodes.push_back(&(*padded_tensor_buffer)[element_index]);
+  element_index++;
+  int next_level_size = 1;
+  while (!current_nodes.empty() && dimension_index < dims_size) {
+    next_level_size *= dims->data[dimension_index];
+    next_level.resize(next_level_size);
+    // Index of elements in next level.
+    int index = 0;
+    for (auto* padded_tensor : current_nodes) {
+      padded_tensor->values.resize(dims->data[dimension_index]);
+      for (int i = 0; i < dims->data[dimension_index]; ++i) {
+        padded_tensor->values[i] = &(*padded_tensor_buffer)[element_index];
+        next_level[index++] = padded_tensor->values[i];
+        element_index++;
+      }
+    }
+    std::swap(current_nodes, next_level);
+    dimension_index++;
   }
-  return element_index;
 }
 
 // Returns pointer to the value at the specified index in 'data'.
@@ -117,20 +124,6 @@ inline const void* GetValuePointerAtIndex(const void* data, int index,
   return nullptr;
 }
 
-// Util method that increment index in the N-d array.
-void IncrementTensorIndex(const TfLiteIntArray* dims,
-                          std::vector<int>* tensor_index_ptr) {
-  int dimension_index = dims->size - 1;
-  auto& tensor_index = *tensor_index_ptr;
-  tensor_index[dimension_index]++;
-  while (dimension_index >= 0 &&
-         tensor_index[dimension_index] == dims->data[dimension_index]) {
-    tensor_index[dimension_index] = 0;
-    dimension_index--;
-    if (dimension_index >= 0) tensor_index[dimension_index]++;
-  }
-}
-
 // Fills the 'padded_tensor' with data from 'input_tensor'.
 TfLiteStatus InitFromInputTensor(const TfLiteTensor* input_tensor,
                                  PaddedTensor* padded_tensor) {
@@ -145,13 +138,13 @@ TfLiteStatus InitFromInputTensor(const TfLiteTensor* input_tensor,
   std::vector<int> tensor_index(dims->size, 0);
   int flat_index = 0;
   const int num_elements = NumElements(input_tensor);
+  auto* tensor = padded_tensor->GetMutable(tensor_index);
   while (flat_index < num_elements) {
-    auto* tensor = padded_tensor->GetMutable(tensor_index);
     if (tensor == nullptr) {
       return kTfLiteError;
     }
     tensor->value = GetValuePointerAtIndex(data, flat_index, data_type);
-    IncrementTensorIndex(dims, &tensor_index);
+    ++tensor;
     ++flat_index;
   }
 
@@ -308,8 +301,8 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
 
   PaddedTensor& padded_tensor = op_data->pad_tensor_buffer[0];
   // Initialize memory.
-  InitializeTensorMemory(input_tensor->dims, 0, input_dims,
-                         &op_data->pad_tensor_buffer, 0, op_data->num_elements);
+  InitializeTensorMemory(input_tensor->dims, input_dims,
+                         &op_data->pad_tensor_buffer);
   // Set the values from the input_tensor.
   TF_LITE_ENSURE_STATUS(InitFromInputTensor(input_tensor, &padded_tensor));
   const int offset =
-- 
GitLab


From 964f0d695939098cca3b77e909f62a696014fb19 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Mon, 4 Feb 2019 12:04:10 -0800
Subject: [PATCH 1858/2345] Updating bug number. The original bug was marked as
 fixed while source datasets still need to be marked as stateful to inhibit
 constant folding.

PiperOrigin-RevId: 232340792
---
 .../contrib/bigtable/ops/bigtable_ops.cc      | 10 +++++-----
 tensorflow/core/ops/dataset_ops.cc            | 20 +++++++++----------
 .../core/ops/experimental_dataset_ops.cc      | 10 +++++-----
 3 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/tensorflow/contrib/bigtable/ops/bigtable_ops.cc b/tensorflow/contrib/bigtable/ops/bigtable_ops.cc
index 416b719e30..39c2a2e775 100644
--- a/tensorflow/contrib/bigtable/ops/bigtable_ops.cc
+++ b/tensorflow/contrib/bigtable/ops/bigtable_ops.cc
@@ -59,7 +59,7 @@ REGISTER_OP("BigtablePrefixKeyDataset")
     .Input("table: resource")
     .Input("prefix: string")
     .Output("handle: variant")
-    .SetIsStateful()  // TODO(b/65524810): Source dataset ops must be marked
+    .SetIsStateful()  // TODO(b/123753214): Source dataset ops must be marked
                       // stateful to inhibit constant folding.
     .SetShapeFn(shape_inference::ScalarShape);
 
@@ -68,14 +68,14 @@ REGISTER_OP("BigtableRangeKeyDataset")
     .Input("start_key: string")
     .Input("end_key: string")
     .Output("handle: variant")
-    .SetIsStateful()  // TODO(b/65524810): Source dataset ops must be marked
+    .SetIsStateful()  // TODO(b/123753214): Source dataset ops must be marked
                       // stateful to inhibit constant folding.
     .SetShapeFn(shape_inference::ScalarShape);
 
 REGISTER_OP("BigtableSampleKeysDataset")
     .Input("table: resource")
     .Output("handle: variant")
-    .SetIsStateful()  // TODO(b/65524810): Source dataset ops must be marked
+    .SetIsStateful()  // TODO(b/123753214): Source dataset ops must be marked
                       // stateful to inhibit constant folding.
     .SetShapeFn(shape_inference::ScalarShape);
 
@@ -85,7 +85,7 @@ REGISTER_OP("BigtableSampleKeyPairsDataset")
     .Input("start_key: string")
     .Input("end_key: string")
     .Output("handle: variant")
-    .SetIsStateful()  // TODO(b/65524810): Source dataset ops must be marked
+    .SetIsStateful()  // TODO(b/123753214): Source dataset ops must be marked
                       // stateful to inhibit constant folding.
     .SetShapeFn(shape_inference::ScalarShape);
 
@@ -100,7 +100,7 @@ REGISTER_OP("BigtableScanDataset")
     .Input("columns: string")
     .Input("probability: float")
     .Output("handle: variant")
-    .SetIsStateful()  // TODO(b/65524810): Source dataset ops must be marked
+    .SetIsStateful()  // TODO(b/123753214): Source dataset ops must be marked
                       // stateful to inhibit constant folding.
     .SetShapeFn(shape_inference::ScalarShape);
 
diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc
index 1c117166de..8494298105 100644
--- a/tensorflow/core/ops/dataset_ops.cc
+++ b/tensorflow/core/ops/dataset_ops.cc
@@ -27,7 +27,7 @@ namespace tensorflow {
 // to a stateful "iterator" by passing the "dataset" to the
 // "MakeIterator" op.
 //
-// TODO(b/65524810): DT_VARIANT tensors that represent "dataset" objects are
+// TODO(b/123753214): DT_VARIANT tensors that represent "dataset" objects are
 // not presently serializable. To avoid issues with constant folding, ensure
 // that any "source dataset" ops (i.e. ops that output a dataset and do not
 // take one as input) are marked "stateful".
@@ -37,7 +37,7 @@ REGISTER_OP("TensorDataset")
     .Output("handle: variant")
     .Attr("Toutput_types: list(type) >= 1")
     .Attr("output_shapes: list(shape) >= 1")
-    .SetIsStateful()  // TODO(b/65524810): Source dataset ops must be marked
+    .SetIsStateful()  // TODO(b/123753214): Source dataset ops must be marked
                       // stateful to inhibit constant folding.
     .SetShapeFn(shape_inference::ScalarShape);  // TODO(mrry): Validate that
                                                 // `components` have shapes
@@ -49,7 +49,7 @@ REGISTER_OP("TensorSliceDataset")
     .Output("handle: variant")
     .Attr("Toutput_types: list(type) >= 1")
     .Attr("output_shapes: list(shape) >= 1")
-    .SetIsStateful()  // TODO(b/65524810): Source dataset ops must be marked
+    .SetIsStateful()  // TODO(b/123753214): Source dataset ops must be marked
                       // stateful to inhibit constant folding.
     .SetShapeFn(shape_inference::ScalarShape);  // TODO(mrry): Validate that the
                                                 // dim-0 slices of `components`
@@ -62,7 +62,7 @@ REGISTER_OP("SparseTensorSliceDataset")
     .Input("dense_shape: int64")
     .Output("handle: variant")
     .Attr("Tvalues: type")
-    .SetIsStateful()  // TODO(b/65524810): Source dataset ops must be marked
+    .SetIsStateful()  // TODO(b/123753214): Source dataset ops must be marked
                       // stateful to inhibit constant folding.
     .SetShapeFn(shape_inference::ScalarShape);
 
@@ -79,7 +79,7 @@ REGISTER_OP("GeneratorDataset")
     .Attr("Tfinalize_func_args: list(type) >= 0")
     .Attr("output_types: list(type) >= 1")
     .Attr("output_shapes: list(shape) >= 1")
-    .SetIsStateful()  // TODO(b/65524810): Source dataset ops must be marked
+    .SetIsStateful()  // TODO(b/123753214): Source dataset ops must be marked
                       // stateful to inhibit constant folding.
     .SetShapeFn(shape_inference::ScalarShape);
 
@@ -322,7 +322,7 @@ REGISTER_OP("RangeDataset")
     .Output("handle: variant")
     .Attr("output_types: list(type) >= 1")
     .Attr("output_shapes: list(shape) >= 1")
-    .SetIsStateful()  // TODO(b/65524810): Source dataset ops must be marked
+    .SetIsStateful()  // TODO(b/123753214): Source dataset ops must be marked
                       // stateful to inhibit constant folding.
     .SetShapeFn([](shape_inference::InferenceContext* c) {
       shape_inference::ShapeHandle unused;
@@ -388,7 +388,7 @@ REGISTER_OP("TextLineDataset")
     .Input("compression_type: string")
     .Input("buffer_size: int64")
     .Output("handle: variant")
-    .SetIsStateful()  // TODO(b/65524810): Source dataset ops must be marked
+    .SetIsStateful()  // TODO(b/123753214): Source dataset ops must be marked
                       // stateful to inhibit constant folding.
     .SetShapeFn([](shape_inference::InferenceContext* c) {
       shape_inference::ShapeHandle unused;
@@ -408,7 +408,7 @@ REGISTER_OP("FixedLengthRecordDataset")
     .Input("footer_bytes: int64")
     .Input("buffer_size: int64")
     .Output("handle: variant")
-    .SetIsStateful()  // TODO(b/65524810): Source dataset ops must be marked
+    .SetIsStateful()  // TODO(b/123753214): Source dataset ops must be marked
                       // stateful to inhibit constant folding.
     .SetShapeFn([](shape_inference::InferenceContext* c) {
       shape_inference::ShapeHandle unused;
@@ -431,7 +431,7 @@ REGISTER_OP("FixedLengthRecordDatasetV2")
     .Input("buffer_size: int64")
     .Input("compression_type: string")
     .Output("handle: variant")
-    .SetIsStateful()  // TODO(b/65524810): Source dataset ops must be marked
+    .SetIsStateful()  // TODO(b/123753214): Source dataset ops must be marked
                       // stateful to inhibit constant folding.
     .SetShapeFn([](shape_inference::InferenceContext* c) {
       shape_inference::ShapeHandle unused;
@@ -451,7 +451,7 @@ REGISTER_OP("TFRecordDataset")
     .Input("compression_type: string")
     .Input("buffer_size: int64")
     .Output("handle: variant")
-    .SetIsStateful()  // TODO(b/65524810): Source dataset ops must be marked
+    .SetIsStateful()  // TODO(b/123753214): Source dataset ops must be marked
                       // stateful to inhibit constant folding.
     .SetShapeFn([](shape_inference::InferenceContext* c) {
       shape_inference::ShapeHandle unused;
diff --git a/tensorflow/core/ops/experimental_dataset_ops.cc b/tensorflow/core/ops/experimental_dataset_ops.cc
index 41348972d3..95230af579 100644
--- a/tensorflow/core/ops/experimental_dataset_ops.cc
+++ b/tensorflow/core/ops/experimental_dataset_ops.cc
@@ -42,7 +42,7 @@ REGISTER_OP("ExperimentalCSVDataset")
     .Output("handle: variant")
     .Attr("output_types: list({float,double,int32,int64,string}) >= 1")
     .Attr("output_shapes: list(shape) >= 1")
-    .SetIsStateful()  // TODO(b/65524810): Source dataset ops must be marked
+    .SetIsStateful()  // TODO(b/123753214): Source dataset ops must be marked
                       // stateful to inhibit constant folding.
     .SetShapeFn([](shape_inference::InferenceContext* c) {
       shape_inference::ShapeHandle unused;
@@ -213,7 +213,7 @@ REGISTER_OP("ExperimentalMapDataset")
 REGISTER_OP("ExperimentalMatchingFilesDataset")
     .Input("patterns: string")
     .Output("handle: variant")
-    .SetIsStateful()  // TODO(b/65524810): Source dataset ops must be marked
+    .SetIsStateful()  // TODO(b/123753214): Source dataset ops must be marked
                       // stateful to inhibit constant folding.
     .SetShapeFn([](shape_inference::InferenceContext* c) {
       shape_inference::ShapeHandle unused;
@@ -267,7 +267,7 @@ REGISTER_OP("ExperimentalRandomDataset")
     .Output("handle: variant")
     .Attr("output_types: list(type) >= 1")
     .Attr("output_shapes: list(shape) >= 1")
-    .SetIsStateful()  // TODO(b/65524810): Source dataset ops must be marked
+    .SetIsStateful()  // TODO(b/123753214): Source dataset ops must be marked
                       // stateful to inhibit constant folding.
     .SetShapeFn([](shape_inference::InferenceContext* c) {
       shape_inference::ShapeHandle unused;
@@ -338,7 +338,7 @@ REGISTER_OP("ExperimentalSqlDataset")
     .Output("handle: variant")
     .Attr("output_types: list(type) >= 1")
     .Attr("output_shapes: list(shape) >= 1")
-    .SetIsStateful()  // TODO(b/65524810): Source dataset ops must be marked
+    .SetIsStateful()  // TODO(b/123753214): Source dataset ops must be marked
                       // stateful to inhibit constant folding.
     .SetShapeFn([](shape_inference::InferenceContext* c) {
       shape_inference::ShapeHandle unused;
@@ -467,7 +467,7 @@ REGISTER_OP("ExperimentalLMDBDataset")
     .Output("handle: variant")
     .Attr("output_types: list(type) >= 1")
     .Attr("output_shapes: list(shape) >= 1")
-    .SetIsStateful()  // TODO(b/65524810): Source dataset ops must be marked
+    .SetIsStateful()  // TODO(b/123753214): Source dataset ops must be marked
                       // stateful to inhibit constant folding.
     .SetShapeFn(shape_inference::ScalarShape);
 
-- 
GitLab


From fb6ce4462da4b6ebe5dfd2199001d5c869b309fc Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Mon, 4 Feb 2019 12:52:40 -0800
Subject: [PATCH 1859/2345] Make v2 Keras Initializer exports backwards
 compatible with V1 exports

PiperOrigin-RevId: 232349234
---
 tensorflow/python/keras/ops.py                | 39 ++++++++++++-------
 ...nsorflow.keras.initializers.constant.pbtxt | 18 +++++++++
 ...low.keras.initializers.glorot_normal.pbtxt | 19 +++++++++
 ...ow.keras.initializers.glorot_uniform.pbtxt | 19 +++++++++
 ...nsorflow.keras.initializers.identity.pbtxt | 18 +++++++++
 .../tensorflow.keras.initializers.ones.pbtxt  | 17 ++++++++
 ...orflow.keras.initializers.orthogonal.pbtxt | 18 +++++++++
 .../v2/tensorflow.keras.initializers.pbtxt    | 28 +++++++++++++
 .../tensorflow.keras.initializers.zeros.pbtxt | 17 ++++++++
 9 files changed, 179 insertions(+), 14 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.constant.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.glorot_normal.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.glorot_uniform.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.identity.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.ones.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.orthogonal.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.zeros.pbtxt

diff --git a/tensorflow/python/keras/ops.py b/tensorflow/python/keras/ops.py
index bc14eef505..44e022828a 100644
--- a/tensorflow/python/keras/ops.py
+++ b/tensorflow/python/keras/ops.py
@@ -56,22 +56,33 @@ keras_export(v1=["keras.initializers.he_uniform"])(
 
 keras_export("keras.initializers.Initializer", v1=[])(
     init_ops_v2.Initializer)
-keras_export("keras.initializers.Zeros", v1=[])(
-    init_ops_v2.Zeros)
-keras_export("keras.initializers.Ones", v1=[])(
-    init_ops_v2.Ones)
-keras_export("keras.initializers.Constant", v1=[])(
-    init_ops_v2.Constant)
+keras_export(
+    "keras.initializers.Zeros", "keras.initializers.zeros", v1=[])(
+        init_ops_v2.Zeros)
+keras_export(
+    "keras.initializers.Ones", "keras.initializers.ones", v1=[])(
+        init_ops_v2.Ones)
+keras_export(
+    "keras.initializers.Constant", "keras.initializers.constant", v1=[])(
+        init_ops_v2.Constant)
 keras_export("keras.initializers.VarianceScaling", v1=[])(
     init_ops_v2.VarianceScaling)
-keras_export("keras.initializers.Orthogonal", v1=[])(
-    init_ops_v2.Orthogonal)
-keras_export("keras.initializers.Identity", v1=[])(
-    init_ops_v2.Identity)
-keras_export("keras.initializers.GlorotUniform", v1=[])(
-    init_ops_v2.GlorotUniform)
-keras_export("keras.initializers.GlorotNormal", v1=[])(
-    init_ops_v2.GlorotNormal)
+keras_export(
+    "keras.initializers.Orthogonal", "keras.initializers.orthogonal", v1=[])(
+        init_ops_v2.Orthogonal)
+keras_export(
+    "keras.initializers.Identity", "keras.initializers.identity", v1=[])(
+        init_ops_v2.Identity)
+keras_export(
+    "keras.initializers.GlorotUniform",
+    "keras.initializers.glorot_uniform",
+    v1=[])(
+        init_ops_v2.GlorotUniform)
+keras_export(
+    "keras.initializers.GlorotNormal",
+    "keras.initializers.glorot_normal",
+    v1=[])(
+        init_ops_v2.GlorotNormal)
 keras_export("keras.initializers.lecun_normal", v1=[])(
     init_ops_v2.lecun_normal)
 keras_export("keras.initializers.lecun_uniform", v1=[])(
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.constant.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.constant.pbtxt
new file mode 100644
index 0000000000..b03cbb8eb8
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.constant.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.keras.initializers.constant"
+tf_class {
+  is_instance: "<class \'tensorflow.python.ops.init_ops_v2.Constant\'>"
+  is_instance: "<class \'tensorflow.python.ops.init_ops_v2.Initializer\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=[\'0\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.glorot_normal.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.glorot_normal.pbtxt
new file mode 100644
index 0000000000..02f8c252bd
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.glorot_normal.pbtxt
@@ -0,0 +1,19 @@
+path: "tensorflow.keras.initializers.glorot_normal"
+tf_class {
+  is_instance: "<class \'tensorflow.python.ops.init_ops_v2.GlorotNormal\'>"
+  is_instance: "<class \'tensorflow.python.ops.init_ops_v2.VarianceScaling\'>"
+  is_instance: "<class \'tensorflow.python.ops.init_ops_v2.Initializer\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.glorot_uniform.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.glorot_uniform.pbtxt
new file mode 100644
index 0000000000..6d18a3b6e7
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.glorot_uniform.pbtxt
@@ -0,0 +1,19 @@
+path: "tensorflow.keras.initializers.glorot_uniform"
+tf_class {
+  is_instance: "<class \'tensorflow.python.ops.init_ops_v2.GlorotUniform\'>"
+  is_instance: "<class \'tensorflow.python.ops.init_ops_v2.VarianceScaling\'>"
+  is_instance: "<class \'tensorflow.python.ops.init_ops_v2.Initializer\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.identity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.identity.pbtxt
new file mode 100644
index 0000000000..dcdb6ddf5f
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.identity.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.keras.initializers.identity"
+tf_class {
+  is_instance: "<class \'tensorflow.python.ops.init_ops_v2.Identity\'>"
+  is_instance: "<class \'tensorflow.python.ops.init_ops_v2.Initializer\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'gain\'], varargs=None, keywords=None, defaults=[\'1.0\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.ones.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.ones.pbtxt
new file mode 100644
index 0000000000..cc2dd171df
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.ones.pbtxt
@@ -0,0 +1,17 @@
+path: "tensorflow.keras.initializers.ones"
+tf_class {
+  is_instance: "<class \'tensorflow.python.ops.init_ops_v2.Ones\'>"
+  is_instance: "<class \'tensorflow.python.ops.init_ops_v2.Initializer\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.orthogonal.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.orthogonal.pbtxt
new file mode 100644
index 0000000000..855065c163
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.orthogonal.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.keras.initializers.orthogonal"
+tf_class {
+  is_instance: "<class \'tensorflow.python.ops.init_ops_v2.Orthogonal\'>"
+  is_instance: "<class \'tensorflow.python.ops.init_ops_v2.Initializer\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'gain\', \'seed\'], varargs=None, keywords=None, defaults=[\'1.0\', \'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.pbtxt
index 7412cd1305..15a56fbb97 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.pbtxt
@@ -48,6 +48,34 @@ tf_module {
     name: "Zeros"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "constant"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "glorot_normal"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "glorot_uniform"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "identity"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "ones"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "orthogonal"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "zeros"
+    mtype: "<type \'type\'>"
+  }
   member_method {
     name: "deserialize"
     argspec: "args=[\'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.zeros.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.zeros.pbtxt
new file mode 100644
index 0000000000..f9b3359d7a
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.initializers.zeros.pbtxt
@@ -0,0 +1,17 @@
+path: "tensorflow.keras.initializers.zeros"
+tf_class {
+  is_instance: "<class \'tensorflow.python.ops.init_ops_v2.Zeros\'>"
+  is_instance: "<class \'tensorflow.python.ops.init_ops_v2.Initializer\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
-- 
GitLab


From 05ca1e9bf3e7a04603eac921c4d95c5dbeca7dd6 Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Mon, 4 Feb 2019 13:05:55 -0800
Subject: [PATCH 1860/2345] [Grappler] Re-enable DependencyOptimizer in
 FunctionTest.testControlFlowStrictness.

PiperOrigin-RevId: 232351788
---
 tensorflow/python/framework/function_test.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/tensorflow/python/framework/function_test.py b/tensorflow/python/framework/function_test.py
index 3d5a5fe797..f541a4fb4a 100644
--- a/tensorflow/python/framework/function_test.py
+++ b/tensorflow/python/framework/function_test.py
@@ -497,8 +497,6 @@ class FunctionTest(test.TestCase):
                                          lambda y: AssertFail(y), [x])
       # pylint: enable=unnecessary-lambda
 
-    rewriter_config = rewriter_config_pb2.RewriterConfig(
-        dependency_optimization=rewriter_config_pb2.RewriterConfig.OFF)
     # Enables inlining.
     config = config_pb2.ConfigProto(
         graph_options=config_pb2.GraphOptions(
@@ -506,8 +504,7 @@ class FunctionTest(test.TestCase):
                 opt_level=config_pb2.OptimizerOptions.L0,
                 do_common_subexpression_elimination=True,
                 do_function_inlining=True,
-                do_constant_folding=True),
-            rewrite_options=rewriter_config))
+                do_constant_folding=True)))
 
     with session.Session(config=config) as sess:
       # Since the 'False' branch is not taken, the assertion should not fire.
-- 
GitLab


From 91da898cb6f6b0e751e15ceb813a37cdfe18a035 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Mon, 4 Feb 2019 13:31:05 -0800
Subject: [PATCH 1861/2345] Bump max bazel version for TF.

I just verified that we can build using bazel 0.22

PiperOrigin-RevId: 232356025
---
 configure.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.py b/configure.py
index 8dcd318220..08226dc132 100644
--- a/configure.py
+++ b/configure.py
@@ -1556,7 +1556,7 @@ def main():
   # environment variables.
   environ_cp = dict(os.environ)
 
-  check_bazel_version('0.19.0', '0.21.0')
+  check_bazel_version('0.19.0', '0.22.0')
 
   reset_tf_configure_bazelrc()
 
-- 
GitLab


From 5437e7644fc59851ed077322cd569f80112611d8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 4 Feb 2019 13:32:22 -0800
Subject: [PATCH 1862/2345] This CL enables tf.print to print to the cell
 output in Jupyter notebooks. It works by adding a log listener registration
 mechanism to the C API, and registering a listener that prints messages
 directly to python's stdout when in an ipython environment.

PiperOrigin-RevId: 232356327
---
 tensorflow/c/BUILD                        |  1 +
 tensorflow/c/c_api.cc                     |  8 ++++++
 tensorflow/c/c_api.h                      |  8 ++++++
 tensorflow/core/kernels/BUILD             |  1 +
 tensorflow/core/kernels/logging_ops.cc    | 26 +++++++++++++++++-
 tensorflow/core/kernels/logging_ops.h     | 33 +++++++++++++++++++++++
 tensorflow/python/eager/pywrap_tfe.h      |  2 ++
 tensorflow/python/eager/pywrap_tfe_src.cc | 33 +++++++++++++++++++++++
 tensorflow/python/ops/logging_ops.py      | 14 +++++++---
 tensorflow/python/pywrap_tfe.i            |  1 +
 10 files changed, 123 insertions(+), 4 deletions(-)
 create mode 100644 tensorflow/core/kernels/logging_ops.h

diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index ef52a28460..6423b4bb20 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -97,6 +97,7 @@ tf_cuda_library(
             "//tensorflow/core:lib",
             "//tensorflow/core:lib_internal",
             "//tensorflow/core/distributed_runtime:server_lib",
+            "//tensorflow/core/kernels:logging_ops",
         ],
     }) + select({
         "//tensorflow:with_xla_support": [
diff --git a/tensorflow/c/c_api.cc b/tensorflow/c/c_api.cc
index 94d9f4a6fa..6d6d02635b 100644
--- a/tensorflow/c/c_api.cc
+++ b/tensorflow/c/c_api.cc
@@ -27,6 +27,7 @@ limitations under the License.
 #include "tensorflow/cc/ops/while_loop.h"
 #include "tensorflow/cc/saved_model/loader.h"
 #include "tensorflow/core/framework/op_gen_lib.h"
+#include "tensorflow/core/kernels/logging_ops.h"
 #endif
 #include "tensorflow/c/c_api_internal.h"
 #include "tensorflow/core/common_runtime/device_mgr.h"
@@ -2954,4 +2955,11 @@ void TF_DeleteServer(TF_Server* server) {
   delete server;
 #endif
 }
+
+void TF_RegisterLogListener(void (*listener)(const char*)) {
+#ifndef __ANDROID__
+  tensorflow::logging::RegisterListener(listener);
+#endif
+}
+
 }  // end extern "C"
diff --git a/tensorflow/c/c_api.h b/tensorflow/c/c_api.h
index 8031928dac..ac04037937 100644
--- a/tensorflow/c/c_api.h
+++ b/tensorflow/c/c_api.h
@@ -1743,6 +1743,14 @@ TF_CAPI_EXPORT extern const char* TF_ServerTarget(TF_Server* server);
 // it will be stopped and joined.
 TF_CAPI_EXPORT extern void TF_DeleteServer(TF_Server* server);
 
+// Register a listener method that processes printed messages.
+//
+// If any listeners are registered, the print operator will call all listeners
+// with the printed messages and immediately return without writing to the
+// logs.
+TF_CAPI_EXPORT extern void TF_RegisterLogListener(
+    void (*listener)(const char*));
+
 #ifdef __cplusplus
 } /* end extern "C" */
 #endif
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index b6a421b795..e5ee109fff 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -5615,6 +5615,7 @@ filegroup(
         "gemm_functors.h",
         "image_resizer_state.h",
         "initializable_lookup_table.h",
+        "logging_ops.h",
         "lookup_table_init_op.h",
         "lookup_table_op.h",
         "lookup_util.h",
diff --git a/tensorflow/core/kernels/logging_ops.cc b/tensorflow/core/kernels/logging_ops.cc
index 2599340d78..bcd9f58642 100644
--- a/tensorflow/core/kernels/logging_ops.cc
+++ b/tensorflow/core/kernels/logging_ops.cc
@@ -13,7 +13,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include "tensorflow/core/kernels/logging_ops.h"
+
 #include <iostream>
+
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_split.h"
 #include "tensorflow/core/framework/op_kernel.h"
@@ -48,6 +51,22 @@ Status AppendStringToFile(const std::string& fname, StringPiece data,
 
 }  // namespace
 
+namespace logging {
+
+typedef std::vector<void (*)(const char*)> Listeners;
+
+Listeners* GetListeners() {
+  static Listeners* listeners = new Listeners;
+  return listeners;
+}
+
+bool RegisterListener(void (*listener)(const char*)) {
+  GetListeners()->push_back(listener);
+  return true;
+}
+
+}  // end namespace logging
+
 class AssertOp : public OpKernel {
  public:
   explicit AssertOp(OpKernelConstruction* ctx) : OpKernel(ctx) {
@@ -157,7 +176,12 @@ class PrintV2Op : public OpKernel {
       OP_REQUIRES_OK(ctx, AppendStringToFile(file_path_, msg, ctx->env()));
       return;
     }
-    if (output_stream_ == "stdout") {
+    auto listeners = logging::GetListeners();
+    if (!listeners->empty()) {
+      for (auto& listener : *listeners) {
+        listener(msg.data());
+      }
+    } else if (output_stream_ == "stdout") {
       std::cout << msg << std::endl;
     } else if (output_stream_ == "stderr") {
       std::cerr << msg << std::endl;
diff --git a/tensorflow/core/kernels/logging_ops.h b/tensorflow/core/kernels/logging_ops.h
new file mode 100644
index 0000000000..92a8d63409
--- /dev/null
+++ b/tensorflow/core/kernels/logging_ops.h
@@ -0,0 +1,33 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_KERNELS_LOGGING_OPS_H_
+#define TENSORFLOW_CORE_KERNELS_LOGGING_OPS_H_
+
+#include "absl/strings/str_cat.h"
+#include "absl/strings/str_split.h"
+
+namespace tensorflow {
+
+namespace logging {
+
+// Register a listener method to call on any printed messages.
+// Returns true if it is successfully registered.
+bool RegisterListener(void (*listener)(const char*));
+
+}  // namespace logging
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_KERNELS_LOGGING_OPS_H_
diff --git a/tensorflow/python/eager/pywrap_tfe.h b/tensorflow/python/eager/pywrap_tfe.h
index 8d6f212499..63440c04b9 100755
--- a/tensorflow/python/eager/pywrap_tfe.h
+++ b/tensorflow/python/eager/pywrap_tfe.h
@@ -234,4 +234,6 @@ PyObject* TFE_Py_TensorShapeOnDevice(PyObject* tensor);
 // for the defun function cache.
 PyObject* TFE_Py_EncodeArg(PyObject*);
 
+void TFE_Py_EnableInteractivePythonLogging();
+
 #endif  // TENSORFLOW_PYTHON_EAGER_PYWRAP_TFE_H_
diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc
index da1bb24fc7..085b3e29eb 100644
--- a/tensorflow/python/eager/pywrap_tfe_src.cc
+++ b/tensorflow/python/eager/pywrap_tfe_src.cc
@@ -3033,3 +3033,36 @@ PyObject* TFE_Py_EncodeArg(PyObject* arg) {
 
   return result.ToPyTuple();
 }
+
+// A method prints incoming messages directly to Python's
+// stdout using Python's C API. This is necessary in Jupyter notebooks
+// and colabs where messages to the C stdout don't go to the notebook
+// cell outputs, but calls to Python's stdout do.
+void PrintToPythonStdout(const char* msg) {
+  if (Py_IsInitialized()) {
+    PyGILState_STATE py_threadstate;
+    py_threadstate = PyGILState_Ensure();
+
+    string string_msg = msg;
+    // PySys_WriteStdout truncates strings over 1000 bytes, so
+    // we write the message in chunks small enough to not be truncated.
+    int CHUNK_SIZE = 900;
+    auto len = string_msg.length();
+    for (int i = 0; i < len; i += CHUNK_SIZE) {
+      PySys_WriteStdout("%s", string_msg.substr(i, CHUNK_SIZE).data());
+    }
+    PySys_WriteStdout("\n");
+
+    PyGILState_Release(py_threadstate);
+  }
+}
+
+// Register PrintToPythonStdout as a log listener, to allow
+// printing in colabs and jupyter notebooks to work.
+void TFE_Py_EnableInteractivePythonLogging() {
+  static bool enabled_interactive_logging = false;
+  if (!enabled_interactive_logging) {
+    enabled_interactive_logging = true;
+    TF_RegisterLogListener(PrintToPythonStdout);
+  }
+}
diff --git a/tensorflow/python/ops/logging_ops.py b/tensorflow/python/ops/logging_ops.py
index 3cb16eb81e..f05fbf4dd5 100644
--- a/tensorflow/python/ops/logging_ops.py
+++ b/tensorflow/python/ops/logging_ops.py
@@ -25,6 +25,7 @@ import sys
 
 import six
 
+from tensorflow.python import pywrap_tensorflow
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
@@ -40,6 +41,14 @@ from tensorflow.python.util import nest
 from tensorflow.python.util.deprecation import deprecated
 from tensorflow.python.util.tf_export import tf_export
 
+# Register printing to the cell output if we are in a Colab or Jupyter Notebook.
+try:
+  get_ipython()  # Exists in an ipython env like Jupyter or Colab
+  pywrap_tensorflow.TFE_Py_EnableInteractivePythonLogging()
+except NameError:
+  pass
+
+
 # The python wrapper for Assert is in control_flow_ops, as the Assert
 # call relies on certain conditionals for its dependencies.  Use
 # control_flow_ops.Assert.
@@ -193,9 +202,8 @@ def print_v2(*inputs, **kwargs):
     (This prints "tensors: [0 1 2 ... 7 8 9] {2: [0 2 4 ... 14 16 18]}" to
     sys.stdout)
 
-  Note: This op is only partially compatible with Jupyter notebooks and colabs.
-    Because it prints to the C++ standard out / standard error, this will go
-    in the notebook kernel's console output, not in the notebook cell output.
+  Note: In Jupyter notebooks and colabs, this operator prints to the notebook
+    cell outputs. It will not write to the notebook kernel's console logs.
 
   Args:
     *inputs: Positional arguments that are the inputs to print. Inputs in the
diff --git a/tensorflow/python/pywrap_tfe.i b/tensorflow/python/pywrap_tfe.i
index bae20aca50..dd74f12317 100755
--- a/tensorflow/python/pywrap_tfe.i
+++ b/tensorflow/python/pywrap_tfe.i
@@ -69,6 +69,7 @@ limitations under the License.
 %rename("%s") TFE_DeleteContextOptions;
 %rename("%s") TFE_Py_TensorShapeSlice;
 %rename("%s") TFE_Py_TensorShapeOnDevice;
+%rename("%s") TFE_Py_EnableInteractivePythonLogging;
 %rename("%s") TFE_ContextStartStep;
 %rename("%s") TFE_ContextEndStep;
 %rename("%s") TFE_Py_RegisterVSpace;
-- 
GitLab


From 3abed38c6ed4144a3a8faac1a818298a3b80c5de Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 4 Feb 2019 13:34:40 -0800
Subject: [PATCH 1863/2345] Internal change

PiperOrigin-RevId: 232356756
---
 tensorflow/BUILD                         |  6 +++
 tensorflow/core/BUILD                    | 60 ++++++++++++++++++++++++
 tensorflow/core/framework/reader_base.cc |  4 +-
 tensorflow/core/platform/logging.h       |  2 +-
 tensorflow/core/platform/platform.h      |  2 +-
 tensorflow/tensorflow.bzl                | 21 +++++++++
 6 files changed, 91 insertions(+), 4 deletions(-)

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 6bc8403d12..0b63ee4056 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -94,6 +94,12 @@ config_setting(
     visibility = ["//visibility:public"],
 )
 
+config_setting(
+    name = "emscripten",
+    values = {"crosstool_top": "//external:android/emscripten"},
+    visibility = ["//visibility:public"],
+)
+
 config_setting(
     name = "raspberry_pi_armeabi",
     values = {
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 3d92a836d1..50e80aef99 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -77,6 +77,7 @@ load(
     "//tensorflow:tensorflow.bzl",
     "cc_header_only_library",
     "if_android",
+    "if_emscripten",
     "if_ios",
     "if_linux_x86_64",
     "if_mobile",
@@ -87,10 +88,12 @@ load(
     "tf_copts",
     "tf_cuda_library",
     "tf_features_nomodules_if_android",
+    "tf_features_nomodules_if_emscripten",
     "tf_gen_op_libs",
     "tf_generate_proto_text_sources",
     "tf_genrule_cmd_append_to_srcs",
     "tf_opts_nortti_if_android",
+    "tf_opts_nortti_if_emscripten",
     "transitive_hdrs",
 )
 load("//tensorflow:tensorflow.bzl", "tf_cc_test_mkl")
@@ -1770,6 +1773,29 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "emscripten_tensorflow_lib_lite_nortti_lite_protos_no_runtime",
+    srcs = if_emscripten(["//tensorflow/core:mobile_srcs_no_runtime"]),
+    copts = ["-DSUPPORT_SELECTIVE_REGISTRATION"] + tf_opts_nortti_if_emscripten(),
+    defines = ["TENSORFLOW_LITE_PROTOS"],
+    linkopts = ["-lz"],
+    tags = [
+        "manual",
+        "notap",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":emscripten_proto_lib_no_rtti_lite_runtime",
+        ":mobile_additional_lib_deps",
+        ":stats_calculator_portable",
+        "//third_party/eigen3",
+        "@double_conversion//:double-conversion",
+        "@nsync//:nsync_cpp",
+        "@zlib_archive//:zlib",
+    ],
+    alwayslink = 1,
+)
+
 # Native library support for iOS applications.
 #
 # bazel  build --config=ios_x86_64 \
@@ -5037,6 +5063,40 @@ transitive_hdrs(
 # -----------------------------------------------------------------------------
 # Google-internal targets go here (must be at the end).
 
+load("//tensorflow:tensorflow.bzl", "tf_portable_proto_library")
+
+genrule(
+    name = "emscripten_proto_config_lite_runtime",
+    srcs = ["android_proto_config.asciipb"],  # Reuse the same file as android.
+    outs = ["emscripten_proto_config_lite_runtime.asciipb"],
+    cmd = tf_genrule_cmd_append_to_srcs("optimize_mode:LITE_RUNTIME"),
+    visibility = ["//visibility:private"],
+)
+
+# We are keeping the "android" version of tf_android_core_proto_headers. All it does is
+# normalize CORE_PROTO_SRCS to generate valid output file names.
+tf_portable_proto_library(
+    name = "emscripten_proto_lib_no_rtti_lite_runtime",
+    config = ":emscripten_proto_config_lite_runtime",
+    copts = tf_opts_nortti_if_emscripten(),
+    features = tf_features_nomodules_if_emscripten(),
+    header_outs = tf_android_core_proto_headers(CORE_PROTO_SRCS) + ["//google/protobuf/any.proto.h"],
+    link_full_protobuf = False,
+    prefix_dir = "emscripten_proto_no_rtti",
+    proto_deps = [
+        ":protos_all_cc",
+        "@protobuf_archive//:protobuf",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+# There is currently no need for a full proto version of emscripten tf lib lite.
+alias(
+    name = "emscripten_lib_lite_no_runtime",
+    actual = "//tensorflow/core:emscripten_tensorflow_lib_lite_nortti_lite_protos_no_runtime",
+    visibility = ["//visibility:public"],
+)
+
 alias(
     name = "android_srcs_no_runtime",
     actual = ":mobile_srcs_no_runtime",
diff --git a/tensorflow/core/framework/reader_base.cc b/tensorflow/core/framework/reader_base.cc
index f84ef0f953..ed4ff24039 100644
--- a/tensorflow/core/framework/reader_base.cc
+++ b/tensorflow/core/framework/reader_base.cc
@@ -241,7 +241,7 @@ Status ReaderBase::RestoreBaseState(const ReaderBaseState& state) {
   num_records_produced_ = state.num_records_produced();
   work_ = state.current_work();
   if (work_started_ < 0 || work_finished_ < 0 || num_records_produced_ < 0) {
-#ifdef __ANDROID__
+#if defined(__ANDROID__) || defined(__EMSCRIPTEN__)
     const string debug_string = "<debug state not available>";
 #else
     const string debug_string = state.DebugString();
@@ -251,7 +251,7 @@ Status ReaderBase::RestoreBaseState(const ReaderBaseState& state) {
         debug_string);
   }
   if (work_started_ > work_finished_) {
-#ifdef __ANDROID__
+#if defined(__ANDROID__) || (__EMSCRIPTEN__)
     const string debug_string = "<debug state not available>";
 #else
     const string debug_string = state.DebugString();
diff --git a/tensorflow/core/platform/logging.h b/tensorflow/core/platform/logging.h
index 17a5d5fb5b..7417ec8aff 100644
--- a/tensorflow/core/platform/logging.h
+++ b/tensorflow/core/platform/logging.h
@@ -19,7 +19,7 @@ limitations under the License.
 #include "tensorflow/core/platform/platform.h"  // To pick up PLATFORM_define
 
 #if defined(PLATFORM_GOOGLE) || defined(PLATFORM_GOOGLE_ANDROID) || \
-    defined(GOOGLE_LOGGING)
+    defined(GOOGLE_LOGGING) || defined(__EMSCRIPTEN__)
 #include "tensorflow/core/platform/google/build_config/logging.h"
 #else
 #include "tensorflow/core/platform/default/logging.h"
diff --git a/tensorflow/core/platform/platform.h b/tensorflow/core/platform/platform.h
index 0481b36871..671e5dd3c8 100644
--- a/tensorflow/core/platform/platform.h
+++ b/tensorflow/core/platform/platform.h
@@ -40,7 +40,7 @@ limitations under the License.
 #elif defined(_WIN32)
 #define PLATFORM_WINDOWS
 
-#elif defined(__arm__)
+#elif defined(__arm__) || defined(__EMSCRIPTEN__)
 #define PLATFORM_POSIX
 
 // Require an outside macro to tell us if we're building for Raspberry Pi or
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index fab1584cbb..6c8b445edd 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -97,6 +97,11 @@ def tf_android_core_proto_headers(core_proto_sources_relative):
         for p in core_proto_sources_relative
     ])
 
+# Wrapper for portable protos which currently just creates an empty rule.
+def tf_portable_proto_library(name, proto_deps, **kwargs):
+    _ignore = [kwargs]
+    native.cc_library(name = name, deps = proto_deps)
+
 # Sanitize a dependency so that it works correctly from code that includes
 # TensorFlow as a submodule.
 def clean_dep(dep):
@@ -146,6 +151,12 @@ def if_android(a):
         "//conditions:default": [],
     })
 
+def if_emscripten(a):
+    return select({
+        clean_dep("//tensorflow:emscripten"): a,
+        "//conditions:default": [],
+    })
+
 def if_ios(a):
     return select({
         clean_dep("//tensorflow:ios"): a,
@@ -306,9 +317,19 @@ def tf_opts_nortti_if_android():
 
 # LINT.ThenChange(//tensorflow/contrib/android/cmake/CMakeLists.txt)
 
+def tf_opts_nortti_if_emscripten():
+    return if_emscripten([
+        "-fno-rtti",
+        "-DGOOGLE_PROTOBUF_NO_RTTI",
+        "-DGOOGLE_PROTOBUF_NO_STATIC_INITIALIZER",
+    ])
+
 def tf_features_nomodules_if_android():
     return if_android(["-use_header_modules"])
 
+def tf_features_nomodules_if_emscripten():
+    return if_emscripten(["-use_header_modules"])
+
 # Given a list of "op_lib_names" (a list of files in the ops directory
 # without their .cc extensions), generate a library for that file.
 def tf_gen_op_libs(op_lib_names, deps = None, is_external = True):
-- 
GitLab


From 90f1af3e4487f5526ab9974ee07c63c125916499 Mon Sep 17 00:00:00 2001
From: Yifei Feng <yifeif@google.com>
Date: Mon, 4 Feb 2019 13:42:20 -0800
Subject: [PATCH 1864/2345] Remove the cudart dependency added in cl/231990786
 Add check for that core shared library never depends on cuda shared
 libraries.

PiperOrigin-RevId: 232358190
---
 .../core/platform/default/build_config/BUILD      | 15 +++++++++++++++
 tensorflow/stream_executor/cuda/BUILD             |  2 +-
 tensorflow/stream_executor/cuda/cudart_stub.cc    |  7 +++++++
 3 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/platform/default/build_config/BUILD b/tensorflow/core/platform/default/build_config/BUILD
index 6faf5c512f..845fe0ec04 100644
--- a/tensorflow/core/platform/default/build_config/BUILD
+++ b/tensorflow/core/platform/default/build_config/BUILD
@@ -7,6 +7,7 @@ licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
+load("//tensorflow:tensorflow.bzl", "check_deps")
 load("//tensorflow:tensorflow.bzl", "if_cuda")
 load("//tensorflow:tensorflow.bzl", "if_rocm")
 load("//tensorflow:tensorflow.bzl", "tf_copts")
@@ -283,6 +284,20 @@ cc_library(
     ],
 )
 
+# Check that libtensorflow_framework.so does not depend on cuda shared libraries.
+check_deps(
+    name = "libtensorflow_cuda_check_deps",
+    disallowed_deps = [
+        ":cuda",
+        "@local_config_cuda//cuda:cublas",
+        "@local_config_cuda//cuda:cuda_driver",
+        "@local_config_cuda//cuda:cudnn",
+        "@local_config_cuda//cuda:curand",
+        "@local_config_cuda//cuda:cusolver",
+    ],
+    deps = ["//tensorflow:libtensorflow_framework.so"],
+)
+
 cc_library(
     name = "rocm",
     data = [],
diff --git a/tensorflow/stream_executor/cuda/BUILD b/tensorflow/stream_executor/cuda/BUILD
index 0e23cc3eb4..36b3b167c7 100644
--- a/tensorflow/stream_executor/cuda/BUILD
+++ b/tensorflow/stream_executor/cuda/BUILD
@@ -83,11 +83,11 @@ cc_library(
     ]),
     deps = if_cuda_is_configured([
         ":cuda_diagnostics",
+        ":cudart_stub",
         "@com_google_absl//absl/base",
         "@com_google_absl//absl/container:inlined_vector",
         "@com_google_absl//absl/strings",
         "@local_config_cuda//cuda:cuda_headers",
-        "//tensorflow/core:cuda",
         "//tensorflow/stream_executor:device_options",
         "//tensorflow/stream_executor/gpu:gpu_driver_header",
         "//tensorflow/stream_executor/lib",
diff --git a/tensorflow/stream_executor/cuda/cudart_stub.cc b/tensorflow/stream_executor/cuda/cudart_stub.cc
index cb35719183..c5fc43d56c 100644
--- a/tensorflow/stream_executor/cuda/cudart_stub.cc
+++ b/tensorflow/stream_executor/cuda/cudart_stub.cc
@@ -111,4 +111,11 @@ cudaError_t CUDART_CB cudaPointerGetAttributes(
   if (!func_ptr) return GetSymbolNotFoundError();
   return func_ptr(attributes, ptr);
 }
+
+cudaError_t CUDART_CB cudaGetLastError() {
+  using FuncPtr = cudaError_t (*)();
+  static auto func_ptr = LoadSymbol<FuncPtr>("cudaGetLastError");
+  if (!func_ptr) return GetSymbolNotFoundError();
+  return func_ptr();
+}
 }  // extern "C"
-- 
GitLab


From 7f026bd2cd7a079e5d862e09c39eab074c7e8fb2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 4 Feb 2019 13:48:44 -0800
Subject: [PATCH 1865/2345] This CL moves tf.contrib.framework.is_tensor to
 core as tf.is_tensor, and updates the upgrade script.

PiperOrigin-RevId: 232359279
---
 tensorflow/python/framework/tensor_util.py           | 12 ++++++++----
 tensorflow/python/framework/tensor_util_test.py      | 10 ++++++++++
 tensorflow/tools/api/golden/v1/tensorflow.pbtxt      |  4 ++++
 tensorflow/tools/api/golden/v2/tensorflow.pbtxt      |  4 ++++
 tensorflow/tools/compatibility/tf_upgrade_v2.py      |  2 ++
 tensorflow/tools/compatibility/tf_upgrade_v2_test.py |  6 ++++++
 6 files changed, 34 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/framework/tensor_util.py b/tensorflow/python/framework/tensor_util.py
index 7de6653d49..21ded1a37f 100644
--- a/tensorflow/python/framework/tensor_util.py
+++ b/tensorflow/python/framework/tensor_util.py
@@ -22,6 +22,7 @@ import six
 
 from tensorflow.core.framework import tensor_pb2
 from tensorflow.core.framework import tensor_shape_pb2
+from tensorflow.python.framework import composite_tensor
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.util import compat
@@ -932,13 +933,15 @@ def constant_value_as_shape(tensor):  # pylint: disable=invalid-name
   return ret
 
 
+@tf_export("is_tensor")
 def is_tensor(x):  # pylint: disable=invalid-name
   """Check whether `x` is of tensor type.
 
-  Check whether an object is a tensor. This check is equivalent to calling
-  `isinstance(x, (tf.Tensor, tf.SparseTensor, tf.Variable))` and also checks
-  if all the component variables of a MirroredVariable or a ReplicaLocalVariable
-  are tensors.
+  Check whether an object is a tensor or a composite tensor. This check is
+  equivalent to calling
+  `isinstance(x, (tf.Tensor, tf.SparseTensor, tf.RaggedTensor, tf.Variable))`
+  and also checks if all the component variables of a MirroredVariable or a
+  ReplicaLocalVariable are tensors.
 
   Args:
     x: A python object to check.
@@ -947,4 +950,5 @@ def is_tensor(x):  # pylint: disable=invalid-name
     `True` if `x` is a tensor, `False` if not.
   """
   return (isinstance(x, ops._TensorLike) or ops.is_dense_tensor_like(x) or  # pylint: disable=protected-access
+          isinstance(x, composite_tensor.CompositeTensor) or
           (hasattr(x, "is_tensor_like") and x.is_tensor_like))
diff --git a/tensorflow/python/framework/tensor_util_test.py b/tensorflow/python/framework/tensor_util_test.py
index cdacdfaaad..e73df39035 100644
--- a/tensorflow/python/framework/tensor_util_test.py
+++ b/tensorflow/python/framework/tensor_util_test.py
@@ -773,6 +773,16 @@ class TensorUtilTest(test.TestCase):
       self.assertAllClose(np.array([10, 20, 30], dtype=np.int64), a)
 
 
+class IsTensorTest(test.TestCase):
+
+  @test_util.run_in_graph_and_eager_modes
+  def testConstantTensor(self):
+    np_val = np.random.rand(3).astype(np.int32)
+    tf_val = constant_op.constant(np_val)
+    self.assertFalse(tensor_util.is_tensor(np_val))
+    self.assertTrue(tensor_util.is_tensor(tf_val))
+
+
 class ConstantValueTest(test.TestCase):
 
   def testConstant(self):
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
index 56d06053f1..64f572022e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
@@ -1460,6 +1460,10 @@ tf_module {
     name: "is_strictly_increasing"
     argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "is_tensor"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "is_variable_initialized"
     argspec: "args=[\'variable\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index bb7ea2e13e..f0206823a7 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -668,6 +668,10 @@ tf_module {
     name: "init_scope"
     argspec: "args=[], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "is_tensor"
+    argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "less"
     argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index b774acd6d2..06edb5837c 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -537,6 +537,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.data.experimental.unbatch",
         "tf.contrib.data.unique":
             "tf.data.experimental.unique",
+        "tf.contrib.framework.is_tensor":
+            "tf.is_tensor",
         "tf.contrib.framework.nest.assert_same_structure":
             "tf.nest.assert_same_structure",
         "tf.contrib.framework.nest.flatten":
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 7668ba2260..3e7041aa03 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -1144,6 +1144,12 @@ def _log_prob(self, x):
     _, _, _, new_text = self._upgrade(text)
     self.assertEqual(expected, new_text)
 
+  def test_is_tensor_upgrade(self):
+    text = "tf.contrib.framework.is_tensor(x)"
+    expected = "tf.is_tensor(x)"
+    _, _, _, new_text = self._upgrade(text)
+    self.assertEqual(expected, new_text)
+
   def test_sample_distorted_bounding_box(self):
     # pylint: disable=line-too-long
     text = "tf.image.sample_distorted_bounding_box(a, b, c, d, e, f, g, h, i, j)"
-- 
GitLab


From df211323b834ead969130146297c9416853f1e65 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 4 Feb 2019 14:19:54 -0800
Subject: [PATCH 1866/2345] Add built-in operator AddN. It only supports float
 and int_32 now.

PiperOrigin-RevId: 232365698
---
 tensorflow/lite/BUILD                         |  4 +-
 tensorflow/lite/build_def.bzl                 |  1 +
 tensorflow/lite/g3doc/tf_ops_compatibility.md | 11 +++
 tensorflow/lite/kernels/BUILD                 | 13 +++
 tensorflow/lite/kernels/add_n.cc              | 88 +++++++++++++++++
 tensorflow/lite/kernels/add_n_test.cc         | 98 +++++++++++++++++++
 .../internal/reference/reference_ops.h        | 16 +++
 tensorflow/lite/kernels/register.cc           |  2 +
 tensorflow/lite/testing/generate_examples.py  | 45 +++++++++
 tensorflow/lite/toco/import_tensorflow.cc     |  2 +-
 tensorflow/lite/toco/tflite/operator.cc       | 21 ++++
 tensorflow/lite/toco/tflite/operator_test.cc  |  7 ++
 12 files changed, 305 insertions(+), 3 deletions(-)
 create mode 100644 tensorflow/lite/kernels/add_n.cc
 create mode 100644 tensorflow/lite/kernels/add_n_test.cc

diff --git a/tensorflow/lite/BUILD b/tensorflow/lite/BUILD
index 7d71ca32d5..8cc91a1a24 100644
--- a/tensorflow/lite/BUILD
+++ b/tensorflow/lite/BUILD
@@ -4,7 +4,7 @@ package(
 
 licenses(["notice"])  # Apache 2.0
 
-load("//tensorflow:tensorflow.bzl", "tf_cc_test", "if_not_windows")
+load("//tensorflow:tensorflow.bzl", "if_not_windows", "tf_cc_test")
 load("//tensorflow/lite:build_def.bzl", "tflite_copts")
 load("//tensorflow/lite:special_rules.bzl", "tflite_portable_test_suite")
 
@@ -190,7 +190,7 @@ cc_library(
         ":string",
         ":util",
         "//tensorflow/lite/c:c_api_internal",
-        "//tensorflow/lite/core/api:api",
+        "//tensorflow/lite/core/api",
         "//tensorflow/lite/nnapi:nnapi_implementation",
         "//tensorflow/lite/profiling:profiler",
         "//tensorflow/lite/schema:schema_fbs",
diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index 181590e5a9..88a8faf02e 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -226,6 +226,7 @@ def generated_test_models():
     return [
         "abs",
         "add",
+        "add_n",
         "arg_min_max",
         "avg_pool",
         "batch_to_space_nd",
diff --git a/tensorflow/lite/g3doc/tf_ops_compatibility.md b/tensorflow/lite/g3doc/tf_ops_compatibility.md
index cff4afc250..d7c71df9d8 100644
--- a/tensorflow/lite/g3doc/tf_ops_compatibility.md
+++ b/tensorflow/lite/g3doc/tf_ops_compatibility.md
@@ -165,6 +165,17 @@ Options {
 }
 ```
 
+**ADD_N**
+
+```
+Inputs {
+  0-N: any number of tensors (must have same size and shape)
+}
+Outputs {
+  0: elementwise sum of the input tensors
+}
+```
+
 **ARG_MAX**
 
 ```
diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index 826bb9a258..90e4d82618 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -152,6 +152,7 @@ cc_library(
     srcs = [
         "activations.cc",
         "add.cc",
+        "add_n.cc",
         "arg_min_max.cc",
         "audio_spectrogram.cc",
         "basic_rnn.cc",
@@ -355,6 +356,18 @@ tf_cc_test(
     ],
 )
 
+tf_cc_test(
+    name = "add_n_test",
+    size = "small",
+    srcs = ["add_n_test.cc"],
+    deps = [
+        ":builtin_ops",
+        ":test_util",
+        "//tensorflow/lite:framework",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
 tf_cc_test(
     name = "arg_min_max_test",
     size = "small",
diff --git a/tensorflow/lite/kernels/add_n.cc b/tensorflow/lite/kernels/add_n.cc
new file mode 100644
index 0000000000..3e9b2ea24a
--- /dev/null
+++ b/tensorflow/lite/kernels/add_n.cc
@@ -0,0 +1,88 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/kernels/internal/reference/reference_ops.h"
+#include "tensorflow/lite/kernels/internal/tensor.h"
+#include "tensorflow/lite/kernels/internal/tensor_ctypes.h"
+#include "tensorflow/lite/kernels/kernel_util.h"
+
+namespace tflite {
+namespace ops {
+namespace builtin {
+namespace add_n {
+
+constexpr int kInputTensor1 = 0;
+constexpr int kOutputTensor = 0;
+
+TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
+  int num_inputs = NumInputs(node);
+  TF_LITE_ENSURE(context, num_inputs >= 2);
+  TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);
+
+  const TfLiteTensor* input1 = GetInput(context, node, kInputTensor1);
+  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+  output->type = input1->type;
+
+  // Check that all input tensors have the same shape and type.
+  for (int i = kInputTensor1 + 1; i < num_inputs; ++i) {
+    const TfLiteTensor* input = GetInput(context, node, i);
+    TF_LITE_ENSURE(context, HaveSameShapes(input1, input));
+    TF_LITE_ENSURE_EQ(context, input1->type, input->type);
+  }
+
+  // Use the first input node's dimension to be the dimension of the output
+  // node.
+  TfLiteIntArray* input1_dims = input1->dims;
+  TfLiteIntArray* output_dims = TfLiteIntArrayCopy(input1_dims);
+  return context->ResizeTensor(context, output, output_dims);
+}
+
+template <typename T>
+void EvalAddN(TfLiteContext* context, TfLiteNode* node) {
+  // TODO(haoliang): Initialize all_inputs only once during init.
+  VectorOfTensors<T> all_inputs(*context, *node->inputs);
+  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+  int num_inputs = NumInputs(node);
+  const TfLiteTensor* input1 = GetInput(context, node, kInputTensor1);
+  reference_ops::AddN<T>(GetTensorShape(input1), num_inputs, all_inputs.data(),
+                         GetTensorData<T>(output));
+}
+
+TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
+  const TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+  if (output->type == kTfLiteFloat32) {
+    EvalAddN<float>(context, node);
+  } else if (output->type == kTfLiteInt32) {
+    EvalAddN<int32_t>(context, node);
+  } else {
+    context->ReportError(context,
+                         "AddN only supports FLOAT32|INT32 now, got %s.",
+                         TfLiteTypeGetName(output->type));
+    return kTfLiteError;
+  }
+  return kTfLiteOk;
+}
+
+}  // namespace add_n
+
+TfLiteRegistration* Register_ADD_N() {
+  static TfLiteRegistration r = {/*init*/ nullptr, /*free*/ nullptr,
+                                 add_n::Prepare, add_n::Eval};
+  return &r;
+}
+
+}  // namespace builtin
+}  // namespace ops
+}  // namespace tflite
diff --git a/tensorflow/lite/kernels/add_n_test.cc b/tensorflow/lite/kernels/add_n_test.cc
new file mode 100644
index 0000000000..ee9477d2ff
--- /dev/null
+++ b/tensorflow/lite/kernels/add_n_test.cc
@@ -0,0 +1,98 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <vector>
+
+#include <gtest/gtest.h>
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/kernels/test_util.h"
+#include "tensorflow/lite/model.h"
+
+namespace tflite {
+namespace {
+
+using ::testing::ElementsAreArray;
+
+class BaseAddNOpModel : public SingleOpModel {
+ public:
+  BaseAddNOpModel(const std::vector<TensorData>& inputs,
+                  const TensorData& output) {
+    int num_inputs = inputs.size();
+    std::vector<std::vector<int>> input_shapes;
+
+    for (int i = 0; i < num_inputs; ++i) {
+      inputs_.push_back(AddInput(inputs[i]));
+      input_shapes.push_back(GetShape(inputs_[i]));
+    }
+
+    output_ = AddOutput(output);
+    SetBuiltinOp(BuiltinOperator_ADD_N, BuiltinOptions_AddNOptions,
+                 CreateAddNOptions(builder_).Union());
+    BuildInterpreter(input_shapes);
+  }
+
+  int input(int i) { return inputs_[i]; }
+
+ protected:
+  std::vector<int> inputs_;
+  int output_;
+};
+
+class FloatAddNOpModel : public BaseAddNOpModel {
+ public:
+  using BaseAddNOpModel::BaseAddNOpModel;
+
+  std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
+};
+
+class IntegerAddNOpModel : public BaseAddNOpModel {
+ public:
+  using BaseAddNOpModel::BaseAddNOpModel;
+
+  std::vector<int32_t> GetOutput() { return ExtractVector<int32_t>(output_); }
+};
+
+TEST(FloatAddNOpModel, AddMultipleTensors) {
+  FloatAddNOpModel m({{TensorType_FLOAT32, {1, 2, 2, 1}},
+                      {TensorType_FLOAT32, {1, 2, 2, 1}},
+                      {TensorType_FLOAT32, {1, 2, 2, 1}}},
+                     {TensorType_FLOAT32, {}});
+  m.PopulateTensor<float>(m.input(0), {-2.0, 0.2, 0.7, 0.8});
+  m.PopulateTensor<float>(m.input(1), {0.1, 0.2, 0.3, 0.5});
+  m.PopulateTensor<float>(m.input(2), {0.5, 0.1, 0.1, 0.2});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutput(), ElementsAreArray({-1.4, 0.5, 1.1, 1.5}));
+}
+
+TEST(IntegerAddNOpModel, AddMultipleTensors) {
+  IntegerAddNOpModel m({{TensorType_INT32, {1, 2, 2, 1}},
+                        {TensorType_INT32, {1, 2, 2, 1}},
+                        {TensorType_INT32, {1, 2, 2, 1}}},
+                       {TensorType_INT32, {}});
+  m.PopulateTensor<int32_t>(m.input(0), {-20, 2, 7, 8});
+  m.PopulateTensor<int32_t>(m.input(1), {1, 2, 3, 5});
+  m.PopulateTensor<int32_t>(m.input(2), {10, -5, 1, -2});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutput(), ElementsAreArray({-9, -1, 11, 11}));
+}
+
+}  // namespace
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/lite/kernels/internal/reference/reference_ops.h b/tensorflow/lite/kernels/internal/reference/reference_ops.h
index ac6905fcd7..84f62b1c97 100644
--- a/tensorflow/lite/kernels/internal/reference/reference_ops.h
+++ b/tensorflow/lite/kernels/internal/reference/reference_ops.h
@@ -702,6 +702,22 @@ inline void Add(const ArithmeticParams& params,
   }
 }
 
+// T is expected to be either float or int.
+template <typename T>
+inline void AddN(const RuntimeShape& input_shape, const size_t num_inputs,
+                 T* const* input_data, T* output_data) {
+  // All inputs and output should have the same shape, this is checked during
+  // Prepare stage.
+  const size_t size = input_shape.FlatSize();
+  for (int i = 0; i < size; ++i) {
+    T x = 0;
+    for (int j = 0; j < num_inputs; ++j) {
+      x += input_data[j][i];
+    }
+    output_data[i] = x;
+  }
+}
+
 // Element-wise add that can often be used for inner loop of broadcast add as
 // well as the non-broadcast add.
 inline void AddElementwise(int size, const ArithmeticParams& params,
diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index 12a53f93b8..aad6deb4d8 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -131,6 +131,7 @@ TfLiteRegistration* Register_FILL();
 TfLiteRegistration* Register_MIRROR_PAD();
 TfLiteRegistration* Register_UNIQUE();
 TfLiteRegistration* Register_REVERSE_V2();
+TfLiteRegistration* Register_ADD_N();
 
 TfLiteStatus UnsupportedTensorFlowOp(TfLiteContext* context, TfLiteNode* node) {
   context->ReportError(
@@ -295,6 +296,7 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_MIRROR_PAD, Register_MIRROR_PAD());
   AddBuiltin(BuiltinOperator_UNIQUE, Register_UNIQUE());
   AddBuiltin(BuiltinOperator_REVERSE_V2, Register_REVERSE_V2());
+  AddBuiltin(BuiltinOperator_ADD_N, Register_ADD_N());
 
   // TODO(andrewharp, ahentz): Move these somewhere more appropriate so that
   // custom ops aren't always included by default.
diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index ee68607a0f..7ebc5950b2 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -1184,6 +1184,51 @@ def make_add_tests(zip_path):
   make_binary_op_tests(zip_path, tf.add)
 
 
+def make_add_n_tests(zip_path):
+  """Make a set of tests for AddN op."""
+
+  test_parameters = [
+      {
+          "dtype": [tf.float32, tf.int32],
+          "input_shape": [[2, 5, 3, 1]],
+          "num_inputs": [2, 3, 4, 5],
+      },
+      {
+          "dtype": [tf.float32, tf.int32],
+          "input_shape": [[5]],
+          "num_inputs": [2, 3, 4, 5],
+      },
+      {
+          "dtype": [tf.float32, tf.int32],
+          "input_shape": [[]],
+          "num_inputs": [2, 3, 4, 5],
+      },
+  ]
+
+  def build_graph(parameters):
+    """Builds the graph given the current parameters."""
+    input_tensors = []
+    for i in range(parameters["num_inputs"]):
+      input_tensors.append(
+          tf.placeholder(
+              dtype=parameters["dtype"],
+              name="input_{}".format(i),
+              shape=parameters["input_shape"]))
+    out = tf.add_n(input_tensors)
+    return input_tensors, [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    """Builds operand inputs for op."""
+    input_data = []
+    for i in range(parameters["num_inputs"]):
+      input_data.append(
+          create_tensor_data(parameters["dtype"], parameters["input_shape"]))
+    return input_data, sess.run(
+        outputs, feed_dict={i: d for i, d in zip(inputs, input_data)})
+
+  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+
+
 def make_div_tests(zip_path):
   make_binary_op_tests(zip_path, tf.div)
 
diff --git a/tensorflow/lite/toco/import_tensorflow.cc b/tensorflow/lite/toco/import_tensorflow.cc
index b1b04949ba..813e439995 100644
--- a/tensorflow/lite/toco/import_tensorflow.cc
+++ b/tensorflow/lite/toco/import_tensorflow.cc
@@ -2375,7 +2375,7 @@ ConverterMapType GetTensorFlowNodeConverterMap() {
   return std::unordered_map<std::string, ConverterType>({
       {"Abs", ConvertSimpleOperator<AbsOperator, kAnyNumInputs, 1>},
       {"Add", ConvertSimpleOperator<AddOperator, 2, 1>},
-      {"AddN", ConvertSimpleOperatorFlexOk<AddNOperator, kAnyNumInputs, 1>},
+      {"AddN", ConvertSimpleOperator<AddNOperator, kAnyNumInputs, 1>},
       {"All", ConvertSimpleOperator<TensorFlowAllOperator, kAnyNumInputs, 1>},
       {"Any", ConvertReduceOperator<TensorFlowAnyOperator>},
       {"ArgMax", ConvertArgMaxOperator},
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index 81203dfa04..4ce3aa9218 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -203,6 +203,25 @@ class Add : public BuiltinOperator<AddOperator, ::tflite::AddOptions,
   }
 };
 
+class AddN : public BuiltinOperator<AddNOperator, ::tflite::AddNOptions,
+                                    ::tflite::BuiltinOptions_AddNOptions> {
+ public:
+  using BuiltinOperator::BuiltinOperator;
+
+  flatbuffers::Offset<TfLiteOptions> WriteOptions(
+      const TocoOperator& op,
+      flatbuffers::FlatBufferBuilder* builder) const override {
+    return ::tflite::CreateAddNOptions(*builder);
+  }
+
+  void ReadOptions(const TfLiteOptions& options,
+                   TocoOperator* op) const override {}
+
+  int GetVersion(const OperatorSignature& op_signature) const override {
+    return 1;
+  }
+};
+
 class SpaceToBatchND
     : public BuiltinOperator<SpaceToBatchNDOperator,
                              ::tflite::SpaceToBatchNDOptions,
@@ -1903,6 +1922,8 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
   // Builtin Operators.
   ops.push_back(
       MakeUnique<Add>(::tflite::BuiltinOperator_ADD, OperatorType::kAdd));
+  ops.push_back(
+      MakeUnique<AddN>(::tflite::BuiltinOperator_ADD_N, OperatorType::kAddN));
   ops.push_back(
       MakeUnique<Div>(::tflite::BuiltinOperator_DIV, OperatorType::kDiv));
   ops.push_back(
diff --git a/tensorflow/lite/toco/tflite/operator_test.cc b/tensorflow/lite/toco/tflite/operator_test.cc
index 88f68f7ebf..43b52c4e93 100644
--- a/tensorflow/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/lite/toco/tflite/operator_test.cc
@@ -164,6 +164,13 @@ TEST_F(OperatorTest, BuiltinAdd) {
             output_toco_op->fused_activation_function);
 }
 
+TEST_F(OperatorTest, BuiltinAddN) {
+  AddNOperator op;
+  auto output_toco_op =
+      SerializeAndDeserialize(GetOperator("ADD_N", OperatorType::kAddN), op);
+  ASSERT_NE(output_toco_op.get(), nullptr);
+}
+
 TEST_F(OperatorTest, BuiltinReducerOps) {
   CheckReducerOperator<MeanOperator>("MEAN", OperatorType::kMean);
   CheckReducerOperator<TensorFlowSumOperator>("SUM", OperatorType::kSum);
-- 
GitLab


From 7889a8d58f2337b5783c975bff408994d6e5aecf Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 4 Feb 2019 14:22:10 -0800
Subject: [PATCH 1867/2345] Allow eager to compile on Android.

PiperOrigin-RevId: 232366131
---
 tensorflow/core/common_runtime/eager/context.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/core/common_runtime/eager/context.cc b/tensorflow/core/common_runtime/eager/context.cc
index 3a9d1e02be..12e6483d6b 100644
--- a/tensorflow/core/common_runtime/eager/context.cc
+++ b/tensorflow/core/common_runtime/eager/context.cc
@@ -20,9 +20,11 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/device_resolver_local.h"
 #include "tensorflow/core/common_runtime/device_set.h"
 #include "tensorflow/core/common_runtime/process_util.h"
+#ifndef __ANDROID__
 #include "tensorflow/core/distributed_runtime/collective_param_resolver_distributed.h"
 #include "tensorflow/core/distributed_runtime/device_resolver_distributed.h"
 #include "tensorflow/core/distributed_runtime/rpc_collective_executor_mgr.h"
+#endif
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/lib/core/blocking_counter.h"
 #include "tensorflow/core/util/env_var.h"
-- 
GitLab


From a0fdd6c68c149b55529ab8da07354256d01ddc6b Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Mon, 4 Feb 2019 14:31:15 -0800
Subject: [PATCH 1868/2345] Update default max_workspace_size_bytes to 1GB. 2MB
 is too small to produce good perf result as a default.

PiperOrigin-RevId: 232367938
---
 .../contrib/tensorrt/python/trt_convert.py    | 29 +++----
 .../python/compiler/tensorrt/trt_convert.py   | 77 ++++++++++---------
 .../compiler/tensorrt/trt_convert_test.py     |  8 ++
 3 files changed, 65 insertions(+), 49 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/python/trt_convert.py b/tensorflow/contrib/tensorrt/python/trt_convert.py
index 42bf1eda51..4a95937813 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert.py
@@ -21,20 +21,21 @@ from __future__ import print_function
 from tensorflow.python.compiler.tensorrt import trt_convert
 
 
-def create_inference_graph(input_graph_def,
-                           outputs,
-                           max_batch_size=1,
-                           max_workspace_size_bytes=2 << 20,
-                           precision_mode=trt_convert.TrtPrecisionMode.FP32,
-                           minimum_segment_size=3,
-                           is_dynamic_op=False,
-                           maximum_cached_engines=1,
-                           cached_engine_batches=None,
-                           use_calibration=True,
-                           input_saved_model_dir=None,
-                           input_saved_model_tags=None,
-                           output_saved_model_dir=None,
-                           session_config=None):
+def create_inference_graph(
+    input_graph_def,
+    outputs,
+    max_batch_size=1,
+    max_workspace_size_bytes=trt_convert.DEFAULT_TRT_MAX_WORKSPACE_SIZE_BYTES,
+    precision_mode=trt_convert.TrtPrecisionMode.FP32,
+    minimum_segment_size=3,
+    is_dynamic_op=False,
+    maximum_cached_engines=1,
+    cached_engine_batches=None,
+    use_calibration=True,
+    input_saved_model_dir=None,
+    input_saved_model_tags=None,
+    output_saved_model_dir=None,
+    session_config=None):
   return trt_convert.create_inference_graph(
       input_graph_def=input_graph_def,
       outputs=outputs,
diff --git a/tensorflow/python/compiler/tensorrt/trt_convert.py b/tensorflow/python/compiler/tensorrt/trt_convert.py
index ca7872b478..cff22ec3ac 100644
--- a/tensorflow/python/compiler/tensorrt/trt_convert.py
+++ b/tensorflow/python/compiler/tensorrt/trt_convert.py
@@ -70,15 +70,21 @@ class TrtPrecisionMode(object):
     return [TrtPrecisionMode.FP32, TrtPrecisionMode.FP16, TrtPrecisionMode.INT8]
 
 
-def get_tensorrt_rewriter_config(rewriter_config=None,
-                                 max_batch_size=1,
-                                 max_workspace_size_bytes=2 << 20,
-                                 precision_mode=TrtPrecisionMode.FP32,
-                                 minimum_segment_size=3,
-                                 is_dynamic_op=False,
-                                 maximum_cached_engines=1,
-                                 cached_engine_batches=None,
-                                 use_calibration=True):
+# Use a large enough number as the default max_workspace_size for TRT engines,
+# so it can produce reasonable performance results with the default.
+DEFAULT_TRT_MAX_WORKSPACE_SIZE_BYTES = 1 << 30
+
+
+def get_tensorrt_rewriter_config(
+    rewriter_config=None,
+    max_batch_size=1,
+    max_workspace_size_bytes=DEFAULT_TRT_MAX_WORKSPACE_SIZE_BYTES,
+    precision_mode=TrtPrecisionMode.FP32,
+    minimum_segment_size=3,
+    is_dynamic_op=False,
+    maximum_cached_engines=1,
+    cached_engine_batches=None,
+    use_calibration=True):
   """Returns a RewriterConfig proto for TRT transformation.
 
   Args:
@@ -97,12 +103,12 @@ def get_tensorrt_rewriter_config(rewriter_config=None,
       If the number of cached engines is already at max but none of them can
       serve the input, the TRTEngineOp will fall back to run the TF function
       based on which the TRTEngineOp is created.
-    cached_engine_batches: a list of batch sizes used to create cached
-      engines, only used when is_dynamic_op is True. The length of the list
-      should be <= maximum_cached_engines, and the dynamic TRT op will
-      use this list to determine the batch sizes of the cached engines, instead
-      of making the decision on the fly. This is useful when we know the most
-      common batch size(s) the application is going to generate.
+    cached_engine_batches: a list of batch sizes used to create cached engines,
+      only used when is_dynamic_op is True. The length of the list should be <=
+      maximum_cached_engines, and the dynamic TRT op will use this list to
+      determine the batch sizes of the cached engines, instead of making the
+      decision on the fly. This is useful when we know the most common batch
+      size(s) the application is going to generate.
     use_calibration: this argument is ignored if precision_mode is not INT8. If
       set to True, a calibration graph will be created to calibrate the missing
       ranges. The calibration graph must be converted to an inference graph
@@ -162,20 +168,21 @@ def get_tensorrt_rewriter_config(rewriter_config=None,
   return rewriter_config_with_trt
 
 
-def create_inference_graph(input_graph_def,
-                           outputs,
-                           max_batch_size=1,
-                           max_workspace_size_bytes=2 << 20,
-                           precision_mode=TrtPrecisionMode.FP32,
-                           minimum_segment_size=3,
-                           is_dynamic_op=False,
-                           maximum_cached_engines=1,
-                           cached_engine_batches=None,
-                           use_calibration=True,
-                           input_saved_model_dir=None,
-                           input_saved_model_tags=None,
-                           output_saved_model_dir=None,
-                           session_config=None):
+def create_inference_graph(
+    input_graph_def,
+    outputs,
+    max_batch_size=1,
+    max_workspace_size_bytes=DEFAULT_TRT_MAX_WORKSPACE_SIZE_BYTES,
+    precision_mode=TrtPrecisionMode.FP32,
+    minimum_segment_size=3,
+    is_dynamic_op=False,
+    maximum_cached_engines=1,
+    cached_engine_batches=None,
+    use_calibration=True,
+    input_saved_model_dir=None,
+    input_saved_model_tags=None,
+    output_saved_model_dir=None,
+    session_config=None):
   """Python wrapper for the TRT transformation.
 
   Args:
@@ -197,12 +204,12 @@ def create_inference_graph(input_graph_def,
       If the number of cached engines is already at max but none of them can
       serve the input, the TRTEngineOp will fall back to run the TF function
       based on which the TRTEngineOp is created.
-    cached_engine_batches: a list of batch sizes used to create cached
-      engines, only used when is_dynamic_op is True. The length of the list
-      should be <= maximum_cached_engines, and the dynamic TRT op will
-      use this list to determine the batch sizes of the cached engines, instead
-      of making the decision on the fly. This is useful when we know the most
-      common batch size(s) the application is going to generate.
+    cached_engine_batches: a list of batch sizes used to create cached engines,
+      only used when is_dynamic_op is True. The length of the list should be <=
+      maximum_cached_engines, and the dynamic TRT op will use this list to
+      determine the batch sizes of the cached engines, instead of making the
+      decision on the fly. This is useful when we know the most common batch
+      size(s) the application is going to generate.
     use_calibration: this argument is ignored if precision_mode is not INT8. If
       set to True, a calibration graph will be created to calibrate the missing
       ranges. The calibration graph must be converted to an inference graph
diff --git a/tensorflow/python/compiler/tensorrt/trt_convert_test.py b/tensorflow/python/compiler/tensorrt/trt_convert_test.py
index 7f839117cd..c665f616fa 100644
--- a/tensorflow/python/compiler/tensorrt/trt_convert_test.py
+++ b/tensorflow/python/compiler/tensorrt/trt_convert_test.py
@@ -47,6 +47,10 @@ from tensorflow.python.tools import saved_model_utils
 class TrtConvertTest(test_util.TensorFlowTestCase):
   """Class to test Tensorflow-TensorRT integration python API."""
 
+  # Use a small max_workspace_size for tests so they don't consume too much GPU
+  # memory.
+  _TRT_MAX_WORKSPACE_SIZE_BYTES = 2 << 20
+
   def testGetTensorrtRewriterConfig(self):
     """Test case for trt_convert.get_tensorrt_rewriter_config()."""
     rewriter_cfg = trt_convert.get_tensorrt_rewriter_config(
@@ -147,6 +151,7 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
     input_graph_def = None if input_saved_model_dir else self._GetGraphDef()
     output_graph_def = trt_convert.create_inference_graph(
         input_graph_def, ["output"],
+        max_workspace_size_bytes=TrtConvertTest._TRT_MAX_WORKSPACE_SIZE_BYTES,
         input_saved_model_dir=input_saved_model_dir,
         output_saved_model_dir=output_saved_model_dir,
         session_config=self._GetConfigProto())
@@ -200,6 +205,7 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
     output_graph_def = trt_convert.create_inference_graph(
         self._GetGraphDef(), ["output"],
         minimum_segment_size=5,
+        max_workspace_size_bytes=TrtConvertTest._TRT_MAX_WORKSPACE_SIZE_BYTES,
         is_dynamic_op=False)
     node_name_to_op = {node.name: node.op for node in output_graph_def.node}
     self.assertEqual({
@@ -223,6 +229,7 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
     output_graph_def = trt_convert.create_inference_graph(
         None,
         None,
+        max_workspace_size_bytes=TrtConvertTest._TRT_MAX_WORKSPACE_SIZE_BYTES,
         is_dynamic_op=True,
         maximum_cached_engines=2,
         input_saved_model_dir=input_saved_model_dir,
@@ -266,6 +273,7 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
         None,
         None,
         max_batch_size=1,
+        max_workspace_size_bytes=TrtConvertTest._TRT_MAX_WORKSPACE_SIZE_BYTES,
         is_dynamic_op=False,
         maximum_cached_engines=2,  # This is noop, added just for testing.
         input_saved_model_dir=input_saved_model_dir,
-- 
GitLab


From 6e0f86d12ac8185d889da02ecddc89b140eb0c81 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Mon, 4 Feb 2019 14:36:31 -0800
Subject: [PATCH 1869/2345] Allow get_concrete_function() in 1.x-style Graphs

Makes it easier to force a trace before running the global variables initializer.

PiperOrigin-RevId: 232369014
---
 tensorflow/python/eager/def_function.py  |  1 -
 tensorflow/python/eager/function_test.py | 19 ++++++++++++++++++-
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py
index 43cb07e79d..59c6608965 100644
--- a/tensorflow/python/eager/def_function.py
+++ b/tensorflow/python/eager/def_function.py
@@ -623,7 +623,6 @@ class Function(object):
     Raises:
       ValueError: if this object has not yet been called on concrete values.
     """
-    assert context.executing_eagerly()
     if self._stateful_fn is None:
       initializer_map = {}
       self._initialize(args, kwargs, add_initializers_to=initializer_map)
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index c8f7ecbca2..33ae062f5d 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -466,6 +466,22 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     value = tensor_init()
     self.assertAllEqual(value, 2.0)
 
+  @test_util.run_in_graph_and_eager_modes
+  def testGetConcreteFunctionCreatesVariables(self):
+
+    v_holder = []
+
+    @def_function.function
+    def tensor_init():
+      if not v_holder:
+        v_holder.append(variables.Variable(5.))
+      return v_holder[0].read_value()
+
+    concrete = tensor_init.get_concrete_function()
+    self.evaluate(variables.global_variables_initializer())
+    self.assertAllEqual(5., self.evaluate(concrete()))
+    self.assertAllEqual(5., self.evaluate(tensor_init()))
+
   def testDefunShapeInferenceWithCapturedResourceVariable(self):
     v = resource_variable_ops.ResourceVariable([[1, 2], [3, 4]])
 
@@ -1057,7 +1073,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     def func():
       return constant_op.constant(0)
 
-    defined = function.defun(func)
+    defined = def_function.function(func)
     with ops.device('cpu:0'):
       cpu_graph_function = defined.get_concrete_function()
 
@@ -1324,6 +1340,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
                                  'tuple or a list.*'):
       function.defun(foo, input_signature=signature)
 
+  @test_util.run_in_graph_and_eager_modes
   def testInputsIncompatibleWithSignatureRaisesError(self):
 
     def foo(a):
-- 
GitLab


From 877e6f3c3d12d424cb9aaa5342a6acc78b25deec Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Mon, 4 Feb 2019 14:49:12 -0800
Subject: [PATCH 1870/2345] [XLA] Operands should have the same element types
 as the corresponding to_apply computation parameters for map, reduce and
 reduce window.

Fix ShapeVerifier for this and add HLO verifier tests.

PiperOrigin-RevId: 232371312
---
 .../compiler/xla/service/hlo_verifier.cc      | 52 +++++++++++++---
 .../compiler/xla/service/hlo_verifier_test.cc | 62 +++++++++++++++++++
 2 files changed, 106 insertions(+), 8 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc
index 4caaa5a32b..12d1b4ac2c 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier.cc
@@ -376,6 +376,24 @@ Status ShapeVerifier::HandleGetTupleElement(HloInstruction* get_tuple_element) {
                         get_tuple_element->tuple_index()));
 }
 
+namespace {
+Status SameElementTypesForOperandsAndToApplyParameters(
+    const HloInstruction& instruction, int64 num_operands_to_check) {
+  const ProgramShape& to_apply = instruction.to_apply()->ComputeProgramShape();
+  for (int i = 0; i < num_operands_to_check; ++i) {
+    const Shape& parameter_shape = to_apply.parameters(i);
+    const Shape& operand_shape = instruction.operands()[i]->shape();
+    if (!ShapeUtil::SameElementType(parameter_shape, operand_shape)) {
+      return InvalidArgument(
+          "Shape mismatch between to_apply computation"
+          " parameter and operand %d in %s.",
+          i, instruction.ToString().c_str());
+    }
+  }
+  return Status::OK();
+}
+}  // namespace
+
 Status ShapeVerifier::HandleReduce(HloInstruction* reduce) {
   if (reduce->operand_count() % 2 != 0) {
     return InternalError(
@@ -387,9 +405,15 @@ Status ShapeVerifier::HandleReduce(HloInstruction* reduce) {
   for (const HloInstruction* operand : reduce->operands()) {
     operand_shapes.push_back(&operand->shape());
   }
-  return CheckShape(reduce, ShapeInference::InferReduceShape(
-                                operand_shapes, reduce->dimensions(),
-                                reduce->to_apply()->ComputeProgramShape()));
+  TF_RETURN_IF_ERROR(
+      CheckShape(reduce, ShapeInference::InferReduceShape(
+                             operand_shapes, reduce->dimensions(),
+                             reduce->to_apply()->ComputeProgramShape())));
+
+  return allow_mixed_precision_
+             ? Status::OK()
+             : SameElementTypesForOperandsAndToApplyParameters(
+                   *reduce, reduce->operands().size() - 1);
 }
 
 Status ShapeVerifier::HandleBitcast(HloInstruction* bitcast) {
@@ -545,19 +569,31 @@ Status ShapeVerifier::HandleMap(HloInstruction* map) {
   // arbitrary map dimensions.
   std::vector<int64> map_dims(max_operand_rank);
   std::iota(map_dims.begin(), map_dims.end(), 0);
-  return CheckShape(map, ShapeInference::InferMapShape(
-                             operand_shapes,
-                             map->to_apply()->ComputeProgramShape(), map_dims));
+
+  TF_RETURN_IF_ERROR(CheckShape(
+      map,
+      ShapeInference::InferMapShape(
+          operand_shapes, map->to_apply()->ComputeProgramShape(), map_dims)));
+
+  return allow_mixed_precision_
+             ? Status::OK()
+             : SameElementTypesForOperandsAndToApplyParameters(
+                   *map, map->operands().size());
 }
 
 Status ShapeVerifier::HandleReduceWindow(HloInstruction* reduce_window) {
   TF_RETURN_IF_ERROR(CheckOperandCount(reduce_window, 2));
-  return CheckShape(
+  TF_RETURN_IF_ERROR(CheckShape(
       reduce_window,
       ShapeInference::InferReduceWindowShape(
           reduce_window->operand(0)->shape(),
           reduce_window->operand(1)->shape(), reduce_window->window(),
-          reduce_window->to_apply()->ComputeProgramShape()));
+          reduce_window->to_apply()->ComputeProgramShape())));
+
+  return allow_mixed_precision_
+             ? Status::OK()
+             : SameElementTypesForOperandsAndToApplyParameters(*reduce_window,
+                                                               1);
 }
 
 Status ShapeVerifier::HandleSelectAndScatter(HloInstruction* instruction) {
diff --git a/tensorflow/compiler/xla/service/hlo_verifier_test.cc b/tensorflow/compiler/xla/service/hlo_verifier_test.cc
index 4f69bd155b..523890b3c7 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier_test.cc
@@ -552,5 +552,67 @@ TEST_F(HloVerifierTest, IotaNonArrayResult) {
               HasSubstr("does not support non-array result"));
 }
 
+static const char* const kMapOperandComputationMismatchHlo = R"(
+  HloModule MapOperandComputationMismatch
+
+  Computation {
+    param0 = f32[] parameter(0)
+    constant = f32[] constant(1)
+    ROOT add = f32[] add(param0, constant)
+  }
+
+  ENTRY kernelEntry {
+  param = f64[] parameter(0)
+  ROOT map = f32[] map(param), dimensions={}, to_apply=Computation
+})";
+
+TEST_F(HloVerifierTest, MapOperandComputationMismatch) {
+  TF_ASSERT_OK_AND_ASSIGN(auto module,
+                          ParseHloString(kMapOperandComputationMismatchHlo));
+  auto status = verifier().Run(module.get()).status();
+  ASSERT_FALSE(status.ok());
+  EXPECT_THAT(
+      status.error_message(),
+      HasSubstr(
+          "Shape mismatch between to_apply computation parameter and operand"));
+}
+
+TEST_F(HloVerifierTestAllowMixedPrecision, MapOperandComputationMismatch) {
+  TF_ASSERT_OK_AND_ASSIGN(auto module,
+                          ParseHloString(kMapOperandComputationMismatchHlo));
+  auto status = verifier().Run(module.get()).status();
+  ASSERT_TRUE(status.ok());
+}
+
+static const char* const kReduceOperandComputationMismatchHlo = R"(
+  HloModule ReduceOperandComputationMismatch
+  computation {
+    x = f32[] parameter(0)
+    y = f32[] parameter(1)
+    ROOT add = f32[] add(x, y)
+  }
+
+  ENTRY kernelEntry {
+    arg0 = f16[64,64,224,224]{3,2,1,0} parameter(0)
+    constant = f16[] constant(0)
+    reduce = f16[64]{0} reduce(arg0, constant), dimensions={0,2,3}, to_apply=computation
+  })";
+
+TEST_F(HloVerifierTest, ReduceOperandComputationMismatch) {
+  TF_ASSERT_OK_AND_ASSIGN(auto module,
+                          ParseHloString(kReduceOperandComputationMismatchHlo));
+  auto status = verifier().Run(module.get()).status();
+  ASSERT_FALSE(status.ok());
+  EXPECT_THAT(status.error_message(),
+              HasSubstr("Expected instruction to have shape equal to f32[64]"));
+}
+
+TEST_F(HloVerifierTestAllowMixedPrecision, ReduceOperandComputationMismatch) {
+  TF_ASSERT_OK_AND_ASSIGN(auto module,
+                          ParseHloString(kReduceOperandComputationMismatchHlo));
+  auto status = verifier().Run(module.get()).status();
+  ASSERT_TRUE(status.ok());
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From e6b48a1bd5c71aad85dead086b5ed8f27b08c9a7 Mon Sep 17 00:00:00 2001
From: Shivani Agrawal <shivaniagrawal@google.com>
Date: Mon, 4 Feb 2019 14:52:47 -0800
Subject: [PATCH 1871/2345] [tf.data] Adds more coverage for Defun-defun
 switch, adds map unit test.

PiperOrigin-RevId: 232371981
---
 tensorflow/python/data/kernel_tests/map_test.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tensorflow/python/data/kernel_tests/map_test.py b/tensorflow/python/data/kernel_tests/map_test.py
index bc9a67e5db..c419c632ff 100644
--- a/tensorflow/python/data/kernel_tests/map_test.py
+++ b/tensorflow/python/data/kernel_tests/map_test.py
@@ -705,6 +705,13 @@ class MapTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.assertRaises(errors.OutOfRangeError):
       self.evaluate(get_next())
 
+  def testNestedListMapDataset(self):
+    dataset = dataset_ops.Dataset.from_tensors(
+        [0, 1, 2]).repeat(10).map(lambda a: ([a[1], a[0] + a[2]], a[1]))
+
+    expected_output = [(np.array([1, 2]), 1)] * 10
+    self.assertDatasetProduces(dataset, expected_output=expected_output)
+
   def testPrefetch(self):
     # We will use this event to test that `_map_py_func()` has been
     # invoked a certain number of times (6 times, to be exact) after
-- 
GitLab


From ae0fd7300f8077a5cca198da9a55fc6f81dbfe48 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 4 Feb 2019 14:53:53 -0800
Subject: [PATCH 1872/2345] Don't let hash function output leak into deadness
 analysis' output

In deadness analysis we try to canonicalize and and or expressions by sorting
them (so that e.g. A&B and B&A map ultimately map to the same Predicate*).  We
used to sort them by their hashes (this was a recursive "content" hash and did
not depend on pointer values) but this meant that changes to the hash function
would result in changes to the Predicates we'd produce for a TensorFlow node.

This CL avoids this unnecessary dependence on hashes by using a deterministic ID
(order of allocation, basically) instead of the hash to sort the predicates.

PiperOrigin-RevId: 232372151
---
 tensorflow/compiler/jit/deadness_analysis.cc  | 69 +++++++------------
 .../compiler/jit/deadness_analysis_test.cc    | 50 +++++++-------
 2 files changed, 49 insertions(+), 70 deletions(-)

diff --git a/tensorflow/compiler/jit/deadness_analysis.cc b/tensorflow/compiler/jit/deadness_analysis.cc
index 0ef0d3db8c..4397eea9af 100644
--- a/tensorflow/compiler/jit/deadness_analysis.cc
+++ b/tensorflow/compiler/jit/deadness_analysis.cc
@@ -113,7 +113,11 @@ class Predicate {
   enum class Kind { kAnd, kOr, kNot, kAndRecurrence, kSymbol };
 
   virtual string ToString() const = 0;
-  int64 hash() const { return hash_; }
+
+  // An ID assigned to the Predicate at construction time.  Conceptually like a
+  // pointer, except that it is stable across runs.
+  int64 id() const { return id_; }
+
   virtual absl::Span<Predicate* const> GetOperands() const = 0;
 
   virtual Kind kind() const = 0;
@@ -126,29 +130,19 @@ class Predicate {
   static void Visit(Predicate* p, const FunctionTy& func);
 
  protected:
-  explicit Predicate(int64 hash) : hash_(hash) {}
+  explicit Predicate(int64 id) : id_(id) {}
 
  private:
-  const int64 hash_;
+  const int64 id_;
 
   TF_DISALLOW_COPY_AND_ASSIGN(Predicate);
 };
 
-int64 HashPredicateSequence(Predicate::Kind kind,
-                            absl::Span<Predicate* const> preds) {
-  int64 hash = ::tensorflow::hash<Predicate::Kind>()(kind);
-  for (Predicate* pred : preds) {
-    hash = Hash64Combine(hash, pred->hash());
-  }
-  return hash;
-}
-
 // Represents a logical conjunction of a set of predicates.
 class AndPredicate : public Predicate {
  public:
-  explicit AndPredicate(std::vector<Predicate*> operands)
-      : Predicate(HashPredicateSequence(Kind::kAnd, operands)),
-        operands_(std::move(operands)) {}
+  explicit AndPredicate(int64 id, std::vector<Predicate*> operands)
+      : Predicate(id), operands_(std::move(operands)) {}
 
   string ToString() const override {
     if (operands().empty()) {
@@ -177,9 +171,8 @@ class AndPredicate : public Predicate {
 // Represents a logical disjunction of a set of predicates.
 class OrPredicate : public Predicate {
  public:
-  explicit OrPredicate(std::vector<Predicate*> operands)
-      : Predicate(HashPredicateSequence(Kind::kOr, operands)),
-        operands_(std::move(operands)) {}
+  explicit OrPredicate(int64 id, std::vector<Predicate*> operands)
+      : Predicate(id), operands_(std::move(operands)) {}
 
   string ToString() const override {
     if (operands().empty()) {
@@ -207,9 +200,8 @@ class OrPredicate : public Predicate {
 // Represents a logical negation of a set of predicates.
 class NotPredicate : public Predicate {
  public:
-  explicit NotPredicate(Predicate* operand)
-      : Predicate(HashPredicateSequence(Kind::kNot, {operand})),
-        operands_({operand}) {}
+  explicit NotPredicate(int64 id, Predicate* operand)
+      : Predicate(id), operands_({operand}) {}
 
   string ToString() const override {
     return absl::StrCat("~", operand()->ToString());
@@ -246,11 +238,9 @@ class NotPredicate : public Predicate {
 // iterations).
 class AndRecurrencePredicate : public Predicate {
  public:
-  explicit AndRecurrencePredicate(Predicate* start, Predicate* step,
+  explicit AndRecurrencePredicate(int64 id, Predicate* start, Predicate* step,
                                   std::vector<string> frame)
-      : Predicate(Hash(start, step, frame)),
-        operands_({start, step}),
-        frame_(std::move(frame)) {}
+      : Predicate(id), operands_({start, step}), frame_(std::move(frame)) {}
 
   Predicate* start() const { return operands_[0]; }
   Predicate* step() const { return operands_[1]; }
@@ -270,16 +260,6 @@ class AndRecurrencePredicate : public Predicate {
  private:
   std::array<Predicate*, 2> operands_;
   std::vector<string> frame_;
-
-  static int64 Hash(Predicate* start, Predicate* step,
-                    const std::vector<string>& frame) {
-    uint64 frame_hash = 0;
-    for (const string& sub_frame : frame) {
-      frame_hash = Hash64Combine(Hash64(sub_frame), frame_hash);
-    }
-    return Hash64Combine(
-        HashPredicateSequence(Kind::kAndRecurrence, {start, step}), frame_hash);
-  }
 };
 
 // Represents an uninterpreted symbol in a logical predicate.
@@ -289,8 +269,8 @@ class AndRecurrencePredicate : public Predicate {
 // symbols.
 class SymbolPredicate : public Predicate {
  public:
-  explicit SymbolPredicate(TensorId tensor_id, bool must_be_true)
-      : Predicate(Hash(tensor_id, must_be_true)),
+  explicit SymbolPredicate(int64 id, TensorId tensor_id, bool must_be_true)
+      : Predicate(id),
         tensor_id_(std::move(tensor_id)),
         must_be_true_(must_be_true) {}
 
@@ -313,13 +293,6 @@ class SymbolPredicate : public Predicate {
  private:
   TensorId tensor_id_;
   bool must_be_true_;
-
-  static int64 Hash(const TensorId tensor_id, bool must_be_true) {
-    return Hash64Combine(
-        ::tensorflow::hash<bool>()(must_be_true),
-        Hash64Combine(::tensorflow::hash<Predicate::Kind>()(Kind::kSymbol),
-                      TensorId::Hasher{}(tensor_id)));
-  }
 };
 
 template <typename FunctionTy>
@@ -477,8 +450,11 @@ class PredicateFactory {
 
   template <typename PredicateT, typename... Args>
   std::unique_ptr<Predicate> Make(Args&&... args) {
+    // If we ever expose the Predicate class outside this .cc file then we may
+    // want to make this hard to misuse (by accidentally passing in an arbitrary
+    // integer to the Predicate constructor for instance).
     return std::unique_ptr<PredicateT>(
-        new PredicateT(std::forward<Args>(args)...));
+        new PredicateT(id_counter_++, std::forward<Args>(args)...));
   }
 
   Predicate* MakeAndOrImpl(absl::Span<Predicate* const> operands, bool is_and);
@@ -559,6 +535,7 @@ class PredicateFactory {
   absl::flat_hash_map<SignatureForSymbol, std::unique_ptr<Predicate>,
                       HashSignatureForSymbol>
       interned_symbol_instances_;
+  int64 id_counter_ = 0;
   int stack_depth_ = 0;
 };
 
@@ -566,7 +543,7 @@ Predicate* PredicateFactory::MakeInternedAndOr(
     std::vector<Predicate*> simplified_ops, Predicate::Kind pred_kind) {
   std::stable_sort(
       simplified_ops.begin(), simplified_ops.end(),
-      [](Predicate* a, Predicate* b) { return a->hash() < b->hash(); });
+      [](Predicate* a, Predicate* b) { return a->id() < b->id(); });
 
   auto it = interned_and_or_instances_.find({pred_kind, simplified_ops});
   if (it != interned_and_or_instances_.end()) {
diff --git a/tensorflow/compiler/jit/deadness_analysis_test.cc b/tensorflow/compiler/jit/deadness_analysis_test.cc
index 16ee8f86d5..38a5118d9a 100644
--- a/tensorflow/compiler/jit/deadness_analysis_test.cc
+++ b/tensorflow/compiler/jit/deadness_analysis_test.cc
@@ -521,7 +521,7 @@ TEST(DeadnessAnalysisTest, Loop) {
     EXPECT_EQ(predicate_map[ControlOutputFor(iv2)],
               "{#true,&,*iv2/cond:0}<fr0>");
     EXPECT_EQ(predicate_map[ControlOutputFor(add0)],
-              "({#true,&,*iv1/cond:0}<fr0> & {#true,&,*iv0/cond:0}<fr0>)");
+              "({#true,&,*iv0/cond:0}<fr0> & {#true,&,*iv1/cond:0}<fr0>)");
     EXPECT_EQ(predicate_map[ControlOutputFor(add1)],
               "({#true,&,*iv1/cond:0}<fr0> & {#true,&,*iv2/cond:0}<fr0>)");
   }
@@ -553,11 +553,11 @@ TEST(DeadnessAnalysisTest, ControlEquivalentLoopBodies) {
     EXPECT_EQ(predicate_map[ControlOutputFor(iv.induction_var)],
               "{#true,&,*iv0/cond:0}<loop>");
     EXPECT_EQ(predicate_map[ControlOutputFor(dependent_iv0)],
-              "{#true,&,(*iv0/cond:0 & iv0/iv:0)}<loop>");
+              "{#true,&,(iv0/iv:0 & *iv0/cond:0)}<loop>");
     EXPECT_EQ(predicate_map[ControlOutputFor(dependent_iv1)],
-              "{#true,&,(*iv0/cond:0 & iv0/iv:0)}<loop>");
+              "{#true,&,(iv0/iv:0 & *iv0/cond:0)}<loop>");
     EXPECT_EQ(predicate_map[ControlOutputFor(add0)],
-              "{#true,&,(*iv0/cond:0 & iv0/iv:0)}<loop>");
+              "{#true,&,(iv0/iv:0 & *iv0/cond:0)}<loop>");
   }
 }
 
@@ -643,22 +643,23 @@ TEST(DeadnessAnalysisTest, ControlEquivalentNestedLoopBodies) {
     EXPECT_EQ(predicate_map[ControlOutputFor(iv_outer.induction_var)],
               "{#true,&,*iv_outer/cond:0}<outer_loop>");
     EXPECT_EQ(predicate_map[ControlOutputFor(iv_inner.induction_var)],
-              "{({#true,&,*iv_outer/cond:0}<outer_loop> & "
-              "*iv_outer/cond:0),&,*iv_inner/cond:0}<inner_loop;outer_loop>");
+              "{(*iv_outer/cond:0 & "
+              "{#true,&,*iv_outer/cond:0}<outer_loop>),&,*iv_inner/"
+              "cond:0}<inner_loop;outer_loop>");
 
     EXPECT_EQ(predicate_map[ControlOutputFor(dependent_inner_iv0)],
               "{{#true,&,(iv_outer/iv:0 & "
-              "*iv_outer/cond:0)}<outer_loop>,&,(*iv_inner/cond:0 & "
-              "iv_inner/iv:0)}<inner_loop;outer_loop>");
+              "*iv_outer/cond:0)}<outer_loop>,&,(iv_inner/iv:0 & "
+              "*iv_inner/cond:0)}<inner_loop;outer_loop>");
 
     EXPECT_EQ(predicate_map[ControlOutputFor(dependent_inner_iv1)],
               "{{#true,&,(iv_outer/iv:0 & "
-              "*iv_outer/cond:0)}<outer_loop>,&,(*iv_inner/cond:0 & "
-              "iv_inner/iv:0)}<inner_loop;outer_loop>");
+              "*iv_outer/cond:0)}<outer_loop>,&,(iv_inner/iv:0 & "
+              "*iv_inner/cond:0)}<inner_loop;outer_loop>");
     EXPECT_EQ(predicate_map[ControlOutputFor(add0)],
               "{{#true,&,(iv_outer/iv:0 & "
-              "*iv_outer/cond:0)}<outer_loop>,&,(*iv_inner/cond:0 & "
-              "iv_inner/iv:0)}<inner_loop;outer_loop>");
+              "*iv_outer/cond:0)}<outer_loop>,&,(iv_inner/iv:0 & "
+              "*iv_inner/cond:0)}<inner_loop;outer_loop>");
   }
 }
 
@@ -702,20 +703,21 @@ TEST(DeadnessAnalysisTest, ControlNonEquivalentNestedLoopBodies) {
     EXPECT_EQ(predicate_map[ControlOutputFor(outer_iv[0])],
               "{#true,&,*iv_outer/cond:0}<outer_loop>");
     EXPECT_EQ(predicate_map[ControlOutputFor(inner_iv[0])],
-              "{({#true,&,*iv_outer/cond:0}<outer_loop> & "
-              "*iv_outer/cond:0),&,*iv_inner/cond:0}<inner_loop;outer_loop>");
+              "{(*iv_outer/cond:0 & "
+              "{#true,&,*iv_outer/cond:0}<outer_loop>),&,*iv_inner/"
+              "cond:0}<inner_loop;outer_loop>");
     EXPECT_EQ(predicate_map[ControlOutputFor(outer_iv[1])],
               "{#true,&,*iv_outer/cond_1:0}<outer_loop>");
-    EXPECT_EQ(
-        predicate_map[ControlOutputFor(inner_iv[1])],
-        "{({#true,&,*iv_outer/cond_1:0}<outer_loop> & "
-        "*iv_outer/cond_1:0),&,*iv_inner/cond_1:0}<inner_loop;outer_loop>");
-    EXPECT_EQ(
-        predicate_map[ControlOutputFor(add0)],
-        "({({#true,&,*iv_outer/cond:0}<outer_loop> & "
-        "*iv_outer/cond:0),&,*iv_inner/cond:0}<inner_loop;outer_loop> & "
-        "{({#true,&,*iv_outer/cond_1:0}<outer_loop> & "
-        "*iv_outer/cond_1:0),&,*iv_inner/cond_1:0}<inner_loop;outer_loop>)");
+    EXPECT_EQ(predicate_map[ControlOutputFor(inner_iv[1])],
+              "{(*iv_outer/cond_1:0 & "
+              "{#true,&,*iv_outer/cond_1:0}<outer_loop>),&,*iv_inner/"
+              "cond_1:0}<inner_loop;outer_loop>");
+    EXPECT_EQ(predicate_map[ControlOutputFor(add0)],
+              "({(*iv_outer/cond:0 & "
+              "{#true,&,*iv_outer/cond:0}<outer_loop>),&,*iv_inner/"
+              "cond:0}<inner_loop;outer_loop> & {(*iv_outer/cond_1:0 & "
+              "{#true,&,*iv_outer/cond_1:0}<outer_loop>),&,*iv_inner/"
+              "cond_1:0}<inner_loop;outer_loop>)");
   }
 }
 
-- 
GitLab


From 1b04355390e6f2e2309a3e03196a2446261aaa03 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 4 Feb 2019 15:06:51 -0800
Subject: [PATCH 1873/2345] Add a version of SerializeToStringDeterministic
 that serializes to a pre-allocated buffer. In the common case, this avoids a
 double write into the string and is in some cases about 30% faster on large
 protobufs.

Benchmark:

Run on *** (72 X 2993 MHz CPUs); 2019-02-01T17:30:47.860707104-08:00
CPU: Intel Skylake Xeon with HyperThreading (36 cores) dL1:32KB dL2:1024KB dL3:24MB
Benchmark                           Time(ns)        CPU(ns)     Iterations
--------------------------------------------------------------------------
BM_ProtoSerializationToString/1           99.1           99.1      7063926
BM_ProtoSerializationToString/8          450            450        1557551
BM_ProtoSerializationToString/64        3154           3154         222251
BM_ProtoSerializationToString/512      24947          24949          28043
BM_ProtoSerializationToString/4k      224626         224626           2995
BM_ProtoSerializationToString/10k     563273         563262           1000
BM_ProtoSerializationToBuffer/1           77.7           77.8      9015139
BM_ProtoSerializationToBuffer/8          351            351        1996330
BM_ProtoSerializationToBuffer/64        2511           2511         278283
BM_ProtoSerializationToBuffer/512      19846          19849          35234
BM_ProtoSerializationToBuffer/4k      184208         184262           3502
BM_ProtoSerializationToBuffer/10k     427867         427938           1622
PiperOrigin-RevId: 232374729
---
 tensorflow/core/BUILD                         | 13 ++++
 .../core/lib/strings/proto_serialization.cc   | 15 ++++-
 .../core/lib/strings/proto_serialization.h    |  4 ++
 .../lib/strings/proto_serialization_test.cc   | 66 +++++++++++++++++++
 4 files changed, 95 insertions(+), 3 deletions(-)
 create mode 100644 tensorflow/core/lib/strings/proto_serialization_test.cc

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 50e80aef99..3a56fa476c 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -3705,6 +3705,19 @@ tf_cc_test(
     ],
 )
 
+tf_cc_test(
+    name = "lib_strings_proto_serialization_test",
+    srcs = ["lib/strings/proto_serialization_test.cc"],
+    deps = [
+        ":lib",
+        ":lib_internal",
+        ":lib_test_internal",
+        ":protos_all_cc",
+        ":test",
+        ":test_main",
+    ],
+)
+
 tf_cc_test(
     name = "lib_random_weighted_picker_test",
     size = "medium",
diff --git a/tensorflow/core/lib/strings/proto_serialization.cc b/tensorflow/core/lib/strings/proto_serialization.cc
index 5c1fbda215..4a64eedef7 100644
--- a/tensorflow/core/lib/strings/proto_serialization.cc
+++ b/tensorflow/core/lib/strings/proto_serialization.cc
@@ -21,9 +21,18 @@ namespace tensorflow {
 bool SerializeToStringDeterministic(const protobuf::MessageLite& msg,
                                     string* result) {
   DCHECK_LE(msg.ByteSizeLong(), static_cast<size_t>(INT_MAX));
-  const int size = static_cast<int>(msg.ByteSizeLong());
-  *result = string(size, '\0');
-  protobuf::io::ArrayOutputStream array_stream(&(*result)[0], size);
+  *result = string(msg.ByteSizeLong(), '\0');
+  return SerializeToBufferDeterministic(msg, const_cast<char*>(result->data()),
+                                        result->size());
+}
+
+// As above, but takes a pre-allocated buffer wrapped by result.
+// PRECONDITION: result.size() == msg.ByteSizeLong().
+bool SerializeToBufferDeterministic(const protobuf::MessageLite& msg,
+                                    char* buffer, int size) {
+  DCHECK_LE(msg.ByteSizeLong(), static_cast<size_t>(INT_MAX));
+  DCHECK_EQ(msg.ByteSizeLong(), size);
+  protobuf::io::ArrayOutputStream array_stream(buffer, size);
   protobuf::io::CodedOutputStream output_stream(&array_stream);
   output_stream.SetSerializationDeterministic(true);
   msg.SerializeWithCachedSizes(&output_stream);
diff --git a/tensorflow/core/lib/strings/proto_serialization.h b/tensorflow/core/lib/strings/proto_serialization.h
index 6664928e28..fa8848a649 100644
--- a/tensorflow/core/lib/strings/proto_serialization.h
+++ b/tensorflow/core/lib/strings/proto_serialization.h
@@ -28,6 +28,10 @@ namespace tensorflow {
 bool SerializeToStringDeterministic(const protobuf::MessageLite& msg,
                                     string* result);
 
+// As above, but serialize to a pre-allocated `buffer` of length `size`.
+// PRECONDITION: size == msg.ByteSizeLong().
+bool SerializeToBufferDeterministic(const protobuf::MessageLite& msg,
+                                    char* buffer, int size);
 }  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_LIB_STRINGS_PROTO_SERIALIZATION_H_
diff --git a/tensorflow/core/lib/strings/proto_serialization_test.cc b/tensorflow/core/lib/strings/proto_serialization_test.cc
new file mode 100644
index 0000000000..466547daae
--- /dev/null
+++ b/tensorflow/core/lib/strings/proto_serialization_test.cc
@@ -0,0 +1,66 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/lib/strings/proto_serialization.h"
+
+#include <string>
+#include "tensorflow/core/framework/graph.pb.h"
+#include "tensorflow/core/framework/node_def.pb.h"
+#include "tensorflow/core/lib/strings/strcat.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/platform/test_benchmark.h"
+
+namespace tensorflow {
+
+static void BM_ProtoSerializationToString(int iters, int num_nodes) {
+  testing::StopTiming();
+  GraphDef graph_def;
+  for (int i = 0; i < num_nodes; ++i) {
+    NodeDef* node = graph_def.add_node();
+    node->set_name(strings::StrCat("node", i));
+    node->set_op(strings::StrCat("op", i % 10));
+  }
+  testing::StartTiming();
+  for (int i = 0; i < iters; ++i) {
+    string serialized;
+    testing::DoNotOptimize(
+        SerializeToStringDeterministic(graph_def, &serialized));
+  }
+  testing::StopTiming();
+}
+BENCHMARK(BM_ProtoSerializationToString)->Range(1, 10000);
+
+static void BM_ProtoSerializationToBuffer(int iters, int num_nodes) {
+  testing::StopTiming();
+  GraphDef graph_def;
+  for (int i = 0; i < num_nodes; ++i) {
+    NodeDef* node = graph_def.add_node();
+    node->set_name(strings::StrCat("node", i));
+    node->set_op(strings::StrCat("op", i % 10));
+  }
+  testing::StartTiming();
+  int size = graph_def.ByteSizeLong();
+  for (int i = 0; i < iters; ++i) {
+    char* buf = new char[size];
+    testing::DoNotOptimize(
+        SerializeToBufferDeterministic(graph_def, buf, size));
+    delete[] buf;
+  }
+  testing::StopTiming();
+}
+
+BENCHMARK(BM_ProtoSerializationToBuffer)->Range(1, 10000);
+
+}  // namespace tensorflow
-- 
GitLab


From 33190103458eb9611025c1a947a2a59fc780aa08 Mon Sep 17 00:00:00 2001
From: Eddie Zhou <eddz@google.com>
Date: Mon, 4 Feb 2019 15:31:40 -0800
Subject: [PATCH 1874/2345] Add swap_memory to
 tf.contrib.rnn.stack_bidirectional_dynamic_rnn, propagating to
 tf.nn.bidirectional_dynamic_rnn.

PiperOrigin-RevId: 232378992
---
 tensorflow/contrib/rnn/python/ops/rnn.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/rnn/python/ops/rnn.py b/tensorflow/contrib/rnn/python/ops/rnn.py
index 0266b72dcb..41b1698321 100644
--- a/tensorflow/contrib/rnn/python/ops/rnn.py
+++ b/tensorflow/contrib/rnn/python/ops/rnn.py
@@ -131,7 +131,8 @@ def stack_bidirectional_dynamic_rnn(cells_fw,
                                     sequence_length=None,
                                     parallel_iterations=None,
                                     time_major=False,
-                                    scope=None):
+                                    scope=None,
+                                    swap_memory=False):
   """Creates a dynamic bidirectional recurrent neural network.
 
   Stacks several bidirectional rnn layers. The combined forward and backward
@@ -171,6 +172,10 @@ def stack_bidirectional_dynamic_rnn(cells_fw,
       data is batch-major, so by default this function accepts input and emits
       output in batch-major form.
     scope: VariableScope for the created subgraph; defaults to None.
+    swap_memory: Transparently swap the tensors produced in forward inference
+      but needed for back prop from GPU to CPU.  This allows training RNNs
+      which would typically not fit on a single GPU, with very minimal (or no)
+      performance penalty.
 
   Returns:
     A tuple (outputs, output_state_fw, output_state_bw) where:
@@ -230,6 +235,7 @@ def stack_bidirectional_dynamic_rnn(cells_fw,
             sequence_length=sequence_length,
             parallel_iterations=parallel_iterations,
             dtype=dtype,
+            swap_memory=swap_memory,
             time_major=time_major)
         # Concat the outputs to create the new input.
         prev_layer = array_ops.concat(outputs, 2)
-- 
GitLab


From 8b491637336083b40e9d4a38524463b18ba2825c Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Mon, 4 Feb 2019 15:39:54 -0800
Subject: [PATCH 1875/2345] [XLA] InferGetTupleElementShape should report an
 error instead of crashing when the tuple index is a negative value.

Add a shape_inference test case.

PiperOrigin-RevId: 232380529
---
 tensorflow/compiler/xla/service/shape_inference.cc |  2 +-
 .../compiler/xla/service/shape_inference_test.cc   | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index bfb18f40b6..12db957b0c 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -2436,7 +2436,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
         ShapeUtil::HumanString(arg));
   }
 
-  if (index >= arg.tuple_shapes_size()) {
+  if (index < 0 || index >= arg.tuple_shapes_size()) {
     return InvalidArgument(
         "Cannot infer shape: attempt to index out of tuple bounds: %d "
         ">= %d in shape %s.",
diff --git a/tensorflow/compiler/xla/service/shape_inference_test.cc b/tensorflow/compiler/xla/service/shape_inference_test.cc
index 26120a06b8..c43db7aae8 100644
--- a/tensorflow/compiler/xla/service/shape_inference_test.cc
+++ b/tensorflow/compiler/xla/service/shape_inference_test.cc
@@ -896,6 +896,20 @@ TEST_F(ShapeInferenceTest, InferConstIndexShape) {
   ASSERT_TRUE(ShapeUtil::Equal(s32_, inferred1_status.ValueOrDie()));
 }
 
+TEST_F(ShapeInferenceTest, InferTupleElementShapeOutOfBound) {
+  Shape tuple_shape = ShapeUtil::MakeTupleShape({f32_, s32_});
+  auto inferredNegative_status =
+      ShapeInference::InferGetTupleElementShape(tuple_shape, -1);
+  auto inferred2_status =
+      ShapeInference::InferGetTupleElementShape(tuple_shape, 2);
+  ASSERT_FALSE(inferredNegative_status.ok());
+  ASSERT_FALSE(inferred2_status.ok());
+  EXPECT_THAT(inferredNegative_status.status().error_message(),
+              HasSubstr("attempt to index out of tuple bounds"));
+  EXPECT_THAT(inferred2_status.status().error_message(),
+              HasSubstr("attempt to index out of tuple bounds"));
+}
+
 TEST_F(ShapeInferenceTest, InferPowShape) {
   auto ten_floats = ShapeUtil::MakeShape(F32, {10});
   auto inferred_status = ShapeInference::InferBinaryOpShape(
-- 
GitLab


From 6fd6bf5b802335afeab73447cd2574e9188c97e9 Mon Sep 17 00:00:00 2001
From: nammbash <niroopshankar.ammbashankar@intel.com>
Date: Mon, 4 Feb 2019 15:55:33 -0800
Subject: [PATCH 1876/2345] omp reduction feature and clang formatting

---
 ...mkl_requantization_range_per_channel_op.cc | 44 +++++++++----------
 .../core/kernels/mkl_requantize_ops_test.cc   | 30 ++++++++-----
 2 files changed, 40 insertions(+), 34 deletions(-)

diff --git a/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc b/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc
index 1de60c67ae..37d6bb1658 100644
--- a/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc
+++ b/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc
@@ -47,12 +47,14 @@ class MklRequantizationRangePerChannelOp : public OpKernel {
     const Tensor& input_max = ctx->input(kInputMaxIndex);
 
     const size_t depth = input_max.NumElements();
-    OP_REQUIRES(ctx, input_min.dim_size(0) == depth,
-                errors::InvalidArgument("input_min has incorrect size, expected ",
-                                        depth, " was ", input_min.dim_size(0)));
-    OP_REQUIRES(ctx, input_max.dim_size(0) == depth,
-                errors::InvalidArgument("input_max has incorrect size, expected ",
-                                        depth, " was ", input_max.dim_size(0)));
+    OP_REQUIRES(
+        ctx, input_min.dim_size(0) == depth,
+        errors::InvalidArgument("input_min has incorrect size, expected ",
+                                depth, " was ", input_min.dim_size(0)));
+    OP_REQUIRES(
+        ctx, input_max.dim_size(0) == depth,
+        errors::InvalidArgument("input_max has incorrect size, expected ",
+                                depth, " was ", input_max.dim_size(0)));
 
     const float* input_min_data = input_min.flat<float>().data();
     const float* input_max_data = input_max.flat<float>().data();
@@ -61,16 +63,18 @@ class MklRequantizationRangePerChannelOp : public OpKernel {
     Eigen::array<int, 2> shuffling({1, 0});
     auto input_matrix = input.flat_inner_dims<qint32>();
 
-    // TODO: verify performance of not transposing and finding the min max directly
+    // TODO: verify performance of not transposing and finding the min max
+    // directly
     // from input_matrix vs the one presentd below of transposing and using the
     // transposed matrix as the transpose operation might be more costly.
-    // Note that this operation is a calibration step for quantization and will cease 
-    // to exist in the final inference graph as it will exist as a const node. 
+    // Note that this operation is a calibration step for quantization and will
+    // cease
+    // to exist in the final inference graph as it will exist as a const node.
     auto transposed_input = input_matrix.shuffle(shuffling);
 
     // Find the ranges of each channel in parallel.
     float out_min_max = std::numeric_limits<float>::min();
-    #pragma omp parallel for //reduction(min:out_min_max)
+#pragma omp parallel for reduction(max : out_min_max)
     for (size_t i = 0; i < depth; ++i) {
       Eigen::Tensor<qint32, 0, Eigen::RowMajor> min =
           transposed_input.chip<0>(i).minimum();
@@ -82,21 +86,17 @@ class MklRequantizationRangePerChannelOp : public OpKernel {
           std::max(std::abs(min_per_channel), std::abs(max_per_channel));
       float scale =
           std::max(std::abs(input_min_data[i]), std::abs(input_max_data[i]));
-      ranges[i] = scale * static_cast<float>(abs_max) / static_cast<float>(1L << 31);
+      ranges[i] =
+          scale * static_cast<float>(abs_max) / static_cast<float>(1L << 31);
       if (min_per_channel < 0) is_non_negative = false;
 
-        // thread-local out_min_max. 
-        //if (out_min_max < ranges[i]) out_min_max = ranges[i];
-      } 
-      // All local out_min_max gets min-reduced into one global out_min_max at 
-      // the end of the loop by specifying reduction(min:out_min_max) along with 
-      // omp parallel for.
-    
-
-    // Obtain ranges of out_min_max after all parallel_for openMP threads have joined.     
-    for (size_t i = 0; i < depth; ++i) {
-      if (out_min_max < ranges[i]) out_min_max = ranges[i];
+      // thread-local out_min_max.
+      out_min_max = std::max(out_min_max, ranges[i]);
     }
+    // All local out_min_max gets max-reduced into one global out_min_max at
+    // the end of the loop by specifying reduction(max:out_min_max) along with
+    // omp parallel for.
+
     // Fixing max to clip_value_max_ (example 6.0 to support relu6)
     if (out_min_max > clip_value_max_) out_min_max = clip_value_max_;
 
diff --git a/tensorflow/core/kernels/mkl_requantize_ops_test.cc b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
index 9ec379c08e..591830f775 100644
--- a/tensorflow/core/kernels/mkl_requantize_ops_test.cc
+++ b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
@@ -56,11 +56,15 @@ void MklRequantizatedOpsTestHelper::Setup(Tensor &input_tensor_qint32,
   // T and W are quantized using a Quantize Op.
   // The input Tensor T (NHWC) is quantized to unsigned int8.
   // The input Filter W (NHWC) is quantized to signed int8.
-  // Hence T's max value is mapped to ((2^8-1) = 255), while W's to ((2^7)-1 = 127)).
+  // Hence T's max value is mapped to ((2^8-1) = 255), while W's to ((2^7)-1 =
+  // 127)).
 
-  // Range of Quantized T  in uint8[0   , 255] maps to orig T  in FP32[0   , 5.0]
-  // Range of Quantized W1 in int8[-127, 127]  maps to orig W1 in FP32[-2.0, 2.0]
-  // Range of Quantized W2 in int8[-127, 127]  maps to orig W2 in FP32[-3.0, 3.0]
+  // Range of Quantized T  in uint8[0   , 255] maps to orig T  in FP32[0   ,
+  // 5.0]
+  // Range of Quantized W1 in int8[-127, 127]  maps to orig W1 in FP32[-2.0,
+  // 2.0]
+  // Range of Quantized W2 in int8[-127, 127]  maps to orig W2 in FP32[-3.0,
+  // 3.0]
 
   // Hence the resolution of Quantized T will be 5.0/255
   // Hence the resolution of Quantized W1 will be 2.0/127
@@ -70,8 +74,10 @@ void MklRequantizatedOpsTestHelper::Setup(Tensor &input_tensor_qint32,
   // ---------------------------------------------------------------------------
   // The input T and weights W1 (or W2) will be convolved.
   // The output Tensor T is in int32 whose range is [-2^31, 2^31).
-  // For simplicity, we truncate the range to (-2^31, 2^31) to make it symmetric.
-  // The range of the convolved T*W1 is ((2^31)-1) * 5.0/255 * 2.0/127 = 663110.59
+  // For simplicity, we truncate the range to (-2^31, 2^31) to make it
+  // symmetric.
+  // The range of the convolved T*W1 is ((2^31)-1) * 5.0/255 * 2.0/127 =
+  // 663110.59
   // So the range of convolved T*W1 in int32(-2^31, 2^31) that maps to
   // orig T range in FP32[0, 5.0] * [-2.0, 2.0] is [-663110.59, 663110.59]
 
@@ -160,8 +166,8 @@ TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_Basic) {
   // Verify the Expected Outputs
   const float output_min = GetOutput(0)->flat<float>()(0);
   const float output_max = GetOutput(1)->flat<float>()(0);
-  EXPECT_NEAR(-14.8217, output_min, 0.02);
-  EXPECT_NEAR(14.8217, output_max, 0.02);
+  EXPECT_NEAR(-14.8217, output_min, 0.002);
+  EXPECT_NEAR(14.8217, output_max, 0.002);
 
   // output range is made use in RequantizePerChannelTest_Basic
 }
@@ -221,8 +227,8 @@ TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_ClipMax) {
   // Verify the Expected Outputs
   const float output_min = GetOutput(0)->flat<float>()(0);
   const float output_max = GetOutput(1)->flat<float>()(0);
-  EXPECT_NEAR(-6.0, output_min, 0.02);  // Values are Max as with clip_value
-  EXPECT_NEAR(6.0, output_max, 0.02);   // Values are Max as with clip_value
+  EXPECT_NEAR(-6.0, output_min, 0.002);  // Values are Max as with clip_value
+  EXPECT_NEAR(6.0, output_max, 0.002);   // Values are Max as with clip_value
 }
 
 TEST_F(MklRequantizatedOpsTest, RequantizePerChannelTest_Basic) {
@@ -290,8 +296,8 @@ TEST_F(MklRequantizatedOpsTest, RequantizePerChannelTest_Basic) {
   Tensor output = *GetOutput(0);
   const float output_min = GetOutput(1)->flat<float>()(0);
   const float output_max = GetOutput(2)->flat<float>()(0);
-  EXPECT_NEAR(range_op_output_min, output_min, 0.02);
-  EXPECT_NEAR(range_op_output_max, output_max, 0.02);
+  EXPECT_NEAR(range_op_output_min, output_min, 0.002);
+  EXPECT_NEAR(range_op_output_max, output_max, 0.002);
 }
 
 }  // namespace tensorflow
-- 
GitLab


From ee9f21c05772fa91cf536fa796cf2ab6e52d1325 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 4 Feb 2019 15:56:14 -0800
Subject: [PATCH 1877/2345] Switch log listener printing to use string.c_str()
 instead of string.data().

PiperOrigin-RevId: 232383289
---
 tensorflow/core/kernels/logging_ops.cc    | 2 +-
 tensorflow/python/eager/pywrap_tfe_src.cc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/kernels/logging_ops.cc b/tensorflow/core/kernels/logging_ops.cc
index bcd9f58642..e611ae28b9 100644
--- a/tensorflow/core/kernels/logging_ops.cc
+++ b/tensorflow/core/kernels/logging_ops.cc
@@ -179,7 +179,7 @@ class PrintV2Op : public OpKernel {
     auto listeners = logging::GetListeners();
     if (!listeners->empty()) {
       for (auto& listener : *listeners) {
-        listener(msg.data());
+        listener(msg.c_str());
       }
     } else if (output_stream_ == "stdout") {
       std::cout << msg << std::endl;
diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc
index 085b3e29eb..eb2f28d8c7 100644
--- a/tensorflow/python/eager/pywrap_tfe_src.cc
+++ b/tensorflow/python/eager/pywrap_tfe_src.cc
@@ -3049,7 +3049,7 @@ void PrintToPythonStdout(const char* msg) {
     int CHUNK_SIZE = 900;
     auto len = string_msg.length();
     for (int i = 0; i < len; i += CHUNK_SIZE) {
-      PySys_WriteStdout("%s", string_msg.substr(i, CHUNK_SIZE).data());
+      PySys_WriteStdout("%s", string_msg.substr(i, CHUNK_SIZE).c_str());
     }
     PySys_WriteStdout("\n");
 
-- 
GitLab


From 26a17c3babbdf197d023139aee9dfdd784498bc1 Mon Sep 17 00:00:00 2001
From: Mahmoud Abuzaina <mahmoud.abuzaina@intel.com>
Date: Mon, 4 Feb 2019 16:02:37 -0800
Subject: [PATCH 1878/2345] Fixing the conv unit test failure caused by MKL pad
 fusion

---
 tensorflow/core/kernels/mkl_conv_ops.h | 39 +++++++++++---------------
 1 file changed, 17 insertions(+), 22 deletions(-)

diff --git a/tensorflow/core/kernels/mkl_conv_ops.h b/tensorflow/core/kernels/mkl_conv_ops.h
index b96cc23186..228397c712 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.h
+++ b/tensorflow/core/kernels/mkl_conv_ops.h
@@ -196,9 +196,8 @@ class MklDnnConvUtil {
                                         filter_shape.DebugString()));
 
     for (int i = 0; i < ((strides_.size() == 4) ? 3 : 5); i++) {
-      OP_REQUIRES(context_,
-                  FastBoundsCheck(filter_shape.dim_size(i),
-                                  std::numeric_limits<int>::max()),
+      OP_REQUIRES(context_, FastBoundsCheck(filter_shape.dim_size(i),
+                                            std::numeric_limits<int>::max()),
                   errors::InvalidArgument("filter too large"));
     }
 
@@ -392,14 +391,24 @@ class MklDnnConvUtil {
     int64 pad_D1, pad_D2;
 
     if (is_conv2d) {
+      Padding padding_type;
+      if (pad_enabled) {
+        padding_type = Padding::EXPLICIT;
+        pad_top = static_cast<int64>((*pad_l)[0]);
+        pad_left = static_cast<int64>((*pad_l)[1]);
+        pad_bottom = static_cast<int64>((*pad_r)[0]);
+        pad_right = static_cast<int64>((*pad_r)[1]);
+      } else {
+        padding_type = padding_;
+      }
       OP_REQUIRES_OK(context_,
                      GetWindowedOutputSizeVerboseV2(
                          input_rows, filter_rows, dilation_rows, stride_rows,
-                         padding_, &out_rows, &pad_top, &pad_bottom));
+                         padding_type, &out_rows, &pad_top, &pad_bottom));
       OP_REQUIRES_OK(context_,
                      GetWindowedOutputSizeVerboseV2(
                          input_cols, filter_cols, dilation_cols, stride_cols,
-                         padding_, &out_cols, &pad_left, &pad_right));
+                         padding_type, &out_cols, &pad_left, &pad_right));
     } else {
       OP_REQUIRES_OK(context_, GetWindowedOutputSizeVerbose(
                                    input_planes, filter_planes, stride_planes,
@@ -413,25 +422,11 @@ class MklDnnConvUtil {
     }
 
     if (is_conv2d) {
-      // Conv + pad fusion is enabled only for 2D
+      // Conv + pad fusion is enabled only for 2D.
       // If pad_enabled, i.e., pad and conv op are fused, then
       // all pads are already passed from pad op through
-      // *pad_l and *pad_r
-      if (pad_enabled) {
-        pad_top = static_cast<int64>((*pad_l)[0]);
-        pad_left = static_cast<int64>((*pad_l)[1]);
-        pad_bottom = static_cast<int64>((*pad_r)[0]);
-        pad_right = static_cast<int64>((*pad_r)[1]);
-        // update the out_rows and out_cols based on all
-        // sides of the pads coming from pad op.
-        out_rows = out_rows + (pad_top + pad_bottom) / stride_rows;
-        out_cols = out_cols + (pad_left + pad_right) / stride_cols;
-      }
-      // Handle padding. MKL-DNN uses asymetric padding.
-      // But, if pad_enabled, i.e., pad and conv op are fused,
-      // then, *pad_l and *pad_r are already set from pad op.
-      // In that case they need not set here.
-      else {
+      // *pad_l and *pad_r and they don't need to be set here.
+      if (!pad_enabled) {
         *pad_l = {static_cast<int>(pad_top), static_cast<int>(pad_left)};
         *pad_r = {static_cast<int>(pad_bottom), static_cast<int>(pad_right)};
       }
-- 
GitLab


From 2a5466b074f4f181300d154f2c75f4a73fbea825 Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Mon, 4 Feb 2019 16:01:43 -0800
Subject: [PATCH 1879/2345] Make `name` and `shape` arguments optional in
 `Layer.add_weight()`.

PiperOrigin-RevId: 232384169
---
 tensorflow/python/keras/engine/base_layer.py  | 15 ++++++++-----
 .../python/keras/engine/base_layer_test.py    | 22 +++++++++++++++++++
 .../golden/v1/tensorflow.keras.-model.pbtxt   |  2 +-
 .../v1/tensorflow.keras.-sequential.pbtxt     |  2 +-
 ....experimental.-peephole-l-s-t-m-cell.pbtxt |  2 +-
 .../tensorflow.keras.layers.-activation.pbtxt |  2 +-
 ...eras.layers.-activity-regularization.pbtxt |  2 +-
 .../v1/tensorflow.keras.layers.-add.pbtxt     |  2 +-
 ...nsorflow.keras.layers.-alpha-dropout.pbtxt |  2 +-
 ...low.keras.layers.-average-pooling1-d.pbtxt |  2 +-
 ...low.keras.layers.-average-pooling2-d.pbtxt |  2 +-
 ...low.keras.layers.-average-pooling3-d.pbtxt |  2 +-
 .../v1/tensorflow.keras.layers.-average.pbtxt |  2 +-
 ...tensorflow.keras.layers.-avg-pool1-d.pbtxt |  2 +-
 ...tensorflow.keras.layers.-avg-pool2-d.pbtxt |  2 +-
 ...tensorflow.keras.layers.-avg-pool3-d.pbtxt |  2 +-
 ...ow.keras.layers.-batch-normalization.pbtxt |  2 +-
 ...nsorflow.keras.layers.-bidirectional.pbtxt |  2 +-
 ...tensorflow.keras.layers.-concatenate.pbtxt |  2 +-
 ...orflow.keras.layers.-conv-l-s-t-m2-d.pbtxt |  2 +-
 .../v1/tensorflow.keras.layers.-conv1-d.pbtxt |  2 +-
 ...flow.keras.layers.-conv2-d-transpose.pbtxt |  2 +-
 .../v1/tensorflow.keras.layers.-conv2-d.pbtxt |  2 +-
 ...flow.keras.layers.-conv3-d-transpose.pbtxt |  2 +-
 .../v1/tensorflow.keras.layers.-conv3-d.pbtxt |  2 +-
 ...sorflow.keras.layers.-convolution1-d.pbtxt |  2 +-
 ...ras.layers.-convolution2-d-transpose.pbtxt |  2 +-
 ...sorflow.keras.layers.-convolution2-d.pbtxt |  2 +-
 ...ras.layers.-convolution3-d-transpose.pbtxt |  2 +-
 ...sorflow.keras.layers.-convolution3-d.pbtxt |  2 +-
 ...tensorflow.keras.layers.-cropping1-d.pbtxt |  2 +-
 ...tensorflow.keras.layers.-cropping2-d.pbtxt |  2 +-
 ...tensorflow.keras.layers.-cropping3-d.pbtxt |  2 +-
 ...sorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt |  2 +-
 ...rflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt |  2 +-
 .../v1/tensorflow.keras.layers.-dense.pbtxt   |  2 +-
 ...flow.keras.layers.-depthwise-conv2-d.pbtxt |  2 +-
 .../v1/tensorflow.keras.layers.-dot.pbtxt     |  2 +-
 .../v1/tensorflow.keras.layers.-dropout.pbtxt |  2 +-
 .../v1/tensorflow.keras.layers.-e-l-u.pbtxt   |  2 +-
 .../tensorflow.keras.layers.-embedding.pbtxt  |  2 +-
 .../v1/tensorflow.keras.layers.-flatten.pbtxt |  2 +-
 .../tensorflow.keras.layers.-g-r-u-cell.pbtxt |  2 +-
 .../v1/tensorflow.keras.layers.-g-r-u.pbtxt   |  2 +-
 ...rflow.keras.layers.-gaussian-dropout.pbtxt |  2 +-
 ...sorflow.keras.layers.-gaussian-noise.pbtxt |  2 +-
 ...as.layers.-global-average-pooling1-d.pbtxt |  2 +-
 ...as.layers.-global-average-pooling2-d.pbtxt |  2 +-
 ...as.layers.-global-average-pooling3-d.pbtxt |  2 +-
 ...low.keras.layers.-global-avg-pool1-d.pbtxt |  2 +-
 ...low.keras.layers.-global-avg-pool2-d.pbtxt |  2 +-
 ...low.keras.layers.-global-avg-pool3-d.pbtxt |  2 +-
 ...low.keras.layers.-global-max-pool1-d.pbtxt |  2 +-
 ...low.keras.layers.-global-max-pool2-d.pbtxt |  2 +-
 ...low.keras.layers.-global-max-pool3-d.pbtxt |  2 +-
 ....keras.layers.-global-max-pooling1-d.pbtxt |  2 +-
 ....keras.layers.-global-max-pooling2-d.pbtxt |  2 +-
 ....keras.layers.-global-max-pooling3-d.pbtxt |  2 +-
 ...tensorflow.keras.layers.-input-layer.pbtxt |  2 +-
 ...ensorflow.keras.layers.-l-s-t-m-cell.pbtxt |  2 +-
 .../v1/tensorflow.keras.layers.-l-s-t-m.pbtxt |  2 +-
 .../v1/tensorflow.keras.layers.-lambda.pbtxt  |  2 +-
 .../v1/tensorflow.keras.layers.-layer.pbtxt   |  2 +-
 ...ensorflow.keras.layers.-leaky-re-l-u.pbtxt |  2 +-
 ...w.keras.layers.-locally-connected1-d.pbtxt |  2 +-
 ...w.keras.layers.-locally-connected2-d.pbtxt |  2 +-
 .../v1/tensorflow.keras.layers.-masking.pbtxt |  2 +-
 ...tensorflow.keras.layers.-max-pool1-d.pbtxt |  2 +-
 ...tensorflow.keras.layers.-max-pool2-d.pbtxt |  2 +-
 ...tensorflow.keras.layers.-max-pool3-d.pbtxt |  2 +-
 ...sorflow.keras.layers.-max-pooling1-d.pbtxt |  2 +-
 ...sorflow.keras.layers.-max-pooling2-d.pbtxt |  2 +-
 ...sorflow.keras.layers.-max-pooling3-d.pbtxt |  2 +-
 .../v1/tensorflow.keras.layers.-maximum.pbtxt |  2 +-
 .../v1/tensorflow.keras.layers.-minimum.pbtxt |  2 +-
 .../tensorflow.keras.layers.-multiply.pbtxt   |  2 +-
 .../tensorflow.keras.layers.-p-re-l-u.pbtxt   |  2 +-
 .../v1/tensorflow.keras.layers.-permute.pbtxt |  2 +-
 .../v1/tensorflow.keras.layers.-r-n-n.pbtxt   |  2 +-
 .../v1/tensorflow.keras.layers.-re-l-u.pbtxt  |  2 +-
 ...nsorflow.keras.layers.-repeat-vector.pbtxt |  2 +-
 .../v1/tensorflow.keras.layers.-reshape.pbtxt |  2 +-
 ...flow.keras.layers.-separable-conv1-d.pbtxt |  2 +-
 ...flow.keras.layers.-separable-conv2-d.pbtxt |  2 +-
 ...ras.layers.-separable-convolution1-d.pbtxt |  2 +-
 ...ras.layers.-separable-convolution2-d.pbtxt |  2 +-
 ...flow.keras.layers.-simple-r-n-n-cell.pbtxt |  2 +-
 ...ensorflow.keras.layers.-simple-r-n-n.pbtxt |  2 +-
 .../v1/tensorflow.keras.layers.-softmax.pbtxt |  2 +-
 ...low.keras.layers.-spatial-dropout1-d.pbtxt |  2 +-
 ...low.keras.layers.-spatial-dropout2-d.pbtxt |  2 +-
 ...low.keras.layers.-spatial-dropout3-d.pbtxt |  2 +-
 ...ow.keras.layers.-stacked-r-n-n-cells.pbtxt |  2 +-
 .../tensorflow.keras.layers.-subtract.pbtxt   |  2 +-
 ...low.keras.layers.-thresholded-re-l-u.pbtxt |  2 +-
 ...rflow.keras.layers.-time-distributed.pbtxt |  2 +-
 ...sorflow.keras.layers.-up-sampling1-d.pbtxt |  2 +-
 ...sorflow.keras.layers.-up-sampling2-d.pbtxt |  2 +-
 ...sorflow.keras.layers.-up-sampling3-d.pbtxt |  2 +-
 .../v1/tensorflow.keras.layers.-wrapper.pbtxt |  2 +-
 ...orflow.keras.layers.-zero-padding1-d.pbtxt |  2 +-
 ...orflow.keras.layers.-zero-padding2-d.pbtxt |  2 +-
 ...orflow.keras.layers.-zero-padding3-d.pbtxt |  2 +-
 ...rs.experimental.-layer-normalization.pbtxt |  2 +-
 .../v1/tensorflow.keras.models.-model.pbtxt   |  2 +-
 .../tensorflow.keras.models.-sequential.pbtxt |  2 +-
 .../golden/v2/tensorflow.keras.-model.pbtxt   |  2 +-
 .../v2/tensorflow.keras.-sequential.pbtxt     |  2 +-
 ....experimental.-peephole-l-s-t-m-cell.pbtxt |  2 +-
 .../tensorflow.keras.layers.-activation.pbtxt |  2 +-
 ...eras.layers.-activity-regularization.pbtxt |  2 +-
 .../v2/tensorflow.keras.layers.-add.pbtxt     |  2 +-
 ...nsorflow.keras.layers.-alpha-dropout.pbtxt |  2 +-
 ...low.keras.layers.-average-pooling1-d.pbtxt |  2 +-
 ...low.keras.layers.-average-pooling2-d.pbtxt |  2 +-
 ...low.keras.layers.-average-pooling3-d.pbtxt |  2 +-
 .../v2/tensorflow.keras.layers.-average.pbtxt |  2 +-
 ...tensorflow.keras.layers.-avg-pool1-d.pbtxt |  2 +-
 ...tensorflow.keras.layers.-avg-pool2-d.pbtxt |  2 +-
 ...tensorflow.keras.layers.-avg-pool3-d.pbtxt |  2 +-
 ...ow.keras.layers.-batch-normalization.pbtxt |  2 +-
 ...nsorflow.keras.layers.-bidirectional.pbtxt |  2 +-
 ...tensorflow.keras.layers.-concatenate.pbtxt |  2 +-
 ...orflow.keras.layers.-conv-l-s-t-m2-d.pbtxt |  2 +-
 .../v2/tensorflow.keras.layers.-conv1-d.pbtxt |  2 +-
 ...flow.keras.layers.-conv2-d-transpose.pbtxt |  2 +-
 .../v2/tensorflow.keras.layers.-conv2-d.pbtxt |  2 +-
 ...flow.keras.layers.-conv3-d-transpose.pbtxt |  2 +-
 .../v2/tensorflow.keras.layers.-conv3-d.pbtxt |  2 +-
 ...sorflow.keras.layers.-convolution1-d.pbtxt |  2 +-
 ...ras.layers.-convolution2-d-transpose.pbtxt |  2 +-
 ...sorflow.keras.layers.-convolution2-d.pbtxt |  2 +-
 ...ras.layers.-convolution3-d-transpose.pbtxt |  2 +-
 ...sorflow.keras.layers.-convolution3-d.pbtxt |  2 +-
 ...tensorflow.keras.layers.-cropping1-d.pbtxt |  2 +-
 ...tensorflow.keras.layers.-cropping2-d.pbtxt |  2 +-
 ...tensorflow.keras.layers.-cropping3-d.pbtxt |  2 +-
 ...sorflow.keras.layers.-dense-features.pbtxt |  2 +-
 .../v2/tensorflow.keras.layers.-dense.pbtxt   |  2 +-
 ...flow.keras.layers.-depthwise-conv2-d.pbtxt |  2 +-
 .../v2/tensorflow.keras.layers.-dot.pbtxt     |  2 +-
 .../v2/tensorflow.keras.layers.-dropout.pbtxt |  2 +-
 .../v2/tensorflow.keras.layers.-e-l-u.pbtxt   |  2 +-
 .../tensorflow.keras.layers.-embedding.pbtxt  |  2 +-
 .../v2/tensorflow.keras.layers.-flatten.pbtxt |  2 +-
 .../tensorflow.keras.layers.-g-r-u-cell.pbtxt |  2 +-
 .../v2/tensorflow.keras.layers.-g-r-u.pbtxt   |  2 +-
 ...rflow.keras.layers.-gaussian-dropout.pbtxt |  2 +-
 ...sorflow.keras.layers.-gaussian-noise.pbtxt |  2 +-
 ...as.layers.-global-average-pooling1-d.pbtxt |  2 +-
 ...as.layers.-global-average-pooling2-d.pbtxt |  2 +-
 ...as.layers.-global-average-pooling3-d.pbtxt |  2 +-
 ...low.keras.layers.-global-avg-pool1-d.pbtxt |  2 +-
 ...low.keras.layers.-global-avg-pool2-d.pbtxt |  2 +-
 ...low.keras.layers.-global-avg-pool3-d.pbtxt |  2 +-
 ...low.keras.layers.-global-max-pool1-d.pbtxt |  2 +-
 ...low.keras.layers.-global-max-pool2-d.pbtxt |  2 +-
 ...low.keras.layers.-global-max-pool3-d.pbtxt |  2 +-
 ....keras.layers.-global-max-pooling1-d.pbtxt |  2 +-
 ....keras.layers.-global-max-pooling2-d.pbtxt |  2 +-
 ....keras.layers.-global-max-pooling3-d.pbtxt |  2 +-
 ...tensorflow.keras.layers.-input-layer.pbtxt |  2 +-
 ...ensorflow.keras.layers.-l-s-t-m-cell.pbtxt |  2 +-
 .../v2/tensorflow.keras.layers.-l-s-t-m.pbtxt |  2 +-
 .../v2/tensorflow.keras.layers.-lambda.pbtxt  |  2 +-
 .../v2/tensorflow.keras.layers.-layer.pbtxt   |  2 +-
 ...ensorflow.keras.layers.-leaky-re-l-u.pbtxt |  2 +-
 ...ensorflow.keras.layers.-linear-model.pbtxt |  2 +-
 ...w.keras.layers.-locally-connected1-d.pbtxt |  2 +-
 ...w.keras.layers.-locally-connected2-d.pbtxt |  2 +-
 .../v2/tensorflow.keras.layers.-masking.pbtxt |  2 +-
 ...tensorflow.keras.layers.-max-pool1-d.pbtxt |  2 +-
 ...tensorflow.keras.layers.-max-pool2-d.pbtxt |  2 +-
 ...tensorflow.keras.layers.-max-pool3-d.pbtxt |  2 +-
 ...sorflow.keras.layers.-max-pooling1-d.pbtxt |  2 +-
 ...sorflow.keras.layers.-max-pooling2-d.pbtxt |  2 +-
 ...sorflow.keras.layers.-max-pooling3-d.pbtxt |  2 +-
 .../v2/tensorflow.keras.layers.-maximum.pbtxt |  2 +-
 .../v2/tensorflow.keras.layers.-minimum.pbtxt |  2 +-
 .../tensorflow.keras.layers.-multiply.pbtxt   |  2 +-
 .../tensorflow.keras.layers.-p-re-l-u.pbtxt   |  2 +-
 .../v2/tensorflow.keras.layers.-permute.pbtxt |  2 +-
 .../v2/tensorflow.keras.layers.-r-n-n.pbtxt   |  2 +-
 .../v2/tensorflow.keras.layers.-re-l-u.pbtxt  |  2 +-
 ...nsorflow.keras.layers.-repeat-vector.pbtxt |  2 +-
 .../v2/tensorflow.keras.layers.-reshape.pbtxt |  2 +-
 ...flow.keras.layers.-separable-conv1-d.pbtxt |  2 +-
 ...flow.keras.layers.-separable-conv2-d.pbtxt |  2 +-
 ...ras.layers.-separable-convolution1-d.pbtxt |  2 +-
 ...ras.layers.-separable-convolution2-d.pbtxt |  2 +-
 ...flow.keras.layers.-simple-r-n-n-cell.pbtxt |  2 +-
 ...ensorflow.keras.layers.-simple-r-n-n.pbtxt |  2 +-
 .../v2/tensorflow.keras.layers.-softmax.pbtxt |  2 +-
 ...low.keras.layers.-spatial-dropout1-d.pbtxt |  2 +-
 ...low.keras.layers.-spatial-dropout2-d.pbtxt |  2 +-
 ...low.keras.layers.-spatial-dropout3-d.pbtxt |  2 +-
 ...ow.keras.layers.-stacked-r-n-n-cells.pbtxt |  2 +-
 .../tensorflow.keras.layers.-subtract.pbtxt   |  2 +-
 ...low.keras.layers.-thresholded-re-l-u.pbtxt |  2 +-
 ...rflow.keras.layers.-time-distributed.pbtxt |  2 +-
 ...sorflow.keras.layers.-up-sampling1-d.pbtxt |  2 +-
 ...sorflow.keras.layers.-up-sampling2-d.pbtxt |  2 +-
 ...sorflow.keras.layers.-up-sampling3-d.pbtxt |  2 +-
 .../v2/tensorflow.keras.layers.-wrapper.pbtxt |  2 +-
 ...orflow.keras.layers.-zero-padding1-d.pbtxt |  2 +-
 ...orflow.keras.layers.-zero-padding2-d.pbtxt |  2 +-
 ...orflow.keras.layers.-zero-padding3-d.pbtxt |  2 +-
 ...rs.experimental.-layer-normalization.pbtxt |  2 +-
 .../v2/tensorflow.keras.models.-model.pbtxt   |  2 +-
 .../tensorflow.keras.models.-sequential.pbtxt |  2 +-
 210 files changed, 239 insertions(+), 214 deletions(-)

diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index e91dce5e5d..ca76823741 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -244,8 +244,8 @@ class Layer(checkpointable.Checkpointable):
 
   @doc_controls.for_subclass_implementers
   def add_weight(self,
-                 name,
-                 shape,
+                 name=None,
+                 shape=None,
                  dtype=None,
                  initializer=None,
                  regularizer=None,
@@ -259,8 +259,8 @@ class Layer(checkpointable.Checkpointable):
     """Adds a new variable to the layer.
 
     Arguments:
-      name: variable name.
-      shape: variable shape.
+      name: Variable name.
+      shape: Variable shape. Defaults to scalar if unspecified.
       dtype: The type of the variable. Defaults to `self.dtype` or `float32`.
       initializer: initializer instance (callable).
       regularizer: regularizer instance (callable).
@@ -297,6 +297,7 @@ class Layer(checkpointable.Checkpointable):
       ValueError: When giving unsupported dtype and no initializer or when
         trainable has been set to True with synchronization set as `ON_READ`.
     """
+    shape = shape or ()
     # Validate optional keyword arguments.
     for kwarg in kwargs:
       if kwarg not in ['getter', 'collections']:
@@ -363,8 +364,10 @@ class Layer(checkpointable.Checkpointable):
       # TODO(fchollet): in the future, this should be handled at the
       # level of variable creation, and weight regularization losses
       # should be variable attributes.
-      self._handle_weight_regularization(name, variable, regularizer)
-
+      name_in_scope = variable.name[:variable.name.find(':')]
+      self._handle_weight_regularization(name_in_scope,
+                                         variable,
+                                         regularizer)
     if trainable:
       self._trainable_weights.append(variable)
     else:
diff --git a/tensorflow/python/keras/engine/base_layer_test.py b/tensorflow/python/keras/engine/base_layer_test.py
index faeb4c5f40..669b1df576 100644
--- a/tensorflow/python/keras/engine/base_layer_test.py
+++ b/tensorflow/python/keras/engine/base_layer_test.py
@@ -225,6 +225,28 @@ class BaseLayerTest(keras_parameterized.TestCase):
     model(np.zeros((2, 4), dtype='float32'))
     self.assertTrue(model.built)
 
+  @test_util.run_in_graph_and_eager_modes
+  def test_default_add_weight(self):
+
+    class TestLayer(keras.layers.Layer):
+
+      def __init__(self):
+        super(TestLayer, self).__init__()
+        self.default_weight = self.add_weight()
+        self.weight_without_name = self.add_weight(shape=(3, 4))
+        self.regularized_weight_without_name = self.add_weight(
+            shape=(3, 4), regularizer='l2')
+
+    layer = TestLayer()
+    self.assertEqual(layer.default_weight.shape.as_list(), [])
+    self.assertEqual(layer.weight_without_name.shape.as_list(), [3, 4])
+    self.assertEqual(layer.default_weight.dtype.name, 'float32')
+    self.assertEqual(layer.weight_without_name.dtype.name, 'float32')
+    self.assertEqual(len(layer.losses), 1)
+    if not context.executing_eagerly():
+      # Cannot access tensor.name in eager execution.
+      self.assertTrue('Variable_2/Regularizer' in layer.losses[0].name)
+
   def test_learning_phase_freezing_for_layers(self):
     # This test is only meant to run in graph functions mode (ambient eager).
     # In forced eager, `model.predict` ignores the global learning phase
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt
index 283cc6a735..bb44ba0b86 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt
@@ -135,7 +135,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt
index 95e405aeba..44fc15ecb5 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt
@@ -140,7 +140,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt
index 1bfd51cdcc..4f7ace494d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activation.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activation.pbtxt
index 8a0b8eb46f..eab888cb80 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activation.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activation.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activity-regularization.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activity-regularization.pbtxt
index abb3c23694..96c7acca9e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activity-regularization.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activity-regularization.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-add.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-add.pbtxt
index b27db4e7f2..9e8aae118e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-add.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-add.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-alpha-dropout.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-alpha-dropout.pbtxt
index 50998ac9d6..01fc730c71 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-alpha-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-alpha-dropout.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling1-d.pbtxt
index be17aeafb5..8b6a151987 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling1-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling2-d.pbtxt
index 7f21b444bc..3c78457507 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling2-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling3-d.pbtxt
index 2ac86f152f..e6e96a08a0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling3-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average.pbtxt
index f6b1dd2f7e..ec2d5b179b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool1-d.pbtxt
index 3da1f43a92..afff7907e8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool1-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool2-d.pbtxt
index a7be5ac818..d7ab835f74 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool2-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool3-d.pbtxt
index c5c29bead3..6654f86025 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool3-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-batch-normalization.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-batch-normalization.pbtxt
index 3af3c2a501..a328d9f656 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-batch-normalization.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-batch-normalization.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-bidirectional.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-bidirectional.pbtxt
index 880d18e1aa..94f3a46e48 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-bidirectional.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-bidirectional.pbtxt
@@ -115,7 +115,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-concatenate.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-concatenate.pbtxt
index 1eb0cf1a18..e0eae17311 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-concatenate.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-concatenate.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt
index d9394e60f5..ec8a44c55c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt
@@ -196,7 +196,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv1-d.pbtxt
index a0f6dc8097..350d49aa22 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv1-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d-transpose.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d-transpose.pbtxt
index 037b92f861..9b48eb6f26 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d-transpose.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d.pbtxt
index 6a0d027d47..1708d6ab73 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d-transpose.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d-transpose.pbtxt
index 66b5bd75fc..50184922a1 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d-transpose.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d.pbtxt
index e73133ff07..fd24af307f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution1-d.pbtxt
index 7af6b2b3c3..fbc7609593 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution1-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt
index baff492dfb..671a004629 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d.pbtxt
index 63d30a6185..dd6519c388 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt
index 7a29cbbec3..648f480ed9 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d.pbtxt
index 87c75c0224..87a07ea77b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping1-d.pbtxt
index f69104ddfe..6f3a15314e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping1-d.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping2-d.pbtxt
index aa05471933..a1c418c446 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping2-d.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping3-d.pbtxt
index d61f1ddc1d..ad98f9c9b9 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping3-d.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt
index e2d05f8298..e35403b8f8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt
@@ -116,7 +116,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt
index f650f48423..90d03ea5a4 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt
@@ -116,7 +116,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dense.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dense.pbtxt
index 06e8b6b314..ef12b2ebce 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dense.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dense.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt
index 9fdf6f66d1..eacfb37c27 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dot.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dot.pbtxt
index cbe1020650..7928ceb2bd 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dot.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dot.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dropout.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dropout.pbtxt
index 0efba09b27..a7fa5451c1 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dropout.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-e-l-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-e-l-u.pbtxt
index b34c499eb2..483ba654a9 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-e-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-e-l-u.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-embedding.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-embedding.pbtxt
index 51dd853127..4d0e5e1bae 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-embedding.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-embedding.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-flatten.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-flatten.pbtxt
index dcd18a9ced..5947047bf8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-flatten.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-flatten.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u-cell.pbtxt
index f029907ee8..b4efdf32dc 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u-cell.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u.pbtxt
index 278ae06cba..db4d981c2c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u.pbtxt
@@ -179,7 +179,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-dropout.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-dropout.pbtxt
index 15cbcfe8ed..168676875a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-dropout.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-noise.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-noise.pbtxt
index 865b898c4c..69bca6ab1d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-noise.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-noise.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt
index 3e17aca17c..9a4119deca 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt
index b160687a2a..2ca1eb1176 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt
index 70e8d51a5a..4331adc14c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt
index 809dc8554b..6e91b4ab42 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt
index 3fbce8cb71..85887a52f3 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt
index 70e4103ea1..dd20fd1bb7 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool1-d.pbtxt
index 000bf54c45..3372ae7c43 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool1-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool2-d.pbtxt
index 8ffbf07f9b..0fb1882393 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool2-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool3-d.pbtxt
index 3803d2b0a8..5b1c850d1e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool3-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt
index 28668224e0..49e59e0872 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt
index b83ed67723..9504f64ed8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt
index e689d69140..42de6aef4f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-layer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-layer.pbtxt
index bb6eddae71..f388b84dc6 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-layer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-layer.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt
index 5fb3f9dd3a..d2634ddf69 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m.pbtxt
index 8eb6dd9f4a..94ec4321f8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m.pbtxt
@@ -179,7 +179,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-lambda.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-lambda.pbtxt
index 376bec0814..da2373cd5d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-lambda.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-lambda.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-layer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-layer.pbtxt
index c5f91a6338..2e47132817 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-layer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-layer.pbtxt
@@ -105,7 +105,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-leaky-re-l-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-leaky-re-l-u.pbtxt
index bde8887359..a74e935f32 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-leaky-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-leaky-re-l-u.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected1-d.pbtxt
index 16945f2c12..0f4c071e86 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected1-d.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected2-d.pbtxt
index f05741ffce..5eea071b64 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected2-d.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-masking.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-masking.pbtxt
index 7885db4ed2..a16ceef5a7 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-masking.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-masking.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool1-d.pbtxt
index 9380d26cf4..e61d7300e8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool1-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool2-d.pbtxt
index 8eb8218df3..a21c403463 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool2-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool3-d.pbtxt
index 0c96f86ed3..fb8613a84d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool3-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling1-d.pbtxt
index 0c6b230eb7..a433d49edb 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling1-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling2-d.pbtxt
index eb7ca52fba..fa6ad6f70f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling2-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling3-d.pbtxt
index e724e9088f..05e2aceafe 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling3-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-maximum.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-maximum.pbtxt
index dafbd09ee2..ce62223ef8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-maximum.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-maximum.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-minimum.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-minimum.pbtxt
index 3122fbec1c..a0ff4f9689 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-minimum.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-minimum.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-multiply.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-multiply.pbtxt
index 0527cda1f0..558cc0da43 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-multiply.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-multiply.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-p-re-l-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-p-re-l-u.pbtxt
index 814e5a5d54..5863fbb20c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-p-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-p-re-l-u.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-permute.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-permute.pbtxt
index aa1731afb8..4d7413b92c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-permute.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-permute.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-r-n-n.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-r-n-n.pbtxt
index 9d7dd85fe0..67ab60b5bf 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-r-n-n.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-r-n-n.pbtxt
@@ -110,7 +110,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-re-l-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-re-l-u.pbtxt
index e9bba298bb..eb32ba2d9f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-re-l-u.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-repeat-vector.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-repeat-vector.pbtxt
index 3c783eb512..81ac2539a0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-repeat-vector.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-repeat-vector.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-reshape.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-reshape.pbtxt
index b8e0882541..dd4dc49b0a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-reshape.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-reshape.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv1-d.pbtxt
index 310f369ed6..c8724f01f8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv1-d.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv2-d.pbtxt
index df19d781c2..8c47395dc0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv2-d.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution1-d.pbtxt
index bf909509bd..c0b6ad428e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution1-d.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution2-d.pbtxt
index 5d66bc6fb6..c5566c120f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution2-d.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt
index 88e9300de9..f91aac802e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n.pbtxt
index 9d81c6d4bc..eb2a7b920e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n.pbtxt
@@ -167,7 +167,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-softmax.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-softmax.pbtxt
index 712eb0c6ec..f0411e2fd4 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-softmax.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-softmax.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt
index dfc4ca2705..2a2fd2e643 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt
index 5e4f727f71..e4d1d43194 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt
index 9d893cb30a..4e641a8ed0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt
index a2ed954e4c..591796e71b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt
@@ -114,7 +114,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-subtract.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-subtract.pbtxt
index 8a0818e78a..67555db2c6 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-subtract.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-subtract.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt
index b5591b4826..0ed7da5128 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-time-distributed.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-time-distributed.pbtxt
index 210e4fd4e6..9492b0be62 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-time-distributed.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-time-distributed.pbtxt
@@ -111,7 +111,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling1-d.pbtxt
index da2213a84f..16c31d3f03 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling1-d.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling2-d.pbtxt
index e2c303d506..cf1a076ef1 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling2-d.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling3-d.pbtxt
index 396e774c8a..5cded98c5a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling3-d.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-wrapper.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-wrapper.pbtxt
index 8b6418d514..16f3f0658e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-wrapper.pbtxt
@@ -110,7 +110,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding1-d.pbtxt
index e8fda4c71a..59997a8968 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding1-d.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding2-d.pbtxt
index 50c52d270b..9a327c2ae9 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding2-d.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding3-d.pbtxt
index 84c6b78a2b..79338685fb 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding3-d.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.-layer-normalization.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.-layer-normalization.pbtxt
index 476c597fe6..6c8faef6de 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.-layer-normalization.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.-layer-normalization.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt
index eb1ab1d9dd..3132e8dba7 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt
@@ -135,7 +135,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt
index c69cf28174..b5ef70e744 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt
@@ -140,7 +140,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt
index 283cc6a735..bb44ba0b86 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt
@@ -135,7 +135,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt
index 95e405aeba..44fc15ecb5 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt
@@ -140,7 +140,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt
index 1bfd51cdcc..4f7ace494d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activation.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activation.pbtxt
index 8a0b8eb46f..eab888cb80 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activation.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activation.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activity-regularization.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activity-regularization.pbtxt
index abb3c23694..96c7acca9e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activity-regularization.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activity-regularization.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-add.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-add.pbtxt
index b27db4e7f2..9e8aae118e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-add.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-add.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-alpha-dropout.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-alpha-dropout.pbtxt
index 50998ac9d6..01fc730c71 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-alpha-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-alpha-dropout.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling1-d.pbtxt
index be17aeafb5..8b6a151987 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling1-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling2-d.pbtxt
index 7f21b444bc..3c78457507 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling2-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling3-d.pbtxt
index 2ac86f152f..e6e96a08a0 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling3-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average.pbtxt
index f6b1dd2f7e..ec2d5b179b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool1-d.pbtxt
index 3da1f43a92..afff7907e8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool1-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool2-d.pbtxt
index a7be5ac818..d7ab835f74 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool2-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool3-d.pbtxt
index c5c29bead3..6654f86025 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool3-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-batch-normalization.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-batch-normalization.pbtxt
index b13f963a6f..05ac793c6b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-batch-normalization.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-batch-normalization.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-bidirectional.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-bidirectional.pbtxt
index 880d18e1aa..94f3a46e48 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-bidirectional.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-bidirectional.pbtxt
@@ -115,7 +115,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-concatenate.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-concatenate.pbtxt
index 1eb0cf1a18..e0eae17311 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-concatenate.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-concatenate.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt
index d9394e60f5..ec8a44c55c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt
@@ -196,7 +196,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv1-d.pbtxt
index a0f6dc8097..350d49aa22 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv1-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d-transpose.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d-transpose.pbtxt
index 037b92f861..9b48eb6f26 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d-transpose.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d.pbtxt
index 6a0d027d47..1708d6ab73 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d-transpose.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d-transpose.pbtxt
index 66b5bd75fc..50184922a1 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d-transpose.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d.pbtxt
index e73133ff07..fd24af307f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution1-d.pbtxt
index 7af6b2b3c3..fbc7609593 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution1-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt
index baff492dfb..671a004629 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d.pbtxt
index 63d30a6185..dd6519c388 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt
index 7a29cbbec3..648f480ed9 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d.pbtxt
index 87c75c0224..87a07ea77b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping1-d.pbtxt
index f69104ddfe..6f3a15314e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping1-d.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping2-d.pbtxt
index aa05471933..a1c418c446 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping2-d.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping3-d.pbtxt
index d61f1ddc1d..ad98f9c9b9 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping3-d.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt
index 3caa3ff4ed..ca6a327a58 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense.pbtxt
index 06e8b6b314..ef12b2ebce 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt
index 9fdf6f66d1..eacfb37c27 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dot.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dot.pbtxt
index cbe1020650..7928ceb2bd 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dot.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dot.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dropout.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dropout.pbtxt
index 0efba09b27..a7fa5451c1 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dropout.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-e-l-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-e-l-u.pbtxt
index b34c499eb2..483ba654a9 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-e-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-e-l-u.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-embedding.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-embedding.pbtxt
index 51dd853127..4d0e5e1bae 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-embedding.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-embedding.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-flatten.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-flatten.pbtxt
index dcd18a9ced..5947047bf8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-flatten.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-flatten.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u-cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u-cell.pbtxt
index f029907ee8..b4efdf32dc 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u-cell.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u.pbtxt
index ac2d8c9aa3..811990a95c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u.pbtxt
@@ -180,7 +180,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-dropout.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-dropout.pbtxt
index 15cbcfe8ed..168676875a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-dropout.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-noise.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-noise.pbtxt
index 865b898c4c..69bca6ab1d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-noise.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-noise.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt
index 3e17aca17c..9a4119deca 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt
index b160687a2a..2ca1eb1176 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt
index 70e8d51a5a..4331adc14c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt
index 809dc8554b..6e91b4ab42 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt
index 3fbce8cb71..85887a52f3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt
index 70e4103ea1..dd20fd1bb7 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool1-d.pbtxt
index 000bf54c45..3372ae7c43 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool1-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool2-d.pbtxt
index 8ffbf07f9b..0fb1882393 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool2-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool3-d.pbtxt
index 3803d2b0a8..5b1c850d1e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool3-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt
index 28668224e0..49e59e0872 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt
index b83ed67723..9504f64ed8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt
index e689d69140..42de6aef4f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-layer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-layer.pbtxt
index bb6eddae71..f388b84dc6 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-layer.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-layer.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt
index 5fb3f9dd3a..d2634ddf69 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m.pbtxt
index 89dfc2a256..622a8e2714 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m.pbtxt
@@ -180,7 +180,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-lambda.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-lambda.pbtxt
index 376bec0814..da2373cd5d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-lambda.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-lambda.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-layer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-layer.pbtxt
index c5f91a6338..2e47132817 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-layer.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-layer.pbtxt
@@ -105,7 +105,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-leaky-re-l-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-leaky-re-l-u.pbtxt
index bde8887359..a74e935f32 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-leaky-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-leaky-re-l-u.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt
index c4726cf824..bf6c84b6b4 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt
@@ -140,7 +140,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected1-d.pbtxt
index 16945f2c12..0f4c071e86 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected1-d.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected2-d.pbtxt
index f05741ffce..5eea071b64 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected2-d.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-masking.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-masking.pbtxt
index 7885db4ed2..a16ceef5a7 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-masking.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-masking.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool1-d.pbtxt
index 9380d26cf4..e61d7300e8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool1-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool2-d.pbtxt
index 8eb8218df3..a21c403463 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool2-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool3-d.pbtxt
index 0c96f86ed3..fb8613a84d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool3-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling1-d.pbtxt
index 0c6b230eb7..a433d49edb 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling1-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling2-d.pbtxt
index eb7ca52fba..fa6ad6f70f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling2-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling3-d.pbtxt
index e724e9088f..05e2aceafe 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling3-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-maximum.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-maximum.pbtxt
index dafbd09ee2..ce62223ef8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-maximum.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-maximum.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-minimum.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-minimum.pbtxt
index 3122fbec1c..a0ff4f9689 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-minimum.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-minimum.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-multiply.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-multiply.pbtxt
index 0527cda1f0..558cc0da43 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-multiply.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-multiply.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-p-re-l-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-p-re-l-u.pbtxt
index 814e5a5d54..5863fbb20c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-p-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-p-re-l-u.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-permute.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-permute.pbtxt
index aa1731afb8..4d7413b92c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-permute.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-permute.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-r-n-n.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-r-n-n.pbtxt
index 9d7dd85fe0..67ab60b5bf 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-r-n-n.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-r-n-n.pbtxt
@@ -110,7 +110,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-re-l-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-re-l-u.pbtxt
index e9bba298bb..eb32ba2d9f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-re-l-u.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-repeat-vector.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-repeat-vector.pbtxt
index 3c783eb512..81ac2539a0 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-repeat-vector.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-repeat-vector.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-reshape.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-reshape.pbtxt
index b8e0882541..dd4dc49b0a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-reshape.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-reshape.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv1-d.pbtxt
index 310f369ed6..c8724f01f8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv1-d.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv2-d.pbtxt
index df19d781c2..8c47395dc0 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv2-d.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution1-d.pbtxt
index bf909509bd..c0b6ad428e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution1-d.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution2-d.pbtxt
index 5d66bc6fb6..c5566c120f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution2-d.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt
index 88e9300de9..f91aac802e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n.pbtxt
index 9d81c6d4bc..eb2a7b920e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n.pbtxt
@@ -167,7 +167,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-softmax.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-softmax.pbtxt
index 712eb0c6ec..f0411e2fd4 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-softmax.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-softmax.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt
index dfc4ca2705..2a2fd2e643 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt
index 5e4f727f71..e4d1d43194 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt
index 9d893cb30a..4e641a8ed0 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt
index a2ed954e4c..591796e71b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt
@@ -114,7 +114,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-subtract.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-subtract.pbtxt
index 8a0818e78a..67555db2c6 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-subtract.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-subtract.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt
index b5591b4826..0ed7da5128 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-time-distributed.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-time-distributed.pbtxt
index 210e4fd4e6..9492b0be62 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-time-distributed.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-time-distributed.pbtxt
@@ -111,7 +111,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling1-d.pbtxt
index da2213a84f..16c31d3f03 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling1-d.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling2-d.pbtxt
index e2c303d506..cf1a076ef1 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling2-d.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling3-d.pbtxt
index 396e774c8a..5cded98c5a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling3-d.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-wrapper.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-wrapper.pbtxt
index 8b6418d514..16f3f0658e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-wrapper.pbtxt
@@ -110,7 +110,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding1-d.pbtxt
index e8fda4c71a..59997a8968 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding1-d.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding2-d.pbtxt
index 50c52d270b..9a327c2ae9 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding2-d.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding3-d.pbtxt
index 84c6b78a2b..79338685fb 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding3-d.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.-layer-normalization.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.-layer-normalization.pbtxt
index 476c597fe6..6c8faef6de 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.-layer-normalization.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.-layer-normalization.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt
index eb1ab1d9dd..3132e8dba7 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt
@@ -135,7 +135,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt
index c69cf28174..b5ef70e744 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt
@@ -140,7 +140,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "apply"
-- 
GitLab


From 9d5975e241cea25d191b89764fe4f0d68e5d17f4 Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Mon, 4 Feb 2019 16:03:20 -0800
Subject: [PATCH 1880/2345] Removing spurious print statements.

PiperOrigin-RevId: 232384630
---
 tensorflow/python/feature_column/feature_column_v2.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/tensorflow/python/feature_column/feature_column_v2.py b/tensorflow/python/feature_column/feature_column_v2.py
index 4b93c2f28d..edc37193e5 100644
--- a/tensorflow/python/feature_column/feature_column_v2.py
+++ b/tensorflow/python/feature_column/feature_column_v2.py
@@ -4022,13 +4022,9 @@ class WeightedCategoricalColumn(
 
   def transform_feature(self, transformation_cache, state_manager):
     """Applies weights to tensor generated from `categorical_column`'."""
-    print('WeightedCategoricalColumn.transform_feature: ', self.name)
-    print('Weight feature key: ', self.weight_feature_key)
     weight_tensor = transformation_cache.get(self.weight_feature_key,
                                              state_manager)
-    print('Weight tensor before: ', weight_tensor)
     weight_tensor = self._transform_weight_tensor(weight_tensor)
-    print('Weight tensor after: ', weight_tensor)
     return (transformation_cache.get(self.categorical_column, state_manager),
             weight_tensor)
 
@@ -4042,9 +4038,7 @@ class WeightedCategoricalColumn(
 
   def get_sparse_tensors(self, transformation_cache, state_manager):
     """See `CategoricalColumn` base class."""
-    print('WeightedCategoricalColumn.get_sparse_tensors: ', self.name)
     tensors = transformation_cache.get(self, state_manager)
-    print('tensors[1]: ', tensors[1])
     return CategoricalColumn.IdWeightPair(tensors[0], tensors[1])
 
   @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE,
-- 
GitLab


From 4fde512c819e781bf276f38028b08c8ae419bf7e Mon Sep 17 00:00:00 2001
From: nammbash <niroopshankar.ammbashankar@intel.com>
Date: Mon, 4 Feb 2019 16:31:16 -0800
Subject: [PATCH 1881/2345] change comments for test

---
 .../core/kernels/mkl_requantize_ops_test.cc       | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/tensorflow/core/kernels/mkl_requantize_ops_test.cc b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
index 591830f775..95ee4fffe7 100644
--- a/tensorflow/core/kernels/mkl_requantize_ops_test.cc
+++ b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
@@ -55,16 +55,13 @@ void MklRequantizatedOpsTestHelper::Setup(Tensor &input_tensor_qint32,
   // ------------------------------------------
   // T and W are quantized using a Quantize Op.
   // The input Tensor T (NHWC) is quantized to unsigned int8.
+  // Hence T's max value is mapped to ((2^8-1) = 255).
   // The input Filter W (NHWC) is quantized to signed int8.
-  // Hence T's max value is mapped to ((2^8-1) = 255), while W's to ((2^7)-1 =
-  // 127)).
-
-  // Range of Quantized T  in uint8[0   , 255] maps to orig T  in FP32[0   ,
-  // 5.0]
-  // Range of Quantized W1 in int8[-127, 127]  maps to orig W1 in FP32[-2.0,
-  // 2.0]
-  // Range of Quantized W2 in int8[-127, 127]  maps to orig W2 in FP32[-3.0,
-  // 3.0]
+  // Hence W's max value is mapped to ((2^7)-1 = 127)).
+
+  // Range of Quantized T  in uint8[0  , 255] maps to orig T  in FP32[0   , 5.0]
+  // Range of Quantized W1 in int8[-127, 127] maps to orig W1 in FP32[-2.0, 2.0]
+  // Range of Quantized W2 in int8[-127, 127] maps to orig W2 in FP32[-3.0, 3.0]
 
   // Hence the resolution of Quantized T will be 5.0/255
   // Hence the resolution of Quantized W1 will be 2.0/127
-- 
GitLab


From 839cb1d099ebc131290b9850bd96404391ca4483 Mon Sep 17 00:00:00 2001
From: Mahmoud Abuzaina <mahmoud.abuzaina@intel.com>
Date: Mon, 4 Feb 2019 16:34:40 -0800
Subject: [PATCH 1882/2345] Added a comment

---
 tensorflow/core/kernels/mkl_quantized_concat_op_test.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/core/kernels/mkl_quantized_concat_op_test.cc b/tensorflow/core/kernels/mkl_quantized_concat_op_test.cc
index d483b85d48..dcd1de95e0 100644
--- a/tensorflow/core/kernels/mkl_quantized_concat_op_test.cc
+++ b/tensorflow/core/kernels/mkl_quantized_concat_op_test.cc
@@ -223,6 +223,7 @@ void QuantizedConcatTest::TestSecondDim8Bit(float first_min, float first_max,
   const float output_max = GetOutput(2)->flat<float>()(0);
   Tensor output_float =
       QuantizedTensorToFloat<quint8>(output_quantized, output_min, output_max);
+  // Using the same error tolerance as in Eigen QuantizedConcat test
   test::ExpectTensorNear<float>(expected_float, output_float, 1.0);
 }
 
-- 
GitLab


From 02f37497bd0b242f4a63a3be3c8c0fe022df4d1f Mon Sep 17 00:00:00 2001
From: nammbash <niroopshankar.ammbashankar@intel.com>
Date: Mon, 4 Feb 2019 16:44:11 -0800
Subject: [PATCH 1883/2345] change comments for test 2

---
 .../core/kernels/mkl_requantize_ops_test.cc   | 66 +++++++++----------
 1 file changed, 33 insertions(+), 33 deletions(-)

diff --git a/tensorflow/core/kernels/mkl_requantize_ops_test.cc b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
index 95ee4fffe7..83577ffb68 100644
--- a/tensorflow/core/kernels/mkl_requantize_ops_test.cc
+++ b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
@@ -45,32 +45,32 @@ void MklRequantizatedOpsTestHelper::Setup(Tensor &input_tensor_qint32,
                                           float &range_weights_ch2) {
   // Step 1: Input range assumptions
   // -------------------------------
-  // Assume input Tensor T (NHWC) in FP32 has range [0, 5.0]   size nt*ht*wt*ct
-  // Assume input Filter W (NHWC) with 2 output channels of    size nw*ht*wt*2
-  // logically,   Filter W has 2 channels W1 and W2 each of    size nw*ht*wt*1
-  // Assume input Filter W1(NHWC) in FP32 has range [-2.0, 2.0]size nw*ht*wt*1
-  // Assume input Filter W2(NHWC) in FP32 has range [-3.0, 3.0]size nw*ht*wt*1
+  // Assume input tensor T (NHWC) in FP32 has range [0, 5.0]   size nt*ht*wt*ct
+  // Assume input filter W (NHWC) with 2 output channels of    size nw*ht*wt*2
+  // logically,   filter W has 2 channels W1 and W2 each of    size nw*ht*wt*1
+  // Assume input filter W1(NHWC) in FP32 has range [-2.0, 2.0]size nw*ht*wt*1
+  // Assume input filter W2(NHWC) in FP32 has range [-3.0, 3.0]size nw*ht*wt*1
 
   // Step 2: Quantization details (per channel)
   // ------------------------------------------
-  // T and W are quantized using a Quantize Op.
-  // The input Tensor T (NHWC) is quantized to unsigned int8.
+  // T and W are quantized using a quantize Op.
+  // The input tensor T (NHWC) is quantized to unsigned int8.
   // Hence T's max value is mapped to ((2^8-1) = 255).
-  // The input Filter W (NHWC) is quantized to signed int8.
+  // The input filter W (NHWC) is quantized to signed int8.
   // Hence W's max value is mapped to ((2^7)-1 = 127)).
 
-  // Range of Quantized T  in uint8[0  , 255] maps to orig T  in FP32[0   , 5.0]
-  // Range of Quantized W1 in int8[-127, 127] maps to orig W1 in FP32[-2.0, 2.0]
-  // Range of Quantized W2 in int8[-127, 127] maps to orig W2 in FP32[-3.0, 3.0]
+  // Range of quantized T  in uint8[0  , 255] maps to orig T  in FP32[0   , 5.0]
+  // Range of quantized W1 in int8[-127, 127] maps to orig W1 in FP32[-2.0, 2.0]
+  // Range of quantized W2 in int8[-127, 127] maps to orig W2 in FP32[-3.0, 3.0]
 
-  // Hence the resolution of Quantized T will be 5.0/255
-  // Hence the resolution of Quantized W1 will be 2.0/127
-  // Hence the resolution of Quantized W2 will be 3.0/127
+  // Hence the resolution of quantized T will be 5.0/255
+  // Hence the resolution of quantized W1 will be 2.0/127
+  // Hence the resolution of quantized W2 will be 3.0/127
 
   // Step 3: Assumption of quantizedconv on quantized input&weights(per channel)
   // ---------------------------------------------------------------------------
   // The input T and weights W1 (or W2) will be convolved.
-  // The output Tensor T is in int32 whose range is [-2^31, 2^31).
+  // The output tensor T is in int32 whose range is [-2^31, 2^31).
   // For simplicity, we truncate the range to (-2^31, 2^31) to make it
   // symmetric.
   // The range of the convolved T*W1 is ((2^31)-1) * 5.0/255 * 2.0/127 =
@@ -82,17 +82,17 @@ void MklRequantizatedOpsTestHelper::Setup(Tensor &input_tensor_qint32,
   // So the range of convolved T*W1 in int32(-2^31, 2^31) that maps to
   // orig T range in FP32 [0, 5.0] * [-3.0, 3.0]  is [-994665.88, 994665.88]
 
-  // Step 4: Assumption output above is fed to Requantization_range_perchannel
+  // Step 4: Assumption output above is fed to requantization_range_perchannel
   // --------------------------------------------------------------------------
   // Here we recalculate the new range for convolved T*W so that we
   // make good use in int8 quantization from int32 to int8.
 
   // We assume the above operations are performed and use these values above
-  // as ranges for Requantization_range_perchannel_op.
+  // as ranges for requantization_range_perchannel_op.
   range_weights_ch1 = 663110.59;  // For W1 channel
   range_weights_ch2 = 994665.88;  // For W2 Channel
 
-  // We Fill the input Tensor T qint32 with arbitrary int32 values
+  // We Fill the input tensor T qint32 with arbitrary int32 values
   test::FillValues<qint32>(
       &input_tensor_qint32,
       {-1000, -2000,  2000,   4000,   -3000,  -6000,  4000,   8000,
@@ -150,14 +150,14 @@ TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_Basic) {
   float ch2_min = -range_weights_ch2;
   float ch2_max = range_weights_ch2;
 
-  // Add the Perchannel range Nodes to the Op.
+  // Add the perchannel range Nodes to the Op.
   AddInputFromArray<float>(TensorShape({input_channels}), {ch1_min, ch2_min});
   AddInputFromArray<float>(TensorShape({input_channels}), {ch1_max, ch2_max});
 
-  // Run the Kernel
+  // Run the kernel
   TF_ASSERT_OK(RunOpKernel());
 
-  // Step 6: Verify Output and Store values to test Requantize_perchannel
+  // Step 6: Verify output and store values to test requantize_perchannel
   // --------------------------------------------------------------------
 
   // Verify the Expected Outputs
@@ -166,7 +166,7 @@ TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_Basic) {
   EXPECT_NEAR(-14.8217, output_min, 0.002);
   EXPECT_NEAR(14.8217, output_max, 0.002);
 
-  // output range is made use in RequantizePerChannelTest_Basic
+  // output range is made use in requantizePerChannelTest_Basic
 }
 
 TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_ClipMax) {
@@ -205,23 +205,23 @@ TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_ClipMax) {
   AddInputFromArray<qint32>(input_tensor_qint32.shape(),
                             input_tensor_qint32.flat<qint32>());
 
-  // Calculate the Min and max from the ranges
+  // Calculate the min and max from the ranges
   float ch1_min = -range_weights_ch1;
   float ch1_max = range_weights_ch1;
   float ch2_min = -range_weights_ch2;
   float ch2_max = range_weights_ch2;
 
-  // Add the Perchannel range Nodes to the Op.
+  // Add the perchannel range nodes to the Op.
   AddInputFromArray<float>(TensorShape({input_channels}), {ch1_min, ch2_min});
   AddInputFromArray<float>(TensorShape({input_channels}), {ch1_max, ch2_max});
 
-  // Run the Kernel
+  // Run the kernel
   TF_ASSERT_OK(RunOpKernel());
 
-  // Step 6: Verify Output and Store values to test Requantize_perchannel
+  // Step 6: Verify output and store values to test requantize_perchannel
   // --------------------------------------------------------------------
 
-  // Verify the Expected Outputs
+  // Verify the expected outputs
   const float output_min = GetOutput(0)->flat<float>()(0);
   const float output_max = GetOutput(1)->flat<float>()(0);
   EXPECT_NEAR(-6.0, output_min, 0.002);  // Values are Max as with clip_value
@@ -248,7 +248,7 @@ TEST_F(MklRequantizatedOpsTest, RequantizePerChannelTest_Basic) {
 
   // Step 7: Define and run requantize_perchannel
   // --------------------------------------------
-  // The output of Requantization_range_op_per_channel which calculated the
+  // The output of requantization_range_op_per_channel which calculated the
   // new ranges of int8 is fed to the requantize per channel op.
   // Here the Values of Convolved T*W is converted from int32 to int8.
 
@@ -263,26 +263,26 @@ TEST_F(MklRequantizatedOpsTest, RequantizePerChannelTest_Basic) {
                    .Finalize(node_def()));
   TF_ASSERT_OK(InitOp());
 
-  // Add the Input Nodes to the Op.
+  // Add the input Nodes to the Op.
   AddInputFromArray<qint32>(input_tensor_qint32.shape(),
                             input_tensor_qint32.flat<qint32>());
 
-  // Calculate the Min and max from the ranges
+  // Calculate the min and max from the ranges
   float ch1_min = -range_weights_ch1;
   float ch1_max = range_weights_ch1;
   float ch2_min = -range_weights_ch2;
   float ch2_max = range_weights_ch2;
 
-  // Add the Perchannel range Nodes to the Op.
+  // Add the perchannel range nodes to the Op.
   AddInputFromArray<float>(TensorShape({input_channels}), {ch1_min, ch2_min});
   AddInputFromArray<float>(TensorShape({input_channels}), {ch1_max, ch2_max});
 
-  // Calculate the Min and max from Step 6 above
+  // Calculate the min and max from Step 6 above
   // in RequantizationRangePerChannelTest_Basic
   float range_op_output_min = -14.8217;
   float range_op_output_max = 14.8217;
 
-  // Add the Requested_min and requested_max stored from Step 6.
+  // Add the requested_min and requested_max stored from Step 6.
   AddInputFromArray<float>(TensorShape({1}), {range_op_output_min});
   AddInputFromArray<float>(TensorShape({1}), {range_op_output_max});
 
-- 
GitLab


From cee14c8eba9ecdefff804b30c670a0de230ef518 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 4 Feb 2019 16:39:25 -0800
Subject: [PATCH 1884/2345] Automated rollback of commit
 90f1af3e4487f5526ab9974ee07c63c125916499

PiperOrigin-RevId: 232390533
---
 .../core/platform/default/build_config/BUILD      | 15 ---------------
 tensorflow/stream_executor/cuda/BUILD             |  2 +-
 tensorflow/stream_executor/cuda/cudart_stub.cc    |  7 -------
 3 files changed, 1 insertion(+), 23 deletions(-)

diff --git a/tensorflow/core/platform/default/build_config/BUILD b/tensorflow/core/platform/default/build_config/BUILD
index 845fe0ec04..6faf5c512f 100644
--- a/tensorflow/core/platform/default/build_config/BUILD
+++ b/tensorflow/core/platform/default/build_config/BUILD
@@ -7,7 +7,6 @@ licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
-load("//tensorflow:tensorflow.bzl", "check_deps")
 load("//tensorflow:tensorflow.bzl", "if_cuda")
 load("//tensorflow:tensorflow.bzl", "if_rocm")
 load("//tensorflow:tensorflow.bzl", "tf_copts")
@@ -284,20 +283,6 @@ cc_library(
     ],
 )
 
-# Check that libtensorflow_framework.so does not depend on cuda shared libraries.
-check_deps(
-    name = "libtensorflow_cuda_check_deps",
-    disallowed_deps = [
-        ":cuda",
-        "@local_config_cuda//cuda:cublas",
-        "@local_config_cuda//cuda:cuda_driver",
-        "@local_config_cuda//cuda:cudnn",
-        "@local_config_cuda//cuda:curand",
-        "@local_config_cuda//cuda:cusolver",
-    ],
-    deps = ["//tensorflow:libtensorflow_framework.so"],
-)
-
 cc_library(
     name = "rocm",
     data = [],
diff --git a/tensorflow/stream_executor/cuda/BUILD b/tensorflow/stream_executor/cuda/BUILD
index 36b3b167c7..0e23cc3eb4 100644
--- a/tensorflow/stream_executor/cuda/BUILD
+++ b/tensorflow/stream_executor/cuda/BUILD
@@ -83,11 +83,11 @@ cc_library(
     ]),
     deps = if_cuda_is_configured([
         ":cuda_diagnostics",
-        ":cudart_stub",
         "@com_google_absl//absl/base",
         "@com_google_absl//absl/container:inlined_vector",
         "@com_google_absl//absl/strings",
         "@local_config_cuda//cuda:cuda_headers",
+        "//tensorflow/core:cuda",
         "//tensorflow/stream_executor:device_options",
         "//tensorflow/stream_executor/gpu:gpu_driver_header",
         "//tensorflow/stream_executor/lib",
diff --git a/tensorflow/stream_executor/cuda/cudart_stub.cc b/tensorflow/stream_executor/cuda/cudart_stub.cc
index c5fc43d56c..cb35719183 100644
--- a/tensorflow/stream_executor/cuda/cudart_stub.cc
+++ b/tensorflow/stream_executor/cuda/cudart_stub.cc
@@ -111,11 +111,4 @@ cudaError_t CUDART_CB cudaPointerGetAttributes(
   if (!func_ptr) return GetSymbolNotFoundError();
   return func_ptr(attributes, ptr);
 }
-
-cudaError_t CUDART_CB cudaGetLastError() {
-  using FuncPtr = cudaError_t (*)();
-  static auto func_ptr = LoadSymbol<FuncPtr>("cudaGetLastError");
-  if (!func_ptr) return GetSymbolNotFoundError();
-  return func_ptr();
-}
 }  // extern "C"
-- 
GitLab


From afddb8acaefb17f82a2f072d56f468ac036975b3 Mon Sep 17 00:00:00 2001
From: Frank Chen <frankchn@google.com>
Date: Mon, 4 Feb 2019 16:44:29 -0800
Subject: [PATCH 1885/2345] Introduce new C++-based Shard Dataset Op and
 un-deprecate .shard()

PiperOrigin-RevId: 232391347
---
 .../base_api/api_def_ShardDataset.pbtxt       |  17 ++
 tensorflow/core/kernels/data/BUILD            |  12 ++
 .../core/kernels/data/shard_dataset_op.cc     | 195 ++++++++++++++++++
 tensorflow/core/ops/dataset_ops.cc            |  16 ++
 .../kernel_tests/cardinality_test.py          |  29 +--
 .../kernel_tests/serialization/BUILD          |  18 ++
 .../shard_dataset_serialization_test.py       |  42 ++++
 .../python/data/kernel_tests/shard_test.py    |  26 ++-
 tensorflow/python/data/ops/dataset_ops.py     | 126 ++++++-----
 .../api/golden/v1/tensorflow.raw_ops.pbtxt    |   4 +
 .../golden/v2/tensorflow.data.-dataset.pbtxt  |   4 +
 ...ow.data.-fixed-length-record-dataset.pbtxt |   4 +
 .../tensorflow.data.-t-f-record-dataset.pbtxt |   4 +
 .../tensorflow.data.-text-line-dataset.pbtxt  |   4 +
 ...rflow.data.experimental.-csv-dataset.pbtxt |   4 +
 ...ow.data.experimental.-random-dataset.pbtxt |   4 +
 ...rflow.data.experimental.-sql-dataset.pbtxt |   4 +
 .../api/golden/v2/tensorflow.raw_ops.pbtxt    |   4 +
 18 files changed, 442 insertions(+), 75 deletions(-)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_ShardDataset.pbtxt
 create mode 100644 tensorflow/core/kernels/data/shard_dataset_op.cc
 create mode 100644 tensorflow/python/data/experimental/kernel_tests/serialization/shard_dataset_serialization_test.py

diff --git a/tensorflow/core/api_def/base_api/api_def_ShardDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ShardDataset.pbtxt
new file mode 100644
index 0000000000..cd537e05d7
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_ShardDataset.pbtxt
@@ -0,0 +1,17 @@
+op {
+  graph_op_name: "ShardDataset"
+  visibility: HIDDEN
+  in_arg {
+    name: "num_shards"
+    description: <<END
+An integer representing the number of shards operating in parallel.
+END
+  }
+  in_arg {
+    name: "index"
+    description: <<END
+An integer representing the current worker index.
+END
+  }
+  summary: "Creates a `Dataset` that includes only 1/`num_shards` of this dataset."
+}
diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD
index cd6803fb3f..56c8399b34 100644
--- a/tensorflow/core/kernels/data/BUILD
+++ b/tensorflow/core/kernels/data/BUILD
@@ -143,6 +143,17 @@ tf_kernel_library(
     ],
 )
 
+tf_kernel_library(
+    name = "shard_dataset_op",
+    srcs = ["shard_dataset_op.cc"],
+    deps = [
+        "//tensorflow/core:dataset_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+    ],
+)
+
 tf_kernel_library(
     name = "window_dataset_op",
     srcs = ["window_dataset_op.cc"],
@@ -633,6 +644,7 @@ tf_kernel_library(
         ":range_dataset_op",
         ":reader_dataset_ops",
         ":repeat_dataset_op",
+        ":shard_dataset_op",
         ":shuffle_dataset_op",
         ":skip_dataset_op",
         ":sparse_tensor_slice_dataset_op",
diff --git a/tensorflow/core/kernels/data/shard_dataset_op.cc b/tensorflow/core/kernels/data/shard_dataset_op.cc
new file mode 100644
index 0000000000..9bb64911aa
--- /dev/null
+++ b/tensorflow/core/kernels/data/shard_dataset_op.cc
@@ -0,0 +1,195 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/framework/dataset.h"
+#include "tensorflow/core/framework/partial_tensor_shape.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/util/batch_util.h"
+
+namespace tensorflow {
+namespace data {
+namespace {
+
+// See documentation in ../../ops/dataset_ops.cc for a high-level
+// description of the following op.
+
+class ShardDatasetOp : public UnaryDatasetOpKernel {
+ public:
+  explicit ShardDatasetOp(OpKernelConstruction* ctx)
+      : UnaryDatasetOpKernel(ctx) {}
+
+  void MakeDataset(OpKernelContext* ctx, DatasetBase* input,
+                   DatasetBase** output) override {
+    int64 index = 0;
+    int64 num_shards = 0;
+
+    OP_REQUIRES_OK(ctx,
+                   ParseScalarArgument<int64>(ctx, "num_shards", &num_shards));
+    OP_REQUIRES(
+        ctx, num_shards > 0,
+        errors::InvalidArgument("Number of shards must be greater than zero "
+                                "(currently num_shards = ",
+                                num_shards, ")."));
+
+    OP_REQUIRES_OK(ctx, ParseScalarArgument<int64>(ctx, "index", &index));
+    OP_REQUIRES(
+        ctx, index >= 0 && index < num_shards,
+        errors::InvalidArgument("Index must be between 0 and ", num_shards - 1,
+                                " (currently index = ", index, ")."));
+
+    *output = new Dataset(ctx, num_shards, index, input);
+  }
+
+ private:
+  class Dataset : public DatasetBase {
+   public:
+    Dataset(OpKernelContext* ctx, int64 num_shards, int64 index,
+            const DatasetBase* input)
+        : DatasetBase(DatasetContext(ctx)),
+          num_shards_(num_shards),
+          index_(index),
+          input_(input) {
+      input_->Ref();
+    }
+
+    ~Dataset() override { input_->Unref(); }
+
+    std::unique_ptr<IteratorBase> MakeIteratorInternal(
+        const string& prefix) const override {
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::Shard")});
+    }
+
+    const DataTypeVector& output_dtypes() const override {
+      return input_->output_dtypes();
+    }
+
+    const std::vector<PartialTensorShape>& output_shapes() const override {
+      return input_->output_shapes();
+    }
+
+    string DebugString() const override {
+      return strings::StrCat("ShardDatasetOp(", num_shards_, ", ", index_,
+                             ")::Dataset");
+    }
+
+    int64 Cardinality() const override {
+      int64 n = input_->Cardinality();
+      if (n == kInfiniteCardinality || n == kUnknownCardinality) {
+        return n;
+      }
+      return n / num_shards_ + (index_ < n % num_shards_ ? 1 : 0);
+    }
+
+   protected:
+    Status AsGraphDefInternal(SerializationContext* ctx,
+                              DatasetGraphDefBuilder* b,
+                              Node** output) const override {
+      Node* input_graph_node = nullptr;
+      TF_RETURN_IF_ERROR(b->AddInputDataset(ctx, input_, &input_graph_node));
+      Node* num_shards = nullptr;
+      TF_RETURN_IF_ERROR(b->AddScalar(num_shards_, &num_shards));
+      Node* index = nullptr;
+      TF_RETURN_IF_ERROR(b->AddScalar(index_, &index));
+      TF_RETURN_IF_ERROR(
+          b->AddDataset(this, {input_graph_node, num_shards, index}, output));
+      return Status::OK();
+    }
+
+   private:
+    class Iterator : public DatasetIterator<Dataset> {
+     public:
+      explicit Iterator(const Params& params)
+          : DatasetIterator<Dataset>(params), next_index_(0) {}
+
+      Status Initialize(IteratorContext* ctx) override {
+        return dataset()->input_->MakeIterator(ctx, prefix(), &input_impl_);
+      }
+
+      Status GetNextInternal(IteratorContext* ctx,
+                             std::vector<Tensor>* out_tensors,
+                             bool* end_of_sequence) override {
+        mutex_lock l(mu_);
+
+        if (!input_impl_) {
+          *end_of_sequence = true;
+          return Status::OK();
+        }
+
+        std::vector<Tensor> result;
+        do {
+          result.clear();
+          TF_RETURN_IF_ERROR(
+              input_impl_->GetNext(ctx, &result, end_of_sequence));
+          if (*end_of_sequence) {
+            input_impl_.reset();
+            return Status::OK();
+          }
+        } while ((next_index_++ % dataset()->num_shards_) != dataset()->index_);
+
+        *out_tensors = std::move(result);
+        return Status::OK();
+      }
+
+     protected:
+      std::shared_ptr<model::Node> CreateNode(
+          IteratorContext* ctx, model::Node::Args args) const override {
+        return model::MakeKnownRatioNode(std::move(args),
+                                         dataset()->num_shards_);
+      }
+
+      Status SaveInternal(IteratorStateWriter* writer) override {
+        mutex_lock l(mu_);
+        if (!input_impl_) {
+          TF_RETURN_IF_ERROR(
+              writer->WriteScalar(full_name("input_impl_empty"), ""));
+        } else {
+          TF_RETURN_IF_ERROR(SaveInput(writer, input_impl_));
+          TF_RETURN_IF_ERROR(
+              writer->WriteScalar(full_name("next_index"), next_index_));
+        }
+        return Status::OK();
+      }
+
+      Status RestoreInternal(IteratorContext* ctx,
+                             IteratorStateReader* reader) override {
+        mutex_lock l(mu_);
+        if (!reader->Contains(full_name("input_impl_empty"))) {
+          TF_RETURN_IF_ERROR(RestoreInput(ctx, reader, input_impl_));
+          TF_RETURN_IF_ERROR(
+              reader->ReadScalar(full_name("next_index"), &next_index_));
+        } else {
+          input_impl_.reset();
+        }
+        return Status::OK();
+      }
+
+     private:
+      mutex mu_;
+      std::unique_ptr<IteratorBase> input_impl_ GUARDED_BY(mu_);
+      int64 next_index_ GUARDED_BY(mu_);
+    };
+
+    const int64 num_shards_;
+    const int64 index_;
+    const DatasetBase* const input_;
+  };
+};
+
+REGISTER_KERNEL_BUILDER(Name("ShardDataset").Device(DEVICE_CPU),
+                        ShardDatasetOp);
+
+}  // namespace
+}  // namespace data
+}  // namespace tensorflow
diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc
index 8494298105..872a6da915 100644
--- a/tensorflow/core/ops/dataset_ops.cc
+++ b/tensorflow/core/ops/dataset_ops.cc
@@ -275,6 +275,22 @@ REGISTER_OP("BatchDatasetV2")
       return shape_inference::ScalarShape(c);
     });
 
+REGISTER_OP("ShardDataset")
+    .Input("input_dataset: variant")
+    .Input("num_shards: int64")
+    .Input("index: int64")
+    .Output("handle: variant")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      shape_inference::ShapeHandle unused;
+      // num_shards should be a scalar.
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused));
+      // index should be a scalar.
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused));
+      return shape_inference::ScalarShape(c);
+    });
+
 // TODO(mrry): Validate that `padded_shapes` are all vectors, the lengths of
 // `output_types` and `output_shapes` are `N` the `output_shapes` are (as far as
 // possible to tell statically) compatible with `padded_shapes`, and that
diff --git a/tensorflow/python/data/experimental/kernel_tests/cardinality_test.py b/tensorflow/python/data/experimental/kernel_tests/cardinality_test.py
index 4a8296d084..993b511d5e 100644
--- a/tensorflow/python/data/experimental/kernel_tests/cardinality_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/cardinality_test.py
@@ -49,8 +49,7 @@ class NumElementsTest(test_base.DatasetTestBase, parameterized.TestCase):
        lambda: dataset_ops.Dataset.range(5).filter(lambda _: True).concatenate(
            dataset_ops.Dataset.range(5)), cardinality.UNKNOWN),
       ("Concatenate3", lambda: dataset_ops.Dataset.range(5).repeat().
-       concatenate(dataset_ops.Dataset.range(5)),
-       cardinality.INFINITE),
+       concatenate(dataset_ops.Dataset.range(5)), cardinality.INFINITE),
       ("Concatenate4", lambda: dataset_ops.Dataset.range(5).concatenate(
           dataset_ops.Dataset.range(5).filter(lambda _: True)),
        cardinality.UNKNOWN),
@@ -70,8 +69,7 @@ class NumElementsTest(test_base.DatasetTestBase, parameterized.TestCase):
        lambda: dataset_ops.Dataset.range(5).repeat().concatenate(
            dataset_ops.Dataset.range(5).repeat()), cardinality.INFINITE),
       ("FlatMap", lambda: dataset_ops.Dataset.range(5).flat_map(
-          lambda _: dataset_ops.Dataset.from_tensors(0)),
-       cardinality.UNKNOWN),
+          lambda _: dataset_ops.Dataset.from_tensors(0)), cardinality.UNKNOWN),
       ("Filter", lambda: dataset_ops.Dataset.range(5).filter(lambda _: True),
        cardinality.UNKNOWN),
       ("FromTensors1", lambda: dataset_ops.Dataset.from_tensors(0), 1),
@@ -117,6 +115,13 @@ class NumElementsTest(test_base.DatasetTestBase, parameterized.TestCase):
        cardinality.INFINITE),
       ("Shuffle", lambda: dataset_ops.Dataset.range(5).shuffle(buffer_size=1),
        5),
+      ("Shard1", lambda: dataset_ops.Dataset.range(5).shard(2, 0), 3),
+      ("Shard2", lambda: dataset_ops.Dataset.range(5).shard(8, 7), 0),
+      ("Shard3",
+       lambda: dataset_ops.Dataset.range(5).filter(lambda _: True).shard(2, 0),
+       cardinality.UNKNOWN),
+      ("Shard4", lambda: dataset_ops.Dataset.range(5).repeat().shard(2, 0),
+       cardinality.INFINITE),
       ("Skip1", lambda: dataset_ops.Dataset.range(5).skip(2), 3),
       ("Skip2", lambda: dataset_ops.Dataset.range(5).skip(8), 0),
       ("Skip3",
@@ -138,15 +143,13 @@ class NumElementsTest(test_base.DatasetTestBase, parameterized.TestCase):
        5),
       ("Zip2", lambda: dataset_ops.Dataset.zip(
           (dataset_ops.Dataset.range(5), dataset_ops.Dataset.range(3))), 3),
-      ("Zip3", lambda: dataset_ops.Dataset.zip(
-          (dataset_ops.Dataset.range(5),
-           dataset_ops.Dataset.range(3).repeat())), 5),
-      ("Zip4", lambda: dataset_ops.Dataset.zip(
-          (dataset_ops.Dataset.range(5).repeat(),
-           dataset_ops.Dataset.range(3).repeat())), cardinality.INFINITE),
-      ("Zip5", lambda: dataset_ops.Dataset.zip(
-          (dataset_ops.Dataset.range(5),
-           dataset_ops.Dataset.range(3).filter(lambda _: True))),
+      ("Zip3", lambda: dataset_ops.Dataset.zip((dataset_ops.Dataset.range(
+          5), dataset_ops.Dataset.range(3).repeat())), 5),
+      ("Zip4", lambda: dataset_ops.Dataset.zip((dataset_ops.Dataset.range(
+          5).repeat(), dataset_ops.Dataset.range(3).repeat())),
+       cardinality.INFINITE),
+      ("Zip5", lambda: dataset_ops.Dataset.zip((dataset_ops.Dataset.range(
+          5), dataset_ops.Dataset.range(3).filter(lambda _: True))),
        cardinality.UNKNOWN),
       # pylint: enable=g-long-lambda
   )
diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD b/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
index dc168d867a..caf571ef4e 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
@@ -623,6 +623,24 @@ py_test(
     ],
 )
 
+py_test(
+    name = "shard_dataset_serialization_test",
+    size = "medium",
+    srcs = ["shard_dataset_serialization_test.py"],
+    srcs_version = "PY2AND3",
+    tags = [
+        "no_oss",
+        "no_pip",
+        "no_windows",
+    ],
+    deps = [
+        ":dataset_serialization_test_base",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "@absl_py//absl/testing:parameterized",
+    ],
+)
+
 py_test(
     name = "shuffle_and_repeat_dataset_serialization_test",
     size = "medium",
diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/shard_dataset_serialization_test.py b/tensorflow/python/data/experimental/kernel_tests/serialization/shard_dataset_serialization_test.py
new file mode 100644
index 0000000000..99674b6910
--- /dev/null
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/shard_dataset_serialization_test.py
@@ -0,0 +1,42 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for the ShardDataset serialization."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+
+from tensorflow.python.data.experimental.kernel_tests.serialization import dataset_serialization_test_base
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.platform import test
+
+
+class ShardDatasetSerializationTest(
+    dataset_serialization_test_base.DatasetSerializationTestBase,
+    parameterized.TestCase):
+
+  def _build_dataset(self, num_elements, num_shards, index):
+    return dataset_ops.Dataset.range(num_elements).shard(num_shards, index)
+
+  @parameterized.parameters((10, 5, 2, 3), (10, 10, 0, 9), (100, 2, 0, 1))
+  def testCore(self, elems, num_shards, index1, index2):
+    self.run_core_tests(lambda: self._build_dataset(elems, num_shards, index1),
+                        lambda: self._build_dataset(elems, num_shards, index2),
+                        elems // num_shards)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/kernel_tests/shard_test.py b/tensorflow/python/data/kernel_tests/shard_test.py
index 928550676d..bdd2e899f5 100644
--- a/tensorflow/python/data/kernel_tests/shard_test.py
+++ b/tensorflow/python/data/kernel_tests/shard_test.py
@@ -19,6 +19,7 @@ from __future__ import print_function
 
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import errors
 from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
@@ -41,20 +42,24 @@ class ShardTest(test_base.DatasetTestBase):
     self.assertDatasetProduces(dataset, expected_output=[0, 5])
 
   def testOffsetGreaterNumShards(self):
-    with self.assertRaises(ValueError):
-      dataset_ops.Dataset.range(10).shard(5, 7)
+    with self.assertRaises(errors.InvalidArgumentError):
+      dataset = dataset_ops.Dataset.range(10).shard(5, 7)
+      self.evaluate(self.getNext(dataset)())
 
   def testNegativeOffset(self):
-    with self.assertRaises(ValueError):
-      dataset_ops.Dataset.range(10).shard(5, -3)
+    with self.assertRaises(errors.InvalidArgumentError):
+      dataset = dataset_ops.Dataset.range(10).shard(5, -3)
+      self.evaluate(self.getNext(dataset)())
 
   def testNegativeNumShards(self):
-    with self.assertRaises(ValueError):
-      dataset_ops.Dataset.range(10).shard(-3, 1)
+    with self.assertRaises(errors.InvalidArgumentError):
+      dataset = dataset_ops.Dataset.range(10).shard(-3, 1)
+      self.evaluate(self.getNext(dataset)())
 
   def testZeroNumShards(self):
-    with self.assertRaises(ValueError):
-      dataset_ops.Dataset.range(10).shard(0, 1)
+    with self.assertRaises(errors.InvalidArgumentError):
+      dataset = dataset_ops.Dataset.range(10).shard(0, 1)
+      self.evaluate(self.getNext(dataset)())
 
   def testIteratorEndsBeforeFirstElem(self):
     dataset = dataset_ops.Dataset.range(1).shard(5, 2)
@@ -72,5 +77,10 @@ class ShardTest(test_base.DatasetTestBase):
     dataset = dataset_ops.Dataset.range(10).shard(4, 3)
     self.assertDatasetProduces(dataset, expected_output=[3, 7])
 
+  def testNumShardsLargerThanDataset(self):
+    dataset = dataset_ops.Dataset.range(10).shard(20, 5)
+    self.assertDatasetProduces(dataset, expected_output=[5])
+
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index 766c6d5395..9694fc2574 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -28,7 +28,6 @@ from six.moves import queue as Queue  # pylint: disable=redefined-builtin
 
 
 from tensorflow.python.compat import compat
-from tensorflow.python.data.experimental.ops import filter_for_shard_ops
 from tensorflow.python.data.experimental.ops import optimization_options
 from tensorflow.python.data.experimental.ops import stats_options
 from tensorflow.python.data.experimental.ops import threading_options
@@ -805,6 +804,59 @@ class DatasetV2(object):
     """
     return SkipDataset(self, count)
 
+  def shard(self, num_shards, index):
+    """Creates a `Dataset` that includes only 1/`num_shards` of this dataset.
+
+    This dataset operator is very useful when running distributed training, as
+    it allows each worker to read a unique subset.
+
+    When reading a single input file, you can skip elements as follows:
+
+    ```python
+    d = tf.data.TFRecordDataset(input_file)
+    d = d.shard(num_workers, worker_index)
+    d = d.repeat(num_epochs)
+    d = d.shuffle(shuffle_buffer_size)
+    d = d.map(parser_fn, num_parallel_calls=num_map_threads)
+    ```
+
+    Important caveats:
+
+    - Be sure to shard before you use any randomizing operator (such as
+      shuffle).
+    - Generally it is best if the shard operator is used early in the dataset
+      pipeline. For example, when reading from a set of TFRecord files, shard
+      before converting the dataset to input samples. This avoids reading every
+      file on every worker. The following is an example of an efficient
+      sharding strategy within a complete pipeline:
+
+    ```python
+    d = Dataset.list_files(pattern)
+    d = d.shard(num_workers, worker_index)
+    d = d.repeat(num_epochs)
+    d = d.shuffle(shuffle_buffer_size)
+    d = d.interleave(tf.data.TFRecordDataset,
+                     cycle_length=num_readers, block_length=1)
+    d = d.map(parser_fn, num_parallel_calls=num_map_threads)
+    ```
+
+    Args:
+      num_shards: A `tf.int64` scalar `tf.Tensor`, representing the number of
+        shards operating in parallel.
+      index: A `tf.int64` scalar `tf.Tensor`, representing the worker index.
+
+    Returns:
+      Dataset: A `Dataset`.
+
+    Raises:
+      InvalidArgumentError: if `num_shards` or `index` are illegal values.
+        Note: error checking is done on a best-effort basis, and errors aren't
+        guaranteed to be caught upon dataset creation. (e.g. providing in a
+        placeholder tensor bypasses the early checking, and will instead result
+        in an error during a session.run call.)
+    """
+    return ShardDataset(self, num_shards, index)
+
   def batch(self, batch_size, drop_remainder=False):
     """Combines consecutive elements of this dataset into batches.
 
@@ -1550,60 +1602,9 @@ class DatasetV1(DatasetV2):
   def skip(self, count):
     return DatasetV1Adapter(super(DatasetV1, self).skip(count))
 
-  @deprecation.deprecated(
-      None, "Use `dataset.apply(tf.data.experimental.filter_for_shard(...))`.")
+  @functools.wraps(DatasetV2.shard)
   def shard(self, num_shards, index):
-    """Creates a `Dataset` that includes only 1/`num_shards` of this dataset.
-
-    This dataset operator is very useful when running distributed training, as
-    it allows each worker to read a unique subset.
-
-    When reading a single input file, you can skip elements as follows:
-
-    ```python
-    d = tf.data.TFRecordDataset(FLAGS.input_file)
-    d = d.shard(FLAGS.num_workers, FLAGS.worker_index)
-    d = d.repeat(FLAGS.num_epochs)
-    d = d.shuffle(FLAGS.shuffle_buffer_size)
-    d = d.map(parser_fn, num_parallel_calls=FLAGS.num_map_threads)
-    ```
-
-    Important caveats:
-
-    - Be sure to shard before you use any randomizing operator (such as
-      shuffle).
-    - Generally it is best if the shard operator is used early in the dataset
-      pipeline. For example, when reading from a set of TFRecord files, shard
-      before converting the dataset to input samples. This avoids reading every
-      file on every worker. The following is an example of an efficient
-      sharding strategy within a complete pipeline:
-
-    ```python
-    d = Dataset.list_files(FLAGS.pattern)
-    d = d.shard(FLAGS.num_workers, FLAGS.worker_index)
-    d = d.repeat(FLAGS.num_epochs)
-    d = d.shuffle(FLAGS.shuffle_buffer_size)
-    d = d.interleave(tf.data.TFRecordDataset,
-                     cycle_length=FLAGS.num_readers, block_length=1)
-    d = d.map(parser_fn, num_parallel_calls=FLAGS.num_map_threads)
-    ```
-
-    Args:
-      num_shards: A `tf.int64` scalar `tf.Tensor`, representing the number of
-        shards operating in parallel.
-      index: A `tf.int64` scalar `tf.Tensor`, representing the worker index.
-
-    Returns:
-      Dataset: A `Dataset`.
-
-    Raises:
-      ValueError: if `num_shards` or `index` are illegal values. Note: error
-        checking is done on a best-effort basis, and errors aren't guaranteed
-        to be caught upon dataset creation. (e.g. providing in a placeholder
-        tensor bypasses the early checking, and will instead result in an error
-        during a session.run call.)
-    """
-    return self.apply(filter_for_shard_ops.filter_for_shard(num_shards, index))
+    return DatasetV1Adapter(super(DatasetV1, self).shard(num_shards, index))
 
   @functools.wraps(DatasetV2.batch)
   def batch(self, batch_size, drop_remainder=False):
@@ -2504,6 +2505,23 @@ class SkipDataset(UnaryUnchangedStructureDataset):
     super(SkipDataset, self).__init__(input_dataset, variant_tensor)
 
 
+class ShardDataset(UnaryUnchangedStructureDataset):
+  """A `Dataset` for sharding its input."""
+
+  def __init__(self, input_dataset, num_shards, index):
+    """See `Dataset.shard()` for details."""
+    self._input_dataset = input_dataset
+    self._num_shards = ops.convert_to_tensor(
+        num_shards, dtype=dtypes.int64, name="num_shards")
+    self._index = ops.convert_to_tensor(index, dtype=dtypes.int64, name="index")
+    variant_tensor = gen_dataset_ops.shard_dataset(
+        input_dataset._variant_tensor,  # pylint: disable=protected-access
+        num_shards=self._num_shards,
+        index=self._index,
+        **flat_structure(self))
+    super(ShardDataset, self).__init__(input_dataset, variant_tensor)
+
+
 class BatchDataset(UnaryDataset):
   """A `Dataset` that batches contiguous elements from its input."""
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt
index 0b2a15f521..28f26fe2de 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt
@@ -3056,6 +3056,10 @@ tf_module {
     name: "ShapeN"
     argspec: "args=[\'input\', \'out_type\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "ShardDataset"
+    argspec: "args=[\'input_dataset\', \'num_shards\', \'index\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "ShardedFilename"
     argspec: "args=[\'basename\', \'shard\', \'num_shards\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-dataset.pbtxt
index 951b2df05a..195c10445d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.-dataset.pbtxt
@@ -90,6 +90,10 @@ tf_class {
     name: "repeat"
     argspec: "args=[\'self\', \'count\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "shard"
+    argspec: "args=[\'self\', \'num_shards\', \'index\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "shuffle"
     argspec: "args=[\'self\', \'buffer_size\', \'seed\', \'reshuffle_each_iteration\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-fixed-length-record-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-fixed-length-record-dataset.pbtxt
index f157351243..043584c233 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.-fixed-length-record-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.-fixed-length-record-dataset.pbtxt
@@ -92,6 +92,10 @@ tf_class {
     name: "repeat"
     argspec: "args=[\'self\', \'count\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "shard"
+    argspec: "args=[\'self\', \'num_shards\', \'index\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "shuffle"
     argspec: "args=[\'self\', \'buffer_size\', \'seed\', \'reshuffle_each_iteration\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-t-f-record-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-t-f-record-dataset.pbtxt
index 690da98b1a..76d15f4634 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.-t-f-record-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.-t-f-record-dataset.pbtxt
@@ -91,6 +91,10 @@ tf_class {
     name: "repeat"
     argspec: "args=[\'self\', \'count\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "shard"
+    argspec: "args=[\'self\', \'num_shards\', \'index\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "shuffle"
     argspec: "args=[\'self\', \'buffer_size\', \'seed\', \'reshuffle_each_iteration\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-text-line-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-text-line-dataset.pbtxt
index fe0bc1a4db..a6c7a2d7c5 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.-text-line-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.-text-line-dataset.pbtxt
@@ -92,6 +92,10 @@ tf_class {
     name: "repeat"
     argspec: "args=[\'self\', \'count\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "shard"
+    argspec: "args=[\'self\', \'num_shards\', \'index\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "shuffle"
     argspec: "args=[\'self\', \'buffer_size\', \'seed\', \'reshuffle_each_iteration\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-csv-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-csv-dataset.pbtxt
index 261129b132..ae177ebc6b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-csv-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-csv-dataset.pbtxt
@@ -92,6 +92,10 @@ tf_class {
     name: "repeat"
     argspec: "args=[\'self\', \'count\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "shard"
+    argspec: "args=[\'self\', \'num_shards\', \'index\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "shuffle"
     argspec: "args=[\'self\', \'buffer_size\', \'seed\', \'reshuffle_each_iteration\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-random-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-random-dataset.pbtxt
index 0b34bbc942..c15c73f3f7 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-random-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-random-dataset.pbtxt
@@ -92,6 +92,10 @@ tf_class {
     name: "repeat"
     argspec: "args=[\'self\', \'count\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "shard"
+    argspec: "args=[\'self\', \'num_shards\', \'index\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "shuffle"
     argspec: "args=[\'self\', \'buffer_size\', \'seed\', \'reshuffle_each_iteration\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-sql-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-sql-dataset.pbtxt
index 0e61890eee..567f48d1aa 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-sql-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-sql-dataset.pbtxt
@@ -92,6 +92,10 @@ tf_class {
     name: "repeat"
     argspec: "args=[\'self\', \'count\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "shard"
+    argspec: "args=[\'self\', \'num_shards\', \'index\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "shuffle"
     argspec: "args=[\'self\', \'buffer_size\', \'seed\', \'reshuffle_each_iteration\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt
index 0b2a15f521..28f26fe2de 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt
@@ -3056,6 +3056,10 @@ tf_module {
     name: "ShapeN"
     argspec: "args=[\'input\', \'out_type\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "ShardDataset"
+    argspec: "args=[\'input_dataset\', \'num_shards\', \'index\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "ShardedFilename"
     argspec: "args=[\'basename\', \'shard\', \'num_shards\'], varargs=None, keywords=None, defaults=None"
-- 
GitLab


From 038a9fd67b7f7e0284bb24269d81ac076409e077 Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Mon, 4 Feb 2019 16:48:27 -0800
Subject: [PATCH 1886/2345] Add minimal console logging functionality for
 TensorFlow lite

By default this logs to stderr with various severities. On Android
it also routes to Android's logcat.

PiperOrigin-RevId: 232391898
---
 tensorflow/lite/BUILD                      | 46 ++++++++++++++---
 tensorflow/lite/minimal_logging.cc         | 44 ++++++++++++++++
 tensorflow/lite/minimal_logging.h          | 56 ++++++++++++++++++++
 tensorflow/lite/minimal_logging_android.cc | 55 ++++++++++++++++++++
 tensorflow/lite/minimal_logging_default.cc | 31 +++++++++++
 tensorflow/lite/minimal_logging_ios.cc     | 47 +++++++++++++++++
 tensorflow/lite/minimal_logging_test.cc    | 60 ++++++++++++++++++++++
 tensorflow/lite/stderr_reporter.cc         | 20 ++------
 8 files changed, 334 insertions(+), 25 deletions(-)
 create mode 100644 tensorflow/lite/minimal_logging.cc
 create mode 100644 tensorflow/lite/minimal_logging.h
 create mode 100644 tensorflow/lite/minimal_logging_android.cc
 create mode 100644 tensorflow/lite/minimal_logging_default.cc
 create mode 100644 tensorflow/lite/minimal_logging_ios.cc
 create mode 100644 tensorflow/lite/minimal_logging_test.cc

diff --git a/tensorflow/lite/BUILD b/tensorflow/lite/BUILD
index 8cc91a1a24..cadc11e227 100644
--- a/tensorflow/lite/BUILD
+++ b/tensorflow/lite/BUILD
@@ -173,18 +173,11 @@ cc_library(
         "stderr_reporter.h",
     ],
     copts = tflite_copts() + TFLITE_DEFAULT_COPTS,
-    linkopts = [
-    ] + select({
-        "//tensorflow:android": [
-            "-llog",
-        ],
-        "//conditions:default": [
-        ],
-    }),
     deps = [
         ":arena_planner",
         ":graph_info",
         ":memory_planner",
+        ":minimal_logging",
         ":schema_fbs_version",
         ":simple_memory_arena",
         ":string",
@@ -368,4 +361,41 @@ cc_test(
     ],
 )
 
+cc_library(
+    name = "minimal_logging",
+    srcs = [
+        "minimal_logging.cc",
+    ] + select({
+        "//tensorflow:android": [
+            "minimal_logging_android.cc",
+        ],
+        "//tensorflow:ios": [
+            "minimal_logging_ios.cc",
+        ],
+        "//conditions:default": [
+            "minimal_logging_default.cc",
+        ],
+    }),
+    hdrs = ["minimal_logging.h"],
+    copts = TFLITE_DEFAULT_COPTS + tflite_copts(),
+    linkopts = select({
+        "//tensorflow:android": ["-llog"],
+        "//conditions:default": [],
+    }),
+    visibility = ["//visibility:private"],
+)
+
+cc_test(
+    name = "minimal_logging_test",
+    size = "small",
+    srcs = ["minimal_logging_test.cc"],
+    tags = [
+        "tflite_not_portable_ios",  # TODO(b/117786830)
+    ],
+    deps = [
+        ":minimal_logging",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
 tflite_portable_test_suite()
diff --git a/tensorflow/lite/minimal_logging.cc b/tensorflow/lite/minimal_logging.cc
new file mode 100644
index 0000000000..8768ef6e31
--- /dev/null
+++ b/tensorflow/lite/minimal_logging.cc
@@ -0,0 +1,44 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/minimal_logging.h"
+
+#include <cstdarg>
+
+namespace tflite {
+namespace logging_internal {
+
+void MinimalLogger::Log(LogSeverity severity, const char* format, ...) {
+  va_list args;
+  va_start(args, format);
+  VLog(severity, format, args);
+  va_end(args);
+}
+
+const char* MinimalLogger::GetSeverityName(LogSeverity severity) {
+  switch (severity) {
+    case TFLITE_LOG_INFO:
+      return "INFO";
+    case TFLITE_LOG_WARNING:
+      return "WARNING";
+    case TFLITE_LOG_ERROR:
+      return "ERROR";
+    default:
+      return "<Unknown severity>";
+  }
+}
+
+}  // namespace logging_internal
+}  // namespace tflite
diff --git a/tensorflow/lite/minimal_logging.h b/tensorflow/lite/minimal_logging.h
new file mode 100644
index 0000000000..7682ed8edc
--- /dev/null
+++ b/tensorflow/lite/minimal_logging.h
@@ -0,0 +1,56 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_MINIMAL_LOGGING_H_
+#define TENSORFLOW_LITE_MINIMAL_LOGGING_H_
+
+#include <cstdarg>
+
+namespace tflite {
+
+enum LogSeverity {
+  TFLITE_LOG_INFO = 0,
+  TFLITE_LOG_WARNING = 1,
+  TFLITE_LOG_ERROR = 2,
+};
+
+namespace logging_internal {
+
+// Helper class for simple platform-specific console logging. Note that we
+// explicitly avoid the convenience of ostream-style logging to minimize binary
+// size impact.
+class MinimalLogger {
+ public:
+  // Logging hook that takes variadic args.
+  static void Log(LogSeverity severity, const char* format, ...);
+
+  // Logging hook that takes a formatted va_list.
+  static void VLog(LogSeverity severity, const char* format, va_list args);
+
+ private:
+  static const char* GetSeverityName(LogSeverity severity);
+};
+
+}  // namespace logging_internal
+}  // namespace tflite
+
+// Convenience macro for basic internal logging in production builds.
+// Note: This should never be used for debug-type logs, as it will *not* be
+// stripped in release optimized builds. In general, prefer the error reporting
+// APIs for developer-facing errors, and only use this for diagnostic output
+// that should always be logged in user builds.
+#define TFLITE_LOG_PROD(severity, format, ...) \
+  tflite::logging_internal::MinimalLogger::Log(severity, format, ##__VA_ARGS__);
+
+#endif  // TENSORFLOW_LITE_MINIMAL_LOGGING_H_
diff --git a/tensorflow/lite/minimal_logging_android.cc b/tensorflow/lite/minimal_logging_android.cc
new file mode 100644
index 0000000000..f87e6fa18e
--- /dev/null
+++ b/tensorflow/lite/minimal_logging_android.cc
@@ -0,0 +1,55 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/minimal_logging.h"
+
+#include <android/log.h>
+#include <cstdio>
+
+namespace tflite {
+namespace logging_internal {
+namespace {
+
+int GetPlatformSeverity(LogSeverity severity) {
+  switch (severity) {
+    case TFLITE_LOG_INFO:
+      return ANDROID_LOG_INFO;
+    case TFLITE_LOG_WARNING:
+      return ANDROID_LOG_WARN;
+    case TFLITE_LOG_ERROR:
+      return ANDROID_LOG_ERROR;
+    default:
+      return ANDROID_LOG_DEBUG;
+  }
+}
+
+}  // namespace
+
+void MinimalLogger::VLog(LogSeverity severity, const char* format,
+                         va_list args) {
+  // First log to Android's explicit log(cat) API.
+  va_list args_for_android_log;
+  va_copy(args_for_android_log, args);
+  __android_log_vprint(GetPlatformSeverity(severity), "tflite", format, args);
+  va_end(args_for_android_log);
+
+  // Also print to stderr for standard console applications.
+  fprintf(stderr, "%s: ", GetSeverityName(severity));
+  vfprintf(stderr, format, args);
+  fputc('\n', stderr);
+}
+
+}  // namespace logging_internal
+}  // namespace tflite
diff --git a/tensorflow/lite/minimal_logging_default.cc b/tensorflow/lite/minimal_logging_default.cc
new file mode 100644
index 0000000000..9fa13e47e6
--- /dev/null
+++ b/tensorflow/lite/minimal_logging_default.cc
@@ -0,0 +1,31 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/minimal_logging.h"
+
+#include <cstdio>
+
+namespace tflite {
+namespace logging_internal {
+
+void MinimalLogger::VLog(LogSeverity severity, const char* format,
+                         va_list args) {
+  fprintf(stderr, "%s: ", GetSeverityName(severity));
+  vfprintf(stderr, format, args);
+  fputc('\n', stderr);
+}
+
+}  // namespace logging_internal
+}  // namespace tflite
diff --git a/tensorflow/lite/minimal_logging_ios.cc b/tensorflow/lite/minimal_logging_ios.cc
new file mode 100644
index 0000000000..a774682a5b
--- /dev/null
+++ b/tensorflow/lite/minimal_logging_ios.cc
@@ -0,0 +1,47 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/minimal_logging.h"
+
+#include <syslog.h>
+#include <cstdarg>
+
+namespace tflite {
+namespace logging_internal {
+namespace {
+
+int GetPlatformSeverity(LogSeverity severity) {
+  switch (severity) {
+    case TFLITE_LOG_INFO:
+      return LOG_INFO;
+    case TFLITE_LOG_WARNING:
+      return LOG_WARNING;
+    case TFLITE_LOG_ERROR:
+      return LOG_ERR;
+    default:
+      return LOG_DEBUG;
+  }
+}
+
+}  // namespace
+
+void MinimalLogger::VLog(LogSeverity severity, const char* format,
+                         va_list args) {
+  // TODO(b/123704468): Use os_log when available.
+  vsyslog(GetPlatformSeverity(severity), format, args);
+}
+
+}  // namespace logging_internal
+}  // namespace tflite
diff --git a/tensorflow/lite/minimal_logging_test.cc b/tensorflow/lite/minimal_logging_test.cc
new file mode 100644
index 0000000000..e59425a2b2
--- /dev/null
+++ b/tensorflow/lite/minimal_logging_test.cc
@@ -0,0 +1,60 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/minimal_logging.h"
+
+#include <string>
+
+#include <gtest/gtest.h>
+
+namespace tflite {
+
+TEST(MinimalLogging, Basic) {
+  testing::internal::CaptureStderr();
+  TFLITE_LOG_PROD(TFLITE_LOG_INFO, "Foo");
+  EXPECT_EQ("INFO: Foo\n", testing::internal::GetCapturedStderr());
+}
+
+TEST(MinimalLogging, BasicFormatted) {
+  testing::internal::CaptureStderr();
+  TFLITE_LOG_PROD(TFLITE_LOG_INFO, "Foo %s %s", "Bar", "Baz");
+  EXPECT_EQ("INFO: Foo Bar Baz\n", testing::internal::GetCapturedStderr());
+}
+
+TEST(MinimalLogging, Warn) {
+  testing::internal::CaptureStderr();
+  TFLITE_LOG_PROD(TFLITE_LOG_WARNING, "One", "");
+  EXPECT_EQ("WARNING: One\n", testing::internal::GetCapturedStderr());
+}
+
+TEST(MinimalLogging, Error) {
+  testing::internal::CaptureStderr();
+  TFLITE_LOG_PROD(TFLITE_LOG_ERROR, "Two");
+  EXPECT_EQ("ERROR: Two\n", testing::internal::GetCapturedStderr());
+}
+
+TEST(MinimalLogging, UnknownSeverity) {
+  testing::internal::CaptureStderr();
+  TFLITE_LOG_PROD(static_cast<LogSeverity>(-1), "Three");
+  EXPECT_EQ("<Unknown severity>: Three\n",
+            testing::internal::GetCapturedStderr());
+}
+
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/lite/stderr_reporter.cc b/tensorflow/lite/stderr_reporter.cc
index 09eb1d254a..366a1816ef 100644
--- a/tensorflow/lite/stderr_reporter.cc
+++ b/tensorflow/lite/stderr_reporter.cc
@@ -13,28 +13,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 #include "tensorflow/lite/stderr_reporter.h"
-#include <cstdarg>
-#include <cstdio>
 
-#ifdef __ANDROID__
-#include <android/log.h>
-#endif
+#include "tensorflow/lite/minimal_logging.h"
 
 namespace tflite {
 
 int StderrReporter::Report(const char* format, va_list args) {
-#ifdef __ANDROID__
-  // On Android stderr is not captured for applications, only for code run from
-  // the shell. Rather than assume all users will set up a custom error
-  // reporter, let's output to logcat here
-  va_list args_for_log;
-  va_copy(args_for_log, args);
-  __android_log_vprint(ANDROID_LOG_ERROR, "tflite", format, args_for_log);
-  va_end(args_for_log);
-#endif
-  const int result = vfprintf(stderr, format, args);
-  fputc('\n', stderr);
-  return result;
+  logging_internal::MinimalLogger::VLog(TFLITE_LOG_ERROR, format, args);
+  return 0;
 }
 
 ErrorReporter* DefaultErrorReporter() {
-- 
GitLab


From 6f848e3e8b7af75eb9d6dcd249c6cab734e41919 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Mon, 4 Feb 2019 17:12:27 -0800
Subject: [PATCH 1887/2345] Minor logging cleanup, add a few extra tests.

PiperOrigin-RevId: 232395958
---
 tensorflow/python/autograph/impl/api.py       |  3 +-
 tensorflow/python/autograph/impl/api_test.py  | 82 +++++++++++++++----
 .../python/autograph/impl/conversion.py       | 18 ++--
 3 files changed, 79 insertions(+), 24 deletions(-)

diff --git a/tensorflow/python/autograph/impl/api.py b/tensorflow/python/autograph/impl/api.py
index 5b6d15f301..abe0f25821 100644
--- a/tensorflow/python/autograph/impl/api.py
+++ b/tensorflow/python/autograph/impl/api.py
@@ -193,16 +193,17 @@ def converted_call(f, owner, options, *args, **kwargs):
         'Entity {} appears to be decorated by wrapt, which is not yet supported'
         ' by AutoGraph. The function will be called without transformation.'
         ' You may however apply AutoGraph before the decorator.'.format(f), 1)
+    logging.log(2, 'Permanently whitelisted: %s: wrapt decorated', f)
     return f(*args, **kwargs)
 
   # Other built-in modules are permanently whitelisted.
   # TODO(mdan): Figure out how to do this consistently for all stdlib modules.
   if (f in collections.__dict__.values() or f in pdb.__dict__.values() or
       f in copy.__dict__.values()):
+    logging.log(2, 'Permanently whitelisted: %s: part of builtin module', f)
     return f(*args, **kwargs)
 
   # TODO(mdan): This needs cleanup.
-  # In particular, we may want to avoid renaming functions altogether.
   if not options.force_conversion and conversion.is_whitelisted_for_graph(f):
 
     # TODO(mdan): This may be inconsistent in certain situations.
diff --git a/tensorflow/python/autograph/impl/api_test.py b/tensorflow/python/autograph/impl/api_test.py
index d5561ba824..51928099e8 100644
--- a/tensorflow/python/autograph/impl/api_test.py
+++ b/tensorflow/python/autograph/impl/api_test.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import collections
 import functools
 import gc
 
@@ -26,6 +27,7 @@ import numpy as np
 from tensorflow.python.autograph import utils
 from tensorflow.python.autograph.core import converter
 from tensorflow.python.autograph.impl import api
+from tensorflow.python.autograph.pyct import inspect_utils
 from tensorflow.python.autograph.pyct import parser
 from tensorflow.python.autograph.utils import py_func
 from tensorflow.python.framework import constant_op
@@ -46,7 +48,7 @@ class TestResource(str):
 class ApiTest(test.TestCase):
 
   @test_util.run_deprecated_v1
-  def test_decorator_recurses(self):
+  def test_decorator_recursive(self):
 
     class TestClass(object):
 
@@ -69,7 +71,7 @@ class ApiTest(test.TestCase):
       self.assertListEqual([0, 1], self.evaluate(x).tolist())
 
   @test_util.run_deprecated_v1
-  def test_decorator_does_not_recurse(self):
+  def test_decorator_not_recursive(self):
 
     class TestClass(object):
 
@@ -90,7 +92,7 @@ class ApiTest(test.TestCase):
       self.assertListEqual([0, 1], self.evaluate(x).tolist())
 
   @test_util.run_deprecated_v1
-  def test_decorator_calls_unconverted_graph(self):
+  def test_convert_then_do_not_convert_graph(self):
 
     class TestClass(object):
 
@@ -105,14 +107,13 @@ class ApiTest(test.TestCase):
         return x
 
     tc = TestClass()
-    with self.cached_session() as sess:
-      x = tc.test_method(
-          constant_op.constant([2, 4]), constant_op.constant(1),
-          constant_op.constant(-2))
-      self.assertListEqual([0, 1], self.evaluate(x).tolist())
+    x = tc.test_method(
+        constant_op.constant((2, 4)), constant_op.constant(1),
+        constant_op.constant(-2))
+    self.assertAllEqual((0, 1), self.evaluate(x))
 
   @test_util.run_deprecated_v1
-  def test_decorator_calls_unconverted_py_func(self):
+  def test_convert_then_do_not_convert_py_func(self):
 
     class TestClass(object):
 
@@ -132,11 +133,10 @@ class ApiTest(test.TestCase):
         return x
 
     tc = TestClass()
-    with self.cached_session() as sess:
-      x = tc.test_method(
-          constant_op.constant([2, 4]), constant_op.constant(1),
-          constant_op.constant(-2))
-      self.assertListEqual([0, 1], self.evaluate(x).tolist())
+    x = tc.test_method(
+        constant_op.constant((2, 4)), constant_op.constant(1),
+        constant_op.constant(-2))
+    self.assertAllEqual((0, 1), self.evaluate(x))
 
   @test_util.run_deprecated_v1
   def test_decorator_calls_decorated(self):
@@ -265,6 +265,26 @@ class ApiTest(test.TestCase):
                              converter.ConversionOptions(), tc)
       self.assertEqual(1, self.evaluate(x))
 
+  def test_converted_call_method_converts_recursively(self):
+
+    class TestClass(object):
+
+      def __init__(self, x):
+        self.x = x
+
+      def other_method(self):
+        if self.x < 0:
+          return -self.x
+        return self.x
+
+      def test_method(self):
+        return self.other_method()
+
+    tc = TestClass(constant_op.constant(-1))
+    x = api.converted_call(tc.test_method, None,
+                           converter.ConversionOptions(recursive=True), tc)
+    self.assertEqual(1, self.evaluate(x))
+
   def test_converted_call_method_by_class(self):
 
     class TestClass(object):
@@ -334,6 +354,22 @@ class ApiTest(test.TestCase):
                              constant_op.constant(0))
       self.assertTrue(self.evaluate(x))
 
+  def test_converted_call_then_already_converted_dynamic(self):
+
+    @api.convert()
+    def g(x):
+      if x > 0:
+        return x
+      else:
+        return -x
+
+    def f(g, x):
+      return g(x)
+
+    x = api.converted_call(f, None, converter.ConversionOptions(),
+                           g, constant_op.constant(1))
+    self.assertEqual(self.evaluate(x), 1)
+
   @test_util.run_deprecated_v1
   def test_converted_call_no_user_code(self):
 
@@ -397,6 +433,24 @@ class ApiTest(test.TestCase):
       self.evaluate(variables.global_variables_initializer())
       self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
 
+  def test_converted_call_namedtuple(self):
+
+    opts = converter.ConversionOptions()
+
+    x = api.converted_call(collections.namedtuple, None, opts,
+                           'TestNamedtuple', ('a', 'b'))
+
+    self.assertTrue(inspect_utils.isnamedtuple(x))
+
+  def test_converted_call_namedtuple_via_collections(self):
+
+    opts = converter.ConversionOptions()
+
+    x = api.converted_call('namedtuple', collections, opts,
+                           'TestNamedtuple', ('a', 'b'))
+
+    self.assertTrue(inspect_utils.isnamedtuple(x))
+
   def test_converted_call_lambda(self):
 
     opts = converter.ConversionOptions()
diff --git a/tensorflow/python/autograph/impl/conversion.py b/tensorflow/python/autograph/impl/conversion.py
index 32a7d89621..1ac2e33a8a 100644
--- a/tensorflow/python/autograph/impl/conversion.py
+++ b/tensorflow/python/autograph/impl/conversion.py
@@ -20,7 +20,6 @@ from __future__ import print_function
 
 import functools
 import imp
-# import types
 import unittest
 
 import gast
@@ -87,17 +86,17 @@ def is_whitelisted_for_graph(o):
     # Builtins typically have unnamed modules.
     for prefix, in config.DEFAULT_UNCOMPILED_MODULES:
       if m.__name__.startswith(prefix):
-        logging.log(2, '%s is whitelisted: name starts with "%s"', o, prefix)
+        logging.log(2, 'Whitelisted: %s: name starts with "%s"', o, prefix)
         return True
 
     # Temporary -- whitelist tensorboard modules.
     # TODO(b/122731813): Remove.
     if m.__name__ == 'tensorboard' or '.tensorboard' in m.__name__:
-      logging.log(2, '%s is whitelisted: name contains "tensorboard"', o)
+      logging.log(2, 'Whitelisted: %s: name contains "tensorboard"', o)
       return True
 
   if hasattr(o, 'autograph_info__') or hasattr(o, '__ag_compiled'):
-    logging.log(2, '%s is whitelisted: already converted', o)
+    logging.log(2, 'Whitelisted: %s: already converted', o)
     return True
 
   if hasattr(o, '__call__'):
@@ -105,9 +104,10 @@ def is_whitelisted_for_graph(o):
     # The type check avoids infinite recursion around the __call__ method
     # of function objects.
     if (type(o) != type(o.__call__)) and is_whitelisted_for_graph(o.__call__):  # pylint: disable=unidiomatic-typecheck
-      logging.log(2, '%s is whitelisted: object __call__ whitelisted', o)
+      logging.log(2, 'Whitelisted: %s: object __call__ whitelisted', o)
       return True
 
+  owner_class = None
   if tf_inspect.ismethod(o):
     # Methods of whitelisted classes are also whitelisted, even if they are
     # bound via user subclasses.
@@ -127,12 +127,12 @@ def is_whitelisted_for_graph(o):
     owner_class = inspect_utils.getmethodclass(o)
     if owner_class is not None:
       if issubclass(owner_class, unittest.TestCase):
-        logging.log(2, '%s is whitelisted: method of TestCase subclass', o)
+        logging.log(2, 'Whitelisted: %s: method of TestCase subclass', o)
         return True
 
       owner_class = inspect_utils.getdefiningclass(o, owner_class)
       if is_whitelisted_for_graph(owner_class):
-        logging.log(2, '%s is whitelisted: owner is whitelisted %s', o,
+        logging.log(2, 'Whitelisted: %s: owner is whitelisted %s', o,
                     owner_class)
         return True
 
@@ -145,10 +145,10 @@ def is_whitelisted_for_graph(o):
           'Entity {} looks like a namedtuple subclass. Its constructor will'
           ' not be converted by AutoGraph, but if it has any custom methods,'
           ' those will be.'.format(o), 1)
-    logging.log(2, '%s is whitelisted: named tuple', o)
+    logging.log(2, 'Whitelisted: %s: named tuple', o)
     return True
 
-  logging.log(2, '%s is NOT whitelisted', o)
+  logging.log(2, 'Not whitelisted: %s: default rule', o)
   return False
 
 
-- 
GitLab


From 40d91d49fc5c38e55d7a333a71e06dc6e283d5bf Mon Sep 17 00:00:00 2001
From: Mahmoud Abuzaina <mahmoud.abuzaina@intel.com>
Date: Mon, 4 Feb 2019 17:24:03 -0800
Subject: [PATCH 1888/2345] Added missing data type for quantized ops

---
 tensorflow/core/graph/mkl_graph_util.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/graph/mkl_graph_util.h b/tensorflow/core/graph/mkl_graph_util.h
index 990b2fe9b0..f36ca8c5a8 100644
--- a/tensorflow/core/graph/mkl_graph_util.h
+++ b/tensorflow/core/graph/mkl_graph_util.h
@@ -96,7 +96,7 @@ static inline bool IsMklOp(const string& op_name, DataType T) {
 
   // Restrict quantized ops to QUINT8 and QINT8 for now
   if (kernel.find(kMklQuantizedOpLabelPattern) != string::npos) {
-    return (T == DT_QUINT8 || T == DT_QINT8);
+    return (T == DT_QUINT8 || T == DT_QINT8 || T == DT_QINT32);
   }
   // Restrict regular ops to FLOAT
   if (kernel.find(kMklOpLabelPattern) != string::npos) {
-- 
GitLab


From 030088358322c883086817a732f1cbe9dd84bb49 Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Mon, 4 Feb 2019 17:19:22 -0800
Subject: [PATCH 1889/2345] Add libtensorflowlite.so target

This target exports the interpreter C++ API and the kernel C API, and
includes the builtin set of ops.

Fixes #25278 and fixes #19642.

PiperOrigin-RevId: 232396927
---
 tensorflow/lite/BUILD                       | 28 ++++++++++++++++++++-
 tensorflow/lite/tflite_exported_symbols.lds |  3 +++
 tensorflow/lite/tflite_version_script.lds   |  8 ++++++
 3 files changed, 38 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/lite/tflite_exported_symbols.lds
 create mode 100644 tensorflow/lite/tflite_version_script.lds

diff --git a/tensorflow/lite/BUILD b/tensorflow/lite/BUILD
index cadc11e227..86eca1d14e 100644
--- a/tensorflow/lite/BUILD
+++ b/tensorflow/lite/BUILD
@@ -5,7 +5,7 @@ package(
 licenses(["notice"])  # Apache 2.0
 
 load("//tensorflow:tensorflow.bzl", "if_not_windows", "tf_cc_test")
-load("//tensorflow/lite:build_def.bzl", "tflite_copts")
+load("//tensorflow/lite:build_def.bzl", "tflite_cc_shared_object", "tflite_copts")
 load("//tensorflow/lite:special_rules.bzl", "tflite_portable_test_suite")
 
 exports_files(glob([
@@ -398,4 +398,30 @@ cc_test(
     ],
 )
 
+# Shared lib target for convenience, pulls in the core runtime and builtin ops.
+# Note: This target is not yet finalized, and the exact set of exported (C/C++)
+# APIs is subject to change.
+tflite_cc_shared_object(
+    name = "libtensorflowlite.so",
+    linkopts = select({
+        "//tensorflow:darwin": [
+            "-Wl,-exported_symbols_list",  # This line must be directly followed by the exported_symbols.lds file
+            "$(location //tensorflow/lite:tflite_exported_symbols.lds)",
+            "-Wl,-install_name,@rpath/libtensorflowlite.so",
+        ],
+        "//tensorflow:windows": [],
+        "//conditions:default": [
+            "-z defs",
+            "-Wl,--version-script",  #  This line must be directly followed by the version_script.lds file
+            "$(location //tensorflow/lite:tflite_version_script.lds)",
+        ],
+    }),
+    deps = [
+        ":framework",
+        ":tflite_exported_symbols.lds",
+        ":tflite_version_script.lds",
+        "//tensorflow/lite/kernels:builtin_ops",
+    ],
+)
+
 tflite_portable_test_suite()
diff --git a/tensorflow/lite/tflite_exported_symbols.lds b/tensorflow/lite/tflite_exported_symbols.lds
new file mode 100644
index 0000000000..b145204aa1
--- /dev/null
+++ b/tensorflow/lite/tflite_exported_symbols.lds
@@ -0,0 +1,3 @@
+*TfLite*
+*tflite*
+*TFL_*
diff --git a/tensorflow/lite/tflite_version_script.lds b/tensorflow/lite/tflite_version_script.lds
new file mode 100644
index 0000000000..1df70705eb
--- /dev/null
+++ b/tensorflow/lite/tflite_version_script.lds
@@ -0,0 +1,8 @@
+VERS_1.0 {
+  global:
+    *TfLite*;
+    *tflite*;
+    *TFL_*;
+  local:
+    *;
+};
-- 
GitLab


From d4b3956c3759afac03f2a21c77399a01150f2928 Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Mon, 4 Feb 2019 17:21:48 -0800
Subject: [PATCH 1890/2345] Make name_scopes work correctly in V2.

PiperOrigin-RevId: 232397249
---
 tensorflow/python/keras/engine/base_layer.py  | 113 +++++++++---------
 .../python/keras/engine/base_layer_test.py    |  28 +++++
 2 files changed, 85 insertions(+), 56 deletions(-)

diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index ca76823741..4eb0be58cf 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -551,64 +551,59 @@ class Layer(checkpointable.Checkpointable):
         # pass to __call__, hence we set previous_mask as the default value.
         kwargs['mask'] = previous_mask
 
-    with ops.name_scope(self._name_scope()):
-      if not self.built:
+    # Check input assumptions set after layer building, e.g. input shape.
+    if build_graph:
+      # Symbolic execution on symbolic tensors. We will attempt to build
+      # the corresponding TF subgraph inside `backend.get_graph()`
+      input_spec.assert_input_compatibility(self.input_spec, inputs, self.name)
+      graph = backend.get_graph()
+      with graph.as_default(), ops.name_scope(self._name_scope()):
         # Build layer if applicable (if the `build` method has been overridden).
         self._maybe_build(inputs)
-        # We must set self.built since user defined build functions are not
-        # constrained to set self.built.
-        self.built = True
-
-      # Check input assumptions set after layer building, e.g. input shape.
-      if build_graph:
-        # Symbolic execution on symbolic tensors. We will attempt to build
-        # the corresponding TF subgraph inside `backend.get_graph()`
-        input_spec.assert_input_compatibility(
-            self.input_spec, inputs, self.name)
-        graph = backend.get_graph()
-        with graph.as_default():
-          if not self.dynamic:
-            try:
-              outputs = self.call(inputs, *args, **kwargs)
-            except TypeError as e:
-              messages = ('`tf.Tensor` as a Python `bool` is not allowed',
-                          'Tensor objects are only iterable when eager')
-              exception_str = str(e)
-              for msg in messages:
-                if msg in exception_str:
-                  raise TypeError('You are attempting to use Python control '
-                                  'flow in a layer that was not declared to be '
-                                  'dynamic. Pass `dynamic=True` to the class '
-                                  'constructor.\nEncountered error:\n"""\n' +
-                                  exception_str + '\n"""')
-              raise
-          else:
-            # We will use static shape inference to return symbolic tensors
-            # matching the specifications of the layer outputs.
-            # Since `self.dynamic` is True, we will never attempt to
-            # run the underlying TF graph (which is disconnected).
-            # TODO(fchollet): consider py_func as an alternative, which
-            # would enable us to run the underlying graph if needed.
-            outputs = self._symbolic_call(inputs)
-
-          if outputs is None:
-            raise ValueError('A layer\'s `call` method should return a '
-                             'Tensor or a list of Tensors, not None '
-                             '(layer: ' + self.name + ').')
-          if base_layer_utils.have_all_keras_metadata(inputs):
-            inputs, outputs = self._set_connectivity_metadata_(
-                inputs, outputs, args, kwargs)
-          self._handle_activity_regularization(inputs, outputs)
-          self._set_mask_metadata(inputs, outputs, previous_mask)
-          if hasattr(self, '_set_inputs') and not self.inputs:
-            # Subclassed network: explicitly set metadata normally set by
-            # a call to self._set_inputs().
-            # TODO(b/120997007): This should be done in Eager as well, but
-            # causes garbage collection issues because of the placeholders
-            # created on the default Keras graph.
-            self._set_inputs(inputs, outputs)
-      else:
-        # Eager execution on data tensors.
+        if not self.dynamic:
+          try:
+            outputs = self.call(inputs, *args, **kwargs)
+          except TypeError as e:
+            messages = ('`tf.Tensor` as a Python `bool` is not allowed',
+                        'Tensor objects are only iterable when eager')
+            exception_str = str(e)
+            for msg in messages:
+              if msg in exception_str:
+                raise TypeError('You are attempting to use Python control '
+                                'flow in a layer that was not declared to be '
+                                'dynamic. Pass `dynamic=True` to the class '
+                                'constructor.\nEncountered error:\n"""\n' +
+                                exception_str + '\n"""')
+            raise
+        else:
+          # We will use static shape inference to return symbolic tensors
+          # matching the specifications of the layer outputs.
+          # Since `self.dynamic` is True, we will never attempt to
+          # run the underlying TF graph (which is disconnected).
+          # TODO(fchollet): consider py_func as an alternative, which
+          # would enable us to run the underlying graph if needed.
+          outputs = self._symbolic_call(inputs)
+
+        if outputs is None:
+          raise ValueError('A layer\'s `call` method should return a '
+                           'Tensor or a list of Tensors, not None '
+                           '(layer: ' + self.name + ').')
+        if base_layer_utils.have_all_keras_metadata(inputs):
+          inputs, outputs = self._set_connectivity_metadata_(
+              inputs, outputs, args, kwargs)
+        self._handle_activity_regularization(inputs, outputs)
+        self._set_mask_metadata(inputs, outputs, previous_mask)
+        if hasattr(self, '_set_inputs') and not self.inputs:
+          # Subclassed network: explicitly set metadata normally set by
+          # a call to self._set_inputs().
+          # TODO(b/120997007): This should be done in Eager as well, but
+          # causes garbage collection issues because of the placeholders
+          # created on the default Keras graph.
+          self._set_inputs(inputs, outputs)
+    else:
+      # Eager execution on data tensors.
+      with ops.name_scope(self._name_scope()):
+        self._maybe_build(inputs)
         outputs = self.call(inputs, *args, **kwargs)
         self._handle_activity_regularization(inputs, outputs)
         self._set_mask_metadata(inputs, outputs, previous_mask)
@@ -1578,6 +1573,9 @@ class Layer(checkpointable.Checkpointable):
 
   def _maybe_build(self, inputs):
     # Check input assumptions set before layer building, e.g. input rank.
+    if self.built:
+      return
+
     input_spec.assert_input_compatibility(
         self.input_spec, inputs, self.name)
     input_list = nest.flatten(inputs)
@@ -1592,6 +1590,9 @@ class Layer(checkpointable.Checkpointable):
     # Only call `build` if the user has manually overridden the build method.
     if not hasattr(self.build, '_is_default'):
       self.build(input_shapes)
+    # We must set self.built since user defined build functions are not
+    # constrained to set self.built.
+    self.built = True
 
   def _symbolic_call(self, inputs):
     input_shapes = nest.map_structure(lambda x: x.shape, inputs)
diff --git a/tensorflow/python/keras/engine/base_layer_test.py b/tensorflow/python/keras/engine/base_layer_test.py
index 669b1df576..ba14f94cc2 100644
--- a/tensorflow/python/keras/engine/base_layer_test.py
+++ b/tensorflow/python/keras/engine/base_layer_test.py
@@ -456,6 +456,34 @@ class NestedTrackingTest(test.TestCase):
       self.assertEqual(len(layer.updates), 3)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class NameScopingTest(keras_parameterized.TestCase):
+
+  def test_name_scope_layer(self):
+    x = keras.backend.placeholder(shape=(10, 10))
+    layer = keras.layers.Dense(10, name='MyName')
+    layer(x)
+    self.assertEqual(layer.bias.name, 'MyName/bias:0')
+    self.assertEqual(layer.kernel.name, 'MyName/kernel:0')
+
+  def test_name_scope_sublayer(self):
+    x = keras.backend.placeholder(shape=(10, 10))
+    layer = keras.layers.Dense(
+        10, activation=keras.layers.ReLU(name='MyAct'), name='MyName2')
+    y = layer(x)
+    self.assertEqual(layer.bias.name, 'MyName2/bias:0')
+    self.assertEqual(layer.kernel.name, 'MyName2/kernel:0')
+    self.assertEqual(y.name, 'MyName2/MyAct/Relu:0')
+
+  def test_name_scope_tf_tensor(self):
+    x = ops.convert_to_tensor(np.ones((10, 10)))
+    layer = keras.layers.Dense(
+        10, activation=keras.layers.ReLU(name='MyAct'), name='MyName3')
+    layer(x)
+    self.assertEqual(layer.bias.name, 'MyName3/bias:0')
+    self.assertEqual(layer.kernel.name, 'MyName3/kernel:0')
+
+
 if __name__ == '__main__':
   ops.enable_eager_execution()
   test.main()
-- 
GitLab


From be6d1285be1a15708a434ba5777a4194113e4a2a Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Mon, 4 Feb 2019 17:42:33 -0800
Subject: [PATCH 1891/2345] Make ColocationGraph::Member a full class

This separation helps with reasoning as ColocationGraph and Member
classes become more complex.

PiperOrigin-RevId: 232400025
---
 tensorflow/core/common_runtime/placer.cc | 473 +++++++++++++++--------
 1 file changed, 304 insertions(+), 169 deletions(-)

diff --git a/tensorflow/core/common_runtime/placer.cc b/tensorflow/core/common_runtime/placer.cc
index 8c0a3fb0c5..72e5efcebf 100644
--- a/tensorflow/core/common_runtime/placer.cc
+++ b/tensorflow/core/common_runtime/placer.cc
@@ -32,6 +32,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/stringpiece.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
+#include "tensorflow/core/util/device_name_utils.h"
 #include "tensorflow/core/util/port.h"
 
 namespace tensorflow {
@@ -115,6 +116,271 @@ bool IsExemptFromResourceInputColocation(const Node* node) {
   return op_type == "PartitionedCall" || op_type == "StatefulPartitionedCall";
 }
 
+bool HasPriorities(const PrioritizedDeviceTypeVector& device_types) {
+  for (const auto& prioritized_device_type : device_types) {
+    if (prioritized_device_type.second != 0) return true;
+  }
+  return false;
+}
+
+bool ArePrioritiesSame(const PrioritizedDeviceTypeVector& a_types,
+                       const PrioritizedDeviceTypeVector& b_types) {
+  if (a_types.size() != b_types.size()) {
+    return false;
+  }
+  for (int i = 0; i < a_types.size(); ++i) {
+    if (a_types[i].first != b_types[i].first) {
+      return false;
+    }
+  }
+  return true;
+}
+
+// Represents a node in the disjoint node set forest, and the
+// accumulated constraints on the device used by that node.
+class Member {
+ public:
+  Member() = default;
+
+  Status SetParentAndSupportedDevices(const Node& node,
+                                      const std::vector<DeviceType>& types) {
+    int id = node.id();
+    if (id < 0) {
+      return errors::Internal(
+          "Placer should not be creating a Member for node: ",
+          node.DebugString());
+    }
+    parent_ = id;
+    return SupportedDeviceTypesForNode(types, node.def(),
+                                       &supported_device_types_);
+  }
+
+  const DeviceNameUtils::ParsedName& device_name() const {
+    return device_name_;
+  }
+
+  Status SetDeviceName(const string& device_name) {
+    if (!DeviceNameUtils::ParseFullName(device_name, &device_name_)) {
+      return errors::Internal("Malformed assigned device '", device_name, "'");
+    }
+    return Status::OK();
+  }
+  void SetDeviceName(const DeviceNameUtils::ParsedName& device_name) {
+    device_name_ = device_name;
+  }
+
+  const PrioritizedDeviceTypeVector& supported_device_types() const {
+    return supported_device_types_;
+  }
+
+  static void Merge(std::vector<Member>* tree, int x_root, int y_root,
+                    Member** new_root, Member** old_root) {
+    Member& x_root_member = (*tree)[x_root];
+    Member& y_root_member = (*tree)[y_root];
+
+    // Merge the sets by setting the parent pointer of the smaller tree's root
+    // node to point to the root of the larger tree. Together with path
+    // compression in ColocationGraph::FindRoot, this ensures that we do not
+    // experience pathological performance on graphs such as chains.
+    int new_root_id, old_root_id;
+    if (x_root_member.rank_ < y_root_member.rank_) {
+      // The tree rooted at x_root is shallower, so connect it to
+      // y_root. The rank of y_root is unchanged because its new
+      // child has strictly less rank.
+      x_root_member.parent_ = y_root;
+      new_root_id = y_root;
+      old_root_id = x_root;
+    } else if (x_root_member.rank_ > y_root_member.rank_) {
+      // The tree rooted at y_root is shallower, so connect it to
+      // x_root. The rank of x_root is unchanged because its new
+      // child has strictly less rank.
+      y_root_member.parent_ = x_root;
+      new_root_id = x_root;
+      old_root_id = y_root;
+    } else {
+      // Both trees have the same rank, so break the tie by choosing
+      // x_root as the new root.
+      y_root_member.parent_ = x_root;
+      // Increment the rank of the tree rooted at x_root, because it
+      // is now strictly deeper than before.
+      ++x_root_member.rank_;
+      new_root_id = x_root;
+      old_root_id = y_root;
+    }
+
+    *new_root = &(*tree)[new_root_id];
+    *old_root = &(*tree)[old_root_id];
+  }
+
+  // tree is non-const because we can change some `parent` pointers in some
+  // members for more efficient future lookups. The vector itself is not
+  // changed.
+  static int FindRoot(std::vector<Member>* tree, int node_id) {
+    Member& member = (*tree)[node_id];
+    if (member.parent_ == node_id) {
+      // member.parent is the root of this disjoint tree.  Do nothing.
+    } else {
+      member.parent_ = FindRoot(tree, member.parent_);
+    }
+    // Now it is guaranteed that member.parent is the root of this disjoint
+    // tree.
+    return member.parent_;
+  }
+
+  Status MergeDeviceNames(const Member& other, bool allow_soft_placement) {
+    return DeviceNameUtils::MergeDevNames(&device_name_, other.device_name_,
+                                          allow_soft_placement);
+  }
+  Status MergeDeviceNames(const string& dev_name, bool allow_soft_placement) {
+    DeviceNameUtils::ParsedName parsed;
+    DeviceNameUtils::ParseFullName(dev_name, &parsed);
+    return DeviceNameUtils::MergeDevNames(&device_name_, parsed,
+                                          allow_soft_placement);
+  }
+
+  // Updates this to contain the intersection of the device types in
+  // this and "other".
+  void MergeSupportedDevices(const Member& other) {
+    PrioritizedDeviceTypeVector temp = supported_device_types_;
+    supported_device_types_.clear();
+
+    // Generate intersection with priorities.
+    PrioritizedDeviceTypeVector target_intersection;
+    PrioritizedDeviceTypeVector other_intersection;
+    for (const auto& prioritized_device_type : temp) {
+      bool found = false;
+      for (const auto& other_prioritized_device_type :
+           other.supported_device_types_) {
+        if (prioritized_device_type.first ==
+            other_prioritized_device_type.first) {
+          found = true;
+          other_intersection.push_back(other_prioritized_device_type);
+          break;
+        }
+      }
+      if (found) {
+        target_intersection.push_back(prioritized_device_type);
+      }
+    }
+
+    // Sort the devices by priority order.
+    auto device_sort = [](const std::pair<DeviceType, int32>& a,
+                          const std::pair<DeviceType, int32>& b) {
+      // First look at set priorities.
+      if (a.second != b.second) {
+        return a.second > b.second;
+      }
+      // Then fallback to default priorities.
+      auto a_priority = DeviceSet::DeviceTypeOrder(a.first);
+      auto b_priority = DeviceSet::DeviceTypeOrder(b.first);
+      if (a_priority != b_priority) {
+        return a_priority > b_priority;
+      }
+      // Finally just look at the Device type strings.
+      return a.first.type_string() < b.first.type_string();
+    };
+
+    std::sort(target_intersection.begin(), target_intersection.end(),
+              device_sort);
+    std::sort(other_intersection.begin(), other_intersection.end(),
+              device_sort);
+
+    bool is_target_prioritized = HasPriorities(target_intersection);
+    bool is_other_prioritized = HasPriorities(other_intersection);
+    // If neither are prioritized then we just return the original i.e. target
+    // prioritization.
+    if (!is_target_prioritized && !is_other_prioritized) {
+      supported_device_types_ = target_intersection;
+    }
+    // If only one is prioritized, then we respect priorities of that in the
+    // intersection.
+    if (is_target_prioritized && !is_other_prioritized) {
+      supported_device_types_ = target_intersection;
+    }
+    if (!is_target_prioritized && is_other_prioritized) {
+      supported_device_types_ = other_intersection;
+    }
+    // If both have priorities and agree then we go with that. If the
+    // prioritization order is different, then we just fallback to the default
+    // i.e. what the DeviceTypeOrder suggests. In that case, we also set the
+    // merged priorities to 0, so that downstream merges work correctly as well.
+    if (is_target_prioritized && is_other_prioritized) {
+      bool priorities_agree =
+          ArePrioritiesSame(target_intersection, other_intersection);
+      if (priorities_agree) {
+        supported_device_types_ = target_intersection;
+      } else {
+        for (const auto& prioritized_device : target_intersection) {
+          supported_device_types_.push_back(
+              std::make_pair(prioritized_device.first, 0));
+        }
+        std::sort(supported_device_types_.begin(),
+                  supported_device_types_.end(), device_sort);
+      }
+    }
+  }
+
+  Status AssignDevice(const Node& node, bool allow_soft_placement) {
+    if (node.assigned_device_name_index() == assigned_device_name_index_) {
+      return Status::OK();
+    }
+    Status s =
+        MergeDeviceNames(node.assigned_device_name(), allow_soft_placement);
+    if (!s.ok()) {
+      return errors::Internal(
+          "Constraining by assigned device should not cause an error. Original "
+          "root device name: ",
+          DeviceNameUtils::ParsedNameToString(device_name_),
+          " assigned device name \"", node.assigned_device_name(),
+          ". Error: ", s.error_message());
+    }
+
+    assigned_device_name_index_ = node.assigned_device_name_index();
+    // Clear cached possible_devices, if any.
+    possible_devices_.clear();
+    return Status::OK();
+  }
+
+  void set_possible_devices(std::vector<Device*>&& devices) {
+    possible_devices_ = devices;
+  }
+  const std::vector<Device*>& possible_devices() { return possible_devices_; }
+
+ private:
+  // The id of the node that is the parent of this one, or its own
+  // id if it is a root. parent <= 0 indicates that this member is invalid.
+  int parent_ = -1;
+
+  // A proxy for the depth of the tree that is used to prefer
+  // connecting smaller trees to larger trees when merging disjoint
+  // sets.
+  int rank_ = 0;
+
+  // Once colocation groups have been formed and we assigned at least
+  // one node in this group to a device, assigned_device_name_index will
+  // contain this device name's index in the graph. The `device_name` will
+  // contain the parsed name of this device and `possible_devices`, if
+  // computed, will contain just this device.
+  // `assigned_device_name_index` is an optimization to avoid parsing and
+  // comparing device names. The value of -1 signals that a single device
+  // has not been chosen yet.
+  int assigned_device_name_index_ = -1;
+
+  // The merged form of the device requested for this node, with
+  // those of all of its children.
+  DeviceNameUtils::ParsedName device_name_;
+
+  // The intersection of all device types supported by this node,
+  // and those of all of its children, in priority order
+  // of the preferred device.
+  PrioritizedDeviceTypeVector supported_device_types_;
+
+  // If this node is a root, stores a list of Devices to which this node
+  // and all of its children have been assigned, or nullptr if this
+  // has not yet been computed.
+  std::vector<Device*> possible_devices_;
+};
+
 // This class maintains the connected components of a colocation
 // constraint graph, and uses this information to assign a satisfying
 // device placement to the nodes of the graph.
@@ -231,8 +497,8 @@ class ColocationGraph {
     // specified a device, then 'dst's device should be
     // cleared: the reference edge forces 'node' to be on the
     // same device as the source node.
-    const auto& source_parsed_name = src_root.device_name;
-    const auto& dest_parsed_name = dst_root.device_name;
+    const auto& source_parsed_name = src_root.device_name();
+    const auto& dest_parsed_name = dst_root.device_name();
     if (DeviceNameUtils::HasSomeDetails(source_parsed_name) &&
         DeviceNameUtils::HasSomeDetails(dest_parsed_name)) {
       // Ignore a specified device for 'dst' if the two names were
@@ -252,7 +518,7 @@ class ColocationGraph {
         }
 
         // Make 'dst' colocated with the source
-        dst_root.device_name = source_parsed_name;
+        dst_root.SetDeviceName(source_parsed_name);
       }
     }
     Status status = ColocateNodes(*src, src_root_id, *dst, dst_root_id);
@@ -337,50 +603,18 @@ class ColocationGraph {
     DCHECK_EQ(x_root, FindRoot(x.id()));
     DCHECK_EQ(y_root, FindRoot(y.id()));
 
-    Member& x_root_member = members_[x_root];
-    Member& y_root_member = members_[y_root];
-
-    // Merge the sets by setting the parent pointer of the smaller tree's root
-    // node to point to the root of the larger tree. Together with path
-    // compression in ColocationGraph::FindRoot, this ensures that we do not
-    // experience pathological performance on graphs such as chains.
-    int new_root, old_root;
-    if (x_root_member.rank < y_root_member.rank) {
-      // The tree rooted at x_root is shallower, so connect it to
-      // y_root. The rank of y_root is unchanged because its new
-      // child has strictly less rank.
-      x_root_member.parent = y_root;
-      new_root = y_root;
-      old_root = x_root;
-    } else if (x_root_member.rank > y_root_member.rank) {
-      // The tree rooted at y_root is shallower, so connect it to
-      // x_root. The rank of x_root is unchanged because its new
-      // child has strictly less rank.
-      y_root_member.parent = x_root;
-      new_root = x_root;
-      old_root = y_root;
-    } else {
-      // Both trees have the same rank, so break the tie by choosing
-      // x_root as the new root.
-      y_root_member.parent = x_root;
-      // Increment the rank of the tree rooted at x_root, because it
-      // is now strictly deeper than before.
-      ++x_root_member.rank;
-      new_root = x_root;
-      old_root = y_root;
-    }
-
-    Member& new_root_member = members_[new_root];
-    Member& old_root_member = members_[old_root];
+    Member* new_root_member;
+    Member* old_root_member;
+    Member::Merge(&members_, x_root, y_root, &new_root_member,
+                  &old_root_member);
 
     // Merge the partial device specifications, and ensure that they are
     // compatible. NULL options_ is treated as allowing soft placement.
     // TODO(mrry): Consider enriching the error message by pointing
     // out which nodes have the explicit partial device
     // specifications that caused this conflict.
-    Status s = DeviceNameUtils::MergeDevNames(&new_root_member.device_name,
-                                              old_root_member.device_name,
-                                              allow_soft_placement_);
+    Status s = new_root_member->MergeDeviceNames(*old_root_member,
+                                                 allow_soft_placement_);
     if (!s.ok()) {
       return errors::InvalidArgument(
           "Cannot colocate nodes ",
@@ -393,9 +627,8 @@ class ColocationGraph {
     // type, by computing the intersection of
     // new_root_member.supported_device_types and
     // old_root_member.supported_device_types.
-    MergeSupportedDevices(&new_root_member.supported_device_types,
-                          old_root_member.supported_device_types);
-    if (new_root_member.supported_device_types.empty()) {
+    new_root_member->MergeSupportedDevices(*old_root_member);
+    if (new_root_member->supported_device_types().empty()) {
       return errors::InvalidArgument(
           "Cannot colocate nodes ",
           errors::FormatColocationNodeForError(x.name()), " and ",
@@ -422,28 +655,7 @@ class ColocationGraph {
     }
     int root = FindRoot(node.id());
     Member& root_member = members_[root];
-    if (node.assigned_device_name_index() ==
-        root_member.assigned_device_name_index) {
-      return Status::OK();
-    }
-    DeviceNameUtils::ParsedName parsed;
-    DeviceNameUtils::ParseFullName(node.assigned_device_name(), &parsed);
-    Status s = DeviceNameUtils::MergeDevNames(&root_member.device_name, parsed,
-                                              allow_soft_placement_);
-    if (!s.ok()) {
-      return errors::Internal(
-          "Constraining by assigned device should not cause an error. Original "
-          "root device name: ",
-          DeviceNameUtils::ParsedNameToString(root_member.device_name),
-          " assigned device name \"", node.assigned_device_name(),
-          ". Error: ", s.error_message());
-    }
-
-    root_member.assigned_device_name_index = node.assigned_device_name_index();
-    // Clear cached possible_devices, if any.
-    root_member.possible_devices.clear();
-
-    return Status::OK();
+    return root_member.AssignDevice(node, allow_soft_placement_);
   }
 
   // For the given node, subject to the constraints previously given
@@ -454,11 +666,11 @@ class ColocationGraph {
   // The caller must not use the returned pointer after there is any possibility
   // that the members_[i].possible_devices field has been modified.
   Status GetDevicesForNode(Node* node,
-                           std::vector<Device*>** possible_devices) {
+                           const std::vector<Device*>** possible_devices) {
     *possible_devices = nullptr;
     const int node_root = FindRoot(node->id());
-    if (!members_[node_root].possible_devices.empty()) {
-      *possible_devices = &members_[node_root].possible_devices;
+    if (!members_[node_root].possible_devices().empty()) {
+      *possible_devices = &members_[node_root].possible_devices();
       return Status::OK();
     }
 
@@ -469,18 +681,18 @@ class ColocationGraph {
     // "devices" will contain the set of feasible placements for the
     // colocated node set containing 'node'.
     std::vector<Device*> devices;
-    if (DeviceNameUtils::HasSomeDetails(members_[node_root].device_name)) {
+    if (DeviceNameUtils::HasSomeDetails(members_[node_root].device_name())) {
       // The root node has a (possibly partial) device
       // specification, so enumerate the physical devices that
       // conform to it.
-      device_set_->FindMatchingDevices(members_[node_root].device_name,
+      device_set_->FindMatchingDevices(members_[node_root].device_name(),
                                        &devices);
 
       if (!devices.empty()) {
         // Filter devices into those that are compatible with the root
         // node (and its children).
         devices = FilterSupportedDevices(
-            devices, members_[node_root].supported_device_types,
+            devices, members_[node_root].supported_device_types(),
             default_device_);
       }
 
@@ -489,14 +701,14 @@ class ColocationGraph {
         // The soft_device_name is the same as the node's device name
         // without specifying the device type or ID.
         DeviceNameUtils::ParsedName soft_device_name =
-            members_[node_root].device_name;
+            members_[node_root].device_name();
         soft_device_name.type.clear();
         soft_device_name.has_type = false;
         soft_device_name.has_id = false;
         device_set_->FindMatchingDevices(soft_device_name, &devices);
         if (!devices.empty()) {
           devices = FilterSupportedDevices(
-              devices, members_[node_root].supported_device_types,
+              devices, members_[node_root].supported_device_types(),
               default_device_);
         }
       }
@@ -510,7 +722,7 @@ class ColocationGraph {
         DeviceNameUtils::ParsedName specified_device_name;
         if (DeviceNameUtils::ParseFullName(node->requested_device(),
                                            &specified_device_name) &&
-            specified_device_name == members_[node_root].device_name) {
+            specified_device_name == members_[node_root].device_name()) {
           // The specified device and merged set device match, and
           // will appear in the GraphDef (for debugging), so just
           // print the specified device.
@@ -562,7 +774,7 @@ class ColocationGraph {
               " was colocated with a group of nodes that ",
               "required incompatible device '",
               DeviceNameUtils::ParsedNameToString(
-                  members_[node_root].device_name),
+                  members_[node_root].device_name()),
               "'", debug_info);
         }
       }
@@ -573,7 +785,7 @@ class ColocationGraph {
         return errors::Internal("No devices are registered");
       }
       devices = FilterSupportedDevices(
-          device_set_->devices(), members_[node_root].supported_device_types,
+          device_set_->devices(), members_[node_root].supported_device_types(),
           default_device_);
 
       if (devices.empty()) {
@@ -585,8 +797,8 @@ class ColocationGraph {
     }
 
     // Cache the result of the possible devices for this node group.
-    members_[node_root].possible_devices = std::move(devices);
-    *possible_devices = &members_[node_root].possible_devices;
+    members_[node_root].set_possible_devices(std::move(devices));
+    *possible_devices = &members_[node_root].possible_devices();
     return Status::OK();
   }
 
@@ -603,44 +815,6 @@ class ColocationGraph {
     return Status::OK();
   }
 
-  // Represents a node in the disjoint node set forest, and the
-  // accumulated constraints on the device used by that node.
-  struct Member {
-    Member() = default;
-    // The id of the node that is the parent of this one, or its own
-    // id if it is a root. parent <= 0 indicates that this member is invalid.
-    int parent = -1;
-
-    // A proxy for the depth of the tree that is used to prefer
-    // connecting smaller trees to larger trees when merging disjoint
-    // sets.
-    int rank = 0;
-
-    // The intersection of all device types supported by this node,
-    // and those of all of its children, in priority order
-    // of the preferred device.
-    PrioritizedDeviceTypeVector supported_device_types;
-
-    // The merged form of the device requested for this node, with
-    // those of all of its children.
-    DeviceNameUtils::ParsedName device_name;
-
-    // Once colocation groups have been formed and we assigned at least
-    // one node in this group to a device, assigned_device_name_index will
-    // contain this device name's index in the graph. The `device_name` will
-    // contain the parsed name of this device and `possible_devices`, if
-    // computed, will contain just this device.
-    // `assigned_device_name_index` is an optimization to avoid parsing and
-    // comparing device names. The value of -1 signals that a single device
-    // has not been chosen yet.
-    int assigned_device_name_index = -1;
-
-    // If this node is a root, stores a list of Devices to which this node
-    // and all of its children have been assigned, or nullptr if this
-    // has not yet been computed.
-    std::vector<Device*> possible_devices;
-  };
-
   // Returns debugging info for the node referred to by 'node_root'.
   string DebugInfo(const int node_root) {
     string text(
@@ -667,7 +841,7 @@ class ColocationGraph {
       colocation_nodes.push_back(node);
       const string& op_type = node->type_string();
       string devices_registered;
-      for (const auto& device_type : members_[id].supported_device_types) {
+      for (const auto& device_type : members_[id].supported_device_types()) {
         strings::StrAppend(&devices_registered,
                            DeviceTypeString(device_type.first), " ");
       }
@@ -693,11 +867,8 @@ class ColocationGraph {
   }
 
   Status InitializeMember(const Node& node, Member* member) {
-    const int id = node.id();
-    DCHECK_GE(id, 0);
-    member->parent = id;
-    TF_RETURN_IF_ERROR(SupportedDeviceTypesForNode(
-        device_types_, node.def(), &member->supported_device_types));
+    TF_RETURN_IF_ERROR(
+        member->SetParentAndSupportedDevices(node, device_types_));
 
     if (node.has_assigned_device_name()) {
       // This node has already been assigned to a device, so we
@@ -710,11 +881,7 @@ class ColocationGraph {
       // the TensorFlow runtime, we consider errors in this branch to
       // be INTERNAL.
       const string& assigned_device_name = node.assigned_device_name();
-      if (!DeviceNameUtils::ParseFullName(assigned_device_name,
-                                          &member->device_name)) {
-        return errors::Internal("Malformed assigned device '",
-                                assigned_device_name, "'");
-      }
+      TF_RETURN_IF_ERROR(member->SetDeviceName(assigned_device_name));
       const Device* assigned_device =
           device_set_->FindDeviceByName(assigned_device_name);
       if (assigned_device == nullptr) {
@@ -722,7 +889,7 @@ class ColocationGraph {
                                 "' does not match any device");
       }
 
-      for (const auto& d : member->supported_device_types) {
+      for (const auto& d : member->supported_device_types()) {
         if (DeviceType(assigned_device->attributes().device_type()) ==
             d.first) {
           return Status::OK();
@@ -740,7 +907,7 @@ class ColocationGraph {
       // in the NodeDef.
 
       // If no kernels are registered for this op type, fail with an error.
-      if (member->supported_device_types.empty()) {
+      if (member->supported_device_types().empty()) {
         std::set<string> registered_device_types;
         for (Device* d : device_set_->devices()) {
           registered_device_types.insert(d->device_type());
@@ -771,36 +938,16 @@ class ColocationGraph {
         // devices.
         // NOTE: The full name may specify a device that is not in
         // n.supported_device_types(), but we check that in AssignDevice().
-        if (!DeviceNameUtils::ParseFullName(node.requested_device(),
-                                            &member->device_name)) {
+        if (!member->SetDeviceName(node.requested_device()).ok()) {
           return errors::InvalidArgument("Malformed device specification '",
-                                         node.requested_device(), "'");
+                                         node.requested_device(),
+                                         "' in node: ", node.DebugString());
         }
       }
     }
     return Status::OK();
   }
 
-  static bool HasPriorities(const PrioritizedDeviceTypeVector& device_types) {
-    for (const auto& prioritized_device_type : device_types) {
-      if (prioritized_device_type.second != 0) return true;
-    }
-    return false;
-  }
-
-  static bool ArePrioritiesSame(const PrioritizedDeviceTypeVector& a_types,
-                                const PrioritizedDeviceTypeVector& b_types) {
-    if (a_types.size() != b_types.size()) {
-      return false;
-    }
-    for (int i = 0; i < a_types.size(); ++i) {
-      if (a_types[i].first != b_types[i].first) {
-        return false;
-      }
-    }
-    return true;
-  }
-
   // Updates target to contain the intersection of the device types in
   // "target" and "other".
   static void MergeSupportedDevices(PrioritizedDeviceTypeVector* target,
@@ -883,19 +1030,7 @@ class ColocationGraph {
 
   // Returns the root node of the disjoint tree to which the node with the
   // given id is connected.
-  int FindRoot(int node_id) {
-    Member& member = members_[node_id];
-    DCHECK_GE(member.parent, 0);
-    if (member.parent == node_id) {
-      // member.parent is the root of this disjoint tree.  Do nothing.
-    } else {
-      member.parent = FindRoot(member.parent);
-    }
-    // Now it is guaranteed that member.parent is the root of this disjoint
-    // tree.
-    DCHECK_GE(member.parent, 0);
-    return member.parent;
-  }
+  int FindRoot(int node_id) { return Member::FindRoot(&members_, node_id); }
 
   // Ensures that the devices of 'dst's resource and reference match the device
   // specified for 'src', which is an input of 'dst' with a partially or fully
@@ -913,7 +1048,7 @@ class ColocationGraph {
           continue;
         }
         const auto& input_root = members_[FindRoot(input_node->id())];
-        const auto& input_parsed_name = input_root.device_name;
+        const auto& input_parsed_name = input_root.device_name();
         if (DeviceNameUtils::HasSomeDetails(input_parsed_name) &&
             !DeviceNameUtils::AreCompatibleDevNames(input_parsed_name,
                                                     src_parsed_name)) {
@@ -1014,7 +1149,7 @@ Status Placer::Run() {
       continue;
     }
 
-    std::vector<Device*>* devices;
+    const std::vector<Device*>* devices;
     Status status = colocation_graph.GetDevicesForNode(node, &devices);
     if (!status.ok()) {
       return AttachDef(
@@ -1062,7 +1197,7 @@ Status Placer::Run() {
   // Perform a second pass assignment for those nodes explicitly
   // skipped during the first pass.
   for (Node* node : second_pass) {
-    std::vector<Device*>* devices;
+    const std::vector<Device*>* devices;
     Status status = colocation_graph.GetDevicesForNode(node, &devices);
     if (!status.ok()) {
       return AttachDef(
-- 
GitLab


From a335040d6605f0a0b5abc59179eff505e171022a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 4 Feb 2019 17:47:57 -0800
Subject: [PATCH 1892/2345] Fix loss reductions to correctly divide by global
 batch size when using a tf.distribute.Strategy, instead of less-reliable
 division by the number of replicas in optimizers.

RELNOTES: Bug fix: loss and gradients should now more reliably be
correctly scaled w.r.t. the global batch size when using a
tf.distribute.Strategy.
PiperOrigin-RevId: 232400597
---
 .../python/keras_optimizer_v2_test.py         |  2 +-
 .../python/losses/python/losses_impl_test.py  |  4 ++-
 .../contrib/optimizer_v2/optimizer_v2.py      | 18 +++-------
 tensorflow/python/distribute/BUILD            |  1 -
 .../python/distribute/distribute_lib.py       | 15 ++++----
 tensorflow/python/framework/ops.py            |  3 --
 tensorflow/python/keras/BUILD                 |  1 +
 .../engine/distributed_training_utils.py      |  4 +--
 .../keras/engine/training_distributed.py      |  5 ++-
 .../python/keras/optimizer_v2/optimizer_v2.py | 36 +++++++++++--------
 tensorflow/python/keras/utils/losses_utils.py | 10 +++---
 tensorflow/python/ops/losses/BUILD            |  1 +
 tensorflow/python/ops/losses/losses_impl.py   | 25 +++++++------
 tensorflow/python/training/optimizer.py       | 18 ----------
 .../api/golden/v2/tensorflow.distribute.pbtxt |  4 ---
 tensorflow/tools/compatibility/renames_v2.py  |  1 +
 16 files changed, 63 insertions(+), 85 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py b/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
index 952b11932b..c93d7afa7c 100644
--- a/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
+++ b/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
@@ -64,7 +64,7 @@ class MirroredStrategyOptimizerV2Test(test.TestCase, parameterized.TestCase):
 
         def loss_fn():
           replica_id = _replica_id()
-          return math_ops.cast(replica_id + 1, dtype=dtypes.float32) * var
+          return math_ops.cast(replica_id + 1, dtype=dtypes.float32) * 0.5 * var
 
         train_op = optimizer.minimize(loss_fn, var_list=[var])
 
diff --git a/tensorflow/contrib/gan/python/losses/python/losses_impl_test.py b/tensorflow/contrib/gan/python/losses/python/losses_impl_test.py
index e3c780ac1a..44ee0f5269 100644
--- a/tensorflow/contrib/gan/python/losses/python/losses_impl_test.py
+++ b/tensorflow/contrib/gan/python/losses/python/losses_impl_test.py
@@ -403,7 +403,9 @@ class _PenaltyTest(object):
   def test_all_correct(self):
     loss = self._penalty_fn(**self._kwargs)
     self.assertEqual(self._expected_dtype, loss.dtype)
-    self.assertEqual(self._expected_op_name, loss.op.name)
+    # NOTE: Op names will change, it is inappropriate to include them in tests.
+    # See go/tf-breaking-change.
+    # self.assertEqual(self._expected_op_name, loss.op.name)
     with self.cached_session():
       variables.global_variables_initializer().run()
       self.assertAlmostEqual(self._expected_loss, loss.eval(), 6)
diff --git a/tensorflow/contrib/optimizer_v2/optimizer_v2.py b/tensorflow/contrib/optimizer_v2/optimizer_v2.py
index 1323ed014c..a49149e592 100644
--- a/tensorflow/contrib/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/contrib/optimizer_v2/optimizer_v2.py
@@ -24,7 +24,6 @@ import abc
 
 import six
 
-from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import distribution_strategy_context as distribute_ctx
 from tensorflow.python.distribute import reduce_util as ds_reduce_util
 from tensorflow.python.eager import backprop
@@ -661,7 +660,7 @@ class OptimizerV2(optimizer_v1.Optimizer):
                name=None,
                grad_loss=None,
                stop_gradients=None,
-               scale_loss_by_num_replicas=None):
+               scale_loss_by_num_replicas=False):
     """Add operations to minimize `loss` by updating `var_list`.
 
     This method simply combines calls `compute_gradients()` and
@@ -685,8 +684,7 @@ class OptimizerV2(optimizer_v1.Optimizer):
       stop_gradients: Optional. A Tensor or list of tensors not to differentiate
         through.
       scale_loss_by_num_replicas: Optional boolean. If true, scale the loss down
-        by the number of replicas. By default, auto-detects whether this is
-        needed.
+        by the number of replicas. DEPRECATED and generally no longer needed.
 
     Returns:
       An Operation that updates the variables in `var_list`.  If `global_step`
@@ -732,7 +730,7 @@ class OptimizerV2(optimizer_v1.Optimizer):
                         aggregation_method=None,
                         grad_loss=None,
                         stop_gradients=None,
-                        scale_loss_by_num_replicas=None):
+                        scale_loss_by_num_replicas=False):
     """Compute gradients of `loss` for the variables in `var_list`.
 
     This is the first part of `minimize()`.  It returns a list
@@ -756,8 +754,7 @@ class OptimizerV2(optimizer_v1.Optimizer):
       stop_gradients: Optional. A Tensor or list of tensors not to differentiate
         through.
       scale_loss_by_num_replicas: Optional boolean. If true, scale the loss down
-        by the number of replicas. By default, auto-detects whether this is
-        needed.
+        by the number of replicas. DEPRECATED and generally no longer needed.
 
     Returns:
       A list of (gradient, variable) pairs. Variable is always present, but
@@ -781,9 +778,7 @@ class OptimizerV2(optimizer_v1.Optimizer):
           tape.watch(var_list)
         loss_value = loss()
 
-        # Scale loss for number of replicas (callable-loss case). In this case,
-        # we have to be careful to call distribute_lib.get_loss_reduction()
-        # *after* loss() is evaluated, so we know what loss reduction it uses.
+        # Scale loss for number of replicas (callable-loss case).
         loss_value = self._scale_loss(loss_value, scale_loss_by_num_replicas)
 
       if var_list is None:
@@ -839,9 +834,6 @@ class OptimizerV2(optimizer_v1.Optimizer):
   @staticmethod
   def _scale_loss(loss_value, scale_loss_by_num_replicas):
     """Scale loss for the number of replicas."""
-    if scale_loss_by_num_replicas is None:
-      scale_loss_by_num_replicas = (
-          distribute_lib.get_loss_reduction() == ds_reduce_util.ReduceOp.MEAN)
     if scale_loss_by_num_replicas:
       num_replicas = distribute_ctx.get_strategy().num_replicas_in_sync
       if num_replicas > 1:
diff --git a/tensorflow/python/distribute/BUILD b/tensorflow/python/distribute/BUILD
index bc465e1fde..9bcdcce302 100644
--- a/tensorflow/python/distribute/BUILD
+++ b/tensorflow/python/distribute/BUILD
@@ -138,7 +138,6 @@ py_library(
         "//tensorflow/python:variable_scope",
         "//tensorflow/python/data",
         "//tensorflow/python/distribute/cluster_resolver:cluster_resolver_lib",
-        "//tensorflow/python/ops/losses",
         "//tensorflow/tools/docs:doc_controls",
     ],
 )
diff --git a/tensorflow/python/distribute/distribute_lib.py b/tensorflow/python/distribute/distribute_lib.py
index 2cc99b3db4..3e48364487 100644
--- a/tensorflow/python/distribute/distribute_lib.py
+++ b/tensorflow/python/distribute/distribute_lib.py
@@ -37,7 +37,6 @@ from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import custom_gradient
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variable_scope
-from tensorflow.python.ops.losses import losses_impl
 from tensorflow.python.platform import tf_logging
 from tensorflow.python.util import nest
 from tensorflow.python.util.tf_export import tf_export
@@ -79,14 +78,14 @@ class UpdateContext(object):
 # Public utility functions.
 
 
-@tf_export("distribute.get_loss_reduction")
+@tf_export(v1=["distribute.get_loss_reduction"])
 def get_loss_reduction():
-  """`tf.distribute.ReduceOp` corresponding to the last loss reduction."""
-  loss_reduction = ops.get_default_graph()._last_loss_reduction  # pylint: disable=protected-access
-  if (loss_reduction == losses_impl.Reduction.SUM or
-      loss_reduction == losses_impl.ReductionV2.SUM):
-    return reduce_util.ReduceOp.SUM
-  return reduce_util.ReduceOp.MEAN
+  """DEPRECATED: Now always returns `tf.distribute.ReduceOp.SUM`.
+
+  We now always make the complete adjustment when computing the loss, so
+  code should always add gradients/losses across replicas, never average.
+  """
+  return reduce_util.ReduceOp.SUM
 
 
 # ------------------------------------------------------------------------------
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index e74b43f93e..8cbdbec657 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -3048,9 +3048,6 @@ class Graph(object):
     # being called inside function definitions behave as if they were seeing the
     # actual outside graph).
     self._graph_key = "grap-key-%d/" % (uid(),)
-    # A string with the last reduction method passed to
-    # losses.compute_weighted_loss(), or None.
-    self._last_loss_reduction = None
     self._container = ""
     self._registered_ops = op_def_registry.get_registered_ops()
     # Set to True if this graph is being built in an
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 5ddf901f20..1c71ff9679 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -292,6 +292,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":backend",
+        "//tensorflow/python/distribute:distribute_lib",
     ],
 )
 
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index 91df7705d1..7df9c2f76b 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -23,7 +23,7 @@ import numpy as np
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.distribute import distribute_coordinator_context as dc_context
-from tensorflow.python.distribute import distribute_lib
+from tensorflow.python.distribute import reduce_util
 from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -104,7 +104,7 @@ def unwrap_values(distribution_strategy, grouped_inputs, grouped_outputs,
                                         grouped_inputs)
   if with_loss_tensor:
     # reduce loss tensor before adding it to the list of fetches
-    loss = distribution_strategy.reduce(distribute_lib.get_loss_reduction(),
+    loss = distribution_strategy.reduce(reduce_util.ReduceOp.SUM,
                                         grouped_outputs[0])
     all_outputs = flatten_perdevice_values(distribution_strategy,
                                            grouped_outputs[1:])
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 6412adc5a6..4a2a337821 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -23,7 +23,6 @@ import numpy as np
 
 from tensorflow.python.data.experimental.ops import batching
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import input_lib
 from tensorflow.python.distribute import reduce_util as ds_reduce_util
 from tensorflow.python.framework import constant_op
@@ -312,7 +311,7 @@ def experimental_tpu_fit_loop(model,
 
     for label, output in zip(out_labels, combined_fn.outputs):
       if label == 'loss':
-        reduce_op = distribute_lib.get_loss_reduction()
+        reduce_op = ds_reduce_util.ReduceOp.SUM
       else:
         # We reduce all other metrics using mean for now. This is temporary
         # workaround until new metrics are in place.
@@ -496,7 +495,7 @@ def experimental_tpu_test_loop(model,
 
     for label, output in zip(model.metrics_names, combined_fn.outputs):
       if label == 'loss':
-        reduce_op = distribute_lib.get_loss_reduction()
+        reduce_op = ds_reduce_util.ReduceOp.SUM
       else:
         # We reduce all other metrics using mean for now. This is temporary
         # workaround until new metrics are in place.
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
index bf6dcaad12..b701c98215 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
@@ -25,7 +25,6 @@ import functools
 
 import six
 
-from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import distribution_strategy_context as distribute_ctx
 from tensorflow.python.distribute import reduce_util as ds_reduce_util
 from tensorflow.python.distribute import values as distributed_values
@@ -126,7 +125,26 @@ class OptimizerV2(checkpointable.Checkpointable):
   opt.apply_gradients(capped_grads_and_vars)
   ```
 
+  ### Use with `tf.distribute.Strategy`.
+
+  This optimizer class is `tf.distribute.Strategy` aware, which means it
+  automatically sums gradients across all replicas. To average gradients,
+  you divide your loss by the global batch size, which is done automatically
+  if you use a member of `tf.keras.losses` or `tf.losses`. See the
+  `reduction` argument of your loss which should be set to
+  `tf.losses.Reduction.SUM_OVER_BATCH_SIZE` for averaging or
+  `tf.losses.Reduction.SUM` for not.
+
+  If you are not using these and you want to average gradients, you should use
+  `tf.math.reduce_sum` to add up your per-example losses and then divide by the
+  global batch size. Note that when using `tf.distribute.Strategy`, the first
+  component of a tensor's shape is the *replica-local* batch size, which is off
+  by a factor equal to the number of replicas being used to compute a single
+  step. As a result, using `tf.math.reduce_mean` will give the wrong answer,
+  resulting in gradients that can be many times too big.
+
   ### Variable Constraint
+
   All Keras optimizers respect variable constraints. If constraint function is
   passed to any variable, the constraint will be applied to the variable after
   the gradient has been applied to the variable.
@@ -174,15 +192,13 @@ class OptimizerV2(checkpointable.Checkpointable):
   ```
 
   ### Write a customized optimizer.
+  If you intend to create your own optimization algorithm, simply inherit from
+  this class and override the following methods:
 
-  This optimizer class updates variables from gradients and is
-  tf.distribute.Strategy aware. If you intend to create your own optimization
-  algorithm, simply inherit from this class and override the following methods:
     - resource_apply_dense (update variable given gradient tensor is dense)
     - resource_apply_sparse (update variable given gradient tensor is sparse)
     - create_slots (if your optimizer algorithm requires additional variables)
     - get_config (serialization of the optimizer, include all hyper parameters)
-
   """
 
   def __init__(self, name, **kwargs):
@@ -310,7 +326,6 @@ class OptimizerV2(checkpointable.Checkpointable):
     with backprop.GradientTape() as tape:
       tape.watch(var_list)
       loss_value = loss()
-      loss_value = self._scale_loss(loss_value)
     grads = tape.gradient(loss_value, var_list, grad_loss)
 
     if hasattr(self, "clipnorm"):
@@ -329,14 +344,6 @@ class OptimizerV2(checkpointable.Checkpointable):
 
     return grads_and_vars
 
-  @staticmethod
-  def _scale_loss(loss_value):
-    if distribute_lib.get_loss_reduction() == ds_reduce_util.ReduceOp.MEAN:
-      num_replicas = distribute_ctx.get_strategy().num_replicas_in_sync
-      if num_replicas > 1:
-        loss_value *= (1. / num_replicas)
-    return loss_value
-
   def get_gradients(self, loss, params):
     """Returns gradients of `loss` with respect to `params`.
 
@@ -351,7 +358,6 @@ class OptimizerV2(checkpointable.Checkpointable):
       ValueError: In case any gradient cannot be computed (e.g. if gradient
         function not implemented).
     """
-    loss = self._scale_loss(loss)
     grads = gradients.gradients(loss, params)
     if None in grads:
       raise ValueError("An operation has `None` for gradient. "
diff --git a/tensorflow/python/keras/utils/losses_utils.py b/tensorflow/python/keras/utils/losses_utils.py
index d42b354fb1..899780e53b 100644
--- a/tensorflow/python/keras/utils/losses_utils.py
+++ b/tensorflow/python/keras/utils/losses_utils.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.keras import backend as K
@@ -148,7 +149,9 @@ def _reduce_weighted_loss(
   else:
     loss = math_ops.reduce_sum(weighted_losses)
     if reduction == losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE:
-      loss = _safe_mean(loss, _num_elements(weighted_losses))
+      num_replicas = (  # Used to convert from local to global batch size.
+          distribution_strategy_context.get_strategy().num_replicas_in_sync)
+      loss = _safe_mean(loss, num_replicas * _num_elements(weighted_losses))
   return loss
 
 
@@ -177,11 +180,6 @@ def compute_weighted_loss(losses,
   if sample_weight is None:
     sample_weight = 1.0
   with ops.name_scope(name, 'weighted_loss', (losses, sample_weight)):
-    # Save the `reduction` argument for loss normalization when distributing
-    # to multiple replicas.
-    # TODO(josh11b): Associate it with the returned op for more precision.
-    ops.get_default_graph()._last_loss_reduction = reduction  # pylint: disable=protected-access
-
     # Update dimensions of `sample_weight` to match with `losses` if possible.
     losses, _, sample_weight = squeeze_or_expand_dimensions(
         losses, None, sample_weight)
diff --git a/tensorflow/python/ops/losses/BUILD b/tensorflow/python/ops/losses/BUILD
index 4aea0265a7..9155d890de 100644
--- a/tensorflow/python/ops/losses/BUILD
+++ b/tensorflow/python/ops/losses/BUILD
@@ -29,6 +29,7 @@ py_library(
         "//tensorflow/python:platform",
         "//tensorflow/python:util",
         "//tensorflow/python:weights_broadcast_ops",
+        "//tensorflow/python/distribute:distribute_lib",
     ],
 )
 
diff --git a/tensorflow/python/ops/losses/losses_impl.py b/tensorflow/python/ops/losses/losses_impl.py
index 3393e75335..1169c452cc 100644
--- a/tensorflow/python/ops/losses/losses_impl.py
+++ b/tensorflow/python/ops/losses/losses_impl.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -43,6 +44,8 @@ class ReductionV2(object):
   * `NONE`: Un-reduced weighted losses with the same shape as input.
   * `SUM`: Scalar sum of weighted losses.
   * `SUM_OVER_BATCH_SIZE`: Scalar `SUM` divided by number of elements in losses.
+     Note that when using `tf.distribute.Strategy`, this is the global batch
+     size across all the replicas that are contributing to a single step.
   """
 
   NONE = "none"
@@ -69,8 +72,13 @@ class Reduction(object):
   * `SUM`: Scalar sum of weighted losses.
   * `MEAN`: Scalar `SUM` divided by sum of weights. DEPRECATED.
   * `SUM_OVER_BATCH_SIZE`: Scalar `SUM` divided by number of elements in losses.
+     Note that when using `tf.distribute.Strategy`, this is the global batch
+     size across all the replicas that are contributing to a single step.
   * `SUM_OVER_NONZERO_WEIGHTS`: Scalar `SUM` divided by number of non-zero
      weights. DEPRECATED.
+     Note that when using `tf.distribute.Strategy`, this is scaled by the
+     number of replicas that are contributing to a single step to get an
+     approximation to the global batch size.
   * `SUM_BY_NONZERO_WEIGHTS`: Same as `SUM_OVER_NONZERO_WEIGHTS`.
   """
 
@@ -198,11 +206,6 @@ def compute_weighted_loss(
   """
   Reduction.validate(reduction)
   with ops.name_scope(scope, "weighted_loss", (losses, weights)):
-    # Save the `reduction` argument for loss normalization when distributing
-    # to multiple replicas.
-    # TODO(josh11b): Associate it with the returned op for more precision.
-    ops.get_default_graph()._last_loss_reduction = reduction  # pylint: disable=protected-access
-
     with ops.control_dependencies((
         weights_broadcast_ops.assert_broadcastable(weights, losses),)):
       losses = ops.convert_to_tensor(losses)
@@ -214,15 +217,17 @@ def compute_weighted_loss(
         loss = weighted_losses
       else:
         loss = math_ops.reduce_sum(weighted_losses)
+        num_replicas = (  # Used to convert from local to global batch size.
+            distribution_strategy_context.get_strategy().num_replicas_in_sync)
         if reduction == Reduction.MEAN:
-          loss = _safe_mean(
-              loss,
-              math_ops.reduce_sum(array_ops.ones_like(losses) * weights))
+          denom = (num_replicas *
+                   math_ops.reduce_sum(array_ops.ones_like(losses) * weights))
+          loss = _safe_mean(loss, denom)
         elif (reduction == Reduction.SUM_BY_NONZERO_WEIGHTS or
               reduction == Reduction.SUM_OVER_NONZERO_WEIGHTS):
-          loss = _safe_mean(loss, _num_present(losses, weights))
+          loss = _safe_mean(loss, num_replicas * _num_present(losses, weights))
         elif reduction == Reduction.SUM_OVER_BATCH_SIZE:
-          loss = _safe_mean(loss, _num_elements(losses))
+          loss = _safe_mean(loss, num_replicas * _num_elements(losses))
 
       # Convert the result back to the input type.
       loss = math_ops.cast(loss, input_dtype)
diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py
index 3742ebb807..a98fcc2dce 100644
--- a/tensorflow/python/training/optimizer.py
+++ b/tensorflow/python/training/optimizer.py
@@ -24,7 +24,6 @@ import abc
 
 import six
 
-from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import distribution_strategy_context as distribute_ctx
 from tensorflow.python.distribute import reduce_util as ds_reduce_util
 from tensorflow.python.eager import backprop
@@ -461,12 +460,6 @@ class Optimizer(
           tape.watch(var_list)
         loss_value = loss()
 
-        # Scale loss if using a "mean" loss reduction and multiple replicas.
-        # Have to be careful to call distribute_lib.get_loss_reduction()
-        # *after* loss() is evaluated, so we know what loss reduction it uses.
-        # TODO(josh11b): Test that we handle weight decay in a reasonable way.
-        loss_value = self._scale_loss(loss_value)
-
       if var_list is None:
         var_list = tape.watched_variables()
       # TODO(jhseu): Figure out why GradientTape's gradients don't require loss
@@ -481,9 +474,6 @@ class Optimizer(
           "`loss` passed to Optimizer.compute_gradients should "
           "be a function when eager execution is enabled.")
 
-    # Scale loss if using a "mean" loss reduction and multiple replicas.
-    loss = self._scale_loss(loss)
-
     if gate_gradients not in [Optimizer.GATE_NONE, Optimizer.GATE_OP,
                               Optimizer.GATE_GRAPH]:
       raise ValueError("gate_gradients must be one of: Optimizer.GATE_NONE, "
@@ -518,14 +508,6 @@ class Optimizer(
          if g is not None and v.dtype != dtypes.resource])
     return grads_and_vars
 
-  @staticmethod
-  def _scale_loss(loss_value):
-    if distribute_lib.get_loss_reduction() == ds_reduce_util.ReduceOp.MEAN:
-      num_replicas = distribute_ctx.get_strategy().num_replicas_in_sync
-      if num_replicas > 1:
-        loss_value *= (1. / num_replicas)
-    return loss_value
-
   def apply_gradients(self, grads_and_vars, global_step=None, name=None):
     """Apply gradients to variables.
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.pbtxt
index 5b5c9e2df1..0d0d35043c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.pbtxt
@@ -36,10 +36,6 @@ tf_module {
     name: "cluster_resolver"
     mtype: "<type \'module\'>"
   }
-  member_method {
-    name: "get_loss_reduction"
-    argspec: "args=[], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "get_replica_context"
     argspec: "args=[], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 3e147d5478..8f83448160 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -150,6 +150,7 @@ renames = {
     'tf.disable_v2_batch_normalization': 'tf.compat.v1.disable_v2_batch_normalization',
     'tf.disable_v2_behavior': 'tf.compat.v1.disable_v2_behavior',
     'tf.disable_v2_tensorshape': 'tf.compat.v1.disable_v2_tensorshape',
+    'tf.distribute.get_loss_reduction': 'tf.compat.v1.distribute.get_loss_reduction',
     'tf.distributions.Bernoulli': 'tf.compat.v1.distributions.Bernoulli',
     'tf.distributions.Beta': 'tf.compat.v1.distributions.Beta',
     'tf.distributions.Categorical': 'tf.compat.v1.distributions.Categorical',
-- 
GitLab


From 7bc89d4bab63b9fe2c1477bb8cc63504ea0fdc72 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 4 Feb 2019 17:55:52 -0800
Subject: [PATCH 1893/2345] Remove deprecated method: WriteToTensor

PiperOrigin-RevId: 232401517
---
 tensorflow/lite/string_util.h | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/tensorflow/lite/string_util.h b/tensorflow/lite/string_util.h
index f076db76f2..cb268ee805 100644
--- a/tensorflow/lite/string_util.h
+++ b/tensorflow/lite/string_util.h
@@ -35,7 +35,7 @@ limitations under the License.
 //   buf.AddString("AB", 2);
 //   # Write content of DynamicBuffer to tensor in format of string tensor
 //   # described above.
-//   buf.WriteToTensor(tensor)
+//   buf.WriteToTensor(tensor, nullptr)
 
 #ifndef TENSORFLOW_LITE_STRING_UTIL_H_
 #define TENSORFLOW_LITE_STRING_UTIL_H_
@@ -83,10 +83,6 @@ class DynamicBuffer {
   // Fill content into a string tensor. Set shape to {num_strings}.
   void WriteToTensorAsVector(TfLiteTensor* tensor);
 
-  // Deprecated. Use WriteToTensorAsVector() or pass in the new shpe.
-  // TODO(b/120230709): remove when people migrate away.
-  void WriteToTensor(TfLiteTensor* tensor) { WriteToTensorAsVector(tensor); }
-
  private:
   // Data buffer to store contents of strings, not including headers.
   std::vector<char> data_;
-- 
GitLab


From 2f070c4b5ba705e2c565602ee55d193428daea72 Mon Sep 17 00:00:00 2001
From: nammbash <niroopshankar.ammbashankar@intel.com>
Date: Mon, 4 Feb 2019 18:12:01 -0800
Subject: [PATCH 1894/2345] review comments changes

---
 ...pi_def_RequantizationRangePerChannel.pbtxt |  2 +-
 .../api_def_RequantizePerChannel.pbtxt        |  3 +-
 ...mkl_requantization_range_per_channel_op.cc |  9 +++--
 .../core/kernels/mkl_requantize_ops_test.cc   | 36 +++++++++----------
 .../kernels/mkl_requantize_per_channel_op.cc  | 23 ++++++------
 5 files changed, 37 insertions(+), 36 deletions(-)

diff --git a/tensorflow/core/api_def/base_api/api_def_RequantizationRangePerChannel.pbtxt b/tensorflow/core/api_def/base_api/api_def_RequantizationRangePerChannel.pbtxt
index bf80a35691..2dfde04721 100644
--- a/tensorflow/core/api_def/base_api/api_def_RequantizationRangePerChannel.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_RequantizationRangePerChannel.pbtxt
@@ -40,7 +40,7 @@ END
   attr {
     name: "clip_value_max"
     description: <<END
-The maximum value of the output that needs to to be clipped.
+The maximum value of the output that needs to be clipped.
 Example: set this to 6 for Relu6. 
 END
   }
diff --git a/tensorflow/core/api_def/base_api/api_def_RequantizePerChannel.pbtxt b/tensorflow/core/api_def/base_api/api_def_RequantizePerChannel.pbtxt
index e46bc77e7a..d62736c2bc 100644
--- a/tensorflow/core/api_def/base_api/api_def_RequantizePerChannel.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_RequantizePerChannel.pbtxt
@@ -58,8 +58,7 @@ END
   attr {
     name: "out_type"
     description: <<END
-The quantized type of output tensor that needs to be converted. 
-input tensor.
+The quantized type of output tensor that needs to be converted.
 END
   }
   summary: "Requantizes input with min and max values known per channel."
diff --git a/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc b/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc
index 37d6bb1658..a325576d60 100644
--- a/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc
+++ b/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc
@@ -64,12 +64,11 @@ class MklRequantizationRangePerChannelOp : public OpKernel {
     auto input_matrix = input.flat_inner_dims<qint32>();
 
     // TODO: verify performance of not transposing and finding the min max
-    // directly
-    // from input_matrix vs the one presentd below of transposing and using the
-    // transposed matrix as the transpose operation might be more costly.
+    // directly from input_matrix vs the one presented below of transposing and
+    // using the transposed matrix as the transposing operation in itself might
+    // be more costly.
     // Note that this operation is a calibration step for quantization and will
-    // cease
-    // to exist in the final inference graph as it will exist as a const node.
+    // cease to exist in the final inference graph(will exist as a const node).
     auto transposed_input = input_matrix.shuffle(shuffling);
 
     // Find the ranges of each channel in parallel.
diff --git a/tensorflow/core/kernels/mkl_requantize_ops_test.cc b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
index 83577ffb68..178634860e 100644
--- a/tensorflow/core/kernels/mkl_requantize_ops_test.cc
+++ b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
@@ -53,7 +53,7 @@ void MklRequantizatedOpsTestHelper::Setup(Tensor &input_tensor_qint32,
 
   // Step 2: Quantization details (per channel)
   // ------------------------------------------
-  // T and W are quantized using a quantize Op.
+  // T and W are quantized using a quantize op.
   // The input tensor T (NHWC) is quantized to unsigned int8.
   // Hence T's max value is mapped to ((2^8-1) = 255).
   // The input filter W (NHWC) is quantized to signed int8.
@@ -79,7 +79,7 @@ void MklRequantizatedOpsTestHelper::Setup(Tensor &input_tensor_qint32,
   // orig T range in FP32[0, 5.0] * [-2.0, 2.0] is [-663110.59, 663110.59]
 
   // The range of the convolved T*W2 is (2^31-1) * 5.0/255 * 3.0/127 = 994665.88
-  // So the range of convolved T*W1 in int32(-2^31, 2^31) that maps to
+  // So the range of convolved T*W2 in int32(-2^31, 2^31) that maps to
   // orig T range in FP32 [0, 5.0] * [-3.0, 3.0]  is [-994665.88, 994665.88]
 
   // Step 4: Assumption output above is fed to requantization_range_perchannel
@@ -106,7 +106,7 @@ void MklRequantizatedOpsTestHelper::Setup(Tensor &input_tensor_qint32,
   // test RequantizationRangePerChannelTest_ClipMax
 }
 
-// Tests the RequantizationRangePerChannel Op wherein the range
+// Tests the RequantizationRangePerChannel op wherein the range
 // of the weights is calculated per channel.
 TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_Basic) {
   // Let us set up the tensor and inputs before we run this op.
@@ -123,13 +123,13 @@ TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_Basic) {
   Tensor input_tensor_qint32(DT_QINT32,
                              {1, input_height, input_width, input_channels});
 
-  // Explanation and setup prior to this Op. Fill T and populate range values.
+  // Explanation and setup prior to this op. Fill T and populate range values.
   MklRequantizatedOpsTestHelper helper;
   helper.Setup(input_tensor_qint32, range_weights_ch1, range_weights_ch2);
 
   // Step 5: Define and run requantization_range_perchannel
   // -------------------------------------------------------
-  // Define, create and initialize the OP in question.
+  // Define, create and initialize the op in question.
   TF_ASSERT_OK(NodeDefBuilder("requantization_range_per_channel",
                               "RequantizationRangePerChannel")
                    .Input(FakeInput(DT_QINT32))
@@ -140,17 +140,17 @@ TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_Basic) {
                    .Finalize(node_def()));
   TF_ASSERT_OK(InitOp());
 
-  // Add the Input Nodes to the Op.
+  // Add the input nodes to the op.
   AddInputFromArray<qint32>(input_tensor_qint32.shape(),
                             input_tensor_qint32.flat<qint32>());
 
-  // Calculate the Min and max from the ranges
+  // Calculate the min and max from the ranges
   float ch1_min = -range_weights_ch1;
   float ch1_max = range_weights_ch1;
   float ch2_min = -range_weights_ch2;
   float ch2_max = range_weights_ch2;
 
-  // Add the perchannel range Nodes to the Op.
+  // Add the perchannel range Nodes to the op.
   AddInputFromArray<float>(TensorShape({input_channels}), {ch1_min, ch2_min});
   AddInputFromArray<float>(TensorShape({input_channels}), {ch1_max, ch2_max});
 
@@ -184,13 +184,13 @@ TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_ClipMax) {
   Tensor input_tensor_qint32(DT_QINT32,
                              {1, input_height, input_width, input_channels});
 
-  // Explanation and setup prior to this Op. Fill T and populate range values.
+  // Explanation and setup prior to this op. Fill T and populate range values.
   MklRequantizatedOpsTestHelper helper;
   helper.Setup(input_tensor_qint32, range_weights_ch1, range_weights_ch2);
 
   // Step 5: Define and run requantization_range_perchannel
   // -------------------------------------------------------
-  // Define, create and initalize the OP in question.
+  // Define, create and initalize the op in question.
   TF_ASSERT_OK(NodeDefBuilder("requantization_range_per_channel",
                               "RequantizationRangePerChannel")
                    .Input(FakeInput(DT_QINT32))
@@ -201,7 +201,7 @@ TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_ClipMax) {
                    .Finalize(node_def()));
   TF_ASSERT_OK(InitOp());
 
-  // Add the Input Nodes to the Op.
+  // Add the input nodes to the op.
   AddInputFromArray<qint32>(input_tensor_qint32.shape(),
                             input_tensor_qint32.flat<qint32>());
 
@@ -211,7 +211,7 @@ TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_ClipMax) {
   float ch2_min = -range_weights_ch2;
   float ch2_max = range_weights_ch2;
 
-  // Add the perchannel range nodes to the Op.
+  // Add the perchannel range nodes to the op.
   AddInputFromArray<float>(TensorShape({input_channels}), {ch1_min, ch2_min});
   AddInputFromArray<float>(TensorShape({input_channels}), {ch1_max, ch2_max});
 
@@ -224,8 +224,8 @@ TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_ClipMax) {
   // Verify the expected outputs
   const float output_min = GetOutput(0)->flat<float>()(0);
   const float output_max = GetOutput(1)->flat<float>()(0);
-  EXPECT_NEAR(-6.0, output_min, 0.002);  // Values are Max as with clip_value
-  EXPECT_NEAR(6.0, output_max, 0.002);   // Values are Max as with clip_value
+  EXPECT_NEAR(-6.0, output_min, 0.002);  // Values are aligned with clip_value.
+  EXPECT_NEAR(6.0, output_max, 0.002);   // Values are aligned with clip_value.
 }
 
 TEST_F(MklRequantizatedOpsTest, RequantizePerChannelTest_Basic) {
@@ -242,7 +242,7 @@ TEST_F(MklRequantizatedOpsTest, RequantizePerChannelTest_Basic) {
   Tensor input_tensor_qint32(DT_QINT32,
                              {1, input_height, input_width, input_channels});
 
-  // Explanation and setup prior to this Op. Fill T and populate range values.
+  // Explanation and setup prior to this op. Fill T and populate range values.
   MklRequantizatedOpsTestHelper helper;
   helper.Setup(input_tensor_qint32, range_weights_ch1, range_weights_ch2);
 
@@ -250,7 +250,7 @@ TEST_F(MklRequantizatedOpsTest, RequantizePerChannelTest_Basic) {
   // --------------------------------------------
   // The output of requantization_range_op_per_channel which calculated the
   // new ranges of int8 is fed to the requantize per channel op.
-  // Here the Values of Convolved T*W is converted from int32 to int8.
+  // Here the values of convolved T*W is converted from int32 to int8.
 
   TF_ASSERT_OK(NodeDefBuilder("requantize_per_channel", "RequantizePerChannel")
                    .Input(FakeInput(DT_QINT32))
@@ -263,7 +263,7 @@ TEST_F(MklRequantizatedOpsTest, RequantizePerChannelTest_Basic) {
                    .Finalize(node_def()));
   TF_ASSERT_OK(InitOp());
 
-  // Add the input Nodes to the Op.
+  // Add the input Nodes to the op.
   AddInputFromArray<qint32>(input_tensor_qint32.shape(),
                             input_tensor_qint32.flat<qint32>());
 
@@ -273,7 +273,7 @@ TEST_F(MklRequantizatedOpsTest, RequantizePerChannelTest_Basic) {
   float ch2_min = -range_weights_ch2;
   float ch2_max = range_weights_ch2;
 
-  // Add the perchannel range nodes to the Op.
+  // Add the perchannel range nodes to the op.
   AddInputFromArray<float>(TensorShape({input_channels}), {ch1_min, ch2_min});
   AddInputFromArray<float>(TensorShape({input_channels}), {ch1_max, ch2_max});
 
diff --git a/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc b/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc
index 16940ff9bb..1f5dd5bcf5 100644
--- a/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc
+++ b/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc
@@ -65,18 +65,19 @@ class MklRequantizePerChannelOp : public OpKernel {
       size_t depth = input_min_vec.NumElements();
       OP_REQUIRES(
           ctx, input_min_vec.dim_size(0) == depth,
-          errors::InvalidArgument("input_min has incorrect size, expected ", depth,
-                                  " was ", input_min_vec.dim_size(0)));
+          errors::InvalidArgument("input_min has incorrect size, expected ",
+                                  depth, " was ", input_min_vec.dim_size(0)));
       OP_REQUIRES(
           ctx, input_max_vec.dim_size(0) == depth,
-          errors::InvalidArgument("input_max has incorrect size, expected ", depth,
-                                  " was ", input_max_vec.dim_size(0)));
+          errors::InvalidArgument("input_max has incorrect size, expected ",
+                                  depth, " was ", input_max_vec.dim_size(0)));
 
       if (out_type_ == DT_QINT8) DCHECK(input_requested_min_float < 0.0f);
 
       const float factor = (out_type_ == DT_QINT8) ? 127.0f : 255.0f;
-      float requested_min_max = std::max(std::abs(input_requested_min_float),
-                                         std::abs(input_requested_max_float));
+      const float requested_min_max =
+          std::max(std::abs(input_requested_min_float),
+                   std::abs(input_requested_max_float));
       Tensor* output = nullptr;
       OP_REQUIRES_OK(ctx, ctx->allocate_output(kOutputTensorIndex,
                                                input.shape(), &output));
@@ -85,8 +86,8 @@ class MklRequantizePerChannelOp : public OpKernel {
       for (int i = 0; i < depth; ++i) {
         float min_max_from_vec = std::max(std::abs(input_min_vec_data[i]),
                                           std::abs(input_max_vec_data[i]));
-        scales[i] =
-            factor * (min_max_from_vec / requested_min_max / static_cast<float>(1L << 31));
+        scales[i] = factor * (min_max_from_vec / requested_min_max /
+                              static_cast<float>(1L << 31));
       }
 
       mkldnn::primitive_attr reorder_attr;
@@ -132,8 +133,10 @@ class MklRequantizePerChannelOp : public OpKernel {
 
       Tensor* output_min = nullptr;
       Tensor* output_max = nullptr;
-      OP_REQUIRES_OK(ctx, ctx->allocate_output(kOutputMinIndex, {}, &output_min));
-      OP_REQUIRES_OK(ctx, ctx->allocate_output(kOutputMaxIndex, {}, &output_max));
+      OP_REQUIRES_OK(ctx,
+                     ctx->allocate_output(kOutputMinIndex, {}, &output_min));
+      OP_REQUIRES_OK(ctx,
+                     ctx->allocate_output(kOutputMaxIndex, {}, &output_max));
 
       output_min->flat<float>()(0) = input_requested_min_float;
       output_max->flat<float>()(0) = input_requested_max_float;
-- 
GitLab


From 0507dba17e223053f7d7f19e8e546ca6a97b1d8b Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Mon, 4 Feb 2019 18:06:20 -0800
Subject: [PATCH 1895/2345] Automated rollback of commit
 f1f1336aef572eb2073c9a488f8408b6404a700b. Revert #25330.

PiperOrigin-RevId: 232403091
---
 tensorflow/core/grappler/op_types.cc          | 14 -------
 tensorflow/core/grappler/op_types.h           |  4 --
 .../optimizers/arithmetic_optimizer.cc        | 22 +---------
 .../optimizers/arithmetic_optimizer_test.cc   | 41 -------------------
 4 files changed, 2 insertions(+), 79 deletions(-)

diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc
index e8179b22a8..e188c15b59 100644
--- a/tensorflow/core/grappler/op_types.cc
+++ b/tensorflow/core/grappler/op_types.cc
@@ -47,24 +47,10 @@ bool IsAnyDiv(const NodeDef& node) {
          node.op() == "FloorDiv" || node.op() == "TruncateDiv";
 }
 
-bool IsAnyMax(const NodeDef& node) {
-  const auto& op = node.op();
-  return op == "Max" || op == "SegmentMax" || op == "UnsortedSegmentMax";
-}
-
-bool IsAnyMin(const NodeDef& node) {
-  const auto& op = node.op();
-  return op == "Min" || op == "SegmentMin" || op == "UnsortedSegmentMin";
-}
-
 bool IsApproximateEqual(const NodeDef& node) {
   return node.op() == "ApproximateEqual";
 }
 
-bool IsArgMax(const NodeDef& node) { return node.op() == "ArgMax"; }
-
-bool IsArgMin(const NodeDef& node) { return node.op() == "ArgMin"; }
-
 bool IsAvgPoolGrad(const NodeDef& node) { return node.op() == "AvgPoolGrad"; }
 
 bool IsAssign(const NodeDef& node) {
diff --git a/tensorflow/core/grappler/op_types.h b/tensorflow/core/grappler/op_types.h
index fc32749d9f..a1ee253369 100644
--- a/tensorflow/core/grappler/op_types.h
+++ b/tensorflow/core/grappler/op_types.h
@@ -28,11 +28,7 @@ bool IsAll(const NodeDef& node);
 bool IsAngle(const NodeDef& node);
 bool IsAny(const NodeDef& node);
 bool IsAnyDiv(const NodeDef& node);
-bool IsAnyMax(const NodeDef& node);
-bool IsAnyMin(const NodeDef& node);
 bool IsApproximateEqual(const NodeDef& node);
-bool IsArgMax(const NodeDef& node);
-bool IsArgMin(const NodeDef& node);
 bool IsAvgPoolGrad(const NodeDef& node);
 bool IsAssert(const NodeDef& node);
 bool IsAssign(const NodeDef& node);
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index e2c8b7cdbc..db7257156e 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -52,7 +52,6 @@ limitations under the License.
 #include "tensorflow/core/util/saved_tensor_slice_util.h"
 #include "tensorflow/core/util/strided_slice_op.h"
 
-using tensorflow::str_util::StringReplace;
 using tensorflow::strings::StrCat;
 
 namespace tensorflow {
@@ -2722,8 +2721,7 @@ class OptimizeMaxOrMinOfMonotonicStage : public ArithmeticOptimizerStage {
   ~OptimizeMaxOrMinOfMonotonicStage() override = default;
 
   bool IsSupported(const NodeDef* node) const override {
-    return IsAnyMax(*node) || IsAnyMin(*node) || IsArgMax(*node) ||
-           IsArgMin(*node);
+    return IsMax(*node) || IsMin(*node);
   }
 
   Status TrySimplify(NodeDef* reduction_node,
@@ -2754,15 +2752,9 @@ class OptimizeMaxOrMinOfMonotonicStage : public ArithmeticOptimizerStage {
       if (!is_non_decreasing) {
         // Flip Min<->Max if the function is non-increasing, e.g.
         // Max(Neg(x)) = Neg(Min(x)).
-        const string opposite = FlipMinMax(*reduction_node);
+        const string opposite = IsMax(*reduction_node) ? "Min" : "Max";
         reduction_node->set_op(opposite);
       }
-
-      if (IsArgMax(*reduction_node) || IsArgMin(*reduction_node)) {
-        // ArgMax(Sqrt(x)) = ArgMax(x)
-        inner_function->set_op("Identity");
-      }
-
       AddToOptimizationQueue(reduction_node);
       AddToOptimizationQueue(inner_function);
       AddToOptimizationQueue(inner_input);
@@ -2783,16 +2775,6 @@ class OptimizeMaxOrMinOfMonotonicStage : public ArithmeticOptimizerStage {
       AddToOptimizationQueue(consumer);
     }
   }
-
- private:
-  string FlipMinMax(const NodeDef& node) {
-    const string& op = node.op();
-    if (IsAnyMax(node) || IsArgMax(node)) {
-      return str_util::StringReplace(op, "Max", "Min", false);
-    } else {
-      return str_util::StringReplace(op, "Min", "Max", false);
-    }
-  }
 };
 
 // Replace a chain of type&shape preserving unary ops with a
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
index dd5ee909b6..94c59c68c8 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
@@ -3466,47 +3466,6 @@ TEST_F(ArithmeticOptimizerTest, OptimizeMaxOrMinOfMonotonicElementWise) {
   EXPECT_EQ(2, required_node_count);
 }
 
-TEST_F(ArithmeticOptimizerTest, OptimizeArgMaxOrArgMinOfMonotonicElementWise) {
-  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
-  const auto x = ops::Const(s.WithOpName("x"), {1.0f, 2.0f}, {1, 2});
-  Output sqrt = ops::Sqrt(s.WithOpName("sqrt"), x);
-  Output arg_max = ops::ArgMax(s.WithOpName("arg_max"), sqrt, 1);
-  Output final_out = ops::Identity(s.WithOpName("final_out"), arg_max);
-
-  GrapplerItem item;
-  item.fetch = {"final_out"};
-  TF_CHECK_OK(s.ToGraphDef(&item.graph));
-  const auto tensors_expected = EvaluateNodes(item.graph, item.fetch);
-  EXPECT_EQ(1, tensors_expected.size());
-
-  GraphDef output;
-  ArithmeticOptimizer optimizer;
-  EnableOnlyOptimizeMaxOrMinOfMonotonic(&optimizer);
-  OptimizeAndPrune(&optimizer, &item, &output);
-  const auto tensors = EvaluateNodes(output, item.fetch);
-  EXPECT_EQ(1, tensors.size());
-
-  test::ExpectTensorEqual<int64>(tensors_expected[0], tensors[0]);
-  EXPECT_EQ(item.graph.node_size() - 1, output.node_size());
-  // Check if the inputs are switched
-  int required_node_count = 0;
-  for (int i = 0; i < output.node_size(); ++i) {
-    const NodeDef& node = output.node(i);
-    if (node.name() == "final_out") {
-      EXPECT_EQ("Identity", node.op());
-      EXPECT_EQ(1, node.input_size());
-      EXPECT_EQ("arg_max", node.input(0));
-      ++required_node_count;
-    } else if (node.name() == "arg_max") {
-      EXPECT_EQ("ArgMax", node.op());
-      EXPECT_EQ(2, node.input_size());
-      EXPECT_EQ("x", node.input(0));
-      ++required_node_count;
-    }
-  }
-  EXPECT_EQ(2, required_node_count);
-}
-
 TEST_F(ArithmeticOptimizerTest,
        OptimizeMaxOrMinOfMonotonicElementWise_DoNotChangeFetchNode) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
-- 
GitLab


From 4a0583cce0a51531bd36f42a4885dcec634ba886 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 4 Feb 2019 18:18:56 -0800
Subject: [PATCH 1896/2345] Update ops-related pbtxt files.

PiperOrigin-RevId: 232404467
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 31 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 31 +++++++++++++++++++
 2 files changed, 62 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index c73ae0cc16..7e5b448f73 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -64552,6 +64552,37 @@ op {
     }
   }
 }
+op {
+  name: "ShardDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "num_shards"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "index"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "ShardedFilename"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 2b75389be9..c0ec6549d4 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -31722,6 +31722,37 @@ op {
     }
   }
 }
+op {
+  name: "ShardDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "num_shards"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "index"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "ShardedFilename"
   input_arg {
-- 
GitLab


From cf549e5f36d122e91c774bd7da44519c9f7792ff Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Mon, 4 Feb 2019 18:42:05 -0800
Subject: [PATCH 1897/2345] Update Softmax interface for fixed point int8
 Op/kernel. Use 1/256 as scale and -128 as zero point for output.

PiperOrigin-RevId: 232406765
---
 tensorflow/lite/kernels/activations.cc        |  2 +-
 .../lite/tools/optimize/subgraph_quantizer.cc | 24 ++++++++++++++++++-
 .../lite/tools/optimize/subgraph_quantizer.h  |  6 +++++
 .../tools/optimize/subgraph_quantizer_test.cc | 11 +++++++--
 4 files changed, 39 insertions(+), 4 deletions(-)

diff --git a/tensorflow/lite/kernels/activations.cc b/tensorflow/lite/kernels/activations.cc
index 4463a6c5a6..2b35cc4915 100644
--- a/tensorflow/lite/kernels/activations.cc
+++ b/tensorflow/lite/kernels/activations.cc
@@ -60,9 +60,9 @@ namespace {
 TfLiteStatus CheckOutputQuantParams(TfLiteContext* context,
                                     const TfLiteTensor* input,
                                     const TfLiteTensor* output) {
+  TF_LITE_ENSURE(context, output->params.scale == 1. / 256);
   if (input->type == kTfLiteUInt8) {
     TF_LITE_ENSURE_EQ(context, output->params.zero_point, 0);
-    TF_LITE_ENSURE(context, output->params.scale == 1. / 256);
   } else {
     TF_LITE_ENSURE_EQ(context, output->params.zero_point, -128);
   }
diff --git a/tensorflow/lite/tools/optimize/subgraph_quantizer.cc b/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
index 118e055058..c1ff444d78 100644
--- a/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
+++ b/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
@@ -325,6 +325,27 @@ TfLiteStatus SubgraphQuantizer::AsymmetricQuantizeSingleInputOutputOp(
   return kTfLiteOk;
 }
 
+TfLiteStatus SubgraphQuantizer::AsymmetricQuantizeSoftmax(
+    BuiltinOperator op_code, OperatorT* op) {
+  TF_LITE_ENSURE_EQ(this->error_reporter_, op->inputs.size(), 1);
+  TF_LITE_ENSURE_EQ(this->error_reporter_, op->outputs.size(), 1);
+
+  if (IsSubgraphInput(op->inputs[0])) {
+    TF_LITE_ENSURE_STATUS(AsymmetricQuantizeTensor(op_code, op->inputs[0]));
+  }
+
+  auto output_tensor = subgraph_->tensors[op->outputs[0]].get();
+  if (output_tensor->type != TensorType_FLOAT32) {
+    return kTfLiteOk;
+  }
+
+  // Softmax output is hardcoded to have 1/256 as scale and -128 as zero point.
+  output_tensor->type = TensorType_INT8;
+  output_tensor->quantization->scale = {1.0f / 256.0f};
+  output_tensor->quantization->zero_point = {-128};
+  return kTfLiteOk;
+}
+
 bool SubgraphQuantizer::IsSubgraphInput(int32_t tensor_idx) const {
   return std::find(subgraph_->inputs.begin(), subgraph_->inputs.end(),
                    tensor_idx) != subgraph_->inputs.end();
@@ -342,8 +363,9 @@ TfLiteStatus SubgraphQuantizer::QuantizeOperator(int op_idx) {
     case BuiltinOperator_MAX_POOL_2D:
       return PropagateMinMaxForAvgAndMaxPool(op_code, op);
     case BuiltinOperator_SQUEEZE:
-    case BuiltinOperator_SOFTMAX:
       return AsymmetricQuantizeSingleInputOutputOp(op_code, op);
+    case BuiltinOperator_SOFTMAX:
+      return AsymmetricQuantizeSoftmax(op_code, op);
     default:
       return kTfLiteError;
   }
diff --git a/tensorflow/lite/tools/optimize/subgraph_quantizer.h b/tensorflow/lite/tools/optimize/subgraph_quantizer.h
index 9d6ca7fad5..fd1c39253c 100644
--- a/tensorflow/lite/tools/optimize/subgraph_quantizer.h
+++ b/tensorflow/lite/tools/optimize/subgraph_quantizer.h
@@ -51,6 +51,12 @@ class SubgraphQuantizer {
   TfLiteStatus AsymmetricQuantizeSingleInputOutputOp(BuiltinOperator op_code,
                                                      OperatorT* op);
 
+  // Asymmetric quantizes inputs and outputs of an Softmax Op.
+  // Input is quantized with the min-max range and output is hardcoded to have
+  // 1/256 as scale and -128 as zero point.
+  TfLiteStatus AsymmetricQuantizeSoftmax(BuiltinOperator op_code,
+                                         OperatorT* op);
+
   TfLiteStatus AsymmetricQuantizeTensor(BuiltinOperator op_code,
                                         int32_t tensor_idx);
 
diff --git a/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc b/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
index 4b23ced0fb..7261d224c3 100644
--- a/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
+++ b/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
@@ -291,6 +291,7 @@ TEST(SubgraphQuantizerTest, VerifySoftmaxQuantization) {
   ASSERT_EQ(op->outputs.size(), 1);
   auto float_graph = readonly_model->subgraphs()->Get(0);
 
+  // Verify input.
   ASSERT_EQ(float_graph->tensors()->Get(op->inputs[0])->type(),
             TensorType_FLOAT32);
   ASSERT_EQ(float_graph->tensors()->Get(op->outputs[0])->type(),
@@ -306,12 +307,18 @@ TEST(SubgraphQuantizerTest, VerifySoftmaxQuantization) {
   VerifyAsymmetricQuantizationScale(*float_input_quant_params,
                                     *input_quant_params);
 
+  // Verify output.
   auto float_output_quant_params =
       float_graph->tensors()->Get(op->outputs[0])->quantization();
   auto output_quant_params =
       subgraph->tensors[op->outputs[0]]->quantization.get();
-  VerifyAsymmetricQuantizationScale(*float_output_quant_params,
-                                    *output_quant_params);
+  ASSERT_EQ(float_output_quant_params->min()->size(), 1);
+  ASSERT_EQ(float_output_quant_params->max()->size(), 1);
+
+  ASSERT_EQ(output_quant_params->scale.size(), 1);
+  ASSERT_EQ(output_quant_params->zero_point.size(), 1);
+  ASSERT_EQ(1.0f / 256.0f, output_quant_params->scale[0]);
+  ASSERT_EQ(-128, output_quant_params->zero_point[0]);
 }
 
 TEST(SubgraphQuantizerTest, VerifyAvgPoolQuantization) {
-- 
GitLab


From ea71fdda0a336a03efd2679cb0559f9d0c293bb9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 4 Feb 2019 18:46:17 -0800
Subject: [PATCH 1898/2345] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 232407120

---
 tensorflow/go/op/wrappers.go | 126 +++++++++++++++++++++--------------
 1 file changed, 75 insertions(+), 51 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 8d8d6474c2..cc0995fe10 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -9285,6 +9285,57 @@ func ReluGrad(scope *Scope, gradients tf.Output, features tf.Output) (backprops
 	return op.Output(0)
 }
 
+// TensorArrayGatherV2Attr is an optional argument to TensorArrayGatherV2.
+type TensorArrayGatherV2Attr func(optionalAttr)
+
+// TensorArrayGatherV2ElementShape sets the optional element_shape attribute to value.
+// If not specified, defaults to <unknown_rank:true >
+func TensorArrayGatherV2ElementShape(value tf.Shape) TensorArrayGatherV2Attr {
+	return func(m optionalAttr) {
+		m["element_shape"] = value
+	}
+}
+
+// Deprecated. Use TensorArrayGatherV3
+//
+// DEPRECATED at GraphDef version 26: Use TensorArrayGatherV3
+func TensorArrayGatherV2(scope *Scope, handle tf.Output, indices tf.Output, flow_in tf.Output, dtype tf.DataType, optional ...TensorArrayGatherV2Attr) (value tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtype": dtype}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorArrayGatherV2",
+		Input: []tf.Input{
+			handle, indices, flow_in,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Returns the truth value of (x == y) element-wise.
+//
+// *NOTE*: `Equal` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func Equal(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Equal",
+		Input: []tf.Input{
+			x, y,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Computes the gradient of morphological 2-D dilation with respect to the input.
 //
 // Arguments:
@@ -27888,6 +27939,30 @@ func LookupTableInsertV2(scope *Scope, table_handle tf.Output, keys tf.Output, v
 	return scope.AddOperation(opspec)
 }
 
+// Creates a `Dataset` that includes only 1/`num_shards` of this dataset.
+//
+// Arguments:
+//
+//	num_shards: An integer representing the number of shards operating in parallel.
+//	index: An integer representing the current worker index.
+//
+//
+func ShardDataset(scope *Scope, input_dataset tf.Output, num_shards tf.Output, index tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "ShardDataset",
+		Input: []tf.Input{
+			input_dataset, num_shards, index,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Creates a dataset that batches and pads `batch_size` elements from the input.
 //
 // Arguments:
@@ -34641,57 +34716,6 @@ func DynamicStitch(scope *Scope, indices []tf.Output, data []tf.Output) (merged
 	return op.Output(0)
 }
 
-// Returns the truth value of (x == y) element-wise.
-//
-// *NOTE*: `Equal` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func Equal(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Equal",
-		Input: []tf.Input{
-			x, y,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// TensorArrayGatherV2Attr is an optional argument to TensorArrayGatherV2.
-type TensorArrayGatherV2Attr func(optionalAttr)
-
-// TensorArrayGatherV2ElementShape sets the optional element_shape attribute to value.
-// If not specified, defaults to <unknown_rank:true >
-func TensorArrayGatherV2ElementShape(value tf.Shape) TensorArrayGatherV2Attr {
-	return func(m optionalAttr) {
-		m["element_shape"] = value
-	}
-}
-
-// Deprecated. Use TensorArrayGatherV3
-//
-// DEPRECATED at GraphDef version 26: Use TensorArrayGatherV3
-func TensorArrayGatherV2(scope *Scope, handle tf.Output, indices tf.Output, flow_in tf.Output, dtype tf.DataType, optional ...TensorArrayGatherV2Attr) (value tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtype": dtype}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "TensorArrayGatherV2",
-		Input: []tf.Input{
-			handle, indices, flow_in,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Interleave the values from the `data` tensors into a single tensor.
 //
 // Builds a merged tensor such that
-- 
GitLab


From d91c4f63a3db7a84b7bf7fc1f79d90c7a6c80a1b Mon Sep 17 00:00:00 2001
From: Amit <30853054+amitsrivastava78@users.noreply.github.com>
Date: Tue, 5 Feb 2019 09:05:38 +0530
Subject: [PATCH 1899/2345] Updated backend.py for the typo error

Fixed one typo error in backend.py
---
 tensorflow/python/keras/backend.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index 9c2b46ee24..f8a2cc09b4 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -1590,7 +1590,7 @@ def min(x, axis=None, keepdims=False):
           the reduced dimension is retained with length 1.
 
   Returns:
-      A tensor with miminum values of `x`.
+      A tensor with minimum values of `x`.
   """
   return math_ops.reduce_min(x, axis, keepdims)
 
-- 
GitLab


From 1659617e80e00a5eb0f0ed037c83ff272b2991ee Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 4 Feb 2019 20:00:49 -0800
Subject: [PATCH 1900/2345] Automated rollback of commit
 da5513d02ff833f85e9f466d38caa24e511c4600

PiperOrigin-RevId: 232413506
---
 .../compiler/jit/mark_for_compilation_pass.cc | 20 +----
 .../compiler/jit/partially_decluster_pass.cc  | 56 +++++++-------
 .../jit/partially_decluster_pass_test.cc      | 43 +----------
 tensorflow/compiler/tests/BUILD               | 16 ----
 .../compiler/tests/control_flow_ops_test.py   | 56 --------------
 tensorflow/compiler/tf2xla/graph_compiler.cc  | 68 ++++++-----------
 tensorflow/compiler/tf2xla/kernels/BUILD      |  1 -
 .../tf2xla/kernels/control_flow_ops.cc        | 74 -------------------
 8 files changed, 49 insertions(+), 285 deletions(-)
 delete mode 100644 tensorflow/compiler/tests/control_flow_ops_test.py
 delete mode 100644 tensorflow/compiler/tf2xla/kernels/control_flow_ops.cc

diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass.cc b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
index 0ad85087cd..20c2cd7e05 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
@@ -74,13 +74,6 @@ struct OperationFilter {
   // have dummy XLA implementations.
   bool allow_dummy_ops;
 
-  // Whether Switch/Merge ops are allowed.  We avoid auto-clustering these ops
-  // for now since they can result in mismatching deadness in the output
-  // produced by an XLA cluster, something the TF executor does not allow today.
-  // However, we plan to auto-cluster these in some limited cases in the near
-  // future (b/123096195).
-  bool allow_switch_merge_ops;
-
   // Whether ops that produce or consume DT_VARIANT values are allowed.  We
   // don't auto-cluster these ops because we don't yet support live-in or
   // live-out DT_VARIANT values.
@@ -262,10 +255,6 @@ bool IsCompilableCall(const NodeDef& call_def,
     if (!op_filter.allow_control_trigger && node->IsControlTrigger()) {
       return false;
     }
-    if (!op_filter.allow_switch_merge_ops &&
-        (node->IsSwitch() || node->IsMerge())) {
-      return false;
-    }
     if (!op_filter.allow_dummy_ops && IsDummyImplOp(node->type_string())) {
       return false;
     }
@@ -504,7 +493,6 @@ Status FindCompilationCandidates(
     op_filter.allow_resource_ops = registration->compile_resource_ops;
     op_filter.allow_stateful_rng_ops = always_auto_cluster;
     op_filter.allow_control_trigger = always_auto_cluster;
-    op_filter.allow_switch_merge_ops = always_auto_cluster;
     op_filter.allow_dummy_ops = always_auto_cluster;
     op_filter.allow_ops_producing_or_consuming_variant = always_auto_cluster;
 
@@ -525,11 +513,6 @@ Status FindCompilationCandidates(
       VLOG(2) << "Rejecting " << node->name() << ": is a control trigger op";
       continue;
     }
-    if (!op_filter.allow_switch_merge_ops &&
-        (node->IsSwitch() || node->IsMerge())) {
-      VLOG(2) << "Rejecting " << node->name() << ": is a switch or a merge op";
-      continue;
-    }
     if (!op_filter.allow_dummy_ops && IsDummyImplOp(node->type_string())) {
       VLOG(2) << "Rejecting " << node->name() << ": dummy op ("
               << node->type_string() << ")";
@@ -676,7 +659,6 @@ bool IsCompilable(FunctionLibraryRuntime* flr, const NodeDef& ndef) {
   op_filter.allow_resource_ops = true;
   op_filter.allow_stateful_rng_ops = true;
   op_filter.allow_control_trigger = true;
-  op_filter.allow_switch_merge_ops = true;
   op_filter.allow_dummy_ops = true;
   op_filter.allow_ops_producing_or_consuming_variant = true;
 
@@ -751,7 +733,7 @@ Status MarkForCompilationPass::Run(
     // deadness semantics of these nodes correctly and auto-clustering these
     // nodes can cause deadness to propagate to nodes that should be live.
     if (!deadness_analysis_disabled) {
-      if (!node->IsMerge() &&
+      if (node->IsMerge() ||
           deadness->HasInputsWithMismatchingDeadness(*node)) {
         VLOG(2) << "Rejecting " << node->name() << ": mismatching deadness.";
         return false;
diff --git a/tensorflow/compiler/jit/partially_decluster_pass.cc b/tensorflow/compiler/jit/partially_decluster_pass.cc
index dd3a122940..e1fd2aaee2 100644
--- a/tensorflow/compiler/jit/partially_decluster_pass.cc
+++ b/tensorflow/compiler/jit/partially_decluster_pass.cc
@@ -29,31 +29,6 @@ namespace {
 
 bool NotBackedge(const Edge& edge) { return !edge.src()->IsNextIteration(); }
 
-bool IsMustCompileDevice(const DeviceType& device_type) {
-  const XlaOpRegistry::DeviceRegistration* registration;
-  if (XlaOpRegistry::GetCompilationDevice(device_type.type(), &registration)) {
-    return registration->autoclustering_policy ==
-           XlaOpRegistry::AutoclusteringPolicy::kAlways;
-  }
-
-  return false;
-}
-
-Status MustCompileNode(const Node* n, bool* must_compile) {
-  DeviceType device_type("");
-  TF_RETURN_IF_ERROR(
-      DeviceToDeviceType(n->assigned_device_name(), &device_type));
-
-  if (IsMustCompileDevice(device_type)) {
-    *must_compile = true;
-    return Status::OK();
-  }
-
-  // We must compile `n` if it does not have a TensorFlow kernel.
-  *must_compile = !FindKernelDef(device_type, n->def(), nullptr, nullptr).ok();
-  return Status::OK();
-}
-
 namespace reduce_device_to_host_copies {
 Status FindNodesToDecluster(const Graph& graph,
                             absl::flat_hash_set<Node*>* result,
@@ -76,12 +51,6 @@ Status FindNodesToDecluster(const Graph& graph,
       continue;
     }
 
-    bool must_compile;
-    TF_RETURN_IF_ERROR(MustCompileNode(n, &must_compile));
-    if (must_compile) {
-      continue;
-    }
-
     DeviceType device_type("");
     TF_RETURN_IF_ERROR(
         DeviceToDeviceType(n->assigned_device_name(), &device_type));
@@ -251,6 +220,31 @@ bool IsIntraClusterEdge(const Edge& edge) {
   return src_cluster_name.has_value() && src_cluster_name == dst_cluster_name;
 }
 
+bool IsMustCompileDevice(const DeviceType& device_type) {
+  const XlaOpRegistry::DeviceRegistration* registration;
+  if (XlaOpRegistry::GetCompilationDevice(device_type.type(), &registration)) {
+    return registration->autoclustering_policy ==
+           XlaOpRegistry::AutoclusteringPolicy::kAlways;
+  }
+
+  return false;
+}
+
+Status MustCompileNode(const Node* n, bool* must_compile) {
+  DeviceType device_type("");
+  TF_RETURN_IF_ERROR(
+      DeviceToDeviceType(n->assigned_device_name(), &device_type));
+
+  if (IsMustCompileDevice(device_type)) {
+    *must_compile = true;
+    return Status::OK();
+  }
+
+  // We must compile `n` if it does not have a TensorFlow kernel.
+  *must_compile = !FindKernelDef(device_type, n->def(), nullptr, nullptr).ok();
+  return Status::OK();
+}
+
 // Declusters nodes to reduce the number of times we think we need to recompile
 // a TensorFlow graph.
 //
diff --git a/tensorflow/compiler/jit/partially_decluster_pass_test.cc b/tensorflow/compiler/jit/partially_decluster_pass_test.cc
index a6035da8bd..1d81a8f4fc 100644
--- a/tensorflow/compiler/jit/partially_decluster_pass_test.cc
+++ b/tensorflow/compiler/jit/partially_decluster_pass_test.cc
@@ -74,12 +74,6 @@ REGISTER_KERNEL_BUILDER(Name("FakeBinary")
                             .HostMemory("host_out"),
                         FakeBinaryOp);
 
-REGISTER_KERNEL_BUILDER(Name("FakeBinary")
-                            .Device(DEVICE_XLA_CPU)
-                            .HostMemory("host_in")
-                            .HostMemory("host_out"),
-                        FakeBinaryOp);
-
 REGISTER_KERNEL_BUILDER(
     Name("FakeResourceUpdate").Device(DEVICE_CPU).HostMemory("something_else"),
     FakeResourceUpdateOp);
@@ -373,7 +367,7 @@ TEST(PartiallyDeclusterPassTest, EdgeAcrossDifferentClusters) {
   EXPECT_EQ(GetXlaClusterForNode(*n), "cluster_1");
 }
 
-TEST(PartiallyDeclusterPassTest, DontDeclusterXlaDeviceOps_A) {
+TEST(PartiallyDeclusterPassTest, DontDeclusterXlaDeviceOps) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output shape_a = ops::Placeholder(s.WithOpName("shape_a"), DT_INT32,
                                     ops::Placeholder::Attrs{});
@@ -407,41 +401,6 @@ TEST(PartiallyDeclusterPassTest, DontDeclusterXlaDeviceOps_A) {
   EXPECT_EQ(GetXlaClusterForNode(*n), "cluster_0");
 }
 
-TEST(PartiallyDeclusterPassTest, DontDeclusterXlaDeviceOps_B) {
-  std::unique_ptr<Graph> graph(new Graph(OpRegistry::Global()));
-  {
-    GraphDefBuilder builder(GraphDefBuilder::kFailImmediately);
-    Node* input =
-        ops::SourceOp("FakeNullary", builder.opts().WithName("Input"));
-    Node* clustered_producer =
-        ops::BinaryOp("FakeBinary", input, input,
-                      builder.opts().WithName("ClusteredProducer"));
-    ops::BinaryOp("FakeBinary", clustered_producer, input,
-                  builder.opts().WithName("UnclusteredConsumer"));
-    Node* clustered_consumer =
-        ops::BinaryOp("FakeBinary", {clustered_producer, 1}, input,
-                      builder.opts().WithName("ClusteredConsumer"));
-    clustered_producer->AddAttr(kXlaClusterAttr, "cluster_0");
-    clustered_consumer->AddAttr(kXlaClusterAttr, "cluster_0");
-    TF_EXPECT_OK(GraphDefBuilderToGraph(builder, graph.get()));
-  }
-
-  // Scope::ToGraph loses the assigned device name since it goes through
-  // GraphDef/NodeDef which does not have a field for the assigned device name.
-  Node* n = FindNodeByName(*graph, "ClusteredProducer");
-  ASSERT_NE(n, nullptr);
-  n->set_assigned_device_name(
-      "/job:localhost/replica:0/task:0/device:XLA_CPU:0");
-
-  TF_ASSERT_OK(PartiallyDecluster(&graph));
-  std::vector<Node*> unclustered_consumer_inputs;
-  ASSERT_TRUE(GetInputsForNode(*graph, "UnclusteredConsumer",
-                               &unclustered_consumer_inputs));
-  ASSERT_EQ(unclustered_consumer_inputs.size(), 2);
-  EXPECT_EQ(unclustered_consumer_inputs[0]->name(), "ClusteredProducer");
-  EXPECT_EQ(GetXlaClusterForNode(*unclustered_consumer_inputs[0]), "cluster_0");
-}
-
 TEST(PartiallyDeclusterPassTest, DontDeclusterNonTensorFlowOps) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output dynamic_slice_operand =
diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD
index b6d4218f1d..9b6ca4092c 100644
--- a/tensorflow/compiler/tests/BUILD
+++ b/tensorflow/compiler/tests/BUILD
@@ -1223,19 +1223,3 @@ tf_xla_py_test(
         "@absl_py//absl/testing:parameterized",
     ],
 )
-
-tf_xla_py_test(
-    name = "control_flow_ops_test",
-    size = "small",
-    srcs = ["control_flow_ops_test.py"],
-    deps = [
-        ":xla_test",
-        "//tensorflow/compiler/tf2xla/python:xla",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:constant_op",
-        "//tensorflow/python:control_flow_ops",
-        "//tensorflow/python:errors",
-        "//tensorflow/python:framework",
-        "//tensorflow/python:platform_test",
-    ],
-)
diff --git a/tensorflow/compiler/tests/control_flow_ops_test.py b/tensorflow/compiler/tests/control_flow_ops_test.py
deleted file mode 100644
index c050290a48..0000000000
--- a/tensorflow/compiler/tests/control_flow_ops_test.py
+++ /dev/null
@@ -1,56 +0,0 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Test cases for Switch and Merge nodes."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np  # pylint: disable=unused-import
-
-from tensorflow.compiler.tests import xla_test
-from tensorflow.python.framework import constant_op
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.platform import googletest
-
-
-class SwitchMergeTest(xla_test.XLATestCase):
-
-  def testSwitchMerge(self):
-    with self.cached_session() as sess:
-      predicate = array_ops.placeholder(dtypes.bool)
-      with self.test_scope():
-        false_output, true_output = control_flow_ops.switch(
-            data=constant_op.constant(42.0), pred=predicate)
-        with ops.control_dependencies([array_ops.identity(false_output)]):
-          five = constant_op.constant(5.0)
-        with ops.control_dependencies([array_ops.identity(true_output)]):
-          ten = constant_op.constant(10.0)
-        result = control_flow_ops.merge([five, ten])
-
-    with_true = sess.run(result, {predicate: True})
-    self.assertEquals(with_true.output, 10.0)
-    self.assertEquals(with_true.value_index, 1)
-
-    with_false = sess.run(result, {predicate: False})
-    self.assertEquals(with_false.output, 5.0)
-    self.assertEquals(with_false.value_index, 0)
-
-
-if __name__ == "__main__":
-  googletest.main()
diff --git a/tensorflow/compiler/tf2xla/graph_compiler.cc b/tensorflow/compiler/tf2xla/graph_compiler.cc
index 19413f6c72..5e4699bbb6 100644
--- a/tensorflow/compiler/tf2xla/graph_compiler.cc
+++ b/tensorflow/compiler/tf2xla/graph_compiler.cc
@@ -119,9 +119,6 @@ Status GraphCompiler::Compile() {
   GetReversePostOrder(*graph_, &topo_sorted_nodes,
                       /*stable_comparator=*/NodeComparatorName());
 
-  std::vector<bool> control_output_dead;
-  control_output_dead.resize(graph_->num_node_ids());
-
   OpKernelContext::Params params;
   PartiallySetupParams(&params);
 
@@ -137,8 +134,8 @@ Status GraphCompiler::Compile() {
       return s;
     }
 
-    TF_RET_CHECK(!n->IsRecv() && !n->IsSend())
-        << "Unsupported node: " << n->DebugString();
+    TF_RET_CHECK(!n->IsRecv() && !n->IsSend() && !n->IsSwitch())
+        << "Not supported node: " << n->DebugString();
     params.op_kernel = op_kernel.get();
     absl::InlinedVector<AllocatorAttributes, 4> output_attr(n->num_outputs());
     params.output_attr_array = output_attr.data();
@@ -148,59 +145,38 @@ Status GraphCompiler::Compile() {
     tensor_inputs_.clear();
     tensor_inputs_.resize(n->num_inputs());
 
-    bool any_data_input_dead = false;
-    bool any_data_input_alive = false;
-    bool any_control_input_dead = false;
-
     // Set up inputs from outputs of previous nodes.
     for (auto* e : n->in_edges()) {
-      if (e->IsControlEdge()) {
-        any_control_input_dead |= control_output_dead[e->src()->id()];
-        continue;
-      }
-
+      if (e->IsControlEdge()) continue;
       const Node* src = e->src();
       TF_RET_CHECK(src->id() < output_registry.size());
       const NodeOutputs& src_outputs = output_registry[src->id()];
 
-      const TensorValue& src_output = src_outputs.at(e->src_output());
-      tensor_inputs_.at(e->dst_input()) = src_output;
-
-      any_data_input_alive |= src_output.tensor != nullptr;
-      any_data_input_dead |= src_output.tensor == nullptr;
+      tensor_inputs_.at(e->dst_input()) = src_outputs.at(e->src_output());
     }
 
-    bool node_is_dead = n->IsMerge()
-                            ? !any_data_input_alive
-                            : (any_data_input_dead || any_control_input_dead);
+    OpKernelContext op_context(&params, n->num_outputs());
+    VLOG(3) << "Translating " << params.op_kernel->name();
+    if (IsFunctional(n)) {
+      TF_RETURN_IF_ERROR(CompileFunctionalNode(n, &op_context));
+    } else {
+      device_->Compute(CHECK_NOTNULL(params.op_kernel), &op_context);
+      Status s = op_context.status();
+      if (!s.ok()) {
+        return AttachDef(s, n->def());
+      }
+    }
 
+    // Set up outputs. Also check if outputs from the previous computation is
+    // valid.
     NodeOutputs& outputs = output_registry[n->id()];
     outputs.resize(n->num_outputs());
-
-    if (!node_is_dead) {
-      OpKernelContext op_context(&params, n->num_outputs());
-      VLOG(3) << "Translating " << params.op_kernel->name();
-      if (IsFunctional(n)) {
-        TF_RETURN_IF_ERROR(CompileFunctionalNode(n, &op_context));
-      } else {
-        device_->Compute(CHECK_NOTNULL(params.op_kernel), &op_context);
-        Status s = op_context.status();
-        if (!s.ok()) {
-          return AttachDef(s, n->def());
-        }
+    for (int o = 0; o < n->num_outputs(); ++o) {
+      outputs[o] = op_context.release_output(o);
+      if (outputs[o].tensor == nullptr) {
+        return errors::Internal("Missing xla_context ", o, "-th output from ",
+                                FormatNodeForError(*n));
       }
-
-      // Set up outputs. Also check if outputs from the previous computation is
-      // valid.
-      for (int o = 0; o < n->num_outputs(); ++o) {
-        outputs[o] = op_context.release_output(o);
-        if (outputs[o].tensor == nullptr && !n->IsSwitch()) {
-          return errors::Internal("Missing xla_context ", o, "-th output from ",
-                                  FormatNodeForError(*n));
-        }
-      }
-    } else {
-      control_output_dead[n->id()] = true;
     }
   }
   return Status::OK();
diff --git a/tensorflow/compiler/tf2xla/kernels/BUILD b/tensorflow/compiler/tf2xla/kernels/BUILD
index e3c1ba48fd..69353fe87d 100644
--- a/tensorflow/compiler/tf2xla/kernels/BUILD
+++ b/tensorflow/compiler/tf2xla/kernels/BUILD
@@ -25,7 +25,6 @@ tf_kernel_library(
         "clip_by_value_op.cc",
         "concat_op.cc",
         "const_op.cc",
-        "control_flow_ops.cc",
         "conv_ops.cc",
         "cross_op.cc",
         "cwise_ops.cc",
diff --git a/tensorflow/compiler/tf2xla/kernels/control_flow_ops.cc b/tensorflow/compiler/tf2xla/kernels/control_flow_ops.cc
deleted file mode 100644
index 225016b392..0000000000
--- a/tensorflow/compiler/tf2xla/kernels/control_flow_ops.cc
+++ /dev/null
@@ -1,74 +0,0 @@
-/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include "tensorflow/compiler/tf2xla/lib/util.h"
-
-#include "tensorflow/compiler/tf2xla/type_util.h"
-#include "tensorflow/compiler/tf2xla/xla_helpers.h"
-#include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
-#include "tensorflow/compiler/tf2xla/xla_op_registry.h"
-#include "tensorflow/compiler/xla/client/lib/constants.h"
-#include "tensorflow/compiler/xla/client/xla_builder.h"
-#include "tensorflow/compiler/xla/primitive_util.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
-#include "tensorflow/core/framework/kernel_def_builder.h"
-
-namespace tensorflow {
-namespace {
-
-class SwitchOp : public XlaOpKernel {
- public:
-  explicit SwitchOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {}
-
-  void Compile(XlaOpKernelContext* ctx) override {
-    OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(ctx->InputShape("pred")),
-                errors::InvalidArgument(
-                    "The second input must be a scalar, but it has shape ",
-                    ctx->InputShape("pred").DebugString()));
-
-    xla::Literal pred_literal;
-    OP_REQUIRES_OK(ctx, ctx->ConstantInput("pred", &pred_literal));
-    bool pred = pred_literal.data<bool>()[0];
-    ctx->SetOutput(pred ? 1 : 0, ctx->Input("data"));
-  }
-};
-
-REGISTER_XLA_OP(
-    Name("Switch").CompileTimeConstantInput("pred").CompilationOnly(),
-    SwitchOp);
-
-class MergeOp : public XlaOpKernel {
- public:
-  explicit MergeOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {}
-
-  void Compile(XlaOpKernelContext* ctx) override {
-    xla::PrimitiveType index_type;
-    OP_REQUIRES_OK(ctx, DataTypeToPrimitiveType(ctx->expected_output_dtype(1),
-                                                &index_type));
-
-    for (int i = 0; i < ctx->num_inputs(); i++) {
-      if (ctx->op_kernel_context()->has_input(i)) {
-        ctx->SetOutput(0, ctx->Input(i));
-        ctx->SetOutput(
-            1, xla::ConstantR0WithType(ctx->Input(i).builder(), index_type, i));
-        return;
-      }
-    }
-  }
-};
-
-REGISTER_XLA_OP(Name("Merge").CompilationOnly(), MergeOp);
-
-}  // anonymous namespace
-}  // namespace tensorflow
-- 
GitLab


From 08dfb770b18f82d11de3c11c491ab9e97820a435 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 4 Feb 2019 20:45:37 -0800
Subject: [PATCH 1901/2345] Rework depthwise conv quantized testing.

PiperOrigin-RevId: 232417265
---
 tensorflow/lite/kernels/internal/BUILD        |   1 +
 .../internal/depthwiseconv_quantized_test.cc  | 328 ++++++++++--------
 .../depthwiseconv_uint8_3x3_filter.h          |  26 +-
 .../internal/reference/depthwiseconv_uint8.h  |   8 +-
 4 files changed, 204 insertions(+), 159 deletions(-)

diff --git a/tensorflow/lite/kernels/internal/BUILD b/tensorflow/lite/kernels/internal/BUILD
index cbe4c9d414..97b322551e 100644
--- a/tensorflow/lite/kernels/internal/BUILD
+++ b/tensorflow/lite/kernels/internal/BUILD
@@ -591,6 +591,7 @@ cc_test(
         ":reference_base",
         ":test_util",
         ":types",
+        "@com_google_absl//absl/strings",
         "@com_google_googletest//:gtest_main",
     ],
 )
diff --git a/tensorflow/lite/kernels/internal/depthwiseconv_quantized_test.cc b/tensorflow/lite/kernels/internal/depthwiseconv_quantized_test.cc
index 3682499d49..963ab4d8ff 100644
--- a/tensorflow/lite/kernels/internal/depthwiseconv_quantized_test.cc
+++ b/tensorflow/lite/kernels/internal/depthwiseconv_quantized_test.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include <cstdlib>
 #include <iterator>
 #include <limits>
+#include <string>
 #include <vector>
 
 #include <gtest/gtest.h>
@@ -26,6 +27,7 @@ limitations under the License.
 #include "tensorflow/lite/kernels/internal/types.h"
 
 #define ALLOW_SLOW_GENERIC_DEPTHWISECONV_FALLBACK
+#include "absl/strings/substitute.h"
 #include "tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8.h"
 #include "tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h"
 #include "tensorflow/lite/kernels/internal/reference/depthwiseconv_uint8.h"
@@ -33,26 +35,65 @@ limitations under the License.
 namespace tflite {
 namespace {
 
-enum class ForceKernelInvocation {
-  // Run all tests against kUseStandardEntry even if also testing another
-  // kernel, since we need to be sure that the main DepthwiseConv() function in
-  // optimized_ops.h dispatches to a correctly-executing kernel.
-  kNone = 0,  // The "default" option: use the normal DepthwiseConv
-              // kernel (entry) function.
-  kUseGenericKernel,
-  kUseNeon3x3,            // 3x3 kernel that uses NEON when available.
-  kUseNeon3x3DotProduct,  // 3x3 kernel that uses dot-product enabled NEON when
-                          // available.
+using ::testing::Bool;
+using ::testing::Values;
+
+// Currently, this is used in place of a Boolean "is symmetric?".
+enum class ParamsSpecialization {
+  kNone = 0,
+  kSymmetric,  // Symmetric quantization: zero represented by 128.
+};
+
+static constexpr int kSymmetricZeroPoint = 128;
+
+// Extend coverage distribution in a specific aspect, either explicitly chosen
+// or randomly chosen as in a mixture distribution.
+enum class CoverageExtension {
+  kNone = 0,
+  kLargeHeights = 1,
+  kLargeWidths = 2,
+  kNumOptions
+};
+
+// The TestParam structure below is the preferred parameterization of tests. A
+// tuple version is defined in order to support value-parameterized tests.
+typedef std::tuple<DepthwiseConvInvocation, int, bool, bool, bool>
+    TestParamTuple;
+
+struct TestParam {
+  TestParam() = default;
+
+  explicit TestParam(TestParamTuple param_tuple)
+      : forced_invocation(::testing::get<0>(param_tuple)),
+        tests_to_run(::testing::get<1>(param_tuple)),
+        test_stride(::testing::get<2>(param_tuple)),
+        test_pad(::testing::get<3>(param_tuple)),
+        test_depth_multiplier(::testing::get<4>(param_tuple)) {}
+
+  static std::string TestNameSuffix(
+      const ::testing::TestParamInfo<TestParamTuple>& info) {
+    const TestParam param(info.param);
+    return absl::Substitute("invocation_$0_stride_$1_pad_$2_depth_mult_$3",
+                            static_cast<int>(param.forced_invocation),
+                            param.test_stride, param.test_pad,
+                            param.test_depth_multiplier);
+  }
+
+  DepthwiseConvInvocation forced_invocation = DepthwiseConvInvocation::kNone;
+  int tests_to_run = 0;
+  bool test_stride = false;
+  bool test_pad = false;
+  bool test_depth_multiplier = false;
 };
 
 inline void DispatchDepthwiseConv(
-    ForceKernelInvocation forced_invocation, const DepthwiseParams& params,
+    const TestParam& test_param, const DepthwiseParams& params,
     const RuntimeShape& input_shape, const uint8* input_data,
     const RuntimeShape& filter_shape, const uint8* filter_data,
     const RuntimeShape& bias_shape, const int32* bias_data,
     const RuntimeShape& output_shape, uint8* output_data) {
-  switch (forced_invocation) {
-    case ForceKernelInvocation::kUseNeon3x3: {
+  switch (test_param.forced_invocation) {
+    case DepthwiseConvInvocation::kUseNeon3x3: {
 // Enable for arm64 except for the Nvidia Linux 4 Tegra (L4T) running on
 // Jetson TX-2. This compiler does not support the offsetof() macro.
 #if defined(__aarch64__) && !defined(GOOGLE_L4T)
@@ -74,10 +115,10 @@ inline void DispatchDepthwiseConv(
       ASSERT_TRUE(basic_3x3_kernel_supported)
           << "pad_width = " << params.padding_values.width
           << " pad_height = " << params.padding_values.height
-          << " input_width = " << input_shape.Dims(1)
-          << " input_height = " << input_shape.Dims(2)
-          << " output_width = " << output_shape.Dims(1)
-          << " output_height = " << output_shape.Dims(2);
+          << " input_width = " << input_shape.Dims(2)
+          << " input_height = " << input_shape.Dims(1)
+          << " output_width = " << output_shape.Dims(2)
+          << " output_height = " << output_shape.Dims(1);
 
       // Call kernel optimized for depthwise convolutions using 3x3 filters.
       optimized_ops::DepthwiseConv3x3Filter(
@@ -88,56 +129,24 @@ inline void DispatchDepthwiseConv(
       break;
 #endif
     }
-    case ForceKernelInvocation::kUseNeon3x3DotProduct: {
-// Enable for arm64 except for the Nvidia Linux 4 Tegra (L4T) running on
-// Jetson TX-2. This compiler does not support the offsetof() macro.
-#if defined(__ARM_FEATURE_DOTPROD) && defined(__aarch64__) && \
-    !defined(GOOGLE_L4T)
-      using optimized_ops::DotProduct3x3KernelType;
-      DotProduct3x3KernelType kernel_type =
-          optimized_ops::CategorizeDotProductKernel(params);
-      switch (kernel_type) {
-        case DotProduct3x3KernelType::kPlain:
-          // TODO(b/118430534): Implement optimized kernel.
-          optimized_ops::DepthwiseConv3x3Filter(
-              params, input_shape, input_data, filter_shape, filter_data,
-              bias_shape, bias_data, output_shape, output_data);
-          return;
-        case DotProduct3x3KernelType::kWithDepthMultiplication:
-          // TODO(b/118430338): Implement optimized kernel.
-          optimized_ops::DepthwiseConvGeneral(
-              params, input_shape, input_data, filter_shape, filter_data,
-              bias_shape, bias_data, output_shape, output_data);
-          return;
-        case DotProduct3x3KernelType::kWithPad0Stride2:
-          // TODO(b/118430338): Implement optimized kernel.
-          optimized_ops::DepthwiseConv3x3Filter(
-              params, input_shape, input_data, filter_shape, filter_data,
-              bias_shape, bias_data, output_shape, output_data);
-          return;
-        case DotProduct3x3KernelType::kWithPad1Stride1:
-          // TODO(b/118430338): Implement optimized kernel.
-          optimized_ops::DepthwiseConvGeneral(
-              params, input_shape, input_data, filter_shape, filter_data,
-              bias_shape, bias_data, output_shape, output_data);
-          return;
-        case DotProduct3x3KernelType::kNone:
-        default:
-          break;
-      }
-#endif
+    case DepthwiseConvInvocation::kUseNeon3x3DotProduct:
+    case DepthwiseConvInvocation::kUseCModel3x3DotProduct:
+    case DepthwiseConvInvocation::kUseUnwound3x3DotProduct:
+    case DepthwiseConvInvocation::kUseIntrinsics3x3DotProduct:
+      // TODO(b/118426582) Placeholder for future dispatches.
       break;
-    }
-    case ForceKernelInvocation::kUseGenericKernel: {
+    case DepthwiseConvInvocation::kUseGenericKernel: {
       optimized_ops::DepthwiseConvGeneral(params, input_shape, input_data,
                                           filter_shape, filter_data, bias_shape,
                                           bias_data, output_shape, output_data);
       return;
     }
-    case ForceKernelInvocation::kNone:
+    case DepthwiseConvInvocation::kNone:
     default:
       break;
   }
+  EXPECT_EQ(test_param.forced_invocation, DepthwiseConvInvocation::kNone)
+      << "TODO(b/118426582) requested kernel was not invoked / available yet";
   optimized_ops::DepthwiseConv(params, input_shape, input_data, filter_shape,
                                filter_data, bias_shape, bias_data, output_shape,
                                output_data);
@@ -145,7 +154,7 @@ inline void DispatchDepthwiseConv(
 
 // Runs the DepthwiseConv and compares against the reference implementation.
 int TestOneDepthwiseConvWithGivenOutputShift(
-    ForceKernelInvocation forced_invocation, const std::uint8_t* input_data,
+    const TestParam& test_param, const std::uint8_t* input_data,
     const RuntimeShape& input_shape, std::int32_t input_offset,
     const std::uint8_t* filter_data, const RuntimeShape& filter_shape,
     std::int32_t filter_offset, const std::int32_t* bias_data,
@@ -177,7 +186,7 @@ int TestOneDepthwiseConvWithGivenOutputShift(
   reference_ops::DepthwiseConv(op_params, input_shape, input_data, filter_shape,
                                filter_data, bias_shape, bias_data, output_shape,
                                reference_output_data.data());
-  DispatchDepthwiseConv(forced_invocation, op_params, input_shape, input_data,
+  DispatchDepthwiseConv(test_param, op_params, input_shape, input_data,
                         filter_shape, filter_data, bias_shape, bias_data,
                         output_shape, output_data.data());
   int saturated_min = 0;
@@ -201,15 +210,26 @@ int TestOneDepthwiseConvWithGivenOutputShift(
   const float mean_diff = static_cast<float>(sum_diff) / output_buffer_size;
   const float mean_abs_diff =
       static_cast<float>(sum_abs_diff) / output_buffer_size;
+
+  constexpr int diff_mean_tolerance = 1;
+  constexpr int diff_median_tolerance = 0;
+  // The tolerance that we apply to means is tight, but we allow for a rounding
+  // difference in one pixel, and loosen by another 1% for float comparison.
+  const float mean_tolerance =
+      std::max(1e-5f, 1.01f * 2.f / output_buffer_size *
+                          std::sqrt(1.f * depth_multiplier));
+
   // Normally we should require bit-for-bit exact results. Unfortunately a bug
   // in the Intel arm_neon_sse.h translation header that we use for x86 tests
   // causes 1-bit inaccuracy in
   // the vqrdmulh_n_s32 intrinsic, which causes off-by-1 errors in quantized
   // DepthwiseConv ops. So we have to live with a few off-by-one errors for now,
   // yet still ensure that no more than a small minority of values are wrong.
-  EXPECT_TRUE(std::abs(mean_diff) < 1e-5f && mean_abs_diff < 1e-5f &&
-              std::abs(median_diff) == 0 && std::abs(min_diff) <= 1 &&
-              std::abs(max_diff) <= 1);
+  EXPECT_LT(std::abs(mean_diff), mean_tolerance);
+  EXPECT_LT(mean_abs_diff, mean_tolerance);
+  EXPECT_LE(std::abs(median_diff), diff_median_tolerance);
+  EXPECT_LE(std::abs(min_diff), diff_mean_tolerance);
+  EXPECT_LE(std::abs(max_diff), diff_mean_tolerance);
   if (saturated_min > 2 * saturated_max) {
     return -1;
   }
@@ -221,13 +241,12 @@ int TestOneDepthwiseConvWithGivenOutputShift(
 
 // The point of this function is that we can't practically know which
 // output_shift value to pass to test DepthwiseConv. It's not easy to guess (we
-// could do some
-// statistics for large size, but they would be fragile at smaller sizes), and
-// guessing wrong would mean that all the values get saturated so the test
-// becomes
-// vacuous. So we just bisect our way to reasonable output_shift values.
+// could do some statistics for large size, but they would be fragile at smaller
+// sizes), and guessing wrong would mean that all the values get saturated so
+// the test becomes vacuous. So we just bisect our way to reasonable
+// output_shift values.
 void TestOneDepthwiseConvBisectOutputShift(
-    ForceKernelInvocation forced_invocation, const std::uint8_t* input_data,
+    const TestParam& test_param, const std::uint8_t* input_data,
     const RuntimeShape& input_shape, std::int32_t input_offset,
     const std::uint8_t* filter_data, const RuntimeShape& filter_shape,
     std::int32_t filter_offset, const std::int32_t* bias_data,
@@ -242,7 +261,7 @@ void TestOneDepthwiseConvBisectOutputShift(
   int output_shift_bisect_midpoint =
       (output_activation_bisect_start + output_activation_bisect_end) / 2;
   int bisect_result = TestOneDepthwiseConvWithGivenOutputShift(
-      forced_invocation, input_data, input_shape, input_offset, filter_data,
+      test_param, input_data, input_shape, input_offset, filter_data,
       filter_shape, filter_offset, bias_data, bias_shape, stride, padding_type,
       pad_width, pad_height, depth_multiplier, output_offset, output_multiplier,
       output_shift_bisect_midpoint, output_activation_min,
@@ -269,7 +288,7 @@ void TestOneDepthwiseConvBisectOutputShift(
                                              ? output_activation_bisect_end
                                              : output_shift_bisect_midpoint;
   TestOneDepthwiseConvBisectOutputShift(
-      forced_invocation, input_data, input_shape, input_offset, filter_data,
+      test_param, input_data, input_shape, input_offset, filter_data,
       filter_shape, filter_offset, bias_data, bias_shape, stride, padding_type,
       pad_width, pad_height, depth_multiplier, output_offset, output_multiplier,
       new_output_activation_bisect_start, new_output_activation_bisect_end,
@@ -277,7 +296,7 @@ void TestOneDepthwiseConvBisectOutputShift(
 }
 
 void TestOneDepthwiseConv(
-    ForceKernelInvocation forced_invocation, const std::uint8_t* input_data,
+    const TestParam& test_param, const std::uint8_t* input_data,
     const RuntimeShape& input_shape, std::int32_t input_offset,
     const std::uint8_t* filter_data, const RuntimeShape& filter_shape,
     std::int32_t filter_offset, const std::int32_t* bias_data,
@@ -287,13 +306,14 @@ void TestOneDepthwiseConv(
     std::int32_t output_activation_min, std::int32_t output_activation_max,
     const RuntimeShape& output_shape) {
   TestOneDepthwiseConvBisectOutputShift(
-      forced_invocation, input_data, input_shape, input_offset, filter_data,
+      test_param, input_data, input_shape, input_offset, filter_data,
       filter_shape, filter_offset, bias_data, bias_shape, stride, padding_type,
       pad_width, pad_height, depth_multiplier, output_offset, output_multiplier,
       0, 32, output_activation_min, output_activation_max, output_shape);
 }
 
-bool TryTestDepthwiseConv(ForceKernelInvocation forced_invocation, int batch,
+bool TryTestDepthwiseConv(const TestParam& test_param,
+                          ParamsSpecialization params_specialization, int batch,
                           int input_depth, int input_width, int input_height,
                           int filter_width, int filter_height,
                           int depth_multiplier, int stride,
@@ -318,9 +338,12 @@ bool TryTestDepthwiseConv(ForceKernelInvocation forced_invocation, int batch,
   }
   const std::int32_t output_multiplier =
       UniformRandomInt(1 << 29, std::numeric_limits<std::int32_t>::max());
-  const std::int32_t input_offset = UniformRandomInt(-256, 0);
-  const std::int32_t filter_offset = UniformRandomInt(-256, 0);
-  const std::int32_t output_offset = UniformRandomInt(-256, 0);
+  std::int32_t filter_offset = -kSymmetricZeroPoint;
+  if (params_specialization != ParamsSpecialization::kSymmetric) {
+    filter_offset = UniformRandomInt(-255, 0);
+  }
+  const std::int32_t input_offset = UniformRandomInt(-255, 0);
+  const std::int32_t output_offset = UniformRandomInt(0, 255);
   RuntimeShape input_shape_inference(
       {batch, input_height, input_width, input_depth});
   RuntimeShape output_shape_inference;
@@ -343,7 +366,7 @@ bool TryTestDepthwiseConv(ForceKernelInvocation forced_invocation, int batch,
   FillRandom(&filter_data);
   FillRandom(&bias_data, -10000, 10000);
   TestOneDepthwiseConv(
-      forced_invocation, input_data.data(), input_shape_inference, input_offset,
+      test_param, input_data.data(), input_shape_inference, input_offset,
       filter_data.data(), filter_shape_inference, filter_offset,
       bias_data.data(), bias_shape_inference, stride, padding_type, pad_width,
       pad_height, depth_multiplier, output_offset, output_multiplier,
@@ -355,7 +378,8 @@ bool TryTestDepthwiseConv(ForceKernelInvocation forced_invocation, int batch,
 // be legal. If they're not legal, it returns false. If they're legal,
 // it runs the DepthwiseConv test and returns true. This allows the caller
 // to loop until a test has been run.
-bool TryTestOneDepthwiseConv(ForceKernelInvocation forced_invocation) {
+bool TryTestOneDepthwiseConv(const TestParam& test_param,
+                             ParamsSpecialization params_specialization) {
   // We have to pick a lot of positive values, where we are particularly
   // interested in small values because they are most likely to be special
   // cases in optimized implementations, and secondarily because they allow
@@ -375,13 +399,14 @@ bool TryTestOneDepthwiseConv(ForceKernelInvocation forced_invocation) {
       UniformRandomInt(0, 1) ? PaddingType::kSame : PaddingType::kValid;
 
   return TryTestDepthwiseConv(
-      forced_invocation, batch, input_depth, input_width, input_height,
-      filter_width, filter_height, depth_multiplier, stride,
+      test_param, params_specialization, batch, input_depth, input_width,
+      input_height, filter_width, filter_height, depth_multiplier, stride,
       dilation_width_factor, dilation_height_factor, padding_type);
 }
 
 // Tests parameters for the 3x3 filter kernel.
-bool TryTestOneDepthwiseConv3x3Filter(ForceKernelInvocation forced_invocation) {
+bool TryTestOneDepthwiseConv3x3Filter(
+    const TestParam& test_param, ParamsSpecialization params_specialization) {
   const int batch = ExponentialRandomPositiveInt(0.9f, 3, 20);
   const int input_depth = 8 * ExponentialRandomPositiveInt(0.9f, 10, 50);
   int input_width = ExponentialRandomPositiveInt(0.9f, 20, 200);
@@ -397,7 +422,7 @@ bool TryTestOneDepthwiseConv3x3Filter(ForceKernelInvocation forced_invocation) {
       UniformRandomInt(0, 1) ? PaddingType::kSame : PaddingType::kValid;
 
   // Adjust for, or reject, special cases.
-  if (forced_invocation != ForceKernelInvocation::kNone) {
+  if (test_param.forced_invocation != DepthwiseConvInvocation::kNone) {
     // With stride == 2 and SAME, padding width and height are the left and top
     // padding amounts. When there is an even input dimension, padding + 1 is
     // required on the right / bottom. This is not handled by these kernels, so
@@ -416,59 +441,71 @@ bool TryTestOneDepthwiseConv3x3Filter(ForceKernelInvocation forced_invocation) {
   }
 
   return TryTestDepthwiseConv(
-      forced_invocation, batch, input_depth, input_width, input_height,
-      filter_width, filter_height, depth_multiplier, stride,
+      test_param, params_specialization, batch, input_depth, input_width,
+      input_height, filter_width, filter_height, depth_multiplier, stride,
       dilation_width_factor, dilation_height_factor, padding_type);
 }
 
 // Tests with parameters suited to dot-product-NEON 3x3 filter kernels.
-bool TryTestOneNeonDot3x3(ForceKernelInvocation forced_invocation,
-                          bool test_stride, bool test_pad,
-                          bool test_depth_multiplier) {
+bool TryTestOneNeonDot3x3(const TestParam& test_param,
+                          ParamsSpecialization params_specialization) {
+  const CoverageExtension coverage_extension = static_cast<CoverageExtension>(
+      UniformRandomInt(0, static_cast<int>(CoverageExtension::kNumOptions)));
+
   const int batch = 1;
-  const int input_depth = test_depth_multiplier
+  const int input_depth = test_param.test_depth_multiplier
                               ? 1
-                              : 8 * ExponentialRandomPositiveInt(0.9f, 10, 50);
-  const int input_width = ExponentialRandomPositiveInt(0.9f, 20, 200);
-  const int input_height = ExponentialRandomPositiveInt(0.9f, 20, 200);
+                              : 8 * ExponentialRandomPositiveInt(0.9f, 3, 50);
+  const int input_width = coverage_extension == CoverageExtension::kLargeWidths
+                              ? ExponentialRandomPositiveInt(0.9f, 50, 200)
+                              : ExponentialRandomPositiveInt(0.9f, 20, 60);
+  const int input_height =
+      coverage_extension == CoverageExtension::kLargeHeights
+          ? ExponentialRandomPositiveInt(0.9f, 50, 200)
+          : ExponentialRandomPositiveInt(0.9f, 20, 60);
   const int filter_width = 3;
   const int filter_height = 3;
   const int depth_multiplier =
-      test_depth_multiplier ? 8 * ExponentialRandomPositiveInt(0.8f, 1, 6) : 1;
-  const int stride = test_stride ? 2 : 1;
+      test_param.test_depth_multiplier
+          ? 8 * ExponentialRandomPositiveInt(0.2f, 1, 9)
+          : 1;
+  const int stride = test_param.test_stride ? 2 : 1;
   // We don't support dilations in the 3x3 filter.
   const int dilation_width_factor = 1;
   const int dilation_height_factor = 1;
-  const auto padding_type = test_pad ? PaddingType::kSame : PaddingType::kValid;
+  const auto padding_type =
+      test_param.test_pad ? PaddingType::kSame : PaddingType::kValid;
 
   return TryTestDepthwiseConv(
-      forced_invocation, batch, input_depth, input_width, input_height,
-      filter_width, filter_height, depth_multiplier, stride,
+      test_param, params_specialization, batch, input_depth, input_width,
+      input_height, filter_width, filter_height, depth_multiplier, stride,
       dilation_width_factor, dilation_height_factor, padding_type);
 }
 
-void TestOneDepthwiseConv(ForceKernelInvocation forced_invocation) {
-  while (!TryTestOneDepthwiseConv(forced_invocation)) {
+void TestOneDepthwiseConv(DepthwiseConvInvocation forced_invocation) {
+  TestParam test_param;
+  test_param.forced_invocation = forced_invocation;
+  while (!TryTestOneDepthwiseConv(test_param, ParamsSpecialization::kNone)) {
   }
 }
 
-void TestOneDepthwiseConv3x3Filter(ForceKernelInvocation forced_invocation) {
-  while (!TryTestOneDepthwiseConv3x3Filter(forced_invocation)) {
+void TestOneDepthwiseConv3x3Filter(DepthwiseConvInvocation forced_invocation) {
+  TestParam test_param;
+  test_param.forced_invocation = forced_invocation;
+  while (!TryTestOneDepthwiseConv3x3Filter(test_param,
+                                           ParamsSpecialization::kNone)) {
   }
 }
 
-void TestOneNeonDot3x3(ForceKernelInvocation forced_invocation,
-                       bool test_stride, bool test_pad,
-                       bool test_depth_multiplier) {
-  while (!TryTestOneNeonDot3x3(forced_invocation, test_stride, test_pad,
-                               test_depth_multiplier)) {
+void TestOneNeonDot3x3(const TestParam& test_param) {
+  while (!TryTestOneNeonDot3x3(test_param, ParamsSpecialization::kSymmetric)) {
   }
 }
 
 TEST(TestDepthwiseConv, TestDepthwiseConv) {
   const int kTestsToRun = 10 * 1000;
   for (int i = 0; i < kTestsToRun; i++) {
-    TestOneDepthwiseConv(ForceKernelInvocation::kNone);
+    TestOneDepthwiseConv(DepthwiseConvInvocation::kNone);
   }
 }
 
@@ -476,69 +513,70 @@ TEST(TestDepthwiseConv, TestDepthwiseConv) {
 TEST(TestDepthwiseConv, TestGenericKernel) {
   const int kTestsToRun = 10 * 1000;
   for (int i = 0; i < kTestsToRun; i++) {
-    TestOneDepthwiseConv(ForceKernelInvocation::kUseGenericKernel);
+    TestOneDepthwiseConv(DepthwiseConvInvocation::kUseGenericKernel);
   }
 }
 
 TEST(TestDepthwiseConv, TestKernel3x3Filter) {
   const int kTestsToRun = 1000;
   for (int i = 0; i < kTestsToRun; i++) {
-    TestOneDepthwiseConv3x3Filter(ForceKernelInvocation::kNone);
+    TestOneDepthwiseConv3x3Filter(DepthwiseConvInvocation::kNone);
   }
 }
 
-// While the 3x3 coverage test is primarily targeted at specialized kernels, we
-// also run it against the generic kernel, optionally with fewer invocations.
+// While 3x3 coverage tests are primarily targeted at specialized kernels, we
+// also run it against the generic kernel.
 TEST(TestDepthwiseConv, TestGenericKernel3x3Filter) {
-  const int kTestsToRun = 1000;
+  const int kTestsToRun = 100;
   for (int i = 0; i < kTestsToRun; i++) {
-    TestOneDepthwiseConv3x3Filter(ForceKernelInvocation::kUseGenericKernel);
+    TestOneDepthwiseConv3x3Filter(DepthwiseConvInvocation::kUseGenericKernel);
   }
 }
 
+#if defined(__aarch64__) && !defined(GOOGLE_L4T)
 TEST(TestDepthwiseConv, TestNeon3x3Filter) {
   const int kTestsToRun = 3 * 1000;
   for (int i = 0; i < kTestsToRun; i++) {
-    TestOneDepthwiseConv3x3Filter(ForceKernelInvocation::kUseNeon3x3);
+    TestOneDepthwiseConv3x3Filter(DepthwiseConvInvocation::kUseNeon3x3);
   }
 }
+#endif
 
-// No stride, no depth multiplier, no pad.
-TEST(TestDepthwiseConv, TestNeonDot3x3Plain) {
-  const int kTestsToRun = 3 * 1000;
-  for (int i = 0; i < kTestsToRun; i++) {
-    TestOneNeonDot3x3(ForceKernelInvocation::kUseNeon3x3DotProduct,
-                      /*test_stride=*/false, /*test_pad=*/false,
-                      /*test_depth_multiplier=*/false);
-  }
-}
+class DepthwiseConvTest : public ::testing::TestWithParam<TestParamTuple> {};
 
-TEST(TestDepthwiseConv, TestNeonDot3x3DepthMultiplier) {
-  const int kTestsToRun = 3 * 1000;
-  for (int i = 0; i < kTestsToRun; i++) {
-    TestOneNeonDot3x3(ForceKernelInvocation::kUseNeon3x3DotProduct,
-                      /*test_stride=*/false, /*test_pad=*/false,
-                      /*test_depth_multiplier=*/true);
+TEST_P(DepthwiseConvTest, NeonDot3x3) {
+  const TestParam param(GetParam());
+  for (int i = 0; i < param.tests_to_run; i++) {
+    TestOneNeonDot3x3(param);
   }
 }
 
-TEST(TestDepthwiseConv, TestNeonDot3x3Stride2) {
-  const int kTestsToRun = 3 * 1000;
-  for (int i = 0; i < kTestsToRun; i++) {
-    TestOneNeonDot3x3(ForceKernelInvocation::kUseNeon3x3DotProduct,
-                      /*test_stride=*/true, /*test_pad=*/false,
-                      /*test_depth_multiplier=*/false);
-  }
-}
+#if defined(__aarch64__) && !defined(GOOGLE_L4T)
+INSTANTIATE_TEST_SUITE_P(
+    Neon3x3Kernel, DepthwiseConvTest,
+    testing::Combine(
+        Values(DepthwiseConvInvocation::kUseNeon3x3),  // forced_invocation
+        Values(1000),                                  // tests_to_run
+        Bool(),                                        // test_stride
+        Values(false),                                 // test_pad
+        Values(false)                                  // test_depth_multiplier
+        ),
+    TestParam::TestNameSuffix);
+#endif
 
-TEST(TestDepthwiseConv, TestNeonDot3x3Pad1) {
-  const int kTestsToRun = 3 * 1000;
-  for (int i = 0; i < kTestsToRun; i++) {
-    TestOneNeonDot3x3(ForceKernelInvocation::kUseNeon3x3DotProduct,
-                      /*test_stride=*/false, /*test_pad=*/true,
-                      /*test_depth_multiplier=*/false);
-  }
-}
+// While 3x3 coverage tests are primarily targeted at specialized kernels, we
+// also run it against the generic kernel.
+INSTANTIATE_TEST_SUITE_P(
+    GenericKernel, DepthwiseConvTest,
+    testing::Combine(
+        Values(
+            DepthwiseConvInvocation::kUseGenericKernel),  // forced_invocation
+        Values(100),                                      // tests_to_run
+        Bool(),                                           // test_stride
+        Bool(),                                           // test_pad
+        Bool()  // test_depth_multiplier
+        ),
+    TestParam::TestNameSuffix);
 
 }  // namespace
 }  // namespace tflite
diff --git a/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h b/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h
index 6bfe27d96d..b7993c3104 100644
--- a/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h
+++ b/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h
@@ -28,33 +28,33 @@ namespace optimized_ops {
 enum class DotProduct3x3KernelType {
   kNone = 0,  // Parameter combination is not supported for dot product kernels.
   kPlain,
-  kWithDepthMultiplication,
-  kWithPad0Stride2,
-  kWithPad1Stride1,
+  kWithDepthMultiplicationStride1,
+  kWithDepthMultiplicationStride2,
+  kStride2,
 };
 
 inline DotProduct3x3KernelType CategorizeDotProductKernel(
     const DepthwiseParams& params) {
-  const int padding = params.padding_values.width;
+  const int padding =
+      std::max(params.padding_values.width, params.padding_values.height);
   const int stride = params.stride_width;
-  if (padding != params.padding_values.height ||
-      stride != params.stride_height) {
+  if (stride != params.stride_height || padding > 1) {
     return DotProduct3x3KernelType::kNone;
   }
 
   if (params.depth_multiplier == 1) {
-    if (padding == 0 && stride == 1) {
+    if (stride == 1) {
       return DotProduct3x3KernelType::kPlain;
-    } else if (padding == 0 && stride == 2) {
-      return DotProduct3x3KernelType::kWithPad0Stride2;
-    } else if (padding == 1 && stride == 1) {
-      return DotProduct3x3KernelType::kWithPad1Stride1;
+    } else if (stride == 2) {
+      return DotProduct3x3KernelType::kStride2;
     } else {
       return DotProduct3x3KernelType::kNone;
     }
   } else {
-    if (padding == 0 && stride == 1) {
-      return DotProduct3x3KernelType::kWithDepthMultiplication;
+    if (stride == 1) {
+      return DotProduct3x3KernelType::kWithDepthMultiplicationStride1;
+    } else if (stride == 2) {
+      return DotProduct3x3KernelType::kWithDepthMultiplicationStride2;
     } else {
       return DotProduct3x3KernelType::kNone;
     }
diff --git a/tensorflow/lite/kernels/internal/reference/depthwiseconv_uint8.h b/tensorflow/lite/kernels/internal/reference/depthwiseconv_uint8.h
index d9a5372e6e..99c03426f5 100644
--- a/tensorflow/lite/kernels/internal/reference/depthwiseconv_uint8.h
+++ b/tensorflow/lite/kernels/internal/reference/depthwiseconv_uint8.h
@@ -27,7 +27,7 @@ namespace tflite {
 // Used in tests and template parameters to control which version of depthwise
 // convolution is called. Primarily for reference code, and specializations
 // forced in tests.
-enum class DepthwiseConvKernelInvocation {
+enum class DepthwiseConvInvocation {
   // Run all tests against kUseStandardEntry even if also testing another
   // kernel, since we need to be sure that the main DepthwiseConv() function in
   // optimized_ops.h dispatches to a correctly-executing kernel.
@@ -44,6 +44,12 @@ enum class DepthwiseConvKernelInvocation {
   kUseIntrinsics3x3DotProduct,  // 3x3 kernel using NEON intrinsics.
 };
 
+// Category of depthwise convolution depth multiplication.
+enum class DepthwiseConvDepthMultiplication {
+  kNoMultiplication = 0,  // Depth multiplier = 1.
+  kUnitInputDepth,        // Input depth = 1, output depth = depth multiplier.
+};
+
 namespace reference_ops {
 
 inline void DepthwiseConv(
-- 
GitLab


From 783cdae78c3fdce1de859f974dfdcafdb38d4251 Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Mon, 4 Feb 2019 22:20:18 -0800
Subject: [PATCH 1902/2345] Fixed the bug that "cluster_spec is not iterable".

PiperOrigin-RevId: 232425018
---
 tensorflow/python/distribute/estimator_training.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/distribute/estimator_training.py b/tensorflow/python/distribute/estimator_training.py
index 7d5f231c37..0ec6703b86 100644
--- a/tensorflow/python/distribute/estimator_training.py
+++ b/tensorflow/python/distribute/estimator_training.py
@@ -24,6 +24,7 @@ import six
 
 from tensorflow.python.distribute import distribute_coordinator as dc
 from tensorflow.python.distribute import distribute_coordinator_context as dc_context
+from tensorflow.python.distribute import multi_worker_util
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import server_lib
 
@@ -296,10 +297,11 @@ def estimator_train(estimator, train_distributed_fn, hooks):
   assert estimator._config._distribute_coordinator_mode
   run_config = estimator._config
   assert estimator._config.cluster_spec
-  cluster_spec = estimator._config.cluster_spec
+  cluster_spec = multi_worker_util.normalize_cluster_spec(
+      estimator._config.cluster_spec)
   assert estimator._config._train_distribute
 
-  if 'evaluator' in cluster_spec:
+  if 'evaluator' in cluster_spec.jobs:
     raise ValueError("'evaluator' job is not supported if you don't use "
                      '`train_and_evaluate`')
 
@@ -344,10 +346,11 @@ def estimator_evaluate(estimator, evaluate_distributed_fn, hooks):
   assert estimator._config._distribute_coordinator_mode
   run_config = estimator._config
   assert estimator._config.cluster_spec
-  cluster_spec = estimator._config.cluster_spec
+  cluster_spec = multi_worker_util.normalize_cluster_spec(
+      estimator._config.cluster_spec)
   assert estimator._config._eval_distribute
 
-  if 'evaluator' in cluster_spec:
+  if 'evaluator' in cluster_spec.jobs:
     raise ValueError("'evaluator' job is not supported if you don't use "
                      '`train_and_evaluate`')
 
-- 
GitLab


From 91c7ffe16cb481bd819cb3bfcc0295140a93b539 Mon Sep 17 00:00:00 2001
From: cclauss <cclauss@me.com>
Date: Tue, 5 Feb 2019 07:49:59 +0100
Subject: [PATCH 1903/2345] Update install_pip_packages.sh

https://github.com/benjaminp/six/blob/master/CHANGES Fixes a metaclass bug and adds __six.ensure_binary()__, __six.ensure_text()__, and
  __six.ensure_str()__.
---
 tensorflow/tools/ci_build/install/install_pip_packages.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/ci_build/install/install_pip_packages.sh b/tensorflow/tools/ci_build/install/install_pip_packages.sh
index 4cd3eb2e0e..b1243be4e5 100755
--- a/tensorflow/tools/ci_build/install/install_pip_packages.sh
+++ b/tensorflow/tools/ci_build/install/install_pip_packages.sh
@@ -40,8 +40,8 @@ pip2 install virtualenv
 pip3 install virtualenv
 
 # Install six.
-pip2 install --upgrade six==1.11.0
-pip3 install --upgrade six==1.11.0
+pip2 install --upgrade six==1.12.0
+pip3 install --upgrade six==1.12.0
 
 # Install absl-py.
 pip2 install --upgrade absl-py
-- 
GitLab


From f4df3fcb00a7b3acd86443c6550c48f890cb71ff Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 5 Feb 2019 01:02:40 -0800
Subject: [PATCH 1904/2345] compat: Update forward compatibility horizon to
 2019-02-05

PiperOrigin-RevId: 232438774
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index a278a01410..262df69275 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 2, 4)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 2, 5)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From f8e851674172108112be04152763192358853873 Mon Sep 17 00:00:00 2001
From: cclauss <cclauss@me.com>
Date: Tue, 5 Feb 2019 10:56:16 +0100
Subject: [PATCH 1905/2345] install_pip_packages.sh: 'pep8' is now
 'pycodestyle'

As discussed at https://pep8.readthedocs.io/en/release-1.7.x/
---
 tensorflow/tools/ci_build/install/install_pip_packages.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/tools/ci_build/install/install_pip_packages.sh b/tensorflow/tools/ci_build/install/install_pip_packages.sh
index 4cd3eb2e0e..0fa8c6caf2 100755
--- a/tensorflow/tools/ci_build/install/install_pip_packages.sh
+++ b/tensorflow/tools/ci_build/install/install_pip_packages.sh
@@ -97,9 +97,9 @@ pip3 install py-cpuinfo
 pip2 install pylint==1.6.4
 pip3 install pylint==1.6.4
 
-# pep8 tests require the following:
-pip2 install pep8
-pip3 install pep8
+# pycodestyle tests require the following:
+pip2 install pycodestyle
+pip3 install pycodestyle
 
 # tf.mock require the following for python2:
 pip2 install mock
-- 
GitLab


From 316d3c0e7e84b477a08deaa47c65de2fca16bd3f Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Tue, 5 Feb 2019 03:41:23 -0800
Subject: [PATCH 1906/2345] Add rank agnostic max_pool operator

PiperOrigin-RevId: 232457083
---
 tensorflow/python/ops/nn_ops.py               | 114 +++++++++++++++++-
 tensorflow/python/ops/nn_test.py              |  76 ++++++++++++
 .../tools/api/golden/v1/tensorflow.nn.pbtxt   |  10 +-
 .../tools/api/golden/v2/tensorflow.nn.pbtxt   |   6 +-
 tensorflow/tools/compatibility/renames_v2.py  |   4 +-
 tensorflow/tools/compatibility/reorders_v2.py |   1 +
 .../tools/compatibility/tf_upgrade_v2.py      |  12 ++
 .../tools/compatibility/tf_upgrade_v2_test.py |   6 +
 8 files changed, 220 insertions(+), 9 deletions(-)

diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index b6f97033b9..290e146ec1 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -3061,8 +3061,77 @@ def avg_pool(value, ksize, strides, padding, data_format="NHWC", name=None):
         name=name)
 
 
-@tf_export("nn.max_pool")
-def max_pool(value, ksize, strides, padding, data_format="NHWC", name=None):
+# pylint: disable=redefined-builtin
+@tf_export("nn.max_pool", v1=["nn.max_pool_v2"])
+def max_pool_v2(input, ksize, strides, padding, data_format=None, name=None):
+  """Performs the max pooling on the input.
+
+  Args:
+    input:  Tensor of rank N+2, of shape `[batch_size] + input_spatial_shape +
+      [num_channels]` if `data_format` does not start with "NC" (default), or
+      `[batch_size, num_channels] + input_spatial_shape` if data_format starts
+      with "NC". Pooling happens over the spatial dimensions only.
+    ksize: An int or list of `ints` that has length `1`, `N` or `N+2`. The size
+      of the window for each dimension of the input tensor.
+    strides: An int or list of `ints` that has length `1`, `N` or `N+2`. The
+      stride of the sliding window for each dimension of the input tensor.
+    padding: A string, either `'VALID'` or `'SAME'`. The padding algorithm. See
+      the "returns" section of `tf.nn.convolution` for details.
+    data_format: A string. Specifies the channel dimension. For N=1 it can be
+      either "NWC" (default) or "NCW", for N=2 it can be either "NHWC" (default)
+      or "NCHW" and for N=3 either "NDHWC" (default) or "NCDHW".
+    name: Optional name for the operation.
+
+  Returns:
+    A `Tensor` of format specified by `data_format`.
+    The max pooled output tensor.
+  """
+  if input.shape is not None:
+    n = len(input.shape) - 2
+  elif data_format is not None:
+    n = len(data_format) - 2
+  else:
+    raise ValueError(
+        "The input must have a rank or a data format must be given.")
+  if n < 1 or n > 3:
+    raise ValueError(
+        "Input tensor must be of rank 3, 4 or 5 but was {}.".format(n + 2))
+
+  if data_format is None:
+    channel_index = n + 1
+  else:
+    channel_index = 1 if data_format.startswith("NC") else n + 1
+
+  ksize = _get_sequence(ksize, n, channel_index, "ksize")
+  strides = _get_sequence(strides, n, channel_index, "strides")
+
+  max_pooling_ops = {
+      1: max_pool1d,
+      2: gen_nn_ops.max_pool,
+      3: gen_nn_ops.max_pool3d
+  }
+
+  op = max_pooling_ops.get(n)
+  return op(
+      input,
+      ksize=ksize,
+      strides=strides,
+      padding=padding,
+      data_format=data_format,
+      name=name)
+
+
+# pylint: enable=redefined-builtin
+
+
+@tf_export(v1=["nn.max_pool"])
+def max_pool(value,
+             ksize,
+             strides,
+             padding,
+             data_format="NHWC",
+             name=None,
+             input=None):  # pylint: disable=redefined-builtin
   """Performs the max pooling on the input.
 
   Args:
@@ -3075,17 +3144,18 @@ def max_pool(value, ksize, strides, padding, data_format="NHWC", name=None):
       See the "returns" section of `tf.nn.convolution` for details.
     data_format: A string. 'NHWC', 'NCHW' and 'NCHW_VECT_C' are supported.
     name: Optional name for the operation.
+    input: Alias for value.
 
   Returns:
     A `Tensor` of format specified by `data_format`.
     The max pooled output tensor.
   """
+  value = deprecation.deprecated_argument_lookup("input", input, "value", value)
   with ops.name_scope(name, "MaxPool", [value]) as name:
-    value = ops.convert_to_tensor(value, name="input")
     if data_format is None:
       data_format = "NHWC"
-
     channel_index = 1 if data_format.startswith("NC") else 3
+
     ksize = _get_sequence(ksize, 2, channel_index, "ksize")
     strides = _get_sequence(strides, 2, channel_index, "strides")
 
@@ -3139,8 +3209,44 @@ def max_pool1d(input, ksize, strides, padding, data_format="NWC", name=None):
         data_format=data_format,
         name=name)
     return array_ops.squeeze(result, expanding_dim)
+# pylint: enable=redefined-builtin
 
 
+# pylint: disable=redefined-builtin
+@tf_export("nn.max_pool2d")
+def max_pool2d(input, ksize, strides, padding, data_format="NHWC", name=None):
+  """Performs the max pooling on the input.
+
+  Args:
+    input: A 4-D `Tensor` of the format specified by `data_format`.
+    ksize: An int or list of `ints` that has length `1`, `2` or `4`. The size of
+      the window for each dimension of the input tensor.
+    strides: An int or list of `ints` that has length `1`, `2` or `4`. The
+      stride of the sliding window for each dimension of the input tensor.
+    padding: A string, either `'VALID'` or `'SAME'`. The padding algorithm. See
+      the "returns" section of `tf.nn.convolution` for details.
+    data_format: A string. 'NHWC', 'NCHW' and 'NCHW_VECT_C' are supported.
+    name: Optional name for the operation.
+
+  Returns:
+    A `Tensor` of format specified by `data_format`.
+    The max pooled output tensor.
+  """
+  with ops.name_scope(name, "MaxPool2d", [input]) as name:
+    if data_format is None:
+      data_format = "NHWC"
+    channel_index = 1 if data_format.startswith("NC") else 3
+
+    ksize = _get_sequence(ksize, 2, channel_index, "ksize")
+    strides = _get_sequence(strides, 2, channel_index, "strides")
+
+    return gen_nn_ops.max_pool(
+        input,
+        ksize=ksize,
+        strides=strides,
+        padding=padding,
+        data_format=data_format,
+        name=name)
 # pylint: enable=redefined-builtin
 
 
diff --git a/tensorflow/python/ops/nn_test.py b/tensorflow/python/ops/nn_test.py
index 74561349ed..3279dcae0c 100644
--- a/tensorflow/python/ops/nn_test.py
+++ b/tensorflow/python/ops/nn_test.py
@@ -1241,5 +1241,81 @@ class DataFormatVectorPermuteTest(test_lib.TestCase):
       self.assertAllEqual(y_val, [[7, 4], [4, 5], [5, 1], [9, 3]])
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class MaxPoolTest(test_lib.TestCase):
+
+  def test1DTensor(self):
+    x = array_ops.ones([3, 6, 5])
+    ksize = 2
+    strides = 2
+
+    y1 = nn_ops.max_pool_v2(x, ksize, strides, "SAME")
+    y2 = nn_ops.max_pool1d(x, ksize, strides, "SAME")
+
+    self.assertAllEqual(self.evaluate(y1), self.evaluate(y2))
+
+  def test1DNumpy(self):
+    x = np.ones([3, 6, 5])
+    ksize = 2
+    strides = 2
+
+    y1 = nn_ops.max_pool_v2(x, ksize, strides, "SAME")
+    y2 = nn_ops.max_pool1d(x, ksize, strides, "SAME")
+
+    self.assertAllEqual(self.evaluate(y1), self.evaluate(y2))
+
+  def test2DTensor(self):
+    x = array_ops.ones([3, 6, 6, 5])
+    ksize = 2
+    strides = 2
+
+    y1 = nn_ops.max_pool_v2(x, ksize, strides, "SAME")
+    y2 = nn_ops.max_pool(x, ksize, strides, "SAME")
+
+    self.assertAllEqual(self.evaluate(y1), self.evaluate(y2))
+
+  def test2DNumpy(self):
+    x = np.ones([3, 6, 6, 5])
+    ksize = 2
+    strides = 2
+
+    y1 = nn_ops.max_pool_v2(x, ksize, strides, "SAME")
+    y2 = nn_ops.max_pool(x, ksize, strides, "SAME")
+
+    self.assertAllEqual(self.evaluate(y1), self.evaluate(y2))
+
+  def test3DTensor(self):
+    x = array_ops.ones([3, 7, 6, 6, 5])
+    ksize = 2
+    strides = 2
+
+    y1 = nn_ops.max_pool_v2(x, ksize, strides, "SAME")
+    y2 = nn_ops.max_pool3d(x, ksize, strides, "SAME")
+
+    self.assertAllEqual(self.evaluate(y1), self.evaluate(y2))
+
+  def test3DNumpy(self):
+    x = np.ones([3, 7, 6, 6, 5], dtype=np.float32)
+    ksize = 2
+    strides = 2
+
+    y1 = nn_ops.max_pool_v2(x, ksize, strides, "SAME")
+    y2 = nn_ops.max_pool3d(x, ksize, strides, "SAME")
+
+    self.assertAllEqual(self.evaluate(y1), self.evaluate(y2))
+
+  def testIncorrectSizeInputSmall(self):
+    x = array_ops.ones([3, 4])
+    with self.assertRaisesRegex(
+        ValueError, "Input tensor must be of rank 3, 4 or 5 but was 2."):
+      nn_ops.max_pool_v2(x, 2, 2, "SAME")
+
+  def testIncorrectSizeInput(self):
+    x = array_ops.ones([3, 4, 1, 2, 1, 2])
+    with self.assertRaisesRegex(
+        ValueError, "Input tensor must be of rank 3, 4 or 5 but was 6."):
+      nn_ops.max_pool_v2(x, 2, 2, "SAME")
+
+
 if __name__ == "__main__":
   test_lib.main()
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
index cf935dcf85..cf0d9b9b6d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
@@ -234,16 +234,24 @@ tf_module {
   }
   member_method {
     name: "max_pool"
-    argspec: "args=[\'value\', \'ksize\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'None\'], "
+    argspec: "args=[\'value\', \'ksize\', \'strides\', \'padding\', \'data_format\', \'name\', \'input\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'None\', \'None\'], "
   }
   member_method {
     name: "max_pool1d"
     argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'NWC\', \'None\'], "
   }
+  member_method {
+    name: "max_pool2d"
+    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'None\'], "
+  }
   member_method {
     name: "max_pool3d"
     argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'NDHWC\', \'None\'], "
   }
+  member_method {
+    name: "max_pool_v2"
+    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "max_pool_with_argmax"
     argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\', \'Targmax\', \'name\', \'output_dtype\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'None\', \'None\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index ac3fd16224..bfe627f651 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -194,12 +194,16 @@ tf_module {
   }
   member_method {
     name: "max_pool"
-    argspec: "args=[\'value\', \'ksize\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'None\'], "
+    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
   member_method {
     name: "max_pool1d"
     argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'NWC\', \'None\'], "
   }
+  member_method {
+    name: "max_pool2d"
+    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'None\'], "
+  }
   member_method {
     name: "max_pool3d"
     argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'NDHWC\', \'None\'], "
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 8f83448160..c29b259027 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -269,7 +269,6 @@ renames = {
     'tf.keras.initializers.Orthogonal': 'tf.compat.v1.keras.initializers.Orthogonal',
     'tf.keras.initializers.TruncatedNormal': 'tf.compat.v1.keras.initializers.TruncatedNormal',
     'tf.keras.initializers.VarianceScaling': 'tf.compat.v1.keras.initializers.VarianceScaling',
-    'tf.keras.initializers.constant': 'tf.compat.v1.keras.initializers.constant',
     'tf.keras.initializers.glorot_normal': 'tf.compat.v1.keras.initializers.glorot_normal',
     'tf.keras.initializers.glorot_uniform': 'tf.compat.v1.keras.initializers.glorot_uniform',
     'tf.keras.initializers.he_normal': 'tf.compat.v1.keras.initializers.he_normal',
@@ -278,13 +277,11 @@ renames = {
     'tf.keras.initializers.lecun_normal': 'tf.compat.v1.keras.initializers.lecun_normal',
     'tf.keras.initializers.lecun_uniform': 'tf.compat.v1.keras.initializers.lecun_uniform',
     'tf.keras.initializers.normal': 'tf.compat.v1.keras.initializers.normal',
-    'tf.keras.initializers.ones': 'tf.compat.v1.keras.initializers.ones',
     'tf.keras.initializers.orthogonal': 'tf.compat.v1.keras.initializers.orthogonal',
     'tf.keras.initializers.random_normal': 'tf.compat.v1.keras.initializers.random_normal',
     'tf.keras.initializers.random_uniform': 'tf.compat.v1.keras.initializers.random_uniform',
     'tf.keras.initializers.truncated_normal': 'tf.compat.v1.keras.initializers.truncated_normal',
     'tf.keras.initializers.uniform': 'tf.compat.v1.keras.initializers.uniform',
-    'tf.keras.initializers.zeros': 'tf.compat.v1.keras.initializers.zeros',
     'tf.layers.AveragePooling1D': 'tf.compat.v1.layers.AveragePooling1D',
     'tf.layers.AveragePooling2D': 'tf.compat.v1.layers.AveragePooling2D',
     'tf.layers.AveragePooling3D': 'tf.compat.v1.layers.AveragePooling3D',
@@ -433,6 +430,7 @@ renames = {
     'tf.nn.depthwise_conv2d_native_backprop_input': 'tf.nn.depthwise_conv2d_backprop_input',
     'tf.nn.dynamic_rnn': 'tf.compat.v1.nn.dynamic_rnn',
     'tf.nn.log_uniform_candidate_sampler': 'tf.random.log_uniform_candidate_sampler',
+    'tf.nn.max_pool_v2': 'tf.nn.max_pool',
     'tf.nn.quantized_avg_pool': 'tf.compat.v1.nn.quantized_avg_pool',
     'tf.nn.quantized_conv2d': 'tf.compat.v1.nn.quantized_conv2d',
     'tf.nn.quantized_max_pool': 'tf.compat.v1.nn.quantized_max_pool',
diff --git a/tensorflow/tools/compatibility/reorders_v2.py b/tensorflow/tools/compatibility/reorders_v2.py
index ac890e2474..e7edf3f724 100644
--- a/tensorflow/tools/compatibility/reorders_v2.py
+++ b/tensorflow/tools/compatibility/reorders_v2.py
@@ -72,6 +72,7 @@ reorders = {
     'tf.nn.fractional_avg_pool': ['value', 'pooling_ratio', 'pseudo_random', 'overlapping', 'deterministic', 'seed', 'seed2', 'name'],
     'tf.nn.fractional_max_pool': ['value', 'pooling_ratio', 'pseudo_random', 'overlapping', 'deterministic', 'seed', 'seed2', 'name'],
     'tf.nn.in_top_k': ['predictions', 'targets', 'k', 'name'],
+    'tf.nn.max_pool': ['value', 'ksize', 'strides', 'padding', 'data_format', 'name', 'input'],
     'tf.nn.moments': ['x', 'axes', 'shift', 'name', 'keep_dims', 'keepdims'],
     'tf.nn.pool': ['input', 'window_shape', 'pooling_type', 'padding', 'dilation_rate', 'strides', 'name', 'data_format', 'dilations'],
     'tf.nn.separable_conv2d': ['input', 'depthwise_filter', 'pointwise_filter', 'strides', 'padding', 'rate', 'name', 'data_format', 'dilations'],
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 06edb5837c..18271ef87f 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -223,6 +223,9 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.nn.max_pool_with_argmax": {
             "Targmax": "output_dtype",
         },
+        "tf.nn.max_pool": {
+            "value": "input"
+        },
         "tf.multinomial": {
             "output_dtype": "dtype",
         },
@@ -699,6 +702,14 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.compat.v1.assert_rank",
         "tf.contrib.framework.argsort":
             "tf.argsort",
+        "tf.nn.max_pool":
+            "tf.nn.max_pool2d",
+        'tf.keras.initializers.zeros':
+            'tf.compat.v1.keras.initializers.zeros',
+        'tf.keras.initializers.ones':
+            'tf.compat.v1.keras.initializers.ones',
+        'tf.keras.initializers.constant':
+            'tf.compat.v1.keras.initializers.constant',
     }
     # pylint: enable=line-too-long
 
@@ -811,6 +822,7 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.image.sample_distorted_bounding_box",
         "tf.gradients",
         "tf.hessians",
+        "tf.nn.max_pool",
     }
 
     # Functions that were reordered should be changed to the new keyword args
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 3e7041aa03..ede0ccfe17 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -1173,6 +1173,12 @@ def _log_prob(self, x):
     _, _, errors, _ = self._upgrade("tf.flags.FLAGS")
     self.assertIn("tf.flags has been removed", errors[0])
 
+  def test_max_pool_2d(self):
+    text = "tf.nn.max_pool(value=4)"
+    expected_text = "tf.nn.max_pool2d(input=4)"
+    _, _, _, new_text = self._upgrade(text)
+    self.assertEqual(expected_text, new_text)
+
 
 class TestUpgradeFiles(test_util.TensorFlowTestCase):
 
-- 
GitLab


From 134f3fbeadacb8a1096a77e36d4208f20086751f Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Tue, 5 Feb 2019 03:42:14 -0800
Subject: [PATCH 1907/2345] Use Sequence instead of Collection to indicate
 defined ordering.

Addresses feedback on tensorflow/community#56.

PiperOrigin-RevId: 232457135
---
 tensorflow/python/module/module.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/tensorflow/python/module/module.py b/tensorflow/python/module/module.py
index da2d39a7f0..67f19a8eec 100644
--- a/tensorflow/python/module/module.py
+++ b/tensorflow/python/module/module.py
@@ -199,27 +199,27 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
 
   @property
   def variables(self):
-    """Collection of variables owned by this module and it's submodules.
+    """Sequence of variables owned by this module and it's submodules.
 
     Note: this method uses reflection to find variables on the current instance
     and submodules. For performance reasons you may wish to cache the result
     of calling this method if you don't expect the return value to change.
 
     Returns:
-      A collection of variables for the current module (sorted by attribute
+      A sequence of variables for the current module (sorted by attribute
       name) followed by variables from all submodules recursively (depth first).
     """
     return tuple(self._flatten(predicate=_IS_VARIABLE))
 
   @property
   def owned_variables(self):
-    """Collection of variables that are attributes of the current module.
+    """Sequence of variables that are attributes of the current module.
 
     See `variables` for a property which returns all variables from the current
     module and all it's submodules recursively.
 
     Returns:
-      A collection of variables which are attributes of the current module. Will
+      A sequence of variables which are attributes of the current module. Will
       yield variables inside nested structures (lists etc) but not in other
       modules.
     """
@@ -227,27 +227,27 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
 
   @property
   def trainable_variables(self):
-    """Collection of variables owned by this module and it's submodules.
+    """Sequence of variables owned by this module and it's submodules.
 
     Note: this method uses reflection to find variables on the current instance
     and submodules. For performance reasons you may wish to cache the result
     of calling this method if you don't expect the return value to change.
 
     Returns:
-      A collection of variables for the current module (sorted by attribute
+      A sequence of variables for the current module (sorted by attribute
       name) followed by variables from all submodules recursively (depth first).
     """
     return tuple(self._flatten(predicate=_IS_TRAINABLE_VARIABLE))
 
   @property
   def owned_trainable_variables(self):
-    """Collection of variables that are attributes of the current module.
+    """Sequence of variables that are attributes of the current module.
 
     See `variables` for a property which returns all variables from the current
     module and all it's submodules recursively.
 
     Returns:
-      A collection of variables which are attributes of the current module. Will
+      A sequence of variables which are attributes of the current module. Will
       yield variables inside nested structures (lists etc) but not in other
       modules.
     """
@@ -256,7 +256,7 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
 
   @property
   def owned_submodules(self):
-    """Collection of immediate child modules.
+    """Sequence of immediate child modules.
 
     Child modules are modules which are found as properties of the current
     module.
@@ -271,13 +271,13 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
     >>> assert list(c.owned_submodules) == []
 
     Returns:
-      A collection of all child modules.
+      A sequence of all child modules.
     """
     return tuple(self._flatten(recursive=False, predicate=_IS_MODULE))
 
   @property
   def submodules(self):
-    """Collection of all sub-modules.
+    """Sequence of all sub-modules.
 
     Submodules are modules which are properties of this module, or found as
     properties of modules which are properties of this module (and so on).
@@ -292,7 +292,7 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
     >>> assert list(c.submodules) == []
 
     Returns:
-      A collection of all submodules.
+      A sequence of all submodules.
     """
     return tuple(self._flatten(predicate=_IS_MODULE))
 
-- 
GitLab


From e462205a6bb0b797af55d6964552ae21e402f2a6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 5 Feb 2019 06:35:05 -0800
Subject: [PATCH 1908/2345] [TF:XLA] Fallback when XLA constant folding fails.

PiperOrigin-RevId: 232475250
---
 tensorflow/compiler/tf2xla/xla_compiler.cc        | 15 +++++++++++++--
 .../python/kernel_tests/signal/dct_ops_test.py    |  1 -
 .../python/kernel_tests/signal/fft_ops_test.py    | 12 +-----------
 .../kernel_tests/signal/spectral_ops_test.py      |  3 ---
 4 files changed, 14 insertions(+), 17 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/xla_compiler.cc b/tensorflow/compiler/tf2xla/xla_compiler.cc
index 514b156deb..0833264523 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler.cc
@@ -43,6 +43,8 @@ limitations under the License.
 #include "tensorflow/core/graph/algorithm.h"
 #include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/graph/node_builder.h"
+#include "tensorflow/core/lib/core/error_codes.pb.h"
+#include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/gtl/cleanup.h"
 #include "tensorflow/core/lib/hash/hash.h"
 #include "tensorflow/core/platform/logging.h"
@@ -1108,8 +1110,17 @@ Status XlaCompiler::CompileGraph(const XlaCompiler::CompileOptions& options,
   result->outputs.resize(context->retvals().size());
   std::vector<XlaExpression> retvals = context->retvals();
   if (options.resolve_compile_time_constants) {
-    TF_RETURN_IF_ERROR(ResolveConstantExpressionsToConstants(
-        client(), absl::Span<XlaExpression>(retvals)));
+    Status status = ResolveConstantExpressionsToConstants(
+        client(), absl::Span<XlaExpression>(retvals));
+
+    // If the HloEvaluator has not implemented an expression, just evaluate it
+    // at runtime.
+    if (status.code() == error::UNIMPLEMENTED) {
+      ConvertConstantsToExpressions(&builder,
+                                    absl::Span<XlaExpression>(retvals));
+    } else {
+      TF_RETURN_IF_ERROR(status);
+    }
   } else {
     ConvertConstantsToExpressions(&builder, absl::Span<XlaExpression>(retvals));
   }
diff --git a/tensorflow/python/kernel_tests/signal/dct_ops_test.py b/tensorflow/python/kernel_tests/signal/dct_ops_test.py
index e698afde63..a3ac15bab8 100644
--- a/tensorflow/python/kernel_tests/signal/dct_ops_test.py
+++ b/tensorflow/python/kernel_tests/signal/dct_ops_test.py
@@ -134,7 +134,6 @@ class DCTOpsTest(parameterized.TestCase, test.TestCase):
   @parameterized.parameters([
       [[2]], [[3]], [[10]], [[2, 20]], [[2, 3, 25]]])
   @test_util.run_deprecated_v1
-  @test_util.disable_xla("This test never passed for XLA")
   def test_random(self, shape):
     """Test randomly generated batches of data."""
     with spectral_ops_test_util.fft_kernel_label_map():
diff --git a/tensorflow/python/kernel_tests/signal/fft_ops_test.py b/tensorflow/python/kernel_tests/signal/fft_ops_test.py
index f3bee87a65..4577587fa4 100644
--- a/tensorflow/python/kernel_tests/signal/fft_ops_test.py
+++ b/tensorflow/python/kernel_tests/signal/fft_ops_test.py
@@ -159,7 +159,6 @@ class FFTOpsTest(BaseFFTOpsTest):
       raise ValueError("invalid rank")
 
   @test_util.run_deprecated_v1
-  @test_util.disable_xla("This test never passed for XLA")
   def testEmpty(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       for np_type in (np.complex64, np.complex128):
@@ -170,7 +169,6 @@ class FFTOpsTest(BaseFFTOpsTest):
             self.assertEqual(x.shape, self._tfIFFT(x, rank).shape)
 
   @test_util.run_deprecated_v1
-  @test_util.disable_xla("This test never passed for XLA")
   def testBasic(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       for np_type, tol in ((np.complex64, 1e-4), (np.complex128, 1e-8)):
@@ -180,7 +178,6 @@ class FFTOpsTest(BaseFFTOpsTest):
                 np.mod(np.arange(np.power(4, dims)), 10).reshape(
                     (4,) * dims).astype(np_type), rank, rtol=tol, atol=tol)
 
-  @test_util.disable_xla("This test never passed for XLA")
   def testLargeBatch(self):
     if test.is_gpu_available(cuda_only=True):
       rank = 1
@@ -212,7 +209,6 @@ class FFTOpsTest(BaseFFTOpsTest):
                 rank, use_placeholder=True, rtol=tol, atol=tol)
 
   @test_util.run_deprecated_v1
-  @test_util.disable_xla("This test never passed for XLA")
   def testRandom(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       for np_type, tol in ((np.complex64, 1e-4), (np.complex128, 5e-6)):
@@ -228,7 +224,6 @@ class FFTOpsTest(BaseFFTOpsTest):
                           rtol=tol, atol=tol)
 
   @test_util.run_deprecated_v1
-  @test_util.disable_xla("This test never passed for XLA")
   def testRandom1D(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       for np_type in (np.complex64, np.complex128):
@@ -345,7 +340,6 @@ class RFFTOpsTest(BaseFFTOpsTest):
       raise ValueError("invalid rank")
 
   @test_util.run_deprecated_v1
-  @test_util.disable_xla("This test never passed for XLA")
   def testEmpty(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       for rank in VALID_FFT_RANKS:
@@ -356,7 +350,6 @@ class RFFTOpsTest(BaseFFTOpsTest):
           self.assertEqual(x.shape, self._tfIFFT(x, rank).shape)
 
   @test_util.run_deprecated_v1
-  @test_util.disable_xla("This test never passed for XLA")
   def testBasic(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       for rank in VALID_FFT_RANKS:
@@ -371,7 +364,6 @@ class RFFTOpsTest(BaseFFTOpsTest):
             self._compareBackward(
                 c2r.astype(np.complex64), rank, (size,) * rank)
 
-  @test_util.disable_xla("This test never passed for XLA")
   def testLargeBatch(self):
     if test.is_gpu_available(cuda_only=True):
       rank = 1
@@ -405,7 +397,6 @@ class RFFTOpsTest(BaseFFTOpsTest):
                 rank, (size,) * rank,
                 use_placeholder=True)
 
-  @test_util.disable_xla("This test never passed for XLA")
   def testFftLength(self):
     if test.is_gpu_available(cuda_only=True):
       with spectral_ops_test_util.fft_kernel_label_map():
@@ -449,7 +440,6 @@ class RFFTOpsTest(BaseFFTOpsTest):
                   use_placeholder=True)
 
   @test_util.run_deprecated_v1
-  @test_util.disable_xla("This test never passed for XLA")
   def testRandom(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       def gen_real(shape):
@@ -475,7 +465,7 @@ class RFFTOpsTest(BaseFFTOpsTest):
                 gen_complex(complex_dims), rank, (size,) * rank)
 
   @test_util.run_deprecated_v1
-  @test_util.disable_xla("This test never passed for XLA")
+  @test_util.disable_xla("b/123738986")  # More assertions needed.
   def testError(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       for rank in VALID_FFT_RANKS:
diff --git a/tensorflow/python/kernel_tests/signal/spectral_ops_test.py b/tensorflow/python/kernel_tests/signal/spectral_ops_test.py
index a72a836592..7b9748c7f2 100644
--- a/tensorflow/python/kernel_tests/signal/spectral_ops_test.py
+++ b/tensorflow/python/kernel_tests/signal/spectral_ops_test.py
@@ -21,7 +21,6 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import math_ops
@@ -116,7 +115,6 @@ class SpectralOpsTest(test.TestCase):
       self.assertAllClose(
           expected_inverse_stft, actual_inverse_stft, 1e-4, 1e-4)
 
-  @test_util.disable_xla("This test never passed for XLA")
   def test_shapes(self):
     with spectral_ops_test_util.fft_kernel_label_map(), (
         self.session(use_gpu=True)):
@@ -152,7 +150,6 @@ class SpectralOpsTest(test.TestCase):
       self.assertAllEqual([256], inverse_stft.shape.as_list())
       self.assertAllEqual([expected_length], self.evaluate(inverse_stft).shape)
 
-  @test_util.disable_xla("This test never passed for XLA")
   def test_stft_and_inverse_stft(self):
     """Test that spectral_ops.stft/inverse_stft match a NumPy implementation."""
     # Tuples of (signal_length, frame_length, frame_step, fft_length).
-- 
GitLab


From 441fb2b81ebd947ab287f0191ed69160e71eece5 Mon Sep 17 00:00:00 2001
From: Shanqing Cai <cais@google.com>
Date: Tue, 5 Feb 2019 06:49:58 -0800
Subject: [PATCH 1909/2345] tfdbg: start deprecating invoke_stepper
 PiperOrigin-RevId: 232476961

---
 tensorflow/python/debug/BUILD              |  6 +++++
 tensorflow/python/debug/cli/stepper_cli.py | 30 +++++++++++++++++++---
 2 files changed, 33 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/debug/BUILD b/tensorflow/python/debug/BUILD
index 4634f505f6..dbe738b703 100644
--- a/tensorflow/python/debug/BUILD
+++ b/tensorflow/python/debug/BUILD
@@ -1015,6 +1015,12 @@ cuda_py_test(
         "//tensorflow/python:training",
         "//tensorflow/python:variables",
     ],
+    tags = [
+        "manual",
+        "no_pip",
+        "no_windows",
+        "notap",
+    ],
 )
 
 cuda_py_test(
diff --git a/tensorflow/python/debug/cli/stepper_cli.py b/tensorflow/python/debug/cli/stepper_cli.py
index 94eb2754da..fe1a012a54 100644
--- a/tensorflow/python/debug/cli/stepper_cli.py
+++ b/tensorflow/python/debug/cli/stepper_cli.py
@@ -251,6 +251,9 @@ class NodeStepperCLI(object):
       lines.extend(
           ["Topologically-sorted transitive input(s) and fetch(es):", ""])
 
+    output = debugger_cli_common.rich_text_lines_from_rich_line_list(lines)
+    self._add_deprecation_warning(output)
+
     for i, element_name in enumerate(self._sorted_nodes):
       if i < index_range[0] or i >= index_range[1]:
         continue
@@ -269,15 +272,36 @@ class NodeStepperCLI(object):
           override_names,
           dirty_variable_names)
 
-      lines.append(node_prefix + "] " + element_name)
-
-    output = debugger_cli_common.rich_text_lines_from_rich_line_list(lines)
+      output.append_rich_line(node_prefix + "] " + element_name)
 
     if verbose:
       output.extend(self._node_status_label_legend())
 
     return output
 
+  def _add_deprecation_warning(self, message):
+    """Add deprecation warning as RichTextLines."""
+    color = "yellow"
+    message.append_rich_line(
+        debugger_cli_common.RichLine(
+            "WARNING: the invoke_stepper feature of tfdbg has been deprecated ",
+            color))
+    message.append_rich_line(
+        debugger_cli_common.RichLine(
+            "and will be removed in the next release of TensorFlow.",
+            color))
+    message.append_rich_line(debugger_cli_common.RichLine("", color))
+    message.append_rich_line(
+        debugger_cli_common.RichLine(
+            "There now exist better alternatives of stepping debugging, "
+            "including:",
+            color))
+    message.append_rich_line(
+        debugger_cli_common.RichLine("- TensorBoard Debugger Plugin", color))
+    message.append_rich_line(
+        debugger_cli_common.RichLine("- Eager Execution", color))
+    message.append_rich_line(debugger_cli_common.RichLine("", color))
+
   def _get_status_labels(self,
                          element_name,
                          handle_node_names,
-- 
GitLab


From 4892db4ddb94ba612eab7fd41e209830dc65441d Mon Sep 17 00:00:00 2001
From: Grzegorz Pawelczak <grzegorzp@graphcore.ai>
Date: Tue, 5 Feb 2019 16:21:41 +0100
Subject: [PATCH 1910/2345] Use asserts rather than expect

---
 .../xla/service/hlo_tfgraph_builder_test.cc         | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_tfgraph_builder_test.cc b/tensorflow/compiler/xla/service/hlo_tfgraph_builder_test.cc
index e5e9517c90..498abcfe04 100644
--- a/tensorflow/compiler/xla/service/hlo_tfgraph_builder_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_tfgraph_builder_test.cc
@@ -17,6 +17,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/tests/hlo_test_base.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/tensor_shape.pb.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
 
 namespace xla {
 namespace hlo_graph_dumper {
@@ -185,14 +186,14 @@ TEST_F(HloTfGraphBuilderTest, TokenHasNoLayout) {
   metadata.set_op_name("x");
   metadata.set_op_type("y");
   token->set_metadata(metadata);
-  TF_CHECK_OK(generator_.AddComputation(*builder.Build()));
+  TF_ASSERT_OK(generator_.AddComputation(*builder.Build()));
   GraphDef graph_def = generator_.GetGraphDef();
-  EXPECT_EQ(graph_def.node_size(), 1);
+  ASSERT_EQ(graph_def.node_size(), 1);
   const auto &node = graph_def.node(0);
-  EXPECT_EQ(GetNodeAttr(node, "type").s(), "TOKEN");
-  EXPECT_EQ(GetNodeAttr(node, "layout").s(), "");
-  EXPECT_EQ(GetNodeAttr(node, "tf_op_name").s(), "x");
-  EXPECT_EQ(GetNodeAttr(node, "tf_op_type").s(), "y");
+  ASSERT_EQ(GetNodeAttr(node, "type").s(), "TOKEN");
+  ASSERT_EQ(GetNodeAttr(node, "layout").s(), "");
+  ASSERT_EQ(GetNodeAttr(node, "tf_op_name").s(), "x");
+  ASSERT_EQ(GetNodeAttr(node, "tf_op_type").s(), "y");
 }
 
 }  // namespace
-- 
GitLab


From 95a722499340d2ee9b9f4d7e3af6d9be3d71276d Mon Sep 17 00:00:00 2001
From: Doe Hyun Yoon <dyoon@google.com>
Date: Tue, 5 Feb 2019 08:45:13 -0800
Subject: [PATCH 1911/2345] Fix the condition in ShouldUpdateOutputValue() in
 static shape inference.

When using aggressive_shape_inference, running EvaluateNode() takes too much resources
(compute and memory) if input and output tensors are large. Previously, set 17 as
threshold for small / large tensors, but the condition check logic was incorrect.

This CL fixes that and avoid costly EvaluateNode() in case output tensor are
huge (e.g., Fill op that takes a shape describing large tensors).

PiperOrigin-RevId: 232493327
---
 .../core/grappler/costs/graph_properties.cc   |  2 +-
 .../grappler/costs/graph_properties_test.cc   | 46 +++++++++++++++++++
 2 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/grappler/costs/graph_properties.cc b/tensorflow/core/grappler/costs/graph_properties.cc
index d0ac87c962..8ec558be7d 100644
--- a/tensorflow/core/grappler/costs/graph_properties.cc
+++ b/tensorflow/core/grappler/costs/graph_properties.cc
@@ -1181,7 +1181,7 @@ class SymbolicShapeRefiner {
     // elements is larger than the given max size.
     for (int i = 0; i < ic->num_outputs(); i++) {
       const ShapeHandle& shape_handle = ic->output(i);
-      if (!ic->FullyDefined(shape_handle) &&
+      if (!ic->FullyDefined(shape_handle) ||
           ic->Value(ic->NumElements(shape_handle)) > max_size) {
         return false;
       }
diff --git a/tensorflow/core/grappler/costs/graph_properties_test.cc b/tensorflow/core/grappler/costs/graph_properties_test.cc
index 0a7697a213..fa6b05bd15 100644
--- a/tensorflow/core/grappler/costs/graph_properties_test.cc
+++ b/tensorflow/core/grappler/costs/graph_properties_test.cc
@@ -975,6 +975,52 @@ TEST_F(GraphPropertiesTest, IdentityPassingShape) {
   EXPECT_EQ("float: [5,5]", PropToString(out_prop0));
 }
 
+TEST_F(GraphPropertiesTest, SkippingValueInferenceForLargeTensors) {
+  // When using aggressive_shape_inference, we run EvaluateNode() for
+  // whitelisted ops and small input / output tensors. For instance, Fill op is
+  // evaluated and produces output tensor value if output tensor size is smal
+  // (currently, fewer than 17 elements); otherwise we don't run EvalauteNode().
+  // This is to avoid wasting time and memory for producing huge tensors (e.g.,
+  // initializing a large table using Fill.
+  {
+    tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+    Output a = ops::Const(s.WithOpName("a"), 4, {2});  // 4x4
+    Output b = ops::Const(s.WithOpName("const"), 0.1f, {});
+    // Shape described by a is small; expect output values of Fill op.
+    Output c = ops::Fill(s.WithOpName("fill"), a, b);
+
+    GrapplerItem item;
+    TF_CHECK_OK(s.ToGraphDef(&item.graph));
+    GraphProperties properties(item);
+    TF_CHECK_OK(properties.InferStatically(
+        /*assume_valid_feeds=*/false,
+        /*aggressive_shape_inference=*/true));
+    const auto out_props = properties.GetOutputProperties("fill");
+    const OpInfo::TensorProperties out_prop0 = out_props[0];
+    EXPECT_EQ("float: [4,4]", PropToString(out_prop0));
+    EXPECT_TRUE(out_prop0.has_value());
+  }
+  {
+    tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+    Output a = ops::Const(s.WithOpName("a"), 1000, {4});  // 1000x1000x1000x1000
+    Output b = ops::Const(s.WithOpName("const"), 0.1f, {});
+    // Shape described by a is huge; in that case we skip value inference.
+    // Otherwise, it'd be too much overhead.
+    Output c = ops::Fill(s.WithOpName("fill"), a, b);
+
+    GrapplerItem item;
+    TF_CHECK_OK(s.ToGraphDef(&item.graph));
+    GraphProperties properties(item);
+    TF_CHECK_OK(properties.InferStatically(
+        /*assume_valid_feeds=*/false,
+        /*aggressive_shape_inference=*/true));
+    const auto out_props = properties.GetOutputProperties("fill");
+    const OpInfo::TensorProperties out_prop0 = out_props[0];
+    EXPECT_EQ("float: [1000,1000,1000,1000]", PropToString(out_prop0));
+    EXPECT_FALSE(out_prop0.has_value());
+  }
+}
+
 TEST_F(GraphPropertiesTest, PackWithConstInput) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output a = ops::Const(s.WithOpName("a"), 1, {});
-- 
GitLab


From af3b41caa4dcda87837bb73b2c6888c6544c2b5d Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Tue, 5 Feb 2019 09:01:09 -0800
Subject: [PATCH 1912/2345] [tf.data] Make map_vectorization test v2 compatible

PiperOrigin-RevId: 232496009
---
 .../optimization/map_vectorization_test.py    | 31 ++++++++++++-------
 .../python/data/kernel_tests/test_base.py     |  2 ++
 2 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
index bba2f352e2..d69043c7ad 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
@@ -26,6 +26,7 @@ from tensorflow.python.data.experimental.ops import batching
 from tensorflow.python.data.experimental.ops import optimization
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -378,6 +379,7 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
     def map_fn(x):
       # x has leading dimension 5, this will raise an error
       return array_ops.gather(x, 10)
+
     with self.assertRaisesRegexp(errors.InvalidArgumentError,
                                  r"indices = 10 is not in \[0, 5\)"):
       base_dataset = dataset_ops.Dataset.range(5).repeat(5).batch(
@@ -478,8 +480,7 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     self.assertDatasetsEqual(optimized, unoptimized)
 
-  # TODO(b/117581999): Add eager coverage for the following tests.
-  def testSkipEagerOptimizationIgnoreStateful(self):
+  def testOptimizationIgnoreStateful(self):
 
     def map_fn(x):
       with ops.control_dependencies([check_ops.assert_equal(x, 0)]):
@@ -489,10 +490,13 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
                                                            [3, 4]]).repeat(5)
     unoptimized, optimized = self._get_test_datasets(
         base_dataset, map_fn, expect_optimized=False)
-    self.assertDatasetsRaiseSameError(
-        unoptimized, optimized, errors.InvalidArgumentError,
-        [("OneShotIterator", "OneShotIterator_1", 1),
-         ("IteratorGetNext", "IteratorGetNext_1", 1)])
+    replacements = None
+    if not context.executing_eagerly():
+      # In graph mode, the ops have unique names.
+      replacements = [("OneShotIterator", "OneShotIterator_1", 1),
+                      ("IteratorGetNext", "IteratorGetNext_1", 1)]
+    self.assertDatasetsRaiseSameError(unoptimized, optimized,
+                                      errors.InvalidArgumentError, replacements)
 
   def testOptimizationIgnoreRagged(self):
     # Make sure we ignore inputs that might not be uniformly sized
@@ -505,8 +509,7 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
         base_dataset, map_fn, expect_optimized=False)
     self.assertDatasetsEqual(unoptimized, optimized)
 
-  # TODO(b/117581999): Add eager coverage for the following tests.
-  def testSkipEagerOptimizationIgnoreRaggedMap(self):
+  def testOptimizationIgnoreRaggedMap(self):
     # Don't optimize when the output of the map fn shapes are unknown.
     def map_fn(x):
       return array_ops.tile(x, x)
@@ -514,10 +517,13 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
     base_dataset = dataset_ops.Dataset.range(20).batch(1, drop_remainder=True)
     unoptimized, optimized = self._get_test_datasets(
         base_dataset, map_fn, expect_optimized=False)
-    self.assertDatasetsRaiseSameError(
-        unoptimized, optimized, errors.InvalidArgumentError,
-        [("OneShotIterator", "OneShotIterator_1", 1),
-         ("IteratorGetNext", "IteratorGetNext_1", 1)])
+    replacements = None
+    if not context.executing_eagerly():
+      # In graph mode, the ops have unique names.
+      replacements = [("OneShotIterator", "OneShotIterator_1", 1),
+                      ("IteratorGetNext", "IteratorGetNext_1", 1)]
+    self.assertDatasetsRaiseSameError(unoptimized, optimized,
+                                      errors.InvalidArgumentError, replacements)
 
   def testOptimizationWithUnknownBatchShape(self):
     tensor = sparse_tensor.SparseTensor(
@@ -535,5 +541,6 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
     optimized = unoptimized.with_options(options)
     self.assertDatasetsEqual(unoptimized, optimized)
 
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/data/kernel_tests/test_base.py b/tensorflow/python/data/kernel_tests/test_base.py
index 7aa7f33003..19a1a0de6d 100644
--- a/tensorflow/python/data/kernel_tests/test_base.py
+++ b/tensorflow/python/data/kernel_tests/test_base.py
@@ -178,6 +178,8 @@ class DatasetTestBase(test.TestCase):
                                    exception_class,
                                    replacements=None):
     """Checks that datasets raise the same error on the first get_next call."""
+    if replacements is None:
+      replacements = []
     next1 = self.getNext(dataset1)
     next2 = self.getNext(dataset2)
     try:
-- 
GitLab


From 43633e0c8119b7285e714617df4cd28550ed4661 Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Tue, 5 Feb 2019 09:04:01 -0800
Subject: [PATCH 1913/2345] [XLA] Clean up some R1
 DynamicSlice/DynamicUpdateSlice code

None of this should be reachable anymore, except when bypassing the HLO optimization stage.

PiperOrigin-RevId: 232496810
---
 .../xla/service/elemental_ir_emitter.cc       | 32 ++++------------
 .../llvm_ir/dynamic_update_slice_util.cc      | 38 +++++--------------
 2 files changed, 17 insertions(+), 53 deletions(-)

diff --git a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
index 727e0bfa52..ccdc12a30a 100644
--- a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
@@ -1767,18 +1767,10 @@ StatusOr<llvm::Value*> ElementalIrEmitter::EmitElementalDynamicSlice(
     auto index_typed_const = [&](uint64 c) -> llvm::Constant* {
       return llvm::ConstantInt::get(index_type, c);
     };
-    // TODO(b/118437727): Remove the R1 path.
-    llvm::Value* start_index_value;
-    if (hlo->operand(1)->shape().rank() == 1) {
-      llvm_ir::IrArray::Index dim_index(1, index_typed_const(i));
-      TF_ASSIGN_OR_RETURN(start_index_value,
-                          operand_to_generator.at(hlo->operand(1))(dim_index));
-    } else {
-      llvm_ir::IrArray::Index zero_index(index_type);
-      TF_ASSIGN_OR_RETURN(
-          start_index_value,
-          operand_to_generator.at(hlo->operand(1 + i))(zero_index));
-    }
+    llvm_ir::IrArray::Index zero_index(index_type);
+    TF_ASSIGN_OR_RETURN(
+        llvm::Value * start_index_value,
+        operand_to_generator.at(hlo->operand(1 + i))(zero_index));
 
     // Clamp the start index so that the sliced portion fits in the operand:
     // start_index = clamp(start_index, 0, operand_dim_size - output_dim_size)
@@ -1924,18 +1916,10 @@ StatusOr<llvm::Value*> ElementalIrEmitter::EmitElementalDynamicUpdateSlice(
       return llvm::ConstantInt::get(index_type, c);
     };
 
-    llvm::Value* start_index_value;
-    // TODO(b/118437727): Remove the R1 path.
-    if (hlo->operand(2)->shape().rank() == 1) {
-      llvm_ir::IrArray::Index dim_index(1, index_typed_const(i));
-      TF_ASSIGN_OR_RETURN(start_index_value,
-                          operand_to_generator.at(hlo->operand(2))(dim_index));
-    } else {
-      llvm_ir::IrArray::Index zero_index(index_type);
-      TF_ASSIGN_OR_RETURN(
-          start_index_value,
-          operand_to_generator.at(hlo->operand(2 + i))(zero_index));
-    }
+    llvm_ir::IrArray::Index zero_index(index_type);
+    TF_ASSIGN_OR_RETURN(
+        llvm::Value * start_index_value,
+        operand_to_generator.at(hlo->operand(2 + i))(zero_index));
 
     // Clamp the start index so that the update region fits in the operand.
     // start_index = clamp(start_index, 0, input_dim_size - update_dim_size)
diff --git a/tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.cc b/tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.cc
index c66eaec8fb..3acceccfa5 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.cc
@@ -113,20 +113,10 @@ Status EmitDynamicUpdateSliceInPlace(absl::Span<const IrArray> operand_arrays,
   Shape output_shape = output_array.GetShape();
   Shape update_shape = update_array.GetShape();
 
-  IndexGenerator start_indices_generator;
-  // TODO(b/118437727): Remove the R1 path, and rename the variables.
-  if (start_indices_array.GetShape().rank() == 1) {
-    start_indices_generator = [&](int64 index) {
-      return start_indices_array.EmitReadArrayElement(
-          IrArray::Index({b->getInt64(index)}), b);
-    };
-  } else {
-    start_indices_generator = [&](int64 index) {
-      return operand_arrays[2 + index].EmitReadArrayElement(
-          IrArray::Index(b->getInt64Ty()), b);
-    };
-  }
-
+  IndexGenerator start_indices_generator = [&](int64 index) {
+    return operand_arrays[2 + index].EmitReadArrayElement(
+        IrArray::Index(b->getInt64Ty()), b);
+  };
   ElementGenerator update_array_generator = [&](const IrArray::Index& index) {
     return update_array.EmitReadArrayElement(index, b);
   };
@@ -178,21 +168,11 @@ static Status EmitFusedDynamicUpdateSliceInPlaceImpl(
   TF_RETURN_IF_ERROR(dynamic_update_slice->Accept(&fused_emitter));
   ElementGenerator update_array_generator = fused_emitter.GetGenerator(update);
 
-  // TODO(b/118437727): Remove the R1 path, and rename the variables.
-  IndexGenerator start_indices_generator;
-  if (start_indices->shape().rank() == 1) {
-    start_indices_generator = [&](int64 index) {
-      return fused_emitter.GetGenerator(start_indices)(
-          IrArray::Index({b->getInt64(index)}));
-    };
-  } else {
-    start_indices_generator = [&](int64 index) {
-      ElementGenerator element_generator =
-          fused_emitter.GetGenerator(dynamic_update_slice->operand(2 + index));
-      return element_generator(IrArray::Index(b->getInt64Ty()));
-    };
-  }
-
+  IndexGenerator start_indices_generator = [&](int64 index) {
+    ElementGenerator element_generator =
+        fused_emitter.GetGenerator(dynamic_update_slice->operand(2 + index));
+    return element_generator(IrArray::Index(b->getInt64Ty()));
+  };
   bool is_signed = ShapeUtil::ElementIsSigned(start_indices->shape());
   return EmitDynamicUpdateSliceInPlaceImpl(
       update_shape, start_indices_generator, is_signed, update_array_generator,
-- 
GitLab


From 930108569819c13e12c8c6919596d2899c410911 Mon Sep 17 00:00:00 2001
From: Andr? Susano Pinto <andresp@google.com>
Date: Tue, 5 Feb 2019 09:17:53 -0800
Subject: [PATCH 1914/2345] Refactor code for listing checkpointable functions
 of an object.

The main goal is to move the logic of "dir+getattr" from all checkpointable
objects to just "AutoCheckpointable". Similar to how checkpointable base
does not enforces

The refactoring also provides an internal mechanism for a class to provide
its own listing of functions if needed and makes obsolete the need to register
a getter and an attribute extractor.

PiperOrigin-RevId: 232499034
---
 tensorflow/python/keras/engine/training.py    |  7 ++--
 .../python/saved_model/revived_types.py       | 20 +----------
 tensorflow/python/saved_model/save.py         | 20 +----------
 .../python/training/checkpointable/base.py    | 13 ++++++++
 .../checkpointable/data_structures.py         | 33 +++++++++++--------
 .../training/checkpointable/tracking.py       | 17 ++++++++++
 6 files changed, 56 insertions(+), 54 deletions(-)

diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 8e1a962610..e93f2e0523 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -1784,9 +1784,10 @@ class Model(Network):
       batch_size = 32
     return batch_size
 
-  @property
-  def _default_save_signature(self):
-    return saving_utils.trace_model_call(self)
+  def _list_functions_for_serialization(self):
+    return {
+        '_default_save_signature': saving_utils.trace_model_call(self)
+    }
 
   def _set_sample_weight_attributes(self, sample_weight_mode,
                                     skip_target_weighing_indices):
diff --git a/tensorflow/python/saved_model/revived_types.py b/tensorflow/python/saved_model/revived_types.py
index 8f82039cfd..ae063202bb 100644
--- a/tensorflow/python/saved_model/revived_types.py
+++ b/tensorflow/python/saved_model/revived_types.py
@@ -26,8 +26,7 @@ class VersionedTypeRegistration(object):
   """Holds information about one version of a revived type."""
 
   def __init__(self, object_factory, version, min_producer_version,
-               min_consumer_version, bad_consumers=None, setter=setattr,
-               getter=getattr, attribute_extractor=dir):
+               min_consumer_version, bad_consumers=None, setter=setattr):
     """Identify a revived type version.
 
     Args:
@@ -60,16 +59,8 @@ class VersionedTypeRegistration(object):
         addition to any version less than `min_consumer_version`).
       setter: A callable with the same signature as `setattr` to use when adding
         dependencies to generated objects.
-      getter: A callable with the same signature as `getattr` to use when
-        retrieving items from objects of this type. Used along with
-        `attribute_extractor` to find functions, which are not Checkpointable
-        objects and so not regular dependencies.
-      attribute_extractor: A callable equivalent of the builtin `dir`, used for
-        listing items in this container (if any).
     """
     self.setter = setter
-    self.getter = getter
-    self.attribute_extractor = attribute_extractor
     self.identifier = None  # Set after registration
     self._object_factory = object_factory
     self.version = version
@@ -146,15 +137,6 @@ def register_revived_type(identifier, predicate, versions):
   _TYPE_IDENTIFIERS.append(identifier)
 
 
-def get_attribute_extractors(obj):
-  """Get a `dir` and `getattr` equivalent for use with `obj`."""
-  for identifier in _TYPE_IDENTIFIERS:
-    predicate, versions = _REVIVED_TYPE_REGISTRY[identifier]
-    if predicate(obj):
-      return versions[0].attribute_extractor, versions[0].getter
-  return dir, getattr
-
-
 def serialize(obj):
   """Create a SavedUserObject from a checkpointable object."""
   for identifier in _TYPE_IDENTIFIERS:
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index 1f16748909..3c8bf1bf57 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -91,7 +91,7 @@ class _SaveableView(object):
     nodes_without_functions = list(self.nodes)
     seen_function_names = set()
     for obj in nodes_without_functions:
-      self.functions[obj] = self._list_functions(obj)
+      self.functions[obj] = obj._list_functions_for_serialization()  # pylint: disable=protected-access
       for function in self.functions[obj].values():
         if function not in self.node_ids:
           self.node_ids[function] = len(self.nodes)
@@ -137,24 +137,6 @@ class _SaveableView(object):
         child_proto.node_id = self.node_ids[ref_function]
         child_proto.local_name = local_name
 
-  def _list_functions(self, checkpointable_object):
-    """Return a dict of `Function`s of a checkpointable."""
-    functions = dict()
-    attribute_extractor, attribute_getter = (
-        revived_types.get_attribute_extractors(checkpointable_object))
-    for attribute_name in attribute_extractor(checkpointable_object):
-      try:
-        attribute_value = attribute_getter(
-            checkpointable_object, attribute_name, None)
-      except Exception:  # pylint: disable=broad-except
-        # We really don't want to throw an exception just because some object's
-        # attribute accessor is broken.
-        attribute_value = None
-      if isinstance(attribute_value, (def_function.Function,
-                                      defun.ConcreteFunction)):
-        functions[attribute_name] = attribute_value
-    return functions
-
   def map_resources(self):
     """Makes new resource handle ops corresponding to existing resource tensors.
 
diff --git a/tensorflow/python/training/checkpointable/base.py b/tensorflow/python/training/checkpointable/base.py
index a2c93cd110..0d659cef20 100644
--- a/tensorflow/python/training/checkpointable/base.py
+++ b/tensorflow/python/training/checkpointable/base.py
@@ -862,3 +862,16 @@ class Checkpointable(object):
     return {OBJECT_CONFIG_JSON_KEY: functools.partial(
         PythonStringStateSaveable,
         state_callback=_state_callback)}
+
+  def _list_functions_for_serialization(self):
+    """Lists the functions of this checkpointable to serialize.
+
+    Internal sub-classes can override this with specific logic. E.g.
+    `AutoCheckpointable` provides an implementation that returns the `attr`
+    that return functions.
+
+    Returns:
+        A dictionary mapping attribute names to `Function` or
+        `ConcreteFunction`.
+    """
+    return dict()
diff --git a/tensorflow/python/training/checkpointable/data_structures.py b/tensorflow/python/training/checkpointable/data_structures.py
index c86846f8e2..ae3ab3fd17 100644
--- a/tensorflow/python/training/checkpointable/data_structures.py
+++ b/tensorflow/python/training/checkpointable/data_structures.py
@@ -24,6 +24,8 @@ import sys
 
 import six
 
+from tensorflow.python.eager import def_function
+from tensorflow.python.eager import function as defun
 from tensorflow.python.ops import variables
 from tensorflow.python.saved_model import revived_types
 from tensorflow.python.training.checkpointable import base
@@ -525,6 +527,12 @@ class _ListWrapper(List, collections.MutableSequence,
   def __repr__(self):
     return "ListWrapper(%s)" % (repr(self._storage),)
 
+  def _list_functions_for_serialization(self):
+    return {
+        str(key): value for key, value in enumerate(self)
+        if _is_function(value)
+    }
+
 
 class Mapping(CheckpointableDataStructure, collections.Mapping):
   """An append-only checkpointable mapping data structure with string keys.
@@ -793,6 +801,16 @@ class _DictWrapper(Mapping, collections.MutableMapping):
     for key, value in dict(*args, **kwargs).items():
       self[key] = value
 
+  def _list_functions_for_serialization(self):
+    return {
+        key: value for key, value in self.items()
+        if _is_function(value)
+    }
+
+
+def _is_function(x):
+  return isinstance(x, (def_function.Function, defun.ConcreteFunction))
+
 revived_types.register_revived_type(
     "checkpointable_dict_wrapper",
     lambda obj: isinstance(obj, _DictWrapper),
@@ -803,9 +821,7 @@ revived_types.register_revived_type(
         version=1,
         min_producer_version=1,
         min_consumer_version=1,
-        setter=operator.setitem,
-        getter=_DictWrapper.get,
-        attribute_extractor=lambda obj: obj.keys())])
+        setter=operator.setitem)])
 
 
 def _set_list_item(list_object, index_string, value):
@@ -815,13 +831,6 @@ def _set_list_item(list_object, index_string, value):
   list_object[item_index] = value
 
 
-def _list_getter(obj, item, default=None):
-  index = int(item)
-  if index < len(obj):
-    return obj[index]
-  return default
-
-
 revived_types.register_revived_type(
     "checkpointable_list_wrapper",
     lambda obj: isinstance(obj, _ListWrapper),
@@ -830,6 +839,4 @@ revived_types.register_revived_type(
         version=1,
         min_producer_version=1,
         min_consumer_version=1,
-        setter=_set_list_item,
-        getter=_list_getter,
-        attribute_extractor=lambda obj: [str(i) for i in range(len(obj))])])
+        setter=_set_list_item)])
diff --git a/tensorflow/python/training/checkpointable/tracking.py b/tensorflow/python/training/checkpointable/tracking.py
index f1ef5cca49..83b0d8bcf2 100644
--- a/tensorflow/python/training/checkpointable/tracking.py
+++ b/tensorflow/python/training/checkpointable/tracking.py
@@ -18,6 +18,8 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.eager import context
+from tensorflow.python.eager import def_function
+from tensorflow.python.eager import function as defun
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import resource_variable_ops
@@ -91,6 +93,21 @@ class AutoCheckpointable(base.Checkpointable):
     """Override to allow CheckpointableBase to disable dependency tracking."""
     return data_structures.NoDependency(value)
 
+  def _list_functions_for_serialization(self):
+    """Return a dict of `Function`s of a checkpointable."""
+    functions = dict()
+    for attribute_name in dir(self):
+      try:
+        attribute_value = getattr(self, attribute_name, None)
+      except Exception:  # pylint: disable=broad-except
+        # We really don't want to throw an exception just because some object's
+        # attribute accessor is broken.
+        attribute_value = None
+      if isinstance(attribute_value, (def_function.Function,
+                                      defun.ConcreteFunction)):
+        functions[attribute_name] = attribute_value
+    return functions
+
 
 class ResourceTracker(object):
   """An object that tracks a list of resources."""
-- 
GitLab


From 15f93e11b05802ca9318a5707cdcd859a15cd019 Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Tue, 5 Feb 2019 09:27:01 -0800
Subject: [PATCH 1915/2345] Standardize the inputs for conv2d and
 conv2d_transpose

PiperOrigin-RevId: 232500504
---
 .../quantize/python/fold_batch_norms.py       |  4 +-
 tensorflow/python/eager/backprop.py           |  2 +
 tensorflow/python/ops/nn_grad.py              | 24 +++----
 tensorflow/python/ops/nn_ops.py               | 63 ++++++++++++-------
 4 files changed, 55 insertions(+), 38 deletions(-)

diff --git a/tensorflow/contrib/quantize/python/fold_batch_norms.py b/tensorflow/contrib/quantize/python/fold_batch_norms.py
index e0c6da00d8..a70f748fad 100644
--- a/tensorflow/contrib/quantize/python/fold_batch_norms.py
+++ b/tensorflow/contrib/quantize/python/fold_batch_norms.py
@@ -454,7 +454,7 @@ def _CloneWithNewOperands(layer_op, input_tensor, weight_tensor,
         strides=layer_op.get_attr('strides'),
         padding=layer_op.get_attr('padding'),
         use_cudnn_on_gpu=layer_op.get_attr('use_cudnn_on_gpu'),
-        data_format=layer_op.get_attr('data_format'),
+        data_format=layer_op.get_attr('data_format').decode(),
         name=new_layer_name)
   elif layer_op.type == 'MatMul':
     return math_ops.matmul(
@@ -867,7 +867,7 @@ class _OpCloner(object):
         strides=op.get_attr('strides'),
         padding=op.get_attr('padding'),
         use_cudnn_on_gpu=op.get_attr('use_cudnn_on_gpu'),
-        data_format=op.get_attr('data_format'),
+        data_format=op.get_attr('data_format').decode(),
         name=new_name).op
 
   def _CloneDepthwiseConv2d(self, op, inputs, new_name):
diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index 6117d8a4ea..694b05c1ee 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -80,6 +80,8 @@ def make_attr(attr_type, value):
     return tensor_shape.as_shape(value).as_proto()
   elif attr_type == [pywrap_tensorflow.TF_ATTR_SHAPE]:
     return [tensor_shape.as_shape(v).as_proto() for v in value]
+  elif isinstance(value, str):
+    return value.encode()
   return value
 
 
diff --git a/tensorflow/python/ops/nn_grad.py b/tensorflow/python/ops/nn_grad.py
index 6ca2b2aafe..a3d3c7b4ef 100644
--- a/tensorflow/python/ops/nn_grad.py
+++ b/tensorflow/python/ops/nn_grad.py
@@ -50,7 +50,7 @@ def _Conv2DBackpropInputGrad(op, grad):
           strides=op.get_attr("strides"),
           padding=op.get_attr("padding"),
           use_cudnn_on_gpu=op.get_attr("use_cudnn_on_gpu"),
-          data_format=op.get_attr("data_format")),
+          data_format=op.get_attr("data_format").decode()),
       nn_ops.conv2d(
           grad,
           op.inputs[1],
@@ -58,7 +58,7 @@ def _Conv2DBackpropInputGrad(op, grad):
           strides=op.get_attr("strides"),
           padding=op.get_attr("padding"),
           use_cudnn_on_gpu=op.get_attr("use_cudnn_on_gpu"),
-          data_format=op.get_attr("data_format"))
+          data_format=op.get_attr("data_format").decode())
   ]
 
 
@@ -73,7 +73,7 @@ def _Conv2DBackpropFilterGrad(op, grad):
           strides=op.get_attr("strides"),
           padding=op.get_attr("padding"),
           use_cudnn_on_gpu=op.get_attr("use_cudnn_on_gpu"),
-          data_format=op.get_attr("data_format")), None,
+          data_format=op.get_attr("data_format").decode()), None,
       nn_ops.conv2d(
           op.inputs[0],
           grad,
@@ -81,13 +81,13 @@ def _Conv2DBackpropFilterGrad(op, grad):
           strides=op.get_attr("strides"),
           padding=op.get_attr("padding"),
           use_cudnn_on_gpu=op.get_attr("use_cudnn_on_gpu"),
-          data_format=op.get_attr("data_format"))
+          data_format=op.get_attr("data_format").decode())
   ]
 
 
 @ops.RegisterGradient("Conv3D")
 def _Conv3DGrad(op, grad):
-  data_format = op.get_attr("data_format")
+  data_format = op.get_attr("data_format").decode()
   return [
       nn_ops.conv3d_backprop_input_v2(
           array_ops.shape(op.inputs[0]),
@@ -110,7 +110,7 @@ def _Conv3DGrad(op, grad):
 
 @ops.RegisterGradient("Conv3DBackpropInputV2")
 def _Conv3DBackpropInputGrad(op, grad):
-  data_format = op.get_attr("data_format")
+  data_format = op.get_attr("data_format").decode()
   return [
       None,
       nn_ops.conv3d_backprop_filter_v2(
@@ -133,7 +133,7 @@ def _Conv3DBackpropInputGrad(op, grad):
 
 @ops.RegisterGradient("Conv3DBackpropFilterV2")
 def _Conv3DBackpropFilterGrad(op, grad):
-  data_format = op.get_attr("data_format")
+  data_format = op.get_attr("data_format").decode()
   return [
       nn_ops.conv3d_backprop_input_v2(
           array_ops.shape(op.inputs[0]),
@@ -161,7 +161,7 @@ def _AvgPool3DGrad(op, grad):
       ksize=op.get_attr("ksize"),
       strides=op.get_attr("strides"),
       padding=op.get_attr("padding"),
-      data_format=op.get_attr("data_format"))
+      data_format=op.get_attr("data_format").decode())
 
 
 @ops.RegisterGradient("AvgPool3DGrad")
@@ -172,7 +172,7 @@ def _AvgPool3DGradGrad(op, grad):
               op.get_attr("ksize"),
               op.get_attr("strides"),
               op.get_attr("padding"),
-              data_format=op.get_attr("data_format")))
+              data_format=op.get_attr("data_format").decode()))
 
 
 @ops.RegisterGradient("MaxPool3D")
@@ -184,7 +184,7 @@ def _MaxPool3DGrad(op, grad):
       ksize=op.get_attr("ksize"),
       strides=op.get_attr("strides"),
       padding=op.get_attr("padding"),
-      data_format=op.get_attr("data_format"))
+      data_format=op.get_attr("data_format").decode())
 
 
 @ops.RegisterGradient("MaxPool3DGrad")
@@ -200,7 +200,7 @@ def _MaxPool3DGradGrad(op, grad):
               op.get_attr("ksize"),
               op.get_attr("strides"),
               padding=op.get_attr("padding"),
-              data_format=op.get_attr("data_format")))
+              data_format=op.get_attr("data_format").decode()))
 
 
 @ops.RegisterGradient("MaxPool3DGradGrad")
@@ -216,7 +216,7 @@ def _MaxPool3DGradGradGrad(op, grad):
               op.get_attr("ksize"),
               op.get_attr("strides"),
               padding=op.get_attr("padding"),
-              data_format=op.get_attr("data_format")))
+              data_format=op.get_attr("data_format").decode()))
 
 
 @ops.RegisterGradient("Softmax")
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 290e146ec1..0d7f6ac65b 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -1503,10 +1503,11 @@ def conv2d_v2(input,  # pylint: disable=redefined-builtin
     filters: A `Tensor`. Must have the same type as `input`.
       A 4-D tensor of shape
       `[filter_height, filter_width, in_channels, out_channels]`
-    strides: A list of `ints`.
-      1-D tensor of length 4.  The stride of the sliding window for each
-      dimension of `input`. The dimension order is determined by the value of
-      `data_format`, see below for details.
+    strides: An int or list of `ints` that has length `1`, `2` or `4`.  The
+      stride of the sliding window for each dimension of `input`. If a single
+      value is given it is replicated in the `H` and `W` dimension. By default
+      the `N` and `C` dimensions are set to 1. The dimension order is determined
+      by the value of `data_format`, see below for details.
     padding: Either the `string `"SAME"` or `"VALID"` indicating the type of
       padding algorithm to use, or a list indicating the explicit paddings at
       the start and end of each dimension. When explicit padding is used and
@@ -1521,20 +1522,20 @@ def conv2d_v2(input,  # pylint: disable=redefined-builtin
           [batch, height, width, channels].
       Alternatively, the format could be "NCHW", the data storage order of:
           [batch, channels, height, width].
-    dilations: An optional list of `ints`. Defaults to `[1, 1, 1, 1]`.
-      1-D tensor of length 4.  The dilation factor for each dimension of
-      `input`. If set to k > 1, there will be k-1 skipped cells between each
-      filter element on that dimension. The dimension order is determined by the
-      value of `data_format`, see above for details. Dilations in the batch and
-      depth dimensions must be 1.
+    dilations: An int or list of `ints` that has length `1`, `2` or `4`,
+      defaults to 1. The dilation factor for each dimension of`input`. If a
+      single value is given it is replicated in the `H` and `W` dimension. By
+      default the `N` and `C` dimensions are set to 1. If set to k > 1, there
+      will be k-1 skipped cells between each filter element on that dimension.
+      The dimension order is determined by the value of `data_format`, see above
+      for details. Dilations in the batch and depth dimensions if a 4-d tensor
+      must be 1.
     name: A name for the operation (optional).
 
   Returns:
     A `Tensor`. Has the same type as `input`.
   """
   # pylint: enable=line-too-long
-  if dilations is None:
-    dilations = [1, 1, 1, 1]
   return conv2d(input,  # pylint: disable=redefined-builtin
                 filters,
                 strides,
@@ -1588,10 +1589,11 @@ def conv2d(  # pylint: disable=redefined-builtin,dangerous-default-value
     filter: A `Tensor`. Must have the same type as `input`.
       A 4-D tensor of shape
       `[filter_height, filter_width, in_channels, out_channels]`
-    strides: A list of `ints`.
-      1-D tensor of length 4.  The stride of the sliding window for each
-      dimension of `input`. The dimension order is determined by the value of
-      `data_format`, see below for details.
+    strides: An int or list of `ints` that has length `1`, `2` or `4`.  The
+      stride of the sliding window for each dimension of `input`. If a single
+      value is given it is replicated in the `H` and `W` dimension. By default
+      the `N` and `C` dimensions are set to 1. The dimension order is determined
+      by the value of `data_format`, see below for details.
     padding: Either the `string `"SAME"` or `"VALID"` indicating the type of
       padding algorithm to use, or a list indicating the explicit paddings at
       the start and end of each dimension. When explicit padding is used and
@@ -1607,12 +1609,14 @@ def conv2d(  # pylint: disable=redefined-builtin,dangerous-default-value
           [batch, height, width, channels].
       Alternatively, the format could be "NCHW", the data storage order of:
           [batch, channels, height, width].
-    dilations: An optional list of `ints`. Defaults to `[1, 1, 1, 1]`.
-      1-D tensor of length 4.  The dilation factor for each dimension of
-      `input`. If set to k > 1, there will be k-1 skipped cells between each
-      filter element on that dimension. The dimension order is determined by the
-      value of `data_format`, see above for details. Dilations in the batch and
-      depth dimensions must be 1.
+    dilations: An int or list of `ints` that has length `1`, `2` or `4`,
+      defaults to 1. The dilation factor for each dimension of`input`. If a
+      single value is given it is replicated in the `H` and `W` dimension. By
+      default the `N` and `C` dimensions are set to 1. If set to k > 1, there
+      will be k-1 skipped cells between each filter element on that dimension.
+      The dimension order is determined by the value of `data_format`, see above
+      for details. Dilations in the batch and depth dimensions if a 4-d tensor
+      must be 1.
     name: A name for the operation (optional).
     filters: Alias for filter.
 
@@ -1622,6 +1626,12 @@ def conv2d(  # pylint: disable=redefined-builtin,dangerous-default-value
   filter = deprecation.deprecated_argument_lookup(
       "filters", filters, "filter", filter)
   padding, explicit_paddings = _convert_padding(padding)
+  if data_format is None:
+    data_format = "NHWC"
+  channel_index = 1 if data_format.startswith("NC") else 3
+
+  strides = _get_sequence(strides, 2, channel_index, "strides")
+  dilations = _get_sequence(dilations, 2, channel_index, "dilations")
   return gen_nn_ops.conv2d(input,  # pylint: disable=redefined-builtin
                            filter,
                            strides,
@@ -1912,8 +1922,11 @@ def conv2d_transpose(
       `in_channels` dimension must match that of `value`.
     output_shape: A 1-D `Tensor` representing the output shape of the
       deconvolution op.
-    strides: A list of ints. The stride of the sliding window for each
-      dimension of the input tensor.
+    strides: An int or list of `ints` that has length `1`, `2` or `4`.  The
+      stride of the sliding window for each dimension of `input`. If a single
+      value is given it is replicated in the `H` and `W` dimension. By default
+      the `N` and `C` dimensions are set to 0. The dimension order is determined
+      by the value of `data_format`, see below for details.
     padding: A string, either `'VALID'` or `'SAME'`. The padding algorithm.
       See the "returns" section of `tf.nn.convolution` for details.
     data_format: A string. 'NHWC' and 'NCHW' are supported.
@@ -1961,6 +1974,8 @@ def conv2d_transpose(
       raise ValueError("padding must be either VALID or SAME:"
                        " {}".format(padding))
 
+    strides = _get_sequence(strides, 2, axis, "strides")
+
     return gen_nn_ops.conv2d_backprop_input(
         input_sizes=output_shape_,
         filter=filter,
-- 
GitLab


From 66561698741796a711bdb0dc0c7f24362423c82a Mon Sep 17 00:00:00 2001
From: Yuanzhong Xu <yuanzx@google.com>
Date: Tue, 5 Feb 2019 09:39:49 -0800
Subject: [PATCH 1916/2345] [XLA] Fix replica-id HLO parsing

PiperOrigin-RevId: 232502680
---
 tensorflow/compiler/xla/service/hlo_parser.cc      |  5 +----
 tensorflow/compiler/xla/service/hlo_parser_test.cc | 11 +++++++++++
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_parser.cc b/tensorflow/compiler/xla/service/hlo_parser.cc
index dd8e8ff3a5..00c810675b 100644
--- a/tensorflow/compiler/xla/service/hlo_parser.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser.cc
@@ -853,13 +853,10 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
       break;
     }
     case HloOpcode::kReplicaId: {
-      if (!ParseOperands(&operands, /*expected_size=*/1) ||
+      if (!ParseOperands(&operands, /*expected_size=*/0) ||
           !ParseAttributes(attrs)) {
         return false;
       }
-      if (!operands.empty()) {
-        return false;
-      }
       instruction = builder->AddInstruction(HloInstruction::CreateReplicaId());
       break;
     }
diff --git a/tensorflow/compiler/xla/service/hlo_parser_test.cc b/tensorflow/compiler/xla/service/hlo_parser_test.cc
index b525f66f9b..b66d84caec 100644
--- a/tensorflow/compiler/xla/service/hlo_parser_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser_test.cc
@@ -1280,6 +1280,17 @@ ENTRY CollectivePermute {
   ROOT root = f32[128,32]{0,1} collective-permute(input), source_target_pairs={{0,1},{1,2},{2,3}}
 }
 
+)"
+},
+// replica-id
+{
+"ReplicaId",
+R"(HloModule replica-id
+
+ENTRY Replica-id {
+  ROOT replica-id = u32[] replica-id()
+}
+
 )"
 },
 // Iota
-- 
GitLab


From 0906bafa73198677d52f8f15599a8bed3fc9b234 Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Tue, 5 Feb 2019 09:40:12 -0800
Subject: [PATCH 1917/2345] [XLA] InferPadShape should report an error when the
 result dimension size is a negative number.

Modify an existing test case to test the situation.

PiperOrigin-RevId: 232502738
---
 tensorflow/compiler/xla/service/shape_inference.cc      | 4 ++++
 tensorflow/compiler/xla/service/shape_inference_test.cc | 8 ++++++++
 2 files changed, 12 insertions(+)

diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index 12db957b0c..a403428d05 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -534,6 +534,10 @@ StatusOr<Shape> InferWindowOutputShape(const Shape& base_shape,
                     p.edge_padding_high() +
                     std::max<int64>(operand_shape.dimensions(i) - 1, 0LL) *
                         p.interior_padding();
+    if (dimensions[i] < 0) {
+      return InvalidArgument("Padding result in negative size for dimension %d",
+                             i);
+    }
     is_dynamic[i] = operand_shape.is_dynamic_dimension(i);
   }
 
diff --git a/tensorflow/compiler/xla/service/shape_inference_test.cc b/tensorflow/compiler/xla/service/shape_inference_test.cc
index c43db7aae8..eabc22307a 100644
--- a/tensorflow/compiler/xla/service/shape_inference_test.cc
+++ b/tensorflow/compiler/xla/service/shape_inference_test.cc
@@ -1481,6 +1481,14 @@ TEST_F(ShapeInferenceTest, Pad) {
   Shape inferred_shape = inferred_status.ValueOrDie();
   ASSERT_TRUE(
       ShapeUtil::Equal(ShapeUtil::MakeShape(F32, {39, 31}), inferred_shape));
+
+  dimension1->set_edge_padding_low(-20);
+  dimension1->set_edge_padding_high(-10);
+  auto negative_dimension_size = ShapeInference::InferPadShape(
+      input_shape, padding_value_shape, padding_config);
+  ASSERT_FALSE(negative_dimension_size.ok());
+  ASSERT_THAT(negative_dimension_size.status().error_message(),
+              HasSubstr("negative size for dimension 1"));
 }
 
 TEST_F(ShapeInferenceTest, Reverse) {
-- 
GitLab


From 32b84541e4eb87bd97ba9557421a92e77bfeedca Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Tue, 5 Feb 2019 09:40:53 -0800
Subject: [PATCH 1918/2345] [XLA:Python] Fail gracefully when an unsupported
 numpy type is converted to/from XLA.

Add support for int8/int16/uint16.

PiperOrigin-RevId: 232502865
---
 .../xla/python/local_computation_builder.i    | 28 +++++------
 .../compiler/xla/python/numpy_bridge.cc       | 50 +++++++++++++++----
 tensorflow/compiler/xla/python/numpy_bridge.h | 16 ++++--
 .../compiler/xla/python/xla_client_test.py    |  6 +++
 4 files changed, 72 insertions(+), 28 deletions(-)

diff --git a/tensorflow/compiler/xla/python/local_computation_builder.i b/tensorflow/compiler/xla/python/local_computation_builder.i
index 7b2f69d6ec..df2ab0b539 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.i
+++ b/tensorflow/compiler/xla/python/local_computation_builder.i
@@ -453,16 +453,6 @@ tensorflow::ImportNumpy();
 
 // Literal
 
-%typemap(out) StatusOr<Literal> {
-  if ($1.ok()) {
-    Literal value = $1.ConsumeValueOrDie();
-    $result = numpy::PyObjectFromXlaLiteral(*value);
-  } else {
-    PyErr_SetString(PyExc_RuntimeError, $1.status().ToString().c_str());
-    SWIG_fail;
-  }
-}
-
 %typemap(in) const Literal& (StatusOr<Literal> literal_status) {
   literal_status = numpy::XlaLiteralFromPyObject($input);
   if (!literal_status.ok()) {
@@ -472,16 +462,26 @@ tensorflow::ImportNumpy();
   $1 = &literal_status.ValueOrDie();
 }
 
-%typemap(out) Literal {
-  $result = numpy::PyObjectFromXlaLiteral(*$1);
+%typemap(out) Literal (StatusOr<numpy::Safe_PyObjectPtr> obj_status) {
+  obj_status = numpy::PyObjectFromXlaLiteral(*$1);
+  if (!obj_status.ok()) {
+    PyErr_SetString(PyExc_RuntimeError, obj_status.status().ToString().c_str());
+    SWIG_fail;
+  }
+  $result = obj_status.ValueOrDie().release();
 }
 
-%typemap(out) StatusOr<Literal> {
+%typemap(out) StatusOr<Literal> (StatusOr<numpy::Safe_PyObjectPtr> obj_status) {
   if (!$1.ok()) {
     PyErr_SetString(PyExc_RuntimeError, $1.status().ToString().c_str());
     SWIG_fail;
   }
-  $result = numpy::PyObjectFromXlaLiteral($1.ValueOrDie());
+  obj_status = numpy::PyObjectFromXlaLiteral($1.ValueOrDie());
+  if (!obj_status.ok()) {
+    PyErr_SetString(PyExc_RuntimeError, obj_status.status().ToString().c_str());
+    SWIG_fail;
+  }
+  $result = obj_status.ValueOrDie().release();
 }
 
 %typemap(in) const std::vector<Literal>& (std::vector<Literal> temps) {
diff --git a/tensorflow/compiler/xla/python/numpy_bridge.cc b/tensorflow/compiler/xla/python/numpy_bridge.cc
index 52c5c621f7..8e056f9725 100644
--- a/tensorflow/compiler/xla/python/numpy_bridge.cc
+++ b/tensorflow/compiler/xla/python/numpy_bridge.cc
@@ -26,6 +26,10 @@ namespace swig {
 
 namespace numpy {
 
+Safe_PyObjectPtr make_safe(PyObject* object) {
+  return Safe_PyObjectPtr(object);
+}
+
 int PrimitiveTypeToNumpyType(PrimitiveType primitive_type) {
   switch (primitive_type) {
     case PRED:
@@ -349,13 +353,17 @@ StatusOr<OpMetadata> OpMetadataFromPyObject(PyObject* o) {
   return result;
 }
 
-PyObject* PyObjectFromXlaLiteral(const LiteralSlice& literal) {
+StatusOr<Safe_PyObjectPtr> PyObjectFromXlaLiteral(const LiteralSlice& literal) {
   if (literal.shape().IsTuple()) {
     int num_elements = ShapeUtil::TupleElementCount(literal.shape());
-    PyObject* tuple = PyTuple_New(num_elements);
+    std::vector<Safe_PyObjectPtr> elems(num_elements);
     for (int i = 0; i < num_elements; i++) {
-      PyTuple_SET_ITEM(tuple, i,
-                       PyObjectFromXlaLiteral(LiteralSlice(literal, {i})));
+      TF_ASSIGN_OR_RETURN(elems[i],
+                          PyObjectFromXlaLiteral(LiteralSlice(literal, {i})));
+    }
+    Safe_PyObjectPtr tuple = make_safe(PyTuple_New(num_elements));
+    for (int i = 0; i < num_elements; i++) {
+      PyTuple_SET_ITEM(tuple.get(), i, elems[i].release());
     }
     return tuple;
   } else {
@@ -365,10 +373,10 @@ PyObject* PyObjectFromXlaLiteral(const LiteralSlice& literal) {
       dimensions[i] = ShapeUtil::GetDimension(literal.shape(), i);
     }
     int np_type = PrimitiveTypeToNumpyType(literal.shape().element_type());
-    PyObject* array =
-        PyArray_EMPTY(rank, dimensions.data(), np_type, /*fortran=*/0);
-    CopyLiteralToNumpyArray(np_type, literal,
-                            reinterpret_cast<PyArrayObject*>(array));
+    Safe_PyObjectPtr array = make_safe(
+        PyArray_EMPTY(rank, dimensions.data(), np_type, /*fortran=*/0));
+    TF_RETURN_IF_ERROR(CopyLiteralToNumpyArray(
+        np_type, literal, reinterpret_cast<PyArrayObject*>(array.get())));
     return array;
   }
 }
@@ -408,6 +416,12 @@ Status CopyNumpyArrayToLiteral(int np_type, PyArrayObject* py_array,
     case NPY_BOOL:
       CopyNumpyArrayToLiteral<bool>(py_array, literal);
       break;
+    case NPY_INT8:
+      CopyNumpyArrayToLiteral<int8>(py_array, literal);
+      break;
+    case NPY_INT16:
+      CopyNumpyArrayToLiteral<int16>(py_array, literal);
+      break;
     case NPY_INT32:
       CopyNumpyArrayToLiteral<int32>(py_array, literal);
       break;
@@ -417,6 +431,9 @@ Status CopyNumpyArrayToLiteral(int np_type, PyArrayObject* py_array,
     case NPY_UINT8:
       CopyNumpyArrayToLiteral<uint8>(py_array, literal);
       break;
+    case NPY_UINT16:
+      CopyNumpyArrayToLiteral<uint16>(py_array, literal);
+      break;
     case NPY_UINT32:
       CopyNumpyArrayToLiteral<uint32>(py_array, literal);
       break;
@@ -445,12 +462,18 @@ Status CopyNumpyArrayToLiteral(int np_type, PyArrayObject* py_array,
   return Status::OK();
 }
 
-void CopyLiteralToNumpyArray(int np_type, const LiteralSlice& literal,
-                             PyArrayObject* py_array) {
+Status CopyLiteralToNumpyArray(int np_type, const LiteralSlice& literal,
+                               PyArrayObject* py_array) {
   switch (np_type) {
     case NPY_BOOL:
       CopyLiteralToNumpyArray<bool>(literal, py_array);
       break;
+    case NPY_INT8:
+      CopyLiteralToNumpyArray<int8>(literal, py_array);
+      break;
+    case NPY_INT16:
+      CopyLiteralToNumpyArray<int16>(literal, py_array);
+      break;
     case NPY_INT32:
       CopyLiteralToNumpyArray<int32>(literal, py_array);
       break;
@@ -460,6 +483,9 @@ void CopyLiteralToNumpyArray(int np_type, const LiteralSlice& literal,
     case NPY_UINT8:
       CopyLiteralToNumpyArray<uint8>(literal, py_array);
       break;
+    case NPY_UINT16:
+      CopyLiteralToNumpyArray<uint16>(literal, py_array);
+      break;
     case NPY_UINT32:
       CopyLiteralToNumpyArray<uint32>(literal, py_array);
       break;
@@ -482,8 +508,10 @@ void CopyLiteralToNumpyArray(int np_type, const LiteralSlice& literal,
       CopyLiteralToNumpyArray<complex128>(literal, py_array);
       break;
     default:
-      LOG(FATAL) << "No XLA literal container for Numpy type" << np_type;
+      return InvalidArgument(
+          "No XLA literal container for Numpy type number: %d", np_type);
   }
+  return Status::OK();
 }
 
 PyObject* LongToPyIntOrPyLong(long x) {  // NOLINT
diff --git a/tensorflow/compiler/xla/python/numpy_bridge.h b/tensorflow/compiler/xla/python/numpy_bridge.h
index 40ff2d9ad2..737fc4b29c 100644
--- a/tensorflow/compiler/xla/python/numpy_bridge.h
+++ b/tensorflow/compiler/xla/python/numpy_bridge.h
@@ -36,6 +36,16 @@ namespace swig {
 
 namespace numpy {
 
+struct PyDecrefDeleter {
+  void operator()(PyObject* p) const { Py_DECREF(p); }
+};
+
+// Safe container for an owned PyObject. On destruction, the reference count of
+// the contained object will be decremented.
+using Safe_PyObjectPtr = std::unique_ptr<PyObject, PyDecrefDeleter>;
+
+Safe_PyObjectPtr make_safe(PyObject* object);
+
 // Maps XLA primitive types (PRED, S8, F32, ..., and TUPLE) to numpy
 // dtypes (NPY_BOOL, NPY_INT8, NPY_FLOAT32, ..., and NPY_OBJECT), and
 // vice versa.
@@ -74,7 +84,7 @@ StatusOr<OpMetadata> OpMetadataFromPyObject(PyObject* o);
 // array data.
 //
 // The return value is a new reference.
-PyObject* PyObjectFromXlaLiteral(const LiteralSlice& literal);
+StatusOr<Safe_PyObjectPtr> PyObjectFromXlaLiteral(const LiteralSlice& literal);
 
 // Converts a Numpy ndarray or a nested Python tuple thereof to a
 // corresponding XLA literal.
@@ -90,8 +100,8 @@ StatusOr<Literal> XlaLiteralFromPyObject(PyObject* o);
 Status CopyNumpyArrayToLiteral(int np_type, PyArrayObject* py_array,
                                Literal* literal);
 
-void CopyLiteralToNumpyArray(int np_type, const LiteralSlice& literal,
-                             PyArrayObject* py_array);
+Status CopyLiteralToNumpyArray(int np_type, const LiteralSlice& literal,
+                               PyArrayObject* py_array);
 
 template <typename NativeT>
 void CopyNumpyArrayToLiteral(PyArrayObject* py_array, Literal* literal) {
diff --git a/tensorflow/compiler/xla/python/xla_client_test.py b/tensorflow/compiler/xla/python/xla_client_test.py
index c80e792464..aa38c06cf9 100644
--- a/tensorflow/compiler/xla/python/xla_client_test.py
+++ b/tensorflow/compiler/xla/python/xla_client_test.py
@@ -88,6 +88,12 @@ def NumpyArrayBool(*args, **kwargs):
 class ComputationsWithConstantsTest(LocalComputationTest):
   """Tests focusing on Constant ops."""
 
+  def testConstantScalarSumS8(self):
+    c = self._NewComputation()
+    root = c.Add(c.Constant(np.int8(1)), c.Constant(np.int8(2)))
+    self.assertEqual(c.GetShape(root), c.GetReturnValueShape())
+    self._ExecuteAndCompareExact(c, expected=np.int8(3))
+
   def testConstantScalarSumF32(self):
     c = self._NewComputation()
     root = c.Add(c.ConstantF32Scalar(1.11), c.ConstantF32Scalar(3.14))
-- 
GitLab


From f6a39be9f488e431bbb891520100e4d290fc07f6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 5 Feb 2019 09:51:21 -0800
Subject: [PATCH 1919/2345] Introduce rounding into depthwise conv reference /
 test.

PiperOrigin-RevId: 232504607
---
 .../internal/depthwiseconv_quantized_test.cc  |  77 +++++--
 .../internal/reference/depthwiseconv_uint8.h  | 197 +++++++++++-------
 2 files changed, 184 insertions(+), 90 deletions(-)

diff --git a/tensorflow/lite/kernels/internal/depthwiseconv_quantized_test.cc b/tensorflow/lite/kernels/internal/depthwiseconv_quantized_test.cc
index 963ab4d8ff..b396e6256c 100644
--- a/tensorflow/lite/kernels/internal/depthwiseconv_quantized_test.cc
+++ b/tensorflow/lite/kernels/internal/depthwiseconv_quantized_test.cc
@@ -57,7 +57,8 @@ enum class CoverageExtension {
 
 // The TestParam structure below is the preferred parameterization of tests. A
 // tuple version is defined in order to support value-parameterized tests.
-typedef std::tuple<DepthwiseConvInvocation, int, bool, bool, bool>
+typedef std::tuple<DepthwiseConvInvocation, int, bool, bool, bool,
+                   DepthwiseConvOutputRounding, bool>
     TestParamTuple;
 
 struct TestParam {
@@ -68,7 +69,9 @@ struct TestParam {
         tests_to_run(::testing::get<1>(param_tuple)),
         test_stride(::testing::get<2>(param_tuple)),
         test_pad(::testing::get<3>(param_tuple)),
-        test_depth_multiplier(::testing::get<4>(param_tuple)) {}
+        test_depth_multiplier(::testing::get<4>(param_tuple)),
+        output_rounding(::testing::get<5>(param_tuple)),
+        loose_tolerance(::testing::get<6>(param_tuple)) {}
 
   static std::string TestNameSuffix(
       const ::testing::TestParamInfo<TestParamTuple>& info) {
@@ -84,6 +87,9 @@ struct TestParam {
   bool test_stride = false;
   bool test_pad = false;
   bool test_depth_multiplier = false;
+  DepthwiseConvOutputRounding output_rounding =
+      DepthwiseConvOutputRounding::kNone;
+  bool loose_tolerance = false;
 };
 
 inline void DispatchDepthwiseConv(
@@ -183,9 +189,30 @@ int TestOneDepthwiseConvWithGivenOutputShift(
   op_params.output_offset = output_offset;
   op_params.output_multiplier = output_multiplier;
   op_params.output_shift = -output_shift;
-  reference_ops::DepthwiseConv(op_params, input_shape, input_data, filter_shape,
-                               filter_data, bias_shape, bias_data, output_shape,
-                               reference_output_data.data());
+  switch (test_param.output_rounding) {
+    case DepthwiseConvOutputRounding::kUpward:
+      reference_ops::DepthwiseConvBasicKernel<
+          DepthwiseConvOutputRounding::kAwayFromZero>::Run(op_params,
+                                                           input_shape,
+                                                           input_data,
+                                                           filter_shape,
+                                                           filter_data,
+                                                           bias_shape,
+                                                           bias_data,
+                                                           output_shape,
+                                                           reference_output_data
+                                                               .data());
+      break;
+    case DepthwiseConvOutputRounding::kAwayFromZero:
+      reference_ops::DepthwiseConv(
+          op_params, input_shape, input_data, filter_shape, filter_data,
+          bias_shape, bias_data, output_shape, reference_output_data.data());
+      break;
+    case DepthwiseConvOutputRounding::kNone:
+    default:
+      EXPECT_NE(test_param.output_rounding, DepthwiseConvOutputRounding::kNone);
+      break;
+  }
   DispatchDepthwiseConv(test_param, op_params, input_shape, input_data,
                         filter_shape, filter_data, bias_shape, bias_data,
                         output_shape, output_data.data());
@@ -221,10 +248,10 @@ int TestOneDepthwiseConvWithGivenOutputShift(
 
   // Normally we should require bit-for-bit exact results. Unfortunately a bug
   // in the Intel arm_neon_sse.h translation header that we use for x86 tests
-  // causes 1-bit inaccuracy in
-  // the vqrdmulh_n_s32 intrinsic, which causes off-by-1 errors in quantized
-  // DepthwiseConv ops. So we have to live with a few off-by-one errors for now,
-  // yet still ensure that no more than a small minority of values are wrong.
+  // causes 1-bit inaccuracy in the vqrdmulh_n_s32 intrinsic, which causes
+  // off-by-1 errors in quantized DepthwiseConv ops. So we have to live with a
+  // few off-by-one errors for now, yet still ensure that no more than a small
+  // minority of values are wrong.
   EXPECT_LT(std::abs(mean_diff), mean_tolerance);
   EXPECT_LT(mean_abs_diff, mean_tolerance);
   EXPECT_LE(std::abs(median_diff), diff_median_tolerance);
@@ -482,16 +509,21 @@ bool TryTestOneNeonDot3x3(const TestParam& test_param,
       dilation_width_factor, dilation_height_factor, padding_type);
 }
 
-void TestOneDepthwiseConv(DepthwiseConvInvocation forced_invocation) {
+void TestOneDepthwiseConv(DepthwiseConvInvocation forced_invocation,
+                          DepthwiseConvOutputRounding output_rounding) {
   TestParam test_param;
   test_param.forced_invocation = forced_invocation;
+  test_param.output_rounding = output_rounding;
   while (!TryTestOneDepthwiseConv(test_param, ParamsSpecialization::kNone)) {
   }
 }
 
-void TestOneDepthwiseConv3x3Filter(DepthwiseConvInvocation forced_invocation) {
+void TestOneDepthwiseConv3x3Filter(
+    DepthwiseConvInvocation forced_invocation,
+    DepthwiseConvOutputRounding output_rounding) {
   TestParam test_param;
   test_param.forced_invocation = forced_invocation;
+  test_param.output_rounding = output_rounding;
   while (!TryTestOneDepthwiseConv3x3Filter(test_param,
                                            ParamsSpecialization::kNone)) {
   }
@@ -505,7 +537,8 @@ void TestOneNeonDot3x3(const TestParam& test_param) {
 TEST(TestDepthwiseConv, TestDepthwiseConv) {
   const int kTestsToRun = 10 * 1000;
   for (int i = 0; i < kTestsToRun; i++) {
-    TestOneDepthwiseConv(DepthwiseConvInvocation::kNone);
+    TestOneDepthwiseConv(DepthwiseConvInvocation::kNone,
+                         DepthwiseConvOutputRounding::kAwayFromZero);
   }
 }
 
@@ -513,14 +546,16 @@ TEST(TestDepthwiseConv, TestDepthwiseConv) {
 TEST(TestDepthwiseConv, TestGenericKernel) {
   const int kTestsToRun = 10 * 1000;
   for (int i = 0; i < kTestsToRun; i++) {
-    TestOneDepthwiseConv(DepthwiseConvInvocation::kUseGenericKernel);
+    TestOneDepthwiseConv(DepthwiseConvInvocation::kUseGenericKernel,
+                         DepthwiseConvOutputRounding::kAwayFromZero);
   }
 }
 
 TEST(TestDepthwiseConv, TestKernel3x3Filter) {
   const int kTestsToRun = 1000;
   for (int i = 0; i < kTestsToRun; i++) {
-    TestOneDepthwiseConv3x3Filter(DepthwiseConvInvocation::kNone);
+    TestOneDepthwiseConv3x3Filter(DepthwiseConvInvocation::kNone,
+                                  DepthwiseConvOutputRounding::kAwayFromZero);
   }
 }
 
@@ -529,7 +564,8 @@ TEST(TestDepthwiseConv, TestKernel3x3Filter) {
 TEST(TestDepthwiseConv, TestGenericKernel3x3Filter) {
   const int kTestsToRun = 100;
   for (int i = 0; i < kTestsToRun; i++) {
-    TestOneDepthwiseConv3x3Filter(DepthwiseConvInvocation::kUseGenericKernel);
+    TestOneDepthwiseConv3x3Filter(DepthwiseConvInvocation::kUseGenericKernel,
+                                  DepthwiseConvOutputRounding::kAwayFromZero);
   }
 }
 
@@ -537,7 +573,8 @@ TEST(TestDepthwiseConv, TestGenericKernel3x3Filter) {
 TEST(TestDepthwiseConv, TestNeon3x3Filter) {
   const int kTestsToRun = 3 * 1000;
   for (int i = 0; i < kTestsToRun; i++) {
-    TestOneDepthwiseConv3x3Filter(DepthwiseConvInvocation::kUseNeon3x3);
+    TestOneDepthwiseConv3x3Filter(DepthwiseConvInvocation::kUseNeon3x3,
+                                  DepthwiseConvOutputRounding::kAwayFromZero);
   }
 }
 #endif
@@ -559,7 +596,9 @@ INSTANTIATE_TEST_SUITE_P(
         Values(1000),                                  // tests_to_run
         Bool(),                                        // test_stride
         Values(false),                                 // test_pad
-        Values(false)                                  // test_depth_multiplier
+        Values(false),                                 // test_depth_multiplier
+        Values(DepthwiseConvOutputRounding::kAwayFromZero),  // output_rounding
+        Values(false)                                        // loose_tolerance
         ),
     TestParam::TestNameSuffix);
 #endif
@@ -574,7 +613,9 @@ INSTANTIATE_TEST_SUITE_P(
         Values(100),                                      // tests_to_run
         Bool(),                                           // test_stride
         Bool(),                                           // test_pad
-        Bool()  // test_depth_multiplier
+        Bool(),                                        // test_depth_multiplier
+        Values(DepthwiseConvOutputRounding::kUpward),  // output_rounding
+        Values(false)                                  // loose_tolerance
         ),
     TestParam::TestNameSuffix);
 
diff --git a/tensorflow/lite/kernels/internal/reference/depthwiseconv_uint8.h b/tensorflow/lite/kernels/internal/reference/depthwiseconv_uint8.h
index 99c03426f5..7cc5679dcb 100644
--- a/tensorflow/lite/kernels/internal/reference/depthwiseconv_uint8.h
+++ b/tensorflow/lite/kernels/internal/reference/depthwiseconv_uint8.h
@@ -44,6 +44,14 @@ enum class DepthwiseConvInvocation {
   kUseIntrinsics3x3DotProduct,  // 3x3 kernel using NEON intrinsics.
 };
 
+// Category of depthwise convolution output rounding.
+enum class DepthwiseConvOutputRounding {
+  kNone = 0,      // Invalid: specific method must be specified.
+  kAwayFromZero,  // Original method: exact halves rounded away from zero.
+  kUpward,        // Halves towards +infinity: adds 0.5 before truncate.
+  // This is where a future kNearestEven would be placed.
+};
+
 // Category of depthwise convolution depth multiplication.
 enum class DepthwiseConvDepthMultiplication {
   kNoMultiplication = 0,  // Depth multiplier = 1.
@@ -52,88 +60,133 @@ enum class DepthwiseConvDepthMultiplication {
 
 namespace reference_ops {
 
-inline void DepthwiseConv(
-    const DepthwiseParams& params, const RuntimeShape& input_shape,
-    const uint8* input_data, const RuntimeShape& filter_shape,
-    const uint8* filter_data, const RuntimeShape& bias_shape,
-    const int32* bias_data, const RuntimeShape& output_shape,
-    uint8* output_data) {
-  const int stride_width = params.stride_width;
-  const int stride_height = params.stride_height;
-  const int dilation_width_factor = params.dilation_width_factor;
-  const int dilation_height_factor = params.dilation_height_factor;
-  const int pad_width = params.padding_values.width;
-  const int pad_height = params.padding_values.height;
-  const int depth_multiplier = params.depth_multiplier;
-  const int32 output_activation_min = params.quantized_activation_min;
-  const int32 output_activation_max = params.quantized_activation_max;
-  const int32 input_offset = params.input_offset;
-  const int32 filter_offset = params.weights_offset;
-  const int32 output_offset = params.output_offset;
-  const int32 output_multiplier = params.output_multiplier;
-  const int output_shift = params.output_shift;
-  TFLITE_DCHECK_EQ(input_shape.DimensionsCount(), 4);
-  TFLITE_DCHECK_EQ(filter_shape.DimensionsCount(), 4);
-  TFLITE_DCHECK_EQ(output_shape.DimensionsCount(), 4);
-
-  TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
-  const int batches = MatchingDim(input_shape, 0, output_shape, 0);
-  const int output_depth = MatchingDim(filter_shape, 3, output_shape, 3);
-  const int input_height = input_shape.Dims(1);
-  const int input_width = input_shape.Dims(2);
-  const int input_depth = input_shape.Dims(3);
-  const int filter_height = filter_shape.Dims(1);
-  const int filter_width = filter_shape.Dims(2);
-  const int output_height = output_shape.Dims(1);
-  const int output_width = output_shape.Dims(2);
-  TFLITE_DCHECK_EQ(output_depth, input_depth * depth_multiplier);
-  TFLITE_DCHECK_EQ(bias_shape.FlatSize(), output_depth);
-
-  for (int b = 0; b < batches; ++b) {
-    for (int out_y = 0; out_y < output_height; ++out_y) {
-      for (int out_x = 0; out_x < output_width; ++out_x) {
-        for (int ic = 0; ic < input_depth; ++ic) {
-          for (int m = 0; m < depth_multiplier; m++) {
-            const int oc = m + ic * depth_multiplier;
-            const int in_x_origin = (out_x * stride_width) - pad_width;
-            const int in_y_origin = (out_y * stride_height) - pad_height;
-            int32 acc = 0;
-            for (int filter_y = 0; filter_y < filter_height; ++filter_y) {
-              for (int filter_x = 0; filter_x < filter_width; ++filter_x) {
-                const int in_x = in_x_origin + dilation_width_factor * filter_x;
-                const int in_y =
-                    in_y_origin + dilation_height_factor * filter_y;
-                // If the location is outside the bounds of the input image,
-                // use zero as a default value.
-                if ((in_x >= 0) && (in_x < input_width) && (in_y >= 0) &&
-                    (in_y < input_height)) {
-                  int32 input_val =
-                      input_data[Offset(input_shape, b, in_y, in_x, ic)];
-                  int32 filter_val = filter_data[Offset(
-                      filter_shape, 0, filter_y, filter_x, oc)];
-                  acc +=
-                      (filter_val + filter_offset) * (input_val + input_offset);
+template <DepthwiseConvOutputRounding output_rounding>
+inline int32 DepthwiseConvRound(int32 x, int32 quantized_multiplier,
+                                int shift) {
+  TFLITE_DCHECK_NE(output_rounding, DepthwiseConvOutputRounding::kNone);
+  return MultiplyByQuantizedMultiplier(x, quantized_multiplier, shift);
+}
+
+template <>
+inline int32 DepthwiseConvRound<DepthwiseConvOutputRounding::kAwayFromZero>(
+    int32 x, int32 quantized_multiplier, int shift) {
+  return MultiplyByQuantizedMultiplier(x, quantized_multiplier, shift);
+}
+
+template <>
+inline int32 DepthwiseConvRound<DepthwiseConvOutputRounding::kUpward>(
+    int32 x, int32 quantized_multiplier, int shift) {
+  using gemmlowp::SaturatingRoundingDoublingHighMul;
+  const int left_shift = shift > 0 ? shift : 0;
+  const int right_shift = shift > 0 ? 0 : -shift;
+  const int rounding_offset = right_shift > 0 ? 1 << (right_shift - 1) : 0;
+  return (SaturatingRoundingDoublingHighMul(x * (1 << left_shift),
+                                            quantized_multiplier) +
+          rounding_offset) >>
+         right_shift;
+}
+
+template <DepthwiseConvOutputRounding output_rounding>
+struct DepthwiseConvBasicKernel {
+  static inline void Run(const DepthwiseParams& params,
+                         const RuntimeShape& input_shape,
+                         const uint8* input_data,
+                         const RuntimeShape& filter_shape,
+                         const uint8* filter_data,
+                         const RuntimeShape& bias_shape, const int32* bias_data,
+                         const RuntimeShape& output_shape, uint8* output_data) {
+    const int stride_width = params.stride_width;
+    const int stride_height = params.stride_height;
+    const int dilation_width_factor = params.dilation_width_factor;
+    const int dilation_height_factor = params.dilation_height_factor;
+    const int pad_width = params.padding_values.width;
+    const int pad_height = params.padding_values.height;
+    const int depth_multiplier = params.depth_multiplier;
+    const int32 output_activation_min = params.quantized_activation_min;
+    const int32 output_activation_max = params.quantized_activation_max;
+    const int32 input_offset = params.input_offset;
+    const int32 filter_offset = params.weights_offset;
+    const int32 output_offset = params.output_offset;
+    const int32 output_multiplier = params.output_multiplier;
+    const int output_shift = params.output_shift;
+    TFLITE_DCHECK_EQ(input_shape.DimensionsCount(), 4);
+    TFLITE_DCHECK_EQ(filter_shape.DimensionsCount(), 4);
+    TFLITE_DCHECK_EQ(output_shape.DimensionsCount(), 4);
+
+    TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
+    const int batches = MatchingDim(input_shape, 0, output_shape, 0);
+    const int output_depth = MatchingDim(filter_shape, 3, output_shape, 3);
+    const int input_height = input_shape.Dims(1);
+    const int input_width = input_shape.Dims(2);
+    const int input_depth = input_shape.Dims(3);
+    const int filter_height = filter_shape.Dims(1);
+    const int filter_width = filter_shape.Dims(2);
+    const int output_height = output_shape.Dims(1);
+    const int output_width = output_shape.Dims(2);
+    TFLITE_DCHECK_EQ(output_depth, input_depth * depth_multiplier);
+    TFLITE_DCHECK_EQ(bias_shape.FlatSize(), output_depth);
+
+    for (int b = 0; b < batches; ++b) {
+      for (int out_y = 0; out_y < output_height; ++out_y) {
+        for (int out_x = 0; out_x < output_width; ++out_x) {
+          for (int ic = 0; ic < input_depth; ++ic) {
+            for (int m = 0; m < depth_multiplier; m++) {
+              const int oc = m + ic * depth_multiplier;
+              const int in_x_origin = (out_x * stride_width) - pad_width;
+              const int in_y_origin = (out_y * stride_height) - pad_height;
+              int32 acc = 0;
+              for (int filter_y = 0; filter_y < filter_height; ++filter_y) {
+                for (int filter_x = 0; filter_x < filter_width; ++filter_x) {
+                  const int in_x =
+                      in_x_origin + dilation_width_factor * filter_x;
+                  const int in_y =
+                      in_y_origin + dilation_height_factor * filter_y;
+                  // If the location is outside the bounds of the input image,
+                  // use zero as a default value.
+                  if ((in_x >= 0) && (in_x < input_width) && (in_y >= 0) &&
+                      (in_y < input_height)) {
+                    int32 input_val =
+                        input_data[Offset(input_shape, b, in_y, in_x, ic)];
+                    int32 filter_val = filter_data[Offset(
+                        filter_shape, 0, filter_y, filter_x, oc)];
+                    acc += (filter_val + filter_offset) *
+                           (input_val + input_offset);
+                  }
                 }
               }
+              if (bias_data) {
+                acc += bias_data[oc];
+              }
+              acc = DepthwiseConvRound<output_rounding>(acc, output_multiplier,
+                                                        output_shift);
+              acc += output_offset;
+              acc = std::max(acc, output_activation_min);
+              acc = std::min(acc, output_activation_max);
+              output_data[Offset(output_shape, b, out_y, out_x, oc)] =
+                  static_cast<uint8>(acc);
             }
-            if (bias_data) {
-              acc += bias_data[oc];
-            }
-            acc = MultiplyByQuantizedMultiplier(acc, output_multiplier,
-                                                output_shift);
-            acc += output_offset;
-            acc = std::max(acc, output_activation_min);
-            acc = std::min(acc, output_activation_max);
-            output_data[Offset(output_shape, b, out_y, out_x, oc)] =
-                static_cast<uint8>(acc);
           }
         }
       }
     }
   }
+};
+
+inline void DepthwiseConv(
+    const DepthwiseParams& params, const RuntimeShape& input_shape,
+    const uint8* input_data, const RuntimeShape& filter_shape,
+    const uint8* filter_data, const RuntimeShape& bias_shape,
+    const int32* bias_data, const RuntimeShape& output_shape,
+    uint8* output_data) {
+  return DepthwiseConvBasicKernel<
+      DepthwiseConvOutputRounding::kAwayFromZero>::Run(params, input_shape,
+                                                       input_data, filter_shape,
+                                                       filter_data, bias_shape,
+                                                       bias_data, output_shape,
+                                                       output_data);
 }
 
-}  // end namespace reference_ops
+}  // namespace reference_ops
 }  // end namespace tflite
 
 #endif  // TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_DEPTHWISECONV_UINT8_H_
-- 
GitLab


From e1f2db44f44addd86374b98be13b08990540020d Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Tue, 5 Feb 2019 10:04:57 -0800
Subject: [PATCH 1920/2345] Standardise inputs for the conv1d and add the
 ability to perform dilations

PiperOrigin-RevId: 232507407
---
 tensorflow/python/ops/nn_ops.py               | 75 ++++++++++++-------
 .../tools/api/golden/v1/tensorflow.nn.pbtxt   |  2 +-
 .../tools/api/golden/v2/tensorflow.nn.pbtxt   |  2 +-
 3 files changed, 49 insertions(+), 30 deletions(-)

diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 0d7f6ac65b..af2bca0a17 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -3999,14 +3999,16 @@ def fractional_avg_pool_v2(value,
     "`NHWC` for data_format is deprecated, use `NWC` instead",
     warn_once=True,
     data_format="NHWC")
-def conv1d(value=None,
-           filters=None,
-           stride=None,
-           padding=None,
-           use_cudnn_on_gpu=None,
-           data_format=None,
-           name=None,
-           input=None):  # pylint: disable=redefined-builtin
+def conv1d(
+    value,
+    filters,
+    stride,
+    padding,
+    use_cudnn_on_gpu=None,
+    data_format=None,
+    name=None,
+    input=None,  # pylint: disable=redefined-builtin
+    dilations=None):
   r"""Computes a 1-D convolution given 3-D input and filter tensors.
 
   Given an input tensor of shape
@@ -4034,8 +4036,8 @@ def conv1d(value=None,
   Args:
     value: A 3D `Tensor`.  Must be of type `float16`, `float32`, or `float64`.
     filters: A 3D `Tensor`.  Must have the same type as `value`.
-    stride: An `integer`.  The number of entries by which
-      the filter is moved right at each step.
+    stride: An int or list of `ints` that has length `1` or `3`.  The number of
+      entries by which the filter is moved right at each step.
     padding: 'SAME' or 'VALID'
     use_cudnn_on_gpu: An optional `bool`.  Defaults to `True`.
     data_format: An optional `string` from `"NWC", "NCW"`.  Defaults
@@ -4044,6 +4046,10 @@ def conv1d(value=None,
       data as [batch, in_channels, in_width].
     name: A name for the operation (optional).
     input: Alias for value.
+    dilations: An int or list of `ints` that has length `1` or `3` which
+      defaults to 1. The dilation factor for each dimension of input. If set to
+      k > 1, there will be k-1 skipped cells between each filter element on that
+      dimension. Dilations in the batch and depth dimensions must be 1.
 
   Returns:
     A `Tensor`.  Has the same type as input.
@@ -4057,13 +4063,16 @@ def conv1d(value=None,
     if data_format is None or data_format == "NHWC" or data_format == "NWC":
       data_format = "NHWC"
       spatial_start_dim = 1
-      strides = [1, 1, stride, 1]
+      channel_index = 2
     elif data_format == "NCHW" or data_format == "NCW":
       data_format = "NCHW"
       spatial_start_dim = 2
-      strides = [1, 1, 1, stride]
+      channel_index = 1
     else:
       raise ValueError("data_format must be \"NWC\" or \"NCW\".")
+    strides = [1] + _get_sequence(stride, 1, channel_index, "stride")
+    dilations = [1] + _get_sequence(dilations, 1, channel_index, "dilations")
+
     value = array_ops.expand_dims(value, spatial_start_dim)
     filters = array_ops.expand_dims(filters, 0)
     result = gen_nn_ops.conv2d(
@@ -4072,17 +4081,21 @@ def conv1d(value=None,
         strides,
         padding,
         use_cudnn_on_gpu=use_cudnn_on_gpu,
-        data_format=data_format)
+        data_format=data_format,
+        dilations=dilations,
+        name=name)
     return array_ops.squeeze(result, [spatial_start_dim])
 
 
 @tf_export("nn.conv1d", v1=[])
-def conv1d_v2(input,  # pylint: disable=redefined-builtin
-              filters,
-              stride,
-              padding,
-              data_format=None,
-              name=None):
+def conv1d_v2(
+    input,  # pylint: disable=redefined-builtin
+    filters,
+    stride,
+    padding,
+    data_format="NWC",
+    dilations=None,
+    name=None):
   r"""Computes a 1-D convolution given 3-D input and filter tensors.
 
   Given an input tensor of shape
@@ -4110,13 +4123,17 @@ def conv1d_v2(input,  # pylint: disable=redefined-builtin
   Args:
     input: A 3D `Tensor`.  Must be of type `float16`, `float32`, or `float64`.
     filters: A 3D `Tensor`.  Must have the same type as `input`.
-    stride: An `integer`.  The number of entries by which
-      the filter is moved right at each step.
+    stride: An int or list of `ints` that has length `1` or `3`.  The number of
+      entries by which the filter is moved right at each step.
     padding: 'SAME' or 'VALID'
     data_format: An optional `string` from `"NWC", "NCW"`.  Defaults
       to `"NWC"`, the data is stored in the order of
       [batch, in_width, in_channels].  The `"NCW"` format stores
       data as [batch, in_channels, in_width].
+    dilations: An int or list of `ints` that has length `1` or `3` which
+      defaults to 1. The dilation factor for each dimension of input. If set to
+      k > 1, there will be k-1 skipped cells between each filter element on that
+      dimension. Dilations in the batch and depth dimensions must be 1.
     name: A name for the operation (optional).
 
   Returns:
@@ -4125,13 +4142,15 @@ def conv1d_v2(input,  # pylint: disable=redefined-builtin
   Raises:
     ValueError: if `data_format` is invalid.
   """
-  return conv1d(input,  # pylint: disable=redefined-builtin
-                filters,
-                stride,
-                padding,
-                use_cudnn_on_gpu=True,
-                data_format=data_format,
-                name=name)
+  return conv1d(
+      input,  # pylint: disable=redefined-builtin
+      filters,
+      stride,
+      padding,
+      use_cudnn_on_gpu=True,
+      data_format=data_format,
+      name=name,
+      dilations=dilations)
 
 
 def conv1d_transpose(
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
index cf0d9b9b6d..b8a56e1cf0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
@@ -54,7 +54,7 @@ tf_module {
   }
   member_method {
     name: "conv1d"
-    argspec: "args=[\'value\', \'filters\', \'stride\', \'padding\', \'use_cudnn_on_gpu\', \'data_format\', \'name\', \'input\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'value\', \'filters\', \'stride\', \'padding\', \'use_cudnn_on_gpu\', \'data_format\', \'name\', \'input\', \'dilations\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "conv2d"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index bfe627f651..930cce32e4 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -50,7 +50,7 @@ tf_module {
   }
   member_method {
     name: "conv1d"
-    argspec: "args=[\'input\', \'filters\', \'stride\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+    argspec: "args=[\'input\', \'filters\', \'stride\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NWC\', \'None\', \'None\'], "
   }
   member_method {
     name: "conv2d"
-- 
GitLab


From f722aee786f51962f07eaf02653ff509a1647b1c Mon Sep 17 00:00:00 2001
From: Doe Hyun Yoon <dyoon@google.com>
Date: Tue, 5 Feb 2019 10:21:32 -0800
Subject: [PATCH 1921/2345] Return minimum costs for all persistent ops: Const
 and Vars.

Currently, PredictCosts() returns minimum costs for Const, Var, and VarV2, but not other var ops. Also, the logic was incorrect in that it sets minimum cost to compute time, but zero to execution time.

This CL fixes these.

PiperOrigin-RevId: 232510666
---
 .../costs/analytical_cost_estimator_test.cc   |  2 +-
 .../grappler/costs/op_level_cost_estimator.cc | 47 +++++++++++--------
 .../grappler/costs/op_level_cost_estimator.h  |  1 +
 .../costs/op_level_cost_estimator_test.cc     | 20 ++++++++
 4 files changed, 50 insertions(+), 20 deletions(-)

diff --git a/tensorflow/core/grappler/costs/analytical_cost_estimator_test.cc b/tensorflow/core/grappler/costs/analytical_cost_estimator_test.cc
index eb7ee8dc0a..2c319b6c6b 100644
--- a/tensorflow/core/grappler/costs/analytical_cost_estimator_test.cc
+++ b/tensorflow/core/grappler/costs/analytical_cost_estimator_test.cc
@@ -102,7 +102,7 @@ TEST_F(AnalyticalCostEstimatorTest, SimpleTest) {
   Costs summary;
   TF_ASSERT_OK(estimator.PredictCosts(item.graph, &run_metadata, &summary));
 
-  EXPECT_EQ(Costs::NanoSeconds(9151), summary.execution_time);
+  EXPECT_EQ(Costs::NanoSeconds(9157), summary.execution_time);
   // Note there are totally 17 nodes (RandomUniform creates 2 nodes), but
   // grappler will not process "label", therefore we have 15 here instead
   EXPECT_EQ(15, summary.num_ops_total);
diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
index 59d20f1fb9..1e2e160955 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
@@ -27,7 +27,6 @@ namespace tensorflow {
 namespace grappler {
 
 constexpr int kOpsPerMac = 2;
-constexpr char kConst[] = "Const";
 constexpr char kGuaranteeConst[] = "GuaranteeConst";
 constexpr char kConv2d[] = "Conv2D";
 constexpr char kConv2dBackpropFilter[] = "Conv2DBackpropFilter";
@@ -50,8 +49,6 @@ constexpr char kSqueeze[] = "Squeeze";
 constexpr char kRecv[] = "_Recv";
 constexpr char kSend[] = "_Send";
 constexpr char kBatchMatMul[] = "BatchMatMul";
-constexpr char kVariable[] = "Variable";
-constexpr char kVariableV2[] = "VariableV2";
 constexpr char kRank[] = "Rank";
 constexpr char kShape[] = "Shape";
 constexpr char kShapeN[] = "ShapeN";
@@ -68,6 +65,13 @@ constexpr char kAvgPoolGrad[] = "AvgPoolGrad";
 constexpr char kFusedBatchNorm[] = "FusedBatchNorm";
 constexpr char kFusedBatchNormGrad[] = "FusedBatchNormGrad";
 constexpr char kQuantizedMatMulV2[] = "QuantizedMatMulV2";
+// Persistent ops.
+constexpr char kConst[] = "Const";
+constexpr char kVariable[] = "Variable";
+constexpr char kVariableV2[] = "VariableV2";
+constexpr char kAutoReloadVariable[] = "AutoReloadVariable";
+constexpr char kVarHandleOp[] = "VarHandleOp";
+constexpr char kReadVariableOp[] = "ReadVariableOp";
 
 static const Costs::Duration kMinComputeTime(1);
 
@@ -259,10 +263,6 @@ OpLevelCostEstimator::OpLevelCostEstimator() {
       {kRecv, wrap(&OpLevelCostEstimator::PredictIdentity)},
       {kSend, wrap(&OpLevelCostEstimator::PredictIdentity)},
 
-      {kConst, wrap(&OpLevelCostEstimator::PredictVariable)},
-      {kVariable, wrap(&OpLevelCostEstimator::PredictVariable)},
-      {kVariableV2, wrap(&OpLevelCostEstimator::PredictVariable)},
-
       {kRank, wrap(&OpLevelCostEstimator::PredictMetadata)},
       {kShape, wrap(&OpLevelCostEstimator::PredictMetadata)},
       {kShapeN, wrap(&OpLevelCostEstimator::PredictMetadata)},
@@ -276,6 +276,11 @@ OpLevelCostEstimator::OpLevelCostEstimator() {
        wrap(&OpLevelCostEstimator::PredictFusedBatchNormGrad)},
   };
 
+  persistent_ops_ = {
+      kConst,       kVariable,       kVariableV2, kAutoReloadVariable,
+      kVarHandleOp, kReadVariableOp,
+  };
+
 #define EIGEN_COST(X) Eigen::internal::functor_traits<Eigen::internal::X>::Cost
 
   // Quantize = apply min and max bounds, multiply by scale factor and round.
@@ -363,21 +368,25 @@ OpLevelCostEstimator::OpLevelCostEstimator() {
 Costs OpLevelCostEstimator::PredictCosts(const OpContext& op_context) const {
   const auto& op_info = op_context.op_info;
   auto it = device_cost_impl_.find(op_info.op());
-  if (it == device_cost_impl_.end()) {
-    if (elementwise_ops_.find(op_info.op()) != elementwise_ops_.end()) {
-      return PredictCwiseOp(op_context);
-    }
+  if (it != device_cost_impl_.end()) {
+    std::function<Costs(const OpContext&)> estimator = it->second;
+    Costs costs = estimator(op_context);
+    VLOG(1) << "Operation " << op_info.op() << " takes "
+            << costs.execution_time.count() << " ns.";
+    return costs;
+  }
 
-    VLOG(1) << "Missing accurate estimator for op: " << op_info.op();
+  if (persistent_ops_.find(op_info.op()) != persistent_ops_.end()) {
+    return PredictVariable(op_context);
+  }
 
-    return PredictCostOfAnUnknownOp(op_context);
+  if (elementwise_ops_.find(op_info.op()) != elementwise_ops_.end()) {
+    return PredictCwiseOp(op_context);
   }
 
-  std::function<Costs(const OpContext&)> estimator = it->second;
-  Costs costs = estimator(op_context);
-  VLOG(1) << "Operation " << op_info.op() << " takes "
-          << costs.execution_time.count() << " ns.";
-  return costs;
+  VLOG(1) << "Missing accurate estimator for op: " << op_info.op();
+
+  return PredictCostOfAnUnknownOp(op_context);
 }
 
 DeviceInfo OpLevelCostEstimator::GetDeviceInfo(
@@ -1240,7 +1249,7 @@ Costs OpLevelCostEstimator::PredictVariable(const OpContext& op_context) const {
   result.num_ops_with_unknown_shapes = result.inaccurate;
 
   result.compute_time = kMinComputeTime;
-  result.execution_time = result.execution_time;
+  result.execution_time = result.compute_time;
   return result;
 }
 
diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator.h b/tensorflow/core/grappler/costs/op_level_cost_estimator.h
index f8ba8c6637..ace8fb218c 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator.h
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator.h
@@ -193,6 +193,7 @@ class OpLevelCostEstimator {
   // If true, assume compute and memory overlap; hence, the op cost is max of
   // compute_time and memory_time, insteaf of sum of those two.
   bool compute_memory_overlap_;
+  std::set<string> persistent_ops_;
 
  private:
   friend class OpLevelCostEstimatorTest;
diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
index aa0fc9d6c2..04c6ada2bf 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
@@ -499,6 +499,26 @@ class OpLevelCostEstimatorTest : public ::testing::Test {
   OpLevelCostEstimator estimator_;
 };
 
+TEST_F(OpLevelCostEstimatorTest, TestPersistentOpCosts) {
+  OpContext op_context;
+  SetCpuDevice(&op_context.op_info);
+  std::unordered_set<string> persisent_ops = {
+      "Const",       "Variable",       "VariableV2", "AutoReloadVariable",
+      "VarHandleOp", "ReadVariableOp",
+  };
+  // Minmum cost for all persistent ops.
+  for (const auto& op : persisent_ops) {
+    op_context.op_info.set_op(op);
+    auto cost = estimator_.PredictCosts(op_context);
+    EXPECT_EQ(Costs::Duration(0), cost.memory_time);
+    EXPECT_EQ(Costs::Duration(1), cost.compute_time);
+    EXPECT_EQ(Costs::Duration(1), cost.execution_time);
+    EXPECT_EQ(1, cost.num_ops_total);
+    EXPECT_FALSE(cost.inaccurate);
+    EXPECT_EQ(0, cost.num_ops_with_unknown_shapes);
+  }
+}
+
 TEST_F(OpLevelCostEstimatorTest, TestGatherCosts) {
   OpContext op_context;
   SetCpuDevice(&op_context.op_info);
-- 
GitLab


From cc79252c0b5949506fce1d01b78e96e564a9c20f Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Tue, 5 Feb 2019 10:39:23 -0800
Subject: [PATCH 1922/2345] Refactor TF-TRT API: - Add a new GraphConverter
 class to provide general support for offline   GraphDef/SavedModel
 conversion. It also supports post-conversion calibration.   This way we can
 make a backend specific converter by inheriting this class   and overwrite
 the get_rewriter_config() method to provide backend specific   rewriter
 options. - Add a new TrtGraphConverter class (inherited from GraphConverter)
 for TRT   conversion.

PiperOrigin-RevId: 232514321
---
 .../test/tf_trt_integration_test_base.py      |   2 +-
 .../python/compiler/tensorrt/trt_convert.py   | 697 ++++++++++++------
 .../compiler/tensorrt/trt_convert_test.py     |   6 +-
 3 files changed, 471 insertions(+), 234 deletions(-)

diff --git a/tensorflow/python/compiler/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/python/compiler/tensorrt/test/tf_trt_integration_test_base.py
index bba661b879..28563f09a1 100644
--- a/tensorflow/python/compiler/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/python/compiler/tensorrt/test/tf_trt_integration_test_base.py
@@ -259,7 +259,7 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
     """Get config proto based on specific settings."""
     conversion_params = self.GetConversionParams(run_params)
     if graph_state != GraphState.ORIGINAL and run_params.use_optimizer:
-      rewriter_cfg = trt_convert.get_tensorrt_rewriter_config(
+      rewriter_cfg = trt_convert.TrtGraphConverter.get_tensorrt_rewriter_config(
           conversion_params.rewriter_config, conversion_params.max_batch_size,
           conversion_params.max_workspace_size_bytes,
           conversion_params.precision_mode,
diff --git a/tensorflow/python/compiler/tensorrt/trt_convert.py b/tensorflow/python/compiler/tensorrt/trt_convert.py
index cff22ec3ac..33b5e50418 100644
--- a/tensorflow/python/compiler/tensorrt/trt_convert.py
+++ b/tensorflow/python/compiler/tensorrt/trt_convert.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 import six as _six
 # pylint: disable=unused-import,line-too-long
+from tensorflow.compiler.tf2tensorrt.python.ops import trt_ops
 from tensorflow.python.compiler.tensorrt.wrap_conversion import add_test_value
 from tensorflow.python.compiler.tensorrt.wrap_conversion import calib_convert
 from tensorflow.python.compiler.tensorrt.wrap_conversion import clear_test_values
@@ -41,7 +42,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.grappler import tf_optimizer
 from tensorflow.python.platform import tf_logging
 from tensorflow.python.saved_model import builder
-from tensorflow.python.saved_model import loader_impl
+from tensorflow.python.saved_model import loader
 from tensorflow.python.saved_model import tag_constants
 from tensorflow.python.training import saver
 
@@ -60,6 +61,228 @@ def _to_string(s):
   return s
 
 
+class GraphConverter(object):
+  """Base class for offline converters to optimize SavedModels/GraphDefs.
+
+  A `GraphConverter` object encapsulates the environment to convert (optimize) a
+  TensorFlow SavedModel or GraphDef.
+
+  To create a custom GraphConverter:
+
+  ```python
+  class MyGraphConverter(GraphConverter):
+    ...
+
+    def get_rewriter_config(self, rewriter_config_template=None):
+      my_rewriter_config = ...
+      return my_rewriter_config
+  ```
+
+  Then to run the conversion without quantization calibration:
+
+  ```python
+  my_converter = MyGraphConverter(input_saved_model_dir="my_dir")
+  converted_graph_def = my_converter.convert()
+  my_converter.save(output_saved_model_dir)  # Optional
+  ```
+
+  TODO(laigd): add calibration support.
+  """
+
+  def __init__(self,
+               input_saved_model_dir=None,
+               input_saved_model_tags=None,
+               input_graph_def=None,
+               nodes_blacklist=None,
+               session_config=None):
+    """Initialize the converter.
+
+    Args:
+      input_saved_model_dir: the directory to load the SavedModel which contains
+        the input graph to transforms. Used only when input_graph_def is None.
+      input_saved_model_tags: list of tags to load the SavedModel.
+      input_graph_def: a GraphDef object containing a model to be transformed.
+        If set to None, the graph will be read from the SavedModel loaded from
+        input_saved_model_dir.
+      nodes_blacklist: list of node names to prevent the converter from
+        touching. Only used when input_graph_def is not None.
+      session_config: the ConfigProto used to create a Session. It's also used
+        as a template to create a RewriterConfig for conversion. If not
+        specified, a default ConfigProto will be used.
+
+    Raises:
+      ValueError: if the combination of the parameters is invalid.
+    """
+    if input_graph_def and input_saved_model_dir:
+      raise ValueError(
+          "Can only specify one of input_graph_def and input_saved_model_dir")
+    if not input_graph_def and not input_saved_model_dir:
+      raise ValueError("Must specify one of input_graph_def and "
+                       "input_saved_model_dir")
+
+    self._input_graph_def = input_graph_def
+    self._nodes_blacklist = nodes_blacklist
+    self._input_saved_model_dir = input_saved_model_dir
+    self._converted = False
+    self._grappler_meta_graph_def = None
+
+    self._input_saved_model_tags = (
+        input_saved_model_tags or [tag_constants.SERVING])
+    self._session_config = session_config or config_pb2.ConfigProto()
+
+  def get_rewriter_config(self, rewriter_config_template=None):
+    """Returns a RewriterConfig proto for TRT transformation.
+
+    Args:
+      rewriter_config_template: a template RewriterConfig proto used to create a
+        RewriterConfig for the conversion. The implementation should not modify
+        the template. If None, it will use a default one.
+
+    Returns:
+      A RewriterConfig proto which will be used to run the conversion using
+      Grappler.
+    """
+    raise NotImplementedError("get_rewriter_config")
+
+  def _run_conversion(self):
+    """Run Grappler's OptimizeGraph() tool to convert the graph."""
+    # Create custom ConfigProto for Grappler.
+    grappler_session_config = config_pb2.ConfigProto()
+    grappler_session_config.CopyFrom(self._session_config)
+    rewriter_config = None
+    if (grappler_session_config.HasField("graph_options") and
+        grappler_session_config.graph_options.HasField("rewrite_options")):
+      rewriter_config = grappler_session_config.graph_options.rewrite_options
+    custom_rewriter_config = self.get_rewriter_config(rewriter_config)
+    grappler_session_config.graph_options.rewrite_options.CopyFrom(
+        custom_rewriter_config)
+
+    # Run Grappler.
+    self._converted_graph_def = tf_optimizer.OptimizeGraph(
+        grappler_session_config,
+        self._grappler_meta_graph_def,
+        graph_id=b"tf_graph")
+    self._converted = True
+
+  def _convert_graph_def(self):
+    """Convert the input GraphDef."""
+    graph = ops.Graph()
+    with graph.as_default():
+      importer.import_graph_def(self._input_graph_def, name="")
+    self._grappler_meta_graph_def = saver.export_meta_graph(
+        graph_def=graph.as_graph_def(add_shapes=True), graph=graph)
+    if self._nodes_blacklist:
+      output_collection = meta_graph_pb2.CollectionDef()
+      output_list = output_collection.node_list.value
+      for i in self._nodes_blacklist:
+        if isinstance(i, ops.Tensor):
+          output_list.append(_to_bytes(i.name))
+        else:
+          output_list.append(_to_bytes(i))
+      # TODO(laigd): use another key as the self._nodes_blacklist are really
+      # not train_op.
+      self._grappler_meta_graph_def.collection_def["train_op"].CopyFrom(
+          output_collection)
+
+    self._run_conversion()
+
+  def _convert_saved_model(self):
+    """Convert the input SavedModel."""
+    graph = ops.Graph()
+    with session.Session(graph=graph, config=self._session_config) as sess:
+      input_meta_graph_def = loader.load(sess, self._input_saved_model_tags,
+                                         self._input_saved_model_dir)
+
+      def _gather_names(tensor_info):
+        """Get the node names from a TensorInfo."""
+        return set([tensor_info[key].name.split(":")[0] for key in tensor_info])
+
+      # Get input and outputs from all SignatureDef.
+      output_node_names = set()
+      for key in input_meta_graph_def.signature_def:
+        signature_def = input_meta_graph_def.signature_def[key]
+        output_node_names.update(_gather_names(signature_def.inputs))
+        output_node_names.update(_gather_names(signature_def.outputs))
+
+      # Freeze the variables in the SavedModel graph and copy the frozen
+      # graph over.
+      frozen_graph_def = graph_util.convert_variables_to_constants(
+          sess, sess.graph.as_graph_def(add_shapes=True),
+          list(output_node_names))
+      self._grappler_meta_graph_def = meta_graph_pb2.MetaGraphDef()
+      self._grappler_meta_graph_def.graph_def.CopyFrom(frozen_graph_def)
+
+      # Copy the collections that are not variables.
+      for key in input_meta_graph_def.collection_def:
+        # TODO(laigd): currently we use the collection key to filter out
+        # collections that depend on variable ops, but this may miss some
+        # other user-defined collections. A better way would be to use
+        # CollectionDef::NodeList for the filtering.
+        if key not in [
+            "variables", "local_variables", "model_variables",
+            "trainable_variables", "train_op", "table_initializer"
+        ]:
+          self._grappler_meta_graph_def.collection_def[key].CopyFrom(
+              input_meta_graph_def.collection_def[key])
+
+      # Copy other information.
+      self._grappler_meta_graph_def.meta_info_def.CopyFrom(
+          input_meta_graph_def.meta_info_def)
+      for key in input_meta_graph_def.signature_def:
+        self._grappler_meta_graph_def.signature_def[key].CopyFrom(
+            input_meta_graph_def.signature_def[key])
+      # TODO(laigd): maybe add back AssetFileDef.
+
+    self._run_conversion()
+
+  def convert(self):
+    """Run the conversion.
+
+    Returns:
+      The converted GraphDef.
+    """
+    assert not self._converted
+
+    if self._input_graph_def:
+      self._convert_graph_def()
+    else:
+      self._convert_saved_model()
+    return self._converted_graph_def
+
+  def save(self, output_saved_model_dir):
+    """Save the converted graph as a SavedModel.
+
+    Args:
+      output_saved_model_dir: construct a SavedModel using the converted
+        GraphDef and save it to the specified directory. This option only works
+        when the input graph is loaded from a SavedModel, i.e. when
+        input_saved_model_dir is specified and input_graph_def is None in
+        __init__().
+
+    Raises:
+      ValueError: if the input to the converter is a GraphDef instead of a
+      SavedModel.
+    """
+    assert self._converted
+
+    if self._input_graph_def:
+      raise ValueError(
+          "Not able to save to a SavedModel since input is a GraphDef")
+
+    # Write the transformed graphdef as SavedModel.
+    saved_model_builder = builder.SavedModelBuilder(output_saved_model_dir)
+    with ops.Graph().as_default():
+      importer.import_graph_def(self._converted_graph_def, name="")
+      # We don't use any specific converter here.
+      with session.Session(config=self._session_config) as sess:
+        saved_model_builder.add_meta_graph_and_variables(
+            sess,
+            self._input_saved_model_tags,
+            signature_def_map=self._grappler_meta_graph_def.signature_def)
+    # Ignore other meta graphs from the input SavedModel.
+    saved_model_builder.save()
+
+
 class TrtPrecisionMode(object):
   FP32 = "FP32"
   FP16 = "FP16"
@@ -75,97 +298,231 @@ class TrtPrecisionMode(object):
 DEFAULT_TRT_MAX_WORKSPACE_SIZE_BYTES = 1 << 30
 
 
-def get_tensorrt_rewriter_config(
-    rewriter_config=None,
-    max_batch_size=1,
-    max_workspace_size_bytes=DEFAULT_TRT_MAX_WORKSPACE_SIZE_BYTES,
-    precision_mode=TrtPrecisionMode.FP32,
-    minimum_segment_size=3,
-    is_dynamic_op=False,
-    maximum_cached_engines=1,
-    cached_engine_batches=None,
-    use_calibration=True):
-  """Returns a RewriterConfig proto for TRT transformation.
-
-  Args:
-    rewriter_config: a template RewriterConfig proto used to create a
-      TRT-enabled RewriterConfig. If None, it will use a default one.
-    max_batch_size: max size for the input batch
-    max_workspace_size_bytes: the maximum GPU temporary memory which the TRT
-      engine can use at execution time. This corresponds to the 'workspaceSize'
-      parameter of nvinfer1::IBuilder::setMaxWorkspaceSize().
-    precision_mode: one of TrtPrecisionMode.supported_precision_modes().
-    minimum_segment_size: the minimum number of nodes required for a subgraph to
-      be replaced by TRTEngineOp.
-    is_dynamic_op: whether to generate dynamic TRT ops which will build the TRT
-      network and engine at run time.
-    maximum_cached_engines: max number of cached TRT engines in dynamic TRT ops.
-      If the number of cached engines is already at max but none of them can
-      serve the input, the TRTEngineOp will fall back to run the TF function
-      based on which the TRTEngineOp is created.
-    cached_engine_batches: a list of batch sizes used to create cached engines,
-      only used when is_dynamic_op is True. The length of the list should be <=
-      maximum_cached_engines, and the dynamic TRT op will use this list to
-      determine the batch sizes of the cached engines, instead of making the
-      decision on the fly. This is useful when we know the most common batch
-      size(s) the application is going to generate.
-    use_calibration: this argument is ignored if precision_mode is not INT8. If
-      set to True, a calibration graph will be created to calibrate the missing
-      ranges. The calibration graph must be converted to an inference graph
-      using calib_graph_to_infer_graph() after running calibration. if set to
-      False, quantization nodes will be expected for every tensor in the graph
-      (exlcuding those which will be fused). If a range is missing, an error
-      will occur. Please note that accuracy may be negatively affected if there
-      is a mismatch between which tensors TRT quantizes and which tensors were
-      trained with fake quantization.
-
-  Returns:
-    A RewriterConfig proto which sets a TensorRTOptimizer to run Grappler.
-
-  Raises:
-    TypeError: if any of the parameters are of unexpected type.
-    ValueError: if any of the parameters are of unexpected value.
-  """
-  if rewriter_config is not None and not isinstance(
-      rewriter_config, rewriter_config_pb2.RewriterConfig):
-    raise TypeError("rewriter_config should be a RewriterConfig proto.")
-
-  rewriter_config_with_trt = rewriter_config_pb2.RewriterConfig()
-  if rewriter_config is None:
-    # Layout optimizer may add Const nodes followed by Reshape nodes, thus we
-    # need to run constant folding again.
-    rewriter_config_with_trt.optimizers.extend(
-        ["constfold", "layout", "constfold"])
-    rewriter_config_with_trt.meta_optimizer_iterations = (
-        rewriter_config_pb2.RewriterConfig.ONE)
-  else:
-    rewriter_config_with_trt.CopyFrom(rewriter_config)
-
-  if precision_mode.upper() not in TrtPrecisionMode.supported_precision_modes():
-    raise ValueError(("precision mode '{}' is not supported."
-                      "It should be one of {}").format(
-                          precision_mode,
-                          TrtPrecisionMode.supported_precision_modes))
-
-  optimizer = rewriter_config_with_trt.custom_optimizers.add()
-  optimizer.name = "TensorRTOptimizer"
-  optimizer.parameter_map["minimum_segment_size"].i = minimum_segment_size
-  optimizer.parameter_map["max_batch_size"].i = max_batch_size
-  optimizer.parameter_map["is_dynamic_op"].b = is_dynamic_op
-  optimizer.parameter_map[
-      "max_workspace_size_bytes"].i = max_workspace_size_bytes
-  optimizer.parameter_map["precision_mode"].s = _to_bytes(precision_mode)
-  optimizer.parameter_map["maximum_cached_engines"].i = maximum_cached_engines
-  if cached_engine_batches:
-    if not isinstance(cached_engine_batches, list):
-      raise TypeError("cached_engine_batches should be a list.")
-    if len(cached_engine_batches) > maximum_cached_engines:
-      raise ValueError("cached_engine_batches should not contain more than "
-                       "maximum_cached_engines items.")
-    optimizer.parameter_map["cached_engine_batches"].list.i.extend(
-        cached_engine_batches)
-  optimizer.parameter_map["use_calibration"].b = use_calibration
-  return rewriter_config_with_trt
+class TrtGraphConverter(GraphConverter):
+  """A GraphConverter for TRT transformation."""
+
+  _TRT_CALIBRATION_RESOURCE_CONTAINER_NAME = "TF_TRT_Calibration"
+
+  @classmethod
+  def get_tensorrt_rewriter_config(
+      cls,
+      rewriter_config_template=None,
+      max_batch_size=1,
+      max_workspace_size_bytes=DEFAULT_TRT_MAX_WORKSPACE_SIZE_BYTES,
+      precision_mode=TrtPrecisionMode.FP32,
+      minimum_segment_size=3,
+      is_dynamic_op=False,
+      maximum_cached_engines=1,
+      cached_engine_batches=None,
+      use_calibration=True):
+    """Returns a RewriterConfig proto for TRT transformation.
+
+    Args:
+      rewriter_config_template: a template RewriterConfig proto used to create a
+        TRT-enabled RewriterConfig. If None, it will use a default one.
+      max_batch_size: max size for the input batch
+      max_workspace_size_bytes: the maximum GPU temporary memory which the TRT
+        engine can use at execution time. This corresponds to the
+        'workspaceSize'
+        parameter of nvinfer1::IBuilder::setMaxWorkspaceSize().
+      precision_mode: one of TrtPrecisionMode.supported_precision_modes().
+      minimum_segment_size: the minimum number of nodes required for a subgraph
+        to be replaced by TRTEngineOp.
+      is_dynamic_op: whether to generate dynamic TRT ops which will build the
+        TRT network and engine at run time.
+      maximum_cached_engines: max number of cached TRT engines in dynamic TRT
+        ops. If the number of cached engines is already at max but none of them
+        can serve the input, the TRTEngineOp will fall back to run the TF
+        function based on which the TRTEngineOp is created.
+      cached_engine_batches: a list of batch sizes used to create cached
+        engines, only used when is_dynamic_op is True. The length of the list
+        should be <= maximum_cached_engines, and the dynamic TRT op will use
+        this list to determine the batch sizes of the cached engines, instead of
+        making the decision on the fly. This is useful when we know the most
+        common batch size(s) the application is going to generate.
+      use_calibration: this argument is ignored if precision_mode is not INT8.
+        If set to True, a calibration graph will be created to calibrate the
+        missing ranges. The calibration graph must be converted to an inference
+        graph using calib_graph_to_infer_graph() after running calibration. if
+        set to False, quantization nodes will be expected for every tensor in
+        the graph (exlcuding those which will be fused). If a range is missing,
+        an error will occur. Please note that accuracy may be negatively
+        affected if there is a mismatch between which tensors TRT quantizes and
+        which tensors were trained with fake quantization.
+
+    Returns:
+      A RewriterConfig proto which sets a TensorRTOptimizer to run Grappler.
+
+    Raises:
+      TypeError: if any of the parameters are of unexpected type.
+      ValueError: if any of the parameters are of unexpected value.
+    """
+    if rewriter_config_template is not None and not isinstance(
+        rewriter_config_template, rewriter_config_pb2.RewriterConfig):
+      raise TypeError(
+          "rewriter_config_template should be a RewriterConfig proto.")
+
+    rewriter_config_with_trt = rewriter_config_pb2.RewriterConfig()
+    if rewriter_config_template is None:
+      # Layout optimizer may add Const nodes followed by Reshape nodes, thus we
+      # need to run constant folding again.
+      rewriter_config_with_trt.optimizers.extend(
+          ["constfold", "layout", "constfold"])
+      rewriter_config_with_trt.meta_optimizer_iterations = (
+          rewriter_config_pb2.RewriterConfig.ONE)
+    else:
+      rewriter_config_with_trt.CopyFrom(rewriter_config_template)
+
+    optimizer = rewriter_config_with_trt.custom_optimizers.add()
+    optimizer.name = "TensorRTOptimizer"
+    optimizer.parameter_map["minimum_segment_size"].i = minimum_segment_size
+    optimizer.parameter_map["max_batch_size"].i = max_batch_size
+    optimizer.parameter_map["is_dynamic_op"].b = is_dynamic_op
+    optimizer.parameter_map[
+        "max_workspace_size_bytes"].i = max_workspace_size_bytes
+    optimizer.parameter_map["precision_mode"].s = _to_bytes(precision_mode)
+    optimizer.parameter_map["maximum_cached_engines"].i = maximum_cached_engines
+    if cached_engine_batches:
+      optimizer.parameter_map["cached_engine_batches"].list.i.extend(
+          cached_engine_batches)
+    optimizer.parameter_map["use_calibration"].b = use_calibration
+    return rewriter_config_with_trt
+
+  def __init__(self,
+               input_saved_model_dir=None,
+               input_saved_model_tags=None,
+               input_graph_def=None,
+               nodes_blacklist=None,
+               session_config=None,
+               max_batch_size=1,
+               max_workspace_size_bytes=DEFAULT_TRT_MAX_WORKSPACE_SIZE_BYTES,
+               precision_mode=TrtPrecisionMode.FP32,
+               minimum_segment_size=3,
+               is_dynamic_op=False,
+               maximum_cached_engines=1,
+               cached_engine_batches=None,
+               use_calibration=True):
+    """Initialize the converter.
+
+    Args:
+      input_saved_model_dir: the directory to load the SavedModel which contains
+        the input graph to transforms. Used only when input_graph_def is None.
+      input_saved_model_tags: list of tags to load the SavedModel.
+      input_graph_def: a GraphDef object containing a model to be transformed.
+        If set to None, the graph will be read from the SavedModel loaded from
+        input_saved_model_dir.
+      nodes_blacklist: list of node names to prevent the converter from
+        touching. Only used when input_graph_def is not None.
+      session_config: the ConfigProto used to create a Session. It's also used
+        as a template to create a TRT-enabled ConfigProto for conversion. If not
+        specified, a default ConfigProto will be used.
+      max_batch_size: max size for the input batch.
+      max_workspace_size_bytes: the maximum GPU temporary memory which the TRT
+        engine can use at execution time. This corresponds to the
+        'workspaceSize'
+        parameter of nvinfer1::IBuilder::setMaxWorkspaceSize().
+      precision_mode: one of TrtPrecisionMode.supported_precision_modes().
+      minimum_segment_size: the minimum number of nodes required for a subgraph
+        to be replaced by TRTEngineOp.
+      is_dynamic_op: whether to generate dynamic TRT ops which will build the
+        TRT network and engine at run time.
+      maximum_cached_engines: max number of cached TRT engines in dynamic TRT
+        ops. If the number of cached engines is already at max but none of them
+        can serve the input, the TRTEngineOp will fall back to run the TF
+        function based on which the TRTEngineOp is created.
+      cached_engine_batches: a list of batch sizes used to create cached
+        engines, only used when is_dynamic_op is True. The length of the list
+        should be <= maximum_cached_engines, and the dynamic TRT op will use
+        this list to determine the batch sizes of the cached engines, instead of
+        making the decision on the fly. This is useful when we know the most
+        common batch size(s) the application is going to generate.
+      use_calibration: this argument is ignored if precision_mode is not INT8.
+        If set to True, a calibration graph will be created to calibrate the
+        missing ranges. The calibration graph must be converted to an inference
+        graph using calib_graph_to_infer_graph() after running calibration. if
+        set to False, quantization nodes will be expected for every tensor in
+        the graph (exlcuding those which will be fused). If a range is missing,
+        an error will occur. Please note that accuracy may be negatively
+        affected if there is a mismatch between which tensors TRT quantizes and
+        which tensors were trained with fake quantization.
+
+    Raises:
+      ValueError: if the combination of the parameters is invalid.
+      RuntimeError: if the TensorRT library version is incompatible.
+    """
+    super(TrtGraphConverter, self).__init__(
+        input_saved_model_dir=input_saved_model_dir,
+        input_saved_model_tags=input_saved_model_tags,
+        input_graph_def=input_graph_def,
+        nodes_blacklist=nodes_blacklist,
+        session_config=session_config)
+
+    # Check compatibility of TensorRT version.
+    compiled_version = get_linked_tensorrt_version()
+    loaded_version = get_loaded_tensorrt_version()
+    version_mismatch = False
+    if loaded_version[0] < compiled_version[0]:
+      tf_logging.error(
+          "TensorRT version mismatch. Tensorflow was compiled against " +
+          "TensorRT %s but library loaded from environment is TensorRT %s" %
+          (".".join([str(x) for x in compiled_version]),
+           ".".join([str(x) for x in loaded_version])) +
+          ". Please make sure that correct version of TensorRT " +
+          "is available in the system and added to ldconfig or LD_LIBRARY_PATH")
+      raise RuntimeError("Incompatible TensorRT library version")
+    for i in zip(loaded_version, compiled_version):
+      if i[0] != i[1]:
+        tf_logging.warn("TensorRT mismatch. Compiled against version " +
+                        "%s, but loaded %s. Things may not work" %
+                        (".".join([str(x) for x in compiled_version]),
+                         ".".join([str(x) for x in loaded_version])))
+        version_mismatch = True
+        break
+    if not version_mismatch:
+      tf_logging.info("Running against TensorRT version %s" % ".".join(
+          [str(x) for x in loaded_version]))
+
+    # Check input arguments.
+    if precision_mode.upper() not in TrtPrecisionMode.supported_precision_modes(
+    ):
+      raise ValueError(("precision mode '{}' is not supported."
+                        "It should be one of {}").format(
+                            precision_mode,
+                            TrtPrecisionMode.supported_precision_modes))
+
+    if cached_engine_batches:
+      if not isinstance(cached_engine_batches, list):
+        raise TypeError("cached_engine_batches should be a list.")
+      if len(cached_engine_batches) > maximum_cached_engines:
+        raise ValueError("cached_engine_batches should not contain more than "
+                         "maximum_cached_engines items.")
+
+    # TODO(laigd):
+    # - Get rid of is_dynamic_op option, it should always be True, and it should
+    #   accept N shapes as input.
+    # - Verify in int8 mode that maximum_cached_engines and
+    #   cached_engine_batches are set appropriately.
+    # - If it fails to build the int8 engine it should return error.
+    self._max_batch_size = max_batch_size
+    self._max_workspace_size_bytes = max_workspace_size_bytes
+    self._precision_mode = precision_mode
+    self._minimum_segment_size = minimum_segment_size
+    self._is_dynamic_op = is_dynamic_op
+    self._maximum_cached_engines = maximum_cached_engines
+    self._cached_engine_batches = cached_engine_batches
+    self._use_calibration = use_calibration
+
+  def get_rewriter_config(self, rewriter_config_template=None):
+    return TrtGraphConverter.get_tensorrt_rewriter_config(
+        rewriter_config_template,
+        max_batch_size=self._max_batch_size,
+        max_workspace_size_bytes=self._max_workspace_size_bytes,
+        precision_mode=self._precision_mode,
+        minimum_segment_size=self._minimum_segment_size,
+        is_dynamic_op=self._is_dynamic_op,
+        maximum_cached_engines=self._maximum_cached_engines,
+        cached_engine_batches=self._cached_engine_batches,
+        use_calibration=self._use_calibration)
 
 
 def create_inference_graph(
@@ -253,144 +610,24 @@ def create_inference_graph(
     ValueError: if the combination of the parameters is invalid.
     RuntimeError: if the TensorRT library version is incompatible.
   """
-  compiled_version = get_linked_tensorrt_version()
-  loaded_version = get_loaded_tensorrt_version()
-  version_mismatch = False
-  if loaded_version[0] < compiled_version[0]:
-    tf_logging.error(
-        "TensorRT version mismatch. Tensorflow was compiled against " +
-        "TensorRT %s but library loaded from environment is TensorRT %s" %
-        (".".join([str(x) for x in compiled_version]),
-         ".".join([str(x) for x in loaded_version])) +
-        ". Please make sure that correct version of TensorRT " +
-        "is available in the system and added to ldconfig or LD_LIBRARY_PATH")
-    raise RuntimeError("Incompatible TensorRT library version")
-  for i in zip(loaded_version, compiled_version):
-    if i[0] != i[1]:
-      tf_logging.warn("TensorRT mismatch. Compiled against version " +
-                      "%s, but loaded %s. Things may not work" %
-                      (".".join([str(x) for x in compiled_version]),
-                       ".".join([str(x) for x in loaded_version])))
-      version_mismatch = True
-      break
-  if not version_mismatch:
-    tf_logging.info("Running against TensorRT version %s" % ".".join(
-        [str(x) for x in loaded_version]))
-
-  if session_config is None:
-    session_config = config_pb2.ConfigProto()
-
-  if input_saved_model_tags is None:
-    input_saved_model_tags = [tag_constants.SERVING]
-  saved_model_loader = None
-  grappler_meta_graph_def = None
-
-  if input_graph_def is None:
-    # Read from SavedModel and freeze the graph if necessary.
-    if input_saved_model_dir is None:
-      raise ValueError("input_graph_def and input_saved_model_dir cannot be "
-                       "both None")
-    with ops.Graph().as_default():
-      with session.Session(config=session_config) as sess:
-        saved_model_loader = loader_impl.SavedModelLoader(input_saved_model_dir)
-        input_meta_graph_def = saved_model_loader.load(sess,
-                                                       input_saved_model_tags)
-        output_node_names = set()
-
-        def _gather_names(tensor_info):
-          """Get the node names from a TensorInfo."""
-          return set(
-              [tensor_info[key].name.split(":")[0] for key in tensor_info])
-
-        # Get input and outputs from all SignatureDef.
-        for key in input_meta_graph_def.signature_def:
-          signature_def = input_meta_graph_def.signature_def[key]
-          output_node_names.update(_gather_names(signature_def.inputs))
-          output_node_names.update(_gather_names(signature_def.outputs))
-
-        # Freeze the variables in the SavedModel graph and copy the frozen
-        # graph over.
-        frozen_graph_def = graph_util.convert_variables_to_constants(
-            sess, sess.graph.as_graph_def(add_shapes=True),
-            list(output_node_names))
-        grappler_meta_graph_def = meta_graph_pb2.MetaGraphDef()
-        grappler_meta_graph_def.graph_def.CopyFrom(frozen_graph_def)
-
-        # Copy the collections that are not variables.
-        for key in input_meta_graph_def.collection_def:
-          # TODO(laigd): currently we use the collection key to filter out
-          # collections that depend on variable ops, but this may miss some
-          # other user-defined collections. A better way would be to use
-          # CollectionDef::NodeList for the filtering.
-          if key not in [
-              "variables", "local_variables", "model_variables",
-              "trainable_variables", "train_op", "table_initializer"
-          ]:
-            grappler_meta_graph_def.collection_def[key].CopyFrom(
-                input_meta_graph_def.collection_def[key])
-
-        # Copy other information.
-        grappler_meta_graph_def.meta_info_def.CopyFrom(
-            input_meta_graph_def.meta_info_def)
-        for key in input_meta_graph_def.signature_def:
-          grappler_meta_graph_def.signature_def[key].CopyFrom(
-              input_meta_graph_def.signature_def[key])
-        # TODO(laigd): maybe add back AssetFileDef.
-  else:
-    if output_saved_model_dir is not None:
-      raise ValueError("output_saved_model_dir cannot be set when "
-                       "input_graph_def is set")
-    # Create MetaGraphDef from input graph.
-    graph = ops.Graph()
-    with graph.as_default():
-      importer.import_graph_def(input_graph_def, name="")
-    grappler_meta_graph_def = saver.export_meta_graph(
-        graph_def=graph.as_graph_def(add_shapes=True), graph=graph)
-    if outputs:
-      output_collection = meta_graph_pb2.CollectionDef()
-      output_list = output_collection.node_list.value
-      for i in outputs:
-        if isinstance(i, ops.Tensor):
-          output_list.append(_to_bytes(i.name))
-        else:
-          output_list.append(_to_bytes(i))
-      # TODO(laigd): use another key as the outputs are really not train_op.
-      grappler_meta_graph_def.collection_def["train_op"].CopyFrom(
-          output_collection)
-
-  # Create TRT-enabled ConfigProto.
-  session_config_with_trt = config_pb2.ConfigProto()
-  session_config_with_trt.CopyFrom(session_config)
-  rewriter_config = None
-  if (session_config_with_trt.HasField("graph_options") and
-      session_config_with_trt.graph_options.HasField("rewrite_options")):
-    rewriter_config = session_config_with_trt.graph_options.rewrite_options
-  rewriter_config_with_trt = get_tensorrt_rewriter_config(
-      rewriter_config, max_batch_size, max_workspace_size_bytes, precision_mode,
-      minimum_segment_size, is_dynamic_op, maximum_cached_engines,
-      cached_engine_batches, use_calibration)
-  session_config_with_trt.graph_options.rewrite_options.CopyFrom(
-      rewriter_config_with_trt)
-
-  # Run Grappler.
-  transformed_graph_def = tf_optimizer.OptimizeGraph(
-      session_config_with_trt, grappler_meta_graph_def, graph_id=b"tf_graph")
-
-  # Optionally write the transformed graphdef as SavedModel.
-  if output_saved_model_dir is not None:
-    saved_model_builder = builder.SavedModelBuilder(output_saved_model_dir)
-    with ops.Graph().as_default():
-      importer.import_graph_def(transformed_graph_def, name="")
-      # We don't use TRT here.
-      with session.Session(config=session_config) as sess:
-        saved_model_builder.add_meta_graph_and_variables(
-            sess,
-            input_saved_model_tags,
-            signature_def_map=grappler_meta_graph_def.signature_def)
-    # Ignore other meta graphs from the input SavedModel.
-    saved_model_builder.save()
-
-  return transformed_graph_def
+  trt_converter = TrtGraphConverter(
+      input_saved_model_dir=input_saved_model_dir,
+      input_saved_model_tags=input_saved_model_tags,
+      input_graph_def=input_graph_def,
+      nodes_blacklist=outputs,
+      session_config=session_config,
+      max_batch_size=max_batch_size,
+      max_workspace_size_bytes=max_workspace_size_bytes,
+      precision_mode=precision_mode,
+      minimum_segment_size=minimum_segment_size,
+      is_dynamic_op=is_dynamic_op,
+      maximum_cached_engines=maximum_cached_engines,
+      cached_engine_batches=cached_engine_batches,
+      use_calibration=use_calibration)
+  converted_graph_def = trt_converter.convert()
+  if output_saved_model_dir:
+    trt_converter.save(output_saved_model_dir)
+  return converted_graph_def
 
 
 def calib_graph_to_infer_graph(calibration_graph_def, is_dynamic_op=False):
diff --git a/tensorflow/python/compiler/tensorrt/trt_convert_test.py b/tensorflow/python/compiler/tensorrt/trt_convert_test.py
index c665f616fa..0dbc5c1970 100644
--- a/tensorflow/python/compiler/tensorrt/trt_convert_test.py
+++ b/tensorflow/python/compiler/tensorrt/trt_convert_test.py
@@ -52,9 +52,9 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
   _TRT_MAX_WORKSPACE_SIZE_BYTES = 2 << 20
 
   def testGetTensorrtRewriterConfig(self):
-    """Test case for trt_convert.get_tensorrt_rewriter_config()."""
-    rewriter_cfg = trt_convert.get_tensorrt_rewriter_config(
-        rewriter_config=None,
+    """Test case for TrtGraphConverter.get_tensorrt_rewriter_config()."""
+    rewriter_cfg = trt_convert.TrtGraphConverter.get_tensorrt_rewriter_config(
+        rewriter_config_template=None,
         max_batch_size=128,
         max_workspace_size_bytes=1234,
         precision_mode="INT8",
-- 
GitLab


From 36e8b05115bba67197fac32ad83c41e9ca2c96d1 Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Tue, 5 Feb 2019 11:04:08 -0800
Subject: [PATCH 1923/2345] [tf.data] Clean up and standardize benchmarks.

PiperOrigin-RevId: 232519626
---
 tensorflow/python/data/benchmarks/BUILD       |  35 ++--
 .../python/data/benchmarks/batch_benchmark.py |  59 ++-----
 .../python/data/benchmarks/benchmark_base.py  |  92 ++++++++++
 .../data/benchmarks/filter_benchmark.py       |  41 +----
 .../from_tensor_slices_benchmark.py           | 164 ++++--------------
 .../python/data/benchmarks/map_benchmark.py   | 131 ++++----------
 .../python/data/benchmarks/range_benchmark.py |  56 ++----
 7 files changed, 210 insertions(+), 368 deletions(-)
 create mode 100644 tensorflow/python/data/benchmarks/benchmark_base.py

diff --git a/tensorflow/python/data/benchmarks/BUILD b/tensorflow/python/data/benchmarks/BUILD
index 739e5bab19..031476100f 100644
--- a/tensorflow/python/data/benchmarks/BUILD
+++ b/tensorflow/python/data/benchmarks/BUILD
@@ -17,15 +17,23 @@ py_test(
     ],
 )
 
+py_library(
+    name = "benchmark_base",
+    srcs = ["benchmark_base.py"],
+    deps = [
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:session",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//third_party/py/numpy",
+    ],
+)
+
 py_test(
     name = "batch_benchmark",
     srcs = ["batch_benchmark.py"],
     srcs_version = "PY2AND3",
     deps = [
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:dtypes",
-        "//tensorflow/python:session",
+        ":benchmark_base",
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python/data/ops:dataset_ops",
         "//third_party/py/numpy",
@@ -37,12 +45,8 @@ py_test(
     srcs = ["filter_benchmark.py"],
     srcs_version = "PY2AND3",
     deps = [
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:session",
+        ":benchmark_base",
         "//tensorflow/python/data/ops:dataset_ops",
-        "//third_party/py/numpy",
     ],
 )
 
@@ -51,9 +55,7 @@ py_test(
     srcs = ["from_tensor_slices_benchmark.py"],
     srcs_version = "PY2AND3",
     deps = [
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:errors",
-        "//tensorflow/python:session",
+        ":benchmark_base",
         "//tensorflow/python/data/ops:dataset_ops",
         "//third_party/py/numpy",
     ],
@@ -64,6 +66,7 @@ py_test(
     srcs = ["list_files_benchmark.py"],
     srcs_version = "PY2AND3",
     deps = [
+        ":benchmark_base",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_ops",
@@ -78,11 +81,8 @@ py_test(
     srcs = ["map_benchmark.py"],
     srcs_version = "PY2AND3",
     deps = [
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:session",
+        ":benchmark_base",
         "//tensorflow/python/data/ops:dataset_ops",
-        "//third_party/py/numpy",
     ],
 )
 
@@ -91,8 +91,7 @@ py_test(
     srcs = ["range_benchmark.py"],
     srcs_version = "PY2AND3",
     deps = [
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:session",
+        ":benchmark_base",
         "//tensorflow/python/data/ops:dataset_ops",
     ],
 )
diff --git a/tensorflow/python/data/benchmarks/batch_benchmark.py b/tensorflow/python/data/benchmarks/batch_benchmark.py
index 0ccf5c57d1..8cad91212a 100644
--- a/tensorflow/python/data/benchmarks/batch_benchmark.py
+++ b/tensorflow/python/data/benchmarks/batch_benchmark.py
@@ -17,70 +17,37 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import time
-
 import numpy as np
 
-from tensorflow.python.client import session
+from tensorflow.python.data.benchmarks import benchmark_base
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import sparse_tensor
-from tensorflow.python.ops import array_ops
-from tensorflow.python.platform import test
 
 
-# TODO(b/119837791): Add eager benchmarks.
-class BatchBenchmark(test.Benchmark):
+class BatchBenchmark(benchmark_base.DatasetBenchmarkBase):
   """Benchmarks for `tf.data.Dataset.batch()`."""
 
-  def benchmarkBatchSparse(self):
+  def benchmark_batch_sparse(self):
     non_zeros_per_row_values = [0, 1, 5, 10, 100]
     batch_size_values = [1, 32, 64, 128, 1024]
 
-    sparse_placeholder = array_ops.sparse_placeholder(dtype=dtypes.int64)
-    batch_size_placeholder = array_ops.placeholder(dtype=dtypes.int64, shape=[])
-
-    dataset = dataset_ops.Dataset.from_tensors(sparse_placeholder).repeat(
-        ).batch(batch_size_placeholder)
-    options = dataset_ops.Options()
-    options.experimental_optimization.apply_default_optimizations = False
-    dataset = dataset.with_options(options)
-    iterator = dataset_ops.make_initializable_iterator(dataset)
-    next_element = iterator.get_next()
-
     for non_zeros_per_row in non_zeros_per_row_values:
 
-      sparse_value = sparse_tensor.SparseTensorValue(
+      tensor = sparse_tensor.SparseTensor(
           indices=np.arange(non_zeros_per_row, dtype=np.int64)[:, np.newaxis],
           values=np.arange(non_zeros_per_row, dtype=np.int64),
           dense_shape=[1000])
 
       for batch_size in batch_size_values:
-
-        with session.Session() as sess:
-          sess.run(iterator.initializer, feed_dict={
-              sparse_placeholder: sparse_value,
-              batch_size_placeholder: batch_size})
-          # Run five steps to warm up the session caches before taking the
-          # first measurement.
-          for _ in range(5):
-            sess.run(next_element.indices.op)
-          deltas = []
-          for _ in range(100):
-            start = time.time()
-            for _ in range(100):
-              sess.run(next_element.indices.op)
-            end = time.time()
-            deltas.append(end - start)
-
-        median_wall_time = np.median(deltas) / 100.0
-
-        self.report_benchmark(
-            iters=10000,
-            wall_time=median_wall_time,
-            name="sparse_num_elements_%d_batch_size_%d" %
-            (non_zeros_per_row, batch_size))
+        dataset = dataset_ops.Dataset.from_tensors(tensor).repeat().batch(
+            batch_size)
+        self.run_and_report_benchmark(
+            dataset,
+            num_elements=100000 // batch_size,
+            iters=1,
+            name="sparse_num_elements_%d_batch_size_%d" % (non_zeros_per_row,
+                                                           batch_size))
 
 
 if __name__ == "__main__":
-  test.main()
+  benchmark_base.test.main()
diff --git a/tensorflow/python/data/benchmarks/benchmark_base.py b/tensorflow/python/data/benchmarks/benchmark_base.py
new file mode 100644
index 0000000000..47f992d955
--- /dev/null
+++ b/tensorflow/python/data/benchmarks/benchmark_base.py
@@ -0,0 +1,92 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test utilities for tf.data benchmarking functionality."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import time
+import numpy as np
+
+from tensorflow.python.client import session
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.util import nest
+from tensorflow.python.platform import test
+
+
+# TODO(b/119837791): Add eager benchmarks.
+class DatasetBenchmarkBase(test.Benchmark):
+  """Base class for dataset benchmarks."""
+
+  def run_benchmark(self, dataset, num_elements, iters=1):
+    """Benchmarks the dataset.
+
+    Runs the dataset `iters` times. In each iteration, the benchmark measures
+    the time it takes to go through `num_elements` elements of the dataset.
+
+    Args:
+      dataset: Dataset to benchmark.
+      num_elements: Number of dataset elements to iterate through each benchmark
+        iteration.
+      iters: Number of times to repeat the timing.
+
+    Returns:
+      A float, representing the per-element wall time of the dataset in seconds.
+      This is the median time (with respect to `iters`) it takes for the dataset
+      to go through `num_elements` elements, divided by `num_elements.`
+    """
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
+    # NOTE: We use `dataset.skip()` to perform the iterations in C++, avoiding
+    # the overhead of multiple `session.run()` calls. Note that this relies on
+    # the underlying implementation of `skip`: if it is optimized in the future,
+    # we will have to change this code.
+    dataset = dataset.skip(num_elements - 1)
+    iterator = dataset_ops.make_initializable_iterator(dataset)
+    next_element = iterator.get_next()
+    next_element = nest.flatten(next_element)[0]
+
+    deltas = []
+    for _ in range(iters):
+      with session.Session() as sess:
+        # Run once to warm up the session caches.
+        sess.run(iterator.initializer)
+        sess.run(next_element)
+
+        sess.run(iterator.initializer)
+        start = time.time()
+        sess.run(next_element.op)
+        end = time.time()
+      deltas.append(end - start)
+    return np.median(deltas) / float(num_elements)
+
+  def run_and_report_benchmark(self,
+                               dataset,
+                               num_elements,
+                               name,
+                               iters=5,
+                               extras=None):
+    # Measure the per-element wall time.
+    wall_time = self.run_benchmark(dataset, num_elements, iters)
+
+    if extras is None:
+      extras = {}
+    extras["elements_per_second"] = 1 / wall_time
+    extras["num_elements"] = num_elements
+    # 'mode' represents the mechanism used for iterating over dataset elements.
+    name = "%s_mode_cpp" % name
+    self.report_benchmark(
+        wall_time=wall_time, iters=iters, name=name, extras=extras)
diff --git a/tensorflow/python/data/benchmarks/filter_benchmark.py b/tensorflow/python/data/benchmarks/filter_benchmark.py
index e0ecf19e11..eb47b4089c 100644
--- a/tensorflow/python/data/benchmarks/filter_benchmark.py
+++ b/tensorflow/python/data/benchmarks/filter_benchmark.py
@@ -17,51 +17,26 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import time
-
-import numpy as np
-
-from tensorflow.python.client import session
+from tensorflow.python.data.benchmarks import benchmark_base
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
-from tensorflow.python.platform import test
 
 
 # TODO(b/119837791): Add eager benchmarks.
-class FilterBenchmark(test.Benchmark):
+class FilterBenchmark(benchmark_base.DatasetBenchmarkBase):
   """Benchmarks for `tf.data.Dataset.filter()`."""
 
   def _benchmark(self, predicate, name):
-    with ops.Graph().as_default():
-      dataset = (
-          dataset_ops.Dataset.from_tensors(True).repeat(None).filter(predicate))
-      options = dataset_ops.Options()
-      options.experimental_optimization.apply_default_optimizations = False
-      dataset = dataset.with_options(options)
-      iterator = dataset_ops.make_one_shot_iterator(dataset)
-      next_element = iterator.get_next()
-
-      with session.Session() as sess:
-        for _ in range(5):
-          sess.run(next_element.op)
-        deltas = []
-        for _ in range(100):
-          start = time.time()
-          for _ in range(100):
-            sess.run(next_element.op)
-          end = time.time()
-          deltas.append(end - start)
-
-        median_wall_time = np.median(deltas) / 100
-        self.report_benchmark(iters=100, wall_time=median_wall_time, name=name)
+    dataset = (
+        dataset_ops.Dataset.from_tensors(True).repeat(None).filter(predicate))
+    self.run_and_report_benchmark(dataset, num_elements=100000, name=name)
 
-  def benchmarkSimpleFunction(self):
+  def benchmark_simple_function(self):
     self._benchmark(array_ops.identity, "simple_function")
 
-  def benchmarkReturnComponentOptimization(self):
+  def benchmark_return_component_optimization(self):
     self._benchmark(lambda x: x, "return_component")
 
 
 if __name__ == "__main__":
-  test.main()
+  benchmark_base.test.main()
diff --git a/tensorflow/python/data/benchmarks/from_tensor_slices_benchmark.py b/tensorflow/python/data/benchmarks/from_tensor_slices_benchmark.py
index 4e5559ddba..3af174acc3 100644
--- a/tensorflow/python/data/benchmarks/from_tensor_slices_benchmark.py
+++ b/tensorflow/python/data/benchmarks/from_tensor_slices_benchmark.py
@@ -17,170 +17,70 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import time
-
 import numpy as np
 
-from tensorflow.python.client import session
+from tensorflow.python.data.benchmarks import benchmark_base
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.framework import errors
-from tensorflow.python.platform import test
 
 
 # TODO(b/119837791): Add eager benchmarks.
-class FromTensorSlicesBenchmark(test.Benchmark):
+class FromTensorSlicesBenchmark(benchmark_base.DatasetBenchmarkBase):
   """Benchmarks for `tf.data.Dataset.from_tensor_slices()`."""
 
-  def benchmarkSliceRepeatBatch(self):
+  def benchmark_slice_repeat_batch(self):
     input_size = 10000
     batch_size = 100
     num_epochs = 100
+    num_elements = input_size * num_epochs // batch_size
 
     input_data = np.random.randn(input_size)
 
     dataset = (
-        dataset_ops.Dataset.from_tensor_slices(input_data)
-        .repeat(num_epochs + 1).batch(batch_size))
-    options = dataset_ops.Options()
-    options.experimental_optimization.apply_default_optimizations = False
-    dataset = dataset.with_options(options)
-    iterator = dataset_ops.make_initializable_iterator(dataset)
-    next_element = iterator.get_next()
-
-    with session.Session() as sess:
-      sess.run(iterator.initializer)
-      # Run one whole epoch to burn in the computation.
-      for _ in range(input_size // batch_size):
-        sess.run(next_element)
-      deltas = []
-      try:
-        while True:
-          start = time.time()
-          sess.run(next_element)
-          deltas.append(time.time() - start)
-      except errors.OutOfRangeError:
-        pass
-
-    median_wall_time = np.median(deltas)
-    self.report_benchmark(
-        iters=len(deltas),
-        wall_time=median_wall_time,
+        dataset_ops.Dataset.from_tensor_slices(input_data).repeat(
+            num_epochs).batch(batch_size))
+
+    self.run_and_report_benchmark(
+        dataset,
+        num_elements=num_elements,
         name="slice_repeat_batch_input_%d_batch_%d" % (input_size, batch_size))
 
-  def benchmarkSliceRepeatBatchCallable(self):
+  def benchmark_reshape_slice_repeat(self):
     input_size = 10000
-    batch_size = 100
+    reshape_dim = [100, 100]
     num_epochs = 100
 
+    num_elements = num_epochs * reshape_dim[0]
+
     input_data = np.random.randn(input_size)
 
     dataset = (
-        dataset_ops.Dataset.from_tensor_slices(input_data)
-        .repeat(num_epochs + 1).batch(batch_size))
-    options = dataset_ops.Options()
-    options.experimental_optimization.apply_default_optimizations = False
-    dataset = dataset.with_options(options)
-    iterator = dataset_ops.make_initializable_iterator(dataset)
-    next_element = iterator.get_next()
-
-    with session.Session() as sess:
-      sess.run(iterator.initializer)
-      get_next_element = sess.make_callable(next_element)
-      # Run one whole epoch to burn in the computation.
-      for _ in range(input_size // batch_size):
-        get_next_element()
-      deltas = []
-      try:
-        while True:
-          start = time.time()
-          get_next_element()
-          deltas.append(time.time() - start)
-      except errors.OutOfRangeError:
-        pass
-
-    median_wall_time = np.median(deltas)
-    self.report_benchmark(
-        iters=len(deltas),
-        wall_time=median_wall_time,
-        name="slice_repeat_batch_callable_input_%d_batch_%d" %
-        (input_size, batch_size))
-
-  def benchmarkReshapeSliceRepeatCallable(self):
-    input_size = 10000
-    batch_size = 100
-    num_epochs = 100
+        dataset_ops.Dataset.from_tensor_slices(
+            input_data.reshape(*reshape_dim)).repeat(num_epochs))
 
-    input_data = np.random.randn(input_size)
+    self.run_and_report_benchmark(
+        dataset,
+        num_elements=num_elements,
+        name="reshape_slice_repeat_input_%d" % input_size,
+    )
 
-    dataset = (
-        dataset_ops.Dataset.from_tensor_slices(input_data.reshape(100, 100))
-        .repeat(num_epochs + 1))
-    options = dataset_ops.Options()
-    options.experimental_optimization.apply_default_optimizations = False
-    dataset = dataset.with_options(options)
-    iterator = dataset_ops.make_initializable_iterator(dataset)
-    next_element = iterator.get_next()
-
-    with session.Session() as sess:
-      sess.run(iterator.initializer)
-      get_next_element = sess.make_callable(next_element)
-      # Run one whole epoch to burn in the computation.
-      for _ in range(input_size // batch_size):
-        get_next_element()
-      deltas = []
-      try:
-        while True:
-          start = time.time()
-          get_next_element()
-          deltas.append(time.time() - start)
-      except errors.OutOfRangeError:
-        pass
-
-    median_wall_time = np.median(deltas)
-    self.report_benchmark(
-        iters=len(deltas),
-        wall_time=median_wall_time,
-        name="reshape_slice_repeat_callable_input_%d_batch_%d" %
-        (input_size, batch_size))
-
-  def benchmarkSliceBatchCacheRepeatCallable(self):
+  def benchmark_slice_batch_cache_repeat(self):
     input_size = 10000
     batch_size = 100
     num_epochs = 100
+    num_elements = input_size * num_epochs // batch_size
 
     input_data = np.random.randn(input_size)
 
     dataset = (
-        dataset_ops.Dataset.from_tensor_slices(input_data).batch(batch_size)
-        .cache().repeat(num_epochs + 1))
-    options = dataset_ops.Options()
-    options.experimental_optimization.apply_default_optimizations = False
-    dataset = dataset.with_options(options)
-    iterator = dataset_ops.make_initializable_iterator(dataset)
-    next_element = iterator.get_next()
-
-    with session.Session() as sess:
-      sess.run(iterator.initializer)
-      get_next_element = sess.make_callable(next_element)
-      # Run one whole epoch to burn in the computation.
-      for _ in range(input_size // batch_size):
-        get_next_element()
-      deltas = []
-      try:
-        while True:
-          start = time.time()
-          get_next_element()
-          deltas.append(time.time() - start)
-      except errors.OutOfRangeError:
-        pass
-
-    median_wall_time = np.median(deltas)
-    self.report_benchmark(
-        iters=len(deltas),
-        wall_time=median_wall_time,
-        name="slice_batch_cache_repeat_callable_input_%d_batch_%d" %
-        (input_size, batch_size))
+        dataset_ops.Dataset.from_tensor_slices(input_data).batch(
+            batch_size).cache().repeat(num_epochs))
+
+    self.run_and_report_benchmark(
+        dataset,
+        num_elements=num_elements,
+        name="slice_batch_cache_repeat_input_%d_batch_%d" % (input_size,
+                                                             batch_size))
 
 
 if __name__ == "__main__":
-  test.main()
+  benchmark_base.test.main()
diff --git a/tensorflow/python/data/benchmarks/map_benchmark.py b/tensorflow/python/data/benchmarks/map_benchmark.py
index b620eaaed5..75b71fffac 100644
--- a/tensorflow/python/data/benchmarks/map_benchmark.py
+++ b/tensorflow/python/data/benchmarks/map_benchmark.py
@@ -17,114 +17,51 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import time
-
-import numpy as np
-
-from tensorflow.python.client import session
+from tensorflow.python.data.benchmarks import benchmark_base
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.framework import ops
-from tensorflow.python.platform import test
 
 
 # TODO(b/119837791): Add eager benchmarks.
-class MapBenchmark(test.Benchmark):
-  """Bechmarks for `tf.data.Dataset.map()`."""
-
-  def benchmarkChainOfMaps(self):
-    chain_lengths = [0, 1, 2, 5, 10, 20, 50]
-    for chain_length in chain_lengths:
-      for mode in ["general", "single-threaded", "short-circuit"]:
-        if mode == "general":
-          map_fn = lambda x: x + 1
-          use_inter_op_parallelism = True
-          benchmark_label = ""
-        if mode == "single-threaded":
-          map_fn = lambda x: x + 1
-          use_inter_op_parallelism = False
-          benchmark_label = "_single_threaded"
-        if mode == "short-circuit":
-          map_fn = lambda x: x
-          use_inter_op_parallelism = True  # should not have any significance
-          benchmark_label = "_short_circuit"
+class MapBenchmark(benchmark_base.DatasetBenchmarkBase):
+  """Benchmarks for `tf.data.Dataset.map()`."""
 
-        with ops.Graph().as_default():
-          dataset = dataset_ops.Dataset.from_tensors(0).repeat(None)
-          for _ in range(chain_length):
-            dataset = dataset_ops.MapDataset(
-                dataset,
-                map_fn,
-                use_inter_op_parallelism=use_inter_op_parallelism)
-          options = dataset_ops.Options()
-          options.experimental_optimization.apply_default_optimizations = False
-          dataset = dataset.with_options(options)
-          iterator = dataset_ops.make_one_shot_iterator(dataset)
-          next_element = iterator.get_next()
+  def benchmark_chain_of_maps(self):
 
-          with session.Session() as sess:
-            for _ in range(5):
-              sess.run(next_element.op)
-            deltas = []
-            for _ in range(100):
-              start = time.time()
-              for _ in range(100):
-                sess.run(next_element.op)
-              end = time.time()
-              deltas.append(end - start)
+    def benchmark_helper(chain_length, map_fn, use_inter_op_parallelism, label):
+      dataset = dataset_ops.Dataset.from_tensors(0).repeat(None)
+      for _ in range(chain_length):
+        dataset = dataset_ops.MapDataset(
+            dataset, map_fn, use_inter_op_parallelism=use_inter_op_parallelism)
+      self.run_and_report_benchmark(
+          dataset,
+          num_elements=10000,
+          name="chain_length_%d%s" % (chain_length, label))
 
-            median_wall_time = np.median(deltas) / 100
-            self.report_benchmark(
-                iters=1000,
-                wall_time=median_wall_time,
-                name="chain_length_%d%s" % (chain_length, benchmark_label))
+    chain_lengths = [0, 1, 2, 5, 10, 20, 50]
+    for chain_length in chain_lengths:
+      benchmark_helper(chain_length, lambda x: x + 1, True, "")
+      benchmark_helper(chain_length, lambda x: x + 1, False, "_single_threaded")
+      benchmark_helper(chain_length, lambda x: x, True, "_short_circuit")
 
-  def benchmarkMapFanOut(self):
+  def benchmark_map_fan_out(self):
     fan_outs = [1, 2, 5, 10, 20, 50, 100]
-    for fan_out in fan_outs:
-      for mode in ["general", "single-threaded", "short-circuit"]:
-        if mode == "general":
-          map_fn = lambda *xs: [x + 1 for x in xs]
-          use_inter_op_parallelism = True
-          benchmark_label = ""
-        if mode == "single-threaded":
-          map_fn = lambda *xs: [x + 1 for x in xs]
-          use_inter_op_parallelism = False
-          benchmark_label = "_single_threaded"
-        if mode == "short-circuit":
-          map_fn = lambda *xs: xs
-          use_inter_op_parallelism = True  # should not have any significance
-          benchmark_label = "_short_circuit"
 
-        with ops.Graph().as_default():
-          dataset = dataset_ops.Dataset.from_tensors(
-              tuple(0 for _ in range(fan_out))).repeat(None)
-          dataset = dataset_ops.MapDataset(
-              dataset,
-              map_fn,
-              use_inter_op_parallelism=use_inter_op_parallelism)
-          options = dataset_ops.Options()
-          options.experimental_optimization.apply_default_optimizations = False
-          dataset = dataset.with_options(options)
-          iterator = dataset_ops.make_one_shot_iterator(dataset)
-          next_element = iterator.get_next()
+    def benchmark_helper(fan_out, map_fn, use_inter_op_parallelism, label):
+      dataset = dataset_ops.Dataset.from_tensors(
+          tuple(0 for _ in range(fan_out))).repeat(None)
+      dataset = dataset_ops.MapDataset(
+          dataset, map_fn, use_inter_op_parallelism=use_inter_op_parallelism)
+      self.run_and_report_benchmark(
+          dataset,
+          num_elements=10000,
+          name="fan_out_%d%s" % (fan_out, label))
 
-          with session.Session() as sess:
-            for _ in range(5):
-              sess.run(next_element[0].op)
-            deltas = []
-            for _ in range(100):
-              start = time.time()
-              for _ in range(100):
-                sess.run(next_element[0].op)
-              end = time.time()
-              deltas.append(end - start)
-
-            median_wall_time = np.median(deltas) / 100
-            self.report_benchmark(
-                iters=1000,
-                wall_time=median_wall_time,
-                name="fan_out_%d%s" % (fan_out, benchmark_label))
+    for fan_out in fan_outs:
+      benchmark_helper(fan_out, lambda *xs: [x + 1 for x in xs], True, "")
+      benchmark_helper(fan_out, lambda *xs: [x + 1 for x in xs], False,
+                       "_single_threaded")
+      benchmark_helper(fan_out, lambda *xs: xs, True, "_short_circuit")
 
 
 if __name__ == "__main__":
-  test.main()
+  benchmark_base.test.main()
diff --git a/tensorflow/python/data/benchmarks/range_benchmark.py b/tensorflow/python/data/benchmarks/range_benchmark.py
index 375ff339a8..80569e4e31 100644
--- a/tensorflow/python/data/benchmarks/range_benchmark.py
+++ b/tensorflow/python/data/benchmarks/range_benchmark.py
@@ -17,54 +17,26 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import time
-
-from tensorflow.python.client import session
+from tensorflow.python.data.benchmarks import benchmark_base
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.platform import test
-
-_NUMPY_RANDOM_SEED = 42
 
 
-class RangeBenchmark(test.Benchmark):
+class RangeBenchmark(benchmark_base.DatasetBenchmarkBase):
   """Benchmarks for `tf.data.Dataset.range()`."""
 
-  def _benchmarkRangeHelper(self, modeling_enabled):
-    num_elements = 10000000 if modeling_enabled else 50000000
-
-    # Use `Dataset.skip()` and `Dataset.take()` to perform the iteration in
-    # C++, and focus on the minimal overheads (excluding Python invocation
-    # costs).
-    dataset = dataset_ops.Dataset.range(num_elements).skip(
-        num_elements - 1).take(1)
-    options = dataset_ops.Options()
-    options.experimental_autotune = modeling_enabled
-    options.experimental_optimization.apply_default_optimizations = False
-    dataset = dataset.with_options(options)
-    iterator = dataset_ops.make_initializable_iterator(dataset)
-    next_element = iterator.get_next()
-
-    with session.Session() as sess:
-      # Run once to warm up the session caches.
-      sess.run(iterator.initializer)
-      sess.run(next_element)
-
-      # Run once for timing.
-      sess.run(iterator.initializer)
-      start = time.time()
-      sess.run(next_element)
-      end = time.time()
-
-      time_per_element = (end - start) / num_elements
-      self.report_benchmark(
-          iters=num_elements,
-          wall_time=time_per_element,
-          name="modeling_%s" % ("on" if modeling_enabled else "off"))
-
-  def benchmarkRange(self):
+  def benchmark_range(self):
     for modeling_enabled in [False, True]:
-      self._benchmarkRangeHelper(modeling_enabled)
+      num_elements = 10000000 if modeling_enabled else 50000000
+      options = dataset_ops.Options()
+      options.experimental_autotune = modeling_enabled
+      dataset = dataset_ops.Dataset.range(num_elements)
+      dataset = dataset.with_options(options)
+
+      self.run_and_report_benchmark(
+          dataset,
+          num_elements=num_elements,
+          name="modeling_%s" % ("on" if modeling_enabled else "off"))
 
 
 if __name__ == "__main__":
-  test.main()
+  benchmark_base.test.main()
-- 
GitLab


From dfcd531df03144d4ca31000efa98e5136b47b15a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 5 Feb 2019 11:19:45 -0800
Subject: [PATCH 1924/2345] Use GRPC_ARG_MAX_RECONNECT_BACKOFF_MS instead of
 "grpc.testing.fixed_reconnect_backoff_ms" to restrict the backoff to 1s

PiperOrigin-RevId: 232522905
---
 tensorflow/core/debug/debug_io_utils.cc                 | 2 +-
 tensorflow/core/distributed_runtime/rpc/grpc_channel.cc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/debug/debug_io_utils.cc b/tensorflow/core/debug/debug_io_utils.cc
index 6994dec3b5..527c935380 100644
--- a/tensorflow/core/debug/debug_io_utils.cc
+++ b/tensorflow/core/debug/debug_io_utils.cc
@@ -730,7 +730,7 @@ Status DebugGrpcChannel::Connect(const int64 timeout_micros) {
   ::grpc::ChannelArguments args;
   args.SetInt(GRPC_ARG_MAX_MESSAGE_LENGTH, std::numeric_limits<int32>::max());
   // Avoid problems where default reconnect backoff is too long (e.g., 20 s).
-  args.SetInt("grpc.testing.fixed_reconnect_backoff_ms", 1000);
+  args.SetInt(GRPC_ARG_MAX_RECONNECT_BACKOFF_MS, 1000);
   channel_ = ::grpc::CreateCustomChannel(
       server_stream_addr_, ::grpc::InsecureChannelCredentials(), args);
   if (!channel_->WaitForConnected(
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_channel.cc b/tensorflow/core/distributed_runtime/rpc/grpc_channel.cc
index 781b7d65cd..64c221805b 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_channel.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_channel.cc
@@ -62,7 +62,7 @@ Status ValidateHostPortPair(const string& host_port) {
   args.SetInt(GRPC_ARG_MAX_MESSAGE_LENGTH, std::numeric_limits<int32>::max());
   // NOTE(mrry): Some versions of gRPC use a 20-second minimum backoff
   // on connection failure, which makes our tests time out.
-  args.SetInt("grpc.testing.fixed_reconnect_backoff_ms", 1000);
+  args.SetInt(GRPC_ARG_MAX_RECONNECT_BACKOFF_MS, 1000);
   if (rpc_options != nullptr) {
     if (rpc_options->compression_algorithm() == "deflate") {
       args.SetCompressionAlgorithm(GRPC_COMPRESS_DEFLATE);
-- 
GitLab


From 87826a32bb0167d7e7437d37f9c4fe94a295b08e Mon Sep 17 00:00:00 2001
From: David Majnemer <majnemer@google.com>
Date: Tue, 5 Feb 2019 11:21:38 -0800
Subject: [PATCH 1925/2345] [XLA] Make kSign return NaN for NaN inputs

Also correct tf.math.sign for NaN inputs, we returned +1.0 instead of 0.0.

PiperOrigin-RevId: 232523293
---
 tensorflow/compiler/tests/unary_ops_test.py   |  9 ++++++++
 .../compiler/tf2xla/kernels/unary_ops.cc      |  5 ++--
 .../compiler/xla/g3doc/operation_semantics.md |  2 +-
 .../xla/service/elemental_ir_emitter.cc       | 14 ++++++-----
 .../xla/service/hlo_evaluator_typed_visitor.h | 23 ++++++++++++++++---
 .../xla/service/llvm_ir/ir_builder_mixin.h    |  5 ++++
 .../compiler/xla/tests/unary_op_test.cc       |  4 +++-
 7 files changed, 49 insertions(+), 13 deletions(-)

diff --git a/tensorflow/compiler/tests/unary_ops_test.py b/tensorflow/compiler/tests/unary_ops_test.py
index 083e2e58ae..978ed667ee 100644
--- a/tensorflow/compiler/tests/unary_ops_test.py
+++ b/tensorflow/compiler/tests/unary_ops_test.py
@@ -391,6 +391,11 @@ class UnaryOpsTest(xla_test.XLATestCase):
           expected=np.array(
               [[-0.66666669, -0.5, 0, 0.5, 0.66666669]], dtype=dtype))
 
+      self._assertOpOutputMatchesExpected(
+          math_ops.sign,
+          np.array([[-2.0, -1.0, -0.0, +0.0, 1.0, 2.0]], dtype=dtype),
+          expected=np.array([[-1.0, -1.0, -0.0, +0.0, 1.0, 1.0]], dtype=dtype))
+
       self._assertOpOutputMatchesExpected(
           math_ops.is_finite,
           np.array(
@@ -743,6 +748,10 @@ class UnaryOpsTest(xla_test.XLATestCase):
           np.array(
               [[np.NINF, -2, -1, 0, 0.5, 1, 2, np.inf, np.nan]], dtype=dtype),
           expected=np.array([[0, 0, 0, 0, 0, 0, 0, 0, 1]], dtype=np.bool))
+      self._assertOpOutputMatchesExpected(
+          math_ops.sign,
+          np.array([[np.nan]], dtype=dtype),
+          expected=np.array([[0.0]], dtype=dtype))
 
   def testLogicalOps(self):
     self._assertOpOutputMatchesExpected(
diff --git a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc
index 4544e03491..62b5cd32da 100644
--- a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc
@@ -89,8 +89,9 @@ xla::XlaOp Sigmoid(xla::XlaOp x) {
 }
 XLAJIT_MAKE_UNARY(Sigmoid, Sigmoid(x));
 
-// Returns 0 if x is 0, -1 if x < 0 and 1 if x > 0.
-XLAJIT_MAKE_UNARY(Sign, xla::Sign(x));
+// Returns 0 if x is NaN, 0 if x is 0, -1 if x < 0 and 1 if x > 0.
+XLAJIT_MAKE_UNARY(Sign,
+                  xla::Select(xla::Ne(x, x), xla::ZerosLike(x), xla::Sign(x)));
 XLAJIT_MAKE_UNARY(Sinh, xla::Sinh(x));
 
 // softplus(x) = log(1 + exp(x))
diff --git a/tensorflow/compiler/xla/g3doc/operation_semantics.md b/tensorflow/compiler/xla/g3doc/operation_semantics.md
index 363fd17b69..db90d184b5 100644
--- a/tensorflow/compiler/xla/g3doc/operation_semantics.md
+++ b/tensorflow/compiler/xla/g3doc/operation_semantics.md
@@ -1186,7 +1186,7 @@ if and only if the corresponding input element is finite.
 
 <b>`Sign(operand)`</b> Element-wise sign operation `x -> sgn(x)` where
 
-$$\text{sgn}(x) = \begin{cases} -1 & x < 0\\ 0 & x = 0\\ 1 & x > 0 \end{cases}$$
+$$\text{sgn}(x) = \begin{cases} -1 & x < 0\\ -0 & x = -0\\ NaN & x = NaN\\ +0 & x = +0\\ 1 & x > 0 \end{cases}$$
 
 using the comparison operator of the element type of `operand`.
 
diff --git a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
index ccdc12a30a..fef84ac80a 100644
--- a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
@@ -440,14 +440,16 @@ StatusOr<llvm::Value*> ElementalIrEmitter::EmitFloatUnaryOp(
                                           {operand_value},
                                           {operand_value->getType()}, b_);
     case HloOpcode::kSign: {
-      // TODO(b/32151903): Ensure consistent sign behavior for -0.0.
       auto type = operand_value->getType();
       auto zero = llvm::ConstantFP::get(type, 0.0);
-      auto oeq = FCmpOEQ(operand_value, zero);
-      auto olt = FCmpOLT(operand_value, zero);
-      return Select(oeq, zero,
-                    Select(olt, llvm::ConstantFP::get(type, -1.0),
-                           llvm::ConstantFP::get(type, 1.0)));
+      auto ne0_i1 = FCmpONE(operand_value, zero);
+      auto ne0_float = UIToFP(ne0_i1, type);
+      llvm::Value* result = llvm_ir::EmitCallToIntrinsic(
+          llvm::Intrinsic::copysign, {ne0_float, operand_value},
+          {operand_value->getType()}, b_);
+      auto is_nan = FCmpUNO(operand_value, operand_value);
+      result = Select(is_nan, operand_value, result);
+      return result;
     }
     case HloOpcode::kIsFinite: {
       // abs(x) o!= inf, this works because the comparison returns false if
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
index 742a389ed0..652042e85f 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
@@ -462,9 +462,9 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
     return HandleNegate<ReturnT>(negate);
   }
 
-  template <
-      typename NativeT,
-      typename std::enable_if<!is_complex_t<NativeT>::value>::type* = nullptr>
+  template <typename NativeT,
+            typename std::enable_if<std::is_integral<NativeT>::value>::type* =
+                nullptr>
   Status HandleSign(HloInstruction* sign) {
     TF_ASSIGN_OR_RETURN(parent_->evaluated_[sign],
                         ElementWiseUnaryOp(sign, [](ElementwiseT elem_operand) {
@@ -474,6 +474,23 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
     return Status::OK();
   }
 
+  template <typename NativeT,
+            typename std::enable_if<
+                std::is_same<NativeT, bfloat16>::value ||
+                std::is_same<NativeT, Eigen::half>::value ||
+                std::is_floating_point<NativeT>::value>::type* = nullptr>
+  Status HandleSign(HloInstruction* sign) {
+    TF_ASSIGN_OR_RETURN(parent_->evaluated_[sign],
+                        ElementWiseUnaryOp(sign, [](ElementwiseT elem_operand) {
+                          return std::isnan(elem_operand)
+                                     ? elem_operand
+                                     : std::copysign(
+                                           elem_operand != ElementwiseT(0),
+                                           elem_operand);
+                        }));
+    return Status::OK();
+  }
+
   template <
       typename NativeT,
       typename std::enable_if<is_complex_t<NativeT>::value>::type* = nullptr>
diff --git a/tensorflow/compiler/xla/service/llvm_ir/ir_builder_mixin.h b/tensorflow/compiler/xla/service/llvm_ir/ir_builder_mixin.h
index cf5083e8c1..02c719502e 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/ir_builder_mixin.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/ir_builder_mixin.h
@@ -269,6 +269,11 @@ class IrBuilderMixin {
     return mixin_builder()->CreateFCmpUNE(std::forward<Args>(args)...);
   }
 
+  template <class... Args>
+  llvm::Value* FCmpUNO(Args&&... args) {
+    return mixin_builder()->CreateFCmpUNO(std::forward<Args>(args)...);
+  }
+
   template <class... Args>
   llvm::Value* FDiv(Args&&... args) {
     return mixin_builder()->CreateFDiv(std::forward<Args>(args)...);
diff --git a/tensorflow/compiler/xla/tests/unary_op_test.cc b/tensorflow/compiler/xla/tests/unary_op_test.cc
index 4fbd7f2fb1..c51f30f3b5 100644
--- a/tensorflow/compiler/xla/tests/unary_op_test.cc
+++ b/tensorflow/compiler/xla/tests/unary_op_test.cc
@@ -64,7 +64,9 @@ class UnaryOpTest : public ClientLibraryTestBase {
         &builder, {-2, 25, 0, static_cast<T>(-0.0), -123, inf<T>(), -inf<T>()});
     Sign(arg);
 
-    ComputeAndCompareR1<T>(&builder, {-1, 1, 0, 0, -1, 1, -1}, {});
+    ComputeAndCompareR1<T>(
+        &builder,
+        {-1, 1, static_cast<T>(+0.0), static_cast<T>(-0.0), -1, 1, -1}, {});
   }
 
   template <typename T>
-- 
GitLab


From 56f29b4deb1d5fca7a10e1af7251fe7361d206fd Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Tue, 5 Feb 2019 11:24:47 -0800
Subject: [PATCH 1926/2345] Remove owned_* methods.

Users who want this functionality can do so by subclassing tf.Module and
implemeting the methods themselves using `_flatten`. For example:

```
class MyModule(tf.Module):
  @property
  def owned_variables(self):
    is_variable = lambda v: isinstance(v, tf.Variable)
    return tuple(self._flatten(recursive=False, predicate=is_variable))
```

PiperOrigin-RevId: 232523894
---
 tensorflow/python/module/module.py            | 50 -------------------
 tensorflow/python/module/module_test.py       | 25 ----------
 .../v1/tensorflow.experimental.-module.pbtxt  | 12 -----
 .../v2/tensorflow.experimental.-module.pbtxt  | 12 -----
 4 files changed, 99 deletions(-)

diff --git a/tensorflow/python/module/module.py b/tensorflow/python/module/module.py
index 67f19a8eec..6bd275538c 100644
--- a/tensorflow/python/module/module.py
+++ b/tensorflow/python/module/module.py
@@ -211,20 +211,6 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
     """
     return tuple(self._flatten(predicate=_IS_VARIABLE))
 
-  @property
-  def owned_variables(self):
-    """Sequence of variables that are attributes of the current module.
-
-    See `variables` for a property which returns all variables from the current
-    module and all it's submodules recursively.
-
-    Returns:
-      A sequence of variables which are attributes of the current module. Will
-      yield variables inside nested structures (lists etc) but not in other
-      modules.
-    """
-    return tuple(self._flatten(recursive=False, predicate=_IS_VARIABLE))
-
   @property
   def trainable_variables(self):
     """Sequence of variables owned by this module and it's submodules.
@@ -239,42 +225,6 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
     """
     return tuple(self._flatten(predicate=_IS_TRAINABLE_VARIABLE))
 
-  @property
-  def owned_trainable_variables(self):
-    """Sequence of variables that are attributes of the current module.
-
-    See `variables` for a property which returns all variables from the current
-    module and all it's submodules recursively.
-
-    Returns:
-      A sequence of variables which are attributes of the current module. Will
-      yield variables inside nested structures (lists etc) but not in other
-      modules.
-    """
-    return tuple(
-        self._flatten(recursive=False, predicate=_IS_TRAINABLE_VARIABLE))
-
-  @property
-  def owned_submodules(self):
-    """Sequence of immediate child modules.
-
-    Child modules are modules which are found as properties of the current
-    module.
-
-    >>> a = tf.experimental.Module()
-    >>> b = tf.experimental.Module()
-    >>> c = tf.experimental.Module()
-    >>> a.b = b
-    >>> b.c = c
-    >>> assert list(a.owned_submodules) == [b]
-    >>> assert list(b.owned_submodules) == [c]
-    >>> assert list(c.owned_submodules) == []
-
-    Returns:
-      A sequence of all child modules.
-    """
-    return tuple(self._flatten(recursive=False, predicate=_IS_MODULE))
-
   @property
   def submodules(self):
     """Sequence of all sub-modules.
diff --git a/tensorflow/python/module/module_test.py b/tensorflow/python/module/module_test.py
index a487660435..da9bcf7268 100644
--- a/tensorflow/python/module/module_test.py
+++ b/tensorflow/python/module/module_test.py
@@ -151,12 +151,6 @@ class VariableTrackingTest(test.TestCase):
     self.assertEqual(m.child.variables, (m.child.w, m.child.child.w))
     self.assertEqual(m.child.child.variables, (m.child.child.w,))
 
-  def test_owned_variables(self):
-    m = RecursiveModule(3)
-    self.assertEqual(m.owned_variables, (m.w,))
-    self.assertEqual(m.child.owned_variables, (m.child.w,))
-    self.assertEqual(m.child.child.owned_variables, (m.child.child.w,))
-
   def test_trainable_variables(self):
     m = RecursiveModule(3)
     self.assertEqual(m.trainable_variables,
@@ -171,28 +165,9 @@ class VariableTrackingTest(test.TestCase):
     self.assertEqual(len(m.child.trainable_variables), 0)
     self.assertEqual(len(m.child.child.trainable_variables), 0)
 
-  def test_owned_trainable_variables(self):
-    m = RecursiveModule(3)
-    self.assertEqual(m.owned_trainable_variables, (m.w,))
-    self.assertEqual(m.child.owned_trainable_variables, (m.child.w,))
-    self.assertEqual(m.child.child.owned_trainable_variables,
-                     (m.child.child.w,))
-
-  def test_owned_trainable_variables_ignores_non_trainable(self):
-    m = RecursiveModule(3, trainable=False)
-    self.assertEqual(len(m.owned_trainable_variables), 0)
-    self.assertEqual(len(m.child.owned_trainable_variables), 0)
-    self.assertEqual(len(m.child.child.owned_trainable_variables), 0)
-
 
 class ModuleTrackingTest(test.TestCase):
 
-  def test_owned_submodules(self):
-    m = RecursiveModule(3)
-    self.assertEqual(list(m.owned_submodules), [m.child])
-    self.assertEqual(list(m.child.owned_submodules), [m.child.child])
-    self.assertEqual(list(m.child.child.owned_submodules), [])
-
   def test_submodules(self):
     m = RecursiveModule(3)
     self.assertEqual(list(m.submodules), [m.child, m.child.child])
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.experimental.-module.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.experimental.-module.pbtxt
index c364b0217a..3c5add1b72 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.experimental.-module.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.experimental.-module.pbtxt
@@ -12,18 +12,6 @@ tf_class {
     name: "name_scope"
     mtype: "<type \'property\'>"
   }
-  member {
-    name: "owned_submodules"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "owned_trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "owned_variables"
-    mtype: "<type \'property\'>"
-  }
   member {
     name: "submodules"
     mtype: "<type \'property\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.experimental.-module.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.experimental.-module.pbtxt
index c364b0217a..3c5add1b72 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.experimental.-module.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.experimental.-module.pbtxt
@@ -12,18 +12,6 @@ tf_class {
     name: "name_scope"
     mtype: "<type \'property\'>"
   }
-  member {
-    name: "owned_submodules"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "owned_trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "owned_variables"
-    mtype: "<type \'property\'>"
-  }
   member {
     name: "submodules"
     mtype: "<type \'property\'>"
-- 
GitLab


From a751c3d1f52ad161ffaa5fd0ef9a3cdbf46bbb8a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 5 Feb 2019 11:32:22 -0800
Subject: [PATCH 1927/2345] Store var_scope.name instead of var_scope in
 embedding variable mapping in brn_tpu_estimator.py.

PiperOrigin-RevId: 232525479
---
 tensorflow/contrib/tpu/python/tpu/feature_column.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/feature_column.py b/tensorflow/contrib/tpu/python/tpu/feature_column.py
index 8edf131bc2..68bcdb5026 100644
--- a/tensorflow/contrib/tpu/python/tpu/feature_column.py
+++ b/tensorflow/contrib/tpu/python/tpu/feature_column.py
@@ -420,7 +420,7 @@ def _record_variable_scope_and_name(embedding_var_name,
   else:
     # scope contains var_scope_name.
     captured_scope = variable_scope.get_variable_scope()
-    var_def_dict[embedding_var_name] = (captured_scope,
+    var_def_dict[embedding_var_name] = (captured_scope.name,
                                         embedding_var_name_in_fc)
 
 
-- 
GitLab


From c5d551751362625690651408df93d907c4b14224 Mon Sep 17 00:00:00 2001
From: Yifei Feng <yifeif@google.com>
Date: Tue, 5 Feb 2019 11:35:56 -0800
Subject: [PATCH 1928/2345] Automated rollback of commit
 90f1af3e4487f5526ab9974ee07c63c125916499

PiperOrigin-RevId: 232526208
---
 .../core/platform/default/build_config/BUILD      | 15 +++++++++++++++
 tensorflow/stream_executor/cuda/BUILD             |  8 ++++++--
 tensorflow/stream_executor/cuda/cudart_stub.cc    |  7 +++++++
 3 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/platform/default/build_config/BUILD b/tensorflow/core/platform/default/build_config/BUILD
index 6faf5c512f..845fe0ec04 100644
--- a/tensorflow/core/platform/default/build_config/BUILD
+++ b/tensorflow/core/platform/default/build_config/BUILD
@@ -7,6 +7,7 @@ licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
+load("//tensorflow:tensorflow.bzl", "check_deps")
 load("//tensorflow:tensorflow.bzl", "if_cuda")
 load("//tensorflow:tensorflow.bzl", "if_rocm")
 load("//tensorflow:tensorflow.bzl", "tf_copts")
@@ -283,6 +284,20 @@ cc_library(
     ],
 )
 
+# Check that libtensorflow_framework.so does not depend on cuda shared libraries.
+check_deps(
+    name = "libtensorflow_cuda_check_deps",
+    disallowed_deps = [
+        ":cuda",
+        "@local_config_cuda//cuda:cublas",
+        "@local_config_cuda//cuda:cuda_driver",
+        "@local_config_cuda//cuda:cudnn",
+        "@local_config_cuda//cuda:curand",
+        "@local_config_cuda//cuda:cusolver",
+    ],
+    deps = ["//tensorflow:libtensorflow_framework.so"],
+)
+
 cc_library(
     name = "rocm",
     data = [],
diff --git a/tensorflow/stream_executor/cuda/BUILD b/tensorflow/stream_executor/cuda/BUILD
index 0e23cc3eb4..03dd92e19c 100644
--- a/tensorflow/stream_executor/cuda/BUILD
+++ b/tensorflow/stream_executor/cuda/BUILD
@@ -87,13 +87,17 @@ cc_library(
         "@com_google_absl//absl/container:inlined_vector",
         "@com_google_absl//absl/strings",
         "@local_config_cuda//cuda:cuda_headers",
-        "//tensorflow/core:cuda",
         "//tensorflow/stream_executor:device_options",
         "//tensorflow/stream_executor/gpu:gpu_driver_header",
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
         "//tensorflow/stream_executor/platform:dso_loader",
-    ] + tf_additional_cuda_driver_deps()),
+    ] + tf_additional_cuda_driver_deps()) + select({
+        # include dynamic loading implementation only when if_cuda_is_configured and build dynamically
+        "//tensorflow:using_cuda_nvcc_with_dynamic_build": ["cudart_stub"],
+        "//tensorflow:using_cuda_clang_with_dynamic_build": ["cudart_stub"],
+        "//conditions:default": ["//tensorflow/core:cuda"],
+    }),
 )
 
 cc_library(
diff --git a/tensorflow/stream_executor/cuda/cudart_stub.cc b/tensorflow/stream_executor/cuda/cudart_stub.cc
index cb35719183..c5fc43d56c 100644
--- a/tensorflow/stream_executor/cuda/cudart_stub.cc
+++ b/tensorflow/stream_executor/cuda/cudart_stub.cc
@@ -111,4 +111,11 @@ cudaError_t CUDART_CB cudaPointerGetAttributes(
   if (!func_ptr) return GetSymbolNotFoundError();
   return func_ptr(attributes, ptr);
 }
+
+cudaError_t CUDART_CB cudaGetLastError() {
+  using FuncPtr = cudaError_t (*)();
+  static auto func_ptr = LoadSymbol<FuncPtr>("cudaGetLastError");
+  if (!func_ptr) return GetSymbolNotFoundError();
+  return func_ptr();
+}
 }  // extern "C"
-- 
GitLab


From d7e835f9d9738723c06a8b97c553f922e0cee191 Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Tue, 5 Feb 2019 11:51:37 -0800
Subject: [PATCH 1929/2345] Allow CriticalSection codes to run on GPU by
 keeping the mutex lock in host memory.

PiperOrigin-RevId: 232529207
---
 tensorflow/contrib/framework/BUILD               |  1 +
 .../python/ops/critical_section_test.py          | 14 ++++++++++++--
 tensorflow/core/kernels/mutex_ops.cc             | 16 +++++++++++++++-
 3 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/framework/BUILD b/tensorflow/contrib/framework/BUILD
index 3f6dbe0cbd..c99f84789e 100644
--- a/tensorflow/contrib/framework/BUILD
+++ b/tensorflow/contrib/framework/BUILD
@@ -179,6 +179,7 @@ cuda_py_test(
     additional_deps = [
         "//tensorflow/python:client_testlib",
         ":framework_py",
+        "//tensorflow/python/data/experimental/ops:prefetching_ops",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:framework_for_generated_wrappers",
diff --git a/tensorflow/contrib/framework/python/ops/critical_section_test.py b/tensorflow/contrib/framework/python/ops/critical_section_test.py
index 34fd5018af..d2bb4f476f 100644
--- a/tensorflow/contrib/framework/python/ops/critical_section_test.py
+++ b/tensorflow/contrib/framework/python/ops/critical_section_test.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.contrib.framework.python.ops import critical_section_ops
+from tensorflow.python.data.experimental.ops import prefetching_ops
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
@@ -334,13 +335,22 @@ class CriticalSectionTest(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
   def testInsideFunction(self):
+    if test_util.is_gpu_available():
+      self.skipTest(
+          "b/123899495: Colocation errors for critical sections in map on GPU")
     cs = critical_section_ops.CriticalSection()
-    v = resource_variable_ops.ResourceVariable(1)
+    with ops.device("/gpu:0" if test_util.is_gpu_available() else "/cpu:0"):
+      v = resource_variable_ops.ResourceVariable(1)
     def fn():
       return v.read_value()
 
     # map() creates a TensorFlow function.
-    ds = dataset_ops.Dataset.range(1).map(lambda _: cs.execute(fn))
+    ds = dataset_ops.Dataset.range(1)
+    if test_util.is_gpu_available():
+      ds = (ds.apply(prefetching_ops.copy_to_device("/gpu:0"))
+            .apply(prefetching_ops.map_on_gpu(lambda _: cs.execute(fn))))
+    else:
+      ds = ds.map(lambda _: cs.execute(fn))
 
     def get_first():
       if context.executing_eagerly():
diff --git a/tensorflow/core/kernels/mutex_ops.cc b/tensorflow/core/kernels/mutex_ops.cc
index 1603a2aa86..b06845f13a 100644
--- a/tensorflow/core/kernels/mutex_ops.cc
+++ b/tensorflow/core/kernels/mutex_ops.cc
@@ -242,10 +242,24 @@ class ConsumeMutexLockOp : public OpKernel {
 
 REGISTER_KERNEL_BUILDER(Name("MutexLock").Device(DEVICE_CPU), MutexLockOp);
 
-REGISTER_KERNEL_BUILDER(Name("MutexV2").Device(DEVICE_CPU),
+REGISTER_KERNEL_BUILDER(Name("MutexLock")
+                            .Device(DEVICE_GPU)
+                            .HostMemory("mutex_lock")
+                            .HostMemory("mutex"),
+                        MutexLockOp);
+
+REGISTER_KERNEL_BUILDER(
+    Name("MutexV2").Device(DEVICE_CPU).HostMemory("resource"),
+    ResourceHandleOp<Mutex>);
+
+REGISTER_KERNEL_BUILDER(Name("MutexV2").Device(DEVICE_GPU),
                         ResourceHandleOp<Mutex>);
 
 REGISTER_KERNEL_BUILDER(Name("ConsumeMutexLock").Device(DEVICE_CPU),
                         ConsumeMutexLockOp);
 
+REGISTER_KERNEL_BUILDER(
+    Name("ConsumeMutexLock").Device(DEVICE_GPU).HostMemory("mutex_lock"),
+    ConsumeMutexLockOp);
+
 }  // namespace tensorflow
-- 
GitLab


From 1b24197e20dbb799a7e624cdf89b5d86e0e5640e Mon Sep 17 00:00:00 2001
From: Shivani Agrawal <shivaniagrawal@google.com>
Date: Tue, 5 Feb 2019 11:57:54 -0800
Subject: [PATCH 1930/2345] [tf.data] remove device placement logging from map
 test.

PiperOrigin-RevId: 232530498
---
 tensorflow/python/data/kernel_tests/map_test.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/data/kernel_tests/map_test.py b/tensorflow/python/data/kernel_tests/map_test.py
index c419c632ff..a56a3afad6 100644
--- a/tensorflow/python/data/kernel_tests/map_test.py
+++ b/tensorflow/python/data/kernel_tests/map_test.py
@@ -395,8 +395,7 @@ class MapTest(test_base.DatasetTestBase, parameterized.TestCase):
   # TODO(b/121264236): add eager mode coverage when we have mutli-device setup.
   @test_util.run_v1_only("b/121264236")
   def testSkipEagerCaptureConstantsWithConflictingDevices(self):
-    config = config_pb2.ConfigProto(
-        device_count={"CPU": 3}, log_device_placement=True)
+    config = config_pb2.ConfigProto(device_count={"CPU": 3})
     with self.cached_session(config=config):
       with ops.device("/device:CPU:0"):
         a = constant_op.constant(3.0)
@@ -414,8 +413,7 @@ class MapTest(test_base.DatasetTestBase, parameterized.TestCase):
   @test_util.run_v1_only(
       "defun will convert RefVariables to ResourceVariables.")
   def testSkipEagerRefVariablesWithConflictingDevices(self):
-    config = config_pb2.ConfigProto(
-        device_count={"CPU": 3}, log_device_placement=True)
+    config = config_pb2.ConfigProto(device_count={"CPU": 3})
     with self.cached_session(config=config):
 
       def func(_):
@@ -436,8 +434,7 @@ class MapTest(test_base.DatasetTestBase, parameterized.TestCase):
   # TODO(b/121264236): add eager mode coverage when we have mutli-device setup.
   @test_util.run_v1_only("b/121264236")
   def testSkipEagerResourceVariablesWithConflictingDevices(self):
-    config = config_pb2.ConfigProto(
-        device_count={"CPU": 3}, log_device_placement=True)
+    config = config_pb2.ConfigProto(device_count={"CPU": 3})
     with self.cached_session(config=config):
 
       def func(_):
-- 
GitLab


From aee80c4b9b1ecd4a2f3f53561dbb774d71848fdd Mon Sep 17 00:00:00 2001
From: Sergio Guadarrama <sguada@google.com>
Date: Tue, 5 Feb 2019 12:02:31 -0800
Subject: [PATCH 1931/2345] Make sure that variables.trainable and considered
 trainable.

When creating models under tf.function the trainable variables don't always end up in tf.trainable_variables() collection.

PiperOrigin-RevId: 232531430
---
 tensorflow/contrib/training/python/training/training.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/training/python/training/training.py b/tensorflow/contrib/training/python/training/training.py
index 093765dc20..4ceb6e9350 100644
--- a/tensorflow/contrib/training/python/training/training.py
+++ b/tensorflow/contrib/training/python/training/training.py
@@ -432,7 +432,7 @@ def create_train_op(total_loss,
   else:
     # Make sure that variables_to_train are in tf.trainable_variables()
     for v in variables_to_train:
-      assert v in tf_variables.trainable_variables()
+      assert v.trainable or v in tf_variables.trainable_variables()
 
   assert variables_to_train
 
-- 
GitLab


From 05777ae33f3f93feae11650c424e5f6d2baa6bd7 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Tue, 5 Feb 2019 12:07:43 -0800
Subject: [PATCH 1932/2345] Fix issue for stateful LSTM/GRU in eager mode for
 2.0.

PiperOrigin-RevId: 232532542
---
 tensorflow/python/keras/layers/recurrent.py   | 38 +++++++++----------
 .../python/keras/layers/unified_gru_test.py   | 23 +++++++++++
 .../python/keras/layers/unified_lstm_test.py  | 23 +++++++++++
 3 files changed, 65 insertions(+), 19 deletions(-)

diff --git a/tensorflow/python/keras/layers/recurrent.py b/tensorflow/python/keras/layers/recurrent.py
index 9404543ed9..79c6197211 100644
--- a/tensorflow/python/keras/layers/recurrent.py
+++ b/tensorflow/python/keras/layers/recurrent.py
@@ -2152,16 +2152,11 @@ class UnifiedGRU(GRU):
             count=3)
 
       inputs *= self._dropout_mask[0]
-    experimental_api_name = 'gru_' + str(uuid.uuid4())
-    defun_standard_gru = _generate_defun_backend(
-        experimental_api_name, _CPU_DEVICE_NAME, standard_gru)
-    defun_cudnn_gru = _generate_defun_backend(
-        experimental_api_name, _GPU_DEVICE_NAME, cudnn_gru)
     if ops.executing_eagerly_outside_functions():
       # Under eager context, the device placement is already known. Prefer the
       # GPU implementation when GPU is available.
       if context.num_gpus() > 0:
-        last_output, outputs, new_h, runtime = defun_cudnn_gru(
+        last_output, outputs, new_h, runtime = cudnn_gru(
             inputs=inputs,
             init_h=initial_state[0],
             kernel=self.cell.kernel,
@@ -2169,7 +2164,7 @@ class UnifiedGRU(GRU):
             bias=self.cell.bias,
             time_major=self.time_major)
       else:
-        last_output, outputs, new_h, runtime = defun_standard_gru(
+        last_output, outputs, new_h, runtime = standard_gru(
             inputs=inputs,
             init_h=initial_state[0],
             kernel=self.cell.kernel,
@@ -2179,6 +2174,11 @@ class UnifiedGRU(GRU):
             recurrent_activation=self.recurrent_activation,
             time_major=self.time_major)
     else:
+      experimental_api_name = 'gru_' + str(uuid.uuid4())
+      defun_standard_gru = _generate_defun_backend(
+          experimental_api_name, _CPU_DEVICE_NAME, standard_gru)
+      defun_cudnn_gru = _generate_defun_backend(
+          experimental_api_name, _GPU_DEVICE_NAME, cudnn_gru)
       # Call the normal GRU impl and register the CuDNN impl function. The
       # grappler will kick in during session execution to optimize the graph.
       last_output, outputs, new_h, runtime = defun_standard_gru(
@@ -3112,29 +3112,29 @@ class UnifiedLSTM(LSTM):
 
         inputs *= self._dropout_mask[0]
 
-      # Each time a defun function is called, we will give a unique identifiable
-      # API name, so that the grappler won't get confused when it sees multiple
-      # LSTM layer added into same graph, and it will be able to pair up the
-      # different implementations across them.
-      experimental_api_name = 'lstm_' + str(uuid.uuid4())
-      defun_standard_lstm = _generate_defun_backend(
-          experimental_api_name, _CPU_DEVICE_NAME, standard_lstm)
-      defun_cudnn_lstm = _generate_defun_backend(
-          experimental_api_name, _GPU_DEVICE_NAME, cudnn_lstm)
-
       if ops.executing_eagerly_outside_functions():
         # Under eager context, the device placement is already known. Prefer the
         # GPU implementation here.
         if context.num_gpus() > 0:
-          last_output, outputs, new_h, new_c, runtime = defun_cudnn_lstm(
+          last_output, outputs, new_h, new_c, runtime = cudnn_lstm(
               inputs, initial_state[0], initial_state[1], self.cell.kernel,
               self.cell.recurrent_kernel, self.cell.bias, self.time_major)
         else:
-          last_output, outputs, new_h, new_c, runtime = defun_standard_lstm(
+          last_output, outputs, new_h, new_c, runtime = standard_lstm(
               inputs, initial_state[0], initial_state[1], self.cell.kernel,
               self.cell.recurrent_kernel, self.cell.bias, self.activation,
               self.recurrent_activation, self.time_major)
       else:
+        # Each time a `tf.function` is called, we will give it a unique
+        # identifiable API name, so that Grappler won't get confused when it
+        # sees multiple LSTM layers added into same graph, and it will be able
+        # to pair up the different implementations across them.
+        experimental_api_name = 'lstm_' + str(uuid.uuid4())
+        defun_standard_lstm = _generate_defun_backend(
+            experimental_api_name, _CPU_DEVICE_NAME, standard_lstm)
+        defun_cudnn_lstm = _generate_defun_backend(
+            experimental_api_name, _GPU_DEVICE_NAME, cudnn_lstm)
+
         # Call the normal LSTM impl and register the CuDNN impl function. The
         # grappler will kick in during session execution to optimize the graph.
         last_output, outputs, new_h, new_c, runtime = defun_standard_lstm(
diff --git a/tensorflow/python/keras/layers/unified_gru_test.py b/tensorflow/python/keras/layers/unified_gru_test.py
index 8259643d7c..b25007e52c 100644
--- a/tensorflow/python/keras/layers/unified_gru_test.py
+++ b/tensorflow/python/keras/layers/unified_gru_test.py
@@ -464,6 +464,29 @@ class UnifiedGRUTest(keras_parameterized.TestCase):
 
     np.testing.assert_allclose(out7, out6, atol=1e-5)
 
+  def test_stateful_GRU_training(self):
+    # See b/123587692 for more context.
+    vocab_size = 20
+    embedding_dim = 10
+    batch_size = 8
+    timestep = 12
+    units = 5
+    x = np.random.randint(0, vocab_size, size=(batch_size, timestep))
+    y = np.random.randint(0, vocab_size, size=(batch_size, timestep))
+
+    model = keras.Sequential([
+        keras.layers.Embedding(vocab_size, embedding_dim,
+                               batch_input_shape=[batch_size, timestep]),
+        keras.layers.UnifiedGRU(units,
+                                return_sequences=True,
+                                stateful=True),
+        keras.layers.Dense(vocab_size)
+    ])
+    model.compile(optimizer='adam',
+                  loss='sparse_categorical_crossentropy',
+                  run_eagerly=testing_utils.should_run_eagerly())
+    model.fit(x, y, epochs=1, shuffle=False)
+
 
 class GRULayerGradientTapeTest(test.TestCase):
 
diff --git a/tensorflow/python/keras/layers/unified_lstm_test.py b/tensorflow/python/keras/layers/unified_lstm_test.py
index 375894b166..08153dbd94 100644
--- a/tensorflow/python/keras/layers/unified_lstm_test.py
+++ b/tensorflow/python/keras/layers/unified_lstm_test.py
@@ -633,6 +633,29 @@ class UnifiedLSTMTest(keras_parameterized.TestCase):
 
     self.assertAllClose(out7, out6, atol=1e-5)
 
+  def test_stateful_LSTM_training(self):
+    # See b/123587692 for more context.
+    vocab_size = 20
+    embedding_dim = 10
+    batch_size = 8
+    timestep = 12
+    units = 5
+    x = np.random.randint(0, vocab_size, size=(batch_size, timestep))
+    y = np.random.randint(0, vocab_size, size=(batch_size, timestep))
+
+    model = keras.Sequential([
+        keras.layers.Embedding(vocab_size, embedding_dim,
+                               batch_input_shape=[batch_size, timestep]),
+        keras.layers.UnifiedLSTM(units,
+                                 return_sequences=True,
+                                 stateful=True),
+        keras.layers.Dense(vocab_size)
+    ])
+    model.compile(optimizer='adam',
+                  loss='sparse_categorical_crossentropy',
+                  run_eagerly=testing_utils.should_run_eagerly())
+    model.fit(x, y, epochs=1, shuffle=False)
+
 
 class LSTMLayerGraphOnlyTest(test.TestCase):
 
-- 
GitLab


From 0bd3a75b2e7c1eb0689712a6e5fff7b0091ddf17 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 5 Feb 2019 12:12:03 -0800
Subject: [PATCH 1933/2345] Simplify dataset shuffling logic in Keras'
 model.fit(), and make it sensitive to the user's batch size.

PiperOrigin-RevId: 232533325
---
 tensorflow/python/keras/engine/training.py | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index e93f2e0523..6530b086ab 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -2291,18 +2291,14 @@ class Model(Network):
         else:
           in_tuple = x
 
-        if shuffle:
-          # 1024 is a good buffer size since it is much larger than the average
-          # batch size provided by the user and provides sufficient randomness.
-          # One thing to keep in mind is the memory usage based on the size of
-          # each sample.
-          shuffle_buffer = 1024
-        else:
-          shuffle_buffer = None
         ds = strategy.extended.experimental_make_numpy_dataset(in_tuple,
                                                                session=session)
-        if shuffle_buffer:
-          ds = ds.shuffle(shuffle_buffer)
+        if shuffle:
+          # We want a buffer size that is larger than the batch size provided by
+          # the user and provides sufficient randomness. Note that larger
+          # numbers introduce more memory usage based on the size of each
+          # sample.
+          ds = ds.shuffle(max(1024, batch_size * 8))
         if repeat:
           ds = ds.repeat()
 
-- 
GitLab


From 112dc292a0661f37ef296a9368e73d71a6edfe40 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Tue, 5 Feb 2019 12:15:05 -0800
Subject: [PATCH 1934/2345] [XLA] Simplify function in literal.cc.  NFC

PiperOrigin-RevId: 232533777
---
 tensorflow/compiler/xla/literal.cc | 32 ++++++++++++------------------
 1 file changed, 13 insertions(+), 19 deletions(-)

diff --git a/tensorflow/compiler/xla/literal.cc b/tensorflow/compiler/xla/literal.cc
index 8600e8752c..bb9bca0986 100644
--- a/tensorflow/compiler/xla/literal.cc
+++ b/tensorflow/compiler/xla/literal.cc
@@ -1628,26 +1628,20 @@ bool LiteralBase::IsAllFloat(float value) const {
           return true;
         }
 
-        auto piece_is_all = [&]() {
-          switch (shape().element_type()) {
-            case F32:
-              return AllElementsEqualValue<float>(piece.data<float>(), value);
-            case F64:
-              return AllElementsEqualValue<double>(piece.data<double>(), value);
-            case F16:
-              return AllElementsEqualValue<half>(piece.data<half>(),
-                                                 static_cast<half>(value));
-            case BF16:
-              return AllElementsEqualValue<bfloat16>(
-                  piece.data<bfloat16>(), static_cast<bfloat16>(value));
-            default:
-              return false;
-          }
-        };
-        if (!piece_is_all()) {
-          return false;
+        switch (shape().element_type()) {
+          case F32:
+            return AllElementsEqualValue<float>(piece.data<float>(), value);
+          case F64:
+            return AllElementsEqualValue<double>(piece.data<double>(), value);
+          case F16:
+            return AllElementsEqualValue<half>(piece.data<half>(),
+                                               static_cast<half>(value));
+          case BF16:
+            return AllElementsEqualValue<bfloat16>(
+                piece.data<bfloat16>(), static_cast<bfloat16>(value));
+          default:
+            return false;
         }
-        return true;
       });
 }
 
-- 
GitLab


From 9efc4910030fb7afe71246482da898040e88dc74 Mon Sep 17 00:00:00 2001
From: Shivani Agrawal <shivaniagrawal@google.com>
Date: Tue, 5 Feb 2019 12:26:11 -0800
Subject: [PATCH 1935/2345] [data-coverage] Adds eager and v2 coverage for
 stats_dataset_op_test.

PiperOrigin-RevId: 232535654
---
 .../kernel_tests/stats_dataset_ops_test.py        | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py b/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py
index 8b330559f5..2e40c5236b 100644
--- a/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py
@@ -35,7 +35,6 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
 
-@test_util.run_all_in_graph_and_eager_modes
 def function_set_stats_aggregator(dataset,
                                   aggregator,
                                   prefix="",
@@ -53,6 +52,7 @@ def function_apply_options(dataset, aggregator, prefix="", counter_prefix=""):
   return dataset.with_options(options)
 
 
+@test_util.run_all_in_graph_and_eager_modes
 @parameterized.named_parameters(
     ("SetStatsAggregator", function_set_stats_aggregator),
     ("StatsOptions", function_apply_options),
@@ -66,7 +66,6 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
             stats_ops.bytes_produced_stats("bytes_produced"))
     dataset = dataset_transformation(dataset, aggregator)
     next_element = self.getNext(dataset, requires_initialization=True)
-    summary_t = aggregator.get_summary()
 
     expected_sum = 0.0
     for i in range(100):
@@ -78,8 +77,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
       self._assertSummaryHasSum(summary_str, "bytes_produced", expected_sum)
     with self.assertRaises(errors.OutOfRangeError):
       self.evaluate(next_element())
-    # TODO(shivaniagrawal): ntentional breaking case
-    summary_str = self.evaluate(summary_t)
+    summary_str = self.evaluate(aggregator.get_summary())
     self._assertSummaryHasCount(summary_str, "bytes_produced", 100.0)
     self._assertSummaryHasSum(summary_str, "bytes_produced", expected_sum)
 
@@ -357,13 +355,10 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
         100.0)
 
 
+@test_util.run_all_in_graph_and_eager_modes
 @parameterized.named_parameters(
-    dict(
-        testcase_name="SetStatsAggregator",
-        dataset_transformation=function_set_stats_aggregator),
-    dict(
-        testcase_name="StatsOptions",
-        dataset_transformation=function_apply_options))
+    ("SetStatsAggregator", function_set_stats_aggregator),
+    ("StatsOptions", function_apply_options))
 class FeatureStatsDatasetTest(
     stats_dataset_test_base.StatsDatasetTestBase,
     reader_dataset_ops_test_base.MakeBatchedFeaturesDatasetTestBase):
-- 
GitLab


From 6925a47c3e18566176d80cb48881dce20387d7fd Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Tue, 5 Feb 2019 12:37:36 -0800
Subject: [PATCH 1936/2345] [XLA] Remove XlaBuilder::Constant...() templates.

These are private and unused.

PiperOrigin-RevId: 232537450
---
 tensorflow/compiler/xla/client/xla_builder.h | 107 -------------------
 1 file changed, 107 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_builder.h b/tensorflow/compiler/xla/client/xla_builder.h
index c429035ad0..07c2496fca 100644
--- a/tensorflow/compiler/xla/client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_builder.h
@@ -315,38 +315,6 @@ class XlaBuilder {
 
   XlaOp ConstantLiteral(const LiteralSlice& literal);
 
-  template <typename NativeT>
-  XlaOp ConstantR0(NativeT value);
-  template <typename NativeT>
-  XlaOp ConstantR1(absl::Span<const NativeT> values);
-  XlaOp ConstantR1(const tensorflow::core::Bitmap& values);
-  template <typename NativeT>
-  XlaOp ConstantR2(
-      std::initializer_list<std::initializer_list<NativeT>> values);
-  template <typename NativeT>
-  XlaOp ConstantFromArrayWithLayout(const Array<NativeT>& values,
-                                    const Layout& layout);
-  template <typename NativeT>
-  XlaOp ConstantFromArray(const Array<NativeT>& values);
-  template <typename NativeT>
-  XlaOp ConstantR2FromArray2DWithLayout(const Array2D<NativeT>& values,
-                                        const Layout& layout);
-  template <typename NativeT>
-  XlaOp ConstantR2FromArray2D(const Array2D<NativeT>& values);
-  template <typename NativeT>
-  XlaOp ConstantR3FromArray3DWithLayout(const Array3D<NativeT>& values,
-                                        const Layout& layout);
-  template <typename NativeT>
-  XlaOp ConstantR3FromArray3D(const Array3D<NativeT>& values);
-  template <typename NativeT>
-  XlaOp ConstantR4FromArray4DWithLayout(const Array4D<NativeT>& values,
-                                        const Layout& layout);
-  template <typename NativeT>
-  XlaOp ConstantR4FromArray4D(const Array4D<NativeT>& values);
-
-  template <typename NativeT>
-  XlaOp ConstantR1(int64 length, NativeT value);
-
   XlaOp Broadcast(const XlaOp& operand,
                   absl::Span<const int64> broadcast_sizes);
 
@@ -1970,81 +1938,6 @@ XlaOp GetDimensionSize(const XlaOp& operand, int64 dimension);
 // Implementation details below this point.
 //
 
-template <typename NativeT>
-XlaOp XlaBuilder::ConstantR0(NativeT value) {
-  return ConstantLiteral(LiteralUtil::CreateR0<NativeT>(value));
-}
-
-template <typename NativeT>
-XlaOp XlaBuilder::ConstantR1(absl::Span<const NativeT> values) {
-  return ConstantLiteral(LiteralUtil::CreateR1<NativeT>(values));
-}
-
-template <typename NativeT>
-XlaOp XlaBuilder::ConstantR1(int64 length, NativeT value) {
-  Literal literal(ShapeUtil::MakeShape(
-      primitive_util::NativeToPrimitiveType<NativeT>(), {length}));
-  literal.PopulateWithValue(value);
-  return ConstantLiteral(literal);
-}
-
-inline XlaOp XlaBuilder::ConstantR1(const tensorflow::core::Bitmap& values) {
-  return ConstantLiteral(LiteralUtil::CreateR1(values));
-}
-
-template <typename NativeT>
-XlaOp XlaBuilder::ConstantR2(
-    std::initializer_list<std::initializer_list<NativeT>> values) {
-  return ConstantLiteral(LiteralUtil::CreateR2<NativeT>(values));
-}
-
-template <typename NativeT>
-XlaOp XlaBuilder::ConstantFromArrayWithLayout(const Array<NativeT>& values,
-                                              const Layout& layout) {
-  return ConstantLiteral(
-      LiteralUtil::CreateFromArrayWithLayout<NativeT>(values, layout));
-}
-
-template <typename NativeT>
-XlaOp XlaBuilder::ConstantFromArray(const Array<NativeT>& values) {
-  return ConstantLiteral(LiteralUtil::CreateFromArray<NativeT>(values));
-}
-
-template <typename NativeT>
-XlaOp XlaBuilder::ConstantR2FromArray2DWithLayout(
-    const Array2D<NativeT>& values, const Layout& layout) {
-  return ConstantLiteral(
-      LiteralUtil::CreateFromArrayWithLayout<NativeT>(values, layout));
-}
-
-template <typename NativeT>
-XlaOp XlaBuilder::ConstantR2FromArray2D(const Array2D<NativeT>& values) {
-  return ConstantLiteral(LiteralUtil::CreateR2FromArray2D<NativeT>(values));
-}
-
-template <typename NativeT>
-XlaOp XlaBuilder::ConstantR3FromArray3DWithLayout(
-    const Array3D<NativeT>& values, const Layout& layout) {
-  return ConstantLiteral(
-      LiteralUtil::CreateR3FromArray3DWithLayout<NativeT>(values, layout));
-}
-
-template <typename NativeT>
-XlaOp XlaBuilder::ConstantR3FromArray3D(const Array3D<NativeT>& values) {
-  return ConstantFromArray(values);
-}
-
-template <typename NativeT>
-XlaOp XlaBuilder::ConstantR4FromArray4DWithLayout(
-    const Array4D<NativeT>& values, const Layout& layout) {
-  return ConstantFromArrayWithLayout(values, layout);
-}
-
-template <typename NativeT>
-XlaOp XlaBuilder::ConstantR4FromArray4D(const Array4D<NativeT>& values) {
-  return ConstantFromArray(values);
-}
-
 // Free function template implementations.
 
 template <typename NativeT>
-- 
GitLab


From 50e51ae4f5555823184453d58c11984765495bfe Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Tue, 5 Feb 2019 12:56:52 -0800
Subject: [PATCH 1937/2345] When init_scoping to an outer eager context,
 attempt to re-apply the innermost device annotation.

We're unable to re-apply it if the annotation is a device function (since eager doesn't support those).

PiperOrigin-RevId: 232540960
---
 tensorflow/python/eager/def_function_test.py | 23 ++++++++++++++++++++
 tensorflow/python/framework/ops.py           | 21 +++++++++++++++++-
 2 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/eager/def_function_test.py b/tensorflow/python/eager/def_function_test.py
index 76e47fce85..1082c90d1b 100644
--- a/tensorflow/python/eager/def_function_test.py
+++ b/tensorflow/python/eager/def_function_test.py
@@ -21,6 +21,7 @@ import functools
 import weakref
 
 from tensorflow.python.eager import backprop
+from tensorflow.python.eager import context
 from tensorflow.python.eager import def_function
 from tensorflow.python.eager import lift_to_graph
 from tensorflow.python.framework import constant_op
@@ -31,6 +32,7 @@ from tensorflow.python.framework import test_util
 from tensorflow.python.keras.engine import training
 from tensorflow.python.keras.layers import core
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
@@ -393,6 +395,27 @@ class DefFunctionTest(test.TestCase):
     v_holder[1].assign(11.)
     self.assertAllClose([14., 15.], wrapper(constant_op.constant(2.)))
 
+  def testDeviceAnnotationRespected(self):
+    if not context.num_gpus():
+      self.skipTest("Needs multiple devices")
+
+    a = []
+
+    @def_function.function()
+    def create_variable():
+      with ops.init_scope():
+        initial_value = random_ops.random_uniform(
+            (2, 2), maxval=1000000, dtype=dtypes.int64)
+
+      if not a:
+        with ops.device("CPU:0"):
+          a.append(resource_variable_ops.ResourceVariable(initial_value))
+
+      return a[0].read_value()
+
+    created_variable_read = create_variable()
+    self.assertRegexpMatches(created_variable_read.device, "CPU")
+
 
 if __name__ == '__main__':
   ops.enable_eager_execution()
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 8cbdbec657..6e64622131 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -95,9 +95,12 @@ class _UserDeviceSpec(object):
         lineno = -1
       self.display_name = "%s<%s, %d>" % (func_name, fname, lineno)
 
+    self.raw_string = None
+
     self.function = self._device_name_or_function
     if not (self._device_name_or_function is None or
             callable(self._device_name_or_function)):
+      self.raw_string = self._device_name_or_function
       self.function = pydev.merge_device(self._device_name_or_function)
 
 
@@ -5544,6 +5547,8 @@ def init_scope():
     try:
       with outer_context(), name_scope(scope), control_dependencies(
           None), tape.stop_recording():
+        context_manager = NullContextmanager
+        context_manager_input = None
         if not context.executing_eagerly():
           # The device stack is preserved when lifting into a graph. Eager
           # execution doesn't implement device stacks and in particular it
@@ -5552,7 +5557,21 @@ def init_scope():
           outer_graph = get_default_graph()
           outer_device_stack = outer_graph._device_function_stack  # pylint: disable=protected-access
           outer_graph._device_function_stack = innermost_nonempty_device_stack  # pylint: disable=protected-access
-        yield
+        elif innermost_nonempty_device_stack is not None:
+          for device_spec in innermost_nonempty_device_stack.peek_objs():
+            if device_spec.function is None:
+              break
+            if device_spec.raw_string:
+              context_manager = context.device
+              context_manager_input = device_spec.raw_string
+              break
+            # It is currently not possible to have a device function in V2,
+            # but in V1 we are unable to apply device functions in eager mode.
+            # This means that we will silently skip some of the entries on the
+            # device stack in V1 + eager mode.
+
+        with context_manager(context_manager_input):
+          yield
     finally:
       # If an exception is raised here it may be hiding a related exception in
       # try-block (just above).
-- 
GitLab


From 761a98c6f12c4e3714d47cc402aef30fafe2ba6b Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Tue, 5 Feb 2019 13:09:22 -0800
Subject: [PATCH 1938/2345] [XLA] Remove XlaBuilder methods for unary and
 binary operators.

These methods (with one exception) had only one caller?the corresponding free function in the xla namespace. This change inlines the methods into their callers.

PiperOrigin-RevId: 232543596
---
 tensorflow/compiler/xla/client/xla_builder.cc | 355 ++++++------------
 tensorflow/compiler/xla/client/xla_builder.h  | 101 -----
 2 files changed, 105 insertions(+), 351 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc
index c00ba26295..b64d352e2c 100644
--- a/tensorflow/compiler/xla/client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_builder.cc
@@ -573,16 +573,6 @@ XlaOp XlaBuilder::TernaryOp(HloOpcode triop, const XlaOp& lhs, const XlaOp& rhs,
   });
 }
 
-XlaOp XlaBuilder::Add(const XlaOp& lhs, const XlaOp& rhs,
-                      absl::Span<const int64> broadcast_dimensions) {
-  return BinaryOp(HloOpcode::kAdd, lhs, rhs, broadcast_dimensions);
-}
-
-XlaOp XlaBuilder::Mul(const XlaOp& lhs, const XlaOp& rhs,
-                      absl::Span<const int64> broadcast_dimensions) {
-  return BinaryOp(HloOpcode::kMultiply, lhs, rhs, broadcast_dimensions);
-}
-
 XlaOp XlaBuilder::ConstantLiteral(const LiteralSlice& literal) {
   return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
     HloInstructionProto instr;
@@ -1004,36 +994,6 @@ XlaOp XlaBuilder::GetTupleElement(const XlaOp& tuple_data, int64 index) {
   });
 }
 
-XlaOp XlaBuilder::Eq(const XlaOp& lhs, const XlaOp& rhs,
-                     absl::Span<const int64> broadcast_dimensions) {
-  return BinaryOp(HloOpcode::kEq, lhs, rhs, broadcast_dimensions);
-}
-
-XlaOp XlaBuilder::Ne(const XlaOp& lhs, const XlaOp& rhs,
-                     absl::Span<const int64> broadcast_dimensions) {
-  return BinaryOp(HloOpcode::kNe, lhs, rhs, broadcast_dimensions);
-}
-
-XlaOp XlaBuilder::Ge(const XlaOp& lhs, const XlaOp& rhs,
-                     absl::Span<const int64> broadcast_dimensions) {
-  return BinaryOp(HloOpcode::kGe, lhs, rhs, broadcast_dimensions);
-}
-
-XlaOp XlaBuilder::Gt(const XlaOp& lhs, const XlaOp& rhs,
-                     absl::Span<const int64> broadcast_dimensions) {
-  return BinaryOp(HloOpcode::kGt, lhs, rhs, broadcast_dimensions);
-}
-
-XlaOp XlaBuilder::Le(const XlaOp& lhs, const XlaOp& rhs,
-                     absl::Span<const int64> broadcast_dimensions) {
-  return BinaryOp(HloOpcode::kLe, lhs, rhs, broadcast_dimensions);
-}
-
-XlaOp XlaBuilder::Lt(const XlaOp& lhs, const XlaOp& rhs,
-                     absl::Span<const int64> broadcast_dimensions) {
-  return BinaryOp(HloOpcode::kLt, lhs, rhs, broadcast_dimensions);
-}
-
 XlaOp XlaBuilder::Dot(const XlaOp& lhs, const XlaOp& rhs,
                       const PrecisionConfig* precision_config) {
   return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
@@ -1549,147 +1509,6 @@ XlaOp XlaBuilder::CustomCall(
   });
 }
 
-XlaOp XlaBuilder::Complex(const XlaOp& real, const XlaOp& imag,
-                          absl::Span<const int64> broadcast_dimensions) {
-  return BinaryOp(HloOpcode::kComplex, real, imag, broadcast_dimensions);
-}
-
-XlaOp XlaBuilder::Conj(const XlaOp& operand) {
-  return Complex(Real(operand), Neg(Imag(operand)));
-}
-
-XlaOp XlaBuilder::Sub(const XlaOp& lhs, const XlaOp& rhs,
-                      absl::Span<const int64> broadcast_dimensions) {
-  return BinaryOp(HloOpcode::kSubtract, lhs, rhs, broadcast_dimensions);
-}
-
-XlaOp XlaBuilder::Div(const XlaOp& lhs, const XlaOp& rhs,
-                      absl::Span<const int64> broadcast_dimensions) {
-  return BinaryOp(HloOpcode::kDivide, lhs, rhs, broadcast_dimensions);
-}
-
-XlaOp XlaBuilder::Rem(const XlaOp& lhs, const XlaOp& rhs,
-                      absl::Span<const int64> broadcast_dimensions) {
-  return BinaryOp(HloOpcode::kRemainder, lhs, rhs, broadcast_dimensions);
-}
-
-XlaOp XlaBuilder::Max(const XlaOp& lhs, const XlaOp& rhs,
-                      absl::Span<const int64> broadcast_dimensions) {
-  return BinaryOp(HloOpcode::kMaximum, lhs, rhs, broadcast_dimensions);
-}
-
-XlaOp XlaBuilder::Min(const XlaOp& lhs, const XlaOp& rhs,
-                      absl::Span<const int64> broadcast_dimensions) {
-  return BinaryOp(HloOpcode::kMinimum, lhs, rhs, broadcast_dimensions);
-}
-
-XlaOp XlaBuilder::And(const XlaOp& lhs, const XlaOp& rhs,
-                      absl::Span<const int64> broadcast_dimensions) {
-  return BinaryOp(HloOpcode::kAnd, lhs, rhs, broadcast_dimensions);
-}
-
-XlaOp XlaBuilder::Or(const XlaOp& lhs, const XlaOp& rhs,
-                     absl::Span<const int64> broadcast_dimensions) {
-  return BinaryOp(HloOpcode::kOr, lhs, rhs, broadcast_dimensions);
-}
-
-XlaOp XlaBuilder::Xor(const XlaOp& lhs, const XlaOp& rhs,
-                      absl::Span<const int64> broadcast_dimensions) {
-  return BinaryOp(HloOpcode::kXor, lhs, rhs, broadcast_dimensions);
-}
-
-XlaOp XlaBuilder::Not(const XlaOp& operand) {
-  return UnaryOp(HloOpcode::kNot, operand);
-}
-
-XlaOp XlaBuilder::ShiftLeft(const XlaOp& lhs, const XlaOp& rhs,
-                            absl::Span<const int64> broadcast_dimensions) {
-  return BinaryOp(HloOpcode::kShiftLeft, lhs, rhs, broadcast_dimensions);
-}
-
-XlaOp XlaBuilder::ShiftRightArithmetic(
-    const XlaOp& lhs, const XlaOp& rhs,
-    absl::Span<const int64> broadcast_dimensions) {
-  return BinaryOp(HloOpcode::kShiftRightArithmetic, lhs, rhs,
-                  broadcast_dimensions);
-}
-
-XlaOp XlaBuilder::ShiftRightLogical(
-    const XlaOp& lhs, const XlaOp& rhs,
-    absl::Span<const int64> broadcast_dimensions) {
-  return BinaryOp(HloOpcode::kShiftRightLogical, lhs, rhs,
-                  broadcast_dimensions);
-}
-
-XlaOp XlaBuilder::Abs(const XlaOp& operand) {
-  return UnaryOp(HloOpcode::kAbs, operand);
-}
-
-XlaOp XlaBuilder::Atan2(const XlaOp& y, const XlaOp& x,
-                        absl::Span<const int64> broadcast_dimensions) {
-  return BinaryOp(HloOpcode::kAtan2, y, x, broadcast_dimensions);
-}
-
-XlaOp XlaBuilder::Exp(const XlaOp& operand) {
-  return UnaryOp(HloOpcode::kExp, operand);
-}
-
-XlaOp XlaBuilder::Expm1(const XlaOp& operand) {
-  return UnaryOp(HloOpcode::kExpm1, operand);
-}
-
-XlaOp XlaBuilder::Floor(const XlaOp& operand) {
-  return UnaryOp(HloOpcode::kFloor, operand);
-}
-
-XlaOp XlaBuilder::Ceil(const XlaOp& operand) {
-  return UnaryOp(HloOpcode::kCeil, operand);
-}
-
-XlaOp XlaBuilder::Round(const XlaOp& operand) {
-  return UnaryOp(HloOpcode::kRoundNearestAfz, operand);
-}
-
-XlaOp XlaBuilder::Log(const XlaOp& operand) {
-  return UnaryOp(HloOpcode::kLog, operand);
-}
-
-XlaOp XlaBuilder::Log1p(const XlaOp& operand) {
-  return UnaryOp(HloOpcode::kLog1p, operand);
-}
-
-XlaOp XlaBuilder::Sign(const XlaOp& operand) {
-  return UnaryOp(HloOpcode::kSign, operand);
-}
-
-XlaOp XlaBuilder::Clz(const XlaOp& operand) {
-  return UnaryOp(HloOpcode::kClz, operand);
-}
-
-XlaOp XlaBuilder::Cos(const XlaOp& operand) {
-  return UnaryOp(HloOpcode::kCos, operand);
-}
-
-XlaOp XlaBuilder::Sin(const XlaOp& operand) {
-  return UnaryOp(HloOpcode::kSin, operand);
-}
-
-XlaOp XlaBuilder::Tanh(const XlaOp& operand) {
-  return UnaryOp(HloOpcode::kTanh, operand);
-}
-
-XlaOp XlaBuilder::Real(const XlaOp& operand) {
-  return UnaryOp(HloOpcode::kReal, operand);
-}
-
-XlaOp XlaBuilder::Imag(const XlaOp& operand) {
-  return UnaryOp(HloOpcode::kImag, operand);
-}
-
-XlaOp XlaBuilder::IsFinite(const XlaOp& operand) {
-  return UnaryOp(HloOpcode::kIsFinite, operand);
-}
-
 XlaOp XlaBuilder::Transpose(const XlaOp& operand,
                             absl::Span<const int64> permutation) {
   return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
@@ -1745,11 +1564,6 @@ XlaOp XlaBuilder::Sort(const XlaOp& keys, absl::Span<const XlaOp> values,
   });
 }
 
-XlaOp XlaBuilder::Pow(const XlaOp& lhs, const XlaOp& rhs,
-                      absl::Span<const int64> broadcast_dimensions) {
-  return BinaryOp(HloOpcode::kPower, lhs, rhs, broadcast_dimensions);
-}
-
 XlaOp XlaBuilder::ConvertElementType(const XlaOp& operand,
                                      PrimitiveType new_element_type) {
   return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
@@ -1775,10 +1589,6 @@ XlaOp XlaBuilder::BitcastConvertType(const XlaOp& operand,
   });
 }
 
-XlaOp XlaBuilder::Neg(const XlaOp& operand) {
-  return UnaryOp(HloOpcode::kNegate, operand);
-}
-
 XlaOp XlaBuilder::Clamp(const XlaOp& min, const XlaOp& operand,
                         const XlaOp& max) {
   return TernaryOp(HloOpcode::kClamp, min, operand, max);
@@ -2159,8 +1969,8 @@ XlaOp XlaBuilder::CrossReplicaSum(
     TF_ASSIGN_OR_RETURN(const Shape& shape, GetShape(operand));
     const Shape& scalar_shape = ShapeUtil::MakeShape(shape.element_type(), {});
     auto b = CreateSubBuilder("sum");
-    b->Add(b->Parameter(/*parameter_number=*/0, scalar_shape, "x"),
-           b->Parameter(/*parameter_number=*/1, scalar_shape, "y"));
+    Add(b->Parameter(/*parameter_number=*/0, scalar_shape, "x"),
+        b->Parameter(/*parameter_number=*/1, scalar_shape, "y"));
     TF_ASSIGN_OR_RETURN(auto computation, b->Build());
     return CrossReplicaSum(operand, computation, replica_groups,
                            /*channel_id=*/absl::nullopt);
@@ -2956,32 +2766,38 @@ XlaOp GetTupleElement(const XlaOp& tuple_data, int64 index) {
 
 XlaOp Eq(const XlaOp& lhs, const XlaOp& rhs,
          absl::Span<const int64> broadcast_dimensions) {
-  return lhs.builder()->Eq(lhs, rhs, broadcast_dimensions);
+  return lhs.builder()->BinaryOp(HloOpcode::kEq, lhs, rhs,
+                                 broadcast_dimensions);
 }
 
 XlaOp Ne(const XlaOp& lhs, const XlaOp& rhs,
          absl::Span<const int64> broadcast_dimensions) {
-  return lhs.builder()->Ne(lhs, rhs, broadcast_dimensions);
+  return lhs.builder()->BinaryOp(HloOpcode::kNe, lhs, rhs,
+                                 broadcast_dimensions);
 }
 
 XlaOp Ge(const XlaOp& lhs, const XlaOp& rhs,
          absl::Span<const int64> broadcast_dimensions) {
-  return lhs.builder()->Ge(lhs, rhs, broadcast_dimensions);
+  return lhs.builder()->BinaryOp(HloOpcode::kGe, lhs, rhs,
+                                 broadcast_dimensions);
 }
 
 XlaOp Gt(const XlaOp& lhs, const XlaOp& rhs,
          absl::Span<const int64> broadcast_dimensions) {
-  return lhs.builder()->Gt(lhs, rhs, broadcast_dimensions);
+  return lhs.builder()->BinaryOp(HloOpcode::kGt, lhs, rhs,
+                                 broadcast_dimensions);
 }
 
-XlaOp Lt(const XlaOp& lhs, const XlaOp& rhs,
+XlaOp Le(const XlaOp& lhs, const XlaOp& rhs,
          absl::Span<const int64> broadcast_dimensions) {
-  return lhs.builder()->Lt(lhs, rhs, broadcast_dimensions);
+  return lhs.builder()->BinaryOp(HloOpcode::kLe, lhs, rhs,
+                                 broadcast_dimensions);
 }
 
-XlaOp Le(const XlaOp& lhs, const XlaOp& rhs,
+XlaOp Lt(const XlaOp& lhs, const XlaOp& rhs,
          absl::Span<const int64> broadcast_dimensions) {
-  return lhs.builder()->Le(lhs, rhs, broadcast_dimensions);
+  return lhs.builder()->BinaryOp(HloOpcode::kLt, lhs, rhs,
+                                 broadcast_dimensions);
 }
 
 XlaOp Dot(const XlaOp& lhs, const XlaOp& rhs,
@@ -3084,78 +2900,96 @@ XlaOp CustomCallWithLayout(XlaBuilder* builder, const string& call_target_name,
                              operand_shapes_with_layout);
 }
 
-XlaOp Complex(const XlaOp& real, const XlaOp& imag,
+XlaOp Complex(const XlaOp& lhs, const XlaOp& rhs,
               absl::Span<const int64> broadcast_dimensions) {
-  return real.builder()->Complex(real, imag, broadcast_dimensions);
+  return lhs.builder()->BinaryOp(HloOpcode::kComplex, lhs, rhs,
+                                 broadcast_dimensions);
 }
 
-XlaOp Conj(const XlaOp& operand) { return operand.builder()->Conj(operand); }
+XlaOp Conj(const XlaOp& operand) {
+  return Complex(Real(operand), Neg(Imag(operand)));
+}
 
 XlaOp Add(const XlaOp& lhs, const XlaOp& rhs,
           absl::Span<const int64> broadcast_dimensions) {
-  return lhs.builder()->Add(lhs, rhs, broadcast_dimensions);
+  return lhs.builder()->BinaryOp(HloOpcode::kAdd, lhs, rhs,
+                                 broadcast_dimensions);
 }
 
 XlaOp Sub(const XlaOp& lhs, const XlaOp& rhs,
           absl::Span<const int64> broadcast_dimensions) {
-  return lhs.builder()->Sub(lhs, rhs, broadcast_dimensions);
+  return lhs.builder()->BinaryOp(HloOpcode::kSubtract, lhs, rhs,
+                                 broadcast_dimensions);
 }
 
 XlaOp Mul(const XlaOp& lhs, const XlaOp& rhs,
           absl::Span<const int64> broadcast_dimensions) {
-  return lhs.builder()->Mul(lhs, rhs, broadcast_dimensions);
+  return lhs.builder()->BinaryOp(HloOpcode::kMultiply, lhs, rhs,
+                                 broadcast_dimensions);
 }
 
 XlaOp Div(const XlaOp& lhs, const XlaOp& rhs,
           absl::Span<const int64> broadcast_dimensions) {
-  return lhs.builder()->Div(lhs, rhs, broadcast_dimensions);
+  return lhs.builder()->BinaryOp(HloOpcode::kDivide, lhs, rhs,
+                                 broadcast_dimensions);
 }
 
 XlaOp Rem(const XlaOp& lhs, const XlaOp& rhs,
           absl::Span<const int64> broadcast_dimensions) {
-  return lhs.builder()->Rem(lhs, rhs, broadcast_dimensions);
+  return lhs.builder()->BinaryOp(HloOpcode::kRemainder, lhs, rhs,
+                                 broadcast_dimensions);
 }
 
 XlaOp Max(const XlaOp& lhs, const XlaOp& rhs,
           absl::Span<const int64> broadcast_dimensions) {
-  return lhs.builder()->Max(lhs, rhs, broadcast_dimensions);
+  return lhs.builder()->BinaryOp(HloOpcode::kMaximum, lhs, rhs,
+                                 broadcast_dimensions);
 }
 
 XlaOp Min(const XlaOp& lhs, const XlaOp& rhs,
           absl::Span<const int64> broadcast_dimensions) {
-  return lhs.builder()->Min(lhs, rhs, broadcast_dimensions);
+  return lhs.builder()->BinaryOp(HloOpcode::kMinimum, lhs, rhs,
+                                 broadcast_dimensions);
 }
 
 XlaOp And(const XlaOp& lhs, const XlaOp& rhs,
           absl::Span<const int64> broadcast_dimensions) {
-  return lhs.builder()->And(lhs, rhs, broadcast_dimensions);
+  return lhs.builder()->BinaryOp(HloOpcode::kAnd, lhs, rhs,
+                                 broadcast_dimensions);
 }
 
 XlaOp Or(const XlaOp& lhs, const XlaOp& rhs,
          absl::Span<const int64> broadcast_dimensions) {
-  return lhs.builder()->Or(lhs, rhs, broadcast_dimensions);
+  return lhs.builder()->BinaryOp(HloOpcode::kOr, lhs, rhs,
+                                 broadcast_dimensions);
 }
 
 XlaOp Xor(const XlaOp& lhs, const XlaOp& rhs,
           absl::Span<const int64> broadcast_dimensions) {
-  return lhs.builder()->Xor(lhs, rhs, broadcast_dimensions);
+  return lhs.builder()->BinaryOp(HloOpcode::kXor, lhs, rhs,
+                                 broadcast_dimensions);
 }
 
-XlaOp Not(const XlaOp& operand) { return operand.builder()->Not(operand); }
+XlaOp Not(const XlaOp& operand) {
+  return operand.builder()->UnaryOp(HloOpcode::kNot, operand);
+}
 
 XlaOp ShiftLeft(const XlaOp& lhs, const XlaOp& rhs,
                 absl::Span<const int64> broadcast_dimensions) {
-  return lhs.builder()->ShiftLeft(lhs, rhs, broadcast_dimensions);
+  return lhs.builder()->BinaryOp(HloOpcode::kShiftLeft, lhs, rhs,
+                                 broadcast_dimensions);
 }
 
 XlaOp ShiftRightArithmetic(const XlaOp& lhs, const XlaOp& rhs,
                            absl::Span<const int64> broadcast_dimensions) {
-  return lhs.builder()->ShiftRightArithmetic(lhs, rhs, broadcast_dimensions);
+  return lhs.builder()->BinaryOp(HloOpcode::kShiftRightArithmetic, lhs, rhs,
+                                 broadcast_dimensions);
 }
 
 XlaOp ShiftRightLogical(const XlaOp& lhs, const XlaOp& rhs,
                         absl::Span<const int64> broadcast_dimensions) {
-  return lhs.builder()->ShiftRightLogical(lhs, rhs, broadcast_dimensions);
+  return lhs.builder()->BinaryOp(HloOpcode::kShiftRightLogical, lhs, rhs,
+                                 broadcast_dimensions);
 }
 
 XlaOp Reduce(const XlaOp& operand, const XlaOp& init_value,
@@ -3250,48 +3084,67 @@ XlaOp SelectAndScatterWithGeneralPadding(
       init_value, scatter);
 }
 
-XlaOp Abs(const XlaOp& operand) { return operand.builder()->Abs(operand); }
+XlaOp Abs(const XlaOp& operand) {
+  return operand.builder()->UnaryOp(HloOpcode::kAbs, operand);
+}
 
-XlaOp Atan2(const XlaOp& y, const XlaOp& x,
+XlaOp Atan2(const XlaOp& lhs, const XlaOp& rhs,
             absl::Span<const int64> broadcast_dimensions) {
-  return y.builder()->Atan2(y, x, broadcast_dimensions);
+  return lhs.builder()->BinaryOp(HloOpcode::kAtan2, lhs, rhs,
+                                 broadcast_dimensions);
 }
 
-XlaOp Exp(const XlaOp& operand) { return operand.builder()->Exp(operand); }
-
-XlaOp Expm1(const XlaOp& operand) { return operand.builder()->Expm1(operand); }
-
-XlaOp Floor(const XlaOp& operand) { return operand.builder()->Floor(operand); }
-
-XlaOp Ceil(const XlaOp& operand) { return operand.builder()->Ceil(operand); }
-
-XlaOp Round(const XlaOp& operand) { return operand.builder()->Round(operand); }
-
-XlaOp Log(const XlaOp& operand) { return operand.builder()->Log(operand); }
-
-XlaOp Log1p(const XlaOp& operand) { return operand.builder()->Log1p(operand); }
-
-XlaOp Sign(const XlaOp& operand) { return operand.builder()->Sign(operand); }
-
-XlaOp Clz(const XlaOp& operand) { return operand.builder()->Clz(operand); }
-
-XlaOp Cos(const XlaOp& operand) { return operand.builder()->Cos(operand); }
-
-XlaOp Sin(const XlaOp& operand) { return operand.builder()->Sin(operand); }
-
-XlaOp Tanh(const XlaOp& operand) { return operand.builder()->Tanh(operand); }
-
-XlaOp Real(const XlaOp& operand) { return operand.builder()->Real(operand); }
-
-XlaOp Imag(const XlaOp& operand) { return operand.builder()->Imag(operand); }
+XlaOp Exp(const XlaOp& operand) {
+  return operand.builder()->UnaryOp(HloOpcode::kExp, operand);
+}
+XlaOp Expm1(const XlaOp& operand) {
+  return operand.builder()->UnaryOp(HloOpcode::kExpm1, operand);
+}
+XlaOp Floor(const XlaOp& operand) {
+  return operand.builder()->UnaryOp(HloOpcode::kFloor, operand);
+}
+XlaOp Ceil(const XlaOp& operand) {
+  return operand.builder()->UnaryOp(HloOpcode::kCeil, operand);
+}
+XlaOp Round(const XlaOp& operand) {
+  return operand.builder()->UnaryOp(HloOpcode::kRoundNearestAfz, operand);
+}
+XlaOp Log(const XlaOp& operand) {
+  return operand.builder()->UnaryOp(HloOpcode::kLog, operand);
+}
+XlaOp Log1p(const XlaOp& operand) {
+  return operand.builder()->UnaryOp(HloOpcode::kLog1p, operand);
+}
+XlaOp Sign(const XlaOp& operand) {
+  return operand.builder()->UnaryOp(HloOpcode::kSign, operand);
+}
+XlaOp Clz(const XlaOp& operand) {
+  return operand.builder()->UnaryOp(HloOpcode::kClz, operand);
+}
+XlaOp Cos(const XlaOp& operand) {
+  return operand.builder()->UnaryOp(HloOpcode::kCos, operand);
+}
+XlaOp Sin(const XlaOp& operand) {
+  return operand.builder()->UnaryOp(HloOpcode::kSin, operand);
+}
+XlaOp Tanh(const XlaOp& operand) {
+  return operand.builder()->UnaryOp(HloOpcode::kTanh, operand);
+}
+XlaOp Real(const XlaOp& operand) {
+  return operand.builder()->UnaryOp(HloOpcode::kReal, operand);
+}
+XlaOp Imag(const XlaOp& operand) {
+  return operand.builder()->UnaryOp(HloOpcode::kImag, operand);
+}
 
 XlaOp Pow(const XlaOp& lhs, const XlaOp& rhs,
           absl::Span<const int64> broadcast_dimensions) {
-  return lhs.builder()->Pow(lhs, rhs, broadcast_dimensions);
+  return lhs.builder()->BinaryOp(HloOpcode::kPower, lhs, rhs,
+                                 broadcast_dimensions);
 }
 
 XlaOp IsFinite(const XlaOp& operand) {
-  return operand.builder()->IsFinite(operand);
+  return operand.builder()->UnaryOp(HloOpcode::kIsFinite, operand);
 }
 
 XlaOp ConvertElementType(const XlaOp& operand, PrimitiveType new_element_type) {
@@ -3302,7 +3155,9 @@ XlaOp BitcastConvertType(const XlaOp& operand, PrimitiveType new_element_type) {
   return operand.builder()->BitcastConvertType(operand, new_element_type);
 }
 
-XlaOp Neg(const XlaOp& operand) { return operand.builder()->Neg(operand); }
+XlaOp Neg(const XlaOp& operand) {
+  return operand.builder()->UnaryOp(HloOpcode::kNegate, operand);
+}
 
 XlaOp Transpose(const XlaOp& operand, absl::Span<const int64> permutation) {
   return operand.builder()->Transpose(operand, permutation);
diff --git a/tensorflow/compiler/xla/client/xla_builder.h b/tensorflow/compiler/xla/client/xla_builder.h
index 07c2496fca..fa742891e6 100644
--- a/tensorflow/compiler/xla/client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_builder.h
@@ -362,24 +362,6 @@ class XlaBuilder {
 
   XlaOp GetTupleElement(const XlaOp& tuple_data, int64 index);
 
-  XlaOp Eq(const XlaOp& lhs, const XlaOp& rhs,
-           absl::Span<const int64> broadcast_dimensions = {});
-
-  XlaOp Ne(const XlaOp& lhs, const XlaOp& rhs,
-           absl::Span<const int64> broadcast_dimensions = {});
-
-  XlaOp Ge(const XlaOp& lhs, const XlaOp& rhs,
-           absl::Span<const int64> broadcast_dimensions = {});
-
-  XlaOp Gt(const XlaOp& lhs, const XlaOp& rhs,
-           absl::Span<const int64> broadcast_dimensions = {});
-
-  XlaOp Lt(const XlaOp& lhs, const XlaOp& rhs,
-           absl::Span<const int64> broadcast_dimensions = {});
-
-  XlaOp Le(const XlaOp& lhs, const XlaOp& rhs,
-           absl::Span<const int64> broadcast_dimensions = {});
-
   XlaOp Dot(const XlaOp& lhs, const XlaOp& rhs,
             const PrecisionConfig* precision_config = nullptr);
 
@@ -444,49 +426,6 @@ class XlaBuilder {
       const Shape& shape_with_layout, const string& opaque,
       absl::optional<absl::Span<const Shape>> operand_shapes_with_layout);
 
-  XlaOp Complex(const XlaOp& real, const XlaOp& imag,
-                absl::Span<const int64> broadcast_dimensions = {});
-
-  XlaOp Conj(const XlaOp& operand);
-
-  XlaOp Add(const XlaOp& lhs, const XlaOp& rhs,
-            absl::Span<const int64> broadcast_dimensions = {});
-
-  XlaOp Sub(const XlaOp& lhs, const XlaOp& rhs,
-            absl::Span<const int64> broadcast_dimensions = {});
-
-  XlaOp Mul(const XlaOp& lhs, const XlaOp& rhs,
-            absl::Span<const int64> broadcast_dimensions = {});
-
-  XlaOp Div(const XlaOp& lhs, const XlaOp& rhs,
-            absl::Span<const int64> broadcast_dimensions = {});
-
-  XlaOp Rem(const XlaOp& lhs, const XlaOp& rhs,
-            absl::Span<const int64> broadcast_dimensions = {});
-
-  XlaOp Max(const XlaOp& lhs, const XlaOp& rhs,
-            absl::Span<const int64> broadcast_dimensions = {});
-
-  XlaOp Min(const XlaOp& lhs, const XlaOp& rhs,
-            absl::Span<const int64> broadcast_dimensions = {});
-
-  XlaOp And(const XlaOp& lhs, const XlaOp& rhs,
-            absl::Span<const int64> broadcast_dimensions = {});
-
-  XlaOp Or(const XlaOp& lhs, const XlaOp& rhs,
-           absl::Span<const int64> broadcast_dimensions = {});
-
-  XlaOp Xor(const XlaOp& lhs, const XlaOp& rhs,
-            absl::Span<const int64> broadcast_dimensions = {});
-
-  XlaOp Not(const XlaOp& operand);
-
-  XlaOp ShiftLeft(const XlaOp& lhs, const XlaOp& rhs,
-                  absl::Span<const int64> broadcast_dimensions = {});
-  XlaOp ShiftRightArithmetic(const XlaOp& lhs, const XlaOp& rhs,
-                             absl::Span<const int64> broadcast_dimensions = {});
-  XlaOp ShiftRightLogical(const XlaOp& lhs, const XlaOp& rhs,
-                          absl::Span<const int64> broadcast_dimensions = {});
 
   XlaOp Reduce(const XlaOp& operand, const XlaOp& init_value,
                const XlaComputation& computation,
@@ -546,44 +485,6 @@ class XlaBuilder {
       absl::Span<const std::pair<int64, int64>> padding, const XlaOp& source,
       const XlaOp& init_value, const XlaComputation& scatter);
 
-  XlaOp Abs(const XlaOp& operand);
-
-  XlaOp Atan2(const XlaOp& y, const XlaOp& x,
-              absl::Span<const int64> broadcast_dimensions = {});
-
-  XlaOp Exp(const XlaOp& operand);
-
-  XlaOp Expm1(const XlaOp& operand);
-
-  XlaOp Floor(const XlaOp& operand);
-
-  XlaOp Ceil(const XlaOp& operand);
-
-  XlaOp Round(const XlaOp& operand);
-
-  XlaOp Log(const XlaOp& operand);
-
-  XlaOp Log1p(const XlaOp& operand);
-
-  XlaOp Sign(const XlaOp& operand);
-
-  XlaOp Clz(const XlaOp& operand);
-
-  XlaOp Cos(const XlaOp& operand);
-
-  XlaOp Sin(const XlaOp& operand);
-
-  XlaOp Tanh(const XlaOp& operand);
-
-  XlaOp Real(const XlaOp& operand);
-
-  XlaOp Imag(const XlaOp& operand);
-
-  XlaOp Pow(const XlaOp& lhs, const XlaOp& rhs,
-            absl::Span<const int64> broadcast_dimensions = {});
-
-  XlaOp IsFinite(const XlaOp& operand);
-
   XlaOp Iota(const Shape& shape, int64 iota_dimension);
 
   XlaOp Iota(PrimitiveType type, int64 size);
@@ -594,8 +495,6 @@ class XlaBuilder {
   XlaOp BitcastConvertType(const XlaOp& operand,
                            PrimitiveType new_element_type);
 
-  XlaOp Neg(const XlaOp& operand);
-
   XlaOp Transpose(const XlaOp& operand, absl::Span<const int64> permutation);
 
   XlaOp Rev(const XlaOp& operand, absl::Span<const int64> dimensions);
-- 
GitLab


From 1506f859603f9293ff6c3b5c0e1e559c48827872 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Tue, 5 Feb 2019 13:18:55 -0800
Subject: [PATCH 1939/2345] Fix Estimator test after checkpointable refactor

PiperOrigin-RevId: 232545235
---
 tensorflow/python/training/monitored_session.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/training/monitored_session.py b/tensorflow/python/training/monitored_session.py
index 41a42bd269..1687898ec7 100644
--- a/tensorflow/python/training/monitored_session.py
+++ b/tensorflow/python/training/monitored_session.py
@@ -41,6 +41,7 @@ from tensorflow.python.training import queue_runner
 from tensorflow.python.training import saver as training_saver
 from tensorflow.python.training import session_manager as sm
 from tensorflow.python.training import session_run_hook
+from tensorflow.python.training.checkpointable import graph_view
 from tensorflow.python.training.checkpointable import util as checkpointable_util
 from tensorflow.python.util import function_utils
 from tensorflow.python.util.tf_export import tf_export
@@ -229,8 +230,8 @@ class Scaffold(object):
     # pylint: enable=g-long-lambda
     if isinstance(self._saver, checkpointable_util.Checkpoint):
       self._saver = training_saver.Saver(
-          var_list=checkpointable_util.CheckpointableSaver(
-              self._saver).gather_objects(),
+          var_list=graph_view.ObjectGraphView(
+              self._saver).frozen_saveable_objects(),
           sharded=True)
     else:
       self._saver.build()
-- 
GitLab


From b593cc75d3ee78d912fd05a275ab371055592095 Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Tue, 5 Feb 2019 13:20:44 -0800
Subject: [PATCH 1940/2345] Allow layers to return variables.

PiperOrigin-RevId: 232545535
---
 tensorflow/python/keras/backend.py            | 17 +++++++++++----
 .../python/keras/engine/base_layer_test.py    | 21 +++++++++++++++++++
 2 files changed, 34 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index 9c2b46ee24..064358cacd 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -2893,7 +2893,7 @@ class GraphExecutionFunction(object):
                       'should be a list or tuple.')
     self.inputs = nest.flatten(inputs)
     self._outputs_structure = outputs
-    self.outputs = nest.flatten(outputs)
+    self.outputs = cast_variables_to_tensor(nest.flatten(outputs))
     with ops.control_dependencies(self.outputs):
       updates_ops = []
       for update in updates:
@@ -3053,14 +3053,13 @@ class EagerExecutionFunction(object):
     if not isinstance(updates, (list, tuple)):
       raise TypeError('`updates` in a Keras backend function '
                       'should be a list or tuple.')
+    self.name = name
     self.inputs = nest.flatten(inputs)
     self._outputs_structure = outputs
-    self.outputs = nest.flatten(outputs)
-    self.name = name
-
     graph = get_graph()
     # Consolidate updates
     with graph.as_default():
+      self.outputs = cast_variables_to_tensor(nest.flatten(outputs))
       with ops.control_dependencies(self.outputs):
         # In general, updates should be run after the outputs have been
         # computed. However, we can only ensure this when we create
@@ -5262,3 +5261,13 @@ def configure_and_create_distributed_session(distribution_strategy):
 def is_tpu_strategy(strategy):
   """We're executing TPU Strategy."""
   return strategy is not None and strategy.__class__.__name__ == 'TPUStrategy'
+
+
+def cast_variables_to_tensor(tensors):
+
+  def _cast_variables_to_tensor(tensor):
+    if isinstance(tensor, variables_module.Variable):
+      return array_ops.identity(tensor)
+    return tensor
+
+  return nest.map_structure(_cast_variables_to_tensor, tensors)
diff --git a/tensorflow/python/keras/engine/base_layer_test.py b/tensorflow/python/keras/engine/base_layer_test.py
index ba14f94cc2..109fc1f25e 100644
--- a/tensorflow/python/keras/engine/base_layer_test.py
+++ b/tensorflow/python/keras/engine/base_layer_test.py
@@ -369,6 +369,27 @@ class BaseLayerTest(keras_parameterized.TestCase):
       function_name = last_entry[2]
       self.assertEqual(function_name, 'easily_identifiable_name')
 
+  # Cannot be enabled with `run_eagerly=True`, see b/123904578
+  @test_util.run_all_in_graph_and_eager_modes
+  def test_layer_can_return_variable(self):
+
+    class ComputeSum(keras.layers.Layer):
+
+      def __init__(self):
+        super(ComputeSum, self).__init__()
+        self.total = variables.Variable(
+            initial_value=array_ops.zeros((1, 1)), trainable=False)
+        if not context.executing_eagerly():
+          keras.backend.get_session().run(self.total.initializer)
+
+      def call(self, inputs):
+        self.total.assign_add(inputs)
+        return self.total
+
+    inputs = keras.Input(shape=(1,))
+    model = keras.Model(inputs, ComputeSum()(inputs))
+    model.predict(np.ones((1, 1)))
+
 
 @test_util.run_all_in_graph_and_eager_modes
 class NestedTrackingTest(test.TestCase):
-- 
GitLab


From 81c60429466be4c5a83776fdbb7638b04dfdb029 Mon Sep 17 00:00:00 2001
From: Amit Patankar <amitpatankar@google.com>
Date: Tue, 5 Feb 2019 13:33:15 -0800
Subject: [PATCH 1941/2345] Skip testNonMatchingVariableCreation.

PiperOrigin-RevId: 232547857
---
 .../contrib/distribute/python/mirrored_strategy_multigpu_test.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index 0b8df787e6..bc0572bb46 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -606,6 +606,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
             aggregation="invalid")
 
   def testNonMatchingVariableCreation(self, distribution):
+    self.skipTest("b/123075960")
     def model_fn(name):
       v = variable_scope.variable(1.0, name=name)
       ds_context.get_replica_context().merge_call(lambda _: _)
-- 
GitLab


From 37479879a931a298986999f15d4c1bd4ec2345a9 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Tue, 5 Feb 2019 13:48:48 -0800
Subject: [PATCH 1942/2345] [XLA] Add TODO about duplicated erfinv function.

Comment-only change.

PiperOrigin-RevId: 232550776
---
 tensorflow/compiler/xla/service/elemental_ir_emitter.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
index fef84ac80a..e868dc6d88 100644
--- a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
@@ -857,6 +857,9 @@ llvm::Value* ElementalIrEmitter::EmitFloatMin(llvm::Value* lhs_value,
   return llvm_ir::EmitFloatMin(lhs_value, rhs_value, b_);
 }
 
+// TODO(b/123355973): We have an implementation of erfinv in math.cc.  We
+// shouldn't have two implementations, especially since this one isn't testable
+// (it's only observable via a normally-distributed RNG).
 StatusOr<llvm::Value*> ElementalIrEmitter::EmitErfInv(PrimitiveType prim_type,
                                                       llvm::Value* x) {
   if (prim_type != F16 && prim_type != F32 && prim_type != F64) {
-- 
GitLab


From e0d42a96356a1b2b19f7a0694263505b4151a623 Mon Sep 17 00:00:00 2001
From: nammbash <niroopshankar.ammbashankar@intel.com>
Date: Tue, 5 Feb 2019 13:54:55 -0800
Subject: [PATCH 1943/2345] some small changes

---
 .../mkl_requantization_range_per_channel_op.cc       |  2 +-
 tensorflow/core/kernels/mkl_requantize_ops_test.cc   | 12 +++++-------
 2 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc b/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc
index a325576d60..e1dd386c7d 100644
--- a/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc
+++ b/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc
@@ -89,7 +89,7 @@ class MklRequantizationRangePerChannelOp : public OpKernel {
           scale * static_cast<float>(abs_max) / static_cast<float>(1L << 31);
       if (min_per_channel < 0) is_non_negative = false;
 
-      // thread-local out_min_max.
+      // Thread-local out_min_max.
       out_min_max = std::max(out_min_max, ranges[i]);
     }
     // All local out_min_max gets max-reduced into one global out_min_max at
diff --git a/tensorflow/core/kernels/mkl_requantize_ops_test.cc b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
index 178634860e..eeda1909f6 100644
--- a/tensorflow/core/kernels/mkl_requantize_ops_test.cc
+++ b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
@@ -71,14 +71,12 @@ void MklRequantizatedOpsTestHelper::Setup(Tensor &input_tensor_qint32,
   // ---------------------------------------------------------------------------
   // The input T and weights W1 (or W2) will be convolved.
   // The output tensor T is in int32 whose range is [-2^31, 2^31).
-  // For simplicity, we truncate the range to (-2^31, 2^31) to make it
-  // symmetric.
-  // The range of the convolved T*W1 is ((2^31)-1) * 5.0/255 * 2.0/127 =
-  // 663110.59
+  // For simplicity and symmetry, we truncate the above range to (-2^31, 2^31).
+  // The range of convolved T*W1 is ((2^31)-1) * 5.0/255 * 2.0/127 = 663110.59
   // So the range of convolved T*W1 in int32(-2^31, 2^31) that maps to
-  // orig T range in FP32[0, 5.0] * [-2.0, 2.0] is [-663110.59, 663110.59]
+  // orig T range in FP32[0, 5.0] * [-2.0, 2.0] is [-663110.59, 663110.59].
 
-  // The range of the convolved T*W2 is (2^31-1) * 5.0/255 * 3.0/127 = 994665.88
+  // The range of convolved T*W2 is (2^31-1) * 5.0/255 * 3.0/127 = 994665.88
   // So the range of convolved T*W2 in int32(-2^31, 2^31) that maps to
   // orig T range in FP32 [0, 5.0] * [-3.0, 3.0]  is [-994665.88, 994665.88]
 
@@ -166,7 +164,7 @@ TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_Basic) {
   EXPECT_NEAR(-14.8217, output_min, 0.002);
   EXPECT_NEAR(14.8217, output_max, 0.002);
 
-  // output range is made use in requantizePerChannelTest_Basic
+  // output range is made use in RequantizePerChannelTest_Basic
 }
 
 TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_ClipMax) {
-- 
GitLab


From 3206bcd6e22c8434ef0ed80d0814560da5b6a088 Mon Sep 17 00:00:00 2001
From: nammbash <niroopshankar.ammbashankar@intel.com>
Date: Tue, 5 Feb 2019 13:56:28 -0800
Subject: [PATCH 1944/2345] some small changes

---
 tensorflow/core/kernels/mkl_requantize_ops_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/mkl_requantize_ops_test.cc b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
index eeda1909f6..a156bd494c 100644
--- a/tensorflow/core/kernels/mkl_requantize_ops_test.cc
+++ b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
@@ -164,7 +164,7 @@ TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_Basic) {
   EXPECT_NEAR(-14.8217, output_min, 0.002);
   EXPECT_NEAR(14.8217, output_max, 0.002);
 
-  // output range is made use in RequantizePerChannelTest_Basic
+  // Output range is made use in RequantizePerChannelTest_Basic
 }
 
 TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_ClipMax) {
-- 
GitLab


From 99762fa064e58058196f7c00cb3e65a01b051684 Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <penporn@google.com>
Date: Tue, 5 Feb 2019 13:53:18 -0800
Subject: [PATCH 1945/2345] Automated rollback of commit
 b1177e81fef8b3b97cb8751c1a43f25ed04af78b. Revert #25146.

PiperOrigin-RevId: 232551587
---
 tensorflow/core/BUILD                         |  14 +-
 tensorflow/core/api_def/excluded_ops.cc       |   6 +-
 tensorflow/core/framework/common_shape_fns.cc |   6 -
 tensorflow/core/framework/common_shape_fns.h  |   2 -
 tensorflow/core/kernels/BUILD                 |  24 --
 tensorflow/core/kernels/mkl_concat_op.cc      |  77 +-----
 .../kernels/mkl_quantized_concat_op_test.cc   | 230 ------------------
 .../core/kernels/quantized_concat_op.cc       |  10 -
 tensorflow/core/ops/mkl_array_ops.cc          |  92 -------
 9 files changed, 6 insertions(+), 455 deletions(-)
 delete mode 100644 tensorflow/core/kernels/mkl_quantized_concat_op_test.cc
 delete mode 100644 tensorflow/core/ops/mkl_array_ops.cc

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 8002d56a15..3a56fa476c 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1144,13 +1144,6 @@ tf_gen_op_libs(
     deps = [":protos_all_cc"],
 )
 
-tf_gen_op_libs(
-    op_lib_names = [
-        "mkl_array_ops",
-    ],
-    deps = [":protos_all_cc"],
-)
-
 tf_gen_op_libs(
     op_lib_names = [
         "audio_ops",
@@ -1290,10 +1283,7 @@ cc_library(
         ":training_ops_op_lib",
         ":user_ops_op_lib",
         ":word2vec_ops",
-    ] + if_mkl([
-        ":mkl_array_ops_op_lib",
-        ":mkl_nn_ops_op_lib",
-    ]) + tf_additional_cloud_op_deps(),
+    ] + if_mkl([":mkl_nn_ops_op_lib"]) + tf_additional_cloud_op_deps(),
     alwayslink = 1,
 )
 
@@ -4533,7 +4523,7 @@ tf_cc_test(
         "//tensorflow/cc:scope",
         "//tensorflow/core/kernels:cwise_op",
         "//third_party/eigen3",
-    ] + if_mkl([":mkl_array_ops_op_lib"]),
+    ],
 )
 
 tf_cc_test(
diff --git a/tensorflow/core/api_def/excluded_ops.cc b/tensorflow/core/api_def/excluded_ops.cc
index 65d2102ac8..02026e94ab 100644
--- a/tensorflow/core/api_def/excluded_ops.cc
+++ b/tensorflow/core/api_def/excluded_ops.cc
@@ -24,9 +24,9 @@ const std::unordered_set<std::string>* GetExcludedOps() {
            "GcsConfigureBlockCache", "GcsConfigureCredentials",
 #ifdef INTEL_MKL
            // QuantizedFusedOps for Intel CPU
-           "QuantizedConcatV2", "QuantizedConv2DAndRequantize",
-           "QuantizedConv2DWithBias", "QuantizedConv2DWithBiasAndRequantize",
-           "QuantizedConv2DAndRelu", "QuantizedConv2DAndReluAndRequantize",
+           "QuantizedConv2DAndRequantize", "QuantizedConv2DWithBias",
+           "QuantizedConv2DWithBiasAndRequantize", "QuantizedConv2DAndRelu",
+           "QuantizedConv2DAndReluAndRequantize",
            "QuantizedConv2DWithBiasAndRelu",
            "QuantizedConv2DWithBiasAndReluAndRequantize",
            "QuantizedConv2DWithBiasSumAndRelu",
diff --git a/tensorflow/core/framework/common_shape_fns.cc b/tensorflow/core/framework/common_shape_fns.cc
index 5c974a76ac..83bc95065e 100644
--- a/tensorflow/core/framework/common_shape_fns.cc
+++ b/tensorflow/core/framework/common_shape_fns.cc
@@ -1303,12 +1303,6 @@ Status ConcatV2Shape(InferenceContext* c) {
                            c->num_inputs() - 1 /* dim_index */);
 }
 
-Status QuantizedConcatV2Shape(InferenceContext* c, int num_inputs_to_concat) {
-  return ConcatShapeHelper(c, 0 /* start_value_index */,
-                           num_inputs_to_concat /* end_value_index */,
-                           num_inputs_to_concat /* dim_index */);
-}
-
 Status BroadcastBinaryOpOutputShapeFnHelper(InferenceContext* c,
                                             ShapeHandle shape_x,
                                             ShapeHandle shape_y,
diff --git a/tensorflow/core/framework/common_shape_fns.h b/tensorflow/core/framework/common_shape_fns.h
index d421844ee6..14b9688bdc 100644
--- a/tensorflow/core/framework/common_shape_fns.h
+++ b/tensorflow/core/framework/common_shape_fns.h
@@ -279,8 +279,6 @@ Status ConcatShape(shape_inference::InferenceContext* c,
 // Shape function for concat operations.
 Status ConcatV2Shape(shape_inference::InferenceContext* c);
 
-Status QuantizedConcatV2Shape(InferenceContext* c, int num_inputs_to_concat);
-
 // Shape function for binary operators that broadcast their inputs
 // and with output to output_index.
 // Note: out cannot be NULL.
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 1749f3a4da..e5ee109fff 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -6561,30 +6561,6 @@ tf_cc_test(
     ],
 )
 
-tf_cc_test_mkl(
-    name = "mkl_quantized_concat_op_test",
-    size = "small",
-    srcs = ["mkl_quantized_concat_op_test.cc"],
-    deps = [
-        ":mkl_concat_op",
-        ":ops_testutil",
-        ":ops_util",
-        ":quantization_utils",
-        ":quantized_ops",
-        "//tensorflow/core:array_ops_op_lib",
-        "//tensorflow/core:core_cpu",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:math_ops_op_lib",
-        "//tensorflow/core:mkl_array_ops_op_lib",
-        "//tensorflow/core:nn_ops_op_lib",
-        "//tensorflow/core:protos_all_cc",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-        "//tensorflow/core:testlib",
-    ],
-)
-
 tf_cc_test(
     name = "quantized_batch_norm_op_test",
     size = "small",
diff --git a/tensorflow/core/kernels/mkl_concat_op.cc b/tensorflow/core/kernels/mkl_concat_op.cc
index b95bbca6b5..3a5c87485c 100644
--- a/tensorflow/core/kernels/mkl_concat_op.cc
+++ b/tensorflow/core/kernels/mkl_concat_op.cc
@@ -25,14 +25,12 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/kernels/concat_lib.h"
-#include "tensorflow/core/kernels/concat_lib_cpu.h"
-#include "tensorflow/core/kernels/quantization_utils.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
-#include "tensorflow/core/util/mkl_util.h"
 
 using mkldnn::concat;
 using mkldnn::stream;
+#include "tensorflow/core/util/mkl_util.h"
 
 namespace tensorflow {
 typedef Eigen::ThreadPoolDevice CPUDevice;
@@ -228,50 +226,8 @@ class MklConcatOp : public OpKernel {
       // format and avoid calling eigen version.
       if (!are_all_tf_inputs && !are_all_mkl_inputs) invoke_eigen = true;
 
-      OpInputList input_mins, input_maxes;
-      if (std::is_same<T, qint8>::value || std::is_same<T, quint8>::value) {
-        // MKL-DNN concat does not support input tensors that have different
-        // ranges. Check if the ranges of the all input tensors are the same.
-        // If not, forward it to Eigen implementation.
-
-        OP_REQUIRES_OK(context, context->input_list("input_mins", &input_mins));
-        OP_REQUIRES(context, (input_mins.size() == N),
-                    errors::InvalidArgument(
-                        "QuantizedConcatOp : Expected mins input list length ",
-                        input_mins.size(), " to equal values length ", N));
-
-        OP_REQUIRES_OK(context,
-                       context->input_list("input_maxes", &input_maxes));
-        OP_REQUIRES(context, (input_maxes.size() == N),
-                    errors::InvalidArgument(
-                        "QuantizedConcatOp : Expected maxes input list length ",
-                        input_maxes.size(), " to equal values length ", N));
-        float input_min = input_mins[0].flat<float>()(0);
-        float input_max = input_maxes[0].flat<float>()(0);
-        const float eps = 1.0e-6;
-        for (int i = 1; i < N; ++i) {
-          float min = input_mins[i].flat<float>()(0);
-          float max = input_maxes[i].flat<float>()(0);
-
-          if (fabs(input_min - min) > eps || fabs(input_max - max) > eps) {
-            invoke_eigen = true;
-            break;
-          }
-        }
-      }
-
       // Call Eigen library
       if (invoke_eigen) {
-        // MKL-DNN quantized concat does not support input tensors with
-        // different ranges.
-        // TODO (mabuzain): Add quantized version of CallEigen() to support
-        // this case.
-        OP_REQUIRES(
-            context,
-            (!std::is_same<T, qint8>::value && !std::is_same<T, quint8>::value),
-            errors::Unimplemented("MKL DNN quantized concat does not "
-                                  "support input tensors that have "
-                                  "different ranges"));
         CallEigenVersion(context, input_tensors, mkl_input_shapes);
         return;
       }
@@ -418,23 +374,6 @@ class MklConcatOp : public OpKernel {
       std::vector<primitive> net;
       net.push_back(concat_op);
       stream(stream::kind::eager).submit(net).wait();
-
-      // For quantized concat, min and max outputs are also computed.
-      if (std::is_same<T, qint8>::value || std::is_same<T, quint8>::value) {
-        Tensor* output_min = nullptr;
-        Tensor* output_max = nullptr;
-        MklDnnShape output_min_mkl_shape, output_max_mkl_shape;
-        output_min_mkl_shape.SetMklTensor(false);
-        output_max_mkl_shape.SetMklTensor(false);
-        AllocateOutputSetMklShape(context, 1, &output_min, {},
-                                  output_min_mkl_shape);
-        AllocateOutputSetMklShape(context, 2, &output_max, {},
-                                  output_max_mkl_shape);
-        // All input tensors should have the same range, just use the
-        // first one
-        output_min->flat<float>()(0) = input_mins[0].flat<float>()(0);
-        output_max->flat<float>()(0) = input_maxes[0].flat<float>()(0);
-      }
     } catch (mkldnn::error& e) {
       string error_msg = "Status: " + std::to_string(e.status) +
                          ", message: " + string(e.message) + ", in file " +
@@ -551,20 +490,6 @@ class MklConcatOp : public OpKernel {
 
 TF_CALL_float(REGISTER_MKL_CPU);
 
-REGISTER_KERNEL_BUILDER(Name("_MklQuantizedConcatV2")
-                            .Device(DEVICE_CPU)
-                            .TypeConstraint<quint8>("T")
-                            .HostMemory("axis")
-                            .Label(mkl_op_registry::kMklQuantizedOpLabel),
-                        MklConcatOp<CPUDevice, quint8, NAME_IS_AXIS>)
-
-REGISTER_KERNEL_BUILDER(Name("_MklQuantizedConcatV2")
-                            .Device(DEVICE_CPU)
-                            .TypeConstraint<qint8>("T")
-                            .HostMemory("axis")
-                            .Label(mkl_op_registry::kMklQuantizedOpLabel),
-                        MklConcatOp<CPUDevice, qint8, NAME_IS_AXIS>)
-
 #undef REGISTER_CONCAT_MKL
 }  // namespace tensorflow
 
diff --git a/tensorflow/core/kernels/mkl_quantized_concat_op_test.cc b/tensorflow/core/kernels/mkl_quantized_concat_op_test.cc
deleted file mode 100644
index dcd1de95e0..0000000000
--- a/tensorflow/core/kernels/mkl_quantized_concat_op_test.cc
+++ /dev/null
@@ -1,230 +0,0 @@
-/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#define EIGEN_USE_THREADS
-
-#include <functional>
-#include <memory>
-#include <vector>
-
-#include "tensorflow/core/common_runtime/kernel_benchmark_testlib.h"
-#include "tensorflow/core/framework/allocator.h"
-#include "tensorflow/core/framework/fake_input.h"
-#include "tensorflow/core/framework/node_def_builder.h"
-#include "tensorflow/core/framework/op_kernel.h"
-#include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/framework/tensor_testutil.h"
-#include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/framework/types.pb.h"
-#include "tensorflow/core/graph/node_builder.h"
-#include "tensorflow/core/kernels/ops_testutil.h"
-#include "tensorflow/core/kernels/ops_util.h"
-#include "tensorflow/core/kernels/quantization_utils.h"
-#include "tensorflow/core/lib/core/status.h"
-#include "tensorflow/core/lib/core/status_test_util.h"
-#include "tensorflow/core/platform/test.h"
-#include "tensorflow/core/platform/test_benchmark.h"
-
-namespace tensorflow {
-
-using test::graph::Constant;
-
-static const uint8 dummy_tensor[] = {0, 0, 0, 0, 0, 0, 0, 0};
-static const TensorShape dummy_shape({8});
-
-// Helper class for converting MKL tensors to TF tensors and comparing to
-// expected values
-
-class ConvMklToTF : public OpsTestBase {
- public:
-  template <typename T>
-  void ConvertMKL2TF(DataType dtype, const Tensor& first, const Tensor& second,
-                     Tensor& output) {
-    // Create an MKL to TF conversion node and execute it
-    TF_EXPECT_OK(NodeDefBuilder("mkl_to_tf_op", "_MklToTf")
-                     .Input(FakeInput(dtype))     // Input
-                     .Input(FakeInput(DT_UINT8))  // MKL second tensor
-                     .Attr("T", dtype)
-                     .Attr("_kernel", "MklOp")
-                     .Finalize(node_def()));
-    TF_EXPECT_OK(InitOp());
-    AddInputFromArray<T>(first.shape(), first.flat<T>());
-    AddInputFromArray<uint8>(second.shape(), second.flat<uint8>());
-    TF_ASSERT_OK(RunOpKernel());
-
-    output = *GetOutput(0);
-  }
-  void TestBody(){};
-};
-
-class QuantizedConcatTest : public OpsTestBase {
- protected:
-  QuantizedConcatTest() {}
-
-  void TestSmall8Bit(float first_min, float first_max, float second_min,
-                     float second_max);
-  void TestSecondDim8Bit(float first_min, float first_max, float second_min,
-                         float second_max);
-};
-
-TEST_F(QuantizedConcatTest, Small8BitSameRange) {
-  // Range for both is the same, so impl can use memcpy.
-  TestSmall8Bit(0.0f, 255.0f, 0.0f, 255.0f);
-}
-
-void QuantizedConcatTest::TestSmall8Bit(float first_min, float first_max,
-                                        float second_min, float second_max) {
-  TF_ASSERT_OK(NodeDefBuilder("quantized_concat_op", "_MklQuantizedConcatV2")
-                   .Input(FakeInput(2, DT_QUINT8))
-                   .Input(FakeInput(DT_INT32))
-                   .Input(FakeInput(2, DT_FLOAT))
-                   .Input(FakeInput(2, DT_FLOAT))
-                   .Input(FakeInput(2, DT_UINT8))  // MKL second tensor
-                   .Input(FakeInput(DT_UINT8))     // MKL second tensor
-                   .Input(FakeInput(2, DT_UINT8))  // MKL second tensor
-                   .Input(FakeInput(2, DT_UINT8))  // MKL second tensor
-                   .Attr("N", 2)
-                   .Attr("T", DataTypeToEnum<quint8>::v())
-                   .Attr("Tidx", DT_INT32)
-                   .Attr("_kernel", "QuantizedMklOp")
-                   .Finalize(node_def()));
-  TF_ASSERT_OK(InitOp());
-  const int first_batch = 2;
-  const int first_height = 2;
-  const int first_width = 3;
-  const int first_depth = 1;
-  Tensor first_float(DT_FLOAT,
-                     {first_batch, first_height, first_width, first_depth});
-  test::FillValues<float>(&first_float,
-                          {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
-  Tensor first_quantized =
-      FloatTensorToQuantized<quint8>(first_float, first_min, first_max);
-
-  const int second_batch = 2;
-  const int second_height = 2;
-  const int second_width = 3;
-  const int second_depth = 1;
-  Tensor second_float(
-      DT_FLOAT, {second_batch, second_height, second_width, second_depth});
-  test::FillValues<float>(&second_float,
-                          {13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24});
-  Tensor second_quantized =
-      FloatTensorToQuantized<quint8>(second_float, second_min, second_max);
-
-  const int expected_batch = first_batch + second_batch;
-  Tensor expected_float(
-      DT_FLOAT, {expected_batch, first_height, first_width, first_depth});
-  test::FillValues<float>(&expected_float,
-                          {1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12,
-                           13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24});
-
-  AddInputFromArray<quint8>(first_quantized.shape(),
-                            first_quantized.flat<quint8>());
-  AddInputFromArray<quint8>(second_quantized.shape(),
-                            second_quantized.flat<quint8>());
-  AddInputFromArray<int32>(TensorShape({}), {0});
-  AddInputFromArray<float>(TensorShape({}), {first_min});
-  AddInputFromArray<float>(TensorShape({}), {second_min});
-  AddInputFromArray<float>(TensorShape({}), {first_max});
-  AddInputFromArray<float>(TensorShape({}), {second_max});
-  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
-  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
-  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
-  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
-  TF_ASSERT_OK(RunOpKernel());
-  const Tensor& output_quantized = *GetOutput(0);
-  const float output_min = GetOutput(1)->flat<float>()(0);
-  const float output_max = GetOutput(2)->flat<float>()(0);
-  Tensor output_float =
-      QuantizedTensorToFloat<quint8>(output_quantized, output_min, output_max);
-  test::ExpectTensorNear<float>(expected_float, output_float, 0.2);
-}
-
-TEST_F(QuantizedConcatTest, SecondDim8BitSameRange) {
-  TestSecondDim8Bit(-10.0f, 150.0f, -10.0f, 150.0f);
-}
-
-void QuantizedConcatTest::TestSecondDim8Bit(float first_min, float first_max,
-                                            float second_min,
-                                            float second_max) {
-  TF_ASSERT_OK(NodeDefBuilder("quantized_concat_op", "_MklQuantizedConcatV2")
-                   .Input(FakeInput(2, DT_QUINT8))
-                   .Input(FakeInput(DT_INT32))
-                   .Input(FakeInput(2, DT_FLOAT))
-                   .Input(FakeInput(2, DT_FLOAT))
-                   .Input(FakeInput(2, DT_UINT8))  // MKL second tensor
-                   .Input(FakeInput(DT_UINT8))     // MKL second tensor
-                   .Input(FakeInput(2, DT_UINT8))  // MKL second tensor
-                   .Input(FakeInput(2, DT_UINT8))  // MKL second tensor
-                   .Attr("N", 2)
-                   .Attr("T", DataTypeToEnum<quint8>::v())
-                   .Attr("Tidx", DT_INT32)
-                   .Attr("_kernel", "QuantizedMklOp")
-                   .Finalize(node_def()));
-  TF_ASSERT_OK(InitOp());
-  const int first_batch = 2;
-  const int first_height = 2;
-  const int first_width = 3;
-  const int first_depth = 1;
-  Tensor first_float(DT_FLOAT,
-                     {first_batch, first_height, first_width, first_depth});
-  test::FillValues<float>(&first_float,
-                          {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
-  Tensor first_quantized =
-      FloatTensorToQuantized<quint8>(first_float, first_min, first_max);
-
-  const int second_batch = 2;
-  const int second_height = 2;
-  const int second_width = 3;
-  const int second_depth = 1;
-
-  Tensor second_float(
-      DT_FLOAT, {second_batch, second_height, second_width, second_depth});
-  test::FillValues<float>(&second_float,
-                          {13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24});
-  Tensor second_quantized =
-      FloatTensorToQuantized<quint8>(second_float, second_min, second_max);
-
-  const int expected_height = first_height + second_height;
-  Tensor expected_float(
-      DT_FLOAT, {first_batch, expected_height, first_width, first_depth});
-  test::FillValues<float>(&expected_float,
-                          {1, 2, 3, 4,  5,  6,  13, 14, 15, 16, 17, 18,
-                           7, 8, 9, 10, 11, 12, 19, 20, 21, 22, 23, 24});
-
-  AddInputFromArray<quint8>(first_quantized.shape(),
-                            first_quantized.flat<quint8>());
-  AddInputFromArray<quint8>(second_quantized.shape(),
-                            second_quantized.flat<quint8>());
-  AddInputFromArray<int32>(TensorShape({}), {1});
-  AddInputFromArray<float>(TensorShape({}), {first_min});
-  AddInputFromArray<float>(TensorShape({}), {second_min});
-  AddInputFromArray<float>(TensorShape({}), {first_max});
-  AddInputFromArray<float>(TensorShape({}), {second_max});
-  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
-  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
-  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
-  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
-  TF_ASSERT_OK(RunOpKernel());
-  const Tensor& output_quantized = *GetOutput(0);
-  const float output_min = GetOutput(1)->flat<float>()(0);
-  const float output_max = GetOutput(2)->flat<float>()(0);
-  Tensor output_float =
-      QuantizedTensorToFloat<quint8>(output_quantized, output_min, output_max);
-  // Using the same error tolerance as in Eigen QuantizedConcat test
-  test::ExpectTensorNear<float>(expected_float, output_float, 1.0);
-}
-
-}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/quantized_concat_op.cc b/tensorflow/core/kernels/quantized_concat_op.cc
index f38791cce4..b03ac8e87d 100644
--- a/tensorflow/core/kernels/quantized_concat_op.cc
+++ b/tensorflow/core/kernels/quantized_concat_op.cc
@@ -246,14 +246,4 @@ REGISTER_QUANTIZED_CONCAT(qint32);
 
 #undef REGISTER_QUANTIZED_CONCAT
 
-#define REGISTER_QUANTIZED_CONCATV2(type)                \
-  REGISTER_KERNEL_BUILDER(Name("QuantizedConcatV2")      \
-                              .Device(DEVICE_CPU)        \
-                              .TypeConstraint<type>("T") \
-                              .HostMemory("axis"),       \
-                          QuantizedConcatOp<type>)
-
-REGISTER_QUANTIZED_CONCATV2(quint8);
-REGISTER_QUANTIZED_CONCATV2(qint32);
-
 }  // namespace tensorflow
diff --git a/tensorflow/core/ops/mkl_array_ops.cc b/tensorflow/core/ops/mkl_array_ops.cc
deleted file mode 100644
index e7ad3be611..0000000000
--- a/tensorflow/core/ops/mkl_array_ops.cc
+++ /dev/null
@@ -1,92 +0,0 @@
-/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifdef INTEL_MKL
-
-// This file contains the registration of MKL-DNN array ops.
-
-#include "tensorflow/core/framework/common_shape_fns.h"
-#include "tensorflow/core/framework/op.h"
-#include "tensorflow/core/framework/shape_inference.h"
-#include "tensorflow/core/framework/tensor.pb.h"
-#include "tensorflow/core/util/mirror_pad_mode.h"
-#include "tensorflow/core/util/padding.h"
-#include "tensorflow/core/util/strided_slice_op.h"
-#include "tensorflow/core/util/tensor_format.h"
-
-namespace tensorflow {
-
-using shape_inference::DimensionHandle;
-using shape_inference::InferenceContext;
-using shape_inference::ShapeHandle;
-using shape_inference::UnchangedShape;
-
-// Adding QuantizedConcatV2 op to be able to replace it by
-// _MklQuantizedConcatV2 in the graph rewrite.
-REGISTER_OP("QuantizedConcatV2")
-    .Input("values: N * T")
-    .Input("axis: Tidx")
-    .Input("input_mins: N * float32")
-    .Input("input_maxes: N * float32")
-    .Output("output: T")
-    .Output("output_min: float")
-    .Output("output_max: float")
-    .Attr("N: int >= 2")
-    .Attr("T: type")
-    .Attr("Tidx: {int32, int64} = DT_INT32")
-    .SetShapeFn([](InferenceContext* c) {
-      const int n = (c->num_inputs() - 1) / 3;
-      TF_RETURN_IF_ERROR(shape_inference::QuantizedConcatV2Shape(c, n));
-      ShapeHandle unused;
-      for (int i = n + 1; i < c->num_inputs(); ++i) {
-        TF_RETURN_IF_ERROR(c->WithRank(c->input(i), 0, &unused));
-      }
-      c->set_output(1, c->Scalar());
-      c->set_output(2, c->Scalar());
-      return Status::OK();
-    });
-
-REGISTER_OP("_MklQuantizedConcatV2")
-    .Input("values: N * T")
-    .Input("axis: Tidx")
-    .Input("input_mins:  N * float32")
-    .Input("input_maxes: N * float32")
-    .Input("mkl_values: N * uint8")
-    .Input("mkl_axis: uint8")
-    .Input("mkl_input_mins:  N * uint8")
-    .Input("mkl_input_maxes: N * uint8")
-    .Output("output: T")
-    .Output("output_min: float")
-    .Output("output_max: float")
-    .Output("mkl_output: uint8")
-    .Output("mkl_output_min: uint8")
-    .Output("mkl_output_max: uint8")
-    .Attr("N: int >= 2")
-    .Attr("T: type")
-    .Attr("Tidx: {int32, int64} = DT_INT32")
-    .SetShapeFn([](InferenceContext* c) {
-      const int n = (c->num_inputs() / 2 - 1) / 3;
-      TF_RETURN_IF_ERROR(shape_inference::QuantizedConcatV2Shape(c, n));
-      ShapeHandle unused;
-      for (int i = n + 1; i < c->num_inputs() / 2; ++i) {
-        TF_RETURN_IF_ERROR(c->WithRank(c->input(i), 0, &unused));
-      }
-      c->set_output(1, c->Scalar());
-      c->set_output(2, c->Scalar());
-      return Status::OK();
-    });
-}  // namespace tensorflow
-
-#endif
-- 
GitLab


From d42c0824c676c1e1bdec638944432f5ce4267713 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 5 Feb 2019 13:56:25 -0800
Subject: [PATCH 1946/2345] Remove unused inclusions and add missing inclusions

PiperOrigin-RevId: 232552217
---
 tensorflow/cc/saved_model/loader.cc                             | 1 -
 tensorflow/compiler/jit/encapsulate_util_test.cc                | 1 -
 tensorflow/compiler/xla/client/lib/matrix_test.cc               | 1 -
 tensorflow/compiler/xla/index_util.cc                           | 1 -
 tensorflow/compiler/xla/service/ar_crs_combiner.cc              | 1 -
 tensorflow/compiler/xla/service/cpu/cpu_hlo_support_checker.cc  | 1 -
 tensorflow/compiler/xla/service/cpu/cpu_transfer_manager.cc     | 1 -
 tensorflow/compiler/xla/service/generic_transfer_manager.cc     | 1 -
 tensorflow/compiler/xla/service/gpu/gpu_hlo_support_checker.cc  | 1 -
 tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc     | 1 -
 tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc        | 1 -
 tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc      | 1 -
 tensorflow/compiler/xla/service/gpu/parallel_loop_emitter.cc    | 1 -
 tensorflow/compiler/xla/service/gpu/tests/gpu_copy_test.cc      | 1 -
 tensorflow/compiler/xla/service/gpu/tests/infeed_test.cc        | 1 -
 tensorflow/compiler/xla/service/gpu/variadic_op_splitter.cc     | 1 -
 tensorflow/compiler/xla/service/hlo_cse.cc                      | 1 -
 .../compiler/xla/service/hlo_input_output_alias_config_test.cc  | 1 -
 tensorflow/compiler/xla/service/llvm_ir/llvm_loop.cc            | 1 -
 tensorflow/compiler/xla/service/llvm_ir/loop_emitter.cc         | 1 -
 tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc     | 1 -
 tensorflow/compiler/xla/tests/reduce_precision_test.cc          | 1 -
 tensorflow/compiler/xla/text_literal_writer.cc                  | 1 -
 .../compiler/xla/tools/dumped_computation_to_operation_list.cc  | 1 -
 tensorflow/compiler/xla/tools/dumped_computation_to_text.cc     | 1 -
 tensorflow/compiler/xla/tools/show_signature.cc                 | 1 -
 tensorflow/compiler/xrt/kernels/xrt_execute_op.cc               | 1 -
 tensorflow/compiler/xrt/xrt_state.cc                            | 1 -
 tensorflow/core/debug/debug_grpc_testlib.cc                     | 1 +
 tensorflow/core/debug/debug_io_utils.cc                         | 1 +
 tensorflow/core/debug/debug_io_utils_test.cc                    | 1 +
 .../core/grappler/costs/analytical_cost_estimator_test.cc       | 1 -
 .../core/grappler/optimizers/data/shuffle_and_repeat_fusion.cc  | 1 -
 .../optimizers/data/vectorization/vectorizer_registry_test.cc   | 1 -
 tensorflow/core/kernels/data/experimental/stats_dataset_ops.cc  | 2 --
 tensorflow/core/profiler/lib/profiler_session.cc                | 1 -
 .../lite/tools/accuracy/ilsvrc/inception_preprocessing.cc       | 1 -
 37 files changed, 3 insertions(+), 35 deletions(-)

diff --git a/tensorflow/cc/saved_model/loader.cc b/tensorflow/cc/saved_model/loader.cc
index 10f7abf09e..66260fcf4a 100644
--- a/tensorflow/cc/saved_model/loader.cc
+++ b/tensorflow/cc/saved_model/loader.cc
@@ -26,7 +26,6 @@ limitations under the License.
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/protobuf_internal.h"
-#include "tensorflow/core/protobuf/saved_model.pb.h"
 #include "tensorflow/core/protobuf/saver.pb.h"
 #include "tensorflow/core/public/session.h"
 #include "tensorflow/core/public/session_options.h"
diff --git a/tensorflow/compiler/jit/encapsulate_util_test.cc b/tensorflow/compiler/jit/encapsulate_util_test.cc
index 3bb979e069..6d1661222e 100644
--- a/tensorflow/compiler/jit/encapsulate_util_test.cc
+++ b/tensorflow/compiler/jit/encapsulate_util_test.cc
@@ -21,7 +21,6 @@ limitations under the License.
 #include "tensorflow/core/framework/node_def_util.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_shape.pb.h"
-#include "tensorflow/core/framework/types.pb.h"
 #include "tensorflow/core/platform/test.h"
 
 namespace tensorflow {
diff --git a/tensorflow/compiler/xla/client/lib/matrix_test.cc b/tensorflow/compiler/xla/client/lib/matrix_test.cc
index 79cf529ee9..61f91a56fb 100644
--- a/tensorflow/compiler/xla/client/lib/matrix_test.cc
+++ b/tensorflow/compiler/xla/client/lib/matrix_test.cc
@@ -24,7 +24,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/tests/client_library_test_base.h"
 #include "tensorflow/compiler/xla/tests/test_macros.h"
 #include "tensorflow/compiler/xla/types.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
 
 namespace xla {
 namespace {
diff --git a/tensorflow/compiler/xla/index_util.cc b/tensorflow/compiler/xla/index_util.cc
index 7e22a32e54..eebd8245ab 100644
--- a/tensorflow/compiler/xla/index_util.cc
+++ b/tensorflow/compiler/xla/index_util.cc
@@ -21,7 +21,6 @@ limitations under the License.
 #include "absl/strings/str_join.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/types.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/platform/logging.h"
 
 namespace xla {
diff --git a/tensorflow/compiler/xla/service/ar_crs_combiner.cc b/tensorflow/compiler/xla/service/ar_crs_combiner.cc
index f8dff6a700..99373dc107 100644
--- a/tensorflow/compiler/xla/service/ar_crs_combiner.cc
+++ b/tensorflow/compiler/xla/service/ar_crs_combiner.cc
@@ -29,7 +29,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/types.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
 
 namespace xla {
 
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_hlo_support_checker.cc b/tensorflow/compiler/xla/service/cpu/cpu_hlo_support_checker.cc
index 7fbe0fa157..4ac61f44d9 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_hlo_support_checker.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_hlo_support_checker.cc
@@ -17,7 +17,6 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/layout_util.h"
 #include "tensorflow/compiler/xla/shape_util.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/lib/core/errors.h"
 
 namespace xla {
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_transfer_manager.cc b/tensorflow/compiler/xla/service/cpu/cpu_transfer_manager.cc
index 3361a5973f..fae9670051 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_transfer_manager.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_transfer_manager.cc
@@ -29,7 +29,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/util.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/gtl/cleanup.h"
 #include "tensorflow/core/platform/logging.h"
diff --git a/tensorflow/compiler/xla/service/generic_transfer_manager.cc b/tensorflow/compiler/xla/service/generic_transfer_manager.cc
index 7d450f4b53..cb43c27be9 100644
--- a/tensorflow/compiler/xla/service/generic_transfer_manager.cc
+++ b/tensorflow/compiler/xla/service/generic_transfer_manager.cc
@@ -26,7 +26,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/util.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/stream_executor_no_cuda.h"
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_hlo_support_checker.cc b/tensorflow/compiler/xla/service/gpu/gpu_hlo_support_checker.cc
index 4268fb2c7a..4765f67c4b 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_hlo_support_checker.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_hlo_support_checker.cc
@@ -17,7 +17,6 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/layout_util.h"
 #include "tensorflow/compiler/xla/shape_util.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/lib/core/errors.h"
 
 namespace xla {
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc b/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc
index 8c6a691479..e593f53564 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc
@@ -30,7 +30,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/util.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/gtl/cleanup.h"
 #include "tensorflow/core/platform/logging.h"
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc b/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc
index 82bdd677d9..3ed6553f92 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc
@@ -20,7 +20,6 @@ limitations under the License.
 
 #include "llvm/IR/Module.h"
 #include "tensorflow/compiler/xla/layout_util.h"
-#include "tensorflow/compiler/xla/service/gpu/backend_configs.pb.h"
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index 294a454931..4fab95920d 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -38,7 +38,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/service/buffer_assignment.h"
 #include "tensorflow/compiler/xla/service/dfs_hlo_visitor.h"
-#include "tensorflow/compiler/xla/service/gpu/backend_configs.pb.h"
 #include "tensorflow/compiler/xla/service/gpu/buffer_allocations.h"
 #include "tensorflow/compiler/xla/service/gpu/conditional_thunk.h"
 #include "tensorflow/compiler/xla/service/gpu/convolution_thunk.h"
diff --git a/tensorflow/compiler/xla/service/gpu/parallel_loop_emitter.cc b/tensorflow/compiler/xla/service/gpu/parallel_loop_emitter.cc
index 8154d75d23..cb01264920 100644
--- a/tensorflow/compiler/xla/service/gpu/parallel_loop_emitter.cc
+++ b/tensorflow/compiler/xla/service/gpu/parallel_loop_emitter.cc
@@ -25,7 +25,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/llvm_ir/llvm_loop.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h"
 #include "tensorflow/compiler/xla/shape_util.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
 
 namespace xla {
 namespace gpu {
diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_copy_test.cc b/tensorflow/compiler/xla/service/gpu/tests/gpu_copy_test.cc
index a1ed849904..d33e9cf714 100644
--- a/tensorflow/compiler/xla/service/gpu/tests/gpu_copy_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_copy_test.cc
@@ -24,7 +24,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/platform/test.h"
 
 namespace xla {
diff --git a/tensorflow/compiler/xla/service/gpu/tests/infeed_test.cc b/tensorflow/compiler/xla/service/gpu/tests/infeed_test.cc
index f91a22d482..06b06a5b1e 100644
--- a/tensorflow/compiler/xla/service/gpu/tests/infeed_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/tests/infeed_test.cc
@@ -25,7 +25,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/test_helpers.h"
 #include "tensorflow/compiler/xla/tests/client_library_test_base.h"
 #include "tensorflow/compiler/xla/tests/literal_test_util.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/lib/math/math_util.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/types.h"
diff --git a/tensorflow/compiler/xla/service/gpu/variadic_op_splitter.cc b/tensorflow/compiler/xla/service/gpu/variadic_op_splitter.cc
index c552c29254..bbbcc2dbb0 100644
--- a/tensorflow/compiler/xla/service/gpu/variadic_op_splitter.cc
+++ b/tensorflow/compiler/xla/service/gpu/variadic_op_splitter.cc
@@ -23,7 +23,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/hlo_module.h"
 #include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/util.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/lib/core/errors.h"
 
 namespace xla {
diff --git a/tensorflow/compiler/xla/service/hlo_cse.cc b/tensorflow/compiler/xla/service/hlo_cse.cc
index e602107cbe..849cac278e 100644
--- a/tensorflow/compiler/xla/service/hlo_cse.cc
+++ b/tensorflow/compiler/xla/service/hlo_cse.cc
@@ -33,7 +33,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/types.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/hash/hash.h"
 
diff --git a/tensorflow/compiler/xla/service/hlo_input_output_alias_config_test.cc b/tensorflow/compiler/xla/service/hlo_input_output_alias_config_test.cc
index a46a107723..265bfdf7f9 100644
--- a/tensorflow/compiler/xla/service/hlo_input_output_alias_config_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_input_output_alias_config_test.cc
@@ -29,7 +29,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/tests/hlo_test_base.h"
 #include "tensorflow/compiler/xla/types.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 
 namespace xla {
diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_loop.cc b/tensorflow/compiler/xla/service/llvm_ir/llvm_loop.cc
index fe320bbe72..3a35405a2d 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/llvm_loop.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_loop.cc
@@ -25,7 +25,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/types.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/platform/logging.h"
 
 namespace xla {
diff --git a/tensorflow/compiler/xla/service/llvm_ir/loop_emitter.cc b/tensorflow/compiler/xla/service/llvm_ir/loop_emitter.cc
index 0dc120e0b0..a689881e65 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/loop_emitter.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/loop_emitter.cc
@@ -23,7 +23,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/types.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/protobuf.h"
diff --git a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
index 7379fbcc22..acdd3c9da9 100644
--- a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
+++ b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
@@ -35,7 +35,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/tests/literal_test_util.h"
 #include "tensorflow/compiler/xla/tests/test_macros.h"
 #include "tensorflow/compiler/xla/types.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace xla {
diff --git a/tensorflow/compiler/xla/tests/reduce_precision_test.cc b/tensorflow/compiler/xla/tests/reduce_precision_test.cc
index f80d29b9de..e2cf4c0be2 100644
--- a/tensorflow/compiler/xla/tests/reduce_precision_test.cc
+++ b/tensorflow/compiler/xla/tests/reduce_precision_test.cc
@@ -34,7 +34,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/tests/literal_test_util.h"
 #include "tensorflow/compiler/xla/tests/test_macros.h"
 #include "tensorflow/compiler/xla/types.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace xla {
diff --git a/tensorflow/compiler/xla/text_literal_writer.cc b/tensorflow/compiler/xla/text_literal_writer.cc
index 7289ae7df6..fc7949d889 100644
--- a/tensorflow/compiler/xla/text_literal_writer.cc
+++ b/tensorflow/compiler/xla/text_literal_writer.cc
@@ -24,7 +24,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/types.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/types.h"
 
diff --git a/tensorflow/compiler/xla/tools/dumped_computation_to_operation_list.cc b/tensorflow/compiler/xla/tools/dumped_computation_to_operation_list.cc
index 4375e7c138..df2d3d18b9 100644
--- a/tensorflow/compiler/xla/tools/dumped_computation_to_operation_list.cc
+++ b/tensorflow/compiler/xla/tools/dumped_computation_to_operation_list.cc
@@ -31,7 +31,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/service.h"
 #include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/types.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/platform/logging.h"
diff --git a/tensorflow/compiler/xla/tools/dumped_computation_to_text.cc b/tensorflow/compiler/xla/tools/dumped_computation_to_text.cc
index 723569862c..35bb82ca22 100644
--- a/tensorflow/compiler/xla/tools/dumped_computation_to_text.cc
+++ b/tensorflow/compiler/xla/tools/dumped_computation_to_text.cc
@@ -26,7 +26,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/service.h"
 #include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/types.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/platform/logging.h"
diff --git a/tensorflow/compiler/xla/tools/show_signature.cc b/tensorflow/compiler/xla/tools/show_signature.cc
index cdf306dfd1..b80d0db8d8 100644
--- a/tensorflow/compiler/xla/tools/show_signature.cc
+++ b/tensorflow/compiler/xla/tools/show_signature.cc
@@ -37,7 +37,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/types.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/platform/logging.h"
diff --git a/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc b/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc
index 116c193cab..42ef88168a 100644
--- a/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc
+++ b/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc
@@ -23,7 +23,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/statusor.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/compiler/xrt/xrt.pb.h"
 #include "tensorflow/compiler/xrt/xrt_compilation_cache.h"
 #include "tensorflow/compiler/xrt/xrt_device.h"
diff --git a/tensorflow/compiler/xrt/xrt_state.cc b/tensorflow/compiler/xrt/xrt_state.cc
index 1e2a9584f8..78a1b6afc0 100644
--- a/tensorflow/compiler/xrt/xrt_state.cc
+++ b/tensorflow/compiler/xrt/xrt_state.cc
@@ -31,7 +31,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/statusor.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/random/random.h"
diff --git a/tensorflow/core/debug/debug_grpc_testlib.cc b/tensorflow/core/debug/debug_grpc_testlib.cc
index f70931e926..4927caf5a3 100644
--- a/tensorflow/core/debug/debug_grpc_testlib.cc
+++ b/tensorflow/core/debug/debug_grpc_testlib.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include "tensorflow/core/debug/debug_graph_utils.h"
 #include "tensorflow/core/debug/debugger_event_metadata.pb.h"
 #include "tensorflow/core/framework/summary.pb.h"
+#include "tensorflow/core/framework/tensor.pb.h"
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/env.h"
diff --git a/tensorflow/core/debug/debug_io_utils.cc b/tensorflow/core/debug/debug_io_utils.cc
index 527c935380..ebcb046003 100644
--- a/tensorflow/core/debug/debug_io_utils.cc
+++ b/tensorflow/core/debug/debug_io_utils.cc
@@ -35,6 +35,7 @@ limitations under the License.
 #include "tensorflow/core/debug/debugger_event_metadata.pb.h"
 #include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/framework/summary.pb.h"
+#include "tensorflow/core/framework/tensor.pb.h"
 #include "tensorflow/core/framework/tensor_shape.pb.h"
 #include "tensorflow/core/lib/core/bits.h"
 #include "tensorflow/core/lib/hash/hash.h"
diff --git a/tensorflow/core/debug/debug_io_utils_test.cc b/tensorflow/core/debug/debug_io_utils_test.cc
index 82e0ae5edb..0926a82fad 100644
--- a/tensorflow/core/debug/debug_io_utils_test.cc
+++ b/tensorflow/core/debug/debug_io_utils_test.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include "tensorflow/core/debug/debug_node_key.h"
 #include "tensorflow/core/debug/debugger_event_metadata.pb.h"
 #include "tensorflow/core/framework/summary.pb.h"
+#include "tensorflow/core/framework/tensor.pb.h"
 #include "tensorflow/core/framework/tensor_testutil.h"
 #include "tensorflow/core/lib/core/notification.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
diff --git a/tensorflow/core/grappler/costs/analytical_cost_estimator_test.cc b/tensorflow/core/grappler/costs/analytical_cost_estimator_test.cc
index 2c319b6c6b..fdc6b79c82 100644
--- a/tensorflow/core/grappler/costs/analytical_cost_estimator_test.cc
+++ b/tensorflow/core/grappler/costs/analytical_cost_estimator_test.cc
@@ -16,7 +16,6 @@ limitations under the License.
 #include "tensorflow/core/grappler/costs/virtual_scheduler.h"
 
 #include "tensorflow/cc/ops/standard_ops.h"
-#include "tensorflow/core/framework/cost_graph.pb.h"
 #include "tensorflow/core/grappler/clusters/virtual_cluster.h"
 #include "tensorflow/core/grappler/costs/analytical_cost_estimator.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
diff --git a/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.cc b/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.cc
index ff64ff1adb..0563460b29 100644
--- a/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.cc
@@ -16,7 +16,6 @@ limitations under the License.
 #include "tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.h"
 
 #include "absl/container/flat_hash_set.h"
-#include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/grappler/clusters/cluster.h"
 #include "tensorflow/core/grappler/grappler_item.h"
diff --git a/tensorflow/core/grappler/optimizers/data/vectorization/vectorizer_registry_test.cc b/tensorflow/core/grappler/optimizers/data/vectorization/vectorizer_registry_test.cc
index 0eee91f241..0f34d2b7eb 100644
--- a/tensorflow/core/grappler/optimizers/data/vectorization/vectorizer_registry_test.cc
+++ b/tensorflow/core/grappler/optimizers/data/vectorization/vectorizer_registry_test.cc
@@ -14,7 +14,6 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/grappler/optimizers/data/vectorization/vectorizer_registry.h"
-#include "tensorflow/core/framework/function.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/platform/test.h"
 
diff --git a/tensorflow/core/kernels/data/experimental/stats_dataset_ops.cc b/tensorflow/core/kernels/data/experimental/stats_dataset_ops.cc
index bf96be4eb0..be5fa4c789 100644
--- a/tensorflow/core/kernels/data/experimental/stats_dataset_ops.cc
+++ b/tensorflow/core/kernels/data/experimental/stats_dataset_ops.cc
@@ -12,8 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tensorflow/core/example/example.pb.h"
-#include "tensorflow/core/example/feature.pb.h"
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/stats_aggregator.h"
diff --git a/tensorflow/core/profiler/lib/profiler_session.cc b/tensorflow/core/profiler/lib/profiler_session.cc
index 31070b76c7..86b1e69bed 100644
--- a/tensorflow/core/profiler/lib/profiler_session.cc
+++ b/tensorflow/core/profiler/lib/profiler_session.cc
@@ -17,7 +17,6 @@ limitations under the License.
 #include <string>
 #include "tensorflow/contrib/tpu/profiler/trace_events.pb.h"
 #include "tensorflow/core/common_runtime/eager/context.h"
-#include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/platform/types.h"
diff --git a/tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing.cc b/tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing.cc
index 04b6cb7558..b730b0804e 100644
--- a/tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing.cc
+++ b/tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing.cc
@@ -19,7 +19,6 @@ limitations under the License.
 
 #include "tensorflow/cc/framework/scope.h"
 #include "tensorflow/cc/ops/standard_ops.h"
-#include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/graph/graph_def_builder.h"
 #include "tensorflow/core/lib/core/errors.h"
-- 
GitLab


From db57de1f1bc5948acc107f28427fa9a2cd20292b Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Tue, 5 Feb 2019 14:01:17 -0800
Subject: [PATCH 1947/2345] [XLA:Python] Fix breakage in replicated computation
 launches.

PiperOrigin-RevId: 232553215
---
 tensorflow/compiler/xla/python/xla_client.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/python/xla_client.py b/tensorflow/compiler/xla/python/xla_client.py
index 34b5079722..43eeebc6d8 100644
--- a/tensorflow/compiler/xla/python/xla_client.py
+++ b/tensorflow/compiler/xla/python/xla_client.py
@@ -728,7 +728,8 @@ class LocalComputation(object):
     ]
 
     # Execute
-    output_buffers = self._backend.execute_replicated(stripped_args)
+    output_buffers = self._backend.execute_replicated(
+        self._c_computation, stripped_args)
 
     # Wrap output handles in LocalBuffer instances
     return tuple(
-- 
GitLab


From 234d42b5d4fd7eeab820c4a9457f320845b17e1c Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Tue, 5 Feb 2019 14:01:24 -0800
Subject: [PATCH 1948/2345] Fix a typo in comment

PiperOrigin-RevId: 232553248
---
 tensorflow/lite/kernels/if_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/kernels/if_test.cc b/tensorflow/lite/kernels/if_test.cc
index 80e4ef133a..6593a58a76 100644
--- a/tensorflow/lite/kernels/if_test.cc
+++ b/tensorflow/lite/kernels/if_test.cc
@@ -26,7 +26,7 @@ namespace builtin {
 // ADD and MUL are used to test simple branch.
 TfLiteRegistration* Register_ADD();
 TfLiteRegistration* Register_MUL();
-// ADD and MUL are used to test dynamic sized subgraphs.
+// Pad is used to test dynamic sized subgraphs.
 TfLiteRegistration* Register_PAD();
 }  // namespace builtin
 namespace custom {
-- 
GitLab


From f77ec5d5ee3eba39f79fb768d9d5b3e3788b7edb Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Tue, 5 Feb 2019 14:05:59 -0800
Subject: [PATCH 1949/2345] [XLA] Fix HloElementTypeConverter's handling of
 bitcast convert.

Previously, if you asked HloElementTypeConverter to e.g. convert BF16 to F32,
it would convert a BF16 input to bitcast-convert to F32.

This don't work, because the output of the bitcast-convert still has width 16
bits, and the input/output widths must match.

The right thing is to ignore bitcast-convert operations for the purposes of
this pass.

PiperOrigin-RevId: 232554328
---
 .../xla/service/hlo_element_type_converter.cc      |  1 +
 .../xla/service/hlo_element_type_converter_test.cc | 14 ++++++++++++++
 2 files changed, 15 insertions(+)

diff --git a/tensorflow/compiler/xla/service/hlo_element_type_converter.cc b/tensorflow/compiler/xla/service/hlo_element_type_converter.cc
index 9b0f2b2a0f..812db465c6 100644
--- a/tensorflow/compiler/xla/service/hlo_element_type_converter.cc
+++ b/tensorflow/compiler/xla/service/hlo_element_type_converter.cc
@@ -127,6 +127,7 @@ StatusOr<bool> HloElementTypeConverter::Run(HloModule* module) {
       // These are ops where it does not make sense to convert them.
       if (opcode == HloOpcode::kParameter || opcode == HloOpcode::kConstant ||
           opcode == HloOpcode::kTuple || opcode == HloOpcode::kConvert ||
+          opcode == HloOpcode::kBitcastConvert ||
           opcode == HloOpcode::kGetTupleElement ||
           opcode == HloOpcode::kInfeed || opcode == HloOpcode::kOutfeed) {
         continue;
diff --git a/tensorflow/compiler/xla/service/hlo_element_type_converter_test.cc b/tensorflow/compiler/xla/service/hlo_element_type_converter_test.cc
index 5b633784e2..4171f73862 100644
--- a/tensorflow/compiler/xla/service/hlo_element_type_converter_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_element_type_converter_test.cc
@@ -176,5 +176,19 @@ ENTRY main {
   EXPECT_THAT(rng1->control_predecessors(), ElementsAre(rng0));
 }
 
+TEST_F(HloElementTypeConverterTest, BitcastConvertIsUnmodified) {
+  const string& hlo_string = R"(
+  HloModule test
+
+  ENTRY test {
+    p = bf16[] parameter(0)
+    ROOT c = u16[] bitcast-convert(p)
+  })";
+  auto module = ParseAndReturnVerifiedModule(hlo_string).ValueOrDie();
+  HloElementTypeConverter converter(BF16, F32);
+  TF_ASSERT_OK_AND_ASSIGN(bool converted, RunHloPass(&converter, module.get()));
+  EXPECT_FALSE(converted);
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 5e0a1a375a795ceb0b8fac65b0a04bfdb124986e Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Tue, 5 Feb 2019 14:07:57 -0800
Subject: [PATCH 1950/2345] TensorSpec equality is more useful if it's
 shape-list-equality and not requires that the shapes are equal.

PiperOrigin-RevId: 232554706
---
 tensorflow/python/eager/def_function_test.py | 23 +++++++++++++++-----
 tensorflow/python/framework/tensor_spec.py   |  4 +++-
 tensorflow/python/ops/tensor_array_ops.py    |  2 +-
 3 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/tensorflow/python/eager/def_function_test.py b/tensorflow/python/eager/def_function_test.py
index 1082c90d1b..b49b1656fc 100644
--- a/tensorflow/python/eager/def_function_test.py
+++ b/tensorflow/python/eager/def_function_test.py
@@ -268,7 +268,8 @@ class DefFunctionTest(test.TestCase):
     self.assertAllClose(4., concrete(constant_op.constant(2.)))
     signature_args, _ = concrete.structured_input_signature
     self.assertEqual(signature_args,
-                     (tensor_spec.TensorSpec(None, dtypes.float32),))
+                     (tensor_spec.TensorSpec(
+                         None, dtypes.float32, name='x'),))
 
   def test_serialization_signature_cache(self):
 
@@ -288,10 +289,10 @@ class DefFunctionTest(test.TestCase):
 
     self.assertEqual(
         signatures_args,
-        set(((tensor_spec.TensorSpec([1, 2], dtypes.float32),
-              tensor_spec.TensorSpec([1], dtypes.float32)),
-             (tensor_spec.TensorSpec([1, 3], dtypes.int32),
-              tensor_spec.TensorSpec([1], dtypes.int32)))))
+        set(((tensor_spec.TensorSpec([1, 2], dtypes.float32, name='x'),
+              tensor_spec.TensorSpec([1], dtypes.float32, name='y')),
+             (tensor_spec.TensorSpec([1, 3], dtypes.int32, name='x'),
+              tensor_spec.TensorSpec([1], dtypes.int32, name='y')))))
 
   @test_util.assert_no_garbage_created
   def testFunctionReferenceCycles(self):
@@ -373,6 +374,18 @@ class DefFunctionTest(test.TestCase):
 
     self.assertAllEqual(add(v, v), 2.0)
 
+  def testShapeCache(self):
+    @def_function.function
+    def func(x):
+      return 2 * x
+
+    func_a = func.get_concrete_function(
+        tensor_spec.TensorSpec([None], dtypes.int32))
+    func_b = func.get_concrete_function(
+        tensor_spec.TensorSpec([None], dtypes.int32))
+
+    self.assertIs(func_a, func_b)
+
   def testInitializationInNestedCall(self):
     v_holder = []
 
diff --git a/tensorflow/python/framework/tensor_spec.py b/tensorflow/python/framework/tensor_spec.py
index c44636edc4..2e847c7a35 100644
--- a/tensorflow/python/framework/tensor_spec.py
+++ b/tensorflow/python/framework/tensor_spec.py
@@ -108,7 +108,9 @@ class TensorSpec(object):
     return hash((self._shape_tuple, self.dtype))
 
   def __eq__(self, other):
-    return self.shape == other.shape and self.dtype == other.dtype
+    return (self._shape_tuple == other._shape_tuple  # pylint: disable=protected-access
+            and self.dtype == other.dtype
+            and self._name == other._name)  # pylint: disable=protected-access
 
   def __ne__(self, other):
     return not self == other
diff --git a/tensorflow/python/ops/tensor_array_ops.py b/tensorflow/python/ops/tensor_array_ops.py
index 3c0f98466b..1a11c332ea 100644
--- a/tensorflow/python/ops/tensor_array_ops.py
+++ b/tensorflow/python/ops/tensor_array_ops.py
@@ -825,7 +825,7 @@ class _EagerTensorArray(object):
     if self._infer_shape:
       if self._element_shape is None:
         self._element_shape = value.shape
-      elif self._element_shape != value.shape:
+      elif not self._element_shape.is_compatible_with(value.shape):
         raise ValueError("Incompatible shape for value (%s), expected (%s)" %
                          (value.shape.as_list(), self._element_shape.as_list()))
 
-- 
GitLab


From be38ca08c57632d082c79aff0f6449ce4468574d Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Tue, 5 Feb 2019 14:13:14 -0800
Subject: [PATCH 1951/2345] [XLA] Remove some unused friend declarations from
 XlaBuilder.

The free xla::Constant* functions all call the free xla::ConstantLiteral function, not XlaBuilder itself.

PiperOrigin-RevId: 232555768
---
 tensorflow/compiler/xla/client/xla_builder.h | 42 --------------------
 1 file changed, 42 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_builder.h b/tensorflow/compiler/xla/client/xla_builder.h
index fa742891e6..ea4dc18c2d 100644
--- a/tensorflow/compiler/xla/client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_builder.h
@@ -692,48 +692,6 @@ class XlaBuilder {
                          const Shape& shape, const string& name);
   friend XlaOp ConstantLiteral(XlaBuilder* builder,
                                const LiteralSlice& literal);
-  template <typename NativeT>
-  friend XlaOp ConstantR0(XlaBuilder* builder, NativeT value);
-  template <typename NativeT>
-  friend XlaOp ConstantR1(XlaBuilder* builder,
-                          absl::Span<const NativeT> values);
-  friend XlaOp ConstantR1(XlaBuilder* builder,
-                          const tensorflow::core::Bitmap& values);
-  template <typename NativeT>
-  friend XlaOp ConstantR2(
-      XlaBuilder* builder,
-      std::initializer_list<std::initializer_list<NativeT>> values);
-  template <typename NativeT>
-  friend XlaOp ConstantFromArrayWithLayout(XlaBuilder* builder,
-                                           const Array<NativeT>& values,
-                                           const Layout& layout);
-  template <typename NativeT>
-  friend XlaOp ConstantFromArray(XlaBuilder* builder,
-                                 const Array<NativeT>& values);
-  template <typename NativeT>
-  friend XlaOp ConstantR2FromArray2DWithLayout(XlaBuilder* builder,
-                                               const Array2D<NativeT>& values,
-                                               const Layout& layout);
-  template <typename NativeT>
-  friend XlaOp ConstantR2FromArray2D(XlaBuilder* builder,
-                                     const Array2D<NativeT>& values);
-  template <typename NativeT>
-  friend XlaOp ConstantR3FromArray3DWithLayout(XlaBuilder* builder,
-                                               const Array3D<NativeT>& values,
-                                               const Layout& layout);
-  template <typename NativeT>
-  friend XlaOp ConstantR3FromArray3D(XlaBuilder* builder,
-                                     const Array3D<NativeT>& values);
-  template <typename NativeT>
-  friend XlaOp ConstantR4FromArray4DWithLayout(XlaBuilder* builder,
-                                               const Array4D<NativeT>& values,
-                                               const Layout& layout);
-  template <typename NativeT>
-  friend XlaOp ConstantR4FromArray4D(XlaBuilder* builder,
-                                     const Array4D<NativeT>& values);
-
-  template <typename NativeT>
-  friend XlaOp ConstantR1(XlaBuilder* builder, int64 length, NativeT value);
 
   friend XlaOp Broadcast(const XlaOp& operand,
                          absl::Span<const int64> broadcast_sizes);
-- 
GitLab


From 3b1c4b4bb3a86a32de3c692a05b5db11501b8c5f Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Tue, 5 Feb 2019 14:22:11 -0800
Subject: [PATCH 1952/2345] [XLA] Add IsNegZero function.

PiperOrigin-RevId: 232557517
---
 tensorflow/compiler/xla/client/lib/math.cc    | 28 +++++++++++++++++++
 tensorflow/compiler/xla/client/lib/math.h     |  5 ++++
 .../compiler/xla/client/lib/math_test.cc      | 17 +++++++++++
 3 files changed, 50 insertions(+)

diff --git a/tensorflow/compiler/xla/client/lib/math.cc b/tensorflow/compiler/xla/client/lib/math.cc
index 1489120685..19d98d1001 100644
--- a/tensorflow/compiler/xla/client/lib/math.cc
+++ b/tensorflow/compiler/xla/client/lib/math.cc
@@ -79,6 +79,34 @@ XlaOp IsNan(XlaOp operand) {
   });
 }
 
+XlaOp IsNegZero(XlaOp operand) {
+  auto& b = *operand.builder();
+  return b.ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    TF_RETURN_IF_ERROR(EnsureOperandIsRealFp("IsNegZero", operand));
+    TF_ASSIGN_OR_RETURN(auto shape, b.GetShape(operand));
+
+    // The bitwise representation of -0 in bfloat16 and IEEE 754 is 0x80...0
+    // (sign bit on, all other bits off).
+    switch (shape.element_type()) {
+      case F64:
+        return Eq(BitcastConvertType(operand, U64),
+                  ConstantR0WithType(&b, U64, uint64{1} << 63));
+      case F32:
+        return Eq(BitcastConvertType(operand, U32),
+                  ConstantR0WithType(&b, U32, uint32{1} << 31));
+      case F16:
+      case BF16:
+        // Not all XLA backends handle U16 well, so we convert to F32/U32.
+        // TODO(jlebar): It would be nice if we could stay in (B)F16/U16 for
+        // backends that *do* support it.
+        return Eq(BitcastConvertType(ConvertElementType(operand, F32), U32),
+                  ConstantR0WithType(&b, U32, uint32{1} << 31));
+      default:
+        LOG(FATAL) << "Expected real fp type.";
+    }
+  });
+}
+
 XlaOp Sqrt(XlaOp operand) { return Pow(operand, ScalarLike(operand, 0.5)); }
 
 XlaOp Rsqrt(XlaOp operand) { return Pow(operand, ScalarLike(operand, -0.5)); }
diff --git a/tensorflow/compiler/xla/client/lib/math.h b/tensorflow/compiler/xla/client/lib/math.h
index 907571c9a3..b036fa299d 100644
--- a/tensorflow/compiler/xla/client/lib/math.h
+++ b/tensorflow/compiler/xla/client/lib/math.h
@@ -28,6 +28,11 @@ XlaOp IsNegInf(XlaOp operand);
 XlaOp IsInf(XlaOp operand);
 XlaOp IsNan(XlaOp operand);
 
+// Determines whether operand is equal to -0.
+//
+// Raises an error for integral or complex values.
+XlaOp IsNegZero(XlaOp operand);
+
 // Returns the next number after 'from' in the direction of 'to' the same way
 // std::nextafter(from, to) would.
 XlaOp NextAfter(XlaOp from, XlaOp to);
diff --git a/tensorflow/compiler/xla/client/lib/math_test.cc b/tensorflow/compiler/xla/client/lib/math_test.cc
index 364ac5876a..bdfb0575f5 100644
--- a/tensorflow/compiler/xla/client/lib/math_test.cc
+++ b/tensorflow/compiler/xla/client/lib/math_test.cc
@@ -88,6 +88,22 @@ class MathTypedTest : public MathTest {
             {false, false, false, false, false, false, false, true, true}));
     ComputeAndCompareLiteral(&b, expected, {});
   }
+
+  void TestIsNegZero() {
+    SetFastMathDisabled(true);
+    XlaBuilder b(TestName());
+    T inf(std::numeric_limits<float>::infinity());
+    T nan(std::numeric_limits<float>::quiet_NaN());
+    IsNegZero(AddParam(
+        LiteralUtil::CreateR1<T>({T{-0.0}, T{0}, T{1}, T{-1}, inf, -inf, nan}),
+        &b));
+
+    ComputeAndCompareLiteral(
+        &b,
+        LiteralUtil::CreateR1<bool>(
+            {true, false, false, false, false, false, false}),
+        {}, error_spec_);
+  }
 };
 
 // TODO(b/123355973): Add bfloat16 to TestTypes once it's working.
@@ -102,6 +118,7 @@ TYPED_TEST_CASE(MathTypedTest, TestTypes);
 XLA_TYPED_TEST(MathTypedTest, LogEdgeCases) { this->TestLogEdgeCases(); }
 XLA_TYPED_TEST(MathTypedTest, Log1pEdgeCases) { this->TestLog1pEdgeCases(); }
 XLA_TYPED_TEST(MathTypedTest, IsInfOrNan) { this->TestIsInfOrNan(); }
+XLA_TYPED_TEST(MathTypedTest, IsNegZero) { this->TestIsNegZero(); }
 
 // Check that certain ops only support real, floating-point inputs.
 //
-- 
GitLab


From 9b73a8ee12dbc4e442baf93e566c8696bdbfebe1 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Tue, 5 Feb 2019 14:41:32 -0800
Subject: [PATCH 1953/2345] [XLA] Print out directories we looked in for
 ptxas/libdevice when we can't find them.

This way users know what where we're looking.

Also inform users that they can pass a flag to cause us to look in a specific directory.

PiperOrigin-RevId: 232561130
---
 .../xla/service/gpu/nvptx_compiler.cc         | 38 +++++++++++++++----
 1 file changed, 31 insertions(+), 7 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
index 1f4f176661..4bead3e6b5 100644
--- a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
+++ b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
@@ -118,6 +118,9 @@ std::vector<string> GetCudaRootCandidates(
     const HloModuleConfig& hlo_module_config) {
   std::vector<string> potential_cuda_roots = tensorflow::CandidateCudaRoots();
 
+  // "." is our last resort, even though it probably won't work.
+  potential_cuda_roots.push_back(".");
+
   // CUDA location explicitly specified by user via --xla_gpu_cuda_data_dir has
   // highest priority.
   string xla_gpu_cuda_data_dir =
@@ -129,9 +132,23 @@ std::vector<string> GetCudaRootCandidates(
   return potential_cuda_roots;
 }
 
+void PrintCantFindCudaMessage(absl::string_view msg,
+                              const HloModuleConfig& hlo_module_config) {
+  LOG(WARNING) << msg;
+  LOG(WARNING) << "Searched in the following directories:";
+  for (const auto& dir : GetCudaRootCandidates(hlo_module_config)) {
+    LOG(WARNING) << "  " << dir;
+  }
+  LOG(WARNING)
+      << "You can choose the search directory by setting xla_gpu_cuda_data_dir "
+         "in HloModule's DebugOptions.  For most apps, setting the environment "
+         "variable XLA_FLAGS=--xla_gpu_cuda_data_dir=/path/to/cuda will work.";
+}
+
 // Returns the directory containing nvvm libdevice files.
 string GetLibdeviceDir(const HloModuleConfig& hlo_module_config) {
-  for (const string& cuda_root : GetCudaRootCandidates(hlo_module_config)) {
+  const auto& candidate_dirs = GetCudaRootCandidates(hlo_module_config);
+  for (const string& cuda_root : candidate_dirs) {
     string libdevice_dir =
         tensorflow::io::JoinPath(cuda_root, "nvvm", "libdevice");
     VLOG(2) << "Looking for libdevice at " << libdevice_dir;
@@ -140,8 +157,14 @@ string GetLibdeviceDir(const HloModuleConfig& hlo_module_config) {
       return libdevice_dir;
     }
   }
-  LOG(WARNING) << "Unable to find libdevice dir. Using '.'";
-  // Last resort: maybe in the current folder.
+  PrintCantFindCudaMessage(
+      "Can't find directory containing CUDA libevice.  This may result in "
+      "compilation or runtime failures, if the program we try to run uses "
+      "routines from libdevice.",
+      hlo_module_config);
+
+  // GetCudaRotCandidates always inclues ".", but but if everything fails, we
+  // return it anyway.  Better than returning the empty string.
   return ".";
 }
 
@@ -843,10 +866,11 @@ std::vector<uint8> NVPTXCompiler::CompilePtxOrGetCachedResult(
             log_warning = !warning_done.exchange(true);
           }
           if (log_warning) {
-            LOG(WARNING)
-                << "Failed to compile ptx to cubin.  Will attempt to let "
-                   "GPU driver compile the ptx. "
-                << maybe_cubin.status();
+            PrintCantFindCudaMessage(
+                "Can't find ptxas binary.  Will back to the GPU driver "
+                "for PTX -> sass compilation.  This is OK so long as you don't "
+                "see a warning below about an out-of-date driver version.",
+                hlo_module_config);
           }
 
           // We're going to use the driver to JIT our PTX->SASS, so warn if
-- 
GitLab


From 5943cddd6b10c85af26d15e9c5a40c1ef7791b03 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 5 Feb 2019 14:52:07 -0800
Subject: [PATCH 1954/2345] Enable easier compilation of tagged specializations
 in generated projects.

PiperOrigin-RevId: 232563264
---
 .../examples/micro_speech/CMSIS/Makefile.inc  | 35 ++++++++++++-------
 .../experimental/micro/tools/make/Makefile    | 11 +++++-
 .../micro/tools/make/helper_functions.inc     | 10 +++---
 .../tools/make/targets/ecm3531_makefile.inc   | 17 ++++++---
 .../micro/tools/make/templates/Makefile.tpl   |  9 ++---
 5 files changed, 54 insertions(+), 28 deletions(-)

diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/Makefile.inc b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/Makefile.inc
index 3d560510ad..2d0deb0c60 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/Makefile.inc
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/Makefile.inc
@@ -5,6 +5,11 @@ ifneq ($(filter CMSIS,$(ALL_TAGS)),)
     -isystem$(MAKEFILE_DIR)/downloads/cmsis/CMSIS/DSP/Include/ \
     -I$(MAKEFILE_DIR)/downloads/CMSIS_ext/
 
+  GENERATED_PROJECT_INCLUDES += \
+    -isystemthird_party/cmsis/CMSIS/Core/Include/ \
+    -isystemthird_party/cmsis/CMSIS/DSP/Include/ \
+    -Ithird_party/CMSIS_ext/
+
   CMSIS_PREPROCESSOR_SRCS := \
     tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/hanning.cc \
     tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/sin_1k.cc \
@@ -24,20 +29,26 @@ ifneq ($(filter CMSIS,$(ALL_TAGS)),)
   MICRO_SPEECH_HDRS += $(CMSIS_PREPROCESSOR_HDRS)
 
   THIRD_PARTY_CC_SRCS += \
-    third_party/CMSIS_ext/arm_cmplx_mag_squared_q10p6.c \
-    third_party/cmsis/CMSIS/DSP/Source/BasicMathFunctions/arm_mult_q15.c \
-    third_party/cmsis/CMSIS/DSP/Source/TransformFunctions/arm_rfft_init_q15.c \
-    third_party/cmsis/CMSIS/DSP/Source/TransformFunctions/arm_rfft_q15.c \
-    third_party/cmsis/CMSIS/DSP/Source/TransformFunctions/arm_cfft_q15.c \
-    third_party/cmsis/CMSIS/DSP/Source/TransformFunctions/arm_cfft_radix4_q15.c \
-    third_party/cmsis/CMSIS/DSP/Source/TransformFunctions/arm_bitreversal2.S \
-    third_party/cmsis/CMSIS/DSP/Source/CommonTables/arm_const_structs.c \
-    third_party/cmsis/CMSIS/DSP/Source/CommonTables/arm_common_tables.c \
-    third_party/cmsis/CMSIS/DSP/Source/StatisticsFunctions/arm_mean_q15.c \
-    third_party/cmsis/CMSIS/DSP/Source/StatisticsFunctions/arm_max_q7.c
+    $(MAKEFILE_DIR)/downloads/CMSIS_ext/arm_cmplx_mag_squared_q10p6.c \
+    $(MAKEFILE_DIR)/downloads/cmsis/CMSIS/DSP/Source/BasicMathFunctions/arm_mult_q15.c \
+    $(MAKEFILE_DIR)/downloads/cmsis/CMSIS/DSP/Source/TransformFunctions/arm_bitreversal.c \
+    $(MAKEFILE_DIR)/downloads/cmsis/CMSIS/DSP/Source/TransformFunctions/arm_rfft_init_q15.c \
+    $(MAKEFILE_DIR)/downloads/cmsis/CMSIS/DSP/Source/TransformFunctions/arm_rfft_q15.c \
+    $(MAKEFILE_DIR)/downloads/cmsis/CMSIS/DSP/Source/TransformFunctions/arm_cfft_q15.c \
+    $(MAKEFILE_DIR)/downloads/cmsis/CMSIS/DSP/Source/TransformFunctions/arm_cfft_radix4_q15.c \
+    $(MAKEFILE_DIR)/downloads/cmsis/CMSIS/DSP/Source/TransformFunctions/arm_bitreversal2.S \
+    $(MAKEFILE_DIR)/downloads/cmsis/CMSIS/DSP/Source/CommonTables/arm_const_structs.c \
+    $(MAKEFILE_DIR)/downloads/cmsis/CMSIS/DSP/Source/CommonTables/arm_common_tables.c \
+    $(MAKEFILE_DIR)/downloads/cmsis/CMSIS/DSP/Source/StatisticsFunctions/arm_mean_q15.c \
+    $(MAKEFILE_DIR)/downloads/cmsis/CMSIS/DSP/Source/StatisticsFunctions/arm_max_q7.c
 
   THIRD_PARTY_CC_HDRS += \
+    third_party/cmsis/CMSIS/Core/Include/cmsis_compiler.h \
+    third_party/cmsis/CMSIS/Core/Include/cmsis_gcc.h \
+    third_party/cmsis/CMSIS/Core/Include/cmsis_version.h \
+    third_party/cmsis/CMSIS/Core/Include/core_cm3.h \
     third_party/cmsis/CMSIS/DSP/Include/arm_common_tables.h \
-    third_party/cmsis/CMSIS/DSP/Include/arm_const_structs.h
+    third_party/cmsis/CMSIS/DSP/Include/arm_const_structs.h \
+    third_party/cmsis/CMSIS/DSP/Include/arm_math.h
 
 endif
diff --git a/tensorflow/lite/experimental/micro/tools/make/Makefile b/tensorflow/lite/experimental/micro/tools/make/Makefile
index 741d50eb87..3457fafb9d 100644
--- a/tensorflow/lite/experimental/micro/tools/make/Makefile
+++ b/tensorflow/lite/experimental/micro/tools/make/Makefile
@@ -30,13 +30,22 @@ TARGET_ARCH := $(HOST_ARCH)
 # STM32F746NG board, using the CMSIS library's implementations where possible.
 ALL_TAGS := $(TAGS) $(TARGET)
 
+# This is obviously horrible.  We need to generate these 3 versions of the
+# include directories from one source.
 INCLUDES := \
 -I. \
 -I$(MAKEFILE_DIR)/downloads/ \
 -I$(MAKEFILE_DIR)/downloads/gemmlowp \
 -I$(MAKEFILE_DIR)/downloads/flatbuffers/include
 
-# These are the include paths added to any generated project file.
+# Same list of paths, but now relative to the generated project files.
+GENERATED_PROJECT_INCLUDES := \
+-I. \
+-I./third_party/gemmlowp \
+-I./third_party/flatbuffers/include
+
+# Same list of paths, but now in the format the generate_keil_project.py
+# script expects them.
 PROJECT_INCLUDES := \
 . \
 third_party/gemmlowp \
diff --git a/tensorflow/lite/experimental/micro/tools/make/helper_functions.inc b/tensorflow/lite/experimental/micro/tools/make/helper_functions.inc
index 4d2465b5e9..28f7618672 100644
--- a/tensorflow/lite/experimental/micro/tools/make/helper_functions.inc
+++ b/tensorflow/lite/experimental/micro/tools/make/helper_functions.inc
@@ -52,6 +52,7 @@ specialize = $(call specialize_on_tags,$(1),$(strip $(call reverse,$(ALL_TAGS)))
 # 5 - List of C/C++ header files needed to build the target.
 # 6 - Linker flags required.
 # 7 - C++ compilation flags needed.
+# 8 - C compilation flags needed.
 # Calling eval on the output will create a <Name>_makefile target that you
 # can invoke to create the standalone project.
 define generate_project
@@ -68,7 +69,8 @@ $(PRJDIR)$(3)/$(1)/%: tensorflow/lite/experimental/micro/tools/make/templates/%.
 	sed -E 's#\%\{SRCS\}\%#$(4)#g' $$< | \
 	sed -E 's#\%\{EXECUTABLE\}\%#$(3)#g' | \
 	sed -E 's#\%\{LINKER_FLAGS\}\%#$(6)#g' | \
-	sed -E 's#\%\{CXX_FLAGS\}\%#$(7)#g' > $$@
+	sed -E 's#\%\{CXX_FLAGS\}\%#$(7)#g' | \
+	sed -E 's#\%\{CC_FLAGS\}\%#$(8)#g' > $$@
 
 $(PRJDIR)$(3)/$(1)/keil_project.uvprojx: tensorflow/lite/experimental/micro/tools/make/templates/keil_project.uvprojx.tpl
 	@mkdir -p $$(dir $$@)
@@ -89,9 +91,9 @@ endef
 # Calling eval on the output will create targets that you can invoke to
 # generate the standalone project.
 define generate_microlite_projects
-$(call generate_project,make,$(MAKE_PROJECT_FILES),$(1),$(MICROLITE_CC_SRCS) $(THIRD_PARTY_CC_SRCS) $(2),$(MICROLITE_CC_HDRS) $(THIRD_PARTY_CC_HDRS) $(MICROLITE_TEST_HDRS) $(3),$(MICROLITE_LIBS),$(CXXFLAGS))
-$(call generate_project,mbed,$(MBED_PROJECT_FILES),$(1),$(MICROLITE_CC_SRCS) $(THIRD_PARTY_CC_SRCS) $(2),$(MICROLITE_CC_HDRS) $(THIRD_PARTY_CC_HDRS) $(MICROLITE_TEST_HDRS) $(3),$(MICROLITE_LIBS),$(CXXFLAGS))
-$(call generate_project,keil,$(KEIL_PROJECT_FILES),$(1),$(MICROLITE_CC_SRCS) $(THIRD_PARTY_CC_SRCS) $(2),$(MICROLITE_CC_HDRS) $(THIRD_PARTY_CC_HDRS) $(MICROLITE_TEST_HDRS) $(3),$(MICROLITE_LIBS),$(CXXFLAGS))
+$(call generate_project,make,$(MAKE_PROJECT_FILES),$(1),$(MICROLITE_CC_SRCS) $(THIRD_PARTY_CC_SRCS) $(2),$(MICROLITE_CC_HDRS) $(THIRD_PARTY_CC_HDRS) $(MICROLITE_TEST_HDRS) $(3),$(LDFLAGS) $(MICROLITE_LIBS),$(CXXFLAGS) $(GENERATED_PROJECT_INCLUDES), $(CCFLAGS) $(GENERATED_PROJECT_INCLUDES))
+$(call generate_project,mbed,$(MBED_PROJECT_FILES),$(1),$(MICROLITE_CC_SRCS) $(THIRD_PARTY_CC_SRCS) $(2),$(MICROLITE_CC_HDRS) $(THIRD_PARTY_CC_HDRS) $(MICROLITE_TEST_HDRS) $(3),$(MICROLITE_LIBS),$(CXXFLAGS),$(CCFLAGS))
+$(call generate_project,keil,$(KEIL_PROJECT_FILES),$(1),$(MICROLITE_CC_SRCS) $(THIRD_PARTY_CC_SRCS) $(2),$(MICROLITE_CC_HDRS) $(THIRD_PARTY_CC_HDRS) $(MICROLITE_TEST_HDRS) $(3),$(MICROLITE_LIBS),$(CXXFLAGS),$(CCFLAGS))
 endef
 
 
diff --git a/tensorflow/lite/experimental/micro/tools/make/targets/ecm3531_makefile.inc b/tensorflow/lite/experimental/micro/tools/make/targets/ecm3531_makefile.inc
index baae58f87e..4ce2f69bc9 100644
--- a/tensorflow/lite/experimental/micro/tools/make/targets/ecm3531_makefile.inc
+++ b/tensorflow/lite/experimental/micro/tools/make/targets/ecm3531_makefile.inc
@@ -14,6 +14,7 @@ ifeq ($(TARGET), ecm3531)
   endif
 
   PLATFORM_FLAGS = \
+    -DARM_MATH_CM3 \
     -DFIRMWARE_BUILD \
     -DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK \
     -DTF_LITE_STATIC_MEMORY \
@@ -59,13 +60,12 @@ ifeq ($(TARGET), ecm3531)
     -fno-exceptions \
     -nostdlib --specs=nano.specs -t -lstdc++ -lc -lnosys -lm \
     -Wl,-T,$(MAKEFILE_DIR)/targets/ecm3531/ecm3531.lds \
-    -Wl,-Map=$(MAKEFILE_DIR)/gen/$(TARGET).map,--cref
+    -Wl,-Map=$(MAKEFILE_DIR)/targets/ecm3531/ecm3531.map,--cref
   BUILD_TYPE := micro
   MICROLITE_LIBS := \
     $(GCC_ARM)/lib/gcc/arm-none-eabi/7.3.1/thumb/v7e-m/fpv4-sp/softfp/crtbegin.o \
     -lm
-  INCLUDES += \
-    -isystem$(MAKEFILE_DIR)/downloads/cmsis/CMSIS/Core/Include/ \
+  ECM3531_INCLUDES := \
     -I$(GCC_ARM)/arm-none-eabi/include/ \
     -I$(ETA_SDK)/ecm3531/boards/eta_evb/projects/m3/common/inc/ \
     -I$(ETA_SDK)/ecm3531/m3/reg/inc/ \
@@ -75,6 +75,9 @@ ifeq ($(TARGET), ecm3531)
     -I$(ETA_SDK)/../utils/inc/  \
     -I$(ETA_SDK)/ecm3531/boards/eta_evb/eta_bsp/inc
 
+  INCLUDES += $(ECM3531_INCLUDES)
+  GENERATED_PROJECT_INCLUDES += $(ECM3531_INCLUDES)
+
   # _main.c contains application and target specific initialization, like
   # setting clock speed, default uart setups, etc. and an implementation
   # of the DebugLog interfaces.
@@ -83,7 +86,13 @@ ifeq ($(TARGET), ecm3531)
     $(MAKEFILE_DIR)/targets/ecm3531/_main.c \
     $(wildcard $(ETA_SDK)/ecm3531/boards/eta_evb/projects/m3/common/src/*.c) \
     $(wildcard $(ETA_SDK)/ecm3531/m3/csp/src/*.c) \
-    $(wildcard $(ETA_SDK)/ecm3531/m3/csp/src/*.s) \
+    $(wildcard $(ETA_SDK)/ecm3531/m3/csp/src/*.s)
+
+  # The linker script isn't a header, but it needs to get copied to the gen/
+  # directory for generated projects.  This is similar to the behavior needed
+  # for headers.
+  MICROLITE_CC_HDRS += \
+    $(MAKEFILE_DIR)/targets/ecm3531/ecm3531.lds
 
   TEST_SCRIPT := tensorflow/lite/experimental/micro/testing/test_ecm3531_binary.sh
   # These are tests that don't currently work on the blue pill.
diff --git a/tensorflow/lite/experimental/micro/tools/make/templates/Makefile.tpl b/tensorflow/lite/experimental/micro/tools/make/templates/Makefile.tpl
index 74d54f1ebe..ca6519c139 100644
--- a/tensorflow/lite/experimental/micro/tools/make/templates/Makefile.tpl
+++ b/tensorflow/lite/experimental/micro/tools/make/templates/Makefile.tpl
@@ -4,12 +4,8 @@ SRCS := \
 OBJS := \
 $(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(SRCS)))
 
-INCLUDES := \
--I. \
--I./third_party/gemmlowp \
--I./third_party/flatbuffers/include
-
 CXXFLAGS += %{CXX_FLAGS}%
+CCFLAGS += %{CC_FLAGS}%
 
 LDFLAGS += %{LINKER_FLAGS}%
 
@@ -20,7 +16,6 @@ LDFLAGS += %{LINKER_FLAGS}%
 	$(CC) $(CCFLAGS) $(INCLUDES) -c $< -o $@
 
 %{EXECUTABLE}% : $(OBJS)
-	$(CXX) $(LDFLAGS) $(OBJS) \
-	-o $@
+	$(CXX) $(CXXFLAGS) -o $@ $(OBJS) $(LDFLAGS)
 
 all: %{EXECUTABLE}%
-- 
GitLab


From b0b8121f95223d436f56f8f858b30369dd71de5f Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Tue, 5 Feb 2019 14:57:47 -0800
Subject: [PATCH 1955/2345] Fix end-of-epoch tf.data.Datasets reset code.

The datasets weren't getting reset, because the loop was was reading `output_generator`,
but the reset path was only resetting `generator`.

PiperOrigin-RevId: 232564221
---
 .../keras/engine/training_dataset_test.py     | 20 +++++++++++++++++--
 .../python/keras/engine/training_generator.py | 20 ++++++++++---------
 2 files changed, 29 insertions(+), 11 deletions(-)

diff --git a/tensorflow/python/keras/engine/training_dataset_test.py b/tensorflow/python/keras/engine/training_dataset_test.py
index 751c265005..4d2d68c03c 100644
--- a/tensorflow/python/keras/engine/training_dataset_test.py
+++ b/tensorflow/python/keras/engine/training_dataset_test.py
@@ -27,6 +27,7 @@ from tensorflow.python.data.experimental.ops import cardinality
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.eager import context
 from tensorflow.python.framework import test_util as tf_test_util
+from tensorflow.python.keras import callbacks
 from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import metrics as metrics_module
 from tensorflow.python.keras import testing_utils
@@ -34,6 +35,15 @@ from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging as logging
 
 
+class BatchCounterCallback(callbacks.Callback):
+
+  def __init__(self):
+    self.batch_count = 0
+
+  def on_batch_end(self, *args, **kwargs):
+    self.batch_count += 1
+
+
 class TestTrainingWithDatasetIterators(keras_parameterized.TestCase):
 
   @keras_parameterized.run_with_all_model_types
@@ -394,8 +404,11 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
     dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
     dataset = dataset.batch(10)
 
-    history = model.fit(dataset, epochs=2, verbose=1)
+    batch_counter = BatchCounterCallback()
+    history = model.fit(dataset, epochs=2, verbose=1, callbacks=[batch_counter])
+
     self.assertEqual(len(history.history['loss']), 2)
+    self.assertEqual(batch_counter.batch_count, 20)
     model.evaluate(dataset)
     out = model.predict(dataset)
     self.assertEqual(out.shape[0], 100)
@@ -415,8 +428,11 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
     self.assertEqual(keras.backend.get_value(cardinality.cardinality(dataset)),
                      cardinality.UNKNOWN)
 
-    history = model.fit(dataset, epochs=2, verbose=1)
+    batch_counter = BatchCounterCallback()
+    history = model.fit(dataset, epochs=2, verbose=1, callbacks=[batch_counter])
+
     self.assertEqual(len(history.history['loss']), 2)
+    self.assertEqual(batch_counter.batch_count, 20)
     model.evaluate(dataset)
     out = model.predict(dataset)
     self.assertEqual(out.shape[0], 100)
diff --git a/tensorflow/python/keras/engine/training_generator.py b/tensorflow/python/keras/engine/training_generator.py
index da460ee205..7c63161024 100644
--- a/tensorflow/python/keras/engine/training_generator.py
+++ b/tensorflow/python/keras/engine/training_generator.py
@@ -149,12 +149,14 @@ def model_iteration(model,
       model, mode, class_weight=class_weight)
 
   # Create the queue for the generator.
-  output_generator, enqueuer = _make_enqueued_generator(
-      generator,
-      workers=workers,
-      use_multiprocessing=use_multiprocessing,
-      max_queue_size=max_queue_size,
-      shuffle=shuffle)
+  enqueuer = None
+  if not is_dataset:
+    generator, enqueuer = _make_enqueued_generator(
+        generator,
+        workers=workers,
+        use_multiprocessing=use_multiprocessing,
+        max_queue_size=max_queue_size,
+        shuffle=shuffle)
 
   num_samples_or_steps, use_steps = _get_num_samples_or_steps(
       data, steps_per_epoch)
@@ -208,7 +210,7 @@ def model_iteration(model,
 
     step = 0
     while step < target_steps:
-      batch_data = _get_next_batch(output_generator, mode)
+      batch_data = _get_next_batch(generator, mode)
       if batch_data is None:
         if not is_dataset:
           # We ran out of batches while the user passed an iterator (legacy).
@@ -317,10 +319,10 @@ predict_generator = functools.partial(
     model_iteration, mode=ModeKeys.PREDICT, shuffle=False)
 
 
-def _get_next_batch(output_generator, mode):
+def _get_next_batch(generator, mode):
   """Retrieves the next batch of input data."""
   try:
-    generator_output = next(output_generator)
+    generator_output = next(generator)
   except (StopIteration, errors.OutOfRangeError):
     return None
   if not isinstance(generator_output, tuple):
-- 
GitLab


From d1dcc5b407b3af4cf8fdf110189f37747250a835 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Tue, 5 Feb 2019 14:58:57 -0800
Subject: [PATCH 1956/2345] Update regularizer test to reduce flake.

PiperOrigin-RevId: 232564438
---
 tensorflow/python/keras/BUILD                |  2 +-
 tensorflow/python/keras/regularizers_test.py | 83 +++++++++++---------
 2 files changed, 45 insertions(+), 40 deletions(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 1c71ff9679..6f3fd226f4 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -411,7 +411,7 @@ tf_py_test(
 
 tf_py_test(
     name = "regularizers_test",
-    size = "small",
+    size = "medium",
     srcs = ["regularizers_test.py"],
     additional_deps = [
         ":keras",
diff --git a/tensorflow/python/keras/regularizers_test.py b/tensorflow/python/keras/regularizers_test.py
index 3d6b259d87..3aca0c7fd3 100644
--- a/tensorflow/python/keras/regularizers_test.py
+++ b/tensorflow/python/keras/regularizers_test.py
@@ -18,9 +18,11 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from absl.testing import parameterized
+
 from tensorflow.python import keras
-from tensorflow.python.keras import testing_utils
 from tensorflow.python.framework import test_util
+from tensorflow.python.keras import testing_utils
 from tensorflow.python.platform import test
 
 
@@ -28,50 +30,53 @@ DATA_DIM = 5
 NUM_CLASSES = 2
 
 
-def get_data():
-  (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
-      train_samples=10,
-      test_samples=10,
-      input_shape=(DATA_DIM,),
-      num_classes=NUM_CLASSES)
-  y_train = keras.utils.to_categorical(y_train, NUM_CLASSES)
-  y_test = keras.utils.to_categorical(y_test, NUM_CLASSES)
-  return (x_train, y_train), (x_test, y_test)
-
-
-def create_model(kernel_regularizer=None, activity_regularizer=None):
-  model = keras.models.Sequential()
-  model.add(keras.layers.Dense(NUM_CLASSES,
-                               kernel_regularizer=kernel_regularizer,
-                               activity_regularizer=activity_regularizer,
-                               input_shape=(DATA_DIM,)))
-  return model
+class KerasRegularizersTest(test.TestCase, parameterized.TestCase):
 
+  def create_model(self, kernel_regularizer=None, activity_regularizer=None):
+    model = keras.models.Sequential()
+    model.add(keras.layers.Dense(NUM_CLASSES,
+                                 kernel_regularizer=kernel_regularizer,
+                                 activity_regularizer=activity_regularizer,
+                                 input_shape=(DATA_DIM,)))
+    return model
 
-class KerasRegularizersTest(test.TestCase):
+  def get_data(self):
+    (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
+        train_samples=10,
+        test_samples=10,
+        input_shape=(DATA_DIM,),
+        num_classes=NUM_CLASSES)
+    y_train = keras.utils.to_categorical(y_train, NUM_CLASSES)
+    y_test = keras.utils.to_categorical(y_test, NUM_CLASSES)
+    return (x_train, y_train), (x_test, y_test)
 
-  def test_kernel_regularization(self):
+  @parameterized.named_parameters([
+      ('l1', keras.regularizers.l1()),
+      ('l2', keras.regularizers.l2()),
+      ('l1_l2', keras.regularizers.l1_l2()),
+  ])
+  def test_kernel_regularization(self, regularizer):
     with self.cached_session():
-      (x_train, y_train), _ = get_data()
-      for reg in [keras.regularizers.l1(),
-                  keras.regularizers.l2(),
-                  keras.regularizers.l1_l2()]:
-        model = create_model(kernel_regularizer=reg)
-        model.compile(loss='categorical_crossentropy', optimizer='sgd')
-        assert len(model.losses) == 1
-        model.fit(x_train, y_train, batch_size=10,
-                  epochs=1, verbose=0)
+      (x_train, y_train), _ = self.get_data()
+      model = self.create_model(kernel_regularizer=regularizer)
+      model.compile(loss='categorical_crossentropy', optimizer='sgd')
+      assert len(model.losses) == 1
+      model.fit(x_train, y_train, batch_size=10,
+                epochs=1, verbose=0)
 
-  @test_util.run_deprecated_v1
-  def test_activity_regularization(self):
+  @parameterized.named_parameters([
+      ('l1', keras.regularizers.l1()),
+      ('l2', keras.regularizers.l2()),
+  ])
+  @test_util.deprecated_graph_mode_only
+  def test_activity_regularization(self, regularizer):
     with self.cached_session():
-      (x_train, y_train), _ = get_data()
-      for reg in [keras.regularizers.l1(), keras.regularizers.l2()]:
-        model = create_model(activity_regularizer=reg)
-        model.compile(loss='categorical_crossentropy', optimizer='sgd')
-        assert len(model.losses) == 1
-        model.fit(x_train, y_train, batch_size=10,
-                  epochs=1, verbose=0)
+      (x_train, y_train), _ = self.get_data()
+      model = self.create_model(activity_regularizer=regularizer)
+      model.compile(loss='categorical_crossentropy', optimizer='sgd')
+      assert len(model.losses) == 1
+      model.fit(x_train, y_train, batch_size=10,
+                epochs=1, verbose=0)
 
   def test_zero_regularization(self):
     inputs = keras.backend.ones(shape=(10, 10))
-- 
GitLab


From afe79b092748bd142d24527fd9ee647f7cba67f9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 5 Feb 2019 15:00:03 -0800
Subject: [PATCH 1957/2345] Replace calls to SerializeToStringDeterministic()
 with SerializeToBufferDeterministic(), which is typically 30% faster.

In addition, this change allows us to detect negatives in proto comparison without actually doing the serialization, in the case where the serialized protos are known to have different lengths.

PiperOrigin-RevId: 232564652
---
 tensorflow/compiler/aot/compile.cc            |  9 ++--
 .../jit/encapsulate_xla_computations_pass.cc  |  7 +--
 tensorflow/compiler/xla/service/executable.cc | 13 +++---
 .../compiler/xrt/kernels/xrt_compile_ops.cc   |  8 ++--
 tensorflow/core/BUILD                         |  4 +-
 tensorflow/core/framework/attr_value_util.cc  | 25 ++--------
 tensorflow/core/framework/op_def_util.cc      | 10 +---
 .../core/lib/strings/proto_serialization.cc   | 46 ++++++++++++++++---
 .../core/lib/strings/proto_serialization.h    | 17 +++++--
 .../lib/strings/proto_serialization_test.cc   |  8 ++--
 10 files changed, 89 insertions(+), 58 deletions(-)

diff --git a/tensorflow/compiler/aot/compile.cc b/tensorflow/compiler/aot/compile.cc
index 9fc223bdc7..0e46a9f5e9 100644
--- a/tensorflow/compiler/aot/compile.cc
+++ b/tensorflow/compiler/aot/compile.cc
@@ -108,10 +108,13 @@ Status CompileGraph(const GraphDef& graph_def, const tf2xla::Config& config,
                         computation.Snapshot());
     // Serialize the HloSnapshot deterministically so that all the outputs of a
     // tf_library genrule are deterministic.
-    string proto;
-    TF_RET_CHECK(SerializeToStringDeterministic(*module, &proto));
+    const size_t size = module->ByteSizeLong();
+    auto serialized = absl::make_unique<char[]>(size);
+    TF_RET_CHECK(
+        SerializeToBufferDeterministic(*module, serialized.get(), size));
     TF_RETURN_IF_ERROR(
-        WriteStringToFile(Env::Default(), flags.out_session_module, proto));
+        WriteStringToFile(Env::Default(), flags.out_session_module,
+                          absl::string_view(serialized.get(), size)));
   }
   xla::cpu::CpuAotCompilationOptions aot_opts(
       flags.target_triple, flags.target_cpu, flags.target_features,
diff --git a/tensorflow/compiler/jit/encapsulate_xla_computations_pass.cc b/tensorflow/compiler/jit/encapsulate_xla_computations_pass.cc
index ec745cdbb7..c9ae717b8d 100644
--- a/tensorflow/compiler/jit/encapsulate_xla_computations_pass.cc
+++ b/tensorflow/compiler/jit/encapsulate_xla_computations_pass.cc
@@ -173,9 +173,10 @@ Status RewriteSubgraph(const std::vector<OutputTensor>& arg_source_tensors,
   // Nondeterminism in serialization would not lead to incorrect results, but
   // may cause spurious cache misses. DeterministicSerialization is a
   // best-effort deterministic serialization.
-  string serialized;
-  TF_RET_CHECK(SerializeToStringDeterministic(gdef, &serialized));
-  uint64 fingerprint = Fingerprint64(serialized);
+  const size_t size = gdef.ByteSizeLong();
+  auto serialized = absl::make_unique<char[]>(size);
+  TF_RET_CHECK(SerializeToBufferDeterministic(gdef, serialized.get(), size));
+  uint64 fingerprint = Fingerprint64(absl::string_view(serialized.get(), size));
   LOG(INFO) << "Subgraph fingerprint:" << fingerprint;
   call_def->set_op(absl::StrCat(call_def->op(), "_", fingerprint));
   return Status::OK();
diff --git a/tensorflow/compiler/xla/service/executable.cc b/tensorflow/compiler/xla/service/executable.cc
index 10b8c01ff1..1518d83083 100644
--- a/tensorflow/compiler/xla/service/executable.cc
+++ b/tensorflow/compiler/xla/service/executable.cc
@@ -26,7 +26,6 @@ limitations under the License.
 #include "tensorflow/core/lib/strings/proto_serialization.h"
 #include "tensorflow/core/platform/env.h"
 
-
 namespace xla {
 
 StatusOr<std::vector<ScopedShapedBuffer>> Executable::ExecuteOnStreams(
@@ -173,11 +172,13 @@ Status Executable::DumpHloSnapshot() {
   }
   filename = SanitizeFileName(std::move(filename));
   string file_path = tensorflow::io::JoinPath(directory_path, filename);
-  string result;
-  TF_RET_CHECK(
-      tensorflow::SerializeToStringDeterministic(hlo_session, &result));
-  return tensorflow::WriteStringToFile(tensorflow::Env::Default(), file_path,
-                                       result);
+  const size_t size = hlo_session.ByteSizeLong();
+  auto serialized = absl::make_unique<char[]>(size);
+  TF_RET_CHECK(tensorflow::SerializeToBufferDeterministic(
+      hlo_session, serialized.get(), size));
+  return tensorflow::WriteStringToFile(
+      tensorflow::Env::Default(), file_path,
+      absl::string_view(serialized.get(), size));
 }
 
 }  // namespace xla
diff --git a/tensorflow/compiler/xrt/kernels/xrt_compile_ops.cc b/tensorflow/compiler/xrt/kernels/xrt_compile_ops.cc
index 2ee1a6cd1a..b791519c09 100644
--- a/tensorflow/compiler/xrt/kernels/xrt_compile_ops.cc
+++ b/tensorflow/compiler/xrt/kernels/xrt_compile_ops.cc
@@ -68,9 +68,11 @@ class XRTCompileOp : public OpKernel {
 
 Status CompilationCacheKey(const xrt::XLAComputation& computation,
                            string* key) {
-  string serialized;
-  TF_RET_CHECK(SerializeToStringDeterministic(computation, &serialized));
-  uint64 fingerprint = Fingerprint64(serialized);
+  const size_t size = computation.ByteSizeLong();
+  auto serialized = absl::make_unique<char[]>(size);
+  TF_RET_CHECK(
+      SerializeToBufferDeterministic(computation, serialized.get(), size));
+  uint64 fingerprint = Fingerprint64(absl::string_view(serialized.get(), size));
   *key = absl::StrCat(fingerprint);
   return Status::OK();
 }
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 3a56fa476c..3b4fe3cc2d 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -2300,6 +2300,8 @@ cc_library(
         ":lib_proto_parsing",
         ":abi",
         ":core_stringpiece",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/strings",
         "//third_party/eigen3",
         "//tensorflow/core/platform/default/build_config:platformlib",
         "@snappy",
@@ -2679,7 +2681,6 @@ tf_cuda_library(
             "example/**/*.cc",
             "framework/**/*.cc",
             "util/**/*.cc",
-        ] + [
             "graph/edgeset.cc",
             "graph/graph.cc",
             "graph/graph_def_builder.cc",
@@ -3715,6 +3716,7 @@ tf_cc_test(
         ":protos_all_cc",
         ":test",
         ":test_main",
+        "@com_google_absl//absl/memory",
     ],
 )
 
diff --git a/tensorflow/core/framework/attr_value_util.cc b/tensorflow/core/framework/attr_value_util.cc
index 79966f0692..43b435270c 100644
--- a/tensorflow/core/framework/attr_value_util.cc
+++ b/tensorflow/core/framework/attr_value_util.cc
@@ -54,9 +54,7 @@ uint64 TensorProtoHash(const TensorProto& tp) {
   DCHECK(success);
   TensorProto p;
   tensor.AsProtoTensorContent(&p);
-  string s;
-  SerializeToStringDeterministic(p, &s);
-  return Hash64(s);
+  return DeterministicProtoHash64(p);
 }
 
 // Do not create large tensors in memory, compute hash based on TensorProto
@@ -64,12 +62,8 @@ uint64 TensorProtoHash(const TensorProto& tp) {
 // different hash code if they are defined with different TensorProto
 // representations.
 uint64 FastTensorProtoHash(const TensorProto& tp) {
-  string s;
   if (TensorByteSize(tp) > kMaxAttrValueTensorByteSize) {
-    string s;
-    bool success = SerializeToStringDeterministic(tp, &s);
-    DCHECK(success);
-    return Hash64(s);
+    return DeterministicProtoHash64(tp);
   } else {
     return TensorProtoHash(tp);
   }
@@ -95,11 +89,7 @@ bool AreTensorProtosEqual(const TensorProto& lhs, const TensorProto& rhs) {
   TensorProto rhs_tp;
   rhs_t.AsProtoTensorContent(&rhs_tp);
 
-  string lhs_str, rhs_str;
-  SerializeToStringDeterministic(lhs_tp, &lhs_str);
-  SerializeToStringDeterministic(rhs_tp, &rhs_str);
-
-  return lhs_str == rhs_str;
+  return AreSerializedProtosEqual(lhs_tp, rhs_tp);
 }
 
 // Do not construct large tensors in memory, compare equality using TensorProto
@@ -139,9 +129,7 @@ uint64 AttrValueHash(const AttrValue& a, const TensorProtoHasher& tensor_hash) {
   }
 
   // If `a` is not a tensor or func, get a hash of serialized string.
-  string s;
-  SerializeToStringDeterministic(a, &s);
-  return Hash64(s);
+  return DeterministicProtoHash64(a);
 }
 
 bool AreAttrValuesEqual(const AttrValue& a, const AttrValue& b,
@@ -175,10 +163,7 @@ bool AreAttrValuesEqual(const AttrValue& a, const AttrValue& b,
 
   // All other fields in AttrValue have deterministic representations.
   // It is safe to compare their serialized strings.
-  string a_str, b_str;
-  SerializeToStringDeterministic(a, &a_str);
-  SerializeToStringDeterministic(b, &b_str);
-  return a_str == b_str;
+  return AreSerializedProtosEqual(a, b);
 }
 
 string SummarizeString(const string& str) {
diff --git a/tensorflow/core/framework/op_def_util.cc b/tensorflow/core/framework/op_def_util.cc
index 3597f43d51..d629719096 100644
--- a/tensorflow/core/framework/op_def_util.cc
+++ b/tensorflow/core/framework/op_def_util.cc
@@ -838,20 +838,14 @@ bool OpDefEqual(const OpDef& o1, const OpDef& o2) {
   OpDef o2_copy = o2;
   o1_copy.clear_attr();
   o2_copy.clear_attr();
-  string s1, s2;
-  SerializeToStringDeterministic(o1_copy, &s1);
-  SerializeToStringDeterministic(o2_copy, &s2);
-  if (s1 != s2) return false;
-  return true;
+  return AreSerializedProtosEqual(o1_copy, o2_copy);
 }
 
 uint64 OpDefHash(const OpDef& o) {
   uint64 h = RepeatedAttrDefHash(o.attr());
   OpDef o_copy = o;
   o_copy.clear_attr();
-  string s;
-  SerializeToStringDeterministic(o_copy, &s);
-  return Hash64(s.data(), s.size(), h);
+  return DeterministicProtoHash64(o_copy, h);
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/lib/strings/proto_serialization.cc b/tensorflow/core/lib/strings/proto_serialization.cc
index 4a64eedef7..a6c321c782 100644
--- a/tensorflow/core/lib/strings/proto_serialization.cc
+++ b/tensorflow/core/lib/strings/proto_serialization.cc
@@ -14,24 +14,27 @@ limitations under the License.
 ==============================================================================*/
 #include "tensorflow/core/lib/strings/proto_serialization.h"
 
+#include <cstring>
+#include "absl/memory/memory.h"
+#include "absl/strings/string_view.h"
+#include "tensorflow/core/lib/hash/hash.h"
 #include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/macros.h"
 
 namespace tensorflow {
 
 bool SerializeToStringDeterministic(const protobuf::MessageLite& msg,
                                     string* result) {
-  DCHECK_LE(msg.ByteSizeLong(), static_cast<size_t>(INT_MAX));
-  *result = string(msg.ByteSizeLong(), '\0');
+  const size_t size = msg.ByteSizeLong();
+  DCHECK_LE(size, static_cast<size_t>(INT_MAX));
+  *result = string(size, '\0');
   return SerializeToBufferDeterministic(msg, const_cast<char*>(result->data()),
                                         result->size());
 }
 
-// As above, but takes a pre-allocated buffer wrapped by result.
-// PRECONDITION: result.size() == msg.ByteSizeLong().
 bool SerializeToBufferDeterministic(const protobuf::MessageLite& msg,
-                                    char* buffer, int size) {
-  DCHECK_LE(msg.ByteSizeLong(), static_cast<size_t>(INT_MAX));
-  DCHECK_EQ(msg.ByteSizeLong(), size);
+                                    char* buffer, size_t size) {
+  DCHECK(msg.ByteSizeLong() == size && size <= static_cast<size_t>(INT_MAX));
   protobuf::io::ArrayOutputStream array_stream(buffer, size);
   protobuf::io::CodedOutputStream output_stream(&array_stream);
   output_stream.SetSerializationDeterministic(true);
@@ -39,4 +42,33 @@ bool SerializeToBufferDeterministic(const protobuf::MessageLite& msg,
   return !output_stream.HadError() && size == output_stream.ByteCount();
 }
 
+bool AreSerializedProtosEqual(const protobuf::MessageLite& x,
+                              const protobuf::MessageLite& y) {
+  const size_t size = x.ByteSizeLong();
+  if (size != y.ByteSizeLong()) return false;
+  if (size == 0) return true;
+  auto x_serialized = absl::make_unique<char[]>(size);
+  bool success_x = SerializeToBufferDeterministic(x, x_serialized.get(), size);
+  DCHECK(success_x);
+  auto y_serialized = absl::make_unique<char[]>(size);
+  bool success_y = SerializeToBufferDeterministic(y, y_serialized.get(), size);
+  DCHECK(success_y);
+  return memcmp(x_serialized.get(), y_serialized.get(), size) == 0;
+}
+
+uint64 DeterministicProtoHash64(const protobuf::MessageLite& proto,
+                                uint64 seed) {
+  const size_t size = proto.ByteSizeLong();
+  auto serialized = absl::make_unique<char[]>(size);
+  SerializeToBufferDeterministic(proto, serialized.get(), size);
+  return Hash64(serialized.get(), size, seed);
+}
+
+uint64 DeterministicProtoHash64(const protobuf::MessageLite& proto) {
+  const size_t size = proto.ByteSizeLong();
+  auto serialized = absl::make_unique<char[]>(size);
+  SerializeToBufferDeterministic(proto, serialized.get(), size);
+  return Hash64(serialized.get(), size);
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/lib/strings/proto_serialization.h b/tensorflow/core/lib/strings/proto_serialization.h
index fa8848a649..763bd68c1b 100644
--- a/tensorflow/core/lib/strings/proto_serialization.h
+++ b/tensorflow/core/lib/strings/proto_serialization.h
@@ -28,10 +28,21 @@ namespace tensorflow {
 bool SerializeToStringDeterministic(const protobuf::MessageLite& msg,
                                     string* result);
 
-// As above, but serialize to a pre-allocated `buffer` of length `size`.
-// PRECONDITION: size == msg.ByteSizeLong().
+// As above, but takes a pre-allocated buffer wrapped by result.
+// PRECONDITION: size == msg.ByteSizeLong() && size <= INT_MAX.
 bool SerializeToBufferDeterministic(const protobuf::MessageLite& msg,
-                                    char* buffer, int size);
+                                    char* buffer, size_t size);
+
+// Returns true if serializing x and y using
+// SerializeToBufferDeterministic() yields identical strings.
+bool AreSerializedProtosEqual(const protobuf::MessageLite& x,
+                              const protobuf::MessageLite& y);
+
+// Computes Hash64 of the output of SerializeToBufferDeterministic().
+uint64 DeterministicProtoHash64(const protobuf::MessageLite& proto);
+uint64 DeterministicProtoHash64(const protobuf::MessageLite& proto,
+                                uint64 seed);
+
 }  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_LIB_STRINGS_PROTO_SERIALIZATION_H_
diff --git a/tensorflow/core/lib/strings/proto_serialization_test.cc b/tensorflow/core/lib/strings/proto_serialization_test.cc
index 466547daae..cbde20eced 100644
--- a/tensorflow/core/lib/strings/proto_serialization_test.cc
+++ b/tensorflow/core/lib/strings/proto_serialization_test.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/core/lib/strings/proto_serialization.h"
 
 #include <string>
+#include "absl/memory/memory.h"
 #include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/lib/strings/strcat.h"
@@ -51,12 +52,11 @@ static void BM_ProtoSerializationToBuffer(int iters, int num_nodes) {
     node->set_op(strings::StrCat("op", i % 10));
   }
   testing::StartTiming();
-  int size = graph_def.ByteSizeLong();
+  const size_t size = graph_def.ByteSizeLong();
   for (int i = 0; i < iters; ++i) {
-    char* buf = new char[size];
+    auto buf = absl::make_unique<char[]>(size);
     testing::DoNotOptimize(
-        SerializeToBufferDeterministic(graph_def, buf, size));
-    delete[] buf;
+        SerializeToBufferDeterministic(graph_def, buf.get(), size));
   }
   testing::StopTiming();
 }
-- 
GitLab


From e6612ea98ca98293e21e83f28b0d433b24b0e755 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 5 Feb 2019 15:03:14 -0800
Subject: [PATCH 1958/2345] * Small refactoring of constant folding of Merge
 nodes. * Small optimization of IsOnes/IsZeros.

PiperOrigin-RevId: 232565389
---
 tensorflow/core/grappler/op_types.cc          |   4 +
 tensorflow/core/grappler/op_types.h           |   2 +
 .../grappler/optimizers/constant_folding.cc   | 179 +++++++++---------
 .../grappler/optimizers/constant_folding.h    |   1 +
 4 files changed, 100 insertions(+), 86 deletions(-)

diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc
index e188c15b59..ee83fe009b 100644
--- a/tensorflow/core/grappler/op_types.cc
+++ b/tensorflow/core/grappler/op_types.cc
@@ -316,6 +316,8 @@ bool IsNextIteration(const NodeDef& node) {
   return op == "NextIteration" || op == "RefNextIteration";
 }
 
+bool IsOnesLike(const NodeDef& node) { return node.op() == "OnesLike"; }
+
 bool IsPack(const NodeDef& node) { return node.op() == "Pack"; }
 
 bool IsPad(const NodeDef& node) {
@@ -536,6 +538,8 @@ bool IsWhile(const NodeDef& node) {
   return op == "While" || op == "StatelessWhile";
 }
 
+bool IsZerosLike(const NodeDef& node) { return node.op() == "ZerosLike"; }
+
 bool IsZeta(const NodeDef& node) { return node.op() == "Zeta"; }
 
 namespace {
diff --git a/tensorflow/core/grappler/op_types.h b/tensorflow/core/grappler/op_types.h
index a1ee253369..7395151d30 100644
--- a/tensorflow/core/grappler/op_types.h
+++ b/tensorflow/core/grappler/op_types.h
@@ -100,6 +100,7 @@ bool IsMod(const NodeDef& node);
 bool IsMul(const NodeDef& node);
 bool IsMatMul(const NodeDef& node);
 bool IsNextIteration(const NodeDef& node);
+bool IsOnesLike(const NodeDef& node);
 bool IsPack(const NodeDef& node);
 bool IsPad(const NodeDef& node);
 bool IsPack(const NodeDef& node);
@@ -170,6 +171,7 @@ bool IsTruncateMod(const NodeDef& node);
 bool IsUnpack(const NodeDef& node);
 bool IsVariable(const NodeDef& node);
 bool IsWhile(const NodeDef& node);
+bool IsZerosLike(const NodeDef& node);
 bool IsZeta(const NodeDef& node);
 
 // Return true if the op is an aggregation (e.g. Add, AddN).
diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc
index b0c3c5b518..da7a328001 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding.cc
@@ -1059,98 +1059,103 @@ Status ConstantFolding::EvaluateOneFoldable(const NodeDef& node,
   return Status::OK();
 }
 
-Status ConstantFolding::FoldNode(NodeDef* node, GraphDef* output_graph,
-                                 bool* result_too_large) {
-  if (IsMerge(*node)) {
-    // Merge nodes are special, in the sense that they execute as soon as one of
-    // their input is ready. We can therefore fold a merge node iff it has at
-    // least one constant input without control dependency.
-    // We still need to ensure that the nodes in the fanin of the merge node are
-    // scheduled. We'll therefore add a control dependency from the merge node
-    // to the folded constant. We end up with:
-    //  * the merge node and its inputs are preserved as is
-    //  * a new constant node C1, driven by the merge node through a control
-    //  dependency, initialized to the value of the folded input
-    //  * a new constant node C2, driven by the merge node through a control
-    //  dependency, initialized to the index of the folded input
-    //  * the fanout of the merge nodes is rewired to be driven by either C1 or
-    //  C2.
-    for (int input_index = 0; input_index < node->input_size(); ++input_index) {
-      const auto& input = node->input(input_index);
-      if (IsControlInput(input)) {
-        // Try the next input.
-        continue;
-      }
-      NodeDef* input_node = node_map_->GetNode(input);
-      if (!IsReallyConstant(*input_node)) {
-        continue;
-      }
-      bool valid_input = true;
-      for (const string& fanin_of_input : input_node->input()) {
-        if (IsControlInput(fanin_of_input)) {
-          valid_input = false;
-          break;
-        }
-      }
-      if (!valid_input) {
-        // Try the next input
-        continue;
+Status ConstantFolding::FoldMergeNode(NodeDef* node, GraphDef* output_graph) {
+  // Merge nodes are special, in the sense that they execute as soon as one of
+  // their input is ready. We can therefore fold a merge node iff it has at
+  // least one constant input without control dependency.
+  // We still need to ensure that the nodes in the fanin of the merge node are
+  // scheduled. We'll therefore add a control dependency from the merge node
+  // to the folded constant. We end up with:
+  //  * the merge node and its inputs are preserved as is
+  //  * a new constant node C1, driven by the merge node through a control
+  //  dependency, initialized to the value of the folded input
+  //  * a new constant node C2, driven by the merge node through a control
+  //  dependency, initialized to the index of the folded input
+  //  * the fanout of the merge nodes is rewired to be driven by either C1 or
+  //  C2.
+  for (int input_index = 0; input_index < node->input_size(); ++input_index) {
+    const auto& input = node->input(input_index);
+    if (IsControlInput(input)) {
+      // Try the next input.
+      continue;
+    }
+    NodeDef* input_node = node_map_->GetNode(input);
+    if (!IsReallyConstant(*input_node)) {
+      continue;
+    }
+    bool valid_input = true;
+    for (const string& fanin_of_input : input_node->input()) {
+      if (IsControlInput(fanin_of_input)) {
+        valid_input = false;
+        break;
       }
+    }
+    if (!valid_input) {
+      // Try the next input
+      continue;
+    }
 
-      string const_out_name = OptimizedNodeName(*node, "_const");
-      string const_index_name = OptimizedNodeName(*node, "_index");
-      if (node_map_->GetNode(const_out_name) ||
-          node_map_->GetNode(const_index_name)) {
-        // Intended name already exists.
-        return errors::AlreadyExists(
-            strings::StrCat(const_out_name, " or ", const_index_name,
-                            " already present in the graph"));
-      }
-
-      NodeDef* const_out = output_graph->add_node();
-      *const_out = *input_node;
-      const_out->set_name(const_out_name);
-      const_out->set_device(node->device());
-      *const_out->add_input() = AsControlDependency(*node);
-      node_map_->AddNode(const_out->name(), const_out);
-      node_map_->AddOutput(node->name(), const_out->name());
-
-      NodeDef* const_index = output_graph->add_node();
-      const_index->set_op("Const");
-      Tensor index(DT_INT32, TensorShape({}));
-      index.flat<int32>()(0) = input_index;
-      (*const_index->mutable_attr())["dtype"].set_type(DT_INT32);
-      index.AsProtoTensorContent(
-          (*const_index->mutable_attr())["value"].mutable_tensor());
-      const_index->set_name(const_index_name);
-      const_index->set_device(node->device());
-      *const_index->add_input() = AsControlDependency(*node);
-      node_map_->AddNode(const_index->name(), const_index);
-      node_map_->AddOutput(node->name(), const_index->name());
-
-      auto outputs = node_map_->GetOutputs(node->name());
-      for (NodeDef* output : outputs) {
-        for (int i = 0; i < output->input_size(); i++) {
-          int port;
-          string node_name = ParseNodeName(output->input(i), &port);
-          if (node_name == node->name()) {
-            if (port == 0) {
-              *output->mutable_input(i) = const_out->name();
-              node_map_->AddOutput(const_out->name(), output->name());
-            } else if (port == 1) {
-              *output->mutable_input(i) = const_index->name();
-              node_map_->AddOutput(const_index->name(), output->name());
-            } else {
-              // This is a control dependency (or an invalid edge since the
-              // merge node has only 2 inputs): preserve them.
-            }
+    string const_out_name = OptimizedNodeName(*node, "_const");
+    string const_index_name = OptimizedNodeName(*node, "_index");
+    if (node_map_->GetNode(const_out_name) ||
+        node_map_->GetNode(const_index_name)) {
+      // Intended name already exists.
+      return errors::AlreadyExists(
+          strings::StrCat(const_out_name, " or ", const_index_name,
+                          " already present in the graph"));
+    }
+
+    NodeDef* const_out = output_graph->add_node();
+    *const_out = *input_node;
+    const_out->set_name(const_out_name);
+    const_out->set_device(node->device());
+    *const_out->add_input() = AsControlDependency(*node);
+    node_map_->AddNode(const_out->name(), const_out);
+    node_map_->AddOutput(node->name(), const_out->name());
+
+    NodeDef* const_index = output_graph->add_node();
+    const_index->set_op("Const");
+    Tensor index(DT_INT32, TensorShape({}));
+    index.flat<int32>()(0) = input_index;
+    (*const_index->mutable_attr())["dtype"].set_type(DT_INT32);
+    index.AsProtoTensorContent(
+        (*const_index->mutable_attr())["value"].mutable_tensor());
+    const_index->set_name(const_index_name);
+    const_index->set_device(node->device());
+    *const_index->add_input() = AsControlDependency(*node);
+    node_map_->AddNode(const_index->name(), const_index);
+    node_map_->AddOutput(node->name(), const_index->name());
+
+    auto outputs = node_map_->GetOutputs(node->name());
+    for (NodeDef* output : outputs) {
+      for (int i = 0; i < output->input_size(); i++) {
+        int port;
+        string node_name = ParseNodeName(output->input(i), &port);
+        if (node_name == node->name()) {
+          if (port == 0) {
+            *output->mutable_input(i) = const_out->name();
+            node_map_->AddOutput(const_out->name(), output->name());
+          } else if (port == 1) {
+            *output->mutable_input(i) = const_index->name();
+            node_map_->AddOutput(const_index->name(), output->name());
+          } else {
+            // This is a control dependency (or an invalid edge since the
+            // merge node has only 2 inputs): preserve them.
           }
         }
       }
-      return Status::OK();
     }
     return Status::OK();
   }
+  return Status::OK();
+}
+
+Status ConstantFolding::FoldNode(NodeDef* node, GraphDef* output_graph,
+                                 bool* result_too_large) {
+  *result_too_large = false;
+  if (IsMerge(*node)) {
+    return FoldMergeNode(node, output_graph);
+  }
 
   std::vector<NodeDef> const_nodes;
   TF_RETURN_IF_ERROR(
@@ -1395,7 +1400,8 @@ bool ConstantFolding::IsOnes(const NodeDef& node) const {
   if (feed_nodes_.find(node.name()) != feed_nodes_.end()) {
     return false;
   }
-  if (node.op() == "OnesLike") return true;
+  if (IsOnesLike(node)) return true;
+  if (IsZerosLike(node)) return false;
   if (node.op() == "Fill") {
     NodeDef* values = node_map_->GetNode(NodeName(node.input(1)));
     return values != nullptr && IsOnes(*values);
@@ -1428,7 +1434,8 @@ bool ConstantFolding::IsZeros(const NodeDef& node) const {
   if (feed_nodes_.find(node.name()) != feed_nodes_.end()) {
     return false;
   }
-  if (node.op() == "ZerosLike") return true;
+  if (IsOnesLike(node)) return false;
+  if (IsZerosLike(node)) return true;
   if (node.op() == "Fill") {
     NodeDef* values = node_map_->GetNode(NodeName(node.input(1)));
     return values != nullptr && IsZeros(*values);
diff --git a/tensorflow/core/grappler/optimizers/constant_folding.h b/tensorflow/core/grappler/optimizers/constant_folding.h
index 99200925cb..4c532d7af1 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding.h
+++ b/tensorflow/core/grappler/optimizers/constant_folding.h
@@ -78,6 +78,7 @@ class ConstantFolding : public GraphOptimizer {
   Status EvaluateOneFoldable(const NodeDef& node, std::vector<NodeDef>* outputs,
                              bool* result_too_large);
 
+  Status FoldMergeNode(NodeDef* node, GraphDef* output_graph);
   Status FoldNode(NodeDef* node, GraphDef* output_graph,
                   bool* result_too_large);
 
-- 
GitLab


From bb79db7ce5f26483d9770f6fe21719a23d5d5e32 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 5 Feb 2019 15:15:24 -0800
Subject: [PATCH 1959/2345] Changed embedding layer gradient accumulation (in
 low-level API) to an enum and defaulted it to enabled.

PiperOrigin-RevId: 232567988
---
 .../tpu/proto/optimization_parameters.proto   | 34 ++++++++++++++-----
 .../contrib/tpu/python/tpu/tpu_embedding.py   |  6 ++++
 2 files changed, 31 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/tpu/proto/optimization_parameters.proto b/tensorflow/contrib/tpu/proto/optimization_parameters.proto
index bc50c613f3..a4ca7f3986 100644
--- a/tensorflow/contrib/tpu/proto/optimization_parameters.proto
+++ b/tensorflow/contrib/tpu/proto/optimization_parameters.proto
@@ -89,11 +89,11 @@ message FtrlParameters {
 // the normal version of Adam that updates all parameters in the embedding
 // table, even for entries that are not used in the current minibatch
 // (https://www.tensorflow.org/api_docs/python/tf/contrib/opt/AdamOptimizer). If
-// use_non_lazy_adam is enabled, use_gradient_accumulation is also required in
-// order to get correct results; a warning will be printed otherwise (which may
-// change to an error in the future). If use_sum_inside_sqrt is set, the Adam
-// variable update formula will be changed from m / (sqrt(v) + epsilon) to
-// m / sqrt(v + epsilon**2); this option improves the performance of TPU
+// use_non_lazy_adam is enabled, gradient accumulation is also required to be
+// enabled in order to get correct results; a warning will be printed otherwise
+// (which may change to an error in the future). If use_sum_inside_sqrt is set,
+// the Adam variable update formula will be changed from m / (sqrt(v) + epsilon)
+// to m / sqrt(v + epsilon**2); this option improves the performance of TPU
 // training and is not expected to harm model quality.
 message AdamParameters {
   float beta1 = 3;
@@ -170,6 +170,20 @@ message ProximalAdagradParameters {
   float initial_accumulator = 3;
 }
 
+// Status of using gradient accumulation (doing two passes over the input
+// gradients: one to accumulate them into a temporary array and another to apply
+// them using the actual optimization algorithm). The extra message is to wrap
+// the enum for scoping.
+message GradientAccumulationStatus {
+  // Defaults to value of use_gradient_accumulation (temporary compatibility
+  // behavior).
+  enum Status {
+    UNSPECIFIED = 0;
+    ENABLED = 1;
+    DISABLED = 2;
+  }
+};
+
 message OptimizationParameters {
   // Learning rate used for updating the embedding layer parameters.
   LearningRate learning_rate = 13;
@@ -191,11 +205,13 @@ message OptimizationParameters {
   // once per minibatch.
   float weight_decay_factor = 16;
 
-  // Whether to use gradient accumulation (do two passes over the input
+  // Status of using gradient accumulation (doing two passes over the input
   // gradients: one to accumulate them into a temporary array and another to
-  // apply them using the actual optimization algorithm). This feature is
-  // experimental -- it has not been fully verified and may cause training
-  // crashes and/or failures.
+  // apply them using the actual optimization algorithm).
+  GradientAccumulationStatus.Status gradient_accumulation_status = 17;
+
+  // Old gradient accumulation flag; overridden by gradient_accumulation_status
+  // when it is set.
   bool use_gradient_accumulation = 15;
 
   // Optimization algorithm parameters; which field is selected determines which
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py b/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py
index 0e4597bd6f..1689126dbc 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py
@@ -26,6 +26,7 @@ import six
 
 from tensorflow.contrib.framework.python.framework import experimental
 from tensorflow.contrib.tpu.ops import gen_tpu_ops
+from tensorflow.contrib.tpu.proto import optimization_parameters_pb2
 from tensorflow.contrib.tpu.proto import tpu_embedding_configuration_pb2 as elc
 from tensorflow.contrib.tpu.python.ops import tpu_ops
 from tensorflow.contrib.tpu.python.tpu import tpu_system_metadata as tpu_system_metadata_lib
@@ -473,6 +474,11 @@ class TPUEmbedding(object):
 
       table_descriptor.optimization_parameters.learning_rate.constant = (
           self._optimization_parameters.learning_rate)
+      table_descriptor.optimization_parameters.gradient_accumulation_status = (
+          optimization_parameters_pb2.GradientAccumulationStatus.ENABLED
+          if self._optimization_parameters.use_gradient_accumulation else
+          optimization_parameters_pb2.GradientAccumulationStatus.DISABLED)
+      # For compatibility with old TPU workers.
       table_descriptor.optimization_parameters.use_gradient_accumulation = (
           self._optimization_parameters.use_gradient_accumulation)
       self._optimizer_handler.set_optimization_parameters(table_descriptor)
-- 
GitLab


From 5b93006575591103f2c88e05470fc8c2161f7be2 Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Tue, 5 Feb 2019 15:32:32 -0800
Subject: [PATCH 1960/2345] Fix TFLite makefile build

References the proper logging implementation on each platform.

PiperOrigin-RevId: 232571245
---
 tensorflow/lite/tools/make/Makefile | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tensorflow/lite/tools/make/Makefile b/tensorflow/lite/tools/make/Makefile
index e98ba9b2e0..5c4bb4dbf8 100644
--- a/tensorflow/lite/tools/make/Makefile
+++ b/tensorflow/lite/tools/make/Makefile
@@ -131,6 +131,14 @@ else
 	CORE_CC_EXCLUDE_SRCS += tensorflow/lite/nnapi_delegate.cc
 endif
 
+ifeq ($(TARGET),ios)
+	CORE_CC_EXCLUDE_SRCS += tensorflow/lite/minimal_logging_android.cc
+	CORE_CC_EXCLUDE_SRCS += tensorflow/lite/minimal_logging_default.cc
+else
+	CORE_CC_EXCLUDE_SRCS += tensorflow/lite/minimal_logging_android.cc
+	CORE_CC_EXCLUDE_SRCS += tensorflow/lite/minimal_logging_ios.cc
+endif
+
 
 # Filter out all the excluded files.
 TF_LITE_CC_SRCS := $(filter-out $(CORE_CC_EXCLUDE_SRCS), $(CORE_CC_ALL_SRCS))
-- 
GitLab


From c9515b97875014cef1fadb86e88031def2b704b3 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Tue, 5 Feb 2019 15:33:07 -0800
Subject: [PATCH 1961/2345] [XLA] Add unit_diagonal option to TriangularSolve.

BLAS implementations of TRSM usually have a "unit_diagonal" option that allows users to specify that the elements on the diagonal should be ignored. This is frequently useful if both lower and upper triangular matrices are packed into the same matrix and the diagonal only belongs to one of them. For example, this is true in the case of LU decomposition in JAX:
https://github.com/google/jax/blob/master/jax/numpy/linalg.py#L153
This change pushes that logic into the TriangularSolve() operator, which will allow us to remove it from JAX.

This change is also in preparation for adding a first class TriangularSolve HLO that calls into BLAS implementations where applicable. We would like the HLO operator to support the full feature set of any underlying BLAS implementations.

PiperOrigin-RevId: 232571346
---
 .../kernels/matrix_triangular_solve_op.cc     |  3 +-
 .../compiler/xla/client/lib/cholesky.cc       |  1 +
 .../xla/client/lib/triangular_solve.cc        | 23 ++++--
 .../xla/client/lib/triangular_solve.h         |  4 +-
 .../xla/client/lib/triangular_solve_test.cc   | 71 ++++++++++++++++++-
 .../xla/python/local_computation_builder.cc   | 10 ++-
 .../xla/python/local_computation_builder.h    |  3 +-
 tensorflow/compiler/xla/python/xla_client.py  | 14 ++--
 8 files changed, 108 insertions(+), 21 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/matrix_triangular_solve_op.cc b/tensorflow/compiler/tf2xla/kernels/matrix_triangular_solve_op.cc
index 90c0ebefb2..b3224953d5 100644
--- a/tensorflow/compiler/tf2xla/kernels/matrix_triangular_solve_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/matrix_triangular_solve_op.cc
@@ -31,7 +31,8 @@ class MatrixTriangularSolveOp : public XlaOpKernel {
   void Compile(XlaOpKernelContext* ctx) override {
     auto result = xla::TriangularSolve(
         ctx->Input(0), ctx->Input(1), /*left_side=*/true,
-        /*lower=*/lower_, /*transpose_a=*/adjoint_, /*conjugate_a=*/adjoint_);
+        /*lower=*/lower_, /*transpose_a=*/adjoint_, /*conjugate_a=*/adjoint_,
+        /*unit_diagonal=*/false);
     ctx->SetOutput(0, result);
   }
 
diff --git a/tensorflow/compiler/xla/client/lib/cholesky.cc b/tensorflow/compiler/xla/client/lib/cholesky.cc
index 414bd1494c..4578fff9d1 100644
--- a/tensorflow/compiler/xla/client/lib/cholesky.cc
+++ b/tensorflow/compiler/xla/client/lib/cholesky.cc
@@ -199,6 +199,7 @@ XlaOp Cholesky(XlaOp a, int64 block_size,
                                       /*lower=*/true,
                                       /*transpose_a=*/true,
                                       /*conjugate_a=*/false,
+                                      /*unit_diagonal=*/false,
                                       /*block_size=*/block_size);
         l = UpdateSliceInMinorDims(l, update, {i + k, i});
       }
diff --git a/tensorflow/compiler/xla/client/lib/triangular_solve.cc b/tensorflow/compiler/xla/client/lib/triangular_solve.cc
index ba7fde118f..1515e9ef7a 100644
--- a/tensorflow/compiler/xla/client/lib/triangular_solve.cc
+++ b/tensorflow/compiler/xla/client/lib/triangular_solve.cc
@@ -346,8 +346,8 @@ XlaOp SolveWithInvertedDiagonalBlocks(XlaOp a, XlaOp b, XlaOp inv_diag_blocks,
 }
 
 XlaOp TriangularSolve(XlaOp a, XlaOp b, bool left_side, bool lower,
-                      bool transpose_a, bool conjugate_a, int64 block_size,
-                      PrecisionConfig::Precision precision) {
+                      bool transpose_a, bool conjugate_a, bool unit_diagonal,
+                      int64 block_size, PrecisionConfig::Precision precision) {
   XlaBuilder* builder = a.builder();
   return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
     TF_ASSIGN_OR_RETURN(Shape a_shape, builder->GetShape(a));
@@ -406,6 +406,20 @@ XlaOp TriangularSolve(XlaOp a, XlaOp b, bool left_side, bool lower,
       return b;
     }
 
+    // TODO(phawkins): consider pushing triangle masking into
+    // InvertDiagonalBlocks.
+    if (unit_diagonal) {
+      // Mask everything but the subdiagonal/superdiagonal elements.
+      a = lower ? Select(TriangleMask(a, -1), a, ZerosLike(a))
+                : Select(TriangleMask(a, 0), ZerosLike(a), a);
+      int64 k = ShapeUtil::GetDimension(a_shape, -1);
+      a = xla::Add(a, IdentityMatrix(builder, a_shape.element_type(), k, k),
+                   /*broadcast_dimensions=*/{ndims - 2, ndims - 1});
+    } else {
+      // Mask off the ignored elements of the triangular matrix a.
+      a = Triangle(a, lower);
+    }
+
     // We find the diagonal blocks of the coefficient matrix
     auto diag_blocks = DiagonalBlocks(a, block_size);
 
@@ -413,11 +427,6 @@ XlaOp TriangularSolve(XlaOp a, XlaOp b, bool left_side, bool lower,
     auto inv_diag_blocks = InvertDiagonalBlocks(diag_blocks, lower, transpose_a,
                                                 conjugate_a, precision);
 
-    // Mask off the ignored elements of the triangular matrix a.
-    // TODO(phawkins): it would probably be preferable to perform this masking
-    // block by block inside SolveWithInvertedDiagonalBlocks.
-    a = Triangle(a, lower);
-
     // We now find the solution using GEMMs
     auto x =
         SolveWithInvertedDiagonalBlocks(a, b, inv_diag_blocks, left_side, lower,
diff --git a/tensorflow/compiler/xla/client/lib/triangular_solve.h b/tensorflow/compiler/xla/client/lib/triangular_solve.h
index 50a3b30ebd..b87ef7288f 100644
--- a/tensorflow/compiler/xla/client/lib/triangular_solve.h
+++ b/tensorflow/compiler/xla/client/lib/triangular_solve.h
@@ -54,12 +54,14 @@ namespace xla {
 // `conjugate_a` is a boolean indicating whether the entries of `a` are complex
 // conjugated (independently of whether they are transposed), so that when both
 // transpose_a and conjugate_a are true the effect is a Hermitian adjoint.
+// If `unit_diagonal` elements on the matrix diagonal are assumed to be '1' and
+// are not read by the triangular solve..
 //
 // Uses a blocked algorithm if `block_size` is > 1; if block_size == 1 then no
 // blocking is used.
 XlaOp TriangularSolve(
     XlaOp a, XlaOp b, bool left_side, bool lower, bool transpose_a,
-    bool conjugate_a, int64 block_size = 128,
+    bool conjugate_a, bool unit_diagonal, int64 block_size = 128,
     PrecisionConfig::Precision precision = PrecisionConfig::HIGHEST);
 
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/client/lib/triangular_solve_test.cc b/tensorflow/compiler/xla/client/lib/triangular_solve_test.cc
index 284a2e9d18..b333ffa917 100644
--- a/tensorflow/compiler/xla/client/lib/triangular_solve_test.cc
+++ b/tensorflow/compiler/xla/client/lib/triangular_solve_test.cc
@@ -54,6 +54,20 @@ Array2D<float> AValsUpper() {
           {kNan, kNan, kNan, 11}};
 }
 
+Array2D<float> AValsLowerUnitDiagonal() {
+  return {{kNan, kNan, kNan, kNan},
+          {3, kNan, kNan, kNan},
+          {4, 7, kNan, kNan},
+          {5, 8, 10, kNan}};
+}
+
+Array2D<float> AValsUpperUnitDiagonal() {
+  return {{kNan, 3, 4, 5},
+          {kNan, kNan, 7, 8},
+          {kNan, kNan, kNan, 10},
+          {kNan, kNan, kNan, kNan}};
+}
+
 Array2D<float> BValsRight() {
   return {{1, 2, 3, 4}, {5, 6, 7, 8}, {9, 10, 11, 12}};
 }
@@ -97,6 +111,7 @@ XLA_TEST_F(TriangularSolveTest, EmptyArrays) {
   TriangularSolve(a, b,
                   /*left_side=*/true, /*lower=*/true,
                   /*transpose_a=*/true, /*conjugate_a=*/false,
+                  /*unit_diagonal=*/false,
                   /*block_size=*/2);
 
   ComputeAndCompareR2<float>(&builder, Array2D<float>(0, 10),
@@ -112,6 +127,7 @@ XLA_TEST_F(TriangularSolveTest, SimpleRightLowerTranspose) {
   TriangularSolve(a, b,
                   /*left_side=*/false, /*lower=*/true,
                   /*transpose_a=*/true, /*conjugate_a=*/false,
+                  /*unit_diagonal=*/false,
                   /*block_size=*/2);
 
   Array2D<float> expected({
@@ -133,6 +149,7 @@ XLA_TEST_F(TriangularSolveTest, SimpleRightLowerNotranspose) {
   TriangularSolve(a, b,
                   /*left_side=*/false, /*lower=*/true,
                   /*transpose_a=*/false, /*conjugate_a=*/false,
+                  /*unit_diagonal=*/false,
                   /*block_size=*/2);
 
   Array2D<float> expected({
@@ -154,6 +171,7 @@ XLA_TEST_F(TriangularSolveTest, SimpleRightUpperTranspose) {
   TriangularSolve(a, b,
                   /*left_side=*/false, /*lower=*/false,
                   /*transpose_a=*/true, /*conjugate_a=*/false,
+                  /*unit_diagonal=*/false,
                   /*block_size=*/2);
 
   Array2D<float> expected({
@@ -175,6 +193,7 @@ XLA_TEST_F(TriangularSolveTest, SimpleRightUpperNotranspose) {
   TriangularSolve(a, b,
                   /*left_side=*/false, /*lower=*/false,
                   /*transpose_a=*/false, /*conjugate_a=*/false,
+                  /*unit_diagonal=*/false,
                   /*block_size=*/2);
 
   Array2D<float> expected({
@@ -196,6 +215,7 @@ XLA_TEST_F(TriangularSolveTest, SimpleLeftLowerTranspose) {
   TriangularSolve(a, b,
                   /*left_side=*/true, /*lower=*/true,
                   /*transpose_a=*/true, /*conjugate_a=*/false,
+                  /*unit_diagonal=*/false,
                   /*block_size=*/2);
 
   Array2D<float> expected({
@@ -218,6 +238,7 @@ XLA_TEST_F(TriangularSolveTest, SimpleLeftLowerNotranspose) {
   TriangularSolve(a, b,
                   /*left_side=*/true, /*lower=*/true,
                   /*transpose_a=*/false, /*conjugate_a=*/false,
+                  /*unit_diagonal=*/false,
                   /*block_size=*/2);
 
   Array2D<float> expected({
@@ -231,6 +252,26 @@ XLA_TEST_F(TriangularSolveTest, SimpleLeftLowerNotranspose) {
                              ErrorSpec(1e-2, 1e-2));
 }
 
+XLA_TEST_F(TriangularSolveTest, SimpleLeftLowerNoTransposeUnitDiagonal) {
+  XlaBuilder builder(TestName());
+
+  XlaOp a, b;
+  auto a_data =
+      CreateR2Parameter<float>(AValsLowerUnitDiagonal(), 0, "a", &builder, &a);
+  auto b_data = CreateR2Parameter<float>(BValsLeft(), 1, "b", &builder, &b);
+  TriangularSolve(a, b,
+                  /*left_side=*/true, /*lower=*/true,
+                  /*transpose_a=*/false, /*conjugate_a=*/false,
+                  /*unit_diagonal=*/true,
+                  /*block_size=*/2);
+
+  Array2D<float> expected(
+      {{1., 2., 3.}, {1., -1., -3.}, {-4., 7., 18.}, {37., -61., -159.}});
+
+  ComputeAndCompareR2<float>(&builder, expected, {a_data.get(), b_data.get()},
+                             ErrorSpec(1e-2, 1e-2));
+}
+
 XLA_TEST_F(TriangularSolveTest, SimpleLeftLowerNotransposeIrregularblock) {
   XlaBuilder builder(TestName());
 
@@ -240,6 +281,7 @@ XLA_TEST_F(TriangularSolveTest, SimpleLeftLowerNotransposeIrregularblock) {
   TriangularSolve(a, b,
                   /*left_side=*/true, /*lower=*/true,
                   /*transpose_a=*/false, /*conjugate_a=*/false,
+                  /*unit_diagonal=*/false,
                   /*block_size=*/3);
 
   Array2D<float> expected({
@@ -262,6 +304,7 @@ XLA_TEST_F(TriangularSolveTest, SimpleLeftUpperTranspose) {
   TriangularSolve(a, b,
                   /*left_side=*/true, /*lower=*/false,
                   /*transpose_a=*/true, /*conjugate_a=*/false,
+                  /*unit_diagonal=*/false,
                   /*block_size=*/2);
 
   Array2D<float> expected({
@@ -284,6 +327,7 @@ XLA_TEST_F(TriangularSolveTest, SimpleLeftUpperNotranspose) {
   TriangularSolve(a, b,
                   /*left_side=*/true, /*lower=*/false,
                   /*transpose_a=*/false, /*conjugate_a=*/false,
+                  /*unit_diagonal=*/false,
                   /*block_size=*/2);
 
   Array2D<float> expected({
@@ -297,6 +341,28 @@ XLA_TEST_F(TriangularSolveTest, SimpleLeftUpperNotranspose) {
                              ErrorSpec(1e-2, 1e-2));
 }
 
+XLA_TEST_F(TriangularSolveTest, SimpleLeftUpperNotransposeUnitDiagonal) {
+  XlaBuilder builder(TestName());
+
+  XlaOp a, b;
+  auto a_data =
+      CreateR2Parameter<float>(AValsUpperUnitDiagonal(), 0, "a", &builder, &a);
+  auto b_data = CreateR2Parameter<float>(BValsLeft(), 1, "b", &builder, &b);
+  TriangularSolve(a, b,
+                  /*left_side=*/true, /*lower=*/false,
+                  /*transpose_a=*/false, /*conjugate_a=*/false,
+                  /*unit_diagonal=*/true,
+                  /*block_size=*/2);
+
+  Array2D<float> expected({{-1402., -1538., -1674.},
+                           {575., 631., 687.},
+                           {-93., -102., -111.},
+                           {10., 11., 12.}});
+
+  ComputeAndCompareR2<float>(&builder, expected, {a_data.get(), b_data.get()},
+                             ErrorSpec(1e-2, 1e-2));
+}
+
 XLA_TEST_F(TriangularSolveTest, SimpleRightLowerTransposeConjugate) {
   XlaBuilder builder(TestName());
 
@@ -308,6 +374,7 @@ XLA_TEST_F(TriangularSolveTest, SimpleRightLowerTransposeConjugate) {
   TriangularSolve(a, b,
                   /*left_side=*/false, /*lower=*/true,
                   /*transpose_a=*/true, /*conjugate_a=*/true,
+                  /*unit_diagonal=*/false,
                   /*block_size=*/2);
 
   Array2D<complex64> expected({
@@ -334,6 +401,7 @@ XLA_TEST_F(TriangularSolveTest, SimpleLeftUpperTransposeNoconjugate) {
   TriangularSolve(a, b,
                   /*left_side=*/true, /*lower=*/false,
                   /*transpose_a=*/true, /*conjugate_a=*/false,
+                  /*unit_diagonal=*/false,
                   /*block_size=*/2);
 
   Array2D<complex64> expected({
@@ -372,6 +440,7 @@ XLA_TEST_F(TriangularSolveTest, BatchedLeftUpper) {
            TriangularSolve(a, b,
                            /*left_side=*/true, /*lower=*/false,
                            /*transpose_a=*/false, /*conjugate_a=*/false,
+                           /*unit_diagonal=*/false,
                            /*block_size=*/2));
 
   ComputeAndCompareR3<float>(&builder, bvals, {a_data.get(), b_data.get()},
@@ -409,7 +478,7 @@ XLA_TEST_P(TriangularSolveParametricTest, Random) {
   auto a_data = CreateR2Parameter<float>(avals, 0, "a", &builder, &a);
   auto b_data = CreateR2Parameter<float>(bvals, 1, "b", &builder, &b);
   auto x = TriangularSolve(a, b, spec.left_side, spec.lower, spec.transpose_a,
-                           /*conjugate_a=*/false,
+                           /*conjugate_a=*/false, /*unit_diagonal=*/false,
                            /*block_size=*/3);
   auto a_tri = Triangle(a, spec.lower);
   a_tri = MaybeTransposeInMinorDims(a_tri, spec.transpose_a);
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.cc b/tensorflow/compiler/xla/python/local_computation_builder.cc
index c0daf3c13e..08913809c4 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.cc
+++ b/tensorflow/compiler/xla/python/local_computation_builder.cc
@@ -928,13 +928,11 @@ LocalOp LocalComputationBuilder::QR(const LocalOp& a, bool full_matrices) {
   });
 }
 
-LocalOp LocalComputationBuilder::TriangularSolve(const LocalOp& a,
-                                                 const LocalOp& b,
-                                                 bool left_side, bool lower,
-                                                 bool transpose_a,
-                                                 bool conjugate_a) {
+LocalOp LocalComputationBuilder::TriangularSolve(
+    const LocalOp& a, const LocalOp& b, bool left_side, bool lower,
+    bool transpose_a, bool conjugate_a, bool unit_diagonal) {
   return xla::TriangularSolve(a.op(), b.op(), left_side, lower, transpose_a,
-                              conjugate_a);
+                              conjugate_a, unit_diagonal);
 }
 
 LocalOp LocalComputationBuilder::Gather(
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.h b/tensorflow/compiler/xla/python/local_computation_builder.h
index e3af88f825..5e65ecfeef 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.h
+++ b/tensorflow/compiler/xla/python/local_computation_builder.h
@@ -425,7 +425,8 @@ class LocalComputationBuilder {
   LocalOp Cholesky(const LocalOp& a);
 
   LocalOp TriangularSolve(const LocalOp& a, const LocalOp& b, bool left_side,
-                          bool lower, bool transpose_a, bool conjugate_a);
+                          bool lower, bool transpose_a, bool conjugate_a,
+                          bool unit_diagonal);
 
   LocalOp Gather(const LocalOp& input, const LocalOp& start_indices,
                  const GatherDimensionNumbers& dimension_numbers,
diff --git a/tensorflow/compiler/xla/python/xla_client.py b/tensorflow/compiler/xla/python/xla_client.py
index 43eeebc6d8..6a7a27dd61 100644
--- a/tensorflow/compiler/xla/python/xla_client.py
+++ b/tensorflow/compiler/xla/python/xla_client.py
@@ -1613,11 +1613,17 @@ class ComputationBuilder(object):
     """Enqueues a QR decomposition onto the computation."""
     return self._client.QR(a, full_matrices)
 
-  def TriangularSolve(self, a, b, left_side=False, lower=False,
-                      transpose_a=False, conjugate_a=False):
+  def TriangularSolve(self,
+                      a,
+                      b,
+                      left_side=False,
+                      lower=False,
+                      transpose_a=False,
+                      conjugate_a=False,
+                      unit_diagonal=False):
     """Enqueues a triangular-solve operation onto the computation."""
-    return self._client.TriangularSolve(
-        a, b, left_side, lower, transpose_a, conjugate_a)
+    return self._client.TriangularSolve(a, b, left_side, lower, transpose_a,
+                                        conjugate_a, unit_diagonal)
 
   def Gather(self, a, start_indices, dimension_numbers, slice_sizes):
     """Enqueues a Gather operation onto the computation."""
-- 
GitLab


From 0dd3a69452500032b04c16e4c6894da47f118a39 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 5 Feb 2019 15:44:26 -0800
Subject: [PATCH 1962/2345] Internal change.

PiperOrigin-RevId: 232573441
---
 tensorflow/lite/kernels/fully_connected.cc    |  4 ++-
 .../lite/kernels/fully_connected_test.cc      | 31 +++++++++++++++++--
 2 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/tensorflow/lite/kernels/fully_connected.cc b/tensorflow/lite/kernels/fully_connected.cc
index dfc9550ed6..7b4d29cc91 100644
--- a/tensorflow/lite/kernels/fully_connected.cc
+++ b/tensorflow/lite/kernels/fully_connected.cc
@@ -212,7 +212,9 @@ TfLiteStatus EvalHybrid(TfLiteContext* context, TfLiteNode* node,
   TF_LITE_ENSURE_EQ(context, input->type, kTfLiteFloat32);
   TF_LITE_ENSURE(context,
                  filter->type == kTfLiteUInt8 || filter->type == kTfLiteInt8);
-  TF_LITE_ENSURE_EQ(context, bias->type, kTfLiteFloat32);
+  if (bias) {
+    TF_LITE_ENSURE_EQ(context, bias->type, kTfLiteFloat32);
+  }
   TF_LITE_ENSURE_EQ(context, output->type, kTfLiteFloat32);
 
   int total_input_size = 1;
diff --git a/tensorflow/lite/kernels/fully_connected_test.cc b/tensorflow/lite/kernels/fully_connected_test.cc
index 03f4ea7143..31aa3f36a3 100644
--- a/tensorflow/lite/kernels/fully_connected_test.cc
+++ b/tensorflow/lite/kernels/fully_connected_test.cc
@@ -137,6 +137,7 @@ class BaseFullyConnectedOpModel : public SingleOpModel {
   BaseFullyConnectedOpModel(
       TfLiteRegistration* registration, int units, int batches,
       const TensorData& input, const TensorData& output = {TensorType_FLOAT32},
+      bool bias_tensor_optional = false,
       ActivationFunctionType activation_func = ActivationFunctionType_RELU,
       FullyConnectedOptionsWeightsFormat weights_format =
           FullyConnectedOptionsWeightsFormat_DEFAULT)
@@ -151,7 +152,9 @@ class BaseFullyConnectedOpModel : public SingleOpModel {
     weights_ =
         AddInput({input.type, {units_, input_size_}, input.min, input.max});
 
-    if (input.type == TensorType_FLOAT32) {
+    if (bias_tensor_optional) {
+      bias_ = AddNullInput();
+    } else if (input.type == TensorType_FLOAT32) {
       bias_ = AddInput({TensorType_FLOAT32, {units_}});
     } else {
       // This is a quantized version. The scale of 'bias' depends on the scales
@@ -173,7 +176,9 @@ class BaseFullyConnectedOpModel : public SingleOpModel {
             .Union());
     resolver_ = absl::make_unique<SingleOpResolver>(
         BuiltinOperator_FULLY_CONNECTED, registration);
-    BuildInterpreter({GetShape(input_), GetShape(weights_), GetShape(bias_)});
+    BuildInterpreter(
+        {GetShape(input_), GetShape(weights_),
+         (bias_ == kOptionalTensor) ? std::vector<int>() : GetShape(bias_)});
   }
 
   int input_size() { return input_size_; }
@@ -397,6 +402,27 @@ TEST_P(FloatFullyConnectedOpTest, SimpleTest2) {
   EXPECT_THAT(m.GetOutput(), ElementsAre(11, 9));
 }
 
+TEST(FloatFullyConnectedOpTest, SimpleTestNoBias) {
+  // The optimized kernel assumes that the bias is specified.
+  FloatFullyConnectedOpModel m(ops::builtin::Register_FULLY_CONNECTED_PIE(),
+                               /*units=*/1, /*batches=*/2,
+                               /*input=*/{TensorType_FLOAT32, {2, 2}},
+                               /*output=*/{TensorType_FLOAT32},
+                               /*bias_tensor_optional=*/true);
+  m.SetWeights({
+      2, 4,  // u = 0
+  });
+
+  m.SetInput({
+      1, 2,  // b = 0
+      2, 1,  // b = 1
+  });
+
+  m.Invoke();
+
+  EXPECT_THAT(m.GetOutput(), ElementsAre(10, 8));
+}
+
 TEST_P(QuantizedFullyConnectedOpTest, SimpleTestQuantized) {
   QuantizedFullyConnectedOpModel m(
       GetRegistration(), /*units=*/3, /*batches*/ 2,
@@ -477,6 +503,7 @@ void SimpleTestQuantizedInt16OutputCase(
       /*input=*/
       {TensorType_UINT8, {batches, input_depth}, kInputMin, kInputMax},
       /*output=*/{TensorType_INT16, {}, kOutputMin, kOutputMax},
+      /*bias_tensor_optional=*/false,
       /*activation_func=*/ActivationFunctionType_NONE, weights_format);
 
   std::mt19937 random_engine;
-- 
GitLab


From a0a4d37e44419edc582c069ecd2de15b6d0c19ac Mon Sep 17 00:00:00 2001
From: Yuan Lin <yulin@nvidia.com>
Date: Tue, 5 Feb 2019 15:55:05 -0800
Subject: [PATCH 1963/2345] [XLA] Add support to specify boundary nodes in
 interactive_grahviz tool.

The enhanced command is `<instr> [<width>] [/ <boundary_instr1> <boundary_instr2> ...]`.

The boundary nodes are optional. This is useful in cases where one wants to furhter prune
a graph when using a large <width>.
---
 .../compiler/xla/service/hlo_graph_dumper.cc  | 11 ++--
 .../compiler/xla/service/hlo_graph_dumper.h   |  9 +++-
 .../xla/tools/interactive_graphviz.cc         | 52 ++++++++++++++-----
 3 files changed, 53 insertions(+), 19 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
index 46ee99923e..810b1e1dfc 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
@@ -1281,8 +1281,9 @@ namespace {
 
 // Gets a NodeFilter that includes roughly all instructions whose distance from
 // root is <= radius.
-NodeFilter MakeNodeRadiusAroundFilter(const HloInstruction* root,
-                                      int64 radius) {
+NodeFilter MakeNodeRadiusAroundFilter(
+    const HloInstruction* root, int64 radius,
+    const std::set<const HloInstruction*>* boundary) {
   // First, find the neighborhood of nodes with distance from root <= radius.
   // These nodes are our initial set of "normal" nodes.
   absl::flat_hash_map<const HloInstruction*, NodeFilterResult> nodes;
@@ -1298,6 +1299,9 @@ NodeFilter MakeNodeRadiusAroundFilter(const HloInstruction* root,
     if (depth == radius) {
       continue;
     }
+    if (boundary->count(instr) != 0) {
+      continue;
+    }
 
     // Traverse into instr's operands.
     //
@@ -1513,11 +1517,12 @@ string DumpGraph(const HloComputation& computation, const string& label,
 }
 
 string DumpNeighborhoodAround(const HloInstruction& node, int radius,
+                              const std::set<const HloInstruction*>* boundary,
                               bool show_backend_config) {
   auto debug_options = node.GetModule()->config().debug_options();
   string label =
       StrCat("Neighborhood of ", radius, " nodes around ", node.name());
-  NodeFilter filter = MakeNodeRadiusAroundFilter(&node, radius);
+  NodeFilter filter = MakeNodeRadiusAroundFilter(&node, radius, boundary);
   string graph =
       HloDotDumper(node.parent(), label, debug_options, show_backend_config,
                    /*profile=*/nullptr, filter)
diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.h b/tensorflow/compiler/xla/service/hlo_graph_dumper.h
index 8e51454ef1..755009b817 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.h
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.h
@@ -63,8 +63,13 @@ string DumpGraph(const HloComputation& computation, const string& label,
 // The number of nodes dumped is controlled by the radius parameter, which
 // (roughly) corresponds to the max distance a node may be from the primary node
 // before it's omitted from the graph.
-string DumpNeighborhoodAround(const HloInstruction& node, int radius,
-                              bool show_backend_config = false);
+// 
+// The optional boundary parameter specifies the set of boundary nodes which
+// will be omitted when they are within the radius.
+string DumpNeighborhoodAround(
+    const HloInstruction& node, int radius,
+    const std::set<const HloInstruction*>* boundary = nullptr,
+    bool show_backend_config = false);
 
 // Dumps nodes on any of the paths from `from` to `to`.  If there are more than
 // max_nodes on all paths, restricts to the max_nodes nodes on the shortest
diff --git a/tensorflow/compiler/xla/tools/interactive_graphviz.cc b/tensorflow/compiler/xla/tools/interactive_graphviz.cc
index ac865707f8..5d90e36975 100644
--- a/tensorflow/compiler/xla/tools/interactive_graphviz.cc
+++ b/tensorflow/compiler/xla/tools/interactive_graphviz.cc
@@ -139,9 +139,10 @@ HloComputation* FindComputation(const HloModule& module,
 // Print a help message describing the various available commands.
 void DoHelpCommand() {
   std::cout << R"(Commands:
-  <instruction> [<width>]
-    Renders a neighborhood of <width> nodes around <instruction>.  If <width>
-    is not provided, the default value is )"
+  <instruction> [<width>] [/ <boundary_instruction>+]
+    Renders a neighborhood of <width> nodes around <instruction>, without going
+    beyond the optional boundary instructions.  If <width> is not provided, 
+    the default value is )"
             << kDefaultWidth << R"(.
   allpaths <instruction> <instruction> [<n>]
     Renders a subset of all paths from one instruction to the other.  Either
@@ -457,12 +458,6 @@ void DoAllPathsCommand(const Options& opts, const HloModule& module,
 // Plot a given instruction neighborhood or computation with graphviz.
 void DoPlotCommand(const Options& opts, const HloModule& module,
                    const std::vector<string>& tokens) {
-  if (tokens.size() > 2) {
-    std::cerr << R"(Illegal input.  Enter e.g. "%fusion.1 42" or "%fusion.1".)"
-              << std::endl;
-    return;
-  }
-
   string node_name = tokens[0];
 
   // Find the node with the given name.
@@ -475,16 +470,43 @@ void DoPlotCommand(const Options& opts, const HloModule& module,
   }
 
   uint64 graph_width = kDefaultWidth;
-  if (tokens.size() == 2) {
+  std::set<const HloInstruction*> boundary;
+  if (tokens.size() >= 2) {
     if (comp) {
       std::cerr << "Can only use graph-size parameter with instructions, but "
                 << node_name << " is a computation." << std::endl;
       return;
     }
+
+    int bound_index = tokens.size();
     if (!absl::SimpleAtoi(tokens[1], &graph_width)) {
-      std::cerr << "Can't parse '" << tokens[1] << "' as an integer."
-                << std::endl;
-      return;
+      if (tokens[1] != "/") {
+        std::cerr << "Can't parse '" << tokens[1] << "' as an integer."
+                  << std::endl;
+        return;
+      }
+      graph_width = kDefaultWidth;
+      bound_index = 2;
+    } else {
+      if (tokens.size() > 2) {
+        if (tokens[2] != "/") {
+          std::cerr << "Expect a /, but get a '" << tokens[1] << "'."
+                    << std::endl;
+          return;
+        }
+        bound_index = 3;
+      }
+    }
+    while (bound_index < tokens.size()) {
+      string bnode_name = tokens[bound_index];
+      const HloInstruction* binstr = FindInstruction(module, bnode_name);
+      if (!binstr) {
+        std::cerr << "Couldn't find HloInstruction named " << node_name << "."
+                  << std::endl;
+        return;
+      }
+      boundary.insert(binstr);
+      bound_index++;
     }
   }
 
@@ -496,7 +518,9 @@ void DoPlotCommand(const Options& opts, const HloModule& module,
         /*show_backend_config=*/show_backend_config));
   } else {
     DisplayGraphHandle(opts, hlo_graph_dumper::DumpNeighborhoodAround(
-        *instr, graph_width, /*show_backend_config=*/show_backend_config));
+                                 *instr, graph_width,
+                                 /*boundary=*/&boundary,
+                                 /*show_backend_config=*/show_backend_config));
   }
 }
 
-- 
GitLab


From e35778a865142289dd749ea2c907e13b7756af8b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 5 Feb 2019 16:01:42 -0800
Subject: [PATCH 1964/2345] Enable OSS Profiler to trace CPU activities.

PiperOrigin-RevId: 232576409
---
 .../core/platform/default/build_config.bzl    |   5 +-
 .../core/platform/default/device_tracer.cc    |  37 +++--
 tensorflow/core/profiler/internal/BUILD       |   1 +
 tensorflow/core/profiler/internal/cpu/BUILD   |  44 ++++++
 .../core/profiler/internal/cpu/host_tracer.cc | 118 ++++++++++++++++
 .../core/profiler/internal/cpu/host_tracer.h  |  67 +++++++++
 .../profiler/internal/cpu/host_tracer_test.cc | 133 ++++++++++++++++++
 tensorflow/core/profiler/internal/gpu/BUILD   |   1 -
 .../core/profiler/lib/profiler_session.cc     |   6 +-
 .../kernel_tests/tensor_array_ops_test.py     |   2 +-
 .../profiler/internal/run_metadata_test.py    |   2 +-
 11 files changed, 403 insertions(+), 13 deletions(-)
 create mode 100644 tensorflow/core/profiler/internal/cpu/BUILD
 create mode 100644 tensorflow/core/profiler/internal/cpu/host_tracer.cc
 create mode 100644 tensorflow/core/profiler/internal/cpu/host_tracer.h
 create mode 100644 tensorflow/core/profiler/internal/cpu/host_tracer_test.cc

diff --git a/tensorflow/core/platform/default/build_config.bzl b/tensorflow/core/platform/default/build_config.bzl
index e17ce4ebf3..f9e3e1dc25 100644
--- a/tensorflow/core/platform/default/build_config.bzl
+++ b/tensorflow/core/platform/default/build_config.bzl
@@ -579,7 +579,10 @@ def tf_additional_device_tracer_cuda_deps():
     return []
 
 def tf_additional_device_tracer_deps():
-    return []
+    return [
+        "//tensorflow/core/profiler/lib:traceme",
+        "//tensorflow/core/profiler/internal/cpu:host_tracer",
+    ]
 
 def tf_additional_device_tracer_test_flags():
     return []
diff --git a/tensorflow/core/platform/default/device_tracer.cc b/tensorflow/core/platform/default/device_tracer.cc
index 8351362e05..0d1bf39739 100644
--- a/tensorflow/core/platform/default/device_tracer.cc
+++ b/tensorflow/core/platform/default/device_tracer.cc
@@ -31,6 +31,8 @@ limitations under the License.
 #include "tensorflow/core/platform/mem.h"
 #include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/platform/tracing.h"
+#include "tensorflow/core/profiler/internal/cpu/host_tracer.h"
+#include "tensorflow/core/profiler/lib/traceme.h"
 
 namespace {
 
@@ -299,6 +301,14 @@ TF_STATIC_THREAD_LOCAL_POD(const char *, tls_current_annotation);
 
 class TraceCollectorImpl : public tracing::TraceCollector {
  public:
+  class ActivityHandle : public Handle {
+   public:
+    ActivityHandle(string &&name, int level)
+        : trace_me_(std::move(name), level) {}
+
+   private:
+    profiler::TraceMe trace_me_;
+  };
   TraceCollectorImpl() { tracing::SetTraceCollector(this); }
 
   ~TraceCollectorImpl() override {
@@ -318,14 +328,16 @@ class TraceCollectorImpl : public tracing::TraceCollector {
       }
       ~Impl() override { tls_current_annotation.get() = nullptr; }
     };
-    return std::unique_ptr<Handle>(
-        new Impl{ConcatenateNames(name_part1, name_part2)});
+    return absl::make_unique<Impl>(ConcatenateNames(name_part1, name_part2));
   }
 
-  virtual std::unique_ptr<Handle> CreateActivityHandle(StringPiece, StringPiece,
-                                                       bool) const {
-    // We don't do anything with 'Activities' yet.
-    return nullptr;
+  virtual std::unique_ptr<Handle> CreateActivityHandle(
+      StringPiece name_part1, StringPiece name_part2, bool is_expensive) const {
+    if (!IsEnabledForActivities(is_expensive)) {
+      return nullptr;
+    }
+    return absl::make_unique<ActivityHandle>(
+        ConcatenateNames(name_part1, name_part2), GetLevel(is_expensive));
   }
 
   bool IsEnabledForAnnotations() const override {
@@ -333,8 +345,7 @@ class TraceCollectorImpl : public tracing::TraceCollector {
   }
 
   bool IsEnabledForActivities(bool is_expensive) const override {
-    // We don't do anything with 'Activities' so we are never 'enabled'.
-    return false;
+    return profiler::TraceMeRecorder::Active(GetLevel(is_expensive));
   }
 
   void Start() {
@@ -349,6 +360,10 @@ class TraceCollectorImpl : public tracing::TraceCollector {
   }
 
  private:
+  static int GetLevel(bool is_expensive) {
+    return profiler::GetTFTraceMeLevel(is_expensive);
+  }
+
   std::atomic<bool> active_trace_session_;
 };
 
@@ -421,6 +436,7 @@ class DeviceTracerImpl : public DeviceTracer, public CUPTIClient {
   int64 end_walltime_us_ GUARDED_BY(mu_);
   uint64_t start_timestamp_ GUARDED_BY(mu_);
   uint64_t end_timestamp_ GUARDED_BY(mu_);
+  std::unique_ptr<profiler::cpu::HostTracer> host_tracer_ GUARDED_BY(mu_);
 
   TF_DISALLOW_COPY_AND_ASSIGN(DeviceTracerImpl);
 };
@@ -493,6 +509,8 @@ Status DeviceTracerImpl::Start() {
 
   CUPTI_CALL(GetTimestamp(&start_timestamp_));
   start_walltime_us_ = NowInUsec();
+  host_tracer_ = profiler::cpu::HostTracer::Create(2);
+  host_tracer_->Start().IgnoreError();
   enabled_ = true;
   return Status::OK();
 }
@@ -510,6 +528,7 @@ Status DeviceTracerImpl::Stop() {
   end_walltime_us_ = NowInUsec();
   CUPTI_CALL(GetTimestamp(&end_timestamp_));
   enabled_ = false;
+  host_tracer_->Stop().IgnoreError();
   return Status::OK();
 }
 
@@ -676,6 +695,8 @@ Status DeviceTracerImpl::Collect(StepStatsCollector *collector) {
     collector->Save(memcpy_device, ns);
     collector->Save(strings::StrCat(stream_device, rec.stream_id), nscopy);
   }
+
+  host_tracer_->CollectDataToCollector(collector).IgnoreError();
   return Status::OK();
 }
 
diff --git a/tensorflow/core/profiler/internal/BUILD b/tensorflow/core/profiler/internal/BUILD
index bae2148bb0..30aca58a41 100644
--- a/tensorflow/core/profiler/internal/BUILD
+++ b/tensorflow/core/profiler/internal/BUILD
@@ -375,6 +375,7 @@ tf_cuda_library(
     visibility = [
         "//learning/brain/runtime:__pkg__",  # xprof_bridge
         "//perftools/accelerators/xprof/xprofilez:__pkg__",  # alias xprof::TraceMeRecorder
+        "//tensorflow/core/profiler/internal/cpu:__pkg__",  # host_tracer
         "//tensorflow/core/profiler/lib:__pkg__",  # traceme
     ],
     deps = [
diff --git a/tensorflow/core/profiler/internal/cpu/BUILD b/tensorflow/core/profiler/internal/cpu/BUILD
new file mode 100644
index 0000000000..b94453c0a4
--- /dev/null
+++ b/tensorflow/core/profiler/internal/cpu/BUILD
@@ -0,0 +1,44 @@
+package(
+    default_visibility = ["//tensorflow:internal"],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+load(
+    "//tensorflow:tensorflow.bzl",
+    "tf_cuda_library",
+)
+load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_test")
+
+tf_cuda_library(
+    name = "host_tracer",
+    srcs = [
+        "host_tracer.cc",
+    ],
+    hdrs = [
+        "host_tracer.h",
+    ],
+    deps = [
+        "//tensorflow/core:core_cpu_lib",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core/profiler/internal:profiler_interface",
+        "//tensorflow/core/profiler/internal:traceme_recorder",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+tf_cuda_cc_test(
+    name = "host_tracer_test",
+    srcs = ["host_tracer_test.cc"],
+    deps = [
+        ":host_tracer",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core/profiler/lib:traceme",
+        "@com_google_absl//absl/types:optional",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
diff --git a/tensorflow/core/profiler/internal/cpu/host_tracer.cc b/tensorflow/core/profiler/internal/cpu/host_tracer.cc
new file mode 100644
index 0000000000..1efc50695a
--- /dev/null
+++ b/tensorflow/core/profiler/internal/cpu/host_tracer.cc
@@ -0,0 +1,118 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/profiler/internal/cpu/host_tracer.h"
+
+#include <utility>
+
+#include "absl/container/flat_hash_map.h"
+#include "absl/strings/str_split.h"
+#include "tensorflow/core/common_runtime/step_stats_collector.h"
+#include "tensorflow/core/platform/env_time.h"
+
+namespace tensorflow {
+namespace profiler {
+namespace cpu {
+
+/* static */ std::unique_ptr<HostTracer> HostTracer::Create(
+    int host_trace_level) {
+  return absl::WrapUnique(new HostTracer(host_trace_level));
+}
+HostTracer::HostTracer(int host_trace_level)
+    : host_trace_level_(host_trace_level) {}
+
+HostTracer::~HostTracer() { Stop().IgnoreError(); }
+
+Status HostTracer::Start() {
+  if (recording_) {
+    return Status(error::INTERNAL, "TraceMeRecorder already started");
+  }
+  recording_ = TraceMeRecorder::Start(host_trace_level_);
+  if (!recording_) {
+    return Status(error::INTERNAL, "Failed to start TraceMeRecorder");
+  }
+  return Status::OK();
+}
+
+Status HostTracer::Stop() {
+  if (!recording_) {
+    return Status(error::INTERNAL, "TraceMeRecorder not started");
+  }
+  events_ = TraceMeRecorder::Stop();
+  recording_ = false;
+  return Status::OK();
+}
+
+constexpr char kUserMetadataMarker = '#';
+
+Status HostTracer::CollectData(RunMetadata* run_metadata) {
+  auto step_stats_collector =
+      absl::make_unique<StepStatsCollector>(run_metadata->mutable_step_stats());
+  return CollectDataToCollector(step_stats_collector.get());
+}
+
+Status HostTracer::CollectDataToCollector(
+    StepStatsCollector* step_stats_collector) {
+  if (events_.empty() && recording_) {
+    events_ = TraceMeRecorder::Collect();
+  }
+  // Pair up start and end events, and add complete events to trace_entries.
+  absl::flat_hash_map<uint64, uint64> end_times;
+  for (const auto& thread : events_) {
+    for (const auto& event : thread.events) {
+      if (event.end_time && !event.start_time) {
+        end_times.emplace(event.activity_id, event.end_time);
+      }
+    }
+  }
+
+  const string cpu_name = "/host:CPU";
+  for (auto& thread : events_) {
+    for (auto& event : thread.events) {
+      if (!event.end_time) {
+        auto it = end_times.find(event.activity_id);
+        if (it != end_times.end()) event.end_time = it->second;
+      }
+      if (event.start_time && event.end_time) {
+        NodeExecStats* ns = new NodeExecStats;
+        if (event.name.back() != kUserMetadataMarker) {
+          ns->set_node_name(std::move(event.name));
+        } else {
+          // Expect the format will be "<name>#<metadata>#"
+          std::vector<absl::string_view> parts =
+              absl::StrSplit(event.name, kUserMetadataMarker);
+          if (parts.size() >= 2) {
+            ns->set_node_name(string(parts[0]));
+            ns->set_timeline_label(string(parts[1]));
+          } else {
+            ns->set_node_name(std::move(event.name));
+          }
+        }
+        ns->set_all_start_micros(event.start_time / EnvTime::kMicrosToNanos);
+        ns->set_all_end_rel_micros((event.end_time - event.start_time) /
+                                   EnvTime::kMicrosToNanos);
+        ns->set_thread_id(thread.thread.tid);
+        // TODO(fishx): Add thread name to RunMetadata
+        step_stats_collector->Save(cpu_name, ns);
+      }
+    }
+  }
+  events_.clear();
+  step_stats_collector->Finalize();
+  return Status::OK();
+}
+
+}  // namespace cpu
+}  // namespace profiler
+}  // namespace tensorflow
diff --git a/tensorflow/core/profiler/internal/cpu/host_tracer.h b/tensorflow/core/profiler/internal/cpu/host_tracer.h
new file mode 100644
index 0000000000..c6340c2edd
--- /dev/null
+++ b/tensorflow/core/profiler/internal/cpu/host_tracer.h
@@ -0,0 +1,67 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_PROFILER_INTERNAL_CPU_HOST_TRACER_H_
+#define TENSORFLOW_CORE_PROFILER_INTERNAL_CPU_HOST_TRACER_H_
+
+#include "tensorflow/core/common_runtime/step_stats_collector.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/profiler/internal/profiler_interface.h"
+#include "tensorflow/core/profiler/internal/traceme_recorder.h"
+#include "tensorflow/core/protobuf/config.pb.h"
+
+namespace tensorflow {
+namespace profiler {
+namespace cpu {
+
+// Controls TraceMeRecorder and converts TraceMeRecorder::Events into
+// RunMetadata messages.
+//
+// Thread-safety: This class is go/thread-compatible.
+class HostTracer : public ProfilerInterface {
+ public:
+  static std::unique_ptr<HostTracer> Create(int host_trace_level);
+
+  ~HostTracer();
+
+  // Starts recording TraceMes.
+  Status Start() override;
+
+  // Stops recording TraceMes.
+  Status Stop() override;
+
+  // Populates user traces and thread names in response.
+  // The user traces and thread names are in no particular order.
+  Status CollectData(RunMetadata* run_metadata) override;
+
+  Status CollectDataToCollector(StepStatsCollector* step_stats_collector);
+
+ private:
+  explicit HostTracer(int host_trace_level);
+
+  // Level of host tracing.
+  const int host_trace_level_;
+
+  // True if currently recording.
+  bool recording_ = false;
+
+  // Container of all traced events.
+  TraceMeRecorder::Events events_;
+};
+
+}  // namespace cpu
+}  // namespace profiler
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_PROFILER_INTERNAL_CPU_HOST_TRACER_H_
diff --git a/tensorflow/core/profiler/internal/cpu/host_tracer_test.cc b/tensorflow/core/profiler/internal/cpu/host_tracer_test.cc
new file mode 100644
index 0000000000..51f9c6a8ca
--- /dev/null
+++ b/tensorflow/core/profiler/internal/cpu/host_tracer_test.cc
@@ -0,0 +1,133 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/profiler/internal/cpu/host_tracer.h"
+
+#include <string>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+#include "absl/types/optional.h"
+#include "tensorflow/core/framework/step_stats.pb.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/profiler/lib/traceme.h"
+#include "tensorflow/core/protobuf/config.pb.h"
+
+namespace tensorflow {
+namespace profiler {
+namespace cpu {
+namespace {
+
+using ::testing::ElementsAre;
+using ::testing::Pair;
+using ::testing::UnorderedElementsAre;
+
+NodeExecStats MakeNodeStats(const string& name, uint64 thread_id,
+                            const string& label = "") {
+  NodeExecStats ns;
+  ns.set_node_name(name);
+  ns.set_thread_id(thread_id);
+  if (!label.empty()) {
+    ns.set_timeline_label(label);
+  }
+  return ns;
+}
+
+class NodeStatsMatcher {
+ public:
+  explicit NodeStatsMatcher(const NodeExecStats& expected)
+      : expected_(expected) {}
+
+  bool MatchAndExplain(const NodeExecStats& p,
+                       ::testing::MatchResultListener* /* listener */) const {
+    return p.node_name() == expected_.node_name() &&
+           p.thread_id() == expected_.thread_id() &&
+           p.timeline_label() == expected_.timeline_label();
+  }
+
+  void DescribeTo(::std::ostream* os) const { *os << expected_.DebugString(); }
+  void DescribeNegationTo(::std::ostream* os) const {
+    *os << "not equal to expected message: " << expected_.DebugString();
+  }
+
+ private:
+  const NodeExecStats expected_;
+};
+
+inline ::testing::PolymorphicMatcher<NodeStatsMatcher> EqualsNodeStats(
+    const NodeExecStats& expected) {
+  return ::testing::MakePolymorphicMatcher(NodeStatsMatcher(expected));
+}
+
+TEST(HostTracerTest, CollectsTraceMeEvents) {
+  uint32 thread_id = Env::Default()->GetCurrentThreadId();
+
+  auto tracer = HostTracer::Create(/*host_trace_level=*/1);
+
+  TF_ASSERT_OK(tracer->Start());
+  { TraceMe traceme("hello"); }
+  { TraceMe traceme("world"); }
+  { TraceMe traceme("contains#inside"); }
+  { TraceMe traceme("good#key1=value1#"); }
+  { TraceMe traceme("morning#key1=value1,key2=value2#"); }
+  { TraceMe traceme("incomplete#key1=value1,key2#"); }
+  TF_ASSERT_OK(tracer->Stop());
+
+  RunMetadata run_metadata;
+  TF_ASSERT_OK(tracer->CollectData(&run_metadata));
+
+  EXPECT_EQ(run_metadata.step_stats().dev_stats_size(), 1);
+  EXPECT_EQ(run_metadata.step_stats().dev_stats(0).node_stats_size(), 6);
+  EXPECT_THAT(
+      run_metadata.step_stats().dev_stats(0).node_stats(),
+      UnorderedElementsAre(
+          EqualsNodeStats(MakeNodeStats("hello", thread_id)),
+          EqualsNodeStats(MakeNodeStats("world", thread_id)),
+          EqualsNodeStats(MakeNodeStats("contains#inside", thread_id)),
+          EqualsNodeStats(MakeNodeStats("good", thread_id, "key1=value1")),
+          EqualsNodeStats(
+              MakeNodeStats("morning", thread_id, "key1=value1,key2=value2")),
+          EqualsNodeStats(
+              MakeNodeStats("incomplete", thread_id, "key1=value1,key2"))));
+}
+
+void ValidateResult(const RunMetadata& run_metadata, const string& trace_name) {
+  uint32 thread_id = Env::Default()->GetCurrentThreadId();
+
+  EXPECT_THAT(
+      run_metadata.step_stats().dev_stats(0).node_stats(),
+      ElementsAre(EqualsNodeStats(MakeNodeStats(trace_name, thread_id))));
+}
+
+TEST(HostTracerTest, CollectsTraceMeEventsBetweenTracing) {
+  auto tracer = HostTracer::Create(/*host_trace_level=*/1);
+  RunMetadata run_metadata;
+  RunMetadata run_metadata2;
+
+  TF_ASSERT_OK(tracer->Start());
+  { TraceMe traceme("hello"); }
+  TF_ASSERT_OK(tracer->CollectData(&run_metadata));
+  { TraceMe traceme("world"); }
+  TF_ASSERT_OK(tracer->CollectData(&run_metadata2));
+  TF_ASSERT_OK(tracer->Stop());
+
+  ValidateResult(run_metadata, "hello");
+  ValidateResult(run_metadata2, "world");
+}
+
+}  // namespace
+}  // namespace cpu
+}  // namespace profiler
+}  // namespace tensorflow
diff --git a/tensorflow/core/profiler/internal/gpu/BUILD b/tensorflow/core/profiler/internal/gpu/BUILD
index c4b5b5243c..35f90e9bfc 100644
--- a/tensorflow/core/profiler/internal/gpu/BUILD
+++ b/tensorflow/core/profiler/internal/gpu/BUILD
@@ -21,6 +21,5 @@ tf_cuda_library(
         "//tensorflow/core:core_cpu_lib",
         "//tensorflow/core:device_tracer",
         "//tensorflow/core/profiler/internal:profiler_interface",
-        "//tensorflow/core/profiler/internal/runtime:eager_profiler",
     ],
 )
diff --git a/tensorflow/core/profiler/lib/profiler_session.cc b/tensorflow/core/profiler/lib/profiler_session.cc
index 86b1e69bed..4641ae2a5f 100644
--- a/tensorflow/core/profiler/lib/profiler_session.cc
+++ b/tensorflow/core/profiler/lib/profiler_session.cc
@@ -55,7 +55,11 @@ void ConvertRunMetadataToTraceEvent(RunMetadata* run_metadata,
       auto* event = trace->add_trace_events();
       auto* args = event->mutable_args();
       event->set_device_id(device_id);
-      event->set_resource_id(0);
+      if (device_stats->device().find("host:CPU") != string::npos) {
+        event->set_resource_id(node.thread_id());
+      } else {
+        event->set_resource_id(0);
+      }
       event->set_name(node.node_name());
       event->set_timestamp_ps(
           (node.all_start_micros() - profile_start_time_micros) *
diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
index 3b2f476e0c..b4544baa5d 100644
--- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py
+++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
@@ -1513,7 +1513,7 @@ class TensorArrayTest(test.TestCase):
       if "/task:1/" in d:
         self.assertTrue(
             [s for s in dev_stats[d] if "/TensorArray" in s.node_name])
-      else:
+      elif "/host:CPU" not in d:
         self.assertFalse(
             [s for s in dev_stats[d] if "/TensorArray" in s.node_name])
 
diff --git a/tensorflow/python/profiler/internal/run_metadata_test.py b/tensorflow/python/profiler/internal/run_metadata_test.py
index f96d721f46..9e92a8f5f3 100644
--- a/tensorflow/python/profiler/internal/run_metadata_test.py
+++ b/tensorflow/python/profiler/internal/run_metadata_test.py
@@ -50,7 +50,7 @@ def _extract_node(run_meta, node_name):
       dev = dev[dev.find('cpu:'):]
     elif dev.find('gpu:') > 0:
       dev = dev[dev.find('gpu:'):]
-    else:
+    elif '/host:cpu' not in dev:
       assert False, 'Unrecognized device name: %s' % dev
 
     for node_stat in dev_stat.node_stats:
-- 
GitLab


From 90cebebed3bd0e413f7f00ab45566d1b28b21fe6 Mon Sep 17 00:00:00 2001
From: Lukas Geiger <lukas.geiger94@gmail.com>
Date: Fri, 1 Feb 2019 23:35:37 +0100
Subject: [PATCH 1965/2345] Support max pooling ops in algorithmic optimizer

This PR adds support for max pooling operations to the algorithmic optimizer for monotonic functions.

This follows up on #25330
/cc @ezhulenev
---
 tensorflow/core/grappler/op_types.cc          |  6 ++
 tensorflow/core/grappler/op_types.h           |  1 +
 .../optimizers/arithmetic_optimizer.cc        |  7 +-
 .../optimizers/arithmetic_optimizer_test.cc   | 70 +++++++++++++++++++
 4 files changed, 82 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc
index ee83fe009b..f403e16492 100644
--- a/tensorflow/core/grappler/op_types.cc
+++ b/tensorflow/core/grappler/op_types.cc
@@ -47,6 +47,12 @@ bool IsAnyDiv(const NodeDef& node) {
          node.op() == "FloorDiv" || node.op() == "TruncateDiv";
 }
 
+bool IsAnyMaxPool(const NodeDef& node) {
+  const auto& op = node.op();
+  return op == "MaxPool" || op == "MaxPoolV2" || op == "MaxPool3D" ||
+         op == "MaxPoolWithArgmax" || op == "FractionalMaxPool";
+}
+
 bool IsApproximateEqual(const NodeDef& node) {
   return node.op() == "ApproximateEqual";
 }
diff --git a/tensorflow/core/grappler/op_types.h b/tensorflow/core/grappler/op_types.h
index 7395151d30..bc1d8c15ac 100644
--- a/tensorflow/core/grappler/op_types.h
+++ b/tensorflow/core/grappler/op_types.h
@@ -28,6 +28,7 @@ bool IsAll(const NodeDef& node);
 bool IsAngle(const NodeDef& node);
 bool IsAny(const NodeDef& node);
 bool IsAnyDiv(const NodeDef& node);
+bool IsAnyMaxPool(const NodeDef& node);
 bool IsApproximateEqual(const NodeDef& node);
 bool IsAvgPoolGrad(const NodeDef& node);
 bool IsAssert(const NodeDef& node);
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index db7257156e..902cb3fdd8 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -2721,7 +2721,7 @@ class OptimizeMaxOrMinOfMonotonicStage : public ArithmeticOptimizerStage {
   ~OptimizeMaxOrMinOfMonotonicStage() override = default;
 
   bool IsSupported(const NodeDef* node) const override {
-    return IsMax(*node) || IsMin(*node);
+    return IsMax(*node) || IsMin(*node) || IsAnyMaxPool(*node);
   }
 
   Status TrySimplify(NodeDef* reduction_node,
@@ -2735,10 +2735,13 @@ class OptimizeMaxOrMinOfMonotonicStage : public ArithmeticOptimizerStage {
     // 0. inner_function is not in the preserve set,
     // 1. inner_function's Op is element-wise monotonic
     // 2. inner_function's output is not being consumed elsewhere.
+    // 3. is monotonic increasing if reduction_node is a pooling operation
+    //    since we don't have MinPool operations.
     bool is_non_decreasing = false;
     if (!IsInPreserveSet(*inner_function) &&
         IsElementWiseMonotonic(*inner_function, &is_non_decreasing) &&
-        ctx().node_map->GetOutputs(inner_function->name()).size() == 1) {
+        ctx().node_map->GetOutputs(inner_function->name()).size() == 1 &&
+        (is_non_decreasing || !IsAnyMaxPool(*reduction_node))) {
       // Swap the first inputs of the inner function Op & the reduction Op.
       NodeDef* inner_input;
       TF_RETURN_IF_ERROR(GetInputNode(inner_function->input(0), &inner_input));
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
index 94c59c68c8..73634b716b 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
@@ -3561,6 +3561,76 @@ TEST_F(ArithmeticOptimizerTest,
   EXPECT_EQ(2, required_node_count);
 }
 
+TEST_F(ArithmeticOptimizerTest,
+       OptimizeMaxOrMinOfMonotonicElementWiseNonIncreasingDoNotChangeMaxPool) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+  auto x = ops::Const(s.WithOpName("x"), 1.5f, {3, 3, 3, 1});
+  Output neg = ops::Neg(s.WithOpName("neg"), x);
+  Output max_pool = ops::MaxPool(
+      s.WithOpName("max_pool"), neg, {1, 2, 2, 1}, {1, 2, 2, 1}, "VALID");
+
+  GrapplerItem item;
+  item.fetch = {"max_pool"};
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+  auto tensors_expected = EvaluateNodes(item.graph, item.fetch);
+  ASSERT_EQ(1, tensors_expected.size());
+
+  GraphDef output;
+  ArithmeticOptimizer optimizer;
+  EnableOnlyOptimizeMaxOrMinOfMonotonic(&optimizer);
+  OptimizeTwice(&optimizer, &item, &output);
+
+  // Should be a NoOp
+  VerifyGraphsMatch(item.graph, output, __LINE__);
+
+  auto tensors = EvaluateNodes(output, item.fetch);
+  ASSERT_EQ(1, tensors.size());
+  test::ExpectTensorNear<float>(tensors_expected[0], tensors[0], 1e-6);
+}
+
+TEST_F(ArithmeticOptimizerTest,
+       OptimizeMaxOrMinOfMonotonicElementWiseMaxPool) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+  auto x = ops::Const(s.WithOpName("x"), 1.5f, {3, 3, 3, 1});
+  Output sqrt = ops::Sqrt(s.WithOpName("sqrt"), x);
+  Output max_pool = ops::MaxPool(
+      s.WithOpName("max_pool"), sqrt, {1, 2, 2, 1}, {1, 2, 2, 1}, "VALID");
+  Output final_out = ops::Identity(s.WithOpName("final_out"), max_pool);
+
+  GrapplerItem item;
+  item.fetch = {"final_out"};
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+  auto tensors_expected = EvaluateNodes(item.graph, item.fetch);
+  EXPECT_EQ(1, tensors_expected.size());
+
+  GraphDef output;
+  ArithmeticOptimizer optimizer;
+  EnableOnlyOptimizeMaxOrMinOfMonotonic(&optimizer);
+  OptimizeAndPrune(&optimizer, &item, &output);
+  auto tensors = EvaluateNodes(output, item.fetch);
+  EXPECT_EQ(1, tensors.size());
+
+  test::ExpectTensorNear<float>(tensors_expected[0], tensors[0], 1e-6);
+  EXPECT_EQ(item.graph.node_size(), output.node_size());
+  // Check if the inputs are switched
+  int required_node_count = 0;
+  for (int i = 0; i < output.node_size(); ++i) {
+    const NodeDef& node = output.node(i);
+    if (node.name() == "sqrt") {
+      EXPECT_EQ("Sqrt", node.op());
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("max_pool", node.input(0));
+      ++required_node_count;
+    } else if (node.name() == "max_pool") {
+      EXPECT_EQ("MaxPool", node.op());
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("x", node.input(0));
+      ++required_node_count;
+    }
+  }
+  EXPECT_EQ(2, required_node_count);
+}
+
 TEST_F(ArithmeticOptimizerTest, UnaryOpsComposition) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
 
-- 
GitLab


From cbcbfe0267aac0eb5d0b9f698ec8da46e390f393 Mon Sep 17 00:00:00 2001
From: Mahmoud Abuzaina <mahmoud.abuzaina@intel.com>
Date: Tue, 5 Feb 2019 16:51:28 -0800
Subject: [PATCH 1966/2345] Quantized Concat with added #ifdef INTEL_MKL

---
 tensorflow/core/BUILD                         |  14 +-
 tensorflow/core/api_def/excluded_ops.cc       |   6 +-
 tensorflow/core/framework/common_shape_fns.cc |  20 +-
 tensorflow/core/framework/common_shape_fns.h  |   2 +
 tensorflow/core/kernels/BUILD                 |  24 ++
 tensorflow/core/kernels/mkl_concat_op.cc      |  98 +++++++-
 .../kernels/mkl_quantized_concat_op_test.cc   | 230 ++++++++++++++++++
 .../core/kernels/quantized_concat_op.cc       |  27 +-
 tensorflow/core/ops/mkl_array_ops.cc          |  92 +++++++
 9 files changed, 479 insertions(+), 34 deletions(-)
 create mode 100644 tensorflow/core/kernels/mkl_quantized_concat_op_test.cc
 create mode 100644 tensorflow/core/ops/mkl_array_ops.cc

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 3a56fa476c..8002d56a15 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1144,6 +1144,13 @@ tf_gen_op_libs(
     deps = [":protos_all_cc"],
 )
 
+tf_gen_op_libs(
+    op_lib_names = [
+        "mkl_array_ops",
+    ],
+    deps = [":protos_all_cc"],
+)
+
 tf_gen_op_libs(
     op_lib_names = [
         "audio_ops",
@@ -1283,7 +1290,10 @@ cc_library(
         ":training_ops_op_lib",
         ":user_ops_op_lib",
         ":word2vec_ops",
-    ] + if_mkl([":mkl_nn_ops_op_lib"]) + tf_additional_cloud_op_deps(),
+    ] + if_mkl([
+        ":mkl_array_ops_op_lib",
+        ":mkl_nn_ops_op_lib",
+    ]) + tf_additional_cloud_op_deps(),
     alwayslink = 1,
 )
 
@@ -4523,7 +4533,7 @@ tf_cc_test(
         "//tensorflow/cc:scope",
         "//tensorflow/core/kernels:cwise_op",
         "//third_party/eigen3",
-    ],
+    ] + if_mkl([":mkl_array_ops_op_lib"]),
 )
 
 tf_cc_test(
diff --git a/tensorflow/core/api_def/excluded_ops.cc b/tensorflow/core/api_def/excluded_ops.cc
index 02026e94ab..65d2102ac8 100644
--- a/tensorflow/core/api_def/excluded_ops.cc
+++ b/tensorflow/core/api_def/excluded_ops.cc
@@ -24,9 +24,9 @@ const std::unordered_set<std::string>* GetExcludedOps() {
            "GcsConfigureBlockCache", "GcsConfigureCredentials",
 #ifdef INTEL_MKL
            // QuantizedFusedOps for Intel CPU
-           "QuantizedConv2DAndRequantize", "QuantizedConv2DWithBias",
-           "QuantizedConv2DWithBiasAndRequantize", "QuantizedConv2DAndRelu",
-           "QuantizedConv2DAndReluAndRequantize",
+           "QuantizedConcatV2", "QuantizedConv2DAndRequantize",
+           "QuantizedConv2DWithBias", "QuantizedConv2DWithBiasAndRequantize",
+           "QuantizedConv2DAndRelu", "QuantizedConv2DAndReluAndRequantize",
            "QuantizedConv2DWithBiasAndRelu",
            "QuantizedConv2DWithBiasAndReluAndRequantize",
            "QuantizedConv2DWithBiasSumAndRelu",
diff --git a/tensorflow/core/framework/common_shape_fns.cc b/tensorflow/core/framework/common_shape_fns.cc
index 83bc95065e..8f2fc418c4 100644
--- a/tensorflow/core/framework/common_shape_fns.cc
+++ b/tensorflow/core/framework/common_shape_fns.cc
@@ -57,9 +57,8 @@ Status GetWindowedOutputSizeVerboseV2(int64 input_size, int64 filter_size,
   if (*output_size < 0) {
     return errors::InvalidArgument(
         "Computed output size would be negative: ", *output_size,
-        " [input_size: ", input_size,
-        ", effective_filter_size: ", effective_filter_size,
-        ", stride: ", stride, "]");
+        " [input_size: ", input_size, ", effective_filter_size: ",
+        effective_filter_size, ", stride: ", stride, "]");
   }
   return Status::OK();
 }
@@ -1303,6 +1302,12 @@ Status ConcatV2Shape(InferenceContext* c) {
                            c->num_inputs() - 1 /* dim_index */);
 }
 
+Status QuantizedConcatV2Shape(InferenceContext* c, int num_inputs_to_concat) {
+  return ConcatShapeHelper(c, 0 /* start_value_index */,
+                           num_inputs_to_concat /* end_value_index */,
+                           num_inputs_to_concat /* dim_index */);
+}
+
 Status BroadcastBinaryOpOutputShapeFnHelper(InferenceContext* c,
                                             ShapeHandle shape_x,
                                             ShapeHandle shape_y,
@@ -1566,11 +1571,10 @@ Status ScatterNdUpdateShape(InferenceContext* c) {
       Status s = c->Merge(prefix_indices, prefix_updates, &unused);
       if (!s.ok()) {
         return errors::InvalidArgument(
-            "The outer ", num_outer_dims,
-            " dimensions of indices.shape=", c->DebugString(indices_shape),
-            " must match the outer ", num_outer_dims,
-            " dimensions of updates.shape=", c->DebugString(updates_shape),
-            ": ", s.error_message());
+            "The outer ", num_outer_dims, " dimensions of indices.shape=",
+            c->DebugString(indices_shape), " must match the outer ",
+            num_outer_dims, " dimensions of updates.shape=",
+            c->DebugString(updates_shape), ": ", s.error_message());
       }
 
       ShapeHandle input_suffix;
diff --git a/tensorflow/core/framework/common_shape_fns.h b/tensorflow/core/framework/common_shape_fns.h
index 14b9688bdc..d421844ee6 100644
--- a/tensorflow/core/framework/common_shape_fns.h
+++ b/tensorflow/core/framework/common_shape_fns.h
@@ -279,6 +279,8 @@ Status ConcatShape(shape_inference::InferenceContext* c,
 // Shape function for concat operations.
 Status ConcatV2Shape(shape_inference::InferenceContext* c);
 
+Status QuantizedConcatV2Shape(InferenceContext* c, int num_inputs_to_concat);
+
 // Shape function for binary operators that broadcast their inputs
 // and with output to output_index.
 // Note: out cannot be NULL.
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index e5ee109fff..1749f3a4da 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -6561,6 +6561,30 @@ tf_cc_test(
     ],
 )
 
+tf_cc_test_mkl(
+    name = "mkl_quantized_concat_op_test",
+    size = "small",
+    srcs = ["mkl_quantized_concat_op_test.cc"],
+    deps = [
+        ":mkl_concat_op",
+        ":ops_testutil",
+        ":ops_util",
+        ":quantization_utils",
+        ":quantized_ops",
+        "//tensorflow/core:array_ops_op_lib",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:mkl_array_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+    ],
+)
+
 tf_cc_test(
     name = "quantized_batch_norm_op_test",
     size = "small",
diff --git a/tensorflow/core/kernels/mkl_concat_op.cc b/tensorflow/core/kernels/mkl_concat_op.cc
index 3a5c87485c..ba6c0f2389 100644
--- a/tensorflow/core/kernels/mkl_concat_op.cc
+++ b/tensorflow/core/kernels/mkl_concat_op.cc
@@ -17,7 +17,6 @@ limitations under the License.
 #include <vector>
 
 #include "mkldnn.hpp"
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
@@ -25,12 +24,15 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/kernels/concat_lib.h"
+#include "tensorflow/core/kernels/concat_lib_cpu.h"
+#include "tensorflow/core/kernels/quantization_utils.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
+#include "tensorflow/core/util/mkl_util.h"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
 using mkldnn::concat;
 using mkldnn::stream;
-#include "tensorflow/core/util/mkl_util.h"
 
 namespace tensorflow {
 typedef Eigen::ThreadPoolDevice CPUDevice;
@@ -78,9 +80,8 @@ class EigenConcatBaseOp : public OpKernel {
     const TensorShape& input_shape = input_shapes[0];
 
     int32 axis = concat_dim < 0 ? concat_dim + input_dims : concat_dim;
-    OP_REQUIRES(c,
-                (0 <= axis && axis < input_dims) ||
-                    (allow_legacy_scalars() && concat_dim == 0),
+    OP_REQUIRES(c, (0 <= axis && axis < input_dims) ||
+                       (allow_legacy_scalars() && concat_dim == 0),
                 errors::InvalidArgument(
                     "ConcatOp : Expected concatenating dimensions in the range "
                     "[",
@@ -102,13 +103,12 @@ class EigenConcatBaseOp : public OpKernel {
       const auto in = values[i];
       const bool in_is_scalar = IsLegacyScalar(input_shapes[i]);
       OP_REQUIRES(
-          c,
-          (input_shapes[i].dims() == input_dims) ||
-              (input_is_scalar && in_is_scalar),
+          c, (input_shapes[i].dims() == input_dims) ||
+                 (input_is_scalar && in_is_scalar),
           errors::InvalidArgument(
               "ConcatOp : Ranks of all input tensors should match: shape[0] = ",
-              input_shape.DebugString(), " vs. shape[", i,
-              "] = ", input_shapes[i].DebugString()));
+              input_shape.DebugString(), " vs. shape[", i, "] = ",
+              input_shapes[i].DebugString()));
       if (in.NumElements() > 0) {
         int64 inputs_flat_dim1 = in.NumElements() / inputs_flat_dim0;
         inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
@@ -226,8 +226,49 @@ class MklConcatOp : public OpKernel {
       // format and avoid calling eigen version.
       if (!are_all_tf_inputs && !are_all_mkl_inputs) invoke_eigen = true;
 
+      OpInputList input_mins, input_maxes;
+      if (std::is_same<T, qint8>::value || std::is_same<T, quint8>::value) {
+        // MKL-DNN concat does not support input tensors that have different
+        // ranges. Check if the ranges of the all input tensors are the same.
+        // If not, forward it to Eigen implementation.
+
+        OP_REQUIRES_OK(context, context->input_list("input_mins", &input_mins));
+        OP_REQUIRES(context, (input_mins.size() == N),
+                    errors::InvalidArgument(
+                        "QuantizedConcatOp : Expected mins input list length ",
+                        input_mins.size(), " to equal values length ", N));
+
+        OP_REQUIRES_OK(context,
+                       context->input_list("input_maxes", &input_maxes));
+        OP_REQUIRES(context, (input_maxes.size() == N),
+                    errors::InvalidArgument(
+                        "QuantizedConcatOp : Expected maxes input list length ",
+                        input_maxes.size(), " to equal values length ", N));
+        float input_min = input_mins[0].flat<float>()(0);
+        float input_max = input_maxes[0].flat<float>()(0);
+        const float eps = 1.0e-6;
+        for (int i = 1; i < N; ++i) {
+          float min = input_mins[i].flat<float>()(0);
+          float max = input_maxes[i].flat<float>()(0);
+
+          if (fabs(input_min - min) > eps || fabs(input_max - max) > eps) {
+            invoke_eigen = true;
+            break;
+          }
+        }
+      }
+
       // Call Eigen library
       if (invoke_eigen) {
+        // MKL-DNN quantized concat does not support input tensors with
+        // different ranges.
+        // TODO (mabuzain): Add quantized version of CallEigen() to support
+        // this case.
+        OP_REQUIRES(context, (!std::is_same<T, qint8>::value &&
+                              !std::is_same<T, quint8>::value),
+                    errors::Unimplemented("MKL DNN quantized concat does not "
+                                          "support input tensors that have "
+                                          "different ranges"));
         CallEigenVersion(context, input_tensors, mkl_input_shapes);
         return;
       }
@@ -374,10 +415,27 @@ class MklConcatOp : public OpKernel {
       std::vector<primitive> net;
       net.push_back(concat_op);
       stream(stream::kind::eager).submit(net).wait();
+
+      // For quantized concat, min and max outputs are also computed.
+      if (std::is_same<T, qint8>::value || std::is_same<T, quint8>::value) {
+        Tensor* output_min = nullptr;
+        Tensor* output_max = nullptr;
+        MklDnnShape output_min_mkl_shape, output_max_mkl_shape;
+        output_min_mkl_shape.SetMklTensor(false);
+        output_max_mkl_shape.SetMklTensor(false);
+        AllocateOutputSetMklShape(context, 1, &output_min, {},
+                                  output_min_mkl_shape);
+        AllocateOutputSetMklShape(context, 2, &output_max, {},
+                                  output_max_mkl_shape);
+        // All input tensors should have the same range, just use the
+        // first one
+        output_min->flat<float>()(0) = input_mins[0].flat<float>()(0);
+        output_max->flat<float>()(0) = input_maxes[0].flat<float>()(0);
+      }
     } catch (mkldnn::error& e) {
-      string error_msg = "Status: " + std::to_string(e.status) +
-                         ", message: " + string(e.message) + ", in file " +
-                         string(__FILE__) + ":" + std::to_string(__LINE__);
+      string error_msg = "Status: " + std::to_string(e.status) + ", message: " +
+                         string(e.message) + ", in file " + string(__FILE__) +
+                         ":" + std::to_string(__LINE__);
       OP_REQUIRES_OK(
           context,
           errors::Aborted("Operation received an exception:", error_msg));
@@ -490,6 +548,20 @@ class MklConcatOp : public OpKernel {
 
 TF_CALL_float(REGISTER_MKL_CPU);
 
+REGISTER_KERNEL_BUILDER(Name("_MklQuantizedConcatV2")
+                            .Device(DEVICE_CPU)
+                            .TypeConstraint<quint8>("T")
+                            .HostMemory("axis")
+                            .Label(mkl_op_registry::kMklQuantizedOpLabel),
+                        MklConcatOp<CPUDevice, quint8, NAME_IS_AXIS>)
+
+REGISTER_KERNEL_BUILDER(Name("_MklQuantizedConcatV2")
+                            .Device(DEVICE_CPU)
+                            .TypeConstraint<qint8>("T")
+                            .HostMemory("axis")
+                            .Label(mkl_op_registry::kMklQuantizedOpLabel),
+                        MklConcatOp<CPUDevice, qint8, NAME_IS_AXIS>)
+
 #undef REGISTER_CONCAT_MKL
 }  // namespace tensorflow
 
diff --git a/tensorflow/core/kernels/mkl_quantized_concat_op_test.cc b/tensorflow/core/kernels/mkl_quantized_concat_op_test.cc
new file mode 100644
index 0000000000..dcd1de95e0
--- /dev/null
+++ b/tensorflow/core/kernels/mkl_quantized_concat_op_test.cc
@@ -0,0 +1,230 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#define EIGEN_USE_THREADS
+
+#include <functional>
+#include <memory>
+#include <vector>
+
+#include "tensorflow/core/common_runtime/kernel_benchmark_testlib.h"
+#include "tensorflow/core/framework/allocator.h"
+#include "tensorflow/core/framework/fake_input.h"
+#include "tensorflow/core/framework/node_def_builder.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_testutil.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/framework/types.pb.h"
+#include "tensorflow/core/graph/node_builder.h"
+#include "tensorflow/core/kernels/ops_testutil.h"
+#include "tensorflow/core/kernels/ops_util.h"
+#include "tensorflow/core/kernels/quantization_utils.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/platform/test_benchmark.h"
+
+namespace tensorflow {
+
+using test::graph::Constant;
+
+static const uint8 dummy_tensor[] = {0, 0, 0, 0, 0, 0, 0, 0};
+static const TensorShape dummy_shape({8});
+
+// Helper class for converting MKL tensors to TF tensors and comparing to
+// expected values
+
+class ConvMklToTF : public OpsTestBase {
+ public:
+  template <typename T>
+  void ConvertMKL2TF(DataType dtype, const Tensor& first, const Tensor& second,
+                     Tensor& output) {
+    // Create an MKL to TF conversion node and execute it
+    TF_EXPECT_OK(NodeDefBuilder("mkl_to_tf_op", "_MklToTf")
+                     .Input(FakeInput(dtype))     // Input
+                     .Input(FakeInput(DT_UINT8))  // MKL second tensor
+                     .Attr("T", dtype)
+                     .Attr("_kernel", "MklOp")
+                     .Finalize(node_def()));
+    TF_EXPECT_OK(InitOp());
+    AddInputFromArray<T>(first.shape(), first.flat<T>());
+    AddInputFromArray<uint8>(second.shape(), second.flat<uint8>());
+    TF_ASSERT_OK(RunOpKernel());
+
+    output = *GetOutput(0);
+  }
+  void TestBody(){};
+};
+
+class QuantizedConcatTest : public OpsTestBase {
+ protected:
+  QuantizedConcatTest() {}
+
+  void TestSmall8Bit(float first_min, float first_max, float second_min,
+                     float second_max);
+  void TestSecondDim8Bit(float first_min, float first_max, float second_min,
+                         float second_max);
+};
+
+TEST_F(QuantizedConcatTest, Small8BitSameRange) {
+  // Range for both is the same, so impl can use memcpy.
+  TestSmall8Bit(0.0f, 255.0f, 0.0f, 255.0f);
+}
+
+void QuantizedConcatTest::TestSmall8Bit(float first_min, float first_max,
+                                        float second_min, float second_max) {
+  TF_ASSERT_OK(NodeDefBuilder("quantized_concat_op", "_MklQuantizedConcatV2")
+                   .Input(FakeInput(2, DT_QUINT8))
+                   .Input(FakeInput(DT_INT32))
+                   .Input(FakeInput(2, DT_FLOAT))
+                   .Input(FakeInput(2, DT_FLOAT))
+                   .Input(FakeInput(2, DT_UINT8))  // MKL second tensor
+                   .Input(FakeInput(DT_UINT8))     // MKL second tensor
+                   .Input(FakeInput(2, DT_UINT8))  // MKL second tensor
+                   .Input(FakeInput(2, DT_UINT8))  // MKL second tensor
+                   .Attr("N", 2)
+                   .Attr("T", DataTypeToEnum<quint8>::v())
+                   .Attr("Tidx", DT_INT32)
+                   .Attr("_kernel", "QuantizedMklOp")
+                   .Finalize(node_def()));
+  TF_ASSERT_OK(InitOp());
+  const int first_batch = 2;
+  const int first_height = 2;
+  const int first_width = 3;
+  const int first_depth = 1;
+  Tensor first_float(DT_FLOAT,
+                     {first_batch, first_height, first_width, first_depth});
+  test::FillValues<float>(&first_float,
+                          {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
+  Tensor first_quantized =
+      FloatTensorToQuantized<quint8>(first_float, first_min, first_max);
+
+  const int second_batch = 2;
+  const int second_height = 2;
+  const int second_width = 3;
+  const int second_depth = 1;
+  Tensor second_float(
+      DT_FLOAT, {second_batch, second_height, second_width, second_depth});
+  test::FillValues<float>(&second_float,
+                          {13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24});
+  Tensor second_quantized =
+      FloatTensorToQuantized<quint8>(second_float, second_min, second_max);
+
+  const int expected_batch = first_batch + second_batch;
+  Tensor expected_float(
+      DT_FLOAT, {expected_batch, first_height, first_width, first_depth});
+  test::FillValues<float>(&expected_float,
+                          {1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12,
+                           13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24});
+
+  AddInputFromArray<quint8>(first_quantized.shape(),
+                            first_quantized.flat<quint8>());
+  AddInputFromArray<quint8>(second_quantized.shape(),
+                            second_quantized.flat<quint8>());
+  AddInputFromArray<int32>(TensorShape({}), {0});
+  AddInputFromArray<float>(TensorShape({}), {first_min});
+  AddInputFromArray<float>(TensorShape({}), {second_min});
+  AddInputFromArray<float>(TensorShape({}), {first_max});
+  AddInputFromArray<float>(TensorShape({}), {second_max});
+  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+  TF_ASSERT_OK(RunOpKernel());
+  const Tensor& output_quantized = *GetOutput(0);
+  const float output_min = GetOutput(1)->flat<float>()(0);
+  const float output_max = GetOutput(2)->flat<float>()(0);
+  Tensor output_float =
+      QuantizedTensorToFloat<quint8>(output_quantized, output_min, output_max);
+  test::ExpectTensorNear<float>(expected_float, output_float, 0.2);
+}
+
+TEST_F(QuantizedConcatTest, SecondDim8BitSameRange) {
+  TestSecondDim8Bit(-10.0f, 150.0f, -10.0f, 150.0f);
+}
+
+void QuantizedConcatTest::TestSecondDim8Bit(float first_min, float first_max,
+                                            float second_min,
+                                            float second_max) {
+  TF_ASSERT_OK(NodeDefBuilder("quantized_concat_op", "_MklQuantizedConcatV2")
+                   .Input(FakeInput(2, DT_QUINT8))
+                   .Input(FakeInput(DT_INT32))
+                   .Input(FakeInput(2, DT_FLOAT))
+                   .Input(FakeInput(2, DT_FLOAT))
+                   .Input(FakeInput(2, DT_UINT8))  // MKL second tensor
+                   .Input(FakeInput(DT_UINT8))     // MKL second tensor
+                   .Input(FakeInput(2, DT_UINT8))  // MKL second tensor
+                   .Input(FakeInput(2, DT_UINT8))  // MKL second tensor
+                   .Attr("N", 2)
+                   .Attr("T", DataTypeToEnum<quint8>::v())
+                   .Attr("Tidx", DT_INT32)
+                   .Attr("_kernel", "QuantizedMklOp")
+                   .Finalize(node_def()));
+  TF_ASSERT_OK(InitOp());
+  const int first_batch = 2;
+  const int first_height = 2;
+  const int first_width = 3;
+  const int first_depth = 1;
+  Tensor first_float(DT_FLOAT,
+                     {first_batch, first_height, first_width, first_depth});
+  test::FillValues<float>(&first_float,
+                          {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
+  Tensor first_quantized =
+      FloatTensorToQuantized<quint8>(first_float, first_min, first_max);
+
+  const int second_batch = 2;
+  const int second_height = 2;
+  const int second_width = 3;
+  const int second_depth = 1;
+
+  Tensor second_float(
+      DT_FLOAT, {second_batch, second_height, second_width, second_depth});
+  test::FillValues<float>(&second_float,
+                          {13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24});
+  Tensor second_quantized =
+      FloatTensorToQuantized<quint8>(second_float, second_min, second_max);
+
+  const int expected_height = first_height + second_height;
+  Tensor expected_float(
+      DT_FLOAT, {first_batch, expected_height, first_width, first_depth});
+  test::FillValues<float>(&expected_float,
+                          {1, 2, 3, 4,  5,  6,  13, 14, 15, 16, 17, 18,
+                           7, 8, 9, 10, 11, 12, 19, 20, 21, 22, 23, 24});
+
+  AddInputFromArray<quint8>(first_quantized.shape(),
+                            first_quantized.flat<quint8>());
+  AddInputFromArray<quint8>(second_quantized.shape(),
+                            second_quantized.flat<quint8>());
+  AddInputFromArray<int32>(TensorShape({}), {1});
+  AddInputFromArray<float>(TensorShape({}), {first_min});
+  AddInputFromArray<float>(TensorShape({}), {second_min});
+  AddInputFromArray<float>(TensorShape({}), {first_max});
+  AddInputFromArray<float>(TensorShape({}), {second_max});
+  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+  TF_ASSERT_OK(RunOpKernel());
+  const Tensor& output_quantized = *GetOutput(0);
+  const float output_min = GetOutput(1)->flat<float>()(0);
+  const float output_max = GetOutput(2)->flat<float>()(0);
+  Tensor output_float =
+      QuantizedTensorToFloat<quint8>(output_quantized, output_min, output_max);
+  // Using the same error tolerance as in Eigen QuantizedConcat test
+  test::ExpectTensorNear<float>(expected_float, output_float, 1.0);
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/quantized_concat_op.cc b/tensorflow/core/kernels/quantized_concat_op.cc
index b03ac8e87d..59a8f28e26 100644
--- a/tensorflow/core/kernels/quantized_concat_op.cc
+++ b/tensorflow/core/kernels/quantized_concat_op.cc
@@ -17,13 +17,13 @@ limitations under the License.
 
 #include <vector>
 
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/kernels/concat_lib_cpu.h"
 #include "tensorflow/core/kernels/quantization_utils.h"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
 namespace tensorflow {
 
@@ -135,8 +135,8 @@ class QuantizedConcatOp : public OpKernel {
           context, in.dims() == input_dims || (input_is_scalar && in_is_scalar),
           errors::InvalidArgument(
               "ConcatOp : Ranks of all input tensors should match: shape[0] = ",
-              input_shape.DebugString(), " vs. shape[", i,
-              "] = ", in.shape().DebugString()));
+              input_shape.DebugString(), " vs. shape[", i, "] = ",
+              in.shape().DebugString()));
       for (int j = 0; j < input_dims; ++j) {
         if (j == concat_dim) {
           continue;
@@ -145,8 +145,8 @@ class QuantizedConcatOp : public OpKernel {
             context, in.dim_size(j) == input_shape.dim_size(j),
             errors::InvalidArgument(
                 "ConcatOp : Dimensions of inputs should match: shape[0] = ",
-                input_shape.DebugString(), " vs. shape[", i,
-                "] = ", in.shape().DebugString()));
+                input_shape.DebugString(), " vs. shape[", i, "] = ",
+                in.shape().DebugString()));
       }
       if (in.NumElements() > 0) {
         int64 inputs_flat_dim1 = in.NumElements() / inputs_flat_dim0;
@@ -184,9 +184,8 @@ class QuantizedConcatOp : public OpKernel {
     const int input_dims = values[0].dims();
     const TensorShape& input_shape = values[0].shape();
     OP_REQUIRES(
-        context,
-        (0 <= concat_dim && concat_dim < input_dims) ||
-            (allow_legacy_scalars() && concat_dim == 0),
+        context, (0 <= concat_dim && concat_dim < input_dims) ||
+                     (allow_legacy_scalars() && concat_dim == 0),
         errors::InvalidArgument(
             "ConcatOp : Expected concatenating dimensions in the range [", 0,
             ", ", input_dims, "), but got ", concat_dim));
@@ -246,4 +245,16 @@ REGISTER_QUANTIZED_CONCAT(qint32);
 
 #undef REGISTER_QUANTIZED_CONCAT
 
+#ifdef INTEL_MKL
+#define REGISTER_QUANTIZED_CONCATV2(type)                \
+  REGISTER_KERNEL_BUILDER(Name("QuantizedConcatV2")      \
+                              .Device(DEVICE_CPU)        \
+                              .TypeConstraint<type>("T") \
+                              .HostMemory("axis"),       \
+                          QuantizedConcatOp<type>)
+
+REGISTER_QUANTIZED_CONCATV2(quint8);
+REGISTER_QUANTIZED_CONCATV2(qint32);
+#endif
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/ops/mkl_array_ops.cc b/tensorflow/core/ops/mkl_array_ops.cc
new file mode 100644
index 0000000000..0279401289
--- /dev/null
+++ b/tensorflow/core/ops/mkl_array_ops.cc
@@ -0,0 +1,92 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifdef INTEL_MKL
+
+// This file contains the registration of MKL-DNN array ops.
+
+#include "tensorflow/core/framework/common_shape_fns.h"
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/shape_inference.h"
+#include "tensorflow/core/framework/tensor.pb.h"
+#include "tensorflow/core/util/mirror_pad_mode.h"
+#include "tensorflow/core/util/padding.h"
+#include "tensorflow/core/util/strided_slice_op.h"
+#include "tensorflow/core/util/tensor_format.h"
+
+namespace tensorflow {
+
+using shape_inference::DimensionHandle;
+using shape_inference::InferenceContext;
+using shape_inference::ShapeHandle;
+using shape_inference::UnchangedShape;
+
+// Adding QuantizedConcatV2 op to be able to replace it by
+// _MklQuantizedConcatV2 in the graph rewrite.
+REGISTER_OP("QuantizedConcatV2")
+    .Input("values: N * T")
+    .Input("axis: Tidx")
+    .Input("input_mins: N * float32")
+    .Input("input_maxes: N * float32")
+    .Output("output: T")
+    .Output("output_min: float")
+    .Output("output_max: float")
+    .Attr("N: int >= 2")
+    .Attr("T: type")
+    .Attr("Tidx: {int32, int64} = DT_INT32")
+    .SetShapeFn([](InferenceContext* c) {
+      const int n = (c->num_inputs() - 1) / 3;
+      TF_RETURN_IF_ERROR(shape_inference::QuantizedConcatV2Shape(c, n));
+      ShapeHandle unused;
+      for (int i = n + 1; i < c->num_inputs(); ++i) {
+        TF_RETURN_IF_ERROR(c->WithRank(c->input(i), 0, &unused));
+      }
+      c->set_output(1, c->Scalar());
+      c->set_output(2, c->Scalar());
+      return Status::OK();
+    });
+
+REGISTER_OP("_MklQuantizedConcatV2")
+    .Input("values: N * T")
+    .Input("axis: Tidx")
+    .Input("input_mins:  N * float32")
+    .Input("input_maxes: N * float32")
+    .Input("mkl_values: N * uint8")
+    .Input("mkl_axis: uint8")
+    .Input("mkl_input_mins:  N * uint8")
+    .Input("mkl_input_maxes: N * uint8")
+    .Output("output: T")
+    .Output("output_min: float")
+    .Output("output_max: float")
+    .Output("mkl_output: uint8")
+    .Output("mkl_output_min: uint8")
+    .Output("mkl_output_max: uint8")
+    .Attr("N: int >= 2")
+    .Attr("T: type")
+    .Attr("Tidx: {int32, int64} = DT_INT32")
+    .SetShapeFn([](InferenceContext* c) {
+      const int n = (c->num_inputs() / 2 - 1) / 3;
+      TF_RETURN_IF_ERROR(shape_inference::QuantizedConcatV2Shape(c, n));
+      ShapeHandle unused;
+      for (int i = n + 1; i < c->num_inputs() / 2; ++i) {
+        TF_RETURN_IF_ERROR(c->WithRank(c->input(i), 0, &unused));
+      }
+      c->set_output(1, c->Scalar());
+      c->set_output(2, c->Scalar());
+      return Status::OK();
+    });
+}
+
+#endif
-- 
GitLab


From 05d9d59f14555c1111f9398bcc267817c981e688 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Tue, 5 Feb 2019 17:20:07 -0800
Subject: [PATCH 1967/2345] [tf.data] Make sure that TF V2 tests use tf.data V2
 API.

PiperOrigin-RevId: 232589427
---
 .../kernel_tests/assert_element_shape_test.py    |  2 ++
 .../python/kernel_tests/lmdb_dataset_op_test.py  |  2 ++
 .../python/kernel_tests/reduce_dataset_test.py   |  2 ++
 .../python/kernel_tests/slide_dataset_op_test.py |  2 ++
 .../kernel_tests/map_defun_op_test.py            |  3 ++-
 .../optimization/optimize_dataset_test.py        | 11 ++++++-----
 .../checkpoint_input_pipeline_hook_test.py       |  7 ++-----
 .../dataset_serialization_test_base.py           |  2 --
 .../kernel_tests/stats_dataset_ops_test.py       |  6 ++++--
 .../kernel_tests/wrap_unwrap_test.py             |  6 +++---
 .../python/data/experimental/ops/resampling.py   |  5 ++---
 .../python/data/kernel_tests/dataset_test.py     | 16 +++++++---------
 .../python/data/kernel_tests/flat_map_test.py    | 10 +++++-----
 .../from_sparse_tensor_slices_test.py            |  3 +--
 tensorflow/python/data/kernel_tests/map_test.py  |  6 ++++--
 .../python/data/kernel_tests/optional_test.py    |  2 +-
 .../python/data/kernel_tests/shard_test.py       |  2 +-
 tensorflow/python/data/kernel_tests/test_base.py |  8 ++++++++
 18 files changed, 54 insertions(+), 41 deletions(-)

diff --git a/tensorflow/contrib/data/python/kernel_tests/assert_element_shape_test.py b/tensorflow/contrib/data/python/kernel_tests/assert_element_shape_test.py
index 6c5f8c6b00..4db711c1f3 100644
--- a/tensorflow/contrib/data/python/kernel_tests/assert_element_shape_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/assert_element_shape_test.py
@@ -25,11 +25,13 @@ from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import script_ops
 from tensorflow.python.platform import test
 
 
+@test_util.run_v1_only("deprecated API, no eager or V2 test coverage")
 class AssertElementShapeTest(test_base.DatasetTestBase):
 
   def test_assert_element_shape(self):
diff --git a/tensorflow/contrib/data/python/kernel_tests/lmdb_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/lmdb_dataset_op_test.py
index b9840b1ff1..220f9934b6 100644
--- a/tensorflow/contrib/data/python/kernel_tests/lmdb_dataset_op_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/lmdb_dataset_op_test.py
@@ -27,12 +27,14 @@ from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 from tensorflow.python.util import compat
 
 prefix_path = "tensorflow/core/lib"
 
 
+@test_util.run_v1_only("deprecated API, no eager or V2 test coverage")
 class LMDBDatasetTest(test_base.DatasetTestBase):
 
   def setUp(self):
diff --git a/tensorflow/contrib/data/python/kernel_tests/reduce_dataset_test.py b/tensorflow/contrib/data/python/kernel_tests/reduce_dataset_test.py
index e7281d5318..78019fcc7d 100644
--- a/tensorflow/contrib/data/python/kernel_tests/reduce_dataset_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/reduce_dataset_test.py
@@ -25,10 +25,12 @@ from tensorflow.contrib.data.python.ops import grouping
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
 
+@test_util.run_v1_only("deprecated API, no eager or V2 test coverage")
 class ReduceDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   @parameterized.named_parameters(
diff --git a/tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py
index 2527706709..9275a36582 100644
--- a/tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py
@@ -26,11 +26,13 @@ from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
 
+@test_util.run_v1_only("deprecated API, no eager or V2 test coverage")
 class SlideDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   @parameterized.named_parameters(
diff --git a/tensorflow/python/data/experimental/kernel_tests/map_defun_op_test.py b/tensorflow/python/data/experimental/kernel_tests/map_defun_op_test.py
index 19830a23bb..a48f0808a6 100644
--- a/tensorflow/python/data/experimental/kernel_tests/map_defun_op_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/map_defun_op_test.py
@@ -28,6 +28,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_spec
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import data_flow_ops
@@ -35,7 +36,7 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
 
-# TODO(b/117581999): add eager coverage.
+@test_util.run_v1_only("b/123903858: Add eager and V2 test coverage")
 class MapDefunTest(test_base.DatasetTestBase):
 
   def testMapDefunSimple(self):
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
index bcd027ebbb..a85e0cf801 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
@@ -112,7 +112,7 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = self.getNext(dataset)
     self.evaluate(get_next())
 
-  # TODO(b/123300735): Add eager coverage for the following tests.
+  @test_util.run_v1_only("b/123902160")
   def testSkipEagerOptimizationLargeInputFromTensor(self):
     input_t = array_ops.placeholder(dtypes.int32, (None, None, None))
     dataset = dataset_ops.Dataset.from_tensors(input_t)
@@ -127,7 +127,7 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       sess.run(init_op, {input_t: np.ones([512, 1024, 1025], np.int32)})
       self.evaluate(get_next)
 
-  # TODO(b/117581999): Add eager coverage for the following tests.
+  @test_util.run_v1_only("b/123902160")
   def testSkipEagerOptimizationLargeInputFromTensorSlices(self):
     input_t = array_ops.placeholder(dtypes.int32, (None, None, None, None))
     dataset = dataset_ops.Dataset.from_tensor_slices(input_t)
@@ -219,7 +219,7 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     self.assertDatasetProduces(dataset, expected_output=[0])
 
   @parameterized.named_parameters(_generate_captured_refvar_test_cases())
-  # Skip eager because RefVariables are not supported in eager mode.
+  @test_util.run_v1_only("RefVariables are not supported in eager mode.")
   def testSkipEagerOptimizationWithCapturedRefVar(self, dataset_fn):
     """Tests that default optimizations are disabled with ref variables."""
     variable = variable_scope.get_variable(
@@ -236,7 +236,7 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       options.experimental_optimization.noop_elimination = True
       options.experimental_optimization.map_and_batch_fusion = True
       optimized_dataset = unoptimized_dataset.with_options(options)
-      optimized_it = optimized_dataset.make_initializable_iterator()
+      optimized_it = dataset_ops.make_initializable_iterator(optimized_dataset)
 
     self.assertGreaterEqual(len(w), 1)
     expected = ("tf.data static optimizations are not compatible with "
@@ -248,7 +248,8 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     # Check that outputs are the same in the optimized and unoptimized cases,
     # when the variable value is changing.
-    unoptimized_it = unoptimized_dataset.make_initializable_iterator()
+    unoptimized_it = dataset_ops.make_initializable_iterator(
+        unoptimized_dataset)
     with ops.control_dependencies([assign_op]):
       unoptimized_output = unoptimized_it.get_next()
       optimized_output = optimized_it.get_next()
diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/checkpoint_input_pipeline_hook_test.py b/tensorflow/python/data/experimental/kernel_tests/serialization/checkpoint_input_pipeline_hook_test.py
index 8cc66d0c29..84b8e5ca36 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/checkpoint_input_pipeline_hook_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/checkpoint_input_pipeline_hook_test.py
@@ -19,7 +19,6 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.data.experimental.ops import iterator_ops
-from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -35,7 +34,8 @@ from tensorflow_estimator.python.estimator import estimator
 from tensorflow_estimator.python.estimator import model_fn
 
 
-class CheckpointInputPipelineHookTest(test_base.DatasetTestBase):
+@test_util.run_v1_only('b/123904664')
+class CheckpointInputPipelineHookTest(test.TestCase):
 
   @staticmethod
   def _model_fn(features, labels, mode, config):
@@ -69,7 +69,6 @@ class CheckpointInputPipelineHookTest(test_base.DatasetTestBase):
   def _build_iterator_saver_hook(self, est):
     return iterator_ops.CheckpointInputPipelineHook(est)
 
-  @test_util.run_deprecated_v1
   def testReturnDatasetFromInputFn(self):
 
     def _input_fn():
@@ -82,7 +81,6 @@ class CheckpointInputPipelineHookTest(test_base.DatasetTestBase):
     est.train(_input_fn, steps=2, hooks=[self._build_iterator_saver_hook(est)])
     self.assertSequenceEqual(self._read_vars(est.model_dir), (4, 3))
 
-  @test_util.run_deprecated_v1
   def testBuildIteratorInInputFn(self):
 
     def _input_fn():
@@ -97,7 +95,6 @@ class CheckpointInputPipelineHookTest(test_base.DatasetTestBase):
     est.train(_input_fn, steps=2, hooks=[self._build_iterator_saver_hook(est)])
     self.assertSequenceEqual(self._read_vars(est.model_dir), (4, 3))
 
-  @test_util.run_deprecated_v1
   def testDoNotRestore(self):
 
     def _input_fn():
diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/dataset_serialization_test_base.py b/tensorflow/python/data/experimental/kernel_tests/serialization/dataset_serialization_test_base.py
index bdbd8702b7..ca45ecca4c 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/dataset_serialization_test_base.py
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/dataset_serialization_test_base.py
@@ -23,7 +23,6 @@ import os
 import numpy as np
 
 from tensorflow.python.data.experimental.ops import iterator_ops as contrib_iterator_ops
-from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.framework import dtypes
@@ -78,7 +77,6 @@ class DatasetSerializationTestBase(test.TestCase):
     # NOTE: We disable all default optimizations in serialization tests in order
     # to test the actual dataset in question.
     options = dataset_ops.Options()
-    options.experimental_optimization = OptimizationOptions()
     options.experimental_optimization.apply_default_optimizations = False
 
     def ds_fn1_no_opt():
diff --git a/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py b/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py
index 2e40c5236b..c53ac82c6e 100644
--- a/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py
@@ -59,7 +59,8 @@ def function_apply_options(dataset, aggregator, prefix="", counter_prefix=""):
 )
 class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
 
-  def testBytesProduced(self, dataset_transformation):
+  @test_util.run_v1_only("b/123901126")
+  def testSkipEagerBytesProduced(self, dataset_transformation):
     aggregator = stats_aggregator.StatsAggregator()
     dataset = dataset_ops.Dataset.range(100).map(
         lambda x: array_ops.tile([x], ops.convert_to_tensor([x]))).apply(
@@ -358,7 +359,8 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
 @test_util.run_all_in_graph_and_eager_modes
 @parameterized.named_parameters(
     ("SetStatsAggregator", function_set_stats_aggregator),
-    ("StatsOptions", function_apply_options))
+    ("StatsOptions", function_apply_options)
+)
 class FeatureStatsDatasetTest(
     stats_dataset_test_base.StatsDatasetTestBase,
     reader_dataset_ops_test_base.MakeBatchedFeaturesDatasetTestBase):
diff --git a/tensorflow/python/data/experimental/kernel_tests/wrap_unwrap_test.py b/tensorflow/python/data/experimental/kernel_tests/wrap_unwrap_test.py
index a8f5050151..e6e77575a6 100644
--- a/tensorflow/python/data/experimental/kernel_tests/wrap_unwrap_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/wrap_unwrap_test.py
@@ -31,7 +31,7 @@ class WrapDatasetVariantTest(test_base.DatasetTestBase):
 
   def testBasic(self):
     ds = dataset_ops.Dataset.range(100)
-    ds_variant = ds._as_variant_tensor()  # pylint: disable=protected-access
+    ds_variant = ds._variant_tensor  # pylint: disable=protected-access
 
     wrapped_variant = gen_dataset_ops.wrap_dataset_variant(ds_variant)
     unwrapped_variant = gen_dataset_ops.unwrap_dataset_variant(wrapped_variant)
@@ -42,10 +42,10 @@ class WrapDatasetVariantTest(test_base.DatasetTestBase):
     for i in range(100):
       self.assertEqual(i, self.evaluate(get_next()))
 
-  # TODO(b/117581999): add eager coverage when supported.
+  @test_util.run_v1_only("b/123901304")
   def testSkipEagerGPU(self):
     ds = dataset_ops.Dataset.range(100)
-    ds_variant = ds._as_variant_tensor()  # pylint: disable=protected-access
+    ds_variant = ds._variant_tensor  # pylint: disable=protected-access
     wrapped_variant = gen_dataset_ops.wrap_dataset_variant(ds_variant)
 
     with ops.device("/gpu:0"):
diff --git a/tensorflow/python/data/experimental/ops/resampling.py b/tensorflow/python/data/experimental/ops/resampling.py
index 3a3040ae9a..6676085ae5 100644
--- a/tensorflow/python/data/experimental/ops/resampling.py
+++ b/tensorflow/python/data/experimental/ops/resampling.py
@@ -168,8 +168,7 @@ def _filter_ds(dataset, acceptance_dist_ds, initial_dist_ds, class_values_ds,
 def _estimate_initial_dist_ds(
     target_dist_t, class_values_ds, dist_estimation_batch_size=32,
     smoothing_constant=10):
-  num_classes = (target_dist_t.shape[0].value or
-                 array_ops.shape(target_dist_t)[0])
+  num_classes = (target_dist_t.shape[0] or array_ops.shape(target_dist_t)[0])
   initial_examples_per_class_seen = array_ops.fill(
       [num_classes], np.int64(smoothing_constant))
 
@@ -207,7 +206,7 @@ def _estimate_data_distribution(c, num_examples_per_class_seen):
       `[num_classes]`.
     dist: The updated distribution.  Type `float32`, shape `[num_classes]`.
   """
-  num_classes = num_examples_per_class_seen.get_shape()[0].value
+  num_classes = num_examples_per_class_seen.get_shape()[0]
   # Update the class-count based on what labels are seen in batch.
   num_examples_per_class_seen = math_ops.add(
       num_examples_per_class_seen, math_ops.reduce_sum(
diff --git a/tensorflow/python/data/kernel_tests/dataset_test.py b/tensorflow/python/data/kernel_tests/dataset_test.py
index f319b24bee..1e764b3e25 100644
--- a/tensorflow/python/data/kernel_tests/dataset_test.py
+++ b/tensorflow/python/data/kernel_tests/dataset_test.py
@@ -92,15 +92,16 @@ class DatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       ("TFRecord", lambda: readers.TFRecordDataset(""), 1),
   )
   def testDatasetSimpleSourceInputs(self, dataset_fn, num_inputs=0):
-    self.assertEqual(num_inputs, len(dataset_fn()._inputs()))
+    self.assertLen(dataset_fn()._inputs(), num_inputs)
 
+  @test_util.run_v1_only("deprecated API, no eager or V2 test coverage")
   def testDatasetComplexSourceInputs(self):
     dataset_fn = dataset_ops.Dataset.from_sparse_tensor_slices(
         sparse_tensor.SparseTensor(
             indices=np.array([[0, 0], [1, 0], [2, 0]]),
             values=np.array([0, 0, 0]),
             dense_shape=np.array([3, 1])))
-    self.assertEqual(0, len(dataset_fn._inputs()))
+    self.assertEmpty(dataset_fn._inputs())
 
   @parameterized.named_parameters(
       ("Batch",
@@ -266,27 +267,24 @@ class DatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
           round_trip_dataset, [self.evaluate(tf_value_fn())],
           requires_initialization=True)
 
-  # NOTE: This test is specific to graph mode and is skipped in eager mode.
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("graph mode specific, no eager or V2 test coverage")
   def testSkipEagerSameGraphErrorOneShot(self):
     dataset = dataset_ops.Dataset.range(10)
     with ops.Graph().as_default():
       with self.assertRaisesRegexp(ValueError, "must be from the same graph"):
         dataset = dataset.batch(2)
 
-  # NOTE: This test is specific to graph mode and is skipped in eager mode.
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("graph mode specific, no eager or V2 test coverage")
   def testSkipEagerSameGraphErrorOneShotSimple(self):
     dataset = dataset_ops.Dataset.range(10)
     with ops.Graph().as_default():
       with test.mock.patch.object(logging, "warning") as mock_log:
-        _ = dataset.make_one_shot_iterator()
+        _ = dataset_ops.make_one_shot_iterator(dataset)
         self.assertRegexpMatches(
             str(mock_log.call_args), "Please ensure that all datasets in the "
             "pipeline are created in the same graph as the iterator.")
 
-  # NOTE: This test is specific to graph mode and is skipped in eager mode.
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("graph mode specific, no eager or V2 test coverage")
   def testSkipEagerSameGraphErrorInitializable(self):
     dataset = dataset_ops.Dataset.range(10)
     with ops.Graph().as_default():
diff --git a/tensorflow/python/data/kernel_tests/flat_map_test.py b/tensorflow/python/data/kernel_tests/flat_map_test.py
index ff52821b10..69b5fd0d77 100644
--- a/tensorflow/python/data/kernel_tests/flat_map_test.py
+++ b/tensorflow/python/data/kernel_tests/flat_map_test.py
@@ -65,11 +65,11 @@ class FlatMapTest(test_base.DatasetTestBase):
     repeats = [[1, 2], [3, 4], [5, 0], [1, 7]]
     components = np.array(repeats, dtype=np.int64)
     iterator = (
-        dataset_ops.Dataset.from_tensor_slices(components)
-        .flat_map(lambda x: dataset_ops.Dataset.from_tensor_slices(x)
-                  .flat_map(lambda y: dataset_ops.Dataset.from_tensors(y)
-                            .repeat(y))).make_initializable_iterator(
-                                shared_name="shared_flat_map_iterator"))
+        dataset_ops.make_initializable_iterator(
+            dataset_ops.Dataset.from_tensor_slices(components).flat_map(
+                lambda x: dataset_ops.Dataset.from_tensor_slices(x).flat_map(
+                    lambda y: dataset_ops.Dataset.from_tensors(y).repeat(y))),
+            shared_name="shared_flat_map_iterator"))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
diff --git a/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py b/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py
index 546c2fb2ed..2ce9c9a061 100644
--- a/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py
+++ b/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py
@@ -29,10 +29,9 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
 
-# NOTE: deprecated method in V2, no eager coverage added.
+@test_util.run_v1_only("deprecated API, no eager or V2 test coverage")
 class FromSparseTensorSlicesTest(test_base.DatasetTestBase):
 
-  @test_util.run_deprecated_v1
   def testFromSparseTensorSlices(self):
     """Test a dataset based on slices of a `tf.SparseTensor`."""
     st = array_ops.sparse_placeholder(dtypes.float64)
diff --git a/tensorflow/python/data/kernel_tests/map_test.py b/tensorflow/python/data/kernel_tests/map_test.py
index a56a3afad6..1cd1ddc351 100644
--- a/tensorflow/python/data/kernel_tests/map_test.py
+++ b/tensorflow/python/data/kernel_tests/map_test.py
@@ -313,8 +313,8 @@ class MapTest(test_base.DatasetTestBase, parameterized.TestCase):
       if context.executing_eagerly():
         captured_iterator = iter(dataset_ops.Dataset.range(10))
       else:
-        captured_iterator = dataset_ops.Dataset.range(
-            10).make_initializable_iterator()
+        captured_iterator = dataset_ops.make_initializable_iterator(
+            dataset_ops.Dataset.range(10))
       ds = _build_ds(captured_iterator)
       return captured_iterator, ds
 
@@ -351,6 +351,7 @@ class MapTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.assertRaises(errors.OutOfRangeError):
       self.evaluate(get_next())
 
+  @test_util.run_v1_only("b/123904513")
   def testCaptureQueue(self):
     elements = np.random.randint(100, size=[200])
     queue = data_flow_ops.FIFOQueue(200, dtypes.int64, shapes=[])
@@ -816,6 +817,7 @@ class MapTest(test_base.DatasetTestBase, parameterized.TestCase):
         dataset,
         expected_output=[self.evaluate(_check(_sparse(i))) for i in range(10)])
 
+  @test_util.run_v1_only("b/123904513")
   def testParallelMapOutOfRangeError(self):
     def raising_py_func(i):
       if i == 100:
diff --git a/tensorflow/python/data/kernel_tests/optional_test.py b/tensorflow/python/data/kernel_tests/optional_test.py
index 2269bb8724..4fde0aa002 100644
--- a/tensorflow/python/data/kernel_tests/optional_test.py
+++ b/tensorflow/python/data/kernel_tests/optional_test.py
@@ -329,7 +329,7 @@ class OptionalTest(test_base.DatasetTestBase, parameterized.TestCase):
     if not works_on_gpu and test.is_gpu_available():
       self.skipTest("Test case not yet supported on GPU.")
     ds = dataset_ops.Dataset.from_tensors(np_value).repeat(3)
-    iterator = ds.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(ds)
     next_elem = iterator_ops.get_next_as_optional(iterator)
     self.assertIsInstance(next_elem, optional_ops.Optional)
     self.assertTrue(
diff --git a/tensorflow/python/data/kernel_tests/shard_test.py b/tensorflow/python/data/kernel_tests/shard_test.py
index bdd2e899f5..9fc70ff607 100644
--- a/tensorflow/python/data/kernel_tests/shard_test.py
+++ b/tensorflow/python/data/kernel_tests/shard_test.py
@@ -24,7 +24,7 @@ from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
-@test_util.run_all_in_graph_and_eager_modes
+@test_util.run_v1_only("deprecated API, no eager or V2 test coverage")
 class ShardTest(test_base.DatasetTestBase):
 
   def testSimpleCase(self):
diff --git a/tensorflow/python/data/kernel_tests/test_base.py b/tensorflow/python/data/kernel_tests/test_base.py
index 19a1a0de6d..57df29ead5 100644
--- a/tensorflow/python/data/kernel_tests/test_base.py
+++ b/tensorflow/python/data/kernel_tests/test_base.py
@@ -19,6 +19,7 @@ from __future__ import print_function
 
 import re
 
+from tensorflow.python import tf2
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import nest
 from tensorflow.python.eager import context
@@ -32,6 +33,13 @@ from tensorflow.python.platform import test
 class DatasetTestBase(test.TestCase):
   """Base class for dataset tests."""
 
+  @classmethod
+  def setUpClass(cls):
+    if tf2.enabled():
+      dataset_ops.Dataset = dataset_ops.DatasetV2
+    else:
+      dataset_ops.Dataset = dataset_ops.DatasetV1
+
   def assertSparseValuesEqual(self, a, b):
     """Asserts that two SparseTensors/SparseTensorValues are equal."""
     self.assertAllEqual(a.indices, b.indices)
-- 
GitLab


From f7fba1b59bc79f99f5624755cf244c873acb9c82 Mon Sep 17 00:00:00 2001
From: Lukas Geiger <lukas.geiger94@gmail.com>
Date: Wed, 6 Feb 2019 02:48:31 +0100
Subject: [PATCH 1968/2345] Remove reciprocal from monotonic decreasing ops

---
 tensorflow/core/grappler/op_types.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc
index e8179b22a8..4323fc4cc4 100644
--- a/tensorflow/core/grappler/op_types.cc
+++ b/tensorflow/core/grappler/op_types.cc
@@ -184,8 +184,7 @@ bool IsElementWiseMonotonic(const NodeDef& node, bool* is_non_decreasing) {
           "Sign",  "Sinh", "Softsign", "Softplus", "Sqrt",  "Tanh",
       }));
   static const gtl::FlatSet<string>* const kMonotonicNonIncreasingOps =
-      CHECK_NOTNULL((new gtl::FlatSet<string>{"Acos", "Erfc", "Inv", "Neg",
-                                              "Reciprocal", "Rsqrt"}));
+      CHECK_NOTNULL((new gtl::FlatSet<string>{"Acos", "Erfc", "Neg", "Rsqrt"}));
   if (kMonotonicNonDecreasingOps->count(node.op()) > 0) {
     if (is_non_decreasing) {
       *is_non_decreasing = true;
-- 
GitLab


From ac70b2793540c43e1342c0895b089a0f787a2e62 Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Tue, 5 Feb 2019 17:46:00 -0800
Subject: [PATCH 1969/2345] Optimize Mirror pad

PiperOrigin-RevId: 232592700
---
 tensorflow/lite/kernels/mirror_pad.cc      | 32 +++++++++++++++++-----
 tensorflow/lite/kernels/mirror_pad_test.cc | 13 +++++++++
 2 files changed, 38 insertions(+), 7 deletions(-)

diff --git a/tensorflow/lite/kernels/mirror_pad.cc b/tensorflow/lite/kernels/mirror_pad.cc
index c0924da2a1..65a98ef450 100644
--- a/tensorflow/lite/kernels/mirror_pad.cc
+++ b/tensorflow/lite/kernels/mirror_pad.cc
@@ -37,6 +37,10 @@ struct PaddedTensor {
   // Note: This is not owned by default. It will point to the value
   // in the input tensor.
   const void* value = nullptr;
+  // The start index of the values of this tensor in the output buffer.
+  int start = -1;
+  // The end index of the values of this tensor in the output buffer.
+  int end = -1;
   // If this tensor is not one value, then this vector will have
   // all the tensors that belongs to this tensor.
   // Pointers are not owned.
@@ -66,8 +70,6 @@ struct PaddedTensor {
 struct OpData {
   // Holds intermediate data structure of the padded tensor.
   std::vector<PaddedTensor> pad_tensor_buffer;
-  // Total number of intermediate elements in the pad_tensor_buffer.
-  int num_elements;
 };
 
 // Util method to initialize the memory of the padded tensor.
@@ -79,6 +81,7 @@ void InitializeTensorMemory(const TfLiteIntArray* const dims, int dims_size,
   // nodes in the next level, and swap while moving on dimensions of the tensor.
   std::vector<PaddedTensor*> current_nodes, next_level;
   current_nodes.push_back(&(*padded_tensor_buffer)[element_index]);
+  current_nodes[0]->start = current_nodes[0]->end = -1;
   element_index++;
   int next_level_size = 1;
   while (!current_nodes.empty() && dimension_index < dims_size) {
@@ -90,6 +93,7 @@ void InitializeTensorMemory(const TfLiteIntArray* const dims, int dims_size,
       padded_tensor->values.resize(dims->data[dimension_index]);
       for (int i = 0; i < dims->data[dimension_index]; ++i) {
         padded_tensor->values[i] = &(*padded_tensor_buffer)[element_index];
+        padded_tensor->values[i]->start = padded_tensor->values[i]->end = -1;
         next_level[index++] = padded_tensor->values[i];
         element_index++;
       }
@@ -238,25 +242,40 @@ TfLiteStatus PadTensor(const TfLiteTensor* padding_matrix, int offset,
 // Fills 'output_data' with data from 'padded_tensor'.
 // The function does this recursively by setting left padding first then
 // original data, followed by the right padding.
+// The functions returns the index in 'output_data' to be filled with data.
 template <typename T>
-int FillOutput(const PaddedTensor* padded_tensor, T* output_data,
+int FillOutput(PaddedTensor* padded_tensor, T* output_data,
                int index_in_output) {
   if (padded_tensor == nullptr || output_data == nullptr) {
     return -1;
   }
+  // Check if this tensor value was computed and written in the output
+  // already. If yes, just copy the values.
+  if (padded_tensor->start != -1) {
+    const int size = padded_tensor->end - padded_tensor->start + 1;
+    memcpy(output_data + index_in_output, output_data + padded_tensor->start,
+           size * sizeof(T));
+    return index_in_output + size;
+  }
+  // Record the start index in the output.
+  padded_tensor->start = index_in_output;
+  // Check for single value.
   if (padded_tensor->value != nullptr) {
     output_data[index_in_output] = *static_cast<const T*>(padded_tensor->value);
+    padded_tensor->end = index_in_output;
     return index_in_output + 1;
   }
-  for (const auto* tensor : padded_tensor->left_pad_ptrs) {
+  for (auto* tensor : padded_tensor->left_pad_ptrs) {
     index_in_output = FillOutput(tensor, output_data, index_in_output);
   }
-  for (const auto& tensor : padded_tensor->values) {
+  for (auto& tensor : padded_tensor->values) {
     index_in_output = FillOutput(tensor, output_data, index_in_output);
   }
-  for (const auto* tensor : padded_tensor->right_pad_ptrs) {
+  for (auto* tensor : padded_tensor->right_pad_ptrs) {
     index_in_output = FillOutput(tensor, output_data, index_in_output);
   }
+  // Record the end index in the output.
+  padded_tensor->end = index_in_output - 1;
   return index_in_output;
 }
 
@@ -373,7 +392,6 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
     num_elements += extra_nodes;
   }
   op_data->pad_tensor_buffer.resize(num_elements);
-  op_data->num_elements = num_elements;
 
   if (!IsConstantTensor(padding_matrix)) {
     SetTensorToDynamic(output_tensor);
diff --git a/tensorflow/lite/kernels/mirror_pad_test.cc b/tensorflow/lite/kernels/mirror_pad_test.cc
index fd09e6e449..91e48fa68a 100644
--- a/tensorflow/lite/kernels/mirror_pad_test.cc
+++ b/tensorflow/lite/kernels/mirror_pad_test.cc
@@ -185,5 +185,18 @@ TEST(MirrorPadTest, Pad_1D_Symmetric) {
   EXPECT_THAT(model.GetOutput(), ElementsAreArray({1, 2, 3, 3, 2}));
 }
 
+TEST(MirrorPadTest, Pad_1D_Symmetric_Multiple_Invoke) {
+  BaseMirrorPadOpModel<int> model(
+      {TensorType_INT32, {3}}, {TensorType_INT32, {1, 2}},
+      {TensorType_INT32, {}}, tflite::MirrorPadMode_SYMMETRIC);
+  model.PopulateTensor<int>(model.input_tensor_id(), {1, 2, 3});
+  model.PopulateTensor<int>(model.padding_matrix_tensor_id(), {0, 2});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({1, 2, 3, 3, 2}));
+  model.PopulateTensor<int>(model.input_tensor_id(), {4, 5, 6});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({4, 5, 6, 6, 5}));
+}
+
 }  // namespace
 }  // namespace tflite
-- 
GitLab


From b7c9ea172f2faedc9257dd406bb1e336c12284fa Mon Sep 17 00:00:00 2001
From: Mahmoud Abuzaina <mahmoud.abuzaina@intel.com>
Date: Tue, 5 Feb 2019 17:57:44 -0800
Subject: [PATCH 1970/2345] Added #ifdef INTEL_MKL to the Quantized Concat test

---
 tensorflow/core/kernels/mkl_quantized_concat_op_test.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/core/kernels/mkl_quantized_concat_op_test.cc b/tensorflow/core/kernels/mkl_quantized_concat_op_test.cc
index dcd1de95e0..fc68480bbe 100644
--- a/tensorflow/core/kernels/mkl_quantized_concat_op_test.cc
+++ b/tensorflow/core/kernels/mkl_quantized_concat_op_test.cc
@@ -13,6 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#if defined(INTEL_MKL) && defined(ENABLE_MKL)
+
 #define EIGEN_USE_THREADS
 
 #include <functional>
@@ -228,3 +230,5 @@ void QuantizedConcatTest::TestSecondDim8Bit(float first_min, float first_max,
 }
 
 }  // namespace tensorflow
+
+#endif  // INTEL_MKL && ENABLE_MKL
-- 
GitLab


From 9c65f0e235db7b91814f7cb565bf32ca9e07cb2e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 5 Feb 2019 17:54:56 -0800
Subject: [PATCH 1971/2345] Add LinearOperator.adjoint method. Also add
 LinearOperator.H as a shorter version of this.

RELNOTES: Added LinearOperator.adjoint and LinearOperator.H (alias).
PiperOrigin-RevId: 232593870
---
 .../linalg/linear_operator_adjoint_test.py    |  24 +++
 .../linalg/linear_operator_algebra_test.py    |  48 +++++-
 .../linalg/linear_operator_block_diag_test.py |  21 +++
 .../linalg/linear_operator_diag_test.py       |   5 +
 .../linalg/linear_operator_identity_test.py   |   6 +
 .../linalg/linear_operator_kronecker_test.py  |  17 ++
 .../ops/linalg/adjoint_registrations.py       | 127 +++++++++++++++
 tensorflow/python/ops/linalg/linalg.py        |   1 +
 .../python/ops/linalg/linear_operator.py      |  20 +++
 .../ops/linalg/linear_operator_adjoint.py     |  23 ++-
 .../ops/linalg/linear_operator_algebra.py     |  71 +++++++++
 .../ops/linalg/linear_operator_test_util.py   |  17 ++
 ...flow.linalg.-linear-operator-adjoint.pbtxt | 150 ++++++++++++++++++
 ...w.linalg.-linear-operator-block-diag.pbtxt |   8 +
 ...ow.linalg.-linear-operator-circulant.pbtxt |   8 +
 ...linalg.-linear-operator-circulant2-d.pbtxt |   8 +
 ...linalg.-linear-operator-circulant3-d.pbtxt |   8 +
 ....linalg.-linear-operator-composition.pbtxt |   8 +
 ...sorflow.linalg.-linear-operator-diag.pbtxt |   8 +
 ....linalg.-linear-operator-full-matrix.pbtxt |   8 +
 ...low.linalg.-linear-operator-identity.pbtxt |   8 +
 ...ow.linalg.-linear-operator-inversion.pbtxt |   8 +
 ...ow.linalg.-linear-operator-kronecker.pbtxt |   8 +
 ...alg.-linear-operator-low-rank-update.pbtxt |   8 +
 ...lg.-linear-operator-lower-triangular.pbtxt |   8 +
 ...alg.-linear-operator-scaled-identity.pbtxt |   8 +
 ...orflow.linalg.-linear-operator-zeros.pbtxt |   8 +
 .../tensorflow.linalg.-linear-operator.pbtxt  |   8 +
 .../api/golden/v1/tensorflow.linalg.pbtxt     |   4 +
 ...flow.linalg.-linear-operator-adjoint.pbtxt | 150 ++++++++++++++++++
 ...w.linalg.-linear-operator-block-diag.pbtxt |   8 +
 ...ow.linalg.-linear-operator-circulant.pbtxt |   8 +
 ...linalg.-linear-operator-circulant2-d.pbtxt |   8 +
 ...linalg.-linear-operator-circulant3-d.pbtxt |   8 +
 ....linalg.-linear-operator-composition.pbtxt |   8 +
 ...sorflow.linalg.-linear-operator-diag.pbtxt |   8 +
 ....linalg.-linear-operator-full-matrix.pbtxt |   8 +
 ...low.linalg.-linear-operator-identity.pbtxt |   8 +
 ...ow.linalg.-linear-operator-inversion.pbtxt |   8 +
 ...ow.linalg.-linear-operator-kronecker.pbtxt |   8 +
 ...alg.-linear-operator-low-rank-update.pbtxt |   8 +
 ...lg.-linear-operator-lower-triangular.pbtxt |   8 +
 ...alg.-linear-operator-scaled-identity.pbtxt |   8 +
 ...orflow.linalg.-linear-operator-zeros.pbtxt |   8 +
 .../tensorflow.linalg.-linear-operator.pbtxt  |   8 +
 .../api/golden/v2/tensorflow.linalg.pbtxt     |   4 +
 46 files changed, 923 insertions(+), 5 deletions(-)
 create mode 100644 tensorflow/python/ops/linalg/adjoint_registrations.py
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-adjoint.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-adjoint.pbtxt

diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_adjoint_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_adjoint_test.py
index 1bed4b5268..f70d8c4e1c 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_adjoint_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_adjoint_test.py
@@ -114,5 +114,29 @@ class LinearOperatorAdjointTest(
     self.assertEqual("my_operator_adjoint", operator.name)
 
 
+class LinearOperatorAdjointNonSquareTest(
+    linear_operator_test_util.NonSquareLinearOperatorDerivedClassTest):
+  """Tests done in the base class NonSquareLinearOperatorDerivedClassTest."""
+
+  def _operator_and_matrix(self, build_info, dtype, use_placeholder):
+    shape_before_adjoint = list(build_info.shape)
+    # We need to swap the last two dimensions because we are taking the adjoint
+    # of this operator
+    shape_before_adjoint[-1], shape_before_adjoint[-2] = (
+        shape_before_adjoint[-2], shape_before_adjoint[-1])
+    matrix = linear_operator_test_util.random_normal(
+        shape_before_adjoint, dtype=dtype)
+
+    lin_op_matrix = matrix
+
+    if use_placeholder:
+      lin_op_matrix = array_ops.placeholder_with_default(matrix, shape=None)
+
+    operator = LinearOperatorAdjoint(
+        linalg.LinearOperatorFullMatrix(lin_op_matrix))
+
+    return operator, linalg.adjoint(matrix)
+
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_algebra_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_algebra_test.py
index ec78a3ffe0..12da8659ca 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_algebra_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_algebra_test.py
@@ -26,15 +26,59 @@ from tensorflow.python.ops.linalg import matmul_registrations  # pylint: disable
 from tensorflow.python.platform import test
 
 # pylint: disable=protected-access
+_ADJOINTS = linear_operator_algebra._ADJOINTS
+_registered_adjoint = linear_operator_algebra._registered_adjoint
 _CHOLESKY_DECOMPS = linear_operator_algebra._CHOLESKY_DECOMPS
-_MATMUL = linear_operator_algebra._MATMUL
 _registered_cholesky = linear_operator_algebra._registered_cholesky
-_registered_matmul = linear_operator_algebra._registered_matmul
 _INVERSES = linear_operator_algebra._INVERSES
 _registered_inverse = linear_operator_algebra._registered_inverse
+_MATMUL = linear_operator_algebra._MATMUL
+_registered_matmul = linear_operator_algebra._registered_matmul
 # pylint: enable=protected-access
 
 
+class AdjointTest(test.TestCase):
+
+  def testRegistration(self):
+
+    class CustomLinOp(linear_operator.LinearOperator):
+
+      def _matmul(self, a):
+        pass
+
+      def _shape(self):
+        return tensor_shape.TensorShape([1, 1])
+
+      def _shape_tensor(self):
+        pass
+
+    # Register Adjoint to a lambda that spits out the name parameter
+    @linear_operator_algebra.RegisterAdjoint(CustomLinOp)
+    def _adjoint(a):  # pylint: disable=unused-argument,unused-variable
+      return "OK"
+
+    self.assertEqual("OK", CustomLinOp(dtype=None).adjoint())
+
+  def testRegistrationFailures(self):
+
+    class CustomLinOp(linear_operator.LinearOperator):
+      pass
+
+    with self.assertRaisesRegexp(TypeError, "must be callable"):
+      linear_operator_algebra.RegisterAdjoint(CustomLinOp)("blah")
+
+    # First registration is OK
+    linear_operator_algebra.RegisterAdjoint(CustomLinOp)(lambda a: None)
+
+    # Second registration fails
+    with self.assertRaisesRegexp(ValueError, "has already been registered"):
+      linear_operator_algebra.RegisterAdjoint(CustomLinOp)(lambda a: None)
+
+  def testExactAdjointRegistrationsAllMatch(self):
+    for (k, v) in _ADJOINTS.items():
+      self.assertEqual(v, _registered_adjoint(k[0]))
+
+
 class CholeskyTest(test.TestCase):
 
   def testRegistration(self):
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_block_diag_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_block_diag_test.py
index 96e6e3c04c..28f8d20f61 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_block_diag_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_block_diag_test.py
@@ -136,6 +136,27 @@ class SquareLinearOperatorBlockDiagTest(
     self.assertTrue(operator.is_non_singular)
     self.assertFalse(operator.is_self_adjoint)
 
+  def test_block_diag_adjoint_type(self):
+    matrix = [[1., 0.], [0., 1.]]
+    operator = block_diag.LinearOperatorBlockDiag(
+        [
+            linalg.LinearOperatorFullMatrix(
+                matrix,
+                is_non_singular=True,
+            ),
+            linalg.LinearOperatorFullMatrix(
+                matrix,
+                is_non_singular=True,
+            ),
+        ],
+        is_non_singular=True,
+    )
+    adjoint = operator.adjoint()
+    self.assertIsInstance(
+        adjoint,
+        block_diag.LinearOperatorBlockDiag)
+    self.assertEqual(2, len(adjoint.operators))
+
   def test_block_diag_cholesky_type(self):
     matrix = [[1., 0.], [0., 1.]]
     operator = block_diag.LinearOperatorBlockDiag(
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py
index 4d7a31be87..5c3220e60f 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py
@@ -187,6 +187,11 @@ class LinearOperatorDiagTest(
         linalg_lib.LinearOperatorDiag))
     self.assertAllClose([6., 9.], self.evaluate(operator_matmul.diag))
 
+  def test_diag_adjoint_type(self):
+    diag = [1., 3., 5., 8.]
+    operator = linalg.LinearOperatorDiag(diag, is_non_singular=True)
+    self.assertIsInstance(operator.adjoint(), linalg.LinearOperatorDiag)
+
   def test_diag_cholesky_type(self):
     diag = [1., 3., 5., 8.]
     operator = linalg.LinearOperatorDiag(
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
index ea9ee99a58..55eff59e03 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
@@ -259,6 +259,12 @@ class LinearOperatorIdentityTest(
           is_non_singular=None,
       )
 
+  def test_identity_adjoint_type(self):
+    operator = linalg_lib.LinearOperatorIdentity(
+        num_rows=2, is_non_singular=True)
+    self.assertIsInstance(
+        operator.adjoint(), linalg_lib.LinearOperatorIdentity)
+
   def test_identity_cholesky_type(self):
     operator = linalg_lib.LinearOperatorIdentity(
         num_rows=2,
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_kronecker_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_kronecker_test.py
index 54ccc0c5f6..166188f6ce 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_kronecker_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_kronecker_test.py
@@ -192,6 +192,23 @@ class SquareLinearOperatorKroneckerTest(
     with self.assertRaisesRegexp(ValueError, ">=1 operators"):
       kronecker.LinearOperatorKronecker([])
 
+  def test_kronecker_adjoint_type(self):
+    matrix = [[1., 0.], [0., 1.]]
+    operator = kronecker.LinearOperatorKronecker(
+        [
+            linalg.LinearOperatorFullMatrix(
+                matrix, is_non_singular=True),
+            linalg.LinearOperatorFullMatrix(
+                matrix, is_non_singular=True),
+        ],
+        is_non_singular=True,
+    )
+    adjoint = operator.adjoint()
+    self.assertIsInstance(
+        adjoint,
+        kronecker.LinearOperatorKronecker)
+    self.assertEqual(2, len(adjoint.operators))
+
   def test_kronecker_cholesky_type(self):
     matrix = [[1., 0.], [0., 1.]]
     operator = kronecker.LinearOperatorKronecker(
diff --git a/tensorflow/python/ops/linalg/adjoint_registrations.py b/tensorflow/python/ops/linalg/adjoint_registrations.py
new file mode 100644
index 0000000000..59ec97d275
--- /dev/null
+++ b/tensorflow/python/ops/linalg/adjoint_registrations.py
@@ -0,0 +1,127 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Registrations for LinearOperator.adjoint."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.linalg import linear_operator
+from tensorflow.python.ops.linalg import linear_operator_adjoint
+from tensorflow.python.ops.linalg import linear_operator_algebra
+from tensorflow.python.ops.linalg import linear_operator_block_diag
+from tensorflow.python.ops.linalg import linear_operator_circulant
+from tensorflow.python.ops.linalg import linear_operator_diag
+from tensorflow.python.ops.linalg import linear_operator_identity
+from tensorflow.python.ops.linalg import linear_operator_kronecker
+
+
+# By default, return LinearOperatorAdjoint which switched the .matmul
+# and .solve methods.
+@linear_operator_algebra.RegisterAdjoint(linear_operator.LinearOperator)
+def _adjoint_linear_operator(linop):
+  return linear_operator_adjoint.LinearOperatorAdjoint(
+      linop,
+      is_non_singular=linop.is_non_singular,
+      is_self_adjoint=linop.is_self_adjoint,
+      is_positive_definite=linop.is_positive_definite,
+      is_square=linop.is_square)
+
+
+@linear_operator_algebra.RegisterAdjoint(
+    linear_operator_adjoint.LinearOperatorAdjoint)
+def _adjoint_adjoint_linear_operator(linop):
+  return linop.operator
+
+
+@linear_operator_algebra.RegisterAdjoint(
+    linear_operator_identity.LinearOperatorIdentity)
+def _adjoint_identity(identity_operator):
+  return identity_operator
+
+
+@linear_operator_algebra.RegisterAdjoint(
+    linear_operator_identity.LinearOperatorScaledIdentity)
+def _adjoint_scaled_identity(identity_operator):
+  multiplier = identity_operator.multiplier
+  if multiplier.dtype.is_complex:
+    multiplier = math_ops.conj(multiplier)
+
+  return linear_operator_identity.LinearOperatorScaledIdentity(
+      num_rows=identity_operator._num_rows,  # pylint: disable=protected-access
+      multiplier=multiplier,
+      is_non_singular=identity_operator.is_non_singular,
+      is_self_adjoint=identity_operator.is_self_adjoint,
+      is_positive_definite=identity_operator.is_positive_definite,
+      is_square=True)
+
+
+@linear_operator_algebra.RegisterAdjoint(
+    linear_operator_diag.LinearOperatorDiag)
+def _adjoint_diag(diag_operator):
+  diag = diag_operator.diag
+  if diag.dtype.is_complex:
+    diag = math_ops.conj(diag)
+
+  return linear_operator_diag.LinearOperatorDiag(
+      diag=diag,
+      is_non_singular=diag_operator.is_non_singular,
+      is_self_adjoint=diag_operator.is_self_adjoint,
+      is_positive_definite=diag_operator.is_positive_definite,
+      is_square=True)
+
+
+@linear_operator_algebra.RegisterAdjoint(
+    linear_operator_block_diag.LinearOperatorBlockDiag)
+def _adjoint_block_diag(block_diag_operator):
+    # We take the adjoint of each block on the diagonal.
+  return linear_operator_block_diag.LinearOperatorBlockDiag(
+      operators=[
+          operator.adjoint() for operator in block_diag_operator.operators],
+      is_non_singular=block_diag_operator.is_non_singular,
+      is_self_adjoint=block_diag_operator.is_self_adjoint,
+      is_positive_definite=block_diag_operator.is_positive_definite,
+      is_square=True)
+
+
+@linear_operator_algebra.RegisterAdjoint(
+    linear_operator_kronecker.LinearOperatorKronecker)
+def _adjoint_kronecker(kronecker_operator):
+    # Adjoint of a Kronecker product is the Kronecker product
+    # of adjoints.
+  return linear_operator_kronecker.LinearOperatorKronecker(
+      operators=[
+          operator.adjoint() for operator in kronecker_operator.operators],
+      is_non_singular=kronecker_operator.is_non_singular,
+      is_self_adjoint=kronecker_operator.is_self_adjoint,
+      is_positive_definite=kronecker_operator.is_positive_definite,
+      is_square=True)
+
+
+@linear_operator_algebra.RegisterAdjoint(
+    linear_operator_circulant.LinearOperatorCirculant)
+def _adjoint_circulant(circulant_operator):
+  spectrum = circulant_operator.spectrum
+  if spectrum.dtype.is_complex:
+    spectrum = math_ops.conj(spectrum)
+
+  # Conjugating the spectrum is sufficient to get the adjoint.
+  return linear_operator_circulant.LinearOperatorCirculant(
+      spectrum=spectrum,
+      is_non_singular=circulant_operator.is_non_singular,
+      is_self_adjoint=circulant_operator.is_self_adjoint,
+      is_positive_definite=circulant_operator.is_positive_definite,
+      is_square=True)
diff --git a/tensorflow/python/ops/linalg/linalg.py b/tensorflow/python/ops/linalg/linalg.py
index eebe741337..b9f8411c93 100644
--- a/tensorflow/python/ops/linalg/linalg.py
+++ b/tensorflow/python/ops/linalg/linalg.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 # go/tf-wildcard-import
 # pylint: disable=wildcard-import,unused-import
+from tensorflow.python.ops.linalg import adjoint_registrations as _adjoint_registrations
 from tensorflow.python.ops.linalg import cholesky_registrations as _cholesky_registrations
 from tensorflow.python.ops.linalg import inverse_registrations as _inverse_registrations
 from tensorflow.python.ops.linalg import linear_operator_algebra as _linear_operator_algebra
diff --git a/tensorflow/python/ops/linalg/linear_operator.py b/tensorflow/python/ops/linalg/linear_operator.py
index 4c99e86dc5..8fa9f63e04 100644
--- a/tensorflow/python/ops/linalg/linear_operator.py
+++ b/tensorflow/python/ops/linalg/linear_operator.py
@@ -847,6 +847,26 @@ class LinearOperator(object):
 
       return self._solvevec(rhs, adjoint=adjoint)
 
+  def adjoint(self, name="adjoint"):
+    """Returns the adjoint of the current `LinearOperator`.
+
+    Given `A` representing this `LinearOperator`, return `A*`.
+    Note that calling `self.adjoint()` and `self.H` are equivalent.
+
+    Args:
+      name:  A name for this `Op`.
+
+    Returns:
+      `LinearOperator` which represents the adjoint of this `LinearOperator`.
+    """
+    if self.is_self_adjoint is True:  # pylint: disable=g-bool-id-comparison
+      return self
+    with self._name_scope(name):
+      return linear_operator_algebra.adjoint(self)
+
+  # self.H is equivalent to self.adjoint().
+  H = property(adjoint, None)
+
   def inverse(self, name="inverse"):
     """Returns the Inverse of this `LinearOperator`.
 
diff --git a/tensorflow/python/ops/linalg/linear_operator_adjoint.py b/tensorflow/python/ops/linalg/linear_operator_adjoint.py
index 858e224b9a..7ee4752d26 100644
--- a/tensorflow/python/ops/linalg/linear_operator_adjoint.py
+++ b/tensorflow/python/ops/linalg/linear_operator_adjoint.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops.linalg import linalg_impl as linalg
 from tensorflow.python.ops.linalg import linear_operator
@@ -83,7 +84,7 @@ class LinearOperatorAdjoint(linear_operator.LinearOperator):
     r"""Initialize a `LinearOperatorAdjoint`.
 
     `LinearOperatorAdjoint` is initialized with an operator `A`.  The `solve`
-    and `matmul` methods effectively flip the `adjoint` argument.  E.g.
+    and `matmul` methods  effectively flip the `adjoint` argument.  E.g.
 
     ```
     A = MyLinearOperator(...)
@@ -175,15 +176,24 @@ class LinearOperatorAdjoint(linear_operator.LinearOperator):
     return self.operator.assert_self_adjoint()
 
   def _shape(self):
-    return self.operator.shape
+    # Rotate last dimension
+    shape = self.operator.shape
+    return shape[:-2].concatenate([shape[-1], shape[-2]])
 
   def _shape_tensor(self):
-    return self.operator.shape_tensor()
+    # Rotate last dimension
+    shape = self.operator.shape_tensor()
+    return array_ops.concat([
+        shape[:-2], [shape[-1], shape[-2]]], axis=-1)
 
   def _matmul(self, x, adjoint=False, adjoint_arg=False):
     return self.operator.matmul(
         x, adjoint=(not adjoint), adjoint_arg=adjoint_arg)
 
+  def _matvec(self, x, adjoint=False, adjoint_arg=False):
+    return self.operator.matvec(
+        x, adjoint=(not adjoint), adjoint_arg=adjoint_arg)
+
   def _determinant(self):
     if self.is_self_adjoint:
       return self.operator.determinant()
@@ -201,7 +211,14 @@ class LinearOperatorAdjoint(linear_operator.LinearOperator):
     return self.operator.solve(
         rhs, adjoint=(not adjoint), adjoint_arg=adjoint_arg)
 
+  def _solvevec(self, rhs, adjoint=False, adjoint_arg=False):
+    return self.operator.solvevec(
+        rhs, adjoint=(not adjoint), adjoint_arg=adjoint_arg)
+
   def _to_dense(self):
     if self.is_self_adjoint:
       return self.operator.to_dense()
     return linalg.adjoint(self.operator.to_dense())
+
+  def _add_to_tensor(self, x):
+    return self.to_dense() + x
diff --git a/tensorflow/python/ops/linalg/linear_operator_algebra.py b/tensorflow/python/ops/linalg/linear_operator_algebra.py
index c1513fdb38..0d1eab4b73 100644
--- a/tensorflow/python/ops/linalg/linear_operator_algebra.py
+++ b/tensorflow/python/ops/linalg/linear_operator_algebra.py
@@ -25,6 +25,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.util import tf_inspect
 
 
+_ADJOINTS = {}
 _CHOLESKY_DECOMPS = {}
 _MATMUL = {}
 _INVERSES = {}
@@ -46,6 +47,11 @@ def _registered_function(type_list, registry):
   return registry.get(tuple(r[1] for r in registered_combination), None)
 
 
+def _registered_adjoint(type_a):
+  """Get the Adjoint function registered for class a."""
+  return _registered_function([type_a], _ADJOINTS)
+
+
 def _registered_cholesky(type_a):
   """Get the Cholesky function registered for class a."""
   return _registered_function([type_a], _CHOLESKY_DECOMPS)
@@ -61,6 +67,29 @@ def _registered_inverse(type_a):
   return _registered_function([type_a], _INVERSES)
 
 
+def adjoint(lin_op_a, name=None):
+  """Get the adjoint associated to lin_op_a.
+
+  Args:
+    lin_op_a: The LinearOperator to take the adjoint of.
+    name: Name to use for this operation.
+
+  Returns:
+    A LinearOperator that represents the adjoint of `lin_op_a`.
+
+  Raises:
+    NotImplementedError: If no Adjoint method is defined for the LinearOperator
+      type of `lin_op_a`.
+  """
+  adjoint_fn = _registered_adjoint(type(lin_op_a))
+  if adjoint_fn is None:
+    raise ValueError("No adjoint registered for {}".format(
+        type(lin_op_a)))
+
+  with ops.name_scope(name, "Adjoint"):
+    return adjoint_fn(lin_op_a)
+
+
 def cholesky(lin_op_a, name=None):
   """Get the Cholesky factor associated to lin_op_a.
 
@@ -132,6 +161,48 @@ def inverse(lin_op_a, name=None):
     return inverse_fn(lin_op_a)
 
 
+class RegisterAdjoint(object):
+  """Decorator to register an Adjoint implementation function.
+
+  Usage:
+
+  @linear_operator_algebra.RegisterAdjoint(lin_op.LinearOperatorIdentity)
+  def _adjoint_identity(lin_op_a):
+    # Return the identity matrix.
+  """
+
+  def __init__(self, lin_op_cls_a):
+    """Initialize the LinearOperator registrar.
+
+    Args:
+      lin_op_cls_a: the class of the LinearOperator to decompose.
+    """
+    self._key = (lin_op_cls_a,)
+
+  def __call__(self, adjoint_fn):
+    """Perform the Adjoint registration.
+
+    Args:
+      adjoint_fn: The function to use for the Adjoint.
+
+    Returns:
+      adjoint_fn
+
+    Raises:
+      TypeError: if adjoint_fn is not a callable.
+      ValueError: if a Adjoint function has already been registered for
+        the given argument classes.
+    """
+    if not callable(adjoint_fn):
+      raise TypeError(
+          "adjoint_fn must be callable, received: {}".format(adjoint_fn))
+    if self._key in _ADJOINTS:
+      raise ValueError("Adjoint({}) has already been registered to: {}".format(
+          self._key[0].__name__, _ADJOINTS[self._key]))
+    _ADJOINTS[self._key] = adjoint_fn
+    return adjoint_fn
+
+
 class RegisterCholesky(object):
   """Decorator to register a Cholesky implementation function.
 
diff --git a/tensorflow/python/ops/linalg/linear_operator_test_util.py b/tensorflow/python/ops/linalg/linear_operator_test_util.py
index a957c84dc1..0383098804 100644
--- a/tensorflow/python/ops/linalg/linear_operator_test_util.py
+++ b/tensorflow/python/ops/linalg/linear_operator_test_util.py
@@ -278,6 +278,23 @@ class LinearOperatorDerivedClassTest(test.TestCase):
     self._skip_if_tests_to_skip_contains("matmul_with_broadcast")
     self._test_matmul(with_batch=False)
 
+  def test_adjoint(self):
+    self._skip_if_tests_to_skip_contains("adjoint")
+    for use_placeholder in self._use_placeholder_options:
+      for build_info in self._operator_build_infos:
+        for dtype in self._dtypes_to_test:
+          with self.test_session(graph=ops.Graph()) as sess:
+            sess.graph.seed = random_seed.DEFAULT_GRAPH_SEED
+            operator, mat = self._operator_and_matrix(
+                build_info, dtype, use_placeholder=use_placeholder)
+            op_adjoint = operator.adjoint().to_dense()
+            op_adjoint_h = operator.H.to_dense()
+            mat_adjoint = linalg.adjoint(mat)
+            op_adjoint_v, op_adjoint_h_v, mat_adjoint_v = sess.run(
+                [op_adjoint, op_adjoint_h, mat_adjoint])
+            self.assertAC(mat_adjoint_v, op_adjoint_v)
+            self.assertAC(mat_adjoint_v, op_adjoint_h_v)
+
   def test_cholesky(self):
     self._skip_if_tests_to_skip_contains("cholesky")
     for use_placeholder in self._use_placeholder_options:
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-adjoint.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-adjoint.pbtxt
new file mode 100644
index 0000000000..37344f7031
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-adjoint.pbtxt
@@ -0,0 +1,150 @@
+path: "tensorflow.linalg.LinearOperatorAdjoint"
+tf_class {
+  is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_adjoint.LinearOperatorAdjoint\'>"
+  is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "batch_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "domain_dimension"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "graph_parents"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_non_singular"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_positive_definite"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_self_adjoint"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_square"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "operator"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "range_dimension"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "tensor_rank"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'operator\', \'is_non_singular\', \'is_self_adjoint\', \'is_positive_definite\', \'is_square\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_to_tensor"
+    argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
+  }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
+  member_method {
+    name: "assert_non_singular"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
+  }
+  member_method {
+    name: "assert_positive_definite"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_positive_definite\'], "
+  }
+  member_method {
+    name: "assert_self_adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_self_adjoint\'], "
+  }
+  member_method {
+    name: "batch_shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
+  }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
+  member_method {
+    name: "determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
+  }
+  member_method {
+    name: "diag_part"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'diag_part\'], "
+  }
+  member_method {
+    name: "domain_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
+  }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
+  member_method {
+    name: "log_abs_determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
+  }
+  member_method {
+    name: "matmul"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'matmul\'], "
+  }
+  member_method {
+    name: "matvec"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'matvec\'], "
+  }
+  member_method {
+    name: "range_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'range_dimension_tensor\'], "
+  }
+  member_method {
+    name: "shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'shape_tensor\'], "
+  }
+  member_method {
+    name: "solve"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'solve\'], "
+  }
+  member_method {
+    name: "solvevec"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'solve\'], "
+  }
+  member_method {
+    name: "tensor_rank_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'tensor_rank_tensor\'], "
+  }
+  member_method {
+    name: "to_dense"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'to_dense\'], "
+  }
+  member_method {
+    name: "trace"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'trace\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-block-diag.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-block-diag.pbtxt
index c7a50969b5..ddef774a75 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-block-diag.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-block-diag.pbtxt
@@ -3,6 +3,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_block_diag.LinearOperatorBlockDiag\'>"
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "batch_shape"
     mtype: "<type \'property\'>"
@@ -63,6 +67,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_non_singular"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant.pbtxt
index 3900c752c8..97a6b1a475 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant.pbtxt
@@ -4,6 +4,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_circulant._BaseLinearOperatorCirculant\'>"
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "batch_shape"
     mtype: "<type \'property\'>"
@@ -72,6 +76,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_hermitian_spectrum"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_hermitian_spectrum\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt
index 7b876099af..e2bfe7e7d8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt
@@ -4,6 +4,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_circulant._BaseLinearOperatorCirculant\'>"
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "batch_shape"
     mtype: "<type \'property\'>"
@@ -72,6 +76,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_hermitian_spectrum"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_hermitian_spectrum\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt
index 5bddba8e79..8885526669 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt
@@ -4,6 +4,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_circulant._BaseLinearOperatorCirculant\'>"
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "batch_shape"
     mtype: "<type \'property\'>"
@@ -72,6 +76,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_hermitian_spectrum"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_hermitian_spectrum\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-composition.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-composition.pbtxt
index 62ba8bb59e..2a017fcb89 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-composition.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-composition.pbtxt
@@ -3,6 +3,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_composition.LinearOperatorComposition\'>"
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "batch_shape"
     mtype: "<type \'property\'>"
@@ -63,6 +67,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_non_singular"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-diag.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-diag.pbtxt
index 0803feeabd..31dcf7b0a6 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-diag.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-diag.pbtxt
@@ -3,6 +3,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_diag.LinearOperatorDiag\'>"
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "batch_shape"
     mtype: "<type \'property\'>"
@@ -63,6 +67,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_non_singular"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-full-matrix.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-full-matrix.pbtxt
index 6def32864b..0ad39b4ba6 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-full-matrix.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-full-matrix.pbtxt
@@ -3,6 +3,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_full_matrix.LinearOperatorFullMatrix\'>"
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "batch_shape"
     mtype: "<type \'property\'>"
@@ -59,6 +63,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_non_singular"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-identity.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-identity.pbtxt
index dbf1ac82d3..f66a5a833a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-identity.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-identity.pbtxt
@@ -4,6 +4,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_identity.BaseLinearOperatorIdentity\'>"
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "batch_shape"
     mtype: "<type \'property\'>"
@@ -60,6 +64,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'mat\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_non_singular"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-inversion.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-inversion.pbtxt
index 6a3fe4dd66..a7eb144d83 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-inversion.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-inversion.pbtxt
@@ -3,6 +3,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_inversion.LinearOperatorInversion\'>"
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "batch_shape"
     mtype: "<type \'property\'>"
@@ -63,6 +67,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_non_singular"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-kronecker.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-kronecker.pbtxt
index 85d902b977..c983f8c6e6 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-kronecker.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-kronecker.pbtxt
@@ -3,6 +3,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_kronecker.LinearOperatorKronecker\'>"
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "batch_shape"
     mtype: "<type \'property\'>"
@@ -63,6 +67,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_non_singular"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt
index 638d82a599..813aec2a13 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt
@@ -3,6 +3,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_low_rank_update.LinearOperatorLowRankUpdate\'>"
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "base_operator"
     mtype: "<type \'property\'>"
@@ -83,6 +87,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_non_singular"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt
index ab1b04bd3c..0bb7a15e13 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt
@@ -3,6 +3,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_lower_triangular.LinearOperatorLowerTriangular\'>"
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "batch_shape"
     mtype: "<type \'property\'>"
@@ -59,6 +63,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_non_singular"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt
index 961969aac5..7747c98540 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt
@@ -4,6 +4,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_identity.BaseLinearOperatorIdentity\'>"
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "batch_shape"
     mtype: "<type \'property\'>"
@@ -64,6 +68,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'mat\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_non_singular"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-zeros.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-zeros.pbtxt
index e76738a964..590782bbc1 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-zeros.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-zeros.pbtxt
@@ -3,6 +3,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_zeros.LinearOperatorZeros\'>"
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "batch_shape"
     mtype: "<type \'property\'>"
@@ -59,6 +63,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'mat\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_non_singular"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator.pbtxt
index b35cd69da4..ed6bfdff28 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator.pbtxt
@@ -2,6 +2,10 @@ path: "tensorflow.linalg.LinearOperator"
 tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "batch_shape"
     mtype: "<type \'property\'>"
@@ -58,6 +62,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_non_singular"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.pbtxt
index 5e49b75c31..973850cfd8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.pbtxt
@@ -4,6 +4,10 @@ tf_module {
     name: "LinearOperator"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "LinearOperatorAdjoint"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "LinearOperatorBlockDiag"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-adjoint.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-adjoint.pbtxt
new file mode 100644
index 0000000000..37344f7031
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-adjoint.pbtxt
@@ -0,0 +1,150 @@
+path: "tensorflow.linalg.LinearOperatorAdjoint"
+tf_class {
+  is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_adjoint.LinearOperatorAdjoint\'>"
+  is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "batch_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "domain_dimension"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "graph_parents"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_non_singular"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_positive_definite"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_self_adjoint"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_square"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "operator"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "range_dimension"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "tensor_rank"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'operator\', \'is_non_singular\', \'is_self_adjoint\', \'is_positive_definite\', \'is_square\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_to_tensor"
+    argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
+  }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
+  member_method {
+    name: "assert_non_singular"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
+  }
+  member_method {
+    name: "assert_positive_definite"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_positive_definite\'], "
+  }
+  member_method {
+    name: "assert_self_adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_self_adjoint\'], "
+  }
+  member_method {
+    name: "batch_shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
+  }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
+  member_method {
+    name: "determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
+  }
+  member_method {
+    name: "diag_part"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'diag_part\'], "
+  }
+  member_method {
+    name: "domain_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
+  }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
+  member_method {
+    name: "log_abs_determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
+  }
+  member_method {
+    name: "matmul"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'matmul\'], "
+  }
+  member_method {
+    name: "matvec"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'matvec\'], "
+  }
+  member_method {
+    name: "range_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'range_dimension_tensor\'], "
+  }
+  member_method {
+    name: "shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'shape_tensor\'], "
+  }
+  member_method {
+    name: "solve"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'solve\'], "
+  }
+  member_method {
+    name: "solvevec"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'solve\'], "
+  }
+  member_method {
+    name: "tensor_rank_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'tensor_rank_tensor\'], "
+  }
+  member_method {
+    name: "to_dense"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'to_dense\'], "
+  }
+  member_method {
+    name: "trace"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'trace\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-block-diag.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-block-diag.pbtxt
index c7a50969b5..ddef774a75 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-block-diag.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-block-diag.pbtxt
@@ -3,6 +3,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_block_diag.LinearOperatorBlockDiag\'>"
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "batch_shape"
     mtype: "<type \'property\'>"
@@ -63,6 +67,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_non_singular"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant.pbtxt
index 3900c752c8..97a6b1a475 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant.pbtxt
@@ -4,6 +4,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_circulant._BaseLinearOperatorCirculant\'>"
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "batch_shape"
     mtype: "<type \'property\'>"
@@ -72,6 +76,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_hermitian_spectrum"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_hermitian_spectrum\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt
index 7b876099af..e2bfe7e7d8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt
@@ -4,6 +4,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_circulant._BaseLinearOperatorCirculant\'>"
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "batch_shape"
     mtype: "<type \'property\'>"
@@ -72,6 +76,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_hermitian_spectrum"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_hermitian_spectrum\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt
index 5bddba8e79..8885526669 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt
@@ -4,6 +4,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_circulant._BaseLinearOperatorCirculant\'>"
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "batch_shape"
     mtype: "<type \'property\'>"
@@ -72,6 +76,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_hermitian_spectrum"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_hermitian_spectrum\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-composition.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-composition.pbtxt
index 62ba8bb59e..2a017fcb89 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-composition.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-composition.pbtxt
@@ -3,6 +3,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_composition.LinearOperatorComposition\'>"
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "batch_shape"
     mtype: "<type \'property\'>"
@@ -63,6 +67,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_non_singular"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-diag.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-diag.pbtxt
index 0803feeabd..31dcf7b0a6 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-diag.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-diag.pbtxt
@@ -3,6 +3,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_diag.LinearOperatorDiag\'>"
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "batch_shape"
     mtype: "<type \'property\'>"
@@ -63,6 +67,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_non_singular"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-full-matrix.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-full-matrix.pbtxt
index 6def32864b..0ad39b4ba6 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-full-matrix.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-full-matrix.pbtxt
@@ -3,6 +3,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_full_matrix.LinearOperatorFullMatrix\'>"
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "batch_shape"
     mtype: "<type \'property\'>"
@@ -59,6 +63,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_non_singular"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-identity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-identity.pbtxt
index dbf1ac82d3..f66a5a833a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-identity.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-identity.pbtxt
@@ -4,6 +4,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_identity.BaseLinearOperatorIdentity\'>"
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "batch_shape"
     mtype: "<type \'property\'>"
@@ -60,6 +64,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'mat\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_non_singular"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-inversion.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-inversion.pbtxt
index 6a3fe4dd66..a7eb144d83 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-inversion.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-inversion.pbtxt
@@ -3,6 +3,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_inversion.LinearOperatorInversion\'>"
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "batch_shape"
     mtype: "<type \'property\'>"
@@ -63,6 +67,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_non_singular"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-kronecker.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-kronecker.pbtxt
index 85d902b977..c983f8c6e6 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-kronecker.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-kronecker.pbtxt
@@ -3,6 +3,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_kronecker.LinearOperatorKronecker\'>"
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "batch_shape"
     mtype: "<type \'property\'>"
@@ -63,6 +67,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_non_singular"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt
index 638d82a599..813aec2a13 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt
@@ -3,6 +3,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_low_rank_update.LinearOperatorLowRankUpdate\'>"
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "base_operator"
     mtype: "<type \'property\'>"
@@ -83,6 +87,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_non_singular"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt
index ab1b04bd3c..0bb7a15e13 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt
@@ -3,6 +3,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_lower_triangular.LinearOperatorLowerTriangular\'>"
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "batch_shape"
     mtype: "<type \'property\'>"
@@ -59,6 +63,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_non_singular"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt
index 961969aac5..7747c98540 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt
@@ -4,6 +4,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_identity.BaseLinearOperatorIdentity\'>"
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "batch_shape"
     mtype: "<type \'property\'>"
@@ -64,6 +68,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'mat\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_non_singular"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-zeros.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-zeros.pbtxt
index e76738a964..590782bbc1 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-zeros.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-zeros.pbtxt
@@ -3,6 +3,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_zeros.LinearOperatorZeros\'>"
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "batch_shape"
     mtype: "<type \'property\'>"
@@ -59,6 +63,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'mat\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_non_singular"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator.pbtxt
index b35cd69da4..ed6bfdff28 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator.pbtxt
@@ -2,6 +2,10 @@ path: "tensorflow.linalg.LinearOperator"
 tf_class {
   is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "H"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "batch_shape"
     mtype: "<type \'property\'>"
@@ -58,6 +62,10 @@ tf_class {
     name: "add_to_tensor"
     argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'adjoint\'], "
+  }
   member_method {
     name: "assert_non_singular"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt
index f9119cdd5f..08b928e02c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt
@@ -4,6 +4,10 @@ tf_module {
     name: "LinearOperator"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "LinearOperatorAdjoint"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "LinearOperatorBlockDiag"
     mtype: "<type \'type\'>"
-- 
GitLab


From f395dbf81347e22608d691657b3674ad347d76dd Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Tue, 5 Feb 2019 18:12:00 -0800
Subject: [PATCH 1972/2345] Disable flaky tests for OSS build.

PiperOrigin-RevId: 232596390
---
 tensorflow/python/keras/BUILD | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 6f3fd226f4..d904193319 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -368,7 +368,10 @@ tf_py_test(
         "//tensorflow/python:nn_ops",
     ],
     shard_count = 12,
-    tags = ["notsan"],
+    tags = [
+        "no_oss",  # b/123899138
+        "notsan",
+    ],
 )
 
 tf_py_test(
@@ -394,6 +397,9 @@ tf_py_test(
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
     ],
+    tags = [
+        "no_oss",  # b/123899138
+    ],
 )
 
 tf_py_test(
@@ -816,6 +822,7 @@ tf_py_test(
     ],
     shard_count = 4,
     tags = [
+        "no_oss",  # b/123899138
         "noasan",  # http://b/78599823
         "notsan",
     ],
-- 
GitLab


From 933f85fce087999f93bde1a9f0ef727e0fb3d46c Mon Sep 17 00:00:00 2001
From: nammbash <niroopshankar.ammbashankar@intel.com>
Date: Tue, 5 Feb 2019 19:03:00 -0800
Subject: [PATCH 1973/2345] MKL conditional compilation

---
 tensorflow/core/kernels/mkl_requantize_ops_test.cc |  2 ++
 .../core/kernels/mkl_requantize_per_channel_op.cc  |  2 +-
 tensorflow/core/kernels/requantize.cc              | 14 +++++++++++---
 3 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/kernels/mkl_requantize_ops_test.cc b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
index a156bd494c..35b8a975b6 100644
--- a/tensorflow/core/kernels/mkl_requantize_ops_test.cc
+++ b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
@@ -12,6 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
+#if defined(INTEL_MKL) && defined(ENABLE_MKL)
 
 #include <cmath>
 
@@ -296,3 +297,4 @@ TEST_F(MklRequantizatedOpsTest, RequantizePerChannelTest_Basic) {
 }
 
 }  // namespace tensorflow
+#endif  // INTEL_MKL && ENABLE_MKL
diff --git a/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc b/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc
index 1f5dd5bcf5..63701126b0 100644
--- a/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc
+++ b/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc
@@ -15,8 +15,8 @@ limitations under the License.
 
 // See docs in ../ops/array_ops.cc.
 
-#define EIGEN_USE_THREADS
 #ifdef INTEL_MKL
+#define EIGEN_USE_THREADS
 #include <math.h>
 
 #include "mkldnn.hpp"
diff --git a/tensorflow/core/kernels/requantize.cc b/tensorflow/core/kernels/requantize.cc
index ab057d5a67..6790229d87 100644
--- a/tensorflow/core/kernels/requantize.cc
+++ b/tensorflow/core/kernels/requantize.cc
@@ -100,10 +100,18 @@ REGISTER_KERNEL_BUILDER(Name("Requantize")
                             .TypeConstraint<quint8>("out_type"),
                         RequantizeOp<qint32, quint8>);
 
-REGISTER_KERNEL_BUILDER(Name("Requantize")
+#ifndef INTEL_MKL
+
+REGISTER_KERNEL_BUILDER(Name("RequantizePerChannel")
                             .Device(DEVICE_CPU)
-                            .TypeConstraint<qint32>("Tinput")
+                            .TypeConstraint<qint32>("T")
                             .TypeConstraint<qint8>("out_type"),
-                        RequantizeOp<qint32, qint8>);
+                        NoOp);
+
+REGISTER_KERNEL_BUILDER(Name("RequantizationRangePerChannel")
+                            .Device(DEVICE_CPU)
+                            .TypeConstraint<qint32>("T"),
+                        NoOp);
+#endif
 
 }  // namespace tensorflow
-- 
GitLab


From ebb2e9f4efe0eb3ff54d159b536ad99b06c8a6fe Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 5 Feb 2019 19:52:32 -0800
Subject: [PATCH 1974/2345] Fix OneDeviceDataset numpy dataset "Error copying
 tensor to device:" bug.  I needed to make sure
 OneDeviceExtended._create_variable() used tf.device() in this case instead of
 tf.colocate_with(). I also needed to remove the device set by one device
 strategy's scope, since otherwise dataset creation gives lots of errors.

Also, expand OneDeviceStrategy eager/GPU test coverage.

PiperOrigin-RevId: 232605692
---
 tensorflow/contrib/distribute/python/BUILD    | 23 +++++++++----------
 .../contrib/distribute/python/combinations.py |  3 +++
 .../contrib/distribute/python/keras_test.py   | 18 +++++++--------
 .../python/one_device_strategy_test.py        |  8 ++++++-
 .../python/distribute/one_device_strategy.py  | 11 +++++----
 5 files changed, 37 insertions(+), 26 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 509eb78128..f4c9e002c1 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -131,6 +131,17 @@ py_library(
     ],
 )
 
+cuda_py_test(
+    name = "one_device_strategy_test",
+    srcs = ["one_device_strategy_test.py"],
+    additional_deps = [
+        ":strategy_test_lib",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python/distribute:one_device_strategy",
+        "//tensorflow/python/eager:test",
+    ],
+)
+
 py_library(
     name = "collective_all_reduce_strategy",
     srcs = ["collective_all_reduce_strategy.py"],
@@ -192,18 +203,6 @@ py_test(
     ],
 )
 
-py_test(
-    name = "one_device_strategy_test",
-    srcs = ["one_device_strategy_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":strategy_test_lib",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python/distribute:one_device_strategy",
-        "//tensorflow/python/eager:test",
-    ],
-)
-
 # TODO(priyag): Rename this test to mirrored_strategy_test
 cuda_py_test(
     name = "mirrored_strategy_multigpu_test",
diff --git a/tensorflow/contrib/distribute/python/combinations.py b/tensorflow/contrib/distribute/python/combinations.py
index 798a1591c7..7c0f8033fb 100644
--- a/tensorflow/contrib/distribute/python/combinations.py
+++ b/tensorflow/contrib/distribute/python/combinations.py
@@ -352,6 +352,9 @@ default_strategy = NamedDistribution(
 one_device_strategy = NamedDistribution(
     "OneDeviceCPU", lambda: one_device_lib.OneDeviceStrategy("/cpu:0"),
     required_gpus=None)
+one_device_strategy_gpu = NamedDistribution(
+    "OneDeviceGPU", lambda: one_device_lib.OneDeviceStrategy("/gpu:0"),
+    required_gpus=1)
 tpu_strategy = NamedDistribution(
     "TPU", _get_tpu_strategy_creator(steps_per_run=2),
     required_tpu=True)
diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index dd975c6c36..6a15083470 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -230,6 +230,8 @@ def multi_input_output_model():
   return model
 
 
+# TODO(josh11b): Add combinations.one_device_strategy_gpu once it works with
+# TestDistributionStrategyWithCallbacks.test_callbacks_in_predict.
 strategies_minus_tpu = [
     combinations.default_strategy,
     combinations.one_device_strategy,
@@ -244,15 +246,13 @@ tpu_strategies = [
 
 
 def strategy_minus_tpu_combinations():
-  return combinations.combine(
-      distribution=strategies_minus_tpu,
-      mode=['graph', 'eager'])
+  return combinations.combine(distribution=strategies_minus_tpu,
+                              mode=['graph', 'eager'])
 
 
 def tpu_strategy_combinations():
-  return combinations.combine(
-      distribution=tpu_strategies,
-      mode=['graph'])
+  return combinations.combine(distribution=tpu_strategies,
+                              mode=['graph'])
 
 
 def all_strategy_combinations():
@@ -287,9 +287,9 @@ def strategy_and_optimizer_combinations():
 
 
 def strategy_for_numpy_input_combinations():
-  return combinations.combine(
-      distribution=strategies_minus_tpu + tpu_strategies,
-      mode=['graph'])
+  one_gpu = combinations.one_device_strategy_gpu
+  return (all_strategy_combinations() +
+          combinations.combine(distribution=[one_gpu], mode=['graph', 'eager']))
 
 
 class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase,
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy_test.py b/tensorflow/contrib/distribute/python/one_device_strategy_test.py
index 906bffc852..93c2447cad 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy_test.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import sys
+
 from tensorflow.contrib.distribute.python import strategy_test_lib
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import one_device_strategy
@@ -29,8 +31,12 @@ class OneDeviceStrategyTest(
     strategy_test_lib.DistributionTestBase,
     strategy_test_lib.OneDeviceDistributionTestBase):
 
+  # TODO(josh11b): Switch to using the combinations library.
   def _get_distribution_strategy(self):
-    return one_device_strategy.OneDeviceStrategy("/device:CPU:0")
+    if "test_gpu" in sys.argv[0]:
+      return one_device_strategy.OneDeviceStrategy("/device:GPU:0")
+    else:
+      return one_device_strategy.OneDeviceStrategy("/device:CPU:0")
 
   def testMinimizeLossEager(self):
     self._test_minimize_loss_eager(self._get_distribution_strategy())
diff --git a/tensorflow/python/distribute/one_device_strategy.py b/tensorflow/python/distribute/one_device_strategy.py
index b2d2f03618..ff9f616967 100644
--- a/tensorflow/python/distribute/one_device_strategy.py
+++ b/tensorflow/python/distribute/one_device_strategy.py
@@ -50,7 +50,6 @@ class OneDeviceExtended(distribute_lib.DistributionStrategyExtended):
   def __init__(self, container_strategy, device):
     super(OneDeviceExtended, self).__init__(container_strategy)
     self._device = device
-    self._default_device = device
     self._input_device = device_util.canonicalize("/device:CPU:0")
     worker_device_pairs = [(self._input_device, [self._device])]
     device_map = values.SingleDeviceMap(device)
@@ -62,8 +61,12 @@ class OneDeviceExtended(distribute_lib.DistributionStrategyExtended):
     if colocate_with is None:
       with ops.device(self._device):
         return next_creator(*args, **kwargs)
-    with ops.colocate_with(colocate_with):
-      return next_creator(*args, **kwargs)
+    elif isinstance(colocate_with, numpy_dataset.SingleDevice):
+      with ops.device(colocate_with.device):
+        return next_creator(*args, **kwargs)
+    else:
+      with ops.colocate_with(colocate_with):
+        return next_creator(*args, **kwargs)
 
   def _validate_colocate_with_variable(self, colocate_with_variable):
     values.validate_colocate(colocate_with_variable, self)
@@ -83,7 +86,7 @@ class OneDeviceExtended(distribute_lib.DistributionStrategyExtended):
 
   def _experimental_make_numpy_dataset(self, numpy_input, session):
     return numpy_dataset.one_host_numpy_dataset(
-        numpy_input, self._input_device, session)
+        numpy_input, numpy_dataset.SingleDevice(self._input_device), session)
 
   def _broadcast_to(self, tensor, destinations):
     del destinations
-- 
GitLab


From 480961d242b2e05c8519243537302db4dd47b39d Mon Sep 17 00:00:00 2001
From: nammbash <niroopshankar.ammbashankar@intel.com>
Date: Tue, 5 Feb 2019 19:59:28 -0800
Subject: [PATCH 1975/2345] cleanup requantize

---
 tensorflow/core/kernels/requantize.cc | 14 --------------
 1 file changed, 14 deletions(-)

diff --git a/tensorflow/core/kernels/requantize.cc b/tensorflow/core/kernels/requantize.cc
index 6790229d87..7b73f8209a 100644
--- a/tensorflow/core/kernels/requantize.cc
+++ b/tensorflow/core/kernels/requantize.cc
@@ -100,18 +100,4 @@ REGISTER_KERNEL_BUILDER(Name("Requantize")
                             .TypeConstraint<quint8>("out_type"),
                         RequantizeOp<qint32, quint8>);
 
-#ifndef INTEL_MKL
-
-REGISTER_KERNEL_BUILDER(Name("RequantizePerChannel")
-                            .Device(DEVICE_CPU)
-                            .TypeConstraint<qint32>("T")
-                            .TypeConstraint<qint8>("out_type"),
-                        NoOp);
-
-REGISTER_KERNEL_BUILDER(Name("RequantizationRangePerChannel")
-                            .Device(DEVICE_CPU)
-                            .TypeConstraint<qint32>("T"),
-                        NoOp);
-#endif
-
 }  // namespace tensorflow
-- 
GitLab


From 2102c1226816b82d2f34b6348b0f2ac302611f02 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Wed, 6 Feb 2019 00:41:51 -0800
Subject: [PATCH 1976/2345] Allow to specify an (optional) comparator
 computation for Sort in HLO text.

Adjust all tests that use HLO text to specify such a comparator computation.
For now, this is simply ignored by the emitters.
In a later change we will actually use it, and make it required to pass a
comparator computation.

PiperOrigin-RevId: 232629373
---
 .../compiler/xla/service/buffer_assignment.cc |  1 +
 tensorflow/compiler/xla/service/call_graph.cc |  1 +
 .../cpu/tests/cpu_key_value_sort_test.cc      |  8 +-
 .../service/gpu/gpu_layout_assignment_test.cc | 11 ++-
 .../xla/service/hlo_element_type_converter.cc |  6 +-
 .../compiler/xla/service/hlo_instruction.cc   | 40 ++++++++--
 .../compiler/xla/service/hlo_instruction.h    |  9 +++
 .../compiler/xla/service/hlo_instructions.cc  | 25 ++++++-
 .../compiler/xla/service/hlo_instructions.h   |  3 +
 tensorflow/compiler/xla/service/hlo_parser.cc | 17 ++++-
 .../compiler/xla/service/hlo_parser_test.cc   | 62 ++++++++++++++--
 .../compiler/xla/service/hlo_verifier.cc      |  1 +
 .../xla/service/sort_simplifier_test.cc       | 74 +++++++++++++------
 .../compiler/xla/tests/test_utils_test.cc     | 30 +++++++-
 14 files changed, 238 insertions(+), 50 deletions(-)

diff --git a/tensorflow/compiler/xla/service/buffer_assignment.cc b/tensorflow/compiler/xla/service/buffer_assignment.cc
index e1b91b5001..cbebbdc8a2 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment.cc
+++ b/tensorflow/compiler/xla/service/buffer_assignment.cc
@@ -191,6 +191,7 @@ Status GatherComputationsByAllocationType(
           case HloOpcode::kReduceWindow:
           case HloOpcode::kScatter:
           case HloOpcode::kSelectAndScatter:
+          case HloOpcode::kSort:
           case HloOpcode::kFusion:
             // Map/reduce etc computations are always thread-local.
             worklist.push_back(std::make_pair(subcomputation,
diff --git a/tensorflow/compiler/xla/service/call_graph.cc b/tensorflow/compiler/xla/service/call_graph.cc
index 94af788c54..98304757ca 100644
--- a/tensorflow/compiler/xla/service/call_graph.cc
+++ b/tensorflow/compiler/xla/service/call_graph.cc
@@ -64,6 +64,7 @@ CallContext GetInstructionCallContext(HloOpcode opcode) {
     case HloOpcode::kReduceWindow:
     case HloOpcode::kScatter:
     case HloOpcode::kSelectAndScatter:
+    case HloOpcode::kSort:
     case HloOpcode::kFusion:
       return CallContext::kParallel;
     default:
diff --git a/tensorflow/compiler/xla/service/cpu/tests/cpu_key_value_sort_test.cc b/tensorflow/compiler/xla/service/cpu/tests/cpu_key_value_sort_test.cc
index 3934c03a04..762ee67db9 100644
--- a/tensorflow/compiler/xla/service/cpu/tests/cpu_key_value_sort_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/tests/cpu_key_value_sort_test.cc
@@ -26,10 +26,16 @@ TEST_F(CpuKeyValueSortTest, SortR1) {
   const string hlo_text = R"(
 HloModule KeyValueSort
 
+compare {
+  p.0.lhs = f32[] parameter(0)
+  p.0.rhs = f32[] parameter(1)
+  ROOT lt = pred[] less-than(p.0.lhs, p.0.rhs)
+}
+
 ENTRY main {
   a = f32[10] parameter(0)
 
-  ROOT result = f32[10] sort(f32[10] a), dimensions={0}
+  ROOT result = f32[10] sort(f32[10] a), dimensions={0}, to_apply=compare
 }
 )";
 
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_layout_assignment_test.cc b/tensorflow/compiler/xla/service/gpu/gpu_layout_assignment_test.cc
index 29756d2726..391029e574 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_layout_assignment_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_layout_assignment_test.cc
@@ -368,12 +368,21 @@ TEST_F(LayoutAssignmentTest, DotLayout) {
 TEST_F(LayoutAssignmentTest, SortLayout) {
   const char* hlo_text = R"(
   HloModule SortLayout
+
+  compare {
+    p.0.lhs = f32[] parameter(0)
+    p.0.rhs = f32[] parameter(1)
+    p.1.lhs = f32[] parameter(2)
+    p.1.rhs = f32[] parameter(3)
+    ROOT lt = pred[] less-than(p.0.lhs, p.0.rhs)
+  }
+
   ENTRY sort {
     keys = f32[3,2]{0,1} constant({{0,1},{0,1},{0,1}})
     values = f32[2,3]{1,0} parameter(0)
     transpose = f32[3,2]{1,0} transpose(values), dimensions={1,0}
     ROOT sort = (f32[3,2]{1,0}, f32[3,2]{1,0}) sort(keys, transpose),
-      dimensions={1}
+      dimensions={1}, to_apply=compare
   })";
 
   TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
diff --git a/tensorflow/compiler/xla/service/hlo_element_type_converter.cc b/tensorflow/compiler/xla/service/hlo_element_type_converter.cc
index 812db465c6..bff467772d 100644
--- a/tensorflow/compiler/xla/service/hlo_element_type_converter.cc
+++ b/tensorflow/compiler/xla/service/hlo_element_type_converter.cc
@@ -149,7 +149,11 @@ StatusOr<bool> HloElementTypeConverter::Run(HloModule* module) {
           opcode == HloOpcode::kConditional) {
         continue;
       }
-      TF_RET_CHECK(hlo->called_computations().empty()) << hlo->ToString();
+      // TODO(b/122298745): Once we don't ignore called computations anymore,
+      // add kSort to the if statement above.
+      if (opcode != HloOpcode::kSort) {
+        TF_RET_CHECK(hlo->called_computations().empty()) << hlo->ToString();
+      }
 
       bool nullary = hlo->operands().empty();
       bool wrong_element_type = hlo->shape().element_type() == eliminate_type_;
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index 1b677bc25c..fd9b8ea352 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -203,9 +203,14 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
           << "Sort instruction should have 1 dimension";
       auto sort_operands = all_operands();
       HloInstruction* keys = sort_operands[0];
-      instruction = CreateSort(
-          shape, proto.dimensions(0), keys,
-          absl::Span<HloInstruction* const>(sort_operands).subspan(1));
+      if (proto.called_computation_ids_size() == 1) {
+        instruction = CreateSort(shape, proto.dimensions(0), all_operands(),
+                                 computations(0));
+      } else {
+        instruction = CreateSort(
+            shape, proto.dimensions(0), keys,
+            absl::Span<HloInstruction* const>(sort_operands).subspan(1));
+      }
       break;
     }
     case HloOpcode::kTranspose:
@@ -1158,6 +1163,13 @@ HloInstruction::CreateBroadcastSequence(
   return absl::make_unique<HloSortInstruction>(shape, dimension, keys, values);
 }
 
+/* static */ std::unique_ptr<HloInstruction> HloInstruction::CreateSort(
+    const Shape& shape, int64 dimension,
+    absl::Span<HloInstruction* const> operands, HloComputation* compare) {
+  return absl::make_unique<HloSortInstruction>(shape, dimension, operands,
+                                               compare);
+}
+
 /* static */ std::unique_ptr<HloInstruction> HloInstruction::CreateFusion(
     const Shape& shape, FusionKind fusion_kind, HloInstruction* fused_root) {
   return absl::make_unique<HloFusionInstruction>(shape, fusion_kind,
@@ -1952,6 +1964,7 @@ HloComputation* HloInstruction::to_apply() const {
     case HloOpcode::kReduce:
     case HloOpcode::kAllReduce:
     case HloOpcode::kScatter:
+    case HloOpcode::kSort:
       CHECK_EQ(called_computations_.size(), 1);
       return called_computations_[0];
     default:
@@ -1971,6 +1984,7 @@ void HloInstruction::set_to_apply(HloComputation* computation) {
     case HloOpcode::kReduce:
     case HloOpcode::kAllReduce:
     case HloOpcode::kScatter:
+    case HloOpcode::kSort:
       CHECK_EQ(called_computations_.size(), 1);
       called_computations_[0] = computation;
       break;
@@ -2243,9 +2257,14 @@ std::vector<string> HloInstruction::ExtraAttributesToString(
                opcode() == HloOpcode::kReduceWindow ||
                opcode() == HloOpcode::kReduce ||
                opcode() == HloOpcode::kAllReduce ||
-               opcode() == HloOpcode::kScatter) {
-      extra.push_back(
-          StrCat("to_apply=", PrintName(to_apply()->name(), options)));
+               opcode() == HloOpcode::kScatter ||
+               opcode() == HloOpcode::kSort) {
+      // TODO(b/122298745): Remove this check when Sort has a required
+      // sub-computation.
+      if (!called_computations().empty()) {
+        extra.push_back(
+            StrCat("to_apply=", PrintName(to_apply()->name(), options)));
+      }
     } else if (!called_computations().empty()) {
       extra.push_back(StrCat(
           "calls=",
@@ -2280,8 +2299,13 @@ std::vector<string> HloInstruction::ExtraAttributesToString(
       case HloOpcode::kReduce:
       case HloOpcode::kAllReduce:
       case HloOpcode::kScatter:
-        extra.push_back(
-            StrCat("to_apply=\n", to_apply()->ToString(new_options)));
+      case HloOpcode::kSort:
+        // TODO(b/122298745): Remove this check once sort has a required
+        // sub-computation.
+        if (to_apply() != nullptr) {
+          extra.push_back(
+              StrCat("to_apply=\n", to_apply()->ToString(new_options)));
+        }
         break;
       default:
         if (!called_computations().empty()) {
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h
index c11d29d33e..79c6c85f72 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.h
+++ b/tensorflow/compiler/xla/service/hlo_instruction.h
@@ -676,6 +676,15 @@ class HloInstruction {
       const Shape& shape, int64 dimension, HloInstruction* keys,
       absl::Span<HloInstruction* const> values = {});
 
+  // Creates a n-ary sort op with a 'compare' computation which is used for
+  // comparisons in the sorting algorithm. 'compare' gets 2 * n parameters,
+  // where parameters 2 * i and 2 * i + 1 are the values of the i-th operand at
+  // specific index positions which should be compared, and should return a
+  // PRED.
+  static std::unique_ptr<HloInstruction> CreateSort(
+      const Shape& shape, int64 dimension,
+      absl::Span<HloInstruction* const> operands, HloComputation* compare);
+
   // Creates a while instruction, given a condition computation, a body
   // computation, and the initial value for the input of the computations. For
   // example, shape: S32, condition: i -> i < 1000, body: i -> i * 2, init: 1
diff --git a/tensorflow/compiler/xla/service/hlo_instructions.cc b/tensorflow/compiler/xla/service/hlo_instructions.cc
index 785206bf77..8e40e54c3d 100644
--- a/tensorflow/compiler/xla/service/hlo_instructions.cc
+++ b/tensorflow/compiler/xla/service/hlo_instructions.cc
@@ -619,6 +619,16 @@ HloSortInstruction::HloSortInstruction(const Shape& shape, int64 dimension,
   }
 }
 
+HloSortInstruction::HloSortInstruction(
+    const Shape& shape, int64 dimension,
+    absl::Span<HloInstruction* const> operands, HloComputation* compare)
+    : HloInstruction(HloOpcode::kSort, shape), dimensions_({dimension}) {
+  for (auto* value : operands) {
+    AppendOperand(value);
+  }
+  AppendComputation(compare);
+}
+
 HloInstructionProto HloSortInstruction::ToProto() const {
   HloInstructionProto proto = HloInstruction::ToProto();
   for (int64 dimension : dimensions_) {
@@ -637,12 +647,25 @@ bool HloSortInstruction::IdenticalSlowPath(
     const std::function<bool(const HloComputation*, const HloComputation*)>&
         eq_computations) const {
   const auto& casted_other = static_cast<const HloSortInstruction&>(other);
-  return dimensions() == casted_other.dimensions();
+  if (dimensions() != casted_other.dimensions()) {
+    return false;
+  }
+  if (called_computations().empty()) {
+    return other.called_computations().empty();
+  }
+  if (other.called_computations().empty()) {
+    return false;
+  }
+  return eq_computations(to_apply(), other.to_apply());
 }
 
 std::unique_ptr<HloInstruction> HloSortInstruction::CloneWithNewOperandsImpl(
     const Shape& shape, absl::Span<HloInstruction* const> new_operands,
     HloCloneContext* context) const {
+  if (!called_computations().empty()) {
+    return absl::make_unique<HloSortInstruction>(shape, dimensions(0),
+                                                 new_operands, to_apply());
+  }
   HloInstruction* keys = new_operands[0];
   return absl::make_unique<HloSortInstruction>(shape, dimensions(0), keys,
                                                new_operands.subspan(1));
diff --git a/tensorflow/compiler/xla/service/hlo_instructions.h b/tensorflow/compiler/xla/service/hlo_instructions.h
index 1b4a94753c..6b6157a776 100644
--- a/tensorflow/compiler/xla/service/hlo_instructions.h
+++ b/tensorflow/compiler/xla/service/hlo_instructions.h
@@ -420,6 +420,9 @@ class HloSortInstruction : public HloInstruction {
   explicit HloSortInstruction(const Shape& shape, int64 dimension,
                               HloInstruction* keys,
                               absl::Span<HloInstruction* const> values = {});
+  explicit HloSortInstruction(const Shape& shape, int64 dimension,
+                              absl::Span<HloInstruction* const> operands,
+                              HloComputation* compare);
   // Returns the dimension sizes or numbers associated with this instruction.
   const std::vector<int64>& dimensions() const override { return dimensions_; }
   int64 dimensions(int64 index) const override { return dimensions()[index]; }
diff --git a/tensorflow/compiler/xla/service/hlo_parser.cc b/tensorflow/compiler/xla/service/hlo_parser.cc
index 00c810675b..b3b24c148a 100644
--- a/tensorflow/compiler/xla/service/hlo_parser.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser.cc
@@ -894,14 +894,23 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
       optional<std::vector<tensorflow::int64>> dimensions;
       attrs["dimensions"] = {/*required=*/true, AttrTy::kBracedInt64List,
                              &dimensions};
+      optional<HloComputation*> to_apply;
+      // TODO(b/122298745): Make this required.
+      attrs["to_apply"] = {/*required=*/false, AttrTy::kHloComputation,
+                           &to_apply};
       if (!ParseOperands(&operands) || !ParseAttributes(attrs) ||
           dimensions->size() != 1) {
         return false;
       }
-      instruction = builder->AddInstruction(HloInstruction::CreateSort(
-          shape, dimensions->at(0),
-          /*keys=*/operands[0],
-          /*values=*/absl::Span<HloInstruction* const>(operands).subspan(1)));
+      if (to_apply.has_value()) {
+        instruction = builder->AddInstruction(HloInstruction::CreateSort(
+            shape, dimensions->at(0), operands, to_apply.value()));
+      } else {
+        instruction = builder->AddInstruction(HloInstruction::CreateSort(
+            shape, dimensions->at(0),
+            /*keys=*/operands[0],
+            /*values=*/absl::Span<HloInstruction* const>(operands).subspan(1)));
+      }
       break;
     }
     case HloOpcode::kTuple: {
diff --git a/tensorflow/compiler/xla/service/hlo_parser_test.cc b/tensorflow/compiler/xla/service/hlo_parser_test.cc
index b66d84caec..cd7effeae0 100644
--- a/tensorflow/compiler/xla/service/hlo_parser_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser_test.cc
@@ -1047,9 +1047,15 @@ ENTRY ReducePrecision {
 "SortKey",
 R"(HloModule sort
 
+compare {
+  p.0.lhs = f32[] parameter(0)
+  p.0.rhs = f32[] parameter(1)
+  ROOT lt = pred[] less-than(p.0.lhs, p.0.rhs)
+}
+
 ENTRY Sort {
   x = f32[1024]{0} parameter(0)
-  ROOT sorted = f32[1024]{0} sort(x), dimensions={0}
+  ROOT sorted = f32[1024]{0} sort(x), dimensions={0}, to_apply=compare
 }
 
 )"
@@ -1059,10 +1065,18 @@ ENTRY Sort {
 "SortKeyValue",
 R"(HloModule sort
 
+compare {
+  p.1.lhs = s32[] parameter(2)
+  p.1.rhs = s32[] parameter(3)
+  p.0.lhs = f32[] parameter(0)
+  p.0.rhs = f32[] parameter(1)
+  ROOT lt = pred[] less-than(p.0.lhs, p.0.rhs)
+}
+
 ENTRY Sort {
   keys = f32[1024]{0} parameter(0)
   values = s32[1024]{0} parameter(1)
-  ROOT sorted = (f32[1024]{0}, s32[1024]{0}) sort(keys, values), dimensions={0}
+  ROOT sorted = (f32[1024]{0}, s32[1024]{0}) sort(keys, values), dimensions={0}, to_apply=compare
 }
 
 )"
@@ -1072,9 +1086,15 @@ ENTRY Sort {
 "SortKeyR2",
 R"(HloModule sort
 
+compare {
+  p.0.lhs = f32[] parameter(0)
+  p.0.rhs = f32[] parameter(1)
+  ROOT lt = pred[] less-than(p.0.lhs, p.0.rhs)
+}
+
 ENTRY Sort {
   x = f32[1024,16]{0,1} parameter(0)
-  ROOT sorted = f32[1024,16]{0,1} sort(x), dimensions={0}
+  ROOT sorted = f32[1024,16]{0,1} sort(x), dimensions={0}, to_apply=compare
 }
 
 )"
@@ -1084,10 +1104,18 @@ ENTRY Sort {
 "SortKeyValueR2",
 R"(HloModule sort
 
+compare {
+  p.1.lhs = s32[] parameter(2)
+  p.1.rhs = s32[] parameter(3)
+  p.0.lhs = f32[] parameter(0)
+  p.0.rhs = f32[] parameter(1)
+  ROOT lt = pred[] less-than(p.0.lhs, p.0.rhs)
+}
+
 ENTRY Sort {
   keys = f32[1024,16]{0,1} parameter(0)
   values = s32[1024,16]{0,1} parameter(1)
-  ROOT sorted = (f32[1024,16]{0,1}, s32[1024,16]{0,1}) sort(keys, values), dimensions={0}
+  ROOT sorted = (f32[1024,16]{0,1}, s32[1024,16]{0,1}) sort(keys, values), dimensions={0}, to_apply=compare
 }
 
 )"
@@ -1097,12 +1125,24 @@ ENTRY Sort {
 "SortManyValues",
 R"(HloModule sort
 
+compare {
+  p.1.lhs = s32[] parameter(2)
+  p.1.rhs = s32[] parameter(3)
+  p.2.lhs = u32[] parameter(4)
+  p.2.rhs = u32[] parameter(5)
+  p.3.lhs = f32[] parameter(6)
+  p.3.rhs = f32[] parameter(7)
+  p.0.lhs = f32[] parameter(0)
+  p.0.rhs = f32[] parameter(1)
+  ROOT lt = pred[] less-than(p.0.lhs, p.0.rhs)
+}
+
 ENTRY Sort {
   keys = f32[1024,16]{0,1} parameter(0)
   values.0 = s32[1024,16]{0,1} parameter(1)
   values.1 = u32[1024,16]{0,1} parameter(2)
   values.2 = f32[1024,16]{0,1} parameter(3)
-  ROOT sorted = (f32[1024,16]{0,1}, s32[1024,16]{0,1}, u32[1024,16]{0,1}, f32[1024,16]{0,1}) sort(keys, values.0, values.1, values.2), dimensions={0}
+  ROOT sorted = (f32[1024,16]{0,1}, s32[1024,16]{0,1}, u32[1024,16]{0,1}, f32[1024,16]{0,1}) sort(keys, values.0, values.1, values.2), dimensions={0}, to_apply=compare
 }
 
 )"
@@ -1320,10 +1360,18 @@ ENTRY Computation {
 "ScheduledModule",
 R"(HloModule scheduled_module, is_scheduled=true
 
+compare {
+  p.1.lhs = s32[] parameter(2)
+  p.1.rhs = s32[] parameter(3)
+  p.0.lhs = f32[] parameter(0)
+  p.0.rhs = f32[] parameter(1)
+  ROOT lhs = pred[] less-than(p.0.lhs, p.0.rhs)
+}
+
 ENTRY Sort {
   keys = f32[1024]{0} parameter(0)
   values = s32[1024]{0} parameter(1)
-  ROOT sorted = (f32[1024]{0}, s32[1024]{0}) sort(keys, values), dimensions={0}
+  ROOT sorted = (f32[1024]{0}, s32[1024]{0}) sort(keys, values), dimensions={0}, to_apply=compare
 }
 
 )"
@@ -1407,7 +1455,7 @@ class HloParameterizedParserTest
  protected:
   // Expects "ToString(ParseHloString(string)) == string", that is, parses the
   // string, asserts that it succeeded, stringifies the parsed module, and
-  // checks that the it equals the original string.
+  // checks that it equals the original string.
   void ExpectEqual() {
     const string& original = GetParam().module_string;
     TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc
index 12d1b4ac2c..9d94c5cf76 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier.cc
@@ -50,6 +50,7 @@ bool IsCallerInstruction(HloInstruction* hlo) {
     case HloOpcode::kReduceWindow:
     case HloOpcode::kScatter:
     case HloOpcode::kSelectAndScatter:
+    case HloOpcode::kSort:
     case HloOpcode::kFusion:
       return true;
     default:
diff --git a/tensorflow/compiler/xla/service/sort_simplifier_test.cc b/tensorflow/compiler/xla/service/sort_simplifier_test.cc
index cd05fcf830..a05bc79989 100644
--- a/tensorflow/compiler/xla/service/sort_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/sort_simplifier_test.cc
@@ -34,13 +34,21 @@ TEST_F(SortSimplifierTest, RemoveUnusedSortOperandArrayResult) {
   const char* hlo_string = R"(
    HloModule permutation_sort
 
-    ENTRY sort_computation {
-      keys = f32[64,8732]{1,0} parameter(0)
-      values = s32[64,8732]{1,0} parameter(1)
-      sort = (f32[64,8732]{1,0}, s32[64,8732]{1,0}) sort(keys, values),
-        dimensions={1}
-      ROOT gte = f32[64,8732]{1,0} get-tuple-element(sort), index=0
-    })";
+   compare {
+     p.0.lhs = f32[] parameter(0)
+     p.0.rhs = f32[] parameter(1)
+     p.1.lhs = s32[] parameter(2)
+     p.1.rhs = s32[] parameter(3)
+     ROOT lt = pred[] less-than(p.0.lhs, p.0.rhs)
+   }
+
+   ENTRY sort_computation {
+     keys = f32[64,8732]{1,0} parameter(0)
+     values = s32[64,8732]{1,0} parameter(1)
+     sort = (f32[64,8732]{1,0}, s32[64,8732]{1,0}) sort(keys, values),
+       dimensions={1}, to_apply=compare
+     ROOT gte = f32[64,8732]{1,0} get-tuple-element(sort), index=0
+   })";
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
@@ -58,17 +66,27 @@ TEST_F(SortSimplifierTest, RemoveUnusedSortOperandTuple) {
   const char* hlo_string = R"(
    HloModule permutation_sort
 
-    ENTRY sort_computation {
-      keys = f32[64,87] parameter(0)
-      values.0 = s32[64,87] parameter(1)
-      values.1 = u32[64,87] parameter(2)
-      sort = (f32[64,87], s32[64,87], u32[64,87]) sort(
-          keys, values.0, values.1),
-        dimensions={1}
-      gte.0 = f32[64,87] get-tuple-element(sort), index=0
-      gte.1 = u32[64,87] get-tuple-element(sort), index=2
-      ROOT tuple = (f32[64,87], u32[64,87]) tuple(gte.0, gte.1)
-    })";
+   compare {
+     p.0.lhs = f32[] parameter(0)
+     p.0.rhs = f32[] parameter(1)
+     p.1.lhs = s32[] parameter(2)
+     p.1.rhs = s32[] parameter(3)
+     p.2.lhs = u32[] parameter(4)
+     p.2.rhs = u32[] parameter(5)
+     ROOT lt = pred[] less-than(p.0.lhs, p.0.rhs)
+   }
+
+   ENTRY sort_computation {
+     keys = f32[64,87] parameter(0)
+     values.0 = s32[64,87] parameter(1)
+     values.1 = u32[64,87] parameter(2)
+     sort = (f32[64,87], s32[64,87], u32[64,87]) sort(
+         keys, values.0, values.1),
+       dimensions={1}, to_apply=compare
+     gte.0 = f32[64,87] get-tuple-element(sort), index=0
+     gte.1 = u32[64,87] get-tuple-element(sort), index=2
+     ROOT tuple = (f32[64,87], u32[64,87]) tuple(gte.0, gte.1)
+   })";
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
@@ -86,12 +104,20 @@ TEST_F(SortSimplifierTest, DontRemoveUnusedSortKey) {
   const char* hlo_string = R"(
    HloModule permutation_sort
 
-    ENTRY sort_computation {
-      keys = f32[64,8732]{1,0} parameter(0)
-      values = s32[64,8732]{1,0} parameter(1)
-      sort = (f32[64,8732]{1,0}, s32[64,8732]{1,0}) sort(keys, values), dimensions={1}
-      ROOT gte = s32[64,8732]{1,0} get-tuple-element(sort), index=1
-    })";
+   compare {
+     p.0.lhs = f32[] parameter(0)
+     p.0.rhs = f32[] parameter(1)
+     p.1.lhs = s32[] parameter(2)
+     p.1.rhs = s32[] parameter(3)
+     ROOT lt = pred[] less-than(p.0.lhs, p.0.rhs)
+   }
+
+   ENTRY sort_computation {
+     keys = f32[64,8732]{1,0} parameter(0)
+     values = s32[64,8732]{1,0} parameter(1)
+     sort = (f32[64,8732]{1,0}, s32[64,8732]{1,0}) sort(keys, values), dimensions={1}, to_apply=compare
+     ROOT gte = s32[64,8732]{1,0} get-tuple-element(sort), index=1
+   })";
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
diff --git a/tensorflow/compiler/xla/tests/test_utils_test.cc b/tensorflow/compiler/xla/tests/test_utils_test.cc
index 591d6c1922..321c3fb2df 100644
--- a/tensorflow/compiler/xla/tests/test_utils_test.cc
+++ b/tensorflow/compiler/xla/tests/test_utils_test.cc
@@ -136,10 +136,18 @@ XLA_TEST_F(TestUtilsTest, NoDuplicatesFloats) {
   auto module = ParseHloString(R"(
 HloModule sort.148.1589
 
+compare {
+  p.0.lhs = f32[] parameter(0)
+  p.0.rhs = f32[] parameter(1)
+  p.1.lhs = s32[] parameter(2)
+  p.1.rhs = s32[] parameter(3)
+  ROOT lt = pred[] less-than(p.0.lhs, p.0.rhs)
+}
+
 ENTRY %sort.148.1589 (parameter.0: f32[1048576], parameter.1: s32[1048576]) -> (f32[1048576], s32[1048576]) {
   %parameter.0 = f32[1048576]{0} parameter(0)
   %parameter.1 = s32[1048576]{0} parameter(1)
-  ROOT %sort.148.1589 = (f32[1048576]{0}, s32[1048576]{0}) sort(f32[1048576]{0} %parameter.0, s32[1048576]{0} %parameter.1), dimensions={0}
+  ROOT %sort.148.1589 = (f32[1048576]{0}, s32[1048576]{0}) sort(f32[1048576]{0} %parameter.0, s32[1048576]{0} %parameter.1), dimensions={0}, to_apply=compare
 }
 )")
                     .ValueOrDie();
@@ -159,10 +167,18 @@ XLA_TEST_F(TestUtilsTest, NoDuplicatesInt32) {
   auto module = ParseHloString(R"(
 HloModule sort.148.1589
 
+compare {
+  p.0.lhs = s32[] parameter(0)
+  p.0.rhs = s32[] parameter(1)
+  p.1.lhs = s32[] parameter(2)
+  p.1.rhs = s32[] parameter(3)
+  ROOT lt = pred[] less-than(p.0.lhs, p.0.rhs)
+}
+
 ENTRY %sort.148.1589 (parameter.0: s32[1048576], parameter.1: s32[1048576]) -> (s32[1048576], s32[1048576]) {
   %parameter.0 = s32[1048576]{0} parameter(0)
   %parameter.1 = s32[1048576]{0} parameter(1)
-  ROOT %sort.148.1589 = (s32[1048576]{0}, s32[1048576]{0}) sort(s32[1048576]{0} %parameter.0, s32[1048576]{0} %parameter.1), dimensions={0}
+  ROOT %sort.148.1589 = (s32[1048576]{0}, s32[1048576]{0}) sort(s32[1048576]{0} %parameter.0, s32[1048576]{0} %parameter.1), dimensions={0}, to_apply=compare
 }
 )")
                     .ValueOrDie();
@@ -182,10 +198,18 @@ XLA_TEST_F(TestUtilsTest, NoDuplicatesBfloat16) {
   auto module = ParseHloString(R"(
 HloModule sort, is_scheduled=true
 
+compare {
+  p.0.lhs = bf16[] parameter(0)
+  p.0.rhs = bf16[] parameter(1)
+  p.1.lhs = s32[] parameter(2)
+  p.1.rhs = s32[] parameter(3)
+  ROOT lt = pred[] less-than(p.0.lhs, p.0.rhs)
+}
+
 ENTRY %sort. (parameter.0: bf16[2,1452], parameter.1: s32[2,1452]) -> (bf16[2,1452], s32[2,1452]) {
   %parameter.0 = bf16[2,1452]{1,0} parameter(0)
   %parameter.1 = s32[2,1452]{1,0} parameter(1)
-  ROOT %sort = (bf16[2,1452]{1,0}, s32[2,1452]{1,0}) sort(bf16[2,1452]{1,0} %parameter.0, s32[2,1452]{1,0} %parameter.1), dimensions={1}
+  ROOT %sort = (bf16[2,1452]{1,0}, s32[2,1452]{1,0}) sort(bf16[2,1452]{1,0} %parameter.0, s32[2,1452]{1,0} %parameter.1), dimensions={1}, to_apply=compare
 }
 )")
                     .ValueOrDie();
-- 
GitLab


From 97c19661e9a50081914169b5a4086916cf1fd962 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 6 Feb 2019 01:02:27 -0800
Subject: [PATCH 1977/2345] compat: Update forward compatibility horizon to
 2019-02-06

PiperOrigin-RevId: 232631405
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 262df69275..3ba54688c7 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 2, 5)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 2, 6)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From e5b9fdd4c77b0240b06996e15122741120d97685 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 6 Feb 2019 01:53:47 -0800
Subject: [PATCH 1978/2345] [TF:XLA] Enable tests and update descriptions.

PiperOrigin-RevId: 232637449
---
 tensorflow/python/kernel_tests/pooling_ops_test.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/kernel_tests/pooling_ops_test.py b/tensorflow/python/kernel_tests/pooling_ops_test.py
index 94a861f554..546b406c20 100644
--- a/tensorflow/python/kernel_tests/pooling_ops_test.py
+++ b/tensorflow/python/kernel_tests/pooling_ops_test.py
@@ -1211,7 +1211,6 @@ class PoolingTest(test.TestCase):
                      [1, window_rows, window_cols, 1],
                      [1, row_stride, col_stride, 1], padding)
 
-  @test_util.disable_xla("This test never passed for XLA")
   def _testMaxPoolGradDirect(self, input_data, output_backprop,
                              expected_input_backprop, input_sizes, output_sizes,
                              window_rows, window_cols, row_stride, col_stride,
@@ -1353,6 +1352,7 @@ class PoolingTest(test.TestCase):
             use_gpu=use_gpu,
             v2=v2)
 
+  @test_util.disable_xla("b/123923733")  # NaNs handled differently
   def _testMaxPoolGradDirectWithNans2_1(self):
     input_data = [float("nan")] * 16
     output_backprop = [11.0, 12.0, 13.0, 15.0, 16.0, 17.0, 19.0, 20.0, 21.0]
@@ -1427,6 +1427,7 @@ class PoolingTest(test.TestCase):
     else:
       del os.environ["TF_ENABLE_MAXPOOL_NANPROP"]
 
+  @test_util.disable_xla("b/123923733")  # NaNs handled differently
   def _testMaxPoolGradDirectWithNans2_2(self):
     input_data = [float("nan")] * 16
     output_backprop = [
@@ -1627,7 +1628,6 @@ class PoolingTest(test.TestCase):
           use_gpu=use_gpu)
 
   @test_util.run_deprecated_v1
-  @test_util.disable_xla("This test never passed for XLA")
   def testMaxPoolGradGrad(self):
     for (data_format, use_gpu) in GetTestConfigs():
       self._testMaxPoolGradGradValidPadding1_1(data_format, use_gpu)
@@ -1662,7 +1662,6 @@ class PoolingTest(test.TestCase):
         [1, row_stride, col_stride, 1], padding)
 
   @test_util.run_deprecated_v1
-  @test_util.disable_xla("This test never passed for XLA")
   def testAvgPoolGrad(self):
     for (data_format, use_gpu) in GetTestConfigs():
       self._testAvgPoolGradValidPadding1_1(data_format, use_gpu)
@@ -1901,9 +1900,10 @@ if __name__ == "__main__":
     if name_ == "maxpool5":
       setattr(
           PoolingTest, "testMaxPoolGrad_" + name_,
-          test_util.disable_xla("maxpool5 fails while all others pass")(
-              GetMaxPoolGradTest(input_size_, filter_size_, output_size_,
-                                 stride_, padding_)))
+          test_util.disable_xla(
+              "b/123926014: incorrect output with only constants")(
+                  GetMaxPoolGradTest(input_size_, filter_size_, output_size_,
+                                     stride_, padding_)))
     else:
       setattr(
           PoolingTest, "testMaxPoolGrad_" + name_,
-- 
GitLab


From 700ae54188ff594059712881543aead199fd2742 Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Wed, 6 Feb 2019 17:05:58 +0530
Subject: [PATCH 1979/2345] Update ring.h

accumululated -> accumulated
---
 tensorflow/contrib/mpi_collectives/ring.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/mpi_collectives/ring.h b/tensorflow/contrib/mpi_collectives/ring.h
index cae57ce60e..9b5d52e1b6 100644
--- a/tensorflow/contrib/mpi_collectives/ring.h
+++ b/tensorflow/contrib/mpi_collectives/ring.h
@@ -129,7 +129,7 @@ cudaStream_t CudaStreamForMPI();
  *  has the fully accumulated Segment 1; and so on. The scatter-reduce is
  * complete.
  *
- *  Next, the allgather distributes these fully accumululated chunks across all
+ *  Next, the allgather distributes these fully accumulated chunks across all
  * nodes. Communication proceeds in the same ring, once again in N-1 steps. At
  * the ith step, node j will send chunk (j - i + 1) and receive chunk (j - i).
  * For example, at the first iteration, the following transfers will occur:
-- 
GitLab


From 8c10269efc125ae285c5943b27b0ca2e6e8428a4 Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Wed, 6 Feb 2019 17:14:38 +0530
Subject: [PATCH 1980/2345] Update bits.h

__buitin_clzll  >> __builtin_clzll
---
 tensorflow/lite/experimental/microfrontend/lib/bits.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/experimental/microfrontend/lib/bits.h b/tensorflow/lite/experimental/microfrontend/lib/bits.h
index bf15466a3d..04b3ba6f05 100644
--- a/tensorflow/lite/experimental/microfrontend/lib/bits.h
+++ b/tensorflow/lite/experimental/microfrontend/lib/bits.h
@@ -63,14 +63,14 @@ static inline int CountLeadingZeros64Slow(uint64_t n) {
 
 static inline int CountLeadingZeros64(uint64_t n) {
 #if defined(_MSC_VER) && defined(_M_X64)
-  // MSVC does not have __buitin_clzll. Use _BitScanReverse64.
+  // MSVC does not have __builtin_clzll. Use _BitScanReverse64.
   unsigned long result = 0;  // NOLINT(runtime/int)
   if (_BitScanReverse64(&result, n)) {
     return 63 - result;
   }
   return 64;
 #elif defined(_MSC_VER)
-  // MSVC does not have __buitin_clzll. Compose two calls to _BitScanReverse
+  // MSVC does not have __builtin_clzll. Compose two calls to _BitScanReverse
   unsigned long result = 0;  // NOLINT(runtime/int)
   if ((n >> 32) && _BitScanReverse(&result, n >> 32)) {
     return 31 - result;
-- 
GitLab


From d20954874dab39dc67fd2222003b9ab5ea4840d5 Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Wed, 6 Feb 2019 05:29:06 -0800
Subject: [PATCH 1981/2345] Use short name alias for
 distribution_strategy_context in tests.

PiperOrigin-RevId: 232658091
---
 .../python/distribute/distribute_lib_test.py  | 61 ++++++++-----------
 1 file changed, 25 insertions(+), 36 deletions(-)

diff --git a/tensorflow/python/distribute/distribute_lib_test.py b/tensorflow/python/distribute/distribute_lib_test.py
index c147849e5d..6876af3775 100644
--- a/tensorflow/python/distribute/distribute_lib_test.py
+++ b/tensorflow/python/distribute/distribute_lib_test.py
@@ -19,7 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.distribute import distribute_lib
-from tensorflow.python.distribute import distribution_strategy_context
+from tensorflow.python.distribute import distribution_strategy_context as ds_context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.ops import variable_scope
@@ -60,13 +60,12 @@ class _TestExtended(distribute_lib.DistributionStrategyExtended):
 
 
 def _assert_in_default_state(t):
-  t.assertIs(distribution_strategy_context._get_default_replica_context(),
-             distribution_strategy_context.get_replica_context())
-  t.assertIs(None, distribution_strategy_context.get_cross_replica_context())
-  t.assertFalse(distribution_strategy_context.in_cross_replica_context())
-  t.assertIs(distribution_strategy_context._get_default_strategy(),
-             distribution_strategy_context.get_strategy())
-  t.assertFalse(distribution_strategy_context.has_strategy())
+  t.assertIs(ds_context._get_default_replica_context(),
+             ds_context.get_replica_context())
+  t.assertIs(None, ds_context.get_cross_replica_context())
+  t.assertFalse(ds_context.in_cross_replica_context())
+  t.assertIs(ds_context._get_default_strategy(), ds_context.get_strategy())
+  t.assertFalse(ds_context.has_strategy())
 
 
 class TestStrategyTest(test.TestCase):
@@ -76,14 +75,12 @@ class TestStrategyTest(test.TestCase):
     dist = _TestStrategy()
 
     def run_fn():
-      replica_context = distribution_strategy_context.get_replica_context()
+      replica_context = ds_context.get_replica_context()
       self.assertTrue(replica_context is not None)
-      self.assertIs(None,
-                    distribution_strategy_context.get_cross_replica_context())
-      self.assertFalse(distribution_strategy_context.in_cross_replica_context())
-      self.assertTrue(distribution_strategy_context.has_strategy())
-      self.assertIs(dist,
-                    distribution_strategy_context.get_strategy())
+      self.assertIs(None, ds_context.get_cross_replica_context())
+      self.assertFalse(ds_context.in_cross_replica_context())
+      self.assertTrue(ds_context.has_strategy())
+      self.assertIs(dist, ds_context.get_strategy())
       self.assertEqual("foo", replica_context.merge_call(None, test_arg="foo"))
       expected_value = _get_test_variable(
           "bar", variable_scope.VariableSynchronization.AUTO,
@@ -101,13 +98,11 @@ class TestStrategyTest(test.TestCase):
     _assert_in_default_state(self)
     dist = _TestStrategy()
     with dist.scope():
-      self.assertIs(None, distribution_strategy_context.get_replica_context())
-      self.assertIs(dist,
-                    distribution_strategy_context.get_cross_replica_context())
-      self.assertTrue(distribution_strategy_context.in_cross_replica_context())
-      self.assertTrue(distribution_strategy_context.has_strategy())
-      self.assertIs(dist,
-                    distribution_strategy_context.get_strategy())
+      self.assertIs(None, ds_context.get_replica_context())
+      self.assertIs(dist, ds_context.get_cross_replica_context())
+      self.assertTrue(ds_context.in_cross_replica_context())
+      self.assertTrue(ds_context.has_strategy())
+      self.assertIs(dist, ds_context.get_strategy())
       expected_value = _get_test_variable(
           "baz", variable_scope.VariableSynchronization.AUTO,
           variable_scope.VariableAggregation.NONE)
@@ -138,22 +133,16 @@ class DefaultDistributionStrategyTest(test.TestCase):
     _assert_in_default_state(self)
 
     def merge_fn(dist, s):
-      self.assertIs(
-          distribution_strategy_context._get_default_strategy(),
-          dist)
-      self.assertIs(None, distribution_strategy_context.get_replica_context())
-      self.assertIs(dist,
-                    distribution_strategy_context.get_cross_replica_context())
-      self.assertTrue(distribution_strategy_context.in_cross_replica_context())
-      self.assertIs(dist,
-                    distribution_strategy_context.get_strategy())
-      self.assertFalse(
-          distribution_strategy_context.has_strategy())
+      self.assertIs(ds_context._get_default_strategy(), dist)
+      self.assertIs(None, ds_context.get_replica_context())
+      self.assertIs(dist, ds_context.get_cross_replica_context())
+      self.assertTrue(ds_context.in_cross_replica_context())
+      self.assertIs(dist, ds_context.get_strategy())
+      self.assertFalse(ds_context.has_strategy())
       return "foo_" + s
 
-    replica_ctx = distribution_strategy_context.get_replica_context()
-    self.assertIs(distribution_strategy_context._get_default_replica_context(),
-                  replica_ctx)
+    replica_ctx = ds_context.get_replica_context()
+    self.assertIs(ds_context._get_default_replica_context(), replica_ctx)
     self.assertEqual("foo_bar", replica_ctx.merge_call(merge_fn, args=("bar",)))
     _assert_in_default_state(self)
 
-- 
GitLab


From 2f672ee9562a452f8dbfa259a8ccec56367e9b17 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Wed, 6 Feb 2019 05:51:10 -0800
Subject: [PATCH 1982/2345] Migrate tests to specify a comparison computation
 for Sort.

Add a helper method to hlo_creation_utils which constructs a Sort op with
a simple less-than comparison computation. Use this in tests that directly
create a Sort HloInstruction.

PiperOrigin-RevId: 232660377
---
 tensorflow/compiler/xla/service/BUILD         |  4 +++
 .../xla/service/algebraic_simplifier_test.cc  | 15 ++++----
 .../service/bfloat16_normalization_test.cc    | 13 ++++---
 .../xla/service/hlo_creation_utils.cc         | 35 +++++++++++++++++++
 .../compiler/xla/service/hlo_creation_utils.h | 12 +++++++
 .../xla/service/hlo_dataflow_analysis_test.cc | 20 +++++++----
 .../service/tuple_points_to_analysis_test.cc  | 20 +++++++----
 7 files changed, 95 insertions(+), 24 deletions(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index e5630decd3..a0cdf342b2 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -114,6 +114,7 @@ tf_cc_test(
         ":bfloat16_normalization",
         ":bfloat16_support",
         ":hlo",
+        ":hlo_creation_utils",
         ":hlo_verifier",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:status_macros",
@@ -1601,6 +1602,7 @@ tf_cc_test(
         ":algebraic_simplifier",
         ":hlo",
         ":hlo_casting_utils",
+        ":hlo_creation_utils",
         ":hlo_matchers",
         ":hlo_parser",
         ":hlo_pass",
@@ -2305,6 +2307,7 @@ tf_cc_test(
     srcs = ["hlo_dataflow_analysis_test.cc"],
     deps = [
         ":hlo",
+        ":hlo_creation_utils",
         ":hlo_dataflow_analysis",
         ":hlo_graph_dumper",
         ":hlo_matchers",
@@ -2475,6 +2478,7 @@ tf_cc_test(
     srcs = ["tuple_points_to_analysis_test.cc"],
     deps = [
         ":hlo",
+        ":hlo_creation_utils",
         ":hlo_matchers",
         ":instruction_fusion",
         ":tuple_points_to_analysis",
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index f55a1886b8..46d600600f 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -25,6 +25,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/service/hlo_casting_utils.h"
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/service/hlo_creation_utils.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_instructions.h"
 #include "tensorflow/compiler/xla/service/hlo_matchers.h"
@@ -2749,12 +2750,13 @@ TEST_F(AlgebraicSimplifierTest, SliceOfReshapeUnchanged) {
 
 TEST_F(AlgebraicSimplifierTest, RemoveNoopSort) {
   auto builder = HloComputation::Builder(TestName());
+  auto module = CreateNewVerifiedModule();
 
   Shape keys_shape = ShapeUtil::MakeShape(F32, {1});
   auto keys = builder.AddInstruction(
       HloInstruction::CreateParameter(0, keys_shape, "keys"));
-  builder.AddInstruction(HloInstruction::CreateSort(keys_shape, 0, keys));
-  auto module = CreateNewVerifiedModule();
+  TF_ASSERT_OK(
+      MakeSortHlo(keys_shape, {keys}, 0, &builder, module.get()).status());
   HloComputation* computation = module->AddEntryComputation(builder.Build());
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
@@ -2763,6 +2765,7 @@ TEST_F(AlgebraicSimplifierTest, RemoveNoopSort) {
 
 TEST_F(AlgebraicSimplifierTest, ReplaceEffectiveScalarKeyValueSortWithTuple) {
   auto builder = HloComputation::Builder(TestName());
+  auto module = CreateNewVerifiedModule();
 
   Shape keys_shape = ShapeUtil::MakeShape(F32, {5, 0});
   Shape values_shape = ShapeUtil::MakeShape(S32, {5, 0});
@@ -2772,10 +2775,10 @@ TEST_F(AlgebraicSimplifierTest, ReplaceEffectiveScalarKeyValueSortWithTuple) {
       HloInstruction::CreateParameter(1, values_shape, "values0"));
   auto values1 = builder.AddInstruction(
       HloInstruction::CreateParameter(2, values_shape, "values1"));
-  builder.AddInstruction(HloInstruction::CreateSort(
-      ShapeUtil::MakeTupleShape({keys_shape, values_shape, values_shape}), 0,
-      keys, {values0, values1}));
-  auto module = CreateNewVerifiedModule();
+  TF_ASSERT_OK(MakeSortHlo(ShapeUtil::MakeTupleShape(
+                               {keys_shape, values_shape, values_shape}),
+                           {keys, values0, values1}, 0, &builder, module.get())
+                   .status());
   HloComputation* computation = module->AddEntryComputation(builder.Build());
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
diff --git a/tensorflow/compiler/xla/service/bfloat16_normalization_test.cc b/tensorflow/compiler/xla/service/bfloat16_normalization_test.cc
index 551ac4be73..2591ff602c 100644
--- a/tensorflow/compiler/xla/service/bfloat16_normalization_test.cc
+++ b/tensorflow/compiler/xla/service/bfloat16_normalization_test.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/bfloat16_normalization.h"
 #include "tensorflow/compiler/xla/service/bfloat16_support.h"
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/service/hlo_creation_utils.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
@@ -282,8 +283,10 @@ TEST_F(BFloat16NormalizationTest, ResolveMixedPrecisionTupleSort) {
   HloInstruction* value = builder.AddInstruction(
       HloInstruction::CreateParameter(1, s32_shape, "value"));
 
-  HloInstruction* sort = builder.AddInstruction(HloInstruction::CreateSort(
-      ShapeUtil::MakeTupleShape({bf16_shape, s32_shape}), 0, key, {value}));
+  TF_ASSERT_OK_AND_ASSIGN(
+      auto* sort,
+      MakeSortHlo(ShapeUtil::MakeTupleShape({bf16_shape, s32_shape}),
+                  {key, value}, 0, &builder, module.get()));
   HloInstruction* gte = builder.AddInstruction(
       HloInstruction::CreateGetTupleElement(bf16_shape, sort, 0));
 
@@ -308,8 +311,10 @@ TEST_F(BFloat16NormalizationTest, ResolveMixedPrecisionTupleSortRoot) {
   HloInstruction* value = builder.AddInstruction(
       HloInstruction::CreateParameter(1, bf16_shape, "value"));
 
-  HloInstruction* sort = builder.AddInstruction(HloInstruction::CreateSort(
-      ShapeUtil::MakeTupleShape({bf16_shape, bf16_shape}), 0, key, {value}));
+  TF_ASSERT_OK_AND_ASSIGN(
+      auto* sort,
+      MakeSortHlo(ShapeUtil::MakeTupleShape({bf16_shape, f32_shape}),
+                  {key, value}, 0, &builder, module.get()));
 
   auto computation = module->AddEntryComputation(builder.Build());
 
diff --git a/tensorflow/compiler/xla/service/hlo_creation_utils.cc b/tensorflow/compiler/xla/service/hlo_creation_utils.cc
index d56f673455..5d3e11f7c4 100644
--- a/tensorflow/compiler/xla/service/hlo_creation_utils.cc
+++ b/tensorflow/compiler/xla/service/hlo_creation_utils.cc
@@ -268,6 +268,41 @@ StatusOr<HloInstruction*> MakeSelectHlo(HloInstruction* pred,
       select_shape, HloOpcode::kSelect, pred, on_true, on_false));
 }
 
+StatusOr<HloInstruction*> MakeSortHlo(
+    const Shape& sort_shape, absl::Span<HloInstruction* const> operands,
+    int64 dimension_to_sort, HloComputation::Builder* builder,
+    HloModule* module) {
+  CHECK(!operands.empty()) << "Sort Hlo requires at least one operand.";
+  HloComputation* compare_computation;
+  {
+    auto b = HloComputation::Builder("Sort.Compare");
+    Shape key_scalar_shape =
+        ShapeUtil::MakeShape(operands[0]->shape().element_type(), {});
+    auto lhs = b.AddInstruction(
+        HloInstruction::CreateParameter(0, key_scalar_shape, "p.0.lhs"));
+    auto rhs = b.AddInstruction(
+        HloInstruction::CreateParameter(1, key_scalar_shape, "p.0.rhs"));
+    int parameter_count = 2;
+    for (const auto* operand : operands.subspan(1)) {
+      Shape scalar_shape =
+          ShapeUtil::MakeShape(operand->shape().element_type(), {});
+      b.AddInstruction(HloInstruction::CreateParameter(
+          parameter_count, scalar_shape,
+          StrCat("p.", parameter_count / 2, ".lhs")));
+      ++parameter_count;
+      b.AddInstruction(HloInstruction::CreateParameter(
+          parameter_count, scalar_shape,
+          StrCat("p.", parameter_count / 2, ".rhs")));
+      ++parameter_count;
+    }
+    b.AddInstruction(HloInstruction::CreateBinary(
+        ShapeUtil::MakeShape(PRED, {}), HloOpcode::kLt, lhs, rhs));
+    compare_computation = module->AddEmbeddedComputation(b.Build());
+  }
+  return builder->AddInstruction(HloInstruction::CreateSort(
+      sort_shape, dimension_to_sort, operands, compare_computation));
+}
+
 StatusOr<HloInstruction*> CollapseFirstNDims(HloInstruction* operand, int64 n) {
   CHECK_GT(n, 0);
 
diff --git a/tensorflow/compiler/xla/service/hlo_creation_utils.h b/tensorflow/compiler/xla/service/hlo_creation_utils.h
index 1c3174e9c8..80f58f68d3 100644
--- a/tensorflow/compiler/xla/service/hlo_creation_utils.h
+++ b/tensorflow/compiler/xla/service/hlo_creation_utils.h
@@ -123,6 +123,18 @@ StatusOr<HloInstruction*> MakeSelectHlo(HloInstruction* pred,
                                         HloInstruction* on_true,
                                         HloInstruction* on_false);
 
+// Creates a Sort HLO instruction and adds it to the computation containing the
+// operands. All operands must be in the same computation. Also creates a
+// default compare sub-computation which sorts the first operand into ascending
+// order.
+// Note that this default compare sub-computation does not have special handling
+// for floating point values and thus can result in undefined behavior in the
+// presence of NaN values.
+StatusOr<HloInstruction*> MakeSortHlo(
+    const Shape& sort_shape, absl::Span<HloInstruction* const> operands,
+    int64 dimension_to_sort, HloComputation::Builder* builder,
+    HloModule* module);
+
 // Creates an R1 Constant HLO instruction of the given PrimitiveType with the
 // given values and adds it to the given computation.
 template <typename NativeT>
diff --git a/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc b/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc
index 4a7c4963b7..e3059e02cf 100644
--- a/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc
@@ -17,6 +17,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/service/hlo_creation_utils.h"
 #include "tensorflow/compiler/xla/service/hlo_graph_dumper.h"
 #include "tensorflow/compiler/xla/service/hlo_matchers.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
@@ -2356,14 +2357,16 @@ TEST_F(CanShareOperandBufferWithUserTest, ScatterCanShare) {
 
 TEST_F(CanShareOperandBufferWithUserTest, SortCanShare) {
   auto builder = HloComputation::Builder(TestName());
+  module_ = CreateNewVerifiedModule();
 
   Shape keys_shape = ShapeUtil::MakeShape(F32, {8});
   auto keys = builder.AddInstruction(
       HloInstruction::CreateParameter(0, keys_shape, "keys"));
-  auto sort =
-      builder.AddInstruction(HloInstruction::CreateSort(keys_shape, 0, keys));
+  TF_ASSERT_OK_AND_ASSIGN(
+      auto* sort, MakeSortHlo(keys_shape, {keys}, -1, &builder, module_.get()));
 
-  BuildModuleAndRunAnalysis(builder.Build());
+  computation_ = module_->AddEntryComputation(builder.Build());
+  RunAnalysis();
 
   EXPECT_TRUE(
       dataflow_analysis_->CanShareOperandBufferWithUser(keys, {}, sort, {}));
@@ -2371,6 +2374,7 @@ TEST_F(CanShareOperandBufferWithUserTest, SortCanShare) {
 
 TEST_F(CanShareOperandBufferWithUserTest, SortCanShareWithTupleUser) {
   auto builder = HloComputation::Builder(TestName());
+  module_ = CreateNewVerifiedModule();
 
   Shape keys_shape = ShapeUtil::MakeShape(F32, {8});
   Shape values_shape = ShapeUtil::MakeShape(F32, {8});
@@ -2378,11 +2382,13 @@ TEST_F(CanShareOperandBufferWithUserTest, SortCanShareWithTupleUser) {
       HloInstruction::CreateParameter(0, keys_shape, "keys"));
   auto values = builder.AddInstruction(
       HloInstruction::CreateParameter(1, values_shape, "values"));
-  auto sort = builder.AddInstruction(HloInstruction::CreateSort(
-      ShapeUtil::MakeTupleShape({keys_shape, values_shape}), 0, keys,
-      {values}));
+  TF_ASSERT_OK_AND_ASSIGN(
+      auto* sort,
+      MakeSortHlo(ShapeUtil::MakeTupleShape({keys_shape, values_shape}),
+                  {keys, values}, 0, &builder, module_.get()));
 
-  BuildModuleAndRunAnalysis(builder.Build());
+  computation_ = module_->AddEntryComputation(builder.Build());
+  RunAnalysis();
 
   // The buffer for the keys can be shared with the first tuple entry.
   EXPECT_TRUE(
diff --git a/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc b/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc
index fd5759e442..5516026139 100644
--- a/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc
+++ b/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include <memory>
 
 #include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/compiler/xla/service/hlo_creation_utils.h"
 #include "tensorflow/compiler/xla/service/hlo_matchers.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
 #include "tensorflow/compiler/xla/service/instruction_fusion.h"
@@ -1065,14 +1066,16 @@ TEST_F(CanShareOperandBufferWithUserTest, ScatterCanShare) {
 
 TEST_F(CanShareOperandBufferWithUserTest, SortCanShare) {
   auto builder = HloComputation::Builder(TestName());
+  module_ = CreateNewVerifiedModule();
 
   Shape keys_shape = ShapeUtil::MakeShape(F32, {8});
   auto keys = builder.AddInstruction(
       HloInstruction::CreateParameter(0, keys_shape, "keys"));
-  auto sort =
-      builder.AddInstruction(HloInstruction::CreateSort(keys_shape, 0, keys));
+  TF_ASSERT_OK_AND_ASSIGN(
+      auto* sort, MakeSortHlo(keys_shape, {keys}, 0, &builder, module_.get()));
 
-  BuildModuleAndRunAnalysis(builder.Build());
+  computation_ = module_->AddEntryComputation(builder.Build());
+  RunAnalysis();
 
   EXPECT_TRUE(
       points_to_analysis_->CanShareOperandBufferWithUser(keys, {}, sort, {}));
@@ -1080,6 +1083,7 @@ TEST_F(CanShareOperandBufferWithUserTest, SortCanShare) {
 
 TEST_F(CanShareOperandBufferWithUserTest, SortCanShareWithTupleUser) {
   auto builder = HloComputation::Builder(TestName());
+  module_ = CreateNewVerifiedModule();
 
   Shape keys_shape = ShapeUtil::MakeShape(F32, {8});
   Shape values_shape = ShapeUtil::MakeShape(F32, {8});
@@ -1087,11 +1091,13 @@ TEST_F(CanShareOperandBufferWithUserTest, SortCanShareWithTupleUser) {
       HloInstruction::CreateParameter(0, keys_shape, "keys"));
   auto values = builder.AddInstruction(
       HloInstruction::CreateParameter(1, values_shape, "values"));
-  auto sort = builder.AddInstruction(HloInstruction::CreateSort(
-      ShapeUtil::MakeTupleShape({keys_shape, values_shape}), 0, keys,
-      {values}));
+  TF_ASSERT_OK_AND_ASSIGN(
+      auto* sort,
+      MakeSortHlo(ShapeUtil::MakeTupleShape({keys_shape, values_shape}),
+                  {keys, values}, 0, &builder, module_.get()));
 
-  BuildModuleAndRunAnalysis(builder.Build());
+  computation_ = module_->AddEntryComputation(builder.Build());
+  RunAnalysis();
 
   // The buffer for the keys can be shared with the first tuple entry.
   EXPECT_TRUE(
-- 
GitLab


From f85b73ea37eb4cfba9e895c6a305a7b0a4493cb4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 6 Feb 2019 07:00:45 -0800
Subject: [PATCH 1983/2345] Cleanup: remove unused captures (of "this"), remove
 redundant get(), pass ints by value

PiperOrigin-RevId: 232668054
---
 .../common_runtime/buf_rendezvous_test.cc     | 24 +++++++++----------
 .../collective_executor_mgr_test.cc           | 12 +++++-----
 .../collective_param_resolver_local.cc        |  2 +-
 .../collective_param_resolver_local_test.cc   |  2 +-
 .../collective_rma_local_test.cc              |  8 +++----
 .../device_resolver_local_test.cc             |  4 ++--
 .../common_runtime/gpu/gpu_process_state.cc   |  2 +-
 .../hierarchical_tree_broadcaster_test.cc     |  2 +-
 8 files changed, 28 insertions(+), 28 deletions(-)

diff --git a/tensorflow/core/common_runtime/buf_rendezvous_test.cc b/tensorflow/core/common_runtime/buf_rendezvous_test.cc
index 0e798235bf..7621787dec 100644
--- a/tensorflow/core/common_runtime/buf_rendezvous_test.cc
+++ b/tensorflow/core/common_runtime/buf_rendezvous_test.cc
@@ -109,7 +109,7 @@ TEST_F(BufRendezvousTest, CorrectUseConsumerFirst) {
 TEST_F(BufRendezvousTest, ErrorDuplicatePut) {
   bool prod_callback_called = false;
   br_->ProvideBuf("key0", fake_dev_ptr_, fake_dev_ctx_, &a_, aa_,
-                  [this, &prod_callback_called](const Status& s) {
+                  [&prod_callback_called](const Status& s) {
                     prod_callback_called = true;
                   });
   Status bad_status;
@@ -129,11 +129,11 @@ TEST_F(BufRendezvousTest, ErrorDuplicatePut) {
 
 TEST_F(BufRendezvousTest, ErrorDeleteNonEmpty) {
   Status cons_status;
-  br_->ConsumeBuf(
-      "key0", [this, &cons_status](const Status& s, BufRendezvous::Hook* h) {
-        cons_status = s;
-        EXPECT_EQ(h, nullptr);
-      });
+  br_->ConsumeBuf("key0",
+                  [&cons_status](const Status& s, BufRendezvous::Hook* h) {
+                    cons_status = s;
+                    EXPECT_EQ(h, nullptr);
+                  });
   EXPECT_TRUE(cons_status.ok());
   br_.reset();
   EXPECT_FALSE(cons_status.ok());
@@ -146,13 +146,13 @@ TEST_F(BufRendezvousTest, AbortNonEmpty) {
   Status prod_status;
   Notification prod_note;
   Notification cons_note;
-  br_->ConsumeBuf("key0", [this, &cons_note, &cons_status](
-                              const Status& s, BufRendezvous::Hook* h) {
+  br_->ConsumeBuf("key0", [&cons_note, &cons_status](const Status& s,
+                                                     BufRendezvous::Hook* h) {
     cons_status = s;
     cons_note.Notify();
   });
   br_->ProvideBuf("key1", fake_dev_ptr_, fake_dev_ctx_, &a_, aa_,
-                  [this, &prod_note, &prod_status](const Status& s) {
+                  [&prod_note, &prod_status](const Status& s) {
                     prod_status = s;
                     prod_note.Notify();
                   });
@@ -175,13 +175,13 @@ TEST_F(BufRendezvousTest, UseAfterAbort) {
   Status prod_status;
   Notification prod_note;
   Notification cons_note;
-  br_->ConsumeBuf("key0", [this, &cons_note, &cons_status](
-                              const Status& s, BufRendezvous::Hook* h) {
+  br_->ConsumeBuf("key0", [&cons_note, &cons_status](const Status& s,
+                                                     BufRendezvous::Hook* h) {
     cons_status = s;
     cons_note.Notify();
   });
   br_->ProvideBuf("key1", fake_dev_ptr_, fake_dev_ctx_, &a_, aa_,
-                  [this, &prod_note, &prod_status](const Status& s) {
+                  [&prod_note, &prod_status](const Status& s) {
                     prod_status = s;
                     prod_note.Notify();
                   });
diff --git a/tensorflow/core/common_runtime/collective_executor_mgr_test.cc b/tensorflow/core/common_runtime/collective_executor_mgr_test.cc
index 80d4166aa6..3eef5ed0a0 100644
--- a/tensorflow/core/common_runtime/collective_executor_mgr_test.cc
+++ b/tensorflow/core/common_runtime/collective_executor_mgr_test.cc
@@ -73,11 +73,11 @@ TEST_F(CollectiveExecutorMgrTest, StepSequenceRelated) {
   EXPECT_EQ(CollectiveExecutor::kInvalidId, cme_->NextStepId(123));
   Notification ss_note;
   Status ss_status;
-  cme_->RefreshStepIdSequenceAsync(
-      123, [this, &ss_status, &ss_note](const Status& s) {
-        ss_status = s;
-        ss_note.Notify();
-      });
+  cme_->RefreshStepIdSequenceAsync(123,
+                                   [&ss_status, &ss_note](const Status& s) {
+                                     ss_status = s;
+                                     ss_note.Notify();
+                                   });
   ss_note.WaitForNotification();
   EXPECT_FALSE(ss_status.ok());
   EXPECT_EQ(ss_status.error_message(),
@@ -87,7 +87,7 @@ TEST_F(CollectiveExecutorMgrTest, StepSequenceRelated) {
   GetStepSequenceRequest* req = nullptr;
   GetStepSequenceResponse* resp = nullptr;
   cme_->GetStepSequenceAsync(req, resp,
-                             [this, &gs_status, &gs_note](const Status& s) {
+                             [&gs_status, &gs_note](const Status& s) {
                                gs_status = s;
                                gs_note.Notify();
                              });
diff --git a/tensorflow/core/common_runtime/collective_param_resolver_local.cc b/tensorflow/core/common_runtime/collective_param_resolver_local.cc
index e286ba2a37..5bb2a7df53 100644
--- a/tensorflow/core/common_runtime/collective_param_resolver_local.cc
+++ b/tensorflow/core/common_runtime/collective_param_resolver_local.cc
@@ -360,7 +360,7 @@ void SortDevicesAndTasks(CollectiveParams* cp) {
   for (int i = 0; i < perm.size(); ++i) {
     perm[i] = i;
   }
-  std::sort(perm.begin(), perm.end(), [cp](const int& a, const int& b) {
+  std::sort(perm.begin(), perm.end(), [cp](int a, int b) {
     return cp->instance.device_names[a] < cp->instance.device_names[b];
   });
   std::vector<string> new_devs;
diff --git a/tensorflow/core/common_runtime/collective_param_resolver_local_test.cc b/tensorflow/core/common_runtime/collective_param_resolver_local_test.cc
index ab91eaba2f..70eb9f8081 100644
--- a/tensorflow/core/common_runtime/collective_param_resolver_local_test.cc
+++ b/tensorflow/core/common_runtime/collective_param_resolver_local_test.cc
@@ -175,7 +175,7 @@ TEST_F(CollectiveParamResolverLocalTest, CompleteParamsReduction1Task) {
     Env::Default()->SchedClosure([this, i, cp, &note, &statuses]() {
       prl_->CompleteParamsAsync(cp->instance.device_names[0], cp,
                                 nullptr /*CancellationManager*/,
-                                [this, &statuses, &note, i](const Status& s) {
+                                [&statuses, &note, i](const Status& s) {
                                   statuses[i] = s;
                                   note[i].Notify();
                                 });
diff --git a/tensorflow/core/common_runtime/collective_rma_local_test.cc b/tensorflow/core/common_runtime/collective_rma_local_test.cc
index 6979f4e63b..2e9d8cd394 100644
--- a/tensorflow/core/common_runtime/collective_rma_local_test.cc
+++ b/tensorflow/core/common_runtime/collective_rma_local_test.cc
@@ -70,7 +70,7 @@ TEST_F(CollectiveRemoteAccessLocalTest, PostRecvCPU0) {
                      "key_0", cpu0 /*to_device*/, nullptr /*to_device_ctx*/,
                      attr /*to_alloc_attr*/, &sink_tensor, dev_locality,
                      0 /*stream_index*/,
-                     [this, &recv_note, &recv_status](const Status& s) {
+                     [&recv_note, &recv_status](const Status& s) {
                        recv_status = s;
                        recv_note.Notify();
                      });
@@ -85,7 +85,7 @@ TEST_F(CollectiveRemoteAccessLocalTest, PostRecvCPU0) {
   rma_->PostToPeer(kTaskName + "/device:CPU:0", kTaskName, "key_0",
                    cpu0 /*from_device*/, nullptr /*from_device_ctx*/,
                    attr /*to_alloc_attr*/, &source_tensor, dev_locality,
-                   [this, &send_note, &send_status](const Status& s) {
+                   [&send_note, &send_status](const Status& s) {
                      send_status = s;
                      send_note.Notify();
                    });
@@ -113,7 +113,7 @@ TEST_F(CollectiveRemoteAccessLocalTest, PostRecvCPU1_2) {
                      "key_0", cpu2 /*to_device*/, nullptr /*to_device_ctx*/,
                      attr /*to_alloc_attr*/, &sink_tensor, dev_locality,
                      0 /*stream_index*/,
-                     [this, &recv_note, &recv_status](const Status& s) {
+                     [&recv_note, &recv_status](const Status& s) {
                        recv_status = s;
                        recv_note.Notify();
                      });
@@ -130,7 +130,7 @@ TEST_F(CollectiveRemoteAccessLocalTest, PostRecvCPU1_2) {
   rma_->PostToPeer(kTaskName + "/device:CPU:2", kTaskName, "key_0",
                    cpu1 /*from_device*/, nullptr /*from_device_ctx*/,
                    attr /*to_alloc_attr*/, &source_tensor, dev_locality,
-                   [this, &send_note, &send_status](const Status& s) {
+                   [&send_note, &send_status](const Status& s) {
                      send_status = s;
                      send_note.Notify();
                    });
diff --git a/tensorflow/core/common_runtime/device_resolver_local_test.cc b/tensorflow/core/common_runtime/device_resolver_local_test.cc
index 54f1119e13..62e82bcc5a 100644
--- a/tensorflow/core/common_runtime/device_resolver_local_test.cc
+++ b/tensorflow/core/common_runtime/device_resolver_local_test.cc
@@ -56,7 +56,7 @@ TEST_F(DeviceResolverLocalTest, GetDeviceLocalitiesKnown) {
   Notification note;
   Status status;
   drl_->GetDeviceLocalitiesAsync(cp.instance, &localities,
-                                 [this, &note, &status](const Status& s) {
+                                 [&note, &status](const Status& s) {
                                    status = s;
                                    note.Notify();
                                  });
@@ -74,7 +74,7 @@ TEST_F(DeviceResolverLocalTest, GetDeviceLocalitiesUnknown) {
   Notification note;
   Status status;
   drl_->GetDeviceLocalitiesAsync(cp.instance, &localities,
-                                 [this, &note, &status](const Status& s) {
+                                 [&note, &status](const Status& s) {
                                    status = s;
                                    note.Notify();
                                  });
diff --git a/tensorflow/core/common_runtime/gpu/gpu_process_state.cc b/tensorflow/core/common_runtime/gpu/gpu_process_state.cc
index 8167cfb9d7..909f14a17b 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_process_state.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_process_state.cc
@@ -90,7 +90,7 @@ Allocator* GPUProcessState::GetGPUAllocator(const GPUOptions& options,
   }
 
   AllocatorParts& allocator_parts = gpu_allocators_[tf_gpu_id.value()];
-  if (allocator_parts.allocator.get() == nullptr) {
+  if (allocator_parts.allocator == nullptr) {
     // Validate allocator types.
     if (!allocator_type.empty() && allocator_type != "BFC") {
       LOG(ERROR) << "Invalid allocator type: " << allocator_type;
diff --git a/tensorflow/core/common_runtime/hierarchical_tree_broadcaster_test.cc b/tensorflow/core/common_runtime/hierarchical_tree_broadcaster_test.cc
index f0656ff533..12af4a8201 100644
--- a/tensorflow/core/common_runtime/hierarchical_tree_broadcaster_test.cc
+++ b/tensorflow/core/common_runtime/hierarchical_tree_broadcaster_test.cc
@@ -616,7 +616,7 @@ class HierarchicalTreeBroadcasterTest : public ::testing::Test {
         auto* dev_info = device_->tensorflow_gpu_device_info();
         CHECK(dev_info);
         dev_info->default_context->CopyCPUTensorToDevice(
-            &cpu_tensor, device_, &tensor_, [this, &notification](Status s) {
+            &cpu_tensor, device_, &tensor_, [&notification](Status s) {
               TF_CHECK_OK(s);
               notification.Notify();
             });
-- 
GitLab


From 4232ed91430498d7f91184995a0a47a1b6c94945 Mon Sep 17 00:00:00 2001
From: James Ring <sjr@google.com>
Date: Wed, 6 Feb 2019 07:02:45 -0800
Subject: [PATCH 1984/2345] introduce //tensorflow/c:c_api_no_xla

This target should contain only the compile-only dependencies of the
TensorFlow C API. This prevents users from inadvertently including
runtime dependencies that they don't necessarily need (i.e. XLA).

PiperOrigin-RevId: 232668346
---
 tensorflow/c/BUILD | 35 ++++++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 13 deletions(-)

diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index 6423b4bb20..10a2992730 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -67,6 +67,23 @@ tf_cuda_library(
 
 tf_cuda_library(
     name = "c_api",
+    hdrs = ["c_api.h"],
+    copts = tf_copts(),
+    visibility = ["//visibility:public"],
+    deps = [
+        ":c_api_no_xla",
+        ":c_api_internal",
+    ] + select({
+        "//tensorflow:with_xla_support": [
+            "//tensorflow/compiler/tf2xla:xla_compiler",
+            "//tensorflow/compiler/jit",
+        ],
+        "//conditions:default": [],
+    }),
+)
+
+tf_cuda_library(
+    name = "c_api_no_xla",
     srcs = [
         "c_api.cc",
         "c_api_function.cc",
@@ -75,14 +92,12 @@ tf_cuda_library(
         "c_api.h",
     ],
     copts = tf_copts(),
-    visibility = ["//visibility:public"],
-    deps = select({
+    visibility = ["//tensorflow/c:__subpackages__"],
+    deps = [":c_api_internal"] + select({
         "//tensorflow:android": [
-            ":c_api_internal",
             "//tensorflow/core:android_tensorflow_lib_lite",
         ],
         "//conditions:default": [
-            ":c_api_internal",
             "//tensorflow/cc/saved_model:loader_lite",
             "//tensorflow/cc:gradients",
             "//tensorflow/cc:ops",
@@ -99,12 +114,6 @@ tf_cuda_library(
             "//tensorflow/core/distributed_runtime:server_lib",
             "//tensorflow/core/kernels:logging_ops",
         ],
-    }) + select({
-        "//tensorflow:with_xla_support": [
-            "//tensorflow/compiler/tf2xla:xla_compiler",
-            "//tensorflow/compiler/jit",
-        ],
-        "//conditions:default": [],
     }),
 )
 
@@ -157,8 +166,8 @@ tf_cuda_library(
     hdrs = ["tf_status_helper.h"],
     visibility = ["//visibility:public"],
     deps = [
-        ":c_api",
         ":c_api_internal",
+        ":c_api_no_xla",
         "//tensorflow/core:lib",
     ],
 )
@@ -214,13 +223,13 @@ tf_cuda_library(
     visibility = ["//visibility:public"],
     deps = select({
         "//tensorflow:android": [
-            ":c_api",
+            ":c_api_no_xla",
             ":c_api_internal",
             ":tf_status_helper",
             "//tensorflow/core:android_tensorflow_lib_lite",
         ],
         "//conditions:default": [
-            ":c_api",
+            ":c_api_no_xla",
             ":c_api_internal",
             ":tf_status_helper",
             "//tensorflow/core:framework",
-- 
GitLab


From 9f6937e0d1d60ff60238c8c949438fc8209d2da7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 6 Feb 2019 07:07:57 -0800
Subject: [PATCH 1985/2345] [TF:XLA] Attach useful information to disabled XLA
 tests.

PiperOrigin-RevId: 232669090
---
 tensorflow/python/framework/function_test.py              | 1 -
 tensorflow/python/framework/test_util.py                  | 2 +-
 tensorflow/python/kernel_tests/benchmark_test.py          | 2 +-
 tensorflow/python/kernel_tests/check_ops_test.py          | 8 ++++----
 tensorflow/python/kernel_tests/cholesky_op_test.py        | 4 +++-
 tensorflow/python/kernel_tests/concat_op_test.py          | 5 +----
 .../python/kernel_tests/control_flow_ops_py_test.py       | 2 --
 tensorflow/python/kernel_tests/depthtospace_op_test.py    | 2 +-
 tensorflow/python/kernel_tests/diag_op_test.py            | 4 ++--
 tensorflow/python/kernel_tests/functional_ops_test.py     | 5 +----
 tensorflow/python/kernel_tests/gather_nd_op_test.py       | 4 +---
 tensorflow/python/kernel_tests/map_fn_test.py             | 1 -
 tensorflow/python/kernel_tests/pool_test.py               | 1 -
 tensorflow/python/kernel_tests/pooling_ops_test.py        | 4 ++--
 tensorflow/python/kernel_tests/reduction_ops_test_big.py  | 3 ++-
 tensorflow/python/kernel_tests/relu_op_test.py            | 2 +-
 tensorflow/python/kernel_tests/scan_ops_test.py           | 3 +--
 tensorflow/python/kernel_tests/scatter_nd_ops_test.py     | 2 +-
 tensorflow/python/kernel_tests/spacetodepth_op_test.py    | 2 +-
 tensorflow/python/kernel_tests/split_op_test.py           | 2 +-
 20 files changed, 24 insertions(+), 35 deletions(-)

diff --git a/tensorflow/python/framework/function_test.py b/tensorflow/python/framework/function_test.py
index f541a4fb4a..7df76efde5 100644
--- a/tensorflow/python/framework/function_test.py
+++ b/tensorflow/python/framework/function_test.py
@@ -284,7 +284,6 @@ class FunctionTest(test.TestCase):
         out, = sess.run(dlogits, {logits: x, labels: y})
       self.assertAllClose(out, np.exp(prob - y))
 
-  @test_util.disable_xla("This test never passed for XLA")
   def testCustomGradientError(self):
     dtype = dtypes.float32
 
diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index d27572f056..1d267830ff 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -1460,7 +1460,7 @@ def disable_all_xla(description):
       value = getattr(cls, name)
       if callable(value) and name.startswith(
           "test") and not name == "test_session":
-        setattr(cls, name, base_decorator(value))
+        setattr(cls, name, base_decorator(description)(value))
     return cls
 
   return disable_all_impl
diff --git a/tensorflow/python/kernel_tests/benchmark_test.py b/tensorflow/python/kernel_tests/benchmark_test.py
index a91f96cf95..3fa2054847 100644
--- a/tensorflow/python/kernel_tests/benchmark_test.py
+++ b/tensorflow/python/kernel_tests/benchmark_test.py
@@ -126,7 +126,7 @@ class BenchmarkTest(test.TestCase):
     self.assertFalse(_ran_somebenchmark_2[0])
     self.assertFalse(_ran_somebenchmark_but_shouldnt[0])
 
-  @test_util.disable_xla("This test never passed for XLA")
+  @test_util.disable_xla("b/123744455")  # GPU memory is incorrect
   def testReportingBenchmark(self):
     tempdir = test.get_temp_dir()
     try:
diff --git a/tensorflow/python/kernel_tests/check_ops_test.py b/tensorflow/python/kernel_tests/check_ops_test.py
index d5f3696a9d..7d00919cc8 100644
--- a/tensorflow/python/kernel_tests/check_ops_test.py
+++ b/tensorflow/python/kernel_tests/check_ops_test.py
@@ -889,8 +889,8 @@ class EnsureShapeTest(test.TestCase):
 
   # Dynamic shape check
   @test_util.run_deprecated_v1
-  @test_util.disable_xla("This test never passed for XLA"
-                        )  # Dynamic shapes not supported now with XLA
+  @test_util.disable_xla(
+      "b/123337890")  # Dynamic shapes not supported now with XLA
   def testEnsuresDynamicShape_RaisesError(self):
     placeholder = array_ops.placeholder(dtypes.int32)
     derived = math_ops.divide(placeholder, 3, name="MyDivide")
@@ -904,8 +904,8 @@ class EnsureShapeTest(test.TestCase):
         sess.run(derived, feed_dict={placeholder: feed_val})
 
   @test_util.run_deprecated_v1
-  @test_util.disable_xla("This test never passed for XLA"
-                        )  # Dynamic shapes not supported now with XLA
+  @test_util.disable_xla(
+      "b/123337890")  # Dynamic shapes not supported now with XLA
   def testEnsuresDynamicShape_RaisesErrorDimUnknown(self):
     placeholder = array_ops.placeholder(dtypes.int32)
     derived = placeholder / 3
diff --git a/tensorflow/python/kernel_tests/cholesky_op_test.py b/tensorflow/python/kernel_tests/cholesky_op_test.py
index abb71a672c..2305c0b568 100644
--- a/tensorflow/python/kernel_tests/cholesky_op_test.py
+++ b/tensorflow/python/kernel_tests/cholesky_op_test.py
@@ -163,7 +163,9 @@ class CholeskyOpTest(test.TestCase):
     with self.assertRaises(ValueError):
       linalg_ops.cholesky(tensor3)
 
-  @test_util.disable_xla("This test never passed for XLA")  # all nan on XLA
+  # The below invalid Cholesky call returns an error with TF Classic and just
+  # returns NaNs with XLA.
+  @test_util.disable_xla("b/123337890")
   def testNotInvertibleCPU(self):
     # The input should be invertible.
     with self.session(use_gpu=True):
diff --git a/tensorflow/python/kernel_tests/concat_op_test.py b/tensorflow/python/kernel_tests/concat_op_test.py
index a968b06127..7e37785344 100644
--- a/tensorflow/python/kernel_tests/concat_op_test.py
+++ b/tensorflow/python/kernel_tests/concat_op_test.py
@@ -33,7 +33,6 @@ from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 
 
-@test_util.disable_all_xla("This test never passed for XLA")
 class ConcatOpTest(test.TestCase):
 
   @test_util.run_deprecated_v1
@@ -642,7 +641,6 @@ class ConcatOpTest(test.TestCase):
         self.assertAllEqual([[1, 2, 3, 7, 8, 9], [4, 5, 6, 10, 11, 12]], output)
 
 
-@test_util.disable_all_xla("This test never passed for XLA")
 class ConcatOffsetTest(test.TestCase):
 
   def testBasic(self):
@@ -686,8 +684,7 @@ class ConcatOffsetTest(test.TestCase):
       self.evaluate(off)
 
   @test_util.run_deprecated_v1
-  @test_util.disable_xla(
-      "This test never passed for XLA")  # Different error message on XLA
+  @test_util.disable_xla("b/123337890")  # Error messages differ
   def testSizeMismatch(self):
     cdim = constant_op.constant(1, dtypes.int32)
     s0 = constant_op.constant([2, 3, 5], dtypes.int32)
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 81a530a263..a073bb3f22 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -2769,8 +2769,6 @@ class ControlFlowTest(test.TestCase):
       self.assertEqual(i_val, 3)
       self.assertAllClose(x_val, 1.0)
 
-  @test_util.disable_xla("This test never passed for XLA"
-                        )  # Resource variable issue for ControlFlowV2
   @test_util.run_gpu_only
   def testGpuResourceAccess(self):
     with ops.device(test.gpu_device_name()):
diff --git a/tensorflow/python/kernel_tests/depthtospace_op_test.py b/tensorflow/python/kernel_tests/depthtospace_op_test.py
index 97d3645b61..b7a865cf13 100644
--- a/tensorflow/python/kernel_tests/depthtospace_op_test.py
+++ b/tensorflow/python/kernel_tests/depthtospace_op_test.py
@@ -295,7 +295,7 @@ class DepthToSpaceTest(test.TestCase):
       actual_vals, expected_vals = self.evaluate([actual, expected])
       self.assertTrue(np.array_equal(actual_vals, expected_vals))
 
-  @test_util.disable_xla("This test never passed for XLA")
+  @test_util.disable_xla("b/123553551")  # Unsupported data format
   def testAgainstTranspose(self):
     self.compareToTranspose(3, 2, 3, 1, 2, "NHWC", False)
     self.compareToTranspose(3, 2, 3, 2, 2, "NHWC", False)
diff --git a/tensorflow/python/kernel_tests/diag_op_test.py b/tensorflow/python/kernel_tests/diag_op_test.py
index b8139918c5..0bf48fd228 100644
--- a/tensorflow/python/kernel_tests/diag_op_test.py
+++ b/tensorflow/python/kernel_tests/diag_op_test.py
@@ -65,7 +65,7 @@ class MatrixDiagTest(test.TestCase):
       array_ops.matrix_diag(0)
 
   @test_util.run_deprecated_v1
-  @test_util.disable_xla("This test never passed for XLA")
+  @test_util.disable_xla("b/123337890")  # Error messages differ
   def testInvalidShapeAtEval(self):
     with self.session(use_gpu=True):
       v = array_ops.placeholder(dtype=dtypes_lib.float32)
@@ -270,7 +270,7 @@ class MatrixDiagPartTest(test.TestCase):
       array_ops.matrix_diag_part(0)
 
   @test_util.run_deprecated_v1
-  @test_util.disable_xla("This test never passed for XLA")
+  @test_util.disable_xla("b/123337890")  # Error messages differ
   def testInvalidShapeAtEval(self):
     with self.session(use_gpu=True):
       v = array_ops.placeholder(dtype=dtypes_lib.float32)
diff --git a/tensorflow/python/kernel_tests/functional_ops_test.py b/tensorflow/python/kernel_tests/functional_ops_test.py
index 13f9acbe1b..48e14d0f50 100644
--- a/tensorflow/python/kernel_tests/functional_ops_test.py
+++ b/tensorflow/python/kernel_tests/functional_ops_test.py
@@ -56,7 +56,6 @@ def simple_scoped_fn(a, x):
 
 
 @test_util.with_control_flow_v2
-@test_util.disable_all_xla("This test never passed for XLA")
 class FunctionalOpsTest(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
@@ -660,8 +659,7 @@ class FunctionalOpsTest(test.TestCase):
     self.assertAllEqual(Run(100., True), 5050.)
 
   @test_util.run_v1_only("b/120545219")
-  @test_util.disable_xla(
-      "This test never passed for XLA")  # Different error message
+  @test_util.disable_xla("b/123337890")  # Different error message
   def testWhileError(self):
     for use_gpu in (True, False):
       with ops.Graph().as_default() as g:
@@ -938,7 +936,6 @@ class FunctionalOpsTest(test.TestCase):
 
 # TODO(akshayka): Replace `function.Defun` with tf.contrib.eager.defun` in the
 # below test cases.
-@test_util.disable_all_xla("This test never passed for XLA")
 class PartitionedCallTest(test.TestCase):
 
   @test_util.run_deprecated_v1
diff --git a/tensorflow/python/kernel_tests/gather_nd_op_test.py b/tensorflow/python/kernel_tests/gather_nd_op_test.py
index 76ae2fcb72..ad8376b48c 100644
--- a/tensorflow/python/kernel_tests/gather_nd_op_test.py
+++ b/tensorflow/python/kernel_tests/gather_nd_op_test.py
@@ -34,7 +34,6 @@ from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 
 
-@test_util.disable_all_xla("This test never passed for XLA")
 class GatherNdTest(test.TestCase):
 
   def _testSimpleDtype(self, dtype):
@@ -57,7 +56,7 @@ class GatherNdTest(test.TestCase):
     self._testSimpleDtype("|S")  # byte strings in python2 + 3
 
   @test_util.run_deprecated_v1
-  @test_util.disable_xla("This test never passed for XLA")
+  @test_util.disable_xla("b/123337890")  # Error messages differ
   def testEmptyIndicesAndParamsOKButJustEmptyParamsFails(self):
     with self.session(use_gpu=True):
       params = np.ones((3, 3), dtype=np.float32)
@@ -360,7 +359,6 @@ class GatherNdTest(test.TestCase):
       self.assertAllEqual(expected_grads, ops.convert_to_tensor(grads).eval())
 
 
-@test_util.disable_all_xla("This test never passed for XLA")
 class GatherNdOpBenchmark(test.Benchmark):
 
   def benchmark_gather_nd_op(self):
diff --git a/tensorflow/python/kernel_tests/map_fn_test.py b/tensorflow/python/kernel_tests/map_fn_test.py
index a31db416a2..41d99ea379 100644
--- a/tensorflow/python/kernel_tests/map_fn_test.py
+++ b/tensorflow/python/kernel_tests/map_fn_test.py
@@ -49,7 +49,6 @@ def simple_scoped_fn(a, x):
 
 
 @test_util.with_control_flow_v2
-@test_util.disable_all_xla("This test never passed for XLA")
 class MapFnTest(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
diff --git a/tensorflow/python/kernel_tests/pool_test.py b/tensorflow/python/kernel_tests/pool_test.py
index 367c94dd1e..78e786f01c 100644
--- a/tensorflow/python/kernel_tests/pool_test.py
+++ b/tensorflow/python/kernel_tests/pool_test.py
@@ -303,7 +303,6 @@ class PoolingTest(test.TestCase):
     self.assertLess(err, err_tolerance)
 
   @test_util.run_deprecated_v1
-  @test_util.disable_xla("This test never passed for XLA")  # Much larger error
   def testGradient1D(self):
     with self.session(use_gpu=test.is_gpu_available()):
       for padding in ["SAME", "VALID"]:
diff --git a/tensorflow/python/kernel_tests/pooling_ops_test.py b/tensorflow/python/kernel_tests/pooling_ops_test.py
index 546b406c20..0cd6495d81 100644
--- a/tensorflow/python/kernel_tests/pooling_ops_test.py
+++ b/tensorflow/python/kernel_tests/pooling_ops_test.py
@@ -730,7 +730,7 @@ class PoolingTest(test.TestCase):
         t = nn_ops.max_pool(
             t, ksize=ksize, strides=strides, padding="SAME").eval()
 
-  @test_util.disable_xla("b/123338077")
+  @test_util.disable_xla("b/123338077")  # Passes with XLA
   def testDepthwiseMaxPoolInvalidConfigs(self):
     self._testDepthwiseMaxPoolInvalidConfig(
         [1, 2, 2, 4], [1, 2, 2, 2], [1, 1, 1, 2],
@@ -1821,7 +1821,7 @@ class PoolingTest(test.TestCase):
             padding="SAME")
 
   @test_util.run_deprecated_v1
-  @test_util.disable_xla("This test never passed for XLA")
+  @test_util.disable_xla("b/123337890")  # Error messages differ
   def testOpEdgeCases(self):
     with self.session(use_gpu=test.is_gpu_available()) as sess:
       pool_funcs = [nn_ops.max_pool, nn_ops.avg_pool]
diff --git a/tensorflow/python/kernel_tests/reduction_ops_test_big.py b/tensorflow/python/kernel_tests/reduction_ops_test_big.py
index 1e8524f72a..73bba54a46 100644
--- a/tensorflow/python/kernel_tests/reduction_ops_test_big.py
+++ b/tensorflow/python/kernel_tests/reduction_ops_test_big.py
@@ -21,16 +21,17 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
-
 class BaseReductionTest(test.TestCase):
 
   def _tf_reduce(self, x, reduction_axes, keepdims):
     raise NotImplementedError()
 
 
+@test_util.disable_all_xla("b/123864762")  # Test times out
 class BigReductionTest(BaseReductionTest):
   """Test reductions for sum and boolean all over a wide range of shapes."""
 
diff --git a/tensorflow/python/kernel_tests/relu_op_test.py b/tensorflow/python/kernel_tests/relu_op_test.py
index 3b8924904c..ca02aa6e4e 100644
--- a/tensorflow/python/kernel_tests/relu_op_test.py
+++ b/tensorflow/python/kernel_tests/relu_op_test.py
@@ -86,7 +86,7 @@ class ReluTest(test.TestCase):
     self.assertAllClose(np_relu, tf_relu)
     self.assertShapeEqual(np_relu, tf_relu)
 
-  @test_util.disable_xla("This test never passed for XLA")
+  @test_util.disable_xla("b/123338077")  # Passes with XLA
   def testReluInt8x4BadShape(self):
     if not test.is_gpu_available(cuda_only=True):
       self.skipTest("No GPU available")
diff --git a/tensorflow/python/kernel_tests/scan_ops_test.py b/tensorflow/python/kernel_tests/scan_ops_test.py
index 4e15894fb4..2a3021f982 100644
--- a/tensorflow/python/kernel_tests/scan_ops_test.py
+++ b/tensorflow/python/kernel_tests/scan_ops_test.py
@@ -70,7 +70,6 @@ def handle_options(func, x, axis, exclusive, reverse):
   return x
 
 
-@test_util.disable_all_xla("This test never passed for XLA")
 class CumsumTest(test.TestCase):
 
   valid_dtypes = [
@@ -135,6 +134,7 @@ class CumsumTest(test.TestCase):
         self._compareAll(x, axis)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("b/123860949")  # The computation is constant folded
   def testLarge(self):
     for dtype in self.valid_dtypes:
       x = np.ones([1000000], dtype=dtype) / 1024
@@ -194,7 +194,6 @@ class CumsumTest(test.TestCase):
           self._compareGradient([5, 10], axis, exclusive, reverse)
 
 
-@test_util.disable_all_xla("This test never passed for XLA")
 class CumprodTest(test.TestCase):
 
   valid_dtypes = [
diff --git a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
index 88f7b27b77..5bc301b613 100644
--- a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
+++ b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
@@ -296,7 +296,7 @@ class StatefulScatterNdTest(test.TestCase):
                                     updates).get_shape().as_list(), shape)
 
   @test_util.run_v1_only("b/120545219")
-  @test_util.disable_xla("This test never passed for XLA")
+  @test_util.disable_xla("b/123337890")  # Error messages differ
   def testResVarInvalidOutputShape(self):
     res = variables.Variable(
         initial_value=lambda: array_ops.zeros(shape=[], dtype=dtypes.float32),
diff --git a/tensorflow/python/kernel_tests/spacetodepth_op_test.py b/tensorflow/python/kernel_tests/spacetodepth_op_test.py
index 7f3c381fa1..69243afb69 100644
--- a/tensorflow/python/kernel_tests/spacetodepth_op_test.py
+++ b/tensorflow/python/kernel_tests/spacetodepth_op_test.py
@@ -285,7 +285,7 @@ class SpaceToDepthTest(test.TestCase):
       actual_vals, expected_vals = self.evaluate([actual, expected])
       self.assertTrue(np.array_equal(actual_vals, expected_vals))
 
-  @test_util.disable_xla("This test never passed for XLA")
+  @test_util.disable_xla("b/123553551")  # Unsupported data format
   def testAgainstTranspose(self):
     self.compareToTranspose(3, 2, 3, 1, 2, "NHWC", False)
     self.compareToTranspose(1, 2, 3, 2, 2, "NHWC", False)
diff --git a/tensorflow/python/kernel_tests/split_op_test.py b/tensorflow/python/kernel_tests/split_op_test.py
index 80004db833..42b4d1b778 100644
--- a/tensorflow/python/kernel_tests/split_op_test.py
+++ b/tensorflow/python/kernel_tests/split_op_test.py
@@ -373,7 +373,7 @@ class SplitOpTest(test.TestCase):
     assert s1.shape.as_list() == [1]
 
   @test_util.run_deprecated_v1
-  @test_util.disable_xla("This test never passed for XLA")
+  @test_util.disable_xla("b/123337890")  # Error messages differ
   def testNonexistentDimTensor(self):
     x = array_ops.placeholder(dtypes.int32)
     values = np.zeros([5, 30])
-- 
GitLab


From 5b5636642dab57f71ff0107c0fc032a2834d4dad Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 6 Feb 2019 07:11:31 -0800
Subject: [PATCH 1986/2345] [TF:XLA] Add explicit padding support for Conv2D in
 tf2xla.

PiperOrigin-RevId: 232669459
---
 .../tf2xla/kernels/conv_op_helpers.cc         | 35 ++++++++++++-------
 .../compiler/tf2xla/kernels/conv_op_helpers.h |  1 +
 tensorflow/core/kernels/conv_grad_ops.cc      |  5 +--
 tensorflow/core/kernels/conv_grad_ops.h       |  4 +--
 .../python/kernel_tests/conv_ops_test.py      | 32 +----------------
 5 files changed, 27 insertions(+), 50 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc b/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
index 5f99b24e22..e8b270c67a 100644
--- a/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
+++ b/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
@@ -203,7 +203,8 @@ Status ConvBackpropComputeDimensionsV2XlaShapes(
     StringPiece label, int num_spatial_dims, const xla::Shape& input_shape,
     const xla::Shape& filter_shape, const xla::Shape& out_backprop_shape,
     absl::Span<const int32> dilations, const std::vector<int32>& strides,
-    Padding padding, TensorFormat data_format, ConvBackpropDimensions* dims) {
+    Padding padding, TensorFormat data_format, ConvBackpropDimensions* dims,
+    absl::Span<const int64> explicit_paddings) {
   TensorShape input_tensor_shape, filter_tensor_shape,
       out_backprop_tensor_shape;
   TF_RETURN_IF_ERROR(XLAShapeToTensorShape(input_shape, &input_tensor_shape));
@@ -212,8 +213,8 @@ Status ConvBackpropComputeDimensionsV2XlaShapes(
       XLAShapeToTensorShape(out_backprop_shape, &out_backprop_tensor_shape));
   return ConvBackpropComputeDimensionsV2(
       label, num_spatial_dims, input_tensor_shape, filter_tensor_shape,
-      out_backprop_tensor_shape, dilations, strides, padding,
-      /*explicit_paddings=*/{}, data_format, dims);
+      out_backprop_tensor_shape, dilations, strides, padding, explicit_paddings,
+      data_format, dims);
 }
 
 }  // anonymous namespace
@@ -227,10 +228,9 @@ xla::StatusOr<ConvOpAttrs> ConvOpAttrs::Create(int num_spatial_dims,
   TF_RETURN_IF_ERROR(ctx->GetAttr("dilations", &attrs.dilations));
   TF_RETURN_IF_ERROR(ctx->GetAttr("strides", &attrs.strides));
   TF_RETURN_IF_ERROR(ctx->GetAttr("padding", &attrs.padding));
-  // TODO(reedwm): Support explicit padding.
   if (attrs.padding == EXPLICIT) {
-    return errors::Unimplemented(
-        "XLA does not yet support Conv2D with explicit padding.");
+    TF_RETURN_IF_ERROR(
+        ctx->GetAttr("explicit_paddings", &attrs.explicit_paddings));
   }
 
   string data_format;
@@ -303,6 +303,11 @@ xla::StatusOr<xla::XlaOp> MakeXlaForwardConvOp(StringPiece /*type_string*/,
     window_strides[i] = attrs.strides.at(dim);
     rhs_dilation[i] = attrs.dilations.at(dim);
 
+    if (attrs.padding == EXPLICIT) {
+      padding[i] = {attrs.explicit_paddings.at(dim * 2),
+                    attrs.explicit_paddings.at(dim * 2 + 1)};
+    }
+
     int64 unused_output_size;
     TF_RETURN_IF_ERROR(GetWindowedOutputSizeVerboseV2(
         input_shape.dimensions(dim), filter_shape.dimensions(i),
@@ -337,7 +342,7 @@ xla::StatusOr<xla::XlaOp> MakeXlaBackpropInputConvOp(
   TF_RETURN_IF_ERROR(ConvBackpropComputeDimensionsV2XlaShapes(
       type_string, attrs.num_spatial_dims, input_shape, expanded_filter_shape,
       out_backprop_shape, attrs.dilations, attrs.strides, attrs.padding,
-      attrs.data_format, &dims));
+      attrs.data_format, &dims, attrs.explicit_paddings));
 
   // The input gradients are computed by a convolution of the output
   // gradients and the filter, with some appropriate padding. See the
@@ -420,7 +425,7 @@ xla::StatusOr<xla::XlaOp> MakeXlaBackpropFilterConvOp(
   TF_RETURN_IF_ERROR(ConvBackpropComputeDimensionsV2XlaShapes(
       type_string, attrs.num_spatial_dims, activations_shape,
       expanded_filter_shape, out_backprop_shape, attrs.dilations, attrs.strides,
-      attrs.padding, attrs.data_format, &dims));
+      attrs.padding, attrs.data_format, &dims, attrs.explicit_paddings));
 
   // The activations (inputs) form the LHS of the convolution.
   // Activations have shape: [batch, in_rows, in_cols, ..., in_depth]
@@ -469,6 +474,8 @@ xla::StatusOr<xla::XlaOp> MakeXlaBackpropFilterConvOp(
     int64 dim = GetTensorSpatialDimIndex(num_dims, attrs.data_format, i);
     dnums.add_input_spatial_dimensions(dim);
     dnums.add_kernel_spatial_dimensions(dim);
+    rhs_dilation[i] = dims.spatial_dims[i].stride;
+    window_strides[i] = attrs.dilations[dim];
 
     // We will also need to pad the input with zeros such that after the
     // convolution, we get the right size for the filter.
@@ -495,6 +502,8 @@ xla::StatusOr<xla::XlaOp> MakeXlaBackpropFilterConvOp(
     // We apply negative padding in this case.
     const int64 pad_total = padded_in_size - dims.spatial_dims[i].input_size;
 
+    // + For the EXPLICIT padding, we pad the top/left side with the explicit
+    //   padding and pad the bottom/right side with the remaining space.
     // + For the VALID padding, we don't pad anything on the top/left side
     //   and pad the bottom/right side with the remaining space.
     // + For the SAME padding, we pad top/left side the same as bottom/right
@@ -503,12 +512,12 @@ xla::StatusOr<xla::XlaOp> MakeXlaBackpropFilterConvOp(
     // In addition, if the padded input size is smaller than the input size,
     // we need to ignore some training elements of the input. We do this by
     // applying negative padding on the right/bottom.
-    const int64 pad_before =
-        attrs.padding == Padding::SAME ? std::max<int64>(pad_total / 2, 0) : 0;
-
+    const int64 pad_before = attrs.padding == Padding::EXPLICIT
+                                 ? attrs.explicit_paddings[2 * dim]
+                                 : attrs.padding == Padding::SAME
+                                       ? std::max<int64>(pad_total / 2, 0)
+                                       : 0;
     padding[i] = {pad_before, pad_total - pad_before};
-    rhs_dilation[i] = dims.spatial_dims[i].stride;
-    window_strides[i] = attrs.dilations[dim];
   }
 
   // Besides padding the input, we will also expand output_rows to
diff --git a/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.h b/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.h
index 6e1b70a478..d893eca7f9 100644
--- a/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.h
+++ b/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.h
@@ -47,6 +47,7 @@ struct ConvOpAttrs {
   std::vector<int32> dilations;
   std::vector<int32> strides;
   Padding padding;
+  std::vector<int64> explicit_paddings;
   TensorFormat data_format;
 };
 
diff --git a/tensorflow/core/kernels/conv_grad_ops.cc b/tensorflow/core/kernels/conv_grad_ops.cc
index 0fd7550830..9ceb51062e 100644
--- a/tensorflow/core/kernels/conv_grad_ops.cc
+++ b/tensorflow/core/kernels/conv_grad_ops.cc
@@ -78,9 +78,6 @@ Status ConvBackpropExtractAndVerifyDimension(
         " stride: ", dim->stride, " dilation: ", dim->dilation);
   }
 
-  // TODO(reedwm): Correctly handle explicit padding here. The rest of the
-  // fields set on 'dim' are only used in XLA. TensorFlow ops do not yet support
-  // explicit padding for XLA.
   int64 effective_filter_size = (dim->filter_size - 1) * dim->dilation + 1;
   dim->expanded_output_size = (dim->output_size - 1) * dim->stride + 1;
   const auto padded_out_size = dim->input_size + effective_filter_size - 1;
@@ -102,7 +99,7 @@ Status ConvBackpropComputeDimensionsV2(
     StringPiece label, int num_spatial_dims, const TensorShape& input_shape,
     const TensorShape& filter_shape, const TensorShape& out_backprop_shape,
     const gtl::ArraySlice<int32>& dilations, const std::vector<int32>& strides,
-    Padding padding, const std::vector<int64>& explicit_paddings,
+    Padding padding, absl::Span<const int64> explicit_paddings,
     TensorFormat data_format, ConvBackpropDimensions* dims) {
   // The + 2 in the following line is for the batch and feature dimensions.
   const int num_dims = num_spatial_dims + 2;
diff --git a/tensorflow/core/kernels/conv_grad_ops.h b/tensorflow/core/kernels/conv_grad_ops.h
index c8e8cf28c5..173f92806f 100644
--- a/tensorflow/core/kernels/conv_grad_ops.h
+++ b/tensorflow/core/kernels/conv_grad_ops.h
@@ -222,7 +222,7 @@ struct ConvBackpropSpatialDimension {
   int64 stride;
   int64 dilation;
 
-  // The following fields are valid only if the padding is not EXPLICIT.
+  // Output size after scaling by the stride.
   int64 expanded_output_size;
 
   // Number of padding elements to be added before/after this dimension of
@@ -270,7 +270,7 @@ Status ConvBackpropComputeDimensionsV2(
     StringPiece label, int num_spatial_dims, const TensorShape& input_shape,
     const TensorShape& filter_shape, const TensorShape& out_backprop_shape,
     const gtl::ArraySlice<int32>& dilations, const std::vector<int32>& strides,
-    Padding padding, const std::vector<int64>& explicit_paddings,
+    Padding padding, absl::Span<const int64> explicit_paddings,
     TensorFormat data_format, ConvBackpropDimensions* dims);
 }  // namespace tensorflow
 
diff --git a/tensorflow/python/kernel_tests/conv_ops_test.py b/tensorflow/python/kernel_tests/conv_ops_test.py
index d9b908be1e..732d870c3a 100644
--- a/tensorflow/python/kernel_tests/conv_ops_test.py
+++ b/tensorflow/python/kernel_tests/conv_ops_test.py
@@ -574,7 +574,6 @@ class Conv2DTest(test.TestCase):
         padding="VALID")
 
   @test_util.run_in_graph_and_eager_modes()
-  @test_util.disable_xla("This test never passed for XLA")
   def testConv2D0x0Padding(self):
     self._VerifyExplicitPaddings(
         tensor_in_sizes=[1, 2, 3, 3],
@@ -589,7 +588,6 @@ class Conv2DTest(test.TestCase):
         padding=[[0, 0], [0, 0]])
 
   @test_util.run_in_graph_and_eager_modes()
-  @test_util.disable_xla("This test never passed for XLA")
   def testConv2D1x1Padding(self):
     self._VerifyExplicitPaddings(
         tensor_in_sizes=[1, 2, 3, 2],
@@ -604,7 +602,6 @@ class Conv2DTest(test.TestCase):
         padding=[[1, 1], [1, 1]])
 
   @test_util.run_in_graph_and_eager_modes()
-  @test_util.disable_xla("This test never passed for XLA")
   def testConv2D2x2Padding(self):
     self._VerifyExplicitPaddings(
         tensor_in_sizes=[1, 2, 1, 2],
@@ -619,7 +616,6 @@ class Conv2DTest(test.TestCase):
         padding=[[2, 2], [2, 2]])
 
   @test_util.run_in_graph_and_eager_modes()
-  @test_util.disable_xla("This test never passed for XLA")
   def testConv2DOnlyBottomPadding(self):
     self._VerifyExplicitPaddings(
         tensor_in_sizes=[1, 2, 3, 3],
@@ -634,7 +630,6 @@ class Conv2DTest(test.TestCase):
         padding=[[0, 3], [0, 0]])
 
   @test_util.run_in_graph_and_eager_modes()
-  @test_util.disable_xla("This test never passed for XLA")
   def testConv2DOnlyTopRightPadding(self):
     self._VerifyExplicitPaddings(
         tensor_in_sizes=[1, 2, 3, 3],
@@ -650,7 +645,6 @@ class Conv2DTest(test.TestCase):
         padding=[[1, 0], [0, 2]])
 
   @test_util.run_in_graph_and_eager_modes()
-  @test_util.disable_xla("This test never passed for XLA")
   def testConv2DLotsPadding(self):
     self._VerifyExplicitPaddings(
         tensor_in_sizes=[1, 1, 1, 3],
@@ -665,7 +659,6 @@ class Conv2DTest(test.TestCase):
         padding=[[3, 4], [4, 2]])
 
   @test_util.run_in_graph_and_eager_modes()
-  @test_util.disable_xla("This test never passed for XLA")
   def testConv2DExplicitPaddingWithDilations(self):
     self._VerifyExplicitPaddings(
         tensor_in_sizes=[1, 3, 2, 1],
@@ -681,7 +674,6 @@ class Conv2DTest(test.TestCase):
         padding=[[2, 1], [1, 2]],
         dilations=[2, 3])
 
-  @test_util.disable_xla("This test never passed for XLA")
   def testConv2DExplicitPaddingWithLayoutOptimizer(self):
     # Test with Grappler's layout optimizer, to ensure the layout optimizer
     # handles explicit padding correctly.
@@ -1349,7 +1341,6 @@ class Conv2DTest(test.TestCase):
         dilations=dilations)
 
   @test_util.run_in_graph_and_eager_modes()
-  @test_util.disable_xla("This test never passed for XLA")
   def testConv2D2x2Depth1Padding0x0BackpropInput(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -1372,7 +1363,6 @@ class Conv2DTest(test.TestCase):
             data_format=data_format)
 
   @test_util.run_in_graph_and_eager_modes()
-  @test_util.disable_xla("This test never passed for XLA")
   def testConv2D2x2Depth1Padding1x1BackpropInput(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -1405,7 +1395,6 @@ class Conv2DTest(test.TestCase):
             dilations=[2, 2])
 
   @test_util.run_in_graph_and_eager_modes()
-  @test_util.disable_xla("This test never passed for XLA")
   def testConv2D2x2Depth1Padding2x2BackpropInput(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -1430,7 +1419,6 @@ class Conv2DTest(test.TestCase):
             dilations=[2, 3])
 
   @test_util.run_in_graph_and_eager_modes()
-  @test_util.disable_xla("This test never passed for XLA")
   def testConv2D2x2Depth1Padding_1_8_4_1_BackpropInput(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -1453,7 +1441,6 @@ class Conv2DTest(test.TestCase):
             data_format=data_format)
 
   @test_util.run_in_graph_and_eager_modes()
-  @test_util.disable_xla("This test never passed for XLA")
   def testConv2D2x2Depth1Padding_5_0_2_2_BackpropInput(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -1512,7 +1499,6 @@ class Conv2DTest(test.TestCase):
         err=err)
 
   @test_util.run_in_graph_and_eager_modes()
-  @test_util.disable_xla("This test never passed for XLA")
   def testConv2D2x2Depth1Padding0x0BackpropFilter(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -1535,7 +1521,6 @@ class Conv2DTest(test.TestCase):
             data_format=data_format)
 
   @test_util.run_in_graph_and_eager_modes()
-  @test_util.disable_xla("This test never passed for XLA")
   def testConv2D2x2Depth1Padding1x1BackpropFilter(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -1569,7 +1554,6 @@ class Conv2DTest(test.TestCase):
             dilations=[2, 2])
 
   @test_util.run_in_graph_and_eager_modes()
-  @test_util.disable_xla("This test never passed for XLA")
   def testConv2D2x2Depth1Padding2x2BackpropFilter(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -1594,7 +1578,6 @@ class Conv2DTest(test.TestCase):
             dilations=[2, 3])
 
   @test_util.run_in_graph_and_eager_modes()
-  @test_util.disable_xla("This test never passed for XLA")
   def testConv2D2x2Depth1Padding_1_8_4_1_BackpropFilter(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -1618,7 +1601,6 @@ class Conv2DTest(test.TestCase):
             data_format=data_format)
 
   @test_util.run_in_graph_and_eager_modes()
-  @test_util.disable_xla("This test never passed for XLA")
   def testConv2D2x2Depth1Padding_5_0_2_2_BackpropFilter(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -1976,7 +1958,6 @@ class Conv2DTest(test.TestCase):
           data_format=data_format,
           use_gpu=use_gpu)
 
-  @test_util.disable_xla("This test never passed for XLA")
   def testInputGradient1x1PaddingStrideOne(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -1998,7 +1979,6 @@ class Conv2DTest(test.TestCase):
             use_gpu=use_gpu,
             max_err=0.0025)
 
-  @test_util.disable_xla("This test never passed for XLA")
   def testFilterGradient1x1PaddingStrideOne(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -2019,7 +1999,6 @@ class Conv2DTest(test.TestCase):
             data_format=data_format,
             use_gpu=use_gpu)
 
-  @test_util.disable_xla("This test never passed for XLA")
   def testInputGradient1x1PaddingStrideTwo(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -2040,7 +2019,6 @@ class Conv2DTest(test.TestCase):
             data_format=data_format,
             use_gpu=use_gpu)
 
-  @test_util.disable_xla("This test never passed for XLA")
   def testFilterGradient1x1PaddingStrideTwo(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -2061,7 +2039,6 @@ class Conv2DTest(test.TestCase):
             data_format=data_format,
             use_gpu=use_gpu)
 
-  @test_util.disable_xla("This test never passed for XLA")
   def testInputGradient2x2PaddingStrideOne(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -2082,7 +2059,6 @@ class Conv2DTest(test.TestCase):
             data_format=data_format,
             use_gpu=use_gpu)
 
-  @test_util.disable_xla("This test never passed for XLA")
   def testFilterGradient2x2PaddingStrideOne(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -2104,7 +2080,6 @@ class Conv2DTest(test.TestCase):
             use_gpu=use_gpu,
             max_err=0.003)
 
-  @test_util.disable_xla("This test never passed for XLA")
   def testInputGradient1_2_3_4PaddingStride3x2(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -2125,7 +2100,6 @@ class Conv2DTest(test.TestCase):
             data_format=data_format,
             use_gpu=use_gpu)
 
-  @test_util.disable_xla("This test never passed for XLA")
   def testFilterGradient1_2_3_4PaddingStride3x2(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -2146,7 +2120,6 @@ class Conv2DTest(test.TestCase):
             data_format=data_format,
             use_gpu=use_gpu)
 
-  @test_util.disable_xla("This test never passed for XLA")
   def testInputGradient4_3_2_1PaddingStride2x1(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -2167,7 +2140,6 @@ class Conv2DTest(test.TestCase):
             data_format=data_format,
             use_gpu=use_gpu)
 
-  @test_util.disable_xla("This test never passed for XLA")
   def testFilterGradient4_3_2_1PaddingStride2x1(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -2188,7 +2160,6 @@ class Conv2DTest(test.TestCase):
             data_format=data_format,
             use_gpu=use_gpu)
 
-  @test_util.disable_xla("This test never passed for XLA")
   def testInputGradient0_0_0_5PaddingStride1x2(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -2209,7 +2180,6 @@ class Conv2DTest(test.TestCase):
             data_format=data_format,
             use_gpu=use_gpu)
 
-  @test_util.disable_xla("This test never passed for XLA")
   def testFilterGradient0_0_0_5PaddingStride1x2(self):
     if not test.is_gpu_available(cuda_only=True):
       return
@@ -2316,7 +2286,7 @@ class Conv2DTest(test.TestCase):
           strides=[1, 1, 1, 1],
           padding=[0, 0, 0, 0])
 
-  @test_util.disable_xla("This test never passed for XLA")
+  @test_util.disable_xla("b/123337890")  # Error messages differ
   def testOpEdgeCases(self):
     with self.cached_session() as sess:
       # Illegal strides.
-- 
GitLab


From e1836047555f19ea371a1e4b46101cf535c4a68b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 6 Feb 2019 07:41:22 -0800
Subject: [PATCH 1987/2345] Add flatten_with_tuple_paths_up_to(), the 'flatten'
 version of map_structure_with_tuple_paths_up_to().

PiperOrigin-RevId: 232673116
---
 tensorflow/contrib/framework/__init__.py |   1 +
 tensorflow/python/util/nest.py           |  94 +++++++++
 tensorflow/python/util/nest_test.py      | 238 +++++++++++++++++++++++
 3 files changed, 333 insertions(+)

diff --git a/tensorflow/contrib/framework/__init__.py b/tensorflow/contrib/framework/__init__.py
index 3784631dcb..fc2334d5d7 100644
--- a/tensorflow/contrib/framework/__init__.py
+++ b/tensorflow/contrib/framework/__init__.py
@@ -139,6 +139,7 @@ _nest_allowed_symbols = [
     'map_structure_with_tuple_paths',
     'assert_shallow_structure',
     'flatten_up_to',
+    'flatten_with_tuple_paths_up_to',
     'map_structure_up_to',
     'map_structure_with_tuple_paths_up_to',
     'get_traverse_shallow_structure',
diff --git a/tensorflow/python/util/nest.py b/tensorflow/python/util/nest.py
index a43ec48589..2435694631 100644
--- a/tensorflow/python/util/nest.py
+++ b/tensorflow/python/util/nest.py
@@ -761,6 +761,100 @@ def flatten_up_to(shallow_tree, input_tree, check_types=True):
   return list(v for _, v in _yield_flat_up_to(shallow_tree, input_tree))
 
 
+def flatten_with_tuple_paths_up_to(shallow_tree, input_tree, check_types=True):
+  """Flattens `input_tree` up to `shallow_tree`.
+
+  Any further depth in structure in `input_tree` is retained as elements in the
+  partially flattened output.
+
+  Returns a list of (path, value) pairs, where value a leaf node in the
+  flattened tree, and path is the tuple path of that leaf in input_tree.
+
+  If `shallow_tree` and `input_tree` are not sequences, this returns a
+  single-element list: `[((), input_tree)]`.
+
+  Use Case:
+
+  Sometimes we may wish to partially flatten a nested sequence, retaining some
+  of the nested structure. We achieve this by specifying a shallow structure,
+  `shallow_tree`, we wish to flatten up to.
+
+  The input, `input_tree`, can be thought of as having the same structure as
+  `shallow_tree`, but with leaf nodes that are themselves tree structures.
+
+  Examples:
+
+  ```python
+  input_tree = [[[2, 2], [3, 3]], [[4, 9], [5, 5]]]
+  shallow_tree = [[True, True], [False, True]]
+
+  flattened_input_tree = flatten_with_tuple_paths_up_to(shallow_tree,
+                                                        input_tree)
+  flattened_shallow_tree = flatten_with_tuple_paths_up_to(shallow_tree,
+                                                          shallow_tree)
+
+  # Output is:
+  # [((0, 0), [2, 2]),
+  #  ((0, 1), [3, 3]),
+  #  ((1, 0), [4, 9]),
+  #  ((1, 1), [5, 5])]
+  #
+  # [((0, 0), True),
+  #  ((0, 1), True),
+  #  ((1, 0), False),
+  #  ((1, 1), True)]
+  ```
+
+  ```python
+  input_tree = [[('a', 1), [('b', 2), [('c', 3), [('d', 4)]]]]]
+  shallow_tree = [['level_1', ['level_2', ['level_3', ['level_4']]]]]
+
+  input_tree_flattened_as_shallow_tree = flatten_up_to(shallow_tree, input_tree)
+  input_tree_flattened = flatten(input_tree)
+
+  # Output is:
+  # [((0, 0), ('a', 1)),
+  #  ((0, 1, 0), ('b', 2)),
+  #  ((0, 1, 1, 0), ('c', 3)),
+  #  ((0, 1, 1, 1), ('d', 4))]
+  # ['a', 1, 'b', 2, 'c', 3, 'd', 4]
+  ```
+
+  Non-Sequence Edge Cases:
+
+  ```python
+  flatten_with_tuple_paths_up_to(0, 0)  # Output: [(), 0]
+
+  flatten_with_tuple_paths_up_to(0, [0, 1, 2])  # Output: [(), [0, 1, 2]]
+
+  flatten_with_tuple_paths_up_to([0, 1, 2], 0)  # Output: TypeError
+
+  flatten_with_tuple_paths_up_to([0, 1, 2], [0, 1, 2])
+  # Output: [((0,) 0), ((1,), 1), ((2,), 2)]
+  ```
+
+  Args:
+    shallow_tree: a possibly pruned structure of input_tree.
+    input_tree: an arbitrarily nested structure or a scalar object.
+      Note, numpy arrays are considered scalars.
+    check_types: bool. If True, check that each node in shallow_tree has the
+      same type as the corresponding node in input_tree.
+
+  Returns:
+    A Python list, the partially flattened version of `input_tree` according to
+    the structure of `shallow_tree`.
+
+  Raises:
+    TypeError: If `shallow_tree` is a sequence but `input_tree` is not.
+    TypeError: If the sequence types of `shallow_tree` are different from
+      `input_tree`.
+    ValueError: If the sequence lengths of `shallow_tree` are different from
+      `input_tree`.
+  """
+  assert_shallow_structure(shallow_tree, input_tree, check_types=check_types)
+  return list(_yield_flat_up_to(shallow_tree, input_tree))
+
+
 def map_structure_up_to(shallow_tree, func, *inputs, **kwargs):
   """Applies a function or op to a number of partially flattened inputs.
 
diff --git a/tensorflow/python/util/nest_test.py b/tensorflow/python/util/nest_test.py
index 71034ffcb6..ec559bd2ab 100644
--- a/tensorflow/python/util/nest_test.py
+++ b/tensorflow/python/util/nest_test.py
@@ -686,6 +686,244 @@ class NestTest(parameterized.TestCase, test.TestCase):
     flattened_shallow_tree = nest.flatten_up_to(shallow_tree, shallow_tree)
     self.assertEqual(flattened_shallow_tree, shallow_tree)
 
+  def testFlattenWithTuplePathsUpTo(self):
+    def get_paths_and_values(shallow_tree, input_tree):
+      path_value_pairs = nest.flatten_with_tuple_paths_up_to(shallow_tree,
+                                                             input_tree)
+      paths = [p for p, _ in path_value_pairs]
+      values = [v for _, v in path_value_pairs]
+      return paths, values
+
+    # Shallow tree ends at scalar.
+    input_tree = [[[2, 2], [3, 3]], [[4, 9], [5, 5]]]
+    shallow_tree = [[True, True], [False, True]]
+    (flattened_input_tree_paths,
+     flattened_input_tree) = get_paths_and_values(shallow_tree, input_tree)
+    (flattened_shallow_tree_paths,
+     flattened_shallow_tree) = get_paths_and_values(shallow_tree, shallow_tree)
+    self.assertEqual(flattened_input_tree_paths,
+                     [(0, 0), (0, 1), (1, 0), (1, 1)])
+    self.assertEqual(flattened_input_tree, [[2, 2], [3, 3], [4, 9], [5, 5]])
+    self.assertEqual(flattened_shallow_tree_paths,
+                     [(0, 0), (0, 1), (1, 0), (1, 1)])
+    self.assertEqual(flattened_shallow_tree, [True, True, False, True])
+
+    # Shallow tree ends at string.
+    input_tree = [[("a", 1), [("b", 2), [("c", 3), [("d", 4)]]]]]
+    shallow_tree = [["level_1", ["level_2", ["level_3", ["level_4"]]]]]
+    (input_tree_flattened_as_shallow_tree_paths,
+     input_tree_flattened_as_shallow_tree) = get_paths_and_values(shallow_tree,
+                                                                  input_tree)
+    input_tree_flattened_paths = [p for p, _ in
+                                  nest.flatten_with_tuple_paths(input_tree)]
+    input_tree_flattened = nest.flatten(input_tree)
+    self.assertEqual(input_tree_flattened_as_shallow_tree_paths,
+                     [(0, 0), (0, 1, 0), (0, 1, 1, 0), (0, 1, 1, 1, 0)])
+    self.assertEqual(input_tree_flattened_as_shallow_tree,
+                     [("a", 1), ("b", 2), ("c", 3), ("d", 4)])
+
+    self.assertEqual(input_tree_flattened_paths,
+                     [(0, 0, 0), (0, 0, 1),
+                      (0, 1, 0, 0), (0, 1, 0, 1),
+                      (0, 1, 1, 0, 0), (0, 1, 1, 0, 1),
+                      (0, 1, 1, 1, 0, 0), (0, 1, 1, 1, 0, 1)])
+    self.assertEqual(input_tree_flattened, ["a", 1, "b", 2, "c", 3, "d", 4])
+
+    # Make sure dicts are correctly flattened, yielding values, not keys.
+    input_tree = {"a": 1, "b": {"c": 2}, "d": [3, (4, 5)]}
+    shallow_tree = {"a": 0, "b": 0, "d": [0, 0]}
+    (input_tree_flattened_as_shallow_tree_paths,
+     input_tree_flattened_as_shallow_tree) = get_paths_and_values(shallow_tree,
+                                                                  input_tree)
+    self.assertEqual(input_tree_flattened_as_shallow_tree_paths,
+                     [("a",), ("b",), ("d", 0), ("d", 1)])
+    self.assertEqual(input_tree_flattened_as_shallow_tree,
+                     [1, {"c": 2}, 3, (4, 5)])
+
+    # Namedtuples.
+    ab_tuple = collections.namedtuple("ab_tuple", "a, b")
+    input_tree = ab_tuple(a=[0, 1], b=2)
+    shallow_tree = ab_tuple(a=0, b=1)
+    (input_tree_flattened_as_shallow_tree_paths,
+     input_tree_flattened_as_shallow_tree) = get_paths_and_values(shallow_tree,
+                                                                  input_tree)
+    self.assertEqual(input_tree_flattened_as_shallow_tree_paths,
+                     [("a",), ("b",)])
+    self.assertEqual(input_tree_flattened_as_shallow_tree,
+                     [[0, 1], 2])
+
+    # Nested dicts, OrderedDicts and namedtuples.
+    input_tree = collections.OrderedDict(
+        [("a", ab_tuple(a=[0, {"b": 1}], b=2)),
+         ("c", {"d": 3, "e": collections.OrderedDict([("f", 4)])})])
+    shallow_tree = input_tree
+    (input_tree_flattened_as_shallow_tree_paths,
+     input_tree_flattened_as_shallow_tree) = get_paths_and_values(shallow_tree,
+                                                                  input_tree)
+    self.assertEqual(input_tree_flattened_as_shallow_tree_paths,
+                     [("a", "a", 0),
+                      ("a", "a", 1, "b"),
+                      ("a", "b"),
+                      ("c", "d"),
+                      ("c", "e", "f")])
+    self.assertEqual(input_tree_flattened_as_shallow_tree, [0, 1, 2, 3, 4])
+    shallow_tree = collections.OrderedDict([("a", 0), ("c", {"d": 3, "e": 1})])
+    (input_tree_flattened_as_shallow_tree_paths,
+     input_tree_flattened_as_shallow_tree) = get_paths_and_values(shallow_tree,
+                                                                  input_tree)
+    self.assertEqual(input_tree_flattened_as_shallow_tree_paths,
+                     [("a",),
+                      ("c", "d"),
+                      ("c", "e")])
+    self.assertEqual(input_tree_flattened_as_shallow_tree,
+                     [ab_tuple(a=[0, {"b": 1}], b=2),
+                      3,
+                      collections.OrderedDict([("f", 4)])])
+    shallow_tree = collections.OrderedDict([("a", 0), ("c", 0)])
+    (input_tree_flattened_as_shallow_tree_paths,
+     input_tree_flattened_as_shallow_tree) = get_paths_and_values(shallow_tree,
+                                                                  input_tree)
+    self.assertEqual(input_tree_flattened_as_shallow_tree_paths,
+                     [("a",), ("c",)])
+    self.assertEqual(input_tree_flattened_as_shallow_tree,
+                     [ab_tuple(a=[0, {"b": 1}], b=2),
+                      {"d": 3, "e": collections.OrderedDict([("f", 4)])}])
+
+    ## Shallow non-list edge-case.
+    # Using iterable elements.
+    input_tree = ["input_tree"]
+    shallow_tree = "shallow_tree"
+    (flattened_input_tree_paths,
+     flattened_input_tree) = get_paths_and_values(shallow_tree, input_tree)
+    (flattened_shallow_tree_paths,
+     flattened_shallow_tree) = get_paths_and_values(shallow_tree, shallow_tree)
+    self.assertEqual(flattened_input_tree_paths, [()])
+    self.assertEqual(flattened_input_tree, [input_tree])
+    self.assertEqual(flattened_shallow_tree_paths, [()])
+    self.assertEqual(flattened_shallow_tree, [shallow_tree])
+
+    input_tree = ["input_tree_0", "input_tree_1"]
+    shallow_tree = "shallow_tree"
+    (flattened_input_tree_paths,
+     flattened_input_tree) = get_paths_and_values(shallow_tree, input_tree)
+    (flattened_shallow_tree_paths,
+     flattened_shallow_tree) = get_paths_and_values(shallow_tree, shallow_tree)
+    self.assertEqual(flattened_input_tree_paths, [()])
+    self.assertEqual(flattened_input_tree, [input_tree])
+    self.assertEqual(flattened_shallow_tree_paths, [()])
+    self.assertEqual(flattened_shallow_tree, [shallow_tree])
+
+    # Test case where len(shallow_tree) < len(input_tree)
+    input_tree = {"a": "A", "b": "B", "c": "C"}
+    shallow_tree = {"a": 1, "c": 2}
+    (flattened_input_tree_paths,
+     flattened_input_tree) = get_paths_and_values(shallow_tree, input_tree)
+    (flattened_shallow_tree_paths,
+     flattened_shallow_tree) = get_paths_and_values(shallow_tree, shallow_tree)
+    self.assertEqual(flattened_input_tree_paths, [("a",), ("c",)])
+    self.assertEqual(flattened_input_tree, ["A", "C"])
+    self.assertEqual(flattened_shallow_tree_paths, [("a",), ("c",)])
+    self.assertEqual(flattened_shallow_tree, [1, 2])
+
+    # Using non-iterable elements.
+    input_tree = [0]
+    shallow_tree = 9
+    (flattened_input_tree_paths,
+     flattened_input_tree) = get_paths_and_values(shallow_tree, input_tree)
+    (flattened_shallow_tree_paths,
+     flattened_shallow_tree) = get_paths_and_values(shallow_tree, shallow_tree)
+    self.assertEqual(flattened_input_tree_paths, [()])
+    self.assertEqual(flattened_input_tree, [input_tree])
+    self.assertEqual(flattened_shallow_tree_paths, [()])
+    self.assertEqual(flattened_shallow_tree, [shallow_tree])
+
+    input_tree = [0, 1]
+    shallow_tree = 9
+    (flattened_input_tree_paths,
+     flattened_input_tree) = get_paths_and_values(shallow_tree, input_tree)
+    (flattened_shallow_tree_paths,
+     flattened_shallow_tree) = get_paths_and_values(shallow_tree, shallow_tree)
+    self.assertEqual(flattened_input_tree_paths, [()])
+    self.assertEqual(flattened_input_tree, [input_tree])
+    self.assertEqual(flattened_shallow_tree_paths, [()])
+    self.assertEqual(flattened_shallow_tree, [shallow_tree])
+
+    ## Both non-list edge-case.
+    # Using iterable elements.
+    input_tree = "input_tree"
+    shallow_tree = "shallow_tree"
+    (flattened_input_tree_paths,
+     flattened_input_tree) = get_paths_and_values(shallow_tree, input_tree)
+    (flattened_shallow_tree_paths,
+     flattened_shallow_tree) = get_paths_and_values(shallow_tree, shallow_tree)
+    self.assertEqual(flattened_input_tree_paths, [()])
+    self.assertEqual(flattened_input_tree, [input_tree])
+    self.assertEqual(flattened_shallow_tree_paths, [()])
+    self.assertEqual(flattened_shallow_tree, [shallow_tree])
+
+    # Using non-iterable elements.
+    input_tree = 0
+    shallow_tree = 0
+    (flattened_input_tree_paths,
+     flattened_input_tree) = get_paths_and_values(shallow_tree, input_tree)
+    (flattened_shallow_tree_paths,
+     flattened_shallow_tree) = get_paths_and_values(shallow_tree, shallow_tree)
+    self.assertEqual(flattened_input_tree_paths, [()])
+    self.assertEqual(flattened_input_tree, [input_tree])
+    self.assertEqual(flattened_shallow_tree_paths, [()])
+    self.assertEqual(flattened_shallow_tree, [shallow_tree])
+
+    ## Input non-list edge-case.
+    # Using iterable elements.
+    input_tree = "input_tree"
+    shallow_tree = ["shallow_tree"]
+    with self.assertRaisesWithLiteralMatch(
+        TypeError,
+        nest._IF_SHALLOW_IS_SEQ_INPUT_MUST_BE_SEQ.format(type(input_tree))):
+      (flattened_input_tree_paths,
+       flattened_input_tree) = get_paths_and_values(shallow_tree, input_tree)
+    (flattened_shallow_tree_paths,
+     flattened_shallow_tree) = get_paths_and_values(shallow_tree, shallow_tree)
+    self.assertEqual(flattened_shallow_tree_paths, [(0,)])
+    self.assertEqual(flattened_shallow_tree, shallow_tree)
+
+    input_tree = "input_tree"
+    shallow_tree = ["shallow_tree_9", "shallow_tree_8"]
+    with self.assertRaisesWithLiteralMatch(
+        TypeError,
+        nest._IF_SHALLOW_IS_SEQ_INPUT_MUST_BE_SEQ.format(type(input_tree))):
+      (flattened_input_tree_paths,
+       flattened_input_tree) = get_paths_and_values(shallow_tree, input_tree)
+    (flattened_shallow_tree_paths,
+     flattened_shallow_tree) = get_paths_and_values(shallow_tree, shallow_tree)
+    self.assertEqual(flattened_shallow_tree_paths, [(0,), (1,)])
+    self.assertEqual(flattened_shallow_tree, shallow_tree)
+
+    # Using non-iterable elements.
+    input_tree = 0
+    shallow_tree = [9]
+    with self.assertRaisesWithLiteralMatch(
+        TypeError,
+        nest._IF_SHALLOW_IS_SEQ_INPUT_MUST_BE_SEQ.format(type(input_tree))):
+      (flattened_input_tree_paths,
+       flattened_input_tree) = get_paths_and_values(shallow_tree, input_tree)
+    (flattened_shallow_tree_paths,
+     flattened_shallow_tree) = get_paths_and_values(shallow_tree, shallow_tree)
+    self.assertEqual(flattened_shallow_tree_paths, [(0,)])
+    self.assertEqual(flattened_shallow_tree, shallow_tree)
+
+    input_tree = 0
+    shallow_tree = [9, 8]
+    with self.assertRaisesWithLiteralMatch(
+        TypeError,
+        nest._IF_SHALLOW_IS_SEQ_INPUT_MUST_BE_SEQ.format(type(input_tree))):
+      (flattened_input_tree_paths,
+       flattened_input_tree) = get_paths_and_values(shallow_tree, input_tree)
+    (flattened_shallow_tree_paths,
+     flattened_shallow_tree) = get_paths_and_values(shallow_tree, shallow_tree)
+    self.assertEqual(flattened_shallow_tree_paths, [(0,), (1,)])
+    self.assertEqual(flattened_shallow_tree, shallow_tree)
+
   def testMapStructureUpTo(self):
     # Named tuples.
     ab_tuple = collections.namedtuple("ab_tuple", "a, b")
-- 
GitLab


From 2ccb446ecf92dc01df0a60d965561944d82e405b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 6 Feb 2019 08:00:49 -0800
Subject: [PATCH 1988/2345] Handle nested code-blocks correctly in lowering
 passes. This fixes issues with the early-return and continue passes which
 were generating incorrect code when interleaved with "with" and "try"
 statements.

PiperOrigin-RevId: 232675424
---
 .../converters/continue_statements.py         | 22 +++++++++++++++
 .../autograph/converters/return_statements.py | 27 +++++++++++++++++++
 2 files changed, 49 insertions(+)

diff --git a/tensorflow/python/autograph/converters/continue_statements.py b/tensorflow/python/autograph/converters/continue_statements.py
index c3b6679e38..725e05360d 100644
--- a/tensorflow/python/autograph/converters/continue_statements.py
+++ b/tensorflow/python/autograph/converters/continue_statements.py
@@ -121,6 +121,28 @@ class ContinueCanonicalizationTransformer(converter.Base):
     node.orelse = self.visit_block(node.orelse)
     return node
 
+  def visit_With(self, node):
+    node.items = self.visit_block(node.items)
+    node.body = self.visit_block(node.body,
+                                 after_visit=self._postprocess_statement)
+    return node
+
+  def visit_Try(self, node):
+    node.body = self.visit_block(node.body,
+                                 after_visit=self._postprocess_statement)
+    node.orelse = self.visit_block(node.orelse,
+                                   after_visit=self._postprocess_statement)
+    # In Python 3.8 and later continue is allowed in finally blocks
+    node.finalbody = self.visit_block(node.finalbody,
+                                      after_visit=self._postprocess_statement)
+    node.handlers = self.visit_block(node.handlers)
+    return node
+
+  def visit_ExceptHandler(self, node):
+    node.body = self.visit_block(node.body,
+                                 after_visit=self._postprocess_statement)
+    return node
+
 
 def transform(node, ctx):
   transformer = ContinueCanonicalizationTransformer(ctx)
diff --git a/tensorflow/python/autograph/converters/return_statements.py b/tensorflow/python/autograph/converters/return_statements.py
index 6cf6f26649..723acab908 100644
--- a/tensorflow/python/autograph/converters/return_statements.py
+++ b/tensorflow/python/autograph/converters/return_statements.py
@@ -115,6 +115,21 @@ class ConditionalReturnRewriter(converter.Base):
       anno.setanno(node, STMT_DEFINITELY_RETURNS, True)
     return node
 
+  def visit_Try(self, node):
+    # We could decide whether a 'try' DEFINITELY_RETURNS based on its components
+    # It is not clear whether we want to do anything with this given
+    # a 'try' is likely to throw an exception in some circumstances.
+    node.body, _ = self._visit_statement_block(node, node.body)
+    node.orelse, _ = self._visit_statement_block(node, node.orelse)
+    node.finalbody, _ = self._visit_statement_block(node, node.finalbody)
+    node.handlers = self.visit_block(node.handlers)
+    return node
+
+  def visit_ExceptHandler(self, node):
+    # To determine whether `try` DEFINITELY_RETURNS we need to revisit this.
+    node.body, _ = self._visit_statement_block(node, node.body)
+    return node
+
   def visit_If(self, node):
     node.test = self.visit(node.test)
 
@@ -317,6 +332,17 @@ class ReturnStatementsTransformer(converter.Base):
     node.body = self._visit_statement_block(node, node.body)
     return node
 
+  def visit_Try(self, node):
+    node.body = self._visit_statement_block(node, node.body)
+    node.orelse = self._visit_statement_block(node, node.orelse)
+    node.finalbody = self._visit_statement_block(node, node.finalbody)
+    node.handlers = self.visit_block(node.handlers)
+    return node
+
+  def visit_ExceptHandler(self, node):
+    node.body = self._visit_statement_block(node, node.body)
+    return node
+
   def visit_If(self, node):
     node.test = self.visit(node.test)
     node.body = self._visit_statement_block(node, node.body)
@@ -382,5 +408,6 @@ def transform(node, ctx, default_to_null_return=True):
   transformer = ReturnStatementsTransformer(
       ctx, default_to_null_return=default_to_null_return)
   node = transformer.visit(node)
+  transformer.debug_print_src(node)
 
   return node
-- 
GitLab


From 4eef6deec6995e07a2d07a94f647f208d43fecfa Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 6 Feb 2019 08:15:03 -0800
Subject: [PATCH 1989/2345] Remove erroneous move; the erstwhile, moved-from
 state is still needed.

This hadn't manifested as an error so far because the moved-from object had accidentally not been movable so far, so the move was ineffective.

PiperOrigin-RevId: 232677767
---
 tensorflow/core/framework/variant_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/framework/variant_test.cc b/tensorflow/core/framework/variant_test.cc
index 08d09de7b8..8947f93887 100644
--- a/tensorflow/core/framework/variant_test.cc
+++ b/tensorflow/core/framework/variant_test.cc
@@ -186,7 +186,7 @@ TEST(VariantTest, TensorListTest) {
   x.Encode(&serialized);
 
   Variant y = TensorList();
-  y.Decode(std::move(serialized));
+  y.Decode(serialized);
 
   const TensorList& decoded_vec = *y.get<TensorList>();
   for (int i = 0; i < 4; ++i) {
-- 
GitLab


From e3f9693c5f5c8d5f7d7f247cd2e55b9645e4f3f6 Mon Sep 17 00:00:00 2001
From: Yuan Lin <yulin@nvidia.com>
Date: Wed, 6 Feb 2019 08:39:32 -0800
Subject: [PATCH 1990/2345] [XLA] interactive_graphviz: pass constant param by
 reference instead of pointer

---
 tensorflow/compiler/xla/service/hlo_graph_dumper.cc   | 6 +++---
 tensorflow/compiler/xla/service/hlo_graph_dumper.h    | 5 +----
 tensorflow/compiler/xla/tools/interactive_graphviz.cc | 2 +-
 3 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
index 810b1e1dfc..6baf357790 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
@@ -1283,7 +1283,7 @@ namespace {
 // root is <= radius.
 NodeFilter MakeNodeRadiusAroundFilter(
     const HloInstruction* root, int64 radius,
-    const std::set<const HloInstruction*>* boundary) {
+    const std::set<const HloInstruction*>& boundary) {
   // First, find the neighborhood of nodes with distance from root <= radius.
   // These nodes are our initial set of "normal" nodes.
   absl::flat_hash_map<const HloInstruction*, NodeFilterResult> nodes;
@@ -1299,7 +1299,7 @@ NodeFilter MakeNodeRadiusAroundFilter(
     if (depth == radius) {
       continue;
     }
-    if (boundary->count(instr) != 0) {
+    if (boundary.count(instr) != 0) {
       continue;
     }
 
@@ -1517,7 +1517,7 @@ string DumpGraph(const HloComputation& computation, const string& label,
 }
 
 string DumpNeighborhoodAround(const HloInstruction& node, int radius,
-                              const std::set<const HloInstruction*>* boundary,
+                              const std::set<const HloInstruction*>& boundary,
                               bool show_backend_config) {
   auto debug_options = node.GetModule()->config().debug_options();
   string label =
diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.h b/tensorflow/compiler/xla/service/hlo_graph_dumper.h
index 755009b817..7547d983cf 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.h
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.h
@@ -63,12 +63,9 @@ string DumpGraph(const HloComputation& computation, const string& label,
 // The number of nodes dumped is controlled by the radius parameter, which
 // (roughly) corresponds to the max distance a node may be from the primary node
 // before it's omitted from the graph.
-// 
-// The optional boundary parameter specifies the set of boundary nodes which
-// will be omitted when they are within the radius.
 string DumpNeighborhoodAround(
     const HloInstruction& node, int radius,
-    const std::set<const HloInstruction*>* boundary = nullptr,
+    const std::set<const HloInstruction*>& boundary = {},
     bool show_backend_config = false);
 
 // Dumps nodes on any of the paths from `from` to `to`.  If there are more than
diff --git a/tensorflow/compiler/xla/tools/interactive_graphviz.cc b/tensorflow/compiler/xla/tools/interactive_graphviz.cc
index 5d90e36975..48013954a8 100644
--- a/tensorflow/compiler/xla/tools/interactive_graphviz.cc
+++ b/tensorflow/compiler/xla/tools/interactive_graphviz.cc
@@ -519,7 +519,7 @@ void DoPlotCommand(const Options& opts, const HloModule& module,
   } else {
     DisplayGraphHandle(opts, hlo_graph_dumper::DumpNeighborhoodAround(
                                  *instr, graph_width,
-                                 /*boundary=*/&boundary,
+                                 /*boundary=*/boundary,
                                  /*show_backend_config=*/show_backend_config));
   }
 }
-- 
GitLab


From ac1919f614fe034a65eeb8d378ad3cf70f2d82aa Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Wed, 6 Feb 2019 08:59:42 -0800
Subject: [PATCH 1991/2345] Automated rollback of commit
 05ca1e9bf3e7a04603eac921c4d95c5dbeca7dd6

PiperOrigin-RevId: 232684115
---
 tensorflow/python/framework/function_test.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/framework/function_test.py b/tensorflow/python/framework/function_test.py
index 7df76efde5..7543376bcf 100644
--- a/tensorflow/python/framework/function_test.py
+++ b/tensorflow/python/framework/function_test.py
@@ -496,6 +496,8 @@ class FunctionTest(test.TestCase):
                                          lambda y: AssertFail(y), [x])
       # pylint: enable=unnecessary-lambda
 
+    rewriter_config = rewriter_config_pb2.RewriterConfig(
+        dependency_optimization=rewriter_config_pb2.RewriterConfig.OFF)
     # Enables inlining.
     config = config_pb2.ConfigProto(
         graph_options=config_pb2.GraphOptions(
@@ -503,7 +505,8 @@ class FunctionTest(test.TestCase):
                 opt_level=config_pb2.OptimizerOptions.L0,
                 do_common_subexpression_elimination=True,
                 do_function_inlining=True,
-                do_constant_folding=True)))
+                do_constant_folding=True),
+            rewrite_options=rewriter_config))
 
     with session.Session(config=config) as sess:
       # Since the 'False' branch is not taken, the assertion should not fire.
-- 
GitLab


From 36d3dc73acf2d54ad2d59ea2c48972ac9e5955a2 Mon Sep 17 00:00:00 2001
From: Chris Jones <cjfj@google.com>
Date: Wed, 6 Feb 2019 09:04:07 -0800
Subject: [PATCH 1992/2345] Fix `TPUStrategy.extended.reduce_to` and allow user
 to specify destinations other than the TPU host.

PiperOrigin-RevId: 232685087
---
 .../contrib/distribute/python/tpu_strategy.py | 25 +++++++++++--------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 818cc40f6b..341d9aeaf9 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -543,19 +543,24 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
       return cross_device_ops_lib.reduce_non_distributed_value(
           reduce_op, self._device_map, value, destinations)
 
-    # Validate that the destination is same as the host device
-    # Note we don't do this when in replicate context as the reduction is
-    # performed on the TPU device itself.
     devices = cross_device_ops_lib.get_devices_from(destinations)
-    if len(devices) == 1:
-      assert device_util.canonicalize(devices[0]) == device_util.canonicalize(
-          self._host_device)
-    else:
+    if len(devices) != 1:
       raise ValueError("Multiple devices are not supported for TPUStrategy")
 
-    output = math_ops.add_n(value)
-    if reduce_op == reduce_util.ReduceOp.MEAN:
-      return output * (1. / len(value))
+    # Always performs the reduction on the TPU host.
+    with ops.device(self._host_device):
+      output = math_ops.add_n(value.values)
+      if reduce_op == reduce_util.ReduceOp.MEAN:
+        output *= (1. / len(value.values))
+
+    # If necessary, copy to requested destination.
+    dest_canonical = device_util.canonicalize(devices[0])
+    host_canonical = device_util.canonicalize(self._host_device)
+
+    if dest_canonical != host_canonical:
+      with ops.device(devices[0]):
+        output = array_ops.identity(output)
+
     return output
 
   def _update(self, var, fn, args, kwargs, group):
-- 
GitLab


From 8f0ab94274bb961245ebf25e9e447412a0b9b26c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 6 Feb 2019 09:09:26 -0800
Subject: [PATCH 1993/2345] Remove ineffective std::moves.

PiperOrigin-RevId: 232685841
---
 tensorflow/core/debug/debug_graph_utils.cc  | 2 +-
 tensorflow/core/grappler/utils/functions.cc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/debug/debug_graph_utils.cc b/tensorflow/core/debug/debug_graph_utils.cc
index 5fc95a8f20..b69eb1da39 100644
--- a/tensorflow/core/debug/debug_graph_utils.cc
+++ b/tensorflow/core/debug/debug_graph_utils.cc
@@ -299,7 +299,7 @@ Status DebugNodeInserter::CreateCopyNode(
 
   auto builder = NodeDefBuilder(copy_node_name, copy_op_name)
                      .Input(src_node_name, src_output, src_dt)
-                     .Attr("debug_ops_spec", std::move(debug_ops_spec));
+                     .Attr("debug_ops_spec", debug_ops_spec);
 
   if (!builder.Finalize(&node_def).ok()) {
     return Status(
diff --git a/tensorflow/core/grappler/utils/functions.cc b/tensorflow/core/grappler/utils/functions.cc
index 5f38aeac54..35979d4a23 100644
--- a/tensorflow/core/grappler/utils/functions.cc
+++ b/tensorflow/core/grappler/utils/functions.cc
@@ -314,7 +314,7 @@ GrapplerFunctionItem::GrapplerFunctionItem(
     std::vector<string> keep_nodes, const int graph_def_version,
     const bool is_stateful, GraphDef&& function_body)
     : description_(std::move(description)),
-      func_attr_(std::move(func_attr)),
+      func_attr_(func_attr),
       input_arg_expansions_(std::move(input_arg_expansions)),
       output_arg_expansions_(std::move(output_arg_expansions)),
       is_stateful_(is_stateful) {
-- 
GitLab


From d1b41c3e29f2d03946bd656f438f7f7768a10916 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 6 Feb 2019 09:16:41 -0800
Subject: [PATCH 1994/2345] Internal change.

PiperOrigin-RevId: 232686835
---
 tensorflow/core/BUILD | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 3b4fe3cc2d..7975b25b06 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -5082,7 +5082,6 @@ load("//tensorflow:tensorflow.bzl", "tf_portable_proto_library")
 
 genrule(
     name = "emscripten_proto_config_lite_runtime",
-    srcs = ["android_proto_config.asciipb"],  # Reuse the same file as android.
     outs = ["emscripten_proto_config_lite_runtime.asciipb"],
     cmd = tf_genrule_cmd_append_to_srcs("optimize_mode:LITE_RUNTIME"),
     visibility = ["//visibility:private"],
-- 
GitLab


From e360a071cf35f67981136f6d9f58d784c854c97d Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Wed, 6 Feb 2019 09:17:14 -0800
Subject: [PATCH 1995/2345] fix pip dependencies

PiperOrigin-RevId: 232686930
---
 tensorflow/tools/pip_package/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index 30a31a35c5..30ee5165af 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -77,6 +77,7 @@ COMMON_PIP_DEPS = [
     "//tensorflow/python:meta_graph_testdata",
     "//tensorflow/python:spectral_ops_test_util",
     "//tensorflow/python:util_example_parser_configuration",
+    "//tensorflow/python/data/benchmarks:benchmark_base",
     "//tensorflow/python/data/experimental/kernel_tests/serialization:dataset_serialization_test_base",
     "//tensorflow/python/data/experimental/kernel_tests:stats_dataset_test_base",
     "//tensorflow/python/data/kernel_tests:test_base",
-- 
GitLab


From c1f508a1e3bee9aae0eb2761511e743963d61659 Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Wed, 6 Feb 2019 09:28:50 -0800
Subject: [PATCH 1996/2345] Automated rollback of commit
 0db554ab7e31311bb6176cbcb7cb046c6a89fdcb

PiperOrigin-RevId: 232688479
---
 tensorflow/core/grappler/optimizers/BUILD     |   9 +-
 .../optimizers/dependency_optimizer.cc        | 515 ++++++++++--------
 .../optimizers/dependency_optimizer.h         |  30 +-
 .../optimizers/dependency_optimizer_test.cc   | 269 +++++----
 4 files changed, 445 insertions(+), 378 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 4c8b4f69d6..cb5d7d6746 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -1,7 +1,9 @@
 licenses(["notice"])  # Apache 2.0
 
-load("//tensorflow:tensorflow.bzl", "tf_cc_test", "tf_kernel_library")
+load("//tensorflow:tensorflow.bzl", "tf_cc_test")
 load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_test")
+load("//tensorflow:tensorflow.bzl", "tf_kernel_library")
+load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda")
 
 # Platform specific build config
 load(
@@ -304,19 +306,14 @@ cc_library(
         ":constant_folding",
         ":graph_optimizer",
         "//tensorflow/core:framework",
-        "//tensorflow/core:graph",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/grappler:grappler_item",
-        "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/costs:graph_properties",
         "//tensorflow/core/grappler/utils:topological_sort",
-        "@com_google_absl//absl/container:flat_hash_map",
-        "@com_google_absl//absl/container:flat_hash_set",
-        "@com_google_absl//absl/strings",
     ],
 )
 
diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer.cc b/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
index 9712ad63b0..8b81cb2430 100644
--- a/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
@@ -15,13 +15,13 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/dependency_optimizer.h"
 
-#include "absl/container/flat_hash_map.h"
+#include <unordered_map>
+#include <unordered_set>
+
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/op.h"
-#include "tensorflow/core/graph/tensor_id.h"
 #include "tensorflow/core/grappler/costs/graph_properties.h"
 #include "tensorflow/core/grappler/grappler_item.h"
-#include "tensorflow/core/grappler/mutable_graph_view.h"
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/grappler/optimizers/constant_folding.h"
 #include "tensorflow/core/grappler/utils.h"
@@ -38,15 +38,20 @@ namespace grappler {
 
 namespace {
 
-// Builds a map from the &graph->node(i) to i.
-absl::flat_hash_map<const NodeDef*, int> BuildNodeToIdx(const GraphDef& graph) {
-  // Set up &node -> index map.
-  absl::flat_hash_map<const NodeDef*, int> node_to_idx;
-  for (int i = 0; i < graph.node_size(); ++i) {
-    const NodeDef& node = graph.node(i);
-    node_to_idx[&node] = i;
+bool RemoveInput(NodeDef* node, const string& input, NodeMap* node_map) {
+  bool removed_input = false;
+  int pos = 0;
+  while (pos < node->input_size()) {
+    if (node->input(pos) == input) {
+      node->mutable_input()->SwapElements(pos, node->input_size() - 1);
+      node->mutable_input()->RemoveLast();
+      node_map->RemoveOutput(NodeName(input), node->name());
+      removed_input = true;
+    } else {
+      ++pos;
+    }
   }
-  return node_to_idx;
+  return removed_input;
 }
 
 }  // namespace
@@ -63,9 +68,7 @@ bool DependencyOptimizer::SafeToRemoveIdentity(const NodeDef& node) const {
     // The output values of this node may be needed.
     return false;
   }
-  MutableGraphView::OutputPort port = graph_view_->GetRegularFanin(
-      MutableGraphView::InputPort(const_cast<NodeDef*>(&node), 0));
-  NodeDef* input = port.node;
+  const NodeDef* input = node_map_->GetNode(NodeName(node.input(0)));
   CHECK(input != nullptr) << "node = " << node.name()
                           << " input = " << node.input(0);
   // Don't remove Identity nodes corresponding to Variable reads or following
@@ -76,23 +79,20 @@ bool DependencyOptimizer::SafeToRemoveIdentity(const NodeDef& node) const {
     // Don't turn Identity nodes following Switch into NoOp or remove them
     // if it requires anchoring a control dependencies the Switch node, which
     // is not valid.
-    MutableGraphView::OutputPort control_port(const_cast<NodeDef*>(&node),
-                                              Graph::kControlSlot);
-    if (!graph_view_->GetFanout(control_port).empty()) {
+    if (str_util::StartsWith(node.name(), kConstantFoldingCtrl)) {
+      // TODO(rmlarsen): Try to remove this artificial contraint.
       return false;
     }
   }
-  bool node_has_multiple_inputs =
-      graph_view_->NumFanins(node, /*include_controlling_nodes=*/true) > 1;
-  for (auto consumer :
-       graph_view_->GetFanouts(node, /*include_controlled_nodes=*/true)) {
-    if (node_has_multiple_inputs && IsMerge(*consumer.node)) {
+  for (auto consumer : node_map_->GetOutputs(node.name())) {
+    if (node.input_size() > 1 && IsMerge(*consumer)) {
       return false;
     }
     if (IsSwitch(*input)) {
-      if (graph_view_->HasFanin(*consumer.node,
-                                {node.name(), Graph::kControlSlot})) {
-        return false;
+      for (const string& consumer_input : consumer->input()) {
+        if (consumer_input == AsControlDependency(node.name())) {
+          return false;
+        }
       }
     }
   }
@@ -116,7 +116,7 @@ bool DependencyOptimizer::SafeToConvertToNoOp(const NodeDef& node) const {
   if (!status.ok() || op_def->output_arg_size() == 0) {
     return false;
   }
-  const absl::flat_hash_set<string> do_not_rewrite_ops{
+  const std::unordered_set<string> do_not_rewrite_ops{
       "Assert",     "CheckNumerics",         "_Retval",
       "_Arg",       "_ParallelConcatUpdate", "TPUExecute",
       "TPUCompile", "ControlTrigger"};
@@ -126,7 +126,7 @@ bool DependencyOptimizer::SafeToConvertToNoOp(const NodeDef& node) const {
   if (!SafeToRemoveIdentity(node)) {
     return false;
   }
-  if (graph_view_->NumFanouts(node, /*include_controlled_nodes=*/false) > 0) {
+  if (NumNonControlOutputs(node, *node_map_) > 0) {
     // The output values of this node may be needed.
     return false;
   }
@@ -134,58 +134,61 @@ bool DependencyOptimizer::SafeToConvertToNoOp(const NodeDef& node) const {
 }
 
 int DependencyOptimizer::NumEdgesIfBypassed(
-    const NodeDef& node,
-    const absl::flat_hash_set<MutableGraphView::Edge>& fanout_edges) const {
+    const NodeDef& node, const std::vector<NodeDef*>& output_nodes) const {
   const bool is_multi_input_identity_n =
       IsIdentityN(node) && !IsIdentityNSingleInput(node);
-  const int num_fanins = node.input_size();
-  const int num_fanouts = fanout_edges.size();
+  const int num_outputs = output_nodes.size();
+  const int num_inputs = node.input_size();
 
   if (is_multi_input_identity_n) {
     // multi-input identity_n with input/output control dependencies will likely
     // increase number of edges after optimization.
     int num_edges_if_bypassed(0);
-    int num_non_controlling_fanins =
-        graph_view_->NumFanins(node, /*include_controlling_nodes=*/false);
-    num_edges_if_bypassed += num_non_controlling_fanins;
-    num_edges_if_bypassed +=
-        (num_fanins - num_non_controlling_fanins) * num_fanouts;
-
-    for (const auto& fanout : fanout_edges) {
-      if (fanout.dst.port_id == Graph::kControlSlot) {
-        num_edges_if_bypassed += num_fanins;
+    for (string input_node_name : node.input()) {
+      if (IsControlInput(input_node_name)) {
+        num_edges_if_bypassed += num_outputs;
       } else {
         ++num_edges_if_bypassed;
       }
     }
+
+    for (auto consumer : output_nodes) {
+      for (int j = 0; j < consumer->input_size(); ++j) {
+        const TensorId consumer_input = ParseTensorName(consumer->input(j));
+        if (consumer_input.node() == node.name()) {
+          if (IsControlInput(consumer_input)) {
+            num_edges_if_bypassed += num_inputs;
+          } else {
+            ++num_edges_if_bypassed;
+          }
+        }
+      }
+    }
     return num_edges_if_bypassed;
   } else {
-    return num_fanins * num_fanouts;
+    return num_inputs * num_outputs;
   }
 }
 
 bool DependencyOptimizer::BypassingNodeIsBeneficial(
-    const NodeDef& node,
-    const absl::flat_hash_set<MutableGraphView::Edge>& fanout_edges) const {
+    const NodeDef& node, const std::vector<NodeDef*>& input_nodes,
+    const std::vector<NodeDef*>& output_nodes) const {
   const bool is_identity = IsIdentity(node) || IsIdentityNSingleInput(node);
   const bool is_multi_input_identity_n =
       IsIdentityN(node) && !IsIdentityNSingleInput(node);
-  const int num_fanins = node.input_size();
-  const int num_fanouts = fanout_edges.size();
+  const int num_outputs = output_nodes.size();
+  const int num_inputs = node.input_size();
 
-  if (NumEdgesIfBypassed(node, fanout_edges) > num_fanins + num_fanouts) {
+  if (NumEdgesIfBypassed(node, output_nodes) > num_inputs + num_outputs) {
     return false;
   }
 
-  auto fanin_edges =
-      graph_view_->GetFaninEdges(node, /*include_controlling_edges=*/true);
-
   // Make sure that we don't increase the number of edges that cross
   // device boundaries.
-  if ((num_fanins == 1 && num_fanouts > 1 &&
-       fanin_edges.begin()->src.node->device() != node.device()) ||
-      (num_fanins > 1 && num_fanouts == 1 &&
-       fanout_edges.begin()->dst.node->device() != node.device())) {
+  if ((num_inputs == 1 && num_outputs > 1 &&
+       input_nodes[0]->device() != node.device()) ||
+      (num_inputs > 1 && num_outputs == 1 &&
+       output_nodes[0]->device() != node.device())) {
     return false;
   }
 
@@ -194,21 +197,21 @@ bool DependencyOptimizer::BypassingNodeIsBeneficial(
   // cost before and after.
   const string& node_dev = node.device();
   int num_cross_in = 0;
-  for (const auto& fanin : fanin_edges) {
-    num_cross_in += static_cast<int>(fanin.src.node->device() != node_dev);
+  for (NodeDef* input_node : input_nodes) {
+    num_cross_in += static_cast<int>(input_node->device() != node_dev);
   }
   int num_cross_out = 0;
-  for (const auto& fanout : fanout_edges) {
-    num_cross_out += static_cast<int>(fanout.dst.node->device() != node_dev);
+  for (NodeDef* output_node : output_nodes) {
+    num_cross_out += static_cast<int>(output_node->device() != node_dev);
   }
 
   // Make sure we do not increase the number of device crossings.
   const int num_cross_before = num_cross_in + num_cross_out;
   int num_cross_after = 0;
-  for (const auto& fanin : fanin_edges) {
-    for (const auto& fanout : fanout_edges) {
-      num_cross_after += static_cast<int>(fanin.src.node->device() !=
-                                          fanout.dst.node->device());
+  for (NodeDef* input_node : input_nodes) {
+    for (NodeDef* output_node : output_nodes) {
+      num_cross_after +=
+          static_cast<int>(input_node->device() != output_node->device());
     }
   }
   if (num_cross_after > num_cross_before) {
@@ -226,34 +229,47 @@ bool DependencyOptimizer::BypassingNodeIsBeneficial(
   return true;
 }
 
-Status DependencyOptimizer::OptimizeNode(
-    const string& node_name, SetVector<string>* nodes_to_simplify,
-    absl::flat_hash_set<string>* nodes_to_delete) {
-  NodeDef* node = graph_view_->GetNode(node_name);
+void DependencyOptimizer::OptimizeNode(int node_idx,
+                                       SetVector<int>* nodes_to_simplify,
+                                       std::set<int>* nodes_to_delete) {
+  NodeDef* node = optimized_graph_->mutable_node(node_idx);
   const bool is_noop = IsNoOp(*node);
   const bool is_identity = IsIdentity(*node) || IsIdentityNSingleInput(*node);
   const bool is_multi_input_identity =
       IsIdentityN(*node) && !IsIdentityNSingleInput(*node);
-  // WARNING: This is a strong assumption based on the executor behavior that
-  // constant nodes with no input control dependency are always executed early.
-  // In this case we then can prune all their output control dependencies.
-  if (IsConstant(*node) &&
-      graph_view_->NumFanins(*node, /*include_controlling_nodes=*/true) == 0) {
-    for (const auto& fanout :
-         graph_view_->GetFanouts(*node, /*include_controlled_nodes=*/true)) {
-      TF_RETURN_IF_ERROR(
-          graph_view_->RemoveControllingFanin(fanout.node->name(), node_name));
-      nodes_to_simplify->PushBack(fanout.node->name());
+  const string node_name = node->name();
+  // Constant nodes with no input control dependency are always executed early,
+  // so we can prune all their output control dependencies.
+  if (IsConstant(*node) && node->input_size() == 0) {
+    const std::set<NodeDef*> output_nodes = node_map_->GetOutputs(node_name);
+    for (NodeDef* fanout : output_nodes) {
+      bool optimize_fanout = false;
+      bool data_connection = false;
+      for (int i = fanout->input_size() - 1; i >= 0; --i) {
+        const TensorId input_tensor = ParseTensorName(fanout->input(i));
+        if (input_tensor.node() == node_name) {
+          if (input_tensor.index() < 0) {
+            fanout->mutable_input()->SwapElements(i, fanout->input_size() - 1);
+            fanout->mutable_input()->RemoveLast();
+            optimize_fanout = true;
+          } else {
+            data_connection = true;
+          }
+        }
+      }
+      if (optimize_fanout) {
+        nodes_to_simplify->PushBack(node_to_idx_[fanout]);
+        if (!data_connection) {
+          node_map_->RemoveOutput(node_name, fanout->name());
+        }
+      }
     }
-
-    if (graph_view_->NumFanouts(*node, /*include_controlled_nodes=*/true) ==
-            0 &&
-        fetch_nodes_known_ &&
+    if (node_map_->GetOutputs(node_name).empty() && fetch_nodes_known_ &&
         nodes_to_preserve_.find(node_name) == nodes_to_preserve_.end()) {
       // Mark the node for deletion.
-      nodes_to_delete->insert(node->name());
+      nodes_to_delete->insert(node_to_idx_[node]);
     }
-    return Status::OK();
+    return;
   }
 
   // Change ops that only have control dependencies as outputs to NoOps.
@@ -262,21 +278,34 @@ Status DependencyOptimizer::OptimizeNode(
             << ") with NoOp.";
     // The outputs of this node are not consumed. Replace its inputs with
     // control dependencies and replace the op itself with the NoOp op.
-    const int num_regular_fanins =
-        graph_view_->NumFanins(*node, /*include_controlling_nodes=*/false);
-    absl::flat_hash_set<string> regular_fanin_names;
-    for (int i = 0; i < num_regular_fanins; ++i) {
-      regular_fanin_names.emplace(ParseTensorName(node->input(i)).node());
-    }
-    TF_RETURN_IF_ERROR(
-        graph_view_->UpdateAllRegularFaninsToControlling(node_name));
-    TF_RETURN_IF_ERROR(
-        graph_view_->UpdateNode(node->name(), "NoOp", node->device(), {}));
-    nodes_to_simplify->PushBack(node->name());
-    for (const string& regular_fanin_name : regular_fanin_names) {
-      nodes_to_simplify->PushBack(regular_fanin_name);
+    std::unordered_set<string> ctrl_inputs;
+    int pos = 0;
+    while (pos < node->input_size()) {
+      const string old_input = node->input(pos);
+      if (IsControlInput(old_input)) {
+        if (!ctrl_inputs.insert(old_input).second) {
+          // We found a duplicate control input. Remove it.
+          node->mutable_input()->SwapElements(pos, node->input_size() - 1);
+          node->mutable_input()->RemoveLast();
+        } else {
+          ++pos;
+        }
+        continue;
+      }
+      // Replace a normal input with a control input.
+      const string ctrl_input = ConstantFolding::AddControlDependency(
+          old_input, optimized_graph_, node_map_.get());
+      ctrl_inputs.insert(ctrl_input);
+      node->set_input(pos, ctrl_input);
+      node_map_->UpdateInput(node_name, old_input, ctrl_input);
+      const NodeDef* old_input_node = node_map_->GetNode(old_input);
+      nodes_to_simplify->PushBack(node_to_idx_[old_input_node]);
+      ++pos;
     }
-    return Status::OK();
+    node->set_op("NoOp");
+    node->clear_attr();
+    nodes_to_simplify->PushBack(node_to_idx_[node]);
+    return;
   }
 
   // Remove NoOp nodes if the product of their fan-in and fan-out is less than
@@ -329,114 +358,128 @@ Status DependencyOptimizer::OptimizeNode(
 
   if (is_noop || ((is_identity || is_multi_input_identity) &&
                   SafeToRemoveIdentity(*node))) {
-    auto fanout_edges =
-        graph_view_->GetFanoutEdges(*node, /*include_controlled_edges=*/true);
-    if (!BypassingNodeIsBeneficial(*node, fanout_edges)) {
-      return Status::OK();
+    const auto& output_node_set = node_map_->GetOutputs(node_name);
+    const std::vector<NodeDef*> output_nodes(output_node_set.begin(),
+                                             output_node_set.end());
+    const int num_inputs = node->input_size();
+    std::vector<NodeDef*> input_nodes;
+    for (int i = 0; i < num_inputs; ++i) {
+      NodeDef* input_node = node_map_->GetNode(node->input(i));
+      if (input_node == nullptr) {
+        LOG(ERROR) << "Invalid input " << node->input(i);
+        return;
+      }
+      input_nodes.push_back(input_node);
     }
 
-    if (!fanout_edges.empty()) {
-      const int num_regular_fanins =
-          graph_view_->NumFanins(*node, /*include_controlling_nodes=*/false);
-      std::vector<TensorId> regular_fanin_ids;
-      regular_fanin_ids.reserve(num_regular_fanins);
-      for (int i = 0; i < num_regular_fanins; ++i) {
-        regular_fanin_ids.push_back(ParseTensorName(node->input(i)));
-      }
+    if (!BypassingNodeIsBeneficial(*node, input_nodes, output_nodes)) {
+      return;
+    }
 
-      absl::flat_hash_set<NodeDef*> processed_consumers;
-      VLOG(1) << "***** Rerouting input around\n" << node->DebugString();
-      // Now remove the node and re-wire its inputs to its outputs.
-      for (auto fanout_edge : fanout_edges) {
-        NodeDef* consumer = fanout_edge.dst.node;
-        VLOG(1) << "consumer before:\n" << consumer->DebugString();
-        if (fanout_edge.src.port_id == Graph::kControlSlot) {
-          TF_RETURN_IF_ERROR(
-              graph_view_->RemoveControllingFanin(consumer->name(), node_name));
-          // Add all non controlling fanins of node as controlling fanins to
-          // consumer.
-          for (const TensorId& regular_fanin_id : regular_fanin_ids) {
-            TF_RETURN_IF_ERROR(graph_view_->AddControllingFanin(
-                consumer->name(), regular_fanin_id));
-            nodes_to_simplify->PushBack(string(regular_fanin_id.node()));
+    VLOG(1) << "***** Rerouting input around\n" << node->DebugString();
+    // Now remove the node and re-wire its inputs to its outputs.
+    for (auto consumer : output_nodes) {
+      bool updated_consumer = false;
+      VLOG(1) << "consumer before:\n" << consumer->DebugString();
+      for (int i = 0; i < num_inputs; ++i) {
+        const NodeDef* input = input_nodes[i];
+        // Forward dependency from input to consumer if it doesn't already
+        // depend on it.
+        if ((is_identity && i == 0) ||
+            (is_multi_input_identity && !IsControlInput(node->input(i)))) {
+          // Replace regular input from Identity node.
+          string new_input;
+          const string& input_to_forward = node->input(i);
+          CHECK(!IsControlInput(input_to_forward));
+          for (int j = 0; j < consumer->input_size(); ++j) {
+            const TensorId old_input = ParseTensorName(consumer->input(j));
+            if (old_input.node() == node_name) {
+              if (old_input.index() == i) {
+                // Regular input
+                new_input = input_to_forward;
+                node_map_->UpdateInput(consumer->name(), old_input.ToString(),
+                                       new_input);
+                consumer->set_input(j, new_input);
+              } else if (old_input.index() == -1) {
+                // Control dependency
+                new_input = AsControlDependency(NodeName(input_to_forward));
+                node_map_->UpdateInput(consumer->name(), old_input.ToString(),
+                                       new_input);
+                consumer->set_input(j, new_input);
+              }
+            }
           }
+          updated_consumer = true;
         } else {
-          TF_RETURN_IF_ERROR(graph_view_->UpdateRegularFaninByPort(
-              consumer->name(), fanout_edge.dst.port_id,
-              ParseTensorName(node->input(fanout_edge.src.port_id))));
-        }
-        processed_consumers.insert(consumer);
-        VLOG(1) << "consumer after:\n" << consumer->DebugString();
-      }
-
-      std::vector<TensorId> controlling_fanin_ids;
-      controlling_fanin_ids.reserve(node->input_size() - num_regular_fanins);
-      for (int i = num_regular_fanins; i < node->input_size(); ++i) {
-        controlling_fanin_ids.push_back(ParseTensorName(node->input(i)));
-      }
-
-      for (NodeDef* consumer : processed_consumers) {
-        // Forward all controlling fanins of node to consumer.
-        VLOG(1) << "consumer before:\n" << consumer->DebugString();
-        for (const TensorId& controlling_fanin_id : controlling_fanin_ids) {
-          TF_RETURN_IF_ERROR(graph_view_->AddControllingFanin(
-              consumer->name(), controlling_fanin_id));
+          // Forward dependency from input to consumer if it doesn't already
+          // depend on it.
+          if (node_map_->GetOutputs(input->name()).count(consumer) == 0) {
+            consumer->add_input(AsControlDependency(input->name()));
+            node_map_->AddOutput(input->name(), consumer->name());
+            nodes_to_simplify->PushBack(node_to_idx_[input]);
+            updated_consumer = true;
+          }
         }
-        nodes_to_simplify->PushBack(consumer->name());
-        VLOG(1) << "consumer after:\n" << consumer->DebugString();
       }
-      for (const TensorId& controlling_fanin_id : controlling_fanin_ids) {
-        nodes_to_simplify->PushBack(string(controlling_fanin_id.node()));
+      // Remove dependency on node from consumer.
+      updated_consumer |= RemoveInput(consumer, AsControlDependency(node_name),
+                                      node_map_.get());
+      if (updated_consumer) {
+        nodes_to_simplify->PushBack(node_to_idx_[consumer]);
       }
+      VLOG(1) << "consumer after:\n" << consumer->DebugString();
     }
-
+    node_map_->RemoveOutputs(node_name);
     if (fetch_nodes_known_ &&
         nodes_to_preserve_.find(node_name) == nodes_to_preserve_.end()) {
       // Mark the node for deletion.
-      nodes_to_delete->insert(node_name);
+      nodes_to_delete->insert(node_idx);
 
       // Disconnect the node from its inputs to enable further optimizations.
-      TF_RETURN_IF_ERROR(
-          graph_view_->RemoveAllFanins(node_name,
-                                       /*keep_controlling_fanins=*/false));
+      node_map_->RemoveInputs(node_name);
+      node->clear_input();
     }
   }
-  return Status::OK();
+}
+
+void DependencyOptimizer::CleanControlInputs() {
+  for (int i = 0; i < optimized_graph_->node_size(); ++i) {
+    DedupControlInputs(optimized_graph_->mutable_node(i));
+  }
 }
 
 Status DependencyOptimizer::OptimizeDependencies() {
-  SetVector<string> nodes_to_simplify;
-  absl::flat_hash_set<string> nodes_to_delete;
-  for (int i = 0; i < graph_view_->graph()->node_size(); ++i) {
-    const NodeDef& node = graph_view_->graph()->node(i);
+  SetVector<int> nodes_to_simplify;
+  std::set<int> nodes_to_delete;
+  for (int i = 0; i < optimized_graph_->node_size(); ++i) {
+    const NodeDef& node = optimized_graph_->node(i);
     if (IsNoOp(node) || IsIdentity(node) || IsIdentityN(node) ||
         IsConstant(node) || SafeToConvertToNoOp(node)) {
-      nodes_to_simplify.PushBack(node.name());
+      nodes_to_simplify.PushBack(i);
     }
   }
   while (!nodes_to_simplify.Empty()) {
-    string node_to_simplify = nodes_to_simplify.PopBack();
+    int node_to_simplify = nodes_to_simplify.PopBack();
     // Discard nodes that were marked for deletion already.
     while (nodes_to_delete.find(node_to_simplify) != nodes_to_delete.end()) {
       node_to_simplify = nodes_to_simplify.PopBack();
     }
-    TF_RETURN_IF_ERROR(
-        OptimizeNode(node_to_simplify, &nodes_to_simplify, &nodes_to_delete));
+    OptimizeNode(node_to_simplify, &nodes_to_simplify, &nodes_to_delete);
   }
 
   if (fetch_nodes_known_) {
     VLOG(1) << "Deleted " << nodes_to_delete.size() << " out of "
-            << graph_view_->graph()->node_size() << " nodes.";
-    TF_RETURN_IF_ERROR(graph_view_->DeleteNodes(nodes_to_delete));
+            << optimized_graph_->node_size() << " nodes.";
+    EraseNodesFromGraph(nodes_to_delete, optimized_graph_);
+    node_map_.reset(new NodeMap(optimized_graph_));
+    BuildNodeToIdx();
   }
   return Status::OK();
 }
 
 Status DependencyOptimizer::TransitiveReduction() {
   // PRECONDITION: optimized_graph_ must be sorted topologically.
-  GraphDef* graph = graph_view_->graph();
-  auto node_to_idx = BuildNodeToIdx(*graph);
-  const int num_nodes = graph->node_size();
+  const int num_nodes = optimized_graph_->node_size();
   // Set up a compressed version of the graph to save a constant factor in the
   // expensive algorithm below. Also cache the set of control outputs and the
   // highest index of a target of any control output from each node.
@@ -445,20 +488,20 @@ Status DependencyOptimizer::TransitiveReduction() {
   std::vector<gtl::InlinedVector<std::pair<int, int>, 2>> control_outputs(
       num_nodes);
   for (int node_idx = 0; node_idx < num_nodes; ++node_idx) {
-    const NodeDef& node = graph->node(node_idx);
+    const NodeDef& node = optimized_graph_->node(node_idx);
     if (ModifiesFrameInfo(node) || !HasOpDef(node)) {
       // Ignore function nodes and nodes that modify frame info.
       continue;
     }
     for (int input_slot = 0; input_slot < node.input_size(); ++input_slot) {
       const string& input = node.input(input_slot);
-      const NodeDef* input_node = graph_view_->GetNode(NodeName(input));
+      const NodeDef* input_node = node_map_->GetNode(input);
       if (ModifiesFrameInfo(*input_node) || IsMerge(*input_node)) {
         // Ignore edges from nodes that modify frame info and from Merge nodes,
         // because we cannot know which of it's input paths executes.
         continue;
       }
-      const int input_node_idx = node_to_idx[input_node];
+      const int input_node_idx = node_to_idx_[input_node];
       inputs[node_idx].push_back(input_node_idx);
       if (IsControlInput(input)) {
         ++num_controls;
@@ -477,7 +520,7 @@ Status DependencyOptimizer::TransitiveReduction() {
   // such that when we swap them out so we don't clobber the
   // node(target).input() repeated field.
   typedef std::pair<int, int> InputSlotAndSource;
-  absl::flat_hash_map<
+  std::unordered_map<
       int, std::set<InputSlotAndSource, std::greater<InputSlotAndSource>>>
       control_edges_to_remove;
   for (int source = 0; source < num_nodes; ++source) {
@@ -524,13 +567,16 @@ Status DependencyOptimizer::TransitiveReduction() {
 
   for (const auto& it : control_edges_to_remove) {
     const int target = it.first;
-    const NodeDef& target_node = graph->node(target);
-    const string target_node_name = target_node.name();
+    NodeDef* target_node = optimized_graph_->mutable_node(target);
     for (const InputSlotAndSource& slot_and_source : it.second) {
       const int input_slot = slot_and_source.first;
-      const TensorId tensor_id = ParseTensorName(target_node.input(input_slot));
-      TF_RETURN_IF_ERROR(graph_view_->RemoveControllingFanin(target_node_name,
-                                                             tensor_id.node()));
+      const int source = slot_and_source.second;
+      const NodeDef& source_node = optimized_graph_->node(source);
+      CHECK_LT(input_slot, target_node->input_size());
+      target_node->mutable_input()->SwapElements(input_slot,
+                                                 target_node->input_size() - 1);
+      node_map_->RemoveOutput(source_node.name(), target_node->name());
+      target_node->mutable_input()->RemoveLast();
       ++num_controls_removed;
     }
   }
@@ -539,17 +585,26 @@ Status DependencyOptimizer::TransitiveReduction() {
   return Status::OK();
 }
 
+void DependencyOptimizer::BuildNodeToIdx() {
+  // Set up &node -> index map.
+  node_to_idx_.clear();
+  for (int i = 0; i < optimized_graph_->node_size(); ++i) {
+    const NodeDef& node = optimized_graph_->node(i);
+    node_to_idx_[&node] = i;
+  }
+}
+
 // Suppose there are cross-device control inputs to node C from multiple nodes
 // that are located on another device, e.g., we have control edges:
 // A->C, B->C
 // where A and B are on device X and C is on device Y.
 // We can reduce cross-device communication by introducing an intermediate
 // NoOp node C' on device X and rewriting the control edges to:
-// A->C', B->C', C'->C
-Status DependencyOptimizer::GroupCrossDeviceControlEdges() {
-  const int num_nodes = graph_view_->graph()->node_size();
+// A->C', B->C', C' -> C
+void DependencyOptimizer::GroupCrossDeviceControlEdges() {
+  const int num_nodes = optimized_graph_->node_size();
   for (int i = 0; i < num_nodes; ++i) {
-    NodeDef* node = graph_view_->graph()->mutable_node(i);
+    NodeDef* node = optimized_graph_->mutable_node(i);
     if (node->device().empty()) continue;
 
     // Creates new noop nodes for devices on which multiple control inputs are
@@ -560,71 +615,88 @@ Status DependencyOptimizer::GroupCrossDeviceControlEdges() {
     // that device.
     std::map<string, NodeDef*> noops;
     int num_noops = 0;
-    auto controlling_fanins = graph_view_->GetFanin(
-        MutableGraphView::InputPort(node, Graph::kControlSlot));
-    for (const auto& controlling_fanin : controlling_fanins) {
-      const NodeDef* fanin_node = controlling_fanin.node;
-      if (!fanin_node->device().empty() &&
-          fanin_node->device() != node->device()) {
-        auto emplace_result = noops.emplace(fanin_node->device(), nullptr);
-        if (!emplace_result.second && emplace_result.first->second == nullptr) {
-          // This is the second cross-device control input from the same
-          // device. Creates an intermediate noop node on that device.
-          string group_name;
-          NodeDef* noop;
-          // Creates a fresh node name; there may be conflicting names from
-          // a previous iteration of the optimizer.
-          do {
-            group_name = AddPrefixToNodeName(
-                node->name(),
-                strings::StrCat("GroupCrossDeviceControlEdges_", num_noops));
-            noop = graph_view_->GetNode(group_name);
-            ++num_noops;
-          } while (noop != nullptr);
-          NodeDef new_node;
-          new_node.set_name(group_name);
-          new_node.set_device(fanin_node->device());
-          new_node.set_op("NoOp");
-          emplace_result.first->second =
-              graph_view_->AddNode(std::move(new_node));
+    for (int j = 0; j < node->input_size(); ++j) {
+      if (IsControlInput(node->input(j))) {
+        const NodeDef* input = node_map_->GetNode(node->input(j));
+        if (input != nullptr && !input->device().empty() &&
+            input->device() != node->device()) {
+          auto emplace_result = noops.emplace(input->device(), nullptr);
+          if (!emplace_result.second &&
+              emplace_result.first->second == nullptr) {
+            // This is the second cross-device control input from the same
+            // device. Creates an intermediate noop node on that device.
+            string group_name;
+            NodeDef* noop;
+            // Creates a fresh node name; there may be conflicting names from
+            // a previous iteration of the optimizer.
+            do {
+              group_name = AddPrefixToNodeName(
+                  node->name(),
+                  strings::StrCat("GroupCrossDeviceControlEdges_", num_noops));
+              noop = node_map_->GetNode(group_name);
+              ++num_noops;
+            } while (noop != nullptr);
+            noop = optimized_graph_->add_node();
+            noop->set_name(group_name);
+            noop->set_device(input->device());
+            noop->set_op("NoOp");
+            node_map_->AddNode(noop->name(), noop);
+            emplace_result.first->second = noop;
+          }
         }
       }
     }
 
     // Reroute existing control edges to go via the newly introduced NoOp nodes.
-    for (const auto& controlling_fanin : controlling_fanins) {
-      auto it = noops.find(controlling_fanin.node->device());
-      if (it != noops.end() && it->second != nullptr) {
-        TF_RETURN_IF_ERROR(graph_view_->RemoveControllingFanin(
-            node->name(), controlling_fanin.node->name()));
-        TF_RETURN_IF_ERROR(graph_view_->AddControllingFanin(
-            it->second->name(),
-            {controlling_fanin.node->name(), Graph::kControlSlot}));
+    int pos = 0;
+    while (pos < node->input_size()) {
+      const string& input_name = node->input(pos);
+      if (IsControlInput(input_name)) {
+        NodeDef* input = node_map_->GetNode(input_name);
+        if (input == nullptr) {
+          ++pos;
+        } else {
+          auto it = noops.find(input->device());
+          if (it == noops.end() || it->second == nullptr) {
+            ++pos;
+          } else {
+            node->mutable_input()->SwapElements(pos, node->input_size() - 1);
+            node->mutable_input()->RemoveLast();
+            it->second->add_input(AsControlDependency(*input));
+            node_map_->UpdateOutput(input_name, node->name(),
+                                    it->second->name());
+          }
+        }
+      } else {
+        ++pos;
       }
     }
     for (const auto& entry : noops) {
       if (entry.second) {
-        TF_RETURN_IF_ERROR(graph_view_->AddControllingFanin(
-            node->name(), {entry.second->name(), Graph::kControlSlot}));
+        node->add_input(AsControlDependency(*entry.second));
+        node_map_->AddOutput(entry.second->name(), node->name());
       }
     }
   }
-  return Status::OK();
 }
 
 Status DependencyOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
                                      GraphDef* optimized_graph) {
-  *optimized_graph = item.graph;
+  optimized_graph_ = optimized_graph;
+  *optimized_graph_ = item.graph;
   nodes_to_preserve_ = item.NodesToPreserve();
   fetch_nodes_known_ = !item.fetch.empty();
-  graph_view_.reset(new MutableGraphView(optimized_graph));
+  CleanControlInputs();
 
   const int num_iterations = 2;
   for (int iteration = 0; iteration < num_iterations; ++iteration) {
     GRAPPLER_RETURN_IF_DEADLINE_EXCEEDED();
     Status topo_sort_status;
     // Perform topological sort to prepare the graph for transitive reduction.
-    topo_sort_status = TopologicalSort(optimized_graph);
+    topo_sort_status = TopologicalSort(optimized_graph_);
+    // Set up index-based graph datastructures to speed up analysis steps below.
+    node_map_.reset(new NodeMap(optimized_graph_));
+    BuildNodeToIdx();
 
     if (topo_sort_status.ok()) {
       // Remove redundant control dependencies.
@@ -638,7 +710,10 @@ Status DependencyOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
     // nodes.
     TF_RETURN_IF_ERROR(OptimizeDependencies());
 
-    TF_RETURN_IF_ERROR(GroupCrossDeviceControlEdges());
+    // Dedup control inputs.
+    CleanControlInputs();
+
+    GroupCrossDeviceControlEdges();
   }
 
   return Status::OK();
diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer.h b/tensorflow/core/grappler/optimizers/dependency_optimizer.h
index 98b69b4b7a..99021b955f 100644
--- a/tensorflow/core/grappler/optimizers/dependency_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/dependency_optimizer.h
@@ -17,10 +17,6 @@ limitations under the License.
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DEPENDENCY_OPTIMIZER_H_
 
 #include <unordered_set>
-
-#include "absl/container/flat_hash_set.h"
-#include "absl/strings/string_view.h"
-#include "tensorflow/core/grappler/mutable_graph_view.h"
 #include "tensorflow/core/grappler/optimizers/graph_optimizer.h"
 #include "tensorflow/core/grappler/utils.h"
 #include "tensorflow/core/protobuf/rewriter_config.pb.h"
@@ -49,22 +45,24 @@ class DependencyOptimizer : public GraphOptimizer {
   // Returns true if bypassing node does not increase the number of edges or
   // number of edges crossing a device boundary.
   bool BypassingNodeIsBeneficial(
-      const NodeDef& node,
-      const absl::flat_hash_set<MutableGraphView::Edge>& fanout_edges) const;
-  int NumEdgesIfBypassed(
-      const NodeDef& node,
-      const absl::flat_hash_set<MutableGraphView::Edge>& fanout_edges) const;
+      const NodeDef& node, const std::vector<NodeDef*>& input_nodes,
+      const std::vector<NodeDef*>& output_nodes) const;
+  int NumEdgesIfBypassed(const NodeDef& node,
+                         const std::vector<NodeDef*>& output_nodes) const;
   // Returns true if node is not an Identity node or if it is an Identity
   // that is safe to remove.
   bool SafeToRemoveIdentity(const NodeDef& node) const;
   // Returns true if it is safe to convert node to NoOp.
   bool SafeToConvertToNoOp(const NodeDef& node) const;
-  // Tries to optimize the node with the given node name, possibly additional
+  // Removes all duplicate control dependencies.
+  void CleanControlInputs();
+  // Builds a map from the &optimized_graph_->node(i) to i.
+  void BuildNodeToIdx();
+  // Tries to optimize the node with the given index, possibly additional
   // optimizations by inserting nodes in nodes_to_simplify, and pruning nodes by
   // inserting them in nodes_to_delete.
-  Status OptimizeNode(const string& node_name,
-                      SetVector<string>* nodes_to_simplify,
-                      absl::flat_hash_set<string>* nodes_to_delete);
+  void OptimizeNode(int node_idx, SetVector<int>* nodes_to_simplify,
+                    std::set<int>* nodes_to_delete);
   // Eliminates redundant control dependencies by computing the transitive
   // reduction of the graph.
   Status TransitiveReduction();
@@ -72,11 +70,13 @@ class DependencyOptimizer : public GraphOptimizer {
   Status OptimizeDependencies();
   // Replaces multiple cross-device control edges from the same device with a
   // single control edge.
-  Status GroupCrossDeviceControlEdges();
+  void GroupCrossDeviceControlEdges();
 
   bool fetch_nodes_known_;
   std::unordered_set<string> nodes_to_preserve_;
-  std::unique_ptr<MutableGraphView> graph_view_;
+  std::unique_ptr<NodeMap> node_map_;
+  std::unordered_map<const NodeDef*, int> node_to_idx_;
+  GraphDef* optimized_graph_;  // Not owned.
 };
 
 }  // end namespace grappler
diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc b/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
index 59512d8b5d..5883fcb926 100644
--- a/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
@@ -41,7 +41,7 @@ void VerifyGraphsEqual(const GraphDef& original_graph,
     const NodeDef& optimized = optimized_graph.node(i);
     EXPECT_EQ(original.name(), optimized.name()) << func;
     EXPECT_EQ(original.op(), optimized.op()) << func;
-    ASSERT_EQ(original.input_size(), optimized.input_size()) << func;
+    EXPECT_EQ(original.input_size(), optimized.input_size()) << func;
     for (int j = 0; j < original.input_size(); ++j) {
       EXPECT_EQ(original.input(j), optimized.input(j)) << func;
     }
@@ -89,12 +89,12 @@ TEST_F(DependencyOptimizerTest, DependenciesDrivenByConstants) {
   TF_EXPECT_OK(status);
 
   // The 'z' node should have been optimized away leaving only 5 nodes.
-  EXPECT_EQ(output.node_size(), 5);
+  EXPECT_EQ(5, output.node_size());
 
-  for (const NodeDef& node : output.node()) {
+  for (const NodeDef& node : item.graph.node()) {
     if (node.name() == "id1" || node.name() == "id2") {
-      ASSERT_EQ(node.input_size(), 1);
-      EXPECT_EQ(node.input(0), "add");
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("add", node.input(0));
     }
   }
 }
@@ -123,30 +123,30 @@ TEST_F(DependencyOptimizerTest, ChangeToNoop) {
   status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
-  EXPECT_EQ(output.node_size(), item.graph.node_size());
+  EXPECT_EQ(item.graph.node_size(), output.node_size());
   int found = 0;
-  for (int i = 0; i < output.node_size(); ++i) {
-    const NodeDef& node = output.node(i);
+  for (int i = 0; i < item.graph.node_size(); ++i) {
+    const NodeDef& node = item.graph.node(i);
     // "add" should get turned into a NoOp and removed.
-    EXPECT_NE(node.name(), "add");
+    EXPECT_NE("add", node.name());
     if (node.name() == "id1") {
-      EXPECT_EQ(node.op(), "Identity");
-      ASSERT_EQ(node.input_size(), 2);
-      EXPECT_EQ(node.input(0), "x");
-      EXPECT_EQ(node.input(1), "^y");
+      EXPECT_EQ("Identity", node.op());
+      EXPECT_EQ(2, node.input_size());
+      EXPECT_EQ("x", node.input(0));
+      EXPECT_EQ("^y", node.input(1));
       ++found;
     } else if (node.name() == "id2") {
-      EXPECT_EQ(node.op(), "Identity");
-      ASSERT_EQ(node.input_size(), 2);
-      EXPECT_EQ(node.input(0), "y");
-      EXPECT_EQ(node.input(1), "^x");
+      EXPECT_EQ("Identity", node.op());
+      EXPECT_EQ(2, node.input_size());
+      EXPECT_EQ("y", node.input(0));
+      EXPECT_EQ("^x", node.input(1));
       ++found;
     }
   }
-  EXPECT_EQ(found, 2);
+  EXPECT_EQ(2, found);
 }
 
-TEST_F(DependencyOptimizerTest, ChangeToNoopRepeatedInput) {
+TEST_F(DependencyOptimizerTest, ChangeToNoop_RepeatedInput) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output x = ops::RandomUniform(s.WithOpName("x"), {1, 2}, DT_FLOAT);
   Output add = ops::Add(s.WithOpName("add"), x, x);
@@ -164,24 +164,25 @@ TEST_F(DependencyOptimizerTest, ChangeToNoopRepeatedInput) {
   item.graph.Swap(&output);
   status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
+  LOG(INFO) << output.DebugString();
 
-  EXPECT_EQ(output.node_size(), item.graph.node_size());
+  EXPECT_EQ(item.graph.node_size(), output.node_size());
   int found = 0;
   for (int i = 0; i < item.graph.node_size(); ++i) {
     const NodeDef& node = item.graph.node(i);
     // "add" should get turned into a NoOp and removed.
-    EXPECT_NE(node.name(), "add");
+    EXPECT_NE("add", node.name());
     if (node.name() == "id1") {
-      EXPECT_EQ(node.op(), "Identity");
-      EXPECT_EQ(node.input_size(), 1);
-      EXPECT_EQ(node.input(0), "x");
+      EXPECT_EQ("Identity", node.op());
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("x", node.input(0));
       ++found;
     }
   }
-  EXPECT_EQ(found, 1);
+  EXPECT_EQ(1, found);
 }
 
-TEST_F(DependencyOptimizerTest, ChangeToNoopSwitchIdentity) {
+TEST_F(DependencyOptimizerTest, ChangeToNoop_SwitchIdentity) {
   // This tests that we don't try to repeatedly add Identity nodes
   // with names like "ConstantFoldingCtrl/foo/bar/switch_$port" when
   // multiple nodes reading the same output of a Switch node get
@@ -219,23 +220,23 @@ TEST_F(DependencyOptimizerTest, ChangeToNoopSwitchIdentity) {
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
-  EXPECT_EQ(output.node_size(), item.graph.node_size() - 1);
+  EXPECT_EQ(item.graph.node_size() - 1, output.node_size());
   for (int i = 0; i < output.node_size(); ++i) {
     const NodeDef& node = output.node(i);
     // "neg" should be eliminated.
-    EXPECT_NE(node.name(), "neg");
+    EXPECT_NE("neg", node.name());
     // A control dep from "^ConstantFoldingCtrl/switch_1"
     // should be attached to "c1".
     if (node.name() == "c1") {
-      EXPECT_EQ(node.op(), "Const");
-      ASSERT_EQ(node.input_size(), 1);
-      EXPECT_EQ(node.input(0), "^ConstantFoldingCtrl/switch_1");
+      EXPECT_EQ("Const", node.op());
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("^ConstantFoldingCtrl/switch_1", node.input(0));
     }
   }
 }
 
 // TODO(rmlarsen): Add test to make sure we skip Switch and Merge.
-TEST_F(DependencyOptimizerTest, ChangeToNoopNoFetch) {
+TEST_F(DependencyOptimizerTest, ChangeToNoop_NoFetch) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output x = ops::RandomUniform(s.WithOpName("x"), {1, 2}, DT_FLOAT);
   Output y = ops::RandomUniform(s.WithOpName("y"), {1, 2}, DT_FLOAT);
@@ -257,7 +258,7 @@ TEST_F(DependencyOptimizerTest, ChangeToNoopNoFetch) {
   VerifyGraphsEqual(item.graph, output, __FUNCTION__);
 }
 
-TEST_F(DependencyOptimizerTest, RemoveNoOpsEmptyInputOrOutput) {
+TEST_F(DependencyOptimizerTest, RemoveNoOps_EmptyInputOrOutput) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output x = ops::RandomUniform(s, {1, 2}, DT_FLOAT);
   auto noop1 = ops::NoOp(s);
@@ -277,18 +278,18 @@ TEST_F(DependencyOptimizerTest, RemoveNoOpsEmptyInputOrOutput) {
   status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
-  EXPECT_EQ(output.node_size(), item.graph.node_size());
+  EXPECT_EQ(item.graph.node_size(), output.node_size());
   for (const NodeDef& node : output.node()) {
     if (node.name() == "NoOp" || node.name() == "NoOp_1") {
       EXPECT_EQ(0, node.input_size());
     } else if (node.name() == "Identity") {
-      ASSERT_EQ(node.input_size(), 1);
-      EXPECT_EQ(node.input(0), "RandomUniform");
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("RandomUniform", node.input(0));
     }
   }
 }
 
-TEST_F(DependencyOptimizerTest, RemoveNoOpsDeviceBoundaries) {
+TEST_F(DependencyOptimizerTest, RemoveNoOps_DeviceBoundaries) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output x = ops::RandomUniform(s.WithOpName("x").WithDevice("/CPU:0"), {1, 2},
                                 DT_FLOAT);
@@ -323,7 +324,7 @@ TEST_F(DependencyOptimizerTest, RemoveNoOpsDeviceBoundaries) {
   VerifyGraphsEqual(item.graph, output, __FUNCTION__);
 }
 
-TEST_F(DependencyOptimizerTest, RemoveIdentityOpsDeviceBoundaries) {
+TEST_F(DependencyOptimizerTest, RemoveIdentityOps_DeviceBoundaries) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output x = ops::RandomUniform(s.WithOpName("x").WithDevice("/CPU:0"), {1, 2},
                                 DT_FLOAT);
@@ -355,7 +356,7 @@ TEST_F(DependencyOptimizerTest, RemoveIdentityOpsDeviceBoundaries) {
   VerifyGraphsEqual(item.graph, output, __FUNCTION__);
 }
 
-TEST_F(DependencyOptimizerTest, RemoveIdentityOpsIdenticalDevices) {
+TEST_F(DependencyOptimizerTest, RemoveIdentityOps_IdenticalDevices) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output x = ops::RandomUniform(s.WithOpName("x").WithDevice("/CPU:0"), {1, 2},
                                 DT_FLOAT);
@@ -372,17 +373,16 @@ TEST_F(DependencyOptimizerTest, RemoveIdentityOpsIdenticalDevices) {
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
-  EXPECT_EQ(output.node_size(), item.graph.node_size() - 1);
+  EXPECT_EQ(item.graph.node_size() - 1, output.node_size());
   for (const NodeDef& node : output.node()) {
     EXPECT_NE(node.name(), "id_a");
     if (node.name() == "Identity") {
-      ASSERT_EQ(node.input_size(), 1);
       EXPECT_EQ(node.input(0), "x");
     }
   }
 }
 
-TEST_F(DependencyOptimizerTest, RemoveNoOpsSingleInputOrOutput) {
+TEST_F(DependencyOptimizerTest, RemoveNoOps_SingleInputOrOutput) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output x = ops::RandomUniform(s.WithOpName("x"), {1, 2}, DT_FLOAT);
   Output y = ops::RandomUniform(s.WithOpName("y"), {1, 2}, DT_FLOAT);
@@ -409,17 +409,15 @@ TEST_F(DependencyOptimizerTest, RemoveNoOpsSingleInputOrOutput) {
   status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
-  EXPECT_EQ(output.node_size(), item.graph.node_size());
+  EXPECT_EQ(item.graph.node_size(), output.node_size());
   for (const NodeDef& node : output.node()) {
     if (node.name() == "NoOp" || node.name() == "NoOp_1") {
-      EXPECT_EQ(node.input_size(), 0);
+      EXPECT_EQ(0, node.input_size());
     } else if (node.name() == "Identity") {
-      ASSERT_EQ(node.input_size(), 1);
-      EXPECT_EQ(node.input(0), "x");
+      EXPECT_EQ("x", node.input(0));
     } else if (node.name() == "Identity_1") {
-      ASSERT_EQ(node.input_size(), 2);
-      EXPECT_EQ(node.input(0), "y");
-      EXPECT_EQ(node.input(1), "^x");
+      EXPECT_EQ("y", node.input(0));
+      EXPECT_EQ("^x", node.input(1));
     }
   }
 }
@@ -464,48 +462,48 @@ TEST_F(DependencyOptimizerTest, RemoveIdentity) {
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
-  EXPECT_EQ(output.node_size(), item.graph.node_size() - 3);
+  EXPECT_EQ(item.graph.node_size() - 3, output.node_size());
   int found = 0;
   for (const NodeDef& node : output.node()) {
-    EXPECT_NE(node.name(), "id_a");
-    EXPECT_NE(node.name(), "id_b");
-    EXPECT_NE(node.name(), "id_c");
+    EXPECT_NE("id_a", node.name());
+    EXPECT_NE("id_b", node.name());
+    EXPECT_NE("id_c", node.name());
     if (node.name() == "a_a" || node.name() == "a_b") {
-      ASSERT_EQ(node.input_size(), 1);
-      EXPECT_EQ(node.input(0), "x");
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("x", node.input(0));
       ++found;
     }
     if (node.name() == "a_c" || node.name() == "a_d") {
-      ASSERT_EQ(node.input_size(), 2);
-      EXPECT_EQ(node.input(0), "z");
-      EXPECT_EQ(node.input(1), "^x");
+      EXPECT_EQ(2, node.input_size());
+      EXPECT_EQ("z", node.input(0));
+      EXPECT_EQ("^x", node.input(1));
       ++found;
     }
     if (node.name() == "b_a") {
-      ASSERT_EQ(node.input_size(), 3);
-      EXPECT_EQ(node.input(0), "x");
-      EXPECT_EQ(node.input(1), "^y");
-      EXPECT_EQ(node.input(2), "^z");
+      EXPECT_EQ(3, node.input_size());
+      EXPECT_EQ("x", node.input(0));
+      EXPECT_EQ("^y", node.input(1));
+      EXPECT_EQ("^z", node.input(2));
       ++found;
     }
     if (node.name() == "c_a") {
-      ASSERT_EQ(node.input_size(), 2);
-      EXPECT_EQ(node.input(0), "x");
-      EXPECT_EQ(node.input(1), "^y");
+      EXPECT_EQ(2, node.input_size());
+      EXPECT_EQ("x", node.input(0));
+      EXPECT_EQ("^y", node.input(1));
       ++found;
     }
     if (node.name() == "c_b") {
-      ASSERT_EQ(node.input_size(), 3);
-      EXPECT_EQ(node.input(0), "z");
-      EXPECT_EQ(node.input(1), "^x");
-      EXPECT_EQ(node.input(2), "^y");
+      EXPECT_EQ(3, node.input_size());
+      EXPECT_EQ("z", node.input(0));
+      EXPECT_EQ("^x", node.input(1));
+      EXPECT_EQ("^y", node.input(2));
       ++found;
     }
   }
   EXPECT_EQ(found, 7);
 }
 
-TEST_F(DependencyOptimizerTest, RemoveIdentityRepeatedInputs) {
+TEST_F(DependencyOptimizerTest, RemoveIdentity_RepeatedInputs) {
   // Corner cases with repeated inputs.
   tensorflow::Scope scope = tensorflow::Scope::NewRootScope();
   ops::Variable x(scope.WithOpName("x"), {}, DT_BOOL);
@@ -531,35 +529,35 @@ TEST_F(DependencyOptimizerTest, RemoveIdentityRepeatedInputs) {
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
-  EXPECT_EQ(output.node_size(), item.graph.node_size() - 1);
+  EXPECT_EQ(item.graph.node_size() - 1, output.node_size());
   int found = 0;
   for (const NodeDef& node : output.node()) {
-    EXPECT_NE(node.name(), "id0");
+    EXPECT_NE("id0", node.name());
     if (node.name() == "or0") {
-      ASSERT_EQ(node.input_size(), 2);
-      EXPECT_EQ(node.input(0), "switch:1");
-      EXPECT_EQ(node.input(1), "switch:1");
+      EXPECT_EQ(2, node.input_size());
+      EXPECT_EQ("switch:1", node.input(0));
+      EXPECT_EQ("switch:1", node.input(1));
       ++found;
     }
     if (node.name() == "or1") {
-      ASSERT_EQ(node.input_size(), 2);
-      EXPECT_EQ(node.input(0), "switch:1");
-      EXPECT_EQ(node.input(1), "y");
+      EXPECT_EQ(2, node.input_size());
+      EXPECT_EQ("switch:1", node.input(0));
+      EXPECT_EQ("y", node.input(1));
       ++found;
     }
     if (node.name() == "or2") {
       // or1 should be unchanged.
-      ASSERT_EQ(node.input_size(), 3);
-      EXPECT_EQ(node.input(0), "y");
-      EXPECT_EQ(node.input(1), "y");
-      EXPECT_EQ(node.input(2), "^id1");
+      EXPECT_EQ(3, node.input_size());
+      EXPECT_EQ("y", node.input(0));
+      EXPECT_EQ("y", node.input(1));
+      EXPECT_EQ("^id1", node.input(2));
       ++found;
     }
   }
   EXPECT_EQ(found, 3);
 }
 
-TEST_F(DependencyOptimizerTest, TransitiveReductionSimple) {
+TEST_F(DependencyOptimizerTest, Transitive_Reduction_Simple) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output c = ops::Const(s.WithOpName("c"), {1.0f, 2.0f}, {1, 2});
   Output x = ops::Square(s.WithOpName("x"), c);
@@ -574,13 +572,13 @@ TEST_F(DependencyOptimizerTest, TransitiveReductionSimple) {
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
-  ASSERT_EQ(output.node_size(), 4);
-  EXPECT_EQ(output.node(3).name(), "neg2");
-  ASSERT_EQ(output.node(3).input_size(), 1);
-  EXPECT_EQ(output.node(3).input(0), "neg1");
+  EXPECT_EQ(4, output.node_size());
+  EXPECT_EQ("neg2", output.node(3).name());
+  EXPECT_EQ(1, output.node(3).input_size());
+  EXPECT_EQ("neg1", output.node(3).input(0));
 }
 
-TEST_F(DependencyOptimizerTest, ChangeToNoopIdentity) {
+TEST_F(DependencyOptimizerTest, ChangeToNoop_Identity) {
   tensorflow::Scope scope = tensorflow::Scope::NewRootScope();
   ops::Variable v_in(scope.WithOpName("v_in"), {3}, DT_FLOAT);
   Output id_after_var = ops::Identity(scope.WithOpName("id_after_var"), v_in);
@@ -611,18 +609,18 @@ TEST_F(DependencyOptimizerTest, ChangeToNoopIdentity) {
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
-  EXPECT_EQ(output.node_size(), item.graph.node_size() - 2);
+  EXPECT_EQ(item.graph.node_size() - 2, output.node_size());
   bool found = false;
   for (int i = 0; i < output.node_size(); ++i) {
     const NodeDef& node = output.node(i);
     // "id0" and "id1" but neither "ConstantFoldingCtrl/switch_1",
     // "id_after_var, nor "id2"" should be eliminated.
-    EXPECT_NE(node.name(), "id0");
-    EXPECT_NE(node.name(), "id1");
+    EXPECT_NE("id0", node.name());
+    EXPECT_NE("id1", node.name());
     if (node.name() == "c1") {
-      EXPECT_EQ(node.op(), "Const");
-      ASSERT_EQ(node.input_size(), 1);
-      EXPECT_EQ(node.input(0), "^ConstantFoldingCtrl/switch_1");
+      EXPECT_EQ("Const", node.op());
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("^ConstantFoldingCtrl/switch_1", node.input(0));
       found = true;
     }
   }
@@ -652,17 +650,17 @@ TEST_F(DependencyOptimizerTest, IdentityInputs) {
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
-  ASSERT_EQ(output.node_size(), 6);
-  EXPECT_EQ(output.node(4).name(), "out2");
-  ASSERT_EQ(output.node(4).input_size(), 1);
-  EXPECT_EQ(output.node(4).input(0), "s:1");
+  EXPECT_EQ(6, output.node_size());
+  EXPECT_EQ("out1", output.node(4).name());
+  EXPECT_EQ(1, output.node(4).input_size());
+  EXPECT_EQ("s", output.node(4).input(0));
 
-  EXPECT_EQ(output.node(5).name(), "out1");
-  ASSERT_EQ(output.node(5).input_size(), 1);
-  EXPECT_EQ(output.node(5).input(0), "s");
+  EXPECT_EQ("out2", output.node(5).name());
+  EXPECT_EQ(1, output.node(5).input_size());
+  EXPECT_EQ("s:1", output.node(5).input(0));
 }
 
-TEST_F(DependencyOptimizerTest, RemoveIdentityNSwitchInput) {
+TEST_F(DependencyOptimizerTest, RemoveIdentityN_SwitchInput) {
   tensorflow::Scope scope = tensorflow::Scope::NewRootScope();
   Output b = ops::Placeholder(scope.WithOpName("b"), DT_BOOL);
   Output x = ops::RandomUniform(scope.WithOpName("x"), {1, 2}, DT_FLOAT);
@@ -689,27 +687,27 @@ TEST_F(DependencyOptimizerTest, RemoveIdentityNSwitchInput) {
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
-  ASSERT_EQ(output.node_size(), 8);
+  EXPECT_EQ(8, output.node_size());
 
   auto out1_node = output.node(7);
-  EXPECT_EQ(out1_node.name(), "out1");
-  ASSERT_EQ(out1_node.input_size(), 1);
-  EXPECT_EQ(out1_node.input(0), "s");
+  EXPECT_EQ("out1", out1_node.name());
+  EXPECT_EQ(1, out1_node.input_size());
+  EXPECT_EQ("s", out1_node.input(0));
 
-  auto out2_node = output.node(6);
-  EXPECT_EQ(out2_node.name(), "out2");
-  ASSERT_EQ(out2_node.input_size(), 1);
-  EXPECT_EQ(out2_node.input(0), "s:1");
+  auto out2_node = output.node(4);
+  EXPECT_EQ("out2", out2_node.name());
+  EXPECT_EQ(1, out2_node.input_size());
+  EXPECT_EQ("s:1", out2_node.input(0));
 
   auto out3_node = output.node(5);
-  EXPECT_EQ(out3_node.name(), "out3");
-  ASSERT_EQ(out3_node.input_size(), 1);
-  EXPECT_EQ(out3_node.input(0), "s");
-
-  auto out4_node = output.node(4);
-  EXPECT_EQ(out4_node.name(), "out4");
-  ASSERT_EQ(out4_node.input_size(), 1);
-  EXPECT_EQ(out4_node.input(0), "s:1");
+  EXPECT_EQ("out3", out3_node.name());
+  EXPECT_EQ(1, out3_node.input_size());
+  EXPECT_EQ("s", out3_node.input(0));
+
+  auto out4_node = output.node(6);
+  EXPECT_EQ("out4", out4_node.name());
+  EXPECT_EQ(1, out4_node.input_size());
+  EXPECT_EQ("s:1", out4_node.input(0));
 }
 
 TEST_F(DependencyOptimizerTest, DoNotRemoveIdentityNWithControlDependency) {
@@ -732,11 +730,11 @@ TEST_F(DependencyOptimizerTest, DoNotRemoveIdentityNWithControlDependency) {
   Status status = optimizer.Optimize(nullptr, item, &optimized_graph_def);
   TF_EXPECT_OK(status);
 
-  EXPECT_EQ(optimized_graph_def.node_size(), 6);
+  EXPECT_EQ(6, optimized_graph_def.node_size());
 }
 
 TEST_F(DependencyOptimizerTest,
-       IdentityDeviceCrossingConsumerOnDifferentDevice) {
+       Identity_DeviceCrossing_ConsumerOnDifferentDevice) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output x_on_1 =
       ops::Const(s.WithOpName("x_on_1").WithDevice("/gpu:1"), {1.0f}, {});
@@ -758,7 +756,7 @@ TEST_F(DependencyOptimizerTest,
   VerifyGraphsEqual(item.graph, output, __FUNCTION__);
 }
 
-TEST_F(DependencyOptimizerTest, IdentityDeviceCrossingConsumerOnSameDevice) {
+TEST_F(DependencyOptimizerTest, Identity_DeviceCrossing_ConsumerOnSameDevice) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output x_on_1 =
       ops::Const(s.WithOpName("x_on_1").WithDevice("/gpu:1"), {1.0f}, {});
@@ -776,15 +774,12 @@ TEST_F(DependencyOptimizerTest, IdentityDeviceCrossingConsumerOnSameDevice) {
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
-
-  EXPECT_EQ(output.node_size(), 3);
+  LOG(INFO) << output.DebugString();
+  EXPECT_EQ(3, output.node_size());
   for (const auto& node : output.node()) {
-    EXPECT_NE(node.name(), "x_on_2");
+    EXPECT_NE("x_on_2", node.name());
     if (node.name() == "result") {
-      LOG(INFO) << node.DebugString();
-      ASSERT_EQ(node.input_size(), 2);
-      EXPECT_EQ(node.input(0), "x_on_1");
-      EXPECT_EQ(node.input(1), "one_on_2");
+      EXPECT_EQ("x_on_1", node.input(0));
     }
   }
 }
@@ -812,25 +807,25 @@ TEST_F(DependencyOptimizerTest, RemoveGreaterEqualWithNoOp) {
   for (const NodeDef& node : output.node()) {
     if (node.name() == "x") {
       count++;
-      EXPECT_EQ(node.op(), "Placeholder");
-      EXPECT_EQ(node.input_size(), 0);
+      EXPECT_EQ("Placeholder", node.op());
+      EXPECT_EQ(0, node.input_size());
     } else if (node.name() == "y") {
       count++;
-      EXPECT_EQ(node.op(), "Placeholder");
-      EXPECT_EQ(node.input_size(), 0);
+      EXPECT_EQ("Placeholder", node.op());
+      EXPECT_EQ(0, node.input_size());
     } else if (node.name() == "GreaterEqual") {
       count++;
     } else if (node.name() == "NoOp") {
       count++;
     } else if (node.name() == "z") {
       count++;
-      EXPECT_EQ(node.op(), "Add");
-      ASSERT_EQ(node.input_size(), 2);
-      EXPECT_EQ(node.input(0), "x");
-      EXPECT_EQ(node.input(1), "y");
+      EXPECT_EQ("Add", node.op());
+      EXPECT_EQ(2, node.input_size());
+      EXPECT_EQ("x", node.input(0));
+      EXPECT_EQ("y", node.input(1));
     }
   }
-  EXPECT_EQ(count, 3);
+  EXPECT_EQ(3, count);
 }
 
 TEST_F(DependencyOptimizerTest, GroupCrossDeviceControlDeps) {
-- 
GitLab


From 1f856fb5d978177123ddd5ac5a3e4bb669288d65 Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Wed, 6 Feb 2019 09:37:35 -0800
Subject: [PATCH 1997/2345] Remove overhead on rank agnostic convoltion
 operator

PiperOrigin-RevId: 232689932
---
 tensorflow/python/keras/backend.py |   4 +-
 tensorflow/python/ops/nn_ops.py    | 113 +++++++++++++++++++++--------
 2 files changed, 86 insertions(+), 31 deletions(-)

diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index 064358cacd..32820fef5f 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -4143,8 +4143,8 @@ def conv1d(x,
   x = nn.convolution(
       input=x,
       filter=kernel,
-      dilation_rate=(dilation_rate,),
-      strides=(strides,),
+      dilation_rate=dilation_rate,
+      strides=strides,
       padding=padding,
       data_format=tf_data_format)
   if data_format == 'channels_first' and tf_data_format == 'NWC':
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index bd70d9f09a..f04a3ae293 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -65,7 +65,9 @@ def _get_sequence(value, n, channel_index, name):
     return value
   elif current_n == 1:
     value = list((value[0],) * n)
-  elif current_n != n:
+  elif current_n == n:
+    value = list(value)
+  else:
     raise ValueError("{} should be of length 1, {} or {} but was {}".format(
         name, n, n + 2, current_n))
 
@@ -881,21 +883,14 @@ def convolution(
   filter = deprecated_argument_lookup("filters", filters, "filter", filter)
   dilation_rate = deprecated_argument_lookup(
       "dilations", dilations, "dilation_rate", dilation_rate)
-  # pylint: enable=line-too-long
-  with ops.name_scope(name, "convolution", [input, filter]) as name:
-    input = ops.convert_to_tensor(input, name="input")  # pylint: disable=redefined-builtin
-    input_shape = input.get_shape()
-    filter = ops.convert_to_tensor(filter, name="filter")  # pylint: disable=redefined-builtin
-    filter_shape = filter.get_shape()
-    op = Convolution(
-        input_shape,
-        filter_shape,
-        padding,
-        strides=strides,
-        dilation_rate=dilation_rate,
-        name=name,
-        data_format=data_format)
-    return op(input, filter)
+  return convolution_internal(
+      input,
+      filter,
+      strides=strides,
+      padding=padding,
+      data_format=data_format,
+      dilations=dilation_rate,
+      name=name)
 
 
 @tf_export("nn.convolution", v1=[])
@@ -907,14 +902,15 @@ def convolution_v2(
     data_format=None,
     dilations=None,
     name=None):
-  return convolution(
+  return convolution_internal(
       input,  # pylint: disable=redefined-builtin
       filters,
-      padding=padding,
       strides=strides,
-      dilation_rate=dilations,
-      name=name,
-      data_format=data_format)
+      padding=padding,
+      data_format=data_format,
+      dilations=dilations,
+      name=name)
+
 
 convolution_v2.__doc__ = deprecation.rewrite_argument_docstring(
     deprecation.rewrite_argument_docstring(
@@ -922,6 +918,67 @@ convolution_v2.__doc__ = deprecation.rewrite_argument_docstring(
     "filter", "filters")
 
 
+def convolution_internal(
+    input,  # pylint: disable=redefined-builtin
+    filters,
+    strides=None,
+    padding="VALID",
+    data_format=None,
+    dilations=None,
+    name=None):
+  """Internal function which performs rank agnostic convolution."""
+  with ops.name_scope(name, "convolution", [input, filter]) as name:
+    if input.shape is not None:
+      n = len(input.shape) - 2
+    elif filters.shape is not None:
+      n = len(filters.shape) - 2
+    else:
+      raise ValueError("rank of input or filter must be known")
+
+    if n < 1 or n > 3:
+      raise ValueError(
+          "Input tensor must be of rank 3, 4 or 5 but was {}.".format(n + 2))
+
+    if data_format is None:
+      channel_index = n + 1
+    else:
+      channel_index = 1 if data_format.startswith("NC") else n + 1
+
+    strides = _get_sequence(strides, n, channel_index, "strides")
+    dilations = _get_sequence(dilations, n, channel_index, "dilations")
+
+    conv_ops = {1: conv1d, 2: gen_nn_ops.conv2d, 3: gen_nn_ops.conv3d}
+
+    if all(i == 1 for i in dilations):
+      # fast path if no dilation as gradient only supported on GPU for dilations
+      op = conv_ops.get(n)
+      return op(
+          input,
+          filters,
+          strides,
+          padding=padding,
+          data_format=data_format,
+          dilations=dilations,
+          name=name)
+    else:
+      if channel_index == 1:
+        strides = strides[2:]
+        dilations = dilations[2:]
+      else:
+        strides = strides[1:-1]
+        dilations = dilations[1:-1]
+
+      op = Convolution(
+          tensor_shape.as_shape(input.shape),
+          tensor_shape.as_shape(filters.shape),
+          padding,
+          strides=strides,
+          dilation_rate=dilations,
+          name=name,
+          data_format=data_format)
+      return op(input, filters)
+
+
 class Convolution(object):
   """Helper class for convolution.
 
@@ -4040,10 +4097,9 @@ def conv1d(
       entries by which the filter is moved right at each step.
     padding: 'SAME' or 'VALID'
     use_cudnn_on_gpu: An optional `bool`.  Defaults to `True`.
-    data_format: An optional `string` from `"NWC", "NCW"`.  Defaults
-      to `"NWC"`, the data is stored in the order of
-      [batch, in_width, in_channels].  The `"NCW"` format stores
-      data as [batch, in_channels, in_width].
+    data_format: An optional `string` from `"NWC", "NCW"`.  Defaults to `"NWC"`,
+      the data is stored in the order of [batch, in_width, in_channels].  The
+      `"NCW"` format stores data as [batch, in_channels, in_width].
     name: A name for the operation (optional).
     input: Alias for value.
     dilations: An int or list of `ints` that has length `1` or `3` which
@@ -4126,10 +4182,9 @@ def conv1d_v2(
     stride: An int or list of `ints` that has length `1` or `3`.  The number of
       entries by which the filter is moved right at each step.
     padding: 'SAME' or 'VALID'
-    data_format: An optional `string` from `"NWC", "NCW"`.  Defaults
-      to `"NWC"`, the data is stored in the order of
-      [batch, in_width, in_channels].  The `"NCW"` format stores
-      data as [batch, in_channels, in_width].
+    data_format: An optional `string` from `"NWC", "NCW"`.  Defaults to `"NWC"`,
+      the data is stored in the order of [batch, in_width, in_channels].  The
+      `"NCW"` format stores data as [batch, in_channels, in_width].
     dilations: An int or list of `ints` that has length `1` or `3` which
       defaults to 1. The dilation factor for each dimension of input. If set to
       k > 1, there will be k-1 skipped cells between each filter element on that
-- 
GitLab


From 0ab6969bc5bb75891687f1bc267c3f764956c4ac Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Wed, 6 Feb 2019 09:40:09 -0800
Subject: [PATCH 1998/2345] nomac for def_function_xla_test

We're not ./configuring XLA on OSX, so we shouldn't test it

PiperOrigin-RevId: 232690418
---
 tensorflow/python/eager/BUILD | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD
index af7930224b..aba342b6d5 100644
--- a/tensorflow/python/eager/BUILD
+++ b/tensorflow/python/eager/BUILD
@@ -527,7 +527,10 @@ tf_py_test(
 tf_xla_py_test(
     name = "def_function_xla_test",
     srcs = ["def_function_xla_test.py"],
-    tags = ["no_pip"],
+    tags = [
+        "no_pip",
+        "nomac",
+    ],
     deps = [
         ":def_function",
         "//tensorflow/compiler/tests:xla_test",
-- 
GitLab


From 539e543ff54577cca43a50a9d99c59d2e9ce2130 Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Wed, 6 Feb 2019 10:13:10 -0800
Subject: [PATCH 1999/2345] Remove superfluous line (repeat of line 163)

PiperOrigin-RevId: 232696399
---
 tensorflow/core/kernels/data/map_dataset_op.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/core/kernels/data/map_dataset_op.cc b/tensorflow/core/kernels/data/map_dataset_op.cc
index 95f4c1c891..e516d7791b 100644
--- a/tensorflow/core/kernels/data/map_dataset_op.cc
+++ b/tensorflow/core/kernels/data/map_dataset_op.cc
@@ -138,7 +138,6 @@ class MapDatasetOp : public UnaryDatasetOpKernel {
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
                               Node** output) const override {
-      TF_RETURN_IF_ERROR(b->AddFunction(ctx, func_.name()));
       Node* input_graph_node = nullptr;
       TF_RETURN_IF_ERROR(b->AddInputDataset(ctx, input_, &input_graph_node));
 
-- 
GitLab


From 1a43c97f079e9f5ec5b2798f053ac5d3aa6b1973 Mon Sep 17 00:00:00 2001
From: James Ring <sjr@google.com>
Date: Wed, 6 Feb 2019 10:18:13 -0800
Subject: [PATCH 2000/2345] Add draft of C BitcastOp implementation.

This CL removes the C++ implementation of BitCast kernel and adds a C
implementation in a new tensorflow/c/kernels. The kernel is registered both for
CPU and, if CUDA is present, GPU.

The Bitcast Op registration is left in place. A C API for registering ops is
coming.

PiperOrigin-RevId: 232697339
---
 tensorflow/c/kernels/BUILD              |  35 +++++
 tensorflow/c/kernels/bitcast_op.cc      | 165 ++++++++++++++++++++++++
 tensorflow/c/kernels/bitcast_op_test.cc | 101 +++++++++++++++
 tensorflow/core/BUILD                   |   1 +
 tensorflow/core/kernels/BUILD           |   7 -
 tensorflow/core/kernels/bitcast_op.cc   |  82 ------------
 tensorflow/core/kernels/bitcast_op.h    |  29 -----
 7 files changed, 302 insertions(+), 118 deletions(-)
 create mode 100644 tensorflow/c/kernels/BUILD
 create mode 100644 tensorflow/c/kernels/bitcast_op.cc
 create mode 100644 tensorflow/c/kernels/bitcast_op_test.cc
 delete mode 100644 tensorflow/core/kernels/bitcast_op.cc
 delete mode 100644 tensorflow/core/kernels/bitcast_op.h

diff --git a/tensorflow/c/kernels/BUILD b/tensorflow/c/kernels/BUILD
new file mode 100644
index 0000000000..e4ce21890d
--- /dev/null
+++ b/tensorflow/c/kernels/BUILD
@@ -0,0 +1,35 @@
+load(
+    "//tensorflow:tensorflow.bzl",
+    "tf_cc_test",
+    "tf_cuda_library",
+)
+
+package(
+    default_visibility = ["//visibility:public"],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+tf_cuda_library(
+    name = "bitcast_op",
+    srcs = ["bitcast_op.cc"],
+    deps = [
+        "//tensorflow/c:kernels",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:ops",
+        "@com_google_absl//absl/strings:str_format",
+    ],
+    alwayslink = 1,
+)
+
+tf_cc_test(
+    name = "bitcast_op_test",
+    srcs = ["bitcast_op_test.cc"],
+    deps = [
+        ":bitcast_op",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
diff --git a/tensorflow/c/kernels/bitcast_op.cc b/tensorflow/c/kernels/bitcast_op.cc
new file mode 100644
index 0000000000..90f32c990e
--- /dev/null
+++ b/tensorflow/c/kernels/bitcast_op.cc
@@ -0,0 +1,165 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "absl/strings/str_format.h"
+#include "tensorflow/c/kernels.h"
+#include "tensorflow/core/framework/common_shape_fns.h"
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/shape_inference.h"
+#include "tensorflow/core/framework/types.h"
+
+// BitcastOp implements a bitcast kernel, creating an output tensor that shares
+// the same data buffer as the input but with a different shape and/or data
+// type. Its inputs are:
+//
+//   * the input tensor
+//   * an attribute named "T" containing the TF_DataType of the input tensor
+//   * an attribute named "type" containing the TF_DataType of the output tensor
+//
+// Given an input tensor of shape [...], if the input DataType "T" is larger
+// than the output DataType "type", then the shape changes from [...]
+// to [..., sizeof(T)/sizeof(type)].
+//
+// If "T" is smaller than "type", the operator requires that the rightmost
+// dimension be equal to sizeof(type)/sizeof(T). The shape then goes from
+// [..., sizeof(type)/sizeof(T)] to [...].
+//
+// Bitcast is implemented as a low-level cast, so machines with different endian
+// orderings will give different results.
+typedef struct BitcastOp {
+  TF_DataType input_data_type;
+  TF_DataType output_data_type;
+  size_t in_size;
+  size_t out_size;
+} BitcastOp;
+
+void* BitcastOp_Create(TF_OpKernelConstruction* ctx) {
+  auto* kernel = new BitcastOp;
+
+  TF_Status* s = TF_NewStatus();
+  TF_OpKernelConstruction_GetAttrType(ctx, "T", &kernel->input_data_type, s);
+
+  if (TF_GetCode(s) == TF_OK) {
+    TF_OpKernelConstruction_GetAttrType(ctx, "type", &kernel->output_data_type,
+                                        s);
+  }
+
+  if (TF_GetCode(s) == TF_OK) {
+    kernel->in_size = TF_DataTypeSize(kernel->input_data_type);
+    kernel->out_size = TF_DataTypeSize(kernel->output_data_type);
+
+    size_t check_size = std::max(kernel->in_size, kernel->out_size) %
+                        std::min(kernel->in_size, kernel->out_size);
+    if (check_size != 0) {
+      std::string err =
+          absl::StrFormat("cannot convert between datatype %d and %d",
+                          kernel->input_data_type, kernel->output_data_type);
+      TF_SetStatus(s, TF_INVALID_ARGUMENT, err.c_str());
+    }
+  }
+
+  if (TF_GetCode(s) != TF_OK) {
+    TF_OpKernelConstruction_Failure(ctx, s);
+    delete kernel;
+    kernel = nullptr;
+  }
+
+  TF_DeleteStatus(s);
+  return kernel;
+}
+
+void BitcastOp_Delete(void* kernel) { delete static_cast<BitcastOp*>(kernel); }
+
+void BitcastOp_Compute(void* kernel, TF_OpKernelContext* ctx) {
+  auto* k = static_cast<BitcastOp*>(kernel);
+  int dim_count = 0;
+
+  TF_Tensor* tensor;
+  TF_Status* status = TF_NewStatus();
+  TF_GetInput(ctx, 0, &tensor, status);
+  if (TF_GetCode(status) == TF_OK) {
+    dim_count = TF_NumDims(tensor);
+    if (!(k->in_size >= k->out_size ||
+          (dim_count > 0 &&
+           TF_Dim(tensor, dim_count - 1) == k->out_size / k->in_size))) {
+      std::string err =
+          absl::StrFormat("Cannot bitcast from %d to %d", k->input_data_type,
+                          k->output_data_type);
+      TF_SetStatus(status, TF_INVALID_ARGUMENT, err.c_str());
+    }
+  }
+
+  if (TF_GetCode(status) == TF_OK) {
+    auto* dims = new int64_t[dim_count + 1];
+    int new_dim_count = dim_count;
+    for (int dim = 0; dim < dim_count; ++dim) {
+      dims[dim] = TF_Dim(tensor, dim);
+    }
+    if (k->out_size < k->in_size) {
+      dims[new_dim_count++] = static_cast<int64_t>(k->in_size / k->out_size);
+    } else if (k->out_size > k->in_size) {
+      --new_dim_count;
+    }
+
+    TF_Tensor* output = TF_AllocateTensor(k->output_data_type, dims, 0,
+                                          TF_DataTypeSize(k->output_data_type));
+    TF_TensorBitcastFrom(tensor, k->output_data_type, output, dims,
+                         new_dim_count, status);
+    if (TF_GetCode(status) == TF_OK) {
+      TF_SetOutput(ctx, 0, output, status);
+    }
+    delete[] dims;
+    TF_DeleteTensor(output);
+  }
+
+  if (TF_GetCode(status) != TF_OK) {
+    TF_OpKernelContext_Failure(ctx, status);
+  }
+  TF_DeleteStatus(status);
+  TF_DeleteTensor(tensor);
+}
+
+void RegisterBitcastOp() {
+  TF_Status* status = TF_NewStatus();
+
+  {
+    auto* builder = TF_NewKernelBuilder("Bitcast", tensorflow::DEVICE_CPU,
+                                        &BitcastOp_Create, &BitcastOp_Compute,
+                                        &BitcastOp_Delete);
+    TF_RegisterKernelBuilder("BitcastOp", builder, status);
+    CHECK_EQ(TF_OK, TF_GetCode(status))
+        << "Error while registering bitcast kernel";
+  }
+
+#if GOOGLE_CUDA
+  {
+    auto* builder = TF_NewKernelBuilder("Bitcast", tensorflow::DEVICE_GPU,
+                                        &BitcastOp_Create, &BitcastOp_Compute,
+                                        &BitcastOp_Delete);
+    TF_RegisterKernelBuilder("BitcastOp", builder, status);
+    CHECK_EQ(TF_OK, TF_GetCode(status))
+        << "Error while registering CUDA bitcast kernel";
+  }
+#endif
+
+  TF_DeleteStatus(status);
+}
+
+// A dummy static variable initialized by a lambda whose side-effect is to
+// register the bitcast kernel.
+static bool BitcastOpIsRegistered = []() {
+  RegisterBitcastOp();
+  return true;
+}();
diff --git a/tensorflow/c/kernels/bitcast_op_test.cc b/tensorflow/c/kernels/bitcast_op_test.cc
new file mode 100644
index 0000000000..06ffcca19d
--- /dev/null
+++ b/tensorflow/c/kernels/bitcast_op_test.cc
@@ -0,0 +1,101 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/framework/attr_value.pb.h"
+#include "tensorflow/core/framework/attr_value_util.h"
+#include "tensorflow/core/framework/node_def.pb.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+namespace {
+
+class DummyDevice : public DeviceBase {
+ public:
+  DummyDevice(Env* env, bool save) : DeviceBase(env), save_(save) {}
+  bool RequiresRecordingAccessedTensors() const override { return save_; }
+  Allocator* GetAllocator(AllocatorAttributes /*attr*/) override {
+    return cpu_allocator();
+  }
+
+ private:
+  bool save_;
+};
+
+void TestBitcastOp(Tensor* input_tensor, DataType out_type,
+                   TensorShape expected_shape, error::Code expected_code) {
+  Status status;
+  NodeDef def;
+  def.set_op("Bitcast");
+  def.set_device(DEVICE_CPU);
+
+  AttrValue typeAttr;
+  SetAttrValue(input_tensor->dtype(), &typeAttr);
+
+  AttrValue outTypeAttr;
+  SetAttrValue(out_type, &outTypeAttr);
+
+  (*def.mutable_attr())["T"] = typeAttr;
+  (*def.mutable_attr())["type"] = outTypeAttr;
+
+  def.add_input(
+      strings::StrCat("input1: ", DataTypeString(input_tensor->dtype())));
+
+  std::unique_ptr<OpKernel> kernel =
+      CreateOpKernel(DeviceType(DEVICE_CPU), nullptr, nullptr, def, 1, &status);
+  ASSERT_TRUE(status.ok()) << status.ToString();
+
+  OpKernelContext::Params params;
+  DummyDevice dummy_device(nullptr, false);
+  params.device = &dummy_device;
+  params.op_kernel = kernel.get();
+  gtl::InlinedVector<TensorValue, 4> inputs;
+  inputs.emplace_back(input_tensor);
+  params.inputs = &inputs;
+
+  OpKernelContext ctx(&params);
+  kernel->Compute(&ctx);
+  ASSERT_EQ(expected_code, ctx.status().code());
+  if (expected_code == error::OK) {
+    ASSERT_EQ(expected_shape, ctx.mutable_output(0)->shape())
+        << ctx.mutable_output(0)->shape().DebugString();
+  }
+}
+
+TEST(BitcastOpTest, TestUpcast) {
+  Tensor int8_input(DT_UINT8, {8});
+  for (int i = 0; i < 8; i++) {
+    int8_input.vec<uint8>()(i) = static_cast<uint8>(1);
+  }
+  TestBitcastOp(&int8_input, DT_UINT64, TensorShape(), error::OK);
+}
+
+TEST(BitcastOpTest, TestDowncast) {
+  Tensor int64_input(static_cast<uint64>(1));
+  TestBitcastOp(&int64_input, DT_UINT8, TensorShape({8}), error::OK);
+}
+
+TEST(BitcastOpTest, TestCastToSameSize) {
+  Tensor int32_input(DT_UINT32, {4, 6});
+  TestBitcastOp(&int32_input, DT_UINT8, TensorShape({4, 6, 4}), error::OK);
+}
+
+TEST(BitcastOpTest, TestImpossibleCast) {
+  Tensor int8_input(DT_UINT8, {1});
+  TestBitcastOp(&int8_input, DT_UINT32, TensorShape(), error::INVALID_ARGUMENT);
+}
+
+}  // namespace
+}  // namespace tensorflow
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 7975b25b06..cf212fc5d7 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1387,6 +1387,7 @@ cc_library(
     name = "all_kernels_impl",
     visibility = ["//visibility:private"],
     deps = [
+        "//tensorflow/c/kernels:bitcast_op",
         "//tensorflow/core/kernels:array",
         "//tensorflow/core/kernels:audio",
         "//tensorflow/core/kernels:batch_kernels",
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index e5ee109fff..ae3bb94e9e 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -838,7 +838,6 @@ cc_library(
     deps = [
         ":batch_space_ops",
         ":bcast_ops",
-        ":bitcast_op",
         ":broadcast_to_op",
         ":concat_op",
         ":constant_op",
@@ -888,12 +887,6 @@ tf_kernel_library(
     deps = ARRAY_DEPS,
 )
 
-tf_kernel_library(
-    name = "bitcast_op",
-    prefix = "bitcast_op",
-    deps = ARRAY_DEPS,
-)
-
 tf_kernel_library(
     name = "broadcast_to_op",
     prefix = "broadcast_to_op",
diff --git a/tensorflow/core/kernels/bitcast_op.cc b/tensorflow/core/kernels/bitcast_op.cc
deleted file mode 100644
index 02c8808809..0000000000
--- a/tensorflow/core/kernels/bitcast_op.cc
+++ /dev/null
@@ -1,82 +0,0 @@
-/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-// See docs in ../ops/array_ops.cc.
-#include "tensorflow/core/framework/op_kernel.h"
-#include "tensorflow/core/framework/register_types.h"
-#include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/framework/types.h"
-
-namespace tensorflow {
-
-class BitcastOp : public OpKernel {
- public:
-  explicit BitcastOp(OpKernelConstruction* context) : OpKernel(context) {
-    OP_REQUIRES_OK(context, context->GetAttr("T", &input_data_type_));
-    OP_REQUIRES_OK(context, context->GetAttr("type", &output_data_type_));
-    in_size_ = DataTypeSize(input_data_type_);
-    out_size_ = DataTypeSize(output_data_type_);
-    int check_size =
-        std::max(in_size_, out_size_) % std::min(in_size_, out_size_);
-    OP_REQUIRES(
-        context, check_size == 0,
-        errors::InvalidArgument("cannot convert between datatype ",
-                                input_data_type_, " and ", output_data_type_));
-  }
-
-  void Compute(OpKernelContext* context) override {
-    const Tensor& input_tensor = context->input(0);
-
-    TensorShape adjusted_shape = input_tensor.shape();
-    OP_REQUIRES(context,
-                in_size_ >= out_size_ ||
-                    (input_tensor.dims() > 0 &&
-                     input_tensor.dim_size(input_tensor.dims() - 1) ==
-                         out_size_ / in_size_),
-                errors::InvalidArgument(
-                    "Cannot bitcast from ", DataTypeString(input_data_type_),
-                    " to ", DataTypeString(output_data_type_), ": shape ",
-                    input_tensor.shape().DebugString()));
-
-    if (out_size_ < in_size_) {
-      adjusted_shape.AddDim(in_size_ / out_size_);
-    } else if (out_size_ > in_size_) {
-      adjusted_shape.RemoveDim(input_tensor.dims() - 1);
-    }
-    Tensor output_tensor;
-
-    OP_REQUIRES_OK(context,
-                   output_tensor.BitcastFrom(input_tensor, output_data_type_,
-                                             adjusted_shape));
-    context->set_output(0, output_tensor);
-  }
-
-  bool IsExpensive() override { return false; }
-
- private:
-  DataType input_data_type_;
-  DataType output_data_type_;
-  int in_size_;
-  int out_size_;
-};
-
-REGISTER_KERNEL_BUILDER(Name("Bitcast").Device(DEVICE_CPU), BitcastOp);
-
-#if GOOGLE_CUDA
-REGISTER_KERNEL_BUILDER(Name("Bitcast").Device(DEVICE_GPU), BitcastOp);
-#endif  // GOOGLE_CUDA
-
-}  // end namespace tensorflow
diff --git a/tensorflow/core/kernels/bitcast_op.h b/tensorflow/core/kernels/bitcast_op.h
deleted file mode 100644
index 1f3659f303..0000000000
--- a/tensorflow/core/kernels/bitcast_op.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-// See docs in ../ops/array_ops.cc.
-
-#ifndef TENSORFLOW_CORE_KERNELS_BITCAST_OP_H_
-#define TENSORFLOW_CORE_KERNELS_BITCAST_OP_H_
-
-#include <string.h>  // for memcpy
-
-#include "tensorflow/core/framework/op_kernel.h"
-#include "tensorflow/core/framework/register_types.h"
-#include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/framework/types.h"
-
-#endif  // TENSORFLOW_CORE_KERNELS_BITCAST_OP_H_
-- 
GitLab


From 218df759b63b4d1ee8b5845112321f55330003a6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 6 Feb 2019 10:26:42 -0800
Subject: [PATCH 2001/2345] Added support for dequantization of inputs for some
 operators.

When an operator's input is floating-point but the weights or bias are quantized, Dequantize operators are added to convert quantized inputs to floating-point since NNAPI does not support mixing float and quantized types. This is needed for models using post-training quantization.

Only CONV_2D and FULLY_CONNECTED operations support this for now.

PiperOrigin-RevId: 232699006
---
 .../lite/delegates/nnapi/nnapi_delegate.cc    | 360 +++++++++++++-----
 .../delegates/nnapi/nnapi_delegate_test.cc    | 245 +++++++-----
 tensorflow/lite/nnapi/NeuralNetworksTypes.h   |   1 +
 3 files changed, 425 insertions(+), 181 deletions(-)

diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
index 8345eb7497..86fe7c5140 100644
--- a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
+++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
@@ -51,8 +51,47 @@ namespace {
   } while (0)
 
 namespace {
+
+bool IsFloat(TfLiteType type) {
+  switch (type) {
+    case kTfLiteFloat32:
+      return true;
+    default:
+      return false;
+  }
+}
+
+bool IsQuantized(TfLiteType type) {
+  switch (type) {
+    case kTfLiteUInt8:
+    case kTfLiteInt8:
+    case kTfLiteInt16:
+      return true;
+    default:
+      return false;
+  }
+}
+
+bool IsHybridOperator(const TfLiteContext* context, int builtin_code,
+                      const TfLiteNode* node) {
+  switch (builtin_code) {
+    case kTfLiteBuiltinConv2d:
+    case kTfLiteBuiltinFullyConnected: {
+      const int input_id = node->inputs->data[0];
+      const int filter_id = node->inputs->data[1];
+      const TfLiteType input_type = context->tensors[input_id].type;
+      const TfLiteType filter_type = context->tensors[filter_id].type;
+      return IsFloat(input_type) && IsQuantized(filter_type);
+    }
+    default:
+      return false;
+  }
+}
+
 constexpr int32_t kMinSdkVersionForNNAPI = 27;
 constexpr int32_t kMinSdkVersionForNNAPI11 = 28;
+constexpr int32_t kMinSdkVersionForNNAPI12 = 29;
+
 }  // namespace
 
 // RAII NN API Model Destructor for use with std::unique_ptr
@@ -147,16 +186,42 @@ class OperandMapping {
   std::vector<int> lite_tensor_to_ann_tensor_;
 };
 
+class DequantizeMapping {
+ public:
+  int DequantizedAnnIndex(int ann_index, TfLiteType type) const {
+    for (const auto& element : mapping_) {
+      if (ann_index == std::get<0>(element) && type == std::get<1>(element)) {
+        return std::get<2>(element);
+      }
+    }
+    return -1;
+  }
+
+  void Add(int ann_index, TfLiteType type, int dequantized_ann_index) {
+    // This assumes it is not already mapped.
+    mapping_.emplace_back(ann_index, type, dequantized_ann_index);
+  }
+
+ private:
+  // Each tuple specifies the ANN (quantized) tensor index, the desired
+  // floating-point type and the matching ANN (dequantized) tensor index. This
+  // could use a map but instead std::vector is used to keep code size lower.
+  std::vector<std::tuple<int, TfLiteType, int>> mapping_;
+};
+
 // Abstract builder for building an op in the NN API graph. This handles
 // the disparity between TFLite and NN API operand types. NN API has singular
 // operands for both tensors and parameters, and TFLite separates the two.
 class NNAPIOpBuilder {
  public:
   NNAPIOpBuilder(const NnApi* nnapi, TfLiteContext* context,
-                 OperandMapping* tensor_mapping, ANeuralNetworksModel* nn_model)
+                 OperandMapping* tensor_mapping,
+                 DequantizeMapping* dequantize_mapping,
+                 ANeuralNetworksModel* nn_model)
       : nnapi_(nnapi),
         context_(context),
         operand_mapping_(tensor_mapping),
+        dequantize_mapping_(dequantize_mapping),
         nn_model_(nn_model) {}
 
   TfLiteStatus AddScalarInt32Operand(int32_t value) {
@@ -190,50 +255,129 @@ class NNAPIOpBuilder {
     return kTfLiteOk;
   }
 
-  TfLiteStatus AddTensorInput(int tensor_index) {
-    int ann_index;
-    TF_LITE_ENSURE_STATUS(AddTensor(tensor_index, &ann_index));
-    augmented_inputs_.push_back(ann_index);
-    return kTfLiteOk;
+  TfLiteStatus AddTensorInput(int tensor_index, bool hybrid_op) {
+    return AddTensor(tensor_index, hybrid_op, &augmented_inputs_);
   }
 
   TfLiteStatus AddTensorOutput(int tensor_index) {
-    int ann_index;
-    TF_LITE_ENSURE_STATUS(AddTensor(tensor_index, &ann_index));
-    augmented_outputs_.push_back(ann_index);
-    return kTfLiteOk;
+    return AddTensor(tensor_index, /*hybrid_op=*/false, &augmented_outputs_);
   }
 
   TfLiteStatus AddAdditionalFloat32OutputTensor(uint32_t dimension_count) {
     std::vector<uint32_t> dims(dimension_count, 0);
-    ANeuralNetworksOperandType operand_type{
-        .type = ANEURALNETWORKS_TENSOR_FLOAT32,
-        .dimensionCount = dimension_count,
-        .dimensions = dims.data()};
+    return AddFloat32OutputTensor(dimension_count, dims.data(), nullptr);
+  }
+
+  TfLiteStatus AddStateFloat32Tensor(int tensor_index,
+                                     int* ann_tensor_index_out) {
+    TfLiteTensor* tensor = &context_->tensors[tensor_index];
+    return AddFloat32OutputTensor(
+        tensor->dims->size, reinterpret_cast<uint32_t*>(tensor->dims->data),
+        ann_tensor_index_out);
+  }
+
+  // Adds a Dequantize operator and replaces the input tensor index with the
+  // dequantized version. If the dequantized version of the operator already
+  // exists then it is not added again.
+  TfLiteStatus AddDequantize(int nn_input_index, int lite_index,
+                             TfLiteType dequantized_type) {
+    const int ann_index = operand_mapping_->lite_index_to_ann(lite_index);
+    int dequantized_ann_index =
+        dequantize_mapping_->DequantizedAnnIndex(ann_index, dequantized_type);
+
+    if (dequantized_ann_index == -1) {
+      // The dequantized version does not exist yet, it has to be added: a new
+      // Dequantize operation is added, yielding a new tensor.
+      const TfLiteTensor& tensor = context_->tensors[lite_index];
+      ANeuralNetworksOperandType operand_type{
+          dequantized_type, static_cast<uint32_t>(tensor.dims->size),
+          reinterpret_cast<uint32_t*>(tensor.dims->data), 0.f, 0};
+      RETURN_TFLITE_ERROR_IF_NN_ERROR(
+          context_,
+          nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
+      dequantized_ann_index = operand_mapping_->add_new_non_tensor_operand();
+
+      // Add Dequantize operation.
+      const uint32_t dequantize_input[1] = {static_cast<uint32_t>(ann_index)};
+      const uint32_t dequantize_output[1] = {
+          static_cast<uint32_t>(dequantized_ann_index)};
+      RETURN_TFLITE_ERROR_IF_NN_ERROR(
+          context_, nnapi_->ANeuralNetworksModel_addOperation(
+                        nn_model_, ANEURALNETWORKS_DEQUANTIZE, 1,
+                        dequantize_input, 1, dequantize_output));
+      dequantize_mapping_->Add(ann_index, dequantized_type,
+                               dequantized_ann_index);
+    }
+
+    // The input for the original operation is modified so that the operation
+    // now uses the dequantized tensor as input.
+    augmented_inputs_[nn_input_index] = dequantized_ann_index;
+
+    return kTfLiteOk;
+  }
+
+  // Finish emitting the op (of type `type`) into the NN API.
+  TfLiteStatus FinalizeAddOperation(ANeuralNetworksOperationType type) {
+    // Actually add a NN API operation
+    RETURN_TFLITE_ERROR_IF_NN_ERROR(
+        context_,
+        nnapi_->ANeuralNetworksModel_addOperation(
+            nn_model_, type, static_cast<uint32_t>(augmented_inputs_.size()),
+            augmented_inputs_.data(),
+            static_cast<uint32_t>(augmented_outputs_.size()),
+            augmented_outputs_.data()));
+    augmented_inputs_.clear();
+    augmented_outputs_.clear();
+    return kTfLiteOk;
+  }
+
+ private:
+  template <typename T>
+  TfLiteStatus AddScalarOperand(T value, int32_t nn_type) {
+    ANeuralNetworksOperandType operand_type{.type = nn_type};
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
         context_,
         nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
-    int ann_operand = operand_mapping_->add_new_non_tensor_operand();
-    augmented_outputs_.push_back(ann_operand);
+    const int ann_index = operand_mapping_->add_new_non_tensor_operand();
+    RETURN_TFLITE_ERROR_IF_NN_ERROR(
+        context_, nnapi_->ANeuralNetworksModel_setOperandValue(
+                      nn_model_, ann_index, &value, sizeof(T)));
+    augmented_inputs_.push_back(ann_index);
     return kTfLiteOk;
   }
 
-  TfLiteStatus AddStateFloat32Tensor(int tensor_index,
-                                     int* ann_tensor_index_out) {
-    TfLiteTensor* tensor = &context_->tensors[tensor_index];
-    int ann_index = operand_mapping_->add_new_non_tensor_operand();
+  template <typename T>
+  TfLiteStatus AddVectorOperand(const T* values, uint32_t num_values,
+                                int32_t nn_type) {
+    ANeuralNetworksOperandType operand_type{
+        .type = nn_type, .dimensionCount = 1, .dimensions = &num_values};
 
+    RETURN_TFLITE_ERROR_IF_NN_ERROR(
+        context_,
+        nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
+
+    const int ann_index = operand_mapping_->add_new_non_tensor_operand();
+    RETURN_TFLITE_ERROR_IF_NN_ERROR(
+        context_, nnapi_->ANeuralNetworksModel_setOperandValue(
+                      nn_model_, ann_index, values, sizeof(T) * num_values));
+    augmented_inputs_.push_back(ann_index);
+    return kTfLiteOk;
+  }
+
+  TfLiteStatus AddFloat32OutputTensor(uint32_t dimension_count,
+                                      const uint32_t* dimension_data,
+                                      int* ann_index_out) {
     ANeuralNetworksOperandType operand_type{
-        ANEURALNETWORKS_TENSOR_FLOAT32,
-        static_cast<uint32_t>(tensor->dims->size),
-        reinterpret_cast<uint32_t*>(tensor->dims->data), tensor->params.scale,
-        tensor->params.zero_point};
+        .type = ANEURALNETWORKS_TENSOR_FLOAT32,
+        .dimensionCount = dimension_count,
+        .dimensions = dimension_data,
+    };
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
         context_,
         nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
+    const int ann_index = operand_mapping_->add_new_non_tensor_operand();
     augmented_outputs_.push_back(ann_index);
-
-    *ann_tensor_index_out = ann_index;
+    if (ann_index_out) *ann_index_out = ann_index;
     return kTfLiteOk;
   }
 
@@ -241,10 +385,11 @@ class NNAPIOpBuilder {
   // This returns the NN API tensor index corresponding to the created tensor.
   // If another caller previously created a NN API tensor for `tensor_index`
   // then the existing one is returned.
-  TfLiteStatus AddTensor(int tensor_index, int* ann_tensor_index_out) {
+  TfLiteStatus AddTensor(int tensor_index, bool hybrid_op,
+                         std::vector<uint32_t>* indices) {
     int ann_tensor_index = operand_mapping_->lite_index_to_ann(tensor_index);
     if (ann_tensor_index != -1) {
-      *ann_tensor_index_out = ann_tensor_index;
+      indices->push_back(ann_tensor_index);
       return kTfLiteOk;
     }
     // Allocate a new tensor index
@@ -255,11 +400,17 @@ class NNAPIOpBuilder {
     float scale = 0.0f;
     int32_t zeroPoint = 0;
     TfLiteTensor* tensor = &context_->tensors[tensor_index];
-    switch (tensor->type) {
+    TfLiteType tensor_type = tensor->type;
+    if (hybrid_op && (tensor_type == kTfLiteUInt8)) {
+      // For legacy reason, UINT8 weights in hybrid operators are actually INT8
+      // values and should be interpreted as such.
+      tensor_type = kTfLiteInt8;
+    }
+    switch (tensor_type) {
       case kTfLiteNoType:
         // Tensors added during initialization of Ops don't have a type yet and
         // should not be registered with the NNAPI.
-        *ann_tensor_index_out = -1;
+        indices->push_back(-1);
         return kTfLiteOk;
       case kTfLiteFloat32:
         nn_type = ANEURALNETWORKS_TENSOR_FLOAT32;
@@ -273,6 +424,10 @@ class NNAPIOpBuilder {
           scale = 1;
         }
         break;
+      case kTfLiteInt8:
+        nn_type = ANEURALNETWORKS_TENSOR_QUANT8_SYMM;
+        scale = tensor->params.scale;
+        break;
       case kTfLiteInt32:
         nn_type = ANEURALNETWORKS_TENSOR_INT32;
         scale = tensor->params.scale;
@@ -298,53 +453,7 @@ class NNAPIOpBuilder {
               nn_model_, ann_tensor_index, tensor->data.raw, tensor->bytes));
     }
 
-    *ann_tensor_index_out = ann_tensor_index;
-    return kTfLiteOk;
-  }
-
-  // Finish emitting the op (of type `type`) into the NN API.
-  TfLiteStatus FinalizeAddOperation(ANeuralNetworksOperationType type) {
-    // Actually add a NN API operation
-    RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_,
-        nnapi_->ANeuralNetworksModel_addOperation(
-            nn_model_, type, static_cast<uint32_t>(augmented_inputs_.size()),
-            augmented_inputs_.data(),
-            static_cast<uint32_t>(augmented_outputs_.size()),
-            augmented_outputs_.data()));
-    augmented_inputs_.clear();
-    augmented_outputs_.clear();
-    return kTfLiteOk;
-  }
-
- private:
-  template <typename T>
-  TfLiteStatus AddScalarOperand(T value, int32_t nn_type) {
-    ANeuralNetworksOperandType operand_type{.type = nn_type};
-    RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_,
-        nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
-    int ann_operand = operand_mapping_->add_new_non_tensor_operand();
-    RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_, nnapi_->ANeuralNetworksModel_setOperandValue(
-                      nn_model_, ann_operand, &value, sizeof(T)));
-    augmented_inputs_.push_back(ann_operand);
-    return kTfLiteOk;
-  }
-
-  template <typename T>
-  TfLiteStatus AddVectorOperand(const T* values, uint32_t num_values,
-                                int32_t nn_type) {
-    ANeuralNetworksOperandType operand_type{
-        .type = nn_type, .dimensionCount = 1, .dimensions = &num_values};
-    RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_,
-        nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
-    int ann_operand = operand_mapping_->add_new_non_tensor_operand();
-    RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_, nnapi_->ANeuralNetworksModel_setOperandValue(
-                      nn_model_, ann_operand, values, sizeof(T) * num_values));
-    augmented_inputs_.push_back(ann_operand);
+    indices->push_back(ann_tensor_index);
     return kTfLiteOk;
   }
 
@@ -355,7 +464,13 @@ class NNAPIOpBuilder {
   TfLiteContext* const context_;
 
   // Tracks relationship between indices.
-  OperandMapping* operand_mapping_;
+  OperandMapping* const operand_mapping_;
+
+  // Keeps mapping of ANN quantized tensor and float data type to equivalent
+  // dequantized ANN tensor. For example, tensor #4 (UINT8) + FLOAT32 could map
+  // to tensor #10 (FLOAT32) because a DEQUANTIZE operator was added to convert
+  // tensor #4 to a FLOAT32 tensor.
+  DequantizeMapping* const dequantize_mapping_;
 
   // The NNAPI model.
   ANeuralNetworksModel* const nn_model_;
@@ -394,8 +509,9 @@ class NNAPIDelegateKernel {
   // Return a function that knows how to translate a node into its operands
   // when called. You can use this function to see if a node is supported
   // (i.e. that MappingFn is not nullptr).
-  static MappingFn Map(TfLiteContext* context, int builtin_code, int version,
-                       int android_sdk_version, TfLiteNode* node) {
+  static MappingFn Map(const TfLiteContext* context, int builtin_code,
+                       int version, int android_sdk_version,
+                       const TfLiteNode* node) {
     switch (builtin_code) {
       case kTfLiteBuiltinAdd:
         if (version == 1) {
@@ -451,6 +567,11 @@ class NNAPIDelegateKernel {
         break;
       case kTfLiteBuiltinConv2d:
         if (version == 1) {
+          if (android_sdk_version < kMinSdkVersionForNNAPI12 &&
+              IsHybridOperator(context, builtin_code, node)) {
+            // Hybrid operators not supported before NNAPI 1.2.
+            return nullptr;
+          }
           auto builtin =
               reinterpret_cast<TfLiteConvParams*>(node->builtin_data);
           if (builtin->dilation_width_factor != 1 ||
@@ -488,6 +609,11 @@ class NNAPIDelegateKernel {
         break;
       case kTfLiteBuiltinFullyConnected:
         if (version == 1) {
+          if (android_sdk_version < kMinSdkVersionForNNAPI12 &&
+              IsHybridOperator(context, builtin_code, node)) {
+            // Hybrid operators not supported before NNAPI 1.2.
+            return nullptr;
+          }
           return [](const NNAPIOpMappingArgs& mapping_args)
                      -> ANeuralNetworksOperationType {
             auto builtin = reinterpret_cast<TfLiteFullyConnectedParams*>(
@@ -957,18 +1083,69 @@ class NNAPIDelegateKernel {
   std::unique_ptr<NNMemory> nn_input_memory_;
   std::unique_ptr<NNMemory> nn_output_memory_;
 
+  void AddDequantizeOperatorsWhereNeeded(const TfLiteContext* context,
+                                         int builtin_code,
+                                         const TfLiteNode* node,
+                                         NNAPIOpBuilder* builder) {
+    // Depending on the operator and the input data format, Dequantize
+    // operators may need to be added. For example when the input is
+    // floating-point but weights are quantized then the weights will first be
+    // dequantized to the same format as the input before being passed to the
+    // operator.
+
+    // The tensor determining whether the inputs should be floating-point.
+    int input_tensor_index = -1;
+    std::vector<int> inputs_to_potentially_dequantize;
+
+    switch (builtin_code) {
+      case kTfLiteBuiltinConv2d:
+      case kTfLiteBuiltinFullyConnected: {
+        input_tensor_index = 0;
+        // Weights and bias are inputs #1 and #2 respectively and may require
+        // dequantization.
+        inputs_to_potentially_dequantize = {1, 2};
+        break;
+      }
+      default:
+        return;
+    }
+
+    int tensor_id = node->inputs->data[input_tensor_index];
+    if (tensor_id < 0) return;
+
+    // Nothing to do if the input is not floating-point.
+    if (!IsFloat(context->tensors[tensor_id].type)) return;
+
+    for (int i : inputs_to_potentially_dequantize) {
+      tensor_id = node->inputs->data[i];
+      if (tensor_id < 0) continue;  // Ignore optional input.
+
+      const TfLiteType type = context->tensors[tensor_id].type;
+      // Nothing to do for this tensor if it's not quantized.
+      if (type != kTfLiteUInt8) continue;
+
+      // Insert Dequantize operator if it hasn't been done already and change
+      // the node's input accordingly.
+      builder->AddDequantize(i, node->inputs->data[i], type);
+    }
+  }
+
   TfLiteStatus AddOpsAndTensors(TfLiteContext* context) {
-    // The operand builder allows creating a single op. We create it at this
-    // reduced power position rather than in the for loop to avoid reallocating
-    // the vectors.
-    NNAPIOpBuilder builder(nnapi_, context, &operand_mapping_, nn_model_.get());
-    // Add Tensors
-    // allocate outside to avoid realloc
+    DequantizeMapping dequantize_mapping;
+    // The operand builder allows creating a single op. It is created outside
+    // the for loop to avoid reallocating the vectors.
+    NNAPIOpBuilder builder(nnapi_, context, &operand_mapping_,
+                           &dequantize_mapping, nn_model_.get());
+    // Add Tensors.
     for (auto node_index : nodes_) {
       // Obtain the op and registration.
       TfLiteNode* node;
       TfLiteRegistration* reg;
-      context->GetNodeAndRegistration(context, node_index, &node, &reg);
+      TF_LITE_ENSURE_STATUS(
+          context->GetNodeAndRegistration(context, node_index, &node, &reg));
+
+      const bool hybrid_op = IsHybridOperator(context, reg->builtin_code, node);
+
       // Map inputs to NN API tensor indices.
       for (auto input_index : TfLiteIntArrayView(node->inputs)) {
         if (input_index == kOptionalTensor &&
@@ -980,7 +1157,7 @@ class NNAPIDelegateKernel {
           // tensor when supported by NNAPI.
           TF_LITE_ENSURE_STATUS(builder.AddVectorFloat32Operand(nullptr, 0));
         } else {
-          TF_LITE_ENSURE_STATUS(builder.AddTensorInput(input_index));
+          TF_LITE_ENSURE_STATUS(builder.AddTensorInput(input_index, hybrid_op));
         }
       }
       // Get op type and operands
@@ -993,6 +1170,11 @@ class NNAPIDelegateKernel {
         TF_LITE_ENSURE_STATUS(builder.AddTensorOutput(output_index));
       }
 
+      // Dequantize operators may have to be added in case inputs are to be
+      // floating-point.
+      AddDequantizeOperatorsWhereNeeded(context, reg->builtin_code, node,
+                                        &builder);
+
       builder.FinalizeAddOperation(nn_op_type);
     }
     return kTfLiteOk;
@@ -1026,7 +1208,7 @@ class NNAPIDelegateKernel {
       total_output_byte_size += context->tensors[i].bytes;
     }
 
-    // Add state output tensors as model inputs
+    // Add state output tensors as model outputs.
     for (int i : model_state_outputs_) {
       outputs.push_back(i);
     }
diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
index 5da052eb42..3751238638 100644
--- a/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
+++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
@@ -49,6 +49,24 @@ class SingleOpModelWithNNAPI : public SingleOpModel {
                                  const std::vector<int>& dims) {
     return interpreter_->ResizeInputTensor(tensor_index, dims);
   }
+
+ protected:
+  void SetData(int index, TensorType type, std::initializer_list<float> data) {
+    switch (type) {
+      case TensorType_FLOAT32:
+        PopulateTensor(index, data);
+        break;
+      case TensorType_INT32:
+        QuantizeAndPopulate<int32_t>(index, data);
+        break;
+      case TensorType_UINT8:
+        QuantizeAndPopulate<uint8_t>(index, data);
+        break;
+      default:
+        FAIL() << "Type not supported: " << type;
+        break;
+    }
+  }
 };
 
 class FloatAddOpModel : public SingleOpModelWithNNAPI {
@@ -225,14 +243,15 @@ TEST(NNAPIDelegate, L2PoolWithNoActivation) {
   EXPECT_THAT(m.GetOutput(), ElementsAreArray({3.5, 6.5}));
 }
 
-class BaseConvolutionOpModel : public SingleOpModelWithNNAPI {
+class ConvolutionOpModel : public SingleOpModelWithNNAPI {
  public:
-  BaseConvolutionOpModel(
+  ConvolutionOpModel(
       const TensorData& input, const TensorData& filter,
       const TensorData& output, int stride_width = 2, int stride_height = 2,
       enum Padding padding = Padding_VALID,
       enum ActivationFunctionType activation = ActivationFunctionType_NONE,
-      int dilation_width_factor = 1, int dilation_height_factor = 1) {
+      int dilation_width_factor = 1, int dilation_height_factor = 1)
+      : input_type_(input.type), filter_type_(filter.type) {
     input_ = AddInput(input);
     filter_ = AddInput(filter);
 
@@ -249,7 +268,8 @@ class BaseConvolutionOpModel : public SingleOpModelWithNNAPI {
     }
 
     output_ = AddOutput(output);
-    if (input.type != TensorType_FLOAT32) {
+
+    if (input_type_ != TensorType_FLOAT32) {
       // The following is required by quantized inference. It is the unittest's
       // responsibility to make sure the output scale falls into the correct
       // range.
@@ -265,56 +285,53 @@ class BaseConvolutionOpModel : public SingleOpModelWithNNAPI {
     BuildInterpreter({GetShape(input_), GetShape(filter_), GetShape(bias_)});
   }
 
- protected:
-  int input_;
-  int filter_;
-  int bias_;
-  int output_;
-};
-
-class ConvolutionOpModel : public BaseConvolutionOpModel {
- public:
-  using BaseConvolutionOpModel::BaseConvolutionOpModel;
-
-  void SetFilter(std::initializer_list<float> f) { PopulateTensor(filter_, f); }
-
-  void SetBias(std::initializer_list<float> f) { PopulateTensor(bias_, f); }
-
-  void SetInput(std::initializer_list<float> data) {
-    PopulateTensor(input_, data);
-  }
-  std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
-};
-
-class QuantizedConvolutionOpModel : public BaseConvolutionOpModel {
- public:
-  using BaseConvolutionOpModel::BaseConvolutionOpModel;
-
   void SetInput(std::initializer_list<float> data) {
-    QuantizeAndPopulate<uint8_t>(input_, data);
+    SetData(input_, input_type_, data);
   }
 
   void SetFilter(std::initializer_list<float> data) {
-    QuantizeAndPopulate<uint8_t>(filter_, data);
+    SetData(filter_, filter_type_, data);
   }
 
   void SetBias(std::initializer_list<float> data) {
-    QuantizeAndPopulate<int32_t>(bias_, data);
+    const auto bias_type =
+        (input_type_ == TensorType_FLOAT32) ? input_type_ : TensorType_INT32;
+    SetData(bias_, bias_type, data);
   }
 
-  std::vector<uint8_t> GetOutput() { return ExtractVector<uint8_t>(output_); }
-  std::vector<float> GetDequantizedOutput() {
-    return Dequantize<uint8_t>(ExtractVector<uint8_t>(output_),
-                               GetScale(output_), GetZeroPoint(output_));
+  std::vector<float> GetOutput() {
+    if (input_type_ == TensorType_FLOAT32) {
+      return ExtractVector<float>(output_);
+    } else {
+      return Dequantize<uint8_t>(ExtractVector<uint8_t>(output_),
+                                 GetScale(output_), GetZeroPoint(output_));
+    }
   }
+
+  std::vector<uint8_t> GetQuantizedOutput() {
+    if (input_type_ == TensorType_FLOAT32) {
+      return {};  // Not supported.
+    } else {
+      return ExtractVector<uint8_t>(output_);
+    }
+  }
+
+ protected:
+  int input_;
+  int filter_;
+  int bias_;
+  int output_;
+
+  const TensorType input_type_;
+  const TensorType filter_type_;
 };
 
 // In this tests we set the input and output scales so that the results
 // match exactly the 'non-quantized' version.
-TEST(NNAPIDelegate, SimpleTestQuantized) {
-  QuantizedConvolutionOpModel m({TensorType_UINT8, {2, 2, 4, 1}, -63.5, 64},
-                                {TensorType_UINT8, {3, 2, 2, 1}, -63.5, 64},
-                                {TensorType_UINT8, {}, -127, 128});
+TEST(ConvolutionOpTest, SimpleTestQuantized) {
+  ConvolutionOpModel m({TensorType_UINT8, {2, 2, 4, 1}, -63.5, 64},
+                       {TensorType_UINT8, {3, 2, 2, 1}, -63.5, 64},
+                       {TensorType_UINT8, {}, -127, 128});
   m.SetInput({
       // First batch
       1, 1, 1, 1,  // row = 1
@@ -332,25 +349,55 @@ TEST(NNAPIDelegate, SimpleTestQuantized) {
 
   m.Invoke();
 
-  EXPECT_THAT(m.GetDequantizedOutput(),
-              ElementsAreArray(ArrayFloatNear(
-                  {
-                      18, 2, 5,  // first batch, left
-                      18, 2, 5,  // first batch, right
-                      17, 4, 3,  // second batch, left
-                      37, 4, 3,  // second batch, right
-                  },
-                  1e-5)));
+  EXPECT_THAT(m.GetOutput(), ElementsAreArray(ArrayFloatNear(
+                                 {
+                                     18, 2, 5,  // first batch, left
+                                     18, 2, 5,  // first batch, right
+                                     17, 4, 3,  // second batch, left
+                                     37, 4, 3,  // second batch, right
+                                 },
+                                 1e-5)));
   // For good  measure, let's also verify the quantized values:
-  EXPECT_THAT(m.GetOutput(), ElementsAreArray({
-                                 145, 129, 132,  //
-                                 145, 129, 132,  //
-                                 144, 131, 130,  //
-                                 164, 131, 130,  //
-                             }));
+  EXPECT_THAT(m.GetQuantizedOutput(), ElementsAreArray({
+                                          145, 129, 132,  //
+                                          145, 129, 132,  //
+                                          144, 131, 130,  //
+                                          164, 131, 130,  //
+                                      }));
 }
 
-TEST(NNAPIDelegate, Conv2DWithNoActivation) {
+TEST(ConvolutionOpTest, FloatInputQuantizedWeights) {
+  ConvolutionOpModel m({TensorType_FLOAT32, {2, 2, 4, 1}},
+                       {TensorType_UINT8, {3, 2, 2, 1}, 0, 64},
+                       {TensorType_FLOAT32, {}});
+  m.SetInput({
+      // First batch
+      1, 1, 1, 2,  // row = 1
+      2, 2, 2, 1,  // row = 2
+      // Second batch
+      1, 2, 3, 4,  // row = 1
+      1, 2, 3, 4,  // row = 2
+  });
+  m.SetFilter({
+      1, 2, 3, 4,  // first 2x2 filter
+      0, 1, 0, 1,  // second 2x2 filter
+      0, 0, 1, 1,  // third 2x2 filter
+  });
+  m.SetBias({1, 2, 3});
+
+  m.Invoke();
+
+  EXPECT_THAT(m.GetOutput(), ElementsAreArray(ArrayFloatNear(
+                                 {
+                                     18, 5, 7,    // first batch, left
+                                     16, 5, 6,    // first batch, right
+                                     17, 6, 6,    // second batch, left
+                                     37, 10, 10,  // second batch, right
+                                 },
+                                 0.2)));
+}
+
+TEST(ConvolutionOpTest, NoActivation) {
   ConvolutionOpModel m({TensorType_FLOAT32, {2, 2, 4, 1}},
                        {TensorType_FLOAT32, {3, 2, 2, 1}},
                        {TensorType_FLOAT32, {}});
@@ -458,56 +505,48 @@ TEST(NNAPIDelegate, DepthwiseConv2DWithNoActivation) {
                              }));
 }
 
-class FloatFullyConnectedOpModel : public SingleOpModelWithNNAPI {
+class FullyConnectedOpModel : public SingleOpModelWithNNAPI {
  public:
-  FloatFullyConnectedOpModel(int units, int batches, const TensorData& input,
-                             const TensorData& output = {TensorType_FLOAT32})
-      : batches_(batches), units_(units) {
-    int total_input_size = 1;
-    for (int i = 0; i < input.shape.size(); ++i) {
-      total_input_size *= input.shape[i];
-    }
-    input_size_ = total_input_size / batches_;
-
+  FullyConnectedOpModel(
+      const TensorData& input, const TensorData& weights,
+      const TensorData& output,
+      enum ActivationFunctionType activation = ActivationFunctionType_NONE)
+      : input_type_(input.type), weights_type_(weights.type) {
     input_ = AddInput(input);
-    weights_ =
-        AddInput({input.type, {units_, input_size_}, input.min, input.max});
+    weights_ = AddInput(weights);
 
+    const int units = weights.shape[0];
     if (input.type == TensorType_FLOAT32) {
-      bias_ = AddInput({TensorType_FLOAT32, {units_}});
+      bias_ = AddInput({TensorType_FLOAT32, {units}});
     } else {
       // This is a quantized version. The scale of 'bias' depends on the scales
       // of input and filter. Supposedly this is correctly set during quantized
       // training.
       auto bias_scale = GetScale(input_) * GetScale(weights_);
-      TensorData bias{TensorType_INT32, {units_}, 0, 0, bias_scale};
+      TensorData bias{TensorType_INT32, {units}, 0, 0, bias_scale};
       bias_ = AddInput(bias);
     }
 
     output_ = AddOutput(output);
 
-    SetBuiltinOp(
-        BuiltinOperator_FULLY_CONNECTED, BuiltinOptions_FullyConnectedOptions,
-        CreateFullyConnectedOptions(builder_, ActivationFunctionType_RELU)
-            .Union());
+    SetBuiltinOp(BuiltinOperator_FULLY_CONNECTED,
+                 BuiltinOptions_FullyConnectedOptions,
+                 CreateFullyConnectedOptions(builder_, activation).Union());
     BuildInterpreter({GetShape(input_), GetShape(weights_), GetShape(bias_)});
   }
 
-  int input_size() { return input_size_; }
-  int num_units() { return units_; }
-  int num_batches() { return batches_; }
-
-  void SetBias(std::initializer_list<float> f) { PopulateTensor(bias_, f); }
-
-  void SetWeights(std::initializer_list<float> f) {
-    PopulateTensor(weights_, f);
+  void SetInput(std::initializer_list<float> data) {
+    SetData(input_, input_type_, data);
   }
 
-  void SetInput(std::initializer_list<float> data) {
-    PopulateTensor(input_, data);
+  void SetWeights(std::initializer_list<float> data) {
+    SetData(weights_, weights_type_, data);
   }
-  void SetInput(int offset, float* begin, float* end) {
-    PopulateTensor(input_, offset, begin, end);
+
+  void SetBias(std::initializer_list<float> data) {
+    const auto bias_type =
+        (input_type_ == TensorType_FLOAT32) ? input_type_ : TensorType_INT32;
+    SetData(bias_, bias_type, data);
   }
 
   std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
@@ -518,14 +557,14 @@ class FloatFullyConnectedOpModel : public SingleOpModelWithNNAPI {
   int bias_;
   int output_;
 
-  int batches_;
-  int units_;
-  int input_size_;
+  const TensorType input_type_;
+  const TensorType weights_type_;
 };
 
-TEST(NNAPIDelegate, FullyConnectedSimpleTest) {
-  FloatFullyConnectedOpModel m(/*units=*/3, /*batches=*/2,
-                               /*input=*/{TensorType_FLOAT32, {2, 10}});
+TEST(FullyConnectedOpTest, SimpleTest) {
+  FullyConnectedOpModel m(/*input=*/{TensorType_FLOAT32, {2, 10}},
+                          /*weights=*/{TensorType_FLOAT32, {3, 10}},
+                          /*output=*/{TensorType_FLOAT32});
   m.SetWeights({
       1, 2, 3, 4, 5, 6, 7, 8, 9, 10,  // u = 0
       1, 2, 3, 4, 5, 6, 7, 8, 9, 10,  // u = 1
@@ -543,6 +582,28 @@ TEST(NNAPIDelegate, FullyConnectedSimpleTest) {
   EXPECT_THAT(m.GetOutput(), ElementsAre(24, 25, 26, 58, 59, 60));
 }
 
+TEST(FullyConnectedOpTest, FloatInputQuantizedWeights) {
+  FullyConnectedOpModel m(/*input=*/{TensorType_FLOAT32, {2, 10}},
+                          /*weights=*/{TensorType_UINT8, {3, 10}, 0, 64},
+                          /*output=*/{TensorType_FLOAT32});
+  m.SetWeights({
+      1, 2, 3, 4, 5, 6, 7, 8, 9, 10,  // u = 0
+      1, 2, 3, 4, 5, 6, 7, 8, 9, 10,  // u = 1
+      1, 2, 3, 4, 5, 6, 7, 8, 9, 10,  // u = 1
+  });
+  m.SetBias({1, 2, 3});
+
+  m.SetInput({
+      1, 2, 3, 4, 5, 6, 7, 8,  -9, -10,  // b = 0
+      1, 2, 3, 4, 5, 6, 7, -8, 9,  -10,  // b = 1
+  });
+
+  m.Invoke();
+
+  EXPECT_THAT(m.GetOutput(),
+              ElementsAreArray(ArrayFloatNear({24, 25, 26, 58, 59, 60}, 1.3)));
+}
+
 class SoftmaxOpModel : public SingleOpModelWithNNAPI {
  public:
   SoftmaxOpModel(int batches, int size, float beta)
diff --git a/tensorflow/lite/nnapi/NeuralNetworksTypes.h b/tensorflow/lite/nnapi/NeuralNetworksTypes.h
index 573500dbb3..ba7eaf6763 100644
--- a/tensorflow/lite/nnapi/NeuralNetworksTypes.h
+++ b/tensorflow/lite/nnapi/NeuralNetworksTypes.h
@@ -39,6 +39,7 @@ enum {
   ANEURALNETWORKS_TENSOR_FLOAT32 = 3,
   ANEURALNETWORKS_TENSOR_INT32 = 4,
   ANEURALNETWORKS_TENSOR_QUANT8_ASYMM = 5,
+  ANEURALNETWORKS_TENSOR_QUANT8_SYMM = 13,
 };
 
 /**
-- 
GitLab


From 3bfbcb2abdd02508ba2224df7315798805e7e198 Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Wed, 6 Feb 2019 10:36:35 -0800
Subject: [PATCH 2002/2345] [XLA] Fix algebraic simplifier to handle
 instruction without layout when transforming A / Const to A * (1 / Const).

If the instruction that contains the constant value doesn't have a layout,
algebraic simplifier crashes when creating the inverse of the constant value.
The change is to always use the shape of the literal to create the new value.

PiperOrigin-RevId: 232700905
---
 .../xla/service/algebraic_simplifier.cc       |  7 +++---
 .../xla/service/algebraic_simplifier_test.cc  | 22 +++++++++++++++++++
 2 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index cd06cfcdd3..acc2c284b2 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -892,7 +892,6 @@ std::unique_ptr<HloInstruction> TryDivideToShift(HloInstruction* divide,
 }  // namespace
 
 Status AlgebraicSimplifierVisitor::HandleDivide(HloInstruction* divide) {
-  Shape* shape;
   HloInstruction *a, *b, *c, *d;
   CHECK(Match(divide, m::Divide(m::Op(&a), m::Op(&b))));
   // A/1 => A
@@ -955,6 +954,7 @@ Status AlgebraicSimplifierVisitor::HandleDivide(HloInstruction* divide) {
       break;
   }
 
+  Shape* shape;
   // exp(A)/exp(B) => exp(A-B)
   if (Match(divide, m::Divide(m::Exp(m::Op(&a)), m::Exp(m::Op(&b)))
                         .WithShape(m::Shape(&shape)))) {
@@ -1005,8 +1005,9 @@ Status AlgebraicSimplifierVisitor::HandleDivide(HloInstruction* divide) {
   // (Backends can do this transformation, but generally only if the constant is
   // a scalar.)
   if (Match(divide, m::Divide(m::NonConstant(&a), m::Constant(&b)))) {
-    Literal new_literal(b->shape());
-    switch (b->shape().element_type()) {
+    Shape result_shape = b->literal().shape();
+    Literal new_literal(result_shape);
+    switch (result_shape.element_type()) {
       case F16:
         TF_RETURN_IF_ERROR(InvertConstant<half>(*b, &new_literal));
         break;
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index 46d600600f..a908051037 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -4795,5 +4795,27 @@ TEST_F(AlgebraicSimplifierTest, ZeroSizedReshapeWithoutLayout) {
   EXPECT_THAT(root, op::Constant());
 }
 
+TEST_F(AlgebraicSimplifierTest, DividedByConstantInstructionWithoutLayout) {
+  Shape shape = ShapeUtil::MakeShape(F32, {});
+  shape.clear_layout();
+  auto builder = HloComputation::Builder(TestName());
+  HloInstruction* param = builder.AddInstruction(
+      HloInstruction::CreateParameter(0, shape, "param"));
+
+  HloInstruction* const_value = builder.AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<float>(20.0f)));
+  builder.AddInstruction(HloInstruction::CreateBinary(shape, HloOpcode::kDivide,
+                                                      param, const_value));
+
+  std::unique_ptr<VerifiedHloModule> module = CreateNewVerifiedModule();
+  module->AddEntryComputation(builder.Build());
+
+  AlgebraicSimplifierOptions options;
+  AlgebraicSimplifier simplifier(options);
+  EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
+  HloInstruction* root = module->entry_computation()->root_instruction();
+  EXPECT_THAT(root, op::Multiply());
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 1362eaa98e3e7778446eb767473450ef484996ca Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 6 Feb 2019 10:37:05 -0800
Subject: [PATCH 2003/2345] [Grappler] In the constant folding optimizer:
 Directly convert Fill, ZerosLike, and OnesLike with known output shape to
 Const nodes in compressed format without materializing the (potentially
 large) tensor value by evaluating the node.

PiperOrigin-RevId: 232701020
---
 tensorflow/core/grappler/optimizers/BUILD     |  1 +
 .../grappler/optimizers/constant_folding.cc   | 59 +++++++++++++++++++
 .../grappler/optimizers/constant_folding.h    |  4 +-
 .../optimizers/constant_folding_test.cc       | 51 +++++++++++++++-
 4 files changed, 113 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index cb5d7d6746..cbf0d680fe 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -103,6 +103,7 @@ cc_library(
         "//tensorflow/core/grappler/costs:graph_properties",
         "//tensorflow/core/grappler/utils:symbolic_shapes",
         "@com_google_absl//absl/container:flat_hash_set",
+        "@com_google_absl//absl/strings",
     ],
 )
 
diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc
index da7a328001..f883f8926f 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding.cc
@@ -17,6 +17,7 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/constant_folding.h"
 
+#include "absl/strings/string_view.h"
 #include "tensorflow/core/framework/allocator.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/function.pb.h"
@@ -716,6 +717,61 @@ Status ConstantFolding::MaterializeReductionIndices(
   return Status::OK();
 }
 
+Status ConstantFolding::MaterializeConstantValuedNode(
+    NodeDef* node, const GraphProperties& properties) {
+  // Nodes that generate constant-valued outputs can be represented compactly in
+  // compressed format, regardless of their shape.
+  const std::vector<OpInfo::TensorProperties>& output_props =
+      properties.GetOutputProperties(node->name());
+  if (output_props.size() != 1) return Status::OK();
+  const auto& output_shape = output_props[0].shape();
+  if (!PartialTensorShape(output_shape).IsFullyDefined()) {
+    return Status::OK();
+  }
+  if (IsFill(*node)) {
+    const auto output_dtype = output_props[0].dtype();
+    NodeDef* input_node = nullptr;
+    for (int i = 0; i < 2; ++i) {
+      input_node = node_map_->GetNode(NodeName(node->input(i)));
+      if (input_node == nullptr || !IsReallyConstant(*input_node)) {
+        return Status::OK();
+      }
+    }
+    TF_RETURN_IF_ERROR(CheckAttrExists(*input_node, "value"));
+    const TensorProto& input_tensor = input_node->attr().at("value").tensor();
+    // TODO(rmlarsen): Handle the case where the value is stored in
+    // tensor_content.
+    if (!input_tensor.tensor_content().empty()) {
+      return Status::OK();
+    }
+    TensorProto* tensor = (*node->mutable_attr())["value"].mutable_tensor();
+    // Copy the input tensor to the fill node, set the output shape, and
+    // change the nodd type to Const.
+    *tensor = input_tensor;
+    *(tensor->mutable_tensor_shape()) = output_shape;
+    (*node->mutable_attr())["dtype"].set_type(output_dtype);
+    node->mutable_attr()->erase("T");
+    node->mutable_attr()->erase("index_type");
+    node->set_op("Const");
+    for (int i = 0; i < 2; i++) {
+      // Change inputs to a control inputs.
+      const string ctrl_dep = AsControlDependency(node->input(i));
+      node_map_->UpdateInput(node->name(), node->input(i), ctrl_dep);
+      node->set_input(i, ctrl_dep);
+    }
+    graph_modified_ = true;
+  } else {
+    double value =
+        (IsZerosLike(*node) ? 0.0 : (IsOnesLike(*node) ? 1.0 : -1.0));
+    bool success = false;
+    if (value >= 0) {
+      TF_RETURN_IF_ERROR(ReplaceOperationWithConstant(
+          value, properties, output_shape, node, graph_, &success));
+    }
+  }
+  return Status::OK();
+}
+
 Status ConstantFolding::MaterializeConstants(
     const GraphProperties& properties) {
   const int node_count = graph_->node_size();
@@ -726,6 +782,8 @@ Status ConstantFolding::MaterializeConstants(
       TF_RETURN_IF_ERROR(MaterializeBroadcastGradientArgs(node, properties));
     } else if (IsReduction(node)) {
       TF_RETURN_IF_ERROR(MaterializeReductionIndices(&node, properties));
+    } else if (IsFill(node) || IsZerosLike(node) || IsOnesLike(node)) {
+      TF_RETURN_IF_ERROR(MaterializeConstantValuedNode(&node, properties));
     }
   }
   return Status::OK();
@@ -1569,6 +1627,7 @@ Status ConstantFolding::ReplaceOperationWithConstant(
     node->set_input(i, ctrl_dep);
   }
   *success = true;
+  graph_modified_ = true;
   return Status::OK();
 }
 
diff --git a/tensorflow/core/grappler/optimizers/constant_folding.h b/tensorflow/core/grappler/optimizers/constant_folding.h
index 4c532d7af1..7cf01b4b62 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding.h
+++ b/tensorflow/core/grappler/optimizers/constant_folding.h
@@ -67,8 +67,10 @@ class ConstantFolding : public GraphOptimizer {
                                           const GraphProperties& properties);
   Status MaterializeReductionIndices(NodeDef* node,
                                      const GraphProperties& properties);
-
+  Status MaterializeConstantValuedNode(NodeDef* node,
+                                       const GraphProperties& properties);
   Status MaterializeConstants(const GraphProperties& properties);
+
   bool IsFoldable(const NodeDef& node) const;
 
   Status EvaluateNode(const NodeDef& node,
diff --git a/tensorflow/core/grappler/optimizers/constant_folding_test.cc b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
index d7cabf5a8b..81d00fa5fb 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding_test.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
@@ -378,7 +378,7 @@ TEST_F(ConstantFoldingTest, NeutralElement) {
     const string ones_name = strings::StrCat("ones", suffix);
     const string ctrl_zeros_name = strings::StrCat("^zeros", suffix);
     const string ctrl_ones_name = strings::StrCat("^ones", suffix);
-    EXPECT_EQ(27, output.node_size());
+    EXPECT_EQ(const_type == kFill ? 31 : 27, output.node_size());
     for (int i = 0; i < output.node_size(); ++i) {
       const NodeDef& node = output.node(i);
       const string& name = node.name();
@@ -3466,6 +3466,55 @@ TEST_F(ConstantFoldingCastConstTest, CastConstFolding) {
   }
 }
 
+TEST_F(ConstantFoldingTest, MaterializeConstantValuedNode) {
+  tensorflow::Scope scope = tensorflow::Scope::NewRootScope();
+
+  Output x =
+      ops::Placeholder(scope.WithOpName("x"), DT_FLOAT,
+                       ops::Placeholder::Shape(TensorShape({1, 2, 3, 4})));
+  Output ones_like = ops::OnesLike(scope.WithOpName("ones_like"), x);
+  Output zeros_like = ops::ZerosLike(scope.WithOpName("zeros_like"), x);
+  Output fill = ops::Fill(scope.WithOpName("fill"), {4, 3, 2, 1}, 42);
+
+  GrapplerItem item;
+  TF_CHECK_OK(scope.ToGraphDef(&item.graph));
+  item.fetch = {"ones_like", "zeros_like", "fill"};
+  auto x_t = GenerateRandomTensor<DT_FLOAT>(TensorShape({1, 2, 3, 4}));
+  auto tensors_expected = EvaluateNodes(item.graph, item.fetch, {{"x", x_t}});
+
+  ConstantFolding optimizer(/*opt_level=*/RewriterConfig::AGGRESSIVE,
+                            /*cpu_device=*/nullptr);
+  GraphDef output;
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
+  TF_EXPECT_OK(status);
+
+  EXPECT_EQ(output.node_size(), 6);
+  for (const auto& node : output.node()) {
+    if (node.name() != "x") {
+      EXPECT_EQ(node.op(), "Const");
+    }
+    if (node.name() == "ones_like" || node.name() == "zeros_like") {
+      ASSERT_EQ(node.input_size(), 1);
+      EXPECT_EQ(node.input(0), "^x");
+    }
+    if (node.name() == "fill") {
+      ASSERT_EQ(node.input_size(), 2);
+      EXPECT_EQ(node.input(0)[0], '^');
+      EXPECT_EQ(node.input(1)[0], '^');
+    }
+  }
+  auto tensors = EvaluateNodes(output, item.fetch, {{"x", x_t}});
+  ASSERT_EQ(item.fetch.size(), tensors.size());
+  ASSERT_EQ(tensors_expected.size(), tensors.size());
+  for (int i = 0; i < tensors.size(); i++) {
+    if (item.fetch[i] == "fill") {
+      test::ExpectTensorEqual<int>(tensors_expected[i], tensors[i]);
+    } else {
+      test::ExpectTensorEqual<float>(tensors_expected[i], tensors[i]);
+    }
+  }
+}
+
 }  // namespace
 }  // namespace grappler
 }  // namespace tensorflow
-- 
GitLab


From 4e928af59b3a9f5ea4fad9a2cb8df3ae5bf8d9d5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 6 Feb 2019 10:46:02 -0800
Subject: [PATCH 2004/2345] Add thread name to profile result.

PiperOrigin-RevId: 232702876
---
 .../common_runtime/step_stats_collector.cc    | 25 +++++++++++++++++++
 .../common_runtime/step_stats_collector.h     |  6 +++++
 tensorflow/core/framework/step_stats.proto    |  2 ++
 .../core/profiler/internal/cpu/host_tracer.cc |  2 ++
 .../core/profiler/lib/profiler_session.cc     |  6 +++++
 5 files changed, 41 insertions(+)

diff --git a/tensorflow/core/common_runtime/step_stats_collector.cc b/tensorflow/core/common_runtime/step_stats_collector.cc
index 4926544565..1bdccf5f0e 100644
--- a/tensorflow/core/common_runtime/step_stats_collector.cc
+++ b/tensorflow/core/common_runtime/step_stats_collector.cc
@@ -409,6 +409,21 @@ void StepStatsCollector::Save(const string& device,
   }
 }
 
+void StepStatsCollector::SaveThreadName(const string& device,
+                                        const uint32 thread_id,
+                                        const string& thread_name) {
+  VLOG(1) << "Save dev " << device << " thread id " << thread_id << " name "
+          << thread_name;
+  {
+    mutex_lock l(mu_);
+    if (finalized_) {
+      LOG(WARNING) << "thread_name saved after finalize will not be collected.";
+    }
+    auto& thread_names_map = thread_names_[device];
+    thread_names_map[thread_id] = thread_name;
+  }
+}
+
 NodeExecStatsInterface* StepStatsCollector::CreateNodeExecStats(
     const Node* node) {
   // Only collect statistics for non-transfer nodes.
@@ -531,5 +546,15 @@ void StepStatsCollector::FinalizeInternal() {
       stats->stats()->Swap(dss->add_node_stats());
     }
   }
+  for (const auto& device_thread : thread_names_) {
+    if (dev_stats_pb.find(device_thread.first) == dev_stats_pb.end()) {
+      // skip device without DeviceStepStats.
+      continue;
+    }
+    DeviceStepStats* dss = dev_stats_pb.at(device_thread.first);
+    for (const auto& thread_name : device_thread.second) {
+      (*dss->mutable_thread_names())[thread_name.first] = thread_name.second;
+    }
+  }
 }
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/step_stats_collector.h b/tensorflow/core/common_runtime/step_stats_collector.h
index 7d34383ce8..dfcc51ff4c 100644
--- a/tensorflow/core/common_runtime/step_stats_collector.h
+++ b/tensorflow/core/common_runtime/step_stats_collector.h
@@ -175,6 +175,10 @@ class StepStatsCollector : public StepStatsCollectorInterface {
   void Save(const string& device, NodeExecStats* node_stats_pb);
   void Save(const string& device, NodeExecStatsWrapper* node_stats);
 
+  // Saves thread name.
+  void SaveThreadName(const string& device, const uint32 thread_id,
+                      const string& thread_name);
+
   NodeExecStatsInterface* CreateNodeExecStats(const Node* node) override;
   string ReportAllocsOnResourceExhausted(const string& err) override;
 
@@ -191,12 +195,14 @@ class StepStatsCollector : public StepStatsCollectorInterface {
   static const uint64 kMaxCollectedNodes = 1 << 20;
 
   typedef std::vector<std::unique_ptr<NodeExecStatsWrapper>> NodeStatsVector;
+  typedef std::unordered_map<uint32, string> ThreadNamesMap;
 
   void FinalizeInternal() EXCLUSIVE_LOCKS_REQUIRED(mu_);
 
   mutex mu_;
   bool finalized_ GUARDED_BY(mu_);
   std::unordered_map<string, NodeStatsVector> dev_stats_ GUARDED_BY(mu_);
+  std::unordered_map<string, ThreadNamesMap> thread_names_ GUARDED_BY(mu_);
   StepStats* step_stats_ GUARDED_BY(mu_);
   uint64 collected_nodes_ GUARDED_BY(mu_) = 0;
 };
diff --git a/tensorflow/core/framework/step_stats.proto b/tensorflow/core/framework/step_stats.proto
index 67cc9e3845..f8cab135ab 100644
--- a/tensorflow/core/framework/step_stats.proto
+++ b/tensorflow/core/framework/step_stats.proto
@@ -77,6 +77,8 @@ message NodeExecStats {
 message DeviceStepStats {
   string device = 1;
   repeated NodeExecStats node_stats = 2;
+  // Its key is thread id.
+  map<uint32, string> thread_names = 3;
 }
 
 message StepStats {
diff --git a/tensorflow/core/profiler/internal/cpu/host_tracer.cc b/tensorflow/core/profiler/internal/cpu/host_tracer.cc
index 1efc50695a..3fb2966468 100644
--- a/tensorflow/core/profiler/internal/cpu/host_tracer.cc
+++ b/tensorflow/core/profiler/internal/cpu/host_tracer.cc
@@ -79,6 +79,8 @@ Status HostTracer::CollectDataToCollector(
 
   const string cpu_name = "/host:CPU";
   for (auto& thread : events_) {
+    step_stats_collector->SaveThreadName(cpu_name, thread.thread.tid,
+                                         thread.thread.name);
     for (auto& event : thread.events) {
       if (!event.end_time) {
         auto it = end_times.find(event.activity_id);
diff --git a/tensorflow/core/profiler/lib/profiler_session.cc b/tensorflow/core/profiler/lib/profiler_session.cc
index 4641ae2a5f..1eb9ed6a99 100644
--- a/tensorflow/core/profiler/lib/profiler_session.cc
+++ b/tensorflow/core/profiler/lib/profiler_session.cc
@@ -47,6 +47,12 @@ void ConvertRunMetadataToTraceEvent(RunMetadata* run_metadata,
     resource.set_name("0");
     resource.set_resource_id(0);
     (*device.mutable_resources())[0] = resource;
+    for (const auto& thread_name : device_stats->thread_names()) {
+      tensorflow::tpu::Resource resource;
+      resource.set_resource_id(thread_name.first);
+      resource.set_name(thread_name.second);
+      (*device.mutable_resources())[thread_name.first] = resource;
+    }
     (*trace_devices)[device_id] = device;
 
     // Emit events.
-- 
GitLab


From 2dcb0a07c88ce34196d8fe55ab8f415e537e1484 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 6 Feb 2019 10:56:19 -0800
Subject: [PATCH 2005/2345] New Timestamped BFCAllocator and GPUKernelTracker.

The first part of this change extends BFCAllocator with an optional
timing counter for recording the time at which each Chunk is freed.
This has no effect for conventional memory management (as
applied to CPU RAM), but can achieve a new behavior when applied
to GPU RAM management.  The default TensorFlow memory allocation
convention for GPU RAM is to Unref the tensors Ref'd by a GPU Op as
soon as the Op has queued its kernel (and before that kernel is known
to have completed execution).  This is safe if the memory is
subsequently allocated to another GPU Op (the usual case) because that
second Op will be sequential on the single GPU compute stream and
hence won't touch the memory until the prior kernel has completed.
But this practice is unsafe if the memory is used for I/O or for an Op
queued on a different compute stream unless some further
synchronization is inserted.

Currently, I/O between a GPU and another device is made safe by
inserting stream dependencies.  Multi-compute-stream computation is
made safe by delaying the Unref of Ref'd tensors until the kernel is
known to have completed, via callback through the GPU-specific
EventMgr.  RDMA networking using GPUDirect is another difficult case
where stream synchronization is not possible and it is necessary to
wait until kernels are known to have completed before allowing
reallocation of the used memory.

Simply delaying the deallocation of memory until kernels are known to
have completed is unsatisfactory because it substantially raises the
high-water memory requirements of a program, drastically affecting the
model architectures that are feasible on a particular GPU model.  The
new freed-at count on BFCAllocator::Chunk is part of a strategy
for maintaining the high-water size efficiency of our current
single-compute-stream GPU memory allocation strategy while reducing
synchronization stalls in I/O uses of GPU RAM.  In the future it
may also be applied to multi-compute-stream execution.

The key idea is that when a request to allocate GPU memory is made we
can also pass along a 'freed-by' count and the allocator is free
to return any Chunk whose freed_count is <= that threshold.
This way we can continue to early-allocate GPU RAM without
restrictions to GPU kernels to be executed on a single compute stream,
while simultaneously satisfying the correctness constraints
needed for off-stream use.

GPUKernelTracker is the other component needed to make this new
strategy work.  It keeps track of the stream queuing and real
completion times of GPU kernels thus making it possible to pick the
largest safe freed-by count when making a request for GPU memory
that must be unemcumbered by other uses immediately.  A secondary
capability of the GPUKernelTracker is that it enables capping the
number of GPU kernels queued on a stream.  Without this cap some TF
models can experience moments when hundreds of kernels are queued on
the single compute stream.  Those queued but-not-executing kernels can
tie up memory that could be used for other purposes before its really
needed, and can delay I/O operations which are queued later and need
to wait for the compute stream to clear, for safety.

The new timestamped memory allocation strategy and pending-kernel
capping are considered experimental features and default off for
now, until more experience is gained.

PiperOrigin-RevId: 232705088
---
 tensorflow/core/BUILD                         |   1 +
 .../core/common_runtime/bfc_allocator.cc      |  64 +++++--
 .../core/common_runtime/bfc_allocator.h       |  26 ++-
 .../core/common_runtime/gpu/gpu_device.cc     | 157 +++++++++++++++++-
 .../core/common_runtime/gpu/gpu_device.h      |  85 ++++++++++
 .../common_runtime/gpu/gpu_device_test.cc     |  66 ++++++++
 .../core/common_runtime/gpu/gpu_event_mgr.cc  |  15 +-
 .../common_runtime/gpu/gpu_process_state.cc   |  31 +++-
 .../common_runtime/gpu/gpu_process_state.h    |   5 +
 .../core/common_runtime/shared_counter.h      |  31 ++++
 .../core/distributed_runtime/tensor_coding.cc |   5 +-
 .../core/distributed_runtime/tensor_coding.h  |   3 +-
 tensorflow/core/framework/allocator.h         |   4 +
 tensorflow/core/framework/device_base.h       |   9 +
 tensorflow/core/protobuf/config.proto         |  10 ++
 .../golden/v1/tensorflow.-g-p-u-options.pbtxt |  12 ++
 16 files changed, 487 insertions(+), 37 deletions(-)
 create mode 100644 tensorflow/core/common_runtime/shared_counter.h

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index cf212fc5d7..4cf37a6758 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -2926,6 +2926,7 @@ tf_cuda_library(
 
 CORE_CPU_LIB_HEADERS = CORE_CPU_BASE_HDRS + [
     "common_runtime/allocator_retry.h",
+    "common_runtime/shared_counter.h",
     "common_runtime/base_collective_executor.h",
     "common_runtime/bfc_allocator.h",
     "common_runtime/hierarchical_tree_broadcaster.h",
diff --git a/tensorflow/core/common_runtime/bfc_allocator.cc b/tensorflow/core/common_runtime/bfc_allocator.cc
index 3843ea9e60..c7e535cc80 100644
--- a/tensorflow/core/common_runtime/bfc_allocator.cc
+++ b/tensorflow/core/common_runtime/bfc_allocator.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/bfc_allocator.h"
 
 #include "tensorflow/core/common_runtime/allocator_retry.h"
+#include "tensorflow/core/framework/device_base.h"
 #include "tensorflow/core/lib/core/bits.h"
 #include "tensorflow/core/lib/gtl/stl_util.h"
 #include "tensorflow/core/lib/strings/numbers.h"
@@ -152,6 +153,7 @@ bool BFCAllocator::Extend(size_t alignment, size_t rounded_bytes) {
   c->allocation_id = -1;
   c->prev = kInvalidChunkHandle;
   c->next = kInvalidChunkHandle;
+  c->freed_count = 0;
 
   region_manager_.set_handle(c->ptr, h);
 
@@ -180,29 +182,46 @@ void BFCAllocator::DeallocateChunk(ChunkHandle h) {
   free_chunks_list_ = h;
 }
 
-void* BFCAllocator::AllocateRaw(size_t unused_alignment, size_t num_bytes) {
+void* BFCAllocator::AllocateRawInternalWithRetry(
+    size_t unused_alignment, size_t num_bytes,
+    const AllocationAttributes& allocation_attr) {
   // Fast path: Try once to allocate without getting the retry_helper_ involved
-  void* r = AllocateRawInternal(unused_alignment, num_bytes, false);
+  uint64 freed_by_count = 0;
+  if (allocation_attr.freed_by_func != nullptr) {
+    freed_by_count = allocation_attr.freed_by_func();
+  }
+  void* r =
+      AllocateRawInternal(unused_alignment, num_bytes, false, freed_by_count);
   if (r != nullptr) {
     return r;
   } else {
     static const int64 kMaxMillisToWait = 10000;  // 10 seconds
-    return retry_helper_.AllocateRaw(
-        [this](size_t a, size_t nb, bool v) {
-          return AllocateRawInternal(a, nb, v);
+    r = retry_helper_.AllocateRaw(
+        [this, &allocation_attr](size_t a, size_t nb, bool v) {
+          uint64 freed_by_count = 0;
+          if (allocation_attr.freed_by_func != nullptr) {
+            freed_by_count = allocation_attr.freed_by_func();
+          }
+          return AllocateRawInternal(a, nb, v, freed_by_count);
         },
         kMaxMillisToWait, unused_alignment, num_bytes);
+    return r;
   }
 }
 
 void* BFCAllocator::AllocateRaw(size_t unused_alignment, size_t num_bytes,
                                 const AllocationAttributes& allocation_attr) {
+  VLOG(1) << "AllocateRaw " << Name() << "  " << num_bytes;
   if (allocation_attr.no_retry_on_failure) {
     // Return immediately upon the first failure if this is for allocating an
     // optional scratch space.
     bool dump_log_on_failure = VLOG_IS_ON(2);
-    void* result =
-        AllocateRawInternal(unused_alignment, num_bytes, dump_log_on_failure);
+    uint64 freed_by_count = 0;
+    if (allocation_attr.freed_by_func != nullptr) {
+      freed_by_count = allocation_attr.freed_by_func();
+    }
+    void* result = AllocateRawInternal(unused_alignment, num_bytes,
+                                       dump_log_on_failure, freed_by_count);
     if (result == nullptr) {
       static std::atomic<int32> log_counter{0};
       int32 counter_value = log_counter.load(std::memory_order_relaxed);
@@ -218,7 +237,8 @@ void* BFCAllocator::AllocateRaw(size_t unused_alignment, size_t num_bytes,
     }
     return result;
   } else {
-    return AllocateRaw(unused_alignment, num_bytes);
+    return AllocateRawInternalWithRetry(unused_alignment, num_bytes,
+                                        allocation_attr);
   }
 }
 
@@ -233,7 +253,8 @@ size_t BFCAllocator::RoundedBytes(size_t bytes) {
 
 void* BFCAllocator::AllocateRawInternal(size_t unused_alignment,
                                         size_t num_bytes,
-                                        bool dump_log_on_failure) {
+                                        bool dump_log_on_failure,
+                                        uint64 freed_before) {
   if (num_bytes == 0) {
     LOG(ERROR) << "tried to allocate 0 bytes";
     return nullptr;
@@ -247,14 +268,14 @@ void* BFCAllocator::AllocateRawInternal(size_t unused_alignment,
   BinNum bin_num = BinNumForSize(rounded_bytes);
 
   mutex_lock l(lock_);
-  void* ptr = FindChunkPtr(bin_num, rounded_bytes, num_bytes);
+  void* ptr = FindChunkPtr(bin_num, rounded_bytes, num_bytes, freed_before);
   if (ptr != nullptr) {
     return ptr;
   }
 
   // Try to extend
   if (Extend(unused_alignment, rounded_bytes)) {
-    ptr = FindChunkPtr(bin_num, rounded_bytes, num_bytes);
+    ptr = FindChunkPtr(bin_num, rounded_bytes, num_bytes, freed_before);
     if (ptr != nullptr) {
       return ptr;
     }
@@ -274,7 +295,7 @@ void* BFCAllocator::AllocateRawInternal(size_t unused_alignment,
 }
 
 void* BFCAllocator::FindChunkPtr(BinNum bin_num, size_t rounded_bytes,
-                                 size_t num_bytes) {
+                                 size_t num_bytes, uint64 freed_before) {
   // First identify the first bin that could satisfy rounded_bytes.
   for (; bin_num < kNumBins; bin_num++) {
     // Start searching from the first bin for the smallest chunk that fits
@@ -285,6 +306,9 @@ void* BFCAllocator::FindChunkPtr(BinNum bin_num, size_t rounded_bytes,
       const BFCAllocator::ChunkHandle h = (*citer);
       BFCAllocator::Chunk* chunk = ChunkFromHandle(h);
       DCHECK(!chunk->in_use());
+      if (freed_before > 0 && freed_before < chunk->freed_count) {
+        continue;
+      }
       if (chunk->size >= rounded_bytes) {
         // We found an existing chunk that fits us that wasn't in use, so remove
         // it from the free bin structure prior to using.
@@ -347,6 +371,9 @@ void BFCAllocator::SplitChunk(BFCAllocator::ChunkHandle h, size_t num_bytes) {
   // The new chunk is not in use.
   new_chunk->allocation_id = -1;
 
+  // It inherits the freed time.
+  new_chunk->freed_count = c->freed_count;
+
   // Maintain the pointers.
   // c <-> c_neighbor becomes
   // c <-> new_chunk <-> c_neighbor
@@ -415,6 +442,9 @@ void BFCAllocator::Merge(BFCAllocator::ChunkHandle h1,
   // Set the new size
   c1->size += c2->size;
 
+  // Pick latest free time.
+  c1->freed_count = std::max(c1->freed_count, c2->freed_count);
+
   DeleteChunk(h2);
 }
 
@@ -460,6 +490,11 @@ void BFCAllocator::FreeAndMaybeCoalesce(BFCAllocator::ChunkHandle h) {
   // Mark the chunk as no longer in use.
   c->allocation_id = -1;
 
+  // Optionally record the free time.
+  if (timing_counter_) {
+    c->freed_count = timing_counter_->next();
+  }
+
   // Updates the stats.
   stats_.bytes_in_use -= c->size;
 
@@ -630,7 +665,10 @@ void BFCAllocator::DumpMemoryLog(size_t num_bytes) {
         in_use_by_size[c->size]++;
       }
       LOG(INFO) << (c->in_use() ? "Chunk" : "Free ") << " at " << c->ptr
-                << " of size " << c->size;
+                << " of size " << c->size
+                << (timing_counter_
+                        ? strings::StrCat(" freed_count ", c->freed_count)
+                        : "");
       h = c->next;
     }
   }
diff --git a/tensorflow/core/common_runtime/bfc_allocator.h b/tensorflow/core/common_runtime/bfc_allocator.h
index 2d74bf2b28..261bacbaac 100644
--- a/tensorflow/core/common_runtime/bfc_allocator.h
+++ b/tensorflow/core/common_runtime/bfc_allocator.h
@@ -23,6 +23,7 @@ limitations under the License.
 #include <vector>
 
 #include "tensorflow/core/common_runtime/allocator_retry.h"
+#include "tensorflow/core/common_runtime/shared_counter.h"
 #include "tensorflow/core/framework/allocator.h"
 #include "tensorflow/core/lib/gtl/stl_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
@@ -50,9 +51,14 @@ class BFCAllocator : public Allocator {
   ~BFCAllocator() override;
 
   string Name() override { return name_; }
-  void* AllocateRaw(size_t alignment, size_t num_bytes) override;
+
+  void* AllocateRaw(size_t alignment, size_t num_bytes) override {
+    return AllocateRaw(alignment, num_bytes, AllocationAttributes());
+  }
+
   void* AllocateRaw(size_t alignment, size_t num_bytes,
                     const AllocationAttributes& allocation_attr) override;
+
   void DeallocateRaw(void* ptr) override;
 
   bool TracksAllocationSizes() override;
@@ -67,11 +73,19 @@ class BFCAllocator : public Allocator {
 
   void ClearStats() override;
 
+  void SetTimingCounter(SharedCounter* sc) { timing_counter_ = sc; }
+
  private:
   struct Bin;
 
   void* AllocateRawInternal(size_t alignment, size_t num_bytes,
-                            bool dump_log_on_failure);
+                            bool dump_log_on_failure,
+                            uint64 freed_before_count);
+
+  void* AllocateRawInternalWithRetry(
+      size_t alignment, size_t num_bytes,
+      const AllocationAttributes& allocation_attr);
+
   void DeallocateRawInternal(void* ptr);
 
   // A ChunkHandle is an index into the chunks_ vector in BFCAllocator
@@ -126,6 +140,9 @@ class BFCAllocator : public Allocator {
     // What bin are we in?
     BinNum bin_num = kInvalidBinNum;
 
+    // Optional count when this chunk was most recently made free.
+    uint64 freed_count = 0;
+
     bool in_use() const { return allocation_id != -1; }
 
     string DebugString(BFCAllocator* a,
@@ -314,8 +331,8 @@ class BFCAllocator : public Allocator {
 
   // Returns a pointer to an underlying allocated chunk of size
   // 'rounded_bytes'.
-  void* FindChunkPtr(BinNum bin_num, size_t rounded_bytes, size_t num_bytes)
-      EXCLUSIVE_LOCKS_REQUIRED(lock_);
+  void* FindChunkPtr(BinNum bin_num, size_t rounded_bytes, size_t num_bytes,
+                     uint64 freed_before) EXCLUSIVE_LOCKS_REQUIRED(lock_);
 
   // Splits the chunk specified by 'h' into two chunks, one at least
   // of size 'num_bytes'.
@@ -420,6 +437,7 @@ class BFCAllocator : public Allocator {
 
   std::unique_ptr<SubAllocator> sub_allocator_;
   string name_;
+  SharedCounter* timing_counter_ = nullptr;
 
   // Structures mutable after construction
   mutable mutex lock_;
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.cc b/tensorflow/core/common_runtime/gpu/gpu_device.cc
index 010fdff4e9..80d221af36 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.cc
@@ -276,6 +276,28 @@ BaseGPUDevice::BaseGPUDevice(const SessionOptions& options, const string& name,
       sync_every_op_(sync_every_op),
       max_streams_(max_streams) {
   GPUProcessState::singleton()->EnableGPUDevice();
+  pending_cap_ = options.config.gpu_options().experimental().pending_cap();
+  timestamped_allocator_ =
+      options.config.gpu_options().experimental().timestamped_allocator();
+  if (timestamped_allocator_ || pending_cap_ > 0) {
+    std::unique_ptr<SharedCounter> timing_counter;
+    if (timestamped_allocator_) {
+      // In this case the SharedCounter was already created and set in the
+      // associated Allocator, with ownership by GPUProcessState.  Here we take
+      // over ownership of that SharedAllocator to transfer it to the
+      // GPUKernelTracker.
+      timing_counter =
+          GPUProcessState::singleton()->ReleaseGPUAllocatorCounter(tf_gpu_id);
+      DCHECK(timing_counter.get());
+    } else {
+      DCHECK_GT(pending_cap_, 0);
+      // In this case we need a SharedCounter to be owned by GPUKernelTracker
+      // but one was not created for use by the Allocator, so we create one.
+      timing_counter.reset(new SharedCounter);
+    }
+    kernel_tracker_.reset(
+        new GPUKernelTracker(Env::Default(), std::move(timing_counter)));
+  }
 }
 
 BaseGPUDevice::~BaseGPUDevice() {
@@ -508,6 +530,10 @@ void BaseGPUDevice::ComputeHelper(OpKernel* op_kernel,
       if (idc->stream() != stream) stream->ThenWaitFor(idc->stream());
     }
   }
+  if (pending_cap_ > 0) {
+    DCHECK(kernel_tracker_);
+    kernel_tracker_->PauseWhilePendingExceeds(pending_cap_);
+  }
   se::cuda::ScopedActivateExecutorContext scoped_activation{stream->parent()};
   op_kernel->Compute(context);
   if (context->status().ok()) {
@@ -525,6 +551,14 @@ void BaseGPUDevice::ComputeHelper(OpKernel* op_kernel,
       VLOG(1) << "GpuDevice::ComputeHelper scheduled "
               << ComputeOpKernelDebugString(*op_kernel, stream_id);
     }
+    if (kernel_tracker_) {
+      GPUKernelTracker* tracker = kernel_tracker_.get();
+      DCHECK(tracker);
+      uint64 queued_count = tracker->RecordQueued();
+      em_->ThenExecute(stream, [op_kernel, tracker, queued_count]() {
+        tracker->RecordTerminated(queued_count);
+      });
+    }
   } else {
     if (vlog_1) {
       VLOG(1) << "GpuDevice::ComputeHelper failed to schedule "
@@ -721,8 +755,8 @@ Status ParseVisibleDeviceList(const string& visible_device_list,
       if (!strings::safe_strto32(platform_gpu_id_str, &platform_gpu_id)) {
         return errors::InvalidArgument(
             "Could not parse entry in 'visible_device_list': '",
-            platform_gpu_id_str, "'. visible_device_list = ",
-            visible_device_list);
+            platform_gpu_id_str,
+            "'. visible_device_list = ", visible_device_list);
       }
       if (platform_gpu_id < 0 ||
           platform_gpu_id >= gpu_manager->VisibleDeviceCount()) {
@@ -957,15 +991,15 @@ Status BaseGPUDeviceFactory::CreateDevices(
     for (PlatformGpuId platform_gpu_id : valid_platform_gpu_ids) {
       err = cudaSetDevice(platform_gpu_id.value());
       if (err != cudaSuccess) {
-        return errors::Internal("cudaSetDevice() on GPU:",
-                                platform_gpu_id.value(), " failed. Status: ",
-                                cudaGetErrorString(err));
+        return errors::Internal(
+            "cudaSetDevice() on GPU:", platform_gpu_id.value(),
+            " failed. Status: ", cudaGetErrorString(err));
       }
       err = cudaFree(nullptr);
       if (err != cudaSuccess) {
         return errors::Internal("CUDA runtime implicit initialization on GPU:",
-                                platform_gpu_id.value(), " failed. Status: ",
-                                cudaGetErrorString(err));
+                                platform_gpu_id.value(),
+                                " failed. Status: ", cudaGetErrorString(err));
       }
     }
     // Reset to the original device.
@@ -1517,6 +1551,115 @@ Status BaseGPUDeviceFactory::GetValidDeviceIds(
   return Status::OK();
 }
 
+uint64 BaseGPUDevice::SafeAllocFrontier() {
+  if (timestamped_allocator_) {
+    return kernel_tracker_->LastTerminatedCount();
+  } else {
+    return 0;
+  }
+}
+
+int BaseGPUDevice::PendingKernels() {
+  if (kernel_tracker_) {
+    return kernel_tracker_->NumPending();
+  }
+  return 0;
+}
+
+uint64 GPUKernelTracker::RecordQueued() {
+  mutex_lock l(mu_);
+  uint64 queued_count = timing_counter_->next();
+  VLOG(2) << "RecordQueued queued_count=" << queued_count
+          << " first_available_=" << first_available_
+          << " last_completed_=" << last_completed_
+          << " num_pending_=" << num_pending_;
+  pending_kernels_[first_available_].queued_count = queued_count;
+  pending_kernels_[first_available_].terminated = false;
+  ++first_available_;
+  ++num_pending_;
+  if (first_available_ >= pending_kernels_.size()) {
+    first_available_ = 0;
+  }
+  if (first_available_ == last_completed_) {
+    // Ring buffer is full: double it.  All of the same valid PendingKernel
+    // entries exist after the copy, they are just shifted to begin
+    // at index 0 in the new array.
+    std::vector<PendingKernel> new_buffer(pending_kernels_.size() * 2);
+    for (int i = 0; i < pending_kernels_.size(); ++i) {
+      int j = (i + last_completed_) % pending_kernels_.size();
+      new_buffer[i] = pending_kernels_[j];
+    }
+    last_completed_ = 0;
+    first_available_ = pending_kernels_.size();
+    pending_kernels_.swap(new_buffer);
+    VLOG(1) << "last_completed_=" << last_completed_
+            << " first_available_=" << first_available_
+            << " num_pending_=" << num_pending_;
+  }
+  DCHECK_NE(first_available_, last_completed_) << "exhausted pending_kernels";
+  return queued_count;
+}
+
+void GPUKernelTracker::RecordTerminated(uint64 queued_count) {
+  mutex_lock l(mu_);
+  VLOG(2) << "RecordTerminated queued_count=" << queued_count
+          << " first_available_=" << first_available_
+          << " last_completed_=" << last_completed_
+          << " num_pending_=" << num_pending_ << " LC="
+          << ((last_completed_ >= 0)
+                  ? pending_kernels_[last_completed_].queued_count
+                  : -1);
+  DCHECK_NE(first_available_, last_completed_);
+  DCHECK_GT(num_pending_, 0);
+  // Starting just past the last completed entry, find the entry with
+  // this queued_count and mark it done.
+  int index = (last_completed_ + 1) % pending_kernels_.size();
+  while (true) {
+    if (index == first_available_) {
+      // This should never happen.
+      LOG(FATAL) << "Failed to find " << queued_count  // Crash OK
+                 << " in queue";
+    }
+    if (pending_kernels_[index].queued_count == queued_count) {
+      pending_kernels_[index].terminated = true;
+      break;
+    }
+    index = (index + 1) % pending_kernels_.size();
+  }
+  // Next move last_completed_ forward past all completed kernels.  In theory
+  // kernels should always complete in queued order so we should be able to
+  // advance the completed frontier to the last queued PendingKernel.  In
+  // practice we occassionally see the termination callbacks arrive out of order
+  // probably because of thread scheduling.  Eventually we may support out-of-
+  // order completion involving multple compute streams so here we follow a
+  // conservative approach and wait for every single callback to arrive before
+  // advancing the frontier.
+  while (true) {
+    int next_index = (last_completed_ + 1) % pending_kernels_.size();
+    if (next_index == first_available_) break;
+    if (pending_kernels_[next_index].terminated) {
+      last_completed_ = next_index;
+    } else {
+      break;
+    }
+  }
+  // Last decrease num_pending before maybe waking a waiter.
+  --num_pending_;
+  pending_decreased_.notify_one();
+}
+
+uint64 GPUKernelTracker::LastTerminatedCount() {
+  mutex_lock l(mu_);
+  if (last_completed_ < 0) {
+    // This is an edge case that can be encountered only at the beginning of
+    // execution.  There's not yet a safe threshold count. We don't want to
+    // return 0 since that bypasses the count mechanism in BFCAllocator, so
+    // return the least non-zero value.
+    return 1;
+  }
+  return pending_kernels_[last_completed_].queued_count;
+}
+
 }  // namespace tensorflow
 
 #endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.h b/tensorflow/core/common_runtime/gpu/gpu_device.h
index d002d02c51..33f0585b4d 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.h
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.h
@@ -34,6 +34,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/gpu_device_context.h"
 #include "tensorflow/core/common_runtime/local_device.h"
 #include "tensorflow/core/common_runtime/scoped_allocator_mgr.h"
+#include "tensorflow/core/common_runtime/shared_counter.h"
 #include "tensorflow/core/framework/allocator.h"
 #include "tensorflow/core/framework/device_base.h"
 #include "tensorflow/core/framework/op_kernel.h"
@@ -46,6 +47,7 @@ limitations under the License.
 #include "tensorflow/core/public/session_options.h"
 
 namespace tensorflow {
+class GPUKernelTracker;
 
 class BaseGPUDevice : public LocalDevice {
  public:
@@ -114,6 +116,17 @@ class BaseGPUDevice : public LocalDevice {
     return scoped_allocator_mgr_.get();
   }
 
+  // The following two functions always return 0 unless one of the
+  // related experimental config options has been specified.
+
+  // If returned value is > 0 then GPU Memory chunks freed before this count
+  // are guaranteed not to be in use by any kernel pending on this device.
+  uint64 SafeAllocFrontier() override;
+
+  // Returns the number of kernels that have been queued for execution on
+  // the compute stream and are not yet known to have completed.
+  int PendingKernels();
+
  protected:
   Allocator* gpu_allocator_;  // not owned
   Allocator* cpu_allocator_;  // not owned
@@ -141,6 +154,9 @@ class BaseGPUDevice : public LocalDevice {
   const int32 max_streams_;
   std::unique_ptr<EventMgr> em_;
   std::unique_ptr<thread::ThreadPool> thread_pool_;
+  std::unique_ptr<GPUKernelTracker> kernel_tracker_;
+  int pending_cap_ = 0;
+  bool timestamped_allocator_ = false;
 
   // Initialize scractch buffers used by Eigen.
   Status InitScratchBuffers();
@@ -163,6 +179,75 @@ class BaseGPUDevice : public LocalDevice {
                               StatusCallback done);
 };
 
+// A per-compute-stream utility that keeps track of kernels that have been
+// queued for execution but may not yet have terminated, and also the queued
+// time of the most recently terminated kernel.
+class GPUKernelTracker {
+ public:
+  explicit GPUKernelTracker(Env* env,
+                            std::unique_ptr<SharedCounter> timing_counter)
+      : env_(env),
+        timing_counter_(std::move(timing_counter)),
+        pending_kernels_(64) {}
+
+  // Record that a GPU kernel has just been enqueued on the compute stream.
+  // Inserts a new timing counter value in a new PendingKernel record appended
+  // to the end of the ring buffer then returns that same count.
+  uint64 RecordQueued();
+
+  // Takes a count value returned by RecordQueued and finds the corresponding
+  // PendingKernel record in the ring buffer.  Marks the kernel as completed and
+  // advances the completion frontier accordingly.
+  void RecordTerminated(uint64 at_count);
+
+  // Returns the largest timing count such that all kernels queued no
+  // later than that count are known to have terminated.
+  uint64 LastTerminatedCount();
+
+  // Returns the number of kernels enqueued that are not yet known to
+  // have terminated.
+  int NumPending() {
+    mutex_lock l(mu_);
+    return num_pending_;
+  }
+
+  // Yield current thread until number of pending kernels no longer
+  // exceeds the cap.
+  void PauseWhilePendingExceeds(int cap) {
+    mutex_lock l(mu_);
+    while (num_pending_ > cap) {
+      pending_decreased_.wait(l);
+    }
+  }
+
+ private:
+  Env* env_;
+  std::unique_ptr<SharedCounter> timing_counter_;
+
+  // Records when a kernel was queued for execution.  Kernel launches are
+  // identified by a unique count value from a per-GPU device timing counter.
+  struct PendingKernel {
+    uint64 queued_count;
+    bool terminated;
+    PendingKernel(const PendingKernel& pk)
+        : queued_count(pk.queued_count), terminated(pk.terminated) {}
+    PendingKernel() : queued_count(0), terminated(false) {}
+  };
+  mutex mu_;
+  // Ring buffer of PendingKernel records.
+  std::vector<PendingKernel> pending_kernels_ GUARDED_BY(mu_);
+  // Next unused slot in pending_kernels_.
+  int first_available_ GUARDED_BY(mu_) = 0;
+  // Last completed PendingKernel such that all prior PendingKernels are
+  // also completed.  With out-of-order completion there may be a mixture
+  // of completed and uncompleted entries between last_completed_ and
+  // first_available_, hence num_pending_ is not guaranteed equal to
+  // their differerence.
+  int last_completed_ GUARDED_BY(mu_) = -1;
+  int num_pending_ GUARDED_BY(mu_) = 0;
+  condition_variable pending_decreased_ GUARDED_BY(mu_);
+};
+
 class BaseGPUDeviceFactory : public DeviceFactory {
  public:
   Status CreateDevices(const SessionOptions& options, const string& name_prefix,
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device_test.cc b/tensorflow/core/common_runtime/gpu/gpu_device_test.cc
index ae623b2adb..fba937ac48 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device_test.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device_test.cc
@@ -24,6 +24,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/gtl/stl_util.h"
+#include "tensorflow/core/lib/random/random.h"
 #include "tensorflow/core/platform/test.h"
 
 namespace tensorflow {
@@ -276,6 +277,71 @@ TEST_F(GPUDeviceTest, UnifiedMemoryAllocation) {
   allocator->DeallocateRaw(ptr);
 }
 
+class GPUKernelTrackerTest : public ::testing::Test {
+ protected:
+  void SetUp() {
+    std::unique_ptr<SharedCounter> counter(new SharedCounter);
+    timing_counter_ = counter.get();
+    kernel_tracker_.reset(
+        new GPUKernelTracker(Env::Default(), std::move(counter)));
+  }
+
+  std::unique_ptr<GPUKernelTracker> kernel_tracker_;
+  SharedCounter* timing_counter_ = nullptr;
+};
+
+TEST_F(GPUKernelTrackerTest, basic) {
+  EXPECT_EQ(0, kernel_tracker_->NumPending());
+  // 1 is the expected value when no kernels have yet terminated.
+  EXPECT_EQ(1, kernel_tracker_->LastTerminatedCount());
+
+  std::deque<int64> queued_counts;
+  for (int i = 0; i < 32; ++i) {
+    queued_counts.push_back(kernel_tracker_->RecordQueued());
+  }
+  EXPECT_EQ(32, kernel_tracker_->NumPending());
+  EXPECT_EQ(1, kernel_tracker_->LastTerminatedCount());
+
+  // Mature the kernels in order until empty.
+  while (!queued_counts.empty()) {
+    int64 x = queued_counts.front();
+    queued_counts.pop_front();
+    kernel_tracker_->RecordTerminated(x);
+    EXPECT_EQ(queued_counts.size(), kernel_tracker_->NumPending());
+    EXPECT_EQ(x, kernel_tracker_->LastTerminatedCount());
+  }
+  EXPECT_EQ(timing_counter_->get(), kernel_tracker_->LastTerminatedCount());
+
+  // Next inject so many kernel events that the ring buffer needs
+  // to grow a couple of times, while maturing a few in random order
+  // to introduce gaps between last_completed_ and first_available_.
+  int64 lower_bound = timing_counter_->get();
+  for (int i = 0; i < 1111; ++i) {
+    queued_counts.push_back(kernel_tracker_->RecordQueued());
+    int64 upper_bound = timing_counter_->get();
+    if (0 == (i % 16)) {
+      size_t index = (random::New64() % queued_counts.size());
+      kernel_tracker_->RecordTerminated(queued_counts[index]);
+      queued_counts.erase(queued_counts.begin() + index);
+      EXPECT_LE(lower_bound, kernel_tracker_->LastTerminatedCount());
+      EXPECT_GE(upper_bound, kernel_tracker_->LastTerminatedCount());
+    }
+  }
+
+  // Next mature the remaining kernels in order until empty.
+  while (!queued_counts.empty()) {
+    int64 x = queued_counts.front();
+    queued_counts.pop_front();
+    kernel_tracker_->RecordTerminated(x);
+    EXPECT_EQ(queued_counts.size(), kernel_tracker_->NumPending());
+    // There may be a gap here where we find a kernel that got terminated
+    // out of order, earlier, so the LastTerminatedCount can actually
+    // jump past x.
+    EXPECT_LE(x, kernel_tracker_->LastTerminatedCount());
+  }
+  EXPECT_EQ(timing_counter_->get(), kernel_tracker_->LastTerminatedCount());
+}
+
 }  // namespace tensorflow
 
 #endif
diff --git a/tensorflow/core/common_runtime/gpu/gpu_event_mgr.cc b/tensorflow/core/common_runtime/gpu/gpu_event_mgr.cc
index 3c1c31aa73..6531d6d367 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_event_mgr.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_event_mgr.cc
@@ -241,7 +241,9 @@ void EventMgr::QueueInUse(se::Stream* stream, InUse iu) {
 // events have recorded, and then retire them.  Initial observations
 // suggest that typical behavior in a TensorFlow program is to have
 // 0-3 events pending most of the time, but there are occasionally
-// spikes of up to several hundred outstanding.
+// spikes of up to several hundred outstanding.  (If GPUKernelTracker
+// is used to cap pending kernels there should never be more than
+// that many.)
 //
 // NOTE: If all events are on the same stream, no later event will
 // complete before an earlier event, except possibly if the earlier
@@ -249,13 +251,10 @@ void EventMgr::QueueInUse(se::Stream* stream, InUse iu) {
 // looking past the first kPending event.  However, if we're using
 // multiple streams there may be some gain in looking deeper.
 // As a compromise, PollEvent() calls that are triggered by the queueing
-// of a single event never look past the first kPending event.  Calls
-// coming from the dedicated polling thread always sweep the full queue.
-//
-// Note that allowing the queue to grow very long could cause overall
-// GPU memory use to spike needlessly.  An alternative strategy would
-// be to throttle new Op execution until the pending event queue
-// clears.
+// of a single event never look past the first kPending event.  Consequently
+// those calls do an expected constant amount of work, unaffected by the
+// length of the pending queue.  Calls coming from the dedicated
+// polling thread always sweep the full queue.
 void EventMgr::PollEvents(bool is_dedicated_poller,
                           gtl::InlinedVector<InUse, 4>* to_free) {
   VLOG(2) << "PollEvents  free_events_ " << free_events_.size()
diff --git a/tensorflow/core/common_runtime/gpu/gpu_process_state.cc b/tensorflow/core/common_runtime/gpu/gpu_process_state.cc
index 909f14a17b..7804596d1a 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_process_state.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_process_state.cc
@@ -27,6 +27,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/gpu/gpu_id_utils.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_init.h"
 #include "tensorflow/core/common_runtime/pool_allocator.h"
+#include "tensorflow/core/common_runtime/shared_counter.h"
 #include "tensorflow/core/framework/allocator.h"
 #include "tensorflow/core/framework/log_memory.h"
 #include "tensorflow/core/framework/tracking_allocator.h"
@@ -110,9 +111,15 @@ Allocator* GPUProcessState::GetGPUAllocator(const GPUOptions& options,
         (options.per_process_gpu_memory_fraction() > 1.0 ||
          options.experimental().use_unified_memory()),
         gpu_visitors_[bus_id], {});
-    Allocator* gpu_allocator =
+    GPUBFCAllocator* gpu_bfc_allocator =
         new GPUBFCAllocator(sub_allocator, total_bytes, options,
                             strings::StrCat("GPU_", tf_gpu_id.value(), "_bfc"));
+    Allocator* gpu_allocator = gpu_bfc_allocator;
+    SharedCounter* timing_counter = nullptr;
+    if (options.experimental().timestamped_allocator()) {
+      timing_counter = new SharedCounter;
+      gpu_bfc_allocator->SetTimingCounter(timing_counter);
+    }
 
     // If true, checks for memory overwrites by writing
     // distinctive patterns on both ends of allocated memory.
@@ -137,7 +144,9 @@ Allocator* GPUProcessState::GetGPUAllocator(const GPUOptions& options,
       recording_allocator = new internal::RecordingAllocator(
           &process_state_->mem_desc_map_, gpu_allocator, md, &mu_);
     }
-    allocator_parts = {std::unique_ptr<Allocator>(gpu_allocator), sub_allocator,
+    allocator_parts = {std::unique_ptr<Allocator>(gpu_allocator),
+                       std::unique_ptr<SharedCounter>(timing_counter),
+                       sub_allocator,
                        std::unique_ptr<Allocator>(recording_allocator)};
   }
   if (process_state_->ProcessState::FLAGS_brain_gpu_record_mem_types) {
@@ -151,6 +160,23 @@ Allocator* GPUProcessState::GetGPUAllocator(const GPUOptions& options,
 #endif  // GOOGLE_CUDA
 }
 
+std::unique_ptr<SharedCounter> GPUProcessState::ReleaseGPUAllocatorCounter(
+    TfGpuId tf_gpu_id) {
+  DCHECK(process_state_);
+#if GOOGLE_CUDA
+  GpuIdUtil::CheckValidTfGpuId(tf_gpu_id);
+  mutex_lock l(mu_);
+  if (tf_gpu_id.value() >= static_cast<int64>(gpu_allocators_.size())) {
+    return nullptr;
+  }
+
+  AllocatorParts& allocator_parts = gpu_allocators_[tf_gpu_id.value()];
+  return std::move(allocator_parts.counter);
+#else
+  return nullptr;
+#endif
+}
+
 Allocator* GPUProcessState::GetCUDAHostAllocator(int numa_node) {
   CHECK(process_state_);
   if (!HasGPUDevice() ||
@@ -224,6 +250,7 @@ Allocator* GPUProcessState::GetCUDAHostAllocator(int numa_node) {
       allocator = new TrackingAllocator(allocator, true);
     }
     cuda_host_allocators_.push_back({std::unique_ptr<Allocator>(allocator),
+                                     std::unique_ptr<SharedCounter>(nullptr),
                                      sub_allocator,
                                      std::unique_ptr<Allocator>(nullptr)});
     AllocatorParts& allocator_parts = cuda_host_allocators_.back();
diff --git a/tensorflow/core/common_runtime/gpu/gpu_process_state.h b/tensorflow/core/common_runtime/gpu/gpu_process_state.h
index df51c10c80..c7c9f3a6b3 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_process_state.h
+++ b/tensorflow/core/common_runtime/gpu/gpu_process_state.h
@@ -23,6 +23,7 @@ limitations under the License.
 
 #include "tensorflow/core/common_runtime/gpu/gpu_id.h"
 #include "tensorflow/core/common_runtime/process_state.h"
+#include "tensorflow/core/common_runtime/shared_counter.h"
 #include "tensorflow/core/framework/allocator.h"
 #include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/platform/thread_annotations.h"
@@ -33,6 +34,7 @@ namespace tensorflow {
 
 class Allocator;
 class PoolAllocator;
+class SharedCounter;
 
 // Singleton that manages per-process state when GPUs are present.
 class GPUProcessState {
@@ -108,6 +110,8 @@ class GPUProcessState {
   // Returns bus_id for the given GPU id.
   virtual int BusIdForGPU(TfGpuId tf_gpu_id);
 
+  std::unique_ptr<SharedCounter> ReleaseGPUAllocatorCounter(TfGpuId tf_gpu_id);
+
  protected:
   // GPUProcessState is a singleton that should not normally be deleted except
   // at process shutdown.
@@ -132,6 +136,7 @@ class GPUProcessState {
 
   struct AllocatorParts {
     std::unique_ptr<Allocator> allocator;
+    std::unique_ptr<SharedCounter> counter;
     SubAllocator* sub_allocator;  // owned by allocator
     std::unique_ptr<Allocator> recording_allocator;
   };
diff --git a/tensorflow/core/common_runtime/shared_counter.h b/tensorflow/core/common_runtime/shared_counter.h
new file mode 100644
index 0000000000..5e378524b2
--- /dev/null
+++ b/tensorflow/core/common_runtime/shared_counter.h
@@ -0,0 +1,31 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_SHARED_COUNTER_H_
+#define TENSORFLOW_CORE_COMMON_RUNTIME_SHARED_COUNTER_H_
+
+namespace tensorflow {
+// A lightweight thread-safe monotone counter for establishing
+// temporal ordering.
+class SharedCounter {
+ public:
+  int64 get() { return value_; }
+  int64 next() { return ++value_; }
+
+ private:
+  std::atomic<int64> value_{0};
+};
+
+}  // namespace tensorflow
+#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_SHARED_COUNTER_H_
diff --git a/tensorflow/core/distributed_runtime/tensor_coding.cc b/tensorflow/core/distributed_runtime/tensor_coding.cc
index fe2d1a1293..6d20e7cfca 100644
--- a/tensorflow/core/distributed_runtime/tensor_coding.cc
+++ b/tensorflow/core/distributed_runtime/tensor_coding.cc
@@ -68,13 +68,14 @@ Status TensorResponse::InitFrom(RecvTensorResponse* response) {
   return s;
 }
 
-void TensorResponse::InitPartial(const RecvTensorResponse& response) {
+void TensorResponse::InitPartial(const RecvTensorResponse& response,
+                                 const AllocationAttributes& allocation_attr) {
   // Everything except content is present in *response.  Content will
   // arrive later; allocate a Tensor with appropriate storage for that
   // content.
   meta_ = response;
   TensorShape shape(meta_.tensor().tensor_shape());
-  Tensor t(allocator_, meta_.tensor().dtype(), shape);
+  Tensor t(allocator_, meta_.tensor().dtype(), shape, allocation_attr);
   tensor_ = std::move(t);
 }
 
diff --git a/tensorflow/core/distributed_runtime/tensor_coding.h b/tensorflow/core/distributed_runtime/tensor_coding.h
index 4c34297990..86d95a3063 100644
--- a/tensorflow/core/distributed_runtime/tensor_coding.h
+++ b/tensorflow/core/distributed_runtime/tensor_coding.h
@@ -76,7 +76,8 @@ class TensorResponse {
 
   // Initialize tensor metadata from response and allocate
   // uninitialized backing storage for actual contents.
-  void InitPartial(const RecvTensorResponse& response);
+  void InitPartial(const RecvTensorResponse& response,
+                   const AllocationAttributes& allocation_attr);
 
   // Return a reference to the parsed tensor.  The tensor will remain
   // live only until *this is destroyed or modified.
diff --git a/tensorflow/core/framework/allocator.h b/tensorflow/core/framework/allocator.h
index 3ded86e8e9..4d0c6d4b19 100644
--- a/tensorflow/core/framework/allocator.h
+++ b/tensorflow/core/framework/allocator.h
@@ -46,6 +46,10 @@ struct AllocationAttributes {
   // which Op is performing the allocation, and sets this flag to
   // true.
   bool allocation_will_be_logged = false;
+  // EXPERIMENTAL: If provided, then evaluates to a timing count such that only
+  // a memory chunk whose last-freed count is at this value or earlier may be
+  // returned.
+  std::function<uint64()> freed_by_func = nullptr;
 };
 
 // Runtime statistics collected by an allocator.
diff --git a/tensorflow/core/framework/device_base.h b/tensorflow/core/framework/device_base.h
index 321947aca8..89ba662b69 100644
--- a/tensorflow/core/framework/device_base.h
+++ b/tensorflow/core/framework/device_base.h
@@ -246,6 +246,15 @@ class DeviceBase {
     return errors::Internal("Device does not implement MakeTensorFromProto()");
   }
 
+  // Some devices (i.e. GPUs) may free device memory prior to its actual use
+  // being completed on the assumption that subsequent allocations can only be
+  // used serially with respect to pending uses.  If this function returns a
+  // non-zero value it is the value of a device-specific counter such that any
+  // device memory tagged with an earlier freed-at count is really unencumbered
+  // by pending uses.  For this to be useful the device memory allocator must
+  // be tagging deallocated memory chunks using the same counter.
+  virtual uint64 SafeAllocFrontier() { return 0; }
+
  protected:
   // Does not take ownership.
   void set_tensorflow_device_thread_pool(thread::ThreadPool* thread_pool) {
diff --git a/tensorflow/core/protobuf/config.proto b/tensorflow/core/protobuf/config.proto
index 2f3dbad661..44e98542ec 100644
--- a/tensorflow/core/protobuf/config.proto
+++ b/tensorflow/core/protobuf/config.proto
@@ -156,6 +156,16 @@ message GPUOptions {
     // CollectiveReduce, and serves as an override to automatic ring order
     // generation in OrderTaskDeviceMap() during CollectiveParam resolution.
     string collective_ring_order = 4;
+
+    // If true then extra work is done by GPUDevice and GPUBFCAllocator to
+    // keep track of when GPU memory is freed and when kernels actually
+    // complete so that we can know when a nominally free memory chunk
+    // is really not subject to pending use.
+    bool timestamped_allocator = 5;
+
+    // If > 0 limit the number of pending kernels on any compute
+    // stream to this number.
+    int32 pending_cap = 6;
   }
 
   // Everything inside experimental is subject to change and is not subject
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.-g-p-u-options.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.-g-p-u-options.pbtxt
index a2cc07483a..6c528dd162 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.-g-p-u-options.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.-g-p-u-options.pbtxt
@@ -84,6 +84,18 @@ tf_proto {
         label: LABEL_OPTIONAL
         type: TYPE_STRING
       }
+      field {
+        name: "timestamped_allocator"
+        number: 5
+        label: LABEL_OPTIONAL
+        type: TYPE_BOOL
+      }
+      field {
+        name: "pending_cap"
+        number: 6
+        label: LABEL_OPTIONAL
+        type: TYPE_INT32
+      }
       nested_type {
         name: "VirtualDevices"
         field {
-- 
GitLab


From 6bc79960aa808f14649bee463bdfd3fc02f1b56b Mon Sep 17 00:00:00 2001
From: Austin Anderson <angerson@google.com>
Date: Wed, 6 Feb 2019 11:17:03 -0800
Subject: [PATCH 2006/2345] Fix a typo to avoid merge delays

---
 .../dockerfiles/partials/ubuntu/devel-cpu.partial.Dockerfile    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/dockerfiles/partials/ubuntu/devel-cpu.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/devel-cpu.partial.Dockerfile
index aaeda0b207..a1fd901b34 100644
--- a/tensorflow/tools/dockerfiles/partials/ubuntu/devel-cpu.partial.Dockerfile
+++ b/tensorflow/tools/dockerfiles/partials/ubuntu/devel-cpu.partial.Dockerfile
@@ -24,6 +24,6 @@ ENV CI_BUILD_PYTHON python
 
 # CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version
 ARG CACHE_STOP=1
-# Check out TensorFlow source code if --build_arg CHECKOUT_TF_SRC=1
+# Check out TensorFlow source code if --build-arg CHECKOUT_TF_SRC=1
 ARG CHECKOUT_TF_SRC=0
 RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src || true
-- 
GitLab


From 3ce43ba44507550e94495396f38582a46127af90 Mon Sep 17 00:00:00 2001
From: Austin Anderson <angerson@google.com>
Date: Wed, 6 Feb 2019 11:18:42 -0800
Subject: [PATCH 2007/2345] Fix a typo to avoid merge delays

---
 .../dockerfiles/partials/ubuntu/devel-nvidia.partial.Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/dockerfiles/partials/ubuntu/devel-nvidia.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/devel-nvidia.partial.Dockerfile
index 8ce4a1879d..cb153fa913 100644
--- a/tensorflow/tools/dockerfiles/partials/ubuntu/devel-nvidia.partial.Dockerfile
+++ b/tensorflow/tools/dockerfiles/partials/ubuntu/devel-nvidia.partial.Dockerfile
@@ -50,6 +50,6 @@ ENV TF_CUDA_VERSION=${CUDA}
 ENV TF_CUDNN_VERSION=${CUDNN%%.*}
 # CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version
 ARG CACHE_STOP=1
-# Check out TensorFlow source code if --build_arg CHECKOUT_TF_SRC=1
+# Check out TensorFlow source code if --build-arg CHECKOUT_TF_SRC=1
 ARG CHECKOUT_TF_SRC=0
 RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src || true
-- 
GitLab


From 982c0bd4940ab1a2f9742b0aa05d9fd06fce6d46 Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Wed, 6 Feb 2019 11:37:04 -0800
Subject: [PATCH 2008/2345] Create int8 split.

PiperOrigin-RevId: 232713680
---
 tensorflow/lite/kernels/register.cc     |   3 +-
 tensorflow/lite/kernels/split.cc        |  18 ++--
 tensorflow/lite/kernels/split_test.cc   | 120 +++++++++++++++---------
 tensorflow/lite/toco/tflite/operator.cc |   6 ++
 4 files changed, 96 insertions(+), 51 deletions(-)

diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index aad6deb4d8..bce406c1bc 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -236,7 +236,8 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_MEAN, Register_MEAN());
   AddBuiltin(BuiltinOperator_DIV, Register_DIV());
   AddBuiltin(BuiltinOperator_SUB, Register_SUB());
-  AddBuiltin(BuiltinOperator_SPLIT, Register_SPLIT());
+  AddBuiltin(BuiltinOperator_SPLIT, Register_SPLIT(), /* min_version */ 1,
+             /* max_version */ 2);
   AddBuiltin(BuiltinOperator_SPLIT_V, Register_SPLIT_V());
   AddBuiltin(BuiltinOperator_SQUEEZE, Register_SQUEEZE());
   AddBuiltin(BuiltinOperator_STRIDED_SLICE, Register_STRIDED_SLICE());
diff --git a/tensorflow/lite/kernels/split.cc b/tensorflow/lite/kernels/split.cc
index 7902ed2a46..4112898290 100644
--- a/tensorflow/lite/kernels/split.cc
+++ b/tensorflow/lite/kernels/split.cc
@@ -76,9 +76,9 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   TF_LITE_ENSURE_EQ(context, NumOutputs(node), op_context.params->num_splits);
 
   auto input_type = op_context.input->type;
-  TF_LITE_ENSURE(context, input_type == kTfLiteFloat32 ||
-                              input_type == kTfLiteUInt8 ||
-                              input_type == kTfLiteInt16);
+  TF_LITE_ENSURE(context,
+                 input_type == kTfLiteFloat32 || input_type == kTfLiteUInt8 ||
+                     input_type == kTfLiteInt8 || input_type == kTfLiteInt16);
   for (int i = 0; i < NumOutputs(node); ++i) {
     GetOutput(context, node, i)->type = input_type;
   }
@@ -137,15 +137,19 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
       TF_LITE_SPLIT(uint8_t);
       break;
     }
+    case kTfLiteInt8: {
+      TF_LITE_SPLIT(int8_t);
+      break;
+    }
     case kTfLiteInt16: {
       TF_LITE_SPLIT(int16_t);
       break;
     }
     default:
-      context->ReportError(
-          context,
-          "Only float32, uint8 and int16 are currently supported, got %d.",
-          op_context.input->type);
+      context->ReportError(context,
+                           "Only float32, uint8, int8 and int16 are currently "
+                           "supported, got %d.",
+                           op_context.input->type);
       return kTfLiteError;
   }
 #undef TF_LITE_SPLIT
diff --git a/tensorflow/lite/kernels/split_test.cc b/tensorflow/lite/kernels/split_test.cc
index f3d9ea3bf4..aa2300722c 100644
--- a/tensorflow/lite/kernels/split_test.cc
+++ b/tensorflow/lite/kernels/split_test.cc
@@ -47,13 +47,15 @@ class SplitOpModel : public SingleOpModel {
     }
   }
 
-  void SetInput(std::initializer_list<float> data) {
+  template <typename T>
+  void SetInput(std::initializer_list<T> data) {
     PopulateTensor(input_, data);
   }
   void SetAxis(int axis) { PopulateTensor(axis_, {axis}); }
 
-  std::vector<float> GetOutput(int i) {
-    return ExtractVector<float>(outputs_[i]);
+  template <typename T>
+  std::vector<T> GetOutput(int i) {
+    return ExtractVector<T>(outputs_[i]);
   }
   std::vector<int> GetOutputShape(int i) { return GetTensorShape(outputs_[i]); }
 
@@ -63,33 +65,34 @@ class SplitOpModel : public SingleOpModel {
   std::vector<int> outputs_;
 };
 
-using TensorValues = std::initializer_list<float>;
-
+template <typename T>
 void Check(int axis, int num_splits, std::initializer_list<int> input_shape,
            std::initializer_list<int> output_shape,
-           const TensorValues& input_data,
-           const std::vector<TensorValues>& output_data) {
+           const std::initializer_list<T>& input_data,
+           const std::vector<std::initializer_list<T>>& output_data,
+           const TensorType& type = TensorType_FLOAT32) {
   auto debug = [&](int i) {
     std::stringstream ss;
     ss << "for output tensor " << i << " axis=" << axis
        << " and num_splits=" << num_splits;
     return ss.str();
   };
-  SplitOpModel m({TensorType_FLOAT32, input_shape}, num_splits);
+  SplitOpModel m({type, input_shape}, num_splits);
   m.SetInput(input_data);
   m.SetAxis(axis);
   m.Invoke();
   for (int i = 0; i < num_splits; ++i) {
-    EXPECT_THAT(m.GetOutput(i), ElementsAreArray(output_data[i])) << debug(i);
+    EXPECT_THAT(m.GetOutput<T>(i), ElementsAreArray(output_data[i]))
+        << debug(i);
     EXPECT_THAT(m.GetOutputShape(i), ElementsAreArray(output_shape))
         << debug(i);
   }
 
-  SplitOpModel const_m({TensorType_FLOAT32, input_shape}, num_splits, axis);
+  SplitOpModel const_m({type, input_shape}, num_splits, axis);
   const_m.SetInput(input_data);
   const_m.Invoke();
   for (int i = 0; i < num_splits; ++i) {
-    EXPECT_THAT(const_m.GetOutput(i), ElementsAreArray(output_data[i]))
+    EXPECT_THAT(const_m.GetOutput<T>(i), ElementsAreArray(output_data[i]))
         << debug(i);
     EXPECT_THAT(const_m.GetOutputShape(i), ElementsAreArray(output_shape))
         << debug(i);
@@ -97,44 +100,75 @@ void Check(int axis, int num_splits, std::initializer_list<int> input_shape,
 }
 
 TEST(SplitOpTest, FourDimensional) {
-  Check(/*axis=*/0, /*num_splits=*/2, {2, 2, 2, 2}, {1, 2, 2, 2},
-        {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16},
-        {
-            {1, 2, 3, 4, 5, 6, 7, 8},
-            {9, 10, 11, 12, 13, 14, 15, 16},
-        });
-  Check(/*axis=*/1, /*num_splits=*/2, {2, 2, 2, 2}, {2, 1, 2, 2},
-        {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16},
-        {
-            {1, 2, 3, 4, 9, 10, 11, 12},
-            {5, 6, 7, 8, 13, 14, 15, 16},
-        });
-  Check(/*axis=*/2, /*num_splits=*/2, {2, 2, 2, 2}, {2, 2, 1, 2},
-        {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16},
-        {
-            {1, 2, 5, 6, 9, 10, 13, 14},
-            {3, 4, 7, 8, 11, 12, 15, 16},
-        });
-  Check(/*axis=*/3, /*num_splits=*/2, {2, 2, 2, 2}, {2, 2, 2, 1},
-        {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16},
-        {
-            {1, 3, 5, 7, 9, 11, 13, 15},
-            {2, 4, 6, 8, 10, 12, 14, 16},
-        });
+  Check<float>(/*axis=*/0, /*num_splits=*/2, {2, 2, 2, 2}, {1, 2, 2, 2},
+               {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16},
+               {
+                   {1, 2, 3, 4, 5, 6, 7, 8},
+                   {9, 10, 11, 12, 13, 14, 15, 16},
+               });
+  Check<float>(/*axis=*/1, /*num_splits=*/2, {2, 2, 2, 2}, {2, 1, 2, 2},
+               {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16},
+               {
+                   {1, 2, 3, 4, 9, 10, 11, 12},
+                   {5, 6, 7, 8, 13, 14, 15, 16},
+               });
+  Check<float>(/*axis=*/2, /*num_splits=*/2, {2, 2, 2, 2}, {2, 2, 1, 2},
+               {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16},
+               {
+                   {1, 2, 5, 6, 9, 10, 13, 14},
+                   {3, 4, 7, 8, 11, 12, 15, 16},
+               });
+  Check<float>(/*axis=*/3, /*num_splits=*/2, {2, 2, 2, 2}, {2, 2, 2, 1},
+               {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16},
+               {
+                   {1, 3, 5, 7, 9, 11, 13, 15},
+                   {2, 4, 6, 8, 10, 12, 14, 16},
+               });
+}
+
+TEST(SplitOpTest, FourDimensionalInt8) {
+  Check<int8_t>(/*axis=*/0, /*num_splits=*/2, {2, 2, 2, 2}, {1, 2, 2, 2},
+                {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16},
+                {
+                    {1, 2, 3, 4, 5, 6, 7, 8},
+                    {9, 10, 11, 12, 13, 14, 15, 16},
+                },
+                TensorType_INT8);
+  Check<int8_t>(/*axis=*/1, /*num_splits=*/2, {2, 2, 2, 2}, {2, 1, 2, 2},
+                {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16},
+                {
+                    {1, 2, 3, 4, 9, 10, 11, 12},
+                    {5, 6, 7, 8, 13, 14, 15, 16},
+                },
+                TensorType_INT8);
+  Check<int8_t>(/*axis=*/2, /*num_splits=*/2, {2, 2, 2, 2}, {2, 2, 1, 2},
+                {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16},
+                {
+                    {1, 2, 5, 6, 9, 10, 13, 14},
+                    {3, 4, 7, 8, 11, 12, 15, 16},
+                },
+                TensorType_INT8);
+  Check<int8_t>(/*axis=*/3, /*num_splits=*/2, {2, 2, 2, 2}, {2, 2, 2, 1},
+                {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16},
+                {
+                    {1, 3, 5, 7, 9, 11, 13, 15},
+                    {2, 4, 6, 8, 10, 12, 14, 16},
+                },
+                TensorType_INT8);
 }
 
 TEST(SplitOpTest, OneDimensional) {
-  Check(/*axis=*/0, /*num_splits=*/8, {8}, {1}, {1, 2, 3, 4, 5, 6, 7, 8},
-        {{1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}});
+  Check<float>(/*axis=*/0, /*num_splits=*/8, {8}, {1}, {1, 2, 3, 4, 5, 6, 7, 8},
+               {{1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}});
 }
 
 TEST(SplitOpTest, NegativeAxis) {
-  Check(/*axis=*/-4, /*num_splits=*/2, {2, 2, 2, 2}, {1, 2, 2, 2},
-        {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16},
-        {
-            {1, 2, 3, 4, 5, 6, 7, 8},
-            {9, 10, 11, 12, 13, 14, 15, 16},
-        });
+  Check<float>(/*axis=*/-4, /*num_splits=*/2, {2, 2, 2, 2}, {1, 2, 2, 2},
+               {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16},
+               {
+                   {1, 2, 3, 4, 5, 6, 7, 8},
+                   {9, 10, 11, 12, 13, 14, 15, 16},
+               });
 }
 
 }  // namespace
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index 4ce3aa9218..d45ecf3139 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -1259,6 +1259,12 @@ class Split
   }
 
   int GetVersion(const OperatorSignature& op_signature) const override {
+    const string& input_name = op_signature.op->inputs[0];
+    const Array& input_array = op_signature.model->GetArray(input_name);
+    // If the op take int8 input, it is version 2.
+    if (input_array.data_type == ArrayDataType::kInt8) {
+      return 2;
+    }
     return 1;
   }
 };
-- 
GitLab


From 1588faf3de27036a855a73266a295be7275ecefc Mon Sep 17 00:00:00 2001
From: nammbash <niroopshankar.ammbashankar@intel.com>
Date: Wed, 6 Feb 2019 11:51:41 -0800
Subject: [PATCH 2009/2345] indentation in pbtxt needs to be corrected

---
 .../base_api/api_def_RequantizationRangePerChannel.pbtxt    | 4 ++--
 .../api_def/base_api/api_def_RequantizePerChannel.pbtxt     | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/core/api_def/base_api/api_def_RequantizationRangePerChannel.pbtxt b/tensorflow/core/api_def/base_api/api_def_RequantizationRangePerChannel.pbtxt
index 2dfde04721..58cf122250 100644
--- a/tensorflow/core/api_def/base_api/api_def_RequantizationRangePerChannel.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_RequantizationRangePerChannel.pbtxt
@@ -1,6 +1,6 @@
 op {
-graph_op_name: "RequantizationRangePerChannel"
-visibility : HIDDEN
+  graph_op_name: "RequantizationRangePerChannel"
+  visibility : HIDDEN
   in_arg {
     name: "input"
     description: <<END
diff --git a/tensorflow/core/api_def/base_api/api_def_RequantizePerChannel.pbtxt b/tensorflow/core/api_def/base_api/api_def_RequantizePerChannel.pbtxt
index d62736c2bc..bce6c3dd5a 100644
--- a/tensorflow/core/api_def/base_api/api_def_RequantizePerChannel.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_RequantizePerChannel.pbtxt
@@ -1,7 +1,7 @@
 op {
-graph_op_name: "RequantizePerChannel"
-visibility : HIDDEN
-in_arg {
+  graph_op_name: "RequantizePerChannel"
+  visibility : HIDDEN
+  in_arg {
     name: "input"
     description: <<END
 The original input tensor.
-- 
GitLab


From 8b633af6ffcf3ec05b05f8fba1a5fa20f70f551d Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Wed, 6 Feb 2019 11:44:44 -0800
Subject: [PATCH 2010/2345] Track requested and assigned devices separately in
 Placer.

Before this change Placer treated assigned and requested devices
almost identically. The Placer::Member::device_name was initialized
either with assigned or requested device. During the placement process,
`device_name` could be overriden to satisfy the resource constraint (
all resource touching ops are run on the resource's device). When all
colocation groups are computed and devices to assign are chosen, Placer
simply skipped the nodes with assigned devices.

This behavior can result in various violations of colocation constraints.
For example, the following would be placed successfully and but raise an
error at runtime:
  VarHandleOp (requested on CPU)
      |
      V
    Read (assigned to GPU)

Another issue with the Placer before this change is the logic in
VerifyResourceAndRefInputsCanBeColocated method. Given a resource edge
from src to dst nodes, that connected incompatible colocation groups,
Placer would normally override the destination's device_name. Before
overriding, Placer would check if device_name of all the other inputs
is compatible with the new value.

The semantics of this check is fairly arbitrary. For example, the following
would be placed successfully by overriding the requested device of Add:

CPU resource    CPU resource
          \     /
	   v   v
	Add (requested on GPU)

On the other hand, the following graph would be rejected even though it
can be placed by overriding the requsted device of Identity - logically
the same operation as above:

               VarHandleOp (unplaced)
                 |
                 v
CPU resource   Identity (requested on GPU)
          \     /
	   v   v
	    Add

This change treats assigned and requested devices separately. Overriding
requested devices to satisfy resource constraints is always permitted.
Overriding assigned devices is permitted only when soft placement is allowed.
Colocation group constraints are always respected (or an error is raised),
even with assigned devices and soft placement.

Finally, the emergent property of VerifyResourceAndRefInputsCanBeColocated and
surrounding logic was that requested devices of resource producing ops were
always respected (and error was raised if they resulted in a conflict). This
change preserves this behavior but makes it explicit. Requested devices on
resource generating nodes are treated as assigned device.

PiperOrigin-RevId: 232715063
---
 tensorflow/core/common_runtime/placer.cc      | 392 +++++++++++-------
 tensorflow/core/common_runtime/placer_test.cc | 183 +++++++-
 tensorflow/core/util/device_name_utils.cc     |  24 ++
 tensorflow/core/util/device_name_utils.h      |   5 +
 .../python/data/kernel_tests/map_test.py      |   3 +-
 tensorflow/python/eager/function_test.py      |   5 +-
 6 files changed, 459 insertions(+), 153 deletions(-)

diff --git a/tensorflow/core/common_runtime/placer.cc b/tensorflow/core/common_runtime/placer.cc
index 72e5efcebf..c9827d190d 100644
--- a/tensorflow/core/common_runtime/placer.cc
+++ b/tensorflow/core/common_runtime/placer.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include <utility>
 #include <vector>
 
+#include "absl/strings/str_join.h"
 #include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/framework/attr_value_util.h"
 #include "tensorflow/core/framework/device_attributes.pb.h"
@@ -33,6 +34,7 @@ limitations under the License.
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/util/device_name_utils.h"
+#include "tensorflow/core/util/dump_graph.h"
 #include "tensorflow/core/util/port.h"
 
 namespace tensorflow {
@@ -96,6 +98,27 @@ std::vector<Device*> FilterSupportedDevices(
   return filtered_devices;
 }
 
+// Using absl::StrJoin with lambda does not work in tf-lite builds.
+std::vector<string> DevicesToString(const std::vector<Device*> devices) {
+  std::vector<string> v;
+  v.reserve(devices.size());
+  for (Device* d : devices) {
+    v.push_back(d->name());
+  }
+  return v;
+}
+
+// Using absl::StrJoin with lambda does not work in tf-lite builds.
+std::vector<string> DeviceTypeAndPriorityToString(
+    const PrioritizedDeviceTypeVector& devices) {
+  std::vector<string> v;
+  v.reserve(devices.size());
+  for (const std::pair<DeviceType, int32>& device_and_type : devices) {
+    v.push_back(DeviceTypeString(device_and_type.first));
+  }
+  return v;
+}
+
 // Returns true if the node has no inputs and produces outputs
 // that are consumed by a single node.
 //
@@ -107,6 +130,16 @@ bool IsGeneratorNode(const Node* node) {
          !IsRefType(node->output_type(0));
 }
 
+// While Placer can override requested device on ops processing
+// resources, i.e. node that take (and potentially return) a resource,
+// it must not override requested device on ops generating a resource,
+// e.g. VarHandleOp, _Arg. Such ops are currently no-input, single resource/ref
+// output nodes.
+bool IsResourceGeneratorNode(const Node& node) {
+  return node.num_inputs() == 0 && node.num_outputs() == 1 &&
+         (IsRefType(node.output_type(0)) || node.output_type(0) == DT_RESOURCE);
+}
+
 bool IsExemptFromResourceInputColocation(const Node* node) {
   // Note: Partitioned function calls, which place and partition their
   // function bodies, are exempt from this check: they forward resource and
@@ -136,7 +169,7 @@ bool ArePrioritiesSame(const PrioritizedDeviceTypeVector& a_types,
   return true;
 }
 
-// Represents a node in the disjoint node set forest, and the
+// Represents a node in the disjoint node forest and the
 // accumulated constraints on the device used by that node.
 class Member {
  public:
@@ -155,18 +188,76 @@ class Member {
                                        &supported_device_types_);
   }
 
-  const DeviceNameUtils::ParsedName& device_name() const {
-    return device_name_;
+  const DeviceNameUtils::ParsedName& requested_device_name() const {
+    return requested_device_name_;
   }
 
-  Status SetDeviceName(const string& device_name) {
-    if (!DeviceNameUtils::ParseFullName(device_name, &device_name_)) {
+  Status SetAssignedDeviceName(const string& device_name) {
+    if (DeviceNameUtils::HasSomeDetails(requested_device_name_)) {
+      return errors::Internal(
+          "Setting assigned device name when there is a requested device set "
+          "is unsupported");
+    }
+    if (!DeviceNameUtils::ParseFullName(device_name, &assigned_device_name_)) {
       return errors::Internal("Malformed assigned device '", device_name, "'");
     }
+    // Set requested device to assigned_device to maintain the invariant that
+    // requested is a specialization of assigned.
+    requested_device_name_ = assigned_device_name_;
     return Status::OK();
   }
-  void SetDeviceName(const DeviceNameUtils::ParsedName& device_name) {
-    device_name_ = device_name;
+
+  Status SetRequestedDeviceName(const Node& node) {
+    if (!DeviceNameUtils::ParseFullName(node.requested_device(),
+                                        &requested_device_name_)) {
+      return errors::InvalidArgument("Malformed device specification '",
+                                     node.requested_device(),
+                                     "' in node: ", node.DebugString());
+    }
+    if (DeviceNameUtils::HasSomeDetails(assigned_device_name_)) {
+      return errors::Internal(
+          "Setting requested device name when there is an assigned device set "
+          "is unsupported");
+    }
+    return Status::OK();
+  }
+
+  Status EnsureCompatibilityAcrossResourceEdge(
+      const Node& src, const Member& src_root,
+      const Node& dst, /*dst_root is this*/
+      bool log_device_placement) {
+    if (!DeviceNameUtils::AreCompatibleDevNames(src_root.assigned_device_name_,
+                                                assigned_device_name_)) {
+      return errors::InvalidArgument(
+          "Cannot place the graph because a reference or resource edge "
+          "connects colocation groups with incompatible assigned devices: ",
+          DeviceNameUtils::ParsedNameToString(src_root.assigned_device_name_),
+          " vs ", DeviceNameUtils::ParsedNameToString(assigned_device_name_));
+    }
+
+    if (DeviceNameUtils::AreCompatibleDevNames(src_root.requested_device_name_,
+                                               requested_device_name_)) {
+      return Status::OK();
+    }
+
+    // If we are here, assigned devices are compatible but requested ones are
+    // not. We will be overriding the requested device for destination node, but
+    // need to preserve the invariant that it will be a specialization of
+    // the assigned device.
+    if (log_device_placement) {
+      LOG(INFO) << "Ignoring device specification "
+                << DeviceNameUtils::ParsedNameToString(requested_device_name_)
+                << " for node '" << dst.name()
+                << "' because the input edge from '" << src.name()
+                << "' is a reference connection and already has a device "
+                   "field set to "
+                << DeviceNameUtils::ParsedNameToString(
+                       src_root.requested_device_name_);
+    }
+    requested_device_name_ = src_root.requested_device_name_;
+    DeviceNameUtils::EnsureSpecification(&requested_device_name_,
+                                         assigned_device_name_);
+    return Status::OK();
   }
 
   const PrioritizedDeviceTypeVector& supported_device_types() const {
@@ -228,13 +319,13 @@ class Member {
   }
 
   Status MergeDeviceNames(const Member& other, bool allow_soft_placement) {
-    return DeviceNameUtils::MergeDevNames(&device_name_, other.device_name_,
-                                          allow_soft_placement);
-  }
-  Status MergeDeviceNames(const string& dev_name, bool allow_soft_placement) {
-    DeviceNameUtils::ParsedName parsed;
-    DeviceNameUtils::ParseFullName(dev_name, &parsed);
-    return DeviceNameUtils::MergeDevNames(&device_name_, parsed,
+    // Assuming the "requested is a specialization of assigned" invariant holds
+    // for this and `other`, it will hold after these two merges.
+    TF_RETURN_IF_ERROR(DeviceNameUtils::MergeDevNames(
+        &requested_device_name_, other.requested_device_name_,
+        allow_soft_placement));
+    return DeviceNameUtils::MergeDevNames(&assigned_device_name_,
+                                          other.assigned_device_name_,
                                           allow_soft_placement);
   }
 
@@ -324,16 +415,29 @@ class Member {
     if (node.assigned_device_name_index() == assigned_device_name_index_) {
       return Status::OK();
     }
-    Status s =
-        MergeDeviceNames(node.assigned_device_name(), allow_soft_placement);
+
+    DeviceNameUtils::ParsedName parsed;
+    DeviceNameUtils::ParseFullName(node.assigned_device_name(), &parsed);
+    Status s = DeviceNameUtils::MergeDevNames(&assigned_device_name_, parsed,
+                                              allow_soft_placement);
     if (!s.ok()) {
       return errors::Internal(
           "Constraining by assigned device should not cause an error. Original "
-          "root device name: ",
-          DeviceNameUtils::ParsedNameToString(device_name_),
-          " assigned device name \"", node.assigned_device_name(),
+          "root's assigned device name: ",
+          DeviceNameUtils::ParsedNameToString(assigned_device_name_),
+          " node's assigned device name \"", node.assigned_device_name(),
           ". Error: ", s.error_message());
     }
+    s = DeviceNameUtils::MergeDevNames(&requested_device_name_, parsed,
+                                       allow_soft_placement);
+    if (!s.ok()) {
+      return errors::Internal(
+          "Constraining by assigned device should not cause an error. Original "
+          "root's requested device name: \"",
+          DeviceNameUtils::ParsedNameToString(requested_device_name_),
+          "\", node's assigned device name \"", node.assigned_device_name(),
+          "\". Error: ", s.error_message());
+    }
 
     assigned_device_name_index_ = node.assigned_device_name_index();
     // Clear cached possible_devices, if any.
@@ -346,6 +450,20 @@ class Member {
   }
   const std::vector<Device*>& possible_devices() { return possible_devices_; }
 
+  string DebugString() {
+    return absl::StrCat(
+        "Member(assigned_device_name_index_=", assigned_device_name_index_,
+        " requested_device_name_=",
+        DeviceNameUtils::ParsedNameToString(requested_device_name_),
+        " assigned_device_name_=",
+        DeviceNameUtils::ParsedNameToString(assigned_device_name_),
+        " supported_device_types_=[",
+        absl::StrJoin(DeviceTypeAndPriorityToString(supported_device_types_),
+                      ", "),
+        "] possible_devices_=[",
+        absl::StrJoin(DevicesToString(possible_devices_), ", "), "]");
+  }
+
  private:
   // The id of the node that is the parent of this one, or its own
   // id if it is a root. parent <= 0 indicates that this member is invalid.
@@ -356,19 +474,37 @@ class Member {
   // sets.
   int rank_ = 0;
 
-  // Once colocation groups have been formed and we assigned at least
-  // one node in this group to a device, assigned_device_name_index will
-  // contain this device name's index in the graph. The `device_name` will
-  // contain the parsed name of this device and `possible_devices`, if
-  // computed, will contain just this device.
+  // Once colocation groups have been formed, the Placer starts actually
+  // choosing devices. All nodes in a group must be assigned to the same
+  // device. Once we assigned the first device to some node in this group,
+  // we set assigned_device_name_index to this device name's index in the
+  // graph.
+  // The `*_device_name_` fields will contain the parsed name of this device
+  // and `possible_devices`, if computed, will contain just this device.
   // `assigned_device_name_index` is an optimization to avoid parsing and
   // comparing device names. The value of -1 signals that a single device
   // has not been chosen yet.
   int assigned_device_name_index_ = -1;
 
-  // The merged form of the device requested for this node, with
+  // The merged form of the device requested for this node, with those of all of
+  // its children. requested_device_name_ is always kept a specialization (i.e.
+  // DeviceNameUtils::IsSpecialization) of assigned_device_name_. When no device
+  // is requested, this field is set to assigned_device_name_.  As a
+  // specialization of assigned_device_name_, requested_device_name_ represents
+  // the most specific form of all assigned and requested devices of this node
+  // and its children, if this node is a root. requested_device_name_ is used
+  // to finally select devices for nodes.  We can override requested devices due
+  // to resource colocation constraints but not assigned devices (unless soft
+  // placement is on).
+  DeviceNameUtils::ParsedName requested_device_name_;
+
+  // The merged form of the device assigned for this node, with
   // those of all of its children.
-  DeviceNameUtils::ParsedName device_name_;
+  // This field is used to raise errors due to unsatisfiable constraints.
+  // Can be a partial specification.
+  // INVARIANT: requested_device_name_ is always a
+  // DeviceNameUtils::IsSpecialization of assigned_device_name_.
+  DeviceNameUtils::ParsedName assigned_device_name_;
 
   // The intersection of all device types supported by this node,
   // and those of all of its children, in priority order
@@ -379,7 +515,7 @@ class Member {
   // and all of its children have been assigned, or nullptr if this
   // has not yet been computed.
   std::vector<Device*> possible_devices_;
-};
+};  // namespace
 
 // This class maintains the connected components of a colocation
 // constraint graph, and uses this information to assign a satisfying
@@ -493,34 +629,9 @@ class ColocationGraph {
     int dst_root_id = FindRoot(dst->id());
     auto& src_root = members_[src_root_id];
     auto& dst_root = members_[dst_root_id];
-    // If both the source node and this node have partially
-    // specified a device, then 'dst's device should be
-    // cleared: the reference edge forces 'node' to be on the
-    // same device as the source node.
-    const auto& source_parsed_name = src_root.device_name();
-    const auto& dest_parsed_name = dst_root.device_name();
-    if (DeviceNameUtils::HasSomeDetails(source_parsed_name) &&
-        DeviceNameUtils::HasSomeDetails(dest_parsed_name)) {
-      // Ignore a specified device for 'dst' if the two names were
-      // incompatible.
-      if (!DeviceNameUtils::AreCompatibleDevNames(source_parsed_name,
-                                                  dest_parsed_name)) {
-        TF_RETURN_IF_ERROR(VerifyResourceAndRefInputsCanBeColocated(
-            dst, src, source_parsed_name));
-        if (log_device_placement_) {
-          LOG(INFO) << "Ignoring device specification "
-                    << DeviceNameUtils::ParsedNameToString(dest_parsed_name)
-                    << " for node '" << dst->name()
-                    << "' because the input edge from '" << src->name()
-                    << "' is a reference connection and already has a device "
-                       "field set to "
-                    << DeviceNameUtils::ParsedNameToString(source_parsed_name);
-        }
 
-        // Make 'dst' colocated with the source
-        dst_root.SetDeviceName(source_parsed_name);
-      }
-    }
+    TF_RETURN_IF_ERROR(dst_root.EnsureCompatibilityAcrossResourceEdge(
+        *src, src_root, *dst, log_device_placement_));
     Status status = ColocateNodes(*src, src_root_id, *dst, dst_root_id);
     if (!status.ok()) {
       return AttachDef(
@@ -681,12 +792,13 @@ class ColocationGraph {
     // "devices" will contain the set of feasible placements for the
     // colocated node set containing 'node'.
     std::vector<Device*> devices;
-    if (DeviceNameUtils::HasSomeDetails(members_[node_root].device_name())) {
+    if (DeviceNameUtils::HasSomeDetails(
+            members_[node_root].requested_device_name())) {
       // The root node has a (possibly partial) device
       // specification, so enumerate the physical devices that
       // conform to it.
-      device_set_->FindMatchingDevices(members_[node_root].device_name(),
-                                       &devices);
+      device_set_->FindMatchingDevices(
+          members_[node_root].requested_device_name(), &devices);
 
       if (!devices.empty()) {
         // Filter devices into those that are compatible with the root
@@ -701,7 +813,7 @@ class ColocationGraph {
         // The soft_device_name is the same as the node's device name
         // without specifying the device type or ID.
         DeviceNameUtils::ParsedName soft_device_name =
-            members_[node_root].device_name();
+            members_[node_root].requested_device_name();
         soft_device_name.type.clear();
         soft_device_name.has_type = false;
         soft_device_name.has_id = false;
@@ -722,7 +834,8 @@ class ColocationGraph {
         DeviceNameUtils::ParsedName specified_device_name;
         if (DeviceNameUtils::ParseFullName(node->requested_device(),
                                            &specified_device_name) &&
-            specified_device_name == members_[node_root].device_name()) {
+            specified_device_name ==
+                members_[node_root].requested_device_name()) {
           // The specified device and merged set device match, and
           // will appear in the GraphDef (for debugging), so just
           // print the specified device.
@@ -774,7 +887,7 @@ class ColocationGraph {
               " was colocated with a group of nodes that ",
               "required incompatible device '",
               DeviceNameUtils::ParsedNameToString(
-                  members_[node_root].device_name()),
+                  members_[node_root].requested_device_name()),
               "'", debug_info);
         }
       }
@@ -803,10 +916,7 @@ class ColocationGraph {
   }
 
   Status InitializeMembers() {
-    for (Node* node : graph_->nodes()) {
-      if (!node->IsOp()) {
-        continue;
-      }
+    for (Node* node : graph_->op_nodes()) {
       Status status = InitializeMember(*node, &members_[node->id()]);
       if (!status.ok()) {
         return AttachDef(status, *node);
@@ -815,6 +925,22 @@ class ColocationGraph {
     return Status::OK();
   }
 
+  string DebugString() {
+    std::unordered_set<int> roots;
+    std::vector<string> root_strings;
+    for (const Node* node : graph_->nodes()) {
+      if (!node->IsOp()) {
+        continue;
+      }
+      int node_root = FindRoot(node->id());
+      if (roots.count(node_root) == 0) {
+        root_strings.push_back(DebugInfo(node_root));
+        roots.insert(node_root);
+      }
+    }
+    return absl::StrJoin(root_strings, "\n");
+  }
+
   // Returns debugging info for the node referred to by 'node_root'.
   string DebugInfo(const int node_root) {
     string text(
@@ -860,46 +986,52 @@ class ColocationGraph {
     }
     strings::StrAppend(&text, "\n");
 
-    if (num_nodes_found <= 1) {
+    if (num_nodes_found <= 0) {
       text.clear();
     }
     return text;
   }
 
+  Status InitializeMemberWithAssignedDevice(const string& assigned_device_name,
+                                            const string& node_type,
+                                            bool must_be_full_name,
+                                            Member* member) {
+    // This node has already been assigned to a device, so we
+    // respect this placement, after sanity-checking it.
+    // NOTE: Since any assignment must have been performed by
+    // the TensorFlow runtime, we consider errors in this branch to
+    // be INTERNAL.
+    TF_RETURN_IF_ERROR(member->SetAssignedDeviceName(assigned_device_name));
+    if (!must_be_full_name) {
+      return Status::OK();
+    }
+    // Since assigned device must be a full specification, do extra checks.
+    const Device* assigned_device =
+        device_set_->FindDeviceByName(assigned_device_name);
+    if (assigned_device == nullptr) {
+      return errors::Internal("Assigned device '", assigned_device_name,
+                              "' does not match any device");
+    }
+
+    for (const auto& d : member->supported_device_types()) {
+      if (DeviceType(assigned_device->attributes().device_type()) == d.first) {
+        return Status::OK();
+      }
+    }
+
+    return errors::Internal("Assigned device '", assigned_device_name,
+                            "' does not have registered OpKernel support "
+                            "for ",
+                            node_type);
+  }
+
   Status InitializeMember(const Node& node, Member* member) {
     TF_RETURN_IF_ERROR(
         member->SetParentAndSupportedDevices(node, device_types_));
 
     if (node.has_assigned_device_name()) {
-      // This node has already been assigned to a device, so we
-      // respect this placement, after sanity-checking it.  The
-      // device_name and supported_device_types for this node reflect
-      // the assigned device, so any nodes colocated with this node
-      // will be assigned to the same device (assuming this is
-      // possible).
-      // NOTE: Since any assignment must have been performed by
-      // the TensorFlow runtime, we consider errors in this branch to
-      // be INTERNAL.
-      const string& assigned_device_name = node.assigned_device_name();
-      TF_RETURN_IF_ERROR(member->SetDeviceName(assigned_device_name));
-      const Device* assigned_device =
-          device_set_->FindDeviceByName(assigned_device_name);
-      if (assigned_device == nullptr) {
-        return errors::Internal("Assigned device '", assigned_device_name,
-                                "' does not match any device");
-      }
-
-      for (const auto& d : member->supported_device_types()) {
-        if (DeviceType(assigned_device->attributes().device_type()) ==
-            d.first) {
-          return Status::OK();
-        }
-      }
-
-      return errors::Internal("Assigned device '", assigned_device_name,
-                              "' does not have registered OpKernel support "
-                              "for ",
-                              node.type_string());
+      TF_RETURN_IF_ERROR(InitializeMemberWithAssignedDevice(
+          node.assigned_device_name(), node.type_string(), true, member));
     } else {
       // This node has not yet been assigned to a device, so we
       // calculate any constraints due to the set of registered
@@ -933,15 +1065,18 @@ class ColocationGraph {
       // If the NodeDef contains a device, then we interpret it as a
       // (partial) device specification.
       if (!node.requested_device().empty()) {
-        // The user has specified a device in the NodeDef, try to find a
-        // valid device matching their specification in the set of
-        // devices.
-        // NOTE: The full name may specify a device that is not in
-        // n.supported_device_types(), but we check that in AssignDevice().
-        if (!member->SetDeviceName(node.requested_device()).ok()) {
-          return errors::InvalidArgument("Malformed device specification '",
-                                         node.requested_device(),
-                                         "' in node: ", node.DebugString());
+        if (IsResourceGeneratorNode(node)) {
+          // Treat requested device on resource generating nodes as assigned
+          // device so that we don't override it.
+          TF_RETURN_IF_ERROR(InitializeMemberWithAssignedDevice(
+              node.requested_device(), node.type_string(), false, member));
+        } else {
+          // The user has specified a device in the NodeDef, try to find a
+          // valid device matching their specification in the set of
+          // devices.
+          // NOTE: The full name may specify a device that is not in
+          // n.supported_device_types(), but we check that in AssignDevice().
+          TF_RETURN_IF_ERROR(member->SetRequestedDeviceName(node));
         }
       }
     }
@@ -1032,40 +1167,6 @@ class ColocationGraph {
   // given id is connected.
   int FindRoot(int node_id) { return Member::FindRoot(&members_, node_id); }
 
-  // Ensures that the devices of 'dst's resource and reference match the device
-  // specified for 'src', which is an input of 'dst' with a partially or fully
-  // specified device.
-  Status VerifyResourceAndRefInputsCanBeColocated(
-      const Node* dst, const Node* src,
-      const DeviceNameUtils::ParsedName& src_parsed_name) {
-    std::vector<const Edge*> edges;
-    TF_RETURN_IF_ERROR(dst->input_edges(&edges));
-    for (const Edge* edge : edges) {
-      DataType input_type = dst->input_type(edge->dst_input());
-      if (input_type == DT_RESOURCE || IsRefType(input_type)) {
-        const Node* input_node = edge->src();
-        if (input_node == src) {
-          continue;
-        }
-        const auto& input_root = members_[FindRoot(input_node->id())];
-        const auto& input_parsed_name = input_root.device_name();
-        if (DeviceNameUtils::HasSomeDetails(input_parsed_name) &&
-            !DeviceNameUtils::AreCompatibleDevNames(input_parsed_name,
-                                                    src_parsed_name)) {
-          return AttachDef(
-              errors::InvalidArgument(
-                  "Could not colocate node with its "
-                  "resource and reference inputs; devices ",
-                  DeviceNameUtils::ParsedNameToString(input_parsed_name),
-                  " and ", DeviceNameUtils::ParsedNameToString(src_parsed_name),
-                  " are not compatible."),
-              *dst);
-        }
-      }
-    }
-    return Status::OK();
-  }
-
   const Graph* const graph_;  // Not owned.
   std::vector<Member> members_;
   const DeviceSet* device_set_;  // Not owned.
@@ -1119,6 +1220,15 @@ Status Placer::Run() {
     return errors::FailedPrecondition("No devices are registered");
   }
 
+  if (VLOG_IS_ON(3)) {
+    DumpGraphToFile("placer_input", *graph_, nullptr, "/tmp");
+    for (const Node* node : graph_->op_nodes()) {
+      VLOG(3) << "    " << node->name() << ": requested: '"
+              << node->requested_device() << "' assigned: '"
+              << node->assigned_device_name() << "'";
+    }
+  }
+
   ColocationGraph colocation_graph(
       graph_, devices_, default_device_,
       options_ == nullptr || options_->config.allow_soft_placement(),
@@ -1126,14 +1236,15 @@ Status Placer::Run() {
 
   TF_RETURN_IF_ERROR(colocation_graph.Initialize());
 
-  // For each node, assign a device based on the constraints in the
-  // disjoint node set.
+  // For each node, assign a device based on the constraints in the disjoint
+  // node set.
   std::vector<Node*> second_pass;
   for (Node* node : graph_->op_nodes()) {
     // The graph may have come pre-populated by the framework with assigned
     // devices (e.g., for stateful placements), so the placer should not try to
     // place nodes that are already placed.
     if (node->has_assigned_device_name()) {
+      TF_RETURN_IF_ERROR(colocation_graph.LimitToAssignedDevice(*node));
       LogDeviceAssignment(node, log_device_placement_);
       continue;
     }
@@ -1234,6 +1345,9 @@ Status Placer::Run() {
                                     log_device_placement_));
   }
 
+  if (VLOG_IS_ON(3)) {
+    DumpGraphToFile("placer_output", *graph_, nullptr, "/tmp");
+  }
   return Status::OK();
 }
 
diff --git a/tensorflow/core/common_runtime/placer_test.cc b/tensorflow/core/common_runtime/placer_test.cc
index 04e77e55f6..3cc5ad88ea 100644
--- a/tensorflow/core/common_runtime/placer_test.cc
+++ b/tensorflow/core/common_runtime/placer_test.cc
@@ -17,6 +17,7 @@ limitations under the License.
 
 #include <memory>
 #include <string>
+#include <unordered_set>
 #include <utility>
 #include <vector>
 
@@ -24,11 +25,15 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/common_runtime/device_set.h"
 #include "tensorflow/core/framework/device_attributes.pb.h"
+#include "tensorflow/core/framework/function.h"
+#include "tensorflow/core/framework/function_testlib.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/op_def_builder.h"
 #include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/types.pb.h"
 #include "tensorflow/core/graph/graph.h"
+#include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/graph/graph_def_builder.h"
 #include "tensorflow/core/graph/graph_def_builder_util.h"
 #include "tensorflow/core/lib/core/error_codes.pb.h"
@@ -40,6 +45,16 @@ limitations under the License.
 
 namespace tensorflow {
 
+using ::tensorflow::test::function::GDef;
+using ::tensorflow::test::function::NDef;
+using FDH = ::tensorflow::FunctionDefHelper;
+
+constexpr char kCPU[] = "/device:fakecpu:0";
+constexpr char kGPU[] = "/device:fakegpu:0";
+
+constexpr char kFullCPU[] = "/job:a/replica:0/task:0/device:fakecpu:0";
+constexpr char kFullGPU[] = "/job:a/replica:0/task:0/device:fakegpu:0";
+
 namespace {
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -210,6 +225,16 @@ class PlacerTest : public ::testing::Test {
     return Status::OK();
   }
 
+  Status BuildGraph(const GraphDef& graph_def, Graph* out_graph) {
+    GraphConstructorOptions opts;
+    TF_RETURN_IF_ERROR(ConvertGraphDefToGraph(opts, graph_def, out_graph));
+    nodes_by_name_.clear();
+    for (Node* node : out_graph->nodes()) {
+      nodes_by_name_[node->name()] = node->id();
+    }
+    return Status::OK();
+  }
+
   // Invokes the Placer on "graph". If no DeviceSet is specified, the
   // placement will use the default DeviceSet (of 10 CPU and 10 GPU devices).
   //
@@ -866,7 +891,7 @@ TEST_F(PlacerTest, TestResourceHandle) {
 }
 
 TEST_F(PlacerTest, TestResourceHandlesOnDifferentDevicesFails) {
-  auto handle_test = [this](bool allow_soft_placement) {
+  auto handle_test = [this](bool allow_soft_placement, bool set_assigned) {
     Graph g(OpRegistry::Global());
     {  // Scope for temporary variables used to construct g.
       GraphDefBuilder b(GraphDefBuilder::kFailImmediately);
@@ -878,27 +903,41 @@ TEST_F(PlacerTest, TestResourceHandlesOnDifferentDevicesFails) {
                     b.opts().WithName("two_handles_in"));
       TF_EXPECT_OK(BuildGraph(b, &g));
 
-      GetNodeByName(g, "var_cpu")
-          ->set_assigned_device_name(
-              "/job:a/replica:0/task:0/device:fakecpu:0");
-      GetNodeByName(g, "var_gpu")
-          ->set_assigned_device_name(
-              "/job:a/replica:0/task:0/device:fakegpu:0");
+      if (set_assigned) {
+        GetNodeByName(g, "var_cpu")
+            ->set_assigned_device_name(
+                "/job:a/replica:0/task:0/device:fakecpu:0");
+        GetNodeByName(g, "var_gpu")
+            ->set_assigned_device_name(
+                "/job:a/replica:0/task:0/device:fakegpu:0");
+      } else {
+        GetNodeByName(g, "var_cpu")
+            ->set_requested_device("/job:a/replica:0/task:0/device:fakecpu:0");
+        GetNodeByName(g, "var_gpu")
+            ->set_requested_device("/job:a/replica:0/task:0/device:fakegpu:0");
+      }
     }
 
     SessionOptions options;
     options.config.set_allow_soft_placement(allow_soft_placement);
     options.config.set_log_device_placement(true);
     Status s = Place(&g, &options);
-    EXPECT_EQ(error::INVALID_ARGUMENT, s.code());
+    EXPECT_EQ(error::INVALID_ARGUMENT, s.code()) << s.ToString();
     EXPECT_TRUE(str_util::StrContains(
         s.error_message(),
-        "Could not colocate node with its resource and reference inputs"));
+        "Cannot place the graph because a reference or resource edge "
+        "connects "
+        "colocation groups with incompatible assigned devices: "
+        "/job:a/replica:0/task:0/device:fakegpu:0 vs "
+        "/job:a/replica:0/task:0/device:fakecpu:0"));
+
     return Status::OK();
   };
 
-  TF_EXPECT_OK(handle_test(false));
-  TF_EXPECT_OK(handle_test(true));
+  TF_EXPECT_OK(handle_test(false, false));
+  TF_EXPECT_OK(handle_test(false, true));
+  TF_EXPECT_OK(handle_test(true, false));
+  TF_EXPECT_OK(handle_test(true, true));
 }
 
 // Test that an assignment of an operator to the wrong device
@@ -1617,5 +1656,127 @@ TEST_F(PlacerTest, TestGeneratorNodeDoesntFollowNonColocatedConsumers) {
   EXPECT_DEVICE_TYPE(g, "in", "FakeGPU");
 }
 
+REGISTER_KERNEL_BUILDER(Name("_Arg").Device("FakeCPU"), DummyOp);
+REGISTER_KERNEL_BUILDER(Name("_Arg").Device("FakeGPU"), DummyOp);
+REGISTER_KERNEL_BUILDER(Name("_Retval").Device("FakeCPU"), DummyOp);
+REGISTER_KERNEL_BUILDER(Name("_Retval").Device("FakeGPU"), DummyOp);
+REGISTER_KERNEL_BUILDER(Name("Identity").Device("FakeCPU"), DummyOp);
+REGISTER_KERNEL_BUILDER(Name("Identity").Device("FakeGPU"), DummyOp);
+REGISTER_KERNEL_BUILDER(Name("Const").Device("FakeCPU"), DummyOp);
+REGISTER_KERNEL_BUILDER(Name("Const").Device("FakeGPU"), DummyOp);
+REGISTER_KERNEL_BUILDER(Name("Mul").Device("FakeCPU"), DummyOp);
+REGISTER_KERNEL_BUILDER(Name("Mul").Device("FakeGPU"), DummyOp);
+REGISTER_KERNEL_BUILDER(Name("Add").Device("FakeCPU"), DummyOp);
+REGISTER_KERNEL_BUILDER(Name("Add").Device("FakeGPU"), DummyOp);
+
+TEST_F(PlacerTest, RequestedDeviceOnResourceGeneratorIsTreatedAsAssigned) {
+  /*
+   *    a:RES:GPU  b:RES:CPU
+   *       |         |
+   *       |         |
+   *       v         v
+   *      id1       id2
+   *     @loc:id2
+   */
+  FunctionDef func = test::function::ResourceOutput();
+  GraphDef graph = GDef(
+      {
+          NDef("a", "_Arg", {}, {{"T", DT_RESOURCE}}, kGPU),
+          NDef("b", "_Arg", {}, {{"T", DT_RESOURCE}}, kCPU),
+          NDef("id1", "Identity", {"a"},
+               {{"T", DT_RESOURCE},
+                {"_class", gtl::ArraySlice<string>({"loc:@id2"})}}),
+          NDef("id2", "Identity", {"b"}, {{"T", DT_RESOURCE}}),
+      },
+      // FunctionLib
+      {func});
+
+  Graph g(OpRegistry::Global());
+  TF_ASSERT_OK(BuildGraph(graph, &g));
+  Status s = Place(&g);
+  EXPECT_EQ(error::INVALID_ARGUMENT, s.code());
+  EXPECT_TRUE(str_util::StrContains(
+      s.error_message(),
+      "Cannot place the graph because a reference or resource edge connects "
+      "colocation groups with incompatible assigned devices:"));
+}
+
+TEST_F(PlacerTest, RequestedDeviceCanBeOverridden) {
+  /*
+   *     a:RES      b:RES
+   *       |         |
+   *     id_a:GPU   id_b:CPU
+   *       |         |
+   *       v         v
+   *      id1       id2
+   *     @loc:id2
+   */
+  FunctionDef func = test::function::ResourceOutput();
+  GraphDef graph = GDef(
+      {
+          NDef("a", "_Arg", {}, {{"T", DT_RESOURCE}}),
+          NDef("b", "_Arg", {}, {{"T", DT_RESOURCE}}),
+          NDef("id_a", "Identity", {"a"}, {{"T", DT_RESOURCE}}, kGPU),
+          NDef("id_b", "Identity", {"b"}, {{"T", DT_RESOURCE}}, kCPU),
+          NDef("id1", "Identity", {"id_a"},
+               {{"T", DT_RESOURCE},
+                {"_class", gtl::ArraySlice<string>({"loc:@id2"})}}),
+          NDef("id2", "Identity", {"id_b"}, {{"T", DT_RESOURCE}}),
+      },
+      // FunctionLib
+      {func});
+
+  Graph g(OpRegistry::Global());
+  TF_ASSERT_OK(BuildGraph(graph, &g));
+  TF_ASSERT_OK(Place(&g));
+
+  // All should be colocated
+  EXPECT_COLOCATED(g, "a", "b");
+  EXPECT_COLOCATED(g, "id_a", "id_b");
+  EXPECT_COLOCATED(g, "id1", "id2");
+  EXPECT_COLOCATED(g, "a", "id_a");
+  EXPECT_COLOCATED(g, "a", "id1");
+}
+
+TEST_F(PlacerTest, AssignedDevicesAreNotOverriddenDueToResourcesAndColocation) {
+  /*
+   *     a:RES      b:RES
+   *       |         |
+   *     id_a:GPU   id_b:CPU
+   *       |         |
+   *       v         v
+   *      id1       id2
+   *     @loc:id2
+   */
+  FunctionDef func = test::function::ResourceOutput();
+  GraphDef graph = GDef(
+      {
+          NDef("a", "_Arg", {}, {{"T", DT_RESOURCE}}),
+          NDef("b", "_Arg", {}, {{"T", DT_RESOURCE}}),
+          NDef("id_a", "Identity", {"a"}, {{"T", DT_RESOURCE}}),
+          NDef("id_b", "Identity", {"b"}, {{"T", DT_RESOURCE}}),
+          NDef("id1", "Identity", {"id_a"},
+               {{"T", DT_RESOURCE},
+                {"_class", gtl::ArraySlice<string>({"loc:@id2"})}}),
+          NDef("id2", "Identity", {"id_b"}, {{"T", DT_RESOURCE}}),
+      },
+      // FunctionLib
+      {func});
+
+  Graph g(OpRegistry::Global());
+  TF_ASSERT_OK(BuildGraph(graph, &g));
+  std::unordered_map<string, Node*> nodes = g.BuildNodeNameIndex();
+  GetNodeByName(g, "id_a")->set_assigned_device_name(kFullGPU);
+  GetNodeByName(g, "id_b")->set_assigned_device_name(kFullCPU);
+  Status s = Place(&g);
+  EXPECT_EQ(error::INVALID_ARGUMENT, s.code());
+  EXPECT_TRUE(str_util::StrContains(
+      s.error_message(),
+      "Cannot place the graph because a reference or resource edge connects "
+      "colocation groups with incompatible assigned devices: "
+      "/job:a/replica:0/task:0/device:fakecpu:0 vs "
+      "/job:a/replica:0/task:0/device:fakegpu:0"));
+}
+
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/core/util/device_name_utils.cc b/tensorflow/core/util/device_name_utils.cc
index cb088faec1..56e618872a 100644
--- a/tensorflow/core/util/device_name_utils.cc
+++ b/tensorflow/core/util/device_name_utils.cc
@@ -289,6 +289,30 @@ bool DeviceNameUtils::IsSpecification(const ParsedName& less_specific,
   return true;
 }
 
+void DeviceNameUtils::EnsureSpecification(ParsedName* more_specific,
+                                          const ParsedName& less_specific) {
+  if (less_specific.has_job) {
+    more_specific->has_job = true;
+    more_specific->job = less_specific.job;
+  }
+  if (less_specific.has_replica) {
+    more_specific->has_replica = true;
+    more_specific->replica = less_specific.replica;
+  }
+  if (less_specific.has_task) {
+    more_specific->has_task = true;
+    more_specific->task = less_specific.task;
+  }
+  if (less_specific.has_type) {
+    more_specific->has_type = true;
+    more_specific->type = less_specific.type;
+  }
+  if (less_specific.has_id) {
+    more_specific->has_id = true;
+    more_specific->id = less_specific.id;
+  }
+}
+
 /* static */
 bool DeviceNameUtils::IsCompleteSpecification(const ParsedName& pattern,
                                               const ParsedName& name) {
diff --git a/tensorflow/core/util/device_name_utils.h b/tensorflow/core/util/device_name_utils.h
index bb5e2b3f0c..b047e814bd 100644
--- a/tensorflow/core/util/device_name_utils.h
+++ b/tensorflow/core/util/device_name_utils.h
@@ -110,6 +110,11 @@ class DeviceNameUtils {
   static bool IsSpecification(const ParsedName& less_specific,
                               const ParsedName& more_specific);
 
+  // Makes minimal changes to more_specific so that it becomes a
+  // specification of less_specific.
+  static void EnsureSpecification(ParsedName* more_specific,
+                                  const ParsedName& less_specific);
+
   // Like IsSpecification, but the second argument "name" must have a
   // non-wildcard value for all of its components.
   static bool IsCompleteSpecification(const ParsedName& pattern,
diff --git a/tensorflow/python/data/kernel_tests/map_test.py b/tensorflow/python/data/kernel_tests/map_test.py
index 1cd1ddc351..7a17dd852a 100644
--- a/tensorflow/python/data/kernel_tests/map_test.py
+++ b/tensorflow/python/data/kernel_tests/map_test.py
@@ -451,7 +451,8 @@ class MapTest(test_base.DatasetTestBase, parameterized.TestCase):
       dataset = dataset_ops.Dataset.from_tensors(0).repeat(10).map(func)
       expected_error = (
           errors.InvalidArgumentError,
-          "Could not colocate node with its resource and reference inputs")
+          "Cannot place the graph because a reference or resource edge "
+          "connects colocation groups with incompatible assigned devices")
       self.assertDatasetProduces(
           dataset, expected_error=expected_error, requires_initialization=True)
 
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index 33ae062f5d..871e4e2842 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -823,8 +823,9 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
       return None
 
     with self.assertRaisesRegexp(
-        errors.InvalidArgumentError, 'Could not colocate node with its '
-        'resource and reference inputs.*'):
+        errors.InvalidArgumentError,
+        'Cannot place the graph because a reference or resource edge connects '
+        'colocation groups with incompatible assigned devices'):
       if not context.executing_eagerly():
         self.evaluate(variables.global_variables_initializer())
       self.evaluate(resource_apply_adam())
-- 
GitLab


From 324c3394864b7d1aa37cf8ac9598f094d768d2be Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 6 Feb 2019 11:57:44 -0800
Subject: [PATCH 2011/2345] Allows tf.convert_to_tensor to work with
 zero-dimensional numpy arrays

PiperOrigin-RevId: 232717709
---
 tensorflow/python/eager/tensor_test.py      | 19 +++++++++++++++-
 tensorflow/python/lib/core/py_seq_tensor.cc | 25 +++++++++++++++++++--
 2 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/eager/tensor_test.py b/tensorflow/python/eager/tensor_test.py
index 0ee2ff68c2..0d8845bd96 100644
--- a/tensorflow/python/eager/tensor_test.py
+++ b/tensorflow/python/eager/tensor_test.py
@@ -339,6 +339,24 @@ class TFETensorTest(test_util.TensorFlowTestCase):
   def testConvertToTensorAllowsOverflow(self):
     _ = ops.convert_to_tensor(123456789, dtype=dtypes.uint8)
 
+  @test_util.assert_no_new_pyobjects_executing_eagerly
+  @test_util.run_in_graph_and_eager_modes
+  def testConvertToTensorNumpyZeroDim(self):
+    for np_type, dtype in [(np.int32, dtypes.int32),
+                           (np.half, dtypes.half),
+                           (np.float32, dtypes.float32)]:
+      x = ops.convert_to_tensor([np.array(65, dtype=np_type),
+                                 np.array(16, dtype=np_type)])
+      self.assertEqual(x.dtype, dtype)
+      self.assertAllEqual(x, [65, 16])
+
+  @test_util.assert_no_new_pyobjects_executing_eagerly
+  @test_util.run_in_graph_and_eager_modes
+  def testConvertToTensorNumpyScalar(self):
+    x = ops.convert_to_tensor([np.asscalar(np.array(321, dtype=np.int)),
+                               np.asscalar(np.array(16, dtype=np.int))])
+    self.assertAllEqual(x, [321, 16])
+
   def testEagerTensorError(self):
     with self.assertRaisesRegexp(
         TypeError,
@@ -347,7 +365,6 @@ class TFETensorTest(test_util.TensorFlowTestCase):
       _ = ops.convert_to_tensor(1., dtype=dtypes.int32)
 
 
-
 class TFETensorUtilTest(test_util.TensorFlowTestCase):
 
   def testListOfThree(self):
diff --git a/tensorflow/python/lib/core/py_seq_tensor.cc b/tensorflow/python/lib/core/py_seq_tensor.cc
index f681cff6cf..6cdf7e7163 100644
--- a/tensorflow/python/lib/core/py_seq_tensor.cc
+++ b/tensorflow/python/lib/core/py_seq_tensor.cc
@@ -64,6 +64,19 @@ bool IsPyFloat(PyObject* obj) {
          PyIsInstance(obj, &PyFloatingArrType_Type);  // NumPy float types
 }
 
+// If the input is a zero dimensional PyArray return it converted to a scalar.
+// Otherwise return the input and increment its reference count.
+// Users must Py_DECREF the output of this method.
+PyObject* ZeroDimArrayToScalar(PyObject* obj) {
+  if (PyArray_IsZeroDim(obj) && !PyArray_IsScalar(obj, Generic)) {
+    auto pyarray_obj = reinterpret_cast<PyArrayObject*>(obj);
+    obj = PyArray_ToScalar(PyArray_DATA(pyarray_obj), pyarray_obj);
+  } else {
+    Py_INCREF(obj);
+  }
+  return obj;
+}
+
 // Converts Python object `c` that should hold a Python string into a
 // C++ string in *out.  Returns nullptr on success, or a message on error.
 // Defined below, but forward declared here for use in PyRepr.
@@ -130,6 +143,10 @@ Status SampleElementFromSequence(PyObject* seq, PyObject** elem) {
 Status InferShapeAndType(PyObject* obj, TensorShape* shape, DataType* dtype) {
   std::vector<Safe_PyObjectPtr> refs_to_clean;
   while (true) {
+    // Convert any zero dimensional numpy arrays to scalars first of all.
+    // We also have to make sure a reference to the safe_obj is kept.
+    obj = ZeroDimArrayToScalar(obj);
+    refs_to_clean.push_back(make_safe(obj));
     // We test strings first, in case a string is considered a sequence.
     if (IsPyString(obj)) {
       *dtype = DT_STRING;
@@ -240,7 +257,9 @@ const char ErrorFoundFloat[] =
       }                                                                   \
       PyObject** l = PySequence_Fast_ITEMS(seq.get());                    \
       for (int64 i = 0; i < s; ++i) {                                     \
-        const char* error = CONVERT(l[i], *buf);                          \
+        auto scalar = ZeroDimArrayToScalar(l[i]);                         \
+        const char* error = CONVERT(scalar, *buf);                        \
+        Py_DECREF(scalar);                                                \
         if (TF_PREDICT_FALSE(error != nullptr)) return error;             \
         ++*buf;                                                           \
       }                                                                   \
@@ -253,7 +272,9 @@ const char ErrorFoundFloat[] =
     Tensor result(TYPE_ENUM, shape);                                      \
     if (shape.dims() == 0) { /* Scalar case */                            \
       TYPE value;                                                         \
-      const char* error = CONVERT(obj, &value);                           \
+      auto scalar = ZeroDimArrayToScalar(obj);                            \
+      const char* error = CONVERT(scalar, &value);                        \
+      Py_DECREF(scalar);                                                  \
       if (error != nullptr) return error;                                 \
       result.scalar<TYPE>()() = value;                                    \
     } else {                                                              \
-- 
GitLab


From 78840bb8646aed747e9bf643a11ed2fa77b4bbf6 Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Wed, 6 Feb 2019 12:01:19 -0800
Subject: [PATCH 2012/2345] Remove flaky timing check.

PiperOrigin-RevId: 232718485
---
 tensorflow/python/keras/layers/tensorflow_op_layer_test.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tensorflow/python/keras/layers/tensorflow_op_layer_test.py b/tensorflow/python/keras/layers/tensorflow_op_layer_test.py
index f01c9d4964..c2f1c79844 100644
--- a/tensorflow/python/keras/layers/tensorflow_op_layer_test.py
+++ b/tensorflow/python/keras/layers/tensorflow_op_layer_test.py
@@ -158,8 +158,6 @@ class AutoLambdaTest(keras_parameterized.TestCase):
     size_50 = _construct_graph_of_size(50)
     size_500 = _construct_graph_of_size(500)
 
-    # Check reasonable graph construction time.
-    self.assertLess(size_50, 5)
     # Check construction time grows approx. linearly with size.
     e = 2  # Fudge factor to prevent flakiness.
     self.assertLess(size_500, (10 * e) * size_50)
-- 
GitLab


From 3f8dcd3e288f213001eace4aea0f22cfb1b65946 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 6 Feb 2019 12:05:05 -0800
Subject: [PATCH 2013/2345] Allow exceptions in code that will be staged.
 Consider all exceptions to be exiting the CFG, with no explicit support for
 exception-based control-flow. It may incidentally work to use exception-based
 control flow in code that is never staged.

PiperOrigin-RevId: 232719435
---
 .../converters/side_effect_guards.py          |  4 +++
 tensorflow/python/autograph/pyct/cfg.py       | 36 ++++++++-----------
 .../pyct/static_analysis/liveness.py          |  4 +++
 .../static_analysis/reaching_definitions.py   |  7 +++-
 4 files changed, 28 insertions(+), 23 deletions(-)

diff --git a/tensorflow/python/autograph/converters/side_effect_guards.py b/tensorflow/python/autograph/converters/side_effect_guards.py
index d7c0951fcc..7e556d9513 100644
--- a/tensorflow/python/autograph/converters/side_effect_guards.py
+++ b/tensorflow/python/autograph/converters/side_effect_guards.py
@@ -125,6 +125,10 @@ class SideEffectGuardTransformer(converter.Base):
     node.orelse = self._visit_and_reindent(node.orelse)
     return node
 
+  # TODO(b/123995141) Remove once ExceptionHandlers are in the CFG
+  def visit_ExceptHandler(self, node):
+    return node
+
   def visit_Expr(self, node):
     self.generic_visit(node)
     if isinstance(node.value, gast.Call):
diff --git a/tensorflow/python/autograph/pyct/cfg.py b/tensorflow/python/autograph/pyct/cfg.py
index fdfcd4dcc1..0cedfa84ab 100644
--- a/tensorflow/python/autograph/pyct/cfg.py
+++ b/tensorflow/python/autograph/pyct/cfg.py
@@ -393,6 +393,8 @@ class GraphBuilder(object):
   def _connect_jump_to_finally_sections(self, node):
     """Connects a jump node to the finally sections protecting it."""
     cursor = set((node,))
+    if node not in self.finally_sections:
+      return cursor
     for guard_section_id in self.finally_sections[node]:
       guard_begin, guard_ends = self.finally_section_subgraphs[guard_section_id]
       self._connect_nodes(cursor, guard_begin)
@@ -620,10 +622,10 @@ class AstToCfg(gast.NodeVisitor):
     leaving_node = self.lexical_scopes.pop()
     assert node == leaving_node
 
-  def _get_enclosing_scopes(self, include, stop_at):
+  def _get_enclosing_finally_scopes(self, stop_at):
     included = []
     for node in reversed(self.lexical_scopes):
-      if isinstance(node, include):
+      if isinstance(node, gast.Try) and node.finalbody:
         included.append(node)
       if isinstance(node, stop_at):
         return node, included
@@ -635,10 +637,8 @@ class AstToCfg(gast.NodeVisitor):
 
   def _process_exit_statement(self, node, *exits_nodes_of_type):
     # Note: this is safe because we process functions separately.
-    try_node, guards = self._get_enclosing_scopes(
-        include=(gast.Try,),
-        stop_at=tuple(exits_nodes_of_type),
-    )
+    try_node, guards = self._get_enclosing_finally_scopes(
+        tuple(exits_nodes_of_type))
     if try_node is None:
       raise ValueError(
           '%s that is not enclosed by any of %s' % (node, exits_nodes_of_type))
@@ -646,10 +646,8 @@ class AstToCfg(gast.NodeVisitor):
 
   def _process_continue_statement(self, node, *loops_to_nodes_of_type):
     # Note: this is safe because we process functions separately.
-    try_node, guards = self._get_enclosing_scopes(
-        include=(gast.Try,),
-        stop_at=tuple(loops_to_nodes_of_type),
-    )
+    try_node, guards = self._get_enclosing_finally_scopes(
+        tuple(loops_to_nodes_of_type))
     if try_node is None:
       raise ValueError('%s that is not enclosed by any of %s' %
                        (node, loops_to_nodes_of_type))
@@ -698,10 +696,7 @@ class AstToCfg(gast.NodeVisitor):
     self._process_basic_statement(node)
 
   def visit_Raise(self, node):
-    try_node, guards = self._get_enclosing_scopes(
-        include=(gast.Try,),
-        stop_at=(gast.FunctionDef,),
-    )
+    try_node, guards = self._get_enclosing_finally_scopes((gast.FunctionDef,))
     if try_node is None:
       raise ValueError('%s that is not enclosed by any FunctionDef' % node)
     self.builder.add_error_node(node, guards)
@@ -797,16 +792,13 @@ class AstToCfg(gast.NodeVisitor):
     for stmt in node.orelse:
       self.visit(stmt)
 
-    if node.handlers:
-      # TODO(mdan): Should we still support bare try/except? Might be confusing.
-      raise NotImplementedError('exceptions are not yet supported')
-
     self._exit_lexical_scope(node)
 
-    self.builder.enter_finally_section(node)
-    for stmt in node.finalbody:
-      self.visit(stmt)
-    self.builder.exit_finally_section(node)
+    if node.finalbody:
+      self.builder.enter_finally_section(node)
+      for stmt in node.finalbody:
+        self.visit(stmt)
+      self.builder.exit_finally_section(node)
 
   def visit_With(self, node):
     # TODO(mdan): Mark the context manager's exit call as exit guard.
diff --git a/tensorflow/python/autograph/pyct/static_analysis/liveness.py b/tensorflow/python/autograph/pyct/static_analysis/liveness.py
index f8b8d7fa77..691b786db0 100644
--- a/tensorflow/python/autograph/pyct/static_analysis/liveness.py
+++ b/tensorflow/python/autograph/pyct/static_analysis/liveness.py
@@ -219,6 +219,10 @@ class Annotator(transformer.Base):
                  frozenset(self.current_analyzer.out[cfg_node]))
     return node
 
+  def visit_ExceptHandler(self, node):
+    # TODO(b/123995141) Add Exception Handlers to the CFG
+    return node
+
 
 def resolve(node, source_info, graphs):
   """Resolves the live symbols at the exit of control flow statements.
diff --git a/tensorflow/python/autograph/pyct/static_analysis/reaching_definitions.py b/tensorflow/python/autograph/pyct/static_analysis/reaching_definitions.py
index d1587d8178..6f0f09ee88 100644
--- a/tensorflow/python/autograph/pyct/static_analysis/reaching_definitions.py
+++ b/tensorflow/python/autograph/pyct/static_analysis/reaching_definitions.py
@@ -223,6 +223,10 @@ class TreeAnnotator(transformer.Base):
   def visit_global(self, node):
     raise NotImplementedError()
 
+  def visit_ExceptHandler(self, node):
+    # TODO(b/123995141) Add Exception Handlers to the CFG
+    return node
+
   def visit_Name(self, node):
     if self.current_analyzer is None:
       # Names may appear outside function defs - for example in class
@@ -232,7 +236,8 @@ class TreeAnnotator(transformer.Base):
     analyzer = self.current_analyzer
     cfg_node = self.current_cfg_node
 
-    assert cfg_node is not None, 'name node outside of any statement?'
+    assert cfg_node is not None, ('name node, %s, outside of any statement?'
+                                  % node.id)
 
     qn = anno.getanno(node, anno.Basic.QN)
     if isinstance(node.ctx, gast.Load):
-- 
GitLab


From 7bbc65be714be8d15378b58d0ea7cbc29e8c3769 Mon Sep 17 00:00:00 2001
From: Ruoxin Sang <rxsang@google.com>
Date: Wed, 6 Feb 2019 12:39:29 -0800
Subject: [PATCH 2014/2345] Support IteratorGetNextAsOptionalOp in TPU.

PiperOrigin-RevId: 232725342
---
 tensorflow/compiler/jit/xla_device_ops.h     |   2 +
 tensorflow/core/kernels/data/iterator_ops.cc | 114 ++++++++-----------
 tensorflow/core/kernels/data/iterator_ops.h  |  20 ++++
 3 files changed, 69 insertions(+), 67 deletions(-)

diff --git a/tensorflow/compiler/jit/xla_device_ops.h b/tensorflow/compiler/jit/xla_device_ops.h
index 927f983ba9..f201f62a78 100644
--- a/tensorflow/compiler/jit/xla_device_ops.h
+++ b/tensorflow/compiler/jit/xla_device_ops.h
@@ -241,6 +241,8 @@ class XlaAssignVariableOp : public OpKernel {
                           data::AnonymousIteratorHandleOp);                    \
   REGISTER_KERNEL_BUILDER(Name("IteratorGetNext").Device(DEVICE),              \
                           data::IteratorGetNextOp);                            \
+  REGISTER_KERNEL_BUILDER(Name("IteratorGetNextAsOptional").Device(DEVICE),    \
+                          data::IteratorGetNextAsOptionalOp);                  \
   REGISTER_KERNEL_BUILDER(Name("IteratorGetNextSync").Device(DEVICE),          \
                           data::IteratorGetNextSyncOp);                        \
   REGISTER_KERNEL_BUILDER(Name("IteratorToStringHandle")                       \
diff --git a/tensorflow/core/kernels/data/iterator_ops.cc b/tensorflow/core/kernels/data/iterator_ops.cc
index 808f834f62..0d2dfd962b 100644
--- a/tensorflow/core/kernels/data/iterator_ops.cc
+++ b/tensorflow/core/kernels/data/iterator_ops.cc
@@ -967,78 +967,58 @@ void IteratorGetNextSyncOp::Compute(OpKernelContext* ctx) {
   }
 }
 
-namespace {
+void IteratorGetNextAsOptionalOp::ComputeAsync(OpKernelContext* ctx,
+                                               DoneCallback done) {
+  IteratorResource* iterator;
+  OP_REQUIRES_OK_ASYNC(
+      ctx, LookupResource(ctx, HandleFromInput(ctx, 0), &iterator), done);
+  // The call to `iterator->GetNext()` may block and depend on an
+  // inter-op thread pool thread, so we issue the call from the
+  // owned thread pool.
+  background_worker_.Schedule(std::bind(
+      [this, ctx, iterator](DoneCallback done) {
+        std::vector<Tensor> components;
+        bool end_of_sequence = false;
 
-class IteratorGetNextAsOptionalOp : public AsyncOpKernel {
- public:
-  explicit IteratorGetNextAsOptionalOp(OpKernelConstruction* ctx)
-      : AsyncOpKernel(ctx),
-        background_worker_(ctx->env(),
-                           "tf_data_iterator_get_next_as_optional") {
-    OP_REQUIRES_OK(ctx, ctx->GetAttr("output_types", &output_types_));
-    OP_REQUIRES_OK(ctx, ctx->GetAttr("output_shapes", &output_shapes_));
-  }
+        Status s = iterator->GetNext(IteratorContext(ctx), &components,
+                                     &end_of_sequence);
+        // NOTE(mrry): We must unref the iterator before calling `done()`, to
+        // avoid destruction races.
+        iterator->Unref();
 
-  void ComputeAsync(OpKernelContext* ctx, DoneCallback done) override {
-    IteratorResource* iterator;
-    OP_REQUIRES_OK_ASYNC(
-        ctx, LookupResource(ctx, HandleFromInput(ctx, 0), &iterator), done);
-    // The call to `iterator->GetNext()` may block and depend on an
-    // inter-op thread pool thread, so we issue the call from the
-    // owned thread pool.
-    background_worker_.Schedule(std::bind(
-        [this, ctx, iterator](DoneCallback done) {
-          std::vector<Tensor> components;
-          bool end_of_sequence = false;
-
-          Status s = iterator->GetNext(IteratorContext(ctx), &components,
-                                       &end_of_sequence);
-          // NOTE(mrry): We must unref the iterator before calling `done()`, to
-          // avoid destruction races.
-          iterator->Unref();
-
-          if (!s.ok()) {
-            ctx->SetStatus(s);
-          } else if (end_of_sequence) {
-            OP_REQUIRES_OK_ASYNC(ctx, WriteOptionalNoneToOutput(ctx, 0), done);
-          } else {
-            for (int i = 0; i < components.size(); ++i) {
-              OP_REQUIRES_ASYNC(
-                  ctx, components[i].dtype() == output_types_[i],
-                  errors::InvalidArgument(
-                      "The given optional does not match the expected type for "
-                      "component ",
-                      i, ". Expected: ", DataTypeString(output_types_[i]),
-                      ". Actual: ", DataTypeString(components[i].dtype()), "."),
-                  done);
-              OP_REQUIRES_ASYNC(
-                  ctx,
-                  output_shapes_[i].IsCompatibleWith(components[i].shape()),
-                  errors::InvalidArgument(
-                      "The given optional does not match the expected shape "
-                      "for component ",
-                      i, ". Expected: ", output_shapes_[i].DebugString(),
-                      ". Actual: ", components[i].shape().DebugString(), "."),
-                  done);
-            }
-
-            OP_REQUIRES_OK_ASYNC(
-                ctx,
-                WriteOptionalWithValueToOutput(ctx, 0, std::move(components)),
+        if (!s.ok()) {
+          ctx->SetStatus(s);
+        } else if (end_of_sequence) {
+          OP_REQUIRES_OK_ASYNC(ctx, WriteOptionalNoneToOutput(ctx, 0), done);
+        } else {
+          for (int i = 0; i < components.size(); ++i) {
+            OP_REQUIRES_ASYNC(
+                ctx, components[i].dtype() == output_types_[i],
+                errors::InvalidArgument(
+                    "The given optional does not match the expected type for "
+                    "component ",
+                    i, ". Expected: ", DataTypeString(output_types_[i]),
+                    ". Actual: ", DataTypeString(components[i].dtype()), "."),
+                done);
+            OP_REQUIRES_ASYNC(
+                ctx, output_shapes_[i].IsCompatibleWith(components[i].shape()),
+                errors::InvalidArgument(
+                    "The given optional does not match the expected shape "
+                    "for component ",
+                    i, ". Expected: ", output_shapes_[i].DebugString(),
+                    ". Actual: ", components[i].shape().DebugString(), "."),
                 done);
           }
-          done();
-        },
-        std::move(done)));
-  }
-
- private:
-  BackgroundWorker background_worker_;
-  DataTypeVector output_types_;
-  std::vector<PartialTensorShape> output_shapes_;
-};
 
-}  // namespace
+          OP_REQUIRES_OK_ASYNC(
+              ctx,
+              WriteOptionalWithValueToOutput(ctx, 0, std::move(components)),
+              done);
+        }
+        done();
+      },
+      std::move(done)));
+}
 
 void IteratorToStringHandleOp::Compute(OpKernelContext* ctx) {
   const Tensor& resource_handle_t = ctx->input(0);
diff --git a/tensorflow/core/kernels/data/iterator_ops.h b/tensorflow/core/kernels/data/iterator_ops.h
index cd72269859..7d769d365e 100644
--- a/tensorflow/core/kernels/data/iterator_ops.h
+++ b/tensorflow/core/kernels/data/iterator_ops.h
@@ -19,6 +19,8 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/kernels/ops_util.h"
 
 namespace tensorflow {
@@ -115,6 +117,24 @@ class IteratorGetNextOp : public AsyncOpKernel {
   BackgroundWorker background_worker_;
 };
 
+class IteratorGetNextAsOptionalOp : public AsyncOpKernel {
+ public:
+  explicit IteratorGetNextAsOptionalOp(OpKernelConstruction* ctx)
+      : AsyncOpKernel(ctx),
+        background_worker_(ctx->env(),
+                           "tf_data_iterator_get_next_as_optional") {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("output_types", &output_types_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("output_shapes", &output_shapes_));
+  }
+
+  void ComputeAsync(OpKernelContext* ctx, DoneCallback done) override;
+
+ private:
+  BackgroundWorker background_worker_;
+  DataTypeVector output_types_;
+  std::vector<PartialTensorShape> output_shapes_;
+};
+
 class IteratorGetNextSyncOp : public OpKernel {
  public:
   explicit IteratorGetNextSyncOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}
-- 
GitLab


From ba9f4c017cdd95e5d507635215133bb97fc7216d Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Wed, 6 Feb 2019 12:40:48 -0800
Subject: [PATCH 2015/2345] Update Eigen to commit 1724f8760da8.

(1) Revert gebp_neon.patch.oss.

PiperOrigin-RevId: 232725552
---
 tensorflow/opensource_only.files              |  2 +-
 tensorflow/workspace.bzl                      | 10 ++--
 third_party/eigen3/gebp_neon.patch            | 11 ----
 third_party/eigen3/gpu_packet_math.patch      | 18 ++++++
 .../CXX11/src/FixedPoint/PacketMathAVX2.h     | 60 +++++++++++++++++--
 5 files changed, 80 insertions(+), 21 deletions(-)
 delete mode 100644 third_party/eigen3/gebp_neon.patch
 create mode 100644 third_party/eigen3/gpu_packet_math.patch

diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index 0312114abe..9264939b3b 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -120,8 +120,8 @@ tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatMatProdu
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/ThreadPool
 tensorflow/third_party/eigen3/unsupported/Eigen/SpecialFunctions
 tensorflow/third_party/eigen3/unsupported/Eigen/MatrixFunctions
+tensorflow/third_party/eigen3/gpu_packet_math.patch
 tensorflow/third_party/eigen3/LICENSE
-tensorflow/third_party/eigen3/gebp_neon.patch
 tensorflow/third_party/eigen3/BUILD
 tensorflow/third_party/systemlibs/build_defs.bzl.tpl
 tensorflow/third_party/systemlibs/absl_py.BUILD
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 93e73df883..d8dc923184 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -140,12 +140,12 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "eigen_archive",
         build_file = clean_dep("//third_party:eigen.BUILD"),
-        patch_file = clean_dep("//third_party/eigen3:gebp_neon.patch"),
-        sha256 = "48678550a32665331d729be87076e576f2502fff325f5b6c2c78ebf7b1b22c7b",
-        strip_prefix = "eigen-eigen-bcc817c0ba98",
+        patch_file = clean_dep("//third_party/eigen3:gpu_packet_math.patch"),
+        sha256 = "d1d2ac19b8ef386ad70b91932a90bfbc3014b801d14723b9c8373239128bd2dd",
+        strip_prefix = "eigen-eigen-1724f8760da8",
         urls = [
-            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/bcc817c0ba98.tar.gz",
-            "https://bitbucket.org/eigen/eigen/get/bcc817c0ba98.tar.gz",
+            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/1724f8760da8.tar.gz",
+            "https://bitbucket.org/eigen/eigen/get/1724f8760da8.tar.gz",
         ],
     )
 
diff --git a/third_party/eigen3/gebp_neon.patch b/third_party/eigen3/gebp_neon.patch
deleted file mode 100644
index d0022e94cc..0000000000
--- a/third_party/eigen3/gebp_neon.patch
+++ /dev/null
@@ -1,11 +0,0 @@
---- a/Eigen/src/Core/products/GeneralBlockPanelKernel.h	2019-01-22 20:46:51.000000000 -0800
-+++ b/Eigen/src/Core/products/GeneralBlockPanelKernel.h	2019-01-25 13:48:49.000000000 -0800
-@@ -1031,7 +1031,7 @@
- 
-   EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacket& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<0>&) const
-   {
--    c += a * b;
-+    c = vfmaq_n_f32(c, a, b);
-   }
- 
-   EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<0>&) const
diff --git a/third_party/eigen3/gpu_packet_math.patch b/third_party/eigen3/gpu_packet_math.patch
new file mode 100644
index 0000000000..06347538e1
--- /dev/null
+++ b/third_party/eigen3/gpu_packet_math.patch
@@ -0,0 +1,18 @@
+--- a/Eigen/src/Core/arch/GPU/PacketMath.h
++++ b/Eigen/src/Core/arch/GPU/PacketMath.h
+@@ -100,6 +100,7 @@
+   return make_double2(from, from);
+ }
+ 
++#if defined(EIGEN_CUDA_ARCH)
+ namespace {
+ 
+ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float bitwise_and(const float& a,
+@@ -211,6 +212,7 @@
+ pcmp_eq<double2>(const double2& a, const double2& b) {
+   return make_double2(eq_mask(a.x, b.x), eq_mask(a.y, b.y));
+ }
++#endif  // EIGEN_CUDA_ARCH
+ 
+ template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 plset<float4>(const float& a) {
+   return make_float4(a, a+1, a+2, a+3);
diff --git a/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX2.h b/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX2.h
index 223ea4d58b..8df6782551 100644
--- a/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX2.h
+++ b/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX2.h
@@ -178,25 +178,37 @@ template <>
 struct unpacket_traits<Packet32q8i> {
   typedef QInt8 type;
   typedef Packet16q8i half;
-  enum { size = 32, alignment = Aligned32 };
+  enum { size = 32, alignment = Aligned32, vectorizable = true };
+};
+template <>
+struct unpacket_traits<Packet16q8i> {
+  typedef QInt8 type;
+  typedef Packet16q8i half;
+  enum { size = 16, alignment = Aligned32, vectorizable = true };
 };
 template <>
 struct unpacket_traits<Packet16q16i> {
   typedef QInt16 type;
   typedef Packet8q16i half;
-  enum { size = 16, alignment = Aligned32 };
+  enum { size = 16, alignment = Aligned32, vectorizable = true };
+};
+template <>
+struct unpacket_traits<Packet8q16i> {
+  typedef QInt16 type;
+  typedef Packet8q16i half;
+  enum { size = 8, alignment = Aligned32, vectorizable = true };
 };
 template <>
 struct unpacket_traits<Packet32q8u> {
   typedef QUInt8 type;
   typedef Packet16q8u half;
-  enum { size = 32, alignment = Aligned32 };
+  enum { size = 32, alignment = Aligned32, vectorizable = true };
 };
 template <>
 struct unpacket_traits<Packet8q32i> {
   typedef QInt32 type;
   typedef Packet4q32i half;
-  enum { size = 8, alignment = Aligned32 };
+  enum { size = 8, alignment = Aligned32, vectorizable = true };
 };
 
 // Unaligned load
@@ -206,6 +218,11 @@ EIGEN_STRONG_INLINE Packet32q8i ploadu<Packet32q8i>(const QInt8* from) {
       reinterpret_cast<const __m256i*>(from));
 }
 template <>
+EIGEN_STRONG_INLINE Packet16q8i ploadu<Packet16q8i>(const QInt8* from) {
+  EIGEN_DEBUG_UNALIGNED_LOAD return _mm_loadu_si128(
+      reinterpret_cast<const __m128i*>(from));
+}
+template <>
 EIGEN_STRONG_INLINE Packet32q8u ploadu<Packet32q8u>(const QUInt8* from) {
   EIGEN_DEBUG_UNALIGNED_LOAD return _mm256_loadu_si256(
       reinterpret_cast<const __m256i*>(from));
@@ -215,6 +232,11 @@ EIGEN_STRONG_INLINE Packet16q16i ploadu<Packet16q16i>(const QInt16* from) {
   EIGEN_DEBUG_UNALIGNED_LOAD return _mm256_loadu_si256(
       reinterpret_cast<const __m256i*>(from));
 }
+template<>
+EIGEN_STRONG_INLINE Packet8q16i ploadu<Packet8q16i>(const QInt16* from) {
+  EIGEN_DEBUG_UNALIGNED_LOAD return _mm_loadu_si128(
+      reinterpret_cast<const __m128i*>(from));
+}
 template <>
 EIGEN_STRONG_INLINE Packet8q32i ploadu<Packet8q32i>(const QInt32* from) {
   EIGEN_DEBUG_UNALIGNED_LOAD return _mm256_loadu_si256(
@@ -228,6 +250,11 @@ EIGEN_STRONG_INLINE Packet32q8i pload<Packet32q8i>(const QInt8* from) {
       reinterpret_cast<const __m256i*>(from));
 }
 template <>
+EIGEN_STRONG_INLINE Packet16q8i pload<Packet16q8i>(const QInt8* from) {
+  EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_si128(
+      reinterpret_cast<const __m128i*>(from));
+}
+template <>
 EIGEN_STRONG_INLINE Packet32q8u pload<Packet32q8u>(const QUInt8* from) {
   EIGEN_DEBUG_ALIGNED_LOAD return _mm256_load_si256(
       reinterpret_cast<const __m256i*>(from));
@@ -238,6 +265,11 @@ EIGEN_STRONG_INLINE Packet16q16i pload<Packet16q16i>(const QInt16* from) {
       reinterpret_cast<const __m256i*>(from));
 }
 template <>
+EIGEN_STRONG_INLINE Packet8q16i pload<Packet8q16i>(const QInt16* from) {
+  EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_si128(
+      reinterpret_cast<const __m128i*>(from));
+}
+template <>
 EIGEN_STRONG_INLINE Packet8q32i pload<Packet8q32i>(const QInt32* from) {
   EIGEN_DEBUG_ALIGNED_LOAD return _mm256_load_si256(
       reinterpret_cast<const __m256i*>(from));
@@ -250,6 +282,11 @@ EIGEN_STRONG_INLINE void pstoreu<QInt8>(QInt8* to, const Packet32q8i& from) {
       reinterpret_cast<__m256i*>(to), from.val);
 }
 template <>
+EIGEN_STRONG_INLINE void pstoreu<QInt8>(QInt8* to, const Packet16q8i& from) {
+  EIGEN_DEBUG_UNALIGNED_STORE _mm_storeu_si128(
+      reinterpret_cast<__m128i*>(to), from.val);
+}
+template <>
 EIGEN_STRONG_INLINE void pstoreu<QUInt8>(QUInt8* to, const Packet32q8u& from) {
   EIGEN_DEBUG_UNALIGNED_STORE _mm256_storeu_si256(
       reinterpret_cast<__m256i*>(to), from.val);
@@ -260,6 +297,11 @@ EIGEN_STRONG_INLINE void pstoreu<QInt16>(QInt16* to, const Packet16q16i& from) {
       reinterpret_cast<__m256i*>(to), from.val);
 }
 template <>
+EIGEN_STRONG_INLINE void pstoreu<QInt16>(QInt16* to, const Packet8q16i& from) {
+  EIGEN_DEBUG_UNALIGNED_STORE _mm_storeu_si128(
+      reinterpret_cast<__m128i*>(to), from.val);
+}
+template <>
 EIGEN_STRONG_INLINE void pstoreu<QInt32>(QInt32* to, const Packet8q32i& from) {
   EIGEN_DEBUG_UNALIGNED_STORE _mm256_storeu_si256(
       reinterpret_cast<__m256i*>(to), from.val);
@@ -277,6 +319,11 @@ EIGEN_STRONG_INLINE void pstore<QInt16>(QInt16* to, const Packet16q16i& from) {
                                                from.val);
 }
 template <>
+EIGEN_STRONG_INLINE void pstore<QInt16>(QInt16* to, const Packet8q16i& from) {
+  EIGEN_DEBUG_ALIGNED_STORE _mm_store_si128(reinterpret_cast<__m128i*>(to),
+                                            from.val);
+}
+template <>
 EIGEN_STRONG_INLINE void pstore<QUInt8>(QUInt8* to, const Packet32q8u& from) {
   EIGEN_DEBUG_ALIGNED_STORE _mm256_store_si256(reinterpret_cast<__m256i*>(to),
                                                from.val);
@@ -286,6 +333,11 @@ EIGEN_STRONG_INLINE void pstore<QInt8>(QInt8* to, const Packet32q8i& from) {
   EIGEN_DEBUG_ALIGNED_STORE _mm256_store_si256(reinterpret_cast<__m256i*>(to),
                                                from.val);
 }
+template <>
+EIGEN_STRONG_INLINE void pstore<QInt8>(QInt8* to, const Packet16q8i& from) {
+  EIGEN_DEBUG_ALIGNED_STORE _mm_store_si128(reinterpret_cast<__m128i*>(to),
+                                            from.val);
+}
 
 // Extract first element.
 template <>
-- 
GitLab


From ca62689febf70c757927a7949ac7d4f82028ee2c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 6 Feb 2019 12:43:48 -0800
Subject: [PATCH 2016/2345] Python API support for string tensors.

PiperOrigin-RevId: 232726064
---
 tensorflow/lite/python/BUILD                  |   1 -
 tensorflow/lite/python/interpreter_test.py    |  35 ++++++
 .../lite/python/interpreter_wrapper/BUILD     |  14 +++
 .../interpreter_wrapper.cc                    | 111 ++++++++++++------
 .../lite/python/interpreter_wrapper/numpy.cc  |  25 ++++
 .../lite/python/interpreter_wrapper/numpy.h   |  62 ++++++++++
 .../interpreter_wrapper/python_utils.cc       |  89 +++++++++++++-
 .../python/interpreter_wrapper/python_utils.h |  13 +-
 tensorflow/lite/python/lite_test.py           |   4 +-
 tensorflow/lite/python/optimize/BUILD         |   1 +
 .../python/optimize/calibration_wrapper.cc    |   9 +-
 tensorflow/lite/python/testdata/BUILD         |  11 ++
 tensorflow/lite/python/testdata/gather.pbtxt  |  93 +++++++++++++++
 13 files changed, 410 insertions(+), 58 deletions(-)
 create mode 100644 tensorflow/lite/python/interpreter_wrapper/numpy.cc
 create mode 100644 tensorflow/lite/python/interpreter_wrapper/numpy.h
 create mode 100644 tensorflow/lite/python/testdata/gather.pbtxt

diff --git a/tensorflow/lite/python/BUILD b/tensorflow/lite/python/BUILD
index aa0ffe90e1..02b8b80be9 100644
--- a/tensorflow/lite/python/BUILD
+++ b/tensorflow/lite/python/BUILD
@@ -13,7 +13,6 @@ py_library(
     visibility = ["//visibility:public"],
     deps = [
         "//tensorflow/lite/python/interpreter_wrapper:tensorflow_wrap_interpreter_wrapper",
-        "//tensorflow/python:util",
         "//third_party/py/numpy",
     ],
 )
diff --git a/tensorflow/lite/python/interpreter_test.py b/tensorflow/lite/python/interpreter_test.py
index 7ec56a21c9..b21779226f 100644
--- a/tensorflow/lite/python/interpreter_test.py
+++ b/tensorflow/lite/python/interpreter_test.py
@@ -91,6 +91,41 @@ class InterpreterTest(test_util.TensorFlowTestCase):
     output_data = interpreter.get_tensor(output_details[0]['index'])
     self.assertTrue((expected_output == output_data).all())
 
+  def testString(self):
+    interpreter = interpreter_wrapper.Interpreter(
+        model_path=resource_loader.get_path_to_datafile(
+            'testdata/gather_string.tflite'))
+    interpreter.allocate_tensors()
+
+    input_details = interpreter.get_input_details()
+    self.assertEqual(2, len(input_details))
+    self.assertEqual('input', input_details[0]['name'])
+    self.assertEqual(np.string_, input_details[0]['dtype'])
+    self.assertTrue(([10] == input_details[0]['shape']).all())
+    self.assertEqual((0.0, 0), input_details[0]['quantization'])
+    self.assertEqual('indices', input_details[1]['name'])
+    self.assertEqual(np.int64, input_details[1]['dtype'])
+    self.assertTrue(([3] == input_details[1]['shape']).all())
+    self.assertEqual((0.0, 0), input_details[1]['quantization'])
+
+    output_details = interpreter.get_output_details()
+    self.assertEqual(1, len(output_details))
+    self.assertEqual('output', output_details[0]['name'])
+    self.assertEqual(np.string_, output_details[0]['dtype'])
+    self.assertTrue(([3] == output_details[0]['shape']).all())
+    self.assertEqual((0.0, 0), output_details[0]['quantization'])
+
+    test_input = np.array([1, 2, 3], dtype=np.int64)
+    interpreter.set_tensor(input_details[1]['index'], test_input)
+
+    test_input = np.array(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j'])
+    expected_output = np.array([b'b', b'c', b'd'])
+    interpreter.set_tensor(input_details[0]['index'], test_input)
+    interpreter.invoke()
+
+    output_data = interpreter.get_tensor(output_details[0]['index'])
+    self.assertTrue((expected_output == output_data).all())
+
 
 class InterpreterTestErrorPropagation(test_util.TensorFlowTestCase):
 
diff --git a/tensorflow/lite/python/interpreter_wrapper/BUILD b/tensorflow/lite/python/interpreter_wrapper/BUILD
index 6de6fb48f7..6ec7ce497a 100644
--- a/tensorflow/lite/python/interpreter_wrapper/BUILD
+++ b/tensorflow/lite/python/interpreter_wrapper/BUILD
@@ -6,14 +6,26 @@ licenses(["notice"])  # Apache 2.0
 
 load("//tensorflow:tensorflow.bzl", "tf_py_wrap_cc")
 
+cc_library(
+    name = "numpy",
+    srcs = ["numpy.cc"],
+    hdrs = ["numpy.h"],
+    deps = [
+        "//third_party/py/numpy:headers",
+        "//third_party/python_runtime:headers",
+    ],
+)
+
 cc_library(
     name = "interpreter_wrapper_lib",
     srcs = ["interpreter_wrapper.cc"],
     hdrs = ["interpreter_wrapper.h"],
     deps = [
+        ":numpy",
         ":python_error_reporter",
         ":python_utils",
         "//tensorflow/lite:framework",
+        "//tensorflow/lite:string_util",
         "//tensorflow/lite/kernels:builtin_ops",
         "//third_party/py/numpy:headers",
         "//third_party/python_runtime:headers",
@@ -36,7 +48,9 @@ cc_library(
     srcs = ["python_utils.cc"],
     hdrs = ["python_utils.h"],
     deps = [
+        ":numpy",
         "//tensorflow/lite:framework",
+        "//tensorflow/lite:string_util",
         "//third_party/py/numpy:headers",
         "//third_party/python_runtime:headers",
     ],
diff --git a/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.cc b/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.cc
index 9ccaabbfe9..41cebf867e 100644
--- a/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.cc
+++ b/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.cc
@@ -21,16 +21,10 @@ limitations under the License.
 #include "tensorflow/lite/interpreter.h"
 #include "tensorflow/lite/kernels/register.h"
 #include "tensorflow/lite/model.h"
+#include "tensorflow/lite/python/interpreter_wrapper/numpy.h"
 #include "tensorflow/lite/python/interpreter_wrapper/python_error_reporter.h"
 #include "tensorflow/lite/python/interpreter_wrapper/python_utils.h"
-
-// Disallow Numpy 1.7 deprecated symbols.
-#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
-
-#include <Python.h>
-
-#include "numpy/arrayobject.h"
-#include "numpy/ufuncobject.h"
+#include "tensorflow/lite/string_util.h"
 
 #if PY_MAJOR_VERSION >= 3
 #define PY_TO_CPPSTRING PyBytes_AsStringAndSize
@@ -64,12 +58,6 @@ namespace interpreter_wrapper {
 
 namespace {
 
-// Calls PyArray's initialization to initialize all the API pointers. Note that
-// this usage implies only this translation unit can use the pointers. See
-// tensorflow/python/core/numpy.cc for a strategy if we ever need to extend
-// this further.
-void ImportNumpy() { import_array1(); }
-
 std::unique_ptr<tflite::Interpreter> CreateInterpreter(
     const tflite::FlatBufferModel* model,
     const tflite::ops::builtin::BuiltinOpResolver& resolver) {
@@ -77,7 +65,7 @@ std::unique_ptr<tflite::Interpreter> CreateInterpreter(
     return nullptr;
   }
 
-  ImportNumpy();
+  ::tflite::python::ImportNumpy();
 
   std::unique_ptr<tflite::Interpreter> interpreter;
   if (tflite::InterpreterBuilder(*model, resolver)(&interpreter) != kTfLiteOk) {
@@ -267,7 +255,7 @@ PyObject* InterpreterWrapper::SetTensor(int i, PyObject* value) {
   }
 
   PyArrayObject* array = reinterpret_cast<PyArrayObject*>(array_safe.get());
-  const TfLiteTensor* tensor = interpreter_->tensor(i);
+  TfLiteTensor* tensor = interpreter_->tensor(i);
 
   if (python_utils::TfLiteTypeFromPyArray(array) != tensor->type) {
     PyErr_Format(PyExc_ValueError,
@@ -279,26 +267,41 @@ PyObject* InterpreterWrapper::SetTensor(int i, PyObject* value) {
   }
 
   if (PyArray_NDIM(array) != tensor->dims->size) {
-    PyErr_SetString(PyExc_ValueError, "Cannot set tensor: Dimension mismatch");
+    PyErr_Format(PyExc_ValueError,
+                 "Cannot set tensor: Dimension mismatch."
+                 " Got %d"
+                 " but expected %d for input %d.",
+                 PyArray_NDIM(array), tensor->dims->size, i);
     return nullptr;
   }
 
   for (int j = 0; j < PyArray_NDIM(array); j++) {
     if (tensor->dims->data[j] != PyArray_SHAPE(array)[j]) {
-      PyErr_SetString(PyExc_ValueError,
-                      "Cannot set tensor: Dimension mismatch");
+      PyErr_Format(PyExc_ValueError,
+                   "Cannot set tensor: Dimension mismatch."
+                   " Got %ld"
+                   " but expected %d for dimension %d of input %d.",
+                   PyArray_SHAPE(array)[j], tensor->dims->data[j], j, i);
       return nullptr;
     }
   }
 
-  size_t size = PyArray_NBYTES(array);
-  if (size != tensor->bytes) {
-    PyErr_Format(PyExc_ValueError,
-                 "numpy array had %zu bytes but expected %zu bytes.", size,
-                 tensor->bytes);
-    return nullptr;
+  if (tensor->type != kTfLiteString) {
+    size_t size = PyArray_NBYTES(array);
+    if (size != tensor->bytes) {
+      PyErr_Format(PyExc_ValueError,
+                   "numpy array had %zu bytes but expected %zu bytes.", size,
+                   tensor->bytes);
+      return nullptr;
+    }
+    memcpy(tensor->data.raw, PyArray_DATA(array), size);
+  } else {
+    DynamicBuffer dynamic_buffer;
+    if (!python_utils::FillStringBufferWithPyArray(value, &dynamic_buffer)) {
+      return nullptr;
+    }
+    dynamic_buffer.WriteToTensor(tensor, nullptr);
   }
-  memcpy(tensor->data.raw, PyArray_DATA(array), size);
   Py_RETURN_NONE;
 }
 
@@ -345,19 +348,51 @@ PyObject* InterpreterWrapper::GetTensor(int i) const {
 
   std::vector<npy_intp> dims(tensor->dims->data,
                              tensor->dims->data + tensor->dims->size);
-  // Make a buffer copy but we must tell Numpy It owns that data or else
-  // it will leak.
-  void* data = malloc(tensor->bytes);
-  if (!data) {
-    PyErr_SetString(PyExc_ValueError, "Malloc to copy tensor failed.");
-    return nullptr;
+  if (tensor->type != kTfLiteString) {
+    // Make a buffer copy but we must tell Numpy It owns that data or else
+    // it will leak.
+    void* data = malloc(tensor->bytes);
+    if (!data) {
+      PyErr_SetString(PyExc_ValueError, "Malloc to copy tensor failed.");
+      return nullptr;
+    }
+    memcpy(data, tensor->data.raw, tensor->bytes);
+    PyObject* np_array =
+        PyArray_SimpleNewFromData(dims.size(), dims.data(), type_num, data);
+    PyArray_ENABLEFLAGS(reinterpret_cast<PyArrayObject*>(np_array),
+                        NPY_ARRAY_OWNDATA);
+    return PyArray_Return(reinterpret_cast<PyArrayObject*>(np_array));
+  } else {
+    // Create a C-order array so the data is contiguous in memory.
+    const int32_t kCOrder = 0;
+    PyObject* py_object =
+        PyArray_EMPTY(dims.size(), dims.data(), NPY_OBJECT, kCOrder);
+
+    if (py_object == nullptr) {
+      PyErr_SetString(PyExc_MemoryError, "Failed to allocate PyArray.");
+      return nullptr;
+    }
+
+    PyArrayObject* py_array = reinterpret_cast<PyArrayObject*>(py_object);
+    PyObject** data = reinterpret_cast<PyObject**>(PyArray_DATA(py_array));
+    auto num_strings = GetStringCount(tensor->data.raw);
+    for (int j = 0; j < num_strings; ++j) {
+      auto ref = GetString(tensor->data.raw, j);
+
+      PyObject* bytes = PyBytes_FromStringAndSize(ref.str, ref.len);
+      if (bytes == nullptr) {
+        Py_DECREF(py_object);
+        PyErr_Format(PyExc_ValueError,
+                     "Could not create PyBytes from string %d of input %d.", j,
+                     i);
+        return nullptr;
+      }
+      // PyArray_EMPTY produces an array full of Py_None, which we must decref.
+      Py_DECREF(data[j]);
+      data[j] = bytes;
+    }
+    return py_object;
   }
-  memcpy(data, tensor->data.raw, tensor->bytes);
-  PyObject* np_array =
-      PyArray_SimpleNewFromData(dims.size(), dims.data(), type_num, data);
-  PyArray_ENABLEFLAGS(reinterpret_cast<PyArrayObject*>(np_array),
-                      NPY_ARRAY_OWNDATA);
-  return PyArray_Return(reinterpret_cast<PyArrayObject*>(np_array));
 }
 
 PyObject* InterpreterWrapper::tensor(PyObject* base_object, int i) {
diff --git a/tensorflow/lite/python/interpreter_wrapper/numpy.cc b/tensorflow/lite/python/interpreter_wrapper/numpy.cc
new file mode 100644
index 0000000000..ff5403d2a6
--- /dev/null
+++ b/tensorflow/lite/python/interpreter_wrapper/numpy.cc
@@ -0,0 +1,25 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#define TFLITE_IMPORT_NUMPY  // See numpy.h for explanation.
+#include "tensorflow/lite/python/interpreter_wrapper/numpy.h"
+
+namespace tflite {
+namespace python {
+
+void ImportNumpy() { import_array1(); }
+
+}  // namespace python
+}  // namespace tflite
diff --git a/tensorflow/lite/python/interpreter_wrapper/numpy.h b/tensorflow/lite/python/interpreter_wrapper/numpy.h
new file mode 100644
index 0000000000..a3b013fcb2
--- /dev/null
+++ b/tensorflow/lite/python/interpreter_wrapper/numpy.h
@@ -0,0 +1,62 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_PYTHON_INTERPRETER_WRAPPER_NUMPY_H_
+#define TENSORFLOW_LITE_PYTHON_INTERPRETER_WRAPPER_NUMPY_H_
+
+#ifdef PyArray_Type
+#error "Numpy cannot be included before numpy.h."
+#endif
+
+// Disallow Numpy 1.7 deprecated symbols.
+#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
+
+// To handle PyArray_* calles, numpy defines a static lookup table called
+// PyArray_API, or PY_ARRAY_UNIQUE_SYMBOL, if defined. This causes the
+// PyArray_* pointers to be different for different translation units, unless
+// we take care of selectivel defined NO_IMPORT_ARRAY.
+//
+// Virtually every usage will define NO_IMPORT_ARRAY, and will have access to
+// the lookup table via:
+//   extern void **PyArray_API;
+// In numpy.cc we will define TFLITE_IMPORT_NUMPY, effectively disabling that
+// and instead using:
+//   void **PyArray_API;
+// which is initialized when ImportNumpy() is called.
+//
+// If we don't define PY_ARRAY_UNIQUE_SYMBOL then PyArray_API is a static
+// variable, which causes strange crashes when the pointers are used across
+// translation unit boundaries.
+//
+// For mone info see https://sourceforge.net/p/numpy/mailman/message/5700519
+// See also tensorflow/python/lib/core/numpy.h for a similar approach.
+#define PY_ARRAY_UNIQUE_SYMBOL _tensorflow_numpy_api
+#ifndef TFLITE_IMPORT_NUMPY
+#define NO_IMPORT_ARRAY
+#endif
+
+#include <Python.h>
+
+#include "numpy/arrayobject.h"
+#include "numpy/ufuncobject.h"
+
+namespace tflite {
+namespace python {
+
+void ImportNumpy();
+
+}  // namespace python
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_PYTHON_INTERPRETER_WRAPPER_NUMPY_H_
diff --git a/tensorflow/lite/python/interpreter_wrapper/python_utils.cc b/tensorflow/lite/python/interpreter_wrapper/python_utils.cc
index 2dc604356a..a052ca320f 100644
--- a/tensorflow/lite/python/interpreter_wrapper/python_utils.cc
+++ b/tensorflow/lite/python/interpreter_wrapper/python_utils.cc
@@ -15,9 +15,19 @@ limitations under the License.
 
 #include "tensorflow/lite/python/interpreter_wrapper/python_utils.h"
 
+#include <memory>
+
+#include "tensorflow/lite/python/interpreter_wrapper/numpy.h"
+
 namespace tflite {
 namespace python_utils {
 
+struct PyObjectDereferencer {
+  void operator()(PyObject* py_object) const { Py_DECREF(py_object); }
+};
+
+using UniquePyObjectRef = std::unique_ptr<PyObject, PyObjectDereferencer>;
+
 int TfLiteTypeToPyArrayType(TfLiteType tf_lite_type) {
   switch (tf_lite_type) {
     case kTfLiteFloat32:
@@ -33,7 +43,7 @@ int TfLiteTypeToPyArrayType(TfLiteType tf_lite_type) {
     case kTfLiteInt64:
       return NPY_INT64;
     case kTfLiteString:
-      return NPY_OBJECT;
+      return NPY_STRING;
     case kTfLiteBool:
       return NPY_BOOL;
     case kTfLiteComplex64:
@@ -73,5 +83,82 @@ TfLiteType TfLiteTypeFromPyArray(PyArrayObject* array) {
   return kTfLiteNoType;
 }
 
+#if PY_VERSION_HEX >= 0x03030000
+bool FillStringBufferFromPyUnicode(PyObject* value,
+                                   DynamicBuffer* dynamic_buffer) {
+  Py_ssize_t len = -1;
+  char* buf = PyUnicode_AsUTF8AndSize(value, &len);
+  if (buf == NULL) {
+    PyErr_SetString(PyExc_ValueError, "PyUnicode_AsUTF8AndSize() failed.");
+    return false;
+  }
+  dynamic_buffer->AddString(buf, len);
+  return true;
+}
+#else
+bool FillStringBufferFromPyUnicode(PyObject* value,
+                                   DynamicBuffer* dynamic_buffer) {
+  UniquePyObjectRef utemp(PyUnicode_AsUTF8String(value));
+  if (!utemp) {
+    PyErr_SetString(PyExc_ValueError, "PyUnicode_AsUTF8String() failed.");
+    return false;
+  }
+  char* buf = nullptr;
+  Py_ssize_t len = -1;
+  if (PyBytes_AsStringAndSize(utemp.get(), &buf, &len) == -1) {
+    PyErr_SetString(PyExc_ValueError, "PyBytes_AsStringAndSize() failed.");
+    return false;
+  }
+  dynamic_buffer->AddString(buf, len);
+  return true;
+}
+#endif
+
+bool FillStringBufferFromPyString(PyObject* value,
+                                  DynamicBuffer* dynamic_buffer) {
+  if (PyUnicode_Check(value)) {
+    return FillStringBufferFromPyUnicode(value, dynamic_buffer);
+  }
+
+  char* buf = nullptr;
+  Py_ssize_t len = -1;
+  if (PyBytes_AsStringAndSize(value, &buf, &len) == -1) {
+    PyErr_SetString(PyExc_ValueError, "PyBytes_AsStringAndSize() failed.");
+    return false;
+  }
+  dynamic_buffer->AddString(buf, len);
+  return true;
+}
+
+bool FillStringBufferWithPyArray(PyObject* value,
+                                 DynamicBuffer* dynamic_buffer) {
+  PyArrayObject* array = reinterpret_cast<PyArrayObject*>(value);
+  switch (PyArray_TYPE(array)) {
+    case NPY_OBJECT:
+    case NPY_STRING:
+    case NPY_UNICODE: {
+      UniquePyObjectRef iter(PyArray_IterNew(value));
+      while (PyArray_ITER_NOTDONE(iter.get())) {
+        UniquePyObjectRef item(PyArray_GETITEM(
+            array, reinterpret_cast<char*>(PyArray_ITER_DATA(iter.get()))));
+
+        if (!FillStringBufferFromPyString(item.get(), dynamic_buffer)) {
+          return false;
+        }
+
+        PyArray_ITER_NEXT(iter.get());
+      }
+      return true;
+    }
+    default:
+      break;
+  }
+
+  PyErr_Format(PyExc_ValueError,
+               "Cannot use numpy array of type %d for string tensor.",
+               PyArray_TYPE(array));
+  return false;
+}
+
 }  // namespace python_utils
 }  // namespace tflite
diff --git a/tensorflow/lite/python/interpreter_wrapper/python_utils.h b/tensorflow/lite/python/interpreter_wrapper/python_utils.h
index 30a44226b8..5ffd231a89 100644
--- a/tensorflow/lite/python/interpreter_wrapper/python_utils.h
+++ b/tensorflow/lite/python/interpreter_wrapper/python_utils.h
@@ -17,14 +17,8 @@ limitations under the License.
 #define TENSORFLOW_LITE_PYTHON_INTERPRETER_WRAPPER_PYTHON_UTILS_H_
 
 #include "tensorflow/lite/context.h"
-
-// Disallow Numpy 1.7 deprecated symbols.
-#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
-
-#include <Python.h>
-
-#include "numpy/arrayobject.h"
-#include "numpy/ufuncobject.h"
+#include "tensorflow/lite/python/interpreter_wrapper/numpy.h"
+#include "tensorflow/lite/string_util.h"
 
 namespace tflite {
 namespace python_utils {
@@ -33,6 +27,9 @@ int TfLiteTypeToPyArrayType(TfLiteType tf_lite_type);
 
 TfLiteType TfLiteTypeFromPyArray(PyArrayObject* array);
 
+bool FillStringBufferWithPyArray(PyObject* value,
+                                 DynamicBuffer* dynamic_buffer);
+
 }  // namespace python_utils
 }  // namespace tflite
 #endif  // TENSORFLOW_LITE_PYTHON_INTERPRETER_WRAPPER_PYTHON_UTILS_H_
diff --git a/tensorflow/lite/python/lite_test.py b/tensorflow/lite/python/lite_test.py
index ff91d5e297..ca6c5b8f13 100644
--- a/tensorflow/lite/python/lite_test.py
+++ b/tensorflow/lite/python/lite_test.py
@@ -131,13 +131,13 @@ class FromSessionTest(test_util.TensorFlowTestCase):
     input_details = interpreter.get_input_details()
     self.assertEqual(1, len(input_details))
     self.assertEqual('Placeholder', input_details[0]['name'])
-    self.assertEqual(np.object_, input_details[0]['dtype'])
+    self.assertEqual(np.string_, input_details[0]['dtype'])
     self.assertTrue(([4] == input_details[0]['shape']).all())
 
     output_details = interpreter.get_output_details()
     self.assertEqual(1, len(output_details))
     self.assertEqual('Reshape', output_details[0]['name'])
-    self.assertEqual(np.object_, output_details[0]['dtype'])
+    self.assertEqual(np.string_, output_details[0]['dtype'])
     self.assertTrue(([2, 2] == output_details[0]['shape']).all())
     # TODO(b/122659643): Test setting/getting string data via the python
     # interpreter API after support has been added.
diff --git a/tensorflow/lite/python/optimize/BUILD b/tensorflow/lite/python/optimize/BUILD
index 74b573b81f..8694ebf1f5 100644
--- a/tensorflow/lite/python/optimize/BUILD
+++ b/tensorflow/lite/python/optimize/BUILD
@@ -13,6 +13,7 @@ cc_library(
     deps = [
         "//tensorflow/lite:framework",
         "//tensorflow/lite/kernels:builtin_ops",
+        "//tensorflow/lite/python/interpreter_wrapper:numpy",
         "//tensorflow/lite/python/interpreter_wrapper:python_error_reporter",
         "//tensorflow/lite/python/interpreter_wrapper:python_utils",
         "//tensorflow/lite/tools/optimize:calibration_reader",
diff --git a/tensorflow/lite/python/optimize/calibration_wrapper.cc b/tensorflow/lite/python/optimize/calibration_wrapper.cc
index d6fe1fbdad..21f96f848c 100644
--- a/tensorflow/lite/python/optimize/calibration_wrapper.cc
+++ b/tensorflow/lite/python/optimize/calibration_wrapper.cc
@@ -22,20 +22,13 @@ limitations under the License.
 #include "tensorflow/lite/interpreter.h"
 #include "tensorflow/lite/kernels/register.h"
 #include "tensorflow/lite/model.h"
+#include "tensorflow/lite/python/interpreter_wrapper/numpy.h"
 #include "tensorflow/lite/python/interpreter_wrapper/python_error_reporter.h"
 #include "tensorflow/lite/python/interpreter_wrapper/python_utils.h"
 #include "tensorflow/lite/tools/optimize/calibration_reader.h"
 #include "tensorflow/lite/tools/optimize/calibrator.h"
 #include "tensorflow/lite/tools/optimize/quantize_model.h"
 
-// Disallow Numpy 1.7 deprecated symbols.
-#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
-
-#include <Python.h>
-
-#include "numpy/arrayobject.h"
-#include "numpy/ufuncobject.h"
-
 #if PY_MAJOR_VERSION >= 3
 #define PY_TO_CPPSTRING PyBytes_AsStringAndSize
 #define CPP_TO_PYSTRING PyBytes_FromStringAndSize
diff --git a/tensorflow/lite/python/testdata/BUILD b/tensorflow/lite/python/testdata/BUILD
index 4689c318b3..2fa08e5326 100644
--- a/tensorflow/lite/python/testdata/BUILD
+++ b/tensorflow/lite/python/testdata/BUILD
@@ -32,9 +32,20 @@ tf_to_tflite(
     ],
 )
 
+tf_to_tflite(
+    name = "gather_string",
+    src = "gather.pbtxt",
+    out = "gather_string.tflite",
+    options = [
+        "--input_arrays=input,indices",
+        "--output_arrays=output",
+    ],
+)
+
 filegroup(
     name = "interpreter_test_data",
     srcs = [
+        ":gather_string",
         ":permute_float",
         ":permute_uint8",
     ],
diff --git a/tensorflow/lite/python/testdata/gather.pbtxt b/tensorflow/lite/python/testdata/gather.pbtxt
new file mode 100644
index 0000000000..0b1193c475
--- /dev/null
+++ b/tensorflow/lite/python/testdata/gather.pbtxt
@@ -0,0 +1,93 @@
+node {
+  name: "input"
+  op: "Placeholder"
+  device: "/device:CPU:0"
+  attr {
+    key: "dtype"
+    value {
+      type: DT_STRING
+    }
+  }
+  attr {
+    key: "shape"
+    value {
+      shape {
+        dim {
+          size: 10
+        }
+      }
+    }
+  }
+}
+node {
+  name: "indices"
+  op: "Placeholder"
+  device: "/device:CPU:0"
+  attr {
+    key: "dtype"
+    value {
+      type: DT_INT64
+    }
+  }
+  attr {
+    key: "shape"
+    value {
+      shape {
+        dim {
+          size: 3
+        }
+      }
+    }
+  }
+}
+node {
+  name: "axis"
+  op: "Const"
+  device: "/device:CPU:0"
+  attr {
+    key: "dtype"
+    value {
+      type: DT_INT32
+    }
+  }
+  attr {
+    key: "value"
+    value {
+      tensor {
+        dtype: DT_INT32
+        tensor_shape {
+        }
+        int_val: 0
+      }
+    }
+  }
+}
+node {
+  name: "output"
+  op: "GatherV2"
+  input: "input"
+  input: "indices"
+  input: "axis"
+  device: "/device:CPU:0"
+  attr {
+    key: "Taxis"
+    value {
+      type: DT_INT32
+    }
+  }
+  attr {
+    key: "Tindices"
+    value {
+      type: DT_INT64
+    }
+  }
+  attr {
+    key: "Tparams"
+    value {
+      type: DT_STRING
+    }
+  }
+}
+versions {
+  producer: 27
+}
-- 
GitLab


From 0c4f5dfea4ceb3d7c0b46fc04828420a344f7598 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 6 Feb 2019 12:56:47 -0800
Subject: [PATCH 2017/2345] Add new op schema for TFlite equivalent of
 tf.gather_nd.

PiperOrigin-RevId: 232728420
---
 tensorflow/lite/builtin_ops.h                 |   1 +
 .../lite/core/api/flatbuffer_conversions.cc   |   1 +
 tensorflow/lite/nnapi_delegate.cc             |   1 +
 tensorflow/lite/schema/schema.fbs             |   5 +
 tensorflow/lite/schema/schema_generated.h     | 124 +++++++++++++++++-
 5 files changed, 126 insertions(+), 6 deletions(-)

diff --git a/tensorflow/lite/builtin_ops.h b/tensorflow/lite/builtin_ops.h
index fc871d3592..1915565f4b 100644
--- a/tensorflow/lite/builtin_ops.h
+++ b/tensorflow/lite/builtin_ops.h
@@ -132,6 +132,7 @@ typedef enum {
   kTfLiteBuiltinCeil = 104,
   kTfLiteBuiltinReverseV2 = 105,
   kTfLiteBuiltinAddN = 106,
+  kTfLiteBuiltinGatherNd = 107,
 } TfLiteBuiltinOperator;
 
 #ifdef __cplusplus
diff --git a/tensorflow/lite/core/api/flatbuffer_conversions.cc b/tensorflow/lite/core/api/flatbuffer_conversions.cc
index 175caa1afa..72667d4260 100644
--- a/tensorflow/lite/core/api/flatbuffer_conversions.cc
+++ b/tensorflow/lite/core/api/flatbuffer_conversions.cc
@@ -728,6 +728,7 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type,
     case BuiltinOperator_SQUARED_DIFFERENCE:
     case BuiltinOperator_REVERSE_V2:
     case BuiltinOperator_ADD_N:
+    case BuiltinOperator_GATHER_ND:
       break;
   }
   return kTfLiteOk;
diff --git a/tensorflow/lite/nnapi_delegate.cc b/tensorflow/lite/nnapi_delegate.cc
index 7c67468879..f69baf1ac8 100644
--- a/tensorflow/lite/nnapi_delegate.cc
+++ b/tensorflow/lite/nnapi_delegate.cc
@@ -665,6 +665,7 @@ TfLiteStatus AddOpsAndParams(
       case tflite::BuiltinOperator_CEIL:
       case tflite::BuiltinOperator_REVERSE_V2:
       case tflite::BuiltinOperator_ADD_N:
+      case tflite::BuiltinOperator_GATHER_ND:
         logError("Op code %d is currently not delegated to NNAPI", builtin);
         return kTfLiteError;
         break;
diff --git a/tensorflow/lite/schema/schema.fbs b/tensorflow/lite/schema/schema.fbs
index 098b7c7193..b9c6e988f9 100644
--- a/tensorflow/lite/schema/schema.fbs
+++ b/tensorflow/lite/schema/schema.fbs
@@ -220,6 +220,7 @@ enum BuiltinOperator : byte {
   CEIL = 104,
   REVERSE_V2 = 105,
   ADD_N = 106,
+  GATHER_ND = 107,
 }
 
 // Options for the builtin operators.
@@ -306,6 +307,7 @@ union BuiltinOptions {
   UniqueOptions,
   ReverseV2Options,
   AddNOptions,
+  GatherNdOptions,
 }
 
 enum Padding : byte { SAME, VALID }
@@ -729,6 +731,9 @@ table ReverseV2Options {
 table AddNOptions {
 }
 
+table GatherNdOptions {
+}
+
 // An OperatorCode can be an enum value (BuiltinOperator) if the operator is a
 // builtin, or a string if the operator is custom.
 table OperatorCode {
diff --git a/tensorflow/lite/schema/schema_generated.h b/tensorflow/lite/schema/schema_generated.h
index f0a6b00f96..3117789529 100755
--- a/tensorflow/lite/schema/schema_generated.h
+++ b/tensorflow/lite/schema/schema_generated.h
@@ -277,6 +277,9 @@ struct ReverseV2OptionsT;
 struct AddNOptions;
 struct AddNOptionsT;
 
+struct GatherNdOptions;
+struct GatherNdOptionsT;
+
 struct OperatorCode;
 struct OperatorCodeT;
 
@@ -533,11 +536,12 @@ enum BuiltinOperator {
   BuiltinOperator_CEIL = 104,
   BuiltinOperator_REVERSE_V2 = 105,
   BuiltinOperator_ADD_N = 106,
+  BuiltinOperator_GATHER_ND = 107,
   BuiltinOperator_MIN = BuiltinOperator_ADD,
-  BuiltinOperator_MAX = BuiltinOperator_ADD_N
+  BuiltinOperator_MAX = BuiltinOperator_GATHER_ND
 };
 
-inline const BuiltinOperator (&EnumValuesBuiltinOperator())[106] {
+inline const BuiltinOperator (&EnumValuesBuiltinOperator())[107] {
   static const BuiltinOperator values[] = {
     BuiltinOperator_ADD,
     BuiltinOperator_AVERAGE_POOL_2D,
@@ -644,7 +648,8 @@ inline const BuiltinOperator (&EnumValuesBuiltinOperator())[106] {
     BuiltinOperator_UNIQUE,
     BuiltinOperator_CEIL,
     BuiltinOperator_REVERSE_V2,
-    BuiltinOperator_ADD_N
+    BuiltinOperator_ADD_N,
+    BuiltinOperator_GATHER_ND
   };
   return values;
 }
@@ -758,6 +763,7 @@ inline const char * const *EnumNamesBuiltinOperator() {
     "CEIL",
     "REVERSE_V2",
     "ADD_N",
+    "GATHER_ND",
     nullptr
   };
   return names;
@@ -852,11 +858,12 @@ enum BuiltinOptions {
   BuiltinOptions_UniqueOptions = 80,
   BuiltinOptions_ReverseV2Options = 81,
   BuiltinOptions_AddNOptions = 82,
+  BuiltinOptions_GatherNdOptions = 83,
   BuiltinOptions_MIN = BuiltinOptions_NONE,
-  BuiltinOptions_MAX = BuiltinOptions_AddNOptions
+  BuiltinOptions_MAX = BuiltinOptions_GatherNdOptions
 };
 
-inline const BuiltinOptions (&EnumValuesBuiltinOptions())[83] {
+inline const BuiltinOptions (&EnumValuesBuiltinOptions())[84] {
   static const BuiltinOptions values[] = {
     BuiltinOptions_NONE,
     BuiltinOptions_Conv2DOptions,
@@ -940,7 +947,8 @@ inline const BuiltinOptions (&EnumValuesBuiltinOptions())[83] {
     BuiltinOptions_SplitVOptions,
     BuiltinOptions_UniqueOptions,
     BuiltinOptions_ReverseV2Options,
-    BuiltinOptions_AddNOptions
+    BuiltinOptions_AddNOptions,
+    BuiltinOptions_GatherNdOptions
   };
   return values;
 }
@@ -1030,6 +1038,7 @@ inline const char * const *EnumNamesBuiltinOptions() {
     "UniqueOptions",
     "ReverseV2Options",
     "AddNOptions",
+    "GatherNdOptions",
     nullptr
   };
   return names;
@@ -1372,6 +1381,10 @@ template<> struct BuiltinOptionsTraits<AddNOptions> {
   static const BuiltinOptions enum_value = BuiltinOptions_AddNOptions;
 };
 
+template<> struct BuiltinOptionsTraits<GatherNdOptions> {
+  static const BuiltinOptions enum_value = BuiltinOptions_GatherNdOptions;
+};
+
 struct BuiltinOptionsUnion {
   BuiltinOptions type;
   void *value;
@@ -2059,6 +2072,14 @@ struct BuiltinOptionsUnion {
     return type == BuiltinOptions_AddNOptions ?
       reinterpret_cast<const AddNOptionsT *>(value) : nullptr;
   }
+  GatherNdOptionsT *AsGatherNdOptions() {
+    return type == BuiltinOptions_GatherNdOptions ?
+      reinterpret_cast<GatherNdOptionsT *>(value) : nullptr;
+  }
+  const GatherNdOptionsT *AsGatherNdOptions() const {
+    return type == BuiltinOptions_GatherNdOptions ?
+      reinterpret_cast<const GatherNdOptionsT *>(value) : nullptr;
+  }
 };
 
 bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *obj, BuiltinOptions type);
@@ -7235,6 +7256,46 @@ inline flatbuffers::Offset<AddNOptions> CreateAddNOptions(
 
 flatbuffers::Offset<AddNOptions> CreateAddNOptions(flatbuffers::FlatBufferBuilder &_fbb, const AddNOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
 
+struct GatherNdOptionsT : public flatbuffers::NativeTable {
+  typedef GatherNdOptions TableType;
+  GatherNdOptionsT() {
+  }
+};
+
+struct GatherNdOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef GatherNdOptionsT NativeTableType;
+  bool Verify(flatbuffers::Verifier &verifier) const {
+    return VerifyTableStart(verifier) &&
+           verifier.EndTable();
+  }
+  GatherNdOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const;
+  void UnPackTo(GatherNdOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const;
+  static flatbuffers::Offset<GatherNdOptions> Pack(flatbuffers::FlatBufferBuilder &_fbb, const GatherNdOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
+};
+
+struct GatherNdOptionsBuilder {
+  flatbuffers::FlatBufferBuilder &fbb_;
+  flatbuffers::uoffset_t start_;
+  explicit GatherNdOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb)
+        : fbb_(_fbb) {
+    start_ = fbb_.StartTable();
+  }
+  GatherNdOptionsBuilder &operator=(const GatherNdOptionsBuilder &);
+  flatbuffers::Offset<GatherNdOptions> Finish() {
+    const auto end = fbb_.EndTable(start_);
+    auto o = flatbuffers::Offset<GatherNdOptions>(end);
+    return o;
+  }
+};
+
+inline flatbuffers::Offset<GatherNdOptions> CreateGatherNdOptions(
+    flatbuffers::FlatBufferBuilder &_fbb) {
+  GatherNdOptionsBuilder builder_(_fbb);
+  return builder_.Finish();
+}
+
+flatbuffers::Offset<GatherNdOptions> CreateGatherNdOptions(flatbuffers::FlatBufferBuilder &_fbb, const GatherNdOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
+
 struct OperatorCodeT : public flatbuffers::NativeTable {
   typedef OperatorCode TableType;
   BuiltinOperator builtin_code;
@@ -7614,6 +7675,9 @@ struct Operator FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
   const AddNOptions *builtin_options_as_AddNOptions() const {
     return builtin_options_type() == BuiltinOptions_AddNOptions ? static_cast<const AddNOptions *>(builtin_options()) : nullptr;
   }
+  const GatherNdOptions *builtin_options_as_GatherNdOptions() const {
+    return builtin_options_type() == BuiltinOptions_GatherNdOptions ? static_cast<const GatherNdOptions *>(builtin_options()) : nullptr;
+  }
   const flatbuffers::Vector<uint8_t> *custom_options() const {
     return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_CUSTOM_OPTIONS);
   }
@@ -7973,6 +8037,10 @@ template<> inline const AddNOptions *Operator::builtin_options_as<AddNOptions>()
   return builtin_options_as_AddNOptions();
 }
 
+template<> inline const GatherNdOptions *Operator::builtin_options_as<GatherNdOptions>() const {
+  return builtin_options_as_GatherNdOptions();
+}
+
 struct OperatorBuilder {
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
@@ -10666,6 +10734,29 @@ inline flatbuffers::Offset<AddNOptions> CreateAddNOptions(flatbuffers::FlatBuffe
       _fbb);
 }
 
+inline GatherNdOptionsT *GatherNdOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
+  auto _o = new GatherNdOptionsT();
+  UnPackTo(_o, _resolver);
+  return _o;
+}
+
+inline void GatherNdOptions::UnPackTo(GatherNdOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const {
+  (void)_o;
+  (void)_resolver;
+}
+
+inline flatbuffers::Offset<GatherNdOptions> GatherNdOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const GatherNdOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) {
+  return CreateGatherNdOptions(_fbb, _o, _rehasher);
+}
+
+inline flatbuffers::Offset<GatherNdOptions> CreateGatherNdOptions(flatbuffers::FlatBufferBuilder &_fbb, const GatherNdOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) {
+  (void)_rehasher;
+  (void)_o;
+  struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const GatherNdOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va;
+  return tflite::CreateGatherNdOptions(
+      _fbb);
+}
+
 inline OperatorCodeT *OperatorCode::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
   auto _o = new OperatorCodeT();
   UnPackTo(_o, _resolver);
@@ -11252,6 +11343,10 @@ inline bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *ob
       auto ptr = reinterpret_cast<const AddNOptions *>(obj);
       return verifier.VerifyTable(ptr);
     }
+    case BuiltinOptions_GatherNdOptions: {
+      auto ptr = reinterpret_cast<const GatherNdOptions *>(obj);
+      return verifier.VerifyTable(ptr);
+    }
     default: return false;
   }
 }
@@ -11598,6 +11693,10 @@ inline void *BuiltinOptionsUnion::UnPack(const void *obj, BuiltinOptions type, c
       auto ptr = reinterpret_cast<const AddNOptions *>(obj);
       return ptr->UnPack(resolver);
     }
+    case BuiltinOptions_GatherNdOptions: {
+      auto ptr = reinterpret_cast<const GatherNdOptions *>(obj);
+      return ptr->UnPack(resolver);
+    }
     default: return nullptr;
   }
 }
@@ -11932,6 +12031,10 @@ inline flatbuffers::Offset<void> BuiltinOptionsUnion::Pack(flatbuffers::FlatBuff
       auto ptr = reinterpret_cast<const AddNOptionsT *>(value);
       return CreateAddNOptions(_fbb, ptr, _rehasher).Union();
     }
+    case BuiltinOptions_GatherNdOptions: {
+      auto ptr = reinterpret_cast<const GatherNdOptionsT *>(value);
+      return CreateGatherNdOptions(_fbb, ptr, _rehasher).Union();
+    }
     default: return 0;
   }
 }
@@ -12266,6 +12369,10 @@ inline BuiltinOptionsUnion::BuiltinOptionsUnion(const BuiltinOptionsUnion &u) FL
       value = new AddNOptionsT(*reinterpret_cast<AddNOptionsT *>(u.value));
       break;
     }
+    case BuiltinOptions_GatherNdOptions: {
+      value = new GatherNdOptionsT(*reinterpret_cast<GatherNdOptionsT *>(u.value));
+      break;
+    }
     default:
       break;
   }
@@ -12683,6 +12790,11 @@ inline void BuiltinOptionsUnion::Reset() {
       delete ptr;
       break;
     }
+    case BuiltinOptions_GatherNdOptions: {
+      auto ptr = reinterpret_cast<GatherNdOptionsT *>(value);
+      delete ptr;
+      break;
+    }
     default: break;
   }
   value = nullptr;
-- 
GitLab


From f11085ebf2f86546ae9418a8ce4078318799720b Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Wed, 6 Feb 2019 12:57:53 -0800
Subject: [PATCH 2018/2345] Add Int8 support for arg_min_max.

PiperOrigin-RevId: 232728637
---
 tensorflow/lite/kernels/arg_min_max.cc      |  9 ++++++++-
 tensorflow/lite/kernels/arg_min_max_test.cc | 22 +++++++++++++++++++++
 tensorflow/lite/kernels/register.cc         |  8 ++++++--
 tensorflow/lite/toco/tflite/operator.cc     | 12 +++++++++++
 4 files changed, 48 insertions(+), 3 deletions(-)

diff --git a/tensorflow/lite/kernels/arg_min_max.cc b/tensorflow/lite/kernels/arg_min_max.cc
index f9adf6bc4e..e5223badc4 100644
--- a/tensorflow/lite/kernels/arg_min_max.cc
+++ b/tensorflow/lite/kernels/arg_min_max.cc
@@ -80,13 +80,14 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   switch (input->type) {
     case kTfLiteFloat32:
     case kTfLiteUInt8:
+    case kTfLiteInt8:
     case kTfLiteInt32:
       break;
 
     default:
       context->ReportError(
           context,
-          "Unkonwn input type: %d, only float32 and int types are supported",
+          "Unknown input type: %d, only float32 and int types are supported",
           input->type);
       return kTfLiteError;
   }
@@ -135,6 +136,9 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node, bool is_arg_max) {
           case kTfLiteUInt8:
             TF_LITE_ARG_MIN_MAX(uint8_t, int32_t, int32_t);
             break;
+          case kTfLiteInt8:
+            TF_LITE_ARG_MIN_MAX(int8_t, int32_t, int32_t);
+            break;
           case kTfLiteInt32:
             TF_LITE_ARG_MIN_MAX(int32_t, int32_t, int32_t);
             break;
@@ -150,6 +154,9 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node, bool is_arg_max) {
           case kTfLiteUInt8:
             TF_LITE_ARG_MIN_MAX(uint8_t, int32_t, int64_t);
             break;
+          case kTfLiteInt8:
+            TF_LITE_ARG_MIN_MAX(int8_t, int32_t, int64_t);
+            break;
           case kTfLiteInt32:
             TF_LITE_ARG_MIN_MAX(int32_t, int32_t, int64_t);
             break;
diff --git a/tensorflow/lite/kernels/arg_min_max_test.cc b/tensorflow/lite/kernels/arg_min_max_test.cc
index 1b1000f29e..01ea923f26 100644
--- a/tensorflow/lite/kernels/arg_min_max_test.cc
+++ b/tensorflow/lite/kernels/arg_min_max_test.cc
@@ -86,6 +86,28 @@ TEST(ArgMaxOpTest, GetMaxArgFloat) {
   EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 1}));
 }
 
+TEST(ArgMaxOpTest, GetMaxArgUInt8) {
+  ArgMaxOpModel<int32_t> model({1, 1, 1, 4}, TensorType_UINT8, TensorType_INT32,
+                               TensorType_INT32);
+  model.PopulateTensor<uint8_t>(model.input(), {1, 9, 7, 3});
+  model.PopulateTensor<int>(model.axis(), {3});
+  model.Invoke();
+
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({1}));
+  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 1}));
+}
+
+TEST(ArgMaxOpTest, GetMaxArgInt8) {
+  ArgMaxOpModel<int32_t> model({1, 1, 1, 4}, TensorType_INT8, TensorType_INT32,
+                               TensorType_INT32);
+  model.PopulateTensor<int8_t>(model.input(), {-1, -9, 7, 3});
+  model.PopulateTensor<int>(model.axis(), {3});
+  model.Invoke();
+
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({2}));
+  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 1}));
+}
+
 TEST(ArgMaxOpTest, GetMaxArgInt) {
   ArgMaxOpModel<int32_t> model({1, 1, 1, 4}, TensorType_INT32, TensorType_INT32,
                                TensorType_INT32);
diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index bce406c1bc..948bf779d6 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -252,8 +252,12 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_PRELU, Register_PRELU());
   AddBuiltin(BuiltinOperator_MAXIMUM, Register_MAXIMUM());
   AddBuiltin(BuiltinOperator_MINIMUM, Register_MINIMUM());
-  AddBuiltin(BuiltinOperator_ARG_MAX, Register_ARG_MAX());
-  AddBuiltin(BuiltinOperator_ARG_MIN, Register_ARG_MIN());
+  AddBuiltin(BuiltinOperator_ARG_MAX, Register_ARG_MAX(),
+             /* min_version */ 1,
+             /* max_version */ 2);
+  AddBuiltin(BuiltinOperator_ARG_MIN, Register_ARG_MIN(),
+             /* min_version */ 1,
+             /* max_version */ 2);
   AddBuiltin(BuiltinOperator_GREATER, Register_GREATER());
   AddBuiltin(BuiltinOperator_GREATER_EQUAL, Register_GREATER_EQUAL());
   AddBuiltin(BuiltinOperator_LESS, Register_LESS());
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index d45ecf3139..58c4a8589b 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -1354,6 +1354,12 @@ class ArgMax : public BuiltinOperator<ArgMaxOperator, ::tflite::ArgMaxOptions,
   }
 
   int GetVersion(const OperatorSignature& op_signature) const override {
+    const string& input_name = op_signature.op->inputs[0];
+    const Array& input_array = op_signature.model->GetArray(input_name);
+    if (input_array.data_type == ArrayDataType::kInt8) {
+      return 2;
+    }
+
     return 1;
   }
 };
@@ -1375,6 +1381,12 @@ class ArgMin : public BuiltinOperator<ArgMinOperator, ::tflite::ArgMinOptions,
   }
 
   int GetVersion(const OperatorSignature& op_signature) const override {
+    const string& input_name = op_signature.op->inputs[0];
+    const Array& input_array = op_signature.model->GetArray(input_name);
+    if (input_array.data_type == ArrayDataType::kInt8) {
+      return 2;
+    }
+
     return 1;
   }
 };
-- 
GitLab


From ec7958da20ab7259b133d4e3c76d170fdb9f699b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 6 Feb 2019 13:05:13 -0800
Subject: [PATCH 2019/2345] Fewer TOCO crashes.

PiperOrigin-RevId: 232730162
---
 .../graph_transformations.cc                  | 17 +++--
 .../graph_transformations.h                   | 12 +++-
 .../toco/graph_transformations/quantize.cc    | 12 ++--
 tensorflow/lite/toco/toco_convert.cc          |  2 +-
 tensorflow/lite/toco/toco_tooling.cc          | 64 ++++++++++---------
 tensorflow/lite/toco/toco_tooling.h           |  7 +-
 6 files changed, 70 insertions(+), 44 deletions(-)

diff --git a/tensorflow/lite/toco/graph_transformations/graph_transformations.cc b/tensorflow/lite/toco/graph_transformations/graph_transformations.cc
index a0260e2401..e4eb769859 100644
--- a/tensorflow/lite/toco/graph_transformations/graph_transformations.cc
+++ b/tensorflow/lite/toco/graph_transformations/graph_transformations.cc
@@ -128,7 +128,8 @@ void DiscardUselessConnectedComponentsAndRNNBackEdges(Model* model) {
 }
 
 bool GraphTransformationsPass(int increment, Model* model,
-                              const GraphTransformationsSet& transformations) {
+                              const GraphTransformationsSet& transformations,
+                              tensorflow::Status* status) {
   CHECK(increment == 1 || increment == -1);
   bool changed = false;
   if (model->operators.empty()) {
@@ -142,7 +143,10 @@ bool GraphTransformationsPass(int increment, Model* model,
     for (const auto& transformation : transformations) {
       CHECK(!changed_now);
       CHECK(transformation->Messages().empty());
-      CHECK(transformation->Run(model, op_index, &changed_now).ok());
+      *status = transformation->Run(model, op_index, &changed_now);
+      if (!status->ok()) {
+        return false;
+      }
       const char* made_a_change_msg =
           changed_now ? "made a change" : "did NOT make a change";
       const int log_level =
@@ -186,18 +190,21 @@ bool GraphTransformationsPass(int increment, Model* model,
 
 }  // namespace
 
-void RunGraphTransformations(Model* model, const string& msg,
-                             const GraphTransformationsSet& transformations) {
+tensorflow::Status RunGraphTransformationsWithStatus(
+    Model* model, const string& msg,
+    const GraphTransformationsSet& transformations) {
   PrintModelStats(toco::port::StringF("Before %s", msg), *model);
   int pass_index = 0;
+  tensorflow::Status status;
   while (GraphTransformationsPass((pass_index % 2) ? -1 : 1, model,
-                                  transformations)) {
+                                  transformations, &status)) {
     pass_index++;
     const auto& label =
         toco::port::StringF("After %s pass %d", msg, pass_index);
     PrintModelStats(label, *model);
     CheckInvariants(*model);
   }
+  return status;
 }
 
 }  // namespace toco
diff --git a/tensorflow/lite/toco/graph_transformations/graph_transformations.h b/tensorflow/lite/toco/graph_transformations/graph_transformations.h
index 4008bbdb4d..491a3e7cfb 100644
--- a/tensorflow/lite/toco/graph_transformations/graph_transformations.h
+++ b/tensorflow/lite/toco/graph_transformations/graph_transformations.h
@@ -102,8 +102,16 @@ class GraphTransformationsSet {
 // construct GraphTransformation objects by using 'new', pass us
 // the resulting raw pointers, and this RunGraphTransformations
 // takes care of delete'ing these pointers.
-void RunGraphTransformations(Model* model, const string& message,
-                             const GraphTransformationsSet& transformations);
+tensorflow::Status RunGraphTransformationsWithStatus(
+    Model* model, const string& msg,
+    const GraphTransformationsSet& transformations);
+
+inline void RunGraphTransformations(
+    Model* model, const string& msg,
+    const GraphTransformationsSet& transformations) {
+  auto s = RunGraphTransformationsWithStatus(model, msg, transformations);
+  CHECK(s.ok()) << s.error_message();
+}
 
 #define DECLARE_GRAPH_TRANSFORMATION(GTName)                     \
   class GTName : public GraphTransformation {                    \
diff --git a/tensorflow/lite/toco/graph_transformations/quantize.cc b/tensorflow/lite/toco/graph_transformations/quantize.cc
index ee65f92e00..5a5c9bbd61 100644
--- a/tensorflow/lite/toco/graph_transformations/quantize.cc
+++ b/tensorflow/lite/toco/graph_transformations/quantize.cc
@@ -489,12 +489,12 @@ void FixMinMaxPostQuantization(GraphTransformation* transformation,
     }
   }
   if (!SupportsQuantization(op)) {
-    LOG(FATAL) << "Unimplemented: this graph contains an operator of type "
-               << HelpfulOperatorTypeName(op)
-               << " for which the quantized form is not yet implemented. "
-                  "Sorry, and patches welcome (that's a relatively fun patch "
-                  "to write, mostly providing the actual quantized arithmetic "
-                  "code for this op).";
+    return tensorflow::errors::InvalidArgument(
+        "Unimplemented: this graph contains an operator of type ",
+        HelpfulOperatorTypeName(op),
+        " for which the quantized form is not yet implemented. Sorry, and "
+        "patches welcome (that's a relatively fun patch to write, mostly "
+        "providing the actual quantized arithmetic code for this op).");
   }
 
   for (const auto& input : op.inputs) {
diff --git a/tensorflow/lite/toco/toco_convert.cc b/tensorflow/lite/toco/toco_convert.cc
index 28e7b10ecd..2adfc1dd23 100644
--- a/tensorflow/lite/toco/toco_convert.cc
+++ b/tensorflow/lite/toco/toco_convert.cc
@@ -77,7 +77,7 @@ tensorflow::Status Convert(const string& graph_def_contents,
                            string* output_file_contents) {
   std::unique_ptr<Model> model =
       Import(toco_flags, model_flags, graph_def_contents);
-  Transform(toco_flags, model.get());
+  TF_RETURN_IF_ERROR(TransformWithStatus(toco_flags, model.get()));
   return Export(toco_flags, *model, toco_flags.allow_custom_ops(),
                 output_file_contents);
 }
diff --git a/tensorflow/lite/toco/toco_tooling.cc b/tensorflow/lite/toco/toco_tooling.cc
index 69d7a7a61a..06f51825fe 100644
--- a/tensorflow/lite/toco/toco_tooling.cc
+++ b/tensorflow/lite/toco/toco_tooling.cc
@@ -236,7 +236,8 @@ std::unique_ptr<Model> Import(const TocoFlags& toco_flags,
   return model;
 }
 
-void Transform(const TocoFlags& toco_flags, Model* model) {
+tensorflow::Status TransformWithStatus(const TocoFlags& toco_flags,
+                                       Model* model) {
   const FileFormat output_format = toco_flags.output_format();
   const IODataType inference_type = toco_flags.inference_type();
 
@@ -258,8 +259,8 @@ void Transform(const TocoFlags& toco_flags, Model* model) {
   // stop optimizations from crossing the input/output boundaries. For example
   // this will stop BatchNorm fusing if the output node is in between a conv
   // and BatchNorm layers.
-  RunGraphTransformations(model, "Removing unused ops",
-                          {new toco::RemoveUnusedOp});
+  TF_RETURN_IF_ERROR(RunGraphTransformationsWithStatus(
+      model, "Removing unused ops", {new toco::RemoveUnusedOp}));
 
   GraphTransformationsSet transformations;
   MakeGeneralGraphTransformationsSet(&transformations);
@@ -307,20 +308,21 @@ void Transform(const TocoFlags& toco_flags, Model* model) {
     identify_dilated_conv->set_identify_depthwise_conv(false);
   }
   transformations.Add(identify_dilated_conv);
-  RunGraphTransformations(model, "general graph transformations",
-                          transformations);
+  TF_RETURN_IF_ERROR(RunGraphTransformationsWithStatus(
+      model, "general graph transformations", transformations));
 
   if (quantize_output) {
     if (toco_flags.propagate_fake_quant_num_bits()) {
-      RunGraphTransformations(model,
-                              "fake quant propagation graph transformations",
-                              {new PropagateFakeQuantNumBits});
+      TF_RETURN_IF_ERROR(RunGraphTransformationsWithStatus(
+          model, "fake quant propagation graph transformations",
+          {new PropagateFakeQuantNumBits}));
     }
-    RunGraphTransformations(model, "pre-quantization graph transformations",
-                            {
-                                new HardcodeMinMax,
-                                new DropFakeQuant,
-                            });
+    TF_RETURN_IF_ERROR(RunGraphTransformationsWithStatus(
+        model, "pre-quantization graph transformations",
+        {
+            new HardcodeMinMax,
+            new DropFakeQuant,
+        }));
   }
 
   // Try to merge bidirectional sequence lstm or rnn if present.
@@ -328,8 +330,9 @@ void Transform(const TocoFlags& toco_flags, Model* model) {
   bidirectional_transformations.Add(new RemoveUnusedOp);
   bidirectional_transformations.Add(new toco::GroupBidirectionalSequenceLstm);
   bidirectional_transformations.Add(new toco::GroupBidirectionalSequenceRnn);
-  RunGraphTransformations(model, "Group bidirectional sequence lstm/rnn",
-                          bidirectional_transformations);
+  TF_RETURN_IF_ERROR(RunGraphTransformationsWithStatus(
+      model, "Group bidirectional sequence lstm/rnn",
+      bidirectional_transformations));
 
   // Fix any issues with IO edges. This must happen after any transform that
   // may modify the structure of the edges.
@@ -357,12 +360,12 @@ void Transform(const TocoFlags& toco_flags, Model* model) {
           toco_flags.default_int16_ranges_max());
     }
     if (propagate_default_min_max->has_any_ranges_defined()) {
-      RunGraphTransformations(
+      TF_RETURN_IF_ERROR(RunGraphTransformationsWithStatus(
           model, "default min-max range propagation graph transformations",
           {
               propagate_default_min_max.release(),
               new HardcodeMinMax,
-          });
+          }));
     }
 
     CheckIsReadyForQuantization(*model);
@@ -372,17 +375,18 @@ void Transform(const TocoFlags& toco_flags, Model* model) {
         toco_flags.allow_nudging_weights_to_use_fast_gemm_kernel());
     ensure_safe_for_int8_kernels->set_has_default_ranges_flag(
         has_default_ranges_flag);
-    RunGraphTransformations(model, "quantization graph transformations",
-                            {
-                                new RemoveTrivialQuantizedActivationFunc,
-                                new RemoveTrivialQuantizedMinMax,
-                                new Quantize,
-                                new RemoveFinalDequantizeOp,
-                                ensure_safe_for_int8_kernels,
-                            });
+    TF_RETURN_IF_ERROR(RunGraphTransformationsWithStatus(
+        model, "quantization graph transformations",
+        {
+            new RemoveTrivialQuantizedActivationFunc,
+            new RemoveTrivialQuantizedMinMax,
+            new Quantize,
+            new RemoveFinalDequantizeOp,
+            ensure_safe_for_int8_kernels,
+        }));
     if (SupportsShuffledFCWeights(output_format)) {
-      RunGraphTransformations(model, "shuffling of FC weights",
-                              {new ShuffleFCWeights});
+      TF_RETURN_IF_ERROR(RunGraphTransformationsWithStatus(
+          model, "shuffling of FC weights", {new ShuffleFCWeights}));
     }
   } else {
     GraphTransformationsSet dequantization_transformations{new Dequantize};
@@ -392,8 +396,9 @@ void Transform(const TocoFlags& toco_flags, Model* model) {
       dequantization_transformations.Add(new DropFakeQuant);
     }
 
-    RunGraphTransformations(model, "dequantization graph transformations",
-                            dequantization_transformations);
+    TF_RETURN_IF_ERROR(RunGraphTransformationsWithStatus(
+        model, "dequantization graph transformations",
+        dequantization_transformations));
   }
 
   if (output_format == TENSORFLOW_GRAPHDEF) {
@@ -425,6 +430,7 @@ void Transform(const TocoFlags& toco_flags, Model* model) {
               << " billion (note that a multiply-add is counted as 2 ops).";
   }
   model->ops_count = ops_count;
+  return tensorflow::Status::OK();
 }
 
 tensorflow::Status Export(const TocoFlags& toco_flags, const Model& model,
diff --git a/tensorflow/lite/toco/toco_tooling.h b/tensorflow/lite/toco/toco_tooling.h
index 742e376926..3699615194 100644
--- a/tensorflow/lite/toco/toco_tooling.h
+++ b/tensorflow/lite/toco/toco_tooling.h
@@ -31,7 +31,12 @@ std::unique_ptr<Model> Import(const TocoFlags& toco_flags,
 
 // Transforms a Model. The resulting Model is ready to be passed
 // to Export with the exact same toco_flags.
-void Transform(const TocoFlags& toco_flags, Model* model);
+tensorflow::Status TransformWithStatus(const TocoFlags& toco_flags,
+                                       Model* model);
+inline void Transform(const TocoFlags& toco_flags, Model* model) {
+  auto s = TransformWithStatus(toco_flags, model);
+  CHECK(s.ok()) << s.error_message();
+}
 
 // Exports the Model, which must be of the 'lowered' form returned by
 // Transform, to a file of the format given by
-- 
GitLab


From 8f78dadeab5c448de86e7de1e45ce6a0f8236960 Mon Sep 17 00:00:00 2001
From: HanGuo97 <alex.guohan1106@gmail.com>
Date: Wed, 6 Feb 2019 16:14:25 -0500
Subject: [PATCH 2020/2345] fixed the reuse argument in Dense layer's
 documentation

---
 tensorflow/python/layers/core.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/layers/core.py b/tensorflow/python/layers/core.py
index b2d54a9827..7e12dcacd8 100644
--- a/tensorflow/python/layers/core.py
+++ b/tensorflow/python/layers/core.py
@@ -64,7 +64,7 @@ class Dense(keras_layers.Dense, base.Layer):
       `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`).
     name: String, the name of the layer. Layers with the same name will
       share weights, but to avoid mistakes we require reuse=True in such cases.
-    reuse: Boolean, whether to reuse the weights of a previous layer
+    _reuse: Boolean, whether to reuse the weights of a previous layer
       by the same name.
 
   Properties:
-- 
GitLab


From 09743df8f2a59b19f9fb75553c8d7ae62e4cf45d Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Wed, 6 Feb 2019 13:16:56 -0800
Subject: [PATCH 2021/2345] Re-use validation execution function instead of
 creating a new one each time.

PiperOrigin-RevId: 232732278
---
 .../engine/distributed_training_utils.py      | 49 +++++++++++++------
 tensorflow/python/keras/engine/training.py    |  1 +
 2 files changed, 35 insertions(+), 15 deletions(-)

diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index 7df9c2f76b..ccbc1c9b71 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -723,21 +723,46 @@ def clone_model_on_replicas(model, strategy, mode, inputs=None, targets=None):
 
 
 def _make_execution_function(model, mode):
-  """Makes function to run one step of distributed model execution."""
+  """Makes or reuses function to run one step of distributed model execution."""
+  strategy = model._distribution_strategy
+
+  distributed_model = get_distributed_model(model, mode)
+  # If distributed model for a particular `mode` is already built, use the
+  # `_distribution_function` on that distributed model.
+  if distributed_model:
+    return distributed_model._distributed_function
+
+  # If distributed_model is not built, create one for `mode`.
+  if model._compile_distribution:
+    clone_model_on_replicas(model, strategy, mode)
+  else:
+    _build_distributed_network(model, strategy, mode)
+
+  # We've just created the distributed model. So `distributed_model` should be
+  # not None.
+  distributed_model = get_distributed_model(model, mode)
+  assert distributed_model
+
+  # Also create an execution fuction on that distributed model.
   if context.executing_eagerly():
-    return _make_eager_execution_function(model, mode)
+    distributed_function = _make_eager_execution_function(model, mode)
+  else:
+    distributed_function = _make_graph_execution_function(model, mode)
+
+  # We cache the distributed execution function on the model since creating
+  # distributed models and exection functions are expensive.
+  distributed_model._distributed_function = distributed_function
+  return distributed_function
 
-  strategy = model._distribution_strategy
-  if not get_distributed_model(model, mode):
-    if model._compile_distribution:
-      clone_model_on_replicas(model, strategy, mode)
-    else:
-      _build_distributed_network(model, strategy, mode)
+
+def _make_graph_execution_function(model, mode):
+  """Makes function to run one step of distributed model in graph mode."""
 
   def _per_device_function(model):
     f = model._make_execution_function(mode)
     return (f.inputs, f.outputs, f.updates_op, f.session_kwargs)
 
+  strategy = model._distribution_strategy
   with strategy.scope():
     # Create train ops on each of the devices when we call
     # `_per_device_fit_function`.
@@ -773,19 +798,13 @@ def _make_execution_function(model, mode):
 
 def _make_eager_execution_function(model, mode):
   """Makes function to run one step of distributed model eager execution."""
-  strategy = model._distribution_strategy
-  if not get_distributed_model(model, mode):
-    if model._compile_distribution:
-      clone_model_on_replicas(model, strategy, mode)
-    else:
-      _build_distributed_network(model, strategy, mode)
-
   def _per_device_function(model):
     f = model._make_execution_function(mode)
     return (f.inputs, f.outputs)
 
   # NOTE(priyag): Try creating a new FuncGraph within DS scope instead of using
   # the global one.
+  strategy = model._distribution_strategy
   with K.get_graph().as_default(), strategy.scope():
     # Create train ops on each of the devices when we call
     # `_per_device_fit_function`.
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 6530b086ab..725e20beec 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -538,6 +538,7 @@ class Model(Network):
 
       self._fit_function = None
       self._eval_function = None
+      self._predict_function = None
       self.train_function = None
       self.test_function = None
       self.predict_function = None
-- 
GitLab


From b981a1757ec4f0134d0fe46cddd16295508b1b54 Mon Sep 17 00:00:00 2001
From: Yifei Feng <yifeif@google.com>
Date: Wed, 6 Feb 2019 13:18:22 -0800
Subject: [PATCH 2022/2345] Add test to sanity to check that GPU version of
 libtensorflow_framework.so does not depend on cuda shared libraries.

PiperOrigin-RevId: 232732522
---
 tensorflow/tools/ci_build/ci_sanity.sh | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/ci_build/ci_sanity.sh b/tensorflow/tools/ci_build/ci_sanity.sh
index 2c348a0e33..afb282715b 100755
--- a/tensorflow/tools/ci_build/ci_sanity.sh
+++ b/tensorflow/tools/ci_build/ci_sanity.sh
@@ -540,9 +540,12 @@ do_check_file_name_test() {
   python file_name_test.py
 }
 
+do_libtensorflow_framework_not_depend_on_cuda_check() {
+  bazel build --action_env=TF_NEED_CUDA=1 --define framework_shared_object=true --config=cuda --nobuild_tests_only tensorflow/core/platform/default/build_config:libtensorflow_cuda_check_deps
+}
 # Supply all sanity step commands and descriptions
-SANITY_STEPS=("do_pylint PYTHON2" "do_pylint PYTHON3" "do_check_futures_test" "do_buildifier" "do_bazel_nobuild" "do_pip_package_licenses_check" "do_lib_package_licenses_check" "do_java_package_licenses_check" "do_pip_smoke_test" "do_check_load_py_test" "do_code_link_check" "do_check_file_name_test")
-SANITY_STEPS_DESC=("Python 2 pylint" "Python 3 pylint" "Check that python files have certain __future__ imports" "buildifier check" "bazel nobuild" "pip: license check for external dependencies" "C library: license check for external dependencies" "Java Native Library: license check for external dependencies" "Pip Smoke Test: Checking py_test dependencies exist in pip package" "Check load py_test: Check that BUILD files with py_test target properly load py_test" "Code Link Check: Check there are no broken links" "Check file names for cases")
+SANITY_STEPS=("do_pylint PYTHON2" "do_pylint PYTHON3" "do_check_futures_test" "do_buildifier" "do_bazel_nobuild" "do_pip_package_licenses_check" "do_lib_package_licenses_check" "do_java_package_licenses_check" "do_pip_smoke_test" "do_check_load_py_test" "do_code_link_check" "do_check_file_name_test" "do_libtensorflow_framework_not_depend_on_cuda_check")
+SANITY_STEPS_DESC=("Python 2 pylint" "Python 3 pylint" "Check that python files have certain __future__ imports" "buildifier check" "bazel nobuild" "pip: license check for external dependencies" "C library: license check for external dependencies" "Java Native Library: license check for external dependencies" "Pip Smoke Test: Checking py_test dependencies exist in pip package" "Check load py_test: Check that BUILD files with py_test target properly load py_test" "Code Link Check: Check there are no broken links" "Check file names for cases" "Check gpu libtensorflow_framework.so does not depend on cuda shared libraries.")
 
 INCREMENTAL_FLAG=""
 DEFAULT_BAZEL_CONFIGS=""
-- 
GitLab


From 3dd576de9d4a0bc5ba5916ec3556a6b344dd2be3 Mon Sep 17 00:00:00 2001
From: James Ring <sjr@google.com>
Date: Wed, 6 Feb 2019 13:21:00 -0800
Subject: [PATCH 2023/2345] Automated rollback of commit
 1a43c97f079e9f5ec5b2798f053ac5d3aa6b1973

PiperOrigin-RevId: 232732980
---
 tensorflow/c/kernels/BUILD              |  35 -----
 tensorflow/c/kernels/bitcast_op.cc      | 165 ------------------------
 tensorflow/c/kernels/bitcast_op_test.cc | 101 ---------------
 tensorflow/core/BUILD                   |   1 -
 tensorflow/core/kernels/BUILD           |   7 +
 tensorflow/core/kernels/bitcast_op.cc   |  82 ++++++++++++
 tensorflow/core/kernels/bitcast_op.h    |  29 +++++
 7 files changed, 118 insertions(+), 302 deletions(-)
 delete mode 100644 tensorflow/c/kernels/BUILD
 delete mode 100644 tensorflow/c/kernels/bitcast_op.cc
 delete mode 100644 tensorflow/c/kernels/bitcast_op_test.cc
 create mode 100644 tensorflow/core/kernels/bitcast_op.cc
 create mode 100644 tensorflow/core/kernels/bitcast_op.h

diff --git a/tensorflow/c/kernels/BUILD b/tensorflow/c/kernels/BUILD
deleted file mode 100644
index e4ce21890d..0000000000
--- a/tensorflow/c/kernels/BUILD
+++ /dev/null
@@ -1,35 +0,0 @@
-load(
-    "//tensorflow:tensorflow.bzl",
-    "tf_cc_test",
-    "tf_cuda_library",
-)
-
-package(
-    default_visibility = ["//visibility:public"],
-)
-
-licenses(["notice"])  # Apache 2.0
-
-tf_cuda_library(
-    name = "bitcast_op",
-    srcs = ["bitcast_op.cc"],
-    deps = [
-        "//tensorflow/c:kernels",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:ops",
-        "@com_google_absl//absl/strings:str_format",
-    ],
-    alwayslink = 1,
-)
-
-tf_cc_test(
-    name = "bitcast_op_test",
-    srcs = ["bitcast_op_test.cc"],
-    deps = [
-        ":bitcast_op",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:protos_all_cc",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-    ],
-)
diff --git a/tensorflow/c/kernels/bitcast_op.cc b/tensorflow/c/kernels/bitcast_op.cc
deleted file mode 100644
index 90f32c990e..0000000000
--- a/tensorflow/c/kernels/bitcast_op.cc
+++ /dev/null
@@ -1,165 +0,0 @@
-/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "absl/strings/str_format.h"
-#include "tensorflow/c/kernels.h"
-#include "tensorflow/core/framework/common_shape_fns.h"
-#include "tensorflow/core/framework/op.h"
-#include "tensorflow/core/framework/shape_inference.h"
-#include "tensorflow/core/framework/types.h"
-
-// BitcastOp implements a bitcast kernel, creating an output tensor that shares
-// the same data buffer as the input but with a different shape and/or data
-// type. Its inputs are:
-//
-//   * the input tensor
-//   * an attribute named "T" containing the TF_DataType of the input tensor
-//   * an attribute named "type" containing the TF_DataType of the output tensor
-//
-// Given an input tensor of shape [...], if the input DataType "T" is larger
-// than the output DataType "type", then the shape changes from [...]
-// to [..., sizeof(T)/sizeof(type)].
-//
-// If "T" is smaller than "type", the operator requires that the rightmost
-// dimension be equal to sizeof(type)/sizeof(T). The shape then goes from
-// [..., sizeof(type)/sizeof(T)] to [...].
-//
-// Bitcast is implemented as a low-level cast, so machines with different endian
-// orderings will give different results.
-typedef struct BitcastOp {
-  TF_DataType input_data_type;
-  TF_DataType output_data_type;
-  size_t in_size;
-  size_t out_size;
-} BitcastOp;
-
-void* BitcastOp_Create(TF_OpKernelConstruction* ctx) {
-  auto* kernel = new BitcastOp;
-
-  TF_Status* s = TF_NewStatus();
-  TF_OpKernelConstruction_GetAttrType(ctx, "T", &kernel->input_data_type, s);
-
-  if (TF_GetCode(s) == TF_OK) {
-    TF_OpKernelConstruction_GetAttrType(ctx, "type", &kernel->output_data_type,
-                                        s);
-  }
-
-  if (TF_GetCode(s) == TF_OK) {
-    kernel->in_size = TF_DataTypeSize(kernel->input_data_type);
-    kernel->out_size = TF_DataTypeSize(kernel->output_data_type);
-
-    size_t check_size = std::max(kernel->in_size, kernel->out_size) %
-                        std::min(kernel->in_size, kernel->out_size);
-    if (check_size != 0) {
-      std::string err =
-          absl::StrFormat("cannot convert between datatype %d and %d",
-                          kernel->input_data_type, kernel->output_data_type);
-      TF_SetStatus(s, TF_INVALID_ARGUMENT, err.c_str());
-    }
-  }
-
-  if (TF_GetCode(s) != TF_OK) {
-    TF_OpKernelConstruction_Failure(ctx, s);
-    delete kernel;
-    kernel = nullptr;
-  }
-
-  TF_DeleteStatus(s);
-  return kernel;
-}
-
-void BitcastOp_Delete(void* kernel) { delete static_cast<BitcastOp*>(kernel); }
-
-void BitcastOp_Compute(void* kernel, TF_OpKernelContext* ctx) {
-  auto* k = static_cast<BitcastOp*>(kernel);
-  int dim_count = 0;
-
-  TF_Tensor* tensor;
-  TF_Status* status = TF_NewStatus();
-  TF_GetInput(ctx, 0, &tensor, status);
-  if (TF_GetCode(status) == TF_OK) {
-    dim_count = TF_NumDims(tensor);
-    if (!(k->in_size >= k->out_size ||
-          (dim_count > 0 &&
-           TF_Dim(tensor, dim_count - 1) == k->out_size / k->in_size))) {
-      std::string err =
-          absl::StrFormat("Cannot bitcast from %d to %d", k->input_data_type,
-                          k->output_data_type);
-      TF_SetStatus(status, TF_INVALID_ARGUMENT, err.c_str());
-    }
-  }
-
-  if (TF_GetCode(status) == TF_OK) {
-    auto* dims = new int64_t[dim_count + 1];
-    int new_dim_count = dim_count;
-    for (int dim = 0; dim < dim_count; ++dim) {
-      dims[dim] = TF_Dim(tensor, dim);
-    }
-    if (k->out_size < k->in_size) {
-      dims[new_dim_count++] = static_cast<int64_t>(k->in_size / k->out_size);
-    } else if (k->out_size > k->in_size) {
-      --new_dim_count;
-    }
-
-    TF_Tensor* output = TF_AllocateTensor(k->output_data_type, dims, 0,
-                                          TF_DataTypeSize(k->output_data_type));
-    TF_TensorBitcastFrom(tensor, k->output_data_type, output, dims,
-                         new_dim_count, status);
-    if (TF_GetCode(status) == TF_OK) {
-      TF_SetOutput(ctx, 0, output, status);
-    }
-    delete[] dims;
-    TF_DeleteTensor(output);
-  }
-
-  if (TF_GetCode(status) != TF_OK) {
-    TF_OpKernelContext_Failure(ctx, status);
-  }
-  TF_DeleteStatus(status);
-  TF_DeleteTensor(tensor);
-}
-
-void RegisterBitcastOp() {
-  TF_Status* status = TF_NewStatus();
-
-  {
-    auto* builder = TF_NewKernelBuilder("Bitcast", tensorflow::DEVICE_CPU,
-                                        &BitcastOp_Create, &BitcastOp_Compute,
-                                        &BitcastOp_Delete);
-    TF_RegisterKernelBuilder("BitcastOp", builder, status);
-    CHECK_EQ(TF_OK, TF_GetCode(status))
-        << "Error while registering bitcast kernel";
-  }
-
-#if GOOGLE_CUDA
-  {
-    auto* builder = TF_NewKernelBuilder("Bitcast", tensorflow::DEVICE_GPU,
-                                        &BitcastOp_Create, &BitcastOp_Compute,
-                                        &BitcastOp_Delete);
-    TF_RegisterKernelBuilder("BitcastOp", builder, status);
-    CHECK_EQ(TF_OK, TF_GetCode(status))
-        << "Error while registering CUDA bitcast kernel";
-  }
-#endif
-
-  TF_DeleteStatus(status);
-}
-
-// A dummy static variable initialized by a lambda whose side-effect is to
-// register the bitcast kernel.
-static bool BitcastOpIsRegistered = []() {
-  RegisterBitcastOp();
-  return true;
-}();
diff --git a/tensorflow/c/kernels/bitcast_op_test.cc b/tensorflow/c/kernels/bitcast_op_test.cc
deleted file mode 100644
index 06ffcca19d..0000000000
--- a/tensorflow/c/kernels/bitcast_op_test.cc
+++ /dev/null
@@ -1,101 +0,0 @@
-/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/core/framework/attr_value.pb.h"
-#include "tensorflow/core/framework/attr_value_util.h"
-#include "tensorflow/core/framework/node_def.pb.h"
-#include "tensorflow/core/framework/op_kernel.h"
-#include "tensorflow/core/platform/test.h"
-
-namespace tensorflow {
-namespace {
-
-class DummyDevice : public DeviceBase {
- public:
-  DummyDevice(Env* env, bool save) : DeviceBase(env), save_(save) {}
-  bool RequiresRecordingAccessedTensors() const override { return save_; }
-  Allocator* GetAllocator(AllocatorAttributes /*attr*/) override {
-    return cpu_allocator();
-  }
-
- private:
-  bool save_;
-};
-
-void TestBitcastOp(Tensor* input_tensor, DataType out_type,
-                   TensorShape expected_shape, error::Code expected_code) {
-  Status status;
-  NodeDef def;
-  def.set_op("Bitcast");
-  def.set_device(DEVICE_CPU);
-
-  AttrValue typeAttr;
-  SetAttrValue(input_tensor->dtype(), &typeAttr);
-
-  AttrValue outTypeAttr;
-  SetAttrValue(out_type, &outTypeAttr);
-
-  (*def.mutable_attr())["T"] = typeAttr;
-  (*def.mutable_attr())["type"] = outTypeAttr;
-
-  def.add_input(
-      strings::StrCat("input1: ", DataTypeString(input_tensor->dtype())));
-
-  std::unique_ptr<OpKernel> kernel =
-      CreateOpKernel(DeviceType(DEVICE_CPU), nullptr, nullptr, def, 1, &status);
-  ASSERT_TRUE(status.ok()) << status.ToString();
-
-  OpKernelContext::Params params;
-  DummyDevice dummy_device(nullptr, false);
-  params.device = &dummy_device;
-  params.op_kernel = kernel.get();
-  gtl::InlinedVector<TensorValue, 4> inputs;
-  inputs.emplace_back(input_tensor);
-  params.inputs = &inputs;
-
-  OpKernelContext ctx(&params);
-  kernel->Compute(&ctx);
-  ASSERT_EQ(expected_code, ctx.status().code());
-  if (expected_code == error::OK) {
-    ASSERT_EQ(expected_shape, ctx.mutable_output(0)->shape())
-        << ctx.mutable_output(0)->shape().DebugString();
-  }
-}
-
-TEST(BitcastOpTest, TestUpcast) {
-  Tensor int8_input(DT_UINT8, {8});
-  for (int i = 0; i < 8; i++) {
-    int8_input.vec<uint8>()(i) = static_cast<uint8>(1);
-  }
-  TestBitcastOp(&int8_input, DT_UINT64, TensorShape(), error::OK);
-}
-
-TEST(BitcastOpTest, TestDowncast) {
-  Tensor int64_input(static_cast<uint64>(1));
-  TestBitcastOp(&int64_input, DT_UINT8, TensorShape({8}), error::OK);
-}
-
-TEST(BitcastOpTest, TestCastToSameSize) {
-  Tensor int32_input(DT_UINT32, {4, 6});
-  TestBitcastOp(&int32_input, DT_UINT8, TensorShape({4, 6, 4}), error::OK);
-}
-
-TEST(BitcastOpTest, TestImpossibleCast) {
-  Tensor int8_input(DT_UINT8, {1});
-  TestBitcastOp(&int8_input, DT_UINT32, TensorShape(), error::INVALID_ARGUMENT);
-}
-
-}  // namespace
-}  // namespace tensorflow
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index e22eca3696..2f066d8a63 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1397,7 +1397,6 @@ cc_library(
     name = "all_kernels_impl",
     visibility = ["//visibility:private"],
     deps = [
-        "//tensorflow/c/kernels:bitcast_op",
         "//tensorflow/core/kernels:array",
         "//tensorflow/core/kernels:audio",
         "//tensorflow/core/kernels:batch_kernels",
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index be4f29bf2e..1749f3a4da 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -838,6 +838,7 @@ cc_library(
     deps = [
         ":batch_space_ops",
         ":bcast_ops",
+        ":bitcast_op",
         ":broadcast_to_op",
         ":concat_op",
         ":constant_op",
@@ -887,6 +888,12 @@ tf_kernel_library(
     deps = ARRAY_DEPS,
 )
 
+tf_kernel_library(
+    name = "bitcast_op",
+    prefix = "bitcast_op",
+    deps = ARRAY_DEPS,
+)
+
 tf_kernel_library(
     name = "broadcast_to_op",
     prefix = "broadcast_to_op",
diff --git a/tensorflow/core/kernels/bitcast_op.cc b/tensorflow/core/kernels/bitcast_op.cc
new file mode 100644
index 0000000000..02c8808809
--- /dev/null
+++ b/tensorflow/core/kernels/bitcast_op.cc
@@ -0,0 +1,82 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// See docs in ../ops/array_ops.cc.
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/register_types.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/framework/types.h"
+
+namespace tensorflow {
+
+class BitcastOp : public OpKernel {
+ public:
+  explicit BitcastOp(OpKernelConstruction* context) : OpKernel(context) {
+    OP_REQUIRES_OK(context, context->GetAttr("T", &input_data_type_));
+    OP_REQUIRES_OK(context, context->GetAttr("type", &output_data_type_));
+    in_size_ = DataTypeSize(input_data_type_);
+    out_size_ = DataTypeSize(output_data_type_);
+    int check_size =
+        std::max(in_size_, out_size_) % std::min(in_size_, out_size_);
+    OP_REQUIRES(
+        context, check_size == 0,
+        errors::InvalidArgument("cannot convert between datatype ",
+                                input_data_type_, " and ", output_data_type_));
+  }
+
+  void Compute(OpKernelContext* context) override {
+    const Tensor& input_tensor = context->input(0);
+
+    TensorShape adjusted_shape = input_tensor.shape();
+    OP_REQUIRES(context,
+                in_size_ >= out_size_ ||
+                    (input_tensor.dims() > 0 &&
+                     input_tensor.dim_size(input_tensor.dims() - 1) ==
+                         out_size_ / in_size_),
+                errors::InvalidArgument(
+                    "Cannot bitcast from ", DataTypeString(input_data_type_),
+                    " to ", DataTypeString(output_data_type_), ": shape ",
+                    input_tensor.shape().DebugString()));
+
+    if (out_size_ < in_size_) {
+      adjusted_shape.AddDim(in_size_ / out_size_);
+    } else if (out_size_ > in_size_) {
+      adjusted_shape.RemoveDim(input_tensor.dims() - 1);
+    }
+    Tensor output_tensor;
+
+    OP_REQUIRES_OK(context,
+                   output_tensor.BitcastFrom(input_tensor, output_data_type_,
+                                             adjusted_shape));
+    context->set_output(0, output_tensor);
+  }
+
+  bool IsExpensive() override { return false; }
+
+ private:
+  DataType input_data_type_;
+  DataType output_data_type_;
+  int in_size_;
+  int out_size_;
+};
+
+REGISTER_KERNEL_BUILDER(Name("Bitcast").Device(DEVICE_CPU), BitcastOp);
+
+#if GOOGLE_CUDA
+REGISTER_KERNEL_BUILDER(Name("Bitcast").Device(DEVICE_GPU), BitcastOp);
+#endif  // GOOGLE_CUDA
+
+}  // end namespace tensorflow
diff --git a/tensorflow/core/kernels/bitcast_op.h b/tensorflow/core/kernels/bitcast_op.h
new file mode 100644
index 0000000000..1f3659f303
--- /dev/null
+++ b/tensorflow/core/kernels/bitcast_op.h
@@ -0,0 +1,29 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// See docs in ../ops/array_ops.cc.
+
+#ifndef TENSORFLOW_CORE_KERNELS_BITCAST_OP_H_
+#define TENSORFLOW_CORE_KERNELS_BITCAST_OP_H_
+
+#include <string.h>  // for memcpy
+
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/register_types.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/framework/types.h"
+
+#endif  // TENSORFLOW_CORE_KERNELS_BITCAST_OP_H_
-- 
GitLab


From 1a5abfb53c00bb37173bfd858be5a20d1d00f39d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 6 Feb 2019 13:54:04 -0800
Subject: [PATCH 2024/2345] Export keras.layers.DenseFeatures in v1

PiperOrigin-RevId: 232739826
---
 .../feature_column/feature_column_v2.py       |   2 +-
 ...sorflow.keras.layers.-dense-features.pbtxt | 184 ++++++++++++++++++
 .../golden/v1/tensorflow.keras.layers.pbtxt   |   4 +
 3 files changed, 189 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dense-features.pbtxt

diff --git a/tensorflow/python/feature_column/feature_column_v2.py b/tensorflow/python/feature_column/feature_column_v2.py
index edc37193e5..5ebbaab775 100644
--- a/tensorflow/python/feature_column/feature_column_v2.py
+++ b/tensorflow/python/feature_column/feature_column_v2.py
@@ -380,7 +380,7 @@ class _BaseFeaturesLayer(Layer):
     return array_ops.concat(output_tensors, -1)
 
 
-@keras_export('keras.layers.DenseFeatures', v1=[])
+@keras_export('keras.layers.DenseFeatures')
 class DenseFeatures(_BaseFeaturesLayer):
   """A layer that produces a dense `Tensor` based on given `feature_columns`.
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dense-features.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dense-features.pbtxt
new file mode 100644
index 0000000000..ca6a327a58
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dense-features.pbtxt
@@ -0,0 +1,184 @@
+path: "tensorflow.keras.layers.DenseFeatures"
+tf_class {
+  is_instance: "<class \'tensorflow.python.feature_column.feature_column_v2.DenseFeatures\'>"
+  is_instance: "<class \'tensorflow.python.feature_column.feature_column_v2._BaseFeaturesLayer\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'feature_columns\', \'trainable\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'True\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'_\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'features\', \'cols_to_output_tensors\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.pbtxt
index ad74ab3f12..cc0fdabee1 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.pbtxt
@@ -124,6 +124,10 @@ tf_module {
     name: "Dense"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "DenseFeatures"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "DepthwiseConv2D"
     mtype: "<type \'type\'>"
-- 
GitLab


From 6c7022a46a79c31828585415181d563f878c237c Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Wed, 6 Feb 2019 14:06:11 -0800
Subject: [PATCH 2025/2345] Never track `mask` for the purposes of
 `TensorFlowOpLayer`.

PiperOrigin-RevId: 232742436
---
 tensorflow/python/keras/engine/base_layer.py  | 49 ++++++++++---------
 .../keras/layers/tensorflow_op_layer_test.py  |  5 ++
 2 files changed, 31 insertions(+), 23 deletions(-)

diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index 4eb0be58cf..eee23b1311 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -1353,32 +1353,35 @@ class Layer(checkpointable.Checkpointable):
           self.add_loss(mean_activity_loss, inputs=inputs)
 
   def _set_mask_metadata(self, inputs, outputs, previous_mask):
-    if getattr(self, '_compute_output_and_mask_jointly', False):
-      # Mask is already computed for Keras Graph Networks.
-      return
-
     flat_outputs = nest.flatten(outputs)
-    if all(getattr(x, '_keras_mask', None) is not None for x in flat_outputs):
-      # Mask is already computed by sublayers.
-      return
-
-    if hasattr(self, 'compute_mask'):
-      output_masks = self.compute_mask(inputs, previous_mask)
-      # `compute_mask` can return a single `None` even when a Layer
-      # has multiple outputs.
-      if output_masks is None:
-        flat_masks = [None for _ in flat_outputs]
+    mask_already_computed = (
+        getattr(self, '_compute_output_and_mask_jointly', False) or
+        all(getattr(x, '_keras_mask', None) is not None for x in flat_outputs))
+
+    if not mask_already_computed:
+      if hasattr(self, 'compute_mask'):
+        output_masks = self.compute_mask(inputs, previous_mask)
+        # `compute_mask` can return a single `None` even when a Layer
+        # has multiple outputs.
+        if output_masks is None:
+          flat_masks = [None for _ in flat_outputs]
+        else:
+          flat_masks = nest.flatten(output_masks)
       else:
-        flat_masks = nest.flatten(output_masks)
-    else:
-      flat_masks = [None for _ in flat_outputs]
+        flat_masks = [None for _ in flat_outputs]
 
-    for output, mask in zip(flat_outputs, flat_masks):
-      try:
-        output._keras_mask = mask
-      except AttributeError:
-        # C Type such as np.ndarray.
-        pass
+      for output, mask in zip(flat_outputs, flat_masks):
+        try:
+          output._keras_mask = mask
+        except AttributeError:
+          # C Type such as np.ndarray.
+          pass
+
+    if tf_utils.are_all_symbolic_tensors(flat_outputs):
+      for output in flat_outputs:
+        if getattr(output, '_keras_mask', None) is not None:
+          # Do not track masks for `TensorFlowOpLayer` construction.
+          output._keras_mask._keras_history_checked = True
 
   def _set_connectivity_metadata_(self, inputs, outputs, args, kwargs):
     call_convention = getattr(
diff --git a/tensorflow/python/keras/layers/tensorflow_op_layer_test.py b/tensorflow/python/keras/layers/tensorflow_op_layer_test.py
index c2f1c79844..8941d86674 100644
--- a/tensorflow/python/keras/layers/tensorflow_op_layer_test.py
+++ b/tensorflow/python/keras/layers/tensorflow_op_layer_test.py
@@ -162,6 +162,11 @@ class AutoLambdaTest(keras_parameterized.TestCase):
     e = 2  # Fudge factor to prevent flakiness.
     self.assertLess(size_500, (10 * e) * size_50)
 
+  def test_no_mask_tracking(self):
+    x = keras.backend.placeholder((10, 10))
+    y = keras.layers.Masking(0.)(x)
+    self.assertTrue(y._keras_mask._keras_history_checked)
+
 
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 5e92f41df836dd634b82067af3603cd50fb59c4f Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Wed, 6 Feb 2019 14:08:08 -0800
Subject: [PATCH 2026/2345] Add signatures to the return value of
 tf.saved_model.load()

Only works with models exported from tf.saved_model.save(). We still need a different path for restoring other SavedModels, but hopefully we can make the signature representations identical (keywords in, dict out).

In order to make this work I had to wrap exported signatures in another layer of function. I should either clean that up, or more likely we should come up with an easy API for inspecting the graphs of nested function calls.

PiperOrigin-RevId: 232742842
---
 tensorflow/python/saved_model/BUILD           |  11 +
 tensorflow/python/saved_model/load_test.py    |  63 +++++
 tensorflow/python/saved_model/save.py         | 211 ++++++---------
 tensorflow/python/saved_model/save_test.py    |   8 +
 .../saved_model/signature_serialization.py    | 244 ++++++++++++++++++
 .../training/checkpointable/graph_view.py     |  43 ++-
 6 files changed, 430 insertions(+), 150 deletions(-)
 create mode 100644 tensorflow/python/saved_model/signature_serialization.py

diff --git a/tensorflow/python/saved_model/BUILD b/tensorflow/python/saved_model/BUILD
index b605aa746c..76670bf309 100644
--- a/tensorflow/python/saved_model/BUILD
+++ b/tensorflow/python/saved_model/BUILD
@@ -261,6 +261,16 @@ tf_py_test(
     ],
 )
 
+py_library(
+    name = "signature_serialization",
+    srcs = [
+        "signature_serialization.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+    ],
+)
+
 py_library(
     name = "save",
     srcs = [
@@ -276,6 +286,7 @@ py_library(
         ":saved_object_graph_py",
         ":signature_constants",
         ":signature_def_utils",
+        ":signature_serialization",
         ":tag_constants",
         ":utils",
         "//tensorflow/core:protos_all_py",
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index d61fa864ad..1edb08943b 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -878,5 +878,68 @@ class LoadTest(test.TestCase, parameterized.TestCase):
     self.assertEqual(
         2, imported.table_user(constant_op.constant("gamma")).numpy())
 
+  def test_functions_accessed_once(self, cycles):
+
+    class Exported(tracking.AutoCheckpointable):
+
+      def __init__(self):
+        self._counter = 0
+
+      @property
+      def make_func(self):
+        @def_function.function
+        def f():
+          return constant_op.constant(self._counter)
+        f.get_concrete_function()  # force a trace
+        self._counter += 1
+        return f
+
+    exported = Exported()
+    imported = self.cycle(exported, cycles)
+    self.assertEqual(0, imported.make_func().numpy())
+    self.assertEqual(1, exported.make_func().numpy())
+
+  def test_overwritten_signatures_error(self, cycles):
+    exported = tracking.AutoCheckpointable()
+    exported.f = def_function.function(lambda: constant_op.constant(1.))
+    imported = self.cycle(
+        exported, cycles,
+        signatures={"key": exported.f.get_concrete_function()})
+    self.assertEqual(1., imported.signatures["key"]()["output_0"].numpy())
+    imported.signatures = {"key1": imported.signatures["key"]}
+    with self.assertRaisesRegexp(ValueError, "signatures"):
+      save.save(imported, tempfile.mkdtemp(prefix=self.get_temp_dir()))
+
+  def test_signature_loading(self, cycles):
+
+    class Exported(tracking.AutoCheckpointable):
+
+      def __init__(self):
+        self.v = variables.Variable(3.)
+
+      @def_function.function
+      def do(self, x):
+        return self.v * x
+
+    exported = Exported()
+    imported = self.cycle(
+        exported,
+        signatures=exported.do.get_concrete_function(
+            tensor_spec.TensorSpec(None, dtypes.float32)))
+    for _ in range(cycles - 1):
+      imported = self.cycle(imported, signatures=imported.signatures)
+    self.assertEqual(["serving_default"], list(imported.signatures.keys()))
+    imported_function = imported.signatures["serving_default"]
+    two = constant_op.constant(2.)
+    self.assertEqual(6., imported_function(x=two)["output_0"].numpy())
+    imported.v.assign(4.)
+    self.assertEqual(8., imported_function(x=two)["output_0"].numpy())
+    with self.assertRaisesRegexp(TypeError, "positional"):
+      imported_function(two)
+    with self.assertRaises(TypeError):
+      # The signatures mapping is immutable
+      imported.signatures["random_key"] = 3
+
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index 3c8bf1bf57..5b4f0161af 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -44,6 +44,7 @@ from tensorflow.python.saved_model import revived_types
 from tensorflow.python.saved_model import saved_object_graph_pb2
 from tensorflow.python.saved_model import signature_constants
 from tensorflow.python.saved_model import signature_def_utils
+from tensorflow.python.saved_model import signature_serialization
 from tensorflow.python.saved_model import tag_constants
 from tensorflow.python.saved_model import utils_impl
 from tensorflow.python.training.checkpointable import base
@@ -53,10 +54,8 @@ from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.training.checkpointable import util
 from tensorflow.python.training.saving import functional_saver
 from tensorflow.python.util import compat
-from tensorflow.python.util import nest
 from tensorflow.python.util.tf_export import tf_export
 
-DEFAULT_SIGNATURE_ATTR = "_default_save_signature"
 _UNCOPIABLE_DTYPES = frozenset((dtypes.resource, dtypes.variant))
 
 
@@ -66,14 +65,68 @@ _CapturedConstant = collections.namedtuple(
     "_CapturedConstant", ["eager_tensor", "graph_tensor"])
 
 
+class _AugmentedGraphView(graph_view.ObjectGraphView):
+  """An extendable graph which also tracks functions attached to objects.
+
+  Extensions through `add_object` appear in the object graph and any checkpoints
+  generated from it, even if they are not dependencies of the node they were
+  attached to in the saving program. For example a `.signatures` attribute is
+  added to exported SavedModel root objects without modifying the root object
+  itself.
+
+  Also tracks functions attached to objects in the graph, through the caching
+  `list_functions` method. Enumerating functions only through this method
+  ensures that we get a consistent view of functions, even if object attributes
+  create new functions every time they are accessed.
+  """
+
+  def __init__(self, root):
+    super(_AugmentedGraphView, self).__init__(root)
+    # Object -> (name -> dep)
+    self._extra_dependencies = object_identity.ObjectIdentityDictionary()
+    self._functions = object_identity.ObjectIdentityDictionary()
+
+  def add_object(self, parent_node, name_in_parent, subgraph_root):
+    """Attach an object to `parent_node`, overriding any existing dependency."""
+    self._extra_dependencies.setdefault(
+        parent_node, {})[name_in_parent] = subgraph_root
+
+  def list_dependencies(self, obj):
+    """Overrides a parent method to include `add_object` objects."""
+    extra_dependencies = self._extra_dependencies.get(obj, {})
+    used_names = set()
+    for name, dep in super(_AugmentedGraphView, self).list_dependencies(obj):
+      used_names.add(name)
+      if name in extra_dependencies:
+        yield base.CheckpointableReference(name, extra_dependencies[name])
+      else:
+        yield base.CheckpointableReference(name, dep)
+    for name, dep in extra_dependencies.items():
+      if name in used_names:
+        continue
+      yield base.CheckpointableReference(name, dep)
+
+  def list_functions(self, obj):
+    obj_functions = self._functions.get(obj, None)
+    if obj_functions is None:
+      obj_functions = obj._list_functions_for_serialization()  # pylint: disable=protected-access
+      self._functions[obj] = obj_functions
+    return obj_functions
+
+
 class _SaveableView(object):
-  """Provides a stable view over a checkpointable root.
+  """Provides a frozen view over a checkpointable root.
 
   This class helps creating a single stable view over an object to save. The
   saving code should access properties and functions via this class and not via
   the original object as there are cases where an object construct their
   checkpointable attributes and functions dynamically per call and will yield
   different objects if invoked more than once.
+
+  Changes to the graph, for example adding objects, must happen in
+  `checkpoint_view` (an `_AugmentedGraphView`) before the `_SaveableView` is
+  constructed. Changes after the `_SaveableView` has been constructed will be
+  ignored.
   """
 
   def __init__(self, checkpoint_view):
@@ -84,22 +137,16 @@ class _SaveableView(object):
     self.node_ids = node_ids
     self.captured_tensor_node_ids = object_identity.ObjectIdentityDictionary()
     self.slot_variables = slot_variables
-    self.functions = object_identity.ObjectIdentityDictionary()
     self.concrete_functions = []
 
     # Also add `Function`s as nodes.
     nodes_without_functions = list(self.nodes)
     seen_function_names = set()
-    for obj in nodes_without_functions:
-      self.functions[obj] = obj._list_functions_for_serialization()  # pylint: disable=protected-access
-      for function in self.functions[obj].values():
+    for node in nodes_without_functions:
+      for function in checkpoint_view.list_functions(node).values():
         if function not in self.node_ids:
           self.node_ids[function] = len(self.nodes)
           self.nodes.append(function)
-          # Avoids recursing into functions to see if other functions are
-          # assigned to attributes. This is sometimes true for concrete
-          # functions but not helpful.
-          self.functions[function] = {}
         if isinstance(function, def_function.Function):
           # Force listing the concrete functions for the side effects:
           #  - populate the cache for functions that have an input_signature
@@ -128,11 +175,12 @@ class _SaveableView(object):
       if isinstance(node, (def_function.Function, defun.ConcreteFunction,
                            _CapturedConstant)):
         continue
-      for child in node._checkpoint_dependencies:  # pylint: disable=protected-access
+      for child in self.checkpoint_view.list_dependencies(node):
         child_proto = object_proto.children.add()
         child_proto.node_id = self.node_ids[child.ref]
         child_proto.local_name = child.name
-      for local_name, ref_function in self.functions[node].items():
+      for local_name, ref_function in (
+          self.checkpoint_view.list_functions(node).items()):
         child_proto = object_proto.children.add()
         child_proto.node_id = self.node_ids[ref_function]
         child_proto.local_name = local_name
@@ -197,108 +245,6 @@ class _SaveableView(object):
     return object_map, resource_map, asset_info
 
 
-def _get_signature(function):
-  if (isinstance(function, (defun.Function, def_function.Function)) and
-      function._input_signature is not None):  # pylint: disable=protected-access
-    function = function.get_concrete_function()
-  if not isinstance(function, defun.ConcreteFunction):
-    return None
-  return function
-
-
-def _valid_signature(concrete_function):
-  """Returns whether concrete function can be converted to a signature."""
-  if not concrete_function.outputs:
-    # Functions without outputs don't make sense as signatures. We just don't
-    # have any way to run an Operation with no outputs as a SignatureDef in the
-    # 1.x style.
-    return False
-  try:
-    _normalize_outputs(concrete_function.structured_outputs, "unused", "unused")
-  except ValueError:
-    return False
-  return True
-
-
-def _find_function_to_export(saveable_view):
-  """Function to export, None if no suitable function was found."""
-  # If the user did not specify signatures, check the root object for a function
-  # that can be made into a signature.
-  functions = saveable_view.functions[saveable_view.root]
-  signature = functions.get(DEFAULT_SIGNATURE_ATTR, None)
-  if signature is not None:
-    return signature
-
-  # TODO(andresp): Discuss removing this behaviour. It can lead to WTFs when a
-  # user decides to annotate more functions with tf.function and suddenly
-  # serving that model way later in the process stops working.
-  if len(functions) == 1:
-    single_function = list(functions.values())[0]
-    signature = _get_signature(single_function)
-    if signature and  _valid_signature(signature):
-      return signature
-  return None
-
-
-def _canonicalize_signatures(signatures):
-  """Converts `signatures` into a dictionary of concrete functions."""
-  if signatures is None:
-    return {}
-  if not isinstance(signatures, collections.Mapping):
-    signatures = {
-        signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signatures}
-  concrete_signatures = {}
-  for signature_key, function in signatures.items():
-    signature_function = _get_signature(function)
-    if signature_function is None:
-      raise ValueError(
-          ("Expected a TensorFlow function to generate a signature for, but "
-           "got {}. Only `tf.functions` with an input signature or "
-           "concrete functions can be used as a signature.").format(function))
-    concrete_signatures[signature_key] = signature_function
-  return concrete_signatures
-
-
-def _is_flat(sequence):
-  sequence_flat = nest.flatten(sequence)
-  try:
-    nest.assert_same_structure(sequence_flat, sequence)
-    return True
-  except ValueError:
-    return False
-  except TypeError:
-    return False
-
-
-def _normalize_outputs(outputs, function_name, signature_key):
-  """Construct an output dictionary from unnormalized function outputs."""
-  if isinstance(outputs, collections.Mapping):
-    for key, value in outputs.items():
-      if not isinstance(value, ops.Tensor):
-        raise ValueError(
-            ("Got a dictionary containing non-Tensor value {} for key {} "
-             "in the output of the function {} used to generate a SavedModel "
-             "signature. Dictionaries outputs for functions used as signatures "
-             "should have one Tensor output per string key.")
-            .format(value, key, compat.as_str_any(function_name)))
-    return outputs
-  else:
-    original_outputs = outputs
-    if not isinstance(outputs, collections.Sequence):
-      outputs = [outputs]
-    if not _is_flat(outputs):
-      raise ValueError(
-          ("Got non-flat outputs '{}' from '{}' for SavedModel "
-           "signature '{}'. Signatures have one Tensor per output, so "
-           "to have predictable names Python functions used to generate "
-           "these signatures should avoid outputting Tensors in nested "
-           "structures.")
-          .format(original_outputs, function_name, signature_key))
-    return {("output_{}".format(output_index)): output
-            for output_index, output
-            in enumerate(outputs)}
-
-
 def _tensor_dict_to_tensorinfo(tensor_dict):
   return {key: utils_impl.build_tensor_info_internal(value)
           for key, value in tensor_dict.items()}
@@ -428,8 +374,8 @@ def _generate_signatures(signature_functions, resource_map):
 
   Args:
     signature_functions: A dictionary mapping string keys to concrete TensorFlow
-      functions (e.g. from `_canonicalize_signatures`) which will be used to
-      generate SignatureDefs.
+      functions (e.g. from `signature_serialization.canonicalize_signatures`)
+      which will be used to generate SignatureDefs.
     resource_map: A dictionary mapping from resource tensors in the eager
       context to resource tensors in the Graph being exported. This dictionary
       is used to re-bind resources captured by functions to tensors which will
@@ -460,10 +406,8 @@ def _generate_signatures(signature_functions, resource_map):
     mapped_inputs, exterior_argument_placeholders = (
         _map_function_arguments_to_created_inputs(
             argument_inputs, signature_key, function.name))
-    outputs = _normalize_outputs(
-        _call_function_with_mapped_captures(
-            function, mapped_inputs, resource_map),
-        function.name, signature_key)
+    outputs = _call_function_with_mapped_captures(
+        function, mapped_inputs, resource_map)
     signatures[signature_key] = signature_def_utils.build_signature_def(
         _tensor_dict_to_tensorinfo(exterior_argument_placeholders),
         _tensor_dict_to_tensorinfo(outputs),
@@ -699,6 +643,10 @@ def save(obj, export_dir, signatures=None):
   which case outputs will be numbered, or a dictionary mapping string keys to
   `Tensor`, in which case the keys will be used to name outputs.
 
+  Signatures are available in objects returned by `tf.saved_model.load` as a
+  `.signatures` attribute. This is a reserved attribute: `tf.saved_model.save`
+  on an object with a custom `.signatures` attribute will raise an exception.
+
   Since `tf.keras.Model` objects are also Checkpointable, this function can be
   used to export Keras models. For example, exporting with a signature
   specified:
@@ -823,18 +771,25 @@ def save(obj, export_dir, signatures=None):
     raise ValueError(
         "Expected a Checkpointable object for export, got {}.".format(obj))
 
-  # Use _SaveableView to provide a stable listing of properties and functions.
+  checkpoint_graph_view = _AugmentedGraphView(obj)
+  if signatures is None:
+    signatures = signature_serialization.find_function_to_export(
+        checkpoint_graph_view)
+
+  signatures = signature_serialization.canonicalize_signatures(signatures)
+  signature_map = signature_serialization.create_signature_map(
+      signatures, checkpoint_graph_view)
+  checkpoint_graph_view.add_object(
+      parent_node=checkpoint_graph_view.root,
+      name_in_parent=signature_serialization.SIGNATURE_ATTRIBUTE_NAME,
+      subgraph_root=signature_map)
+
+  # Use _SaveableView to provide a frozen listing of properties and functions.
   # Note we run this twice since, while constructing the view the first time
   # there can be side effects of creating variables.
-  checkpoint_graph_view = graph_view.ObjectGraphView(obj)
   _ = _SaveableView(checkpoint_graph_view)
   saveable_view = _SaveableView(checkpoint_graph_view)
 
-  if signatures is None:
-    signatures = _find_function_to_export(saveable_view)
-
-  signatures = _canonicalize_signatures(signatures)
-
   # TODO(allenl): Factor out some subset of SavedModelBuilder which is 2.x
   # compatible (no sessions) and share it with this export API rather than
   # making a SavedModel proto and writing it directly.
diff --git a/tensorflow/python/saved_model/save_test.py b/tensorflow/python/saved_model/save_test.py
index cbca51aea7..e1b2f5da6f 100644
--- a/tensorflow/python/saved_model/save_test.py
+++ b/tensorflow/python/saved_model/save_test.py
@@ -303,6 +303,14 @@ class SaveTest(test.TestCase):
       self.assertNotIn("T", complex_node.attr)
       self.assertNotIn("Tout", complex_node.attr)
 
+  def test_signature_attribute_reserved(self):
+    root = util.Checkpoint(signatures=variables.Variable(1.))
+    save_dir = os.path.join(self.get_temp_dir(), "saved_model")
+    with self.assertRaisesRegexp(ValueError, "del obj.signatures"):
+      save.save(root, save_dir)
+    del root.signatures
+    save.save(root, save_dir)
+
 
 class AssetTests(test.TestCase):
 
diff --git a/tensorflow/python/saved_model/signature_serialization.py b/tensorflow/python/saved_model/signature_serialization.py
new file mode 100644
index 0000000000..19f15f1886
--- /dev/null
+++ b/tensorflow/python/saved_model/signature_serialization.py
@@ -0,0 +1,244 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Helpers for working with signatures in tf.saved_model.save."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import collections
+
+from tensorflow.python.eager import def_function
+from tensorflow.python.eager import function as defun
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_spec
+from tensorflow.python.saved_model import revived_types
+from tensorflow.python.saved_model import signature_constants
+from tensorflow.python.training.checkpointable import base
+from tensorflow.python.util import compat
+from tensorflow.python.util import nest
+
+
+DEFAULT_SIGNATURE_ATTR = "_default_save_signature"
+SIGNATURE_ATTRIBUTE_NAME = "signatures"
+
+
+def _get_signature(function):
+  if (isinstance(function, (defun.Function, def_function.Function)) and
+      function._input_signature is not None):  # pylint: disable=protected-access
+    function = function.get_concrete_function()
+  if not isinstance(function, defun.ConcreteFunction):
+    return None
+  return function
+
+
+def _valid_signature(concrete_function):
+  """Returns whether concrete function can be converted to a signature."""
+  if not concrete_function.outputs:
+    # Functions without outputs don't make sense as signatures. We just don't
+    # have any way to run an Operation with no outputs as a SignatureDef in the
+    # 1.x style.
+    return False
+  try:
+    _normalize_outputs(concrete_function.structured_outputs, "unused", "unused")
+  except ValueError:
+    return False
+  return True
+
+
+def find_function_to_export(saveable_view):
+  """Function to export, None if no suitable function was found."""
+  # If the user did not specify signatures, check the root object for a function
+  # that can be made into a signature.
+  functions = saveable_view.list_functions(saveable_view.root)
+  signature = functions.get(DEFAULT_SIGNATURE_ATTR, None)
+  if signature is not None:
+    return signature
+
+  # TODO(andresp): Discuss removing this behaviour. It can lead to WTFs when a
+  # user decides to annotate more functions with tf.function and suddenly
+  # serving that model way later in the process stops working.
+  possible_signatures = []
+  for function in functions.values():
+    concrete = _get_signature(function)
+    if concrete is not None and _valid_signature(concrete):
+      possible_signatures.append(concrete)
+  if len(possible_signatures) == 1:
+    single_function = possible_signatures[0]
+    signature = _get_signature(single_function)
+    if signature and  _valid_signature(signature):
+      return signature
+  return None
+
+
+def canonicalize_signatures(signatures):
+  """Converts `signatures` into a dictionary of concrete functions."""
+  if signatures is None:
+    return {}
+  if not isinstance(signatures, collections.Mapping):
+    signatures = {
+        signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signatures}
+  concrete_signatures = {}
+  for signature_key, function in signatures.items():
+    signature_function = _get_signature(function)
+    if signature_function is None:
+      raise ValueError(
+          ("Expected a TensorFlow function to generate a signature for, but "
+           "got {}. Only `tf.functions` with an input signature or "
+           "concrete functions can be used as a signature.").format(function))
+
+    # Re-wrap the function so that it only takes keyword arguments and it
+    # returns a dictionary of Tensors. This matches the format of 1.x-style
+    # signatures.
+    # pylint: disable=cell-var-from-loop
+    @def_function.function
+    def signature_wrapper(**kwargs):
+      structured_outputs = signature_function(**kwargs)
+      return _normalize_outputs(
+          structured_outputs, signature_function.name, signature_key)
+    # TODO(b/123902469): Use ConcreteFunction.structured_inputs once their names
+    # always match keyword arguments.
+    tensor_spec_signature = {}
+    for keyword, tensor in zip(
+        signature_function._arg_keywords,  # pylint: disable=protected-access
+        signature_function.inputs):
+      keyword = compat.as_str(keyword)
+      tensor_spec_signature[keyword] = tensor_spec.TensorSpec.from_tensor(
+          tensor, name=keyword)
+    concrete_signatures[signature_key] = (
+        signature_wrapper.get_concrete_function(**tensor_spec_signature))
+    # pylint: enable=cell-var-from-loop
+  return concrete_signatures
+
+
+def _is_flat(sequence):
+  sequence_flat = nest.flatten(sequence)
+  try:
+    nest.assert_same_structure(sequence_flat, sequence)
+    return True
+  except ValueError:
+    return False
+  except TypeError:
+    return False
+
+
+def _normalize_outputs(outputs, function_name, signature_key):
+  """Construct an output dictionary from unnormalized function outputs."""
+  if isinstance(outputs, collections.Mapping):
+    for key, value in outputs.items():
+      if not isinstance(value, ops.Tensor):
+        raise ValueError(
+            ("Got a dictionary containing non-Tensor value {} for key {} "
+             "in the output of the function {} used to generate a SavedModel "
+             "signature. Dictionaries outputs for functions used as signatures "
+             "should have one Tensor output per string key.")
+            .format(value, key, compat.as_str_any(function_name)))
+    return outputs
+  else:
+    original_outputs = outputs
+    if not isinstance(outputs, collections.Sequence):
+      outputs = [outputs]
+    if not _is_flat(outputs):
+      raise ValueError(
+          ("Got non-flat outputs '{}' from '{}' for SavedModel "
+           "signature '{}'. Signatures have one Tensor per output, so "
+           "to have predictable names Python functions used to generate "
+           "these signatures should avoid outputting Tensors in nested "
+           "structures.")
+          .format(original_outputs, function_name, signature_key))
+    return {("output_{}".format(output_index)): output
+            for output_index, output
+            in enumerate(outputs)}
+
+
+# _SignatureMap is immutable to ensure that users do not expect changes to be
+# reflected in the SavedModel. Using public APIs, tf.saved_model.load() is the
+# only way to create a _SignatureMap and there is no way to modify it. So we can
+# safely ignore/overwrite ".signatures" attributes attached to objects being
+# saved if they contain a _SignatureMap. A ".signatures" attribute containing
+# any other type (e.g. a regular dict) will raise an exception asking the user
+# to first "del obj.signatures" if they want it overwritten.
+class _SignatureMap(collections.Mapping, base.Checkpointable):
+  """A collection of SavedModel signatures."""
+
+  def __init__(self):
+    self._signatures = {}
+
+  def _add_signature(self, name, concrete_function):
+    """Adds a signature to the _SignatureMap."""
+    # Ideally this object would be immutable, but restore is streaming so we do
+    # need a private API for adding new signatures to an existing object.
+    self._signatures[name] = concrete_function
+
+  def __getitem__(self, key):
+    return self._signatures[key]
+
+  def __iter__(self):
+    return iter(self._signatures)
+
+  def __len__(self):
+    return len(self._signatures)
+
+  def __repr__(self):
+    return "_SignatureMap({})".format(self._signatures)
+
+  def _list_functions_for_serialization(self):
+    return {
+        key: value for key, value in self.items()
+        if isinstance(value, (def_function.Function, defun.ConcreteFunction))
+    }
+
+
+revived_types.register_revived_type(
+    "signature_map",
+    lambda obj: isinstance(obj, _SignatureMap),
+    versions=[revived_types.VersionedTypeRegistration(
+        # Standard dependencies are enough to reconstruct the checkpointable
+        # items in dictionaries, so we don't need to save any extra information.
+        object_factory=lambda proto: _SignatureMap(),
+        version=1,
+        min_producer_version=1,
+        min_consumer_version=1,
+        setter=_SignatureMap._add_signature  # pylint: disable=protected-access
+    )])
+
+
+def create_signature_map(signatures, saveable_view):
+  """Performs sanity checks and creates an object containing `signatures`."""
+  for name, dep in saveable_view.list_dependencies(
+      saveable_view.root):
+    if name == SIGNATURE_ATTRIBUTE_NAME:
+      if not isinstance(dep, _SignatureMap):
+        raise ValueError(
+            ("Exporting an object {} which has an attribute named "
+             "'{signatures}'. This is a reserved attribute used to store "
+             "SavedModel signatures in objects which come from "
+             "`tf.saved_model.load`. Delete this attribute "
+             "(e.g. 'del obj.{signatures}') before saving if this shadowing is "
+             "acceptable.").format(
+                 saveable_view.root,
+                 signatures=SIGNATURE_ATTRIBUTE_NAME))
+      break
+  signature_map = _SignatureMap()
+  for name, func in signatures.items():
+    # This true of any signature that came from canonicalize_signatures. Here as
+    # a sanity check on saving; crashing on load (e.g. in _add_signature) would
+    # be more problematic in case future export changes violated these
+    # assertions.
+    assert isinstance(func, defun.ConcreteFunction)
+    assert isinstance(func.structured_outputs, collections.Mapping)
+    assert 0 == func._num_positional_args  # pylint: disable=protected-access
+    signature_map._add_signature(name, func)  # pylint: disable=protected-access
+  return signature_map
diff --git a/tensorflow/python/training/checkpointable/graph_view.py b/tensorflow/python/training/checkpointable/graph_view.py
index 1f5f867fb2..46c6289dfc 100644
--- a/tensorflow/python/training/checkpointable/graph_view.py
+++ b/tensorflow/python/training/checkpointable/graph_view.py
@@ -137,25 +137,6 @@ def _serialize_slot_variables(checkpointable_objects, node_ids, object_names):
   return slot_variables
 
 
-def _fill_object_graph_proto(checkpointable_objects,
-                             node_ids,
-                             slot_variables,
-                             object_graph_proto=None):
-  """Name non-slot `Checkpointable`s and add them to `object_graph_proto`."""
-  if object_graph_proto is None:
-    object_graph_proto = (
-        checkpointable_object_graph_pb2.CheckpointableObjectGraph())
-  for checkpoint_id, checkpointable in enumerate(checkpointable_objects):
-    assert node_ids[checkpointable] == checkpoint_id
-    object_proto = object_graph_proto.nodes.add()
-    object_proto.slot_variables.extend(slot_variables.get(checkpointable, ()))
-    for child in checkpointable._checkpoint_dependencies:  # pylint: disable=protected-access
-      child_proto = object_proto.children.add()
-      child_proto.node_id = node_ids[child.ref]
-      child_proto.local_name = child.name
-  return object_graph_proto
-
-
 class ObjectGraphView(object):
   """Gathers and serializes an object graph."""
 
@@ -172,7 +153,7 @@ class ObjectGraphView(object):
     self._root_ref = root
     self._saveables_cache = saveables_cache
 
-  def _list_dependencies(self, obj):
+  def list_dependencies(self, obj):
     # pylint: disable=protected-access
     obj._maybe_initialize_checkpointable()
     return obj._checkpoint_dependencies
@@ -215,7 +196,7 @@ class ObjectGraphView(object):
              "else.")
             % (current_checkpointable,))
       bfs_sorted.append(current_checkpointable)
-      for name, dependency in self._list_dependencies(current_checkpointable):
+      for name, dependency in self.list_dependencies(current_checkpointable):
         if dependency not in path_to_root:
           path_to_root[dependency] = (
               path_to_root[current_checkpointable] + (
@@ -330,6 +311,24 @@ class ObjectGraphView(object):
 
     return named_saveable_objects, feed_additions
 
+  def _fill_object_graph_proto(self, checkpointable_objects,
+                               node_ids,
+                               slot_variables,
+                               object_graph_proto=None):
+    """Name non-slot `Checkpointable`s and add them to `object_graph_proto`."""
+    if object_graph_proto is None:
+      object_graph_proto = (
+          checkpointable_object_graph_pb2.CheckpointableObjectGraph())
+    for checkpoint_id, checkpointable in enumerate(checkpointable_objects):
+      assert node_ids[checkpointable] == checkpoint_id
+      object_proto = object_graph_proto.nodes.add()
+      object_proto.slot_variables.extend(slot_variables.get(checkpointable, ()))
+      for child in self.list_dependencies(checkpointable):
+        child_proto = object_proto.children.add()
+        child_proto.node_id = node_ids[child.ref]
+        child_proto.local_name = child.name
+    return object_graph_proto
+
   def _serialize_gathered_objects(self, checkpointable_objects, path_to_root,
                                   object_map=None):
     """Create SaveableObjects and protos for gathered objects."""
@@ -343,7 +342,7 @@ class ObjectGraphView(object):
         checkpointable_objects=checkpointable_objects,
         node_ids=node_ids,
         object_names=object_names)
-    object_graph_proto = _fill_object_graph_proto(
+    object_graph_proto = self._fill_object_graph_proto(
         checkpointable_objects=checkpointable_objects,
         node_ids=node_ids,
         slot_variables=slot_variables)
-- 
GitLab


From c0b3295568bfcdf71bfba905903930ebf9b79226 Mon Sep 17 00:00:00 2001
From: Jonathan Hseu <jhseu@google.com>
Date: Wed, 6 Feb 2019 14:12:56 -0800
Subject: [PATCH 2027/2345] Move TPU protos to TF core

PiperOrigin-RevId: 232743902
---
 tensorflow/contrib/cmake/python_modules.txt   |  2 +-
 tensorflow/contrib/cmake/python_protos.txt    |  2 +-
 .../contrib/cmake/tf_core_framework.cmake     |  2 +-
 tensorflow/contrib/tpu/BUILD                  | 20 +++++++++----------
 .../contrib/tpu/ops/tpu_embedding_ops.cc      |  4 ++--
 tensorflow/contrib/tpu/python/ops/tpu_ops.py  |  2 +-
 .../contrib/tpu/python/tpu/keras_support.py   |  2 +-
 tensorflow/contrib/tpu/python/tpu/topology.py |  2 +-
 tensorflow/contrib/tpu/python/tpu/tpu.py      |  3 +--
 .../contrib/tpu/python/tpu/tpu_embedding.py   |  4 ++--
 .../contrib/tpu/python/tpu/tpu_estimator.py   |  2 +-
 tensorflow/contrib/tpu/utils/BUILD            |  6 +++---
 ...embedding_optimization_parameters_utils.cc |  2 +-
 ..._embedding_optimization_parameters_utils.h |  2 +-
 .../tpu_embedding_output_layout_utils.cc      |  2 +-
 .../utils/tpu_embedding_output_layout_utils.h |  2 +-
 .../tpu/proto => core/protobuf/tpu}/BUILD     |  0
 .../protobuf/tpu}/compilation_result.proto    |  0
 .../protobuf/tpu}/dynamic_padding.proto       |  0
 .../tpu}/optimization_parameters.proto        |  0
 .../protobuf/tpu}/topology.proto              |  0
 .../tpu}/tpu_embedding_configuration.proto    |  4 ++--
 .../tpu}/tpu_embedding_output_layout.proto    |  0
 23 files changed, 31 insertions(+), 32 deletions(-)
 rename tensorflow/{contrib/tpu/proto => core/protobuf/tpu}/BUILD (100%)
 rename tensorflow/{contrib/tpu/proto => core/protobuf/tpu}/compilation_result.proto (100%)
 rename tensorflow/{contrib/tpu/proto => core/protobuf/tpu}/dynamic_padding.proto (100%)
 rename tensorflow/{contrib/tpu/proto => core/protobuf/tpu}/optimization_parameters.proto (100%)
 rename tensorflow/{contrib/tpu/proto => core/protobuf/tpu}/topology.proto (100%)
 rename tensorflow/{contrib/tpu/proto => core/protobuf/tpu}/tpu_embedding_configuration.proto (97%)
 rename tensorflow/{contrib/tpu/proto => core/protobuf/tpu}/tpu_embedding_output_layout.proto (100%)

diff --git a/tensorflow/contrib/cmake/python_modules.txt b/tensorflow/contrib/cmake/python_modules.txt
index 21ae9a08a6..8b6395304b 100644
--- a/tensorflow/contrib/cmake/python_modules.txt
+++ b/tensorflow/contrib/cmake/python_modules.txt
@@ -13,6 +13,7 @@ tensorflow/core/lib
 tensorflow/core/lib/core
 tensorflow/core/profiler
 tensorflow/core/protobuf
+tensorflow/core/protobuf/tpu
 tensorflow/core/util
 tensorflow/examples
 tensorflow/examples/tutorials
@@ -437,7 +438,6 @@ tensorflow/contrib/timeseries/python/timeseries/state_space_models
 tensorflow/contrib/tpu
 tensorflow/contrib/tpu/ops
 tensorflow/contrib/tpu/profiler
-tensorflow/contrib/tpu/proto
 tensorflow/contrib/tpu/python
 tensorflow/contrib/tpu/python/ops
 tensorflow/contrib/tpu/python/profiler
diff --git a/tensorflow/contrib/cmake/python_protos.txt b/tensorflow/contrib/cmake/python_protos.txt
index 013180c890..b4603206da 100644
--- a/tensorflow/contrib/cmake/python_protos.txt
+++ b/tensorflow/contrib/cmake/python_protos.txt
@@ -1,6 +1,7 @@
 tensorflow/core
 tensorflow/core/kernels/boosted_trees
 tensorflow/core/profiler
+tensorflow/core/protobuf/tpu
 tensorflow/python
 tensorflow/contrib/boosted_trees/proto
 tensorflow/contrib/cloud/kernels
@@ -12,7 +13,6 @@ tensorflow/contrib/mpi_collectives
 tensorflow/contrib/session_bundle
 tensorflow/contrib/tensor_forest/proto
 tensorflow/contrib/tensorboard/plugins/projector
-tensorflow/contrib/tpu/proto
 tensorflow/contrib/tpu/profiler
 tensorflow/contrib/training/python/training
 tensorflow/contrib/verbs
diff --git a/tensorflow/contrib/cmake/tf_core_framework.cmake b/tensorflow/contrib/cmake/tf_core_framework.cmake
index d8d1cc3aa2..cc263d7995 100644
--- a/tensorflow/contrib/cmake/tf_core_framework.cmake
+++ b/tensorflow/contrib/cmake/tf_core_framework.cmake
@@ -125,9 +125,9 @@ endfunction()
 
 file(GLOB_RECURSE tf_protos_cc_srcs RELATIVE ${tensorflow_source_dir}
     "${tensorflow_source_dir}/tensorflow/core/*.proto"
+    "${tensorflow_source_dir}/tensorflow/core/protobuf/tpu/*.proto"
     "${tensorflow_source_dir}/tensorflow/compiler/xla/*.proto"
     "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/proto/*.proto"
-    "${tensorflow_source_dir}/tensorflow/contrib/tpu/proto/*.proto"
 )
 
 RELATIVE_PROTOBUF_GENERATE_CPP(PROTO_SRCS PROTO_HDRS
diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index 1859dee9d0..7b1a501fb7 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -112,12 +112,12 @@ tf_gen_op_libs(
         "functional_ops",
     ],
     deps = [
-        "//tensorflow/contrib/tpu/proto:tpu_embedding_configuration_proto_cc",
         "//tensorflow/contrib/tpu/utils:tpu_embedding_optimization_parameters_utils",
         "//tensorflow/contrib/tpu/utils:tpu_embedding_output_layout_utils",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_proto_parsing",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core/protobuf/tpu:tpu_embedding_configuration_proto_cc",
     ],
 )
 
@@ -134,10 +134,10 @@ tf_custom_op_library(
         "ops/tpu_embedding_ops.cc",
     ],
     deps = [
-        "//tensorflow/contrib/tpu/proto:tpu_embedding_configuration_proto_cc",
         "//tensorflow/contrib/tpu/utils:tpu_embedding_optimization_parameters_utils",
         "//tensorflow/contrib/tpu/utils:tpu_embedding_output_layout_utils",
         "//tensorflow/core:lib_proto_parsing",
+        "//tensorflow/core/protobuf/tpu:tpu_embedding_configuration_proto_cc",
     ],
 )
 
@@ -284,8 +284,8 @@ py_library(
         "//tensorflow/contrib/cluster_resolver:cluster_resolver_py",
         "//tensorflow/contrib/distribute",
         "//tensorflow/contrib/framework:framework_py",
-        "//tensorflow/contrib/tpu/proto:compilation_result_proto_py",
         "//tensorflow/core:protos_all_py",
+        "//tensorflow/core/protobuf/tpu:compilation_result_proto_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
@@ -331,13 +331,13 @@ py_library(
         "//tensorflow/compiler/xla/python_api:xla_shape",
         "//tensorflow/contrib/cluster_resolver:cluster_resolver_py",
         "//tensorflow/contrib/compiler:xla",
-        "//tensorflow/contrib/tpu/proto:compilation_result_proto_py",
-        "//tensorflow/contrib/tpu/proto:dynamic_padding_proto_py",
-        "//tensorflow/contrib/tpu/proto:optimization_parameters_proto_py",
-        "//tensorflow/contrib/tpu/proto:topology_proto_py",
-        "//tensorflow/contrib/tpu/proto:tpu_embedding_configuration_proto_py",
-        "//tensorflow/contrib/tpu/proto:tpu_embedding_output_layout_proto_py",
         "//tensorflow/core:protos_all_py",
+        "//tensorflow/core/protobuf/tpu:compilation_result_proto_py",
+        "//tensorflow/core/protobuf/tpu:dynamic_padding_proto_py",
+        "//tensorflow/core/protobuf/tpu:optimization_parameters_proto_py",
+        "//tensorflow/core/protobuf/tpu:topology_proto_py",
+        "//tensorflow/core/protobuf/tpu:tpu_embedding_configuration_proto_py",
+        "//tensorflow/core/protobuf/tpu:tpu_embedding_output_layout_proto_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:control_flow_util",
@@ -469,7 +469,7 @@ py_library(
     deps = [
         ":tpu_lib",
         ":tpu_ops",
-        "//tensorflow/contrib/tpu/proto:tpu_embedding_configuration_proto_py",
+        "//tensorflow/core/protobuf/tpu:tpu_embedding_configuration_proto_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:init_ops",
diff --git a/tensorflow/contrib/tpu/ops/tpu_embedding_ops.cc b/tensorflow/contrib/tpu/ops/tpu_embedding_ops.cc
index 676aed0b7b..b991698359 100644
--- a/tensorflow/contrib/tpu/ops/tpu_embedding_ops.cc
+++ b/tensorflow/contrib/tpu/ops/tpu_embedding_ops.cc
@@ -13,7 +13,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tpu/proto/tpu_embedding_configuration.pb.h"
 #include "tensorflow/contrib/tpu/utils/tpu_embedding_optimization_parameters_utils.h"
 #include "tensorflow/contrib/tpu/utils/tpu_embedding_output_layout_utils.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
@@ -23,6 +22,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/lib/strings/stringprintf.h"
+#include "tensorflow/core/protobuf/tpu/tpu_embedding_configuration.pb.h"
 
 namespace tensorflow {
 
@@ -466,7 +466,7 @@ inputs: A TensorList of gradients with which to update embedding tables.
     configuration given to tpu.initialize_system.
 learning_rates: A TensorList of float32 scalars, one for each dynamic learning
     rate tag: see the comments in
-    //third_party/tensorflow/contrib/tpu/proto/optimization_parameters.proto.
+    //third_party/tensorflow/core/protobuf/tpu/optimization_parameters.proto.
     Multiple tables can share the same dynamic learning rate tag as specified
     in the configuration. If the learning rates for all tables are constant,
     this list should be empty.
diff --git a/tensorflow/contrib/tpu/python/ops/tpu_ops.py b/tensorflow/contrib/tpu/python/ops/tpu_ops.py
index 500dd2cd39..55f7c6bcbc 100644
--- a/tensorflow/contrib/tpu/python/ops/tpu_ops.py
+++ b/tensorflow/contrib/tpu/python/ops/tpu_ops.py
@@ -225,7 +225,7 @@ if platform.system() != "Windows":
       config: Serialized TPUEmbeddingConfiguration proto.
       learning_rates: A TensorList of float32 scalars, one for each dynamic
           learning rate tag: see the comments in
-          //third_party/tensorflow/contrib/tpu/proto/
+          //third_party/tensorflow/core/protobuf/tpu/
                                                optimization_parameters.proto.
           Multiple tables can share the same dynamic learning rate tag as
           specified in the configuration. If the learning rates for all tables
diff --git a/tensorflow/contrib/tpu/python/tpu/keras_support.py b/tensorflow/contrib/tpu/python/tpu/keras_support.py
index 37fe9af8c4..4322d1708e 100644
--- a/tensorflow/contrib/tpu/python/tpu/keras_support.py
+++ b/tensorflow/contrib/tpu/python/tpu/keras_support.py
@@ -56,7 +56,6 @@ import six
 
 from tensorflow.contrib.cluster_resolver.python.training import tpu_cluster_resolver as tpu_cluster_resolver_lib
 from tensorflow.contrib.framework.python.framework import experimental
-from tensorflow.contrib.tpu.proto import compilation_result_pb2 as tpu_compilation_result
 from tensorflow.contrib.tpu.python.ops import tpu_ops
 from tensorflow.contrib.tpu.python.tpu import keras_tpu_variables
 from tensorflow.contrib.tpu.python.tpu import tpu
@@ -64,6 +63,7 @@ from tensorflow.contrib.tpu.python.tpu import tpu_function
 from tensorflow.contrib.tpu.python.tpu import tpu_optimizer
 from tensorflow.contrib.tpu.python.tpu import tpu_system_metadata as tpu_system_metadata_lib
 from tensorflow.core.protobuf import config_pb2
+from tensorflow.core.protobuf.tpu import compilation_result_pb2 as tpu_compilation_result
 from tensorflow.python.client import session as tf_session
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
diff --git a/tensorflow/contrib/tpu/python/tpu/topology.py b/tensorflow/contrib/tpu/python/tpu/topology.py
index 6ae718cc2c..00ee21e694 100644
--- a/tensorflow/contrib/tpu/python/tpu/topology.py
+++ b/tensorflow/contrib/tpu/python/tpu/topology.py
@@ -21,7 +21,7 @@ from __future__ import print_function
 import numpy as np
 from six.moves import xrange  # pylint: disable=redefined-builtin
 
-from tensorflow.contrib.tpu.proto import topology_pb2
+from tensorflow.core.protobuf.tpu import topology_pb2
 
 
 def _tpu_device_name(job, task, device):
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu.py b/tensorflow/contrib/tpu/python/tpu/tpu.py
index de2bfd49ec..673129b4be 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu.py
@@ -24,11 +24,10 @@ from six.moves import xrange  # pylint: disable=redefined-builtin
 
 from tensorflow.contrib.compiler import xla
 from tensorflow.contrib.framework.python.framework import experimental
-from tensorflow.contrib.tpu.proto import dynamic_padding_pb2 as dynamic_padding
 from tensorflow.contrib.tpu.python.ops import tpu_ops
 from tensorflow.contrib.tpu.python.tpu import tpu_function
-
 from tensorflow.core.framework import attr_value_pb2
+from tensorflow.core.protobuf.tpu import dynamic_padding_pb2 as dynamic_padding
 from tensorflow.python.compat import compat as api_compat
 from tensorflow.python.framework import device as pydev
 from tensorflow.python.framework import dtypes
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py b/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py
index 1689126dbc..8731b93411 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py
@@ -26,10 +26,10 @@ import six
 
 from tensorflow.contrib.framework.python.framework import experimental
 from tensorflow.contrib.tpu.ops import gen_tpu_ops
-from tensorflow.contrib.tpu.proto import optimization_parameters_pb2
-from tensorflow.contrib.tpu.proto import tpu_embedding_configuration_pb2 as elc
 from tensorflow.contrib.tpu.python.ops import tpu_ops
 from tensorflow.contrib.tpu.python.tpu import tpu_system_metadata as tpu_system_metadata_lib
+from tensorflow.core.protobuf.tpu import optimization_parameters_pb2
+from tensorflow.core.protobuf.tpu import tpu_embedding_configuration_pb2 as elc
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 14dca03568..a372d44b2c 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -31,7 +31,6 @@ import six
 from six.moves import queue as Queue  # pylint: disable=redefined-builtin
 from six.moves import xrange  # pylint: disable=redefined-builtin
 
-from tensorflow.contrib.tpu.proto import compilation_result_pb2 as tpu_compilation_result
 from tensorflow.contrib.tpu.python.ops import tpu_ops
 from tensorflow.contrib.tpu.python.ops import tpu_ordinal_selector_op
 from tensorflow.contrib.tpu.python.tpu import _tpu_estimator_embedding
@@ -51,6 +50,7 @@ from tensorflow.contrib.training.python.training import hparam
 from tensorflow.core.framework import variable_pb2
 from tensorflow.core.framework.summary_pb2 import Summary
 from tensorflow.core.protobuf import config_pb2
+from tensorflow.core.protobuf.tpu import compilation_result_pb2 as tpu_compilation_result
 from tensorflow.python.client import session as tf_session
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import nest as data_nest
diff --git a/tensorflow/contrib/tpu/utils/BUILD b/tensorflow/contrib/tpu/utils/BUILD
index c27b737287..5cbed402f7 100644
--- a/tensorflow/contrib/tpu/utils/BUILD
+++ b/tensorflow/contrib/tpu/utils/BUILD
@@ -8,9 +8,9 @@ cc_library(
     hdrs = ["tpu_embedding_optimization_parameters_utils.h"],
     visibility = ["//visibility:public"],
     deps = [
-        "//tensorflow/contrib/tpu/proto:optimization_parameters_proto_cc",
         "//tensorflow/core:framework_headers_lib",
         "//tensorflow/core:lib_proto_parsing",
+        "//tensorflow/core/protobuf/tpu:optimization_parameters_proto_cc",
         "@com_google_absl//absl/base",
     ],
 )
@@ -21,10 +21,10 @@ cc_library(
     hdrs = ["tpu_embedding_output_layout_utils.h"],
     visibility = ["//visibility:public"],
     deps = [
-        "//tensorflow/contrib/tpu/proto:tpu_embedding_configuration_proto_cc",
-        "//tensorflow/contrib/tpu/proto:tpu_embedding_output_layout_proto_cc",
         "//tensorflow/core:framework_headers_lib",
         "//tensorflow/core:lib_proto_parsing",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core/protobuf/tpu:tpu_embedding_configuration_proto_cc",
+        "//tensorflow/core/protobuf/tpu:tpu_embedding_output_layout_proto_cc",
     ],
 )
diff --git a/tensorflow/contrib/tpu/utils/tpu_embedding_optimization_parameters_utils.cc b/tensorflow/contrib/tpu/utils/tpu_embedding_optimization_parameters_utils.cc
index d98e0b7a5e..d1df7e78ab 100644
--- a/tensorflow/contrib/tpu/utils/tpu_embedding_optimization_parameters_utils.cc
+++ b/tensorflow/contrib/tpu/utils/tpu_embedding_optimization_parameters_utils.cc
@@ -135,7 +135,7 @@ Status GetGradientAccumulationSupport(OptimizationAlgorithm alg,
 }
 namespace {
 // Make a normal state variable specification. Please refer to
-// //third_party/tensorflow/contrib/tpu/proto/optimization_parameters.proto
+// //tensorflow/core/protobuf/tpu/optimization_parameters.proto
 // (StateVariableSpecification message) for instructions on how to set the
 // padding_initial_value field.
 StateVariableSpecification MakeStandardStateVariableSpecification(
diff --git a/tensorflow/contrib/tpu/utils/tpu_embedding_optimization_parameters_utils.h b/tensorflow/contrib/tpu/utils/tpu_embedding_optimization_parameters_utils.h
index 81d50264ed..7a7833bf2d 100644
--- a/tensorflow/contrib/tpu/utils/tpu_embedding_optimization_parameters_utils.h
+++ b/tensorflow/contrib/tpu/utils/tpu_embedding_optimization_parameters_utils.h
@@ -18,8 +18,8 @@ limitations under the License.
 
 #include <string>
 #include "absl/base/casts.h"
-#include "tensorflow/contrib/tpu/proto/optimization_parameters.pb.h"
 #include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/protobuf/tpu/optimization_parameters.pb.h"
 
 namespace tensorflow {
 namespace tpu {
diff --git a/tensorflow/contrib/tpu/utils/tpu_embedding_output_layout_utils.cc b/tensorflow/contrib/tpu/utils/tpu_embedding_output_layout_utils.cc
index 8480ec4b8b..e65abe3894 100644
--- a/tensorflow/contrib/tpu/utils/tpu_embedding_output_layout_utils.cc
+++ b/tensorflow/contrib/tpu/utils/tpu_embedding_output_layout_utils.cc
@@ -14,8 +14,8 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/contrib/tpu/utils/tpu_embedding_output_layout_utils.h"
-#include "tensorflow/contrib/tpu/proto/tpu_embedding_output_layout.pb.h"
 #include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/protobuf/tpu/tpu_embedding_output_layout.pb.h"
 
 namespace tensorflow {
 namespace tpu {
diff --git a/tensorflow/contrib/tpu/utils/tpu_embedding_output_layout_utils.h b/tensorflow/contrib/tpu/utils/tpu_embedding_output_layout_utils.h
index c10fbeeff2..1a04c7bdb4 100644
--- a/tensorflow/contrib/tpu/utils/tpu_embedding_output_layout_utils.h
+++ b/tensorflow/contrib/tpu/utils/tpu_embedding_output_layout_utils.h
@@ -16,9 +16,9 @@ limitations under the License.
 #ifndef TENSORFLOW_CONTRIB_TPU_UTILS_TPU_EMBEDDING_OUTPUT_LAYOUT_UTILS_H_
 #define TENSORFLOW_CONTRIB_TPU_UTILS_TPU_EMBEDDING_OUTPUT_LAYOUT_UTILS_H_
 
-#include "tensorflow/contrib/tpu/proto/tpu_embedding_configuration.pb.h"
 #include "tensorflow/core/framework/tensor_shape.pb.h"
 #include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/protobuf/tpu/tpu_embedding_configuration.pb.h"
 
 namespace tensorflow {
 namespace tpu {
diff --git a/tensorflow/contrib/tpu/proto/BUILD b/tensorflow/core/protobuf/tpu/BUILD
similarity index 100%
rename from tensorflow/contrib/tpu/proto/BUILD
rename to tensorflow/core/protobuf/tpu/BUILD
diff --git a/tensorflow/contrib/tpu/proto/compilation_result.proto b/tensorflow/core/protobuf/tpu/compilation_result.proto
similarity index 100%
rename from tensorflow/contrib/tpu/proto/compilation_result.proto
rename to tensorflow/core/protobuf/tpu/compilation_result.proto
diff --git a/tensorflow/contrib/tpu/proto/dynamic_padding.proto b/tensorflow/core/protobuf/tpu/dynamic_padding.proto
similarity index 100%
rename from tensorflow/contrib/tpu/proto/dynamic_padding.proto
rename to tensorflow/core/protobuf/tpu/dynamic_padding.proto
diff --git a/tensorflow/contrib/tpu/proto/optimization_parameters.proto b/tensorflow/core/protobuf/tpu/optimization_parameters.proto
similarity index 100%
rename from tensorflow/contrib/tpu/proto/optimization_parameters.proto
rename to tensorflow/core/protobuf/tpu/optimization_parameters.proto
diff --git a/tensorflow/contrib/tpu/proto/topology.proto b/tensorflow/core/protobuf/tpu/topology.proto
similarity index 100%
rename from tensorflow/contrib/tpu/proto/topology.proto
rename to tensorflow/core/protobuf/tpu/topology.proto
diff --git a/tensorflow/contrib/tpu/proto/tpu_embedding_configuration.proto b/tensorflow/core/protobuf/tpu/tpu_embedding_configuration.proto
similarity index 97%
rename from tensorflow/contrib/tpu/proto/tpu_embedding_configuration.proto
rename to tensorflow/core/protobuf/tpu/tpu_embedding_configuration.proto
index da19b135d7..53280edfa6 100644
--- a/tensorflow/contrib/tpu/proto/tpu_embedding_configuration.proto
+++ b/tensorflow/core/protobuf/tpu/tpu_embedding_configuration.proto
@@ -2,8 +2,8 @@ syntax = "proto3";
 
 package tensorflow.tpu;
 
-import "tensorflow/contrib/tpu/proto/optimization_parameters.proto";
-import "tensorflow/contrib/tpu/proto/tpu_embedding_output_layout.proto";
+import "tensorflow/core/protobuf/tpu/optimization_parameters.proto";
+import "tensorflow/core/protobuf/tpu/tpu_embedding_output_layout.proto";
 
 message TPUEmbeddingConfiguration {
   // Description of the various embedding tables.
diff --git a/tensorflow/contrib/tpu/proto/tpu_embedding_output_layout.proto b/tensorflow/core/protobuf/tpu/tpu_embedding_output_layout.proto
similarity index 100%
rename from tensorflow/contrib/tpu/proto/tpu_embedding_output_layout.proto
rename to tensorflow/core/protobuf/tpu/tpu_embedding_output_layout.proto
-- 
GitLab


From 4b327d4fb57f0e67dc100fd9da162bf780d6f882 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Wed, 6 Feb 2019 14:22:13 -0800
Subject: [PATCH 2028/2345] Minor changes to _EagerContext to make construction
 lazy

Since any new python thread (or PyThreadState) might try to create this state,
and not use all of it, we try to delay construction of the entire thing till it
is used.

PiperOrigin-RevId: 232745940
---
 tensorflow/python/eager/context.py | 43 ++++++++++++++++++++++++------
 1 file changed, 35 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py
index fd9be060ac..23184143d6 100644
--- a/tensorflow/python/eager/context.py
+++ b/tensorflow/python/eager/context.py
@@ -44,6 +44,7 @@ default_execution_mode = EAGER_MODE if tf2.enabled() else GRAPH_MODE
 # Note that we do not protect this with a lock and instead rely on python's GIL
 # and the idempotent nature of writes to provide thread safety.
 _device_parsing_cache = {}
+_starting_device_spec = pydev.DeviceSpec.from_string("")
 
 _MAXINT32 = 2**31 - 1
 
@@ -135,26 +136,52 @@ class _EagerContext(threading.local):
 
   def __init__(self, config=None):
     super(_EagerContext, self).__init__()
-    self.device_spec = pydev.DeviceSpec.from_string("")
-    self.device_name = self.device_spec.to_string()
+    self.device_spec = _starting_device_spec
+    self.device_name = ""
     self.mode = default_execution_mode
     self.is_eager = default_execution_mode == EAGER_MODE
     self.scope_name = ""
     self.recording_summaries = False
     self.summary_writer_resource = None
     self.scalar_cache = {}
-    self.ones_rank_cache = _EagerTensorCache()
-    self.zeros_cache = _EagerTensorCache()
+    self._ones_rank_cache = None
+    self._zeros_cache = None
     self.execution_mode = None
 
     # Default rewriter config corresponds to turning all default grappler
     # optimizations on.
-    base_config = config_pb2.ConfigProto()
+    self._config = config
+
+    self._function_call_options = None
+
+  @property
+  def function_call_options(self):
+    if self._function_call_options is None:
+      base_config = config_pb2.ConfigProto()
+      if self._config is not None:
+        base_config.MergeFrom(self._config)
+      self._config = None
+      self._function_call_options = FunctionCallOptions(
+          config_proto=base_config)
+
+    return self._function_call_options
+
+  @function_call_options.setter
+  def function_call_options(self, function_call_options):
+    self._function_call_options = function_call_options
+    self._config = None
 
-    if config is not None:
-      base_config.MergeFrom(config)
+  @property
+  def ones_rank_cache(self):
+    if not self._ones_rank_cache:
+      self._ones_rank_cache = _EagerTensorCache()
+    return self._ones_rank_cache
 
-    self.function_call_options = FunctionCallOptions(config_proto=base_config)
+  @property
+  def zeros_cache(self):
+    if not self._zeros_cache:
+      self._zeros_cache = _EagerTensorCache()
+    return self._zeros_cache
 
 
 ContextSwitch = collections.namedtuple(
-- 
GitLab


From 929d4992784b2dca96cc6f0f4bc6d9f4fcac493d Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Wed, 6 Feb 2019 14:23:04 -0800
Subject: [PATCH 2029/2345] Don't add unnecessary Merge nodes when building XLA
 ops

PiperOrigin-RevId: 232746100
---
 tensorflow/compiler/jit/build_xla_ops_pass.cc |  7 +++++++
 .../compiler/jit/build_xla_ops_pass_test.cc   | 20 +++++++++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/tensorflow/compiler/jit/build_xla_ops_pass.cc b/tensorflow/compiler/jit/build_xla_ops_pass.cc
index 9f4042630e..285b1efa53 100644
--- a/tensorflow/compiler/jit/build_xla_ops_pass.cc
+++ b/tensorflow/compiler/jit/build_xla_ops_pass.cc
@@ -115,6 +115,13 @@ void MergeOutgoingControlEdges(const Scope& s, Node* old_node, Node* new_node) {
     return;
   }
 
+  if (ctrl_edges.size() == 1 && ctrl_edges.front()->dst()->IsSink()) {
+    // Avoid creating a Merge node if we can just add an edge to _SINK
+    // instead.
+    s.graph()->AddControlEdge(new_node, s.graph()->sink_node());
+    return;
+  }
+
   // We can't merge control edges directly so we instead first "convert" them to
   // normal values that can be merged, merge the values and then "convert" the
   // merged value back into control.
diff --git a/tensorflow/compiler/jit/build_xla_ops_pass_test.cc b/tensorflow/compiler/jit/build_xla_ops_pass_test.cc
index 390ffa694b..c14c7465c5 100644
--- a/tensorflow/compiler/jit/build_xla_ops_pass_test.cc
+++ b/tensorflow/compiler/jit/build_xla_ops_pass_test.cc
@@ -68,6 +68,8 @@ Status BuildXlaOps(const Scope& s, std::unique_ptr<Graph>* result) {
     }
   }
 
+  FixupSourceAndSinkEdges(graph.get());
+
   GraphOptimizationPassOptions opt_options;
   opt_options.graph = &graph;
   BuildXlaOpsPass pass(/*enable_lazy_compilation=*/true);
@@ -223,5 +225,23 @@ TEST_F(BuildXlaOpsTest, OnXlaDevice) {
   ASSERT_NE(write_op_new, nullptr);
   EXPECT_THAT(write_op_new, assign_var);
 }
+
+TEST_F(BuildXlaOpsTest, NoExtraMergeForEdgeToSink) {
+  Scope root = Scope::NewRootScope().ExitOnError();
+
+  FunctionDefLibrary flib_def =
+      CreateFunctionDefLibWithConstFunction("cluster_0");
+  TF_ASSERT_OK(root.graph()->AddFunctionLibrary(flib_def));
+  Node* call;
+  TF_ASSERT_OK(MakeXlaCompiledKernel(root.graph(), "cluster_0", "C", &call));
+
+  std::unique_ptr<Graph> graph;
+  TF_ASSERT_OK(BuildXlaOps(root, &graph));
+
+  Node* sink_node = graph->sink_node();
+  EXPECT_THAT(sink_node, NodeWith(CtrlDeps(NodeWith(Op("_XlaRun")),
+                                           NodeWith(Op("cluster_0")),
+                                           NodeWith(Op("NoOp")))));
+}
 }  // namespace
 }  // namespace tensorflow
-- 
GitLab


From c855674062cb0aee26198741244b8d23f7ab3f35 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 6 Feb 2019 14:23:31 -0800
Subject: [PATCH 2030/2345] Handle symbols that are only defined inside loops.
 This is valid in Python as long as the loop always runs at least once. A
 runtime error is thrown if the loop doesn't execute and the undefined symbol
 is accessed later in the program. An error will be thrown by the TensorFlow
 runtime even if the undefined symbols would not be used at runtime.

PiperOrigin-RevId: 232746169
---
 .../autograph/converters/control_flow.py      | 33 +++++++------------
 .../autograph/converters/control_flow_test.py | 21 ------------
 2 files changed, 12 insertions(+), 42 deletions(-)

diff --git a/tensorflow/python/autograph/converters/control_flow.py b/tensorflow/python/autograph/converters/control_flow.py
index 0416820594..2526d30257 100644
--- a/tensorflow/python/autograph/converters/control_flow.py
+++ b/tensorflow/python/autograph/converters/control_flow.py
@@ -278,26 +278,15 @@ class ControlFlowTransformer(converter.Base):
     # the loop state, regardless of whether they are later used or not.
     loop_state = body_scope.modified & live_in
 
-    undefined_lives = loop_state - defined_in
+    # Variable that are used or defined inside the loop, but not defined
+    # before entering the loop
+    undefined_lives = ((loop_state - defined_in)
+                       | ((body_scope.modified - live_in) & live_out))
     # Only simple variables must be defined. The composite ones will be
     # implicitly checked at runtime.
     undefined_simple_lives = {v for v in undefined_lives if v.is_simple()}
-    if undefined_simple_lives:
-      raise NameError(
-          'cannot convert loop: it includes symbols that are undefined'
-          ' when entering the loop: {}'.format(
-              self._fmt_symbols(undefined_simple_lives)))
-
-    live_defs_in_loop = (body_scope.modified - live_in) & live_out
-    if live_defs_in_loop:
-      # TODO(mdan): Include reference to explanation why.
-      raise NotImplementedError(
-          'cannot convert loop: it includes symbols that are defined'
-          ' inside the loop, but used later: {}. To fix, initialize'
-          ' these symbols before the loop'.format(
-              self._fmt_symbols(live_defs_in_loop)))
-
-    return loop_state, reserved_symbols
+
+    return loop_state, reserved_symbols, undefined_simple_lives
 
   def _state_constructs(self, loop_state, reserved_symbols):
     loop_state = list(loop_state)
@@ -321,7 +310,7 @@ class ControlFlowTransformer(converter.Base):
   def visit_While(self, node):
     self.generic_visit(node)
 
-    loop_state, reserved_symbols = self._get_loop_state(node)
+    loop_state, reserved_symbols, possibly_undef = self._get_loop_state(node)
 
     # Note: one might expect we can dispatch based on the loop condition.
     # But because that is dependent on the state, it cannot be evaluated ahead
@@ -386,12 +375,13 @@ class ControlFlowTransformer(converter.Base):
           extra_deps=tuple(s.ast() for s in cond_closure),
       )
 
-    return node
+    undefined_assigns = self._create_undefined_assigns(possibly_undef)
+    return undefined_assigns + node
 
   def visit_For(self, node):
     self.generic_visit(node)
 
-    loop_state, reserved_symbols = self._get_loop_state(node)
+    loop_state, reserved_symbols, possibly_undef = self._get_loop_state(node)
     loop_state, state_ssf, state_ast_tuple, ssf_map = self._state_constructs(
         loop_state, reserved_symbols)
     node_body = ast_util.rename_symbols(node.body, ssf_map)
@@ -446,7 +436,8 @@ class ControlFlowTransformer(converter.Base):
           body_name=self.ctx.namer.new_symbol('loop_body', reserved_symbols),
           body=node_body)
 
-    return node
+    undefined_assigns = self._create_undefined_assigns(possibly_undef)
+    return undefined_assigns + node
 
 
 def transform(node, ctx):
diff --git a/tensorflow/python/autograph/converters/control_flow_test.py b/tensorflow/python/autograph/converters/control_flow_test.py
index 1e73dcafa1..d3accd3c1e 100644
--- a/tensorflow/python/autograph/converters/control_flow_test.py
+++ b/tensorflow/python/autograph/converters/control_flow_test.py
@@ -78,17 +78,6 @@ class ControlFlowTest(converter_testing.TestCase):
 
     self.assertTransformedResult(test_fn, constant_op.constant(5), 0)
 
-  def test_while_variable_defined_in_body(self):
-    def bad_while_loop(n):
-      while n > 0:
-        n -= 1
-        s = n
-      return s
-
-    node, ctx = self.prepare(bad_while_loop, {})
-    with self.assertRaises(NameError):
-      control_flow.transform(node, ctx)
-
   @test_util.run_deprecated_v1
   def test_if_basic(self):
 
@@ -225,16 +214,6 @@ class ControlFlowTest(converter_testing.TestCase):
       self.assertEqual(result.test_fn(5), 10)
       self.assertEqual(eval_count[0], 1)
 
-  def test_for_variable_defined_in_body(self):
-    def bad_for_loop(n):
-      for i in range(n):
-        s = i
-      return s
-
-    node, ctx = self.prepare(bad_for_loop, {})
-    with self.assertRaises(NameError):
-      control_flow.transform(node, ctx)
-
   @test_util.run_deprecated_v1
   def test_for_tuple_unpacking(self):
     def test_fn(x_list):
-- 
GitLab


From 0064ad1e1c0fe79832cd8cbb588338683b38c937 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Wed, 6 Feb 2019 14:24:03 -0800
Subject: [PATCH 2031/2345] Don't rewrite slices that have a constant begin
 input

PiperOrigin-RevId: 232746272
---
 .../increase_dynamism_for_auto_jit_pass.cc    | 17 ++++++++++-----
 ...ncrease_dynamism_for_auto_jit_pass_test.cc | 21 +++++++++++++++++++
 2 files changed, 33 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass.cc b/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass.cc
index ebfffc3267..5287fd175d 100644
--- a/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass.cc
+++ b/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass.cc
@@ -247,6 +247,7 @@ Status ConvertTensorFlowSliceToStaticShapedSlice(
           .NewSubScope(absl::StrCat(slice->name(), "/static_shaped_slice"));
   Scope host_scope = main_scope.WithAssignedDevice(host_name);
 
+  // In the future we may want to be clever here and avoid the extra Cast ops.
   SliceInputs slice_inputs_int64 =
       MakeSliceIndexAndSizeInt64(host_scope, slice_inputs);
 
@@ -312,9 +313,9 @@ Status RewriteSlice(Graph* g, Node* slice, const SliceInputs& slice_inputs,
   return Status::OK();
 }
 
-// Return true if `n` is a slice we can rewrite to have a static shape
+// Return true if `n` is a slice we should rewrite to have a static shape
 // (i.e. have the output shape only depend on the "size" input).
-xla::StatusOr<bool> IsRewritableSlice(Node* n) {
+xla::StatusOr<bool> ShouldRewriteSlice(Node* n) {
   if (n->type_string() != "Slice") {
     return false;
   }
@@ -332,14 +333,20 @@ xla::StatusOr<bool> IsRewritableSlice(Node* n) {
 
   // If slice_size[i] < -1 for any i then executing the slice will throw an
   // error, and we don't do anything here.
-  return absl::c_all_of(slice_inputs->size_as_vector,
-                        [](int64 size_i) { return size_i >= -1; });
+  bool slice_size_has_error = absl::c_all_of(
+      slice_inputs->size_as_vector, [](int64 size_i) { return size_i >= -1; });
+  if (!slice_size_has_error) {
+    return false;
+  }
+
+  // No point in rewriting slices that have both size and begin as constants.
+  return !slice_inputs->begin.node()->IsConstant();
 }
 
 Status FindAndRewriteSlices(Graph* g, bool* changed) {
   std::vector<Node*> slices_to_rewrite;
   for (Node* n : g->nodes()) {
-    TF_ASSIGN_OR_RETURN(bool is_rewritable, IsRewritableSlice(n));
+    TF_ASSIGN_OR_RETURN(bool is_rewritable, ShouldRewriteSlice(n));
     if (is_rewritable) {
       slices_to_rewrite.push_back(n);
     }
diff --git a/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass_test.cc b/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass_test.cc
index 32e30216de..2add2c13f9 100644
--- a/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass_test.cc
+++ b/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass_test.cc
@@ -432,5 +432,26 @@ TEST(SliceToDynamicSliceRewriteTest, WithControlDepsToConstant) {
                                                       Name("dependency")))));
 }
 
+TEST(SliceToDynamicSliceRewriteTest, DontRewriteSliceWithConstBegin) {
+  Scope root = Scope::NewRootScope()
+                   .ExitOnError()
+                   .WithAssignedDevice(kDeviceName)
+                   .WithXlaCluster("cluster_0");
+
+  Output input = ops::Placeholder(root.WithOpName("input"), DT_FLOAT);
+  Output begin = ops::Const(root.WithOpName("begin"), {10, 10});
+  Output size = ops::Const(root.WithOpName("size"), {-1, 500});
+  Output slice = ops::Slice(root.WithOpName("slice"), input, begin, size);
+
+  std::unique_ptr<Graph> result;
+  TF_ASSERT_OK(IncreaseDynamismForAutoJit(root, &result));
+
+  Node* slice_node = testing::FindNodeByName(result.get(), "slice");
+  EXPECT_THAT(slice_node,
+              NodeWith(Op("Slice"), Inputs(Out(NodeWith(Op("Placeholder"))),
+                                           Out(NodeWith(Op("Const"))),
+                                           Out(NodeWith(Op("Const"))))));
+}
+
 }  // namespace
 }  // namespace tensorflow
-- 
GitLab


From 95d958336276d1b054242ca2506f267c7de145dc Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Wed, 6 Feb 2019 14:35:03 -0800
Subject: [PATCH 2032/2345] Update documentation with correct path.

PiperOrigin-RevId: 232748497
---
 tensorflow/lite/tools/benchmark/ios/README.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/lite/tools/benchmark/ios/README.md b/tensorflow/lite/tools/benchmark/ios/README.md
index 8142f48ad9..ee880f005d 100644
--- a/tensorflow/lite/tools/benchmark/ios/README.md
+++ b/tensorflow/lite/tools/benchmark/ios/README.md
@@ -27,9 +27,9 @@ Running
 tensorflow/lite/tools/make/build_ios_universal_lib.sh
 ```
 
-will also build `tensorflow/lite/gen/lib/benchmark-lib.a` .
+will also build `tensorflow/lite/tools/make/gen/lib/benchmark-lib.a` .
 
-- Now copy the downloaded model file to `benchmark_data` directory. 
+- Now copy the downloaded model file to `benchmark_data` directory.
 
 - Modify `benchmark_params.json` change the `input_layer`, `input_layer_shape`
 and other benchmark parameters.
@@ -37,8 +37,8 @@ and other benchmark parameters.
 - Change `Build Phases -> Copy Bundle Resources` and add the model file to the
 resources that need to be copied.
 
-- Ensure that `Build Phases -> Link Binary With Library` contains the 
-`Accelerate framework` and `tensorflow/lite/gen/lib/benchmark-lib.a`.
+- Ensure that `Build Phases -> Link Binary With Library` contains the
+`Accelerate framework` and `tensorflow/lite/tools/make/gen/lib/benchmark-lib.a`.
 
 - Now try running the app. The app has a single button that runs the benchmark
   on the model and displays results in a text view below.
@@ -48,7 +48,7 @@ resources that need to be copied.
 If you want detailed profiling, use the following command:
 
 ```bash
-tensorflow/lite/build_ios_universal_lib.sh -p
+tensorflow/lite/tools/make/build_ios_universal_lib.sh -p
 ```
 
 Then following the same steps above and run the benchmark app. You will see the
-- 
GitLab


From 4900e8a17367aa8c158e5e783d4776ffc206a77b Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Wed, 6 Feb 2019 14:47:48 -0800
Subject: [PATCH 2033/2345] Automated rollback of commit
 1f856fb5d978177123ddd5ac5a3e4bb669288d65

PiperOrigin-RevId: 232750954
---
 tensorflow/python/keras/backend.py |   4 +-
 tensorflow/python/ops/nn_ops.py    | 113 ++++++++---------------------
 2 files changed, 31 insertions(+), 86 deletions(-)

diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index 32820fef5f..064358cacd 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -4143,8 +4143,8 @@ def conv1d(x,
   x = nn.convolution(
       input=x,
       filter=kernel,
-      dilation_rate=dilation_rate,
-      strides=strides,
+      dilation_rate=(dilation_rate,),
+      strides=(strides,),
       padding=padding,
       data_format=tf_data_format)
   if data_format == 'channels_first' and tf_data_format == 'NWC':
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index f04a3ae293..bd70d9f09a 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -65,9 +65,7 @@ def _get_sequence(value, n, channel_index, name):
     return value
   elif current_n == 1:
     value = list((value[0],) * n)
-  elif current_n == n:
-    value = list(value)
-  else:
+  elif current_n != n:
     raise ValueError("{} should be of length 1, {} or {} but was {}".format(
         name, n, n + 2, current_n))
 
@@ -883,14 +881,21 @@ def convolution(
   filter = deprecated_argument_lookup("filters", filters, "filter", filter)
   dilation_rate = deprecated_argument_lookup(
       "dilations", dilations, "dilation_rate", dilation_rate)
-  return convolution_internal(
-      input,
-      filter,
-      strides=strides,
-      padding=padding,
-      data_format=data_format,
-      dilations=dilation_rate,
-      name=name)
+  # pylint: enable=line-too-long
+  with ops.name_scope(name, "convolution", [input, filter]) as name:
+    input = ops.convert_to_tensor(input, name="input")  # pylint: disable=redefined-builtin
+    input_shape = input.get_shape()
+    filter = ops.convert_to_tensor(filter, name="filter")  # pylint: disable=redefined-builtin
+    filter_shape = filter.get_shape()
+    op = Convolution(
+        input_shape,
+        filter_shape,
+        padding,
+        strides=strides,
+        dilation_rate=dilation_rate,
+        name=name,
+        data_format=data_format)
+    return op(input, filter)
 
 
 @tf_export("nn.convolution", v1=[])
@@ -902,15 +907,14 @@ def convolution_v2(
     data_format=None,
     dilations=None,
     name=None):
-  return convolution_internal(
+  return convolution(
       input,  # pylint: disable=redefined-builtin
       filters,
-      strides=strides,
       padding=padding,
-      data_format=data_format,
-      dilations=dilations,
-      name=name)
-
+      strides=strides,
+      dilation_rate=dilations,
+      name=name,
+      data_format=data_format)
 
 convolution_v2.__doc__ = deprecation.rewrite_argument_docstring(
     deprecation.rewrite_argument_docstring(
@@ -918,67 +922,6 @@ convolution_v2.__doc__ = deprecation.rewrite_argument_docstring(
     "filter", "filters")
 
 
-def convolution_internal(
-    input,  # pylint: disable=redefined-builtin
-    filters,
-    strides=None,
-    padding="VALID",
-    data_format=None,
-    dilations=None,
-    name=None):
-  """Internal function which performs rank agnostic convolution."""
-  with ops.name_scope(name, "convolution", [input, filter]) as name:
-    if input.shape is not None:
-      n = len(input.shape) - 2
-    elif filters.shape is not None:
-      n = len(filters.shape) - 2
-    else:
-      raise ValueError("rank of input or filter must be known")
-
-    if n < 1 or n > 3:
-      raise ValueError(
-          "Input tensor must be of rank 3, 4 or 5 but was {}.".format(n + 2))
-
-    if data_format is None:
-      channel_index = n + 1
-    else:
-      channel_index = 1 if data_format.startswith("NC") else n + 1
-
-    strides = _get_sequence(strides, n, channel_index, "strides")
-    dilations = _get_sequence(dilations, n, channel_index, "dilations")
-
-    conv_ops = {1: conv1d, 2: gen_nn_ops.conv2d, 3: gen_nn_ops.conv3d}
-
-    if all(i == 1 for i in dilations):
-      # fast path if no dilation as gradient only supported on GPU for dilations
-      op = conv_ops.get(n)
-      return op(
-          input,
-          filters,
-          strides,
-          padding=padding,
-          data_format=data_format,
-          dilations=dilations,
-          name=name)
-    else:
-      if channel_index == 1:
-        strides = strides[2:]
-        dilations = dilations[2:]
-      else:
-        strides = strides[1:-1]
-        dilations = dilations[1:-1]
-
-      op = Convolution(
-          tensor_shape.as_shape(input.shape),
-          tensor_shape.as_shape(filters.shape),
-          padding,
-          strides=strides,
-          dilation_rate=dilations,
-          name=name,
-          data_format=data_format)
-      return op(input, filters)
-
-
 class Convolution(object):
   """Helper class for convolution.
 
@@ -4097,9 +4040,10 @@ def conv1d(
       entries by which the filter is moved right at each step.
     padding: 'SAME' or 'VALID'
     use_cudnn_on_gpu: An optional `bool`.  Defaults to `True`.
-    data_format: An optional `string` from `"NWC", "NCW"`.  Defaults to `"NWC"`,
-      the data is stored in the order of [batch, in_width, in_channels].  The
-      `"NCW"` format stores data as [batch, in_channels, in_width].
+    data_format: An optional `string` from `"NWC", "NCW"`.  Defaults
+      to `"NWC"`, the data is stored in the order of
+      [batch, in_width, in_channels].  The `"NCW"` format stores
+      data as [batch, in_channels, in_width].
     name: A name for the operation (optional).
     input: Alias for value.
     dilations: An int or list of `ints` that has length `1` or `3` which
@@ -4182,9 +4126,10 @@ def conv1d_v2(
     stride: An int or list of `ints` that has length `1` or `3`.  The number of
       entries by which the filter is moved right at each step.
     padding: 'SAME' or 'VALID'
-    data_format: An optional `string` from `"NWC", "NCW"`.  Defaults to `"NWC"`,
-      the data is stored in the order of [batch, in_width, in_channels].  The
-      `"NCW"` format stores data as [batch, in_channels, in_width].
+    data_format: An optional `string` from `"NWC", "NCW"`.  Defaults
+      to `"NWC"`, the data is stored in the order of
+      [batch, in_width, in_channels].  The `"NCW"` format stores
+      data as [batch, in_channels, in_width].
     dilations: An int or list of `ints` that has length `1` or `3` which
       defaults to 1. The dilation factor for each dimension of input. If set to
       k > 1, there will be k-1 skipped cells between each filter element on that
-- 
GitLab


From b6c0b195751d2eb933cb007a066f1f57df8b306d Mon Sep 17 00:00:00 2001
From: Ayush Dubey <ayushd@google.com>
Date: Wed, 6 Feb 2019 14:49:17 -0800
Subject: [PATCH 2034/2345] Add VLOGs in key areas to enable easier debugging
 of collective execution.

To turn on logging for individual components, set the TF_CPP_VMODULE
environment variable to a comma-separated list of log levels.  e.g.
TF_CPP_VMODULE="a=1,b=2" would write all VLOGs upto level 1 for a.{h,cc} and
upto level 2 for b.{h,cc}.
PiperOrigin-RevId: 232751196
---
 .../base_collective_executor.cc               |  5 +++
 .../collective_param_resolver_local.cc        |  2 +-
 .../collective_param_resolver_distributed.cc  |  8 ++++-
 .../core/kernels/collective_nccl_reducer.cc   |  4 ++-
 tensorflow/core/kernels/collective_ops.cc     | 32 +++++++++++++++++--
 5 files changed, 45 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/common_runtime/base_collective_executor.cc b/tensorflow/core/common_runtime/base_collective_executor.cc
index 03c0e9ce2b..8870a5399c 100644
--- a/tensorflow/core/common_runtime/base_collective_executor.cc
+++ b/tensorflow/core/common_runtime/base_collective_executor.cc
@@ -301,6 +301,8 @@ bool BaseCollectiveExecutor::CheckDependencies(
   for (int32 instance : col_params.instance.impl_details.dependencies) {
     auto find_iter = launched_.find(instance);
     if (find_iter == launched_.end() || find_iter->second != 0) {
+      VLOG(1) << "Collective " << col_params.ToString()
+              << " blocked by instance " << instance;
       return false;
     }
   }
@@ -313,6 +315,7 @@ void BaseCollectiveExecutor::WaitForDependencies(
   while (!CheckDependencies(col_params)) {
     launch_cv_.wait(l);
   }
+  VLOG(1) << "Unblocking collective " << col_params.ToString();
 }
 
 void BaseCollectiveExecutor::Launched(const CollectiveParams& col_params) {
@@ -325,6 +328,8 @@ void BaseCollectiveExecutor::Launched(const CollectiveParams& col_params) {
     launched_[col_params.instance.instance_key] = num_devices;
   }
   if (--launched_[col_params.instance.instance_key] == 0) {
+    VLOG(1) << "Unblocking dependencies for collective instance "
+            << col_params.instance.instance_key;
     launch_cv_.notify_all();
   }
 }
diff --git a/tensorflow/core/common_runtime/collective_param_resolver_local.cc b/tensorflow/core/common_runtime/collective_param_resolver_local.cc
index 5bb2a7df53..5acba6e093 100644
--- a/tensorflow/core/common_runtime/collective_param_resolver_local.cc
+++ b/tensorflow/core/common_runtime/collective_param_resolver_local.cc
@@ -584,7 +584,7 @@ void CollectiveParamResolverLocal::CallInitInstanceSharedParams(
 void CollectiveParamResolverLocal::CompleteParamsAsync(
     const string& device, CollectiveParams* cp, CancellationManager* cancel_mgr,
     const StatusCallback& done) {
-  VLOG(1) << "CompleteParams " << device << " for " << cp << ": "
+  VLOG(1) << "CompleteParams local " << device << " for " << cp << ": "
           << cp->ToString();
   CompleteGroupLocal(
       device, cp,
diff --git a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc
index df735b1c39..443759ab74 100644
--- a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc
+++ b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc
@@ -86,11 +86,17 @@ CollectiveParamResolverDistributed::CollectiveParamResolverDistributed(
       worker_cache_(worker_cache),
       group_leader_(task_name == config.experimental().collective_group_leader()
                         ? ""
-                        : config.experimental().collective_group_leader()) {}
+                        : config.experimental().collective_group_leader()) {
+  VLOG(1) << "CompleteParamResolverDistributed ctor task={" << task_name
+          << "} config.collective_group_leader={"
+          << config.experimental().collective_group_leader() << "}";
+}
 
 void CollectiveParamResolverDistributed::CompleteParamsAsync(
     const string& device, CollectiveParams* cp, CancellationManager* cancel_mgr,
     const StatusCallback& done) {
+  VLOG(1) << "CompleteParams distributed " << device << " for " << cp << ": "
+          << cp->ToString();
   CompleteGroupDistributed(device, cp, cancel_mgr,
                            [this, device, cp, cancel_mgr, done](
                                const Status& s, const GroupRec* gr) {
diff --git a/tensorflow/core/kernels/collective_nccl_reducer.cc b/tensorflow/core/kernels/collective_nccl_reducer.cc
index 113f1487f0..c5e6f06c65 100644
--- a/tensorflow/core/kernels/collective_nccl_reducer.cc
+++ b/tensorflow/core/kernels/collective_nccl_reducer.cc
@@ -149,7 +149,9 @@ void NcclReducer::Run(StatusCallback done) {
           << col_params_->group.num_tasks << " current task "
           << col_params_->instance.task_names[col_params_->default_rank]
           << " num local devices " << num_local_devices
-          << " num global devices " << num_global_devices;
+          << " num global devices " << num_global_devices << " device "
+          << col_ctx_->device_name << " instance "
+          << col_params_->instance.instance_key;
   NcclManager::instance()->AddToAllReduce(
       std::move(participant),
       {nccl_collective_key, num_local_devices, num_global_devices,
diff --git a/tensorflow/core/kernels/collective_ops.cc b/tensorflow/core/kernels/collective_ops.cc
index 04e37e8edc..56843eb773 100644
--- a/tensorflow/core/kernels/collective_ops.cc
+++ b/tensorflow/core/kernels/collective_ops.cc
@@ -43,6 +43,10 @@ class CollectiveOpKernel : public AsyncOpKernel {
       // Call in a blockable thread because it's not guaranteed that
       // this call cannot block.
       c->env()->SchedClosure([this, c, done, col_exec]() {
+        VLOG(1) << "CollectiveOpKernel CompleteParams for collective "
+                << col_params_.name << " device " << c->device()->name()
+                << " group " << col_params_.group.group_key << " instance "
+                << col_params_.instance.instance_key;
         col_exec->CompleteParamsAsync(
             c->device()->name(), &col_params_, c->cancellation_manager(),
             [this, c, done](const Status& s) {
@@ -149,10 +153,18 @@ class CollectiveReduceOpKernel : public CollectiveOpKernel {
       col_params_.instance.shape = c->input(0).shape();
     }
     if (!CanProceedWithCompute(c, col_exec, done)) return;
-    auto actual_done = [c, col_exec, done](const Status& s) {
+
+    int32 instance_key = col_params_.instance.instance_key;
+    auto actual_done = [c, instance_key, done](const Status& s) {
       OP_REQUIRES_OK_ASYNC(c, s, done);
       done();
+      VLOG(1) << "CollectiveReduceKernel ExecuteAsync done for device "
+              << c->device()->name() << " instance " << instance_key;
     };
+    VLOG(1) << "CollectiveReduceKernel ExecuteAsync start for collective "
+            << col_params_.name << " device " << c->device()->name()
+            << " group " << col_params_.group.group_key << " instance "
+            << instance_key;
     col_exec->ExecuteAsync(c, col_params_, GetCollectiveKey(c), actual_done);
   }
 
@@ -211,10 +223,17 @@ class CollectiveBcastSendOpKernel : public CollectiveOpKernel {
                          " does not match shape of input"),
         done);
 
-    auto actual_done = [c, col_exec, done](const Status& s) {
+    int32 instance_key = col_params_.instance.instance_key;
+    auto actual_done = [c, instance_key, done](const Status& s) {
       OP_REQUIRES_OK_ASYNC(c, s, done);
       done();
+      VLOG(1) << "CollectiveBcastSendOpKernel ExecuteAsync done for device "
+              << c->device()->name() << " instance " << instance_key;
     };
+    VLOG(1) << "CollectiveBcastSendOpKernel ExecuteAsync start for collective "
+            << col_params_.name << " device " << c->device()->name()
+            << " group " << col_params_.group.group_key << " instance "
+            << instance_key;
     col_exec->ExecuteAsync(c, col_params_, GetCollectiveKey(c), actual_done);
   }
 
@@ -266,10 +285,17 @@ class CollectiveBcastRecvOpKernel : public CollectiveOpKernel {
     }
     if (!CanProceedWithCompute(c, col_exec, done)) return;
 
-    auto actual_done = [c, col_exec, done](const Status& s) {
+    int32 instance_key = col_params_.instance.instance_key;
+    auto actual_done = [c, instance_key, done](const Status& s) {
       OP_REQUIRES_OK_ASYNC(c, s, done);
       done();
+      VLOG(1) << "CollectiveBcastRecvOpKernel ExecuteAsync done for device "
+              << c->device()->name() << " instance " << instance_key;
     };
+    VLOG(1) << "CollectiveBcastRecvOpKernel ExecuteAsync start for collective "
+            << col_params_.name << " device " << c->device()->name()
+            << " group " << col_params_.group.group_key << " instance "
+            << instance_key;
     col_exec->ExecuteAsync(c, col_params_, GetCollectiveKey(c), actual_done);
   }
 
-- 
GitLab


From e67caacbf9e04358a5b0a368b5bc772091616bfd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 6 Feb 2019 15:08:12 -0800
Subject: [PATCH 2035/2345] Fix bugs in profiler.

PiperOrigin-RevId: 232755426
---
 .../core/platform/default/device_tracer.cc    |  2 +-
 tensorflow/core/profiler/internal/BUILD       |  1 -
 .../internal/traceme_recorder_test.cc         | 19 +++++++++----------
 tensorflow/core/profiler/lib/BUILD            |  1 -
 tensorflow/core/profiler/lib/traceme.cc       | 12 +++++-------
 tensorflow/core/profiler/lib/traceme.h        | 13 ++++++-------
 6 files changed, 21 insertions(+), 27 deletions(-)

diff --git a/tensorflow/core/platform/default/device_tracer.cc b/tensorflow/core/platform/default/device_tracer.cc
index 0d1bf39739..ffcb38fdcd 100644
--- a/tensorflow/core/platform/default/device_tracer.cc
+++ b/tensorflow/core/platform/default/device_tracer.cc
@@ -445,6 +445,7 @@ DeviceTracerImpl::DeviceTracerImpl(CUPTIManager *cupti_manager)
     : cupti_manager_(cupti_manager) {
   VLOG(1) << "DeviceTracer created.";
   cupti_wrapper_.reset(new perftools::gputools::profiler::CuptiWrapper());
+  host_tracer_ = profiler::cpu::HostTracer::Create(2);
   enabled_ = false;
 }
 
@@ -509,7 +510,6 @@ Status DeviceTracerImpl::Start() {
 
   CUPTI_CALL(GetTimestamp(&start_timestamp_));
   start_walltime_us_ = NowInUsec();
-  host_tracer_ = profiler::cpu::HostTracer::Create(2);
   host_tracer_->Start().IgnoreError();
   enabled_ = true;
   return Status::OK();
diff --git a/tensorflow/core/profiler/internal/BUILD b/tensorflow/core/profiler/internal/BUILD
index 30aca58a41..da3039ae3c 100644
--- a/tensorflow/core/profiler/internal/BUILD
+++ b/tensorflow/core/profiler/internal/BUILD
@@ -393,7 +393,6 @@ tf_cuda_cc_test(
         ":traceme_recorder",
         "//tensorflow/core:lib",
         "@com_google_absl//absl/synchronization",
-        "@com_google_absl//absl/time",
         "@com_google_googletest//:gtest_main",
     ],
 )
diff --git a/tensorflow/core/profiler/internal/traceme_recorder_test.cc b/tensorflow/core/profiler/internal/traceme_recorder_test.cc
index bea3f7c9bd..ec588af1d6 100644
--- a/tensorflow/core/profiler/internal/traceme_recorder_test.cc
+++ b/tensorflow/core/profiler/internal/traceme_recorder_test.cc
@@ -18,9 +18,8 @@ limitations under the License.
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
 #include "absl/synchronization/notification.h"
-#include "absl/time/clock.h"
-#include "absl/time/time.h"
 #include "tensorflow/core/lib/core/threadpool.h"
+#include "tensorflow/core/platform/env_time.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
@@ -32,7 +31,7 @@ MATCHER_P(Named, name, "") { return arg.name == name; }
 constexpr static uint64 kNanosInSec = 1000000000;
 
 TEST(RecorderTest, SingleThreaded) {
-  uint64 start_time = static_cast<uint64>(absl::GetCurrentTimeNanos());
+  uint64 start_time = Env::Default()->NowNanos();
   uint64 end_time = start_time + kNanosInSec;
 
   TraceMeRecorder::Record({1, "before", start_time, end_time});
@@ -48,7 +47,7 @@ TEST(RecorderTest, SingleThreaded) {
 }
 
 TEST(RecorderTest, CollectionBeforeStop) {
-  uint64 start_time = static_cast<uint64>(absl::GetCurrentTimeNanos());
+  uint64 start_time = Env::Default()->NowNanos();
   uint64 end_time = start_time + kNanosInSec;
 
   TraceMeRecorder::Record({1, "ignored", start_time, end_time});
@@ -73,8 +72,8 @@ TEST(RecorderTest, CollectionBeforeStop) {
 }
 
 void SpinNanos(int nanos) {
-  uint64 deadline = absl::GetCurrentTimeNanos() + nanos;
-  while (absl::GetCurrentTimeNanos() < deadline) {
+  uint64 deadline = Env::Default()->NowNanos() + nanos;
+  while (Env::Default()->NowNanos() < deadline) {
   }
 }
 
@@ -103,7 +102,7 @@ TEST(RecorderTest, Multithreaded) {
       uint64 j = 0;
       bool was_active = false;
       auto record_event = [&j, i]() {
-        uint64 start_time = static_cast<uint64>(absl::GetCurrentTimeNanos());
+        uint64 start_time = Env::Default()->NowNanos();
         uint64 end_time = start_time + kNanosInSec;
         TraceMeRecorder::Record({/*activity_id=*/j++,
                                  /*name=*/strings::StrCat(i), start_time,
@@ -154,7 +153,7 @@ TEST(RecorderTest, Multithreaded) {
   // Wait while all the threads are spun up.
   while (thread_count.load(std::memory_order_relaxed) < kNumThreads) {
     LOG(INFO) << "Waiting for all threads to spin up...";
-    absl::SleepFor(absl::Milliseconds(1));
+    Env::Default()->SleepForMicroseconds(1 * EnvTime::kMillisToMicros);
   }
 
   // We will probably be done after two iterations (with each thread getting
@@ -165,7 +164,7 @@ TEST(RecorderTest, Multithreaded) {
   for (int iters = 0; iters < kMaxIters && !done(); ++iters) {
     LOG(INFO) << "Looping until convergence, iteration: " << iters;
     TraceMeRecorder::Start(/*level=*/1);
-    absl::SleepFor(absl::Milliseconds(100));
+    Env::Default()->SleepForMicroseconds(100 * EnvTime::kMillisToMicros);
     auto results = TraceMeRecorder::Stop();
     for (const auto& thread : results) {
       if (thread.events.empty()) continue;
@@ -194,7 +193,7 @@ TEST(RecorderTest, Multithreaded) {
         }
       }
     }
-    absl::SleepFor(absl::Milliseconds(1));
+    Env::Default()->SleepForMicroseconds(1 * EnvTime::kMillisToMicros);
   }
   stop.Notify();
 
diff --git a/tensorflow/core/profiler/lib/BUILD b/tensorflow/core/profiler/lib/BUILD
index 697076c56a..e12cc1e921 100644
--- a/tensorflow/core/profiler/lib/BUILD
+++ b/tensorflow/core/profiler/lib/BUILD
@@ -53,7 +53,6 @@ tf_cuda_library(
         "//tensorflow/core:lib",
         "//tensorflow/core/profiler/internal:traceme_recorder",
         "@com_google_absl//absl/strings",
-        "@com_google_absl//absl/time",
         "@com_google_absl//absl/types:optional",
     ],
 )
diff --git a/tensorflow/core/profiler/lib/traceme.cc b/tensorflow/core/profiler/lib/traceme.cc
index ae1c9215e2..90272b8bf5 100644
--- a/tensorflow/core/profiler/lib/traceme.cc
+++ b/tensorflow/core/profiler/lib/traceme.cc
@@ -31,17 +31,15 @@ uint64 NewActivityId() {
 /* static */ uint64 TraceMe::ActivityStartImpl(
     absl::string_view activity_name) {
   uint64 activity_id = NewActivityId();
-  TraceMeRecorder::Record(
-      {activity_id, string(activity_name),
-       /*start_time=*/static_cast<uint64>(absl::GetCurrentTimeNanos()),
-       /*end_time=*/0});
+  TraceMeRecorder::Record({activity_id, string(activity_name),
+                           /*start_time=*/Env::Default()->NowNanos(),
+                           /*end_time=*/0});
   return activity_id;
 }
 
 /* static */ void TraceMe::ActivityEndImpl(uint64 activity_id) {
-  TraceMeRecorder::Record(
-      {activity_id, /*name=*/"", /*start_time=*/0,
-       /*end_time=*/static_cast<uint64>(absl::GetCurrentTimeNanos())});
+  TraceMeRecorder::Record({activity_id, /*name=*/"", /*start_time=*/0,
+                           /*end_time=*/Env::Default()->NowNanos()});
 }
 
 }  // namespace profiler
diff --git a/tensorflow/core/profiler/lib/traceme.h b/tensorflow/core/profiler/lib/traceme.h
index 098c531092..b9fae3d37f 100644
--- a/tensorflow/core/profiler/lib/traceme.h
+++ b/tensorflow/core/profiler/lib/traceme.h
@@ -18,8 +18,8 @@ limitations under the License.
 #include <string>
 
 #include "absl/strings/string_view.h"
-#include "absl/time/clock.h"
 #include "absl/types/optional.h"
+#include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/types.h"
 #include "tensorflow/core/profiler/internal/traceme_recorder.h"
@@ -74,7 +74,7 @@ class TraceMe {
     DCHECK_GE(level, 1);
     if (TraceMeRecorder::Active(level)) {
       new (&no_init_.name) string(activity_name);
-      start_time_ = absl::GetCurrentTimeNanos();
+      start_time_ = Env::Default()->NowNanos();
     } else {
       start_time_ = kUntracedActivity;
     }
@@ -89,7 +89,7 @@ class TraceMe {
     DCHECK_GE(level, 1);
     if (TraceMeRecorder::Active(level)) {
       new (&no_init_.name) string(std::move(activity_name));
-      start_time_ = absl::GetCurrentTimeNanos();
+      start_time_ = Env::Default()->NowNanos();
     } else {
       start_time_ = kUntracedActivity;
     }
@@ -119,7 +119,7 @@ class TraceMe {
     DCHECK_GE(level, 1);
     if (TraceMeRecorder::Active(level)) {
       new (&no_init_.name) string(name_generator());
-      start_time_ = absl::GetCurrentTimeNanos();
+      start_time_ = Env::Default()->NowNanos();
     } else {
       start_time_ = kUntracedActivity;
     }
@@ -136,9 +136,8 @@ class TraceMe {
     //   start/stop session timestamp (recorded in XprofResponse).
     if (start_time_ != kUntracedActivity) {
       if (TraceMeRecorder::Active()) {
-        TraceMeRecorder::Record(
-            {kCompleteActivity, std::move(no_init_.name), start_time_,
-             static_cast<uint64>(absl::GetCurrentTimeNanos())});
+        TraceMeRecorder::Record({kCompleteActivity, std::move(no_init_.name),
+                                 start_time_, Env::Default()->NowNanos()});
       }
       no_init_.name.~string();
     }
-- 
GitLab


From 3d704b754fd18f22a41336274bec9b5a85c3e4db Mon Sep 17 00:00:00 2001
From: Tong Shen <endlessroad@google.com>
Date: Wed, 6 Feb 2019 15:17:32 -0800
Subject: [PATCH 2036/2345] GraphToFunctionDef() should add node's placeholder
 attr to FunctionDef's signature.

PiperOrigin-RevId: 232757344
---
 tensorflow/c/BUILD                         |  1 +
 tensorflow/c/c_api.cc                      |  7 ++++
 tensorflow/c/c_api.h                       |  4 ++
 tensorflow/c/c_api_function.cc             | 38 +++++++++++++++++
 tensorflow/c/c_api_function_test.cc        | 48 ++++++++++++++++++++++
 tensorflow/core/framework/node_def_util.cc | 11 +++--
 6 files changed, 105 insertions(+), 4 deletions(-)

diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index 10a2992730..ef7863dc0d 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -356,6 +356,7 @@ tf_cc_test(
     srcs = ["c_api_function_test.cc"],
     deps = [
         ":c_api",
+        ":c_api_internal",
         ":c_test_util",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
diff --git a/tensorflow/c/c_api.cc b/tensorflow/c/c_api.cc
index 6d6d02635b..ef22b67fe9 100644
--- a/tensorflow/c/c_api.cc
+++ b/tensorflow/c/c_api.cc
@@ -1311,6 +1311,13 @@ void TF_SetAttrTypeList(TF_OperationDescription* desc, const char* attr_name,
                      reinterpret_cast<const DataType*>(values), num_values));
 }
 
+void TF_SetAttrPlaceholder(TF_OperationDescription* desc, const char* attr_name,
+                           const char* placeholder) {
+  tensorflow::AttrValue attr_value;
+  attr_value.set_placeholder(placeholder);
+  desc->node_builder.Attr(attr_name, attr_value);
+}
+
 void TF_SetAttrFuncName(TF_OperationDescription* desc, const char* attr_name,
                         const char* value, size_t length) {
   tensorflow::NameAttrList func_name;
diff --git a/tensorflow/c/c_api.h b/tensorflow/c/c_api.h
index ac04037937..88b8b49b01 100644
--- a/tensorflow/c/c_api.h
+++ b/tensorflow/c/c_api.h
@@ -549,6 +549,10 @@ TF_CAPI_EXPORT extern void TF_SetAttrTypeList(TF_OperationDescription* desc,
                                               const char* attr_name,
                                               const TF_DataType* values,
                                               int num_values);
+TF_CAPI_EXPORT extern void TF_SetAttrPlaceholder(TF_OperationDescription* desc,
+                                                 const char* attr_name,
+                                                 const char* placeholder);
+
 // Set a 'func' attribute to the specified name.
 // `value` must point to a string of length `length` bytes.
 TF_CAPI_EXPORT extern void TF_SetAttrFuncName(TF_OperationDescription* desc,
diff --git a/tensorflow/c/c_api_function.cc b/tensorflow/c/c_api_function.cc
index 28b9f8df9c..1477d643b5 100644
--- a/tensorflow/c/c_api_function.cc
+++ b/tensorflow/c/c_api_function.cc
@@ -162,6 +162,11 @@ Status FillFunctionBody(
     const std::vector<const Node*>& body_nodes,
     const std::unordered_map<string, string>& tensor_renaming,
     FunctionDef* fdef) {
+  std::unordered_set<string> func_attr_names;
+  for (const auto& func_attr : fdef->signature().attr()) {
+    func_attr_names.insert(func_attr.name());
+  }
+
   std::vector<const Edge*> in_edges;
   std::vector<const Edge*> control_edges;
   for (const Node* node : body_nodes) {
@@ -243,6 +248,39 @@ Status FillFunctionBody(
     if (node->op_def().is_stateful()) {
       fdef->mutable_signature()->set_is_stateful(true);
     }
+
+    // If this node has any attributes with placeholder value, add the
+    // attribute to FunctionDef signature.
+    for (const auto& iter : node->attrs()) {
+      if (iter.second.placeholder().empty()) {
+        continue;
+      }
+
+      // If we already added the attribute, skip it.
+      string func_attr_name = iter.second.placeholder();
+      if (func_attr_names.find(func_attr_name) != func_attr_names.end()) {
+        continue;
+      }
+
+      // This node's attribute is a placeholder value, so it does not have type
+      // information. We check node's OpDef for attribute type.
+      string node_attr_name = iter.first;
+      const OpDef_AttrDef* node_attr_def = nullptr;
+      for (const auto& node_attr : node->op_def().attr()) {
+        if (node_attr.name() == node_attr_name) {
+          node_attr_def = &node_attr;
+        }
+      }
+      if (!node_attr_def) {
+        return errors::Internal("Cannot find attr ", node_attr_name,
+                                " in OpDef ", node->op_def().DebugString());
+      }
+      OpDef_AttrDef* attr_def = fdef->mutable_signature()->add_attr();
+      attr_def->set_name(func_attr_name);
+      attr_def->set_type(node_attr_def->type());
+
+      func_attr_names.insert(func_attr_name);
+    }
   }
   return Status::OK();
 }
diff --git a/tensorflow/c/c_api_function_test.cc b/tensorflow/c/c_api_function_test.cc
index 73fe73769b..946f8c4a2c 100644
--- a/tensorflow/c/c_api_function_test.cc
+++ b/tensorflow/c/c_api_function_test.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/c/c_api.h"
 
+#include "tensorflow/c/c_api_internal.h"
 #include "tensorflow/c/c_test_util.h"
 #include "tensorflow/core/framework/function.pb.h"
 #include "tensorflow/core/framework/op_def.pb.h"
@@ -1230,6 +1231,53 @@ void DefineFunction(const char* name, TF_Function** func,
   ASSERT_NE(*func, nullptr);
 }
 
+REGISTER_OP("CustomOp")
+    .Output("output: float32")
+    .Attr("index: int")
+    .SetShapeFn(tensorflow::shape_inference::UnknownShape);
+
+void NodeWithPlaceholderAttrHelper(TF_Graph* graph, TF_Status* s,
+                                   const char* name, const char* placeholder,
+                                   TF_Operation** op) {
+  TF_OperationDescription* desc = TF_NewOperation(graph, "CustomOp", name);
+  TF_SetAttrPlaceholder(desc, "index", placeholder);
+  *op = TF_FinishOperation(desc, s);
+  ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s);
+  ASSERT_NE(*op, nullptr);
+}
+
+TEST_F(CApiFunctionTest, GraphToFunctionDefWithPlaceholderAttr) {
+  std::unique_ptr<TF_Graph, decltype(&TF_DeleteGraph)> func_graph(
+      TF_NewGraph(), TF_DeleteGraph);
+  std::unique_ptr<TF_Status, decltype(&TF_DeleteStatus)> s(TF_NewStatus(),
+                                                           TF_DeleteStatus);
+
+  TF_Operation *node1, *node2, *node3;
+  NodeWithPlaceholderAttrHelper(func_graph.get(), s.get(), "node1", "v1",
+                                &node1);
+  NodeWithPlaceholderAttrHelper(func_graph.get(), s.get(), "node2", "v1",
+                                &node2);
+  NodeWithPlaceholderAttrHelper(func_graph.get(), s.get(), "node3", "v2",
+                                &node3);
+
+  TF_Output inputs[] = {};
+  TF_Output outputs[] = {{node1, 0}, {node2, 0}, {node3, 0}};
+  func_ = TF_GraphToFunction(
+      func_graph.get(), "func", /*append_hash_to_fn_name=*/false, -1,
+      /*opers=*/nullptr, 0, inputs, 3, outputs,
+      /*output_names=*/nullptr,
+      /*opts=*/nullptr, /*description=*/nullptr, s.get());
+  ASSERT_EQ(TF_OK, TF_GetCode(s.get())) << TF_Message(s.get());
+  ASSERT_NE(func_, nullptr);
+
+  // Verify that FunctionDef has 2 attributes, "v1" and "v2".
+  ASSERT_EQ(func_->fdef.signature().attr().size(), 2);
+  EXPECT_EQ(func_->fdef.signature().attr(0).name(), "v1");
+  EXPECT_EQ(func_->fdef.signature().attr(0).type(), "int");
+  EXPECT_EQ(func_->fdef.signature().attr(1).name(), "v2");
+  EXPECT_EQ(func_->fdef.signature().attr(1).type(), "int");
+}
+
 TEST_F(CApiFunctionTest, SetGradientAndRun) {
   // Define the function and its grad
   DefineFunction(func_name_, &func_);
diff --git a/tensorflow/core/framework/node_def_util.cc b/tensorflow/core/framework/node_def_util.cc
index e369e882a0..fee52375c1 100644
--- a/tensorflow/core/framework/node_def_util.cc
+++ b/tensorflow/core/framework/node_def_util.cc
@@ -515,10 +515,13 @@ Status ValidateNodeDef(const NodeDef& node_def, const OpDef& op_def) {
           ". (Check whether your GraphDef-interpreting binary is up to date "
           "with your GraphDef-generating binary.).");
     }
-    TF_RETURN_WITH_CONTEXT_IF_ERROR(
-        ValidateAttrValue(attr.second, *iter->second),
-        "; NodeDef: ", FormatNodeDefForError(node_def), "; ",
-        SummarizeOpDef(op_def));
+    // If attr value is placeholder, do not check it.
+    if (attr.second.placeholder().empty()) {
+      TF_RETURN_WITH_CONTEXT_IF_ERROR(
+          ValidateAttrValue(attr.second, *iter->second),
+          "; NodeDef: ", FormatNodeDefForError(node_def), "; ",
+          SummarizeOpDef(op_def));
+    }
     // Keep track of which attr names have (not) been found in the NodeDef.
     op_attrs.erase(iter);
   }
-- 
GitLab


From 6103a3d46a8c5a7156931251ca5f7d5711e40492 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Wed, 6 Feb 2019 15:22:23 -0800
Subject: [PATCH 2037/2345] Refactoring: Move subgraph testing utility function
 to subgraph_test_util module.

This is a no-op change to improve code quality.

PiperOrigin-RevId: 232758236
---
 tensorflow/lite/kernels/BUILD                 |  31 ++-
 tensorflow/lite/kernels/if_test.cc            | 195 ++----------------
 tensorflow/lite/kernels/subgraph_test_util.cc | 159 ++++++++++++++
 tensorflow/lite/kernels/subgraph_test_util.h  |  62 ++++++
 .../lite/kernels/subgraph_test_util_test.cc   |  84 ++++++++
 5 files changed, 355 insertions(+), 176 deletions(-)
 create mode 100644 tensorflow/lite/kernels/subgraph_test_util.cc
 create mode 100644 tensorflow/lite/kernels/subgraph_test_util.h
 create mode 100644 tensorflow/lite/kernels/subgraph_test_util_test.cc

diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index 90e4d82618..795bd80788 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -1218,9 +1218,10 @@ tf_cc_test(
     deps = [
         ":builtin_ops",
         ":kernel_util",
+        ":subgraph_test_util",
+        ":test_util",
         "//tensorflow/lite:builtin_op_data",
         "//tensorflow/lite:framework",
-        "//tensorflow/lite/kernels:test_util",
         "@com_google_googletest//:gtest",
         "@flatbuffers",
     ],
@@ -1285,3 +1286,31 @@ tf_cc_test(
         "@com_google_googletest//:gtest_main",
     ],
 )
+
+cc_library(
+    name = "subgraph_test_util",
+    testonly = 1,
+    srcs = ["subgraph_test_util.cc"],
+    hdrs = ["subgraph_test_util.h"],
+    deps = [
+        ":builtin_ops",
+        ":kernel_util",
+        ":test_util",
+        "//tensorflow/lite:builtin_op_data",
+        "//tensorflow/lite:framework",
+        "@com_google_googletest//:gtest",
+        "@flatbuffers",
+    ],
+)
+
+tf_cc_test(
+    name = "subgraph_test_util_test",
+    size = "small",
+    srcs = ["subgraph_test_util_test.cc"],
+    deps = [
+        ":subgraph_test_util",
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite/kernels:test_util",
+        "@com_google_googletest//:gtest",
+    ],
+)
diff --git a/tensorflow/lite/kernels/if_test.cc b/tensorflow/lite/kernels/if_test.cc
index 6593a58a76..a1460cdb42 100644
--- a/tensorflow/lite/kernels/if_test.cc
+++ b/tensorflow/lite/kernels/if_test.cc
@@ -1,4 +1,4 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -12,191 +12,26 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
+
 #include <gtest/gtest.h>
 #include "flatbuffers/flexbuffers.h"  // TF:flatbuffers
 #include "tensorflow/lite/interpreter.h"
 #include "tensorflow/lite/kernels/kernel_util.h"
 #include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/kernels/subgraph_test_util.h"
 #include "tensorflow/lite/kernels/test_util.h"
 #include "tensorflow/lite/model.h"
 
 namespace tflite {
-namespace ops {
-namespace builtin {
-// ADD and MUL are used to test simple branch.
-TfLiteRegistration* Register_ADD();
-TfLiteRegistration* Register_MUL();
-// Pad is used to test dynamic sized subgraphs.
-TfLiteRegistration* Register_PAD();
-}  // namespace builtin
-namespace custom {
-TfLiteRegistration* Register_IF();
-}  // namespace custom
-}  // namespace ops
-
-namespace {
-
-void SetupTensor(Subgraph* subgraph, int tensor_index,
-                 const std::vector<int>& shape, TfLiteType type) {
-  ASSERT_EQ(subgraph->SetTensorParametersReadWrite(
-                tensor_index, type, "", shape.size(), shape.data(), {}, false),
-            kTfLiteOk);
-}
-
-// TODO(ycling): Consider to move all the test helper functions to another
-// build target (e.g. subgraph_test_util).
-// Build a subgraph with an add op. Helper function for testing.
-void BuildAddSubgraph(Subgraph* subgraph) {
-  int first_new_tensor_index;
-  ASSERT_EQ(subgraph->AddTensors(3, &first_new_tensor_index), kTfLiteOk);
-  ASSERT_EQ(first_new_tensor_index, 0);
-  ASSERT_EQ(subgraph->SetInputs({0, 1}), kTfLiteOk);
-  ASSERT_EQ(subgraph->SetOutputs({2}), kTfLiteOk);
-
-  SetupTensor(subgraph, 0, {2}, kTfLiteInt32);
-  SetupTensor(subgraph, 1, {1, 2}, kTfLiteInt32);
-  // Intentionally set the wrong output size for testing. This should be
-  // overridden by Prepare function.
-  SetupTensor(subgraph, 2, {100}, kTfLiteInt32);
-
-  TfLiteAddParams* params =
-      reinterpret_cast<TfLiteAddParams*>(malloc(sizeof(TfLiteAddParams)));
-  params->activation = kTfLiteActNone;
-  int node_index;
-  subgraph->AddNodeWithParameters({0, 1}, {2}, nullptr, 0, params,
-                                  ::tflite::ops::builtin::Register_ADD(),
-                                  &node_index);
-}
-
-// Build a subgraph with an mul op. Helper function for testing.
-void BuildMulSubgraph(Subgraph* subgraph) {
-  int first_new_tensor_index;
-  ASSERT_EQ(subgraph->AddTensors(3, &first_new_tensor_index), kTfLiteOk);
-  ASSERT_EQ(first_new_tensor_index, 0);
-  ASSERT_EQ(subgraph->SetInputs({0, 1}), kTfLiteOk);
-  ASSERT_EQ(subgraph->SetOutputs({2}), kTfLiteOk);
-
-  SetupTensor(subgraph, 0, {2}, kTfLiteInt32);
-  SetupTensor(subgraph, 1, {1, 2}, kTfLiteInt32);
-  // Intentionally set the wrong output size for testing. This should be
-  // overridden by Prepare function.
-  SetupTensor(subgraph, 2, {100}, kTfLiteInt32);
 
-  TfLiteMulParams* params =
-      reinterpret_cast<TfLiteMulParams*>(malloc(sizeof(TfLiteMulParams)));
-  params->activation = kTfLiteActNone;
-  int node_index;
-  subgraph->AddNodeWithParameters({0, 1}, {2}, nullptr, 0, params,
-                                  ::tflite::ops::builtin::Register_MUL(),
-                                  &node_index);
-}
-
-// Build a subgraph with a pad op. Helper function for testing.
-void BuildPadSubgraph(Subgraph* subgraph) {
-  int first_new_tensor_index;
-  ASSERT_EQ(subgraph->AddTensors(3, &first_new_tensor_index), kTfLiteOk);
-  ASSERT_EQ(first_new_tensor_index, 0);
-  ASSERT_EQ(subgraph->SetInputs({0, 1}), kTfLiteOk);
-  ASSERT_EQ(subgraph->SetOutputs({2}), kTfLiteOk);
-
-  SetupTensor(subgraph, 0, {2}, kTfLiteInt32);
-  SetupTensor(subgraph, 1, {1, 2}, kTfLiteInt32);
-  // Intentionally set the wrong output size for testing. This should be
-  // overridden by Prepare function.
-  SetupTensor(subgraph, 2, {100}, kTfLiteInt32);
-
-  TfLitePadParams* params =
-      reinterpret_cast<TfLitePadParams*>(malloc(sizeof(TfLitePadParams)));
-  int node_index;
-  subgraph->AddNodeWithParameters({0, 1}, {2}, nullptr, 0, params,
-                                  ::tflite::ops::builtin::Register_PAD(),
-                                  &node_index);
-}
-
-void BuildIfSubgraph(Subgraph* subgraph) {
-  int first_new_tensor_index;
-  ASSERT_EQ(subgraph->AddTensors(4, &first_new_tensor_index), kTfLiteOk);
-  ASSERT_EQ(first_new_tensor_index, 0);
-  ASSERT_EQ(subgraph->SetInputs({0, 1, 2}), kTfLiteOk);
-  ASSERT_EQ(subgraph->SetOutputs({3}), kTfLiteOk);
-
-  SetupTensor(subgraph, 0, {1}, kTfLiteBool);
-  SetupTensor(subgraph, 1, {2}, kTfLiteInt32);
-  SetupTensor(subgraph, 2, {1, 2}, kTfLiteInt32);
-  // Intentionally set the wrong output size for testing. This should be
-  // overridden by Prepare function.
-  SetupTensor(subgraph, 3, {100}, kTfLiteInt32);
-
-  flexbuffers::Builder fbb;
-  fbb.Map([&]() {
-    fbb.Int("then_subgraph_index", 1);
-    fbb.Int("else_subgraph_index", 2);
-  });
-  fbb.Finish();
-  const auto& buffer = fbb.GetBuffer();
-
-  int node_index;
-  subgraph->AddNodeWithParameters(
-      {0, 1, 2}, {3}, reinterpret_cast<const char*>(buffer.data()),
-      buffer.size(), nullptr, ::tflite::ops::custom::Register_IF(),
-      &node_index);
-}
+using subgraph_test_util::BuildAddSubgraph;
+using subgraph_test_util::BuildIfSubgraph;
+using subgraph_test_util::BuildMulSubgraph;
+using subgraph_test_util::BuildPadSubgraph;
+using subgraph_test_util::CheckIntTensor;
+using subgraph_test_util::FillIntTensor;
 
-void FillIntTensor(TfLiteTensor* tensor, const std::vector<int32_t>& data) {
-  int count = NumElements(tensor);
-  ASSERT_EQ(count, data.size());
-  for (int i = 0; i < count; ++i) {
-    tensor->data.i32[i] = data[i];
-  }
-}
-
-void CheckIntTensor(const TfLiteTensor* tensor, const std::vector<int>& shape,
-                    const std::vector<int32_t>& data) {
-  ASSERT_EQ(tensor->dims->size, shape.size());
-  for (int i = 0; i < tensor->dims->size; ++i) {
-    ASSERT_EQ(tensor->dims->data[i], shape[i]);
-  }
-  ASSERT_EQ(tensor->type, kTfLiteInt32);
-  int count = NumElements(tensor);
-  ASSERT_EQ(count, data.size());
-  for (int i = 0; i < count; ++i) {
-    EXPECT_EQ(tensor->data.i32[i], data[i]);
-  }
-}
-
-// TestHelperfunctionTest tests the helper functions defined in this file.
-TEST(TestHelperfunctionTest, TestBuildAddSubgraph) {
-  std::unique_ptr<Interpreter> interpreter(new Interpreter);
-  BuildAddSubgraph(&interpreter->primary_subgraph());
-  ASSERT_EQ(interpreter->AllocateTensors(), kTfLiteOk);
-  FillIntTensor(interpreter->tensor(interpreter->inputs()[0]), {5, 7});
-  FillIntTensor(interpreter->tensor(interpreter->inputs()[1]), {1, 2});
-  ASSERT_EQ(interpreter->Invoke(), kTfLiteOk);
-  TfLiteTensor* output = interpreter->tensor(interpreter->outputs()[0]);
-  CheckIntTensor(output, {1, 2}, {6, 9});
-}
-
-TEST(TestHelperfunctionTest, TestBuildMulSubgraph) {
-  std::unique_ptr<Interpreter> interpreter(new Interpreter);
-  BuildMulSubgraph(&interpreter->primary_subgraph());
-  ASSERT_EQ(interpreter->AllocateTensors(), kTfLiteOk);
-  FillIntTensor(interpreter->tensor(interpreter->inputs()[0]), {5, 7});
-  FillIntTensor(interpreter->tensor(interpreter->inputs()[1]), {1, 2});
-  ASSERT_EQ(interpreter->Invoke(), kTfLiteOk);
-  TfLiteTensor* output = interpreter->tensor(interpreter->outputs()[0]);
-  CheckIntTensor(output, {1, 2}, {5, 14});
-}
-
-TEST(TestHelperfunctionTest, TestBuildPadSubgraph) {
-  std::unique_ptr<Interpreter> interpreter(new Interpreter);
-  BuildPadSubgraph(&interpreter->primary_subgraph());
-  ASSERT_EQ(interpreter->AllocateTensors(), kTfLiteOk);
-  FillIntTensor(interpreter->tensor(interpreter->inputs()[0]), {5, 7});
-  FillIntTensor(interpreter->tensor(interpreter->inputs()[1]), {1, 2});
-  ASSERT_EQ(interpreter->Invoke(), kTfLiteOk);
-  TfLiteTensor* output = interpreter->tensor(interpreter->outputs()[0]);
-  CheckIntTensor(output, {5}, {0, 5, 7, 0, 0});
-}
+namespace {
 
 // A simple test that performs `ADD` if condition is true, and `MUL` otherwise.
 // The computation is: `cond ? a + b : a * b`.
@@ -208,7 +43,12 @@ class SimpleIfTest : public ::testing::Test {
     BuildAddSubgraph(interpreter_->subgraph(1));
     BuildMulSubgraph(interpreter_->subgraph(2));
     BuildIfSubgraph(&interpreter_->primary_subgraph());
+
+    interpreter_->ResizeInputTensor(interpreter_->inputs()[0], {1});
+    interpreter_->ResizeInputTensor(interpreter_->inputs()[1], {2});
+    interpreter_->ResizeInputTensor(interpreter_->inputs()[2], {1, 2});
     ASSERT_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
+
     FillIntTensor(interpreter_->tensor(interpreter_->inputs()[1]), {5, 7});
     FillIntTensor(interpreter_->tensor(interpreter_->inputs()[2]), {1, 2});
   }
@@ -239,7 +79,12 @@ class DynamicSubgraphIfTest : public ::testing::Test {
     BuildAddSubgraph(interpreter_->subgraph(1));
     BuildPadSubgraph(interpreter_->subgraph(2));
     BuildIfSubgraph(&interpreter_->primary_subgraph());
+
+    interpreter_->ResizeInputTensor(interpreter_->inputs()[0], {1});
+    interpreter_->ResizeInputTensor(interpreter_->inputs()[1], {2});
+    interpreter_->ResizeInputTensor(interpreter_->inputs()[2], {1, 2});
     ASSERT_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
+
     FillIntTensor(interpreter_->tensor(interpreter_->inputs()[1]), {5, 7});
     FillIntTensor(interpreter_->tensor(interpreter_->inputs()[2]), {1, 2});
   }
diff --git a/tensorflow/lite/kernels/subgraph_test_util.cc b/tensorflow/lite/kernels/subgraph_test_util.cc
new file mode 100644
index 0000000000..0f41f9cc90
--- /dev/null
+++ b/tensorflow/lite/kernels/subgraph_test_util.cc
@@ -0,0 +1,159 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/kernels/subgraph_test_util.h"
+
+#include "flatbuffers/flexbuffers.h"  // TF:flatbuffers
+#include "tensorflow/lite/core/subgraph.h"
+#include "tensorflow/lite/kernels/kernel_util.h"
+#include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/kernels/test_util.h"
+#include "tensorflow/lite/model.h"
+
+namespace tflite {
+
+namespace ops {
+namespace builtin {
+// ADD and MUL are used to test simple branch.
+TfLiteRegistration* Register_ADD();
+TfLiteRegistration* Register_MUL();
+// ADD and MUL are used to test dynamic sized subgraphs.
+TfLiteRegistration* Register_PAD();
+}  // namespace builtin
+namespace custom {
+TfLiteRegistration* Register_IF();
+}  // namespace custom
+}  // namespace ops
+
+namespace subgraph_test_util {
+
+void SetupTensor(Subgraph* subgraph, int tensor_index, TfLiteType type) {
+  ASSERT_EQ(subgraph->SetTensorParametersReadWrite(tensor_index, type, "", 0,
+                                                   nullptr, {}, false),
+            kTfLiteOk);
+}
+
+void BuildAddSubgraph(Subgraph* subgraph) {
+  int first_new_tensor_index;
+  ASSERT_EQ(subgraph->AddTensors(3, &first_new_tensor_index), kTfLiteOk);
+  ASSERT_EQ(first_new_tensor_index, 0);
+  ASSERT_EQ(subgraph->SetInputs({0, 1}), kTfLiteOk);
+  ASSERT_EQ(subgraph->SetOutputs({2}), kTfLiteOk);
+
+  SetupTensor(subgraph, 0, kTfLiteInt32);
+  SetupTensor(subgraph, 1, kTfLiteInt32);
+  SetupTensor(subgraph, 2, kTfLiteInt32);
+
+  TfLiteAddParams* params =
+      reinterpret_cast<TfLiteAddParams*>(malloc(sizeof(TfLiteAddParams)));
+  params->activation = kTfLiteActNone;
+  int node_index;
+  subgraph->AddNodeWithParameters({0, 1}, {2}, nullptr, 0, params,
+                                  ::tflite::ops::builtin::Register_ADD(),
+                                  &node_index);
+}
+
+// Build a subgraph with an mul op. Helper function for testing.
+void BuildMulSubgraph(Subgraph* subgraph) {
+  int first_new_tensor_index;
+  ASSERT_EQ(subgraph->AddTensors(3, &first_new_tensor_index), kTfLiteOk);
+  ASSERT_EQ(first_new_tensor_index, 0);
+  ASSERT_EQ(subgraph->SetInputs({0, 1}), kTfLiteOk);
+  ASSERT_EQ(subgraph->SetOutputs({2}), kTfLiteOk);
+
+  SetupTensor(subgraph, 0, kTfLiteInt32);
+  SetupTensor(subgraph, 1, kTfLiteInt32);
+  SetupTensor(subgraph, 2, kTfLiteInt32);
+
+  TfLiteMulParams* params =
+      reinterpret_cast<TfLiteMulParams*>(malloc(sizeof(TfLiteMulParams)));
+  params->activation = kTfLiteActNone;
+  int node_index;
+  subgraph->AddNodeWithParameters({0, 1}, {2}, nullptr, 0, params,
+                                  ::tflite::ops::builtin::Register_MUL(),
+                                  &node_index);
+}
+
+// Build a subgraph with a pad op. Helper function for testing.
+void BuildPadSubgraph(Subgraph* subgraph) {
+  int first_new_tensor_index;
+  ASSERT_EQ(subgraph->AddTensors(3, &first_new_tensor_index), kTfLiteOk);
+  ASSERT_EQ(first_new_tensor_index, 0);
+  ASSERT_EQ(subgraph->SetInputs({0, 1}), kTfLiteOk);
+  ASSERT_EQ(subgraph->SetOutputs({2}), kTfLiteOk);
+
+  SetupTensor(subgraph, 0, kTfLiteInt32);
+  SetupTensor(subgraph, 1, kTfLiteInt32);
+  SetupTensor(subgraph, 2, kTfLiteInt32);
+
+  TfLitePadParams* params =
+      reinterpret_cast<TfLitePadParams*>(malloc(sizeof(TfLitePadParams)));
+  int node_index;
+  subgraph->AddNodeWithParameters({0, 1}, {2}, nullptr, 0, params,
+                                  ::tflite::ops::builtin::Register_PAD(),
+                                  &node_index);
+}
+
+void BuildIfSubgraph(Subgraph* subgraph) {
+  int first_new_tensor_index;
+  ASSERT_EQ(subgraph->AddTensors(4, &first_new_tensor_index), kTfLiteOk);
+  ASSERT_EQ(first_new_tensor_index, 0);
+  ASSERT_EQ(subgraph->SetInputs({0, 1, 2}), kTfLiteOk);
+  ASSERT_EQ(subgraph->SetOutputs({3}), kTfLiteOk);
+
+  SetupTensor(subgraph, 0, kTfLiteBool);
+  SetupTensor(subgraph, 1, kTfLiteInt32);
+  SetupTensor(subgraph, 2, kTfLiteInt32);
+  SetupTensor(subgraph, 3, kTfLiteInt32);
+
+  flexbuffers::Builder fbb;
+  fbb.Map([&]() {
+    fbb.Int("then_subgraph_index", 1);
+    fbb.Int("else_subgraph_index", 2);
+  });
+  fbb.Finish();
+  const auto& buffer = fbb.GetBuffer();
+
+  int node_index;
+  subgraph->AddNodeWithParameters(
+      {0, 1, 2}, {3}, reinterpret_cast<const char*>(buffer.data()),
+      buffer.size(), nullptr, ::tflite::ops::custom::Register_IF(),
+      &node_index);
+}
+
+void FillIntTensor(TfLiteTensor* tensor, const std::vector<int32_t>& data) {
+  int count = NumElements(tensor);
+  ASSERT_EQ(count, data.size());
+  for (int i = 0; i < count; ++i) {
+    tensor->data.i32[i] = data[i];
+  }
+}
+
+void CheckIntTensor(const TfLiteTensor* tensor, const std::vector<int>& shape,
+                    const std::vector<int32_t>& data) {
+  ASSERT_EQ(tensor->dims->size, shape.size());
+  for (int i = 0; i < tensor->dims->size; ++i) {
+    ASSERT_EQ(tensor->dims->data[i], shape[i]);
+  }
+  ASSERT_EQ(tensor->type, kTfLiteInt32);
+  int count = NumElements(tensor);
+  ASSERT_EQ(count, data.size());
+  for (int i = 0; i < count; ++i) {
+    EXPECT_EQ(tensor->data.i32[i], data[i]);
+  }
+}
+
+}  // namespace subgraph_test_util
+}  // namespace tflite
diff --git a/tensorflow/lite/kernels/subgraph_test_util.h b/tensorflow/lite/kernels/subgraph_test_util.h
new file mode 100644
index 0000000000..6dc5207c4b
--- /dev/null
+++ b/tensorflow/lite/kernels/subgraph_test_util.h
@@ -0,0 +1,62 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// This module provides helper functions for testing the interaction between
+// control flow ops and subgraphs.
+// For convenience, we mostly only use `kTfLiteInt32` in this module.
+
+#ifndef TENSORFLOW_LITE_KERNELS_SUBGRAPH_TEST_UTIL_H_
+#define TENSORFLOW_LITE_KERNELS_SUBGRAPH_TEST_UTIL_H_
+
+#include "tensorflow/lite/core/subgraph.h"
+
+namespace tflite {
+namespace subgraph_test_util {
+
+// Build a subgraph with a single Add op.
+// 2 inputs. 1 output.
+void BuildAddSubgraph(Subgraph* subgraph);
+
+// Build a subgraph with a single Mul op.
+// 2 inputs. 1 output.
+void BuildMulSubgraph(Subgraph* subgraph);
+
+// Build a subgraph with a single Pad op.
+// 2 inputs. 1 output.
+void BuildPadSubgraph(Subgraph* subgraph);
+
+// Build a subgraph with a single If op.
+// 3 inputs:
+//   The 1st input is condition with boolean type.
+//   The 2nd and 3rd inputs are feed input the branch subgraphs.
+// 1 output.
+void BuildIfSubgraph(Subgraph* subgraph);
+
+// Fill a `TfLiteTensor` with a 32-bits integer vector.
+// Preconditions:
+// * The tensor must have `kTfLiteInt32` type.
+// * The tensor must be allocated.
+// * The element count of the tensor must be equal to the length or
+//   the vector.
+void FillIntTensor(TfLiteTensor* tensor, const std::vector<int32_t>& data);
+
+// Check if the shape and data of a tensor is as expected.
+void CheckIntTensor(const TfLiteTensor* tensor, const std::vector<int>& shape,
+                    const std::vector<int32_t>& data);
+
+}  // namespace subgraph_test_util
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_KERNELS_SUBGRAPH_TEST_UTIL_H_
diff --git a/tensorflow/lite/kernels/subgraph_test_util_test.cc b/tensorflow/lite/kernels/subgraph_test_util_test.cc
new file mode 100644
index 0000000000..88cd8f580a
--- /dev/null
+++ b/tensorflow/lite/kernels/subgraph_test_util_test.cc
@@ -0,0 +1,84 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/kernels/subgraph_test_util.h"
+#include <gtest/gtest.h>
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/kernels/test_util.h"
+
+namespace tflite {
+
+namespace subgraph_test_util {
+
+namespace {
+
+// SubGraphTestUtilTest tests the helper functions defined in this file.
+TEST(SubGraphTestUtilTest, TestBuildAddSubgraph) {
+  std::unique_ptr<Interpreter> interpreter(new Interpreter);
+  BuildAddSubgraph(&interpreter->primary_subgraph());
+
+  interpreter->ResizeInputTensor(interpreter->inputs()[0], {2});
+  interpreter->ResizeInputTensor(interpreter->inputs()[1], {1, 2});
+  ASSERT_EQ(interpreter->AllocateTensors(), kTfLiteOk);
+
+  FillIntTensor(interpreter->tensor(interpreter->inputs()[0]), {5, 7});
+  FillIntTensor(interpreter->tensor(interpreter->inputs()[1]), {1, 2});
+  ASSERT_EQ(interpreter->Invoke(), kTfLiteOk);
+
+  TfLiteTensor* output = interpreter->tensor(interpreter->outputs()[0]);
+  CheckIntTensor(output, {1, 2}, {6, 9});
+}
+
+TEST(SubGraphTestUtilTest, TestBuildMulSubgraph) {
+  std::unique_ptr<Interpreter> interpreter(new Interpreter);
+  BuildMulSubgraph(&interpreter->primary_subgraph());
+
+  interpreter->ResizeInputTensor(interpreter->inputs()[0], {2});
+  interpreter->ResizeInputTensor(interpreter->inputs()[1], {1, 2});
+  ASSERT_EQ(interpreter->AllocateTensors(), kTfLiteOk);
+
+  FillIntTensor(interpreter->tensor(interpreter->inputs()[0]), {5, 7});
+  FillIntTensor(interpreter->tensor(interpreter->inputs()[1]), {1, 2});
+  ASSERT_EQ(interpreter->Invoke(), kTfLiteOk);
+
+  TfLiteTensor* output = interpreter->tensor(interpreter->outputs()[0]);
+  CheckIntTensor(output, {1, 2}, {5, 14});
+}
+
+TEST(SubGraphTestUtilTest, TestBuildPadSubgraph) {
+  std::unique_ptr<Interpreter> interpreter(new Interpreter);
+  BuildPadSubgraph(&interpreter->primary_subgraph());
+
+  interpreter->ResizeInputTensor(interpreter->inputs()[0], {2});
+  interpreter->ResizeInputTensor(interpreter->inputs()[1], {1, 2});
+  ASSERT_EQ(interpreter->AllocateTensors(), kTfLiteOk);
+
+  FillIntTensor(interpreter->tensor(interpreter->inputs()[0]), {5, 7});
+  FillIntTensor(interpreter->tensor(interpreter->inputs()[1]), {1, 2});
+  ASSERT_EQ(interpreter->Invoke(), kTfLiteOk);
+
+  TfLiteTensor* output = interpreter->tensor(interpreter->outputs()[0]);
+  CheckIntTensor(output, {5}, {0, 5, 7, 0, 0});
+}
+
+}  // namespace
+}  // namespace subgraph_test_util
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
-- 
GitLab


From b0022ada8fd3ba6ec1c359e5d4e374b4632d70aa Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 6 Feb 2019 15:41:09 -0800
Subject: [PATCH 2038/2345] Internal changes only.

PiperOrigin-RevId: 232761942
---
 WORKSPACE                                     | 25 +++++--------------
 tensorflow/workspace.bzl                      | 10 --------
 .../preconfig/generate/archives.bzl           |  9 +++----
 .../preconfig/generate/workspace.bzl          |  5 +++-
 4 files changed, 14 insertions(+), 35 deletions(-)

diff --git a/WORKSPACE b/WORKSPACE
index 957b8d8528..9f07b9fd47 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -29,7 +29,7 @@ load(
 bazel_toolchains_repositories()
 
 load(
-    "@io_bazel_rules_docker//container:container.bzl",
+    "@io_bazel_rules_docker//repositories:repositories.bzl",
     container_repositories = "repositories",
 )
 
@@ -43,29 +43,17 @@ remote_config_workspace()
 # Apple and Swift rules.
 http_archive(
     name = "build_bazel_rules_apple",
-    sha256 = "4fe4ee824200b48821730f89ff260984332dc3551db587c24691235d1d96a8a7",
-    strip_prefix = "rules_apple-0.10.0",
-    urls = ["https://github.com/bazelbuild/rules_apple/archive/0.10.0.tar.gz"],
-)
-http_archive(
-    name = "build_bazel_rules_swift",
-    sha256 = "6544ff5615febec0342de1127144d2f3e43ea80fb7f9b1ade65e6a184e39e618",
-    strip_prefix = "rules_swift-0.5.0",
-    urls = ["https://github.com/bazelbuild/rules_swift/archive/0.5.0.tar.gz"],
-)
-http_archive(
-    name = "bazel_skylib",
-    sha256 = "eb5c57e4c12e68c0c20bc774bfbc60a568e800d025557bc4ea022c6479acc867",
-    strip_prefix = "bazel-skylib-0.6.0",
-    urls = ["https://github.com/bazelbuild/bazel-skylib/archive/0.6.0.tar.gz"],
+    sha256 = "73b4980a318d203d3307f850e27e66ec5cc8d223147a3475a6f11597eb6438a5",
+    strip_prefix = "rules_apple-0.13.0",
+    urls = ["https://github.com/bazelbuild/rules_apple/archive/0.13.0.tar.gz"],
 )
 http_file(
     name = "xctestrunner",
     executable = 1,
-    urls = ["https://github.com/google/xctestrunner/releases/download/0.2.5/ios_test_runner.par"],
+    urls = ["https://github.com/google/xctestrunner/releases/download/0.2.6/ios_test_runner.par"],
 )
 load("@build_bazel_rules_apple//apple:repositories.bzl", "apple_rules_dependencies")
-apple_rules_dependencies(ignore_version_differences = True)
+apple_rules_dependencies()
 load("@build_bazel_rules_swift//swift:repositories.bzl", "swift_rules_dependencies")
 swift_rules_dependencies()
 
@@ -134,4 +122,3 @@ http_archive(
         "http://download.tensorflow.org/models/speech_commands_v0.01.zip",
     ],
 )
-
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index d8dc923184..e97041d14d 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -718,16 +718,6 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
         ],
     )
 
-    tf_http_archive(
-        name = "bazel_toolchains",
-        sha256 = "07dfbe80638eb1fe681f7c07e61b34b579c6710c691e49ee90ccdc6e9e75ebbb",
-        strip_prefix = "bazel-toolchains-9a111bd82161c1fbe8ed17a593ca1023fd941c70",
-        urls = [
-            "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/9a111bd82161c1fbe8ed17a593ca1023fd941c70.tar.gz",
-            "https://github.com/bazelbuild/bazel-toolchains/archive/9a111bd82161c1fbe8ed17a593ca1023fd941c70.tar.gz",
-        ],
-    )
-
     tf_http_archive(
         name = "arm_neon_2_x86_sse",
         build_file = clean_dep("//third_party:arm_neon_2_x86_sse.BUILD"),
diff --git a/third_party/toolchains/preconfig/generate/archives.bzl b/third_party/toolchains/preconfig/generate/archives.bzl
index bafc7d4943..d20432e1de 100644
--- a/third_party/toolchains/preconfig/generate/archives.bzl
+++ b/third_party/toolchains/preconfig/generate/archives.bzl
@@ -1,13 +1,12 @@
-load("//tensorflow:version_check.bzl", "parse_bazel_version")
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
 def bazel_toolchains_archive():
     http_archive(
       name = "bazel_toolchains",
-      sha256 = "ee854b5de299138c1f4a2edb5573d22b21d975acfc7aa938f36d30b49ef97498",
-      strip_prefix = "bazel-toolchains-37419a124bdb9af2fec5b99a973d359b6b899b61",
+      sha256 = "109a99384f9d08f9e75136d218ebaebc68cc810c56897aea2224c57932052d30",
+      strip_prefix = "bazel-toolchains-94d31935a2c94fe7e7c7379a0f3393e181928ff7",
       urls = [
-        "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/37419a124bdb9af2fec5b99a973d359b6b899b61.tar.gz",
-        "https://github.com/bazelbuild/bazel-toolchains/archive/37419a124bdb9af2fec5b99a973d359b6b899b61.tar.gz",
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/94d31935a2c94fe7e7c7379a0f3393e181928ff7.tar.gz",
+        "https://github.com/bazelbuild/bazel-toolchains/archive/94d31935a2c94fe7e7c7379a0f3393e181928ff7.tar.gz",
       ],
     )
diff --git a/third_party/toolchains/preconfig/generate/workspace.bzl b/third_party/toolchains/preconfig/generate/workspace.bzl
index 0495173786..bce2d5bdbf 100644
--- a/third_party/toolchains/preconfig/generate/workspace.bzl
+++ b/third_party/toolchains/preconfig/generate/workspace.bzl
@@ -1,7 +1,10 @@
+load(
+    "@io_bazel_rules_docker//repositories:repositories.bzl",
+    container_repositories = "repositories",
+)
 load(
     "@io_bazel_rules_docker//container:container.bzl",
     "container_pull",
-    container_repositories = "repositories",
 )
 load(":containers.bzl", "container_digests")
 
-- 
GitLab


From 06d3c5c4f2b5f68c80fb0b8d19efb8de3685314e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 6 Feb 2019 15:53:27 -0800
Subject: [PATCH 2039/2345] Add quantize_and_dequantize_op_hdrs library for
 compiling the ops directly.

PiperOrigin-RevId: 232764250
---
 tensorflow/core/kernels/BUILD | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 1749f3a4da..ea651f166c 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -7171,3 +7171,12 @@ cc_header_only_library(
         ":cwise_lib",
     ],
 )
+
+# Library to link with when compiling the quantize and dequantize kernels directly,
+# e.g. for selective registration.
+cc_header_only_library(
+    name = "quantize_and_dequantize_op_hdrs",
+    deps = [
+        ":quantize_and_dequantize_op",
+    ],
+)
-- 
GitLab


From 2b7686e953b9964addb95170b6f613eb2d3d9f41 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 6 Feb 2019 15:55:51 -0800
Subject: [PATCH 2040/2345] Remove user-defined special members from
 VariantTensorData. This has the side effect of making the class
 move-optimized.

PiperOrigin-RevId: 232764696
---
 tensorflow/core/framework/variant_tensor_data.cc | 4 ----
 tensorflow/core/framework/variant_tensor_data.h  | 4 ++--
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/framework/variant_tensor_data.cc b/tensorflow/core/framework/variant_tensor_data.cc
index c169e867ec..993a8989b7 100644
--- a/tensorflow/core/framework/variant_tensor_data.cc
+++ b/tensorflow/core/framework/variant_tensor_data.cc
@@ -20,14 +20,10 @@ limitations under the License.
 
 namespace tensorflow {
 
-VariantTensorData::VariantTensorData() {}
-
 VariantTensorData::VariantTensorData(VariantTensorDataProto proto) {
   FromProto(std::move(proto));
 }
 
-VariantTensorData::~VariantTensorData() {}
-
 int VariantTensorData::tensors_size() const { return tensors_.size(); }
 
 const Tensor& VariantTensorData::tensors(int index) const {
diff --git a/tensorflow/core/framework/variant_tensor_data.h b/tensorflow/core/framework/variant_tensor_data.h
index ca99e83f9c..d98cf6b5e1 100644
--- a/tensorflow/core/framework/variant_tensor_data.h
+++ b/tensorflow/core/framework/variant_tensor_data.h
@@ -37,11 +37,11 @@ class VariantTensorDataProto;
 // separate so that kernels do not need to depend on protos.
 class VariantTensorData {
  public:
-  VariantTensorData();
+  VariantTensorData() = default;
+
   // TODO(b/118823936): This silently returns if the proto is invalid.
   // Consider calling FromProto explicitly instead.
   VariantTensorData(VariantTensorDataProto proto);
-  ~VariantTensorData();
 
   // Name of the type of objects being serialized.
   const string& type_name() const { return type_name_; }
-- 
GitLab


From d0a4836f912afb1ecad199205ff6c7cdf2066663 Mon Sep 17 00:00:00 2001
From: Katherine Wu <kathywu@google.com>
Date: Wed, 6 Feb 2019 16:00:42 -0800
Subject: [PATCH 2041/2345] Internal change

PiperOrigin-RevId: 232765574
---
 .../python/keras/saving/saved_model_test.py   |  6 +---
 .../saved_model/model_utils/__init__.py       |  1 +
 .../saved_model/model_utils/export_utils.py   | 29 ++++++++++++++-----
 .../python/saved_model/signature_constants.py |  4 +++
 4 files changed, 28 insertions(+), 12 deletions(-)

diff --git a/tensorflow/python/keras/saving/saved_model_test.py b/tensorflow/python/keras/saving/saved_model_test.py
index 8063b8af4d..6ecb4ed6ea 100644
--- a/tensorflow/python/keras/saving/saved_model_test.py
+++ b/tensorflow/python/keras/saving/saved_model_test.py
@@ -38,7 +38,6 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 from tensorflow.python.saved_model import loader_impl
 from tensorflow.python.saved_model import model_utils
-from tensorflow.python.saved_model import signature_constants
 from tensorflow.python.training import mode_keys
 from tensorflow.python.training import training as training_module
 
@@ -264,10 +263,7 @@ def subclassed_model():
 
 def load_model(sess, path, mode):
   tags = model_utils.EXPORT_TAG_MAP[mode]
-  if mode == mode_keys.ModeKeys.PREDICT:
-    sig_def_key = signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
-  else:
-    sig_def_key = mode
+  sig_def_key = model_utils.SIGNATURE_KEY_MAP[mode]
 
   meta_graph_def = loader_impl.load(sess, tags, path)
   inputs = {
diff --git a/tensorflow/python/saved_model/model_utils/__init__.py b/tensorflow/python/saved_model/model_utils/__init__.py
index 84540badb4..3f54c96def 100644
--- a/tensorflow/python/saved_model/model_utils/__init__.py
+++ b/tensorflow/python/saved_model/model_utils/__init__.py
@@ -25,4 +25,5 @@ from tensorflow.python.saved_model.model_utils.export_utils import EXPORT_TAG_MA
 from tensorflow.python.saved_model.model_utils.export_utils import get_export_outputs
 from tensorflow.python.saved_model.model_utils.export_utils import get_temp_export_dir
 from tensorflow.python.saved_model.model_utils.export_utils import get_timestamped_export_dir
+from tensorflow.python.saved_model.model_utils.export_utils import SIGNATURE_KEY_MAP
 # pylint: enable=wildcard-import
diff --git a/tensorflow/python/saved_model/model_utils/export_utils.py b/tensorflow/python/saved_model/model_utils/export_utils.py
index 4f8933758d..343a3362c4 100644
--- a/tensorflow/python/saved_model/model_utils/export_utils.py
+++ b/tensorflow/python/saved_model/model_utils/export_utils.py
@@ -41,6 +41,18 @@ EXPORT_TAG_MAP = {
     mode_keys.ModeKeys.TEST: [tag_constants.EVAL],
 }
 
+# For every exported mode, a SignatureDef map should be created using the
+# functions `export_outputs_for_mode` and `build_all_signature_defs`. By
+# default, this map will contain a single Signature that defines the input
+# tensors and output predictions, losses, and/or metrics (depending on the mode)
+# The default keys used in the SignatureDef map are defined below.
+SIGNATURE_KEY_MAP = {
+    mode_keys.ModeKeys.PREDICT:
+        signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY,
+    mode_keys.ModeKeys.TRAIN:
+        signature_constants.DEFAULT_TRAIN_SIGNATURE_DEF_KEY,
+    mode_keys.ModeKeys.TEST: signature_constants.DEFAULT_EVAL_SIGNATURE_DEF_KEY
+}
 
 _SINGLE_FEATURE_DEFAULT_NAME = 'feature'
 _SINGLE_RECEIVER_DEFAULT_NAME = 'input'
@@ -262,18 +274,21 @@ def export_outputs_for_mode(
   Raises:
     ValueError: if an appropriate ExportOutput cannot be found for the mode.
   """
-  # TODO(b/113185250): move all model export helper functions into an util file.
+  if mode not in SIGNATURE_KEY_MAP:
+    raise ValueError(
+        'Export output type not found for mode: {}. Expected one of: {}.\n'
+        'One likely error is that V1 Estimator Modekeys were somehow passed to '
+        'this function. Please ensure that you are using the new ModeKeys.'
+        .format(mode, SIGNATURE_KEY_MAP.keys()))
+  signature_key = SIGNATURE_KEY_MAP[mode]
   if mode == mode_keys.ModeKeys.PREDICT:
     return get_export_outputs(serving_export_outputs, predictions)
   elif mode == mode_keys.ModeKeys.TRAIN:
-    return {mode: export_output_lib.TrainOutput(
-        loss=loss, predictions=predictions, metrics=metrics)}
-  elif mode == mode_keys.ModeKeys.TEST:
-    return {mode: export_output_lib.EvalOutput(
+    return {signature_key: export_output_lib.TrainOutput(
         loss=loss, predictions=predictions, metrics=metrics)}
   else:
-    raise ValueError(
-        'Export output type not found for mode: {}'.format(mode))
+    return {signature_key: export_output_lib.EvalOutput(
+        loss=loss, predictions=predictions, metrics=metrics)}
 
 
 def get_export_outputs(export_outputs, predictions):
diff --git a/tensorflow/python/saved_model/signature_constants.py b/tensorflow/python/saved_model/signature_constants.py
index 0efe176343..8047d0d5c7 100644
--- a/tensorflow/python/saved_model/signature_constants.py
+++ b/tensorflow/python/saved_model/signature_constants.py
@@ -136,6 +136,10 @@ tf_export(
 ################################################################################
 # Train/Eval API constants.
 # Not exported while export_all_saved_models is experimental.
+DEFAULT_TRAIN_SIGNATURE_DEF_KEY = "train"
+# TODO(b/123998850): Change default signature key to "test" after making sure
+# that TFMA use cases won't break.
+DEFAULT_EVAL_SIGNATURE_DEF_KEY = "eval"
 
 SUPERVISED_TRAIN_METHOD_NAME = "tensorflow/supervised/training"
 
-- 
GitLab


From 3138e8e9e7791dec85d0ae540348ec6742c5ac58 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 6 Feb 2019 16:05:52 -0800
Subject: [PATCH 2042/2345] Add methods for testing HLO backend configurations
 and checking runtime variation.

PiperOrigin-RevId: 232766807
---
 tensorflow/compiler/xla/service/hlo_module.h  |   1 +
 tensorflow/compiler/xla/service/hlo_runner.cc |  29 +++++
 tensorflow/compiler/xla/service/hlo_runner.h  |   8 ++
 .../compiler/xla/tests/hlo_test_base.cc       | 106 ++++++++++++++++--
 tensorflow/compiler/xla/tests/hlo_test_base.h |  10 +-
 5 files changed, 144 insertions(+), 10 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_module.h b/tensorflow/compiler/xla/service/hlo_module.h
index f1310e4b27..20d98b96a9 100644
--- a/tensorflow/compiler/xla/service/hlo_module.h
+++ b/tensorflow/compiler/xla/service/hlo_module.h
@@ -187,6 +187,7 @@ class HloModule {
   std::vector<HloComputation*> MakeNonfusionComputations() const;
 
   const HloModuleConfig& config() const { return config_; }
+  void set_config(HloModuleConfig& config) { config_ = config; }
 
   // Return a string representation of the module.
   //
diff --git a/tensorflow/compiler/xla/service/hlo_runner.cc b/tensorflow/compiler/xla/service/hlo_runner.cc
index d7d66ae1c4..84399f17e5 100644
--- a/tensorflow/compiler/xla/service/hlo_runner.cc
+++ b/tensorflow/compiler/xla/service/hlo_runner.cc
@@ -168,6 +168,35 @@ StatusOr<Literal> HloRunner::Execute(std::unique_ptr<HloModule> module,
       /*profile=*/profile);
 }
 
+StatusOr<Literal> HloRunner::Execute(
+    std::unique_ptr<Executable> executable,
+    const absl::Span<const Literal* const> arguments,
+    ExecutionProfile* profile) {
+  TF_ASSIGN_OR_RETURN(std::vector<ScopedShapedBuffer> argument_buffers,
+                      TransferLiteralsToDevice(arguments));
+  TF_ASSIGN_OR_RETURN(ScopedShapedBuffer result,
+                      ExecuteWithDeviceBuffers(
+                          /*module=*/std::move(executable),
+                          /*arguments=*/argument_buffers,
+                          /*profile=*/profile));
+  return TransferLiteralFromDevice(result);
+}
+
+StatusOr<Literal> HloRunner::Execute(std::unique_ptr<Executable> executable,
+                                     const absl::Span<const Literal> arguments,
+                                     ExecutionProfile* profile) {
+  // Construct a vector of plain pointers for the arguments.
+  std::vector<const Literal*> argument_pointers;
+  argument_pointers.reserve(arguments.size());
+  for (const auto& argument : arguments) {
+    argument_pointers.push_back(&argument);
+  }
+  return Execute(
+      /*module=*/std::move(executable),
+      /*arguments=*/argument_pointers,
+      /*profile=*/profile);
+}
+
 StatusOr<ScopedShapedBuffer> HloRunner::ExecuteWithDeviceBuffers(
     std::unique_ptr<HloModule> module,
     const absl::Span<const ShapedBuffer* const> arguments, bool run_hlo_passes,
diff --git a/tensorflow/compiler/xla/service/hlo_runner.h b/tensorflow/compiler/xla/service/hlo_runner.h
index bb792cf8c9..a6e6015d6a 100644
--- a/tensorflow/compiler/xla/service/hlo_runner.h
+++ b/tensorflow/compiler/xla/service/hlo_runner.h
@@ -124,6 +124,14 @@ class HloRunner {
                             bool run_hlo_passes = true,
                             ExecutionProfile* profile = nullptr);
 
+  StatusOr<Literal> Execute(std::unique_ptr<Executable> executable,
+                            const absl::Span<const Literal* const> arguments,
+                            ExecutionProfile* profile = nullptr);
+
+  StatusOr<Literal> Execute(std::unique_ptr<Executable> executable,
+                            const absl::Span<const Literal> arguments,
+                            ExecutionProfile* profile = nullptr);
+
   // As Execute(), but accepts and returns device buffers instead of host
   // buffers.
   StatusOr<ScopedShapedBuffer> ExecuteWithDeviceBuffers(
diff --git a/tensorflow/compiler/xla/tests/hlo_test_base.cc b/tensorflow/compiler/xla/tests/hlo_test_base.cc
index 66f72ba8d2..38466b175e 100644
--- a/tensorflow/compiler/xla/tests/hlo_test_base.cc
+++ b/tensorflow/compiler/xla/tests/hlo_test_base.cc
@@ -313,7 +313,10 @@ StatusOr<::testing::AssertionResult> HloTestBase::RunAndCompareInternal(
                        reference_preprocessor);
 }
 
-::testing::AssertionResult HloTestBase::Run(string_view hlo_string) {
+::testing::AssertionResult HloTestBase::Run(string_view hlo_string,
+                                            bool run_hlo_passes,
+                                            ExecutionProfile* profile,
+                                            string backend_config) {
   auto module_or_status =
       HloRunner::CreateModuleFromString(hlo_string, GetDebugOptionsForTest());
   if (!module_or_status.ok()) {
@@ -321,19 +324,106 @@ StatusOr<::testing::AssertionResult> HloTestBase::RunAndCompareInternal(
            << "Error while parsing HLO text format: "
            << module_or_status.status().ToString();
   }
+
+  std::unique_ptr<HloModule> module = std::move(module_or_status.ValueOrDie());
   const auto& fake_arguments =
-      MakeFakeArguments(module_or_status.ValueOrDie().get())
-          .ConsumeValueOrDie();
+      MakeFakeArguments(module.get()).ConsumeValueOrDie();
   std::vector<Literal*> fake_argument_ptrs;
   absl::c_transform(
       fake_arguments, std::back_inserter(fake_argument_ptrs),
       [](const Literal& literal) { return const_cast<Literal*>(&literal); });
-  return test_runner_
-                 .Execute(std::move(module_or_status.ValueOrDie()),
-                          fake_argument_ptrs, /*run_hlo_passes=*/true)
-                 .ok()
+
+  if (profile != nullptr) {
+    // We have to enable HLO profiling since otherwise currently the
+    // ExecutionProfile is not correct.
+    //
+    // TODO(b/119432044): Fix collection of the ExecutionProfile
+    // so that this is not necessary.
+    HloModuleConfig config = module->config();
+    DebugOptions debug_options = config.debug_options();
+    debug_options.set_xla_hlo_profile(true);
+    config.set_debug_options(debug_options);
+    module->set_config(config);
+  }
+
+  if (!backend_config.empty()) {
+    // Set backend configuration if it is given.
+    HloInstruction* instruction =
+        module->entry_computation()->root_instruction();
+    instruction->set_raw_backend_config_string(backend_config);
+  }
+
+  // return ::testing::AssertionSuccess();
+  auto output = test_runner_.Execute(std::move(module), fake_argument_ptrs,
+                                     /*run_hlo_passes=*/run_hlo_passes,
+                                     /*profile=*/profile);
+
+  return output.ok()
              ? ::testing::AssertionSuccess()
-             : ::testing::AssertionFailure();
+             : ::testing::AssertionFailure() << output.status().error_message();
+}
+
+::testing::AssertionResult HloTestBase::RunMultipleTimes(
+    string_view hlo_string, bool run_hlo_passes,
+    std::vector<ExecutionProfile>* profiles, string backend_config) {
+  int n = profiles->size();
+  std::vector<std::vector<Literal*>> fake_argument_ptrs(n);
+  std::vector<std::vector<Literal>> fake_arguments(n);
+  std::vector<std::unique_ptr<Executable>> executables(n);
+
+  for (int i = 0; i < n; ++i) {
+    auto module_or_status =
+        HloRunner::CreateModuleFromString(hlo_string, GetDebugOptionsForTest());
+    if (!module_or_status.ok()) {
+      return ::testing::AssertionFailure()
+             << "Error while parsing HLO text format: "
+             << module_or_status.status().ToString();
+    }
+    std::unique_ptr<HloModule> module =
+        std::move(module_or_status.ValueOrDie());
+
+    fake_arguments[i] = MakeFakeArguments(module.get()).ConsumeValueOrDie();
+    absl::c_transform(
+        fake_arguments[i], std::back_inserter(fake_argument_ptrs[i]),
+        [](const Literal& literal) { return const_cast<Literal*>(&literal); });
+
+    if (profiles != nullptr) {
+      // We have to enable HLO profiling since otherwise currently the
+      // ExecutionProfile is not correct.
+      //
+      // TODO(b/119432044): Fix collection of the ExecutionProfile
+      // so that this is not necessary.
+      HloModuleConfig config = module->config();
+      DebugOptions debug_options = config.debug_options();
+      debug_options.set_xla_hlo_profile(true);
+      config.set_debug_options(debug_options);
+      module->set_config(config);
+    }
+
+    if (backend_config != "") {
+      // Set backend configuration if it is given.
+      HloInstruction* instruction =
+          module->entry_computation()->root_instruction();
+      instruction->set_raw_backend_config_string(backend_config);
+    }
+
+    auto executable =
+        test_runner_.CreateExecutable(std::move(module), run_hlo_passes);
+    if (!executable.ok())
+      return ::testing::AssertionFailure()
+             << executable.status().error_message();
+    executables[i] = std::move(executable.ValueOrDie());
+  }
+
+  for (int i = 0; i < n; ++i) {
+    auto output =
+        test_runner_.Execute(std::move(executables[i]), fake_argument_ptrs[i],
+                             /*profile=*/&((*profiles)[i]));
+    if (!output.ok())
+      return ::testing::AssertionFailure() << output.status().error_message();
+  }
+
+  return ::testing::AssertionSuccess();
 }
 
 ::testing::AssertionResult HloTestBase::RunAndCompareFromFile(
diff --git a/tensorflow/compiler/xla/tests/hlo_test_base.h b/tensorflow/compiler/xla/tests/hlo_test_base.h
index 69a4f96288..78bdd336e0 100644
--- a/tensorflow/compiler/xla/tests/hlo_test_base.h
+++ b/tensorflow/compiler/xla/tests/hlo_test_base.h
@@ -221,8 +221,14 @@ class HloTestBase : public ::testing::Test {
       const absl::optional<ErrorSpec>& error,
       const std::function<void(HloModule*)>& reference_preprocessor = nullptr)
       TF_MUST_USE_RESULT;
-  ::testing::AssertionResult Run(const absl::string_view hlo_string)
-      TF_MUST_USE_RESULT;
+  ::testing::AssertionResult Run(const absl::string_view hlo_string,
+                                 bool run_hlo_passes = true,
+                                 ExecutionProfile* profile = nullptr,
+                                 string backend_config = "") TF_MUST_USE_RESULT;
+  ::testing::AssertionResult RunMultipleTimes(
+      const absl::string_view hlo_string, bool run_hlo_passes,
+      std::vector<ExecutionProfile>* profiles,
+      string backend_config = "") TF_MUST_USE_RESULT;
   ::testing::AssertionResult RunAndCompareFromFile(
       const string& filename, const absl::optional<ErrorSpec>& error,
       const std::function<void(HloModule*)>& reference_preprocessor = nullptr)
-- 
GitLab


From d4c33e5148fa4143ec483b6f0f2bbe899d8e6da5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 6 Feb 2019 17:34:19 -0800
Subject: [PATCH 2043/2345] Cleaning up pip.sh for CI builds & testing.

PiperOrigin-RevId: 232781773
---
 tensorflow/tools/ci_build/builds/pip_new.sh | 655 ++++++++++++++++++++
 1 file changed, 655 insertions(+)
 create mode 100755 tensorflow/tools/ci_build/builds/pip_new.sh

diff --git a/tensorflow/tools/ci_build/builds/pip_new.sh b/tensorflow/tools/ci_build/builds/pip_new.sh
new file mode 100755
index 0000000000..78e226eaf7
--- /dev/null
+++ b/tensorflow/tools/ci_build/builds/pip_new.sh
@@ -0,0 +1,655 @@
+#!/usr/bin/env bash
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# Build the Python PIP installation package for TensorFlow and install
+# the package.
+#
+# Usage:
+#   pip.sh
+#
+# Required step(s):
+#   Run configure.py prior to running this script.
+#
+# Required environment variable(s):
+#   CONTAINER_TYPE:      (CPU | GPU)
+#   OS_TYPE:             (UBUNTU | MACOS)
+#   TF_PYTHON_VERSION:   (python2 | python2.7 | python3.5 | python3.7)
+#
+# Optional environment variables. If provided, overwrites any default values.
+#   TF_BUILD_FLAGS:      Bazel build flags excluding `--test_tag_filters` and
+#                        test targets.
+#                          e.g. TF_BUILD_FLAGS="--verbose_failures=true \
+#                               --build_tests_only --test_output=errors"
+#   TF_TEST_FILTER_TAGS: Filtering tags for bazel tests. More specifically,
+#                        input tags for `--test_filter_tags` flag.
+#                          e.g. TF_TEST_FILTER_TAGS="no-pip,-nomac,no_oss"
+#   TF_TEST_TARGETS:     Bazel test targets.
+#                          e.g. TF_TEST_TARGETS="//tensorflow/contrib/... \
+#                               //tensorflow/... \
+#                               //tensorflow/python/... "
+#   TF_PIP_TESTS:        PIP tests to run. If NOT specified, skips all tests.
+#                          e.g. TF_PIP_TESTS="test_pip_virtualenv_clean \
+#                               test_pip_virtualenv_clean \
+#                               test_pip_virtualenv_oss_serial"
+#   IS_NIGHTLY:          Nightly run flag.
+#                          e.g. IS_NIGHTLY=1  # nightly runs
+#                               IS_NIGHTLY=0  # non-nightly runs
+#   TF_PROJECT_NAME:     Name of the project. This string will be pass onto
+#                        the wheel file name. For nightly builds, it will be
+#                        overwritten to 'tf_nightly'. For gpu builds, '_gpu'
+#                        will be appended.
+#                          e.g. TF_PROJECT_NAME="tensorflow"
+# 			                   e.g. TF_PROJECT_NAME="tf_nightly_gpu"
+#   TF_PIP_TEST_ROOT:    Root directory for building and testing pip pkgs.
+#                          e.g. TF_PIP_TEST_ROOT="pip_test"
+#
+# To-be-deprecated variable(s).
+#   GIT_TAG_OVERRIDE:    Values for `--git_tag_override`. This flag gets passed
+#                        in as `--action_env` for bazel build and tests.
+#   TF_BUILD_INSTALL_EXTRA_PIP_PACKAGES
+#                        Additonal pip packages to be installed.
+#                        Caveat: pip version needs to be checked prior.
+
+# set bash options
+set -e
+set -x
+
+###########################################################################
+# General helper function(s)
+###########################################################################
+
+# Strip leading and trailing whitespaces
+str_strip () {
+  echo -e "$1" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//'
+}
+
+# Convert string to all lower case
+lowercase() {
+  if [[ -z "${1}" ]]; then
+    die "Nothing to convert to lowercase. No argument given."
+  fi
+  echo "${1}" | tr '[:upper:]' '[:lower:]'
+}
+
+check_global_vars() {
+  # Check container type
+  if ! [[ ${CONTAINER_TYPE} == "cpu" ]] && \
+     ! [[ ${CONTAINER_TYPE} == "rocm" ]] && \
+     ! [[ ${CONTAINER_TYPE} == "gpu" ]]; then
+    die "Error: Provided CONTAINER_TYPE \"${CONTAINER_TYPE}\" "\
+        "is not supported."
+  fi
+  # Check OS type
+  if ! [[ ${OS_TYPE} == "ubuntu" ]] && \
+     ! [[ ${OS_TYPE} == "macos" ]]; then
+    die"Error: Provided OS_TYPE \"${OS_TYPE}\" is not supported."
+  fi
+}
+
+add_test_filter_tag() {
+  EMPTY=""
+  while true; do
+    FILTER="${1:$EMPTY}"
+    if ! [[ $BAZEL_TEST_FILTER_TAGS == *"${FILTER}"* ]]; then
+      BAZEL_TEST_FILTER_TAGS="${FILTER},${BAZEL_TEST_FILTER_TAGS}"
+    fi
+    shift
+    if [[ -z "${1}" ]]; then
+      break
+    fi
+  done
+}
+
+remove_test_filter_tag() {
+  EMPTY=""
+  while true; do
+    FILTER="${1:$EMPTY}"
+    BAZEL_TEST_FILTER_TAGS="$(echo ${BAZEL_TEST_FILTER_TAGS} | sed -e 's/^'${FILTER}',//g' -e 's/,'${FILTER}'//g')"
+    shift
+    if [[ -z "${1}" ]]; then
+      break
+    fi
+  done
+}
+
+# Clean up bazel build & test flags with proper configuration.
+update_bazel_flags() {
+  # Add git tag override flag if necessary.
+  GIT_TAG_STR=" --action_env=GIT_TAG_OVERRIDE"
+  if [[ -z "${GIT_TAG_OVERRIDE}" ]] && \
+    ! [[ ${BAZEL_BUILD_FLAGS} = *${GIT_TAG_STR}* ]]; then
+    BAZEL_BUILD_FLAGS+="${GIT_TAG_STR}"
+  fi
+  # Clean up whitespaces
+  BAZEL_BUILD_FLAGS=$(str_strip "${BAZEL_BUILD_FLAGS}")
+  # Cleaned bazel flags
+  echo "Bazel build flags (cleaned):\n" "${BAZEL_BUILD_FLAGS}"
+}
+
+update_test_filter_tags() {
+  # Add test filter tags
+  # This script is for PIP version of the installation. Add pip related tags.
+  add_test_filter_tag -no_pip -nopip
+  # MacOS filter tags
+  if [[ ${OS_TYPE} == "macos" ]]; then
+    remove_test_filter_tag nomac no_mac
+    add_test_filter_tag -nomac -no_mac
+  fi
+  # GPU or CPU tags
+  if [[ "${CONTAINER_TYPE}" == "gpu" ]]; then
+    remove_test_filter_tag no_gpu -requires-gpu
+    add_test_filter_tag requires-gpu
+  else
+    remove_test_filter_tag -no_gpu requires-gpu
+    add_test_filter_tag no_gpu -requires-gpu
+  fi
+  echo "Final test filter tags: ${BAZEL_TEST_FILTER_TAGS}"
+}
+
+# Check currently running python and pip version
+check_python_pip_version() {
+  # Check if only the major version of python is provided by the user.
+  MAJOR_VER_ONLY=0
+  if [[ ${#PYTHON_VER} -lt 9 ]]; then
+    # User only provided major version (e.g. 'python2' instead of 'python2.7')
+    MAJOR_VER_ONLY=1
+  fi
+
+  # Retrieve only the version number of the user requested python.
+  PYTHON_VER_REQUESTED=${PYTHON_VER:6:3}
+  echo "PYTHON_VER_REQUESTED: ${PYTHON_VER_REQUESTED}"
+
+  # Retrieve only the version numbers of the python & pip in use currently.
+  PYTHON_VER_IN_USE=$(python --version 2>&1)
+  PYTHON_VER_IN_USE=${PYTHON_VER_IN_USE:7:3}
+  PIP_VER_IN_USE=$(pip --version)
+  PIP_VER_IN_USE=${PIP_VER_IN_USE:${#PIP_VER_IN_USE}-4:3}
+
+  # If only major versions are applied, drop minor versions.
+  if [[ $MAJOR_VER_ONLY == 1 ]]; then
+    PYTHON_VER_IN_USE=${PYTHON_VER_IN_USE:0:1}
+    PIP_VER_IN_USE=${PIP_VER_IN_USE:0:1}
+  fi
+
+  # Check if all versions match.
+  echo -e "User requested python version: '${PYTHON_VER_REQUESTED}'\n" \
+    "Detected python version in use: '${PYTHON_VER_IN_USE}'\n"\
+    "Detected pip version in use: '${PIP_VER_IN_USE}'"
+  if ! [[ $PYTHON_VER_REQUESTED == $PYTHON_VER_IN_USE ]]; then
+    die "Error: Mismatch in python versions detected."
+  else:
+    echo "Python and PIP versions in use match the requested."
+  fi
+}
+
+###########################################################################
+# Setup: directories, local/global variables
+###########################################################################
+
+# Script directory and source necessary files.
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${SCRIPT_DIR}/builds_common.sh"
+
+# Required global variables
+# Checks on values for these vars are done in "Build TF PIP Package" section.
+CONTAINER_TYPE=$(lowercase "${CONTAINER_TYPE}")
+OS_TYPE=$(lowercase "${OS_TYPE}")
+PYTHON_VER=$(lowercase "${TF_PYTHON_VERSION}")
+
+# Python bin path
+if [[ -z "$PYTHON_BIN_PATH" ]]; then
+  die "Error: PYTHON_BIN_PATH was not provided. Did you run configure?"
+fi
+# Get python version for configuring pip later in installation.
+PYTHON_VER_CFG=$(${PYTHON_BIN_PATH} -V 2>&1 | awk '{print $NF}' | cut -d. -f-2)
+echo "PYTHON_BIN_PATH: ${PYTHON_BIN_PATH} (version: ${PYTHON_VER_CFG})"
+
+# Default values for optional global variables in case they are not user
+# defined.
+DEFAULT_BAZEL_BUILD_FLAGS='--test_output=errors --verbose_failures=true'
+DEFAULT_BAZEL_TEST_FILTERS='-no_oss,-oss_serial'
+DEFAULT_BAZEL_TEST_TARGETS='//tensorflow/python/... -//tensorflow/core/... -//tensorflow/compiler/... '
+DEFAULT_PIP_TESTS="" # Do not run any tests by default
+DEFAULT_IS_NIGHTLY=0 # Not nightly by default
+DEFAULT_PROJECT_NAME="tensorflow"
+DEFAULT_PIP_TEST_ROOT="pip_test"
+
+# Take in optional global variables
+BAZEL_BUILD_FLAGS=${TF_BUILD_FLAGS:-$DEFAULT_BAZEL_BUILD_FLAGS}
+BAZEL_TEST_TARGETS=${TF_TEST_TARGETS:-$DEFAULT_BAZEL_TEST_TARGETS}
+BAZEL_TEST_FILTER_TAGS=${TF_TEST_FILTER_TAGS:-$DEFAULT_BAZEL_TEST_FILTERS}
+PIP_TESTS=${TF_PIP_TESTS:-$DEFAULT_PIP_TESTS}
+IS_NIGHTLY=${IS_NIGHTLY:-$DEFAULT_IS_NIGHTLY}
+PROJECT_NAME=${TF_PROJECT_NAME:-$DEFAULT_PROJECT_NAME}
+PIP_TEST_ROOT=${TF_PIP_TEST_ROOT:-$DEFAULT_PIP_TEST_ROOT}
+
+# Local variables
+PIP_WHL_DIR="${KOKORO_ARTIFACTS_DIR}/tensorflow/${PIP_TEST_ROOT}/whl"
+mkdir -p "${PIP_WHL_DIR}"
+PIP_WHL_DIR=$(realpath "${PIP_WHL_DIR}") # Get absolute path
+WHL_PATH=""
+# Determine the major.minor versions of python being used (e.g., 2.7).
+# Useful for determining the directory of the local pip installation.
+PY_MAJOR_MINOR_VER=$(${PYTHON_BIN_PATH} -V 2>&1 | awk '{print $NF}' | cut -d. -f-2)
+if [[ -z "${PY_MAJOR_MINOR_VER}" ]]; then
+  die "ERROR: Unable to determine the major.minor version of Python."
+fi
+echo "Python binary path to be used in PIP install: ${PYTHON_BIN_PATH} "\
+"(Major.Minor version: ${PY_MAJOR_MINOR_VER})"
+
+# PIP packages
+INSTALL_EXTRA_PIP_PACKAGES=${TF_BUILD_INSTALL_EXTRA_PIP_PACKAGES}
+
+###########################################################################
+# Build TF PIP Package
+###########################################################################
+
+# First, check that global variables are properly set.
+check_global_vars
+
+# Check if in a virtualenv and exit if yes.
+IN_VENV=$(python -c 'import sys; print("1" if hasattr(sys, "real_prefix") else "0")')
+if [[ "$IN_VENV" == "1" ]]; then
+  echo "It appears that we are already in a virtualenv. Deactivating..."
+  deactivate || source deactivate || die "FAILED: Unable to deactivate from existing virtualenv."
+fi
+
+# Configure python. Obtain the path to python binary.
+source tools/python_bin_path.sh
+# Assume PYTHON_BIN_PATH is exported by the script above.
+if [[ -z "$PYTHON_BIN_PATH" ]]; then
+  die "PYTHON_BIN_PATH was not provided. Did you run configure?"
+fi
+
+# Bazel build the file.
+PIP_BUILD_TARGET="//tensorflow/tools/pip_package:build_pip_package"
+# Clean bazel cache.
+bazel clean
+# Clean up and update bazel flags
+update_bazel_flags
+# Build. This outputs the file `build_pip_package`.
+bazel build ${BAZEL_BUILD_FLAGS} ${PIP_BUILD_TARGET} || \
+  die "Error: Bazel build failed for target: '${PIP_BUILD_TARGET}'"
+
+###########################################################################
+# Test function(s)
+###########################################################################
+
+test_pip_virtualenv_clean() {
+  # Create a clean directory.
+  CLEAN_VENV_DIR="${PIP_TEST_ROOT}/venv_clean"
+
+  # activate virtual environment and install tensorflow with PIP.
+  create_activate_virtualenv --clean "${CLEAN_VENV_DIR}"
+  install_tensorflow_pip "${WHL_PATH}"
+
+  # cd to a temporary directory to avoid picking up Python files in the source
+  # tree.
+  TMP_DIR=$(mktemp -d)
+  pushd "${TMP_DIR}"
+
+  # Run a quick check on tensorflow installation.
+  RET_VAL=$(python -c "import tensorflow as tf; print(tf.Session().run(tf.constant(42)))")
+
+  # Deactivate virtualenv.
+  deactivate || source deactivate || die "FAILED: Unable to deactivate from existing virtualenv."
+
+  # Return to original directory. Remove temp dirs.
+  popd
+  sudo rm -rf "${TMP_DIR}" "${CLEAN_VENV_DIR}"
+
+  # Check result to see if tensorflow is properly installed.
+  if [[ ${RET_VAL} == 42 ]]; then
+    echo "PIP test on clean virtualenv PASSED."
+    return 0
+  else
+    echo "PIP test on clean virtualenv FAILED."
+    return 1
+  fi
+}
+
+test_pip_virtualenv_non_clean() {
+  # Create virtualenv directory for install test
+  VENV_DIR="${PIP_TEST_ROOT}/venv"
+
+  # Activate virtualenv
+  create_activate_virtualenv "${VENV_DIR}"
+  # Install TF with pip
+  install_tensorflow_pip "${WHL_PATH}"
+
+  # cd to a temporary directory to avoid picking up Python files in the source
+  # tree.
+  TMP_DIR=$(mktemp -d)
+  pushd "${TMP_DIR}"
+
+  # Run a quick check on tensorflow installation.
+  RET_VAL=$(python -c "import tensorflow as tf; print(tf.Session().run(tf.constant(42)))")
+
+  # Return to original directory. Remove temp dirs.
+  popd
+  sudo rm -rf "${TMP_DIR}"
+
+  # Check result to see if tensorflow is properly installed.
+  if [[ ${RET_VAL} -ne 42 ]]; then
+    echo "PIP test on virtualenv (non-clean) FAILED"
+    return 1
+  fi
+
+  # Install extra pip packages, if specified.
+  for PACKAGE in ${INSTALL_EXTRA_PIP_PACKAGES}; do
+    echo "Installing extra pip package required by test-on-install: ${PACKAGE}"
+
+    ${PIP_BIN_PATH} install ${PACKAGE}
+    if [[ $? != 0 ]]; then
+      echo "${PIP_BIN_PATH} install ${PACKAGE} FAILED."
+      deactivate || source deactivate || die "FAILED: Unable to deactivate from existing virtualenv."
+      return 1
+    fi
+  done
+
+  # Run bazel test.
+  run_test_with_bazel
+  RESULT=$?
+
+  # Deactivate from virtualenv.
+  deactivate || source deactivate || die "FAILED: Unable to deactivate from existing virtualenv."
+  sudo rm -rf "${VENV_DIR}"
+
+  if [[ $RESULT -ne 0 ]]; then
+    echo "PIP test on virtualenv (non-clean) FAILED."
+    return 1
+  else
+    echo "PIP test on virtualenv (non-clean) PASSED."
+    return 0
+  fi
+}
+
+test_pip_virtualenv_oss_serial() {
+  # Create virtualenv directory
+  VENV_DIR="${PIP_TEST_ROOT}/venv"
+
+  create_activate_virtualenv "${VENV_DIR}"
+  run_test_with_bazel --oss_serial
+  RESULT=$?
+
+  # deactivate virtualenv
+  deactivate || source deactivate || die "FAILED: Unable to deactivate from existing virtualenv."
+
+  if [[ ${RESULT} -ne 0 ]]; then
+    echo "PIP test on virtualenv (oss-serial) FAILED."
+    return 1
+  else
+    echo "PIP test on virtualenv (oss-serial) PASSED."
+    return 0
+  fi
+}
+
+###########################################################################
+# Test helper function(s)
+###########################################################################
+
+create_activate_virtualenv() {
+  VIRTUALENV_FLAGS="--system-site-packages"
+  if [[ "${1}" == "--clean" ]]; then
+    shift
+  fi
+
+  VIRTUALENV_DIR="${1}"
+  if [[ -d "${VIRTUALENV_DIR}" ]]; then
+    if sudo rm -rf "${VIRTUALENV_DIR}"
+    then
+      echo "Removed existing virtualenv directory: ${VIRTUALENV_DIR}"
+    else
+      die "Failed to remove existing virtualenv directory: ${VIRTUALENV_DIR}"
+    fi
+  fi
+
+  if mkdir -p "${VIRTUALENV_DIR}"
+  then
+    echo "Created virtualenv directory: ${VIRTUALENV_DIR}"
+  else
+    die "FAILED to create virtualenv directory: ${VIRTUALENV_DIR}"
+  fi
+
+  # Use the virtualenv from the default python version (i.e., python-virtualenv)
+  # to create the virtualenv directory for testing. Use the -p flag to specify
+  # the python version inside the to-be-created virtualenv directory.
+  ${PYTHON_BIN_PATH} -m virtualenv -p ${PYTHON_BIN_PATH} ${VIRTUALENV_FLAGS} ${VIRTUALENV_DIR} || \
+    die "FAILED: Unable to create virtualenv"
+
+  source "${VIRTUALENV_DIR}/bin/activate" || \
+    die "FAILED: Unable to activate virtualenv in ${VIRTUALENV_DIR}"
+}
+
+install_tensorflow_pip() {
+  if [[ -z "${1}" ]]; then
+    die "Please provide a proper wheel file path."
+  fi
+
+  TF_WHEEL_PATH="${1}"
+
+  # Upgrade pip so it supports tags such as cp27mu, manylinux1 etc.
+  echo "Upgrade pip in virtualenv"
+
+  # NOTE: pip install --upgrade pip leads to a documented TLS issue for
+  # some versions in python
+  curl https://bootstrap.pypa.io/get-pip.py | ${PYTHON_BIN_PATH}
+
+  # Configure matching pip version with python.
+  PIP_BIN_PATH="$(which pip${PYTHON_VER_CFG})"
+  echo "PIP_BIN_PATH: ${PIP_BIN_PATH}"
+
+  # Check that requested python version matches configured one.
+  check_python_pip_version
+
+  # Force upgrade of setuptools. This must happen before the pip install of the
+  # WHL_PATH, which pulls in absl-py, which uses install_requires notation
+  # introduced in setuptools >=20.5. The default version of setuptools is 5.5.1,
+  # which is too old for absl-py.
+  ${PIP_BIN_PATH} install --upgrade setuptools==39.1.0
+
+  # Force tensorflow reinstallation. Otherwise it may not get installed from
+  # last build if it had the same version number as previous build.
+  PIP_FLAGS="--upgrade --force-reinstall"
+  ${PIP_BIN_PATH} install -v ${PIP_FLAGS} ${WHL_PATH} || \
+    die "pip install (forcing to reinstall tensorflow) FAILED"
+  echo "Successfully installed pip package ${TF_WHEEL_PATH}"
+
+  # Force downgrade of setuptools. This must happen after the pip install of the
+  # WHL_PATH, which ends up upgrading to the latest version of setuptools.
+  # Versions of setuptools >= 39.1.0 will cause tests to fail like this:
+  #   ImportError: cannot import name py31compat
+  ${PIP_BIN_PATH} install --upgrade setuptools==39.1.0
+}
+
+run_test_with_bazel() {
+  IS_OSS_SERIAL=0
+  if [[ "${1}" == "--oss_serial" ]]; then
+    IS_OSS_SERIAL=1
+  fi
+  TF_GPU_COUNT=${TF_GPU_COUNT:-4}
+
+  # PIP tests should have a "different" path. Different than the one we place
+  # virtualenv, because we are deleting and recreating it here.
+  PIP_TEST_PREFIX=bazel_pip
+  PIP_TEST_ROOT=$(pwd)/${PIP_TEST_PREFIX}
+  sudo rm -rf $PIP_TEST_ROOT
+  mkdir -p $PIP_TEST_ROOT
+  ln -s $(pwd)/tensorflow ${PIP_TEST_ROOT}/tensorflow
+
+  if [[ "${IS_OSS_SERIAL}" == "1" ]]; then
+    remove_test_filter_tag -no_oss
+    add_test_filter_tag oss_serial
+  else
+    add_test_filter_tag -oss_serial
+  fi
+
+  # Clean the bazel cache
+  bazel clean
+  # Clean up flags before running bazel commands
+  update_bazel_flags
+  # Clean up and update test filter tags
+  update_test_filter_tags
+
+  # Figure out how many concurrent tests we can run and do run the tests.
+  BAZEL_PARALLEL_TEST_FLAGS=""
+  if [[ $CONTAINER_TYPE == "gpu" ]]; then
+    # Number of test threads is the number of GPU cards available.
+    if [[ $OS_TYPE == "macos" ]]; then
+      BAZEL_PARALLEL_TEST_FLAGS="--local_test_jobs=1"
+    else
+      BAZEL_PARALLEL_TEST_FLAGS="--local_test_jobs=${TF_GPU_COUNT} \
+        --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute"
+    fi
+  else
+    # Number of test threads is the number of physical CPUs.
+    if [[ $OS_TYPE == "macos" ]]; then
+      BAZEL_PARALLEL_TEST_FLAGS="--local_test_jobs=$(sysctl -n hw.ncpu)"
+    else
+      BAZEL_PARALLEL_TEST_FLAGS="--local_test_jobs=$(grep -c ^processor /proc/cpuinfo)"
+    fi
+  fi
+
+  if [[ ${IS_OSS_SERIAL} == 1 ]]; then
+    BAZEL_PARALLEL_TEST_FLAGS="--local_test_jobs=1"
+  fi
+
+  # Run the test.
+  bazel test ${BAZEL_BUILD_FLAGS} ${BAZEL_PARALLEL_TEST_FLAGS} --test_tag_filters=${BAZEL_TEST_FILTER_TAGS} -- ${BAZEL_TEST_TARGETS}
+}
+
+run_all_tests() {
+  if [[ -z "${PIP_TESTS}" ]]; then
+    echo "No test was specified to run. Skipping all tests."
+    return 0
+  fi
+  FAIL_COUNTER=0
+  PASS_COUNTER=0
+  for TEST in ${PIP_TESTS[@]}; do
+
+    # Run tests.
+    case "${TEST}" in
+    "test_pip_virtualenv_clean")
+      test_pip_virtualenv_clean
+      ;;
+    "test_pip_virtualenv_non_clean")
+      test_pip_virtualenv_non_clean
+      ;;
+    "test_pip_virtualenv_oss_serial")
+      test_pip_virtualenv_oss_serial
+      ;;
+    *)
+      die "No matching test ${TEST} was found. Stopping test."
+      ;;
+    esac
+
+    # Check and update the results.
+    RETVAL=$?
+
+    # Update results counter
+    if [ ${RETVAL} -eq 0 ]; then
+      echo "Test (${TEST}) PASSED. (PASS COUNTER: ${PASS_COUNTER})"
+      PASS_COUNTER=$(($PASS_COUNTER+1))
+    else
+      echo "Test (${TEST}) FAILED. (FAIL COUNTER: ${FAIL_COUNTER})"
+      FAIL_COUNTER=$(($FAIL_COUNTER+1))
+    fi
+  done
+  printf "${PASS_COUNTER} PASSED | ${FAIL_COUNTER} FAILED"
+  if [[ "${FAIL_COUNTER}" == "0" ]]; then
+    printf "PIP tests ${COLOR_GREEN}PASSED${COLOR_NC}\n"
+    return 0
+  else:
+    printf "PIP tests ${COLOR_RED}FAILED${COLOR_NC}\n"
+    return 1
+  fi
+}
+
+###########################################################################
+# Build TF PIP Wheel file
+###########################################################################
+
+# Update the build flags for building whl.
+# Flags: GPU, OS, tf_nightly, project name
+GPU_FLAG=""
+NIGHTLY_FLAG=""
+
+# TF Nightly flag
+if [[ "$IS_NIGHTLY" == 1 ]]; then
+  # If 'nightly' is not specified in the project name already, then add.
+  if ! [[ $PROJECT_NAME == *"nightly"* ]]; then
+    echo "WARNING: IS_NIGHTLY=${IS_NIGHTLY} but requested project name \
+    (PROJECT_NAME=${PROJECT_NAME}) does not include 'nightly' string. \
+    Renaming it to 'tf_nightly'."
+    PROJECT_NAME="tf_nightly"
+  fi
+  NIGHTLY_FLAG="--nightly_flag"
+fi
+
+# CPU / GPU flag
+if [[ ${CONTAINER_TYPE} == "gpu" ]]; then
+  GPU_FLAG="--gpu"
+  if ! [[ $PROJECT_NAME == *"gpu"* ]]; then
+    echo "WARNING: GPU is specified but requested project name (PROJECT_NAME=${PROJECT_NAME}) \
+    does not include 'gpu'. Appending '_gpu' to the project name."
+    PROJECT_NAME="${PROJECT_NAME}_gpu"
+  fi
+fi
+
+./bazel-bin/tensorflow/tools/pip_package/build_pip_package ${PIP_WHL_DIR} ${GPU_FLAG} ${NIGHTLY_FLAG} "--project_name" ${PROJECT_NAME} || die "build_pip_package FAILED"
+
+PY_MAJOR_MINOR_VER=$(echo $PY_MAJOR_MINOR_VER | tr -d '.')
+if [[ $PY_MAJOR_MINOR_VER == "2" ]]; then
+  PY_MAJOR_MINOR_VER="27"
+fi
+
+# Set wheel path and verify that there is only one .whl file in the path.
+WHL_PATH=$(ls "${PIP_WHL_DIR}"/"${PROJECT_NAME}"-*"${PY_MAJOR_MINOR_VER}"*"${PY_MAJOR_MINOR_VER}"*.whl)
+if [[ $(echo "${WHL_PATH}" | wc -w) -ne 1 ]]; then
+  echo "ERROR: Failed to find exactly one built TensorFlow .whl file in "\
+  "directory: ${PIP_WHL_DIR}"
+fi
+
+WHL_DIR=$(dirname "${WHL_PATH}")
+WHL_BASE_NAME=$(basename "${WHL_PATH}")
+AUDITED_WHL_NAME="${WHL_DIR}"/$(echo "${WHL_BASE_NAME//linux/manylinux1}")
+
+# Print the size of the wheel file.
+echo "Size of the PIP wheel file built: $(ls -l ${WHL_PATH} | awk '{print $5}')"
+
+# Run tests (if any is specified).
+run_all_tests
+
+for WHL_PATH in $(ls ${PIP_TEST_ROOT}/${PROJECT_NAME}*.whl); do
+  if [[ "${TF_NEED_CUDA}" -eq "1" ]]; then
+    # Copy and rename for gpu manylinux as we do not want auditwheel to package in libcudart.so
+    WHL_PATH=${AUDITED_WHL_NAME}
+    cp "${WHL_DIR}"/"${WHL_BASE_NAME}" "${WHL_PATH}"
+    echo "Copied manylinux1 wheel file at ${WHL_PATH}"
+  else
+    # Repair the wheels for cpu manylinux1
+    echo "auditwheel repairing ${WHL_PATH}"
+    auditwheel repair -w "${WHL_DIR}" "${WHL_PATH}"
+
+    if [[ -f ${AUDITED_WHL_NAME} ]]; then
+      WHL_PATH=${AUDITED_WHL_NAME}
+      echo "Repaired manylinux1 wheel file at: ${WHL_PATH}"
+    else
+      die "ERROR: Cannot find repaired wheel."
+    fi
+  fi
+done
+
+echo "EOF: Successfully ran pip_new.sh"
-- 
GitLab


From 1d179685a7a5fd4477c37a20aaad6ad1eacb4d66 Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Wed, 6 Feb 2019 17:36:06 -0800
Subject: [PATCH 2044/2345] Fix serialization bug in optimize dataset

PiperOrigin-RevId: 232782055
---
 tensorflow/core/kernels/data/BUILD            |  1 +
 tensorflow/core/kernels/data/dataset_utils.cc | 30 +++++++
 tensorflow/core/kernels/data/dataset_utils.h  |  7 ++
 .../core/kernels/data/dataset_utils_test.cc   | 86 +++++++++++++++++++
 .../kernels/data/graph_rewrite_dataset.cc     | 26 ++++--
 tensorflow/core/kernels/data/iterator_ops.cc  | 21 ++++-
 .../optimize_dataset_serialization_test.py    | 14 +++
 7 files changed, 174 insertions(+), 11 deletions(-)

diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD
index 56c8399b34..18114d6421 100644
--- a/tensorflow/core/kernels/data/BUILD
+++ b/tensorflow/core/kernels/data/BUILD
@@ -564,6 +564,7 @@ cc_library(
     srcs = ["graph_rewrite_dataset.cc"],
     hdrs = ["graph_rewrite_dataset.h"],
     deps = [
+        ":dataset_utils",
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
         "//tensorflow/core:protos_all_cc",
diff --git a/tensorflow/core/kernels/data/dataset_utils.cc b/tensorflow/core/kernels/data/dataset_utils.cc
index 5a6c8bac08..9def8c9161 100644
--- a/tensorflow/core/kernels/data/dataset_utils.cc
+++ b/tensorflow/core/kernels/data/dataset_utils.cc
@@ -231,5 +231,35 @@ Status VariantTensorDataWriter::WriteTensorInternal(StringPiece key,
   return Status::OK();
 }
 
+Status AddToFunctionLibrary(FunctionLibraryDefinition* base,
+                            const FunctionLibraryDefinition& to_add) {
+  for (const auto& fn : to_add.ListFunctionNames()) {
+    if (auto found = base->Find(fn)) {
+      if (!OpDefEqual(found->signature(), to_add.Find(fn)->signature())) {
+        return errors::InvalidArgument("Cannot add function '", fn,
+                                       "' because a different function with "
+                                       "the same signature already exists.");
+      }
+      TF_RETURN_IF_ERROR(base->RemoveFunction(fn));
+    }
+  }
+  return base->AddLibrary(to_add);
+}
+
+Status AddToFunctionLibrary(FunctionLibraryDefinition* base,
+                            const FunctionDefLibrary& to_add) {
+  for (const auto& fd : to_add.function()) {
+    if (auto found = base->Find(fd.signature().name())) {
+      if (!OpDefEqual(found->signature(), fd.signature())) {
+        return errors::InvalidArgument("Cannot add function '",
+                                       fd.signature().name(),
+                                       "' because a different function with "
+                                       "the same signature already exists.");
+      }
+      TF_RETURN_IF_ERROR(base->RemoveFunction(fd.signature().name()));
+    }
+  }
+  return base->AddLibrary(to_add);
+}
 }  // namespace data
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/data/dataset_utils.h b/tensorflow/core/kernels/data/dataset_utils.h
index 0754d1d266..d85e87ca09 100644
--- a/tensorflow/core/kernels/data/dataset_utils.h
+++ b/tensorflow/core/kernels/data/dataset_utils.h
@@ -98,6 +98,13 @@ class VariantTensorDataWriter : public IteratorStateWriter {
   std::vector<string> keys_;
 };
 
+// Adds the functions in `to_add` to `base`. If a function with a matching
+// signature already exists in `base`, replaces it with the function from
+// `to_add`.
+Status AddToFunctionLibrary(FunctionLibraryDefinition* base,
+                            const FunctionLibraryDefinition& to_add);
+Status AddToFunctionLibrary(FunctionLibraryDefinition* base,
+                            const FunctionDefLibrary& to_add);
 }  // namespace data
 }  // namespace tensorflow
 
diff --git a/tensorflow/core/kernels/data/dataset_utils_test.cc b/tensorflow/core/kernels/data/dataset_utils_test.cc
index 290399b3e2..bddd2d455e 100644
--- a/tensorflow/core/kernels/data/dataset_utils_test.cc
+++ b/tensorflow/core/kernels/data/dataset_utils_test.cc
@@ -14,6 +14,8 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/kernels/data/dataset_utils.h"
+#include "tensorflow/core/framework/function.h"
+#include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/variant.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/platform/test.h"
@@ -77,6 +79,90 @@ TEST(DatasetUtilsTest, VariantTensorDataNonExistentKey) {
             reader.ReadTensor("NonExistentKey", &val_tensor).code());
 }
 
+TEST(DatasetUtilsTest, AddToFunctionLibrary) {
+  auto make_fn_a = [](const string& fn_name) {
+    return FunctionDefHelper::Create(
+        /*function_name=*/fn_name,
+        /*in_def=*/{"arg: int64"},
+        /*out_def=*/{"ret: int64"},
+        /*attr_def=*/{},
+        /*node_def=*/{{{"node"}, "Identity", {"arg"}, {{"T", DT_INT64}}}},
+        /*ret_def=*/{{"ret", "node:output:0"}});
+  };
+
+  auto make_fn_b = [](const string& fn_name) {
+    return FunctionDefHelper::Create(
+        /*function_name=*/fn_name,
+        /*in_def=*/{"arg: int64"},
+        /*out_def=*/{"ret: int64"},
+        /*attr_def=*/{},
+        /*node_def=*/
+        {{{"node"}, "Identity", {"arg"}, {{"T", DT_INT64}}},
+         {{"node2"}, "Identity", {"node:output:0"}, {{"T", DT_INT64}}}},
+        /*ret_def=*/{{"ret", "node2:output:0"}});
+  };
+
+  FunctionDefLibrary fdef_base;
+  *fdef_base.add_function() = make_fn_a("0");
+  *fdef_base.add_function() = make_fn_a("1");
+  *fdef_base.add_function() = make_fn_a("2");
+
+  FunctionDefLibrary fdef_to_add;
+  *fdef_to_add.add_function() = make_fn_b("0");  // Override
+  *fdef_to_add.add_function() = make_fn_a("1");  // Do nothing
+  *fdef_to_add.add_function() = make_fn_b("3");  // Add new function
+
+  FunctionLibraryDefinition flib_0(OpRegistry::Global(), fdef_base);
+  TF_ASSERT_OK(AddToFunctionLibrary(&flib_0, fdef_to_add));
+
+  FunctionLibraryDefinition flib_1(OpRegistry::Global(), fdef_base);
+  FunctionLibraryDefinition flib_to_add(OpRegistry::Global(), fdef_to_add);
+  TF_ASSERT_OK(AddToFunctionLibrary(&flib_1, flib_to_add));
+
+  for (const auto& flib : {flib_0, flib_1}) {
+    EXPECT_TRUE(FunctionDefsEqual(*flib.Find("0"), make_fn_b("0")));
+    EXPECT_TRUE(FunctionDefsEqual(*flib.Find("1"), make_fn_a("1")));
+    EXPECT_TRUE(FunctionDefsEqual(*flib.Find("2"), make_fn_a("2")));
+    EXPECT_TRUE(FunctionDefsEqual(*flib.Find("3"), make_fn_b("3")));
+  }
+}
+
+TEST(DatasetUtilsTest, AddToFunctionLibraryWithConflictingSignatures) {
+  FunctionDefLibrary fdef_base;
+  *fdef_base.add_function() = FunctionDefHelper::Create(
+      /*function_name=*/"0",
+      /*in_def=*/{"arg: int64"},
+      /*out_def=*/{"ret: int64"},
+      /*attr_def=*/{},
+      /*node_def=*/{},
+      /*ret_def=*/{{"ret", "arg"}});
+
+  FunctionDefLibrary fdef_to_add;
+  *fdef_to_add.add_function() = FunctionDefHelper::Create(
+      /*function_name=*/"0",
+      /*in_def=*/{"arg: int64"},
+      /*out_def=*/{"ret: int64", "ret2: int64"},
+      /*attr_def=*/{},
+      /*node_def=*/{},
+      /*ret_def=*/{{"ret", "arg"}, {"ret2", "arg"}});
+
+  FunctionLibraryDefinition flib_0(OpRegistry::Global(), fdef_base);
+  Status s = AddToFunctionLibrary(&flib_0, fdef_to_add);
+  EXPECT_EQ(error::Code::INVALID_ARGUMENT, s.code());
+  EXPECT_EQ(
+      "Cannot add function '0' because a different function with the same "
+      "signature already exists.",
+      s.error_message());
+
+  FunctionLibraryDefinition flib_1(OpRegistry::Global(), fdef_base);
+  FunctionLibraryDefinition flib_to_add(OpRegistry::Global(), fdef_to_add);
+  s = AddToFunctionLibrary(&flib_1, flib_to_add);
+  EXPECT_EQ(error::Code::INVALID_ARGUMENT, s.code());
+  EXPECT_EQ(
+      "Cannot add function '0' because a different function with the same "
+      "signature already exists.",
+      s.error_message());
+}
 }  // namespace
 }  // namespace data
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/data/graph_rewrite_dataset.cc b/tensorflow/core/kernels/data/graph_rewrite_dataset.cc
index bc4bb46010..29deac9ad4 100644
--- a/tensorflow/core/kernels/data/graph_rewrite_dataset.cc
+++ b/tensorflow/core/kernels/data/graph_rewrite_dataset.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 #include "tensorflow/core/kernels/data/graph_rewrite_dataset.h"
 
+#include "tensorflow/core/kernels/data/dataset_utils.h"
 #include "tensorflow/core/protobuf/meta_graph.pb.h"
 #include "tensorflow/core/protobuf/rewriter_config.pb.h"
 
@@ -57,14 +58,9 @@ Status GraphRewriteDataset::Optimize(OpKernelContext* ctx) {
 
   // Some functions may have been modified without having their names
   // changed (for example, nested dataset graphs from FlatMap or
-  // Interleave). To avoid name conflicts, we remove these functions from
-  // flib_def_ before adding the optimized function library.
-  for (const FunctionDef& fd : graph_def.library().function()) {
-    if (flib_def_->Find(fd.signature().name()) != nullptr) {
-      TF_RETURN_IF_ERROR(flib_def_->RemoveFunction(fd.signature().name()));
-    }
-  }
-  TF_RETURN_IF_ERROR(flib_def_->AddLibrary(graph_def.library()));
+  // Interleave).
+  TF_RETURN_IF_ERROR(
+      AddToFunctionLibrary(flib_def_.get(), graph_def.library()));
 
   Graph graph(OpRegistry::Global());
   TF_RETURN_IF_ERROR(ImportGraphDef({}, graph_def, &graph, nullptr));
@@ -82,9 +78,21 @@ Status GraphRewriteDataset::Optimize(OpKernelContext* ctx) {
 Status GraphRewriteDataset::AsGraphDefInternal(SerializationContext* ctx,
                                                DatasetGraphDefBuilder* b,
                                                Node** output) const {
+  SerializationContext::Params params;
+  // The optimized input needs access to the newly optimized functions when
+  // it is serialized. Here, we use the optimized function library for
+  // serialization, which is the union of the function library from the
+  // OpKernelContext at dataset creation time and newly optimized functions.
+  // This includes all functions that optimized_input_ may use.
+  params.flib_def = flib_def_.get();
+  params.input_list = ctx->input_list();
+  params.optimization_only = ctx->optimization_only();
+  SerializationContext optimized_ctx(params);
+
   // We only serialize the optimized dataset to avoid re-running
   // optimizations when the input pipeline is restored from a checkpoint.
-  TF_RETURN_IF_ERROR(b->AddInputDataset(ctx, optimized_input_, output));
+  TF_RETURN_IF_ERROR(
+      b->AddInputDataset(&optimized_ctx, optimized_input_, output));
   return Status::OK();
 }
 
diff --git a/tensorflow/core/kernels/data/iterator_ops.cc b/tensorflow/core/kernels/data/iterator_ops.cc
index 0d2dfd962b..990cc75721 100644
--- a/tensorflow/core/kernels/data/iterator_ops.cc
+++ b/tensorflow/core/kernels/data/iterator_ops.cc
@@ -99,7 +99,17 @@ class IteratorResource : public ResourceBase {
       captured_state = iterator_state_;
     }
     if (captured_state) {
-      return captured_state->iterator->Save(ctx, writer);
+      SerializationContext::Params params;
+      // The iterator state may contain functions that are not present
+      // in ctx's function library. Namely, an iterator may be restored from
+      // a serialized iterator with a modified function library (for example, as
+      // a result of OptimizeDataset). These modified functions are needed
+      // to serialize the iterator again.
+      params.flib_def = captured_state->flib_def.get();
+      params.input_list = ctx->input_list();
+      params.optimization_only = ctx->optimization_only();
+      SerializationContext ctx_with_functions(params);
+      return captured_state->iterator->Save(&ctx_with_functions, writer);
     } else {
       return errors::FailedPrecondition(
           "Save() failed because the iterator has not been initialized. "
@@ -134,7 +144,14 @@ class IteratorResource : public ResourceBase {
     std::unique_ptr<FunctionLibraryDefinition> flib_def(nullptr);
     std::unique_ptr<ProcessFunctionLibraryRuntime> pflr(nullptr);
     TF_RETURN_IF_ERROR(ctx->function_library()->Clone(&flib_def, &pflr, &lib));
-    TF_RETURN_IF_ERROR(flib_def->AddLibrary(graph_def.library()));
+
+    // Some function names may be duplicated (for example, if the serialized
+    // graph has an optimized function that retains its original name). We
+    // override functions in flib_def in the event of conflict. It is
+    // safe to assume that any node in the serialized graph is referring to the
+    // serialized function when there is a conflict.
+    TF_RETURN_IF_ERROR(
+        AddToFunctionLibrary(flib_def.get(), graph_def.library()));
     std::unique_ptr<State> new_state = absl::make_unique<State>(
         std::move(flib_def), std::move(pflr), lib, nullptr /* iterator */);
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/optimize_dataset_serialization_test.py b/tensorflow/python/data/experimental/kernel_tests/serialization/optimize_dataset_serialization_test.py
index ed4a1da596..aaa46bacef 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/optimize_dataset_serialization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/optimize_dataset_serialization_test.py
@@ -34,6 +34,20 @@ class OptimizeDatasetSerializationTest(
 
     self.run_core_tests(lambda: build_dataset(200, 10), None, 20)
 
+  def testWithNewFunction(self):
+    """Tests that optimized datasets with new functions work."""
+
+    def build_dataset():
+      dataset = dataset_ops.Dataset.range(100)
+      dataset = dataset.map(lambda x: x)
+      dataset = dataset.batch(5)
+      # map_vectorization adds a new vectorized function to the function
+      # library.
+      dataset = dataset.apply(optimization.optimize(["map_vectorization"]))
+      return dataset
+
+    self.run_core_tests(build_dataset, None, 20)
+
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 9e9f25fbe8dab6013448e6ceae36f60115065f41 Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Wed, 6 Feb 2019 17:54:58 -0800
Subject: [PATCH 2045/2345] Add int8 support for topk_v2.

PiperOrigin-RevId: 232784488
---
 tensorflow/lite/kernels/register.cc     |  4 +++-
 tensorflow/lite/kernels/topk_v2.cc      |  4 ++++
 tensorflow/lite/kernels/topk_v2_test.cc | 16 ++++++++++++++++
 tensorflow/lite/toco/tflite/operator.cc |  5 +++++
 4 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index 948bf779d6..27b34c8f31 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -242,7 +242,9 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_SQUEEZE, Register_SQUEEZE());
   AddBuiltin(BuiltinOperator_STRIDED_SLICE, Register_STRIDED_SLICE());
   AddBuiltin(BuiltinOperator_EXP, Register_EXP());
-  AddBuiltin(BuiltinOperator_TOPK_V2, Register_TOPK_V2());
+  AddBuiltin(BuiltinOperator_TOPK_V2, Register_TOPK_V2(),
+             /* min_version */ 1,
+             /* max_version */ 2);
   AddBuiltin(BuiltinOperator_LOG, Register_LOG());
   AddBuiltin(BuiltinOperator_LOG_SOFTMAX, Register_LOG_SOFTMAX());
   AddBuiltin(BuiltinOperator_CAST, Register_CAST());
diff --git a/tensorflow/lite/kernels/topk_v2.cc b/tensorflow/lite/kernels/topk_v2.cc
index 444b01e7b2..64973d7b86 100644
--- a/tensorflow/lite/kernels/topk_v2.cc
+++ b/tensorflow/lite/kernels/topk_v2.cc
@@ -207,6 +207,10 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
       TopK(row_size, num_rows, input->data.uint8, k, output_indexes->data.i32,
            output_values->data.uint8);
       break;
+    case kTfLiteInt8:
+      TopK(row_size, num_rows, input->data.int8, k, output_indexes->data.i32,
+           output_values->data.int8);
+      break;
     case kTfLiteInt32:
       TopK(row_size, num_rows, input->data.i32, k, output_indexes->data.i32,
            output_values->data.i32);
diff --git a/tensorflow/lite/kernels/topk_v2_test.cc b/tensorflow/lite/kernels/topk_v2_test.cc
index 108b812366..0097ae2f9a 100644
--- a/tensorflow/lite/kernels/topk_v2_test.cc
+++ b/tensorflow/lite/kernels/topk_v2_test.cc
@@ -46,6 +46,10 @@ class TopKV2OpModel : public SingleOpModel {
     PopulateTensor<uint8_t>(input_, data);
   }
 
+  void SetInputInt8(std::initializer_list<int8_t> data) {
+    PopulateTensor<int8_t>(input_, data);
+  }
+
   void SetInputInt32(std::initializer_list<int32_t> data) {
     PopulateTensor<int32_t>(input_, data);
   }
@@ -66,6 +70,10 @@ class TopKV2OpModel : public SingleOpModel {
     return ExtractVector<uint8_t>(output_values_);
   }
 
+  std::vector<int8_t> GetValuesInt8() {
+    return ExtractVector<int8_t>(output_values_);
+  }
+
   std::vector<int32_t> GetValuesInt32() {
     return ExtractVector<int32_t>(output_values_);
   }
@@ -128,6 +136,14 @@ TEST(TopKV2OpTest, TypeUint8) {
   EXPECT_THAT(m.GetValuesUInt8(), ElementsAreArray({3, 2, 251, 250}));
 }
 
+TEST(TopKV2OpTest, TypeInt8) {
+  TopKV2OpModel m({2, 3}, TensorType_INT8, 2);
+  m.SetInputInt8({1, 2, 3, -126, 125, -24});
+  m.Invoke();
+  EXPECT_THAT(m.GetIndexes(), ElementsAreArray({2, 1, 1, 2}));
+  EXPECT_THAT(m.GetValuesInt8(), ElementsAreArray({3, 2, 125, -24}));
+}
+
 // Check that int32_t works.
 TEST(TopKV2OpTest, TypeInt32) {
   TopKV2OpModel m({2, 3}, TensorType_INT32, 2);
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index 58c4a8589b..50242531e7 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -1333,6 +1333,11 @@ class TopK_V2 : public BuiltinOperator<TopKV2Operator, ::tflite::TopKV2Options,
                    TocoOperator* op) const override {}
 
   int GetVersion(const OperatorSignature& op_signature) const override {
+    const string& input_name = op_signature.op->inputs[0];
+    const Array& input_array = op_signature.model->GetArray(input_name);
+    if (input_array.data_type == ArrayDataType::kInt8) {
+      return 2;
+    }
     return 1;
   }
 };
-- 
GitLab


From 30dd4d04192118154cd2f95a9a72cb70fb7b15a7 Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Wed, 6 Feb 2019 19:29:25 -0800
Subject: [PATCH 2046/2345] Add int8 support for ResizeNearestNeighbor.

PiperOrigin-RevId: 232794817
---
 tensorflow/lite/kernels/register.cc           |   4 +-
 .../lite/kernels/resize_nearest_neighbor.cc   |   8 +-
 .../kernels/resize_nearest_neighbor_test.cc   | 131 +++++++++++++++++-
 tensorflow/lite/toco/tflite/operator.cc       |   6 +
 4 files changed, 142 insertions(+), 7 deletions(-)

diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index 27b34c8f31..7998d3a756 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -226,7 +226,9 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_RESHAPE, Register_RESHAPE());
   AddBuiltin(BuiltinOperator_RESIZE_BILINEAR, Register_RESIZE_BILINEAR());
   AddBuiltin(BuiltinOperator_RESIZE_NEAREST_NEIGHBOR,
-             Register_RESIZE_NEAREST_NEIGHBOR());
+             Register_RESIZE_NEAREST_NEIGHBOR(),
+             /* min_version */ 1,
+             /* max_version */ 2);
   AddBuiltin(BuiltinOperator_SKIP_GRAM, Register_SKIP_GRAM());
   AddBuiltin(BuiltinOperator_SPACE_TO_DEPTH, Register_SPACE_TO_DEPTH());
   AddBuiltin(BuiltinOperator_GATHER, Register_GATHER(),
diff --git a/tensorflow/lite/kernels/resize_nearest_neighbor.cc b/tensorflow/lite/kernels/resize_nearest_neighbor.cc
index a48d8004f8..3030a4f28e 100644
--- a/tensorflow/lite/kernels/resize_nearest_neighbor.cc
+++ b/tensorflow/lite/kernels/resize_nearest_neighbor.cc
@@ -106,8 +106,14 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
           GetTensorShape(size), GetTensorData<int32>(size),
           GetTensorShape(output), GetTensorData<uint8_t>(output));
     }
+  } else if (output->type == kTfLiteInt8) {
+    reference_ops::ResizeNearestNeighbor(
+        op_params, GetTensorShape(input), GetTensorData<int8_t>(input),
+        GetTensorShape(size), GetTensorData<int32>(size),
+        GetTensorShape(output), GetTensorData<int8_t>(output));
   } else {
-    context->ReportError(context, "Output type is %d, requires float or uint8.",
+    context->ReportError(context,
+                         "Output type is %d, requires float, uint8 or int8.",
                          output->type);
     return kTfLiteError;
   }
diff --git a/tensorflow/lite/kernels/resize_nearest_neighbor_test.cc b/tensorflow/lite/kernels/resize_nearest_neighbor_test.cc
index 03e2effd84..63b4f13d64 100644
--- a/tensorflow/lite/kernels/resize_nearest_neighbor_test.cc
+++ b/tensorflow/lite/kernels/resize_nearest_neighbor_test.cc
@@ -79,7 +79,7 @@ TEST(ResizeNearestNeighborOpTest, HorizontalResize) {
               ElementsAreArray(ArrayFloatNear({3, 3, 6})));
 }
 
-TEST(ResizeNearestNeighborOpTest, HorizontalResize8Bit) {
+TEST(ResizeNearestNeighborOpTest, HorizontalResizeUInt8) {
   ResizeNearestNeighborOpModel m({TensorType_UINT8, {1, 1, 2, 1}});
   m.SetInput<uint8>({3, 6});
   m.SetSize({1, 3});
@@ -95,6 +95,21 @@ TEST(ResizeNearestNeighborOpTest, HorizontalResize8Bit) {
               ElementsAreArray(ArrayFloatNear({3, 3, 6})));
 }
 
+TEST(ResizeNearestNeighborOpTest, HorizontalResizeInt8) {
+  ResizeNearestNeighborOpModel m({TensorType_INT8, {1, 1, 2, 1}});
+  m.SetInput<int8_t>({-3, 6});
+  m.SetSize({1, 3});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutput<int8_t>(),
+              ElementsAreArray(ArrayFloatNear({-3, -3, 6})));
+
+  ResizeNearestNeighborOpModel const_m({TensorType_INT8, {1, 1, 2, 1}}, {1, 3});
+  const_m.SetInput<int8_t>({-3, 6});
+  const_m.Invoke();
+  EXPECT_THAT(const_m.GetOutput<int8_t>(),
+              ElementsAreArray(ArrayFloatNear({-3, -3, 6})));
+}
+
 TEST(ResizeNearestNeighborOpTest, VerticalResize) {
   ResizeNearestNeighborOpModel m({TensorType_FLOAT32, {1, 2, 1, 1}});
   m.SetInput<float>({3, 9});
@@ -111,7 +126,7 @@ TEST(ResizeNearestNeighborOpTest, VerticalResize) {
               ElementsAreArray(ArrayFloatNear({3, 3, 9})));
 }
 
-TEST(ResizeNearestNeighborOpTest, VerticalResize8Bit) {
+TEST(ResizeNearestNeighborOpTest, VerticalResizeUInt8) {
   ResizeNearestNeighborOpModel m({TensorType_UINT8, {1, 2, 1, 1}});
   m.SetInput<uint8>({3, 9});
   m.SetSize({3, 1});
@@ -127,6 +142,21 @@ TEST(ResizeNearestNeighborOpTest, VerticalResize8Bit) {
               ElementsAreArray(ArrayFloatNear({3, 3, 9})));
 }
 
+TEST(ResizeNearestNeighborOpTest, VerticalResizeInt8) {
+  ResizeNearestNeighborOpModel m({TensorType_INT8, {1, 2, 1, 1}});
+  m.SetInput<int8_t>({3, -9});
+  m.SetSize({3, 1});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutput<int8_t>(),
+              ElementsAreArray(ArrayFloatNear({3, 3, -9})));
+
+  ResizeNearestNeighborOpModel const_m({TensorType_INT8, {1, 2, 1, 1}}, {3, 1});
+  const_m.SetInput<int8_t>({3, -9});
+  const_m.Invoke();
+  EXPECT_THAT(const_m.GetOutput<int8_t>(),
+              ElementsAreArray(ArrayFloatNear({3, 3, -9})));
+}
+
 TEST(ResizeNearestNeighborOpTest, TwoDimensionalResize) {
   ResizeNearestNeighborOpModel m({TensorType_FLOAT32, {1, 2, 2, 1}});
   m.SetInput<float>({
@@ -155,7 +185,7 @@ TEST(ResizeNearestNeighborOpTest, TwoDimensionalResize) {
                                           })));
 }
 
-TEST(ResizeNearestNeighborOpTest, TwoDimensionalResize8Bit) {
+TEST(ResizeNearestNeighborOpTest, TwoDimensionalResizeUInt8) {
   ResizeNearestNeighborOpModel m({TensorType_UINT8, {1, 2, 2, 1}});
   m.SetInput<uint8>({
       3, 6,  //
@@ -183,6 +213,33 @@ TEST(ResizeNearestNeighborOpTest, TwoDimensionalResize8Bit) {
                                           })));
 }
 
+TEST(ResizeNearestNeighborOpTest, TwoDimensionalResizeInt8) {
+  ResizeNearestNeighborOpModel m({TensorType_INT8, {1, 2, 2, 1}});
+  m.SetInput<int8_t>({
+      3, -6,  //
+      9, 12   //
+  });
+  m.SetSize({3, 3});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutput<int8_t>(), ElementsAreArray(ArrayFloatNear({
+                                         3, 3, -6,  //
+                                         3, 3, -6,  //
+                                         9, 9, 12,  //
+                                     })));
+
+  ResizeNearestNeighborOpModel const_m({TensorType_INT8, {1, 2, 2, 1}}, {3, 3});
+  const_m.SetInput<int8_t>({
+      3, -6,  //
+      9, 12   //
+  });
+  const_m.Invoke();
+  EXPECT_THAT(const_m.GetOutput<int8_t>(), ElementsAreArray(ArrayFloatNear({
+                                               3, 3, -6,  //
+                                               3, 3, -6,  //
+                                               9, 9, 12,  //
+                                           })));
+}
+
 TEST(ResizeNearestNeighborOpTest, TwoDimensionalResizeWithTwoBatches) {
   ResizeNearestNeighborOpModel m({TensorType_FLOAT32, {2, 2, 2, 1}});
   m.SetInput<float>({
@@ -249,7 +306,7 @@ TEST(ResizeNearestNeighborOpTest, ThreeDimensionalResize) {
                                           })));
 }
 
-TEST(ResizeNearestNeighborOpTest, TwoDimensionalResizeWithTwoBatches8Bit) {
+TEST(ResizeNearestNeighborOpTest, TwoDimensionalResizeWithTwoBatchesUInt8) {
   ResizeNearestNeighborOpModel m({TensorType_UINT8, {2, 2, 2, 1}});
   m.SetInput<uint8>({
       3, 6,   //
@@ -287,7 +344,44 @@ TEST(ResizeNearestNeighborOpTest, TwoDimensionalResizeWithTwoBatches8Bit) {
                                           })));
 }
 
-TEST(ResizeNearestNeighborOpTest, ThreeDimensionalResize8Bit) {
+TEST(ResizeNearestNeighborOpTest, TwoDimensionalResizeWithTwoBatchesInt8) {
+  ResizeNearestNeighborOpModel m({TensorType_INT8, {2, 2, 2, 1}});
+  m.SetInput<int8_t>({
+      3, 6,    //
+      9, -12,  //
+      -4, 10,  //
+      12, 16   //
+  });
+  m.SetSize({3, 3});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutput<int8_t>(), ElementsAreArray(ArrayFloatNear({
+                                         3, 3, 6,     //
+                                         3, 3, 6,     //
+                                         9, 9, -12,   //
+                                         -4, -4, 10,  //
+                                         -4, -4, 10,  //
+                                         12, 12, 16,  //
+                                     })));
+
+  ResizeNearestNeighborOpModel const_m({TensorType_INT8, {2, 2, 2, 1}}, {3, 3});
+  const_m.SetInput<int8_t>({
+      3, 6,    //
+      9, -12,  //
+      -4, 10,  //
+      12, 16   //
+  });
+  const_m.Invoke();
+  EXPECT_THAT(const_m.GetOutput<int8_t>(), ElementsAreArray(ArrayFloatNear({
+                                               3, 3, 6,     //
+                                               3, 3, 6,     //
+                                               9, 9, -12,   //
+                                               -4, -4, 10,  //
+                                               -4, -4, 10,  //
+                                               12, 12, 16,  //
+                                           })));
+}
+
+TEST(ResizeNearestNeighborOpTest, ThreeDimensionalResizeUInt8) {
   ResizeNearestNeighborOpModel m({TensorType_UINT8, {1, 2, 2, 2}});
   m.SetInput<uint8>({
       3, 4, 6, 10,     //
@@ -315,6 +409,33 @@ TEST(ResizeNearestNeighborOpTest, ThreeDimensionalResize8Bit) {
                                           })));
 }
 
+TEST(ResizeNearestNeighborOpTest, ThreeDimensionalResizeInt8) {
+  ResizeNearestNeighborOpModel m({TensorType_INT8, {1, 2, 2, 2}});
+  m.SetInput<int8_t>({
+      3, 4, -6, 10,     //
+      10, 12, -14, 16,  //
+  });
+  m.SetSize({3, 3});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutput<int8_t>(), ElementsAreArray(ArrayFloatNear({
+                                         3, 4, 3, 4, -6, 10,       //
+                                         3, 4, 3, 4, -6, 10,       //
+                                         10, 12, 10, 12, -14, 16,  //
+                                     })));
+
+  ResizeNearestNeighborOpModel const_m({TensorType_INT8, {1, 2, 2, 2}}, {3, 3});
+  const_m.SetInput<int8_t>({
+      3, 4, -6, 10,     //
+      10, 12, -14, 16,  //
+  });
+  const_m.Invoke();
+  EXPECT_THAT(const_m.GetOutput<int8_t>(), ElementsAreArray(ArrayFloatNear({
+                                               3, 4, 3, 4, -6, 10,       //
+                                               3, 4, 3, 4, -6, 10,       //
+                                               10, 12, 10, 12, -14, 16,  //
+                                           })));
+}
+
 }  // namespace
 }  // namespace tflite
 
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index 50242531e7..deff93db62 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -1212,6 +1212,12 @@ class ResizeNearestNeighbor
   }
 
   int GetVersion(const OperatorSignature& op_signature) const override {
+    const string& input_name = op_signature.op->inputs[0];
+    const Array& input_array = op_signature.model->GetArray(input_name);
+    // Version 2 supports signed int8 input types.
+    if (input_array.data_type == ArrayDataType::kInt8) {
+      return 2;
+    }
     return 1;
   }
 };
-- 
GitLab


From be3d1dc8354f9edde67bb0071b73e4cf22d15348 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 6 Feb 2019 19:46:25 -0800
Subject: [PATCH 2047/2345] Make the split op work with int32 in tf-lite.

PiperOrigin-RevId: 232796544
---
 tensorflow/lite/kernels/register.cc     |  2 +-
 tensorflow/lite/kernels/split.cc        | 11 ++++++---
 tensorflow/lite/kernels/split_test.cc   | 31 +++++++++++++++++++++++++
 tensorflow/lite/toco/tflite/operator.cc |  4 +++-
 4 files changed, 43 insertions(+), 5 deletions(-)

diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index 7998d3a756..9a824b49a0 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -239,7 +239,7 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_DIV, Register_DIV());
   AddBuiltin(BuiltinOperator_SUB, Register_SUB());
   AddBuiltin(BuiltinOperator_SPLIT, Register_SPLIT(), /* min_version */ 1,
-             /* max_version */ 2);
+             /* max_version */ 3);
   AddBuiltin(BuiltinOperator_SPLIT_V, Register_SPLIT_V());
   AddBuiltin(BuiltinOperator_SQUEEZE, Register_SQUEEZE());
   AddBuiltin(BuiltinOperator_STRIDED_SLICE, Register_STRIDED_SLICE());
diff --git a/tensorflow/lite/kernels/split.cc b/tensorflow/lite/kernels/split.cc
index 4112898290..c0f701f55d 100644
--- a/tensorflow/lite/kernels/split.cc
+++ b/tensorflow/lite/kernels/split.cc
@@ -78,7 +78,8 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   auto input_type = op_context.input->type;
   TF_LITE_ENSURE(context,
                  input_type == kTfLiteFloat32 || input_type == kTfLiteUInt8 ||
-                     input_type == kTfLiteInt8 || input_type == kTfLiteInt16);
+                     input_type == kTfLiteInt8 || input_type == kTfLiteInt16 ||
+                     input_type == kTfLiteInt32);
   for (int i = 0; i < NumOutputs(node); ++i) {
     GetOutput(context, node, i)->type = input_type;
   }
@@ -145,10 +146,14 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
       TF_LITE_SPLIT(int16_t);
       break;
     }
+    case kTfLiteInt32: {
+      TF_LITE_SPLIT(int32_t);
+      break;
+    }
     default:
       context->ReportError(context,
-                           "Only float32, uint8, int8 and int16 are currently "
-                           "supported, got %d.",
+                           "Only float32, uint8, int8, int16 and int32 are "
+                           "currently supported, got %d.",
                            op_context.input->type);
       return kTfLiteError;
   }
diff --git a/tensorflow/lite/kernels/split_test.cc b/tensorflow/lite/kernels/split_test.cc
index aa2300722c..fa313d4b18 100644
--- a/tensorflow/lite/kernels/split_test.cc
+++ b/tensorflow/lite/kernels/split_test.cc
@@ -157,6 +157,37 @@ TEST(SplitOpTest, FourDimensionalInt8) {
                 TensorType_INT8);
 }
 
+TEST(SplitOpTest, FourDimensionalInt32) {
+  Check<int32_t>(/*axis=*/0, /*num_splits=*/2, {2, 2, 2, 2}, {1, 2, 2, 2},
+                 {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16},
+                 {
+                     {1, 2, 3, 4, 5, 6, 7, 8},
+                     {9, 10, 11, 12, 13, 14, 15, 16},
+                 },
+                 TensorType_INT32);
+  Check<int32_t>(/*axis=*/1, /*num_splits=*/2, {2, 2, 2, 2}, {2, 1, 2, 2},
+                 {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16},
+                 {
+                     {1, 2, 3, 4, 9, 10, 11, 12},
+                     {5, 6, 7, 8, 13, 14, 15, 16},
+                 },
+                 TensorType_INT32);
+  Check<int32_t>(/*axis=*/2, /*num_splits=*/2, {2, 2, 2, 2}, {2, 2, 1, 2},
+                 {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16},
+                 {
+                     {1, 2, 5, 6, 9, 10, 13, 14},
+                     {3, 4, 7, 8, 11, 12, 15, 16},
+                 },
+                 TensorType_INT32);
+  Check<int32_t>(/*axis=*/3, /*num_splits=*/2, {2, 2, 2, 2}, {2, 2, 2, 1},
+                 {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16},
+                 {
+                     {1, 3, 5, 7, 9, 11, 13, 15},
+                     {2, 4, 6, 8, 10, 12, 14, 16},
+                 },
+                 TensorType_INT32);
+}
+
 TEST(SplitOpTest, OneDimensional) {
   Check<float>(/*axis=*/0, /*num_splits=*/8, {8}, {1}, {1, 2, 3, 4, 5, 6, 7, 8},
                {{1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}});
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index deff93db62..07ca10da7c 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -1267,9 +1267,11 @@ class Split
   int GetVersion(const OperatorSignature& op_signature) const override {
     const string& input_name = op_signature.op->inputs[0];
     const Array& input_array = op_signature.model->GetArray(input_name);
-    // If the op take int8 input, it is version 2.
+    // If the op take int8 input, it is version 2, for int32 it's version 3.
     if (input_array.data_type == ArrayDataType::kInt8) {
       return 2;
+    } else if (input_array.data_type == ArrayDataType::kInt32) {
+      return 3;
     }
     return 1;
   }
-- 
GitLab


From dd1d9f654988695b86e854818b65ccda8e84515d Mon Sep 17 00:00:00 2001
From: Tong Shen <endlessroad@google.com>
Date: Wed, 6 Feb 2019 19:49:43 -0800
Subject: [PATCH 2048/2345] Better error message.

PiperOrigin-RevId: 232796868
---
 tensorflow/c/c_api_function.cc | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/tensorflow/c/c_api_function.cc b/tensorflow/c/c_api_function.cc
index 1477d643b5..45d6c33a1e 100644
--- a/tensorflow/c/c_api_function.cc
+++ b/tensorflow/c/c_api_function.cc
@@ -265,17 +265,19 @@ Status FillFunctionBody(
       // This node's attribute is a placeholder value, so it does not have type
       // information. We check node's OpDef for attribute type.
       string node_attr_name = iter.first;
-      const OpDef_AttrDef* node_attr_def = nullptr;
+      const OpDef::AttrDef* node_attr_def = nullptr;
       for (const auto& node_attr : node->op_def().attr()) {
         if (node_attr.name() == node_attr_name) {
           node_attr_def = &node_attr;
         }
       }
       if (!node_attr_def) {
-        return errors::Internal("Cannot find attr ", node_attr_name,
-                                " in OpDef ", node->op_def().DebugString());
+        return errors::Unimplemented(
+            "Placeholder value is not supported for attributes not in OpDef. "
+            "Attribute: ",
+            node_attr_name, ", OpDef: ", node->op_def().DebugString());
       }
-      OpDef_AttrDef* attr_def = fdef->mutable_signature()->add_attr();
+      OpDef::AttrDef* attr_def = fdef->mutable_signature()->add_attr();
       attr_def->set_name(func_attr_name);
       attr_def->set_type(node_attr_def->type());
 
-- 
GitLab


From a252fe83f4dc29c0614983046825ec75a3529cb9 Mon Sep 17 00:00:00 2001
From: Craig Citro <craigcitro@google.com>
Date: Wed, 6 Feb 2019 20:32:43 -0800
Subject: [PATCH 2049/2345] Add a hook to skip checking for GCE credentials.

Currently, when attempting to fetch an object from GCS, TF will attempt to
detect the GCE metadata service, and (if available) use the metadata service
for fetching auth tokens.

However, in many environments (such as Colab), all attempts to reach the
metadata service will fail by timing out. TF will valiantly retry, leading to
a situation where calling `tf.io.gfile.exists('gs://some-public-bucket')` will
take almost 700s (!) to finish all retries. Once these retries complete, we
attempt the request with an empty bearer token, and the request succeeds.

Once the request succeeds, TF sets an indefinite expiration time, meaning that
an interactive user can't (say) call `gcloud auth` and try again.

This change addresses this problem by adding a new hook for completely
skipping the GCE credential fetch, in the form of the `$NO_GCE_CHECK`
environment variable. This already exists in other Google auth libraries, eg
the Java client:
  https://github.com/googleapis/google-auth-library-java/blob/999de3b11de320354a8ff80a8dc906723d708cf4/oauth2_http/java/com/google/auth/oauth2/DefaultCredentialsProvider.java#L79
When set to any value (even the empty string), the google auth provider
completely skips attempts to talk to the GCE metadata service. In addition, we
don't set an indefinite expiration time in this case, so that future attempts
to fetch credentials aren't skipped.

Fixes #25463. (At least, provides the hook for Colab to use.)
Offers one potential solution to #25464.

PiperOrigin-RevId: 232800897
---
 .../platform/cloud/google_auth_provider.cc    | 32 ++++++++++++++++---
 .../platform/cloud/google_auth_provider.h     |  2 +-
 .../cloud/google_auth_provider_test.cc        | 25 ++++++++++++++-
 3 files changed, 52 insertions(+), 7 deletions(-)

diff --git a/tensorflow/core/platform/cloud/google_auth_provider.cc b/tensorflow/core/platform/cloud/google_auth_provider.cc
index e15400780a..c27fb5e105 100644
--- a/tensorflow/core/platform/cloud/google_auth_provider.cc
+++ b/tensorflow/core/platform/cloud/google_auth_provider.cc
@@ -44,6 +44,10 @@ constexpr char kGoogleAuthTokenForTesting[] = "GOOGLE_AUTH_TOKEN_FOR_TESTING";
 // The environment variable which can override '~/.config/gcloud' if set.
 constexpr char kCloudSdkConfig[] = "CLOUDSDK_CONFIG";
 
+// The environment variable used to skip attempting to fetch GCE credentials
+// if set.
+constexpr char kNoGceCheck[] = "NO_GCE_CHECK";
+
 // The default path to the gcloud config folder, relative to the home folder.
 constexpr char kGCloudConfigFolder[] = ".config/gcloud/";
 
@@ -146,10 +150,23 @@ Status GoogleAuthProvider::GetToken(string* t) {
   }
 
   auto token_from_files_status = GetTokenFromFiles();
-  auto token_from_gce_status =
-      token_from_files_status.ok() ? Status::OK() : GetTokenFromGce();
+  if (token_from_files_status.ok()) {
+    *t = current_token_;
+    return Status::OK();
+  }
+
+  bool skip_gce_check = std::getenv(kNoGceCheck) != nullptr;
+  Status token_from_gce_status;
+  if (skip_gce_check) {
+    token_from_gce_status =
+        Status(error::CANCELLED,
+               strings::StrCat("GCE check skipped due to presence of $",
+                               kNoGceCheck, " environment variable."));
+  } else {
+    token_from_gce_status = GetTokenFromGce();
+  }
 
-  if (token_from_files_status.ok() || token_from_gce_status.ok()) {
+  if (token_from_gce_status.ok()) {
     *t = current_token_;
     return Status::OK();
   }
@@ -165,8 +182,13 @@ Status GoogleAuthProvider::GetToken(string* t) {
   // so return an empty token instead of failing.
   *t = "";
 
-  // From now on, always return the empty token.
-  expiration_timestamp_sec_ = UINT64_MAX;
+  // We only want to keep returning our empty token if we've tried and failed
+  // the (potentially slow) task of detecting GCE.
+  if (skip_gce_check) {
+    expiration_timestamp_sec_ = 0;
+  } else {
+    expiration_timestamp_sec_ = UINT64_MAX;
+  }
   current_token_ = "";
 
   return Status::OK();
diff --git a/tensorflow/core/platform/cloud/google_auth_provider.h b/tensorflow/core/platform/cloud/google_auth_provider.h
index 3755b124a8..4ab816d54c 100644
--- a/tensorflow/core/platform/cloud/google_auth_provider.h
+++ b/tensorflow/core/platform/cloud/google_auth_provider.h
@@ -51,7 +51,7 @@ class GoogleAuthProvider : public AuthProvider {
   /// Gets the bearer token from Google Compute Engine environment.
   Status GetTokenFromGce() EXCLUSIVE_LOCKS_REQUIRED(mu_);
 
-  /// Gets the bearer token from the systen env variable, for testing purposes.
+  /// Gets the bearer token from the system env variable, for testing purposes.
   Status GetTokenForTesting() EXCLUSIVE_LOCKS_REQUIRED(mu_);
 
   std::unique_ptr<OAuthClient> oauth_client_;
diff --git a/tensorflow/core/platform/cloud/google_auth_provider_test.cc b/tensorflow/core/platform/cloud/google_auth_provider_test.cc
index ec31c5ee8c..0a6b108718 100644
--- a/tensorflow/core/platform/cloud/google_auth_provider_test.cc
+++ b/tensorflow/core/platform/cloud/google_auth_provider_test.cc
@@ -69,9 +69,10 @@ class GoogleAuthProviderTest : public ::testing::Test {
   void TearDown() override { ClearEnvVars(); }
 
   void ClearEnvVars() {
-    unsetenv("GOOGLE_APPLICATION_CREDENTIALS");
     unsetenv("CLOUDSDK_CONFIG");
+    unsetenv("GOOGLE_APPLICATION_CREDENTIALS");
     unsetenv("GOOGLE_AUTH_TOKEN_FOR_TESTING");
+    unsetenv("NO_GCE_CHECK");
   }
 };
 
@@ -238,4 +239,26 @@ TEST_F(GoogleAuthProviderTest, NothingAvailable) {
   EXPECT_EQ("", token);
 }
 
+TEST_F(GoogleAuthProviderTest, NoGceCheckEnvironmentVariable) {
+  setenv("NO_GCE_CHECK", "", 1);
+  auto oauth_client = new FakeOAuthClient;
+
+  FakeEnv env;
+  // If the env var above isn't respected, attempting to fetch a token
+  // from GCE will segfault (as the metadata client is null).
+  GoogleAuthProvider provider(std::unique_ptr<OAuthClient>(oauth_client),
+                              nullptr, &env);
+
+  string token;
+  TF_EXPECT_OK(provider.GetToken(&token));
+  EXPECT_EQ("", token);
+
+  // We also want to confirm that our empty token has a short expiration set: we
+  // now set a testing token, and confirm that it's returned instead of our
+  // empty token.
+  setenv("GOOGLE_AUTH_TOKEN_FOR_TESTING", "newToken", 1);
+  TF_EXPECT_OK(provider.GetToken(&token));
+  EXPECT_EQ("newToken", token);
+}
+
 }  // namespace tensorflow
-- 
GitLab


From 6a7c03870ade37d2f29b8314a1628af71c6bf094 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 6 Feb 2019 21:01:38 -0800
Subject: [PATCH 2050/2345] Fix large tests to size medium.

PiperOrigin-RevId: 232803438
---
 tensorflow/contrib/losses/BUILD        | 3 ++-
 tensorflow/contrib/opt/BUILD           | 3 ++-
 tensorflow/contrib/tensor_forest/BUILD | 3 ++-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/losses/BUILD b/tensorflow/contrib/losses/BUILD
index 728f75f8ef..f4ebbdeee8 100644
--- a/tensorflow/contrib/losses/BUILD
+++ b/tensorflow/contrib/losses/BUILD
@@ -82,10 +82,11 @@ py_library(
 
 py_test(
     name = "metric_loss_ops_test",
-    size = "large",
+    size = "medium",
     srcs = [
         "python/metric_learning/metric_loss_ops_test.py",
     ],
+    shard_count = 4,
     srcs_version = "PY2AND3",
     deps = [
         ":metric_learning_py",
diff --git a/tensorflow/contrib/opt/BUILD b/tensorflow/contrib/opt/BUILD
index 12320d9e45..f30643cf30 100644
--- a/tensorflow/contrib/opt/BUILD
+++ b/tensorflow/contrib/opt/BUILD
@@ -413,8 +413,9 @@ py_test(
 
 py_test(
     name = "shampoo_test",
-    size = "large",
+    size = "medium",
     srcs = ["python/training/shampoo_test.py"],
+    shard_count = 4,
     srcs_version = "PY2AND3",
     deps = [
         ":opt_py",
diff --git a/tensorflow/contrib/tensor_forest/BUILD b/tensorflow/contrib/tensor_forest/BUILD
index 398ac314f4..583bbf97c5 100644
--- a/tensorflow/contrib/tensor_forest/BUILD
+++ b/tensorflow/contrib/tensor_forest/BUILD
@@ -537,8 +537,9 @@ py_library(
 
 py_test(
     name = "random_forest_test",
-    size = "large",
+    size = "medium",
     srcs = ["client/random_forest_test.py"],
+    shard_count = 6,
     srcs_version = "PY2AND3",
     tags = [
         "noasan",
-- 
GitLab


From d32e7484dcbfcbe0993a78f84300e32a0649bd62 Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Thu, 7 Feb 2019 10:54:24 +0530
Subject: [PATCH 2051/2345] Docstring updated for missed 'Raises"

---
 tensorflow/python/keras/backend.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index 37e605ea2c..0f86e31127 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -4365,6 +4365,7 @@ def separable_conv2d(x,
   Raises:
       ValueError: if `data_format` is neither `channels_last` or
       `channels_first`.
+      ValueError: if `strides` is not a tuple of 2 integers.
   """
   if data_format is None:
     data_format = image_data_format()
@@ -4572,6 +4573,8 @@ def pool2d(x,
   Raises:
       ValueError: if `data_format` is neither `"channels_last"` or
       `"channels_first"`.
+      ValueError: if `pool_size` is not a tuple of 2 integers.
+      ValueError: if `strides` is not a tuple of 2 integers.
       ValueError: if `pool_mode` is neither `"max"` or `"avg"`.
   """
   if data_format is None:
-- 
GitLab


From ab842858b00d3fed43c3476990c13bdb0086b8ce Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <penporn@google.com>
Date: Wed, 6 Feb 2019 22:49:00 -0800
Subject: [PATCH 2052/2345] Internal change.

PiperOrigin-RevId: 232812429
---
 tensorflow/core/kernels/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index ea651f166c..c3400fbf1b 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -6833,6 +6833,7 @@ tf_cc_test_mkl(
     name = "mkl_conv_ops_test",
     size = "small",
     srcs = ["mkl_conv_ops_test.cc"],
+    tags = ["no_mac"],
     deps = [
         ":ops_testutil",
         ":ops_util",
-- 
GitLab


From 4987df47a13b8243900ab78eee9a14a7c589880d Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Wed, 6 Feb 2019 23:02:56 -0800
Subject: [PATCH 2053/2345] [XLA] Outline the code that checks the computations
 and instructions of a module have unique names and IDs to a member function
 of HloModule.

PiperOrigin-RevId: 232813715
---
 tensorflow/compiler/xla/service/hlo_module.cc | 52 +++++++++++--------
 tensorflow/compiler/xla/service/hlo_module.h  |  2 +
 2 files changed, 32 insertions(+), 22 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_module.cc b/tensorflow/compiler/xla/service/hlo_module.cc
index 759ce6541e..8322870cfd 100644
--- a/tensorflow/compiler/xla/service/hlo_module.cc
+++ b/tensorflow/compiler/xla/service/hlo_module.cc
@@ -246,6 +246,34 @@ HloModuleProto HloModule::ToProto() const {
   return proto;
 }
 
+Status HloModule::CheckUniqueNamesAndIdsForComputationsAndInstructions() const {
+  absl::flat_hash_set<string> computation_names;
+  absl::flat_hash_set<int> computation_ids;
+  absl::flat_hash_set<string> instruction_names;
+  absl::flat_hash_set<int> instruction_ids;
+
+  for (const HloComputation* computation : computations()) {
+    TF_RET_CHECK(!ContainsKey(computation_names, computation->name()))
+        << "Computation name is not unique: " << computation->name();
+    computation_names.insert(computation->name());
+
+    TF_RET_CHECK(!ContainsKey(computation_ids, computation->unique_id()))
+        << "Computation id is not unique: " << computation->unique_id();
+    computation_ids.insert(computation->unique_id());
+
+    for (const HloInstruction* instruction : computation->instructions()) {
+      TF_RET_CHECK(!ContainsKey(instruction_names, instruction->name()))
+          << "Instruction name is not unique: " << instruction->name();
+      instruction_names.insert(instruction->name());
+
+      TF_RET_CHECK(!ContainsKey(instruction_ids, instruction->unique_id()))
+          << "Instruction id is not unique: " << instruction->unique_id();
+      instruction_ids.insert(instruction->unique_id());
+    }
+  }
+  return Status::OK();
+}
+
 /* static */
 StatusOr<std::unique_ptr<HloModule>> HloModule::CreateFromProto(
     const HloModuleProto& proto, const HloModuleConfig& module_config) {
@@ -329,28 +357,8 @@ StatusOr<std::unique_ptr<HloModule>> HloModule::CreateFromProto(
                       DynamicParameterBinding::CreateFromProto(
                           proto.dynamic_parameter_binding()));
 
-  absl::flat_hash_set<string> computation_names;
-  absl::flat_hash_set<string> instruction_names;
-  absl::flat_hash_set<int> computation_ids;
-  absl::flat_hash_set<int> instruction_ids;
-  for (HloComputation* computation : module->computations()) {
-    TF_RET_CHECK(!ContainsKey(computation_names, computation->name()))
-        << "Computation name is not unique: " << computation->name();
-    computation_names.insert(computation->name());
-
-    TF_RET_CHECK(!ContainsKey(computation_ids, computation->unique_id()))
-        << "Computation id is not unique: " << computation->unique_id();
-    computation_ids.insert(computation->unique_id());
-    for (HloInstruction* instruction : computation->instructions()) {
-      TF_RET_CHECK(!ContainsKey(instruction_names, instruction->name()))
-          << "Instruction name is not unique: " << instruction->name();
-      instruction_names.insert(instruction->name());
-
-      TF_RET_CHECK(!ContainsKey(instruction_ids, instruction->unique_id()))
-          << "Instruction id is not unique: " << instruction->unique_id();
-      instruction_ids.insert(instruction->unique_id());
-    }
-  }
+  TF_RETURN_IF_ERROR(
+      module->CheckUniqueNamesAndIdsForComputationsAndInstructions());
 
   if (proto.has_schedule()) {
     TF_ASSIGN_OR_RETURN(
diff --git a/tensorflow/compiler/xla/service/hlo_module.h b/tensorflow/compiler/xla/service/hlo_module.h
index 20d98b96a9..8a2a0c8da2 100644
--- a/tensorflow/compiler/xla/service/hlo_module.h
+++ b/tensorflow/compiler/xla/service/hlo_module.h
@@ -265,6 +265,8 @@ class HloModule {
   const HloSchedule& schedule() const { return *schedule_; }
   HloSchedule& schedule() { return *schedule_; }
 
+  Status CheckUniqueNamesAndIdsForComputationsAndInstructions() const;
+
  private:
   HloComputation* AddComputationInternal(
       std::unique_ptr<HloComputation> computation, bool is_entry,
-- 
GitLab


From 121dd0762284314d739296de70b1ea1979cd5949 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Wed, 6 Feb 2019 23:48:29 -0800
Subject: [PATCH 2054/2345] Dispatch while loops based exclusively on the test
 closure, not the modified state. Upgrade the tests from v1-only mode.

PiperOrigin-RevId: 232817358
---
 .../autograph/converters/control_flow.py      |  1 -
 .../autograph/converters/control_flow_test.py | 41 +++++++++++++++----
 .../autograph/operators/control_flow.py       |  3 +-
 .../autograph/operators/control_flow_test.py  | 37 +++++++++++------
 4 files changed, 59 insertions(+), 23 deletions(-)

diff --git a/tensorflow/python/autograph/converters/control_flow.py b/tensorflow/python/autograph/converters/control_flow.py
index 2526d30257..aa7bf4666c 100644
--- a/tensorflow/python/autograph/converters/control_flow.py
+++ b/tensorflow/python/autograph/converters/control_flow.py
@@ -329,7 +329,6 @@ class ControlFlowTransformer(converter.Base):
     cond_closure = set()
     for s in cond_scope.read:
       cond_closure |= s.support_set
-    cond_closure -= loop_state
 
     loop_state, state_ssf, state_ast_tuple, ssf_map = self._state_constructs(
         loop_state, reserved_symbols)
diff --git a/tensorflow/python/autograph/converters/control_flow_test.py b/tensorflow/python/autograph/converters/control_flow_test.py
index d3accd3c1e..37ea4c2ae8 100644
--- a/tensorflow/python/autograph/converters/control_flow_test.py
+++ b/tensorflow/python/autograph/converters/control_flow_test.py
@@ -33,8 +33,7 @@ class ControlFlowTest(converter_testing.TestCase):
       inputs = (inputs,)
     with self.converted(test_fn, control_flow, {},
                         constant_op.constant) as result:
-      with self.cached_session() as sess:
-        self.assertEqual(sess.run(result.test_fn(*inputs)), expected)
+      self.assertEqual(self.evaluate(result.test_fn(*inputs)), expected)
 
   @test_util.run_deprecated_v1
   def test_while_basic(self):
@@ -78,6 +77,33 @@ class ControlFlowTest(converter_testing.TestCase):
 
     self.assertTransformedResult(test_fn, constant_op.constant(5), 0)
 
+  @test_util.run_deprecated_v1
+  def test_while_dispatches_by_cond_only(self):
+
+    class TensorIncompatibleNumeric(object):
+      """Works in arithmetic expression, but errors out with TF ops."""
+
+      def __init__(self, val):
+        self.val = val
+
+      def __add__(self, other):
+        return TensorIncompatibleNumeric(self.val + other)
+
+    def test_fn(n, s):
+      while n > 0:
+        n -= 1
+        s += n
+      return s
+
+    self.assertTransformedResult(test_fn, (constant_op.constant(5), 0), 10)
+    with self.converted(test_fn, control_flow, {}) as result:
+      # n alone controls the staging. When the loop is not staged, Python
+      # knows how to add the two objects. But when staged, tf.while_loop will
+      # not know how to deal with the TensorIncompatibleNumeric object.
+      self.assertEqual(result.test_fn(5, TensorIncompatibleNumeric(0)).val, 10)
+      with self.assertRaises(TypeError):
+        result.test_fn(constant_op.constant(5), TensorIncompatibleNumeric(0))
+
   @test_util.run_deprecated_v1
   def test_if_basic(self):
 
@@ -112,11 +138,10 @@ class ControlFlowTest(converter_testing.TestCase):
       return obj
 
     with self.converted(test_fn, control_flow, {}) as result:
-      with self.cached_session() as sess:
-        res_obj = result.test_fn(constant_op.constant(1), TestClass(0, 0))
-        self.assertEqual(sess.run((res_obj.a, res_obj.b)), (-1, 0))
-        res_obj = result.test_fn(constant_op.constant(-1), TestClass(0, 0))
-        self.assertEqual(sess.run((res_obj.a, res_obj.b)), (0, -2))
+      res_obj = result.test_fn(constant_op.constant(1), TestClass(0, 0))
+      self.assertEqual(self.evaluate((res_obj.a, res_obj.b)), (-1, 0))
+      res_obj = result.test_fn(constant_op.constant(-1), TestClass(0, 0))
+      self.assertEqual(self.evaluate((res_obj.a, res_obj.b)), (0, -2))
 
   @test_util.run_deprecated_v1
   def test_if_single_output(self):
@@ -223,5 +248,7 @@ class ControlFlowTest(converter_testing.TestCase):
       return z
 
     self.assertTransformedResult(test_fn, [3, 3], 7)
+
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/autograph/operators/control_flow.py b/tensorflow/python/autograph/operators/control_flow.py
index f046000720..707064cefa 100644
--- a/tensorflow/python/autograph/operators/control_flow.py
+++ b/tensorflow/python/autograph/operators/control_flow.py
@@ -153,8 +153,7 @@ def while_stmt(test, body, init_state, extra_deps, opts=None):
   # TODO(mdan): Consider adding a generic mechanism for dynamic dispatch.
   # That could be something as simple as a collection of dispatch rules, with
   # some prioritization.
-  if any(tensor_util.is_tensor(v)
-         for v in nest.flatten(init_state + extra_deps)):
+  if any(tensor_util.is_tensor(v) for v in nest.flatten(extra_deps)):
     return _tf_while_stmt(test, body, init_state, opts)
   else:
     return _py_while_stmt(test, body, init_state, opts)
diff --git a/tensorflow/python/autograph/operators/control_flow_test.py b/tensorflow/python/autograph/operators/control_flow_test.py
index 590b51820f..316748f534 100644
--- a/tensorflow/python/autograph/operators/control_flow_test.py
+++ b/tensorflow/python/autograph/operators/control_flow_test.py
@@ -65,30 +65,41 @@ class WhileLoopTest(test.TestCase):
   def test_tensor(self):
     n = constant_op.constant(5)
     results = control_flow.while_stmt(
-        test=lambda i, sum: i < n,
-        body=lambda i, sum: (i + 1, sum + i,),
+        test=lambda i, s: i < n,
+        body=lambda i, s: (i + 1, s + i,),
         init_state=(0, 0),
         extra_deps=(n,))
-    with self.cached_session():
-      self.assertEqual((5, 10), self.evaluate(results))
+    self.assertEqual((5, 10), self.evaluate(results))
 
   @test_util.run_deprecated_v1
-  def test_tensor_dict_state(self):
+  def test_python_with_tensor_state(self):
     n = 5
-    init_state = {'i': constant_op.constant(0), 'sum': constant_op.constant(0)}
     results = control_flow.while_stmt(
-        test=lambda s: s['i'] < n,
-        body=lambda s: ({'i': s['i'] + 1, 'sum': s['sum'] + s['i']},),
-        init_state=(init_state,),
+        test=lambda i, s: i < n,
+        body=lambda i, s: (i + 1, s + i),
+        init_state=(0, constant_op.constant(0)),
         extra_deps=())
-    with self.cached_session():
-      self.assertEqual(({'i': 5, 'sum': 10},), self.evaluate(results))
+    result_i, result_s = results
+    self.assertEqual(5, result_i)
+    self.assertEqual(10, self.evaluate(result_s))
+
+  @test_util.run_deprecated_v1
+  def test_python_due_to_hidden_cond_type(self):
+    n = 5
+
+    # TODO(b/124002646): Improve the error message.
+    with self.assertRaises(Exception):
+      control_flow.while_stmt(
+          test=lambda i, s: i < n,
+          body=lambda i, s: (i + 1, s + i),
+          init_state=(constant_op.constant(0), constant_op.constant(0)),
+          extra_deps=())
 
   def test_python(self):
     n = 5
     results = control_flow.while_stmt(
-        test=lambda i, sum: i < n,
-        body=lambda i, sum: (i + 1, sum + i),
+        test=lambda i, s: i < n,
+        body=lambda i, s: (i + 1, s + i),
         init_state=(0, 0),
         extra_deps=(n,))
     self.assertEqual((5, 10), results)
-- 
GitLab


From 0e8de82ceba176f2e15ed2b316382ab0b30ea41d Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Thu, 7 Feb 2019 00:21:51 -0800
Subject: [PATCH 2055/2345] Use the dtype of the inputs to convert into
 placeholders. Otherwise we can get tf.float32 from tf.string or tf.int32.

PiperOrigin-RevId: 232821743
---
 tensorflow/python/keras/engine/training_utils.py      | 6 +++++-
 tensorflow/python/keras/engine/training_utils_test.py | 6 +++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index 22f976a705..25c2f1f774 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -31,6 +31,7 @@ from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.data.ops import readers
 from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
@@ -1416,7 +1417,10 @@ class ModelInputs(object):
         # we have. The user should call `model._set_inputs(placeholders)`
         # to specify custom placeholders if the need arises.
         shape = (None,) + tuple(v.shape[1:])
-        v = K.placeholder(shape=shape, name=k)
+        dtype = dtypes.as_dtype(v.dtype)
+        if dtype.is_floating:
+          dtype = K.floatx()
+        v = K.placeholder(shape=shape, name=k, dtype=dtype)
       elif isinstance(v, tensor_shape.TensorShape):
         shape = (None,) + tuple(v.as_list()[1:])
         v = K.placeholder(shape=shape, name=k)
diff --git a/tensorflow/python/keras/engine/training_utils_test.py b/tensorflow/python/keras/engine/training_utils_test.py
index e472dc3f2c..30e3d1f3e4 100644
--- a/tensorflow/python/keras/engine/training_utils_test.py
+++ b/tensorflow/python/keras/engine/training_utils_test.py
@@ -25,7 +25,9 @@ import numpy as np
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import readers
 from tensorflow.python.eager import context
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import tensor_util
+from tensorflow.python.keras import backend
 from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.keras.engine import training_utils
@@ -45,10 +47,11 @@ class ModelInputsTest(test.TestCase):
     vals = model_inputs.get_symbolic_inputs(return_single_as_list=True)
     self.assertEqual(1, len(vals))
     self.assertTrue(tensor_util.is_tensor(vals[0]))
+    self.assertEqual(backend.floatx(), vals[0].dtype)
 
   def test_single_thing_eager(self):
     with context.eager_mode():
-      a = np.ones(10)
+      a = np.ones(10, dtype=np.int32)
       model_inputs = training_utils.ModelInputs(a)
       self.assertEqual(['input_1'], model_inputs.get_input_names())
       val = model_inputs.get_symbolic_inputs()
@@ -56,6 +59,7 @@ class ModelInputsTest(test.TestCase):
       vals = model_inputs.get_symbolic_inputs(return_single_as_list=True)
       self.assertEqual(1, len(vals))
       self.assertTrue(tf_utils.is_symbolic_tensor(vals[0]))
+      self.assertEqual(dtypes.int32, vals[0].dtype)
 
   def test_list(self):
     a = [np.ones(10), np.ones(20)]
-- 
GitLab


From 3a4b3241eec5fe75c55bb4844966700ba7cb4906 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Thu, 7 Feb 2019 00:40:13 -0800
Subject: [PATCH 2056/2345] Use Scatter to implement InvertPermutationOp for
 XLA.

This avoids that we need the CompileTimeConstantInput constraint.
Add a test with non-constant input.

PiperOrigin-RevId: 232824152
---
 tensorflow/compiler/tests/unary_ops_test.py   |  6 +++
 .../compiler/tf2xla/kernels/transpose_op.cc   | 54 ++++++++++++-------
 2 files changed, 42 insertions(+), 18 deletions(-)

diff --git a/tensorflow/compiler/tests/unary_ops_test.py b/tensorflow/compiler/tests/unary_ops_test.py
index 978ed667ee..831e203f49 100644
--- a/tensorflow/compiler/tests/unary_ops_test.py
+++ b/tensorflow/compiler/tests/unary_ops_test.py
@@ -820,6 +820,12 @@ class UnaryOpsTest(xla_test.XLATestCase):
         np.array([1, 2, 0], np.int32),
         expected=np.array([2, 0, 1], dtype=np.int32))
 
+  def testInvertPermutationTwiceIsNoop(self):
+    self._assertOpOutputMatchesExpected(
+        lambda x: array_ops.invert_permutation(array_ops.invert_permutation(x)),
+        np.array([1, 2, 0], np.int32),
+        expected=np.array([1, 2, 0], dtype=np.int32))
+
   def testRank(self):
     rank_op = lambda x: array_ops.rank_internal(x, optimize=False)
     for dtype in self.numeric_types:
diff --git a/tensorflow/compiler/tf2xla/kernels/transpose_op.cc b/tensorflow/compiler/tf2xla/kernels/transpose_op.cc
index 76793d677b..4ac7143062 100644
--- a/tensorflow/compiler/tf2xla/kernels/transpose_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/transpose_op.cc
@@ -19,6 +19,7 @@ limitations under the License.
 // helper.
 
 #include "tensorflow/core/kernels/transpose_op.h"
+#include "tensorflow/compiler/tf2xla/lib/scatter.h"
 #include "tensorflow/compiler/tf2xla/type_util.h"
 #include "tensorflow/compiler/tf2xla/xla_helpers.h"
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
@@ -128,29 +129,46 @@ class InvertPermutationOp : public XlaOpKernel {
                 errors::InvalidArgument("permutation of nonnegative int32s "
                                         "must have <= int32 max elements"));
 
-    std::vector<int64> perm;
-    OP_REQUIRES_OK(ctx, ctx->ConstantInputAsIntVector(0, &perm));
-
-    int size = perm.size();
+    auto e = ctx->InputExpression(0);
+    auto tensor_or_status = e.ResolveConstant(ctx->compiler()->client());
+    OP_REQUIRES_OK(ctx, tensor_or_status.status());
+    // If the input is a constant, we also want the output to be a constant.
+    // Some models rely on the result of InvertPermutation being a constant.
+    // TODO(b/32495713): Remove this when we can check whether Scatter is
+    // constant. Right now, we always assume it is non-constant because we don't
+    // check the embedded computation.
+    if (tensor_or_status.ValueOrDie().has_value()) {
+      std::vector<int64> perm;
+      OP_REQUIRES_OK(ctx, ctx->ConstantInputAsIntVector(0, &perm));
+
+      int size = perm.size();
+
+      std::vector<int32> output(size);
+      std::fill_n(output.data(), size, -1);
+      for (int i = 0; i < size; ++i) {
+        const int64 d = perm[i];
+        OP_REQUIRES(ctx, FastBoundsCheck(d, size),
+                    errors::InvalidArgument(d, " is not between 0 and ", size));
+        OP_REQUIRES(ctx, output[d] == -1,
+                    errors::InvalidArgument(d, " is duplicated in the input."));
+        output[d] = i;
+      }
 
-    std::vector<int32> output(size);
-    std::fill_n(output.data(), size, -1);
-    for (int i = 0; i < size; ++i) {
-      const int64 d = perm[i];
-      OP_REQUIRES(ctx, FastBoundsCheck(d, size),
-                  errors::InvalidArgument(d, " is not between 0 and ", size));
-      OP_REQUIRES(ctx, output[d] == -1,
-                  errors::InvalidArgument(d, " is duplicated in the input."));
-      output[d] = i;
+      ctx->SetOutput(0, xla::ConstantR1<int32>(ctx->builder(), output));
+    } else {
+      auto indices = ctx->Input(0);
+      int size = ctx->InputShape(0).num_elements();
+      auto iota = xla::Iota(ctx->builder(), xla::S32, size);
+      auto result = XlaScatter(iota, iota, indices,
+                               /*indices_are_vectors=*/false, /*combiner=*/{},
+                               ctx->builder());
+      OP_REQUIRES_OK(ctx, result.status());
+      ctx->SetOutput(0, result.ValueOrDie());
     }
-
-    ctx->SetOutput(0, xla::ConstantR1<int32>(ctx->builder(), output));
   }
 };
 
-REGISTER_XLA_OP(Name("InvertPermutation")
-                    .TypeConstraint("T", DT_INT32)
-                    .CompileTimeConstantInput("x"),
+REGISTER_XLA_OP(Name("InvertPermutation").TypeConstraint("T", DT_INT32),
                 InvertPermutationOp);
 
 }  // namespace
-- 
GitLab


From 42d8403ace8f494dfd1207a6a1f641f1ad6706ea Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 7 Feb 2019 01:02:28 -0800
Subject: [PATCH 2057/2345] compat: Update forward compatibility horizon to
 2019-02-07

PiperOrigin-RevId: 232826432
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 3ba54688c7..f49e7602ce 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 2, 6)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 2, 7)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From dde1ed7309bcc4e9f1176a158dc63f0ce8263b6b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 7 Feb 2019 02:35:14 -0800
Subject: [PATCH 2058/2345] [TF:XLA] Add error message about the non-strict XLA
 jit.

PiperOrigin-RevId: 232836591
---
 tensorflow/compiler/jit/BUILD                 |  2 --
 .../jit/encapsulate_xla_computations_pass.cc  | 34 +++++++++++++++++--
 tensorflow/compiler/jit/kernels/BUILD         |  1 +
 tensorflow/compiler/jit/kernels/xla_ops.cc    | 20 ++++++++---
 tensorflow/compiler/xla/BUILD                 |  4 ---
 tensorflow/compiler/xla/status_macros.cc      |  7 ++++
 tensorflow/compiler/xla/status_macros.h       |  4 +++
 7 files changed, 60 insertions(+), 12 deletions(-)

diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index 3cae081ce7..4e3229bc70 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -282,7 +282,6 @@ cc_library(
         "//tensorflow/compiler/tf2xla:common",
         "//tensorflow/compiler/tf2xla:dump_graph",
         "//tensorflow/compiler/tf2xla:xla_compiler",
-        "//tensorflow/compiler/xla:debug_options_flags",
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla/client:client_library",
         "//tensorflow/compiler/xla/client:local_client",
@@ -465,7 +464,6 @@ cc_library(
         "//tensorflow/compiler/tf2xla:tf2xla_util",
         "//tensorflow/core:framework",
         "//tensorflow/core:graph",
-        "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/types:optional",
diff --git a/tensorflow/compiler/jit/encapsulate_xla_computations_pass.cc b/tensorflow/compiler/jit/encapsulate_xla_computations_pass.cc
index c9ae717b8d..411ca75cd8 100644
--- a/tensorflow/compiler/jit/encapsulate_xla_computations_pass.cc
+++ b/tensorflow/compiler/jit/encapsulate_xla_computations_pass.cc
@@ -15,13 +15,17 @@ limitations under the License.
 
 #include "tensorflow/compiler/jit/encapsulate_xla_computations_pass.h"
 
+#include "absl/algorithm/container.h"
 #include "absl/container/flat_hash_set.h"
 #include "absl/memory/memory.h"
+#include "absl/strings/ascii.h"
 #include "absl/strings/str_cat.h"
 #include "tensorflow/compiler/jit/encapsulate_subgraphs_pass.h"
 #include "tensorflow/compiler/tf2xla/dump_graph.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/core/framework/node_def.pb.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/lib/core/stringpiece.h"
 #include "tensorflow/core/lib/hash/hash.h"
 #include "tensorflow/core/lib/strings/proto_serialization.h"
 #include "tensorflow/core/lib/strings/str_util.h"
@@ -36,6 +40,25 @@ namespace {
 
 const char* const kXlaClusterOutput = "XlaClusterOutput";
 
+bool IsCpuGpuCompile(const Graph* graph) {
+  for (Node* n : graph->nodes()) {
+    string name;
+    // Only consider nodes being compiled.
+    if (!GetNodeAttr(n->attrs(),
+                     EncapsulateXlaComputationsPass::kXlaClusterAttr, &name)
+             .ok())
+      continue;
+    // Early return for any node with a device that is not a CPU or GPU.
+    DeviceNameUtils::ParsedName parsed;
+    if (DeviceNameUtils::ParseFullName(n->def().device(), &parsed)) {
+      if (parsed.type != DEVICE_CPU && parsed.type != DEVICE_GPU) {
+        return false;
+      }
+    }
+  }
+  return true;
+}
+
 // Checks if a graph node is marked to be a guaranteed constant.
 bool is_guaranteed_constant(const Node& n) {
   bool guaranteed_constant = false;
@@ -352,12 +375,19 @@ Status EncapsulateXlaComputationsPass::Run(
           << dump_graph::DumpGraphToFile("encapsulate_xla_computations_before",
                                          **options.graph, options.flib_def);
 
-  TF_RETURN_IF_ERROR(Encapsulate(options.graph, options.flib_def));
+  const char* additional_help =
+      IsCpuGpuCompile(options.graph->get())
+          ? xla::status_macros::kPossibleAutoJitAlternative
+          : "";
+
+  TF_RETURN_WITH_CONTEXT_IF_ERROR(Encapsulate(options.graph, options.flib_def),
+                                  additional_help);
   VLOG(1) << "EncapsulateXlaComputations() half-way: "
           << dump_graph::DumpGraphToFile("encapsulate_xla_computations_halfway",
                                          **options.graph, options.flib_def);
 
-  TF_RETURN_IF_ERROR(BuildXlaLaunchOps(options.graph->get()));
+  TF_RETURN_WITH_CONTEXT_IF_ERROR(BuildXlaLaunchOps(options.graph->get()),
+                                  additional_help);
   VLOG(1) << "EncapsulateXlaComputations() finished: "
           << dump_graph::DumpGraphToFile("encapsulate_xla_computations_after",
                                          **options.graph, options.flib_def);
diff --git a/tensorflow/compiler/jit/kernels/BUILD b/tensorflow/compiler/jit/kernels/BUILD
index bab824c15f..d0fa2c40be 100644
--- a/tensorflow/compiler/jit/kernels/BUILD
+++ b/tensorflow/compiler/jit/kernels/BUILD
@@ -19,6 +19,7 @@ cc_library(
         "//tensorflow/compiler/tf2xla:common",
         "//tensorflow/compiler/tf2xla:tf2xla_util",
         "//tensorflow/compiler/tf2xla:xla_compiler",
+        "//tensorflow/compiler/xla:status_macros",
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla/client:client_library",
         "//tensorflow/compiler/xla/client:local_client",
diff --git a/tensorflow/compiler/jit/kernels/xla_ops.cc b/tensorflow/compiler/jit/kernels/xla_ops.cc
index ad71df5a69..997ef6e14b 100644
--- a/tensorflow/compiler/jit/kernels/xla_ops.cc
+++ b/tensorflow/compiler/jit/kernels/xla_ops.cc
@@ -25,6 +25,7 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/client_library.h"
 #include "tensorflow/compiler/xla/client/local_client.h"
+#include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/core/common_runtime/dma_helper.h"
 #include "tensorflow/core/common_runtime/function.h"
@@ -35,6 +36,8 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/kernels/variable_ops.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/stream_executor_no_cuda.h"
 #include "tensorflow/core/util/stream_executor_util.h"
@@ -304,10 +307,19 @@ void XlaLocalLaunchBase::Compute(OpKernelContext* ctx) {
   xla::LocalExecutable* executable;
   std::map<int, OptionalTensor> variables;
 
-  OP_REQUIRES_OK(
-      ctx, CompileToLocalExecutable(ctx, function_, platform_info_, resources_,
-                                    constants_, /*lazy=*/false, &client,
-                                    &variables, &kernel, &executable));
+  {
+    Status s = CompileToLocalExecutable(
+        ctx, function_, platform_info_, resources_, constants_, /*lazy=*/false,
+        &client, &variables, &kernel, &executable);
+    if (!s.ok() && (platform_info_.device_type().type_string() == DEVICE_CPU ||
+                    platform_info_.device_type().type_string() == DEVICE_GPU)) {
+      // Suggest auto jit if the failure was with GPU or CPU.
+      errors::AppendToMessage(&s,
+                              xla::status_macros::kPossibleAutoJitAlternative);
+    }
+
+    OP_REQUIRES_OK(ctx, s);
+  }
 
   se::Stream* stream =
       ctx->op_device_context() ? ctx->op_device_context()->stream() : nullptr;
diff --git a/tensorflow/compiler/xla/BUILD b/tensorflow/compiler/xla/BUILD
index 636e5ef721..5b8a4c51fe 100644
--- a/tensorflow/compiler/xla/BUILD
+++ b/tensorflow/compiler/xla/BUILD
@@ -150,8 +150,6 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":status",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:lib_internal",
         "//tensorflow/stream_executor/lib",
     ],
 )
@@ -194,7 +192,6 @@ cc_library(
         ":types",
         ":util",
         "//tensorflow/core:lib",
-        "@com_google_absl//absl/synchronization",
     ],
 )
 
@@ -833,7 +830,6 @@ cc_library(
             "//tensorflow/compiler/xla:xla_proto",
             "//tensorflow/compiler/xla/service:hlo",
             "//tensorflow/core:framework_internal",
-            "//tensorflow/core:lib",
             "@com_google_absl//absl/strings",
         ],
 )
diff --git a/tensorflow/compiler/xla/status_macros.cc b/tensorflow/compiler/xla/status_macros.cc
index b88fe367d7..aa7238f07d 100644
--- a/tensorflow/compiler/xla/status_macros.cc
+++ b/tensorflow/compiler/xla/status_macros.cc
@@ -25,6 +25,13 @@ limitations under the License.
 namespace xla {
 namespace status_macros {
 
+ABSL_CONST_INIT const char kPossibleAutoJitAlternative[] =
+    "This error might be occurring with the use of xla.compile. If it is not "
+    "necessary that every Op be compiled with XLA, an alternative is to use "
+    "auto_jit with OptimizerOptions.global_jit_level = ON_2 or the environment "
+    "variable TF_XLA_FLAGS=\"tf_xla_auto_jit=2\" which will attempt to use xla "
+    "to compile as much of the graph as the compiler is able to.";
+
 static Status MakeStatus(tensorflow::error::Code code, const string& message) {
   return Status(code, message);
 }
diff --git a/tensorflow/compiler/xla/status_macros.h b/tensorflow/compiler/xla/status_macros.h
index e51dd64e2a..315136acc7 100644
--- a/tensorflow/compiler/xla/status_macros.h
+++ b/tensorflow/compiler/xla/status_macros.h
@@ -30,6 +30,10 @@ limitations under the License.
 namespace xla {
 namespace status_macros {
 
+// This is a useful error message when encountering XLA Compiler errors that
+// could be handled with the non-strict AutoJit mode.
+extern const char kPossibleAutoJitAlternative[];
+
 // Stream object used to collect error messages in MAKE_ERROR macros
 // or append error messages with APPEND_ERROR.  It accepts any
 // arguments with operator<< to build an error string, and then has an
-- 
GitLab


From 101143e8d6cf124efc357858b5f60373aed834b9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 7 Feb 2019 03:00:53 -0800
Subject: [PATCH 2059/2345] Fix preformance bug in MapAndBatchDatasetOp

Previously we were creating a copy of the result tesor to pass it into
CopyElementSlice what prevented the fast path what doesn't copy the actual
underlying data to trigger. Adding an extra std::move should fix this issue.

PiperOrigin-RevId: 232839237
---
 .../kernels/data/experimental/map_and_batch_dataset_op.cc   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc b/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
index 0ef56915eb..77c1749632 100644
--- a/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
@@ -450,7 +450,7 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
               result->UpdateStatus(allocate_status, offset);
             } else {
               for (size_t i = 0; i < return_values->size(); ++i) {
-                const Tensor& tensor = return_values->at(i);
+                Tensor& tensor = return_values->at(i);
                 Tensor* batch = &(result->output)[i];
                 if (tensor.NumElements() !=
                     (batch->NumElements() / batch->dim_size(0))) {
@@ -468,8 +468,8 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
                 // TODO(mrry): Add a version of DoParallelConcat that allows us
                 // to move `tensor` where possible, to speed up string tensor
                 // batching.
-                Status copy_status =
-                    batch_util::CopyElementToSlice(tensor, batch, offset);
+                Status copy_status = batch_util::CopyElementToSlice(
+                    std::move(tensor), batch, offset);
                 if (!copy_status.ok()) {
                   result->UpdateStatus(copy_status, offset);
                   break;
-- 
GitLab


From e199bac06979bdfc621758a4306a95ac24b8c467 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Thu, 7 Feb 2019 05:23:15 -0800
Subject: [PATCH 2060/2345] Reuse the default comparators computation for Sort
 in MakeSortHlo.

This has the advantage that we handle NaN values in the same way.

PiperOrigin-RevId: 232853920
---
 tensorflow/compiler/xla/service/BUILD         |  4 ++
 .../xla/service/hlo_creation_utils.cc         | 41 ++++++++-----------
 .../compiler/xla/service/hlo_creation_utils.h |  3 --
 3 files changed, 21 insertions(+), 27 deletions(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index a0cdf342b2..f5fbe642d0 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -1461,11 +1461,15 @@ cc_library(
     hdrs = ["hlo_creation_utils.h"],
     deps = [
         ":hlo",
+        ":hlo_module_config",
         ":shape_inference",
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:literal_util",
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:util",
+        "//tensorflow/compiler/xla/client:xla_builder",
+        "//tensorflow/compiler/xla/client:xla_computation",
+        "//tensorflow/compiler/xla/client/lib:comparators",
         "@com_google_absl//absl/algorithm:container",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
diff --git a/tensorflow/compiler/xla/service/hlo_creation_utils.cc b/tensorflow/compiler/xla/service/hlo_creation_utils.cc
index 5d3e11f7c4..070115604b 100644
--- a/tensorflow/compiler/xla/service/hlo_creation_utils.cc
+++ b/tensorflow/compiler/xla/service/hlo_creation_utils.cc
@@ -17,10 +17,15 @@ limitations under the License.
 #include "absl/algorithm/container.h"
 #include "absl/memory/memory.h"
 #include "absl/strings/str_cat.h"
+#include "tensorflow/compiler/xla/client/lib/comparators.h"
+#include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/compiler/xla/client/xla_computation.h"
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/compiler/xla/service/hlo_clone_context.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/service/hlo_module_config.h"
 #include "tensorflow/compiler/xla/service/shape_inference.h"
 #include "tensorflow/compiler/xla/util.h"
 
@@ -274,31 +279,19 @@ StatusOr<HloInstruction*> MakeSortHlo(
     HloModule* module) {
   CHECK(!operands.empty()) << "Sort Hlo requires at least one operand.";
   HloComputation* compare_computation;
-  {
-    auto b = HloComputation::Builder("Sort.Compare");
-    Shape key_scalar_shape =
-        ShapeUtil::MakeShape(operands[0]->shape().element_type(), {});
-    auto lhs = b.AddInstruction(
-        HloInstruction::CreateParameter(0, key_scalar_shape, "p.0.lhs"));
-    auto rhs = b.AddInstruction(
-        HloInstruction::CreateParameter(1, key_scalar_shape, "p.0.rhs"));
-    int parameter_count = 2;
-    for (const auto* operand : operands.subspan(1)) {
-      Shape scalar_shape =
-          ShapeUtil::MakeShape(operand->shape().element_type(), {});
-      b.AddInstruction(HloInstruction::CreateParameter(
-          parameter_count, scalar_shape,
-          StrCat("p.", parameter_count / 2, ".lhs")));
-      ++parameter_count;
-      b.AddInstruction(HloInstruction::CreateParameter(
-          parameter_count, scalar_shape,
-          StrCat("p.", parameter_count / 2, ".rhs")));
-      ++parameter_count;
-    }
-    b.AddInstruction(HloInstruction::CreateBinary(
-        ShapeUtil::MakeShape(PRED, {}), HloOpcode::kLt, lhs, rhs));
-    compare_computation = module->AddEmbeddedComputation(b.Build());
+  XlaBuilder b("Sort.Compare");
+  std::vector<PrimitiveType> operand_types(operands.size());
+  for (int64 i = 0; i < operands.size(); ++i) {
+    operand_types[i] = operands[i]->shape().element_type();
   }
+  XlaComputation comparator = CreateScalarLtComputation(operand_types, &b);
+  TF_ASSIGN_OR_RETURN(ProgramShape program_shape, comparator.GetProgramShape());
+  HloModuleConfig config(program_shape);
+  TF_ASSIGN_OR_RETURN(auto new_module,
+                      HloModule::CreateFromProto(comparator.proto(), config));
+  HloCloneContext context(module);
+  compare_computation =
+      module->DeepCloneComputation(new_module->entry_computation(), &context);
   return builder->AddInstruction(HloInstruction::CreateSort(
       sort_shape, dimension_to_sort, operands, compare_computation));
 }
diff --git a/tensorflow/compiler/xla/service/hlo_creation_utils.h b/tensorflow/compiler/xla/service/hlo_creation_utils.h
index 80f58f68d3..36b8cdc7fe 100644
--- a/tensorflow/compiler/xla/service/hlo_creation_utils.h
+++ b/tensorflow/compiler/xla/service/hlo_creation_utils.h
@@ -127,9 +127,6 @@ StatusOr<HloInstruction*> MakeSelectHlo(HloInstruction* pred,
 // operands. All operands must be in the same computation. Also creates a
 // default compare sub-computation which sorts the first operand into ascending
 // order.
-// Note that this default compare sub-computation does not have special handling
-// for floating point values and thus can result in undefined behavior in the
-// presence of NaN values.
 StatusOr<HloInstruction*> MakeSortHlo(
     const Shape& sort_shape, absl::Span<HloInstruction* const> operands,
     int64 dimension_to_sort, HloComputation::Builder* builder,
-- 
GitLab


From 4a02ee7d49a7463c0abf7ef8453b6a497f09ca01 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Thu, 7 Feb 2019 06:56:32 -0800
Subject: [PATCH 2061/2345] Remove debug code that spams the output.

PiperOrigin-RevId: 232866668
---
 tensorflow/python/autograph/converters/return_statements.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/python/autograph/converters/return_statements.py b/tensorflow/python/autograph/converters/return_statements.py
index 723acab908..3173e676e5 100644
--- a/tensorflow/python/autograph/converters/return_statements.py
+++ b/tensorflow/python/autograph/converters/return_statements.py
@@ -408,6 +408,5 @@ def transform(node, ctx, default_to_null_return=True):
   transformer = ReturnStatementsTransformer(
       ctx, default_to_null_return=default_to_null_return)
   node = transformer.visit(node)
-  transformer.debug_print_src(node)
 
   return node
-- 
GitLab


From 06344b1e0e550ce43532bdfa5190f564e832be99 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 7 Feb 2019 07:04:00 -0800
Subject: [PATCH 2062/2345] Add copy constructor and assignment to XlaOp.

PiperOrigin-RevId: 232868062
---
 tensorflow/compiler/xla/client/xla_builder.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/compiler/xla/client/xla_builder.h b/tensorflow/compiler/xla/client/xla_builder.h
index ea4dc18c2d..cd18a9fab0 100644
--- a/tensorflow/compiler/xla/client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_builder.h
@@ -56,6 +56,9 @@ class XlaOp {
   }
   ~XlaOp() = default;
 
+  XlaOp(const XlaOp& other) = default;
+  XlaOp& operator=(const XlaOp& other) = default;
+
   // Precondition: !IsUninitialized().
   //
   // It's very common to do foo.builder()->bar().  Without this precondition, if
-- 
GitLab


From f321e3c900dfda328edb946ff95c4eaba4cf4928 Mon Sep 17 00:00:00 2001
From: Noah Eisen <ncteisen@google.com>
Date: Thu, 7 Feb 2019 07:25:17 -0800
Subject: [PATCH 2063/2345] Fix small typo

PiperOrigin-RevId: 232870506
---
 tensorflow/core/distributed_runtime/message_wrappers.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/distributed_runtime/message_wrappers.h b/tensorflow/core/distributed_runtime/message_wrappers.h
index 474ac0e186..92a9994ac3 100644
--- a/tensorflow/core/distributed_runtime/message_wrappers.h
+++ b/tensorflow/core/distributed_runtime/message_wrappers.h
@@ -234,7 +234,7 @@ class ProtoRunStepRequest : public RunStepRequestWrapper {
 //
 ////////////////////////////////////////////////////////////////////////////////
 
-// Abstract interface for an immutable RunStepRequest message.
+// Abstract interface for an immutable RunGraphRequest message.
 //
 // This interface is typically used by server-side components in the
 // TensorFlow worker.
-- 
GitLab


From 382a627639e2d7a48b37437cea08d14e02d960d1 Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Thu, 7 Feb 2019 08:15:23 -0800
Subject: [PATCH 2064/2345] Add int8 support for comparison operators.

PiperOrigin-RevId: 232877482
---
 tensorflow/lite/kernels/comparisons.cc       |  65 +++++++---
 tensorflow/lite/kernels/comparisons_test.cc  | 130 ++++++++++++++++++-
 tensorflow/lite/kernels/register.cc          |  24 +++-
 tensorflow/lite/toco/tflite/operator.cc      | 104 +++++++++++++--
 tensorflow/lite/toco/tflite/operator_test.cc |  44 +++++++
 5 files changed, 323 insertions(+), 44 deletions(-)

diff --git a/tensorflow/lite/kernels/comparisons.cc b/tensorflow/lite/kernels/comparisons.cc
index a914449ae5..d492419255 100644
--- a/tensorflow/lite/kernels/comparisons.cc
+++ b/tensorflow/lite/kernels/comparisons.cc
@@ -59,11 +59,12 @@ TfLiteStatus ComparisonPrepare(TfLiteContext* context, TfLiteNode* node) {
 
 // TODO(ruic): optimize macros below to using template functions.
 #define TF_LITE_QUANTIZE_COMPARISON(opname)                                    \
+  template <typename input_dtype>                                              \
   void EvalQuantized##opname(TfLiteContext* context, TfLiteNode* node,         \
                              const TfLiteTensor* input1,                       \
                              const TfLiteTensor* input2, TfLiteTensor* output, \
                              bool requires_broadcast) {                        \
-    if (input1->type == kTfLiteUInt8) {                                        \
+    if (input1->type == kTfLiteUInt8 || input1->type == kTfLiteInt8) {         \
       auto input1_offset = -input1->params.zero_point;                         \
       auto input2_offset = -input2->params.zero_point;                         \
       const int left_shift = 8;                                                \
@@ -87,14 +88,16 @@ TfLiteStatus ComparisonPrepare(TfLiteContext* context, TfLiteNode* node) {
       op_params.input2_shift = input2_shift;                                   \
       if (requires_broadcast) {                                                \
         reference_ops::Broadcast4DSlow##opname##WithScaling(                   \
-            op_params, GetTensorShape(input1), GetTensorData<uint8_t>(input1), \
-            GetTensorShape(input2), GetTensorData<uint8_t>(input2),            \
-            GetTensorShape(output), GetTensorData<bool>(output));              \
+            op_params, GetTensorShape(input1),                                 \
+            GetTensorData<input_dtype>(input1), GetTensorShape(input2),        \
+            GetTensorData<input_dtype>(input2), GetTensorShape(output),        \
+            GetTensorData<bool>(output));                                      \
       } else {                                                                 \
         reference_ops::opname##WithScaling(                                    \
-            op_params, GetTensorShape(input1), GetTensorData<uint8_t>(input1), \
-            GetTensorShape(input2), GetTensorData<uint8_t>(input2),            \
-            GetTensorShape(output), GetTensorData<bool>(output));              \
+            op_params, GetTensorShape(input1),                                 \
+            GetTensorData<input_dtype>(input1), GetTensorShape(input2),        \
+            GetTensorData<input_dtype>(input2), GetTensorShape(output),        \
+            GetTensorData<bool>(output));                                      \
       }                                                                        \
     }                                                                          \
   }
@@ -136,8 +139,12 @@ TfLiteStatus EqualEval(TfLiteContext* context, TfLiteNode* node) {
       TF_LITE_COMPARISON(int64_t, Equal, requires_broadcast);
       break;
     case kTfLiteUInt8:
-      EvalQuantizedEqual(context, node, input1, input2, output,
-                         requires_broadcast);
+      EvalQuantizedEqual<uint8_t>(context, node, input1, input2, output,
+                                  requires_broadcast);
+      break;
+    case kTfLiteInt8:
+      EvalQuantizedEqual<int8_t>(context, node, input1, input2, output,
+                                 requires_broadcast);
       break;
     default:
       context->ReportError(context,
@@ -165,8 +172,12 @@ TfLiteStatus NotEqualEval(TfLiteContext* context, TfLiteNode* node) {
       TF_LITE_COMPARISON(int64_t, NotEqual, requires_broadcast);
       break;
     case kTfLiteUInt8:
-      EvalQuantizedNotEqual(context, node, input1, input2, output,
-                            requires_broadcast);
+      EvalQuantizedNotEqual<uint8_t>(context, node, input1, input2, output,
+                                     requires_broadcast);
+      break;
+    case kTfLiteInt8:
+      EvalQuantizedNotEqual<int8_t>(context, node, input1, input2, output,
+                                    requires_broadcast);
       break;
     default:
       context->ReportError(context,
@@ -193,8 +204,12 @@ TfLiteStatus GreaterEval(TfLiteContext* context, TfLiteNode* node) {
       TF_LITE_COMPARISON(int64_t, Greater, requires_broadcast);
       break;
     case kTfLiteUInt8:
-      EvalQuantizedGreater(context, node, input1, input2, output,
-                           requires_broadcast);
+      EvalQuantizedGreater<uint8_t>(context, node, input1, input2, output,
+                                    requires_broadcast);
+      break;
+    case kTfLiteInt8:
+      EvalQuantizedGreater<int8_t>(context, node, input1, input2, output,
+                                   requires_broadcast);
       break;
     default:
       context->ReportError(context,
@@ -221,8 +236,12 @@ TfLiteStatus GreaterEqualEval(TfLiteContext* context, TfLiteNode* node) {
       TF_LITE_COMPARISON(int64_t, GreaterEqual, requires_broadcast);
       break;
     case kTfLiteUInt8:
-      EvalQuantizedGreaterEqual(context, node, input1, input2, output,
-                                requires_broadcast);
+      EvalQuantizedGreaterEqual<uint8_t>(context, node, input1, input2, output,
+                                         requires_broadcast);
+      break;
+    case kTfLiteInt8:
+      EvalQuantizedGreaterEqual<int8_t>(context, node, input1, input2, output,
+                                        requires_broadcast);
       break;
     default:
       context->ReportError(context,
@@ -249,8 +268,12 @@ TfLiteStatus LessEval(TfLiteContext* context, TfLiteNode* node) {
       TF_LITE_COMPARISON(int64_t, Less, requires_broadcast);
       break;
     case kTfLiteUInt8:
-      EvalQuantizedLess(context, node, input1, input2, output,
-                        requires_broadcast);
+      EvalQuantizedLess<uint8_t>(context, node, input1, input2, output,
+                                 requires_broadcast);
+      break;
+    case kTfLiteInt8:
+      EvalQuantizedLess<int8_t>(context, node, input1, input2, output,
+                                requires_broadcast);
       break;
     default:
       context->ReportError(context,
@@ -277,8 +300,12 @@ TfLiteStatus LessEqualEval(TfLiteContext* context, TfLiteNode* node) {
       TF_LITE_COMPARISON(int64_t, LessEqual, requires_broadcast);
       break;
     case kTfLiteUInt8:
-      EvalQuantizedLessEqual(context, node, input1, input2, output,
-                             requires_broadcast);
+      EvalQuantizedLessEqual<uint8_t>(context, node, input1, input2, output,
+                                      requires_broadcast);
+      break;
+    case kTfLiteInt8:
+      EvalQuantizedLessEqual<int8_t>(context, node, input1, input2, output,
+                                     requires_broadcast);
       break;
     default:
       context->ReportError(context,
diff --git a/tensorflow/lite/kernels/comparisons_test.cc b/tensorflow/lite/kernels/comparisons_test.cc
index ab10c959a4..6ec1f09a6c 100644
--- a/tensorflow/lite/kernels/comparisons_test.cc
+++ b/tensorflow/lite/kernels/comparisons_test.cc
@@ -363,7 +363,7 @@ TEST(ComparisonsTest, LessEqualBroadcastTwoD) {
   EXPECT_THAT(model.GetOutputShape(), ElementsAre(1, 1, 2, 4));
 }
 
-TEST(QuantizedComparisonsTest, EqualQuantized) {
+TEST(QuantizedComparisonsTest, EqualUInt8Quantized) {
   const float kMin = -1.f;
   const float kMax = 128.f;
   ComparisonOpModel model({TensorType_UINT8, {1, 2, 2, 1}, kMin, kMax},
@@ -376,7 +376,20 @@ TEST(QuantizedComparisonsTest, EqualQuantized) {
   EXPECT_THAT(model.GetOutput(), ElementsAre(true, false, true, false));
 }
 
-TEST(QuantizedComparisonsTest, NotEqualQuantized) {
+TEST(QuantizedComparisonsTest, EqualInt8Quantized) {
+  const float kMin = -127.f;
+  const float kMax = 127.f;
+  ComparisonOpModel model({TensorType_INT8, {1, 2, 2, 1}, kMin, kMax},
+                          {TensorType_INT8, {1, 2, 2, 1}, kMin, kMax},
+                          TensorType_INT8, BuiltinOperator_EQUAL);
+  model.QuantizeAndPopulate<int8_t>(model.input1(), {1, -9, 7, 3});
+  model.QuantizeAndPopulate<int8_t>(model.input2(), {-1, 2, 7, 5});
+  model.Invoke();
+
+  EXPECT_THAT(model.GetOutput(), ElementsAre(false, false, true, false));
+}
+
+TEST(QuantizedComparisonsTest, NotEqualUInt8Quantized) {
   const float kMin = -1.f;
   const float kMax = 128.f;
   ComparisonOpModel model({TensorType_UINT8, {1, 2, 2, 1}, kMin, kMax},
@@ -389,6 +402,19 @@ TEST(QuantizedComparisonsTest, NotEqualQuantized) {
   EXPECT_THAT(model.GetOutput(), ElementsAre(false, true, false, true));
 }
 
+TEST(QuantizedComparisonsTest, NotEqualInt8Quantized) {
+  const float kMin = -127.f;
+  const float kMax = 127.f;
+  ComparisonOpModel model({TensorType_INT8, {1, 2, 2, 1}, kMin, kMax},
+                          {TensorType_INT8, {1, 2, 2, 1}, kMin, kMax},
+                          TensorType_INT8, BuiltinOperator_NOT_EQUAL);
+  model.QuantizeAndPopulate<int8_t>(model.input1(), {1, -9, 7, 3});
+  model.QuantizeAndPopulate<int8_t>(model.input2(), {1, 2, 7, 5});
+  model.Invoke();
+
+  EXPECT_THAT(model.GetOutput(), ElementsAre(false, true, false, true));
+}
+
 TEST(ComparisonsTest, GreaterQuantized) {
   const float kMin = -1.f;
   const float kMax = 128.f;
@@ -470,7 +496,7 @@ TEST(ComparisonsTest, QuantizedEqualWithBroadcast) {
   }
 }
 
-TEST(ComparisonsTest, QuantizedNotEqualWithBroadcast) {
+TEST(ComparisonsTest, QuantizedUInt8NotEqualWithBroadcast) {
   const float kMin = -1.f;
   const float kMax = 128.f;
   std::vector<std::vector<int>> test_shapes = {
@@ -488,7 +514,25 @@ TEST(ComparisonsTest, QuantizedNotEqualWithBroadcast) {
   }
 }
 
-TEST(ComparisonsTest, QuantizedGreaterWithBroadcast) {
+TEST(ComparisonsTest, QuantizedInt8NotEqualWithBroadcast) {
+  const float kMin = -127.f;
+  const float kMax = 127.f;
+  std::vector<std::vector<int>> test_shapes = {
+      {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}};
+  for (int i = 0; i < test_shapes.size(); ++i) {
+    ComparisonOpModel model({TensorType_INT8, test_shapes[i], kMin, kMax},
+                            {TensorType_INT8, {}, kMin, kMax}, TensorType_INT8,
+                            BuiltinOperator_NOT_EQUAL);
+    model.QuantizeAndPopulate<int8_t>(model.input1(), {-20, 2, 7, -8, 11, 20});
+    model.QuantizeAndPopulate<int8_t>(model.input2(), {2});
+    model.Invoke();
+    EXPECT_THAT(model.GetOutput(),
+                ElementsAre(true, false, true, true, true, true))
+        << "With shape number " << i;
+  }
+}
+
+TEST(ComparisonsTest, QuantizedUInt8GreaterWithBroadcast) {
   const float kMin = -1.f;
   const float kMax = 128.f;
   std::vector<std::vector<int>> test_shapes = {
@@ -506,7 +550,25 @@ TEST(ComparisonsTest, QuantizedGreaterWithBroadcast) {
   }
 }
 
-TEST(ComparisonsTest, QuantizedGreaterEqualWithBroadcast) {
+TEST(ComparisonsTest, QuantizedInt8GreaterWithBroadcast) {
+  const float kMin = -127.f;
+  const float kMax = 127.f;
+  std::vector<std::vector<int>> test_shapes = {
+      {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}};
+  for (int i = 0; i < test_shapes.size(); ++i) {
+    ComparisonOpModel model({TensorType_INT8, test_shapes[i], kMin, kMax},
+                            {TensorType_INT8, {}, kMin, kMax}, TensorType_INT8,
+                            BuiltinOperator_GREATER);
+    model.QuantizeAndPopulate<int8_t>(model.input1(), {20, -2, -71, 8, 11, 20});
+    model.QuantizeAndPopulate<int8_t>(model.input2(), {8});
+    model.Invoke();
+    EXPECT_THAT(model.GetOutput(),
+                ElementsAre(true, false, false, false, true, true))
+        << "With shape number " << i;
+  }
+}
+
+TEST(ComparisonsTest, QuantizedUInt8GreaterEqualWithBroadcast) {
   const float kMin = -1.f;
   const float kMax = 128.f;
   std::vector<std::vector<int>> test_shapes = {
@@ -524,7 +586,25 @@ TEST(ComparisonsTest, QuantizedGreaterEqualWithBroadcast) {
   }
 }
 
-TEST(ComparisonsTest, QuantizedLessWithBroadcast) {
+TEST(ComparisonsTest, QuantizedInt8GreaterEqualWithBroadcast) {
+  const float kMin = -127.f;
+  const float kMax = 127.f;
+  std::vector<std::vector<int>> test_shapes = {
+      {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}};
+  for (int i = 0; i < test_shapes.size(); ++i) {
+    ComparisonOpModel model({TensorType_INT8, test_shapes[i], kMin, kMax},
+                            {TensorType_INT8, {}, kMin, kMax}, TensorType_INT8,
+                            BuiltinOperator_GREATER_EQUAL);
+    model.QuantizeAndPopulate<int8_t>(model.input1(), {20, -2, -71, 8, 11, 20});
+    model.QuantizeAndPopulate<int8_t>(model.input2(), {8});
+    model.Invoke();
+    EXPECT_THAT(model.GetOutput(),
+                ElementsAre(true, false, false, true, true, true))
+        << "With shape number " << i;
+  }
+}
+
+TEST(ComparisonsTest, QuantizedUInt8LessWithBroadcast) {
   const float kMin = -1.f;
   const float kMax = 128.f;
   std::vector<std::vector<int>> test_shapes = {
@@ -542,7 +622,25 @@ TEST(ComparisonsTest, QuantizedLessWithBroadcast) {
   }
 }
 
-TEST(ComparisonsTest, QuantizedLessEqualWithBroadcast) {
+TEST(ComparisonsTest, QuantizedInt8LessWithBroadcast) {
+  const float kMin = -127.f;
+  const float kMax = 127.f;
+  std::vector<std::vector<int>> test_shapes = {
+      {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}};
+  for (int i = 0; i < test_shapes.size(); ++i) {
+    ComparisonOpModel model({TensorType_INT8, test_shapes[i], kMin, kMax},
+                            {TensorType_INT8, {}, kMin, kMax}, TensorType_INT8,
+                            BuiltinOperator_LESS);
+    model.QuantizeAndPopulate<int8_t>(model.input1(), {20, -2, -71, 8, 11, 20});
+    model.QuantizeAndPopulate<int8_t>(model.input2(), {8});
+    model.Invoke();
+    EXPECT_THAT(model.GetOutput(),
+                ElementsAre(false, true, true, false, false, false))
+        << "With shape number " << i;
+  }
+}
+
+TEST(ComparisonsTest, QuantizedUInt8LessEqualWithBroadcast) {
   const float kMin = -1.f;
   const float kMax = 128.f;
   std::vector<std::vector<int>> test_shapes = {
@@ -560,6 +658,24 @@ TEST(ComparisonsTest, QuantizedLessEqualWithBroadcast) {
   }
 }
 
+TEST(ComparisonsTest, QuantizedInt8LessEqualWithBroadcast) {
+  const float kMin = -127.f;
+  const float kMax = 127.f;
+  std::vector<std::vector<int>> test_shapes = {
+      {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}};
+  for (int i = 0; i < test_shapes.size(); ++i) {
+    ComparisonOpModel model({TensorType_INT8, test_shapes[i], kMin, kMax},
+                            {TensorType_INT8, {}, kMin, kMax}, TensorType_INT8,
+                            BuiltinOperator_LESS_EQUAL);
+    model.QuantizeAndPopulate<int8_t>(model.input1(), {20, -2, -71, 8, 11, 20});
+    model.QuantizeAndPopulate<int8_t>(model.input2(), {8});
+    model.Invoke();
+    EXPECT_THAT(model.GetOutput(),
+                ElementsAre(false, true, true, true, false, false))
+        << "With shape number " << i;
+  }
+}
+
 }  // namespace
 }  // namespace tflite
 
diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index 9a824b49a0..799811dc25 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -262,10 +262,18 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_ARG_MIN, Register_ARG_MIN(),
              /* min_version */ 1,
              /* max_version */ 2);
-  AddBuiltin(BuiltinOperator_GREATER, Register_GREATER());
-  AddBuiltin(BuiltinOperator_GREATER_EQUAL, Register_GREATER_EQUAL());
-  AddBuiltin(BuiltinOperator_LESS, Register_LESS());
-  AddBuiltin(BuiltinOperator_LESS_EQUAL, Register_LESS_EQUAL());
+  AddBuiltin(BuiltinOperator_GREATER, Register_GREATER(),
+             /* min_version */ 1,
+             /* max_version */ 2);
+  AddBuiltin(BuiltinOperator_GREATER_EQUAL, Register_GREATER_EQUAL(),
+             /* min_version */ 1,
+             /* max_version */ 2);
+  AddBuiltin(BuiltinOperator_LESS, Register_LESS(),
+             /* min_version */ 1,
+             /* max_version */ 2);
+  AddBuiltin(BuiltinOperator_LESS_EQUAL, Register_LESS_EQUAL(),
+             /* min_version */ 1,
+             /* max_version */ 2);
   AddBuiltin(BuiltinOperator_FLOOR, Register_FLOOR());
   AddBuiltin(BuiltinOperator_CEIL, Register_CEIL());
   AddBuiltin(BuiltinOperator_NEG, Register_NEG());
@@ -281,8 +289,12 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_REDUCE_ANY, Register_REDUCE_ANY());
   AddBuiltin(BuiltinOperator_EXPAND_DIMS, Register_EXPAND_DIMS());
   AddBuiltin(BuiltinOperator_SPARSE_TO_DENSE, Register_SPARSE_TO_DENSE());
-  AddBuiltin(BuiltinOperator_EQUAL, Register_EQUAL());
-  AddBuiltin(BuiltinOperator_NOT_EQUAL, Register_NOT_EQUAL());
+  AddBuiltin(BuiltinOperator_EQUAL, Register_EQUAL(),
+             /* min_version */ 1,
+             /* max_version */ 2);
+  AddBuiltin(BuiltinOperator_NOT_EQUAL, Register_NOT_EQUAL(),
+             /* min_version */ 1,
+             /* max_version */ 2);
   AddBuiltin(BuiltinOperator_SQRT, Register_SQRT());
   AddBuiltin(BuiltinOperator_RSQRT, Register_RSQRT());
   AddBuiltin(BuiltinOperator_SHAPE, Register_SHAPE());
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index 07ca10da7c..7aaad7f61e 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -1944,6 +1944,92 @@ class Dequantize
   }
 };
 
+class Equal : public SimpleOperator<TensorFlowEqualOperator> {
+ public:
+  explicit Equal() : SimpleOperator("EQUAL", OperatorType::kEqual) {}
+  int GetVersion(const OperatorSignature& op_signature) const override {
+    const string& input_name = op_signature.op->inputs[0];
+    const Array& input_array = op_signature.model->GetArray(input_name);
+    // Version 2 supports signed int8 input types.
+    if (input_array.data_type == ArrayDataType::kInt8) {
+      return 2;
+    }
+    return 1;
+  }
+};
+
+class NotEqual : public SimpleOperator<TensorFlowNotEqualOperator> {
+ public:
+  explicit NotEqual() : SimpleOperator("NOT_EQUAL", OperatorType::kNotEqual) {}
+  int GetVersion(const OperatorSignature& op_signature) const override {
+    const string& input_name = op_signature.op->inputs[0];
+    const Array& input_array = op_signature.model->GetArray(input_name);
+    // Version 2 supports signed int8 input types.
+    if (input_array.data_type == ArrayDataType::kInt8) {
+      return 2;
+    }
+    return 1;
+  }
+};
+
+class Greater : public SimpleOperator<TensorFlowGreaterOperator> {
+ public:
+  explicit Greater() : SimpleOperator("GREATER", OperatorType::kGreater) {}
+  int GetVersion(const OperatorSignature& op_signature) const override {
+    const string& input_name = op_signature.op->inputs[0];
+    const Array& input_array = op_signature.model->GetArray(input_name);
+    // Version 2 supports signed int8 input types.
+    if (input_array.data_type == ArrayDataType::kInt8) {
+      return 2;
+    }
+    return 1;
+  }
+};
+
+class GreaterEqual : public SimpleOperator<TensorFlowGreaterEqualOperator> {
+ public:
+  explicit GreaterEqual()
+      : SimpleOperator("GREATER_EQUAL", OperatorType::kGreaterEqual) {}
+  int GetVersion(const OperatorSignature& op_signature) const override {
+    const string& input_name = op_signature.op->inputs[0];
+    const Array& input_array = op_signature.model->GetArray(input_name);
+    // Version 2 supports signed int8 input types.
+    if (input_array.data_type == ArrayDataType::kInt8) {
+      return 2;
+    }
+    return 1;
+  }
+};
+
+class Less : public SimpleOperator<TensorFlowLessOperator> {
+ public:
+  explicit Less() : SimpleOperator("LESS", OperatorType::kLess) {}
+  int GetVersion(const OperatorSignature& op_signature) const override {
+    const string& input_name = op_signature.op->inputs[0];
+    const Array& input_array = op_signature.model->GetArray(input_name);
+    // Version 2 supports signed int8 input types.
+    if (input_array.data_type == ArrayDataType::kInt8) {
+      return 2;
+    }
+    return 1;
+  }
+};
+
+class LessEqual : public SimpleOperator<TensorFlowLessEqualOperator> {
+ public:
+  explicit LessEqual()
+      : SimpleOperator("LESS_EQUAL", OperatorType::kLessEqual) {}
+  int GetVersion(const OperatorSignature& op_signature) const override {
+    const string& input_name = op_signature.op->inputs[0];
+    const Array& input_array = op_signature.model->GetArray(input_name);
+    // Version 2 supports signed int8 input types.
+    if (input_array.data_type == ArrayDataType::kInt8) {
+      return 2;
+    }
+    return 1;
+  }
+};
+
 namespace {
 // Build a vector containing all the known operators.
 std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
@@ -2122,18 +2208,12 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
       "MAXIMUM", OperatorType::kMaximum));  //  Element-wise Maximum
   ops.push_back(MakeUnique<SimpleOperator<TensorFlowMinimumOperator>>(
       "MINIMUM", OperatorType::kMinimum));  //  Element-wise Minimum
-  ops.push_back(MakeUnique<SimpleOperator<TensorFlowGreaterOperator>>(
-      "GREATER", OperatorType::kGreater));
-  ops.push_back(MakeUnique<SimpleOperator<TensorFlowGreaterEqualOperator>>(
-      "GREATER_EQUAL", OperatorType::kGreaterEqual));
-  ops.push_back(MakeUnique<SimpleOperator<TensorFlowLessOperator>>(
-      "LESS", OperatorType::kLess));
-  ops.push_back(MakeUnique<SimpleOperator<TensorFlowLessEqualOperator>>(
-      "LESS_EQUAL", OperatorType::kLessEqual));
-  ops.push_back(MakeUnique<SimpleOperator<TensorFlowEqualOperator>>(
-      "EQUAL", OperatorType::kEqual));
-  ops.push_back(MakeUnique<SimpleOperator<TensorFlowNotEqualOperator>>(
-      "NOT_EQUAL", OperatorType::kNotEqual));
+  ops.push_back(MakeUnique<Greater>());
+  ops.push_back(MakeUnique<GreaterEqual>());
+  ops.push_back(MakeUnique<Less>());
+  ops.push_back(MakeUnique<LessEqual>());
+  ops.push_back(MakeUnique<Equal>());
+  ops.push_back(MakeUnique<NotEqual>());
   ops.push_back(
       MakeUnique<SimpleOperator<NegOperator>>("NEG", OperatorType::kNeg));
   ops.push_back(MakeUnique<SimpleOperator<SelectOperator>>(
diff --git a/tensorflow/lite/toco/tflite/operator_test.cc b/tensorflow/lite/toco/tflite/operator_test.cc
index 43b52c4e93..13da1682d3 100644
--- a/tensorflow/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/lite/toco/tflite/operator_test.cc
@@ -677,6 +677,50 @@ TEST_F(OperatorTest, BuiltinUnique) {
   EXPECT_EQ(output_toco_op->idx_out_type, op.idx_out_type);
 }
 
+template <typename Op>
+void VersioningTest() {
+  Op op;
+  op.inputs = {"input1"};
+  auto operator_by_type_map = BuildOperatorByTypeMap(false /*enable_flex_ops*/);
+  const BaseOperator* base_op = operator_by_type_map.at(op.type).get();
+
+  Model uint8_model;
+  Array& uint8_array = uint8_model.GetOrCreateArray(op.inputs[0]);
+  uint8_array.data_type = ArrayDataType::kUint8;
+  OperatorSignature uint8_signature = {.model = &uint8_model, .op = &op};
+  EXPECT_EQ(base_op->GetVersion(uint8_signature), 1);
+
+  Model int8_model;
+  Array& int8_array = int8_model.GetOrCreateArray(op.inputs[0]);
+  int8_array.data_type = ArrayDataType::kInt8;
+  OperatorSignature int8_signature = {.model = &int8_model, .op = &op};
+  EXPECT_EQ(base_op->GetVersion(int8_signature), 2);
+}
+
+TEST_F(OperatorTest, VersioningEqualTest) {
+  VersioningTest<TensorFlowEqualOperator>();
+}
+
+TEST_F(OperatorTest, VersioningNotEqualTest) {
+  VersioningTest<TensorFlowNotEqualOperator>();
+}
+
+TEST_F(OperatorTest, VersioningLessTest) {
+  VersioningTest<TensorFlowLessOperator>();
+}
+
+TEST_F(OperatorTest, VersioningLessEqualTest) {
+  VersioningTest<TensorFlowLessEqualOperator>();
+}
+
+TEST_F(OperatorTest, VersioningGreaterTest) {
+  VersioningTest<TensorFlowGreaterOperator>();
+}
+
+TEST_F(OperatorTest, VersioningGreaterEqualTest) {
+  VersioningTest<TensorFlowGreaterEqualOperator>();
+}
+
 }  // namespace
 }  // namespace tflite
 
-- 
GitLab


From 9596d6dfaed8cac0f03279bd51e9839ff3bfe085 Mon Sep 17 00:00:00 2001
From: Andr? Susano Pinto <andresp@google.com>
Date: Thu, 7 Feb 2019 08:28:59 -0800
Subject: [PATCH 2065/2345] Refactor setup of revived functions from a
 SavedModel.

PiperOrigin-RevId: 232879303
---
 tensorflow/python/saved_model/load.py | 90 +++++++++++++++------------
 1 file changed, 50 insertions(+), 40 deletions(-)

diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
index e8474f2312..9fb1fe9fb8 100644
--- a/tensorflow/python/saved_model/load.py
+++ b/tensorflow/python/saved_model/load.py
@@ -56,49 +56,59 @@ class _Loader(object):
         function_deserialization.load_function_def_library(
             meta_graph.graph_def.library))
     self._load_all()
-    self._setup_functions()
+    # TODO(b/124045874): There are limitations with functions whose captures
+    # trigger other functions to be executed. For now it is only guaranteed to
+    # work if the captures of a function only trigger functions without
+    # captures.
+    self._setup_functions_structures()
+    self._setup_functions_captures()
     self._restore_checkpoint()
 
-  def _setup_concrete_function(self, proto, concrete_function, coder):
-    """Setup captured tensors and outputs for a single concrete function."""
-    bound_inputs = [
-        self._get_tensor_from_node(node_id)
-        for node_id in proto.bound_inputs]
-    bound_variables = [
-        self._nodes[node_id]
-        for node_id in proto.bound_inputs
-        if self._proto.nodes[node_id].WhichOneof("kind") == "variable"
-    ]
-    # TODO(andresp): This is only injecting the captured inputs into the
-    # concrete function, note that we did not modify the FuncGraph
-    # itself.
-    concrete_function._captured_inputs = bound_inputs  # pylint: disable=protected-access
-    concrete_function._func_graph.variables = bound_variables  # pylint: disable=protected-access
-    # By setting the structured_outputs directly, we can rely on this
-    # function_lib.ConcreteFunction object to perform the output repacking
-    # logic. The only limitation of that logic is that it only works
-    # with output that is convertible to Tensors and the conversion
-    # always happens. For example tf.TensorShape([2, 3]) will be
-    # converted to Tensor representing [2, 3].
-    original_outputs = coder.decode_proto(proto.output_signature)
-    # The original_outputs here had Tensors converted to TensorSpecs, so
-    # the restored function's structured_outputs field will not be
-    # exactly the same. Fortunately the repacking logic cares only about
-    # the structure.
-    # TODO(vbardiovsky): Should we just replicate the structures, with
-    # Nones instead of real objects?
-    concrete_function._func_graph.structured_outputs = original_outputs  # pylint: disable=protected-access
-    concrete_function._func_graph.structured_input_signature = (  # pylint: disable=protected-access
-        coder.decode_proto(proto.canonicalized_input_signature))
-
-  def _setup_functions(self):
-    """Setup captures and output structure in restored functions."""
+    for node in self._nodes:
+      if isinstance(node, tracking.TrackableResource):
+        init_op = node.initialize()
+        ops.add_to_collection(ops.GraphKeys.TABLE_INITIALIZERS, init_op)
+
+  def _setup_functions_structures(self):
+    """Setup structure for inputs and outputs of restored functions."""
     coder = nested_structure_coder.StructureCoder()
-    for name, concrete_function_proto in self._proto.concrete_functions.items():
-      self._setup_concrete_function(
-          concrete_function_proto,
-          self._concrete_functions[name],
-          coder)
+    for name, proto in sorted(self._proto.concrete_functions.items()):
+      concrete_function = self._concrete_functions[name]
+      # By setting the structured_outputs directly, we can rely on this
+      # function_lib.ConcreteFunction object to perform the output repacking
+      # logic. The only limitation of that logic is that it only works
+      # with output that is convertible to Tensors and the conversion
+      # always happens. For example tf.TensorShape([2, 3]) will be
+      # converted to Tensor representing [2, 3].
+      original_outputs = coder.decode_proto(proto.output_signature)
+      # The original_outputs here had Tensors converted to TensorSpecs, so
+      # the restored function's structured_outputs field will not be
+      # exactly the same. Fortunately the repacking logic cares only about
+      # the structure.
+      # TODO(vbardiovsky): Should we just replicate the structures, with
+      # Nones instead of real objects?
+      concrete_function._func_graph.structured_outputs = original_outputs  # pylint: disable=protected-access
+      concrete_function._func_graph.structured_input_signature = (  # pylint: disable=protected-access
+          coder.decode_proto(proto.canonicalized_input_signature))
+
+  def _setup_functions_captures(self):
+    """Setup captures and variables in restored functions."""
+    concrete_functions = sorted(self._proto.concrete_functions.items())
+    for name, proto in concrete_functions:
+      concrete_function = self._concrete_functions[name]
+      bound_inputs = [
+          self._get_tensor_from_node(node_id)
+          for node_id in proto.bound_inputs]
+      bound_variables = [
+          self._nodes[node_id]
+          for node_id in proto.bound_inputs
+          if self._proto.nodes[node_id].WhichOneof("kind") == "variable"
+      ]
+      # TODO(andresp): This is only injecting the captured inputs into the
+      # concrete function, note that we did not modify the FuncGraph
+      # itself.
+      concrete_function._captured_inputs = bound_inputs  # pylint: disable=protected-access
+      concrete_function._func_graph.variables = bound_variables  # pylint: disable=protected-access
 
   def _get_tensor_from_node(self, node_id):
     obj = self._nodes[node_id]
-- 
GitLab


From 60b3c5f84d12f0d404da48f808f01f7474bb18d0 Mon Sep 17 00:00:00 2001
From: Arpit Shah <ashah@ambiqmicro.com>
Date: Thu, 7 Feb 2019 11:14:04 -0600
Subject: [PATCH 2066/2345] Support for public Ambiq SDK

---
 tensorflow/lite/experimental/micro/README.md  | 18 +++++++------
 .../micro_speech/apollo3/pushbutton_main.c    |  2 +-
 .../experimental/micro/tools/make/Makefile    |  8 ++++++
 .../micro/tools/make/download_dependencies.sh | 25 ++++++++++++-------
 .../make/targets/apollo3evb_makefile.inc      | 10 ++++----
 5 files changed, 40 insertions(+), 23 deletions(-)

diff --git a/tensorflow/lite/experimental/micro/README.md b/tensorflow/lite/experimental/micro/README.md
index 30070bf008..31ddb7ce06 100644
--- a/tensorflow/lite/experimental/micro/README.md
+++ b/tensorflow/lite/experimental/micro/README.md
@@ -265,12 +265,10 @@ You should see the following log with the magic string `~~~ALL TEST PASSED~~~`:
 
 Follow these steps to get the pushbutton yes/no example working on Apollo 3:
 
-1.  Make sure to run the "Getting Started" section before performing the
-    following steps
-2.  Download Apollo3-SDK-2018.08.13 and place in
-    `tensorflow/lite/experimental/micro/tools/make/downloads`. This is not yet
-    publicly released, but you can contact ashah@ambiqmicro.com to request a
-    copy.
+1.  Make sure to run the "Building Portable Reference Code using Make" section 
+    before performing the following steps
+2.  The Ambiq Micro SDK is downloaded into
+    `tensorflow/lite/experimental/micro/tools/make/downloads` by 'download_dependencies.sh'.
 3.  Compile the project with the following command: make -f
     tensorflow/lite/experimental/micro/tools/make/Makefile TARGET=apollo3evb
     pushbutton_cmsis_speech_test_bin
@@ -296,8 +294,12 @@ Follow these steps to get the pushbutton yes/no example working on Apollo 3:
         5.  Press Ctrl+c to exit
     4.  Press BTN2. An LED will flash for 1 second. Speak your utterance during
         this one second
-    5.  The debugger will print out four numbers. They are the probabilites for
-        1) no speech, 2) unknown speech, 3) yes, 4) no
+    5.  The debugger will print out four numbers.
+        They are the probabilites for
+        1.  no speech
+        2.  unkown speech
+        3.  yes
+        4.  no
     6.  The EVB LEDs will indicate detection.
         1.  LED0 (rightmost LED) - ON when capturing 1sec of audio
         2.  LED1 - ON when detecting silence
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/pushbutton_main.c b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/pushbutton_main.c
index afee38343b..4f70d47c3e 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/pushbutton_main.c
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/pushbutton_main.c
@@ -157,7 +157,7 @@ void pdm_data_get(void) {
 // PDM interrupt handler.
 //
 //*****************************************************************************
-void am_pdm_isr(void) {
+void am_pdm0_isr(void) {
   uint32_t ui32Status;
   //
   // Read the interrupt status.
diff --git a/tensorflow/lite/experimental/micro/tools/make/Makefile b/tensorflow/lite/experimental/micro/tools/make/Makefile
index 3457fafb9d..19a464f726 100644
--- a/tensorflow/lite/experimental/micro/tools/make/Makefile
+++ b/tensorflow/lite/experimental/micro/tools/make/Makefile
@@ -209,6 +209,14 @@ $(BINDIR)%_test : $(OBJDIR)%_test.o $(MICROLITE_LIB_PATH)
 $(BINDIR)%.test_target: $(BINDIR)%_test
 	$(TEST_SCRIPT) $< '~~~ALL TESTS PASSED~~~'
 
+# snease: Add %.bin rule here since BINDIR is now defined
+# These are microcontroller-specific rules for converting the ELF output
+# of the linker into a binary image that can be loaded directly.
+OBJCOPY := $(TARGET_TOOLCHAIN_PREFIX)objcopy
+$(BINDIR)%.bin: $(BINDIR)%
+	@mkdir -p $(dir $@)
+	$(OBJCOPY) $< $@ -O binary
+
 # Generate standalone makefile projects for all of the test targets.
 $(foreach TEST_TARGET,$(MICROLITE_TEST_SRCS),\
 $(eval $(call microlite_test,$(notdir $(basename $(TEST_TARGET))),$(TEST_TARGET))))
diff --git a/tensorflow/lite/experimental/micro/tools/make/download_dependencies.sh b/tensorflow/lite/experimental/micro/tools/make/download_dependencies.sh
index 82c15e32f6..4c747ac397 100755
--- a/tensorflow/lite/experimental/micro/tools/make/download_dependencies.sh
+++ b/tensorflow/lite/experimental/micro/tools/make/download_dependencies.sh
@@ -35,6 +35,7 @@ CMSIS_URL="https://github.com/ARM-software/CMSIS_5/archive/5.4.0.zip"
 STM32_BARE_LIB_URL="https://github.com/google/stm32_bare_lib/archive/c07d611fb0af58450c5a3e0ab4d52b47f99bc82d.zip"
 SIFIVE_FE310_LIB_URL="https://github.com/sifive/freedom-e-sdk/archive/baeeb8fd497a99b3c141d7494309ec2e64f19bdf.zip"
 RISCV_TOOLCHAIN_URL="https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-20181030-x86_64-linux-ubuntu14.tar.gz"
+AM_SDK_URL="http://s3.asia.ambiqmicro.com/downloads/AmbiqSuite-Rel2.0.0.zip"
 AP3_URL="https://github.com/AmbiqMicro/TFLiteMicro_Apollo3/archive/dfbcef9a57276c087d95aab7cb234f1d4c9eaaba.zip"
 CUST_CMSIS_URL="https://github.com/AmbiqMicro/TFLiteMicro_CustCMSIS/archive/8f63966c5692e6a3a83956efd2e4aed77c4c9949.zip"
 GCC_EMBEDDED_URL="https://developer.arm.com/-/media/Files/downloads/gnu-rm/7-2018q2/gcc-arm-none-eabi-7-2018-q2-update-linux.tar.bz2"
@@ -72,25 +73,30 @@ download_and_extract() {
   find "${dir}" -type f -name '*BUILD' -delete
 }
 
-patch_apollo3_sdk() {
-  local ap3_dir="${1}"
-  if [ ! -f ${ap3_dir}/VERSION.txt ]; then
-    echo "Could not find ${ap3_dir}, skipping Apollo3 SDK";
+patch_am_sdk() {
+  local am_dir="${1}"
+  if [ ! -f ${am_dir}/VERSION.txt ]; then
+    echo "Could not find ${am_dir}, skipping AmbiqMicro SDK patch";
     return;
   fi
-  local src_dir=${ap3_dir}/boards/apollo3_evb/examples/hello_world/gcc
-  local dest_dir=${ap3_dir}/boards/apollo3_evb/examples/hello_world/gcc_patched
+
+  local src_dir=${am_dir}/boards/apollo3_evb/examples/hello_world/gcc
+  local dest_dir=${am_dir}/boards/apollo3_evb/examples/hello_world/gcc_patched
+
   rm -rf ${dest_dir}
   mkdir ${dest_dir}
+
   cp "${src_dir}/startup_gcc.c" "${dest_dir}/startup_gcc.c"
   cp "${src_dir}/hello_world.ld" "${dest_dir}/apollo3evb.ld"
-  sed -i -e '131s/1024/1024\*20/g' "${dest_dir}/startup_gcc.c"
+
+  sed -i -e '114s/1024/1024\*20/g' "${dest_dir}/startup_gcc.c"
   sed -i -e 's/main/_main/g' "${dest_dir}/startup_gcc.c"
+
   sed -i -e '3s/hello_world.ld/apollo3evb.ld/g' "${dest_dir}/apollo3evb.ld"
   sed -i -e '3s/startup_gnu/startup_gcc/g' "${dest_dir}/apollo3evb.ld"
-  sed -i -e '6s/am_reset_isr/Reset_Handler/g' "${dest_dir}/apollo3evb.ld"
   sed -i -e '22s/\*(.text\*)/\*(.text\*)\n\n\t\/\* These are the C++ global constructors.  Stick them all here and\n\t \* then walk through the array in main() calling them all.\n\t \*\/\n\t_init_array_start = .;\n\tKEEP (\*(SORT(.init_array\*)))\n\t_init_array_end = .;\n\n\t\/\* XXX Currently not doing anything for global destructors. \*\/\n/g' "${dest_dir}/apollo3evb.ld"
   sed -i -e "70s/} > SRAM/} > SRAM\n    \/\* Add this to satisfy reference to symbol 'end' from libnosys.a(sbrk.o)\n     \* to denote the HEAP start.\n     \*\/\n   end = .;/g" "${dest_dir}/apollo3evb.ld"
+
   echo "Finished preparing Apollo3 files"
 }
 
@@ -100,8 +106,9 @@ download_and_extract "${CMSIS_URL}" "${DOWNLOADS_DIR}/cmsis"
 download_and_extract "${STM32_BARE_LIB_URL}" "${DOWNLOADS_DIR}/stm32_bare_lib"
 download_and_extract "${SIFIVE_FE310_LIB_URL}" "${DOWNLOADS_DIR}/sifive_fe310_lib"
 download_and_extract "${RISCV_TOOLCHAIN_URL}" "${DOWNLOADS_DIR}/riscv_toolchain"
+download_and_extract "${AM_SDK_URL}" "${DOWNLOADS_DIR}/AmbiqSuite-Rel2.0.0"
+patch_am_sdk "${DOWNLOADS_DIR}/AmbiqSuite-Rel2.0.0"
 download_and_extract "${AP3_URL}" "${DOWNLOADS_DIR}/apollo3_ext"
-patch_apollo3_sdk "${DOWNLOADS_DIR}/Apollo3-SDK-2018.08.13"
 download_and_extract "${CUST_CMSIS_URL}" "${DOWNLOADS_DIR}/CMSIS_ext"
 download_and_extract "${GCC_EMBEDDED_URL}" "${DOWNLOADS_DIR}/gcc_embedded"
 
diff --git a/tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb_makefile.inc b/tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb_makefile.inc
index 6ed402a623..fa9dc9c99b 100644
--- a/tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb_makefile.inc
+++ b/tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb_makefile.inc
@@ -5,9 +5,9 @@ ifeq ($(TARGET), apollo3evb)
   TARGET_TOOLCHAIN_PREFIX := arm-none-eabi-
   # Download the Ambiq Apollo3 SDK and set this variable to find the header
   # files:
-  APOLLO3_SDK := $(MAKEFILE_DIR)/downloads/Apollo3-SDK-2018.08.13
+  APOLLO3_SDK := $(MAKEFILE_DIR)/downloads/AmbiqSuite-Rel2.0.0
   # Need a pointer to the GNU ARM toolchain for crtbegin.o for the fp functions
-  # with the softfp interfaces.
+  # with the hard interfaces.
   GCC_ARM := $(MAKEFILE_DIR)/downloads/gcc_embedded/
 
   PLATFORM_FLAGS = \
@@ -31,7 +31,7 @@ ifeq ($(TARGET), apollo3evb)
     -mcpu=cortex-m4 \
     -mthumb \
     -mfpu=fpv4-sp-d16 \
-    -mfloat-abi=softfp \
+    -mfloat-abi=hard \
     -std=gnu++11 \
     -Wvla \
     -Wall \
@@ -49,7 +49,7 @@ ifeq ($(TARGET), apollo3evb)
   CXXFLAGS += $(PLATFORM_FLAGS)
   CCFLAGS += $(PLATFORM_FLAGS)
   LDFLAGS += \
-    -mthumb -mcpu=cortex-m4 -mfpu=fpv4-sp-d16 -mfloat-abi=softfp \
+    -mthumb -mcpu=cortex-m4 -mfpu=fpv4-sp-d16 -mfloat-abi=hard \
     -nostartfiles -static \
     -Wl,--gc-sections -Wl,--entry,Reset_Handler \
     -Wl,--start-group -lm -lc -lgcc -Wl,--end-group \
@@ -61,7 +61,7 @@ ifeq ($(TARGET), apollo3evb)
   MICROLITE_LIBS := \
     $(APOLLO3_SDK)/boards/apollo3_evb/bsp/gcc/bin/libam_bsp.a \
     $(APOLLO3_SDK)/mcu/apollo3/hal/gcc/bin/libam_hal.a \
-    $(GCC_ARM)/lib/gcc/arm-none-eabi/7.3.1/thumb/v7e-m/fpv4-sp/softfp/crtbegin.o \
+    $(GCC_ARM)/lib/gcc/arm-none-eabi/7.3.1/thumb/v7e-m/fpv4-sp/hard/crtbegin.o \
     -lm
   INCLUDES += \
     -isystem$(MAKEFILE_DIR)/downloads/cmsis/CMSIS/Core/Include/ \
-- 
GitLab


From 9b0414dc8c51ba9deb4313da8106ab2ec9b622be Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Thu, 7 Feb 2019 09:23:44 -0800
Subject: [PATCH 2067/2345] Fix TensorFlow Lite Makefile build

PiperOrigin-RevId: 232887546
---
 tensorflow/lite/tools/make/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/tools/make/Makefile b/tensorflow/lite/tools/make/Makefile
index 5c4bb4dbf8..4a22d7630a 100644
--- a/tensorflow/lite/tools/make/Makefile
+++ b/tensorflow/lite/tools/make/Makefile
@@ -109,7 +109,7 @@ $(wildcard tensorflow/lite/*test.cc) \
 $(wildcard tensorflow/lite/*/*test.cc) \
 $(wildcard tensorflow/lite/*/*/*test.cc) \
 $(wildcard tensorflow/lite/*/*/*/*test.cc) \
-$(wildcard tensorflow/lite/kernels/test_util.cc) \
+$(wildcard tensorflow/lite/kernels/*test_util.cc) \
 $(MINIMAL_SRCS)
 
 ifeq ($(BUILD_TYPE),micro)
-- 
GitLab


From 9495d4760917adc259e3c1692b636b2c361c6a1a Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Thu, 7 Feb 2019 09:34:53 -0800
Subject: [PATCH 2068/2345] Fix bullet list in docstring.

PiperOrigin-RevId: 232889358
---
 tensorflow/python/autograph/utils/ag_logging.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/python/autograph/utils/ag_logging.py b/tensorflow/python/autograph/utils/ag_logging.py
index cd737a8290..037b0615f6 100644
--- a/tensorflow/python/autograph/utils/ag_logging.py
+++ b/tensorflow/python/autograph/utils/ag_logging.py
@@ -46,8 +46,11 @@ def set_verbosity(level, alsologtostdout=False):
   more in-depth debugging.
 
   There are two controls that control the logging verbosity:
+
    * The `set_verbosity` function
+
    * The `AUTOGRAPH_VERBOSITY` environment variable
+
   `set_verbosity` takes precedence over the environment variable.
 
   For example:
-- 
GitLab


From b50b7911ec301959b66919f2765f68eaa177fe5e Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Thu, 7 Feb 2019 09:35:29 -0800
Subject: [PATCH 2069/2345] Propagate explicit device placement in more places.

Specifically when lowering While ops and for NoOps in InlineFunctionBody.

PiperOrigin-RevId: 232889472
---
 tensorflow/core/common_runtime/function.cc    |  1 +
 .../core/common_runtime/lower_while_op.cc     | 11 +++++
 .../kernel_tests/control_flow_ops_py_test.py  | 41 +++++++++++++++++++
 3 files changed, 53 insertions(+)

diff --git a/tensorflow/core/common_runtime/function.cc b/tensorflow/core/common_runtime/function.cc
index 82bbb7e689..72aee7b280 100644
--- a/tensorflow/core/common_runtime/function.cc
+++ b/tensorflow/core/common_runtime/function.cc
@@ -1424,6 +1424,7 @@ void InlineFunctionBody(const FunctionLibraryDefinition& flib_def, Graph* g,
     if (e->IsControlEdge()) {
       if (input_control_node == nullptr) {
         input_control_node = AddNoOp(g);
+        input_control_node->set_requested_device(caller->def().device());
       }
       g->AddControlEdge(e->src(), input_control_node);
     } else {
diff --git a/tensorflow/core/common_runtime/lower_while_op.cc b/tensorflow/core/common_runtime/lower_while_op.cc
index 7552838ca1..f1c3bbd552 100644
--- a/tensorflow/core/common_runtime/lower_while_op.cc
+++ b/tensorflow/core/common_runtime/lower_while_op.cc
@@ -200,6 +200,7 @@ Status LowerWhileHelper::CreateEnterNodes() {
                            .Input(NodeOut(edge->src(), edge->src_output()))
                            .Attr("frame_name", name_)
                            .Attr("parallel_iterations", parallel_iterations_)
+                           .Device(while_op_->requested_device())
                            .Finalize(graph_, &enter_node));
     enter_nodes_[edge->dst_input()] = enter_node;
   }
@@ -216,6 +217,7 @@ Status LowerWhileHelper::CreateEnterNodes() {
     TF_RETURN_IF_ERROR(NodeBuilder(NewName("LoopControlInputs"), "NoOp",
                                    graph_->op_registry(), &debug_info_)
                            .ControlInputs(control_inputs)
+                           .Device(while_op_->requested_device())
                            .Finalize(graph_, &incoming_control_node));
     for (Node* n : enter_nodes_) {
       graph_->AddControlEdge(incoming_control_node, n);
@@ -231,6 +233,7 @@ Status LowerWhileHelper::CreateMergeNodes() {
         NodeBuilder(NewName("merge"), "Merge", graph_->op_registry(),
                     &debug_info_)
             .Input({NodeOut(enter_node, 0), NodeOut(enter_node, 0)})
+            .Device(while_op_->requested_device())
             .Finalize(graph_, &merge_node));
     merge_nodes_.emplace_back(merge_node);
   }
@@ -241,6 +244,7 @@ Status LowerWhileHelper::CreateCondFuncCallNode() {
   for (Node* merge_node : merge_nodes_) {
     cond_call_builder_.Input(NodeOut(merge_node, 0));
   }
+  cond_call_builder_.Device(while_op_->requested_device());
   TF_RETURN_IF_ERROR(cond_call_builder_.Finalize(graph_, &cond_call_node_));
   // Add a control edge to make sure the Const nodes in the cond function
   // are in the same frame as the rest of the function, otherwise
@@ -249,6 +253,7 @@ Status LowerWhileHelper::CreateCondFuncCallNode() {
   TF_RETURN_IF_ERROR(NodeBuilder(NewName("LoopCond"), "LoopCond",
                                  graph_->op_registry(), &debug_info_)
                          .Input(NodeOut(cond_call_node_, 0))
+                         .Device(while_op_->requested_device())
                          .Finalize(graph_, &loop_cond_node_));
   return Status::OK();
 }
@@ -270,6 +275,7 @@ Status LowerWhileHelper::CreateSwitchNodes() {
                                    graph_->op_registry(), &debug_info_)
                            .Input(NodeOut(merge_nodes_[i], 0))
                            .Input(NodeOut(loop_cond_node_, 0))
+                           .Device(while_op_->requested_device())
                            .Finalize(graph_, &switch_node));
     switch_nodes_.emplace_back(switch_node);
   }
@@ -280,6 +286,7 @@ Status LowerWhileHelper::CreateBodyFuncCallNode() {
   for (Node* switch_node : switch_nodes_) {
     body_call_builder_.Input(NodeOut(switch_node, 1));
   }
+  body_call_builder_.Device(while_op_->requested_device());
   TF_RETURN_IF_ERROR(body_call_builder_.Finalize(graph_, &body_call_node_));
   // Add a control edge to make sure the Const nodes in the body function
   // are in the same frame as the rest of the function, otherwise
@@ -296,6 +303,7 @@ Status LowerWhileHelper::CreateBodyFuncCallNode() {
   TF_RETURN_IF_ERROR(NodeBuilder(NewName("loop_body_control"), op_type,
                                  graph_->op_registry(), &debug_info_)
                          .Input(NodeOut(switch_nodes_[0], 1))
+                         .Device(while_op_->requested_device())
                          .Finalize(graph_, &body_control_node_));
   graph_->AddControlEdge(body_control_node_, body_call_node_);
   return Status::OK();
@@ -309,6 +317,7 @@ Status LowerWhileHelper::CreateExitNodes() {
     TF_RETURN_IF_ERROR(NodeBuilder(NewName("exit"), "Exit",
                                    graph_->op_registry(), &debug_info_)
                            .Input(NodeOut(switch_node, 0))
+                           .Device(while_op_->requested_device())
                            .Finalize(graph_, &exit_node));
     exit_nodes_.emplace_back(exit_node);
     outputs.emplace_back(NodeOut(exit_node, 0));
@@ -320,6 +329,7 @@ Status LowerWhileHelper::CreateExitNodes() {
   // 2. Fetching the output of the While node by name in calls to sess.run.
   NodeBuilder ib(name_, "IdentityN", OpRegistry::Global(), &debug_info_);
   ib.Input(outputs);
+  ib.Device(while_op_->requested_device());
   TF_RETURN_IF_ERROR(ib.Finalize(graph_, &lowered_while_output_));
   return Status::OK();
 }
@@ -330,6 +340,7 @@ Status LowerWhileHelper::CreateNextIterationNodes() {
     TF_RETURN_IF_ERROR(NodeBuilder(NewName("next_iteration"), "NextIteration",
                                    graph_->op_registry(), &debug_info_)
                            .Input(NodeOut(body_call_node_, i))
+                           .Device(while_op_->requested_device())
                            .Finalize(graph_, &next_iteration));
     next_iterations_nodes_.emplace_back(next_iteration);
   }
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index a073bb3f22..7b14145bd4 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -654,6 +654,27 @@ class ControlFlowTest(test.TestCase):
       r = control_flow_ops.cond(pred, fn1, fn2)
       self.assertAllEqual([11, 12], self.evaluate(r))
 
+  @test_util.run_gpu_only
+  @test_util.run_deprecated_v1
+  def testCond_Device(self):
+    x = constant_op.constant(-10)
+
+    # True branch function defined outside of device scope
+    def true_fn():
+      return math_ops.exp(x)
+
+    with ops.device("CPU:0"):
+      r = control_flow_ops.cond(
+          constant_op.constant(True), true_fn, lambda: 0.)
+      self.assertIn("cpu", r.device.lower())
+
+    with session.Session() as sess:
+      options = config_pb2.RunOptions(output_partition_graphs=True)
+      run_metadata = config_pb2.RunMetadata()
+      sess.run(r, options=options, run_metadata=run_metadata)
+      # We expect that everything runs on CPU, even if GPU is available.
+      self.assertEqual(len(run_metadata.partition_graphs), 1)
+
   def testCondListOutput(self):
     with self.cached_session() as sess:
       x = constant_op.constant(10)
@@ -1503,6 +1524,26 @@ class ControlFlowTest(test.TestCase):
       result = r[2]
     self.assertAllEqual(np.array([0, 1, 2, 3, 4, 5, 6]), result)
 
+  @test_util.run_gpu_only
+  @test_util.run_deprecated_v1
+  def testWhile_Device(self):
+
+    # Body function defined outside of device scope
+    def body(x):
+      return math_ops.exp(x)
+
+    with ops.device("CPU:0"):
+      r = control_flow_ops.while_loop(
+          lambda x: x < 10, body, [constant_op.constant(-10.)])
+      self.assertIn("cpu", r.device.lower())
+
+    with session.Session() as sess:
+      options = config_pb2.RunOptions(output_partition_graphs=True)
+      run_metadata = config_pb2.RunMetadata()
+      sess.run(r, options=options, run_metadata=run_metadata)
+      # We expect that everything runs on CPU, even if GPU is available.
+      self.assertEqual(len(run_metadata.partition_graphs), 1)
+
   @test_util.disable_control_flow_v2("b/116338794 (buffer_reuse)")
   @test_util.run_v1_only("b/120545219")
   def testBufferForwarding(self):
-- 
GitLab


From 81b9a950453a5ce4964f2e7bb535589da3708840 Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Thu, 7 Feb 2019 09:52:45 -0800
Subject: [PATCH 2070/2345] Update select to support int8.

PiperOrigin-RevId: 232892451
---
 tensorflow/lite/kernels/register.cc          |  4 +++-
 tensorflow/lite/kernels/select.cc            |  3 +++
 tensorflow/lite/kernels/select_test.cc       | 13 +++++++++++++
 tensorflow/lite/toco/tflite/operator.cc      | 17 +++++++++++++++--
 tensorflow/lite/toco/tflite/operator_test.cc | 19 +++++++++++++++++++
 5 files changed, 53 insertions(+), 3 deletions(-)

diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index 799811dc25..66c95ffb6f 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -277,7 +277,9 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_FLOOR, Register_FLOOR());
   AddBuiltin(BuiltinOperator_CEIL, Register_CEIL());
   AddBuiltin(BuiltinOperator_NEG, Register_NEG());
-  AddBuiltin(BuiltinOperator_SELECT, Register_SELECT());
+  AddBuiltin(BuiltinOperator_SELECT, Register_SELECT(),
+             /* min_version */ 1,
+             /* max_version */ 2);
   AddBuiltin(BuiltinOperator_SLICE, Register_SLICE());
   AddBuiltin(BuiltinOperator_SIN, Register_SIN());
   AddBuiltin(BuiltinOperator_TRANSPOSE_CONV, Register_TRANSPOSE_CONV());
diff --git a/tensorflow/lite/kernels/select.cc b/tensorflow/lite/kernels/select.cc
index 4687ab4417..d1c63d887d 100644
--- a/tensorflow/lite/kernels/select.cc
+++ b/tensorflow/lite/kernels/select.cc
@@ -89,6 +89,9 @@ TfLiteStatus SelectEval(TfLiteContext* context, TfLiteNode* node) {
     case kTfLiteUInt8:                                                         \
       TF_LITE_SELECT(uint8_t, op);                                             \
       break;                                                                   \
+    case kTfLiteInt8:                                                          \
+      TF_LITE_SELECT(int8_t, op);                                              \
+      break;                                                                   \
     case kTfLiteInt16:                                                         \
       TF_LITE_SELECT(int16_t, op);                                             \
       break;                                                                   \
diff --git a/tensorflow/lite/kernels/select_test.cc b/tensorflow/lite/kernels/select_test.cc
index 5111300e47..d7cadeb51e 100644
--- a/tensorflow/lite/kernels/select_test.cc
+++ b/tensorflow/lite/kernels/select_test.cc
@@ -96,6 +96,19 @@ TEST(SelectOpTest, SelectUInt8) {
   EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 1, 4}));
 }
 
+TEST(SelectOpTest, SelectInt8) {
+  SelectOpModel model({1, 1, 1, 4}, {1, 1, 1, 4}, {1, 1, 1, 4},
+                      TensorType_INT8);
+
+  model.PopulateTensor<bool>(model.input1(), {false, true, false, false});
+  model.PopulateTensor<int8_t>(model.input2(), {1, -2, 3, 4});
+  model.PopulateTensor<int8_t>(model.input3(), {5, 6, 7, -8});
+  model.Invoke();
+
+  EXPECT_THAT(model.GetOutput<int8_t>(), ElementsAreArray({5, -2, 7, -8}));
+  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 1, 4}));
+}
+
 TEST(SelectOpTest, SelectInt16) {
   SelectOpModel model({1, 1, 1, 4}, {1, 1, 1, 4}, {1, 1, 1, 4},
                       TensorType_INT16);
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index 7aaad7f61e..773be365bc 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -2030,6 +2030,20 @@ class LessEqual : public SimpleOperator<TensorFlowLessEqualOperator> {
   }
 };
 
+class Select : public SimpleOperator<SelectOperator> {
+ public:
+  explicit Select() : SimpleOperator("SELECT", OperatorType::kSelect) {}
+  int GetVersion(const OperatorSignature& op_signature) const override {
+    const string& input_name = op_signature.op->inputs[0];
+    const Array& input_array = op_signature.model->GetArray(input_name);
+    // Version 2 supports signed int8 input types.
+    if (input_array.data_type == ArrayDataType::kInt8) {
+      return 2;
+    }
+    return 1;
+  }
+};
+
 namespace {
 // Build a vector containing all the known operators.
 std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
@@ -2216,8 +2230,7 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
   ops.push_back(MakeUnique<NotEqual>());
   ops.push_back(
       MakeUnique<SimpleOperator<NegOperator>>("NEG", OperatorType::kNeg));
-  ops.push_back(MakeUnique<SimpleOperator<SelectOperator>>(
-      "SELECT", OperatorType::kSelect));
+  ops.push_back(MakeUnique<Select>());
   ops.push_back(
       MakeUnique<SimpleOperator<SliceOperator>>("SLICE", OperatorType::kSlice));
   ops.push_back(
diff --git a/tensorflow/lite/toco/tflite/operator_test.cc b/tensorflow/lite/toco/tflite/operator_test.cc
index 13da1682d3..903da16d74 100644
--- a/tensorflow/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/lite/toco/tflite/operator_test.cc
@@ -721,6 +721,25 @@ TEST_F(OperatorTest, VersioningGreaterEqualTest) {
   VersioningTest<TensorFlowGreaterEqualOperator>();
 }
 
+TEST_F(OperatorTest, VersioningSelectTest) {
+  SelectOperator select_op;
+  select_op.inputs = {"input1"};
+  auto operator_by_type_map = BuildOperatorByTypeMap(false /*enable_flex_ops*/);
+  const BaseOperator* op = operator_by_type_map.at(select_op.type).get();
+
+  Model uint8_model;
+  Array& uint8_array = uint8_model.GetOrCreateArray(select_op.inputs[0]);
+  uint8_array.data_type = ArrayDataType::kUint8;
+  OperatorSignature uint8_signature = {.model = &uint8_model, .op = &select_op};
+  EXPECT_EQ(op->GetVersion(uint8_signature), 1);
+
+  Model int8_model;
+  Array& int8_array = int8_model.GetOrCreateArray(select_op.inputs[0]);
+  int8_array.data_type = ArrayDataType::kInt8;
+  OperatorSignature int8_signature = {.model = &int8_model, .op = &select_op};
+  EXPECT_EQ(op->GetVersion(int8_signature), 2);
+}
+
 }  // namespace
 }  // namespace tflite
 
-- 
GitLab


From ae8eddd16fab29fea2c73aa5fbdc48175d23f959 Mon Sep 17 00:00:00 2001
From: James Ring <sjr@google.com>
Date: Thu, 7 Feb 2019 10:36:05 -0800
Subject: [PATCH 2071/2345] Fix generation of bad GetAttr calls

OpKernelContext does not have GetAttr(). Previously, the
DEFINE_TF_GETATTR_ macro would define a function called
TF_OpKernelContext_GetWhateverAttr, but its implementation was
erroneously casting to OpKernelConstruction. This was never going to
work.

PiperOrigin-RevId: 232901435
---
 tensorflow/c/c_test.c        | 17 +++++++++++------
 tensorflow/c/kernels.cc      | 11 ++++-------
 tensorflow/c/kernels.h       |  9 ---------
 tensorflow/c/kernels_test.cc | 19 +++++++++----------
 4 files changed, 24 insertions(+), 32 deletions(-)

diff --git a/tensorflow/c/c_test.c b/tensorflow/c/c_test.c
index b86d8eb8e3..7468122cd5 100644
--- a/tensorflow/c/c_test.c
+++ b/tensorflow/c/c_test.c
@@ -25,6 +25,16 @@ limitations under the License.
 #include "tensorflow/c/env.h"
 #include "tensorflow/c/kernels.h"
 
+// A create function. This will never actually get called in this test, it's
+// just nice to know that it compiles.
+void* create(TF_OpKernelConstruction* ctx) {
+  TF_DataType type;
+  TF_Status* s = TF_NewStatus();
+  TF_OpKernelConstruction_GetAttrType(ctx, "foobar", &type, s);
+  TF_DeleteStatus(s);
+  return NULL;
+}
+
 // A compute function. This will never actually get called in this test, it's
 // just nice to know that it compiles.
 void compute(void* kernel, TF_OpKernelContext* ctx) {
@@ -32,12 +42,7 @@ void compute(void* kernel, TF_OpKernelContext* ctx) {
   TF_Status* s = TF_NewStatus();
   TF_GetInput(ctx, 0, &input, s);
   TF_DeleteTensor(input);
-
-  TF_DataType type;
-  TF_OpKernelContext_GetAttrType(ctx, "foobar", &type, s);
-
   TF_DeleteStatus(s);
-
 }
 
 // Exercises tensorflow's C API.
@@ -80,7 +85,7 @@ int main(int argc, char** argv) {
   TF_StringStreamDone(s);
 
   TF_KernelBuilder* b =
-      TF_NewKernelBuilder("SomeOp", "SomeDevice", NULL, &compute, NULL);
+      TF_NewKernelBuilder("SomeOp", "SomeDevice", &create, &compute, NULL);
   TF_RegisterKernelBuilder("someKernel", b, status);
 
   TF_DeleteStatus(status);
diff --git a/tensorflow/c/kernels.cc b/tensorflow/c/kernels.cc
index 9505bf9dda..71181ae430 100644
--- a/tensorflow/c/kernels.cc
+++ b/tensorflow/c/kernels.cc
@@ -173,9 +173,10 @@ void TF_OpKernelContext_Failure(TF_OpKernelContext* ctx, TF_Status* status) {
   cc_ctx->CtxFailure(s);
 }
 
-#define DEFINE_TF_GETATTR_(struct_name, func, c_type, cc_type)                 \
-  void struct_name##_GetAttr##func(struct_name* ctx, const char* attr_name,    \
-                                   c_type* val, TF_Status* status) {           \
+#define DEFINE_TF_GETATTR(func, c_type, cc_type)                               \
+  void TF_OpKernelConstruction_GetAttr##func(TF_OpKernelConstruction* ctx,     \
+                                             const char* attr_name,            \
+                                             c_type* val, TF_Status* status) { \
     TF_SetStatus(status, TF_OK, "");                                           \
     cc_type v;                                                                 \
     auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelConstruction*>(ctx); \
@@ -186,10 +187,6 @@ void TF_OpKernelContext_Failure(TF_OpKernelContext* ctx, TF_Status* status) {
     }                                                                          \
   }
 
-#define DEFINE_TF_GETATTR(func, c_type, cc_type)                     \
-  DEFINE_TF_GETATTR_(TF_OpKernelConstruction, func, c_type, cc_type) \
-  DEFINE_TF_GETATTR_(TF_OpKernelContext, func, c_type, cc_type)
-
 DEFINE_TF_GETATTR(Type, TF_DataType, tensorflow::DataType)
 
 TF_DataType TF_ExpectedOutputDataType(TF_OpKernelContext* ctx, int i) {
diff --git a/tensorflow/c/kernels.h b/tensorflow/c/kernels.h
index b015d01039..c47bfa8aa3 100644
--- a/tensorflow/c/kernels.h
+++ b/tensorflow/c/kernels.h
@@ -137,15 +137,6 @@ TF_CAPI_EXPORT extern void TF_OpKernelConstruction_GetAttrType(
     TF_OpKernelConstruction* ctx, const char* attr_name, TF_DataType* val,
     TF_Status* status);
 
-// Interprets the named kernel context attribute as a TF_DataType and places it
-// into *val. *status is set to TF_OK.
-//
-// If the attribute could not be found or could not be interpreted as
-// TF_DataType, *status is populated with an error.
-TF_CAPI_EXPORT extern void TF_OpKernelContext_GetAttrType(
-    TF_OpKernelContext* ctx, const char* attr_name, TF_DataType* val,
-    TF_Status* status);
-
 #ifdef __cplusplus
 } /* end extern "C" */
 #endif
diff --git a/tensorflow/c/kernels_test.cc b/tensorflow/c/kernels_test.cc
index 0d2954717e..608887722f 100644
--- a/tensorflow/c/kernels_test.cc
+++ b/tensorflow/c/kernels_test.cc
@@ -36,6 +36,15 @@ static void* MyCreateFunc(TF_OpKernelConstruction* ctx) {
   struct MyCustomKernel* s = new struct MyCustomKernel;
   s->created = true;
   s->compute_called = false;
+
+  // Exercise attribute reads.
+  TF_DataType type;
+  TF_Status* status = TF_NewStatus();
+  TF_OpKernelConstruction_GetAttrType(ctx, "SomeDataTypeAttr", &type, status);
+  EXPECT_EQ(TF_OK, TF_GetCode(status));
+  EXPECT_EQ(TF_FLOAT, type);
+  TF_DeleteStatus(status);
+
   return s;
 }
 
@@ -43,17 +52,7 @@ static void MyComputeFunc(void* kernel, TF_OpKernelContext* ctx) {
   struct MyCustomKernel* s = static_cast<struct MyCustomKernel*>(kernel);
   s->compute_called = true;
   if (ctx != nullptr) {
-    TF_Status* status = TF_NewStatus();
-
     EXPECT_EQ(43, TF_StepId(ctx));
-
-    // Exercise attribute reads.
-    TF_DataType type;
-    TF_OpKernelContext_GetAttrType(ctx, "SomeDataTypeAttr", &type, status);
-    EXPECT_EQ(TF_OK, TF_GetCode(status));
-    EXPECT_EQ(TF_FLOAT, type);
-
-    TF_DeleteStatus(status);
   }
 }
 
-- 
GitLab


From 345788754c025534d487bb8cd01f535b840474b4 Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Thu, 7 Feb 2019 10:37:46 -0800
Subject: [PATCH 2072/2345] Update one device strategy tests to use
 combinations. This correctly skips the GPU tests when none is available.

PiperOrigin-RevId: 232901810
---
 tensorflow/contrib/distribute/python/BUILD    |  3 +-
 .../python/one_device_strategy_test.py        | 80 ++++++++-----------
 2 files changed, 36 insertions(+), 47 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index f4c9e002c1..26ebc3dc2e 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -136,8 +136,7 @@ cuda_py_test(
     srcs = ["one_device_strategy_test.py"],
     additional_deps = [
         ":strategy_test_lib",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python/distribute:one_device_strategy",
+        ":combinations",
         "//tensorflow/python/eager:test",
     ],
 )
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy_test.py b/tensorflow/contrib/distribute/python/one_device_strategy_test.py
index 93c2447cad..0e56f663d6 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy_test.py
@@ -18,42 +18,35 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import sys
-
+from tensorflow.contrib.distribute.python import combinations
 from tensorflow.contrib.distribute.python import strategy_test_lib
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.distribute import one_device_strategy
+from tensorflow.python.eager import context
 from tensorflow.python.eager import test
-from tensorflow.python.framework import test_util
 
 
+@combinations.generate(combinations.combine(
+    distribution=[
+        combinations.one_device_strategy,
+        combinations.one_device_strategy_gpu],
+    mode=["eager", "graph"]))
 class OneDeviceStrategyTest(
     strategy_test_lib.DistributionTestBase,
     strategy_test_lib.OneDeviceDistributionTestBase):
 
-  # TODO(josh11b): Switch to using the combinations library.
-  def _get_distribution_strategy(self):
-    if "test_gpu" in sys.argv[0]:
-      return one_device_strategy.OneDeviceStrategy("/device:GPU:0")
+  def testMinimizeLoss(self, distribution):
+    if context.executing_eagerly():
+      self._test_minimize_loss_eager(distribution)
     else:
-      return one_device_strategy.OneDeviceStrategy("/device:CPU:0")
-
-  def testMinimizeLossEager(self):
-    self._test_minimize_loss_eager(self._get_distribution_strategy())
-
-  def testMinimizeLossGraph(self):
-    self._test_minimize_loss_graph(self._get_distribution_strategy())
+      self._test_minimize_loss_graph(distribution)
 
-  def testReplicaId(self):
-    self._test_replica_id(self._get_distribution_strategy())
+  def testReplicaId(self, distribution):
+    self._test_replica_id(distribution)
 
-  @test_util.run_in_graph_and_eager_modes
-  def testCallAndMergeExceptions(self):
-    self._test_call_and_merge_exceptions(self._get_distribution_strategy())
+  def testCallAndMergeExceptions(self, distribution):
+    self._test_call_and_merge_exceptions(distribution)
 
-  @test_util.run_in_graph_and_eager_modes
-  def testMakeInputFnIteratorWithDataset(self):
-    d = one_device_strategy.OneDeviceStrategy("/device:CPU:0")
+  def testMakeInputFnIteratorWithDataset(self, distribution):
     dataset_fn = lambda: dataset_ops.Dataset.range(10)
     expected_values = [[i] for i in range(10)]
     input_fn = self._input_fn_to_test_input_context(
@@ -61,13 +54,11 @@ class OneDeviceStrategyTest(
         expected_num_replicas_in_sync=1,
         expected_num_input_pipelines=1,
         expected_input_pipeline_id=0)
-    iterator = d.make_input_fn_iterator(input_fn)
+    iterator = distribution.make_input_fn_iterator(input_fn)
     self._test_input_fn_iterator(
-        iterator, d.extended.worker_devices, expected_values)
+        iterator, distribution.extended.worker_devices, expected_values)
 
-  @test_util.run_in_graph_and_eager_modes
-  def testMakeInputFnIteratorWithCallable(self):
-    d = one_device_strategy.OneDeviceStrategy("/device:CPU:0")
+  def testMakeInputFnIteratorWithCallable(self, distribution):
     def fn():
       dataset = dataset_ops.Dataset.range(10)
       it = dataset.make_one_shot_iterator()
@@ -78,32 +69,31 @@ class OneDeviceStrategyTest(
         expected_num_replicas_in_sync=1,
         expected_num_input_pipelines=1,
         expected_input_pipeline_id=0)
-    iterator = d.make_input_fn_iterator(input_fn)
+    iterator = distribution.make_input_fn_iterator(input_fn)
     self._test_input_fn_iterator(
-        iterator, d.extended.worker_devices, expected_values,
+        iterator, distribution.extended.worker_devices, expected_values,
         test_reinitialize=False)
 
-  @test_util.run_in_graph_and_eager_modes
-  def testNumpyIterator(self):
-    self._test_numpy_iterator(self._get_distribution_strategy())
+  def testNumpyIterator(self, distribution):
+    self._test_numpy_iterator(distribution)
 
-  def testAllReduceSum(self):
-    self._test_all_reduce_sum(self._get_distribution_strategy())
+  def testAllReduceSum(self, distribution):
+    self._test_all_reduce_sum(distribution)
 
-  def testAllReduceSumGradients(self):
-    self._test_all_reduce_sum_gradients(self._get_distribution_strategy())
+  def testAllReduceSumGradients(self, distribution):
+    self._test_all_reduce_sum_gradients(distribution)
 
-  def testAllReduceSumGradientTape(self):
-    self._test_all_reduce_sum_gradient_tape(self._get_distribution_strategy())
+  def testAllReduceSumGradientTape(self, distribution):
+    self._test_all_reduce_sum_gradient_tape(distribution)
 
-  def testAllReduceMean(self):
-    self._test_all_reduce_mean(self._get_distribution_strategy())
+  def testAllReduceMean(self, distribution):
+    self._test_all_reduce_mean(distribution)
 
-  def testAllReduceMeanGradients(self):
-    self._test_all_reduce_mean_gradients(self._get_distribution_strategy())
+  def testAllReduceMeanGradients(self, distribution):
+    self._test_all_reduce_mean_gradients(distribution)
 
-  def testAllReduceMeanGradientTape(self):
-    self._test_all_reduce_mean_gradient_tape(self._get_distribution_strategy())
+  def testAllReduceMeanGradientTape(self, distribution):
+    self._test_all_reduce_mean_gradient_tape(distribution)
 
 
 if __name__ == "__main__":
-- 
GitLab


From ef0badfc26fa6ddb2cd7acad5db1cde1d094dc15 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Thu, 7 Feb 2019 11:01:24 -0800
Subject: [PATCH 2073/2345] Fix bugs in cache key encoding.

Add a delimiter between the dimensions of the shape, and add a character when the sequence ends.

PiperOrigin-RevId: 232906693
---
 tensorflow/python/eager/function_test.py  | 31 ++++++++++++++++++++---
 tensorflow/python/eager/pywrap_tfe_src.cc | 18 ++++++++-----
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index 871e4e2842..0572192c86 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -1204,17 +1204,17 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
       return tuple(key[0] for key in defined._function_cache)
 
     # `True` corresponds to the fact that we're executing eagerly
-    self.assertIn(('URRR', (0, 1, 20)), cache_keys())
+    self.assertIn(('URRRu', (0, 1, 20)), cache_keys())
 
     defined(1)  # bar=1, baz=2
-    self.assertIn(('URRR', (1, 1, 2)), cache_keys())
+    self.assertIn(('URRRu', (1, 1, 2)), cache_keys())
 
     # This matches the previous call.
     defined(foo=1)
     self.assertEqual(len(defined._function_cache), 2)
 
     defined(1, 2, 3)
-    self.assertIn(('URRR', (1, 2, 3)), cache_keys())
+    self.assertIn(('URRRu', (1, 2, 3)), cache_keys())
 
     # This matches the previous call.
     defined(1, bar=2, baz=3)
@@ -1865,6 +1865,31 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
         self.assertEqual(len(maybe_add._function_cache), 3)
         self.assertEqual(len(add._function_cache), 2)
 
+  def testCacheKeyOverlappingShapes(self):
+    @function.defun
+    def defined(t):
+      return t
+
+    defined(array_ops.zeros([12, 1]))
+    self.assertLen(defined._function_cache, 1)
+
+    defined(array_ops.zeros([1, 21]))
+    self.assertLen(defined._function_cache, 2)
+
+  def testCacheKeyNestedLists(self):
+    @function.defun
+    def defined(l):
+      return l
+
+    a = constant_op.constant(1.)
+    b = constant_op.constant(2.)
+    c = constant_op.constant(3.)
+    defined([[a], b, c])
+    self.assertLen(defined._function_cache, 1)
+
+    defined([[a, b], c])
+    self.assertLen(defined._function_cache, 2)
+
   def testDecoratedMethod(self):
     m = DefunnedMiniModel()
     instance_call_one = m.call(array_ops.ones([1, 2]), training=True)
diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc
index eb2f28d8c7..6679e9cc48 100644
--- a/tensorflow/python/eager/pywrap_tfe_src.cc
+++ b/tensorflow/python/eager/pywrap_tfe_src.cc
@@ -2676,9 +2676,10 @@ PyObject* TFE_Py_FastPathExecute_C(PyObject*, PyObject* args) {
       for (Py_ssize_t j = 0; j < len; j++) {
         PyObject* py_input = PySequence_Fast_GET_ITEM(input, j);
         tensorflow::Safe_PyObjectPtr py_eager_tensor;
-        if (!ConvertToTensor(op_exec_info, py_input, &py_eager_tensor,
-                             []() { Py_RETURN_NONE; },
-                             [](const TF_DataType& dtype) {}, status)) {
+        if (!ConvertToTensor(
+                op_exec_info, py_input, &py_eager_tensor,
+                []() { Py_RETURN_NONE; }, [](const TF_DataType& dtype) {},
+                status)) {
           return nullptr;
         }
 
@@ -2823,10 +2824,13 @@ namespace {
 const char kTensor[] = "T";
 const char kIndexedSlices[] = "I";
 const char kList[] = "L";
+const char kListEnd[] = "l";
 const char kTuple[] = "U";
+const char kTupleEnd[] = "u";
 const char kDict[] = "D";
 const char kRaw[] = "R";
 const char kShape[] = "s";
+const char kShapeDelim[] = "-";
 const char kDType[] = "d";
 const char kNone[] = "n";
 
@@ -2866,7 +2870,7 @@ tensorflow::Status TFE_Py_EncodeTensor(PyObject* arg, EncodeResult* result) {
 
     absl::StrAppend(&result->str, kShape);
     for (tensorflow::int64 dim_size : tensor_shape.dim_sizes()) {
-      absl::StrAppend(&result->str, dim_size);
+      absl::StrAppend(&result->str, dim_size, kShapeDelim);
     }
 
     return tensorflow::Status::OK();
@@ -2928,6 +2932,7 @@ tensorflow::Status TFE_Py_EncodeArgHelper(PyObject* arg, EncodeResult* result);
 
 // This function doesn't set the type of sequence before
 tensorflow::Status TFE_Py_EncodeSequence(PyObject* arg, const char* type,
+                                         const char* end_type,
                                          EncodeResult* result) {
   tensorflow::Safe_PyObjectPtr arg_seq(
       PySequence_Fast(arg, "unable to create seq from list/tuple"));
@@ -2942,6 +2947,7 @@ tensorflow::Status TFE_Py_EncodeSequence(PyObject* arg, const char* type,
       TF_RETURN_IF_ERROR(TFE_Py_EncodeArgHelper(item, result));
     }
   }
+  absl::StrAppend(&result->str, end_type);
 
   return tensorflow::Status::OK();
 }
@@ -2980,9 +2986,9 @@ tensorflow::Status TFE_Py_EncodeArgHelper(PyObject* arg, EncodeResult* result) {
       TF_RETURN_IF_ERROR(TFE_Py_EncodeTensor(dense_shape.get(), result));
     }
   } else if (PyList_Check(arg)) {
-    TF_RETURN_IF_ERROR(TFE_Py_EncodeSequence(arg, kList, result));
+    TF_RETURN_IF_ERROR(TFE_Py_EncodeSequence(arg, kList, kListEnd, result));
   } else if (PyTuple_Check(arg)) {
-    TF_RETURN_IF_ERROR(TFE_Py_EncodeSequence(arg, kTuple, result));
+    TF_RETURN_IF_ERROR(TFE_Py_EncodeSequence(arg, kTuple, kTupleEnd, result));
   } else if (PyDict_Check(arg)) {
     tensorflow::Safe_PyObjectPtr keys(PyDict_Keys(arg));
     if (PyList_Sort(keys.get()) == -1) {
-- 
GitLab


From 567e02c3de3167bf0c0b986546360a9a18f467e0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 7 Feb 2019 11:05:45 -0800
Subject: [PATCH 2074/2345] Keras should not cache sessions across different
 graphs.

PiperOrigin-RevId: 232907810
---
 .../contrib/distribute/python/keras_test.py   | 22 +++++++----------
 tensorflow/python/keras/backend.py            | 24 ++++++++++++-------
 tensorflow/python/keras/backend_test.py       |  6 +++++
 3 files changed, 30 insertions(+), 22 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 6a15083470..e06363a1e8 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -235,6 +235,7 @@ def multi_input_output_model():
 strategies_minus_tpu = [
     combinations.default_strategy,
     combinations.one_device_strategy,
+    combinations.one_device_strategy_gpu,
     combinations.mirrored_strategy_with_gpu_and_cpu,
     combinations.mirrored_strategy_with_two_gpus,
     combinations.core_mirrored_strategy_with_gpu_and_cpu,
@@ -263,6 +264,7 @@ def all_strategy_combinations_minus_default():
   strategy_minus_default_combinations = combinations.combine(
       distribution=[
           combinations.one_device_strategy,
+          combinations.one_device_strategy_gpu,
           combinations.mirrored_strategy_with_gpu_and_cpu,
           combinations.mirrored_strategy_with_two_gpus,
           combinations.core_mirrored_strategy_with_gpu_and_cpu,
@@ -286,12 +288,6 @@ def strategy_and_optimizer_combinations():
       ]))
 
 
-def strategy_for_numpy_input_combinations():
-  one_gpu = combinations.one_device_strategy_gpu
-  return (all_strategy_combinations() +
-          combinations.combine(distribution=[one_gpu], mode=['graph', 'eager']))
-
-
 class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase,
                                         parameterized.TestCase):
 
@@ -447,7 +443,7 @@ class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase,
 class TestDistributionStrategyWithNumpyArrays(test.TestCase,
                                               parameterized.TestCase):
 
-  @combinations.generate(strategy_for_numpy_input_combinations())
+  @combinations.generate(all_strategy_combinations())
   def test_calculating_input_params_no_steps_no_batch_size(self, distribution):
     # Calculate the per_replica_batch_size scaling factor for strategies
     # that use per_core_batch_size
@@ -478,7 +474,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
         distributed_training_utils.get_input_params(
             distribution, input_63_samples, steps=None, batch_size=None)
 
-  @combinations.generate(strategy_for_numpy_input_combinations())
+  @combinations.generate(all_strategy_combinations())
   def test_calculating_input_params_with_steps_no_batch_size(self,
                                                              distribution):
     # Calculate the per_replica_batch_size scaling factor for strategies
@@ -524,7 +520,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
           distributed_training_utils.get_input_params(
               distribution, input_63_samples, steps=1, batch_size=None)
 
-  @combinations.generate(strategy_for_numpy_input_combinations())
+  @combinations.generate(all_strategy_combinations())
   def test_calculating_input_params_no_steps_with_batch_size(self,
                                                              distribution):
     # Calculate the per_replica_batch_size scaling factor for strategies
@@ -558,7 +554,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
         distributed_training_utils.get_input_params(
             distribution, input_64_samples, steps=None, batch_size=3)
 
-  @combinations.generate(strategy_for_numpy_input_combinations())
+  @combinations.generate(all_strategy_combinations())
   def test_calculating_input_params_with_steps_with_batch_size(self,
                                                                distribution):
     with self.cached_session():
@@ -575,7 +571,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
         distributed_training_utils.get_input_params(
             distribution, input_64_samples, steps=10, batch_size=13)
 
-  @combinations.generate(strategy_for_numpy_input_combinations())
+  @combinations.generate(all_strategy_combinations())
   def test_calling_model_with_numpy_arrays(self, distribution):
     with self.cached_session():
       with distribution.scope():
@@ -606,7 +602,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
         # with batch_size
         model.predict(inputs, batch_size=8)
 
-  @combinations.generate(strategy_for_numpy_input_combinations())
+  @combinations.generate(all_strategy_combinations())
   def test_calling_model_with_nested_numpy_arrays(self, distribution):
     with self.cached_session():
       with distribution.scope():
@@ -658,7 +654,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
       model.fit(inputs, targets, sample_weight=sample_weights, epochs=1,
                 steps_per_epoch=2, verbose=1)
 
-  @combinations.generate(strategy_for_numpy_input_combinations())
+  @combinations.generate(all_strategy_combinations())
   def test_flatten_predict_outputs(self, distribution):
     with self.cached_session():
       with distribution.scope():
diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index 37e605ea2c..c48fa9d683 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -396,15 +396,21 @@ def _get_session():
   if default_session is not None:
     session = default_session
   else:
-    if getattr(_SESSION, 'session', None) is None:
-      # We are creating the Session inside a Distribution
-      # Strategy scope.
-      if distribution_strategy_context.has_strategy():
-        configure_and_create_distributed_session(
-            distribution_strategy_context.get_strategy())
-      else:
-        _SESSION.session = session_module.Session(
-            config=get_default_session_config())
+    # We use an init_scope since want to use the outermost graph even if we are
+    # in the middle of defining a function.
+    with ops.init_scope():
+      # If we don't have a session, or that session does not match the current
+      # graph, create and cache a new session.
+      if (getattr(_SESSION, 'session', None) is None or
+          _SESSION.session.graph is not ops.get_default_graph()):
+        # If we are creating the Session inside a tf.distribute.Strategy scope,
+        # we ask the strategy for the right session options to use.
+        if distribution_strategy_context.has_strategy():
+          configure_and_create_distributed_session(
+              distribution_strategy_context.get_strategy())
+        else:
+          _SESSION.session = session_module.Session(
+              config=get_default_session_config())
     session = _SESSION.session
   return session
 
diff --git a/tensorflow/python/keras/backend_test.py b/tensorflow/python/keras/backend_test.py
index 263a1dc461..9a39d18f1e 100644
--- a/tensorflow/python/keras/backend_test.py
+++ b/tensorflow/python/keras/backend_test.py
@@ -1741,6 +1741,12 @@ class BackendGraphTests(test.TestCase):
         x, mean, var, beta, gamma, axis=1, epsilon=1e-3)
     self.assertEqual(normed.shape.as_list(), [10, 3, 5, 5])
 
+  def test_get_session_different_graphs(self):
+    with ops.Graph().as_default():
+      session = keras.backend.get_session()
+    with ops.Graph().as_default():
+      self.assertIsNot(session, keras.backend.get_session())
+
 
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From f47f02cec0ec05a823f5ac8b5fc8373d598d7f1a Mon Sep 17 00:00:00 2001
From: nammbash <niroopshankar.ammbashankar@intel.com>
Date: Thu, 7 Feb 2019 11:16:43 -0800
Subject: [PATCH 2075/2345] additional golden files for fixing

---
 .../golden/v2/tensorflow.estimator.-mode-keys.pbtxt  |  2 +-
 .../v2/tensorflow.estimator.experimental.pbtxt       | 12 ------------
 .../api/golden/v2/tensorflow.estimator.inputs.pbtxt  | 11 +++++++++++
 .../tools/api/golden/v2/tensorflow.estimator.pbtxt   |  4 ++++
 4 files changed, 16 insertions(+), 13 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.estimator.inputs.pbtxt

diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-mode-keys.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-mode-keys.pbtxt
index f3dfe7296f..bf7c1abcd8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-mode-keys.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-mode-keys.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.estimator.ModeKeys"
 tf_class {
-  is_instance: "<class \'tensorflow_estimator.python.estimator.mode_keys.ModeKeysV2\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.model_fn.ModeKeys\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "EVAL"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.pbtxt
index b1bd5a2661..f0fd7ce782 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.pbtxt
@@ -36,16 +36,4 @@ tf_module {
     name: "stop_if_higher_hook"
     argspec: "args=[\'estimator\', \'metric_name\', \'threshold\', \'eval_dir\', \'min_steps\', \'run_every_secs\', \'run_every_steps\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'60\', \'None\'], "
   }
-  member_method {
-    name: "stop_if_lower_hook"
-    argspec: "args=[\'estimator\', \'metric_name\', \'threshold\', \'eval_dir\', \'min_steps\', \'run_every_secs\', \'run_every_steps\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'60\', \'None\'], "
-  }
-  member_method {
-    name: "stop_if_no_decrease_hook"
-    argspec: "args=[\'estimator\', \'metric_name\', \'max_steps_without_decrease\', \'eval_dir\', \'min_steps\', \'run_every_secs\', \'run_every_steps\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'60\', \'None\'], "
-  }
-  member_method {
-    name: "stop_if_no_increase_hook"
-    argspec: "args=[\'estimator\', \'metric_name\', \'max_steps_without_increase\', \'eval_dir\', \'min_steps\', \'run_every_secs\', \'run_every_steps\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'60\', \'None\'], "
-  }
 }
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.inputs.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.inputs.pbtxt
new file mode 100644
index 0000000000..b318fea1f8
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.inputs.pbtxt
@@ -0,0 +1,11 @@
+path: "tensorflow.estimator.inputs"
+tf_module {
+  member_method {
+    name: "numpy_input_fn"
+    argspec: "args=[\'x\', \'y\', \'batch_size\', \'num_epochs\', \'shuffle\', \'queue_capacity\', \'num_threads\'], varargs=None, keywords=None, defaults=[\'None\', \'128\', \'1\', \'None\', \'1000\', \'1\'], "
+  }
+  member_method {
+    name: "pandas_input_fn"
+    argspec: "args=[\'x\', \'y\', \'batch_size\', \'num_epochs\', \'shuffle\', \'queue_capacity\', \'num_threads\', \'target_column\'], varargs=None, keywords=None, defaults=[\'None\', \'128\', \'1\', \'None\', \'1000\', \'1\', \'target\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.pbtxt
index add8ef5e65..6f57505afe 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.pbtxt
@@ -180,6 +180,10 @@ tf_module {
     name: "export"
     mtype: "<type \'module\'>"
   }
+  member {
+    name: "inputs"
+    mtype: "<type \'module\'>"
+  }
   member_method {
     name: "add_metrics"
     argspec: "args=[\'estimator\', \'metric_fn\'], varargs=None, keywords=None, defaults=None"
-- 
GitLab


From 1d5fd921901af8b796e9b3d9c51ccf917b7f79d1 Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Thu, 7 Feb 2019 11:06:59 -0800
Subject: [PATCH 2076/2345] Add with_path to allow modules to be flattened with
 key paths.

```
>>> class MyModule(tf.Module):
...   @property
...   def state_dict(self):
...     return dict(self._flatten(
...         predicate=lambda v: isinstance(v, tf.Variable), with_path=True))

>>> mod = MyModule()
>>> mod.encoder = Encoder()
>>> mod.decoder = mod.encoder
>>> mod.state_dict
{('encoder', 'w'), <tf.Variable: ...>,
 ('decoder', 'w'), <tf.Variable: ...>}
```

h/t tensorflow/community#56

PiperOrigin-RevId: 232908045
---
 tensorflow/python/module/module.py      | 67 +++++++++++++++++++------
 tensorflow/python/module/module_test.py | 21 +++++++-
 2 files changed, 70 insertions(+), 18 deletions(-)

diff --git a/tensorflow/python/module/module.py b/tensorflow/python/module/module.py
index 6bd275538c..13891339c1 100644
--- a/tensorflow/python/module/module.py
+++ b/tensorflow/python/module/module.py
@@ -207,7 +207,8 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
 
     Returns:
       A sequence of variables for the current module (sorted by attribute
-      name) followed by variables from all submodules recursively (depth first).
+      name) followed by variables from all submodules recursively (breadth
+      first).
     """
     return tuple(self._flatten(predicate=_IS_VARIABLE))
 
@@ -221,7 +222,8 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
 
     Returns:
       A sequence of variables for the current module (sorted by attribute
-      name) followed by variables from all submodules recursively (depth first).
+      name) followed by variables from all submodules recursively (breadth
+      first).
     """
     return tuple(self._flatten(predicate=_IS_TRAINABLE_VARIABLE))
 
@@ -249,7 +251,8 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
   def _flatten(self,
                recursive=True,
                predicate=None,
-               attribute_traversal_key=None):
+               attribute_traversal_key=None,
+               with_path=False):
     """Flattened attribute values in sorted order by attribute name.
 
     Modules are flattened by first walking their attributes in name order.
@@ -267,11 +270,15 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
     ...
     ...   @property
     ...   def tensors(self):
-    ...     return tuple(self._flatten(predicate=is_tensor))
+    ...     return tuple(self._flatten(predicate=is_tensor, with_path=True))
 
     >>> foo = Foo()
     >>> foo.tensors
-    (<tf.Tensor...'a'>, <tf.Tensor...'b'>, ...'c'>, ...'d'>, ...'e'>)
+    ((('x', 0),   <tf.Tensor: ...'a'>),
+     (('x', 1),   <tf.Tensor: ...'b'>),
+     (('y', 'i'), <tf.Tensor: ...'c'>),
+     (('y', 'j'), <tf.Tensor: ...'d'>),
+     (('z',),     <tf.Tensor: ...'e'>))
 
     `attribute_traversal_key` controls the order object properties are visited.
     If not set objects are visited in ascending order by name.
@@ -284,6 +291,10 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
       attribute_traversal_key: (Optional) Method to rekey object attributes
         before they are sorted. Contract is the same as `key` argument to
         builtin `sorted` and only applies to object properties.
+      with_path: (Optional) Whether to include the path to the object as well
+        as the object itself. If `with_path` is `True` then leaves will not be
+        de-duplicated (e.g. if the same leaf instance is reachable via multiple
+        modules then it will be yielded multiple times with different paths).
 
     Returns:
       Flat generator for leaves of the current module and optionally all
@@ -297,7 +308,7 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
         recursive=recursive,
         predicate=predicate,
         attribute_traversal_key=attribute_traversal_key,
-        seen=set())
+        with_path=with_path)
 
   @classmethod
   def no_name_scope(cls, method):
@@ -337,8 +348,20 @@ def camel_to_snake(value):
   return _CAMEL_TO_SNAKE_R.sub(r"_\1", value).lower()
 
 
-def _flatten_module(module, recursive, predicate, attribute_traversal_key,
-                    seen):
+# AutoCheckpointable adds object attributes that users will not expect us to
+# include when flattening (these reference dependencies reachable via other
+# object attributes).
+AUTO_CHECKPOINTABLE_ATTRS = ("_unconditional_checkpoint_dependencies",
+                             "_unconditional_dependency_names")
+
+
+def _flatten_module(module,
+                    recursive,
+                    predicate,
+                    attribute_traversal_key,
+                    with_path,
+                    module_path=(),
+                    seen=None):
   """Implementation of `flatten`."""
   if seen is None:
     seen = set([id(module)])
@@ -347,25 +370,37 @@ def _flatten_module(module, recursive, predicate, attribute_traversal_key,
   submodules = []
 
   for key in sorted(module_dict, key=attribute_traversal_key):
-    for leaf in nest.flatten(module_dict[key]):
-      leaf_id = id(leaf)
-      if leaf_id in seen:
-        continue
+    if key in AUTO_CHECKPOINTABLE_ATTRS:
+      continue
+
+    for leaf_path, leaf in nest.flatten_with_tuple_paths(module_dict[key]):
+      leaf_path = (key,) + leaf_path
+
+      # TODO(tomhennigan) Handle cycles for `with_path=True` (e.g. `a.a = a`).
+      if not with_path:
+        leaf_id = id(leaf)
+        if leaf_id in seen:
+          continue
+        seen.add(leaf_id)
 
-      seen.add(leaf_id)
       if predicate(leaf):
-        yield leaf
+        if with_path:
+          yield module_path + leaf_path, leaf
+        else:
+          yield leaf
 
       if recursive and isinstance(leaf, Module):
         # Walk direct properties first then recurse.
-        submodules.append(leaf)
+        submodules.append((module_path + leaf_path, leaf))
 
-  for submodule in submodules:
+  for submodule_path, submodule in submodules:
     subvalues = _flatten_module(
         submodule,
         recursive=recursive,
         predicate=predicate,
         attribute_traversal_key=attribute_traversal_key,
+        with_path=with_path,
+        module_path=submodule_path,
         seen=seen)
 
     for subvalue in subvalues:
diff --git a/tensorflow/python/module/module_test.py b/tensorflow/python/module/module_test.py
index da9bcf7268..08a63537e3 100644
--- a/tensorflow/python/module/module_test.py
+++ b/tensorflow/python/module/module_test.py
@@ -299,10 +299,10 @@ NamedPair = collections.namedtuple("NamedPair", ("first", "second"))
 mk_index_dict = lambda v: dict(enumerate(v))
 
 
-class WalkTest(parameterized.TestCase, test.TestCase):
+class FlattenTest(parameterized.TestCase, test.TestCase):
 
   @parameterized.parameters(lambda v: NamedPair(*v), list, tuple, mk_index_dict)
-  def test_walk(self, container_type):
+  def test_flatten(self, container_type):
     parent = SimpleModule(container_type=container_type)
     child = parent.c
 
@@ -320,6 +320,23 @@ class WalkTest(parameterized.TestCase, test.TestCase):
         mod.variables,
         mod._trainable_variables + mod._non_trainable_variables + [mod._bonus])
 
+  def test_with_path(self):
+    mod = module.Module()
+    mod.w = variables.Variable(1.)
+    mod.encoder = module.Module()
+    mod.encoder.w = [({"k": mod.w}, {"k": mod.w})]
+    mod.decoder = mod.encoder
+
+    state_dict = dict(
+        mod._flatten(with_path=True, predicate=module._IS_VARIABLE))
+
+    self.assertEqual(state_dict,
+                     {("w",): mod.w,
+                      ("encoder", "w", 0, 0, "k"): mod.encoder.w[0][0]["k"],
+                      ("encoder", "w", 0, 1, "k"): mod.encoder.w[0][1]["k"],
+                      ("decoder", "w", 0, 0, "k"): mod.decoder.w[0][0]["k"],
+                      ("decoder", "w", 0, 1, "k"): mod.decoder.w[0][1]["k"]},)
+
 
 class LayerModule(module.Module):
 
-- 
GitLab


From 7d659b81fbe57d121f840519a89af5b8a56eb001 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Thu, 7 Feb 2019 11:20:36 -0800
Subject: [PATCH 2077/2345] Disable flaky test.

PiperOrigin-RevId: 232910820
---
 .../seq2seq/python/kernel_tests/basic_decoder_v2_test.py        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/basic_decoder_v2_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/basic_decoder_v2_test.py
index 15fb688fc4..2341ebb77a 100644
--- a/tensorflow/contrib/seq2seq/python/kernel_tests/basic_decoder_v2_test.py
+++ b/tensorflow/contrib/seq2seq/python/kernel_tests/basic_decoder_v2_test.py
@@ -127,7 +127,7 @@ class BasicDecoderTest(keras_parameterized.TestCase):
           np.argmax(eval_result["step_outputs"].rnn_output, -1),
           eval_result["step_outputs"].sample_id)
 
-  def testStepWithGreedyEmbeddingHelper(self):
+  def DISABLED_testStepWithGreedyEmbeddingHelper(self):
     batch_size = 5
     vocabulary_size = 7
     cell_depth = vocabulary_size  # cell's logits must match vocabulary size
-- 
GitLab


From 4129fff6b4e1399be2f61f512560409ee1db50fe Mon Sep 17 00:00:00 2001
From: Tong Shen <endlessroad@google.com>
Date: Thu, 7 Feb 2019 11:41:14 -0800
Subject: [PATCH 2078/2345] Ensure copied const node has unique node name.

PiperOrigin-RevId: 232915090
---
 tensorflow/compiler/tf2xla/tf2xla_util.cc     |  7 ++-
 .../compiler/tf2xla/tf2xla_util_test.cc       | 53 +++++++++++++++++++
 2 files changed, 59 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/tf2xla/tf2xla_util.cc b/tensorflow/compiler/tf2xla/tf2xla_util.cc
index c64f78e1a1..88c03a6056 100644
--- a/tensorflow/compiler/tf2xla/tf2xla_util.cc
+++ b/tensorflow/compiler/tf2xla/tf2xla_util.cc
@@ -122,7 +122,12 @@ Status ReplaceArgUsageWithConstNode(
 
   for (const auto& iter : const_input_index_to_node) {
     int arg_index = iter.first;
-    Node* const_node = g->CopyNode(iter.second);
+    NodeDef const_def = iter.second->def();
+    const_def.set_name(g->NewName(const_def.name()));
+    Status s;
+    Node* const_node = g->AddNode(const_def, &s);
+    TF_RETURN_IF_ERROR(s);
+
     Node* arg_node = arg_nodes[arg_index];
 
     // Collect all usages of the _Arg node.
diff --git a/tensorflow/compiler/tf2xla/tf2xla_util_test.cc b/tensorflow/compiler/tf2xla/tf2xla_util_test.cc
index 9e9c3cecee..28b4744470 100644
--- a/tensorflow/compiler/tf2xla/tf2xla_util_test.cc
+++ b/tensorflow/compiler/tf2xla/tf2xla_util_test.cc
@@ -363,5 +363,58 @@ TEST(PropagateConstIntoFunctionalNodes, WhileLoopWithResourceInput) {
   TF_EXPECT_OK(PropagateConstIntoFunctionalNodes(&graph, &fld, &fld));
 }
 
+TEST(PropagateConstIntoFunctionalNodes, CopiedConstNodeHasUniqueName) {
+  FunctionLibraryDefinition fld(OpRegistry::Global(), {});
+  {
+    // Cond graph & body graph.
+    Scope scope = Scope::NewRootScope().ExitOnError();
+    auto pred = ops::_Arg(scope.WithOpName("arg0"), DT_BOOL, 0);
+    auto input = ops::_Arg(scope.WithOpName("arg1"), DT_BOOL, 1);
+    auto duplicate_name = ops::NoOp(scope.WithOpName("duplicate_name"));
+    auto ret = ops::_Retval(scope.WithOpName("ret"), pred, 0);
+    Graph graph(OpRegistry::Global());
+    TF_ASSERT_OK(scope.ToGraph(&graph));
+    FunctionDef cond_fdef;
+    TF_ASSERT_OK(GraphToFunctionDef(graph, "cond", &cond_fdef));
+    TF_ASSERT_OK(fld.AddFunctionDef(cond_fdef));
+    FunctionDef body_fdef;
+    TF_ASSERT_OK(GraphToFunctionDef(graph, "body", &body_fdef));
+    TF_ASSERT_OK(fld.AddFunctionDef(body_fdef));
+  }
+  Scope scope = Scope::NewRootScope().ExitOnError();
+  auto pred =
+      ops::Const(scope.WithOpName("duplicate_name"), false, TensorShape({}));
+  auto input = ops::Const(scope.WithOpName("input"), false, TensorShape({}));
+  NameAttrList cond_fn, body_fn;
+  cond_fn.set_name("cond");
+  body_fn.set_name("body");
+  auto while_op =
+      ops::While(scope.WithOpName("while"),
+                 std::initializer_list<Input>{pred, input}, cond_fn, body_fn);
+  Graph graph(OpRegistry::Global());
+  TF_ASSERT_OK(scope.ToGraph(&graph));
+
+  TF_EXPECT_OK(PropagateConstIntoFunctionalNodes(&graph, &fld, &fld));
+
+  // Check that in rewritten body function, the NoOp node still has name
+  // "duplicate_name", and the copied Const node has name "duplicate_name/_0".
+  auto node_name_index = graph.BuildNodeNameIndex();
+  Node* while_node = node_name_index["while"];
+  ASSERT_NE(while_node, nullptr);
+  TF_ASSERT_OK(GetNodeAttr(while_node->def(), "body", &body_fn));
+  const FunctionDef* rewritten_body_fn = fld.Find(body_fn.name());
+  ASSERT_NE(rewritten_body_fn, nullptr);
+  std::unordered_map<string, NodeDef> nodes;
+  for (const NodeDef& node_def : rewritten_body_fn->node_def()) {
+    nodes[node_def.name()] = node_def;
+  }
+  auto noop_def = nodes.find("duplicate_name");
+  ASSERT_NE(noop_def, nodes.end());
+  EXPECT_EQ(noop_def->second.op(), "NoOp");
+  auto const_def = nodes.find("duplicate_name/_0");
+  ASSERT_NE(const_def, nodes.end());
+  EXPECT_EQ(const_def->second.op(), "Const");
+}
+
 }  // namespace
 }  // namespace tensorflow
-- 
GitLab


From 2cb71d047330780e33b0b4b80a26f2cdd405ddf7 Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Thu, 7 Feb 2019 11:44:41 -0800
Subject: [PATCH 2079/2345] [XLA] Add routine
 HloModule::AddComputationAndUnifyNamesAndIds.

This routine clears the ID for the computation and the instructions inside the
computation and calls routine AddComputationInternal for adding the computation
and sanitizing the names and IDs for the computation and the instruction inside
the computation.

PiperOrigin-RevId: 232915664
---
 tensorflow/compiler/xla/service/hlo_computation.h |  4 ++++
 tensorflow/compiler/xla/service/hlo_instruction.h |  4 ++++
 tensorflow/compiler/xla/service/hlo_module.h      | 10 ++++++++++
 3 files changed, 18 insertions(+)

diff --git a/tensorflow/compiler/xla/service/hlo_computation.h b/tensorflow/compiler/xla/service/hlo_computation.h
index 0cb9caddd0..fd1f990431 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.h
+++ b/tensorflow/compiler/xla/service/hlo_computation.h
@@ -391,6 +391,10 @@ class HloComputation {
     fusion_instruction_ = fusion_instruction;
   }
 
+  // Clear the unique ID of the computation so that it can be re-assigned, such
+  // as for the purpose of compacting the unique IDs.
+  void ClearUniqueIdInternal() { unique_id_ = -1; }
+
   // The id of this computation should be unique within the module.
   void SetUniqueId(int64 id) {
     CHECK_EQ(unique_id_, -1);
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h
index 79c6c85f72..2ef72a8705 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.h
+++ b/tensorflow/compiler/xla/service/hlo_instruction.h
@@ -1251,6 +1251,10 @@ class HloInstruction {
   // on the instruction's existing name.
   void UniquifyName(NameUniquer* name_uniquer);
 
+  // Clear the unique ID of the instruction so that it can be re-assigned, such
+  // as for the purpose of compacting the instruction unique IDs.
+  void ClearUniqueIdInternal() { unique_id_ = -1; }
+
   // Set the unique id for this instruction to "id"
   void SetUniqueId(int id) {
     CHECK_EQ(unique_id_, -1);  // Should not be assigned already
diff --git a/tensorflow/compiler/xla/service/hlo_module.h b/tensorflow/compiler/xla/service/hlo_module.h
index 8a2a0c8da2..b6fe6a5cdb 100644
--- a/tensorflow/compiler/xla/service/hlo_module.h
+++ b/tensorflow/compiler/xla/service/hlo_module.h
@@ -265,6 +265,16 @@ class HloModule {
   const HloSchedule& schedule() const { return *schedule_; }
   HloSchedule& schedule() { return *schedule_; }
 
+  HloComputation* AddComputationAndUnifyNamesAndIds(
+      std::unique_ptr<HloComputation> computation, bool is_entry) {
+    computation->ClearUniqueIdInternal();
+    for (auto* instruction : computation->instructions()) {
+      instruction->ClearUniqueIdInternal();
+    }
+    return AddComputationInternal(std::move(computation), is_entry,
+                                  /*uniquify_identifiers=*/true);
+  }
+
   Status CheckUniqueNamesAndIdsForComputationsAndInstructions() const;
 
  private:
-- 
GitLab


From 45a97891f26621e75754b08d01f75eff273554cb Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Thu, 7 Feb 2019 11:47:05 -0800
Subject: [PATCH 2080/2345] Expose API for passing delegates to the TfLite
 benchmark model, to be able to benchmark performance with delegates.

PiperOrigin-RevId: 232916082
---
 .../lite/tools/benchmark/benchmark_tflite_model.cc | 14 +++++++++++++-
 .../lite/tools/benchmark/benchmark_tflite_model.h  |  6 +++++-
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc b/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc
index 0bc7565e82..83e0ff1f87 100644
--- a/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc
+++ b/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc
@@ -356,11 +356,23 @@ void BenchmarkTfLiteModel::Init() {
     }
   }
 
-  if (interpreter->AllocateTensors() != kTfLiteOk) {
+  // Don't allocate tensors if we have delegates.
+  if (delegates_.empty() && interpreter->AllocateTensors() != kTfLiteOk) {
     TFLITE_LOG(FATAL) << "Failed to allocate tensors!";
   }
 }
 
+void BenchmarkTfLiteModel::ApplyDelegates() {
+  for (int i = 0; i < delegates_.size(); ++i) {
+    if (interpreter->ModifyGraphWithDelegate(delegates_[i].get()) !=
+        kTfLiteOk) {
+      TFLITE_LOG(FATAL) << "Failed to apply delegate # " << i;
+    } else {
+      TFLITE_LOG(INFO) << "Applied Delegate # " << i;
+    }
+  }
+}
+
 void BenchmarkTfLiteModel::RunImpl() {
   if (interpreter->Invoke() != kTfLiteOk) {
     TFLITE_LOG(FATAL) << "Failed to invoke!";
diff --git a/tensorflow/lite/tools/benchmark/benchmark_tflite_model.h b/tensorflow/lite/tools/benchmark/benchmark_tflite_model.h
index 83599e644d..3532d2a5dd 100644
--- a/tensorflow/lite/tools/benchmark/benchmark_tflite_model.h
+++ b/tensorflow/lite/tools/benchmark/benchmark_tflite_model.h
@@ -70,6 +70,9 @@ class BenchmarkTfLiteModel : public BenchmarkModel {
   uint64_t ComputeInputBytes() override;
   void Init() override;
   void RunImpl() override;
+  void SetDelegates(std::vector<std::unique_ptr<TfLiteDelegate>> delegates) {
+    delegates_ = std::move(delegates);
+  }
 
   struct InputLayerInfo {
     std::string name;
@@ -81,7 +84,7 @@ class BenchmarkTfLiteModel : public BenchmarkModel {
   void PrepareInputsAndOutputs() override;
 
   // Allows installation of custom delegates during initialization
-  virtual void ApplyDelegates() {}
+  virtual void ApplyDelegates();
 
   std::unique_ptr<tflite::FlatBufferModel> model;
   std::unique_ptr<tflite::Interpreter> interpreter;
@@ -90,6 +93,7 @@ class BenchmarkTfLiteModel : public BenchmarkModel {
   std::vector<InputLayerInfo> inputs;
   ProfilingListener profiling_listener_;
   GemmlowpProfilingListener gemmlowp_profiling_listener_;
+  std::vector<std::unique_ptr<TfLiteDelegate>> delegates_;
 };
 
 }  // namespace benchmark
-- 
GitLab


From e8a82ff6b0d1d2770a4f24cb5bdc0d25d2fa80c1 Mon Sep 17 00:00:00 2001
From: Blake Hechtman <blakehechtman@google.com>
Date: Thu, 7 Feb 2019 12:08:44 -0800
Subject: [PATCH 2081/2345] [XLA] Support scalar dots and dots without
 contracting dimensions general with a multiply.

PiperOrigin-RevId: 232920143
---
 tensorflow/compiler/xla/client/xla_builder.cc | 12 ++++++
 .../xla/service/algebraic_simplifier.cc       | 38 +++++++++++++++++--
 .../compiler/xla/service/dot_decomposer.cc    |  2 +-
 .../compiler/xla/tests/dot_operation_test.cc  |  6 +++
 4 files changed, 54 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc
index b64d352e2c..29b71bea5d 100644
--- a/tensorflow/compiler/xla/client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_builder.cc
@@ -1014,6 +1014,18 @@ XlaOp XlaBuilder::DotGeneral(const XlaOp& lhs, const XlaOp& rhs,
     HloInstructionProto instr;
     TF_ASSIGN_OR_RETURN(const Shape& lhs_shape, GetShape(lhs));
     TF_ASSIGN_OR_RETURN(const Shape& rhs_shape, GetShape(rhs));
+    // If one operand is a scalar, just multiply the two operands.
+    if (ShapeUtil::IsScalar(lhs_shape) || ShapeUtil::IsScalar(rhs_shape)) {
+      if (dimension_numbers.rhs_batch_dimensions_size() != 0 ||
+          dimension_numbers.lhs_batch_dimensions_size() != 0 ||
+          dimension_numbers.rhs_contracting_dimensions_size() != 0 ||
+          dimension_numbers.lhs_contracting_dimensions_size() != 0) {
+        return InvalidArgument(
+            "Dots with scalar operands must have no contracting or batch "
+            "dimensions");
+      }
+      return xla::Mul(lhs, rhs);
+    }
     TF_ASSIGN_OR_RETURN(Shape shape,
                         ShapeInference::InferDotOpShape(lhs_shape, rhs_shape,
                                                         dimension_numbers));
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index acc2c284b2..51d924f34d 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -1090,7 +1090,7 @@ StatusOr<bool> AlgebraicSimplifierVisitor::HandleDotStrengthReduction(
   const int64 rhs_rank = rhs->shape().rank();
   const int64 lhs_rank = lhs->shape().rank();
   const auto& dnums = dot->dot_dimension_numbers();
-  if (dnums.rhs_contracting_dimensions_size() > 1) {
+  if (dnums.rhs_contracting_dimensions_size() != 1) {
     return false;
   }
   if (dot_rank > 2 && (lhs_rank != rhs_rank || lhs_rank != dot_rank)) {
@@ -1585,7 +1585,9 @@ StatusOr<HloInstruction*> AlgebraicSimplifierVisitor::OptimizeDotOfGather(
 Status AlgebraicSimplifierVisitor::HandleDot(HloInstruction* dot) {
   HloInstruction *lhs, *rhs;
   CHECK(Match(dot, m::Dot(m::Op(&lhs), m::Op(&rhs))));
-
+  if (options_.is_layout_sensitive()) {
+    return Status::OK();
+  }
   // Replace a zero element dot with a broadcast of the constant 0.
   if (ShapeUtil::IsZeroElementArray(dot->shape()) ||
       ShapeUtil::IsZeroElementArray(lhs->shape()) ||
@@ -1602,6 +1604,32 @@ Status AlgebraicSimplifierVisitor::HandleDot(HloInstruction* dot) {
       dot->shape().element_type() != BF16) {
     return Status::OK();
   }
+
+  // If a dot only contains batch dimensions, then tranpose the rhs and lhs
+  // acording to the batch dimension numbers and do a simple multiply.
+  if (lhs->shape().rank() ==
+          dot->dot_dimension_numbers().lhs_batch_dimensions_size() &&
+      rhs->shape().rank() ==
+          dot->dot_dimension_numbers().rhs_batch_dimensions_size()) {
+    HloInstruction* new_rhs = rhs;
+    HloInstruction* new_lhs = lhs;
+    if (lhs->shape().rank() > 1) {
+      TF_ASSIGN_OR_RETURN(
+          new_rhs,
+          MakeTransposeHlo(
+              rhs, AsInt64Slice(
+                       dot->dot_dimension_numbers().rhs_batch_dimensions())));
+      TF_ASSIGN_OR_RETURN(
+          new_lhs,
+          MakeTransposeHlo(
+              lhs, AsInt64Slice(
+                       dot->dot_dimension_numbers().lhs_batch_dimensions())));
+    }
+    TF_ASSIGN_OR_RETURN(auto new_dot,
+                        MakeBinaryHlo(HloOpcode::kMultiply, new_lhs, new_rhs));
+    return ReplaceInstruction(dot, new_dot);
+  }
+
   if (lhs->shape().rank() > 2 || rhs->shape().rank() > 2 ||
       dot->shape().rank() > 2) {
     if (options_.enable_dot_strength_reduction() &&
@@ -1640,7 +1668,11 @@ Status AlgebraicSimplifierVisitor::HandleDot(HloInstruction* dot) {
   }
 
   // Simplify dot(transpose(a), transpose(b)) to transpose(dot(b,a)).
-  if (lhs->IsRank2Transpose() && rhs->IsRank2Transpose()) {
+  if (dot->dot_dimension_numbers().lhs_batch_dimensions_size() == 0 &&
+      dot->dot_dimension_numbers().lhs_contracting_dimensions_size() == 1 &&
+      dot->dot_dimension_numbers().lhs_contracting_dimensions(0) == 1 &&
+      dot->dot_dimension_numbers().rhs_contracting_dimensions(0) == 0 &&
+      lhs->IsRank2Transpose() && rhs->IsRank2Transpose()) {
     DotDimensionNumbers dot_dimension_numbers;
     dot_dimension_numbers.add_lhs_contracting_dimensions(1);
     dot_dimension_numbers.add_rhs_contracting_dimensions(0);
diff --git a/tensorflow/compiler/xla/service/dot_decomposer.cc b/tensorflow/compiler/xla/service/dot_decomposer.cc
index e8bc6d0571..559b9c1f2c 100644
--- a/tensorflow/compiler/xla/service/dot_decomposer.cc
+++ b/tensorflow/compiler/xla/service/dot_decomposer.cc
@@ -297,7 +297,7 @@ StatusOr<bool> DotDecomposer::Run(HloModule* module) {
       const DotDimensionNumbers& dnums = instruction->dot_dimension_numbers();
       // A dot it not canonical if there are more than one contracting
       // dimension.
-      if (dnums.lhs_contracting_dimensions_size() > 1) {
+      if (dnums.lhs_contracting_dimensions_size() != 1) {
         non_canonical_dots.push_back(instruction);
         continue;
       }
diff --git a/tensorflow/compiler/xla/tests/dot_operation_test.cc b/tensorflow/compiler/xla/tests/dot_operation_test.cc
index f740f48158..2e35805c6a 100644
--- a/tensorflow/compiler/xla/tests/dot_operation_test.cc
+++ b/tensorflow/compiler/xla/tests/dot_operation_test.cc
@@ -1188,6 +1188,12 @@ std::vector<EinsumParamType> GetEinsumTestCases() {
       p{v{8, 55, 11, 3}, v{55, 11, 3, 29}, "mkBC,kBCn->BCnm"},
       p{v{5, 6}, v{6, 7}, "ab,cd->dcba"},
       p{v{6}, v{6, 7}, "b,bc->c"},
+      p{v{77}, v{77}, "a,a->a"},
+      p{v{55}, v{}, "a,->a"},
+      p{v{11, 111}, v{11}, "ab,a->ab"},
+      p{v{16, 34}, v{16, 34}, "ab,ab->ab"},
+      p{v{16, 3, 34}, v{3, 16, 34}, "abc,bac->abc"},
+      p{v{5, 19}, v{}, "ab,->ab"},
   };
   return test_cases;
 }
-- 
GitLab


From 5b7063a3cfdc7710721924c5b4350c8265d5ff2d Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Thu, 7 Feb 2019 12:22:29 -0800
Subject: [PATCH 2082/2345] Create TensorBoardV2 to simplify implementation and
 support more functionality in V2.

PiperOrigin-RevId: 232922346
---
 tensorflow/python/keras/BUILD                 |  27 +
 tensorflow/python/keras/__init__.py           |   1 +
 tensorflow/python/keras/callbacks.py          | 448 ++++--------
 tensorflow/python/keras/callbacks_test.py     | 660 +++++-------------
 tensorflow/python/keras/callbacks_v1.py       | 424 +++++++++++
 tensorflow/python/keras/callbacks_v1_test.py  | 562 +++++++++++++++
 .../engine/distributed_training_utils.py      |   4 +-
 ...orflow.keras.callbacks.-tensor-board.pbtxt |   2 +-
 ...orflow.keras.callbacks.-tensor-board.pbtxt |   2 +-
 9 files changed, 1296 insertions(+), 834 deletions(-)
 create mode 100644 tensorflow/python/keras/callbacks_v1.py
 create mode 100644 tensorflow/python/keras/callbacks_v1_test.py

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index d904193319..01603e070b 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -156,6 +156,7 @@ py_library(
         ":activations",
         ":backend",
         ":callbacks",
+        ":callbacks_v1",
         ":constraints",
         ":engine_utils",
         ":initializers",
@@ -220,6 +221,18 @@ py_library(
     ],
 )
 
+py_library(
+    name = "callbacks_v1",
+    srcs = [
+        "callbacks_v1.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":backend",
+        ":engine_utils",
+    ],
+)
+
 py_library(
     name = "constraints",
     srcs = [
@@ -1007,6 +1020,20 @@ tf_py_test(
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
     ],
+    shard_count = 4,
+    tags = ["notsan"],
+)
+
+tf_py_test(
+    name = "callbacks_v1_test",
+    size = "medium",
+    srcs = ["callbacks_v1_test.py"],
+    additional_deps = [
+        ":keras",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+    ],
     tags = ["notsan"],
 )
 
diff --git a/tensorflow/python/keras/__init__.py b/tensorflow/python/keras/__init__.py
index f024b9b59a..b7ec63837d 100644
--- a/tensorflow/python/keras/__init__.py
+++ b/tensorflow/python/keras/__init__.py
@@ -25,6 +25,7 @@ from tensorflow.python.keras import activations
 from tensorflow.python.keras import applications
 from tensorflow.python.keras import backend
 from tensorflow.python.keras import callbacks
+from tensorflow.python.keras import callbacks_v1
 from tensorflow.python.keras import constraints
 from tensorflow.python.keras import datasets
 from tensorflow.python.keras import estimator
diff --git a/tensorflow/python/keras/callbacks.py b/tensorflow/python/keras/callbacks.py
index 41fedbb31f..c5ae62c2f5 100644
--- a/tensorflow/python/keras/callbacks.py
+++ b/tensorflow/python/keras/callbacks.py
@@ -32,17 +32,13 @@ import six
 
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.eager import context
-from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras.utils.data_utils import Sequence
 from tensorflow.python.keras.utils.generic_utils import Progbar
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import summary_ops_v2
-from tensorflow.python.ops import variables
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.summary import summary as tf_summary
-from tensorflow.python.training import saver
 from tensorflow.python.training.mode_keys import ModeKeys
 from tensorflow.python.util.tf_export import keras_export
 
@@ -1054,10 +1050,10 @@ class LearningRateScheduler(Callback):
     logs['lr'] = K.get_value(self.model.optimizer.lr)
 
 
-@keras_export('keras.callbacks.TensorBoard')
+@keras_export('keras.callbacks.TensorBoard', v1=[])
 class TensorBoard(Callback):
   # pylint: disable=line-too-long
-  """Tensorboard basic visualizations.
+  """TensorBoard basic visualizations.
 
   This callback writes a log for TensorBoard, which allows
   you to visualize dynamic graphs of your training and test
@@ -1077,51 +1073,25 @@ class TensorBoard(Callback):
   [here](https://www.tensorflow.org/get_started/summaries_and_tensorboard).
 
   Arguments:
-      log_dir: the path of the directory where to save the log
-          files to be parsed by TensorBoard.
-      histogram_freq: frequency (in epochs) at which to compute activation
-          and weight histograms for the layers of the model. If set to 0,
-          histograms won't be computed. Validation data (or split) must be
-          specified for histogram visualizations.
-      write_graph: whether to visualize the graph in TensorBoard.
-          The log file can become quite large when
-          write_graph is set to True.
-      write_grads: whether to visualize gradient histograms in TensorBoard.
-          `histogram_freq` must be greater than 0.
-      batch_size: size of batch of inputs to feed to the network
-          for histograms computation.
-      write_images: whether to write model weights to visualize as
-          image in TensorBoard.
-      embeddings_freq: frequency (in epochs) at which selected embedding
-          layers will be saved. If set to 0, embeddings won't be computed.
-          Data to be visualized in TensorBoard's Embedding tab must be passed
-          as `embeddings_data`.
-      embeddings_layer_names: a list of names of layers to keep eye on. If
-          None or empty list all the embedding layer will be watched.
-      embeddings_metadata: a dictionary which maps layer name to a file name
-          in which metadata for this embedding layer is saved. See the
-          [details](https://www.tensorflow.org/how_tos/embedding_viz/#metadata_optional)
-          about metadata files format. In case if the same metadata file is
-          used for all embedding layers, string can be passed.
-      embeddings_data: data to be embedded at layers specified in
-          `embeddings_layer_names`. Numpy array (if the model has a single
-          input) or list of Numpy arrays (if the model has multiple inputs).
-          Learn [more about embeddings](https://www.tensorflow.org/programmers_guide/embedding)
+      log_dir: the path of the directory where to save the log files to be
+        parsed by TensorBoard.
+      histogram_freq: frequency (in epochs) at which to compute activation and
+        weight histograms for the layers of the model. If set to 0, histograms
+        won't be computed. Validation data (or split) must be specified for
+        histogram visualizations.
+      write_graph: whether to visualize the graph in TensorBoard. The log file
+        can become quite large when write_graph is set to True.
+      write_images: whether to write model weights to visualize as image in
+        TensorBoard.
       update_freq: `'batch'` or `'epoch'` or integer. When using `'batch'`,
-          writes the losses and metrics to TensorBoard after each batch.
-          The same applies for `'epoch'`. If using an integer, let's say `1000`,
-          the callback will write the metrics and losses to TensorBoard every
-          1000 samples. Note that writing too frequently to TensorBoard
-          can slow down your training.
+        writes the losses and metrics to TensorBoard after each batch. The same
+        applies for `'epoch'`. If using an integer, let's say `1000`, the
+        callback will write the metrics and losses to TensorBoard every 1000
+        samples. Note that writing too frequently to TensorBoard can slow down
+        your training.
 
   Raises:
       ValueError: If histogram_freq is set and no validation data is provided.
-
-  @compatibility(eager)
-  Using the `TensorBoard` callback will work when eager execution is enabled,
-  with the restriction that outputting histogram summaries of weights and
-  gradients is not supported. Consequently, `histogram_freq` will be ignored.
-  @end_compatibility
   """
 
   # pylint: enable=line-too-long
@@ -1129,316 +1099,142 @@ class TensorBoard(Callback):
   def __init__(self,
                log_dir='./logs',
                histogram_freq=0,
-               batch_size=32,
                write_graph=True,
-               write_grads=False,
                write_images=False,
-               embeddings_freq=0,
-               embeddings_layer_names=None,
-               embeddings_metadata=None,
-               embeddings_data=None,
-               update_freq='epoch'):
+               update_freq='epoch',
+               **kwargs):
     super(TensorBoard, self).__init__()
+    self._validate_kwargs(kwargs)
+
     self.log_dir = log_dir
     self.histogram_freq = histogram_freq
-    if self.histogram_freq and context.executing_eagerly():
-      logging.warning(
-          UserWarning('Weight and gradient histograms not supported for eager'
-                      'execution, setting `histogram_freq` to `0`.'))
-      self.histogram_freq = 0
-    self.merged = None
     self.write_graph = write_graph
-    self.write_grads = write_grads
     self.write_images = write_images
-    self.batch_size = batch_size
-    self._current_batch = 0
-    self._total_batches_seen = 0
-    self._total_val_batches_seen = 0
-    self.embeddings_freq = embeddings_freq
-    self.embeddings_layer_names = embeddings_layer_names
-    self.embeddings_metadata = embeddings_metadata
-    self.embeddings_data = embeddings_data
     if update_freq == 'batch':
       self.update_freq = 1
     else:
       self.update_freq = update_freq
+
     self._samples_seen = 0
     self._samples_seen_at_last_write = 0
+    self._current_batch = 0
+    self._total_batches_seen = 0
+    self._total_val_batches_seen = 0
 
-  def _init_writer(self, model):
-    """Sets file writer."""
-    if context.executing_eagerly():
-      self.writer = summary_ops_v2.create_file_writer(self.log_dir)
-      if not model.run_eagerly and self.write_graph:
-        with self.writer.as_default():
-          summary_ops_v2.graph(K.get_graph())
-    elif self.write_graph:
-      self.writer = tf_summary.FileWriter(self.log_dir, K.get_graph())
-    else:
-      self.writer = tf_summary.FileWriter(self.log_dir)
-
-  def _make_histogram_ops(self, model):
-    """Defines histogram ops when histogram_freq > 0."""
-    # only make histogram summary op if it hasn't already been made
-    if self.histogram_freq and self.merged is None:
-      for layer in self.model.layers:
-        for weight in layer.weights:
-          mapped_weight_name = weight.name.replace(':', '_')
-          tf_summary.histogram(mapped_weight_name, weight)
-          if self.write_images:
-            w_img = array_ops.squeeze(weight)
-            shape = K.int_shape(w_img)
-            if len(shape) == 2:  # dense layer kernel case
-              if shape[0] > shape[1]:
-                w_img = array_ops.transpose(w_img)
-                shape = K.int_shape(w_img)
-              w_img = array_ops.reshape(w_img, [1, shape[0], shape[1], 1])
-            elif len(shape) == 3:  # convnet case
-              if K.image_data_format() == 'channels_last':
-                # switch to channels_first to display
-                # every kernel as a separate image
-                w_img = array_ops.transpose(w_img, perm=[2, 0, 1])
-                shape = K.int_shape(w_img)
-              w_img = array_ops.reshape(w_img,
-                                        [shape[0], shape[1], shape[2], 1])
-            elif len(shape) == 1:  # bias case
-              w_img = array_ops.reshape(w_img, [1, shape[0], 1, 1])
-            else:
-              # not possible to handle 3D convnets etc.
-              continue
-
-            shape = K.int_shape(w_img)
-            assert len(shape) == 4 and shape[-1] in [1, 3, 4]
-            tf_summary.image(mapped_weight_name, w_img)
-
-        if self.write_grads:
-          for weight in layer.trainable_weights:
-            mapped_weight_name = weight.name.replace(':', '_')
-            grads = model.optimizer.get_gradients(model.total_loss, weight)
-
-            def is_indexed_slices(grad):
-              return type(grad).__name__ == 'IndexedSlices'
-
-            grads = [
-                grad.values if is_indexed_slices(grad) else grad
-                for grad in grads
-            ]
-            tf_summary.histogram('{}_grad'.format(mapped_weight_name), grads)
-
-        if hasattr(layer, 'output'):
-          if isinstance(layer.output, list):
-            for i, output in enumerate(layer.output):
-              tf_summary.histogram('{}_out_{}'.format(layer.name, i), output)
-          else:
-            tf_summary.histogram('{}_out'.format(layer.name), layer.output)
+  def _validate_kwargs(self, kwargs):
+    """Handle arguments were supported in V1."""
+    if kwargs.get('write_grads', False):
+      logging.warning('`write_grads` will be ignored in TensorFlow 2.0 '
+                      'for the `TensorBoard` Callback.')
+    if kwargs.get('embeddings_freq', False):
+      logging.warning('Embeddings will be ignored in TensorFlow 2.0 '
+                      'for the `TensorBoard` Callback.')
+
+    unrecognized_kwargs = set(kwargs.keys()) - {
+        'write_grads', 'embeddings_freq', 'embeddings_layer_names',
+        'embeddings_metadata', 'embeddings_data'
+    }
+
+    # Only allow kwargs that were supported in V1.
+    if unrecognized_kwargs:
+      raise ValueError('Unrecognized arguments in `TensorBoard` '
+                       'Callback: ' + str(unrecognized_kwargs))
 
   def set_model(self, model):
-    """Sets Keras model and creates summary ops."""
-
+    """Sets Keras model and writes graph if specified."""
     self.model = model
-    self._init_writer(model)
-    # histogram summaries only enabled in graph mode
-    if not context.executing_eagerly():
-      self._make_histogram_ops(model)
-      self.merged = tf_summary.merge_all()
-
-    # If both embedding_freq and embeddings_data are available, we will
-    # visualize embeddings.
-    if self.embeddings_freq and self.embeddings_data is not None:
-      # Avoid circular dependency.
-      from tensorflow.python.keras.engine import training_utils  # pylint: disable=g-import-not-at-top
-      self.embeddings_data = training_utils.standardize_input_data(
-          self.embeddings_data, model.input_names)
-
-      # If embedding_layer_names are not provided, get all of the embedding
-      # layers from the model.
-      embeddings_layer_names = self.embeddings_layer_names
-      if not embeddings_layer_names:
-        embeddings_layer_names = [
-            layer.name
-            for layer in self.model.layers
-            if type(layer).__name__ == 'Embedding'
-        ]
-
-      self.assign_embeddings = []
-      embeddings_vars = {}
-
-      self.batch_id = batch_id = array_ops.placeholder(dtypes.int32)
-      self.step = step = array_ops.placeholder(dtypes.int32)
-
-      for layer in self.model.layers:
-        if layer.name in embeddings_layer_names:
-          embedding_input = self.model.get_layer(layer.name).output
-          embedding_size = np.prod(embedding_input.shape[1:])
-          embedding_input = array_ops.reshape(embedding_input,
-                                              (step, int(embedding_size)))
-          shape = (self.embeddings_data[0].shape[0], int(embedding_size))
-          embedding = variables.Variable(
-              array_ops.zeros(shape), name=layer.name + '_embedding')
-          embeddings_vars[layer.name] = embedding
-          batch = state_ops.assign(embedding[batch_id:batch_id + step],
-                                   embedding_input)
-          self.assign_embeddings.append(batch)
-
-      self.saver = saver.Saver(list(embeddings_vars.values()))
-
-      # Create embeddings_metadata dictionary
-      if isinstance(self.embeddings_metadata, str):
-        embeddings_metadata = {
-            layer_name: self.embeddings_metadata
-            for layer_name in embeddings_vars.keys()
-        }
-      else:
-        # If embedding_metadata is already a dictionary
-        embeddings_metadata = self.embeddings_metadata
-
-      try:
-        from tensorboard.plugins import projector
-      except ImportError:
-        raise ImportError('Failed to import TensorBoard. Please make sure that '
-                          'TensorBoard integration is complete."')
-
-      # TODO(psv): Add integration tests to test embedding visualization
-      # with TensorBoard callback. We are unable to write a unit test for this
-      # because TensorBoard dependency assumes TensorFlow package is installed.
-      config = projector.ProjectorConfig()
-      for layer_name, tensor in embeddings_vars.items():
-        embedding = config.embeddings.add()
-        embedding.tensor_name = tensor.name
-
-        if (embeddings_metadata is not None and
-            layer_name in embeddings_metadata):
-          embedding.metadata_path = embeddings_metadata[layer_name]
-
-      projector.visualize_embeddings(self.writer, config)
-
-  def _fetch_callback(self, summary):
-    self.writer.add_summary(summary, self._total_val_batches_seen)
-    self._total_val_batches_seen += 1
-
-  def _write_custom_summaries(self, step, logs=None):
-    """Writes metrics out as custom scalar summaries.
-
-    Arguments:
-        step: the global step to use for Tensorboard.
-        logs: dict. Keys are scalar summary names, values are
-            NumPy scalars.
-
-    """
-    logs = logs or {}
-    if context.executing_eagerly():
-      # use v2 summary ops
-      with self.writer.as_default(), summary_ops_v2.always_record_summaries():
-        for name, value in logs.items():
-          if isinstance(value, np.ndarray):
-            value = value.item()
-          summary_ops_v2.scalar(name, value, step=step)
-    else:
-      # use FileWriter from v1 summary
-      for name, value in logs.items():
-        if isinstance(value, np.ndarray):
-          value = value.item()
-        summary = tf_summary.Summary()
-        summary_value = summary.value.add()
-        summary_value.simple_value = value
-        summary_value.tag = name
-        self.writer.add_summary(summary, step)
-    self.writer.flush()
+    with context.eager_mode():
+      self.writer = summary_ops_v2.create_file_writer(self.log_dir)
+      if self.write_graph:
+        if model.run_eagerly:
+          logging.warning('TensorBoard Callback will ignore `write_graph=True`'
+                          'when `Model.run_eagerly=True`.`')
+        else:
+          with self.writer.as_default():
+            with summary_ops_v2.always_record_summaries():
+              summary_ops_v2.graph(K.get_graph())
 
   def on_batch_end(self, batch, logs=None):
     """Writes scalar summaries for metrics on every training batch."""
-    # Don't output batch_size and batch number as Tensorboard summaries
+    # Don't output batch_size and batch number as TensorBoard summaries
     logs = logs or {}
     self._samples_seen += logs.get('size', 1)
     samples_seen_since = self._samples_seen - self._samples_seen_at_last_write
     if self.update_freq != 'epoch' and samples_seen_since >= self.update_freq:
-      batch_logs = {('batch_' + k): v
-                    for k, v in logs.items()
-                    if k not in ['batch', 'size', 'num_steps']}
-      self._write_custom_summaries(self._total_batches_seen, batch_logs)
+      self._log_metrics(logs, prefix='batch_', step=self._total_batches_seen)
       self._samples_seen_at_last_write = self._samples_seen
     self._total_batches_seen += 1
 
-  def on_epoch_begin(self, epoch, logs=None):
-    """Add histogram op to Model eval_function callbacks, reset batch count."""
-
-    # check if histogram summary should be run for this epoch
-    if self.histogram_freq and epoch % self.histogram_freq == 0:
-      self._epoch = epoch
-      # pylint: disable=protected-access
-      # add the histogram summary op if it should run this epoch
-      self.model._make_eval_function()
-      if self.merged not in self.model._eval_function.fetches:
-        self.model._eval_function.fetches.append(self.merged)
-        self.model._eval_function.fetch_callbacks[
-            self.merged] = self._fetch_callback
-      # pylint: enable=protected-access
-
   def on_epoch_end(self, epoch, logs=None):
-    """Checks if summary ops should run next epoch, logs scalar summaries."""
-
-    # don't output batch_size and
-    # batch number as Tensorboard summaries
-    logs = {('epoch_' + k): v
-            for k, v in logs.items()
-            if k not in ['batch', 'size', 'num_steps']}
-    if self.update_freq == 'epoch':
-      step = epoch
-    else:
-      step = self._samples_seen
-    self._write_custom_summaries(step, logs)
-
-    # pop the histogram summary op after each epoch
-    if self.histogram_freq:
-      # pylint: disable=protected-access
-      if self.merged in self.model._eval_function.fetches:
-        self.model._eval_function.fetches.remove(self.merged)
-      if self.merged in self.model._eval_function.fetch_callbacks:
-        self.model._eval_function.fetch_callbacks.pop(self.merged)
-      # pylint: enable=protected-access
-
-    if self.embeddings_data is None and self.embeddings_freq:
-      raise ValueError('To visualize embeddings, embeddings_data must '
-                       'be provided.')
-
-    if self.embeddings_freq and self.embeddings_data is not None:
-      if epoch % self.embeddings_freq == 0:
-        # We need a second forward-pass here because we're passing
-        # the `embeddings_data` explicitly. This design allows to pass
-        # arbitrary data as `embeddings_data` and results from the fact
-        # that we need to know the size of the `tf.Variable`s which
-        # hold the embeddings in `set_model`. At this point, however,
-        # the `validation_data` is not yet set.
-
-        embeddings_data = self.embeddings_data
-        n_samples = embeddings_data[0].shape[0]
-        i = 0
-        while i < n_samples:
-          step = min(self.batch_size, n_samples - i)
-          batch = slice(i, i + step)
-
-          if isinstance(self.model.input, list):
-            feed_dict = {
-                model_input: embeddings_data[idx][batch]
-                for idx, model_input in enumerate(self.model.input)
-            }
-          else:
-            feed_dict = {self.model.input: embeddings_data[0][batch]}
-
-          feed_dict.update({self.batch_id: i, self.step: step})
+    """Runs metrics and histogram summaries at epoch end."""
+    step = epoch if self.update_freq == 'epoch' else self._samples_seen
+    self._log_metrics(logs, prefix='epoch_', step=step)
 
-          if not isinstance(K.learning_phase(), int):
-            feed_dict[K.learning_phase()] = False
+    if self.histogram_freq and epoch % self.histogram_freq == 0:
+      self._log_weights(epoch)
 
-          self.sess.run(self.assign_embeddings, feed_dict=feed_dict)
-          self.saver.save(self.sess,
-                          os.path.join(self.log_dir, 'keras_embedding.ckpt'),
-                          epoch)
+  def on_train_end(self, logs=None):
+    with context.eager_mode():
+      self.writer.close()
 
-          i += self.batch_size
+  def _log_metrics(self, logs, prefix, step):
+    """Writes metrics out as custom scalar summaries.
 
-  def on_train_end(self, logs=None):
-    self.writer.close()
+    Arguments:
+        logs: Dict. Keys are scalar summary names, values are NumPy scalars.
+        prefix: String. The prefix to apply to the scalar summary names.
+        step: Int. The global step to use for TensorBoard.
+    """
+    if logs is None:
+      logs = {}
+    # Scrub non-metric items and assign batch or epoch prefix.
+    metric_logs = {(prefix + k): v
+                   for k, v in logs.items()
+                   if k not in ['batch', 'size', 'num_steps']}
+    with context.eager_mode(), \
+          self.writer.as_default(), \
+          summary_ops_v2.always_record_summaries():
+      for name, value in metric_logs.items():
+        summary_ops_v2.scalar(name, value, step=step)
+
+  def _log_weights(self, epoch):
+    """Logs the weights of the Model to TensorBoard."""
+    with context.eager_mode(), \
+          self.writer.as_default(), \
+          summary_ops_v2.always_record_summaries():
+      for layer in self.model.layers:
+        for weight in layer.weights:
+          weight_name = weight.name.replace(':', '_')
+          with ops.init_scope():
+            weight = K.get_value(weight)
+          summary_ops_v2.histogram(weight_name, weight, step=epoch)
+          if self.write_images:
+            self._log_weight_as_image(weight, weight_name, epoch)
+      self.writer.flush()
+
+  def _log_weight_as_image(self, weight, weight_name, epoch):
+    """Logs a weight as a TensorBoard image."""
+    w_img = array_ops.squeeze(weight)
+    shape = K.int_shape(w_img)
+    if len(shape) == 1:  # Bias case
+      w_img = array_ops.reshape(w_img, [1, shape[0], 1, 1])
+    elif len(shape) == 2:  # Dense layer kernel case
+      if shape[0] > shape[1]:
+        w_img = array_ops.transpose(w_img)
+        shape = K.int_shape(w_img)
+      w_img = array_ops.reshape(w_img, [1, shape[0], shape[1], 1])
+    elif len(shape) == 3:  # ConvNet case
+      if K.image_data_format() == 'channels_last':
+        # Switch to channels_first to display every kernel as a separate
+        # image.
+        w_img = array_ops.transpose(w_img, perm=[2, 0, 1])
+        shape = K.int_shape(w_img)
+      w_img = array_ops.reshape(w_img, [shape[0], shape[1], shape[2], 1])
+
+    shape = K.int_shape(w_img)
+    # Not possible to handle 3D convnets etc.
+    if len(shape) == 4 and shape[-1] in [1, 3, 4]:
+      summary_ops_v2.image(weight_name, w_img, step=epoch)
 
 
 @keras_export('keras.callbacks.ReduceLROnPlateau')
diff --git a/tensorflow/python/keras/callbacks_test.py b/tensorflow/python/keras/callbacks_test.py
index 6aba08b95f..cced204750 100644
--- a/tensorflow/python/keras/callbacks_test.py
+++ b/tensorflow/python/keras/callbacks_test.py
@@ -23,22 +23,21 @@ import csv
 import os
 import re
 import shutil
-import tempfile
 import threading
 import unittest
 
 from absl.testing import parameterized
 import numpy as np
 
-from tensorflow.core.framework import summary_pb2
 from tensorflow.python import keras
 from tensorflow.python.framework import random_seed
-from tensorflow.python.framework import test_util
 from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import testing_utils
+from tensorflow.python.ops import summary_ops_v2
 from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import adam
+from tensorflow.python.util import tf_contextlib
 
 try:
   import h5py  # pylint:disable=g-import-not-at-top
@@ -836,310 +835,6 @@ class KerasCallbacksTest(test.TestCase):
       self.assertEqual(len(loss), 1)
       self.assertEqual(loss[0], np.inf)
 
-  @test_util.run_deprecated_v1
-  def test_TensorBoard(self):
-    np.random.seed(1337)
-
-    temp_dir = self.get_temp_dir()
-    self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True)
-
-    (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
-        train_samples=TRAIN_SAMPLES,
-        test_samples=TEST_SAMPLES,
-        input_shape=(INPUT_DIM,),
-        num_classes=NUM_CLASSES)
-    y_test = keras.utils.to_categorical(y_test)
-    y_train = keras.utils.to_categorical(y_train)
-
-    def data_generator(train):
-      if train:
-        max_batch_index = len(x_train) // BATCH_SIZE
-      else:
-        max_batch_index = len(x_test) // BATCH_SIZE
-      i = 0
-      while 1:
-        if train:
-          yield (x_train[i * BATCH_SIZE:(i + 1) * BATCH_SIZE],
-                 y_train[i * BATCH_SIZE:(i + 1) * BATCH_SIZE])
-        else:
-          yield (x_test[i * BATCH_SIZE:(i + 1) * BATCH_SIZE],
-                 y_test[i * BATCH_SIZE:(i + 1) * BATCH_SIZE])
-        i += 1
-        i %= max_batch_index
-
-    # case: Sequential
-    with self.cached_session():
-      model = keras.models.Sequential()
-      model.add(
-          keras.layers.Dense(
-              NUM_HIDDEN, input_dim=INPUT_DIM, activation='relu'))
-      # non_trainable_weights: moving_variance, moving_mean
-      model.add(keras.layers.BatchNormalization())
-      model.add(keras.layers.Dense(NUM_CLASSES, activation='softmax'))
-      model.compile(
-          loss='categorical_crossentropy',
-          optimizer='sgd',
-          metrics=['accuracy'])
-      tsb = keras.callbacks.TensorBoard(
-          log_dir=temp_dir, histogram_freq=1, write_images=True,
-          write_grads=True, batch_size=5)
-      cbks = [tsb]
-
-      # fit with validation data
-      model.fit(
-          x_train,
-          y_train,
-          batch_size=BATCH_SIZE,
-          validation_data=(x_test, y_test),
-          callbacks=cbks,
-          epochs=3,
-          verbose=0)
-
-      # fit with validation data and accuracy
-      model.fit(
-          x_train,
-          y_train,
-          batch_size=BATCH_SIZE,
-          validation_data=(x_test, y_test),
-          callbacks=cbks,
-          epochs=2,
-          verbose=0)
-
-      # fit generator with validation data
-      model.fit_generator(
-          data_generator(True),
-          len(x_train),
-          epochs=2,
-          validation_data=(x_test, y_test),
-          callbacks=cbks,
-          verbose=0)
-
-      # fit generator without validation data
-      # histogram_freq must be zero
-      tsb.histogram_freq = 0
-      model.fit_generator(
-          data_generator(True),
-          len(x_train),
-          epochs=2,
-          callbacks=cbks,
-          verbose=0)
-
-      # fit generator with validation data and accuracy
-      tsb.histogram_freq = 1
-      model.fit_generator(
-          data_generator(True),
-          len(x_train),
-          epochs=2,
-          validation_data=(x_test, y_test),
-          callbacks=cbks,
-          verbose=0)
-
-      # fit generator without validation data and accuracy
-      tsb.histogram_freq = 0
-      model.fit_generator(
-          data_generator(True), len(x_train), epochs=2, callbacks=cbks)
-      assert os.path.exists(temp_dir)
-
-  @test_util.run_deprecated_v1
-  def test_TensorBoard_multi_input_output(self):
-    np.random.seed(1337)
-    tmpdir = self.get_temp_dir()
-    self.addCleanup(shutil.rmtree, tmpdir, ignore_errors=True)
-
-    with self.cached_session():
-      filepath = os.path.join(tmpdir, 'logs')
-
-      (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
-          train_samples=TRAIN_SAMPLES,
-          test_samples=TEST_SAMPLES,
-          input_shape=(INPUT_DIM,),
-          num_classes=NUM_CLASSES)
-      y_test = keras.utils.to_categorical(y_test)
-      y_train = keras.utils.to_categorical(y_train)
-
-      def data_generator(train):
-        if train:
-          max_batch_index = len(x_train) // BATCH_SIZE
-        else:
-          max_batch_index = len(x_test) // BATCH_SIZE
-        i = 0
-        while 1:
-          if train:
-            # simulate multi-input/output models
-            yield ([x_train[i * BATCH_SIZE: (i + 1) * BATCH_SIZE]] * 2,
-                   [y_train[i * BATCH_SIZE: (i + 1) * BATCH_SIZE]] * 2)
-          else:
-            yield ([x_test[i * BATCH_SIZE: (i + 1) * BATCH_SIZE]] * 2,
-                   [y_test[i * BATCH_SIZE: (i + 1) * BATCH_SIZE]] * 2)
-          i += 1
-          i %= max_batch_index
-
-      inp1 = keras.Input((INPUT_DIM,))
-      inp2 = keras.Input((INPUT_DIM,))
-      inp = keras.layers.add([inp1, inp2])
-      hidden = keras.layers.Dense(2, activation='relu')(inp)
-      hidden = keras.layers.Dropout(0.1)(hidden)
-      output1 = keras.layers.Dense(NUM_CLASSES, activation='softmax')(hidden)
-      output2 = keras.layers.Dense(NUM_CLASSES, activation='softmax')(hidden)
-      model = keras.models.Model([inp1, inp2], [output1, output2])
-      model.compile(loss='categorical_crossentropy',
-                    optimizer='sgd',
-                    metrics=['accuracy'])
-
-      # we must generate new callbacks for each test, as they aren't stateless
-      def callbacks_factory(histogram_freq):
-        return [keras.callbacks.TensorBoard(log_dir=filepath,
-                                            histogram_freq=histogram_freq,
-                                            write_images=True, write_grads=True,
-                                            batch_size=5)]
-
-      # fit without validation data
-      model.fit([x_train] * 2, [y_train] * 2, batch_size=BATCH_SIZE,
-                callbacks=callbacks_factory(histogram_freq=0), epochs=3)
-
-      # fit with validation data and accuracy
-      model.fit([x_train] * 2, [y_train] * 2, batch_size=BATCH_SIZE,
-                validation_data=([x_test] * 2, [y_test] * 2),
-                callbacks=callbacks_factory(histogram_freq=1), epochs=2)
-
-      # fit generator without validation data
-      model.fit_generator(data_generator(True), len(x_train), epochs=2,
-                          callbacks=callbacks_factory(histogram_freq=0))
-
-      # fit generator with validation data and accuracy
-      model.fit_generator(data_generator(True), len(x_train), epochs=2,
-                          validation_data=([x_test] * 2, [y_test] * 2),
-                          callbacks=callbacks_factory(histogram_freq=1))
-      assert os.path.isdir(filepath)
-
-  @test_util.run_deprecated_v1
-  def test_Tensorboard_histogram_summaries_in_test_function(self):
-
-    class FileWriterStub(object):
-
-      def __init__(self, logdir, graph=None):
-        self.logdir = logdir
-        self.graph = graph
-        self.steps_seen = []
-
-      def add_summary(self, summary, global_step):
-        summary_obj = summary_pb2.Summary()
-
-        # ensure a valid Summary proto is being sent
-        if isinstance(summary, bytes):
-          summary_obj.ParseFromString(summary)
-        else:
-          assert isinstance(summary, summary_pb2.Summary)
-          summary_obj = summary
-
-        # keep track of steps seen for the merged_summary op,
-        # which contains the histogram summaries
-        if len(summary_obj.value) > 1:
-          self.steps_seen.append(global_step)
-
-      def flush(self):
-        pass
-
-      def close(self):
-        pass
-
-    def _init_writer(obj, _):
-      obj.writer = FileWriterStub(obj.log_dir)
-
-    np.random.seed(1337)
-    tmpdir = self.get_temp_dir()
-    self.addCleanup(shutil.rmtree, tmpdir, ignore_errors=True)
-    (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
-        train_samples=TRAIN_SAMPLES,
-        test_samples=TEST_SAMPLES,
-        input_shape=(INPUT_DIM,),
-        num_classes=NUM_CLASSES)
-    y_test = keras.utils.to_categorical(y_test)
-    y_train = keras.utils.to_categorical(y_train)
-
-    with self.cached_session():
-      model = keras.models.Sequential()
-      model.add(
-          keras.layers.Dense(
-              NUM_HIDDEN, input_dim=INPUT_DIM, activation='relu'))
-      # non_trainable_weights: moving_variance, moving_mean
-      model.add(keras.layers.BatchNormalization())
-      model.add(keras.layers.Dense(NUM_CLASSES, activation='softmax'))
-      model.compile(
-          loss='categorical_crossentropy',
-          optimizer='sgd',
-          metrics=['accuracy'])
-      keras.callbacks.TensorBoard._init_writer = _init_writer
-      tsb = keras.callbacks.TensorBoard(
-          log_dir=tmpdir,
-          histogram_freq=1,
-          write_images=True,
-          write_grads=True,
-          batch_size=5)
-      cbks = [tsb]
-
-      # fit with validation data
-      model.fit(
-          x_train,
-          y_train,
-          batch_size=BATCH_SIZE,
-          validation_data=(x_test, y_test),
-          callbacks=cbks,
-          epochs=3,
-          verbose=0)
-
-      self.assertAllEqual(tsb.writer.steps_seen, [0, 1, 2, 3, 4, 5])
-
-  @test_util.run_deprecated_v1
-  def test_Tensorboard_histogram_summaries_with_generator(self):
-    np.random.seed(1337)
-    tmpdir = self.get_temp_dir()
-    self.addCleanup(shutil.rmtree, tmpdir, ignore_errors=True)
-
-    def generator():
-      x = np.random.randn(10, 100).astype(np.float32)
-      y = np.random.randn(10, 10).astype(np.float32)
-      while True:
-        yield x, y
-
-    with self.cached_session():
-      model = testing_utils.get_small_sequential_mlp(
-          num_hidden=10, num_classes=10, input_dim=100)
-      model.compile(
-          loss='categorical_crossentropy',
-          optimizer='sgd',
-          metrics=['accuracy'])
-      tsb = keras.callbacks.TensorBoard(
-          log_dir=tmpdir,
-          histogram_freq=1,
-          write_images=True,
-          write_grads=True,
-          batch_size=5)
-      cbks = [tsb]
-
-      # fit with validation generator
-      model.fit_generator(
-          generator(),
-          steps_per_epoch=2,
-          epochs=2,
-          validation_data=generator(),
-          validation_steps=2,
-          callbacks=cbks,
-          verbose=0)
-
-      with self.assertRaises(ValueError):
-        # fit with validation generator but no
-        # validation_steps
-        model.fit_generator(
-            generator(),
-            steps_per_epoch=2,
-            epochs=2,
-            validation_data=generator(),
-            callbacks=cbks,
-            verbose=0)
-
-      self.assertTrue(os.path.exists(tmpdir))
-
   @unittest.skipIf(
       os.name == 'nt',
       'use_multiprocessing=True does not work on windows properly.')
@@ -1187,239 +882,196 @@ class KerasCallbacksTest(test.TestCase):
       t.join()
       assert not t.is_alive()
 
-  def test_TensorBoard_with_ReduceLROnPlateau(self):
+  def test_RemoteMonitorWithJsonPayload(self):
+    if requests is None:
+      self.skipTest('`requests` required to run this test')
     with self.cached_session():
-      temp_dir = self.get_temp_dir()
-      self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True)
-
       (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
           train_samples=TRAIN_SAMPLES,
           test_samples=TEST_SAMPLES,
           input_shape=(INPUT_DIM,),
           num_classes=NUM_CLASSES)
-      y_test = keras.utils.to_categorical(y_test)
-      y_train = keras.utils.to_categorical(y_train)
-
-      model = testing_utils.get_small_sequential_mlp(
-          num_hidden=NUM_HIDDEN, num_classes=NUM_CLASSES, input_dim=INPUT_DIM)
+      y_test = keras.utils.np_utils.to_categorical(y_test)
+      y_train = keras.utils.np_utils.to_categorical(y_train)
+      model = keras.models.Sequential()
+      model.add(
+          keras.layers.Dense(
+              NUM_HIDDEN, input_dim=INPUT_DIM, activation='relu'))
+      model.add(keras.layers.Dense(NUM_CLASSES, activation='softmax'))
       model.compile(
-          loss='binary_crossentropy', optimizer='sgd', metrics=['accuracy'])
+          loss='categorical_crossentropy',
+          optimizer='rmsprop',
+          metrics=['accuracy'])
+      cbks = [keras.callbacks.RemoteMonitor(send_as_json=True)]
 
-      cbks = [
-          keras.callbacks.ReduceLROnPlateau(
-              monitor='val_loss', factor=0.5, patience=4, verbose=1),
-          keras.callbacks.TensorBoard(log_dir=temp_dir)
-      ]
+      with test.mock.patch.object(requests, 'post'):
+        model.fit(
+            x_train,
+            y_train,
+            batch_size=BATCH_SIZE,
+            validation_data=(x_test, y_test),
+            callbacks=cbks,
+            epochs=1)
 
-      model.fit(
-          x_train,
-          y_train,
-          batch_size=BATCH_SIZE,
-          validation_data=(x_test, y_test),
-          callbacks=cbks,
-          epochs=2,
-          verbose=0)
 
-      assert os.path.exists(temp_dir)
+class _MockSummaryFile(object):
+  """Mocks a TensorBoard summary file, recording the tag names it sees."""
+
+  def __init__(self):
+    self.scalar_names = set()
+    self.hist_names = set()
+    self.image_names = set()
+
 
-  @test_util.run_deprecated_v1
-  def test_Tensorboard_batch_logging(self):
+def _make_mock_scalar_summary(summary_file):
 
-    class FileWriterStub(object):
+  def _mock_scalar_summary(name, *args, **kwargs):  # pylint: disable=unused-argument
+    summary_file.scalar_names.update({name})
 
-      def __init__(self, logdir, graph=None):
-        self.logdir = logdir
-        self.graph = graph
-        self.batches_logged = []
-        self.summary_values = []
-        self.summary_tags = []
+  return _mock_scalar_summary
 
-      def add_summary(self, summary, step):
-        self.summary_values.append(summary.value[0].simple_value)
-        self.summary_tags.append(summary.value[0].tag)
-        self.batches_logged.append(step)
 
-      def flush(self):
-        pass
+def _make_mock_hist_summary(summary_file):
 
-      def close(self):
-        pass
+  def _mock_hist_summary(name, *args, **kwargs):  # pylint: disable=unused-argument
+    summary_file.hist_names.update({name})
 
-    temp_dir = self.get_temp_dir()
-    self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True)
+  return _mock_hist_summary
 
-    tb_cbk = keras.callbacks.TensorBoard(temp_dir, update_freq='batch')
-    tb_cbk.writer = FileWriterStub(temp_dir)
 
-    for batch in range(5):
-      tb_cbk.on_batch_end(batch, {'acc': batch})
-    self.assertEqual(tb_cbk.writer.batches_logged, [0, 1, 2, 3, 4])
-    self.assertEqual(tb_cbk.writer.summary_values, [0., 1., 2., 3., 4.])
-    self.assertEqual(tb_cbk.writer.summary_tags, ['batch_acc'] * 5)
+def _make_mock_image_summary(summary_file):
 
-  @test_util.run_deprecated_v1
-  def test_Tensorboard_epoch_and_batch_logging(self):
+  def _mock_image_summary(name, *args, **kwargs):  # pylint: disable=unused-argument
+    summary_file.image_names.update({name})
 
-    class FileWriterStub(object):
+  return _mock_image_summary
 
-      def __init__(self, logdir, graph=None):
-        self.logdir = logdir
-        self.graph = graph
 
-      def add_summary(self, summary, step):
-        if 'batch_' in summary.value[0].tag:
-          self.batch_summary = (step, summary)
-        elif 'epoch_' in summary.value[0].tag:
-          self.epoch_summary = (step, summary)
+@tf_contextlib.contextmanager
+def _mock_summary_api(summary_file):
+  with test.mock.patch.object(summary_ops_v2,
+                              'scalar',
+                              _make_mock_scalar_summary(summary_file)), \
+        test.mock.patch.object(summary_ops_v2,
+                               'histogram',
+                               _make_mock_hist_summary(summary_file)), \
+        test.mock.patch.object(summary_ops_v2,
+                               'image',
+                               _make_mock_image_summary(summary_file)):
+    yield
 
-      def flush(self):
-        pass
 
-      def close(self):
-        pass
+@keras_parameterized.run_with_all_model_types
+@keras_parameterized.run_all_keras_modes(always_skip_v1=True)
+class TestTensorBoardV2(keras_parameterized.TestCase):
 
-    temp_dir = self.get_temp_dir()
-    self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True)
+  def _get_model(self):
+    layers = [
+        keras.layers.Conv2D(8, (3, 3)),
+        keras.layers.Flatten(),
+        keras.layers.Dense(1)
+    ]
+    model = testing_utils.get_model_from_layers(layers, input_shape=(10, 10, 1))
+    model.compile('sgd', 'mse', run_eagerly=testing_utils.should_run_eagerly())
+    return model
 
-    tb_cbk = keras.callbacks.TensorBoard(temp_dir, update_freq='batch')
-    tb_cbk.writer = FileWriterStub(temp_dir)
+  def test_TensorBoard_basic(self):
+    summary_file = _MockSummaryFile()
+    model = self._get_model()
+    x, y = np.ones((10, 10, 10, 1)), np.ones((10, 1))
+    temp_dir = self.get_temp_dir() + '/tb'
+    tb_cbk = keras.callbacks.TensorBoard(temp_dir)
 
-    tb_cbk.on_batch_end(0, {'acc': 5.0})
-    batch_step, batch_summary = tb_cbk.writer.batch_summary
-    self.assertEqual(batch_step, 0)
-    self.assertEqual(batch_summary.value[0].simple_value, 5.0)
+    with _mock_summary_api(summary_file):  # pylint: disable=not-context-manager
+      model.fit(
+          x,
+          y,
+          batch_size=2,
+          epochs=2,
+          validation_data=(x, y),
+          callbacks=[tb_cbk])
 
-    tb_cbk = keras.callbacks.TensorBoard(temp_dir, update_freq='epoch')
-    tb_cbk.writer = FileWriterStub(temp_dir)
-    tb_cbk.on_epoch_end(0, {'acc': 10.0})
-    epoch_step, epoch_summary = tb_cbk.writer.epoch_summary
-    self.assertEqual(epoch_step, 0)
-    self.assertEqual(epoch_summary.value[0].simple_value, 10.0)
+    self.assertEqual(summary_file.scalar_names,
+                     {'epoch_loss', 'epoch_val_loss'})
 
-  @test_util.run_in_graph_and_eager_modes
-  def test_Tensorboard_eager(self):
-    temp_dir = tempfile.mkdtemp(dir=self.get_temp_dir())
-    self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True)
+  def test_TensorBoard_batch_metrics(self):
+    summary_file = _MockSummaryFile()
+    model = self._get_model()
+    x, y = np.ones((10, 10, 10, 1)), np.ones((10, 1))
+    temp_dir = self.get_temp_dir() + '/tb'
+    tb_cbk = keras.callbacks.TensorBoard(temp_dir, update_freq=1)
 
-    (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
-        train_samples=TRAIN_SAMPLES,
-        test_samples=TEST_SAMPLES,
-        input_shape=(INPUT_DIM,),
-        num_classes=NUM_CLASSES)
-    y_test = keras.utils.to_categorical(y_test)
-    y_train = keras.utils.to_categorical(y_train)
+    with _mock_summary_api(summary_file):  # pylint: disable=not-context-manager
+      model.fit(
+          x,
+          y,
+          batch_size=2,
+          epochs=2,
+          validation_data=(x, y),
+          callbacks=[tb_cbk])
 
-    model = testing_utils.get_small_sequential_mlp(
-        num_hidden=NUM_HIDDEN, num_classes=NUM_CLASSES, input_dim=INPUT_DIM)
-    model.compile(
-        loss='binary_crossentropy',
-        optimizer=adam.AdamOptimizer(0.01),
-        metrics=['accuracy'])
+    self.assertEqual(summary_file.scalar_names,
+                     {'batch_loss', 'epoch_loss', 'epoch_val_loss'})
 
-    cbks = [keras.callbacks.TensorBoard(log_dir=temp_dir)]
+  def test_TensorBoard_weight_histograms(self):
+    summary_file = _MockSummaryFile()
+    model = self._get_model()
+    x, y = np.ones((10, 10, 10, 1)), np.ones((10, 1))
+    temp_dir = self.get_temp_dir() + '/tb'
+    tb_cbk = keras.callbacks.TensorBoard(temp_dir, histogram_freq=1)
 
-    model.fit(
-        x_train,
-        y_train,
-        batch_size=BATCH_SIZE,
-        validation_data=(x_test, y_test),
-        callbacks=cbks,
-        epochs=2,
-        verbose=0)
-
-    self.assertTrue(os.path.exists(temp_dir))
-
-  @test_util.run_deprecated_v1
-  def test_TensorBoard_update_freq(self):
-
-    class FileWriterStub(object):
-
-      def __init__(self, logdir, graph=None):
-        self.logdir = logdir
-        self.graph = graph
-        self.batch_summaries = []
-        self.epoch_summaries = []
-
-      def add_summary(self, summary, step):
-        if 'batch_' in summary.value[0].tag:
-          self.batch_summaries.append((step, summary))
-        elif 'epoch_' in summary.value[0].tag:
-          self.epoch_summaries.append((step, summary))
-
-      def flush(self):
-        pass
-
-      def close(self):
-        pass
-
-    temp_dir = self.get_temp_dir()
-    self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True)
-
-    # Epoch mode
-    tb_cbk = keras.callbacks.TensorBoard(temp_dir, update_freq='epoch')
-    tb_cbk.writer = FileWriterStub(temp_dir)
-
-    tb_cbk.on_batch_end(0, {'acc': 5.0, 'size': 1})
-    self.assertEqual(tb_cbk.writer.batch_summaries, [])
-    tb_cbk.on_epoch_end(0, {'acc': 10.0, 'size': 1})
-    self.assertEqual(len(tb_cbk.writer.epoch_summaries), 1)
-
-    # Batch mode
-    tb_cbk = keras.callbacks.TensorBoard(temp_dir, update_freq='batch')
-    tb_cbk.writer = FileWriterStub(temp_dir)
-
-    tb_cbk.on_batch_end(0, {'acc': 5.0, 'size': 1})
-    self.assertEqual(len(tb_cbk.writer.batch_summaries), 1)
-    tb_cbk.on_batch_end(0, {'acc': 5.0, 'size': 1})
-    self.assertEqual(len(tb_cbk.writer.batch_summaries), 2)
-    self.assertFalse(tb_cbk.writer.epoch_summaries)
-
-    # Integer mode
-    tb_cbk = keras.callbacks.TensorBoard(temp_dir, update_freq=20)
-    tb_cbk.writer = FileWriterStub(temp_dir)
-
-    tb_cbk.on_batch_end(0, {'acc': 5.0, 'size': 10})
-    self.assertFalse(tb_cbk.writer.batch_summaries)
-    tb_cbk.on_batch_end(0, {'acc': 5.0, 'size': 10})
-    self.assertEqual(len(tb_cbk.writer.batch_summaries), 1)
-    tb_cbk.on_batch_end(0, {'acc': 5.0, 'size': 10})
-    self.assertEqual(len(tb_cbk.writer.batch_summaries), 1)
-    tb_cbk.on_batch_end(0, {'acc': 5.0, 'size': 10})
-    self.assertEqual(len(tb_cbk.writer.batch_summaries), 2)
-    tb_cbk.on_batch_end(0, {'acc': 10.0, 'size': 10})
-    self.assertEqual(len(tb_cbk.writer.batch_summaries), 2)
-    self.assertFalse(tb_cbk.writer.epoch_summaries)
+    with _mock_summary_api(summary_file):  # pylint: disable=not-context-manager
+      model.fit(
+          x,
+          y,
+          batch_size=2,
+          epochs=2,
+          validation_data=(x, y),
+          callbacks=[tb_cbk])
 
-  def test_RemoteMonitorWithJsonPayload(self):
-    if requests is None:
-      self.skipTest('`requests` required to run this test')
-    with self.cached_session():
-      (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
-          train_samples=TRAIN_SAMPLES,
-          test_samples=TEST_SAMPLES,
-          input_shape=(INPUT_DIM,),
-          num_classes=NUM_CLASSES)
-      y_test = keras.utils.np_utils.to_categorical(y_test)
-      y_train = keras.utils.np_utils.to_categorical(y_train)
-      model = keras.models.Sequential()
-      model.add(
-          keras.layers.Dense(
-              NUM_HIDDEN, input_dim=INPUT_DIM, activation='relu'))
-      model.add(keras.layers.Dense(NUM_CLASSES, activation='softmax'))
-      model.compile(
-          loss='categorical_crossentropy',
-          optimizer='rmsprop',
-          metrics=['accuracy'])
-      cbks = [keras.callbacks.RemoteMonitor(send_as_json=True)]
+    self.assertEqual(summary_file.scalar_names,
+                     {'epoch_loss', 'epoch_val_loss'})
 
-      with test.mock.patch.object(requests, 'post'):
-        model.fit(
-            x_train,
-            y_train,
-            batch_size=BATCH_SIZE,
-            validation_data=(x_test, y_test),
-            callbacks=cbks,
-            epochs=1)
+    # Strip Layer names as Layers are created multiple times in test.
+    hist_names = {
+        name[name.rfind('/') + 1:] for name in summary_file.hist_names
+    }
+    self.assertEqual(hist_names, {'bias_0', 'kernel_0'})
+
+  def test_TensorBoard_weight_images(self):
+    summary_file = _MockSummaryFile()
+    model = self._get_model()
+    x, y = np.ones((10, 10, 10, 1)), np.ones((10, 1))
+    temp_dir = self.get_temp_dir() + '/tb'
+    tb_cbk = keras.callbacks.TensorBoard(
+        temp_dir, histogram_freq=1, write_images=True)
+
+    with _mock_summary_api(summary_file):  # pylint: disable=not-context-manager
+      model.fit(
+          x,
+          y,
+          batch_size=2,
+          epochs=2,
+          validation_data=(x, y),
+          callbacks=[tb_cbk])
+
+    self.assertEqual(summary_file.scalar_names,
+                     {'epoch_loss', 'epoch_val_loss'})
+
+    # Strip Layer names as Layers are created multiple times in test.
+    hist_names = {
+        name[name.rfind('/') + 1:] for name in summary_file.hist_names
+    }
+    self.assertEqual(hist_names, {'bias_0', 'kernel_0'})
+
+    image_names = {
+        name[name.rfind('/') + 1:] for name in summary_file.image_names
+    }
+    self.assertEqual(image_names, {'bias_0', 'kernel_0'})
+
+  def test_TensorBoard_invalid_argument(self):
+    with self.assertRaisesRegexp(ValueError, 'Unrecognized arguments'):
+      keras.callbacks.TensorBoard(wwrite_images=True)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/keras/callbacks_v1.py b/tensorflow/python/keras/callbacks_v1.py
new file mode 100644
index 0000000000..980eacd356
--- /dev/null
+++ b/tensorflow/python/keras/callbacks_v1.py
@@ -0,0 +1,424 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# pylint: disable=g-import-not-at-top
+"""Callbacks: utilities called at certain points during model training.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+
+import numpy as np
+
+from tensorflow.python.eager import context
+from tensorflow.python.framework import dtypes
+from tensorflow.python.keras import backend as K
+from tensorflow.python.keras import callbacks
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import state_ops
+from tensorflow.python.ops import summary_ops_v2
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.summary import summary as tf_summary
+from tensorflow.python.training import saver
+from tensorflow.python.util.tf_export import keras_export
+
+
+@keras_export(v1=['keras.callbacks.TensorBoard'])
+class TensorBoard(callbacks.Callback):
+  # pylint: disable=line-too-long
+  """TensorBoard basic visualizations.
+
+  This callback writes a log for TensorBoard, which allows
+  you to visualize dynamic graphs of your training and test
+  metrics, as well as activation histograms for the different
+  layers in your model.
+
+  TensorBoard is a visualization tool provided with TensorFlow.
+
+  If you have installed TensorFlow with pip, you should be able
+  to launch TensorBoard from the command line:
+
+  ```sh
+  tensorboard --logdir=/full_path_to_your_logs
+  ```
+
+  You can find more information about TensorBoard
+  [here](https://www.tensorflow.org/get_started/summaries_and_tensorboard).
+
+  Arguments:
+      log_dir: the path of the directory where to save the log files to be
+        parsed by TensorBoard.
+      histogram_freq: frequency (in epochs) at which to compute activation and
+        weight histograms for the layers of the model. If set to 0, histograms
+        won't be computed. Validation data (or split) must be specified for
+        histogram visualizations.
+      write_graph: whether to visualize the graph in TensorBoard. The log file
+        can become quite large when write_graph is set to True.
+      write_grads: whether to visualize gradient histograms in TensorBoard.
+        `histogram_freq` must be greater than 0.
+      batch_size: size of batch of inputs to feed to the network for histograms
+        computation.
+      write_images: whether to write model weights to visualize as image in
+        TensorBoard.
+      embeddings_freq: frequency (in epochs) at which selected embedding layers
+        will be saved. If set to 0, embeddings won't be computed. Data to be
+        visualized in TensorBoard's Embedding tab must be passed as
+        `embeddings_data`.
+      embeddings_layer_names: a list of names of layers to keep eye on. If None
+        or empty list all the embedding layer will be watched.
+      embeddings_metadata: a dictionary which maps layer name to a file name in
+        which metadata for this embedding layer is saved. See the
+          [details](https://www.tensorflow.org/how_tos/embedding_viz/#metadata_optional)
+            about metadata files format. In case if the same metadata file is
+            used for all embedding layers, string can be passed.
+      embeddings_data: data to be embedded at layers specified in
+        `embeddings_layer_names`. Numpy array (if the model has a single input)
+        or list of Numpy arrays (if the model has multiple inputs). Learn [more
+        about
+            embeddings](https://www.tensorflow.org/programmers_guide/embedding)
+      update_freq: `'batch'` or `'epoch'` or integer. When using `'batch'`,
+        writes the losses and metrics to TensorBoard after each batch. The same
+        applies for `'epoch'`. If using an integer, let's say `1000`, the
+        callback will write the metrics and losses to TensorBoard every 1000
+        samples. Note that writing too frequently to TensorBoard can slow down
+        your training.
+
+  Raises:
+      ValueError: If histogram_freq is set and no validation data is provided.
+
+  @compatibility(eager)
+  Using the `TensorBoard` callback will work when eager execution is enabled,
+  with the restriction that outputting histogram summaries of weights and
+  gradients is not supported. Consequently, `histogram_freq` will be ignored.
+  @end_compatibility
+  """
+
+  # pylint: enable=line-too-long
+
+  def __init__(self,
+               log_dir='./logs',
+               histogram_freq=0,
+               batch_size=32,
+               write_graph=True,
+               write_grads=False,
+               write_images=False,
+               embeddings_freq=0,
+               embeddings_layer_names=None,
+               embeddings_metadata=None,
+               embeddings_data=None,
+               update_freq='epoch'):
+    super(TensorBoard, self).__init__()
+    self.log_dir = log_dir
+    self.histogram_freq = histogram_freq
+    if self.histogram_freq and context.executing_eagerly():
+      logging.warning(
+          UserWarning('Weight and gradient histograms not supported for eager'
+                      'execution, setting `histogram_freq` to `0`.'))
+      self.histogram_freq = 0
+    self.merged = None
+    self.write_graph = write_graph
+    self.write_grads = write_grads
+    self.write_images = write_images
+    self.batch_size = batch_size
+    self._current_batch = 0
+    self._total_batches_seen = 0
+    self._total_val_batches_seen = 0
+    self.embeddings_freq = embeddings_freq
+    self.embeddings_layer_names = embeddings_layer_names
+    self.embeddings_metadata = embeddings_metadata
+    self.embeddings_data = embeddings_data
+    if update_freq == 'batch':
+      self.update_freq = 1
+    else:
+      self.update_freq = update_freq
+    self._samples_seen = 0
+    self._samples_seen_at_last_write = 0
+
+  def _init_writer(self, model):
+    """Sets file writer."""
+    if context.executing_eagerly():
+      self.writer = summary_ops_v2.create_file_writer(self.log_dir)
+      if not model.run_eagerly and self.write_graph:
+        with self.writer.as_default():
+          summary_ops_v2.graph(K.get_graph())
+    elif self.write_graph:
+      self.writer = tf_summary.FileWriter(self.log_dir, K.get_graph())
+    else:
+      self.writer = tf_summary.FileWriter(self.log_dir)
+
+  def _make_histogram_ops(self, model):
+    """Defines histogram ops when histogram_freq > 0."""
+    # only make histogram summary op if it hasn't already been made
+    if self.histogram_freq and self.merged is None:
+      for layer in self.model.layers:
+        for weight in layer.weights:
+          mapped_weight_name = weight.name.replace(':', '_')
+          tf_summary.histogram(mapped_weight_name, weight)
+          if self.write_images:
+            w_img = array_ops.squeeze(weight)
+            shape = K.int_shape(w_img)
+            if len(shape) == 2:  # dense layer kernel case
+              if shape[0] > shape[1]:
+                w_img = array_ops.transpose(w_img)
+                shape = K.int_shape(w_img)
+              w_img = array_ops.reshape(w_img, [1, shape[0], shape[1], 1])
+            elif len(shape) == 3:  # convnet case
+              if K.image_data_format() == 'channels_last':
+                # switch to channels_first to display
+                # every kernel as a separate image
+                w_img = array_ops.transpose(w_img, perm=[2, 0, 1])
+                shape = K.int_shape(w_img)
+              w_img = array_ops.reshape(w_img,
+                                        [shape[0], shape[1], shape[2], 1])
+            elif len(shape) == 1:  # bias case
+              w_img = array_ops.reshape(w_img, [1, shape[0], 1, 1])
+            else:
+              # not possible to handle 3D convnets etc.
+              continue
+
+            shape = K.int_shape(w_img)
+            assert len(shape) == 4 and shape[-1] in [1, 3, 4]
+            tf_summary.image(mapped_weight_name, w_img)
+
+        if self.write_grads:
+          for weight in layer.trainable_weights:
+            mapped_weight_name = weight.name.replace(':', '_')
+            grads = model.optimizer.get_gradients(model.total_loss, weight)
+
+            def is_indexed_slices(grad):
+              return type(grad).__name__ == 'IndexedSlices'
+
+            grads = [
+                grad.values if is_indexed_slices(grad) else grad
+                for grad in grads
+            ]
+            tf_summary.histogram('{}_grad'.format(mapped_weight_name), grads)
+
+        if hasattr(layer, 'output'):
+          if isinstance(layer.output, list):
+            for i, output in enumerate(layer.output):
+              tf_summary.histogram('{}_out_{}'.format(layer.name, i), output)
+          else:
+            tf_summary.histogram('{}_out'.format(layer.name), layer.output)
+
+  def set_model(self, model):
+    """Sets Keras model and creates summary ops."""
+
+    self.model = model
+    self._init_writer(model)
+    # histogram summaries only enabled in graph mode
+    if not context.executing_eagerly():
+      self._make_histogram_ops(model)
+      self.merged = tf_summary.merge_all()
+
+    # If both embedding_freq and embeddings_data are available, we will
+    # visualize embeddings.
+    if self.embeddings_freq and self.embeddings_data is not None:
+      # Avoid circular dependency.
+      from tensorflow.python.keras.engine import training_utils  # pylint: disable=g-import-not-at-top
+      self.embeddings_data = training_utils.standardize_input_data(
+          self.embeddings_data, model.input_names)
+
+      # If embedding_layer_names are not provided, get all of the embedding
+      # layers from the model.
+      embeddings_layer_names = self.embeddings_layer_names
+      if not embeddings_layer_names:
+        embeddings_layer_names = [
+            layer.name
+            for layer in self.model.layers
+            if type(layer).__name__ == 'Embedding'
+        ]
+
+      self.assign_embeddings = []
+      embeddings_vars = {}
+
+      self.batch_id = batch_id = array_ops.placeholder(dtypes.int32)
+      self.step = step = array_ops.placeholder(dtypes.int32)
+
+      for layer in self.model.layers:
+        if layer.name in embeddings_layer_names:
+          embedding_input = self.model.get_layer(layer.name).output
+          embedding_size = np.prod(embedding_input.shape[1:])
+          embedding_input = array_ops.reshape(embedding_input,
+                                              (step, int(embedding_size)))
+          shape = (self.embeddings_data[0].shape[0], int(embedding_size))
+          embedding = variables.Variable(
+              array_ops.zeros(shape), name=layer.name + '_embedding')
+          embeddings_vars[layer.name] = embedding
+          batch = state_ops.assign(embedding[batch_id:batch_id + step],
+                                   embedding_input)
+          self.assign_embeddings.append(batch)
+
+      self.saver = saver.Saver(list(embeddings_vars.values()))
+
+      # Create embeddings_metadata dictionary
+      if isinstance(self.embeddings_metadata, str):
+        embeddings_metadata = {
+            layer_name: self.embeddings_metadata
+            for layer_name in embeddings_vars.keys()
+        }
+      else:
+        # If embedding_metadata is already a dictionary
+        embeddings_metadata = self.embeddings_metadata
+
+      try:
+        from tensorboard.plugins import projector
+      except ImportError:
+        raise ImportError('Failed to import TensorBoard. Please make sure that '
+                          'TensorBoard integration is complete."')
+
+      # TODO(psv): Add integration tests to test embedding visualization
+      # with TensorBoard callback. We are unable to write a unit test for this
+      # because TensorBoard dependency assumes TensorFlow package is installed.
+      config = projector.ProjectorConfig()
+      for layer_name, tensor in embeddings_vars.items():
+        embedding = config.embeddings.add()
+        embedding.tensor_name = tensor.name
+
+        if (embeddings_metadata is not None and
+            layer_name in embeddings_metadata):
+          embedding.metadata_path = embeddings_metadata[layer_name]
+
+      projector.visualize_embeddings(self.writer, config)
+
+  def _fetch_callback(self, summary):
+    self.writer.add_summary(summary, self._total_val_batches_seen)
+    self._total_val_batches_seen += 1
+
+  def _write_custom_summaries(self, step, logs=None):
+    """Writes metrics out as custom scalar summaries.
+
+    Arguments:
+        step: the global step to use for TensorBoard.
+        logs: dict. Keys are scalar summary names, values are
+            NumPy scalars.
+
+    """
+    logs = logs or {}
+    if context.executing_eagerly():
+      # use v2 summary ops
+      with self.writer.as_default(), summary_ops_v2.always_record_summaries():
+        for name, value in logs.items():
+          if isinstance(value, np.ndarray):
+            value = value.item()
+          summary_ops_v2.scalar(name, value, step=step)
+    else:
+      # use FileWriter from v1 summary
+      for name, value in logs.items():
+        if isinstance(value, np.ndarray):
+          value = value.item()
+        summary = tf_summary.Summary()
+        summary_value = summary.value.add()
+        summary_value.simple_value = value
+        summary_value.tag = name
+        self.writer.add_summary(summary, step)
+    self.writer.flush()
+
+  def on_batch_end(self, batch, logs=None):
+    """Writes scalar summaries for metrics on every training batch."""
+    # Don't output batch_size and batch number as TensorBoard summaries
+    logs = logs or {}
+    self._samples_seen += logs.get('size', 1)
+    samples_seen_since = self._samples_seen - self._samples_seen_at_last_write
+    if self.update_freq != 'epoch' and samples_seen_since >= self.update_freq:
+      batch_logs = {('batch_' + k): v
+                    for k, v in logs.items()
+                    if k not in ['batch', 'size', 'num_steps']}
+      self._write_custom_summaries(self._total_batches_seen, batch_logs)
+      self._samples_seen_at_last_write = self._samples_seen
+    self._total_batches_seen += 1
+
+  def on_epoch_begin(self, epoch, logs=None):
+    """Add histogram op to Model eval_function callbacks, reset batch count."""
+
+    # check if histogram summary should be run for this epoch
+    if self.histogram_freq and epoch % self.histogram_freq == 0:
+      self._epoch = epoch
+      # pylint: disable=protected-access
+      # add the histogram summary op if it should run this epoch
+      self.model._make_eval_function()
+      if self.merged not in self.model._eval_function.fetches:
+        self.model._eval_function.fetches.append(self.merged)
+        self.model._eval_function.fetch_callbacks[
+            self.merged] = self._fetch_callback
+      # pylint: enable=protected-access
+
+  def on_epoch_end(self, epoch, logs=None):
+    """Checks if summary ops should run next epoch, logs scalar summaries."""
+
+    # don't output batch_size and
+    # batch number as TensorBoard summaries
+    logs = {('epoch_' + k): v
+            for k, v in logs.items()
+            if k not in ['batch', 'size', 'num_steps']}
+    if self.update_freq == 'epoch':
+      step = epoch
+    else:
+      step = self._samples_seen
+    self._write_custom_summaries(step, logs)
+
+    # pop the histogram summary op after each epoch
+    if self.histogram_freq:
+      # pylint: disable=protected-access
+      if self.merged in self.model._eval_function.fetches:
+        self.model._eval_function.fetches.remove(self.merged)
+      if self.merged in self.model._eval_function.fetch_callbacks:
+        self.model._eval_function.fetch_callbacks.pop(self.merged)
+      # pylint: enable=protected-access
+
+    if self.embeddings_data is None and self.embeddings_freq:
+      raise ValueError('To visualize embeddings, embeddings_data must '
+                       'be provided.')
+
+    if self.embeddings_freq and self.embeddings_data is not None:
+      if epoch % self.embeddings_freq == 0:
+        # We need a second forward-pass here because we're passing
+        # the `embeddings_data` explicitly. This design allows to pass
+        # arbitrary data as `embeddings_data` and results from the fact
+        # that we need to know the size of the `tf.Variable`s which
+        # hold the embeddings in `set_model`. At this point, however,
+        # the `validation_data` is not yet set.
+
+        embeddings_data = self.embeddings_data
+        n_samples = embeddings_data[0].shape[0]
+        i = 0
+        while i < n_samples:
+          step = min(self.batch_size, n_samples - i)
+          batch = slice(i, i + step)
+
+          if isinstance(self.model.input, list):
+            feed_dict = {
+                model_input: embeddings_data[idx][batch]
+                for idx, model_input in enumerate(self.model.input)
+            }
+          else:
+            feed_dict = {self.model.input: embeddings_data[0][batch]}
+
+          feed_dict.update({self.batch_id: i, self.step: step})
+
+          if not isinstance(K.learning_phase(), int):
+            feed_dict[K.learning_phase()] = False
+
+          self.sess.run(self.assign_embeddings, feed_dict=feed_dict)
+          self.saver.save(self.sess,
+                          os.path.join(self.log_dir, 'keras_embedding.ckpt'),
+                          epoch)
+
+          i += self.batch_size
+
+  def on_train_end(self, logs=None):
+    self.writer.close()
diff --git a/tensorflow/python/keras/callbacks_v1_test.py b/tensorflow/python/keras/callbacks_v1_test.py
new file mode 100644
index 0000000000..fdb6af938b
--- /dev/null
+++ b/tensorflow/python/keras/callbacks_v1_test.py
@@ -0,0 +1,562 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for Keras callbacks."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import shutil
+import tempfile
+
+import numpy as np
+
+from tensorflow.core.framework import summary_pb2
+from tensorflow.python import keras
+from tensorflow.python.framework import test_util
+from tensorflow.python.keras import callbacks_v1
+from tensorflow.python.keras import testing_utils
+from tensorflow.python.platform import test
+from tensorflow.python.training import adam
+
+
+TRAIN_SAMPLES = 10
+TEST_SAMPLES = 10
+NUM_CLASSES = 2
+INPUT_DIM = 3
+NUM_HIDDEN = 5
+BATCH_SIZE = 5
+
+
+class TestTensorBoardV1(test.TestCase):
+
+  @test_util.run_deprecated_v1
+  def test_TensorBoard(self):
+    np.random.seed(1337)
+
+    temp_dir = self.get_temp_dir()
+    self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True)
+
+    (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
+        train_samples=TRAIN_SAMPLES,
+        test_samples=TEST_SAMPLES,
+        input_shape=(INPUT_DIM,),
+        num_classes=NUM_CLASSES)
+    y_test = keras.utils.to_categorical(y_test)
+    y_train = keras.utils.to_categorical(y_train)
+
+    def data_generator(train):
+      if train:
+        max_batch_index = len(x_train) // BATCH_SIZE
+      else:
+        max_batch_index = len(x_test) // BATCH_SIZE
+      i = 0
+      while 1:
+        if train:
+          yield (x_train[i * BATCH_SIZE:(i + 1) * BATCH_SIZE],
+                 y_train[i * BATCH_SIZE:(i + 1) * BATCH_SIZE])
+        else:
+          yield (x_test[i * BATCH_SIZE:(i + 1) * BATCH_SIZE],
+                 y_test[i * BATCH_SIZE:(i + 1) * BATCH_SIZE])
+        i += 1
+        i %= max_batch_index
+
+    # case: Sequential
+    with self.cached_session():
+      model = keras.models.Sequential()
+      model.add(
+          keras.layers.Dense(
+              NUM_HIDDEN, input_dim=INPUT_DIM, activation='relu'))
+      # non_trainable_weights: moving_variance, moving_mean
+      model.add(keras.layers.BatchNormalization())
+      model.add(keras.layers.Dense(NUM_CLASSES, activation='softmax'))
+      model.compile(
+          loss='categorical_crossentropy',
+          optimizer='sgd',
+          metrics=['accuracy'])
+      tsb = callbacks_v1.TensorBoard(
+          log_dir=temp_dir,
+          histogram_freq=1,
+          write_images=True,
+          write_grads=True,
+          batch_size=5)
+      cbks = [tsb]
+
+      # fit with validation data
+      model.fit(
+          x_train,
+          y_train,
+          batch_size=BATCH_SIZE,
+          validation_data=(x_test, y_test),
+          callbacks=cbks,
+          epochs=3,
+          verbose=0)
+
+      # fit with validation data and accuracy
+      model.fit(
+          x_train,
+          y_train,
+          batch_size=BATCH_SIZE,
+          validation_data=(x_test, y_test),
+          callbacks=cbks,
+          epochs=2,
+          verbose=0)
+
+      # fit generator with validation data
+      model.fit_generator(
+          data_generator(True),
+          len(x_train),
+          epochs=2,
+          validation_data=(x_test, y_test),
+          callbacks=cbks,
+          verbose=0)
+
+      # fit generator without validation data
+      # histogram_freq must be zero
+      tsb.histogram_freq = 0
+      model.fit_generator(
+          data_generator(True),
+          len(x_train),
+          epochs=2,
+          callbacks=cbks,
+          verbose=0)
+
+      # fit generator with validation data and accuracy
+      tsb.histogram_freq = 1
+      model.fit_generator(
+          data_generator(True),
+          len(x_train),
+          epochs=2,
+          validation_data=(x_test, y_test),
+          callbacks=cbks,
+          verbose=0)
+
+      # fit generator without validation data and accuracy
+      tsb.histogram_freq = 0
+      model.fit_generator(
+          data_generator(True), len(x_train), epochs=2, callbacks=cbks)
+      assert os.path.exists(temp_dir)
+
+  @test_util.run_deprecated_v1
+  def test_TensorBoard_multi_input_output(self):
+    np.random.seed(1337)
+    tmpdir = self.get_temp_dir()
+    self.addCleanup(shutil.rmtree, tmpdir, ignore_errors=True)
+
+    with self.cached_session():
+      filepath = os.path.join(tmpdir, 'logs')
+
+      (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
+          train_samples=TRAIN_SAMPLES,
+          test_samples=TEST_SAMPLES,
+          input_shape=(INPUT_DIM,),
+          num_classes=NUM_CLASSES)
+      y_test = keras.utils.to_categorical(y_test)
+      y_train = keras.utils.to_categorical(y_train)
+
+      def data_generator(train):
+        if train:
+          max_batch_index = len(x_train) // BATCH_SIZE
+        else:
+          max_batch_index = len(x_test) // BATCH_SIZE
+        i = 0
+        while 1:
+          if train:
+            # simulate multi-input/output models
+            yield ([x_train[i * BATCH_SIZE: (i + 1) * BATCH_SIZE]] * 2,
+                   [y_train[i * BATCH_SIZE: (i + 1) * BATCH_SIZE]] * 2)
+          else:
+            yield ([x_test[i * BATCH_SIZE: (i + 1) * BATCH_SIZE]] * 2,
+                   [y_test[i * BATCH_SIZE: (i + 1) * BATCH_SIZE]] * 2)
+          i += 1
+          i %= max_batch_index
+
+      inp1 = keras.Input((INPUT_DIM,))
+      inp2 = keras.Input((INPUT_DIM,))
+      inp = keras.layers.add([inp1, inp2])
+      hidden = keras.layers.Dense(2, activation='relu')(inp)
+      hidden = keras.layers.Dropout(0.1)(hidden)
+      output1 = keras.layers.Dense(NUM_CLASSES, activation='softmax')(hidden)
+      output2 = keras.layers.Dense(NUM_CLASSES, activation='softmax')(hidden)
+      model = keras.models.Model([inp1, inp2], [output1, output2])
+      model.compile(loss='categorical_crossentropy',
+                    optimizer='sgd',
+                    metrics=['accuracy'])
+
+      # we must generate new callbacks for each test, as they aren't stateless
+      def callbacks_factory(histogram_freq):
+        return [
+            callbacks_v1.TensorBoard(
+                log_dir=filepath,
+                histogram_freq=histogram_freq,
+                write_images=True,
+                write_grads=True,
+                batch_size=5)
+        ]
+
+      # fit without validation data
+      model.fit([x_train] * 2, [y_train] * 2, batch_size=BATCH_SIZE,
+                callbacks=callbacks_factory(histogram_freq=0), epochs=3)
+
+      # fit with validation data and accuracy
+      model.fit([x_train] * 2, [y_train] * 2, batch_size=BATCH_SIZE,
+                validation_data=([x_test] * 2, [y_test] * 2),
+                callbacks=callbacks_factory(histogram_freq=1), epochs=2)
+
+      # fit generator without validation data
+      model.fit_generator(data_generator(True), len(x_train), epochs=2,
+                          callbacks=callbacks_factory(histogram_freq=0))
+
+      # fit generator with validation data and accuracy
+      model.fit_generator(data_generator(True), len(x_train), epochs=2,
+                          validation_data=([x_test] * 2, [y_test] * 2),
+                          callbacks=callbacks_factory(histogram_freq=1))
+      assert os.path.isdir(filepath)
+
+  @test_util.run_deprecated_v1
+  def test_Tensorboard_histogram_summaries_in_test_function(self):
+
+    class FileWriterStub(object):
+
+      def __init__(self, logdir, graph=None):
+        self.logdir = logdir
+        self.graph = graph
+        self.steps_seen = []
+
+      def add_summary(self, summary, global_step):
+        summary_obj = summary_pb2.Summary()
+
+        # ensure a valid Summary proto is being sent
+        if isinstance(summary, bytes):
+          summary_obj.ParseFromString(summary)
+        else:
+          assert isinstance(summary, summary_pb2.Summary)
+          summary_obj = summary
+
+        # keep track of steps seen for the merged_summary op,
+        # which contains the histogram summaries
+        if len(summary_obj.value) > 1:
+          self.steps_seen.append(global_step)
+
+      def flush(self):
+        pass
+
+      def close(self):
+        pass
+
+    def _init_writer(obj, _):
+      obj.writer = FileWriterStub(obj.log_dir)
+
+    np.random.seed(1337)
+    tmpdir = self.get_temp_dir()
+    self.addCleanup(shutil.rmtree, tmpdir, ignore_errors=True)
+    (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
+        train_samples=TRAIN_SAMPLES,
+        test_samples=TEST_SAMPLES,
+        input_shape=(INPUT_DIM,),
+        num_classes=NUM_CLASSES)
+    y_test = keras.utils.to_categorical(y_test)
+    y_train = keras.utils.to_categorical(y_train)
+
+    with self.cached_session():
+      model = keras.models.Sequential()
+      model.add(
+          keras.layers.Dense(
+              NUM_HIDDEN, input_dim=INPUT_DIM, activation='relu'))
+      # non_trainable_weights: moving_variance, moving_mean
+      model.add(keras.layers.BatchNormalization())
+      model.add(keras.layers.Dense(NUM_CLASSES, activation='softmax'))
+      model.compile(
+          loss='categorical_crossentropy',
+          optimizer='sgd',
+          metrics=['accuracy'])
+      callbacks_v1.TensorBoard._init_writer = _init_writer
+      tsb = callbacks_v1.TensorBoard(
+          log_dir=tmpdir,
+          histogram_freq=1,
+          write_images=True,
+          write_grads=True,
+          batch_size=5)
+      cbks = [tsb]
+
+      # fit with validation data
+      model.fit(
+          x_train,
+          y_train,
+          batch_size=BATCH_SIZE,
+          validation_data=(x_test, y_test),
+          callbacks=cbks,
+          epochs=3,
+          verbose=0)
+
+      self.assertAllEqual(tsb.writer.steps_seen, [0, 1, 2, 3, 4, 5])
+
+  @test_util.run_deprecated_v1
+  def test_Tensorboard_histogram_summaries_with_generator(self):
+    np.random.seed(1337)
+    tmpdir = self.get_temp_dir()
+    self.addCleanup(shutil.rmtree, tmpdir, ignore_errors=True)
+
+    def generator():
+      x = np.random.randn(10, 100).astype(np.float32)
+      y = np.random.randn(10, 10).astype(np.float32)
+      while True:
+        yield x, y
+
+    with self.cached_session():
+      model = testing_utils.get_small_sequential_mlp(
+          num_hidden=10, num_classes=10, input_dim=100)
+      model.compile(
+          loss='categorical_crossentropy',
+          optimizer='sgd',
+          metrics=['accuracy'])
+      tsb = callbacks_v1.TensorBoard(
+          log_dir=tmpdir,
+          histogram_freq=1,
+          write_images=True,
+          write_grads=True,
+          batch_size=5)
+      cbks = [tsb]
+
+      # fit with validation generator
+      model.fit_generator(
+          generator(),
+          steps_per_epoch=2,
+          epochs=2,
+          validation_data=generator(),
+          validation_steps=2,
+          callbacks=cbks,
+          verbose=0)
+
+      with self.assertRaises(ValueError):
+        # fit with validation generator but no
+        # validation_steps
+        model.fit_generator(
+            generator(),
+            steps_per_epoch=2,
+            epochs=2,
+            validation_data=generator(),
+            callbacks=cbks,
+            verbose=0)
+
+      self.assertTrue(os.path.exists(tmpdir))
+
+  def test_TensorBoard_with_ReduceLROnPlateau(self):
+    with self.cached_session():
+      temp_dir = self.get_temp_dir()
+      self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True)
+
+      (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
+          train_samples=TRAIN_SAMPLES,
+          test_samples=TEST_SAMPLES,
+          input_shape=(INPUT_DIM,),
+          num_classes=NUM_CLASSES)
+      y_test = keras.utils.to_categorical(y_test)
+      y_train = keras.utils.to_categorical(y_train)
+
+      model = testing_utils.get_small_sequential_mlp(
+          num_hidden=NUM_HIDDEN, num_classes=NUM_CLASSES, input_dim=INPUT_DIM)
+      model.compile(
+          loss='binary_crossentropy', optimizer='sgd', metrics=['accuracy'])
+
+      cbks = [
+          keras.callbacks.ReduceLROnPlateau(
+              monitor='val_loss', factor=0.5, patience=4, verbose=1),
+          callbacks_v1.TensorBoard(log_dir=temp_dir)
+      ]
+
+      model.fit(
+          x_train,
+          y_train,
+          batch_size=BATCH_SIZE,
+          validation_data=(x_test, y_test),
+          callbacks=cbks,
+          epochs=2,
+          verbose=0)
+
+      assert os.path.exists(temp_dir)
+
+  @test_util.run_deprecated_v1
+  def test_Tensorboard_batch_logging(self):
+
+    class FileWriterStub(object):
+
+      def __init__(self, logdir, graph=None):
+        self.logdir = logdir
+        self.graph = graph
+        self.batches_logged = []
+        self.summary_values = []
+        self.summary_tags = []
+
+      def add_summary(self, summary, step):
+        self.summary_values.append(summary.value[0].simple_value)
+        self.summary_tags.append(summary.value[0].tag)
+        self.batches_logged.append(step)
+
+      def flush(self):
+        pass
+
+      def close(self):
+        pass
+
+    temp_dir = self.get_temp_dir()
+    self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True)
+
+    tb_cbk = callbacks_v1.TensorBoard(temp_dir, update_freq='batch')
+    tb_cbk.writer = FileWriterStub(temp_dir)
+
+    for batch in range(5):
+      tb_cbk.on_batch_end(batch, {'acc': batch})
+    self.assertEqual(tb_cbk.writer.batches_logged, [0, 1, 2, 3, 4])
+    self.assertEqual(tb_cbk.writer.summary_values, [0., 1., 2., 3., 4.])
+    self.assertEqual(tb_cbk.writer.summary_tags, ['batch_acc'] * 5)
+
+  @test_util.run_deprecated_v1
+  def test_Tensorboard_epoch_and_batch_logging(self):
+
+    class FileWriterStub(object):
+
+      def __init__(self, logdir, graph=None):
+        self.logdir = logdir
+        self.graph = graph
+
+      def add_summary(self, summary, step):
+        if 'batch_' in summary.value[0].tag:
+          self.batch_summary = (step, summary)
+        elif 'epoch_' in summary.value[0].tag:
+          self.epoch_summary = (step, summary)
+
+      def flush(self):
+        pass
+
+      def close(self):
+        pass
+
+    temp_dir = self.get_temp_dir()
+    self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True)
+
+    tb_cbk = callbacks_v1.TensorBoard(temp_dir, update_freq='batch')
+    tb_cbk.writer = FileWriterStub(temp_dir)
+
+    tb_cbk.on_batch_end(0, {'acc': 5.0})
+    batch_step, batch_summary = tb_cbk.writer.batch_summary
+    self.assertEqual(batch_step, 0)
+    self.assertEqual(batch_summary.value[0].simple_value, 5.0)
+
+    tb_cbk = callbacks_v1.TensorBoard(temp_dir, update_freq='epoch')
+    tb_cbk.writer = FileWriterStub(temp_dir)
+    tb_cbk.on_epoch_end(0, {'acc': 10.0})
+    epoch_step, epoch_summary = tb_cbk.writer.epoch_summary
+    self.assertEqual(epoch_step, 0)
+    self.assertEqual(epoch_summary.value[0].simple_value, 10.0)
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_Tensorboard_eager(self):
+    temp_dir = tempfile.mkdtemp(dir=self.get_temp_dir())
+    self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True)
+
+    (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
+        train_samples=TRAIN_SAMPLES,
+        test_samples=TEST_SAMPLES,
+        input_shape=(INPUT_DIM,),
+        num_classes=NUM_CLASSES)
+    y_test = keras.utils.to_categorical(y_test)
+    y_train = keras.utils.to_categorical(y_train)
+
+    model = testing_utils.get_small_sequential_mlp(
+        num_hidden=NUM_HIDDEN, num_classes=NUM_CLASSES, input_dim=INPUT_DIM)
+    model.compile(
+        loss='binary_crossentropy',
+        optimizer=adam.AdamOptimizer(0.01),
+        metrics=['accuracy'])
+
+    cbks = [callbacks_v1.TensorBoard(log_dir=temp_dir)]
+
+    model.fit(
+        x_train,
+        y_train,
+        batch_size=BATCH_SIZE,
+        validation_data=(x_test, y_test),
+        callbacks=cbks,
+        epochs=2,
+        verbose=0)
+
+    self.assertTrue(os.path.exists(temp_dir))
+
+  @test_util.run_deprecated_v1
+  def test_TensorBoard_update_freq(self):
+
+    class FileWriterStub(object):
+
+      def __init__(self, logdir, graph=None):
+        self.logdir = logdir
+        self.graph = graph
+        self.batch_summaries = []
+        self.epoch_summaries = []
+
+      def add_summary(self, summary, step):
+        if 'batch_' in summary.value[0].tag:
+          self.batch_summaries.append((step, summary))
+        elif 'epoch_' in summary.value[0].tag:
+          self.epoch_summaries.append((step, summary))
+
+      def flush(self):
+        pass
+
+      def close(self):
+        pass
+
+    temp_dir = self.get_temp_dir()
+    self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True)
+
+    # Epoch mode
+    tb_cbk = callbacks_v1.TensorBoard(temp_dir, update_freq='epoch')
+    tb_cbk.writer = FileWriterStub(temp_dir)
+
+    tb_cbk.on_batch_end(0, {'acc': 5.0, 'size': 1})
+    self.assertEqual(tb_cbk.writer.batch_summaries, [])
+    tb_cbk.on_epoch_end(0, {'acc': 10.0, 'size': 1})
+    self.assertEqual(len(tb_cbk.writer.epoch_summaries), 1)
+
+    # Batch mode
+    tb_cbk = callbacks_v1.TensorBoard(temp_dir, update_freq='batch')
+    tb_cbk.writer = FileWriterStub(temp_dir)
+
+    tb_cbk.on_batch_end(0, {'acc': 5.0, 'size': 1})
+    self.assertEqual(len(tb_cbk.writer.batch_summaries), 1)
+    tb_cbk.on_batch_end(0, {'acc': 5.0, 'size': 1})
+    self.assertEqual(len(tb_cbk.writer.batch_summaries), 2)
+    self.assertFalse(tb_cbk.writer.epoch_summaries)
+
+    # Integer mode
+    tb_cbk = callbacks_v1.TensorBoard(temp_dir, update_freq=20)
+    tb_cbk.writer = FileWriterStub(temp_dir)
+
+    tb_cbk.on_batch_end(0, {'acc': 5.0, 'size': 10})
+    self.assertFalse(tb_cbk.writer.batch_summaries)
+    tb_cbk.on_batch_end(0, {'acc': 5.0, 'size': 10})
+    self.assertEqual(len(tb_cbk.writer.batch_summaries), 1)
+    tb_cbk.on_batch_end(0, {'acc': 5.0, 'size': 10})
+    self.assertEqual(len(tb_cbk.writer.batch_summaries), 1)
+    tb_cbk.on_batch_end(0, {'acc': 5.0, 'size': 10})
+    self.assertEqual(len(tb_cbk.writer.batch_summaries), 2)
+    tb_cbk.on_batch_end(0, {'acc': 10.0, 'size': 10})
+    self.assertEqual(len(tb_cbk.writer.batch_summaries), 2)
+    self.assertFalse(tb_cbk.writer.epoch_summaries)
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index ccbc1c9b71..700b956da4 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -196,14 +196,14 @@ def validate_callbacks(input_callbacks, optimizer):
       # features of the callback that involve accessing model attributes and
       # running ops.
       if isinstance(callback, callbacks.TensorBoard):
-        if callback.__getattribute__('histogram_freq'):
+        if getattr(callback, 'histogram_freq', False):
           logging.warning(
               UserWarning(
                   '`histogram_freq` in the TensorBoard callback is not '
                   'supported when using DistributionStrategy. Setting '
                   '`histogram_freq` to `0`.'))
           callback.histogram_freq = 0
-        if callback.__getattribute__('write_grads'):
+        if getattr(callback, 'write_grads', False):
           logging.warning(
               UserWarning(
                   '`write_grads` in the TensorBoard callback is not supported '
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.callbacks.-tensor-board.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.callbacks.-tensor-board.pbtxt
index 2e0f77eda8..66bacd8094 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.callbacks.-tensor-board.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.callbacks.-tensor-board.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.keras.callbacks.TensorBoard"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.callbacks.TensorBoard\'>"
+  is_instance: "<class \'tensorflow.python.keras.callbacks_v1.TensorBoard\'>"
   is_instance: "<class \'tensorflow.python.keras.callbacks.Callback\'>"
   is_instance: "<type \'object\'>"
   member_method {
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.callbacks.-tensor-board.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.callbacks.-tensor-board.pbtxt
index 2e0f77eda8..abf4286b69 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.callbacks.-tensor-board.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.callbacks.-tensor-board.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'log_dir\', \'histogram_freq\', \'batch_size\', \'write_graph\', \'write_grads\', \'write_images\', \'embeddings_freq\', \'embeddings_layer_names\', \'embeddings_metadata\', \'embeddings_data\', \'update_freq\'], varargs=None, keywords=None, defaults=[\'./logs\', \'0\', \'32\', \'True\', \'False\', \'False\', \'0\', \'None\', \'None\', \'None\', \'epoch\'], "
+    argspec: "args=[\'self\', \'log_dir\', \'histogram_freq\', \'write_graph\', \'write_images\', \'update_freq\'], varargs=None, keywords=kwargs, defaults=[\'./logs\', \'0\', \'True\', \'False\', \'epoch\'], "
   }
   member_method {
     name: "on_batch_begin"
-- 
GitLab


From 8773ca70dc7c143068a4b086bf32300720d08d1b Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Thu, 7 Feb 2019 12:48:15 -0800
Subject: [PATCH 2083/2345] [tf.data] Switching tf.data functions to use
 `defun`, providing an escape hatch to continue using the legacy `Defun`.

There are subtle differences between the implementation of `defun` and `Defun` (such as resources handling or control flow) and it is possible that input pipelines that use control flow or resources in their functions might be affected by this change. To migrate majority of existing pipelines to the recommended way of creating functions in TF 2.0 world, while allowing (a small number of) existing pipelines to continue relying on the deprecated behavior, this CL provides an escape hatch.

If your input pipeline is affected by this CL, it should apply the escape hatch by replacing `foo.map(...)` with `foo.map_with_legacy_function(...)`.

PiperOrigin-RevId: 232926599
---
 .../python/data/experimental/__init__.py      |   2 +
 .../python/data/experimental/ops/batching.py  |  64 +++++-
 .../python/data/kernel_tests/map_test.py      |  14 +-
 tensorflow/python/data/ops/dataset_ops.py     | 188 +++++++++++++-----
 .../golden/v1/tensorflow.data.-dataset.pbtxt  |   4 +
 ...ow.data.-fixed-length-record-dataset.pbtxt |   4 +
 .../tensorflow.data.-t-f-record-dataset.pbtxt |   4 +
 .../tensorflow.data.-text-line-dataset.pbtxt  |   4 +
 ...rflow.data.experimental.-csv-dataset.pbtxt |   4 +
 ...ow.data.experimental.-random-dataset.pbtxt |   4 +
 ...rflow.data.experimental.-sql-dataset.pbtxt |   4 +
 .../v1/tensorflow.data.experimental.pbtxt     |   4 +
 .../tools/compatibility/tf_upgrade_v2.py      |   2 +
 .../tools/compatibility/tf_upgrade_v2_test.py |   7 +
 14 files changed, 249 insertions(+), 60 deletions(-)

diff --git a/tensorflow/python/data/experimental/__init__.py b/tensorflow/python/data/experimental/__init__.py
index 3c1d798bd2..275bdf7ef4 100644
--- a/tensorflow/python/data/experimental/__init__.py
+++ b/tensorflow/python/data/experimental/__init__.py
@@ -58,6 +58,7 @@ See [Importing Data](https://tensorflow.org/guide/datasets) for an overview.
 @@make_csv_dataset
 @@make_saveable_from_iterator
 @@map_and_batch
+@@map_and_batch_with_legacy_function
 @@parallel_interleave
 @@parse_example_dataset
 @@prefetch_to_device
@@ -82,6 +83,7 @@ from __future__ import print_function
 
 from tensorflow.python.data.experimental.ops.batching import dense_to_sparse_batch
 from tensorflow.python.data.experimental.ops.batching import map_and_batch
+from tensorflow.python.data.experimental.ops.batching import map_and_batch_with_legacy_function
 from tensorflow.python.data.experimental.ops.batching import unbatch
 from tensorflow.python.data.experimental.ops.cardinality import cardinality
 from tensorflow.python.data.experimental.ops.cardinality import INFINITE as INFINITE_CARDINALITY
diff --git a/tensorflow/python/data/experimental/ops/batching.py b/tensorflow/python/data/experimental/ops/batching.py
index 39cb0a68f8..a4ef71f796 100644
--- a/tensorflow/python/data/experimental/ops/batching.py
+++ b/tensorflow/python/data/experimental/ops/batching.py
@@ -38,6 +38,7 @@ from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import gen_experimental_dataset_ops as ged_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import sparse_ops
+from tensorflow.python.util import deprecation
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -549,12 +550,15 @@ class _MapAndBatchDataset(dataset_ops.UnaryDataset):
   """A `Dataset` that maps a function over a batch of elements."""
 
   def __init__(self, input_dataset, map_func, batch_size, num_parallel_calls,
-               drop_remainder):
+               drop_remainder, use_legacy_function=False):
     """See `Dataset.map()` for details."""
     self._input_dataset = input_dataset
 
     self._map_func = dataset_ops.StructuredFunctionWrapper(
-        map_func, "tf.data.experimental.map_and_batch()", dataset=input_dataset)
+        map_func,
+        "tf.data.experimental.map_and_batch()",
+        dataset=input_dataset,
+        use_legacy_function=use_legacy_function)
     self._batch_size_t = ops.convert_to_tensor(
         batch_size, dtype=dtypes.int64, name="batch_size")
     self._num_parallel_calls_t = ops.convert_to_tensor(
@@ -589,6 +593,62 @@ class _MapAndBatchDataset(dataset_ops.UnaryDataset):
     return self._structure
 
 
+@deprecation.deprecated(None, "Use `tf.data.experimental.map_and_batch()")
+@tf_export(v1=["data.experimental.map_and_batch_with_legacy_function"])
+def map_and_batch_with_legacy_function(map_func,
+                                       batch_size,
+                                       num_parallel_batches=None,
+                                       drop_remainder=False,
+                                       num_parallel_calls=None):
+  """Fused implementation of `map` and `batch`.
+
+  NOTE: This is an escape hatch for existing uses of `map_and_batch` that do not
+  work with V2 functions. New uses are strongly discouraged and existing uses
+  should migrate to `map_and_batch` as this method will not be removed in V2.
+
+  Args:
+    map_func: A function mapping a nested structure of tensors to another
+      nested structure of tensors.
+    batch_size: A `tf.int64` scalar `tf.Tensor`, representing the number of
+      consecutive elements of this dataset to combine in a single batch.
+    num_parallel_batches: (Optional.) A `tf.int64` scalar `tf.Tensor`,
+      representing the number of batches to create in parallel. On one hand,
+      higher values can help mitigate the effect of stragglers. On the other
+      hand, higher values can increase contention if CPU is scarce.
+    drop_remainder: (Optional.) A `tf.bool` scalar `tf.Tensor`, representing
+      whether the last batch should be dropped in case its size is smaller than
+      desired; the default behavior is not to drop the smaller batch.
+    num_parallel_calls: (Optional.) A `tf.int32` scalar `tf.Tensor`,
+      representing the number of elements to process in parallel. If not
+      specified, `batch_size * num_parallel_batches` elements will be processed
+      in parallel. If the value `tf.data.experimental.AUTOTUNE` is used, then
+      the number of parallel calls is set dynamically based on available CPU.
+
+  Returns:
+    A `Dataset` transformation function, which can be passed to
+    `tf.data.Dataset.apply`.
+
+  Raises:
+    ValueError: If both `num_parallel_batches` and `num_parallel_calls` are
+      specified.
+  """
+
+  if num_parallel_batches is None and num_parallel_calls is None:
+    num_parallel_calls = batch_size
+  elif num_parallel_batches is not None and num_parallel_calls is None:
+    num_parallel_calls = batch_size * num_parallel_batches
+  elif num_parallel_batches is not None and num_parallel_calls is not None:
+    raise ValueError("The `num_parallel_batches` and `num_parallel_calls` "
+                     "arguments are mutually exclusive.")
+
+  def _apply_fn(dataset):
+    return _MapAndBatchDataset(dataset, map_func, batch_size,
+                               num_parallel_calls, drop_remainder,
+                               use_legacy_function=True)
+
+  return _apply_fn
+
+
 @tf_export("data.experimental.map_and_batch")
 def map_and_batch(map_func,
                   batch_size,
diff --git a/tensorflow/python/data/kernel_tests/map_test.py b/tensorflow/python/data/kernel_tests/map_test.py
index 7a17dd852a..2d421a02d4 100644
--- a/tensorflow/python/data/kernel_tests/map_test.py
+++ b/tensorflow/python/data/kernel_tests/map_test.py
@@ -393,7 +393,7 @@ class MapTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.assertRaises(errors.OutOfRangeError):
       self.evaluate(get_next())
 
-  # TODO(b/121264236): add eager mode coverage when we have mutli-device setup.
+  # TODO(b/121264236): add eager mode coverage when we have multi-device setup.
   @test_util.run_v1_only("b/121264236")
   def testSkipEagerCaptureConstantsWithConflictingDevices(self):
     config = config_pb2.ConfigProto(device_count={"CPU": 3})
@@ -410,9 +410,8 @@ class MapTest(test_base.DatasetTestBase, parameterized.TestCase):
       expected_output = [8.0] * 10
       self.assertDatasetProduces(dataset, expected_output=expected_output)
 
-  # TODO(b/121264236): add eager mode coverage when we have mutli-device setup.
-  @test_util.run_v1_only(
-      "defun will convert RefVariables to ResourceVariables.")
+  # TODO(b/121264236): add eager mode coverage when we have multi-device setup.
+  @test_util.run_v1_only("b/121264236")
   def testSkipEagerRefVariablesWithConflictingDevices(self):
     config = config_pb2.ConfigProto(device_count={"CPU": 3})
     with self.cached_session(config=config):
@@ -424,7 +423,10 @@ class MapTest(test_base.DatasetTestBase, parameterized.TestCase):
           b = variables.VariableV1(5.0)
         return math_ops.add(a, b)
 
-      dataset = dataset_ops.Dataset.from_tensors(0).repeat(10).map(func)
+      # Use the legacy function implementation as eager function will convert
+      # RefVariables to ResourceVariables.
+      dataset = dataset_ops.Dataset.from_tensors(0).repeat(10)
+      dataset = dataset.map_with_legacy_function(func)
       self.evaluate(variables.global_variables_initializer())
       expected_output = [8.0] * 10
       self.assertDatasetProduces(
@@ -432,7 +434,7 @@ class MapTest(test_base.DatasetTestBase, parameterized.TestCase):
           expected_output=expected_output,
           requires_initialization=True)
 
-  # TODO(b/121264236): add eager mode coverage when we have mutli-device setup.
+  # TODO(b/121264236): add eager mode coverage when we have multi-device setup.
   @test_util.run_v1_only("b/121264236")
   def testSkipEagerResourceVariablesWithConflictingDevices(self):
     config = config_pb2.ConfigProto(device_count={"CPU": 3})
diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index 7cd838b926..32a9c755e4 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -39,6 +39,7 @@ from tensorflow.python.data.util import sparse
 from tensorflow.python.data.util import structure as structure_lib
 from tensorflow.python.data.util import traverse
 from tensorflow.python.eager import context
+from tensorflow.python.eager import function as eager_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import function
@@ -47,6 +48,7 @@ from tensorflow.python.framework import random_seed as core_random_seed
 from tensorflow.python.framework import smart_cond
 from tensorflow.python.framework import sparse_tensor as sparse_tensor_lib
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import tensor_spec
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
@@ -85,12 +87,12 @@ class DatasetV2(object):
     Args:
       variant_tensor: A DT_VARIANT tensor that represents the dataset.
     """
-    self._dataset_variant_tensor = variant_tensor
+    self._variant_tensor_attr = variant_tensor
     self._graph_attr = ops.get_default_graph()
 
   @property
   def _variant_tensor(self):
-    return self._dataset_variant_tensor
+    return self._variant_tensor_attr
 
   @_variant_tensor.setter
   def _variant_tensor(self, _):
@@ -142,6 +144,8 @@ class DatasetV2(object):
     return any(
         [input_dataset._has_captured_ref() for input_dataset in self._inputs()])  # pylint: disable=protected-access
 
+  # TODO(jsimsa): Change this to be the transitive closure of functions used
+  # by this dataset and its inputs.
   def _functions(self):
     """Returns a list of functions associated with this dataset.
 
@@ -1642,6 +1646,43 @@ class DatasetV1(DatasetV2):
           ParallelMapDataset(
               self, map_func, num_parallel_calls, preserve_cardinality=False))
 
+  @deprecation.deprecated(None, "Use `tf.data.Dataset.map()")
+  def map_with_legacy_function(self, map_func, num_parallel_calls=None):
+    """Maps `map_func` across the elements of this dataset.
+
+    NOTE: This is an escape hatch for existing uses of `map` that do not work
+    with V2 functions. New uses are strongly discouraged and existing uses
+    should migrate to `map` as this method will not be removed in V2.
+
+    Args:
+      map_func: A function mapping a nested structure of tensors (having shapes
+        and types defined by `self.output_shapes` and `self.output_types`) to
+        another nested structure of tensors.
+      num_parallel_calls: (Optional.) A `tf.int32` scalar `tf.Tensor`,
+        representing the number elements to process asynchronously in parallel.
+        If not specified, elements will be processed sequentially. If the value
+        `tf.data.experimental.AUTOTUNE` is used, then the number of parallel
+        calls is set dynamically based on available CPU.
+
+    Returns:
+      Dataset: A `Dataset`.
+    """
+    if num_parallel_calls is None:
+      return DatasetV1Adapter(
+          MapDataset(
+              self,
+              map_func,
+              preserve_cardinality=False,
+              use_legacy_function=True))
+    else:
+      return DatasetV1Adapter(
+          ParallelMapDataset(
+              self,
+              map_func,
+              num_parallel_calls,
+              preserve_cardinality=False,
+              use_legacy_function=True))
+
   @functools.wraps(DatasetV2.flat_map)
   def flat_map(self, map_func):
     return DatasetV1Adapter(super(DatasetV1, self).flat_map(map_func))
@@ -2070,9 +2111,9 @@ structure_lib.Structure._register_custom_converter(DatasetV2,
 
 
 class StructuredFunctionWrapper(object):
-  """A wrapper for `Defun` that supports structured arguments and return values.
-  """
+  """A function wrapper that supports structured arguments and return values."""
 
+  # pylint: disable=protected-access
   def __init__(self,
                func,
                transformation_name,
@@ -2082,6 +2123,7 @@ class StructuredFunctionWrapper(object):
                input_types=None,
                input_structure=None,
                add_to_graph=True,
+               use_legacy_function=False,
                defun_kwargs=None):
     """Creates a new `StructuredFunctionWrapper` for the given function.
 
@@ -2103,9 +2145,12 @@ class StructuredFunctionWrapper(object):
         defines the element types and structure for `func` arguments.
       add_to_graph: (Optional.) If `True`, the function will be added to the
         default graph.
+      use_legacy_function: (Optional.) A boolean that determines whether the
+        function be created using `tensorflow.python.eager.function.defun`
+        (default behavior) or `tensorflow.python.eager.function.Defun` (legacy
+        beheavior).
       defun_kwargs: (Optional.) A dictionary mapping string argument names to
-        values. If supplied, will be passed to `function.Defun()` as keyword
-        arguments.
+        values. If supplied, will be passed to `function` as keyword arguments.
 
     Raises:
       ValueError: If an invalid combination of `dataset`, `input_classes`,
@@ -2125,7 +2170,7 @@ class StructuredFunctionWrapper(object):
           raise ValueError("Either `dataset`, `input_structure` or all of "
                            "`input_classes`, `input_shapes`, and `input_types` "
                            "must be specified.")
-        self._input_structure = dataset._element_structure  # pylint: disable=protected-access
+        self._input_structure = dataset._element_structure
     else:
       if not (dataset is None and input_classes is None and input_shapes is None
               and input_types is None):
@@ -2134,24 +2179,38 @@ class StructuredFunctionWrapper(object):
                          "must be specified.")
       self._input_structure = input_structure
 
-    self._transformation_name = transformation_name
+    if defun_kwargs is None:
+      defun_kwargs = {}
+
     readable_transformation_name = transformation_name.replace(
         ".", "_")[:-2] if len(transformation_name) > 2 else ""
-    self._func_name = "_".join([
-        readable_transformation_name,
-        function_utils.get_func_name(func),
-        str(ops.uid())
-    ])
 
-    if defun_kwargs is None:
-      defun_kwargs = {}
+    func_name = "_".join(
+        [readable_transformation_name,
+         function_utils.get_func_name(func)])
+
+    def _warn_if_collections(transformation_name, graph, initial_length):
+      """Prints a warning if the given graph uses common graph collections.
 
-    @function.Defun(
-        *self._input_structure._flat_types, func_name=self._func_name,  # pylint: disable=protected-access
-        **defun_kwargs)
-    def tf_data_structured_function_wrapper(*args):
+      NOTE(mrry): Currently a warning is only generated for lookup tables. Any
+      variables created will be automatically hoisted out to the outermost scope
+      using `init_scope()`. Some collections (such as for control-flow contexts)
+      are benign and should not generate a warning.
+
+      Args:
+        transformation_name: A human-readable name for the transformation.
+        graph: The graph to check for collections.
+        initial_length: The initial length of the lookup table collection.
+      """
+      length = len(graph.get_collection(ops.GraphKeys.TABLE_INITIALIZERS))
+      if length != initial_length:
+        warnings.warn("Creating lookup tables inside a function passed to %s "
+                      "is not supported. Create each table outside the "
+                      "function, and capture it inside the function to use it."
+                      % transformation_name)
+
+    def _wrapper_helper(*args):
       """Wrapper for passing nested structures to and from tf.data functions."""
-      # pylint: disable=protected-access
       nested_args = self._input_structure._from_compatible_tensor_list(args)
       if not _should_unpack_args(nested_args):
         nested_args = (nested_args,)
@@ -2175,18 +2234,53 @@ class StructuredFunctionWrapper(object):
       except (ValueError, TypeError):
         raise TypeError("Unsupported return value from function passed to "
                         "%s: %s." % (transformation_name, ret))
+      return ret
+
+    if use_legacy_function:
+      func_name = func_name + "_" + str(ops.uid())
+
+      @function.Defun(
+          *self._input_structure._flat_types,
+          func_name=func_name,
+          **defun_kwargs)
+      def wrapper_fn(*args):
+        ret = _wrapper_helper(*args)
+        _warn_if_collections(transformation_name, ops.get_default_graph(), 0)
+        return self._output_structure._to_tensor_list(ret)
+
+      self._function = wrapper_fn
+      if add_to_graph:
+        self._function.add_to_graph(ops.get_default_graph())
+      else:
+        # Use the private method that will execute `wrapper_fn` but delay adding
+        # it to the graph in case (e.g.) we need to rerun the function.
+        self._function._create_definition_if_needed()
+    else:
+      defun_kwargs.update({"func_name": func_name})
+
+      @eager_function.defun_with_attributes(
+          input_signature=[
+              tensor_spec.TensorSpec(input_shape, input_type)  # pylint: disable=g-complex-comprehension
+              for input_shape, input_type in zip(
+                  self._input_structure._flat_shapes,
+                  self._input_structure._flat_types)
+          ],
+          attributes=defun_kwargs)
+      def wrapper_fn(*args):  # pylint: disable=missing-docstring
+        ret = _wrapper_helper(*args)
+        ret = self._output_structure._to_tensor_list(ret)
+        return [ops.convert_to_tensor(t) for t in ret]
 
-      _warn_if_collections(transformation_name)
-      return self._output_structure._to_tensor_list(ret)
+      initial_length = len(ops.get_default_graph().get_collection(
+          ops.GraphKeys.TABLE_INITIALIZERS))
 
-    self._function = tf_data_structured_function_wrapper
-    if add_to_graph:
-      self._function.add_to_graph(ops.get_default_graph())
-    else:
-      # Use the private method that will execute
-      # `tf_data_structured_function_wrapper` but delay adding it to the graph
-      # in case (e.g.) we need to rerun the function.
-      self._function._create_definition_if_needed()  # pylint: disable=protected-access
+      self._function = wrapper_fn._get_concrete_function_internal()
+      if add_to_graph:
+        self._function.add_to_graph(ops.get_default_graph())
+
+      _warn_if_collections(transformation_name, self._function.graph,
+                           initial_length)
+  # pylint: enable=protected-access
 
   @property
   def output_structure(self):
@@ -2759,24 +2853,6 @@ def _should_unpack_args(args):
   return type(args) is tuple  # pylint: disable=unidiomatic-typecheck
 
 
-def _warn_if_collections(transformation_name):
-  """Prints warning message if the current graph uses common graph collections.
-
-  NOTE(mrry): Currently a warning is only generated for lookup tables. Any
-  variables created will be automatically hoisted out to the outermost scope
-  using `init_scope()`. Some collections (such as for control-flow contexts)
-  are benign and should not generate a warning.
-
-  Args:
-    transformation_name: A human-readable name for the transformation.
-  """
-  if ops.get_default_graph().get_collection(ops.GraphKeys.TABLE_INITIALIZERS):
-    warnings.warn("Creating lookup tables inside a function passed to %s is not"
-                  " supported. Create each table outside the function, and "
-                  "capture it inside the function to use it."
-                  % transformation_name)
-
-
 class MapDataset(UnaryDataset):
   """A `Dataset` that maps a function over elements in its input."""
 
@@ -2784,13 +2860,17 @@ class MapDataset(UnaryDataset):
                input_dataset,
                map_func,
                use_inter_op_parallelism=True,
-               preserve_cardinality=False):
+               preserve_cardinality=False,
+               use_legacy_function=False):
     """See `Dataset.map()` for details."""
     self._input_dataset = input_dataset
     self._use_inter_op_parallelism = use_inter_op_parallelism
     self._preserve_cardinality = preserve_cardinality
     self._map_func = StructuredFunctionWrapper(
-        map_func, self._transformation_name(), dataset=input_dataset)
+        map_func,
+        self._transformation_name(),
+        dataset=input_dataset,
+        use_legacy_function=use_legacy_function)
     variant_tensor = gen_dataset_ops.map_dataset(
         input_dataset._variant_tensor,  # pylint: disable=protected-access
         self._map_func.function.captured_inputs,
@@ -2819,12 +2899,16 @@ class ParallelMapDataset(UnaryDataset):
                map_func,
                num_parallel_calls,
                use_inter_op_parallelism=True,
-               preserve_cardinality=False):
+               preserve_cardinality=False,
+               use_legacy_function=False):
     """See `Dataset.map()` for details."""
     self._input_dataset = input_dataset
     self._use_inter_op_parallelism = use_inter_op_parallelism
     self._map_func = StructuredFunctionWrapper(
-        map_func, self._transformation_name(), dataset=input_dataset)
+        map_func,
+        self._transformation_name(),
+        dataset=input_dataset,
+        use_legacy_function=use_legacy_function)
     self._num_parallel_calls = ops.convert_to_tensor(
         num_parallel_calls, dtype=dtypes.int32, name="num_parallel_calls")
     self._preserve_cardinality = preserve_cardinality
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.-dataset.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.-dataset.pbtxt
index f7d388d33d..e638d822bc 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.-dataset.pbtxt
@@ -79,6 +79,10 @@ tf_class {
     name: "map"
     argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "map_with_legacy_function"
+    argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "options"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.-fixed-length-record-dataset.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.-fixed-length-record-dataset.pbtxt
index d73168b070..c0952b6f9a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.-fixed-length-record-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.-fixed-length-record-dataset.pbtxt
@@ -81,6 +81,10 @@ tf_class {
     name: "map"
     argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "map_with_legacy_function"
+    argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "options"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.-t-f-record-dataset.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.-t-f-record-dataset.pbtxt
index 51224cd6b4..a509ba5f9a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.-t-f-record-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.-t-f-record-dataset.pbtxt
@@ -81,6 +81,10 @@ tf_class {
     name: "map"
     argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "map_with_legacy_function"
+    argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "options"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.-text-line-dataset.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.-text-line-dataset.pbtxt
index a10add1b7e..c27998e8c9 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.-text-line-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.-text-line-dataset.pbtxt
@@ -81,6 +81,10 @@ tf_class {
     name: "map"
     argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "map_with_legacy_function"
+    argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "options"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-csv-dataset.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-csv-dataset.pbtxt
index 71b597c19c..2a899d8d06 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-csv-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-csv-dataset.pbtxt
@@ -81,6 +81,10 @@ tf_class {
     name: "map"
     argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "map_with_legacy_function"
+    argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "options"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-random-dataset.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-random-dataset.pbtxt
index 20646e87b5..3c4727f959 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-random-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-random-dataset.pbtxt
@@ -81,6 +81,10 @@ tf_class {
     name: "map"
     argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "map_with_legacy_function"
+    argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "options"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-sql-dataset.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-sql-dataset.pbtxt
index 86c5ff5b0b..353c8fc618 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-sql-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-sql-dataset.pbtxt
@@ -81,6 +81,10 @@ tf_class {
     name: "map"
     argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "map_with_legacy_function"
+    argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "options"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
index abc98a74b6..df58bef739 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
@@ -152,6 +152,10 @@ tf_module {
     name: "map_and_batch"
     argspec: "args=[\'map_func\', \'batch_size\', \'num_parallel_batches\', \'drop_remainder\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
+  member_method {
+    name: "map_and_batch_with_legacy_function"
+    argspec: "args=[\'map_func\', \'batch_size\', \'num_parallel_batches\', \'drop_remainder\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "parallel_interleave"
     argspec: "args=[\'map_func\', \'cycle_length\', \'block_length\', \'sloppy\', \'buffer_output_elements\', \'prefetch_input_elements\'], varargs=None, keywords=None, defaults=[\'1\', \'False\', \'None\', \'None\'], "
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 18271ef87f..7e86d6cfa5 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -710,6 +710,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             'tf.compat.v1.keras.initializers.ones',
         'tf.keras.initializers.constant':
             'tf.compat.v1.keras.initializers.constant',
+        "tf.data.experimental.map_and_batch_with_legacy_function":
+            "tf.compat.v1.data.experimental.map_and_batch_with_legacy_function",
     }
     # pylint: enable=line-too-long
 
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index ede0ccfe17..bf1e80a681 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -1037,6 +1037,13 @@ tf.print('abc')
       _, unused_report, unused_errors, actual = self._upgrade(text)
       self.assertEqual(actual, expected)
 
+  def testMapAndBatch(self):
+    suffix = ".data.experimental.map_and_batch_with_legacy_function(args)"
+    text = "tf" + suffix
+    expected = "tf.compat.v1" + suffix
+    _, unused_report, unused_errors, actual = self._upgrade(text)
+    self.assertEqual(actual, expected)
+
   def testCast(self):
     for (name, dtype) in [("int32", "int32"),
                           ("int64", "int64"),
-- 
GitLab


From 243118bbbfc3d3d5c1f2d19fb70a3e61786e1655 Mon Sep 17 00:00:00 2001
From: Noah Eisen <ncteisen@google.com>
Date: Thu, 7 Feb 2019 12:51:13 -0800
Subject: [PATCH 2084/2345] Implement duplicate checking on Master methods

PiperOrigin-RevId: 232927103
---
 tensorflow/core/distributed_runtime/BUILD     |  4 ++-
 tensorflow/core/distributed_runtime/master.cc | 27 +++++++++++++-
 tensorflow/core/distributed_runtime/master.h  |  4 +++
 .../distributed_runtime/master_interface.h    |  5 ++-
 .../distributed_runtime/message_wrappers.cc   | 10 ++++++
 .../distributed_runtime/message_wrappers.h    |  6 ++++
 .../distributed_runtime/recent_request_ids.cc | 35 ++++++++++++++-----
 .../distributed_runtime/recent_request_ids.h  |  6 ++++
 tensorflow/core/distributed_runtime/rpc/BUILD |  1 +
 .../distributed_runtime/rpc/grpc_session.cc   |  4 +++
 tensorflow/core/protobuf/master.proto         | 33 +++++++++++++----
 11 files changed, 117 insertions(+), 18 deletions(-)

diff --git a/tensorflow/core/distributed_runtime/BUILD b/tensorflow/core/distributed_runtime/BUILD
index da88f9cb93..1f85bd8e78 100644
--- a/tensorflow/core/distributed_runtime/BUILD
+++ b/tensorflow/core/distributed_runtime/BUILD
@@ -17,7 +17,6 @@ filegroup(
 
 load("//tensorflow:tensorflow.bzl", "tf_cc_test", "tf_cuda_library")
 load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_test")
-load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_tests")
 load("//tensorflow:tensorflow.bzl", "tf_copts")
 
 # For platform specific build config
@@ -298,6 +297,7 @@ cc_library(
     deps = [
         ":call_options",
         ":message_wrappers",
+        ":request_id",
         "//tensorflow/core:lib",
         "//tensorflow/core:master_proto_cc",
     ],
@@ -311,6 +311,7 @@ cc_library(
         ":call_options",
         ":master_env",
         ":master_session",
+        ":recent_request_ids",
         ":remote_device",
         ":worker_cache",
         ":worker_interface",
@@ -765,6 +766,7 @@ cc_library(
     srcs = ["recent_request_ids.cc"],
     hdrs = ["recent_request_ids.h"],
     deps = [
+        ":message_wrappers",
         "//tensorflow/core:lib",
         "//tensorflow/core:worker_proto_cc",
     ],
diff --git a/tensorflow/core/distributed_runtime/master.cc b/tensorflow/core/distributed_runtime/master.cc
index 269f620e42..fc8d2871ac 100644
--- a/tensorflow/core/distributed_runtime/master.cc
+++ b/tensorflow/core/distributed_runtime/master.cc
@@ -65,7 +65,8 @@ Master::Master(MasterEnv* env, double session_gc_seconds)
     : env_(env),
       last_1000_steps_(1000),
       step_count_(0),
-      session_gc_seconds_(session_gc_seconds) {
+      session_gc_seconds_(session_gc_seconds),
+      recent_request_ids_(10000) {
   // Right now, a master service must be co-located with a device.
   // Otherwise, fetches do not work.
   CHECK(!env->local_devices.empty());
@@ -510,6 +511,12 @@ void Master::ExtendSession(const ExtendSessionRequest* req,
 
 void Master::PartialRunSetup(const PartialRunSetupRequest* req,
                              PartialRunSetupResponse* resp, MyClosure done) {
+  Status s = recent_request_ids_.TrackUnique(req->request_id(),
+                                             "PartialRunSetup (Master)", *req);
+  if (!s.ok()) {
+    done(s);
+    return;
+  }
   auto session = FindMasterSession(req->session_handle());
   if (session == nullptr) {
     done(errors::Aborted("Session ", req->session_handle(), " is not found."));
@@ -525,6 +532,12 @@ void Master::PartialRunSetup(const PartialRunSetupRequest* req,
 
 void Master::RunStep(CallOptions* opts, const RunStepRequestWrapper* req,
                      MutableRunStepResponseWrapper* resp, MyClosure done) {
+  Status s = recent_request_ids_.TrackUnique(req->request_id(),
+                                             "RunStep (Master)", req);
+  if (!s.ok()) {
+    done(s);
+    return;
+  }
   auto start_time = env_->env->NowMicros();
   auto session = FindMasterSession(req->session_handle());
   if (session == nullptr) {
@@ -664,6 +677,12 @@ void Master::Reset(const ResetRequest* req, ResetResponse* resp,
 
 void Master::MakeCallable(const MakeCallableRequest* req,
                           MakeCallableResponse* resp, MyClosure done) {
+  Status s = recent_request_ids_.TrackUnique(req->request_id(),
+                                             "MakeCallable (Master)", *req);
+  if (!s.ok()) {
+    done(s);
+    return;
+  }
   auto session = FindMasterSession(req->session_handle());
   if (session == nullptr) {
     done(errors::Aborted("Session ", req->session_handle(), " is not found."));
@@ -681,6 +700,12 @@ void Master::MakeCallable(const MakeCallableRequest* req,
 
 void Master::RunCallable(CallOptions* opts, const RunCallableRequest* req,
                          RunCallableResponse* resp, MyClosure done) {
+  Status s = recent_request_ids_.TrackUnique(req->request_id(),
+                                             "RunCallable (Master)", *req);
+  if (!s.ok()) {
+    done(s);
+    return;
+  }
   auto session = FindMasterSession(req->session_handle());
   if (session == nullptr) {
     done(errors::Aborted("Session ", req->session_handle(), " is not found."));
diff --git a/tensorflow/core/distributed_runtime/master.h b/tensorflow/core/distributed_runtime/master.h
index dbb337fd48..0524582ac7 100644
--- a/tensorflow/core/distributed_runtime/master.h
+++ b/tensorflow/core/distributed_runtime/master.h
@@ -22,6 +22,7 @@ limitations under the License.
 #include "tensorflow/core/distributed_runtime/call_options.h"
 #include "tensorflow/core/distributed_runtime/master_env.h"
 #include "tensorflow/core/distributed_runtime/master_session.h"
+#include "tensorflow/core/distributed_runtime/recent_request_ids.h"
 #include "tensorflow/core/lib/core/notification.h"
 #include "tensorflow/core/lib/gtl/map_util.h"
 #include "tensorflow/core/platform/macros.h"
@@ -95,6 +96,9 @@ class Master {
   // closed automatically.
   const double session_gc_seconds_;
 
+  // Used to track ids for incoming requests so we can detect duplicates.
+  RecentRequestIds recent_request_ids_;
+
   // Call CleanupAll on all workers.
   void CleanupWorkers(const ResetRequest& reset);
 
diff --git a/tensorflow/core/distributed_runtime/master_interface.h b/tensorflow/core/distributed_runtime/master_interface.h
index a8ae3cba3c..cde47fb9ca 100644
--- a/tensorflow/core/distributed_runtime/master_interface.h
+++ b/tensorflow/core/distributed_runtime/master_interface.h
@@ -18,6 +18,7 @@ limitations under the License.
 
 #include "tensorflow/core/distributed_runtime/call_options.h"
 #include "tensorflow/core/distributed_runtime/message_wrappers.h"
+#include "tensorflow/core/distributed_runtime/request_id.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/protobuf/master.pb.h"
@@ -66,7 +67,9 @@ class MasterInterface {
   // The message returned from this method must only be used in a
   // `RunStep()` call on the same `MasterInterface` instance.
   virtual MutableRunStepRequestWrapper* CreateRunStepRequest() {
-    return new MutableProtoRunStepRequest;
+    MutableProtoRunStepRequest* ret = new MutableProtoRunStepRequest;
+    ret->request_.set_request_id(GetUniqueRequestId());
+    return ret;
   }
 
   // Returns a response object for use in calls to
diff --git a/tensorflow/core/distributed_runtime/message_wrappers.cc b/tensorflow/core/distributed_runtime/message_wrappers.cc
index 40bf564cab..c9bc558964 100644
--- a/tensorflow/core/distributed_runtime/message_wrappers.cc
+++ b/tensorflow/core/distributed_runtime/message_wrappers.cc
@@ -97,6 +97,10 @@ bool InMemoryRunStepRequest::store_errors_in_response_body() const {
   return store_errors_in_response_body_;
 }
 
+int64 InMemoryRunStepRequest::request_id() const {
+  return 0;  // no need to track request id for local version.
+}
+
 void InMemoryRunStepRequest::set_store_errors_in_response_body(
     bool store_errors) {
   store_errors_in_response_body_ = store_errors;
@@ -210,6 +214,10 @@ void MutableProtoRunStepRequest::set_store_errors_in_response_body(
   request_.set_store_errors_in_response_body(store_errors);
 }
 
+int64 MutableProtoRunStepRequest::request_id() const {
+  return request_.request_id();
+}
+
 string MutableProtoRunStepRequest::DebugString() const {
   return request_.DebugString();
 }
@@ -272,6 +280,8 @@ bool ProtoRunStepRequest::store_errors_in_response_body() const {
   return request_->store_errors_in_response_body();
 }
 
+int64 ProtoRunStepRequest::request_id() const { return request_->request_id(); }
+
 string ProtoRunStepRequest::DebugString() const {
   return request_->DebugString();
 }
diff --git a/tensorflow/core/distributed_runtime/message_wrappers.h b/tensorflow/core/distributed_runtime/message_wrappers.h
index 92a9994ac3..2cdbd1bfaf 100644
--- a/tensorflow/core/distributed_runtime/message_wrappers.h
+++ b/tensorflow/core/distributed_runtime/message_wrappers.h
@@ -87,6 +87,8 @@ class RunStepRequestWrapper {
   // truncate long metadata messages.
   virtual bool store_errors_in_response_body() const = 0;
 
+  virtual int64 request_id() const = 0;
+
   // Returns a human-readable representation of this message for debugging.
   virtual string DebugString() const = 0;
 
@@ -127,6 +129,7 @@ class InMemoryRunStepRequest : public MutableRunStepRequestWrapper {
   string DebugString() const override;
   const RunStepRequest& ToProto() const override;
   bool store_errors_in_response_body() const override;
+  int64 request_id() const override;
 
   // MutableRunStepRequestWrapper methods.
   void set_session_handle(const string& handle) override;
@@ -177,6 +180,7 @@ class MutableProtoRunStepRequest : public MutableRunStepRequestWrapper {
   string DebugString() const override;
   const RunStepRequest& ToProto() const override;
   bool store_errors_in_response_body() const override;
+  int64 request_id() const override;
 
   // MutableRunStepRequestWrapper methods.
   void set_session_handle(const string& handle) override;
@@ -189,6 +193,7 @@ class MutableProtoRunStepRequest : public MutableRunStepRequestWrapper {
 
  private:
   RunStepRequest request_;
+  friend class MasterInterface;
 };
 
 // Wrapper for immutable RunStep requests that use a non-owned
@@ -216,6 +221,7 @@ class ProtoRunStepRequest : public RunStepRequestWrapper {
   string DebugString() const override;
   const RunStepRequest& ToProto() const override;
   bool store_errors_in_response_body() const override;
+  int64 request_id() const override;
 
  private:
   const RunStepRequest* const request_;  // Not owned.
diff --git a/tensorflow/core/distributed_runtime/recent_request_ids.cc b/tensorflow/core/distributed_runtime/recent_request_ids.cc
index 4f6866c5d1..2c953e12c0 100644
--- a/tensorflow/core/distributed_runtime/recent_request_ids.cc
+++ b/tensorflow/core/distributed_runtime/recent_request_ids.cc
@@ -28,12 +28,10 @@ RecentRequestIds::RecentRequestIds(int num_tracked_request_ids)
   set_.reserve(num_tracked_request_ids);
 }
 
-Status RecentRequestIds::TrackUnique(int64 request_id,
-                                     const string& method_name,
-                                     const protobuf::Message& request) {
+bool RecentRequestIds::Insert(int64 request_id) {
   if (request_id == 0) {
     // For backwards compatibility, allow all requests with request_id 0.
-    return Status::OK();
+    return true;
   }
 
   mutex_lock l(mu_);
@@ -43,9 +41,7 @@ Status RecentRequestIds::TrackUnique(int64 request_id,
     // request_id's age in the circular_buffer_ if it's tracked again. Strict
     // LRU is not useful here because returning this error will close the
     // current Session.
-    return errors::Aborted("The same ", method_name,
-                           " request was received twice. ",
-                           request.ShortDebugString());
+    return false;
   }
 
   // Remove the oldest request_id from the set_. circular_buffer_ is
@@ -54,7 +50,30 @@ Status RecentRequestIds::TrackUnique(int64 request_id,
   set_.erase(circular_buffer_[next_index_]);
   circular_buffer_[next_index_] = request_id;
   next_index_ = (next_index_ + 1) % circular_buffer_.size();
-  return Status::OK();
+  return true;
+}
+
+Status RecentRequestIds::TrackUnique(int64 request_id,
+                                     const string& method_name,
+                                     const protobuf::Message& request) {
+  if (Insert(request_id)) {
+    return Status::OK();
+  } else {
+    return errors::Aborted("The same ", method_name,
+                           " request was received twice. ",
+                           request.ShortDebugString());
+  }
+}
+Status RecentRequestIds::TrackUnique(int64 request_id,
+                                     const string& method_name,
+                                     const RunStepRequestWrapper* wrapper) {
+  if (Insert(request_id)) {
+    return Status::OK();
+  } else {
+    return errors::Aborted("The same ", method_name,
+                           " request was received twice. ",
+                           wrapper->ToProto().ShortDebugString());
+  }
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/distributed_runtime/recent_request_ids.h b/tensorflow/core/distributed_runtime/recent_request_ids.h
index 11cf937c94..4094fcbde7 100644
--- a/tensorflow/core/distributed_runtime/recent_request_ids.h
+++ b/tensorflow/core/distributed_runtime/recent_request_ids.h
@@ -20,6 +20,7 @@ limitations under the License.
 #include <unordered_set>
 #include <vector>
 
+#include "tensorflow/core/distributed_runtime/message_wrappers.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/platform/protobuf.h"
@@ -58,8 +59,13 @@ class RecentRequestIds {
   // ShortDebugString are added to returned errors.
   Status TrackUnique(int64 request_id, const string& method_name,
                      const protobuf::Message& request);
+  // Overloaded versions of the above function for wrapped protos.
+  Status TrackUnique(int64 request_id, const string& method_name,
+                     const RunStepRequestWrapper* wrapper);
 
  private:
+  bool Insert(int64 request_id);
+
   mutex mu_;
   // next_index_ indexes into circular_buffer_, and points to the next storage
   // space to use. When the buffer is full, next_index_ points at the oldest
diff --git a/tensorflow/core/distributed_runtime/rpc/BUILD b/tensorflow/core/distributed_runtime/rpc/BUILD
index 607963484d..3f261119b6 100644
--- a/tensorflow/core/distributed_runtime/rpc/BUILD
+++ b/tensorflow/core/distributed_runtime/rpc/BUILD
@@ -408,6 +408,7 @@ cc_library(
         "//tensorflow/core/distributed_runtime:local_master",
         "//tensorflow/core/distributed_runtime:master_interface",
         "//tensorflow/core/distributed_runtime:message_wrappers",
+        "//tensorflow/core/distributed_runtime:request_id",
     ],
     alwayslink = 1,
 )
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_session.cc b/tensorflow/core/distributed_runtime/rpc/grpc_session.cc
index 32063fecbb..c14bfd2155 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_session.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_session.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include "tensorflow/core/distributed_runtime/call_options.h"
 #include "tensorflow/core/distributed_runtime/local_master.h"
 #include "tensorflow/core/distributed_runtime/master_interface.h"
+#include "tensorflow/core/distributed_runtime/request_id.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_channel.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_remote_master.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
@@ -312,6 +313,7 @@ Status GrpcSession::PRunSetup(const std::vector<string>& input_names,
   for (const string& target : target_nodes) {
     req.add_target(target);
   }
+  req.set_request_id(GetUniqueRequestId());
   call_options.SetTimeout(options_.config.operation_timeout_in_ms());
   TF_RETURN_IF_ERROR(master_->PartialRunSetup(&call_options, &req, &resp));
   *handle = resp.partial_run_handle();
@@ -408,6 +410,7 @@ Status GrpcSession::MakeCallable(const CallableOptions& callable_options,
   MakeCallableRequest req;
   TF_RETURN_IF_ERROR(Handle(req.mutable_session_handle()));
   *req.mutable_options() = callable_options;
+  req.set_request_id(GetUniqueRequestId());
   MakeCallableResponse resp;
   CallOptions call_options;
   call_options.SetTimeout(options_.config.operation_timeout_in_ms());
@@ -423,6 +426,7 @@ Status GrpcSession::RunCallable(CallableHandle handle,
   RunCallableRequest req;
   TF_RETURN_IF_ERROR(Handle(req.mutable_session_handle()));
   req.set_handle(handle);
+  req.set_request_id(GetUniqueRequestId());
   for (const Tensor& feed : feed_tensors) {
     feed.AsProtoTensorContent(req.mutable_feed()->Add());
   }
diff --git a/tensorflow/core/protobuf/master.proto b/tensorflow/core/protobuf/master.proto
index c104463c51..4a998c5bfc 100644
--- a/tensorflow/core/protobuf/master.proto
+++ b/tensorflow/core/protobuf/master.proto
@@ -16,11 +16,13 @@ limitations under the License.
 syntax = "proto3";
 
 package tensorflow;
+
 option cc_enable_arenas = true;
 option java_outer_classname = "DistributedRuntimeProtos";
 option java_multiple_files = true;
 option java_package = "org.tensorflow.distruntime";
-option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf";
+
+// add go_package externally with copybara
 import "tensorflow/core/framework/device_attributes.proto";
 import "tensorflow/core/framework/graph.proto";
 import "tensorflow/core/framework/tensor.proto";
@@ -138,6 +140,11 @@ message RunStepRequest {
   // response body. This is a workaround since the RPC subsystem may
   // truncate long metadata messages.
   bool store_errors_in_response_body = 7;
+
+  // Unique identifier for this request. Every RunStepRequest must
+  // have a unique request_id, and retried RunStepRequest must have
+  // the same request_id. If request_id is zero, retry detection is disabled.
+  int64 request_id = 8;
 }
 
 message RunStepResponse {
@@ -183,6 +190,11 @@ message PartialRunSetupRequest {
   // Target Nodes. A list of node names. The named nodes will be run in future
   // steps, but their outputs will not be fetched.
   repeated string target = 4;
+
+  // Unique identifier for this request. Every PartialRunSetupRequest must
+  // have a unique request_id, and retried PartialRunSetupRequest must have
+  // the same request_id. If request_id is zero, retry detection is disabled.
+  int64 request_id = 5;
 }
 
 message PartialRunSetupResponse {
@@ -204,8 +216,7 @@ message CloseSessionRequest {
   string session_handle = 1;
 }
 
-message CloseSessionResponse {
-}
+message CloseSessionResponse {}
 
 // Reset() allows misbehaving or slow sessions to be aborted and closed, and
 // causes their resources eventually to be released.  Reset() does not wait
@@ -237,8 +248,7 @@ message ResetRequest {
   repeated string device_filters = 2;
 }
 
-message ResetResponse {
-}
+message ResetResponse {}
 
 ////////////////////////////////////////////////////////////////////////////////
 //
@@ -279,6 +289,11 @@ message MakeCallableRequest {
 
   // Options that define the behavior of the created callable.
   CallableOptions options = 2;
+
+  // Unique identifier for this request. Every MakeCallableRequest must
+  // have a unique request_id, and retried MakeCallableRequest must have
+  // the same request_id. If request_id is zero, retry detection is disabled.
+  int64 request_id = 3;
 }
 
 message MakeCallableResponse {
@@ -303,6 +318,11 @@ message RunCallableRequest {
   // Values of the tensors passed as arguments to the callable, in the order
   // defined in the CallableOptions.feed field passed to MakeCallable.
   repeated TensorProto feed = 3;
+
+  // Unique identifier for this request. Every RunCallableRequest must
+  // have a unique request_id, and retried RunCallableRequest must have
+  // the same request_id. If request_id is zero, retry detection is disabled.
+  int64 request_id = 4;
 }
 
 message RunCallableResponse {
@@ -330,5 +350,4 @@ message ReleaseCallableRequest {
   int64 handle = 2;
 }
 
-message ReleaseCallableResponse {
-}
+message ReleaseCallableResponse {}
-- 
GitLab


From 1ae58bc787db0d265da1e85f5acb9fa4ee701fb9 Mon Sep 17 00:00:00 2001
From: nammbash <niroopshankar.ammbashankar@intel.com>
Date: Thu, 7 Feb 2019 13:02:21 -0800
Subject: [PATCH 2085/2345] Revert "additional golden files for fixing"

This reverts commit f47f02cec0ec05a823f5ac8b5fc8373d598d7f1a.
---
 .../golden/v2/tensorflow.estimator.-mode-keys.pbtxt  |  2 +-
 .../v2/tensorflow.estimator.experimental.pbtxt       | 12 ++++++++++++
 .../api/golden/v2/tensorflow.estimator.inputs.pbtxt  | 11 -----------
 .../tools/api/golden/v2/tensorflow.estimator.pbtxt   |  4 ----
 4 files changed, 13 insertions(+), 16 deletions(-)
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.estimator.inputs.pbtxt

diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-mode-keys.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-mode-keys.pbtxt
index bf7c1abcd8..f3dfe7296f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-mode-keys.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-mode-keys.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.estimator.ModeKeys"
 tf_class {
-  is_instance: "<class \'tensorflow_estimator.python.estimator.model_fn.ModeKeys\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.mode_keys.ModeKeysV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "EVAL"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.pbtxt
index f0fd7ce782..b1bd5a2661 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.pbtxt
@@ -36,4 +36,16 @@ tf_module {
     name: "stop_if_higher_hook"
     argspec: "args=[\'estimator\', \'metric_name\', \'threshold\', \'eval_dir\', \'min_steps\', \'run_every_secs\', \'run_every_steps\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'60\', \'None\'], "
   }
+  member_method {
+    name: "stop_if_lower_hook"
+    argspec: "args=[\'estimator\', \'metric_name\', \'threshold\', \'eval_dir\', \'min_steps\', \'run_every_secs\', \'run_every_steps\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'60\', \'None\'], "
+  }
+  member_method {
+    name: "stop_if_no_decrease_hook"
+    argspec: "args=[\'estimator\', \'metric_name\', \'max_steps_without_decrease\', \'eval_dir\', \'min_steps\', \'run_every_secs\', \'run_every_steps\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'60\', \'None\'], "
+  }
+  member_method {
+    name: "stop_if_no_increase_hook"
+    argspec: "args=[\'estimator\', \'metric_name\', \'max_steps_without_increase\', \'eval_dir\', \'min_steps\', \'run_every_secs\', \'run_every_steps\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'60\', \'None\'], "
+  }
 }
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.inputs.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.inputs.pbtxt
deleted file mode 100644
index b318fea1f8..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.inputs.pbtxt
+++ /dev/null
@@ -1,11 +0,0 @@
-path: "tensorflow.estimator.inputs"
-tf_module {
-  member_method {
-    name: "numpy_input_fn"
-    argspec: "args=[\'x\', \'y\', \'batch_size\', \'num_epochs\', \'shuffle\', \'queue_capacity\', \'num_threads\'], varargs=None, keywords=None, defaults=[\'None\', \'128\', \'1\', \'None\', \'1000\', \'1\'], "
-  }
-  member_method {
-    name: "pandas_input_fn"
-    argspec: "args=[\'x\', \'y\', \'batch_size\', \'num_epochs\', \'shuffle\', \'queue_capacity\', \'num_threads\', \'target_column\'], varargs=None, keywords=None, defaults=[\'None\', \'128\', \'1\', \'None\', \'1000\', \'1\', \'target\'], "
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.pbtxt
index 6f57505afe..add8ef5e65 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.pbtxt
@@ -180,10 +180,6 @@ tf_module {
     name: "export"
     mtype: "<type \'module\'>"
   }
-  member {
-    name: "inputs"
-    mtype: "<type \'module\'>"
-  }
   member_method {
     name: "add_metrics"
     argspec: "args=[\'estimator\', \'metric_fn\'], varargs=None, keywords=None, defaults=None"
-- 
GitLab


From 25fda0885d09ec9756a451840ea3a40a2978070a Mon Sep 17 00:00:00 2001
From: "Wen-Heng (Jack) Chung" <whchung@gmail.com>
Date: Fri, 1 Feb 2019 18:54:23 +0000
Subject: [PATCH 2086/2345] Finalize GPU kernels early with -fno-gpu-rdc

- finalize GPU kernels at compile-time with -fno-gpu-rdc
- abolish using hipcc as linker
- use gcc as linker, link with HIP/HCC runtime
- do NOT use --gc-sections at link time to prevese .kernel section for GPU kernels

- amend missing bazel logic between downstream ROCm fork and TF mainline
---
 .../gpus/crosstool/CROSSTOOL_hipcc.tpl        |  6 +-
 .../bin/crosstool_wrapper_driver_rocm.tpl     | 70 ++++++++++++-------
 third_party/gpus/rocm_configure.bzl           | 37 ++++++++++
 3 files changed, 87 insertions(+), 26 deletions(-)

diff --git a/third_party/gpus/crosstool/CROSSTOOL_hipcc.tpl b/third_party/gpus/crosstool/CROSSTOOL_hipcc.tpl
index 0e175b3ef6..221ccb2fe4 100644
--- a/third_party/gpus/crosstool/CROSSTOOL_hipcc.tpl
+++ b/third_party/gpus/crosstool/CROSSTOOL_hipcc.tpl
@@ -152,7 +152,11 @@ toolchain {
     # Removal of unused code and data at link time (can this increase binary size in some cases?).
     compiler_flag: "-ffunction-sections"
     compiler_flag: "-fdata-sections"
-    linker_flag: "-Wl,--gc-sections"
+
+    # With hipcc -fno-gpu-rdc, objects and libraries would contain a .kernel
+    # section which would be removed by ld. Therefore --gc-sections shall be
+    # abolished here.
+    #linker_flag: "-Wl,--gc-sections"
   }
   linking_mode_flags { mode: DYNAMIC }
 }
diff --git a/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_rocm.tpl b/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_rocm.tpl
index 57c7b635c4..c159531263 100755
--- a/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_rocm.tpl
+++ b/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_rocm.tpl
@@ -30,6 +30,12 @@ GCC_HOST_COMPILER_PATH = ('%{gcc_host_compiler_path}')
 
 HIPCC_PATH = '%{hipcc_path}'
 PREFIX_DIR = os.path.dirname(GCC_HOST_COMPILER_PATH)
+HIPCC_ENV = '%{hipcc_env}'
+HIP_RUNTIME_PATH = '%{hip_runtime_path}'
+HIP_RUNTIME_LIBRARY = '%{hip_runtime_library}'
+HCC_RUNTIME_PATH = '%{hcc_runtime_path}'
+HCC_RUNTIME_LIBRARY = '%{hcc_runtime_library}'
+VERBOSE = '%{crosstool_verbose}'=='1'
 
 def Log(s):
   print('gpus/crosstool: {0}'.format(s))
@@ -165,6 +171,11 @@ def InvokeHipcc(argv, log=False):
 
   hipccopts = ' '
   hipccopts += ' ' + hipcc_compiler_options
+  # Use -fno-gpu-rdc by default for early GPU kernel finalization
+  # This flag would trigger GPU kernels be generated at compile time, instead
+  # of link time. This allows the default host compiler (gcc) be used as the
+  # linker for TensorFlow on ROCm platform.
+  hipccopts += ' -fno-gpu-rdc '
   hipccopts += undefines
   hipccopts += defines
   hipccopts += std_options
@@ -205,37 +216,46 @@ def main():
   args, leftover = parser.parse_known_args(sys.argv[1:])
 
   if args.x and args.x[0] == 'rocm':
-    # XXX use hipcc to link
-    if args.pass_exit_codes:
-      gpu_compiler_flags = [flag for flag in sys.argv[1:]
-                                 if not flag.startswith(('-pass-exit-codes'))]
-
-      # special handling for $ORIGIN
-      # - guard every argument with ''
-      modified_gpu_compiler_flags = []
-      for flag in gpu_compiler_flags:
-        modified_gpu_compiler_flags.append("'" + flag + "'")
-
-      if args.rocm_log: Log('Link with hipcc: %s' % (' '.join([HIPCC_PATH] + modified_gpu_compiler_flags)))
-      return subprocess.call([HIPCC_PATH] + modified_gpu_compiler_flags)
-
+    # compilation for GPU objects
     if args.rocm_log: Log('-x rocm')
     leftover = [pipes.quote(s) for s in leftover]
     if args.rocm_log: Log('using hipcc')
     return InvokeHipcc(leftover, log=args.rocm_log)
 
-  # Strip our flags before passing through to the CPU compiler for files which
-  # are not -x rocm. We can't just pass 'leftover' because it also strips -x.
-  # We not only want to pass -x to the CPU compiler, but also keep it in its
-  # relative location in the argv list (the compiler is actually sensitive to
-  # this).
-  cpu_compiler_flags = [flag for flag in sys.argv[1:]
-                             if not flag.startswith(('--rocm_log'))]
+  elif args.pass_exit_codes:
+    # link
+    # with hipcc compiler invoked with -fno-gpu-rdc by default now, it's ok to 
+    # use host compiler as linker, but we have to link with HCC/HIP runtime.
+    # Such restriction would be revised further as the bazel script get
+    # improved to fine tune dependencies to ROCm libraries.
+    gpu_linker_flags = [flag for flag in sys.argv[1:]
+                               if not flag.startswith(('--rocm_log'))]
+
+    gpu_linker_flags.append('-L' + HCC_RUNTIME_PATH)
+    gpu_linker_flags.append('-Wl,-rpath=' + HCC_RUNTIME_PATH)
+    gpu_linker_flags.append('-l' + HCC_RUNTIME_LIBRARY)
+    gpu_linker_flags.append('-L' + HIP_RUNTIME_PATH)
+    gpu_linker_flags.append('-Wl,-rpath=' + HIP_RUNTIME_PATH)
+    gpu_linker_flags.append('-l' + HIP_RUNTIME_LIBRARY)
+
+    if VERBOSE: print(' '.join([CPU_COMPILER] + gpu_linker_flags))
+    return subprocess.call([CPU_COMPILER] + gpu_linker_flags)
 
-  # XXX: SE codes need to be built with gcc, but need this macro defined
-  cpu_compiler_flags.append("-D__HIP_PLATFORM_HCC__")
-
-  return subprocess.call([CPU_COMPILER] + cpu_compiler_flags)
+  else:
+    # compilation for host objects
+
+    # Strip our flags before passing through to the CPU compiler for files which
+    # are not -x rocm. We can't just pass 'leftover' because it also strips -x.
+    # We not only want to pass -x to the CPU compiler, but also keep it in its
+    # relative location in the argv list (the compiler is actually sensitive to
+    # this).
+    cpu_compiler_flags = [flag for flag in sys.argv[1:]
+                               if not flag.startswith(('--rocm_log'))]
+
+    # XXX: SE codes need to be built with gcc, but need this macro defined
+    cpu_compiler_flags.append("-D__HIP_PLATFORM_HCC__")
+    if VERBOSE: print(' '.join([CPU_COMPILER] + cpu_compiler_flags))
+    return subprocess.call([CPU_COMPILER] + cpu_compiler_flags)
 
 if __name__ == '__main__':
   sys.exit(main())
diff --git a/third_party/gpus/rocm_configure.bzl b/third_party/gpus/rocm_configure.bzl
index f67fb8ae38..fa6b942164 100644
--- a/third_party/gpus/rocm_configure.bzl
+++ b/third_party/gpus/rocm_configure.bzl
@@ -232,6 +232,37 @@ def _amdgpu_targets(repository_ctx):
             auto_configure_fail("Invalid AMDGPU target: %s" % amdgpu_target)
     return amdgpu_targets
 
+def _hipcc_env(repository_ctx):
+    """Returns the environment variable string for hipcc.
+
+    Args:
+        repository_ctx: The repository context.
+
+    Returns:
+        A string containing environment variables for hipcc.
+    """
+    hipcc_env = ""
+    for name in ["HIP_CLANG_PATH", "DEVICE_LIB_PATH", "HIP_VDI_HOME",\
+                 "HIPCC_VERBOSE", "HIPCC_COMPILE_FLAGS_APPEND"]:
+        if name in repository_ctx.os.environ:
+            hipcc_env = hipcc_env + " " + name + "=\"" + \
+                    repository_ctx.os.environ[name].strip() + "\";"
+    return hipcc_env.strip()
+
+def _crosstool_verbose(repository_ctx):
+    """Returns the environment variable value CROSSTOOL_VERBOSE.
+
+    Args:
+        repository_ctx: The repository context.
+
+    Returns:
+        A string containing value of environment variable CROSSTOOL_VERBOSE.
+    """
+    name = "CROSSTOOL_VERBOSE"
+    if name in repository_ctx.os.environ:
+        return repository_ctx.os.environ[name].strip()
+    return "0"
+
 def _cpu_value(repository_ctx):
     """Returns the name of the host operating system.
 
@@ -656,6 +687,12 @@ def _create_local_rocm_repository(repository_ctx):
         {
             "%{cpu_compiler}": str(cc),
             "%{hipcc_path}": "/opt/rocm/bin/hipcc",
+            "%{hipcc_env}": _hipcc_env(repository_ctx),
+            "%{hip_runtime_path}": "/opt/rocm/hip/lib",
+            "%{hip_runtime_library}": "hip_hcc",
+            "%{hcc_runtime_path}": "/opt/rocm/hcc/lib",
+            "%{hcc_runtime_library}": "mcwamp",
+            "%{crosstool_verbose}": _crosstool_verbose(repository_ctx),
             "%{gcc_host_compiler_path}": str(cc),
             "%{rocm_amdgpu_targets}": ",".join(
                 ["\"%s\"" % c for c in rocm_config.amdgpu_targets],
-- 
GitLab


From c3a9f923dceaa0cf70f2ba7f228f9fecbd7468c3 Mon Sep 17 00:00:00 2001
From: Tong Shen <endlessroad@google.com>
Date: Thu, 7 Feb 2019 13:13:24 -0800
Subject: [PATCH 2087/2345] In FunctionalizeCond, cluster Merge nodes by
 (CondId, AncestorId, predicate).

Original implementation cluster Merge nodes by (CondId, AncestorId), which is not enough. For the following case:
  pred1 = array_ops.placeholder(dtypes.bool, name='pred1')
  pred2 = array_ops.placeholder(dtypes.bool, name='pred2')
  cond1 = control_flow_ops.cond(pred1, ...)
  cond2 = control_flow_ops.cond(pred2, ...)
  cond3 = control_flow_ops.cond(pred1, use cond1 and cond2)
  cond4 = control_flow_ops.cond(pred2, use cond1 and cond2)

cond3 and cond4 have the same CondId (empty) and the same AncestorId ({pred1, PRED}, {merge_of_cond1, MERGE}), but they should not be merged into one "If" node (because they have different predicates).

This does not happen before cl/231844072, because back then AncestorId uses Switch node in AncestorId, and cond3/cond4 has different Switch nodes.

PiperOrigin-RevId: 232931242
---
 .../compiler/tf2xla/functionalize_cond.cc     | 43 ++++++++++++++++---
 1 file changed, 37 insertions(+), 6 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/functionalize_cond.cc b/tensorflow/compiler/tf2xla/functionalize_cond.cc
index c8341a2c6b..0e75d38d66 100644
--- a/tensorflow/compiler/tf2xla/functionalize_cond.cc
+++ b/tensorflow/compiler/tf2xla/functionalize_cond.cc
@@ -63,6 +63,23 @@ size_t AncestorNode::Hash::operator()(const AncestorNode& ancestor) const {
   return Hash64Combine(h, std::hash<int>()(static_cast<int>(ancestor.type)));
 }
 
+typedef std::tuple<StateMap::CondId, StateMap::AncestorId, OutputTensor>
+    ClusterTuple;
+
+struct ClusterTupleLessThan {
+  bool operator()(const ClusterTuple& a, const ClusterTuple& b) const {
+    if (std::tie(std::get<0>(a), std::get<1>(a)) <
+        std::tie(std::get<0>(b), std::get<1>(b))) {
+      return true;
+    } else if (std::tie(std::get<0>(a), std::get<1>(a)) ==
+               std::tie(std::get<0>(b), std::get<1>(b))) {
+      return StateMap::OutputTensorLess()(std::get<2>(a), std::get<2>(b));
+    } else {
+      return false;
+    }
+  }
+};
+
 // TODO(jpienaar): Move to OutputTensor.
 string DebugString(const OutputTensor& tensor) {
   return absl::StrCat(tensor.node->name(), ":", tensor.index);
@@ -1393,16 +1410,30 @@ Status FunctionalizeCond::FunctionalizeInternal() {
   // Sort the merge nodes from innermost outwards.
   SortMergeNodes(&merge_order);
 
-  // Cluster merge nodes by CondId and AncestorId in order of nesting.
-  using ClusterPair = std::pair<StateMap::CondId, StateMap::AncestorId>;
+  // Cluster merge nodes by (CondId, AncestorId, predicate) in order of
+  // nesting. (CondId, AncestorId) is not enough, e.g.
+  //   pred1 = array_ops.placeholder(dtypes.bool, name='pred1')
+  //   pred2 = array_ops.placeholder(dtypes.bool, name='pred2')
+  //   cond1 = control_flow_ops.cond(pred1, ...)
+  //   cond2 = control_flow_ops.cond(pred2, ...)
+  //   cond3 = control_flow_ops.cond(pred1, use cond1 and cond2)
+  //   cond4 = control_flow_ops.cond(pred2, use cond1 and cond2)
+  // cond3 and cond4 have the same (CondId, AncestorId), but they should not
+  // be merged into one "If" node (because they have different predicates).
   std::deque<std::vector<Node*>> merge_clusters;
-  std::map<ClusterPair, int> merge_cluster_index;
+  std::map<ClusterTuple, int, ClusterTupleLessThan> merge_cluster_index;
   for (Node* merge : merge_order) {
     auto cond_id = state_map_.LookupCondId(merge);
     if (state_map_.IsDead(cond_id)) continue;
 
-    ClusterPair key =
-        std::make_pair(cond_id, state_map_.LookupAncestorId(merge));
+    auto predicate = merge_to_predicate_.find(merge);
+    if (predicate == merge_to_predicate_.end()) {
+      return errors::Internal("Cannot find predicate for Merge node ",
+                              merge->name());
+    }
+
+    ClusterTuple key = std::make_tuple(
+        cond_id, state_map_.LookupAncestorId(merge), predicate->second);
     auto idx = merge_cluster_index.find(key);
     if (idx == merge_cluster_index.end()) {
       merge_cluster_index[key] = merge_clusters.size();
@@ -1422,7 +1453,7 @@ Status FunctionalizeCond::FunctionalizeInternal() {
                      &state_map_);
     for (Node* merge : cluster) TF_RETURN_IF_ERROR(cond.AddMerge(merge));
     TF_RETURN_IF_ERROR(
-        cond.BuildAndReplace(graph_, library_, &merge_to_predicate_));
+        cond.BuildAndReplace(graph_, library_, &merge_to_replacement_));
 
     if (VLOG_IS_ON(4)) DumpGraphWithCondState("after_extract");
   }
-- 
GitLab


From 6537a601e38c7e0757f5985b5f0119a3f56fed93 Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Thu, 7 Feb 2019 13:23:14 -0800
Subject: [PATCH 2088/2345] [tf.data] Map vectorization benchmark fix, to
 actually measure the `cast` function.

PiperOrigin-RevId: 232932944
---
 .../data/experimental/benchmarks/map_vectorization_benchmark.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/data/experimental/benchmarks/map_vectorization_benchmark.py b/tensorflow/python/data/experimental/benchmarks/map_vectorization_benchmark.py
index 50e3a5c469..8f8fbe86df 100644
--- a/tensorflow/python/data/experimental/benchmarks/map_vectorization_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/map_vectorization_benchmark.py
@@ -159,7 +159,7 @@ class MapVectorizationBenchmark(test.Benchmark):
 
   def benchmarkCast(self):
     self._benchmark_helper(
-        lambda *args: [math_ops.cast(x, dtypes.float64) for x in args], "cast")
+        lambda *args: [math_ops.cast(x, dtypes.float32) for x in args], "cast")
 
   def benchmarkReshape(self):
     self._benchmark_helper(
-- 
GitLab


From 552fc09415f4ebe3266c8084cb967fa839fcfb69 Mon Sep 17 00:00:00 2001
From: Bruce Fontaine <bfontain@google.com>
Date: Thu, 7 Feb 2019 13:53:38 -0800
Subject: [PATCH 2089/2345] Fix typo in TPUEstimator code and make embedding
 load/retrieve ops dependent on loss.

PiperOrigin-RevId: 232938753
---
 .../python/tpu/_tpu_estimator_embedding.py    |   2 +-
 .../contrib/tpu/python/tpu/tpu_embedding.py   | 252 ++++++++++++------
 .../contrib/tpu/python/tpu/tpu_estimator.py   |   7 +-
 3 files changed, 173 insertions(+), 88 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/_tpu_estimator_embedding.py b/tensorflow/contrib/tpu/python/tpu/_tpu_estimator_embedding.py
index 6ce96e5bcd..2b71ce6fd8 100644
--- a/tensorflow/contrib/tpu/python/tpu/_tpu_estimator_embedding.py
+++ b/tensorflow/contrib/tpu/python/tpu/_tpu_estimator_embedding.py
@@ -89,7 +89,7 @@ def _get_tpu_embedding_optimization_parameters(embedding_config_spec):
   elif embedding_config_spec.optimizer_type == 'sgd':
     return tpu_embedding.StochasticGradientDescentParameters(
         embedding_config_spec.learning_rate,
-        embedding_config_spec.use_gradient_accumulattion)
+        embedding_config_spec.use_gradient_accumulation)
   elif embedding_config_spec.optimizer_type == 'adam':
     return tpu_embedding.AdamParameters(
         embedding_config_spec.learning_rate,
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py b/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py
index 8731b93411..2fb0ae9517 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py
@@ -282,7 +282,7 @@ class TPUEmbedding(object):
                                      embedding.config_proto))
       sess.run(variables.global_variables_initializer())
       sess.run(embedding.init_ops)
-      sess.run(embedding_variables_and_ops.load_ops)
+      sess.run(embedding_variables_and_ops.load_ops())
       sess.run(enqueue_ops)
       loss_val = sess.run(loss)
     ```
@@ -510,13 +510,15 @@ class TPUEmbedding(object):
         A dictionary mapping from string of table name to embedding variables,
         A dictionary mapping from string of table name to AdagradSlotVariable,
          AdamSlotVariables etc with slot variables,
-        A list of ops to load embedding and slot variables on CPU to TPU,
-        A list of ops to retrieve embedding and slot variables from TPU to CPU.
+        A function which returns a list of ops to load embedding and slot
+         variables from TPU to CPU.
+        A function which returns a list of ops to retrieve embedding and slot
+         variables from TPU to CPU.
     """
     embedding_variables_by_table = {}
     slot_variables_by_table = {}
-    load_ops = []
-    retrieve_ops = []
+    load_op_fns = []
+    retrieve_op_fns = []
     for table in self._table_to_config_dict:
       if embedding_variable_name_by_table:
         embedding_variable_name = embedding_variable_name_by_table[table]
@@ -539,14 +541,37 @@ class TPUEmbedding(object):
             collections=[ops.GraphKeys.GLOBAL_VARIABLES])
         embedding_variables_by_table[table] = table_variables
 
-        slot_variables_for_table, load_ops_for_table, retrieve_ops_for_table = (
+        slot_variables_for_table, load_ops_fn, retrieve_ops_fn = (
             self._optimizer_handler.create_variables_and_ops(
                 table, slot_variable_names, self._num_hosts,
                 self._table_to_config_dict[table], table_variables)
         )
         slot_variables_by_table[table] = slot_variables_for_table
-        load_ops.extend(load_ops_for_table)
-        retrieve_ops.extend(retrieve_ops_for_table)
+        load_op_fns.append(load_ops_fn)
+        retrieve_op_fns.append(retrieve_ops_fn)
+
+    def load_ops():
+      """Calls and returns the load ops for each embedding table.
+
+      Returns:
+        A list of ops to load embedding and slot variables from CPU to TPU.
+      """
+      load_ops_list = []
+      for load_op_fn in load_op_fns:
+        load_ops_list.extend(load_op_fn())
+      return load_ops_list
+
+    def retrieve_ops():
+      """Calls and returns the retrieve ops for each embedding table.
+
+      Returns:
+        A list of ops to retrieve embedding and slot variables from TPU to CPU.
+      """
+      retrieve_ops_list = []
+      for retrieve_op_fn in retrieve_op_fns:
+        retrieve_ops_list.extend(retrieve_op_fn())
+      return retrieve_ops_list
+
     return VariablesAndOps(embedding_variables_by_table,
                            slot_variables_by_table,
                            load_ops, retrieve_ops)
@@ -877,30 +902,48 @@ class _AdagradHandler(_OptimizerHandler):
         initializer=accumulator_initializer)
     slot_variables = AdagradSlotVariable(accumulator_variables)
 
-    load_ops = []
-    retrieve_ops = []
-    for host_id, table_variable, accumulator_variable in (zip(
-        range(num_hosts), table_variables, accumulator_variables)):
-      with ops.colocate_with(table_variable):
-        load_parameters_op = (
-            tpu_ops.load_tpu_embedding_adagrad_parameters(
-                parameters=table_variable,
-                accumulators=accumulator_variable,
-                table_name=table,
-                num_shards=num_hosts,
-                shard_id=host_id))
-        retrieved_table, retrieved_accumulator = (
-            tpu_ops.retrieve_tpu_embedding_adagrad_parameters(
-                table_name=table,
-                num_shards=num_hosts,
-                shard_id=host_id))
-        retrieve_parameters_op = control_flow_ops.group(
-            state_ops.assign(table_variable, retrieved_table),
-            state_ops.assign(accumulator_variable, retrieved_accumulator))
-
-      load_ops.append(load_parameters_op)
-      retrieve_ops.append(retrieve_parameters_op)
-    return slot_variables, load_ops, retrieve_ops
+    def load_ops_fn():
+      """Returns the retrieve ops for AdaGrad embedding tables.
+
+      Returns:
+        A list of ops to load embedding and slot variables from CPU to TPU.
+      """
+      load_op_list = []
+      for host_id, table_variable, accumulator_variable in (zip(
+          range(num_hosts), table_variables, accumulator_variables)):
+        with ops.colocate_with(table_variable):
+          load_parameters_op = (
+              tpu_ops.load_tpu_embedding_adagrad_parameters(
+                  parameters=table_variable,
+                  accumulators=accumulator_variable,
+                  table_name=table,
+                  num_shards=num_hosts,
+                  shard_id=host_id))
+        load_op_list.append(load_parameters_op)
+      return load_op_list
+
+    def retrieve_ops_fn():
+      """Returns the retrieve ops for AdaGrad embedding tables.
+
+      Returns:
+        A list of ops to retrieve embedding and slot variables from TPU to CPU.
+      """
+      retrieve_op_list = []
+      for host_id, table_variable, accumulator_variable in (zip(
+          range(num_hosts), table_variables, accumulator_variables)):
+        with ops.colocate_with(table_variable):
+          retrieved_table, retrieved_accumulator = (
+              tpu_ops.retrieve_tpu_embedding_adagrad_parameters(
+                  table_name=table,
+                  num_shards=num_hosts,
+                  shard_id=host_id))
+          retrieve_parameters_op = control_flow_ops.group(
+              state_ops.assign(table_variable, retrieved_table),
+              state_ops.assign(accumulator_variable, retrieved_accumulator))
+        retrieve_op_list.append(retrieve_parameters_op)
+      return retrieve_op_list
+
+    return slot_variables, load_ops_fn, retrieve_ops_fn
 
 
 class _AdamHandler(_OptimizerHandler):
@@ -947,33 +990,55 @@ class _AdamHandler(_OptimizerHandler):
         initializer=v_initializer)
     slot_variables = AdamSlotVariables(m_variables, v_variables)
 
-    load_ops = []
-    retrieve_ops = []
-    for host_id, table_variable, m_variable, v_variable in (zip(
-        range(num_hosts), table_variables,
-        m_variables, v_variables)):
-      with ops.colocate_with(table_variable):
-        load_parameters_op = (
-            tpu_ops.load_tpu_embedding_adam_parameters(
-                parameters=table_variable,
-                momenta=m_variable,
-                velocities=v_variable,
-                table_name=table,
-                num_shards=num_hosts,
-                shard_id=host_id))
-        retrieved_table, retrieved_m, retrieved_v = (
-            tpu_ops.retrieve_tpu_embedding_adam_parameters(
-                table_name=table,
-                num_shards=num_hosts,
-                shard_id=host_id))
-        retrieve_parameters_op = control_flow_ops.group(
-            state_ops.assign(table_variable, retrieved_table),
-            state_ops.assign(m_variable, retrieved_m),
-            state_ops.assign(v_variable, retrieved_v))
-
-      load_ops.append(load_parameters_op)
-      retrieve_ops.append(retrieve_parameters_op)
-    return slot_variables, load_ops, retrieve_ops
+    def load_ops_fn():
+      """Returns the retrieve ops for AdaGrad embedding tables.
+
+      Returns:
+        A list of ops to load embedding and slot variables from CPU to TPU.
+      """
+      load_op_list = []
+      for host_id, table_variable, m_variable, v_variable in (zip(
+          range(num_hosts), table_variables,
+          m_variables, v_variables)):
+        with ops.colocate_with(table_variable):
+          load_parameters_op = (
+              tpu_ops.load_tpu_embedding_adam_parameters(
+                  parameters=table_variable,
+                  momenta=m_variable,
+                  velocities=v_variable,
+                  table_name=table,
+                  num_shards=num_hosts,
+                  shard_id=host_id))
+
+      load_op_list.append(load_parameters_op)
+      return load_op_list
+
+    def retrieve_ops_fn():
+      """Returns the retrieve ops for Adam embedding tables.
+
+      Returns:
+        A list of ops to retrieve embedding and slot variables from TPU to CPU.
+      """
+
+      retrieve_op_list = []
+      for host_id, table_variable, m_variable, v_variable in (zip(
+          range(num_hosts), table_variables,
+          m_variables, v_variables)):
+        with ops.colocate_with(table_variable):
+          retrieved_table, retrieved_m, retrieved_v = (
+              tpu_ops.retrieve_tpu_embedding_adam_parameters(
+                  table_name=table,
+                  num_shards=num_hosts,
+                  shard_id=host_id))
+          retrieve_parameters_op = control_flow_ops.group(
+              state_ops.assign(table_variable, retrieved_table),
+              state_ops.assign(m_variable, retrieved_m),
+              state_ops.assign(v_variable, retrieved_v))
+
+        retrieve_op_list.append(retrieve_parameters_op)
+      return retrieve_op_list
+
+    return slot_variables, load_ops_fn, retrieve_ops_fn
 
 
 class _StochasticGradientDescentHandler(_OptimizerHandler):
@@ -990,30 +1055,51 @@ class _StochasticGradientDescentHandler(_OptimizerHandler):
                                table_config, table_variables):
     del table_config
 
-    load_ops = []
-    retrieve_ops = []
-    for host_id, table_variable in (zip(
-        range(num_hosts), table_variables)):
-      with ops.colocate_with(table_variable):
-        load_parameters_op = (
-            tpu_ops
-            .load_tpu_embedding_stochastic_gradient_descent_parameters(
-                parameters=table_variable,
-                table_name=table,
-                num_shards=num_hosts,
-                shard_id=host_id))
-        retrieved_table = (
-            tpu_ops
-            .retrieve_tpu_embedding_stochastic_gradient_descent_parameters(
-                table_name=table,
-                num_shards=num_hosts,
-                shard_id=host_id))
-        retrieve_parameters_op = control_flow_ops.group(
-            state_ops.assign(table_variable, retrieved_table))
-
-      load_ops.append(load_parameters_op)
-      retrieve_ops.append(retrieve_parameters_op)
-    return None, load_ops, retrieve_ops
+    def load_ops_fn():
+      """Returns the retrieve ops for AdaGrad embedding tables.
+
+      Returns:
+        A list of ops to load embedding and slot variables from CPU to TPU.
+      """
+      load_op_list = []
+      for host_id, table_variable in (zip(
+          range(num_hosts), table_variables)):
+        with ops.colocate_with(table_variable):
+          load_parameters_op = (
+              tpu_ops
+              .load_tpu_embedding_stochastic_gradient_descent_parameters(
+                  parameters=table_variable,
+                  table_name=table,
+                  num_shards=num_hosts,
+                  shard_id=host_id))
+
+        load_op_list.append(load_parameters_op)
+      return load_op_list
+
+    def retrieve_ops_fn():
+      """Returns the retrieve ops for SGD embedding tables.
+
+      Returns:
+        A list of ops to retrieve embedding and slot variables from TPU to CPU.
+      """
+
+      retrieve_op_list = []
+      for host_id, table_variable in (zip(
+          range(num_hosts), table_variables)):
+        with ops.colocate_with(table_variable):
+          retrieved_table = (
+              tpu_ops
+              .retrieve_tpu_embedding_stochastic_gradient_descent_parameters(
+                  table_name=table,
+                  num_shards=num_hosts,
+                  shard_id=host_id))
+          retrieve_parameters_op = control_flow_ops.group(
+              state_ops.assign(table_variable, retrieved_table))
+
+        retrieve_op_list.append(retrieve_parameters_op)
+      return retrieve_op_list
+
+    return None, load_ops_fn, retrieve_ops_fn
 
 
 def _get_optimization_handler(optimization_parameters):
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index a372d44b2c..41e2d1dd10 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -2790,7 +2790,7 @@ class TPUEstimator(estimator_lib.Estimator):
           tpu_init_ops.extend(ctx.embedding_config.tpu_embedding.init_ops)
           embedding_variables_and_ops = (
               ctx.embedding_config.tpu_embedding.create_variables_and_ops())
-          tpu_init_ops.extend(embedding_variables_and_ops.load_ops)
+          tpu_init_ops.extend(embedding_variables_and_ops.load_ops())
 
         input_holders = _InputPipeline(input_fn, batch_axis, ctx)
         enqueue_ops, dequeue_fn, input_hooks, run_infeed_loop_on_coordinator = (
@@ -2881,9 +2881,8 @@ class TPUEstimator(estimator_lib.Estimator):
           summary.scalar(model_fn_lib.LOSS_METRIC_KEY, loss)
           with ops.control_dependencies([loss]):
             update_ops = _sync_variables_ops(ctx)
-
-          if ctx.embedding_config:
-            update_ops.extend(embedding_variables_and_ops.retrieve_ops)
+            if ctx.embedding_config:
+              update_ops.extend(embedding_variables_and_ops.retrieve_ops())
 
           # Validate the TPU training graph to catch basic errors
           _validate_tpu_training_graph()
-- 
GitLab


From 670207218cd0af11c9fba210b4d4478fd731cf2a Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Thu, 7 Feb 2019 14:02:48 -0800
Subject: [PATCH 2090/2345] Update applications test.

PiperOrigin-RevId: 232940552
---
 tensorflow/python/keras/BUILD                             | 4 ++--
 tensorflow/python/keras/applications/applications_test.py | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 01603e070b..e0794c2b32 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -518,14 +518,14 @@ tf_py_test(
 
 tf_py_test(
     name = "applications_test",
-    size = "enormous",
+    size = "medium",
     srcs = ["applications/applications_test.py"],
     additional_deps = [
         ":keras",
         "@absl_py//absl/testing:parameterized",
         "//tensorflow/python:client_testlib",
     ],
-    shard_count = 2,
+    shard_count = 11,
 )
 
 tf_py_test(
diff --git a/tensorflow/python/keras/applications/applications_test.py b/tensorflow/python/keras/applications/applications_test.py
index b15ca5990a..41c508d274 100644
--- a/tensorflow/python/keras/applications/applications_test.py
+++ b/tensorflow/python/keras/applications/applications_test.py
@@ -38,7 +38,6 @@ MODEL_LIST = [
     (applications.DenseNet169, 1664),
     (applications.DenseNet201, 1920),
     (applications.NASNetMobile, 1056),
-    (applications.NASNetLarge, 4032),
 ]
 
 
@@ -47,7 +46,8 @@ class ApplicationsTest(test.TestCase, parameterized.TestCase):
   @parameterized.parameters(*MODEL_LIST)
   def test_feature_extration_model(self, model_fn, output_dim):
     model = model_fn(include_top=False, weights=None)
-    self.assertEqual(model.output_shape, (None, None, None, output_dim))
+    self.assertLen(model.output_shape, 4)
+    self.assertEqual(model.output_shape[-1], output_dim)
 
 
 if __name__ == '__main__':
-- 
GitLab


From 08e1a3dbc9dfee7cbd9da741a1c862457440e4c9 Mon Sep 17 00:00:00 2001
From: Brian Patton <bjp@google.com>
Date: Thu, 7 Feb 2019 14:14:09 -0800
Subject: [PATCH 2091/2345] Adds python support for ops that take a list of
 functions as an attr.

PiperOrigin-RevId: 232942930
---
 tensorflow/python/framework/op_def_library.py | 26 ++++++++----
 .../python/framework/op_def_library_test.py   | 41 +++++++++++++++++++
 tensorflow/python/framework/python_op_gen.cc  |  2 +-
 tensorflow/python/framework/test_ops.cc       |  4 ++
 4 files changed, 65 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/framework/op_def_library.py b/tensorflow/python/framework/op_def_library.py
index 2318b32ef1..b843e1f68b 100644
--- a/tensorflow/python/framework/op_def_library.py
+++ b/tensorflow/python/framework/op_def_library.py
@@ -212,6 +212,22 @@ def _MakeTensor(v, arg_name):
       (repr(v), arg_name))
 
 
+def _MakeFunc(v, arg_name):
+  """Ensure v is a func."""
+  if isinstance(v, attr_value_pb2.NameAttrList):
+    return v
+  fn_attr = attr_value_pb2.NameAttrList()
+  if isinstance(v, compat.bytes_or_text_types):
+    fn_attr.name = v
+  elif hasattr(v, "add_to_graph"):
+    v.add_to_graph(ops.get_default_graph())
+    fn_attr.name = v.name
+  else:
+    raise TypeError("Don't know how to convert {} to a func for "
+                    "argument {}".format(v, arg_name))
+  return fn_attr
+
+
 class _OpInfo(object):
   """All per-Op state we would like to precompute/validate."""
 
@@ -733,13 +749,9 @@ class OpDefLibrary(object):
           attr_value.list.tensor.extend(
               [_MakeTensor(x, key) for x in value])
         elif attr_def.type == "func":
-          if isinstance(value, attr_value_pb2.NameAttrList):
-            attr_value.func.CopyFrom(value)
-          elif isinstance(value, compat.bytes_or_text_types):
-            attr_value.func.name = value
-          else:
-            value.add_to_graph(ops.get_default_graph())
-            attr_value.func.name = value.name
+          attr_value.func.CopyFrom(_MakeFunc(value, key))
+        elif attr_def.type == "list(func)":
+          attr_value.list.func.extend([_MakeFunc(x, key) for x in value])
         else:
           raise TypeError("Unrecognized Attr type " + attr_def.type)
 
diff --git a/tensorflow/python/framework/op_def_library_test.py b/tensorflow/python/framework/op_def_library_test.py
index 66cfe213b3..aed056a95c 100644
--- a/tensorflow/python/framework/op_def_library_test.py
+++ b/tensorflow/python/framework/op_def_library_test.py
@@ -24,6 +24,7 @@ from google.protobuf import text_format
 from tensorflow.core.framework import op_def_pb2
 from tensorflow.core.framework import tensor_shape_pb2
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import function
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import test_ops
@@ -466,6 +467,46 @@ class OpDefLibraryTest(test_util.TensorFlowTestCase):
       self.assertEqual(str(cm.exception),
                        "Expected float for argument 'a' not 'bad'.")
 
+  def testAttrFunc(self):
+    with ops.Graph().as_default():
+      @function.Defun(dtypes.float32, func_name="MyFn")
+      def fn(x):
+        return 2 + x
+      op = self._lib.apply_op("FuncAttr", f=fn, name="t")
+      self.assertProtoEquals("""
+        name: 't' op: 'FuncAttr' attr { key: 'f'
+                                        value { func { name: 'MyFn' } } }
+        """, op.node_def)
+
+      with self.assertRaises(TypeError) as cm:
+        self._lib.apply_op("FuncAttr", f=3)
+      self.assertEqual(str(cm.exception),
+                       "Don't know how to convert 3 to a func for argument f")
+
+  def testAttrFuncList(self):
+    with ops.Graph().as_default():
+      @function.Defun(dtypes.float32, func_name="MyFn")
+      def fn1(x):
+        return 2 + x
+      @function.Defun(dtypes.int32, dtypes.float32, func_name="MyFn2")
+      def fn2(x, y):
+        return 2 + x, y * 3
+      @function.Defun(dtypes.int32, func_name="MyFn3")
+      def fn3(y):
+        return 2 + y
+      op = self._lib.apply_op("FuncListAttr", f=[fn1, fn2, fn3], name="t")
+      self.assertProtoEquals("""
+        name: 't' op: 'FuncListAttr'
+        attr { key: 'f' value { list { func { name: 'MyFn' }
+                                       func { name: 'MyFn2' }
+                                       func { name: 'MyFn3' } } } }
+        """, op.node_def)
+
+      with self.assertRaises(TypeError) as cm:
+        self._lib.apply_op("FuncListAttr", f=[fn1, 3, fn2])
+      self.assertEqual(str(cm.exception),
+                       "Don't know how to convert 3 to a func for argument f")
+
   def testAttrBool(self):
     with ops.Graph().as_default():
       op = self._lib.apply_op("AttrBool", a=True, name="t")
diff --git a/tensorflow/python/framework/python_op_gen.cc b/tensorflow/python/framework/python_op_gen.cc
index b0b9ce7055..c27f014014 100644
--- a/tensorflow/python/framework/python_op_gen.cc
+++ b/tensorflow/python/framework/python_op_gen.cc
@@ -547,7 +547,7 @@ bool GenEagerPythonOp::GetEagerFunctionSetup(const string& indentation,
       strings::StrAppend(function_setup, indentation, attr_api_name,
                          " = [_execute.make_tensor(_t, \"", attr_api_name,
                          "\") for _t in ", attr_api_name, "]\n");
-    } else if (attr_type != "func") {
+    } else if (attr_type != "func" && attr_type != "list(func)") {
       *function_setup =
           strings::StrCat("# No definition for ", function_name_,
                           " since we don't support attrs with type\n"
diff --git a/tensorflow/python/framework/test_ops.cc b/tensorflow/python/framework/test_ops.cc
index 1d0145f61c..5d1386c26d 100644
--- a/tensorflow/python/framework/test_ops.cc
+++ b/tensorflow/python/framework/test_ops.cc
@@ -406,6 +406,10 @@ REGISTER_OP("FuncAttr")
     .Attr("f: func")
     .SetShapeFn(shape_inference::UnknownShape);
 
+REGISTER_OP("FuncListAttr")
+    .Attr("f: list(func)")
+    .SetShapeFn(shape_inference::UnknownShape);
+
 REGISTER_OP("Simple")
     .Input("a: int32")
     .Output("out: float")
-- 
GitLab


From a69a0cf96a449a8d01f00ebfb7c243e4ba50d923 Mon Sep 17 00:00:00 2001
From: Yuan Lin <yulin@nvidia.com>
Date: Thu, 7 Feb 2019 14:32:30 -0800
Subject: [PATCH 2092/2345] [XLA] improve interactive_graphviz boundary change
 based on review feedbacks

---
 .../compiler/xla/service/hlo_graph_dumper.cc  | 10 +++---
 .../compiler/xla/service/hlo_graph_dumper.h   | 10 +++---
 .../xla/tools/interactive_graphviz.cc         | 34 +++++++++----------
 3 files changed, 28 insertions(+), 26 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
index 6baf357790..d3cf139e1a 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
@@ -1281,9 +1281,9 @@ namespace {
 
 // Gets a NodeFilter that includes roughly all instructions whose distance from
 // root is <= radius.
-NodeFilter MakeNodeRadiusAroundFilter(
-    const HloInstruction* root, int64 radius,
-    const std::set<const HloInstruction*>& boundary) {
+NodeFilter MakeNodeRadiusAroundFilter(const HloInstruction* root,
+                                      int64 radius,
+				      const absl::flat_hash_set<const HloInstruction*>& boundary) {
   // First, find the neighborhood of nodes with distance from root <= radius.
   // These nodes are our initial set of "normal" nodes.
   absl::flat_hash_map<const HloInstruction*, NodeFilterResult> nodes;
@@ -1299,7 +1299,7 @@ NodeFilter MakeNodeRadiusAroundFilter(
     if (depth == radius) {
       continue;
     }
-    if (boundary.count(instr) != 0) {
+    if (boundary.contains(instr)) {
       continue;
     }
 
@@ -1517,7 +1517,7 @@ string DumpGraph(const HloComputation& computation, const string& label,
 }
 
 string DumpNeighborhoodAround(const HloInstruction& node, int radius,
-                              const std::set<const HloInstruction*>& boundary,
+			      const absl::flat_hash_set<const HloInstruction*>& boundary,
                               bool show_backend_config) {
   auto debug_options = node.GetModule()->config().debug_options();
   string label =
diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.h b/tensorflow/compiler/xla/service/hlo_graph_dumper.h
index 7547d983cf..dd94256aef 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.h
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.h
@@ -63,10 +63,12 @@ string DumpGraph(const HloComputation& computation, const string& label,
 // The number of nodes dumped is controlled by the radius parameter, which
 // (roughly) corresponds to the max distance a node may be from the primary node
 // before it's omitted from the graph.
-string DumpNeighborhoodAround(
-    const HloInstruction& node, int radius,
-    const std::set<const HloInstruction*>& boundary = {},
-    bool show_backend_config = false);
+//
+// The optional boundary specifies a set of boundary nodes, beyond which nodes
+// will be omitted even if they  are within the radius.
+string DumpNeighborhoodAround(const HloInstruction& node, int radius,
+			      const absl::flat_hash_set<const HloInstruction*>& boundary={},
+                              bool show_backend_config = false);
 
 // Dumps nodes on any of the paths from `from` to `to`.  If there are more than
 // max_nodes on all paths, restricts to the max_nodes nodes on the shortest
diff --git a/tensorflow/compiler/xla/tools/interactive_graphviz.cc b/tensorflow/compiler/xla/tools/interactive_graphviz.cc
index 48013954a8..94674a8176 100644
--- a/tensorflow/compiler/xla/tools/interactive_graphviz.cc
+++ b/tensorflow/compiler/xla/tools/interactive_graphviz.cc
@@ -470,7 +470,7 @@ void DoPlotCommand(const Options& opts, const HloModule& module,
   }
 
   uint64 graph_width = kDefaultWidth;
-  std::set<const HloInstruction*> boundary;
+  absl::flat_hash_set<const HloInstruction*> boundary;
   if (tokens.size() >= 2) {
     if (comp) {
       std::cerr << "Can only use graph-size parameter with instructions, but "
@@ -478,31 +478,31 @@ void DoPlotCommand(const Options& opts, const HloModule& module,
       return;
     }
 
-    int bound_index = tokens.size();
+    int bound_index;
+    // get the width if present
     if (!absl::SimpleAtoi(tokens[1], &graph_width)) {
-      if (tokens[1] != "/") {
-        std::cerr << "Can't parse '" << tokens[1] << "' as an integer."
-                  << std::endl;
-        return;
-      }
+      bound_index = 1;
       graph_width = kDefaultWidth;
+    }
+    else {
       bound_index = 2;
-    } else {
-      if (tokens.size() > 2) {
-        if (tokens[2] != "/") {
+    }
+    // get the '/'
+    if (bound_index < tokens.size()) {
+      if (tokens[bound_index] != "/") {
           std::cerr << "Expect a /, but get a '" << tokens[1] << "'."
                     << std::endl;
           return;
-        }
-        bound_index = 3;
       }
+      bound_index++;
     }
+    // get the boundary nodes
     while (bound_index < tokens.size()) {
       string bnode_name = tokens[bound_index];
       const HloInstruction* binstr = FindInstruction(module, bnode_name);
       if (!binstr) {
-        std::cerr << "Couldn't find HloInstruction named " << node_name << "."
-                  << std::endl;
+        std::cerr << "Couldn't find HloInstruction named "
+                  << node_name << "." << std::endl;
         return;
       }
       boundary.insert(binstr);
@@ -518,9 +518,9 @@ void DoPlotCommand(const Options& opts, const HloModule& module,
         /*show_backend_config=*/show_backend_config));
   } else {
     DisplayGraphHandle(opts, hlo_graph_dumper::DumpNeighborhoodAround(
-                                 *instr, graph_width,
-                                 /*boundary=*/boundary,
-                                 /*show_backend_config=*/show_backend_config));
+        *instr, graph_width, 
+        /*boundary=*/boundary,
+        /*show_backend_config=*/show_backend_config));
   }
 }
 
-- 
GitLab


From 1874181691fd90a49ac5b4f75777d3acbdb575f6 Mon Sep 17 00:00:00 2001
From: Frank Chen <frankchn@google.com>
Date: Thu, 7 Feb 2019 14:27:16 -0800
Subject: [PATCH 2093/2345] Add better output shape handling to RebatchDataset

PiperOrigin-RevId: 232945496
---
 .../kernel_tests/rebatch_dataset_test.py      | 17 +++++++++++
 .../python/data/experimental/ops/batching.py  | 30 +++++++++++--------
 2 files changed, 35 insertions(+), 12 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/rebatch_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/rebatch_dataset_test.py
index 0dcbd563fa..486099bbb1 100644
--- a/tensorflow/python/data/experimental/kernel_tests/rebatch_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/rebatch_dataset_test.py
@@ -55,6 +55,23 @@ class RebatchDatasetTest(test_base.DatasetTestBase):
     with self.assertRaisesRegexp(ValueError, "not divisible by"):
       batching._RebatchDataset(dataset, num_workers=5)
 
+  def testTupleOutput(self):
+    dataset = (dataset_ops.Dataset.range(1024).map(lambda x: (x, x))
+               .batch(32, drop_remainder=True))
+    rebatched_dataset = batching._RebatchDataset(dataset, num_workers=4)
+    expected_output = [([k for k in range(i, i + 8)],  # pylint: disable=g-complex-comprehension
+                        [k for k in range(i, i + 8)])
+                       for i in range(0, 1024, 8)]
+    self.assertDatasetProduces(rebatched_dataset, expected_output)
+
+  def testNestedDictionaryOutput(self):
+    dataset = dataset_ops.Dataset.range(1024).map(
+        lambda x: {"a": x, "b": {"c": x}}).batch(32, drop_remainder=True)
+    rebatched_dataset = batching._RebatchDataset(dataset, num_workers=4)
+    expected_output = [{"a": [k for k in range(i, i + 8)],  # pylint: disable=g-complex-comprehension
+                        "b": {"c": [k for k in range(i, i + 8)]}}
+                       for i in range(0, 1024, 8)]
+    self.assertDatasetProduces(rebatched_dataset, expected_output)
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/data/experimental/ops/batching.py b/tensorflow/python/data/experimental/ops/batching.py
index a4ef71f796..5e28e0c001 100644
--- a/tensorflow/python/data/experimental/ops/batching.py
+++ b/tensorflow/python/data/experimental/ops/batching.py
@@ -712,18 +712,24 @@ class _RebatchDataset(dataset_ops.UnaryDataset):
 
   def __init__(self, input_dataset, num_workers):
     self._input_dataset = input_dataset
-    output_shapes = input_dataset.output_shapes
-    if len(output_shapes) < 1:
-      raise ValueError("Input shape should have at least one dimension.")
-    if not output_shapes.dims[0].value:
-      raise ValueError("Cannot rebatch unknown batch size datasets.")
-    if output_shapes.dims[0].value % num_workers != 0:
-      raise ValueError(
-          "First dim of input shape: %d is not divisible by num_workers: %d" %
-          (output_shapes[0], num_workers))
-    output_dims = [d for d in output_shapes.dims]
-    output_dims[0] = output_dims[0] // num_workers
-    output_shapes = tensor_shape.TensorShapeV1(output_dims)
+
+    def recalculate_output_shapes(output_shapes):
+      """Recalculates the output_shapes after dividing it by num_workers."""
+      if len(output_shapes) < 1:
+        raise ValueError("Input shape should have at least one dimension.")
+      if not output_shapes.dims[0].value:
+        raise ValueError("Cannot rebatch unknown batch size datasets.")
+      if output_shapes.dims[0].value % num_workers != 0:
+        raise ValueError(
+            "First dim of input shape: %d is not divisible by num_workers: %d" %
+            (output_shapes[0], num_workers))
+      output_dims = [d for d in output_shapes.dims]
+      output_dims[0] = output_dims[0] // num_workers
+      return tensor_shape.TensorShapeV1(output_dims)
+
+    output_shapes = nest.map_structure(recalculate_output_shapes,
+                                       input_dataset.output_shapes)
+
     self._structure = structure.convert_legacy_structure(
         self._input_dataset.output_types, output_shapes,
         self._input_dataset.output_classes)
-- 
GitLab


From e10dc81fbe28e8153aceaa29a684993cbdc8c00e Mon Sep 17 00:00:00 2001
From: Yuan Lin <yulin@nvidia.com>
Date: Thu, 7 Feb 2019 14:34:29 -0800
Subject: [PATCH 2094/2345] apply clang format

---
 .../compiler/xla/service/hlo_graph_dumper.cc  | 13 +++++++------
 .../compiler/xla/service/hlo_graph_dumper.h   |  7 ++++---
 .../xla/tools/interactive_graphviz.cc         | 19 +++++++++----------
 3 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
index d3cf139e1a..c7973112d0 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
@@ -1281,9 +1281,9 @@ namespace {
 
 // Gets a NodeFilter that includes roughly all instructions whose distance from
 // root is <= radius.
-NodeFilter MakeNodeRadiusAroundFilter(const HloInstruction* root,
-                                      int64 radius,
-				      const absl::flat_hash_set<const HloInstruction*>& boundary) {
+NodeFilter MakeNodeRadiusAroundFilter(
+    const HloInstruction* root, int64 radius,
+    const absl::flat_hash_set<const HloInstruction*>& boundary) {
   // First, find the neighborhood of nodes with distance from root <= radius.
   // These nodes are our initial set of "normal" nodes.
   absl::flat_hash_map<const HloInstruction*, NodeFilterResult> nodes;
@@ -1516,9 +1516,10 @@ string DumpGraph(const HloComputation& computation, const string& label,
   return graph_url;
 }
 
-string DumpNeighborhoodAround(const HloInstruction& node, int radius,
-			      const absl::flat_hash_set<const HloInstruction*>& boundary,
-                              bool show_backend_config) {
+string DumpNeighborhoodAround(
+    const HloInstruction& node, int radius,
+    const absl::flat_hash_set<const HloInstruction*>& boundary,
+    bool show_backend_config) {
   auto debug_options = node.GetModule()->config().debug_options();
   string label =
       StrCat("Neighborhood of ", radius, " nodes around ", node.name());
diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.h b/tensorflow/compiler/xla/service/hlo_graph_dumper.h
index dd94256aef..1b90a36706 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.h
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.h
@@ -66,9 +66,10 @@ string DumpGraph(const HloComputation& computation, const string& label,
 //
 // The optional boundary specifies a set of boundary nodes, beyond which nodes
 // will be omitted even if they  are within the radius.
-string DumpNeighborhoodAround(const HloInstruction& node, int radius,
-			      const absl::flat_hash_set<const HloInstruction*>& boundary={},
-                              bool show_backend_config = false);
+string DumpNeighborhoodAround(
+    const HloInstruction& node, int radius,
+    const absl::flat_hash_set<const HloInstruction*>& boundary = {},
+    bool show_backend_config = false);
 
 // Dumps nodes on any of the paths from `from` to `to`.  If there are more than
 // max_nodes on all paths, restricts to the max_nodes nodes on the shortest
diff --git a/tensorflow/compiler/xla/tools/interactive_graphviz.cc b/tensorflow/compiler/xla/tools/interactive_graphviz.cc
index 94674a8176..0b86ac6088 100644
--- a/tensorflow/compiler/xla/tools/interactive_graphviz.cc
+++ b/tensorflow/compiler/xla/tools/interactive_graphviz.cc
@@ -483,16 +483,15 @@ void DoPlotCommand(const Options& opts, const HloModule& module,
     if (!absl::SimpleAtoi(tokens[1], &graph_width)) {
       bound_index = 1;
       graph_width = kDefaultWidth;
-    }
-    else {
+    } else {
       bound_index = 2;
     }
     // get the '/'
     if (bound_index < tokens.size()) {
       if (tokens[bound_index] != "/") {
-          std::cerr << "Expect a /, but get a '" << tokens[1] << "'."
-                    << std::endl;
-          return;
+        std::cerr << "Expect a /, but get a '" << tokens[1] << "'."
+                  << std::endl;
+        return;
       }
       bound_index++;
     }
@@ -501,8 +500,8 @@ void DoPlotCommand(const Options& opts, const HloModule& module,
       string bnode_name = tokens[bound_index];
       const HloInstruction* binstr = FindInstruction(module, bnode_name);
       if (!binstr) {
-        std::cerr << "Couldn't find HloInstruction named "
-                  << node_name << "." << std::endl;
+        std::cerr << "Couldn't find HloInstruction named " << node_name << "."
+                  << std::endl;
         return;
       }
       boundary.insert(binstr);
@@ -518,9 +517,9 @@ void DoPlotCommand(const Options& opts, const HloModule& module,
         /*show_backend_config=*/show_backend_config));
   } else {
     DisplayGraphHandle(opts, hlo_graph_dumper::DumpNeighborhoodAround(
-        *instr, graph_width, 
-        /*boundary=*/boundary,
-        /*show_backend_config=*/show_backend_config));
+                                 *instr, graph_width,
+                                 /*boundary=*/boundary,
+                                 /*show_backend_config=*/show_backend_config));
   }
 }
 
-- 
GitLab


From 3f177961073bb38539abd0824d38f2d07a9ab560 Mon Sep 17 00:00:00 2001
From: Austin Anderson <angerson@google.com>
Date: Thu, 7 Feb 2019 14:47:10 -0800
Subject: [PATCH 2095/2345] Add TF_PACKAGE_VERSION to Dockerfiles that install
 via pip

This change adds `--build-arg TF_PACKAGE_VERSION=...` to the standard
Dockerfiles. It also picks up a tiny --build-arg documentation typo.

I also merged the dockerfile specs into one spec. Since the Dockerfiles
are usually going to be built all at once (and the default behavior of
the assembler is to wipe the build directory), it seems more convenient
to put them all in one place.
---
 .../tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile     | 5 ++++-
 tensorflow/tools/dockerfiles/dockerfiles/cpu.Dockerfile      | 5 ++++-
 .../dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile     | 2 +-
 .../tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile       | 2 +-
 .../tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile     | 5 ++++-
 tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile      | 5 ++++-
 .../dockerfiles/ppc64le/cpu-ppc64le-jupyter.Dockerfile       | 2 ++
 .../dockerfiles/ppc64le/devel-cpu-ppc64le-jupyter.Dockerfile | 4 +++-
 .../dockerfiles/ppc64le/devel-cpu-ppc64le.Dockerfile         | 2 +-
 .../dockerfiles/ppc64le/devel-gpu-ppc64le-jupyter.Dockerfile | 4 +++-
 .../dockerfiles/ppc64le/devel-gpu-ppc64le.Dockerfile         | 2 +-
 .../dockerfiles/ppc64le/gpu-ppc64le-jupyter.Dockerfile       | 2 ++
 .../tools/dockerfiles/partials/tensorflow.partial.Dockerfile | 5 ++++-
 tensorflow/tools/dockerfiles/spec.yml                        | 5 -----
 14 files changed, 34 insertions(+), 16 deletions(-)

diff --git a/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile
index c09a95858b..c806fa4eac 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile
@@ -47,8 +47,11 @@ RUN ln -s $(which ${PYTHON}) /usr/local/bin/python
 #   tensorflow-gpu
 #   tf-nightly
 #   tf-nightly-gpu
+# Set --build-arg TF_PACKAGE_VERSION=1.11.0rc0 to install a specific version.
+# Installs the latest version by default.
 ARG TF_PACKAGE=tensorflow
-RUN ${PIP} install ${TF_PACKAGE}
+ARG TF_PACKAGE_VERSION=
+RUN ${PIP} install ${TF_PACKAGE}${TF_PACKAGE_VERSION:+==${TF_PACKAGE_VERSION}}
 
 COPY bashrc /etc/bash.bashrc
 RUN chmod a+rwx /etc/bash.bashrc
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/cpu.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/cpu.Dockerfile
index 857b5e2047..a82577b53b 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/cpu.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/cpu.Dockerfile
@@ -47,8 +47,11 @@ RUN ln -s $(which ${PYTHON}) /usr/local/bin/python
 #   tensorflow-gpu
 #   tf-nightly
 #   tf-nightly-gpu
+# Set --build-arg TF_PACKAGE_VERSION=1.11.0rc0 to install a specific version.
+# Installs the latest version by default.
 ARG TF_PACKAGE=tensorflow
-RUN ${PIP} install ${TF_PACKAGE}
+ARG TF_PACKAGE_VERSION=
+RUN ${PIP} install ${TF_PACKAGE}${TF_PACKAGE_VERSION:+==${TF_PACKAGE_VERSION}}
 
 COPY bashrc /etc/bash.bashrc
 RUN chmod a+rwx /etc/bash.bashrc
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile
index 73f7a74d08..5c01623f81 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile
@@ -73,7 +73,7 @@ ENV TF_CUDA_VERSION=${CUDA}
 ENV TF_CUDNN_VERSION=${CUDNN%%.*}
 # CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version
 ARG CACHE_STOP=1
-# Check out TensorFlow source code if --build_arg CHECKOUT_TF_SRC=1
+# Check out TensorFlow source code if --build-arg CHECKOUT_TF_SRC=1
 ARG CHECKOUT_TF_SRC=0
 RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src || true
 
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile
index 7ae5010079..56d582d7e7 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile
@@ -73,7 +73,7 @@ ENV TF_CUDA_VERSION=${CUDA}
 ENV TF_CUDNN_VERSION=${CUDNN%%.*}
 # CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version
 ARG CACHE_STOP=1
-# Check out TensorFlow source code if --build_arg CHECKOUT_TF_SRC=1
+# Check out TensorFlow source code if --build-arg CHECKOUT_TF_SRC=1
 ARG CHECKOUT_TF_SRC=0
 RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src || true
 
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile
index f686a7107c..335bbb5efe 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile
@@ -80,8 +80,11 @@ RUN ln -s $(which ${PYTHON}) /usr/local/bin/python
 #   tensorflow-gpu
 #   tf-nightly
 #   tf-nightly-gpu
+# Set --build-arg TF_PACKAGE_VERSION=1.11.0rc0 to install a specific version.
+# Installs the latest version by default.
 ARG TF_PACKAGE=tensorflow
-RUN ${PIP} install ${TF_PACKAGE}
+ARG TF_PACKAGE_VERSION=
+RUN ${PIP} install ${TF_PACKAGE}${TF_PACKAGE_VERSION:+==${TF_PACKAGE_VERSION}}
 
 COPY bashrc /etc/bash.bashrc
 RUN chmod a+rwx /etc/bash.bashrc
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile
index 00664b6b73..3b1e3e9f49 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile
@@ -80,8 +80,11 @@ RUN ln -s $(which ${PYTHON}) /usr/local/bin/python
 #   tensorflow-gpu
 #   tf-nightly
 #   tf-nightly-gpu
+# Set --build-arg TF_PACKAGE_VERSION=1.11.0rc0 to install a specific version.
+# Installs the latest version by default.
 ARG TF_PACKAGE=tensorflow
-RUN ${PIP} install ${TF_PACKAGE}
+ARG TF_PACKAGE_VERSION=
+RUN ${PIP} install ${TF_PACKAGE}${TF_PACKAGE_VERSION:+==${TF_PACKAGE_VERSION}}
 
 COPY bashrc /etc/bash.bashrc
 RUN chmod a+rwx /etc/bash.bashrc
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/cpu-ppc64le-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/cpu-ppc64le-jupyter.Dockerfile
index beb3292a9d..63bf205a7a 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/cpu-ppc64le-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/cpu-ppc64le-jupyter.Dockerfile
@@ -75,6 +75,8 @@ COPY bashrc /etc/bash.bashrc
 RUN chmod a+rwx /etc/bash.bashrc
 
 RUN ${PIP} install jupyter matplotlib
+RUN ${PIP} install jupyter_http_over_ws
+RUN jupyter serverextension enable --py jupyter_http_over_ws
 
 RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/
 RUN mkdir /.local && chmod a+rwx /.local
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le-jupyter.Dockerfile
index 1f32849735..c8384f7af2 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le-jupyter.Dockerfile
@@ -47,7 +47,7 @@ ENV CI_BUILD_PYTHON python
 
 # CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version
 ARG CACHE_STOP=1
-# Check out TensorFlow source code if --build_arg CHECKOUT_TF_SRC=1
+# Check out TensorFlow source code if --build-arg CHECKOUT_TF_SRC=1
 ARG CHECKOUT_TF_SRC=0
 RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src || true
 
@@ -108,6 +108,8 @@ COPY bashrc /etc/bash.bashrc
 RUN chmod a+rwx /etc/bash.bashrc
 
 RUN ${PIP} install jupyter matplotlib
+RUN ${PIP} install jupyter_http_over_ws
+RUN jupyter serverextension enable --py jupyter_http_over_ws
 
 RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/
 RUN mkdir /.local && chmod a+rwx /.local
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le.Dockerfile
index cda51c371d..08f880e7b5 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le.Dockerfile
@@ -47,7 +47,7 @@ ENV CI_BUILD_PYTHON python
 
 # CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version
 ARG CACHE_STOP=1
-# Check out TensorFlow source code if --build_arg CHECKOUT_TF_SRC=1
+# Check out TensorFlow source code if --build-arg CHECKOUT_TF_SRC=1
 ARG CHECKOUT_TF_SRC=0
 RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src || true
 
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le-jupyter.Dockerfile
index d8ee19f66e..7113bb2967 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le-jupyter.Dockerfile
@@ -73,7 +73,7 @@ ENV TF_CUDA_VERSION=${CUDA}
 ENV TF_CUDNN_VERSION=${CUDNN%%.*}
 # CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version
 ARG CACHE_STOP=1
-# Check out TensorFlow source code if --build_arg CHECKOUT_TF_SRC=1
+# Check out TensorFlow source code if --build-arg CHECKOUT_TF_SRC=1
 ARG CHECKOUT_TF_SRC=0
 RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src || true
 
@@ -134,6 +134,8 @@ COPY bashrc /etc/bash.bashrc
 RUN chmod a+rwx /etc/bash.bashrc
 
 RUN ${PIP} install jupyter matplotlib
+RUN ${PIP} install jupyter_http_over_ws
+RUN jupyter serverextension enable --py jupyter_http_over_ws
 
 RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/
 RUN mkdir /.local && chmod a+rwx /.local
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le.Dockerfile
index 966070634b..a2f6fc5cba 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le.Dockerfile
@@ -73,7 +73,7 @@ ENV TF_CUDA_VERSION=${CUDA}
 ENV TF_CUDNN_VERSION=${CUDNN%%.*}
 # CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version
 ARG CACHE_STOP=1
-# Check out TensorFlow source code if --build_arg CHECKOUT_TF_SRC=1
+# Check out TensorFlow source code if --build-arg CHECKOUT_TF_SRC=1
 ARG CHECKOUT_TF_SRC=0
 RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src || true
 
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le-jupyter.Dockerfile
index 449a8d8aa8..9d0112c9c8 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le-jupyter.Dockerfile
@@ -108,6 +108,8 @@ COPY bashrc /etc/bash.bashrc
 RUN chmod a+rwx /etc/bash.bashrc
 
 RUN ${PIP} install jupyter matplotlib
+RUN ${PIP} install jupyter_http_over_ws
+RUN jupyter serverextension enable --py jupyter_http_over_ws
 
 RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/
 RUN mkdir /.local && chmod a+rwx /.local
diff --git a/tensorflow/tools/dockerfiles/partials/tensorflow.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/tensorflow.partial.Dockerfile
index 76758bd147..2ae840687d 100644
--- a/tensorflow/tools/dockerfiles/partials/tensorflow.partial.Dockerfile
+++ b/tensorflow/tools/dockerfiles/partials/tensorflow.partial.Dockerfile
@@ -3,5 +3,8 @@
 #   tensorflow-gpu
 #   tf-nightly
 #   tf-nightly-gpu
+# Set --build-arg TF_PACKAGE_VERSION=1.11.0rc0 to install a specific version.
+# Installs the latest version by default.
 ARG TF_PACKAGE=tensorflow
-RUN ${PIP} install ${TF_PACKAGE}
+ARG TF_PACKAGE_VERSION=
+RUN ${PIP} install ${TF_PACKAGE}${TF_PACKAGE_VERSION:+==${TF_PACKAGE_VERSION}}
diff --git a/tensorflow/tools/dockerfiles/spec.yml b/tensorflow/tools/dockerfiles/spec.yml
index d19b1d15fc..6fddfe000c 100644
--- a/tensorflow/tools/dockerfiles/spec.yml
+++ b/tensorflow/tools/dockerfiles/spec.yml
@@ -55,11 +55,6 @@ releases:
         tag_specs:
             - "{ubuntu}{jupyter}"
             - "{ubuntu-devel}{jupyter}"
-
-    ppc64le-dockerfiles:
-        is_dockerfiles: true
-        upload_images: false
-        tag_specs:
             - "{ubuntu-ppc64le}{jupyter}"
             - "{ubuntu-devel-ppc64le}{jupyter}"
 
-- 
GitLab


From 92b4da0f8e34139135c40c5b54a9dce2ddff9dc3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 7 Feb 2019 14:40:53 -0800
Subject: [PATCH 2096/2345] Automated rollback of commit
 567e02c3de3167bf0c0b986546360a9a18f467e0

PiperOrigin-RevId: 232948023
---
 .../contrib/distribute/python/keras_test.py   | 22 ++++++++++-------
 tensorflow/python/keras/backend.py            | 24 +++++++------------
 tensorflow/python/keras/backend_test.py       |  6 -----
 3 files changed, 22 insertions(+), 30 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index e06363a1e8..6a15083470 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -235,7 +235,6 @@ def multi_input_output_model():
 strategies_minus_tpu = [
     combinations.default_strategy,
     combinations.one_device_strategy,
-    combinations.one_device_strategy_gpu,
     combinations.mirrored_strategy_with_gpu_and_cpu,
     combinations.mirrored_strategy_with_two_gpus,
     combinations.core_mirrored_strategy_with_gpu_and_cpu,
@@ -264,7 +263,6 @@ def all_strategy_combinations_minus_default():
   strategy_minus_default_combinations = combinations.combine(
       distribution=[
           combinations.one_device_strategy,
-          combinations.one_device_strategy_gpu,
           combinations.mirrored_strategy_with_gpu_and_cpu,
           combinations.mirrored_strategy_with_two_gpus,
           combinations.core_mirrored_strategy_with_gpu_and_cpu,
@@ -288,6 +286,12 @@ def strategy_and_optimizer_combinations():
       ]))
 
 
+def strategy_for_numpy_input_combinations():
+  one_gpu = combinations.one_device_strategy_gpu
+  return (all_strategy_combinations() +
+          combinations.combine(distribution=[one_gpu], mode=['graph', 'eager']))
+
+
 class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase,
                                         parameterized.TestCase):
 
@@ -443,7 +447,7 @@ class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase,
 class TestDistributionStrategyWithNumpyArrays(test.TestCase,
                                               parameterized.TestCase):
 
-  @combinations.generate(all_strategy_combinations())
+  @combinations.generate(strategy_for_numpy_input_combinations())
   def test_calculating_input_params_no_steps_no_batch_size(self, distribution):
     # Calculate the per_replica_batch_size scaling factor for strategies
     # that use per_core_batch_size
@@ -474,7 +478,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
         distributed_training_utils.get_input_params(
             distribution, input_63_samples, steps=None, batch_size=None)
 
-  @combinations.generate(all_strategy_combinations())
+  @combinations.generate(strategy_for_numpy_input_combinations())
   def test_calculating_input_params_with_steps_no_batch_size(self,
                                                              distribution):
     # Calculate the per_replica_batch_size scaling factor for strategies
@@ -520,7 +524,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
           distributed_training_utils.get_input_params(
               distribution, input_63_samples, steps=1, batch_size=None)
 
-  @combinations.generate(all_strategy_combinations())
+  @combinations.generate(strategy_for_numpy_input_combinations())
   def test_calculating_input_params_no_steps_with_batch_size(self,
                                                              distribution):
     # Calculate the per_replica_batch_size scaling factor for strategies
@@ -554,7 +558,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
         distributed_training_utils.get_input_params(
             distribution, input_64_samples, steps=None, batch_size=3)
 
-  @combinations.generate(all_strategy_combinations())
+  @combinations.generate(strategy_for_numpy_input_combinations())
   def test_calculating_input_params_with_steps_with_batch_size(self,
                                                                distribution):
     with self.cached_session():
@@ -571,7 +575,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
         distributed_training_utils.get_input_params(
             distribution, input_64_samples, steps=10, batch_size=13)
 
-  @combinations.generate(all_strategy_combinations())
+  @combinations.generate(strategy_for_numpy_input_combinations())
   def test_calling_model_with_numpy_arrays(self, distribution):
     with self.cached_session():
       with distribution.scope():
@@ -602,7 +606,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
         # with batch_size
         model.predict(inputs, batch_size=8)
 
-  @combinations.generate(all_strategy_combinations())
+  @combinations.generate(strategy_for_numpy_input_combinations())
   def test_calling_model_with_nested_numpy_arrays(self, distribution):
     with self.cached_session():
       with distribution.scope():
@@ -654,7 +658,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
       model.fit(inputs, targets, sample_weight=sample_weights, epochs=1,
                 steps_per_epoch=2, verbose=1)
 
-  @combinations.generate(all_strategy_combinations())
+  @combinations.generate(strategy_for_numpy_input_combinations())
   def test_flatten_predict_outputs(self, distribution):
     with self.cached_session():
       with distribution.scope():
diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index c48fa9d683..37e605ea2c 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -396,21 +396,15 @@ def _get_session():
   if default_session is not None:
     session = default_session
   else:
-    # We use an init_scope since want to use the outermost graph even if we are
-    # in the middle of defining a function.
-    with ops.init_scope():
-      # If we don't have a session, or that session does not match the current
-      # graph, create and cache a new session.
-      if (getattr(_SESSION, 'session', None) is None or
-          _SESSION.session.graph is not ops.get_default_graph()):
-        # If we are creating the Session inside a tf.distribute.Strategy scope,
-        # we ask the strategy for the right session options to use.
-        if distribution_strategy_context.has_strategy():
-          configure_and_create_distributed_session(
-              distribution_strategy_context.get_strategy())
-        else:
-          _SESSION.session = session_module.Session(
-              config=get_default_session_config())
+    if getattr(_SESSION, 'session', None) is None:
+      # We are creating the Session inside a Distribution
+      # Strategy scope.
+      if distribution_strategy_context.has_strategy():
+        configure_and_create_distributed_session(
+            distribution_strategy_context.get_strategy())
+      else:
+        _SESSION.session = session_module.Session(
+            config=get_default_session_config())
     session = _SESSION.session
   return session
 
diff --git a/tensorflow/python/keras/backend_test.py b/tensorflow/python/keras/backend_test.py
index 9a39d18f1e..263a1dc461 100644
--- a/tensorflow/python/keras/backend_test.py
+++ b/tensorflow/python/keras/backend_test.py
@@ -1741,12 +1741,6 @@ class BackendGraphTests(test.TestCase):
         x, mean, var, beta, gamma, axis=1, epsilon=1e-3)
     self.assertEqual(normed.shape.as_list(), [10, 3, 5, 5])
 
-  def test_get_session_different_graphs(self):
-    with ops.Graph().as_default():
-      session = keras.backend.get_session()
-    with ops.Graph().as_default():
-      self.assertIsNot(session, keras.backend.get_session())
-
 
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 03d9fdf18ac40cecfb53a75d6932d8479bca6099 Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Thu, 7 Feb 2019 14:42:04 -0800
Subject: [PATCH 2097/2345] Guard against unsupported types before triggering
 resize

PiperOrigin-RevId: 232948223
---
 .../main/java/org/tensorflow/lite/Tensor.java | 34 +++++++++++++------
 .../org/tensorflow/lite/InterpreterTest.java  | 13 +++++++
 2 files changed, 36 insertions(+), 11 deletions(-)

diff --git a/tensorflow/lite/java/src/main/java/org/tensorflow/lite/Tensor.java b/tensorflow/lite/java/src/main/java/org/tensorflow/lite/Tensor.java
index 725bb326ba..16cca45f38 100644
--- a/tensorflow/lite/java/src/main/java/org/tensorflow/lite/Tensor.java
+++ b/tensorflow/lite/java/src/main/java/org/tensorflow/lite/Tensor.java
@@ -107,7 +107,7 @@ public final class Tensor {
       throw new IllegalArgumentException(
           "Null inputs are allowed only if the Tensor is bound to a buffer handle.");
     }
-    throwExceptionIfTypeIsIncompatible(src);
+    throwIfDataIsIncompatible(src);
     if (isByteBuffer(src)) {
       ByteBuffer srcBuffer = (ByteBuffer) src;
       // For direct ByteBuffer instances we support zero-copy. Note that this assumes the caller
@@ -138,7 +138,7 @@ public final class Tensor {
       throw new IllegalArgumentException(
           "Null outputs are allowed only if the Tensor is bound to a buffer handle.");
     }
-    throwExceptionIfTypeIsIncompatible(dst);
+    throwIfDataIsIncompatible(dst);
     if (dst instanceof ByteBuffer) {
       ByteBuffer dstByteBuffer = (ByteBuffer) dst;
       dstByteBuffer.put(buffer());
@@ -159,6 +159,7 @@ public final class Tensor {
     if (isByteBuffer(input)) {
       return null;
     }
+    throwIfTypeIsIncompatible(input);
     int[] inputShape = computeShapeOf(input);
     if (Arrays.equals(shapeCopy, inputShape)) {
       return null;
@@ -243,16 +244,14 @@ public final class Tensor {
     }
   }
 
-  private void throwExceptionIfTypeIsIncompatible(Object o) {
+  private void throwIfDataIsIncompatible(Object o) {
+    throwIfTypeIsIncompatible(o);
+    throwIfShapeIsIncompatible(o);
+  }
+
+  private void throwIfTypeIsIncompatible(Object o) {
+    // ByteBuffer payloads can map to any type, so exempt it from the check.
     if (isByteBuffer(o)) {
-      ByteBuffer oBuffer = (ByteBuffer) o;
-      if (oBuffer.capacity() != numBytes()) {
-        throw new IllegalArgumentException(
-            String.format(
-                "Cannot convert between a TensorFlowLite buffer with %d bytes and a "
-                    + "ByteBuffer with %d bytes.",
-                numBytes(), oBuffer.capacity()));
-      }
       return;
     }
     DataType oType = dataTypeOf(o);
@@ -263,7 +262,20 @@ public final class Tensor {
                   + "object of type %s (which is compatible with the TensorFlowLite type %s).",
               dtype, o.getClass().getName(), oType));
     }
+  }
 
+  private void throwIfShapeIsIncompatible(Object o) {
+    if (isByteBuffer(o)) {
+      ByteBuffer oBuffer = (ByteBuffer) o;
+      if (oBuffer.capacity() != numBytes()) {
+        throw new IllegalArgumentException(
+            String.format(
+                "Cannot convert between a TensorFlowLite buffer with %d bytes and a "
+                    + "ByteBuffer with %d bytes.",
+                numBytes(), oBuffer.capacity()));
+      }
+      return;
+    }
     int[] oShape = computeShapeOf(o);
     if (!Arrays.equals(oShape, shapeCopy)) {
       throw new IllegalArgumentException(
diff --git a/tensorflow/lite/java/src/test/java/org/tensorflow/lite/InterpreterTest.java b/tensorflow/lite/java/src/test/java/org/tensorflow/lite/InterpreterTest.java
index c5496e3a21..ff3325633c 100644
--- a/tensorflow/lite/java/src/test/java/org/tensorflow/lite/InterpreterTest.java
+++ b/tensorflow/lite/java/src/test/java/org/tensorflow/lite/InterpreterTest.java
@@ -21,6 +21,7 @@ import static org.junit.Assert.fail;
 import java.io.File;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.nio.FloatBuffer;
 import java.nio.MappedByteBuffer;
 import java.nio.channels.FileChannel;
 import java.nio.file.Files;
@@ -247,6 +248,18 @@ public final class InterpreterTest {
     interpreter.close();
   }
 
+  @Test
+  public void testRunWithUnsupportedInputType() {
+    FloatBuffer floatBuffer = FloatBuffer.allocate(10);
+    float[][][][] parsedOutputs = new float[2][8][8][3];
+    try (Interpreter interpreter = new Interpreter(MODEL_FILE)) {
+      interpreter.run(floatBuffer, parsedOutputs);
+      fail();
+    } catch (IllegalArgumentException e) {
+      assertThat(e).hasMessageThat().contains("DataType error: cannot resolve DataType of");
+    }
+  }
+
   @Test
   public void testRunWithWrongOutputType() {
     Interpreter interpreter = new Interpreter(MODEL_FILE);
-- 
GitLab


From e7a429ddf8a6fe1c2359bde17c425080ad2bbc1d Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Thu, 7 Feb 2019 15:07:47 -0800
Subject: [PATCH 2098/2345] [XRT] Add Python op wrappers for XRT.

PiperOrigin-RevId: 232953220
---
 tensorflow/compiler/xrt/BUILD         | 31 +++++++++++++++++++++++++++
 tensorflow/compiler/xrt/kernels/BUILD |  3 +++
 2 files changed, 34 insertions(+)

diff --git a/tensorflow/compiler/xrt/BUILD b/tensorflow/compiler/xrt/BUILD
index 2dae746d03..b2718c5c28 100644
--- a/tensorflow/compiler/xrt/BUILD
+++ b/tensorflow/compiler/xrt/BUILD
@@ -11,9 +11,15 @@ package(
 
 load(
     "//tensorflow:tensorflow.bzl",
+    "tf_custom_op_py_library",
     "tf_gen_op_libs",
+    "tf_gen_op_wrapper_py",
 )
 load("//tensorflow/compiler/xla:xla.bzl", "xla_proto_library")
+load(
+    "//tensorflow/core:platform/default/build_config.bzl",
+    "tf_proto_library_py",
+)
 
 xla_proto_library(
     name = "xrt_proto",
@@ -27,6 +33,12 @@ xla_proto_library(
     ],
 )
 
+tf_proto_library_py(
+    name = "xrt_proto",  # bzl adds a _py suffix
+    srcs = ["xrt.proto"],
+    visibility = ["//visibility:public"],
+)
+
 cc_library(
     name = "xrt_utils",
     srcs = [
@@ -78,6 +90,25 @@ tf_gen_op_libs(
     ],
 )
 
+tf_gen_op_wrapper_py(
+    name = "xrt_ops_wrapper_py",
+    out = "xrt_ops.py",
+    deps = [
+        ":xrt_compile_ops_op_lib",
+        ":xrt_execute_op_op_lib",
+        ":xrt_state_ops_op_lib",
+    ],
+)
+
+tf_custom_op_py_library(
+    name = "xrt_ops",
+    kernels = ["//tensorflow/compiler/xrt/kernels:xrt_ops"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":xrt_ops_wrapper_py",
+    ],
+)
+
 cc_library(
     name = "xrt_server",
     visibility = ["//visibility:public"],
diff --git a/tensorflow/compiler/xrt/kernels/BUILD b/tensorflow/compiler/xrt/kernels/BUILD
index dc02fd272f..1e325191bb 100644
--- a/tensorflow/compiler/xrt/kernels/BUILD
+++ b/tensorflow/compiler/xrt/kernels/BUILD
@@ -51,7 +51,10 @@ cc_library(
         "//tensorflow/compiler/xla/service:compiler",
         "//tensorflow/compiler/xla/service:computation_placer",
         "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/compiler/xrt:xrt_compile_ops_op_lib",
+        "//tensorflow/compiler/xrt:xrt_execute_op_op_lib",
         "//tensorflow/compiler/xrt:xrt_proto",
+        "//tensorflow/compiler/xrt:xrt_state_ops_op_lib",
         "//tensorflow/compiler/xrt:xrt_utils",
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
-- 
GitLab


From 4a2b474da2f22c6a9d3e93445624493f082d339b Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Thu, 7 Feb 2019 15:11:17 -0800
Subject: [PATCH 2099/2345] [XLA] Point some TODOs to the right bugs.

Use more-specific bug rather than the umbrella one.

PiperOrigin-RevId: 232953874
---
 tensorflow/compiler/xla/service/cpu/ir_emitter.cc           | 2 +-
 tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc | 2 +-
 tensorflow/compiler/xla/service/gpu/ir_emitter.cc           | 2 +-
 tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc  | 4 ++--
 tensorflow/compiler/xla/service/hlo_instruction.h           | 1 -
 5 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index efdda8599a..4b96f31fbe 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -1914,7 +1914,7 @@ StatusOr<llvm::Value*> IrEmitter::EmitTargetElementLoopBodyForReduce(
 }
 
 Status IrEmitter::HandleReduce(HloInstruction* reduce) {
-  // TODO(b/112040122): Support variadic reduce.
+  // TODO(b/118333695): Support variadic reduce.
   if (!reduce->shape().IsArray()) {
     return Unimplemented("Variadic reduce is not supported on CPU");
   }
diff --git a/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc
index ffd4214958..80ddb3e5cd 100644
--- a/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc
@@ -448,7 +448,7 @@ llvm_ir::ElementGenerator GpuElementalIrEmitter::MakeElementGenerator(
         return Load(accum_ptr);
       };
     case HloOpcode::kReduce:
-      // TODO(b/112040122): This should be supported.
+      // TODO(b/118332391): This should be supported.
       CHECK_EQ(hlo->operand_count(), 2) << "Did not expect variadic reduce";
       return [=, &operand_to_generator](
                  const IrArray::Index& output_index) -> StatusOr<llvm::Value*> {
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
index 0007a9a8a3..cb13e727a4 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
@@ -647,7 +647,7 @@ Status IrEmitter::HandleParameter(HloInstruction* parameter) {
 }
 
 Status IrEmitter::HandleReduce(HloInstruction* reduce) {
-  // TODO(b/112040122): Support variadic reduce.
+  // TODO(b/118332391): Support variadic reduce.
   if (!reduce->shape().IsArray()) {
     return Unimplemented("Variadic reduce is not supported on GPU");
   }
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index 4fab95920d..ee8db1fa36 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -545,7 +545,7 @@ Status IrEmitterUnnested::HandleFusion(HloInstruction* fusion) {
         // a 1D array. The specialized version requires a initializer thunk that
         // initializes the output array to the initial value of the reduce.
         if (root->opcode() == HloOpcode::kReduce && root->shape().IsTuple()) {
-          // TODO(b/112040122): Support variadic reduce.
+          // TODO(b/118332391): Support variadic reduce.
           return Unimplemented("Variadic reduce is not supported on GPU");
         }
         return EmitReductionToVector(fusion);
@@ -634,7 +634,7 @@ Status IrEmitterUnnested::EmitExtraOutputsForReduce(
 }
 
 Status IrEmitterUnnested::HandleReduce(HloInstruction* reduce) {
-  // TODO(b/112040122): Support multi-output reduce.
+  // TODO(b/118332391): Support multi-output reduce.
   if (!reduce->shape().IsArray()) {
     return Unimplemented("Multi-output reduce is not supported on GPU");
   }
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h
index 2ef72a8705..92c25ef006 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.h
+++ b/tensorflow/compiler/xla/service/hlo_instruction.h
@@ -598,7 +598,6 @@ class HloInstruction {
   // f_2 = f(f_1.tuple_element(0), ..., f_1.tuple_element(N), input0.value1,
   // ..., inputN.value1)
   // ...
-  // TODO(b/112040122): Add support to this in HLO passes and in backends.
   static std::unique_ptr<HloInstruction> CreateReduce(
       const Shape& shape, absl::Span<HloInstruction* const> operands,
       absl::Span<HloInstruction* const> init_values,
-- 
GitLab


From f841ca55bb41a0a817a5080d18270a124aa9fd1d Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Thu, 7 Feb 2019 15:24:50 -0800
Subject: [PATCH 2100/2345] Avoid converting constructors, which would require
 converting the entire class - that process is currently unreliable.

PiperOrigin-RevId: 232956583
---
 tensorflow/python/autograph/impl/api.py      | 10 ++++++++++
 tensorflow/python/autograph/impl/api_test.py | 15 +++++++++------
 2 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/autograph/impl/api.py b/tensorflow/python/autograph/impl/api.py
index abe0f25821..0f874a138a 100644
--- a/tensorflow/python/autograph/impl/api.py
+++ b/tensorflow/python/autograph/impl/api.py
@@ -196,6 +196,13 @@ def converted_call(f, owner, options, *args, **kwargs):
     logging.log(2, 'Permanently whitelisted: %s: wrapt decorated', f)
     return f(*args, **kwargs)
 
+  # Constructors are permanently whitelisted.
+  # TODO(mdan): Toggle as experimental feature instead.
+  # TODO(b/124016764): Remove this limitation.
+  if tf_inspect.isclass(f):
+    logging.log(2, 'Permanently whitelisted: %s: constructor', f)
+    return f(*args, **kwargs)
+
   # Other built-in modules are permanently whitelisted.
   # TODO(mdan): Figure out how to do this consistently for all stdlib modules.
   if (f in collections.__dict__.values() or f in pdb.__dict__.values() or
@@ -276,6 +283,9 @@ def converted_call(f, owner, options, *args, **kwargs):
 
     elif tf_inspect.isclass(f):
       # Constructors
+      # Note: Until we support class constructurs, and enable whole-class
+      # conversion with an experimental flag, this branch is dead code.
+      # TODO(mdan): Consider removing unless there is a compelling use case.
       target_entity = f
       arg_map_target = f.__init__
       effective_args = args
diff --git a/tensorflow/python/autograph/impl/api_test.py b/tensorflow/python/autograph/impl/api_test.py
index 51928099e8..c89ac579c0 100644
--- a/tensorflow/python/autograph/impl/api_test.py
+++ b/tensorflow/python/autograph/impl/api_test.py
@@ -320,6 +320,7 @@ class ApiTest(test.TestCase):
       x = api.converted_call(tc, None, converter.ConversionOptions())
       self.assertEqual(1, self.evaluate(x))
 
+  @test_util.run_deprecated_v1
   def test_converted_call_constructor(self):
 
     class TestClass(object):
@@ -332,12 +333,14 @@ class ApiTest(test.TestCase):
           return -self.x
         return self.x
 
-    with self.cached_session() as sess:
-      tc = api.converted_call(TestClass, None, converter.ConversionOptions(),
-                              constant_op.constant(-1))
-      # tc is now a converted object.
-      x = tc.test_method()
-      self.assertEqual(1, self.evaluate(x))
+    tc = api.converted_call(TestClass, None, converter.ConversionOptions(),
+                            constant_op.constant(-1))
+    # tc is still a TestClass - constructors are whitelisted.
+    # TODO(b/124016764): Support this use case.
+    # The error below is specific to the `if` statement not being converted.
+    with self.assertRaisesRegex(
+        TypeError, 'Using a `tf.Tensor` as a Python `bool`'):
+      tc.test_method()
 
   def test_converted_call_already_converted(self):
 
-- 
GitLab


From c639c153a093a24c382b8fdff56d8b3ee77cfd53 Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Thu, 7 Feb 2019 15:31:17 -0800
Subject: [PATCH 2101/2345] [tf.data] vectorization bug fix

PiperOrigin-RevId: 232957781
---
 .../optimizers/data/vectorization_utils.cc    |  4 ++++
 .../optimization/map_vectorization_test.py    | 19 +++++++++++++++++++
 2 files changed, 23 insertions(+)

diff --git a/tensorflow/core/grappler/optimizers/data/vectorization_utils.cc b/tensorflow/core/grappler/optimizers/data/vectorization_utils.cc
index c57a7b1256..1969ff00e4 100644
--- a/tensorflow/core/grappler/optimizers/data/vectorization_utils.cc
+++ b/tensorflow/core/grappler/optimizers/data/vectorization_utils.cc
@@ -415,6 +415,10 @@ Status Vectorization::Initialize(const FunctionDef& outer_scope,
 // NodeBuilder
 Status Vectorization::StackTensor(WrappedTensor* unstacked,
                                   TensorDesc* result) {
+  if (unstacked->node->output_type(unstacked->output_index) == DT_VARIANT) {
+    // TODO(b/124069171): "ExpandDims" doesn't work with Variant tensors.
+    return errors::Unimplemented("Cannot stack tensor with Variant type.");
+  }
   // Note that all these nodes are necessary as the size of the batch may not be
   // constant.
   if (unstacked->stacked) {
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
index d69043c7ad..0e14f75ead 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
@@ -541,6 +541,25 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
     optimized = unoptimized.with_options(options)
     self.assertDatasetsEqual(unoptimized, optimized)
 
+  def testOptimizationWithSparseTensor(self):
+    base_dataset = dataset_ops.Dataset.from_tensors(0)
+
+    def map_fn(x):
+      del x
+      return sparse_tensor.SparseTensor(
+          indices=[[0, 0], [1, 2]], values=[1, 2], dense_shape=[3, 4])
+
+    # Datasets with sparse tensors have unknown output shapes.
+    unoptimized = base_dataset.apply(batching.map_and_batch(map_fn, 2))
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    unoptimized = unoptimized.with_options(options)
+
+    options = dataset_ops.Options()
+    options.experimental_optimization.map_vectorization = True
+    optimized = unoptimized.with_options(options)
+    self.assertDatasetsEqual(unoptimized, optimized)
+
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 4856bbd5a575b78ca7a194e7e420702096d90aab Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Thu, 7 Feb 2019 15:32:16 -0800
Subject: [PATCH 2102/2345] [XLA:Python] Check for c_api being None during
 LocalComputation's finalizer.

During process shutdown, it seems sometimes c_api can be None, leading to noisy warnings.

PiperOrigin-RevId: 232957980
---
 tensorflow/compiler/xla/python/xla_client.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/tensorflow/compiler/xla/python/xla_client.py b/tensorflow/compiler/xla/python/xla_client.py
index 6a7a27dd61..54fd7a4f38 100644
--- a/tensorflow/compiler/xla/python/xla_client.py
+++ b/tensorflow/compiler/xla/python/xla_client.py
@@ -757,12 +757,13 @@ class LocalComputation(object):
     return [out.to_py() for out in self.ExecutePerReplica(arguments)]
 
   def __del__(self):
-    # Ensure a reference to C-based destructor for use in __del__.
-    if self._is_compiled:
-      self._backend.delete_executable(self._c_computation)
-    else:
-      assert isinstance(self._c_computation, c_api.LocalComputation)
-      c_api.DeleteLocalComputation(self._c_computation)
+    # Python may have freed c_api first.
+    if c_api and self._c_computation:
+      if self._is_compiled:
+        self._backend.delete_executable(self._c_computation)
+      else:
+        assert isinstance(self._c_computation, c_api.LocalComputation)
+        c_api.DeleteLocalComputation(self._c_computation)
 
 
 def _make_replica_group_proto(replica_group):
-- 
GitLab


From 5c909541ee86bb1130f11a5c3b6a51ee8c7e12c9 Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Thu, 7 Feb 2019 15:41:41 -0800
Subject: [PATCH 2103/2345] [Grappler] Add node name mutations to
 MutableGraphView.

PiperOrigin-RevId: 232959856
---
 .../core/grappler/mutable_graph_view.cc       | 341 ++++++++++++-
 tensorflow/core/grappler/mutable_graph_view.h |  38 +-
 .../core/grappler/mutable_graph_view_test.cc  | 474 ++++++++++++++++--
 3 files changed, 810 insertions(+), 43 deletions(-)

diff --git a/tensorflow/core/grappler/mutable_graph_view.cc b/tensorflow/core/grappler/mutable_graph_view.cc
index 64098e7e28..6a5e60edcc 100644
--- a/tensorflow/core/grappler/mutable_graph_view.cc
+++ b/tensorflow/core/grappler/mutable_graph_view.cc
@@ -106,6 +106,146 @@ bool HasRegularFaninNode(const MutableGraphView& graph, const NodeDef& node,
   return false;
 }
 
+using FanoutsMap =
+    absl::flat_hash_map<MutableGraphView::OutputPort,
+                        absl::flat_hash_set<MutableGraphView::InputPort>>;
+
+void SwapControlledFanoutInputs(const MutableGraphView& graph,
+                                const FanoutsMap::iterator& control_fanouts,
+                                absl::string_view to_node_name) {
+  absl::string_view from_node_name(control_fanouts->first.node->name());
+  string control = TensorIdToString({to_node_name, Graph::kControlSlot});
+  for (const auto& control_fanout : control_fanouts->second) {
+    const int start = graph.NumFanins(*control_fanout.node,
+                                      /*include_controlling_nodes=*/false);
+    for (int i = start; i < control_fanout.node->input_size(); ++i) {
+      TensorId tensor_id = ParseTensorName(control_fanout.node->input(i));
+      if (tensor_id.node() == from_node_name) {
+        control_fanout.node->set_input(i, control);
+        break;
+      }
+    }
+  }
+}
+
+void SwapRegularFanoutInputs(FanoutsMap* fanouts, NodeDef* from_node,
+                             absl::string_view to_node_name, int max_port) {
+  MutableGraphView::OutputPort port;
+  port.node = from_node;
+  for (int i = 0; i <= max_port; ++i) {
+    port.port_id = i;
+    auto it = fanouts->find(port);
+    if (it == fanouts->end()) {
+      continue;
+    }
+    string input = TensorIdToString({to_node_name, i});
+    for (const auto& fanout : it->second) {
+      fanout.node->set_input(fanout.port_id, input);
+    }
+  }
+}
+
+using MaxOutputPortsMap = absl::flat_hash_map<const NodeDef*, int>;
+
+void SwapFanoutInputs(const MutableGraphView& graph, FanoutsMap* fanouts,
+                      MaxOutputPortsMap* max_output_ports, NodeDef* from_node,
+                      NodeDef* to_node) {
+  auto from_control_fanouts = fanouts->find({from_node, Graph::kControlSlot});
+  if (from_control_fanouts != fanouts->end()) {
+    SwapControlledFanoutInputs(graph, from_control_fanouts, to_node->name());
+  }
+  auto to_control_fanouts = fanouts->find({to_node, Graph::kControlSlot});
+  if (to_control_fanouts != fanouts->end()) {
+    SwapControlledFanoutInputs(graph, to_control_fanouts, from_node->name());
+  }
+  auto from_max_port = max_output_ports->find(from_node);
+  if (from_max_port != max_output_ports->end()) {
+    SwapRegularFanoutInputs(fanouts, from_node, to_node->name(),
+                            from_max_port->second);
+  }
+  auto to_max_port = max_output_ports->find(to_node);
+  if (to_max_port != max_output_ports->end()) {
+    SwapRegularFanoutInputs(fanouts, to_node, from_node->name(),
+                            to_max_port->second);
+  }
+}
+
+void SwapFanoutsMapValues(FanoutsMap* fanouts,
+                          const MutableGraphView::OutputPort& from_port,
+                          const FanoutsMap::iterator& from_fanouts,
+                          const MutableGraphView::OutputPort& to_port,
+                          const FanoutsMap::iterator& to_fanouts) {
+  const bool from_exists = from_fanouts != fanouts->end();
+  const bool to_exists = to_fanouts != fanouts->end();
+
+  if (from_exists && to_exists) {
+    std::swap(from_fanouts->second, to_fanouts->second);
+  } else if (from_exists) {
+    fanouts->emplace(to_port, std::move(from_fanouts->second));
+    fanouts->erase(from_port);
+  } else if (to_exists) {
+    fanouts->emplace(from_port, std::move(to_fanouts->second));
+    fanouts->erase(to_port);
+  }
+}
+
+void SwapRegularFanoutsAndMaxPortValues(FanoutsMap* fanouts,
+                                        MaxOutputPortsMap* max_output_ports,
+                                        NodeDef* from_node, NodeDef* to_node) {
+  auto from_max_port = max_output_ports->find(from_node);
+  auto to_max_port = max_output_ports->find(to_node);
+  bool from_exists = from_max_port != max_output_ports->end();
+  bool to_exists = to_max_port != max_output_ports->end();
+
+  auto forward_fanouts = [fanouts](NodeDef* from, NodeDef* to, int start,
+                                   int end) {
+    for (int i = start; i <= end; ++i) {
+      MutableGraphView::OutputPort from_port(from, i);
+      auto from_fanouts = fanouts->find(from_port);
+      if (from_fanouts != fanouts->end()) {
+        MutableGraphView::OutputPort to_port(to, i);
+        fanouts->emplace(to_port, std::move(from_fanouts->second));
+        fanouts->erase(from_port);
+      }
+    }
+  };
+
+  if (from_exists && to_exists) {
+    const int from = from_max_port->second;
+    const int to = to_max_port->second;
+    const int shared = std::min(from, to);
+    for (int i = 0; i <= shared; ++i) {
+      MutableGraphView::OutputPort from_port(from_node, i);
+      auto from_fanouts = fanouts->find(from_port);
+      MutableGraphView::OutputPort to_port(to_node, i);
+      auto to_fanouts = fanouts->find(to_port);
+      SwapFanoutsMapValues(fanouts, from_port, from_fanouts, to_port,
+                           to_fanouts);
+    }
+    if (to > from) {
+      forward_fanouts(to_node, from_node, shared + 1, to);
+    } else if (from > to) {
+      forward_fanouts(from_node, to_node, shared + 1, from);
+    }
+
+    std::swap(from_max_port->second, to_max_port->second);
+  } else if (from_exists) {
+    forward_fanouts(from_node, to_node, 0, from_max_port->second);
+
+    max_output_ports->emplace(to_node, from_max_port->second);
+    max_output_ports->erase(from_node);
+  } else if (to_exists) {
+    forward_fanouts(to_node, from_node, 0, to_max_port->second);
+
+    max_output_ports->emplace(from_node, to_max_port->second);
+    max_output_ports->erase(to_node);
+  }
+}
+
+bool HasFanoutValue(const FanoutsMap& fanouts, const FanoutsMap::iterator& it) {
+  return it != fanouts.end() && !it->second.empty();
+}
+
 Status MutationError(absl::string_view function_name, absl::string_view params,
                      absl::string_view msg) {
   return errors::InvalidArgument(absl::Substitute(
@@ -180,6 +320,12 @@ Status CheckPortRange(int port, int min, int max, ErrorHandler handler) {
   return Status::OK();
 }
 
+string SwapNodeNamesSwitchControlErrorMsg(absl::string_view node_name) {
+  return absl::Substitute(
+      "can't swap node name '$0' as it will become a Switch control dependency",
+      node_name);
+}
+
 string GeneratedNameForIdentityConsumingSwitch(
     const MutableGraphView::OutputPort& fanin) {
   return AddPrefixToNodeName(
@@ -398,6 +544,182 @@ Status MutableGraphView::UpdateNode(
   return Status::OK();
 }
 
+Status MutableGraphView::UpdateNodeName(absl::string_view from_node_name,
+                                        absl::string_view to_node_name,
+                                        bool update_fanouts) {
+  auto error_status = [from_node_name, to_node_name,
+                       update_fanouts](absl::string_view msg) {
+    string params = absl::Substitute(
+        "from_node_name='$0', to_node_name='$1', update_fanouts=$2",
+        from_node_name, to_node_name, update_fanouts);
+    return MutationError("UpdateNodeName", params, msg);
+  };
+
+  NodeDef* node = GetNode(from_node_name);
+  TF_RETURN_IF_ERROR(CheckNodeExists(from_node_name, node, error_status));
+
+  if (node->name() == to_node_name) {
+    return Status::OK();
+  }
+  if (HasNode(to_node_name)) {
+    return error_status(
+        "can't update node name because new node name is in use");
+  }
+  auto max_output_port = max_regular_output_port().find(node);
+  const bool has_max_output_port =
+      max_output_port != max_regular_output_port().end();
+  auto control_fanouts = fanouts().find({node, Graph::kControlSlot});
+
+  if (update_fanouts) {
+    SwapControlledFanoutInputs(*this, control_fanouts, to_node_name);
+    if (has_max_output_port) {
+      SwapRegularFanoutInputs(&fanouts(), node, to_node_name,
+                              max_output_port->second);
+    }
+  } else if (has_max_output_port ||
+             HasFanoutValue(fanouts(), control_fanouts)) {
+    return error_status("can't update node name because node has fanouts");
+  }
+
+  nodes().erase(node->name());
+  node->set_name(string(to_node_name));
+  nodes().emplace(node->name(), node);
+  return Status::OK();
+}
+
+Status MutableGraphView::SwapNodeNames(absl::string_view from_node_name,
+                                       absl::string_view to_node_name,
+                                       bool update_fanouts) {
+  auto error_status = [from_node_name, to_node_name,
+                       update_fanouts](absl::string_view msg) {
+    string params = absl::Substitute(
+        "from_node_name='$0', to_node_name='$1', update_fanouts=$2",
+        from_node_name, to_node_name, update_fanouts);
+    return MutationError("SwapNodeNames", params, msg);
+  };
+
+  NodeDef* from_node = GetNode(from_node_name);
+  TF_RETURN_IF_ERROR(CheckNodeExists(from_node_name, from_node, error_status));
+  if (from_node_name == to_node_name) {
+    return Status::OK();
+  }
+  NodeDef* to_node = GetNode(to_node_name);
+  TF_RETURN_IF_ERROR(CheckNodeExists(to_node_name, to_node, error_status));
+
+  auto swap_names = [this, from_node, to_node]() {
+    nodes().erase(from_node->name());
+    nodes().erase(to_node->name());
+    std::swap(*from_node->mutable_name(), *to_node->mutable_name());
+    nodes().emplace(from_node->name(), from_node);
+    nodes().emplace(to_node->name(), to_node);
+  };
+
+  if (update_fanouts) {
+    SwapFanoutInputs(*this, &fanouts(), &max_regular_output_port(), from_node,
+                     to_node);
+    swap_names();
+    return Status::OK();
+  }
+
+  bool from_is_switch = IsSwitch(*from_node);
+  MutableGraphView::OutputPort to_control(to_node, Graph::kControlSlot);
+  auto to_control_fanouts = fanouts().find(to_control);
+  if (from_is_switch && HasFanoutValue(fanouts(), to_control_fanouts)) {
+    return error_status(SwapNodeNamesSwitchControlErrorMsg(from_node_name));
+  }
+
+  bool to_is_switch = IsSwitch(*to_node);
+  MutableGraphView::OutputPort from_control(from_node, Graph::kControlSlot);
+  auto from_control_fanouts = fanouts().find(from_control);
+  if (to_is_switch && HasFanoutValue(fanouts(), from_control_fanouts)) {
+    return error_status(SwapNodeNamesSwitchControlErrorMsg(to_node_name));
+  }
+
+  // Swap node names.
+  swap_names();
+
+  // Swap controlling fanouts.
+  SwapFanoutsMapValues(&fanouts(), from_control, from_control_fanouts,
+                       to_control, to_control_fanouts);
+
+  // Swap regular fanouts.
+  SwapRegularFanoutsAndMaxPortValues(&fanouts(), &max_regular_output_port(),
+                                     from_node, to_node);
+
+  // Update fanins to remove self loops.
+  auto update_fanins = [this](NodeDef* node, absl::string_view old_node_name) {
+    for (int i = 0; i < node->input_size(); ++i) {
+      TensorId tensor_id = ParseTensorName(node->input(i));
+      if (tensor_id.node() == node->name()) {
+        const int idx = tensor_id.index();
+        const int node_idx =
+            IsTensorIdControlling(tensor_id) ? Graph::kControlSlot : i;
+
+        MutableGraphView::OutputPort from_fanin(node, idx);
+        absl::flat_hash_set<InputPort>* from_fanouts = &fanouts()[from_fanin];
+        from_fanouts->erase({node, node_idx});
+        UpdateMaxRegularOutputPortForRemovedFanin(from_fanin, *from_fanouts);
+
+        MutableGraphView::OutputPort to_fanin(nodes().at(old_node_name), idx);
+        fanouts()[to_fanin].insert({node, node_idx});
+        UpdateMaxRegularOutputPortForAddedFanin(to_fanin);
+        node->set_input(i, TensorIdToString({old_node_name, idx}));
+      }
+    }
+  };
+  update_fanins(from_node, to_node->name());
+  update_fanins(to_node, from_node->name());
+
+  // Dedup control dependencies.
+  auto dedup_control_fanouts =
+      [this](NodeDef* node, const FanoutsMap::iterator& control_fanouts) {
+        if (CanDedupControlWithRegularInput(*this, *node) &&
+            control_fanouts != fanouts().end()) {
+          for (const auto& control_fanout : control_fanouts->second) {
+            if (HasRegularFaninNode(*this, *control_fanout.node,
+                                    node->name())) {
+              RemoveControllingFaninInternal(control_fanout.node, node);
+            }
+          }
+        }
+      };
+  auto dedup_switch_control = [this, dedup_control_fanouts](NodeDef* node) {
+    OutputPort port;
+    port.node = node;
+    const int max_port =
+        gtl::FindWithDefault(max_regular_output_port(), node, -1);
+    for (int i = 0; i <= max_port; ++i) {
+      port.port_id = i;
+      auto it = fanouts().find(port);
+      if (it == fanouts().end()) {
+        continue;
+      }
+      for (const auto& fanout : it->second) {
+        auto fanout_controls =
+            fanouts().find({fanout.node, Graph::kControlSlot});
+        dedup_control_fanouts(fanout.node, fanout_controls);
+      }
+    }
+  };
+
+  if (!from_is_switch) {
+    if (to_is_switch) {
+      dedup_switch_control(from_node);
+    } else {
+      dedup_control_fanouts(from_node, from_control_fanouts);
+    }
+  }
+  if (!to_is_switch) {
+    if (from_is_switch) {
+      dedup_switch_control(to_node);
+    } else {
+      dedup_control_fanouts(to_node, to_control_fanouts);
+    }
+  }
+
+  return Status::OK();
+}
+
 Status MutableGraphView::UpdateFanouts(absl::string_view from_node_name,
                                        absl::string_view to_node_name) {
   NodeDef* from_node = GetNode(from_node_name);
@@ -625,9 +947,9 @@ NodeDef* MutableGraphView::GetControllingFaninToAdd(absl::string_view node_name,
   if (!IsSwitch(*fanin.node)) {
     return fanin.node;
   } else {
-    TensorId tensor_id(fanin.node->name(), fanin.port_id);
     if (IsOutputPortControlling(fanin)) {
       // Can't add a Switch node control dependency.
+      TensorId tensor_id(fanin.node->name(), fanin.port_id);
       *error_msg = absl::Substitute(
           "can't add fanin '$0' as it will become a Switch control dependency",
           tensor_id.ToString());
@@ -639,18 +961,15 @@ NodeDef* MutableGraphView::GetControllingFaninToAdd(absl::string_view node_name,
     // dependency is only triggered when the corresponding output is triggered.
     // We start by looking for an identity node connected to the output of the
     // switch node, and use it to anchor the control dependency.
-    auto fanouts = GetFanouts(*fanin.node, /*include_controlled_nodes=*/false);
-    for (auto fanout : fanouts) {
+    for (const auto& fanout : GetFanout(fanin)) {
       if (IsIdentity(*fanout.node) || IsIdentityNSingleInput(*fanout.node)) {
-        if (ParseTensorName(fanout.node->input(0)) == tensor_id) {
-          if (fanout.node->name() == node_name) {
-            *error_msg =
-                absl::Substitute("can't add found fanin '$0' to self",
-                                 AsControlDependency(fanout.node->name()));
-            return nullptr;
-          }
-          return fanout.node;
+        if (fanout.node->name() == node_name) {
+          *error_msg =
+              absl::Substitute("can't add found fanin '$0' to self",
+                               AsControlDependency(fanout.node->name()));
+          return nullptr;
         }
+        return fanout.node;
       }
     }
 
diff --git a/tensorflow/core/grappler/mutable_graph_view.h b/tensorflow/core/grappler/mutable_graph_view.h
index 08a1cd18a1..a09c147be6 100644
--- a/tensorflow/core/grappler/mutable_graph_view.h
+++ b/tensorflow/core/grappler/mutable_graph_view.h
@@ -84,6 +84,40 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
                     absl::string_view device,
                     absl::Span<const std::pair<string, AttrValue>> attrs);
 
+  // Updates node `from_node_name` name to `to_node_name`. If `to_node_name` is
+  // in use, node `from_node_name` does not exist, or node `from_node_name` has
+  // fanouts and `update_fanouts` is set to false, an error will be returned and
+  // nothing will be modified in the graph.
+  Status UpdateNodeName(absl::string_view from_node_name,
+                        absl::string_view to_node_name, bool update_fanouts);
+
+  // Swap node names `from_node_name` and `to_node_name`. Self loops of one node
+  // are removed by updating the inputs introducing self loops to use the other
+  // node's name. Setting `update_fanouts` to false will exclude other fanouts
+  // from having their inputs updated, but inputs introducing self loops will
+  // always be updated regardless of `update_fanouts.
+  //
+  // Example:
+  //   1. foo(other:3, bar:2, ^bar)
+  //   2. bar(foo:3, other:1, foo:1, ^foo)
+  //   3. other(foo:5, bar:6)
+  //
+  // After calling SwapNodeNames("foo", "bar", false):
+  //   1. bar(other:3, foo:2, ^foo)
+  //   2. foo(bar:3, other:1, bar:1, ^bar)
+  //   3. other(foo:5, bar:6)
+  //
+  // After calling SwapNodeNames("foo", "bar", true):
+  //   1. bar(other:3, foo:2, ^foo)
+  //   2. foo(bar:3, other:1, bar:1, ^bar)
+  //   3. other(bar:5, foo:6)
+  //
+  // If it is not possible to swap node names (i.e. nodes do not exist or Switch
+  // control dependency may be introduced), an error will be returned and
+  // nothing will be modified in the graph.
+  Status SwapNodeNames(absl::string_view from_node_name,
+                       absl::string_view to_node_name, bool update_fanouts);
+
   // Updates all fanouts (input ports fetching output tensors) from
   // `from_node_name` to the `to_node_name`, including control dependencies.
   //
@@ -92,7 +126,7 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   //   2. foo2(bar:1, other:1)
   //   3. foo3(other:2, ^bar)
   //
-  // After calling ForwardOutputs(bar, new_bar):
+  // After calling UpdateFanouts(bar, new_bar):
   //   1. foo1(new_bar:0, new_bar:1, other:0)
   //   2. foo2(new_bar:1, other:1)
   //   3. foo3(other:2, ^new_bar)
@@ -240,7 +274,7 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   //   2. foo2(bar:1, other:1)
   //   3. foo3(other:2, ^bar)
   //
-  // After calling ForwardOutputs(bar, new_bar):
+  // After calling UpdateFanouts(bar, new_bar):
   //   1. foo1(new_bar:0, new_bar:1, other:0)
   //   2. foo2(new_bar:1, other:1)
   //   3. foo3(other:2, ^new_bar)
diff --git a/tensorflow/core/grappler/mutable_graph_view_test.cc b/tensorflow/core/grappler/mutable_graph_view_test.cc
index 06333d3a34..07818d1f52 100644
--- a/tensorflow/core/grappler/mutable_graph_view_test.cc
+++ b/tensorflow/core/grappler/mutable_graph_view_test.cc
@@ -305,6 +305,450 @@ TEST(MutableGraphViewTest, UpdateNodeSwitchControlDependency) {
   CheckGraph(graph);
 }
 
+absl::flat_hash_map<string, std::vector<string>> GetNodeInputsFromGraph(
+    const GraphDef& graph, absl::string_view node_to_exclude) {
+  absl::flat_hash_map<string, std::vector<string>> node_inputs;
+  for (const auto& node : graph.node()) {
+    if (node.name() == node_to_exclude) {
+      continue;
+    }
+    node_inputs[node.name()] =
+        std::vector<string>(node.input().begin(), node.input().end());
+  }
+  return node_inputs;
+}
+
+void CheckUnmodifiedNodeFanins(
+    const GraphDef& graph, absl::string_view node_to_exclude,
+    const absl::flat_hash_map<string, std::vector<string>>&
+        unmodified_node_inputs) {
+  for (const auto& node : graph.node()) {
+    if (node.name() == node_to_exclude) {
+      continue;
+    }
+    auto it = unmodified_node_inputs.find(node.name());
+    ASSERT_NE(it, unmodified_node_inputs.end());
+    ASSERT_EQ(it->second.size(), node.input_size());
+    for (int i = 0; i < node.input_size(); ++i) {
+      EXPECT_EQ(node.input(i), it->second[i]);
+    }
+  }
+}
+
+void TestUpdateNodeName(absl::string_view from_node_name, bool node_exists,
+                        absl::string_view to_node_name, bool update_fanouts,
+                        bool success, const string& error_msg,
+                        absl::Span<const string> expected_fanins) {
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {"a"}),
+       NDef("c", "NotImportant", {}, {})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  NodeDef* node = graph.GetNode(from_node_name);
+  if (node_exists) {
+    EXPECT_NE(node, nullptr);
+  } else {
+    EXPECT_EQ(node, nullptr);
+  }
+
+  absl::flat_hash_map<string, std::vector<string>> unmodified_node_inputs =
+      GetNodeInputsFromGraph(graph_def, from_node_name);
+
+  Status s = graph.UpdateNodeName(from_node_name, to_node_name, update_fanouts);
+  EXPECT_EQ(s.ok(), success);
+  string updated_node_name;
+  if (success) {
+    updated_node_name = string(to_node_name);
+  } else {
+    updated_node_name = string(from_node_name);
+    EXPECT_EQ(s.error_message(), error_msg);
+  }
+  if (node_exists) {
+    EXPECT_EQ(node->name(), updated_node_name);
+    CompareNodeFanins(graph, node, expected_fanins);
+  }
+
+  CheckUnmodifiedNodeFanins(graph_def, updated_node_name,
+                            unmodified_node_inputs);
+
+  CheckGraph(graph);
+}
+
+TEST(MutableGraphViewTest, UpdateNodeName) {
+  string error_msg;
+  // Node has no fanouts.
+  TestUpdateNodeName("b", /*node_exists=*/true, "d", /*update_fanouts=*/false,
+                     /*success=*/true, error_msg, {"a"});
+  // Node has fanouts and rename to self.
+  TestUpdateNodeName("b", /*node_exists=*/true, "b", /*update_fanouts=*/false,
+                     /*success=*/true, error_msg, {"a"});
+  // Node has no fanouts and rename to self.
+  TestUpdateNodeName("a", /*node_exists=*/true, "a", /*update_fanouts=*/false,
+                     /*success=*/true, error_msg, {});
+
+  // New node name is in use.
+  error_msg =
+      "MutableGraphView::UpdateNodeName(from_node_name='c', to_node_name='b', "
+      "update_fanouts=false) error: can't update node name because new node "
+      "name is in use.";
+  TestUpdateNodeName("c", /*node_exists=*/true, "b", /*update_fanouts=*/false,
+                     /*success=*/false, error_msg, {});
+  error_msg =
+      "MutableGraphView::UpdateNodeName(from_node_name='a', to_node_name='b', "
+      "update_fanouts=true) error: can't update node name because new node "
+      "name is in use.";
+  TestUpdateNodeName("a", /*node_exists=*/true, "b", /*update_fanouts=*/true,
+                     /*success=*/false, error_msg, {});
+  // Node has fanouts.
+  error_msg =
+      "MutableGraphView::UpdateNodeName(from_node_name='a', to_node_name='d', "
+      "update_fanouts=false) error: can't update node name because node has "
+      "fanouts.";
+  TestUpdateNodeName("a", /*node_exists=*/true, "d", /*update_fanouts=*/false,
+                     /*success=*/false, error_msg, {});
+  // Node does not exist.
+  error_msg =
+      "MutableGraphView::UpdateNodeName(from_node_name='d', to_node_name='e', "
+      "update_fanouts=false) error: node 'd' was not found.";
+  TestUpdateNodeName("d", /*node_exists=*/false, "e", /*update_fanouts=*/false,
+                     /*success=*/false, error_msg, {});
+  error_msg =
+      "MutableGraphView::UpdateNodeName(from_node_name='d', to_node_name='e', "
+      "update_fanouts=true) error: node 'd' was not found.";
+  TestUpdateNodeName("d", /*node_exists=*/false, "e", /*update_fanouts=*/true,
+                     /*success=*/false, error_msg, {});
+}
+
+TEST(MutableGraphViewTest, UpdateNodeNameWithFanouts) {
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {"a:2"}),
+       NDef("c", "NotImportant", {"b", "^a"}),
+       NDef("d", "NotImportant", {"^b", "^a"}),
+       NDef("e", "NotImportant", {"b:2", "c:4", "b:1", "^a"})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  TF_EXPECT_OK(graph.UpdateNodeName("b", "f", /*update_fanouts=*/true));
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"f", "^c", "^d", "^e"});
+  CheckNode(graph, "f", "NotImportant", "", {}, {"a:2"},
+            {"c", "^d", "e", "e:2"});
+  CheckNode(graph, "c", "NotImportant", "", {}, {"f", "^a"}, {"e:1"});
+  CheckNode(graph, "d", "NotImportant", "", {}, {"^f", "^a"}, {});
+  CheckNode(graph, "e", "NotImportant", "", {}, {"f:2", "c:4", "f:1", "^a"},
+            {});
+
+  CheckGraph(graph);
+}
+
+GraphDef SimpleSwapNodeNamesMutationGraph() {
+  return test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("switch_1", "Switch", {"a"}),
+       NDef("identity_1", "Identity", {"switch_1:1"}),
+       NDef("b", "NotImportant", {}, {}), NDef("switch_2", "Switch", {"b"}),
+       NDef("identity_2", "Identity", {"switch_2:0"}),
+       NDef("foo_1", "NotImportant", {"identity_1", "^identity_1"}),
+       NDef("foo_2", "NotImportant", {"identity_2", "^identity_2"})},
+      /*funcs=*/{});
+}
+
+void TestSwapNodeNames(bool update_fanouts) {
+  GraphDef graph_def = SimpleSwapNodeNamesMutationGraph();
+
+  MutableGraphView graph(&graph_def);
+
+  TF_EXPECT_OK(graph.SwapNodeNames("foo_1", "foo_2", update_fanouts));
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"switch_1"});
+  CheckNode(graph, "switch_1", "Switch", "", {}, {"a"}, {"identity_1"});
+  CheckNode(graph, "identity_1", "Identity", "", {}, {"switch_1:1"},
+            {"foo_2", "^foo_2"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {}, {"switch_2"});
+  CheckNode(graph, "switch_2", "Switch", "", {}, {"b"}, {"identity_2"});
+  CheckNode(graph, "identity_2", "Identity", "", {}, {"switch_2:0"},
+            {"foo_1", "^foo_1"});
+  CheckNode(graph, "foo_2", "NotImportant", "", {},
+            {"identity_1", "^identity_1"}, {});
+  CheckNode(graph, "foo_1", "NotImportant", "", {},
+            {"identity_2", "^identity_2"}, {});
+
+  CheckGraph(graph);
+}
+
+TEST(MutableGraphView, SwapNodeNames) {
+  TestSwapNodeNames(/*update_fanouts=*/false);
+  TestSwapNodeNames(/*update_fanouts=*/true);
+}
+
+void TestSwapNodeNamesWithSameNames(bool update_fanouts) {
+  GraphDef graph_def = SimpleSwapNodeNamesMutationGraph();
+
+  MutableGraphView graph(&graph_def);
+
+  TF_EXPECT_OK(graph.SwapNodeNames("identity_1", "identity_1", update_fanouts));
+
+  // No changes to graph.
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"switch_1"});
+  CheckNode(graph, "switch_1", "Switch", "", {}, {"a"}, {"identity_1"});
+  CheckNode(graph, "identity_1", "Identity", "", {}, {"switch_1:1"},
+            {"foo_1", "^foo_1"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {}, {"switch_2"});
+  CheckNode(graph, "switch_2", "Switch", "", {}, {"b"}, {"identity_2"});
+  CheckNode(graph, "identity_2", "Identity", "", {}, {"switch_2:0"},
+            {"foo_2", "^foo_2"});
+  CheckNode(graph, "foo_1", "NotImportant", "", {},
+            {"identity_1", "^identity_1"}, {});
+  CheckNode(graph, "foo_2", "NotImportant", "", {},
+            {"identity_2", "^identity_2"}, {});
+
+  CheckGraph(graph);
+}
+
+TEST(MutableGraphView, SwapNodeNamesSameName) {
+  TestSwapNodeNamesWithSameNames(/*update_fanouts=*/false);
+  TestSwapNodeNamesWithSameNames(/*update_fanouts=*/true);
+}
+
+TEST(MutableGraphView, SwapNodeNamesBetweenSwitches) {
+  GraphDef graph_def = SimpleSwapNodeNamesMutationGraph();
+
+  MutableGraphView graph(&graph_def);
+
+  TF_EXPECT_OK(
+      graph.SwapNodeNames("switch_1", "switch_2", /*update_fanouts=*/false));
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"switch_2"});
+  CheckNode(graph, "switch_2", "Switch", "", {}, {"a"}, {"identity_2"});
+  CheckNode(graph, "identity_1", "Identity", "", {}, {"switch_1:1"},
+            {"foo_1", "^foo_1"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {}, {"switch_1"});
+  CheckNode(graph, "switch_1", "Switch", "", {}, {"b"}, {"identity_1"});
+  CheckNode(graph, "identity_2", "Identity", "", {}, {"switch_2:0"},
+            {"foo_2", "^foo_2"});
+  CheckNode(graph, "foo_1", "NotImportant", "", {},
+            {"identity_1", "^identity_1"}, {});
+  CheckNode(graph, "foo_2", "NotImportant", "", {},
+            {"identity_2", "^identity_2"}, {});
+
+  CheckGraph(graph);
+}
+
+TEST(MutableGraphView, SwapNodeNamesBetweenSwitchesAndUpdateFanouts) {
+  GraphDef graph_def = SimpleSwapNodeNamesMutationGraph();
+
+  MutableGraphView graph(&graph_def);
+
+  TF_EXPECT_OK(
+      graph.SwapNodeNames("switch_1", "switch_2", /*update_fanouts=*/true));
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"switch_2"});
+  CheckNode(graph, "switch_2", "Switch", "", {}, {"a"}, {"identity_1"});
+  CheckNode(graph, "identity_1", "Identity", "", {}, {"switch_2:1"},
+            {"foo_1", "^foo_1"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {}, {"switch_1"});
+  CheckNode(graph, "switch_1", "Switch", "", {}, {"b"}, {"identity_2"});
+  CheckNode(graph, "identity_2", "Identity", "", {}, {"switch_1:0"},
+            {"foo_2", "^foo_2"});
+  CheckNode(graph, "foo_1", "NotImportant", "", {},
+            {"identity_1", "^identity_1"}, {});
+  CheckNode(graph, "foo_2", "NotImportant", "", {},
+            {"identity_2", "^identity_2"}, {});
+
+  CheckGraph(graph);
+}
+
+TEST(MutableGraphView, SwapNodeNamesSwitchAndNonSwitch) {
+  GraphDef graph_def = SimpleSwapNodeNamesMutationGraph();
+
+  MutableGraphView graph(&graph_def);
+
+  TF_EXPECT_OK(graph.SwapNodeNames("a", "switch_1", /*update_fanouts=*/false));
+
+  // Dedup controls and fix self loop.
+  CheckNode(graph, "switch_1", "NotImportant", "", {}, {}, {"a", "identity_1"});
+  CheckNode(graph, "a", "Switch", "", {}, {"switch_1"}, {});
+  CheckNode(graph, "identity_1", "Identity", "", {}, {"switch_1:1"}, {"foo_1"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {}, {"switch_2"});
+  CheckNode(graph, "switch_2", "Switch", "", {}, {"b"}, {"identity_2"});
+  CheckNode(graph, "identity_2", "Identity", "", {}, {"switch_2:0"},
+            {"foo_2", "^foo_2"});
+  CheckNode(graph, "foo_1", "NotImportant", "", {}, {"identity_1"}, {});
+  CheckNode(graph, "foo_2", "NotImportant", "", {},
+            {"identity_2", "^identity_2"}, {});
+
+  CheckGraph(graph);
+}
+
+TEST(MutableGraphView, SwapNodeNamesSwitchAndNonSwitchAndUpdateFanouts) {
+  GraphDef graph_def = SimpleSwapNodeNamesMutationGraph();
+
+  MutableGraphView graph(&graph_def);
+
+  TF_EXPECT_OK(graph.SwapNodeNames("a", "switch_1", /*update_fanouts=*/true));
+
+  CheckNode(graph, "switch_1", "NotImportant", "", {}, {}, {"a"});
+  CheckNode(graph, "a", "Switch", "", {}, {"switch_1"}, {"identity_1"});
+  CheckNode(graph, "identity_1", "Identity", "", {}, {"a:1"},
+            {"foo_1", "^foo_1"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {}, {"switch_2"});
+  CheckNode(graph, "switch_2", "Switch", "", {}, {"b"}, {"identity_2"});
+  CheckNode(graph, "identity_2", "Identity", "", {}, {"switch_2:0"},
+            {"foo_2", "^foo_2"});
+  CheckNode(graph, "foo_1", "NotImportant", "", {},
+            {"identity_1", "^identity_1"}, {});
+  CheckNode(graph, "foo_2", "NotImportant", "", {},
+            {"identity_2", "^identity_2"}, {});
+
+  CheckGraph(graph);
+}
+
+TEST(MutableGraphView, SwapNodeNamesNonSwitchAndSwitch) {
+  GraphDef graph_def = SimpleSwapNodeNamesMutationGraph();
+
+  MutableGraphView graph(&graph_def);
+
+  TF_EXPECT_OK(graph.SwapNodeNames("switch_2", "b", /*update_fanouts=*/false));
+
+  // Dedup controls and fix self loop.
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"switch_1"});
+  CheckNode(graph, "switch_1", "Switch", "", {}, {"a"}, {"identity_1"});
+  CheckNode(graph, "identity_1", "Identity", "", {}, {"switch_1:1"},
+            {"foo_1", "^foo_1"});
+  CheckNode(graph, "switch_2", "NotImportant", "", {}, {}, {"b", "identity_2"});
+  CheckNode(graph, "b", "Switch", "", {}, {"switch_2"}, {});
+  CheckNode(graph, "identity_2", "Identity", "", {}, {"switch_2:0"}, {"foo_2"});
+  CheckNode(graph, "foo_1", "NotImportant", "", {},
+            {"identity_1", "^identity_1"}, {});
+  CheckNode(graph, "foo_2", "NotImportant", "", {}, {"identity_2"}, {});
+
+  CheckGraph(graph);
+}
+
+TEST(MutableGraphView, SwapNodeNamesNonSwitchAndSwitchAndUpdateFanouts) {
+  GraphDef graph_def = SimpleSwapNodeNamesMutationGraph();
+
+  MutableGraphView graph(&graph_def);
+
+  TF_EXPECT_OK(graph.SwapNodeNames("switch_2", "b", /*update_fanouts=*/true));
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"switch_1"});
+  CheckNode(graph, "switch_1", "Switch", "", {}, {"a"}, {"identity_1"});
+  CheckNode(graph, "identity_1", "Identity", "", {}, {"switch_1:1"},
+            {"foo_1", "^foo_1"});
+  CheckNode(graph, "switch_2", "NotImportant", "", {}, {}, {"b"});
+  CheckNode(graph, "b", "Switch", "", {}, {"switch_2"}, {"identity_2"});
+  CheckNode(graph, "identity_2", "Identity", "", {}, {"b:0"},
+            {"foo_2", "^foo_2"});
+  CheckNode(graph, "foo_1", "NotImportant", "", {},
+            {"identity_1", "^identity_1"}, {});
+  CheckNode(graph, "foo_2", "NotImportant", "", {},
+            {"identity_2", "^identity_2"}, {});
+
+  CheckGraph(graph);
+}
+
+void TestSwapNodeNamesSimpleSelfLoop(bool update_fanouts) {
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {"b:7"}), NDef("b", "NotImportant", {"a:10"})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  TF_EXPECT_OK(graph.SwapNodeNames("a", "b", update_fanouts));
+
+  // No self loops.
+  CheckNode(graph, "a", "NotImportant", "", {}, {"b:10"}, {"b:0"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {"a:7"}, {"a:0"});
+
+  CheckGraph(graph);
+}
+
+TEST(MutableGraphView, SwapNodeNamesSelfLoops) {
+  TestSwapNodeNamesSimpleSelfLoop(/*update_fanouts=*/false);
+  TestSwapNodeNamesSimpleSelfLoop(/*update_fanouts=*/true);
+}
+
+void TestSwapNodeNamesError(absl::string_view from_node_name,
+                            absl::string_view to_node_name, bool update_fanouts,
+                            const string& error_msg) {
+  GraphDef graph_def = SimpleSwapNodeNamesMutationGraph();
+
+  MutableGraphView graph(&graph_def);
+
+  Status s = graph.SwapNodeNames(from_node_name, to_node_name, update_fanouts);
+  EXPECT_EQ(s.ok(), false);
+  EXPECT_EQ(s.error_message(), error_msg);
+
+  // No changes to graph.
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"switch_1"});
+  CheckNode(graph, "switch_1", "Switch", "", {}, {"a"}, {"identity_1"});
+  CheckNode(graph, "identity_1", "Identity", "", {}, {"switch_1:1"},
+            {"foo_1", "^foo_1"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {}, {"switch_2"});
+  CheckNode(graph, "switch_2", "Switch", "", {}, {"b"}, {"identity_2"});
+  CheckNode(graph, "identity_2", "Identity", "", {}, {"switch_2:0"},
+            {"foo_2", "^foo_2"});
+  CheckNode(graph, "foo_1", "NotImportant", "", {},
+            {"identity_1", "^identity_1"}, {});
+  CheckNode(graph, "foo_2", "NotImportant", "", {},
+            {"identity_2", "^identity_2"}, {});
+
+  CheckGraph(graph);
+}
+
+// TODO(lyandy): add tests with update_fanouts == true.
+TEST(MutableGraphView, SwapNodeNamesError) {
+  string error_msg;
+  // Missing nodes.
+  error_msg =
+      "MutableGraphView::SwapNodeNames(from_node_name='foo_3', "
+      "to_node_name='foo_2', update_fanouts=false) error: node 'foo_3' was not "
+      "found.";
+  TestSwapNodeNamesError("foo_3", "foo_2", /*update_fanouts=*/false, error_msg);
+  error_msg =
+      "MutableGraphView::SwapNodeNames(from_node_name='foo_3', "
+      "to_node_name='foo_2', update_fanouts=true) error: node 'foo_3' was not "
+      "found.";
+  TestSwapNodeNamesError("foo_3", "foo_2", /*update_fanouts=*/true, error_msg);
+  error_msg =
+      "MutableGraphView::SwapNodeNames(from_node_name='foo_1', "
+      "to_node_name='foo_4', update_fanouts=false) error: node 'foo_4' was not "
+      "found.";
+  TestSwapNodeNamesError("foo_1", "foo_4", /*update_fanouts=*/false, error_msg);
+  error_msg =
+      "MutableGraphView::SwapNodeNames(from_node_name='foo_1', "
+      "to_node_name='foo_4', update_fanouts=true) error: node 'foo_4' was not "
+      "found.";
+  TestSwapNodeNamesError("foo_1", "foo_4", /*update_fanouts=*/true, error_msg);
+  error_msg =
+      "MutableGraphView::SwapNodeNames(from_node_name='foo_5', "
+      "to_node_name='foo_6', update_fanouts=false) error: node 'foo_5' was not "
+      "found.";
+  TestSwapNodeNamesError("foo_5", "foo_6", /*update_fanouts=*/false, error_msg);
+  error_msg =
+      "MutableGraphView::SwapNodeNames(from_node_name='foo_5', "
+      "to_node_name='foo_6', update_fanouts=true) error: node 'foo_5' was not "
+      "found.";
+  TestSwapNodeNamesError("foo_5", "foo_6", /*update_fanouts=*/true, error_msg);
+
+  // Switch control dependencies.
+  error_msg =
+      "MutableGraphView::SwapNodeNames(from_node_name='switch_2', "
+      "to_node_name='identity_1', update_fanouts=false) error: can't swap node "
+      "name 'switch_2' as it will become a Switch control dependency.";
+  TestSwapNodeNamesError("switch_2", "identity_1", /*update_fanouts=*/false,
+                         error_msg);
+  error_msg =
+      "MutableGraphView::SwapNodeNames(from_node_name='identity_2', "
+      "to_node_name='switch_1', update_fanouts=false) error: can't swap node "
+      "name 'switch_1' as it will become a Switch control dependency.";
+  TestSwapNodeNamesError("identity_2", "switch_1", /*update_fanouts=*/false,
+                         error_msg);
+}
+
 TEST(MutableGraphViewTest, AddAndUpdateFanouts) {
   // Actual node.op() is not important in this test.
   GraphDef graph_def = test::function::GDef(
@@ -461,36 +905,6 @@ GraphDef SimpleMutateFaninGraph() {
   return graph_def;
 }
 
-absl::flat_hash_map<string, std::vector<string>> GetNodeInputsFromGraph(
-    const GraphDef& graph, absl::string_view node_to_exclude) {
-  absl::flat_hash_map<string, std::vector<string>> node_inputs;
-  for (const auto& node : graph.node()) {
-    if (node.name() == node_to_exclude) {
-      continue;
-    }
-    node_inputs[node.name()] =
-        std::vector<string>(node.input().begin(), node.input().end());
-  }
-  return node_inputs;
-}
-
-void CheckUnmodifiedNodeFanins(
-    const GraphDef& graph, absl::string_view node_to_exclude,
-    const absl::flat_hash_map<string, std::vector<string>>&
-        unmodified_node_inputs) {
-  for (const auto& node : graph.node()) {
-    if (node.name() == node_to_exclude) {
-      continue;
-    }
-    auto it = unmodified_node_inputs.find(node.name());
-    ASSERT_NE(it, unmodified_node_inputs.end());
-    ASSERT_EQ(it->second.size(), node.input_size());
-    for (int i = 0; i < node.input_size(); ++i) {
-      EXPECT_EQ(node.input(i), it->second[i]);
-    }
-  }
-}
-
 void TestAddRegularFanin(absl::string_view node_name, bool node_exists,
                          const TensorId& fanin_to_add, bool success,
                          const string& error_msg,
-- 
GitLab


From 53601bb2682358cb8beeea00d19989f3dd85d2a5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 7 Feb 2019 15:49:31 -0800
Subject: [PATCH 2104/2345] Expand coverage by adding tests to
 function_test.py.

PiperOrigin-RevId: 232961152
---
 tensorflow/python/eager/function_test.py | 175 ++++++++++++++---------
 1 file changed, 111 insertions(+), 64 deletions(-)

diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index 0572192c86..a790df7d8f 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -381,13 +381,13 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     x = random_ops.random_uniform([2, 2]).numpy()
     defined = function.defun(f)
     defined(x)
-    self.assertEqual(len(defined._function_cache), 1)
+    self.assertLen(defined._function_cache, 1)
 
     x = random_ops.random_uniform([2, 2]).numpy()
     defined(x)
     # A NumPy array with different values but the same shape and dtype
     # shouldn't trigger another function definition.
-    self.assertEqual(len(defined._function_cache), 1)
+    self.assertLen(defined._function_cache, 1)
 
     # Test that the numpy array is properly an argument to the graph function.
     self.assertEqual(1., defined(numpy.ones([])).numpy())
@@ -595,7 +595,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     run_metadata = context.export_run_metadata()
     context.disable_run_metadata()
     step_stats = run_metadata.step_stats
-    self.assertGreater(len(step_stats.dev_stats), 0)
+    self.assertNotEmpty(step_stats.dev_stats)
     cpu_stats = step_stats.dev_stats[0]
     self.assertEqual('/job:localhost/replica:0/task:0/device:CPU:0',
                      cpu_stats.device)
@@ -604,10 +604,10 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     # arbitrarily many (placeholders, return identities, etc, might be included
     # or not in the future, so shouldn't be tested for exactly.
     self.assertGreaterEqual(len(cpu_stats.node_stats), 2)
-    self.assertEqual(len(run_metadata.partition_graphs), 1)
+    self.assertLen(run_metadata.partition_graphs, 1)
 
   def testGraphModeCaptureVariable(self):
-    with context.graph_mode(), self.cached_session() as sess:
+    with context.graph_mode(), self.cached_session():
 
       class HasAVar(object):
 
@@ -932,7 +932,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
                                   constant_op.constant(4)],
         constant_op.constant(5)
     ])
-    self.assertEqual(len(ret), 2)
+    self.assertLen(ret, 2)
     self.assertAllEqual(ret[0][0], 2)
     self.assertAllEqual(ret[0][1][0][0], 8)
     self.assertAllEqual(ret[0][1][0][1], 4)
@@ -1044,7 +1044,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
 
     defined = function.defun(multi_device_fn)
     outputs = self.evaluate(defined())
-    self.assertEqual(len(defined._function_cache), 1)
+    self.assertLen(defined._function_cache, 1)
     self.assertIn(compat.as_bytes('CPU:0'), outputs[0])
     self.assertIn(compat.as_bytes('CPU:1'), outputs[1])
     self.assertIn(compat.as_bytes('CPU:2'), outputs[2])
@@ -1052,7 +1052,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     with ops.device('/cpu:3'):
       outputs = self.evaluate(defined())
     # All function definitions are agnostic to call site devices.
-    self.assertEqual(len(defined._function_cache), 1)
+    self.assertLen(defined._function_cache, 1)
     self.assertIn(compat.as_bytes('CPU:0'), outputs[0])
     self.assertIn(compat.as_bytes('CPU:1'), outputs[1])
     self.assertIn(compat.as_bytes('CPU:2'), outputs[2])
@@ -1060,7 +1060,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
 
     with ops.device('/cpu:0'):
       outputs = self.evaluate(defined())
-    self.assertEqual(len(defined._function_cache), 1)
+    self.assertLen(defined._function_cache, 1)
     self.assertIn(compat.as_bytes('CPU:0'), outputs[0])
     self.assertIn(compat.as_bytes('CPU:1'), outputs[1])
     self.assertIn(compat.as_bytes('CPU:2'), outputs[2])
@@ -1145,10 +1145,38 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
 
     defined = function.defun(func)
     defined(Foo())
-    self.assertEqual(len(defined._function_cache), 1)
+    self.assertLen(defined._function_cache, 1)
 
     defined(Foo())
-    self.assertEqual(len(defined._function_cache), 2)
+    self.assertLen(defined._function_cache, 2)
+
+  def testCacheTensorDtypeCollision(self):
+
+    def func(t):
+      return t + t
+
+    defined = function.defun(func)
+    t = constant_op.constant([[1.0]], dtype=dtypes.complex64)
+    defined(t)
+    self.assertLen(defined._function_cache, 1)
+
+    t = constant_op.constant([[1.0]], dtype=dtypes.complex128)
+    defined(t)
+    self.assertLen(defined._function_cache, 2)
+
+  def testCacheTensorShapeCollision(self):
+
+    def func(t):
+      return t + t
+
+    defined = function.defun(func)
+    t = constant_op.constant([[1.0]], dtype=dtypes.complex64)
+    defined(t)
+    self.assertLen(defined._function_cache, 1)
+
+    t = constant_op.constant([1.0], dtype=dtypes.complex64)
+    defined(t)
+    self.assertLen(defined._function_cache, 2)
 
   def testCacheTensorShapeDtypeCollision(self):
 
@@ -1158,11 +1186,11 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     defined = function.defun(func)
     t = constant_op.constant([[1.0]], dtype=dtypes.complex64)
     defined(t)
-    self.assertEqual(len(defined._function_cache), 1)
+    self.assertLen(defined._function_cache, 1)
 
     t = constant_op.constant([1.0], dtype=dtypes.complex128)
     defined(t)
-    self.assertEqual(len(defined._function_cache), 2)
+    self.assertLen(defined._function_cache, 2)
 
   def testCacheTensorUnknownShapesCollision(self):
 
@@ -1174,19 +1202,19 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
 
       p = array_ops.placeholder(dtype=dtypes.float32, shape=None)
       defined(p)
-      self.assertEqual(len(defined._function_cache), 1)
+      self.assertLen(defined._function_cache, 1)
 
       p = array_ops.placeholder(dtype=dtypes.float32, shape=[None])
       defined(p)
-      self.assertEqual(len(defined._function_cache), 2)
+      self.assertLen(defined._function_cache, 2)
 
       p = array_ops.placeholder(dtype=dtypes.float32, shape=[None, None])
       defined(p)
-      self.assertEqual(len(defined._function_cache), 3)
+      self.assertLen(defined._function_cache, 3)
 
       t = constant_op.constant(1.0, dtype=dtypes.float32)
       defined(t)
-      self.assertEqual(len(defined._function_cache), 4)
+      self.assertLen(defined._function_cache, 4)
 
   def testPythonFunctionWithDefaultArgs(self):
 
@@ -1211,25 +1239,26 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
 
     # This matches the previous call.
     defined(foo=1)
-    self.assertEqual(len(defined._function_cache), 2)
+    self.assertLen(defined._function_cache, 2)
 
     defined(1, 2, 3)
+    self.assertLen(defined._function_cache, 3)
     self.assertIn(('URRRu', (1, 2, 3)), cache_keys())
 
     # This matches the previous call.
     defined(1, bar=2, baz=3)
-    self.assertEqual(len(defined._function_cache), 3)
+    self.assertLen(defined._function_cache, 3)
 
     # This matches the previous call.
     defined(1, baz=3, bar=2)
-    self.assertEqual(len(defined._function_cache), 3)
+    self.assertLen(defined._function_cache, 3)
 
   def testFunctoolsPartialUnwrappedCorrectly(self):
 
     def full_function(a, b, c=3):
       return a, b, c
 
-    partial = functools.partial(full_function, 1, c=3)
+    partial = functools.partial(full_function, 1, c=4)
     a, b, c = partial(2)
 
     defined = function.defun(partial)
@@ -1248,12 +1277,12 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     defined = function.defun(foo, input_signature=signature)
     a = array_ops.ones([2])
     self.assertAllEqual(a, defined(a))
-    self.assertEqual(len(defined._function_cache), 1)
+    self.assertLen(defined._function_cache, 1)
     self.assertAllEqual(a, defined.get_concrete_function()(a))
     self.assertAllEqual(a, defined.get_concrete_function(a)(a))
     self.assertAllEqual(a, defined.get_concrete_function(
         tensor_spec.TensorSpec((2,), dtype=dtypes.float32))(a))
-    self.assertEqual(len(defined._function_cache), 1)
+    self.assertLen(defined._function_cache, 1)
 
     def bar(a):
       self.assertEqual(a._shape_tuple(), (2, None))
@@ -1263,13 +1292,13 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     defined = function.defun(bar, input_signature=signature)
     a = array_ops.ones([2, 1])
     out = defined(a)
-    self.assertEqual(len(defined._function_cache), 1)
+    self.assertLen(defined._function_cache, 1)
     self.assertAllEqual(out, a)
 
     # Changing the second dimension shouldn't create a new function.
     b = array_ops.ones([2, 3])
     out = defined(b)
-    self.assertEqual(len(defined._function_cache), 1)
+    self.assertLen(defined._function_cache, 1)
     self.assertAllEqual(out, b)
 
   def testNestedInputSignatures(self):
@@ -1286,7 +1315,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     a = array_ops.ones([2, 1])
     b = array_ops.ones([1])
     out = defined([a, a], b)
-    self.assertEqual(len(defined._function_cache), 1)
+    self.assertLen(defined._function_cache, 1)
     nest.assert_same_structure(out, [[a, a], b])
     self.assertAllEqual(out[0][0], a)
     self.assertAllEqual(out[0][1], a)
@@ -1297,7 +1326,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     b = array_ops.ones([2, 5])
     c = array_ops.ones([1])
     out = defined([a, b], c)
-    self.assertEqual(len(defined._function_cache), 1)
+    self.assertLen(defined._function_cache, 1)
     nest.assert_same_structure(out, [[a, b], c])
     self.assertAllEqual(out[0][0], a)
     self.assertAllEqual(out[0][1], b)
@@ -1371,6 +1400,25 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
       defined.get_concrete_function(
           tensor_spec.TensorSpec(shape=(3,), dtype=dtypes.float32))
 
+  def testInputsIncompatibleWithNestedSignatureRaisesError(self):
+
+    def foo(a, b):
+      return [a, b]
+
+    signature = [[tensor_spec.TensorSpec((1,), dtypes.float32)] * 2,
+                 [tensor_spec.TensorSpec((1,), dtypes.float32)] * 2]
+    defined = function.defun(foo, input_signature=signature)
+    a = array_ops.ones([1])
+
+    with self.assertRaisesRegexp(ValueError,
+                                 'Structure of Python function inputs.*'):
+      defined([a, a, a], [a])
+
+    with self.assertRaisesRegexp(ValueError,
+                                 'Structure of Python function inputs.*'):
+      defined([a], [a, a, a])
+    defined([a, a], [a, a])
+
   def testInputSignatureForFunctionWithNonTensorInputsNotAllowed(self):
 
     def foo(a, training=True):
@@ -1404,22 +1452,22 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     integer = constant_op.constant(2, dtypes.int64)
 
     out1, out2 = foo(flt, integer)
-    self.assertEqual(len(foo._function_cache), 1)
+    self.assertLen(foo._function_cache, 1)
     self.assertEqual(out1.numpy(), 1.0)
     self.assertEqual(out2.numpy(), 2)
 
     out1, out2 = foo(flt=flt, integer=integer)
-    self.assertEqual(len(foo._function_cache), 1)
+    self.assertLen(foo._function_cache, 1)
     self.assertEqual(out1.numpy(), 1.0)
     self.assertEqual(out2.numpy(), 2)
 
     out1, out2 = foo(integer=integer, flt=flt)
-    self.assertEqual(len(foo._function_cache), 1)
+    self.assertLen(foo._function_cache, 1)
     self.assertEqual(out1.numpy(), 1.0)
     self.assertEqual(out2.numpy(), 2)
 
     out1, out2 = foo(flt, integer=integer)
-    self.assertEqual(len(foo._function_cache), 1)
+    self.assertLen(foo._function_cache, 1)
     self.assertEqual(out1.numpy(), 1.0)
     self.assertEqual(out2.numpy(), 2)
 
@@ -1449,27 +1497,27 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     a = constant_op.constant(2.0)
     b = constant_op.constant([1.0, 2.0])
     one = defined(a, b)
-    self.assertEqual(len(defined._function_cache), 1)
+    self.assertLen(defined._function_cache, 1)
 
     two = defined(a=a, b=b)
-    self.assertEqual(len(defined._function_cache), 1)
+    self.assertLen(defined._function_cache, 1)
 
     three = defined(b=b, a=a)
-    self.assertEqual(len(defined._function_cache), 1)
+    self.assertLen(defined._function_cache, 1)
 
     four = defined(a, b=b)
-    self.assertEqual(len(defined._function_cache), 1)
+    self.assertLen(defined._function_cache, 1)
 
     # The next call corresponds to a new input signature, hence
     # we expect another function to be defined.
     five = defined(b, a)
-    self.assertEqual(len(defined._function_cache), 2)
+    self.assertLen(defined._function_cache, 2)
 
     six = defined(a=b, b=a)
-    self.assertEqual(len(defined._function_cache), 2)
+    self.assertLen(defined._function_cache, 2)
 
     seven = defined(b=a, a=b)
-    self.assertEqual(len(defined._function_cache), 2)
+    self.assertLen(defined._function_cache, 2)
 
     self.assertAllEqual(one, [1.0, 2.0])
     self.assertAllEqual(two, [1.0, 2.0])
@@ -1551,19 +1599,19 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
 
         graph = ops.get_default_graph()
         # pylint: disable=protected-access
-        self.assertEqual(len(graph._functions), 2)
+        self.assertLen(graph._functions, 2)
         functions = list(graph._functions.values())
         self.assertRegexpMatches(
             functions[0].definition.signature.name, '.*matmul.*')
         attrs = functions[0].definition.attr
-        self.assertEqual(len(attrs), 2)
+        self.assertLen(attrs, 2)
         self.assertEqual(attrs['experimental_1'].s, b'value1')
         self.assertEqual(attrs['experimental_2'].i, 2)
 
         self.assertRegexpMatches(
             functions[1].definition.signature.name, '.*add.*')
         attrs = functions[1].definition.attr
-        self.assertEqual(len(attrs), 2)
+        self.assertLen(attrs, 2)
         self.assertEqual(attrs['experimental_3'].b, True)
         self.assertEqual(attrs['experimental_4'].f, 1.0)
         # pylint: enable=protected-access
@@ -1609,7 +1657,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
 
         graph = ops.get_default_graph()
         # pylint: disable=protected-access
-        self.assertEqual(len(graph._functions), 6)
+        self.assertLen(graph._functions, 6)
         # two sets of functions, each of them are (inference, forward, backward)
         functions = list(graph._functions.values())
         captured_function_names = [
@@ -1648,7 +1696,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
         self.assertAllEqual(double.eval().reshape(-1), [2, 4, 6, 8])
         # Make sure the pre registered function is used, and no other function
         # is added.
-        self.assertEqual(len(graph._functions), 6)
+        self.assertLen(graph._functions, 6)
         functions = list(graph._functions.values())
         for i in range(len(functions)):
           self.assertEqual(captured_function_names[i],
@@ -1685,7 +1733,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
 
         graph = ops.get_default_graph()
         # pylint: disable=protected-access
-        self.assertEqual(len(graph._functions), 6)
+        self.assertLen(graph._functions, 6)
         # two sets of functions, each of them are (inference, forward, backward)
         functions = list(graph._functions.values())
         captured_function_names = [
@@ -1711,7 +1759,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
         self.assertAllEqual([[1, 2], [3, 4]], self.evaluate(composite_t))
         # Make sure the pre registered function is used, and no other function
         # is added.
-        self.assertEqual(len(graph._functions), 6)
+        self.assertLen(graph._functions, 6)
 
   def testRegisterFunctionWithInputSignature(self):
     def matmul(x, y):
@@ -1729,12 +1777,12 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
 
         graph = ops.get_default_graph()
         # pylint: disable=protected-access
-        self.assertEqual(len(graph._functions), 3)
+        self.assertLen(graph._functions, 3)
 
         # Test register function with cache, note inputs are ignored.
         function.register(defun_matmul)
         graph = ops.get_default_graph()
-        self.assertEqual(len(graph._functions), 3)
+        self.assertLen(graph._functions, 3)
 
   def testRegisterFunctionWithCache(self):
     def matmul(x, y):
@@ -1751,7 +1799,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
         graph = ops.get_default_graph()
         # Only one function is registered since the input param are in same type
         # pylint: disable=protected-access
-        self.assertEqual(len(graph._functions), 3)
+        self.assertLen(graph._functions, 3)
 
   def testCallingFunctionWithDifferentVariables(self):
 
@@ -1762,8 +1810,8 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
 
     v = resource_variable_ops.ResourceVariable(0.0)
     graph_function = foo.get_concrete_function(v)
-    self.assertEqual(len(graph_function.inputs), 1)
-    self.assertEqual(len(graph_function.captured_inputs), 0)
+    self.assertLen(graph_function.inputs, 1)
+    self.assertEmpty(graph_function.captured_inputs)
 
     self.assertEqual(float(graph_function(v)), 1.0)
     self.assertEqual(float(graph_function(v)), 2.0)
@@ -1805,7 +1853,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     config = config_pb2.ConfigProto(graph_options=graph_options)
 
     with context.graph_mode(), self.cached_session(
-        config=config, graph=ops.Graph(), use_gpu=True) as sess:
+        config=config, graph=ops.Graph(), use_gpu=True):
 
       @function.defun_with_attributes(
           attributes={
@@ -1852,18 +1900,18 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
       with ops.Graph().as_default():
         x = constant_op.constant(11)
         maybe_add(x, True)
-        self.assertEqual(len(maybe_add._function_cache), 1)
-        self.assertEqual(len(add._function_cache), 1)
+        self.assertLen(maybe_add._function_cache, 1)
+        self.assertLen(add._function_cache, 1)
 
         maybe_add(x, False)
-        self.assertEqual(len(maybe_add._function_cache), 2)
-        self.assertEqual(len(add._function_cache), 1)
+        self.assertLen(maybe_add._function_cache, 2)
+        self.assertLen(add._function_cache, 1)
 
       with ops.Graph().as_default():
         x = constant_op.constant(11)
         maybe_add(x, True)
-        self.assertEqual(len(maybe_add._function_cache), 3)
-        self.assertEqual(len(add._function_cache), 2)
+        self.assertLen(maybe_add._function_cache, 3)
+        self.assertLen(add._function_cache, 2)
 
   def testCacheKeyOverlappingShapes(self):
     @function.defun
@@ -2110,7 +2158,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     m = DefunnedMiniModel()
     m(array_ops.ones([1, 2]))
     weak_variables = weakref.WeakSet(m.variables)
-    self.assertEqual(2, len(weak_variables))
+    self.assertLen(weak_variables, 2)
     del m
     self.assertEqual([], list(weak_variables))
 
@@ -2165,13 +2213,12 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     def fn(x):
       return fn2(x)
 
-    try:
+    with self.assertRaises(errors.InvalidArgumentError) as cm:
       fn(2)
-      self.assertFail()
-    except errors.InvalidArgumentError as e:
-      self.assertIn('fn -> fn2', e.message)
-      self.assertIn('node assert_equal/Assert/Assert (defined at', e.message)
-      self.assertNotIn('fn3', e.message)
+    e = cm.exception
+    self.assertIn('fn -> fn2', e.message)
+    self.assertIn('node assert_equal/Assert/Assert (defined at', e.message)
+    self.assertNotIn('fn3', e.message)
 
   def testFunctionIsNotPinned(self):
     """Tests that functions aren't pinned to the CPU by the eager runtime."""
-- 
GitLab


From 20e1089c40941f585f0e4a1918b29272f078d5d0 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Thu, 7 Feb 2019 16:03:17 -0800
Subject: [PATCH 2105/2345] Add back tool scripts to pip packages.

PiperOrigin-RevId: 232963771
---
 tensorflow/python/tools/BUILD | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/tensorflow/python/tools/BUILD b/tensorflow/python/tools/BUILD
index b8ad31e88b..f3db3b5952 100644
--- a/tensorflow/python/tools/BUILD
+++ b/tensorflow/python/tools/BUILD
@@ -13,21 +13,26 @@ load("//tensorflow:tensorflow.bzl", "py_binary")
 # Transitive dependencies of this target will be included in the pip package.
 py_library(
     name = "tools_pip",
+    data = [
+        ":freeze_graph",
+        ":import_pb_to_tensorboard",
+        ":inspect_checkpoint",
+        ":optimize_for_inference",
+        ":print_selective_registration_header",
+        ":saved_model_cli",
+        ":strip_unused",
+        # Include the TF upgrade script to users can run it directly after install TF
+        "//tensorflow/tools/compatibility:tf_upgrade_v2",
+    ],
     deps = [
-        ":freeze_graph_lib",
-        ":import_pb_to_tensorboard_lib",
-        ":inspect_checkpoint_lib",
-        ":optimize_for_inference_lib",
-        ":print_selective_registration_header_lib",
-        ":saved_model_cli_lib",
         ":saved_model_utils",
-        ":strip_unused_lib",
         # The following py_library are needed because
         # py_binary may not depend on them when --define=no_tensorflow_py_deps=true
         # is specified. See https://github.com/tensorflow/tensorflow/issues/22390
+        ":freeze_graph_lib",
+        ":optimize_for_inference_lib",
         ":selective_registration_header_lib",
-        # Include the TF upgrade script to users can run it directly after install TF
-        "//tensorflow/tools/compatibility:tf_upgrade_v2_lib",
+        ":strip_unused_lib",
     ],
 )
 
-- 
GitLab


From c935938d55014ab73d0657255ffcc1d9bc00dc98 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 7 Feb 2019 16:12:31 -0800
Subject: [PATCH 2106/2345] More strict error checks in tpu feature columns.
 Each embedding var name i.e. the key in var_def_dict uniquely must map to the
 same var scope and var name tuple, for both the shared embedding case and
 normal embedding case.

PiperOrigin-RevId: 232965482
---
 .../contrib/tpu/python/tpu/feature_column.py   | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/feature_column.py b/tensorflow/contrib/tpu/python/tpu/feature_column.py
index 68bcdb5026..e80d0de705 100644
--- a/tensorflow/contrib/tpu/python/tpu/feature_column.py
+++ b/tensorflow/contrib/tpu/python/tpu/feature_column.py
@@ -408,19 +408,25 @@ def _record_variable_scope_and_name(embedding_var_name,
 
   var_def_dict = collection[0]
 
-  captured_scope = None
+  captured_scope = variable_scope.get_variable_scope()
+  captured_scope_name = captured_scope.name
 
-  if is_shared_embedding and (embedding_var_name in var_def_dict):
+  if embedding_var_name in var_def_dict:
+    if (var_def_dict[embedding_var_name][0] != captured_scope_name
+        and not is_shared_embedding):
+      raise ValueError(
+          'For embedding var name {}, the variable scope name is different, '
+          'got {}; expected {}'.format(embedding_var_name,
+                                       captured_scope_name,
+                                       var_def_dict[embedding_var_name][0]))
     if var_def_dict[embedding_var_name][1] != embedding_var_name_in_fc:
       raise ValueError(
-          'For embedding var name {}, the shared embedding name is different, '
+          'For embedding var name {}, the embedding name is different, '
           'got {}; expected {}'.format(embedding_var_name,
                                        embedding_var_name_in_fc,
                                        var_def_dict[embedding_var_name][1]))
   else:
-    # scope contains var_scope_name.
-    captured_scope = variable_scope.get_variable_scope()
-    var_def_dict[embedding_var_name] = (captured_scope.name,
+    var_def_dict[embedding_var_name] = (captured_scope_name,
                                         embedding_var_name_in_fc)
 
 
-- 
GitLab


From 38a7273fb383bbf0e2992f9d8bc010c1c3c291ca Mon Sep 17 00:00:00 2001
From: Noah Eisen <ncteisen@google.com>
Date: Thu, 7 Feb 2019 16:13:03 -0800
Subject: [PATCH 2107/2345] Add retry support for MasterService.

PiperOrigin-RevId: 232965574
---
 tensorflow/core/distributed_runtime/rpc/BUILD |   3 +-
 .../rpc/grpc_remote_master.cc                 | 132 +++++++++++-------
 .../core/distributed_runtime/rpc/grpc_util.cc |  52 +++++++
 .../core/distributed_runtime/rpc/grpc_util.h  |   9 ++
 4 files changed, 146 insertions(+), 50 deletions(-)

diff --git a/tensorflow/core/distributed_runtime/rpc/BUILD b/tensorflow/core/distributed_runtime/rpc/BUILD
index 3f261119b6..dd22e74ac5 100644
--- a/tensorflow/core/distributed_runtime/rpc/BUILD
+++ b/tensorflow/core/distributed_runtime/rpc/BUILD
@@ -46,6 +46,7 @@ cc_library(
         "//tensorflow/core:lib",
         # Required to be able to overload TensorResponse parsing.
         "//tensorflow/core/distributed_runtime:tensor_coding",
+        "//tensorflow/core:lib_internal",
     ],
 )
 
@@ -166,7 +167,6 @@ cc_library(
     deps = [
         ":grpc_util",
         "//tensorflow/core:lib",
-        "@com_google_absl//absl/time",
     ],
 )
 
@@ -196,7 +196,6 @@ tf_cuda_library(
         "//tensorflow/core/distributed_runtime:worker_env",
         "//tensorflow/core/distributed_runtime:worker_session",
         "@com_google_absl//absl/container:flat_hash_map",
-        "@com_google_absl//absl/time",
     ],
 )
 
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_remote_master.cc b/tensorflow/core/distributed_runtime/rpc/grpc_remote_master.cc
index b832a2115c..a84559098a 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_remote_master.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_remote_master.cc
@@ -24,6 +24,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/strings/strcat.h"
+#include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/tracing.h"
 #include "tensorflow/core/protobuf/master.pb.h"
 
@@ -43,104 +44,139 @@ class GrpcRemoteMaster : public MasterInterface {
   Status CreateSession(CallOptions* call_options,
                        const CreateSessionRequest* request,
                        CreateSessionResponse* response) override {
-    ::grpc::ClientContext ctx;
-    return Call(&ctx, call_options, request, response,
-                &MasterServiceStub::CreateSession);
+    return CallWithRetry(call_options, request, response,
+                         &MasterServiceStub::CreateSession);
   }
 
   Status ExtendSession(CallOptions* call_options,
                        const ExtendSessionRequest* request,
                        ExtendSessionResponse* response) override {
-    ::grpc::ClientContext ctx;
-    return Call(&ctx, call_options, request, response,
-                &MasterServiceStub::ExtendSession);
+    return CallWithRetry(call_options, request, response,
+                         &MasterServiceStub::ExtendSession);
   }
 
   Status PartialRunSetup(CallOptions* call_options,
                          const PartialRunSetupRequest* request,
                          PartialRunSetupResponse* response) override {
-    ::grpc::ClientContext ctx;
-    return Call(&ctx, call_options, request, response,
-                &MasterServiceStub::PartialRunSetup);
+    return CallWithRetry(call_options, request, response,
+                         &MasterServiceStub::PartialRunSetup);
   }
 
   Status RunStep(CallOptions* call_options, RunStepRequestWrapper* request,
                  MutableRunStepResponseWrapper* response) override {
-    ::grpc::ClientContext ctx;
-    auto trace = TraceRpc("RunStep/Client", &ctx);
-    return Call(&ctx, call_options, &request->ToProto(),
-                get_proto_from_wrapper(response), &MasterServiceStub::RunStep);
+    return CallWithRetry(call_options, &request->ToProto(),
+                         get_proto_from_wrapper(response),
+                         &MasterServiceStub::RunStep, "RunStep/Client");
   }
 
   Status CloseSession(CallOptions* call_options,
                       const CloseSessionRequest* request,
                       CloseSessionResponse* response) override {
-    ::grpc::ClientContext ctx;
-    return Call(&ctx, call_options, request, response,
-                &MasterServiceStub::CloseSession);
+    return CallWithRetry(call_options, request, response,
+                         &MasterServiceStub::CloseSession);
   }
 
   Status ListDevices(CallOptions* call_options,
                      const ListDevicesRequest* request,
                      ListDevicesResponse* response) override {
-    ::grpc::ClientContext ctx;
-    return Call(&ctx, call_options, request, response,
-                &MasterServiceStub::ListDevices);
+    return CallWithRetry(call_options, request, response,
+                         &MasterServiceStub::ListDevices);
   }
 
   Status Reset(CallOptions* call_options, const ResetRequest* request,
                ResetResponse* response) override {
-    ::grpc::ClientContext ctx;
-    return Call(&ctx, call_options, request, response,
-                &MasterServiceStub::Reset);
+    return CallWithRetry(call_options, request, response,
+                         &MasterServiceStub::Reset);
   }
 
   Status MakeCallable(CallOptions* call_options,
                       const MakeCallableRequest* request,
                       MakeCallableResponse* response) override {
-    ::grpc::ClientContext ctx;
-    return Call(&ctx, call_options, request, response,
-                &MasterServiceStub::MakeCallable);
+    return CallWithRetry(call_options, request, response,
+                         &MasterServiceStub::MakeCallable);
   }
   Status RunCallable(CallOptions* call_options,
                      const RunCallableRequest* request,
                      RunCallableResponse* response) override {
-    ::grpc::ClientContext ctx;
-    return Call(&ctx, call_options, request, response,
-                &MasterServiceStub::RunCallable);
+    return CallWithRetry(call_options, request, response,
+                         &MasterServiceStub::RunCallable);
   }
   Status ReleaseCallable(CallOptions* call_options,
                          const ReleaseCallableRequest* request,
                          ReleaseCallableResponse* response) override {
-    ::grpc::ClientContext ctx;
-    return Call(&ctx, call_options, request, response,
-                &MasterServiceStub::ReleaseCallable);
+    return CallWithRetry(call_options, request, response,
+                         &MasterServiceStub::ReleaseCallable);
   }
 
  private:
   // Start tracing, attaching a unique ID to both the trace and the RPC.
-  tracing::ScopedActivity TraceRpc(StringPiece name,
-                                   ::grpc::ClientContext* ctx) {
+  tracing::ScopedActivity* NewTraceRpc(StringPiece name,
+                                       ::grpc::ClientContext* ctx) {
     string trace_id = strings::StrCat(tracing::GetUniqueArg());
     ctx->AddMetadata(GrpcIdKey(), trace_id);
-    return tracing::ScopedActivity(name, trace_id);
-  }
-
-  void SetDeadline(::grpc::ClientContext* ctx, int64 time_in_ms) {
-    if (time_in_ms > 0) {
-      ctx->set_deadline(gpr_time_from_millis(time_in_ms, GPR_TIMESPAN));
-    }
+    return new tracing::ScopedActivity(name, trace_id);
   }
 
   template <typename Request, typename Response>
-  Status Call(::grpc::ClientContext* ctx, CallOptions* call_options,
-              const Request* request, Response* response,
-              ::grpc::Status (MasterServiceStub::*pfunc)(::grpc::ClientContext*,
-                                                         const Request&,
-                                                         Response*)) {
-    ctx->set_fail_fast(false);
-    SetDeadline(ctx, call_options->GetTimeout());
-    return FromGrpcStatus((stub_.get()->*pfunc)(ctx, *request, response));
+  Status CallWithRetry(CallOptions* call_options, const Request* request,
+                       Response* response,
+                       ::grpc::Status (MasterServiceStub::*pfunc)(
+                           ::grpc::ClientContext*, const Request&, Response*),
+                       string trace_string = {}) {
+    int64 timeout_in_ms = call_options->GetTimeout();
+    int64 expired_time_micros = Env::Default()->NowMicros();
+    if (timeout_in_ms > 0) {
+      expired_time_micros += (timeout_in_ms / 1000.);
+    }
+    Status s;
+    for (int num_retries = 0;; ++num_retries) {
+      ::grpc::ClientContext ctx;
+      std::unique_ptr<tracing::ScopedActivity> trace;
+      if (!trace_string.empty()) {
+        trace.reset(NewTraceRpc(trace_string, &ctx));
+      }
+      ctx.set_fail_fast(false);
+      if (timeout_in_ms > 0) {
+        // We do not modify the timeout here to match legacy behavior. However,
+        // this could violate the contract of tensorflow::Session. If we retry
+        // an RPC just before the deadline is exceeded, we will still set the
+        // timeout to the original value. This leads to the overall timeout
+        // being double what was expected.
+        // TODO(b/117162170): investigate fixing this behavior for legacy and
+        // gRPC RPC layers.
+        ctx.set_deadline(gpr_time_from_millis(timeout_in_ms, GPR_TIMESPAN));
+      }
+      s = FromGrpcStatus((stub_.get()->*pfunc)(&ctx, *request, response));
+      if (!errors::IsUnavailable(s)) {
+        return s;
+      }
+      // TODO(b/117162170): we may want to make this configurable.
+      constexpr int kMaxRetries = 10;
+      LOG(WARNING) << "RPC failed with status = \"" << s
+                   << "\" and grpc_error_string = \""
+                   << ctx.debug_error_string() << "\", maybe retrying the RPC";
+      if (num_retries >= kMaxRetries) {
+        LOG(WARNING) << "Too many retries, returning last status: " << s;
+        return s;
+      }
+      const int64 now_micros = Env::Default()->NowMicros();
+      const int64 deadline_with_backoff_micros =
+          now_micros + ComputeBackoffMicroseconds(num_retries);
+      // Wait for a short period of time before retrying the RPC.  If our
+      // backoff would put us past the RPC deadline, we truncate it to ensure
+      // our RPC starts before the deadline.
+      const auto backoff_until =
+          (timeout_in_ms <= 0 ||
+           expired_time_micros > deadline_with_backoff_micros)
+              ? deadline_with_backoff_micros
+              : expired_time_micros;
+      Env::Default()->SleepForMicroseconds(backoff_until - now_micros);
+      if (Env::Default()->NowMicros() > expired_time_micros &&
+          timeout_in_ms > 0) {
+        // If timeout_in_ms is set, exit the retry loop on timeout.
+        return errors::DeadlineExceeded(ctx.debug_error_string());
+      }
+    }
   }
 
   std::unique_ptr<MasterServiceStub> stub_;
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_util.cc b/tensorflow/core/distributed_runtime/rpc/grpc_util.cc
index e211c33732..471e2c16b3 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_util.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_util.cc
@@ -15,9 +15,61 @@ limitations under the License.
 
 #include "tensorflow/core/distributed_runtime/rpc/grpc_util.h"
 #include "tensorflow/core/distributed_runtime/tensor_coding.h"
+#include "tensorflow/core/lib/random/random.h"
 
 namespace tensorflow {
 
+namespace {
+
+double GenerateUniformRandomNumber() {
+  return random::New64() * (1.0 / std::numeric_limits<uint64>::max());
+}
+
+double GenerateUniformRandomNumberBetween(double a, double b) {
+  if (a == b) return a;
+  DCHECK_LT(a, b);
+  return a + GenerateUniformRandomNumber() * (b - a);
+}
+
+}  // namespace
+
+int64 ComputeBackoffMicroseconds(int current_retry_attempt, int64 min_delay,
+                                 int64 max_delay) {
+  DCHECK_GE(current_retry_attempt, 0);
+
+  // This function with the constants below is calculating:
+  //
+  // (0.4 * min_delay) + (random[0.6,1.0] * min_delay * 1.3^retries)
+  //
+  // Note that there is an extra truncation that occurs and is documented in
+  // comments below.
+  constexpr double kBackoffBase = 1.3;
+  constexpr double kBackoffRandMult = 0.4;
+
+  // This first term does not vary with current_retry_attempt or a random
+  // number. It exists to ensure the final term is >= min_delay
+  const double first_term = kBackoffRandMult * min_delay;
+
+  // This is calculating min_delay * 1.3^retries
+  double uncapped_second_term = min_delay;
+  while (current_retry_attempt > 0 &&
+         uncapped_second_term < max_delay - first_term) {
+    current_retry_attempt--;
+    uncapped_second_term *= kBackoffBase;
+  }
+  // Note that first_term + uncapped_second_term can exceed max_delay here
+  // because of the final multiply by kBackoffBase.  We fix that problem with
+  // the min() below.
+  double second_term = std::min(uncapped_second_term, max_delay - first_term);
+
+  // This supplies the random jitter to ensure that retried don't cause a
+  // thundering herd problem.
+  second_term *=
+      GenerateUniformRandomNumberBetween(1.0 - kBackoffRandMult, 1.0);
+
+  return std::max(static_cast<int64>(first_term + second_term), min_delay);
+}
+
 ::grpc::Status GrpcMaybeUnparseProto(const protobuf::Message& src,
                                      grpc::ByteBuffer* dst) {
   bool own_buffer;
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_util.h b/tensorflow/core/distributed_runtime/rpc/grpc_util.h
index 45259aa2ec..976f3e6452 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_util.h
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_util.h
@@ -29,6 +29,15 @@ limitations under the License.
 
 namespace tensorflow {
 
+// Given the total number of RPC retries attempted, return a randomized
+// amount of time to delay before retrying the request.
+//
+// The average computed backoff increases with the number of RPCs attempted.
+// See implementation for details on the calculations.
+int64 ComputeBackoffMicroseconds(int current_retry_attempt,
+                                 int64 min_delay = 1000,
+                                 int64 max_delay = 10000000);
+
 // Thin wrapper around ::grpc::ProtoBufferReader to give TensorResponse an
 // efficient byte reader from which to decode a RecvTensorResponse.
 class GrpcByteSource : public TensorResponse::Source {
-- 
GitLab


From b63e41693e6cc8b4a7f8b82f128cc309c2a88e36 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 7 Feb 2019 16:13:16 -0800
Subject: [PATCH 2108/2345] [XLA] Use == for layout comparison in shape
 comparison.

PiperOrigin-RevId: 232965633
---
 tensorflow/compiler/xla/shape.cc | 20 +-------------------
 1 file changed, 1 insertion(+), 19 deletions(-)

diff --git a/tensorflow/compiler/xla/shape.cc b/tensorflow/compiler/xla/shape.cc
index a36d3547a0..b88f489450 100644
--- a/tensorflow/compiler/xla/shape.cc
+++ b/tensorflow/compiler/xla/shape.cc
@@ -143,28 +143,10 @@ bool Shape::Equal::operator()(const Shape& lhs, const Shape& rhs) {
       return false;
     }
     if (LayoutUtil::IsDenseArray(lhs)) {
-      if (!absl::c_equal(LayoutUtil::MinorToMajor(lhs),
-                         LayoutUtil::MinorToMajor(rhs))) {
+      if (lhs.layout() != rhs.layout()) {
         VLOG(3) << "CompareShapes: lhs layout != rhs layout";
         return false;
       }
-
-      const auto& lhs_tiles = lhs.layout().tiles();
-      const auto& rhs_tiles = rhs.layout().tiles();
-      if (lhs_tiles.size() != rhs_tiles.size()) {
-        return false;
-      }
-      for (int64 i = 0; i < lhs_tiles.size(); i++) {
-        if (!absl::c_equal(lhs_tiles[i].dimensions(),
-                           rhs_tiles[i].dimensions())) {
-          return false;
-        }
-      }
-
-      if (lhs.layout().element_size_in_bits() !=
-          rhs.layout().element_size_in_bits()) {
-        return false;
-      }
     }
   }
 
-- 
GitLab


From 49853d70188fececa52ca9deb068912602ba9063 Mon Sep 17 00:00:00 2001
From: nammbash <niroopshankar.ammbashankar@intel.com>
Date: Thu, 7 Feb 2019 16:34:31 -0800
Subject: [PATCH 2109/2345] adding new ops api to the new shared raw ops file
 in tensorflow/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt

---
 tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt
index 28f26fe2de..2a63036ffa 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt
@@ -2644,9 +2644,17 @@ tf_module {
     name: "RequantizationRange"
     argspec: "args=[\'input\', \'input_min\', \'input_max\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "RequantizationRangePerChannel"
+    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'clip_value_max\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "Requantize"
     argspec: "args=[\'input\', \'input_min\', \'input_max\', \'requested_output_min\', \'requested_output_max\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }  
+  member_method {
+    name: "RequantizePerChannel"
+    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'requested_output_min\', \'requested_output_max\', \'out_type\'], varargs=None, keywords=None, defaults=None"
   }
   member_method {
     name: "Reshape"
-- 
GitLab


From 543b9f88b5a118ebc651ed4e9cecc93c8f4b97a0 Mon Sep 17 00:00:00 2001
From: Craig Citro <craigcitro@google.com>
Date: Thu, 7 Feb 2019 16:14:53 -0800
Subject: [PATCH 2110/2345] Be more strict about values that skip the GCE
 credential checks.

In commit a252fe83f4dc29c0614983046825ec75a3529cb9, I added support for a
`$NO_GCE_CHECK` environment variable whose presence would skip attempting to
use GCE credentials. Part of the motivation was symmetry with other Google
auth libraries, such as the Java client and Python's oauth2client.

However, both of those libraries are more precise about the values: they only
skip the GCE check if `$NO_GCE_CHECK` is set to `true` (case insensitive in
Java, and specifically `True` in Python).

This change just refines the checking to do a case-insensitive check for
`true`, bringing its behavior into line with the existing implementations. (I
also updated the test.)

PiperOrigin-RevId: 232965965
---
 tensorflow/core/platform/cloud/BUILD                   |  1 +
 tensorflow/core/platform/cloud/google_auth_provider.cc | 10 +++++++---
 .../core/platform/cloud/google_auth_provider_test.cc   |  7 ++++++-
 3 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/platform/cloud/BUILD b/tensorflow/core/platform/cloud/BUILD
index 647a797b82..40a808b661 100644
--- a/tensorflow/core/platform/cloud/BUILD
+++ b/tensorflow/core/platform/cloud/BUILD
@@ -151,6 +151,7 @@ cc_library(
         ":retrying_utils",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "@com_google_absl//absl/strings",
         "@jsoncpp_git//:jsoncpp",
     ],
 )
diff --git a/tensorflow/core/platform/cloud/google_auth_provider.cc b/tensorflow/core/platform/cloud/google_auth_provider.cc
index c27fb5e105..e91a9f8975 100644
--- a/tensorflow/core/platform/cloud/google_auth_provider.cc
+++ b/tensorflow/core/platform/cloud/google_auth_provider.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #endif
 #include <fstream>
 #include <utility>
+#include "absl/strings/match.h"
 #include "include/json/json.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/io/path.h"
@@ -44,8 +45,9 @@ constexpr char kGoogleAuthTokenForTesting[] = "GOOGLE_AUTH_TOKEN_FOR_TESTING";
 // The environment variable which can override '~/.config/gcloud' if set.
 constexpr char kCloudSdkConfig[] = "CLOUDSDK_CONFIG";
 
-// The environment variable used to skip attempting to fetch GCE credentials
-// if set.
+// The environment variable used to skip attempting to fetch GCE credentials:
+// setting this to 'true' (case insensitive) will skip attempting to contact
+// the GCE metadata service.
 constexpr char kNoGceCheck[] = "NO_GCE_CHECK";
 
 // The default path to the gcloud config folder, relative to the home folder.
@@ -155,7 +157,9 @@ Status GoogleAuthProvider::GetToken(string* t) {
     return Status::OK();
   }
 
-  bool skip_gce_check = std::getenv(kNoGceCheck) != nullptr;
+  char* no_gce_check_var = std::getenv(kNoGceCheck);
+  bool skip_gce_check = no_gce_check_var != nullptr &&
+                        absl::EqualsIgnoreCase(no_gce_check_var, "true");
   Status token_from_gce_status;
   if (skip_gce_check) {
     token_from_gce_status =
diff --git a/tensorflow/core/platform/cloud/google_auth_provider_test.cc b/tensorflow/core/platform/cloud/google_auth_provider_test.cc
index 0a6b108718..d2db59200a 100644
--- a/tensorflow/core/platform/cloud/google_auth_provider_test.cc
+++ b/tensorflow/core/platform/cloud/google_auth_provider_test.cc
@@ -240,7 +240,7 @@ TEST_F(GoogleAuthProviderTest, NothingAvailable) {
 }
 
 TEST_F(GoogleAuthProviderTest, NoGceCheckEnvironmentVariable) {
-  setenv("NO_GCE_CHECK", "", 1);
+  setenv("NO_GCE_CHECK", "True", 1);
   auto oauth_client = new FakeOAuthClient;
 
   FakeEnv env;
@@ -253,6 +253,11 @@ TEST_F(GoogleAuthProviderTest, NoGceCheckEnvironmentVariable) {
   TF_EXPECT_OK(provider.GetToken(&token));
   EXPECT_EQ("", token);
 
+  // We confirm that our env var is case insensitive.
+  setenv("NO_GCE_CHECK", "true", 1);
+  TF_EXPECT_OK(provider.GetToken(&token));
+  EXPECT_EQ("", token);
+
   // We also want to confirm that our empty token has a short expiration set: we
   // now set a testing token, and confirm that it's returned instead of our
   // empty token.
-- 
GitLab


From cb5f4b3f553bbae29e45561476ce2fa54654bd9a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 7 Feb 2019 16:20:32 -0800
Subject: [PATCH 2111/2345] Add an optional prefix param to ScopedStepContainer
 to avoid possible name clashes

PiperOrigin-RevId: 232966938
---
 tensorflow/core/framework/resource_mgr.h | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tensorflow/core/framework/resource_mgr.h b/tensorflow/core/framework/resource_mgr.h
index 9c381e7d6b..18a21d744b 100644
--- a/tensorflow/core/framework/resource_mgr.h
+++ b/tensorflow/core/framework/resource_mgr.h
@@ -89,9 +89,17 @@ class ScopedStepContainer {
   // step_id: the unique ID of this step. Doesn't have to be sequential, just
   // has to be unique.
   // cleanup: callback to delete a container of this name.
+  // prefix: optional string prefix to disambiguate step containers.
   ScopedStepContainer(const int64 step_id,
                       std::function<void(const string&)> cleanup)
       : name_(strings::StrCat("__per_step_", step_id)), cleanup_(cleanup) {}
+
+  ScopedStepContainer(const int64 step_id,
+                      std::function<void(const string&)> cleanup,
+                      const string& prefix)
+      : name_(strings::StrCat("__", prefix, "_per_step_", step_id)),
+        cleanup_(cleanup) {}
+
   ~ScopedStepContainer() { cleanup_(name_); }
 
   const string& name() const { return name_; }
-- 
GitLab


From 3984ce80d24aeed6a446131bef03e7ca160f7651 Mon Sep 17 00:00:00 2001
From: Yuan Lin <yulin@nvidia.com>
Date: Thu, 7 Feb 2019 16:40:32 -0800
Subject: [PATCH 2112/2345] [XLA] Interactive_graphviz: improve comments and
 skip empty fields when splitting the command

---
 .../compiler/xla/service/hlo_graph_dumper.h   |  2 +-
 .../xla/tools/interactive_graphviz.cc         | 23 ++++++++++---------
 2 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.h b/tensorflow/compiler/xla/service/hlo_graph_dumper.h
index 1b90a36706..9432a2a193 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.h
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.h
@@ -65,7 +65,7 @@ string DumpGraph(const HloComputation& computation, const string& label,
 // before it's omitted from the graph.
 //
 // The optional boundary specifies a set of boundary nodes, beyond which nodes
-// will be omitted even if they  are within the radius.
+// will be omitted even if they are within the radius.
 string DumpNeighborhoodAround(
     const HloInstruction& node, int radius,
     const absl::flat_hash_set<const HloInstruction*>& boundary = {},
diff --git a/tensorflow/compiler/xla/tools/interactive_graphviz.cc b/tensorflow/compiler/xla/tools/interactive_graphviz.cc
index 0b86ac6088..faf0c46757 100644
--- a/tensorflow/compiler/xla/tools/interactive_graphviz.cc
+++ b/tensorflow/compiler/xla/tools/interactive_graphviz.cc
@@ -478,29 +478,30 @@ void DoPlotCommand(const Options& opts, const HloModule& module,
       return;
     }
 
-    int bound_index;
-    // get the width if present
-    if (!absl::SimpleAtoi(tokens[1], &graph_width)) {
-      bound_index = 1;
-      graph_width = kDefaultWidth;
+    int bound_index = 1;
+    // Get the <width> if present.
+    if (absl::SimpleAtoi(tokens[bound_index], &graph_width)) {
+      bound_index++;
     } else {
-      bound_index = 2;
+      // <width> not found, need to reset graph_width.
+      graph_width = kDefaultWidth;
     }
-    // get the '/'
+    // Get the '/'.
     if (bound_index < tokens.size()) {
+      // This token must be a '/'.
       if (tokens[bound_index] != "/") {
-        std::cerr << "Expect a /, but get a '" << tokens[1] << "'."
+        std::cerr << "Expect a /, but get a '" << tokens[bound_index] << "'."
                   << std::endl;
         return;
       }
       bound_index++;
     }
-    // get the boundary nodes
+    // Get the boundary nodes.
     while (bound_index < tokens.size()) {
       string bnode_name = tokens[bound_index];
       const HloInstruction* binstr = FindInstruction(module, bnode_name);
       if (!binstr) {
-        std::cerr << "Couldn't find HloInstruction named " << node_name << "."
+        std::cerr << "Couldn't find HloInstruction named " << bnode_name << "."
                   << std::endl;
         return;
       }
@@ -538,7 +539,7 @@ void InteractiveDumpGraphs(const Options& opts, const HloModule& module) {
                 << std::endl;
       continue;
     }
-    std::vector<string> tokens = absl::StrSplit(line, ' ');
+    std::vector<string> tokens = absl::StrSplit(line, ' ', absl::SkipEmpty());
     if (tokens[0] == "quit" || tokens[0] == "exit") {
       break;
     } else if (tokens[0] == "help") {
-- 
GitLab


From 308c957525acdc68bb43c0e92404fbf34cb8cbb8 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Thu, 7 Feb 2019 16:36:35 -0800
Subject: [PATCH 2113/2345] [XLA:Python] Add a version() method to
 xla_client.py that reports the library version.

PiperOrigin-RevId: 232969741
---
 tensorflow/compiler/xla/python/xla_client.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tensorflow/compiler/xla/python/xla_client.py b/tensorflow/compiler/xla/python/xla_client.py
index 54fd7a4f38..49f1f33c11 100644
--- a/tensorflow/compiler/xla/python/xla_client.py
+++ b/tensorflow/compiler/xla/python/xla_client.py
@@ -41,6 +41,18 @@ from tensorflow.compiler.xla.service import hlo_pb2
 # pylint: disable=invalid-name
 
 
+# Version of the XLA Python client.
+#
+# JAX packages the XLA python plugin as a binary pip module (jaxlib) that is
+# packaged separately from the Python code that consumes it (jax).
+#
+# We occasionally need to make backwards-incompatible changes to jaxlib, in
+# which case we need to be able to detect when incompatible versions are
+# installed.
+def version():
+  return (0, 1, 7)
+
+
 _OP_METADATA_FIELDS = [
     'op_type',
     'op_name',
-- 
GitLab


From 2bc20aa8db09e8c662a1579b236be80df264fba2 Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Thu, 7 Feb 2019 16:49:26 -0800
Subject: [PATCH 2114/2345] In case we delete the iterator before the
 GeneratorDataset is initialized, we still ran the finalize function. This
 causes errors because state_ hasn't been computed.

PiperOrigin-RevId: 232971830
---
 tensorflow/core/kernels/data/generator_dataset_op.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/data/generator_dataset_op.cc b/tensorflow/core/kernels/data/generator_dataset_op.cc
index 5dff2be39d..3469743af6 100644
--- a/tensorflow/core/kernels/data/generator_dataset_op.cc
+++ b/tensorflow/core/kernels/data/generator_dataset_op.cc
@@ -71,7 +71,7 @@ class GeneratorDatasetOp::Dataset : public DatasetBase {
         : DatasetIterator<Dataset>(params) {}
 
     ~Iterator() override {
-      if (!finalized_) {
+      if (!finalized_ && initialized_) {
         std::vector<Tensor> ignored;
         Status s =
             instantiated_finalize_func_->RunInstantiated(state_, &ignored);
-- 
GitLab


From 51d9b3103862dcccb0fe01f8fa66309663424db4 Mon Sep 17 00:00:00 2001
From: Noah Eisen <ncteisen@google.com>
Date: Thu, 7 Feb 2019 16:52:06 -0800
Subject: [PATCH 2115/2345] Upgrade gRPC version used in OSS Tensorflow

PiperOrigin-RevId: 232972228
---
 tensorflow/contrib/cmake/external/grpc.cmake | 2 +-
 tensorflow/workspace.bzl                     | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/cmake/external/grpc.cmake b/tensorflow/contrib/cmake/external/grpc.cmake
index e570c09ecb..dd8cca809e 100644
--- a/tensorflow/contrib/cmake/external/grpc.cmake
+++ b/tensorflow/contrib/cmake/external/grpc.cmake
@@ -17,7 +17,7 @@ include (ExternalProject)
 set(GRPC_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/include)
 set(GRPC_URL https://github.com/grpc/grpc.git)
 set(GRPC_BUILD ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc)
-set(GRPC_TAG 69b6c047bc767b4d80e7af4d00ccb7c45b683dae)
+set(GRPC_TAG 1bd32fb4d582e3d182786b10d63dd048ba11b678)
 
 if(WIN32)
   # We use unsecure gRPC because boringssl does not build on windows
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index e97041d14d..65ee35f9c1 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -478,12 +478,12 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     # WARNING: make sure ncteisen@ and vpai@ are cc-ed on any CL to change the below rule
     tf_http_archive(
         name = "grpc",
-        sha256 = "1aa84387232dda273ea8fdfe722622084f72c16f7b84bfc519ac7759b71cdc91",
-        strip_prefix = "grpc-69b6c047bc767b4d80e7af4d00ccb7c45b683dae",
+        sha256 = "1b7178e42f6f766cf56425fab4f1ba5ddf668b209a997ba62ffca232dc43688c",
+        strip_prefix = "grpc-1bd32fb4d582e3d182786b10d63dd048ba11b678",
         system_build_file = clean_dep("//third_party/systemlibs:grpc.BUILD"),
         urls = [
-            "https://mirror.bazel.build/github.com/grpc/grpc/archive/69b6c047bc767b4d80e7af4d00ccb7c45b683dae.tar.gz",
-            "https://github.com/grpc/grpc/archive/69b6c047bc767b4d80e7af4d00ccb7c45b683dae.tar.gz",
+            "https://mirror.bazel.build/github.com/grpc/grpc/archive/1bd32fb4d582e3d182786b10d63dd048ba11b678.tar.gz",
+            "https://github.com/grpc/grpc/archive/1bd32fb4d582e3d182786b10d63dd048ba11b678.tar.gz",
         ],
     )
 
-- 
GitLab


From d1823e2e966e96ee4ea7baa202ad9f292ac7427b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 7 Feb 2019 17:07:32 -0800
Subject: [PATCH 2116/2345] Introduce TF_NUM_INTEROP_THREADS and
 TF_NUM_INTRAOP_THREADS to allow explicit configuration.

PiperOrigin-RevId: 232974975
---
 .../core/common_runtime/local_device.cc       | 20 ++++++++----
 .../core/common_runtime/process_util.cc       | 32 +++++++++++++++----
 tensorflow/core/common_runtime/process_util.h | 13 +++++++-
 3 files changed, 52 insertions(+), 13 deletions(-)

diff --git a/tensorflow/core/common_runtime/local_device.cc b/tensorflow/core/common_runtime/local_device.cc
index f1fcca194e..2a6d6f5a7a 100644
--- a/tensorflow/core/common_runtime/local_device.cc
+++ b/tensorflow/core/common_runtime/local_device.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/common_runtime/eigen_thread_pool.h"
 #include "tensorflow/core/common_runtime/process_state.h"
+#include "tensorflow/core/common_runtime/process_util.h"
 #include "tensorflow/core/lib/core/threadpool.h"
 #include "tensorflow/core/platform/byte_order.h"
 #include "tensorflow/core/platform/cpu_feature_guard.h"
@@ -53,15 +54,22 @@ struct LocalDevice::EigenThreadPoolInfo {
 
   explicit EigenThreadPoolInfo(const SessionOptions& options, int numa_node,
                                Allocator* allocator) {
+    // Use session setting if specified.
     int32 intra_op_parallelism_threads =
         options.config.intra_op_parallelism_threads();
+    // If no session setting, use environment setting.
     if (intra_op_parallelism_threads == 0) {
-      intra_op_parallelism_threads = port::NumSchedulableCPUs();
-      if (numa_node != port::kNUMANoAffinity) {
-        // Assume that CPUs are equally distributed over available NUMA nodes.
-        // This may not be true, but there isn't currently a better way of
-        // determining the number of CPUs specific to the requested node.
-        intra_op_parallelism_threads /= port::NUMANumNodes();
+      static int env_num_threads = NumIntraOpThreadsFromEnvironment();
+      intra_op_parallelism_threads = env_num_threads;
+      // If no session setting or environment, compute a reasonable default.
+      if (intra_op_parallelism_threads == 0) {
+        intra_op_parallelism_threads = port::NumSchedulableCPUs();
+        if (numa_node != port::kNUMANoAffinity) {
+          // Assume that CPUs are equally distributed over available NUMA nodes.
+          // This may not be true, but there isn't currently a better way of
+          // determining the number of CPUs specific to the requested node.
+          intra_op_parallelism_threads /= port::NUMANumNodes();
+        }
       }
     }
     ThreadOptions thread_opts;
diff --git a/tensorflow/core/common_runtime/process_util.cc b/tensorflow/core/common_runtime/process_util.cc
index e1dc08d645..d42b8d55e4 100644
--- a/tensorflow/core/common_runtime/process_util.cc
+++ b/tensorflow/core/common_runtime/process_util.cc
@@ -34,14 +34,23 @@ namespace tensorflow {
 
 namespace {
 
+int32 DefaultNumInterOpThreads() {
+  // Use environment setting if specified (init once)
+  static int env_num_threads = NumInterOpThreadsFromEnvironment();
+  if (env_num_threads > 0) {
+    return env_num_threads;
+  }
+
+  // Default to using the number of cores available in the process.
+  return port::NumSchedulableCPUs();
+}
+
 static thread::ThreadPool* InitComputePool(const SessionOptions& options) {
   int32 inter_op_parallelism_threads =
       options.config.inter_op_parallelism_threads();
   if (inter_op_parallelism_threads == 0) {
-    // Default to using the number of cores available in the process.
-    inter_op_parallelism_threads = port::NumSchedulableCPUs();
+    inter_op_parallelism_threads = DefaultNumInterOpThreads();
   }
-
   return new thread::ThreadPool(Env::Default(), "Compute",
                                 inter_op_parallelism_threads);
 }
@@ -53,6 +62,18 @@ thread::ThreadPool* ComputePool(const SessionOptions& options) {
   return compute_pool;
 }
 
+int32 NumInterOpThreadsFromEnvironment() {
+  int32 num;
+  const char* val = std::getenv("TF_NUM_INTEROP_THREADS");
+  return (val && strings::safe_strto32(val, &num)) ? num : 0;
+}
+
+int32 NumIntraOpThreadsFromEnvironment() {
+  int32 num;
+  const char* val = std::getenv("TF_NUM_INTRAOP_THREADS");
+  return (val && strings::safe_strto32(val, &num)) ? num : 0;
+}
+
 int32 NumInterOpThreadsFromSessionOptions(const SessionOptions& options) {
   const int32 inter_op = options.config.inter_op_parallelism_threads();
   if (inter_op != 0) return inter_op;
@@ -67,7 +88,7 @@ int32 NumInterOpThreadsFromSessionOptions(const SessionOptions& options) {
 #endif  // _OPENMP
     DCHECK_GE(mkl_intra_op, 1);
     const int32 mkl_inter_op = std::max(
-        (port::NumSchedulableCPUs() + mkl_intra_op - 1) / mkl_intra_op, 2);
+        (DefaultNumInterOpThreads() + mkl_intra_op - 1) / mkl_intra_op, 2);
     VLOG(0)
         << "Creating new thread pool with default inter op setting: "
         << mkl_inter_op
@@ -75,8 +96,7 @@ int32 NumInterOpThreadsFromSessionOptions(const SessionOptions& options) {
     return mkl_inter_op;
   }
 #endif  // INTEL_MKL
-  // Default to using the number of cores available in the process.
-  return port::NumSchedulableCPUs();
+  return DefaultNumInterOpThreads();
 }
 
 thread::ThreadPool* NewThreadPoolFromSessionOptions(
diff --git a/tensorflow/core/common_runtime/process_util.h b/tensorflow/core/common_runtime/process_util.h
index 5d92666716..7ad658be9f 100644
--- a/tensorflow/core/common_runtime/process_util.h
+++ b/tensorflow/core/common_runtime/process_util.h
@@ -30,7 +30,18 @@ namespace tensorflow {
 // using 'options'.  Caller does not take ownership over threadpool.
 thread::ThreadPool* ComputePool(const SessionOptions& options);
 
-// Returns number of inter op threads.
+// Returns the TF_NUM_INTEROP_THREADS environment value, or 0 if not specified.
+int32 NumInterOpThreadsFromEnvironment();
+
+// Returns the TF_NUM_INTRAOP_THREADS environment value, or 0 if not specified.
+int32 NumIntraOpThreadsFromEnvironment();
+
+// Returns the number of inter op threads specified in `options` or a default.
+// If no value is specified in the provided options, then the function returns
+// the value defined in the TF_NUM_INTEROP_THREADS environment variable.
+// If neither a value is specified in the options or in the environment,
+// this function will return a reasonable default value based on the number
+// of schedulable CPUs, and any MKL and OpenMP configurations.
 int32 NumInterOpThreadsFromSessionOptions(const SessionOptions& options);
 
 // Creates a thread pool with number of inter op threads.
-- 
GitLab


From 99121926a8d774f1e205c00108396bc955346e3d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 7 Feb 2019 17:12:38 -0800
Subject: [PATCH 2117/2345] Implement tf.broadcast_to in a way the can handle
 higher ranks. This version uses the broadcast optimizations in
 core/util/bcast.h, which is used for most broadcasting ops in TensorFlow.

PiperOrigin-RevId: 232975751
---
 tensorflow/core/kernels/broadcast_to_op.cc    |  58 ++++-
 tensorflow/core/kernels/broadcast_to_op.h     | 225 +++++-------------
 tensorflow/core/util/bcast.h                  |  12 +-
 .../kernel_tests/broadcast_to_ops_test.py     |  14 +-
 tensorflow/python/ops/array_grad.py           |   2 +
 5 files changed, 124 insertions(+), 187 deletions(-)

diff --git a/tensorflow/core/kernels/broadcast_to_op.cc b/tensorflow/core/kernels/broadcast_to_op.cc
index 2810925bbc..8c4341335f 100644
--- a/tensorflow/core/kernels/broadcast_to_op.cc
+++ b/tensorflow/core/kernels/broadcast_to_op.cc
@@ -15,13 +15,17 @@ limitations under the License.
 
 #define EIGEN_USE_THREADS
 
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+#endif  // GOOGLE_CUDA
 
+#include "tensorflow/core/kernels/broadcast_to_op.h"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/broadcast_to_op.h"
+#include "tensorflow/core/util/bcast.h"
 
 namespace tensorflow {
 
@@ -43,12 +47,42 @@ class BroadcastToOp : public OpKernel {
     OP_REQUIRES_OK(ctx,
                    ctx->op_kernel().MakeShape(shape_tensor, &output_shape));
 
+    // Handle copy.
+    if (output_shape == input_shape) {
+      ctx->set_output(0, input_tensor);
+      return;
+    }
+
+    OP_REQUIRES(ctx, input_shape.dims() <= output_shape.dims(),
+                errors::InvalidArgument(
+                    "Rank of input (", input_shape.dims(),
+                    ") must be no greater than rank of output shape (",
+                    output_shape.dims(), ")."));
+
     Tensor* output_tensor = nullptr;
     OP_REQUIRES_OK(ctx, ctx->allocate_output(0, output_shape, &output_tensor));
-
-    const Device& d = ctx->eigen_device<Device>();
-    functor::BroadcastTo<Device, T>()(d, ctx, *output_tensor, output_shape,
-                                      input_tensor, input_shape);
+    // Handle empty case.
+    if (output_shape.num_elements() == 0) {
+      return;
+    }
+
+    // Handle broadcast from Scalar.
+    const Device& device = ctx->eigen_device<Device>();
+    if (input_shape.dims() == 0) {
+      functor::FillFunctor<Device, T>()(device, output_tensor->flat<T>(),
+                                        input_tensor.scalar<T>());
+      return;
+    }
+
+    BCast bcast(BCast::FromShape(input_shape), BCast::FromShape(output_shape),
+                /*fewer_dims_optimization=*/true);
+    OP_REQUIRES(ctx, bcast.IsValid(),
+                errors::InvalidArgument(
+                    "Incompatible shapes: ", input_shape.DebugString(), " vs. ",
+                    output_shape.DebugString()));
+
+    functor::BroadcastTo<Device, T>()(device, ctx, *output_tensor, output_shape,
+                                      input_tensor, input_shape, bcast);
   }
 };
 
@@ -65,12 +99,12 @@ TF_CALL_ALL_TYPES(REGISTER_KERNEL);
 #if GOOGLE_CUDA
 
 namespace functor {
-#define DECLARE_GPU_TEMPLATE(Type)                              \
-  template <>                                                   \
-  void BroadcastTo<GPUDevice, Type>::operator()(                \
-      const GPUDevice& d, OpKernelContext* ctx, Tensor& output, \
-      const TensorShape& output_shape, const Tensor& input,     \
-      const TensorShape& input_shape);                          \
+#define DECLARE_GPU_TEMPLATE(Type)                               \
+  template <>                                                    \
+  void BroadcastTo<GPUDevice, Type>::operator()(                 \
+      const GPUDevice& d, OpKernelContext* ctx, Tensor& output,  \
+      const TensorShape& output_shape, const Tensor& input,      \
+      const TensorShape& input_shape, const BCast& bcast) const; \
   extern template struct BroadcastTo<GPUDevice, Type>;
 
 TF_CALL_GPU_ALL_TYPES(DECLARE_GPU_TEMPLATE);
diff --git a/tensorflow/core/kernels/broadcast_to_op.h b/tensorflow/core/kernels/broadcast_to_op.h
index bc11c5f914..6ae860c2b2 100644
--- a/tensorflow/core/kernels/broadcast_to_op.h
+++ b/tensorflow/core/kernels/broadcast_to_op.h
@@ -23,196 +23,81 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/kernels/fill_functor.h"
+#include "tensorflow/core/util/bcast.h"
 
 namespace tensorflow {
 
 namespace functor {
 
-#define BROADCAST_SHAPE(NDIMS, input_shape, output_shape)                 \
-  auto reshape = AsEigenDSizesWithPrefix<NDIMS>(input_shape);             \
-  auto broadcast = output_shape.AsEigenDSizes<NDIMS>();                   \
-  auto reshape_32bit = AsEigenDSizesWithPrefix<NDIMS, int>(input_shape);  \
-  auto broadcast_32bit = output_shape.AsEigenDSizes<NDIMS, int>();        \
-  if (input_shape.dims() > 0) {                                           \
-    for (int i = 0; i < NDIMS; i++) {                                     \
-      if (reshape[i] != broadcast[i]) {                                   \
-        OP_REQUIRES(                                                      \
-            ctx, ((reshape[i] != 0) && (broadcast[i] % reshape[i] == 0)), \
-            errors::InvalidArgument("invalid shape to broadcast from ",   \
-                                    input_shape.DebugString(), " to ",    \
-                                    output_shape.DebugString()));         \
-        broadcast[i] = broadcast[i] / reshape[i];                         \
-      } else {                                                            \
-        broadcast[i] = 1;                                                 \
-      }                                                                   \
-      if (can_use_32bit) {                                                \
-        broadcast_32bit[i] = static_cast<int>(broadcast[i]);              \
-      }                                                                   \
-    }                                                                     \
+template <typename Device, typename T>
+struct BroadcastTo {
+  template <int NDIMS>
+  void DoBCast32Bit(const Device &device, typename TTypes<T, NDIMS>::Tensor out,
+                    typename TTypes<T, NDIMS>::ConstTensor in,
+                    const typename Eigen::array<int, NDIMS> &bcast) const {
+    To32Bit(out).device(device) = To32Bit(in).broadcast(bcast);
   }
 
-#define HANDLE_BROADCAST_FROM_SCALAR()                              \
-  if (std::is_same<Eigen::GpuDevice, Device>::value) {              \
-    FillFunctor<Device, T>()(d, output_tensor.flat<T>(),            \
-                             input_tensor.scalar<T>());             \
-  } else {                                                          \
-    output.device(d) = output.constant(input_tensor.scalar<T>()()); \
+  template <int NDIMS>
+  void DoBCast(
+      const Device &device, typename TTypes<T, NDIMS>::Tensor out,
+      typename TTypes<T, NDIMS>::ConstTensor in,
+      const typename Eigen::array<Eigen::DenseIndex, NDIMS> &bcast) const {
+    out.device(device) = in.broadcast(bcast);
   }
 
-#define HANDLE_BROADCAST_CASE(dim_i)                                        \
-  case dim_i: {                                                             \
-    if (can_use_32bit) {                                                    \
-      auto input = input_tensor.tensor<T, dim_i>();                         \
-      To32Bit(output).device(d) =                                           \
-          To32Bit(input).reshape(reshape_32bit).broadcast(broadcast_32bit); \
-    } else {                                                                \
-      auto input = input_tensor.tensor<T, dim_i>();                         \
-      output.device(d) = input.reshape(reshape).broadcast(broadcast);       \
-    }                                                                       \
-  } break
-
-template <typename Device, typename T>
-struct BroadcastTo {
-  void operator()(const Device &d, OpKernelContext *ctx, Tensor &output_tensor,
-                  const TensorShape &output_shape, const Tensor &input_tensor,
-                  const TensorShape &input_shape) {
-    if (output_shape.num_elements() == 0) {
-      return;
-    }
-    if (output_shape == input_shape) {
-      output_tensor.flat<T>().device(d) = input_tensor.flat<T>();
-      return;
-    }
-
+  template <int NDIMS>
+  void ReshapeAndBCast(const Device &device, Tensor &output_tensor,
+                       const Tensor &input_tensor, const BCast &bcast) const {
     const bool can_use_32bit = std::is_same<Eigen::GpuDevice, Device>::value &&
                                output_tensor.NumElements() < kint32max &&
                                input_tensor.NumElements() < kint32max;
+    if (can_use_32bit) {
+      DoBCast32Bit<NDIMS>(
+          device, output_tensor.template shaped<T, NDIMS>(bcast.result_shape()),
+          input_tensor.template shaped<T, NDIMS>(bcast.x_reshape()),
+          BCast::ToIndexArrayType<int, NDIMS>(bcast.x_bcast()));
+    } else {
+      DoBCast<NDIMS>(
+          device, output_tensor.template shaped<T, NDIMS>(bcast.result_shape()),
+          input_tensor.template shaped<T, NDIMS>(bcast.x_reshape()),
+          BCast::ToIndexArrayType<Eigen::DenseIndex, NDIMS>(bcast.x_bcast()));
+    }
+  }
 
-    switch (output_shape.dims()) {
-      case 0: {
-        if (input_shape.dims() > 0) {
-          ctx->CtxFailure(errors::InvalidArgument(
-              "invalid shape to broadcast from ", input_shape.DebugString(),
-              " to ", output_shape.DebugString()));
-          break;
-        }
-        output_tensor.scalar<T>().device(d) = input_tensor.scalar<T>();
+  // PRECONDITION: rank(input_shape) > 0 &&
+  //               rank(input_shape) <= rank(output_shape)  &&
+  //               output_shape.num_elements() > 0.
+  void operator()(const Device &device, OpKernelContext *ctx,
+                  Tensor &output_tensor, const TensorShape &output_shape,
+                  const Tensor &input_tensor, const TensorShape &input_shape,
+                  const BCast &bcast) const {
+    const int ndims = bcast.y_reshape().size();
+    switch (ndims) {
+      case 1:
+        ReshapeAndBCast<1>(device, output_tensor, input_tensor, bcast);
+        break;
+      case 2:
+        ReshapeAndBCast<2>(device, output_tensor, input_tensor, bcast);
+        break;
+      case 3:
+        ReshapeAndBCast<3>(device, output_tensor, input_tensor, bcast);
+        break;
+      case 4:
+        ReshapeAndBCast<4>(device, output_tensor, input_tensor, bcast);
+        break;
+      case 5:
+        ReshapeAndBCast<5>(device, output_tensor, input_tensor, bcast);
         break;
-      }
-      case 1: {
-        BROADCAST_SHAPE(1, input_shape, output_shape);
-
-        auto output = output_tensor.tensor<T, 1>();
-        switch (input_shape.dims()) {
-          case 0: {
-            HANDLE_BROADCAST_FROM_SCALAR();
-          } break;
-            HANDLE_BROADCAST_CASE(1);
-          default:
-            ctx->CtxFailure(errors::InvalidArgument(
-                "invalid shape to broadcast from ", input_shape.DebugString(),
-                " to ", output_shape.DebugString()));
-            break;
-        }
-      } break;
-      case 2: {
-        BROADCAST_SHAPE(2, input_shape, output_shape);
-        auto output = output_tensor.tensor<T, 2>();
-        switch (input_shape.dims()) {
-          case 0: {
-            HANDLE_BROADCAST_FROM_SCALAR();
-          } break;
-            HANDLE_BROADCAST_CASE(1);
-            HANDLE_BROADCAST_CASE(2);
-          default:
-            ctx->CtxFailure(errors::InvalidArgument(
-                "invalid shape to broadcast from ", input_shape.DebugString(),
-                " to ", output_shape.DebugString()));
-            break;
-        }
-      } break;
-      case 3: {
-        BROADCAST_SHAPE(3, input_shape, output_shape);
-        auto output = output_tensor.tensor<T, 3>();
-        switch (input_shape.dims()) {
-          case 0: {
-            HANDLE_BROADCAST_FROM_SCALAR();
-          } break;
-            HANDLE_BROADCAST_CASE(1);
-            HANDLE_BROADCAST_CASE(2);
-            HANDLE_BROADCAST_CASE(3);
-          default:
-            ctx->CtxFailure(errors::InvalidArgument(
-                "invalid shape to broadcast from ", input_shape.DebugString(),
-                " to ", output_shape.DebugString()));
-            break;
-        }
-      } break;
-      case 4: {
-        BROADCAST_SHAPE(4, input_shape, output_shape);
-        auto output = output_tensor.tensor<T, 4>();
-        switch (input_shape.dims()) {
-          case 0: {
-            HANDLE_BROADCAST_FROM_SCALAR();
-          } break;
-            HANDLE_BROADCAST_CASE(1);
-            HANDLE_BROADCAST_CASE(2);
-            HANDLE_BROADCAST_CASE(3);
-            HANDLE_BROADCAST_CASE(4);
-          default:
-            ctx->CtxFailure(errors::InvalidArgument(
-                "invalid shape to broadcast from ", input_shape.DebugString(),
-                " to ", output_shape.DebugString()));
-            break;
-        }
-      } break;
-      case 5: {
-        BROADCAST_SHAPE(5, input_shape, output_shape);
-        auto output = output_tensor.tensor<T, 5>();
-        switch (input_shape.dims()) {
-          case 0: {
-            HANDLE_BROADCAST_FROM_SCALAR();
-          } break;
-            HANDLE_BROADCAST_CASE(1);
-            HANDLE_BROADCAST_CASE(2);
-            HANDLE_BROADCAST_CASE(3);
-            HANDLE_BROADCAST_CASE(4);
-            HANDLE_BROADCAST_CASE(5);
-          default:
-            ctx->CtxFailure(errors::InvalidArgument(
-                "invalid shape to broadcast from ", input_shape.DebugString(),
-                " to ", output_shape.DebugString()));
-            break;
-        }
-      } break;
       default:
-        ctx->CtxFailure(errors::InvalidArgument(
-            "invalid shape to broadcast from ", input_shape.DebugString(),
-            " to ", output_shape.DebugString()));
+        ctx->SetStatus(errors::Unimplemented(
+            "Broadcast between ", input_shape.DebugString(), " and ",
+            output_shape.DebugString(), " is not supported yet."));
         break;
     }
   }
-
- private:
-  template <int NDIMS, typename DimType = Eigen::DenseIndex>
-  Eigen::DSizes<DimType, NDIMS> AsEigenDSizesWithPrefix(
-      const TensorShape &shape) const {
-    Eigen::DSizes<DimType, NDIMS> dsizes;
-    for (int d = 0; d < NDIMS - shape.dims(); d++) {
-      dsizes[d] = 1;
-    }
-    for (int d = NDIMS - shape.dims(); d < NDIMS; d++) {
-      dsizes[d] =
-          static_cast<DimType>(shape.dim_size(d - (NDIMS - shape.dims())));
-    }
-    return dsizes;
-  }
 };
 
-#undef BROADCAST_SHAPE
-#undef HANDLE_BROADCAST_FROM_SCALAR
-#undef HANDLE_BROADCAST_CASE
-
 }  // namespace functor
 }  // namespace tensorflow
 
diff --git a/tensorflow/core/util/bcast.h b/tensorflow/core/util/bcast.h
index 6d73c38e3c..2d647fd8d8 100644
--- a/tensorflow/core/util/bcast.h
+++ b/tensorflow/core/util/bcast.h
@@ -105,15 +105,21 @@ class BCast {
   static Vec FromShape(const TensorShape& shape);
   static TensorShape ToShape(const BCast::Vec& vec);
 
-  template <int NDIMS>
-  static Eigen::array<Eigen::DenseIndex, NDIMS> ToIndexArray(
+  template <typename IndexType, int NDIMS>
+  static Eigen::array<IndexType, NDIMS> ToIndexArrayType(
       const BCast::Vec& vec) {
     CHECK_EQ(vec.size(), NDIMS);
-    Eigen::array<Eigen::DenseIndex, NDIMS> ret;
+    Eigen::array<IndexType, NDIMS> ret;
     for (int i = 0; i < NDIMS; ++i) ret[i] = vec[i];
     return ret;
   }
 
+  template <int NDIMS>
+  static Eigen::array<Eigen::DenseIndex, NDIMS> ToIndexArray(
+      const BCast::Vec& vec) {
+    return ToIndexArrayType<Eigen::DenseIndex, NDIMS>(vec);
+  }
+
  private:
   bool valid_ = true;
   Vec x_reshape_;
diff --git a/tensorflow/python/kernel_tests/broadcast_to_ops_test.py b/tensorflow/python/kernel_tests/broadcast_to_ops_test.py
index b9eb2391b4..366300ec01 100644
--- a/tensorflow/python/kernel_tests/broadcast_to_ops_test.py
+++ b/tensorflow/python/kernel_tests/broadcast_to_ops_test.py
@@ -66,6 +66,16 @@ class BroadcastToTest(test_util.TensorFlowTestCase):
           v_np = np.broadcast_to(x, output_shape)
           self.assertAllEqual(v_tf.eval(), v_np)
 
+  @test_util.run_deprecated_v1
+  def testBroadcastToShapeInnerDim(self):
+    input_shape = [2, 1, 3]
+    output_shape = [2, 5, 3]
+    with self.cached_session(use_gpu=True):
+      x = np.array(np.random.randint(5, size=input_shape), dtype=np.int32)
+      v_tf = array_ops.broadcast_to(constant_op.constant(x), output_shape)
+      v_np = np.broadcast_to(x, output_shape)
+      self.assertAllEqual(v_tf.eval(), v_np)
+
   @test_util.run_deprecated_v1
   def testBroadcastToScalar(self):
     with self.session(use_gpu=True):
@@ -130,8 +140,8 @@ class BroadcastToTest(test_util.TensorFlowTestCase):
 
   @test_util.run_deprecated_v1
   def testGradientWithBroadcastAllDimensions(self):
-    x = constant_op.constant([[1, 2, 3], [4, 5, 6]], dtype=dtypes.float32)
-    v = array_ops.broadcast_to(x, [5, 4, 6])
+    x = constant_op.constant([1], dtype=dtypes.float32)
+    v = array_ops.broadcast_to(x, [5, 2, 3])
     out = 2 * v
     with self.cached_session():
       err = gradient_checker.compute_gradient_error(x, x.get_shape(),
diff --git a/tensorflow/python/ops/array_grad.py b/tensorflow/python/ops/array_grad.py
index 45e741ef22..06f0a45f6a 100644
--- a/tensorflow/python/ops/array_grad.py
+++ b/tensorflow/python/ops/array_grad.py
@@ -847,6 +847,8 @@ def _BroadcastToGrad(op, grad):
   # Group by ids and sum its gradients.
   grad_flatten = array_ops.reshape(grad, [-1])
   broadcast_ids_flatten = array_ops.reshape(broadcast_ids, [-1])
+  # TODO(apassos): Use reduce_sum for gradient now that we only support
+  # the usual numpy broadcast semantics.
   updates_grad_flatten = math_ops.unsorted_segment_sum(grad_flatten,
                                                        broadcast_ids_flatten,
                                                        input_value_size)
-- 
GitLab


From fc4005be8eed17eccce613b52888c0b783565800 Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Thu, 7 Feb 2019 17:24:16 -0800
Subject: [PATCH 2118/2345] Resize bilinear int8 support.

PiperOrigin-RevId: 232977313
---
 tensorflow/lite/kernels/register.cc           |   4 +-
 tensorflow/lite/kernels/resize_bilinear.cc    |   2 +
 .../lite/kernels/resize_bilinear_test.cc      | 131 +++++++++++++++++-
 tensorflow/lite/toco/tflite/operator.cc       |   6 +
 4 files changed, 137 insertions(+), 6 deletions(-)

diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index 66c95ffb6f..fec8e63ee0 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -224,7 +224,9 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_PAD, Register_PAD());
   AddBuiltin(BuiltinOperator_PADV2, Register_PADV2());
   AddBuiltin(BuiltinOperator_RESHAPE, Register_RESHAPE());
-  AddBuiltin(BuiltinOperator_RESIZE_BILINEAR, Register_RESIZE_BILINEAR());
+  AddBuiltin(BuiltinOperator_RESIZE_BILINEAR, Register_RESIZE_BILINEAR(),
+             /* min_version */ 1,
+             /* max_version */ 2);
   AddBuiltin(BuiltinOperator_RESIZE_NEAREST_NEIGHBOR,
              Register_RESIZE_NEAREST_NEIGHBOR(),
              /* min_version */ 1,
diff --git a/tensorflow/lite/kernels/resize_bilinear.cc b/tensorflow/lite/kernels/resize_bilinear.cc
index d42cb18866..7383d03438 100644
--- a/tensorflow/lite/kernels/resize_bilinear.cc
+++ b/tensorflow/lite/kernels/resize_bilinear.cc
@@ -109,6 +109,8 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
     if (kernel_type == kGenericOptimized || kernel_type == kNeonOptimized) {
       TF_LITE_RESIZE_BILINEAR(optimized_ops, uint8_t);
     }
+  } else if (output->type == kTfLiteInt8) {
+    TF_LITE_RESIZE_BILINEAR(reference_ops, int8_t);
 #undef TF_LITE_RESIZE_BILINEAR
   } else {
     context->ReportError(context, "Output type is %d, requires float.",
diff --git a/tensorflow/lite/kernels/resize_bilinear_test.cc b/tensorflow/lite/kernels/resize_bilinear_test.cc
index d3f4837a28..a5ead9c874 100644
--- a/tensorflow/lite/kernels/resize_bilinear_test.cc
+++ b/tensorflow/lite/kernels/resize_bilinear_test.cc
@@ -78,7 +78,7 @@ TEST(ResizeBilinearOpTest, HorizontalResize) {
               ElementsAreArray(ArrayFloatNear({3, 5, 6})));
 }
 
-TEST(ResizeBilinearOpTest, HorizontalResize8Bit) {
+TEST(ResizeBilinearOpTest, HorizontalResizeUInt8) {
   ResizeBilinearOpModel m({TensorType_UINT8, {1, 1, 2, 1}});
   m.SetInput<uint8>({3, 6});
   m.SetSize({1, 3});
@@ -93,6 +93,21 @@ TEST(ResizeBilinearOpTest, HorizontalResize8Bit) {
               ElementsAreArray(ArrayFloatNear({3, 5, 6})));
 }
 
+TEST(ResizeBilinearOpTest, HorizontalResizeInt8) {
+  ResizeBilinearOpModel m({TensorType_INT8, {1, 1, 2, 1}});
+  m.SetInput<int8_t>({3, 6});
+  m.SetSize({1, 3});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutput<int8_t>(),
+              ElementsAreArray(ArrayFloatNear({3, 5, 6})));
+
+  ResizeBilinearOpModel const_m({TensorType_INT8, {1, 1, 2, 1}}, {1, 3});
+  const_m.SetInput<int8_t>({3, 6});
+  const_m.Invoke();
+  EXPECT_THAT(const_m.GetOutput<int8_t>(),
+              ElementsAreArray(ArrayFloatNear({3, 5, 6})));
+}
+
 TEST(ResizeBilinearOpTest, VerticalResize) {
   ResizeBilinearOpModel m({TensorType_FLOAT32, {1, 2, 1, 1}});
   m.SetInput<float>({3, 9});
@@ -108,7 +123,7 @@ TEST(ResizeBilinearOpTest, VerticalResize) {
               ElementsAreArray(ArrayFloatNear({3, 7, 9})));
 }
 
-TEST(ResizeBilinearOpTest, VerticalResize8Bit) {
+TEST(ResizeBilinearOpTest, VerticalResizeUInt8) {
   ResizeBilinearOpModel m({TensorType_UINT8, {1, 2, 1, 1}});
   m.SetInput<uint8>({3, 9});
   m.SetSize({3, 1});
@@ -123,6 +138,21 @@ TEST(ResizeBilinearOpTest, VerticalResize8Bit) {
               ElementsAreArray(ArrayFloatNear({3, 7, 9})));
 }
 
+TEST(ResizeBilinearOpTest, VerticalResizeInt8) {
+  ResizeBilinearOpModel m({TensorType_INT8, {1, 2, 1, 1}});
+  m.SetInput<int8_t>({3, 9});
+  m.SetSize({3, 1});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutput<int8_t>(),
+              ElementsAreArray(ArrayFloatNear({3, 7, 9})));
+
+  ResizeBilinearOpModel const_m({TensorType_INT8, {1, 2, 1, 1}}, {3, 1});
+  const_m.SetInput<int8_t>({3, 9});
+  const_m.Invoke();
+  EXPECT_THAT(const_m.GetOutput<int8_t>(),
+              ElementsAreArray(ArrayFloatNear({3, 7, 9})));
+}
+
 TEST(ResizeBilinearOpTest, TwoDimensionalResize) {
   ResizeBilinearOpModel m({TensorType_FLOAT32, {1, 2, 2, 1}});
   m.SetInput<float>({
@@ -150,7 +180,7 @@ TEST(ResizeBilinearOpTest, TwoDimensionalResize) {
                                           })));
 }
 
-TEST(ResizeBilinearOpTest, TwoDimensionalResize8Bit) {
+TEST(ResizeBilinearOpTest, TwoDimensionalResizeUInt8) {
   ResizeBilinearOpModel m({TensorType_UINT8, {1, 2, 2, 1}});
   m.SetInput<uint8>({
       3, 6,  //
@@ -177,6 +207,33 @@ TEST(ResizeBilinearOpTest, TwoDimensionalResize8Bit) {
                                           })));
 }
 
+TEST(ResizeBilinearOpTest, TwoDimensionalResizeInt8) {
+  ResizeBilinearOpModel m({TensorType_INT8, {1, 2, 2, 1}});
+  m.SetInput<int8_t>({
+      3, 6,  //
+      9, 12  //
+  });
+  m.SetSize({3, 3});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutput<int8_t>(), ElementsAreArray(ArrayFloatNear({
+                                         3, 5, 6,    //
+                                         7, 9, 10,   //
+                                         9, 11, 12,  //
+                                     })));
+
+  ResizeBilinearOpModel const_m({TensorType_INT8, {1, 2, 2, 1}}, {3, 3});
+  const_m.SetInput<int8_t>({
+      3, 6,  //
+      9, 12  //
+  });
+  const_m.Invoke();
+  EXPECT_THAT(const_m.GetOutput<int8_t>(), ElementsAreArray(ArrayFloatNear({
+                                               3, 5, 6,    //
+                                               7, 9, 10,   //
+                                               9, 11, 12,  //
+                                           })));
+}
+
 TEST(ResizeBilinearOpTest, TwoDimensionalResizeWithTwoBatches) {
   ResizeBilinearOpModel m({TensorType_FLOAT32, {2, 2, 2, 1}});
   m.SetInput<float>({
@@ -241,7 +298,7 @@ TEST(ResizeBilinearOpTest, ThreeDimensionalResize) {
                                           })));
 }
 
-TEST(ResizeBilinearOpTest, TwoDimensionalResizeWithTwoBatches8Bit) {
+TEST(ResizeBilinearOpTest, TwoDimensionalResizeWithTwoBatchesUInt8) {
   ResizeBilinearOpModel m({TensorType_UINT8, {2, 2, 2, 1}});
   m.SetInput<uint8>({
       3, 6,   //
@@ -278,7 +335,44 @@ TEST(ResizeBilinearOpTest, TwoDimensionalResizeWithTwoBatches8Bit) {
                                           })));
 }
 
-TEST(ResizeBilinearOpTest, ThreeDimensionalResize8Bit) {
+TEST(ResizeBilinearOpTest, TwoDimensionalResizeWithTwoBatchesInt8) {
+  ResizeBilinearOpModel m({TensorType_INT8, {2, 2, 2, 1}});
+  m.SetInput<int8_t>({
+      3, 6,   //
+      9, 12,  //
+      4, 10,  //
+      12, 16  //
+  });
+  m.SetSize({3, 3});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutput<int8_t>(), ElementsAreArray(ArrayFloatNear({
+                                         3, 5, 6,     //
+                                         7, 9, 10,    //
+                                         9, 11, 12,   //
+                                         4, 8, 10,    //
+                                         9, 12, 13,   //
+                                         12, 14, 16,  //
+                                     })));
+
+  ResizeBilinearOpModel const_m({TensorType_INT8, {2, 2, 2, 1}}, {3, 3});
+  const_m.SetInput<int8_t>({
+      3, 6,   //
+      9, 12,  //
+      4, 10,  //
+      12, 16  //
+  });
+  const_m.Invoke();
+  EXPECT_THAT(const_m.GetOutput<int8_t>(), ElementsAreArray(ArrayFloatNear({
+                                               3, 5, 6,     //
+                                               7, 9, 10,    //
+                                               9, 11, 12,   //
+                                               4, 8, 10,    //
+                                               9, 12, 13,   //
+                                               12, 14, 16,  //
+                                           })));
+}
+
+TEST(ResizeBilinearOpTest, ThreeDimensionalResizeUInt8) {
   ResizeBilinearOpModel m({TensorType_UINT8, {1, 2, 2, 2}});
   m.SetInput<uint8>({
       3, 4, 6, 10,     //
@@ -304,6 +398,33 @@ TEST(ResizeBilinearOpTest, ThreeDimensionalResize8Bit) {
                                               10, 12, 12, 14, 14, 16,  //
                                           })));
 }
+
+TEST(ResizeBilinearOpTest, ThreeDimensionalResizeInt8) {
+  ResizeBilinearOpModel m({TensorType_INT8, {1, 2, 2, 2}});
+  m.SetInput<int8_t>({
+      3, 4, 6, 10,     //
+      10, 12, 14, 16,  //
+  });
+  m.SetSize({3, 3});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutput<int8_t>(), ElementsAreArray(ArrayFloatNear({
+                                         3, 4, 5, 8, 6, 10,       //
+                                         7, 9, 10, 12, 11, 13,    //
+                                         10, 12, 12, 14, 14, 16,  //
+                                     })));
+
+  ResizeBilinearOpModel const_m({TensorType_INT8, {1, 2, 2, 2}}, {3, 3});
+  const_m.SetInput<int8_t>({
+      3, 4, 6, 10,     //
+      10, 12, 14, 16,  //
+  });
+  const_m.Invoke();
+  EXPECT_THAT(const_m.GetOutput<int8_t>(), ElementsAreArray(ArrayFloatNear({
+                                               3, 4, 5, 8, 6, 10,       //
+                                               7, 9, 10, 12, 11, 13,    //
+                                               10, 12, 12, 14, 14, 16,  //
+                                           })));
+}
 }  // namespace
 }  // namespace tflite
 
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index 773be365bc..fe40e71847 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -1189,6 +1189,12 @@ class ResizeBilinear
   }
 
   int GetVersion(const OperatorSignature& op_signature) const override {
+    const string& input_name = op_signature.op->inputs[0];
+    const Array& input_array = op_signature.model->GetArray(input_name);
+    // If the op takes int8 input, it is version 2.
+    if (input_array.data_type == ArrayDataType::kInt8) {
+      return 2;
+    }
     return 1;
   }
 };
-- 
GitLab


From ec5e355a1df575b85f9d0114e0522b64e1b9c6ff Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Thu, 7 Feb 2019 17:45:38 -0800
Subject: [PATCH 2119/2345] Make `ModelCheckpoint` callback default to using
 `save_weights` instead of `save` when receiving a subclassed model.

PiperOrigin-RevId: 232980088
---
 tensorflow/python/keras/callbacks.py      |  10 +-
 tensorflow/python/keras/callbacks_test.py | 331 +++++++++++-----------
 2 files changed, 178 insertions(+), 163 deletions(-)

diff --git a/tensorflow/python/keras/callbacks.py b/tensorflow/python/keras/callbacks.py
index c5ae62c2f5..1c2ed81367 100644
--- a/tensorflow/python/keras/callbacks.py
+++ b/tensorflow/python/keras/callbacks.py
@@ -391,7 +391,7 @@ class Callback(object):
   take as argument will contain keys for quantities relevant to
   the current batch or epoch.
 
-  Currently, the `.fit()` method of the `Sequential` model class
+  Currently, the `.fit()` method of the `Model` class
   will include the following quantities in the `logs` that
   it passes to its callbacks:
 
@@ -811,6 +811,14 @@ class ModelCheckpoint(Callback):
         self.monitor_op = np.less
         self.best = np.Inf
 
+  def set_model(self, model):
+    self.model = model
+    # Use name matching rather than `isinstance` to avoid circular dependencies.
+    if (not self.save_weights_only and
+        not model._is_graph_network and  # pylint: disable=protected-access
+        model.__class__.__name__ != 'Sequential'):
+      self.save_weights_only = True
+
   def on_epoch_end(self, epoch, logs=None):
     logs = logs or {}
     self.epochs_since_last_save += 1
diff --git a/tensorflow/python/keras/callbacks_test.py b/tensorflow/python/keras/callbacks_test.py
index cced204750..e188c68c0b 100644
--- a/tensorflow/python/keras/callbacks_test.py
+++ b/tensorflow/python/keras/callbacks_test.py
@@ -217,173 +217,180 @@ class CallbackCountsTest(keras_parameterized.TestCase):
         })
 
 
-class KerasCallbacksTest(test.TestCase):
+class KerasCallbacksTest(keras_parameterized.TestCase):
 
+  @keras_parameterized.run_with_all_model_types
   def test_ModelCheckpoint(self):
     if h5py is None:
       return  # Skip test if models cannot be saved.
 
-    with self.cached_session():
-      np.random.seed(1337)
-
-      temp_dir = self.get_temp_dir()
-      self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True)
-
-      filepath = os.path.join(temp_dir, 'checkpoint.h5')
-      (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
-          train_samples=TRAIN_SAMPLES,
-          test_samples=TEST_SAMPLES,
-          input_shape=(INPUT_DIM,),
-          num_classes=NUM_CLASSES)
-      y_test = keras.utils.to_categorical(y_test)
-      y_train = keras.utils.to_categorical(y_train)
-      # case 1
-      monitor = 'val_loss'
-      save_best_only = False
-      mode = 'auto'
-
-      model = keras.models.Sequential()
-      model.add(
-          keras.layers.Dense(
-              NUM_HIDDEN, input_dim=INPUT_DIM, activation='relu'))
-      model.add(keras.layers.Dense(NUM_CLASSES, activation='softmax'))
-      model.compile(
-          loss='categorical_crossentropy',
-          optimizer='rmsprop',
-          metrics=['accuracy'])
-
-      cbks = [
-          keras.callbacks.ModelCheckpoint(
-              filepath,
-              monitor=monitor,
-              save_best_only=save_best_only,
-              mode=mode)
-      ]
-      model.fit(
-          x_train,
-          y_train,
-          batch_size=BATCH_SIZE,
-          validation_data=(x_test, y_test),
-          callbacks=cbks,
-          epochs=1,
-          verbose=0)
-      assert os.path.exists(filepath)
-      os.remove(filepath)
-
-      # case 2
-      mode = 'min'
-      cbks = [
-          keras.callbacks.ModelCheckpoint(
-              filepath,
-              monitor=monitor,
-              save_best_only=save_best_only,
-              mode=mode)
-      ]
-      model.fit(
-          x_train,
-          y_train,
-          batch_size=BATCH_SIZE,
-          validation_data=(x_test, y_test),
-          callbacks=cbks,
-          epochs=1,
-          verbose=0)
-      assert os.path.exists(filepath)
-      os.remove(filepath)
-
-      # case 3
-      mode = 'max'
-      monitor = 'val_acc'
-      cbks = [
-          keras.callbacks.ModelCheckpoint(
-              filepath,
-              monitor=monitor,
-              save_best_only=save_best_only,
-              mode=mode)
-      ]
-      model.fit(
-          x_train,
-          y_train,
-          batch_size=BATCH_SIZE,
-          validation_data=(x_test, y_test),
-          callbacks=cbks,
-          epochs=1,
-          verbose=0)
-      assert os.path.exists(filepath)
-      os.remove(filepath)
-
-      # case 4
-      save_best_only = True
-      cbks = [
-          keras.callbacks.ModelCheckpoint(
-              filepath,
-              monitor=monitor,
-              save_best_only=save_best_only,
-              mode=mode)
-      ]
-      model.fit(
-          x_train,
-          y_train,
-          batch_size=BATCH_SIZE,
-          validation_data=(x_test, y_test),
-          callbacks=cbks,
-          epochs=1,
-          verbose=0)
-      assert os.path.exists(filepath)
-      os.remove(filepath)
-
-      # Case: metric not available.
-      cbks = [
-          keras.callbacks.ModelCheckpoint(
-              filepath,
-              monitor='unknown',
-              save_best_only=True)
-      ]
-      model.fit(
-          x_train,
-          y_train,
-          batch_size=BATCH_SIZE,
-          validation_data=(x_test, y_test),
-          callbacks=cbks,
-          epochs=1,
-          verbose=0)
-      # File won't be written.
-      assert not os.path.exists(filepath)
-
-      # case 5
-      save_best_only = False
-      period = 2
-      mode = 'auto'
-
-      filepath = os.path.join(temp_dir, 'checkpoint.{epoch:02d}.h5')
-      cbks = [
-          keras.callbacks.ModelCheckpoint(
-              filepath,
-              monitor=monitor,
-              save_best_only=save_best_only,
-              mode=mode,
-              period=period)
-      ]
-      model.fit(
-          x_train,
-          y_train,
-          batch_size=BATCH_SIZE,
-          validation_data=(x_test, y_test),
-          callbacks=cbks,
-          epochs=4,
-          verbose=1)
-      assert os.path.exists(filepath.format(epoch=2))
-      assert os.path.exists(filepath.format(epoch=4))
-      os.remove(filepath.format(epoch=2))
-      os.remove(filepath.format(epoch=4))
-      assert not os.path.exists(filepath.format(epoch=1))
-      assert not os.path.exists(filepath.format(epoch=3))
-
-      # Invalid use: this will raise a warning but not an Exception.
-      keras.callbacks.ModelCheckpoint(
-          filepath,
-          monitor=monitor,
-          save_best_only=save_best_only,
-          mode='unknown')
+    layers = [
+        keras.layers.Dense(NUM_HIDDEN, input_dim=INPUT_DIM, activation='relu'),
+        keras.layers.Dense(NUM_CLASSES, activation='softmax')
+    ]
+    model = testing_utils.get_model_from_layers(layers, input_shape=(10,))
+    model.compile(loss='categorical_crossentropy',
+                  optimizer='rmsprop',
+                  metrics=['accuracy'])
+
+    temp_dir = self.get_temp_dir()
+    self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True)
+
+    filepath = os.path.join(temp_dir, 'checkpoint')
+    (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
+        train_samples=TRAIN_SAMPLES,
+        test_samples=TEST_SAMPLES,
+        input_shape=(INPUT_DIM,),
+        num_classes=NUM_CLASSES)
+    y_test = keras.utils.to_categorical(y_test)
+    y_train = keras.utils.to_categorical(y_train)
+    # case 1
+    monitor = 'val_loss'
+    save_best_only = False
+    mode = 'auto'
+
+    model = keras.models.Sequential()
+    model.add(
+        keras.layers.Dense(
+            NUM_HIDDEN, input_dim=INPUT_DIM, activation='relu'))
+    model.add(keras.layers.Dense(NUM_CLASSES, activation='softmax'))
+    model.compile(
+        loss='categorical_crossentropy',
+        optimizer='rmsprop',
+        metrics=['accuracy'])
+
+    cbks = [
+        keras.callbacks.ModelCheckpoint(
+            filepath,
+            monitor=monitor,
+            save_best_only=save_best_only,
+            mode=mode)
+    ]
+    model.fit(
+        x_train,
+        y_train,
+        batch_size=BATCH_SIZE,
+        validation_data=(x_test, y_test),
+        callbacks=cbks,
+        epochs=1,
+        verbose=0)
+    assert os.path.exists(filepath)
+    os.remove(filepath)
+
+    # case 2
+    mode = 'min'
+    cbks = [
+        keras.callbacks.ModelCheckpoint(
+            filepath,
+            monitor=monitor,
+            save_best_only=save_best_only,
+            mode=mode)
+    ]
+    model.fit(
+        x_train,
+        y_train,
+        batch_size=BATCH_SIZE,
+        validation_data=(x_test, y_test),
+        callbacks=cbks,
+        epochs=1,
+        verbose=0)
+    assert os.path.exists(filepath)
+    os.remove(filepath)
+
+    # case 3
+    mode = 'max'
+    monitor = 'val_acc'
+    cbks = [
+        keras.callbacks.ModelCheckpoint(
+            filepath,
+            monitor=monitor,
+            save_best_only=save_best_only,
+            mode=mode)
+    ]
+    model.fit(
+        x_train,
+        y_train,
+        batch_size=BATCH_SIZE,
+        validation_data=(x_test, y_test),
+        callbacks=cbks,
+        epochs=1,
+        verbose=0)
+    assert os.path.exists(filepath)
+    os.remove(filepath)
+
+    # case 4
+    save_best_only = True
+    cbks = [
+        keras.callbacks.ModelCheckpoint(
+            filepath,
+            monitor=monitor,
+            save_best_only=save_best_only,
+            mode=mode)
+    ]
+    model.fit(
+        x_train,
+        y_train,
+        batch_size=BATCH_SIZE,
+        validation_data=(x_test, y_test),
+        callbacks=cbks,
+        epochs=1,
+        verbose=0)
+    assert os.path.exists(filepath)
+    os.remove(filepath)
+
+    # Case: metric not available.
+    cbks = [
+        keras.callbacks.ModelCheckpoint(
+            filepath,
+            monitor='unknown',
+            save_best_only=True)
+    ]
+    model.fit(
+        x_train,
+        y_train,
+        batch_size=BATCH_SIZE,
+        validation_data=(x_test, y_test),
+        callbacks=cbks,
+        epochs=1,
+        verbose=0)
+    # File won't be written.
+    assert not os.path.exists(filepath)
+
+    # case 5
+    save_best_only = False
+    period = 2
+    mode = 'auto'
+
+    filepath = os.path.join(temp_dir, 'checkpoint.{epoch:02d}.h5')
+    cbks = [
+        keras.callbacks.ModelCheckpoint(
+            filepath,
+            monitor=monitor,
+            save_best_only=save_best_only,
+            mode=mode,
+            period=period)
+    ]
+    model.fit(
+        x_train,
+        y_train,
+        batch_size=BATCH_SIZE,
+        validation_data=(x_test, y_test),
+        callbacks=cbks,
+        epochs=4,
+        verbose=1)
+    assert os.path.exists(filepath.format(epoch=2))
+    assert os.path.exists(filepath.format(epoch=4))
+    os.remove(filepath.format(epoch=2))
+    os.remove(filepath.format(epoch=4))
+    assert not os.path.exists(filepath.format(epoch=1))
+    assert not os.path.exists(filepath.format(epoch=3))
+
+    # Invalid use: this will raise a warning but not an Exception.
+    keras.callbacks.ModelCheckpoint(
+        filepath,
+        monitor=monitor,
+        save_best_only=save_best_only,
+        mode='unknown')
 
   def test_EarlyStopping(self):
     with self.cached_session():
-- 
GitLab


From 9e10fc685fd922c07bbd4d9b3fa26d83742d09ae Mon Sep 17 00:00:00 2001
From: Nupur Garg <nupurgarg@google.com>
Date: Thu, 7 Feb 2019 17:53:28 -0800
Subject: [PATCH 2120/2345] Makes exporting MetaGraphDefs possible in
 TensorFlow 2.0.

PiperOrigin-RevId: 232980970
---
 tensorflow/python/framework/meta_graph.py | 5 ++++-
 tensorflow/python/training/saver.py       | 7 ++++---
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/framework/meta_graph.py b/tensorflow/python/framework/meta_graph.py
index 0febab506e..9e16ab78a0 100644
--- a/tensorflow/python/framework/meta_graph.py
+++ b/tensorflow/python/framework/meta_graph.py
@@ -972,8 +972,11 @@ def export_scoped_meta_graph(filename=None,
 
   Raises:
     ValueError: When the `GraphDef` is larger than 2GB.
+    ValueError: When executing in Eager mode and either `graph_def` or `graph`
+      is undefined.
   """
-  if context.executing_eagerly():
+  if context.executing_eagerly() and not (graph_def is not None and
+                                          graph is not None):
     raise ValueError("Exporting/importing meta graphs is not supported when "
                      "Eager Execution is enabled.")
   graph = graph or ops.get_default_graph()
diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py
index ee0e2154e8..256b22b04c 100644
--- a/tensorflow/python/training/saver.py
+++ b/tensorflow/python/training/saver.py
@@ -1552,12 +1552,13 @@ def export_meta_graph(filename=None,
     RuntimeError: If called with eager execution enabled.
 
   @compatibility(eager)
-  Exporting/importing meta graphs is not supported. No graph exists when eager
-  execution is enabled.
+  Exporting/importing meta graphs is not supported unless both `graph_def` and
+  `graph` are provided. No graph exists when eager execution is enabled.
   @end_compatibility
   """
   # pylint: enable=line-too-long
-  if context.executing_eagerly():
+  if context.executing_eagerly() and not (graph_def is not None and
+                                          graph is not None):
     raise RuntimeError("Exporting/importing meta graphs is not supported when "
                        "eager execution is enabled. No graph exists when eager "
                        "execution is enabled.")
-- 
GitLab


From 9edfbd753c403613d56f51893271a74acdaa46ef Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 7 Feb 2019 18:02:03 -0800
Subject: [PATCH 2121/2345] Add profiler benchmark.

PiperOrigin-RevId: 232981975
---
 tensorflow/python/eager/BUILD              |  1 +
 tensorflow/python/eager/benchmarks_test.py | 17 +++++++++++++++++
 2 files changed, 18 insertions(+)

diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD
index aba342b6d5..47a7e9ae2f 100644
--- a/tensorflow/python/eager/BUILD
+++ b/tensorflow/python/eager/BUILD
@@ -384,6 +384,7 @@ cuda_py_test(
         ":context",
         ":function",
         ":test",
+        ":profiler",
         "//third_party/py/numpy",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:pywrap_tensorflow",
diff --git a/tensorflow/python/eager/benchmarks_test.py b/tensorflow/python/eager/benchmarks_test.py
index 16b52c9baa..f44185df0b 100644
--- a/tensorflow/python/eager/benchmarks_test.py
+++ b/tensorflow/python/eager/benchmarks_test.py
@@ -38,6 +38,7 @@ from tensorflow.python.eager import backprop  # pylint: disable=unused-import
 from tensorflow.python.eager import context
 from tensorflow.python.eager import core
 from tensorflow.python.eager import function
+from tensorflow.python.eager import profiler
 from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -815,10 +816,26 @@ class MicroBenchmarks(test.Benchmark):
       model = make_keras_model(initializer="glorot_uniform")
       self._benchmark_keras_model_fit(model)
 
+  def benchmark_keras_model_functional_fit_graph_mode_with_profiler(self):
+    profiler.start()
+    with context.graph_mode():
+      model = make_keras_model(initializer="glorot_uniform")
+      self._benchmark_keras_model_fit(model)
+    result = profiler.stop()
+    assert result is not None
+
   def benchmark_keras_model_functional_fit_run_model_eagerly(self):
     model = make_keras_model(initializer="glorot_uniform")
     self._benchmark_keras_model_fit(model, run_eagerly=True)
 
+  def benchmark_keras_model_functional_fit_run_model_eagerly_with_profiler(
+      self):
+    profiler.start()
+    model = make_keras_model(initializer="glorot_uniform")
+    self._benchmark_keras_model_fit(model, run_eagerly=True)
+    result = profiler.stop()
+    assert result is not None
+
   def benchmark_keras_model_sequential_fit(self):
     model = make_sequential_keras_model(initializer="glorot_uniform")
     self._benchmark_keras_model_fit(model)
-- 
GitLab


From ebbf93511f891523551c89c0e465aca92f65d3f1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 7 Feb 2019 18:20:29 -0800
Subject: [PATCH 2122/2345] Allow using Profiler server in graph mode.

In Eager mode, profiler can collect trace from EagerContext, device tracer (CUPTI) and cpu tracer.
In Graph mode, profiler can only collect trace from device tracer and cpu tracer.

PiperOrigin-RevId: 232984228
---
 tensorflow/c/eager/c_api_experimental.cc       | 16 +++++++++++++---
 tensorflow/c/eager/c_api_experimental.h        | 18 +++++++++++++++---
 tensorflow/c/eager/c_api_internal.h            | 11 +++++++++--
 .../core/profiler/lib/profiler_session.cc      | 10 +++++-----
 .../core/profiler/lib/profiler_session.h       |  9 +++++++--
 .../core/profiler/rpc/profiler_server.cc       | 13 ++++++++-----
 tensorflow/core/profiler/rpc/profiler_server.h |  7 ++++---
 .../core/profiler/rpc/profiler_service_impl.cc | 16 +++++++++-------
 .../core/profiler/rpc/profiler_service_impl.h  |  4 +++-
 tensorflow/python/eager/profiler.py            | 12 ++++++++----
 tensorflow/python/pywrap_tfe.i                 |  2 ++
 11 files changed, 83 insertions(+), 35 deletions(-)

diff --git a/tensorflow/c/eager/c_api_experimental.cc b/tensorflow/c/eager/c_api_experimental.cc
index dab1750564..e67e54b152 100644
--- a/tensorflow/c/eager/c_api_experimental.cc
+++ b/tensorflow/c/eager/c_api_experimental.cc
@@ -46,7 +46,17 @@ void TFE_ProfilerSerializeToString(TFE_Context* ctx, TFE_Profiler* profiler,
   };
 }
 
-void TFE_StartProfilerServer(TFE_Context* ctx, int port) {
-  auto server_thread = tensorflow::StartProfilerServer(&ctx->context, port);
-  ctx->context.AddChildThread(std::move(server_thread));
+TFE_ProfilerServerOptions* TFE_NewProfilerServerOptions() {
+  return new TFE_ProfilerServerOptions;
+}
+
+void TFE_ProfilerServerOptionsSetEagerContext(
+    TFE_ProfilerServerOptions* options, TFE_Context* ctx) {
+  options->profiler_context.eager_context = &ctx->context;
+}
+
+void TFE_StartProfilerServer(TFE_ProfilerServerOptions* options, int port) {
+  // Release child thread intentionally. The child thread can be terminate by
+  // terminating the main thread.
+  tensorflow::StartProfilerServer(&options->profiler_context, port).release();
 }
diff --git a/tensorflow/c/eager/c_api_experimental.h b/tensorflow/c/eager/c_api_experimental.h
index 8c85d0e516..ed65374438 100644
--- a/tensorflow/c/eager/c_api_experimental.h
+++ b/tensorflow/c/eager/c_api_experimental.h
@@ -42,14 +42,26 @@ TF_CAPI_EXPORT extern void TFE_ProfilerSerializeToString(TFE_Context* ctx,
                                                          TF_Buffer* buf,
                                                          TF_Status* status);
 
+typedef struct TFE_ProfilerServerOptions TFE_ProfilerServerOptions;
+
+// Return a new Profiler server options object.
+TF_CAPI_EXPORT extern TFE_ProfilerServerOptions* TFE_NewProfilerServerOptions(
+    void);
+
+// Set the eager context in TFE_ProfilerServerOptions
+TF_CAPI_EXPORT extern void TFE_ProfilerServerOptionsSetEagerContext(
+    TFE_ProfilerServerOptions* options, TFE_Context* ctx);
+
 // Start a profiler grpc server which listens to specified port. It will start
-// the server on its own thread. It can be shutdown by destructing TFE_Context.
-// Creating multiple profiler server is allowed. The service defined in
+// the server on its own thread. It can be shutdown by terminating tensorflow.
+// It can be used in both Eager mode and graph mode. Creating multiple profiler
+// server is allowed. The service defined in
 // tensorflow/contrib/tpu/profiler/tpu_profiler.proto. Please use
 // tensorflow/contrib/tpu/profiler/capture_tpu_profile to capture tracable
 // file following
 // https://cloud.google.com/tpu/docs/cloud-tpu-tools#capture_trace.
-TF_CAPI_EXPORT extern void TFE_StartProfilerServer(TFE_Context* ctx, int port);
+TF_CAPI_EXPORT extern void TFE_StartProfilerServer(
+    TFE_ProfilerServerOptions* options, int port);
 
 #ifdef __cplusplus
 } /* end extern "C" */
diff --git a/tensorflow/c/eager/c_api_internal.h b/tensorflow/c/eager/c_api_internal.h
index b70c0f1c11..b070e1cb1f 100644
--- a/tensorflow/c/eager/c_api_internal.h
+++ b/tensorflow/c/eager/c_api_internal.h
@@ -108,12 +108,19 @@ struct TFE_Op {
 };
 
 struct TFE_Profiler {
-  TFE_Profiler(TFE_Context* ctx)
-      : profiler(tensorflow::ProfilerSession::Create(&ctx->context)) {}
+  TFE_Profiler(TFE_Context* ctx) {
+    tensorflow::ProfilerContext profiler_context;
+    profiler_context.eager_context = &ctx->context;
+    profiler = tensorflow::ProfilerSession::Create(&profiler_context);
+  }
 
   std::unique_ptr<tensorflow::ProfilerSession> profiler;
 };
 
+struct TFE_ProfilerServerOptions {
+  tensorflow::ProfilerContext profiler_context;
+};
+
 namespace tensorflow {
 // Set an AttrValue on the op. Doesn't handle the list types.
 void SetOpAttrValueScalar(TFE_Context* ctx, TFE_Op* op,
diff --git a/tensorflow/core/profiler/lib/profiler_session.cc b/tensorflow/core/profiler/lib/profiler_session.cc
index 1eb9ed6a99..6ef779d5e1 100644
--- a/tensorflow/core/profiler/lib/profiler_session.cc
+++ b/tensorflow/core/profiler/lib/profiler_session.cc
@@ -82,7 +82,7 @@ void ConvertRunMetadataToTraceEvent(RunMetadata* run_metadata,
 }  // namespace
 
 /*static*/ std::unique_ptr<ProfilerSession> ProfilerSession::Create(
-    EagerContext* const context) {
+    ProfilerContext* const context) {
   return absl::WrapUnique(new ProfilerSession(context));
 }
 
@@ -110,13 +110,13 @@ Status ProfilerSession::SerializeToString(string* content) {
   return Status::OK();
 }
 
-ProfilerSession::ProfilerSession(EagerContext* const context)
+ProfilerSession::ProfilerSession(ProfilerContext* const context)
     : start_time_micros_(Env::Default()->NowNanos() / EnvTime::kMicrosToNanos) {
   LOG(INFO) << "Profile Session started.";
 
-  if (context != nullptr) {
-    profilers_.push_back(
-        tensorflow::profiler::runtime::EagerProfiler::Create(context));
+  if (context->eager_context != nullptr) {
+    profilers_.push_back(tensorflow::profiler::runtime::EagerProfiler::Create(
+        context->eager_context));
   }
   profilers_.push_back(tensorflow::profiler::gpu::Tracer::Create());
 
diff --git a/tensorflow/core/profiler/lib/profiler_session.h b/tensorflow/core/profiler/lib/profiler_session.h
index 1ab4825055..e764852ae3 100644
--- a/tensorflow/core/profiler/lib/profiler_session.h
+++ b/tensorflow/core/profiler/lib/profiler_session.h
@@ -22,6 +22,10 @@ limitations under the License.
 
 namespace tensorflow {
 
+struct ProfilerContext {
+  EagerContext* eager_context = nullptr;
+};
+
 // A profiler which will start profiling when creating the object and will stop
 // when either the object is destroyed or SerializedToString is called. It will
 // profile all operations run under the given EagerContext.
@@ -32,7 +36,8 @@ namespace tensorflow {
 class ProfilerSession {
  public:
   // Creates and ProfilerSession and starts profiling.
-  static std::unique_ptr<ProfilerSession> Create(EagerContext* const context);
+  static std::unique_ptr<ProfilerSession> Create(
+      ProfilerContext* const context);
 
   // Deletes an exsiting Profiler and enables starting a new one.
   ~ProfilerSession();
@@ -43,7 +48,7 @@ class ProfilerSession {
 
  private:
   // Constructs an instance of the class and starts profiling
-  explicit ProfilerSession(EagerContext* const context);
+  explicit ProfilerSession(ProfilerContext* const context);
 
   // Profiler is neither copyable or movable.
   ProfilerSession(const ProfilerSession&) = delete;
diff --git a/tensorflow/core/profiler/rpc/profiler_server.cc b/tensorflow/core/profiler/rpc/profiler_server.cc
index 835aa1e3f5..bef1e3c519 100644
--- a/tensorflow/core/profiler/rpc/profiler_server.cc
+++ b/tensorflow/core/profiler/rpc/profiler_server.cc
@@ -23,13 +23,16 @@ limitations under the License.
 
 namespace tensorflow {
 
-std::unique_ptr<Thread> StartProfilerServer(EagerContext* const eager_context,
-                                            int32 port) {
-  return WrapUnique(eager_context->TFEnv()->StartThread(
-      {}, "profiler_server", [eager_context, port]() {
+std::unique_ptr<Thread> StartProfilerServer(
+    ProfilerContext* const profiler_context, int32 port) {
+  Env* env = profiler_context->eager_context != nullptr
+                 ? profiler_context->eager_context->TFEnv()
+                 : Env::Default();
+  return WrapUnique(
+      env->StartThread({}, "profiler server", [profiler_context, port]() {
         string server_address = strings::StrCat("0.0.0.0:", port);
         std::unique_ptr<TPUProfiler::Service> service =
-            CreateProfilerService(eager_context);
+            CreateProfilerService(profiler_context);
         ::grpc::ServerBuilder builder;
         builder.AddListeningPort(server_address,
                                  ::grpc::InsecureServerCredentials());
diff --git a/tensorflow/core/profiler/rpc/profiler_server.h b/tensorflow/core/profiler/rpc/profiler_server.h
index ebefdd6cb0..4e8c715ac7 100644
--- a/tensorflow/core/profiler/rpc/profiler_server.h
+++ b/tensorflow/core/profiler/rpc/profiler_server.h
@@ -15,10 +15,11 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_PROFILER_RPC_PROFILER_SERVER_H_
 #define TENSORFLOW_CORE_PROFILER_RPC_PROFILER_SERVER_H_
 
-#include "tensorflow/core/common_runtime/eager/context.h"
+#include "tensorflow/core/profiler/lib/profiler_session.h"
 
 namespace tensorflow {
-std::unique_ptr<Thread> StartProfilerServer(EagerContext* const eager_context,
-                                            int32 port);
+
+std::unique_ptr<Thread> StartProfilerServer(
+    ProfilerContext* const profiler_context, int32 port);
 }  // namespace tensorflow
 #endif  // TENSORFLOW_CORE_PROFILER_RPC_PROFILER_SERVER_H_
diff --git a/tensorflow/core/profiler/rpc/profiler_service_impl.cc b/tensorflow/core/profiler/rpc/profiler_service_impl.cc
index 872ef2ada9..8704d9b64c 100644
--- a/tensorflow/core/profiler/rpc/profiler_service_impl.cc
+++ b/tensorflow/core/profiler/rpc/profiler_service_impl.cc
@@ -26,8 +26,8 @@ namespace {
 // TODO(fishx): Rename TPUProfiler to something more generic.
 class ProfilerServiceImpl : public TPUProfiler::Service {
  public:
-  explicit ProfilerServiceImpl(EagerContext* const eager_context)
-      : eager_context_(eager_context) {}
+  explicit ProfilerServiceImpl(ProfilerContext* const profiler_context)
+      : profiler_context_(*profiler_context) {}
   ~ProfilerServiceImpl() override {}
 
   ::grpc::Status Monitor(::grpc::ServerContext* ctx, const MonitorRequest* req,
@@ -39,13 +39,15 @@ class ProfilerServiceImpl : public TPUProfiler::Service {
                          ProfileResponse* response) override {
     LOG(INFO) << "Received a profile request.";
     std::unique_ptr<ProfilerSession> profiler =
-        ProfilerSession::Create(eager_context_);
+        ProfilerSession::Create(&profiler_context_);
     if (!profiler->Status().ok()) {
       return ::grpc::Status(::grpc::StatusCode::INTERNAL,
                             profiler->Status().error_message());
     }
 
-    Env* env = eager_context_->TFEnv();
+    Env* env = profiler_context_.eager_context != nullptr
+                   ? profiler_context_.eager_context->TFEnv()
+                   : Env::Default();
     for (size_t i = 0; i < req->duration_ms(); ++i) {
       env->SleepForMicroseconds(1000);
       if (ctx->IsCancelled()) {
@@ -62,13 +64,13 @@ class ProfilerServiceImpl : public TPUProfiler::Service {
   }
 
  private:
-  EagerContext* const eager_context_;
+  ProfilerContext profiler_context_;
 };
 }  // namespace
 
 std::unique_ptr<TPUProfiler::Service> CreateProfilerService(
-    EagerContext* const eager_context) {
-  return MakeUnique<ProfilerServiceImpl>(eager_context);
+    ProfilerContext* const profiler_context) {
+  return MakeUnique<ProfilerServiceImpl>(profiler_context);
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/profiler/rpc/profiler_service_impl.h b/tensorflow/core/profiler/rpc/profiler_service_impl.h
index 311a267451..9d27f71dfa 100644
--- a/tensorflow/core/profiler/rpc/profiler_service_impl.h
+++ b/tensorflow/core/profiler/rpc/profiler_service_impl.h
@@ -20,10 +20,12 @@ limitations under the License.
 #include "grpcpp/support/status.h"
 #include "tensorflow/contrib/tpu/profiler/tpu_profiler.grpc.pb.h"
 #include "tensorflow/core/common_runtime/eager/context.h"
+#include "tensorflow/core/profiler/lib/profiler_session.h"
 
 namespace tensorflow {
+
 std::unique_ptr<TPUProfiler::Service> CreateProfilerService(
-    EagerContext* const eager_context);
+    ProfilerContext* const profiler_context);
 }  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_PROFILER_RPC_PROFILER_SERVICE_IMPL_H_
diff --git a/tensorflow/python/eager/profiler.py b/tensorflow/python/eager/profiler.py
index a883122643..eaba0fa245 100644
--- a/tensorflow/python/eager/profiler.py
+++ b/tensorflow/python/eager/profiler.py
@@ -79,7 +79,8 @@ def start_profiler_server(port):
   """Start a profiler grpc server that listens to given port.
 
   The profiler server will keep the program running even the training finishes.
-  Please shutdown the server with CTRL-C. The service defined in
+  Please shutdown the server with CTRL-C. It can be used in both eager mode and
+  graph mode. The service defined in
   tensorflow/contrib/tpu/profiler/tpu_profiler.proto. Please use
   tensorflow/contrib/tpu/profiler/capture_tpu_profile to capture tracable
   file following https://cloud.google.com/tpu/docs/cloud-tpu-tools#capture_trace
@@ -87,9 +88,12 @@ def start_profiler_server(port):
   Args:
     port: port profiler server listens to.
   """
-  pywrap_tensorflow.TFE_StartProfilerServer(
-      context.context()._handle,  # pylint: disable=protected-access
-      port)
+  opts = pywrap_tensorflow.TFE_NewProfilerServerOptions()
+  if context.default_execution_mode == context.EAGER_MODE:
+    pywrap_tensorflow.TFE_ProfilerServerOptionsSetEagerContext(
+        opts,
+        context.context()._handle)  # pylint: disable=protected-access
+  pywrap_tensorflow.TFE_StartProfilerServer(opts, port)
 
 
 class Profiler(object):
diff --git a/tensorflow/python/pywrap_tfe.i b/tensorflow/python/pywrap_tfe.i
index dd74f12317..b19cc536b7 100755
--- a/tensorflow/python/pywrap_tfe.i
+++ b/tensorflow/python/pywrap_tfe.i
@@ -36,6 +36,8 @@ limitations under the License.
 %rename("%s") TFE_NewProfiler;
 %rename("%s") TFE_DeleteProfiler;
 %rename("%s") TFE_ProfilerSerializeToString;
+%rename("%s") TFE_NewProfilerServerOptions;
+%rename("%s") TFE_ProfilerServerOptionsSetEagerContext;
 %rename("%s") TFE_StartProfilerServer;
 %rename("%s") TFE_OpNameGetAttrType;
 %rename("%s") TFE_Py_InitEagerTensor;
-- 
GitLab


From 831f1d3aaef4cbce6ea61c229bd5531f35fc71dc Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Thu, 7 Feb 2019 18:26:48 -0800
Subject: [PATCH 2123/2345] Add int8 support for max pool.

PiperOrigin-RevId: 232984847
---
 .../internal/reference/integer_ops/pooling.h  | 58 +++++++++++++++++++
 tensorflow/lite/kernels/pooling.cc            | 38 ++++++++++--
 tensorflow/lite/kernels/pooling_test.cc       | 21 ++++++-
 tensorflow/lite/kernels/register.cc           |  4 +-
 tensorflow/lite/toco/tflite/operator.cc       |  5 ++
 5 files changed, 120 insertions(+), 6 deletions(-)

diff --git a/tensorflow/lite/kernels/internal/reference/integer_ops/pooling.h b/tensorflow/lite/kernels/internal/reference/integer_ops/pooling.h
index 22750bc91a..2762bec8e6 100644
--- a/tensorflow/lite/kernels/internal/reference/integer_ops/pooling.h
+++ b/tensorflow/lite/kernels/internal/reference/integer_ops/pooling.h
@@ -15,6 +15,7 @@ limitations under the License.
 #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_POOLING_H_
 #define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_POOLING_H_
 
+#include <limits>
 #include "tensorflow/lite/kernels/internal/common.h"
 
 namespace tflite {
@@ -77,6 +78,63 @@ inline void AveragePool(const PoolParams& params,
   }
 }
 
+inline void MaxPool(const PoolParams& params, const RuntimeShape& input_shape,
+                    const int8* input_data, const RuntimeShape& output_shape,
+                    int8* output_data) {
+  TFLITE_DCHECK_LE(params.quantized_activation_min,
+                   params.quantized_activation_max);
+  TFLITE_DCHECK_GE(params.quantized_activation_min,
+                   std::numeric_limits<int8_t>::min());
+  TFLITE_DCHECK_LE(params.quantized_activation_max,
+                   std::numeric_limits<int8_t>::max());
+  TFLITE_DCHECK_EQ(input_shape.DimensionsCount(), 4);
+  TFLITE_DCHECK_EQ(output_shape.DimensionsCount(), 4);
+  const int batches = MatchingDim(input_shape, 0, output_shape, 0);
+  const int depth = MatchingDim(input_shape, 3, output_shape, 3);
+  const int input_height = input_shape.Dims(1);
+  const int input_width = input_shape.Dims(2);
+  const int output_height = output_shape.Dims(1);
+  const int output_width = output_shape.Dims(2);
+  const int stride_height = params.stride_height;
+  const int stride_width = params.stride_width;
+  for (int batch = 0; batch < batches; ++batch) {
+    for (int out_y = 0; out_y < output_height; ++out_y) {
+      for (int out_x = 0; out_x < output_width; ++out_x) {
+        for (int channel = 0; channel < depth; ++channel) {
+          const int in_x_origin =
+              (out_x * stride_width) - params.padding_values.width;
+          const int in_y_origin =
+              (out_y * stride_height) - params.padding_values.height;
+          // Compute the boundaries of the filter region clamped so as to
+          // ensure that the filter window fits in the input array.
+          const int filter_x_start = std::max(0, -in_x_origin);
+          const int filter_x_end =
+              std::min(params.filter_width, input_width - in_x_origin);
+          const int filter_y_start = std::max(0, -in_y_origin);
+          const int filter_y_end =
+              std::min(params.filter_height, input_height - in_y_origin);
+          int8_t max = std::numeric_limits<int8_t>::lowest();
+          for (int filter_y = filter_y_start; filter_y < filter_y_end;
+               ++filter_y) {
+            for (int filter_x = filter_x_start; filter_x < filter_x_end;
+                 ++filter_x) {
+              const int in_x = in_x_origin + filter_x;
+              const int in_y = in_y_origin + filter_y;
+              max = std::max(
+                  max,
+                  input_data[Offset(input_shape, batch, in_y, in_x, channel)]);
+            }
+          }
+          max = std::max<int8_t>(max, params.quantized_activation_min);
+          max = std::min<int8_t>(max, params.quantized_activation_max);
+          output_data[Offset(output_shape, batch, out_y, out_x, channel)] =
+              static_cast<int8_t>(max);
+        }
+      }
+    }
+  }
+}
+
 }  // namespace reference_integer_ops
 }  // namespace tflite
 
diff --git a/tensorflow/lite/kernels/pooling.cc b/tensorflow/lite/kernels/pooling.cc
index e6155fcb8c..bdf736dcfb 100644
--- a/tensorflow/lite/kernels/pooling.cc
+++ b/tensorflow/lite/kernels/pooling.cc
@@ -226,9 +226,9 @@ void MaxEvalFloat(TfLiteContext* context, TfLiteNode* node,
 }
 
 template <KernelType kernel_type>
-void MaxEvalQuantized(TfLiteContext* context, TfLiteNode* node,
-                      TfLitePoolParams* params, OpData* data,
-                      const TfLiteTensor* input, TfLiteTensor* output) {
+void MaxEvalQuantizedUInt8(TfLiteContext* context, TfLiteNode* node,
+                           TfLitePoolParams* params, OpData* data,
+                           const TfLiteTensor* input, TfLiteTensor* output) {
   int32_t activation_min;
   int32_t activation_max;
   CalculateActivationRangeUint8(params->activation, output, &activation_min,
@@ -254,6 +254,31 @@ void MaxEvalQuantized(TfLiteContext* context, TfLiteNode* node,
 #undef TF_LITE_MAX_POOL
 }
 
+template <KernelType kernel_type>
+void MaxEvalQuantizedInt8(TfLiteContext* context, TfLiteNode* node,
+                          TfLitePoolParams* params, OpData* data,
+                          const TfLiteTensor* input, TfLiteTensor* output) {
+  int32_t activation_min;
+  int32_t activation_max;
+  CalculateActivationRangeInt8(params->activation, output, &activation_min,
+                               &activation_max);
+#define TF_LITE_MAX_POOL(type)                                        \
+  tflite::PoolParams op_params;                                       \
+  op_params.stride_height = params->stride_height;                    \
+  op_params.stride_width = params->stride_width;                      \
+  op_params.filter_height = params->filter_height;                    \
+  op_params.filter_width = params->filter_width;                      \
+  op_params.padding_values.height = data->padding.height;             \
+  op_params.padding_values.width = data->padding.width;               \
+  op_params.quantized_activation_min = activation_min;                \
+  op_params.quantized_activation_max = activation_max;                \
+  type::MaxPool(op_params, GetTensorShape(input),                     \
+                GetTensorData<int8_t>(input), GetTensorShape(output), \
+                GetTensorData<int8_t>(output))
+  TF_LITE_MAX_POOL(reference_integer_ops);
+#undef TF_LITE_MAX_POOL
+}
+
 template <KernelType kernel_type>
 void L2EvalFloat(TfLiteContext* context, TfLiteNode* node,
                  TfLitePoolParams* params, OpData* data,
@@ -321,7 +346,12 @@ TfLiteStatus MaxEval(TfLiteContext* context, TfLiteNode* node) {
       MaxEvalFloat<kernel_type>(context, node, params, data, input, output);
       break;
     case kTfLiteUInt8:
-      MaxEvalQuantized<kernel_type>(context, node, params, data, input, output);
+      MaxEvalQuantizedUInt8<kernel_type>(context, node, params, data, input,
+                                         output);
+      break;
+    case kTfLiteInt8:
+      MaxEvalQuantizedInt8<kernel_type>(context, node, params, data, input,
+                                        output);
       break;
     default:
       context->ReportError(context, "Type %d not currently supported.",
diff --git a/tensorflow/lite/kernels/pooling_test.cc b/tensorflow/lite/kernels/pooling_test.cc
index e1b7934011..3e19bd2f64 100644
--- a/tensorflow/lite/kernels/pooling_test.cc
+++ b/tensorflow/lite/kernels/pooling_test.cc
@@ -204,7 +204,7 @@ TEST(FloatPoolingOpTest, MaxPool) {
   EXPECT_THAT(m.GetOutput(), ElementsAreArray({6, 10}));
 }
 
-TEST(QuantizedPoolingOpTest, MaxPool) {
+TEST(QuantizedUInt8PoolingOpTest, MaxPool) {
   // Choose the input ranges carefully so that the dequantized output matches
   // the results of the float model above.
   QuantizedPoolingOpModel m(
@@ -223,6 +223,25 @@ TEST(QuantizedPoolingOpTest, MaxPool) {
   EXPECT_THAT(m.GetOutput(), ElementsAreArray({96, 160}));
 }
 
+TEST(QuantizedInt8PoolingOpTest, MaxPool) {
+  // Choose the input ranges carefully so that the dequantized output matches
+  // the results of the float model above.
+  SymmetricQuantizedPoolingOpModel m(
+      BuiltinOperator_MAX_POOL_2D,
+      /*input=*/{TensorType_INT8, {1, 2, 4, 1}, 0, 15.9375},
+      /*filter_width=*/2, /*filter_height=*/2,
+      /*output=*/{TensorType_INT8, {}, 0, 15.9375});
+  m.SetInput({
+      0, -6, 2, 4,   //
+      3, 2, -10, 7,  //
+  });
+  m.Invoke();
+
+  EXPECT_THAT(m.GetDequantizedOutput(),
+              ElementsAreArray(ArrayFloatNear({3, 7})));
+  EXPECT_THAT(m.GetOutput(), ElementsAreArray({-80, -16}));
+}
+
 TEST(FloatPoolingOpTest, L2Pool) {
   FloatPoolingOpModel m(BuiltinOperator_L2_POOL_2D,
                         /*input=*/{TensorType_FLOAT32, {1, 2, 4, 1}},
diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index fec8e63ee0..355d66f64d 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -170,7 +170,9 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_AVERAGE_POOL_2D, Register_AVERAGE_POOL_2D(),
              /* min_version */ 1,
              /* max_version */ 2);
-  AddBuiltin(BuiltinOperator_MAX_POOL_2D, Register_MAX_POOL_2D());
+  AddBuiltin(BuiltinOperator_MAX_POOL_2D, Register_MAX_POOL_2D(),
+             /* min_version */ 1,
+             /* max_version */ 2);
   AddBuiltin(BuiltinOperator_L2_POOL_2D, Register_L2_POOL_2D());
   AddBuiltin(BuiltinOperator_CONV_2D, Register_CONV_2D(),
              /* min_version */ 1,
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index fe40e71847..8a9cd7fded 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -643,6 +643,11 @@ class MaxPool : public BuiltinOperator<MaxPoolOperator, ::tflite::Pool2DOptions,
   }
 
   int GetVersion(const OperatorSignature& op_signature) const override {
+    const string& input_name = op_signature.op->inputs[0];
+    const Array& input_array = op_signature.model->GetArray(input_name);
+    if (input_array.data_type == ArrayDataType::kInt8) {
+      return 2;
+    }
     return 1;
   }
 };
-- 
GitLab


From c0eab335f9c872502a159577de03079b22751767 Mon Sep 17 00:00:00 2001
From: nammbash <niroopshankar.ammbashankar@intel.com>
Date: Thu, 7 Feb 2019 18:47:57 -0800
Subject: [PATCH 2124/2345] build file buildifier formatter

---
 tensorflow/core/kernels/BUILD | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index ee54d9c578..c90fc37644 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -6997,7 +6997,8 @@ tf_mkl_kernel_library(
         "no_op.h",
         "reference_gemm.h",
     ],
-    deps = if_mkl([
+    deps = if_mkl(
+        [
             ":concat_lib_hdrs",
             ":conv_ops",
             ":cwise_op",
@@ -7016,7 +7017,7 @@ tf_mkl_kernel_library(
             "//third_party/eigen3",
             "//third_party/mkl:intel_binary_blob",
             "@gemmlowp",
-            "@mkl_dnn",            
+            "@mkl_dnn",
         ],
     ),
 )
@@ -7025,6 +7026,7 @@ tf_cc_test_mkl(
     name = "mkl_requantize_ops_test",
     size = "small",
     srcs = ["mkl_requantize_ops_test.cc"],
+    tags = ["no_mac"],  #TODO(penporn): Re-enable the test on MacOS.
     deps = [
         ":mkl_requantize_ops",
         ":ops_testutil",
-- 
GitLab


From 9bb620cbf9a9246bb1c3eb7ce797f7d2d525887c Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Thu, 7 Feb 2019 18:54:28 -0800
Subject: [PATCH 2125/2345] [XLA] Add a TriangularSolve HLO.

Previously TriangularSolve() existed in the XLA client library, but was lowered into other HLO ops immediately, preventing us from well-tuned existing BLAS implementations. This change adds a first-class HLO for TriangularSolve.

The API of TriangularSolve is chose to match the BLAS TRSM API closely.

On the CPU and interpreter backends, the TriangularSolve HLO is immediately expanded to a Call operator that runs the same computation the existing client library would have built. With some cunning, we are able to use XlaBuilder inside a lowering pass, allowing us to keep using the much simpler XlaBuilder API to express the triangular solve computation.

Adds a generic OpExpander pass superclass, and refactors GatherExpander to use it. OpExpander is used as the superclass of the new TriangularSolveExpander.

On GPU, add direct implementation of TriangularSolve in terms of the cuBlas TRSM implementation.

PiperOrigin-RevId: 232987494
---
 tensorflow/compiler/tf2xla/kernels/BUILD      |   1 -
 .../kernels/matrix_triangular_solve_op.cc     |   9 +-
 tensorflow/compiler/xla/client/lib/BUILD      |  46 ------
 .../compiler/xla/client/lib/cholesky.cc       |  14 +-
 .../xla/client/lib/triangular_solve.h         |  69 --------
 tensorflow/compiler/xla/client/xla_builder.cc |  23 +++
 tensorflow/compiler/xla/client/xla_builder.h  |  30 +++-
 tensorflow/compiler/xla/python/BUILD          |   1 -
 .../xla/python/local_computation_builder.cc   |  14 +-
 .../xla/python/local_computation_builder.h    |   6 +-
 tensorflow/compiler/xla/python/xla_client.py  |  10 +-
 tensorflow/compiler/xla/service/BUILD         |  37 +++++
 tensorflow/compiler/xla/service/cpu/BUILD     |   1 +
 .../compiler/xla/service/cpu/cpu_compiler.cc  |   3 +
 .../compiler/xla/service/dfs_hlo_visitor.h    |   1 +
 .../service/dfs_hlo_visitor_with_default.h    |   3 +
 .../compiler/xla/service/gather_expander.cc   |  28 +---
 .../compiler/xla/service/gather_expander.h    |  10 +-
 tensorflow/compiler/xla/service/gpu/BUILD     |   4 +
 .../xla/service/gpu/gpu_layout_assignment.cc  |  26 +++
 .../xla/service/gpu/ir_emitter_unnested.cc    |  95 ++++++++---
 .../xla/service/gpu/ir_emitter_unnested.h     |   4 +
 tensorflow/compiler/xla/service/gpu/thunk.cc  |   2 +
 tensorflow/compiler/xla/service/gpu/thunk.h   |   1 +
 .../xla/service/gpu/triangular_solve_thunk.cc | 149 ++++++++++++++++++
 .../xla/service/gpu/triangular_solve_thunk.h  |  75 +++++++++
 tensorflow/compiler/xla/service/hlo.proto     |   5 +-
 .../compiler/xla/service/hlo_cost_analysis.cc |  15 ++
 .../compiler/xla/service/hlo_cost_analysis.h  |   1 +
 .../compiler/xla/service/hlo_graph_dumper.cc  |   1 +
 .../compiler/xla/service/hlo_instruction.cc   |  23 +++
 .../compiler/xla/service/hlo_instruction.h    |   7 +
 .../compiler/xla/service/hlo_instructions.cc  |  48 ++++++
 .../compiler/xla/service/hlo_instructions.h   |  32 +++-
 tensorflow/compiler/xla/service/hlo_opcode.h  |   1 +
 tensorflow/compiler/xla/service/hlo_parser.cc |  59 +++++++
 .../compiler/xla/service/hlo_verifier.cc      |   9 ++
 .../compiler/xla/service/hlo_verifier.h       |   1 +
 .../xla/service/instruction_fusion.cc         |   1 +
 .../compiler/xla/service/interpreter/BUILD    |   1 +
 .../xla/service/interpreter/compiler.cc       |   2 +
 .../compiler/xla/service/layout_assignment.cc |   1 +
 .../compiler/xla/service/op_expander_pass.cc  |  44 ++++++
 .../compiler/xla/service/op_expander_pass.h   |  43 +++++
 .../compiler/xla/service/shape_inference.cc   |  49 +++++-
 .../compiler/xla/service/shape_inference.h    |   4 +
 .../triangular_solve_expander.cc}             |  73 ++++++++-
 .../xla/service/triangular_solve_expander.h   |  43 +++++
 .../xla/service/tuple_points_to_analysis.cc   |  10 ++
 tensorflow/compiler/xla/tests/BUILD           |  23 +++
 .../lib => tests}/triangular_solve_test.cc    |  70 ++++----
 tensorflow/compiler/xla/xla_data.proto        |  20 +++
 52 files changed, 1012 insertions(+), 236 deletions(-)
 delete mode 100644 tensorflow/compiler/xla/client/lib/triangular_solve.h
 create mode 100644 tensorflow/compiler/xla/service/gpu/triangular_solve_thunk.cc
 create mode 100644 tensorflow/compiler/xla/service/gpu/triangular_solve_thunk.h
 create mode 100644 tensorflow/compiler/xla/service/op_expander_pass.cc
 create mode 100644 tensorflow/compiler/xla/service/op_expander_pass.h
 rename tensorflow/compiler/xla/{client/lib/triangular_solve.cc => service/triangular_solve_expander.cc} (85%)
 create mode 100644 tensorflow/compiler/xla/service/triangular_solve_expander.h
 rename tensorflow/compiler/xla/{client/lib => tests}/triangular_solve_test.cc (88%)

diff --git a/tensorflow/compiler/tf2xla/kernels/BUILD b/tensorflow/compiler/tf2xla/kernels/BUILD
index 69353fe87d..b3f050c52b 100644
--- a/tensorflow/compiler/tf2xla/kernels/BUILD
+++ b/tensorflow/compiler/tf2xla/kernels/BUILD
@@ -144,7 +144,6 @@ tf_kernel_library(
         "//tensorflow/compiler/xla/client/lib:qr",
         "//tensorflow/compiler/xla/client/lib:quantize",
         "//tensorflow/compiler/xla/client/lib:sorting",
-        "//tensorflow/compiler/xla/client/lib:triangular_solve",
         "//tensorflow/core:bitwise_ops_op_lib",
         "//tensorflow/core:control_flow_ops_op_lib",
         "//tensorflow/core:data_flow_ops_op_lib",
diff --git a/tensorflow/compiler/tf2xla/kernels/matrix_triangular_solve_op.cc b/tensorflow/compiler/tf2xla/kernels/matrix_triangular_solve_op.cc
index b3224953d5..5a6569c895 100644
--- a/tensorflow/compiler/tf2xla/kernels/matrix_triangular_solve_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/matrix_triangular_solve_op.cc
@@ -15,7 +15,8 @@ limitations under the License.
 
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
-#include "tensorflow/compiler/xla/client/lib/triangular_solve.h"
+#include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
 
 namespace tensorflow {
 namespace {
@@ -31,8 +32,10 @@ class MatrixTriangularSolveOp : public XlaOpKernel {
   void Compile(XlaOpKernelContext* ctx) override {
     auto result = xla::TriangularSolve(
         ctx->Input(0), ctx->Input(1), /*left_side=*/true,
-        /*lower=*/lower_, /*transpose_a=*/adjoint_, /*conjugate_a=*/adjoint_,
-        /*unit_diagonal=*/false);
+        /*lower=*/lower_, /*unit_diagonal=*/false,
+        /*transpose_a=*/
+        adjoint_ ? xla::TriangularSolveOptions::ADJOINT
+                 : xla::TriangularSolveOptions::NO_TRANSPOSE);
     ctx->SetOutput(0, result);
   }
 
diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD
index 6be8154221..84b542d3d3 100644
--- a/tensorflow/compiler/xla/client/lib/BUILD
+++ b/tensorflow/compiler/xla/client/lib/BUILD
@@ -65,7 +65,6 @@ cc_library(
         "//tensorflow/compiler/xla/client/lib:loops",
         "//tensorflow/compiler/xla/client/lib:matrix",
         "//tensorflow/compiler/xla/client/lib:slicing",
-        "//tensorflow/compiler/xla/client/lib:triangular_solve",
         "//tensorflow/core:lib",
     ],
 )
@@ -450,51 +449,6 @@ cc_library(
     ],
 )
 
-cc_library(
-    name = "triangular_solve",
-    srcs = ["triangular_solve.cc"],
-    hdrs = ["triangular_solve.h"],
-    deps = [
-        "//tensorflow/compiler/xla:literal",
-        "//tensorflow/compiler/xla:shape_util",
-        "//tensorflow/compiler/xla:status_macros",
-        "//tensorflow/compiler/xla:statusor",
-        "//tensorflow/compiler/xla:util",
-        "//tensorflow/compiler/xla:xla_data_proto",
-        "//tensorflow/compiler/xla/client:xla_builder",
-        "//tensorflow/compiler/xla/client:xla_computation",
-        "//tensorflow/compiler/xla/client/lib:constants",
-        "//tensorflow/compiler/xla/client/lib:math",
-        "//tensorflow/compiler/xla/client/lib:matrix",
-        "//tensorflow/compiler/xla/client/lib:slicing",
-        "//tensorflow/core:lib",
-    ],
-)
-
-xla_test(
-    name = "triangular_solve_test",
-    srcs = ["triangular_solve_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-        "noasan",  # sometimes times out, http://b/78650012
-    ],
-    deps = [
-        ":math",
-        ":matrix",
-        ":triangular_solve",
-        "//tensorflow/compiler/xla:array2d",
-        "//tensorflow/compiler/xla:literal",
-        "//tensorflow/compiler/xla:statusor",
-        "//tensorflow/compiler/xla:test",
-        "//tensorflow/compiler/xla:types",
-        "//tensorflow/compiler/xla/client:xla_builder",
-        "//tensorflow/compiler/xla/tests:client_library_test_base",
-        "//tensorflow/compiler/xla/tests:literal_test_util",
-        "//tensorflow/compiler/xla/tests:xla_internal_test_main",
-        "//tensorflow/core:test",
-    ],
-)
-
 cc_library(
     name = "self_adjoint_eigen",
     srcs = ["self_adjoint_eigen.cc"],
diff --git a/tensorflow/compiler/xla/client/lib/cholesky.cc b/tensorflow/compiler/xla/client/lib/cholesky.cc
index 4578fff9d1..bb41f9932d 100644
--- a/tensorflow/compiler/xla/client/lib/cholesky.cc
+++ b/tensorflow/compiler/xla/client/lib/cholesky.cc
@@ -23,7 +23,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/lib/math.h"
 #include "tensorflow/compiler/xla/client/lib/matrix.h"
 #include "tensorflow/compiler/xla/client/lib/slicing.h"
-#include "tensorflow/compiler/xla/client/lib/triangular_solve.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/primitive_util.h"
@@ -194,13 +193,12 @@ XlaOp Cholesky(XlaOp a, int64 block_size,
         // l[i+k:, i:i+k] =
         //     trsm_right_transpose(l[i:i+k, i:i+k], a[i+k:, i:i+k])
         auto panel = SliceInMinorDims(a, {i + k, i}, {n, i + k});
-        auto update = TriangularSolve(factorized, panel,
-                                      /*left_side=*/false,
-                                      /*lower=*/true,
-                                      /*transpose_a=*/true,
-                                      /*conjugate_a=*/false,
-                                      /*unit_diagonal=*/false,
-                                      /*block_size=*/block_size);
+        auto update =
+            TriangularSolve(factorized, panel,
+                            /*left_side=*/false,
+                            /*lower=*/true,
+                            /*unit_diagonal=*/false,
+                            /*transpose_a=*/TriangularSolveOptions::TRANSPOSE);
         l = UpdateSliceInMinorDims(l, update, {i + k, i});
       }
     }
diff --git a/tensorflow/compiler/xla/client/lib/triangular_solve.h b/tensorflow/compiler/xla/client/lib/triangular_solve.h
deleted file mode 100644
index b87ef7288f..0000000000
--- a/tensorflow/compiler/xla/client/lib/triangular_solve.h
+++ /dev/null
@@ -1,69 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_COMPILER_XLA_CLIENT_LIB_TRIANGULAR_SOLVE_H_
-#define TENSORFLOW_COMPILER_XLA_CLIENT_LIB_TRIANGULAR_SOLVE_H_
-
-#include "tensorflow/compiler/xla/client/xla_builder.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
-
-namespace xla {
-
-// Solves systems of linear equations with lower or upper triangular coefficient
-// matrices by forward- or back-substitution. Broadcasting along leading
-// dimensions, this routine solves one of the matrix systems
-//   `op(a) * x = b`,  or `x * op(a) = b`,
-// for the variable `x` given `a` and `b`, where `op(a)` is either
-//   `op(a) = a`,  or `op(a) = transpose(a)`,  or `op(a) = conj(transpose(a))`.
-// That is, the innermost matrices in the output satisfy a scalar system
-// depending on the value of the value of (left_side, transpose_a, conjugate_a)
-// according to:
-//   (F, F, F) => `output[..., i, k]  a[..., k, j] = b[..., i, j]`,
-//   (F, F, T) => `output[..., i, k] a*[..., k, j] = b[..., i, j]`,
-//   (F, T, F) => `output[..., i, k]  a[..., j, k] = b[..., i, j]`,
-//   (F, T, T) => `output[..., i, k] a*[..., j, k] = b[..., i, j]`,
-//   (T, F, F) => ` a[..., i, k] output[..., k, j] = b[..., i, j]`,
-//   (T, F, T) => `a*[..., i, k] output[..., k, j] = b[..., i, j]`,
-//   (T, T, F) => ` a[..., i, k] output[..., j, k] = b[..., i, j]`,
-//   (T, T, T) => `a*[..., i, k] output[..., j, k] = b[..., i, j]`,
-// where * denotes complex conjugation and where the index `k` is summed over.
-//
-// `a` is a tensor of shape `[..., M, M]` whose innermost 2 dimensions form
-// square matrices. If lower is true (false), then the strictly upper (lower)
-// triangular part of each innermost matrix in `a` is assumed to be zero and is
-// not accessed.
-// `b` is a tensor of shape `[..., M, K]` if left_side is true, otherwise a
-// tensor of shape `[..., K, M]`.
-// `left_side` is a boolean, indicating whether to solve a system of the form
-// op(a) * x = b (true) or x * op(a) = b (false).
-// `lower` is a boolean, indicating whether the argument `a` is lower-triangular
-// (true) or upper-triangular (false).
-// `transpose_a` is a boolean indicating whether the matrix `a` is transposed.
-// `conjugate_a` is a boolean indicating whether the entries of `a` are complex
-// conjugated (independently of whether they are transposed), so that when both
-// transpose_a and conjugate_a are true the effect is a Hermitian adjoint.
-// If `unit_diagonal` elements on the matrix diagonal are assumed to be '1' and
-// are not read by the triangular solve..
-//
-// Uses a blocked algorithm if `block_size` is > 1; if block_size == 1 then no
-// blocking is used.
-XlaOp TriangularSolve(
-    XlaOp a, XlaOp b, bool left_side, bool lower, bool transpose_a,
-    bool conjugate_a, bool unit_diagonal, int64 block_size = 128,
-    PrecisionConfig::Precision precision = PrecisionConfig::HIGHEST);
-
-}  // namespace xla
-
-#endif  // TENSORFLOW_COMPILER_XLA_CLIENT_LIB_TRIANGULAR_SOLVE_H_
diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc
index 29b71bea5d..82375f16fd 100644
--- a/tensorflow/compiler/xla/client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_builder.cc
@@ -2883,6 +2883,29 @@ XlaOp Fft(const XlaOp& operand, FftType fft_type,
   return operand.builder()->Fft(operand, fft_type, fft_length);
 }
 
+XlaOp TriangularSolve(XlaOp a, XlaOp b, bool left_side, bool lower,
+                      bool unit_diagonal,
+                      TriangularSolveOptions::Transpose transpose_a) {
+  XlaBuilder* builder = a.builder();
+  return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    HloInstructionProto instr;
+    TF_ASSIGN_OR_RETURN(const Shape& a_shape, builder->GetShape(a));
+    TF_ASSIGN_OR_RETURN(const Shape& b_shape, builder->GetShape(b));
+    xla::TriangularSolveOptions& options =
+        *instr.mutable_triangular_solve_options();
+    options.set_left_side(left_side);
+    options.set_lower(lower);
+    options.set_unit_diagonal(unit_diagonal);
+    options.set_transpose_a(transpose_a);
+    TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferTriangularSolveShape(
+                                         a_shape, b_shape, options));
+    *instr.mutable_shape() = shape.ToProto();
+
+    return builder->AddInstruction(std::move(instr),
+                                   HloOpcode::kTriangularSolve, {a, b});
+  });
+}
+
 XlaOp Infeed(XlaBuilder* builder, const Shape& shape, const string& config) {
   return builder->Infeed(shape, config);
 }
diff --git a/tensorflow/compiler/xla/client/xla_builder.h b/tensorflow/compiler/xla/client/xla_builder.h
index cd18a9fab0..384ccfa25a 100644
--- a/tensorflow/compiler/xla/client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_builder.h
@@ -429,7 +429,6 @@ class XlaBuilder {
       const Shape& shape_with_layout, const string& opaque,
       absl::optional<absl::Span<const Shape>> operand_shapes_with_layout);
 
-
   XlaOp Reduce(const XlaOp& operand, const XlaOp& init_value,
                const XlaComputation& computation,
                absl::Span<const int64> dimensions_to_reduce);
@@ -792,6 +791,9 @@ class XlaBuilder {
       const PrecisionConfig* precision_config);
   friend XlaOp Fft(const XlaOp& operand, FftType fft_type,
                    absl::Span<const int64> fft_length);
+  friend XlaOp TriangularSolve(XlaOp a, XlaOp b, bool left_side, bool lower,
+                               bool unit_diagonal,
+                               TriangularSolveOptions::Transpose transpose_a);
   friend XlaOp Infeed(XlaBuilder* builder, const Shape& shape,
                       const string& config);
   friend void Outfeed(const XlaOp& operand, const Shape& shape_with_layout,
@@ -1312,6 +1314,32 @@ XlaOp ConvGeneralDilated(const XlaOp& lhs, const XlaOp& rhs,
 XlaOp Fft(const XlaOp& operand, FftType fft_type,
           absl::Span<const int64> fft_length);
 
+// Solves systems of linear equations with lower or upper triangular coefficient
+// matrices by forward- or back-substitution. Broadcasting along leading
+// dimensions, this routine solves for x in one of the matrix systems
+//   `op(a) * x = b`,  or `x * op(a) = b`,
+// for the variable `x` given `a` and `b`, where `op(a)` is either
+//   `op(a) = a`,  or `op(a) = transpose(a)`,  or `op(a) = conj(transpose(a))`.
+//
+// * `a` is a tensor of shape `[..., M, M]` whose innermost 2 dimensions form
+//   square matrices. If `lower` is true (false), then the strictly upper
+//   (lower) triangular part of each innermost matrix in `a` is assumed to be
+//   zero and is not accessed.
+// * `b` is a tensor of shape `[..., M, K]` if `left_side` is true, otherwise a
+//   tensor of shape `[..., K, M]`.
+// * `left_side` is a boolean, indicating whether to solve a system of the form
+//   op(a) * x = b (true) or x * op(a) = b (false).
+// * `lower` is a boolean, indicating whether the argument `a` is
+// lower-triangular
+//   (true) or upper-triangular (false).
+// * If `unit_diagonal` is true, the diagonal elements of `a` are assumed to be
+//   1 and not accessed.
+// * `transpose_a` indicates which function `op` we use to transform the tensor
+//   `a`: the identity function, transpose(a), or conjugate(transpose(a))
+XlaOp TriangularSolve(XlaOp a, XlaOp b, bool left_side, bool lower,
+                      bool unit_diagonal,
+                      TriangularSolveOptions::Transpose transpose_a);
+
 // Enqueues an infeed instruction onto the computation, which writes data of
 // the given shape to the infeed buffer of the device.
 XlaOp Infeed(XlaBuilder* builder, const Shape& shape,
diff --git a/tensorflow/compiler/xla/python/BUILD b/tensorflow/compiler/xla/python/BUILD
index 4afb21d5c8..f7e2d26b7a 100644
--- a/tensorflow/compiler/xla/python/BUILD
+++ b/tensorflow/compiler/xla/python/BUILD
@@ -77,7 +77,6 @@ cc_library(
         "//tensorflow/compiler/xla/client/lib:cholesky",
         "//tensorflow/compiler/xla/client/lib:math",
         "//tensorflow/compiler/xla/client/lib:qr",
-        "//tensorflow/compiler/xla/client/lib:triangular_solve",
         "//tensorflow/compiler/xla/service:platform_util",
         "//tensorflow/compiler/xla/service:shaped_buffer",
         "//tensorflow/compiler/xla/service/cpu:custom_call_target_registry",
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.cc b/tensorflow/compiler/xla/python/local_computation_builder.cc
index 08913809c4..24138a173d 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.cc
+++ b/tensorflow/compiler/xla/python/local_computation_builder.cc
@@ -27,7 +27,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/lib/cholesky.h"
 #include "tensorflow/compiler/xla/client/lib/math.h"
 #include "tensorflow/compiler/xla/client/lib/qr.h"
-#include "tensorflow/compiler/xla/client/lib/triangular_solve.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/client/xla_computation.h"
 #include "tensorflow/compiler/xla/executable_run_options.h"
@@ -928,11 +927,14 @@ LocalOp LocalComputationBuilder::QR(const LocalOp& a, bool full_matrices) {
   });
 }
 
-LocalOp LocalComputationBuilder::TriangularSolve(
-    const LocalOp& a, const LocalOp& b, bool left_side, bool lower,
-    bool transpose_a, bool conjugate_a, bool unit_diagonal) {
-  return xla::TriangularSolve(a.op(), b.op(), left_side, lower, transpose_a,
-                              conjugate_a, unit_diagonal);
+LocalOp LocalComputationBuilder::TriangularSolve(const LocalOp& a,
+                                                 const LocalOp& b,
+                                                 bool left_side, bool lower,
+                                                 bool unit_diagonal,
+                                                 int transpose_a) {
+  return xla::TriangularSolve(
+      a.op(), b.op(), left_side, lower, unit_diagonal,
+      xla::TriangularSolveOptions::Transpose(transpose_a));
 }
 
 LocalOp LocalComputationBuilder::Gather(
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.h b/tensorflow/compiler/xla/python/local_computation_builder.h
index 5e65ecfeef..bc8b7e610c 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.h
+++ b/tensorflow/compiler/xla/python/local_computation_builder.h
@@ -424,9 +424,11 @@ class LocalComputationBuilder {
 
   LocalOp Cholesky(const LocalOp& a);
 
+  // `transpose_a` is the integer value of a TriangularSolveOptions::Transpose
+  // enum. We use an integer here so we don't have to teach SWIG about the
+  // enum.
   LocalOp TriangularSolve(const LocalOp& a, const LocalOp& b, bool left_side,
-                          bool lower, bool transpose_a, bool conjugate_a,
-                          bool unit_diagonal);
+                          bool lower, bool unit_diagonal, int transpose_a);
 
   LocalOp Gather(const LocalOp& input, const LocalOp& start_indices,
                  const GatherDimensionNumbers& dimension_numbers,
diff --git a/tensorflow/compiler/xla/python/xla_client.py b/tensorflow/compiler/xla/python/xla_client.py
index 49f1f33c11..c7afed300b 100644
--- a/tensorflow/compiler/xla/python/xla_client.py
+++ b/tensorflow/compiler/xla/python/xla_client.py
@@ -1635,8 +1635,14 @@ class ComputationBuilder(object):
                       conjugate_a=False,
                       unit_diagonal=False):
     """Enqueues a triangular-solve operation onto the computation."""
-    return self._client.TriangularSolve(a, b, left_side, lower, transpose_a,
-                                        conjugate_a, unit_diagonal)
+    if not transpose_a:
+      transpose = 1
+      if conjugate_a:
+        a = self.Conj(a)
+    else:
+      transpose = 3 if conjugate_a else 2
+    return self._client.TriangularSolve(a, b, left_side, lower, unit_diagonal,
+                                        transpose)
 
   def Gather(self, a, start_indices, dimension_numbers, slice_sizes):
     """Enqueues a Gather operation onto the computation."""
diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index f5fbe642d0..ecaf99311b 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -1514,6 +1514,20 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "op_expander_pass",
+    srcs = ["op_expander_pass.cc"],
+    hdrs = ["op_expander_pass.h"],
+    deps = [
+        ":hlo",
+        ":hlo_creation_utils",
+        ":hlo_pass",
+        "//tensorflow/compiler/xla:statusor",
+        "//tensorflow/compiler/xla:util",
+        "@com_google_absl//absl/algorithm:container",
+    ],
+)
+
 cc_library(
     name = "gather_expander",
     srcs = ["gather_expander.cc"],
@@ -1522,6 +1536,7 @@ cc_library(
         ":hlo",
         ":hlo_creation_utils",
         ":hlo_pass",
+        ":op_expander_pass",
         ":while_util",
         "//tensorflow/compiler/xla:literal_util",
         "//tensorflow/compiler/xla:statusor",
@@ -1545,6 +1560,28 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "triangular_solve_expander",
+    srcs = ["triangular_solve_expander.cc"],
+    hdrs = ["triangular_solve_expander.h"],
+    deps = [
+        ":op_expander_pass",
+        "//tensorflow/compiler/xla:literal",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:status_macros",
+        "//tensorflow/compiler/xla:statusor",
+        "//tensorflow/compiler/xla:util",
+        "//tensorflow/compiler/xla/client:xla_builder",
+        "//tensorflow/compiler/xla/client:xla_computation",
+        "//tensorflow/compiler/xla/client/lib:constants",
+        "//tensorflow/compiler/xla/client/lib:math",
+        "//tensorflow/compiler/xla/client/lib:matrix",
+        "//tensorflow/compiler/xla/client/lib:slicing",
+        "//tensorflow/core:lib",
+        "@com_google_absl//absl/container:flat_hash_map",
+    ],
+)
+
 tf_cc_test(
     name = "batchnorm_expander_test",
     size = "small",
diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index d4535b204d..42672bc387 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -136,6 +136,7 @@ cc_library(
         "//tensorflow/compiler/xla/service:reshape_mover",
         "//tensorflow/compiler/xla/service:sort_simplifier",
         "//tensorflow/compiler/xla/service:transpose_folding",
+        "//tensorflow/compiler/xla/service:triangular_solve_expander",
         "//tensorflow/compiler/xla/service:tuple_simplifier",
         "//tensorflow/compiler/xla/service:while_loop_constant_sinking",
         "//tensorflow/compiler/xla/service:while_loop_invariant_code_motion",
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
index eafda68510..3676de56a3 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
@@ -95,6 +95,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/scatter_expander.h"
 #include "tensorflow/compiler/xla/service/sort_simplifier.h"
 #include "tensorflow/compiler/xla/service/transpose_folding.h"
+#include "tensorflow/compiler/xla/service/triangular_solve_expander.h"
 #include "tensorflow/compiler/xla/service/tuple_simplifier.h"
 #include "tensorflow/compiler/xla/service/while_loop_constant_sinking.h"
 #include "tensorflow/compiler/xla/service/while_loop_invariant_code_motion.h"
@@ -255,6 +256,8 @@ Status CpuCompiler::RunHloPassesThroughLayoutAssn(
 
   pipeline.AddPass<MapInliner>();
 
+  pipeline.AddPass<TriangularSolveExpander>();
+
   // TODO(b/65775800): Fix wrong output bug in Call and remove the CallInliner
   // pass.
   pipeline.AddPass<CallInliner>();
diff --git a/tensorflow/compiler/xla/service/dfs_hlo_visitor.h b/tensorflow/compiler/xla/service/dfs_hlo_visitor.h
index c54e954c22..af039776b7 100644
--- a/tensorflow/compiler/xla/service/dfs_hlo_visitor.h
+++ b/tensorflow/compiler/xla/service/dfs_hlo_visitor.h
@@ -105,6 +105,7 @@ class DfsHloVisitorBase {
   }
   virtual Status HandleConvolution(HloInstructionPtr hlo) = 0;
   virtual Status HandleFft(HloInstructionPtr fft) = 0;
+  virtual Status HandleTriangularSolve(HloInstructionPtr hlo) = 0;
   virtual Status HandleAllReduce(HloInstructionPtr hlo) = 0;
   virtual Status HandleAllToAll(HloInstructionPtr hlo) = 0;
   virtual Status HandleCollectivePermute(HloInstructionPtr hlo) = 0;
diff --git a/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h b/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h
index 33b2cc3fb0..341bb37b83 100644
--- a/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h
+++ b/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h
@@ -91,6 +91,9 @@ class DfsHloVisitorWithDefaultBase
   Status HandleFft(HloInstructionPtr fft) override {
     return DefaultAction(fft);
   }
+  Status HandleTriangularSolve(HloInstructionPtr hlo) override {
+    return DefaultAction(hlo);
+  }
   Status HandleAllReduce(HloInstructionPtr crs) override {
     return DefaultAction(crs);
   }
diff --git a/tensorflow/compiler/xla/service/gather_expander.cc b/tensorflow/compiler/xla/service/gather_expander.cc
index 590942cddc..a58ac39dff 100644
--- a/tensorflow/compiler/xla/service/gather_expander.cc
+++ b/tensorflow/compiler/xla/service/gather_expander.cc
@@ -296,7 +296,7 @@ static StatusOr<HloInstruction*> PermuteBatchAndOffsetDims(
 // [3,1] out of operand into an accumulator of shape [4,3,1].  We then
 // reshape this result to [2,2,3] and finally transpose it to [2,3,2].
 
-StatusOr<HloInstruction*> GatherExpander::ExpandGather(
+StatusOr<HloInstruction*> GatherExpander::ExpandInstruction(
     HloInstruction* gather_instr) {
   CHECK(!ShapeUtil::IsZeroElementArray(gather_instr->shape()));
 
@@ -361,25 +361,11 @@ StatusOr<HloInstruction*> GatherExpander::ExpandGather(
                                    output_rank);
 }
 
-StatusOr<bool> GatherExpander::Run(HloModule* module) {
-  auto is_nontrivial_gather = [](HloInstruction* inst) {
-    return inst->opcode() == HloOpcode::kGather &&
-           // Avoid expanding gather ops that produce zero sized tensors,
-           // instead punt these to ZeroSizedHloElimination.
-           !ShapeUtil::IsZeroElementArray(inst->shape());
-  };
-
-  std::vector<HloInstruction*> gather_instrs;
-  for (HloComputation* computation : module->MakeNonfusionComputations()) {
-    absl::c_copy_if(computation->instructions(),
-                    std::back_inserter(gather_instrs), is_nontrivial_gather);
-  }
-
-  for (HloInstruction* inst : gather_instrs) {
-    TF_ASSIGN_OR_RETURN(HloInstruction * expanded_root, ExpandGather(inst));
-    TF_RETURN_IF_ERROR(inst->parent()->ReplaceInstruction(inst, expanded_root));
-  }
-
-  return !gather_instrs.empty();
+bool GatherExpander::InstructionMatchesPattern(HloInstruction* inst) {
+  return inst->opcode() == HloOpcode::kGather &&
+         // Avoid expanding gather ops that produce zero sized tensors,
+         // instead punt these to ZeroSizedHloElimination.
+         !ShapeUtil::IsZeroElementArray(inst->shape());
 }
+
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/gather_expander.h b/tensorflow/compiler/xla/service/gather_expander.h
index 8af9c6b71f..5625a37cb4 100644
--- a/tensorflow/compiler/xla/service/gather_expander.h
+++ b/tensorflow/compiler/xla/service/gather_expander.h
@@ -16,20 +16,22 @@ limitations under the License.
 #ifndef TENSORFLOW_COMPILER_XLA_SERVICE_GATHER_EXPANDER_H_
 #define TENSORFLOW_COMPILER_XLA_SERVICE_GATHER_EXPANDER_H_
 
-#include "tensorflow/compiler/xla/service/hlo_pass_interface.h"
+#include "tensorflow/compiler/xla/service/op_expander_pass.h"
 
 namespace xla {
 
 // This pass rewrites gather operations into (roughly) while loops of dynamic
 // slices.  This lets backends that don't support gather directly to
 // nevertheless have a minimum level of support.
-class GatherExpander : public HloModulePass {
+class GatherExpander : public OpExpanderPass {
  public:
   absl::string_view name() const override { return "gather_expander"; }
-  StatusOr<bool> Run(HloModule* module) override;
 
  protected:
-  StatusOr<HloInstruction*> ExpandGather(HloInstruction* gather_instr);
+  bool InstructionMatchesPattern(HloInstruction* instruction) override;
+
+  StatusOr<HloInstruction*> ExpandInstruction(
+      HloInstruction* gather_inst) override;
 };
 
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD
index 85fb2dd47a..17274b46ab 100644
--- a/tensorflow/compiler/xla/service/gpu/BUILD
+++ b/tensorflow/compiler/xla/service/gpu/BUILD
@@ -305,6 +305,7 @@ cc_library(
         "sequential_thunk.cc",
         "thunk.cc",
         "thunk_schedule.cc",
+        "triangular_solve_thunk.cc",
         "tuple_thunk.cc",
         "while_thunk.cc",
     ],
@@ -324,6 +325,7 @@ cc_library(
         "sequential_thunk.h",
         "thunk.h",
         "thunk_schedule.h",
+        "triangular_solve_thunk.h",
         "tuple_thunk.h",
         "while_thunk.h",
     ],
@@ -364,6 +366,8 @@ cc_library(
         "//tensorflow/core/platform/default/build_config:cufft_plugin",
         "//tensorflow/core/platform/default/build_config:stream_executor_cuda",  # build_cleaner: keep
         "//tensorflow/stream_executor",
+        "//tensorflow/stream_executor:blas",
+        "//tensorflow/stream_executor:device_memory",
         "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/memory",
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_layout_assignment.cc b/tensorflow/compiler/xla/service/gpu/gpu_layout_assignment.cc
index 58bdd4209a..a6d80f0b6d 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_layout_assignment.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_layout_assignment.cc
@@ -240,6 +240,32 @@ Status GpuLayoutAssignment::AddBackendConstraints(
         TF_RETURN_IF_ERROR(
             constraints->SetBufferLayout(keys_layout, *output_buffer));
       }
+    } else if (instruction->opcode() == HloOpcode::kTriangularSolve) {
+      // TODO(phawkins): Ideally we would relax this constraint. What we
+      // actually want is that:
+      // a) the batch dimensions are major, in no particular order.
+      // b) the two minor dimensions are in fortran (column-major) order,
+      // although for the 'a' argument we could potentially accept row-major
+      // order and fold the transpose into the operator.
+      auto set_fortran_layout = [](Shape* shape) {
+        LayoutUtil::SetToDefaultLayout(shape);
+        int n = shape->mutable_layout()->minor_to_major_size();
+        CHECK_GE(n, 2);
+        std::swap(shape->mutable_layout()->mutable_minor_to_major()->at(0),
+                  shape->mutable_layout()->mutable_minor_to_major()->at(1));
+      };
+      Shape op0_shape = instruction->operand(0)->shape();
+      Shape op1_shape = instruction->operand(1)->shape();
+      Shape output_shape = instruction->shape();
+      set_fortran_layout(&op0_shape);
+      set_fortran_layout(&op1_shape);
+      set_fortran_layout(&output_shape);
+      TF_RETURN_IF_ERROR(
+          constraints->SetOperandLayout(op0_shape, instruction, 0));
+      TF_RETURN_IF_ERROR(
+          constraints->SetOperandLayout(op1_shape, instruction, 1));
+      TF_RETURN_IF_ERROR(
+          constraints->SetInstructionLayout(output_shape, instruction));
     }
   }
   return Status::OK();
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index ee8db1fa36..26b266a066 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -59,6 +59,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/gpu/partition_assignment.h"
 #include "tensorflow/compiler/xla/service/gpu/sequential_thunk.h"
 #include "tensorflow/compiler/xla/service/gpu/thunk.h"
+#include "tensorflow/compiler/xla/service/gpu/triangular_solve_thunk.h"
 #include "tensorflow/compiler/xla/service/gpu/tuple_thunk.h"
 #include "tensorflow/compiler/xla/service/gpu/while_thunk.h"
 #include "tensorflow/compiler/xla/service/hlo_casting_utils.h"
@@ -486,6 +487,41 @@ Status IrEmitterUnnested::HandleFft(HloInstruction* fft) {
   return Status::OK();
 }
 
+Status IrEmitterUnnested::HandleTriangularSolve(HloInstruction* hlo) {
+  auto has_fortran_layout = [](const Layout& layout) {
+    int n = layout.minor_to_major_size();
+    return layout.minor_to_major(0) == n - 2 &&
+           layout.minor_to_major(1) == n - 1;
+  };
+  TF_RET_CHECK(has_fortran_layout(hlo->operand(0)->shape().layout()));
+  TF_RET_CHECK(has_fortran_layout(hlo->operand(1)->shape().layout()));
+  TF_RET_CHECK(has_fortran_layout(hlo->shape().layout()));
+
+  std::vector<std::unique_ptr<Thunk>> thunks;
+
+  // Triangular solve is in-place on 'b', so copy 'b' to the output if they
+  // aren't the same buffer.
+  auto operand_buffer = GetAllocationSlice(*hlo->operand(1));
+  auto destination_buffer = GetAllocationSlice(*hlo);
+  if (operand_buffer != destination_buffer) {
+    thunks.push_back(absl::make_unique<DeviceToDeviceCopyThunk>(
+        /*source_address=*/operand_buffer,
+        /*destination_buffer=*/destination_buffer,
+        /*mem_size=*/ShapeUtil::ByteSizeOf(hlo->operand(1)->shape()), hlo));
+  }
+
+  thunks.push_back(BuildTriangularSolveThunk(hlo));
+
+  // Elide the sequential thunk if there's no copy.
+  if (thunks.size() == 1) {
+    AddThunkToThunkSequence(std::move(thunks[0]));
+  } else {
+    AddThunkToThunkSequence(
+        absl::make_unique<SequentialThunk>(std::move(thunks), hlo));
+  }
+  return Status::OK();
+}
+
 Status IrEmitterUnnested::HandleFusion(HloInstruction* fusion) {
   HloInstruction* root = fusion->fused_expression_root();
   if (HloInstruction::FusionKind::kInput == fusion->fusion_kind()) {
@@ -958,18 +994,18 @@ Status IrEmitterUnnested::HandleScatter(HloInstruction* scatter) {
       BuildKernelThunk(scatter,
                        /*implements_whole_instruction=*/thunks.empty()));
 
-  TF_RETURN_IF_ERROR(
-      EmitScatter(thunks.back().get(), scatter,
-                  /*scatter_indices_gen=*/
-                  [=](const IrArray::Index& index) {
-                    return GetIrArray(*scatter_indices, *scatter)
-                        .EmitReadArrayElement(index, &b_, "scatter_index");
-                  },
-                  /*updates_gen=*/
-                  [=](const IrArray::Index& index) {
-                    return GetIrArray(*updates, *scatter)
-                        .EmitReadArrayElement(index, &b_, "update");
-                  }));
+  TF_RETURN_IF_ERROR(EmitScatter(
+      thunks.back().get(), scatter,
+      /*scatter_indices_gen=*/
+      [=](const IrArray::Index& index) {
+        return GetIrArray(*scatter_indices, *scatter)
+            .EmitReadArrayElement(index, &b_, "scatter_index");
+      },
+      /*updates_gen=*/
+      [=](const IrArray::Index& index) {
+        return GetIrArray(*updates, *scatter)
+            .EmitReadArrayElement(index, &b_, "update");
+      }));
 
   // Elide the sequential thunk if there's no copy.
   if (thunks.size() == 1) {
@@ -1757,6 +1793,29 @@ std::unique_ptr<Thunk> IrEmitterUnnested::BuildFftThunk(
       /*output_shape=*/inst->shape(), inst);
 }
 
+std::unique_ptr<Thunk> IrEmitterUnnested::BuildTriangularSolveThunk(
+    const HloInstruction* inst) {
+  const HloInstruction* a = inst->operand(0);
+  const HloInstruction* b = inst->operand(1);
+  int64 m = b->shape().dimensions(b->shape().rank() - 2);
+  int64 n = b->shape().dimensions(b->shape().rank() - 1);
+  int64 batch_size = std::accumulate(
+      b->shape().dimensions().begin(), b->shape().dimensions().end() - 2,
+      int64{1}, [](int64 a, int64 b) { return a * b; });
+  int64 elem_size =
+      ShapeUtil::ByteSizeOfPrimitiveType(inst->shape().element_type());
+  int64 a_batch_stride = inst->triangular_solve_options().left_side()
+                             ? m * m * elem_size
+                             : n * n * elem_size;
+  int64 b_batch_stride = m * n * elem_size;
+  return absl::make_unique<TriangularSolveThunk>(
+      inst->triangular_solve_options(),
+      /*a_input_buffer=*/GetAllocationSlice(*a),
+      /*b_input_buffer=*/GetAllocationSlice(*inst),
+      inst->shape().element_type(), batch_size, m, n, a_batch_stride,
+      b_batch_stride, inst);
+}
+
 StatusOr<std::unique_ptr<Thunk>> IrEmitterUnnested::BuildInitializerThunk(
     HloInstruction* hlo, const ShapeIndex& index) {
   bool fused = HloOpcode::kFusion == hlo->opcode();
@@ -2132,7 +2191,6 @@ std::vector<IrArray> IrEmitterUnnested::ConstructIrArrayForInputs(
   return param_arrays;
 }
 
-
 int IrEmitterUnnested::ConstructInputReducedShapeAndCastInputIrArrayToShape(
     const HloInstruction& hlo, const std::vector<IrArray>& param_arrays,
     const std::vector<llvm::Value*>& param_buffers,
@@ -2966,12 +3024,11 @@ LaunchDimensions IrEmitterUnnested::EmitKernel(
   // since we touch threadIdx.x and blockIdx.x at the beginning of the kernel
   // *anyway*.
   if (!reduction_info && unnested_hlo->IsMultiOutputFusion()) {
-    KernelSupportLibrary{&b_}.If(
-        "emit_mof_tuple", IsBlock0Thread0(&b_), [&] {
-          llvm_ir::EmitTuple(GetIrArray(*unnested_hlo, *unnested_hlo),
-                             ConstructIrArrayForOutputs(*unnested_hlo), &b_,
-                             module_);
-        });
+    KernelSupportLibrary{&b_}.If("emit_mof_tuple", IsBlock0Thread0(&b_), [&] {
+      llvm_ir::EmitTuple(GetIrArray(*unnested_hlo, *unnested_hlo),
+                         ConstructIrArrayForOutputs(*unnested_hlo), &b_,
+                         module_);
+    });
   }
 
   // For each tiled parameter, cast its input IrArray to the corresponding
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
index 21b842bb2c..f85e18bbf0 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
@@ -176,6 +176,7 @@ class IrEmitterUnnested : public IrEmitter {
   Status HandleScatter(HloInstruction* scatter) override;
   Status HandleSelect(HloInstruction* select) override;
   Status HandleSort(HloInstruction* sort) override;
+  Status HandleTriangularSolve(HloInstruction* hlo) override;
   Status HandleTupleSelect(HloInstruction* tuple_select) override;
   Status HandleAllReduce(HloInstruction* crs) override;
   Status HandleAfterAll(HloInstruction* after_all) override;
@@ -319,6 +320,9 @@ class IrEmitterUnnested : public IrEmitter {
   // Returns a FftThunk that calls cuFFT to implement `inst`.
   std::unique_ptr<Thunk> BuildFftThunk(const HloInstruction* inst);
 
+  // Returns a TriangularSolveThunk that calls cuBlas to implement `inst`.
+  std::unique_ptr<Thunk> BuildTriangularSolveThunk(const HloInstruction* inst);
+
   // Returns a GemmThunk that calls gemm to implement `inst`. The caller needs
   // to make sure `inst` outlives the lifetime of the returned Thunk object.
   std::unique_ptr<Thunk> BuildGemmThunk(const HloInstruction* inst);
diff --git a/tensorflow/compiler/xla/service/gpu/thunk.cc b/tensorflow/compiler/xla/service/gpu/thunk.cc
index c78605cebb..a677617727 100644
--- a/tensorflow/compiler/xla/service/gpu/thunk.cc
+++ b/tensorflow/compiler/xla/service/gpu/thunk.cc
@@ -48,6 +48,8 @@ std::ostream& operator<<(std::ostream& os, Thunk::Kind kind) {
       return os << "kOutfeed";
     case Thunk::kSequential:
       return os << "kSequential";
+    case Thunk::kTriangularSolve:
+      return os << "kTriangularSolve";
     case Thunk::kTuple:
       return os << "kTuple";
     case Thunk::kWhile:
diff --git a/tensorflow/compiler/xla/service/gpu/thunk.h b/tensorflow/compiler/xla/service/gpu/thunk.h
index e68bee035a..bc69af897a 100644
--- a/tensorflow/compiler/xla/service/gpu/thunk.h
+++ b/tensorflow/compiler/xla/service/gpu/thunk.h
@@ -56,6 +56,7 @@ class Thunk {
     kMemzero,
     kOutfeed,
     kSequential,
+    kTriangularSolve,
     kTuple,
     kWhile,
   };
diff --git a/tensorflow/compiler/xla/service/gpu/triangular_solve_thunk.cc b/tensorflow/compiler/xla/service/gpu/triangular_solve_thunk.cc
new file mode 100644
index 0000000000..5200a2af41
--- /dev/null
+++ b/tensorflow/compiler/xla/service/gpu/triangular_solve_thunk.cc
@@ -0,0 +1,149 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/gpu/triangular_solve_thunk.h"
+
+#include <string>
+
+#include "absl/strings/str_cat.h"
+#include "absl/strings/str_format.h"
+#include "tensorflow/compiler/xla/service/gpu/hlo_execution_profiler.h"
+#include "tensorflow/compiler/xla/types.h"
+#include "tensorflow/compiler/xla/util.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/stream_executor_no_cuda.h"
+#include "tensorflow/stream_executor/blas.h"
+#include "tensorflow/stream_executor/device_memory.h"
+
+namespace xla {
+namespace gpu {
+
+TriangularSolveThunk::TriangularSolveThunk(
+    const TriangularSolveOptions& options,
+    const BufferAllocation::Slice& a_buffer,
+    const BufferAllocation::Slice& b_buffer, PrimitiveType type,
+    int64 batch_size, int64 m, int64 n, int64 a_batch_stride,
+    int64 b_batch_stride, const HloInstruction* hlo)
+    : Thunk(Kind::kTriangularSolve, hlo),
+      uplo_(options.lower() ? se::blas::UpperLower::kLower
+                            : se::blas::UpperLower::kUpper),
+      side_(options.left_side() ? se::blas::Side::kLeft
+                                : se::blas::Side::kRight),
+      unit_diagonal_(options.unit_diagonal() ? se::blas::Diagonal::kUnit
+                                             : se::blas::Diagonal::kNonUnit),
+      a_buffer_(a_buffer),
+      b_buffer_(b_buffer),
+      type_(type),
+      batch_size_(batch_size),
+      m_(m),
+      n_(n),
+      a_batch_stride_(a_batch_stride),
+      b_batch_stride_(b_batch_stride) {
+  transpose_a_ = [&] {
+    switch (options.transpose_a()) {
+      case TriangularSolveOptions::NO_TRANSPOSE:
+        return se::blas::Transpose::kNoTranspose;
+      case TriangularSolveOptions::TRANSPOSE:
+        return se::blas::Transpose::kTranspose;
+      case TriangularSolveOptions::ADJOINT:
+        return se::blas::Transpose::kConjugateTranspose;
+      default:
+        LOG(ERROR) << "Invalid triangular solve transpose value "
+                   << options.transpose_a();
+        return se::blas::Transpose::kNoTranspose;
+    }
+  }();
+}
+
+Status TriangularSolveThunk::ExecuteOnStream(
+    const BufferAllocations& buffer_allocations, se::Stream* stream,
+    HloExecutionProfiler* profiler) {
+  VLOG(3) << "uplo=" << se::blas::UpperLowerString(uplo_)
+          << " side=" << se::blas::SideString(side_)
+          << " diagonal=" << se::blas::DiagonalString(unit_diagonal_)
+          << " batch_size=" << batch_size_ << " m=" << m_ << " n=" << n_
+          << " a_batch_stride=" << a_batch_stride_
+          << " b_batch_stride=" << b_batch_stride_;
+
+  const int lda = side_ == se::blas::Side::kLeft ? m_ : n_;
+  const int ldb = m_;
+
+  char* a_base = static_cast<char*>(
+      buffer_allocations.GetDeviceAddress(a_buffer_).opaque());
+  char* b_base = static_cast<char*>(
+      buffer_allocations.GetDeviceAddress(b_buffer_).opaque());
+  for (int64 i = 0; i < batch_size_; ++i) {
+    bool launch_ok;
+    se::DeviceMemoryBase a_data =
+        se::DeviceMemoryBase(a_base + i * a_batch_stride_, a_batch_stride_);
+    se::DeviceMemoryBase b_data =
+        se::DeviceMemoryBase(b_base + i * b_batch_stride_, b_batch_stride_);
+    switch (type_) {
+      case F32: {
+        se::DeviceMemory<float> b_data_typed(b_data);
+        launch_ok = stream
+                        ->ThenBlasTrsm(side_, uplo_, transpose_a_,
+                                       unit_diagonal_, m_, n_, /*alpha=*/1.0f,
+                                       se::DeviceMemory<float>(a_data), lda,
+                                       &b_data_typed, ldb)
+                        .ok();
+        break;
+      }
+      case F64: {
+        se::DeviceMemory<double> b_data_typed(b_data);
+        launch_ok = stream
+                        ->ThenBlasTrsm(side_, uplo_, transpose_a_,
+                                       unit_diagonal_, m_, n_, /*alpha=*/1.0,
+                                       se::DeviceMemory<double>(a_data), lda,
+                                       &b_data_typed, ldb)
+                        .ok();
+        break;
+      }
+      case C64: {
+        se::DeviceMemory<std::complex<float>> b_data_typed(b_data);
+        launch_ok =
+            stream
+                ->ThenBlasTrsm(side_, uplo_, transpose_a_, unit_diagonal_, m_,
+                               n_, /*alpha=*/1.0f,
+                               se::DeviceMemory<std::complex<float>>(a_data),
+                               lda, &b_data_typed, ldb)
+                .ok();
+        break;
+      }
+      case C128: {
+        se::DeviceMemory<std::complex<double>> b_data_typed(b_data);
+        launch_ok =
+            stream
+                ->ThenBlasTrsm(side_, uplo_, transpose_a_, unit_diagonal_, m_,
+                               n_, /*alpha=*/1.0,
+                               se::DeviceMemory<std::complex<double>>(a_data),
+                               lda, &b_data_typed, ldb)
+                .ok();
+        break;
+      }
+      default:
+        return InvalidArgument("Invalid type for triangular solve %d", type_);
+    }
+    if (!launch_ok) {
+      return InternalError("Unable to launch triangular solve for thunk %p",
+                           this);
+    }
+  }
+  return Status::OK();
+}
+
+}  // namespace gpu
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/gpu/triangular_solve_thunk.h b/tensorflow/compiler/xla/service/gpu/triangular_solve_thunk.h
new file mode 100644
index 0000000000..c947162ea3
--- /dev/null
+++ b/tensorflow/compiler/xla/service/gpu/triangular_solve_thunk.h
@@ -0,0 +1,75 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_GPU_TRIANGULAR_SOLVE_THUNK_H_
+#define TENSORFLOW_COMPILER_XLA_SERVICE_GPU_TRIANGULAR_SOLVE_THUNK_H_
+
+#include "absl/types/optional.h"
+#include "tensorflow/compiler/xla/service/buffer_assignment.h"
+#include "tensorflow/compiler/xla/service/gpu/buffer_allocations.h"
+#include "tensorflow/compiler/xla/service/gpu/gpu_executable.h"
+#include "tensorflow/compiler/xla/service/gpu/hlo_execution_profiler.h"
+#include "tensorflow/compiler/xla/service/gpu/thunk.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/types.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/platform/stream_executor_no_cuda.h"
+#include "tensorflow/stream_executor/blas.h"
+
+namespace xla {
+namespace gpu {
+
+// This class stores everything that StreamExecutor needs to launch a triangular
+// solve (BlasTrsm). It is generated by IrEmitter.
+//
+// Thread-compatible.
+class TriangularSolveThunk : public Thunk {
+ public:
+  TriangularSolveThunk(const TriangularSolveOptions& options,
+                       const BufferAllocation::Slice& a_buffer,
+                       const BufferAllocation::Slice& b_buffer,
+                       PrimitiveType type, int64 batch_size, int64 m, int64 n,
+                       int64 a_batch_stride, int64 b_batch_stride,
+                       const HloInstruction* hlo);
+
+  TriangularSolveThunk(const TriangularSolveThunk&) = delete;
+  TriangularSolveThunk& operator=(const TriangularSolveThunk&) = delete;
+
+  Status ExecuteOnStream(const BufferAllocations& buffer_allocations,
+                         se::Stream* stream,
+                         HloExecutionProfiler* profiler) override;
+
+ private:
+  const se::blas::UpperLower uplo_;
+  const se::blas::Side side_;
+  const se::blas::Diagonal unit_diagonal_;
+  se::blas::Transpose transpose_a_;
+
+  const BufferAllocation::Slice a_buffer_;
+  const BufferAllocation::Slice b_buffer_;
+
+  const PrimitiveType type_;
+  const int64 batch_size_;
+  const int64 m_;
+  const int64 n_;
+  const int64 a_batch_stride_;
+  const int64 b_batch_stride_;
+};
+
+}  // namespace gpu
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_GPU_TRIANGULAR_SOLVE_THUNK_H_
diff --git a/tensorflow/compiler/xla/service/hlo.proto b/tensorflow/compiler/xla/service/hlo.proto
index 263b42a29d..6e64549e7e 100644
--- a/tensorflow/compiler/xla/service/hlo.proto
+++ b/tensorflow/compiler/xla/service/hlo.proto
@@ -34,7 +34,7 @@ import "tensorflow/compiler/xla/xla_data.proto";
 option cc_enable_arenas = true;
 
 // Serialization of HloInstruction.
-// Next ID: 59
+// Next ID: 60
 message HloInstructionProto {
   reserved 10;
   reserved "parameter_name";
@@ -193,6 +193,9 @@ message HloInstructionProto {
   // operand.
   bool constrain_layout = 56;
   repeated xla.ShapeProto operand_shapes_with_layout = 57;
+
+  // Options for TriangularSolve
+  xla.TriangularSolveOptions triangular_solve_options = 59;
 }
 
 // Serialization of HloComputation.
diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
index 1ee958114e..d7c547ff26 100644
--- a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
+++ b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
@@ -546,6 +546,21 @@ Status HloCostAnalysis::HandleFft(const HloInstruction* fft) {
   return Status::OK();
 }
 
+Status HloCostAnalysis::HandleTriangularSolve(const HloInstruction* hlo) {
+  float bytes_accessed = GetShapeSize(hlo->operand(0)->shape()) / 2.0f;
+  bytes_accessed += GetShapeSize(hlo->operand(1)->shape());
+  current_properties_[kBytesAccessedKey] = bytes_accessed;
+
+  const Shape& a_shape = hlo->operand(0)->shape();
+  const Shape& b_shape = hlo->operand(1)->shape();
+  // Estimate as batch * mn^2 / 2 flops.
+  int64 elems = a_shape.dimensions(a_shape.dimensions_size() - 1);
+  elems *= ShapeUtil::ElementsIn(b_shape);
+  // Each output elment requires reduction_widht FMA operations.
+  current_properties_[kFlopsKey] = kFmaFlops * elems;
+  return Status::OK();
+}
+
 Status HloCostAnalysis::HandleAllReduce(const HloInstruction* crs) {
   // We assume 2 replicas, so that each output element is the sum of two input
   // elements.
diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis.h b/tensorflow/compiler/xla/service/hlo_cost_analysis.h
index 421786e20a..96357dec68 100644
--- a/tensorflow/compiler/xla/service/hlo_cost_analysis.h
+++ b/tensorflow/compiler/xla/service/hlo_cost_analysis.h
@@ -71,6 +71,7 @@ class HloCostAnalysis : public ConstDfsHloVisitor {
   Status HandleDot(const HloInstruction* dot) override;
   Status HandleConvolution(const HloInstruction* convolution) override;
   Status HandleFft(const HloInstruction* fft) override;
+  Status HandleTriangularSolve(const HloInstruction* hlo) override;
   Status HandleAllReduce(const HloInstruction* crs) override;
   Status HandleAllToAll(const HloInstruction* hlo) override;
   Status HandleCollectivePermute(const HloInstruction* hlo) override;
diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
index 46ee99923e..dcf00650d0 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
@@ -1011,6 +1011,7 @@ ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) {
     case HloOpcode::kConvolution:
     case HloOpcode::kDot:
     case HloOpcode::kFft:
+    case HloOpcode::kTriangularSolve:
       return kDarkBlue;
     case HloOpcode::kReducePrecision:
       return kRed;
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index fd9b8ea352..acf881ac6a 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -132,6 +132,14 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
                               absl::Span<const int64>(fft_length));
       break;
     }
+    case HloOpcode::kTriangularSolve: {
+      TF_RET_CHECK(proto.operand_ids_size() == 2)
+          << "Triangular solve instruction should have 2 operands but sees "
+          << proto.operand_ids_size();
+      instruction = CreateTriangularSolve(shape, operands(0), operands(1),
+                                          proto.triangular_solve_options());
+      break;
+    }
     case HloOpcode::kSend:
       TF_RET_CHECK(proto.operand_ids_size() == 2)
           << "Send instruction should have 2 operand but sees "
@@ -795,6 +803,13 @@ HloInstruction::CreateGetTupleElement(const Shape& shape,
                                               fft_length);
 }
 
+/* static */ std::unique_ptr<HloInstruction>
+HloInstruction::CreateTriangularSolve(const Shape& shape, HloInstruction* a,
+                                      HloInstruction* b,
+                                      const TriangularSolveOptions& options) {
+  return absl::make_unique<HloTriangularSolveInstruction>(shape, a, b, options);
+}
+
 /* static */ std::unique_ptr<HloInstruction> HloInstruction::CreateDot(
     const Shape& shape, HloInstruction* lhs, HloInstruction* rhs,
     const DotDimensionNumbers& dimension_numbers,
@@ -1359,6 +1374,7 @@ std::unique_ptr<HloInstruction> HloInstruction::CloneWithNewOperands(
     case HloOpcode::kDot:
     case HloOpcode::kDomain:
     case HloOpcode::kGetDimensionSize:
+    case HloOpcode::kTriangularSolve:
       clone = CloneWithNewOperandsImpl(shape, new_operands, context);
       break;
     // Unary ops.
@@ -1814,6 +1830,7 @@ bool HloInstruction::IdenticalSlowPath(
     case HloOpcode::kDot:
     case HloOpcode::kDomain:
     case HloOpcode::kGetDimensionSize:
+    case HloOpcode::kTriangularSolve:
       LOG(FATAL) << "Base class impl called for opcode with subclass: "
                  << opcode();
   }
@@ -2609,6 +2626,8 @@ Status HloInstruction::Visit(DfsHloVisitorBase<HloInstructionPtr>* visitor) {
       return visitor->HandleIota(this);
     case HloOpcode::kGetDimensionSize:
       return visitor->HandleGetDimensionSize(this);
+    case HloOpcode::kTriangularSolve:
+      return visitor->HandleTriangularSolve(this);
 
     // These opcodes are not handled here.
     case HloOpcode::kTrace:
@@ -3476,4 +3495,8 @@ const DomainMetadata& HloInstruction::operand_side_metadata() const {
 const DomainMetadata& HloInstruction::user_side_metadata() const {
   return Cast<HloDomainInstruction>(this)->user_side_metadata();
 }
+
+const TriangularSolveOptions& HloInstruction::triangular_solve_options() const {
+  return Cast<HloTriangularSolveInstruction>(this)->triangular_solve_options();
+}
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h
index 92c25ef006..d7da21619f 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.h
+++ b/tensorflow/compiler/xla/service/hlo_instruction.h
@@ -435,6 +435,10 @@ class HloInstruction {
       const Shape& shape, HloInstruction* operand, FftType fft_type,
       absl::Span<const int64> fft_length);
 
+  static std::unique_ptr<HloInstruction> CreateTriangularSolve(
+      const Shape& shape, HloInstruction* a, HloInstruction* b,
+      const TriangularSolveOptions& options);
+
   // Creates a dot op with operands 'lhs' and 'rhs' with contracting and batch
   // dimensions specified in 'dimension_numbers'.
   static std::unique_ptr<HloInstruction> CreateDot(
@@ -1566,6 +1570,9 @@ class HloInstruction {
   // Delegates to HloDomainInstruction::user_side_metadata().
   const DomainMetadata& user_side_metadata() const;
 
+  // Delegates to HloTriangularSolveInstruction::triangular_solve_options().
+  const TriangularSolveOptions& triangular_solve_options() const;
+
   // Old methods kept for smooth subclassing transition END.
 
  protected:
diff --git a/tensorflow/compiler/xla/service/hlo_instructions.cc b/tensorflow/compiler/xla/service/hlo_instructions.cc
index 8e40e54c3d..27b46b272a 100644
--- a/tensorflow/compiler/xla/service/hlo_instructions.cc
+++ b/tensorflow/compiler/xla/service/hlo_instructions.cc
@@ -201,6 +201,54 @@ std::unique_ptr<HloInstruction> HloFftInstruction::CloneWithNewOperandsImpl(
                                               fft_length_);
 }
 
+HloTriangularSolveInstruction::HloTriangularSolveInstruction(
+    const Shape& shape, HloInstruction* a, HloInstruction* b,
+    const TriangularSolveOptions& options)
+    : HloInstruction(HloOpcode::kTriangularSolve, shape),
+      triangular_solve_options_(options) {
+  AppendOperand(a);
+  AppendOperand(b);
+}
+
+HloInstructionProto HloTriangularSolveInstruction::ToProto() const {
+  HloInstructionProto proto = HloInstruction::ToProto();
+  *proto.mutable_triangular_solve_options() = triangular_solve_options_;
+  return proto;
+}
+
+std::vector<string> HloTriangularSolveInstruction::ExtraAttributesToStringImpl(
+    const HloPrintOptions& options) const {
+  return {StrCat("left_side=", triangular_solve_options_.left_side()),
+          StrCat("lower=", triangular_solve_options_.lower()),
+          StrCat("unit_diagonal=", triangular_solve_options_.unit_diagonal()),
+          StrCat("transpose_a=", TriangularSolveOptions_Transpose_Name(
+                                     triangular_solve_options_.transpose_a()))};
+}
+
+bool HloTriangularSolveInstruction::IdenticalSlowPath(
+    const HloInstruction& other,
+    const std::function<bool(const HloComputation*, const HloComputation*)>&
+        eq_computations) const {
+  const auto& casted_other =
+      static_cast<const HloTriangularSolveInstruction&>(other);
+  const auto& options = triangular_solve_options();
+  const auto& other_options = casted_other.triangular_solve_options();
+
+  return options.left_side() == other_options.left_side() &&
+         options.lower() == other_options.lower() &&
+         options.unit_diagonal() == other_options.unit_diagonal() &&
+         options.transpose_a() == other_options.transpose_a();
+}
+
+std::unique_ptr<HloInstruction>
+HloTriangularSolveInstruction::CloneWithNewOperandsImpl(
+    const Shape& shape, absl::Span<HloInstruction* const> new_operands,
+    HloCloneContext* context) const {
+  CHECK_EQ(new_operands.size(), 2);
+  return absl::make_unique<HloTriangularSolveInstruction>(
+      shape, new_operands[0], new_operands[1], triangular_solve_options());
+}
+
 HloSendRecvInstruction::HloSendRecvInstruction(HloOpcode opcode,
                                                const Shape& shape,
                                                int64 channel_id,
diff --git a/tensorflow/compiler/xla/service/hlo_instructions.h b/tensorflow/compiler/xla/service/hlo_instructions.h
index 6b6157a776..6cf6dab309 100644
--- a/tensorflow/compiler/xla/service/hlo_instructions.h
+++ b/tensorflow/compiler/xla/service/hlo_instructions.h
@@ -131,6 +131,34 @@ class HloFftInstruction : public HloInstruction {
   std::vector<int64> fft_length_;
 };
 
+class HloTriangularSolveInstruction : public HloInstruction {
+ public:
+  explicit HloTriangularSolveInstruction(const Shape& shape, HloInstruction* a,
+                                         HloInstruction* b,
+                                         const TriangularSolveOptions& options);
+  const TriangularSolveOptions& triangular_solve_options() const {
+    return triangular_solve_options_;
+  }
+
+  // Returns a serialized representation of this instruction.
+  HloInstructionProto ToProto() const override;
+
+ private:
+  std::vector<string> ExtraAttributesToStringImpl(
+      const HloPrintOptions& options) const override;
+  bool IdenticalSlowPath(
+      const HloInstruction& other,
+      const std::function<bool(const HloComputation*, const HloComputation*)>&
+          eq_computations) const override;
+
+  // Implementation for non-common logic of CloneWithNewOperands.
+  std::unique_ptr<HloInstruction> CloneWithNewOperandsImpl(
+      const Shape& shape, absl::Span<HloInstruction* const> new_operands,
+      HloCloneContext* context) const override;
+
+  TriangularSolveOptions triangular_solve_options_;
+};
+
 class HloSendRecvInstruction : public HloInstruction {
  public:
   // Returns the channel id associated with the instruction. The id is
@@ -906,9 +934,7 @@ class HloOutfeedInstruction : public HloInstruction {
                                  HloInstruction* token_operand,
                                  absl::string_view outfeed_config);
   // Returns the shape for the Outfeed instruction.
-  const Shape& outfeed_shape() const {
-    return outfeed_shape_;
-  }
+  const Shape& outfeed_shape() const { return outfeed_shape_; }
   // Returns the config for the Outfeed instruction.
   const string& outfeed_config() const { return outfeed_config_; }
   // Returns a serialized representation of this instruction.
diff --git a/tensorflow/compiler/xla/service/hlo_opcode.h b/tensorflow/compiler/xla/service/hlo_opcode.h
index bf9b3c8117..35626ba375 100644
--- a/tensorflow/compiler/xla/service/hlo_opcode.h
+++ b/tensorflow/compiler/xla/service/hlo_opcode.h
@@ -137,6 +137,7 @@ namespace xla {
   V(kTanh, "tanh")                                           \
   V(kTrace, "trace")                                         \
   V(kTranspose, "transpose")                                 \
+  V(kTriangularSolve, "triangular-solve")                    \
   V(kTuple, "tuple", kHloOpcodeIsVariadic)                   \
   V(kTupleSelect, "tuple-select")                            \
   V(kWhile, "while")                                         \
diff --git a/tensorflow/compiler/xla/service/hlo_parser.cc b/tensorflow/compiler/xla/service/hlo_parser.cc
index b3b24c148a..08fc728e3d 100644
--- a/tensorflow/compiler/xla/service/hlo_parser.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser.cc
@@ -180,6 +180,7 @@ class HloParser {
     kBracedInt64ListList,
     kHloComputation,
     kFftType,
+    kTriangularSolveTranspose,
     kWindow,
     kConvolutionDimensionNumbers,
     kSharding,
@@ -278,6 +279,7 @@ class HloParser {
   bool ParseLayout(Layout* layout);
   bool ParseOpcode(HloOpcode* result);
   bool ParseFftType(FftType* result);
+  bool ParseTriangularSolveTranspose(TriangularSolveOptions::Transpose* result);
   bool ParseFusionKind(HloInstruction::FusionKind* result);
   bool ParseRandomDistribution(RandomDistribution* result);
   bool ParsePrecision(PrecisionConfig::Precision* result);
@@ -1098,6 +1100,38 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
           shape, operands[0], *fft_type, *fft_length));
       break;
     }
+    case HloOpcode::kTriangularSolve: {
+      optional<bool> left_side;
+      optional<bool> lower;
+      optional<bool> unit_diagonal;
+      optional<TriangularSolveOptions::Transpose> transpose_a;
+      attrs["left_side"] = {/*required=*/false, AttrTy::kBool, &left_side};
+      attrs["lower"] = {/*required=*/false, AttrTy::kBool, &lower};
+      attrs["unit_diagonal"] = {/*required=*/false, AttrTy::kBool,
+                                &unit_diagonal};
+      attrs["transpose_a"] = {/*required=*/false,
+                              AttrTy::kTriangularSolveTranspose, &transpose_a};
+      if (!ParseOperands(&operands, /*expected_size=*/2) ||
+          !ParseAttributes(attrs)) {
+        return false;
+      }
+      TriangularSolveOptions options;
+      if (left_side) {
+        options.set_left_side(*left_side);
+      }
+      if (lower) {
+        options.set_lower(*lower);
+      }
+      if (unit_diagonal) {
+        options.set_unit_diagonal(*unit_diagonal);
+      }
+      options.set_transpose_a(
+          transpose_a ? *transpose_a : TriangularSolveOptions::NO_TRANSPOSE);
+      instruction =
+          builder->AddInstruction(HloInstruction::CreateTriangularSolve(
+              shape, operands[0], operands[1], options));
+      break;
+    }
     case HloOpcode::kBroadcast: {
       optional<std::vector<tensorflow::int64>> broadcast_dimensions;
       attrs["dimensions"] = {/*required=*/true, AttrTy::kBracedInt64List,
@@ -2630,6 +2664,15 @@ bool HloParser::ParseAttributeHelper(
         static_cast<optional<FftType>*>(attr_out_ptr)->emplace(result);
         return true;
       }
+      case AttrTy::kTriangularSolveTranspose: {
+        TriangularSolveOptions::Transpose result;
+        if (!ParseTriangularSolveTranspose(&result)) {
+          return false;
+        }
+        static_cast<optional<TriangularSolveOptions::Transpose>*>(attr_out_ptr)
+            ->emplace(result);
+        return true;
+      }
       case AttrTy::kWindow: {
         Window result;
         if (!ParseWindow(&result, /*expect_outer_curlies=*/true)) {
@@ -3447,6 +3490,22 @@ bool HloParser::ParseFftType(FftType* result) {
   return true;
 }
 
+bool HloParser::ParseTriangularSolveTranspose(
+    TriangularSolveOptions::Transpose* result) {
+  VLOG(1) << "ParseTriangularSolveTranspose";
+  if (lexer_.GetKind() != TokKind::kIdent) {
+    return TokenError("expects triangular solve transpose type");
+  }
+  string val = lexer_.GetStrVal();
+  if (!TriangularSolveOptions_Transpose_Parse(val, result) ||
+      !TriangularSolveOptions_Transpose_IsValid(*result)) {
+    return TokenError(
+        StrFormat("expects triangular solve transpose type but sees: %s", val));
+  }
+  lexer_.Lex();
+  return true;
+}
+
 bool HloParser::ParseFusionKind(HloInstruction::FusionKind* result) {
   VLOG(1) << "ParseFusionKind";
   if (lexer_.GetKind() != TokKind::kIdent) {
diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc
index 9d94c5cf76..2ca62c6794 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier.cc
@@ -168,6 +168,15 @@ Status ShapeVerifier::HandleFft(HloInstruction* fft) {
   return CheckShape(fft, expected);
 }
 
+Status ShapeVerifier::HandleTriangularSolve(HloInstruction* hlo) {
+  TF_RETURN_IF_ERROR(CheckOperandCount(hlo, 2));
+  TF_ASSIGN_OR_RETURN(const Shape expected,
+                      ShapeInference::InferTriangularSolveShape(
+                          hlo->operand(0)->shape(), hlo->operand(1)->shape(),
+                          hlo->triangular_solve_options()));
+  return CheckShape(hlo, expected);
+}
+
 Status ShapeVerifier::HandleAllReduce(HloInstruction* crs) {
   std::vector<const Shape*> operand_shapes;
   for (const HloInstruction* operand : crs->operands()) {
diff --git a/tensorflow/compiler/xla/service/hlo_verifier.h b/tensorflow/compiler/xla/service/hlo_verifier.h
index facb76a124..a9b5e9a3e6 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier.h
+++ b/tensorflow/compiler/xla/service/hlo_verifier.h
@@ -52,6 +52,7 @@ class ShapeVerifier : public DfsHloVisitor {
   Status HandleDot(HloInstruction* dot) override;
   Status HandleConvolution(HloInstruction* convolution) override;
   Status HandleFft(HloInstruction* fft) override;
+  Status HandleTriangularSolve(HloInstruction* hlo) override;
   Status HandleAllReduce(HloInstruction* crs) override;
   Status HandleAllToAll(HloInstruction* hlo) override;
   Status HandleCollectivePermute(HloInstruction* hlo) override;
diff --git a/tensorflow/compiler/xla/service/instruction_fusion.cc b/tensorflow/compiler/xla/service/instruction_fusion.cc
index d4794acb2f..a5767774f2 100644
--- a/tensorflow/compiler/xla/service/instruction_fusion.cc
+++ b/tensorflow/compiler/xla/service/instruction_fusion.cc
@@ -158,6 +158,7 @@ bool IsAlwaysDuplicable(const HloInstruction& instruction) {
     case HloOpcode::kSort:
     case HloOpcode::kTanh:
     case HloOpcode::kTrace:
+    case HloOpcode::kTriangularSolve:
     case HloOpcode::kWhile:
     case HloOpcode::kGetDimensionSize:
       return true;
diff --git a/tensorflow/compiler/xla/service/interpreter/BUILD b/tensorflow/compiler/xla/service/interpreter/BUILD
index a305c6e800..8cd9362689 100644
--- a/tensorflow/compiler/xla/service/interpreter/BUILD
+++ b/tensorflow/compiler/xla/service/interpreter/BUILD
@@ -50,6 +50,7 @@ cc_library(
         "//tensorflow/compiler/xla/service:map_inliner",
         "//tensorflow/compiler/xla/service:reduce_precision_insertion",
         "//tensorflow/compiler/xla/service:reshape_mover",
+        "//tensorflow/compiler/xla/service:triangular_solve_expander",
         "//tensorflow/compiler/xla/service:while_loop_simplifier",
         "//tensorflow/compiler/xla/service/cpu:custom_call_target_registry",
         "//tensorflow/core:lib",
diff --git a/tensorflow/compiler/xla/service/interpreter/compiler.cc b/tensorflow/compiler/xla/service/interpreter/compiler.cc
index 0827b1daf8..792773c676 100644
--- a/tensorflow/compiler/xla/service/interpreter/compiler.cc
+++ b/tensorflow/compiler/xla/service/interpreter/compiler.cc
@@ -35,6 +35,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/map_inliner.h"
 #include "tensorflow/compiler/xla/service/reduce_precision_insertion.h"
 #include "tensorflow/compiler/xla/service/reshape_mover.h"
+#include "tensorflow/compiler/xla/service/triangular_solve_expander.h"
 #include "tensorflow/compiler/xla/service/while_loop_simplifier.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/core/lib/core/errors.h"
@@ -79,6 +80,7 @@ Status InterpreterCompiler::RunHloOptimization(HloModule* hlo_module) {
   HloPassPipeline pipeline("Interpreter");
 
   pipeline.AddPass<DynamicIndexSplitter>();
+  pipeline.AddPass<TriangularSolveExpander>();
   pipeline.AddPass<LayoutAssignment>(
       hlo_module->mutable_entry_computation_layout(),
       LayoutAssignment::InstructionCanChangeLayout);
diff --git a/tensorflow/compiler/xla/service/layout_assignment.cc b/tensorflow/compiler/xla/service/layout_assignment.cc
index 9376a3c8f8..2dd9e05550 100644
--- a/tensorflow/compiler/xla/service/layout_assignment.cc
+++ b/tensorflow/compiler/xla/service/layout_assignment.cc
@@ -2070,6 +2070,7 @@ bool LayoutAssignment::InstructionCanChangeLayout(
     case HloOpcode::kSort:
     case HloOpcode::kSubtract:
     case HloOpcode::kTanh:
+    case HloOpcode::kTriangularSolve:
     case HloOpcode::kTupleSelect:
     case HloOpcode::kWhile:
       return false;
diff --git a/tensorflow/compiler/xla/service/op_expander_pass.cc b/tensorflow/compiler/xla/service/op_expander_pass.cc
new file mode 100644
index 0000000000..87f0886a97
--- /dev/null
+++ b/tensorflow/compiler/xla/service/op_expander_pass.cc
@@ -0,0 +1,44 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/op_expander_pass.h"
+
+#include <utility>
+
+#include "absl/algorithm/container.h"
+#include "tensorflow/compiler/xla/service/hlo_creation_utils.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/statusor.h"
+#include "tensorflow/compiler/xla/util.h"
+
+namespace xla {
+
+StatusOr<bool> OpExpanderPass::Run(HloModule* module) {
+  std::vector<HloInstruction*> matching_instructions;
+  for (HloComputation* computation : module->MakeNonfusionComputations()) {
+    absl::c_copy_if(
+        computation->instructions(), std::back_inserter(matching_instructions),
+        [&](HloInstruction* inst) { return InstructionMatchesPattern(inst); });
+  }
+
+  for (HloInstruction* inst : matching_instructions) {
+    TF_ASSIGN_OR_RETURN(HloInstruction * expanded_root,
+                        ExpandInstruction(inst));
+    TF_RETURN_IF_ERROR(inst->parent()->ReplaceInstruction(inst, expanded_root));
+  }
+
+  return !matching_instructions.empty();
+}
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/op_expander_pass.h b/tensorflow/compiler/xla/service/op_expander_pass.h
new file mode 100644
index 0000000000..794849d354
--- /dev/null
+++ b/tensorflow/compiler/xla/service/op_expander_pass.h
@@ -0,0 +1,43 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_OP_EXPANDER_PASS_H_
+#define TENSORFLOW_COMPILER_XLA_SERVICE_OP_EXPANDER_PASS_H_
+
+#include "tensorflow/compiler/xla/service/hlo_pass_interface.h"
+
+namespace xla {
+
+// This pass is an abstract superclass for passes that replace operations that
+// match a pattern. It is intended to be subclassed, not used directly.
+//
+// This pass is useful for legalizing HLO instructions that a particular backend
+// does not support into other HLO instructions.
+class OpExpanderPass : public HloModulePass {
+ public:
+  StatusOr<bool> Run(HloModule* module) override;
+
+ protected:
+  // Returns `true` if `instruction` should be expanded by this pass.
+  virtual bool InstructionMatchesPattern(HloInstruction* instruction) = 0;
+
+  // Returns a replacement for `instruction`.
+  virtual StatusOr<HloInstruction*> ExpandInstruction(
+      HloInstruction* instruction) = 0;
+};
+
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_OP_EXPANDER_PASS_H_
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index a403428d05..061dc46f7d 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -1906,6 +1906,49 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
 #undef RET_CHECK_RANK
 }
 
+/* static */ StatusOr<Shape> ShapeInference::InferTriangularSolveShape(
+    const Shape& a, const Shape& b, const TriangularSolveOptions& options) {
+  if (a.rank() < 2) {
+    return InvalidArgument(
+        "The 'a' argument to TriangularSolve must have rank >= 2, got shape %s",
+        a.ToString());
+  }
+  if (b.rank() != a.rank()) {
+    return InvalidArgument(
+        "Arguments to triangular solve must have equal rank; got %s and %s.",
+        b.ToString(), a.ToString());
+  }
+  if (a.dimensions(a.rank() - 2) != a.dimensions(a.rank() - 1)) {
+    return InvalidArgument(
+        "The two minor dimensions of 'a' must have equal size, got %s.",
+        a.ToString());
+  }
+  if (a.dimensions(a.rank() - 1) !=
+      b.dimensions(b.rank() - (options.left_side() ? 2 : 1))) {
+    return InvalidArgument(
+        "The shared dimension of 'a' and 'b' does not match, got shapes %s and "
+        "%s",
+        a.ToString(), b.ToString());
+  }
+  absl::Span<const int64> a_batch_dims(a.dimensions());
+  absl::Span<const int64> b_batch_dims(b.dimensions());
+  a_batch_dims.remove_suffix(2);
+  b_batch_dims.remove_suffix(2);
+  if (a_batch_dims != b_batch_dims) {
+    return InvalidArgument(
+        "The leading batch dimensions of the arguments to triangular solve "
+        "must be equal; got %s and %s.",
+        b.ToString(), a.ToString());
+  }
+  if (!TriangularSolveOptions_Transpose_IsValid(options.transpose_a()) ||
+      options.transpose_a() == TriangularSolveOptions::TRANSPOSE_INVALID) {
+    return InvalidArgument(
+        "Invalid transpose option value for triangular solve (%d).\n",
+        options.transpose_a());
+  }
+  return b;
+}
+
 /* static */ StatusOr<Shape> ShapeInference::InferAllReduceShape(
     absl::Span<const Shape* const> operand_shapes) {
   for (const Shape* operand_shape : operand_shapes) {
@@ -2352,8 +2395,8 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
 
     if (operand_shape.rank() != number_of_indices) {
       return InvalidArgument(
-          "Dynamic update slice start number of dimensions %d must match rank "
-          "%d of slice input (%s).",
+          "Dynamic update slice start number of dimensions %d must match "
+          "rank %d of slice input (%s).",
           number_of_indices, operand_shape.rank(),
           ShapeUtil::HumanString(operand_shape));
     }
@@ -2589,7 +2632,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
         operand_shape.dimensions(i) != 1) {
       return InvalidArgument(
           "Input dimension should be either 1 or equal to the output dimension "
-          "it's broadcasting into; the %lldth operand dimension is %lld, the "
+          "it is broadcasting into; the %lldth operand dimension is %lld, the "
           "%lldth output dimension is %lld.",
           i, operand_shape.dimensions(i), broadcast_dimensions[i],
           output_shape.dimensions(broadcast_dimensions[i]));
diff --git a/tensorflow/compiler/xla/service/shape_inference.h b/tensorflow/compiler/xla/service/shape_inference.h
index 7d39ef38e0..acb071ab18 100644
--- a/tensorflow/compiler/xla/service/shape_inference.h
+++ b/tensorflow/compiler/xla/service/shape_inference.h
@@ -116,6 +116,10 @@ class ShapeInference {
   static StatusOr<Shape> InferFftShape(const Shape& in, FftType fft_type,
                                        absl::Span<const int64> fft_length);
 
+  // Infers the shape produced by the given triangular solve operation.
+  static StatusOr<Shape> InferTriangularSolveShape(
+      const Shape& a, const Shape& b, const TriangularSolveOptions& options);
+
   // Infers the shape produced by a cross replica sum with the given operand
   // shapes.
   static StatusOr<Shape> InferAllReduceShape(
diff --git a/tensorflow/compiler/xla/client/lib/triangular_solve.cc b/tensorflow/compiler/xla/service/triangular_solve_expander.cc
similarity index 85%
rename from tensorflow/compiler/xla/client/lib/triangular_solve.cc
rename to tensorflow/compiler/xla/service/triangular_solve_expander.cc
index 1515e9ef7a..b26cdc1db5 100644
--- a/tensorflow/compiler/xla/client/lib/triangular_solve.cc
+++ b/tensorflow/compiler/xla/service/triangular_solve_expander.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/compiler/xla/client/lib/triangular_solve.h"
+#include "tensorflow/compiler/xla/service/triangular_solve_expander.h"
 
 #include <memory>
 #include <vector>
@@ -33,6 +33,8 @@ limitations under the License.
 
 namespace xla {
 
+namespace {
+
 // Get the diagonal blocks of the coefficient matrix
 XlaOp DiagonalBlocks(XlaOp a, int64 block_size) {
   XlaBuilder* builder = a.builder();
@@ -345,9 +347,10 @@ XlaOp SolveWithInvertedDiagonalBlocks(XlaOp a, XlaOp b, XlaOp inv_diag_blocks,
   });
 }
 
-XlaOp TriangularSolve(XlaOp a, XlaOp b, bool left_side, bool lower,
-                      bool transpose_a, bool conjugate_a, bool unit_diagonal,
-                      int64 block_size, PrecisionConfig::Precision precision) {
+XlaOp BuildTriangularSolve(XlaOp a, XlaOp b, bool left_side, bool lower,
+                           bool transpose_a, bool conjugate_a,
+                           bool unit_diagonal, int64 block_size,
+                           PrecisionConfig::Precision precision) {
   XlaBuilder* builder = a.builder();
   return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
     TF_ASSIGN_OR_RETURN(Shape a_shape, builder->GetShape(a));
@@ -436,4 +439,66 @@ XlaOp TriangularSolve(XlaOp a, XlaOp b, bool left_side, bool lower,
   });
 }
 
+}  // namespace
+
+bool TriangularSolveExpander::InstructionMatchesPattern(
+    HloInstruction* instruction) {
+  return instruction->opcode() == HloOpcode::kTriangularSolve;
+}
+
+StatusOr<HloInstruction*> TriangularSolveExpander::ExpandInstruction(
+    HloInstruction* instruction) {
+  const TriangularSolveOptions& options =
+      instruction->triangular_solve_options();
+  const string name = absl::StrFormat(
+      "xla.triangular_solve_%s_%s_%s_%s_%s_%s",
+      instruction->operand(0)->shape().ToString(),
+      instruction->operand(1)->shape().ToString(),
+      options.left_side() ? "left" : "right",
+      options.lower() ? "lower" : "upper",
+      TriangularSolveOptions_Transpose_Name(options.transpose_a()),
+      options.unit_diagonal() ? "unit" : "nonunit");
+
+  HloModule* module = instruction->parent()->parent();
+
+  HloComputation*& computation =
+      computation_cache_.emplace(name, nullptr).first->second;
+  if (!computation) {
+    // Builds a new expansion.
+    //
+    // We do something unusual here: we build the computation using the
+    // XlaBuilder API, which is nominally an XLA client API. We do this because
+    // the external APIs for building complicated computations (XlaBuilder)
+    // are much more ergonomic than the internal ones. As it turns out,
+    // XlaBuilder isn't really a client API—what it does is build a
+    // HloModuleProto protocol buffer, that we can then deserialize and clone
+    // into our HloModule. Ideally we would avoid the protocol buffer step;
+    // that is left as an exercise for future work.
+    XlaBuilder builder(name);
+    XlaOp a = Parameter(&builder, 0, instruction->operand(0)->shape(), "a");
+    XlaOp b = Parameter(&builder, 1, instruction->operand(1)->shape(), "b");
+    bool transpose_a =
+        options.transpose_a() != TriangularSolveOptions::NO_TRANSPOSE;
+    bool conjugate_a = options.transpose_a() == TriangularSolveOptions::ADJOINT;
+
+    BuildTriangularSolve(a, b, options.left_side(), options.lower(),
+                         transpose_a, conjugate_a, options.unit_diagonal(),
+                         /*block_size=*/128,
+                         /*precision=*/PrecisionConfig::HIGHEST);
+    TF_ASSIGN_OR_RETURN(XlaComputation xla_computation, builder.Build());
+
+    TF_ASSIGN_OR_RETURN(ProgramShape program_shape,
+                        xla_computation.GetProgramShape());
+    HloModuleConfig config(program_shape);
+    TF_ASSIGN_OR_RETURN(auto new_module, HloModule::CreateFromProto(
+                                             xla_computation.proto(), config));
+    HloCloneContext context(module);
+    computation =
+        module->DeepCloneComputation(new_module->entry_computation(), &context);
+  }
+
+  return instruction->parent()->AddInstruction(HloInstruction::CreateCall(
+      instruction->shape(), instruction->operands(), computation));
+}
+
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/triangular_solve_expander.h b/tensorflow/compiler/xla/service/triangular_solve_expander.h
new file mode 100644
index 0000000000..be2374ef8c
--- /dev/null
+++ b/tensorflow/compiler/xla/service/triangular_solve_expander.h
@@ -0,0 +1,43 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_TRIANGULAR_SOLVE_EXPANDER_H_
+#define TENSORFLOW_COMPILER_XLA_SERVICE_TRIANGULAR_SOLVE_EXPANDER_H_
+
+#include "absl/container/flat_hash_map.h"
+#include "tensorflow/compiler/xla/service/op_expander_pass.h"
+
+namespace xla {
+
+class TriangularSolveExpander : public OpExpanderPass {
+ public:
+  absl::string_view name() const override {
+    return "triangular_solve_expander";
+  }
+
+ protected:
+  bool InstructionMatchesPattern(HloInstruction* instruction) override;
+
+  StatusOr<HloInstruction*> ExpandInstruction(
+      HloInstruction* instruction) override;
+
+ private:
+  // Mapping from op signatures to existing computations.
+  absl::flat_hash_map<string, HloComputation*> computation_cache_;
+};
+
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_TRIANGULAR_SOLVE_EXPANDER_H_
diff --git a/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc b/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc
index 5e505aaf02..cc82e9bb02 100644
--- a/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc
+++ b/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc
@@ -699,6 +699,8 @@ bool TuplePointsToAnalysis::HasUniqueFusedUseOfOperandAt(
 // (4) The 'user' of 'operand' is DynamicUpdateSlice or While at operand index
 //     0.
 // (5) The 'user' of 'operand' is Sort, and it is the only user.
+// (6) The 'user' of 'operand' is TriangularSolve, it is the second operand,
+//     and it is the only user.
 //
 // (2) and (3) can only be determined if points-to analysis is available.
 bool TuplePointsToAnalysis::CanShareOperandBufferWithUser(
@@ -779,6 +781,14 @@ bool TuplePointsToAnalysis::CanShareOperandBufferWithUser(
     std::vector<int64> operand_indices = user->OperandIndices(operand);
     return operand_indices.size() == 1 && user_index[0] == operand_indices[0];
   }
+  if (user->opcode() == HloOpcode::kTriangularSolve) {
+    // Only valid if there are no other users.
+    if (operand->users().size() != 1) {
+      return false;
+    }
+    std::vector<int64> operand_indices = user->OperandIndices(operand);
+    return operand_indices.size() == 1 && operand_indices[0] == 1;
+  }
   if (user->opcode() == HloOpcode::kCall) {
     // TODO(b/62548313): Remove when buffer assignment is module scoped and
     // does not assign buffers to calls.
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index e8e779fb2a..9a59371074 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -2155,3 +2155,26 @@ xla_test(
         "//tensorflow/compiler/xla:test",
     ],
 )
+
+xla_test(
+    name = "triangular_solve_test",
+    srcs = ["triangular_solve_test.cc"],
+    tags = [
+        "enable_for_xla_interpreter",
+        "noasan",  # sometimes times out, http://b/78650012
+    ],
+    deps = [
+        "//tensorflow/compiler/xla:array2d",
+        "//tensorflow/compiler/xla:literal",
+        "//tensorflow/compiler/xla:statusor",
+        "//tensorflow/compiler/xla:test",
+        "//tensorflow/compiler/xla:types",
+        "//tensorflow/compiler/xla/client:xla_builder",
+        "//tensorflow/compiler/xla/client/lib:math",
+        "//tensorflow/compiler/xla/client/lib:matrix",
+        "//tensorflow/compiler/xla/tests:client_library_test_base",
+        "//tensorflow/compiler/xla/tests:literal_test_util",
+        "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+        "//tensorflow/core:test",
+    ],
+)
diff --git a/tensorflow/compiler/xla/client/lib/triangular_solve_test.cc b/tensorflow/compiler/xla/tests/triangular_solve_test.cc
similarity index 88%
rename from tensorflow/compiler/xla/client/lib/triangular_solve_test.cc
rename to tensorflow/compiler/xla/tests/triangular_solve_test.cc
index b333ffa917..24ab12136f 100644
--- a/tensorflow/compiler/xla/client/lib/triangular_solve_test.cc
+++ b/tensorflow/compiler/xla/tests/triangular_solve_test.cc
@@ -13,8 +13,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/compiler/xla/client/lib/triangular_solve.h"
-
 #include <memory>
 #include <numeric>
 #include <vector>
@@ -110,9 +108,8 @@ XLA_TEST_F(TriangularSolveTest, EmptyArrays) {
       CreateR2Parameter<float>(Array2D<float>(0, 10), 1, "b", &builder, &b);
   TriangularSolve(a, b,
                   /*left_side=*/true, /*lower=*/true,
-                  /*transpose_a=*/true, /*conjugate_a=*/false,
                   /*unit_diagonal=*/false,
-                  /*block_size=*/2);
+                  /*transpose_a=*/TriangularSolveOptions::TRANSPOSE);
 
   ComputeAndCompareR2<float>(&builder, Array2D<float>(0, 10),
                              {a_data.get(), b_data.get()});
@@ -126,9 +123,8 @@ XLA_TEST_F(TriangularSolveTest, SimpleRightLowerTranspose) {
   auto b_data = CreateR2Parameter<float>(BValsRight(), 1, "b", &builder, &b);
   TriangularSolve(a, b,
                   /*left_side=*/false, /*lower=*/true,
-                  /*transpose_a=*/true, /*conjugate_a=*/false,
                   /*unit_diagonal=*/false,
-                  /*block_size=*/2);
+                  /*transpose_a=*/TriangularSolveOptions::TRANSPOSE);
 
   Array2D<float> expected({
       {0.5, 0.08333334, 0.04629629, 0.03367003},
@@ -148,9 +144,8 @@ XLA_TEST_F(TriangularSolveTest, SimpleRightLowerNotranspose) {
   auto b_data = CreateR2Parameter<float>(BValsRight(), 1, "b", &builder, &b);
   TriangularSolve(a, b,
                   /*left_side=*/false, /*lower=*/true,
-                  /*transpose_a=*/false, /*conjugate_a=*/false,
                   /*unit_diagonal=*/false,
-                  /*block_size=*/2);
+                  /*transpose_a=*/TriangularSolveOptions::NO_TRANSPOSE);
 
   Array2D<float> expected({
       {-0.16414141, -0.06902357, -0.07070707, 0.36363636},
@@ -170,9 +165,8 @@ XLA_TEST_F(TriangularSolveTest, SimpleRightUpperTranspose) {
   auto b_data = CreateR2Parameter<float>(BValsRight(), 1, "b", &builder, &b);
   TriangularSolve(a, b,
                   /*left_side=*/false, /*lower=*/false,
-                  /*transpose_a=*/true, /*conjugate_a=*/false,
                   /*unit_diagonal=*/false,
-                  /*block_size=*/2);
+                  /*transpose_a=*/TriangularSolveOptions::TRANSPOSE);
 
   Array2D<float> expected({
       {-0.16414141, -0.06902357, -0.07070707, 0.36363636},
@@ -192,9 +186,8 @@ XLA_TEST_F(TriangularSolveTest, SimpleRightUpperNotranspose) {
   auto b_data = CreateR2Parameter<float>(BValsRight(), 1, "b", &builder, &b);
   TriangularSolve(a, b,
                   /*left_side=*/false, /*lower=*/false,
-                  /*transpose_a=*/false, /*conjugate_a=*/false,
                   /*unit_diagonal=*/false,
-                  /*block_size=*/2);
+                  /*transpose_a=*/TriangularSolveOptions::NO_TRANSPOSE);
 
   Array2D<float> expected({
       {0.5, 0.08333334, 0.04629629, 0.03367003},
@@ -214,9 +207,8 @@ XLA_TEST_F(TriangularSolveTest, SimpleLeftLowerTranspose) {
   auto b_data = CreateR2Parameter<float>(BValsLeft(), 1, "b", &builder, &b);
   TriangularSolve(a, b,
                   /*left_side=*/true, /*lower=*/true,
-                  /*transpose_a=*/true, /*conjugate_a=*/false,
                   /*unit_diagonal=*/false,
-                  /*block_size=*/2);
+                  /*transpose_a=*/TriangularSolveOptions::TRANSPOSE);
 
   Array2D<float> expected({
       {-0.89646465, -0.69444444, -0.49242424},
@@ -237,9 +229,8 @@ XLA_TEST_F(TriangularSolveTest, SimpleLeftLowerNotranspose) {
   auto b_data = CreateR2Parameter<float>(BValsLeft(), 1, "b", &builder, &b);
   TriangularSolve(a, b,
                   /*left_side=*/true, /*lower=*/true,
-                  /*transpose_a=*/false, /*conjugate_a=*/false,
                   /*unit_diagonal=*/false,
-                  /*block_size=*/2);
+                  /*transpose_a=*/TriangularSolveOptions::NO_TRANSPOSE);
 
   Array2D<float> expected({
       {0.5, 1.0, 1.5},
@@ -261,9 +252,8 @@ XLA_TEST_F(TriangularSolveTest, SimpleLeftLowerNoTransposeUnitDiagonal) {
   auto b_data = CreateR2Parameter<float>(BValsLeft(), 1, "b", &builder, &b);
   TriangularSolve(a, b,
                   /*left_side=*/true, /*lower=*/true,
-                  /*transpose_a=*/false, /*conjugate_a=*/false,
                   /*unit_diagonal=*/true,
-                  /*block_size=*/2);
+                  /*transpose_a=*/TriangularSolveOptions::NO_TRANSPOSE);
 
   Array2D<float> expected(
       {{1., 2., 3.}, {1., -1., -3.}, {-4., 7., 18.}, {37., -61., -159.}});
@@ -280,9 +270,8 @@ XLA_TEST_F(TriangularSolveTest, SimpleLeftLowerNotransposeIrregularblock) {
   auto b_data = CreateR2Parameter<float>(BValsLeft(), 1, "b", &builder, &b);
   TriangularSolve(a, b,
                   /*left_side=*/true, /*lower=*/true,
-                  /*transpose_a=*/false, /*conjugate_a=*/false,
                   /*unit_diagonal=*/false,
-                  /*block_size=*/3);
+                  /*transpose_a=*/TriangularSolveOptions::NO_TRANSPOSE);
 
   Array2D<float> expected({
       {0.5, 1.0, 1.5},
@@ -303,9 +292,8 @@ XLA_TEST_F(TriangularSolveTest, SimpleLeftUpperTranspose) {
   auto b_data = CreateR2Parameter<float>(BValsLeft(), 1, "b", &builder, &b);
   TriangularSolve(a, b,
                   /*left_side=*/true, /*lower=*/false,
-                  /*transpose_a=*/true, /*conjugate_a=*/false,
                   /*unit_diagonal=*/false,
-                  /*block_size=*/2);
+                  /*transpose_a=*/TriangularSolveOptions::TRANSPOSE);
 
   Array2D<float> expected({
       {0.5, 1.0, 1.5},
@@ -326,9 +314,8 @@ XLA_TEST_F(TriangularSolveTest, SimpleLeftUpperNotranspose) {
   auto b_data = CreateR2Parameter<float>(BValsLeft(), 1, "b", &builder, &b);
   TriangularSolve(a, b,
                   /*left_side=*/true, /*lower=*/false,
-                  /*transpose_a=*/false, /*conjugate_a=*/false,
                   /*unit_diagonal=*/false,
-                  /*block_size=*/2);
+                  /*transpose_a=*/TriangularSolveOptions::NO_TRANSPOSE);
 
   Array2D<float> expected({
       {-0.89646465, -0.69444444, -0.49242424},
@@ -350,9 +337,8 @@ XLA_TEST_F(TriangularSolveTest, SimpleLeftUpperNotransposeUnitDiagonal) {
   auto b_data = CreateR2Parameter<float>(BValsLeft(), 1, "b", &builder, &b);
   TriangularSolve(a, b,
                   /*left_side=*/true, /*lower=*/false,
-                  /*transpose_a=*/false, /*conjugate_a=*/false,
                   /*unit_diagonal=*/true,
-                  /*block_size=*/2);
+                  /*transpose_a=*/TriangularSolveOptions::NO_TRANSPOSE);
 
   Array2D<float> expected({{-1402., -1538., -1674.},
                            {575., 631., 687.},
@@ -373,9 +359,8 @@ XLA_TEST_F(TriangularSolveTest, SimpleRightLowerTransposeConjugate) {
       CreateR2Parameter<complex64>(BValsRightComplex(), 1, "b", &builder, &b);
   TriangularSolve(a, b,
                   /*left_side=*/false, /*lower=*/true,
-                  /*transpose_a=*/true, /*conjugate_a=*/true,
                   /*unit_diagonal=*/false,
-                  /*block_size=*/2);
+                  /*transpose_a=*/TriangularSolveOptions::ADJOINT);
 
   Array2D<complex64> expected({
       {0.5, complex64(0.08333333, 0.08333333),
@@ -400,9 +385,8 @@ XLA_TEST_F(TriangularSolveTest, SimpleLeftUpperTransposeNoconjugate) {
       CreateR2Parameter<complex64>(BValsLeftComplex(), 1, "b", &builder, &b);
   TriangularSolve(a, b,
                   /*left_side=*/true, /*lower=*/false,
-                  /*transpose_a=*/true, /*conjugate_a=*/false,
                   /*unit_diagonal=*/false,
-                  /*block_size=*/2);
+                  /*transpose_a=*/TriangularSolveOptions::TRANSPOSE);
 
   Array2D<complex64> expected({
       {0.5, 1., 1.5},
@@ -436,12 +420,12 @@ XLA_TEST_F(TriangularSolveTest, BatchedLeftUpper) {
   XlaOp a, b;
   auto a_data = CreateR3Parameter<float>(avals, 0, "a", &builder, &a);
   auto b_data = CreateR3Parameter<float>(bvals, 1, "b", &builder, &b);
-  BatchDot(ConstantR3FromArray3D(&builder, avals),
-           TriangularSolve(a, b,
-                           /*left_side=*/true, /*lower=*/false,
-                           /*transpose_a=*/false, /*conjugate_a=*/false,
-                           /*unit_diagonal=*/false,
-                           /*block_size=*/2));
+  BatchDot(
+      ConstantR3FromArray3D(&builder, avals),
+      TriangularSolve(a, b,
+                      /*left_side=*/true, /*lower=*/false,
+                      /*unit_diagonal=*/false,
+                      /*transpose_a=*/TriangularSolveOptions::NO_TRANSPOSE));
 
   ComputeAndCompareR3<float>(&builder, bvals, {a_data.get(), b_data.get()},
                              ErrorSpec(1e-2, 1e-2));
@@ -451,7 +435,7 @@ struct TriangularSolveTestSpec {
   int m, n;  // A is mxm, B is mxn
   bool left_side;
   bool lower;
-  bool transpose_a;
+  TriangularSolveOptions::Transpose transpose_a;
 };
 
 class TriangularSolveParametricTest
@@ -477,11 +461,11 @@ XLA_TEST_P(TriangularSolveParametricTest, Random) {
   XlaOp a, b;
   auto a_data = CreateR2Parameter<float>(avals, 0, "a", &builder, &a);
   auto b_data = CreateR2Parameter<float>(bvals, 1, "b", &builder, &b);
-  auto x = TriangularSolve(a, b, spec.left_side, spec.lower, spec.transpose_a,
-                           /*conjugate_a=*/false, /*unit_diagonal=*/false,
-                           /*block_size=*/3);
+  auto x = TriangularSolve(a, b, spec.left_side, spec.lower,
+                           /*unit_diagonal=*/false, spec.transpose_a);
   auto a_tri = Triangle(a, spec.lower);
-  a_tri = MaybeTransposeInMinorDims(a_tri, spec.transpose_a);
+  a_tri = MaybeTransposeInMinorDims(
+      a_tri, spec.transpose_a != TriangularSolveOptions::NO_TRANSPOSE);
   if (spec.left_side) {
     BatchDot(a_tri, x);
   } else {
@@ -498,7 +482,9 @@ std::vector<TriangularSolveTestSpec> TriangularSolveTests() {
     for (int n : {5, 10}) {
       for (bool left_side : {false, true}) {
         for (bool lower : {false, true}) {
-          for (bool transpose_a : {false, true}) {
+          for (TriangularSolveOptions::Transpose transpose_a :
+               {TriangularSolveOptions::NO_TRANSPOSE,
+                TriangularSolveOptions::TRANSPOSE}) {
             specs.push_back({m, n, left_side, lower, transpose_a});
           }
         }
diff --git a/tensorflow/compiler/xla/xla_data.proto b/tensorflow/compiler/xla/xla_data.proto
index a64e2f5df5..4e127356a9 100644
--- a/tensorflow/compiler/xla/xla_data.proto
+++ b/tensorflow/compiler/xla/xla_data.proto
@@ -545,6 +545,26 @@ enum RandomDistribution {
   // Next: 4
 }
 
+message TriangularSolveOptions {
+  // If true, solves ax = b. If false, solves xa = b.
+  bool left_side = 1;
+
+  // If true, 'a' is lower triangular. If false, 'a' is upper triangular.
+  bool lower = 2;
+
+  // If true, the diagonal elements of 'a' are assumed to be 1 and not accessed.
+  bool unit_diagonal = 3;
+
+  // Should we transpose or use the adjoint of 'a'?
+  enum Transpose {
+    TRANSPOSE_INVALID = 0;
+    NO_TRANSPOSE = 1;  // Don't transpose 'a'.
+    TRANSPOSE = 2;     // Transpose 'a'.
+    ADJOINT = 3;       // Complex conjugate and transpose 'a'.
+  };
+  Transpose transpose_a = 4;
+}
+
 message OpSharding {
   enum Type {
     // This sharding is replicated across all devices (implies maximal,
-- 
GitLab


From 5800d6be717be80a668a8f7635a6d8c706100388 Mon Sep 17 00:00:00 2001
From: Smit Hinsu <hinsu@google.com>
Date: Thu, 7 Feb 2019 19:04:02 -0800
Subject: [PATCH 2126/2345] Remove TestParams class constructors in TensorRT
 tests

Aggregate initialization is available by default

PiperOrigin-RevId: 232988505
---
 .../tf2tensorrt/convert/convert_nodes_test.cc | 120 +++++++-----------
 1 file changed, 43 insertions(+), 77 deletions(-)

diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
index 77221f6d9a..bb1341ada3 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
@@ -1439,11 +1439,6 @@ TEST_F(OpConverterTest, ConvertReshape) {
   }
 
   struct TestParams {
-    TestParams(int input_batch_size, const std::vector<int>& input_tensor_dims,
-               const std::vector<int>& input_shape)
-        : batch_size(input_batch_size),
-          tensor_dims(input_tensor_dims),
-          shape(input_shape) {}
     int batch_size;
     std::vector<int> tensor_dims;
     std::vector<int> shape;
@@ -2381,11 +2376,6 @@ TEST_F(OpConverterTest, ConvertExpandDims) {
   }
 
   struct TestParams {
-    TestParams(const std::vector<int>& input_dims, int axis,
-               const std::vector<int>& expected_output_dims)
-        : input_dims(input_dims),
-          axis(axis),
-          expected_output_dims(expected_output_dims) {}
     std::vector<int> input_dims;
     int axis;
     std::vector<int> expected_output_dims;
@@ -2498,11 +2488,6 @@ TEST_F(OpConverterTest, ConvertSqueeze) {
   }
 
   struct TestParams {
-    TestParams(const std::vector<int>& input_dims, const std::vector<int>& axis,
-               const std::vector<int>& expected_output_dims)
-        : input_dims(input_dims),
-          axis(axis),
-          expected_output_dims(expected_output_dims) {}
     std::vector<int> input_dims;
     std::vector<int> axis;
     std::vector<int> expected_output_dims;
@@ -2675,29 +2660,6 @@ TEST_F(OpConverterTest, ConvertStridedSlice) {
   }
 
   struct TestParams {
-    TestParams(const std::vector<int>& input_dims,
-               const std::vector<int>& expected_output_dims,
-               const std::vector<int>& begin, const std::vector<int>& end,
-               const std::vector<int>& begin_mask,
-               const std::vector<int>& end_mask,
-               const std::vector<int>& expected_output)
-        : input_dims(input_dims),
-          expected_output_dims(expected_output_dims),
-          begin(begin),
-          end(end),
-          expected_output(expected_output) {
-      // Masks are provided in terms of vectors for readability. Convert them to
-      // binary here.
-      this->begin_mask = 0;
-      for (int i = 0; i < begin_mask.size(); i++) {
-        if (begin_mask[i]) this->begin_mask |= (1 << i);
-      }
-      this->end_mask = 0;
-      for (int i = 0; i < end_mask.size(); i++) {
-        if (end_mask[i]) this->end_mask |= (1 << i);
-      }
-    }
-
     std::vector<int> input_dims;
     std::vector<int> expected_output_dims;
     std::vector<int> begin;
@@ -2707,87 +2669,112 @@ TEST_F(OpConverterTest, ConvertStridedSlice) {
     std::vector<int> expected_output;
   };
 
+  auto get_mask = [](const std::vector<int>& mask) {
+    int result = 0;
+    for (int i = 0; i < mask.size(); i++) {
+      if (mask[i]) result += (1 << i);
+    }
+    return result;
+  };
+
   // Ok.
   const int kStridedSliceOKCases = 18;
   TestParams ok_params[kStridedSliceOKCases] = {
       // 2D Crop.
       TestParams{/*input_dims=*/{1, 2, 3}, /*expected_output_dims=*/{1, 1, 2},
                  /*begin=*/{0, 0, 0, 0}, /*end=*/{0, 0, 1, 2},
-                 /*begin_mask=*/{0, 0, 0, 0}, /*end_mask=*/{1, 1, 0, 0},
+                 /*begin_mask=*/get_mask({0, 0, 0, 0}),
+                 /*end_mask=*/get_mask({1, 1, 0, 0}),
                  /*expected_output=*/{1, 2}},
       TestParams{/*input_dims=*/{1, 2, 3}, /*expected_output_dims=*/{1, 1, 2},
                  /*begin=*/{0, 0, 1, 1}, /*end=*/{0, 0, 0, 0},
-                 /*begin_mask=*/{0, 0, 0, 0}, /*end_mask=*/{1, 1, 1, 1},
+                 /*begin_mask=*/get_mask({0, 0, 0, 0}),
+                 /*end_mask=*/get_mask({1, 1, 1, 1}),
                  /*expected_output=*/{5, 6}},
       TestParams{/*input_dims=*/{1, 2, 3}, /*expected_output_dims=*/{1, 1, 2},
                  /*begin=*/{0, 0, 1, 1}, /*end=*/{0, 1, 2, 3},
-                 /*begin_mask=*/{0, 0, 0, 0}, /*end_mask=*/{1, 1, 0, 0},
+                 /*begin_mask=*/get_mask({0, 0, 0, 0}),
+                 /*end_mask=*/get_mask({1, 1, 0, 0}),
                  /*expected_output=*/{5, 6}},
       // 2D Crop, with transpose.
       TestParams{/*input_dims=*/{2, 3, 1}, /*expected_output_dims=*/{1, 2, 1},
                  /*begin=*/{0, 0, 0, 0}, /*end=*/{0, 1, 2, 1},
-                 /*begin_mask=*/{0, 0, 0, 0}, /*end_mask=*/{1, 0, 0, 0},
+                 /*begin_mask=*/get_mask({0, 0, 0, 0}),
+                 /*end_mask=*/get_mask({1, 0, 0, 0}),
                  /*expected_output=*/{1, 2}},
       TestParams{/*input_dims=*/{2, 3, 1}, /*expected_output_dims=*/{1, 2, 1},
                  /*begin=*/{0, 1, 1, 0}, /*end=*/{0, 2, 3, 1},
-                 /*begin_mask=*/{0, 0, 0, 0}, /*end_mask=*/{1, 0, 0, 0},
+                 /*begin_mask=*/get_mask({0, 0, 0, 0}),
+                 /*end_mask=*/get_mask({1, 0, 0, 0}),
                  /*expected_output=*/{5, 6}},
       TestParams{/*input_dims=*/{2, 1, 3}, /*expected_output_dims=*/{1, 1, 2},
                  /*begin=*/{0, 0, 0, 0}, /*end=*/{0, 1, 1, 2},
-                 /*begin_mask=*/{0, 0, 0, 0}, /*end_mask=*/{1, 0, 0, 0},
+                 /*begin_mask=*/get_mask({0, 0, 0, 0}),
+                 /*end_mask=*/get_mask({1, 0, 0, 0}),
                  /*expected_output=*/{1, 2}},
       TestParams{/*input_dims=*/{2, 1, 3}, /*expected_output_dims=*/{1, 1, 2},
                  /*begin=*/{0, 1, 0, 1}, /*end=*/{0, 2, 1, 3},
-                 /*begin_mask=*/{0, 0, 0, 0}, /*end_mask=*/{1, 0, 0, 0},
+                 /*begin_mask=*/get_mask({0, 0, 0, 0}),
+                 /*end_mask=*/get_mask({1, 0, 0, 0}),
                  /*expected_output=*/{5, 6}},
       // 2D Crop, with reshape.
       TestParams{/*input_dims=*/{2, 3}, /*expected_output_dims=*/{1, 2},
                  /*begin=*/{0, 0, 0}, /*end=*/{0, 1, 2},
-                 /*begin_mask=*/{0, 0, 0}, /*end_mask=*/{1, 0, 0},
+                 /*begin_mask=*/get_mask({0, 0, 0}),
+                 /*end_mask=*/get_mask({1, 0, 0}),
                  /*expected_output=*/{1, 2}},
       TestParams{/*input_dims=*/{2, 3}, /*expected_output_dims=*/{1, 2},
                  /*begin=*/{0, 1, 1}, /*end=*/{0, 0, 0},
-                 /*begin_mask=*/{0, 0, 0}, /*end_mask=*/{1, 1, 1},
+                 /*begin_mask=*/get_mask({0, 0, 0}),
+                 /*end_mask=*/get_mask({1, 1, 1}),
                  /*expected_output=*/{5, 6}},
       // 1D Crop.
       TestParams{/*input_dims=*/{1, 2, 3}, /*expected_output_dims=*/{1, 2, 2},
                  /*begin=*/{0, 0, 0, 0}, /*end=*/{0, 0, 0, 2},
-                 /*begin_mask=*/{0, 0, 0, 0}, /*end_mask=*/{1, 1, 1, 0},
+                 /*begin_mask=*/get_mask({0, 0, 0, 0}),
+                 /*end_mask=*/get_mask({1, 1, 1, 0}),
                  /*expected_output=*/{1, 2, 4, 5}},
       TestParams{/*input_dims=*/{1, 2, 3}, /*expected_output_dims=*/{1, 1, 3},
                  /*begin=*/{0, 0, 1, 0}, /*end=*/{0, 0, 0, 0},
-                 /*begin_mask=*/{0, 0, 0, 0}, /*end_mask=*/{1, 1, 1, 1},
+                 /*begin_mask=*/get_mask({0, 0, 0, 0}),
+                 /*end_mask=*/get_mask({1, 1, 1, 1}),
                  /*expected_output=*/{4, 5, 6}},
       // 1D Crop, with transpose.
       TestParams{/*input_dims=*/{2, 3, 1}, /*expected_output_dims=*/{1, 3, 1},
                  /*begin=*/{0, 0, 0, 0}, /*end=*/{0, 1, 0, 0},
-                 /*begin_mask=*/{0, 0, 0, 0}, /*end_mask=*/{1, 0, 1, 1},
+                 /*begin_mask=*/get_mask({0, 0, 0, 0}),
+                 /*end_mask=*/get_mask({1, 0, 1, 1}),
                  /*expected_output=*/{1, 2, 3}},
       TestParams{/*input_dims=*/{2, 3, 1}, /*expected_output_dims=*/{1, 3, 1},
                  /*begin=*/{0, 1, 0, 0}, /*end=*/{0, 0, 0, 0},
-                 /*begin_mask=*/{0, 0, 0, 0}, /*end_mask=*/{1, 1, 1, 1},
+                 /*begin_mask=*/get_mask({0, 0, 0, 0}),
+                 /*end_mask=*/get_mask({1, 1, 1, 1}),
                  /*expected_output=*/{4, 5, 6}},
       // 1D Crop, with reshape.
       TestParams{/*input_dims=*/{6}, /*expected_output_dims=*/{3},
                  /*begin=*/{0, 0}, /*end=*/{0, 3},
-                 /*begin_mask=*/{0, 0}, /*end_mask=*/{1, 0},
+                 /*begin_mask=*/get_mask({0, 0}), /*end_mask=*/get_mask({1, 0}),
                  /*expected_output=*/{1, 2, 3}},
       TestParams{/*input_dims=*/{1, 6}, /*expected_output_dims=*/{1, 3},
                  /*begin=*/{0, 0, 2}, /*end=*/{0, 0, 5},
-                 /*begin_mask=*/{0, 0, 0}, /*end_mask=*/{1, 1, 0},
+                 /*begin_mask=*/get_mask({0, 0, 0}),
+                 /*end_mask=*/get_mask({1, 1, 0}),
                  /*expected_output=*/{3, 4, 5}},
       TestParams{/*input_dims=*/{6, 1}, /*expected_output_dims=*/{3, 1},
                  /*begin=*/{0, 2, 0}, /*end=*/{0, 5, 0},
-                 /*begin_mask=*/{0, 0, 0}, /*end_mask=*/{1, 0, 1},
+                 /*begin_mask=*/get_mask({0, 0, 0}),
+                 /*end_mask=*/get_mask({1, 0, 1}),
                  /*expected_output=*/{3, 4, 5}},
       // Negative axis.
       TestParams{/*input_dims=*/{6, 1}, /*expected_output_dims=*/{3, 1},
                  /*begin=*/{0, -6, 0}, /*end=*/{0, -3, 0},
-                 /*begin_mask=*/{0, 0, 0}, /*end_mask=*/{1, 0, 1},
+                 /*begin_mask=*/get_mask({0, 0, 0}),
+                 /*end_mask=*/get_mask({1, 0, 1}),
                  /*expected_output=*/{1, 2, 3}},
       TestParams{/*input_dims=*/{6, 1}, /*expected_output_dims=*/{5, 1},
                  /*begin=*/{0, 0, 0}, /*end=*/{0, -1, 0},
-                 /*begin_mask=*/{0, 0, 0}, /*end_mask=*/{1, 0, 1},
+                 /*begin_mask=*/get_mask({0, 0, 0}),
+                 /*end_mask=*/get_mask({1, 0, 1}),
                  /*expected_output=*/{1, 2, 3, 4, 5}},
   };
 
@@ -2956,27 +2943,6 @@ TEST_F(OpConverterTest, ConvertConv2D) {
   }
 
   struct TestParams {
-    TestParams(const std::vector<int>& input_dims,
-               const std::vector<float>& input,
-               const std::vector<int>& filter_dims,
-               const std::vector<float>& filter,
-               const std::vector<int>& strides, const string& padding,
-               const string& data_format, const std::vector<int>& dilations,
-               bool is_conv2d_backprop_input,
-               const std::vector<int>& expected_output_dims,
-               const std::vector<float>& expected_output)
-        : input_dims(input_dims),
-          input(input),
-          filter_dims(filter_dims),
-          filter(filter),
-          strides(strides),
-          padding(padding),
-          data_format(data_format),
-          dilations(dilations),
-          is_conv2d_backprop_input(is_conv2d_backprop_input),
-          expected_output_dims(expected_output_dims),
-          expected_output(expected_output) {}
-
     std::vector<int> input_dims;
     std::vector<float> input;
     std::vector<int> filter_dims;
-- 
GitLab


From 063c51df5dfbca6bc0805118819aa8184292966a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 7 Feb 2019 19:15:33 -0800
Subject: [PATCH 2127/2345] Get rid of remaining test_session().

PiperOrigin-RevId: 232989493
---
 .../gan/python/eval/python/classifier_metrics_test.py       | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py
index bd17571a05..bc7c1057b4 100644
--- a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py
+++ b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py
@@ -365,7 +365,7 @@ class ClassifierMetricsTest(test.TestCase, parameterized.TestCase):
     unused_image = array_ops.zeros([2, 299, 299, 3])
     incscore = _run_with_mock(classifier_metrics.inception_score, unused_image)
 
-    with self.test_session(use_gpu=True) as sess:
+    with self.cached_session(use_gpu=True) as sess:
       incscore_np = sess.run(incscore, {'concat:0': logits})
 
     self.assertAllClose(_expected_inception_score(logits), incscore_np)
@@ -473,7 +473,7 @@ class ClassifierMetricsTest(test.TestCase, parameterized.TestCase):
         classifier_fn=lambda x: x,
         max_block_size=600)
 
-    with self.test_session() as sess:
+    with self.cached_session() as sess:
       actual_kid, actual_std = sess.run(kid_op)
 
     expected_kid, expected_std = _expected_kid_and_std(test_pool_real_a,
@@ -500,7 +500,7 @@ class ClassifierMetricsTest(test.TestCase, parameterized.TestCase):
         max_block_size=max_block_size)
 
     for block_size in [50, 512, 1000]:
-      with self.test_session() as sess:
+      with self.cached_session() as sess:
         actual_kid, actual_std = sess.run(kid_op, {max_block_size: block_size})
 
       expected_kid, expected_std = _expected_kid_and_std(
-- 
GitLab


From 228421cebf60190bf2b8b300b34941e0a837757b Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne <reedwm@google.com>
Date: Thu, 7 Feb 2019 21:38:45 -0800
Subject: [PATCH 2128/2345] Fix "No duplicates allowed" error message

PiperOrigin-RevId: 233001561
---
 tensorflow/python/distribute/mirrored_strategy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/distribute/mirrored_strategy.py b/tensorflow/python/distribute/mirrored_strategy.py
index df96f83a58..96c7191652 100644
--- a/tensorflow/python/distribute/mirrored_strategy.py
+++ b/tensorflow/python/distribute/mirrored_strategy.py
@@ -454,7 +454,7 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
     assert devices, "Must specify at least one device."
     devices = tuple(device_util.resolve(d) for d in devices)
     assert len(set(devices)) == len(devices), (
-        "No duplicates allowed in `devices` argument: %s" % devices)
+        "No duplicates allowed in `devices` argument: %s" % (devices,))
     # TODO(josh11b): Require at least 2 devices?
     self._device_map = values.ReplicaDeviceMap(devices)
     self._input_workers = input_lib.InputWorkers(self._device_map)
-- 
GitLab


From 20bb92cb6a73b895d6d4edda8d4ff48c2c68287c Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Thu, 7 Feb 2019 22:08:25 -0800
Subject: [PATCH 2129/2345] Create int8 transpose.

PiperOrigin-RevId: 233003958
---
 tensorflow/lite/kernels/register.cc       |  4 ++-
 tensorflow/lite/kernels/transpose.cc      |  5 ++++
 tensorflow/lite/kernels/transpose_test.cc | 31 ++++++++++++++++++++---
 tensorflow/lite/toco/tflite/operator.cc   |  6 +++++
 4 files changed, 41 insertions(+), 5 deletions(-)

diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index 355d66f64d..1b4504e7a5 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -238,7 +238,9 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_GATHER, Register_GATHER(),
              /* min_version */ 1,
              /* max_version */ 2);
-  AddBuiltin(BuiltinOperator_TRANSPOSE, Register_TRANSPOSE());
+  AddBuiltin(BuiltinOperator_TRANSPOSE, Register_TRANSPOSE(),
+             /* min_version */ 1,
+             /* max_version */ 2);
   AddBuiltin(BuiltinOperator_MEAN, Register_MEAN());
   AddBuiltin(BuiltinOperator_DIV, Register_DIV());
   AddBuiltin(BuiltinOperator_SUB, Register_SUB());
diff --git a/tensorflow/lite/kernels/transpose.cc b/tensorflow/lite/kernels/transpose.cc
index 7a6d320674..0ef4972d1a 100644
--- a/tensorflow/lite/kernels/transpose.cc
+++ b/tensorflow/lite/kernels/transpose.cc
@@ -117,6 +117,11 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
         TF_LITE_TRANSPOSE(reference_ops, uint8_t);
       }
       break;
+    case kTfLiteInt8:
+      if (kernel_type == kReference) {
+        TF_LITE_TRANSPOSE(reference_ops, int8_t);
+      }
+      break;
     case kTfLiteInt32:
       if (kernel_type == kReference) {
         TF_LITE_TRANSPOSE(reference_ops, int32_t);
diff --git a/tensorflow/lite/kernels/transpose_test.cc b/tensorflow/lite/kernels/transpose_test.cc
index 93df2c81db..7164415920 100644
--- a/tensorflow/lite/kernels/transpose_test.cc
+++ b/tensorflow/lite/kernels/transpose_test.cc
@@ -25,16 +25,17 @@ namespace {
 
 using ::testing::ElementsAreArray;
 
+template <typename T>
 void RunTestPermutation(const std::vector<int>& shape,
                         const std::vector<int>& perms,
-                        std::vector<float>* input_transposed) {
+                        std::vector<T>* input_transposed) {
   // Count elements and allocate output.
   int count = 1;
   for (auto factor : shape) count *= factor;
   input_transposed->resize(count);
 
   // Create the dummy data
-  std::vector<float> input(count);
+  std::vector<T> input(count);
   for (int i = 0; i < input.size(); i++) {
     input[i] = i;
   }
@@ -64,8 +65,8 @@ void RunTestPermutation(const std::vector<int>& shape,
     params.perm[i] = perms[i];
   }
 
-  reference_ops::Transpose<float>(params, input_shape, input.data(),
-                                  output_shape, input_transposed->data());
+  reference_ops::Transpose<T>(params, input_shape, input.data(), output_shape,
+                              input_transposed->data());
 }
 
 TEST(TransposeTest, TestRefOps1D) {
@@ -125,6 +126,28 @@ TEST(TransposeTest, TestRefOps4D) {
   ASSERT_EQ(out, ref);
 }
 
+TEST(TransposeTest, TestRefOps4DInt8) {
+  std::vector<int8_t> out;
+  // Basic 4d.
+  RunTestPermutation({2, 3, 4, 5}, {2, 0, 1, 3}, &out);
+  ASSERT_EQ(
+      out,
+      std::vector<int8_t>(
+          {0,  1,  2,  3,  4,  20, 21, 22, 23, 24, 40,  41,  42,  43,  44,
+           60, 61, 62, 63, 64, 80, 81, 82, 83, 84, 100, 101, 102, 103, 104,
+           5,  6,  7,  8,  9,  25, 26, 27, 28, 29, 45,  46,  47,  48,  49,
+           65, 66, 67, 68, 69, 85, 86, 87, 88, 89, 105, 106, 107, 108, 109,
+           10, 11, 12, 13, 14, 30, 31, 32, 33, 34, 50,  51,  52,  53,  54,
+           70, 71, 72, 73, 74, 90, 91, 92, 93, 94, 110, 111, 112, 113, 114,
+           15, 16, 17, 18, 19, 35, 36, 37, 38, 39, 55,  56,  57,  58,  59,
+           75, 76, 77, 78, 79, 95, 96, 97, 98, 99, 115, 116, 117, 118, 119}));
+  RunTestPermutation({2, 3, 4, 5}, {0, 1, 2, 3}, &out);
+  // Basic identity.
+  std::vector<int8_t> ref(out.size());
+  for (int k = 0; k < ref.size(); k++) ref[k] = k;
+  ASSERT_EQ(out, ref);
+}
+
 class TransposeOpModel : public SingleOpModel {
  public:
   void SetInput(std::initializer_list<float> data) {
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index 8a9cd7fded..bdebee839c 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -820,6 +820,12 @@ class Transpose
                    TocoOperator* op) const override {}
 
   int GetVersion(const OperatorSignature& op_signature) const override {
+    const string& input_name = op_signature.op->inputs[0];
+    const Array& input_array = op_signature.model->GetArray(input_name);
+    // If the op take int8 input, it is version 2.
+    if (input_array.data_type == ArrayDataType::kInt8) {
+      return 2;
+    }
     return 1;
   }
 };
-- 
GitLab


From 4604d1e52351ebc47a535ab82393e9bd0744ce73 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Thu, 7 Feb 2019 22:16:23 -0800
Subject: [PATCH 2130/2345] Move V2 loss reduction to the keras module as the
 distribute lib dependency on it has been removed.

PiperOrigin-RevId: 233004569
---
 tensorflow/python/keras/engine/training.py    | 10 +--
 tensorflow/python/keras/losses.py             | 40 +++++-----
 tensorflow/python/keras/losses_test.py        | 78 +++++++++----------
 tensorflow/python/keras/ops.py                |  4 -
 tensorflow/python/keras/utils/losses_utils.py | 41 ++++++++--
 tensorflow/python/ops/losses/losses_impl.py   | 27 -------
 .../tools/api/generator/api_init_files.bzl    |  1 -
 .../tensorflow.keras.losses.-reduction.pbtxt  |  2 +-
 .../v2/tensorflow.losses.-reduction.pbtxt     | 28 -------
 .../api/golden/v2/tensorflow.losses.pbtxt     |  7 --
 .../tools/api/golden/v2/tensorflow.pbtxt      |  4 -
 tensorflow/tools/compatibility/renames_v2.py  |  1 +
 12 files changed, 100 insertions(+), 143 deletions(-)
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.losses.-reduction.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.losses.pbtxt

diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 725e20beec..ae9992a9b4 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -44,10 +44,9 @@ from tensorflow.python.keras.engine.network import Network
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
 from tensorflow.python.keras.saving import saving_utils
 from tensorflow.python.keras.utils import data_utils
+from tensorflow.python.keras.utils import losses_utils
 from tensorflow.python.keras.utils.generic_utils import slice_arrays
-from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions
 from tensorflow.python.ops import math_ops
-from tensorflow.python.ops.losses import losses_impl
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import optimizer as tf_optimizer_module
 from tensorflow.python.training.checkpointable import base as checkpointable
@@ -476,8 +475,9 @@ class Model(Network):
                 sample_weight = mask
               else:
                 # Update dimensions of weights to match with mask if possible.
-                mask, _, sample_weight = squeeze_or_expand_dimensions(
-                    mask, None, sample_weight)
+                mask, _, sample_weight = (
+                    losses_utils.squeeze_or_expand_dimensions(
+                        mask, None, sample_weight))
                 sample_weight *= mask
 
             output_loss = loss_fn(y_true, y_pred, sample_weight=sample_weight)
@@ -489,7 +489,7 @@ class Model(Network):
 
             # Keep track of stateful result tensor and function for the loss.
             # Reset reduction here as metric wrapper will take care of that.
-            loss_fn.reduction = losses_impl.ReductionV2.NONE
+            loss_fn.reduction = losses_utils.ReductionV2.NONE
             output_loss_metric = metrics_module.SumOverBatchSizeMetricWrapper(
                 loss_fn, name=loss_fn.name)
             result_tensor = self._call_metric_fn(output_loss_metric, y_true,
diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index 366469d9dd..f27c5f5583 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -27,9 +27,9 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import smart_cond
 from tensorflow.python.keras import backend as K
+from tensorflow.python.keras.utils import losses_utils
 from tensorflow.python.keras.utils.generic_utils import deserialize_keras_object
 from tensorflow.python.keras.utils.generic_utils import serialize_keras_object
-from tensorflow.python.keras.utils.losses_utils import compute_weighted_loss
 from tensorflow.python.keras.utils.tf_utils import is_tensor_or_variable
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
@@ -63,7 +63,7 @@ class Loss(object):
   """
 
   def __init__(self,
-               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               reduction=losses_utils.ReductionV2.SUM_OVER_BATCH_SIZE,
                name=None):
     self.reduction = reduction
     self.name = name
@@ -97,7 +97,7 @@ class Loss(object):
     with ops.name_scope(scope_name, format(self.__class__.__name__),
                         (y_pred, y_true, sample_weight)):
       losses = self.call(y_true, y_pred)
-      return compute_weighted_loss(
+      return losses_utils.compute_weighted_loss(
           losses, sample_weight, reduction=self.reduction)
 
   @classmethod
@@ -141,7 +141,7 @@ class LossFunctionWrapper(Loss):
 
   def __init__(self,
                fn,
-               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               reduction=losses_utils.ReductionV2.SUM_OVER_BATCH_SIZE,
                name=None,
                **kwargs):
     super(LossFunctionWrapper, self).__init__(reduction=reduction, name=name)
@@ -192,7 +192,7 @@ class MeanSquaredError(LossFunctionWrapper):
   """
 
   def __init__(self,
-               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               reduction=losses_utils.ReductionV2.SUM_OVER_BATCH_SIZE,
                name='mean_squared_error'):
     super(MeanSquaredError, self).__init__(
         mean_squared_error, name=name, reduction=reduction)
@@ -222,7 +222,7 @@ class MeanAbsoluteError(LossFunctionWrapper):
   """
 
   def __init__(self,
-               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               reduction=losses_utils.ReductionV2.SUM_OVER_BATCH_SIZE,
                name='mean_absolute_error'):
     super(MeanAbsoluteError, self).__init__(
         mean_absolute_error, name=name, reduction=reduction)
@@ -252,7 +252,7 @@ class MeanAbsolutePercentageError(LossFunctionWrapper):
   """
 
   def __init__(self,
-               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               reduction=losses_utils.ReductionV2.SUM_OVER_BATCH_SIZE,
                name='mean_absolute_percentage_error'):
     super(MeanAbsolutePercentageError, self).__init__(
         mean_absolute_percentage_error, name=name, reduction=reduction)
@@ -282,7 +282,7 @@ class MeanSquaredLogarithmicError(LossFunctionWrapper):
   """
 
   def __init__(self,
-               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               reduction=losses_utils.ReductionV2.SUM_OVER_BATCH_SIZE,
                name='mean_squared_logarithmic_error'):
     super(MeanSquaredLogarithmicError, self).__init__(
         mean_squared_logarithmic_error, name=name, reduction=reduction)
@@ -326,7 +326,7 @@ class BinaryCrossentropy(LossFunctionWrapper):
   def __init__(self,
                from_logits=False,
                label_smoothing=0,
-               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               reduction=losses_utils.ReductionV2.SUM_OVER_BATCH_SIZE,
                name='binary_crossentropy'):
     super(BinaryCrossentropy, self).__init__(
         binary_crossentropy,
@@ -382,7 +382,7 @@ class CategoricalCrossentropy(LossFunctionWrapper):
   def __init__(self,
                from_logits=False,
                label_smoothing=0,
-               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               reduction=losses_utils.ReductionV2.SUM_OVER_BATCH_SIZE,
                name='categorical_crossentropy'):
     super(CategoricalCrossentropy, self).__init__(
         categorical_crossentropy,
@@ -434,7 +434,7 @@ class SparseCategoricalCrossentropy(LossFunctionWrapper):
 
   def __init__(self,
                from_logits=False,
-               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               reduction=losses_utils.ReductionV2.SUM_OVER_BATCH_SIZE,
                name=None):
     super(SparseCategoricalCrossentropy, self).__init__(
         sparse_categorical_crossentropy,
@@ -470,7 +470,7 @@ class Hinge(LossFunctionWrapper):
   """
 
   def __init__(self,
-               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               reduction=losses_utils.ReductionV2.SUM_OVER_BATCH_SIZE,
                name=None):
     super(Hinge, self).__init__(hinge, name=name, reduction=reduction)
 
@@ -502,7 +502,7 @@ class SquaredHinge(LossFunctionWrapper):
   """
 
   def __init__(self,
-               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               reduction=losses_utils.ReductionV2.SUM_OVER_BATCH_SIZE,
                name='squared_hinge'):
     super(SquaredHinge, self).__init__(
         squared_hinge, name=name, reduction=reduction)
@@ -529,7 +529,7 @@ class CategoricalHinge(LossFunctionWrapper):
   """
 
   def __init__(self,
-               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               reduction=losses_utils.ReductionV2.SUM_OVER_BATCH_SIZE,
                name='categorical_hinge'):
     super(CategoricalHinge, self).__init__(
         categorical_hinge, name=name, reduction=reduction)
@@ -558,7 +558,7 @@ class LogLoss(LossFunctionWrapper):
   """
 
   def __init__(self,
-               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               reduction=losses_utils.ReductionV2.SUM_OVER_BATCH_SIZE,
                name='logloss'):
     super(LogLoss, self).__init__(logloss, name=name, reduction=reduction)
 
@@ -586,7 +586,7 @@ class Poisson(LossFunctionWrapper):
   """
 
   def __init__(self,
-               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               reduction=losses_utils.ReductionV2.SUM_OVER_BATCH_SIZE,
                name='poisson'):
     super(Poisson, self).__init__(poisson, name=name, reduction=reduction)
 
@@ -614,7 +614,7 @@ class LogCosh(LossFunctionWrapper):
   """
 
   def __init__(self,
-               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               reduction=losses_utils.ReductionV2.SUM_OVER_BATCH_SIZE,
                name='logcosh'):
     super(LogCosh, self).__init__(logcosh, name=name, reduction=reduction)
 
@@ -642,7 +642,7 @@ class KLDivergence(LossFunctionWrapper):
   """
 
   def __init__(self,
-               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               reduction=losses_utils.ReductionV2.SUM_OVER_BATCH_SIZE,
                name='kullback_leibler_divergence'):
     super(KLDivergence, self).__init__(
         kullback_leibler_divergence, name=name, reduction=reduction)
@@ -685,7 +685,7 @@ class Huber(LossFunctionWrapper):
 
   def __init__(self,
                delta=1.0,
-               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               reduction=losses_utils.ReductionV2.SUM_OVER_BATCH_SIZE,
                name='huber_loss'):
     super(Huber, self).__init__(
         huber_loss, name=name, reduction=reduction, delta=delta)
@@ -985,7 +985,7 @@ class CosineProximity(Loss):
 
   def __init__(self,
                axis=-1,
-               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               reduction=losses_utils.ReductionV2.SUM_OVER_BATCH_SIZE,
                name=None):
     super(CosineProximity, self).__init__(reduction=reduction, name=name)
     self.axis = axis
diff --git a/tensorflow/python/keras/losses_test.py b/tensorflow/python/keras/losses_test.py
index e72209059b..48e4b68fc9 100644
--- a/tensorflow/python/keras/losses_test.py
+++ b/tensorflow/python/keras/losses_test.py
@@ -27,7 +27,7 @@ from tensorflow.python import keras
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import test_util
-from tensorflow.python.ops.losses import losses_impl
+from tensorflow.python.keras.utils import losses_utils
 from tensorflow.python.platform import test
 
 try:
@@ -186,7 +186,7 @@ class KerasLossesTest(test.TestCase):
 
     self.assertEqual(mse_obj.name, 'mean_squared_error')
     self.assertEqual(mse_obj.reduction,
-                     losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE)
+                     losses_utils.ReductionV2.SUM_OVER_BATCH_SIZE)
 
     y_true = constant_op.constant([[1., 9.], [2., 5.]])
     y_pred = constant_op.constant([[4., 8.], [12., 3.]])
@@ -205,9 +205,9 @@ class MeanSquaredErrorTest(test.TestCase):
 
   def test_config(self):
     mse_obj = keras.losses.MeanSquaredError(
-        reduction=losses_impl.ReductionV2.SUM, name='mse_1')
+        reduction=losses_utils.ReductionV2.SUM, name='mse_1')
     self.assertEqual(mse_obj.name, 'mse_1')
-    self.assertEqual(mse_obj.reduction, losses_impl.ReductionV2.SUM)
+    self.assertEqual(mse_obj.reduction, losses_utils.ReductionV2.SUM)
 
   def test_all_correct_unweighted(self):
     mse_obj = keras.losses.MeanSquaredError()
@@ -273,7 +273,7 @@ class MeanSquaredErrorTest(test.TestCase):
 
   def test_no_reduction(self):
     mse_obj = keras.losses.MeanSquaredError(
-        reduction=losses_impl.ReductionV2.NONE)
+        reduction=losses_utils.ReductionV2.NONE)
     y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
     y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
                                   shape=(2, 3),
@@ -284,7 +284,7 @@ class MeanSquaredErrorTest(test.TestCase):
 
   def test_sum_reduction(self):
     mse_obj = keras.losses.MeanSquaredError(
-        reduction=losses_impl.ReductionV2.SUM)
+        reduction=losses_utils.ReductionV2.SUM)
     y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
     y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
                                   shape=(2, 3),
@@ -298,9 +298,9 @@ class MeanAbsoluteErrorTest(test.TestCase):
 
   def test_config(self):
     mae_obj = keras.losses.MeanAbsoluteError(
-        reduction=losses_impl.ReductionV2.SUM, name='mae_1')
+        reduction=losses_utils.ReductionV2.SUM, name='mae_1')
     self.assertEqual(mae_obj.name, 'mae_1')
-    self.assertEqual(mae_obj.reduction, losses_impl.ReductionV2.SUM)
+    self.assertEqual(mae_obj.reduction, losses_utils.ReductionV2.SUM)
 
   def test_all_correct_unweighted(self):
     mae_obj = keras.losses.MeanAbsoluteError()
@@ -366,7 +366,7 @@ class MeanAbsoluteErrorTest(test.TestCase):
 
   def test_no_reduction(self):
     mae_obj = keras.losses.MeanAbsoluteError(
-        reduction=losses_impl.ReductionV2.NONE)
+        reduction=losses_utils.ReductionV2.NONE)
     y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
     y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
                                   shape=(2, 3),
@@ -377,7 +377,7 @@ class MeanAbsoluteErrorTest(test.TestCase):
 
   def test_sum_reduction(self):
     mae_obj = keras.losses.MeanAbsoluteError(
-        reduction=losses_impl.ReductionV2.SUM)
+        reduction=losses_utils.ReductionV2.SUM)
     y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
     y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
                                   shape=(2, 3),
@@ -391,9 +391,9 @@ class MeanAbsolutePercentageErrorTest(test.TestCase):
 
   def test_config(self):
     mape_obj = keras.losses.MeanAbsolutePercentageError(
-        reduction=losses_impl.ReductionV2.SUM, name='mape_1')
+        reduction=losses_utils.ReductionV2.SUM, name='mape_1')
     self.assertEqual(mape_obj.name, 'mape_1')
-    self.assertEqual(mape_obj.reduction, losses_impl.ReductionV2.SUM)
+    self.assertEqual(mape_obj.reduction, losses_utils.ReductionV2.SUM)
 
   def test_unweighted(self):
     mape_obj = keras.losses.MeanAbsolutePercentageError()
@@ -448,9 +448,9 @@ class MeanSquaredLogarithmicErrorTest(test.TestCase):
 
   def test_config(self):
     msle_obj = keras.losses.MeanSquaredLogarithmicError(
-        reduction=losses_impl.ReductionV2.SUM, name='mape_1')
+        reduction=losses_utils.ReductionV2.SUM, name='mape_1')
     self.assertEqual(msle_obj.name, 'mape_1')
-    self.assertEqual(msle_obj.reduction, losses_impl.ReductionV2.SUM)
+    self.assertEqual(msle_obj.reduction, losses_utils.ReductionV2.SUM)
 
   def test_unweighted(self):
     msle_obj = keras.losses.MeanSquaredLogarithmicError()
@@ -522,9 +522,9 @@ class CosineProximityTest(test.TestCase):
 
   def test_config(self):
     cosine_obj = keras.losses.CosineProximity(
-        axis=2, reduction=losses_impl.ReductionV2.SUM, name='cosine_loss')
+        axis=2, reduction=losses_utils.ReductionV2.SUM, name='cosine_loss')
     self.assertEqual(cosine_obj.name, 'cosine_loss')
-    self.assertEqual(cosine_obj.reduction, losses_impl.ReductionV2.SUM)
+    self.assertEqual(cosine_obj.reduction, losses_utils.ReductionV2.SUM)
     self.assertEqual(cosine_obj.axis, 2)
 
   def test_unweighted(self):
@@ -591,9 +591,9 @@ class BinaryCrossentropyTest(test.TestCase):
 
   def test_config(self):
     bce_obj = keras.losses.BinaryCrossentropy(
-        reduction=losses_impl.ReductionV2.SUM, name='bce_1')
+        reduction=losses_utils.ReductionV2.SUM, name='bce_1')
     self.assertEqual(bce_obj.name, 'bce_1')
-    self.assertEqual(bce_obj.reduction, losses_impl.ReductionV2.SUM)
+    self.assertEqual(bce_obj.reduction, losses_utils.ReductionV2.SUM)
 
   def test_all_correct_unweighted(self):
     y_true = constant_op.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]],
@@ -722,7 +722,7 @@ class BinaryCrossentropyTest(test.TestCase):
     logits = constant_op.constant([[100.0, -100.0, 100.0],
                                    [100.0, 100.0, -100.0]])
     bce_obj = keras.losses.BinaryCrossentropy(
-        from_logits=True, reduction=losses_impl.ReductionV2.NONE)
+        from_logits=True, reduction=losses_utils.ReductionV2.NONE)
     loss = bce_obj(y_true, logits)
 
     # Loss = max(x, 0) - x * z + log(1 + exp(-abs(x)))
@@ -757,9 +757,9 @@ class CategoricalCrossentropyTest(test.TestCase):
 
   def test_config(self):
     cce_obj = keras.losses.CategoricalCrossentropy(
-        reduction=losses_impl.ReductionV2.SUM, name='bce_1')
+        reduction=losses_utils.ReductionV2.SUM, name='bce_1')
     self.assertEqual(cce_obj.name, 'bce_1')
-    self.assertEqual(cce_obj.reduction, losses_impl.ReductionV2.SUM)
+    self.assertEqual(cce_obj.reduction, losses_utils.ReductionV2.SUM)
 
   def test_all_correct_unweighted(self):
     y_true = constant_op.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]],
@@ -823,7 +823,7 @@ class CategoricalCrossentropyTest(test.TestCase):
     y_true = constant_op.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]])
     logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
     cce_obj = keras.losses.CategoricalCrossentropy(
-        from_logits=True, reduction=losses_impl.ReductionV2.NONE)
+        from_logits=True, reduction=losses_utils.ReductionV2.NONE)
     loss = cce_obj(y_true, logits)
     self.assertAllClose((0.001822, 0.000459, 0.169846), self.evaluate(loss), 3)
 
@@ -914,7 +914,7 @@ class SparseCategoricalCrossentropyTest(test.TestCase):
     y_true = constant_op.constant([[0], [1], [2]])
     logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
     cce_obj = keras.losses.SparseCategoricalCrossentropy(
-        from_logits=True, reduction=losses_impl.ReductionV2.NONE)
+        from_logits=True, reduction=losses_utils.ReductionV2.NONE)
     loss = cce_obj(y_true, logits)
     self.assertAllClose((0.001822, 0.000459, 0.169846), self.evaluate(loss), 3)
 
@@ -924,9 +924,9 @@ class HingeTest(test.TestCase):
 
   def test_config(self):
     hinge_obj = keras.losses.Hinge(
-        reduction=losses_impl.ReductionV2.SUM, name='hinge_loss')
+        reduction=losses_utils.ReductionV2.SUM, name='hinge_loss')
     self.assertEqual(hinge_obj.name, 'hinge_loss')
-    self.assertEqual(hinge_obj.reduction, losses_impl.ReductionV2.SUM)
+    self.assertEqual(hinge_obj.reduction, losses_utils.ReductionV2.SUM)
 
   def test_unweighted(self):
     hinge_obj = keras.losses.Hinge()
@@ -1024,9 +1024,9 @@ class SquaredHingeTest(test.TestCase):
 
   def test_config(self):
     sq_hinge_obj = keras.losses.SquaredHinge(
-        reduction=losses_impl.ReductionV2.SUM, name='sq_hinge_loss')
+        reduction=losses_utils.ReductionV2.SUM, name='sq_hinge_loss')
     self.assertEqual(sq_hinge_obj.name, 'sq_hinge_loss')
-    self.assertEqual(sq_hinge_obj.reduction, losses_impl.ReductionV2.SUM)
+    self.assertEqual(sq_hinge_obj.reduction, losses_utils.ReductionV2.SUM)
 
   def test_unweighted(self):
     sq_hinge_obj = keras.losses.SquaredHinge()
@@ -1133,9 +1133,9 @@ class CategoricalHingeTest(test.TestCase):
 
   def test_config(self):
     cat_hinge_obj = keras.losses.CategoricalHinge(
-        reduction=losses_impl.ReductionV2.SUM, name='cat_hinge_loss')
+        reduction=losses_utils.ReductionV2.SUM, name='cat_hinge_loss')
     self.assertEqual(cat_hinge_obj.name, 'cat_hinge_loss')
-    self.assertEqual(cat_hinge_obj.reduction, losses_impl.ReductionV2.SUM)
+    self.assertEqual(cat_hinge_obj.reduction, losses_utils.ReductionV2.SUM)
 
   def test_unweighted(self):
     cat_hinge_obj = keras.losses.CategoricalHinge()
@@ -1214,9 +1214,9 @@ class LogLossTest(test.TestCase):
 
   def test_config(self):
     log_loss_obj = keras.losses.LogLoss(
-        reduction=losses_impl.ReductionV2.SUM, name='log')
+        reduction=losses_utils.ReductionV2.SUM, name='log')
     self.assertEqual(log_loss_obj.name, 'log')
-    self.assertEqual(log_loss_obj.reduction, losses_impl.ReductionV2.SUM)
+    self.assertEqual(log_loss_obj.reduction, losses_utils.ReductionV2.SUM)
 
   def test_all_correct(self):
     self.setup()
@@ -1301,9 +1301,9 @@ class LogCoshTest(test.TestCase):
 
   def test_config(self):
     logcosh_obj = keras.losses.LogCosh(
-        reduction=losses_impl.ReductionV2.SUM, name='logcosh_loss')
+        reduction=losses_utils.ReductionV2.SUM, name='logcosh_loss')
     self.assertEqual(logcosh_obj.name, 'logcosh_loss')
-    self.assertEqual(logcosh_obj.reduction, losses_impl.ReductionV2.SUM)
+    self.assertEqual(logcosh_obj.reduction, losses_utils.ReductionV2.SUM)
 
   def test_unweighted(self):
     self.setup()
@@ -1382,9 +1382,9 @@ class PoissonTest(test.TestCase):
 
   def test_config(self):
     poisson_obj = keras.losses.Poisson(
-        reduction=losses_impl.ReductionV2.SUM, name='poisson')
+        reduction=losses_utils.ReductionV2.SUM, name='poisson')
     self.assertEqual(poisson_obj.name, 'poisson')
-    self.assertEqual(poisson_obj.reduction, losses_impl.ReductionV2.SUM)
+    self.assertEqual(poisson_obj.reduction, losses_utils.ReductionV2.SUM)
 
   def test_unweighted(self):
     self.setup()
@@ -1463,9 +1463,9 @@ class KLDivergenceTest(test.TestCase):
 
   def test_config(self):
     k_obj = keras.losses.KLDivergence(
-        reduction=losses_impl.ReductionV2.SUM, name='kld')
+        reduction=losses_utils.ReductionV2.SUM, name='kld')
     self.assertEqual(k_obj.name, 'kld')
-    self.assertEqual(k_obj.reduction, losses_impl.ReductionV2.SUM)
+    self.assertEqual(k_obj.reduction, losses_utils.ReductionV2.SUM)
 
   def test_unweighted(self):
     self.setup()
@@ -1553,9 +1553,9 @@ class HuberLossTest(test.TestCase):
 
   def test_config(self):
     h_obj = keras.losses.Huber(
-        reduction=losses_impl.ReductionV2.SUM, name='huber')
+        reduction=losses_utils.ReductionV2.SUM, name='huber')
     self.assertEqual(h_obj.name, 'huber')
-    self.assertEqual(h_obj.reduction, losses_impl.ReductionV2.SUM)
+    self.assertEqual(h_obj.reduction, losses_utils.ReductionV2.SUM)
 
   def test_all_correct(self):
     self.setup()
diff --git a/tensorflow/python/keras/ops.py b/tensorflow/python/keras/ops.py
index 44e022828a..b2d8520544 100644
--- a/tensorflow/python/keras/ops.py
+++ b/tensorflow/python/keras/ops.py
@@ -20,7 +20,6 @@ from __future__ import print_function
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import init_ops_v2
-from tensorflow.python.ops.losses import losses_impl
 from tensorflow.python.util.tf_export import keras_export
 
 
@@ -101,6 +100,3 @@ keras_export("keras.initializers.TruncatedNormal", v1=[])(
 
 
 keras_export("keras.backend.name_scope")(ops.name_scope)
-
-keras_export("keras.losses.Reduction", v1=[])(
-    losses_impl.ReductionV2)
diff --git a/tensorflow/python/keras/utils/losses_utils.py b/tensorflow/python/keras/utils/losses_utils.py
index 899780e53b..163d36102c 100644
--- a/tensorflow/python/keras/utils/losses_utils.py
+++ b/tensorflow/python/keras/utils/losses_utils.py
@@ -27,7 +27,34 @@ from tensorflow.python.ops import confusion_matrix
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import weights_broadcast_ops
-from tensorflow.python.ops.losses import losses_impl
+from tensorflow.python.util.tf_export import keras_export
+
+
+@keras_export('keras.losses.Reduction', v1=[])
+class ReductionV2(object):
+  """Types of loss reduction.
+
+  Contains the following values:
+
+  * `NONE`: Un-reduced weighted losses with the same shape as input.
+  * `SUM`: Scalar sum of weighted losses.
+  * `SUM_OVER_BATCH_SIZE`: Scalar `SUM` divided by number of elements in losses.
+     Note that when using `tf.distribute.Strategy`, this is the global batch
+     size across all the replicas that are contributing to a single step.
+  """
+
+  NONE = 'none'
+  SUM = 'sum'
+  SUM_OVER_BATCH_SIZE = 'sum_over_batch_size'
+
+  @classmethod
+  def all(cls):
+    return (cls.NONE, cls.SUM, cls.SUM_OVER_BATCH_SIZE)
+
+  @classmethod
+  def validate(cls, key):
+    if key not in cls.all():
+      raise ValueError('Invalid Reduction Key %s.' % key)
 
 
 def squeeze_or_expand_dimensions(y_pred, y_true, sample_weight):
@@ -141,14 +168,14 @@ def _num_elements(losses):
     return math_ops.cast(array_ops.size(losses, name=scope), dtype=losses.dtype)
 
 
-def _reduce_weighted_loss(
-    weighted_losses, reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE):
+def _reduce_weighted_loss(weighted_losses,
+                          reduction=ReductionV2.SUM_OVER_BATCH_SIZE):
   """Reduces the individual weighted loss measurements."""
-  if reduction == losses_impl.ReductionV2.NONE:
+  if reduction == ReductionV2.NONE:
     loss = weighted_losses
   else:
     loss = math_ops.reduce_sum(weighted_losses)
-    if reduction == losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE:
+    if reduction == ReductionV2.SUM_OVER_BATCH_SIZE:
       num_replicas = (  # Used to convert from local to global batch size.
           distribution_strategy_context.get_strategy().num_replicas_in_sync)
       loss = _safe_mean(loss, num_replicas * _num_elements(weighted_losses))
@@ -157,7 +184,7 @@ def _reduce_weighted_loss(
 
 def compute_weighted_loss(losses,
                           sample_weight=None,
-                          reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+                          reduction=ReductionV2.SUM_OVER_BATCH_SIZE,
                           name=None):
   """Computes the weighted loss.
 
@@ -176,7 +203,7 @@ def compute_weighted_loss(losses,
     Weighted loss `Tensor` of the same type as `losses`. If `reduction` is
     `NONE`, this has the same shape as `losses`; otherwise, it is scalar.
   """
-  losses_impl.ReductionV2.validate(reduction)
+  ReductionV2.validate(reduction)
   if sample_weight is None:
     sample_weight = 1.0
   with ops.name_scope(name, 'weighted_loss', (losses, sample_weight)):
diff --git a/tensorflow/python/ops/losses/losses_impl.py b/tensorflow/python/ops/losses/losses_impl.py
index 1169c452cc..6cd1d8e5f8 100644
--- a/tensorflow/python/ops/losses/losses_impl.py
+++ b/tensorflow/python/ops/losses/losses_impl.py
@@ -35,33 +35,6 @@ from tensorflow.python.util.deprecation import deprecated_argument_lookup
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export("losses.Reduction", v1=[])
-class ReductionV2(object):
-  """Types of loss reduction.
-
-  Contains the following values:
-
-  * `NONE`: Un-reduced weighted losses with the same shape as input.
-  * `SUM`: Scalar sum of weighted losses.
-  * `SUM_OVER_BATCH_SIZE`: Scalar `SUM` divided by number of elements in losses.
-     Note that when using `tf.distribute.Strategy`, this is the global batch
-     size across all the replicas that are contributing to a single step.
-  """
-
-  NONE = "none"
-  SUM = "sum"
-  SUM_OVER_BATCH_SIZE = "sum_over_batch_size"
-
-  @classmethod
-  def all(cls):
-    return (cls.NONE, cls.SUM, cls.SUM_OVER_BATCH_SIZE)
-
-  @classmethod
-  def validate(cls, key):
-    if key not in cls.all():
-      raise ValueError("Invalid Reduction Key %s." % key)
-
-
 @tf_export(v1=["losses.Reduction"])
 class Reduction(object):
   """Types of loss reduction.
diff --git a/tensorflow/python/tools/api/generator/api_init_files.bzl b/tensorflow/python/tools/api/generator/api_init_files.bzl
index 5095f9fdbe..19af602cd2 100644
--- a/tensorflow/python/tools/api/generator/api_init_files.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files.bzl
@@ -27,7 +27,6 @@ TENSORFLOW_API_INIT_FILES = [
     "linalg/__init__.py",
     "lite/__init__.py",
     "lite/constants/__init__.py",
-    "losses/__init__.py",
     "math/__init__.py",
     "nest/__init__.py",
     "nn/__init__.py",
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-reduction.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-reduction.pbtxt
index f20ed26e2e..e93be80f1f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-reduction.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-reduction.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.keras.losses.Reduction"
 tf_class {
-  is_instance: "<class \'tensorflow.python.ops.losses.losses_impl.ReductionV2\'>"
+  is_instance: "<class \'tensorflow.python.keras.utils.losses_utils.ReductionV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "NONE"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.-reduction.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.losses.-reduction.pbtxt
deleted file mode 100644
index 6a44e4ce66..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.losses.-reduction.pbtxt
+++ /dev/null
@@ -1,28 +0,0 @@
-path: "tensorflow.losses.Reduction"
-tf_class {
-  is_instance: "<class \'tensorflow.python.ops.losses.losses_impl.ReductionV2\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "NONE"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "SUM"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "SUM_OVER_BATCH_SIZE"
-    mtype: "<type \'str\'>"
-  }
-  member_method {
-    name: "__init__"
-  }
-  member_method {
-    name: "all"
-    argspec: "args=[\'cls\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "validate"
-    argspec: "args=[\'cls\', \'key\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.losses.pbtxt
deleted file mode 100644
index 36007d3ca6..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.losses.pbtxt
+++ /dev/null
@@ -1,7 +0,0 @@
-path: "tensorflow.losses"
-tf_module {
-  member {
-    name: "Reduction"
-    mtype: "<type \'type\'>"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index f0206823a7..1ebbf602c9 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -200,10 +200,6 @@ tf_module {
     name: "lite"
     mtype: "<type \'module\'>"
   }
-  member {
-    name: "losses"
-    mtype: "<type \'module\'>"
-  }
   member {
     name: "math"
     mtype: "<type \'module\'>"
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index c29b259027..a1c47a1efa 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -349,6 +349,7 @@ renames = {
     'tf.logging.warn': 'tf.compat.v1.logging.warn',
     'tf.logging.warning': 'tf.compat.v1.logging.warning',
     'tf.logical_xor': 'tf.math.logical_xor',
+    'tf.losses.Reduction': 'tf.compat.v1.losses.Reduction',
     'tf.losses.absolute_difference': 'tf.compat.v1.losses.absolute_difference',
     'tf.losses.add_loss': 'tf.compat.v1.losses.add_loss',
     'tf.losses.compute_weighted_loss': 'tf.compat.v1.losses.compute_weighted_loss',
-- 
GitLab


From 6b83e5ba308df15db0705c5662145b386b741280 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Thu, 7 Feb 2019 22:22:50 -0800
Subject: [PATCH 2131/2345] Unit tests for eager support in TPU Strategy

PiperOrigin-RevId: 233005064
---
 .../contrib/distribute/python/tpu_strategy.py | 41 ++++++++++++++++---
 tensorflow/contrib/tpu/BUILD                  |  2 +
 2 files changed, 37 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 341d9aeaf9..2ce127c855 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -26,6 +26,7 @@ import copy
 
 from tensorflow.contrib.tpu.python.ops import tpu_ops
 from tensorflow.contrib.tpu.python.tpu import device_assignment as device_assignment_lib
+from tensorflow.contrib.tpu.python.tpu import functional as tpu_functional_ops
 from tensorflow.contrib.tpu.python.tpu import topology
 from tensorflow.contrib.tpu.python.tpu import tpu
 from tensorflow.contrib.tpu.python.tpu import tpu_system_metadata as tpu_system_metadata_lib
@@ -41,6 +42,7 @@ from tensorflow.python.distribute import reduce_util
 from tensorflow.python.distribute import values
 from tensorflow.python.distribute.cluster_resolver import TPUClusterResolver
 from tensorflow.python.eager import context
+from tensorflow.python.eager import function
 from tensorflow.python.eager import tape
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import device as tf_device
@@ -52,6 +54,7 @@ from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variable_scope as vs
 from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.util import compat
 from tensorflow.python.util import nest
 
 
@@ -69,11 +72,36 @@ def initialize_tpu_system(cluster_resolver=None):
   master = cluster_resolver.master()
 
   logging.info("Initializing the TPU system.")
-  session_config = config_pb2.ConfigProto(allow_soft_placement=True)
 
-  with ops.Graph().as_default():
-    with session_lib.Session(config=session_config, target=master) as sess:
-      serialized_topology = sess.run(tpu.initialize_system())
+  if context.executing_eagerly():
+    # This function looks as it is for the following non-intuitive reasons.
+    # tpu.initialize_system creates a dummy op whose sole purpose is to trigger
+    # DistributedTPURewritePass. This pass actually adds real ops that
+    # initialize the TPU system. Thus, we can't simply run tpu.initialize_system
+    # eagerly. We need to wrap it in defun and trigger the rewrite passes on it.
+    # The easiest way to trigger a rewrite is to run the function with
+    # TPUPartitionedCallOp.
+    @function.defun
+    def _tpu_init_fn():
+      return tpu.initialize_system()
+
+    # We can't call _tpu_init_fn normally (because it contains just a dummy op,
+    # see above) but need to define it to get it added to eager context
+    # and get its assigned name.
+    # pylint: disable=protected-access
+    graph_func = _tpu_init_fn._get_concrete_function_internal()
+    func_name = compat.as_str(graph_func._inference_function.name)
+    # pylint: enable=protected-access
+
+    output = tpu_functional_ops.TPUPartitionedCall(
+        args=[], device_ordinal=0, Tout=[dtypes.string], f=func_name)
+    serialized_topology = output[0].numpy()
+  else:
+    session_config = config_pb2.ConfigProto(allow_soft_placement=True)
+    with ops.Graph().as_default():
+      with session_lib.Session(config=session_config, target=master) as sess:
+        serialized_topology = sess.run(tpu.initialize_system())
+
   logging.info("Finished initializing TPU system.")
   return topology.Topology(serialized=serialized_topology)
 
@@ -183,8 +211,9 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
   # This implementation runs a single step. It does not use infeed or outfeed.
   def experimental_run(self, fn, input_iterator=None):
     """See base class."""
-    if context.executing_eagerly():
-      raise NotImplementedError("Eager mode not supported in TPUStrategy.")
+    if context.executing_eagerly() and not ops.inside_function():
+      raise NotImplementedError(
+          "Eager mode not supported in TPUStrategy outside TF functions.")
 
     if input_iterator is None:
       inputs = []
diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index 7b1a501fb7..294dbddcb5 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -325,7 +325,9 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":datasets",
+        ":functional",
         ":profiler",
+        ":tpu_ordinal_selector_py",
         ":tpu_py",
         "//tensorflow/compiler/xla/experimental/xla_sharding",
         "//tensorflow/compiler/xla/python_api:xla_shape",
-- 
GitLab


From 4301ac109e5873682072b15ebebcb27916c32a4a Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Thu, 7 Feb 2019 23:37:16 -0800
Subject: [PATCH 2132/2345] Create int8 logistic.

PiperOrigin-RevId: 233010183
---
 tensorflow/lite/kernels/activations.cc        | 21 +++++-
 tensorflow/lite/kernels/activations_test.cc   | 24 ++++++-
 tensorflow/lite/kernels/internal/BUILD        |  1 +
 .../internal/reference/integer_ops/logistic.h | 64 +++++++++++++++++++
 tensorflow/lite/kernels/register.cc           |  4 +-
 tensorflow/lite/toco/tflite/operator.cc       | 17 ++++-
 tensorflow/lite/toco/tflite/operator_test.cc  | 20 ++++++
 7 files changed, 145 insertions(+), 6 deletions(-)
 create mode 100644 tensorflow/lite/kernels/internal/reference/integer_ops/logistic.h

diff --git a/tensorflow/lite/kernels/activations.cc b/tensorflow/lite/kernels/activations.cc
index 2b35cc4915..b98ae662a6 100644
--- a/tensorflow/lite/kernels/activations.cc
+++ b/tensorflow/lite/kernels/activations.cc
@@ -23,6 +23,7 @@ limitations under the License.
 #include "tensorflow/lite/c/c_api_internal.h"
 #include "tensorflow/lite/kernels/internal/optimized/optimized_ops.h"
 #include "tensorflow/lite/kernels/internal/quantization_util.h"
+#include "tensorflow/lite/kernels/internal/reference/integer_ops/logistic.h"
 #include "tensorflow/lite/kernels/internal/reference/integer_ops/softmax.h"
 #include "tensorflow/lite/kernels/internal/reference/reference_ops.h"
 #include "tensorflow/lite/kernels/internal/tensor.h"
@@ -177,8 +178,15 @@ TfLiteStatus SigmoidPrepare(TfLiteContext* context, TfLiteNode* node) {
   TfLiteTensor* output = GetOutput(context, node, 0);
   TF_LITE_ENSURE_EQ(context, input->type, output->type);
 
-  if (input->type == kTfLiteUInt8) {
-    TF_LITE_ENSURE_EQ(context, output->params.zero_point, 0);
+  if (input->type == kTfLiteUInt8 || input->type == kTfLiteInt8) {
+    if (input->type == kTfLiteUInt8) {
+      TF_LITE_ENSURE_EQ(context, output->params.zero_point,
+                        std::numeric_limits<uint8_t>::min());
+    }
+    if (input->type == kTfLiteInt8) {
+      TF_LITE_ENSURE_EQ(context, output->params.zero_point,
+                        std::numeric_limits<int8_t>::min());
+    }
     TF_LITE_ENSURE(context, output->params.scale == 1. / 256);
 
     static constexpr int kInputIntegerBits = 4;
@@ -493,6 +501,15 @@ TfLiteStatus SigmoidEval(TfLiteContext* context, TfLiteNode* node) {
       }
       break;
     }
+    case kTfLiteInt8: {
+      const int input_size =
+          MatchingFlatSize(GetTensorShape(input), GetTensorShape(output));
+      reference_integer_ops::Logistic(
+          input->params.zero_point, data->input_range_radius,
+          data->input_multiplier, data->input_left_shift, input_size,
+          GetTensorData<int8_t>(input), GetTensorData<int8_t>(output));
+      break;
+    }
     default:
       context->ReportError(context, "Only float32 supported currently, got %s.",
                            TfLiteTypeGetName(input->type));
diff --git a/tensorflow/lite/kernels/activations_test.cc b/tensorflow/lite/kernels/activations_test.cc
index 5e3c56ed5b..38903f5acb 100644
--- a/tensorflow/lite/kernels/activations_test.cc
+++ b/tensorflow/lite/kernels/activations_test.cc
@@ -32,6 +32,8 @@ class BaseActivationsOpModel : public SingleOpModel {
     input_ = AddInput(input);
     if (input.type == TensorType_UINT8) {
       output_ = AddOutput({input.type, {}, 0, 0, 1. / 256});
+    } else if (input.type == TensorType_INT8) {
+      output_ = AddOutput({input.type, {}, 0, 0, 1. / 256, -128});
     } else {
       output_ = AddOutput({input.type, {}});
     }
@@ -253,7 +255,7 @@ TEST(FloatActivationsOpTest, Sigmoid) {
                              })));
 }
 
-TEST(QuantizedActivationsOpTest, Sigmoid) {
+TEST(QuantizedActivationsOpTest, SigmoidUint8) {
   QuantizedActivationsOpModel m(
       BuiltinOperator_LOGISTIC,
       /*input=*/{TensorType_UINT8, {1, 2, 4, 1}, -10, 10});
@@ -273,6 +275,26 @@ TEST(QuantizedActivationsOpTest, Sigmoid) {
               ElementsAreArray({128, 1, 227, 251, 244, 32, 255, 188}));
 }
 
+TEST(QuantizedActivationsOpTest, SigmoidInt8) {
+  QuantizedActivationsOpModel m(
+      BuiltinOperator_LOGISTIC,
+      /*input=*/{TensorType_INT8, {1, 2, 4, 1}, -10, 10});
+  m.SetInput<int8_t>({
+      0, -6, 2, 4,   //
+      3, -2, 10, 1,  //
+  });
+  m.Invoke();
+  EXPECT_THAT(m.GetDequantizedOutput<int8_t>(),
+              ElementsAreArray(ArrayFloatNear(
+                  {
+                      0.5, 0.002473, 0.880797, 0.982014,       //
+                      0.952574, 0.119203, 0.999955, 0.731059,  //
+                  },
+                  kQuantizedTolerance)));
+  EXPECT_THAT(m.GetOutput<int8_t>(),
+              ElementsAreArray({0, -127, 99, 123, 116, -99, 127, 60}));
+}
+
 TEST(QuantizedActivationsOpTest, SigmoidInt16) {
   const float kMin = -1;
   const float kMax = 32767.f / 32768.f;
diff --git a/tensorflow/lite/kernels/internal/BUILD b/tensorflow/lite/kernels/internal/BUILD
index 97b322551e..0248ed9725 100644
--- a/tensorflow/lite/kernels/internal/BUILD
+++ b/tensorflow/lite/kernels/internal/BUILD
@@ -309,6 +309,7 @@ cc_library(
         "reference/integer_ops/conv.h",
         "reference/integer_ops/depthwise_conv.h",
         "reference/integer_ops/dequantize.h",
+        "reference/integer_ops/logistic.h",
         "reference/integer_ops/pooling.h",
         "reference/integer_ops/softmax.h",
         "reference/reference_ops.h",
diff --git a/tensorflow/lite/kernels/internal/reference/integer_ops/logistic.h b/tensorflow/lite/kernels/internal/reference/integer_ops/logistic.h
new file mode 100644
index 0000000000..8277c3b3d5
--- /dev/null
+++ b/tensorflow/lite/kernels/internal/reference/integer_ops/logistic.h
@@ -0,0 +1,64 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_LOGISTIC_H_
+#define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_LOGISTIC_H_
+
+#include <limits>
+#include "tensorflow/lite/kernels/internal/common.h"
+
+namespace tflite {
+namespace reference_integer_ops {
+
+inline void Logistic(int32_t input_zero_point, int32_t input_range_radius,
+                     int32_t input_multiplier, int32_t input_left_shift,
+                     int32_t input_size, const int8_t* input_data,
+                     int8_t* output_data) {
+  // Integer bits must be in sync with Prepare() function.
+  static constexpr int32_t kInputIntegerBits = 4;
+  static constexpr int32_t kOutputIntegerBits = 8;
+  static constexpr int8_t kMinInt8 = std::numeric_limits<int8_t>::min();
+  static constexpr int8_t kMaxInt8 = std::numeric_limits<int8_t>::max();
+  static constexpr int32_t kOutputZeroPoint = -128;
+
+  for (int i = 0; i < input_size; ++i) {
+    const int32_t input =
+        static_cast<int32_t>(input_data[i]) - input_zero_point;
+    if (input <= -input_range_radius) {
+      output_data[i] = kMinInt8;
+    } else if (input >= input_range_radius) {
+      output_data[i] = kMaxInt8;
+    } else {
+      const int32_t input_in_q4 = MultiplyByQuantizedMultiplier(
+          input, input_multiplier, input_left_shift);
+      using FixedPoint4 = gemmlowp::FixedPoint<int32_t, kInputIntegerBits>;
+      const int32_t output_in_q0 =
+          gemmlowp::logistic(FixedPoint4::FromRaw(input_in_q4)).raw();
+
+      // Rescale and downcast.
+      using gemmlowp::RoundingDivideByPOT;
+      int32_t output_in_q23 =
+          RoundingDivideByPOT(output_in_q0, 31 - kOutputIntegerBits);
+      output_in_q23 = std::min(std::max(output_in_q23 + kOutputZeroPoint,
+                                        static_cast<int32_t>(kMinInt8)),
+                               static_cast<int32_t>(kMaxInt8));
+      output_data[i] = static_cast<int8_t>(output_in_q23);
+    }
+  }
+}
+
+}  // namespace reference_integer_ops
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_LOGISTIC_H_
diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index 1b4504e7a5..525b97f8e1 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -166,7 +166,9 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_RELU_N1_TO_1, Register_RELU_N1_TO_1());
   AddBuiltin(BuiltinOperator_RELU6, Register_RELU6());
   AddBuiltin(BuiltinOperator_TANH, Register_TANH());
-  AddBuiltin(BuiltinOperator_LOGISTIC, Register_LOGISTIC());
+  AddBuiltin(BuiltinOperator_LOGISTIC, Register_LOGISTIC(),
+             /* min_version */ 1,
+             /* max_version */ 2);
   AddBuiltin(BuiltinOperator_AVERAGE_POOL_2D, Register_AVERAGE_POOL_2D(),
              /* min_version */ 1,
              /* max_version */ 2);
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index bdebee839c..79125a44f9 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -1617,6 +1617,20 @@ class LeakyRelu
   }
 };
 
+class Logistic : public SimpleOperator<LogisticOperator> {
+ public:
+  explicit Logistic() : SimpleOperator("LOGISTIC", OperatorType::kLogistic) {}
+  int GetVersion(const OperatorSignature& op_signature) const override {
+    const string& input_name = op_signature.op->inputs[0];
+    const Array& input_array = op_signature.model->GetArray(input_name);
+    // Version 2 supports signed int8 input types.
+    if (input_array.data_type == ArrayDataType::kInt8) {
+      return 2;
+    }
+    return 1;
+  }
+};
+
 class SquaredDifference
     : public BuiltinOperator<
           SquaredDifferenceOperator, ::tflite::SquaredDifferenceOptions,
@@ -2227,8 +2241,7 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
       MakeUnique<SimpleOperator<Relu6Operator>>("RELU6", OperatorType::kRelu6));
   ops.push_back(
       MakeUnique<SimpleOperator<PReluOperator>>("PRELU", OperatorType::kPRelu));
-  ops.push_back(MakeUnique<SimpleOperator<LogisticOperator>>(
-      "LOGISTIC", OperatorType::kLogistic));
+  ops.push_back(MakeUnique<Logistic>());
   ops.push_back(
       MakeUnique<SimpleOperator<TanhOperator>>("TANH", OperatorType::kTanh));
   ops.push_back(
diff --git a/tensorflow/lite/toco/tflite/operator_test.cc b/tensorflow/lite/toco/tflite/operator_test.cc
index 903da16d74..5c7028bee1 100644
--- a/tensorflow/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/lite/toco/tflite/operator_test.cc
@@ -548,6 +548,26 @@ TEST_F(OperatorTest, BuiltinLeakyRelu) {
   EXPECT_EQ(op.alpha, output_toco_op->alpha);
 }
 
+TEST_F(OperatorTest, VersioningLogisticTest) {
+  LogisticOperator logistic_op;
+  logistic_op.inputs = {"input1"};
+  auto operator_by_type_map = BuildOperatorByTypeMap(false /*enable_flex_ops*/);
+  const BaseOperator* op = operator_by_type_map.at(logistic_op.type).get();
+
+  Model uint8_model;
+  Array& uint8_array = uint8_model.GetOrCreateArray(logistic_op.inputs[0]);
+  uint8_array.data_type = ArrayDataType::kUint8;
+  OperatorSignature uint8_signature = {.model = &uint8_model,
+                                       .op = &logistic_op};
+  EXPECT_EQ(op->GetVersion(uint8_signature), 1);
+
+  Model int8_model;
+  Array& int8_array = int8_model.GetOrCreateArray(logistic_op.inputs[0]);
+  int8_array.data_type = ArrayDataType::kInt8;
+  OperatorSignature int8_signature = {.model = &int8_model, .op = &logistic_op};
+  EXPECT_EQ(op->GetVersion(int8_signature), 2);
+}
+
 TEST_F(OperatorTest, BuiltinSquaredDifference) {
   SquaredDifferenceOperator op;
   auto output_toco_op = SerializeAndDeserialize(
-- 
GitLab


From f7b8f95d5ce66e2f1b7d273ccb73b0f532faf3a6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 8 Feb 2019 01:02:47 -0800
Subject: [PATCH 2133/2345] compat: Update forward compatibility horizon to
 2019-02-08

PiperOrigin-RevId: 233019624
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index f49e7602ce..68ca77dceb 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 2, 7)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 2, 8)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 20c87f420bc16222356644ceee565e7d43149707 Mon Sep 17 00:00:00 2001
From: Albin Joy <albin.joy@huawei.com>
Date: Fri, 8 Feb 2019 15:02:34 +0530
Subject: [PATCH 2134/2345] Typo error fix devguide.md

---
 tensorflow/lite/g3doc/devguide.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/g3doc/devguide.md b/tensorflow/lite/g3doc/devguide.md
index cbad036407..7e23f1d1b6 100644
--- a/tensorflow/lite/g3doc/devguide.md
+++ b/tensorflow/lite/g3doc/devguide.md
@@ -92,7 +92,7 @@ formats:
 
 TensorFlow models may be saved as a .pb or .pbtxt `tf.GraphDef` file. In order
 to convert the `tf.GraphDef` file to TensorFlow Lite, the model must first be
-frozen. This process invovles several file formats including the `frozen
+frozen. This process involves several file formats including the `frozen
 GraphDef`:
 
 *   `tf.GraphDef` (.pb or .pbtxt) — A protobuf that represents the TensorFlow
@@ -166,7 +166,7 @@ Refer to the [ops compatibility guide](tf_ops_compatibility.md) for
 troubleshooting help, and if that doesn't help, please
 [file an issue](https://github.com/tensorflow/tensorflow/issues).
 
-### Graph vizualization tool
+### Graph Visualization tool
 
 The [development repo](https://github.com/tensorflow/tensorflow) contains a tool
 to visualize TensorFlow Lite models after conversion. To build the
-- 
GitLab


From 246f6482a240f5693bb28c326b9aa36ba48098ec Mon Sep 17 00:00:00 2001
From: Andr? Susano Pinto <andresp@google.com>
Date: Fri, 8 Feb 2019 02:12:48 -0800
Subject: [PATCH 2135/2345] Switch TrackableAsset in eager mode to keep the
 path as a tensor or tf.constant which is the behaviour in graph mode.

This is needed to be able to share traces created for eager mode and graph
mode and so allow "tf.saved_model.load()" to work in both modes.

PiperOrigin-RevId: 233026763
---
 .../python/kernel_tests/lookup_ops_test.py    |  6 +++--
 tensorflow/python/saved_model/load.py         | 18 +++++++------
 tensorflow/python/saved_model/load_test.py    | 27 +++++++++++++++++--
 tensorflow/python/saved_model/save.py         |  6 ++---
 .../saved_model/saved_object_graph.proto      |  6 ++---
 .../training/checkpointable/tracking.py       | 21 +++------------
 6 files changed, 49 insertions(+), 35 deletions(-)

diff --git a/tensorflow/python/kernel_tests/lookup_ops_test.py b/tensorflow/python/kernel_tests/lookup_ops_test.py
index 0b78115eeb..a05f0d09a6 100644
--- a/tensorflow/python/kernel_tests/lookup_ops_test.py
+++ b/tensorflow/python/kernel_tests/lookup_ops_test.py
@@ -1123,22 +1123,24 @@ class InitializeTableFromFileOpTest(test.TestCase):
 
       # Invalid data type
       other_type = constant_op.constant(1)
-      with self.assertRaises(ValueError):
+      with self.assertRaises(Exception) as cm:
         lookup_ops.HashTable(
             lookup_ops.TextFileInitializer(
                 other_type, dtypes.string, lookup_ops.TextFileIndex.WHOLE_LINE,
                 dtypes.int64, lookup_ops.TextFileIndex.LINE_NUMBER),
             default_value)
+      self.assertTrue(isinstance(cm.exception, (ValueError, TypeError)))
 
       # Non-scalar filename
       filenames = constant_op.constant([vocabulary_file, vocabulary_file])
       if not context.executing_eagerly():
-        with self.assertRaises(ValueError):
+        with self.assertRaises(Exception) as cm:
           lookup_ops.HashTable(
               lookup_ops.TextFileInitializer(
                   filenames, dtypes.string, lookup_ops.TextFileIndex.WHOLE_LINE,
                   dtypes.int64, lookup_ops.TextFileIndex.LINE_NUMBER),
               default_value)
+        self.assertTrue(isinstance(cm.exception, (ValueError, TypeError)))
       else:
         with self.assertRaises(errors_impl.InvalidArgumentError):
           lookup_ops.HashTable(
diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
index 9fb1fe9fb8..8376d1d6e2 100644
--- a/tensorflow/python/saved_model/load.py
+++ b/tensorflow/python/saved_model/load.py
@@ -111,14 +111,16 @@ class _Loader(object):
       concrete_function._func_graph.variables = bound_variables  # pylint: disable=protected-access
 
   def _get_tensor_from_node(self, node_id):
-    obj = self._nodes[node_id]
-    if resource_variable_ops.is_resource_variable(obj):
-      return obj.handle
-    elif isinstance(obj, tracking.TrackableAsset):
-      return obj.asset_path.handle
-    elif tensor_util.is_tensor(obj):
-      return obj
-    raise ValueError("Can't convert node %s to tensor" % (type(obj)))
+    """Resolves a node id into a tensor to be captured for a function."""
+    with ops.init_scope():
+      obj = self._nodes[node_id]
+      if resource_variable_ops.is_resource_variable(obj):
+        return obj.handle
+      elif isinstance(obj, tracking.TrackableAsset):
+        return obj.asset_path
+      elif tensor_util.is_tensor(obj):
+        return obj
+      raise ValueError("Can't convert node %s to tensor" % (type(obj)))
 
   def _load_all(self):
     """Load all saved objects and wire their properties."""
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index 1edb08943b..e5b07e8619 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -126,12 +126,35 @@ class LoadTest(test.TestCase, parameterized.TestCase):
         lambda: root.vocab.asset_path,
         input_signature=[])
     imported = self.cycle(root, cycles)
-    origin_output = root.f().numpy()
+    original_output = root.f().numpy()
     imported_output = imported.f().numpy()
-    self.assertNotEqual(origin_output, imported_output)
+    self.assertNotEqual(original_output, imported_output)
     with open(imported_output, "r") as f:
       self.assertEqual("contents", f.read())
 
+  def test_capture_assets_in_graph(self, cycles):
+    root = tracking.AutoCheckpointable()
+    root.vocab = tracking.TrackableAsset(self._make_asset("contents"))
+    root.f = def_function.function(
+        lambda: root.vocab.asset_path,
+        input_signature=[])
+
+    original_output = root.f().numpy()
+
+    if cycles > 1:
+      root = self.cycle(root, cycles - 1)
+    path = tempfile.mkdtemp(prefix=self.get_temp_dir())
+    save.save(root, path)
+
+    with ops.Graph().as_default():
+      imported = load.load(path)
+      imported_tensor = imported.f()
+      with monitored_session.MonitoredSession() as sess:
+        imported_output = sess.run(imported_tensor)
+        self.assertNotEqual(original_output, imported_output)
+        with open(imported_output, "r") as f:
+          self.assertEqual("contents", f.read())
+
   def test_dedup_assets(self, cycles):
     vocab = self._make_asset("contents")
     root = tracking.AutoCheckpointable()
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index 5b4f0161af..b50bb61d9c 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -225,7 +225,7 @@ class _SaveableView(object):
         self.captured_tensor_node_ids[obj.handle] = node_id
       elif isinstance(obj, tracking.TrackableAsset):
         _process_asset(obj, asset_info, resource_map)
-        self.captured_tensor_node_ids[obj.asset_path.handle] = node_id
+        self.captured_tensor_node_ids[obj.asset_path] = node_id
 
     for concrete_function in self.concrete_functions:
       for capture in concrete_function.captured_inputs:
@@ -466,10 +466,10 @@ def _process_asset(trackable_asset, asset_info, resource_map):
   asset_def.filename = path
   asset_def.tensor_info.name = asset_path_initializer.name
   asset_info.asset_defs.append(asset_def)
-  asset_info.asset_initializers_by_resource[original_variable.handle] = (
+  asset_info.asset_initializers_by_resource[original_variable] = (
       asset_variable.initializer)
   asset_info.asset_index[trackable_asset] = len(asset_info.asset_defs) - 1
-  resource_map[original_variable.handle] = asset_variable.handle
+  resource_map[original_variable] = asset_variable
 
 
 def _fill_meta_graph_def(meta_graph_def, saveable_view, signature_functions):
diff --git a/tensorflow/python/saved_model/saved_object_graph.proto b/tensorflow/python/saved_model/saved_object_graph.proto
index c57bf15efa..fe7346dc32 100644
--- a/tensorflow/python/saved_model/saved_object_graph.proto
+++ b/tensorflow/python/saved_model/saved_object_graph.proto
@@ -78,9 +78,9 @@ message SavedUserObject {
 
 // A SavedAsset represents a file in a SavedModel.
 //
-// When bound to a function this object evaluates to a Variable from which the
-// absolute filename can be read. Users should not expect the filename to be
-// maintained.
+// When bound to a function this object evaluates to a tensor with the absolute
+// filename. Users should not depend on a particular part of the filename to
+// remain stable (e.g. basename could be changed).
 message SavedAsset {
   // Index into `MetaGraphDef.asset_file_def[]` that describes the Asset.
   //
diff --git a/tensorflow/python/training/checkpointable/tracking.py b/tensorflow/python/training/checkpointable/tracking.py
index 83b0d8bcf2..b17b963dc4 100644
--- a/tensorflow/python/training/checkpointable/tracking.py
+++ b/tensorflow/python/training/checkpointable/tracking.py
@@ -17,12 +17,10 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.python.eager import context
 from tensorflow.python.eager import def_function
 from tensorflow.python.eager import function as defun
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
-from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.training.checkpointable import base
 from tensorflow.python.training.checkpointable import data_structures
 from tensorflow.python.util import tf_contextlib
@@ -184,23 +182,12 @@ class TrackableAsset(base.Checkpointable):
 
   def __init__(self, path):
     """Record the full path to the asset."""
-    # We use a variable here so that @tf.functions do not capture a literal
-    # value. The init_scope prevents functions from capturing `path` in an
+    # The init_scope prevents functions from capturing `path` in an
     # initialization graph, since it is transient and should not end up in a
-    # serialized function body. When serialized in a SavedModel, the variable
-    # will be set during the loading process to its location in the assets/
-    # directory.
+    # serialized function body.
     with ops.init_scope():
-      if context.executing_eagerly():
-        self._path = self._no_dependency(
-            resource_variable_ops.ResourceVariable(
-                path, dtype=dtypes.string,
-                name="asset_path"))
-      else:
-        # Adding a variable is too disruptive when v1-style graph building,
-        # since things may get fed and local variable initializers would then
-        # need to be run.
-        self._path = path
+      self._path = ops.internal_convert_to_tensor(path, dtype=dtypes.string,
+                                                  name="asset_path")
 
   @property
   def asset_path(self):
-- 
GitLab


From efed3f6fc94048764ddc8e1fc5eeee59b82a39a3 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Fri, 8 Feb 2019 02:25:28 -0800
Subject: [PATCH 2136/2345] Make the comparison computation for Sort required.

This is used by the Sort emitters for comparisons instead of
the hard-coded comparisons of the first operands.
Adapt all HLO passes which work on Sort ops to handle the comparison
computation as well.

PiperOrigin-RevId: 233027849
---
 .../compiler/xla/client/lib/sorting_test.cc   |   4 +-
 tensorflow/compiler/xla/client/xla_builder.cc | 134 +++++++++++-
 tensorflow/compiler/xla/client/xla_builder.h  |  34 ++-
 .../compiler/xla/service/cpu/ir_emitter.cc    |  71 +-----
 .../xla/service/cpu/runtime_key_value_sort.cc |  18 +-
 .../xla/service/cpu/runtime_key_value_sort.h  |  14 +-
 .../xla/service/gpu/ir_emitter_unnested.cc    |  28 +--
 .../xla/service/hlo_element_type_converter.cc |   8 +-
 .../compiler/xla/service/hlo_evaluator.cc     | 111 ++++------
 .../xla/service/hlo_evaluator_typed_visitor.h |  40 ----
 .../compiler/xla/service/hlo_instruction.cc   |  36 +---
 .../compiler/xla/service/hlo_instruction.h    |   5 -
 .../compiler/xla/service/hlo_instructions.cc  |  25 +--
 .../compiler/xla/service/hlo_instructions.h   |   3 -
 tensorflow/compiler/xla/service/hlo_parser.cc |  14 +-
 .../compiler/xla/service/hlo_verifier.cc      |  41 +++-
 .../compiler/xla/service/llvm_ir/sort_util.cc | 202 ++++++++----------
 .../compiler/xla/service/llvm_ir/sort_util.h  |  23 +-
 .../compiler/xla/service/sort_simplifier.cc   |  59 ++++-
 .../xla/service/sort_simplifier_test.cc       |  32 +++
 20 files changed, 474 insertions(+), 428 deletions(-)

diff --git a/tensorflow/compiler/xla/client/lib/sorting_test.cc b/tensorflow/compiler/xla/client/lib/sorting_test.cc
index 0fbd138aca..ae78910a5b 100644
--- a/tensorflow/compiler/xla/client/lib/sorting_test.cc
+++ b/tensorflow/compiler/xla/client/lib/sorting_test.cc
@@ -81,7 +81,9 @@ XLA_TEST_F(SortingTest, TopKFullSort) {
   ComputeAndCompareR1<float>(&builder, inputs, {});
 }
 
-XLA_TEST_F(SortingTest, TopKFullSortWithDuplicates) {
+// TODO(b/122298745): Enable this test when the GPU backend supports stable
+// sorting.
+XLA_TEST_F(SortingTest, DISABLED_ON_GPU(TopKFullSortWithDuplicates)) {
   XlaBuilder builder(TestName());
   XlaOp a;
   auto a_data = CreateR1Parameter<int>({1, 1, 2, 2, 1}, 0, "a", &builder, &a);
diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc
index 82375f16fd..fb9dbe851e 100644
--- a/tensorflow/compiler/xla/client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_builder.cc
@@ -1551,27 +1551,140 @@ XlaOp XlaBuilder::Rev(const XlaOp& operand,
   });
 }
 
+namespace {
+// Switch from a floating point value to a integer value in such a way that when
+// using the integer value to compare, we get the same result for normal values,
+// and -Nan is treated as the smallest value, and Nan is treated as the largest
+// value.
+// If f is a float, and
+// x = bit_cast<int32>(f);
+// y = x < 0 ? numeric_limits<int32>::max() - x : x;
+// then y is ordered as an int32 such that finite values have the obvious order,
+// -0 is ordered before 0, and -NaN and NaN appear at the beginning and end of
+// the ordering.
+// Note that in order to avoid -x to overflow, we calculate
+// numeric_limits<int32>::max() - x as unsigned, and then convert back to
+// signed.
+XlaOp BitcastConvertFloatingPointToIntegral(const XlaOp& value,
+                                            int64 bit_width) {
+  PrimitiveType signed_type;
+  PrimitiveType unsigned_type;
+  XlaOp max_value;
+  switch (bit_width) {
+    case 16:
+      max_value =
+          ConstantR0(value.builder(),
+                     static_cast<uint16>(std::numeric_limits<int16>::max()));
+      signed_type = S16;
+      unsigned_type = U16;
+      break;
+    case 32:
+      max_value =
+          ConstantR0(value.builder(),
+                     static_cast<uint32>(std::numeric_limits<int32>::max()));
+      signed_type = S32;
+      unsigned_type = U32;
+      break;
+    case 64:
+      max_value =
+          ConstantR0(value.builder(),
+                     static_cast<uint64>(std::numeric_limits<int64>::max()));
+      signed_type = S64;
+      unsigned_type = U64;
+      break;
+    default:
+      return value.builder()->ReportError(
+          InvalidArgument("Invalid bit width %lld for Comparator floating "
+                          "point parameter.",
+                          bit_width));
+  }
+  auto signed_value = BitcastConvertType(value, signed_type);
+  auto unsigned_value = BitcastConvertType(value, unsigned_type);
+  auto flipped_value =
+      BitcastConvertType(Sub(max_value, unsigned_value), signed_type);
+  auto is_negative =
+      Lt(signed_value,
+         ConstantLiteral(value.builder(), LiteralUtil::Zero(signed_type)));
+  return Select(is_negative, flipped_value, signed_value);
+}
+}  // namespace
+
 XlaOp XlaBuilder::Sort(const XlaOp& keys, absl::Span<const XlaOp> values,
                        int64 dimension) {
+  return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    std::vector<XlaOp> operands{keys};
+    for (const XlaOp& value : values) {
+      operands.push_back(value);
+    }
+    // Build the default less-than comparator (copied from lib/comparators.cc).
+    // TODO(b/122298745): Remove the deprecated API method so that this code
+    // duplication can be deleted.
+    auto b = this->CreateSubBuilder("comparator");
+    std::vector<PrimitiveType> operand_types;
+    for (const XlaOp& operand : operands) {
+      TF_ASSIGN_OR_RETURN(auto operand_shape, GetShape(operand));
+      operand_types.push_back(operand_shape.element_type());
+    }
+
+    int64 parameter_count = 0;
+    XlaOp first_lhs_param;
+    XlaOp first_rhs_param;
+
+    for (auto operand_type : operand_types) {
+      auto scalar_shape = ShapeUtil::MakeShape(operand_type, {});
+      auto lhs_param =
+          b->Parameter(parameter_count * 2, scalar_shape,
+                       absl::StrCat("p.", parameter_count, ".lhs"));
+      auto rhs_param =
+          b->Parameter(parameter_count * 2 + 1, scalar_shape,
+                       absl::StrCat("p.", parameter_count, ".rhs"));
+      if (parameter_count == 0) {
+        first_lhs_param = lhs_param;
+        first_rhs_param = rhs_param;
+      }
+      ++parameter_count;
+    }
+    if (primitive_util::IsFloatingPointType(operand_types[0])) {
+      PrimitiveType compare_type = operand_types[0];
+      // Special-case handling for BF16. We currently do not support direct
+      // comparisons with BF16, so we convert to F32 and then use the F32
+      // comparison logic.
+      if (compare_type == BF16) {
+        compare_type = F32;
+        first_lhs_param = b->ConvertElementType(first_lhs_param, F32);
+        first_rhs_param = b->ConvertElementType(first_rhs_param, F32);
+      }
+      int64 bit_width = primitive_util::BitWidth(compare_type);
+      first_lhs_param =
+          BitcastConvertFloatingPointToIntegral(first_lhs_param, bit_width);
+      first_rhs_param =
+          BitcastConvertFloatingPointToIntegral(first_rhs_param, bit_width);
+    }
+    Lt(first_lhs_param, first_rhs_param);
+
+    TF_ASSIGN_OR_RETURN(auto comparator, b->Build());
+    return Sort(operands, comparator, dimension);
+  });
+}
+
+XlaOp XlaBuilder::Sort(absl::Span<const XlaOp> operands,
+                       const XlaComputation& comparator, int64 dimension) {
   return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
     HloInstructionProto instr;
     std::vector<const Shape*> operand_shape_ptrs;
-    TF_ASSIGN_OR_RETURN(const Shape& keys_shape, GetShape(keys));
-    operand_shape_ptrs.push_back(&keys_shape);
-    TF_ASSIGN_OR_RETURN(std::vector<Shape> values_shapes,
-                        GetOperandShapes(values));
-    absl::c_transform(values_shapes, std::back_inserter(operand_shape_ptrs),
+    TF_ASSIGN_OR_RETURN(std::vector<Shape> operand_shapes,
+                        GetOperandShapes(operands));
+    absl::c_transform(operand_shapes, std::back_inserter(operand_shape_ptrs),
                       [](const Shape& shape) { return &shape; });
     TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferVariadicOpShape(
                                          HloOpcode::kSort, operand_shape_ptrs));
     *instr.mutable_shape() = shape.ToProto();
     if (dimension == -1) {
-      TF_ASSIGN_OR_RETURN(const Shape& keys_shape, GetShape(keys));
+      TF_ASSIGN_OR_RETURN(const Shape& keys_shape, GetShape(operands[0]));
       dimension = keys_shape.rank() - 1;
     }
     instr.add_dimensions(dimension);
-    std::vector<XlaOp> operands{keys};
-    operands.insert(operands.end(), values.begin(), values.end());
+    AddCalledComputation(comparator, &instr);
     return AddInstruction(std::move(instr), HloOpcode::kSort, operands);
   });
 }
@@ -3206,6 +3319,11 @@ XlaOp Sort(const XlaOp& keys, absl::Span<const XlaOp> values, int64 dimension) {
   return keys.builder()->Sort(keys, values, dimension);
 }
 
+XlaOp Sort(absl::Span<const XlaOp> operands, const XlaComputation& comparator,
+           int64 dimension) {
+  return operands[0].builder()->Sort(operands, comparator, dimension);
+}
+
 XlaOp Clamp(const XlaOp& min, const XlaOp& operand, const XlaOp& max) {
   return min.builder()->Clamp(min, operand, max);
 }
diff --git a/tensorflow/compiler/xla/client/xla_builder.h b/tensorflow/compiler/xla/client/xla_builder.h
index 384ccfa25a..1e39c8766f 100644
--- a/tensorflow/compiler/xla/client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_builder.h
@@ -501,8 +501,11 @@ class XlaBuilder {
 
   XlaOp Rev(const XlaOp& operand, absl::Span<const int64> dimensions);
 
+  ABSL_DEPRECATED("Use form with comparator computation instead")
   XlaOp Sort(const XlaOp& keys, absl::Span<const XlaOp> values = {},
              int64 dimension = -1);
+  XlaOp Sort(absl::Span<const XlaOp> operands, const XlaComputation& comparator,
+             int64 dimension = -1);
 
   XlaOp Clamp(const XlaOp& min, const XlaOp& operand, const XlaOp& max);
 
@@ -919,6 +922,8 @@ class XlaBuilder {
   friend XlaOp Rev(const XlaOp& operand, absl::Span<const int64> dimensions);
   friend XlaOp Sort(const XlaOp& keys, absl::Span<const XlaOp> values,
                     int64 dimension);
+  friend XlaOp Sort(absl::Span<const XlaOp> operands,
+                    const XlaComputation& comparator, int64 dimension);
   friend XlaOp Clamp(const XlaOp& min, const XlaOp& operand, const XlaOp& max);
   friend XlaOp Map(XlaBuilder* builder, absl::Span<const XlaOp> operands,
                    const XlaComputation& computation,
@@ -1675,7 +1680,7 @@ XlaOp Rev(const XlaOp& operand, absl::Span<const int64> dimensions);
 // of keys, in ascending order.
 // * If the keys have higher rank, the keys are sorted along the provided
 // dimension. For example, for a rank-2 tensor (a matrix) of keys, a dimension
-// value of 0 will indepenently sort every column, and a dimension value of 1
+// value of 0 will independently sort every column, and a dimension value of 1
 // will independently sort each row. If no dimension number is provided, then
 // the last dimension is chosen by default.
 //
@@ -1685,9 +1690,36 @@ XlaOp Rev(const XlaOp& operand, absl::Span<const int64> dimensions);
 // * The result is a tuple that consists of a sorted tensor of keys (along the
 // provided dimension, as above) as the first element, and tensors with their
 // corresponding values as the other elements.
+ABSL_DEPRECATED("Use form with comparator computation instead")
 XlaOp Sort(const XlaOp& keys, absl::Span<const XlaOp> values = {},
            int64 dimension = -1);
 
+// Enqueues a sort instruction onto the computation, using 'comparator' for
+// comparisons. 'comparator' needs to define a strict weak order.
+// If only one operand is provided:
+// * If the operand is a rank-1 tensor (an array), the result is a sorted array.
+//   The resulting sorting order has the property that for all index positions
+//   i, j with i < j, either
+//   comparator(value[i], value[j]) = comparator(value[j], value[i]) = false or
+//   comparator(value[i], value[j]) = true.
+// * If the operand has higher rank, the operand is sorted along the provided
+//   dimension. For example, for a rank-2 tensor (a matrix), a dimension value
+//   of 0 will independently sort every column, and a dimension value of 1 will
+//   independently sort each row. If no dimension number is provided, then the
+//   last dimension is chosen by default. For the dimension which is sorted, the
+//   same sorting order applies as in the rank-1 case.
+//
+// If more than one operand is provided:
+// * All operands must be tensors with the same dimensions. The element types of
+//   the tensors may be different.
+// * The result is a tuple that consists of the operands in sorted order (along
+//   the provided dimension, as above). When comparing, 'comparator' is called
+//   with 2 * n scalar parameters, where parameter 2 * i and 2 * i + 1
+//   correspond to the value of operand i at two index positions.
+// Default comparator computations can be found in lib/comparators.h
+XlaOp Sort(absl::Span<const XlaOp> operands, const XlaComputation& comparator,
+           int64 dimension = -1);
+
 // Enqueues a clamp instruction onto the computation.
 XlaOp Clamp(const XlaOp& min, const XlaOp& operand, const XlaOp& max);
 
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index 4b96f31fbe..dd631896ba 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -517,6 +517,7 @@ Status IrEmitter::HandleSort(HloInstruction* hlo) {
     case U8:
     case S16:
     case U16:
+    case BF16:
     case F16:
     case S32:
     case U32:
@@ -577,72 +578,14 @@ Status IrEmitter::HandleSort(HloInstruction* hlo) {
     lower_dimensions *= normalized_keys_shape.dimensions(i);
   }
 
-  llvm::FunctionType* less_than_type = llvm::FunctionType::get(
-      b_.getInt1Ty(), {b_.getInt8PtrTy(), b_.getInt8PtrTy()},
-      /*isVarArg=*/false);
-  auto less_than_function = llvm_ir::CreateFunction(
-      less_than_type, llvm::GlobalValue::InternalLinkage,
-      /*enable_fast_math=*/false,
-      /*optimize_for_size=*/true, absl::StrCat(IrName(sort), "_comparator"),
-      module_);
-  // Emit the code for the less_than function.
-  {
-    llvm::IRBuilder<>::InsertPointGuard guard(b_);
-
-    auto* entry_bb =
-        llvm::BasicBlock::Create(b_.getContext(), "entry", less_than_function);
-
-    b_.SetInsertPoint(entry_bb);
-    auto keys_ir_type = llvm_ir::PrimitiveTypeToIrType(keys_type, module_);
-    CHECK_EQ(less_than_function->arg_size(), 2);
-    llvm::Value* keys_lhs_ptr = less_than_function->arg_begin();
-    keys_lhs_ptr = PointerCast(keys_lhs_ptr, keys_ir_type->getPointerTo());
-    llvm::Value* keys_rhs_ptr = less_than_function->arg_begin() + 1;
-    keys_rhs_ptr = PointerCast(keys_rhs_ptr, keys_ir_type->getPointerTo());
-
-    // TODO(b/122298745): Replace the custom compare logic with a call to the
-    // computation specified for the Sort op.
-    llvm::Value* keys_lhs = Load(keys_ir_type, keys_lhs_ptr);
-    llvm::Value* keys_rhs = Load(keys_ir_type, keys_rhs_ptr);
-    bool is_signed_comparison = true;
-    if (primitive_util::IsFloatingPointType(keys_type)) {
-      // We would like a total order of floating point numbers so that the
-      // sort has a predictable behavior in the presence of NaNs. Rather
-      // than using floating point comparison, we use the following trick:
-      // If f is a float, and
-      // x = bit_cast<int32>(f);
-      // y = x < 0 ? 0x7FFFFFFF - x : x;
-      // then y is ordered as an int32 such that finite values have the
-      // obvious order, -0 is ordered before 0, and -NaN and NaN appear at
-      // the beginning and end of the ordering.
-      auto k = b_.getInt(llvm::APInt::getSignedMaxValue(
-          keys_lhs->getType()->getPrimitiveSizeInBits()));
-      auto comparison_type = k->getType();
-      auto zero = llvm::ConstantInt::get(comparison_type, 0);
-      auto maybe_flip = [&](llvm::Value* v) {
-        return b_.CreateSelect(b_.CreateICmp(llvm::ICmpInst::ICMP_SLT, v, zero),
-                               b_.CreateSub(k, v), v);
-      };
-      keys_lhs = b_.CreateBitCast(keys_lhs, comparison_type);
-      keys_rhs = b_.CreateBitCast(keys_rhs, comparison_type);
-      keys_lhs = maybe_flip(keys_lhs);
-      keys_rhs = maybe_flip(keys_rhs);
-    } else if (!primitive_util::IsSignedIntegralType(keys_type)) {
-      is_signed_comparison = false;
-    }
-    llvm::Value* result =
-        b_.CreateICmp(is_signed_comparison ? llvm::ICmpInst::ICMP_SLT
-                                           : llvm::ICmpInst::ICMP_ULT,
-                      keys_lhs, keys_rhs);
-    llvm::ReturnInst::Create(b_.getContext(),
-                             /*retVal=*/result, entry_bb);
-  }
-
+  auto less_than_function = FindOrDie(emitted_functions_, sort->to_apply());
+  CHECK(absl::c_binary_search(thread_local_computations_, sort->to_apply()));
   llvm::FunctionType* key_value_sort_type = llvm::FunctionType::get(
       b_.getVoidTy(),
       {b_.getInt64Ty(), b_.getInt64Ty(), b_.getInt64Ty(),
        b_.getInt8PtrTy()->getPointerTo(), b_.getInt32Ty(),
-       b_.getInt32Ty()->getPointerTo(), less_than_function->getType()},
+       b_.getInt32Ty()->getPointerTo(), b_.getInt8PtrTy(),
+       b_.getInt64Ty()->getPointerTo(), less_than_function->getType()},
       /*isVarArg=*/false);
   auto* key_value_sort_func = llvm::dyn_cast<llvm::Function>(
       module_
@@ -673,7 +616,9 @@ Status IrEmitter::HandleSort(HloInstruction* hlo) {
   Call(key_value_sort_func,
        {b_.getInt64(higher_dimensions), b_.getInt64(sort_dimension_elements),
         b_.getInt64(lower_dimensions), values,
-        b_.getInt32(sort->operand_count()), sizes, less_than_function});
+        b_.getInt32(sort->operand_count()), sizes,
+        GetExecutableRunOptionsArgument(), GetProfileCountersArgument(),
+        less_than_function});
 
   if (sort->values_count() > 0) {
     llvm_ir::EmitTuple(GetIrArrayFor(sort), destination_addresses, &b_,
diff --git a/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.cc b/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.cc
index a0667d0d9d..2cd2f323c7 100644
--- a/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.cc
+++ b/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.cc
@@ -32,8 +32,9 @@ using tensorflow::int64;
 
 TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSort(
     int64 a, int64 b, int64 c, char** values, int32 values_count,
-    int32* values_primitive_type_size_in_bytes,
-    bool (*less_than)(char*, char*)) {
+    int32* values_primitive_type_size_in_bytes, char* run_options,
+    int64* prof_counters,
+    void (*less_than)(char*, char*, char**, char**, tensorflow::int64*)) {
   // 'values' and 'values_primitive_type_size_in_bytes' are managed by the JIT
   // code, so msan can't tell they are initialized.
   TF_ANNOTATE_MEMORY_IS_INITIALIZED(values, values_count * sizeof(char*));
@@ -54,6 +55,7 @@ TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSort(
   int64 sort_dimension_offset = c;
 
   std::unique_ptr<int64[]> indices(new int64[sort_dimension_elements]);
+  std::unique_ptr<char*[]> comparison_values(new char*[2 * values_count]);
   std::iota(indices.get(), indices.get() + sort_dimension_elements, 0);
   std::unique_ptr<std::string[]> reordered_values(
       new std::string[sort_dimension_elements]);
@@ -69,13 +71,19 @@ TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSort(
         (index - index % sort_dimension_offset) * sort_dimension_elements;
     std::stable_sort(
         indices.get(), indices.get() + sort_dimension_elements,
-        [&](int64 a, int64 b) {
+        [&](int64 a, int64 b) -> bool {
           int64 memory_index_lhs = (base_offset + a * sort_dimension_offset) *
                                    values_primitive_type_size_in_bytes[0];
           int64 memory_index_rhs = (base_offset + b * sort_dimension_offset) *
                                    values_primitive_type_size_in_bytes[0];
-          return less_than(values[0] + memory_index_lhs,
-                           values[0] + memory_index_rhs);
+          for (int32 i = 0; i < values_count; ++i) {
+            comparison_values[i * 2] = values[i] + memory_index_lhs;
+            comparison_values[i * 2 + 1] = values[i] + memory_index_rhs;
+          }
+          char result;
+          less_than(&result, run_options, comparison_values.get(), nullptr,
+                    prof_counters);
+          return result != 0u;
         });
 
     // Reorder the values according to the order defined by 'indices'.
diff --git a/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.h b/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.h
index 5460af3485..4813de9ee6 100644
--- a/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.h
+++ b/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.h
@@ -28,12 +28,20 @@ extern "C" {
 // 'values' and 'values_primitive_type_size_in_bytes'. The size of the primitive
 // type of the i-th shape has exactly 'values_primitive_type_size_in_bytes[i]'
 // bytes. The elements in each 'values' shape are reordered in the same way
-// according to the comparisons using the first shape.
+// according to the comparisons using the first shape. 'run_options' and
+// 'prof_counters' are passed through to the less-than function, which expects
+// the following arguments:
+// - pointer to the return value buffer (char*)
+// - xla::ExecutableRunOptions = 'run_options' (char*)
+// - pointers to the parameter buffers (char**)
+// - pointers to the buffer tables = nullptr for thread local functions (char**)
+// - profile counters = 'prof_counters' (int64*)
 extern void __xla_cpu_runtime_KeyValueSort(
     tensorflow::int64 a, tensorflow::int64 b, tensorflow::int64 c,
     char** values, tensorflow::int32 values_count,
-    tensorflow::int32* values_primitive_type_size_in_bytes,
-    bool (*less_than)(char*, char*));
+    tensorflow::int32* values_primitive_type_size_in_bytes, char* run_options,
+    tensorflow::int64* prof_counters,
+    void (*less_than)(char*, char*, char**, char**, tensorflow::int64*));
 }
 
 #endif  // TENSORFLOW_COMPILER_XLA_SERVICE_CPU_RUNTIME_KEY_VALUE_SORT_H_
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index 26b266a066..0cc65ebb52 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -1153,17 +1153,7 @@ Status IrEmitterUnnested::HandleSort(HloInstruction* sort) {
   std::vector<std::unique_ptr<Thunk>> thunks;
   Shape keys_shape = sort->operand(0)->shape();
   int64 dimension_to_sort = sort->dimensions(0);
-  // In case there is a 'values' parameter that is a iota, we take note and use
-  // it later to ensure a stable sort. Otherwise, we don't guarantee a stable
-  // sort.
-  int64 iota_values_parameter_index = -1;
   for (int64 i = 0; i < sort->operand_count(); ++i) {
-    if (i > 0 && sort->operand(i)->opcode() == HloOpcode::kIota &&
-        ShapeUtil::ElementIsIntegral(sort->operand(i)->shape()) &&
-        Cast<HloIotaInstruction>(sort->operand(i))->iota_dimension() ==
-            dimension_to_sort) {
-      iota_values_parameter_index = i;
-    }
     ShapeIndex shape_index =
         sort->operand_count() > 1 ? ShapeIndex({i}) : ShapeIndex({});
     // We assume that the layout of all involved operands and outputs is the
@@ -1276,25 +1266,23 @@ Status IrEmitterUnnested::HandleSort(HloInstruction* sort) {
                                              : standard_launch_dimensions;
     UpdateLaunchDimensions(launch_dimensions, thunks.back().get(),
                            ir_emitter_context_->llvm_module());
-    IrArray keys_array;
     std::vector<IrArray> values_arrays;
-    values_arrays.reserve(sort->operand_count() - 1);
+    values_arrays.reserve(sort->operand_count());
     for (int64 i = 0; i < sort->operand_count(); ++i) {
       ShapeIndex shape_index =
           sort->operand_count() > 1 ? ShapeIndex({i}) : ShapeIndex({});
-      if (i == 0) {
-        keys_array = GetIrArray(*sort, *sort, shape_index);
-      } else {
-        values_arrays.push_back(GetIrArray(*sort, *sort, shape_index));
-      }
+      values_arrays.push_back(GetIrArray(*sort, *sort, shape_index));
     }
     return llvm_ir::EmitSortInPlace(
-        dimension_to_sort, keys_array, values_arrays,
-        iota_values_parameter_index, IrName(sort), xor_masks, &b_,
+        dimension_to_sort, values_arrays, IrName(sort), xor_masks, &b_,
         launch_dimensions,
         xor_masks.size() > 1 ? num_iterations_in_sort_dim
                              : standard_num_iterations_in_sort_dim,
-        kTileSize);
+        kTileSize,
+        [&](absl::Span<llvm::Value* const> operands, llvm::Value* output) {
+          return EmitCallToNestedComputation(*sort->to_apply(), operands,
+                                             output);
+        });
   };
   std::vector<int64> xor_masks;
   for (int64 stage = 0; stage < num_stages; ++stage) {
diff --git a/tensorflow/compiler/xla/service/hlo_element_type_converter.cc b/tensorflow/compiler/xla/service/hlo_element_type_converter.cc
index bff467772d..7d6b86056a 100644
--- a/tensorflow/compiler/xla/service/hlo_element_type_converter.cc
+++ b/tensorflow/compiler/xla/service/hlo_element_type_converter.cc
@@ -146,14 +146,10 @@ StatusOr<bool> HloElementTypeConverter::Run(HloModule* module) {
           opcode == HloOpcode::kMap || opcode == HloOpcode::kReduce ||
           opcode == HloOpcode::kReduceWindow || opcode == HloOpcode::kScatter ||
           opcode == HloOpcode::kSelectAndScatter ||
-          opcode == HloOpcode::kConditional) {
+          opcode == HloOpcode::kSort || opcode == HloOpcode::kConditional) {
         continue;
       }
-      // TODO(b/122298745): Once we don't ignore called computations anymore,
-      // add kSort to the if statement above.
-      if (opcode != HloOpcode::kSort) {
-        TF_RET_CHECK(hlo->called_computations().empty()) << hlo->ToString();
-      }
+      TF_RET_CHECK(hlo->called_computations().empty()) << hlo->ToString();
 
       bool nullary = hlo->operands().empty();
       bool wrong_element_type = hlo->shape().element_type() == eliminate_type_;
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index 56a1b6f439..691d5c1bbc 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -1461,14 +1461,6 @@ Status HloEvaluator::HandleSort(HloInstruction* sort) {
   }
   Shape key_shape = sort->operand(0)->shape();
   auto rank = key_shape.rank();
-  PrimitiveType keys_type = key_shape.element_type();
-  if (keys_type != F64 && keys_type != U64 && keys_type != S64 &&
-      keys_type != F32 && keys_type != U32 && keys_type != S32 &&
-      keys_type != BF16 && keys_type != F16 && keys_type != U16 &&
-      keys_type != S16 && keys_type != U8 && keys_type != S8) {
-    return InvalidArgument("Unsupported type for Sort: %s",
-                           PrimitiveType_Name(keys_type));
-  }
   std::vector<Literal> result_literals;
   result_literals.reserve(sort->operand_count());
   for (int64 i = 0; i < sort->operand_count(); ++i) {
@@ -1479,6 +1471,7 @@ Status HloEvaluator::HandleSort(HloInstruction* sort) {
   int64 sort_dim = sort->dimensions(0);
   int64 sort_dim_elements = key_shape.dimensions(sort_dim);
   increment[sort_dim] = sort_dim_elements;
+  HloEvaluator embedded_evaluator(max_loop_iterations_);
   // Iterate through each dimension except 'sort_dim'.
   TF_RETURN_IF_ERROR(ShapeUtil::ForEachIndexWithStatus(
       key_shape, zero_base, AsInt64Slice(key_shape.dimensions()), increment,
@@ -1499,78 +1492,48 @@ Status HloEvaluator::HandleSort(HloInstruction* sort) {
         }
         std::vector<int64> indices_to_sort(sort_dim_elements);
         std::iota(indices_to_sort.begin(), indices_to_sort.end(), 0);
+        Status compare_status = Status::OK();
         std::stable_sort(
             indices_to_sort.begin(), indices_to_sort.end(),
-            [keys_type, &literals_to_sort](int64 a, int64 b) {
-              switch (keys_type) {
-                case F64: {
-                  auto key_lhs = literals_to_sort[0].Get<double>({a});
-                  auto key_rhs = literals_to_sort[0].Get<double>({b});
-                  return SafeLess(key_lhs, key_rhs);
-                }
-                case U64: {
-                  auto key_lhs = literals_to_sort[0].Get<uint64>({a});
-                  auto key_rhs = literals_to_sort[0].Get<uint64>({b});
-                  return SafeLess(key_lhs, key_rhs);
-                }
-                case S64: {
-                  auto key_lhs = literals_to_sort[0].Get<int64>({a});
-                  auto key_rhs = literals_to_sort[0].Get<int64>({b});
-                  return SafeLess(key_lhs, key_rhs);
-                }
-                case F32: {
-                  auto key_lhs = literals_to_sort[0].Get<float>({a});
-                  auto key_rhs = literals_to_sort[0].Get<float>({b});
-                  return SafeLess(key_lhs, key_rhs);
-                }
-                case U32: {
-                  auto key_lhs = literals_to_sort[0].Get<uint32>({a});
-                  auto key_rhs = literals_to_sort[0].Get<uint32>({b});
-                  return SafeLess(key_lhs, key_rhs);
-                }
-                case S32: {
-                  auto key_lhs = literals_to_sort[0].Get<int32>({a});
-                  auto key_rhs = literals_to_sort[0].Get<int32>({b});
-                  return SafeLess(key_lhs, key_rhs);
-                }
-                case BF16: {
-                  auto key_lhs = literals_to_sort[0].Get<bfloat16>({a});
-                  auto key_rhs = literals_to_sort[0].Get<bfloat16>({b});
-                  return SafeLess(key_lhs, key_rhs);
-                }
-                case F16: {
-                  auto key_lhs = literals_to_sort[0].Get<Eigen::half>({a});
-                  auto key_rhs = literals_to_sort[0].Get<Eigen::half>({b});
-                  return SafeLess(key_lhs, key_rhs);
-                }
-                case U16: {
-                  auto key_lhs = literals_to_sort[0].Get<uint16>({a});
-                  auto key_rhs = literals_to_sort[0].Get<uint16>({b});
-                  return SafeLess(key_lhs, key_rhs);
-                }
-                case S16: {
-                  auto key_lhs = literals_to_sort[0].Get<int16>({a});
-                  auto key_rhs = literals_to_sort[0].Get<int16>({b});
-                  return SafeLess(key_lhs, key_rhs);
-                }
-                case U8: {
-                  auto key_lhs = literals_to_sort[0].Get<uint8>({a});
-                  auto key_rhs = literals_to_sort[0].Get<uint8>({b});
-                  return SafeLess(key_lhs, key_rhs);
-                }
-                case S8: {
-                  auto key_lhs = literals_to_sort[0].Get<int8>({a});
-                  auto key_rhs = literals_to_sort[0].Get<int8>({b});
-                  return SafeLess(key_lhs, key_rhs);
+            [sort, &compare_status, &embedded_evaluator, &literals_to_sort](
+                int64 a, int64 b) {
+              std::vector<Literal> literals;
+              literals.reserve(2 * sort->operand_count());
+              for (int64 i = 0; i < sort->operand_count(); ++i) {
+                auto lhs = ExtractFromIndexPositions(
+                    literals_to_sort[i], {a}, /*extract_as_scalar=*/true);
+                if (!lhs.ok()) {
+                  compare_status = lhs.status();
+                  return false;
                 }
-                default:
-                  // We should never reach here, because we checked earlier
-                  // that 'key_type' is one of the cases above.
-                  LOG(FATAL) << "Invalid key type in Sort: %s",
-                      PrimitiveType_Name(keys_type);
+                literals.push_back(std::move(lhs.ValueOrDie()));
+                auto rhs = ExtractFromIndexPositions(
+                    literals_to_sort[i], {b}, /*extract_as_scalar=*/true);
+                if (!rhs.ok()) {
+                  compare_status = rhs.status();
                   return false;
+                }
+                literals.push_back(std::move(rhs.ValueOrDie()));
               }
+              std::vector<const Literal*> literal_ptrs;
+              absl::c_transform(
+                  literals, std::back_inserter(literal_ptrs),
+                  [](const Literal& literal) { return &literal; });
+
+              auto computed_result =
+                  embedded_evaluator.Evaluate(*sort->to_apply(), literal_ptrs);
+              // Clear visit states so that we can use the evaluator again
+              // on the same computation.
+              embedded_evaluator.ResetVisitStates();
+              if (!computed_result.ok()) {
+                compare_status = computed_result.status();
+                return false;
+              }
+              return computed_result.ValueOrDie().Get<bool>({});
             });
+        if (!compare_status.ok()) {
+          return compare_status;
+        }
         std::vector<int64> slice_dimensions(rank, 1);
         slice_dimensions[sort_dim] = sort_dim_elements;
         std::vector<int64> start_indices(rank, 0);
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
index 652042e85f..68b67a9621 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
@@ -43,46 +43,6 @@ template <typename T>
 using is_complex_t =
     absl::disjunction<std::is_same<T, complex64>, std::is_same<T, complex128>>;
 
-// It's UB to use std::sort with std::less<float>, because of NaNs. Define
-// "safe" less functions which are actually strict weak orders. -NaN and NaN
-// should appear at the beginning and end of the ordering, and -0.0 should
-// appear before 0.0.
-template <
-    typename NativeT,
-    typename std::enable_if<std::is_integral<NativeT>::value>::type* = nullptr>
-bool SafeLess(const NativeT& a, const NativeT& b) {
-  return a < b;
-}
-
-template <typename NativeT, typename std::enable_if<std::is_floating_point<
-                                NativeT>::value>::type* = nullptr>
-bool SafeLess(const NativeT& a, const NativeT& b) {
-  bool lhs_is_negative = std::signbit(a);
-  bool rhs_is_negative = std::signbit(b);
-  // If the signs are different, we can just compare the signs.
-  if (lhs_is_negative != rhs_is_negative) {
-    return lhs_is_negative && !rhs_is_negative;
-  }
-  bool lhs_nan = std::isnan(a);
-  bool rhs_nan = std::isnan(b);
-  // Exactly one number is nan?
-  if (lhs_nan != rhs_nan) {
-    if (lhs_nan) {
-      return lhs_is_negative;
-    }
-    return !rhs_is_negative;
-  }
-  return a < b;
-}
-
-template <typename NativeT,
-          typename std::enable_if<
-              std::is_same<NativeT, bfloat16>::value ||
-              std::is_same<NativeT, Eigen::half>::value>::type* = nullptr>
-bool SafeLess(const NativeT& a, const NativeT& b) {
-  return SafeLess(static_cast<float>(a), static_cast<float>(b));
-}
-
 // ToArithmeticSafeType(T t):
 //  - converts `t` to the bitwise-equivalent `unsigned T` if T is a signed
 //    integer, and
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index acf881ac6a..aa1f3a2421 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -209,16 +209,12 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
           << proto.operand_ids_size();
       TF_RET_CHECK(proto.dimensions().size() == 1)
           << "Sort instruction should have 1 dimension";
+      TF_RET_CHECK(proto.called_computation_ids_size() == 1)
+          << "Sort instruction should one called computation but sees "
+          << proto.called_computation_ids_size();
       auto sort_operands = all_operands();
-      HloInstruction* keys = sort_operands[0];
-      if (proto.called_computation_ids_size() == 1) {
-        instruction = CreateSort(shape, proto.dimensions(0), all_operands(),
-                                 computations(0));
-      } else {
-        instruction = CreateSort(
-            shape, proto.dimensions(0), keys,
-            absl::Span<HloInstruction* const>(sort_operands).subspan(1));
-      }
+      instruction = CreateSort(shape, proto.dimensions(0), all_operands(),
+                               computations(0));
       break;
     }
     case HloOpcode::kTranspose:
@@ -1172,12 +1168,6 @@ HloInstruction::CreateBroadcastSequence(
   return absl::make_unique<HloTransposeInstruction>(shape, operand, dimensions);
 }
 
-/* static */ std::unique_ptr<HloInstruction> HloInstruction::CreateSort(
-    const Shape& shape, int64 dimension, HloInstruction* keys,
-    absl::Span<HloInstruction* const> values) {
-  return absl::make_unique<HloSortInstruction>(shape, dimension, keys, values);
-}
-
 /* static */ std::unique_ptr<HloInstruction> HloInstruction::CreateSort(
     const Shape& shape, int64 dimension,
     absl::Span<HloInstruction* const> operands, HloComputation* compare) {
@@ -2276,12 +2266,8 @@ std::vector<string> HloInstruction::ExtraAttributesToString(
                opcode() == HloOpcode::kAllReduce ||
                opcode() == HloOpcode::kScatter ||
                opcode() == HloOpcode::kSort) {
-      // TODO(b/122298745): Remove this check when Sort has a required
-      // sub-computation.
-      if (!called_computations().empty()) {
-        extra.push_back(
-            StrCat("to_apply=", PrintName(to_apply()->name(), options)));
-      }
+      extra.push_back(
+          StrCat("to_apply=", PrintName(to_apply()->name(), options)));
     } else if (!called_computations().empty()) {
       extra.push_back(StrCat(
           "calls=",
@@ -2317,12 +2303,8 @@ std::vector<string> HloInstruction::ExtraAttributesToString(
       case HloOpcode::kAllReduce:
       case HloOpcode::kScatter:
       case HloOpcode::kSort:
-        // TODO(b/122298745): Remove this check once sort has a required
-        // sub-computation.
-        if (to_apply() != nullptr) {
-          extra.push_back(
-              StrCat("to_apply=\n", to_apply()->ToString(new_options)));
-        }
+        extra.push_back(
+            StrCat("to_apply=\n", to_apply()->ToString(new_options)));
         break;
       default:
         if (!called_computations().empty()) {
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h
index d7da21619f..e8ade80ef3 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.h
+++ b/tensorflow/compiler/xla/service/hlo_instruction.h
@@ -674,11 +674,6 @@ class HloInstruction {
       const Shape& shape, HloInstruction* operand,
       absl::Span<const int64> dimensions);
 
-  // Creates a sort op, with a keys operand, and optional values operands.
-  static std::unique_ptr<HloInstruction> CreateSort(
-      const Shape& shape, int64 dimension, HloInstruction* keys,
-      absl::Span<HloInstruction* const> values = {});
-
   // Creates a n-ary sort op with a 'compare' computation which is used for
   // comparisons in the sorting algorithm. 'compare' gets 2 * n parameters,
   // where parameters 2 * i and 2 * i + 1 are the values of the i-th operand at
diff --git a/tensorflow/compiler/xla/service/hlo_instructions.cc b/tensorflow/compiler/xla/service/hlo_instructions.cc
index 27b46b272a..92a74187c5 100644
--- a/tensorflow/compiler/xla/service/hlo_instructions.cc
+++ b/tensorflow/compiler/xla/service/hlo_instructions.cc
@@ -657,16 +657,6 @@ std::unique_ptr<HloInstruction> HloReduceInstruction::CloneWithNewOperandsImpl(
                                                  dimensions(), to_apply());
 }
 
-HloSortInstruction::HloSortInstruction(const Shape& shape, int64 dimension,
-                                       HloInstruction* keys,
-                                       absl::Span<HloInstruction* const> values)
-    : HloInstruction(HloOpcode::kSort, shape), dimensions_({dimension}) {
-  AppendOperand(keys);
-  for (auto* value : values) {
-    AppendOperand(value);
-  }
-}
-
 HloSortInstruction::HloSortInstruction(
     const Shape& shape, int64 dimension,
     absl::Span<HloInstruction* const> operands, HloComputation* compare)
@@ -698,25 +688,14 @@ bool HloSortInstruction::IdenticalSlowPath(
   if (dimensions() != casted_other.dimensions()) {
     return false;
   }
-  if (called_computations().empty()) {
-    return other.called_computations().empty();
-  }
-  if (other.called_computations().empty()) {
-    return false;
-  }
   return eq_computations(to_apply(), other.to_apply());
 }
 
 std::unique_ptr<HloInstruction> HloSortInstruction::CloneWithNewOperandsImpl(
     const Shape& shape, absl::Span<HloInstruction* const> new_operands,
     HloCloneContext* context) const {
-  if (!called_computations().empty()) {
-    return absl::make_unique<HloSortInstruction>(shape, dimensions(0),
-                                                 new_operands, to_apply());
-  }
-  HloInstruction* keys = new_operands[0];
-  return absl::make_unique<HloSortInstruction>(shape, dimensions(0), keys,
-                                               new_operands.subspan(1));
+  return absl::make_unique<HloSortInstruction>(shape, dimensions(0),
+                                               new_operands, to_apply());
 }
 
 HloTransposeInstruction::HloTransposeInstruction(
diff --git a/tensorflow/compiler/xla/service/hlo_instructions.h b/tensorflow/compiler/xla/service/hlo_instructions.h
index 6cf6dab309..a0f2b46ba4 100644
--- a/tensorflow/compiler/xla/service/hlo_instructions.h
+++ b/tensorflow/compiler/xla/service/hlo_instructions.h
@@ -445,9 +445,6 @@ class HloReduceInstruction : public HloInstruction {
 
 class HloSortInstruction : public HloInstruction {
  public:
-  explicit HloSortInstruction(const Shape& shape, int64 dimension,
-                              HloInstruction* keys,
-                              absl::Span<HloInstruction* const> values = {});
   explicit HloSortInstruction(const Shape& shape, int64 dimension,
                               absl::Span<HloInstruction* const> operands,
                               HloComputation* compare);
diff --git a/tensorflow/compiler/xla/service/hlo_parser.cc b/tensorflow/compiler/xla/service/hlo_parser.cc
index 08fc728e3d..7c59c3a86b 100644
--- a/tensorflow/compiler/xla/service/hlo_parser.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser.cc
@@ -897,22 +897,14 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
       attrs["dimensions"] = {/*required=*/true, AttrTy::kBracedInt64List,
                              &dimensions};
       optional<HloComputation*> to_apply;
-      // TODO(b/122298745): Make this required.
-      attrs["to_apply"] = {/*required=*/false, AttrTy::kHloComputation,
+      attrs["to_apply"] = {/*required=*/true, AttrTy::kHloComputation,
                            &to_apply};
       if (!ParseOperands(&operands) || !ParseAttributes(attrs) ||
           dimensions->size() != 1) {
         return false;
       }
-      if (to_apply.has_value()) {
-        instruction = builder->AddInstruction(HloInstruction::CreateSort(
-            shape, dimensions->at(0), operands, to_apply.value()));
-      } else {
-        instruction = builder->AddInstruction(HloInstruction::CreateSort(
-            shape, dimensions->at(0),
-            /*keys=*/operands[0],
-            /*values=*/absl::Span<HloInstruction* const>(operands).subspan(1)));
-      }
+      instruction = builder->AddInstruction(HloInstruction::CreateSort(
+          shape, dimensions->at(0), operands, to_apply.value()));
       break;
     }
     case HloOpcode::kTuple: {
diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc
index 2ca62c6794..39153cc06c 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier.cc
@@ -337,13 +337,48 @@ Status ShapeVerifier::HandleSort(HloInstruction* sort) {
     return InternalError("Expected at least 1 operand for %s instruction: %s",
                          HloOpcodeString(sort->opcode()), sort->ToString());
   }
+  HloComputation* compare = sort->to_apply();
+
+  // Check that the 'compare' computation returns a PRED.
+  Shape compare_shape = compare->root_instruction()->shape();
+  if (!ShapesSame(compare_shape, ShapeUtil::MakeShape(PRED, {}))) {
+    return InternalError(
+        "The Sort compare computation shape does not lead to a scalar "
+        "predicate shape: %s",
+        StringifyShape(compare_shape));
+  }
+
+  // Check that the number of parameters of the 'compare' computation is
+  // correct.
+  TF_RETURN_IF_ERROR(
+      CheckParameterCount(sort, compare, sort->operand_count() * 2));
+
+  // Verify that the operands of the compare computation have the correct scalar
+  // shapes.
+  for (int64 parameter_idx = 0; parameter_idx < compare->num_parameters();
+       ++parameter_idx) {
+    int64 operand_idx = parameter_idx / 2;
+    Shape expected_scalar_shape = ShapeUtil::MakeShape(
+        sort->operand(operand_idx)->shape().element_type(), {});
+    Shape actual_parameter_shape =
+        compare->parameter_instruction(parameter_idx)->shape();
+    if (!ShapeUtil::CompatibleIgnoringFpPrecision(expected_scalar_shape,
+                                                  actual_parameter_shape)) {
+      return InternalError(
+          "Expected the %lld-th parameter of the compare computation of sort "
+          "to have shape %s, but got %s",
+          parameter_idx, StringifyShape(expected_scalar_shape),
+          StringifyShape(actual_parameter_shape));
+    }
+  }
+
+  // Verify that all operand shapes have the same dimensions.
   for (int64 operand = 1; operand < sort->operand_count(); ++operand) {
     if (!ShapeUtil::SameDimensions(sort->operand(0)->shape(),
                                    sort->operand(operand)->shape())) {
       return InternalError(
-          "Expected sort to have to have the same dimensions for the keys "
-          "and the values. Keys shape is: %s\n, Values shape (operand index "
-          "%lld) is: %s",
+          "Expected sort to have to have the same dimensions for all operands. "
+          "First operand shape is: %s\n, shape (operand index %lld) is: %s",
           StringifyShape(sort->operand(0)->shape()), operand,
           StringifyShape(sort->operand(operand)->shape()));
     }
diff --git a/tensorflow/compiler/xla/service/llvm_ir/sort_util.cc b/tensorflow/compiler/xla/service/llvm_ir/sort_util.cc
index 89b6a36f96..d71addec9b 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/sort_util.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/sort_util.cc
@@ -24,6 +24,7 @@ limitations under the License.
 #include "llvm/ADT/APInt.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constants.h"
+#include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/Value.h"
 #include "tensorflow/compiler/xla/primitive_util.h"
@@ -45,13 +46,14 @@ namespace llvm_ir {
 namespace {
 
 // Adds the inner comparison loop body where we compare elements.
-void EmitCompareLoopBody(
-    int64 iteration_bound, PrimitiveType key_type, int64 num_values,
-    int64 iota_values_parameter_index, llvm::Value* element_pair_index,
+Status EmitCompareLoopBody(
+    int64 iteration_bound, int64 num_values, llvm::Value* element_pair_index,
     int64 xor_mask, llvm::Type* index_type,
-    std::function<llvm::Value*(int64 operand, llvm::Value* index)> read_element,
+    std::function<llvm::Value*(int64 operand, llvm::Value* index)>
+        element_address,
     std::function<void(int64 operand, llvm::Value* index, llvm::Value* value)>
         write_element,
+    const EmitCallToNestedComputationCallback& emit_compare_callback,
     llvm::IRBuilder<>* b, bool needs_bounds_checks = true) {
   auto index_typed_constant = [&](int64 value) {
     return llvm::ConstantInt::get(index_type, value);
@@ -108,74 +110,44 @@ void EmitCompareLoopBody(
 
   // if (is_smaller_index && index_is_inbounds)
   KernelSupportLibrary ksl(b);
-  ksl.If("smaller_comparison_index", do_comparison, [&]() {
-    auto key1 = read_element(0, current_keys_index);
-    auto key2 = read_element(0, compare_keys_index);
-    auto compare_key1 = key1;
-    auto compare_key2 = key2;
-    bool is_signed_comparison = true;
-    if (primitive_util::IsFloatingPointType(key_type)) {
-      // We would like a total order of floating point numbers so that the
-      // sort has a predictable behavior in the presence of NaNs. Rather
-      // than using floating point comparison, we use the following trick:
-      // If f is a float, and
-      // x = bit_cast<int32>(f);
-      // y = x < 0 ? 0x7FFFFFFF - x : x;
-      // then y is ordered as an int32 such that finite values have the
-      // obvious order, -0 is ordered before 0, and -NaN and NaN appear at
-      // the beginning and end of the ordering.
-      auto k = b->getInt(llvm::APInt::getSignedMaxValue(
-          key1->getType()->getPrimitiveSizeInBits()));
-      auto comparison_type = k->getType();
-      auto zero = llvm::ConstantInt::get(comparison_type, 0);
-      auto maybe_flip = [&](llvm::Value* v) {
-        return b->CreateSelect(b->CreateICmp(llvm::ICmpInst::ICMP_SLT, v, zero),
-                               b->CreateSub(k, v), v);
-      };
-      compare_key1 = b->CreateBitCast(key1, comparison_type);
-      compare_key2 = b->CreateBitCast(key2, comparison_type);
-      compare_key1 = maybe_flip(compare_key1);
-      compare_key2 = maybe_flip(compare_key2);
-    } else if (!primitive_util::IsSignedIntegralType(key_type)) {
-      is_signed_comparison = false;
-    }
-    // If key2 < key1
-    auto is_smaller_than =
-        b->CreateICmp(is_signed_comparison ? llvm::ICmpInst::ICMP_SLT
-                                           : llvm::ICmpInst::ICMP_ULT,
-                      compare_key2, compare_key1);
-    if (iota_values_parameter_index >= 0) {
-      auto keys_equal = b->CreateICmpEQ(compare_key1, compare_key2);
-      auto key_index1 =
-          read_element(iota_values_parameter_index, current_keys_index);
-      auto key_index2 =
-          read_element(iota_values_parameter_index, compare_keys_index);
-      auto index_is_smaller_than =
-          b->CreateICmp(llvm::ICmpInst::ICMP_ULT, key_index2, key_index1);
-      is_smaller_than = b->CreateOr(
-          is_smaller_than, b->CreateAnd(keys_equal, index_is_smaller_than));
+  return ksl.IfWithStatus("smaller_comparison_index", do_comparison, [&]() {
+    std::vector<llvm::Value*> values_to_compare;
+    for (int i = 0; i < num_values; ++i) {
+      values_to_compare.push_back(element_address(i, compare_keys_index));
+      values_to_compare.push_back(element_address(i, current_keys_index));
     }
+    llvm::Module* module = b->GetInsertBlock()->getParent()->getParent();
+    llvm::Value* compare_return_buffer = llvm_ir::EmitAllocaAtFunctionEntry(
+        llvm_ir::PrimitiveTypeToIrType(PRED, module), "compare_return_buffer",
+        b);
+    TF_RETURN_IF_ERROR(
+        emit_compare_callback(values_to_compare, compare_return_buffer));
+    llvm::Value* result = b->CreateLoad(compare_return_buffer);
+
+    // Check if the 'compare' function returns true.
+    llvm::Value* is_smaller_than =
+        b->CreateICmpNE(result, llvm::ConstantInt::get(result->getType(), 0),
+                        "boolean_predicate");
     ksl.If("is_smaller_than", is_smaller_than, [&]() {
-      // Swap key1 with key2.
-      write_element(0, current_keys_index, key2);
-      write_element(0, compare_keys_index, key1);
-      for (int64 i = 1; i <= num_values; ++i) {
-        // Also swap the values.
-        auto value1 = read_element(i, current_keys_index);
-        auto value2 = read_element(i, compare_keys_index);
-        write_element(i, current_keys_index, value2);
-        write_element(i, compare_keys_index, value1);
+      for (int64 i = 0; i < num_values; ++i) {
+        // Swap the values.
+        auto value1 = b->CreateLoad(values_to_compare[i * 2]);
+        auto value2 = b->CreateLoad(values_to_compare[i * 2 + 1]);
+        write_element(i, current_keys_index, value1);
+        write_element(i, compare_keys_index, value2);
       }
     });
+    return Status::OK();
   });
 }
 
-void EmitTiledCompareLoop(
+Status EmitTiledCompareLoop(
     const IrArray::Index& tiled_keys_index, int64 dimension_to_sort,
-    int64 dimension_to_sort_bound, PrimitiveType keys_type,
-    absl::Span<const int64> xor_masks, const std::vector<IrArray>& params,
-    const std::vector<llvm::Value*>& param_shmem_buffers,
-    int64 iota_values_parameter_index, int64 tile_size, llvm::IRBuilder<>* b) {
+    int64 dimension_to_sort_bound, absl::Span<const int64> xor_masks,
+    const std::vector<IrArray>& params,
+    const std::vector<llvm::Value*>& param_shmem_buffers, int64 tile_size,
+    const EmitCallToNestedComputationCallback& emit_compare_callback,
+    llvm::IRBuilder<>* b) {
   KernelSupportLibrary ksl(b);
   llvm::Value* thread_id = llvm_ir::EmitCallToIntrinsic(
       llvm::Intrinsic::nvvm_read_ptx_sreg_tid_x, {}, {}, b);
@@ -200,7 +172,7 @@ void EmitTiledCompareLoop(
             [&]() {
               auto cache_index = b->CreateShl(thread_id, value_one);
               read_or_write(cache_index, current_keys_index);
-              // Increment to go the next index position.
+              // Increment to go to the next index position.
               current_keys_index = b->CreateAdd(current_keys_index, value_one);
               // Here we check whether the next index position is within bounds.
               ksl.If("inner_smaller_keys_index",
@@ -230,10 +202,18 @@ void EmitTiledCompareLoop(
   llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::nvvm_barrier0, {}, {}, b);
 
   // Now emit the bodies of the comparison loops.
-  auto read_element = [&](int64 operand, llvm::Value* index) {
-    return b->CreateLoad(
+  auto element_address = [&](int64 operand, llvm::Value* index) {
+    auto shared_memory_address =
         b->CreateGEP(param_shmem_buffers[operand],
-                     {tiled_keys_index.GetConstantWithIndexType(0), index}));
+                     {tiled_keys_index.GetConstantWithIndexType(0), index});
+    auto ptr_type = shared_memory_address->getType();
+    // We need a generic pointer with address space 0 instead of a pointer to
+    // shared memory (address space 3) so that we can pass it to the comparison
+    // computation.
+    return b->CreateAddrSpaceCast(
+        shared_memory_address,
+        llvm::PointerType::get(ptr_type->getPointerElementType(),
+                               /*AddressSpace=*/0));
   };
   auto write_element = [&](int64 operand, llvm::Value* index,
                            llvm::Value* value) {
@@ -252,7 +232,7 @@ void EmitTiledCompareLoop(
     if (dimension_to_sort_bound % tile_size) {
       // Otherwise we need a bounds check for the last tile. The last tile has
       // size 'dimension_to_sort_bound' % 'tile_size'.
-      ksl.If(
+      TF_RETURN_IF_ERROR(ksl.IfWithStatus(
           "is_last_tile",
           b->CreateICmpUGE(
               b->CreateMul(tiled_keys_index[dimension_to_sort],
@@ -260,24 +240,24 @@ void EmitTiledCompareLoop(
               tiled_keys_index.GetConstantWithIndexType(
                   RoundDownToNearest(dimension_to_sort_bound, tile_size))),
           [&]() {
-            EmitCompareLoopBody(dimension_to_sort_bound % tile_size, keys_type,
-                                params.size() - 1, iota_values_parameter_index,
-                                element_pair_index, xor_mask,
-                                tiled_keys_index.GetType(), read_element,
-                                write_element, b);
+            return EmitCompareLoopBody(
+                dimension_to_sort_bound % tile_size, params.size(),
+                element_pair_index, xor_mask, tiled_keys_index.GetType(),
+                element_address, write_element, emit_compare_callback, b);
           },
           [&]() {
-            EmitCompareLoopBody(tile_size, keys_type, params.size() - 1,
-                                iota_values_parameter_index, element_pair_index,
-                                xor_mask, tiled_keys_index.GetType(),
-                                read_element, write_element, b,
-                                /*needs_bounds_checks=*/false);
-          });
+            return EmitCompareLoopBody(
+                tile_size, params.size(), element_pair_index, xor_mask,
+                tiled_keys_index.GetType(), element_address, write_element,
+                emit_compare_callback, b,
+                /*needs_bounds_checks=*/false);
+          }));
     } else {
-      EmitCompareLoopBody(tile_size, keys_type, params.size() - 1,
-                          iota_values_parameter_index, element_pair_index,
-                          xor_mask, tiled_keys_index.GetType(), read_element,
-                          write_element, b, /*needs_bounds_checks=*/false);
+      TF_RETURN_IF_ERROR(EmitCompareLoopBody(
+          tile_size, params.size(), element_pair_index, xor_mask,
+          tiled_keys_index.GetType(), element_address, write_element,
+          emit_compare_callback, b,
+          /*needs_bounds_checks=*/false));
     }
     // Wait until all comparisons have happened.
     llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::nvvm_barrier0, {}, {}, b);
@@ -301,17 +281,16 @@ void EmitTiledCompareLoop(
   // same location in shared memory because we have exactly tile_size / 2 many
   // threads, and the linear index calculated by ParallelLoopEmitter uses
   // linear_index = blockIdx.x * blockDim.x + threadIdx.x;
+  return Status::OK();
 }
 }  // namespace
 
-Status EmitSortInPlace(int64 dimension_to_sort, const IrArray& keys_array,
-                       const std::vector<IrArray>& values_arrays,
-                       int64 iota_values_parameter_index,
-                       absl::string_view name,
-                       absl::Span<const int64> xor_masks, llvm::IRBuilder<>* b,
-                       const gpu::LaunchDimensions& launch_dimensions,
-                       int64 num_iterations_in_sort_dim,
-                       const int64 tile_size) {
+Status EmitSortInPlace(
+    int64 dimension_to_sort, const std::vector<IrArray>& values_arrays,
+    absl::string_view name, absl::Span<const int64> xor_masks,
+    llvm::IRBuilder<>* b, const gpu::LaunchDimensions& launch_dimensions,
+    int64 num_iterations_in_sort_dim, const int64 tile_size,
+    const EmitCallToNestedComputationCallback& emit_compare_callback) {
   // Iterate through the keys shape in physical order, but skip the dimension to
   // sort and make it the innermost loop which is the loop where the comparisons
   // happen. In the dimension to sort, if we use tiling, we iterate through it
@@ -321,7 +300,7 @@ Status EmitSortInPlace(int64 dimension_to_sort, const IrArray& keys_array,
   // within those 64 elements and are therefore independent of the other
   // comparisons).
 
-  const Shape& keys_shape = keys_array.GetShape();
+  const Shape& keys_shape = values_arrays[0].GetShape();
   int64 rank = keys_shape.rank();
   int64 dimension_to_sort_bound = keys_shape.dimensions(dimension_to_sort);
   std::vector<int64> dimensions_in_iteration_order(rank);
@@ -338,18 +317,16 @@ Status EmitSortInPlace(int64 dimension_to_sort, const IrArray& keys_array,
 
   Shape iteration_shape = ShapeUtil::MakeShape(keys_shape.element_type(),
                                                dimensions_in_iteration_order);
-  std::vector<IrArray> params(1, keys_array);
-  params.insert(params.end(), values_arrays.begin(), values_arrays.end());
 
   // Allocate shared memory for the tiled compare loop.
-  std::vector<llvm::Value*> param_shmem_buffers(params.size(), nullptr);
+  std::vector<llvm::Value*> param_shmem_buffers(values_arrays.size(), nullptr);
   if (xor_masks.size() > 1) {
     llvm::Module* module = b->GetInsertBlock()->getParent()->getParent();
-    for (int64 i = 0; i < params.size(); ++i) {
-      llvm::Type* tile_type =
-          llvm::ArrayType::get(llvm_ir::PrimitiveTypeToIrType(
-                                   params[i].GetShape().element_type(), module),
-                               tile_size);
+    for (int64 i = 0; i < values_arrays.size(); ++i) {
+      llvm::Type* tile_type = llvm::ArrayType::get(
+          llvm_ir::PrimitiveTypeToIrType(
+              values_arrays[i].GetShape().element_type(), module),
+          tile_size);
       param_shmem_buffers[i] = llvm_ir::AllocateSharedMemoryTile(
           module, tile_type, absl::StrCat(name, "_tile_param_", i));
     }
@@ -376,25 +353,24 @@ Status EmitSortInPlace(int64 dimension_to_sort, const IrArray& keys_array,
       keys_index[iteration_order_to_logical_order[i]] = tiles_index[i];
     }
     if (xor_masks.size() > 1) {
-      EmitTiledCompareLoop(keys_index, dimension_to_sort,
-                           dimension_to_sort_bound, keys_shape.element_type(),
-                           xor_masks, params, param_shmem_buffers,
-                           iota_values_parameter_index, tile_size, b);
+      TF_RETURN_IF_ERROR(EmitTiledCompareLoop(
+          keys_index, dimension_to_sort, dimension_to_sort_bound, xor_masks,
+          values_arrays, param_shmem_buffers, tile_size, emit_compare_callback,
+          b));
     } else {
-      auto read_element = [&](int64 operand, llvm::Value* index) {
+      auto element_address = [&](int64 operand, llvm::Value* index) {
         keys_index[dimension_to_sort] = index;
-        return params[operand].EmitReadArrayElement(keys_index, b);
+        return values_arrays[operand].EmitArrayElementAddress(keys_index, b);
       };
       auto write_element = [&](int64 operand, llvm::Value* index,
                                llvm::Value* value) {
         keys_index[dimension_to_sort] = index;
-        params[operand].EmitWriteArrayElement(keys_index, value, b);
+        values_arrays[operand].EmitWriteArrayElement(keys_index, value, b);
       };
-      EmitCompareLoopBody(dimension_to_sort_bound, keys_shape.element_type(),
-                          values_arrays.size(), iota_values_parameter_index,
-                          tiles_index[rank - 1], xor_masks[0],
-                          tiles_index.GetType(), read_element, write_element,
-                          b);
+      TF_RETURN_IF_ERROR(EmitCompareLoopBody(
+          dimension_to_sort_bound, values_arrays.size(), tiles_index[rank - 1],
+          xor_masks[0], tiles_index.GetType(), element_address, write_element,
+          emit_compare_callback, b));
     }
     return Status::OK();
   };
diff --git a/tensorflow/compiler/xla/service/llvm_ir/sort_util.h b/tensorflow/compiler/xla/service/llvm_ir/sort_util.h
index 685f9383ac..b9341a34d1 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/sort_util.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/sort_util.h
@@ -28,19 +28,18 @@ limitations under the License.
 
 namespace xla {
 namespace llvm_ir {
+using EmitCallToNestedComputationCallback =
+    std::function<Status(absl::Span<llvm::Value* const>, llvm::Value*)>;
 // Emits llvm IR to do pairwise comparisons/swaps in the 'dimension_to_sort'
-// dimension of 'keys_array'. All other dimensions are kept as-is. This
-// implements the inner loop of BitonicSort. It is assumed that 'xor_masks'
-// contains only powers of 2, or values 2^k - 1 (k > 0). If
-// 'iota_values_parameter_index' is >= 0, it points at a 'values_arrays' operand
-// that is a iota and can be used to make the sorting stable.
-Status EmitSortInPlace(int64 dimension_to_sort, const IrArray& keys_array,
-                       const std::vector<IrArray>& values_arrays,
-                       int64 iota_values_parameter_index,
-                       absl::string_view name,
-                       absl::Span<const int64> xor_masks, llvm::IRBuilder<>* b,
-                       const gpu::LaunchDimensions& launch_dimensions,
-                       int64 num_iterations_in_sort_dim, int64 tile_size);
+// dimension of each array in 'values_arrays'. All other dimensions are kept
+// as-is. This implements the inner loop of BitonicSort. It is assumed that
+// 'xor_masks' contains only powers of 2, or values 2^k - 1 (k > 0).
+Status EmitSortInPlace(
+    int64 dimension_to_sort, const std::vector<IrArray>& values_arrays,
+    absl::string_view name, absl::Span<const int64> xor_masks,
+    llvm::IRBuilder<>* b, const gpu::LaunchDimensions& launch_dimensions,
+    int64 num_iterations_in_sort_dim, int64 tile_size,
+    const EmitCallToNestedComputationCallback& emit_compare_callback);
 }  // namespace llvm_ir
 }  // namespace xla
 
diff --git a/tensorflow/compiler/xla/service/sort_simplifier.cc b/tensorflow/compiler/xla/service/sort_simplifier.cc
index 4a00e8d7b2..122366a0f3 100644
--- a/tensorflow/compiler/xla/service/sort_simplifier.cc
+++ b/tensorflow/compiler/xla/service/sort_simplifier.cc
@@ -14,12 +14,15 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/compiler/xla/service/sort_simplifier.h"
-#include "tensorflow/compiler/xla/service/hlo_computation.h"
-#include "tensorflow/compiler/xla/service/hlo_instruction.h"
-#include "tensorflow/compiler/xla/statusor.h"
+
+#include <memory>
+#include <vector>
 
 #include "absl/container/flat_hash_map.h"
 #include "absl/container/flat_hash_set.h"
+#include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/statusor.h"
 
 namespace xla {
 namespace {
@@ -39,8 +42,7 @@ StatusOr<bool> RemoveUnusedOperandFromSort(HloInstruction* sort) {
     return false;
   }
 
-  // Index 0 is the sorting key used by the sort HLO itself.
-  absl::flat_hash_set<int64> used_indices{0};
+  absl::flat_hash_set<int64> used_indices;
   for (const HloInstruction* user : sort->users()) {
     if (user->opcode() != HloOpcode::kGetTupleElement) {
       // Can't analyse users other then get-tuple-element.
@@ -49,15 +51,25 @@ StatusOr<bool> RemoveUnusedOperandFromSort(HloInstruction* sort) {
     used_indices.insert(user->tuple_index());
   }
 
+  // Also note which parameters are used by the comparator computation.
+  auto comparator = sort->to_apply();
+  for (int64 i = 0; i < sort->operand_count() * 2; ++i) {
+    if (comparator->parameter_instruction(i)->user_count() > 0) {
+      // operand i corresponds to parameters 2 * i and 2 * i + 1 of the
+      // computation.
+      used_indices.insert(i / 2);
+    }
+  }
+
   if (used_indices.size() == sort->operand_count()) {
     // All operands are used.
     return false;
   }
 
-  std::vector<HloInstruction*> operands{sort->mutable_operand(0)};
-  std::vector<Shape> new_shapes{sort->operand(0)->shape()};
-  for (int64 i = 1; i < sort->operand_count(); ++i) {
-    if (used_indices.count(i)) {
+  std::vector<HloInstruction*> operands;
+  std::vector<Shape> new_shapes;
+  for (int64 i = 0; i < sort->operand_count(); ++i) {
+    if (used_indices.contains(i)) {
       operands.push_back(sort->mutable_operand(i));
       new_shapes.push_back(sort->operand(i)->shape());
     }
@@ -68,6 +80,32 @@ StatusOr<bool> RemoveUnusedOperandFromSort(HloInstruction* sort) {
                              : ShapeUtil::MakeTupleShape(new_shapes);
   HloInstruction* new_sort = computation->AddInstruction(
       sort->CloneWithNewOperands(new_sort_shape, operands));
+  absl::flat_hash_map<const HloInstruction*, std::unique_ptr<HloInstruction>>
+      replacements;
+  int64 parameter_number = 0;
+  for (int64 i = 0; i < sort->operand_count(); ++i) {
+    auto* old_lhs_parameter = comparator->parameter_instruction(i * 2);
+    auto* old_rhs_parameter = comparator->parameter_instruction(i * 2 + 1);
+    if (used_indices.contains(i)) {
+      Shape scalar_shape =
+          ShapeUtil::MakeShape(sort->operand(i)->shape().element_type(), {});
+      replacements[old_lhs_parameter] = HloInstruction::CreateParameter(
+          parameter_number, scalar_shape,
+          absl::StrCat("p.", parameter_number / 2, ".lhs"));
+      ++parameter_number;
+      replacements[old_rhs_parameter] = HloInstruction::CreateParameter(
+          parameter_number, scalar_shape,
+          absl::StrCat("p.", parameter_number / 2, ".rhs"));
+      ++parameter_number;
+    } else {
+      replacements[old_lhs_parameter] = nullptr;
+      replacements[old_rhs_parameter] = nullptr;
+    }
+  }
+  HloModule* module = sort->GetModule();
+  HloComputation* new_compare = module->AddEmbeddedComputation(
+      comparator->CloneWithReplacements(std::move(replacements)));
+  new_sort->set_to_apply(new_compare);
 
   // Map from original get-tuple-element tuple index to new HLO instruction
   absl::flat_hash_map<int64, HloInstruction*> result_map;
@@ -83,7 +121,8 @@ StatusOr<bool> RemoveUnusedOperandFromSort(HloInstruction* sort) {
       }
     }
   } else {
-    result_map[0] = new_sort;
+    CHECK_EQ(used_indices.size(), 1);
+    result_map[*used_indices.begin()] = new_sort;
   }
   std::vector<HloInstruction*> users(sort->users().begin(),
                                      sort->users().end());
diff --git a/tensorflow/compiler/xla/service/sort_simplifier_test.cc b/tensorflow/compiler/xla/service/sort_simplifier_test.cc
index a05bc79989..696ac1b465 100644
--- a/tensorflow/compiler/xla/service/sort_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/sort_simplifier_test.cc
@@ -124,5 +124,37 @@ TEST_F(SortSimplifierTest, DontRemoveUnusedSortKey) {
   SortSimplifier simplifier;
   EXPECT_FALSE(simplifier.Run(module.get()).ValueOrDie());
 }
+
+TEST_F(SortSimplifierTest, RemoveUnusedFirstOperand) {
+  const char* hlo_string = R"(
+   HloModule permutation_sort
+
+   compare {
+     p.0.lhs = f32[] parameter(0)
+     p.0.rhs = f32[] parameter(1)
+     p.1.lhs = s32[] parameter(2)
+     p.1.rhs = s32[] parameter(3)
+     ROOT lt = pred[] less-than(p.1.lhs, p.1.rhs)
+   }
+
+   ENTRY sort_computation {
+     keys = f32[64,8732]{1,0} parameter(0)
+     values = s32[64,8732]{1,0} parameter(1)
+     sort = (f32[64,8732]{1,0}, s32[64,8732]{1,0}) sort(keys, values),
+       dimensions={1}, to_apply=compare
+     ROOT gte = s32[64,8732]{1,0} get-tuple-element(sort), index=1
+   })";
+  TF_ASSERT_OK_AND_ASSIGN(auto module,
+                          ParseAndReturnVerifiedModule(hlo_string));
+
+  SortSimplifier simplifier;
+  uint64 num_executions = 0;
+  do {
+    num_executions++;
+  } while (simplifier.Run(module.get()).ValueOrDie());
+  EXPECT_EQ(num_executions, 2);
+  auto root = module->entry_computation()->root_instruction();
+  EXPECT_THAT(root, GmockMatch(m::Sort(m::Parameter(1))));
+}
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 42854734506d5fc8dfabc813101b9a9addcefa59 Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Fri, 8 Feb 2019 02:28:18 -0800
Subject: [PATCH 2137/2345] Allow dilations in the conv2d transpose op

PiperOrigin-RevId: 233028096
---
 tensorflow/python/ops/nn_ops.py               | 126 +++++++++++-------
 .../tools/api/golden/v1/tensorflow.nn.pbtxt   |   2 +-
 .../tools/api/golden/v2/tensorflow.nn.pbtxt   |   2 +-
 3 files changed, 78 insertions(+), 52 deletions(-)

diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index bd70d9f09a..d65a16cd49 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -1905,7 +1905,8 @@ def conv2d_transpose(
     data_format="NHWC",
     name=None,
     input=None,  # pylint: disable=redefined-builtin
-    filters=None):
+    filters=None,
+    dilations=None):
   """The transpose of `conv2d`.
 
   This operation is sometimes called "deconvolution" after [Deconvolutional
@@ -1933,6 +1934,14 @@ def conv2d_transpose(
     name: Optional name for the returned tensor.
     input: Alias for value.
     filters: Alias for filter.
+    dilations: An int or list of `ints` that has length `1`, `2` or `4`,
+      defaults to 1. The dilation factor for each dimension of`input`. If a
+      single value is given it is replicated in the `H` and `W` dimension. By
+      default the `N` and `C` dimensions are set to 1. If set to k > 1, there
+      will be k-1 skipped cells between each filter element on that dimension.
+      The dimension order is determined by the value of `data_format`, see above
+      for details. Dilations in the batch and depth dimensions if a 4-d tensor
+      must be 1.
 
   Returns:
     A `Tensor` with the same type as `value`.
@@ -1945,44 +1954,14 @@ def conv2d_transpose(
   filter = deprecated_argument_lookup("filters", filters, "filter", filter)
   with ops.name_scope(name, "conv2d_transpose",
                       [value, filter, output_shape]) as name:
-    if data_format not in ("NCHW", "NHWC"):
-      raise ValueError("data_format has to be either NCHW or NHWC.")
-    value = ops.convert_to_tensor(value, name="value")
-    filter = ops.convert_to_tensor(filter, name="filter")  # pylint: disable=redefined-builtin
-    axis = 3 if data_format == "NHWC" else 1
-    if not value.get_shape().dims[axis].is_compatible_with(
-        filter.get_shape()[3]):
-      raise ValueError("input channels does not match filter's input channels, "
-                       "{} != {}".format(value.get_shape()[axis],
-                                         filter.get_shape()[3]))
-
-    output_shape_ = ops.convert_to_tensor(output_shape, name="output_shape")
-    if not output_shape_.get_shape().is_compatible_with(tensor_shape.vector(4)):
-      raise ValueError("output_shape must have shape (4,), got {}".format(
-          output_shape_.get_shape()))
-
-    if isinstance(output_shape, (list, np.ndarray)):
-      # output_shape's shape should be == [4] if reached this point.
-      if not filter.get_shape().dims[2].is_compatible_with(
-          output_shape[axis]):
-        raise ValueError(
-            "output_shape does not match filter's output channels, "
-            "{} != {}".format(output_shape[axis],
-                              filter.get_shape()[2]))
-
-    if padding != "VALID" and padding != "SAME":
-      raise ValueError("padding must be either VALID or SAME:"
-                       " {}".format(padding))
-
-    strides = _get_sequence(strides, 2, axis, "strides")
-
-    return gen_nn_ops.conv2d_backprop_input(
-        input_sizes=output_shape_,
-        filter=filter,
-        out_backprop=value,
-        strides=strides,
+    return conv2d_transpose_v2(
+        value,
+        filter,
+        output_shape,
+        strides,
         padding=padding,
         data_format=data_format,
+        dilations=dilations,
         name=name)
 
 
@@ -1995,20 +1974,67 @@ def conv2d_transpose_v2(
     strides,
     padding="SAME",
     data_format="NHWC",
+    dilations=None,
     name=None):
-  return conv2d_transpose(
-      input,
-      filters,
-      output_shape,
-      strides,
-      padding=padding,
-      data_format=data_format,
-      name=name)
-# pylint: enable=redefined-builtin
-conv2d_transpose_v2.__doc__ = deprecation.rewrite_argument_docstring(
-    deprecation.rewrite_argument_docstring(
-        conv2d_transpose.__doc__, "filter", "filters"),
-    "value", "input")
+  """The transpose of `conv2d`.
+
+  This operation is sometimes called "deconvolution" after [Deconvolutional
+  Networks](http://www.matthewzeiler.com/pubs/cvpr2010/cvpr2010.pdf), but is
+  actually the transpose (gradient) of `conv2d` rather than an actual
+  deconvolution.
+
+  Args:
+    input: A 4-D `Tensor` of type `float` and shape `[batch, height, width,
+      in_channels]` for `NHWC` data format or `[batch, in_channels, height,
+      width]` for `NCHW` data format.
+    filters: A 4-D `Tensor` with the same type as `value` and shape `[height,
+      width, output_channels, in_channels]`.  `filter`'s `in_channels` dimension
+      must match that of `value`.
+    output_shape: A 1-D `Tensor` representing the output shape of the
+      deconvolution op.
+    strides: An int or list of `ints` that has length `1`, `2` or `4`.  The
+      stride of the sliding window for each dimension of `input`. If a single
+      value is given it is replicated in the `H` and `W` dimension. By default
+      the `N` and `C` dimensions are set to 0. The dimension order is determined
+      by the value of `data_format`, see below for details.
+    padding: A string, either `'VALID'` or `'SAME'`. The padding algorithm. See
+      the "returns" section of `tf.nn.convolution` for details.
+    data_format: A string. 'NHWC' and 'NCHW' are supported.
+    dilations: An int or list of `ints` that has length `1`, `2` or `4`,
+      defaults to 1. The dilation factor for each dimension of`input`. If a
+      single value is given it is replicated in the `H` and `W` dimension. By
+      default the `N` and `C` dimensions are set to 1. If set to k > 1, there
+      will be k-1 skipped cells between each filter element on that dimension.
+      The dimension order is determined by the value of `data_format`, see above
+      for details. Dilations in the batch and depth dimensions if a 4-d tensor
+      must be 1.
+    name: Optional name for the returned tensor.
+
+  Returns:
+    A `Tensor` with the same type as `value`.
+
+  Raises:
+    ValueError: If input/output depth does not match `filter`'s shape, or if
+      padding is other than `'VALID'` or `'SAME'`.
+  """
+  with ops.name_scope(name, "conv2d_transpose",
+                      [input, filter, output_shape]) as name:
+    if data_format is None:
+      data_format = "NHWC"
+    channel_index = 1 if data_format.startswith("NC") else 3
+
+    strides = _get_sequence(strides, 2, channel_index, "strides")
+    dilations = _get_sequence(dilations, 2, channel_index, "dilations")
+
+    return gen_nn_ops.conv2d_backprop_input(
+        input_sizes=output_shape,
+        filter=filters,
+        out_backprop=input,
+        strides=strides,
+        padding=padding,
+        data_format=data_format,
+        dilations=dilations,
+        name=name)
 
 
 @tf_export("nn.atrous_conv2d_transpose")
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
index b8a56e1cf0..e46f1bc935 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
@@ -70,7 +70,7 @@ tf_module {
   }
   member_method {
     name: "conv2d_transpose"
-    argspec: "args=[\'value\', \'filter\', \'output_shape\', \'strides\', \'padding\', \'data_format\', \'name\', \'input\', \'filters\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'SAME\', \'NHWC\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'value\', \'filter\', \'output_shape\', \'strides\', \'padding\', \'data_format\', \'name\', \'input\', \'filters\', \'dilations\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'SAME\', \'NHWC\', \'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "conv3d"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index 930cce32e4..92709e3410 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -66,7 +66,7 @@ tf_module {
   }
   member_method {
     name: "conv2d_transpose"
-    argspec: "args=[\'input\', \'filters\', \'output_shape\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'SAME\', \'NHWC\', \'None\'], "
+    argspec: "args=[\'input\', \'filters\', \'output_shape\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'SAME\', \'NHWC\', \'None\', \'None\'], "
   }
   member_method {
     name: "conv3d"
-- 
GitLab


From 875785cd913b1b6b0d3eedac5b524a94207fda94 Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Fri, 8 Feb 2019 02:32:02 -0800
Subject: [PATCH 2138/2345] Add integration test with tf.keras.Sequential.

PiperOrigin-RevId: 233028411
---
 .../saved_model/integration_tests/BUILD       | 18 +++++
 .../export_simple_text_embedding.py           | 81 +++++++++++++++++++
 .../integration_tests/saved_model_test.py     | 11 +++
 .../use_model_in_sequential_keras.py          | 81 +++++++++++++++++++
 4 files changed, 191 insertions(+)
 create mode 100644 tensorflow/examples/saved_model/integration_tests/export_simple_text_embedding.py
 create mode 100644 tensorflow/examples/saved_model/integration_tests/use_model_in_sequential_keras.py

diff --git a/tensorflow/examples/saved_model/integration_tests/BUILD b/tensorflow/examples/saved_model/integration_tests/BUILD
index f40c6d9d28..cb0592c2a2 100644
--- a/tensorflow/examples/saved_model/integration_tests/BUILD
+++ b/tensorflow/examples/saved_model/integration_tests/BUILD
@@ -36,6 +36,22 @@ py_binary(
     ],
 )
 
+py_binary(
+    name = "export_simple_text_embedding",
+    srcs = ["export_simple_text_embedding.py"],
+    deps = [
+        "//tensorflow:tensorflow_py",
+    ],
+)
+
+py_binary(
+    name = "use_model_in_sequential_keras",
+    srcs = ["use_model_in_sequential_keras.py"],
+    deps = [
+        "//tensorflow:tensorflow_py",
+    ],
+)
+
 py_test(
     name = "saved_model_test",
     srcs = [
@@ -43,7 +59,9 @@ py_test(
     ],
     data = [
         ":export_rnn_cell",
+        ":export_simple_text_embedding",
         ":export_text_rnn_model",
+        ":use_model_in_sequential_keras",
         ":use_rnn_cell",
         ":use_text_rnn_model",
     ],
diff --git a/tensorflow/examples/saved_model/integration_tests/export_simple_text_embedding.py b/tensorflow/examples/saved_model/integration_tests/export_simple_text_embedding.py
new file mode 100644
index 0000000000..5149536592
--- /dev/null
+++ b/tensorflow/examples/saved_model/integration_tests/export_simple_text_embedding.py
@@ -0,0 +1,81 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Text embedding model stored as a SavedModel."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl import app
+from absl import flags
+
+import tensorflow as tf
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_string("export_dir", None, "Directory to export SavedModel.")
+
+
+class TextEmbeddingModel(tf.train.Checkpoint):
+  """Text embedding model.
+
+  A text embeddings model that takes a sentences on input and outputs the
+  sentence embedding.
+  """
+
+  def __init__(self, emb_dim, buckets):
+    super(TextEmbeddingModel, self).__init__()
+    self._buckets = buckets
+    self._embeddings = tf.Variable(tf.random.uniform(shape=[buckets, emb_dim]))
+
+  def _tokenize(self, sentences):
+    # Perform a minimalistic text preprocessing by removing punctuation and
+    # splitting on spaces.
+    normalized_sentences = tf.strings.regex_replace(
+        input=sentences, pattern=r"\pP", rewrite="")
+    normalized_sentences = tf.reshape(normalized_sentences, [-1])
+    sparse_tokens = tf.string_split(normalized_sentences, " ")
+
+    # Deal with a corner case: there is one empty sentence.
+    sparse_tokens, _ = tf.sparse.fill_empty_rows(sparse_tokens, tf.constant(""))
+    # Deal with a corner case: all sentences are empty.
+    sparse_tokens = tf.sparse.reset_shape(sparse_tokens)
+
+    return (sparse_tokens.indices, self._words_to_indices(sparse_tokens.values),
+            sparse_tokens.dense_shape)
+
+  def _words_to_indices(self, words):
+    return tf.strings.to_hash_bucket(words, self._buckets)
+
+  @tf.function(input_signature=[tf.TensorSpec([None], tf.dtypes.string)])
+  def __call__(self, sentences):
+    token_ids, token_values, token_dense_shape = self._tokenize(sentences)
+
+    return tf.nn.safe_embedding_lookup_sparse(
+        embedding_weights=self._embeddings,
+        sparse_ids=tf.SparseTensor(token_ids, token_values, token_dense_shape),
+        sparse_weights=None,
+        combiner="sqrtn")
+
+
+def main(argv):
+  del argv
+
+  module = TextEmbeddingModel(emb_dim=10, buckets=100)
+  tf.saved_model.save(module, FLAGS.export_dir)
+
+
+if __name__ == "__main__":
+  app.run(main)
diff --git a/tensorflow/examples/saved_model/integration_tests/saved_model_test.py b/tensorflow/examples/saved_model/integration_tests/saved_model_test.py
index 770e809ffc..4b48bcbb22 100644
--- a/tensorflow/examples/saved_model/integration_tests/saved_model_test.py
+++ b/tensorflow/examples/saved_model/integration_tests/saved_model_test.py
@@ -59,6 +59,17 @@ class SavedModelTest(tf.test.TestCase):
     use_binary = resource_loader.get_path_to_datafile("use_rnn_cell")
     self.assertCommandSucceeded(use_binary, model_dir=export_dir)
 
+  @test_util.run_v2_only
+  def test_text_embedding_in_sequential_keras(self):
+    export_dir = self.get_temp_dir()
+    export_binary = resource_loader.get_path_to_datafile(
+        "export_simple_text_embedding")
+    self.assertCommandSucceeded(export_binary, export_dir=export_dir)
+
+    use_binary = resource_loader.get_path_to_datafile(
+        "use_model_in_sequential_keras")
+    self.assertCommandSucceeded(use_binary, model_dir=export_dir)
+
 if __name__ == "__main__":
   # tf.enable_v2_behavior()
   tf.test.main()
diff --git a/tensorflow/examples/saved_model/integration_tests/use_model_in_sequential_keras.py b/tensorflow/examples/saved_model/integration_tests/use_model_in_sequential_keras.py
new file mode 100644
index 0000000000..9e74dae255
--- /dev/null
+++ b/tensorflow/examples/saved_model/integration_tests/use_model_in_sequential_keras.py
@@ -0,0 +1,81 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Load and use RNN model stored as a SavedModel."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl import app
+from absl import flags
+
+import numpy as np
+
+import tensorflow as tf
+# TODO(vbardiovsky): Remove when load is available.
+from tensorflow.python.saved_model.load import load
+
+tf.saved_model.load = load
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_string("model_dir", None, "Directory to load SavedModel from.")
+
+
+# TODO(vbardiovsky): We should just reuse Keras's Lambda layer, when that
+# enables to get trainable variables.
+class LambdaLayer(tf.keras.layers.Layer):
+  """Lambda layer with output shape inference."""
+
+  def __init__(self, func, **kwargs):
+    self._func = func
+    super(LambdaLayer, self).__init__(**kwargs)
+
+  def call(self, x):
+    result = self._func(x)
+    # TODO(vbardiovsky): Polymorphic function should return shaped tensor.
+    result.set_shape(self.compute_output_shape(x.shape))
+    return result
+
+  def compute_output_shape(self, input_shape):
+    # TODO(vbardiovsky): We should be able to get the embedding dimension from
+    # the restored model.
+    return (input_shape[0], 10)
+
+
+def main(argv):
+  del argv
+
+  features = np.array(["my first sentence", "my second sentence"])
+  labels = np.array([1, 0])
+
+  dataset = tf.data.Dataset.from_tensor_slices((features, labels))
+
+  embedding = tf.saved_model.load(FLAGS.model_dir)
+
+  # Create the sequential keras model.
+  model = tf.keras.Sequential()
+  model.add(LambdaLayer(embedding, batch_input_shape=[None], dtype=tf.string))
+  model.add(tf.keras.layers.Dense(100, activation="relu"))
+  model.add(tf.keras.layers.Dense(50, activation="relu"))
+  model.add(tf.keras.layers.Dense(1, activation="sigmoid"))
+  model.compile(
+      optimizer="adam", loss="binary_crossentropy", metrics=["accuracy"])
+
+  model.fit_generator(generator=dataset.batch(1), epochs=5)
+
+
+if __name__ == "__main__":
+  app.run(main)
-- 
GitLab


From 4f1ca2f277e029f89baae63dad858beb30872878 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 8 Feb 2019 03:01:01 -0800
Subject: [PATCH 2139/2345] [TF:XLA] Fix ModuleGroupMetadata building for some
 nested while loops.

PiperOrigin-RevId: 233031044
---
 .../xla/service/hlo_module_group_metadata.cc         | 12 ++++++++----
 .../compiler/xla/service/hlo_module_group_metadata.h |  3 ++-
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_module_group_metadata.cc b/tensorflow/compiler/xla/service/hlo_module_group_metadata.cc
index 47734bc55c..b877081be5 100644
--- a/tensorflow/compiler/xla/service/hlo_module_group_metadata.cc
+++ b/tensorflow/compiler/xla/service/hlo_module_group_metadata.cc
@@ -389,9 +389,10 @@ Status HloModuleGroupMetadata::AddCompanion(HloInstruction* instruction1,
                instruction1->opcode() == HloOpcode::kCall);
   VLOG(2) << "adding as companions:" << instruction1->ToString() << " and "
           << instruction2->ToString();
-
-  if (!ContainsKey(companion_set_index_, instruction1) &&
-      !ContainsKey(companion_set_index_, instruction2)) {
+  if (instruction1 == instruction2) {
+    return Status::OK();
+  } else if (!ContainsKey(companion_set_index_, instruction1) &&
+             !ContainsKey(companion_set_index_, instruction2)) {
     companion_sets_.push_back(
         absl::make_unique<std::vector<HloInstruction*>>());
     auto companion_set = companion_sets_.back().get();
@@ -419,7 +420,10 @@ Status HloModuleGroupMetadata::AddCompanion(HloInstruction* instruction1,
     for (HloInstruction* hlo : Companions(instruction2)) {
       companion_set_index_[hlo] = companion_set_index_[instruction1];
     }
-    companion_sets_.erase(companion_sets_.begin() + index_to_remove);
+    // We can't remove the set from the vector because companion_set_index_
+    // references sets by their index in this vector, so we reset to nullptr
+    // instead.
+    companion_sets_[index_to_remove].reset(nullptr);
   }
   return Status::OK();
 }
diff --git a/tensorflow/compiler/xla/service/hlo_module_group_metadata.h b/tensorflow/compiler/xla/service/hlo_module_group_metadata.h
index 3ed95c1050..84f7f2f313 100644
--- a/tensorflow/compiler/xla/service/hlo_module_group_metadata.h
+++ b/tensorflow/compiler/xla/service/hlo_module_group_metadata.h
@@ -173,7 +173,8 @@ class HloModuleGroupMetadata {
   // Returns the number of modules for devices (excluding the host module).
   int64 GetDeviceModulesCount() const;
 
-  // Returns the companion instructions for the given instruction.
+  // Returns the companion set for the given instruction, including the
+  // instruction itself.
   //
   // Precondition: IsCompanionWhile(instruction) is true.
   const std::vector<HloInstruction*>& Companions(
-- 
GitLab


From 810a46c005339674baec2933463f4a710dac0c0e Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Fri, 8 Feb 2019 03:32:36 -0800
Subject: [PATCH 2140/2345] Expose @tf.function annotated methods on tf.Module
 as functions.

An issue with decorating methods in the metaclass is that when users annotate
methods with decorators that return Objects (e.g. `tf.function`) they lose the
ability to access methods on those objects (other than call) because our
`with_name_scope` decorator is "on the outside".

A simple answer to this from other decorators is to copy the __dict__ from the
wrapped object to the wrapper (or use `__getattr__`). However in our case this
is not safe, since properties like `get_concrete_function` on `Function` would
need special casing (so the returned callable also entered the module name
scope before tracing).

After a bunch of discussion (thanks superbobry@ and chr1sj0nes@!) about how we
could do this in horrible ways, it seems that the cleanest solution is to allow
a Function to be rewrapped (iff it has not been traced). This pushes the
decorator down the stack of decorators and allows tf.Module to preserve
@tf.function annoated methods as Functions.

h/t tensorflow/community#56

PiperOrigin-RevId: 233034181
---
 tensorflow/python/eager/def_function.py      | 26 +++++++++++++++
 tensorflow/python/eager/def_function_test.py | 14 ++++++++
 tensorflow/python/module/module.py           | 26 +++++++++++++--
 tensorflow/python/module/module_test.py      | 35 ++++++++++++++++++++
 4 files changed, 99 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py
index 59c6608965..57d9eb4b36 100644
--- a/tensorflow/python/eager/def_function.py
+++ b/tensorflow/python/eager/def_function.py
@@ -349,6 +349,32 @@ class Function(object):
     self._stateless_fn = self._defun_with_scope(invalid_creator_scope)
     self._stateless_fn._name = self._name  # pylint: disable=protected-access
 
+  def _decorate(self, decorator):
+    """Allows the captured Python function to be decorated in place.
+
+    This method is only safe to call when the Function has not been called by a
+    user. It makes sense to use this method to push a decorator into the
+    function rather than wrapping the function in the decorator.
+
+    We use this in tf.Module to allow user annotated `tf.functions` to remain as
+    `Function` objects but still automatically enter the Module name_scope
+    when they are evaluated like all other methods.
+
+    Args:
+      decorator: A callable accepting a single argument which is the function
+        to decorate and returning a callable result.
+
+    Raises:
+      ValueError: If the function has been called a ValueError is raised.
+    """
+    if self._stateful_fn is not None or self._stateless_fn is not None:
+      raise ValueError(
+          "Functions cannot be decorated after they have been traced.")
+
+    self._python_function = decorator(self._python_function)
+    self._function_spec = function_lib.FunctionSpec.from_function_and_signature(
+        self._python_function, self._input_signature)
+
   def __call__(self, *args, **kwds):
     """Calls the graph function."""
     if self._created_variables:
diff --git a/tensorflow/python/eager/def_function_test.py b/tensorflow/python/eager/def_function_test.py
index b49b1656fc..2c9da8b4ee 100644
--- a/tensorflow/python/eager/def_function_test.py
+++ b/tensorflow/python/eager/def_function_test.py
@@ -429,6 +429,20 @@ class DefFunctionTest(test.TestCase):
     created_variable_read = create_variable()
     self.assertRegexpMatches(created_variable_read.device, "CPU")
 
+  def testDecorate(self):
+    func = def_function.function(lambda: 1)
+    def decorator(f):
+      return lambda: 1 + f()
+
+    func._decorate(decorator)
+    self.assertEqual(func().numpy(), 2)
+
+  def testDecorate_rejectedAfterTrace(self):
+    func = def_function.function(lambda: 1)
+    self.assertEqual(func().numpy(), 1)
+    msg = 'Functions cannot be decorated after they have been traced.'
+    with self.assertRaisesRegexp(ValueError, msg):
+      func._decorate(lambda f: f)
 
 if __name__ == '__main__':
   ops.enable_eager_execution()
diff --git a/tensorflow/python/module/module.py b/tensorflow/python/module/module.py
index 13891339c1..0905df779f 100644
--- a/tensorflow/python/module/module.py
+++ b/tensorflow/python/module/module.py
@@ -23,6 +23,7 @@ import sys
 
 import six
 
+from tensorflow.python.eager import def_function
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import variables
 from tensorflow.python.training.checkpointable import tracking
@@ -91,7 +92,7 @@ class ModuleMetaclass(type):
     return module
 
 
-def with_name_scope(unbound_method):
+def wrap_with_name_scope(unbound_method):
   """Patches the given method so it enters the modules name scope."""
   def enter_name_scope(self, *args, **kwargs):
     """Decorator that calls the given function in the module name scope.
@@ -118,7 +119,28 @@ def with_name_scope(unbound_method):
       # for a particular method annotate it with `@no_module_name_scope`.
       return unbound_method(self, *args, **kwargs)
 
-  return tf_decorator.make_decorator(unbound_method, enter_name_scope)
+  return enter_name_scope
+
+
+def wrap_with_name_scope_no_exception(unbound_method):
+  """Patches the given method so it enters the modules name scope."""
+  def enter_name_scope(self, *args, **kwargs):
+    with self.name_scope:
+      # tf.Module enters the module name scope for all methods. To disable this
+      # for a particular method annotate it with `@no_module_name_scope`.
+      return unbound_method(self, *args, **kwargs)
+  return enter_name_scope
+
+
+def with_name_scope(unbound_method):
+  """Patches the given method so it enters the modules name scope."""
+  if isinstance(unbound_method, def_function.Function):
+    # Autograph cannot convert functions that have try/catch.
+    unbound_method._decorate(wrap_with_name_scope_no_exception)  # pylint: disable=protected-access
+    return unbound_method
+  else:
+    return tf_decorator.make_decorator(unbound_method,
+                                       wrap_with_name_scope(unbound_method))
 
 
 @tf_export("experimental.Module")
diff --git a/tensorflow/python/module/module_test.py b/tensorflow/python/module/module_test.py
index 08a63537e3..21cc6c569f 100644
--- a/tensorflow/python/module/module_test.py
+++ b/tensorflow/python/module/module_test.py
@@ -23,6 +23,7 @@ import collections
 from absl.testing import parameterized
 
 from tensorflow.python.compat import v2_compat
+from tensorflow.python.eager import def_function
 from tensorflow.python.framework import ops
 from tensorflow.python.module import module
 from tensorflow.python.ops import variables
@@ -200,6 +201,28 @@ class CommonErrorsTest(test.TestCase):
         CallsMethodBeforeSuperConstructorModule(allowed_method=True))
 
 
+class ForwardMethodsTest(test.TestCase):
+
+  def testFunctionType(self):
+    mod = ModuleWithFunctionAnnotatedCall()
+    self.assertTrue(isinstance(mod.forward, def_function.Function))
+    self.assertTrue(isinstance(mod.forward_ag, def_function.Function))
+
+  def testEntersNameScope_call(self):
+    mod = ModuleWithFunctionAnnotatedCall()
+    self.assertEqual(mod.forward().numpy(),
+                     b"module_with_function_annotated_call/")
+    self.assertEqual(mod.forward_ag().numpy(),
+                     b"module_with_function_annotated_call/")
+
+  def testEntersNameScope_concreteFunction(self):
+    mod = ModuleWithFunctionAnnotatedCall()
+    self.assertEqual(mod.forward.get_concrete_function()().numpy(),
+                     b"module_with_function_annotated_call/")
+    self.assertEqual(mod.forward_ag.get_concrete_function()().numpy(),
+                     b"module_with_function_annotated_call/")
+
+
 def get_name_scope():
   with ops.name_scope("x") as ns:
     return ns[:-2]
@@ -295,6 +318,18 @@ class CallsMethodBeforeSuperConstructorModule(module.Module):
   def with_name_scope(self):
     pass
 
+
+class ModuleWithFunctionAnnotatedCall(module.Module):
+
+  @def_function.function(autograph=False)
+  def forward(self):
+    return get_name_scope()
+
+  @def_function.function(autograph=True)
+  def forward_ag(self):
+    return get_name_scope()
+
+
 NamedPair = collections.namedtuple("NamedPair", ("first", "second"))
 mk_index_dict = lambda v: dict(enumerate(v))
 
-- 
GitLab


From a6671ccfc6d46f38eddfd80e506773d4fbd75e77 Mon Sep 17 00:00:00 2001
From: Thomas Joerg <tjoerg@google.com>
Date: Fri, 8 Feb 2019 03:33:18 -0800
Subject: [PATCH 2141/2345] [XLA:GPU] Log the memory read+written for each
 module on VLOG(1).

PiperOrigin-RevId: 233034252
---
 .../compiler/xla/service/gpu/nvptx_compiler.cc      | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
index 4bead3e6b5..9c8a181604 100644
--- a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
+++ b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
@@ -795,14 +795,19 @@ StatusOr<std::unique_ptr<Executable>> NVPTXCompiler::RunBackend(
   std::unique_ptr<HloProfileIndexMap> profile_index_map;
   std::unique_ptr<HloProfilePrinterData> profile_printer;
 
-  if (module->config().hlo_profiling_enabled()) {
+  if (module->config().hlo_profiling_enabled() || VLOG_IS_ON(1)) {
     HloCostAnalysis cost_analysis(ShapeSizeBytesFunction());
     cost_analysis.set_bytes_per_second(
         stream_exec->GetDeviceDescription().memory_bandwidth());
     TF_RETURN_IF_ERROR(module->entry_computation()->Accept(&cost_analysis));
-    profile_index_map = absl::make_unique<HloProfileIndexMap>(*module);
-    profile_printer = CreateHloProfilePrinterData(
-        *profile_index_map, cost_analysis, entry_computation->name());
+    VLOG(1) << "HLO memory read+written: "
+            << tensorflow::strings::HumanReadableNumBytes(
+                   cost_analysis.bytes_accessed());
+    if (module->config().hlo_profiling_enabled()) {
+      profile_index_map = absl::make_unique<HloProfileIndexMap>(*module);
+      profile_printer = CreateHloProfilePrinterData(
+          *profile_index_map, cost_analysis, entry_computation->name());
+    }
   }
 
   auto* gpu_executable = new GpuExecutable(
-- 
GitLab


From 34c4e686f4a5c17c88c64ee9825ef6ad92d05c32 Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Fri, 8 Feb 2019 04:04:27 -0800
Subject: [PATCH 2142/2345] Add dilations arg to conv_3d_transpse in v2

PiperOrigin-RevId: 233036976
---
 tensorflow/python/ops/nn_ops.py               | 131 ++++++++++--------
 .../tools/api/golden/v1/tensorflow.nn.pbtxt   |   2 +-
 .../tools/api/golden/v2/tensorflow.nn.pbtxt   |   2 +-
 3 files changed, 79 insertions(+), 56 deletions(-)

diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index d65a16cd49..d2b865f771 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -2234,7 +2234,8 @@ def conv3d_transpose(
     data_format="NDHWC",
     name=None,
     input=None,  # pylint: disable=redefined-builtin
-    filters=None):
+    filters=None,
+    dilations=None):
   """The transpose of `conv3d`.
 
   This operation is sometimes called "deconvolution" after [Deconvolutional
@@ -2259,6 +2260,14 @@ def conv3d_transpose(
     name: Optional name for the returned tensor.
     input: Alias of value.
     filters: Alias of filter.
+    dilations: An int or list of `ints` that has length `1`, `3` or `5`,
+      defaults to 1. The dilation factor for each dimension of`input`. If a
+      single value is given it is replicated in the `D`, `H` and `W` dimension.
+      By default the `N` and `C` dimensions are set to 1. If set to k > 1, there
+      will be k-1 skipped cells between each filter element on that dimension.
+      The dimension order is determined by the value of `data_format`, see above
+      for details. Dilations in the batch and depth dimensions if a 5-d tensor
+      must be 1.
 
   Returns:
     A `Tensor` with the same type as `value`.
@@ -2269,70 +2278,84 @@ def conv3d_transpose(
   """
   filter = deprecated_argument_lookup("filters", filters, "filter", filter)
   value = deprecated_argument_lookup("input", input, "value", value)
-  with ops.name_scope(name, "conv3d_transpose",
-                      [value, filter, output_shape]) as name:
-    value = ops.convert_to_tensor(value, name="value")
-    filter = ops.convert_to_tensor(filter, name="filter")  # pylint: disable=redefined-builtin
-    axis = 1 if data_format == "NCDHW" else 4
-    if not value.get_shape().dims[axis].is_compatible_with(
-        filter.get_shape()[4]):
-      raise ValueError("input channels does not match filter's input channels, "
-                       "{} != {}".format(value.get_shape()[axis],
-                                         filter.get_shape()[4]))
+  return conv3d_transpose_v2(
+      value,
+      filter,
+      output_shape,
+      strides,
+      padding=padding,
+      data_format=data_format,
+      dilations=dilations,
+      name=name)
 
-    output_shape_ = ops.convert_to_tensor(output_shape, name="output_shape")
-    if not output_shape_.get_shape().is_compatible_with(tensor_shape.vector(5)):
-      raise ValueError("output_shape must have shape (5,), got {}".format(
-          output_shape_.get_shape()))
 
-    if isinstance(output_shape, (list, np.ndarray)):
-      # output_shape's shape should be == [5] if reached this point.
-      if not filter.get_shape().dims[3].is_compatible_with(
-          output_shape[axis]):
-        raise ValueError(
-            "output_shape does not match filter's output channels, "
-            "{} != {}".format(output_shape[axis],
-                              filter.get_shape()[3]))
+# pylint: disable=redefined-builtin
+@tf_export("nn.conv3d_transpose", v1=[])
+def conv3d_transpose_v2(input,
+                        filters,
+                        output_shape,
+                        strides,
+                        padding="SAME",
+                        data_format="NDHWC",
+                        dilations=None,
+                        name=None):
+  """The transpose of `conv3d`.
 
-    if padding != "VALID" and padding != "SAME":
-      raise ValueError("padding must be either VALID or SAME:"
-                       " {}".format(padding))
+  This operation is sometimes called "deconvolution" after [Deconvolutional
+  Networks](http://www.matthewzeiler.com/pubs/cvpr2010/cvpr2010.pdf), but is
+  actually the transpose (gradient) of `conv2d` rather than an actual
+  deconvolution.
+
+  Args:
+    input: A 5-D `Tensor` of type `float` and shape `[batch, height, width,
+      in_channels]` for `NHWC` data format or `[batch, in_channels, height,
+      width]` for `NCHW` data format.
+    filters: A 5-D `Tensor` with the same type as `value` and shape `[height,
+      width, output_channels, in_channels]`.  `filter`'s `in_channels` dimension
+      must match that of `value`.
+    output_shape: A 1-D `Tensor` representing the output shape of the
+      deconvolution op.
+    strides: An int or list of `ints` that has length `1`, `3` or `5`.  The
+      stride of the sliding window for each dimension of `input`. If a single
+      value is given it is replicated in the `D`, `H` and `W` dimension. By
+      default the `N` and `C` dimensions are set to 0. The dimension order is
+      determined by the value of `data_format`, see below for details.
+    padding: A string, either `'VALID'` or `'SAME'`. The padding algorithm. See
+      the "returns" section of `tf.nn.convolution` for details.
+    data_format: A string. 'NDHWC' and 'NCDHW' are supported.
+    dilations: An int or list of `ints` that has length `1`, `3` or `5`,
+      defaults to 1. The dilation factor for each dimension of`input`. If a
+      single value is given it is replicated in the `D`, `H` and `W` dimension.
+      By default the `N` and `C` dimensions are set to 1. If set to k > 1, there
+      will be k-1 skipped cells between each filter element on that dimension.
+      The dimension order is determined by the value of `data_format`, see above
+      for details. Dilations in the batch and depth dimensions if a 5-d tensor
+      must be 1.
+    name: Optional name for the returned tensor.
+
+  Returns:
+    A `Tensor` with the same type as `value`.
+  """
+  with ops.name_scope(name, "conv3d_transpose",
+                      [input, filter, output_shape]) as name:
+    if data_format is None:
+      data_format = "NDHWC"
+    channel_index = 1 if data_format.startswith("NC") else 4
+
+    strides = _get_sequence(strides, 3, channel_index, "strides")
+    dilations = _get_sequence(dilations, 3, channel_index, "dilations")
 
     return gen_nn_ops.conv3d_backprop_input_v2(
-        input_sizes=output_shape_,
-        filter=filter,
-        out_backprop=value,
+        input_sizes=output_shape,
+        filter=filters,
+        out_backprop=input,
         strides=strides,
         padding=padding,
         data_format=data_format,
+        dilations=dilations,
         name=name)
 
 
-# pylint: disable=redefined-builtin
-@tf_export("nn.conv3d_transpose", v1=[])
-def conv3d_transpose_v2(
-    input,
-    filters,
-    output_shape,
-    strides,
-    padding="SAME",
-    data_format="NDHWC",
-    name=None):
-  return conv3d_transpose(
-      input,
-      filters,
-      output_shape,
-      strides,
-      padding=padding,
-      data_format=data_format,
-      name=name)
-# pylint: enable=redefined-builtin
-conv3d_transpose_v2.__doc__ = deprecation.rewrite_argument_docstring(
-    deprecation.rewrite_argument_docstring(
-        conv3d_transpose.__doc__, "filter", "filters"),
-    "value", "input")
-
-
 @tf_export("nn.bias_add")
 def bias_add(value, bias, data_format=None, name=None):
   """Adds `bias` to `value`.
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
index e46f1bc935..d68db966fd 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
@@ -86,7 +86,7 @@ tf_module {
   }
   member_method {
     name: "conv3d_transpose"
-    argspec: "args=[\'value\', \'filter\', \'output_shape\', \'strides\', \'padding\', \'data_format\', \'name\', \'input\', \'filters\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'SAME\', \'NDHWC\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'value\', \'filter\', \'output_shape\', \'strides\', \'padding\', \'data_format\', \'name\', \'input\', \'filters\', \'dilations\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'SAME\', \'NDHWC\', \'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "convolution"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index 92709e3410..6cba569bf3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -78,7 +78,7 @@ tf_module {
   }
   member_method {
     name: "conv3d_transpose"
-    argspec: "args=[\'input\', \'filters\', \'output_shape\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'SAME\', \'NDHWC\', \'None\'], "
+    argspec: "args=[\'input\', \'filters\', \'output_shape\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'SAME\', \'NDHWC\', \'None\', \'None\'], "
   }
   member_method {
     name: "convolution"
-- 
GitLab


From dae9cabbe8fa18cee697c197f10c3a12658f7f10 Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <kramerb@google.com>
Date: Fri, 8 Feb 2019 05:06:12 -0800
Subject: [PATCH 2143/2345] [XLA:AOT] Add support for resource variables as
 inputs/outputs

This is what I think is the minimum viable solution and also serves as an RFC
for the direction in which this API should evolve.

With this change resource variables behave exactly the same way as regular
input/outputs, any variable gets translated into a pair of argument/result
accessors. Initial values coming from the graph are ignored, the user has to
supply their own instead.

For the configuration this requires specifying name/shape/type for all
variables used in the model. I tried to infer the shape from the graph but it's
rather annoying because the arguments are create very early in the process.

This is intentionally lacking documentation, setting the API in stone now seems
too early. It's unclear to me whether we want to keep the argument/result
matching long-term or switch to something that supports in-place updates. The
current API cannot do that because it would violate const correctness.

PiperOrigin-RevId: 233042296
---
 tensorflow/compiler/aot/BUILD                 |  11 +++
 .../compiler/aot/aot_only_var_handle_op.cc    |  56 +++++++++++
 tensorflow/compiler/aot/codegen.cc            |  90 ++++++++++++++++--
 tensorflow/compiler/aot/codegen_test.cc       |  20 +++-
 tensorflow/compiler/aot/codegen_test_h.golden |  56 ++++++++++-
 tensorflow/compiler/aot/codegen_test_o.golden | Bin 728 -> 800 bytes
 tensorflow/compiler/aot/tests/BUILD           |  13 +++
 .../compiler/aot/tests/make_test_graphs.py    |   9 ++
 .../tests/test_graph_tfvariable.config.pbtxt  |  12 +++
 .../compiler/aot/tests/tfcompile_test.cc      |  23 +++++
 tensorflow/compiler/tf2xla/tf2xla.cc          |  19 +++-
 tensorflow/compiler/tf2xla/tf2xla.proto       |  11 +++
 12 files changed, 305 insertions(+), 15 deletions(-)
 create mode 100644 tensorflow/compiler/aot/aot_only_var_handle_op.cc
 create mode 100644 tensorflow/compiler/aot/tests/test_graph_tfvariable.config.pbtxt

diff --git a/tensorflow/compiler/aot/BUILD b/tensorflow/compiler/aot/BUILD
index 16151e7773..af016bf80e 100644
--- a/tensorflow/compiler/aot/BUILD
+++ b/tensorflow/compiler/aot/BUILD
@@ -30,6 +30,7 @@ cc_library(
         "flags.h",
     ],
     deps = [
+        ":aot_only_var_handle_op",
         ":embedded_protocol_buffers",
         "//tensorflow/compiler/tf2xla",
         "//tensorflow/compiler/tf2xla:cpu_function_runtime",
@@ -71,6 +72,7 @@ tf_cc_test(
         ":tfcompile_lib",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "@com_google_absl//absl/strings",
@@ -205,6 +207,15 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "aot_only_var_handle_op",
+    srcs = ["aot_only_var_handle_op.cc"],
+    deps = [
+        "//tensorflow/compiler/tf2xla:xla_compiler",
+    ],
+    alwayslink = 1,
+)
+
 tf_cc_test(
     name = "benchmark_test",
     srcs = ["benchmark_test.cc"],
diff --git a/tensorflow/compiler/aot/aot_only_var_handle_op.cc b/tensorflow/compiler/aot/aot_only_var_handle_op.cc
new file mode 100644
index 0000000000..0ce36a979f
--- /dev/null
+++ b/tensorflow/compiler/aot/aot_only_var_handle_op.cc
@@ -0,0 +1,56 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/tf2xla/xla_context.h"
+#include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
+#include "tensorflow/compiler/tf2xla/xla_op_registry.h"
+
+namespace tensorflow {
+namespace {
+
+// Implementation of varhandle that binds a VarHandleOp to an XlaResource of the
+// same name. It is not safe to use this op in a JIT context.
+class XlaAotOnlyVarHandleOp : public XlaOpKernel {
+ public:
+  explicit XlaAotOnlyVarHandleOp(OpKernelConstruction* c);
+  void Compile(XlaOpKernelContext* context) override;
+
+ private:
+  string name_;
+};
+
+XlaAotOnlyVarHandleOp::XlaAotOnlyVarHandleOp(OpKernelConstruction* c)
+    : XlaOpKernel(c) {
+  OP_REQUIRES_OK(c, c->GetAttr("shared_name", &name_));
+}
+
+void XlaAotOnlyVarHandleOp::Compile(XlaOpKernelContext* context) {
+  // Look for a resource of the same name. TF also keys that on the container
+  // and type attributes, but that doesn't seem necessary.
+  for (const auto& resource : context->xla_context()->resources()) {
+    if (resource->kind() == XlaResource::kVariable &&
+        resource->name() == name_) {
+      context->SetResourceOutput(0, resource.get());
+      return;
+    }
+  }
+  context->SetStatus(
+      errors::InvalidArgument("Variable: ", name_, " not configured"));
+}
+}  // namespace
+
+REGISTER_XLA_OP(Name("VarHandleOp").CompilationOnly(), XlaAotOnlyVarHandleOp);
+
+}  // namespace tensorflow
diff --git a/tensorflow/compiler/aot/codegen.cc b/tensorflow/compiler/aot/codegen.cc
index d016632da2..da0598736a 100644
--- a/tensorflow/compiler/aot/codegen.cc
+++ b/tensorflow/compiler/aot/codegen.cc
@@ -168,12 +168,12 @@ Status GenArgMethods(const tf2xla::Config& config,
                      const xla::ProgramShapeProto& ps,
                      const CompileResult& compile_result, string* methods) {
   size_t num_args = ps.parameters_size();
-  if (config.feed_size() != num_args) {
-    return errors::InvalidArgument("mismatch between feed_size(",
-                                   config.feed_size(), ") and num_args(",
-                                   num_args, ")");
+  if (config.feed_size() + config.variable_size() != num_args) {
+    return errors::InvalidArgument(
+        "mismatch between feed_size(", config.feed_size(), ")+variable_size(",
+        config.variable_size(), ") and num_args(", num_args, ")");
   }
-  for (int i = 0; i < num_args; ++i) {
+  for (int i = 0; i < config.feed_size(); ++i) {
     std::vector<std::pair<string, string>> rewrites;
     TF_RETURN_IF_ERROR(
         AddRewritesForShape(i, xla::Shape(ps.parameters(i)), &rewrites));
@@ -212,12 +212,14 @@ Status GenResultMethods(const tf2xla::Config& config,
     // tuple result, and we rely on this to simplify code generation.
     return errors::Internal("codegen requires the XLA result to be a tuple");
   }
-  if (config.fetch_size() != ps.result().tuple_shapes_size()) {
+  size_t num_results = ps.result().tuple_shapes_size();
+  if (config.fetch_size() + config.variable_size() != num_results) {
     return errors::InvalidArgument("mismatch between fetch_size(",
-                                   config.feed_size(), ") and tuple_size(",
+                                   config.fetch_size(), ")+variable_size(",
+                                   config.variable_size(), ") and tuple_size(",
                                    ps.result().tuple_shapes_size(), ")");
   }
-  for (int i = 0; i < ps.result().tuple_shapes_size(); ++i) {
+  for (int i = 0; i < config.fetch_size(); ++i) {
     std::vector<std::pair<string, string>> rewrites;
     TF_RETURN_IF_ERROR(AddRewritesForShape(
         i, xla::Shape(ps.result().tuple_shapes(i)), &rewrites));
@@ -245,6 +247,51 @@ Status GenResultMethods(const tf2xla::Config& config,
   return Status::OK();
 }
 
+// Generate methods for variables.
+Status GenVariableMethods(const tf2xla::Config& config,
+                          const xla::ProgramShapeProto& ps, string* methods) {
+  size_t num_args = ps.parameters_size();
+  for (int i = config.feed_size(); i < num_args; ++i) {
+    std::vector<std::pair<string, string>> rewrites;
+    TF_RETURN_IF_ERROR(
+        AddRewritesForShape(i, xla::Shape(ps.parameters(i)), &rewrites));
+    const string code = R"(
+  void set_var_{{NAME}}_input_data({{TYPE}}* data) {
+    set_arg_data({{I}}, data);
+  }
+)";
+    const tf2xla::Variable& var = config.variable(i - config.feed_size());
+    *methods += RewriteWithName(
+        var.name().empty() ? var.node_name() : var.name(), code, rewrites);
+  }
+  size_t num_results = ps.result().tuple_shapes_size();
+  for (int i = config.fetch_size(); i < num_results; ++i) {
+    std::vector<std::pair<string, string>> rewrites;
+    TF_RETURN_IF_ERROR(AddRewritesForShape(
+        i, xla::Shape(ps.result().tuple_shapes(i)), &rewrites));
+    string code = R"(
+  {{TYPE}}* var_{{NAME}}_result_data() {
+    return static_cast<{{TYPE}}*>(result_data({{I}}));
+  }
+  {{TYPE}}& var_{{NAME}}_result({{DIM_VARS}}) {
+    return (*static_cast<{{TYPE}}(*){{DIM_SIZES}}>(
+        result_data({{I}}))){{INDICES}};
+  }
+  const {{TYPE}}* var_{{NAME}}_result_data() const {
+    return static_cast<const {{TYPE}}*>(result_data({{I}}));
+  }
+  const {{TYPE}}& var_{{NAME}}_result({{DIM_VARS}}) const {
+    return (*static_cast<const {{TYPE}}(*){{DIM_SIZES}}>(
+        result_data({{I}}))){{INDICES}};
+  }
+)";
+    const tf2xla::Variable& var = config.variable(i - config.fetch_size());
+    *methods += RewriteWithName(
+        var.name().empty() ? var.node_name() : var.name(), code, rewrites);
+  }
+  return Status::OK();
+}
+
 // Generates code implementing {Arg,Result}Names(), where T is one of
 // tf2xla::{Feed,Fetch}. Each feed or fetch name results in a C-style string
 // literal in the array, with nullptr terminating the array.
@@ -291,6 +338,14 @@ Status ValidateFeedFetchCppNames(const tf2xla::Config& config) {
       TF_RETURN_IF_ERROR(ValidateCppIdent(fetch.name(), "fetch name"));
     }
   }
+  for (const tf2xla::Variable& variable : config.variable()) {
+    if (!variable.name().empty()) {
+      TF_RETURN_IF_ERROR(ValidateCppIdent(variable.name(), "variable name"));
+    } else {
+      TF_RETURN_IF_ERROR(
+          ValidateCppIdent(variable.node_name(), "variable name"));
+    }
+  }
   return Status::OK();
 }
 
@@ -339,9 +394,10 @@ Status GenerateHeader(const CodegenOpts& opts, const tf2xla::Config& config,
   std::vector<BufferInfo> buffer_infos_for_temps =
       ExtractTempBufferInfos(buffer_infos);
   const xla::ProgramShapeProto& ps = compile_result.program_shape;
-  string methods_arg, methods_result;
+  string methods_arg, methods_result, methods_variable;
   TF_RETURN_IF_ERROR(GenArgMethods(config, ps, compile_result, &methods_arg));
   TF_RETURN_IF_ERROR(GenResultMethods(config, ps, &methods_result));
+  TF_RETURN_IF_ERROR(GenVariableMethods(config, ps, &methods_variable));
   const size_t arg_bytes_aligned = cpu_function_runtime::AlignedBufferBytes(
       buffer_infos_for_args.data(), buffer_infos_for_args.size(),
       /*allocate_entry_params=*/true);
@@ -523,6 +579,21 @@ class {{CLASS}} final : public tensorflow::XlaCompiledCpuFunction {
   // buffers are managed internally, and may change after each call to Run.
 {{METHODS_RESULT}}
 
+  // Methods for managing variable buffers. Buffers are in row-major order. The
+  // input and output buffers may or may not be identical.
+  //
+  // void set_var_X_data(T* data)
+  //   Sets the buffer for variable X.
+  //
+  // T* var_X_data()
+  //   Returns the buffer of type T for variable X.
+  //
+  // T& var_X(...dim indices...)
+  //   Returns a reference to the value of type T for variable X,
+  //   with dim indices specifying which value. No bounds checking is performed
+  //   on dim indices.
+{{METHODS_VARIABLE}}
+
  private:
   // Number of buffers for the compiled computation.
   static constexpr size_t kNumBuffers = {{NUM_BUFFERS}};
@@ -589,6 +660,7 @@ class {{CLASS}} final : public tensorflow::XlaCompiledCpuFunction {
        include_hlo_profile_printer_data_proto},
       {"{{METHODS_ARG}}\n", methods_arg},
       {"{{METHODS_RESULT}}\n", methods_result},
+      {"{{METHODS_VARIABLE}}\n", methods_variable},
       {"{{NS_END}}\n", ns_end},
       {"{{NS_START}}\n", ns_start},
       {"{{PROGRAM_SHAPE}}", xla::ShapeUtil::HumanString(xla::ProgramShape(ps))},
diff --git a/tensorflow/compiler/aot/codegen_test.cc b/tensorflow/compiler/aot/codegen_test.cc
index c1788ca32a..5580e55b69 100644
--- a/tensorflow/compiler/aot/codegen_test.cc
+++ b/tensorflow/compiler/aot/codegen_test.cc
@@ -22,6 +22,8 @@ limitations under the License.
 #include "absl/strings/string_view.h"
 #include "llvm/Support/TargetSelect.h"
 #include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/core/framework/tensor_shape.pb.h"
+#include "tensorflow/core/framework/types.pb.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/io/path.h"
@@ -172,6 +174,15 @@ TEST(CodegenTest, Golden) {
   tf2xla::Fetch* fetch = config.add_fetch();
   fetch->mutable_id()->set_node_name("fetch0");
   fetch->set_name("myfetch");
+  tf2xla::Variable* variable = config.add_variable();
+  variable->set_node_name("myvar");
+  variable->mutable_shape()->add_dim()->set_size(1);
+  variable->set_type(DT_FLOAT);
+  tf2xla::Variable* variable2 = config.add_variable();
+  variable2->set_node_name("my/var");
+  variable2->set_name("myvar2");
+  variable2->mutable_shape()->add_dim()->set_size(5);
+  variable2->set_type(DT_INT32);
   CompileResult compile_result;
   compile_result.aot.reset(new xla::cpu::CpuAotCompilationResult(
       {},
@@ -186,9 +197,14 @@ TEST(CodegenTest, Golden) {
           {
               xla::ShapeUtil::MakeShape(xla::F32, {1, 2}),
               xla::ShapeUtil::MakeShape(xla::S64, {3, 4}),
+              xla::ShapeUtil::MakeShape(xla::F32, {1}),
+              xla::ShapeUtil::MakeShape(xla::S32, {5}),
           },
-          xla::ShapeUtil::MakeTupleShape(
-              {xla::ShapeUtil::MakeShape(xla::U32, {5, 6})}))
+          xla::ShapeUtil::MakeTupleShape({
+              xla::ShapeUtil::MakeShape(xla::U32, {5, 6}),
+              xla::ShapeUtil::MakeShape(xla::F32, {1}),
+              xla::ShapeUtil::MakeShape(xla::S32, {5}),
+          }))
           .ToProto();
   compile_result.entry_point = "entry_point";
   compile_result.pointer_size = 8;
diff --git a/tensorflow/compiler/aot/codegen_test_h.golden b/tensorflow/compiler/aot/codegen_test_h.golden
index 35994fc785..b5f33d690d 100644
--- a/tensorflow/compiler/aot/codegen_test_h.golden
+++ b/tensorflow/compiler/aot/codegen_test_h.golden
@@ -52,7 +52,7 @@ namespace bar {
 //   is guaranteed that no thread may call a non-const method.
 //
 // The logical function signature is:
-//   ((unknown): f32[1,2], (unknown): s64[3,4]) -> (u32[5,6])
+//   ((unknown): f32[1,2], (unknown): s64[3,4], (unknown): f32[1], (unknown): s32[5]) -> (u32[5,6], f32[1], s32[5])
 //
 // Memory stats:
 //   arg bytes total:    104
@@ -214,6 +214,58 @@ class MyClass final : public tensorflow::XlaCompiledCpuFunction {
         result_data(0)))[dim0][dim1];
   }
 
+  // Methods for managing variable buffers. Buffers are in row-major order. The
+  // input and output buffers may or may not be identical.
+  //
+  // void set_var_X_data(T* data)
+  //   Sets the buffer for variable X.
+  //
+  // T* var_X_data()
+  //   Returns the buffer of type T for variable X.
+  //
+  // T& var_X(...dim indices...)
+  //   Returns a reference to the value of type T for variable X,
+  //   with dim indices specifying which value. No bounds checking is performed
+  //   on dim indices.
+
+  void set_var_myvar_input_data(float* data) {
+    set_arg_data(2, data);
+  }
+
+  void set_var_myvar2_input_data(tensorflow::int32* data) {
+    set_arg_data(3, data);
+  }
+
+  float* var_myvar_result_data() {
+    return static_cast<float*>(result_data(1));
+  }
+  float& var_myvar_result() {
+    return (*static_cast<float(*)[1]>(
+        result_data(1)))[0];
+  }
+  const float* var_myvar_result_data() const {
+    return static_cast<const float*>(result_data(1));
+  }
+  const float& var_myvar_result() const {
+    return (*static_cast<const float(*)[1]>(
+        result_data(1)))[0];
+  }
+
+  tensorflow::int32* var_myvar2_result_data() {
+    return static_cast<tensorflow::int32*>(result_data(2));
+  }
+  tensorflow::int32& var_myvar2_result(size_t dim0) {
+    return (*static_cast<tensorflow::int32(*)[5]>(
+        result_data(2)))[dim0];
+  }
+  const tensorflow::int32* var_myvar2_result_data() const {
+    return static_cast<const tensorflow::int32*>(result_data(2));
+  }
+  const tensorflow::int32& var_myvar2_result(size_t dim0) const {
+    return (*static_cast<const tensorflow::int32(*)[5]>(
+        result_data(2)))[dim0];
+  }
+
  private:
   // Number of buffers for the compiled computation.
   static constexpr size_t kNumBuffers = 6;
@@ -257,7 +309,7 @@ class MyClass final : public tensorflow::XlaCompiledCpuFunction {
   static const xla::ProgramShapeProto* StaticProgramShape() {
     static const xla::ProgramShapeProto* kShape = []() {
     xla::ProgramShapeProto* proto = new xla::ProgramShapeProto;
-    proto->ParseFromArray(&__tfcompile_foo_bar_MyClass_ProgramShapeProto_protobuf_array_contents[0], 64);
+    proto->ParseFromArray(&__tfcompile_foo_bar_MyClass_ProgramShapeProto_protobuf_array_contents[0], 132);
     return proto;
   }();
     return kShape;
diff --git a/tensorflow/compiler/aot/codegen_test_o.golden b/tensorflow/compiler/aot/codegen_test_o.golden
index 7f7b96428572705f30144e6c95cd4cf9c44ce2a3..2884597abcf29583e6192296b0e4ce6825d7c01a 100644
GIT binary patch
delta 142
zcmcb?x`1th2IGQ>n)cCL`~uukjEq{WT#O6~j7E$MT>JtoQjDw!0U-+kUL_#`4k;#9
zHZ3+TCa^jt1_mW`ol0mrCkHahPMo2^*fR0pdnN(K$(tD47+WS=GMO_TnOw+ZKY0Tq
J$K)MM3IO`>7K8u*

delta 78
zcmZ3$c7t_-24loTO?xpRF#%pBAps64CRR2rHZCSc1_eeVrpXH#r6xYmV04(w$@re>
g1jFQCjBS%Gm;@$IVA5c0n7oiteDVb_>nD>s0JBOII{*Lx

diff --git a/tensorflow/compiler/aot/tests/BUILD b/tensorflow/compiler/aot/tests/BUILD
index 10fa33ab5e..444264ba6e 100644
--- a/tensorflow/compiler/aot/tests/BUILD
+++ b/tensorflow/compiler/aot/tests/BUILD
@@ -69,6 +69,7 @@ genrule(
         "test_graph_tfmatmulandadd.pb",
         "test_graph_tfsplits.pb",
         "test_graph_tftop_k.pb",
+        "test_graph_tfvariable.pb",
     ],
     # Set CUDA_VISIBLE_DEVICES='' to prevent the code we launch from using any
     # GPUs which might be present.  This is important because builds may run
@@ -222,6 +223,17 @@ tf_library(
     ],
 )
 
+tf_library(
+    name = "test_graph_tfvariable",
+    testonly = 1,
+    config = "test_graph_tfvariable.config.pbtxt",
+    cpp_class = "VariableComp",
+    graph = "test_graph_tfvariable.pb",
+    tags = [
+        "manual",
+    ],
+)
+
 tf_cc_test(
     name = "tfcompile_test",
     srcs = ["tfcompile_test.cc"],
@@ -241,6 +253,7 @@ tf_cc_test(
         ":test_graph_tfmatmulandadd_with_profiling",
         ":test_graph_tfsplits",
         ":test_graph_tftop_k",
+        ":test_graph_tfvariable",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:test",
         "//tensorflow/compiler/xla:xla_data_proto",
diff --git a/tensorflow/compiler/aot/tests/make_test_graphs.py b/tensorflow/compiler/aot/tests/make_test_graphs.py
index 7bac79ec06..42f8812def 100644
--- a/tensorflow/compiler/aot/tests/make_test_graphs.py
+++ b/tensorflow/compiler/aot/tests/make_test_graphs.py
@@ -149,6 +149,14 @@ def tftop_k(_):
   array_ops.identity(output[1], name='indices')
 
 
+def tfvariable(_):
+  x = variables.Variable(1000.0, name='x')
+  old_x = x.value()
+  with ops.control_dependencies([old_x]):
+    new_x = x.assign_add(42.0)
+  array_ops.stack([old_x, new_x], name='result')
+
+
 def write_graph(build_graph, out_dir):
   """Build a graph using build_graph and write it out."""
   g = ops.Graph()
@@ -171,6 +179,7 @@ def main(_):
   write_graph(tfmatmulandadd, FLAGS.out_dir)
   write_graph(tfsplits, FLAGS.out_dir)
   write_graph(tftop_k, FLAGS.out_dir)
+  write_graph(tfvariable, FLAGS.out_dir)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/compiler/aot/tests/test_graph_tfvariable.config.pbtxt b/tensorflow/compiler/aot/tests/test_graph_tfvariable.config.pbtxt
new file mode 100644
index 0000000000..9b4c4215a3
--- /dev/null
+++ b/tensorflow/compiler/aot/tests/test_graph_tfvariable.config.pbtxt
@@ -0,0 +1,12 @@
+# Text form of tensorflow.tf2xla.Config proto.
+fetch {
+  id { node_name: "result" }
+}
+
+variable {
+  node_name: "x"
+  shape {
+    dim { size: 1 }
+  }
+  type: DT_FLOAT
+}
diff --git a/tensorflow/compiler/aot/tests/tfcompile_test.cc b/tensorflow/compiler/aot/tests/tfcompile_test.cc
index 4dd79e5882..5f9316f393 100644
--- a/tensorflow/compiler/aot/tests/tfcompile_test.cc
+++ b/tensorflow/compiler/aot/tests/tfcompile_test.cc
@@ -30,6 +30,7 @@ limitations under the License.
 #include "tensorflow/compiler/aot/tests/test_graph_tfmatmulandadd_with_profiling.h"
 #include "tensorflow/compiler/aot/tests/test_graph_tfsplits.h"
 #include "tensorflow/compiler/aot/tests/test_graph_tftop_k.h"
+#include "tensorflow/compiler/aot/tests/test_graph_tfvariable.h"
 #include "tensorflow/compiler/xla/service/hlo_profile_printer.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/test.h"
@@ -473,6 +474,28 @@ TEST(TFCompileTest, TopK) {
   EXPECT_EQ(expected_indices[1], fn.result1(1));
 }
 
+TEST(TFCompileTest, Variable) {
+  Eigen::ThreadPool tp(1);
+  Eigen::ThreadPoolDevice device(&tp, tp.NumThreads());
+
+  VariableComp fn;
+  float x = 23;
+  fn.set_var_x_input_data(&x);
+
+  fn.set_thread_pool(&device);
+  fn.Run();
+  EXPECT_EQ(fn.result0(0, 0), 23);
+  EXPECT_EQ(fn.result0(1, 0), 65);
+  EXPECT_EQ(fn.var_x_result(), 65);
+
+  EXPECT_EQ(x, 23);
+  x = fn.var_x_result();
+  fn.Run();
+  EXPECT_EQ(fn.result0(0, 0), 65);
+  EXPECT_EQ(fn.result0(1, 0), 107);
+  EXPECT_EQ(fn.var_x_result(), 107);
+}
+
 TEST(TFCompileTest, AssertEqAndReturnDiff) {
   // Assert is converted into a no-op in XLA, so there is no failure even if the
   // two args are different.
diff --git a/tensorflow/compiler/tf2xla/tf2xla.cc b/tensorflow/compiler/tf2xla/tf2xla.cc
index cf48576ec2..28a4566c9d 100644
--- a/tensorflow/compiler/tf2xla/tf2xla.cc
+++ b/tensorflow/compiler/tf2xla/tf2xla.cc
@@ -254,7 +254,8 @@ Status CreateXlaArgs(const Graph& graph,
 
 // Converts the TensorFlow graph into an XLA computation, by executing the
 // graph symbolically, with each op building up the XLA HLO.
-Status ConvertGraphToXla(std::unique_ptr<Graph> graph, xla::Client* client,
+Status ConvertGraphToXla(std::unique_ptr<Graph> graph,
+                         const tf2xla::Config& config, xla::Client* client,
                          xla::XlaComputation* computation) {
   XlaOpRegistry::RegisterCompilationKernels();
   for (Node* node : graph->nodes()) {
@@ -264,6 +265,19 @@ Status ConvertGraphToXla(std::unique_ptr<Graph> graph, xla::Client* client,
   std::vector<XlaCompiler::Argument> xla_args;
   TF_RETURN_IF_ERROR(CreateXlaArgs(*graph, &xla_args));
 
+  // Populate arguments with resource variables from the config. The variables
+  // get turned into inputs and outputs.
+  for (const tf2xla::Variable& variable : config.variable()) {
+    XlaCompiler::Argument arg;
+    arg.type = variable.type();
+    arg.kind = XlaCompiler::Argument::kResource;
+    arg.shape = variable.shape();
+    arg.name = variable.node_name();
+    arg.resource_kind = XlaResource::kVariable;
+    arg.initialized = true;
+    xla_args.push_back(std::move(arg));
+  }
+
   // Compile the graph into an XLA computation.
   XlaCompiler::Options compiler_options;
   compiler_options.client = client;
@@ -361,7 +375,8 @@ Status ConvertGraphDefToXla(const GraphDef& graph_def,
                             xla::XlaComputation* computation) {
   std::unique_ptr<Graph> graph;
   TF_RETURN_IF_ERROR(InitGraph(graph_def, config, &graph));
-  TF_RETURN_IF_ERROR(ConvertGraphToXla(std::move(graph), client, computation));
+  TF_RETURN_IF_ERROR(
+      ConvertGraphToXla(std::move(graph), config, client, computation));
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/tf2xla/tf2xla.proto b/tensorflow/compiler/tf2xla/tf2xla.proto
index 18c9089f5f..5627af7452 100644
--- a/tensorflow/compiler/tf2xla/tf2xla.proto
+++ b/tensorflow/compiler/tf2xla/tf2xla.proto
@@ -39,6 +39,15 @@ message Fetch {
   string name = 2;  // Optional name for generated code.
 };
 
+// Variable represents a resource variable with the given name, shape and type.
+message Variable {
+  string node_name = 1;
+  string name =
+      2;  // Optional name for generated code. If empty, node_name will be used.
+  TensorShapeProto shape = 3;
+  DataType type = 4;
+}
+
 // Config represents configuration information for tf2xla conversion.
 message Config {
   // Each feed is a positional input argument for the generated computation.
@@ -47,4 +56,6 @@ message Config {
   // Each fetch is a positional output argument for the generated computation.
   // The order of each entry matches the order of each output argument.
   repeated Fetch fetch = 2;
+  // Each variable is a named input and output of the generated computation.
+  repeated Variable variable = 3;
 };
-- 
GitLab


From 942cc64aae9d78bdc7a7b9769f0b605b08ebf7b9 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Fri, 8 Feb 2019 06:13:24 -0800
Subject: [PATCH 2144/2345] Switch XLA TopK to use the new API method for Sort.

PiperOrigin-RevId: 233048377
---
 tensorflow/compiler/xla/client/lib/BUILD      | 1 +
 tensorflow/compiler/xla/client/lib/sorting.cc | 8 +++++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD
index 84b542d3d3..7e7b5732ae 100644
--- a/tensorflow/compiler/xla/client/lib/BUILD
+++ b/tensorflow/compiler/xla/client/lib/BUILD
@@ -375,6 +375,7 @@ cc_library(
     srcs = ["sorting.cc"],
     hdrs = ["sorting.h"],
     deps = [
+        ":comparators",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:types",
         "//tensorflow/compiler/xla:util",
diff --git a/tensorflow/compiler/xla/client/lib/sorting.cc b/tensorflow/compiler/xla/client/lib/sorting.cc
index e8553a08bb..3245f46e6f 100644
--- a/tensorflow/compiler/xla/client/lib/sorting.cc
+++ b/tensorflow/compiler/xla/client/lib/sorting.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/compiler/xla/client/lib/sorting.h"
+#include "tensorflow/compiler/xla/client/lib/comparators.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/util.h"
@@ -30,7 +31,12 @@ XlaOp TopK(XlaOp input, int64 k) {
         ShapeUtil::MakeShape(S32, AsInt64Slice(input_shape.dimensions()));
     XlaOp iota_s32 = Iota(builder, iota_shape, last_dim);
     auto input_dims = input_shape.dimensions();
-    XlaOp sort_result = Sort(Neg(input), {iota_s32});
+    // TODO(b/122298745): Get rid of Neg() and use CreateScalarGtComputation
+    // once the TPU backend supports the comparison computations.
+    XlaOp sort_result =
+        Sort({Neg(input), iota_s32},
+             CreateScalarLtComputation({input_shape.element_type(), S32},
+                                       iota_s32.builder()));
     std::vector<int64> start_indices(input_shape.dimensions_size(), 0);
     std::vector<int64> limit_indices(input_dims.begin(), input_dims.end());
     limit_indices[last_dim] = k;
-- 
GitLab


From f259cb4ac2cbe69c450d956bd3262a23d4a44291 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Fri, 8 Feb 2019 06:34:13 -0800
Subject: [PATCH 2145/2345] Have the command do what the message tells.

PiperOrigin-RevId: 233050417
---
 tensorflow/python/autograph/pyct/errors.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/autograph/pyct/errors.py b/tensorflow/python/autograph/pyct/errors.py
index 3de3fadc88..973c60dcde 100644
--- a/tensorflow/python/autograph/pyct/errors.py
+++ b/tensorflow/python/autograph/pyct/errors.py
@@ -44,5 +44,5 @@ def report_internal_error(entity, exception):
   raise AutoGraphError(
       'Unexpected error transforming %s. If you believe this is due to a bug,'
       ' please set the verbosity to 10 (on Linux, `export '
-      'AUTOGRAPH_VERBOSITY=1`) and attach the full output when filing the bug '
+      'AUTOGRAPH_VERBOSITY=10`) and attach the full output when filing the bug '
       'report. Caused by %s' % (entity, exception))
-- 
GitLab


From bd8eb07ad5e9a0191a013c29342d9ac2af9942fa Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Fri, 8 Feb 2019 07:10:32 -0800
Subject: [PATCH 2146/2345] Method for conv_1d transpose updated to use
 dilations and exported

PiperOrigin-RevId: 233054694
---
 tensorflow/python/kernel_tests/BUILD          |  14 +
 tensorflow/python/kernel_tests/conv1d_test.py |   2 +-
 .../kernel_tests/conv1d_transpose_test.py     | 260 ++++++++++++++++++
 tensorflow/python/ops/nn_ops.py               |  86 +++---
 .../tools/api/golden/v1/tensorflow.nn.pbtxt   |   4 +
 .../tools/api/golden/v2/tensorflow.nn.pbtxt   |   4 +
 6 files changed, 316 insertions(+), 54 deletions(-)
 create mode 100644 tensorflow/python/kernel_tests/conv1d_transpose_test.py

diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 60126bb9fa..c660633a76 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -1678,6 +1678,20 @@ cuda_py_test(
     xla_enable_strict_auto_jit = True,
 )
 
+cuda_py_test(
+    name = "conv1d_transpose_test",
+    size = "small",
+    srcs = ["conv1d_transpose_test.py"],
+    additional_deps = [
+        "//third_party/py/numpy",
+        "//tensorflow/python:client",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:nn_ops",
+    ],
+    xla_enable_strict_auto_jit = True,
+)
+
 cuda_py_test(
     name = "conv2d_transpose_test",
     size = "small",
diff --git a/tensorflow/python/kernel_tests/conv1d_test.py b/tensorflow/python/kernel_tests/conv1d_test.py
index e8463323df..4b44bb6c91 100644
--- a/tensorflow/python/kernel_tests/conv1d_test.py
+++ b/tensorflow/python/kernel_tests/conv1d_test.py
@@ -68,7 +68,7 @@ class Conv1DTest(test.TestCase):
       f = constant_op.constant(
           1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
       output = nn_ops.conv1d_transpose(
-          x, f, y_shape, stride=stride, padding="VALID")
+          x, f, y_shape, strides=stride, padding="VALID")
       value = self.evaluate(output)
 
       cache_values = np.zeros(y_shape, dtype=np.float32)
diff --git a/tensorflow/python/kernel_tests/conv1d_transpose_test.py b/tensorflow/python/kernel_tests/conv1d_transpose_test.py
new file mode 100644
index 0000000000..02ac5af7aa
--- /dev/null
+++ b/tensorflow/python/kernel_tests/conv1d_transpose_test.py
@@ -0,0 +1,260 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for convolution related functionality in tensorflow.ops.nn."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+from six.moves import xrange  # pylint: disable=redefined-builtin
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import gradient_checker
+from tensorflow.python.ops import nn_ops
+import tensorflow.python.ops.nn_grad  # pylint: disable=unused-import
+from tensorflow.python.platform import test
+
+
+class Conv1DTransposeTest(test.TestCase):
+
+  def testConv1DTransposeSingleStride(self):
+    with self.cached_session():
+      strides = [1, 1, 1]
+
+      # Input, output: [batch, width, depth]
+      x_shape = [2, 6, 3]
+      y_shape = [2, 6, 2]
+
+      # Filter: [kernel_width, output_depth, input_depth]
+      f_shape = [3, 2, 3]
+
+      x = constant_op.constant(
+          1.0, shape=x_shape, name="x", dtype=dtypes.float32)
+      f = constant_op.constant(
+          1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
+      output = nn_ops.conv1d_transpose(
+          x, f, y_shape, strides=strides, padding="SAME")
+      value = self.evaluate(output)
+
+      for n in xrange(y_shape[0]):
+        for w in xrange(y_shape[1]):
+          for c in xrange(y_shape[2]):
+            target = 2 * 3.0
+            w_in = w > 0 and w < y_shape[1] - 1
+            if w_in:
+              target += 3.0
+            self.assertAllClose(target, value[n, w, c])
+
+  def testConv1DTransposeSame(self):
+    with self.cached_session():
+      strides = [1, 2, 1]
+
+      # Input, output: [batch, width, depth]
+      x_shape = [2, 4, 3]
+      y_shape = [2, 8, 2]
+
+      # Filter: [kernel_width, output_depth, input_depth]
+      f_shape = [3, 2, 3]
+
+      x = constant_op.constant(
+          1.0, shape=x_shape, name="x", dtype=dtypes.float32)
+      f = constant_op.constant(
+          1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
+      output = nn_ops.conv1d_transpose(
+          x, f, y_shape, strides=strides, padding="SAME")
+      value = self.evaluate(output)
+
+      for n in xrange(x_shape[0]):
+        for k in xrange(f_shape[1]):
+          for w in xrange(y_shape[1]):
+            target = 3.0
+            # We add a case for locations divisible by the stride.
+            w_in = w % strides[1] == 0 and w > 0 and w < y_shape[1] - 1
+            if w_in:
+              target += 3.0
+            self.assertAllClose(target, value[n, w, k])
+
+  def testConv1DTransposeValid(self):
+    with self.cached_session():
+      strides = [1, 2, 1]
+
+      # Input, output: [batch, width, depth]
+      x_shape = [2, 4, 3]
+      y_shape = [2, 9, 2]
+
+      # Filter: [kernel_width, output_depth, input_depth]
+      f_shape = [3, 2, 3]
+
+      x = constant_op.constant(
+          1.0, shape=x_shape, name="x", dtype=dtypes.float32)
+      f = constant_op.constant(
+          1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
+      output = nn_ops.conv1d_transpose(
+          x, f, y_shape, strides=strides, padding="VALID")
+      value = self.evaluate(output)
+
+      cache_values = np.zeros(y_shape, dtype=np.float32)
+
+      # The amount of padding added
+      pad = 1
+
+      for n in xrange(x_shape[0]):
+        for k in xrange(f_shape[1]):
+          for w in xrange(pad, y_shape[1] - pad):
+            target = 3.0
+            # We add a case for locations divisible by the stride.
+            w_in = w % strides[1] == 0 and w > pad and w < y_shape[1] - 1 - pad
+            if w_in:
+              target += 3.0
+            cache_values[n, w, k] = target
+
+          # copy values in the border
+          cache_values[n, 0, k] = cache_values[n, 1, k]
+          cache_values[n, -1, k] = cache_values[n, -2, k]
+          cache_values[n, :, k] = cache_values[n, :, k]
+
+    self.assertAllClose(cache_values, value)
+
+  @test_util.run_deprecated_v1
+  def testGradient(self):
+    x_shape = [2, 4, 3]
+    f_shape = [3, 2, 3]
+    y_shape = [2, 8, 2]
+    strides = [1, 2, 1]
+    np.random.seed(1)  # Make it reproducible.
+    x_val = np.random.random_sample(x_shape).astype(np.float64)
+    f_val = np.random.random_sample(f_shape).astype(np.float64)
+    with self.cached_session():
+      x = constant_op.constant(x_val, name="x", dtype=dtypes.float32)
+      f = constant_op.constant(f_val, name="f", dtype=dtypes.float32)
+      output = nn_ops.conv1d_transpose(
+          x, f, y_shape, strides=strides, padding="SAME")
+      err = gradient_checker.compute_gradient_error([x, f], [x_shape, f_shape],
+                                                    output, y_shape)
+    print("conv1d_transpose gradient err = %g " % err)
+    err_tolerance = 0.0005
+    self.assertLess(err, err_tolerance)
+
+  def testConv1DTransposeSingleStrideNCW(self):
+    # `NCW` data format is only supported for CUDA device.
+    if test.is_gpu_available(cuda_only=True):
+      with self.session(use_gpu=True):
+        strides = [1, 1, 1]
+
+        # Input, output: [batch, depth, width]
+        x_shape = [2, 3, 4]
+        y_shape = [2, 2, 4]
+
+        # Filter: [kernel_width, output_depth, input_depth]
+        f_shape = [3, 2, 3]
+
+        x = constant_op.constant(
+            1.0, shape=x_shape, name="x", dtype=dtypes.float32)
+        f = constant_op.constant(
+            1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
+
+        output = nn_ops.conv1d_transpose(
+            x, f, y_shape, strides=strides, padding="SAME", data_format="NCW")
+
+        value = self.evaluate(output)
+        for n in xrange(x_shape[0]):
+          for k in xrange(f_shape[1]):
+            for w in xrange(y_shape[2]):
+              target = 2 * 3.0
+              w_in = w > 0 and w < y_shape[2] - 1
+              if w_in:
+                target += 3.0
+              self.assertAllClose(target, value[n, k, w])
+
+  def testConv1DTransposeSameNCW(self):
+    # `NCW` data format is only supported for CUDA device.
+    if test.is_gpu_available(cuda_only=True):
+      with self.session(use_gpu=True):
+        strides = [1, 1, 2]
+
+        # Input, output: [batch, depth, width]
+        x_shape = [2, 3, 4]
+        y_shape = [2, 2, 8]
+
+        # Filter: [kernel_width, output_depth, input_depth]
+        f_shape = [3, 2, 3]
+
+        x = constant_op.constant(
+            1.0, shape=x_shape, name="x", dtype=dtypes.float32)
+        f = constant_op.constant(
+            1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
+
+        output = nn_ops.conv1d_transpose(
+            x, f, y_shape, strides=strides, padding="SAME", data_format="NCW")
+
+        value = self.evaluate(output)
+        for n in xrange(x_shape[0]):
+          for k in xrange(f_shape[1]):
+            for w in xrange(y_shape[2]):
+              target = 3.0
+              # We add a case for locations divisible by the stride.
+              w_in = w % strides[2] == 0 and w > 0 and w < y_shape[2] - 1
+              if w_in:
+                target += 3.0
+              self.assertAllClose(target, value[n, k, w])
+
+  def testConv1DTransposeValidNCW(self):
+    # `NCW` data format is only supported for CUDA device.
+    if test.is_gpu_available(cuda_only=True):
+      with self.session(use_gpu=True):
+        strides = [1, 1, 2]
+
+        # Input, output: [batch, depth, width]
+        x_shape = [2, 3, 4]
+        y_shape = [2, 2, 9]
+
+        # Filter: [kernel_width, output_depth, input_depth]
+        f_shape = [3, 2, 3]
+
+        x = constant_op.constant(
+            1.0, shape=x_shape, name="x", dtype=dtypes.float32)
+        f = constant_op.constant(
+            1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
+        output = nn_ops.conv1d_transpose(
+            x, f, y_shape, strides=strides, padding="VALID", data_format="NCW")
+
+        value = self.evaluate(output)
+        cache_values = np.zeros(y_shape, dtype=np.float32)
+        # The amount of padding added
+        pad = 1
+        for n in xrange(x_shape[0]):
+          for k in xrange(f_shape[1]):
+            for w in xrange(pad, y_shape[2] - pad):
+              target = 3.0
+              # We add a case for locations divisible by the stride.
+              w_in = w % strides[2] == 0 and w > pad and \
+                     w < y_shape[2] - 1 - pad
+              if w_in:
+                target += 3.0
+              cache_values[n, k, w] = target
+
+            # copy values in the border
+            cache_values[n, k, 0] = cache_values[n, k, 1]
+            cache_values[n, k, -1] = cache_values[n, k, -2]
+            cache_values[n, k, :] = cache_values[n, k, :]
+
+        self.assertAllClose(cache_values, value)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index d2b865f771..bc9e404d16 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -4202,13 +4202,15 @@ def conv1d_v2(
       dilations=dilations)
 
 
+@tf_export("nn.conv1d_transpose")
 def conv1d_transpose(
-    value,
-    filter,  # pylint: disable=redefined-builtin
+    input,  # pylint: disable=redefined-builtin
+    filters,
     output_shape,
-    stride,
+    strides,
     padding="SAME",
     data_format="NWC",
+    dilations=None,
     name=None):
   """The transpose of `conv1d`.
 
@@ -4218,19 +4220,23 @@ def conv1d_transpose(
   deconvolution.
 
   Args:
-    value: A 3-D `Tensor` of type `float` and shape
+    input: A 3-D `Tensor` of type `float` and shape
       `[batch, in_width, in_channels]` for `NWC` data format or
       `[batch, in_channels, in_width]` for `NCW` data format.
-    filter: A 3-D `Tensor` with the same type as `value` and shape
+    filters: A 3-D `Tensor` with the same type as `value` and shape
       `[filter_width, output_channels, in_channels]`.  `filter`'s
       `in_channels` dimension must match that of `value`.
     output_shape: A 1-D `Tensor`, containing three elements, representing the
       output shape of the deconvolution op.
-    stride: An `integer`.  The number of entries by which
-      the filter is moved right at each step.
+    strides: An int or list of `ints` that has length `1` or `3`.  The number of
+      entries by which the filter is moved right at each step.
     padding: A string, either `'VALID'` or `'SAME'`. The padding algorithm.
       See the "returns" section of `tf.nn.convolution` for details.
     data_format: A string. `'NWC'` and `'NCW'` are supported.
+    dilations: An int or list of `ints` that has length `1` or `3` which
+      defaults to 1. The dilation factor for each dimension of input. If set to
+      k > 1, there will be k-1 skipped cells between each filter element on that
+      dimension. Dilations in the batch and depth dimensions must be 1.
     name: Optional name for the returned tensor.
 
   Returns:
@@ -4242,64 +4248,38 @@ def conv1d_transpose(
       `'VALID'` or `'SAME'`, or if `data_format` is invalid.
   """
   with ops.name_scope(name, "conv1d_transpose",
-                      [value, filter, output_shape]) as name:
-    output_shape_ = ops.convert_to_tensor(output_shape, name="output_shape")
-    if not output_shape_.get_shape().is_compatible_with(tensor_shape.vector(3)):
-      raise ValueError("output_shape must have shape (3,), got {}".format(
-          output_shape_.get_shape()))
-
+                      [input, filters, output_shape]) as name:
     # The format could be either NWC or NCW, map to NHWC or NCHW
     if data_format is None or data_format == "NWC":
-      data_format_2d = "NHWC"
-      axis = 2
+      data_format = "NHWC"
+      spatial_start_dim = 1
+      channel_index = 2
     elif data_format == "NCW":
-      data_format_2d = "NCHW"
-      axis = 1
+      data_format = "NCHW"
+      spatial_start_dim = 2
+      channel_index = 1
     else:
       raise ValueError("data_format must be \"NWC\" or \"NCW\".")
 
-    if not value.get_shape().dims[axis].is_compatible_with(
-        filter.get_shape()[2]):
-      raise ValueError("input channels does not match filter's input channels, "
-                       "{} != {}".format(value.get_shape()[axis],
-                                         filter.get_shape()[2]))
-
-    if isinstance(output_shape, (list, np.ndarray)):
-      # output_shape's shape should be == [3] if reached this point.
-      if not filter.get_shape().dims[1].is_compatible_with(
-          output_shape[axis]):
-        raise ValueError(
-            "output_shape does not match filter's output channels, "
-            "{} != {}".format(output_shape[axis],
-                              filter.get_shape()[1]))
-
-    if padding != "VALID" and padding != "SAME":
-      raise ValueError("padding must be either VALID or SAME:"
-                       " {}".format(padding))
-
     # Reshape the input tensor to [batch, 1, in_width, in_channels]
-    if data_format_2d == "NHWC":
-      output_shape_ = array_ops.concat(
-          [output_shape_[:1], [1], output_shape_[1:]], axis=0)
-      spatial_start_dim = 1
-      strides = [1, 1, stride, 1]
-    else:
-      output_shape_ = array_ops.concat(
-          [output_shape_[:2], [1], output_shape_[2:]], axis=0)
-      spatial_start_dim = 2
-      strides = [1, 1, 1, stride]
-    value = array_ops.expand_dims(value, spatial_start_dim)
-    filter = array_ops.expand_dims(filter, 0)  # pylint: disable=redefined-builtin
+    strides = [1] + _get_sequence(strides, 1, channel_index, "stride")
+    dilations = [1] + _get_sequence(dilations, 1, channel_index, "dilations")
+
+    input = array_ops.expand_dims(input, spatial_start_dim)
+    filters = array_ops.expand_dims(filters, 0)  # pylint: disable=redefined-builtin
+    output_shape = output_shape[: spatial_start_dim] + [1] + \
+                   output_shape[spatial_start_dim:]
 
     result = gen_nn_ops.conv2d_backprop_input(
-        input_sizes=output_shape_,
-        filter=filter,
-        out_backprop=value,
+        input_sizes=output_shape,
+        filter=filters,
+        out_backprop=input,
         strides=strides,
         padding=padding,
-        data_format=data_format_2d,
+        data_format=data_format,
+        dilations=dilations,
         name=name)
-    return array_ops.squeeze(result, [spatial_start_dim])
+    return array_ops.squeeze(result, spatial_start_dim)
 
 
 @ops.RegisterStatistics("Dilation2D", "flops")
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
index d68db966fd..c7061a7f79 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
@@ -56,6 +56,10 @@ tf_module {
     name: "conv1d"
     argspec: "args=[\'value\', \'filters\', \'stride\', \'padding\', \'use_cudnn_on_gpu\', \'data_format\', \'name\', \'input\', \'dilations\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "conv1d_transpose"
+    argspec: "args=[\'input\', \'filters\', \'output_shape\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'SAME\', \'NWC\', \'None\', \'None\'], "
+  }
   member_method {
     name: "conv2d"
     argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'use_cudnn_on_gpu\', \'data_format\', \'dilations\', \'name\', \'filters\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'NHWC\', \'[1, 1, 1, 1]\', \'None\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index 6cba569bf3..24ed1d0e43 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -52,6 +52,10 @@ tf_module {
     name: "conv1d"
     argspec: "args=[\'input\', \'filters\', \'stride\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NWC\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "conv1d_transpose"
+    argspec: "args=[\'input\', \'filters\', \'output_shape\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'SAME\', \'NWC\', \'None\', \'None\'], "
+  }
   member_method {
     name: "conv2d"
     argspec: "args=[\'input\', \'filters\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'None\', \'None\'], "
-- 
GitLab


From 294858ccb2a5cd44072378afd6d18cc61ab83e76 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Fri, 8 Feb 2019 07:33:39 -0800
Subject: [PATCH 2147/2345] [XLA:Python] Fix missing init_pywrap_xla symbol on
 Python 2.7 on Mac.

JAX bug: https://github.com/google/jax/issues/342

PiperOrigin-RevId: 233057401
---
 tensorflow/compiler/xla/python/pywrap_xla_exported_symbols.lds | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/compiler/xla/python/pywrap_xla_exported_symbols.lds b/tensorflow/compiler/xla/python/pywrap_xla_exported_symbols.lds
index bce6c1acf8..ef77ed3d95 100644
--- a/tensorflow/compiler/xla/python/pywrap_xla_exported_symbols.lds
+++ b/tensorflow/compiler/xla/python/pywrap_xla_exported_symbols.lds
@@ -1 +1,2 @@
 _PyInit__pywrap_xla
+_init_pywrap_xla
-- 
GitLab


From ede109fc7bf32f7a7ef2763908b2606cfa4fa303 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 8 Feb 2019 07:39:02 -0800
Subject: [PATCH 2148/2345] [convergence_tools]: tensor tracer interface
 updates.

PiperOrigin-RevId: 233058179
---
 .../contrib/tpu/python/tpu/tensor_tracer.py   | 346 ++++++++++--------
 .../contrib/tpu/python/tpu/tpu_estimator.py   |  25 +-
 2 files changed, 200 insertions(+), 171 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py b/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py
index 2c5ea65182..8c25c98c3b 100644
--- a/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py
+++ b/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py
@@ -114,6 +114,7 @@ _TENSOR_TRACER_STORAGE = 'tensor_tracer_storage'
 _TENSOR_VALUES_CACHE = 'tensor_values_cache'
 _REPLICA_ID_TAG = '#replica-id: '
 
+
 def tensor_tracepoint(tensor, checkpoint_name):
   """Adds a checkpoint with the given checkpoint name for the given tensor.
 
@@ -200,7 +201,6 @@ def _get_tensor_values_cache(graph=None):
 
 def _create_tensor_values_cache(graph, num_tensors):
   """Creates a variable as the cache to store intermediate tensor values."""
-
   graph = graph or ops.get_default_graph()
   # Create in proper graph and base name_scope.
   with graph.as_default() as g, g.name_scope(None):
@@ -215,19 +215,6 @@ def _create_tensor_values_cache(graph, num_tensors):
         collections=[_TENSOR_TRACER_STORAGE, ops.GraphKeys.GLOBAL_VARIABLES])
 
 
-def _set_fetches(result_tensor, train_op):
-  """Sets the fetches from the result tensor and training op."""
-
-  fetches = []
-  if result_tensor is not None:
-    fetches.append(result_tensor)
-  if train_op is not None:
-    fetches.append(train_op)
-  if not fetches:
-    return None
-  return fetches
-
-
 class TensorTracer(object):
   """A software construct for tracing tensor values in a TF graph on TPU.
 
@@ -629,23 +616,17 @@ class TensorTracer(object):
     self._num_hosts = None
     self._replica_id = None
 
-  def _add_replica_id_to_graph(self, result_tensor):
+  def _add_replica_id_to_graph(self):
     """Adds nodes for computing the replica ID to the graph."""
 
-    if not self._num_replicas:
+    if self._num_replicas:
+      with ops.control_dependencies(None):
+        # Uses None as dependency to run outside of TPU graph rewrites.
+        self._replica_id = tpu_ops.tpu_replicated_input(
+            list(range(self._num_replicas)),
+            name='tt_replica_id')
+    else:
       self._replica_id = 'unknown'
-      return result_tensor
-
-    with ops.control_dependencies(None):
-      # Uses None as dependency to run outside of TPU graph rewrites.
-      self._replica_id = tpu_ops.tpu_replicated_input(
-          list(range(self._num_replicas)),
-          name='tt_replica_id')
-    use_replica_id = array_ops.identity(self._replica_id).op
-    with ops.control_dependencies([use_replica_id]):
-      # Adds a control dependency from the result_tensor to
-      # the replica_id to ensure that replica_id will be added to the graph.
-      return array_ops.identity(result_tensor)
 
   def _set_trace_dir(self):
     found, self._trace_dir = TensorTracer.get_flag_value(_FLAG_NAME_TRACE_DIR)
@@ -955,7 +936,6 @@ class TensorTracer(object):
       with ops.control_dependencies([print_op]):
         return array_ops.identity(tensor).op
 
-
     def _show_part_tensor(tensor):
       """Trace function for printing part of the tensor."""
 
@@ -1104,16 +1084,12 @@ class TensorTracer(object):
           traverse_stack.append(input_op)
     return execution_path_operations
 
-  def _determine_traced_tensors(self, graph, fetches):
+  def _determine_traced_tensors(self, graph, ops_in_exec_path):
     """Determines the tensors that will be traced."""
 
     self._traced_tensorname_to_cache_idx_map = {}
     self._cache_idx_to_tensor_idx = []
     operations = graph.get_operations()
-    # Filter out the operations that won't be executed.
-    # if fetches=None, then ops_in_exec_path = set(operations)
-    ops_in_exec_path = self._filter_execution_path_operations(operations,
-                                                              fetches)
     checkpoint_operations = self._get_checkpoints(graph)
     for op_id, op in enumerate(operations):
       if checkpoint_operations and op.name not in checkpoint_operations:
@@ -1176,14 +1152,18 @@ class TensorTracer(object):
     self._write_config_section()
     self._write_op_list_section(operations)
     self._write_tensor_list_section(tensor_list, opname_idx_map)
-    self._determine_traced_tensors(graph, fetches)
+    # Filter out the operations that won't be executed.
+    # if fetches=None, then ops_in_exec_path = set(operations)
+    ops_in_exec_path = self._filter_execution_path_operations(operations,
+                                                              fetches)
+    self._determine_traced_tensors(graph, ops_in_exec_path)
     self._write_cache_index_map_section()
     # Does the topological sort before adding any nodes to the graph.
     (succeed, sorted_or_cycle) = TensorTracer.topological_sort(graph)
     if self._use_tensor_values_cache():
       _create_tensor_values_cache(graph,
                                   len(self._cache_idx_to_tensor_idx))
-    return (operations, succeed, sorted_or_cycle)
+    return (ops_in_exec_path, succeed, sorted_or_cycle)
 
   def _post_tracing(self, succeed, sorted_or_cycle):
     """Work needs to be done after TPU or CPU tracing."""
@@ -1285,60 +1265,108 @@ class TensorTracer(object):
                                  default=_do_nothing,
                                  exclusive=True).op
 
-  def _flush_tensor_values_cache(self, graph, result_tensor, train_op, on_tpu):
+  def _flush_tensor_values_cache(self, graph, tensor_fetches, op_fetches,
+                                 on_tpu):
     """Flushes the intermediate tensor values in the graph to the cache.
 
     Args:
       graph: the graph of Ops
-      result_tensor: a result tensor of evaluating the graph.
-      train_op: the training op.
+      tensor_fetches: list of tensor results returned by the model_fn.
+      op_fetches: list of ops that are returned by the model_fn, e.g., train_op.
       on_tpu: if the graph is executed on TPU.
 
     Returns:
-      An identical copy of result tensor.
+      An identical copy of tensor_fetches.
     """
-
-    train_op_list = []
-    if train_op is not None:
-      train_op_list.append(train_op)
-    with ops.control_dependencies(train_op_list):
+    # Add a dependency to op and tensor fetches to make sure that all tracing
+    # ops are executed before flushing trace results.
+    with ops.control_dependencies(op_fetches +
+                                  [tensor.op for tensor in tensor_fetches]):
       flush_cache_op_list = []
       for host in range(self._num_hosts):
         start_replica = host * 8
         flush_op = self._generate_flush_cache_op(graph, start_replica, on_tpu)
         flush_cache_op_list.append(flush_op)
-      with ops.control_dependencies(flush_cache_op_list):
-        return array_ops.identity(result_tensor)
+      return control_flow_ops.tuple(tensor_fetches,
+                                    control_inputs=flush_cache_op_list)
+
+  def _process_tensor_fetches(self, tensor_fetches):
+    """Check that tensor_fetches is not empty and have valid tensors."""
+    # If none or empty list.
+    if tensor_fetches is None:
+      raise RuntimeError('tensor_fetches provided to tensor_tracer cannot be '
+                         'None.')
+    if not isinstance(tensor_fetches, (list, tuple)):
+      tensor_fetches = [tensor_fetches]
+    elif not tensor_fetches:
+      raise RuntimeError('tensor_fetches provided to tensor_tracer cannot be '
+                         'empty list.')
+    fetches = []
+    for fetch in tensor_fetches:
+      if isinstance(fetch, ops.Tensor):
+        fetches.append(fetch)
+      else:
+        raise RuntimeError('Given tensor_fetch:%s is not a tensor.' % fetch)
+    return fetches
+
+  def _process_op_fetches(self, op_fetches):
+    """Check that op_fetches have valid ops."""
+    if op_fetches is None:
+      return []
+
+    if not isinstance(op_fetches, (list, tuple)):
+      op_fetches = [op_fetches]
+
+    fetches = []
+    for fetch in op_fetches:
+      if isinstance(fetch, ops.Operation):
+        fetches.append(fetch)
+      else:
+        logging.warning('Ignoring the given op_fetch:%s, which is not an op.' %
+                        fetch)
+    return fetches
+
+  def _convert_fetches_to_input_format(self, input_fetches, current_fetches):
+    """Changes current_fetches' format, so that it matches input_fetches."""
+    if isinstance(input_fetches, ops.Tensor):
+      if len(current_fetches) != 1:
+        raise RuntimeError('Tensor tracer input/output fetches do not match.')
+      return current_fetches[0]
+    else:
+      if len(current_fetches) != len(current_fetches):
+        raise RuntimeError('Tensor tracer input/output fetches do not match.')
+      elif isinstance(input_fetches, tuple):
+        return tuple(current_fetches)
+      else:
+        return current_fetches
+
+  def _trace_execution(self, graph,
+                       tensor_fetches,
+                       op_fetches=None,
+                       on_tpu=True):
+    """Commong tracing function for both CPU and TPUs.
+
+    The caller function should set _device_type, _num_replicas,
+    _num_replicas_per_host, _num_hosts and _replica_id before calling
+    _trace_execution.
 
-  def trace_tpu(self, graph,
-                result_tensor,
-                train_op,
-                num_replicas=None,
-                num_replicas_per_host=None,
-                num_hosts=None):
-    """Traces the tensors generated by TPU Ops in a TF graph.
 
     Args:
       graph: the graph of Ops executed on the TPU.
-      result_tensor: a result tensor of evaluating the graph.
-      train_op: the training op.
-      num_replicas: number of replicas used on the TPU.
-      num_replicas_per_host: number of replicas per TPU host.
-      num_hosts: total number of TPU hosts.
+      tensor_fetches: a (list,tuple,or a single object) of tensor fetches
+        returned by model_fn given to session.run. Function must be provided
+        with as least one tensor to fetch.
+      op_fetches: A list of op fetches returned by model_fn given to
+        session.run. op_fetches and tensor_fetches are used to determine the
+        nodes that will be executed. Can be None.
+      on_tpu: True if executing on TPU.
 
     Returns:
-      A tuple (result_tensor_copy, tracing_ops), where:
-        result_tensor_copy: an exact copy of result_tensor
-        tracing_ops: a list of tracing ops. If this list
-                     is non empty, the caller of this function
-                     should pose control dependencies upon these
-                     Ops so that they will be executed when the
-                     graph is evaluated.
-
+      tensor_fetches: an exact copy of tensor_fetches that has additional
+                      dependencies.
     Raises:
-      RuntimeError: If num_replicas_per_host > 8.
+      RuntimeError: If tensor_fetches is None or empty.
     """
-
     def _cast_unsupported_dtypes(tensor):
       """Casts tensor to a supported type."""
 
@@ -1351,23 +1379,20 @@ class TensorTracer(object):
         return math_ops.cast(tensor, dtypes.float32)
       return tensor
 
-    self._device_type = _DEVICE_TYPE_TPU
-    self._num_replicas = num_replicas
-    self._num_replicas_per_host = num_replicas_per_host
-    self._num_hosts = num_hosts
-    if self._num_replicas_per_host > 8:
-      # Checks for the assumption in _generate_flush_cache_op().
-      raise RuntimeError(
-          'num_replicas_per_host (%d) is '
-          'greater than 8'%self._num_replicas_per_host)
-
     TensorTracer.check_device_type(self._device_type)
-    result_tensor_copy = self._add_replica_id_to_graph(result_tensor)
-    fetches = _set_fetches(result_tensor, train_op)
-    (operations, succeed, sorted_or_cycle) = self._pre_tracing(graph, fetches)
+    # Check in_tensor_fetches, and op_fetches and convert them to lists.
+    processed_t_fetches = self._process_tensor_fetches(tensor_fetches)
+    op_fetches = self._process_op_fetches(op_fetches)
+    all_fetches = op_fetches + [tensor.op for tensor in processed_t_fetches]
+
+    # Filter the set of ops that will be executed, and topological sort.
+    (exec_op_set, succeed, sorted_or_cycle) = self._pre_tracing(graph,
+                                                                all_fetches)
 
+    tensor_fetch_set = set(processed_t_fetches)
     tracing_ops = []
-    for op in operations:
+    # Trace ops only if they are in the execution path.
+    for op in exec_op_set:
       for i in range(len(op.outputs)):
         out_tensor = op.outputs[i]
         tensor_name = out_tensor.name
@@ -1377,105 +1402,118 @@ class TensorTracer(object):
         # Otherwise, adding control input below, will introduce a cycle in the
         # graph.
         consumers = out_tensor.consumers()
-        if not consumers:
+        # Not all consumers may be in the exec path. Filter out the consumers
+        # to keep the graph simpler.
+        consumers = [cop for cop in consumers if cop in exec_op_set]
+
+        # If there is no consumer of the tensor, there is no need to trace it;
+        # unless the tensor itself is one of the fetches.
+        is_a_fetched_tensor = out_tensor in tensor_fetch_set
+        if (not consumers) and (not is_a_fetched_tensor):
           continue
         processed_out_tensor = self._preprocess_traced_tensor(out_tensor)
-        processed_out_tensor = _cast_unsupported_dtypes(processed_out_tensor)
+
+        if on_tpu:
+          processed_out_tensor = _cast_unsupported_dtypes(processed_out_tensor)
+
         if self._use_tensor_values_cache():
           cache_idx = self._traced_tensorname_to_cache_idx_map[tensor_name]
           trace_op = self._save_tensor_value_to_cache_op(graph,
                                                          cache_idx,
                                                          processed_out_tensor)
-        else:
+        elif on_tpu:
           trace_op = tpu.outside_compilation(
               self._make_tensor_trace_fun(tensor_name), processed_out_tensor)
+        else:
+          trace_fun = self._make_tensor_trace_fun(tensor_name)
+          traced_tensor = trace_fun(processed_out_tensor)
+          trace_op = traced_tensor.op
+
+        if is_a_fetched_tensor:
+          tracing_ops.append(trace_op)
+          continue
+        # Add it to all consumers, as some consumers may not be executed if they
+        # are in a control flow.
         for consumer_op in consumers:
           # pylint: disable=protected-access
           consumer_op._add_control_input(trace_op)
           # pylint: enable=protected-access
+    if tracing_ops:
+      # If we are tracing a fetched tensor, their dependency is stored in
+      # tracing_ops.
+      processed_t_fetches = control_flow_ops.tuple(processed_t_fetches,
+                                                   control_inputs=tracing_ops)
     if self._use_tensor_values_cache():
-      result_tensor_final = self._flush_tensor_values_cache(graph,
-                                                            result_tensor_copy,
-                                                            train_op,
+      processed_t_fetches = self._flush_tensor_values_cache(graph,
+                                                            processed_t_fetches,
+                                                            op_fetches,
                                                             on_tpu=True)
-    else:
-      result_tensor_final = result_tensor_copy
     self._post_tracing(succeed, sorted_or_cycle)
-    return (result_tensor_final, tracing_ops)
+    # processed_t_fetches is a list at this point. Convert it to the same
+    # format as given in tensor_fetches.
+    return self._convert_fetches_to_input_format(tensor_fetches,
+                                                 processed_t_fetches)
 
-  def _generate_cpu_result(self, result_tensor, train_op, graph):
-    """Generates the final CPU result."""
+  def trace_tpu(self, graph,
+                tensor_fetches,
+                op_fetches=None,
+                num_replicas=None,
+                num_replicas_per_host=None,
+                num_hosts=None):
+    """Traces the tensors generated by TPU Ops in a TF graph.
 
-    if self._use_tensor_values_cache():
-      result_tensor_final = self._flush_tensor_values_cache(graph,
-                                                            result_tensor,
-                                                            train_op,
-                                                            on_tpu=False)
-    else:
-      result_tensor_final = array_ops.identity(result_tensor)
-    return result_tensor_final
+    Args:
+      graph: the graph of Ops executed on the TPU.
+      tensor_fetches: a (list,tuple,or a single object) of tensor fetches
+        returned by model_fn given to session.run. Function must be provided
+        with as least one tensor to fetch.
+      op_fetches: A list of op fetches returned by model_fn given to
+        session.run. op_fetches and tensor_fetches are used to determine the
+        nodes that will be executed. Can be None.
+      num_replicas: number of replicas used on the TPU.
+      num_replicas_per_host: number of replicas per TPU host.
+      num_hosts: total number of TPU hosts.
+
+    Returns:
+      tensor_fetches: an exact copy of tensor_fetches that has additional
+                      dependencies.
+    Raises:
+      RuntimeError: If num_replicas_per_host > 8.
+      RuntimeError: If tensor_fetches is None or empty.
+    """
+
+    self._device_type = _DEVICE_TYPE_TPU
+    self._num_replicas = num_replicas
+    self._num_replicas_per_host = num_replicas_per_host
+    self._num_hosts = num_hosts
+    if self._num_replicas_per_host > 8:
+      # Checks for the assumption in _generate_flush_cache_op().
+      raise RuntimeError('num_replicas_per_host (%d) is '
+                         'greater than 8'%self._num_replicas_per_host)
+    self._add_replica_id_to_graph()
+    return self._trace_execution(graph, tensor_fetches, op_fetches, on_tpu=True)
 
-  def trace_cpu(self, graph, result_tensor, train_op):
+  def trace_cpu(self, graph, tensor_fetches, op_fetches=None):
     """Traces the tensors generated by CPU Ops in a TF graph.
 
     Args:
       graph: the graph of Ops executed on the CPU.
-      result_tensor: a result tensor of evaluating the graph.
-      train_op: the training op.
+      tensor_fetches: a (list,tuple,or a single object) of tensor fetches
+        returned by model_fn given to session.run. Function must be provided
+        with as least one tensor to fetch.
+      op_fetches: A list of op fetches returned by model_fn given to
+        session.run. op_fetches and tensor_fetches are used to determine the
+        nodes that will be executed. Can be None.
 
     Returns:
-      A pair (final_result_tensor, tracing_calls) where:
-         final_result_tensor: an identical copy of result_tensor.
-         tracing_calls: a map from keys to trace calls.
-                     A key is constructed from an Op's name.
-                     A trace call consists of a function and a tensor (
-                     the function will be invoked with the tensor).
+      tensor_fetches: an exact copy of tensor_fetches that has additional
+                      dependencies.
+    Raises:
+      RuntimeError: If tensor_fetches is None or empty.
     """
-
-    if result_tensor is None:
-      raise ValueError(
-          'The result_tensor passed to trace_cpu should not be None')
-
     self._device_type = _DEVICE_TYPE_CPU
-    TensorTracer.check_device_type(self._device_type)
     self._num_replicas = 1
     self._num_replicas_per_host = 1
     self._num_hosts = 1
     self._replica_id = 0
-    fetches = _set_fetches(result_tensor, train_op)
-    (operations, succeed, sorted_or_cycle) = self._pre_tracing(graph, fetches)
-
-    tracing_calls = {}
-    for op in operations:
-      for i in range(len(op.outputs)):
-        out_tensor = op.outputs[i]
-        tensor_name = out_tensor.name
-        if tensor_name not in self._traced_tensorname_to_cache_idx_map:
-          continue
-        # Create the list of consumers before calling _preprocess_traced_tensor.
-        # Otherwise, adding control input below, will introduce a cycle in the
-        # graph.
-        consumers = out_tensor.consumers()
-        if not consumers:
-          continue
-        processed_out_tensor = self._preprocess_traced_tensor(out_tensor)
-        if self._use_tensor_values_cache():
-          cache_idx = self._traced_tensorname_to_cache_idx_map[tensor_name]
-          trace_op = self._save_tensor_value_to_cache_op(graph,
-                                                         cache_idx,
-                                                         processed_out_tensor)
-          for consumer_op in consumers:
-            # pylint: disable=protected-access
-            consumer_op._add_control_input(trace_op)
-            # pylint: enable=protected-access
-        else:
-          trace_fun = self._make_tensor_trace_fun(tensor_name)
-          trace_call = (trace_fun, [processed_out_tensor])
-          trace_call_key = 'tensor_tracing_cpu-%s:%d'%(op.name, i)
-          tracing_calls[trace_call_key] = trace_call
-
-    self._post_tracing(succeed, sorted_or_cycle)
-    final_result_tensor = self._generate_cpu_result(result_tensor,
-                                                    train_op,
-                                                    graph)
-    return (final_result_tensor, tracing_calls)
+    return self._trace_execution(graph, tensor_fetches, op_fetches, on_tpu=True)
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 41e2d1dd10..80adfa1244 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -358,15 +358,8 @@ class TPUEstimatorSpec(model_fn_lib._TPUEstimatorSpec):  # pylint: disable=prote
     if tensor_tracer.TensorTracer.is_enabled() \
        and self.train_op is not None:
       tt = tensor_tracer.TensorTracer()
-      (loss, tracing_calls) = tt.trace_cpu(ops.get_default_graph(),
-                                           loss, self.train_op)
-      tracing_call_ret = _OutfeedHostCall.create_cpu_hostcall(tracing_calls)
-      tracing_functions = tracing_call_ret.values()
-      if tracing_functions:
-        if hooks:
-          hooks.extend([_OutfeedHostCallHook(tracing_functions)])
-        else:
-          hooks = [_OutfeedHostCallHook(tracing_functions)]
+      loss = tt.trace_cpu(ops.get_default_graph(), loss, self.train_op)
+
     hooks = tuple(hooks or [])
     scaffold = self.scaffold_fn() if self.scaffold_fn else None
     return model_fn_lib.EstimatorSpec(
@@ -1454,14 +1447,13 @@ class _ModelFnWrapper(object):
 
       captured_training_hooks.capture(estimator_spec.training_hooks)
 
-      tracing_ops = []
       if tensor_tracer.TensorTracer.is_enabled():
         tt = tensor_tracer.TensorTracer()
-        loss, tracing_ops = tt.trace_tpu(ops.get_default_graph(),
-                                         loss, train_op,
-                                         self._ctx.num_replicas,
-                                         self._ctx.num_of_replicas_per_host,
-                                         self._ctx.num_hosts)
+        loss = tt.trace_tpu(ops.get_default_graph(),
+                            loss, train_op,
+                            self._ctx.num_replicas,
+                            self._ctx.num_of_replicas_per_host,
+                            self._ctx.num_hosts)
 
       if self._ctx.embedding_config is None:
         apply_sparse_grads = []
@@ -1471,8 +1463,7 @@ class _ModelFnWrapper(object):
 
       # We must run train_op to update the variables prior to running the
       # outfeed.
-      with ops.control_dependencies([train_op] + tracing_ops +
-                                    apply_sparse_grads):
+      with ops.control_dependencies([train_op] + apply_sparse_grads):
         host_call_outfeed_ops = []
         if (isinstance(estimator_spec, model_fn_lib._TPUEstimatorSpec)  # pylint: disable=protected-access
             and estimator_spec.host_call is not None):
-- 
GitLab


From 82cae29a5e82a241c5704998bd19457c56dbcee3 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Fri, 8 Feb 2019 07:47:23 -0800
Subject: [PATCH 2149/2345] Allow string leaves in the AST; they are not very
 frequent.

PiperOrigin-RevId: 233059092
---
 tensorflow/python/autograph/pyct/ast_util.py      | 11 ++++++++---
 tensorflow/python/autograph/pyct/ast_util_test.py | 15 ++++++++++++---
 2 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/autograph/pyct/ast_util.py b/tensorflow/python/autograph/pyct/ast_util.py
index 3496c514ee..b091285cab 100644
--- a/tensorflow/python/autograph/pyct/ast_util.py
+++ b/tensorflow/python/autograph/pyct/ast_util.py
@@ -283,13 +283,18 @@ def parallel_walk(node, other):
     n = node_stack.pop()
     o = other_stack.pop()
 
-    if (not isinstance(n, (ast.AST, gast.AST)) or
-        not isinstance(o, (ast.AST, gast.AST)) or
+    if (not isinstance(n, (ast.AST, gast.AST, str)) or
+        not isinstance(o, (ast.AST, gast.AST, str)) or
         n.__class__.__name__ != o.__class__.__name__):
-      raise ValueError('inconsistent nodes: {} and {}'.format(n, o))
+      raise ValueError('inconsistent nodes: {} ({}) and {} ({})'.format(
+          n, n.__class__.__name__, o, o.__class__.__name__))
 
     yield n, o
 
+    if isinstance(n, str):
+      assert isinstance(o, str), 'The check above should have ensured this'
+      continue
+
     for f in n._fields:
       n_child = getattr(n, f, None)
       o_child = getattr(o, f, None)
diff --git a/tensorflow/python/autograph/pyct/ast_util_test.py b/tensorflow/python/autograph/pyct/ast_util_test.py
index 00ef7f5e05..c6c1132dd6 100644
--- a/tensorflow/python/autograph/pyct/ast_util_test.py
+++ b/tensorflow/python/autograph/pyct/ast_util_test.py
@@ -159,11 +159,20 @@ class AstUtilTest(test.TestCase):
     })
 
   def test_parallel_walk(self):
-    node = parser.parse_str(
-        textwrap.dedent("""
+    src = """
       def f(a):
         return a + 1
-    """))
+    """
+    node = parser.parse_str(textwrap.dedent(src))
+    for child_a, child_b in ast_util.parallel_walk(node, node):
+      self.assertEqual(child_a, child_b)
+
+  def test_parallel_walk_string_leaves(self):
+    src = """
+      def f(a):
+        global g
+    """
+    node = parser.parse_str(textwrap.dedent(src))
     for child_a, child_b in ast_util.parallel_walk(node, node):
       self.assertEqual(child_a, child_b)
 
-- 
GitLab


From e3b41eb1b26aa9e4d668f6df5408743679c20f67 Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Fri, 8 Feb 2019 07:57:24 -0800
Subject: [PATCH 2150/2345] Automated rollback of commit
 1f856fb5d978177123ddd5ac5a3e4bb669288d65

PiperOrigin-RevId: 233060179
---
 tensorflow/python/keras/backend.py |   4 +-
 tensorflow/python/ops/nn_ops.py    | 121 ++++++++++++++++++++++-------
 tensorflow/python/ops/nn_test.py   |  16 ++++
 3 files changed, 110 insertions(+), 31 deletions(-)

diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index 37e605ea2c..d09d959b41 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -4143,8 +4143,8 @@ def conv1d(x,
   x = nn.convolution(
       input=x,
       filter=kernel,
-      dilation_rate=(dilation_rate,),
-      strides=(strides,),
+      dilation_rate=dilation_rate,
+      strides=strides,
       padding=padding,
       data_format=tf_data_format)
   if data_format == 'channels_first' and tf_data_format == 'NWC':
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index bc9e404d16..967b29f17e 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -65,7 +65,9 @@ def _get_sequence(value, n, channel_index, name):
     return value
   elif current_n == 1:
     value = list((value[0],) * n)
-  elif current_n != n:
+  elif current_n == n:
+    value = list(value)
+  else:
     raise ValueError("{} should be of length 1, {} or {} but was {}".format(
         name, n, n + 2, current_n))
 
@@ -881,21 +883,14 @@ def convolution(
   filter = deprecated_argument_lookup("filters", filters, "filter", filter)
   dilation_rate = deprecated_argument_lookup(
       "dilations", dilations, "dilation_rate", dilation_rate)
-  # pylint: enable=line-too-long
-  with ops.name_scope(name, "convolution", [input, filter]) as name:
-    input = ops.convert_to_tensor(input, name="input")  # pylint: disable=redefined-builtin
-    input_shape = input.get_shape()
-    filter = ops.convert_to_tensor(filter, name="filter")  # pylint: disable=redefined-builtin
-    filter_shape = filter.get_shape()
-    op = Convolution(
-        input_shape,
-        filter_shape,
-        padding,
-        strides=strides,
-        dilation_rate=dilation_rate,
-        name=name,
-        data_format=data_format)
-    return op(input, filter)
+  return convolution_internal(
+      input,
+      filter,
+      strides=strides,
+      padding=padding,
+      data_format=data_format,
+      dilations=dilation_rate,
+      name=name)
 
 
 @tf_export("nn.convolution", v1=[])
@@ -907,14 +902,15 @@ def convolution_v2(
     data_format=None,
     dilations=None,
     name=None):
-  return convolution(
+  return convolution_internal(
       input,  # pylint: disable=redefined-builtin
       filters,
-      padding=padding,
       strides=strides,
-      dilation_rate=dilations,
-      name=name,
-      data_format=data_format)
+      padding=padding,
+      data_format=data_format,
+      dilations=dilations,
+      name=name)
+
 
 convolution_v2.__doc__ = deprecation.rewrite_argument_docstring(
     deprecation.rewrite_argument_docstring(
@@ -922,6 +918,75 @@ convolution_v2.__doc__ = deprecation.rewrite_argument_docstring(
     "filter", "filters")
 
 
+def convolution_internal(
+    input,  # pylint: disable=redefined-builtin
+    filters,
+    strides=None,
+    padding="VALID",
+    data_format=None,
+    dilations=None,
+    name=None):
+  """Internal function which performs rank agnostic convolution."""
+  with ops.name_scope(name, "convolution", [input, filters]) as name:
+    if isinstance(input.shape, tensor_shape.TensorShapeV1) and \
+        input.shape.rank is not None:
+      n = len(input.shape) - 2
+    elif not isinstance(input.shape, tensor_shape.TensorShapeV1) and \
+        input.shape is not None:
+      n = len(input.shape) - 2
+    elif isinstance(filters.shape, tensor_shape.TensorShapeV1) and \
+        filters.shape.rank is not None:
+      n = len(filters.shape) - 2
+    elif not isinstance(filters.shape, tensor_shape.TensorShapeV1) and \
+        filters.shape is not None:
+      n = len(filters.shape) - 2
+    else:
+      raise ValueError("rank of input or filter must be known")
+
+    if n < 1 or n > 3:
+      raise ValueError(
+          "Input tensor must be of rank 3, 4 or 5 but was {}.".format(n + 2))
+
+    if data_format is None:
+      channel_index = n + 1
+    else:
+      channel_index = 1 if data_format.startswith("NC") else n + 1
+
+    strides = _get_sequence(strides, n, channel_index, "strides")
+    dilations = _get_sequence(dilations, n, channel_index, "dilations")
+
+    conv_ops = {1: conv1d, 2: gen_nn_ops.conv2d, 3: gen_nn_ops.conv3d}
+
+    if all(i == 1 for i in dilations):
+      # fast path if no dilation as gradient only supported on GPU for dilations
+      op = conv_ops.get(n)
+      return op(
+          input,
+          filters,
+          strides,
+          padding=padding,
+          data_format=data_format,
+          dilations=dilations,
+          name=name)
+    else:
+      if channel_index == 1:
+        strides = strides[2:]
+        dilations = dilations[2:]
+      else:
+        strides = strides[1:-1]
+        dilations = dilations[1:-1]
+
+      op = Convolution(
+          tensor_shape.as_shape(input.shape),
+          tensor_shape.as_shape(filters.shape),
+          padding,
+          strides=strides,
+          dilation_rate=dilations,
+          name=name,
+          data_format=data_format)
+      return op(input, filters)
+
+
 class Convolution(object):
   """Helper class for convolution.
 
@@ -4089,10 +4154,9 @@ def conv1d(
       entries by which the filter is moved right at each step.
     padding: 'SAME' or 'VALID'
     use_cudnn_on_gpu: An optional `bool`.  Defaults to `True`.
-    data_format: An optional `string` from `"NWC", "NCW"`.  Defaults
-      to `"NWC"`, the data is stored in the order of
-      [batch, in_width, in_channels].  The `"NCW"` format stores
-      data as [batch, in_channels, in_width].
+    data_format: An optional `string` from `"NWC", "NCW"`.  Defaults to `"NWC"`,
+      the data is stored in the order of [batch, in_width, in_channels].  The
+      `"NCW"` format stores data as [batch, in_channels, in_width].
     name: A name for the operation (optional).
     input: Alias for value.
     dilations: An int or list of `ints` that has length `1` or `3` which
@@ -4175,10 +4239,9 @@ def conv1d_v2(
     stride: An int or list of `ints` that has length `1` or `3`.  The number of
       entries by which the filter is moved right at each step.
     padding: 'SAME' or 'VALID'
-    data_format: An optional `string` from `"NWC", "NCW"`.  Defaults
-      to `"NWC"`, the data is stored in the order of
-      [batch, in_width, in_channels].  The `"NCW"` format stores
-      data as [batch, in_channels, in_width].
+    data_format: An optional `string` from `"NWC", "NCW"`.  Defaults to `"NWC"`,
+      the data is stored in the order of [batch, in_width, in_channels].  The
+      `"NCW"` format stores data as [batch, in_channels, in_width].
     dilations: An int or list of `ints` that has length `1` or `3` which
       defaults to 1. The dilation factor for each dimension of input. If set to
       k > 1, there will be k-1 skipped cells between each filter element on that
diff --git a/tensorflow/python/ops/nn_test.py b/tensorflow/python/ops/nn_test.py
index 3279dcae0c..a3cdeaa42c 100644
--- a/tensorflow/python/ops/nn_test.py
+++ b/tensorflow/python/ops/nn_test.py
@@ -24,9 +24,11 @@ from absl.testing import parameterized
 import numpy as np
 from six.moves import xrange  # pylint: disable=redefined-builtin
 
+from tensorflow.python.eager import def_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_spec
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
@@ -1317,5 +1319,19 @@ class MaxPoolTest(test_lib.TestCase):
       nn_ops.max_pool_v2(x, 2, 2, "SAME")
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class ConvolutionTest(test_lib.TestCase):
+
+  def testUnknownSize(self):
+    x = tensor_spec.TensorSpec(None, dtypes.float32, name="x")
+    k = np.ones([3, 6, 6, 5])
+
+    @def_function.function
+    def F(value):
+      return nn_ops.convolution(value, k, "SAME")
+
+    F.get_concrete_function(x)
+
+
 if __name__ == "__main__":
   test_lib.main()
-- 
GitLab


From 30244afbb5f41567262d4a10a80636f79f281614 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 8 Feb 2019 08:46:02 -0800
Subject: [PATCH 2151/2345] [XLA:TPU] Implement another layout for batch
 grouped convolutions.

PiperOrigin-RevId: 233066927
---
 tensorflow/compiler/xla/tests/BUILD           |  5 ++
 .../conv_depthwise_backprop_filter_test.cc    | 67 +++++++++++++------
 2 files changed, 50 insertions(+), 22 deletions(-)

diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 9a59371074..db1c927469 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -317,6 +317,11 @@ xla_test(
     name = "conv_depthwise_backprop_filter_test",
     timeout = "long",
     srcs = ["conv_depthwise_backprop_filter_test.cc"],
+    # these backends do not natively handle batch group counts.
+    blacklisted_backends = [
+        "gpu",
+        "cpu",
+    ],
     shard_count = 6,
     deps = [
         "//tensorflow/compiler/xla:execution_options_util",
diff --git a/tensorflow/compiler/xla/tests/conv_depthwise_backprop_filter_test.cc b/tensorflow/compiler/xla/tests/conv_depthwise_backprop_filter_test.cc
index 96f4aedf8b..dfbf0478e6 100644
--- a/tensorflow/compiler/xla/tests/conv_depthwise_backprop_filter_test.cc
+++ b/tensorflow/compiler/xla/tests/conv_depthwise_backprop_filter_test.cc
@@ -37,6 +37,8 @@ struct BatchGroupedConvolution2DSpec {
   std::vector<int64> activation_dims;
   std::vector<int64> kernel_dims;
   std::vector<int64> output_dims;
+  std::vector<int64> activation_and_kernel_layout;
+  std::vector<int64> output_layout;
 };
 
 class BatchGroupedConvolution2DTest
@@ -47,8 +49,9 @@ class BatchGroupedConvolution2DTest
 static std::vector<BatchGroupedConvolution2DSpec> GetConv2DTestCases() {
   std::vector<BatchGroupedConvolution2DSpec> config_set;
   std::vector<std::vector<int64>> config_options = {
-      {8, 5, 3, 2},   {4, 5, 5, 2},   {8, 7, 4, 128},  {16, 20, 20, 256},
-      {256, 7, 5, 4}, {256, 6, 6, 4}, {256, 8, 8, 512}};
+      {8, 5, 3, 2},      {4, 5, 5, 2},    {8, 7, 4, 128},
+      {16, 20, 20, 256}, {256, 7, 5, 4},  {256, 6, 6, 4},
+      {256, 8, 8, 512},  {64, 7, 7, 960}, {64, 14, 14, 576}};
 
   for (auto option : config_options) {
     int64 feature = option[3];
@@ -68,8 +71,14 @@ static std::vector<BatchGroupedConvolution2DSpec> GetConv2DTestCases() {
     int64 output_space_size = 3 + activation_size - kernel_size;
     config.output_dims = {output_space_size, output_space_size, feature, 1};
 
+    config.activation_and_kernel_layout = {0, 3, 1, 2};
+    config.output_layout = {2, 3, 0, 1};
     config_set.push_back(config);
 
+    BatchGroupedConvolution2DSpec different_layout_config = config;
+    different_layout_config.activation_and_kernel_layout = {3, 0, 1, 2};
+    config_set.push_back(different_layout_config);
+
     // Add configurations for window dilation cases.
     if (activation_size % 2 == 0 && activation_size == kernel_size) {
       BatchGroupedConvolution2DSpec config;
@@ -79,11 +88,17 @@ static std::vector<BatchGroupedConvolution2DSpec> GetConv2DTestCases() {
       config.activation_dims = {batch, activation_size, activation_size,
                                 feature};
       config.kernel_dims = {batch, kernel_size / 2, kernel_size / 2, feature};
+      config.activation_and_kernel_layout = {0, 3, 1, 2};
+      config.output_layout = {2, 3, 0, 1};
 
       int64 output_space_size = 5;
       config.output_dims = {output_space_size, output_space_size, feature, 1};
 
       config_set.push_back(config);
+
+      BatchGroupedConvolution2DSpec different_layout_config = config;
+      different_layout_config.activation_and_kernel_layout = {3, 0, 1, 2};
+      config_set.push_back(different_layout_config);
     }
   }
 
@@ -97,8 +112,11 @@ string BatchGroupedConvolution2DTestDataToString(
   const string data_type = GetFloatDataType(::testing::get<1>(data.param));
   string str = absl::StrCat(
       "activation_dims_", absl::StrJoin(spec.activation_dims, "x"),
-      "_kernel_dims_", absl::StrJoin(spec.kernel_dims, "x"), "_output_dims_",
-      absl::StrJoin(spec.output_dims, "x"), data_type);
+      "_kernel_dims_", absl::StrJoin(spec.kernel_dims, "x"),
+      "_activation_layout_",
+      absl::StrJoin(spec.activation_and_kernel_layout, "_"), "_output_dims_",
+      absl::StrJoin(spec.output_dims, "x"), data_type, "_output_layout_",
+      absl::StrJoin(spec.output_layout, "_"));
 
   // Test names are not allowed to contain the '-' character.
   absl::c_replace(str, '-', 'n');
@@ -110,23 +128,28 @@ string BuildHloTextBatchGroupedConvolution2D(
   const string data_type = GetFloatDataType(use_bfloat16);
   return absl::StrFormat(
       R"(
-    HloModule TensorFlowDepthwiseConv
+    HloModule TensorFlowDepthwiseConv, is_scheduled=true
 
     ENTRY main {
-      activation = %s[%s] parameter(0)
-      kernel = %s[%s] parameter(1)
-      ROOT conv = %s[%s] convolution(%s[%s] activation, %s[%s] kernel),
+      activation = %s[%s]{%s} parameter(0)
+      kernel = %s[%s]{%s} parameter(1)
+      ROOT conv = %s[%s]{%s} convolution(%s[%s]{%s} activation, %s[%s]{%s} kernel),
           window={size=%dx%d pad=1_%dx1_%d rhs_dilate=%dx%d}, dim_labels=f01b_i01o->01fb,
           batch_group_count=%d
     }
     )",
-      data_type, absl::StrJoin(spec.activation_dims, ","), data_type,
-      absl::StrJoin(spec.kernel_dims, ","), data_type,
-      absl::StrJoin(spec.output_dims, ","), data_type,
-      absl::StrJoin(spec.activation_dims, ","), data_type,
-      absl::StrJoin(spec.kernel_dims, ","), spec.window, spec.window,
-      spec.window_dilation, spec.window_dilation, spec.window_dilation,
-      spec.window_dilation, spec.output_batch);
+      data_type, absl::StrJoin(spec.activation_dims, ","),
+      absl::StrJoin(spec.activation_and_kernel_layout, ","), data_type,
+      absl::StrJoin(spec.kernel_dims, ","),
+      absl::StrJoin(spec.activation_and_kernel_layout, ","), data_type,
+      absl::StrJoin(spec.output_dims, ","),
+      absl::StrJoin(spec.output_layout, ","), data_type,
+      absl::StrJoin(spec.activation_dims, ","),
+      absl::StrJoin(spec.activation_and_kernel_layout, ","), data_type,
+      absl::StrJoin(spec.kernel_dims, ","),
+      absl::StrJoin(spec.activation_and_kernel_layout, ","), spec.window,
+      spec.window, spec.window_dilation, spec.window_dilation,
+      spec.window_dilation, spec.window_dilation, spec.output_batch);
 }
 
 XLA_TEST_P(BatchGroupedConvolution2DTest, DoIt) {
@@ -135,13 +158,13 @@ XLA_TEST_P(BatchGroupedConvolution2DTest, DoIt) {
   const string hlo_text =
       BuildHloTextBatchGroupedConvolution2D(spec, use_bfloat16);
 
-  EXPECT_TRUE(RunAndCompare(hlo_text, ErrorSpec{0.01, 0.01},
-                            [](HloModule* module) -> Status {
-                              BFloat16MixedPrecisionRemoval remover;
-                              TF_RETURN_IF_ERROR(remover.Run(module).status());
-                              Despecializer despecializer;
-                              return despecializer.Run(module).status();
-                            }));
+  EXPECT_TRUE(RunAndCompareNoHloPasses(
+      hlo_text, ErrorSpec{0.01, 0.01}, [](HloModule* module) -> Status {
+        BFloat16MixedPrecisionRemoval remover;
+        TF_RETURN_IF_ERROR(remover.Run(module).status());
+        Despecializer despecializer;
+        return despecializer.Run(module).status();
+      }));
 }
 
 INSTANTIATE_TEST_CASE_P(
-- 
GitLab


From d7df4b9d0c82cec238c7c8bc5f6d93f95ba0705d Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Fri, 8 Feb 2019 10:19:33 -0800
Subject: [PATCH 2152/2345] Fix initialization of TrackableAssets which aren't
 used to initialize other resources

Previously we weren't adding control dependencies to the init_op for these resources.

Adds a test for a function which captures and returns an asset path, which seems like a reasonable thing to support.

PiperOrigin-RevId: 233081963
---
 tensorflow/python/saved_model/save.py      |  2 ++
 tensorflow/python/saved_model/save_test.py | 11 +++++++++++
 2 files changed, 13 insertions(+)

diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index b50bb61d9c..dbfb5dd6df 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -504,6 +504,8 @@ def _fill_meta_graph_def(meta_graph_def, saveable_view, signature_functions):
         resource_initializer_ops.append(
             _call_function_with_mapped_captures(
                 resource_initializer_function, [], resource_map))
+    resource_initializer_ops.extend(
+        asset_info.asset_initializers_by_resource.values())
     with ops.control_dependencies(resource_initializer_ops):
       init_op = control_flow_ops.no_op()
     # Add the same op to the main_op collection and to the init_op
diff --git a/tensorflow/python/saved_model/save_test.py b/tensorflow/python/saved_model/save_test.py
index e1b2f5da6f..37e8dc6884 100644
--- a/tensorflow/python/saved_model/save_test.py
+++ b/tensorflow/python/saved_model/save_test.py
@@ -320,6 +320,17 @@ class AssetTests(test.TestCase):
     with open(self._vocab_path, "w") as f:
       f.write("alpha\nbeta\ngamma\n")
 
+  def test_asset_path_returned(self):
+    root = tracking.AutoCheckpointable()
+    root.path = tracking.TrackableAsset(self._vocab_path)
+    save_dir = os.path.join(self.get_temp_dir(), "saved_model")
+    root.get_asset = def_function.function(lambda: root.path.asset_path)
+    save.save(root, save_dir, signatures=root.get_asset.get_concrete_function())
+    second_dir = os.path.join(self.get_temp_dir(), "second_dir")
+    file_io.rename(save_dir, second_dir)
+    imported_path = _import_and_infer(second_dir, {})["output_0"]
+    self.assertIn(str(second_dir), str(imported_path))
+
   def test_table(self):
     initializer = lookup_ops.TextFileInitializer(
         self._vocab_path,
-- 
GitLab


From a3d38e4805ab9b3d7c0d4e8ceef8f90dbaca5e78 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Fri, 8 Feb 2019 10:20:33 -0800
Subject: [PATCH 2153/2345] Allow tf.train.Checkpoint.write in tf.functions

We can't update the Checkpoint proto state (e.g. for tf.train.latest_checkpoint), but this at least throws an informative error and gives the user the option to write a checkpoint from a function without updating the Checkpoint proto.

Restore is more complicated, since we do restore-on-create. I think the right thing to do is to inherit the eagerness of the context where restore() was first called. Too much for this change.

PiperOrigin-RevId: 233082153
---
 .../python/training/checkpointable/util.py    | 37 ++++++++++++++-----
 .../training/checkpointable/util_test.py      | 23 ++++++++++++
 .../training/saving/saveable_object_util.py   |  5 ++-
 3 files changed, 55 insertions(+), 10 deletions(-)

diff --git a/tensorflow/python/training/checkpointable/util.py b/tensorflow/python/training/checkpointable/util.py
index 1ba9720fb5..dea8942948 100644
--- a/tensorflow/python/training/checkpointable/util.py
+++ b/tensorflow/python/training/checkpointable/util.py
@@ -31,6 +31,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import tensor_util
 from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_io_ops as io_ops
@@ -913,7 +914,8 @@ class CheckpointableSaver(object):
         # save() is called so they pick up new Tensors passed to their
         # constructors. That means the Saver needs to be copied with a new
         # var_list.
-        or context.executing_eagerly()):
+        or context.executing_eagerly()
+        or ops.inside_function()):
       saver = functional_saver.Saver(named_saveable_objects)
       with ops.device("/cpu:0"):
         self._cached_save_operation = saver.save(file_prefix)
@@ -943,10 +945,11 @@ class CheckpointableSaver(object):
       The full path to the checkpoint.
     """
     feed_dict = {}
-    graph_building = not context.executing_eagerly()
+    use_session = (not context.executing_eagerly()
+                   and not ops.inside_function())
     if checkpoint_number:
       file_prefix = "%s-%d" % (file_prefix, checkpoint_number)
-    if graph_building:
+    if use_session:
       if self._object_graph_feed_tensor is None:
         with ops.device("/cpu:0"):
           self._object_graph_feed_tensor = constant_op.constant(
@@ -968,16 +971,15 @@ class CheckpointableSaver(object):
         object_graph_tensor=object_graph_tensor)
     if new_feed_additions:
       feed_dict.update(new_feed_additions)
-    if not graph_building:
+    if not use_session:
       session = None
     elif session is None:
       session = ops.get_default_session()
 
     if session:
-      save_path = session.run(save_path, feed_dict=feed_dict)
+      return session.run(save_path, feed_dict=feed_dict)
     else:
-      save_path = save_path.numpy()
-    return save_path
+      return save_path
 
   def restore(self, save_path):
     """Restore a training checkpoint.
@@ -1277,9 +1279,18 @@ class Checkpoint(tracking.AutoCheckpointable):
     Returns:
       The full path to the checkpoint (i.e. `file_prefix`).
     """
-    return compat.as_str(self._saver.save(
+    output = self._saver.save(
         file_prefix=file_prefix,
-        session=session))
+        session=session)
+    if tensor_util.is_tensor(output):
+      if context.executing_eagerly():
+        return compat.as_str(output.numpy())
+      else:
+        # Function building
+        return output
+    else:
+      # Graph + Session, so we already session.ran it.
+      return compat.as_str(output)
 
   @property
   def save_counter(self):
@@ -1320,6 +1331,14 @@ class Checkpoint(tracking.AutoCheckpointable):
     """
     graph_building = not context.executing_eagerly()
     if graph_building:
+      if ops.inside_function():
+        raise NotImplementedError(
+            "Calling tf.train.Checkpoint.save() from a function is not "
+            "supported, as save() modifies saving metadata in ways not "
+            "supported by TensorFlow Operations. Consider using "
+            "tf.train.Checkpoint.write(), a lower-level API which does not "
+            "update metadata. tf.train.latest_checkpoint and related APIs will "
+            "not see this checkpoint.")
       if session is None:
         session = ops.get_default_session()
       if self._save_counter is None:
diff --git a/tensorflow/python/training/checkpointable/util_test.py b/tensorflow/python/training/checkpointable/util_test.py
index 4f9abccfe4..0894072b22 100644
--- a/tensorflow/python/training/checkpointable/util_test.py
+++ b/tensorflow/python/training/checkpointable/util_test.py
@@ -1262,6 +1262,29 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
     load_status = checkpoint.restore(save_path)
     load_status.assert_existing_objects_matched().run_restore_ops()
 
+  @test_util.run_in_graph_and_eager_modes
+  def test_write_checkpoint_from_function(self):
+    checkpoint_prefix = os.path.join(self.get_temp_dir(), "ckpt")
+    save_checkpoint = checkpointable_utils.Checkpoint(
+        v=variables_lib.Variable(1.))
+
+    @def_function.function
+    def _write_checkpoint():
+      save_path = save_checkpoint.write(checkpoint_prefix)
+      return save_path
+
+    self.evaluate([save_checkpoint.v.initializer])
+    self.evaluate(_write_checkpoint())
+    load_checkpoint = checkpointable_utils.Checkpoint(
+        v=variables_lib.Variable(0.))
+    load_checkpoint.restore(checkpoint_prefix).run_restore_ops()
+    self.assertEqual(1., self.evaluate(load_checkpoint.v))
+    self.evaluate(save_checkpoint.v.assign(3.))
+    self.evaluate(_write_checkpoint())
+    self.evaluate(save_checkpoint.v.assign(0.))
+    load_checkpoint.restore(checkpoint_prefix).run_restore_ops()
+    self.assertEqual(3., self.evaluate(load_checkpoint.v))
+
 
 class _ManualScope(tracking.AutoCheckpointable):
 
diff --git a/tensorflow/python/training/saving/saveable_object_util.py b/tensorflow/python/training/saving/saveable_object_util.py
index b8cc66249b..e923c03411 100644
--- a/tensorflow/python/training/saving/saveable_object_util.py
+++ b/tensorflow/python/training/saving/saveable_object_util.py
@@ -258,7 +258,10 @@ def op_list_to_dict(op_list, convert_variable_to_tensor=True):
       names_to_saveables.update(
           op_list_to_dict(checkpointable_saveables))
     else:
-      if context.executing_eagerly():
+      # Variables (reference and resource) have an _in_graph_mode property
+      # indicating whether they were created in a graph building context. We
+      # also get Tensors when graph building, which do not have this property.
+      if not getattr(var, "_in_graph_mode", True):
         if not isinstance(var, resource_variable_ops.ResourceVariable):
           raise ValueError(
               "Can only save/restore ResourceVariables when eager execution "
-- 
GitLab


From 63c0707cedfe2ad4a489193feb3a4861d60d900c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 8 Feb 2019 10:38:06 -0800
Subject: [PATCH 2154/2345] Add vectorized version of div_no_nan using the new
 pcmp_eq and pandnot packet ops in Eigen.

Benchmark                          Base (ns)  New (ns) Improvement
------------------------------------------------------------------
BM_cpu_DivNoNan_scalar/4k              13574     12943     +4.6%
BM_cpu_DivNoNan_scalar/32k             31168     19213    +38.4%
BM_cpu_DivNoNan_scalar/128k            50737     42902    +15.4%
BM_cpu_DivNoNan_scalar/1M             137965    110706    +19.8%

PiperOrigin-RevId: 233085406
---
 tensorflow/core/kernels/cwise_ops.h | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/cwise_ops.h b/tensorflow/core/kernels/cwise_ops.h
index b03ee937ca..9657e37d54 100644
--- a/tensorflow/core/kernels/cwise_ops.h
+++ b/tensorflow/core/kernels/cwise_ops.h
@@ -158,13 +158,20 @@ struct div_no_nan_op {
       return 0;
     }
   }
+  template <typename Packet>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet
+  packetOp(const Packet& a, const Packet& b) const {
+    const Packet mask = pcmp_eq(b, pzero(b));
+    const Packet quotient = scalar_quotient_op<T>().packetOp(a, b);
+    return pandnot(quotient, mask);
+  }
 };
 
 template <typename T>
 struct functor_traits<div_no_nan_op<T>> {
   enum {
     Cost = functor_traits<scalar_quotient_op<T>>::Cost + NumTraits<T>::AddCost,
-    PacketAccess = false,
+    PacketAccess = true,
   };
 };
 
-- 
GitLab


From 00c5d87cc7e4dd03859759d9820b9728b3756552 Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Fri, 8 Feb 2019 10:48:29 -0800
Subject: [PATCH 2155/2345] Create int8 reduce min and reduce max.

PiperOrigin-RevId: 233087624
---
 tensorflow/lite/kernels/reduce.cc       |   6 +
 tensorflow/lite/kernels/reduce_test.cc  | 201 ++++++++++++++++++++----
 tensorflow/lite/kernels/register.cc     |   8 +-
 tensorflow/lite/toco/tflite/operator.cc |  12 ++
 4 files changed, 198 insertions(+), 29 deletions(-)

diff --git a/tensorflow/lite/kernels/reduce.cc b/tensorflow/lite/kernels/reduce.cc
index 1193f84f6f..3fb2715e48 100644
--- a/tensorflow/lite/kernels/reduce.cc
+++ b/tensorflow/lite/kernels/reduce.cc
@@ -180,6 +180,9 @@ TfLiteStatus InitializeTemporaries(TfLiteContext* context, TfLiteNode* node,
     case kTfLiteUInt8:
       temp_sum->type = kTfLiteInt32;
       break;
+    case kTfLiteInt8:
+      temp_sum->type = kTfLiteInt32;
+      break;
     case kTfLiteBool:
       temp_sum->type = kTfLiteBool;
       break;
@@ -465,6 +468,9 @@ TfLiteStatus EvalGeneric(TfLiteContext* context, TfLiteNode* node) {
     case kTfLiteUInt8:
       return EvalType<uint8_t>(context, node, &op_context, reduce_type);
       break;
+    case kTfLiteInt8:
+      return EvalType<int8_t>(context, node, &op_context, reduce_type);
+      break;
     case kTfLiteBool:
       return EvalType<bool>(context, node, &op_context, reduce_type);
       break;
diff --git a/tensorflow/lite/kernels/reduce_test.cc b/tensorflow/lite/kernels/reduce_test.cc
index 16c59396dd..373fa56fc1 100644
--- a/tensorflow/lite/kernels/reduce_test.cc
+++ b/tensorflow/lite/kernels/reduce_test.cc
@@ -38,9 +38,10 @@ class BaseOpModel : public SingleOpModel {
     return ExtractVector<T>(output_);
   }
 
+  template <typename T>
   std::vector<float> GetDequantizedOutput() {
-    return Dequantize<uint8_t>(ExtractVector<uint8_t>(output_),
-                               GetScale(output_), GetZeroPoint(output_));
+    return Dequantize<T>(ExtractVector<T>(output_), GetScale(output_),
+                         GetZeroPoint(output_));
   }
 
   std::vector<int> GetOutputShape() { return GetTensorShape(output_); }
@@ -280,7 +281,7 @@ TEST(ConstFloatMeanOpTest, KeepDims_4DMean_UInt8) {
   m.QuantizeAndPopulate<uint8_t>(m.Input(), data);
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 1, 1, 3}));
-  EXPECT_THAT(m.GetDequantizedOutput(),
+  EXPECT_THAT(m.GetDequantizedOutput<uint8_t>(),
               ElementsAreArray(ArrayFloatNear({0.25098, 0.25098, 0.25098},
                                               kQuantizedTolerance)));
 }
@@ -294,7 +295,7 @@ TEST(ConstFloatMeanOpTest, KeepDims_4DMean_Quantized) {
   m.QuantizeAndPopulate<uint8_t>(m.Input(), data);
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 1, 1, 2}));
-  EXPECT_THAT(m.GetDequantizedOutput(),
+  EXPECT_THAT(m.GetDequantizedOutput<uint8_t>(),
               ElementsAreArray(
                   ArrayFloatNear({0.235294, 0.313726}, kQuantizedTolerance)));
 }
@@ -361,8 +362,9 @@ TEST(ConstUint8MeanOpTest, NotKeepDims) {
   m.QuantizeAndPopulate<uint8_t>(m.Input(), data);
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 2}));
-  EXPECT_THAT(m.GetDequantizedOutput(), ElementsAreArray(ArrayFloatNear(
-                                            {0.4, 0.4}, kQuantizedTolerance)));
+  EXPECT_THAT(
+      m.GetDequantizedOutput<uint8_t>(),
+      ElementsAreArray(ArrayFloatNear({0.4, 0.4}, kQuantizedTolerance)));
 }
 
 TEST(ConstUint8MeanOpTest, KeepDims) {
@@ -374,7 +376,7 @@ TEST(ConstUint8MeanOpTest, KeepDims) {
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({3, 1}));
   EXPECT_THAT(
-      m.GetDequantizedOutput(),
+      m.GetDequantizedOutput<uint8_t>(),
       ElementsAreArray(ArrayFloatNear({0.3, 0.35, 0.55}, kQuantizedTolerance)));
 }
 
@@ -390,7 +392,7 @@ TEST(DynamicUint8MeanOpTest, NotKeepDims) {
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({2}));
   EXPECT_THAT(
-      m.GetDequantizedOutput(),
+      m.GetDequantizedOutput<uint8_t>(),
       ElementsAreArray(ArrayFloatNear({-1.75, -1.68}, kQuantizedTolerance)));
 }
 
@@ -406,7 +408,7 @@ TEST(DynamicUint8MeanOpTest, KeepDims) {
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 2}));
   EXPECT_THAT(
-      m.GetDequantizedOutput(),
+      m.GetDequantizedOutput<uint8_t>(),
       ElementsAreArray(ArrayFloatNear({9.2815, 0.3695}, kQuantizedTolerance)));
 }
 
@@ -420,7 +422,7 @@ TEST(DynamicUint8MeanOpTest, QuantizedScalar) {
   m.QuantizeAndPopulate<uint8_t>(m.Input(), data);
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), IsEmpty());
-  EXPECT_THAT(m.GetDequantizedOutput(),
+  EXPECT_THAT(m.GetDequantizedOutput<uint8_t>(),
               ElementsAreArray(ArrayFloatNear({0.643}, kQuantizedTolerance)));
 }
 
@@ -433,7 +435,7 @@ TEST(ConstUint8MeanOpTest, QuantizedKeepDims) {
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({3, 1}));
   EXPECT_THAT(
-      m.GetDequantizedOutput(),
+      m.GetDequantizedOutput<uint8_t>(),
       ElementsAreArray(ArrayFloatNear({0.3, 0.35, 0.55}, kQuantizedTolerance)));
 }
 
@@ -526,7 +528,7 @@ TEST(ConstUint8SumOpTest, NotKeepDims) {
   m.QuantizeAndPopulate<uint8_t>(m.Input(), data);
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 2}));
-  EXPECT_THAT(m.GetDequantizedOutput(),
+  EXPECT_THAT(m.GetDequantizedOutput<uint8_t>(),
               ElementsAreArray(
                   ArrayFloatNear({-0.823529, -0.815686}, kQuantizedTolerance)));
 }
@@ -539,8 +541,9 @@ TEST(ConstUint8SumOpTest, NotKeepDimsRescaling) {
   m.QuantizeAndPopulate<uint8_t>(m.Input(), data);
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 2}));
-  EXPECT_THAT(m.GetDequantizedOutput(), ElementsAreArray(ArrayFloatNear(
-                                            {1.2, 1.2}, kQuantizedTolerance)));
+  EXPECT_THAT(
+      m.GetDequantizedOutput<uint8_t>(),
+      ElementsAreArray(ArrayFloatNear({1.2, 1.2}, kQuantizedTolerance)));
 }
 
 TEST(ConstUint8SumOpTest, KeepDims) {
@@ -551,7 +554,7 @@ TEST(ConstUint8SumOpTest, KeepDims) {
   m.QuantizeAndPopulate<uint8_t>(m.Input(), data);
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({3, 1}));
-  EXPECT_THAT(m.GetDequantizedOutput(),
+  EXPECT_THAT(m.GetDequantizedOutput<uint8_t>(),
               ElementsAreArray(ArrayFloatNear({-0.407843, -0.313726, 0.0941177},
                                               kQuantizedTolerance)));
 }
@@ -567,7 +570,7 @@ TEST(DynamicUint8SumOpTest, NotKeepDims) {
   m.QuantizeAndPopulate<uint8_t>(m.Input(), data);
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({2}));
-  EXPECT_THAT(m.GetDequantizedOutput(),
+  EXPECT_THAT(m.GetDequantizedOutput<uint8_t>(),
               ElementsAreArray(
                   ArrayFloatNear({1.48235, 1.64706}, kQuantizedTolerance)));
 }
@@ -584,7 +587,7 @@ TEST(DynamicUint8SumOpTest, KeepDims) {
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 2}));
   EXPECT_THAT(
-      m.GetDequantizedOutput(),
+      m.GetDequantizedOutput<uint8_t>(),
       ElementsAreArray(ArrayFloatNear({6.47059, 10.698}, kQuantizedTolerance)));
 }
 
@@ -741,7 +744,20 @@ TEST(ConstUint8MaxOpTest, NotKeepDims) {
   m.QuantizeAndPopulate<uint8_t>(m.Input(), data);
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 2}));
-  EXPECT_THAT(m.GetDequantizedOutput(),
+  EXPECT_THAT(m.GetDequantizedOutput<uint8_t>(),
+              ElementsAreArray(
+                  ArrayFloatNear({0.501961, 0.603922}, kQuantizedTolerance)));
+}
+
+TEST(ConstInt8MaxOpTest, NotKeepDims) {
+  float kQuantizedTolerance = GetTolerance(-1.0, 1.0);
+  std::vector<float> data = {0.4, 0.2, 0.3, 0.4, 0.5, 0.6};
+  MaxOpConstModel m({TensorType_INT8, {1, 3, 2}, -1.0, 1.0},
+                    {TensorType_INT8, {2}, -1.0, 1.0}, {1}, {1}, false);
+  m.QuantizeAndPopulate<int8_t>(m.Input(), data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 2}));
+  EXPECT_THAT(m.GetDequantizedOutput<int8_t>(),
               ElementsAreArray(
                   ArrayFloatNear({0.501961, 0.603922}, kQuantizedTolerance)));
 }
@@ -754,7 +770,20 @@ TEST(ConstUint8MaxOpTest, KeepDims) {
   m.QuantizeAndPopulate<uint8_t>(m.Input(), data);
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({3, 1}));
-  EXPECT_THAT(m.GetDequantizedOutput(),
+  EXPECT_THAT(m.GetDequantizedOutput<uint8_t>(),
+              ElementsAreArray(
+                  ArrayFloatNear({0.4, 0.4, 0.603922}, kQuantizedTolerance)));
+}
+
+TEST(ConstInt8MaxOpTest, KeepDims) {
+  float kQuantizedTolerance = GetTolerance(-1.0, 1.0);
+  std::vector<float> data = {0.4, 0.2, 0.3, 0.4, 0.5, 0.6};
+  MaxOpConstModel m({TensorType_INT8, {3, 2}, -1.0, 1.0},
+                    {TensorType_INT8, {3}, -1.0, 1.0}, {1}, {1}, true);
+  m.QuantizeAndPopulate<int8_t>(m.Input(), data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({3, 1}));
+  EXPECT_THAT(m.GetDequantizedOutput<int8_t>(),
               ElementsAreArray(
                   ArrayFloatNear({0.4, 0.4, 0.603922}, kQuantizedTolerance)));
 }
@@ -770,7 +799,23 @@ TEST(DynamicUint8MaxOpTest, NotKeepDims) {
   m.QuantizeAndPopulate<uint8_t>(m.Input(), data);
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({2}));
-  EXPECT_THAT(m.GetDequantizedOutput(),
+  EXPECT_THAT(m.GetDequantizedOutput<uint8_t>(),
+              ElementsAreArray(
+                  ArrayFloatNear({1.2902, 0.247059}, kQuantizedTolerance)));
+}
+
+TEST(DynamicInt8MaxOpTest, NotKeepDims) {
+  float kQuantizedTolerance = GetTolerance(-5.0, 2.0);
+  std::vector<float> data = {1.3, -4.8, -3.6, 0.24};
+  MaxOpDynamicModel m({TensorType_INT8, {2, 2}, -5.0, 2.0},
+                      {TensorType_INT8, {2}, -5.0, 2.0},
+                      {TensorType_INT32, {1}}, false);
+  std::vector<int> axis = {1};
+  m.SetAxis(axis);
+  m.QuantizeAndPopulate<int8_t>(m.Input(), data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({2}));
+  EXPECT_THAT(m.GetDequantizedOutput<int8_t>(),
               ElementsAreArray(
                   ArrayFloatNear({1.2902, 0.247059}, kQuantizedTolerance)));
 }
@@ -786,7 +831,23 @@ TEST(DynamicUint8MaxOpTest, KeepDims) {
   m.QuantizeAndPopulate<uint8_t>(m.Input(), data);
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 2}));
-  EXPECT_THAT(m.GetDequantizedOutput(),
+  EXPECT_THAT(m.GetDequantizedOutput<uint8_t>(),
+              ElementsAreArray(
+                  ArrayFloatNear({11.1294, 0.862745}, kQuantizedTolerance)));
+}
+
+TEST(DynamicInt8MaxOpTest, KeepDims) {
+  float kQuantizedTolerance = GetTolerance(-10.0, 12.0);
+  std::vector<float> data = {11.14, -0.14, 7.423, 0.879};
+  MaxOpDynamicModel m({TensorType_INT8, {2, 2}, -10.0, 12.0},
+                      {TensorType_INT8, {2}, -10.0, 12.0},
+                      {TensorType_INT32, {1}}, true);
+  std::vector<int> axis = {0};
+  m.SetAxis(axis);
+  m.QuantizeAndPopulate<int8_t>(m.Input(), data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 2}));
+  EXPECT_THAT(m.GetDequantizedOutput<int8_t>(),
               ElementsAreArray(
                   ArrayFloatNear({11.1294, 0.862745}, kQuantizedTolerance)));
 }
@@ -801,7 +862,21 @@ TEST(DynamicUint8MaxOpTest, Scalar) {
   m.QuantizeAndPopulate<uint8_t>(m.Input(), data);
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), IsEmpty());
-  EXPECT_THAT(m.GetDequantizedOutput(),
+  EXPECT_THAT(m.GetDequantizedOutput<uint8_t>(),
+              ElementsAreArray(ArrayFloatNear({11.1294}, kQuantizedTolerance)));
+}
+
+TEST(DynamicInt8MaxOpTest, Scalar) {
+  float kQuantizedTolerance = GetTolerance(-10.0, 12.0);
+  std::vector<float> data = {11.14};
+  MaxOpDynamicModel m({TensorType_INT8, {}, -10.0, 12.0},
+                      {TensorType_INT8, {}, -10.0, 12.0},
+                      {TensorType_INT32, {1}}, true);
+  std::vector<int> axis = {0};
+  m.QuantizeAndPopulate<int8_t>(m.Input(), data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), IsEmpty());
+  EXPECT_THAT(m.GetDequantizedOutput<int8_t>(),
               ElementsAreArray(ArrayFloatNear({11.1294}, kQuantizedTolerance)));
 }
 
@@ -883,7 +958,20 @@ TEST(ConstUint8MinOpTest, NotKeepDims) {
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 2}));
   EXPECT_THAT(
-      m.GetDequantizedOutput(),
+      m.GetDequantizedOutput<uint8_t>(),
+      ElementsAreArray(ArrayFloatNear({0.294117, 0.2}, kQuantizedTolerance)));
+}
+
+TEST(ConstInt8MinOpTest, NotKeepDims) {
+  float kQuantizedTolerance = GetTolerance(-1.0, 1.0);
+  std::vector<float> data = {0.4, 0.2, 0.3, 0.4, 0.5, 0.6};
+  MinOpConstModel m({TensorType_INT8, {1, 3, 2}, -1.0, 1.0},
+                    {TensorType_INT8, {2}, -1.0, 1.0}, {1}, {1}, false);
+  m.QuantizeAndPopulate<int8_t>(m.Input(), data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 2}));
+  EXPECT_THAT(
+      m.GetDequantizedOutput<int8_t>(),
       ElementsAreArray(ArrayFloatNear({0.294117, 0.2}, kQuantizedTolerance)));
 }
 
@@ -896,7 +984,20 @@ TEST(ConstUint8MinOpTest, KeepDims) {
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({3, 1}));
   EXPECT_THAT(
-      m.GetDequantizedOutput(),
+      m.GetDequantizedOutput<uint8_t>(),
+      ElementsAreArray(ArrayFloatNear({0.2, 0.3, 0.5}, kQuantizedTolerance)));
+}
+
+TEST(ConstInt8MinOpTest, KeepDims) {
+  float kQuantizedTolerance = GetTolerance(-1.0, 1.0);
+  std::vector<float> data = {0.4, 0.2, 0.3, 0.4, 0.5, 0.6};
+  MinOpConstModel m({TensorType_INT8, {3, 2}, -1.0, 1.0},
+                    {TensorType_INT8, {3}, -1.0, 1.0}, {1}, {1}, true);
+  m.QuantizeAndPopulate<int8_t>(m.Input(), data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({3, 1}));
+  EXPECT_THAT(
+      m.GetDequantizedOutput<int8_t>(),
       ElementsAreArray(ArrayFloatNear({0.2, 0.3, 0.5}, kQuantizedTolerance)));
 }
 
@@ -912,7 +1013,23 @@ TEST(DynamicUint8MinOpTest, NotKeepDims) {
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({2}));
   EXPECT_THAT(
-      m.GetDequantizedOutput(),
+      m.GetDequantizedOutput<uint8_t>(),
+      ElementsAreArray(ArrayFloatNear({-4.807843, -3.6}, kQuantizedTolerance)));
+}
+
+TEST(DynamicInt8MinOpTest, NotKeepDims) {
+  float kQuantizedTolerance = GetTolerance(-5.0, 2.0);
+  std::vector<float> data = {1.3, -4.8, -3.6, 0.24};
+  MinOpDynamicModel m({TensorType_INT8, {2, 2}, -5.0, 2.0},
+                      {TensorType_INT8, {2}, -5.0, 2.0},
+                      {TensorType_INT32, {1}}, false);
+  std::vector<int> axis = {1};
+  m.SetAxis(axis);
+  m.QuantizeAndPopulate<int8_t>(m.Input(), data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({2}));
+  EXPECT_THAT(
+      m.GetDequantizedOutput<int8_t>(),
       ElementsAreArray(ArrayFloatNear({-4.807843, -3.6}, kQuantizedTolerance)));
 }
 
@@ -927,7 +1044,23 @@ TEST(DynamicUint8MinOpTest, KeepDims) {
   m.QuantizeAndPopulate<uint8_t>(m.Input(), data);
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 2}));
-  EXPECT_THAT(m.GetDequantizedOutput(),
+  EXPECT_THAT(m.GetDequantizedOutput<uint8_t>(),
+              ElementsAreArray(
+                  ArrayFloatNear({7.427451, -0.164706}, kQuantizedTolerance)));
+}
+
+TEST(DynamicInt8MinOpTest, KeepDims) {
+  float kQuantizedTolerance = GetTolerance(-10.0, 12.0);
+  std::vector<float> data = {11.14, -0.14, 7.423, 0.879};
+  MinOpDynamicModel m({TensorType_INT8, {2, 2}, -10.0, 12.0},
+                      {TensorType_INT8, {2}, -10.0, 12.0},
+                      {TensorType_INT32, {1}}, true);
+  std::vector<int> axis = {0};
+  m.SetAxis(axis);
+  m.QuantizeAndPopulate<int8_t>(m.Input(), data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 2}));
+  EXPECT_THAT(m.GetDequantizedOutput<int8_t>(),
               ElementsAreArray(
                   ArrayFloatNear({7.427451, -0.164706}, kQuantizedTolerance)));
 }
@@ -942,7 +1075,21 @@ TEST(DynamicUint8MinOpTest, Scalar) {
   m.QuantizeAndPopulate<uint8_t>(m.Input(), data);
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), IsEmpty());
-  EXPECT_THAT(m.GetDequantizedOutput(),
+  EXPECT_THAT(m.GetDequantizedOutput<uint8_t>(),
+              ElementsAreArray(ArrayFloatNear({11.1294}, kQuantizedTolerance)));
+}
+
+TEST(DynamicInt8MinOpTest, Scalar) {
+  float kQuantizedTolerance = GetTolerance(-10.0, 12.0);
+  std::vector<float> data = {11.14};
+  MinOpDynamicModel m({TensorType_INT8, {}, -10.0, 12.0},
+                      {TensorType_INT8, {}, -10.0, 12.0},
+                      {TensorType_INT32, {1}}, true);
+  std::vector<int> axis = {0};
+  m.QuantizeAndPopulate<int8_t>(m.Input(), data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), IsEmpty());
+  EXPECT_THAT(m.GetDequantizedOutput<int8_t>(),
               ElementsAreArray(ArrayFloatNear({11.1294}, kQuantizedTolerance)));
 }
 
diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index 525b97f8e1..2c9851b05c 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -294,8 +294,12 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_TILE, Register_TILE());
   AddBuiltin(BuiltinOperator_SUM, Register_SUM());
   AddBuiltin(BuiltinOperator_REDUCE_PROD, Register_REDUCE_PROD());
-  AddBuiltin(BuiltinOperator_REDUCE_MAX, Register_REDUCE_MAX());
-  AddBuiltin(BuiltinOperator_REDUCE_MIN, Register_REDUCE_MIN());
+  AddBuiltin(BuiltinOperator_REDUCE_MAX, Register_REDUCE_MAX(),
+             /* min_version */ 1,
+             /* max_version */ 2);
+  AddBuiltin(BuiltinOperator_REDUCE_MIN, Register_REDUCE_MIN(),
+             /* min_version */ 1,
+             /* max_version */ 2);
   AddBuiltin(BuiltinOperator_REDUCE_ANY, Register_REDUCE_ANY());
   AddBuiltin(BuiltinOperator_EXPAND_DIMS, Register_EXPAND_DIMS());
   AddBuiltin(BuiltinOperator_SPARSE_TO_DENSE, Register_SPARSE_TO_DENSE());
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index 79125a44f9..b26ed6a3ec 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -1115,6 +1115,12 @@ class ReduceMax
   }
 
   int GetVersion(const OperatorSignature& op_signature) const override {
+    const string& input_name = op_signature.op->inputs[0];
+    const Array& input_array = op_signature.model->GetArray(input_name);
+    // If the op take int8 input, it is version 2.
+    if (input_array.data_type == ArrayDataType::kInt8) {
+      return 2;
+    }
     return 1;
   }
 };
@@ -1136,6 +1142,12 @@ class ReduceMin
   }
 
   int GetVersion(const OperatorSignature& op_signature) const override {
+    const string& input_name = op_signature.op->inputs[0];
+    const Array& input_array = op_signature.model->GetArray(input_name);
+    // If the op take int8 input, it is version 2.
+    if (input_array.data_type == ArrayDataType::kInt8) {
+      return 2;
+    }
     return 1;
   }
 };
-- 
GitLab


From a22e9ad5c9886c00536fcef3282c8dd31249c6b1 Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Fri, 8 Feb 2019 10:53:37 -0800
Subject: [PATCH 2156/2345] Make `plot_model` more delightful for notebook
 users.

PiperOrigin-RevId: 233088643
---
 tensorflow/python/keras/utils/vis_utils.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tensorflow/python/keras/utils/vis_utils.py b/tensorflow/python/keras/utils/vis_utils.py
index 9394cc114e..d396851a62 100644
--- a/tensorflow/python/keras/utils/vis_utils.py
+++ b/tensorflow/python/keras/utils/vis_utils.py
@@ -145,6 +145,10 @@ def plot_model(model,
           a string specifying the format of the plot:
           'TB' creates a vertical plot;
           'LR' creates a horizontal plot.
+
+  Returns:
+      A Jupyter notebook Image object if Jupyter is installed.
+      This enables in-line display of the model plots in notebooks.
   """
   dot = model_to_dot(model, show_shapes, show_layer_names, rankdir)
   _, extension = os.path.splitext(to_file)
@@ -152,4 +156,13 @@ def plot_model(model,
     extension = 'png'
   else:
     extension = extension[1:]
+  # Save image to disk.
   dot.write(to_file, format=extension)
+  # Return the image as a Jupyter Image object, to be displayed in-line.
+  # Note that we cannot easily detect whether the code is running in a
+  # notebook, and thus we always return the Image if Jupyter is available.
+  try:
+    from IPython import display
+    return display.Image(filename=to_file)
+  except ImportError:
+    pass
-- 
GitLab


From d815b49f4cf8bbbc7fd648cd36a53c1450e5ff4a Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Fri, 8 Feb 2019 10:58:08 -0800
Subject: [PATCH 2157/2345] Use tf.lite.OpHint instead of importing it directly

After this change, users can import tensorflow then
import these 2 modules directly.

PiperOrigin-RevId: 233089555
---
 tensorflow/lite/experimental/examples/lstm/tflite_lstm.py | 5 ++---
 tensorflow/lite/experimental/examples/lstm/tflite_rnn.py  | 4 ++--
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/tensorflow/lite/experimental/examples/lstm/tflite_lstm.py b/tensorflow/lite/experimental/examples/lstm/tflite_lstm.py
index e6d329ffd1..5ef1a8cc1c 100644
--- a/tensorflow/lite/experimental/examples/lstm/tflite_lstm.py
+++ b/tensorflow/lite/experimental/examples/lstm/tflite_lstm.py
@@ -21,7 +21,6 @@ from __future__ import division
 from __future__ import print_function
 import tensorflow as tf
 
-from tensorflow.lite.python import lite
 from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
 from tensorflow.python.keras import activations
@@ -142,7 +141,7 @@ class TFLiteLSTMCell(rnn_cell_impl.LayerRNNCell):
     # TODO(raziel): layers stuff -- chop if un-layerizing Op.
     self.input_spec = base_layer.InputSpec(ndim=2)
 
-    self._tflite_wrapper = lite.OpHint("UnidirectionalSequenceLstm")
+    self._tflite_wrapper = tf.lite.OpHint("UnidirectionalSequenceLstm")
 
     self._num_units = num_units
     self._use_peepholes = use_peepholes
@@ -557,7 +556,7 @@ def dynamic_rnn(cell,
       "parent_last_child_output": parent_last_child_output,
       "internal_children_input_output": internal_children_input_output
   }
-  tflite_wrapper = lite.OpHint(
+  tflite_wrapper = tf.lite.OpHint(
       "TfLiteDynamicRnn",
       level=2,
       children_inputs_mappings=inputs_outputs_mappings)
diff --git a/tensorflow/lite/experimental/examples/lstm/tflite_rnn.py b/tensorflow/lite/experimental/examples/lstm/tflite_rnn.py
index e4aad18367..b7a89056ac 100644
--- a/tensorflow/lite/experimental/examples/lstm/tflite_rnn.py
+++ b/tensorflow/lite/experimental/examples/lstm/tflite_rnn.py
@@ -20,8 +20,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 import itertools
+import tensorflow as tf
 
-from tensorflow.lite.python import lite
 from tensorflow.python.keras import activations
 from tensorflow.python.layers import base as base_layer
 from tensorflow.python.ops import array_ops
@@ -70,7 +70,7 @@ class TfLiteRNNCell(rnn_cell_impl.LayerRNNCell):
     # Inputs must be Rank-2.
     self.input_spec = base_layer.InputSpec(ndim=2)
 
-    self._tflite_wrapper = lite.OpHint("UnidirectionalSequenceRnn")
+    self._tflite_wrapper = tf.lite.OpHint("UnidirectionalSequenceRnn")
     self._num_units = num_units
     if activation:
       self._activation = activations.get(activation)
-- 
GitLab


From d5fb65ff33389d045222f47ac04d77e1b4275f3c Mon Sep 17 00:00:00 2001
From: Yunlu Li <yunluli@google.com>
Date: Fri, 8 Feb 2019 10:59:29 -0800
Subject: [PATCH 2158/2345] Op level diff test.

PiperOrigin-RevId: 233089842
---
 tensorflow/lite/BUILD                         |   2 +
 tensorflow/lite/c/BUILD                       |   1 +
 tensorflow/lite/testdata/test_input.csv       |   1 +
 tensorflow/lite/testing/BUILD                 |   1 +
 tensorflow/lite/testing/kernel_test/BUILD     | 124 +++++++++++
 .../lite/testing/kernel_test/diff_analyzer.cc | 115 ++++++++++
 .../lite/testing/kernel_test/diff_analyzer.h  |  42 ++++
 .../testing/kernel_test/diff_analyzer_test.cc |  47 ++++
 .../kernel_test/generate_diff_report.cc       |  34 +++
 .../testing/kernel_test/input_generator.cc    | 208 ++++++++++++++++++
 .../testing/kernel_test/input_generator.h     |  50 +++++
 .../kernel_test/input_generator_test.cc       |  81 +++++++
 .../kernel_test/tflite_kernel_runner.cc       |  32 +++
 tensorflow/lite/testing/kernel_test/util.h    | 122 ++++++++++
 .../lite/testing/kernel_test/util_test.cc     |  52 +++++
 tensorflow/lite/testing/tflite_driver.cc      |  30 +++
 tensorflow/lite/testing/tflite_driver.h       |   2 +-
 tensorflow/lite/testing/tflite_driver_test.cc |   4 +
 18 files changed, 947 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/lite/testdata/test_input.csv
 create mode 100644 tensorflow/lite/testing/kernel_test/BUILD
 create mode 100644 tensorflow/lite/testing/kernel_test/diff_analyzer.cc
 create mode 100644 tensorflow/lite/testing/kernel_test/diff_analyzer.h
 create mode 100644 tensorflow/lite/testing/kernel_test/diff_analyzer_test.cc
 create mode 100644 tensorflow/lite/testing/kernel_test/generate_diff_report.cc
 create mode 100644 tensorflow/lite/testing/kernel_test/input_generator.cc
 create mode 100644 tensorflow/lite/testing/kernel_test/input_generator.h
 create mode 100644 tensorflow/lite/testing/kernel_test/input_generator_test.cc
 create mode 100644 tensorflow/lite/testing/kernel_test/tflite_kernel_runner.cc
 create mode 100644 tensorflow/lite/testing/kernel_test/util.h
 create mode 100644 tensorflow/lite/testing/kernel_test/util_test.cc

diff --git a/tensorflow/lite/BUILD b/tensorflow/lite/BUILD
index 86eca1d14e..b79386545b 100644
--- a/tensorflow/lite/BUILD
+++ b/tensorflow/lite/BUILD
@@ -11,6 +11,8 @@ load("//tensorflow/lite:special_rules.bzl", "tflite_portable_test_suite")
 exports_files(glob([
     "testdata/*.bin",
     "testdata/*.pb",
+    "testdata/*.tflite",
+    "testdata/*.csv",
     "models/testdata/*",
 ]))
 
diff --git a/tensorflow/lite/c/BUILD b/tensorflow/lite/c/BUILD
index 680addfa2f..661b648550 100644
--- a/tensorflow/lite/c/BUILD
+++ b/tensorflow/lite/c/BUILD
@@ -12,6 +12,7 @@ cc_library(
         "c_api_internal.h",
     ],
     visibility = [
+        "//learning/brain/mobile/kernel_test:__subpackages__",
         "//tensorflow/lite:__subpackages__",
     ],
 )
diff --git a/tensorflow/lite/testdata/test_input.csv b/tensorflow/lite/testdata/test_input.csv
new file mode 100644
index 0000000000..33894d3063
--- /dev/null
+++ b/tensorflow/lite/testdata/test_input.csv
@@ -0,0 +1 @@
+1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1
\ No newline at end of file
diff --git a/tensorflow/lite/testing/BUILD b/tensorflow/lite/testing/BUILD
index 3840b52a0a..e8b14cd2a7 100644
--- a/tensorflow/lite/testing/BUILD
+++ b/tensorflow/lite/testing/BUILD
@@ -159,6 +159,7 @@ cc_library(
     srcs = ["tflite_driver.cc"],
     hdrs = ["tflite_driver.h"],
     deps = [
+        ":join",
         ":split",
         ":test_runner",
         "//tensorflow/lite:builtin_op_data",
diff --git a/tensorflow/lite/testing/kernel_test/BUILD b/tensorflow/lite/testing/kernel_test/BUILD
new file mode 100644
index 0000000000..c46e80cc36
--- /dev/null
+++ b/tensorflow/lite/testing/kernel_test/BUILD
@@ -0,0 +1,124 @@
+package(default_visibility = [
+    "//visibility:public",
+])
+
+licenses(["notice"])  # Apache 2.0
+
+load(
+    "//tensorflow:tensorflow.bzl",
+    "tf_cc_binary",
+    "tf_cc_test",
+)
+
+cc_library(
+    name = "util",
+    hdrs = ["util.h"],
+    deps = [
+        ":input_generator",
+        "//tensorflow/lite/c:c_api_internal",
+        "//tensorflow/lite/testing:split",
+        "//tensorflow/lite/testing:tflite_driver",
+    ] + select({
+        "//conditions:default": [
+            "//tensorflow/core:framework_internal",
+            "//tensorflow/core:lib",
+        ],
+        "//tensorflow:android": [
+            "//tensorflow/core:android_tensorflow_lib",
+        ],
+    }),
+)
+
+tf_cc_test(
+    name = "util_test",
+    size = "small",
+    srcs = ["util_test.cc"],
+    data = [
+        "//tensorflow/lite:testdata/add.bin",
+        "//tensorflow/lite:testdata/test_input.csv",
+    ],
+    tags = [
+        "no_oss",
+    ],
+    deps = [
+        ":util",
+        "//tensorflow/lite/testing:tflite_driver",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
+
+tf_cc_binary(
+    name = "tflite_kernel_runner",
+    srcs = ["tflite_kernel_runner.cc"],
+    deps = [
+        ":util",
+    ],
+)
+
+tf_cc_binary(
+    name = "generate_diff_report",
+    srcs = ["generate_diff_report.cc"],
+    deps = [
+        ":diff_analyzer",
+        "//tensorflow/core:framework_internal",
+    ],
+)
+
+cc_library(
+    name = "input_generator",
+    srcs = ["input_generator.cc"],
+    hdrs = ["input_generator.h"],
+    deps = [
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite:string",
+        "//tensorflow/lite/c:c_api_internal",
+        "//tensorflow/lite/kernels:builtin_ops",
+        "//tensorflow/lite/testing:join",
+        "//tensorflow/lite/testing:split",
+    ],
+)
+
+tf_cc_test(
+    name = "input_generator_test",
+    size = "small",
+    srcs = ["input_generator_test.cc"],
+    data = [
+        "//tensorflow/lite:testdata/multi_add.bin",
+        "//tensorflow/lite:testdata/test_input.csv",
+    ],
+    tags = [
+        "no_oss",
+    ],
+    deps = [
+        ":input_generator",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
+
+cc_library(
+    name = "diff_analyzer",
+    srcs = ["diff_analyzer.cc"],
+    hdrs = ["diff_analyzer.h"],
+    deps = [
+        "//tensorflow/lite:string",
+        "//tensorflow/lite/c:c_api_internal",
+        "//tensorflow/lite/testing:split",
+    ],
+)
+
+tf_cc_test(
+    name = "diff_analyzer_test",
+    size = "small",
+    srcs = ["diff_analyzer_test.cc"],
+    data = [
+        "//tensorflow/lite:testdata/test_input.csv",
+    ],
+    tags = [
+        "no_oss",
+    ],
+    deps = [
+        ":diff_analyzer",
+        "//tensorflow/core:lib",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
diff --git a/tensorflow/lite/testing/kernel_test/diff_analyzer.cc b/tensorflow/lite/testing/kernel_test/diff_analyzer.cc
new file mode 100644
index 0000000000..7d6fcc80be
--- /dev/null
+++ b/tensorflow/lite/testing/kernel_test/diff_analyzer.cc
@@ -0,0 +1,115 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/testing/kernel_test/diff_analyzer.h"
+
+#include <cmath>
+#include <fstream>
+#include "tensorflow/lite/testing/split.h"
+
+namespace tflite {
+namespace testing {
+
+namespace {
+float CalculateNormalizedMaxDiff(const std::vector<float>& base,
+                                 const std::vector<float>& test) {
+  float diff = 0;
+  // For numerical stability in case the tensor is all 0.
+  float base_max = 1e-6;
+
+  for (int i = 0; i < base.size(); i++) {
+    diff = std::max(diff, std::abs(base[i] - test[i]));
+    base_max = std::max(base_max, base[i]);
+  }
+
+  return diff / base_max;
+}
+
+float CalculateNormalizedL2Norm(const std::vector<float>& base,
+                                const std::vector<float>& test) {
+  float l2_error = 0;
+  // For numerical stability in case the tensor is all 0.
+  float base_max = 1e-6;
+
+  for (int i = 0; i < base.size(); i++) {
+    float diff = base[i] - test[i];
+    l2_error += diff * diff;
+    base_max = std::max(base_max, base[i]);
+  }
+
+  l2_error /= base.size();
+
+  return std::sqrt(l2_error) / base_max;
+}
+
+TfLiteStatus Populate(const string& filename,
+                      std::vector<std::vector<float>>* tensors) {
+  if (filename.empty()) {
+    fprintf(stderr, "Empty input file name.");
+    return kTfLiteError;
+  }
+
+  std::ifstream file(filename);
+  string content;
+  while (std::getline(file, content, '\n')) {
+    tensors->push_back(Split<float>(content, ","));
+  }
+
+  file.close();
+  return kTfLiteOk;
+}
+}  // namespace
+
+TfLiteStatus DiffAnalyzer::ReadFiles(const string& base, const string& test) {
+  TF_LITE_ENSURE_STATUS(Populate(base, &base_tensors_));
+  TF_LITE_ENSURE_STATUS(Populate(test, &test_tensors_));
+
+  if (base_tensors_.size() != test_tensors_.size()) {
+    fprintf(stderr, "Golden and test tensor dimensions don't match.");
+    return kTfLiteError;
+  }
+
+  return kTfLiteOk;
+}
+
+TfLiteStatus DiffAnalyzer::WriteReport(const string& filename) {
+  if (filename.empty()) {
+    fprintf(stderr, "Empty output file name.");
+    return kTfLiteError;
+  }
+
+  std::ofstream output_file;
+  output_file.open(filename, std::fstream::out | std::fstream::trunc);
+  if (!output_file) {
+    fprintf(stderr, "Failed to open output file %s.", filename.c_str());
+    return kTfLiteError;
+  }
+
+  output_file << "Normalized L2 Error"
+              << ","
+              << "Normalized Max Diff"
+              << "\n";
+  for (int i = 0; i < base_tensors_.size(); i++) {
+    float l2_error =
+        CalculateNormalizedL2Norm(base_tensors_[i], test_tensors_[i]);
+    float max_diff =
+        CalculateNormalizedMaxDiff(base_tensors_[i], test_tensors_[i]);
+    output_file << l2_error << "," << max_diff << "\n";
+  }
+
+  output_file.close();
+  return kTfLiteOk;
+}
+}  // namespace testing
+}  // namespace tflite
diff --git a/tensorflow/lite/testing/kernel_test/diff_analyzer.h b/tensorflow/lite/testing/kernel_test/diff_analyzer.h
new file mode 100644
index 0000000000..aecbaea449
--- /dev/null
+++ b/tensorflow/lite/testing/kernel_test/diff_analyzer.h
@@ -0,0 +1,42 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TESTING_KERNEL_TEST_DIFF_ANALYZER_H_
+#define TENSORFLOW_LITE_TESTING_KERNEL_TEST_DIFF_ANALYZER_H_
+
+#include <vector>
+
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/string.h"
+
+namespace tflite {
+namespace testing {
+
+// Reads the baseline and test files with output tensor values, and calculates
+// the diff metrics.
+class DiffAnalyzer {
+ public:
+  DiffAnalyzer() = default;
+  TfLiteStatus ReadFiles(const string& base, const string& test);
+  TfLiteStatus WriteReport(const string& filename);
+
+ private:
+  std::vector<std::vector<float>> base_tensors_;
+  std::vector<std::vector<float>> test_tensors_;
+};
+
+}  // namespace testing
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TESTING_KERNEL_TEST_DIFF_ANALYZER_H_
diff --git a/tensorflow/lite/testing/kernel_test/diff_analyzer_test.cc b/tensorflow/lite/testing/kernel_test/diff_analyzer_test.cc
new file mode 100644
index 0000000000..f92a3e6af7
--- /dev/null
+++ b/tensorflow/lite/testing/kernel_test/diff_analyzer_test.cc
@@ -0,0 +1,47 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/testing/kernel_test/diff_analyzer.h"
+
+#include <fstream>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+#include "tensorflow/core/lib/io/path.h"
+
+namespace tflite {
+namespace testing {
+
+namespace {
+
+TEST(DiffAnalyzerTest, ZeroDiff) {
+  DiffAnalyzer diff_analyzer;
+  string filename = "third_party/tensorflow/lite/testdata/test_input.csv";
+  ASSERT_EQ(diff_analyzer.ReadFiles(filename, filename), kTfLiteOk);
+
+  string output_file =
+      tensorflow::io::JoinPath(FLAGS_test_tmpdir + "diff_report.csv");
+  ASSERT_EQ(diff_analyzer.WriteReport(output_file), kTfLiteOk);
+
+  std::string content;
+  std::ifstream file(output_file);
+  std::getline(file, content);
+  std::getline(file, content);
+  ASSERT_EQ(content, "0,0");
+}
+
+}  // namespace
+
+}  // namespace testing
+}  // namespace tflite
diff --git a/tensorflow/lite/testing/kernel_test/generate_diff_report.cc b/tensorflow/lite/testing/kernel_test/generate_diff_report.cc
new file mode 100644
index 0000000000..afa6a9a94e
--- /dev/null
+++ b/tensorflow/lite/testing/kernel_test/generate_diff_report.cc
@@ -0,0 +1,34 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <vector>
+
+#include "tensorflow/core/util/command_line_flags.h"
+#include "tensorflow/lite/testing/kernel_test/diff_analyzer.h"
+
+int main(int argc, char** argv) {
+  string base, test, output;
+  std::vector<tensorflow::Flag> flag_list = {
+      tensorflow::Flag("base", &base, "Path to the base serialized tensor."),
+      tensorflow::Flag("test", &test, "Path to the test serialized tensor."),
+      tensorflow::Flag("output", &output, "Path to the output file."),
+  };
+  tensorflow::Flags::Parse(&argc, argv, flag_list);
+
+  tflite::testing::DiffAnalyzer diff_analyzer;
+  diff_analyzer.ReadFiles(base, test);
+  diff_analyzer.WriteReport(output);
+  return 0;
+}
diff --git a/tensorflow/lite/testing/kernel_test/input_generator.cc b/tensorflow/lite/testing/kernel_test/input_generator.cc
new file mode 100644
index 0000000000..897e18560d
--- /dev/null
+++ b/tensorflow/lite/testing/kernel_test/input_generator.cc
@@ -0,0 +1,208 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/testing/kernel_test/input_generator.h"
+
+#include <fstream>
+#include <limits>
+#include <random>
+
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/testing/join.h"
+#include "tensorflow/lite/testing/split.h"
+
+namespace tflite {
+namespace testing {
+
+namespace {
+
+template <typename T>
+std::vector<T> GenerateRandomTensor(TfLiteIntArray* dims,
+                                    const std::function<T(int)>& random_func) {
+  int64_t num_elements = 1;
+  for (int i = 0; i < dims->size; i++) {
+    num_elements *= dims->data[i];
+  }
+
+  std::vector<T> result(num_elements);
+  for (int i = 0; i < num_elements; i++) {
+    result[i] = random_func(i);
+  }
+  return result;
+}
+
+template <typename T>
+std::vector<T> GenerateUniform(TfLiteIntArray* dims, float min, float max) {
+  auto random_float = [](float min, float max) {
+    // TODO(yunluli): Change seed for each invocation if needed.
+    static unsigned int seed;
+    return min + (max - min) * static_cast<float>(rand_r(&seed)) / RAND_MAX;
+  };
+
+  std::function<T(int)> random_t = [&](int) {
+    return static_cast<T>(random_float(min, max));
+  };
+  std::vector<T> data = GenerateRandomTensor(dims, random_t);
+  return data;
+}
+
+template <typename T>
+std::vector<T> GenerateGaussian(TfLiteIntArray* dims, float min, float max) {
+  auto random_float = [](float min, float max) {
+    static std::default_random_engine generator;
+    // We generate a float number within [0, 1) following a mormal distribution
+    // with mean = 0.5 and stddev = 1/3, and use it to scale the final random
+    // number into the desired range.
+    static std::normal_distribution<double> distribution(0.5, 1.0 / 3);
+    auto rand_n = distribution(generator);
+    while (rand_n < 0 || rand_n >= 1) {
+      rand_n = distribution(generator);
+    }
+
+    return min + (max - min) * static_cast<float>(rand_n);
+  };
+
+  std::function<T(int)> random_t = [&](int) {
+    return static_cast<T>(random_float(min, max));
+  };
+  std::vector<T> data = GenerateRandomTensor(dims, random_t);
+  return data;
+}
+
+}  // namespace
+
+TfLiteStatus InputGenerator::LoadModel(const string& model_dir) {
+  model_ = FlatBufferModel::BuildFromFile(model_dir.c_str());
+  if (!model_) {
+    fprintf(stderr, "Cannot load model %s", model_dir.c_str());
+    return kTfLiteError;
+  }
+
+  ::tflite::ops::builtin::BuiltinOpResolver builtin_ops;
+  InterpreterBuilder(*model_, builtin_ops)(&interpreter_);
+  if (!interpreter_) {
+    fprintf(stderr, "Failed to build interpreter.");
+    return kTfLiteError;
+  }
+
+  return kTfLiteOk;
+}
+
+TfLiteStatus InputGenerator::ReadInputsFromFile(const string& filename) {
+  if (filename.empty()) {
+    fprintf(stderr, "Empty input file name.");
+    return kTfLiteError;
+  }
+
+  std::ifstream input_file(filename);
+  string input;
+  while (std::getline(input_file, input, '\n')) {
+    inputs_.push_back(input);
+  }
+  input_file.close();
+  return kTfLiteOk;
+}
+
+TfLiteStatus InputGenerator::WriteInputsToFile(const string& filename) {
+  if (filename.empty()) {
+    fprintf(stderr, "Empty input file name.");
+    return kTfLiteError;
+  }
+
+  std::ofstream output_file;
+  output_file.open(filename, std::fstream::out | std::fstream::trunc);
+  if (!output_file) {
+    fprintf(stderr, "Failed to open output file %s.", filename.c_str());
+    return kTfLiteError;
+  }
+
+  for (const auto& input : inputs_) {
+    output_file << input << "\n";
+  }
+  output_file.close();
+
+  return kTfLiteOk;
+}
+
+// TODO(yunluli): Support more tensor types when needed.
+TfLiteStatus InputGenerator::GenerateInput(const string& distribution) {
+  auto input_tensor_ids = interpreter_->inputs();
+  for (auto id : input_tensor_ids) {
+    auto* tensor = interpreter_->tensor(id);
+    if (distribution == "UNIFORM") {
+      switch (tensor->type) {
+        case kTfLiteInt8: {
+          auto data = GenerateUniform<int8_t>(
+              tensor->dims, std::numeric_limits<int8_t>::min(),
+              std::numeric_limits<int8_t>::max());
+          inputs_.push_back(Join(data.data(), data.size(), ","));
+          break;
+        }
+        case kTfLiteUInt8: {
+          auto data = GenerateUniform<uint8_t>(
+              tensor->dims, std::numeric_limits<uint8_t>::min(),
+              std::numeric_limits<uint8_t>::max());
+          inputs_.push_back(Join(data.data(), data.size(), ","));
+          break;
+        }
+        case kTfLiteFloat32: {
+          auto data = GenerateUniform<float>(tensor->dims, -1, 1);
+          inputs_.push_back(JoinDefault(data.data(), data.size(), ","));
+          break;
+        }
+        default:
+          fprintf(stderr, "Unsupported input tensor type %s.",
+                  TfLiteTypeGetName(tensor->type));
+          break;
+      }
+    } else if (distribution == "GAUSSIAN") {
+      switch (tensor->type) {
+        case kTfLiteInt8: {
+          auto data = GenerateGaussian<int8_t>(
+              tensor->dims, std::numeric_limits<int8_t>::min(),
+              std::numeric_limits<int8_t>::max());
+          inputs_.push_back(Join(data.data(), data.size(), ","));
+          break;
+        }
+        case kTfLiteUInt8: {
+          auto data = GenerateGaussian<uint8_t>(
+              tensor->dims, std::numeric_limits<uint8_t>::min(),
+              std::numeric_limits<uint8_t>::max());
+          inputs_.push_back(Join(data.data(), data.size(), ","));
+          break;
+        }
+        case kTfLiteFloat32: {
+          auto data = GenerateGaussian<float>(tensor->dims, -1, 1);
+          inputs_.push_back(JoinDefault(data.data(), data.size(), ","));
+          break;
+        }
+        default:
+          fprintf(stderr, "Unsupported input tensor type %s.",
+                  TfLiteTypeGetName(tensor->type));
+          break;
+      }
+    } else {
+      fprintf(stderr, "Unsupported distribution %s.", distribution.c_str());
+      return kTfLiteError;
+    }
+  }
+
+  return kTfLiteOk;
+}
+
+std::vector<string> InputGenerator::GetInputs() { return inputs_; }
+
+}  // namespace testing
+}  // namespace tflite
diff --git a/tensorflow/lite/testing/kernel_test/input_generator.h b/tensorflow/lite/testing/kernel_test/input_generator.h
new file mode 100644
index 0000000000..859c7068e5
--- /dev/null
+++ b/tensorflow/lite/testing/kernel_test/input_generator.h
@@ -0,0 +1,50 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TESTING_KERNEL_TEST_INPUT_GENERATOR_H_
+#define TENSORFLOW_LITE_TESTING_KERNEL_TEST_INPUT_GENERATOR_H_
+
+#include <memory>
+#include <vector>
+
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/model.h"
+#include "tensorflow/lite/string.h"
+
+namespace tflite {
+namespace testing {
+
+// Generate random input, or read input from a file for kernel diff test.
+// Needs to load the tflite graph to get information like tensor shape and
+// data type.
+class InputGenerator {
+ public:
+  InputGenerator() = default;
+  TfLiteStatus LoadModel(const string& model_dir);
+  TfLiteStatus ReadInputsFromFile(const string& filename);
+  TfLiteStatus GenerateInput(const string& distribution);
+  std::vector<string> GetInputs();
+  TfLiteStatus WriteInputsToFile(const string& filename);
+
+ private:
+  std::unique_ptr<FlatBufferModel> model_;
+  std::unique_ptr<Interpreter> interpreter_;
+  std::vector<string> inputs_;
+};
+
+}  // namespace testing
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TESTING_KERNEL_TEST_INPUT_GENERATOR_H_
diff --git a/tensorflow/lite/testing/kernel_test/input_generator_test.cc b/tensorflow/lite/testing/kernel_test/input_generator_test.cc
new file mode 100644
index 0000000000..b6bd8b94fb
--- /dev/null
+++ b/tensorflow/lite/testing/kernel_test/input_generator_test.cc
@@ -0,0 +1,81 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/testing/kernel_test/input_generator.h"
+
+#include <fstream>
+#include <map>
+
+#include <gmock/gmock.h>
+#include "testing/base/public/googletest.h"
+#include <gtest/gtest.h>
+
+namespace tflite {
+namespace testing {
+
+namespace {
+
+TEST(InputGeneratorTest, LoadModel) {
+  InputGenerator input_generator;
+  ASSERT_EQ(input_generator.LoadModel(
+                "third_party/tensorflow/lite/testdata/multi_add.bin"),
+            kTfLiteOk);
+}
+
+TEST(InputGeneratorTest, ReadWriteSimpleFile) {
+  InputGenerator input_generator;
+  ASSERT_EQ(input_generator.ReadInputsFromFile(
+                "third_party/tensorflow/lite/testdata/test_input.csv"),
+            kTfLiteOk);
+
+  std::vector<string> inputs;
+  std::string content = "1";
+  for (int i = 0; i < 1 * 8 * 8 * 3 - 1; i++) {
+    content.append(",1");
+  }
+  inputs.push_back(content);
+  ASSERT_EQ(input_generator.GetInputs(), inputs);
+
+  auto output_filename = FLAGS_test_tmpdir + "/out.csv";
+  ASSERT_EQ(input_generator.WriteInputsToFile(output_filename), kTfLiteOk);
+
+  std::ifstream in(output_filename);
+  std::string out;
+  std::getline(in, out, '\n');
+  ASSERT_EQ(out, content);
+}
+
+TEST(InputGeneratorTest, GenerateUniformInput) {
+  InputGenerator input_generator;
+  ASSERT_EQ(input_generator.LoadModel(
+                "third_party/tensorflow/lite/testdata/multi_add.bin"),
+            kTfLiteOk);
+  input_generator.GenerateInput("UNIFORM");
+  auto inputs = input_generator.GetInputs();
+  ASSERT_EQ(inputs.size(), 4);
+}
+
+TEST(InputGeneratorTest, GenerateGaussianInput) {
+  InputGenerator input_generator;
+  ASSERT_EQ(input_generator.LoadModel(
+                "third_party/tensorflow/lite/testdata/multi_add.bin"),
+            kTfLiteOk);
+  input_generator.GenerateInput("GAUSSIAN");
+  auto inputs = input_generator.GetInputs();
+  ASSERT_EQ(inputs.size(), 4);
+}
+
+}  // namespace
+}  // namespace testing
+}  // namespace tflite
diff --git a/tensorflow/lite/testing/kernel_test/tflite_kernel_runner.cc b/tensorflow/lite/testing/kernel_test/tflite_kernel_runner.cc
new file mode 100644
index 0000000000..34c1728ed1
--- /dev/null
+++ b/tensorflow/lite/testing/kernel_test/tflite_kernel_runner.cc
@@ -0,0 +1,32 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/testing/kernel_test/util.h"
+
+int main(int argc, char** argv) {
+  tflite::testing::kernel_test::TestOptions options =
+      tflite::testing::kernel_test::ParseTfliteKernelTestFlags(&argc, argv);
+  const bool run_reference_kernel = options.kernel_type == "REFERENCE";
+  const bool use_nnapi = options.kernel_type == "NNAPI";
+
+  auto runner = absl::make_unique<tflite::testing::TfLiteDriver>(
+      use_nnapi, "", run_reference_kernel);
+  if (tflite::testing::kernel_test::RunKernelTest(options, runner.get()) ==
+      kTfLiteOk) {
+    return 0;
+  }
+
+  return -1;
+}
diff --git a/tensorflow/lite/testing/kernel_test/util.h b/tensorflow/lite/testing/kernel_test/util.h
new file mode 100644
index 0000000000..d940e5ad12
--- /dev/null
+++ b/tensorflow/lite/testing/kernel_test/util.h
@@ -0,0 +1,122 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TESTING_KERNEL_TEST_UTIL_H_
+#define TENSORFLOW_LITE_TESTING_KERNEL_TEST_UTIL_H_
+
+#include <fstream>
+
+#include "tensorflow/core/util/command_line_flags.h"
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/testing/kernel_test/input_generator.h"
+#include "tensorflow/lite/testing/split.h"
+#include "tensorflow/lite/testing/tflite_driver.h"
+
+namespace tflite {
+namespace testing {
+namespace kernel_test {
+
+struct TestOptions {
+  // Path of tensorflow lite model.
+  string tflite_model;
+  // Path of the input file. If empty, generate at runtime.
+  string read_input_from_file;
+  // Path to dump the input file.
+  string dump_input_to_file;
+  // Path to dump the output.
+  string dump_output_to_file;
+  // Input distribution.
+  string input_distribution;
+  // Kernel type.
+  string kernel_type;
+};
+
+TestOptions ParseTfliteKernelTestFlags(int* argc, char** argv) {
+  TestOptions options;
+  std::vector<tensorflow::Flag> flags = {
+      tensorflow::Flag("tflite_model", &options.tflite_model,
+                       "Path of tensorflow lite model."),
+      tensorflow::Flag("read_input_from_file", &options.read_input_from_file,
+                       "File to read input data from. If empty, generates "
+                       "input at runtime."),
+      tensorflow::Flag("dump_input_to_file", &options.dump_input_to_file,
+                       "File to dump randomly generated input."),
+      tensorflow::Flag("dump_output_to_file", &options.dump_output_to_file,
+                       "File to dump output."),
+      tensorflow::Flag("input_distribution", &options.input_distribution,
+                       "Input distribution. Default: Gaussian."),
+      tensorflow::Flag("kernel_type", &options.kernel_type, "Kernel type."),
+  };
+
+  tensorflow::Flags::Parse(argc, argv, flags);
+
+  return options;
+}
+
+TfLiteStatus RunKernelTest(const kernel_test::TestOptions& options,
+                           TestRunner* runner) {
+  InputGenerator input_generator;
+
+  if (options.read_input_from_file.empty()) {
+    TF_LITE_ENSURE_STATUS(input_generator.LoadModel(options.tflite_model));
+    TF_LITE_ENSURE_STATUS(
+        input_generator.GenerateInput(options.input_distribution));
+  } else {
+    TF_LITE_ENSURE_STATUS(
+        input_generator.ReadInputsFromFile(options.read_input_from_file));
+  }
+
+  runner->LoadModel(options.tflite_model);
+  runner->AllocateTensors();
+  if (!runner->IsValid()) return kTfLiteError;
+  auto input_tensor_ids = runner->GetInputs();
+  auto inputs = input_generator.GetInputs();
+  if (inputs.size() != input_tensor_ids.size()) {
+    fprintf(stderr,
+            "Number of input tensors generated doesn't match what the model "
+            "asks for.");
+  }
+  for (int i = 0; i < inputs.size(); i++) {
+    runner->SetInput(input_tensor_ids[i], inputs[i]);
+  }
+
+  runner->Invoke();
+
+  if (!options.dump_input_to_file.empty()) {
+    TF_LITE_ENSURE_STATUS(
+        input_generator.WriteInputsToFile(options.dump_input_to_file));
+  }
+
+  if (!options.dump_output_to_file.empty()) {
+    std::ofstream output_file;
+    output_file.open(options.dump_output_to_file,
+                     std::fstream::out | std::fstream::trunc);
+    if (!output_file) {
+      return kTfLiteError;
+    }
+
+    for (auto id : runner->GetOutputs()) {
+      output_file << runner->ReadOutput(id) << "\n";
+    }
+    output_file.close();
+  }
+
+  return kTfLiteOk;
+}
+
+}  // namespace kernel_test
+}  // namespace testing
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TESTING_KERNEL_TEST_UTIL_H_
diff --git a/tensorflow/lite/testing/kernel_test/util_test.cc b/tensorflow/lite/testing/kernel_test/util_test.cc
new file mode 100644
index 0000000000..751c77e136
--- /dev/null
+++ b/tensorflow/lite/testing/kernel_test/util_test.cc
@@ -0,0 +1,52 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/testing/kernel_test/util.h"
+
+#include <fstream>
+#include <memory>
+
+#include <gmock/gmock.h>
+#include "testing/base/public/googletest.h"
+#include <gtest/gtest.h>
+#include "tensorflow/lite/testing/tflite_driver.h"
+
+namespace tflite {
+namespace testing {
+namespace kernel_test {
+namespace {
+
+TEST(UtilTest, SimpleE2ETest) {
+  TestOptions options;
+  options.tflite_model = "third_party/tensorflow/lite/testdata/add.bin";
+  options.read_input_from_file =
+      "third_party/tensorflow/lite/testdata/test_input.csv";
+  options.dump_output_to_file = FLAGS_test_tmpdir + "/test_out.csv";
+  options.kernel_type = "REFERENCE";
+  std::unique_ptr<TestRunner> runner(new TfLiteDriver(false, "", true));
+  RunKernelTest(options, runner.get());
+  std::string expected = "3";
+  for (int i = 0; i < 1 * 8 * 8 * 3 - 1; i++) {
+    expected.append(",3");
+  }
+  std::string content;
+  std::ifstream file(options.dump_output_to_file);
+  std::getline(file, content);
+  EXPECT_EQ(content, expected);
+}
+
+}  // namespace
+}  // namespace kernel_test
+}  // namespace testing
+}  // namespace tflite
diff --git a/tensorflow/lite/testing/tflite_driver.cc b/tensorflow/lite/testing/tflite_driver.cc
index a637dc86c0..5567085833 100644
--- a/tensorflow/lite/testing/tflite_driver.cc
+++ b/tensorflow/lite/testing/tflite_driver.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include "tensorflow/lite/kernels/register.h"
 #include "tensorflow/lite/kernels/register_ref.h"
 #include "tensorflow/lite/string_util.h"
+#include "tensorflow/lite/testing/join.h"
 #include "tensorflow/lite/testing/split.h"
 
 namespace tflite {
@@ -383,5 +384,34 @@ void TfLiteDriver::ResetLSTMStateTensors() {
   interpreter_->ResetVariableTensors();
 }
 
+string TfLiteDriver::ReadOutput(int id) {
+  auto* tensor = interpreter_->tensor(id);
+  int num_elements = 1;
+
+  for (int i = 0; i < tensor->dims->size; ++i) {
+    num_elements *= tensor->dims->data[i];
+  }
+
+  switch (tensor->type) {
+    case kTfLiteFloat32:
+      return JoinDefault(tensor->data.f, num_elements, ",");
+    case kTfLiteInt32:
+      return JoinDefault(tensor->data.i32, num_elements, ",");
+    case kTfLiteInt64:
+      return JoinDefault(tensor->data.i64, num_elements, ",");
+    case kTfLiteUInt8:
+      return Join(tensor->data.uint8, num_elements, ",");
+    case kTfLiteInt8:
+      return JoinDefault(tensor->data.int8, num_elements, ",");
+    case kTfLiteBool:
+      return JoinDefault(tensor->data.b, num_elements, ",");
+    default:
+      Invalidate(absl::StrCat("Unsupported tensor type ",
+                              TfLiteTypeGetName(tensor->type),
+                              " in TfLiteDriver::ReadOutput"));
+      return "";
+  }
+}
+
 }  // namespace testing
 }  // namespace tflite
diff --git a/tensorflow/lite/testing/tflite_driver.h b/tensorflow/lite/testing/tflite_driver.h
index 537f20dfbf..3cce6c4222 100644
--- a/tensorflow/lite/testing/tflite_driver.h
+++ b/tensorflow/lite/testing/tflite_driver.h
@@ -49,7 +49,7 @@ class TfLiteDriver : public TestRunner {
   void SetExpectation(int id, const string& csv_values) override;
   void Invoke() override;
   bool CheckResults() override;
-  string ReadOutput(int id) override { return "no-op"; }
+  string ReadOutput(int id) override;
 
  private:
   void DeallocateStringTensor(TfLiteTensor* t) {
diff --git a/tensorflow/lite/testing/tflite_driver_test.cc b/tensorflow/lite/testing/tflite_driver_test.cc
index 81bf6700cb..e80816bdf5 100644
--- a/tensorflow/lite/testing/tflite_driver_test.cc
+++ b/tensorflow/lite/testing/tflite_driver_test.cc
@@ -54,6 +54,8 @@ TEST(TfliteDriverTest, SimpleTest) {
   ASSERT_TRUE(runner->IsValid());
 
   ASSERT_TRUE(runner->CheckResults());
+  EXPECT_EQ(runner->ReadOutput(5), "0.101,0.202,0.303,0.404");
+  EXPECT_EQ(runner->ReadOutput(6), "0.011,0.022,0.033,0.044");
 }
 
 TEST(TfliteDriverTest, SingleAddOpTest) {
@@ -88,6 +90,8 @@ TEST(TfliteDriverTest, SingleAddOpTest) {
   ASSERT_TRUE(runner->IsValid());
 
   ASSERT_TRUE(runner->CheckResults());
+  EXPECT_EQ(runner->ReadOutput(5), "0.101,0.202,0.303,0.404");
+  EXPECT_EQ(runner->ReadOutput(6), "0.011,0.022,0.033,0.044");
 }
 
 }  // namespace
-- 
GitLab


From 06bbefd8675f721096daf65b7334d175121ebef0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 8 Feb 2019 11:00:12 -0800
Subject: [PATCH 2159/2345] Leave ownership of SharedCounter with
 GPUProcessState, when it's being used by GPUBFCAllocators, instead of
 transfering it to GPUKernelTracker. GPUProcessState has a process lifespan,
 but in some circumstances GPUDevice (and its owned GPUKernelTracker) are
 recreated for new Sessions.

PiperOrigin-RevId: 233089978
---
 .../core/common_runtime/gpu/gpu_device.cc     | 18 +++++++----------
 .../core/common_runtime/gpu/gpu_device.h      | 20 +++++++++++++------
 .../common_runtime/gpu/gpu_device_test.cc     |  7 +++----
 .../common_runtime/gpu/gpu_process_state.cc   |  5 ++---
 .../common_runtime/gpu/gpu_process_state.h    |  2 +-
 5 files changed, 27 insertions(+), 25 deletions(-)

diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.cc b/tensorflow/core/common_runtime/gpu/gpu_device.cc
index 80d221af36..179f6132fa 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.cc
@@ -280,23 +280,19 @@ BaseGPUDevice::BaseGPUDevice(const SessionOptions& options, const string& name,
   timestamped_allocator_ =
       options.config.gpu_options().experimental().timestamped_allocator();
   if (timestamped_allocator_ || pending_cap_ > 0) {
-    std::unique_ptr<SharedCounter> timing_counter;
+    SharedCounter* timing_counter = nullptr;
     if (timestamped_allocator_) {
       // In this case the SharedCounter was already created and set in the
-      // associated Allocator, with ownership by GPUProcessState.  Here we take
-      // over ownership of that SharedAllocator to transfer it to the
-      // GPUKernelTracker.
+      // associated Allocator, with ownership by GPUProcessState.
+      // The GPUKernelTracker will use this SharedCounter, instead of
+      // owning its own.
       timing_counter =
-          GPUProcessState::singleton()->ReleaseGPUAllocatorCounter(tf_gpu_id);
-      DCHECK(timing_counter.get());
+          GPUProcessState::singleton()->GPUAllocatorCounter(tf_gpu_id);
+      DCHECK(timing_counter);
     } else {
       DCHECK_GT(pending_cap_, 0);
-      // In this case we need a SharedCounter to be owned by GPUKernelTracker
-      // but one was not created for use by the Allocator, so we create one.
-      timing_counter.reset(new SharedCounter);
     }
-    kernel_tracker_.reset(
-        new GPUKernelTracker(Env::Default(), std::move(timing_counter)));
+    kernel_tracker_.reset(new GPUKernelTracker(Env::Default(), timing_counter));
   }
 }
 
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.h b/tensorflow/core/common_runtime/gpu/gpu_device.h
index 33f0585b4d..f8f2a2e2f3 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.h
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.h
@@ -184,11 +184,18 @@ class BaseGPUDevice : public LocalDevice {
 // time of the most recently terminated kernel.
 class GPUKernelTracker {
  public:
-  explicit GPUKernelTracker(Env* env,
-                            std::unique_ptr<SharedCounter> timing_counter)
-      : env_(env),
-        timing_counter_(std::move(timing_counter)),
-        pending_kernels_(64) {}
+  // If we're going to share a SharedCounter with an allocator, it's owned
+  // by the allocator because allocators are initialized once per process.
+  // Devices are per-session.
+  explicit GPUKernelTracker(Env* env, SharedCounter* timing_counter)
+      : env_(env), timing_counter_(timing_counter), pending_kernels_(64) {
+    if (!timing_counter_) {
+      // There's not a preexisting counter owned by GPUProcessState, i.e.
+      // pending_cap > 0 but timestamped_allocator == false.
+      owned_counter_.reset(new SharedCounter);
+      timing_counter_ = owned_counter_.get();
+    }
+  }
 
   // Record that a GPU kernel has just been enqueued on the compute stream.
   // Inserts a new timing counter value in a new PendingKernel record appended
@@ -222,7 +229,8 @@ class GPUKernelTracker {
 
  private:
   Env* env_;
-  std::unique_ptr<SharedCounter> timing_counter_;
+  SharedCounter* timing_counter_;
+  std::unique_ptr<SharedCounter> owned_counter_;
 
   // Records when a kernel was queued for execution.  Kernel launches are
   // identified by a unique count value from a per-GPU device timing counter.
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device_test.cc b/tensorflow/core/common_runtime/gpu/gpu_device_test.cc
index fba937ac48..2628cd413f 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device_test.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device_test.cc
@@ -280,14 +280,13 @@ TEST_F(GPUDeviceTest, UnifiedMemoryAllocation) {
 class GPUKernelTrackerTest : public ::testing::Test {
  protected:
   void SetUp() {
-    std::unique_ptr<SharedCounter> counter(new SharedCounter);
-    timing_counter_ = counter.get();
+    timing_counter_.reset(new SharedCounter);
     kernel_tracker_.reset(
-        new GPUKernelTracker(Env::Default(), std::move(counter)));
+        new GPUKernelTracker(Env::Default(), timing_counter_.get()));
   }
 
   std::unique_ptr<GPUKernelTracker> kernel_tracker_;
-  SharedCounter* timing_counter_ = nullptr;
+  std::unique_ptr<SharedCounter> timing_counter_;
 };
 
 TEST_F(GPUKernelTrackerTest, basic) {
diff --git a/tensorflow/core/common_runtime/gpu/gpu_process_state.cc b/tensorflow/core/common_runtime/gpu/gpu_process_state.cc
index 7804596d1a..39883d34f9 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_process_state.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_process_state.cc
@@ -160,8 +160,7 @@ Allocator* GPUProcessState::GetGPUAllocator(const GPUOptions& options,
 #endif  // GOOGLE_CUDA
 }
 
-std::unique_ptr<SharedCounter> GPUProcessState::ReleaseGPUAllocatorCounter(
-    TfGpuId tf_gpu_id) {
+SharedCounter* GPUProcessState::GPUAllocatorCounter(TfGpuId tf_gpu_id) {
   DCHECK(process_state_);
 #if GOOGLE_CUDA
   GpuIdUtil::CheckValidTfGpuId(tf_gpu_id);
@@ -171,7 +170,7 @@ std::unique_ptr<SharedCounter> GPUProcessState::ReleaseGPUAllocatorCounter(
   }
 
   AllocatorParts& allocator_parts = gpu_allocators_[tf_gpu_id.value()];
-  return std::move(allocator_parts.counter);
+  return allocator_parts.counter.get();
 #else
   return nullptr;
 #endif
diff --git a/tensorflow/core/common_runtime/gpu/gpu_process_state.h b/tensorflow/core/common_runtime/gpu/gpu_process_state.h
index c7c9f3a6b3..861157ca0c 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_process_state.h
+++ b/tensorflow/core/common_runtime/gpu/gpu_process_state.h
@@ -110,7 +110,7 @@ class GPUProcessState {
   // Returns bus_id for the given GPU id.
   virtual int BusIdForGPU(TfGpuId tf_gpu_id);
 
-  std::unique_ptr<SharedCounter> ReleaseGPUAllocatorCounter(TfGpuId tf_gpu_id);
+  SharedCounter* GPUAllocatorCounter(TfGpuId tf_gpu_id);
 
  protected:
   // GPUProcessState is a singleton that should not normally be deleted except
-- 
GitLab


From 8448467496587ad483b250bf688fe2d966c53aee Mon Sep 17 00:00:00 2001
From: Jeffrey Poznanovic <Jeffrey.Poznanovic@amd.com>
Date: Fri, 8 Feb 2019 11:44:30 -0800
Subject: [PATCH 2160/2345] Misc updates to the ROCm-specific ci_build files
 Dockerfile.rocm and rocm/run_py3_core.sh

---
 tensorflow/tools/ci_build/Dockerfile.rocm     | 15 ++--
 .../tools/ci_build/linux/rocm/run_py3_core.sh | 70 +++++++++++++++++--
 2 files changed, 70 insertions(+), 15 deletions(-)

diff --git a/tensorflow/tools/ci_build/Dockerfile.rocm b/tensorflow/tools/ci_build/Dockerfile.rocm
index aadaa8bac1..f190199643 100644
--- a/tensorflow/tools/ci_build/Dockerfile.rocm
+++ b/tensorflow/tools/ci_build/Dockerfile.rocm
@@ -44,9 +44,12 @@ RUN apt-get update --allow-insecure-repositories && DEBIAN_FRONTEND=noninteracti
   libboost-filesystem-dev \
   rpm \
   libnuma-dev \
+  pciutils \
   virtualenv \
   python-pip \
   python3-pip \
+  libxml2 \
+  libxml2-dev \
   wget && \
   apt-get clean && \
   rm -rf /var/lib/apt/lists/*
@@ -54,15 +57,12 @@ RUN apt-get update --allow-insecure-repositories && DEBIAN_FRONTEND=noninteracti
 # Install rocm pkgs
 RUN apt-get update --allow-insecure-repositories && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \
-    rocm-dev rocm-libs rocm-utils \
+    rocm-dev rocm-libs rocm-utils rocm-cmake \
     rocfft miopen-hip miopengemm rocblas hipblas rocrand \
     rocm-profiler cxlactivitylogger && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-RUN cd ~ && git clone https://github.com/GPUOpen-ProfessionalCompute-Tools/HIP.git
-RUN cd ~/HIP && mkdir -p build && cd build && cmake .. && make package -j && dpkg -i *.deb
-
 ENV HCC_HOME=$ROCM_PATH/hcc
 ENV HIP_PATH=$ROCM_PATH/hip
 ENV OPENCL_ROOT=$ROCM_PATH/opencl
@@ -71,12 +71,7 @@ ENV PATH="$ROCM_PATH/bin:${PATH}"
 ENV PATH="$OPENCL_ROOT/bin:${PATH}"
 
 # Add target file to help determine which device(s) to build for
-RUN echo -e "gfx803\ngfx900" >> /opt/rocm/bin/target.lst
-
-# Setup environment variables, and add those environment variables at the end of ~/.bashrc 
-ARG HCC_HOME=/opt/rocm/hcc
-ARG HIP_PATH=/opt/rocm/hip
-ARG PATH=$HCC_HOME/bin:$HIP_PATH/bin:$PATH
+RUN bash -c 'echo -e "gfx803\ngfx900\ngfx906" >> /opt/rocm/bin/target.lst'
 
 # Copy and run the install scripts.
 COPY install/*.sh /install/
diff --git a/tensorflow/tools/ci_build/linux/rocm/run_py3_core.sh b/tensorflow/tools/ci_build/linux/rocm/run_py3_core.sh
index 1d0b838c1b..00a46af711 100755
--- a/tensorflow/tools/ci_build/linux/rocm/run_py3_core.sh
+++ b/tensorflow/tools/ci_build/linux/rocm/run_py3_core.sh
@@ -19,9 +19,10 @@ set -e
 set -x
 
 N_JOBS=$(grep -c ^processor /proc/cpuinfo)
+N_GPUS=$(lspci|grep 'VGA'|grep 'AMD/ATI'|wc -l)
 
 echo ""
-echo "Bazel will use ${N_JOBS} concurrent job(s)."
+echo "Bazel will use ${N_JOBS} concurrent build job(s) and ${N_GPUS} concurrent test job(s)."
 echo ""
 
 # Run configure.
@@ -29,11 +30,70 @@ export PYTHON_BIN_PATH=`which python3`
 export CC_OPT_FLAGS='-mavx'
 
 export TF_NEED_ROCM=1
+export TF_GPU_COUNT=${N_GPUS}
 
 yes "" | $PYTHON_BIN_PATH configure.py
 
 # Run bazel test command. Double test timeouts to avoid flakes.
-bazel test --config=rocm --test_tag_filters=-no_oss,-oss_serial,-no_gpu,-benchmark-test -k \
-    --test_lang_filters=py --jobs=${N_JOBS} --test_timeout 300,450,1200,3600 \
-    --build_tests_only --test_output=errors --local_test_jobs=1 --config=opt \
-    //tensorflow/... -//tensorflow/compiler/... -//tensorflow/contrib/...
+bazel test --test_sharding_strategy=disabled --config=rocm --test_tag_filters=-no_oss,-oss_serial,-no_gpu,-benchmark-test -k \
+    --test_lang_filters=py --jobs=${N_JOBS} --test_timeout 600,900,2400,7200 \
+    --build_tests_only --test_output=errors --local_test_jobs=${TF_GPU_COUNT} --config=opt \
+    --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute -- \
+    //tensorflow/... -//tensorflow/compiler/... -//tensorflow/contrib/... \
+    -//tensorflow/python/distribute:distribute_coordinator_test \
+    -//tensorflow/python/feature_column:feature_column_v2_test \
+    -//tensorflow/python/keras:cudnn_recurrent_test \
+    -//tensorflow/python/keras:cudnn_recurrent_test_gpu \
+    -//tensorflow/python/kernel_tests:batch_matmul_op_test \
+    -//tensorflow/python/kernel_tests:batch_matmul_op_test_gpu \
+    -//tensorflow/python/kernel_tests:conv1d_test \
+    -//tensorflow/python/kernel_tests:conv1d_test_gpu \
+    -//tensorflow/python/kernel_tests:conv2d_backprop_filter_grad_test \
+    -//tensorflow/python/kernel_tests:conv2d_backprop_filter_grad_test_gpu \
+    -//tensorflow/python/kernel_tests:conv_ops_3d_test \
+    -//tensorflow/python/kernel_tests:conv_ops_3d_test_gpu \
+    -//tensorflow/python/kernel_tests:conv_ops_test \
+    -//tensorflow/python/kernel_tests:conv_ops_test_gpu \
+    -//tensorflow/python/kernel_tests:depthwise_conv_op_test \
+    -//tensorflow/python/kernel_tests:depthwise_conv_op_test_gpu \
+    -//tensorflow/python/kernel_tests:list_ops_test \
+    -//tensorflow/python/kernel_tests:list_ops_test_gpu \
+    -//tensorflow/python/kernel_tests:matmul_op_test \
+    -//tensorflow/python/kernel_tests:matmul_op_test_gpu \
+    -//tensorflow/python/kernel_tests:pool_test \
+    -//tensorflow/python/kernel_tests:pool_test_gpu \
+    -//tensorflow/python/kernel_tests:pooling_ops_3d_test \
+    -//tensorflow/python/kernel_tests:pooling_ops_3d_test_gpu \
+    -//tensorflow/python/kernel_tests:pooling_ops_test \
+    -//tensorflow/python/kernel_tests:pooling_ops_test_gpu \
+    -//tensorflow/python/ops/parallel_for:control_flow_ops_test \
+    -//tensorflow/python/ops/parallel_for:gradients_test \
+    -//tensorflow/python:layers_normalization_test \
+    -//tensorflow/python:layers_normalization_test_gpu \
+    -//tensorflow/python:layout_optimizer_test \
+    -//tensorflow/python:nn_fused_batchnorm_test \
+    -//tensorflow/python:nn_fused_batchnorm_test_gpu \
+    -//tensorflow/python/eager:backprop_test \
+    -//tensorflow/python/eager:backprop_test_gpu \
+    -//tensorflow/python/kernel_tests:qr_op_test \
+    -//tensorflow/python/kernel_tests:qr_op_test_gpu \
+    -//tensorflow/python/kernel_tests/signal:dct_ops_test \
+    -//tensorflow/python/kernel_tests/signal:fft_ops_test \
+    -//tensorflow/python/kernel_tests/signal:spectral_ops_test \
+    -//tensorflow/python/ops/parallel_for:control_flow_ops_test_gpu \
+    -//tensorflow/python/kernel_tests:tensor_array_ops_test \
+    -//tensorflow/python/kernel_tests:tensor_array_ops_test_gpu \
+    -//tensorflow/python/keras:unified_lstm_test \
+    -//tensorflow/python/keras:unified_lstm_test_gpu \
+    -//tensorflow/python/keras:unified_gru_test \
+    -//tensorflow/python/keras:unified_gru_test_gpu \
+    -//tensorflow/python/keras:training_generator_test \
+    -//tensorflow/python/keras/optimizer_v2:adamax_test \
+    -//tensorflow/python/keras/optimizer_v2:adamax_test_gpu \
+    -//tensorflow/tools/api/tests:api_compatibility_test \
+    -//tensorflow/python/training/checkpointable:util_xla_test_cpu \
+    -//tensorflow/python/kernel_tests/signal:mfcc_ops_test \
+    -//tensorflow/python/kernel_tests:self_adjoint_eig_op_test_gpu \
+    -//tensorflow/python/keras:normalization_test \
+    -//tensorflow/python/keras:base_layer_test \
+    -//tensorflow/python/eager:def_function_xla_test_cpu \
-- 
GitLab


From c66e72e4dcef0ce30fe984b2c24296071cf9fcca Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 8 Feb 2019 11:01:18 -0800
Subject: [PATCH 2161/2345] BUILD fix

PiperOrigin-RevId: 233090248
---
 tensorflow/core/profiler/rpc/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/core/profiler/rpc/BUILD b/tensorflow/core/profiler/rpc/BUILD
index 6b4576ec9b..3e1c654f5b 100644
--- a/tensorflow/core/profiler/rpc/BUILD
+++ b/tensorflow/core/profiler/rpc/BUILD
@@ -30,6 +30,7 @@ tf_cuda_library(
         "//tensorflow/contrib/tpu/profiler:tpu_profiler_proto_cc",
         "//tensorflow/core:framework",
         "//tensorflow/core/common_runtime/eager:context",
+        "//tensorflow/core/profiler/lib:profiler_session",
     ],
     alwayslink = 1,
 )
-- 
GitLab


From 4ee206a60f8a2d14a55ad1a8767a9f78ad027913 Mon Sep 17 00:00:00 2001
From: Billy Lamberta <blamb@google.com>
Date: Fri, 8 Feb 2019 11:03:49 -0800
Subject: [PATCH 2162/2345] Rename to tiled_layout.md and add to nav

PiperOrigin-RevId: 233090869
---
 tensorflow/compiler/xla/g3doc/_book.yaml                    | 2 ++
 .../xla/g3doc/{layout_with_tiling.md => tiled_layout.md}    | 6 ++----
 2 files changed, 4 insertions(+), 4 deletions(-)
 rename tensorflow/compiler/xla/g3doc/{layout_with_tiling.md => tiled_layout.md} (96%)

diff --git a/tensorflow/compiler/xla/g3doc/_book.yaml b/tensorflow/compiler/xla/g3doc/_book.yaml
index 267701e9c0..d756cd74c9 100644
--- a/tensorflow/compiler/xla/g3doc/_book.yaml
+++ b/tensorflow/compiler/xla/g3doc/_book.yaml
@@ -25,6 +25,8 @@ upper_tabs:
         path: /xla/operation_semantics
       - title: Shapes and layout
         path: /xla/shapes
+      - title: Tiled layout
+        path: /xla/tiled_layout
       - title: Using AOT compilation
         path: /xla/tfcompile
       - heading: Tutorials
diff --git a/tensorflow/compiler/xla/g3doc/layout_with_tiling.md b/tensorflow/compiler/xla/g3doc/tiled_layout.md
similarity index 96%
rename from tensorflow/compiler/xla/g3doc/layout_with_tiling.md
rename to tensorflow/compiler/xla/g3doc/tiled_layout.md
index 5e990851af..21e88ceab6 100644
--- a/tensorflow/compiler/xla/g3doc/layout_with_tiling.md
+++ b/tensorflow/compiler/xla/g3doc/tiled_layout.md
@@ -1,9 +1,7 @@
 # Tiled layout
 
-*Note: This doc describes how tiled layout is intended to work. Tiling is being
-implemented, but this is an early effort and it is currently not even guaranteed
-to get an Unimplemented error if one tries to use tiling - it may be just
-silently ignored.*
+Caution: Tiled layout is *pre-release* and this describes how it's intended to
+work. Errors may be silently ignored.
 
 <center> ![](images/xla_array_layout_figure1.png)
 
-- 
GitLab


From 6f6acd41e6f6f19754397f5a4d0d80b61dfc66ae Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 8 Feb 2019 11:07:49 -0800
Subject: [PATCH 2163/2345] Make embedding variable names with TPU embedding
 consistent with the ones on CPU.

PiperOrigin-RevId: 233091645
---
 .../python/tpu/_tpu_estimator_embedding.py    | 74 ++++++++++++++++++-
 .../contrib/tpu/python/tpu/tpu_embedding.py   |  8 ++
 .../contrib/tpu/python/tpu/tpu_estimator.py   | 21 +++++-
 3 files changed, 99 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/_tpu_estimator_embedding.py b/tensorflow/contrib/tpu/python/tpu/_tpu_estimator_embedding.py
index 2b71ce6fd8..9a927c637e 100644
--- a/tensorflow/contrib/tpu/python/tpu/_tpu_estimator_embedding.py
+++ b/tensorflow/contrib/tpu/python/tpu/_tpu_estimator_embedding.py
@@ -36,6 +36,77 @@ _SUPPORTED_FEATURE_COLUMNS = (core_fc._NumericColumn, core_fc_lib.NumericColumn)
 
 # pylint: enable=protected-access
 
+_TABLE_NAME_PREFIX = 'tbl_'
+_LEN_TABLE_NAME_PREFIX = len(_TABLE_NAME_PREFIX)
+
+
+def _get_table_name_from_embedding_var_name(embedding_var_name):
+  return '{}{}'.format(_TABLE_NAME_PREFIX, embedding_var_name)
+
+
+def _get_embedding_var_name_from_table_name(table_name):
+  return table_name[_LEN_TABLE_NAME_PREFIX:]
+
+
+def _get_embedding_variable_name(scope_name, var_name):
+  return '{}/{}'.format(scope_name, var_name)
+
+
+def _get_slot_variable_names(scope_name, var_name, optimization_parameters):
+  """Return embedding variable names which are consistent with CPU runs."""
+  if isinstance(optimization_parameters, tpu_embedding.AdagradParameters):
+    return tpu_embedding.AdagradSlotVariableName(
+        '{}/{}/Adagrad'.format(scope_name, var_name)
+    )
+  elif isinstance(optimization_parameters, tpu_embedding.AdamParameters):
+    return tpu_embedding.AdamSlotVariableNames(
+        '{}/{}/Adam/m'.format(scope_name, var_name),
+        '{}/{}/Adam/v'.format(scope_name, var_name)
+    )
+  elif isinstance(optimization_parameters,
+                  tpu_embedding.StochasticGradientDescentParameters):
+    return None
+  else:
+    raise ValueError('Support to infer full variable name '
+                     'for optimization_parameter {} has not been added.'
+                     .format(optimization_parameters))
+
+
+def get_full_variable_names(
+    graph, table_to_config_dict, optimization_parameters):
+  """Return embedding variable names and slot variables which are consistent with CPU runs."""
+  collection = graph.get_collection_ref(tpu_fc._TPU_FC_TO_SCOPE)  # pylint: disable=protected-access
+  if not collection:
+    raise RuntimeError(
+        'Embedding feature column did not capture any thing. Make sure the '
+        'feature columns passed to TPUEstimator constructor is properly '
+        'used in model_fn.')
+
+  embedding_variable_name_by_table = {}
+  slot_variable_names_by_table = {}
+  for table_name in table_to_config_dict:
+    embedding_var_name = _get_embedding_var_name_from_table_name(table_name)
+    (scope_name, var_name) = collection[0][embedding_var_name]
+
+    if table_name in embedding_variable_name_by_table:
+      existed_scope_name, existed_var_name = (
+          embedding_variable_name_by_table[table_name])
+      if existed_scope_name != scope_name or existed_var_name != var_name:
+        raise RuntimeError(
+            'Internal error: For table {}, scope and/or var_name mismatch(es). '
+            'expected scope {} var_name {}; got scope {} var_name {}'.format(
+                table_name, existed_scope_name, existed_var_name, scope_name,
+                var_name))
+    else:
+      embedding_variable_name_by_table[table_name] = (
+          _get_embedding_variable_name(scope_name, var_name)
+      )
+      slot_variable_names_by_table[table_name] = _get_slot_variable_names(
+          scope_name, var_name, optimization_parameters)
+
+  graph.clear_collection(tpu_fc._TPU_FC_TO_SCOPE)  # pylint: disable=protected-access
+  return embedding_variable_name_by_table, slot_variable_names_by_table
+
 
 def get_tpu_embedding_config_from_feature_columns(feature_columns):
   """Create configs for TPUEmbedding from a list of feature columns.
@@ -63,7 +134,8 @@ def get_tpu_embedding_config_from_feature_columns(feature_columns):
   feature_to_table = {}
   for column in feature_columns:
     feature_name = column.get_feature_key_name()
-    table_name = 'tbl_{}'.format(column.get_embedding_var_name())
+    table_name = _get_table_name_from_embedding_var_name(
+        column.get_embedding_var_name())
     if feature_name in feature_to_table:
       raise ValueError(
           'Feature column {} is used with multiple embeddings and this is '
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py b/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py
index 2fb0ae9517..eb99a18d83 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py
@@ -454,10 +454,18 @@ class TPUEmbedding(object):
     """
     return self._init_ops
 
+  @property
+  def table_to_config_dict(self):
+    return copy.copy(self._table_to_config_dict)
+
   @property
   def feature_to_table_dict(self):
     return copy.copy(self._feature_to_table_dict)
 
+  @property
+  def optimization_parameters(self):
+    return self._optimization_parameters
+
   def _create_config_proto(self):
     """Create `TPUEmbeddingConfiguration`."""
     config_proto = elc.TPUEmbeddingConfiguration()
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 80adfa1244..a6b3b13173 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -2779,9 +2779,6 @@ class TPUEstimator(estimator_lib.Estimator):
         tpu_init_ops = []
         if ctx.embedding_config:
           tpu_init_ops.extend(ctx.embedding_config.tpu_embedding.init_ops)
-          embedding_variables_and_ops = (
-              ctx.embedding_config.tpu_embedding.create_variables_and_ops())
-          tpu_init_ops.extend(embedding_variables_and_ops.load_ops())
 
         input_holders = _InputPipeline(input_fn, batch_axis, ctx)
         enqueue_ops, dequeue_fn, input_hooks, run_infeed_loop_on_coordinator = (
@@ -2797,6 +2794,24 @@ class TPUEstimator(estimator_lib.Estimator):
         if mode == model_fn_lib.ModeKeys.TRAIN:
           compile_op, loss, host_call, scaffold, training_hooks = (
               _train_on_tpu_system(ctx, model_fn_wrapper, dequeue_fn))
+          if ctx.embedding_config:
+            g = ops.get_default_graph()
+            table_to_config_dict = (
+                ctx.embedding_config.tpu_embedding.table_to_config_dict)
+            optimization_parameters = (
+                ctx.embedding_config.tpu_embedding.optimization_parameters)
+            embedding_variable_name_by_table, slot_variable_names_by_table = (
+                _tpu_estimator_embedding.get_full_variable_names(
+                    g, table_to_config_dict, optimization_parameters
+                )
+            )
+            embedding_variables_and_ops = (
+                ctx.embedding_config.tpu_embedding.create_variables_and_ops(
+                    embedding_variable_name_by_table,
+                    slot_variable_names_by_table
+                ))
+            tpu_init_ops.extend(embedding_variables_and_ops.load_ops())
+
           host_ops = host_call.create_tpu_hostcall()
           if host_ops is None:
             host_ops = []
-- 
GitLab


From 5c738693ad5d714eeba413b53a0f037f12fc2c82 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Fri, 8 Feb 2019 11:09:56 -0800
Subject: [PATCH 2164/2345] Minor tf.function documentation update.

PiperOrigin-RevId: 233092058
---
 tensorflow/python/eager/def_function.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py
index 57d9eb4b36..f5bb8cfd57 100644
--- a/tensorflow/python/eager/def_function.py
+++ b/tensorflow/python/eager/def_function.py
@@ -752,6 +752,9 @@ def function(func=None,
       l.append(i)                           # Caution! Doesn't work.
   ```
 
+  Note that unlike other TensorFlow operations, we don't convert python
+  numerical inputs to tensors.
+
   _Referencing `tf.Variable`s_
 
   The Python function `func` may reference stateful objects (such as
-- 
GitLab


From 89a949ec78154a65cac0bda8b528d2741ad1e96b Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Fri, 8 Feb 2019 11:12:24 -0800
Subject: [PATCH 2165/2345] Rename variable for more clarity

PiperOrigin-RevId: 233092472
---
 tensorflow/contrib/distribute/python/tpu_strategy.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 2ce127c855..158247295b 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -221,13 +221,13 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
       inputs = input_iterator.get_next()
 
     result = [None]
-    def replicated_fn(replica_id, inputs):
+    def replicated_fn(replica_id, replica_input):
       """Wraps user function to provide replica ID and `Tensor` inputs."""
       with _TPUReplicaContext(self, replica_id_in_sync_group=replica_id):
         if input_iterator is None:
           result[0] = fn()
         else:
-          result[0] = fn(inputs)
+          result[0] = fn(replica_input)
       return result[0]
 
     replicate_inputs = []  # By replica.
-- 
GitLab


From 21715d6fe1c090e55d38dbf88a425c54d7dd0ad7 Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Fri, 8 Feb 2019 11:36:15 -0800
Subject: [PATCH 2166/2345] Optimize the `reset_state` in keras.metrics to use
 batch operations.

PiperOrigin-RevId: 233097010
---
 tensorflow/python/keras/metrics.py | 27 +++++++++++++--------------
 1 file changed, 13 insertions(+), 14 deletions(-)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 63ff5e63e8..d3be3ef6a0 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -203,8 +203,7 @@ class Metric(Layer):
     This function is called between epochs/steps,
     when a metric is evaluated during training.
     """
-    for v in self.variables:
-      K.set_value(v, 0)
+    K.batch_set_value([(v, 0) for v in self.variables])
 
   @abc.abstractmethod
   def update_state(self, *args, **kwargs):
@@ -864,8 +863,8 @@ class _ConfusionMatrixConditionCount(Metric):
 
   def reset_states(self):
     num_thresholds = len(to_list(self.thresholds))
-    for v in self.variables:
-      K.set_value(v, np.zeros((num_thresholds,)))
+    K.batch_set_value(
+        [(v, np.zeros((num_thresholds,))) for v in self.variables])
 
   def get_config(self):
     config = {'thresholds': self.init_thresholds}
@@ -1187,8 +1186,8 @@ class Precision(Metric):
 
   def reset_states(self):
     num_thresholds = len(to_list(self.thresholds))
-    for v in self.variables:
-      K.set_value(v, np.zeros((num_thresholds,)))
+    K.batch_set_value(
+        [(v, np.zeros((num_thresholds,))) for v in self.variables])
 
   def get_config(self):
     config = {
@@ -1313,8 +1312,8 @@ class Recall(Metric):
 
   def reset_states(self):
     num_thresholds = len(to_list(self.thresholds))
-    for v in self.variables:
-      K.set_value(v, np.zeros((num_thresholds,)))
+    K.batch_set_value(
+        [(v, np.zeros((num_thresholds,))) for v in self.variables])
 
   def get_config(self):
     config = {
@@ -1391,8 +1390,8 @@ class SensitivitySpecificityBase(Metric):
 
   def reset_states(self):
     num_thresholds = len(self.thresholds)
-    for v in self.variables:
-      K.set_value(v, np.zeros((num_thresholds,)))
+    K.batch_set_value(
+        [(v, np.zeros((num_thresholds,))) for v in self.variables])
 
 
 @keras_export('keras.metrics.SensitivityAtSpecificity')
@@ -1809,8 +1808,8 @@ class AUC(Metric):
 
   def reset_states(self):
     num_thresholds = len(self.thresholds)
-    for v in self.variables:
-      K.set_value(v, np.zeros((num_thresholds,)))
+    K.batch_set_value(
+        [(v, np.zeros((num_thresholds,))) for v in self.variables])
 
   def get_config(self):
     config = {
@@ -2425,8 +2424,8 @@ class MeanTensor(Metric):
 
   def reset_states(self):
     if self._built:
-      for v in self.variables:
-        K.set_value(v, np.zeros(self._shape.as_list()))
+      K.batch_set_value(
+          [(v, np.zeros(self._shape.as_list())) for v in self.variables])
 
 
 @keras_export('keras.metrics.BinaryCrossentropy')
-- 
GitLab


From 2f1bc0c3dc19586ef362aa51cfaa926cf4882c50 Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Fri, 8 Feb 2019 11:49:32 -0800
Subject: [PATCH 2167/2345] Create int8 max min.

PiperOrigin-RevId: 233099428
---
 tensorflow/lite/kernels/maximum_minimum.cc    |  3 ++
 .../lite/kernels/maximum_minimum_test.cc      | 11 ++++++
 tensorflow/lite/kernels/register.cc           |  8 +++-
 tensorflow/lite/toco/tflite/operator.cc       | 34 +++++++++++++++--
 tensorflow/lite/toco/tflite/operator_test.cc  | 38 +++++++++++++++++++
 5 files changed, 88 insertions(+), 6 deletions(-)

diff --git a/tensorflow/lite/kernels/maximum_minimum.cc b/tensorflow/lite/kernels/maximum_minimum.cc
index 3bcaabf675..0e15254f47 100644
--- a/tensorflow/lite/kernels/maximum_minimum.cc
+++ b/tensorflow/lite/kernels/maximum_minimum.cc
@@ -108,6 +108,9 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
       case kTfLiteUInt8:
         TFLiteOperation<uint8_t, OpType>(context, node, op_context);
         break;
+      case kTfLiteInt8:
+        TFLiteOperation<int8_t, OpType>(context, node, op_context);
+        break;
       case kTfLiteInt32:
        TFLiteOperation<int32_t, OpType>(context, node, op_context);
         break;
diff --git a/tensorflow/lite/kernels/maximum_minimum_test.cc b/tensorflow/lite/kernels/maximum_minimum_test.cc
index acb74e09d3..6567c8f361 100644
--- a/tensorflow/lite/kernels/maximum_minimum_test.cc
+++ b/tensorflow/lite/kernels/maximum_minimum_test.cc
@@ -112,6 +112,17 @@ TEST(MaxMinOpTest, Uint8Test) {
                      {0, 0, 1, 11, 2, 1});
 }
 
+TEST(MaxMinOpTest, Int8Test) {
+  std::initializer_list<int8_t> data1 = {1, 0, 2, 11, 2, 23};
+  std::initializer_list<int8_t> data2 = {0, 0, 1, 12, 123, 1};
+  TestModel<int8_t>(BuiltinOperator_MAXIMUM, {TensorType_INT8, {3, 1, 2}},
+                    {TensorType_INT8, {3, 1, 2}}, {TensorType_INT8, {3, 1, 2}},
+                    data1, data2, {1, 0, 2, 12, 123, 23});
+  TestModel<int8_t>(BuiltinOperator_MINIMUM, {TensorType_INT8, {3, 1, 2}},
+                    {TensorType_INT8, {3, 1, 2}}, {TensorType_INT8, {3, 1, 2}},
+                    data1, data2, {0, 0, 1, 11, 2, 1});
+}
+
 TEST(MaximumOpTest, FloatWithBroadcastTest) {
   std::initializer_list<float> data1 = {1.0, 0.0, -1.0, -2.0, -1.44, 11.0};
   std::initializer_list<float> data2 = {0.5, 2.0};
diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index 2c9851b05c..970fdba1af 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -262,8 +262,12 @@ BuiltinOpResolver::BuiltinOpResolver() {
              /* min_version */ 1,
              /* max_version */ 2);
   AddBuiltin(BuiltinOperator_PRELU, Register_PRELU());
-  AddBuiltin(BuiltinOperator_MAXIMUM, Register_MAXIMUM());
-  AddBuiltin(BuiltinOperator_MINIMUM, Register_MINIMUM());
+  AddBuiltin(BuiltinOperator_MAXIMUM, Register_MAXIMUM(),
+             /* min_version */ 1,
+             /* max_version */ 2);
+  AddBuiltin(BuiltinOperator_MINIMUM, Register_MINIMUM(),
+             /* min_version */ 1,
+             /* max_version */ 2);
   AddBuiltin(BuiltinOperator_ARG_MAX, Register_ARG_MAX(),
              /* min_version */ 1,
              /* max_version */ 2);
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index b26ed6a3ec..ad0e4addd0 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -652,6 +652,34 @@ class MaxPool : public BuiltinOperator<MaxPoolOperator, ::tflite::Pool2DOptions,
   }
 };
 
+class Maximum : public SimpleOperator<TensorFlowMaximumOperator> {
+ public:
+  explicit Maximum() : SimpleOperator("MAXIMUM", OperatorType::kMaximum) {}
+  int GetVersion(const OperatorSignature& op_signature) const override {
+    const string& input_name = op_signature.op->inputs[0];
+    const Array& input_array = op_signature.model->GetArray(input_name);
+    // Version 2 supports signed int8 input types.
+    if (input_array.data_type == ArrayDataType::kInt8) {
+      return 2;
+    }
+    return 1;
+  }
+};
+
+class Minimum : public SimpleOperator<TensorFlowMinimumOperator> {
+ public:
+  explicit Minimum() : SimpleOperator("MINIMUM", OperatorType::kMinimum) {}
+  int GetVersion(const OperatorSignature& op_signature) const override {
+    const string& input_name = op_signature.op->inputs[0];
+    const Array& input_array = op_signature.model->GetArray(input_name);
+    // Version 2 supports signed int8 input types.
+    if (input_array.data_type == ArrayDataType::kInt8) {
+      return 2;
+    }
+    return 1;
+  }
+};
+
 class Mul : public BuiltinOperator<MulOperator, ::tflite::MulOptions,
                                    ::tflite::BuiltinOptions_MulOptions> {
  public:
@@ -2260,10 +2288,8 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
       MakeUnique<SimpleOperator<ExpOperator>>("EXP", OperatorType::kExp));
   ops.push_back(MakeUnique<SimpleOperator<LogSoftmaxOperator>>(
       "LOG_SOFTMAX", OperatorType::kLogSoftmax));
-  ops.push_back(MakeUnique<SimpleOperator<TensorFlowMaximumOperator>>(
-      "MAXIMUM", OperatorType::kMaximum));  //  Element-wise Maximum
-  ops.push_back(MakeUnique<SimpleOperator<TensorFlowMinimumOperator>>(
-      "MINIMUM", OperatorType::kMinimum));  //  Element-wise Minimum
+  ops.push_back(MakeUnique<Maximum>());  //  Element-wise Maximum
+  ops.push_back(MakeUnique<Minimum>());  //  Element-wise Minimum
   ops.push_back(MakeUnique<Greater>());
   ops.push_back(MakeUnique<GreaterEqual>());
   ops.push_back(MakeUnique<Less>());
diff --git a/tensorflow/lite/toco/tflite/operator_test.cc b/tensorflow/lite/toco/tflite/operator_test.cc
index 5c7028bee1..eaf971419c 100644
--- a/tensorflow/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/lite/toco/tflite/operator_test.cc
@@ -287,6 +287,44 @@ TEST_F(OperatorTest, BuiltinMaxPool) {
   EXPECT_EQ(op.kheight, output_toco_op->kheight);
 }
 
+TEST_F(OperatorTest, VersioningMaxTest) {
+  TensorFlowMaximumOperator max_op;
+  max_op.inputs = {"input1"};
+  auto operator_by_type_map = BuildOperatorByTypeMap(false /*enable_flex_ops*/);
+  const BaseOperator* op = operator_by_type_map.at(max_op.type).get();
+
+  Model uint8_model;
+  Array& uint8_array = uint8_model.GetOrCreateArray(max_op.inputs[0]);
+  uint8_array.data_type = ArrayDataType::kUint8;
+  OperatorSignature uint8_signature = {.model = &uint8_model, .op = &max_op};
+  EXPECT_EQ(op->GetVersion(uint8_signature), 1);
+
+  Model int8_model;
+  Array& int8_array = int8_model.GetOrCreateArray(max_op.inputs[0]);
+  int8_array.data_type = ArrayDataType::kInt8;
+  OperatorSignature int8_signature = {.model = &int8_model, .op = &max_op};
+  EXPECT_EQ(op->GetVersion(int8_signature), 2);
+}
+
+TEST_F(OperatorTest, VersioningMinTest) {
+  TensorFlowMinimumOperator min_op;
+  min_op.inputs = {"input1"};
+  auto operator_by_type_map = BuildOperatorByTypeMap(false /*enable_flex_ops*/);
+  const BaseOperator* op = operator_by_type_map.at(min_op.type).get();
+
+  Model uint8_model;
+  Array& uint8_array = uint8_model.GetOrCreateArray(min_op.inputs[0]);
+  uint8_array.data_type = ArrayDataType::kUint8;
+  OperatorSignature uint8_signature = {.model = &uint8_model, .op = &min_op};
+  EXPECT_EQ(op->GetVersion(uint8_signature), 1);
+
+  Model int8_model;
+  Array& int8_array = int8_model.GetOrCreateArray(min_op.inputs[0]);
+  int8_array.data_type = ArrayDataType::kInt8;
+  OperatorSignature int8_signature = {.model = &int8_model, .op = &min_op};
+  EXPECT_EQ(op->GetVersion(int8_signature), 2);
+}
+
 TEST_F(OperatorTest, BuiltinReshape) {
   TensorFlowReshapeOperator op;
   op.shape = {1, 2, 4, 5, 8};
-- 
GitLab


From 588b577e2cfb1848a48c11914bf4a7d580e377d6 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Fri, 8 Feb 2019 12:03:06 -0800
Subject: [PATCH 2168/2345] Fix test case; NFC

The test was checking for a PartionedCallOp op when the op name is just
"PartionedCall".

PiperOrigin-RevId: 233101888
---
 tensorflow/core/grappler/optimizers/function_optimizer_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
index c7373989dc..2787d9d525 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
@@ -1378,7 +1378,7 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithFunctionalControlFlow) {
 
   // All `PartitionedCall` nodes in the optimized graph must be inlined, and
   // `If` node must be lowered to `Switch` and `Merge` nodes.
-  EXPECT_EQ(count_nodes_with_op("PartitionedCallOp"), 0);
+  EXPECT_EQ(count_nodes_with_op("PartitionedCall"), 0);
   EXPECT_EQ(count_nodes_with_op("If"), 0);
   EXPECT_EQ(count_nodes_with_op("Switch"), 3);
   EXPECT_EQ(count_nodes_with_op("Merge"), 1);
-- 
GitLab


From 38d0634d698f4b00191d8fd515ad821463f971a7 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 8 Feb 2019 12:19:23 -0800
Subject: [PATCH 2169/2345] Get rid of tensorflow::int64 in HLO parser/lexer.

Plain int64 works just as well, as xla::int64 is an alias for tensorflow::int64.

PiperOrigin-RevId: 233104711
---
 tensorflow/compiler/xla/service/hlo_lexer.h   |   4 +-
 tensorflow/compiler/xla/service/hlo_parser.cc | 196 +++++++++---------
 2 files changed, 95 insertions(+), 105 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_lexer.h b/tensorflow/compiler/xla/service/hlo_lexer.h
index 16eed21617..9b440db8b7 100644
--- a/tensorflow/compiler/xla/service/hlo_lexer.h
+++ b/tensorflow/compiler/xla/service/hlo_lexer.h
@@ -108,7 +108,7 @@ class HloLexer {
         LOG(FATAL) << "This token does not have string value";
     }
   }
-  tensorflow::int64 GetInt64Val() const {
+  int64 GetInt64Val() const {
     CHECK(GetKind() == TokKind::kInt);
     return token_state_.int64_val;
   }
@@ -171,7 +171,7 @@ class HloLexer {
     const char* token_start = nullptr;
     TokKind current_kind;
     string str_val;
-    tensorflow::int64 int64_val;
+    int64 int64_val;
     double decimal_val;
     PrimitiveType primitive_type_val;
   };
diff --git a/tensorflow/compiler/xla/service/hlo_parser.cc b/tensorflow/compiler/xla/service/hlo_parser.cc
index 7c59c3a86b..0ab0054dac 100644
--- a/tensorflow/compiler/xla/service/hlo_parser.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser.cc
@@ -63,8 +63,7 @@ HloSchedule ScheduleFromInstructionOrder(HloModule* module) {
 
 // Some functions accept either a linear index or a multi-dimensional index
 // (used for indexing into sparse literals).
-using LinearOrMultiIndex =
-    absl::variant<tensorflow::int64, absl::Span<const int64>>;
+using LinearOrMultiIndex = absl::variant<int64, absl::Span<const int64>>;
 
 // Parser for the HloModule::ToString() format text.
 class HloParser {
@@ -129,8 +128,8 @@ class HloParser {
   // given value. If the literal is dense, it myst have the default layout.
   //
   // `loc` should be the source location of the value.
-  bool SetValueInLiteral(LocTy loc, tensorflow::int64 value,
-                         LinearOrMultiIndex index, Literal* literal);
+  bool SetValueInLiteral(LocTy loc, int64 value, LinearOrMultiIndex index,
+                         Literal* literal);
   bool SetValueInLiteral(LocTy loc, double value, LinearOrMultiIndex index,
                          Literal* literal);
   bool SetValueInLiteral(LocTy loc, bool value, LinearOrMultiIndex index,
@@ -158,9 +157,9 @@ class HloParser {
   // Describes the start, limit, and stride on every dimension of the operand
   // being sliced.
   struct SliceRanges {
-    std::vector<tensorflow::int64> starts;
-    std::vector<tensorflow::int64> limits;
-    std::vector<tensorflow::int64> strides;
+    std::vector<int64> starts;
+    std::vector<int64> limits;
+    std::vector<int64> strides;
   };
 
   // The data parsed for the kDomain instruction.
@@ -253,16 +252,15 @@ class HloParser {
   bool ParseDomain(DomainData* domain);
 
   // Parses a sub-attribute of the window attribute, e.g.,size=1x2x3.
-  bool ParseDxD(const string& name, std::vector<tensorflow::int64>* result);
+  bool ParseDxD(const string& name, std::vector<int64>* result);
   // Parses window's pad sub-attriute, e.g., pad=0_0x3x3.
-  bool ParseWindowPad(std::vector<std::vector<tensorflow::int64>>* pad);
+  bool ParseWindowPad(std::vector<std::vector<int64>>* pad);
 
   bool ParseSliceRanges(SliceRanges* result);
   bool ParsePrecisionList(std::vector<PrecisionConfig::Precision>* result);
   bool ParseShapeList(std::vector<Shape>* result);
   bool ParseInt64List(const TokKind start, const TokKind end,
-                      const TokKind delim,
-                      std::vector<tensorflow::int64>* result);
+                      const TokKind delim, std::vector<int64>* result);
   // 'parse_and_add_item' is an lambda to parse an element in the list and add
   // the parsed element to the result. It's supposed to capture the result.
   bool ParseList(const TokKind start, const TokKind end, const TokKind delim,
@@ -283,7 +281,7 @@ class HloParser {
   bool ParseFusionKind(HloInstruction::FusionKind* result);
   bool ParseRandomDistribution(RandomDistribution* result);
   bool ParsePrecision(PrecisionConfig::Precision* result);
-  bool ParseInt64(tensorflow::int64* result);
+  bool ParseInt64(int64* result);
   bool ParseDouble(double* result);
   bool ParseComplex(std::complex<double>* result);
   bool ParseBool(bool* result);
@@ -658,7 +656,7 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
   HloInstruction* instruction;
   switch (opcode) {
     case HloOpcode::kParameter: {
-      tensorflow::int64 parameter_number;
+      int64 parameter_number;
       if (!ParseToken(TokKind::kLparen,
                       "expects '(' before parameter number") ||
           !ParseInt64(&parameter_number)) {
@@ -690,7 +688,7 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
       break;
     }
     case HloOpcode::kIota: {
-      optional<tensorflow::int64> iota_dimension;
+      optional<int64> iota_dimension;
       attrs["iota_dimension"] = {/*required=*/true, AttrTy::kInt64,
                                  &iota_dimension};
       if (!ParseOperands(&operands, /*expected_size=*/0) ||
@@ -893,7 +891,7 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
       break;
     }
     case HloOpcode::kSort: {
-      optional<std::vector<tensorflow::int64>> dimensions;
+      optional<std::vector<int64>> dimensions;
       attrs["dimensions"] = {/*required=*/true, AttrTy::kBracedInt64List,
                              &dimensions};
       optional<HloComputation*> to_apply;
@@ -930,7 +928,7 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
       break;
     }
     case HloOpcode::kRecv: {
-      optional<tensorflow::int64> channel_id;
+      optional<int64> channel_id;
       // If the is_host_transfer attribute is not present then default to false.
       optional<bool> is_host_transfer = false;
       attrs["channel_id"] = {/*required=*/true, AttrTy::kInt64, &channel_id};
@@ -946,7 +944,7 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
       break;
     }
     case HloOpcode::kRecvDone: {
-      optional<tensorflow::int64> channel_id;
+      optional<int64> channel_id;
       // If the is_host_transfer attribute is not present then default to false.
       optional<bool> is_host_transfer = false;
       attrs["channel_id"] = {/*required=*/true, AttrTy::kInt64, &channel_id};
@@ -964,7 +962,7 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
       break;
     }
     case HloOpcode::kSend: {
-      optional<tensorflow::int64> channel_id;
+      optional<int64> channel_id;
       // If the is_host_transfer attribute is not present then default to false.
       optional<bool> is_host_transfer = false;
       attrs["channel_id"] = {/*required=*/true, AttrTy::kInt64, &channel_id};
@@ -979,7 +977,7 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
       break;
     }
     case HloOpcode::kSendDone: {
-      optional<tensorflow::int64> channel_id;
+      optional<int64> channel_id;
       // If the is_host_transfer attribute is not present then default to false.
       optional<bool> is_host_transfer = false;
       attrs["channel_id"] = {/*required=*/true, AttrTy::kInt64, &channel_id};
@@ -997,7 +995,7 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
       break;
     }
     case HloOpcode::kGetTupleElement: {
-      optional<tensorflow::int64> index;
+      optional<int64> index;
       attrs["index"] = {/*required=*/true, AttrTy::kInt64, &index};
       if (!ParseOperands(&operands, /*expected_size=*/1) ||
           !ParseAttributes(attrs)) {
@@ -1080,7 +1078,7 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
     }
     case HloOpcode::kFft: {
       optional<FftType> fft_type;
-      optional<std::vector<tensorflow::int64>> fft_length;
+      optional<std::vector<int64>> fft_length;
       attrs["fft_type"] = {/*required=*/true, AttrTy::kFftType, &fft_type};
       attrs["fft_length"] = {/*required=*/true, AttrTy::kBracedInt64List,
                              &fft_length};
@@ -1125,7 +1123,7 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
       break;
     }
     case HloOpcode::kBroadcast: {
-      optional<std::vector<tensorflow::int64>> broadcast_dimensions;
+      optional<std::vector<int64>> broadcast_dimensions;
       attrs["dimensions"] = {/*required=*/true, AttrTy::kBracedInt64List,
                              &broadcast_dimensions};
       if (!ParseOperands(&operands, /*expected_size=*/1) ||
@@ -1137,7 +1135,7 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
       break;
     }
     case HloOpcode::kConcatenate: {
-      optional<std::vector<tensorflow::int64>> dimensions;
+      optional<std::vector<int64>> dimensions;
       attrs["dimensions"] = {/*required=*/true, AttrTy::kBracedInt64List,
                              &dimensions};
       if (!ParseOperands(&operands) || !ParseAttributes(attrs) ||
@@ -1152,7 +1150,7 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
       optional<HloComputation*> to_apply;
       attrs["to_apply"] = {/*required=*/true, AttrTy::kHloComputation,
                            &to_apply};
-      optional<std::vector<tensorflow::int64>> dimensions;
+      optional<std::vector<int64>> dimensions;
       attrs["dimensions"] = {/*required=*/false, AttrTy::kBracedInt64List,
                              &dimensions};
       if (!ParseOperands(&operands) || !ParseAttributes(attrs)) {
@@ -1168,7 +1166,7 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
       optional<HloComputation*> reduce_computation;
       attrs["to_apply"] = {/*required=*/true, AttrTy::kHloComputation,
                            &reduce_computation};
-      optional<std::vector<tensorflow::int64>> dimensions_to_reduce;
+      optional<std::vector<int64>> dimensions_to_reduce;
       attrs["dimensions"] = {/*required=*/true, AttrTy::kBracedInt64List,
                              &dimensions_to_reduce};
       if (!ParseOperands(&operands) || !ParseAttributes(attrs)) {
@@ -1189,7 +1187,7 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
       break;
     }
     case HloOpcode::kReverse: {
-      optional<std::vector<tensorflow::int64>> dimensions;
+      optional<std::vector<int64>> dimensions;
       attrs["dimensions"] = {/*required=*/true, AttrTy::kBracedInt64List,
                              &dimensions};
       if (!ParseOperands(&operands, /*expected_size=*/1) ||
@@ -1233,7 +1231,7 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
       break;
     }
     case HloOpcode::kDynamicSlice: {
-      optional<std::vector<tensorflow::int64>> dynamic_slice_sizes;
+      optional<std::vector<int64>> dynamic_slice_sizes;
       attrs["dynamic_slice_sizes"] = {
           /*required=*/true, AttrTy::kBracedInt64List, &dynamic_slice_sizes};
       LocTy loc = lexer_.GetLoc();
@@ -1272,7 +1270,7 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
       break;
     }
     case HloOpcode::kTranspose: {
-      optional<std::vector<tensorflow::int64>> dimensions;
+      optional<std::vector<int64>> dimensions;
       attrs["dimensions"] = {/*required=*/true, AttrTy::kBracedInt64List,
                              &dimensions};
       if (!ParseOperands(&operands, /*expected_size=*/1) ||
@@ -1286,7 +1284,7 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
     case HloOpcode::kBatchNormTraining: {
       optional<float> epsilon;
       attrs["epsilon"] = {/*required=*/true, AttrTy::kFloat, &epsilon};
-      optional<tensorflow::int64> feature_index;
+      optional<int64> feature_index;
       attrs["feature_index"] = {/*required=*/true, AttrTy::kInt64,
                                 &feature_index};
       if (!ParseOperands(&operands, /*expected_size=*/3) ||
@@ -1302,7 +1300,7 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
     case HloOpcode::kBatchNormInference: {
       optional<float> epsilon;
       attrs["epsilon"] = {/*required=*/true, AttrTy::kFloat, &epsilon};
-      optional<tensorflow::int64> feature_index;
+      optional<int64> feature_index;
       attrs["feature_index"] = {/*required=*/true, AttrTy::kInt64,
                                 &feature_index};
       if (!ParseOperands(&operands, /*expected_size=*/5) ||
@@ -1319,7 +1317,7 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
     case HloOpcode::kBatchNormGrad: {
       optional<float> epsilon;
       attrs["epsilon"] = {/*required=*/true, AttrTy::kFloat, &epsilon};
-      optional<tensorflow::int64> feature_index;
+      optional<int64> feature_index;
       attrs["feature_index"] = {/*required=*/true, AttrTy::kInt64,
                                 &feature_index};
       if (!ParseOperands(&operands, /*expected_size=*/5) ||
@@ -1401,8 +1399,8 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
       break;
     }
     case HloOpcode::kReducePrecision: {
-      optional<tensorflow::int64> exponent_bits;
-      optional<tensorflow::int64> mantissa_bits;
+      optional<int64> exponent_bits;
+      optional<int64> mantissa_bits;
       attrs["exponent_bits"] = {/*required=*/true, AttrTy::kInt64,
                                 &exponent_bits};
       attrs["mantissa_bits"] = {/*required=*/true, AttrTy::kInt64,
@@ -1513,16 +1511,16 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
       break;
     }
     case HloOpcode::kDot: {
-      optional<std::vector<tensorflow::int64>> lhs_contracting_dims;
+      optional<std::vector<int64>> lhs_contracting_dims;
       attrs["lhs_contracting_dims"] = {
           /*required=*/false, AttrTy::kBracedInt64List, &lhs_contracting_dims};
-      optional<std::vector<tensorflow::int64>> rhs_contracting_dims;
+      optional<std::vector<int64>> rhs_contracting_dims;
       attrs["rhs_contracting_dims"] = {
           /*required=*/false, AttrTy::kBracedInt64List, &rhs_contracting_dims};
-      optional<std::vector<tensorflow::int64>> lhs_batch_dims;
+      optional<std::vector<int64>> lhs_batch_dims;
       attrs["lhs_batch_dims"] = {/*required=*/false, AttrTy::kBracedInt64List,
                                  &lhs_batch_dims};
-      optional<std::vector<tensorflow::int64>> rhs_batch_dims;
+      optional<std::vector<int64>> rhs_batch_dims;
       attrs["rhs_batch_dims"] = {/*required=*/false, AttrTy::kBracedInt64List,
                                  &rhs_batch_dims};
       optional<std::vector<PrecisionConfig::Precision>> operand_precision;
@@ -1566,19 +1564,19 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
       break;
     }
     case HloOpcode::kGather: {
-      optional<std::vector<tensorflow::int64>> offset_dims;
+      optional<std::vector<int64>> offset_dims;
       attrs["offset_dims"] = {/*required=*/true, AttrTy::kBracedInt64List,
                               &offset_dims};
-      optional<std::vector<tensorflow::int64>> collapsed_slice_dims;
+      optional<std::vector<int64>> collapsed_slice_dims;
       attrs["collapsed_slice_dims"] = {
           /*required=*/true, AttrTy::kBracedInt64List, &collapsed_slice_dims};
-      optional<std::vector<tensorflow::int64>> start_index_map;
+      optional<std::vector<int64>> start_index_map;
       attrs["start_index_map"] = {/*required=*/true, AttrTy::kBracedInt64List,
                                   &start_index_map};
-      optional<tensorflow::int64> index_vector_dim;
+      optional<int64> index_vector_dim;
       attrs["index_vector_dim"] = {/*required=*/true, AttrTy::kInt64,
                                    &index_vector_dim};
-      optional<std::vector<tensorflow::int64>> slice_sizes;
+      optional<std::vector<int64>> slice_sizes;
       attrs["slice_sizes"] = {/*required=*/true, AttrTy::kBracedInt64List,
                               &slice_sizes};
 
@@ -1600,17 +1598,17 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
       break;
     }
     case HloOpcode::kScatter: {
-      optional<std::vector<tensorflow::int64>> update_window_dims;
+      optional<std::vector<int64>> update_window_dims;
       attrs["update_window_dims"] = {
           /*required=*/true, AttrTy::kBracedInt64List, &update_window_dims};
-      optional<std::vector<tensorflow::int64>> inserted_window_dims;
+      optional<std::vector<int64>> inserted_window_dims;
       attrs["inserted_window_dims"] = {
           /*required=*/true, AttrTy::kBracedInt64List, &inserted_window_dims};
-      optional<std::vector<tensorflow::int64>> scatter_dims_to_operand_dims;
+      optional<std::vector<int64>> scatter_dims_to_operand_dims;
       attrs["scatter_dims_to_operand_dims"] = {/*required=*/true,
                                                AttrTy::kBracedInt64List,
                                                &scatter_dims_to_operand_dims};
-      optional<tensorflow::int64> index_vector_dim;
+      optional<int64> index_vector_dim;
       attrs["index_vector_dim"] = {/*required=*/true, AttrTy::kInt64,
                                    &index_vector_dim};
 
@@ -1651,7 +1649,7 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
       return TokenError(StrCat("parsing not yet implemented for op: ",
                                HloOpcodeString(opcode)));
     case HloOpcode::kGetDimensionSize:
-      optional<std::vector<tensorflow::int64>> dimensions;
+      optional<std::vector<int64>> dimensions;
       attrs["dimensions"] = {/*required=*/true, AttrTy::kBracedInt64List,
                              &dimensions};
       if (!ParseOperands(&operands, /*expected_size=*/1) ||
@@ -1740,8 +1738,8 @@ bool HloParser::ParseSingleSharding(OpSharding* sharding,
   LocTy loc = lexer_.GetLoc();
   bool maximal = false;
   bool replicated = false;
-  std::vector<tensorflow::int64> devices;
-  std::vector<tensorflow::int64> tile_assignment_dimensions;
+  std::vector<int64> devices;
+  std::vector<int64> tile_assignment_dimensions;
   while (lexer_.GetKind() != TokKind::kRbrace) {
     switch (lexer_.GetKind()) {
       case TokKind::kw_maximal:
@@ -1767,7 +1765,7 @@ bool HloParser::ParseSingleSharding(OpSharding* sharding,
           }
 
           do {
-            tensorflow::int64 dim;
+            int64 dim;
             if (!ParseInt64(&dim)) {
               return false;
             }
@@ -1779,7 +1777,7 @@ bool HloParser::ParseSingleSharding(OpSharding* sharding,
             return false;
           }
           do {
-            tensorflow::int64 device;
+            int64 device;
             if (!ParseInt64(&device)) {
               return false;
             }
@@ -1823,10 +1821,10 @@ bool HloParser::ParseSingleSharding(OpSharding* sharding,
           "dimensions");
     }
     sharding->set_type(OpSharding::Type::OpSharding_Type_OTHER);
-    for (tensorflow::int64 dim : tile_assignment_dimensions) {
+    for (int64 dim : tile_assignment_dimensions) {
       sharding->add_tile_assignment_dimensions(dim);
     }
-    for (tensorflow::int64 device : devices) {
+    for (int64 device : devices) {
       sharding->add_tile_assignment_devices(device);
     }
   }
@@ -1887,22 +1885,18 @@ bool HloParser::ParseInstructionNames(
                     "expects '}' at the end of instruction name list");
 }
 
-bool HloParser::SetValueInLiteral(LocTy loc, tensorflow::int64 value,
+bool HloParser::SetValueInLiteral(LocTy loc, int64 value,
                                   LinearOrMultiIndex index, Literal* literal) {
   const Shape& shape = literal->shape();
   switch (shape.element_type()) {
     case S8:
-      return SetValueInLiteralHelper<tensorflow::int8>(loc, value, index,
-                                                       literal);
+      return SetValueInLiteralHelper<int8>(loc, value, index, literal);
     case S16:
-      return SetValueInLiteralHelper<tensorflow::int16>(loc, value, index,
-                                                        literal);
+      return SetValueInLiteralHelper<int16>(loc, value, index, literal);
     case S32:
-      return SetValueInLiteralHelper<tensorflow::int32>(loc, value, index,
-                                                        literal);
+      return SetValueInLiteralHelper<int32>(loc, value, index, literal);
     case S64:
-      return SetValueInLiteralHelper<tensorflow::int64>(loc, value, index,
-                                                        literal);
+      return SetValueInLiteralHelper<int64>(loc, value, index, literal);
     case U8:
       return SetValueInLiteralHelper<tensorflow::uint8>(loc, value, index,
                                                         literal);
@@ -1990,7 +1984,7 @@ bool HloParser::SetValueInLiteralHelper(LocTy loc, ParsedElemT value,
   }
 
   // Check that the index is in range and assign into the literal
-  if (auto* linear_index = absl::get_if<tensorflow::int64>(&index)) {
+  if (auto* linear_index = absl::get_if<int64>(&index)) {
     if (*linear_index >= ShapeUtil::ElementsIn(literal->shape())) {
       return Error(loc, StrCat("trys to set value ", StringifyValue(value),
                                " to a literal in shape ",
@@ -2095,8 +2089,8 @@ bool HloParser::ParseDenseLiteral(Literal* literal, const Shape& shape) {
   // Create a literal with the given shape in default layout.
   *literal = LiteralUtil::CreateFromDimensions(
       shape.element_type(), AsInt64Slice(shape.dimensions()));
-  tensorflow::int64 nest_level = 0;
-  tensorflow::int64 linear_index = 0;
+  int64 nest_level = 0;
+  int64 linear_index = 0;
   // elems_seen_per_dim[i] is how many elements or sub-arrays we have seen for
   // the dimension i. For example, to parse f32[2,3] {{1, 2, 3}, {4, 5, 6}},
   // when we are parsing the 2nd '{' (right before '1'), we are seeing a
@@ -2104,13 +2098,13 @@ bool HloParser::ParseDenseLiteral(Literal* literal, const Shape& shape) {
   // the first '}' (right after '3'), it means the sub-array ends, and the
   // sub-array is supposed to contain exactly 3 elements, so check if
   // elems_seen_per_dim[1] is 3.
-  std::vector<tensorflow::int64> elems_seen_per_dim(rank);
+  std::vector<int64> elems_seen_per_dim(rank);
   auto get_index_str = [&elems_seen_per_dim](int dim) -> string {
-    std::vector<tensorflow::int64> elems_seen_until_dim(
-        elems_seen_per_dim.begin(), elems_seen_per_dim.begin() + dim);
+    std::vector<int64> elems_seen_until_dim(elems_seen_per_dim.begin(),
+                                            elems_seen_per_dim.begin() + dim);
     return StrCat("[",
                   StrJoin(elems_seen_until_dim, ",",
-                          [](string* out, const tensorflow::int64& num_elems) {
+                          [](string* out, const int64& num_elems) {
                             StrAppend(out, num_elems - 1);
                           }),
                   "]");
@@ -2216,7 +2210,7 @@ bool HloParser::ParseDenseLiteral(Literal* literal, const Shape& shape) {
         } else if (primitive_util::IsIntegralType(shape.element_type()) ||
                    shape.element_type() == PRED) {
           LocTy loc = lexer_.GetLoc();
-          tensorflow::int64 value;
+          int64 value;
           if (!ParseInt64(&value)) {
             return Error(loc, StrCat("expects integer for primitive type: ",
                                      PrimitiveType_Name(shape.element_type())));
@@ -2261,9 +2255,9 @@ bool HloParser::ParseSparseLiteral(Literal* literal, const Shape& shape) {
       break;
     }
 
-    std::vector<tensorflow::int64> index;
+    std::vector<int64> index;
     if (lexer_.GetKind() == TokKind::kInt) {
-      tensorflow::int64 single_index = lexer_.GetInt64Val();
+      int64 single_index = lexer_.GetInt64Val();
       lexer_.Lex();
       index.push_back(single_index);
     } else {
@@ -2287,7 +2281,7 @@ bool HloParser::ParseSparseLiteral(Literal* literal, const Shape& shape) {
       }
       lexer_.Lex();
     } else if (primitive_util::IsIntegralType(shape.element_type())) {
-      tensorflow::int64 value;
+      int64 value;
       if (!ParseInt64(&value)) {
         return Error(value_loc,
                      StrCat("expects integer for primitive type: ",
@@ -2373,7 +2367,7 @@ bool HloParser::CheckParsedValueIsInRange(LocTy loc, ParsedElemT value) {
         -std::numeric_limits<ParsedElemT>::infinity() == value))) {
     // Skip range checking for non-finite value.
   } else if (std::is_unsigned<LiteralNativeT>::value) {
-    CHECK((std::is_same<ParsedElemT, tensorflow::int64>::value ||
+    CHECK((std::is_same<ParsedElemT, int64>::value ||
            std::is_same<ParsedElemT, bool>::value))
         << "Unimplemented checking for ParsedElemT";
 
@@ -2599,24 +2593,23 @@ bool HloParser::ParseAttributeHelper(
         return true;
       }
       case AttrTy::kInt64: {
-        tensorflow::int64 result;
+        int64 result;
         if (!ParseInt64(&result)) {
           return false;
         }
-        static_cast<optional<tensorflow::int64>*>(attr_out_ptr)
-            ->emplace(result);
+        static_cast<optional<int64>*>(attr_out_ptr)->emplace(result);
         return true;
       }
       case AttrTy::kInt32: {
-        tensorflow::int64 result;
+        int64 result;
         if (!ParseInt64(&result)) {
           return false;
         }
-        if (result != static_cast<tensorflow::int32>(result)) {
+        if (result != static_cast<int32>(result)) {
           return Error(attr_loc, "value out of range for int32");
         }
-        static_cast<optional<tensorflow::int32>*>(attr_out_ptr)
-            ->emplace(static_cast<tensorflow::int32>(result));
+        static_cast<optional<int32>*>(attr_out_ptr)
+            ->emplace(static_cast<int32>(result));
         return true;
       }
       case AttrTy::kFloat: {
@@ -2709,19 +2702,19 @@ bool HloParser::ParseAttributeHelper(
         return true;
       }
       case AttrTy::kBracedInt64List: {
-        std::vector<tensorflow::int64> result;
+        std::vector<int64> result;
         if (!ParseInt64List(TokKind::kLbrace, TokKind::kRbrace, TokKind::kComma,
                             &result)) {
           return false;
         }
-        static_cast<optional<std::vector<tensorflow::int64>>*>(attr_out_ptr)
+        static_cast<optional<std::vector<int64>>*>(attr_out_ptr)
             ->emplace(result);
         return true;
       }
       case AttrTy::kBracedInt64ListList: {
-        std::vector<std::vector<tensorflow::int64>> result;
+        std::vector<std::vector<int64>> result;
         auto parse_and_add_item = [&]() {
-          std::vector<tensorflow::int64> item;
+          std::vector<int64> item;
           if (!ParseInt64List(TokKind::kLbrace, TokKind::kRbrace,
                               TokKind::kComma, &item)) {
             return false;
@@ -2733,8 +2726,7 @@ bool HloParser::ParseAttributeHelper(
                        parse_and_add_item)) {
           return false;
         }
-        static_cast<optional<std::vector<std::vector<tensorflow::int64>>>*>(
-            attr_out_ptr)
+        static_cast<optional<std::vector<std::vector<int64>>>*>(attr_out_ptr)
             ->emplace(result);
         return true;
       }
@@ -2935,7 +2927,7 @@ bool HloParser::ParseConvolutionDimensionNumbers(
   absl::string_view rhs = split2[0];
   absl::string_view out = split2[1];
 
-  const tensorflow::int64 rank = lhs.length();
+  const int64 rank = lhs.length();
   if (rank != rhs.length() || rank != out.length()) {
     return TokenError(
         "convolution lhs, rhs, and output must have the same rank");
@@ -3046,7 +3038,7 @@ bool HloParser::ParseSliceRanges(SliceRanges* result) {
   if (!ParseToken(TokKind::kLbrace, "expects '{' to start ranges")) {
     return false;
   }
-  std::vector<std::vector<tensorflow::int64>> ranges;
+  std::vector<std::vector<int64>> ranges;
   if (lexer_.GetKind() == TokKind::kRbrace) {
     // empty
     return ParseToken(TokKind::kRbrace, "expects '}' to end ranges");
@@ -3116,9 +3108,9 @@ bool HloParser::ParseShapeList(std::vector<Shape>* result) {
 //   ::= int64_val (delim int64_val)*
 bool HloParser::ParseInt64List(const TokKind start, const TokKind end,
                                const TokKind delim,
-                               std::vector<tensorflow::int64>* result) {
+                               std::vector<int64>* result) {
   auto parse_and_add_item = [&]() {
-    tensorflow::int64 i;
+    int64 i;
     if (!ParseInt64(&i)) {
       return false;
     }
@@ -3194,7 +3186,7 @@ bool HloParser::ParseParamList() {
 bool HloParser::ParseDimensionSizes(std::vector<int64>* dimension_sizes,
                                     std::vector<bool>* dynamic_dimensions) {
   auto parse_and_add_item = [&]() {
-    tensorflow::int64 i;
+    int64 i;
     bool is_dynamic = false;
     if (lexer_.GetKind() == TokKind::kLeq) {
       is_dynamic = true;
@@ -3215,7 +3207,7 @@ bool HloParser::ParseDimensionSizes(std::vector<int64>* dimension_sizes,
 bool HloParser::ParseLayout(Layout* layout) {
   std::vector<int64> minor_to_major;
   auto parse_and_add_item = [&]() {
-    tensorflow::int64 i;
+    int64 i;
     if (!ParseInt64(&i)) {
       return false;
     }
@@ -3278,7 +3270,7 @@ bool HloParser::ParseShape(Shape* result) {
     lexer_.Lex();
     const string message =
         "expects a brace-bracketed integer for sparse layout";
-    tensorflow::int64 max_sparse_elements;
+    int64 max_sparse_elements;
     if (!ParseToken(TokKind::kLbrace, message) ||
         !ParseInt64(&max_sparse_elements) ||
         !ParseToken(TokKind::kRbrace, message)) {
@@ -3347,15 +3339,14 @@ bool HloParser::ParseString(string* result) {
   return true;
 }
 
-bool HloParser::ParseDxD(const string& name,
-                         std::vector<tensorflow::int64>* result) {
+bool HloParser::ParseDxD(const string& name, std::vector<int64>* result) {
   LocTy loc = lexer_.GetLoc();
   if (!result->empty()) {
     return Error(loc, StrFormat("sub-attribute '%s=' already exists", name));
   }
   // 1D
   if (lexer_.GetKind() == TokKind::kInt) {
-    tensorflow::int64 number;
+    int64 number;
     if (!ParseInt64(&number)) {
       return Error(loc, StrFormat("expects sub-attribute '%s=i'", name));
     }
@@ -3374,8 +3365,7 @@ bool HloParser::ParseDxD(const string& name,
   return TokenError("expects token type kInt or kDxD");
 }
 
-bool HloParser::ParseWindowPad(
-    std::vector<std::vector<tensorflow::int64>>* pad) {
+bool HloParser::ParseWindowPad(std::vector<std::vector<int64>>* pad) {
   LocTy loc = lexer_.GetLoc();
   if (!pad->empty()) {
     return Error(loc, "sub-attribute 'pad=' already exists");
@@ -3385,7 +3375,7 @@ bool HloParser::ParseWindowPad(
   }
   string str = lexer_.GetStrVal();
   for (const auto& padding_dim_str : absl::StrSplit(str, 'x')) {
-    std::vector<tensorflow::int64> low_high;
+    std::vector<int64> low_high;
     if (!SplitToInt64s(padding_dim_str, '_', &low_high) ||
         low_high.size() != 2) {
       return Error(loc,
@@ -3408,7 +3398,7 @@ bool HloParser::ParsePaddingConfig(PaddingConfig* padding) {
   LocTy loc = lexer_.GetLoc();
   string str = lexer_.GetStrVal();
   for (const auto& padding_dim_str : absl::StrSplit(str, 'x')) {
-    std::vector<tensorflow::int64> padding_dim;
+    std::vector<int64> padding_dim;
     if (!SplitToInt64s(padding_dim_str, '_', &padding_dim) ||
         (padding_dim.size() != 2 && padding_dim.size() != 3)) {
       return Error(loc,
@@ -3430,7 +3420,7 @@ bool HloParser::ParseMetadata(OpMetadata* metadata) {
   optional<string> op_type;
   optional<string> op_name;
   optional<string> source_file;
-  optional<tensorflow::int32> source_line;
+  optional<int32> source_line;
   attrs["op_type"] = {/*required=*/false, AttrTy::kString, &op_type};
   attrs["op_name"] = {/*required=*/false, AttrTy::kString, &op_name};
   attrs["source_file"] = {/*required=*/false, AttrTy::kString, &source_file};
@@ -3549,7 +3539,7 @@ bool HloParser::ParsePrecision(PrecisionConfig::Precision* result) {
   return true;
 }
 
-bool HloParser::ParseInt64(tensorflow::int64* result) {
+bool HloParser::ParseInt64(int64* result) {
   VLOG(1) << "ParseInt64";
   if (lexer_.GetKind() != TokKind::kInt) {
     return TokenError("expects integer");
-- 
GitLab


From 3de71d7b96b1d271e6db302dd906883e612ad858 Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Fri, 8 Feb 2019 12:28:25 -0800
Subject: [PATCH 2170/2345] Export GPU convolution autotuning results from XLA.

PiperOrigin-RevId: 233106101
---
 tensorflow/compiler/xla/BUILD                 |   1 +
 tensorflow/compiler/xla/protobuf_util.h       |  16 ++
 tensorflow/compiler/xla/service/gpu/BUILD     |  12 +
 .../compiler/xla/service/gpu/autotuning.proto |  81 +++++++
 .../gpu/cudnn_conv_algorithm_picker.cc        | 206 ++++++++++++------
 .../service/gpu/cudnn_conv_algorithm_picker.h |   8 +-
 tensorflow/compiler/xla/xla.bzl               |   4 +-
 tensorflow/core/BUILD                         |   5 +-
 tensorflow/stream_executor/logging.proto      |  12 +
 9 files changed, 261 insertions(+), 84 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/gpu/autotuning.proto

diff --git a/tensorflow/compiler/xla/BUILD b/tensorflow/compiler/xla/BUILD
index 5b8a4c51fe..ee6f7d5956 100644
--- a/tensorflow/compiler/xla/BUILD
+++ b/tensorflow/compiler/xla/BUILD
@@ -192,6 +192,7 @@ cc_library(
         ":types",
         ":util",
         "//tensorflow/core:lib",
+        "@com_google_absl//absl/time",
     ],
 )
 
diff --git a/tensorflow/compiler/xla/protobuf_util.h b/tensorflow/compiler/xla/protobuf_util.h
index f22fc8b849..4a88a48f28 100644
--- a/tensorflow/compiler/xla/protobuf_util.h
+++ b/tensorflow/compiler/xla/protobuf_util.h
@@ -16,6 +16,8 @@ limitations under the License.
 #ifndef TENSORFLOW_COMPILER_XLA_PROTOBUF_UTIL_H_
 #define TENSORFLOW_COMPILER_XLA_PROTOBUF_UTIL_H_
 
+#include "google/protobuf/duration.pb.h"
+#include "absl/time/time.h"
 #include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/core/platform/protobuf.h"
@@ -43,6 +45,20 @@ Status DumpProtoToDirectory(const tensorflow::protobuf::Message& message,
 // dirpath along as-is.
 void RegisterDirectoryExpander(const std::function<string(string)>& expander);
 
+// Converts an absl::Duration to a google::protobuf::Duration.
+inline google::protobuf::Duration ToDurationProto(absl::Duration duration) {
+  google::protobuf::Duration proto;
+  proto.set_seconds(absl::IDivDuration(duration, absl::Seconds(1), &duration));
+  proto.set_nanos(
+      absl::IDivDuration(duration, absl::Nanoseconds(1), &duration));
+  return proto;
+}
+
+// Converts a google::protobuf::Duration to an absl::Duration.
+inline absl::Duration FromDurationProto(google::protobuf::Duration proto) {
+  return absl::Seconds(proto.seconds()) + absl::Nanoseconds(proto.nanos());
+}
+
 }  // namespace protobuf_util
 }  // namespace xla
 
diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD
index 17274b46ab..05980fe549 100644
--- a/tensorflow/compiler/xla/service/gpu/BUILD
+++ b/tensorflow/compiler/xla/service/gpu/BUILD
@@ -400,18 +400,21 @@ cc_library(
     srcs = ["cudnn_conv_algorithm_picker.cc"],
     hdrs = ["cudnn_conv_algorithm_picker.h"],
     deps = [
+        ":autotuning_proto",
         ":backend_configs",
         ":buffer_comparator",
         ":cudnn_conv_runner",
         ":gpu_executable",
         ":ir_emission_utils",
         "//tensorflow/compiler/xla:literal_util",
+        "//tensorflow/compiler/xla:protobuf_util",
         "//tensorflow/compiler/xla/service:compiler",
         "//tensorflow/compiler/xla/service:device_memory_allocator",
         "//tensorflow/compiler/xla/service:hlo",
         "//tensorflow/compiler/xla/service:hlo_casting_utils",
         "//tensorflow/compiler/xla/service:hlo_pass",
         "//tensorflow/core:lib",
+        "//tensorflow/core:logger",
         "//tensorflow/core:stream_executor_no_cuda",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/strings:str_format",
@@ -1085,3 +1088,12 @@ tf_cc_test(
         "//tensorflow/compiler/xla/tests:hlo_test_base",
     ],
 )
+
+xla_proto_library(
+    name = "autotuning_proto",
+    srcs = ["autotuning.proto"],
+    deps = [
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/service:hlo_proto",
+    ],
+)
diff --git a/tensorflow/compiler/xla/service/gpu/autotuning.proto b/tensorflow/compiler/xla/service/gpu/autotuning.proto
new file mode 100644
index 0000000000..b4a08963b4
--- /dev/null
+++ b/tensorflow/compiler/xla/service/gpu/autotuning.proto
@@ -0,0 +1,81 @@
+// This file defines protos that store the results of autotuning XLA:GPU
+// operations.
+//
+// They are in proto format because we want to log them structured. They offer
+// tremendous statistical, testing, and debugging value.
+syntax = "proto3";
+
+package xla.gpu;
+
+import "google/protobuf/duration.proto";
+import "tensorflow/compiler/xla/xla_data.proto";
+import "tensorflow/compiler/xla/service/hlo.proto";
+
+message CudnnVersion {
+  int32 major = 1;
+  int32 minor = 2;
+  int32 patch = 3;
+}
+
+message ComputeCapability {
+  int32 major = 1;
+  int32 minor = 2;
+}
+
+message AutotuneResult {
+  message SuccessResult {
+    int64 scratch_bytes = 1;
+    google.protobuf.Duration run_time = 2;
+  }
+
+  message ConvKey {
+    int64 algorithm = 1;
+    bool tensor_ops_enabled = 2;
+  }
+
+  // If the conv runs successfully, success will be populated with the
+  // autotuning result. Otherwise, the error message is propagated.
+  oneof result {
+    SuccessResult success = 3;
+    string error_string = 4;
+  }
+
+  oneof key {
+    ConvKey conv = 5;
+  }
+
+  // Sometimes we run a correctness checker during autotuning. It compares the
+  // result buffer content between two algorithms, say, "reference" and "test"
+  // algorithms. The "test" algorithm is the one associated with this
+  // AutotuneResult.
+  //
+  // This field records the reference algorithm used. Notice that naming it
+  // "reference" doesn't mean it's always correct. However, empirically it's
+  // more correct, as it's "algo 0", less fancy than the compared one.
+  //
+  // Notice that the checker_failure may exist even in the success case.
+  // This is because the error string in `result` comes from the underlying
+  // implementation like cuDNN, which isn't aware that it produced an incorrect
+  // result. And even if the checker detects an incorrect result, we can still
+  // retrieve scratch_bytes and runtime_ms.
+  oneof checker_failure {
+    ConvKey reference_conv = 6;
+  }
+}
+
+message AutotuneLog {
+  message Instruction {
+    xla.HloInstructionProto instruction = 1;
+    repeated xla.ShapeProto operand_shapes = 2;
+  }
+
+  oneof instr_oneof {
+    Instruction instr = 1;
+  }
+
+  // Records all auto-tuning results per algorithm.
+  repeated AutotuneResult results = 3;
+
+  CudnnVersion cudnn_version = 4;
+  ComputeCapability compute_capability = 5;
+}
diff --git a/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.cc b/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.cc
index 309b0aca64..bc5170a9b1 100644
--- a/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.cc
+++ b/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.cc
@@ -16,14 +16,17 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_format.h"
+#include "absl/time/time.h"
 #include "absl/types/optional.h"
 #include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/compiler/xla/protobuf_util.h"
 #include "tensorflow/compiler/xla/service/gpu/backend_configs.pb.h"
 #include "tensorflow/compiler/xla/service/gpu/buffer_comparator.h"
 #include "tensorflow/compiler/xla/service/gpu/convolution_thunk.h"
 #include "tensorflow/compiler/xla/service/gpu/ir_emission_utils.h"
 #include "tensorflow/compiler/xla/service/hlo_casting_utils.h"
 #include "tensorflow/core/lib/strings/numbers.h"
+#include "tensorflow/core/platform/logger.h"
 #include "tensorflow/core/platform/mutex.h"
 
 namespace xla {
@@ -132,6 +135,31 @@ tensorflow::mutex_lock LockGpu(const se::StreamExecutor* stream_exec) {
   return tensorflow::mutex_lock{it->second};
 }
 
+xla::gpu::CudnnVersion GetCudnnVersion(se::StreamExecutor* stream_executor) {
+  xla::gpu::CudnnVersion cudnn_version;
+  if (auto* dnn = stream_executor->AsDnn()) {
+    StatusOr<se::dnn::VersionInfo> version_or = dnn->GetVersion();
+    if (version_or.ok()) {
+      const auto& version = version_or.ValueOrDie();
+      cudnn_version.set_major(version.major_version());
+      cudnn_version.set_minor(version.minor_version());
+      cudnn_version.set_patch(version.patch());
+    }
+  }
+  return cudnn_version;
+}
+
+xla::gpu::ComputeCapability GetComputeCapability(
+    se::StreamExecutor* stream_executor) {
+  xla::gpu::ComputeCapability cc;
+  int cc_major, cc_minor;
+  stream_executor->GetDeviceDescription().cuda_compute_capability(&cc_major,
+                                                                  &cc_minor);
+  cc.set_major(cc_major);
+  cc.set_minor(cc_minor);
+  return cc;
+}
+
 }  // anonymous namespace
 
 // We could have caching here so that we don't redo this work for two identical
@@ -145,8 +173,7 @@ tensorflow::mutex_lock LockGpu(const se::StreamExecutor* stream_exec) {
 // cache misses and doing extra work.  Overall, caching doesn't seem worth the
 // trouble, but we may want to revisit this if we ever find a model where
 // caching would speed up compilation a lot.
-StatusOr<CudnnConvAlgorithmPicker::AutotuneResult>
-CudnnConvAlgorithmPicker::PickBestAlgorithm(
+StatusOr<AutotuneResult> CudnnConvAlgorithmPicker::PickBestAlgorithm(
     const HloCustomCallInstruction* instr) {
   // TODO(timshen): for now only check fp16. It can be expanded to other types,
   // with some work on the HLO routines.
@@ -233,8 +260,6 @@ CudnnConvAlgorithmPicker::PickBestAlgorithm(
           &stream, ShapeUtil::ByteSizeOf(instr->shape().tuple_shapes(0))));
   initialize_buffer(result_buffer);
 
-  se::dnn::ProfileResult best_result;
-  int64 best_result_bytes_used = 0;
   TF_ASSIGN_OR_RETURN(auto backend_config,
                       instr->backend_config<CudnnConvBackendConfig>());
 
@@ -244,6 +269,7 @@ CudnnConvAlgorithmPicker::PickBestAlgorithm(
   // this algorithm considered correct, though.
   optional<AlgorithmDesc> first_algorithm;
   TF_ASSIGN_OR_RETURN(CudnnConvKind kind, GetCudnnConvKind(instr));
+  std::vector<AutotuneResult> profile_results;
   for (const AlgorithmDesc& alg : GetAlgorithms(kind, stream_exec_)) {
     ScratchAllocator scratch_allocator(device_ordinal, allocator);
     se::dnn::ProfileResult profile_result;
@@ -254,73 +280,108 @@ CudnnConvAlgorithmPicker::PickBestAlgorithm(
     RunConvOptions options;
     options.profile_result = &profile_result;
     options.algo_override = alg;
-    bool launch_ok =
+    Status launch_status =
         RunCudnnConv(instr, absl::MakeSpan(operand_buffers), result_buffer,
-                     &scratch_allocator, &stream, options)
-            .ok();
-
-    if (launch_ok && profile_result.is_valid()) {
-      const bool crash_on_checking_failure =
-          instr->GetModule()
-              ->config()
-              .debug_options()
-              .xla_gpu_crash_on_verification_failures();
-      if (comparator.has_value()) {
-        StatusOr<bool> result = comparator->CompareEqual(
-            se::DeviceMemory<Eigen::half>(result_buffer));
-        if (!result.ok()) {
-          LOG(ERROR) << "Unable to compare "
-                     << AlgorithmToString(*first_algorithm) << " against "
-                     << AlgorithmToString(alg) << " for " << instr->ToString()
-                     << ": " << result.status();
-          CHECK(!crash_on_checking_failure);
-        } else if (!result.ValueOrDie()) {
-          LOG(ERROR) << "Results mismatch between different convolution "
-                        "algorithms. This is likely a bug in convolution, or "
-                        "an excessive loss of precision in convolution. "
-                     << instr->ToString() << " for "
-                     << AlgorithmToString(*first_algorithm) << " vs "
-                     << AlgorithmToString(alg);
-          CHECK(!crash_on_checking_failure);
-        }
-      } else if (cross_check_enabled) {
-        auto comp = F16BufferComparator::Create(
-            se::DeviceMemory<Eigen::half>(result_buffer), compiler_, allocator,
-            &stream);
-        if (comp.ok()) {
-          comparator.emplace(comp.ConsumeValueOrDie());
-          first_algorithm.emplace(alg);
-        } else {
-          LOG(ERROR) << "Fail to initialize buffer comparator: "
-                     << comp.status() << ", instruction: " << instr->ToString();
-          CHECK(!crash_on_checking_failure);
-        }
+                     &scratch_allocator, &stream, options);
+
+    profile_results.emplace_back();
+    AutotuneResult& result = profile_results.back();
+    result.mutable_conv()->set_algorithm(alg.algo_id());
+    result.mutable_conv()->set_tensor_ops_enabled(alg.tensor_ops_enabled());
+
+    if (!launch_status.ok()) {
+      result.set_error_string(launch_status.error_message());
+      continue;
+    }
+
+    if (!profile_result.is_valid()) {
+      result.set_error_string("Invalid profile result");
+      continue;
+    }
+
+    int64 scratch_bytes_used = scratch_allocator.TotalAllocatedBytes();
+    result.mutable_success()->set_scratch_bytes(scratch_bytes_used);
+    *result.mutable_success()->mutable_run_time() =
+        protobuf_util::ToDurationProto(
+            absl::Milliseconds(profile_result.elapsed_time_in_ms()));
+
+    const bool crash_on_checking_failure =
+        instr->GetModule()
+            ->config()
+            .debug_options()
+            .xla_gpu_crash_on_verification_failures();
+
+    if (comparator.has_value()) {
+      StatusOr<bool> compare_result = comparator->CompareEqual(
+          se::DeviceMemory<Eigen::half>(result_buffer));
+      if (!compare_result.ok()) {
+        LOG(ERROR) << "Unable to compare "
+                   << AlgorithmToString(*first_algorithm) << " against "
+                   << AlgorithmToString(alg) << " for " << instr->ToString()
+                   << ": " << compare_result.status();
+        CHECK(!crash_on_checking_failure);
+      } else if (!compare_result.ValueOrDie()) {
+        LOG(ERROR) << "Results mismatch between different convolution "
+                      "algorithms. This is likely a bug in convolution, or "
+                      "an excessive loss of precision in convolution. "
+                   << instr->ToString() << " for "
+                   << AlgorithmToString(*first_algorithm) << " vs "
+                   << AlgorithmToString(alg);
+        CHECK(!crash_on_checking_failure);
+        auto* failure = result.mutable_reference_conv();
+        failure->set_algorithm(first_algorithm->algo_id());
+        failure->set_tensor_ops_enabled(first_algorithm->tensor_ops_enabled());
       }
-      int64 scratch_bytes_used = scratch_allocator.TotalAllocatedBytes();
-      VLOG(3) << "Run of algorithm " << AlgorithmToString(alg)
-              << " succeeded, taking " << profile_result.elapsed_time_in_ms()
-              << "ms and using " << NumBytesToString(scratch_bytes_used)
-              << " of scratch (Best result: "
-              << best_result.elapsed_time_in_ms() << "ms, "
-              << NumBytesToString(best_result_bytes_used) << " of scratch)";
-      if (profile_result.elapsed_time_in_ms() <
-          best_result.elapsed_time_in_ms()) {
-        best_result = profile_result;
-        best_result_bytes_used = scratch_bytes_used;
+    } else if (cross_check_enabled) {
+      auto comp = F16BufferComparator::Create(
+          se::DeviceMemory<Eigen::half>(result_buffer), compiler_, allocator,
+          &stream);
+      if (comp.ok()) {
+        comparator.emplace(comp.ConsumeValueOrDie());
+        first_algorithm.emplace(alg);
+      } else {
+        LOG(ERROR) << "Fail to initialize buffer comparator: " << comp.status()
+                   << ", instruction: " << instr->ToString();
+        CHECK(!crash_on_checking_failure);
       }
-    } else {
-      VLOG(3) << "Run of algorithm " << AlgorithmToString(alg) << " failed.";
     }
   }
-  if (best_result.is_valid()) {
-    VLOG(2) << "Best algorithm for " << instr->ToString() << ": "
-            << AlgorithmToString(best_result.algorithm()) << ", takes "
-            << best_result.elapsed_time_in_ms() << "ms, and uses "
-            << best_result_bytes_used << "B of scratch memory.";
-    return AutotuneResult{best_result.algorithm().algo_id(),
-                          best_result.algorithm().tensor_ops_enabled(),
-                          best_result_bytes_used,
-                          absl::Milliseconds(best_result.elapsed_time_in_ms())};
+
+  // Log the autotuning result.
+  {
+    AutotuneLog log;
+    *log.mutable_instr()->mutable_instruction() = instr->ToProto();
+    for (const auto* op : instr->operands()) {
+      *log.mutable_instr()->add_operand_shapes() = op->shape().ToProto();
+    }
+    for (const auto& profile : profile_results) {
+      *log.add_results() = profile;
+    }
+    *log.mutable_compute_capability() = GetComputeCapability(stream_exec_);
+    *log.mutable_cudnn_version() = GetCudnnVersion(stream_exec_);
+    VLOG(2) << "Autotuning result:\n" << log.DebugString();
+    tensorflow::Logger::Singleton()->LogProto(log);
+  }
+
+  auto* profile_results_end = profile_results.data() + profile_results.size();
+
+  const AutotuneResult* best_result = std::min_element(
+      profile_results.data(), profile_results_end,
+      [](const AutotuneResult& lhs, const AutotuneResult& rhs) {
+        // The successful one should have a smaller key, since we are doing
+        // min_element. If they are both unsuccessful, keep the earlier one in
+        // the vector by comparing pointers.
+        return std::make_tuple(
+                   !lhs.has_success(),
+                   protobuf_util::FromDurationProto(lhs.success().run_time()),
+                   &lhs) < std::make_tuple(!rhs.has_success(),
+                                           protobuf_util::FromDurationProto(
+                                               rhs.success().run_time()),
+                                           &rhs);
+      });
+
+  if (best_result != profile_results_end && best_result->has_success()) {
+    return *best_result;
   }
 
   return InternalError(
@@ -341,22 +402,23 @@ StatusOr<bool> CudnnConvAlgorithmPicker::RunOnInstruction(
   }
 
   auto best_algo = std::move(best_algo_or).ValueOrDie();
-  VLOG(1) << "Setting cudnn conv to use algorithm " << best_algo.algorithm
-          << " and " << NumBytesToString(best_algo.scratch_bytes)
+  VLOG(1) << "Setting cudnn conv to use algorithm "
+          << best_algo.conv().algorithm() << " and "
+          << NumBytesToString(best_algo.success().scratch_bytes())
           << " of scratch memory: " << instr->ToString()
-          << " tensor_ops_enabled: " << best_algo.tensor_ops_enabled;
+          << " tensor_ops_enabled: " << best_algo.conv().tensor_ops_enabled();
 
   // Replace instr with a new CustomCall which has the correct algorithm, and
   // whose output shape has the appropriate amount of scratch memory.
   HloComputation* computation = instr->parent();
   Shape new_call_shape = ShapeUtil::MakeTupleShape(
       {instr->shape().tuple_shapes(0),
-       ShapeUtil::MakeShape(U8, {best_algo.scratch_bytes})});
+       ShapeUtil::MakeShape(U8, {best_algo.success().scratch_bytes()})});
 
   TF_ASSIGN_OR_RETURN(CudnnConvBackendConfig backend_config,
                       instr->backend_config<CudnnConvBackendConfig>());
-  backend_config.set_algorithm(best_algo.algorithm);
-  backend_config.set_tensor_ops_enabled(best_algo.tensor_ops_enabled);
+  backend_config.set_algorithm(best_algo.conv().algorithm());
+  backend_config.set_tensor_ops_enabled(best_algo.conv().tensor_ops_enabled());
 
   HloInstruction* new_call = computation->AddInstruction(
       instr->CloneWithNewOperands(new_call_shape, instr->operands()));
diff --git a/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.h b/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.h
index 4991db0948..2e34ba9672 100644
--- a/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.h
+++ b/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.h
@@ -20,6 +20,7 @@ limitations under the License.
 #include "absl/types/optional.h"
 #include "tensorflow/compiler/xla/service/compiler.h"
 #include "tensorflow/compiler/xla/service/device_memory_allocator.h"
+#include "tensorflow/compiler/xla/service/gpu/autotuning.pb.h"
 #include "tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.h"
 #include "tensorflow/compiler/xla/service/hlo_instructions.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
@@ -47,13 +48,6 @@ class CudnnConvAlgorithmPicker : public HloModulePass {
   StatusOr<bool> Run(HloModule* module) override;
 
  private:
-  struct AutotuneResult {
-    int64 algorithm;
-    bool tensor_ops_enabled;
-    int64 scratch_bytes;
-    absl::Duration runtime;
-  };
-
   StatusOr<bool> RunOnComputation(HloComputation* computation);
   StatusOr<bool> RunOnInstruction(HloInstruction* instr);
   StatusOr<AutotuneResult> PickBestAlgorithm(
diff --git a/tensorflow/compiler/xla/xla.bzl b/tensorflow/compiler/xla/xla.bzl
index 60adea5a4a..c743dfd32b 100644
--- a/tensorflow/compiler/xla/xla.bzl
+++ b/tensorflow/compiler/xla/xla.bzl
@@ -17,7 +17,9 @@ def xla_proto_library(name, srcs = [], deps = [], visibility = None, testonly =
     cc_proto_library(
         name = name,
         srcs = srcs,
-        deps = deps,
+        # Append well-known proto dep. As far as I know this is the only way
+        # for xla_proto_library to access google.protobuf.{Any,Duration,...}.
+        deps = deps + ["@protobuf_archive//:cc_wkt_protos"],
         cc_libs = if_static(
             ["@protobuf_archive//:protobuf"],
             otherwise = ["@protobuf_archive//:protobuf_headers"],
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 2f066d8a63..e6d2c53b4a 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -457,10 +457,7 @@ cc_library(
     hdrs = ["platform/logger.h"],
     copts = tf_copts(),
     visibility = ["//visibility:public"],
-    deps = [
-        ":lib_proto_parsing",
-        "@protobuf_archive//:protobuf",
-    ],
+    deps = [":lib_proto_parsing"],
 )
 
 filegroup(
diff --git a/tensorflow/stream_executor/logging.proto b/tensorflow/stream_executor/logging.proto
index 68021d2b31..3351752788 100644
--- a/tensorflow/stream_executor/logging.proto
+++ b/tensorflow/stream_executor/logging.proto
@@ -15,6 +15,18 @@ message ComputeCapability {
   int32 minor = 2;
 }
 
+// NOTE: this proto is temporarily duplicated in other places, outside of
+// stream_executor. The plan is to move all custom logging (tensorflow::Logger
+// related) behavior out of StreamExecutor. There are two reasons:
+// * Technical: stream_executor is part of libtensorflow_framework.so. It's
+//   extremely hard to have a single definition of the protos in the .so, and
+//   let the callers call into those definitions. The complication lives in
+//   cc_proto_library where we have a header-only version and impl version.
+// * Functional: we want to log autotuning stats from the callers. The
+//   autotuning stats are not available in SE.
+//
+// TODO(timshen): remove this proto once both XLA and TF log autotuning
+// results.
 message CudaInfo {
   CudnnVersion cudnn_version = 1;
   ComputeCapability compute_capability = 2;
-- 
GitLab


From c2fef7cb15f78f9797a6a28aded46ae041df89f7 Mon Sep 17 00:00:00 2001
From: Austin Anderson <angerson@google.com>
Date: Fri, 8 Feb 2019 13:01:57 -0800
Subject: [PATCH 2171/2345] Fix a typo preventing GPU images from installing
 CUDA

Due to an incredibly obtuse quirk of the Dockerfile language, the GPU
images weren't getting built properly after the recent ARG changes.
---
 .../partials/ubuntu/devel-nvidia.partial.Dockerfile           | 4 ++++
 .../dockerfiles/partials/ubuntu/nvidia.partial.Dockerfile     | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/tensorflow/tools/dockerfiles/partials/ubuntu/devel-nvidia.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/devel-nvidia.partial.Dockerfile
index cb153fa913..b381ffe779 100644
--- a/tensorflow/tools/dockerfiles/partials/ubuntu/devel-nvidia.partial.Dockerfile
+++ b/tensorflow/tools/dockerfiles/partials/ubuntu/devel-nvidia.partial.Dockerfile
@@ -1,6 +1,10 @@
 ARG ARCH=
 ARG CUDA=10.0
 FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}-base-ubuntu${UBUNTU_VERSION} as base
+# ARCH and CUDA are specified again because the FROM directive resets ARGs
+# (but their default value is retained if set previously)
+ARG ARCH
+ARG CUDA
 ARG CUDNN=7.4.1.5-1
 ARG LIB_DIR_PREFIX=x84_64
 
diff --git a/tensorflow/tools/dockerfiles/partials/ubuntu/nvidia.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/nvidia.partial.Dockerfile
index 1d40ed5f98..041ee87839 100644
--- a/tensorflow/tools/dockerfiles/partials/ubuntu/nvidia.partial.Dockerfile
+++ b/tensorflow/tools/dockerfiles/partials/ubuntu/nvidia.partial.Dockerfile
@@ -1,6 +1,10 @@
 ARG ARCH=
 ARG CUDA=10.0
 FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}-base-ubuntu${UBUNTU_VERSION} as base
+# ARCH and CUDA are specified again because the FROM directive resets ARGs
+# (but their default value is retained if set previously)
+ARG ARCH
+ARG CUDA
 ARG CUDNN=7.4.1.5-1
 
 # Needed for string substitution 
-- 
GitLab


From 421ef8cad68e7323d774075da0c268bd50c62c9a Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 8 Feb 2019 12:29:32 -0800
Subject: [PATCH 2172/2345] Use new HLO matchers in algebraic_simplifier_test.

PiperOrigin-RevId: 233106263
---
 tensorflow/compiler/xla/service/BUILD                       | 1 -
 .../compiler/xla/service/algebraic_simplifier_test.cc       | 6 ++----
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index ecaf99311b..a5eae6d396 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -1644,7 +1644,6 @@ tf_cc_test(
         ":hlo",
         ":hlo_casting_utils",
         ":hlo_creation_utils",
-        ":hlo_matchers",
         ":hlo_parser",
         ":hlo_pass",
         ":pattern_matcher",
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index a908051037..2ace9330e7 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -28,7 +28,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/hlo_creation_utils.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_instructions.h"
-#include "tensorflow/compiler/xla/service/hlo_matchers.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
 #include "tensorflow/compiler/xla/service/hlo_parser.h"
 #include "tensorflow/compiler/xla/service/hlo_pass_fix.h"
@@ -47,7 +46,6 @@ namespace {
 
 using ::testing::ElementsAre;
 namespace m = match;
-namespace op = xla::testing::opcode_matchers;
 
 class AlgebraicSimplifierTest : public HloTestBase {
  protected:
@@ -4792,7 +4790,7 @@ TEST_F(AlgebraicSimplifierTest, ZeroSizedReshapeWithoutLayout) {
   AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   HloInstruction* root = module->entry_computation()->root_instruction();
-  EXPECT_THAT(root, op::Constant());
+  EXPECT_THAT(root, GmockMatch(m::Constant()));
 }
 
 TEST_F(AlgebraicSimplifierTest, DividedByConstantInstructionWithoutLayout) {
@@ -4814,7 +4812,7 @@ TEST_F(AlgebraicSimplifierTest, DividedByConstantInstructionWithoutLayout) {
   AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   HloInstruction* root = module->entry_computation()->root_instruction();
-  EXPECT_THAT(root, op::Multiply());
+  EXPECT_THAT(root, GmockMatch(m::Multiply()));
 }
 
 }  // namespace
-- 
GitLab


From abe13563c66fc165688976b8de2ff993898d5d51 Mon Sep 17 00:00:00 2001
From: Austin Anderson <angerson@google.com>
Date: Fri, 8 Feb 2019 13:08:56 -0800
Subject: [PATCH 2173/2345] Rebuild dockerfiles

---
 .../dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile      | 4 ++++
 tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile | 4 ++++
 .../tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile      | 4 ++++
 tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile       | 4 ++++
 .../dockerfiles/ppc64le/devel-gpu-ppc64le-jupyter.Dockerfile  | 4 ++++
 .../dockerfiles/ppc64le/devel-gpu-ppc64le.Dockerfile          | 4 ++++
 .../dockerfiles/ppc64le/gpu-ppc64le-jupyter.Dockerfile        | 4 ++++
 .../dockerfiles/dockerfiles/ppc64le/gpu-ppc64le.Dockerfile    | 4 ++++
 8 files changed, 32 insertions(+)

diff --git a/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile
index 5c01623f81..d193e4c379 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile
@@ -24,6 +24,10 @@ ARG UBUNTU_VERSION=16.04
 ARG ARCH=
 ARG CUDA=10.0
 FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}-base-ubuntu${UBUNTU_VERSION} as base
+# ARCH and CUDA are specified again because the FROM directive resets ARGs
+# (but their default value is retained if set previously)
+ARG ARCH
+ARG CUDA
 ARG CUDNN=7.4.1.5-1
 ARG LIB_DIR_PREFIX=x84_64
 
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile
index 56d582d7e7..00d3997e6b 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile
@@ -24,6 +24,10 @@ ARG UBUNTU_VERSION=16.04
 ARG ARCH=
 ARG CUDA=10.0
 FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}-base-ubuntu${UBUNTU_VERSION} as base
+# ARCH and CUDA are specified again because the FROM directive resets ARGs
+# (but their default value is retained if set previously)
+ARG ARCH
+ARG CUDA
 ARG CUDNN=7.4.1.5-1
 ARG LIB_DIR_PREFIX=x84_64
 
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile
index 335bbb5efe..85a32fae1b 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile
@@ -24,6 +24,10 @@ ARG UBUNTU_VERSION=16.04
 ARG ARCH=
 ARG CUDA=10.0
 FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}-base-ubuntu${UBUNTU_VERSION} as base
+# ARCH and CUDA are specified again because the FROM directive resets ARGs
+# (but their default value is retained if set previously)
+ARG ARCH
+ARG CUDA
 ARG CUDNN=7.4.1.5-1
 
 # Needed for string substitution 
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile
index 3b1e3e9f49..c661341eae 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile
@@ -24,6 +24,10 @@ ARG UBUNTU_VERSION=16.04
 ARG ARCH=
 ARG CUDA=10.0
 FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}-base-ubuntu${UBUNTU_VERSION} as base
+# ARCH and CUDA are specified again because the FROM directive resets ARGs
+# (but their default value is retained if set previously)
+ARG ARCH
+ARG CUDA
 ARG CUDNN=7.4.1.5-1
 
 # Needed for string substitution 
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le-jupyter.Dockerfile
index 7113bb2967..a077737b6c 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le-jupyter.Dockerfile
@@ -24,6 +24,10 @@ ARG UBUNTU_VERSION=16.04
 ARG ARCH=
 ARG CUDA=10.0
 FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}-base-ubuntu${UBUNTU_VERSION} as base
+# ARCH and CUDA are specified again because the FROM directive resets ARGs
+# (but their default value is retained if set previously)
+ARG ARCH
+ARG CUDA
 ARG CUDNN=7.4.1.5-1
 ARG LIB_DIR_PREFIX=x84_64
 
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le.Dockerfile
index a2f6fc5cba..d046cf8584 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le.Dockerfile
@@ -24,6 +24,10 @@ ARG UBUNTU_VERSION=16.04
 ARG ARCH=
 ARG CUDA=10.0
 FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}-base-ubuntu${UBUNTU_VERSION} as base
+# ARCH and CUDA are specified again because the FROM directive resets ARGs
+# (but their default value is retained if set previously)
+ARG ARCH
+ARG CUDA
 ARG CUDNN=7.4.1.5-1
 ARG LIB_DIR_PREFIX=x84_64
 
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le-jupyter.Dockerfile
index 9d0112c9c8..1e82ca2822 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le-jupyter.Dockerfile
@@ -24,6 +24,10 @@ ARG UBUNTU_VERSION=16.04
 ARG ARCH=
 ARG CUDA=10.0
 FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}-base-ubuntu${UBUNTU_VERSION} as base
+# ARCH and CUDA are specified again because the FROM directive resets ARGs
+# (but their default value is retained if set previously)
+ARG ARCH
+ARG CUDA
 ARG CUDNN=7.4.1.5-1
 
 # Needed for string substitution 
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le.Dockerfile
index f01a47f1c0..e9572756e5 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le.Dockerfile
@@ -24,6 +24,10 @@ ARG UBUNTU_VERSION=16.04
 ARG ARCH=
 ARG CUDA=10.0
 FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}-base-ubuntu${UBUNTU_VERSION} as base
+# ARCH and CUDA are specified again because the FROM directive resets ARGs
+# (but their default value is retained if set previously)
+ARG ARCH
+ARG CUDA
 ARG CUDNN=7.4.1.5-1
 
 # Needed for string substitution 
-- 
GitLab


From b0b6bbd5b9b28a2892fc9a4419d7833672e4a52b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 8 Feb 2019 12:30:18 -0800
Subject: [PATCH 2174/2345] Add parameter to ensure TensorFlow failures are as
 expected.

PiperOrigin-RevId: 233106383
---
 tensorflow/lite/testing/generate_examples.py | 272 ++++++++++++++-----
 1 file changed, 200 insertions(+), 72 deletions(-)

diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index 7ebc5950b2..23191edb17 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -300,8 +300,13 @@ def make_control_dep_tests(zip_path):
 
   extra_toco_options = ExtraTocoOptions()
   extra_toco_options.drop_control_dependency = True
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs,
-                    extra_toco_options)
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      extra_toco_options,
+      expected_tf_failures=3)
 
 
 def toco_convert(graph_def_str, input_tensors, output_tensors,
@@ -369,7 +374,7 @@ def make_zip_of_tests(zip_path,
                       make_test_inputs,
                       extra_toco_options=ExtraTocoOptions(),
                       use_frozen_graph=False,
-                      expected_tf_success=None):
+                      expected_tf_failures=0):
   """Helper to make a zip file of a bunch of TensorFlow models.
 
   This does a cartestian product of the dictionary of test_parameters and
@@ -389,8 +394,9 @@ def make_zip_of_tests(zip_path,
       `output_tensors` and returns tuple `(input_values, output_values)`.
     extra_toco_options: Additional toco options.
     use_frozen_graph: Whether or not freeze graph before toco converter.
-    expected_tf_success: Number of times tensorflow is supposed to succeed in
-      executing the input graphs. `None` means "unknown".
+    expected_tf_failures: Number of times tensorflow is expected to fail in
+      executing the input graphs. In some cases it is OK for TensorFlow to
+      fail because the one or more combination of parameters is invalid.
 
   Raises:
     RuntimeError: if there are toco errors that can't be ignored.
@@ -551,10 +557,17 @@ def make_zip_of_tests(zip_path,
                    " and %d TOCO converted graphs (%.1f%%"), zip_path,
                   total_conversions, tf_success, toco_success, percent)
 
-  if expected_tf_success is not None and tf_success != expected_tf_success:
-    raise RuntimeError(
-        "Expected TF to succeed %d times, but that happened %d times" %
-        (expected_tf_success, tf_success))
+  tf_failures = parameter_count - tf_success
+
+  if tf_failures / parameter_count > 0.8:
+    raise RuntimeError(("Test for '%s' is not very useful. "
+                        "TensorFlow fails in %d percent of the cases.") %
+                       (zip_path, int(100 * tf_failures / parameter_count)))
+
+  if tf_failures != expected_tf_failures:
+    raise RuntimeError(("Expected TF to fail %d times while generating '%s', "
+                        "but that happened %d times") % (expected_tf_failures,
+                                                         zip_path, tf_failures))
 
   if not FLAGS.ignore_toco_errors and toco_errors > 0:
     raise RuntimeError(
@@ -573,11 +586,12 @@ def make_pool_tests(pool_op_in):
 
   pool_op = pool_op_in
 
-  def f(zip_path):
+  def f(zip_path, expected_tf_failures=0):
     """Actual function that generates examples.
 
     Args:
       zip_path: path to write zip to.
+      expected_tf_failures: number of expected tensorflow failures.
     """
 
     # Chose a set of parameters
@@ -606,20 +620,26 @@ def make_pool_tests(pool_op_in):
       return [input_values], sess.run(
           outputs, feed_dict=dict(zip(inputs, [input_values])))
 
-    make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+    make_zip_of_tests(
+        zip_path,
+        test_parameters,
+        build_graph,
+        build_inputs,
+        expected_tf_failures=expected_tf_failures)
+
   return f
 
 
 def make_l2_pool_tests(zip_path):
-  make_pool_tests(make_l2_pool)(zip_path)
+  make_pool_tests(make_l2_pool)(zip_path, expected_tf_failures=80)
 
 
 def make_avg_pool_tests(zip_path):
-  make_pool_tests(tf.nn.avg_pool)(zip_path)
+  make_pool_tests(tf.nn.avg_pool)(zip_path, expected_tf_failures=80)
 
 
 def make_max_pool_tests(zip_path):
-  make_pool_tests(tf.nn.max_pool)(zip_path)
+  make_pool_tests(tf.nn.max_pool)(zip_path, expected_tf_failures=80)
 
 
 def make_abs_tests(zip_path):
@@ -834,11 +854,10 @@ def make_constant_tests(zip_path):
         parameters["input_shape"], dtype=_TF_TYPE_INFO[parameters["dtype"]][0])
     return [dummy_input], sess.run(outputs, feed_dict={inputs[0]: dummy_input})
 
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs,
-                    expected_tf_success=20)
+  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
 
 
-def make_binary_op_tests(zip_path, binary_operator):
+def make_binary_op_tests(zip_path, binary_operator, expected_tf_failures=0):
   """Make a set of tests to do binary ops with and without broadcast."""
 
   test_parameters = [
@@ -908,7 +927,12 @@ def make_binary_op_tests(zip_path, binary_operator):
             inputs[1]: input2
         })
 
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=expected_tf_failures)
 
 
 def make_reduce_tests(reduce_op,
@@ -1137,7 +1161,12 @@ def make_maximum_tests(zip_path):
     ]
     return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
 
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=8)
 
 
 def make_minimum_tests(zip_path):
@@ -1172,7 +1201,12 @@ def make_minimum_tests(zip_path):
     ]
     return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
 
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=8)
 
 
 def make_binary_op_tests_func(binary_operator):
@@ -1242,7 +1276,7 @@ def make_mul_tests(zip_path):
 
 
 def make_pow_tests(zip_path):
-  make_binary_op_tests(zip_path, tf.pow)
+  make_binary_op_tests(zip_path, tf.pow, expected_tf_failures=7)
 
 
 def make_floor_div_tests(zip_path):
@@ -1300,7 +1334,7 @@ def make_gather_tests(zip_path):
       test_parameters,
       build_graph,
       build_inputs,
-      expected_tf_success=60)
+      expected_tf_failures=12)
 
 
 def make_gather_with_constant_tests(zip_path):
@@ -1329,8 +1363,7 @@ def make_gather_with_constant_tests(zip_path):
     return [reference_values], sess.run(
         outputs, feed_dict={inputs[0]: reference_values})
 
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs,
-                    expected_tf_success=2)
+  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
 
 
 def make_global_batch_norm_tests(zip_path):
@@ -1466,7 +1499,12 @@ def make_conv_tests(zip_path):
       values.append(create_tensor_data(np.float32, filter_shape))
     return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
 
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=40)
 
 
 # Note: This is a regression test for a bug (b/122651451) that Toco incorrectly
@@ -1708,7 +1746,12 @@ def make_depthwiseconv_tests(zip_path):
       values.append(create_tensor_data(np.float32, filter_shape))
     return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
 
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=4)
 
 
 def make_split_tests(zip_path):
@@ -1731,7 +1774,12 @@ def make_split_tests(zip_path):
     values = [create_tensor_data(np.float32, parameters["input_shape"])]
     return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
 
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=112)
 
 
 def make_splitv_tests(zip_path):
@@ -1754,7 +1802,12 @@ def make_splitv_tests(zip_path):
     values = [create_tensor_data(np.float32, parameters["input_shape"])]
     return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
 
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=158)
 
 
 def make_concat_tests(zip_path):
@@ -1796,7 +1849,12 @@ def make_concat_tests(zip_path):
     return all_values, sess.run(
         outputs, feed_dict=dict(zip(inputs, all_values)))
 
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=60)
 
 
 def make_fully_connected_tests(zip_path):
@@ -1857,7 +1915,12 @@ def make_fully_connected_tests(zip_path):
       values.append(create_tensor_data(np.float32, parameters["shape2"]))
     return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
 
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=10)
 
 
 def make_l2norm_tests(zip_path):
@@ -1887,7 +1950,12 @@ def make_l2norm_tests(zip_path):
     return [input_values], sess.run(
         outputs, feed_dict=dict(zip(inputs, [input_values])))
 
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=9)
 
 
 def make_local_response_norm_tests(zip_path):
@@ -2372,7 +2440,12 @@ def make_space_to_batch_nd_tests(zip_path):
       values.append(np.array(parameters["paddings"]))
     return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
 
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=56)
 
 
 def make_batch_to_space_nd_tests(zip_path):
@@ -2485,7 +2558,12 @@ def make_transpose_tests(zip_path):
       values.append(np.array(parameters["perm"]))
     return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
 
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=9)
 
 
 def make_squeeze_tests(zip_path):
@@ -2523,10 +2601,16 @@ def make_squeeze_tests(zip_path):
     return [input_values], sess.run(
         outputs, feed_dict=dict(zip(inputs, [input_values])))
 
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=12)
 
 
-def _make_strided_slice_tests(zip_path, test_parameters):
+def _make_strided_slice_tests(zip_path, test_parameters,
+                              expected_tf_failures=0):
   """Utility function to make strided_slice_tests based on parameters."""
 
   def build_graph(parameters):
@@ -2586,7 +2670,12 @@ def _make_strided_slice_tests(zip_path, test_parameters):
 
     return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
 
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=expected_tf_failures)
 
 
 def make_strided_slice_tests(zip_path):
@@ -2660,7 +2749,7 @@ def make_strided_slice_tests(zip_path):
           "constant_indices": [False],
       },
   ]
-  _make_strided_slice_tests(zip_path, test_parameters)
+  _make_strided_slice_tests(zip_path, test_parameters, expected_tf_failures=2)
 
 
 def make_strided_slice_1d_exhaustive_tests(zip_path):
@@ -2862,7 +2951,12 @@ def make_arg_min_max_tests(zip_path):
     return [input_value], sess.run(
         outputs, feed_dict=dict(zip(inputs, [input_value])))
 
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=4)
 
 
 def make_equal_tests(zip_path):
@@ -2897,7 +2991,12 @@ def make_equal_tests(zip_path):
     return [input_value1, input_value2], sess.run(
         outputs, feed_dict=dict(zip(inputs, [input_value1, input_value2])))
 
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=3)
 
 
 def make_not_equal_tests(zip_path):
@@ -2931,7 +3030,12 @@ def make_not_equal_tests(zip_path):
     return [input_value1, input_value2], sess.run(
         outputs, feed_dict=dict(zip(inputs, [input_value1, input_value2])))
 
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=3)
 
 
 def make_greater_tests(zip_path):
@@ -2965,7 +3069,12 @@ def make_greater_tests(zip_path):
     return [input_value1, input_value2], sess.run(
         outputs, feed_dict=dict(zip(inputs, [input_value1, input_value2])))
 
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=3)
 
 
 def make_greater_equal_tests(zip_path):
@@ -2999,7 +3108,12 @@ def make_greater_equal_tests(zip_path):
     return [input_value1, input_value2], sess.run(
         outputs, feed_dict=dict(zip(inputs, [input_value1, input_value2])))
 
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=3)
 
 
 def make_less_tests(zip_path):
@@ -3033,7 +3147,12 @@ def make_less_tests(zip_path):
     return [input_value1, input_value2], sess.run(
         outputs, feed_dict=dict(zip(inputs, [input_value1, input_value2])))
 
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=3)
 
 
 def make_less_equal_tests(zip_path):
@@ -3067,7 +3186,12 @@ def make_less_equal_tests(zip_path):
     return [input_value1, input_value2], sess.run(
         outputs, feed_dict=dict(zip(inputs, [input_value1, input_value2])))
 
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=3)
 
 
 def make_floor_tests(zip_path):
@@ -3319,7 +3443,12 @@ def make_slice_tests(zip_path):
 
     return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
 
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=18)
 
 
 def make_conv2d_transpose_tests(zip_path):
@@ -3356,12 +3485,7 @@ def make_conv2d_transpose_tests(zip_path):
     ]
     return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
 
-  make_zip_of_tests(
-      zip_path,
-      test_parameters,
-      build_graph,
-      build_inputs,
-      expected_tf_success=4)
+  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
 
 
 # Since compute output_shape is fairly complicated for
@@ -3611,7 +3735,12 @@ def make_pack_tests(zip_path):
     return all_values, sess.run(
         outputs, feed_dict=dict(zip(inputs, all_values)))
 
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=72)
 
 
 def make_unpack_tests(zip_path):
@@ -3702,13 +3831,18 @@ def make_fill_tests(zip_path):
     return [input1, input2], sess.run(
         outputs, feed_dict=dict(zip(inputs, [input1, input2])))
 
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=12)
 
 
 def _make_logical_tests(op):
   """Make a set of tests to do logical operations."""
 
-  def logical(zip_path):
+  def logical(zip_path, expected_tf_failures=0):
     """Generate examples."""
     test_parameters = [{
         "input_shape_pair": [([], []), ([1, 1, 1, 3], [1, 1, 1, 3]),
@@ -3733,19 +3867,24 @@ def _make_logical_tests(op):
       return [input_value1, input_value2], sess.run(
           outputs, feed_dict=dict(zip(inputs, [input_value1, input_value2])))
 
-    make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+    make_zip_of_tests(
+        zip_path,
+        test_parameters,
+        build_graph,
+        build_inputs,
+        expected_tf_failures=expected_tf_failures)
 
   return logical
 
 
 def make_logical_or_tests(zip_path):
   """Make a set of tests to do logical_or."""
-  return _make_logical_tests(tf.logical_or)(zip_path)
+  return _make_logical_tests(tf.logical_or)(zip_path, expected_tf_failures=1)
 
 
 def make_logical_and_tests(zip_path):
   """Make a set of tests to do logical_and."""
-  return _make_logical_tests(tf.logical_and)(zip_path)
+  return _make_logical_tests(tf.logical_and)(zip_path, expected_tf_failures=1)
 
 
 def make_logical_xor_tests(zip_path):
@@ -3753,7 +3892,7 @@ def make_logical_xor_tests(zip_path):
 
     Test logical_not as well.
   """
-  return _make_logical_tests(tf.logical_xor)(zip_path)
+  return _make_logical_tests(tf.logical_xor)(zip_path, expected_tf_failures=1)
 
 
 def make_mirror_pad_tests(zip_path):
@@ -3836,12 +3975,7 @@ def make_mirror_pad_tests(zip_path):
     return input_values, sess.run(
         outputs, feed_dict=dict(zip(inputs, input_values)))
 
-  make_zip_of_tests(
-      zip_path,
-      test_parameters,
-      build_graph,
-      build_inputs,
-      expected_tf_success=8)
+  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
 
 
 def make_unroll_batch_matmul_tests(zip_path):
@@ -3893,8 +4027,7 @@ def make_placeholder_with_default_tests(zip_path):
     return [input_value], sess.run(
         outputs, feed_dict=dict(zip(inputs, [input_value])))
 
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs,
-                    expected_tf_success=3)
+  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
 
 
 def make_unique_tests(zip_path):
@@ -3939,12 +4072,7 @@ def make_unique_tests(zip_path):
     return input_values, sess.run(
         outputs, feed_dict=dict(zip(inputs, input_values)))
 
-  make_zip_of_tests(
-      zip_path,
-      test_parameters,
-      build_graph,
-      build_inputs,
-      expected_tf_success=9)
+  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
 
 
 def make_reverse_v2_tests(zip_path):
-- 
GitLab


From 3d4d2b2dd72d67b2a6e9b6c98b5a0c2275c48535 Mon Sep 17 00:00:00 2001
From: Jeff Poznanovic <jeffrey.poznanovic@amd.com>
Date: Fri, 8 Feb 2019 14:17:00 -0700
Subject: [PATCH 2175/2345] Update rocm/run_py3_core.sh

Update rocm/run_py3_core.sh to minimize the unit test exclusion list, to add a "-no_rocm" tag filter, and to rearrange the order of bazel flags.
---
 .../tools/ci_build/linux/rocm/run_py3_core.sh | 62 +------------------
 1 file changed, 3 insertions(+), 59 deletions(-)

diff --git a/tensorflow/tools/ci_build/linux/rocm/run_py3_core.sh b/tensorflow/tools/ci_build/linux/rocm/run_py3_core.sh
index 00a46af711..1944183c0e 100755
--- a/tensorflow/tools/ci_build/linux/rocm/run_py3_core.sh
+++ b/tensorflow/tools/ci_build/linux/rocm/run_py3_core.sh
@@ -35,65 +35,9 @@ export TF_GPU_COUNT=${N_GPUS}
 yes "" | $PYTHON_BIN_PATH configure.py
 
 # Run bazel test command. Double test timeouts to avoid flakes.
-bazel test --test_sharding_strategy=disabled --config=rocm --test_tag_filters=-no_oss,-oss_serial,-no_gpu,-benchmark-test -k \
+bazel test --config=rocm --test_tag_filters=-no_oss,-oss_serial,-no_gpu,-no_rocm,-benchmark-test -k \
     --test_lang_filters=py --jobs=${N_JOBS} --test_timeout 600,900,2400,7200 \
     --build_tests_only --test_output=errors --local_test_jobs=${TF_GPU_COUNT} --config=opt \
+    --test_sharding_strategy=disabled \
     --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute -- \
-    //tensorflow/... -//tensorflow/compiler/... -//tensorflow/contrib/... \
-    -//tensorflow/python/distribute:distribute_coordinator_test \
-    -//tensorflow/python/feature_column:feature_column_v2_test \
-    -//tensorflow/python/keras:cudnn_recurrent_test \
-    -//tensorflow/python/keras:cudnn_recurrent_test_gpu \
-    -//tensorflow/python/kernel_tests:batch_matmul_op_test \
-    -//tensorflow/python/kernel_tests:batch_matmul_op_test_gpu \
-    -//tensorflow/python/kernel_tests:conv1d_test \
-    -//tensorflow/python/kernel_tests:conv1d_test_gpu \
-    -//tensorflow/python/kernel_tests:conv2d_backprop_filter_grad_test \
-    -//tensorflow/python/kernel_tests:conv2d_backprop_filter_grad_test_gpu \
-    -//tensorflow/python/kernel_tests:conv_ops_3d_test \
-    -//tensorflow/python/kernel_tests:conv_ops_3d_test_gpu \
-    -//tensorflow/python/kernel_tests:conv_ops_test \
-    -//tensorflow/python/kernel_tests:conv_ops_test_gpu \
-    -//tensorflow/python/kernel_tests:depthwise_conv_op_test \
-    -//tensorflow/python/kernel_tests:depthwise_conv_op_test_gpu \
-    -//tensorflow/python/kernel_tests:list_ops_test \
-    -//tensorflow/python/kernel_tests:list_ops_test_gpu \
-    -//tensorflow/python/kernel_tests:matmul_op_test \
-    -//tensorflow/python/kernel_tests:matmul_op_test_gpu \
-    -//tensorflow/python/kernel_tests:pool_test \
-    -//tensorflow/python/kernel_tests:pool_test_gpu \
-    -//tensorflow/python/kernel_tests:pooling_ops_3d_test \
-    -//tensorflow/python/kernel_tests:pooling_ops_3d_test_gpu \
-    -//tensorflow/python/kernel_tests:pooling_ops_test \
-    -//tensorflow/python/kernel_tests:pooling_ops_test_gpu \
-    -//tensorflow/python/ops/parallel_for:control_flow_ops_test \
-    -//tensorflow/python/ops/parallel_for:gradients_test \
-    -//tensorflow/python:layers_normalization_test \
-    -//tensorflow/python:layers_normalization_test_gpu \
-    -//tensorflow/python:layout_optimizer_test \
-    -//tensorflow/python:nn_fused_batchnorm_test \
-    -//tensorflow/python:nn_fused_batchnorm_test_gpu \
-    -//tensorflow/python/eager:backprop_test \
-    -//tensorflow/python/eager:backprop_test_gpu \
-    -//tensorflow/python/kernel_tests:qr_op_test \
-    -//tensorflow/python/kernel_tests:qr_op_test_gpu \
-    -//tensorflow/python/kernel_tests/signal:dct_ops_test \
-    -//tensorflow/python/kernel_tests/signal:fft_ops_test \
-    -//tensorflow/python/kernel_tests/signal:spectral_ops_test \
-    -//tensorflow/python/ops/parallel_for:control_flow_ops_test_gpu \
-    -//tensorflow/python/kernel_tests:tensor_array_ops_test \
-    -//tensorflow/python/kernel_tests:tensor_array_ops_test_gpu \
-    -//tensorflow/python/keras:unified_lstm_test \
-    -//tensorflow/python/keras:unified_lstm_test_gpu \
-    -//tensorflow/python/keras:unified_gru_test \
-    -//tensorflow/python/keras:unified_gru_test_gpu \
-    -//tensorflow/python/keras:training_generator_test \
-    -//tensorflow/python/keras/optimizer_v2:adamax_test \
-    -//tensorflow/python/keras/optimizer_v2:adamax_test_gpu \
-    -//tensorflow/tools/api/tests:api_compatibility_test \
-    -//tensorflow/python/training/checkpointable:util_xla_test_cpu \
-    -//tensorflow/python/kernel_tests/signal:mfcc_ops_test \
-    -//tensorflow/python/kernel_tests:self_adjoint_eig_op_test_gpu \
-    -//tensorflow/python/keras:normalization_test \
-    -//tensorflow/python/keras:base_layer_test \
-    -//tensorflow/python/eager:def_function_xla_test_cpu \
+    //tensorflow/... -//tensorflow/compiler/... -//tensorflow/contrib/...
-- 
GitLab


From 30fcd7ef3ee2a8433841fd1c1a3cda239dac5188 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 8 Feb 2019 12:32:03 -0800
Subject: [PATCH 2176/2345] Use gtl::InlinedVector to avoid heap allocations
 when serializing, hashing, and comparing small protocol buffers.

Benchmark numbers for serialization:

name                                          old time/op             new time/op             delta
BM_ProtoSerializationToBuffer/1                74.8ns ? 0%             60.3ns ? 0%   -19.41%          (p=0.008 n=5+5)
BM_ProtoSerializationToBuffer/8                 361ns ? 0%              346ns ? 0%    -4.20%          (p=0.016 n=5+4)
BM_ProtoSerializationToBuffer/64              2.57?s ? 0%             2.55?s ? 1%    -1.05%          (p=0.016 n=4+5)
BM_ProtoSerializationToBuffer/512             20.4?s ? 0%             20.5?s ? 1%      ~             (p=0.841 n=5+5)
BM_ProtoSerializationToBuffer/4k               173?s ? 2%              170?s ? 2%      ~             (p=0.095 n=5+5)
BM_ProtoSerializationToBuffer/10k              432?s ? 1%              420?s ? 0%    -2.85%          (p=0.008 n=5+5)

name                                          old allocs/op           new allocs/op           delta
BM_ProtoSerializationToBuffer/1                  1.00 ? 0%              0.00 ?NaN%  -100.00%          (p=0.008 n=5+5)
BM_ProtoSerializationToBuffer/8                  1.00 ? 0%              0.00 ?NaN%  -100.00%          (p=0.008 n=5+5)
BM_ProtoSerializationToBuffer/64                 1.00 ? 0%              0.00 ?NaN%  -100.00%          (p=0.008 n=5+5)
BM_ProtoSerializationToBuffer/512                1.00 ? 0%               1.00 ? 0%      ~     (all samples are equal)
BM_ProtoSerializationToBuffer/4k                 1.00 ? 0%               1.00 ? 0%      ~     (all samples are equal)
BM_ProtoSerializationToBuffer/10k                1.00 ? 0%               1.00 ? 0%      ~     (all samples are equal)

name                                          old peak-mem(Bytes)/op  new peak-mem(Bytes)/op  delta
BM_ProtoSerializationToBuffer/1                  36.0 ? 0%               36.0 ? 0%      ~     (all samples are equal)
BM_ProtoSerializationToBuffer/8                   112 ? 0%                 36 ? 0%   -67.86%          (p=0.008 n=5+5)
BM_ProtoSerializationToBuffer/64                  950 ? 0%                 37 ? 0%   -96.11%          (p=0.008 n=5+5)
BM_ProtoSerializationToBuffer/512               8.08k ? 0%              8.08k ? 0%      ~     (all samples are equal)
BM_ProtoSerializationToBuffer/4k                68.5k ? 0%              68.5k ? 0%      ~     (all samples are equal)
BM_ProtoSerializationToBuffer/10k                169k ? 0%               169k ? 0%      ~     (all samples are equal)

Benchmark numbers for hashing and comparison.
BM_DeterministicProtoHash64/1            278       260     +6.5%
BM_DeterministicProtoHash64/8           1969      1976     -0.4%
BM_DeterministicProtoHash64/64         16021     16043     -0.1%
BM_DeterministicProtoHash64/512       127188    127400     -0.2%
BM_DeterministicProtoHash64/4k       1089472   1104291     -1.4%
BM_DeterministicProtoHash64/10k      2594413   2621487     -1.0%
BM_AreSerializedProtosEqual/1            535       514     +3.9%
BM_AreSerializedProtosEqual/8           3853      3849     +0.1%
BM_AreSerializedProtosEqual/64         31212     31121     +0.3%
BM_AreSerializedProtosEqual/512       249554    249010     +0.2%
BM_AreSerializedProtosEqual/4k       2096963   2133310     -1.7%
BM_AreSerializedProtosEqual/10k      5027297   5100622     -1.5%

PiperOrigin-RevId: 233106678
---
 .../core/lib/strings/proto_serialization.cc   | 26 ++++++----
 .../lib/strings/proto_serialization_test.cc   | 52 ++++++++++++++-----
 2 files changed, 55 insertions(+), 23 deletions(-)

diff --git a/tensorflow/core/lib/strings/proto_serialization.cc b/tensorflow/core/lib/strings/proto_serialization.cc
index a6c321c782..2341d3e341 100644
--- a/tensorflow/core/lib/strings/proto_serialization.cc
+++ b/tensorflow/core/lib/strings/proto_serialization.cc
@@ -17,11 +17,15 @@ limitations under the License.
 #include <cstring>
 #include "absl/memory/memory.h"
 #include "absl/strings/string_view.h"
+#include "tensorflow/core/lib/gtl/inlined_vector.h"
 #include "tensorflow/core/lib/hash/hash.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/macros.h"
 
 namespace tensorflow {
+namespace {
+static const int kInlinedBufferSize = 256;
+}  // namespace
 
 bool SerializeToStringDeterministic(const protobuf::MessageLite& msg,
                                     string* result) {
@@ -47,28 +51,28 @@ bool AreSerializedProtosEqual(const protobuf::MessageLite& x,
   const size_t size = x.ByteSizeLong();
   if (size != y.ByteSizeLong()) return false;
   if (size == 0) return true;
-  auto x_serialized = absl::make_unique<char[]>(size);
-  bool success_x = SerializeToBufferDeterministic(x, x_serialized.get(), size);
+  gtl::InlinedVector<char, kInlinedBufferSize> x_serialized(size);
+  bool success_x = SerializeToBufferDeterministic(x, x_serialized.data(), size);
   DCHECK(success_x);
-  auto y_serialized = absl::make_unique<char[]>(size);
-  bool success_y = SerializeToBufferDeterministic(y, y_serialized.get(), size);
+  gtl::InlinedVector<char, kInlinedBufferSize> y_serialized(size);
+  bool success_y = SerializeToBufferDeterministic(y, y_serialized.data(), size);
   DCHECK(success_y);
-  return memcmp(x_serialized.get(), y_serialized.get(), size) == 0;
+  return memcmp(x_serialized.data(), y_serialized.data(), size) == 0;
 }
 
 uint64 DeterministicProtoHash64(const protobuf::MessageLite& proto,
                                 uint64 seed) {
   const size_t size = proto.ByteSizeLong();
-  auto serialized = absl::make_unique<char[]>(size);
-  SerializeToBufferDeterministic(proto, serialized.get(), size);
-  return Hash64(serialized.get(), size, seed);
+  gtl::InlinedVector<char, kInlinedBufferSize> serialized(size);
+  SerializeToBufferDeterministic(proto, serialized.data(), size);
+  return Hash64(serialized.data(), size, seed);
 }
 
 uint64 DeterministicProtoHash64(const protobuf::MessageLite& proto) {
   const size_t size = proto.ByteSizeLong();
-  auto serialized = absl::make_unique<char[]>(size);
-  SerializeToBufferDeterministic(proto, serialized.get(), size);
-  return Hash64(serialized.get(), size);
+  gtl::InlinedVector<char, kInlinedBufferSize> serialized(size);
+  SerializeToBufferDeterministic(proto, serialized.data(), size);
+  return Hash64(serialized.data(), size);
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/lib/strings/proto_serialization_test.cc b/tensorflow/core/lib/strings/proto_serialization_test.cc
index cbde20eced..81a6f08ae9 100644
--- a/tensorflow/core/lib/strings/proto_serialization_test.cc
+++ b/tensorflow/core/lib/strings/proto_serialization_test.cc
@@ -17,22 +17,32 @@ limitations under the License.
 
 #include <string>
 #include "absl/memory/memory.h"
+#include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
+#include "tensorflow/core/lib/gtl/inlined_vector.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/platform/test_benchmark.h"
 
 namespace tensorflow {
-
-static void BM_ProtoSerializationToString(int iters, int num_nodes) {
-  testing::StopTiming();
+namespace {
+GraphDef MakeGraphDef(int num_nodes) {
   GraphDef graph_def;
   for (int i = 0; i < num_nodes; ++i) {
     NodeDef* node = graph_def.add_node();
     node->set_name(strings::StrCat("node", i));
     node->set_op(strings::StrCat("op", i % 10));
+    (*node->mutable_attr())["foo"].set_f(3.14f);
+    (*node->mutable_attr())["bar"].set_s("baz");
   }
+  return graph_def;
+}
+}  // namespace
+
+static void BM_ProtoSerializationToString(int iters, int num_nodes) {
+  testing::StopTiming();
+  GraphDef graph_def = MakeGraphDef(num_nodes);
   testing::StartTiming();
   for (int i = 0; i < iters; ++i) {
     string serialized;
@@ -45,22 +55,40 @@ BENCHMARK(BM_ProtoSerializationToString)->Range(1, 10000);
 
 static void BM_ProtoSerializationToBuffer(int iters, int num_nodes) {
   testing::StopTiming();
-  GraphDef graph_def;
-  for (int i = 0; i < num_nodes; ++i) {
-    NodeDef* node = graph_def.add_node();
-    node->set_name(strings::StrCat("node", i));
-    node->set_op(strings::StrCat("op", i % 10));
-  }
+  GraphDef graph_def = MakeGraphDef(num_nodes);
   testing::StartTiming();
   const size_t size = graph_def.ByteSizeLong();
   for (int i = 0; i < iters; ++i) {
-    auto buf = absl::make_unique<char[]>(size);
+    gtl::InlinedVector<char, 1024> buf(size);
     testing::DoNotOptimize(
-        SerializeToBufferDeterministic(graph_def, buf.get(), size));
+        SerializeToBufferDeterministic(graph_def, buf.data(), size));
   }
   testing::StopTiming();
 }
-
 BENCHMARK(BM_ProtoSerializationToBuffer)->Range(1, 10000);
 
+static void BM_DeterministicProtoHash64(int iters, int num_nodes) {
+  testing::StopTiming();
+  GraphDef graph_def = MakeGraphDef(num_nodes);
+  testing::StartTiming();
+  for (int i = 0; i < iters; ++i) {
+    testing::DoNotOptimize(DeterministicProtoHash64(graph_def));
+  }
+  testing::StopTiming();
+}
+BENCHMARK(BM_DeterministicProtoHash64)->Range(1, 10000);
+
+static void BM_AreSerializedProtosEqual(int iters, int num_nodes) {
+  testing::StopTiming();
+  GraphDef graph_def_a = MakeGraphDef(num_nodes);
+  GraphDef graph_def_b = MakeGraphDef(num_nodes);
+  graph_def_b.mutable_node(0)->mutable_name()[0] = 'l';
+  testing::StartTiming();
+  for (int i = 0; i < iters; ++i) {
+    testing::DoNotOptimize(AreSerializedProtosEqual(graph_def_a, graph_def_a));
+  }
+  testing::StopTiming();
+}
+BENCHMARK(BM_AreSerializedProtosEqual)->Range(1, 10000);
+
 }  // namespace tensorflow
-- 
GitLab


From b874e59e5cd25ac292149283ec86fe4437e97cca Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Fri, 8 Feb 2019 12:41:27 -0800
Subject: [PATCH 2177/2345] TF/GCS: Adjust default cache block/maximum sizes to
 avoid using too much memory.

PiperOrigin-RevId: 233108117
---
 tensorflow/core/platform/cloud/gcs_file_system.cc      | 6 +++---
 tensorflow/core/platform/cloud/gcs_file_system_test.cc | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/core/platform/cloud/gcs_file_system.cc b/tensorflow/core/platform/cloud/gcs_file_system.cc
index d1e5779f02..10b57df183 100644
--- a/tensorflow/core/platform/cloud/gcs_file_system.cc
+++ b/tensorflow/core/platform/cloud/gcs_file_system.cc
@@ -64,7 +64,7 @@ constexpr int kGetChildrenDefaultPageSize = 1000;
 // The HTTP response code "308 Resume Incomplete".
 constexpr uint64 HTTP_CODE_RESUME_INCOMPLETE = 308;
 // The environment variable that overrides the size of the readahead buffer.
-ABSL_DEPRECATED("Use GCS_BLOCK_SIZE_MB instead.")
+ABSL_DEPRECATED("Use GCS_READ_CACHE_BLOCK_SIZE_MB instead.")
 constexpr char kReadaheadBufferSize[] = "GCS_READAHEAD_BUFFER_SIZE_BYTES";
 // The environment variable that disables the GCS block cache for reads.
 // This is the explicit alternative to setting BLOCK_SIZE or MAX_SIZE to 0, and
@@ -73,11 +73,11 @@ constexpr char kReadCacheDisabled[] = "GCS_READ_CACHE_DISABLED";
 // The environment variable that overrides the block size for aligned reads from
 // GCS. Specified in MB (e.g. "16" = 16 x 1024 x 1024 = 16777216 bytes).
 constexpr char kBlockSize[] = "GCS_READ_CACHE_BLOCK_SIZE_MB";
-constexpr size_t kDefaultBlockSize = 128 * 1024 * 1024;
+constexpr size_t kDefaultBlockSize = 16 * 1024 * 1024;
 // The environment variable that overrides the max size of the LRU cache of
 // blocks read from GCS. Specified in MB.
 constexpr char kMaxCacheSize[] = "GCS_READ_CACHE_MAX_SIZE_MB";
-constexpr size_t kDefaultMaxCacheSize = 2 * kDefaultBlockSize;
+constexpr size_t kDefaultMaxCacheSize = kDefaultBlockSize;
 // The environment variable that overrides the maximum staleness of cached file
 // contents. Once any block of a file reaches this staleness, all cached blocks
 // will be evicted on the next read.
diff --git a/tensorflow/core/platform/cloud/gcs_file_system_test.cc b/tensorflow/core/platform/cloud/gcs_file_system_test.cc
index dc9eb7796f..a998f8e3ad 100644
--- a/tensorflow/core/platform/cloud/gcs_file_system_test.cc
+++ b/tensorflow/core/platform/cloud/gcs_file_system_test.cc
@@ -3155,8 +3155,8 @@ TEST(GcsFileSystemTest, AdditionalRequestHeaderTest) {
 TEST(GcsFileSystemTest, OverrideCacheParameters) {
   // Verify defaults are propagated correctly.
   GcsFileSystem fs1;
-  EXPECT_EQ(128 * 1024 * 1024, fs1.block_size());
-  EXPECT_EQ(2 * fs1.block_size(), fs1.max_bytes());
+  EXPECT_EQ(16 * 1024 * 1024, fs1.block_size());
+  EXPECT_EQ(fs1.block_size(), fs1.max_bytes());
   EXPECT_EQ(0, fs1.max_staleness());
   EXPECT_EQ(120, fs1.timeouts().connect);
   EXPECT_EQ(60, fs1.timeouts().idle);
-- 
GitLab


From e94e267677bb7afbcd6426545bb43a32c6b7cdcd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 8 Feb 2019 12:45:12 -0800
Subject: [PATCH 2178/2345] Fix build: PyUnicode_AsUTF8AndSize returns a const
 char* in Python 3.7

PiperOrigin-RevId: 233108654
---
 tensorflow/lite/python/interpreter_wrapper/python_utils.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/python/interpreter_wrapper/python_utils.cc b/tensorflow/lite/python/interpreter_wrapper/python_utils.cc
index a052ca320f..526488a9f9 100644
--- a/tensorflow/lite/python/interpreter_wrapper/python_utils.cc
+++ b/tensorflow/lite/python/interpreter_wrapper/python_utils.cc
@@ -87,7 +87,7 @@ TfLiteType TfLiteTypeFromPyArray(PyArrayObject* array) {
 bool FillStringBufferFromPyUnicode(PyObject* value,
                                    DynamicBuffer* dynamic_buffer) {
   Py_ssize_t len = -1;
-  char* buf = PyUnicode_AsUTF8AndSize(value, &len);
+  const char* buf = PyUnicode_AsUTF8AndSize(value, &len);
   if (buf == NULL) {
     PyErr_SetString(PyExc_ValueError, "PyUnicode_AsUTF8AndSize() failed.");
     return false;
-- 
GitLab


From cd734fefb91d62cf94a3929890ac2d8eb138792b Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Fri, 8 Feb 2019 12:53:43 -0800
Subject: [PATCH 2179/2345] str() -> compat.as_str_any()

I have no idea whether this will fix the Windows build or not. But it looked like one side of the comparison was still bytes, so maybe this will make it a string.

PiperOrigin-RevId: 233110061
---
 tensorflow/python/saved_model/save_test.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/saved_model/save_test.py b/tensorflow/python/saved_model/save_test.py
index 37e8dc6884..b76a986578 100644
--- a/tensorflow/python/saved_model/save_test.py
+++ b/tensorflow/python/saved_model/save_test.py
@@ -43,6 +43,7 @@ from tensorflow.python.saved_model import signature_constants
 from tensorflow.python.saved_model import tag_constants
 from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.training.checkpointable import util
+from tensorflow.python.util import compat
 
 
 class _ModelWithOptimizer(util.Checkpoint):
@@ -329,7 +330,8 @@ class AssetTests(test.TestCase):
     second_dir = os.path.join(self.get_temp_dir(), "second_dir")
     file_io.rename(save_dir, second_dir)
     imported_path = _import_and_infer(second_dir, {})["output_0"]
-    self.assertIn(str(second_dir), str(imported_path))
+    self.assertIn(compat.as_str_any(second_dir),
+                  compat.as_str_any(imported_path))
 
   def test_table(self):
     initializer = lookup_ops.TextFileInitializer(
-- 
GitLab


From ad9b2df6a6eac0511f39e1f6bbfc45bea76f84ee Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Fri, 8 Feb 2019 12:57:23 -0800
Subject: [PATCH 2180/2345] Adding the output loss metric wrappers for eager
 mode.

PiperOrigin-RevId: 233110700
---
 tensorflow/python/keras/engine/training.py    | 183 +++++++++++------
 .../python/keras/engine/training_eager.py     | 187 ++++++++++++------
 tensorflow/python/keras/losses.py             |   3 -
 .../python/keras/metrics_correctness_test.py  |  21 +-
 tensorflow/python/keras/models.py             |   3 +-
 tensorflow/python/keras/utils/losses_utils.py |   6 +-
 6 files changed, 251 insertions(+), 152 deletions(-)

diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index ae9992a9b4..66c29bbfdd 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -374,15 +374,9 @@ class Model(Network):
         raise ValueError('target_tensors are not currently supported in Eager '
                          'mode.')
       self.total_loss = None
-      for i in range(len(self.outputs)):
-        if len(self.outputs) > 1:
-          self._compile_metrics_names.append(self.output_names[i] + '_loss')
 
       # Set metric attributes on model.
-      self._set_metric_attributes(
-          self.outputs,
-          skip_target_indices=skip_target_indices,
-      )
+      self._set_metric_attributes(skip_target_indices=skip_target_indices)
 
       self.targets = []
       for i in range(len(self.outputs)):
@@ -455,7 +449,22 @@ class Model(Network):
       # Save all metric attributes per output of the model.
       self._cache_output_metric_attributes(metrics, weighted_metrics)
 
+      # Set metric attributes on model.
+      self._set_metric_attributes(skip_target_indices=skip_target_indices)
+
+      # Invoke metric functions for all the outputs.
+      self._handle_metrics(
+          self.outputs,
+          masks=masks,
+          targets=self.targets,
+          skip_target_indices=skip_target_indices,
+          sample_weights=self.sample_weights)
+
       # Compute total loss.
+      # Used to keep track of the total loss value (stateless).
+      # eg., total_loss = loss_weight_1 * output_1_loss_fn(...) +
+      #                   loss_weight_2 * output_2_loss_fn(...) +
+      #                   layer losses.
       total_loss = None
       with K.name_scope('loss'):
         for i in range(len(self.outputs)):
@@ -480,25 +489,49 @@ class Model(Network):
                         mask, None, sample_weight))
                 sample_weight *= mask
 
-            output_loss = loss_fn(y_true, y_pred, sample_weight=sample_weight)
+            # Reset reduction on the loss so that we can get the per sample loss
+            # value. We use this to get both the stateless and stateful loss
+            # values without having to compute the underlying loss function
+            # twice.
+            weighted_losses = None
+            if hasattr(loss_fn, 'reduction'):
+              current_loss_reduction = loss_fn.reduction
+              loss_fn.reduction = losses_utils.ReductionV2.NONE
+              weighted_losses = loss_fn(
+                  y_true, y_pred, sample_weight=sample_weight)
+              loss_fn.reduction = current_loss_reduction
+
+              # Compute the stateless loss value.
+              output_loss = losses_utils.reduce_weighted_loss(
+                  weighted_losses, reduction=current_loss_reduction)
+            else:
+              # Compute the stateless loss value for a custom loss class.
+              # Here we assume that the class takes care of loss reduction
+              # because if this class returns a vector value we cannot
+              # differentiate between use case where a custom optimizer
+              # expects a vector loss value vs unreduced per-sample loss value.
+              output_loss = loss_fn(y_true, y_pred, sample_weight=sample_weight)
 
           if len(self.outputs) > 1:
             # Keep track of the un-aggregated loss result tensor.
-            self._compile_metrics_tensors[self.output_names[i] +
-                                          '_loss'] = output_loss
+            output_name = self.output_names[i] + '_loss'
+            self._compile_metrics_tensors[output_name] = output_loss
 
             # Keep track of stateful result tensor and function for the loss.
-            # Reset reduction here as metric wrapper will take care of that.
-            loss_fn.reduction = losses_utils.ReductionV2.NONE
-            output_loss_metric = metrics_module.SumOverBatchSizeMetricWrapper(
-                loss_fn, name=loss_fn.name)
-            result_tensor = self._call_metric_fn(output_loss_metric, y_true,
-                                                 y_pred, sample_weight, mask)
-            self._compile_stateful_metrics_tensors[self.output_names[i] +
-                                                   '_loss'] = result_tensor
-            self._compile_stateful_metric_functions.append(output_loss_metric)
-
-            self._compile_metrics_names.append(self.output_names[i] + '_loss')
+            # Compute the stateful loss value.
+            if weighted_losses is not None:
+              # TODO(b/120571621): Directly call metric when the bug is fixed.
+              aggregated_output_loss = self._call_fn_for_each_replica(
+                  self._output_loss_metrics[i], weighted_losses)
+            else:
+              # Custom loss class.
+              aggregated_output_loss = self._call_metric_fn(
+                  self._output_loss_metrics[i], y_true, y_pred, sample_weight)
+            self._compile_stateful_metrics_tensors[
+                output_name] = aggregated_output_loss
+            self._compile_stateful_metric_functions.append(
+                self._output_loss_metrics[i])
+
           if total_loss is None:
             total_loss = loss_weight * output_loss
           else:
@@ -515,19 +548,6 @@ class Model(Network):
         for loss_tensor in self.losses:
           total_loss += loss_tensor
 
-      # Set metric attributes on model.
-      self._set_metric_attributes(
-          self.outputs,
-          skip_target_indices=skip_target_indices,
-      )
-      # Invoke metric functions for all the outputs.
-      self._handle_metrics(
-          self.outputs,
-          masks=masks,
-          targets=self.targets,
-          skip_target_indices=skip_target_indices,
-          sample_weights=self.sample_weights)
-
       # Prepare gradient updates and state updates.
       self.total_loss = total_loss
 
@@ -1261,8 +1281,16 @@ class Model(Network):
     if hasattr(self, 'metrics'):
       for m in self.metrics:
         m.reset_states()
-      if self._distribution_strategy:
-        distributed_training_utils._reset_metrics(self)  # pylint: disable=protected-access
+
+    # Reset the state of loss metric wrappers.
+    if hasattr(
+        self, '_output_loss_metrics') and self._output_loss_metrics is not None:
+      for m in self._output_loss_metrics:
+        m.reset_states()
+
+    # Reset metrics on all the distributed (cloned) models.
+    if self._distribution_strategy:
+      distributed_training_utils._reset_metrics(self)  # pylint: disable=protected-access
 
   def train_on_batch(self,
                      x,
@@ -1321,7 +1349,12 @@ class Model(Network):
 
     if self.run_eagerly:
       outputs = training_eager.train_on_batch(
-          self, x, y, sample_weights=sample_weights)
+          self,
+          x,
+          y,
+          sample_weights=sample_weights,
+          reset_metrics=reset_metrics,
+          output_loss_metrics=self._output_loss_metrics)
     else:
       if not isinstance(K.symbolic_learning_phase(), int):
         ins = x + y + sample_weights + [True]
@@ -1390,7 +1423,12 @@ class Model(Network):
 
     if self.run_eagerly:
       outputs = training_eager.test_on_batch(
-          self, x, y, sample_weights=sample_weights)
+          self,
+          x,
+          y,
+          sample_weights=sample_weights,
+          reset_metrics=reset_metrics,
+          output_loss_metrics=self._output_loss_metrics)
     else:
       inputs = x + y + sample_weights
       if reset_metrics:
@@ -1818,10 +1856,13 @@ class Model(Network):
         output_shapes.append(output.shape.as_list())
     self._per_output_metrics = training_utils.collect_per_output_metric_info(
         metrics, self.output_names, output_shapes, self.loss_functions)
-    self._per_output_weighted_metrics = \
+    self._per_output_weighted_metrics = (
         training_utils.collect_per_output_metric_info(
-            weighted_metrics, self.output_names, output_shapes,
-            self.loss_functions, is_weighted=True)
+            weighted_metrics,
+            self.output_names,
+            output_shapes,
+            self.loss_functions,
+            is_weighted=True))
 
   def _add_unique_metric_name(self, metric_name, output_index):
     """Makes the metric name unique and adds it to the model's metric name list.
@@ -1868,19 +1909,19 @@ class Model(Network):
 
   def _init_metric_attributes(self):
     """Initialized model metric attributes."""
-    # List of all metric names in the model.
+    # List of all metric names in the model. This includes loss metrics.
     self._compile_metrics_names = ['loss']
     # List of stateful metric functions. Used for resetting metric state during
-    # training/eval.
-    # This includes loss functions when there are multiple outputs.
+    # training/eval. This includes loss metric functions.
     self._compile_stateful_metric_functions = []
     # Dict of all aggregated metric result tensors. This includes aggregated
-    # loss result tensors when there are multiple outputs.
+    # loss result tensors.
     self._compile_stateful_metrics_tensors = {}
     # Dict of all metric result tensors (aggregated or not - based on the
-    # values given in compile.). This includes aggregated loss result tensors
-    # when there are multiple outputs.
+    # values given in compile.). This includes aggregated loss result tensors.
     self._compile_metrics_tensors = {}
+    # List of metric wrappers on output losses.
+    self._output_loss_metrics = None
 
   def _set_per_output_metric_attributes(self, metrics_dict, output_index):
     """Sets the metric attributes on the model for the given output.
@@ -1902,12 +1943,21 @@ class Model(Network):
       self._compile_stateful_metric_functions.append(stateful_metric_fn)
     return updated_metrics_dict
 
-  def _set_metric_attributes(self, outputs, skip_target_indices=None):
+  def _set_metric_attributes(self, skip_target_indices=None):
     """Sets the metric attributes on the model for all the model outputs."""
+    # Add loss metric names to the model metric names list.
+    if len(self.outputs) > 1:
+      output_names = [
+          self.output_names[i] + '_loss'
+          for i in range(len(self.outputs))
+          if i not in skip_target_indices
+      ]
+      self._compile_metrics_names.extend(output_names)
+
     skip_target_indices = skip_target_indices or []
     updated_per_output_metrics = []
     updated_per_output_weighted_metrics = []
-    for i in range(len(outputs)):
+    for i in range(len(self.outputs)):
       if i in skip_target_indices:
         updated_per_output_metrics.append(self._per_output_metrics[i])
         updated_per_output_weighted_metrics.append(
@@ -1920,11 +1970,28 @@ class Model(Network):
           self._set_per_output_metric_attributes(
               self._per_output_weighted_metrics[i], i))
 
+    # Create a metric wrapper for each output loss.
+    self._output_loss_metrics = [
+        metrics_module.SumOverBatchSize() if hasattr(loss_fn, 'reduction') else
+        metrics_module.SumOverBatchSizeMetricWrapper(loss_fn)
+        for loss_fn in self.loss_functions
+    ]
+
     self._per_output_metrics = updated_per_output_metrics
     self._per_output_weighted_metrics = updated_per_output_weighted_metrics
 
-  def _call_metric_fn(self, fn, y_true, y_pred, weights, mask):
+  def _call_metric_fn(self, metric_fn, y_true, y_pred, weights, mask=None):
+    # TODO(b/120571621): Remove this function when the bug is fixed.
     """Helper function to call metric function with distribution strategy."""
+    return self._call_fn_for_each_replica(
+        training_utils.call_metric_function,
+        metric_fn,
+        y_true,
+        y_pred,
+        weights=weights,
+        mask=mask)
+
+  def _call_fn_for_each_replica(self, fn, *args, **kwargs):
     # TODO(b/120571621): We want to avoid metric reductions here since
     # since TPUStrategy does not implement replica local variables.
     # Remove this hack once we support TPUReplicaLocalVariables.
@@ -1934,10 +2001,8 @@ class Model(Network):
         distribution_strategy_context.in_cross_replica_context()):
       with self._distribution_strategy.scope():
         return self._distribution_strategy.extended.call_for_each_replica(
-            training_utils.call_metric_function,
-            (fn, y_true, y_pred, weights, mask))
-    return training_utils.call_metric_function(
-        fn, y_true, y_pred, weights=weights, mask=mask)
+            fn, args, kwargs)
+    return fn(*args, **kwargs)
 
   def _handle_per_output_metrics(self,
                                  metrics_dict,
@@ -2072,7 +2137,7 @@ class Model(Network):
           ' trainable weights, did you set `model.trainable`'
           ' without calling `model.compile` after ?', 1)
 
-  def _make_train_function_helper(self, fn_name, outputs, metric_updates=None):
+  def _make_train_function_helper(self, fn_name, outputs):
     if not self._is_compiled:
       raise RuntimeError('You must compile your model before using it.')
     self._check_trainable_weights_consistency()
@@ -2093,9 +2158,6 @@ class Model(Network):
       updates += self.get_updates_for(None)
       # Conditional updates relevant to this model
       updates += self.get_updates_for(self.inputs)
-      # Add stateful metrics updates.
-      if metric_updates is not None:
-        updates += metric_updates
 
       with K.name_scope('training'):
         # Gets loss and metrics. Updates weights at each call.
@@ -2121,7 +2183,7 @@ class Model(Network):
     self._make_train_function_helper(
         '_fit_function', [self.total_loss] + metrics_tensors)
 
-  def _make_test_function_helper(self, fn_name, outputs, metric_updates=None):
+  def _make_test_function_helper(self, fn_name, outputs):
     if not self._is_compiled:
       raise RuntimeError('You must compile your model before using it.')
     if getattr(self, fn_name) is None:
@@ -2131,9 +2193,6 @@ class Model(Network):
 
       with K.name_scope('evaluation'):
         updates = self.state_updates
-        # Add stateful metrics updates.
-        if metric_updates is not None:
-          updates += metric_updates
         # Return loss and metrics, no gradient updates.
         # Does update the network states.
         fn = K.function(
diff --git a/tensorflow/python/keras/engine/training_eager.py b/tensorflow/python/keras/engine/training_eager.py
index 7f80f94d48..4574b194f3 100644
--- a/tensorflow/python/keras/engine/training_eager.py
+++ b/tensorflow/python/keras/engine/training_eager.py
@@ -28,7 +28,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.keras import backend
 from tensorflow.python.keras.engine import training_utils
-from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions
+from tensorflow.python.keras.utils import losses_utils
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import nest
@@ -96,6 +96,10 @@ def _model_loss(model,
      regularization losses and applies masking and sample weighting
      to the loss value.
   """
+  # Used to keep track of the total loss value (stateless).
+  # eg., total_loss = loss_weight_1 * output_1_loss_fn(...) +
+  #                   loss_weight_2 * output_2_loss_fn(...) +
+  #                   layer losses.
   total_loss = 0
   kwargs = {}
   if model._expects_training_arg:
@@ -116,14 +120,14 @@ def _model_loss(model,
   masks = [t._keras_mask for t in outs]
   targets = nest.flatten(targets)
 
-  loss_metrics = []
-  aggregated_loss_metrics = []
+  # Used to keep track of individual output losses (stateless).
+  output_losses = []
+  # Used to keep track of individual output losses (stateful).
+  aggregated_output_losses = []
+
   with backend.name_scope('loss'):
     for i, loss_fn in enumerate(model.loss_functions):
-      if sample_weights:
-        weights = sample_weights[i]
-      else:
-        weights = None
+      weights = sample_weights[i] if sample_weights else None
       mask = masks[i]
       with backend.name_scope(model.output_names[i] + '_loss'):
         if mask is not None:
@@ -133,26 +137,47 @@ def _model_loss(model,
             weights = mask
           else:
             # Update dimensions of weights to match with mask if possible.
-            mask, _, weights = squeeze_or_expand_dimensions(mask, None, weights)
+            mask, _, weights = (
+                losses_utils.squeeze_or_expand_dimensions(mask, None, weights))
             weights *= mask
-        output_loss = loss_fn(targets[i], outs[i], sample_weight=weights)
+
+        # Reset reduction on the loss so that we can get the per sample loss
+        # value. We use this to get both the stateless and stateful loss
+        # values without having to compute the underlying loss function
+        # twice.
+        weighted_losses = None
+        if hasattr(loss_fn, 'reduction'):
+          current_loss_reduction = loss_fn.reduction
+          loss_fn.reduction = losses_utils.ReductionV2.NONE
+          weighted_losses = loss_fn(targets[i], outs[i], sample_weight=weights)
+          loss_fn.reduction = current_loss_reduction
+
+          # Compute the stateless loss value.
+          output_loss = losses_utils.reduce_weighted_loss(weighted_losses)
+        else:
+          # Compute the stateless loss value for a custom loss class.
+          # Here we assume that the class takes care of loss reduction
+          # because if this class returns a vector value we cannot
+          # differentiate between use case where a custom optimizer
+          # expects a vector loss value vs unreduced per-sample loss value.
+          output_loss = loss_fn(targets[i], outs[i], sample_weight=weights)
 
       # If the number of outputs is 1 then we don't append the loss metric
       # associated with each model output. When there are multiple outputs
       # associated with a model, each output's loss is calculated and returned
       # as part of the loss_metrics.
       if len(model.outputs) > 1:
-        loss_metrics.append(backend.mean(output_loss))
-
+        output_losses.append(backend.mean(output_loss))
         if output_loss_metrics is not None:
-          # Keep track of the stateful loss result.
-          aggregated_loss_metrics.append(
-              training_utils.call_metric_function(
-                  output_loss_metrics[i],
-                  targets[i],
-                  outs[i],
-                  weights=weights,
-                  mask=mask))
+          # Compute the stateful loss value.
+          if weighted_losses is not None:
+            aggregated_output_loss = output_loss_metrics[i](weighted_losses)
+          else:
+            # Custom loss class.
+            aggregated_output_loss = training_utils.call_metric_function(
+                output_loss_metrics[i], targets[i], outs[i], weights=weights)
+          # Keep track of the stateful output loss result.
+          aggregated_output_losses.append(aggregated_output_loss)
 
       loss_weight = model.loss_weights_list[i]
       if total_loss is None:
@@ -167,7 +192,7 @@ def _model_loss(model,
       total_loss += math_ops.add_n(custom_losses)
     model._clear_losses()
 
-  return outs, total_loss, loss_metrics, aggregated_loss_metrics, masks
+  return outs, total_loss, output_losses, aggregated_output_losses, masks
 
 
 def _process_single_batch(model,
@@ -200,15 +225,15 @@ def _process_single_batch(model,
   """
   with backend.eager_learning_phase_scope(1 if training else 0):
     with GradientTape() as tape:
-      outs, loss, loss_metrics, aggregated_loss_metrics, masks\
-        = _model_loss(
-            model,
-            inputs,
-            targets,
-            output_loss_metrics=output_loss_metrics,
-            sample_weights=sample_weights,
-            training=training)
-      if loss is None:
+      outs, total_loss, output_losses, aggregated_output_losses, masks = (
+          _model_loss(
+              model,
+              inputs,
+              targets,
+              output_loss_metrics=output_loss_metrics,
+              sample_weights=sample_weights,
+              training=training))
+      if total_loss is None:
         raise ValueError('The model cannot be run '
                          'because it has no loss to optimize.')
     if training:
@@ -217,13 +242,18 @@ def _process_single_batch(model,
                         ' you are not setting model.trainable to False before '
                         'compiling the model.')
       else:
-        grads = tape.gradient(loss, model._collected_trainable_weights)
+        grads = tape.gradient(total_loss, model._collected_trainable_weights)
         model.optimizer.apply_gradients(zip(grads,
                                             model._collected_trainable_weights))
-    return outs, loss, loss_metrics, aggregated_loss_metrics, masks
+    return outs, total_loss, output_losses, aggregated_output_losses, masks
 
 
-def train_on_batch(model, inputs, targets, sample_weights=None):
+def train_on_batch(model,
+                   inputs,
+                   targets,
+                   sample_weights=None,
+                   reset_metrics=True,
+                   output_loss_metrics=None):
   """Calculates the loss and gradient updates for one input batch.
 
   Arguments:
@@ -231,6 +261,11 @@ def train_on_batch(model, inputs, targets, sample_weights=None):
       inputs: Input batch data.
       targets: Target batch data.
       sample_weights: Sample weight batch data.
+      reset_metrics: If `True`, the metrics returned will be only for this
+        batch. If `False`, the metrics will be statefully accumulated across
+        batches.
+      output_loss_metrics: List of metrics that are used to aggregated output
+        loss values.
 
   Returns:
       total loss and the loss associated with each output.
@@ -240,20 +275,24 @@ def train_on_batch(model, inputs, targets, sample_weights=None):
       inputs = training_utils.cast_if_floating_dtype(inputs)
       targets = training_utils.cast_if_floating_dtype(targets)
     else:
-      inputs = training_utils.cast_if_floating_dtype([
-          ops.convert_to_tensor(val) for val in inputs
-      ])
-      targets = training_utils.cast_if_floating_dtype([
-          ops.convert_to_tensor(val) for val in targets
-      ])
+      inputs = training_utils.cast_if_floating_dtype(
+          [ops.convert_to_tensor(val) for val in inputs])
+      targets = training_utils.cast_if_floating_dtype(
+          [ops.convert_to_tensor(val) for val in targets])
   if sample_weights:
     sample_weights = [
         training_utils.cast_if_floating_dtype(ops.convert_to_tensor(val))
         if val is not None else None for val in sample_weights
     ]
 
-  outs, loss, loss_metrics, _, masks = _process_single_batch(
-      model, inputs, targets, sample_weights=sample_weights, training=True)
+  outs, total_loss, output_losses, aggregated_output_losses, masks = (
+      _process_single_batch(
+          model,
+          inputs,
+          targets,
+          sample_weights=sample_weights,
+          training=True,
+          output_loss_metrics=output_loss_metrics))
   if not isinstance(outs, list):
     outs = [outs]
   metrics_results = _eager_metrics_fn(
@@ -262,16 +301,23 @@ def train_on_batch(model, inputs, targets, sample_weights=None):
       targets,
       sample_weights=sample_weights,
       masks=masks,
-      return_stateful_result=True)
-  loss = nest.flatten(loss)
-
-  return [
-      tensor_util.constant_value(v)
-      for v in loss + loss_metrics + metrics_results
-  ]
-
-
-def test_on_batch(model, inputs, targets, sample_weights=None):
+      return_stateful_result=not reset_metrics)
+  total_loss = nest.flatten(total_loss)
+  if reset_metrics:
+    final_output_losses = output_losses
+  else:
+    final_output_losses = aggregated_output_losses
+  results = total_loss + final_output_losses + metrics_results
+
+  return [tensor_util.constant_value(v) for v in results]
+
+
+def test_on_batch(model,
+                  inputs,
+                  targets,
+                  sample_weights=None,
+                  reset_metrics=True,
+                  output_loss_metrics=None):
   """Calculates the loss for one input batch.
 
   Arguments:
@@ -279,6 +325,11 @@ def test_on_batch(model, inputs, targets, sample_weights=None):
       inputs: Input batch data.
       targets: Target batch data.
       sample_weights: Sample weight batch data.
+      reset_metrics: If `True`, the metrics returned will be only for this
+        batch. If `False`, the metrics will be statefully accumulated across
+        batches.
+      output_loss_metrics: List of metrics that are used to aggregated output
+        loss values.
 
   Returns:
       total loss, loss and metrics associated with each output.
@@ -288,19 +339,23 @@ def test_on_batch(model, inputs, targets, sample_weights=None):
       inputs = training_utils.cast_if_floating_dtype(inputs)
       targets = training_utils.cast_if_floating_dtype(targets)
     else:
-      inputs = training_utils.cast_if_floating_dtype([
-          ops.convert_to_tensor(val) for val in inputs
-      ])
-      targets = training_utils.cast_if_floating_dtype([
-          ops.convert_to_tensor(val) for val in targets
-      ])
+      inputs = training_utils.cast_if_floating_dtype(
+          [ops.convert_to_tensor(val) for val in inputs])
+      targets = training_utils.cast_if_floating_dtype(
+          [ops.convert_to_tensor(val) for val in targets])
   if sample_weights:
     sample_weights = [
         training_utils.cast_if_floating_dtype(ops.convert_to_tensor(val))
         if val is not None else None for val in sample_weights
     ]
-  outs, loss, loss_metrics, _, masks = _model_loss(
-      model, inputs, targets, sample_weights=sample_weights, training=False)
+  outs, total_loss, output_losses, aggregated_output_losses, masks = (
+      _model_loss(
+          model,
+          inputs,
+          targets,
+          sample_weights=sample_weights,
+          training=False,
+          output_loss_metrics=output_loss_metrics))
   if not isinstance(outs, list):
     outs = [outs]
   metrics_results = _eager_metrics_fn(
@@ -309,10 +364,12 @@ def test_on_batch(model, inputs, targets, sample_weights=None):
       targets,
       sample_weights=sample_weights,
       masks=masks,
-      return_stateful_result=True)
-  loss = nest.flatten(loss)
-
-  return [
-      tensor_util.constant_value(v)
-      for v in loss + loss_metrics + metrics_results
-  ]
+      return_stateful_result=not reset_metrics)
+  total_loss = nest.flatten(total_loss)
+  if reset_metrics:
+    final_output_losses = output_losses
+  else:
+    final_output_losses = aggregated_output_losses
+  results = total_loss + final_output_losses + metrics_results
+
+  return [tensor_util.constant_value(v) for v in results]
diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index f27c5f5583..f694384075 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-# pylint: disable=unused-import
 """Built-in loss functions.
 """
 from __future__ import absolute_import
@@ -23,7 +22,6 @@ import abc
 
 import six
 
-from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import smart_cond
 from tensorflow.python.keras import backend as K
@@ -32,7 +30,6 @@ from tensorflow.python.keras.utils.generic_utils import deserialize_keras_object
 from tensorflow.python.keras.utils.generic_utils import serialize_keras_object
 from tensorflow.python.keras.utils.tf_utils import is_tensor_or_variable
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
 from tensorflow.python.ops.losses import losses_impl
diff --git a/tensorflow/python/keras/metrics_correctness_test.py b/tensorflow/python/keras/metrics_correctness_test.py
index b2385aa611..6cb3b36183 100644
--- a/tensorflow/python/keras/metrics_correctness_test.py
+++ b/tensorflow/python/keras/metrics_correctness_test.py
@@ -30,7 +30,6 @@ from tensorflow.python.platform import test
 @keras_parameterized.run_with_all_model_types(exclude_models=['sequential'])
 @keras_parameterized.run_all_keras_modes
 class TestMetricsCorrectnessMultiIO(keras_parameterized.TestCase):
-  # TODO(psv): Remove the run_eagerly checks here when b/123082095 is fixed.
 
   def _get_multi_io_model(self):
     inp_1 = layers.Input(shape=(1,), name='input_1')
@@ -119,7 +118,9 @@ class TestMetricsCorrectnessMultiIO(keras_parameterized.TestCase):
         'output_2_mean_squared_error': [7.5, 7.5],
         'output_1_weighted_mean_squared_error': [9.286, 9.286],
         'output_2_weighted_mean_squared_error': [4.375, 4.375],
-        'loss': [41.25, 41.25]
+        'loss': [41.25, 41.25],
+        'output_1_loss': [32.5, 32.5],
+        'output_2_loss': [8.75, 8.75],
     }
 
     # In the order: 'loss', 'output_1_loss', 'output_2_loss',
@@ -137,11 +138,6 @@ class TestMetricsCorrectnessMultiIO(keras_parameterized.TestCase):
                         batch_size=2,
                         epochs=2,
                         shuffle=False)
-
-    if not model.run_eagerly:
-      self.expected_fit_result['output_1_loss'] = [32.5, 32.5]
-      self.expected_fit_result['output_2_loss'] = [8.75, 8.75]
-
     for key, value in self.expected_fit_result.items():
       self.assertAllClose(history.history[key], value, 1e-3)
 
@@ -153,13 +149,8 @@ class TestMetricsCorrectnessMultiIO(keras_parameterized.TestCase):
                                      'output_1': self.weights_1,
                                      'output_2': self.weights_2,
                                  })
-
-    if model.run_eagerly:
-      self.expected_batch_result = [41.25, 58, 10.75, 7.5, 9.286, 7.5, 4.375]
     self.assertAllClose(eval_result, self.expected_batch_result, 1e-3)
 
-    if model.run_eagerly:
-      return
     # Verify that metric value is same with arbitrary weights and batch size.
     x = np.random.random((50, 1))
     y = np.random.random((50, 1))
@@ -191,18 +182,12 @@ class TestMetricsCorrectnessMultiIO(keras_parameterized.TestCase):
     model = self._get_multi_io_model()
     history = model.fit_generator(
         self._custom_generator(), steps_per_epoch=2, epochs=2)
-
-    if not model.run_eagerly:
-      self.expected_fit_result['output_1_loss'] = [32.5, 32.5]
-      self.expected_fit_result['output_2_loss'] = [8.75, 8.75]
     for key, value in self.expected_fit_result.items():
       self.assertAllClose(history.history[key], value, 1e-3)
 
   def test_eval_generator(self):
     model = self._get_multi_io_model()
     eval_result = model.evaluate_generator(self._custom_generator(), steps=2)
-    if model.run_eagerly:
-      self.expected_batch_result = [41.25, 58, 10.75, 7.5, 9.286, 7.5, 4.375]
     self.assertAllClose(eval_result, self.expected_batch_result, 1e-3)
 
 
diff --git a/tensorflow/python/keras/models.py b/tensorflow/python/keras/models.py
index 9bc5aa2be5..d709782ddb 100644
--- a/tensorflow/python/keras/models.py
+++ b/tensorflow/python/keras/models.py
@@ -321,7 +321,8 @@ def _in_place_subclassed_model_reset(model):
     elif isinstance(
         value,
         (list, tuple)) and name not in ('layers', '_layers', 'metrics',
-                                        '_compile_stateful_metric_functions'):
+                                        '_compile_stateful_metric_functions',
+                                        '_output_loss_metrics'):
       # Handle case: list/tuple of layers (also tracked by the Network API).
       if value and all(isinstance(val, Layer) for val in value):
         raise ValueError('We do not support the use of list-of-layers '
diff --git a/tensorflow/python/keras/utils/losses_utils.py b/tensorflow/python/keras/utils/losses_utils.py
index 163d36102c..c25f4f780d 100644
--- a/tensorflow/python/keras/utils/losses_utils.py
+++ b/tensorflow/python/keras/utils/losses_utils.py
@@ -168,8 +168,8 @@ def _num_elements(losses):
     return math_ops.cast(array_ops.size(losses, name=scope), dtype=losses.dtype)
 
 
-def _reduce_weighted_loss(weighted_losses,
-                          reduction=ReductionV2.SUM_OVER_BATCH_SIZE):
+def reduce_weighted_loss(weighted_losses,
+                         reduction=ReductionV2.SUM_OVER_BATCH_SIZE):
   """Reduces the individual weighted loss measurements."""
   if reduction == ReductionV2.NONE:
     loss = weighted_losses
@@ -228,7 +228,7 @@ def compute_weighted_loss(losses,
     sample_weight.get_shape().assert_is_compatible_with(losses.get_shape())
     weighted_losses = math_ops.multiply(losses, sample_weight)
     # Apply reduction function to the individual weighted losses.
-    loss = _reduce_weighted_loss(weighted_losses, reduction)
+    loss = reduce_weighted_loss(weighted_losses, reduction)
     # Convert the result back to the input type.
     loss = math_ops.cast(loss, input_dtype)
     return loss
-- 
GitLab


From e7c810049fb4bd35787577054d83956d0fcddb60 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Fri, 8 Feb 2019 13:14:20 -0800
Subject: [PATCH 2181/2345] Update Eigen up to f5ef1f33d885 commit.

PiperOrigin-RevId: 233113684
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 65ee35f9c1..081ebb9738 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -141,11 +141,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
         name = "eigen_archive",
         build_file = clean_dep("//third_party:eigen.BUILD"),
         patch_file = clean_dep("//third_party/eigen3:gpu_packet_math.patch"),
-        sha256 = "d1d2ac19b8ef386ad70b91932a90bfbc3014b801d14723b9c8373239128bd2dd",
-        strip_prefix = "eigen-eigen-1724f8760da8",
+        sha256 = "c1f026562efcbca7c338bfa38751470d13dd3fd1f1925ca4fafe42603d0e7693",
+        strip_prefix = "eigen-eigen-f5ef1f33d885",
         urls = [
-            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/1724f8760da8.tar.gz",
-            "https://bitbucket.org/eigen/eigen/get/1724f8760da8.tar.gz",
+            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/f5ef1f33d885.tar.gz",
+            "https://bitbucket.org/eigen/eigen/get/f5ef1f33d885.tar.gz",
         ],
     )
 
-- 
GitLab


From 96837731fec65f64c6ede565d895e9b38b22d13b Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Fri, 8 Feb 2019 13:16:02 -0800
Subject: [PATCH 2182/2345] Add a test that includes global. It shows how
 global is allowed through, but mutations to the global variable are not
 applied.

PiperOrigin-RevId: 233113931
---
 tensorflow/python/autograph/impl/api_test.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/tensorflow/python/autograph/impl/api_test.py b/tensorflow/python/autograph/impl/api_test.py
index c89ac579c0..73e470a000 100644
--- a/tensorflow/python/autograph/impl/api_test.py
+++ b/tensorflow/python/autograph/impl/api_test.py
@@ -41,6 +41,9 @@ from tensorflow.python.util import tf_inspect
 tf = utils.fake_tf()
 
 
+testing_global_numeric = 2
+
+
 class TestResource(str):
   pass
 
@@ -496,6 +499,20 @@ class ApiTest(test.TestCase):
       x = compiled_fn(constant_op.constant([4, 8]))
       self.assertListEqual([1, 2], self.evaluate(x).tolist())
 
+  def test_to_graph_with_globals(self):
+
+    def test_fn(x):
+      global testing_global_numeric
+      testing_global_numeric = x + testing_global_numeric
+      return testing_global_numeric
+
+    compiled_fn = api.to_graph(test_fn)
+
+    x = compiled_fn(constant_op.constant(3))
+    self.assertEqual(5, self.evaluate(x))
+    # TODO(b/122368197): This should be the constant 5!
+    self.assertEqual(2, testing_global_numeric)
+
   def test_to_code_basic(self):
 
     def test_fn(x, s):
-- 
GitLab


From 7a8b28be83816d7151427b7caa2e676fbb0fef40 Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <38085909+penpornk@users.noreply.github.com>
Date: Fri, 8 Feb 2019 14:04:37 -0800
Subject: [PATCH 2183/2345] Update
 tensorflow/core/kernels/mkl_requantize_ops_test.cc

Co-Authored-By: nammbash <38869685+nammbash@users.noreply.github.com>
---
 tensorflow/core/kernels/mkl_requantize_ops_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/mkl_requantize_ops_test.cc b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
index 35b8a975b6..ff30dfcfb0 100644
--- a/tensorflow/core/kernels/mkl_requantize_ops_test.cc
+++ b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
@@ -189,7 +189,7 @@ TEST_F(MklRequantizatedOpsTest, RequantizationRangePerChannelTest_ClipMax) {
 
   // Step 5: Define and run requantization_range_perchannel
   // -------------------------------------------------------
-  // Define, create and initalize the op in question.
+  // Define, create and initialize the op in question.
   TF_ASSERT_OK(NodeDefBuilder("requantization_range_per_channel",
                               "RequantizationRangePerChannel")
                    .Input(FakeInput(DT_QINT32))
-- 
GitLab


From 1f1b7996a179c7464d1b3f837d829d4e4c2e7705 Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <38085909+penpornk@users.noreply.github.com>
Date: Fri, 8 Feb 2019 14:04:53 -0800
Subject: [PATCH 2184/2345] Update
 tensorflow/core/kernels/mkl_requantize_ops_test.cc

Co-Authored-By: nammbash <38869685+nammbash@users.noreply.github.com>
---
 tensorflow/core/kernels/mkl_requantize_ops_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/mkl_requantize_ops_test.cc b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
index ff30dfcfb0..9961462754 100644
--- a/tensorflow/core/kernels/mkl_requantize_ops_test.cc
+++ b/tensorflow/core/kernels/mkl_requantize_ops_test.cc
@@ -38,7 +38,7 @@ class MklRequantizatedOpsTestHelper : public OpsTestBase {
  public:
   void Setup(Tensor &input_tensor_qint32, float &range_weights_ch1,
              float &range_weights_ch2);
-  void TestBody(){};
+  void TestBody() {}
 };
 
 void MklRequantizatedOpsTestHelper::Setup(Tensor &input_tensor_qint32,
-- 
GitLab


From 65a1094924bf2698fbca5d3b72f23e09ec5433ca Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Fri, 8 Feb 2019 13:16:36 -0800
Subject: [PATCH 2185/2345] Add module aliases for losses and metrics in the
 api template files.  - tf.losses = tf.keras.losses  - tf.metrics =
 tf.keras.metrics

PiperOrigin-RevId: 233114056
---
 tensorflow/api_template.__init__.py           |   4 +
 tensorflow/compat_template.__init__.py        |   7 +
 ...nsorflow.losses.-binary-crossentropy.pbtxt |  23 ++
 ...low.losses.-categorical-crossentropy.pbtxt |  23 ++
 ...tensorflow.losses.-categorical-hinge.pbtxt |  23 ++
 .../tensorflow.losses.-cosine-proximity.pbtxt |  22 ++
 .../golden/v2/tensorflow.losses.-hinge.pbtxt  |  23 ++
 .../golden/v2/tensorflow.losses.-huber.pbtxt  |  23 ++
 .../tensorflow.losses.-k-l-divergence.pbtxt   |  23 ++
 .../v2/tensorflow.losses.-log-cosh.pbtxt      |  23 ++
 .../v2/tensorflow.losses.-log-loss.pbtxt      |  23 ++
 .../golden/v2/tensorflow.losses.-loss.pbtxt   |  21 ++
 ...nsorflow.losses.-mean-absolute-error.pbtxt |  23 ++
 ...sses.-mean-absolute-percentage-error.pbtxt |  23 ++
 ...ensorflow.losses.-mean-squared-error.pbtxt |  23 ++
 ...sses.-mean-squared-logarithmic-error.pbtxt |  23 ++
 .../v2/tensorflow.losses.-poisson.pbtxt       |  23 ++
 .../v2/tensorflow.losses.-reduction.pbtxt     |  28 ++
 ...ses.-sparse-categorical-crossentropy.pbtxt |  23 ++
 .../v2/tensorflow.losses.-squared-hinge.pbtxt |  23 ++
 .../api/golden/v2/tensorflow.losses.pbtxt     | 187 ++++++++++++
 .../golden/v2/tensorflow.metrics.-a-u-c.pbtxt | 200 +++++++++++++
 .../v2/tensorflow.metrics.-accuracy.pbtxt     | 199 +++++++++++++
 .../tensorflow.metrics.-binary-accuracy.pbtxt | 199 +++++++++++++
 ...sorflow.metrics.-binary-crossentropy.pbtxt | 199 +++++++++++++
 ...orflow.metrics.-categorical-accuracy.pbtxt | 199 +++++++++++++
 ...ow.metrics.-categorical-crossentropy.pbtxt | 199 +++++++++++++
 ...ensorflow.metrics.-categorical-hinge.pbtxt | 199 +++++++++++++
 ...tensorflow.metrics.-cosine-proximity.pbtxt | 199 +++++++++++++
 .../tensorflow.metrics.-false-negatives.pbtxt | 197 +++++++++++++
 .../tensorflow.metrics.-false-positives.pbtxt | 197 +++++++++++++
 .../golden/v2/tensorflow.metrics.-hinge.pbtxt | 199 +++++++++++++
 .../tensorflow.metrics.-k-l-divergence.pbtxt  | 199 +++++++++++++
 .../tensorflow.metrics.-log-cosh-error.pbtxt  | 199 +++++++++++++
 ...sorflow.metrics.-mean-absolute-error.pbtxt | 199 +++++++++++++
 ...rics.-mean-absolute-percentage-error.pbtxt | 199 +++++++++++++
 .../v2/tensorflow.metrics.-mean-io-u.pbtxt    | 196 +++++++++++++
 ...sorflow.metrics.-mean-relative-error.pbtxt | 198 +++++++++++++
 ...nsorflow.metrics.-mean-squared-error.pbtxt | 199 +++++++++++++
 ...rics.-mean-squared-logarithmic-error.pbtxt | 199 +++++++++++++
 .../v2/tensorflow.metrics.-mean-tensor.pbtxt  | 204 +++++++++++++
 .../golden/v2/tensorflow.metrics.-mean.pbtxt  | 197 +++++++++++++
 .../v2/tensorflow.metrics.-metric.pbtxt       | 195 +++++++++++++
 .../v2/tensorflow.metrics.-poisson.pbtxt      | 199 +++++++++++++
 .../v2/tensorflow.metrics.-precision.pbtxt    | 196 +++++++++++++
 .../v2/tensorflow.metrics.-recall.pbtxt       | 196 +++++++++++++
 ...low.metrics.-root-mean-squared-error.pbtxt | 198 +++++++++++++
 ....metrics.-sensitivity-at-specificity.pbtxt | 197 +++++++++++++
 ...metrics.-sparse-categorical-accuracy.pbtxt | 199 +++++++++++++
 ...ics.-sparse-categorical-crossentropy.pbtxt | 199 +++++++++++++
 ...s.-sparse-top-k-categorical-accuracy.pbtxt | 199 +++++++++++++
 ....metrics.-specificity-at-sensitivity.pbtxt | 197 +++++++++++++
 .../tensorflow.metrics.-squared-hinge.pbtxt   | 199 +++++++++++++
 .../golden/v2/tensorflow.metrics.-sum.pbtxt   | 197 +++++++++++++
 ....metrics.-top-k-categorical-accuracy.pbtxt | 199 +++++++++++++
 .../tensorflow.metrics.-true-negatives.pbtxt  | 197 +++++++++++++
 .../tensorflow.metrics.-true-positives.pbtxt  | 197 +++++++++++++
 .../api/golden/v2/tensorflow.metrics.pbtxt    | 271 ++++++++++++++++++
 .../tools/api/golden/v2/tensorflow.pbtxt      |   8 +
 59 files changed, 8032 insertions(+)
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.losses.-binary-crossentropy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.losses.-categorical-crossentropy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.losses.-categorical-hinge.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.losses.-cosine-proximity.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.losses.-hinge.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.losses.-huber.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.losses.-k-l-divergence.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.losses.-log-cosh.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.losses.-log-loss.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.losses.-loss.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.losses.-mean-absolute-error.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.losses.-mean-absolute-percentage-error.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.losses.-mean-squared-error.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.losses.-mean-squared-logarithmic-error.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.losses.-poisson.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.losses.-reduction.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.losses.-sparse-categorical-crossentropy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.losses.-squared-hinge.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.losses.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-a-u-c.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-accuracy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-binary-accuracy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-binary-crossentropy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-categorical-accuracy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-categorical-crossentropy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-categorical-hinge.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-cosine-proximity.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-negatives.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-positives.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-hinge.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-k-l-divergence.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-log-cosh-error.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean-absolute-error.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean-absolute-percentage-error.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean-io-u.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean-relative-error.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean-squared-error.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean-squared-logarithmic-error.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean-tensor.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-metric.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-poisson.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-precision.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-recall.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-root-mean-squared-error.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-sensitivity-at-specificity.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-sparse-categorical-accuracy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-sparse-categorical-crossentropy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-sparse-top-k-categorical-accuracy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-specificity-at-sensitivity.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-squared-hinge.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-sum.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-top-k-categorical-accuracy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-negatives.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-positives.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.pbtxt

diff --git a/tensorflow/api_template.__init__.py b/tensorflow/api_template.__init__.py
index a93799bfe8..8a101e282e 100644
--- a/tensorflow/api_template.__init__.py
+++ b/tensorflow/api_template.__init__.py
@@ -111,5 +111,9 @@ try:
 except NameError:
   pass
 
+# Add module aliases
+if hasattr(_current_module, 'keras'):
+  losses = keras.losses
+  metrics = keras.metrics
 
 # pylint: enable=undefined-variable
diff --git a/tensorflow/compat_template.__init__.py b/tensorflow/compat_template.__init__.py
index 1833b6a65e..953c70ab1e 100644
--- a/tensorflow/compat_template.__init__.py
+++ b/tensorflow/compat_template.__init__.py
@@ -19,6 +19,7 @@ from __future__ import division as _division
 from __future__ import print_function as _print_function
 
 import os as _os
+import sys as _sys
 
 # pylint: disable=g-bad-import-order
 from tensorflow.python import pywrap_tensorflow  # pylint: disable=unused-import
@@ -41,3 +42,9 @@ _component_api_helper.package_hook(
 #
 # This make this one symbol available directly.
 from tensorflow.python.compat.v2_compat import enable_v2_behavior  # pylint: disable=g-import-not-at-top
+
+# Add module aliases
+_current_module = _sys.modules[__name__]
+if hasattr(_current_module, 'keras'):
+  losses = keras.losses
+  metrics = keras.metrics
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.-binary-crossentropy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.losses.-binary-crossentropy.pbtxt
new file mode 100644
index 0000000000..1d180a9fba
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.losses.-binary-crossentropy.pbtxt
@@ -0,0 +1,23 @@
+path: "tensorflow.losses.BinaryCrossentropy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.BinaryCrossentropy\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'from_logits\', \'label_smoothing\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'0\', \'sum_over_batch_size\', \'binary_crossentropy\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.-categorical-crossentropy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.losses.-categorical-crossentropy.pbtxt
new file mode 100644
index 0000000000..3937dfa153
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.losses.-categorical-crossentropy.pbtxt
@@ -0,0 +1,23 @@
+path: "tensorflow.losses.CategoricalCrossentropy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.CategoricalCrossentropy\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'from_logits\', \'label_smoothing\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'0\', \'sum_over_batch_size\', \'categorical_crossentropy\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.-categorical-hinge.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.losses.-categorical-hinge.pbtxt
new file mode 100644
index 0000000000..d2a064dd64
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.losses.-categorical-hinge.pbtxt
@@ -0,0 +1,23 @@
+path: "tensorflow.losses.CategoricalHinge"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.CategoricalHinge\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'categorical_hinge\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.-cosine-proximity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.losses.-cosine-proximity.pbtxt
new file mode 100644
index 0000000000..3ec290213d
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.losses.-cosine-proximity.pbtxt
@@ -0,0 +1,22 @@
+path: "tensorflow.losses.CosineProximity"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.CosineProximity\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'axis\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.-hinge.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.losses.-hinge.pbtxt
new file mode 100644
index 0000000000..155154c312
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.losses.-hinge.pbtxt
@@ -0,0 +1,23 @@
+path: "tensorflow.losses.Hinge"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.Hinge\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.-huber.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.losses.-huber.pbtxt
new file mode 100644
index 0000000000..5052c19a04
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.losses.-huber.pbtxt
@@ -0,0 +1,23 @@
+path: "tensorflow.losses.Huber"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.Huber\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'delta\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'1.0\', \'sum_over_batch_size\', \'huber_loss\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.-k-l-divergence.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.losses.-k-l-divergence.pbtxt
new file mode 100644
index 0000000000..b739c057b2
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.losses.-k-l-divergence.pbtxt
@@ -0,0 +1,23 @@
+path: "tensorflow.losses.KLDivergence"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.KLDivergence\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'kullback_leibler_divergence\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.-log-cosh.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.losses.-log-cosh.pbtxt
new file mode 100644
index 0000000000..557cc21045
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.losses.-log-cosh.pbtxt
@@ -0,0 +1,23 @@
+path: "tensorflow.losses.LogCosh"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.LogCosh\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'logcosh\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.-log-loss.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.losses.-log-loss.pbtxt
new file mode 100644
index 0000000000..65b1feaefd
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.losses.-log-loss.pbtxt
@@ -0,0 +1,23 @@
+path: "tensorflow.losses.LogLoss"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.LogLoss\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'logloss\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.-loss.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.losses.-loss.pbtxt
new file mode 100644
index 0000000000..2bcc6f8854
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.losses.-loss.pbtxt
@@ -0,0 +1,21 @@
+path: "tensorflow.losses.Loss"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.-mean-absolute-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.losses.-mean-absolute-error.pbtxt
new file mode 100644
index 0000000000..a33db29d42
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.losses.-mean-absolute-error.pbtxt
@@ -0,0 +1,23 @@
+path: "tensorflow.losses.MeanAbsoluteError"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.MeanAbsoluteError\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'mean_absolute_error\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.-mean-absolute-percentage-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.losses.-mean-absolute-percentage-error.pbtxt
new file mode 100644
index 0000000000..4c79a5e8b8
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.losses.-mean-absolute-percentage-error.pbtxt
@@ -0,0 +1,23 @@
+path: "tensorflow.losses.MeanAbsolutePercentageError"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.MeanAbsolutePercentageError\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'mean_absolute_percentage_error\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.-mean-squared-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.losses.-mean-squared-error.pbtxt
new file mode 100644
index 0000000000..b99e19413e
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.losses.-mean-squared-error.pbtxt
@@ -0,0 +1,23 @@
+path: "tensorflow.losses.MeanSquaredError"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.MeanSquaredError\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'mean_squared_error\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.-mean-squared-logarithmic-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.losses.-mean-squared-logarithmic-error.pbtxt
new file mode 100644
index 0000000000..e01827391a
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.losses.-mean-squared-logarithmic-error.pbtxt
@@ -0,0 +1,23 @@
+path: "tensorflow.losses.MeanSquaredLogarithmicError"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.MeanSquaredLogarithmicError\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'mean_squared_logarithmic_error\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.-poisson.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.losses.-poisson.pbtxt
new file mode 100644
index 0000000000..b6603cb407
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.losses.-poisson.pbtxt
@@ -0,0 +1,23 @@
+path: "tensorflow.losses.Poisson"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.Poisson\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'poisson\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.-reduction.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.losses.-reduction.pbtxt
new file mode 100644
index 0000000000..e4ae87ea29
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.losses.-reduction.pbtxt
@@ -0,0 +1,28 @@
+path: "tensorflow.losses.Reduction"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.utils.losses_utils.ReductionV2\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "NONE"
+    mtype: "<type \'str\'>"
+  }
+  member {
+    name: "SUM"
+    mtype: "<type \'str\'>"
+  }
+  member {
+    name: "SUM_OVER_BATCH_SIZE"
+    mtype: "<type \'str\'>"
+  }
+  member_method {
+    name: "__init__"
+  }
+  member_method {
+    name: "all"
+    argspec: "args=[\'cls\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "validate"
+    argspec: "args=[\'cls\', \'key\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.-sparse-categorical-crossentropy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.losses.-sparse-categorical-crossentropy.pbtxt
new file mode 100644
index 0000000000..5e3ce6fc72
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.losses.-sparse-categorical-crossentropy.pbtxt
@@ -0,0 +1,23 @@
+path: "tensorflow.losses.SparseCategoricalCrossentropy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.SparseCategoricalCrossentropy\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'from_logits\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.-squared-hinge.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.losses.-squared-hinge.pbtxt
new file mode 100644
index 0000000000..b5e3757143
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.losses.-squared-hinge.pbtxt
@@ -0,0 +1,23 @@
+path: "tensorflow.losses.SquaredHinge"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.SquaredHinge\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.LossFunctionWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'squared_hinge\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.losses.pbtxt
new file mode 100644
index 0000000000..d906734542
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.losses.pbtxt
@@ -0,0 +1,187 @@
+path: "tensorflow.losses"
+tf_module {
+  member {
+    name: "BinaryCrossentropy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "CategoricalCrossentropy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "CategoricalHinge"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "CosineProximity"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Hinge"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Huber"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "KLDivergence"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "LogCosh"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "LogLoss"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Loss"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "MeanAbsoluteError"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "MeanAbsolutePercentageError"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "MeanSquaredError"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "MeanSquaredLogarithmicError"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Poisson"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Reduction"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "SparseCategoricalCrossentropy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "SquaredHinge"
+    mtype: "<type \'type\'>"
+  }
+  member_method {
+    name: "KLD"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MAE"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MAPE"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MSE"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MSLE"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "binary_crossentropy"
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\', \'label_smoothing\'], varargs=None, keywords=None, defaults=[\'False\', \'0\'], "
+  }
+  member_method {
+    name: "categorical_crossentropy"
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\', \'label_smoothing\'], varargs=None, keywords=None, defaults=[\'False\', \'0\'], "
+  }
+  member_method {
+    name: "categorical_hinge"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "cosine"
+    argspec: "args=[\'y_true\', \'y_pred\', \'axis\'], varargs=None, keywords=None, defaults=[\'-1\'], "
+  }
+  member_method {
+    name: "cosine_proximity"
+    argspec: "args=[\'y_true\', \'y_pred\', \'axis\'], varargs=None, keywords=None, defaults=[\'-1\'], "
+  }
+  member_method {
+    name: "deserialize"
+    argspec: "args=[\'name\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "get"
+    argspec: "args=[\'identifier\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "hinge"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "kld"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "kullback_leibler_divergence"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "logcosh"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "mae"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "mape"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "mean_absolute_error"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "mean_absolute_percentage_error"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "mean_squared_error"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "mean_squared_logarithmic_error"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "mse"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "msle"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "poisson"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "serialize"
+    argspec: "args=[\'loss\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "sparse_categorical_crossentropy"
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\', \'axis\'], varargs=None, keywords=None, defaults=[\'False\', \'-1\'], "
+  }
+  member_method {
+    name: "squared_hinge"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-a-u-c.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-a-u-c.pbtxt
new file mode 100644
index 0000000000..c70a33d5f2
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-a-u-c.pbtxt
@@ -0,0 +1,200 @@
+path: "tensorflow.metrics.AUC"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.AUC\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'num_thresholds\', \'curve\', \'summation_method\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'200\', \'ROC\', \'interpolation\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "interpolate_pr_auc"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-accuracy.pbtxt
new file mode 100644
index 0000000000..fb8c9568bf
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-accuracy.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.metrics.Accuracy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Accuracy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'accuracy\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-binary-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-binary-accuracy.pbtxt
new file mode 100644
index 0000000000..08f905221a
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-binary-accuracy.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.metrics.BinaryAccuracy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.BinaryAccuracy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\', \'threshold\'], varargs=None, keywords=None, defaults=[\'binary_accuracy\', \'None\', \'0.5\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-binary-crossentropy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-binary-crossentropy.pbtxt
new file mode 100644
index 0000000000..6420766a7f
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-binary-crossentropy.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.metrics.BinaryCrossentropy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.BinaryCrossentropy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\', \'from_logits\', \'label_smoothing\'], varargs=None, keywords=None, defaults=[\'binary_crossentropy\', \'None\', \'False\', \'0\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-categorical-accuracy.pbtxt
new file mode 100644
index 0000000000..a1e1a2ad83
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-categorical-accuracy.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.metrics.CategoricalAccuracy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.CategoricalAccuracy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'categorical_accuracy\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-categorical-crossentropy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-categorical-crossentropy.pbtxt
new file mode 100644
index 0000000000..765cc70dc0
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-categorical-crossentropy.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.metrics.CategoricalCrossentropy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.CategoricalCrossentropy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\', \'from_logits\', \'label_smoothing\'], varargs=None, keywords=None, defaults=[\'categorical_crossentropy\', \'None\', \'False\', \'0\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-categorical-hinge.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-categorical-hinge.pbtxt
new file mode 100644
index 0000000000..7f5dd75b70
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-categorical-hinge.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.metrics.CategoricalHinge"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.CategoricalHinge\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'categorical_hinge\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-cosine-proximity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-cosine-proximity.pbtxt
new file mode 100644
index 0000000000..2424ebc822
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-cosine-proximity.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.metrics.CosineProximity"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.CosineProximity\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\', \'axis\'], varargs=None, keywords=None, defaults=[\'cosine_proximity\', \'None\', \'-1\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-negatives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-negatives.pbtxt
new file mode 100644
index 0000000000..1f45f2d9bb
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-negatives.pbtxt
@@ -0,0 +1,197 @@
+path: "tensorflow.metrics.FalseNegatives"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.FalseNegatives\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-positives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-positives.pbtxt
new file mode 100644
index 0000000000..f9ceff6478
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-positives.pbtxt
@@ -0,0 +1,197 @@
+path: "tensorflow.metrics.FalsePositives"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.FalsePositives\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-hinge.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-hinge.pbtxt
new file mode 100644
index 0000000000..0b278e6865
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-hinge.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.metrics.Hinge"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Hinge\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'hinge\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-k-l-divergence.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-k-l-divergence.pbtxt
new file mode 100644
index 0000000000..a79ff5ef10
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-k-l-divergence.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.metrics.KLDivergence"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.KLDivergence\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'kullback_leibler_divergence\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-log-cosh-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-log-cosh-error.pbtxt
new file mode 100644
index 0000000000..ee78badd97
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-log-cosh-error.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.metrics.LogCoshError"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.LogCoshError\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'logcosh\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean-absolute-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean-absolute-error.pbtxt
new file mode 100644
index 0000000000..d7edbea85b
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean-absolute-error.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.metrics.MeanAbsoluteError"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanAbsoluteError\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'mean_absolute_error\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean-absolute-percentage-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean-absolute-percentage-error.pbtxt
new file mode 100644
index 0000000000..f6becda1fd
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean-absolute-percentage-error.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.metrics.MeanAbsolutePercentageError"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanAbsolutePercentageError\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'mean_absolute_percentage_error\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean-io-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean-io-u.pbtxt
new file mode 100644
index 0000000000..1e61cb759d
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean-io-u.pbtxt
@@ -0,0 +1,196 @@
+path: "tensorflow.metrics.MeanIoU"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanIoU\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'num_classes\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean-relative-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean-relative-error.pbtxt
new file mode 100644
index 0000000000..99cf4910e3
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean-relative-error.pbtxt
@@ -0,0 +1,198 @@
+path: "tensorflow.metrics.MeanRelativeError"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanRelativeError\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'normalizer\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean-squared-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean-squared-error.pbtxt
new file mode 100644
index 0000000000..a8f18e99c8
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean-squared-error.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.metrics.MeanSquaredError"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanSquaredError\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'mean_squared_error\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean-squared-logarithmic-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean-squared-logarithmic-error.pbtxt
new file mode 100644
index 0000000000..ba2a31a86e
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean-squared-logarithmic-error.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.metrics.MeanSquaredLogarithmicError"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanSquaredLogarithmicError\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'mean_squared_logarithmic_error\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean-tensor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean-tensor.pbtxt
new file mode 100644
index 0000000000..55dc84d2e3
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean-tensor.pbtxt
@@ -0,0 +1,204 @@
+path: "tensorflow.metrics.MeanTensor"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanTensor\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "count"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "total"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'mean_tensor\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'values\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean.pbtxt
new file mode 100644
index 0000000000..92aca4c791
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean.pbtxt
@@ -0,0 +1,197 @@
+path: "tensorflow.metrics.Mean"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'mean\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'values\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-metric.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-metric.pbtxt
new file mode 100644
index 0000000000..43a9699546
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-metric.pbtxt
@@ -0,0 +1,195 @@
+path: "tensorflow.metrics.Metric"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-poisson.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-poisson.pbtxt
new file mode 100644
index 0000000000..fe5c394a24
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-poisson.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.metrics.Poisson"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Poisson\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'poisson\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-precision.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-precision.pbtxt
new file mode 100644
index 0000000000..ea7ca6c0a4
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-precision.pbtxt
@@ -0,0 +1,196 @@
+path: "tensorflow.metrics.Precision"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Precision\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'top_k\', \'class_id\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-recall.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-recall.pbtxt
new file mode 100644
index 0000000000..bea427b3d4
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-recall.pbtxt
@@ -0,0 +1,196 @@
+path: "tensorflow.metrics.Recall"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Recall\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'top_k\', \'class_id\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-root-mean-squared-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-root-mean-squared-error.pbtxt
new file mode 100644
index 0000000000..37a6281857
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-root-mean-squared-error.pbtxt
@@ -0,0 +1,198 @@
+path: "tensorflow.metrics.RootMeanSquaredError"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.RootMeanSquaredError\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'root_mean_squared_error\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-sensitivity-at-specificity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-sensitivity-at-specificity.pbtxt
new file mode 100644
index 0000000000..b5f31469b0
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-sensitivity-at-specificity.pbtxt
@@ -0,0 +1,197 @@
+path: "tensorflow.metrics.SensitivityAtSpecificity"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.SensitivityAtSpecificity\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.SensitivitySpecificityBase\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'specificity\', \'num_thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'200\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-sparse-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-sparse-categorical-accuracy.pbtxt
new file mode 100644
index 0000000000..4578c69189
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-sparse-categorical-accuracy.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.metrics.SparseCategoricalAccuracy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.SparseCategoricalAccuracy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'sparse_categorical_accuracy\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-sparse-categorical-crossentropy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-sparse-categorical-crossentropy.pbtxt
new file mode 100644
index 0000000000..de8fdd9712
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-sparse-categorical-crossentropy.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.metrics.SparseCategoricalCrossentropy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.SparseCategoricalCrossentropy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\', \'from_logits\', \'axis\'], varargs=None, keywords=None, defaults=[\'sparse_categorical_crossentropy\', \'None\', \'False\', \'-1\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-sparse-top-k-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-sparse-top-k-categorical-accuracy.pbtxt
new file mode 100644
index 0000000000..071a34ca12
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-sparse-top-k-categorical-accuracy.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.metrics.SparseTopKCategoricalAccuracy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.SparseTopKCategoricalAccuracy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'k\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'5\', \'sparse_top_k_categorical_accuracy\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-specificity-at-sensitivity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-specificity-at-sensitivity.pbtxt
new file mode 100644
index 0000000000..cdb85b1011
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-specificity-at-sensitivity.pbtxt
@@ -0,0 +1,197 @@
+path: "tensorflow.metrics.SpecificityAtSensitivity"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.SpecificityAtSensitivity\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.SensitivitySpecificityBase\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'sensitivity\', \'num_thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'200\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-squared-hinge.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-squared-hinge.pbtxt
new file mode 100644
index 0000000000..da081b855d
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-squared-hinge.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.metrics.SquaredHinge"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.SquaredHinge\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'squared_hinge\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-sum.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-sum.pbtxt
new file mode 100644
index 0000000000..81ebb07717
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-sum.pbtxt
@@ -0,0 +1,197 @@
+path: "tensorflow.metrics.Sum"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Sum\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'sum\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'values\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-top-k-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-top-k-categorical-accuracy.pbtxt
new file mode 100644
index 0000000000..234a79f89b
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-top-k-categorical-accuracy.pbtxt
@@ -0,0 +1,199 @@
+path: "tensorflow.metrics.TopKCategoricalAccuracy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.TopKCategoricalAccuracy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Reduce\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'k\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'5\', \'top_k_categorical_accuracy\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-negatives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-negatives.pbtxt
new file mode 100644
index 0000000000..2285368287
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-negatives.pbtxt
@@ -0,0 +1,197 @@
+path: "tensorflow.metrics.TrueNegatives"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.TrueNegatives\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-positives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-positives.pbtxt
new file mode 100644
index 0000000000..85dd4edd01
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-positives.pbtxt
@@ -0,0 +1,197 @@
+path: "tensorflow.metrics.TruePositives"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.TruePositives\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.pbtxt
new file mode 100644
index 0000000000..639186a979
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.pbtxt
@@ -0,0 +1,271 @@
+path: "tensorflow.metrics"
+tf_module {
+  member {
+    name: "AUC"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Accuracy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "BinaryAccuracy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "BinaryCrossentropy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "CategoricalAccuracy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "CategoricalCrossentropy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "CategoricalHinge"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "CosineProximity"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "FalseNegatives"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "FalsePositives"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Hinge"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "KLDivergence"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "LogCoshError"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Mean"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "MeanAbsoluteError"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "MeanAbsolutePercentageError"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "MeanIoU"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "MeanRelativeError"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "MeanSquaredError"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "MeanSquaredLogarithmicError"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "MeanTensor"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Metric"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Poisson"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Precision"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Recall"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "RootMeanSquaredError"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "SensitivityAtSpecificity"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "SparseCategoricalAccuracy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "SparseCategoricalCrossentropy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "SparseTopKCategoricalAccuracy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "SpecificityAtSensitivity"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "SquaredHinge"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Sum"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "TopKCategoricalAccuracy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "TrueNegatives"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "TruePositives"
+    mtype: "<type \'type\'>"
+  }
+  member_method {
+    name: "KLD"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MAE"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MAPE"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MSE"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "MSLE"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "binary_accuracy"
+    argspec: "args=[\'y_true\', \'y_pred\', \'threshold\'], varargs=None, keywords=None, defaults=[\'0.5\'], "
+  }
+  member_method {
+    name: "binary_crossentropy"
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\', \'label_smoothing\'], varargs=None, keywords=None, defaults=[\'False\', \'0\'], "
+  }
+  member_method {
+    name: "categorical_accuracy"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "categorical_crossentropy"
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\', \'label_smoothing\'], varargs=None, keywords=None, defaults=[\'False\', \'0\'], "
+  }
+  member_method {
+    name: "cosine"
+    argspec: "args=[\'y_true\', \'y_pred\', \'axis\'], varargs=None, keywords=None, defaults=[\'-1\'], "
+  }
+  member_method {
+    name: "cosine_proximity"
+    argspec: "args=[\'y_true\', \'y_pred\', \'axis\'], varargs=None, keywords=None, defaults=[\'-1\'], "
+  }
+  member_method {
+    name: "deserialize"
+    argspec: "args=[\'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "get"
+    argspec: "args=[\'identifier\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "hinge"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "kld"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "kullback_leibler_divergence"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "mae"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "mape"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "mean_absolute_error"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "mean_absolute_percentage_error"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "mean_squared_error"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "mean_squared_logarithmic_error"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "mse"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "msle"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "poisson"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "serialize"
+    argspec: "args=[\'metric\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "sparse_categorical_accuracy"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "sparse_categorical_crossentropy"
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\', \'axis\'], varargs=None, keywords=None, defaults=[\'False\', \'-1\'], "
+  }
+  member_method {
+    name: "sparse_top_k_categorical_accuracy"
+    argspec: "args=[\'y_true\', \'y_pred\', \'k\'], varargs=None, keywords=None, defaults=[\'5\'], "
+  }
+  member_method {
+    name: "squared_hinge"
+    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "top_k_categorical_accuracy"
+    argspec: "args=[\'y_true\', \'y_pred\', \'k\'], varargs=None, keywords=None, defaults=[\'5\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 1ebbf602c9..f28a3a5308 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -200,10 +200,18 @@ tf_module {
     name: "lite"
     mtype: "<type \'module\'>"
   }
+  member {
+    name: "losses"
+    mtype: "<type \'module\'>"
+  }
   member {
     name: "math"
     mtype: "<type \'module\'>"
   }
+  member {
+    name: "metrics"
+    mtype: "<type \'module\'>"
+  }
   member {
     name: "name_scope"
     mtype: "<type \'type\'>"
-- 
GitLab


From b8a19fabb65d571f6a798f2f1bf05bfff93c7146 Mon Sep 17 00:00:00 2001
From: nammbash <niroopshankar.ammbashankar@intel.com>
Date: Fri, 8 Feb 2019 14:11:10 -0800
Subject: [PATCH 2186/2345] remove balnk line

---
 tensorflow/core/kernels/mkl_requantize_per_channel_op.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc b/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc
index 63701126b0..abd05e0ed5 100644
--- a/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc
+++ b/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc
@@ -140,7 +140,6 @@ class MklRequantizePerChannelOp : public OpKernel {
 
       output_min->flat<float>()(0) = input_requested_min_float;
       output_max->flat<float>()(0) = input_requested_max_float;
-
     } catch (mkldnn::error& e) {
       string error_msg = "Status: " + std::to_string(e.status) + ", message: " +
                          std::string(e.message) + ", in file " +
-- 
GitLab


From 4c84dec7464258fde904a7555a325d1d8d94b747 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Fri, 8 Feb 2019 13:22:57 -0800
Subject: [PATCH 2187/2345] [tf.data] Adding an option to
 `tf.data.experimental.make_csv_dataset` to skip over invalid records.

PiperOrigin-RevId: 233115089
---
 .../python/data/experimental/ops/readers.py      | 16 +++++++++++++---
 .../golden/v1/tensorflow.data.experimental.pbtxt |  2 +-
 .../golden/v2/tensorflow.data.experimental.pbtxt |  2 +-
 3 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/data/experimental/ops/readers.py b/tensorflow/python/data/experimental/ops/readers.py
index 177886e64b..24a399ab4e 100644
--- a/tensorflow/python/data/experimental/ops/readers.py
+++ b/tensorflow/python/data/experimental/ops/readers.py
@@ -24,6 +24,7 @@ import functools
 import numpy as np
 
 from tensorflow.python.data.experimental.ops import batching
+from tensorflow.python.data.experimental.ops import error_ops
 from tensorflow.python.data.experimental.ops import interleave_ops
 from tensorflow.python.data.experimental.ops import optimization
 from tensorflow.python.data.experimental.ops import parsing_ops
@@ -328,6 +329,7 @@ def make_csv_dataset_v2(
     sloppy=False,
     num_rows_for_inference=100,
     compression_type=None,
+    ignore_errors=False,
 ):
   """Reads CSV files into a dataset.
 
@@ -402,6 +404,10 @@ def make_csv_dataset_v2(
       the files. Defaults to 100.
     compression_type: (Optional.) A `tf.string` scalar evaluating to one of
       `""` (no compression), `"ZLIB"`, or `"GZIP"`. Defaults to no compression.
+    ignore_errors: (Optional.) If `True`, ignores errors with CSV file parsing,
+      such as malformed data or empty lines, and moves on to the next valid
+      CSV record. Otherwise, the dataset raises an error and stops processing
+      when encountering any invalid records. Defaults to `False`.
 
   Returns:
     A dataset, where each element is a (features, labels) tuple that corresponds
@@ -457,7 +463,7 @@ def make_csv_dataset_v2(
     raise ValueError("`label_name` provided must be one of the columns.")
 
   def filename_to_dataset(filename):
-    return CsvDataset(
+    dataset = CsvDataset(
         filename,
         record_defaults=column_defaults,
         field_delim=field_delim,
@@ -465,8 +471,11 @@ def make_csv_dataset_v2(
         na_value=na_value,
         select_cols=select_columns,
         header=header,
-        compression_type=compression_type,
+        compression_type=compression_type
     )
+    if ignore_errors:
+      dataset = dataset.apply(error_ops.ignore_errors())
+    return dataset
 
   def map_fn(*columns):
     """Organizes columns into a features dictionary.
@@ -528,13 +537,14 @@ def make_csv_dataset_v1(
     sloppy=False,
     num_rows_for_inference=100,
     compression_type=None,
+    ignore_errors=False,
 ):  # pylint: disable=missing-docstring
   return dataset_ops.DatasetV1Adapter(make_csv_dataset_v2(
       file_pattern, batch_size, column_names, column_defaults, label_name,
       select_columns, field_delim, use_quote_delim, na_value, header,
       num_epochs, shuffle, shuffle_buffer_size, shuffle_seed,
       prefetch_buffer_size, num_parallel_reads, sloppy, num_rows_for_inference,
-      compression_type))
+      compression_type, ignore_errors))
 make_csv_dataset_v1.__doc__ = make_csv_dataset_v2.__doc__
 
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
index df58bef739..0e3999c0f4 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
@@ -142,7 +142,7 @@ tf_module {
   }
   member_method {
     name: "make_csv_dataset"
-    argspec: "args=[\'file_pattern\', \'batch_size\', \'column_names\', \'column_defaults\', \'label_name\', \'select_columns\', \'field_delim\', \'use_quote_delim\', \'na_value\', \'header\', \'num_epochs\', \'shuffle\', \'shuffle_buffer_size\', \'shuffle_seed\', \'prefetch_buffer_size\', \'num_parallel_reads\', \'sloppy\', \'num_rows_for_inference\', \'compression_type\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \',\', \'True\', \'\', \'True\', \'None\', \'True\', \'10000\', \'None\', \'-1\', \'1\', \'False\', \'100\', \'None\'], "
+    argspec: "args=[\'file_pattern\', \'batch_size\', \'column_names\', \'column_defaults\', \'label_name\', \'select_columns\', \'field_delim\', \'use_quote_delim\', \'na_value\', \'header\', \'num_epochs\', \'shuffle\', \'shuffle_buffer_size\', \'shuffle_seed\', \'prefetch_buffer_size\', \'num_parallel_reads\', \'sloppy\', \'num_rows_for_inference\', \'compression_type\', \'ignore_errors\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \',\', \'True\', \'\', \'True\', \'None\', \'True\', \'10000\', \'None\', \'-1\', \'1\', \'False\', \'100\', \'None\', \'False\'], "
   }
   member_method {
     name: "make_saveable_from_iterator"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
index abc98a74b6..48d9002864 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
@@ -142,7 +142,7 @@ tf_module {
   }
   member_method {
     name: "make_csv_dataset"
-    argspec: "args=[\'file_pattern\', \'batch_size\', \'column_names\', \'column_defaults\', \'label_name\', \'select_columns\', \'field_delim\', \'use_quote_delim\', \'na_value\', \'header\', \'num_epochs\', \'shuffle\', \'shuffle_buffer_size\', \'shuffle_seed\', \'prefetch_buffer_size\', \'num_parallel_reads\', \'sloppy\', \'num_rows_for_inference\', \'compression_type\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \',\', \'True\', \'\', \'True\', \'None\', \'True\', \'10000\', \'None\', \'-1\', \'1\', \'False\', \'100\', \'None\'], "
+    argspec: "args=[\'file_pattern\', \'batch_size\', \'column_names\', \'column_defaults\', \'label_name\', \'select_columns\', \'field_delim\', \'use_quote_delim\', \'na_value\', \'header\', \'num_epochs\', \'shuffle\', \'shuffle_buffer_size\', \'shuffle_seed\', \'prefetch_buffer_size\', \'num_parallel_reads\', \'sloppy\', \'num_rows_for_inference\', \'compression_type\', \'ignore_errors\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \',\', \'True\', \'\', \'True\', \'None\', \'True\', \'10000\', \'None\', \'-1\', \'1\', \'False\', \'100\', \'None\', \'False\'], "
   }
   member_method {
     name: "make_saveable_from_iterator"
-- 
GitLab


From 93c81e3388593595c3af1fbc463640e99a4634d5 Mon Sep 17 00:00:00 2001
From: Sachin Joglekar <srjoglekar@google.com>
Date: Fri, 8 Feb 2019 13:28:29 -0800
Subject: [PATCH 2188/2345] Adds checks for graph consistency in tflite::Verify

PiperOrigin-RevId: 233115965
---
 tensorflow/lite/toco/tflite/import_test.cc |  2 +-
 tensorflow/lite/tools/BUILD                |  1 +
 tensorflow/lite/tools/verifier.cc          | 92 ++++++++++++++++++++
 tensorflow/lite/tools/verifier_test.cc     | 98 ++++++++++++++++++++--
 4 files changed, 185 insertions(+), 8 deletions(-)

diff --git a/tensorflow/lite/toco/tflite/import_test.cc b/tensorflow/lite/toco/tflite/import_test.cc
index 93ab5141ab..b00c4124d8 100644
--- a/tensorflow/lite/toco/tflite/import_test.cc
+++ b/tensorflow/lite/toco/tflite/import_test.cc
@@ -60,7 +60,7 @@ class ImportTest : public ::testing::Test {
                                builder_.CreateString("tensor_one"), q);
     auto t2 =
         ::tflite::CreateTensor(builder_, builder_.CreateVector<int>({2, 1}),
-                               ::tflite::TensorType_FLOAT32, 2,
+                               ::tflite::TensorType_FLOAT32, 0,
                                builder_.CreateString("tensor_two"), q);
     return builder_.CreateVector(
         std::vector<Offset<::tflite::Tensor>>({t1, t2}));
diff --git a/tensorflow/lite/tools/BUILD b/tensorflow/lite/tools/BUILD
index 1d141b5dd0..82ac13af00 100644
--- a/tensorflow/lite/tools/BUILD
+++ b/tensorflow/lite/tools/BUILD
@@ -71,6 +71,7 @@ cc_library(
         "//tensorflow/lite:schema_fbs_version",
         "//tensorflow/lite:string_util",
         "//tensorflow/lite/schema:schema_fbs",
+        "@com_google_absl//absl/container:flat_hash_set",
     ],
 )
 
diff --git a/tensorflow/lite/tools/verifier.cc b/tensorflow/lite/tools/verifier.cc
index 8631bef67b..b267dddaae 100644
--- a/tensorflow/lite/tools/verifier.cc
+++ b/tensorflow/lite/tools/verifier.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/lite/tools/verifier.h"
 #include <climits>
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/lite/schema/schema_generated.h"
 #include "tensorflow/lite/string_util.h"
 #include "tensorflow/lite/version.h"
@@ -170,6 +171,93 @@ bool VerifyOperators(const Vector<Offset<Operator>>& operators,
   return true;
 }
 
+bool IsConstantTensor(const Tensor& tensor, const Model& model) {
+  if (!tensor.buffer() || !model.buffers()) return false;
+  if (tensor.buffer() > 0 && tensor.buffer() < model.buffers()->size()) {
+    auto* buffer = model.buffers()->Get(tensor.buffer());
+    if (buffer && buffer->data()) {
+      return true;
+    }
+  }
+  return false;
+}
+
+// Performs basic consistency checks on a sub-graph.
+bool VerifySubGraphConsistency(const Model& model, const SubGraph& subgraph,
+                               ErrorReporter* error_reporter) {
+  absl::flat_hash_set<int> subgraph_input_tensors, constant_tensors,
+      variable_tensors, output_tensors;
+  for (int i = 0; i < subgraph.tensors()->Length(); ++i) {
+    const auto* tensor = subgraph.tensors()->Get(i);
+    bool is_constant_tensor = false;
+    if (model.buffers() && tensor->buffer() > 0 &&
+        tensor->buffer() < model.buffers()->size()) {
+      auto* buffer = model.buffers()->Get(tensor->buffer());
+      if (buffer && buffer->data()) {
+        is_constant_tensor = true;
+      }
+    }
+    if (IsConstantTensor(*tensor, model)) {
+      constant_tensors.insert(i);
+    } else if (tensor->is_variable()) {
+      variable_tensors.insert(i);
+    }
+  }
+  for (const int tensor_idx : *subgraph.inputs()) {
+    subgraph_input_tensors.insert(tensor_idx);
+  }
+
+  for (int op_idx = 0; op_idx < subgraph.operators()->Length(); ++op_idx) {
+    const auto* op = subgraph.operators()->Get(op_idx);
+    const auto& opcode = model.operator_codes()->Get(op->opcode_index());
+    // Check for invalid inputs by ensuring all exist in produced_tensors.
+    for (const int input_idx : *op->inputs()) {
+      if (constant_tensors.find(input_idx) == constant_tensors.end() &&
+          variable_tensors.find(input_idx) == variable_tensors.end() &&
+          subgraph_input_tensors.find(input_idx) ==
+              subgraph_input_tensors.end() &&
+          output_tensors.find(input_idx) == output_tensors.end()) {
+        ReportError(error_reporter,
+                    "Input tensor %d to op %d (%s) is not produced", input_idx,
+                    op_idx, EnumNameBuiltinOperator(opcode->builtin_code()));
+        return false;
+      }
+    }
+    // Check for cycles/invalid outputs by ensuring that none exist in
+    // produced_tensors.
+    for (const int output_idx : *op->outputs()) {
+      if (constant_tensors.find(output_idx) != constant_tensors.end()) {
+        ReportError(error_reporter,
+                    "Output tensor %d to op %d (%s) is a constant", output_idx,
+                    op_idx, EnumNameBuiltinOperator(opcode->builtin_code()));
+        return false;
+      } else if (variable_tensors.find(output_idx) != variable_tensors.end()) {
+        ReportError(error_reporter,
+                    "Output tensor %d to op %d (%s) is a variable", output_idx,
+                    op_idx, EnumNameBuiltinOperator(opcode->builtin_code()));
+        return false;
+      } else if (subgraph_input_tensors.find(output_idx) !=
+                 subgraph_input_tensors.end()) {
+        ReportError(error_reporter,
+                    "Output tensor %d to op %d (%s) is a subgraph input",
+                    output_idx, op_idx,
+                    EnumNameBuiltinOperator(opcode->builtin_code()));
+        return false;
+      } else if (output_tensors.find(output_idx) != output_tensors.end()) {
+        ReportError(error_reporter,
+                    "Output tensor %d to op %d (%s) is an output from "
+                    "another op. There is a cycle in the graph",
+                    output_idx, op_idx,
+                    EnumNameBuiltinOperator(opcode->builtin_code()));
+        return false;
+      }
+      // This can be an input to a subsequent op.
+      output_tensors.insert(output_idx);
+    }
+  }
+  return true;
+}
+
 bool VerifySubGraphs(const Model& model, ErrorReporter* error_reporter) {
   if (!model.subgraphs()) {
     ReportError(error_reporter, "Missing 'subgraphs' section.");
@@ -184,6 +272,10 @@ bool VerifySubGraphs(const Model& model, ErrorReporter* error_reporter) {
     if (!VerifyOperators(*subgraph->operators(), error_reporter)) {
       return false;
     }
+
+    if (!VerifySubGraphConsistency(model, *subgraph, error_reporter)) {
+      return false;
+    }
   }
   return true;
 }
diff --git a/tensorflow/lite/tools/verifier_test.cc b/tensorflow/lite/tools/verifier_test.cc
index 083f00d445..17577f7490 100644
--- a/tensorflow/lite/tools/verifier_test.cc
+++ b/tensorflow/lite/tools/verifier_test.cc
@@ -72,7 +72,8 @@ class TfLiteFlatbufferModelBuilder {
   }
 
   void AddTensor(const std::vector<int>& shape, tflite::TensorType type,
-                 const std::vector<uint8_t>& buffer, const char* name) {
+                 const std::vector<uint8_t>& buffer, const char* name,
+                 const bool is_variable = false) {
     int buffer_index = 0;
     if (!buffer.empty()) {
       buffer_index = buffers_.size();
@@ -81,11 +82,12 @@ class TfLiteFlatbufferModelBuilder {
     if (shape.empty()) {
       tensors_.push_back(CreateTensorDirect(builder_, /*shape=*/nullptr, type,
                                             buffer_index, name,
-                                            /*quantization=*/0));
+                                            /*quantization=*/0, is_variable));
       return;
     }
     tensors_.push_back(CreateTensorDirect(builder_, &shape, type, buffer_index,
-                                          name, /*quantization=*/0));
+                                          name, /*quantization=*/0,
+                                          is_variable));
   }
 
   void AddOperator(const std::vector<int32_t>& inputs,
@@ -148,7 +150,7 @@ TEST(VerifyModel, TestEmptyModel) {
 
 TEST(VerifyModel, TestEmptyShape) {
   TfLiteFlatbufferModelBuilder builder({}, {"test"});
-  builder.AddOperator({0, 1}, {2}, BuiltinOperator_CUSTOM, "test");
+  builder.AddOperator({0, 1}, {3}, BuiltinOperator_CUSTOM, "test");
   builder.AddTensor({2, 3}, TensorType_UINT8, {1, 2, 3, 4, 5, 6}, "input");
   builder.AddTensor({}, TensorType_UINT8, {1, 2, 3, 4, 5, 6}, "inputtwo");
   builder.AddTensor(
@@ -156,7 +158,7 @@ TEST(VerifyModel, TestEmptyShape) {
       {2, 0, 0, 0, 16, 0, 0, 0, 17, 0, 0, 0, 19, 0, 0, 0, 'A', 'B', 'C'},
       "data");
   builder.AddTensor({2, 3}, TensorType_INT32, {}, "output");
-  builder.FinishModel({0, 1}, {2});
+  builder.FinishModel({0, 1}, {3});
   ASSERT_FALSE(builder.Verify());
   EXPECT_THAT(builder.GetErrorString(),
               ::testing::ContainsRegex("Tensor shape is empty"));
@@ -329,9 +331,10 @@ TEST(VerifyModel, AllOpsAreSupported) {
   TfLiteFlatbufferModelBuilder builder({BuiltinOperator_ADD}, {"CustomOp"});
   builder.AddTensor({2, 2}, TensorType_UINT8, {1, 2, 3, 4}, "input1");
   builder.AddTensor({2, 2}, TensorType_UINT8, {1, 2, 3, 4}, "input2");
-  builder.AddTensor({2, 2}, TensorType_UINT8, {}, "output");
+  builder.AddTensor({2, 2}, TensorType_UINT8, {}, "output1");
+  builder.AddTensor({2, 2}, TensorType_UINT8, {}, "output2");
   builder.AddOperator({0, 1}, {2}, BuiltinOperator_ADD, nullptr);
-  builder.AddOperator({0, 1}, {2}, BuiltinOperator_CUSTOM, "CustomOp");
+  builder.AddOperator({0, 1}, {3}, BuiltinOperator_CUSTOM, "CustomOp");
   builder.FinishModel({}, {});
   ASSERT_TRUE(builder.Verify());
   EXPECT_EQ("", builder.GetErrorString());
@@ -363,6 +366,87 @@ TEST(VerifyModel, UseUnsupportedCustomOps) {
                   "Unsupported custom op: Not supported, version: 1"));
 }
 
+TEST(VerifyModel, UnpopulatedInputToOp) {
+  TfLiteFlatbufferModelBuilder builder({}, {"test"});
+  builder.AddOperator({1, 2}, {3}, BuiltinOperator_CUSTOM, "test");
+  builder.AddTensor({2, 3}, TensorType_UINT8, {1, 2, 3, 4, 5, 6}, "input");
+  // This tensor will never be populated.
+  builder.AddTensor({2, 3}, TensorType_UINT8, {}, "invalid_input");
+  builder.AddTensor(
+      {2}, TensorType_STRING,
+      {2, 0, 0, 0, 16, 0, 0, 0, 17, 0, 0, 0, 19, 0, 0, 0, 'A', 'B', 'C'},
+      "data");
+  builder.AddTensor({2, 3}, TensorType_INT32, {}, "output");
+  builder.FinishModel({0, 2}, {3});
+  ASSERT_FALSE(builder.Verify());
+  EXPECT_EQ("Input tensor 1 to op 0 (CUSTOM) is not produced",
+            builder.GetErrorString());
+}
+
+TEST(VerifyModel, MultipleOpsOutputToSameTensor) {
+  TfLiteFlatbufferModelBuilder builder({BuiltinOperator_ADD}, {"CustomOp"});
+  builder.AddTensor({2, 2}, TensorType_UINT8, {1, 2, 3, 4}, "input1");
+  builder.AddTensor({2, 2}, TensorType_UINT8, {1, 2, 3, 4}, "input2");
+  builder.AddTensor({2, 2}, TensorType_UINT8, {}, "output1");
+  builder.AddOperator({0, 1}, {2}, BuiltinOperator_ADD, nullptr);
+  // This can't output to "output1", since the first operator does that.
+  builder.AddOperator({0, 1}, {2}, BuiltinOperator_CUSTOM, "CustomOp");
+  builder.FinishModel({}, {});
+  ASSERT_FALSE(builder.Verify());
+  EXPECT_EQ(
+      "Output tensor 2 to op 1 (CUSTOM) is an output from another op. "
+      "There is a cycle in the graph",
+      builder.GetErrorString());
+}
+
+TEST(VerifyModel, OutputIsAConstantTensor) {
+  TfLiteFlatbufferModelBuilder builder({}, {"test"});
+  builder.AddOperator({0, 1}, {2}, BuiltinOperator_CUSTOM, "test");
+  builder.AddTensor({2, 3}, TensorType_UINT8, {1, 2, 3, 4, 5, 6}, "input");
+  builder.AddTensor(
+      {2}, TensorType_STRING,
+      {2, 0, 0, 0, 16, 0, 0, 0, 17, 0, 0, 0, 19, 0, 0, 0, 'A', 'B', 'C'},
+      "data");
+  // Output shouldn't be populated with constant value.
+  builder.AddTensor({2, 3}, TensorType_INT32, {1, 2, 3, 4, 5, 6}, "output");
+  builder.FinishModel({0, 1}, {2});
+  ASSERT_FALSE(builder.Verify());
+  EXPECT_EQ("Output tensor 2 to op 0 (CUSTOM) is a constant",
+            builder.GetErrorString());
+}
+
+TEST(VerifyModel, OutputIsSubgraphInput) {
+  TfLiteFlatbufferModelBuilder builder({}, {"test"});
+  builder.AddOperator({0, 1}, {2}, BuiltinOperator_CUSTOM, "test");
+  builder.AddTensor({2, 3}, TensorType_UINT8, {1, 2, 3, 4, 5, 6}, "input");
+  builder.AddTensor(
+      {2}, TensorType_STRING,
+      {2, 0, 0, 0, 16, 0, 0, 0, 17, 0, 0, 0, 19, 0, 0, 0, 'A', 'B', 'C'},
+      "data");
+  builder.AddTensor({2, 3}, TensorType_INT32, {}, "output");
+  // Output shouldn't be a subgraph input.
+  builder.FinishModel({0, 1, 2}, {2});
+  ASSERT_FALSE(builder.Verify());
+  EXPECT_EQ("Output tensor 2 to op 0 (CUSTOM) is a subgraph input",
+            builder.GetErrorString());
+}
+
+TEST(VerifyModel, OutputIsAVariable) {
+  TfLiteFlatbufferModelBuilder builder({}, {"test"});
+  builder.AddOperator({0, 1}, {2}, BuiltinOperator_CUSTOM, "test");
+  builder.AddTensor({2, 3}, TensorType_UINT8, {1, 2, 3, 4, 5, 6}, "input");
+  builder.AddTensor(
+      {2}, TensorType_STRING,
+      {2, 0, 0, 0, 16, 0, 0, 0, 17, 0, 0, 0, 19, 0, 0, 0, 'A', 'B', 'C'},
+      "data");
+  // Output shouldn't be a variable.
+  builder.AddTensor({2, 3}, TensorType_INT32, {}, "output", /*variable*/ true);
+  builder.FinishModel({0, 1}, {2});
+  ASSERT_FALSE(builder.Verify());
+  EXPECT_EQ("Output tensor 2 to op 0 (CUSTOM) is a variable",
+            builder.GetErrorString());
+}
+
 // TODO(yichengfan): make up malicious files to test with.
 
 }  // namespace tflite
-- 
GitLab


From ed415ab0927024b12e9cc919e6af6314fd10659e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 8 Feb 2019 13:28:44 -0800
Subject: [PATCH 2189/2345] Updates TPUEstimator documentation; TPUEstimator
 works fine with TPUEstimatorSpec when eval_on_tpu=False.

PiperOrigin-RevId: 233116009
---
 tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index a6b3b13173..4f761e3599 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -2019,9 +2019,9 @@ class TPUEstimator(estimator_lib.Estimator):
   ==========
 
   `model_fn` should return `TPUEstimatorSpec`, which expects the `eval_metrics`
-  for TPU evaluation. However, if eval_on_tpu is False, `model_fn` must return
-  `EstimatorSpec` and the evaluation will execute on CPU or GPU; in this case
-  the following discussion on TPU evaluation does not apply.
+  for TPU evaluation. If eval_on_tpu is False, the evaluation will execute on
+  CPU or GPU; in this case the following discussion on TPU evaluation does not
+  apply.
 
   `TPUEstimatorSpec.eval_metrics` is a tuple of `metric_fn` and `tensors`, where
   `tensors` could be a list of any nested structure of `Tensor`s (See
-- 
GitLab


From 6eb24926d8564dd3a8433f0a217b66165eb92b3e Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Fri, 8 Feb 2019 13:30:15 -0800
Subject: [PATCH 2190/2345] [XLA] Remove R1-form DS/DUS support from the
 Evaluator

PiperOrigin-RevId: 233116272
---
 .../xla/service/hlo_evaluator_typed_visitor.h | 38 ++++---------------
 1 file changed, 8 insertions(+), 30 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
index 68b67a9621..8def61dc63 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
@@ -2809,21 +2809,10 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
       absl::Span<HloInstruction* const> start_indices,
       const Shape& result_shape) {
     std::vector<int64> start;
-    // TODO(b/118437727): Remove the R1 code-path. Note that to distinguish
-    // between the cases, this currently assumes there is at least 1 index. That
-    // is wrong in the general case, because for scalar indices, if the operand
-    // is scalar, then there are no indices. This problem with resolve itself.
-    const HloInstruction* first_index = start_indices[0];
-    if (first_index->shape().rank() == 1) {
-      auto start_indices_typed =
-          parent_->GetEvaluatedLiteralFor(first_index).data<IndexT>();
-      start = std::vector<int64>(start_indices_typed.begin(),
-                                 start_indices_typed.end());
-    } else {
-      for (HloInstruction* index : start_indices) {
-        start.push_back(
-            parent_->GetEvaluatedLiteralFor(index).GetFirstElement<IndexT>());
-      }
+
+    for (HloInstruction* index : start_indices) {
+      start.push_back(
+          parent_->GetEvaluatedLiteralFor(index).GetFirstElement<IndexT>());
     }
 
     // Clamp the start indices so the slice is in-bounds w.r.t the operand.
@@ -2856,22 +2845,11 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
     auto result = operand_literal.Clone();
     const auto rank = result.shape().rank();
     std::vector<int64> start;
-    // TODO(b/118437727): Remove the R1 code-path. Note that to distinguish
-    // between the cases, this currently assumes there is at least 1 index. That
-    // is wrong in the general case, because for scalar indices, if the operand
-    // is scalar, then there are no indices. This problem with resolve itself.
-    const HloInstruction* first_index = start_indices[0];
-    if (first_index->shape().rank() == 1) {
-      auto start_indices_typed =
-          parent_->GetEvaluatedLiteralFor(first_index).data<IndexT>();
-      start = std::vector<int64>(start_indices_typed.begin(),
-                                 start_indices_typed.end());
-    } else {
-      for (HloInstruction* index : start_indices) {
-        start.push_back(
-            parent_->GetEvaluatedLiteralFor(index).GetFirstElement<IndexT>());
-      }
+    for (HloInstruction* index : start_indices) {
+      start.push_back(
+          parent_->GetEvaluatedLiteralFor(index).GetFirstElement<IndexT>());
     }
+
     // Clamp the update start indices so the slice is in-bounds w.r.t the
     // operand.
     for (int64 i = 0; i < rank; ++i) {
-- 
GitLab


From 414296b1a47e6331c14a4b37af1ae260579ea948 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Fri, 8 Feb 2019 13:33:14 -0800
Subject: [PATCH 2191/2345] Another corner case for the print function: `print
 (`.

PiperOrigin-RevId: 233116839
---
 tensorflow/python/autograph/pyct/parser.py      | 2 +-
 tensorflow/python/autograph/pyct/parser_test.py | 9 +++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/autograph/pyct/parser.py b/tensorflow/python/autograph/pyct/parser.py
index 8b73440280..f6b2a7863b 100644
--- a/tensorflow/python/autograph/pyct/parser.py
+++ b/tensorflow/python/autograph/pyct/parser.py
@@ -111,7 +111,7 @@ def parse_str(src):
   """Returns the AST of given piece of code."""
   # TODO(mdan): This should exclude the module things are autowrapped in.
 
-  if six.PY2 and re.search('\\Wprint\\(', src):
+  if six.PY2 and re.search('\\Wprint\\s*\\(', src):
     # This special treatment is required because gast.parse is not aware of
     # whether print_function was present in the original context.
     src = 'from __future__ import print_function\n' + src
diff --git a/tensorflow/python/autograph/pyct/parser_test.py b/tensorflow/python/autograph/pyct/parser_test.py
index cc0123408d..e7fa3c7aeb 100644
--- a/tensorflow/python/autograph/pyct/parser_test.py
+++ b/tensorflow/python/autograph/pyct/parser_test.py
@@ -51,6 +51,15 @@ class ParserTest(test.TestCase):
     """))
     self.assertEqual('f', mod.body[0].name)
 
+  def test_parse_str_weird_print(self):
+    mod = parser.parse_str(
+        textwrap.dedent("""
+            def f(x):
+              print (x)
+              return x + 1
+    """))
+    self.assertEqual('f', mod.body[0].name)
+
   def test_parse_comments(self):
     def f():
 # unindented comment
-- 
GitLab


From fa4ac2ba72e87eb203a2379c5ff946433533db69 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Fri, 8 Feb 2019 13:48:35 -0800
Subject: [PATCH 2192/2345] Add module aliases for v2 optimizers tf.optimizers
 = tf.keras.optimizers

PiperOrigin-RevId: 233119697
---
 tensorflow/api_template.__init__.py           |  1 +
 tensorflow/compat_template.__init__.py        |  1 +
 .../v2/tensorflow.optimizers.-adadelta.pbtxt  | 71 +++++++++++++++++++
 .../v2/tensorflow.optimizers.-adagrad.pbtxt   | 71 +++++++++++++++++++
 .../v2/tensorflow.optimizers.-adam.pbtxt      | 71 +++++++++++++++++++
 .../v2/tensorflow.optimizers.-adamax.pbtxt    | 71 +++++++++++++++++++
 .../v2/tensorflow.optimizers.-nadam.pbtxt     | 71 +++++++++++++++++++
 .../v2/tensorflow.optimizers.-optimizer.pbtxt | 70 ++++++++++++++++++
 .../v2/tensorflow.optimizers.-r-m-sprop.pbtxt | 71 +++++++++++++++++++
 .../v2/tensorflow.optimizers.-s-g-d.pbtxt     | 71 +++++++++++++++++++
 .../api/golden/v2/tensorflow.optimizers.pbtxt | 51 +++++++++++++
 ...imizers.schedules.-exponential-decay.pbtxt | 18 +++++
 ...mizers.schedules.-inverse-time-decay.pbtxt | 18 +++++
 ...rs.schedules.-learning-rate-schedule.pbtxt | 16 +++++
 ....schedules.-piecewise-constant-decay.pbtxt | 18 +++++
 ...timizers.schedules.-polynomial-decay.pbtxt | 18 +++++
 .../v2/tensorflow.optimizers.schedules.pbtxt  | 31 ++++++++
 .../tools/api/golden/v2/tensorflow.pbtxt      |  4 ++
 18 files changed, 743 insertions(+)
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adadelta.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adagrad.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adam.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adamax.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.optimizers.-nadam.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.optimizers.-optimizer.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.optimizers.-r-m-sprop.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.optimizers.-s-g-d.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.optimizers.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.optimizers.schedules.-exponential-decay.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.optimizers.schedules.-inverse-time-decay.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.optimizers.schedules.-learning-rate-schedule.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.optimizers.schedules.-piecewise-constant-decay.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.optimizers.schedules.-polynomial-decay.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.optimizers.schedules.pbtxt

diff --git a/tensorflow/api_template.__init__.py b/tensorflow/api_template.__init__.py
index 8a101e282e..a6eb4755f3 100644
--- a/tensorflow/api_template.__init__.py
+++ b/tensorflow/api_template.__init__.py
@@ -115,5 +115,6 @@ except NameError:
 if hasattr(_current_module, 'keras'):
   losses = keras.losses
   metrics = keras.metrics
+  optimizers = keras.optimizers
 
 # pylint: enable=undefined-variable
diff --git a/tensorflow/compat_template.__init__.py b/tensorflow/compat_template.__init__.py
index 953c70ab1e..05fd9cd981 100644
--- a/tensorflow/compat_template.__init__.py
+++ b/tensorflow/compat_template.__init__.py
@@ -48,3 +48,4 @@ _current_module = _sys.modules[__name__]
 if hasattr(_current_module, 'keras'):
   losses = keras.losses
   metrics = keras.metrics
+  optimizers = keras.optimizers
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adadelta.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adadelta.pbtxt
new file mode 100644
index 0000000000..ff95b1fa02
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adadelta.pbtxt
@@ -0,0 +1,71 @@
+path: "tensorflow.optimizers.Adadelta"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.adadelta.Adadelta\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "iterations"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'learning_rate\', \'rho\', \'epsilon\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'0.001\', \'0.95\', \'1e-07\', \'Adadelta\'], "
+  }
+  member_method {
+    name: "add_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\', \'initializer\'], varargs=None, keywords=None, defaults=[\'zeros\'], "
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'trainable\', \'synchronization\', \'aggregation\'], varargs=None, keywords=None, defaults=[\'None\', \'zeros\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply_gradients"
+    argspec: "args=[\'self\', \'grads_and_vars\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_gradients"
+    argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates"
+    argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "minimize"
+    argspec: "args=[\'self\', \'loss\', \'var_list\', \'grad_loss\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "variables"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adagrad.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adagrad.pbtxt
new file mode 100644
index 0000000000..5341772a25
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adagrad.pbtxt
@@ -0,0 +1,71 @@
+path: "tensorflow.optimizers.Adagrad"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.adagrad.Adagrad\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "iterations"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'learning_rate\', \'initial_accumulator_value\', \'epsilon\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'0.001\', \'0.1\', \'1e-07\', \'Adagrad\'], "
+  }
+  member_method {
+    name: "add_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\', \'initializer\'], varargs=None, keywords=None, defaults=[\'zeros\'], "
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'trainable\', \'synchronization\', \'aggregation\'], varargs=None, keywords=None, defaults=[\'None\', \'zeros\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply_gradients"
+    argspec: "args=[\'self\', \'grads_and_vars\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_gradients"
+    argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates"
+    argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "minimize"
+    argspec: "args=[\'self\', \'loss\', \'var_list\', \'grad_loss\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "variables"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adam.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adam.pbtxt
new file mode 100644
index 0000000000..61f685705f
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adam.pbtxt
@@ -0,0 +1,71 @@
+path: "tensorflow.optimizers.Adam"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.adam.Adam\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "iterations"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'learning_rate\', \'beta_1\', \'beta_2\', \'epsilon\', \'amsgrad\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'0.001\', \'0.9\', \'0.999\', \'1e-07\', \'False\', \'Adam\'], "
+  }
+  member_method {
+    name: "add_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\', \'initializer\'], varargs=None, keywords=None, defaults=[\'zeros\'], "
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'trainable\', \'synchronization\', \'aggregation\'], varargs=None, keywords=None, defaults=[\'None\', \'zeros\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply_gradients"
+    argspec: "args=[\'self\', \'grads_and_vars\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_gradients"
+    argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates"
+    argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "minimize"
+    argspec: "args=[\'self\', \'loss\', \'var_list\', \'grad_loss\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "variables"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adamax.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adamax.pbtxt
new file mode 100644
index 0000000000..9458b431a4
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adamax.pbtxt
@@ -0,0 +1,71 @@
+path: "tensorflow.optimizers.Adamax"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.adamax.Adamax\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "iterations"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'learning_rate\', \'beta_1\', \'beta_2\', \'epsilon\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'0.001\', \'0.9\', \'0.999\', \'1e-07\', \'Adamax\'], "
+  }
+  member_method {
+    name: "add_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\', \'initializer\'], varargs=None, keywords=None, defaults=[\'zeros\'], "
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'trainable\', \'synchronization\', \'aggregation\'], varargs=None, keywords=None, defaults=[\'None\', \'zeros\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply_gradients"
+    argspec: "args=[\'self\', \'grads_and_vars\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_gradients"
+    argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates"
+    argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "minimize"
+    argspec: "args=[\'self\', \'loss\', \'var_list\', \'grad_loss\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "variables"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-nadam.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-nadam.pbtxt
new file mode 100644
index 0000000000..05ccf489e3
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-nadam.pbtxt
@@ -0,0 +1,71 @@
+path: "tensorflow.optimizers.Nadam"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.nadam.Nadam\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "iterations"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'learning_rate\', \'beta_1\', \'beta_2\', \'epsilon\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'0.001\', \'0.9\', \'0.999\', \'1e-07\', \'Nadam\'], "
+  }
+  member_method {
+    name: "add_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\', \'initializer\'], varargs=None, keywords=None, defaults=[\'zeros\'], "
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'trainable\', \'synchronization\', \'aggregation\'], varargs=None, keywords=None, defaults=[\'None\', \'zeros\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply_gradients"
+    argspec: "args=[\'self\', \'grads_and_vars\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_gradients"
+    argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates"
+    argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "minimize"
+    argspec: "args=[\'self\', \'loss\', \'var_list\', \'grad_loss\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "variables"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-optimizer.pbtxt
new file mode 100644
index 0000000000..54a983e7b1
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-optimizer.pbtxt
@@ -0,0 +1,70 @@
+path: "tensorflow.optimizers.Optimizer"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "iterations"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\', \'initializer\'], varargs=None, keywords=None, defaults=[\'zeros\'], "
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'trainable\', \'synchronization\', \'aggregation\'], varargs=None, keywords=None, defaults=[\'None\', \'zeros\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply_gradients"
+    argspec: "args=[\'self\', \'grads_and_vars\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_gradients"
+    argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates"
+    argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "minimize"
+    argspec: "args=[\'self\', \'loss\', \'var_list\', \'grad_loss\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "variables"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-r-m-sprop.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-r-m-sprop.pbtxt
new file mode 100644
index 0000000000..35ee0855fc
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-r-m-sprop.pbtxt
@@ -0,0 +1,71 @@
+path: "tensorflow.optimizers.RMSprop"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.rmsprop.RMSprop\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "iterations"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'learning_rate\', \'rho\', \'momentum\', \'epsilon\', \'centered\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'0.001\', \'0.9\', \'0.0\', \'1e-07\', \'False\', \'RMSprop\'], "
+  }
+  member_method {
+    name: "add_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\', \'initializer\'], varargs=None, keywords=None, defaults=[\'zeros\'], "
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'trainable\', \'synchronization\', \'aggregation\'], varargs=None, keywords=None, defaults=[\'None\', \'zeros\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply_gradients"
+    argspec: "args=[\'self\', \'grads_and_vars\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_gradients"
+    argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates"
+    argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "minimize"
+    argspec: "args=[\'self\', \'loss\', \'var_list\', \'grad_loss\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "variables"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-s-g-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-s-g-d.pbtxt
new file mode 100644
index 0000000000..ddbba60e59
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-s-g-d.pbtxt
@@ -0,0 +1,71 @@
+path: "tensorflow.optimizers.SGD"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.gradient_descent.SGD\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "iterations"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'learning_rate\', \'momentum\', \'nesterov\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'0.001\', \'0.0\', \'False\', \'SGD\'], "
+  }
+  member_method {
+    name: "add_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\', \'initializer\'], varargs=None, keywords=None, defaults=[\'zeros\'], "
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'trainable\', \'synchronization\', \'aggregation\'], varargs=None, keywords=None, defaults=[\'None\', \'zeros\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply_gradients"
+    argspec: "args=[\'self\', \'grads_and_vars\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_gradients"
+    argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates"
+    argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "minimize"
+    argspec: "args=[\'self\', \'loss\', \'var_list\', \'grad_loss\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "variables"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.pbtxt
new file mode 100644
index 0000000000..ed846c7da7
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.pbtxt
@@ -0,0 +1,51 @@
+path: "tensorflow.optimizers"
+tf_module {
+  member {
+    name: "Adadelta"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Adagrad"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Adam"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Adamax"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Nadam"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Optimizer"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "RMSprop"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "SGD"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "schedules"
+    mtype: "<type \'module\'>"
+  }
+  member_method {
+    name: "deserialize"
+    argspec: "args=[\'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "get"
+    argspec: "args=[\'identifier\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "serialize"
+    argspec: "args=[\'optimizer\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.schedules.-exponential-decay.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.schedules.-exponential-decay.pbtxt
new file mode 100644
index 0000000000..f174d6cfc3
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.schedules.-exponential-decay.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.optimizers.schedules.ExponentialDecay"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.ExponentialDecay\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.LearningRateSchedule\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'initial_learning_rate\', \'decay_steps\', \'decay_rate\', \'staircase\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.schedules.-inverse-time-decay.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.schedules.-inverse-time-decay.pbtxt
new file mode 100644
index 0000000000..851d21c5cc
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.schedules.-inverse-time-decay.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.optimizers.schedules.InverseTimeDecay"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.InverseTimeDecay\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.LearningRateSchedule\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'initial_learning_rate\', \'decay_steps\', \'decay_rate\', \'staircase\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.schedules.-learning-rate-schedule.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.schedules.-learning-rate-schedule.pbtxt
new file mode 100644
index 0000000000..36db36e4f4
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.schedules.-learning-rate-schedule.pbtxt
@@ -0,0 +1,16 @@
+path: "tensorflow.optimizers.schedules.LearningRateSchedule"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.LearningRateSchedule\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.schedules.-piecewise-constant-decay.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.schedules.-piecewise-constant-decay.pbtxt
new file mode 100644
index 0000000000..6613bedef5
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.schedules.-piecewise-constant-decay.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.optimizers.schedules.PiecewiseConstantDecay"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.PiecewiseConstantDecay\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.LearningRateSchedule\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'boundaries\', \'values\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.schedules.-polynomial-decay.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.schedules.-polynomial-decay.pbtxt
new file mode 100644
index 0000000000..fbd5bcef8f
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.schedules.-polynomial-decay.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.optimizers.schedules.PolynomialDecay"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.PolynomialDecay\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.learning_rate_schedule.LearningRateSchedule\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'initial_learning_rate\', \'decay_steps\', \'end_learning_rate\', \'power\', \'cycle\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0001\', \'1.0\', \'False\', \'None\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.schedules.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.schedules.pbtxt
new file mode 100644
index 0000000000..8e1e61b766
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.schedules.pbtxt
@@ -0,0 +1,31 @@
+path: "tensorflow.optimizers.schedules"
+tf_module {
+  member {
+    name: "ExponentialDecay"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "InverseTimeDecay"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "LearningRateSchedule"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "PiecewiseConstantDecay"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "PolynomialDecay"
+    mtype: "<type \'type\'>"
+  }
+  member_method {
+    name: "deserialize"
+    argspec: "args=[\'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "serialize"
+    argspec: "args=[\'learning_rate_schedule\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index f28a3a5308..a75c0c27dc 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -232,6 +232,10 @@ tf_module {
     name: "ones_initializer"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "optimizers"
+    mtype: "<type \'module\'>"
+  }
   member {
     name: "qint16"
     mtype: "<class \'tensorflow.python.framework.dtypes.DType\'>"
-- 
GitLab


From 0f5fd00006e6ef8bae5a5f62304f1cbb3c78e584 Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Fri, 8 Feb 2019 14:06:13 -0800
Subject: [PATCH 2193/2345] Recurrent layer: Force runtime type string on CPUs,
 as otherwise it breaks when the containing code is explicitly placed on a
 GPU, for example when using distribution strategy.

PiperOrigin-RevId: 233123011
---
 tensorflow/python/keras/layers/recurrent.py | 24 ++++++++++-----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/tensorflow/python/keras/layers/recurrent.py b/tensorflow/python/keras/layers/recurrent.py
index 79c6197211..ff5a37b627 100644
--- a/tensorflow/python/keras/layers/recurrent.py
+++ b/tensorflow/python/keras/layers/recurrent.py
@@ -2114,8 +2114,7 @@ class UnifiedGRU(GRU):
           time_major=self.time_major,
           zero_output_for_mask=self.zero_output_for_mask)
       # This is a dummy tensor for testing purpose.
-      runtime = constant_op.constant(
-          'unknown', dtype=dtypes.string, name='runtime')
+      runtime = _runtime('unknown')
     else:
       last_output, outputs, runtime, states = self._defun_gru_call(
           inputs, initial_state, training)
@@ -2266,8 +2265,7 @@ def standard_gru(inputs, init_h, kernel, recurrent_kernel, bias, activation,
       unroll=False,
       time_major=time_major,
       input_length=timesteps)
-  return last_output, outputs, new_states[0], constant_op.constant(
-      'cpu', dtype=dtypes.string, name='runtime')
+  return last_output, outputs, new_states[0], _runtime('cpu')
 
 
 def cudnn_gru(inputs, init_h, kernel, recurrent_kernel, bias, time_major):
@@ -2309,8 +2307,7 @@ def cudnn_gru(inputs, init_h, kernel, recurrent_kernel, bias, time_major):
   if not time_major:
     outputs = array_ops.transpose(outputs, perm=[1, 0, 2])
   h = h[0]
-  return last_output, outputs, h, constant_op.constant(
-      'cudnn', dtype=dtypes.string, name='runtime')
+  return last_output, outputs, h, _runtime('cudnn')
 
 
 @keras_export('keras.layers.LSTMCell')
@@ -3090,8 +3087,7 @@ class UnifiedLSTM(LSTM):
           input_length=timesteps,
           time_major=self.time_major,
           zero_output_for_mask=self.zero_output_for_mask)
-      runtime = constant_op.constant(
-          'unknown', dtype=dtypes.string, name='runtime')
+      runtime = _runtime('unknown')
     else:
       # Use the new defun approach for backend implementation swap.
       # Note that different implementations need to have same function
@@ -3266,8 +3262,7 @@ def standard_lstm(inputs, init_h, init_c, kernel, recurrent_kernel, bias,
       unroll=False,
       time_major=time_major,
       input_length=timesteps)
-  return last_output, outputs, new_states[0], new_states[
-      1], constant_op.constant('cpu', dtype=dtypes.string, name='runtime')
+  return last_output, outputs, new_states[0], new_states[1], _runtime('cpu')
 
 
 def cudnn_lstm(inputs, input_h, input_c, kernel, recurrent_kernel, bias,
@@ -3298,8 +3293,7 @@ def cudnn_lstm(inputs, input_h, input_c, kernel, recurrent_kernel, bias,
   h = h[0]
   c = c[0]
 
-  return last_output, outputs, h, c, constant_op.constant(
-      'cudnn', dtype=dtypes.string, name='runtime')
+  return last_output, outputs, h, c, _runtime('cudnn')
 
 
 def _generate_dropout_mask(ones, rate, training=None, count=1):
@@ -3418,3 +3412,9 @@ def _generate_defun_backend(unique_api_name, preferred_device, func):
   }
   return function.defun_with_attributes(func=func,
                                         attributes=function_attributes)
+
+
+def _runtime(runtime_name):
+  with ops.device('/cpu:0'):
+    return constant_op.constant(
+        runtime_name, dtype=dtypes.string, name='runtime')
-- 
GitLab


From 6802d3b3669cce0ff4df3c18f59d19e1d9f85fb2 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Fri, 8 Feb 2019 14:14:17 -0800
Subject: [PATCH 2194/2345] Better error message when trying to capture tensors
 from inner graphs.

PiperOrigin-RevId: 233124512
---
 tensorflow/python/eager/def_function_test.py | 14 ++++++++++++++
 tensorflow/python/framework/func_graph.py    | 16 ++++++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/tensorflow/python/eager/def_function_test.py b/tensorflow/python/eager/def_function_test.py
index 2c9da8b4ee..6680f0b3e6 100644
--- a/tensorflow/python/eager/def_function_test.py
+++ b/tensorflow/python/eager/def_function_test.py
@@ -31,6 +31,7 @@ from tensorflow.python.framework import tensor_spec
 from tensorflow.python.framework import test_util
 from tensorflow.python.keras.engine import training
 from tensorflow.python.keras.layers import core
+from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import resource_variable_ops
@@ -271,6 +272,19 @@ class DefFunctionTest(test.TestCase):
                      (tensor_spec.TensorSpec(
                          None, dtypes.float32, name='x'),))
 
+  def test_error_inner_capture(self):
+
+    @def_function.function
+    def f(inputs):
+      num_steps, _ = inputs.shape[:2]
+      outputs = []
+      for t in math_ops.range(num_steps):
+        outputs.append(inputs[t])
+      return outputs
+
+    with self.assertRaisesRegexp(ValueError, 'inner'):
+      f(array_ops.zeros(shape=(8, 42, 3)))
+
   def test_serialization_signature_cache(self):
 
     @def_function.function
diff --git a/tensorflow/python/framework/func_graph.py b/tensorflow/python/framework/func_graph.py
index 3f81466cd4..465d97999a 100644
--- a/tensorflow/python/framework/func_graph.py
+++ b/tensorflow/python/framework/func_graph.py
@@ -195,6 +195,9 @@ class FuncGraph(ops.Graph):
     else:
       self._collections = collections
 
+  def __str__(self):
+    return "FuncGraph(name=%s, id=%s)" % (self.name, id(self))
+
   def as_default(self):
     outer_cm = super(FuncGraph, self).as_default()
 
@@ -366,6 +369,19 @@ class FuncGraph(ops.Graph):
     if tensor.graph is not self:
       if name is None:
         name = tensor.op.name
+      inner_graph = tensor.graph
+      while inner_graph is not None and isinstance(inner_graph, FuncGraph):
+        if inner_graph is self:
+          raise ValueError(
+              "Trying to capture a tensor from an inner function. This can be "
+              "caused by accessing a tensor defined inside a loop or "
+              "conditional body, or a subfunction, from a calling function, "
+              "without going through the proper return value mechanism. "
+              "Consider using TensorFlow mechanisms such as TensorArrays "
+              "to return tensors from inner functions or loop / conditional "
+              "bodies. Tensor: %s; tensor graph: %s; this graph: %s"
+              % (tensor, tensor.graph, self))
+        inner_graph = inner_graph.outer_graph
       return self._capture_helper(tensor, name)
     return tensor
 
-- 
GitLab


From afef2025d1e35fa276027f2c2f9620a6f555f5bc Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Fri, 8 Feb 2019 14:14:31 -0800
Subject: [PATCH 2195/2345] Create int8 Relu6.

PiperOrigin-RevId: 233124578
---
 tensorflow/lite/kernels/activations.cc        | 37 ++++++++++++-------
 tensorflow/lite/kernels/activations_test.cc   | 24 +++++++++++-
 .../internal/reference/reference_ops.h        | 13 ++++---
 tensorflow/lite/kernels/register.cc           |  3 +-
 tensorflow/lite/toco/tflite/operator.cc       | 17 ++++++++-
 tensorflow/lite/toco/tflite/operator_test.cc  | 19 ++++++++++
 6 files changed, 90 insertions(+), 23 deletions(-)

diff --git a/tensorflow/lite/kernels/activations.cc b/tensorflow/lite/kernels/activations.cc
index b98ae662a6..c0a34957df 100644
--- a/tensorflow/lite/kernels/activations.cc
+++ b/tensorflow/lite/kernels/activations.cc
@@ -361,6 +361,24 @@ TfLiteStatus Relu1Eval(TfLiteContext* context, TfLiteNode* node) {
   }
 }
 
+namespace {
+template <typename T>
+void QuantizedRelu6(const TfLiteTensor* input, TfLiteTensor* output) {
+  ActivationParams params;
+  params.activation_type = FusedActivationFunctionType::kRelu6;
+  params.quantized_activation_min =
+      std::max(static_cast<int32_t>(std::numeric_limits<T>::min()),
+               output->params.zero_point +
+                   static_cast<int32>(roundf(0.f / output->params.scale)));
+  params.quantized_activation_max =
+      std::min(static_cast<int32_t>(std::numeric_limits<T>::max()),
+               output->params.zero_point +
+                   static_cast<int32>(roundf(6.f / output->params.scale)));
+  optimized_ops::ReluX(params, GetTensorShape(input), GetTensorData<T>(input),
+                       GetTensorShape(output), GetTensorData<T>(output));
+}
+}  // namespace
+
 TfLiteStatus Relu6Eval(TfLiteContext* context, TfLiteNode* node) {
   const TfLiteTensor* input = GetInput(context, node, 0);
   TfLiteTensor* output = GetOutput(context, node, 0);
@@ -373,23 +391,16 @@ TfLiteStatus Relu6Eval(TfLiteContext* context, TfLiteNode* node) {
       for (; in < in_end; in++, out++) *out = std::min(std::max(0.f, *in), 6.f);
       return kTfLiteOk;
     } break;
-    case kTfLiteUInt8: {
-      ActivationParams params;
-      params.activation_type = FusedActivationFunctionType::kRelu6;
-      params.quantized_activation_min = std::max(
-          0, output->params.zero_point +
-                 static_cast<int32>(roundf(0.f / output->params.scale)));
-      params.quantized_activation_max = std::min(
-          255, output->params.zero_point +
-                   static_cast<int32>(roundf(6.f / output->params.scale)));
-      optimized_ops::ReluX(params, GetTensorShape(input),
-                           GetTensorData<uint8>(input), GetTensorShape(output),
-                           GetTensorData<uint8>(output));
+    case kTfLiteUInt8:
+      QuantizedRelu6<uint8_t>(input, output);
+      return kTfLiteOk;
+    case kTfLiteInt8: {
+      QuantizedRelu6<int8_t>(input, output);
       return kTfLiteOk;
     } break;
     default:
       context->ReportError(
-          context, "Only float32 and uint8 supported currently, got %s.",
+          context, "Only float32, uint8 and int8 supported currently, got %s.",
           TfLiteTypeGetName(input->type));
       return kTfLiteError;
   }
diff --git a/tensorflow/lite/kernels/activations_test.cc b/tensorflow/lite/kernels/activations_test.cc
index 38903f5acb..e30e62d75a 100644
--- a/tensorflow/lite/kernels/activations_test.cc
+++ b/tensorflow/lite/kernels/activations_test.cc
@@ -174,7 +174,7 @@ TEST(FloatActivationsOpTest, Tanh) {
                              })));
 }
 
-TEST(QuantizedActivationsOpTest, Relu6) {
+TEST(QuantizedActivationsOpTest, Relu6Uint8) {
   const float kMin = -1;
   const float kMax = 127.f / 128.f;
   QuantizedActivationsOpModel m(
@@ -197,6 +197,28 @@ TEST(QuantizedActivationsOpTest, Relu6) {
               ElementsAreArray({128, 128, 160, 192, 176, 128, 224, 144}));
 }
 
+TEST(QuantizedActivationsOpTest, Relu6Int8) {
+  const float kMin = -1;
+  const float kMax = 127.f / 128.f;
+  QuantizedActivationsOpModel m(
+      BuiltinOperator_RELU6,
+      /*input=*/{TensorType_INT8, {1, 2, 4, 1}, 8 * kMin, 8 * kMax},
+      /*output=*/{TensorType_INT8, {1, 2, 4, 1}, 8 * kMin, 8 * kMax});
+  m.SetInput<int8_t>({
+      0, -6, 2, 4,   //
+      3, -2, 10, 1,  //
+  });
+  m.Invoke();
+  EXPECT_THAT(m.GetDequantizedOutput<int8_t>(), ElementsAreArray(ArrayFloatNear(
+                                                    {
+                                                        0, 0, 2, 4,  //
+                                                        3, 0, 6, 1,  //
+                                                    },
+                                                    kQuantizedTolerance)));
+  EXPECT_THAT(m.GetOutput<int8_t>(),
+              ElementsAreArray({0, 0, 32, 64, 48, 0, 96, 16}));
+}
+
 TEST(QuantizedActivationsOpTest, Tanh) {
   const float kMin = -1;
   const float kMax = 127.f / 128.f;
diff --git a/tensorflow/lite/kernels/internal/reference/reference_ops.h b/tensorflow/lite/kernels/internal/reference/reference_ops.h
index 84f62b1c97..229ecc2ab2 100644
--- a/tensorflow/lite/kernels/internal/reference/reference_ops.h
+++ b/tensorflow/lite/kernels/internal/reference/reference_ops.h
@@ -543,16 +543,17 @@ inline void Relu6(const RuntimeShape& input_shape, const float* input_data,
   }
 }
 
+template <typename T>
 inline void ReluX(const tflite::ActivationParams& params,
-                  const RuntimeShape& input_shape, const uint8* input_data,
-                  const RuntimeShape& output_shape, uint8* output_data) {
+                  const RuntimeShape& input_shape, const T* input_data,
+                  const RuntimeShape& output_shape, T* output_data) {
   gemmlowp::ScopedProfilingLabel label("Quantized ReluX (not fused)");
   const int flat_size = MatchingFlatSize(input_shape, output_shape);
-  const uint8 max_value = params.quantized_activation_max;
-  const uint8 min_value = params.quantized_activation_min;
+  const T max_value = params.quantized_activation_max;
+  const T min_value = params.quantized_activation_min;
   for (int i = 0; i < flat_size; ++i) {
-    const uint8 val = input_data[i];
-    const uint8 clamped =
+    const T val = input_data[i];
+    const T clamped =
         val > max_value ? max_value : val < min_value ? min_value : val;
     output_data[i] = clamped;
   }
diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index 970fdba1af..36feba8556 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -164,7 +164,8 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_ABS, Register_ABS());
   AddBuiltin(BuiltinOperator_RELU, Register_RELU());
   AddBuiltin(BuiltinOperator_RELU_N1_TO_1, Register_RELU_N1_TO_1());
-  AddBuiltin(BuiltinOperator_RELU6, Register_RELU6());
+  AddBuiltin(BuiltinOperator_RELU6, Register_RELU6(), /* min_version */ 1,
+             /* max_version */ 2);
   AddBuiltin(BuiltinOperator_TANH, Register_TANH());
   AddBuiltin(BuiltinOperator_LOGISTIC, Register_LOGISTIC(),
              /* min_version */ 1,
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index ad0e4addd0..5f2de4a696 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -1222,6 +1222,20 @@ class ReduceAny
   }
 };
 
+class Relu6 : public SimpleOperator<Relu6Operator> {
+ public:
+  explicit Relu6() : SimpleOperator("RELU6", OperatorType::kRelu6) {}
+  int GetVersion(const OperatorSignature& op_signature) const override {
+    const string& input_name = op_signature.op->inputs[0];
+    const Array& input_array = op_signature.model->GetArray(input_name);
+    // Version 2 supports signed int8 input types.
+    if (input_array.data_type == ArrayDataType::kInt8) {
+      return 2;
+    }
+    return 1;
+  }
+};
+
 class ResizeBilinear
     : public BuiltinOperator<ResizeBilinearOperator,
                              ::tflite::ResizeBilinearOptions,
@@ -2277,8 +2291,7 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
       MakeUnique<SimpleOperator<ReluOperator>>("RELU", OperatorType::kRelu));
   ops.push_back(MakeUnique<SimpleOperator<Relu1Operator>>(
       "RELU_N1_TO_1", OperatorType::kRelu1));
-  ops.push_back(
-      MakeUnique<SimpleOperator<Relu6Operator>>("RELU6", OperatorType::kRelu6));
+  ops.push_back(MakeUnique<Relu6>());
   ops.push_back(
       MakeUnique<SimpleOperator<PReluOperator>>("PRELU", OperatorType::kPRelu));
   ops.push_back(MakeUnique<Logistic>());
diff --git a/tensorflow/lite/toco/tflite/operator_test.cc b/tensorflow/lite/toco/tflite/operator_test.cc
index eaf971419c..eb1c5e0ddf 100644
--- a/tensorflow/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/lite/toco/tflite/operator_test.cc
@@ -568,6 +568,25 @@ TEST_F(OperatorTest, BuiltinOneHot) {
   EXPECT_EQ(op.axis, output_toco_op->axis);
 }
 
+TEST_F(OperatorTest, VersioningRelu6Test) {
+  Relu6Operator relu6_op;
+  relu6_op.inputs = {"input1"};
+  auto operator_by_type_map = BuildOperatorByTypeMap(false /*enable_flex_ops*/);
+  const BaseOperator* op = operator_by_type_map.at(relu6_op.type).get();
+
+  Model uint8_model;
+  Array& uint8_array = uint8_model.GetOrCreateArray(relu6_op.inputs[0]);
+  uint8_array.data_type = ArrayDataType::kUint8;
+  OperatorSignature uint8_signature = {.model = &uint8_model, .op = &relu6_op};
+  EXPECT_EQ(op->GetVersion(uint8_signature), 1);
+
+  Model int8_model;
+  Array& int8_array = int8_model.GetOrCreateArray(relu6_op.inputs[0]);
+  int8_array.data_type = ArrayDataType::kInt8;
+  OperatorSignature int8_signature = {.model = &int8_model, .op = &relu6_op};
+  EXPECT_EQ(op->GetVersion(int8_signature), 2);
+}
+
 TEST_F(OperatorTest, BuiltinUnpack) {
   UnpackOperator op;
   op.num = 5;
-- 
GitLab


From b53b442cf31b8f64193aeaa260f628156c6520b8 Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Fri, 8 Feb 2019 14:18:17 -0800
Subject: [PATCH 2196/2345] Move ColocationGraph and Member out of placer.cc

This is a pure refactoring change. There is no behavior change.
Placer is getting more complex. This split makes the diff of following
CLs cleaner.

PiperOrigin-RevId: 233125235
---
 tensorflow/core/BUILD                         |    2 +
 .../core/common_runtime/colocation_graph.cc   | 1047 ++++++++++++++++
 .../core/common_runtime/colocation_graph.h    |  255 ++++
 tensorflow/core/common_runtime/placer.cc      | 1115 +----------------
 4 files changed, 1305 insertions(+), 1114 deletions(-)
 create mode 100644 tensorflow/core/common_runtime/colocation_graph.cc
 create mode 100644 tensorflow/core/common_runtime/colocation_graph.h

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index e6d2c53b4a..c35b1bc651 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -2992,6 +2992,8 @@ tf_cuda_library(
         "common_runtime/collective_param_resolver_local.cc",
         "common_runtime/collective_rma_local.cc",
         "common_runtime/collective_util.cc",
+        "common_runtime/colocation_graph.cc",
+        "common_runtime/colocation_graph.h",
         "common_runtime/constant_folding.cc",
         "common_runtime/copy_tensor.cc",
         "common_runtime/costmodel_manager.cc",
diff --git a/tensorflow/core/common_runtime/colocation_graph.cc b/tensorflow/core/common_runtime/colocation_graph.cc
new file mode 100644
index 0000000000..2bdc7ec5ac
--- /dev/null
+++ b/tensorflow/core/common_runtime/colocation_graph.cc
@@ -0,0 +1,1047 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/common_runtime/colocation_graph.h"
+
+#include <memory>
+#include <set>
+#include <utility>
+#include <vector>
+
+#include "absl/strings/str_join.h"
+#include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/common_runtime/device_set.h"
+#include "tensorflow/core/framework/attr_value_util.h"
+#include "tensorflow/core/framework/device_attributes.pb.h"
+#include "tensorflow/core/framework/graph.pb.h"
+#include "tensorflow/core/framework/node_def_util.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/framework/types.pb.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/core/stringpiece.h"
+#include "tensorflow/core/lib/strings/str_util.h"
+#include "tensorflow/core/lib/strings/strcat.h"
+#include "tensorflow/core/util/device_name_utils.h"
+#include "tensorflow/core/util/dump_graph.h"
+#include "tensorflow/core/util/port.h"
+
+namespace tensorflow {
+
+namespace {
+
+// We hoist the conversion from C-style string literal to StringPiece here,
+// so that we can avoid the many repeated calls to strlen().
+const StringPiece kColocationAttrNameStringPiece(kColocationAttrName);
+const StringPiece kColocationGroupPrefixStringPiece(kColocationGroupPrefix);
+
+// Returns a list of devices having type in supported_device_types.  The
+// returned list is sorted by preferred type (higher numeric type is preferred).
+std::vector<Device*> FilterSupportedDevices(
+    const std::vector<Device*>& devices,
+    const PrioritizedDeviceTypeVector& supported_device_types,
+    const Device* default_device) {
+  Device* filtered_default_device = nullptr;
+  std::vector<std::pair<Device*, int32>> prioritized_filtered_devices;
+  for (const auto& supported_device_type : supported_device_types) {
+    for (Device* device : devices) {
+      if (DeviceType(device->attributes().device_type()) ==
+          supported_device_type.first) {
+        if (device == default_device) {
+          filtered_default_device = device;
+        } else {
+          prioritized_filtered_devices.emplace_back(
+              device, supported_device_type.second);
+        }
+      }
+    }
+  }
+
+  auto device_sort = [](const std::pair<Device*, int32>& a,
+                        const std::pair<Device*, int32>& b) {
+    if (a.second != b.second) {
+      return a.second > b.second;
+    }
+
+    auto a_priority =
+        DeviceSet::DeviceTypeOrder(DeviceType(a.first->device_type()));
+    auto b_priority =
+        DeviceSet::DeviceTypeOrder(DeviceType(b.first->device_type()));
+    // First sort by prioritized device type (higher is preferred) and
+    // then by device name (lexicographically).
+    if (a_priority != b_priority) {
+      return a_priority > b_priority;
+    }
+    return StringPiece(a.first->name()) < StringPiece(b.first->name());
+  };
+  std::sort(prioritized_filtered_devices.begin(),
+            prioritized_filtered_devices.end(), device_sort);
+
+  std::vector<Device*> filtered_devices;
+  if (filtered_default_device != nullptr) {
+    filtered_devices.emplace_back(filtered_default_device);
+  }
+  for (const auto& prioritized_filtered_device : prioritized_filtered_devices) {
+    filtered_devices.push_back(prioritized_filtered_device.first);
+  }
+  return filtered_devices;
+}
+
+// Using absl::StrJoin with lambda does not work in tf-lite builds.
+std::vector<string> DevicesToString(const std::vector<Device*> devices) {
+  std::vector<string> v;
+  v.reserve(devices.size());
+  for (Device* d : devices) {
+    v.push_back(d->name());
+  }
+  return v;
+}
+
+// Using absl::StrJoin with lambda does not work in tf-lite builds.
+std::vector<string> DeviceTypeAndPriorityToString(
+    const PrioritizedDeviceTypeVector& devices) {
+  std::vector<string> v;
+  v.reserve(devices.size());
+  for (const std::pair<DeviceType, int32>& device_and_type : devices) {
+    v.push_back(DeviceTypeString(device_and_type.first));
+  }
+  return v;
+}
+
+// While Placer can override requested device on ops processing
+// resources, i.e. node that take (and potentially return) a resource,
+// it must not override requested device on ops generating a resource,
+// e.g. VarHandleOp, _Arg. Such ops are currently no-input, single resource/ref
+// output nodes.
+bool IsResourceGeneratorNode(const Node& node) {
+  return node.num_inputs() == 0 && node.num_outputs() == 1 &&
+         (IsRefType(node.output_type(0)) || node.output_type(0) == DT_RESOURCE);
+}
+
+bool IsExemptFromResourceInputColocation(const Node* node) {
+  // Note: Partitioned function calls, which place and partition their
+  // function bodies, are exempt from this check: they forward resource and
+  // ref inputs to operations that are appropriately placed, instead of
+  // dereferencing them.
+  const string& op_type = node->op_def().name();
+  return op_type == "PartitionedCall" || op_type == "StatefulPartitionedCall";
+}
+
+bool HasPriorities(const PrioritizedDeviceTypeVector& device_types) {
+  for (const auto& prioritized_device_type : device_types) {
+    if (prioritized_device_type.second != 0) return true;
+  }
+  return false;
+}
+
+bool ArePrioritiesSame(const PrioritizedDeviceTypeVector& a_types,
+                       const PrioritizedDeviceTypeVector& b_types) {
+  if (a_types.size() != b_types.size()) {
+    return false;
+  }
+  for (int i = 0; i < a_types.size(); ++i) {
+    if (a_types[i].first != b_types[i].first) {
+      return false;
+    }
+  }
+  return true;
+}
+
+}  // namespace
+
+Status Member::SetParentAndSupportedDevices(
+    const Node& node, const std::vector<DeviceType>& types) {
+  int id = node.id();
+  if (id < 0) {
+    return errors::Internal("Placer should not be creating a Member for node: ",
+                            node.DebugString());
+  }
+  parent_ = id;
+  return SupportedDeviceTypesForNode(types, node.def(),
+                                     &supported_device_types_);
+}
+
+Status Member::SetAssignedDeviceName(const string& device_name) {
+  if (DeviceNameUtils::HasSomeDetails(requested_device_name_)) {
+    return errors::Internal(
+        "Setting assigned device name when there is a requested device set "
+        "is unsupported");
+  }
+  if (!DeviceNameUtils::ParseFullName(device_name, &assigned_device_name_)) {
+    return errors::Internal("Malformed assigned device '", device_name, "'");
+  }
+  // Set requested device to assigned_device to maintain the invariant that
+  // requested is a specialization of assigned.
+  requested_device_name_ = assigned_device_name_;
+  return Status::OK();
+}
+
+Status Member::SetRequestedDeviceName(const Node& node) {
+  if (!DeviceNameUtils::ParseFullName(node.requested_device(),
+                                      &requested_device_name_)) {
+    return errors::InvalidArgument("Malformed device specification '",
+                                   node.requested_device(),
+                                   "' in node: ", node.DebugString());
+  }
+  if (DeviceNameUtils::HasSomeDetails(assigned_device_name_)) {
+    return errors::Internal(
+        "Setting requested device name when there is an assigned device set "
+        "is unsupported");
+  }
+  return Status::OK();
+}
+
+Status Member::EnsureCompatibilityAcrossResourceEdge(
+    const Node& src, const Member& src_root,
+    const Node& dst, /*dst_root is this*/
+    bool log_device_placement) {
+  if (!DeviceNameUtils::AreCompatibleDevNames(src_root.assigned_device_name_,
+                                              assigned_device_name_)) {
+    return errors::InvalidArgument(
+        "Cannot place the graph because a reference or resource edge "
+        "connects colocation groups with incompatible assigned devices: ",
+        DeviceNameUtils::ParsedNameToString(src_root.assigned_device_name_),
+        " vs ", DeviceNameUtils::ParsedNameToString(assigned_device_name_));
+  }
+
+  if (DeviceNameUtils::AreCompatibleDevNames(src_root.requested_device_name_,
+                                             requested_device_name_)) {
+    return Status::OK();
+  }
+
+  // If we are here, assigned devices are compatible but requested ones are
+  // not. We will be overriding the requested device for destination node, but
+  // need to preserve the invariant that it will be a specialization of
+  // the assigned device.
+  if (log_device_placement) {
+    LOG(INFO) << "Ignoring device specification "
+              << DeviceNameUtils::ParsedNameToString(requested_device_name_)
+              << " for node '" << dst.name()
+              << "' because the input edge from '" << src.name()
+              << "' is a reference connection and already has a device "
+                 "field set to "
+              << DeviceNameUtils::ParsedNameToString(
+                     src_root.requested_device_name_);
+  }
+  requested_device_name_ = src_root.requested_device_name_;
+  DeviceNameUtils::EnsureSpecification(&requested_device_name_,
+                                       assigned_device_name_);
+  return Status::OK();
+}
+
+void Member::Merge(std::vector<Member>* tree, int x_root, int y_root,
+                   Member** new_root, Member** old_root) {
+  Member& x_root_member = (*tree)[x_root];
+  Member& y_root_member = (*tree)[y_root];
+
+  // Merge the sets by setting the parent pointer of the smaller tree's root
+  // node to point to the root of the larger tree. Together with path
+  // compression in ColocationGraph::FindRoot, this ensures that we do not
+  // experience pathological performance on graphs such as chains.
+  int new_root_id, old_root_id;
+  if (x_root_member.rank_ < y_root_member.rank_) {
+    // The tree rooted at x_root is shallower, so connect it to
+    // y_root. The rank of y_root is unchanged because its new
+    // child has strictly less rank.
+    x_root_member.parent_ = y_root;
+    new_root_id = y_root;
+    old_root_id = x_root;
+  } else if (x_root_member.rank_ > y_root_member.rank_) {
+    // The tree rooted at y_root is shallower, so connect it to
+    // x_root. The rank of x_root is unchanged because its new
+    // child has strictly less rank.
+    y_root_member.parent_ = x_root;
+    new_root_id = x_root;
+    old_root_id = y_root;
+  } else {
+    // Both trees have the same rank, so break the tie by choosing
+    // x_root as the new root.
+    y_root_member.parent_ = x_root;
+    // Increment the rank of the tree rooted at x_root, because it
+    // is now strictly deeper than before.
+    ++x_root_member.rank_;
+    new_root_id = x_root;
+    old_root_id = y_root;
+  }
+
+  *new_root = &(*tree)[new_root_id];
+  *old_root = &(*tree)[old_root_id];
+}
+
+// tree is non-const because we can change some `parent` pointers in some
+// members for more efficient future lookups. The vector itself is not
+// changed.
+int Member::FindRoot(std::vector<Member>* tree, int node_id) {
+  Member& member = (*tree)[node_id];
+  if (member.parent_ == node_id) {
+    // member.parent is the root of this disjoint tree.  Do nothing.
+  } else {
+    member.parent_ = FindRoot(tree, member.parent_);
+  }
+  // Now it is guaranteed that member.parent is the root of this disjoint
+  // tree.
+  return member.parent_;
+}
+
+Status Member::MergeDeviceNames(const Member& other,
+                                bool allow_soft_placement) {
+  // Assuming the "requested is a specialization of assigned" invariant holds
+  // for this and `other`, it will hold after these two merges.
+  TF_RETURN_IF_ERROR(DeviceNameUtils::MergeDevNames(
+      &requested_device_name_, other.requested_device_name_,
+      allow_soft_placement));
+  return DeviceNameUtils::MergeDevNames(&assigned_device_name_,
+                                        other.assigned_device_name_,
+                                        allow_soft_placement);
+}
+
+// Updates this to contain the intersection of the device types in
+// this and "other".
+void Member::MergeSupportedDevices(const Member& other) {
+  PrioritizedDeviceTypeVector temp = supported_device_types_;
+  supported_device_types_.clear();
+
+  // Generate intersection with priorities.
+  PrioritizedDeviceTypeVector target_intersection;
+  PrioritizedDeviceTypeVector other_intersection;
+  for (const auto& prioritized_device_type : temp) {
+    bool found = false;
+    for (const auto& other_prioritized_device_type :
+         other.supported_device_types_) {
+      if (prioritized_device_type.first ==
+          other_prioritized_device_type.first) {
+        found = true;
+        other_intersection.push_back(other_prioritized_device_type);
+        break;
+      }
+    }
+    if (found) {
+      target_intersection.push_back(prioritized_device_type);
+    }
+  }
+
+  // Sort the devices by priority order.
+  auto device_sort = [](const std::pair<DeviceType, int32>& a,
+                        const std::pair<DeviceType, int32>& b) {
+    // First look at set priorities.
+    if (a.second != b.second) {
+      return a.second > b.second;
+    }
+    // Then fallback to default priorities.
+    auto a_priority = DeviceSet::DeviceTypeOrder(a.first);
+    auto b_priority = DeviceSet::DeviceTypeOrder(b.first);
+    if (a_priority != b_priority) {
+      return a_priority > b_priority;
+    }
+    // Finally just look at the Device type strings.
+    return a.first.type_string() < b.first.type_string();
+  };
+
+  std::sort(target_intersection.begin(), target_intersection.end(),
+            device_sort);
+  std::sort(other_intersection.begin(), other_intersection.end(), device_sort);
+
+  bool is_target_prioritized = HasPriorities(target_intersection);
+  bool is_other_prioritized = HasPriorities(other_intersection);
+  // If neither are prioritized then we just return the original i.e. target
+  // prioritization.
+  if (!is_target_prioritized && !is_other_prioritized) {
+    supported_device_types_ = target_intersection;
+  }
+  // If only one is prioritized, then we respect priorities of that in the
+  // intersection.
+  if (is_target_prioritized && !is_other_prioritized) {
+    supported_device_types_ = target_intersection;
+  }
+  if (!is_target_prioritized && is_other_prioritized) {
+    supported_device_types_ = other_intersection;
+  }
+  // If both have priorities and agree then we go with that. If the
+  // prioritization order is different, then we just fallback to the default
+  // i.e. what the DeviceTypeOrder suggests. In that case, we also set the
+  // merged priorities to 0, so that downstream merges work correctly as well.
+  if (is_target_prioritized && is_other_prioritized) {
+    bool priorities_agree =
+        ArePrioritiesSame(target_intersection, other_intersection);
+    if (priorities_agree) {
+      supported_device_types_ = target_intersection;
+    } else {
+      for (const auto& prioritized_device : target_intersection) {
+        supported_device_types_.push_back(
+            std::make_pair(prioritized_device.first, 0));
+      }
+      std::sort(supported_device_types_.begin(), supported_device_types_.end(),
+                device_sort);
+    }
+  }
+}
+
+Status Member::AssignDevice(const Node& node, bool allow_soft_placement) {
+  if (node.assigned_device_name_index() == assigned_device_name_index_) {
+    return Status::OK();
+  }
+
+  DeviceNameUtils::ParsedName parsed;
+  DeviceNameUtils::ParseFullName(node.assigned_device_name(), &parsed);
+  Status s = DeviceNameUtils::MergeDevNames(&assigned_device_name_, parsed,
+                                            allow_soft_placement);
+  if (!s.ok()) {
+    return errors::Internal(
+        "Constraining by assigned device should not cause an error. Original "
+        "root's assigned device name: ",
+        DeviceNameUtils::ParsedNameToString(assigned_device_name_),
+        " node's assigned device name \"", node.assigned_device_name(),
+        ". Error: ", s.error_message());
+  }
+  s = DeviceNameUtils::MergeDevNames(&requested_device_name_, parsed,
+                                     allow_soft_placement);
+  if (!s.ok()) {
+    return errors::Internal(
+        "Constraining by assigned device should not cause an error. Original "
+        "root's requested device name: \"",
+        DeviceNameUtils::ParsedNameToString(requested_device_name_),
+        "\", node's assigned device name \"", node.assigned_device_name(),
+        "\". Error: ", s.error_message());
+  }
+
+  assigned_device_name_index_ = node.assigned_device_name_index();
+  // Clear cached possible_devices, if any.
+  possible_devices_.clear();
+  return Status::OK();
+}
+string Member::DebugString() {
+  return absl::StrCat(
+      "Member(assigned_device_name_index_=", assigned_device_name_index_,
+      " requested_device_name_=",
+      DeviceNameUtils::ParsedNameToString(requested_device_name_),
+      " assigned_device_name_=",
+      DeviceNameUtils::ParsedNameToString(assigned_device_name_),
+      " supported_device_types_=[",
+      absl::StrJoin(DeviceTypeAndPriorityToString(supported_device_types_),
+                    ", "),
+      "] possible_devices_=[",
+      absl::StrJoin(DevicesToString(possible_devices_), ", "), "]");
+}
+ColocationGraph::ColocationGraph(const Graph* graph,
+                                 const DeviceSet* device_set,
+                                 const Device* default_device,
+                                 bool allow_soft_placement,
+                                 bool log_device_placement)
+    : graph_(graph),
+      device_set_(device_set),
+      device_types_(device_set->PrioritizedDeviceTypeList()),
+      default_device_(default_device),
+      allow_soft_placement_(allow_soft_placement),
+      log_device_placement_(log_device_placement) {
+  members_.resize(graph->num_node_ids());
+}
+
+// Adds each node of the Graph to this ColocationGraph as a singleton.
+//
+// NOTE: The implementation assumes that the ids of nodes passed to
+// this method are dense and zero-based; the memory used will be linear in
+// the largest node ID.
+// NOTE: If this method returns an error, *this is left in an undefined
+// state.
+Status ColocationGraph::ColocateAllNodes() {
+  // This maps from a colocation group identifier to the 'root' of that
+  // colocation group.  Note that the keys in this map are StringPiece; the
+  // actual strings are stored under the NodeDef.  The lifetime of this map
+  // is limited to this ColocateAllNodes() method, and no part of the
+  // NodeDef trees are changed during the lifetime of this method, so using
+  // StringPiece as a key is safe.
+  //
+  // Also, as a further optimization, we remove the "loc:@" prefix from
+  // "class" attribute values, when they are used as keys in this table.
+  // This allows us to use StringPiece values that refer to substrings of
+  // 'string' values stored in NodeDef attribute lists, as well as StringPiece
+  // values that refer to 'string' values from NodeDef::name(), without
+  // performing any string allocations.
+  std::unordered_map<StringPiece, const Node*, StringPieceHasher>
+      colocation_group_root;
+
+  for (const Node* node : graph_->op_nodes()) {
+    // When adding the node, identify whether it is part of a colocation
+    // group.
+
+    // This code is effectively the equivalent of GetNodeAttr() for a string
+    // array, but it avoids all internal allocations (the allocation of the
+    // backing store of the std::vector<string> as well as the copies of the
+    // strings within it).  Instead, we combine the query of the colocation
+    // attribute with the calls to ColocateNodeToGroup.
+    bool found_spec = false;
+    const AttrValue* attr_value =
+        node->attrs().Find(kColocationAttrNameStringPiece);
+    if (attr_value != nullptr && attr_value->has_list()) {
+      for (const string& class_spec : attr_value->list().s()) {
+        StringPiece spec(class_spec);
+        if (str_util::ConsumePrefix(&spec, kColocationGroupPrefixStringPiece)) {
+          found_spec = true;
+          TF_RETURN_IF_ERROR(
+              ColocateNodeToGroup(&colocation_group_root, node, spec));
+        }
+      }
+    }
+
+    if (!found_spec) {
+      // If the node does not specify a colocation group, then use the
+      // name of this node as the colocation group.
+      TF_RETURN_IF_ERROR(
+          ColocateNodeToGroup(&colocation_group_root, node, node->name()));
+    }
+  }
+
+  return Status::OK();
+}
+
+Status ColocationGraph::ColocateResourceOrRefEdge(Node* src, Node* dst) {
+  // Colocate `src` and `dst` to maintain the invariant that nodes
+  // connected by reference edges are colocated.
+  int src_root_id = FindRoot(src->id());
+  int dst_root_id = FindRoot(dst->id());
+  auto& src_root = members_[src_root_id];
+  auto& dst_root = members_[dst_root_id];
+
+  TF_RETURN_IF_ERROR(dst_root.EnsureCompatibilityAcrossResourceEdge(
+      *src, src_root, *dst, log_device_placement_));
+  Status status = ColocateNodes(*src, src_root_id, *dst, dst_root_id);
+  if (!status.ok()) {
+    return AttachDef(
+        errors::InvalidArgument("Nodes were connected by a "
+                                "reference connection (requiring them to "
+                                "be on the same device), but the two nodes "
+                                "were assigned two different devices: ",
+                                status.error_message()),
+        *dst);
+  }
+  return Status::OK();
+}
+
+Status ColocationGraph::ColocateResourceAndRefEdges() {
+  // Enumerate the constraint edges, and use them to update the disjoint
+  // node set.
+  // If `node` has an input edge with reference type, add an edge from the
+  // source of that edge to `node`.
+  for (const Edge* edge : graph_->edges()) {
+    if (edge->IsControlEdge()) {
+      continue;
+    }
+    Node* src = edge->src();
+    Node* dst = edge->dst();
+    DataType input_type = dst->input_type(edge->dst_input());
+    if ((input_type == DT_RESOURCE || IsRefType(input_type)) &&
+        !IsExemptFromResourceInputColocation(dst)) {
+      TF_RETURN_IF_ERROR(ColocateResourceOrRefEdge(src, dst));
+    }
+  }
+  return Status::OK();
+}
+
+Status ColocationGraph::Initialize() {
+  TF_RETURN_IF_ERROR(InitializeMembers());
+  TF_RETURN_IF_ERROR(ColocateAllNodes());
+  return ColocateResourceAndRefEdges();
+}
+
+Status ColocationGraph::ColocateNodeToGroup(
+    std::unordered_map<StringPiece, const Node*, StringPieceHasher>*
+        colocation_group_root,
+    const Node* node, StringPiece colocation_group) {
+  const Node*& root_node = (*colocation_group_root)[colocation_group];
+  if (root_node == nullptr) {
+    // This is the first node of the colocation group, so
+    // designate this node as the 'root' of that colocation group.
+    root_node = node;
+  } else {
+    // Try to colocate the node with the root.  If there is an
+    // error, return it.
+    Status s = ColocateNodes(*node, *root_node);
+    if (!s.ok()) {
+      return AttachDef(s, *node);
+    }
+  }
+  return Status::OK();
+}
+
+// Merge the (possibly disjoint) sets containing nodes "x" and
+// "y". Returns OK if the all nodes in the union of these sets can
+// be placed on the same device type.
+//
+// NOTE: If this method returns an error, *this is left in an undefined
+// state.
+Status ColocationGraph::ColocateNodes(const Node& x, const Node& y) {
+  int x_root = FindRoot(x.id());
+  int y_root = FindRoot(y.id());
+  return ColocateNodes(x, x_root, y, y_root);
+}
+
+// This overload of ColocateNodes() allows a caller to provide the root node
+// ids for the two nodes. For large graphs, this noticeably reduces the
+// graph load time.
+Status ColocationGraph::ColocateNodes(const Node& x, int x_root, const Node& y,
+                                      int y_root) {
+  if (x_root == y_root) {
+    return Status::OK();
+  }
+
+  DCHECK_EQ(x_root, FindRoot(x.id()));
+  DCHECK_EQ(y_root, FindRoot(y.id()));
+
+  Member* new_root_member;
+  Member* old_root_member;
+  Member::Merge(&members_, x_root, y_root, &new_root_member, &old_root_member);
+
+  // Merge the partial device specifications, and ensure that they are
+  // compatible. NULL options_ is treated as allowing soft placement.
+  // TODO(mrry): Consider enriching the error message by pointing
+  // out which nodes have the explicit partial device
+  // specifications that caused this conflict.
+  Status s = new_root_member->MergeDeviceNames(*old_root_member,
+                                               allow_soft_placement_);
+  if (!s.ok()) {
+    return errors::InvalidArgument(
+        "Cannot colocate nodes ",
+        errors::FormatColocationNodeForError(x.name()), " and ",
+        errors::FormatColocationNodeForError(y.name()), ": ",
+        s.error_message());
+  }
+
+  // Ensure that the common root has at least one supported device
+  // type, by computing the intersection of
+  // new_root_member.supported_device_types and
+  // old_root_member.supported_device_types.
+  new_root_member->MergeSupportedDevices(*old_root_member);
+  if (new_root_member->supported_device_types().empty()) {
+    return errors::InvalidArgument(
+        "Cannot colocate nodes ",
+        errors::FormatColocationNodeForError(x.name()), " and ",
+        errors::FormatColocationNodeForError(y.name()),
+        " because no device type supports both of those nodes and the "
+        "other nodes colocated with them.",
+        DebugInfo(x_root), DebugInfo(y_root));
+  }
+
+  return Status::OK();
+}
+
+// Limits the possible devices of `node`'s colocation group to the device
+// to which `node` is assigned. This makes sure that all nodes in this
+// colocation group will be assigned to the same device. Without this
+// explicit restriction, heuristics can choose a different possible device
+// for other nodes in the group.
+Status ColocationGraph::LimitToAssignedDevice(const Node& node) {
+  if (node.assigned_device_name_index() < 0) {
+    return errors::Internal(
+        "Expected an assigned node as argument to LimitToAssignedDevice but "
+        "got: ",
+        node.DebugString());
+  }
+  int root = FindRoot(node.id());
+  Member& root_member = members_[root];
+  return root_member.AssignDevice(node, allow_soft_placement_);
+}
+
+// For the given node, subject to the constraints previously given
+// to this ColocationGraph, set its assigned_device_name. Returns OK
+// if a satisfying device can be found, otherwise an error.
+//
+// Note: This method returns a pointer to a field within members_.
+// The caller must not use the returned pointer after there is any possibility
+// that the members_[i].possible_devices field has been modified.
+Status ColocationGraph::GetDevicesForNode(
+    Node* node, const std::vector<Device*>** possible_devices) {
+  *possible_devices = nullptr;
+  const int node_root = FindRoot(node->id());
+  if (!members_[node_root].possible_devices().empty()) {
+    *possible_devices = &members_[node_root].possible_devices();
+    return Status::OK();
+  }
+
+  // We have not yet computed the possible devices for the
+  // colocated node set containing 'node', so we do so now using the
+  // constraints on the root node.
+
+  // "devices" will contain the set of feasible placements for the
+  // colocated node set containing 'node'.
+  std::vector<Device*> devices;
+  if (DeviceNameUtils::HasSomeDetails(
+          members_[node_root].requested_device_name())) {
+    // The root node has a (possibly partial) device
+    // specification, so enumerate the physical devices that
+    // conform to it.
+    device_set_->FindMatchingDevices(
+        members_[node_root].requested_device_name(), &devices);
+
+    if (!devices.empty()) {
+      // Filter devices into those that are compatible with the root
+      // node (and its children).
+      devices = FilterSupportedDevices(
+          devices, members_[node_root].supported_device_types(),
+          default_device_);
+    }
+
+    // Perform soft placement if allow_soft_placement_ is set.
+    if (devices.empty() && allow_soft_placement_) {
+      // The soft_device_name is the same as the node's device name
+      // without specifying the device type or ID.
+      DeviceNameUtils::ParsedName soft_device_name =
+          members_[node_root].requested_device_name();
+      soft_device_name.type.clear();
+      soft_device_name.has_type = false;
+      soft_device_name.has_id = false;
+      device_set_->FindMatchingDevices(soft_device_name, &devices);
+      if (!devices.empty()) {
+        devices = FilterSupportedDevices(
+            devices, members_[node_root].supported_device_types(),
+            default_device_);
+      }
+    }
+
+    if (devices.empty()) {
+      // Return an error when a physical device that matches an explicit
+      // device specification is not found. This ensures that we don't
+      // assign a node to GPU when the user wanted to force it on CPU.
+      string debug_info = DebugInfo(node_root);
+
+      DeviceNameUtils::ParsedName specified_device_name;
+      if (DeviceNameUtils::ParseFullName(node->requested_device(),
+                                         &specified_device_name) &&
+          specified_device_name ==
+              members_[node_root].requested_device_name()) {
+        // The specified device and merged set device match, and
+        // will appear in the GraphDef (for debugging), so just
+        // print the specified device.
+        std::vector<Device*> devices_matching_nodedef;
+        device_set_->FindMatchingDevices(specified_device_name,
+                                         &devices_matching_nodedef);
+        if (devices_matching_nodedef.empty()) {
+          // Sometimes it is almost impossible to understand the problem
+          // without a list of available devices.
+          std::vector<string> device_names;
+          for (const Device* device : device_set_->devices()) {
+            device_names.push_back(device->name());
+          }
+          std::sort(device_names.begin(), device_names.end());
+
+          string gpu_msg = "";
+          if (!IsGoogleCudaEnabled() &&
+              str_util::Lowercase(specified_device_name.type) == "gpu") {
+            gpu_msg =
+                " The requested device appears to be a GPU, but CUDA is not "
+                "enabled.";
+          }
+
+          return errors::InvalidArgument(
+              errors::FormatNodeNameForError(node->name()),
+              "was explicitly assigned to ", node->requested_device(),
+              " but available devices are [ ",
+              str_util::Join(device_names, ", "), " ]. Make sure ",
+              "the device specification refers to a valid device.", gpu_msg);
+        } else if (specified_device_name.has_type) {
+          return errors::InvalidArgument(
+              "Could not satisfy explicit device specification '",
+              node->requested_device(), "' because no supported kernel for ",
+              specified_device_name.type, " devices is available.", debug_info,
+              "\nRegistered kernels:\n",
+              KernelsRegisteredForOp(node->type_string()));
+        } else {
+          return errors::InvalidArgument(
+              "Could not satisfy explicit device specification '",
+              node->requested_device(), debug_info);
+        }
+      } else {
+        // The specified device may be a valid device but the
+        // merged set device is different, so print both.
+        return errors::InvalidArgument(
+            "Could not satisfy explicit device specification '",
+            node->requested_device(), "' because the node ",
+            errors::FormatColocationNodeForError(node->name()),
+            " was colocated with a group of nodes that ",
+            "required incompatible device '",
+            DeviceNameUtils::ParsedNameToString(
+                members_[node_root].requested_device_name()),
+            "'", debug_info);
+      }
+    }
+  } else {
+    // The device is completely unspecified, so enumerate the devices that
+    // support all of the nodes in the set.
+    if (device_set_->devices().empty()) {
+      return errors::Internal("No devices are registered");
+    }
+    devices = FilterSupportedDevices(
+        device_set_->devices(), members_[node_root].supported_device_types(),
+        default_device_);
+
+    if (devices.empty()) {
+      return errors::InvalidArgument(
+          "Node had no OpKernel registered to support this operation: ",
+          "Operation was ", node->type_string(), " and inputs were ",
+          DataTypeVectorString(node->input_types()), DebugInfo(node_root));
+    }
+  }
+
+  // Cache the result of the possible devices for this node group.
+  members_[node_root].set_possible_devices(std::move(devices));
+  *possible_devices = &members_[node_root].possible_devices();
+  return Status::OK();
+}
+
+Status ColocationGraph::InitializeMembers() {
+  for (Node* node : graph_->op_nodes()) {
+    Status status = InitializeMember(*node, &members_[node->id()]);
+    if (!status.ok()) {
+      return AttachDef(status, *node);
+    }
+  }
+  return Status::OK();
+}
+
+string ColocationGraph::DebugString() {
+  std::unordered_set<int> roots;
+  std::vector<string> root_strings;
+  for (const Node* node : graph_->nodes()) {
+    if (!node->IsOp()) {
+      continue;
+    }
+    int node_root = FindRoot(node->id());
+    if (roots.count(node_root) == 0) {
+      root_strings.push_back(DebugInfo(node_root));
+      roots.insert(node_root);
+    }
+  }
+  return absl::StrJoin(root_strings, "\n");
+}
+
+// Returns debugging info for the node referred to by 'node_root'.
+string ColocationGraph::DebugInfo(const int node_root) {
+  string text(
+      "\nColocation Debug Info:\n"
+      "Colocation group had the following types and devices: ");
+
+  // If this node is part of a colocation group, then we want to
+  // collect the mapping of ops to supported devices, so that
+  // the user can see why an unsatisfiable placement occurred.
+
+  std::unordered_map<string, string> type_to_devices;
+  std::vector<const Node*> colocation_nodes;
+  int num_nodes_found = 0;
+
+  for (const Node* node : graph_->nodes()) {
+    if (!node->IsOp()) {
+      continue;
+    }
+    int id = node->id();
+    if (FindRoot(id) != node_root) {
+      continue;
+    }
+    ++num_nodes_found;
+    colocation_nodes.push_back(node);
+    const string& op_type = node->type_string();
+    string devices_registered;
+    for (const auto& device_type : members_[id].supported_device_types()) {
+      strings::StrAppend(&devices_registered,
+                         DeviceTypeString(device_type.first), " ");
+    }
+
+    type_to_devices[op_type] = std::move(devices_registered);
+  }
+
+  for (const auto& td : type_to_devices) {
+    strings::StrAppend(&text, "\n", td.first, ": ", td.second);
+  }
+  strings::StrAppend(&text,
+                     "\n\nColocation members and user-requested devices:");
+  for (const Node* node : colocation_nodes) {
+    strings::StrAppend(&text, "\n  ", node->name(), " (", node->type_string(),
+                       ") ", node->requested_device());
+  }
+  strings::StrAppend(&text, "\n");
+
+  if (num_nodes_found <= 0) {
+    text.clear();
+  }
+  return text;
+}
+
+Status ColocationGraph::InitializeMemberWithAssignedDevice(
+    const string& assigned_device_name, const string& node_type,
+    bool must_be_full_name, Member* member) {
+  // This node has already been assigned to a device, so we
+  // respect this placement, after sanity-checking it.
+  // NOTE: Since any assignment must have been performed by
+  // the TensorFlow runtime, we consider errors in this branch to
+  // be INTERNAL.
+  TF_RETURN_IF_ERROR(member->SetAssignedDeviceName(assigned_device_name));
+  if (!must_be_full_name) {
+    return Status::OK();
+  }
+  // Since assigned device must be a full specification, do extra checks.
+  const Device* assigned_device =
+      device_set_->FindDeviceByName(assigned_device_name);
+  if (assigned_device == nullptr) {
+    return errors::Internal("Assigned device '", assigned_device_name,
+                            "' does not match any device");
+  }
+
+  for (const auto& d : member->supported_device_types()) {
+    if (DeviceType(assigned_device->attributes().device_type()) == d.first) {
+      return Status::OK();
+    }
+  }
+
+  return errors::Internal("Assigned device '", assigned_device_name,
+                          "' does not have registered OpKernel support "
+                          "for ",
+                          node_type);
+}
+
+Status ColocationGraph::InitializeMember(const Node& node, Member* member) {
+  TF_RETURN_IF_ERROR(member->SetParentAndSupportedDevices(node, device_types_));
+
+  if (node.has_assigned_device_name()) {
+    TF_RETURN_IF_ERROR(InitializeMemberWithAssignedDevice(
+        node.assigned_device_name(), node.type_string(), true, member));
+  } else {
+    // This node has not yet been assigned to a device, so we
+    // calculate any constraints due to the set of registered
+    // kernels and any (partial) user-provided device specification
+    // in the NodeDef.
+
+    // If no kernels are registered for this op type, fail with an error.
+    if (member->supported_device_types().empty()) {
+      std::set<string> registered_device_types;
+      for (Device* d : device_set_->devices()) {
+        registered_device_types.insert(d->device_type());
+      }
+      std::vector<string> attr_key_vals;
+      for (const auto& it : node.attrs()) {
+        const string& name = it.first;
+        const AttrValue& attr_value = it.second;
+        attr_key_vals.push_back(
+            strings::StrCat(name, "=", SummarizeAttrValue(attr_value)));
+      }
+      return errors::InvalidArgument(
+          "No OpKernel was registered to support Op '", node.type_string(),
+          "' used by ", errors::FormatNodeNameForError(node.name()),
+          "with these attrs: [", str_util::Join(attr_key_vals, ", "),
+          "]\n"
+          "Registered devices: [",
+          str_util::Join(registered_device_types, ", "), "]\n",
+          "Registered kernels:\n", KernelsRegisteredForOp(node.type_string()));
+    }
+
+    // If the NodeDef contains a device, then we interpret it as a
+    // (partial) device specification.
+    if (!node.requested_device().empty()) {
+      if (IsResourceGeneratorNode(node)) {
+        // Treat requested device on resource generating nodes as assigned
+        // device so that we don't override it.
+        TF_RETURN_IF_ERROR(InitializeMemberWithAssignedDevice(
+            node.requested_device(), node.type_string(), false, member));
+      } else {
+        // The user has specified a device in the NodeDef, try to find a
+        // valid device matching their specification in the set of
+        // devices.
+        // NOTE: The full name may specify a device that is not in
+        // n.supported_device_types(), but we check that in AssignDevice().
+        TF_RETURN_IF_ERROR(member->SetRequestedDeviceName(node));
+      }
+    }
+  }
+  return Status::OK();
+}
+
+// Updates target to contain the intersection of the device types in
+// "target" and "other".
+void ColocationGraph::MergeSupportedDevices(
+    PrioritizedDeviceTypeVector* target,
+    const PrioritizedDeviceTypeVector& other) {
+  PrioritizedDeviceTypeVector temp = *target;
+  target->clear();
+
+  // Generate intersection with priorities.
+  PrioritizedDeviceTypeVector target_intersection;
+  PrioritizedDeviceTypeVector other_intersection;
+  for (const auto& prioritized_device_type : temp) {
+    bool found = false;
+    for (const auto& other_prioritized_device_type : other) {
+      if (prioritized_device_type.first ==
+          other_prioritized_device_type.first) {
+        found = true;
+        other_intersection.push_back(other_prioritized_device_type);
+        break;
+      }
+    }
+    if (found) {
+      target_intersection.push_back(prioritized_device_type);
+    }
+  }
+
+  // Sort the devices by priority order.
+  auto device_sort = [](const std::pair<DeviceType, int32>& a,
+                        const std::pair<DeviceType, int32>& b) {
+    // First look at set priorities.
+    if (a.second != b.second) {
+      return a.second > b.second;
+    }
+    // Then fallback to default priorities.
+    auto a_priority = DeviceSet::DeviceTypeOrder(a.first);
+    auto b_priority = DeviceSet::DeviceTypeOrder(b.first);
+    if (a_priority != b_priority) {
+      return a_priority > b_priority;
+    }
+    // Finally just look at the Device type strings.
+    return a.first.type_string() < b.first.type_string();
+  };
+
+  std::sort(target_intersection.begin(), target_intersection.end(),
+            device_sort);
+  std::sort(other_intersection.begin(), other_intersection.end(), device_sort);
+
+  bool is_target_prioritized = HasPriorities(target_intersection);
+  bool is_other_prioritized = HasPriorities(other_intersection);
+  // If neither are prioritized then we just return the original i.e. target
+  // prioritization.
+  if (!is_target_prioritized && !is_other_prioritized) {
+    *target = target_intersection;
+  }
+  // If only one is prioritized, then we respect priorities of that in the
+  // intersection.
+  if (is_target_prioritized && !is_other_prioritized) {
+    *target = target_intersection;
+  }
+  if (!is_target_prioritized && is_other_prioritized) {
+    *target = other_intersection;
+  }
+  // If both have priorities and agree then we go with that. If the
+  // prioritization order is different, then we just fallback to the default
+  // i.e. what the DeviceTypeOrder suggests. In that case, we also set the
+  // merged priorities to 0, so that downstream merges work correctly as well.
+  if (is_target_prioritized && is_other_prioritized) {
+    bool priorities_agree =
+        ArePrioritiesSame(target_intersection, other_intersection);
+    if (priorities_agree) {
+      *target = target_intersection;
+    } else {
+      for (const auto& prioritized_device : target_intersection) {
+        target->push_back(std::make_pair(prioritized_device.first, 0));
+      }
+      std::sort(target->begin(), target->end(), device_sort);
+    }
+  }
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/colocation_graph.h b/tensorflow/core/common_runtime/colocation_graph.h
new file mode 100644
index 0000000000..8ad54c3232
--- /dev/null
+++ b/tensorflow/core/common_runtime/colocation_graph.h
@@ -0,0 +1,255 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_COLOCATION_GRAPH_H_
+#define TENSORFLOW_CORE_COMMON_RUNTIME_COLOCATION_GRAPH_H_
+
+#include <unordered_map>
+#include <vector>
+
+#include "absl/strings/str_join.h"
+#include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/lib/core/stringpiece.h"
+#include "tensorflow/core/util/device_name_utils.h"
+#include "tensorflow/core/util/port.h"
+
+namespace tensorflow {
+
+// Represents a node in the disjoint node forest and the
+// accumulated constraints on the device used by that node.
+class Member {
+ public:
+  Member() = default;
+
+  Status SetParentAndSupportedDevices(const Node& node,
+                                      const std::vector<DeviceType>& types);
+
+  const DeviceNameUtils::ParsedName& requested_device_name() const {
+    return requested_device_name_;
+  }
+
+  Status SetAssignedDeviceName(const string& device_name);
+
+  Status SetRequestedDeviceName(const Node& node);
+
+  Status EnsureCompatibilityAcrossResourceEdge(
+      const Node& src, const Member& src_root,
+      const Node& dst, /*dst_root is this*/
+      bool log_device_placement);
+
+  const PrioritizedDeviceTypeVector& supported_device_types() const {
+    return supported_device_types_;
+  }
+
+  static void Merge(std::vector<Member>* tree, int x_root, int y_root,
+                    Member** new_root, Member** old_root);
+
+  // tree is non-const because we can change some `parent` pointers in some
+  // members for more efficient future lookups. The vector itself is not
+  // changed.
+  static int FindRoot(std::vector<Member>* tree, int node_id);
+
+  Status MergeDeviceNames(const Member& other, bool allow_soft_placement);
+
+  // Updates this to contain the intersection of the device types in
+  // this and "other".
+  void MergeSupportedDevices(const Member& other);
+
+  Status AssignDevice(const Node& node, bool allow_soft_placement);
+
+  void set_possible_devices(std::vector<Device*>&& devices) {
+    possible_devices_ = devices;
+  }
+  const std::vector<Device*>& possible_devices() { return possible_devices_; }
+
+  string DebugString();
+
+ private:
+  // The id of the node that is the parent of this one, or its own
+  // id if it is a root. parent <= 0 indicates that this member is invalid.
+  int parent_ = -1;
+
+  // A proxy for the depth of the tree that is used to prefer
+  // connecting smaller trees to larger trees when merging disjoint
+  // sets.
+  int rank_ = 0;
+
+  // Once colocation groups have been formed, the Placer starts actually
+  // choosing devices. All nodes in a group must be assigned to the same
+  // device. Once we assigned the first device to some node in this group,
+  // we set assigned_device_name_index to this device name's index in the
+  // graph.
+  // The `*_device_name_` fields will contain the parsed name of this device
+  // and `possible_devices`, if computed, will contain just this device.
+  // `assigned_device_name_index` is an optimization to avoid parsing and
+  // comparing device names. The value of -1 signals that a single device
+  // has not been chosen yet.
+  int assigned_device_name_index_ = -1;
+
+  // The merged form of the device requested for this node, with those of all of
+  // its children. requested_device_name_ is always kept a specialization (i.e.
+  // DeviceNameUtils::IsSpecialization) of assigned_device_name_. When no device
+  // is requested, this field is set to assigned_device_name_.  As a
+  // specialization of assigned_device_name_, requested_device_name_ represents
+  // the most specific form of all assigned and requested devices of this node
+  // and its children, if this node is a root. requested_device_name_ is used
+  // to finally select devices for nodes.  We can override requested devices due
+  // to resource colocation constraints but not assigned devices (unless soft
+  // placement is on).
+  DeviceNameUtils::ParsedName requested_device_name_;
+
+  // The merged form of the device assigned for this node, with
+  // those of all of its children.
+  // This field is used to raise errors due to unsatisfiable constraints.
+  // Can be a partial specification.
+  // INVARIANT: requested_device_name_ is always a
+  // DeviceNameUtils::IsSpecialization of assigned_device_name_.
+  DeviceNameUtils::ParsedName assigned_device_name_;
+
+  // The intersection of all device types supported by this node,
+  // and those of all of its children, in priority order
+  // of the preferred device.
+  PrioritizedDeviceTypeVector supported_device_types_;
+
+  // If this node is a root, stores a list of Devices to which this node
+  // and all of its children have been assigned, or nullptr if this
+  // has not yet been computed.
+  std::vector<Device*> possible_devices_;
+};  // namespace
+
+// This class maintains the connected components of a colocation
+// constraint graph, and uses this information to assign a satisfying
+// device placement to the nodes of the graph.
+//
+// The typical usage pattern is:
+//
+//   Graph graph = ...;
+//   DeviceSet device_set = ...;
+//   ColocationGraph colocation_graph(graph, device_set);
+//
+//   // Add all the nodes of the `graph` to the `colocation_graph`.
+//   for (Node* node : graph.nodes()) {
+//     TF_RETURN_IF_ERROR(colocation_graph.AddNode(*node));
+//   }
+//
+//   // Add one or more colocation constraints.
+//   Node node_1 = *graph.FindNodeId(...);
+//   Node node_2 = *graph.FindNodeId(...);
+//   TF_RETURN_IF_ERROR(colocation_graph.ColocateNodes(node_1, node_2));
+//
+//   // Assign devices based on the accumulated constraints.
+//   for (Node* node : graph.nodes()) {
+//     TF_RETURN_IF_ERROR(colocation_graph.AssignDevice(node));
+//   }
+//
+// This implementation uses the Union-Find algorithm to efficiently maintain the
+// connected components and incrementally adds edges via
+// ColocationGraph::ColocateNodes() invocations.
+//
+// ColocationGraph does not assign any devices to graph nodes. The
+// `log_device_placement` argument is used to log messages when requested
+// device is ignored.
+class ColocationGraph {
+ public:
+  ColocationGraph(const Graph* graph, const DeviceSet* device_set,
+                  const Device* default_device, bool allow_soft_placement,
+                  bool log_device_placement);
+
+  // Adds each node of the Graph to this ColocationGraph as a singleton.
+  //
+  // NOTE: The implementation assumes that the ids of nodes passed to
+  // this method are dense and zero-based; the memory used will be linear in
+  // the largest node ID.
+  // NOTE: If this method returns an error, *this is left in an undefined
+  // state.
+  Status ColocateAllNodes();
+
+  Status ColocateResourceOrRefEdge(Node* src, Node* dst);
+
+  Status ColocateResourceAndRefEdges();
+
+  Status Initialize();
+
+  Status ColocateNodeToGroup(
+      std::unordered_map<StringPiece, const Node*, StringPieceHasher>*
+          colocation_group_root,
+      const Node* node, StringPiece colocation_group);
+
+  // Merge the (possibly disjoint) sets containing nodes "x" and
+  // "y". Returns OK if the all nodes in the union of these sets can
+  // be placed on the same device type.
+  //
+  // NOTE: If this method returns an error, *this is left in an undefined
+  // state.
+  Status ColocateNodes(const Node& x, const Node& y);
+
+  // This overload of ColocateNodes() allows a caller to provide the root node
+  // ids for the two nodes. For large graphs, this noticeably reduces the
+  // graph load time.
+  Status ColocateNodes(const Node& x, int x_root, const Node& y, int y_root);
+
+  // Limits the possible devices of `node`'s colocation group to the device
+  // to which `node` is assigned. This makes sure that all nodes in this
+  // colocation group will be assigned to the same device. Without this
+  // explicit restriction, heuristics can choose a different possible device
+  // for other nodes in the group.
+  Status LimitToAssignedDevice(const Node& node);
+
+  // For the given node, subject to the constraints previously given
+  // to this ColocationGraph, set its assigned_device_name. Returns OK
+  // if a satisfying device can be found, otherwise an error.
+  //
+  // Note: This method returns a pointer to a field within members_.
+  // The caller must not use the returned pointer after there is any possibility
+  // that the members_[i].possible_devices field has been modified.
+  Status GetDevicesForNode(Node* node,
+                           const std::vector<Device*>** possible_devices);
+
+  Status InitializeMembers();
+
+  string DebugString();
+
+  // Returns debugging info for the node referred to by 'node_root'.
+  string DebugInfo(const int node_root);
+
+  Status InitializeMemberWithAssignedDevice(const string& assigned_device_name,
+                                            const string& node_type,
+                                            bool must_be_full_name,
+                                            Member* member);
+
+  Status InitializeMember(const Node& node, Member* member);
+
+  // Updates target to contain the intersection of the device types in
+  // "target" and "other".
+  static void MergeSupportedDevices(PrioritizedDeviceTypeVector* target,
+                                    const PrioritizedDeviceTypeVector& other);
+
+  // Returns the root node of the disjoint tree to which the node with the
+  // given id is connected.
+  int FindRoot(int node_id) { return Member::FindRoot(&members_, node_id); }
+
+  const Graph* const graph_;  // Not owned.
+  std::vector<Member> members_;
+  const DeviceSet* device_set_;  // Not owned.
+  const std::vector<DeviceType> device_types_;
+  const Device* default_device_;
+  const bool allow_soft_placement_;
+  const bool log_device_placement_;
+};
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_COLOCATION_GRAPH_H_
diff --git a/tensorflow/core/common_runtime/placer.cc b/tensorflow/core/common_runtime/placer.cc
index c9827d190d..de2f6e31da 100644
--- a/tensorflow/core/common_runtime/placer.cc
+++ b/tensorflow/core/common_runtime/placer.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include <vector>
 
 #include "absl/strings/str_join.h"
+#include "tensorflow/core/common_runtime/colocation_graph.h"
 #include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/framework/attr_value_util.h"
 #include "tensorflow/core/framework/device_attributes.pb.h"
@@ -41,84 +42,6 @@ namespace tensorflow {
 
 namespace {
 
-// We hoist the conversion from C-style string literal to StringPiece here,
-// so that we can avoid the many repeated calls to strlen().
-const StringPiece kColocationAttrNameStringPiece(kColocationAttrName);
-const StringPiece kColocationGroupPrefixStringPiece(kColocationGroupPrefix);
-
-// Returns a list of devices having type in supported_device_types.  The
-// returned list is sorted by preferred type (higher numeric type is preferred).
-std::vector<Device*> FilterSupportedDevices(
-    const std::vector<Device*>& devices,
-    const PrioritizedDeviceTypeVector& supported_device_types,
-    const Device* default_device) {
-  Device* filtered_default_device = nullptr;
-  std::vector<std::pair<Device*, int32>> prioritized_filtered_devices;
-  for (const auto& supported_device_type : supported_device_types) {
-    for (Device* device : devices) {
-      if (DeviceType(device->attributes().device_type()) ==
-          supported_device_type.first) {
-        if (device == default_device) {
-          filtered_default_device = device;
-        } else {
-          prioritized_filtered_devices.emplace_back(
-              device, supported_device_type.second);
-        }
-      }
-    }
-  }
-
-  auto device_sort = [](const std::pair<Device*, int32>& a,
-                        const std::pair<Device*, int32>& b) {
-    if (a.second != b.second) {
-      return a.second > b.second;
-    }
-
-    auto a_priority =
-        DeviceSet::DeviceTypeOrder(DeviceType(a.first->device_type()));
-    auto b_priority =
-        DeviceSet::DeviceTypeOrder(DeviceType(b.first->device_type()));
-    // First sort by prioritized device type (higher is preferred) and
-    // then by device name (lexicographically).
-    if (a_priority != b_priority) {
-      return a_priority > b_priority;
-    }
-    return StringPiece(a.first->name()) < StringPiece(b.first->name());
-  };
-  std::sort(prioritized_filtered_devices.begin(),
-            prioritized_filtered_devices.end(), device_sort);
-
-  std::vector<Device*> filtered_devices;
-  if (filtered_default_device != nullptr) {
-    filtered_devices.emplace_back(filtered_default_device);
-  }
-  for (const auto& prioritized_filtered_device : prioritized_filtered_devices) {
-    filtered_devices.push_back(prioritized_filtered_device.first);
-  }
-  return filtered_devices;
-}
-
-// Using absl::StrJoin with lambda does not work in tf-lite builds.
-std::vector<string> DevicesToString(const std::vector<Device*> devices) {
-  std::vector<string> v;
-  v.reserve(devices.size());
-  for (Device* d : devices) {
-    v.push_back(d->name());
-  }
-  return v;
-}
-
-// Using absl::StrJoin with lambda does not work in tf-lite builds.
-std::vector<string> DeviceTypeAndPriorityToString(
-    const PrioritizedDeviceTypeVector& devices) {
-  std::vector<string> v;
-  v.reserve(devices.size());
-  for (const std::pair<DeviceType, int32>& device_and_type : devices) {
-    v.push_back(DeviceTypeString(device_and_type.first));
-  }
-  return v;
-}
-
 // Returns true if the node has no inputs and produces outputs
 // that are consumed by a single node.
 //
@@ -140,1042 +63,6 @@ bool IsResourceGeneratorNode(const Node& node) {
          (IsRefType(node.output_type(0)) || node.output_type(0) == DT_RESOURCE);
 }
 
-bool IsExemptFromResourceInputColocation(const Node* node) {
-  // Note: Partitioned function calls, which place and partition their
-  // function bodies, are exempt from this check: they forward resource and
-  // ref inputs to operations that are appropriately placed, instead of
-  // dereferencing them.
-  const string& op_type = node->op_def().name();
-  return op_type == "PartitionedCall" || op_type == "StatefulPartitionedCall";
-}
-
-bool HasPriorities(const PrioritizedDeviceTypeVector& device_types) {
-  for (const auto& prioritized_device_type : device_types) {
-    if (prioritized_device_type.second != 0) return true;
-  }
-  return false;
-}
-
-bool ArePrioritiesSame(const PrioritizedDeviceTypeVector& a_types,
-                       const PrioritizedDeviceTypeVector& b_types) {
-  if (a_types.size() != b_types.size()) {
-    return false;
-  }
-  for (int i = 0; i < a_types.size(); ++i) {
-    if (a_types[i].first != b_types[i].first) {
-      return false;
-    }
-  }
-  return true;
-}
-
-// Represents a node in the disjoint node forest and the
-// accumulated constraints on the device used by that node.
-class Member {
- public:
-  Member() = default;
-
-  Status SetParentAndSupportedDevices(const Node& node,
-                                      const std::vector<DeviceType>& types) {
-    int id = node.id();
-    if (id < 0) {
-      return errors::Internal(
-          "Placer should not be creating a Member for node: ",
-          node.DebugString());
-    }
-    parent_ = id;
-    return SupportedDeviceTypesForNode(types, node.def(),
-                                       &supported_device_types_);
-  }
-
-  const DeviceNameUtils::ParsedName& requested_device_name() const {
-    return requested_device_name_;
-  }
-
-  Status SetAssignedDeviceName(const string& device_name) {
-    if (DeviceNameUtils::HasSomeDetails(requested_device_name_)) {
-      return errors::Internal(
-          "Setting assigned device name when there is a requested device set "
-          "is unsupported");
-    }
-    if (!DeviceNameUtils::ParseFullName(device_name, &assigned_device_name_)) {
-      return errors::Internal("Malformed assigned device '", device_name, "'");
-    }
-    // Set requested device to assigned_device to maintain the invariant that
-    // requested is a specialization of assigned.
-    requested_device_name_ = assigned_device_name_;
-    return Status::OK();
-  }
-
-  Status SetRequestedDeviceName(const Node& node) {
-    if (!DeviceNameUtils::ParseFullName(node.requested_device(),
-                                        &requested_device_name_)) {
-      return errors::InvalidArgument("Malformed device specification '",
-                                     node.requested_device(),
-                                     "' in node: ", node.DebugString());
-    }
-    if (DeviceNameUtils::HasSomeDetails(assigned_device_name_)) {
-      return errors::Internal(
-          "Setting requested device name when there is an assigned device set "
-          "is unsupported");
-    }
-    return Status::OK();
-  }
-
-  Status EnsureCompatibilityAcrossResourceEdge(
-      const Node& src, const Member& src_root,
-      const Node& dst, /*dst_root is this*/
-      bool log_device_placement) {
-    if (!DeviceNameUtils::AreCompatibleDevNames(src_root.assigned_device_name_,
-                                                assigned_device_name_)) {
-      return errors::InvalidArgument(
-          "Cannot place the graph because a reference or resource edge "
-          "connects colocation groups with incompatible assigned devices: ",
-          DeviceNameUtils::ParsedNameToString(src_root.assigned_device_name_),
-          " vs ", DeviceNameUtils::ParsedNameToString(assigned_device_name_));
-    }
-
-    if (DeviceNameUtils::AreCompatibleDevNames(src_root.requested_device_name_,
-                                               requested_device_name_)) {
-      return Status::OK();
-    }
-
-    // If we are here, assigned devices are compatible but requested ones are
-    // not. We will be overriding the requested device for destination node, but
-    // need to preserve the invariant that it will be a specialization of
-    // the assigned device.
-    if (log_device_placement) {
-      LOG(INFO) << "Ignoring device specification "
-                << DeviceNameUtils::ParsedNameToString(requested_device_name_)
-                << " for node '" << dst.name()
-                << "' because the input edge from '" << src.name()
-                << "' is a reference connection and already has a device "
-                   "field set to "
-                << DeviceNameUtils::ParsedNameToString(
-                       src_root.requested_device_name_);
-    }
-    requested_device_name_ = src_root.requested_device_name_;
-    DeviceNameUtils::EnsureSpecification(&requested_device_name_,
-                                         assigned_device_name_);
-    return Status::OK();
-  }
-
-  const PrioritizedDeviceTypeVector& supported_device_types() const {
-    return supported_device_types_;
-  }
-
-  static void Merge(std::vector<Member>* tree, int x_root, int y_root,
-                    Member** new_root, Member** old_root) {
-    Member& x_root_member = (*tree)[x_root];
-    Member& y_root_member = (*tree)[y_root];
-
-    // Merge the sets by setting the parent pointer of the smaller tree's root
-    // node to point to the root of the larger tree. Together with path
-    // compression in ColocationGraph::FindRoot, this ensures that we do not
-    // experience pathological performance on graphs such as chains.
-    int new_root_id, old_root_id;
-    if (x_root_member.rank_ < y_root_member.rank_) {
-      // The tree rooted at x_root is shallower, so connect it to
-      // y_root. The rank of y_root is unchanged because its new
-      // child has strictly less rank.
-      x_root_member.parent_ = y_root;
-      new_root_id = y_root;
-      old_root_id = x_root;
-    } else if (x_root_member.rank_ > y_root_member.rank_) {
-      // The tree rooted at y_root is shallower, so connect it to
-      // x_root. The rank of x_root is unchanged because its new
-      // child has strictly less rank.
-      y_root_member.parent_ = x_root;
-      new_root_id = x_root;
-      old_root_id = y_root;
-    } else {
-      // Both trees have the same rank, so break the tie by choosing
-      // x_root as the new root.
-      y_root_member.parent_ = x_root;
-      // Increment the rank of the tree rooted at x_root, because it
-      // is now strictly deeper than before.
-      ++x_root_member.rank_;
-      new_root_id = x_root;
-      old_root_id = y_root;
-    }
-
-    *new_root = &(*tree)[new_root_id];
-    *old_root = &(*tree)[old_root_id];
-  }
-
-  // tree is non-const because we can change some `parent` pointers in some
-  // members for more efficient future lookups. The vector itself is not
-  // changed.
-  static int FindRoot(std::vector<Member>* tree, int node_id) {
-    Member& member = (*tree)[node_id];
-    if (member.parent_ == node_id) {
-      // member.parent is the root of this disjoint tree.  Do nothing.
-    } else {
-      member.parent_ = FindRoot(tree, member.parent_);
-    }
-    // Now it is guaranteed that member.parent is the root of this disjoint
-    // tree.
-    return member.parent_;
-  }
-
-  Status MergeDeviceNames(const Member& other, bool allow_soft_placement) {
-    // Assuming the "requested is a specialization of assigned" invariant holds
-    // for this and `other`, it will hold after these two merges.
-    TF_RETURN_IF_ERROR(DeviceNameUtils::MergeDevNames(
-        &requested_device_name_, other.requested_device_name_,
-        allow_soft_placement));
-    return DeviceNameUtils::MergeDevNames(&assigned_device_name_,
-                                          other.assigned_device_name_,
-                                          allow_soft_placement);
-  }
-
-  // Updates this to contain the intersection of the device types in
-  // this and "other".
-  void MergeSupportedDevices(const Member& other) {
-    PrioritizedDeviceTypeVector temp = supported_device_types_;
-    supported_device_types_.clear();
-
-    // Generate intersection with priorities.
-    PrioritizedDeviceTypeVector target_intersection;
-    PrioritizedDeviceTypeVector other_intersection;
-    for (const auto& prioritized_device_type : temp) {
-      bool found = false;
-      for (const auto& other_prioritized_device_type :
-           other.supported_device_types_) {
-        if (prioritized_device_type.first ==
-            other_prioritized_device_type.first) {
-          found = true;
-          other_intersection.push_back(other_prioritized_device_type);
-          break;
-        }
-      }
-      if (found) {
-        target_intersection.push_back(prioritized_device_type);
-      }
-    }
-
-    // Sort the devices by priority order.
-    auto device_sort = [](const std::pair<DeviceType, int32>& a,
-                          const std::pair<DeviceType, int32>& b) {
-      // First look at set priorities.
-      if (a.second != b.second) {
-        return a.second > b.second;
-      }
-      // Then fallback to default priorities.
-      auto a_priority = DeviceSet::DeviceTypeOrder(a.first);
-      auto b_priority = DeviceSet::DeviceTypeOrder(b.first);
-      if (a_priority != b_priority) {
-        return a_priority > b_priority;
-      }
-      // Finally just look at the Device type strings.
-      return a.first.type_string() < b.first.type_string();
-    };
-
-    std::sort(target_intersection.begin(), target_intersection.end(),
-              device_sort);
-    std::sort(other_intersection.begin(), other_intersection.end(),
-              device_sort);
-
-    bool is_target_prioritized = HasPriorities(target_intersection);
-    bool is_other_prioritized = HasPriorities(other_intersection);
-    // If neither are prioritized then we just return the original i.e. target
-    // prioritization.
-    if (!is_target_prioritized && !is_other_prioritized) {
-      supported_device_types_ = target_intersection;
-    }
-    // If only one is prioritized, then we respect priorities of that in the
-    // intersection.
-    if (is_target_prioritized && !is_other_prioritized) {
-      supported_device_types_ = target_intersection;
-    }
-    if (!is_target_prioritized && is_other_prioritized) {
-      supported_device_types_ = other_intersection;
-    }
-    // If both have priorities and agree then we go with that. If the
-    // prioritization order is different, then we just fallback to the default
-    // i.e. what the DeviceTypeOrder suggests. In that case, we also set the
-    // merged priorities to 0, so that downstream merges work correctly as well.
-    if (is_target_prioritized && is_other_prioritized) {
-      bool priorities_agree =
-          ArePrioritiesSame(target_intersection, other_intersection);
-      if (priorities_agree) {
-        supported_device_types_ = target_intersection;
-      } else {
-        for (const auto& prioritized_device : target_intersection) {
-          supported_device_types_.push_back(
-              std::make_pair(prioritized_device.first, 0));
-        }
-        std::sort(supported_device_types_.begin(),
-                  supported_device_types_.end(), device_sort);
-      }
-    }
-  }
-
-  Status AssignDevice(const Node& node, bool allow_soft_placement) {
-    if (node.assigned_device_name_index() == assigned_device_name_index_) {
-      return Status::OK();
-    }
-
-    DeviceNameUtils::ParsedName parsed;
-    DeviceNameUtils::ParseFullName(node.assigned_device_name(), &parsed);
-    Status s = DeviceNameUtils::MergeDevNames(&assigned_device_name_, parsed,
-                                              allow_soft_placement);
-    if (!s.ok()) {
-      return errors::Internal(
-          "Constraining by assigned device should not cause an error. Original "
-          "root's assigned device name: ",
-          DeviceNameUtils::ParsedNameToString(assigned_device_name_),
-          " node's assigned device name \"", node.assigned_device_name(),
-          ". Error: ", s.error_message());
-    }
-    s = DeviceNameUtils::MergeDevNames(&requested_device_name_, parsed,
-                                       allow_soft_placement);
-    if (!s.ok()) {
-      return errors::Internal(
-          "Constraining by assigned device should not cause an error. Original "
-          "root's requested device name: \"",
-          DeviceNameUtils::ParsedNameToString(requested_device_name_),
-          "\", node's assigned device name \"", node.assigned_device_name(),
-          "\". Error: ", s.error_message());
-    }
-
-    assigned_device_name_index_ = node.assigned_device_name_index();
-    // Clear cached possible_devices, if any.
-    possible_devices_.clear();
-    return Status::OK();
-  }
-
-  void set_possible_devices(std::vector<Device*>&& devices) {
-    possible_devices_ = devices;
-  }
-  const std::vector<Device*>& possible_devices() { return possible_devices_; }
-
-  string DebugString() {
-    return absl::StrCat(
-        "Member(assigned_device_name_index_=", assigned_device_name_index_,
-        " requested_device_name_=",
-        DeviceNameUtils::ParsedNameToString(requested_device_name_),
-        " assigned_device_name_=",
-        DeviceNameUtils::ParsedNameToString(assigned_device_name_),
-        " supported_device_types_=[",
-        absl::StrJoin(DeviceTypeAndPriorityToString(supported_device_types_),
-                      ", "),
-        "] possible_devices_=[",
-        absl::StrJoin(DevicesToString(possible_devices_), ", "), "]");
-  }
-
- private:
-  // The id of the node that is the parent of this one, or its own
-  // id if it is a root. parent <= 0 indicates that this member is invalid.
-  int parent_ = -1;
-
-  // A proxy for the depth of the tree that is used to prefer
-  // connecting smaller trees to larger trees when merging disjoint
-  // sets.
-  int rank_ = 0;
-
-  // Once colocation groups have been formed, the Placer starts actually
-  // choosing devices. All nodes in a group must be assigned to the same
-  // device. Once we assigned the first device to some node in this group,
-  // we set assigned_device_name_index to this device name's index in the
-  // graph.
-  // The `*_device_name_` fields will contain the parsed name of this device
-  // and `possible_devices`, if computed, will contain just this device.
-  // `assigned_device_name_index` is an optimization to avoid parsing and
-  // comparing device names. The value of -1 signals that a single device
-  // has not been chosen yet.
-  int assigned_device_name_index_ = -1;
-
-  // The merged form of the device requested for this node, with those of all of
-  // its children. requested_device_name_ is always kept a specialization (i.e.
-  // DeviceNameUtils::IsSpecialization) of assigned_device_name_. When no device
-  // is requested, this field is set to assigned_device_name_.  As a
-  // specialization of assigned_device_name_, requested_device_name_ represents
-  // the most specific form of all assigned and requested devices of this node
-  // and its children, if this node is a root. requested_device_name_ is used
-  // to finally select devices for nodes.  We can override requested devices due
-  // to resource colocation constraints but not assigned devices (unless soft
-  // placement is on).
-  DeviceNameUtils::ParsedName requested_device_name_;
-
-  // The merged form of the device assigned for this node, with
-  // those of all of its children.
-  // This field is used to raise errors due to unsatisfiable constraints.
-  // Can be a partial specification.
-  // INVARIANT: requested_device_name_ is always a
-  // DeviceNameUtils::IsSpecialization of assigned_device_name_.
-  DeviceNameUtils::ParsedName assigned_device_name_;
-
-  // The intersection of all device types supported by this node,
-  // and those of all of its children, in priority order
-  // of the preferred device.
-  PrioritizedDeviceTypeVector supported_device_types_;
-
-  // If this node is a root, stores a list of Devices to which this node
-  // and all of its children have been assigned, or nullptr if this
-  // has not yet been computed.
-  std::vector<Device*> possible_devices_;
-};  // namespace
-
-// This class maintains the connected components of a colocation
-// constraint graph, and uses this information to assign a satisfying
-// device placement to the nodes of the graph.
-//
-// The typical usage pattern is:
-//
-//   Graph graph = ...;
-//   DeviceSet device_set = ...;
-//   ColocationGraph colocation_graph(graph, device_set);
-//
-//   // Add all the nodes of the `graph` to the `colocation_graph`.
-//   for (Node* node : graph.nodes()) {
-//     TF_RETURN_IF_ERROR(colocation_graph.AddNode(*node));
-//   }
-//
-//   // Add one or more colocation constraints.
-//   Node node_1 = *graph.FindNodeId(...);
-//   Node node_2 = *graph.FindNodeId(...);
-//   TF_RETURN_IF_ERROR(colocation_graph.ColocateNodes(node_1, node_2));
-//
-//   // Assign devices based on the accumulated constraints.
-//   for (Node* node : graph.nodes()) {
-//     TF_RETURN_IF_ERROR(colocation_graph.AssignDevice(node));
-//   }
-//
-// This implementation uses the Union-Find algorithm to efficiently maintain the
-// connected components and incrementally adds edges via
-// ColocationGraph::ColocateNodes() invocations.
-//
-// ColocationGraph does not assign any devices to graph nodes. The
-// `log_device_placement` argument is used to log messages when requested
-// device is ignored.
-class ColocationGraph {
- public:
-  ColocationGraph(const Graph* graph, const DeviceSet* device_set,
-                  const Device* default_device, bool allow_soft_placement,
-                  bool log_device_placement)
-      : graph_(graph),
-        device_set_(device_set),
-        device_types_(device_set->PrioritizedDeviceTypeList()),
-        default_device_(default_device),
-        allow_soft_placement_(allow_soft_placement),
-        log_device_placement_(log_device_placement) {
-    members_.resize(graph->num_node_ids());
-  }
-
-  // Adds each node of the Graph to this ColocationGraph as a singleton.
-  //
-  // NOTE: The implementation assumes that the ids of nodes passed to
-  // this method are dense and zero-based; the memory used will be linear in
-  // the largest node ID.
-  // NOTE: If this method returns an error, *this is left in an undefined
-  // state.
-  Status ColocateAllNodes() {
-    // This maps from a colocation group identifier to the 'root' of that
-    // colocation group.  Note that the keys in this map are StringPiece; the
-    // actual strings are stored under the NodeDef.  The lifetime of this map
-    // is limited to this ColocateAllNodes() method, and no part of the
-    // NodeDef trees are changed during the lifetime of this method, so using
-    // StringPiece as a key is safe.
-    //
-    // Also, as a further optimization, we remove the "loc:@" prefix from
-    // "class" attribute values, when they are used as keys in this table.
-    // This allows us to use StringPiece values that refer to substrings of
-    // 'string' values stored in NodeDef attribute lists, as well as StringPiece
-    // values that refer to 'string' values from NodeDef::name(), without
-    // performing any string allocations.
-    std::unordered_map<StringPiece, const Node*, StringPieceHasher>
-        colocation_group_root;
-
-    for (const Node* node : graph_->op_nodes()) {
-      // When adding the node, identify whether it is part of a colocation
-      // group.
-
-      // This code is effectively the equivalent of GetNodeAttr() for a string
-      // array, but it avoids all internal allocations (the allocation of the
-      // backing store of the std::vector<string> as well as the copies of the
-      // strings within it).  Instead, we combine the query of the colocation
-      // attribute with the calls to ColocateNodeToGroup.
-      bool found_spec = false;
-      const AttrValue* attr_value =
-          node->attrs().Find(kColocationAttrNameStringPiece);
-      if (attr_value != nullptr && attr_value->has_list()) {
-        for (const string& class_spec : attr_value->list().s()) {
-          StringPiece spec(class_spec);
-          if (str_util::ConsumePrefix(&spec,
-                                      kColocationGroupPrefixStringPiece)) {
-            found_spec = true;
-            TF_RETURN_IF_ERROR(
-                ColocateNodeToGroup(&colocation_group_root, node, spec));
-          }
-        }
-      }
-
-      if (!found_spec) {
-        // If the node does not specify a colocation group, then use the
-        // name of this node as the colocation group.
-        TF_RETURN_IF_ERROR(
-            ColocateNodeToGroup(&colocation_group_root, node, node->name()));
-      }
-    }
-
-    return Status::OK();
-  }
-
-  Status ColocateResourceOrRefEdge(Node* src, Node* dst) {
-    // Colocate `src` and `dst` to maintain the invariant that nodes
-    // connected by reference edges are colocated.
-    int src_root_id = FindRoot(src->id());
-    int dst_root_id = FindRoot(dst->id());
-    auto& src_root = members_[src_root_id];
-    auto& dst_root = members_[dst_root_id];
-
-    TF_RETURN_IF_ERROR(dst_root.EnsureCompatibilityAcrossResourceEdge(
-        *src, src_root, *dst, log_device_placement_));
-    Status status = ColocateNodes(*src, src_root_id, *dst, dst_root_id);
-    if (!status.ok()) {
-      return AttachDef(
-          errors::InvalidArgument("Nodes were connected by a "
-                                  "reference connection (requiring them to "
-                                  "be on the same device), but the two nodes "
-                                  "were assigned two different devices: ",
-                                  status.error_message()),
-          *dst);
-    }
-    return Status::OK();
-  }
-
-  Status ColocateResourceAndRefEdges() {
-    // Enumerate the constraint edges, and use them to update the disjoint
-    // node set.
-    // If `node` has an input edge with reference type, add an edge from the
-    // source of that edge to `node`.
-    for (const Edge* edge : graph_->edges()) {
-      if (edge->IsControlEdge()) {
-        continue;
-      }
-      Node* src = edge->src();
-      Node* dst = edge->dst();
-      DataType input_type = dst->input_type(edge->dst_input());
-      if ((input_type == DT_RESOURCE || IsRefType(input_type)) &&
-          !IsExemptFromResourceInputColocation(dst)) {
-        TF_RETURN_IF_ERROR(ColocateResourceOrRefEdge(src, dst));
-      }
-    }
-    return Status::OK();
-  }
-
-  Status Initialize() {
-    TF_RETURN_IF_ERROR(InitializeMembers());
-    TF_RETURN_IF_ERROR(ColocateAllNodes());
-    return ColocateResourceAndRefEdges();
-  }
-
-  Status ColocateNodeToGroup(
-      std::unordered_map<StringPiece, const Node*, StringPieceHasher>*
-          colocation_group_root,
-      const Node* node, StringPiece colocation_group) {
-    const Node*& root_node = (*colocation_group_root)[colocation_group];
-    if (root_node == nullptr) {
-      // This is the first node of the colocation group, so
-      // designate this node as the 'root' of that colocation group.
-      root_node = node;
-    } else {
-      // Try to colocate the node with the root.  If there is an
-      // error, return it.
-      Status s = ColocateNodes(*node, *root_node);
-      if (!s.ok()) {
-        return AttachDef(s, *node);
-      }
-    }
-    return Status::OK();
-  }
-
-  // Merge the (possibly disjoint) sets containing nodes "x" and
-  // "y". Returns OK if the all nodes in the union of these sets can
-  // be placed on the same device type.
-  //
-  // NOTE: If this method returns an error, *this is left in an undefined
-  // state.
-  Status ColocateNodes(const Node& x, const Node& y) {
-    int x_root = FindRoot(x.id());
-    int y_root = FindRoot(y.id());
-    return ColocateNodes(x, x_root, y, y_root);
-  }
-
-  // This overload of ColocateNodes() allows a caller to provide the root node
-  // ids for the two nodes. For large graphs, this noticeably reduces the
-  // graph load time.
-  Status ColocateNodes(const Node& x, int x_root, const Node& y, int y_root) {
-    if (x_root == y_root) {
-      return Status::OK();
-    }
-
-    DCHECK_EQ(x_root, FindRoot(x.id()));
-    DCHECK_EQ(y_root, FindRoot(y.id()));
-
-    Member* new_root_member;
-    Member* old_root_member;
-    Member::Merge(&members_, x_root, y_root, &new_root_member,
-                  &old_root_member);
-
-    // Merge the partial device specifications, and ensure that they are
-    // compatible. NULL options_ is treated as allowing soft placement.
-    // TODO(mrry): Consider enriching the error message by pointing
-    // out which nodes have the explicit partial device
-    // specifications that caused this conflict.
-    Status s = new_root_member->MergeDeviceNames(*old_root_member,
-                                                 allow_soft_placement_);
-    if (!s.ok()) {
-      return errors::InvalidArgument(
-          "Cannot colocate nodes ",
-          errors::FormatColocationNodeForError(x.name()), " and ",
-          errors::FormatColocationNodeForError(y.name()), ": ",
-          s.error_message());
-    }
-
-    // Ensure that the common root has at least one supported device
-    // type, by computing the intersection of
-    // new_root_member.supported_device_types and
-    // old_root_member.supported_device_types.
-    new_root_member->MergeSupportedDevices(*old_root_member);
-    if (new_root_member->supported_device_types().empty()) {
-      return errors::InvalidArgument(
-          "Cannot colocate nodes ",
-          errors::FormatColocationNodeForError(x.name()), " and ",
-          errors::FormatColocationNodeForError(y.name()),
-          " because no device type supports both of those nodes and the "
-          "other nodes colocated with them.",
-          DebugInfo(x_root), DebugInfo(y_root));
-    }
-
-    return Status::OK();
-  }
-
-  // Limits the possible devices of `node`'s colocation group to the device
-  // to which `node` is assigned. This makes sure that all nodes in this
-  // colocation group will be assigned to the same device. Without this
-  // explicit restriction, heuristics can choose a different possible device
-  // for other nodes in the group.
-  Status LimitToAssignedDevice(const Node& node) {
-    if (node.assigned_device_name_index() < 0) {
-      return errors::Internal(
-          "Expected an assigned node as argument to LimitToAssignedDevice but "
-          "got: ",
-          node.DebugString());
-    }
-    int root = FindRoot(node.id());
-    Member& root_member = members_[root];
-    return root_member.AssignDevice(node, allow_soft_placement_);
-  }
-
-  // For the given node, subject to the constraints previously given
-  // to this ColocationGraph, set its assigned_device_name. Returns OK
-  // if a satisfying device can be found, otherwise an error.
-  //
-  // Note: This method returns a pointer to a field within members_.
-  // The caller must not use the returned pointer after there is any possibility
-  // that the members_[i].possible_devices field has been modified.
-  Status GetDevicesForNode(Node* node,
-                           const std::vector<Device*>** possible_devices) {
-    *possible_devices = nullptr;
-    const int node_root = FindRoot(node->id());
-    if (!members_[node_root].possible_devices().empty()) {
-      *possible_devices = &members_[node_root].possible_devices();
-      return Status::OK();
-    }
-
-    // We have not yet computed the possible devices for the
-    // colocated node set containing 'node', so we do so now using the
-    // constraints on the root node.
-
-    // "devices" will contain the set of feasible placements for the
-    // colocated node set containing 'node'.
-    std::vector<Device*> devices;
-    if (DeviceNameUtils::HasSomeDetails(
-            members_[node_root].requested_device_name())) {
-      // The root node has a (possibly partial) device
-      // specification, so enumerate the physical devices that
-      // conform to it.
-      device_set_->FindMatchingDevices(
-          members_[node_root].requested_device_name(), &devices);
-
-      if (!devices.empty()) {
-        // Filter devices into those that are compatible with the root
-        // node (and its children).
-        devices = FilterSupportedDevices(
-            devices, members_[node_root].supported_device_types(),
-            default_device_);
-      }
-
-      // Perform soft placement if allow_soft_placement_ is set.
-      if (devices.empty() && allow_soft_placement_) {
-        // The soft_device_name is the same as the node's device name
-        // without specifying the device type or ID.
-        DeviceNameUtils::ParsedName soft_device_name =
-            members_[node_root].requested_device_name();
-        soft_device_name.type.clear();
-        soft_device_name.has_type = false;
-        soft_device_name.has_id = false;
-        device_set_->FindMatchingDevices(soft_device_name, &devices);
-        if (!devices.empty()) {
-          devices = FilterSupportedDevices(
-              devices, members_[node_root].supported_device_types(),
-              default_device_);
-        }
-      }
-
-      if (devices.empty()) {
-        // Return an error when a physical device that matches an explicit
-        // device specification is not found. This ensures that we don't
-        // assign a node to GPU when the user wanted to force it on CPU.
-        string debug_info = DebugInfo(node_root);
-
-        DeviceNameUtils::ParsedName specified_device_name;
-        if (DeviceNameUtils::ParseFullName(node->requested_device(),
-                                           &specified_device_name) &&
-            specified_device_name ==
-                members_[node_root].requested_device_name()) {
-          // The specified device and merged set device match, and
-          // will appear in the GraphDef (for debugging), so just
-          // print the specified device.
-          std::vector<Device*> devices_matching_nodedef;
-          device_set_->FindMatchingDevices(specified_device_name,
-                                           &devices_matching_nodedef);
-          if (devices_matching_nodedef.empty()) {
-            // Sometimes it is almost impossible to understand the problem
-            // without a list of available devices.
-            std::vector<string> device_names;
-            for (const Device* device : device_set_->devices()) {
-              device_names.push_back(device->name());
-            }
-            std::sort(device_names.begin(), device_names.end());
-
-            string gpu_msg = "";
-            if (!IsGoogleCudaEnabled() &&
-                str_util::Lowercase(specified_device_name.type) == "gpu") {
-              gpu_msg =
-                  " The requested device appears to be a GPU, but CUDA is not "
-                  "enabled.";
-            }
-
-            return errors::InvalidArgument(
-                errors::FormatNodeNameForError(node->name()),
-                "was explicitly assigned to ", node->requested_device(),
-                " but available devices are [ ",
-                str_util::Join(device_names, ", "), " ]. Make sure ",
-                "the device specification refers to a valid device.", gpu_msg);
-          } else if (specified_device_name.has_type) {
-            return errors::InvalidArgument(
-                "Could not satisfy explicit device specification '",
-                node->requested_device(), "' because no supported kernel for ",
-                specified_device_name.type, " devices is available.",
-                debug_info, "\nRegistered kernels:\n",
-                KernelsRegisteredForOp(node->type_string()));
-          } else {
-            return errors::InvalidArgument(
-                "Could not satisfy explicit device specification '",
-                node->requested_device(), debug_info);
-          }
-        } else {
-          // The specified device may be a valid device but the
-          // merged set device is different, so print both.
-          return errors::InvalidArgument(
-              "Could not satisfy explicit device specification '",
-              node->requested_device(), "' because the node ",
-              errors::FormatColocationNodeForError(node->name()),
-              " was colocated with a group of nodes that ",
-              "required incompatible device '",
-              DeviceNameUtils::ParsedNameToString(
-                  members_[node_root].requested_device_name()),
-              "'", debug_info);
-        }
-      }
-    } else {
-      // The device is completely unspecified, so enumerate the devices that
-      // support all of the nodes in the set.
-      if (device_set_->devices().empty()) {
-        return errors::Internal("No devices are registered");
-      }
-      devices = FilterSupportedDevices(
-          device_set_->devices(), members_[node_root].supported_device_types(),
-          default_device_);
-
-      if (devices.empty()) {
-        return errors::InvalidArgument(
-            "Node had no OpKernel registered to support this operation: ",
-            "Operation was ", node->type_string(), " and inputs were ",
-            DataTypeVectorString(node->input_types()), DebugInfo(node_root));
-      }
-    }
-
-    // Cache the result of the possible devices for this node group.
-    members_[node_root].set_possible_devices(std::move(devices));
-    *possible_devices = &members_[node_root].possible_devices();
-    return Status::OK();
-  }
-
-  Status InitializeMembers() {
-    for (Node* node : graph_->op_nodes()) {
-      Status status = InitializeMember(*node, &members_[node->id()]);
-      if (!status.ok()) {
-        return AttachDef(status, *node);
-      }
-    }
-    return Status::OK();
-  }
-
-  string DebugString() {
-    std::unordered_set<int> roots;
-    std::vector<string> root_strings;
-    for (const Node* node : graph_->nodes()) {
-      if (!node->IsOp()) {
-        continue;
-      }
-      int node_root = FindRoot(node->id());
-      if (roots.count(node_root) == 0) {
-        root_strings.push_back(DebugInfo(node_root));
-        roots.insert(node_root);
-      }
-    }
-    return absl::StrJoin(root_strings, "\n");
-  }
-
-  // Returns debugging info for the node referred to by 'node_root'.
-  string DebugInfo(const int node_root) {
-    string text(
-        "\nColocation Debug Info:\n"
-        "Colocation group had the following types and devices: ");
-
-    // If this node is part of a colocation group, then we want to
-    // collect the mapping of ops to supported devices, so that
-    // the user can see why an unsatisfiable placement occurred.
-
-    std::unordered_map<string, string> type_to_devices;
-    std::vector<const Node*> colocation_nodes;
-    int num_nodes_found = 0;
-
-    for (const Node* node : graph_->nodes()) {
-      if (!node->IsOp()) {
-        continue;
-      }
-      int id = node->id();
-      if (FindRoot(id) != node_root) {
-        continue;
-      }
-      ++num_nodes_found;
-      colocation_nodes.push_back(node);
-      const string& op_type = node->type_string();
-      string devices_registered;
-      for (const auto& device_type : members_[id].supported_device_types()) {
-        strings::StrAppend(&devices_registered,
-                           DeviceTypeString(device_type.first), " ");
-      }
-
-      type_to_devices[op_type] = std::move(devices_registered);
-    }
-
-    for (const auto& td : type_to_devices) {
-      strings::StrAppend(&text, "\n", td.first, ": ", td.second);
-    }
-    strings::StrAppend(&text,
-                       "\n\nColocation members and user-requested devices:");
-    for (const Node* node : colocation_nodes) {
-      strings::StrAppend(&text, "\n  ", node->name(), " (", node->type_string(),
-                         ") ", node->requested_device());
-    }
-    strings::StrAppend(&text, "\n");
-
-    if (num_nodes_found <= 0) {
-      text.clear();
-    }
-    return text;
-  }
-
-  Status InitializeMemberWithAssignedDevice(const string& assigned_device_name,
-                                            const string& node_type,
-                                            bool must_be_full_name,
-                                            Member* member) {
-    // This node has already been assigned to a device, so we
-    // respect this placement, after sanity-checking it.
-    // NOTE: Since any assignment must have been performed by
-    // the TensorFlow runtime, we consider errors in this branch to
-    // be INTERNAL.
-    TF_RETURN_IF_ERROR(member->SetAssignedDeviceName(assigned_device_name));
-    if (!must_be_full_name) {
-      return Status::OK();
-    }
-    // Since assigned device must be a full specification, do extra checks.
-    const Device* assigned_device =
-        device_set_->FindDeviceByName(assigned_device_name);
-    if (assigned_device == nullptr) {
-      return errors::Internal("Assigned device '", assigned_device_name,
-                              "' does not match any device");
-    }
-
-    for (const auto& d : member->supported_device_types()) {
-      if (DeviceType(assigned_device->attributes().device_type()) == d.first) {
-        return Status::OK();
-      }
-    }
-
-    return errors::Internal("Assigned device '", assigned_device_name,
-                            "' does not have registered OpKernel support "
-                            "for ",
-                            node_type);
-  }
-
-  Status InitializeMember(const Node& node, Member* member) {
-    TF_RETURN_IF_ERROR(
-        member->SetParentAndSupportedDevices(node, device_types_));
-
-    if (node.has_assigned_device_name()) {
-      TF_RETURN_IF_ERROR(InitializeMemberWithAssignedDevice(
-          node.assigned_device_name(), node.type_string(), true, member));
-    } else {
-      // This node has not yet been assigned to a device, so we
-      // calculate any constraints due to the set of registered
-      // kernels and any (partial) user-provided device specification
-      // in the NodeDef.
-
-      // If no kernels are registered for this op type, fail with an error.
-      if (member->supported_device_types().empty()) {
-        std::set<string> registered_device_types;
-        for (Device* d : device_set_->devices()) {
-          registered_device_types.insert(d->device_type());
-        }
-        std::vector<string> attr_key_vals;
-        for (const auto& it : node.attrs()) {
-          const string& name = it.first;
-          const AttrValue& attr_value = it.second;
-          attr_key_vals.push_back(
-              strings::StrCat(name, "=", SummarizeAttrValue(attr_value)));
-        }
-        return errors::InvalidArgument(
-            "No OpKernel was registered to support Op '", node.type_string(),
-            "' used by ", errors::FormatNodeNameForError(node.name()),
-            "with these attrs: [", str_util::Join(attr_key_vals, ", "),
-            "]\n"
-            "Registered devices: [",
-            str_util::Join(registered_device_types, ", "), "]\n",
-            "Registered kernels:\n",
-            KernelsRegisteredForOp(node.type_string()));
-      }
-
-      // If the NodeDef contains a device, then we interpret it as a
-      // (partial) device specification.
-      if (!node.requested_device().empty()) {
-        if (IsResourceGeneratorNode(node)) {
-          // Treat requested device on resource generating nodes as assigned
-          // device so that we don't override it.
-          TF_RETURN_IF_ERROR(InitializeMemberWithAssignedDevice(
-              node.requested_device(), node.type_string(), false, member));
-        } else {
-          // The user has specified a device in the NodeDef, try to find a
-          // valid device matching their specification in the set of
-          // devices.
-          // NOTE: The full name may specify a device that is not in
-          // n.supported_device_types(), but we check that in AssignDevice().
-          TF_RETURN_IF_ERROR(member->SetRequestedDeviceName(node));
-        }
-      }
-    }
-    return Status::OK();
-  }
-
-  // Updates target to contain the intersection of the device types in
-  // "target" and "other".
-  static void MergeSupportedDevices(PrioritizedDeviceTypeVector* target,
-                                    const PrioritizedDeviceTypeVector& other) {
-    PrioritizedDeviceTypeVector temp = *target;
-    target->clear();
-
-    // Generate intersection with priorities.
-    PrioritizedDeviceTypeVector target_intersection;
-    PrioritizedDeviceTypeVector other_intersection;
-    for (const auto& prioritized_device_type : temp) {
-      bool found = false;
-      for (const auto& other_prioritized_device_type : other) {
-        if (prioritized_device_type.first ==
-            other_prioritized_device_type.first) {
-          found = true;
-          other_intersection.push_back(other_prioritized_device_type);
-          break;
-        }
-      }
-      if (found) {
-        target_intersection.push_back(prioritized_device_type);
-      }
-    }
-
-    // Sort the devices by priority order.
-    auto device_sort = [](const std::pair<DeviceType, int32>& a,
-                          const std::pair<DeviceType, int32>& b) {
-      // First look at set priorities.
-      if (a.second != b.second) {
-        return a.second > b.second;
-      }
-      // Then fallback to default priorities.
-      auto a_priority = DeviceSet::DeviceTypeOrder(a.first);
-      auto b_priority = DeviceSet::DeviceTypeOrder(b.first);
-      if (a_priority != b_priority) {
-        return a_priority > b_priority;
-      }
-      // Finally just look at the Device type strings.
-      return a.first.type_string() < b.first.type_string();
-    };
-
-    std::sort(target_intersection.begin(), target_intersection.end(),
-              device_sort);
-    std::sort(other_intersection.begin(), other_intersection.end(),
-              device_sort);
-
-    bool is_target_prioritized = HasPriorities(target_intersection);
-    bool is_other_prioritized = HasPriorities(other_intersection);
-    // If neither are prioritized then we just return the original i.e. target
-    // prioritization.
-    if (!is_target_prioritized && !is_other_prioritized) {
-      *target = target_intersection;
-    }
-    // If only one is prioritized, then we respect priorities of that in the
-    // intersection.
-    if (is_target_prioritized && !is_other_prioritized) {
-      *target = target_intersection;
-    }
-    if (!is_target_prioritized && is_other_prioritized) {
-      *target = other_intersection;
-    }
-    // If both have priorities and agree then we go with that. If the
-    // prioritization order is different, then we just fallback to the default
-    // i.e. what the DeviceTypeOrder suggests. In that case, we also set the
-    // merged priorities to 0, so that downstream merges work correctly as well.
-    if (is_target_prioritized && is_other_prioritized) {
-      bool priorities_agree =
-          ArePrioritiesSame(target_intersection, other_intersection);
-      if (priorities_agree) {
-        *target = target_intersection;
-      } else {
-        for (const auto& prioritized_device : target_intersection) {
-          target->push_back(std::make_pair(prioritized_device.first, 0));
-        }
-        std::sort(target->begin(), target->end(), device_sort);
-      }
-    }
-  }
-
-  // Returns the root node of the disjoint tree to which the node with the
-  // given id is connected.
-  int FindRoot(int node_id) { return Member::FindRoot(&members_, node_id); }
-
-  const Graph* const graph_;  // Not owned.
-  std::vector<Member> members_;
-  const DeviceSet* device_set_;  // Not owned.
-  const std::vector<DeviceType> device_types_;
-  const Device* default_device_;
-  const bool allow_soft_placement_;
-  const bool log_device_placement_;
-};
-
 void LogDeviceAssignment(const Node* node, bool log_device_placement) {
   // Log placement if log_device_placement is set.
   if (log_device_placement) {
-- 
GitLab


From 6d85abf25052efa6f72a8ee671fbaf5a18a85ae0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 8 Feb 2019 14:18:25 -0800
Subject: [PATCH 2197/2345] Add parsing/printing of tiles and element size in
 bits (if necessary) for Layout.

PiperOrigin-RevId: 233125262
---
 tensorflow/compiler/xla/layout.cc             |  26 ++++-
 tensorflow/compiler/xla/layout.h              |  14 +++
 tensorflow/compiler/xla/layout_test.cc        |   4 +-
 tensorflow/compiler/xla/layout_util.cc        |  14 ++-
 tensorflow/compiler/xla/layout_util.h         |   4 +-
 tensorflow/compiler/xla/layout_util_test.cc   |  75 +++++++++++++
 tensorflow/compiler/xla/service/hlo_lexer.cc  |   4 +
 tensorflow/compiler/xla/service/hlo_lexer.h   |   7 +-
 tensorflow/compiler/xla/service/hlo_parser.cc | 106 +++++++++++++++++-
 .../compiler/xla/service/hlo_parser_test.cc   |  54 +++++++++
 tensorflow/compiler/xla/shape_util.cc         |  16 +--
 tensorflow/compiler/xla/shape_util.h          |   4 +-
 12 files changed, 303 insertions(+), 25 deletions(-)

diff --git a/tensorflow/compiler/xla/layout.cc b/tensorflow/compiler/xla/layout.cc
index e3b5fcd527..d2f7985aab 100644
--- a/tensorflow/compiler/xla/layout.cc
+++ b/tensorflow/compiler/xla/layout.cc
@@ -30,7 +30,19 @@ TileProto Tile::ToProto() const {
 }
 
 string Tile::ToString() const {
-  return absl::StrCat("(", absl::StrJoin(dimensions(), ","), ")");
+  std::vector<string> elements;
+  for (auto dim : dimensions()) {
+    if (dim >= 0) {
+      elements.push_back(std::to_string(dim));
+    } else {
+      CHECK_EQ(dim, kCombineDimension)
+          << "Tile dimension size needs to be mininum int64 value if it's "
+             "negative. Value is "
+          << dim;
+      elements.push_back("*");
+    }
+  }
+  return absl::StrCat("(", absl::StrJoin(elements, ","), ")");
 }
 
 /* static */ Layout Layout::CreateFromProto(const LayoutProto& proto) {
@@ -64,11 +76,19 @@ LayoutProto Layout::ToProto() const {
 }
 
 string Layout::ToString() const {
-  // TODO(b/119839262): Emit tiles in string.
   if (format() == SPARSE) {
+    CHECK_EQ(tiles_size(), 0) << "Sparse layout should not be tiled.";
     return absl::StrCat("sparse{", max_sparse_elements(), "}");
   } else if (format() == DENSE) {
-    return absl::StrCat("{", absl::StrJoin(minor_to_major(), ","), "}");
+    string colon_string = tiles().empty() ? "" : "T";
+    for (Tile tile : tiles()) {
+      absl::StrAppend(&colon_string, tile.ToString());
+    }
+    if (element_size_in_bits() != 0) {
+      absl::StrAppend(&colon_string, "E(", element_size_in_bits(), ")");
+    }
+    return absl::StrCat("{", absl::StrJoin(minor_to_major(), ","),
+                        colon_string.empty() ? "" : ":", colon_string, "}");
   } else {
     CHECK_EQ(format(), INVALID_FORMAT);
     return "invalid{}";
diff --git a/tensorflow/compiler/xla/layout.h b/tensorflow/compiler/xla/layout.h
index 313368c39e..1faa162998 100644
--- a/tensorflow/compiler/xla/layout.h
+++ b/tensorflow/compiler/xla/layout.h
@@ -55,6 +55,20 @@ class Tile {
   // Returns the dimensions of the tile.
   const std::vector<int64>& dimensions() const { return dimensions_; }
 
+  Tile& add_dimensions(int64 value) {
+    dimensions_.push_back(value);
+    return *this;
+  }
+
+  Tile& clear_dimensions() {
+    dimensions_.clear();
+    return *this;
+  }
+
+  // This dimension size means the corresponding dimension in the shape is
+  // combined with the next minor dimension before tiling is applied.
+  static constexpr int64 kCombineDimension = std::numeric_limits<int64>::min();
+
  private:
   // The bounds of the tile.
   std::vector<int64> dimensions_;
diff --git a/tensorflow/compiler/xla/layout_test.cc b/tensorflow/compiler/xla/layout_test.cc
index fb6abd3f65..7d43b0b87c 100644
--- a/tensorflow/compiler/xla/layout_test.cc
+++ b/tensorflow/compiler/xla/layout_test.cc
@@ -38,10 +38,10 @@ TEST_F(LayoutTest, ToString) {
             "sparse{123}");
   EXPECT_EQ(Layout({4, 5, 6}).ToString(), "{4,5,6}");
   EXPECT_EQ(Layout({3, 2, 1, 0}, {Tile({42, 123}), Tile({4, 5})}).ToString(),
-            "{3,2,1,0}");
+            "{3,2,1,0:T(42,123)(4,5)}");
   EXPECT_EQ(
       Layout({1, 0}, {Tile({2, 55})}).set_element_size_in_bits(42).ToString(),
-      "{1,0}");
+      "{1,0:T(2,55)E(42)}");
 }
 
 TEST_F(LayoutTest, StreamOut) {
diff --git a/tensorflow/compiler/xla/layout_util.cc b/tensorflow/compiler/xla/layout_util.cc
index 2fe9b56c6b..5d85fd7942 100644
--- a/tensorflow/compiler/xla/layout_util.cc
+++ b/tensorflow/compiler/xla/layout_util.cc
@@ -54,12 +54,24 @@ void SetDefaultLayoutToContainer(std::vector<int64>* minor_to_major) {
 }  // namespace
 
 /* static */ Layout LayoutUtil::MakeLayout(
-    absl::Span<const int64> minor_to_major) {
+    absl::Span<const int64> minor_to_major, absl::Span<const Tile> tiles,
+    int64 element_size_in_bits) {
   Layout layout;
   layout.set_format(DENSE);
   for (int64 dimension_number : minor_to_major) {
     layout.add_minor_to_major(dimension_number);
   }
+  for (Tile tile : tiles) {
+    for (int64 dim : tile.dimensions()) {
+      if (dim < 0 && dim != Tile::kCombineDimension) {
+        LOG(FATAL) << "Tile dimension size needs to be mininum int64 value if "
+                      "it's negative. Value is "
+                   << dim;
+      }
+    }
+    *layout.add_tiles() = tile;
+  }
+  layout.set_element_size_in_bits(element_size_in_bits);
   return layout;
 }
 
diff --git a/tensorflow/compiler/xla/layout_util.h b/tensorflow/compiler/xla/layout_util.h
index 609dba67bc..9997aef465 100644
--- a/tensorflow/compiler/xla/layout_util.h
+++ b/tensorflow/compiler/xla/layout_util.h
@@ -36,7 +36,9 @@ class LayoutUtil {
  public:
   // Creates a layout with the given minor-to-major dimension order. (This is a
   // convenience function for protobuf construction.)
-  static Layout MakeLayout(absl::Span<const int64> minor_to_major);
+  static Layout MakeLayout(absl::Span<const int64> minor_to_major,
+                           absl::Span<const Tile> tiles = {},
+                           int64 element_size_in_bits = 0);
 
   // Similar to MakeLayout, but take indices in reverse order.
   static Layout MakeLayoutFromMajorToMinor(
diff --git a/tensorflow/compiler/xla/layout_util_test.cc b/tensorflow/compiler/xla/layout_util_test.cc
index 4cc94c270c..12da214063 100644
--- a/tensorflow/compiler/xla/layout_util_test.cc
+++ b/tensorflow/compiler/xla/layout_util_test.cc
@@ -317,6 +317,81 @@ TEST_F(LayoutUtilTest, DefaultLayoutGettersMajorToMinor) {
                             ShapeUtil::MakeShape(F32, {10, 20, 30, 15, 25}))));
 }
 
+TEST_F(LayoutUtilTest, HumanStringWithTiling) {
+  Shape shape = ShapeUtil::MakeShapeWithLayout(F32, {2, 3, 4}, {0, 1, 2});
+  Tile* tile;
+
+  // No tiling.
+  EXPECT_EQ(ShapeUtil::HumanStringWithLayout(shape), "f32[2,3,4]{0,1,2}");
+
+  // 2D tile.
+  tile = shape.mutable_layout()->add_tiles();
+  tile->add_dimensions(512);
+  tile->add_dimensions(1024);
+  EXPECT_EQ(ShapeUtil::HumanStringWithLayout(shape),
+            "f32[2,3,4]{0,1,2:T(512,1024)}");
+
+  // 1D tile.
+  shape.mutable_layout()->clear_tiles();
+  tile = shape.mutable_layout()->add_tiles();
+  tile->add_dimensions(512);
+  EXPECT_EQ(ShapeUtil::HumanStringWithLayout(shape),
+            "f32[2,3,4]{0,1,2:T(512)}");
+
+  // 2 tiles.
+  shape = ShapeUtil::MakeShapeWithLayout(BF16, {2, 3, 4}, {1, 2, 0});
+  tile = shape.mutable_layout()->add_tiles();
+  tile->add_dimensions(16);
+  tile->add_dimensions(256);
+  tile = shape.mutable_layout()->add_tiles();
+  tile->add_dimensions(2);
+  tile->add_dimensions(1);
+  EXPECT_EQ(ShapeUtil::HumanStringWithLayout(shape),
+            "bf16[2,3,4]{1,2,0:T(16,256)(2,1)}");
+
+  // PRED with element size of 8 bits.
+  shape = ShapeUtil::MakeShapeWithLayout(PRED, {8, 8, 8}, {0, 2, 1});
+  tile = shape.mutable_layout()->add_tiles();
+  tile->add_dimensions(8);
+  tile->add_dimensions(128);
+  EXPECT_EQ(ShapeUtil::HumanStringWithLayout(shape),
+            "pred[8,8,8]{0,2,1:T(8,128)}");
+
+  // PRED with element size of 32 bits.
+  shape.mutable_layout()->clear_tiles();
+  tile = shape.mutable_layout()->add_tiles();
+  tile->add_dimensions(8);
+  tile->add_dimensions(128);
+  shape.mutable_layout()->set_element_size_in_bits(32);
+  EXPECT_EQ(ShapeUtil::HumanStringWithLayout(shape),
+            "pred[8,8,8]{0,2,1:T(8,128)E(32)}");
+
+  // No tile. PRED with element size of 32 bits.
+  shape.mutable_layout()->clear_tiles();
+  shape.mutable_layout()->set_element_size_in_bits(32);
+  EXPECT_EQ(ShapeUtil::HumanStringWithLayout(shape),
+            "pred[8,8,8]{0,2,1:E(32)}");
+
+  // Tile with negative dimension size for combining dimensions.
+  shape = ShapeUtil::MakeShapeWithLayout(BF16, {2, 3, 1004}, {2, 1, 0});
+  tile = shape.mutable_layout()->add_tiles();
+  tile->add_dimensions(2);
+  tile->add_dimensions(Tile::kCombineDimension);
+  tile->add_dimensions(128);
+  EXPECT_EQ(ShapeUtil::HumanStringWithLayout(shape),
+            "bf16[2,3,1004]{2,1,0:T(2,*,128)}");
+
+  // Tile with two negative dimensions.
+  shape = ShapeUtil::MakeShapeWithLayout(BF16, {8, 2, 3, 1004}, {3, 2, 1, 0});
+  tile = shape.mutable_layout()->add_tiles();
+  tile->add_dimensions(2);
+  tile->add_dimensions(Tile::kCombineDimension);
+  tile->add_dimensions(Tile::kCombineDimension);
+  tile->add_dimensions(128);
+  EXPECT_EQ(ShapeUtil::HumanStringWithLayout(shape),
+            "bf16[8,2,3,1004]{3,2,1,0:T(2,*,*,128)}");
+}
+
 TEST_F(LayoutUtilTest, ValidateLayout_ValidArrayLayout) {
   Shape shape = ShapeUtil::MakeShapeWithLayout(F32, {2, 3}, {0, 1});
   auto status =
diff --git a/tensorflow/compiler/xla/service/hlo_lexer.cc b/tensorflow/compiler/xla/service/hlo_lexer.cc
index c1a642dfea..2255383322 100644
--- a/tensorflow/compiler/xla/service/hlo_lexer.cc
+++ b/tensorflow/compiler/xla/service/hlo_lexer.cc
@@ -153,6 +153,8 @@ TokKind HloLexer::LexToken() {
         return LexPercent();
       case ':':
         return TokKind::kColon;
+      case '*':
+        return TokKind::kAsterisk;
       case '[':
         return TokKind::kLsquare;
       case ']':
@@ -464,6 +466,8 @@ string TokKindToString(TokKind kind) {
       return "kComma";
     case TokKind::kColon:
       return "kColon";
+    case TokKind::kAsterisk:
+      return "kAsterisk";
     case TokKind::kLsquare:
       return "kLsquare";
     case TokKind::kRsquare:
diff --git a/tensorflow/compiler/xla/service/hlo_lexer.h b/tensorflow/compiler/xla/service/hlo_lexer.h
index 9b440db8b7..383fb4e862 100644
--- a/tensorflow/compiler/xla/service/hlo_lexer.h
+++ b/tensorflow/compiler/xla/service/hlo_lexer.h
@@ -38,9 +38,10 @@ enum class TokKind {
   kError,
 
   // Tokens with no info.
-  kEqual,  // =
-  kComma,  // ,
-  kColon,  // :
+  kEqual,     // =
+  kComma,     // ,
+  kColon,     // :
+  kAsterisk,  // *
   kLsquare,
   kRsquare,  // [  ]
   kLbrace,
diff --git a/tensorflow/compiler/xla/service/hlo_parser.cc b/tensorflow/compiler/xla/service/hlo_parser.cc
index 0ab0054dac..20dbed07c5 100644
--- a/tensorflow/compiler/xla/service/hlo_parser.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser.cc
@@ -275,6 +275,7 @@ class HloParser {
                            std::vector<bool>* dynamic_dimensions);
   bool ParseShape(Shape* result);
   bool ParseLayout(Layout* layout);
+  bool ParseTiles(std::vector<Tile>* tiles);
   bool ParseOpcode(HloOpcode* result);
   bool ParseFftType(FftType* result);
   bool ParseTriangularSolveTranspose(TriangularSolveOptions::Transpose* result);
@@ -3203,9 +3204,46 @@ bool HloParser::ParseDimensionSizes(std::vector<int64>* dimension_sizes,
                    parse_and_add_item);
 }
 
-// layout ::= '{' int64_list '}'
+// tiles
+//   ::= /*empty*/
+//   ::= 'T' '(' dim_list ')'
+// dim_list
+//   ::= /*empty*/
+//   ::= (int64 | '*') (',' (int64 | '*'))*
+bool HloParser::ParseTiles(std::vector<Tile>* tiles) {
+  auto parse_and_add_tile_dimension = [&]() {
+    tensorflow::int64 i;
+    if (ParseInt64(&i)) {
+      tiles->back().add_dimensions(i);
+      return true;
+    }
+    if (lexer_.GetKind() == TokKind::kAsterisk) {
+      tiles->back().add_dimensions(Tile::kCombineDimension);
+      lexer_.Lex();
+      return true;
+    }
+    return false;
+  };
+
+  do {
+    tiles->push_back(Tile());
+    if (!ParseList(TokKind::kLparen, TokKind::kRparen, TokKind::kComma,
+                   parse_and_add_tile_dimension)) {
+      return false;
+    }
+  } while (lexer_.GetKind() == TokKind::kLparen);
+  return true;
+}
+
+// layout ::= '{' int64_list (':' tiles element_size_in_bits)? '}'
+// element_size_in_bits
+//   ::= /*empty*/
+//   ::= 'E' '(' int64 ')'
 bool HloParser::ParseLayout(Layout* layout) {
   std::vector<int64> minor_to_major;
+  std::vector<Tile> tiles;
+  tensorflow::int64 element_size_in_bits = 0;
+
   auto parse_and_add_item = [&]() {
     int64 i;
     if (!ParseInt64(&i)) {
@@ -3214,11 +3252,60 @@ bool HloParser::ParseLayout(Layout* layout) {
     minor_to_major.push_back(i);
     return true;
   };
-  if (!ParseList(TokKind::kLbrace, TokKind::kRbrace, TokKind::kComma,
-                 parse_and_add_item)) {
+
+  if (!ParseToken(TokKind::kLbrace,
+                  StrCat("expects layout to start with ",
+                         TokKindToString(TokKind::kLbrace)))) {
+    return false;
+  }
+  if (lexer_.GetKind() != TokKind::kRbrace) {
+    if (lexer_.GetKind() == TokKind::kInt) {
+      // Parse minor to major.
+      do {
+        if (!parse_and_add_item()) {
+          return false;
+        }
+      } while (EatIfPresent(TokKind::kComma));
+    }
+
+    if (lexer_.GetKind() == TokKind::kColon) {
+      lexer_.Lex();
+      if (lexer_.GetKind() == TokKind::kIdent && lexer_.GetStrVal() == "T") {
+        lexer_.Lex();
+        ParseTiles(&tiles);
+      }
+
+      if (lexer_.GetKind() == TokKind::kIdent && lexer_.GetStrVal() == "E") {
+        // Parse element size in bits.
+        lexer_.Lex();
+        if (!ParseToken(TokKind::kLparen,
+                        StrCat("expects element size in bits to start with ",
+                               TokKindToString(TokKind::kLparen)))) {
+          return false;
+        }
+        if (!ParseInt64(&element_size_in_bits)) {
+          return false;
+        }
+        if (!ParseToken(TokKind::kRparen,
+                        StrCat("expects element size in bits to end with ",
+                               TokKindToString(TokKind::kRparen)))) {
+          return false;
+        }
+      }
+    }
+  }
+  if (!ParseToken(TokKind::kRbrace,
+                  StrCat("expects layout to end with ",
+                         TokKindToString(TokKind::kRbrace)))) {
     return false;
   }
-  *layout = LayoutUtil::MakeLayout(minor_to_major);
+
+  std::vector<Tile> vec_tiles(tiles.size());
+  for (int i = 0; i < tiles.size(); i++) {
+    vec_tiles[i] = Tile(tiles[i]);
+  }
+  *layout =
+      LayoutUtil::MakeLayout(minor_to_major, vec_tiles, element_size_in_bits);
   return true;
 }
 
@@ -3290,13 +3377,20 @@ bool HloParser::ParseShape(Shape* result) {
   //
   // The open brace could either be the start of a computation or the start of a
   // layout for the f32[123] shape. We consider it the start of a layout if the
-  // next token after the open brace is a integer
+  // next token after the open brace is an integer or a colon.
   if (lexer_.GetKind() == TokKind::kLbrace &&
-      lexer_.LookAhead() == TokKind::kInt) {
+      (lexer_.LookAhead() == TokKind::kInt ||
+       lexer_.LookAhead() == TokKind::kColon)) {
     Layout layout;
     if (!ParseLayout(&layout)) {
       return false;
     }
+    if (layout.minor_to_major_size() != result->rank()) {
+      return Error(
+          lexer_.GetLoc(),
+          StrFormat("Dimensions size is %ld, but minor to major size is %ld.",
+                    result->rank(), layout.minor_to_major_size()));
+    }
     *result->mutable_layout() = layout;
   }
   return true;
diff --git a/tensorflow/compiler/xla/service/hlo_parser_test.cc b/tensorflow/compiler/xla/service/hlo_parser_test.cc
index cd7effeae0..203a7dba22 100644
--- a/tensorflow/compiler/xla/service/hlo_parser_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser_test.cc
@@ -2574,6 +2574,60 @@ TEST_F(HloParserTest, ParseShapeStringWithLayout) {
       << "actual:   " << ShapeUtil::HumanString(actual);
 }
 
+TEST_F(HloParserTest, ParseShapeStringWithTilingLayout) {
+  // One tile.
+  string shape_string = "f32[123,456]{0,1:T(2,128)}";
+  TF_ASSERT_OK_AND_ASSIGN(Shape actual, ParseShape(shape_string));
+  Shape expected =
+      ShapeUtil::MakeShapeWithLayout(F32, {123, 456}, {0, 1}, {Tile({2, 128})});
+  EXPECT_EQ(expected, actual)
+      << "expected: " << ShapeUtil::HumanStringWithLayout(expected)
+      << "actual:   " << ShapeUtil::HumanStringWithLayout(actual);
+
+  // Tile with negative dimension size for combining dimensions.
+  shape_string = "f32[123,456,789]{0,1,2:T(2, * , 128)}";
+  TF_ASSERT_OK_AND_ASSIGN(actual, ParseShape(shape_string));
+  expected =
+      ShapeUtil::MakeShapeWithLayout(F32, {123, 456, 789}, {0, 1, 2},
+                                     {Tile({2, Tile::kCombineDimension, 128})});
+  EXPECT_EQ(expected, actual)
+      << "expected: " << ShapeUtil::HumanStringWithLayout(expected)
+      << "actual:   " << ShapeUtil::HumanStringWithLayout(actual);
+
+  // Two tiles.
+  shape_string = "bf16[123,456,789]{2,1,0:T(2,*,128)(2,1)}";
+  TF_ASSERT_OK_AND_ASSIGN(actual, ParseShape(shape_string));
+  expected = ShapeUtil::MakeShapeWithLayout(
+      BF16, {123, 456, 789}, {2, 1, 0},
+      {Tile({2, Tile::kCombineDimension, 128}), Tile({2, 1})});
+  EXPECT_EQ(expected, actual)
+      << "expected: " << ShapeUtil::HumanStringWithLayout(expected)
+      << "actual:   " << ShapeUtil::HumanStringWithLayout(actual);
+
+  // Tile with element size in bits.
+  shape_string = "pred[123,456]{1,0:T(2,128)E(1)}";
+  TF_ASSERT_OK_AND_ASSIGN(actual, ParseShape(shape_string));
+  expected = ShapeUtil::MakeShapeWithLayout(PRED, {123, 456}, {1, 0},
+                                            {Tile({2, 128})}, 1);
+  EXPECT_EQ(expected, actual)
+      << "expected: " << ShapeUtil::HumanStringWithLayout(expected)
+      << "actual:   " << ShapeUtil::HumanStringWithLayout(actual);
+
+  // Element size in bits without tile.
+  shape_string = "pred[123,456]{1,0:E(1)}";
+  TF_ASSERT_OK_AND_ASSIGN(actual, ParseShape(shape_string));
+  expected = ShapeUtil::MakeShapeWithLayout(PRED, {123, 456}, {1, 0}, {}, 1);
+  EXPECT_EQ(expected, actual)
+      << "expected: " << ShapeUtil::HumanStringWithLayout(expected)
+      << "actual:   " << ShapeUtil::HumanStringWithLayout(actual);
+
+  // Wrong minor_to_major.
+  shape_string = "f32[123,456,789]{1:T(2, * , 128)}";
+  auto result = ParseShape(shape_string);
+  ExpectHasSubstr(result.status().error_message(),
+                  "Dimensions size is 3, but minor to major size is 1.");
+}
+
 TEST_F(HloParserTest, ParseShapeStringWithSparseLayout) {
   string shape_string = "f32[123,456]sparse{10}";
   TF_ASSERT_OK_AND_ASSIGN(Shape actual, ParseShape(shape_string));
diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc
index 1ada4bc036..e6273c4e7f 100644
--- a/tensorflow/compiler/xla/shape_util.cc
+++ b/tensorflow/compiler/xla/shape_util.cc
@@ -89,7 +89,8 @@ namespace {
 // its Layout.
 StatusOr<Shape> MakeShapeWithLayoutInternal(
     PrimitiveType element_type, absl::Span<const int64> dimensions,
-    absl::Span<const int64> minor_to_major) {
+    absl::Span<const int64> minor_to_major, absl::Span<const Tile> tiles,
+    int64 element_size_in_bits) {
   if (dimensions.size() != minor_to_major.size()) {
     return InvalidArgument("Dimensions size is %ld, but layout size is %ld.",
                            dimensions.size(), minor_to_major.size());
@@ -100,11 +101,8 @@ StatusOr<Shape> MakeShapeWithLayoutInternal(
   }
   TF_ASSIGN_OR_RETURN(Shape shape,
                       ShapeUtil::MakeValidatedShape(element_type, dimensions));
-  auto min2maj = shape.mutable_layout()->mutable_minor_to_major();
-  min2maj->clear();
-  for (int64 value : minor_to_major) {
-    min2maj->push_back(value);
-  }
+  *shape.mutable_layout() =
+      LayoutUtil::MakeLayout(minor_to_major, tiles, element_size_in_bits);
   if (!shape.has_layout()) {
     return InvalidArgument("Shape has no layout.");
   }
@@ -189,8 +187,10 @@ StatusOr<Shape> MakeShapeWithLayoutInternal(
 
 /* static */ Shape ShapeUtil::MakeShapeWithLayout(
     PrimitiveType element_type, absl::Span<const int64> dimensions,
-    absl::Span<const int64> minor_to_major) {
-  return MakeShapeWithLayoutInternal(element_type, dimensions, minor_to_major)
+    absl::Span<const int64> minor_to_major, absl::Span<const Tile> tiles,
+    int64 element_size_in_bits) {
+  return MakeShapeWithLayoutInternal(element_type, dimensions, minor_to_major,
+                                     tiles, element_size_in_bits)
       .ValueOrDie();
 }
 
diff --git a/tensorflow/compiler/xla/shape_util.h b/tensorflow/compiler/xla/shape_util.h
index fb6da7460e..7f610a6085 100644
--- a/tensorflow/compiler/xla/shape_util.h
+++ b/tensorflow/compiler/xla/shape_util.h
@@ -398,7 +398,9 @@ class ShapeUtil {
   // Returns a value shape such that shape.has_layout().
   static Shape MakeShapeWithLayout(PrimitiveType element_type,
                                    absl::Span<const int64> dimensions,
-                                   absl::Span<const int64> minor_to_major);
+                                   absl::Span<const int64> minor_to_major,
+                                   absl::Span<const Tile> tiles = {},
+                                   int64 element_size_in_bits = 0);
 
   static Shape MakeShapeWithSparseLayout(PrimitiveType element_type,
                                          absl::Span<const int64> dimensions,
-- 
GitLab


From 11a08b0cdda81c95b50e84768487ddae1c0dd5de Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Fri, 8 Feb 2019 14:24:23 -0800
Subject: [PATCH 2198/2345] Use mkldnn sgemm instead of Eigen single threaded
 matmul if available

PiperOrigin-RevId: 233126334
---
 .../core/kernels/conv_grad_input_ops.cc       | 91 +++++++++++++++----
 1 file changed, 75 insertions(+), 16 deletions(-)

diff --git a/tensorflow/core/kernels/conv_grad_input_ops.cc b/tensorflow/core/kernels/conv_grad_input_ops.cc
index 5520cf7524..a9da05d4c4 100644
--- a/tensorflow/core/kernels/conv_grad_input_ops.cc
+++ b/tensorflow/core/kernels/conv_grad_input_ops.cc
@@ -184,6 +184,76 @@ struct LaunchXsmmBackwardInputConvolution<CPUDevice, float> {
 };
 #endif
 
+template <typename T>
+struct Conv2DCustomBackpropInputMatMulFunctor {
+  using MatrixMap = Eigen::Map<
+      Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>>;
+  using ConstMatrixMap = Eigen::Map<
+      const Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>>;
+
+  void operator()(OpKernelContext* ctx, const T* out_data, const T* filter_data,
+                  const int filter_total_size, const int output_image_size,
+                  const int dims_out_depth, T* im2col_buf) {
+    // Compute gradient into 'im2col_buf'.
+    MatrixMap C(im2col_buf, output_image_size, filter_total_size);
+
+    ConstMatrixMap A(out_data, output_image_size, dims_out_depth);
+    ConstMatrixMap B(filter_data, filter_total_size, dims_out_depth);
+
+    C.noalias() = A * B.transpose();
+  }
+};
+
+#if defined(TENSORFLOW_USE_MKLDNN_CONTRACTION_KERNEL)
+template <>
+struct Conv2DCustomBackpropInputMatMulFunctor<float> {
+  using T = float;
+
+  void operator()(OpKernelContext* ctx, const T* out_data, const T* filter_data,
+                  const int filter_total_size, const int output_image_size,
+                  const int dims_out_depth, T* im2col_buf) {
+    // Inputs are in RowMajor order, we "cheat" by swapping the LHS and RHS:
+    //   RowMajor: C   = A   * B
+    //   ColMajor: C^T = B^T * A^T
+    //
+    // Dimension names:
+    //   out_image_size    -> ois
+    //   filter_total_size -> fts
+    //   dims_out_depth    -> dod
+    //
+    // RowMajor:
+    //   im2col      = out_data    * filter_data^T
+    //   [ois x fts] = [ois x dod] * [fts x dod]^T
+    //
+    // ColMajor:
+    //   im2col^T    = filter_data *  out_data^T
+    //   [fts x ois] = [fts x dod] * [dod x ois]*
+
+    const int m = filter_total_size;
+    const int n = output_image_size;
+    const int k = dims_out_depth;  // contraction dim
+
+    const char transposeA = 'T';  // sgemm(A) == filter_data
+    const char transposeB = 'N';  // sgemm(B) == out_data
+
+    const int ldA = dims_out_depth;
+    const int ldB = dims_out_depth;
+    const int ldC = filter_total_size;
+
+    const float alpha = 1.0;
+    const float beta = 0.0;
+
+    mkldnn_status_t st =
+        mkldnn_sgemm(&transposeA, &transposeB, &m, &n, &k, &alpha, filter_data,
+                     &ldA, out_data, &ldB, &beta, im2col_buf, &ldC);
+
+    OP_REQUIRES(
+        ctx, st == 0,
+        errors::Internal("Failed to call mkldnn_sgemm. Error code: ", st));
+  }
+};
+#endif
+
 // Based on implementation written by Yangqing Jia (jiayq).
 template <typename Device, class T>
 class Conv2DCustomBackpropInputOp : public OpKernel {
@@ -417,21 +487,14 @@ class Conv2DCustomBackpropInputOp : public OpKernel {
         input_backprop_data += input_offset;
       }
     } else {
-      typedef Eigen::Map<
-          Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>>
-          MatrixMap;
-      typedef Eigen::Map<const Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic,
-                                             Eigen::RowMajor>>
-          ConstMatrixMap;
-
       for (int image_id = 0; image_id < dims.batch_size;
            image_id += shard_size) {
         const int shard_limit =
             std::min(static_cast<int>(shard_size),
                      static_cast<int>(dims.batch_size) - image_id);
 
-        auto shard = [&dims, &pad_top, &pad_left, &pad_bottom, &pad_right,
-                      &output_image_size, &filter_total_size,
+        auto shard = [&context, &dims, &pad_top, &pad_left, &pad_bottom,
+                      &pad_right, &output_image_size, &filter_total_size,
                       &input_backprop_data, &col_buffer_data,
                       &out_backprop_data, &filter_data, &input_offset,
                       &output_offset, &size_C](int64 start, int64 limit) {
@@ -440,13 +503,9 @@ class Conv2DCustomBackpropInputOp : public OpKernel {
             T* input_data = input_backprop_data + shard_id * input_offset;
             const T* out_data = out_backprop_data + shard_id * output_offset;
 
-            // Compute gradient into 'im2col_buf'.
-            MatrixMap C(im2col_buf, output_image_size, filter_total_size);
-
-            ConstMatrixMap A(out_data, output_image_size, dims.out_depth);
-            ConstMatrixMap B(filter_data, filter_total_size, dims.out_depth);
-
-            C.noalias() = A * B.transpose();
+            Conv2DCustomBackpropInputMatMulFunctor<T>()(
+                context, out_data, filter_data, filter_total_size,
+                output_image_size, dims.out_depth, im2col_buf);
 
             Col2im<T>(im2col_buf, dims.in_depth,
                       dims.spatial_dims[0].input_size,
-- 
GitLab


From 046f1b74a0c229580aa6c818b2a8610e5e0ec22c Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Fri, 8 Feb 2019 14:31:33 -0800
Subject: [PATCH 2199/2345] [Grappler] Do not prevent inlining of
 PartitionedCall ops with undefined side-effects order.

PiperOrigin-RevId: 233127565
---
 .../grappler/optimizers/function_optimizer.cc | 24 ++++---------------
 1 file changed, 5 insertions(+), 19 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index b722e6b332..ac0831b105 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -1398,14 +1398,10 @@ Status IsInlinableIndirectFunctionCall(const FunctionOptimizerContext& ctx,
 // Checks that all side-effects will be executed in well defined order. We do it
 // by checking if there is a path from stateful/dataset ops to one of the output
 // nodes.
-Status CheckThatSideEffectsWillExecute(
+void CheckThatSideEffectsWillExecute(
     const FunctionOptimizerContext& ctx,
     const GraphTopologyView& graph_topo_view,
     const absl::flat_hash_set<string> output_nodes) {
-  // In aggressive mode we just print a warning for side-effectful nodes that
-  // might not be executed after inlining.
-  const bool aggressive = ctx.opt_level() == RewriterConfig::AGGRESSIVE;
-
   for (const NodeDef& func_body_node : graph_topo_view.graph()->node()) {
     const bool node_must_execute =
         IsDataset(func_body_node) ||
@@ -1438,20 +1434,11 @@ Status CheckThatSideEffectsWillExecute(
                  TraversalDirection::kFollowOutputs, predicates, callbacks);
 
     if (!will_execute) {
-      const string error_message = absl::StrCat(
-          "Can't guarantee execution of a side-effectful node, that is not "
-          "reachable from function outputs. Function body node: ",
-          SummarizeNodeDef(func_body_node));
-
-      if (aggressive) {
-        LOG(WARNING) << error_message;
-      } else {
-        return errors::Internal(error_message);
-      }
+      VLOG(1) << "Can't guarantee execution of a side-effectful node, that is "
+                 "not reachable from function outputs. Function body node: "
+              << SummarizeNodeDef(func_body_node);
     }
   }
-
-  return Status::OK();
 }
 
 Status PlaceInlinedFunctionBody(
@@ -1803,8 +1790,7 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
   // input nodes connected to nodes in the optimized graph).
   GraphTopologyView placed_topo_view(/*skip_invalid_edges=*/true);
   TF_RETURN_IF_ERROR(placed_topo_view.InitializeFromGraph(placed_graph_def));
-  TF_RETURN_IF_ERROR(CheckThatSideEffectsWillExecute(*ctx, placed_topo_view,
-                                                     inlined_output_nodes));
+  CheckThatSideEffectsWillExecute(*ctx, placed_topo_view, inlined_output_nodes);
 
   // ------------------------------------------------------------------------ //
   // Move all the nodes to the optimized graph after successful preprocessing.
-- 
GitLab


From 4d6fd23bcbc3007172f0dc84e3116fd4497b0e50 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 8 Feb 2019 14:33:47 -0800
Subject: [PATCH 2200/2345] Add string support to generate_examples.py

PiperOrigin-RevId: 233127983
---
 .../interpreter_wrapper/python_utils.cc       | 16 ++++++
 .../python/interpreter_wrapper/python_utils.h |  7 +++
 tensorflow/lite/testing/BUILD                 | 27 +++++++++
 tensorflow/lite/testing/generate_examples.py  | 56 +++++++++++++------
 tensorflow/lite/testing/string_util.cc        | 45 +++++++++++++++
 tensorflow/lite/testing/string_util.h         | 33 +++++++++++
 tensorflow/lite/testing/string_util.i         | 31 ++++++++++
 7 files changed, 198 insertions(+), 17 deletions(-)
 create mode 100644 tensorflow/lite/testing/string_util.cc
 create mode 100644 tensorflow/lite/testing/string_util.h
 create mode 100644 tensorflow/lite/testing/string_util.i

diff --git a/tensorflow/lite/python/interpreter_wrapper/python_utils.cc b/tensorflow/lite/python/interpreter_wrapper/python_utils.cc
index 526488a9f9..e3d713630f 100644
--- a/tensorflow/lite/python/interpreter_wrapper/python_utils.cc
+++ b/tensorflow/lite/python/interpreter_wrapper/python_utils.cc
@@ -160,5 +160,21 @@ bool FillStringBufferWithPyArray(PyObject* value,
   return false;
 }
 
+int ConvertFromPyString(PyObject* obj, char** data, Py_ssize_t* length) {
+#if PY_MAJOR_VERSION >= 3
+  return PyBytes_AsStringAndSize(obj, data, length);
+#else
+  return PyString_AsStringAndSize(obj, data, length);
+#endif
+}
+
+PyObject* ConvertToPyString(const char* data, size_t length) {
+#if PY_MAJOR_VERSION >= 3
+  return PyBytes_FromStringAndSize(data, length);
+#else
+  return PyString_FromStringAndSize(data, length);
+#endif
+}
+
 }  // namespace python_utils
 }  // namespace tflite
diff --git a/tensorflow/lite/python/interpreter_wrapper/python_utils.h b/tensorflow/lite/python/interpreter_wrapper/python_utils.h
index 5ffd231a89..f4677378cb 100644
--- a/tensorflow/lite/python/interpreter_wrapper/python_utils.h
+++ b/tensorflow/lite/python/interpreter_wrapper/python_utils.h
@@ -23,6 +23,10 @@ limitations under the License.
 namespace tflite {
 namespace python_utils {
 
+struct PyDecrefDeleter {
+  void operator()(PyObject* p) const { Py_DECREF(p); }
+};
+
 int TfLiteTypeToPyArrayType(TfLiteType tf_lite_type);
 
 TfLiteType TfLiteTypeFromPyArray(PyArrayObject* array);
@@ -30,6 +34,9 @@ TfLiteType TfLiteTypeFromPyArray(PyArrayObject* array);
 bool FillStringBufferWithPyArray(PyObject* value,
                                  DynamicBuffer* dynamic_buffer);
 
+int ConvertFromPyString(PyObject* obj, char** data, Py_ssize_t* length);
+PyObject* ConvertToPyString(const char* data, size_t length);
+
 }  // namespace python_utils
 }  // namespace tflite
 #endif  // TENSORFLOW_LITE_PYTHON_INTERPRETER_WRAPPER_PYTHON_UTILS_H_
diff --git a/tensorflow/lite/testing/BUILD b/tensorflow/lite/testing/BUILD
index e8b14cd2a7..68512b952a 100644
--- a/tensorflow/lite/testing/BUILD
+++ b/tensorflow/lite/testing/BUILD
@@ -10,6 +10,7 @@ load(
     "generated_test_models_all",
 )
 load("//tensorflow/lite:special_rules.bzl", "tflite_portable_test_suite")
+load("//tensorflow:tensorflow.bzl", "tf_py_wrap_cc")
 load(
     "//tensorflow:tensorflow.bzl",
     "tf_cc_binary",
@@ -78,6 +79,7 @@ py_binary(
     srcs_version = "PY2AND3",
     deps = [
         ":generate_examples_report",
+        ":string_util_wrapper",
         "//tensorflow:tensorflow_py",
         "//tensorflow/python:graph_util",
         "//third_party/py/numpy",
@@ -392,4 +394,29 @@ tf_cc_binary(
     ],
 )
 
+cc_library(
+    name = "string_util_lib",
+    srcs = ["string_util.cc"],
+    hdrs = ["string_util.h"],
+    deps = [
+        "//tensorflow/lite:string_util",
+        "//tensorflow/lite/python/interpreter_wrapper:numpy",
+        "//tensorflow/lite/python/interpreter_wrapper:python_utils",
+        "//third_party/py/numpy:headers",
+        "//third_party/python_runtime:headers",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+tf_py_wrap_cc(
+    name = "string_util_wrapper",
+    srcs = [
+        "string_util.i",
+    ],
+    deps = [
+        ":string_util_lib",
+        "//third_party/python_runtime:headers",
+    ],
+)
+
 tflite_portable_test_suite()
diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index 23191edb17..5cd47eb1af 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -36,6 +36,7 @@ import operator
 import os
 import random
 import re
+import string
 import sys
 import tempfile
 import traceback
@@ -52,6 +53,7 @@ import tensorflow as tf
 from google.protobuf import text_format
 # TODO(aselle): switch to TensorFlow's resource_loader
 from tensorflow.lite.testing import generate_examples_report as report_lib
+from tensorflow.lite.testing import string_util_wrapper
 from tensorflow.python.framework import graph_util as tf_graph_util
 from tensorflow.python.ops import rnn
 
@@ -163,6 +165,16 @@ def toco_options(data_types,
   return s
 
 
+def format_result(t):
+  """Convert a tensor to a format that can be used in test specs."""
+  if np.issubdtype(t.dtype, np.number):
+    # Output 9 digits after the point to ensure the precision is good enough.
+    values = ["{:.9f}".format(value) for value in list(t.flatten())]
+    return ",".join(values)
+  else:
+    return string_util_wrapper.SerializeAsHexString(t.flatten())
+
+
 def write_examples(fp, examples):
   """Given a list `examples`, write a text format representation.
 
@@ -179,9 +191,7 @@ def write_examples(fp, examples):
     """Write tensor in file format supported by TFLITE example."""
     fp.write("dtype,%s\n" % x.dtype)
     fp.write("shape," + ",".join(map(str, x.shape)) + "\n")
-    # Output 9 digits after the point to ensure the precision is good enough.
-    values = ["{:.9f}".format(value) for value in list(x.flatten())]
-    fp.write("values," + ",".join(values) + "\n")
+    fp.write("values," + format_result(x) + "\n")
 
   fp.write("test_cases,%d\n" % len(examples))
   for example in examples:
@@ -214,11 +224,9 @@ def write_test_cases(fp, model_name, examples):
     fp.write("invoke {\n")
 
     for t in example["inputs"]:
-      values = ["{:.9f}".format(value) for value in list(t.flatten())]
-      fp.write("  input: \"" + ",".join(values) + "\"\n")
+      fp.write("  input: \"" + format_result(t) + "\"\n")
     for t in example["outputs"]:
-      values = ["{:.9f}".format(value) for value in list(t.flatten())]
-      fp.write("  output: \"" + ",".join(values) + "\"\n")
+      fp.write("  output: \"" + format_result(t) + "\"\n")
     fp.write("}\n")
 
 
@@ -230,6 +238,7 @@ _TF_TYPE_INFO = {
     tf.int16: (np.int16, "QUANTIZED_INT16"),
     tf.int64: (np.int64, "INT64"),
     tf.bool: (np.bool, "BOOL"),
+    tf.string: (np.string_, "STRING"),
 }
 
 
@@ -245,6 +254,10 @@ def create_tensor_data(dtype, shape, min_value=-100, max_value=100):
     value = np.random.randint(min_value, max_value+1, shape)
   elif dtype == tf.bool:
     value = np.random.choice([True, False], size=shape)
+  elif dtype == np.string_:
+    # Not the best strings, but they will do for some basic testing.
+    letters = list(string.ascii_uppercase)
+    return np.random.choice(letters, size=shape).astype(dtype)
   return np.dtype(dtype).type(value) if np.isscalar(value) else value.astype(
       dtype)
 
@@ -1294,16 +1307,25 @@ def make_squared_difference_tests(zip_path):
 def make_gather_tests(zip_path):
   """Make a set of tests to do gather."""
 
-  test_parameters = [{
-      # TODO(mgubin): add string tests when they are supported by Toco.
-      # TODO(mgubin): add tests for Nd indices when they are supported by
-      # TfLite.
-      "params_dtype": [tf.float32, tf.int32, tf.int64],
-      "params_shape": [[10], [1, 2, 20]],
-      "indices_dtype": [tf.int32, tf.int64],
-      "indices_shape": [[3], [5]],
-      "axis": [-1, 0, 1],
-  }]
+  test_parameters = [
+      {
+          # TODO(b/110347007): add tests for Nd indices when they are supported
+          # by TfLite.
+          "params_dtype": [tf.float32, tf.int32, tf.int64],
+          "params_shape": [[10], [1, 2, 20]],
+          "indices_dtype": [tf.int32, tf.int64],
+          "indices_shape": [[3], [5]],
+          "axis": [-1, 0, 1],
+      },
+      {
+          # TODO(b/123895910): add Nd support for strings.
+          "params_dtype": [tf.string],
+          "params_shape": [[8]],
+          "indices_dtype": [tf.int32],
+          "indices_shape": [[3]],
+          "axis": [0],
+      }
+  ]
 
   def build_graph(parameters):
     """Build the gather op testing graph."""
diff --git a/tensorflow/lite/testing/string_util.cc b/tensorflow/lite/testing/string_util.cc
new file mode 100644
index 0000000000..cf9d508764
--- /dev/null
+++ b/tensorflow/lite/testing/string_util.cc
@@ -0,0 +1,45 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <memory>
+
+#include "tensorflow/lite/testing/string_util.h"
+
+#include "absl/strings/escaping.h"
+#include "tensorflow/lite/python/interpreter_wrapper/numpy.h"
+#include "tensorflow/lite/python/interpreter_wrapper/python_utils.h"
+#include "tensorflow/lite/string_util.h"
+
+namespace tflite {
+namespace testing {
+namespace python {
+
+PyObject* SerializeAsHexString(PyObject* value) {
+  DynamicBuffer dynamic_buffer;
+  if (!python_utils::FillStringBufferWithPyArray(value, &dynamic_buffer)) {
+    return nullptr;
+  }
+
+  char* char_buffer = nullptr;
+  size_t size = dynamic_buffer.WriteToBuffer(&char_buffer);
+  string s = absl::BytesToHexString({char_buffer, size});
+  free(char_buffer);
+
+  return python_utils::ConvertToPyString(s.data(), s.size());
+}
+
+}  // namespace python
+}  // namespace testing
+}  // namespace tflite
diff --git a/tensorflow/lite/testing/string_util.h b/tensorflow/lite/testing/string_util.h
new file mode 100644
index 0000000000..56c024d918
--- /dev/null
+++ b/tensorflow/lite/testing/string_util.h
@@ -0,0 +1,33 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TESTING_STRING_UTIL_H_
+#define TENSORFLOW_LITE_TESTING_STRING_UTIL_H_
+
+#include <Python.h>
+#include <string>
+
+namespace tflite {
+namespace testing {
+namespace python {
+
+// Take a python string array, convert it to TF Lite dynamic buffer format and
+// serialize it as a HexString.
+PyObject* SerializeAsHexString(PyObject* value);
+
+}  // namespace python
+}  // namespace testing
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TESTING_STRING_UTIL_H_
diff --git a/tensorflow/lite/testing/string_util.i b/tensorflow/lite/testing/string_util.i
new file mode 100644
index 0000000000..574abb7965
--- /dev/null
+++ b/tensorflow/lite/testing/string_util.i
@@ -0,0 +1,31 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+%{
+
+#define SWIG_FILE_WITH_INIT
+#include "tensorflow/lite/testing/string_util.h"
+
+%}
+
+namespace tflite {
+namespace testing {
+namespace python {
+
+PyObject* SerializeAsHexString(PyObject* string_tensor);
+
+}  // namespace python
+}  // namespace testing
+}  // namespace tflite
-- 
GitLab


From 39a6469767425876c10d1064b66fb66c39586f5d Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Fri, 8 Feb 2019 14:42:22 -0800
Subject: [PATCH 2201/2345] Don't use magic string for API symbol "tensorflow".

PiperOrigin-RevId: 233129516
---
 tensorflow/python/ops/ragged/ragged_dispatch.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/ragged/ragged_dispatch.py b/tensorflow/python/ops/ragged/ragged_dispatch.py
index 0c9e6efe8b..3bda777482 100644
--- a/tensorflow/python/ops/ragged/ragged_dispatch.py
+++ b/tensorflow/python/ops/ragged/ragged_dispatch.py
@@ -470,7 +470,8 @@ def register_dispatchers():
       _BINARY_ELEMENTWISE_OPS + [x[0] for x in _RAGGED_DISPATCH_OPS])
   for op in op_list:
     _, undecorated_op = tf_decorator.unwrap(op)
-    if not hasattr(undecorated_op, tf_export.API_ATTRS['tensorflow'].names):
+    if not hasattr(undecorated_op,
+                   tf_export.API_ATTRS[tf_export.TENSORFLOW_API_NAME].names):
       raise AssertionError('Expected %s to be an exported symbol '
                            '(while adding a RaggedTensor dispatcher)')
 
-- 
GitLab


From 882bf2ecae29d8f7c22942ac40110c13e1321380 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Fri, 8 Feb 2019 14:57:56 -0800
Subject: [PATCH 2202/2345] Pack function call arguments and keywords into
 containers, to avoid clashing with keywords of converted_call itself. It
 might have been easier to just give converted_call weird names, but this is
 more robust.

PiperOrigin-RevId: 233132284
---
 .../python/autograph/converters/call_trees.py |  44 +++++-
 .../autograph/converters/call_trees_test.py   |  39 +++++-
 tensorflow/python/autograph/impl/api.py       |   4 +-
 tensorflow/python/autograph/impl/api_test.py  | 129 +++++++++---------
 tensorflow/python/framework/func_graph.py     |   2 +-
 5 files changed, 143 insertions(+), 75 deletions(-)

diff --git a/tensorflow/python/autograph/converters/call_trees.py b/tensorflow/python/autograph/converters/call_trees.py
index 04439ba516..e53eeb86d9 100644
--- a/tensorflow/python/autograph/converters/call_trees.py
+++ b/tensorflow/python/autograph/converters/call_trees.py
@@ -26,6 +26,7 @@ import gast
 
 from tensorflow.python.autograph.core import converter
 from tensorflow.python.autograph.pyct import anno
+from tensorflow.python.autograph.pyct import ast_util
 from tensorflow.python.autograph.pyct import parser
 from tensorflow.python.autograph.pyct import templates
 
@@ -62,9 +63,6 @@ class CallTreeTransformer(converter.Base):
         not self.ctx.program.options.uses(converter.Feature.BUILTIN_FUNCTIONS)):
       return self.generic_visit(node)
 
-    template = """
-      ag__.converted_call(func, owner, options, args)
-    """
     if isinstance(node.func, gast.Attribute):
       func = gast.Str(node.func.attr)
       owner = node.func.value
@@ -72,6 +70,41 @@ class CallTreeTransformer(converter.Base):
       func = node.func
       owner = parser.parse_expression('None')
 
+    starred_arg = None
+    normal_args = []
+    for a in node.args:
+      if isinstance(a, gast.Starred):
+        assert starred_arg is None, 'Multiple *args should be impossible.'
+        starred_arg = a
+      else:
+        normal_args.append(a)
+    if starred_arg is None:
+      args = templates.replace_as_expression('(args,)', args=normal_args)
+    else:
+      args = templates.replace_as_expression(
+          '(args,) + stararg',
+          stararg=starred_arg.value,
+          args=normal_args)
+
+    kwargs_arg = None
+    normal_keywords = []
+    for k in node.keywords:
+      if k.arg is None:
+        assert kwargs_arg is None, 'Multiple **kwargs should be impossible.'
+        kwargs_arg = k
+      else:
+        normal_keywords.append(k)
+    if kwargs_arg is None:
+      kwargs = ast_util.keywords_to_dict(normal_keywords)
+    else:
+      kwargs = templates.replace_as_expression(
+          'dict(kwargs, **keywords)',
+          kwargs=kwargs_arg.value,
+          keywords=ast_util.keywords_to_dict(normal_keywords))
+
+    template = """
+      ag__.converted_call(func, owner, options, args, kwargs)
+    """
     new_call = templates.replace_as_expression(
         template,
         func=func,
@@ -79,9 +112,8 @@ class CallTreeTransformer(converter.Base):
         options=self.ctx.program.options.to_ast(
             self.ctx,
             internal_convert_user_code=self.ctx.program.options.recursive),
-        args=node.args)
-    # TODO(mdan): Improve the template mechanism to better support this.
-    new_call.keywords = node.keywords
+        args=args,
+        kwargs=kwargs)
 
     return new_call
 
diff --git a/tensorflow/python/autograph/converters/call_trees_test.py b/tensorflow/python/autograph/converters/call_trees_test.py
index 6ee56bf6be..47acda9b46 100644
--- a/tensorflow/python/autograph/converters/call_trees_test.py
+++ b/tensorflow/python/autograph/converters/call_trees_test.py
@@ -34,7 +34,40 @@ class CallTreesTest(converter_testing.TestCase):
       self.assertEquals(
           result.test_fn(None),
           converter_testing.RESULT_OF_MOCK_CONVERTED_CALL + 3)
-      self.assertListEqual(self.dynamic_calls, [()])
+      self.assertListEqual(self.dynamic_calls, [((), {})])
+
+  def test_function_with_kwarg(self):
+
+    def test_fn(f, a, b):
+      return f(a, c=b) + 3
+
+    with self.converted(test_fn, call_trees, {}) as result:
+      self.assertEquals(
+          result.test_fn(None, 1, 2),
+          converter_testing.RESULT_OF_MOCK_CONVERTED_CALL + 3)
+      self.assertListEqual(self.dynamic_calls, [((1,), {'c': 2})])
+
+  def test_function_with_kwargs_starargs(self):
+
+    def test_fn(f, a, *args, **kwargs):
+      return f(a, *args, **kwargs) + 5
+
+    with self.converted(test_fn, call_trees, {}) as result:
+      self.assertEquals(
+          result.test_fn(None, 1, *[2, 3], **{'b': 4, 'c': 5}),
+          converter_testing.RESULT_OF_MOCK_CONVERTED_CALL + 5)
+      self.assertListEqual(self.dynamic_calls, [((1, 2, 3), {'b': 4, 'c': 5})])
+
+  def test_function_with_kwargs_keywords(self):
+
+    def test_fn(f, a, b, **kwargs):
+      return f(a, b=b, **kwargs) + 5
+
+    with self.converted(test_fn, call_trees, {}) as result:
+      self.assertEquals(
+          result.test_fn(None, 1, 2, **{'c': 3}),
+          converter_testing.RESULT_OF_MOCK_CONVERTED_CALL + 5)
+      self.assertListEqual(self.dynamic_calls, [((1,), {'b': 2, 'c': 3})])
 
   def test_class_method(self):
 
@@ -47,7 +80,7 @@ class CallTreesTest(converter_testing.TestCase):
     with self.converted(TestClass.test_method, call_trees, {}) as result:
       self.assertEquals(converter_testing.RESULT_OF_MOCK_CONVERTED_CALL + 1,
                         result.test_method(tc, 1))
-      self.assertListEqual(self.dynamic_calls, [(1,)])
+      self.assertListEqual(self.dynamic_calls, [((1,), {})])
 
   def test_object_method(self):
 
@@ -60,7 +93,7 @@ class CallTreesTest(converter_testing.TestCase):
     with self.converted(tc.test_method, call_trees, {}) as result:
       self.assertEquals(converter_testing.RESULT_OF_MOCK_CONVERTED_CALL + 1,
                         result.test_method(tc, 1))
-      self.assertListEqual(self.dynamic_calls, [(1,)])
+      self.assertListEqual(self.dynamic_calls, [((1,), {})])
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/autograph/impl/api.py b/tensorflow/python/autograph/impl/api.py
index 0f874a138a..471ba3b795 100644
--- a/tensorflow/python/autograph/impl/api.py
+++ b/tensorflow/python/autograph/impl/api.py
@@ -90,7 +90,7 @@ def convert(
               verbose=verbose,
               force_conversion=True,
               optional_features=optional_features,
-          ), *args, **kwargs)
+          ), args, kwargs)
 
     wrapper = tf_decorator.make_decorator(f, wrapper)
 
@@ -162,7 +162,7 @@ def do_not_convert(run_as=RunMode.GRAPH, return_dtypes=None):
   return decorator
 
 
-def converted_call(f, owner, options, *args, **kwargs):
+def converted_call(f, owner, options, args, kwargs):
   """Compiles a function call inline. For internal use only."""
   logging.log(1,
               'Converted call: %s; owner: %s\n    args: %s\n    kwargs: %s\n',
diff --git a/tensorflow/python/autograph/impl/api_test.py b/tensorflow/python/autograph/impl/api_test.py
index 73e470a000..3e17669f0a 100644
--- a/tensorflow/python/autograph/impl/api_test.py
+++ b/tensorflow/python/autograph/impl/api_test.py
@@ -195,18 +195,17 @@ class ApiTest(test.TestCase):
       def test_method(self, x, s, a):
         while tf.reduce_sum(x) > s:
           x //= api.converted_call(self.called_member, None,
-                                   converter.ConversionOptions(), self, a)
+                                   converter.ConversionOptions(), (self, a), {})
         return x
 
     tc = TestClass()
-    with self.cached_session() as sess:
-      x = tc.test_method(
-          constant_op.constant([2, 4]), constant_op.constant(1),
-          constant_op.constant(-2))
-      self.assertListEqual([0, 1], self.evaluate(x).tolist())
+    x = tc.test_method(
+        constant_op.constant([2, 4]), constant_op.constant(1),
+        constant_op.constant(-2))
+    self.assertListEqual([0, 1], self.evaluate(x).tolist())
 
   def test_converted_call_builtin(self):
-    x = api.converted_call(range, None, converter.ConversionOptions(), 3)
+    x = api.converted_call(range, None, converter.ConversionOptions(), (3,), {})
     self.assertEqual((0, 1, 2), tuple(x))
 
   def test_converted_call_function(self):
@@ -216,10 +215,9 @@ class ApiTest(test.TestCase):
         return -x
       return x
 
-    with self.cached_session() as sess:
-      x = api.converted_call(test_fn, None, converter.ConversionOptions(),
-                             constant_op.constant(-1))
-      self.assertEqual(1, self.evaluate(x))
+    x = api.converted_call(test_fn, None, converter.ConversionOptions(),
+                           (constant_op.constant(-1),), {})
+    self.assertEqual(1, self.evaluate(x))
 
   @test_util.run_v1_only('b/120545219')
   def test_converted_call_functools_partial(self):
@@ -230,16 +228,14 @@ class ApiTest(test.TestCase):
       return x, y, z
 
     x = api.converted_call(
-        functools.partial(test_fn, constant_op.constant(-1), z=-3),
-        None, converter.ConversionOptions(),
-        constant_op.constant(-2))
+        functools.partial(test_fn, constant_op.constant(-1), z=-3), None,
+        converter.ConversionOptions(), (constant_op.constant(-2),), {})
     self.assertEqual((1, 2, 3), self.evaluate(x))
 
     x = api.converted_call(
         functools.partial(
-            functools.partial(test_fn, constant_op.constant(-1)), z=-3),
-        None, converter.ConversionOptions(),
-        constant_op.constant(-2))
+            functools.partial(test_fn, constant_op.constant(-1)), z=-3), None,
+        converter.ConversionOptions(), (constant_op.constant(-2),), {})
     self.assertEqual((1, 2, 3), self.evaluate(x))
 
   def test_converted_call_method_explicit_owner(self):
@@ -262,11 +258,10 @@ class ApiTest(test.TestCase):
           return -self.x
         return self.x
 
-    with self.cached_session() as sess:
-      tc = TestClass(constant_op.constant(-1))
-      x = api.converted_call(tc.test_method, None,
-                             converter.ConversionOptions(), tc)
-      self.assertEqual(1, self.evaluate(x))
+    tc = TestClass(constant_op.constant(-1))
+    x = api.converted_call(tc.test_method, None, converter.ConversionOptions(),
+                           (tc,), {})
+    self.assertEqual(1, self.evaluate(x))
 
   def test_converted_call_method_converts_recursively(self):
 
@@ -285,7 +280,8 @@ class ApiTest(test.TestCase):
 
     tc = TestClass(constant_op.constant(-1))
     x = api.converted_call(tc.test_method, None,
-                           converter.ConversionOptions(recursive=True), tc)
+                           converter.ConversionOptions(recursive=True), (tc,),
+                           {})
     self.assertEqual(1, self.evaluate(x))
 
   def test_converted_call_method_by_class(self):
@@ -300,11 +296,10 @@ class ApiTest(test.TestCase):
           return -self.x
         return self.x
 
-    with self.cached_session() as sess:
-      tc = TestClass(constant_op.constant(-1))
-      x = api.converted_call(TestClass.test_method, None,
-                             converter.ConversionOptions(), tc)
-      self.assertEqual(1, self.evaluate(x))
+    tc = TestClass(constant_op.constant(-1))
+    x = api.converted_call(TestClass.test_method, None,
+                           converter.ConversionOptions(), (tc,), {})
+    self.assertEqual(1, self.evaluate(x))
 
   def test_converted_call_callable_object(self):
 
@@ -318,10 +313,9 @@ class ApiTest(test.TestCase):
           return -self.x
         return self.x
 
-    with self.cached_session() as sess:
-      tc = TestClass(constant_op.constant(-1))
-      x = api.converted_call(tc, None, converter.ConversionOptions())
-      self.assertEqual(1, self.evaluate(x))
+    tc = TestClass(constant_op.constant(-1))
+    x = api.converted_call(tc, None, converter.ConversionOptions(), (), {})
+    self.assertEqual(1, self.evaluate(x))
 
   @test_util.run_deprecated_v1
   def test_converted_call_constructor(self):
@@ -337,7 +331,7 @@ class ApiTest(test.TestCase):
         return self.x
 
     tc = api.converted_call(TestClass, None, converter.ConversionOptions(),
-                            constant_op.constant(-1))
+                            (constant_op.constant(-1),), {})
     # tc is still a TestClass - constructors are whitelisted.
     # TODO(b/124016764): Support this use case.
     # The error below is specific to the `if` statement not being converted.
@@ -350,15 +344,14 @@ class ApiTest(test.TestCase):
     def f(x):
       return x == 0
 
-    with self.cached_session() as sess:
-      x = api.converted_call(f, None, converter.ConversionOptions(),
-                             constant_op.constant(0))
-      self.assertTrue(self.evaluate(x))
+    x = api.converted_call(f, None, converter.ConversionOptions(),
+                           (constant_op.constant(0),), {})
+    self.assertTrue(self.evaluate(x))
 
-      converted_f = api.to_graph(f)
-      x = api.converted_call(converted_f, None, converter.ConversionOptions(),
-                             constant_op.constant(0))
-      self.assertTrue(self.evaluate(x))
+    converted_f = api.to_graph(f)
+    x = api.converted_call(converted_f, None, converter.ConversionOptions(),
+                           (constant_op.constant(0),), {})
+    self.assertTrue(self.evaluate(x))
 
   def test_converted_call_then_already_converted_dynamic(self):
 
@@ -373,7 +366,7 @@ class ApiTest(test.TestCase):
       return g(x)
 
     x = api.converted_call(f, None, converter.ConversionOptions(),
-                           g, constant_op.constant(1))
+                           (g, constant_op.constant(1)), {})
     self.assertEqual(self.evaluate(x), 1)
 
   @test_util.run_deprecated_v1
@@ -387,10 +380,10 @@ class ApiTest(test.TestCase):
     # f should not be converted, causing len to error out.
     with self.assertRaisesRegexp(Exception,
                                  'object of type \'Tensor\' has no len()'):
-      api.converted_call(f, None, opts, constant_op.constant([0]))
+      api.converted_call(f, None, opts, (constant_op.constant([0]),), {})
 
     # len on the other hand should work fine.
-    x = api.converted_call(len, None, opts, constant_op.constant([0]))
+    x = api.converted_call(len, None, opts, (constant_op.constant([0]),), {})
     # The constant has static shape so the result is a primitive not a Tensor.
     self.assertEqual(x, 1)
 
@@ -403,11 +396,10 @@ class ApiTest(test.TestCase):
     ])
 
     x = api.converted_call(model.call, None, opts,
-                           constant_op.constant([[0.0]]), training=True)
+                           (constant_op.constant([[0.0]]),), {'training': True})
 
-    with self.cached_session() as sess:
-      self.evaluate(variables.global_variables_initializer())
-      self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
+    self.evaluate(variables.global_variables_initializer())
+    self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
 
   def test_converted_call_whitelisted_method_extra_self(self):
 
@@ -418,11 +410,11 @@ class ApiTest(test.TestCase):
     ])
 
     x = api.converted_call(model.call, None, opts,
-                           model, constant_op.constant([[0.0]]), training=True)
+                           (model, constant_op.constant([[0.0]])),
+                           {'training': True})
 
-    with self.cached_session() as sess:
-      self.evaluate(variables.global_variables_initializer())
-      self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
+    self.evaluate(variables.global_variables_initializer())
+    self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
 
   def test_converted_call_whitelisted_method_via_owner(self):
 
@@ -433,18 +425,17 @@ class ApiTest(test.TestCase):
     ])
 
     x = api.converted_call('call', model, opts,
-                           constant_op.constant([[0.0]]), training=True)
+                           (constant_op.constant([[0.0]]),), {'training': True})
 
-    with self.cached_session() as sess:
-      self.evaluate(variables.global_variables_initializer())
-      self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
+    self.evaluate(variables.global_variables_initializer())
+    self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
 
   def test_converted_call_namedtuple(self):
 
     opts = converter.ConversionOptions()
 
     x = api.converted_call(collections.namedtuple, None, opts,
-                           'TestNamedtuple', ('a', 'b'))
+                           ('TestNamedtuple', ('a', 'b')), {})
 
     self.assertTrue(inspect_utils.isnamedtuple(x))
 
@@ -452,8 +443,8 @@ class ApiTest(test.TestCase):
 
     opts = converter.ConversionOptions()
 
-    x = api.converted_call('namedtuple', collections, opts,
-                           'TestNamedtuple', ('a', 'b'))
+    x = api.converted_call('namedtuple', collections, opts, ('TestNamedtuple',
+                                                             ('a', 'b')), {})
 
     self.assertTrue(inspect_utils.isnamedtuple(x))
 
@@ -463,11 +454,10 @@ class ApiTest(test.TestCase):
 
     l = lambda x: x == 0
 
-    x = api.converted_call(l, None, opts, constant_op.constant(0))
+    x = api.converted_call(l, None, opts, (constant_op.constant(0),), {})
 
-    with self.cached_session() as sess:
-      self.evaluate(variables.global_variables_initializer())
-      self.assertAllEqual(True, self.evaluate(x))
+    self.evaluate(variables.global_variables_initializer())
+    self.assertAllEqual(True, self.evaluate(x))
 
   @test_util.run_deprecated_v1
   def test_to_graph_basic(self):
@@ -513,6 +503,19 @@ class ApiTest(test.TestCase):
     # TODO(b/122368197): This should be the constant 5!
     self.assertEqual(2, testing_global_numeric)
 
+  def test_to_graph_with_kwargs_clashing_converted_call(self):
+
+    def f(**kwargs):
+      return kwargs['f'] + kwargs['owner']
+
+    def test_fn():
+      # These arg names intentionally match converted_call's
+      return f(f=1, owner=2)
+
+    compiled_fn = api.to_graph(test_fn)
+
+    self.assertEqual(compiled_fn(), 3)
+
   def test_to_code_basic(self):
 
     def test_fn(x, s):
diff --git a/tensorflow/python/framework/func_graph.py b/tensorflow/python/framework/func_graph.py
index 465d97999a..60d97d50d7 100644
--- a/tensorflow/python/framework/func_graph.py
+++ b/tensorflow/python/framework/func_graph.py
@@ -543,7 +543,7 @@ def func_graph_from_py_func(name,
                   strip_decorators=(def_function.function,),
                   optional_features=autograph_options,
                   force_conversion=True,
-              ), *args, **kwargs)
+              ), args, kwargs)
 
         # Wrapping around a decorator allows checks like tf_inspect.getargspec
         # to be accurate.
-- 
GitLab


From d94afafddb45c17bd932d5802ab7c50a9833d705 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Fri, 8 Feb 2019 14:59:35 -0800
Subject: [PATCH 2203/2345] Update 'tf.losses'/'tf.metrics' namespace in
 examples to 'tf.keras.losses'/'tf.keras.metrics'

PiperOrigin-RevId: 233132595
---
 tensorflow/python/keras/losses.py             | 92 +++++++++----------
 tensorflow/python/keras/metrics.py            | 76 +++++++--------
 .../python/keras/optimizer_v2/optimizer_v2.py |  4 +-
 tensorflow/python/keras/utils/losses_utils.py |  4 +-
 4 files changed, 88 insertions(+), 88 deletions(-)

diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index f694384075..1bffcab5c7 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -54,8 +54,8 @@ class Loss(object):
   ```
 
   Args:
-    reduction: Type of `tf.losses.Reduction` to apply to loss. Default value is
-      `SUM_OVER_BATCH_SIZE`.
+    reduction: (Optional) Type of `tf.keras.losses.Reduction` to apply to loss.
+      Default value is `SUM_OVER_BATCH_SIZE`.
     name: Optional name for the op.
   """
 
@@ -130,7 +130,7 @@ class LossFunctionWrapper(Loss):
   Args:
     fn: The loss function to wrap, with signature `fn(y_true, y_pred,
       **kwargs)`.
-    reduction: (Optional) Type of `tf.losses.Reduction` to apply to loss.
+    reduction: (Optional) Type of `tf.keras.losses.Reduction` to apply to loss.
       Default value is `SUM_OVER_BATCH_SIZE`.
     name: (Optional) name for the loss.
     **kwargs: The keyword arguments that are passed on to `fn`.
@@ -183,7 +183,7 @@ class MeanSquaredError(LossFunctionWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile('sgd', loss=tf.keras.losses.MeanSquaredError())
   ```
   """
@@ -213,7 +213,7 @@ class MeanAbsoluteError(LossFunctionWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile('sgd', loss=tf.keras.losses.MeanAbsoluteError())
   ```
   """
@@ -243,7 +243,7 @@ class MeanAbsolutePercentageError(LossFunctionWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile('sgd', loss=tf.keras.losses.MeanAbsolutePercentageError())
   ```
   """
@@ -273,7 +273,7 @@ class MeanSquaredLogarithmicError(LossFunctionWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile('sgd', loss=tf.keras.losses.MeanSquaredLogarithmicError())
   ```
   """
@@ -307,7 +307,7 @@ class BinaryCrossentropy(LossFunctionWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile('sgd', loss=tf.keras.losses.BinaryCrossentropy())
   ```
 
@@ -315,8 +315,8 @@ class BinaryCrossentropy(LossFunctionWrapper):
     from_logits: Whether `y_pred` is expected to be a logits tensor. By default,
       we assume that `y_pred` encodes a probability distribution.
     label_smoothing: Float in [0, 1]. If > `0` then smooth the labels.
-    reduction: Type of `tf.losses.Reduction` to apply to loss. Default value is
-      `SUM_OVER_BATCH_SIZE`.
+    reduction: (Optional) Type of `tf.keras.losses.Reduction` to apply to loss.
+      Default value is `SUM_OVER_BATCH_SIZE`.
     name: Optional name for the op.
   """
 
@@ -360,7 +360,7 @@ class CategoricalCrossentropy(LossFunctionWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile('sgd', loss=tf.keras.losses.CategoricalCrossentropy())
   ```
 
@@ -371,8 +371,8 @@ class CategoricalCrossentropy(LossFunctionWrapper):
       meaning the confidence on label values are relaxed. e.g.
       `label_smoothing=0.2` means that we will use a value of `0.1` for label
       `0` and `0.9` for label `1`"
-    reduction: Type of `tf.losses.Reduction` to apply to loss. Default value is
-      `SUM_OVER_BATCH_SIZE`.
+    reduction: (Optional) Type of `tf.keras.losses.Reduction` to apply to loss.
+      Default value is `SUM_OVER_BATCH_SIZE`.
     name: Optional name for the op.
   """
 
@@ -417,15 +417,15 @@ class SparseCategoricalCrossentropy(LossFunctionWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile('sgd', loss=tf.keras.losses.SparseCategoricalCrossentropy())
   ````
 
   Args:
     from_logits: Whether `y_pred` is expected to be a logits tensor. By default,
       we assume that `y_pred` encodes a probability distribution.
-    reduction: Type of `tf.losses.Reduction` to apply to loss. Default value is
-      `SUM_OVER_BATCH_SIZE`.
+    reduction: (Optional) Type of `tf.keras.losses.Reduction` to apply to loss.
+      Default value is `SUM_OVER_BATCH_SIZE`.
     name: Optional name for the op.
   """
 
@@ -450,7 +450,7 @@ class Hinge(LossFunctionWrapper):
   Usage:
 
   ```python
-  h = tf.losses.Hinge()
+  h = tf.keras.losses.Hinge()
   loss = h([-1., 1., 1.], [0.6, -0.7, -0.5])
 
   # loss = max(0, 1 - y_true * y_pred) = [1.6 + 1.7 + 1.5] / 3
@@ -461,8 +461,8 @@ class Hinge(LossFunctionWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.losses.Hinge())
+  model = tf.keras.Model(inputs, outputs)
+  model.compile('sgd', loss=tf.keras.losses.Hinge())
   ```
   """
 
@@ -482,7 +482,7 @@ class SquaredHinge(LossFunctionWrapper):
   Usage:
 
   ```python
-  sh = tf.losses.SquaredHinge()
+  sh = tf.keras.losses.SquaredHinge()
   loss = sh([-1., 1., 1.], [0.6, -0.7, -0.5])
 
   # loss = (max(0, 1 - y_true * y_pred))^2 = [1.6^2 + 1.7^2 + 1.5^2] / 3
@@ -493,8 +493,8 @@ class SquaredHinge(LossFunctionWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.losses.SquaredHinge())
+  model = tf.keras.Model(inputs, outputs)
+  model.compile('sgd', loss=tf.keras.losses.SquaredHinge())
   ```
   """
 
@@ -512,7 +512,7 @@ class CategoricalHinge(LossFunctionWrapper):
   Usage:
 
   ```python
-  ch = tf.losses.CategoricalHinge()
+  ch = tf.keras.losses.CategoricalHinge()
   loss = ch([0., 1., 1.], [1., 0., 1.])
   print('Loss: ', loss.numpy())  # Loss: 1.0
   ```
@@ -520,8 +520,8 @@ class CategoricalHinge(LossFunctionWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.losses.CategoricalHinge())
+  model = tf.keras.Model(inputs, outputs)
+  model.compile('sgd', loss=tf.keras.losses.CategoricalHinge())
   ```
   """
 
@@ -541,7 +541,7 @@ class LogLoss(LossFunctionWrapper):
   Usage:
 
   ```python
-  l = tf.losses.LogLoss()
+  l = tf.keras.losses.LogLoss()
   loss = l([0., 1., 1.], [1., 0., 1.])
   print('Loss: ', loss.numpy())  # Loss: 10.745
   ```
@@ -549,8 +549,8 @@ class LogLoss(LossFunctionWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.losses.LogLoss())
+  model = tf.keras.Model(inputs, outputs)
+  model.compile('sgd', loss=tf.keras.losses.LogLoss())
   ```
   """
 
@@ -569,7 +569,7 @@ class Poisson(LossFunctionWrapper):
   Usage:
 
   ```python
-  p = tf.losses.Poisson()
+  p = tf.keras.losses.Poisson()
   loss = p([1, 9, 2], [4, 8, 12])
   print('Loss: ', loss.numpy())  # Loss: -4.63
   ```
@@ -577,8 +577,8 @@ class Poisson(LossFunctionWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.losses.Poisson())
+  model = tf.keras.Model(inputs, outputs)
+  model.compile('sgd', loss=tf.keras.losses.Poisson())
   ```
   """
 
@@ -597,7 +597,7 @@ class LogCosh(LossFunctionWrapper):
   Usage:
 
   ```python
-  l = tf.losses.LogCosh()
+  l = tf.keras.losses.LogCosh()
   loss = l([0., 1., 1.], [1., 0., 1.])
   print('Loss: ', loss.numpy())  # Loss: 0.289
   ```
@@ -605,8 +605,8 @@ class LogCosh(LossFunctionWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.losses.LogCosh())
+  model = tf.keras.Model(inputs, outputs)
+  model.compile('sgd', loss=tf.keras.losses.LogCosh())
   ```
   """
 
@@ -625,7 +625,7 @@ class KLDivergence(LossFunctionWrapper):
   Usage:
 
   ```python
-  k = tf.losses.KLDivergence()
+  k = tf.keras.losses.KLDivergence()
   loss = k([.4, .9, .2], [.5, .8, .12])
   print('Loss: ', loss.numpy())  # Loss: -0.043
   ```
@@ -633,8 +633,8 @@ class KLDivergence(LossFunctionWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.losses.KLDivergence())
+  model = tf.keras.Model(inputs, outputs)
+  model.compile('sgd', loss=tf.keras.losses.KLDivergence())
   ```
   """
 
@@ -660,7 +660,7 @@ class Huber(LossFunctionWrapper):
   Usage:
 
   ```python
-  l = tf.losses.Huber()
+  l = tf.keras.losses.Huber()
   loss = l([0., 1., 1.], [1., 0., 1.])
   print('Loss: ', loss.numpy())  # Loss: 0.333
   ```
@@ -668,15 +668,15 @@ class Huber(LossFunctionWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.losses.Huber())
+  model = tf.keras.Model(inputs, outputs)
+  model.compile('sgd', loss=tf.keras.losses.Huber())
   ```
 
   Args:
     delta: A float, the point where the Huber loss function changes from a
       quadratic to linear.
-    reduction: Type of `tf.losses.Reduction` to apply to loss. Default value is
-      `SUM_OVER_BATCH_SIZE`.
+    reduction: (Optional) Type of `tf.keras.losses.Reduction` to apply to loss.
+      Default value is `SUM_OVER_BATCH_SIZE`.
     name: Optional name for the op.
   """
 
@@ -960,7 +960,7 @@ class CosineProximity(Loss):
   Usage:
 
   ```python
-  cosine_loss = tf.losses.CosineProximity()
+  cosine_loss = tf.keras.losses.CosineProximity()
   loss = cosine_loss([0., 1., 1.], [1., 0., 1.])
   print('Loss: ', loss.numpy())  # Loss: -0.5
   ```
@@ -968,14 +968,14 @@ class CosineProximity(Loss):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.losses.CosineProximity())
+  model = tf.keras.Model(inputs, outputs)
+  model.compile('sgd', loss=tf.keras.losses.CosineProximity())
   ```
 
   Args:
     axis: (Optional) Defaults to -1. The dimension along which the cosine
       proximity is computed.
-    reduction: (Optional) Type of `tf.losses.Reduction` to apply to loss.
+    reduction: (Optional) Type of `tf.keras.losses.Reduction` to apply to loss.
       Default value is `SUM_OVER_BATCH_SIZE`.
     name: Optional name for the op.
   """
diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index d3be3ef6a0..f24f8685d6 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -373,7 +373,7 @@ class Sum(Reduce):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.add_metric(tf.keras.metrics.Sum(name='sum_1')(outputs))
   model.compile('sgd', loss='mse')
   ```
@@ -415,7 +415,7 @@ class Mean(Reduce):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.add_metric(tf.keras.metrics.Mean(name='mean_1')(outputs))
   model.compile('sgd', loss='mse')
   ```
@@ -459,7 +459,7 @@ class MeanRelativeError(Mean):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile(
     'sgd',
     loss='mse',
@@ -589,7 +589,7 @@ class Accuracy(MeanMetricWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile('sgd', loss='mse', metrics=[tf.keras.metrics.Accuracy()])
   ```
   """
@@ -625,7 +625,7 @@ class BinaryAccuracy(MeanMetricWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile('sgd', loss='mse', metrics=[tf.keras.metrics.BinaryAccuracy()])
   ```
   """
@@ -675,7 +675,7 @@ class CategoricalAccuracy(MeanMetricWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile(
     'sgd',
     loss='mse',
@@ -723,7 +723,7 @@ class SparseCategoricalAccuracy(MeanMetricWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile(
       'sgd',
       loss='mse',
@@ -751,7 +751,7 @@ class TopKCategoricalAccuracy(MeanMetricWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile('sgd', metrics=[tf.keras.metrics.TopKCategoricalAccuracy()])
   ```
   """
@@ -784,7 +784,7 @@ class SparseTopKCategoricalAccuracy(MeanMetricWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile(
     'sgd',
     metrics=[tf.keras.metrics.SparseTopKCategoricalAccuracy()])
@@ -898,7 +898,7 @@ class FalsePositives(_ConfusionMatrixConditionCount):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile('sgd', loss='mse', metrics=[tf.keras.metrics.FalsePositives()])
   ```
   """
@@ -948,7 +948,7 @@ class FalseNegatives(_ConfusionMatrixConditionCount):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile('sgd', loss='mse', metrics=[tf.keras.metrics.FalseNegatives()])
   ```
   """
@@ -998,7 +998,7 @@ class TrueNegatives(_ConfusionMatrixConditionCount):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile('sgd', loss='mse', metrics=[tf.keras.metrics.TrueNegatives()])
   ```
   """
@@ -1048,7 +1048,7 @@ class TruePositives(_ConfusionMatrixConditionCount):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile('sgd', loss='mse', metrics=[tf.keras.metrics.TruePositives()])
   ```
   """
@@ -1108,7 +1108,7 @@ class Precision(Metric):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile('sgd', loss='mse', metrics=[tf.keras.metrics.Precision()])
   ```
   """
@@ -1234,7 +1234,7 @@ class Recall(Metric):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile('sgd', loss='mse', metrics=[tf.keras.metrics.Recall()])
   ```
   """
@@ -1425,7 +1425,7 @@ class SensitivityAtSpecificity(SensitivitySpecificityBase):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile(
       'sgd',
       loss='mse',
@@ -1506,7 +1506,7 @@ class SpecificityAtSensitivity(SensitivitySpecificityBase):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile(
       'sgd',
       loss='mse',
@@ -1601,7 +1601,7 @@ class AUC(Metric):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile('sgd', loss='mse', metrics=[tf.keras.metrics.AUC()])
   ```
   """
@@ -1833,7 +1833,7 @@ class CosineProximity(MeanMetricWrapper):
 
   Usage:
   ```python
-  m = tf.metrics.CosineProximity()
+  m = tf.keras.metrics.CosineProximity()
   m.update_state([0, 1, 1], [1, 0, 1])
   print('Final result: ', m.result().numpy())  # Final result: -0.5
   ```
@@ -1841,11 +1841,11 @@ class CosineProximity(MeanMetricWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile(
       'sgd',
       loss='mse',
-      metrics=[tf.metrics.CosineProximity()])
+      metrics=[tf.keras.metrics.CosineProximity()])
   ```
   """
 
@@ -1870,7 +1870,7 @@ class MeanAbsoluteError(MeanMetricWrapper):
 
   Usage:
   ```python
-  m = tf.metrics.MeanAbsoluteError()
+  m = tf.keras.metrics.MeanAbsoluteError()
   m.update_state([0., 0., 1., 1.], [1., 1., 1., 0.])
   print('Final result: ', m.result().numpy())  # Final result: 0.75
   ```
@@ -1878,7 +1878,7 @@ class MeanAbsoluteError(MeanMetricWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile('sgd', metrics=[tf.keras.metrics.MeanAbsoluteError()])
   ```
   """
@@ -1906,7 +1906,7 @@ class MeanAbsolutePercentageError(MeanMetricWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile('sgd', metrics=[tf.keras.metrics.MeanAbsolutePercentageError()])
   ```
   """
@@ -1934,7 +1934,7 @@ class MeanSquaredError(MeanMetricWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile('sgd', metrics=[tf.keras.metrics.MeanSquaredError()])
   ```
   """
@@ -1962,7 +1962,7 @@ class MeanSquaredLogarithmicError(MeanMetricWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile('sgd', metrics=[tf.keras.metrics.MeanSquaredLogarithmicError()])
   ```
   """
@@ -1996,7 +1996,7 @@ class Hinge(MeanMetricWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile('sgd', metrics=[tf.keras.metrics.Hinge()])
   ```
   """
@@ -2029,7 +2029,7 @@ class SquaredHinge(MeanMetricWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile('sgd', metrics=[tf.keras.metrics.SquaredHinge()])
   ```
   """
@@ -2056,7 +2056,7 @@ class CategoricalHinge(MeanMetricWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile('sgd', metrics=[tf.keras.metrics.CategoricalHinge()])
   ```
   """
@@ -2080,7 +2080,7 @@ class RootMeanSquaredError(Mean):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile('sgd', metrics=[tf.keras.metrics.RootMeanSquaredError()])
   ```
   """
@@ -2130,7 +2130,7 @@ class LogCoshError(MeanMetricWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile('sgd', metrics=[tf.keras.metrics.LogCoshError()])
   ```
   """
@@ -2156,7 +2156,7 @@ class Poisson(MeanMetricWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile('sgd', metrics=[tf.keras.metrics.Poisson()])
   ```
   """
@@ -2182,7 +2182,7 @@ class KLDivergence(MeanMetricWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile('sgd', metrics=[tf.keras.metrics.KLDivergence()])
   ```
   """
@@ -2223,7 +2223,7 @@ class MeanIoU(Metric):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile(
     'sgd',
     loss='mse',
@@ -2329,7 +2329,7 @@ class MeanTensor(Metric):
   Usage:
 
   ```python
-  m = tf.metrics.MeanTensor()
+  m = tf.keras.metrics.MeanTensor()
   m.update_state([0, 1, 2, 3])
   m.update_state([4, 5, 6, 7])
   print('Result: ', m.result().numpy())  # Result: [2, 3, 4, 5]
@@ -2457,7 +2457,7 @@ class BinaryCrossentropy(MeanMetricWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile(
       'sgd',
       loss='mse',
@@ -2523,7 +2523,7 @@ class CategoricalCrossentropy(MeanMetricWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile(
     'sgd',
     loss='mse',
@@ -2597,7 +2597,7 @@ class SparseCategoricalCrossentropy(MeanMetricWrapper):
   Usage with tf.keras API:
 
   ```python
-  model = keras.models.Model(inputs, outputs)
+  model = tf.keras.Model(inputs, outputs)
   model.compile(
     'sgd',
     loss='mse',
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
index b701c98215..5a1b5f2505 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
@@ -132,8 +132,8 @@ class OptimizerV2(checkpointable.Checkpointable):
   you divide your loss by the global batch size, which is done automatically
   if you use a member of `tf.keras.losses` or `tf.losses`. See the
   `reduction` argument of your loss which should be set to
-  `tf.losses.Reduction.SUM_OVER_BATCH_SIZE` for averaging or
-  `tf.losses.Reduction.SUM` for not.
+  `tf.keras.losses.Reduction.SUM_OVER_BATCH_SIZE` for averaging or
+  `tf.keras.losses.Reduction.SUM` for not.
 
   If you are not using these and you want to average gradients, you should use
   `tf.math.reduce_sum` to add up your per-example losses and then divide by the
diff --git a/tensorflow/python/keras/utils/losses_utils.py b/tensorflow/python/keras/utils/losses_utils.py
index c25f4f780d..fd15bd3f3a 100644
--- a/tensorflow/python/keras/utils/losses_utils.py
+++ b/tensorflow/python/keras/utils/losses_utils.py
@@ -192,8 +192,8 @@ def compute_weighted_loss(losses,
     losses: `Tensor` of shape `[batch_size, d1, ... dN]`.
     sample_weight: Optional `Tensor` whose rank is either 0, or the same rank as
       `losses`, or be broadcastable to `losses`.
-    reduction: Type of `tf.losses.Reduction` to apply to loss. Default value is
-      `SUM_OVER_BATCH_SIZE`.
+    reduction: (Optional) Type of `tf.keras.losses.Reduction` to apply to loss.
+      Default value is `SUM_OVER_BATCH_SIZE`.
     name: Optional name for the op.
 
   Raises:
-- 
GitLab


From 36a64ceeed0ed755ce3435e098e8f7335ae65d68 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 8 Feb 2019 15:04:57 -0800
Subject: [PATCH 2204/2345] Optimize constant proto creation.

PiperOrigin-RevId: 233133692
---
 .../grappler/optimizers/constant_folding.cc   | 45 +++++++++----------
 1 file changed, 22 insertions(+), 23 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc
index f883f8926f..98517b74d4 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding.cc
@@ -975,29 +975,28 @@ Status ConstantFolding::CreateNodeDef(const string& name,
   // Use the packed representation whenever possible to avoid generating large
   // graphdefs. Moreover, avoid repeating the last values if they're equal.
   if (tensor->NumElements() > 4) {
-#define POPULATE_TENSOR_PROTO(tensor, t, TYPE, NAME)                  \
-  {                                                                   \
-    const TYPE* val_ptr = tensor->flat<TYPE>().data();                \
-    TYPE last = *val_ptr;                                             \
-    int64 last_index = 0;                                             \
-    for (int64 i = 0; i < tensor->NumElements(); ++i) {               \
-      TYPE cur = *val_ptr++;                                          \
-      if (cur != last) {                                              \
-        last = cur;                                                   \
-        last_index = i;                                               \
-      }                                                               \
-    }                                                                 \
-    if (last_index < kint32max) {                                     \
-      optimized = true;                                               \
-      encoded_size = (last_index + 1) * sizeof(NAME);                 \
-      t->mutable_##NAME##_val()->Reserve(last_index + 1);             \
-      t->mutable_##NAME##_val()->AddNAlreadyReserved(last_index + 1); \
-      val_ptr = tensor->flat<TYPE>().data();                          \
-      for (int64 i = 0; i <= last_index; ++i) {                       \
-        t->set_##NAME##_val(i, *val_ptr++);                           \
-      }                                                               \
-    }                                                                 \
-  }                                                                   \
+#define POPULATE_TENSOR_PROTO(tensor, t, TYPE, NAME)                      \
+  {                                                                       \
+    const auto* val_ptr = tensor->flat<TYPE>().data();                    \
+    auto last = *val_ptr;                                                 \
+    int64 last_index = 0;                                                 \
+    for (int64 i = 0; i < tensor->NumElements(); ++i) {                   \
+      TYPE cur = *val_ptr++;                                              \
+      if (cur != last) {                                                  \
+        last = cur;                                                       \
+        last_index = i;                                                   \
+      }                                                                   \
+    }                                                                     \
+    if (last_index < kint32max) {                                         \
+      optimized = true;                                                   \
+      encoded_size = (last_index + 1) * sizeof(NAME);                     \
+      t->mutable_##NAME##_val()->Reserve(last_index + 1);                 \
+      const auto* src_ptr = tensor->flat<TYPE>().data();                  \
+      auto* dst_ptr =                                                     \
+          t->mutable_##NAME##_val()->AddNAlreadyReserved(last_index + 1); \
+      std::copy(src_ptr, src_ptr + last_index + 1, dst_ptr);              \
+    }                                                                     \
+  }                                                                       \
   break
 
     switch (tensor->dtype()) {
-- 
GitLab


From a9b3f469cae307993d37c708e1062fa84e44d723 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Fri, 8 Feb 2019 15:07:43 -0800
Subject: [PATCH 2205/2345] Fix tensor lifetime issue in TensorListDeviceCopy

PiperOrigin-RevId: 233134162
---
 tensorflow/core/kernels/list_kernels.cc | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/kernels/list_kernels.cc b/tensorflow/core/kernels/list_kernels.cc
index 12863815fb..b584b34016 100644
--- a/tensorflow/core/kernels/list_kernels.cc
+++ b/tensorflow/core/kernels/list_kernels.cc
@@ -79,12 +79,10 @@ static Status TensorListDeviceCopy(
   to->max_num_elements = from.max_num_elements;
   to->tensors.reserve(from.tensors.size());
   for (const Tensor& t : from.tensors) {
-    Tensor tmp(t.dtype());
-    // Do not copy uninitialized tensors.
+    to->tensors.emplace_back(t.dtype());
     if (t.dtype() != DT_INVALID) {
-      TF_RETURN_IF_ERROR(copy(t, &tmp));
+      TF_RETURN_IF_ERROR(copy(t, &to->tensors.back()));
     }
-    to->tensors.push_back(tmp);
   }
   return Status::OK();
 }
-- 
GitLab


From 16106ddfd2bdce83f89525577a2999f9d38063cb Mon Sep 17 00:00:00 2001
From: nammbash <niroopshankar.ammbashankar@intel.com>
Date: Fri, 8 Feb 2019 16:28:03 -0800
Subject: [PATCH 2206/2345] Added the new ops to the golden v1

---
 tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt
index 28f26fe2de..2a63036ffa 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt
@@ -2644,9 +2644,17 @@ tf_module {
     name: "RequantizationRange"
     argspec: "args=[\'input\', \'input_min\', \'input_max\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "RequantizationRangePerChannel"
+    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'clip_value_max\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "Requantize"
     argspec: "args=[\'input\', \'input_min\', \'input_max\', \'requested_output_min\', \'requested_output_max\', \'out_type\'], varargs=None, keywords=None, defaults=None"
+  }  
+  member_method {
+    name: "RequantizePerChannel"
+    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'requested_output_min\', \'requested_output_max\', \'out_type\'], varargs=None, keywords=None, defaults=None"
   }
   member_method {
     name: "Reshape"
-- 
GitLab


From e9330cec5397ef4f465ed736baccfd9a1c0b81e6 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Fri, 8 Feb 2019 15:13:04 -0800
Subject: [PATCH 2207/2345] Start on a 1.x-compat codepath for
 tf.saved_model.load

Allows selecting between multiple MetaGraphs and loading them into 2.x. Variables are OK as long as they're resource variables. Signatures are available in the same format as for 2.x-exported SavedModels, concrete functions which take only keyword arguments and return dictionaries.

Stuff that still needs doing:
  - Lift any variables at all out into the eager context
  - Fix ref variables (running the restore function isn't initializing them for some reason)
  - Assets

PiperOrigin-RevId: 233135086
---
 tensorflow/python/eager/function.py           |  15 +-
 tensorflow/python/saved_model/BUILD           |  51 +++++++
 tensorflow/python/saved_model/load.py         |  63 +++++++-
 tensorflow/python/saved_model/load_test.py    |  32 +++++
 .../python/saved_model/load_v1_in_v2.py       | 118 +++++++++++++++
 .../python/saved_model/load_v1_in_v2_test.py  | 134 ++++++++++++++++++
 tensorflow/python/saved_model/save.py         |   4 +-
 .../saved_model/signature_serialization.py    |  30 ++--
 8 files changed, 424 insertions(+), 23 deletions(-)
 create mode 100644 tensorflow/python/saved_model/load_v1_in_v2.py
 create mode 100644 tensorflow/python/saved_model/load_v1_in_v2_test.py

diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 04f1999a84..7a118368d4 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -431,13 +431,24 @@ class ConcreteFunction(object):
                args))
     args = list(args)
     for keyword in self._arg_keywords[len(args):]:
-      args.append(kwargs.pop(compat.as_str(keyword)))
+      try:
+        args.append(kwargs.pop(compat.as_str(keyword)))
+      except KeyError:
+        specified_keywords = (list(self._arg_keywords[:len(args)])
+                              + list(kwargs.keys()))
+        raise TypeError(
+            "Expected argument names {} but got values for {}. Missing: {}."
+            .format(
+                list(self._arg_keywords),
+                specified_keywords,
+                list(set(self._arg_keywords) - set(specified_keywords))))
     if kwargs:
       positional_arg_keywords = set(self._arg_keywords[:len(args)])
       for unused_key in kwargs:
         if unused_key in positional_arg_keywords:
           raise TypeError("Got two values for keyword '{}'.".format(unused_key))
-      raise TypeError("Keyword arguments {} unknown.".format(kwargs.keys()))
+      raise TypeError("Keyword arguments {} unknown. Expected {}.".format(
+          list(kwargs.keys()), list(self._arg_keywords)))
     return self._call_flat(args)
 
   def _filtered_call(self, args, kwargs):
diff --git a/tensorflow/python/saved_model/BUILD b/tensorflow/python/saved_model/BUILD
index 76670bf309..405319f323 100644
--- a/tensorflow/python/saved_model/BUILD
+++ b/tensorflow/python/saved_model/BUILD
@@ -268,6 +268,14 @@ py_library(
     ],
     srcs_version = "PY2AND3",
     deps = [
+        ":revived_types",
+        ":signature_constants",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:tensor_spec",
+        "//tensorflow/python:util",
+        "//tensorflow/python/eager:def_function",
+        "//tensorflow/python/eager:function",
+        "//tensorflow/python/training/checkpointable:base",
     ],
 )
 
@@ -334,6 +342,7 @@ py_library(
     deps = [
         ":constants",
         ":function_deserialization",
+        ":load_v1_in_v2",
         ":loader",
         ":nested_structure_coder",
         ":revived_types",
@@ -354,6 +363,22 @@ py_library(
     ],
 )
 
+py_library(
+    name = "load_v1_in_v2",
+    srcs = [
+        "load_v1_in_v2.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":loader",
+        ":signature_serialization",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:saver",
+        "//tensorflow/python/eager:wrap_function",
+        "//tensorflow/python/training/checkpointable:tracking",
+    ],
+)
+
 tf_py_test(
     name = "load_test",
     srcs = ["load_test.py"],
@@ -371,6 +396,32 @@ tf_py_test(
     ],
 )
 
+tf_py_test(
+    name = "load_v1_in_v2_test",
+    srcs = ["load_v1_in_v2_test.py"],
+    additional_deps = [
+        ":builder",
+        ":load",
+        ":save",
+        ":signature_def_utils",
+        ":simple_save",
+        ":utils",
+        "@absl_py//absl/testing:parameterized",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:lib",
+        "//tensorflow/python:resource_variable_ops",
+        "//tensorflow/python:session",
+        "//tensorflow/python:tensor_spec",
+        "//tensorflow/python/eager:def_function",
+        "//tensorflow/python/eager:test",
+        "//tensorflow/python/training/checkpointable:tracking",
+        "//tensorflow/python:variables",
+    ],
+)
+
 py_library(
     name = "revived_types",
     srcs = [
diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
index 8376d1d6e2..4511d3a91c 100644
--- a/tensorflow/python/saved_model/load.py
+++ b/tensorflow/python/saved_model/load.py
@@ -30,6 +30,7 @@ from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.saved_model import constants
 from tensorflow.python.saved_model import function_deserialization
+from tensorflow.python.saved_model import load_v1_in_v2
 from tensorflow.python.saved_model import loader_impl
 from tensorflow.python.saved_model import nested_structure_coder
 from tensorflow.python.saved_model import revived_types
@@ -40,6 +41,7 @@ from tensorflow.python.training.checkpointable import graph_view
 from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.training.checkpointable import util
 from tensorflow.python.util import compat
+from tensorflow.python.util import nest
 
 
 class _Loader(object):
@@ -241,14 +243,64 @@ def _load_saved_object_graph_proto(filename):
     return saved_object_graph_pb2.SavedObjectGraph.FromString(contents)
 
 
-def load(export_dir):
-  """Load a SavedModel from `export_dir`."""
+def load(export_dir, tags=None):
+  """Load a SavedModel from `export_dir`.
+
+  Signatures associated with the SavedModel are available as functions:
+
+  ```python
+  imported = tf.saved_model.load(path)
+  f = imported.signatures["serving_default"]
+  print(f(x=tf.constant([[1.]])))
+  ```
+
+  Objects exported with `tf.saved_model.save` additionally have checkpointable
+  objects and functions assigned to attributes:
+
+  ```python
+  exported = tf.train.Checkpoint(v=tf.Variable(3.))
+  exported.f = tf.function(
+      lambda x: exported.v * x,
+      input_signature=[tf.TensorSpec(shape=None, dtype=tf.float32)])
+  tf.saved_model.save(exported, path)
+  imported = tf.saved_model.load(path)
+  assert 3. == imported.v.numpy()
+  assert 6. == imported.f(x=tf.constant(2.)).numpy()
+  ```
+
+  Args:
+    export_dir: The SavedModel directory to load from.
+    tags: A tag or sequence of tags identifying the MetaGraph to load. Optional
+      if the SavedModel contains a single MetaGraph, as for those exported from
+      `tf.saved_model.load`.
+
+  Returns:
+    A checkpointable object with a `signatures` attribute mapping from signature
+    keys to functions. If the SavedModel was exported by `tf.saved_model.load`,
+    it also points to checkpointable objects and functions which were attached
+    to the exported object.
+
+  Raises:
+    ValueError: If `tags` don't match a MetaGraph in the SavedModel.
+  """
+  if tags is not None:
+    # Supports e.g. tags=SERVING and tags=[SERVING]
+    tags = nest.flatten(tags)
   saved_model_proto = loader_impl.parse_saved_model(export_dir)
   object_graph_filename = os.path.join(
       compat.as_bytes(export_dir),
       compat.as_bytes(constants.EXTRA_ASSETS_DIRECTORY),
       compat.as_bytes("object_graph.pb"))
-  if file_io.file_exists(object_graph_filename):
+  if (file_io.file_exists(object_graph_filename)
+      and len(saved_model_proto.meta_graphs) == 1):
+    meta_graph_def = saved_model_proto.meta_graphs[0]
+    if (tags is not None
+        and set(tags) != set(meta_graph_def.meta_info_def.tags)):
+      raise ValueError(
+          ("The SavedModel at {} has one MetaGraph with tags {}, but got an "
+           "incompatible argument tags={} to tf.saved_model.load. You may omit "
+           "it, pass 'None', or pass matching tags.")
+          .format(export_dir, meta_graph_def.meta_info_def.tags, tags))
     object_graph_proto = _load_saved_object_graph_proto(object_graph_filename)
     with ops.init_scope():
       loader = _Loader(object_graph_proto,
@@ -256,7 +308,6 @@ def load(export_dir):
                        export_dir)
       root = loader.get(0)
   else:
-    raise NotImplementedError(
-        "Currently only SavedModels exported with `tf.saved_model.save` may be "
-        "imported. Other SavedModels may eventually be supported via load().")
+    with ops.init_scope():
+      root = load_v1_in_v2.load(export_dir, tags)
   return root
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index e5b07e8619..8149290f12 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -38,6 +38,7 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.saved_model import load
 from tensorflow.python.saved_model import save
+from tensorflow.python.saved_model import tag_constants
 from tensorflow.python.training import monitored_session
 from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.training.checkpointable import util
@@ -964,5 +965,36 @@ class LoadTest(test.TestCase, parameterized.TestCase):
       imported.signatures["random_key"] = 3
 
 
+class SingleCycleTests(test.TestCase, parameterized.TestCase):
+
+  def test_load_with_tags(self):
+    root = tracking.AutoCheckpointable()
+    path = tempfile.mkdtemp(prefix=self.get_temp_dir())
+    save.save(root, path)
+    with self.assertRaises(ValueError):
+      load.load(path, tags=[tag_constants.EVAL])
+    load.load(path, tags=[tag_constants.SERVING])
+    load.load(path, tags=tag_constants.SERVING)
+
+  def test_docstring_examples(self):
+    path = tempfile.mkdtemp(prefix=self.get_temp_dir())
+    exported = util.Checkpoint(v=variables.Variable(3.))
+    exported.f = def_function.function(
+        lambda x: exported.v * x,
+        input_signature=[
+            tensor_spec.TensorSpec(shape=None, dtype=dtypes.float32)])
+    save.save(exported, path)
+    imported = load.load(path)
+    self.assertEqual(3., imported.v.numpy())
+    self.assertEqual(6., imported.f(x=constant_op.constant(2.)).numpy())
+
+    save.save(exported, path, exported.f.get_concrete_function())
+    imported = load.load(path)
+    f = imported.signatures["serving_default"]
+    self.assertAllEqual(
+        [[-3.]],
+        f(x=constant_op.constant([[-1.]]))["output_0"].numpy())
+
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/saved_model/load_v1_in_v2.py b/tensorflow/python/saved_model/load_v1_in_v2.py
new file mode 100644
index 0000000000..c7c8d8d804
--- /dev/null
+++ b/tensorflow/python/saved_model/load_v1_in_v2.py
@@ -0,0 +1,118 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Import a TF v1-style SavedModel when executing eagerly."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import functools
+
+from tensorflow.python.eager import wrap_function
+from tensorflow.python.framework import constant_op
+from tensorflow.python.saved_model import loader_impl
+from tensorflow.python.saved_model import signature_serialization
+from tensorflow.python.training import saver as tf_saver
+from tensorflow.python.training.checkpointable import tracking
+
+
+class _EagerSavedModelLoader(loader_impl.SavedModelLoader):
+  """Loads a SavedModel without using Sessions."""
+
+  def get_meta_graph_def_from_tags(self, tags):
+    """Override to support implicit one-MetaGraph loading with tags=None."""
+    if tags is None:
+      if len(self._saved_model.meta_graphs) != 1:
+        tag_sets = [mg.meta_info_def.tags
+                    for mg in self._saved_model.meta_graphs]
+        raise ValueError(
+            ("Importing a SavedModel with tf.saved_model.load requires a "
+             "'tags=' argument if there is more than one MetaGraph. Got "
+             "'tags=None', but there are {} MetaGraphs in the SavedModel with "
+             "tag sets {}. Pass a 'tags=' argument to load this SavedModel.")
+            .format(len(self._saved_model.meta_graphs), tag_sets))
+      return self._saved_model.meta_graphs[0]
+    return super(_EagerSavedModelLoader, self).get_meta_graph_def_from_tags(
+        tags)
+
+  def load_graph(self, returns, meta_graph_def):
+    """Called from wrap_function to import `meta_graph_def`."""
+    # pylint: disable=protected-access
+    saver, _ = tf_saver._import_meta_graph_with_return_elements(
+        meta_graph_def)
+    # pylint: enable=protected-access
+    returns[0] = saver
+
+  def restore_variables(self, wrapped, saver):
+    """Restores variables from the checkpoint."""
+    if saver is not None:
+      saver_def = saver.saver_def
+      restore_fn = wrapped.prune(
+          feeds=[wrapped.graph.as_graph_element(
+              saver_def.filename_tensor_name)],
+          fetches=[wrapped.graph.as_graph_element(saver_def.restore_op_name)])
+      restore_fn(constant_op.constant(self._variables_path))
+
+  def _extract_signatures(self, wrapped, meta_graph_def):
+    """Creates ConcreteFunctions for signatures in `meta_graph_def`."""
+    signature_functions = {}
+    for signature_key, signature_def in meta_graph_def.signature_def.items():
+      input_names, input_specs = zip(*signature_def.inputs.items())
+      # TODO(allenl): Support optional arguments
+      signature_fn = wrapped.prune(
+          feeds=[wrapped.graph.as_graph_element(inp.name)
+                 for inp in input_specs],
+          fetches={name: wrapped.graph.as_graph_element(out.name)
+                   for name, out in signature_def.outputs.items()})
+      # pylint: disable=protected-access
+      signature_fn._arg_keywords = input_names
+      signature_fn._num_positional_args = 0
+      # pylint: enable=protected-access
+      signature_functions[signature_key] = signature_fn
+    return signature_functions
+
+  def load(self, tags):
+    """Creates an object from the MetaGraph identified by `tags`."""
+    meta_graph_def = self.get_meta_graph_def_from_tags(tags)
+    for node in meta_graph_def.graph_def.node:
+      if node.op == "VariableV2":
+        raise NotImplementedError(
+            "Importing a SavedModel which contains RefVariables. This is not "
+            "currently supported. Running tf.enable_resource_variables() "
+            "before creating exported variables will fix this.")
+    load_graph_returns = [None]
+    wrapped = wrap_function.wrap_function(
+        functools.partial(self.load_graph, load_graph_returns, meta_graph_def),
+        signature=[])
+    saver, = load_graph_returns
+    self.restore_variables(wrapped, saver)
+    with wrapped.graph.as_default():
+      init_op = loader_impl.get_init_op(meta_graph_def)
+    if init_op is not None:
+      # TODO(allenl): Deal with assets
+      wrapped.prune(feeds=[],
+                    fetches=[wrapped.graph.as_graph_element(init_op)])()
+    signature_functions = self._extract_signatures(wrapped, meta_graph_def)
+    root = tracking.AutoCheckpointable()
+    root.signatures = signature_serialization.create_signature_map(
+        signature_functions)
+    # TODO(allenl): Lift out variables and make a ".variables" property
+    return root
+
+
+def load(export_dir, tags):
+  """Load a v1-style SavedModel as an object."""
+  loader = _EagerSavedModelLoader(export_dir)
+  return loader.load(tags=tags)
diff --git a/tensorflow/python/saved_model/load_v1_in_v2_test.py b/tensorflow/python/saved_model/load_v1_in_v2_test.py
new file mode 100644
index 0000000000..42e13e782f
--- /dev/null
+++ b/tensorflow/python/saved_model/load_v1_in_v2_test.py
@@ -0,0 +1,134 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for importing a TF v1-style SavedModel when executing eagerly."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+
+from tensorflow.python.client import session as session_lib
+from tensorflow.python.eager import test
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.saved_model import builder_impl
+from tensorflow.python.saved_model import load
+from tensorflow.python.saved_model import signature_def_utils
+from tensorflow.python.saved_model import simple_save
+from tensorflow.python.saved_model import utils_impl
+
+
+class LoadTest(test.TestCase):
+
+  def _v1_single_metagraph_saved_model(self, use_resource):
+    export_graph = ops.Graph()
+    with export_graph.as_default():
+      start = array_ops.placeholder(
+          shape=[None], dtype=dtypes.float32, name="start")
+      if use_resource:
+        v = resource_variable_ops.ResourceVariable(3.)
+      else:
+        v = variables.RefVariable(3.)
+      local_variable = variables.VariableV1(
+          1.,
+          collections=[ops.GraphKeys.LOCAL_VARIABLES],
+          use_resource=True)
+      output = array_ops.identity(start * v * local_variable, name="output")
+      with session_lib.Session() as session:
+        session.run([v.initializer, local_variable.initializer])
+        path = os.path.join(self.get_temp_dir(), "saved_model", str(ops.uid()))
+        simple_save.simple_save(
+            session,
+            path,
+            inputs={"start": start},
+            outputs={"output": output},
+            legacy_init_op=local_variable.initializer)
+    return path
+
+  def test_resource_variable_import(self):
+    imported = load.load(self._v1_single_metagraph_saved_model(
+        use_resource=True))
+    fn = imported.signatures["serving_default"]
+    with self.assertRaisesRegexp(TypeError, "positional"):
+      fn(constant_op.constant(2.))
+    self.assertEqual({"output": 6.},
+                     self.evaluate(fn(start=constant_op.constant(2.))))
+
+  def test_ref_variable_import(self):
+    with self.assertRaises(NotImplementedError):
+      imported = load.load(self._v1_single_metagraph_saved_model(
+          use_resource=False))
+    # TODO(allenl): Support ref variables
+    self.skipTest("Ref variables aren't working yet")
+    fn = imported.signatures["serving_default"]
+    self.assertEqual(6., fn(start=constant_op.constant(2.)))
+
+  def _v1_multi_metagraph_saved_model(self):
+    export_graph = ops.Graph()
+    with export_graph.as_default():
+      start = array_ops.placeholder(
+          shape=[None], dtype=dtypes.float32, name="start")
+      v = resource_variable_ops.ResourceVariable(21.)
+      first_output = array_ops.identity(start * v, name="first_output")
+      second_output = array_ops.identity(v, name="second_output")
+      with session_lib.Session() as session:
+        session.run(v.initializer)
+        path = os.path.join(self.get_temp_dir(), "saved_model", str(ops.uid()))
+        builder = builder_impl.SavedModelBuilder(path)
+        builder.add_meta_graph_and_variables(
+            session, tags=["first"],
+            signature_def_map={
+                "first_key": signature_def_utils.build_signature_def(
+                    {"first_start": utils_impl.build_tensor_info(start)},
+                    {"first_output": utils_impl.build_tensor_info(
+                        first_output)})})
+        builder.add_meta_graph(
+            tags=["second"],
+            signature_def_map={
+                "second_key": signature_def_utils.build_signature_def(
+                    {"second_start": utils_impl.build_tensor_info(start)},
+                    {"second_output": utils_impl.build_tensor_info(
+                        second_output)})})
+        builder.save()
+    return path
+
+  def test_multi_meta_graph_loading(self):
+    with self.assertRaisesRegexp(ValueError, "2 MetaGraphs"):
+      load.load(self._v1_multi_metagraph_saved_model())
+    first_imported = load.load(self._v1_multi_metagraph_saved_model(),
+                               tags=["first"])
+    self.assertEqual({"first_output": 42.},
+                     self.evaluate(first_imported.signatures["first_key"](
+                         first_start=constant_op.constant(2.))))
+    second_imported = load.load(self._v1_multi_metagraph_saved_model(),
+                                tags=["second"])
+    with self.assertRaisesRegexp(TypeError, "second_start"):
+      second_imported.signatures["second_key"](x=constant_op.constant(2.))
+    with self.assertRaisesRegexp(TypeError, "second_start"):
+      second_imported.signatures["second_key"](
+          second_start=constant_op.constant(2.),
+          x=constant_op.constant(2.))
+    self.assertEqual({"second_output": 21.},
+                     self.evaluate(second_imported.signatures["second_key"](
+                         second_start=constant_op.constant(2.))))
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index dbfb5dd6df..29d8a89482 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -779,8 +779,8 @@ def save(obj, export_dir, signatures=None):
         checkpoint_graph_view)
 
   signatures = signature_serialization.canonicalize_signatures(signatures)
-  signature_map = signature_serialization.create_signature_map(
-      signatures, checkpoint_graph_view)
+  signature_serialization.validate_saveable_view(checkpoint_graph_view)
+  signature_map = signature_serialization.create_signature_map(signatures)
   checkpoint_graph_view.add_object(
       parent_node=checkpoint_graph_view.root,
       name_in_parent=signature_serialization.SIGNATURE_ATTRIBUTE_NAME,
diff --git a/tensorflow/python/saved_model/signature_serialization.py b/tensorflow/python/saved_model/signature_serialization.py
index 19f15f1886..862e78fdeb 100644
--- a/tensorflow/python/saved_model/signature_serialization.py
+++ b/tensorflow/python/saved_model/signature_serialization.py
@@ -215,8 +215,23 @@ revived_types.register_revived_type(
     )])
 
 
-def create_signature_map(signatures, saveable_view):
-  """Performs sanity checks and creates an object containing `signatures`."""
+def create_signature_map(signatures):
+  """Creates an object containing `signatures`."""
+  signature_map = _SignatureMap()
+  for name, func in signatures.items():
+    # This true of any signature that came from canonicalize_signatures. Here as
+    # a sanity check on saving; crashing on load (e.g. in _add_signature) would
+    # be more problematic in case future export changes violated these
+    # assertions.
+    assert isinstance(func, defun.ConcreteFunction)
+    assert isinstance(func.structured_outputs, collections.Mapping)
+    assert 0 == func._num_positional_args  # pylint: disable=protected-access
+    signature_map._add_signature(name, func)  # pylint: disable=protected-access
+  return signature_map
+
+
+def validate_saveable_view(saveable_view):
+  """Performs signature-related sanity checks on `saveable_view`."""
   for name, dep in saveable_view.list_dependencies(
       saveable_view.root):
     if name == SIGNATURE_ATTRIBUTE_NAME:
@@ -231,14 +246,3 @@ def create_signature_map(signatures, saveable_view):
                  saveable_view.root,
                  signatures=SIGNATURE_ATTRIBUTE_NAME))
       break
-  signature_map = _SignatureMap()
-  for name, func in signatures.items():
-    # This true of any signature that came from canonicalize_signatures. Here as
-    # a sanity check on saving; crashing on load (e.g. in _add_signature) would
-    # be more problematic in case future export changes violated these
-    # assertions.
-    assert isinstance(func, defun.ConcreteFunction)
-    assert isinstance(func.structured_outputs, collections.Mapping)
-    assert 0 == func._num_positional_args  # pylint: disable=protected-access
-    signature_map._add_signature(name, func)  # pylint: disable=protected-access
-  return signature_map
-- 
GitLab


From 9b55ee111536324046feed65480c6cdbfdaffebc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 8 Feb 2019 15:14:56 -0800
Subject: [PATCH 2208/2345] Make `steps` and `steps_per_epoch` optional for
 TPUStrategy when runs_per_step is 1

Note:
If the user passes a dataset which is repeated indefinitely and has unknown cardinality, and does not specify `steps`/`steps_per_epoch` parameter, then predict()/fit()/evaluate() will run forever.
PiperOrigin-RevId: 233135393
---
 .../contrib/distribute/python/keras_test.py   |  78 +++++++-
 tensorflow/python/keras/engine/training.py    |  13 --
 .../keras/engine/training_distributed.py      | 168 +++++++++++-------
 3 files changed, 177 insertions(+), 82 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 6a15083470..eb515441e2 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -27,6 +27,7 @@ from tensorflow.contrib.distribute.python import combinations
 from tensorflow.contrib.distribute.python import mirrored_strategy
 from tensorflow.contrib.distribute.python import tpu_strategy
 from tensorflow.python import keras
+from tensorflow.python.data.experimental.ops import cardinality
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import values
 from tensorflow.python.eager import test
@@ -216,6 +217,22 @@ def get_predict_dataset(distribution):
   return dataset
 
 
+def convert_numpy_to_dataset_with_unknown_cardinality(inputs,
+                                                      targets=None):
+  if targets is not None:
+    input_slices = (inputs, targets)
+    dummy_op = (lambda inp, target: True)
+  else:
+    input_slices = inputs
+    dummy_op = (lambda inp: True)
+
+  original_dataset = (dataset_ops.Dataset.from_tensor_slices(
+      input_slices))
+  ds_with_unknown_cardinality = (original_dataset.filter(dummy_op).
+                                 batch(10, drop_remainder=True))
+  return ds_with_unknown_cardinality
+
+
 def multi_input_output_model():
   a = keras.layers.Input(shape=(3,), name='input_a')
   b = keras.layers.Input(shape=(5,), name='input_b')
@@ -841,9 +858,7 @@ class TestDistributionStrategyWithDatasets(test.TestCase,
 
       model.fit(dataset_dict, epochs=1, steps_per_epoch=2, verbose=1)
 
-  # TODO(b/122743976): Include TPUStrategy for this test as well once
-  # step inference is supported.
-  @combinations.generate(strategy_minus_tpu_combinations())
+  @combinations.generate(all_strategy_combinations())
   def test_fit_eval_and_predict_methods_on_dataset_without_steps(
       self, distribution):
     with self.cached_session():
@@ -864,7 +879,7 @@ class TestDistributionStrategyWithDatasets(test.TestCase,
       predict_with_numpy = model.predict(inputs, batch_size=10)
 
       dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
-      dataset = dataset.batch(10)
+      dataset = dataset.batch(10, drop_remainder=True)
       fit_with_ds = model.fit(dataset, epochs=1).history
       eval_with_ds = model.evaluate(dataset)
       predict_dataset = dataset_ops.Dataset.from_tensor_slices(inputs)
@@ -877,6 +892,61 @@ class TestDistributionStrategyWithDatasets(test.TestCase,
       self.assertAllClose(
           predict_with_numpy, predict_with_ds, atol=1e-4, rtol=1e-4)
 
+  @combinations.generate(all_strategy_combinations())
+  def test_on_dataset_with_unknown_cardinality_without_steps(
+      self, distribution):
+    with self.cached_session():
+      with distribution.scope():
+        model = get_model()
+        optimizer = gradient_descent.GradientDescentOptimizer(0.001)
+        loss = 'mse'
+        metrics = ['mae', keras.metrics.CategoricalAccuracy()]
+        model.compile(optimizer, loss, metrics=metrics)
+
+      inputs = np.zeros((1000, 3), dtype=np.float32)
+      targets = np.zeros((1000, 4), dtype=np.float32)
+      # steps/steps_per_epoch are calculated when using numpy arrays as
+      # input data.
+      fit_with_numpy = model.fit(inputs, targets, epochs=1,
+                                 batch_size=10).history
+      fit_with_numpy_multiple_epochs = model.fit(
+          inputs, targets, epochs=2, batch_size=10).history
+      eval_with_numpy = model.evaluate(inputs, targets, batch_size=10)
+      predict_with_numpy = model.predict(inputs, batch_size=10)
+
+      dataset = convert_numpy_to_dataset_with_unknown_cardinality(
+          inputs, targets)
+      predict_dataset = convert_numpy_to_dataset_with_unknown_cardinality(
+          inputs)
+
+      self.assertEqual(keras.backend.get_value(cardinality.cardinality(
+          dataset)), cardinality.UNKNOWN)
+      self.assertEqual(keras.backend.get_value(cardinality.cardinality(
+          predict_dataset)), cardinality.UNKNOWN)
+
+      eval_with_ds = model.evaluate(dataset)
+      predict_with_ds = model.predict(predict_dataset)
+      self.assertAllClose(
+          eval_with_numpy, eval_with_ds, atol=1e-4, rtol=1e-4)
+      self.assertAllClose(
+          predict_with_numpy, predict_with_ds, atol=1e-4, rtol=1e-4)
+
+      if (distributed_training_utils.is_tpu_strategy(distribution) and
+          distribution.extended.steps_per_run != 1):
+        with self.assertRaisesRegexp(ValueError, '`steps_per_epoch` '
+                                     'should be specified'):
+          fit_with_ds = model.fit(dataset, epochs=1)
+      else:
+        fit_with_ds = model.fit(dataset,
+                                epochs=1).history
+        fit_with_ds_multiple_epochs = model.fit(dataset,
+                                                epochs=2).history
+        self.assertAllClose(
+            fit_with_numpy, fit_with_ds, atol=1e-4, rtol=1e-4)
+        self.assertAllClose(
+            fit_with_numpy_multiple_epochs,
+            fit_with_ds_multiple_epochs, atol=1e-4, rtol=1e-4)
+
   @combinations.generate(all_strategy_combinations())
   def test_fit_eval_and_predict_methods_on_dataset(self, distribution):
     with self.cached_session():
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 66c29bbfdd..b81713b2b2 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -2250,9 +2250,6 @@ class Model(Network):
                                           sample_weight=None,
                                           class_weight=None,
                                           batch_size=None,
-                                          check_steps=False,
-                                          steps_name='steps',
-                                          steps=None,
                                           validation_split=0,
                                           shuffle=False,
                                           repeat=True,
@@ -2272,11 +2269,6 @@ class Model(Network):
         to, as conveyed by `y`.
       batch_size: Integer batch size. If provided, it is used to run additional
         validation checks on stateful models.
-      check_steps: boolean, True if we want to check for validity of `steps` and
-        False, otherwise.
-      steps_name: The public API's parameter name for `steps`.
-      steps: Integer or `None`. Total number of steps (batches of samples) to
-        execute.
       validation_split: Float between 0 and 1.
         Fraction of the training data to be used as validation data.
       shuffle: Boolean whether to shuffle the training data before each epoch.
@@ -2325,11 +2317,6 @@ class Model(Network):
       if shuffle:
         training_utils.verify_dataset_shuffled(x)
 
-      if check_steps and steps is None:
-        raise ValueError('When using Datasets as input, '
-                         'you should specify the `{steps_name}` argument.'
-                         .format(steps_name=steps_name))
-
     if ops.executing_eagerly_outside_functions():
       session = None
     else:
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 4a2a337821..e05b6a08f8 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -22,7 +22,6 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.data.experimental.ops import batching
-from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import input_lib
 from tensorflow.python.distribute import reduce_util as ds_reduce_util
 from tensorflow.python.framework import constant_op
@@ -73,18 +72,11 @@ def fit_distributed(model,
             batch_size, mode=ModeKeys.TRAIN))
   batch_size = model._validate_or_infer_batch_size(
       batch_size, steps_per_epoch, x)
-  steps_name = 'steps_per_epoch'
-  if isinstance(x, dataset_ops.DatasetV2):
-    steps_per_epoch = training_utils.infer_steps_for_dataset(
-        x, steps_per_epoch, steps_name=steps_name)
   dataset = model._distribution_standardize_user_data(
       x, y,
       sample_weight=sample_weight,
       class_weight=class_weight,
       batch_size=batch_size,
-      check_steps=True,
-      steps_name=steps_name,
-      steps=steps_per_epoch,
       validation_split=validation_split,
       shuffle=shuffle)
 
@@ -99,18 +91,11 @@ def fit_distributed(model,
       validation_steps, _ = distributed_training_utils.get_input_params(
           model._distribution_strategy, first_valx_value, validation_steps,
           batch_size)
-    steps_name = 'validation_steps'
-    if isinstance(val_x, dataset_ops.DatasetV2):
-      validation_steps = training_utils.infer_steps_for_dataset(
-          val_x, validation_steps, steps_name=steps_name)
     val_dataset = model._distribution_standardize_user_data(
         val_x, val_y,
         sample_weight=val_sample_weights,
         class_weight=None,
         batch_size=batch_size,
-        check_steps=True,
-        steps_name=steps_name,
-        steps=validation_steps,
         validation_split=validation_split,
         shuffle=shuffle)
   elif validation_split:
@@ -142,7 +127,8 @@ def fit_distributed(model,
         initial_epoch=initial_epoch,
         steps_per_epoch=steps_per_epoch,
         validation_steps=validation_steps,
-        validation_freq=validation_freq)
+        validation_freq=validation_freq,
+        steps_name='steps_per_epoch')
 
 
 def evaluate_distributed(model,
@@ -160,18 +146,10 @@ def evaluate_distributed(model,
     steps, batch_size = distributed_training_utils.get_input_params(
         model._distribution_strategy, first_x_value, steps, batch_size)
   batch_size = model._validate_or_infer_batch_size(batch_size, steps, x)
-  steps_name = 'steps'
-
-  if isinstance(x, dataset_ops.DatasetV2):
-    steps = training_utils.infer_steps_for_dataset(x, steps,
-                                                   steps_name=steps_name)
   dataset = model._distribution_standardize_user_data(
       x, y,
       sample_weight=sample_weight,
-      batch_size=batch_size,
-      check_steps=True,
-      steps_name=steps_name,
-      steps=steps)
+      batch_size=batch_size)
 
   if distributed_training_utils.is_tpu_strategy(model._distribution_strategy):
     return experimental_tpu_test_loop(
@@ -201,16 +179,9 @@ def predict_distributed(model,
         model._distribution_strategy, first_x_value, steps,
         batch_size, mode=ModeKeys.PREDICT)
   batch_size = model._validate_or_infer_batch_size(batch_size, steps, x)
-  steps_name = 'steps'
-  if isinstance(x, dataset_ops.DatasetV2):
-    steps = training_utils.infer_steps_for_dataset(x, steps,
-                                                   steps_name=steps_name)
   dataset = model._distribution_standardize_user_data(
       x,
       batch_size=batch_size,
-      check_steps=True,
-      steps_name=steps_name,
-      steps=steps,
       repeat=False,
       allow_partial_batch=True)
   if distributed_training_utils.is_tpu_strategy(model._distribution_strategy):
@@ -271,6 +242,13 @@ def experimental_tpu_fit_loop(model,
   # TODO(fchollet): add support for `steps_per_epoch=None` in TPU loops.
   current_strategy = model._distribution_strategy
   iterator = distributed_training_utils.get_iterator(dataset, current_strategy)
+  steps_per_epoch = training_utils.infer_steps_for_dataset(
+      dataset, steps_per_epoch, epochs, steps_name='steps_per_epoch')
+  if (current_strategy.extended.steps_per_run != 1 and
+      steps_per_epoch is None):
+    raise ValueError('`steps_per_epoch` should be specified when calling '
+                     '`fit` on the model with TPUStrategy when '
+                     '`steps_per_run` != 1 .')
 
   scope = distributed_training_utils.distributed_scope(
       strategy=current_strategy, learning_phase=1)
@@ -330,18 +308,20 @@ def experimental_tpu_fit_loop(model,
     tensor = model._all_stateful_metrics_tensors[name]
     initial_loop_values[name] = array_ops.zeros(tensor.shape, tensor.dtype)
 
-  if steps_per_epoch is None:
-    raise ValueError('`steps_per_epoch` should be specified when calling '
-                     '`fit` on the model.')
+  use_steps = steps_per_epoch is not None
+  if use_steps:
+    iteration_value = min(steps_per_epoch,
+                          current_strategy.extended.steps_per_run)
+  else:
+    iteration_value = current_strategy.extended.steps_per_run
+
   steps_per_run = K.variable(
-      value=min(steps_per_epoch, current_strategy.extended.steps_per_run),
+      value=iteration_value,
       dtype='int32',
       name='steps_per_run')
-
   ctx = current_strategy.extended.experimental_run_steps_on_iterator(
       step_fn, iterator, iterations=steps_per_run,
       initial_loop_values=initial_loop_values)
-
   train_op = ctx.run_op
   output_tensors = ctx.last_step_outputs
 
@@ -361,11 +341,16 @@ def experimental_tpu_fit_loop(model,
       mode=mode)
 
   # Calculate the steps each time on the device.
-  steps_to_run = [current_strategy.extended.steps_per_run] * (
-      steps_per_epoch // current_strategy.extended.steps_per_run)
-  if steps_per_epoch % current_strategy.extended.steps_per_run:
-    steps_to_run.append(
-        steps_per_epoch % current_strategy.extended.steps_per_run)
+  if use_steps:
+    steps_to_run = ([current_strategy.extended.steps_per_run] *
+                    (steps_per_epoch //
+                     current_strategy.extended.steps_per_run))
+    if steps_per_epoch % current_strategy.extended.steps_per_run:
+      steps_to_run.append(
+          steps_per_epoch % current_strategy.extended.steps_per_run)
+    target_steps = len(steps_to_run)
+  else:
+    target_steps = np.inf
 
   callbacks._call_begin_hook(mode)
   for epoch in range(initial_epoch, epochs):
@@ -374,7 +359,9 @@ def experimental_tpu_fit_loop(model,
     epoch_logs = {}
     step_index = 0
     prev_step_count = None
-    for step_count in steps_to_run:
+    current_step = 0
+    while current_step < target_steps:
+      step_count = steps_to_run[current_step] if use_steps else 1
       batch_logs = {'batch': step_index, 'size': 1, 'num_steps': step_count}
       callbacks._call_batch_hook(mode, 'begin', step_index, batch_logs)
       if prev_step_count is None or step_count != prev_step_count:
@@ -383,16 +370,25 @@ def experimental_tpu_fit_loop(model,
       try:
         _, outputs = K.get_session().run([train_op, output_tensors])
       except errors.OutOfRangeError:
-        logging.warning('Your dataset iterator ran out of data; '
-                        'interrupting training. Make sure that your dataset '
-                        'can generate at least `steps_per_epoch * epochs` '
-                        'batches (in this case, %d batches).' %
-                        steps_per_epoch * epochs)
+        if use_steps:
+          logging.warning('Your dataset iterator ran out of data; '
+                          'interrupting training. Make sure that your dataset '
+                          'can generate at least `steps_per_epoch * epochs` '
+                          'batches (in this case, %d batches).' %
+                          steps_per_epoch * epochs)
+        else:
+          target_steps = current_step
+          logging.info('Dataset iterator ran out of data. Inferring the '
+                       'value of `steps_per_epoch` as %s  .' % target_steps)
+          distributed_training_utils.initialize_iterator(iterator,
+                                                         current_strategy)
         break
 
       batch_logs.update(outputs)
       callbacks._call_batch_hook(mode, 'end', step_index, batch_logs)
       step_index = step_index + step_count
+      current_step += 1
+
       if callbacks.model.stop_training:
         break
 
@@ -455,7 +451,11 @@ def experimental_tpu_test_loop(model,
   """
   mode = ModeKeys.TEST
   current_strategy = model._distribution_strategy
-  iterator = distributed_training_utils.get_iterator(dataset, current_strategy)
+  iterator = distributed_training_utils.get_iterator(dataset,
+                                                     current_strategy)
+  steps = training_utils.infer_steps_for_dataset(dataset, steps,
+                                                 steps_name='steps')
+
   scope = distributed_training_utils.distributed_scope(
       strategy=current_strategy, learning_phase=0)
   scope.__enter__()
@@ -539,12 +539,29 @@ def experimental_tpu_test_loop(model,
       mode=ModeKeys.TEST)
   callbacks._call_begin_hook(mode)
 
-  assert steps is not None
   outs = [0.] * len(model.metrics_names)
-  for step in range(steps):
-    batch_logs = {'batch': step, 'size': 1}
-    callbacks._call_batch_hook(mode, 'begin', step, batch_logs)
-    _, batch_outs = K.get_session().run([test_op, output_tensors])
+  if steps is not None:
+    target_steps = steps
+  else:
+    target_steps = np.inf
+
+  current_step = 0
+  while current_step < target_steps:
+    batch_logs = {'batch': current_step, 'size': 1}
+    callbacks._call_batch_hook(mode, 'begin', current_step, batch_logs)
+    try:
+      _, batch_outs = K.get_session().run([test_op, output_tensors])
+    except errors.OutOfRangeError:
+      if steps is not None:
+        warning_msg = 'Make sure that your dataset can generate at least '
+        '`steps` batches (in this case, {} batches).'.format(steps)
+      else:
+        warning_msg = 'Number of steps ran: {} steps'.format(current_step)
+
+      logging.warning('Your dataset iterator ran out of data; '
+                      'interrupting evaluation. ' + warning_msg)
+      target_steps = current_step
+      break
     for i, label in enumerate(model.metrics_names):
       if i == 0:
         # Loss is stateless metrics.
@@ -554,15 +571,16 @@ def experimental_tpu_test_loop(model,
         outs[i] = batch_outs[label]
 
     batch_logs = cbks.make_logs(model, batch_logs, outs, mode)
-    callbacks._call_batch_hook(mode, 'end', step, batch_logs)
+    callbacks._call_batch_hook(mode, 'end', current_step, batch_logs)
     if verbose >= 1:
-      progbar.update(step + 1)
+      progbar.update(current_step + 1)
+    current_step += 1
 
   callbacks._call_end_hook(mode)
 
   scope.__exit__(None, None, None)
   if len(outs) >= 0:
-    outs[0] /= (steps)
+    outs[0] /= (target_steps)
 
   if len(outs) == 1:
     return outs[0]
@@ -591,6 +609,8 @@ def experimental_tpu_predict_loop(model,
       (if the model has multiple outputs).
   """
   mode = ModeKeys.PREDICT
+  steps = training_utils.infer_steps_for_dataset(dataset, steps,
+                                                 steps_name='steps')
   dataset_fully_shaped = (distributed_training_utils.
                           is_dataset_shape_fully_defined(dataset))
   padding_handler = None
@@ -697,22 +717,40 @@ def experimental_tpu_predict_loop(model,
       mode=mode)
   callbacks._call_begin_hook(mode)
 
-  assert steps is not None
   # Since we do not know how many samples we will see, we cannot pre-allocate
   # the returned Numpy arrays. Instead, we store one array per batch seen
   # and concatenate them upon returning.
   unconcatenated_outs = [[] for _ in model.outputs]
-  for step in range(steps):
-    batch_logs = {'batch': step, 'size': 1}
-    callbacks._call_batch_hook(mode, 'begin', step, batch_logs)
-    _, batch_outs = K.get_session().run([predict_op, output_tensors])
+  if steps is not None:
+    target_steps = steps
+  else:
+    target_steps = np.inf
+
+  current_step = 0
+  while current_step < target_steps:
+    batch_logs = {'batch': current_step, 'size': 1}
+    callbacks._call_batch_hook(mode, 'begin', current_step, batch_logs)
+    try:
+      _, batch_outs = K.get_session().run([predict_op, output_tensors])
+    except errors.OutOfRangeError:
+      if steps is not None:
+        warning_msg = 'Make sure that your dataset can generate at least '
+        '`steps` batches (in this case, {} batches).'.format(steps)
+      else:
+        warning_msg = 'Number of steps ran: {} steps'.format(current_step)
+
+      logging.warning('Your dataset iterator ran out of data; '
+                      'interrupting evaluation. ' + warning_msg)
+      break
+
     # TODO(priyag): maybe need to unwrap the outputs first for MirroredStrategy.
     for i, label in enumerate(model.output_names):
       unconcatenated_outs[i].extend(batch_outs[label])
     batch_logs = cbks.make_logs(model, batch_logs, batch_outs, mode)
-    callbacks._call_batch_hook(mode, 'end', step, batch_logs)
+    callbacks._call_batch_hook(mode, 'end', current_step, batch_logs)
     if verbose >= 1:
-      progbar.update(step + 1)
+      progbar.update(current_step + 1)
+    current_step += 1
 
   callbacks._call_end_hook(mode)
 
-- 
GitLab


From 62c846cf33eb744778b0b81125e489b3e35c942b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 8 Feb 2019 15:19:16 -0800
Subject: [PATCH 2209/2345] Reuse python_utils!

PiperOrigin-RevId: 233136100
---
 .../interpreter_wrapper.cc                    | 17 ++++-------------
 .../python/optimize/calibration_wrapper.cc    | 18 +++++-------------
 tensorflow/lite/toco/python/BUILD             |  1 +
 .../lite/toco/python/toco_python_api.cc       | 19 ++++++-------------
 4 files changed, 16 insertions(+), 39 deletions(-)

diff --git a/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.cc b/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.cc
index 41cebf867e..6023587d3b 100644
--- a/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.cc
+++ b/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.cc
@@ -26,14 +26,6 @@ limitations under the License.
 #include "tensorflow/lite/python/interpreter_wrapper/python_utils.h"
 #include "tensorflow/lite/string_util.h"
 
-#if PY_MAJOR_VERSION >= 3
-#define PY_TO_CPPSTRING PyBytes_AsStringAndSize
-#define CPP_TO_PYSTRING PyBytes_FromStringAndSize
-#else
-#define PY_TO_CPPSTRING PyString_AsStringAndSize
-#define CPP_TO_PYSTRING PyString_FromStringAndSize
-#endif
-
 #define TFLITE_PY_CHECK(x)               \
   if ((x) != kTfLiteOk) {                \
     return error_reporter_->exception(); \
@@ -58,6 +50,8 @@ namespace interpreter_wrapper {
 
 namespace {
 
+using python_utils::PyDecrefDeleter;
+
 std::unique_ptr<tflite::Interpreter> CreateInterpreter(
     const tflite::FlatBufferModel* model,
     const tflite::ops::builtin::BuiltinOpResolver& resolver) {
@@ -74,10 +68,6 @@ std::unique_ptr<tflite::Interpreter> CreateInterpreter(
   return interpreter;
 }
 
-struct PyDecrefDeleter {
-  void operator()(PyObject* p) const { Py_DECREF(p); }
-};
-
 PyObject* PyArrayFromIntVector(const int* data, npy_intp size) {
   void* pydata = malloc(size * sizeof(int));
   memcpy(pydata, data, size * sizeof(int));
@@ -429,7 +419,8 @@ InterpreterWrapper* InterpreterWrapper::CreateWrapperCPPFromBuffer(
   char * buf = nullptr;
   Py_ssize_t length;
   std::unique_ptr<PythonErrorReporter> error_reporter(new PythonErrorReporter);
-  if (PY_TO_CPPSTRING(data, &buf, &length) == -1) {
+
+  if (python_utils::ConvertFromPyString(data, &buf, &length) == -1) {
     return nullptr;
   }
   std::unique_ptr<tflite::FlatBufferModel> model =
diff --git a/tensorflow/lite/python/optimize/calibration_wrapper.cc b/tensorflow/lite/python/optimize/calibration_wrapper.cc
index 21f96f848c..8bbe9cc962 100644
--- a/tensorflow/lite/python/optimize/calibration_wrapper.cc
+++ b/tensorflow/lite/python/optimize/calibration_wrapper.cc
@@ -29,14 +29,6 @@ limitations under the License.
 #include "tensorflow/lite/tools/optimize/calibrator.h"
 #include "tensorflow/lite/tools/optimize/quantize_model.h"
 
-#if PY_MAJOR_VERSION >= 3
-#define PY_TO_CPPSTRING PyBytes_AsStringAndSize
-#define CPP_TO_PYSTRING PyBytes_FromStringAndSize
-#else
-#define PY_TO_CPPSTRING PyString_AsStringAndSize
-#define CPP_TO_PYSTRING PyString_FromStringAndSize
-#endif
-
 #define TFLITE_PY_CHECK(x)               \
   if ((x) != kTfLiteOk) {                \
     return error_reporter_->exception(); \
@@ -53,9 +45,7 @@ namespace calibration_wrapper {
 
 namespace {
 
-struct PyDecrefDeleter {
-  void operator()(PyObject* p) const { Py_DECREF(p); }
-};
+using python_utils::PyDecrefDeleter;
 
 std::unique_ptr<tflite::ModelT> CreateMutableModel(const tflite::Model& model) {
   std::unique_ptr<tflite::ModelT> copied_model =
@@ -178,7 +168,8 @@ PyObject* CalibrationWrapper::QuantizeModel() {
     error_reporter_->exception();
     return nullptr;
   }
-  return CPP_TO_PYSTRING(
+
+  return python_utils::ConvertToPyString(
       reinterpret_cast<const char*>(builder.GetCurrentBufferPointer()),
       builder.GetSize());
 }
@@ -189,7 +180,8 @@ PyObject* CalibrationWrapper::QuantizeModel() {
   char* buf = nullptr;
   Py_ssize_t length;
   std::unique_ptr<PythonErrorReporter> error_reporter(new PythonErrorReporter);
-  if (PY_TO_CPPSTRING(data, &buf, &length) == -1) {
+
+  if (python_utils::ConvertFromPyString(data, &buf, &length) == -1) {
     return nullptr;
   }
   std::unique_ptr<tflite::FlatBufferModel> model =
diff --git a/tensorflow/lite/toco/python/BUILD b/tensorflow/lite/toco/python/BUILD
index 6da48333a2..2f5654c56e 100644
--- a/tensorflow/lite/toco/python/BUILD
+++ b/tensorflow/lite/toco/python/BUILD
@@ -25,6 +25,7 @@ cc_library(
     deps = [
         "//third_party/python_runtime:headers",
         "//tensorflow/core:lib",
+        "//tensorflow/lite/python/interpreter_wrapper:python_utils",
         "//tensorflow/lite/toco:model_flags_proto_cc",
         "//tensorflow/lite/toco:toco_flags_proto_cc",
         "//tensorflow/lite/toco:toco_graphviz_dump_options",
diff --git a/tensorflow/lite/toco/python/toco_python_api.cc b/tensorflow/lite/toco/python/toco_python_api.cc
index ce8e3c9df8..6fad092f35 100644
--- a/tensorflow/lite/toco/python/toco_python_api.cc
+++ b/tensorflow/lite/toco/python/toco_python_api.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include <vector>
 #include "tensorflow/core/platform/logging.h"
 
+#include "tensorflow/lite/python/interpreter_wrapper/python_utils.h"
 #include "tensorflow/lite/toco/model_flags.pb.h"
 #include "tensorflow/lite/toco/python/toco_python_api.h"
 #include "tensorflow/lite/toco/toco_flags.pb.h"
@@ -26,14 +27,6 @@ limitations under the License.
 
 namespace toco {
 
-#if PY_MAJOR_VERSION >= 3
-#define TOCO_PY_TO_CPPSTRING PyBytes_AsStringAndSize
-#define TOCO_FROM_CPPSTRING_TO_PY PyBytes_FromStringAndSize
-#else
-#define TOCO_PY_TO_CPPSTRING PyString_AsStringAndSize
-#define TOCO_FROM_CPPSTRING_TO_PY PyString_FromStringAndSize
-#endif
-
 // NOTE(aselle): We are using raw PyObject's here because we want to make
 // sure we input and output bytes rather than unicode strings for Python3.
 PyObject* TocoConvert(PyObject* model_flags_proto_txt_raw,
@@ -44,7 +37,7 @@ PyObject* TocoConvert(PyObject* model_flags_proto_txt_raw,
   auto ConvertArg = [&](PyObject* obj, bool* error) {
     char* buf;
     Py_ssize_t len;
-    if (TOCO_PY_TO_CPPSTRING(obj, &buf, &len) == -1) {
+    if (::tflite::python_utils::ConvertFromPyString(obj, &buf, &len) == -1) {
       *error = true;
       return std::string();
     } else {
@@ -96,15 +89,15 @@ PyObject* TocoConvert(PyObject* model_flags_proto_txt_raw,
     PyObject* dict = PyDict_New();
     PyDict_SetItemString(
         dict, "flatbuffer",
-        TOCO_FROM_CPPSTRING_TO_PY(output_file_contents_txt.data(),
-                                  output_file_contents_txt.size()));
+        ::tflite::python_utils::ConvertToPyString(
+            output_file_contents_txt.data(), output_file_contents_txt.size()));
     PyDict_SetItemString(dict, "arithmetic_ops",
                          PyLong_FromLong(model->ArithmeticOpsCount()));
     return dict;
   }
   // Convert arguments back to byte (py3) or str (py2)
-  return TOCO_FROM_CPPSTRING_TO_PY(output_file_contents_txt.data(),
-                                   output_file_contents_txt.size());
+  return ::tflite::python_utils::ConvertToPyString(
+      output_file_contents_txt.data(), output_file_contents_txt.size());
 }
 
 }  // namespace toco
-- 
GitLab


From 9913382f56957d2880a1990a3eff97303373ffe8 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Fri, 8 Feb 2019 15:34:29 -0800
Subject: [PATCH 2210/2345] Initial implementation of WHILE op

PiperOrigin-RevId: 233138666
---
 tensorflow/lite/context_util.h                |   1 +
 tensorflow/lite/kernels/BUILD                 |  18 +
 tensorflow/lite/kernels/if_test.cc            |  25 +-
 tensorflow/lite/kernels/subgraph_test_util.cc | 330 +++++++++++++++---
 tensorflow/lite/kernels/subgraph_test_util.h  |  93 ++++-
 .../lite/kernels/subgraph_test_util_test.cc   | 141 ++++++--
 tensorflow/lite/kernels/while.cc              | 193 ++++++++++
 tensorflow/lite/kernels/while_test.cc         |  85 +++++
 8 files changed, 780 insertions(+), 106 deletions(-)
 create mode 100644 tensorflow/lite/kernels/while.cc
 create mode 100644 tensorflow/lite/kernels/while_test.cc

diff --git a/tensorflow/lite/context_util.h b/tensorflow/lite/context_util.h
index 68b91ea0b9..2f846cc259 100644
--- a/tensorflow/lite/context_util.h
+++ b/tensorflow/lite/context_util.h
@@ -38,6 +38,7 @@ class TfLiteIntArrayView {
   const_iterator begin() const { return int_array_->data; }
   const_iterator end() const { return &int_array_->data[int_array_->size]; }
   size_t size() const { return end() - begin(); }
+  int operator[](size_t pos) const { return int_array_->data[pos]; }
 
  private:
   const TfLiteIntArray* int_array_;
diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index 795bd80788..5bd58b501a 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -225,6 +225,7 @@ cc_library(
         "unidirectional_sequence_rnn.cc",
         "unique.cc",
         "unpack.cc",
+        "while.cc",
         "zeros_like.cc",
     ],
     hdrs = [
@@ -1227,6 +1228,23 @@ tf_cc_test(
     ],
 )
 
+tf_cc_test(
+    name = "while_test",
+    size = "small",
+    srcs = ["while_test.cc"],
+    tags = ["tflite_not_portable_ios"],
+    deps = [
+        ":builtin_ops",
+        ":kernel_util",
+        ":subgraph_test_util",
+        "//tensorflow/lite:builtin_op_data",
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite/kernels:test_util",
+        "@com_google_googletest//:gtest",
+        "@flatbuffers",
+    ],
+)
+
 tf_cc_test(
     name = "fill_test",
     size = "small",
diff --git a/tensorflow/lite/kernels/if_test.cc b/tensorflow/lite/kernels/if_test.cc
index a1460cdb42..0f90db131b 100644
--- a/tensorflow/lite/kernels/if_test.cc
+++ b/tensorflow/lite/kernels/if_test.cc
@@ -24,25 +24,21 @@ limitations under the License.
 
 namespace tflite {
 
-using subgraph_test_util::BuildAddSubgraph;
-using subgraph_test_util::BuildIfSubgraph;
-using subgraph_test_util::BuildMulSubgraph;
-using subgraph_test_util::BuildPadSubgraph;
 using subgraph_test_util::CheckIntTensor;
+using subgraph_test_util::ControlFlowOpTest;
 using subgraph_test_util::FillIntTensor;
 
 namespace {
 
 // A simple test that performs `ADD` if condition is true, and `MUL` otherwise.
 // The computation is: `cond ? a + b : a * b`.
-class SimpleIfTest : public ::testing::Test {
+class SimpleIfTest : public ControlFlowOpTest {
  protected:
   void SetUp() override {
-    interpreter_.reset(new Interpreter);
     interpreter_->AddSubgraphs(2);
-    BuildAddSubgraph(interpreter_->subgraph(1));
-    BuildMulSubgraph(interpreter_->subgraph(2));
-    BuildIfSubgraph(&interpreter_->primary_subgraph());
+    builder_->BuildAddSubgraph(interpreter_->subgraph(1));
+    builder_->BuildMulSubgraph(interpreter_->subgraph(2));
+    builder_->BuildIfSubgraph(&interpreter_->primary_subgraph());
 
     interpreter_->ResizeInputTensor(interpreter_->inputs()[0], {1});
     interpreter_->ResizeInputTensor(interpreter_->inputs()[1], {2});
@@ -52,7 +48,6 @@ class SimpleIfTest : public ::testing::Test {
     FillIntTensor(interpreter_->tensor(interpreter_->inputs()[1]), {5, 7});
     FillIntTensor(interpreter_->tensor(interpreter_->inputs()[2]), {1, 2});
   }
-  std::unique_ptr<Interpreter> interpreter_;
 };
 
 TEST_F(SimpleIfTest, TestIfTrue) {
@@ -71,14 +66,13 @@ TEST_F(SimpleIfTest, TestIfFalse) {
 
 // Test IF op using subgraphs with dynamically sized outputs.
 // The computation is: `cond ? a + b : pad(a, b)`.
-class DynamicSubgraphIfTest : public ::testing::Test {
+class DynamicSubgraphIfTest : public ControlFlowOpTest {
  protected:
   void SetUp() override {
-    interpreter_.reset(new Interpreter);
     interpreter_->AddSubgraphs(2);
-    BuildAddSubgraph(interpreter_->subgraph(1));
-    BuildPadSubgraph(interpreter_->subgraph(2));
-    BuildIfSubgraph(&interpreter_->primary_subgraph());
+    builder_->BuildAddSubgraph(interpreter_->subgraph(1));
+    builder_->BuildPadSubgraph(interpreter_->subgraph(2));
+    builder_->BuildIfSubgraph(&interpreter_->primary_subgraph());
 
     interpreter_->ResizeInputTensor(interpreter_->inputs()[0], {1});
     interpreter_->ResizeInputTensor(interpreter_->inputs()[1], {2});
@@ -88,7 +82,6 @@ class DynamicSubgraphIfTest : public ::testing::Test {
     FillIntTensor(interpreter_->tensor(interpreter_->inputs()[1]), {5, 7});
     FillIntTensor(interpreter_->tensor(interpreter_->inputs()[2]), {1, 2});
   }
-  std::unique_ptr<Interpreter> interpreter_;
 };
 
 TEST_F(DynamicSubgraphIfTest, TestIfTrue) {
diff --git a/tensorflow/lite/kernels/subgraph_test_util.cc b/tensorflow/lite/kernels/subgraph_test_util.cc
index 0f41f9cc90..e712be1b51 100644
--- a/tensorflow/lite/kernels/subgraph_test_util.cc
+++ b/tensorflow/lite/kernels/subgraph_test_util.cc
@@ -31,92 +31,142 @@ TfLiteRegistration* Register_ADD();
 TfLiteRegistration* Register_MUL();
 // ADD and MUL are used to test dynamic sized subgraphs.
 TfLiteRegistration* Register_PAD();
+TfLiteRegistration* Register_LESS_EQUAL();
 }  // namespace builtin
 namespace custom {
 TfLiteRegistration* Register_IF();
+TfLiteRegistration* Register_WHILE();
 }  // namespace custom
 }  // namespace ops
 
 namespace subgraph_test_util {
 
+namespace {
+
 void SetupTensor(Subgraph* subgraph, int tensor_index, TfLiteType type) {
   ASSERT_EQ(subgraph->SetTensorParametersReadWrite(tensor_index, type, "", 0,
                                                    nullptr, {}, false),
             kTfLiteOk);
 }
 
-void BuildAddSubgraph(Subgraph* subgraph) {
+}  // namespace
+
+SubgraphBuilder::~SubgraphBuilder() {
+  for (auto buffer : buffers_) {
+    free(buffer);
+  }
+}
+
+void SubgraphBuilder::BuildAddSubgraph(Subgraph* subgraph) {
+  const int kInput1 = 0;
+  const int kInput2 = 1;
+  const int kOutput = 2;
+  const int kTensorCount = 3;
+  // kInput1(0) --> +---+
+  //                |ADD| --> kOutput(2)
+  // kInput2(1) --> +---+
+
   int first_new_tensor_index;
-  ASSERT_EQ(subgraph->AddTensors(3, &first_new_tensor_index), kTfLiteOk);
+  ASSERT_EQ(subgraph->AddTensors(kTensorCount, &first_new_tensor_index),
+            kTfLiteOk);
   ASSERT_EQ(first_new_tensor_index, 0);
-  ASSERT_EQ(subgraph->SetInputs({0, 1}), kTfLiteOk);
-  ASSERT_EQ(subgraph->SetOutputs({2}), kTfLiteOk);
+  ASSERT_EQ(subgraph->SetInputs({kInput1, kInput2}), kTfLiteOk);
+  ASSERT_EQ(subgraph->SetOutputs({kOutput}), kTfLiteOk);
 
-  SetupTensor(subgraph, 0, kTfLiteInt32);
-  SetupTensor(subgraph, 1, kTfLiteInt32);
-  SetupTensor(subgraph, 2, kTfLiteInt32);
+  SetupTensor(subgraph, kInput1, kTfLiteInt32);
+  SetupTensor(subgraph, kInput2, kTfLiteInt32);
+  SetupTensor(subgraph, kOutput, kTfLiteInt32);
 
   TfLiteAddParams* params =
       reinterpret_cast<TfLiteAddParams*>(malloc(sizeof(TfLiteAddParams)));
   params->activation = kTfLiteActNone;
   int node_index;
-  subgraph->AddNodeWithParameters({0, 1}, {2}, nullptr, 0, params,
-                                  ::tflite::ops::builtin::Register_ADD(),
-                                  &node_index);
+  subgraph->AddNodeWithParameters(
+      {kInput1, kInput2}, {kOutput}, nullptr, 0, params,
+      ::tflite::ops::builtin::Register_ADD(), &node_index);
 }
 
 // Build a subgraph with an mul op. Helper function for testing.
-void BuildMulSubgraph(Subgraph* subgraph) {
+void SubgraphBuilder::BuildMulSubgraph(Subgraph* subgraph) {
+  const int kInput1 = 0;
+  const int kInput2 = 1;
+  const int kOutput = 2;
+  const int kTensorCount = 3;
+  // kInput1(0) --> +---+
+  //                |MUL| --> kOutput(2)
+  // kInput2(1) --> +---+
+
   int first_new_tensor_index;
-  ASSERT_EQ(subgraph->AddTensors(3, &first_new_tensor_index), kTfLiteOk);
+  ASSERT_EQ(subgraph->AddTensors(kTensorCount, &first_new_tensor_index),
+            kTfLiteOk);
   ASSERT_EQ(first_new_tensor_index, 0);
-  ASSERT_EQ(subgraph->SetInputs({0, 1}), kTfLiteOk);
-  ASSERT_EQ(subgraph->SetOutputs({2}), kTfLiteOk);
+  ASSERT_EQ(subgraph->SetInputs({kInput1, kInput2}), kTfLiteOk);
+  ASSERT_EQ(subgraph->SetOutputs({kOutput}), kTfLiteOk);
 
-  SetupTensor(subgraph, 0, kTfLiteInt32);
-  SetupTensor(subgraph, 1, kTfLiteInt32);
-  SetupTensor(subgraph, 2, kTfLiteInt32);
+  SetupTensor(subgraph, kInput1, kTfLiteInt32);
+  SetupTensor(subgraph, kInput2, kTfLiteInt32);
+  SetupTensor(subgraph, kOutput, kTfLiteInt32);
 
   TfLiteMulParams* params =
       reinterpret_cast<TfLiteMulParams*>(malloc(sizeof(TfLiteMulParams)));
   params->activation = kTfLiteActNone;
   int node_index;
-  subgraph->AddNodeWithParameters({0, 1}, {2}, nullptr, 0, params,
-                                  ::tflite::ops::builtin::Register_MUL(),
-                                  &node_index);
+  subgraph->AddNodeWithParameters(
+      {kInput1, kInput2}, {kOutput}, nullptr, 0, params,
+      ::tflite::ops::builtin::Register_MUL(), &node_index);
 }
 
 // Build a subgraph with a pad op. Helper function for testing.
-void BuildPadSubgraph(Subgraph* subgraph) {
+void SubgraphBuilder::BuildPadSubgraph(Subgraph* subgraph) {
+  const int kInput1 = 0;
+  const int kInput2 = 1;
+  const int kOutput = 2;
+  const int kTensorCount = 3;
+  // kInput1(0) --> +---+
+  //                |PAD| --> kOutput(2)
+  // kInput2(1) --> +---+
+
   int first_new_tensor_index;
-  ASSERT_EQ(subgraph->AddTensors(3, &first_new_tensor_index), kTfLiteOk);
+  ASSERT_EQ(subgraph->AddTensors(kTensorCount, &first_new_tensor_index),
+            kTfLiteOk);
   ASSERT_EQ(first_new_tensor_index, 0);
-  ASSERT_EQ(subgraph->SetInputs({0, 1}), kTfLiteOk);
-  ASSERT_EQ(subgraph->SetOutputs({2}), kTfLiteOk);
+  ASSERT_EQ(subgraph->SetInputs({kInput1, kInput2}), kTfLiteOk);
+  ASSERT_EQ(subgraph->SetOutputs({kOutput}), kTfLiteOk);
 
-  SetupTensor(subgraph, 0, kTfLiteInt32);
-  SetupTensor(subgraph, 1, kTfLiteInt32);
-  SetupTensor(subgraph, 2, kTfLiteInt32);
+  SetupTensor(subgraph, kInput1, kTfLiteInt32);
+  SetupTensor(subgraph, kInput2, kTfLiteInt32);
+  SetupTensor(subgraph, kOutput, kTfLiteInt32);
 
   TfLitePadParams* params =
       reinterpret_cast<TfLitePadParams*>(malloc(sizeof(TfLitePadParams)));
   int node_index;
-  subgraph->AddNodeWithParameters({0, 1}, {2}, nullptr, 0, params,
-                                  ::tflite::ops::builtin::Register_PAD(),
-                                  &node_index);
+  subgraph->AddNodeWithParameters(
+      {kInput1, kInput2}, {kOutput}, nullptr, 0, params,
+      ::tflite::ops::builtin::Register_PAD(), &node_index);
 }
 
-void BuildIfSubgraph(Subgraph* subgraph) {
+void SubgraphBuilder::BuildIfSubgraph(Subgraph* subgraph) {
+  const int kCondInput = 0;
+  const int kInput1 = 1;
+  const int kInput2 = 2;
+  const int kOutput = 3;
+  const int kTensorCount = 4;
+
+  // kCondInput(0) --> +----+
+  // kInput1(1)  ----> | IF | --> kOutput(3)
+  // kInput2(2)  ----> +----+
+
   int first_new_tensor_index;
-  ASSERT_EQ(subgraph->AddTensors(4, &first_new_tensor_index), kTfLiteOk);
+  ASSERT_EQ(subgraph->AddTensors(kTensorCount, &first_new_tensor_index),
+            kTfLiteOk);
   ASSERT_EQ(first_new_tensor_index, 0);
-  ASSERT_EQ(subgraph->SetInputs({0, 1, 2}), kTfLiteOk);
-  ASSERT_EQ(subgraph->SetOutputs({3}), kTfLiteOk);
+  ASSERT_EQ(subgraph->SetInputs({kCondInput, kInput1, kInput2}), kTfLiteOk);
+  ASSERT_EQ(subgraph->SetOutputs({kOutput}), kTfLiteOk);
 
-  SetupTensor(subgraph, 0, kTfLiteBool);
-  SetupTensor(subgraph, 1, kTfLiteInt32);
-  SetupTensor(subgraph, 2, kTfLiteInt32);
-  SetupTensor(subgraph, 3, kTfLiteInt32);
+  SetupTensor(subgraph, kCondInput, kTfLiteBool);
+  SetupTensor(subgraph, kInput1, kTfLiteInt32);
+  SetupTensor(subgraph, kInput2, kTfLiteInt32);
+  SetupTensor(subgraph, kOutput, kTfLiteInt32);
 
   flexbuffers::Builder fbb;
   fbb.Map([&]() {
@@ -128,11 +178,197 @@ void BuildIfSubgraph(Subgraph* subgraph) {
 
   int node_index;
   subgraph->AddNodeWithParameters(
-      {0, 1, 2}, {3}, reinterpret_cast<const char*>(buffer.data()),
-      buffer.size(), nullptr, ::tflite::ops::custom::Register_IF(),
+      {kCondInput, kInput1, kInput2}, {kOutput},
+      reinterpret_cast<const char*>(buffer.data()), buffer.size(), nullptr,
+      ::tflite::ops::custom::Register_IF(), &node_index);
+}
+
+void SubgraphBuilder::BuildLessEqualCondSubgraph(Subgraph* subgraph, int rhs) {
+  const int kInput1 = 0;
+  const int kInput2 = 1;
+  const int kOutput = 2;
+  const int kConstRhs = 3;
+  const int kTensorCount = 4;
+
+  // kInput1(0) ----> +------------+
+  //                  | LESS_EQUAL | --> kOutput(2)
+  // kConstRhs(3) --> +------------+
+  //
+  // kInput2(1) --> (unused)
+
+  int first_new_tensor_index;
+  ASSERT_EQ(subgraph->AddTensors(kTensorCount, &first_new_tensor_index),
+            kTfLiteOk);
+  ASSERT_EQ(first_new_tensor_index, 0);
+  ASSERT_EQ(subgraph->SetInputs({kInput1, kInput2}), kTfLiteOk);
+  ASSERT_EQ(subgraph->SetOutputs({kOutput}), kTfLiteOk);
+
+  SetupTensor(subgraph, kInput1, kTfLiteInt32);
+  SetupTensor(subgraph, kInput2, kTfLiteInt32);
+  SetupTensor(subgraph, kOutput, kTfLiteBool);
+
+  CreateConstantInt32Tensor(subgraph, kConstRhs, {1}, {rhs});
+  int node_index;
+  subgraph->AddNodeWithParameters(
+      {kInput1, kConstRhs}, {kOutput}, nullptr, 0, nullptr,
+      ::tflite::ops::builtin::Register_LESS_EQUAL(), &node_index);
+}
+
+void SubgraphBuilder::BuildAccumulateLoopBodySubgraph(Subgraph* subgraph) {
+  const int kInputCounter = 0;
+  const int kInputValue = 1;
+  const int kOutputCounter = 2;
+  const int kOutputValue = 3;
+  const int kConstStep = 4;
+  const int kTensorCount = 5;
+
+  // kInputCounter(0) --> +-----+
+  //                      | ADD | --> kOutputCounter(2)
+  // kConstStep(4) -----> +-----+            |
+  //                                         |
+  //                                         v
+  //                                      +-----+
+  //                                      | ADD | --> kOutputValue(3)
+  // kInputValue(1) ----------------------+-----+
+
+  int first_new_tensor_index;
+  ASSERT_EQ(subgraph->AddTensors(kTensorCount, &first_new_tensor_index),
+            kTfLiteOk);
+  ASSERT_EQ(first_new_tensor_index, 0);
+  ASSERT_EQ(subgraph->SetInputs({kInputCounter, kInputValue}), kTfLiteOk);
+  ASSERT_EQ(subgraph->SetOutputs({kOutputCounter, kOutputValue}), kTfLiteOk);
+
+  SetupTensor(subgraph, kInputCounter, kTfLiteInt32);
+  SetupTensor(subgraph, kInputValue, kTfLiteInt32);
+  SetupTensor(subgraph, kOutputCounter, kTfLiteInt32);
+  SetupTensor(subgraph, kOutputValue, kTfLiteInt32);
+  CreateConstantInt32Tensor(subgraph, kConstStep, {1}, {1});
+
+  int node_index;
+  TfLiteAddParams* params =
+      reinterpret_cast<TfLiteAddParams*>(malloc(sizeof(TfLiteAddParams)));
+  params->activation = kTfLiteActNone;
+  subgraph->AddNodeWithParameters({0, 4}, {2}, nullptr, 0, params,
+                                  ::tflite::ops::builtin::Register_ADD(),
+                                  &node_index);
+  params = reinterpret_cast<TfLiteAddParams*>(malloc(sizeof(TfLiteAddParams)));
+  params->activation = kTfLiteActNone;
+  subgraph->AddNodeWithParameters({2, 1}, {3}, nullptr, 0, params,
+                                  ::tflite::ops::builtin::Register_ADD(),
+                                  &node_index);
+}
+
+void SubgraphBuilder::BuildPadLoopBodySubgraph(Subgraph* subgraph,
+                                               const std::vector<int> padding) {
+  const int kInputCounter = 0;
+  const int kInputValue = 1;
+  const int kOutputCounter = 2;
+  const int kOutputValue = 3;
+  const int kConstStep = 4;
+  const int kConstPadding = 5;
+  const int kTensorCount = 6;
+
+  // kInputCounter(0) --> +-----+
+  //                      | ADD | --> kOutputCounter(2)
+  // kConstStep(4) -----> +-----+
+  //
+  // kInputValue(1) ----> +-----+
+  //                      | PAD | --> kOutputValue(3)
+  // kConstPadding(5) --> +-----+
+
+  int first_new_tensor_index;
+  ASSERT_EQ(subgraph->AddTensors(kTensorCount, &first_new_tensor_index),
+            kTfLiteOk);
+  ASSERT_EQ(first_new_tensor_index, 0);
+  ASSERT_EQ(subgraph->SetInputs({kInputCounter, kInputValue}), kTfLiteOk);
+  ASSERT_EQ(subgraph->SetOutputs({kOutputCounter, kOutputValue}), kTfLiteOk);
+
+  SetupTensor(subgraph, kInputCounter, kTfLiteInt32);
+  SetupTensor(subgraph, kInputValue, kTfLiteInt32);
+  SetupTensor(subgraph, kOutputCounter, kTfLiteInt32);
+  SetupTensor(subgraph, kOutputValue, kTfLiteInt32);
+
+  CreateConstantInt32Tensor(subgraph, kConstStep, {1}, {1});
+  ASSERT_EQ(padding.size() % 2, 0);
+  int padding_dims = padding.size();
+  CreateConstantInt32Tensor(subgraph, kConstPadding, {1, padding_dims},
+                            padding);
+
+  int node_index;
+  TfLiteAddParams* add_params =
+      reinterpret_cast<TfLiteAddParams*>(malloc(sizeof(TfLiteAddParams)));
+  add_params->activation = kTfLiteActNone;
+  subgraph->AddNodeWithParameters(
+      {kInputCounter, kConstStep}, {kOutputCounter}, nullptr, 0, add_params,
+      ::tflite::ops::builtin::Register_ADD(), &node_index);
+  TfLitePadParams* pad_params =
+      reinterpret_cast<TfLitePadParams*>(malloc(sizeof(TfLiteAddParams)));
+  subgraph->AddNodeWithParameters(
+      {kInputValue, kConstPadding}, {kOutputValue}, nullptr, 0, pad_params,
+      ::tflite::ops::builtin::Register_PAD(), &node_index);
+}
+
+void SubgraphBuilder::BuildWhileSubgraph(Subgraph* subgraph) {
+  const int kInput1 = 0;
+  const int kInput2 = 1;
+  const int kOutput1 = 2;
+  const int kOutput2 = 3;
+  const int kTensorCount = 4;
+
+  // kInput1(0) --> +-------+ --> kOutput1(2)
+  //                | WHILE |
+  // kInput2(1) --> +-------+ --> kOutput2(3)
+
+  int first_new_tensor_index;
+  ASSERT_EQ(subgraph->AddTensors(kTensorCount, &first_new_tensor_index),
+            kTfLiteOk);
+  ASSERT_EQ(first_new_tensor_index, 0);
+  ASSERT_EQ(subgraph->SetInputs({kInput1, kInput2}), kTfLiteOk);
+  ASSERT_EQ(subgraph->SetOutputs({kOutput1, kOutput2}), kTfLiteOk);
+
+  SetupTensor(subgraph, kInput1, kTfLiteInt32);
+  SetupTensor(subgraph, kInput2, kTfLiteInt32);
+  SetupTensor(subgraph, kOutput1, kTfLiteInt32);
+  SetupTensor(subgraph, kOutput2, kTfLiteInt32);
+
+  flexbuffers::Builder fbb;
+  fbb.Map([&]() {
+    fbb.Int("cond_subgraph_index", 1);
+    fbb.Int("body_subgraph_index", 2);
+  });
+  fbb.Finish();
+  const auto& buffer = fbb.GetBuffer();
+
+  int node_index;
+  subgraph->AddNodeWithParameters(
+      {0, 1}, {2, 3}, reinterpret_cast<const char*>(buffer.data()),
+      buffer.size(), nullptr, ::tflite::ops::custom::Register_WHILE(),
       &node_index);
 }
 
+void SubgraphBuilder::CreateConstantInt32Tensor(Subgraph* subgraph,
+                                                int tensor_index,
+                                                const std::vector<int>& shape,
+                                                const std::vector<int>& data) {
+  ASSERT_GT(shape.size(), 0);
+  int num_elements = 1;
+  for (int dim : shape) {
+    num_elements *= dim;
+  }
+  ASSERT_EQ(data.size(), num_elements);
+  size_t size_in_bytes = sizeof(int32_t) * num_elements;
+  // Maybe aligned.
+  int32_t* buffer = reinterpret_cast<int32_t*>(malloc(size_in_bytes));
+  for (int i = 0; i < num_elements; ++i) {
+    buffer[i] = data[i];
+  }
+  buffers_.push_back(buffer);
+  ASSERT_EQ(subgraph->SetTensorParametersReadOnly(
+                tensor_index, kTfLiteInt32, "", shape, {},
+                reinterpret_cast<const char*>(buffer), size_in_bytes),
+            kTfLiteOk);
+}
+
 void FillIntTensor(TfLiteTensor* tensor, const std::vector<int32_t>& data) {
   int count = NumElements(tensor);
   ASSERT_EQ(count, data.size());
@@ -155,5 +391,19 @@ void CheckIntTensor(const TfLiteTensor* tensor, const std::vector<int>& shape,
   }
 }
 
+void CheckBoolTensor(const TfLiteTensor* tensor, const std::vector<int>& shape,
+                     const std::vector<bool>& data) {
+  ASSERT_EQ(tensor->dims->size, shape.size());
+  for (int i = 0; i < tensor->dims->size; ++i) {
+    ASSERT_EQ(tensor->dims->data[i], shape[i]);
+  }
+  ASSERT_EQ(tensor->type, kTfLiteBool);
+  int count = NumElements(tensor);
+  ASSERT_EQ(count, data.size());
+  for (int i = 0; i < count; ++i) {
+    EXPECT_EQ(tensor->data.b[i], data[i]);
+  }
+}
+
 }  // namespace subgraph_test_util
 }  // namespace tflite
diff --git a/tensorflow/lite/kernels/subgraph_test_util.h b/tensorflow/lite/kernels/subgraph_test_util.h
index 6dc5207c4b..972f1381af 100644
--- a/tensorflow/lite/kernels/subgraph_test_util.h
+++ b/tensorflow/lite/kernels/subgraph_test_util.h
@@ -20,29 +20,87 @@ limitations under the License.
 #ifndef TENSORFLOW_LITE_KERNELS_SUBGRAPH_TEST_UTIL_H_
 #define TENSORFLOW_LITE_KERNELS_SUBGRAPH_TEST_UTIL_H_
 
+#include <gtest/gtest.h>
 #include "tensorflow/lite/core/subgraph.h"
+#include "tensorflow/lite/interpreter.h"
 
 namespace tflite {
 namespace subgraph_test_util {
 
-// Build a subgraph with a single Add op.
-// 2 inputs. 1 output.
-void BuildAddSubgraph(Subgraph* subgraph);
+// TODO(ycling): This file should be renamed as
+// `control_flow_test_util` to avoid confusion. I'll do it immediately
+// in a separated change.
+class SubgraphBuilder {
+ public:
+  ~SubgraphBuilder();
 
-// Build a subgraph with a single Mul op.
-// 2 inputs. 1 output.
-void BuildMulSubgraph(Subgraph* subgraph);
+  // Build a subgraph with a single Add op.
+  // 2 inputs. 1 output.
+  void BuildAddSubgraph(Subgraph* subgraph);
 
-// Build a subgraph with a single Pad op.
-// 2 inputs. 1 output.
-void BuildPadSubgraph(Subgraph* subgraph);
+  // Build a subgraph with a single Mul op.
+  // 2 inputs. 1 output.
+  void BuildMulSubgraph(Subgraph* subgraph);
 
-// Build a subgraph with a single If op.
-// 3 inputs:
-//   The 1st input is condition with boolean type.
-//   The 2nd and 3rd inputs are feed input the branch subgraphs.
-// 1 output.
-void BuildIfSubgraph(Subgraph* subgraph);
+  // Build a subgraph with a single Pad op.
+  // 2 inputs. 1 output.
+  void BuildPadSubgraph(Subgraph* subgraph);
+
+  // Build a subgraph with a single If op.
+  // 3 inputs:
+  //   The 1st input is condition with boolean type.
+  //   The 2nd and 3rd inputs are feed input the branch subgraphs.
+  // 1 output.
+  void BuildIfSubgraph(Subgraph* subgraph);
+
+  // Build a subgraph with a single Less op.
+  // The subgraph is used as the condition subgraph for testing `While` op.
+  // 2 inputs:
+  //   The 1st input is a counter with `kTfLiteInt32` type.
+  //   The 2nd input is ignored in this subgraph.
+  // 1 output with `kTfLiteBool` type.
+  //   Equivalent to (input < rhs).
+  void BuildLessEqualCondSubgraph(Subgraph* subgraph, int rhs);
+
+  // An accumulate loop body subgraph. Used to produce triangle number
+  // seqeuence. 2 inputs and 2 outpus
+  //   Equivalent to (counter, value) -> (counter + 1, counter + 1 + value)
+  void BuildAccumulateLoopBodySubgraph(Subgraph* subgraph);
+
+  // A pad loop body subgraph. When used in a loop it will repeatively enlarge
+  // the
+  //   tensor.
+  // 2 inputs and 2 outputs.
+  //   Equivalent to (counter, value) -> (counter + 1, tf.pad(value, padding))
+  // Note the padding is created as a constant tensor.
+  void BuildPadLoopBodySubgraph(Subgraph* subgraph,
+                                const std::vector<int> padding);
+
+  // Build a subgraph with a single While op.
+  // 2 inputs, 2 outputs.
+  void BuildWhileSubgraph(Subgraph* subgraph);
+
+ private:
+  void CreateConstantInt32Tensor(Subgraph* subgraph, int tensor_index,
+                                 const std::vector<int>& shape,
+                                 const std::vector<int>& data);
+  std::vector<void*> buffers_;
+};
+
+class ControlFlowOpTest : public ::testing::Test {
+ public:
+  ControlFlowOpTest()
+      : interpreter_(new Interpreter), builder_(new SubgraphBuilder) {}
+
+  ~ControlFlowOpTest() override {
+    interpreter_.reset();
+    builder_.reset();
+  }
+
+ protected:
+  std::unique_ptr<Interpreter> interpreter_;
+  std::unique_ptr<SubgraphBuilder> builder_;
+};
 
 // Fill a `TfLiteTensor` with a 32-bits integer vector.
 // Preconditions:
@@ -52,9 +110,12 @@ void BuildIfSubgraph(Subgraph* subgraph);
 //   the vector.
 void FillIntTensor(TfLiteTensor* tensor, const std::vector<int32_t>& data);
 
-// Check if the shape and data of a tensor is as expected.
+// Check if the shape and int32 data of a tensor is as expected.
 void CheckIntTensor(const TfLiteTensor* tensor, const std::vector<int>& shape,
                     const std::vector<int32_t>& data);
+// Check if the shape and bool data of a tensor is as expected.
+void CheckBoolTensor(const TfLiteTensor* tensor, const std::vector<int>& shape,
+                     const std::vector<bool>& data);
 
 }  // namespace subgraph_test_util
 }  // namespace tflite
diff --git a/tensorflow/lite/kernels/subgraph_test_util_test.cc b/tensorflow/lite/kernels/subgraph_test_util_test.cc
index 88cd8f580a..04e5118b54 100644
--- a/tensorflow/lite/kernels/subgraph_test_util_test.cc
+++ b/tensorflow/lite/kernels/subgraph_test_util_test.cc
@@ -24,55 +24,128 @@ namespace subgraph_test_util {
 
 namespace {
 
-// SubGraphTestUtilTest tests the helper functions defined in this file.
-TEST(SubGraphTestUtilTest, TestBuildAddSubgraph) {
-  std::unique_ptr<Interpreter> interpreter(new Interpreter);
-  BuildAddSubgraph(&interpreter->primary_subgraph());
-
-  interpreter->ResizeInputTensor(interpreter->inputs()[0], {2});
-  interpreter->ResizeInputTensor(interpreter->inputs()[1], {1, 2});
-  ASSERT_EQ(interpreter->AllocateTensors(), kTfLiteOk);
-
-  FillIntTensor(interpreter->tensor(interpreter->inputs()[0]), {5, 7});
-  FillIntTensor(interpreter->tensor(interpreter->inputs()[1]), {1, 2});
-  ASSERT_EQ(interpreter->Invoke(), kTfLiteOk);
-
-  TfLiteTensor* output = interpreter->tensor(interpreter->outputs()[0]);
+class SubgraphBuilderTest : public ::testing::Test {
+ public:
+  SubgraphBuilderTest()
+      : interpreter_(new Interpreter), builder_(new SubgraphBuilder) {}
+
+  ~SubgraphBuilderTest() override {
+    interpreter_.reset();
+    builder_.reset();
+  }
+
+ protected:
+  void TestAccumelateLoopBody(int input1, int input2, int output1,
+                              int output2) {
+    interpreter_.reset(new Interpreter);
+    builder_->BuildAccumulateLoopBodySubgraph(
+        &interpreter_->primary_subgraph());
+
+    interpreter_->ResizeInputTensor(interpreter_->inputs()[0], {1});
+    interpreter_->ResizeInputTensor(interpreter_->inputs()[1], {1});
+    ASSERT_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
+
+    FillIntTensor(interpreter_->tensor(interpreter_->inputs()[0]), {input1});
+    FillIntTensor(interpreter_->tensor(interpreter_->inputs()[1]), {input2});
+
+    ASSERT_EQ(interpreter_->Invoke(), kTfLiteOk);
+    TfLiteTensor* output_tensor1 =
+        interpreter_->tensor(interpreter_->outputs()[0]);
+    CheckIntTensor(output_tensor1, {1}, {output1});
+    TfLiteTensor* output_tensor2 =
+        interpreter_->tensor(interpreter_->outputs()[1]);
+    CheckIntTensor(output_tensor2, {1}, {output2});
+  }
+
+  std::unique_ptr<Interpreter> interpreter_;
+  std::unique_ptr<SubgraphBuilder> builder_;
+};
+
+TEST_F(SubgraphBuilderTest, TestBuildAddSubgraph) {
+  builder_->BuildAddSubgraph(&interpreter_->primary_subgraph());
+
+  interpreter_->ResizeInputTensor(interpreter_->inputs()[0], {2});
+  interpreter_->ResizeInputTensor(interpreter_->inputs()[1], {1, 2});
+  ASSERT_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
+
+  FillIntTensor(interpreter_->tensor(interpreter_->inputs()[0]), {5, 7});
+  FillIntTensor(interpreter_->tensor(interpreter_->inputs()[1]), {1, 2});
+  ASSERT_EQ(interpreter_->Invoke(), kTfLiteOk);
+
+  TfLiteTensor* output = interpreter_->tensor(interpreter_->outputs()[0]);
   CheckIntTensor(output, {1, 2}, {6, 9});
 }
 
-TEST(SubGraphTestUtilTest, TestBuildMulSubgraph) {
-  std::unique_ptr<Interpreter> interpreter(new Interpreter);
-  BuildMulSubgraph(&interpreter->primary_subgraph());
+TEST_F(SubgraphBuilderTest, TestBuildMulSubgraph) {
+  builder_->BuildMulSubgraph(&interpreter_->primary_subgraph());
 
-  interpreter->ResizeInputTensor(interpreter->inputs()[0], {2});
-  interpreter->ResizeInputTensor(interpreter->inputs()[1], {1, 2});
-  ASSERT_EQ(interpreter->AllocateTensors(), kTfLiteOk);
+  interpreter_->ResizeInputTensor(interpreter_->inputs()[0], {2});
+  interpreter_->ResizeInputTensor(interpreter_->inputs()[1], {1, 2});
+  ASSERT_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
 
-  FillIntTensor(interpreter->tensor(interpreter->inputs()[0]), {5, 7});
-  FillIntTensor(interpreter->tensor(interpreter->inputs()[1]), {1, 2});
-  ASSERT_EQ(interpreter->Invoke(), kTfLiteOk);
+  FillIntTensor(interpreter_->tensor(interpreter_->inputs()[0]), {5, 7});
+  FillIntTensor(interpreter_->tensor(interpreter_->inputs()[1]), {1, 2});
+  ASSERT_EQ(interpreter_->Invoke(), kTfLiteOk);
 
-  TfLiteTensor* output = interpreter->tensor(interpreter->outputs()[0]);
+  TfLiteTensor* output = interpreter_->tensor(interpreter_->outputs()[0]);
   CheckIntTensor(output, {1, 2}, {5, 14});
 }
 
-TEST(SubGraphTestUtilTest, TestBuildPadSubgraph) {
-  std::unique_ptr<Interpreter> interpreter(new Interpreter);
-  BuildPadSubgraph(&interpreter->primary_subgraph());
+TEST_F(SubgraphBuilderTest, TestBuildPadSubgraph) {
+  builder_->BuildPadSubgraph(&interpreter_->primary_subgraph());
 
-  interpreter->ResizeInputTensor(interpreter->inputs()[0], {2});
-  interpreter->ResizeInputTensor(interpreter->inputs()[1], {1, 2});
-  ASSERT_EQ(interpreter->AllocateTensors(), kTfLiteOk);
+  interpreter_->ResizeInputTensor(interpreter_->inputs()[0], {2});
+  interpreter_->ResizeInputTensor(interpreter_->inputs()[1], {1, 2});
+  ASSERT_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
 
-  FillIntTensor(interpreter->tensor(interpreter->inputs()[0]), {5, 7});
-  FillIntTensor(interpreter->tensor(interpreter->inputs()[1]), {1, 2});
-  ASSERT_EQ(interpreter->Invoke(), kTfLiteOk);
+  FillIntTensor(interpreter_->tensor(interpreter_->inputs()[0]), {5, 7});
+  FillIntTensor(interpreter_->tensor(interpreter_->inputs()[1]), {1, 2});
+  ASSERT_EQ(interpreter_->Invoke(), kTfLiteOk);
 
-  TfLiteTensor* output = interpreter->tensor(interpreter->outputs()[0]);
+  TfLiteTensor* output = interpreter_->tensor(interpreter_->outputs()[0]);
   CheckIntTensor(output, {5}, {0, 5, 7, 0, 0});
 }
 
+TEST_F(SubgraphBuilderTest, TestBuildLessEqualCondSubgraph) {
+  builder_->BuildLessEqualCondSubgraph(&interpreter_->primary_subgraph(), 3);
+
+  interpreter_->ResizeInputTensor(interpreter_->inputs()[0], {5});
+  interpreter_->ResizeInputTensor(interpreter_->inputs()[1], {10, 10});
+  ASSERT_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
+
+  // Test [1, 2, 3, 4, 5] <= 3 == [true, true, true, false, false]
+  // (with broadcasting).
+  FillIntTensor(interpreter_->tensor(interpreter_->inputs()[0]),
+                {1, 2, 3, 4, 5});
+  ASSERT_EQ(interpreter_->Invoke(), kTfLiteOk);
+  TfLiteTensor* output = interpreter_->tensor(interpreter_->outputs()[0]);
+  CheckBoolTensor(output, {5}, {true, true, true, false, false});
+}
+
+TEST_F(SubgraphBuilderTest, TestBuildAccumulateLoopBodySubgraph) {
+  TestAccumelateLoopBody(1, 1, 2, 3);
+  TestAccumelateLoopBody(2, 3, 3, 6);
+  TestAccumelateLoopBody(3, 6, 4, 10);
+}
+
+TEST_F(SubgraphBuilderTest, TestBuildPadLoopBodySubgraph) {
+  builder_->BuildPadLoopBodySubgraph(&interpreter_->primary_subgraph(), {1, 2});
+
+  interpreter_->ResizeInputTensor(interpreter_->inputs()[0], {1});
+  interpreter_->ResizeInputTensor(interpreter_->inputs()[1], {5});
+  ASSERT_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
+
+  FillIntTensor(interpreter_->tensor(interpreter_->inputs()[0]), {1});
+  FillIntTensor(interpreter_->tensor(interpreter_->inputs()[1]),
+                {0, 5, 7, 0, 0});
+
+  ASSERT_EQ(interpreter_->Invoke(), kTfLiteOk);
+  TfLiteTensor* output1 = interpreter_->tensor(interpreter_->outputs()[0]);
+  CheckIntTensor(output1, {1}, {2});
+  TfLiteTensor* output2 = interpreter_->tensor(interpreter_->outputs()[1]);
+  CheckIntTensor(output2, {8}, {0, 0, 5, 7, 0, 0, 0, 0});
+}
+
 }  // namespace
 }  // namespace subgraph_test_util
 }  // namespace tflite
diff --git a/tensorflow/lite/kernels/while.cc b/tensorflow/lite/kernels/while.cc
new file mode 100644
index 0000000000..237bec9b04
--- /dev/null
+++ b/tensorflow/lite/kernels/while.cc
@@ -0,0 +1,193 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "flatbuffers/flexbuffers.h"  // TF:flatbuffers
+#include "tensorflow/lite/c/builtin_op_data.h"
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/context_util.h"
+#include "tensorflow/lite/core/subgraph.h"
+#include "tensorflow/lite/kernels/kernel_util.h"
+
+namespace tflite {
+namespace ops {
+namespace custom {
+namespace while_kernel {
+
+namespace {
+
+TfLiteStatus ResizeSubgraphInputs(TfLiteContext* context, TfLiteNode* node,
+                                  Subgraph* subgraph) {
+  int num_inputs = node->inputs->size;
+  for (int i = 0; i < num_inputs; ++i) {
+    const TfLiteTensor* input = GetInput(context, node, i);
+    std::vector<int> dims(input->dims->data,
+                          input->dims->data + input->dims->size);
+    subgraph->ResizeInputTensor(i, dims);
+    TfLiteTensor* subgraph_input = subgraph->tensor(subgraph->inputs()[i]);
+    TF_LITE_ENSURE_EQ(context, input->type, subgraph_input->type);
+  }
+  return kTfLiteOk;
+}
+
+template <typename SrcVector, typename DstVector>
+TfLiteStatus CopyTensors(TfLiteContext* context, Subgraph* src_subgraph,
+                         const SrcVector& src_tensor_indices,
+                         Subgraph* dst_subgraph,
+                         const DstVector& dst_tensor_indices) {
+  TF_LITE_ENSURE_EQ(context, src_tensor_indices.size(),
+                    dst_tensor_indices.size());
+  for (int i = 0; i < src_tensor_indices.size(); ++i) {
+    const TfLiteTensor* src_tensor =
+        src_subgraph->tensor(src_tensor_indices[i]);
+    TfLiteTensor* dst_tensor = dst_subgraph->tensor(dst_tensor_indices[i]);
+    TF_LITE_ENSURE_EQ(context, src_tensor->bytes, dst_tensor->bytes);
+    memcpy(dst_tensor->data.raw, src_tensor->data.raw, src_tensor->bytes);
+  }
+  return kTfLiteOk;
+}
+
+}  // namespace
+
+struct OpData {
+  int cond_subgraph_index;
+  int body_subgraph_index;
+};
+
+void* Init(TfLiteContext* context, const char* buffer, size_t length) {
+  auto* op_data = new OpData;
+  const uint8_t* buffer_t = reinterpret_cast<const uint8_t*>(buffer);
+  const flexbuffers::Map& m = flexbuffers::GetRoot(buffer_t, length).AsMap();
+  op_data->cond_subgraph_index = m["cond_subgraph_index"].AsInt32();
+  op_data->body_subgraph_index = m["body_subgraph_index"].AsInt32();
+  return op_data;
+}
+
+void Free(TfLiteContext* context, void* buffer) {
+  delete reinterpret_cast<OpData*>(buffer);
+}
+
+TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
+  const OpData* op_data = reinterpret_cast<OpData*>(node->user_data);
+  int num_inputs = node->inputs->size;
+  // The number of outputs should be the same as number of inputs.
+  TF_LITE_ENSURE_EQ(context, node->outputs->size, num_inputs);
+
+  // Check subgraph indices and get subgraphs.
+  Subgraph* this_subgraph = reinterpret_cast<Subgraph*>(context->impl_);
+  auto* subgraphs = this_subgraph->GetSubgraphs();
+  TF_LITE_ENSURE(context, op_data->cond_subgraph_index < subgraphs->size());
+  TF_LITE_ENSURE(context, op_data->body_subgraph_index < subgraphs->size());
+
+  Subgraph* cond_subgraph = (*subgraphs)[op_data->cond_subgraph_index].get();
+  Subgraph* body_subgraph = (*subgraphs)[op_data->body_subgraph_index].get();
+
+  // Check input & output count of the condition subgraph.
+  TF_LITE_ENSURE_EQ(context, cond_subgraph->inputs().size(), num_inputs);
+  TF_LITE_ENSURE_EQ(context, cond_subgraph->outputs().size(), 1);
+
+  // Check input & output count of the body subgraph.
+  TF_LITE_ENSURE_EQ(context, body_subgraph->inputs().size(), num_inputs);
+  TF_LITE_ENSURE_EQ(context, body_subgraph->outputs().size(), num_inputs);
+
+  // Prepare and check the condition subgraph.
+  ResizeSubgraphInputs(context, node, cond_subgraph);
+  TF_LITE_ENSURE_OK(context, cond_subgraph->AllocateTensors());
+  TfLiteTensor* cond_output =
+      cond_subgraph->tensor(cond_subgraph->outputs()[0]);
+  // The condition output must be a single boolean value.
+  TF_LITE_ENSURE_EQ(context, cond_output->type, kTfLiteBool);
+  TF_LITE_ENSURE_EQ(context, cond_output->dims->size, 1);
+  TF_LITE_ENSURE_EQ(context, cond_output->dims->data[0], 1);
+
+  // TODO(ycling): Handle the case where condition graph has dynamic
+  // sized tensors.
+
+  // Prepare and check the body subgraph.
+  ResizeSubgraphInputs(context, node, body_subgraph);
+  TF_LITE_ENSURE_OK(context, body_subgraph->AllocateTensors());
+  for (int i = 0; i < num_inputs; ++i) {
+    TfLiteTensor* body_input =
+        body_subgraph->tensor(body_subgraph->inputs()[i]);
+    TfLiteTensor* body_output =
+        body_subgraph->tensor(body_subgraph->outputs()[i]);
+    TF_LITE_ENSURE_EQ(context, body_input->type, body_output->type);
+
+    // TODO(ycling): Support dynamic sized body subgraph.
+    TF_LITE_ENSURE(context, !IsDynamicTensor(body_output));
+    TF_LITE_ENSURE(context,
+                   TfLiteIntArrayEqual(body_input->dims, body_output->dims));
+
+    TfLiteTensor* output = GetOutput(context, node, i);
+    TfLiteIntArray* output_size = TfLiteIntArrayCopy(body_output->dims);
+    TF_LITE_ENSURE_OK(context,
+                      context->ResizeTensor(context, output, output_size));
+  }
+  return kTfLiteOk;
+}
+
+TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
+  const OpData* op_data = reinterpret_cast<OpData*>(node->user_data);
+  Subgraph* this_subgraph = reinterpret_cast<Subgraph*>(context->impl_);
+  auto* subgraphs = this_subgraph->GetSubgraphs();
+  Subgraph* cond_subgraph = (*subgraphs)[op_data->cond_subgraph_index].get();
+  Subgraph* body_subgraph = (*subgraphs)[op_data->body_subgraph_index].get();
+
+  // Currently we copy the input / output between the subgraphs. This isn't
+  // optimized yet.
+  // TODO(b/120234921): Optimize and avoid copying tensors between subgraphs.
+  TF_LITE_ENSURE_OK(
+      context,
+      CopyTensors(context, this_subgraph, TfLiteIntArrayView(node->inputs),
+                  cond_subgraph, cond_subgraph->inputs()));
+  TF_LITE_ENSURE_OK(
+      context,
+      CopyTensors(context, this_subgraph, TfLiteIntArrayView(node->inputs),
+                  body_subgraph, body_subgraph->inputs()));
+
+  while (true) {
+    TF_LITE_ENSURE_OK(context, cond_subgraph->Invoke());
+    TfLiteTensor* cond_output =
+        cond_subgraph->tensor(cond_subgraph->outputs()[0]);
+    if (!cond_output->data.b[0]) {
+      break;
+    }
+    TF_LITE_ENSURE_OK(context, body_subgraph->Invoke());
+
+    TF_LITE_ENSURE_OK(
+        context, CopyTensors(context, body_subgraph, body_subgraph->outputs(),
+                             body_subgraph, body_subgraph->inputs()));
+    TF_LITE_ENSURE_OK(
+        context, CopyTensors(context, body_subgraph, body_subgraph->outputs(),
+                             cond_subgraph, cond_subgraph->inputs()));
+  }
+
+  // Note that copying from body's output will fail if body is never invoked.
+  // TODO(b/120234921): Optimize and avoid copying tensors between subgraphs.
+  TF_LITE_ENSURE_OK(
+      context, CopyTensors(context, body_subgraph, body_subgraph->inputs(),
+                           this_subgraph, TfLiteIntArrayView(node->outputs)));
+  return kTfLiteOk;
+}
+
+}  // namespace while_kernel
+
+TfLiteRegistration* Register_WHILE() {
+  static TfLiteRegistration r = {while_kernel::Init, while_kernel::Free,
+                                 while_kernel::Prepare, while_kernel::Eval};
+  return &r;
+}
+
+}  // namespace custom
+}  // namespace ops
+}  // namespace tflite
diff --git a/tensorflow/lite/kernels/while_test.cc b/tensorflow/lite/kernels/while_test.cc
new file mode 100644
index 0000000000..970ec7033b
--- /dev/null
+++ b/tensorflow/lite/kernels/while_test.cc
@@ -0,0 +1,85 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <gtest/gtest.h>
+#include "flatbuffers/flexbuffers.h"  // TF:flatbuffers
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/kernels/kernel_util.h"
+#include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/kernels/subgraph_test_util.h"
+#include "tensorflow/lite/kernels/test_util.h"
+#include "tensorflow/lite/model.h"
+
+namespace tflite {
+
+using subgraph_test_util::CheckIntTensor;
+using subgraph_test_util::ControlFlowOpTest;
+using subgraph_test_util::FillIntTensor;
+
+namespace {
+
+class WhileTest : public ControlFlowOpTest {};
+
+// The test builds a model that produces the i-th number of
+// triangular number sequence.
+//
+// TODO(ycling): Consider to improve this test case by adding a
+// concat into the body subgraph.
+TEST_F(WhileTest, TestTriangularNumberSequence) {
+  const std::vector<int> expected = {1, 3, 6, 10, 15, 21, 28};
+  for (int i = 0; i < expected.size(); ++i) {
+    interpreter_.reset(new Interpreter);
+    interpreter_->AddSubgraphs(2);
+    builder_->BuildLessEqualCondSubgraph(interpreter_->subgraph(1), i);
+    builder_->BuildAccumulateLoopBodySubgraph(interpreter_->subgraph(2));
+    builder_->BuildWhileSubgraph(&interpreter_->primary_subgraph());
+
+    interpreter_->ResizeInputTensor(interpreter_->inputs()[0], {1});
+    interpreter_->ResizeInputTensor(interpreter_->inputs()[1], {1});
+    ASSERT_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
+    FillIntTensor(interpreter_->tensor(interpreter_->inputs()[0]), {1});
+    FillIntTensor(interpreter_->tensor(interpreter_->inputs()[1]), {1});
+
+    ASSERT_EQ(interpreter_->Invoke(), kTfLiteOk);
+    TfLiteTensor* output1 = interpreter_->tensor(interpreter_->outputs()[0]);
+    CheckIntTensor(output1, {1}, {i + 1});
+    TfLiteTensor* output2 = interpreter_->tensor(interpreter_->outputs()[1]);
+    CheckIntTensor(output2, {1}, {expected[i]});
+  }
+}
+
+// This requires dynamic sized subgraphs and it's not supported right now.
+// TODO(ycling): Support dynamic sized subgraphs.
+TEST_F(WhileTest, TestPadLoop) {
+  interpreter_.reset(new Interpreter);
+  interpreter_->AddSubgraphs(2);
+  builder_->BuildLessEqualCondSubgraph(interpreter_->subgraph(1), 3);
+  builder_->BuildPadLoopBodySubgraph(interpreter_->subgraph(2), {1, 2});
+  builder_->BuildWhileSubgraph(&interpreter_->primary_subgraph());
+
+  interpreter_->ResizeInputTensor(interpreter_->inputs()[0], {1});
+  interpreter_->ResizeInputTensor(interpreter_->inputs()[1], {2});
+  // This is not supported yet. The test ensures thatit doesn't crash and raises
+  // an error properly.
+  ASSERT_NE(interpreter_->AllocateTensors(), kTfLiteOk);
+}
+
+}  // namespace
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
-- 
GitLab


From 0598c2111a760cc1312b68bcd322b5953f9f44b9 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Fri, 8 Feb 2019 15:35:57 -0800
Subject: [PATCH 2211/2345] fix the for statement in creating hypers.

PiperOrigin-RevId: 233138945
---
 tensorflow/python/keras/optimizer_v2/optimizer_v2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
index 5a1b5f2505..41d813e087 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
@@ -559,7 +559,7 @@ class OptimizerV2(checkpointable.Checkpointable):
         self._weights.append(self._iterations)
     for name, value in self._hyper.items():
       if isinstance(value, ops.Tensor) or callable(value):
-        pass
+        continue
       else:
         self._hyper[name] = self.add_weight(
             name,
-- 
GitLab


From 90f2aa53455a85e8e37cb4ef676a8e0b1b47eec4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 8 Feb 2019 15:52:13 -0800
Subject: [PATCH 2212/2345] Change show_side_navs: true|false to nav:
 both|none.

PiperOrigin-RevId: 233141605
---
 tensorflow/lite/g3doc/models/_index.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/g3doc/models/_index.yaml b/tensorflow/lite/g3doc/models/_index.yaml
index f4d8bc40a9..1c6f41bdea 100644
--- a/tensorflow/lite/g3doc/models/_index.yaml
+++ b/tensorflow/lite/g3doc/models/_index.yaml
@@ -4,7 +4,7 @@ description: <!--no description-->
 landing_page:
   body_class: tfo-hide-page-nav
   custom_css_path: /site-assets/css/style.css
-  show_side_navs: true
+  nav: both
   rows:
 
   # Hero
-- 
GitLab


From 1798f30745c1c1cb88eb23e0332686a09aac772d Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Fri, 8 Feb 2019 15:53:26 -0800
Subject: [PATCH 2213/2345] Update python example for keras backend dtype

PiperOrigin-RevId: 233141796
---
 tensorflow/python/keras/backend.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index 89aa2c04ce..9e85e27d48 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -956,10 +956,10 @@ def dtype(x):
       # Keras variable
       >>> kvar = K.variable(np.array([[1, 2], [3, 4]]))
       >>> K.dtype(kvar)
-      'float32_ref'
+      'float32'
       >>> kvar = K.variable(np.array([[1, 2], [3, 4]]), dtype='float32')
       >>> K.dtype(kvar)
-      'float32_ref'
+      'float32'
   ```
   """
   return x.dtype.base_dtype.name
-- 
GitLab


From ce5a8d8ff726e210ecb0d0402eb4f1de43b04346 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Fri, 8 Feb 2019 16:01:19 -0800
Subject: [PATCH 2214/2345] Allow users to interact with resource variables in
 imported 1.x SavedModels

Modifies wrap_function to search for unlifted variables and lift them out as captures. Adds a test for training an imported model.

PiperOrigin-RevId: 233142944
---
 tensorflow/python/eager/wrap_function.py      | 52 +++++++++++++++++++
 .../python/saved_model/load_v1_in_v2.py       |  2 +-
 .../python/saved_model/load_v1_in_v2_test.py  | 15 ++++++
 3 files changed, 68 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/eager/wrap_function.py b/tensorflow/python/eager/wrap_function.py
index f8fbda861c..de384bd456 100644
--- a/tensorflow/python/eager/wrap_function.py
+++ b/tensorflow/python/eager/wrap_function.py
@@ -19,12 +19,16 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import weakref
+
+from tensorflow.python.eager import def_function
 from tensorflow.python.eager import function
 from tensorflow.python.eager import lift_to_graph
 from tensorflow.python.framework import func_graph
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.util import nest
 from tensorflow.python.util.tf_export import tf_export
@@ -67,6 +71,54 @@ class WrappedFunction(function.ConcreteFunction):
     super(WrappedFunction, self).__init__(
         fn_graph, attrs=attrs, signature=signature)
     self._variable_holder = variable_holder
+    if ops.executing_eagerly_outside_functions():
+      # TODO(allenl): Make this work in 1.x?
+      self._lift_unlifted_variables()
+
+  def _lift_unlifted_variables(self):
+    """Finds resource variables and lifts them into the outer context.
+
+    When we import a GraphDef inside a wrap_function, no Python graph building
+    code runs. This means we get VarHandleOps which create variable resources,
+    but no corresponding Python objects. Leaving them like this works but gives
+    the user no way to interact with or modify the variables outside the graph.
+
+    This method searches for variables and lifts them out as regular variable
+    objects when possible, indicating to the FuncGraph that they are captures.
+    """
+    with self.graph.as_default():
+      collection_variables = (
+          ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)
+          + ops.get_collection(ops.GraphKeys.LOCAL_VARIABLES))
+      existing_captures = set(self.graph.internal_captures)
+      lifted_variables = {}
+      for old_variable in collection_variables:
+        if (old_variable._in_graph_mode  # pylint: disable=protected-access
+            and isinstance(old_variable,
+                           resource_variable_ops.ResourceVariable)):
+          if old_variable.handle in existing_captures:
+            continue
+          new_variable = def_function.UnliftedInitializerVariable(
+              array_ops.placeholder(
+                  name="unused_{}_initializer".format(old_variable.op.name),
+                  shape=old_variable.shape,
+                  dtype=old_variable.dtype),
+              name=old_variable.op.name,
+              trainable=old_variable.trainable)
+          self.graph.captures[new_variable.handle] = old_variable.handle
+          existing_captures.add(old_variable.handle)
+          lifted_variables[old_variable] = new_variable
+          # pylint: disable=protected-access
+          self._variable_holder._variables.append(new_variable)
+          self.graph._weak_variables.append(weakref.ref(new_variable))
+          # pylint: enable=protected-access
+      # Update the graph's collections, partly for the user and partly so this
+      # function is idempotent when it runs again in prune() calls.
+      for collection_name in [ops.GraphKeys.GLOBAL_VARIABLES,
+                              ops.GraphKeys.LOCAL_VARIABLES]:
+        mutable_collection = ops.get_collection_ref(collection_name)
+        for index, current in enumerate(mutable_collection):
+          mutable_collection[index] = lifted_variables.get(current, current)
 
   def prune(self, feeds, fetches):
     flat_feeds, flat_fetches = nest.flatten(feeds), nest.flatten(fetches)
diff --git a/tensorflow/python/saved_model/load_v1_in_v2.py b/tensorflow/python/saved_model/load_v1_in_v2.py
index c7c8d8d804..5980b84878 100644
--- a/tensorflow/python/saved_model/load_v1_in_v2.py
+++ b/tensorflow/python/saved_model/load_v1_in_v2.py
@@ -108,7 +108,7 @@ class _EagerSavedModelLoader(loader_impl.SavedModelLoader):
     root = tracking.AutoCheckpointable()
     root.signatures = signature_serialization.create_signature_map(
         signature_functions)
-    # TODO(allenl): Lift out variables and make a ".variables" property
+    root.variables = list(wrapped.graph.variables)
     return root
 
 
diff --git a/tensorflow/python/saved_model/load_v1_in_v2_test.py b/tensorflow/python/saved_model/load_v1_in_v2_test.py
index 42e13e782f..2bb6b93b3b 100644
--- a/tensorflow/python/saved_model/load_v1_in_v2_test.py
+++ b/tensorflow/python/saved_model/load_v1_in_v2_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import os
 
 from tensorflow.python.client import session as session_lib
+from tensorflow.python.eager import backprop
 from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -49,6 +50,7 @@ class LoadTest(test.TestCase):
       local_variable = variables.VariableV1(
           1.,
           collections=[ops.GraphKeys.LOCAL_VARIABLES],
+          trainable=False,
           use_resource=True)
       output = array_ops.identity(start * v * local_variable, name="output")
       with session_lib.Session() as session:
@@ -70,6 +72,19 @@ class LoadTest(test.TestCase):
       fn(constant_op.constant(2.))
     self.assertEqual({"output": 6.},
                      self.evaluate(fn(start=constant_op.constant(2.))))
+    self.assertAllEqual([3., 1.], self.evaluate(imported.variables))
+    imported.variables[0].assign(4.)
+    self.assertEqual({"output": 8.},
+                     self.evaluate(fn(start=constant_op.constant(2.))))
+    imported.variables[1].assign(2.)
+    self.assertEqual({"output": 24.},
+                     self.evaluate(fn(start=constant_op.constant(3.))))
+    self.assertTrue(imported.variables[0].trainable)
+    self.assertFalse(imported.variables[1].trainable)
+    with backprop.GradientTape() as tape:
+      output = fn(start=constant_op.constant(4.))
+    self.assertEqual(imported.variables[:1], list(tape.watched_variables()))
+    self.assertEqual(8., tape.gradient(output, imported.variables[0]).numpy())
 
   def test_ref_variable_import(self):
     with self.assertRaises(NotImplementedError):
-- 
GitLab


From c7621b86bc425245eb01557faded795cebc649ee Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Fri, 8 Feb 2019 16:24:59 -0800
Subject: [PATCH 2215/2345] Expose CriticalSection in core as
 tf.CriticalSection.

PiperOrigin-RevId: 233146763
---
 tensorflow/contrib/framework/BUILD            | 22 ------
 tensorflow/contrib/framework/__init__.py      |  2 -
 .../contrib/framework/python/ops/__init__.py  |  1 -
 tensorflow/python/BUILD                       | 17 +++++
 tensorflow/python/kernel_tests/BUILD          | 21 ++++++
 .../kernel_tests}/critical_section_test.py    | 67 ++++++++++++++-----
 .../python/ops/critical_section_ops.py        | 48 +++++++------
 tensorflow/python/ops/standard_ops.py         |  1 +
 .../v1/tensorflow.-critical-section.pbtxt     | 17 +++++
 .../tools/api/golden/v1/tensorflow.pbtxt      |  4 ++
 .../v2/tensorflow.-critical-section.pbtxt     | 17 +++++
 .../tools/api/golden/v2/tensorflow.pbtxt      |  4 ++
 .../tools/compatibility/tf_upgrade_v2.py      |  2 +
 .../tools/compatibility/tf_upgrade_v2_test.py |  6 ++
 14 files changed, 164 insertions(+), 65 deletions(-)
 rename tensorflow/{contrib/framework/python/ops => python/kernel_tests}/critical_section_test.py (87%)
 rename tensorflow/{contrib/framework => }/python/ops/critical_section_ops.py (92%)
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.-critical-section.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-critical-section.pbtxt

diff --git a/tensorflow/contrib/framework/BUILD b/tensorflow/contrib/framework/BUILD
index c99f84789e..8fd2b5f39b 100644
--- a/tensorflow/contrib/framework/BUILD
+++ b/tensorflow/contrib/framework/BUILD
@@ -32,7 +32,6 @@ tf_custom_op_py_library(
         "python/ops/arg_scope.py",
         "python/ops/audio_ops.py",
         "python/ops/checkpoint_ops.py",
-        "python/ops/critical_section_ops.py",
         "python/ops/ops.py",
         "python/ops/prettyprint_ops.py",
         "python/ops/script_ops.py",
@@ -172,27 +171,6 @@ py_test(
     ],
 )
 
-cuda_py_test(
-    name = "critical_section_test",
-    size = "medium",
-    srcs = ["python/ops/critical_section_test.py"],
-    additional_deps = [
-        "//tensorflow/python:client_testlib",
-        ":framework_py",
-        "//tensorflow/python/data/experimental/ops:prefetching_ops",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:control_flow_ops",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:gradients",
-        "//tensorflow/python:platform_test",
-        "//tensorflow/python:resource_variable_ops",
-        "//tensorflow/python:tensor_array_ops",
-        "//tensorflow/python/data/ops:dataset_ops",
-        "//tensorflow/python/eager:context",
-    ],
-)
-
 py_test(
     name = "ops_test",
     size = "small",
diff --git a/tensorflow/contrib/framework/__init__.py b/tensorflow/contrib/framework/__init__.py
index fc2334d5d7..94fb35b334 100644
--- a/tensorflow/contrib/framework/__init__.py
+++ b/tensorflow/contrib/framework/__init__.py
@@ -94,8 +94,6 @@
 @@smart_constant_value
 @@smart_case
 
-@@CriticalSection
-
 @@BoundedTensorSpec
 @@TensorSpec
 
diff --git a/tensorflow/contrib/framework/python/ops/__init__.py b/tensorflow/contrib/framework/python/ops/__init__.py
index c4976497f5..8113bf7c09 100644
--- a/tensorflow/contrib/framework/python/ops/__init__.py
+++ b/tensorflow/contrib/framework/python/ops/__init__.py
@@ -22,7 +22,6 @@ from __future__ import print_function
 # pylint: disable=wildcard-import
 from tensorflow.contrib.framework.python.ops.arg_scope import *
 from tensorflow.contrib.framework.python.ops.checkpoint_ops import *
-from tensorflow.contrib.framework.python.ops.critical_section_ops import *
 from tensorflow.contrib.framework.python.ops.ops import *
 from tensorflow.contrib.framework.python.ops.prettyprint_ops import *
 from tensorflow.contrib.framework.python.ops.script_ops import *
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index ca6a09cdd4..d3254cc2e4 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -2737,6 +2737,22 @@ py_library(
     ],
 )
 
+py_library(
+    name = "critical_section_ops",
+    srcs = ["ops/critical_section_ops.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":array_ops",
+        ":control_flow_ops",
+        ":dtypes",
+        ":framework_ops",
+        ":resource_variable_ops_gen",
+        ":tensor_array_ops",
+        ":util",
+        "//tensorflow/python/eager:context",
+    ],
+)
+
 py_library(
     name = "list_ops",
     srcs = ["ops/list_ops.py"],
@@ -3148,6 +3164,7 @@ py_library(
         ":clip_ops",
         ":confusion_matrix",
         ":control_flow_ops",
+        ":critical_section_ops",
         ":cudnn_rnn_grad",
         ":data_flow_grad",
         ":data_flow_ops",
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index c660633a76..d98a6dd59c 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -3592,3 +3592,24 @@ cuda_py_test(
     grpc_enabled = True,
     xla_enable_strict_auto_jit = True,
 )
+
+cuda_py_test(
+    name = "critical_section_test",
+    size = "medium",
+    srcs = ["critical_section_test.py"],
+    additional_deps = [
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python/data/experimental/ops:prefetching_ops",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:control_flow_ops",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:gradients",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:resource_variable_ops",
+        "//tensorflow/python:critical_section_ops",
+        "//tensorflow/python:tensor_array_ops",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/eager:context",
+    ],
+)
diff --git a/tensorflow/contrib/framework/python/ops/critical_section_test.py b/tensorflow/python/kernel_tests/critical_section_test.py
similarity index 87%
rename from tensorflow/contrib/framework/python/ops/critical_section_test.py
rename to tensorflow/python/kernel_tests/critical_section_test.py
index d2bb4f476f..7b1519c5e3 100644
--- a/tensorflow/contrib/framework/python/ops/critical_section_test.py
+++ b/tensorflow/python/kernel_tests/critical_section_test.py
@@ -18,14 +18,15 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.framework.python.ops import critical_section_ops
 from tensorflow.python.data.experimental.ops import prefetching_ops
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.eager import context
+from tensorflow.python.eager import def_function
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import critical_section_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging as logging
@@ -48,7 +49,7 @@ class CriticalSectionTest(test.TestCase):
           return array_ops.identity(c)
 
     num_concurrent = 100
-    r = [cs.execute(fn, 1.0, 2.0) for _ in range(num_concurrent)]
+    r = [cs.execute(lambda: fn(1.0, 2.0)) for _ in range(num_concurrent)]
     self.evaluate(v.initializer)
     r_value = self.evaluate(r)
     self.assertAllClose([2.0 * i for i in range(num_concurrent)],
@@ -74,7 +75,7 @@ class CriticalSectionTest(test.TestCase):
               array_ops.identity(inner_cond), true_fn, lambda: c)
 
         def execute():
-          return cs.execute(fn, 1.0, 2.0)
+          return cs.execute(lambda: fn(1.0, 2.0))
 
         r = [
             control_flow_ops.cond(array_ops.identity(outer_cond),
@@ -92,6 +93,7 @@ class CriticalSectionTest(test.TestCase):
         else:
           self.assertAllClose([0] * num_concurrent, r_value)
 
+  @test_util.run_v1_only("b/123990562 Sees CancelledError on some calls")
   def testCriticalSectionInParallelDoesntDeadlockOnError(self):
     # No eager mode execution of this test because eager does not
     # run fn() in parallel, which is where the deadlock could
@@ -103,12 +105,23 @@ class CriticalSectionTest(test.TestCase):
       error = control_flow_ops.Assert((i % 2) == 1, ["Error"])
       with ops.control_dependencies([error]):
         return v.read_value()
+
     num_concurrent = 2
-    r = [cs.execute(fn, i) for i in range(num_concurrent)]
+
+    @def_function.function(autograph=False)
+    def run_concurrently():
+      return [cs.execute(lambda: fn(i)) for i in range(num_concurrent)]
+
+    if not context.executing_eagerly():
+      run_concurrently = run_concurrently()
+
     self.evaluate(v.initializer)
     for _ in range(100):
       with self.assertRaisesOpError("Error"):
-        self.evaluate(r)
+        if context.executing_eagerly():
+          run_concurrently()
+        else:
+          self.evaluate(run_concurrently)
 
   @test_util.run_in_graph_and_eager_modes
   def testCreateCriticalSectionFnReturnsOp(self):
@@ -123,17 +136,20 @@ class CriticalSectionTest(test.TestCase):
           return control_flow_ops.no_op()
 
     num_concurrent = 100
-    r = [cs.execute(fn_return_op, 1.0, 2.0) for _ in range(num_concurrent)]
+    r = [cs.execute(lambda: fn_return_op(1.0, 2.0))
+         for _ in range(num_concurrent)]
     self.evaluate(v.initializer)
     self.evaluate(r)
     final_v = self.evaluate(v)
     self.assertAllClose(2.0 * num_concurrent, final_v)
 
+  @test_util.run_v1_only("Collections don't exist in TF2")
   def testCollection(self):
     cs = critical_section_ops.CriticalSection(shared_name="cs")
     self.assertIn(
         cs, ops.get_collection(critical_section_ops.CRITICAL_SECTIONS))
-    execute = cs.execute(lambda x: x + 1, 1.0, name="my_execute")
+    add = lambda x: x + 1
+    execute = cs.execute(lambda: add(1.0), name="my_execute")
     execute_op = [
         x for x in execute.graph.get_operations()
         if "my_execute" in x.name and "MutexLock" in x.type
@@ -143,18 +159,21 @@ class CriticalSectionTest(test.TestCase):
         [signature.op for signature in
          ops.get_collection(critical_section_ops.CRITICAL_SECTION_EXECUTIONS)])
 
+  @test_util.run_v1_only("b/123955885 Can't identify deadlocks in eager mode")
   def testRecursiveCriticalSectionAccessIsIllegal(self):
     # This does not work properly in eager mode.  Eager users will
     # just hit a deadlock if they do this.  But at least it'll be easier
     # to debug.
     cs = critical_section_ops.CriticalSection()
+    add = lambda y: y + 1
     def fn(x):
-      return cs.execute(lambda y: y + 1, x)
+      return cs.execute(lambda: add(x))
+
     with self.assertRaisesRegexp(
         ValueError,
         r"attempts to directly access the CriticalSection in which it "
         r"would be running"):
-      cs.execute(fn, 1.0)
+      cs.execute(lambda: fn(1.0))
 
   def testRecursiveCriticalSectionAccessViaCapturedTensorIsProtected(self):
     # This one is subtle; and we're being overly cautious here.  The
@@ -174,24 +193,24 @@ class CriticalSectionTest(test.TestCase):
     # operations are finished before anything runs within the critical section.
     cs = critical_section_ops.CriticalSection(shared_name="cs")
     fn = array_ops.identity
-    to_capture = cs.execute(fn, 1.0)
+    to_capture = cs.execute(lambda: fn(1.0))
     fn_captures = lambda x: x + to_capture
     to_capture_too = array_ops.identity(to_capture)
 
-    ex_0 = cs.execute(fn_captures, 1.0)
+    ex_0 = cs.execute(lambda: fn_captures(1.0))
 
     with ops.control_dependencies([to_capture]):
       # This is OK because to_capture will execute before this next call
-      ex_1 = cs.execute(fn_captures, 1.0)
+      ex_1 = cs.execute(lambda: fn_captures(1.0))
 
     dependency = array_ops.identity(to_capture)
 
     fn_captures_dependency = lambda x: x + dependency
 
-    ex_2 = cs.execute(fn_captures_dependency, 1.0)
+    ex_2 = cs.execute(lambda: fn_captures_dependency(1.0))
 
     with ops.control_dependencies([to_capture_too]):
-      ex_3 = cs.execute(fn_captures_dependency, 1.0)
+      ex_3 = cs.execute(lambda: fn_captures_dependency(1.0))
 
     # Ensure there's no actual deadlock on to_execute.
     self.assertEquals(2.0, self.evaluate(ex_0))
@@ -217,6 +236,8 @@ class CriticalSectionTest(test.TestCase):
         body_implicit_capture,
         [0, 0],
         parallel_iterations=25)
+    # For consistency between eager and graph mode.
+    i_n = array_ops.identity(i_n)
     logging.warn(
         "\n==============\nRunning "
         "'testRecursiveCriticalSectionAccessWithinLoopDoesNotDeadlock "
@@ -242,6 +263,8 @@ class CriticalSectionTest(test.TestCase):
         body_implicit_capture_protected,
         [0, 0],
         parallel_iterations=25)
+    # For consistency between eager and graph mode.
+    i_n = array_ops.identity(i_n)
     logging.warn(
         "\n==============\nRunning "
         "'testRecursiveCriticalSectionAccessWithinLoopDoesNotDeadlock "
@@ -258,13 +281,15 @@ class CriticalSectionTest(test.TestCase):
       # This version is ok because j is an argument to fn and we can
       # ensure there's a control dependency on j.
       fn = lambda x: x + 1
-      return (i + 1, cs.execute(fn, j))
+      return (i + 1, cs.execute(lambda: fn(j)))
 
     (i_n, j_n) = control_flow_ops.while_loop(
         lambda i, _: i < 1000,
         body_args_capture,
         [0, 0],
         parallel_iterations=25)
+    # For consistency between eager and graph mode.
+    i_n = array_ops.identity(i_n)
     logging.warn(
         "\n==============\nRunning "
         "'testRecursiveCriticalSectionAccessWithinLoopDoesNotDeadlock "
@@ -277,20 +302,23 @@ class CriticalSectionTest(test.TestCase):
         "body_args_capture'\n"
         "==============\n")
 
+  @test_util.run_v1_only("b/123955885 Can't identify deadlocks in eager mode")
   def testRecursiveCriticalSectionAccessIsIllegalSameSharedName(self):
     # This does not work properly in eager mode.  Eager users will
     # just hit a deadlock if they do this.  But at least it'll be easier
     # to debug.
     cs = critical_section_ops.CriticalSection(shared_name="cs")
     cs_same = critical_section_ops.CriticalSection(shared_name="cs")
+    add = lambda x: x + 1
     def fn(x):
-      return cs_same.execute(lambda x: x+1, x)
+      return cs_same.execute(lambda: add(x))
     with self.assertRaisesRegexp(
         ValueError,
         r"attempts to directly access the CriticalSection in which it "
         r"would be running"):
-      cs.execute(fn, 1.0)
+      cs.execute(lambda: fn(1.0))
 
+  @test_util.run_v1_only("b/123955885 Can't identify deadlocks in eager mode")
   def testMultipleCSExecutionsRequestSameResource(self):
     cs0 = critical_section_ops.CriticalSection()
     cs1 = critical_section_ops.CriticalSection()
@@ -328,8 +356,11 @@ class CriticalSectionTest(test.TestCase):
     # Note, here v must be a resource variable (or something similar),
     # otherwise it gets hoisted into the while_loop by the time we add
     # control dependencies to the lock_op.
+    def body(i):
+      add_j = lambda j: v + j + 1
+      return cs.execute(lambda: add_j(i))
     out = control_flow_ops.while_loop(
-        lambda i: i < 10, lambda i: cs.execute(lambda j: v + j + 1, i), [0])
+        lambda i: i < 10, body, [0])
     self.evaluate(v.initializer)
     self.assertEqual(10, self.evaluate(out))
 
diff --git a/tensorflow/contrib/framework/python/ops/critical_section_ops.py b/tensorflow/python/ops/critical_section_ops.py
similarity index 92%
rename from tensorflow/contrib/framework/python/ops/critical_section_ops.py
rename to tensorflow/python/ops/critical_section_ops.py
index 71ab755aa2..21872ffff1 100644
--- a/tensorflow/contrib/framework/python/ops/critical_section_ops.py
+++ b/tensorflow/python/ops/critical_section_ops.py
@@ -31,6 +31,10 @@ from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gen_resource_variable_ops
 from tensorflow.python.ops import tensor_array_ops
 from tensorflow.python.util import nest
+from tensorflow.python.util.tf_export import tf_export
+
+
+__all__ = ["CriticalSection"]
 
 
 # Graph Keys
@@ -66,6 +70,7 @@ def _get_colocation(op):
     return None
 
 
+@tf_export("CriticalSection")
 class CriticalSection(object):
   """Critical section.
 
@@ -179,37 +184,36 @@ class CriticalSection(object):
   def name(self):
     return self._handle.op.name
 
-  def execute(self, fn, *args, **kwargs):
-    """Execute function `fn(*args, **kwargs)` inside the CriticalSection.
+  def execute(self, fn, exclusive_resource_access=True, name=None):
+    """Execute function `fn()` inside the critical section.
+
+    `fn` should not accept any arguments.  To add extra arguments to when
+    calling `fn` in the critical section, create a lambda:
+
+    ```python
+    critical_section.execute(lambda: fn(*my_args, **my_kwargs))
+    ```
 
     Args:
       fn: The function to execute.  Must return at least one tensor.
-      *args: Additional positional arguments to `fn`.
-      **kwargs: Additional keyword arguments to `fn`.
-        Several keywords are reserved for `execute`.  These are:
-
-        - name; The name to use when creating the execute operation.
-        - exclusive_resource_access; Whether the resources required by
-          `fn` should be exclusive to this `CriticalSection`.  Default: `True`.
-          You may want to set this to `False` if you will be accessing a
-          resource in read-only mode in two different CriticalSections.
+      exclusive_resource_access: Whether the resources required by
+        `fn` should be exclusive to this `CriticalSection`.  Default: `True`.
+        You may want to set this to `False` if you will be accessing a
+        resource in read-only mode in two different CriticalSections.
+      name: The name to use when creating the execute operation.
 
     Returns:
-      The tensors returned from `fn(*args, **kwargs)`.
+      The tensors returned from `fn()`.
 
     Raises:
       ValueError: If `fn` attempts to lock this `CriticalSection` in any nested
         or lazy way that may cause a deadlock.
-      ValueError: If `exclusive_resource_access` is not provided (is `True`) and
+      ValueError: If `exclusive_resource_access == True` and
         another `CriticalSection` has an execution requesting the same
-        resources as in `*args`, `**kwargs`, and any additionally captured
-        inputs in `fn`.  Note, even if `exclusive_resource_access` is `True`,
-        if another execution in another `CriticalSection` was created without
-        `exclusive_resource_access=True`, a `ValueError` will be raised.
+        resources as `fn``.  Note, even if `exclusive_resource_access` is
+        `True`, if another execution in another `CriticalSection` was created
+        without `exclusive_resource_access=True`, a `ValueError` will be raised.
     """
-    name = kwargs.pop("name", None)
-    exclusive_resource_access = kwargs.pop("exclusive_resource_access", True)
-
     with ops.name_scope(name, "critical_section_execute", []):
 
       # Ensure that mutex locking only happens *after* all args and
@@ -222,7 +226,7 @@ class CriticalSection(object):
         with ops.get_default_graph()._lock:  # pylint: disable=protected-access
           existing_ops = ops.get_default_graph().get_operations()
           with ops.control_dependencies([lock]):
-            r = fn(*args, **kwargs)
+            r = fn()
           # TODO(ebrevdo): If creating critical sections in a python loop, this
           # makes graph creation time quadratic.  Revisit if this
           # becomes a problem.
@@ -230,7 +234,7 @@ class CriticalSection(object):
                          .difference(existing_ops))
       else:
         with ops.control_dependencies([lock]):
-          r = fn(*args, **kwargs)
+          r = fn()
 
       if not context.executing_eagerly():
         self._add_control_dependencies_to_lock(created_ops, lock.op)
diff --git a/tensorflow/python/ops/standard_ops.py b/tensorflow/python/ops/standard_ops.py
index ba3bd09492..5e217d8ed2 100644
--- a/tensorflow/python/ops/standard_ops.py
+++ b/tensorflow/python/ops/standard_ops.py
@@ -54,6 +54,7 @@ from tensorflow.python.ops.control_flow_ops import tuple  # pylint: disable=rede
 # pylint: enable=redefined-builtin
 from tensorflow.python.eager import wrap_function
 from tensorflow.python.ops.control_flow_ops import while_loop
+from tensorflow.python.ops.critical_section_ops import *
 from tensorflow.python.ops.data_flow_ops import *
 from tensorflow.python.ops.functional_ops import *
 from tensorflow.python.ops.gradients import *
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.-critical-section.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.-critical-section.pbtxt
new file mode 100644
index 0000000000..024a208346
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.-critical-section.pbtxt
@@ -0,0 +1,17 @@
+path: "tensorflow.CriticalSection"
+tf_class {
+  is_instance: "<class \'tensorflow.python.ops.critical_section_ops.CriticalSection\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'shared_name\', \'critical_section_def\', \'import_scope\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "execute"
+    argspec: "args=[\'self\', \'fn\', \'exclusive_resource_access\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
index 64f572022e..cb9d6a907f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
@@ -32,6 +32,10 @@ tf_module {
     name: "ConfigProto"
     mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
   }
+  member {
+    name: "CriticalSection"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "DType"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-critical-section.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-critical-section.pbtxt
new file mode 100644
index 0000000000..024a208346
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.-critical-section.pbtxt
@@ -0,0 +1,17 @@
+path: "tensorflow.CriticalSection"
+tf_class {
+  is_instance: "<class \'tensorflow.python.ops.critical_section_ops.CriticalSection\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'shared_name\', \'critical_section_def\', \'import_scope\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "execute"
+    argspec: "args=[\'self\', \'fn\', \'exclusive_resource_access\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index a75c0c27dc..5db60220c7 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -4,6 +4,10 @@ tf_module {
     name: "AggregationMethod"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "CriticalSection"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "DType"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 7e86d6cfa5..080e9420a1 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -540,6 +540,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.data.experimental.unbatch",
         "tf.contrib.data.unique":
             "tf.data.experimental.unique",
+        "tf.contrib.framework.CriticalSection":
+            "tf.CriticalSection",
         "tf.contrib.framework.is_tensor":
             "tf.is_tensor",
         "tf.contrib.framework.nest.assert_same_structure":
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index bf1e80a681..c78db3fdf6 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -1157,6 +1157,12 @@ def _log_prob(self, x):
     _, _, _, new_text = self._upgrade(text)
     self.assertEqual(expected, new_text)
 
+  def test_CriticalSection_upgrade(self):
+    text = "tf.contrib.framework.CriticalSection(shared_name='blah')"
+    expected = "tf.CriticalSection(shared_name='blah')"
+    _, _, _, new_text = self._upgrade(text)
+    self.assertEqual(expected, new_text)
+
   def test_sample_distorted_bounding_box(self):
     # pylint: disable=line-too-long
     text = "tf.image.sample_distorted_bounding_box(a, b, c, d, e, f, g, h, i, j)"
-- 
GitLab


From 2a7dc4b341600180d57cca1599a161a0d7980f28 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Fri, 8 Feb 2019 16:28:22 -0800
Subject: [PATCH 2216/2345] Internal flag to capture by value in
 eager.function.Function.

This is accomplished by disallowing creation of "read" operations inside
FuncGraphs when capture_by_value is true, executing those operations
outside the graph instead and bringing the resulting value into the
right context.

This can misbehave somewhat, but should be enough to migrate the few
users of tf.data which rely on capturing by value.

PiperOrigin-RevId: 233147220
---
 tensorflow/python/eager/function.py           | 15 +++--
 tensorflow/python/eager/function_test.py      | 28 +++++++++
 tensorflow/python/framework/func_graph.py     | 61 +++++++++++++++++--
 tensorflow/python/framework/op_def_library.py |  4 +-
 .../python/framework/op_def_library_test.py   | 45 +++++++-------
 5 files changed, 119 insertions(+), 34 deletions(-)

diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 7a118368d4..f10363b14e 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -1044,7 +1044,8 @@ class Function(object):
                input_signature=None,
                attributes=None,
                autograph=True,
-               autograph_options=None):
+               autograph_options=None,
+               capture_by_value=None):
     """Initializes a `Function`.
 
     Args:
@@ -1061,6 +1062,9 @@ class Function(object):
       autograph_options: Experimental knobs to control behavior
         `when autograph=True`. See https://www.tensorflow.org/guide/autograph
         for more information.
+      capture_by_value: Experimental. Whether to capture resource variables by
+        value or reference. If None, will inherit from a parent context or
+        default to False.
 
     Raises:
       ValueError: if `input_signature` is not None and the `python_function`'s
@@ -1078,6 +1082,7 @@ class Function(object):
     self._function_cache = collections.OrderedDict()
     self._garbage_collector = _FunctionGarbageCollector(self._function_cache)
     self._function_attributes = attributes or {}
+    self._capture_by_value = capture_by_value
 
     self._lock = threading.Lock()
     # _descriptor_cache is a of instance of a class to an instance-specific
@@ -1340,13 +1345,15 @@ class Function(object):
                 self._input_signature,
                 autograph=self._autograph,
                 autograph_options=self._autograph_options,
-                arg_names=arg_names), self._function_attributes)
+                arg_names=arg_names,
+                capture_by_value=self._capture_by_value),
+            self._function_attributes)
         # pylint: disable=protected-access
         # Tell the ConcreteFunction to clean up its graph once it goes out of
         # scope. ConcreteFunction does not do this in its constructor since it
         # gets used in some places (like Keras) where the FuncGraph lives
         # longer than the ConcreteFunction.
-        graph_function._garbage_collector = _ConcreteFunctionGarbageCollector(
+        graph_function._garbage_collector = ConcreteFunctionGarbageCollector(
             graph_function.graph)
         # pylint: enable=protected-access
         self._function_cache[cache_key] = graph_function
@@ -1869,7 +1876,7 @@ class _FunctionGarbageCollector(object):
       pass
 
 
-class _ConcreteFunctionGarbageCollector(object):
+class ConcreteFunctionGarbageCollector(object):
   """Cleans up reference cycles when a `ConcreteFunction` goes out of scope."""
 
   def __init__(self, func_graph):
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index a790df7d8f..834691eedd 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -482,6 +482,34 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     self.assertAllEqual(5., self.evaluate(concrete()))
     self.assertAllEqual(5., self.evaluate(tensor_init()))
 
+  def testFuncGraphCaptureByValue(self):
+    v = variables.Variable(1.0)
+
+    def trivial_function():
+      return v.read_value()
+
+    graph_function = function.Function(
+        trivial_function, 'test', capture_by_value=True)
+
+    self.assertAllEqual(graph_function(), 1.0)
+    v.assign(2.0)
+    self.assertAllEqual(graph_function(), 1.0)
+
+  def testFuncGraphCaptureByValueNested(self):
+    v = variables.Variable(1.0)
+
+    def trivial_function():
+      return control_flow_ops.cond(
+          array_ops.placeholder_with_default(True, ()),
+          v.read_value, v.read_value)
+
+    graph_function = function.Function(
+        trivial_function, 'test', capture_by_value=True)
+
+    self.assertAllEqual(graph_function(), 1.0)
+    v.assign(2.0)
+    self.assertAllEqual(graph_function(), 1.0)
+
   def testDefunShapeInferenceWithCapturedResourceVariable(self):
     v = resource_variable_ops.ResourceVariable([[1, 2], [3, 4]])
 
diff --git a/tensorflow/python/framework/func_graph.py b/tensorflow/python/framework/func_graph.py
index 60d97d50d7..5896d8370e 100644
--- a/tensorflow/python/framework/func_graph.py
+++ b/tensorflow/python/framework/func_graph.py
@@ -23,6 +23,7 @@ import weakref
 
 from tensorflow.core.framework import attr_value_pb2
 from tensorflow.python.eager import context
+from tensorflow.python.eager import execute
 from tensorflow.python.eager import tape
 from tensorflow.python.eager.graph_only_ops import graph_placeholder
 from tensorflow.python.framework import dtypes
@@ -135,9 +136,11 @@ class FuncGraph(ops.Graph):
     captures: Maps external tensor -> internal tensor (i.e. input placeholder).
       The entries are in the order they were captured.
     seed: The graph-level random seed.
+    capture_by_value: If True, the func graph will capture Variables by value
+      instead of reference.
   """
 
-  def __init__(self, name, collections=None):
+  def __init__(self, name, collections=None, capture_by_value=None):
     """Construct a new FuncGraph.
 
     The graph will inherit its graph key, collections, seed, and distribution
@@ -152,6 +155,9 @@ class FuncGraph(ops.Graph):
         The current whitelisted collections are the global variables, the
         local variables, and the trainable variables.
         Defaults to None.
+      capture_by_value: An optional boolean. If True, the func graph will
+        capture Variables by value instead of reference. By default inherit
+        from outer graphs, and failing that will default to False.
     """
     super(FuncGraph, self).__init__()
 
@@ -163,6 +169,14 @@ class FuncGraph(ops.Graph):
     self._weak_variables = []
     self.outer_graph = ops.get_default_graph()
     self.captures = py_collections.OrderedDict()
+    # Inherit capture-by-value from outer graph.
+    if capture_by_value is not None:
+      self.capture_by_value = capture_by_value
+    elif self.outer_graph is not None and isinstance(
+        self.outer_graph, FuncGraph):
+      self.capture_by_value = self.outer_graph.capture_by_value
+    else:
+      self.capture_by_value = False
 
     self._building_function = True
     # Map from resource tensor name to last op (in program order) which uses
@@ -281,11 +295,39 @@ class FuncGraph(ops.Graph):
   def variables(self, var_list):
     self._weak_variables = [weakref.ref(v) for v in var_list]
 
+  def _capture_by_value(
+      self,
+      op_type,
+      inputs,
+      dtypes,  # pylint: disable=redefined-outer-name
+      input_types=None,
+      name=None,
+      attrs=None,
+      op_def=None,
+      compute_shapes=True,
+      compute_device=True):
+    # When capturing by value, do the read outside
+    reverse_captures = dict((v, k) for k, v in self.captures.items())
+    uncaptured_inputs = [reverse_captures.get(t, t) for t in inputs]
+    with ops.init_scope():
+      if context.executing_eagerly():
+        attr_list = ("dtype", int(attrs["dtype"].type))
+        value, = execute.execute(
+            compat.as_bytes(op_type), 1, uncaptured_inputs, attr_list,
+            context.context())
+      else:
+        op = ops.get_default_graph().create_op(
+            op_type, uncaptured_inputs, dtypes, input_types, name, attrs,
+            op_def, compute_shapes, compute_device)
+        value = op.outputs[0]
+    captured_value = self.capture(value)
+    return captured_value.op
+
   def create_op(
       self,
       op_type,
       inputs,
-      dtypes,
+      dtypes,  # pylint: disable=redefined-outer-name
       input_types=None,
       name=None,
       attrs=None,
@@ -324,6 +366,12 @@ class FuncGraph(ops.Graph):
     Returns:
       An `Operation` object.
     """
+    if self.capture_by_value and op_type in ["ReadVariableOp",
+                                             "ResourceGather"]:
+      return self._capture_by_value(
+          op_type, inputs, dtypes, input_types, name, attrs, op_def,
+          compute_shapes, compute_device)
+
     # This capturing logic interacts poorly with control flow contexts which
     # want to replace inputs of ops far too late in the process. This can lead
     # the context to get confused and try to create an Enter for an Enter. We
@@ -418,7 +466,8 @@ def func_graph_from_py_func(name,
                             add_control_dependencies=True,
                             arg_names=None,
                             op_return_value=None,
-                            collections=None):
+                            collections=None,
+                            capture_by_value=None):
   """Returns a `FuncGraph` generated from `python_func`.
 
   Args:
@@ -454,6 +503,9 @@ def func_graph_from_py_func(name,
       The current whitelisted collections are the global variables, the
       local variables, and the trainable variables.
       Defaults to None.
+    capture_by_value: An optional boolean. If True, the func graph will capture
+      Variables by value instead of reference. By default inherit from outer
+      graphs, and failing that will default to False.
 
   Returns:
     A FuncGraph.
@@ -465,7 +517,8 @@ def func_graph_from_py_func(name,
   if op_return_value is not None:
     assert isinstance(op_return_value, ops.Tensor), op_return_value
   if func_graph is None:
-    func_graph = FuncGraph(name, collections=collections)
+    func_graph = FuncGraph(name, collections=collections,
+                           capture_by_value=capture_by_value)
   assert isinstance(func_graph, FuncGraph)
   if add_control_dependencies:
     control_manager = AutomaticControlDependencies
diff --git a/tensorflow/python/framework/op_def_library.py b/tensorflow/python/framework/op_def_library.py
index b843e1f68b..372763a862 100644
--- a/tensorflow/python/framework/op_def_library.py
+++ b/tensorflow/python/framework/op_def_library.py
@@ -531,9 +531,9 @@ class OpDefLibrary(object):
             else:
               raise TypeError(
                   "Expected %s passed to parameter '%s' of op '%s', got %s of "
-                  "type '%s' instead." %
+                  "type '%s' instead. Error: %s" %
                   (dtypes.as_dtype(dtype).name, input_arg.name, op_type_name,
-                   repr(values), type(values).__name__))
+                   repr(values), type(values).__name__, err))
           except ValueError:
             # What type does convert_to_tensor think it has?
             try:
diff --git a/tensorflow/python/framework/op_def_library_test.py b/tensorflow/python/framework/op_def_library_test.py
index aed056a95c..71d708dd89 100644
--- a/tensorflow/python/framework/op_def_library_test.py
+++ b/tensorflow/python/framework/op_def_library_test.py
@@ -141,40 +141,43 @@ class OpDefLibraryTest(test_util.TensorFlowTestCase):
     with ops.Graph().as_default():
       with self.assertRaises(TypeError) as cm:
         self._lib.apply_op("Simple", a="Bad string")
-      self.assertEqual(str(cm.exception),
-                       "Expected int32 passed to parameter 'a' of op 'Simple', "
-                       "got 'Bad string' of type 'str' instead.")
+      self.assertTrue(
+          "Expected int32 passed to parameter 'a' of op 'Simple', "
+          "got 'Bad string' of type 'str' instead." in str(cm.exception))
 
       with self.assertRaises(TypeError) as cm:
         self._lib.apply_op("Simple", a=self.Tensor(dtypes.string))
-      self.assertEqual(str(cm.exception),
-                       "Input 'a' of 'Simple' Op has type string "
-                       "that does not match expected type of int32.")
+      self.assertTrue(
+          "Input 'a' of 'Simple' Op has type string "
+          "that does not match expected type of int32." in str(cm.exception))
 
       with self.assertRaises(TypeError) as cm:
         self._lib.apply_op("Simple", a=6, extra="bogus")
-      self.assertEqual(str(cm.exception),
-                       "apply_op() got unexpected keyword arguments: extra")
+      self.assertTrue(
+          "apply_op() got unexpected keyword arguments: extra"
+          in str(cm.exception))
 
       with self.assertRaises(TypeError) as cm:
         self._lib.apply_op("Simple", a=6, extra1="bogus", extra2="also_bogus")
-      self.assertEqual(str(cm.exception),
-                       "apply_op() got unexpected keyword arguments: extra1, "
-                       "extra2")
+      self.assertTrue(
+          "apply_op() got unexpected keyword arguments: extra1, "
+          "extra2" in str(cm.exception))
 
       with self.assertRaises(TypeError) as cm:
         self._lib.apply_op("Simple")
-      self.assertEqual(str(cm.exception), "No argument for input a")
+      self.assertTrue(
+          "No argument for input a" in str(cm.exception))
 
       with self.assertRaises(TypeError) as cm:
         self._lib.apply_op("Simple", wrong=7)
-      self.assertEqual(str(cm.exception), "No argument for input a")
+      self.assertTrue(
+          "No argument for input a" in str(cm.exception))
 
       with self.assertRaises(TypeError) as cm:
         self._lib.apply_op("Simple", a={"label": 1})
-      self.assertEqual(str(cm.exception),
-                       "Expected int32 passed to parameter 'a' of op 'Simple', "
-                       "got {'label': 1} of type 'dict' instead.")
+      self.assertTrue(
+          "Expected int32 passed to parameter 'a' of op 'Simple', "
+          "got {'label': 1} of type 'dict' instead." in str(cm.exception))
 
   def testReservedInput(self):
     with ops.Graph().as_default():
@@ -269,19 +272,13 @@ class OpDefLibraryTest(test_util.TensorFlowTestCase):
         attr { key: 'T' value { type: DT_STRING } }
         """, out.op.node_def)
 
-      with self.assertRaises(TypeError) as cm:
+      with self.assertRaises(TypeError):
         self._lib.apply_op("Binary", a="left", b=12)
-      self.assertEqual(str(cm.exception),
-                       "Expected string passed to parameter 'b' of op 'Binary',"
-                       " got 12 of type 'int' instead.")
 
-      with self.assertRaises(TypeError) as cm:
+      with self.assertRaises(TypeError):
         self._lib.apply_op("Binary",
                            a=self.Tensor(dtypes.string),
                            b=self.Tensor(dtypes.int32))
-      self.assertEqual(str(cm.exception),
-                       "Input 'b' of 'Binary' Op has type int32 "
-                       "that does not match type string of argument 'a'.")
 
   def testRestrict(self):
     with ops.Graph().as_default():
-- 
GitLab


From ff841b6dce21666ee373886a8a6dcc56093a1286 Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Fri, 8 Feb 2019 16:44:23 -0800
Subject: [PATCH 2217/2345] Scrub term "graph network" from error messages and
 docstrings; standardize on "Functional Model"

PiperOrigin-RevId: 233149597
---
 tensorflow/python/keras/engine/network.py        | 9 ++++++---
 tensorflow/python/keras/engine/training_utils.py | 2 +-
 tensorflow/python/keras/models.py                | 6 +++---
 3 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/tensorflow/python/keras/engine/network.py b/tensorflow/python/keras/engine/network.py
index 559df56357..0e1908a4a3 100644
--- a/tensorflow/python/keras/engine/network.py
+++ b/tensorflow/python/keras/engine/network.py
@@ -170,7 +170,7 @@ class Network(base_layer.Layer):
     self._is_compiled = False
     self._expects_training_arg = False
 
-    # This is True for Sequential networks and graph networks.
+    # This is True for Sequential networks and Functional networks.
     self._compute_output_and_mask_jointly = False
 
     self.supports_masking = False
@@ -723,7 +723,7 @@ class Network(base_layer.Layer):
         A list of `InputSpec` instances (one per input to the model)
             or a single instance if the model has only one input.
     """
-    # If not a graph network, can't assume anything.
+    # If subclassed model, can't assume anything.
     if not self._is_graph_network:
       return None
 
@@ -1298,7 +1298,10 @@ class Network(base_layer.Layer):
     """
     if not self._is_graph_network:
       raise NotImplementedError(
-          'Currently `save` requires model to be a graph network. Consider '
+          'The `save` method requires the model to be a Functional model or a '
+          'Sequential model. It does not work for subclassed models, '
+          'because such models are defined via the body of a Python method, '
+          'which isn\'t safely serializable. Consider '
           'using `save_weights`, in order to save the weights of the model.')
 
     from tensorflow.python.keras.models import save_model  # pylint: disable=g-import-not-at-top
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index 25c2f1f774..47c1ba885a 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -1456,7 +1456,7 @@ def get_input_shape_and_dtype(layer):
       does not have a defined input shape.
 
   Raises:
-    ValueError: in case an empty Sequential or Graph Network is passed.
+    ValueError: in case an empty Sequential or Functional model is passed.
   """
 
   def _is_graph_model(layer):
diff --git a/tensorflow/python/keras/models.py b/tensorflow/python/keras/models.py
index d709782ddb..844ce9c9a4 100644
--- a/tensorflow/python/keras/models.py
+++ b/tensorflow/python/keras/models.py
@@ -462,7 +462,7 @@ def clone_and_build_model(
       or functions.
     compile_clone: Boolean, whether to compile model clone (default `True`).
     in_place_reset: Boolean, whether to reset the model in place. Only used if
-      the model is not a graph network. If the model is a subclassed model, then
+      the model is a subclassed model. In the case of a subclassed model,
       this argument must be set to `True` (default `False`). To restore the
       original model, use the function
       `in_place_subclassed_model_state_restoration(model)`.
@@ -505,8 +505,8 @@ def clone_and_build_model(
   else:
     if not in_place_reset:
       raise ValueError(
-          'Model is not a graph network (usually means that it is a subclassed '
-          'model). The model cannot be cloned, but there is a workaround where '
+          'This model is a subclassed model. '
+          'Such a model cannot be cloned, but there is a workaround where '
           'the model is reset in-place. To use this, please set the argument '
           '`in_place_reset` to `True`. This will reset the attributes in the '
           'original model. To restore the attributes, call '
-- 
GitLab


From f2ff301db51c7891b75f60f0ac7c0c1cf8c0f49c Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Fri, 8 Feb 2019 17:21:43 -0800
Subject: [PATCH 2218/2345] Fix tf.contrib.util.constant_value rewrite rule

PiperOrigin-RevId: 233154795
---
 tensorflow/tools/compatibility/tf_upgrade_v2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 080e9420a1..007f9b40ba 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -552,7 +552,7 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.nest.map_structure",
         "tf.contrib.framework.nest.pack_sequence_as":
             "tf.nest.pack_sequence_as",
-        "tf.contrib.framework.constant_value":
+        "tf.contrib.util.constant_value":
             "tf.get_static_value",
         "tf.contrib.saved_model.load_keras_model":
             "tf.keras.experimental.load_from_saved_model",
-- 
GitLab


From 0954d21de9e2d0793e97911885eb12e402d2c340 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Fri, 8 Feb 2019 17:43:50 -0800
Subject: [PATCH 2219/2345] Automated rollback of commit
 882bf2ecae29d8f7c22942ac40110c13e1321380

PiperOrigin-RevId: 233157326
---
 .../python/autograph/converters/call_trees.py |  44 +-----
 .../autograph/converters/call_trees_test.py   |  39 +-----
 tensorflow/python/autograph/impl/api.py       |   4 +-
 tensorflow/python/autograph/impl/api_test.py  | 129 +++++++++---------
 tensorflow/python/framework/func_graph.py     |   2 +-
 5 files changed, 75 insertions(+), 143 deletions(-)

diff --git a/tensorflow/python/autograph/converters/call_trees.py b/tensorflow/python/autograph/converters/call_trees.py
index e53eeb86d9..04439ba516 100644
--- a/tensorflow/python/autograph/converters/call_trees.py
+++ b/tensorflow/python/autograph/converters/call_trees.py
@@ -26,7 +26,6 @@ import gast
 
 from tensorflow.python.autograph.core import converter
 from tensorflow.python.autograph.pyct import anno
-from tensorflow.python.autograph.pyct import ast_util
 from tensorflow.python.autograph.pyct import parser
 from tensorflow.python.autograph.pyct import templates
 
@@ -63,6 +62,9 @@ class CallTreeTransformer(converter.Base):
         not self.ctx.program.options.uses(converter.Feature.BUILTIN_FUNCTIONS)):
       return self.generic_visit(node)
 
+    template = """
+      ag__.converted_call(func, owner, options, args)
+    """
     if isinstance(node.func, gast.Attribute):
       func = gast.Str(node.func.attr)
       owner = node.func.value
@@ -70,41 +72,6 @@ class CallTreeTransformer(converter.Base):
       func = node.func
       owner = parser.parse_expression('None')
 
-    starred_arg = None
-    normal_args = []
-    for a in node.args:
-      if isinstance(a, gast.Starred):
-        assert starred_arg is None, 'Multiple *args should be impossible.'
-        starred_arg = a
-      else:
-        normal_args.append(a)
-    if starred_arg is None:
-      args = templates.replace_as_expression('(args,)', args=normal_args)
-    else:
-      args = templates.replace_as_expression(
-          '(args,) + stararg',
-          stararg=starred_arg.value,
-          args=normal_args)
-
-    kwargs_arg = None
-    normal_keywords = []
-    for k in node.keywords:
-      if k.arg is None:
-        assert kwargs_arg is None, 'Multiple **kwargs should be impossible.'
-        kwargs_arg = k
-      else:
-        normal_keywords.append(k)
-    if kwargs_arg is None:
-      kwargs = ast_util.keywords_to_dict(normal_keywords)
-    else:
-      kwargs = templates.replace_as_expression(
-          'dict(kwargs, **keywords)',
-          kwargs=kwargs_arg.value,
-          keywords=ast_util.keywords_to_dict(normal_keywords))
-
-    template = """
-      ag__.converted_call(func, owner, options, args, kwargs)
-    """
     new_call = templates.replace_as_expression(
         template,
         func=func,
@@ -112,8 +79,9 @@ class CallTreeTransformer(converter.Base):
         options=self.ctx.program.options.to_ast(
             self.ctx,
             internal_convert_user_code=self.ctx.program.options.recursive),
-        args=args,
-        kwargs=kwargs)
+        args=node.args)
+    # TODO(mdan): Improve the template mechanism to better support this.
+    new_call.keywords = node.keywords
 
     return new_call
 
diff --git a/tensorflow/python/autograph/converters/call_trees_test.py b/tensorflow/python/autograph/converters/call_trees_test.py
index 47acda9b46..6ee56bf6be 100644
--- a/tensorflow/python/autograph/converters/call_trees_test.py
+++ b/tensorflow/python/autograph/converters/call_trees_test.py
@@ -34,40 +34,7 @@ class CallTreesTest(converter_testing.TestCase):
       self.assertEquals(
           result.test_fn(None),
           converter_testing.RESULT_OF_MOCK_CONVERTED_CALL + 3)
-      self.assertListEqual(self.dynamic_calls, [((), {})])
-
-  def test_function_with_kwarg(self):
-
-    def test_fn(f, a, b):
-      return f(a, c=b) + 3
-
-    with self.converted(test_fn, call_trees, {}) as result:
-      self.assertEquals(
-          result.test_fn(None, 1, 2),
-          converter_testing.RESULT_OF_MOCK_CONVERTED_CALL + 3)
-      self.assertListEqual(self.dynamic_calls, [((1,), {'c': 2})])
-
-  def test_function_with_kwargs_starargs(self):
-
-    def test_fn(f, a, *args, **kwargs):
-      return f(a, *args, **kwargs) + 5
-
-    with self.converted(test_fn, call_trees, {}) as result:
-      self.assertEquals(
-          result.test_fn(None, 1, *[2, 3], **{'b': 4, 'c': 5}),
-          converter_testing.RESULT_OF_MOCK_CONVERTED_CALL + 5)
-      self.assertListEqual(self.dynamic_calls, [((1, 2, 3), {'b': 4, 'c': 5})])
-
-  def test_function_with_kwargs_keywords(self):
-
-    def test_fn(f, a, b, **kwargs):
-      return f(a, b=b, **kwargs) + 5
-
-    with self.converted(test_fn, call_trees, {}) as result:
-      self.assertEquals(
-          result.test_fn(None, 1, 2, **{'c': 3}),
-          converter_testing.RESULT_OF_MOCK_CONVERTED_CALL + 5)
-      self.assertListEqual(self.dynamic_calls, [((1,), {'b': 2, 'c': 3})])
+      self.assertListEqual(self.dynamic_calls, [()])
 
   def test_class_method(self):
 
@@ -80,7 +47,7 @@ class CallTreesTest(converter_testing.TestCase):
     with self.converted(TestClass.test_method, call_trees, {}) as result:
       self.assertEquals(converter_testing.RESULT_OF_MOCK_CONVERTED_CALL + 1,
                         result.test_method(tc, 1))
-      self.assertListEqual(self.dynamic_calls, [((1,), {})])
+      self.assertListEqual(self.dynamic_calls, [(1,)])
 
   def test_object_method(self):
 
@@ -93,7 +60,7 @@ class CallTreesTest(converter_testing.TestCase):
     with self.converted(tc.test_method, call_trees, {}) as result:
       self.assertEquals(converter_testing.RESULT_OF_MOCK_CONVERTED_CALL + 1,
                         result.test_method(tc, 1))
-      self.assertListEqual(self.dynamic_calls, [((1,), {})])
+      self.assertListEqual(self.dynamic_calls, [(1,)])
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/autograph/impl/api.py b/tensorflow/python/autograph/impl/api.py
index 471ba3b795..0f874a138a 100644
--- a/tensorflow/python/autograph/impl/api.py
+++ b/tensorflow/python/autograph/impl/api.py
@@ -90,7 +90,7 @@ def convert(
               verbose=verbose,
               force_conversion=True,
               optional_features=optional_features,
-          ), args, kwargs)
+          ), *args, **kwargs)
 
     wrapper = tf_decorator.make_decorator(f, wrapper)
 
@@ -162,7 +162,7 @@ def do_not_convert(run_as=RunMode.GRAPH, return_dtypes=None):
   return decorator
 
 
-def converted_call(f, owner, options, args, kwargs):
+def converted_call(f, owner, options, *args, **kwargs):
   """Compiles a function call inline. For internal use only."""
   logging.log(1,
               'Converted call: %s; owner: %s\n    args: %s\n    kwargs: %s\n',
diff --git a/tensorflow/python/autograph/impl/api_test.py b/tensorflow/python/autograph/impl/api_test.py
index 3e17669f0a..73e470a000 100644
--- a/tensorflow/python/autograph/impl/api_test.py
+++ b/tensorflow/python/autograph/impl/api_test.py
@@ -195,17 +195,18 @@ class ApiTest(test.TestCase):
       def test_method(self, x, s, a):
         while tf.reduce_sum(x) > s:
           x //= api.converted_call(self.called_member, None,
-                                   converter.ConversionOptions(), (self, a), {})
+                                   converter.ConversionOptions(), self, a)
         return x
 
     tc = TestClass()
-    x = tc.test_method(
-        constant_op.constant([2, 4]), constant_op.constant(1),
-        constant_op.constant(-2))
-    self.assertListEqual([0, 1], self.evaluate(x).tolist())
+    with self.cached_session() as sess:
+      x = tc.test_method(
+          constant_op.constant([2, 4]), constant_op.constant(1),
+          constant_op.constant(-2))
+      self.assertListEqual([0, 1], self.evaluate(x).tolist())
 
   def test_converted_call_builtin(self):
-    x = api.converted_call(range, None, converter.ConversionOptions(), (3,), {})
+    x = api.converted_call(range, None, converter.ConversionOptions(), 3)
     self.assertEqual((0, 1, 2), tuple(x))
 
   def test_converted_call_function(self):
@@ -215,9 +216,10 @@ class ApiTest(test.TestCase):
         return -x
       return x
 
-    x = api.converted_call(test_fn, None, converter.ConversionOptions(),
-                           (constant_op.constant(-1),), {})
-    self.assertEqual(1, self.evaluate(x))
+    with self.cached_session() as sess:
+      x = api.converted_call(test_fn, None, converter.ConversionOptions(),
+                             constant_op.constant(-1))
+      self.assertEqual(1, self.evaluate(x))
 
   @test_util.run_v1_only('b/120545219')
   def test_converted_call_functools_partial(self):
@@ -228,14 +230,16 @@ class ApiTest(test.TestCase):
       return x, y, z
 
     x = api.converted_call(
-        functools.partial(test_fn, constant_op.constant(-1), z=-3), None,
-        converter.ConversionOptions(), (constant_op.constant(-2),), {})
+        functools.partial(test_fn, constant_op.constant(-1), z=-3),
+        None, converter.ConversionOptions(),
+        constant_op.constant(-2))
     self.assertEqual((1, 2, 3), self.evaluate(x))
 
     x = api.converted_call(
         functools.partial(
-            functools.partial(test_fn, constant_op.constant(-1)), z=-3), None,
-        converter.ConversionOptions(), (constant_op.constant(-2),), {})
+            functools.partial(test_fn, constant_op.constant(-1)), z=-3),
+        None, converter.ConversionOptions(),
+        constant_op.constant(-2))
     self.assertEqual((1, 2, 3), self.evaluate(x))
 
   def test_converted_call_method_explicit_owner(self):
@@ -258,10 +262,11 @@ class ApiTest(test.TestCase):
           return -self.x
         return self.x
 
-    tc = TestClass(constant_op.constant(-1))
-    x = api.converted_call(tc.test_method, None, converter.ConversionOptions(),
-                           (tc,), {})
-    self.assertEqual(1, self.evaluate(x))
+    with self.cached_session() as sess:
+      tc = TestClass(constant_op.constant(-1))
+      x = api.converted_call(tc.test_method, None,
+                             converter.ConversionOptions(), tc)
+      self.assertEqual(1, self.evaluate(x))
 
   def test_converted_call_method_converts_recursively(self):
 
@@ -280,8 +285,7 @@ class ApiTest(test.TestCase):
 
     tc = TestClass(constant_op.constant(-1))
     x = api.converted_call(tc.test_method, None,
-                           converter.ConversionOptions(recursive=True), (tc,),
-                           {})
+                           converter.ConversionOptions(recursive=True), tc)
     self.assertEqual(1, self.evaluate(x))
 
   def test_converted_call_method_by_class(self):
@@ -296,10 +300,11 @@ class ApiTest(test.TestCase):
           return -self.x
         return self.x
 
-    tc = TestClass(constant_op.constant(-1))
-    x = api.converted_call(TestClass.test_method, None,
-                           converter.ConversionOptions(), (tc,), {})
-    self.assertEqual(1, self.evaluate(x))
+    with self.cached_session() as sess:
+      tc = TestClass(constant_op.constant(-1))
+      x = api.converted_call(TestClass.test_method, None,
+                             converter.ConversionOptions(), tc)
+      self.assertEqual(1, self.evaluate(x))
 
   def test_converted_call_callable_object(self):
 
@@ -313,9 +318,10 @@ class ApiTest(test.TestCase):
           return -self.x
         return self.x
 
-    tc = TestClass(constant_op.constant(-1))
-    x = api.converted_call(tc, None, converter.ConversionOptions(), (), {})
-    self.assertEqual(1, self.evaluate(x))
+    with self.cached_session() as sess:
+      tc = TestClass(constant_op.constant(-1))
+      x = api.converted_call(tc, None, converter.ConversionOptions())
+      self.assertEqual(1, self.evaluate(x))
 
   @test_util.run_deprecated_v1
   def test_converted_call_constructor(self):
@@ -331,7 +337,7 @@ class ApiTest(test.TestCase):
         return self.x
 
     tc = api.converted_call(TestClass, None, converter.ConversionOptions(),
-                            (constant_op.constant(-1),), {})
+                            constant_op.constant(-1))
     # tc is still a TestClass - constructors are whitelisted.
     # TODO(b/124016764): Support this use case.
     # The error below is specific to the `if` statement not being converted.
@@ -344,14 +350,15 @@ class ApiTest(test.TestCase):
     def f(x):
       return x == 0
 
-    x = api.converted_call(f, None, converter.ConversionOptions(),
-                           (constant_op.constant(0),), {})
-    self.assertTrue(self.evaluate(x))
+    with self.cached_session() as sess:
+      x = api.converted_call(f, None, converter.ConversionOptions(),
+                             constant_op.constant(0))
+      self.assertTrue(self.evaluate(x))
 
-    converted_f = api.to_graph(f)
-    x = api.converted_call(converted_f, None, converter.ConversionOptions(),
-                           (constant_op.constant(0),), {})
-    self.assertTrue(self.evaluate(x))
+      converted_f = api.to_graph(f)
+      x = api.converted_call(converted_f, None, converter.ConversionOptions(),
+                             constant_op.constant(0))
+      self.assertTrue(self.evaluate(x))
 
   def test_converted_call_then_already_converted_dynamic(self):
 
@@ -366,7 +373,7 @@ class ApiTest(test.TestCase):
       return g(x)
 
     x = api.converted_call(f, None, converter.ConversionOptions(),
-                           (g, constant_op.constant(1)), {})
+                           g, constant_op.constant(1))
     self.assertEqual(self.evaluate(x), 1)
 
   @test_util.run_deprecated_v1
@@ -380,10 +387,10 @@ class ApiTest(test.TestCase):
     # f should not be converted, causing len to error out.
     with self.assertRaisesRegexp(Exception,
                                  'object of type \'Tensor\' has no len()'):
-      api.converted_call(f, None, opts, (constant_op.constant([0]),), {})
+      api.converted_call(f, None, opts, constant_op.constant([0]))
 
     # len on the other hand should work fine.
-    x = api.converted_call(len, None, opts, (constant_op.constant([0]),), {})
+    x = api.converted_call(len, None, opts, constant_op.constant([0]))
     # The constant has static shape so the result is a primitive not a Tensor.
     self.assertEqual(x, 1)
 
@@ -396,10 +403,11 @@ class ApiTest(test.TestCase):
     ])
 
     x = api.converted_call(model.call, None, opts,
-                           (constant_op.constant([[0.0]]),), {'training': True})
+                           constant_op.constant([[0.0]]), training=True)
 
-    self.evaluate(variables.global_variables_initializer())
-    self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
+    with self.cached_session() as sess:
+      self.evaluate(variables.global_variables_initializer())
+      self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
 
   def test_converted_call_whitelisted_method_extra_self(self):
 
@@ -410,11 +418,11 @@ class ApiTest(test.TestCase):
     ])
 
     x = api.converted_call(model.call, None, opts,
-                           (model, constant_op.constant([[0.0]])),
-                           {'training': True})
+                           model, constant_op.constant([[0.0]]), training=True)
 
-    self.evaluate(variables.global_variables_initializer())
-    self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
+    with self.cached_session() as sess:
+      self.evaluate(variables.global_variables_initializer())
+      self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
 
   def test_converted_call_whitelisted_method_via_owner(self):
 
@@ -425,17 +433,18 @@ class ApiTest(test.TestCase):
     ])
 
     x = api.converted_call('call', model, opts,
-                           (constant_op.constant([[0.0]]),), {'training': True})
+                           constant_op.constant([[0.0]]), training=True)
 
-    self.evaluate(variables.global_variables_initializer())
-    self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
+    with self.cached_session() as sess:
+      self.evaluate(variables.global_variables_initializer())
+      self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
 
   def test_converted_call_namedtuple(self):
 
     opts = converter.ConversionOptions()
 
     x = api.converted_call(collections.namedtuple, None, opts,
-                           ('TestNamedtuple', ('a', 'b')), {})
+                           'TestNamedtuple', ('a', 'b'))
 
     self.assertTrue(inspect_utils.isnamedtuple(x))
 
@@ -443,8 +452,8 @@ class ApiTest(test.TestCase):
 
     opts = converter.ConversionOptions()
 
-    x = api.converted_call('namedtuple', collections, opts, ('TestNamedtuple',
-                                                             ('a', 'b')), {})
+    x = api.converted_call('namedtuple', collections, opts,
+                           'TestNamedtuple', ('a', 'b'))
 
     self.assertTrue(inspect_utils.isnamedtuple(x))
 
@@ -454,10 +463,11 @@ class ApiTest(test.TestCase):
 
     l = lambda x: x == 0
 
-    x = api.converted_call(l, None, opts, (constant_op.constant(0),), {})
+    x = api.converted_call(l, None, opts, constant_op.constant(0))
 
-    self.evaluate(variables.global_variables_initializer())
-    self.assertAllEqual(True, self.evaluate(x))
+    with self.cached_session() as sess:
+      self.evaluate(variables.global_variables_initializer())
+      self.assertAllEqual(True, self.evaluate(x))
 
   @test_util.run_deprecated_v1
   def test_to_graph_basic(self):
@@ -503,19 +513,6 @@ class ApiTest(test.TestCase):
     # TODO(b/122368197): This should be the constant 5!
     self.assertEqual(2, testing_global_numeric)
 
-  def test_to_graph_with_kwargs_clashing_converted_call(self):
-
-    def f(**kwargs):
-      return kwargs['f'] + kwargs['owner']
-
-    def test_fn():
-      # These arg names intentionally match converted_call's
-      return f(f=1, owner=2)
-
-    compiled_fn = api.to_graph(test_fn)
-
-    self.assertEqual(compiled_fn(), 3)
-
   def test_to_code_basic(self):
 
     def test_fn(x, s):
diff --git a/tensorflow/python/framework/func_graph.py b/tensorflow/python/framework/func_graph.py
index 5896d8370e..f60cf9350c 100644
--- a/tensorflow/python/framework/func_graph.py
+++ b/tensorflow/python/framework/func_graph.py
@@ -596,7 +596,7 @@ def func_graph_from_py_func(name,
                   strip_decorators=(def_function.function,),
                   optional_features=autograph_options,
                   force_conversion=True,
-              ), args, kwargs)
+              ), *args, **kwargs)
 
         # Wrapping around a decorator allows checks like tf_inspect.getargspec
         # to be accurate.
-- 
GitLab


From 65be9a9c06e1fbd7c55f1b7bc468881a6b70a368 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Fri, 8 Feb 2019 18:12:33 -0800
Subject: [PATCH 2220/2345] Update attention_mechanism and related tests for
 2.0 compatible.

PiperOrigin-RevId: 233160266
---
 .../kernel_tests/attention_wrapper_test.py    |  20 +-
 .../kernel_tests/attention_wrapper_v2_test.py | 679 +++++++++++++++++-
 .../seq2seq/python/ops/attention_wrapper.py   | 459 ++++++++----
 3 files changed, 1012 insertions(+), 146 deletions(-)

diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py
index d815f81f84..1a5692f7b5 100644
--- a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py
+++ b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py
@@ -358,7 +358,7 @@ class AttentionWrapperTest(test.TestCase):
         rnn_output=ResultSummary(
             shape=(5, 3, 6), dtype=dtype('float32'), mean=-0.00597103),
         sample_id=ResultSummary(
-            shape=(5, 3), dtype=dtype('int32'), mean=1.6))
+            shape=(5, 3), dtype=dtype('int32'), mean=1.4))
     expected_final_state = AttentionWrapperState(
         cell_state=LSTMStateTuple(
             c=ResultSummary(
@@ -387,7 +387,7 @@ class AttentionWrapperTest(test.TestCase):
         rnn_output=ResultSummary(
             shape=(5, 3, 6), dtype=dtype('float32'), mean=-0.0052615386),
         sample_id=ResultSummary(
-            shape=(5, 3), dtype=dtype('int32'), mean=1.3333333333))
+            shape=(5, 3), dtype=dtype('int32'), mean=1.4))
     expected_final_state = AttentionWrapperState(
         cell_state=LSTMStateTuple(
             c=ResultSummary(
@@ -454,7 +454,7 @@ class AttentionWrapperTest(test.TestCase):
         rnn_output=ResultSummary(
             shape=(5, 3, 6), dtype=dtype('float32'), mean=-0.0052615386),
         sample_id=ResultSummary(
-            shape=(5, 3), dtype=dtype('int32'), mean=1.3333333333333333))
+            shape=(5, 3), dtype=dtype('int32'), mean=1.4))
     expected_final_state = AttentionWrapperState(
         cell_state=LSTMStateTuple(
             c=ResultSummary(
@@ -696,7 +696,7 @@ class AttentionWrapperTest(test.TestCase):
         rnn_output=ResultSummary(
             shape=(5, 3, 6), dtype=dtype('float32'), mean=-0.0025896581),
         sample_id=ResultSummary(
-            shape=(5, 3), dtype=dtype('int32'), mean=1.6))
+            shape=(5, 3), dtype=dtype('int32'), mean=1.73333333))
     expected_final_state = AttentionWrapperState(
         cell_state=LSTMStateTuple(
             c=ResultSummary(
@@ -707,12 +707,12 @@ class AttentionWrapperTest(test.TestCase):
             shape=(5, 6), dtype=dtype('float32'), mean=-0.00069823361),
         time=3,
         alignments=ResultSummary(
-            shape=(5, 8), dtype=dtype('float32'), mean=0.028698336),
+            shape=(5, 8), dtype=dtype('float32'), mean=0.029914695),
         attention_state=ResultSummary(
-            shape=(5, 8), dtype=dtype('float32'), mean=0.028698336),
+            shape=(5, 8), dtype=dtype('float32'), mean=0.029914695),
         alignment_history=())
     expected_final_alignment_history = ResultSummary(
-        shape=(3, 5, 8), dtype=dtype('float32'), mean=0.04865776002407074)
+        shape=(3, 5, 8), dtype=dtype('float32'), mean=0.0465225502849)
 
     self._testWithAttention(
         create_attention_mechanism,
@@ -921,9 +921,9 @@ class AttentionWrapperTest(test.TestCase):
 
     expected_final_output = BasicDecoderOutput(
         rnn_output=ResultSummary(
-            shape=(5, 3, 20), dtype=dtype('float32'), mean=0.11723966),
+            shape=(5, 3, 20), dtype=dtype('float32'), mean=0.115853324533),
         sample_id=ResultSummary(
-            shape=(5, 3), dtype=dtype('int32'), mean=7.266666666666667))
+            shape=(5, 3), dtype=dtype('int32'), mean=8.6))
     expected_final_state = AttentionWrapperState(
         cell_state=LSTMStateTuple(
             c=ResultSummary(
@@ -931,7 +931,7 @@ class AttentionWrapperTest(test.TestCase):
             h=ResultSummary(
                 shape=(5, 9), dtype=dtype('float32'), mean=-0.0018327223)),
         attention=ResultSummary(
-            shape=(5, 20), dtype=dtype('float32'), mean=0.11601614207),
+            shape=(5, 20), dtype=dtype('float32'), mean=0.11462739855),
         time=3,
         alignments=(ResultSummary(
             shape=(5, 8), dtype=dtype('float32'), mean=0.125),
diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_v2_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_v2_test.py
index 7ff04e1780..5ee01f66f1 100644
--- a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_v2_test.py
+++ b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_v2_test.py
@@ -17,14 +17,23 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import collections
+
 from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.contrib.seq2seq.python.ops import attention_wrapper as wrapper
-from tensorflow.python.framework import ops
+from tensorflow.contrib.seq2seq.python.ops import basic_decoder
+from tensorflow.contrib.seq2seq.python.ops import sampler as sampler_py
+from tensorflow.python import keras
+from tensorflow.python.eager import context
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import test_util
+from tensorflow.python.keras import initializers
+from tensorflow.python.ops import rnn_cell
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
+from tensorflow.python.util import nest
 
 
 @test_util.run_all_in_graph_and_eager_modes
@@ -37,13 +46,10 @@ class AttentionMechanismTest(test.TestCase, parameterized.TestCase):
     self.memory_size = 6
     self.units = 8
 
-    self.memory = ops.convert_to_tensor(
-        np.random.random((self.batch, self.timestep, self.memory_size)),
-        dtype=np.float32)
-    self.query = ops.convert_to_tensor(
-        np.random.random((self.batch, self.units)), dtype=np.float32)
-    self.state = ops.convert_to_tensor(
-        np.random.random((self.batch, self.timestep)), dtype=np.float32)
+    self.memory = np.random.randn(self.batch, self.timestep,
+                                  self.memory_size).astype(np.float32)
+    self.query = np.random.randn(self.batch, self.units).astype(np.float32)
+    self.state = np.random.randn(self.batch, self.timestep).astype(np.float32)
 
   @parameterized.named_parameters(
       ("luong", wrapper.LuongAttentionV2),
@@ -52,8 +58,8 @@ class AttentionMechanismTest(test.TestCase, parameterized.TestCase):
       ("bahdanau_monotonic", wrapper.BahdanauMonotonicAttentionV2),
   )
   def test_attention_shape_inference(self, attention_cls):
-    attention = attention_cls(self.units)
-    attention_score = attention([self.query, self.state, self.memory])
+    attention = attention_cls(self.units, self.memory)
+    attention_score = attention([self.query, self.state])
     self.assertLen(attention_score, 2)
     self.assertEqual(attention_score[0].shape, (self.batch, self.timestep))
     self.assertEqual(attention_score[1].shape, (self.batch, self.timestep))
@@ -65,7 +71,7 @@ class AttentionMechanismTest(test.TestCase, parameterized.TestCase):
       ("bahdanau_monotonic", wrapper.BahdanauMonotonicAttentionV2),
   )
   def test_get_config(self, attention_cls):
-    attention = attention_cls(self.units)
+    attention = attention_cls(self.units, self.memory)
     config = attention.get_config()
 
     attention_from_config = attention_cls.from_config(config)
@@ -80,9 +86,8 @@ class AttentionMechanismTest(test.TestCase, parameterized.TestCase):
       ("bahdanau_monotonic", wrapper.BahdanauMonotonicAttentionV2),
   )
   def test_layer_output(self, attention_cls):
-    attention = attention_cls(self.units)
-
-    score = attention([self.query, self.state, self.memory])
+    attention = attention_cls(self.units, self.memory)
+    score = attention([self.query, self.state])
     self.evaluate(variables.variables_initializer(attention.variables))
 
     score_val = self.evaluate(score)
@@ -90,5 +95,651 @@ class AttentionMechanismTest(test.TestCase, parameterized.TestCase):
     self.assertEqual(score_val[0].shape, (self.batch, self.timestep))
     self.assertEqual(score_val[1].shape, (self.batch, self.timestep))
 
+  @parameterized.named_parameters(
+      ("luong", wrapper.LuongAttentionV2),
+      ("luong_monotonic", wrapper.LuongMonotonicAttentionV2),
+      ("bahdanau", wrapper.BahdanauAttentionV2),
+      ("bahdanau_monotonic", wrapper.BahdanauMonotonicAttentionV2),
+  )
+  def test_passing_memory_from_call(self, attention_cls):
+    attention = attention_cls(self.units, self.memory)
+    weights_before_query = attention.get_weights()
+    ref_score = attention([self.query, self.state])
+
+    self.evaluate(variables.global_variables_initializer())
+    ref_score_val = self.evaluate(ref_score)
+
+    all_weights = attention.get_weights()
+    config = attention.get_config()
+    # Simulate the twice invocation of calls here.
+    attention_from_config = attention_cls.from_config(config)
+    attention_from_config.build(self.memory.shape)
+    attention_from_config.set_weights(weights_before_query)
+    attention_from_config(self.memory, setup_memory=True)
+    attention_from_config.build([self.query.shape, self.state.shape])
+    attention_from_config.set_weights(all_weights)
+    score = attention_from_config([self.query, self.state])
+
+    score_val = self.evaluate(score)
+    self.assertAllClose(ref_score_val, score_val)
+
+  @parameterized.named_parameters(
+      ("luong", wrapper.LuongAttentionV2),
+      ("luong_monotonic", wrapper.LuongMonotonicAttentionV2),
+      ("bahdanau", wrapper.BahdanauAttentionV2),
+      ("bahdanau_monotonic", wrapper.BahdanauMonotonicAttentionV2),
+  )
+  def test_save_load_layer(self, attention_cls):
+    vocab = 20
+    embedding_dim = 6
+    inputs = keras.layers.Input(shape=[self.timestep])
+    encoder_input = keras.layers.Embedding(
+        vocab, embedding_dim, mask_zero=True)(
+            inputs)
+    encoder_output = keras.layers.UnifiedLSTM(
+        self.memory_size, return_sequences=True)(
+            encoder_input)
+
+    attention = attention_cls(self.units, encoder_output)
+    query = keras.layers.Input(shape=[self.units])
+    state = keras.layers.Input(shape=[self.timestep])
+
+    score = attention([query, state])
+
+    x = np.random.randint(vocab, size=(self.batch, self.timestep))
+    x_test = np.random.randint(vocab, size=(self.batch, self.timestep))
+    y = np.random.randn(self.batch, self.timestep)
+    model = keras.models.Model([inputs, query, state], score)
+    model.compile("rmsprop", "mse")
+    model.fit([x, self.query, self.state], (y, y))
+    y_ref = model.predict_on_batch([x_test, self.query, self.state])
+
+    config = model.get_config()
+    weights = model.get_weights()
+    loaded_model = keras.models.Model.from_config(
+        config, custom_objects={attention_cls.__name__: attention_cls})
+    loaded_model.set_weights(weights)
+
+    y = loaded_model.predict_on_batch([x_test, self.query, self.state])
+
+    self.assertAllClose(y_ref, y)
+
+  # TODO(scottzhu): Add tests for model.compile(run_eagerly=True)
+
+
+class ResultSummary(
+    collections.namedtuple("ResultSummary", ("shape", "dtype", "mean"))):
+  pass
+
+
+def get_result_summary(x):
+  if isinstance(x, np.ndarray):
+    return ResultSummary(x.shape, x.dtype, x.mean())
+  return x
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class AttentionWrapperV2Test(test.TestCase, parameterized.TestCase):
+
+  def assertAllCloseOrEqual(self, x, y, **kwargs):
+    if isinstance(x, np.ndarray) or isinstance(x, float):
+      return super(AttentionWrapperV2Test, self).assertAllClose(
+          x, y, atol=1e-3, **kwargs)
+    else:
+      self.assertAllEqual(x, y, **kwargs)
+
+  def setUp(self):
+    super(AttentionWrapperV2Test, self).setUp()
+    self.batch = 64
+    self.units = 128
+    self.encoder_timestep = 10
+    self.encoder_dim = 256
+    self.decoder_timestep = 12
+    self.encoder_outputs = np.random.randn(self.batch, self.encoder_timestep,
+                                           self.encoder_dim)
+    self.encoder_sequence_length = np.random.randint(
+        self.encoder_timestep, size=(self.batch,)).astype(np.int32)
+    self.decoder_inputs = np.random.randn(self.batch, self.decoder_timestep,
+                                          self.units)
+    self.decoder_sequence_length = np.random.randint(
+        self.decoder_timestep, size=(self.batch,)).astype(np.int32)
+
+  def _testWithAttention(self,
+                         create_attention_mechanism,
+                         expected_final_output,
+                         expected_final_state,
+                         attention_mechanism_depth=3,
+                         alignment_history=False,
+                         expected_final_alignment_history=None,
+                         attention_layer_size=6,
+                         attention_layer=None,
+                         create_query_layer=False,
+                         create_memory_layer=True,
+                         create_attention_kwargs=None):
+    attention_layer_sizes = ([attention_layer_size]
+                             if attention_layer_size is not None else None)
+    attention_layers = ([attention_layer]
+                        if attention_layer is not None else None)
+    self._testWithMaybeMultiAttention(
+        is_multi=False,
+        create_attention_mechanisms=[create_attention_mechanism],
+        expected_final_output=expected_final_output,
+        expected_final_state=expected_final_state,
+        attention_mechanism_depths=[attention_mechanism_depth],
+        alignment_history=alignment_history,
+        expected_final_alignment_history=expected_final_alignment_history,
+        attention_layer_sizes=attention_layer_sizes,
+        attention_layers=attention_layers,
+        create_query_layer=create_query_layer,
+        create_memory_layer=create_memory_layer,
+        create_attention_kwargs=create_attention_kwargs)
+
+  def _testWithMaybeMultiAttention(self,
+                                   is_multi,
+                                   create_attention_mechanisms,
+                                   expected_final_output,
+                                   expected_final_state,
+                                   attention_mechanism_depths,
+                                   alignment_history=False,
+                                   expected_final_alignment_history=None,
+                                   attention_layer_sizes=None,
+                                   attention_layers=None,
+                                   create_query_layer=False,
+                                   create_memory_layer=True,
+                                   create_attention_kwargs=None):
+    # Allow is_multi to be True with a single mechanism to enable test for
+    # passing in a single mechanism in a list.
+    assert len(create_attention_mechanisms) == 1 or is_multi
+    encoder_sequence_length = [3, 2, 3, 1, 1]
+    decoder_sequence_length = [2, 0, 1, 2, 3]
+    batch_size = 5
+    encoder_max_time = 8
+    decoder_max_time = 4
+    input_depth = 7
+    encoder_output_depth = 10
+    cell_depth = 9
+    create_attention_kwargs = create_attention_kwargs or {}
+
+    if attention_layer_sizes is not None:
+      # Compute sum of attention_layer_sizes. Use encoder_output_depth if None.
+      attention_depth = sum(attention_layer_size or encoder_output_depth
+                            for attention_layer_size in attention_layer_sizes)
+    elif attention_layers is not None:
+      # Compute sum of attention_layers output depth.
+      attention_depth = sum(
+          attention_layer.compute_output_shape(
+              [batch_size, cell_depth + encoder_output_depth]).dims[-1].value
+          for attention_layer in attention_layers)
+    else:
+      attention_depth = encoder_output_depth * len(create_attention_mechanisms)
+
+    decoder_inputs = np.random.randn(batch_size, decoder_max_time,
+                                     input_depth).astype(np.float32)
+    encoder_outputs = np.random.randn(batch_size, encoder_max_time,
+                                      encoder_output_depth).astype(np.float32)
+
+    attention_mechanisms = []
+    for creator, depth in zip(create_attention_mechanisms,
+                              attention_mechanism_depths):
+      # Create a memory layer with deterministic initializer to avoid randomness
+      # in the test between graph and eager.
+      if create_query_layer:
+        create_attention_kwargs["query_layer"] = keras.layers.Dense(
+            depth, kernel_initializer="ones", use_bias=False)
+      if create_memory_layer:
+        create_attention_kwargs["memory_layer"] = keras.layers.Dense(
+            depth, kernel_initializer="ones", use_bias=False)
+
+      attention_mechanisms.append(
+          creator(
+              units=depth,
+              memory=encoder_outputs,
+              memory_sequence_length=encoder_sequence_length,
+              **create_attention_kwargs))
+
+    with self.cached_session(use_gpu=True):
+      attention_layer_size = attention_layer_sizes
+      attention_layer = attention_layers
+      if not is_multi:
+        if attention_layer_size is not None:
+          attention_layer_size = attention_layer_size[0]
+        if attention_layer is not None:
+          attention_layer = attention_layer[0]
+      cell = rnn_cell.LSTMCell(cell_depth, initializer="ones")
+      cell = wrapper.AttentionWrapper(
+          cell,
+          attention_mechanisms if is_multi else attention_mechanisms[0],
+          attention_layer_size=attention_layer_size,
+          alignment_history=alignment_history,
+          attention_layer=attention_layer)
+      # Set the attention_layer within AttentionWrapper to have deterministic
+      # kernel initializer, for testing purpose.
+      if cell._attention_layers is not None:
+        for layer in cell._attention_layers:
+          if getattr(layer, "kernel_initializer") is None:
+            layer.kernel_initializer = initializers.ones()
+
+      sampler = sampler_py.TrainingSampler()
+      my_decoder = basic_decoder.BasicDecoderV2(cell=cell, sampler=sampler)
+      initial_state = cell.zero_state(
+          dtype=dtypes.float32, batch_size=batch_size)
+      final_outputs, final_state, _ = my_decoder(
+          decoder_inputs,
+          initial_state=initial_state,
+          sequence_length=decoder_sequence_length)
+
+      self.assertIsInstance(final_outputs, basic_decoder.BasicDecoderOutput)
+      self.assertIsInstance(final_state, wrapper.AttentionWrapperState)
+      self.assertIsInstance(final_state.cell_state, rnn_cell.LSTMStateTuple)
+
+      expected_time = (
+          expected_final_state.time if context.executing_eagerly() else None)
+      self.assertEqual((batch_size, expected_time, attention_depth),
+                       tuple(final_outputs.rnn_output.get_shape().as_list()))
+      self.assertEqual((batch_size, expected_time),
+                       tuple(final_outputs.sample_id.get_shape().as_list()))
+
+      self.assertEqual((batch_size, attention_depth),
+                       tuple(final_state.attention.get_shape().as_list()))
+      self.assertEqual((batch_size, cell_depth),
+                       tuple(final_state.cell_state.c.get_shape().as_list()))
+      self.assertEqual((batch_size, cell_depth),
+                       tuple(final_state.cell_state.h.get_shape().as_list()))
+
+      if alignment_history:
+        if is_multi:
+          state_alignment_history = []
+          for history_array in final_state.alignment_history:
+            history = history_array.stack()
+            self.assertEqual((expected_time, batch_size, encoder_max_time),
+                             tuple(history.get_shape().as_list()))
+            state_alignment_history.append(history)
+          state_alignment_history = tuple(state_alignment_history)
+        else:
+          state_alignment_history = final_state.alignment_history.stack()
+          self.assertEqual((expected_time, batch_size, encoder_max_time),
+                           tuple(state_alignment_history.get_shape().as_list()))
+        nest.assert_same_structure(cell.state_size,
+                                   cell.zero_state(batch_size, dtypes.float32))
+        # Remove the history from final_state for purposes of the
+        # remainder of the tests.
+        final_state = final_state._replace(alignment_history=())  # pylint: disable=protected-access
+      else:
+        state_alignment_history = ()
+
+      self.evaluate(variables.global_variables_initializer())
+      eval_result = self.evaluate({
+          "final_outputs": final_outputs,
+          "final_state": final_state,
+          "state_alignment_history": state_alignment_history,
+      })
+
+      final_output_info = nest.map_structure(get_result_summary,
+                                             eval_result["final_outputs"])
+      final_state_info = nest.map_structure(get_result_summary,
+                                            eval_result["final_state"])
+      print("final_output_info: ", final_output_info)
+      print("final_state_info: ", final_state_info)
+
+      nest.map_structure(self.assertAllCloseOrEqual, expected_final_output,
+                         final_output_info)
+      nest.map_structure(self.assertAllCloseOrEqual, expected_final_state,
+                         final_state_info)
+      if alignment_history:  # by default, the wrapper emits attention as output
+        final_alignment_history_info = nest.map_structure(
+            get_result_summary, eval_result["state_alignment_history"])
+        print("final_alignment_history_info: ", final_alignment_history_info)
+        nest.map_structure(
+            self.assertAllCloseOrEqual,
+            # outputs are batch major but the stacked TensorArray is time major
+            expected_final_alignment_history,
+            final_alignment_history_info)
+
+  @parameterized.parameters([np.float16, np.float32, np.float64])
+  def _testBahdanauNormalizedDType(self, dtype):
+    encoder_outputs = self.encoder_outputs.astype(dtype)
+    decoder_inputs = self.decoder_inputs.astype(dtype)
+    attention_mechanism = wrapper.BahdanauAttentionV2(
+        units=self.units,
+        memory=encoder_outputs,
+        memory_sequence_length=self.encoder_sequence_length,
+        normalize=True,
+        dtype=dtype)
+    cell = rnn_cell.LSTMCell(self.units)
+    cell = wrapper.AttentionWrapper(cell, attention_mechanism)
+
+    sampler = sampler_py.TrainingSampler()
+    my_decoder = basic_decoder.BasicDecoderV2(cell=cell, sampler=sampler)
+
+    final_outputs, final_state, _ = my_decoder(
+        decoder_inputs,
+        initial_state=cell.zero_state(dtype=dtype, batch_size=self.batch),
+        sequence_length=self.decoder_sequence_length)
+    self.assertIsInstance(final_outputs, basic_decoder.BasicDecoderOutput)
+    self.assertEqual(final_outputs.rnn_output.dtype, dtype)
+    self.assertIsInstance(final_state, wrapper.AttentionWrapperState)
+    self.assertIsInstance(final_state.cell_state, rnn_cell.LSTMStateTuple)
+
+  @parameterized.parameters([np.float16, np.float32, np.float64])
+  def testLuongScaledDType(self, dtype):
+    # Test case for GitHub issue 18099
+    encoder_outputs = self.encoder_outputs.astype(dtype)
+    decoder_inputs = self.decoder_inputs.astype(dtype)
+    attention_mechanism = wrapper.LuongAttentionV2(
+        units=self.units,
+        memory=encoder_outputs,
+        memory_sequence_length=self.encoder_sequence_length,
+        scale=True,
+        dtype=dtype,
+    )
+    cell = rnn_cell.LSTMCell(self.units)
+    cell = wrapper.AttentionWrapper(cell, attention_mechanism)
+
+    sampler = sampler_py.TrainingSampler()
+    my_decoder = basic_decoder.BasicDecoderV2(cell=cell, sampler=sampler)
+
+    final_outputs, final_state, _ = my_decoder(
+        decoder_inputs,
+        initial_state=cell.zero_state(dtype=dtype, batch_size=self.batch),
+        sequence_length=self.decoder_sequence_length)
+    self.assertIsInstance(final_outputs, basic_decoder.BasicDecoderOutput)
+    self.assertEqual(final_outputs.rnn_output.dtype, dtype)
+    self.assertIsInstance(final_state, wrapper.AttentionWrapperState)
+    self.assertIsInstance(final_state.cell_state, rnn_cell.LSTMStateTuple)
+
+  def testBahdanauNotNormalized(self):
+    create_attention_mechanism = wrapper.BahdanauAttentionV2
+    create_attention_kwargs = {"kernel_initializer": "ones"}
+    expected_final_output = basic_decoder.BasicDecoderOutput(
+        rnn_output=ResultSummary(
+            shape=(5, 3, 6), dtype=np.dtype(np.float32), mean=4.8290324),
+        sample_id=ResultSummary(shape=(5, 3), dtype=np.dtype(np.int32), mean=0))
+    expected_final_state = wrapper.AttentionWrapperState(
+        cell_state=rnn_cell.LSTMStateTuple(
+            c=ResultSummary(
+                shape=(5, 9), dtype=np.dtype(np.float32), mean=1.6432636),
+            h=ResultSummary(
+                shape=(5, 9), dtype=np.dtype(np.float32), mean=0.75866824)),
+        attention=ResultSummary(
+            shape=(5, 6), dtype=np.dtype(np.float32), mean=6.7445569),
+        time=3,
+        alignments=ResultSummary(
+            shape=(5, 8), dtype=np.dtype(np.float32), mean=0.125),
+        attention_state=ResultSummary(
+            shape=(5, 8), dtype=np.dtype(np.float32), mean=0.125),
+        alignment_history=())
+    expected_final_alignment_history = ResultSummary(
+        shape=(3, 5, 8), dtype=np.dtype(np.float32), mean=0.125)
+
+    self._testWithAttention(
+        create_attention_mechanism,
+        expected_final_output,
+        expected_final_state,
+        alignment_history=True,
+        create_query_layer=True,
+        expected_final_alignment_history=expected_final_alignment_history,
+        create_attention_kwargs=create_attention_kwargs)
+
+  def testBahdanauNormalized(self):
+    create_attention_mechanism = wrapper.BahdanauAttentionV2
+    create_attention_kwargs = {"kernel_initializer": "ones", "normalize": True}
+
+    expected_final_output = basic_decoder.BasicDecoderOutput(
+        rnn_output=ResultSummary(
+            shape=(5, 3, 6), dtype=np.dtype("float32"), mean=3.9548259),
+        sample_id=ResultSummary(
+            shape=(5, 3), dtype=np.dtype("int32"), mean=0.0))
+    expected_final_state = wrapper.AttentionWrapperState(
+        cell_state=rnn_cell.LSTMStateTuple(
+            c=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=1.4652209),
+            h=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=0.70997983)),
+        attention=ResultSummary(
+            shape=(5, 6), dtype=np.dtype("float32"), mean=6.3075728),
+        time=3,
+        alignments=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.125),
+        attention_state=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.125),
+        alignment_history=())
+
+    self._testWithAttention(
+        create_attention_mechanism,
+        expected_final_output,
+        expected_final_state,
+        create_query_layer=True,
+        create_attention_kwargs=create_attention_kwargs)
+
+  def testLuongNotNormalized(self):
+    create_attention_mechanism = wrapper.LuongAttentionV2
+
+    expected_final_output = basic_decoder.BasicDecoderOutput(
+        rnn_output=ResultSummary(
+            shape=(5, 3, 6), dtype=np.dtype("float32"), mean=2.6605489),
+        sample_id=ResultSummary(
+            shape=(5, 3), dtype=np.dtype("int32"), mean=0.0))
+    expected_final_state = wrapper.AttentionWrapperState(
+        cell_state=rnn_cell.LSTMStateTuple(
+            c=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=0.88403547),
+            h=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=0.37819088)),
+        attention=ResultSummary(
+            shape=(5, 6), dtype=np.dtype("float32"), mean=4.084631),
+        time=3,
+        alignments=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.125),
+        attention_state=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.125),
+        alignment_history=())
+
+    self._testWithAttention(
+        create_attention_mechanism,
+        expected_final_output,
+        expected_final_state,
+        attention_mechanism_depth=9)
+
+  def testLuongScaled(self):
+    create_attention_mechanism = wrapper.LuongAttentionV2
+    create_attention_kwargs = {"scale": True}
+
+    expected_final_output = basic_decoder.BasicDecoderOutput(
+        rnn_output=ResultSummary(
+            shape=(5, 3, 6), dtype=np.dtype("float32"), mean=2.6605489),
+        sample_id=ResultSummary(
+            shape=(5, 3), dtype=np.dtype("int32"), mean=0.0))
+    expected_final_state = wrapper.AttentionWrapperState(
+        cell_state=rnn_cell.LSTMStateTuple(
+            c=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=0.88403547),
+            h=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=0.37819088)),
+        attention=ResultSummary(
+            shape=(5, 6), dtype=np.dtype("float32"), mean=4.0846314),
+        time=3,
+        alignments=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.125),
+        attention_state=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.125),
+        alignment_history=())
+
+    self._testWithAttention(
+        create_attention_mechanism,
+        expected_final_output,
+        expected_final_state,
+        attention_mechanism_depth=9,
+        create_attention_kwargs=create_attention_kwargs)
+
+  def testNotUseAttentionLayer(self):
+    create_attention_mechanism = wrapper.BahdanauAttentionV2
+    create_attention_kwargs = {"kernel_initializer": "ones"}
+
+    expected_final_output = basic_decoder.BasicDecoderOutput(
+        rnn_output=ResultSummary(
+            shape=(5, 3, 10), dtype=np.dtype("float32"), mean=0.072406612),
+        sample_id=ResultSummary(
+            shape=(5, 3), dtype=np.dtype("int32"), mean=3.86666666))
+    expected_final_state = wrapper.AttentionWrapperState(
+        cell_state=rnn_cell.LSTMStateTuple(
+            c=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=1.032002),
+            h=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=0.61177742)),
+        attention=ResultSummary(
+            shape=(5, 10), dtype=np.dtype("float32"), mean=0.011346335),
+        time=3,
+        alignments=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.125),
+        attention_state=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.125),
+        alignment_history=())
+
+    self._testWithAttention(
+        create_attention_mechanism,
+        expected_final_output,
+        expected_final_state,
+        attention_layer_size=None,
+        create_query_layer=True,
+        create_attention_kwargs=create_attention_kwargs)
+
+  def testBahdanauMonotonicNotNormalized(self):
+    create_attention_mechanism = wrapper.BahdanauMonotonicAttentionV2
+    create_attention_kwargs = {"kernel_initializer": "ones"}
+
+    expected_final_output = basic_decoder.BasicDecoderOutput(
+        rnn_output=ResultSummary(
+            shape=(5, 3, 6), dtype=np.dtype("float32"), mean=5.9850435),
+        sample_id=ResultSummary(
+            shape=(5, 3), dtype=np.dtype("int32"), mean=0.0))
+    expected_final_state = wrapper.AttentionWrapperState(
+        cell_state=rnn_cell.LSTMStateTuple(
+            c=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=1.6752492),
+            h=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=0.76052248)),
+        attention=ResultSummary(
+            shape=(5, 6), dtype=np.dtype("float32"), mean=8.361186),
+        time=3,
+        alignments=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.10989678),
+        attention_state=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.10989678),
+        alignment_history=())
+    expected_final_alignment_history = ResultSummary(
+        shape=(3, 5, 8), dtype=np.dtype("float32"), mean=0.117412611)
+
+    self._testWithAttention(
+        create_attention_mechanism,
+        expected_final_output,
+        expected_final_state,
+        alignment_history=True,
+        expected_final_alignment_history=expected_final_alignment_history,
+        create_query_layer=True,
+        create_attention_kwargs=create_attention_kwargs)
+
+  def testBahdanauMonotonicNormalized(self):
+    create_attention_mechanism = wrapper.BahdanauMonotonicAttentionV2
+    create_attention_kwargs = {"kernel_initializer": "ones",
+                               "normalize": True}
+    expected_final_output = basic_decoder.BasicDecoderOutput(
+        rnn_output=ResultSummary(
+            shape=(5, 3, 6), dtype=np.dtype("float32"), mean=4.5706983),
+        sample_id=ResultSummary(
+            shape=(5, 3), dtype=np.dtype("int32"), mean=0.0))
+    expected_final_state = wrapper.AttentionWrapperState(
+        cell_state=rnn_cell.LSTMStateTuple(
+            c=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=1.6005473),
+            h=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=0.77863038)),
+        attention=ResultSummary(
+            shape=(5, 6), dtype=np.dtype("float32"), mean=7.3326721),
+        time=3,
+        alignments=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.12258384),
+        attention_state=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.12258384),
+        alignment_history=())
+    expected_final_alignment_history = ResultSummary(
+        shape=(3, 5, 8), dtype=np.dtype("float32"), mean=0.12258384)
+
+    self._testWithAttention(
+        create_attention_mechanism,
+        expected_final_output,
+        expected_final_state,
+        alignment_history=True,
+        expected_final_alignment_history=expected_final_alignment_history,
+        create_query_layer=True,
+        create_attention_kwargs=create_attention_kwargs)
+
+  def testLuongMonotonicNotNormalized(self):
+    create_attention_mechanism = wrapper.LuongMonotonicAttentionV2
+
+    expected_final_output = basic_decoder.BasicDecoderOutput(
+        rnn_output=ResultSummary(
+            shape=(5, 3, 6), dtype=np.dtype("float32"), mean=3.159497),
+        sample_id=ResultSummary(
+            shape=(5, 3), dtype=np.dtype("int32"), mean=0.0))
+    expected_final_state = wrapper.AttentionWrapperState(
+        cell_state=rnn_cell.LSTMStateTuple(
+            c=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=1.072384),
+            h=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=0.50331038)),
+        attention=ResultSummary(
+            shape=(5, 6), dtype=np.dtype("float32"), mean=5.3079605),
+        time=3,
+        alignments=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.11467695),
+        attention_state=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.11467695),
+        alignment_history=())
+    expected_final_alignment_history = ResultSummary(
+        shape=(3, 5, 8), dtype=np.dtype("float32"), mean=0.11899644)
+
+    self._testWithAttention(
+        create_attention_mechanism,
+        expected_final_output,
+        expected_final_state,
+        attention_mechanism_depth=9,
+        alignment_history=True,
+        expected_final_alignment_history=expected_final_alignment_history)
+
+  def testLuongMonotonicScaled(self):
+    create_attention_mechanism = wrapper.LuongMonotonicAttentionV2
+    create_attention_kwargs = {"scale": True}
+
+    expected_final_output = basic_decoder.BasicDecoderOutput(
+        rnn_output=ResultSummary(
+            shape=(5, 3, 6), dtype=np.dtype("float32"), mean=3.159497),
+        sample_id=ResultSummary(
+            shape=(5, 3), dtype=np.dtype("int32"), mean=0.0))
+    expected_final_state = wrapper.AttentionWrapperState(
+        cell_state=rnn_cell.LSTMStateTuple(
+            c=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=1.072384),
+            h=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=0.50331038)),
+        attention=ResultSummary(
+            shape=(5, 6), dtype=np.dtype("float32"), mean=5.3079605),
+        time=3,
+        alignments=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.11467695),
+        attention_state=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.11467695),
+        alignment_history=())
+    expected_final_alignment_history = ResultSummary(
+        shape=(3, 5, 8), dtype=np.dtype("float32"), mean=0.11899644)
+
+    self._testWithAttention(
+        create_attention_mechanism,
+        expected_final_output,
+        expected_final_state,
+        attention_mechanism_depth=9,
+        alignment_history=True,
+        expected_final_alignment_history=expected_final_alignment_history,
+        create_attention_kwargs=create_attention_kwargs)
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
index ae3e7f1b5d..07635e4dde 100644
--- a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
+++ b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
@@ -28,7 +28,9 @@ from tensorflow.contrib.framework.python.framework import tensor_util
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.keras import initializers
 from tensorflow.python.keras import layers
+from tensorflow.python.keras.engine import base_layer_utils
 from tensorflow.python.layers import base as layers_base
 from tensorflow.python.layers import core as layers_core
 from tensorflow.python.ops import array_ops
@@ -225,18 +227,39 @@ class _BaseAttentionMechanismV2(AttentionMechanism, layers.Layer):
     1. Storing the query and memory layers.
     2. Preprocessing and storing the memory.
 
-  Note that this layer only support Keras functional API since it takes multiple
-  input tensors, which is not available in sequential model.
+  Note that this layer takes memory as its init parameter, which is an
+  anti-pattern of Keras API, we have to keep the memory as init parameter for
+  performance and dependency reason. Under the hood, during `__init__()`, it
+  will invoke `base_layer.__call__(memory, setup_memory=True)`. This will let
+  keras to keep track of the memory tensor as the input of this layer. Once
+  the `__init__()` is done, then user can query the attention by
+  `score = att_obj([query, state])`, and use it as a normal keras layer.
+
+  Special attention is needed when adding using this class as the base layer for
+  new attention:
+    1. Build() could be invoked at least twice. So please make sure weights are
+       not duplicated.
+    2. Layer.get_weights() might return different set of weights if the instance
+       has `query_layer`. The query_layer weights is not initialized until the
+       memory is configured.
+
+  Also note that this layer does not work with Keras model when
+  `model.compile(run_eagerly=True)` due to the fact that this layer is stateful.
+  The support for that will be added in a future version.
   """
 
   def __init__(self,
+               memory,
                probability_fn,
                query_layer=None,
                memory_layer=None,
+               memory_sequence_length=None,
                **kwargs):
     """Construct base AttentionMechanism class.
 
     Args:
+      memory: The memory to query; usually the output of an RNN encoder.  This
+        tensor should be shaped `[batch_size, max_time, ...]`.
       probability_fn: A `callable`. Converts the score and previous alignments
         to probabilities. Its signature should be:
         `probabilities = probability_fn(score, state)`.
@@ -247,6 +270,9 @@ class _BaseAttentionMechanismV2(AttentionMechanism, layers.Layer):
         depth must match the depth of `query_layer`.
         If `memory_layer` is not provided, the shape of `memory` must match
         that of `query_layer`.
+      memory_sequence_length (optional): Sequence lengths for the batch entries
+        in memory. If provided, the memory tensor rows are masked with zeros
+        for values past the respective sequence lengths.
       **kwargs: Dictionary that contains other common arguments for layer
         creation.
     """
@@ -273,20 +299,128 @@ class _BaseAttentionMechanismV2(AttentionMechanism, layers.Layer):
     self.batch_size = None
     self._memory_initialized = False
     self._check_inner_dims_defined = True
+    self.supports_masking = True
+    self.score_mask_value = dtypes.as_dtype(self.dtype).as_numpy_dtype(-np.inf)
+
+    if memory is not None:
+      # Setup the memory by self.__call__() with memory and memory_seq_length.
+      # This will make the attention follow the keras convention which takes
+      # all the tensor inputs via __call__().
+      if memory_sequence_length is None:
+        inputs = memory
+      else:
+        inputs = [memory, memory_sequence_length]
+      # The returned tensor is ignored here since we already record it as an
+      # attribute.
+      super(_BaseAttentionMechanismV2, self).__call__(
+          inputs, setup_memory=True)
 
   def build(self, input_shape):
-    # The layer suppose to take 3 inputs, [query, state, memory].
-    query_input_shape, _, memory_input_shape = input_shape
-    if self.query_layer is not None:
-      self.query_layer.build(query_input_shape)
-    if self.memory_layer is not None:
-      self.memory_layer.build(memory_input_shape)
-    # dtype of the layer is known at this moment, create the score_mask_value if
-    # needed.
-    self.score_mask_value = dtypes.as_dtype(self.dtype).as_numpy_dtype(-np.inf)
-    self.built = True
+    if not self._memory_initialized:
+      # This is for setting up the memory, which contains memory and optional
+      # memory_sequence_length. Build the memory_layer with memory shape.
+      if self.memory_layer is not None and not self.memory_layer.built:
+        if isinstance(input_shape, list):
+          self.memory_layer.build(input_shape[0])
+        else:
+          self.memory_layer.build(input_shape)
+    else:
+      # The input_shape should be query.shape and state.shape. Use the query
+      # to init the query layer.
+      if self.query_layer is not None and not self.query_layer.built:
+        self.query_layer.build(input_shape[0])
+
+  def __call__(self, inputs, **kwargs):
+    """Preprocess the inputs before calling `base_layer.__call__()`.
+
+    Note that there are situation here, one for setup memory, and one with
+    actual query and state.
+    1. When the memory has not been configured, we just pass all the param to
+    base_layer.__call__(), which will then invoke self.call() with proper
+    inputs, which allows this class to setup memory.
+    2. When the memory has already been setup, the input should contain query
+    and state, and optionally processed memory. If the processed memory is
+    not included in the input, we will have to append it to the inputs and
+    give it to the base_layer.__call__(). The processed memory is the output
+    of first invocation of self.__call__(). If we don't add it here, then from
+    keras perspective, the graph is disconnected since the output from
+    previous call is never used.
+
+    Args:
+      inputs: the inputs tensors.
+      **kwargs: dict, other keyeword arguments for the `__call__()`
+    """
+    if self._memory_initialized:
+      if len(inputs) not in (2, 3):
+        raise ValueError("Expect the inputs to have 2 or 3 tensors, got %d" %
+                         len(inputs))
+      if len(inputs) == 2:
+        # We append the calculated memory here so that the graph will be
+        # connected.
+        inputs.append(self.values)
+    return super(_BaseAttentionMechanismV2, self).__call__(inputs, **kwargs)
+
+  def call(self, inputs, mask=None, setup_memory=False, **kwargs):
+    """Setup the memory or query the attention.
+
+    There are two case here, one for setup memory, and the second is query the
+    attention score. `setup_memory` is the flag to indicate which mode it is.
+    The input list will be treated differently based on that flag.
 
-  def _setup_memory(self, memory, memory_mask=None):
+    Args:
+      inputs: a list of tensor that could either be `query` and `state`, or
+        `memory` and `memory_sequence_length`.
+        `query` is the tensor of dtype matching `memory` and shape
+        `[batch_size, query_depth]`.
+        `state` is the tensor of dtype matching `memory` and shape
+        `[batch_size, alignments_size]`. (`alignments_size` is memory's
+        `max_time`).
+        `memory` is the memory to query; usually the output of an RNN encoder.
+        The tensor should be shaped `[batch_size, max_time, ...]`.
+        `memory_sequence_length` (optional) is the sequence lengths for the
+         batch entries in memory. If provided, the memory tensor rows are masked
+        with zeros for values past the respective sequence lengths.
+      mask: optional bool tensor with shape `[batch, max_time]` for the mask of
+        memory. If it is not None, the corresponding item of the memory should
+        be filtered out during calculation.
+      setup_memory: boolean, whether the input is for setting up memory, or
+        query attention.
+      **kwargs: Dict, other keyword arguments for the call method.
+    Returns:
+      Either processed memory or attention score, based on `setup_memory`.
+    """
+    if setup_memory:
+      if isinstance(inputs, list):
+        if len(inputs) not in (1, 2):
+          raise ValueError("Expect inputs to have 1 or 2 tensors, got %d" %
+                           len(inputs))
+        memory = inputs[0]
+        memory_sequence_length = inputs[1] if len(inputs) == 2 else None
+        memory_mask = mask
+      else:
+        memory, memory_sequence_length = inputs, None
+        memory_mask = mask
+      self._setup_memory(memory, memory_sequence_length, memory_mask)
+      # We force the self.built to false here since only memory is initialized,
+      # but the real query/state has not been call() yet. The layer should be
+      # build and call again.
+      self.built = False
+      # We have to return something here since layers are required to have some
+      # output. Use self.value here since it is the processed memory.
+      return self.values
+    else:
+      if not self._memory_initialized:
+        raise ValueError("Cannot query the attention before the setup of "
+                         "memory")
+      if len(inputs) not in (2, 3):
+        raise ValueError("Expect the inputs to have query, state, and optional "
+                         "processed memory, got %d items" % len(inputs))
+      # Ignore the rest of the inputs and only care about the query and state
+      query, state = inputs[0], inputs[1]
+      return self._calculate_attention(query, state)
+
+  def _setup_memory(self, memory, memory_sequence_length=None,
+                    memory_mask=None):
     """Pre-process the memory before actually query the memory.
 
     This should only be called once at the first invocation of call().
@@ -294,17 +428,30 @@ class _BaseAttentionMechanismV2(AttentionMechanism, layers.Layer):
     Args:
       memory: The memory to query; usually the output of an RNN encoder. This
         tensor should be shaped `[batch_size, max_time, ...]`.
-      memory_mask: The boolean tensor with shape `[batch_size, max_time]`. For
-        any value equal to False, the corresponding value in memory should be
-        ignored.
+      memory_sequence_length (optional): Sequence lengths for the batch entries
+        in memory. If provided, the memory tensor rows are masked with zeros for
+        values past the respective sequence lengths.
+      memory_mask: (Optional) The boolean tensor with shape `[batch_size,
+        max_time]`. For any value equal to False, the corresponding value in
+        memory should be ignored.
     """
     if self._memory_initialized:
       raise ValueError("The memory for the attention has already been setup.")
+    if memory_sequence_length is not None and memory_mask is not None:
+      raise ValueError("memory_sequence_length and memory_mask cannot be "
+                       "used at same time for attention.")
     with ops.name_scope(
         self.name, "BaseAttentionMechanismInit", nest.flatten(memory)):
       self.values = _prepare_memory(
-          memory, memory_mask=memory_mask,
+          memory,
+          memory_sequence_length=memory_sequence_length,
+          memory_mask=memory_mask,
           check_inner_dims_defined=self._check_inner_dims_defined)
+      # Mark the value as check since the memory and memory mask might not
+      # passed from __call__(), which does not have proper keras metadata.
+      # TODO(omalleyt): Remove this hack once the mask the has proper keras
+      # history.
+      base_layer_utils.mark_checked(self.values)
       if self.memory_layer is not None:
         self.keys = self.memory_layer(self.values)
       else:
@@ -315,36 +462,25 @@ class _BaseAttentionMechanismV2(AttentionMechanism, layers.Layer):
       self._alignments_size = (tensor_shape.dimension_value(self.keys.shape[1])
                                or array_ops.shape(self.keys)[1])
       if memory_mask is not None:
-        self.probability_fn = lambda score, prev: (  # pylint:disable=g-long-lambda
-            self.probability_fn(_maybe_mask_score(
-                score, self.score_mask_value, memory_mask=memory_mask), prev))
+        unwrapped_probability_fn = self.probability_fn
+        def _mask_probability_fn(score, prev):
+          return unwrapped_probability_fn(
+              _maybe_mask_score(
+                  score,
+                  memory_mask=memory_mask,
+                  memory_sequence_length=memory_sequence_length,
+                  score_mask_value=self.score_mask_value), prev)
+        self.probability_fn = _mask_probability_fn
     self._memory_initialized = True
 
-  def call(self, inputs, mask=None, **kwargs):
-    """Base method to calculate the attention score.
-
-    Args:
-      inputs: a list of tensor that contains `query`, `state`, and `memory`.
-        `query` is the tensor of dtype matching `memory` and shape
-        `[batch_size, query_depth]`.
-        `state` is the tensor of dtype matching `memory` and shape
-        `[batch_size, alignments_size]`. (`alignments_size` is memory's
-        `max_time`).
-        `memory` is the memory to query; usually the output of an RNN encoder.
-        This tensor should be shaped `[batch_size, max_time, feature]`.
-      mask: optional bool tensor with shape `[batch, max_time]` for the mask of
-        memory. If it is not None, the corresponding item of the memory should
-        be filtered out during calculation.
-      **kwargs: Dict, other keyword arguments for the call method.
-    """
-    query, state, memory, memory_mask = self._process_inputs(inputs, mask)
-    if not self._memory_initialized:
-      self._setup_memory(memory, memory_mask=memory_mask)
-    return self.calculate_attention(query, state)
-
-  def calculate_attention(self, query, state):
+  def _calculate_attention(self, query, state):
     raise NotImplementedError(
-        "calculate_attention need to be implemented by subclasses.")
+        "_calculate_attention need to be implemented by subclasses.")
+
+  def compute_mask(self, inputs, mask=None):
+    # There real input of the attention is query and state, and the memory layer
+    # mask shouldn't be pass down. Returning None for all output mask here.
+    return None, None
 
   def get_config(self):
     config = {}
@@ -361,16 +497,12 @@ class _BaseAttentionMechanismV2(AttentionMechanism, layers.Layer):
           "class_name": self.memory_layer.__class__.__name__,
           "config": self.memory_layer.get_config(),
       }
+    # memory is a required init parameter and its a tensor. It cannot be
+    # serialized to config, so we put a placeholder for it.
+    config["memory"] = None
     base_config = super(_BaseAttentionMechanismV2, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
 
-  def _process_inputs(self, inputs, mask):
-    if len(inputs) != 3:
-      raise ValueError(
-          "Expect to have 3 inputs for attention, got %d" % len(inputs))
-    query, state, memory = inputs
-    return query, state, memory, mask
-
   def _process_probability_fn(self, func_name):
     """Helper method to retrieve the probably function by string input."""
     valid_probability_fns = {
@@ -418,6 +550,46 @@ class _BaseAttentionMechanismV2(AttentionMechanism, layers.Layer):
   def alignments_size(self):
     return self._alignments_size
 
+  @property
+  def state_size(self):
+    return self._alignments_size
+
+  def initial_alignments(self, batch_size, dtype):
+    """Creates the initial alignment values for the `AttentionWrapper` class.
+
+    This is important for AttentionMechanisms that use the previous alignment
+    to calculate the alignment at the next time step (e.g. monotonic attention).
+
+    The default behavior is to return a tensor of all zeros.
+
+    Args:
+      batch_size: `int32` scalar, the batch_size.
+      dtype: The `dtype`.
+
+    Returns:
+      A `dtype` tensor shaped `[batch_size, alignments_size]`
+      (`alignments_size` is the values' `max_time`).
+    """
+    max_time = self._alignments_size
+    return _zero_state_tensors(max_time, batch_size, dtype)
+
+  def initial_state(self, batch_size, dtype):
+    """Creates the initial state values for the `AttentionWrapper` class.
+
+    This is important for AttentionMechanisms that use the previous alignment
+    to calculate the alignment at the next time step (e.g. monotonic attention).
+
+    The default behavior is to return the same output as initial_alignments.
+
+    Args:
+      batch_size: `int32` scalar, the batch_size.
+      dtype: The `dtype`.
+
+    Returns:
+      A structure of all-zero tensors with shapes as described by `state_size`.
+    """
+    return self.initial_alignments(batch_size, dtype)
+
 
 def _luong_score(query, keys, scale):
   """Implements Luong-style (multiplicative) scoring function.
@@ -587,6 +759,8 @@ class LuongAttentionV2(_BaseAttentionMechanismV2):
 
   def __init__(self,
                units,
+               memory,
+               memory_sequence_length=None,
                scale=False,
                probability_fn="softmax",
                dtype=None,
@@ -596,6 +770,11 @@ class LuongAttentionV2(_BaseAttentionMechanismV2):
 
     Args:
       units: The depth of the attention mechanism.
+      memory: The memory to query; usually the output of an RNN encoder.  This
+        tensor should be shaped `[batch_size, max_time, ...]`.
+      memory_sequence_length: (optional): Sequence lengths for the batch entries
+        in memory.  If provided, the memory tensor rows are masked with zeros
+        for values past the respective sequence lengths.
       scale: Python boolean. Whether to scale the energy term.
       probability_fn: (optional) string, the name of function to convert the
         attention score to probabilities. The default is `softmax` which is
@@ -618,26 +797,27 @@ class LuongAttentionV2(_BaseAttentionMechanismV2):
     if not memory_layer:
       memory_layer = layers.Dense(
           units, name="memory_layer", use_bias=False, dtype=dtype)
+    self.units = units
+    self.scale = scale
+    self.scale_weight = None
     super(LuongAttentionV2, self).__init__(
+        memory=memory,
+        memory_sequence_length=memory_sequence_length,
         query_layer=None,
         memory_layer=memory_layer,
         probability_fn=wrapped_probability_fn,
         name=name,
         dtype=dtype,
         **kwargs)
-    self.units = units
-    self.scale = scale
 
   def build(self, input_shape):
     super(LuongAttentionV2, self).build(input_shape)
-    if self.scale:
+    if self.scale and self.scale_weight is None:
       self.scale_weight = self.add_weight(
           "attention_g", initializer=init_ops.ones_initializer, shape=())
-    else:
-      self.scale_weight = None
     self.built = True
 
-  def calculate_attention(self, query, state):
+  def _calculate_attention(self, query, state):
     """Score the query based on the keys and values.
 
     Args:
@@ -851,8 +1031,11 @@ class BahdanauAttentionV2(_BaseAttentionMechanismV2):
 
   def __init__(self,
                units,
+               memory,
+               memory_sequence_length=None,
                normalize=False,
                probability_fn="softmax",
+               kernel_initializer="glorot_uniform",
                dtype=None,
                name="BahdanauAttention",
                **kwargs):
@@ -860,12 +1043,19 @@ class BahdanauAttentionV2(_BaseAttentionMechanismV2):
 
     Args:
       units: The depth of the query mechanism.
+      memory: The memory to query; usually the output of an RNN encoder.  This
+        tensor should be shaped `[batch_size, max_time, ...]`.
+      memory_sequence_length: (optional): Sequence lengths for the batch entries
+        in memory.  If provided, the memory tensor rows are masked with zeros
+        for values past the respective sequence lengths.
       normalize: Python boolean.  Whether to normalize the energy term.
       probability_fn: (optional) string, the name of function to convert the
         attention score to probabilities. The default is `softmax` which is
         `tf.nn.softmax`. Other options is `hardmax`, which is hardmax() within
         this module. Any other value will result into validation error. Default
         to use `softmax`.
+      kernel_initializer: (optional), the name of the initializer for the
+        attention kernel.
       dtype: The data type for the query and memory layers of the attention
         mechanism.
       name: Name to use when creating ops.
@@ -885,33 +1075,39 @@ class BahdanauAttentionV2(_BaseAttentionMechanismV2):
     if not memory_layer:
       memory_layer = layers.Dense(
           units, name="memory_layer", use_bias=False, dtype=dtype)
+    self.units = units
+    self.normalize = normalize
+    self.kernel_initializer = initializers.get(kernel_initializer)
+    self.attention_v = None
+    self.attention_g = None
+    self.attention_b = None
     super(BahdanauAttentionV2, self).__init__(
+        memory=memory,
+        memory_sequence_length=memory_sequence_length,
         query_layer=query_layer,
         memory_layer=memory_layer,
         probability_fn=wrapped_probability_fn,
         name=name,
         dtype=dtype,
         **kwargs)
-    self.units = units
-    self.normalize = normalize
 
   def build(self, input_shape):
     super(BahdanauAttentionV2, self).build(input_shape)
-    self.attention_v = self.add_weight(
-        "attention_v", [self.units], dtype=self.dtype)
-    if self.normalize:
+    if self.attention_v is None:
+      self.attention_v = self.add_weight(
+          "attention_v", [self.units],
+          dtype=self.dtype,
+          initializer=self.kernel_initializer)
+    if self.normalize and self.attention_g is None and self.attention_b is None:
       self.attention_g = self.add_weight(
           "attention_g", initializer=init_ops.constant_initializer(
               math.sqrt((1. / self.units))), shape=())
       self.attention_b = self.add_weight(
           "attention_b", shape=[self.units],
           initializer=init_ops.zeros_initializer())
-    else:
-      self.attention_g = None
-      self.attention_b = None
     self.built = True
 
-  def calculate_attention(self, query, state):
+  def _calculate_attention(self, query, state):
     """Score the query based on the keys and values.
 
     Args:
@@ -940,6 +1136,7 @@ class BahdanauAttentionV2(_BaseAttentionMechanismV2):
         "units": self.units,
         "normalize": self.normalize,
         "probability_fn": self.probability_fn_name,
+        "kernel_initializer": initializers.serialize(self.kernel_initializer)
     }
     base_config = super(BahdanauAttentionV2, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
@@ -1299,11 +1496,14 @@ class BahdanauMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
 
   def __init__(self,
                units,
+               memory,
+               memory_sequence_length=None,
                normalize=False,
                sigmoid_noise=0.,
                sigmoid_noise_seed=None,
                score_bias_init=0.,
                mode="parallel",
+               kernel_initializer="glorot_uniform",
                dtype=None,
                name="BahdanauMonotonicAttention",
                **kwargs):
@@ -1311,6 +1511,11 @@ class BahdanauMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
 
     Args:
       units: The depth of the query mechanism.
+      memory: The memory to query; usually the output of an RNN encoder.  This
+        tensor should be shaped `[batch_size, max_time, ...]`.
+      memory_sequence_length: (optional): Sequence lengths for the batch entries
+        in memory.  If provided, the memory tensor rows are masked with zeros
+        for values past the respective sequence lengths.
       normalize: Python boolean. Whether to normalize the energy term.
       sigmoid_noise: Standard deviation of pre-sigmoid noise. See the docstring
         for `_monotonic_probability_fn` for more information.
@@ -1321,6 +1526,8 @@ class BahdanauMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
       mode: How to compute the attention distribution. Must be one of
         'recursive', 'parallel', or 'hard'. See the docstring for
         `tf.contrib.seq2seq.monotonic_attention` for more information.
+      kernel_initializer: (optional), the name of the initializer for the
+        attention kernel.
       dtype: The data type for the query and memory layers of the attention
         mechanism.
       name: Name to use when creating ops.
@@ -1341,32 +1548,39 @@ class BahdanauMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
     if not memory_layer:
       memory_layer = layers.Dense(
           units, name="memory_layer", use_bias=False, dtype=dtype)
+    self.units = units
+    self.normalize = normalize
+    self.sigmoid_noise = sigmoid_noise
+    self.sigmoid_noise_seed = sigmoid_noise_seed
+    self.score_bias_init = score_bias_init
+    self.mode = mode
+    self.kernel_initializer = initializers.get(kernel_initializer)
+    self.attention_v = None
+    self.attention_score_bias = None
+    self.attention_g = None
+    self.attention_b = None
     super(BahdanauMonotonicAttentionV2, self).__init__(
+        memory=memory,
+        memory_sequence_length=memory_sequence_length,
         query_layer=query_layer,
         memory_layer=memory_layer,
         probability_fn=wrapped_probability_fn,
         name=name,
         dtype=dtype,
         **kwargs)
-    self.units = units
-    self.normalize = normalize
-    self.sigmoid_noise = sigmoid_noise
-    self.sigmoid_noise_seed = sigmoid_noise_seed
-    self.score_bias_init = score_bias_init
-    self.mode = mode
 
   def build(self, input_shape):
     super(BahdanauMonotonicAttentionV2, self).build(input_shape)
-    self.attention_v = self.add_weight(
-        "attention_v", [self.units], dtype=self.dtype)
-    self.attention_score_bias = self.add_weight(
-        "attention_score_bias", shape=(), dtype=self.dtype,
-        initializer=init_ops.constant_initializer(
-            self.score_bias_init, dtype=self.dtype))
-    if not self.normalize:
-      self.attention_g = None
-      self.attention_b = None
-    else:
+    if self.attention_v is None:
+      self.attention_v = self.add_weight(
+          "attention_v", [self.units], dtype=self.dtype,
+          initializer=self.kernel_initializer)
+    if self.attention_score_bias is None:
+      self.attention_score_bias = self.add_weight(
+          "attention_score_bias", shape=(), dtype=self.dtype,
+          initializer=init_ops.constant_initializer(
+              self.score_bias_init, dtype=self.dtype))
+    if self.normalize and self.attention_g is None and self.attention_b is None:
       self.attention_g = self.add_weight(
           "attention_g", dtype=self.dtype,
           initializer=init_ops.constant_initializer(
@@ -1377,7 +1591,7 @@ class BahdanauMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
           initializer=init_ops.zeros_initializer())
     self.built = True
 
-  def calculate_attention(self, query, state):
+  def _calculate_attention(self, query, state):
     """Score the query based on the keys and values.
 
     Args:
@@ -1409,6 +1623,7 @@ class BahdanauMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
         "sigmoid_noise_seed": self.sigmoid_noise_seed,
         "score_bias_init": self.score_bias_init,
         "mode": self.mode,
+        "kernel_initializer": initializers.serialize(self.kernel_initializer),
     }
     base_config = super(BahdanauMonotonicAttentionV2, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
@@ -1542,6 +1757,8 @@ class LuongMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
 
   def __init__(self,
                units,
+               memory,
+               memory_sequence_length=None,
                scale=False,
                sigmoid_noise=0.,
                sigmoid_noise_seed=None,
@@ -1554,6 +1771,11 @@ class LuongMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
 
     Args:
       units: The depth of the query mechanism.
+      memory: The memory to query; usually the output of an RNN encoder.  This
+        tensor should be shaped `[batch_size, max_time, ...]`.
+      memory_sequence_length: (optional): Sequence lengths for the batch entries
+        in memory.  If provided, the memory tensor rows are masked with zeros
+        for values past the respective sequence lengths.
       scale: Python boolean.  Whether to scale the energy term.
       sigmoid_noise: Standard deviation of pre-sigmoid noise.  See the docstring
         for `_monotonic_probability_fn` for more information.
@@ -1580,34 +1802,37 @@ class LuongMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
     if not memory_layer:
       memory_layer = layers.Dense(
           units, name="memory_layer", use_bias=False, dtype=dtype)
+    self.units = units
+    self.scale = scale
+    self.sigmoid_noise = sigmoid_noise
+    self.sigmoid_noise_seed = sigmoid_noise_seed
+    self.score_bias_init = score_bias_init
+    self.mode = mode
+    self.attention_g = None
+    self.attention_score_bias = None
     super(LuongMonotonicAttentionV2, self).__init__(
+        memory=memory,
+        memory_sequence_length=memory_sequence_length,
         query_layer=None,
         memory_layer=memory_layer,
         probability_fn=wrapped_probability_fn,
         name=name,
         dtype=dtype,
         **kwargs)
-    self.units = units
-    self.scale = scale
-    self.sigmoid_noise = sigmoid_noise
-    self.sigmoid_noise_seed = sigmoid_noise_seed
-    self.score_bias_init = score_bias_init
-    self.mode = mode
 
   def build(self, input_shape):
     super(LuongMonotonicAttentionV2, self).build(input_shape)
-    if self.scale:
+    if self.scale and self.attention_g is None:
       self.attention_g = self.add_weight(
           "attention_g", initializer=init_ops.ones_initializer, shape=())
-    else:
-      self.attention_g = None
-    self.attention_score_bias = self.add_weight(
-        "attention_score_bias", shape=(),
-        initializer=init_ops.constant_initializer(
-            self.score_bias_init, dtype=self.dtype))
+    if self.attention_score_bias is None:
+      self.attention_score_bias = self.add_weight(
+          "attention_score_bias", shape=(),
+          initializer=init_ops.constant_initializer(
+              self.score_bias_init, dtype=self.dtype))
     self.built = True
 
-  def calculate_attention(self, query, state):
+  def _calculate_attention(self, query, state):
     """Score the query based on the keys and values.
 
     Args:
@@ -1739,41 +1964,26 @@ def _prepare_memory(memory, memory_sequence_length=None, memory_mask=None,
                          "but saw shape: %s" % (m.name, m.get_shape()))
     nest.map_structure(_check_dims, memory)
   if memory_sequence_length is None and memory_mask is None:
-    seq_len_mask = None
-    seq_len_batch_size = None
+    return memory
   elif memory_sequence_length is not None:
     seq_len_mask = array_ops.sequence_mask(
         memory_sequence_length,
         maxlen=array_ops.shape(nest.flatten(memory)[0])[1],
         dtype=nest.flatten(memory)[0].dtype)
-    seq_len_batch_size = (
-        tensor_shape.dimension_value(memory_sequence_length.shape[0])
-        or array_ops.shape(memory_sequence_length)[0])
   else:
     # For memory_mask is not None
-    seq_len_mask = memory_mask
-    seq_len_batch_size = (
-        tensor_shape.dimension_value(memory_mask.shape[0])
-        or array_ops.shape(memory_mask)[0])
+    seq_len_mask = math_ops.cast(
+        memory_mask, dtype=nest.flatten(memory)[0].dtype)
   def _maybe_mask(m, seq_len_mask):
     """Mask the memory based on the memory mask."""
     rank = m.get_shape().ndims
     rank = rank if rank is not None else array_ops.rank(m)
     extra_ones = array_ops.ones(rank - 2, dtype=dtypes.int32)
-    m_batch_size = tensor_shape.dimension_value(
-        m.shape[0]) or array_ops.shape(m)[0]
-    if seq_len_batch_size is not None:
-      message = ("memory_sequence_length and memory tensor batch sizes do not "
-                 "match.")
-      with ops.control_dependencies([
-          check_ops.assert_equal(
-              seq_len_batch_size, m_batch_size, message=message)]):
-        seq_len_mask = array_ops.reshape(
-            seq_len_mask,
-            array_ops.concat((array_ops.shape(seq_len_mask), extra_ones), 0))
-        return m * seq_len_mask
-    else:
-      return m
+    seq_len_mask = array_ops.reshape(
+        seq_len_mask,
+        array_ops.concat((array_ops.shape(seq_len_mask), extra_ones), 0))
+    return m * seq_len_mask
+
   return nest.map_structure(lambda m: _maybe_mask(m, seq_len_mask), memory)
 
 
@@ -1819,8 +2029,13 @@ def hardmax(logits, name=None):
 def _compute_attention(attention_mechanism, cell_output, attention_state,
                        attention_layer):
   """Computes the attention and alignments for a given attention_mechanism."""
-  alignments, next_attention_state = attention_mechanism(
-      cell_output, state=attention_state)
+  if isinstance(attention_mechanism, _BaseAttentionMechanism):
+    alignments, next_attention_state = attention_mechanism(
+        cell_output, state=attention_state)
+  else:
+    # For AttentionMechanismV2, which takes inputs as a list.
+    alignments, next_attention_state = attention_mechanism(
+        [cell_output, attention_state])
 
   # Reshape from [batch_size, memory_time] to [batch_size, 1, memory_time]
   expanded_alignments = array_ops.expand_dims(alignments, 1)
-- 
GitLab


From a8e6f8e65efaff7d7e357e1b6aadaf49d5c7286d Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Fri, 8 Feb 2019 18:27:34 -0800
Subject: [PATCH 2221/2345] Automated rollback of commit
 046f1b74a0c229580aa6c818b2a8610e5e0ec22c

PiperOrigin-RevId: 233161483
---
 .../grappler/optimizers/function_optimizer.cc | 24 +++++++++++++++----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index ac0831b105..b722e6b332 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -1398,10 +1398,14 @@ Status IsInlinableIndirectFunctionCall(const FunctionOptimizerContext& ctx,
 // Checks that all side-effects will be executed in well defined order. We do it
 // by checking if there is a path from stateful/dataset ops to one of the output
 // nodes.
-void CheckThatSideEffectsWillExecute(
+Status CheckThatSideEffectsWillExecute(
     const FunctionOptimizerContext& ctx,
     const GraphTopologyView& graph_topo_view,
     const absl::flat_hash_set<string> output_nodes) {
+  // In aggressive mode we just print a warning for side-effectful nodes that
+  // might not be executed after inlining.
+  const bool aggressive = ctx.opt_level() == RewriterConfig::AGGRESSIVE;
+
   for (const NodeDef& func_body_node : graph_topo_view.graph()->node()) {
     const bool node_must_execute =
         IsDataset(func_body_node) ||
@@ -1434,11 +1438,20 @@ void CheckThatSideEffectsWillExecute(
                  TraversalDirection::kFollowOutputs, predicates, callbacks);
 
     if (!will_execute) {
-      VLOG(1) << "Can't guarantee execution of a side-effectful node, that is "
-                 "not reachable from function outputs. Function body node: "
-              << SummarizeNodeDef(func_body_node);
+      const string error_message = absl::StrCat(
+          "Can't guarantee execution of a side-effectful node, that is not "
+          "reachable from function outputs. Function body node: ",
+          SummarizeNodeDef(func_body_node));
+
+      if (aggressive) {
+        LOG(WARNING) << error_message;
+      } else {
+        return errors::Internal(error_message);
+      }
     }
   }
+
+  return Status::OK();
 }
 
 Status PlaceInlinedFunctionBody(
@@ -1790,7 +1803,8 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
   // input nodes connected to nodes in the optimized graph).
   GraphTopologyView placed_topo_view(/*skip_invalid_edges=*/true);
   TF_RETURN_IF_ERROR(placed_topo_view.InitializeFromGraph(placed_graph_def));
-  CheckThatSideEffectsWillExecute(*ctx, placed_topo_view, inlined_output_nodes);
+  TF_RETURN_IF_ERROR(CheckThatSideEffectsWillExecute(*ctx, placed_topo_view,
+                                                     inlined_output_nodes));
 
   // ------------------------------------------------------------------------ //
   // Move all the nodes to the optimized graph after successful preprocessing.
-- 
GitLab


From 9b41335f9d131ad97a41aec290aa788e403c178d Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Fri, 8 Feb 2019 18:48:37 -0800
Subject: [PATCH 2222/2345] Update Eigen to commit 5a4931dafc1c.

PiperOrigin-RevId: 233163061
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 081ebb9738..bbdabd7d47 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -141,11 +141,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
         name = "eigen_archive",
         build_file = clean_dep("//third_party:eigen.BUILD"),
         patch_file = clean_dep("//third_party/eigen3:gpu_packet_math.patch"),
-        sha256 = "c1f026562efcbca7c338bfa38751470d13dd3fd1f1925ca4fafe42603d0e7693",
-        strip_prefix = "eigen-eigen-f5ef1f33d885",
+        sha256 = "61f0017318a24cf940db14e57febecc524b24a9faa8ff4fa7f9f91630c4cd09d",
+        strip_prefix = "eigen-eigen-5a4931dafc1c",
         urls = [
-            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/f5ef1f33d885.tar.gz",
-            "https://bitbucket.org/eigen/eigen/get/f5ef1f33d885.tar.gz",
+            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/5a4931dafc1c.tar.gz",
+            "https://bitbucket.org/eigen/eigen/get/5a4931dafc1c.tar.gz",
         ],
     )
 
-- 
GitLab


From 57b954d6b9f979c6ef3bb527d8e0e881799adf64 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 8 Feb 2019 21:35:24 -0800
Subject: [PATCH 2223/2345] Upgrade gRPC version used in OSS Tensorflow

PiperOrigin-RevId: 233174188
---
 tensorflow/contrib/cmake/external/grpc.cmake | 2 +-
 tensorflow/workspace.bzl                     | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/cmake/external/grpc.cmake b/tensorflow/contrib/cmake/external/grpc.cmake
index dd8cca809e..22f1170f39 100644
--- a/tensorflow/contrib/cmake/external/grpc.cmake
+++ b/tensorflow/contrib/cmake/external/grpc.cmake
@@ -17,7 +17,7 @@ include (ExternalProject)
 set(GRPC_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/include)
 set(GRPC_URL https://github.com/grpc/grpc.git)
 set(GRPC_BUILD ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc)
-set(GRPC_TAG 1bd32fb4d582e3d182786b10d63dd048ba11b678)
+set(GRPC_TAG c61d3e8308df248a13e94e9eebc674d6481e9ca4)
 
 if(WIN32)
   # We use unsecure gRPC because boringssl does not build on windows
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index bbdabd7d47..bf25d2e1b7 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -478,12 +478,12 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     # WARNING: make sure ncteisen@ and vpai@ are cc-ed on any CL to change the below rule
     tf_http_archive(
         name = "grpc",
-        sha256 = "1b7178e42f6f766cf56425fab4f1ba5ddf668b209a997ba62ffca232dc43688c",
-        strip_prefix = "grpc-1bd32fb4d582e3d182786b10d63dd048ba11b678",
+        sha256 = "c1960c2d96914431bcaec97179f025868e6d641b7ae9b13d2cf94b161139bd67",
+        strip_prefix = "grpc-c61d3e8308df248a13e94e9eebc674d6481e9ca4",
         system_build_file = clean_dep("//third_party/systemlibs:grpc.BUILD"),
         urls = [
-            "https://mirror.bazel.build/github.com/grpc/grpc/archive/1bd32fb4d582e3d182786b10d63dd048ba11b678.tar.gz",
-            "https://github.com/grpc/grpc/archive/1bd32fb4d582e3d182786b10d63dd048ba11b678.tar.gz",
+            "https://mirror.bazel.build/github.com/grpc/grpc/archive/c61d3e8308df248a13e94e9eebc674d6481e9ca4.tar.gz",
+            "https://github.com/grpc/grpc/archive/c61d3e8308df248a13e94e9eebc674d6481e9ca4.tar.gz",
         ],
     )
 
-- 
GitLab


From ac4ffac6251983404b8161e02eedcf9f480bfbef Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Fri, 8 Feb 2019 23:22:24 -0800
Subject: [PATCH 2224/2345] Fox for tff mnist test failure. This fixes the test
 case but does not solve the underlying issue of using the incorrect graph.

PiperOrigin-RevId: 233180048
---
 tensorflow/python/keras/engine/training.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index b81713b2b2..a7e5750361 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -1971,11 +1971,12 @@ class Model(Network):
               self._per_output_weighted_metrics[i], i))
 
     # Create a metric wrapper for each output loss.
-    self._output_loss_metrics = [
-        metrics_module.SumOverBatchSize() if hasattr(loss_fn, 'reduction') else
-        metrics_module.SumOverBatchSizeMetricWrapper(loss_fn)
-        for loss_fn in self.loss_functions
-    ]
+    if len(self.outputs) > 1:
+      self._output_loss_metrics = [
+          metrics_module.SumOverBatchSize() if hasattr(loss_fn, 'reduction')
+          else metrics_module.SumOverBatchSizeMetricWrapper(loss_fn)
+          for loss_fn in self.loss_functions
+      ]
 
     self._per_output_metrics = updated_per_output_metrics
     self._per_output_weighted_metrics = updated_per_output_weighted_metrics
-- 
GitLab


From d3fb42e1c3b5eb3c10b6c8b8e58666b1fb6bef30 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Fri, 8 Feb 2019 23:40:48 -0800
Subject: [PATCH 2225/2345] Automated rollback of commit
 65be9a9c06e1fbd7c55f1b7bc468881a6b70a368

PiperOrigin-RevId: 233181864
---
 .../kernel_tests/attention_wrapper_test.py    |  20 +-
 .../kernel_tests/attention_wrapper_v2_test.py | 679 +-----------------
 .../seq2seq/python/ops/attention_wrapper.py   | 459 ++++--------
 3 files changed, 146 insertions(+), 1012 deletions(-)

diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py
index 1a5692f7b5..d815f81f84 100644
--- a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py
+++ b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py
@@ -358,7 +358,7 @@ class AttentionWrapperTest(test.TestCase):
         rnn_output=ResultSummary(
             shape=(5, 3, 6), dtype=dtype('float32'), mean=-0.00597103),
         sample_id=ResultSummary(
-            shape=(5, 3), dtype=dtype('int32'), mean=1.4))
+            shape=(5, 3), dtype=dtype('int32'), mean=1.6))
     expected_final_state = AttentionWrapperState(
         cell_state=LSTMStateTuple(
             c=ResultSummary(
@@ -387,7 +387,7 @@ class AttentionWrapperTest(test.TestCase):
         rnn_output=ResultSummary(
             shape=(5, 3, 6), dtype=dtype('float32'), mean=-0.0052615386),
         sample_id=ResultSummary(
-            shape=(5, 3), dtype=dtype('int32'), mean=1.4))
+            shape=(5, 3), dtype=dtype('int32'), mean=1.3333333333))
     expected_final_state = AttentionWrapperState(
         cell_state=LSTMStateTuple(
             c=ResultSummary(
@@ -454,7 +454,7 @@ class AttentionWrapperTest(test.TestCase):
         rnn_output=ResultSummary(
             shape=(5, 3, 6), dtype=dtype('float32'), mean=-0.0052615386),
         sample_id=ResultSummary(
-            shape=(5, 3), dtype=dtype('int32'), mean=1.4))
+            shape=(5, 3), dtype=dtype('int32'), mean=1.3333333333333333))
     expected_final_state = AttentionWrapperState(
         cell_state=LSTMStateTuple(
             c=ResultSummary(
@@ -696,7 +696,7 @@ class AttentionWrapperTest(test.TestCase):
         rnn_output=ResultSummary(
             shape=(5, 3, 6), dtype=dtype('float32'), mean=-0.0025896581),
         sample_id=ResultSummary(
-            shape=(5, 3), dtype=dtype('int32'), mean=1.73333333))
+            shape=(5, 3), dtype=dtype('int32'), mean=1.6))
     expected_final_state = AttentionWrapperState(
         cell_state=LSTMStateTuple(
             c=ResultSummary(
@@ -707,12 +707,12 @@ class AttentionWrapperTest(test.TestCase):
             shape=(5, 6), dtype=dtype('float32'), mean=-0.00069823361),
         time=3,
         alignments=ResultSummary(
-            shape=(5, 8), dtype=dtype('float32'), mean=0.029914695),
+            shape=(5, 8), dtype=dtype('float32'), mean=0.028698336),
         attention_state=ResultSummary(
-            shape=(5, 8), dtype=dtype('float32'), mean=0.029914695),
+            shape=(5, 8), dtype=dtype('float32'), mean=0.028698336),
         alignment_history=())
     expected_final_alignment_history = ResultSummary(
-        shape=(3, 5, 8), dtype=dtype('float32'), mean=0.0465225502849)
+        shape=(3, 5, 8), dtype=dtype('float32'), mean=0.04865776002407074)
 
     self._testWithAttention(
         create_attention_mechanism,
@@ -921,9 +921,9 @@ class AttentionWrapperTest(test.TestCase):
 
     expected_final_output = BasicDecoderOutput(
         rnn_output=ResultSummary(
-            shape=(5, 3, 20), dtype=dtype('float32'), mean=0.115853324533),
+            shape=(5, 3, 20), dtype=dtype('float32'), mean=0.11723966),
         sample_id=ResultSummary(
-            shape=(5, 3), dtype=dtype('int32'), mean=8.6))
+            shape=(5, 3), dtype=dtype('int32'), mean=7.266666666666667))
     expected_final_state = AttentionWrapperState(
         cell_state=LSTMStateTuple(
             c=ResultSummary(
@@ -931,7 +931,7 @@ class AttentionWrapperTest(test.TestCase):
             h=ResultSummary(
                 shape=(5, 9), dtype=dtype('float32'), mean=-0.0018327223)),
         attention=ResultSummary(
-            shape=(5, 20), dtype=dtype('float32'), mean=0.11462739855),
+            shape=(5, 20), dtype=dtype('float32'), mean=0.11601614207),
         time=3,
         alignments=(ResultSummary(
             shape=(5, 8), dtype=dtype('float32'), mean=0.125),
diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_v2_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_v2_test.py
index 5ee01f66f1..7ff04e1780 100644
--- a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_v2_test.py
+++ b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_v2_test.py
@@ -17,23 +17,14 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import collections
-
 from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.contrib.seq2seq.python.ops import attention_wrapper as wrapper
-from tensorflow.contrib.seq2seq.python.ops import basic_decoder
-from tensorflow.contrib.seq2seq.python.ops import sampler as sampler_py
-from tensorflow.python import keras
-from tensorflow.python.eager import context
-from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
-from tensorflow.python.keras import initializers
-from tensorflow.python.ops import rnn_cell
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
-from tensorflow.python.util import nest
 
 
 @test_util.run_all_in_graph_and_eager_modes
@@ -46,10 +37,13 @@ class AttentionMechanismTest(test.TestCase, parameterized.TestCase):
     self.memory_size = 6
     self.units = 8
 
-    self.memory = np.random.randn(self.batch, self.timestep,
-                                  self.memory_size).astype(np.float32)
-    self.query = np.random.randn(self.batch, self.units).astype(np.float32)
-    self.state = np.random.randn(self.batch, self.timestep).astype(np.float32)
+    self.memory = ops.convert_to_tensor(
+        np.random.random((self.batch, self.timestep, self.memory_size)),
+        dtype=np.float32)
+    self.query = ops.convert_to_tensor(
+        np.random.random((self.batch, self.units)), dtype=np.float32)
+    self.state = ops.convert_to_tensor(
+        np.random.random((self.batch, self.timestep)), dtype=np.float32)
 
   @parameterized.named_parameters(
       ("luong", wrapper.LuongAttentionV2),
@@ -58,8 +52,8 @@ class AttentionMechanismTest(test.TestCase, parameterized.TestCase):
       ("bahdanau_monotonic", wrapper.BahdanauMonotonicAttentionV2),
   )
   def test_attention_shape_inference(self, attention_cls):
-    attention = attention_cls(self.units, self.memory)
-    attention_score = attention([self.query, self.state])
+    attention = attention_cls(self.units)
+    attention_score = attention([self.query, self.state, self.memory])
     self.assertLen(attention_score, 2)
     self.assertEqual(attention_score[0].shape, (self.batch, self.timestep))
     self.assertEqual(attention_score[1].shape, (self.batch, self.timestep))
@@ -71,7 +65,7 @@ class AttentionMechanismTest(test.TestCase, parameterized.TestCase):
       ("bahdanau_monotonic", wrapper.BahdanauMonotonicAttentionV2),
   )
   def test_get_config(self, attention_cls):
-    attention = attention_cls(self.units, self.memory)
+    attention = attention_cls(self.units)
     config = attention.get_config()
 
     attention_from_config = attention_cls.from_config(config)
@@ -86,8 +80,9 @@ class AttentionMechanismTest(test.TestCase, parameterized.TestCase):
       ("bahdanau_monotonic", wrapper.BahdanauMonotonicAttentionV2),
   )
   def test_layer_output(self, attention_cls):
-    attention = attention_cls(self.units, self.memory)
-    score = attention([self.query, self.state])
+    attention = attention_cls(self.units)
+
+    score = attention([self.query, self.state, self.memory])
     self.evaluate(variables.variables_initializer(attention.variables))
 
     score_val = self.evaluate(score)
@@ -95,651 +90,5 @@ class AttentionMechanismTest(test.TestCase, parameterized.TestCase):
     self.assertEqual(score_val[0].shape, (self.batch, self.timestep))
     self.assertEqual(score_val[1].shape, (self.batch, self.timestep))
 
-  @parameterized.named_parameters(
-      ("luong", wrapper.LuongAttentionV2),
-      ("luong_monotonic", wrapper.LuongMonotonicAttentionV2),
-      ("bahdanau", wrapper.BahdanauAttentionV2),
-      ("bahdanau_monotonic", wrapper.BahdanauMonotonicAttentionV2),
-  )
-  def test_passing_memory_from_call(self, attention_cls):
-    attention = attention_cls(self.units, self.memory)
-    weights_before_query = attention.get_weights()
-    ref_score = attention([self.query, self.state])
-
-    self.evaluate(variables.global_variables_initializer())
-    ref_score_val = self.evaluate(ref_score)
-
-    all_weights = attention.get_weights()
-    config = attention.get_config()
-    # Simulate the twice invocation of calls here.
-    attention_from_config = attention_cls.from_config(config)
-    attention_from_config.build(self.memory.shape)
-    attention_from_config.set_weights(weights_before_query)
-    attention_from_config(self.memory, setup_memory=True)
-    attention_from_config.build([self.query.shape, self.state.shape])
-    attention_from_config.set_weights(all_weights)
-    score = attention_from_config([self.query, self.state])
-
-    score_val = self.evaluate(score)
-    self.assertAllClose(ref_score_val, score_val)
-
-  @parameterized.named_parameters(
-      ("luong", wrapper.LuongAttentionV2),
-      ("luong_monotonic", wrapper.LuongMonotonicAttentionV2),
-      ("bahdanau", wrapper.BahdanauAttentionV2),
-      ("bahdanau_monotonic", wrapper.BahdanauMonotonicAttentionV2),
-  )
-  def test_save_load_layer(self, attention_cls):
-    vocab = 20
-    embedding_dim = 6
-    inputs = keras.layers.Input(shape=[self.timestep])
-    encoder_input = keras.layers.Embedding(
-        vocab, embedding_dim, mask_zero=True)(
-            inputs)
-    encoder_output = keras.layers.UnifiedLSTM(
-        self.memory_size, return_sequences=True)(
-            encoder_input)
-
-    attention = attention_cls(self.units, encoder_output)
-    query = keras.layers.Input(shape=[self.units])
-    state = keras.layers.Input(shape=[self.timestep])
-
-    score = attention([query, state])
-
-    x = np.random.randint(vocab, size=(self.batch, self.timestep))
-    x_test = np.random.randint(vocab, size=(self.batch, self.timestep))
-    y = np.random.randn(self.batch, self.timestep)
-    model = keras.models.Model([inputs, query, state], score)
-    model.compile("rmsprop", "mse")
-    model.fit([x, self.query, self.state], (y, y))
-    y_ref = model.predict_on_batch([x_test, self.query, self.state])
-
-    config = model.get_config()
-    weights = model.get_weights()
-    loaded_model = keras.models.Model.from_config(
-        config, custom_objects={attention_cls.__name__: attention_cls})
-    loaded_model.set_weights(weights)
-
-    y = loaded_model.predict_on_batch([x_test, self.query, self.state])
-
-    self.assertAllClose(y_ref, y)
-
-  # TODO(scottzhu): Add tests for model.compile(run_eagerly=True)
-
-
-class ResultSummary(
-    collections.namedtuple("ResultSummary", ("shape", "dtype", "mean"))):
-  pass
-
-
-def get_result_summary(x):
-  if isinstance(x, np.ndarray):
-    return ResultSummary(x.shape, x.dtype, x.mean())
-  return x
-
-
-@test_util.run_all_in_graph_and_eager_modes
-class AttentionWrapperV2Test(test.TestCase, parameterized.TestCase):
-
-  def assertAllCloseOrEqual(self, x, y, **kwargs):
-    if isinstance(x, np.ndarray) or isinstance(x, float):
-      return super(AttentionWrapperV2Test, self).assertAllClose(
-          x, y, atol=1e-3, **kwargs)
-    else:
-      self.assertAllEqual(x, y, **kwargs)
-
-  def setUp(self):
-    super(AttentionWrapperV2Test, self).setUp()
-    self.batch = 64
-    self.units = 128
-    self.encoder_timestep = 10
-    self.encoder_dim = 256
-    self.decoder_timestep = 12
-    self.encoder_outputs = np.random.randn(self.batch, self.encoder_timestep,
-                                           self.encoder_dim)
-    self.encoder_sequence_length = np.random.randint(
-        self.encoder_timestep, size=(self.batch,)).astype(np.int32)
-    self.decoder_inputs = np.random.randn(self.batch, self.decoder_timestep,
-                                          self.units)
-    self.decoder_sequence_length = np.random.randint(
-        self.decoder_timestep, size=(self.batch,)).astype(np.int32)
-
-  def _testWithAttention(self,
-                         create_attention_mechanism,
-                         expected_final_output,
-                         expected_final_state,
-                         attention_mechanism_depth=3,
-                         alignment_history=False,
-                         expected_final_alignment_history=None,
-                         attention_layer_size=6,
-                         attention_layer=None,
-                         create_query_layer=False,
-                         create_memory_layer=True,
-                         create_attention_kwargs=None):
-    attention_layer_sizes = ([attention_layer_size]
-                             if attention_layer_size is not None else None)
-    attention_layers = ([attention_layer]
-                        if attention_layer is not None else None)
-    self._testWithMaybeMultiAttention(
-        is_multi=False,
-        create_attention_mechanisms=[create_attention_mechanism],
-        expected_final_output=expected_final_output,
-        expected_final_state=expected_final_state,
-        attention_mechanism_depths=[attention_mechanism_depth],
-        alignment_history=alignment_history,
-        expected_final_alignment_history=expected_final_alignment_history,
-        attention_layer_sizes=attention_layer_sizes,
-        attention_layers=attention_layers,
-        create_query_layer=create_query_layer,
-        create_memory_layer=create_memory_layer,
-        create_attention_kwargs=create_attention_kwargs)
-
-  def _testWithMaybeMultiAttention(self,
-                                   is_multi,
-                                   create_attention_mechanisms,
-                                   expected_final_output,
-                                   expected_final_state,
-                                   attention_mechanism_depths,
-                                   alignment_history=False,
-                                   expected_final_alignment_history=None,
-                                   attention_layer_sizes=None,
-                                   attention_layers=None,
-                                   create_query_layer=False,
-                                   create_memory_layer=True,
-                                   create_attention_kwargs=None):
-    # Allow is_multi to be True with a single mechanism to enable test for
-    # passing in a single mechanism in a list.
-    assert len(create_attention_mechanisms) == 1 or is_multi
-    encoder_sequence_length = [3, 2, 3, 1, 1]
-    decoder_sequence_length = [2, 0, 1, 2, 3]
-    batch_size = 5
-    encoder_max_time = 8
-    decoder_max_time = 4
-    input_depth = 7
-    encoder_output_depth = 10
-    cell_depth = 9
-    create_attention_kwargs = create_attention_kwargs or {}
-
-    if attention_layer_sizes is not None:
-      # Compute sum of attention_layer_sizes. Use encoder_output_depth if None.
-      attention_depth = sum(attention_layer_size or encoder_output_depth
-                            for attention_layer_size in attention_layer_sizes)
-    elif attention_layers is not None:
-      # Compute sum of attention_layers output depth.
-      attention_depth = sum(
-          attention_layer.compute_output_shape(
-              [batch_size, cell_depth + encoder_output_depth]).dims[-1].value
-          for attention_layer in attention_layers)
-    else:
-      attention_depth = encoder_output_depth * len(create_attention_mechanisms)
-
-    decoder_inputs = np.random.randn(batch_size, decoder_max_time,
-                                     input_depth).astype(np.float32)
-    encoder_outputs = np.random.randn(batch_size, encoder_max_time,
-                                      encoder_output_depth).astype(np.float32)
-
-    attention_mechanisms = []
-    for creator, depth in zip(create_attention_mechanisms,
-                              attention_mechanism_depths):
-      # Create a memory layer with deterministic initializer to avoid randomness
-      # in the test between graph and eager.
-      if create_query_layer:
-        create_attention_kwargs["query_layer"] = keras.layers.Dense(
-            depth, kernel_initializer="ones", use_bias=False)
-      if create_memory_layer:
-        create_attention_kwargs["memory_layer"] = keras.layers.Dense(
-            depth, kernel_initializer="ones", use_bias=False)
-
-      attention_mechanisms.append(
-          creator(
-              units=depth,
-              memory=encoder_outputs,
-              memory_sequence_length=encoder_sequence_length,
-              **create_attention_kwargs))
-
-    with self.cached_session(use_gpu=True):
-      attention_layer_size = attention_layer_sizes
-      attention_layer = attention_layers
-      if not is_multi:
-        if attention_layer_size is not None:
-          attention_layer_size = attention_layer_size[0]
-        if attention_layer is not None:
-          attention_layer = attention_layer[0]
-      cell = rnn_cell.LSTMCell(cell_depth, initializer="ones")
-      cell = wrapper.AttentionWrapper(
-          cell,
-          attention_mechanisms if is_multi else attention_mechanisms[0],
-          attention_layer_size=attention_layer_size,
-          alignment_history=alignment_history,
-          attention_layer=attention_layer)
-      # Set the attention_layer within AttentionWrapper to have deterministic
-      # kernel initializer, for testing purpose.
-      if cell._attention_layers is not None:
-        for layer in cell._attention_layers:
-          if getattr(layer, "kernel_initializer") is None:
-            layer.kernel_initializer = initializers.ones()
-
-      sampler = sampler_py.TrainingSampler()
-      my_decoder = basic_decoder.BasicDecoderV2(cell=cell, sampler=sampler)
-      initial_state = cell.zero_state(
-          dtype=dtypes.float32, batch_size=batch_size)
-      final_outputs, final_state, _ = my_decoder(
-          decoder_inputs,
-          initial_state=initial_state,
-          sequence_length=decoder_sequence_length)
-
-      self.assertIsInstance(final_outputs, basic_decoder.BasicDecoderOutput)
-      self.assertIsInstance(final_state, wrapper.AttentionWrapperState)
-      self.assertIsInstance(final_state.cell_state, rnn_cell.LSTMStateTuple)
-
-      expected_time = (
-          expected_final_state.time if context.executing_eagerly() else None)
-      self.assertEqual((batch_size, expected_time, attention_depth),
-                       tuple(final_outputs.rnn_output.get_shape().as_list()))
-      self.assertEqual((batch_size, expected_time),
-                       tuple(final_outputs.sample_id.get_shape().as_list()))
-
-      self.assertEqual((batch_size, attention_depth),
-                       tuple(final_state.attention.get_shape().as_list()))
-      self.assertEqual((batch_size, cell_depth),
-                       tuple(final_state.cell_state.c.get_shape().as_list()))
-      self.assertEqual((batch_size, cell_depth),
-                       tuple(final_state.cell_state.h.get_shape().as_list()))
-
-      if alignment_history:
-        if is_multi:
-          state_alignment_history = []
-          for history_array in final_state.alignment_history:
-            history = history_array.stack()
-            self.assertEqual((expected_time, batch_size, encoder_max_time),
-                             tuple(history.get_shape().as_list()))
-            state_alignment_history.append(history)
-          state_alignment_history = tuple(state_alignment_history)
-        else:
-          state_alignment_history = final_state.alignment_history.stack()
-          self.assertEqual((expected_time, batch_size, encoder_max_time),
-                           tuple(state_alignment_history.get_shape().as_list()))
-        nest.assert_same_structure(cell.state_size,
-                                   cell.zero_state(batch_size, dtypes.float32))
-        # Remove the history from final_state for purposes of the
-        # remainder of the tests.
-        final_state = final_state._replace(alignment_history=())  # pylint: disable=protected-access
-      else:
-        state_alignment_history = ()
-
-      self.evaluate(variables.global_variables_initializer())
-      eval_result = self.evaluate({
-          "final_outputs": final_outputs,
-          "final_state": final_state,
-          "state_alignment_history": state_alignment_history,
-      })
-
-      final_output_info = nest.map_structure(get_result_summary,
-                                             eval_result["final_outputs"])
-      final_state_info = nest.map_structure(get_result_summary,
-                                            eval_result["final_state"])
-      print("final_output_info: ", final_output_info)
-      print("final_state_info: ", final_state_info)
-
-      nest.map_structure(self.assertAllCloseOrEqual, expected_final_output,
-                         final_output_info)
-      nest.map_structure(self.assertAllCloseOrEqual, expected_final_state,
-                         final_state_info)
-      if alignment_history:  # by default, the wrapper emits attention as output
-        final_alignment_history_info = nest.map_structure(
-            get_result_summary, eval_result["state_alignment_history"])
-        print("final_alignment_history_info: ", final_alignment_history_info)
-        nest.map_structure(
-            self.assertAllCloseOrEqual,
-            # outputs are batch major but the stacked TensorArray is time major
-            expected_final_alignment_history,
-            final_alignment_history_info)
-
-  @parameterized.parameters([np.float16, np.float32, np.float64])
-  def _testBahdanauNormalizedDType(self, dtype):
-    encoder_outputs = self.encoder_outputs.astype(dtype)
-    decoder_inputs = self.decoder_inputs.astype(dtype)
-    attention_mechanism = wrapper.BahdanauAttentionV2(
-        units=self.units,
-        memory=encoder_outputs,
-        memory_sequence_length=self.encoder_sequence_length,
-        normalize=True,
-        dtype=dtype)
-    cell = rnn_cell.LSTMCell(self.units)
-    cell = wrapper.AttentionWrapper(cell, attention_mechanism)
-
-    sampler = sampler_py.TrainingSampler()
-    my_decoder = basic_decoder.BasicDecoderV2(cell=cell, sampler=sampler)
-
-    final_outputs, final_state, _ = my_decoder(
-        decoder_inputs,
-        initial_state=cell.zero_state(dtype=dtype, batch_size=self.batch),
-        sequence_length=self.decoder_sequence_length)
-    self.assertIsInstance(final_outputs, basic_decoder.BasicDecoderOutput)
-    self.assertEqual(final_outputs.rnn_output.dtype, dtype)
-    self.assertIsInstance(final_state, wrapper.AttentionWrapperState)
-    self.assertIsInstance(final_state.cell_state, rnn_cell.LSTMStateTuple)
-
-  @parameterized.parameters([np.float16, np.float32, np.float64])
-  def testLuongScaledDType(self, dtype):
-    # Test case for GitHub issue 18099
-    encoder_outputs = self.encoder_outputs.astype(dtype)
-    decoder_inputs = self.decoder_inputs.astype(dtype)
-    attention_mechanism = wrapper.LuongAttentionV2(
-        units=self.units,
-        memory=encoder_outputs,
-        memory_sequence_length=self.encoder_sequence_length,
-        scale=True,
-        dtype=dtype,
-    )
-    cell = rnn_cell.LSTMCell(self.units)
-    cell = wrapper.AttentionWrapper(cell, attention_mechanism)
-
-    sampler = sampler_py.TrainingSampler()
-    my_decoder = basic_decoder.BasicDecoderV2(cell=cell, sampler=sampler)
-
-    final_outputs, final_state, _ = my_decoder(
-        decoder_inputs,
-        initial_state=cell.zero_state(dtype=dtype, batch_size=self.batch),
-        sequence_length=self.decoder_sequence_length)
-    self.assertIsInstance(final_outputs, basic_decoder.BasicDecoderOutput)
-    self.assertEqual(final_outputs.rnn_output.dtype, dtype)
-    self.assertIsInstance(final_state, wrapper.AttentionWrapperState)
-    self.assertIsInstance(final_state.cell_state, rnn_cell.LSTMStateTuple)
-
-  def testBahdanauNotNormalized(self):
-    create_attention_mechanism = wrapper.BahdanauAttentionV2
-    create_attention_kwargs = {"kernel_initializer": "ones"}
-    expected_final_output = basic_decoder.BasicDecoderOutput(
-        rnn_output=ResultSummary(
-            shape=(5, 3, 6), dtype=np.dtype(np.float32), mean=4.8290324),
-        sample_id=ResultSummary(shape=(5, 3), dtype=np.dtype(np.int32), mean=0))
-    expected_final_state = wrapper.AttentionWrapperState(
-        cell_state=rnn_cell.LSTMStateTuple(
-            c=ResultSummary(
-                shape=(5, 9), dtype=np.dtype(np.float32), mean=1.6432636),
-            h=ResultSummary(
-                shape=(5, 9), dtype=np.dtype(np.float32), mean=0.75866824)),
-        attention=ResultSummary(
-            shape=(5, 6), dtype=np.dtype(np.float32), mean=6.7445569),
-        time=3,
-        alignments=ResultSummary(
-            shape=(5, 8), dtype=np.dtype(np.float32), mean=0.125),
-        attention_state=ResultSummary(
-            shape=(5, 8), dtype=np.dtype(np.float32), mean=0.125),
-        alignment_history=())
-    expected_final_alignment_history = ResultSummary(
-        shape=(3, 5, 8), dtype=np.dtype(np.float32), mean=0.125)
-
-    self._testWithAttention(
-        create_attention_mechanism,
-        expected_final_output,
-        expected_final_state,
-        alignment_history=True,
-        create_query_layer=True,
-        expected_final_alignment_history=expected_final_alignment_history,
-        create_attention_kwargs=create_attention_kwargs)
-
-  def testBahdanauNormalized(self):
-    create_attention_mechanism = wrapper.BahdanauAttentionV2
-    create_attention_kwargs = {"kernel_initializer": "ones", "normalize": True}
-
-    expected_final_output = basic_decoder.BasicDecoderOutput(
-        rnn_output=ResultSummary(
-            shape=(5, 3, 6), dtype=np.dtype("float32"), mean=3.9548259),
-        sample_id=ResultSummary(
-            shape=(5, 3), dtype=np.dtype("int32"), mean=0.0))
-    expected_final_state = wrapper.AttentionWrapperState(
-        cell_state=rnn_cell.LSTMStateTuple(
-            c=ResultSummary(
-                shape=(5, 9), dtype=np.dtype("float32"), mean=1.4652209),
-            h=ResultSummary(
-                shape=(5, 9), dtype=np.dtype("float32"), mean=0.70997983)),
-        attention=ResultSummary(
-            shape=(5, 6), dtype=np.dtype("float32"), mean=6.3075728),
-        time=3,
-        alignments=ResultSummary(
-            shape=(5, 8), dtype=np.dtype("float32"), mean=0.125),
-        attention_state=ResultSummary(
-            shape=(5, 8), dtype=np.dtype("float32"), mean=0.125),
-        alignment_history=())
-
-    self._testWithAttention(
-        create_attention_mechanism,
-        expected_final_output,
-        expected_final_state,
-        create_query_layer=True,
-        create_attention_kwargs=create_attention_kwargs)
-
-  def testLuongNotNormalized(self):
-    create_attention_mechanism = wrapper.LuongAttentionV2
-
-    expected_final_output = basic_decoder.BasicDecoderOutput(
-        rnn_output=ResultSummary(
-            shape=(5, 3, 6), dtype=np.dtype("float32"), mean=2.6605489),
-        sample_id=ResultSummary(
-            shape=(5, 3), dtype=np.dtype("int32"), mean=0.0))
-    expected_final_state = wrapper.AttentionWrapperState(
-        cell_state=rnn_cell.LSTMStateTuple(
-            c=ResultSummary(
-                shape=(5, 9), dtype=np.dtype("float32"), mean=0.88403547),
-            h=ResultSummary(
-                shape=(5, 9), dtype=np.dtype("float32"), mean=0.37819088)),
-        attention=ResultSummary(
-            shape=(5, 6), dtype=np.dtype("float32"), mean=4.084631),
-        time=3,
-        alignments=ResultSummary(
-            shape=(5, 8), dtype=np.dtype("float32"), mean=0.125),
-        attention_state=ResultSummary(
-            shape=(5, 8), dtype=np.dtype("float32"), mean=0.125),
-        alignment_history=())
-
-    self._testWithAttention(
-        create_attention_mechanism,
-        expected_final_output,
-        expected_final_state,
-        attention_mechanism_depth=9)
-
-  def testLuongScaled(self):
-    create_attention_mechanism = wrapper.LuongAttentionV2
-    create_attention_kwargs = {"scale": True}
-
-    expected_final_output = basic_decoder.BasicDecoderOutput(
-        rnn_output=ResultSummary(
-            shape=(5, 3, 6), dtype=np.dtype("float32"), mean=2.6605489),
-        sample_id=ResultSummary(
-            shape=(5, 3), dtype=np.dtype("int32"), mean=0.0))
-    expected_final_state = wrapper.AttentionWrapperState(
-        cell_state=rnn_cell.LSTMStateTuple(
-            c=ResultSummary(
-                shape=(5, 9), dtype=np.dtype("float32"), mean=0.88403547),
-            h=ResultSummary(
-                shape=(5, 9), dtype=np.dtype("float32"), mean=0.37819088)),
-        attention=ResultSummary(
-            shape=(5, 6), dtype=np.dtype("float32"), mean=4.0846314),
-        time=3,
-        alignments=ResultSummary(
-            shape=(5, 8), dtype=np.dtype("float32"), mean=0.125),
-        attention_state=ResultSummary(
-            shape=(5, 8), dtype=np.dtype("float32"), mean=0.125),
-        alignment_history=())
-
-    self._testWithAttention(
-        create_attention_mechanism,
-        expected_final_output,
-        expected_final_state,
-        attention_mechanism_depth=9,
-        create_attention_kwargs=create_attention_kwargs)
-
-  def testNotUseAttentionLayer(self):
-    create_attention_mechanism = wrapper.BahdanauAttentionV2
-    create_attention_kwargs = {"kernel_initializer": "ones"}
-
-    expected_final_output = basic_decoder.BasicDecoderOutput(
-        rnn_output=ResultSummary(
-            shape=(5, 3, 10), dtype=np.dtype("float32"), mean=0.072406612),
-        sample_id=ResultSummary(
-            shape=(5, 3), dtype=np.dtype("int32"), mean=3.86666666))
-    expected_final_state = wrapper.AttentionWrapperState(
-        cell_state=rnn_cell.LSTMStateTuple(
-            c=ResultSummary(
-                shape=(5, 9), dtype=np.dtype("float32"), mean=1.032002),
-            h=ResultSummary(
-                shape=(5, 9), dtype=np.dtype("float32"), mean=0.61177742)),
-        attention=ResultSummary(
-            shape=(5, 10), dtype=np.dtype("float32"), mean=0.011346335),
-        time=3,
-        alignments=ResultSummary(
-            shape=(5, 8), dtype=np.dtype("float32"), mean=0.125),
-        attention_state=ResultSummary(
-            shape=(5, 8), dtype=np.dtype("float32"), mean=0.125),
-        alignment_history=())
-
-    self._testWithAttention(
-        create_attention_mechanism,
-        expected_final_output,
-        expected_final_state,
-        attention_layer_size=None,
-        create_query_layer=True,
-        create_attention_kwargs=create_attention_kwargs)
-
-  def testBahdanauMonotonicNotNormalized(self):
-    create_attention_mechanism = wrapper.BahdanauMonotonicAttentionV2
-    create_attention_kwargs = {"kernel_initializer": "ones"}
-
-    expected_final_output = basic_decoder.BasicDecoderOutput(
-        rnn_output=ResultSummary(
-            shape=(5, 3, 6), dtype=np.dtype("float32"), mean=5.9850435),
-        sample_id=ResultSummary(
-            shape=(5, 3), dtype=np.dtype("int32"), mean=0.0))
-    expected_final_state = wrapper.AttentionWrapperState(
-        cell_state=rnn_cell.LSTMStateTuple(
-            c=ResultSummary(
-                shape=(5, 9), dtype=np.dtype("float32"), mean=1.6752492),
-            h=ResultSummary(
-                shape=(5, 9), dtype=np.dtype("float32"), mean=0.76052248)),
-        attention=ResultSummary(
-            shape=(5, 6), dtype=np.dtype("float32"), mean=8.361186),
-        time=3,
-        alignments=ResultSummary(
-            shape=(5, 8), dtype=np.dtype("float32"), mean=0.10989678),
-        attention_state=ResultSummary(
-            shape=(5, 8), dtype=np.dtype("float32"), mean=0.10989678),
-        alignment_history=())
-    expected_final_alignment_history = ResultSummary(
-        shape=(3, 5, 8), dtype=np.dtype("float32"), mean=0.117412611)
-
-    self._testWithAttention(
-        create_attention_mechanism,
-        expected_final_output,
-        expected_final_state,
-        alignment_history=True,
-        expected_final_alignment_history=expected_final_alignment_history,
-        create_query_layer=True,
-        create_attention_kwargs=create_attention_kwargs)
-
-  def testBahdanauMonotonicNormalized(self):
-    create_attention_mechanism = wrapper.BahdanauMonotonicAttentionV2
-    create_attention_kwargs = {"kernel_initializer": "ones",
-                               "normalize": True}
-    expected_final_output = basic_decoder.BasicDecoderOutput(
-        rnn_output=ResultSummary(
-            shape=(5, 3, 6), dtype=np.dtype("float32"), mean=4.5706983),
-        sample_id=ResultSummary(
-            shape=(5, 3), dtype=np.dtype("int32"), mean=0.0))
-    expected_final_state = wrapper.AttentionWrapperState(
-        cell_state=rnn_cell.LSTMStateTuple(
-            c=ResultSummary(
-                shape=(5, 9), dtype=np.dtype("float32"), mean=1.6005473),
-            h=ResultSummary(
-                shape=(5, 9), dtype=np.dtype("float32"), mean=0.77863038)),
-        attention=ResultSummary(
-            shape=(5, 6), dtype=np.dtype("float32"), mean=7.3326721),
-        time=3,
-        alignments=ResultSummary(
-            shape=(5, 8), dtype=np.dtype("float32"), mean=0.12258384),
-        attention_state=ResultSummary(
-            shape=(5, 8), dtype=np.dtype("float32"), mean=0.12258384),
-        alignment_history=())
-    expected_final_alignment_history = ResultSummary(
-        shape=(3, 5, 8), dtype=np.dtype("float32"), mean=0.12258384)
-
-    self._testWithAttention(
-        create_attention_mechanism,
-        expected_final_output,
-        expected_final_state,
-        alignment_history=True,
-        expected_final_alignment_history=expected_final_alignment_history,
-        create_query_layer=True,
-        create_attention_kwargs=create_attention_kwargs)
-
-  def testLuongMonotonicNotNormalized(self):
-    create_attention_mechanism = wrapper.LuongMonotonicAttentionV2
-
-    expected_final_output = basic_decoder.BasicDecoderOutput(
-        rnn_output=ResultSummary(
-            shape=(5, 3, 6), dtype=np.dtype("float32"), mean=3.159497),
-        sample_id=ResultSummary(
-            shape=(5, 3), dtype=np.dtype("int32"), mean=0.0))
-    expected_final_state = wrapper.AttentionWrapperState(
-        cell_state=rnn_cell.LSTMStateTuple(
-            c=ResultSummary(
-                shape=(5, 9), dtype=np.dtype("float32"), mean=1.072384),
-            h=ResultSummary(
-                shape=(5, 9), dtype=np.dtype("float32"), mean=0.50331038)),
-        attention=ResultSummary(
-            shape=(5, 6), dtype=np.dtype("float32"), mean=5.3079605),
-        time=3,
-        alignments=ResultSummary(
-            shape=(5, 8), dtype=np.dtype("float32"), mean=0.11467695),
-        attention_state=ResultSummary(
-            shape=(5, 8), dtype=np.dtype("float32"), mean=0.11467695),
-        alignment_history=())
-    expected_final_alignment_history = ResultSummary(
-        shape=(3, 5, 8), dtype=np.dtype("float32"), mean=0.11899644)
-
-    self._testWithAttention(
-        create_attention_mechanism,
-        expected_final_output,
-        expected_final_state,
-        attention_mechanism_depth=9,
-        alignment_history=True,
-        expected_final_alignment_history=expected_final_alignment_history)
-
-  def testLuongMonotonicScaled(self):
-    create_attention_mechanism = wrapper.LuongMonotonicAttentionV2
-    create_attention_kwargs = {"scale": True}
-
-    expected_final_output = basic_decoder.BasicDecoderOutput(
-        rnn_output=ResultSummary(
-            shape=(5, 3, 6), dtype=np.dtype("float32"), mean=3.159497),
-        sample_id=ResultSummary(
-            shape=(5, 3), dtype=np.dtype("int32"), mean=0.0))
-    expected_final_state = wrapper.AttentionWrapperState(
-        cell_state=rnn_cell.LSTMStateTuple(
-            c=ResultSummary(
-                shape=(5, 9), dtype=np.dtype("float32"), mean=1.072384),
-            h=ResultSummary(
-                shape=(5, 9), dtype=np.dtype("float32"), mean=0.50331038)),
-        attention=ResultSummary(
-            shape=(5, 6), dtype=np.dtype("float32"), mean=5.3079605),
-        time=3,
-        alignments=ResultSummary(
-            shape=(5, 8), dtype=np.dtype("float32"), mean=0.11467695),
-        attention_state=ResultSummary(
-            shape=(5, 8), dtype=np.dtype("float32"), mean=0.11467695),
-        alignment_history=())
-    expected_final_alignment_history = ResultSummary(
-        shape=(3, 5, 8), dtype=np.dtype("float32"), mean=0.11899644)
-
-    self._testWithAttention(
-        create_attention_mechanism,
-        expected_final_output,
-        expected_final_state,
-        attention_mechanism_depth=9,
-        alignment_history=True,
-        expected_final_alignment_history=expected_final_alignment_history,
-        create_attention_kwargs=create_attention_kwargs)
-
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
index 07635e4dde..ae3e7f1b5d 100644
--- a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
+++ b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
@@ -28,9 +28,7 @@ from tensorflow.contrib.framework.python.framework import tensor_util
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
-from tensorflow.python.keras import initializers
 from tensorflow.python.keras import layers
-from tensorflow.python.keras.engine import base_layer_utils
 from tensorflow.python.layers import base as layers_base
 from tensorflow.python.layers import core as layers_core
 from tensorflow.python.ops import array_ops
@@ -227,39 +225,18 @@ class _BaseAttentionMechanismV2(AttentionMechanism, layers.Layer):
     1. Storing the query and memory layers.
     2. Preprocessing and storing the memory.
 
-  Note that this layer takes memory as its init parameter, which is an
-  anti-pattern of Keras API, we have to keep the memory as init parameter for
-  performance and dependency reason. Under the hood, during `__init__()`, it
-  will invoke `base_layer.__call__(memory, setup_memory=True)`. This will let
-  keras to keep track of the memory tensor as the input of this layer. Once
-  the `__init__()` is done, then user can query the attention by
-  `score = att_obj([query, state])`, and use it as a normal keras layer.
-
-  Special attention is needed when adding using this class as the base layer for
-  new attention:
-    1. Build() could be invoked at least twice. So please make sure weights are
-       not duplicated.
-    2. Layer.get_weights() might return different set of weights if the instance
-       has `query_layer`. The query_layer weights is not initialized until the
-       memory is configured.
-
-  Also note that this layer does not work with Keras model when
-  `model.compile(run_eagerly=True)` due to the fact that this layer is stateful.
-  The support for that will be added in a future version.
+  Note that this layer only support Keras functional API since it takes multiple
+  input tensors, which is not available in sequential model.
   """
 
   def __init__(self,
-               memory,
                probability_fn,
                query_layer=None,
                memory_layer=None,
-               memory_sequence_length=None,
                **kwargs):
     """Construct base AttentionMechanism class.
 
     Args:
-      memory: The memory to query; usually the output of an RNN encoder.  This
-        tensor should be shaped `[batch_size, max_time, ...]`.
       probability_fn: A `callable`. Converts the score and previous alignments
         to probabilities. Its signature should be:
         `probabilities = probability_fn(score, state)`.
@@ -270,9 +247,6 @@ class _BaseAttentionMechanismV2(AttentionMechanism, layers.Layer):
         depth must match the depth of `query_layer`.
         If `memory_layer` is not provided, the shape of `memory` must match
         that of `query_layer`.
-      memory_sequence_length (optional): Sequence lengths for the batch entries
-        in memory. If provided, the memory tensor rows are masked with zeros
-        for values past the respective sequence lengths.
       **kwargs: Dictionary that contains other common arguments for layer
         creation.
     """
@@ -299,128 +273,20 @@ class _BaseAttentionMechanismV2(AttentionMechanism, layers.Layer):
     self.batch_size = None
     self._memory_initialized = False
     self._check_inner_dims_defined = True
-    self.supports_masking = True
-    self.score_mask_value = dtypes.as_dtype(self.dtype).as_numpy_dtype(-np.inf)
-
-    if memory is not None:
-      # Setup the memory by self.__call__() with memory and memory_seq_length.
-      # This will make the attention follow the keras convention which takes
-      # all the tensor inputs via __call__().
-      if memory_sequence_length is None:
-        inputs = memory
-      else:
-        inputs = [memory, memory_sequence_length]
-      # The returned tensor is ignored here since we already record it as an
-      # attribute.
-      super(_BaseAttentionMechanismV2, self).__call__(
-          inputs, setup_memory=True)
 
   def build(self, input_shape):
-    if not self._memory_initialized:
-      # This is for setting up the memory, which contains memory and optional
-      # memory_sequence_length. Build the memory_layer with memory shape.
-      if self.memory_layer is not None and not self.memory_layer.built:
-        if isinstance(input_shape, list):
-          self.memory_layer.build(input_shape[0])
-        else:
-          self.memory_layer.build(input_shape)
-    else:
-      # The input_shape should be query.shape and state.shape. Use the query
-      # to init the query layer.
-      if self.query_layer is not None and not self.query_layer.built:
-        self.query_layer.build(input_shape[0])
-
-  def __call__(self, inputs, **kwargs):
-    """Preprocess the inputs before calling `base_layer.__call__()`.
-
-    Note that there are situation here, one for setup memory, and one with
-    actual query and state.
-    1. When the memory has not been configured, we just pass all the param to
-    base_layer.__call__(), which will then invoke self.call() with proper
-    inputs, which allows this class to setup memory.
-    2. When the memory has already been setup, the input should contain query
-    and state, and optionally processed memory. If the processed memory is
-    not included in the input, we will have to append it to the inputs and
-    give it to the base_layer.__call__(). The processed memory is the output
-    of first invocation of self.__call__(). If we don't add it here, then from
-    keras perspective, the graph is disconnected since the output from
-    previous call is never used.
-
-    Args:
-      inputs: the inputs tensors.
-      **kwargs: dict, other keyeword arguments for the `__call__()`
-    """
-    if self._memory_initialized:
-      if len(inputs) not in (2, 3):
-        raise ValueError("Expect the inputs to have 2 or 3 tensors, got %d" %
-                         len(inputs))
-      if len(inputs) == 2:
-        # We append the calculated memory here so that the graph will be
-        # connected.
-        inputs.append(self.values)
-    return super(_BaseAttentionMechanismV2, self).__call__(inputs, **kwargs)
-
-  def call(self, inputs, mask=None, setup_memory=False, **kwargs):
-    """Setup the memory or query the attention.
-
-    There are two case here, one for setup memory, and the second is query the
-    attention score. `setup_memory` is the flag to indicate which mode it is.
-    The input list will be treated differently based on that flag.
+    # The layer suppose to take 3 inputs, [query, state, memory].
+    query_input_shape, _, memory_input_shape = input_shape
+    if self.query_layer is not None:
+      self.query_layer.build(query_input_shape)
+    if self.memory_layer is not None:
+      self.memory_layer.build(memory_input_shape)
+    # dtype of the layer is known at this moment, create the score_mask_value if
+    # needed.
+    self.score_mask_value = dtypes.as_dtype(self.dtype).as_numpy_dtype(-np.inf)
+    self.built = True
 
-    Args:
-      inputs: a list of tensor that could either be `query` and `state`, or
-        `memory` and `memory_sequence_length`.
-        `query` is the tensor of dtype matching `memory` and shape
-        `[batch_size, query_depth]`.
-        `state` is the tensor of dtype matching `memory` and shape
-        `[batch_size, alignments_size]`. (`alignments_size` is memory's
-        `max_time`).
-        `memory` is the memory to query; usually the output of an RNN encoder.
-        The tensor should be shaped `[batch_size, max_time, ...]`.
-        `memory_sequence_length` (optional) is the sequence lengths for the
-         batch entries in memory. If provided, the memory tensor rows are masked
-        with zeros for values past the respective sequence lengths.
-      mask: optional bool tensor with shape `[batch, max_time]` for the mask of
-        memory. If it is not None, the corresponding item of the memory should
-        be filtered out during calculation.
-      setup_memory: boolean, whether the input is for setting up memory, or
-        query attention.
-      **kwargs: Dict, other keyword arguments for the call method.
-    Returns:
-      Either processed memory or attention score, based on `setup_memory`.
-    """
-    if setup_memory:
-      if isinstance(inputs, list):
-        if len(inputs) not in (1, 2):
-          raise ValueError("Expect inputs to have 1 or 2 tensors, got %d" %
-                           len(inputs))
-        memory = inputs[0]
-        memory_sequence_length = inputs[1] if len(inputs) == 2 else None
-        memory_mask = mask
-      else:
-        memory, memory_sequence_length = inputs, None
-        memory_mask = mask
-      self._setup_memory(memory, memory_sequence_length, memory_mask)
-      # We force the self.built to false here since only memory is initialized,
-      # but the real query/state has not been call() yet. The layer should be
-      # build and call again.
-      self.built = False
-      # We have to return something here since layers are required to have some
-      # output. Use self.value here since it is the processed memory.
-      return self.values
-    else:
-      if not self._memory_initialized:
-        raise ValueError("Cannot query the attention before the setup of "
-                         "memory")
-      if len(inputs) not in (2, 3):
-        raise ValueError("Expect the inputs to have query, state, and optional "
-                         "processed memory, got %d items" % len(inputs))
-      # Ignore the rest of the inputs and only care about the query and state
-      query, state = inputs[0], inputs[1]
-      return self._calculate_attention(query, state)
-
-  def _setup_memory(self, memory, memory_sequence_length=None,
-                    memory_mask=None):
+  def _setup_memory(self, memory, memory_mask=None):
     """Pre-process the memory before actually query the memory.
 
     This should only be called once at the first invocation of call().
@@ -428,30 +294,17 @@ class _BaseAttentionMechanismV2(AttentionMechanism, layers.Layer):
     Args:
       memory: The memory to query; usually the output of an RNN encoder. This
         tensor should be shaped `[batch_size, max_time, ...]`.
-      memory_sequence_length (optional): Sequence lengths for the batch entries
-        in memory. If provided, the memory tensor rows are masked with zeros for
-        values past the respective sequence lengths.
-      memory_mask: (Optional) The boolean tensor with shape `[batch_size,
-        max_time]`. For any value equal to False, the corresponding value in
-        memory should be ignored.
+      memory_mask: The boolean tensor with shape `[batch_size, max_time]`. For
+        any value equal to False, the corresponding value in memory should be
+        ignored.
     """
     if self._memory_initialized:
       raise ValueError("The memory for the attention has already been setup.")
-    if memory_sequence_length is not None and memory_mask is not None:
-      raise ValueError("memory_sequence_length and memory_mask cannot be "
-                       "used at same time for attention.")
     with ops.name_scope(
         self.name, "BaseAttentionMechanismInit", nest.flatten(memory)):
       self.values = _prepare_memory(
-          memory,
-          memory_sequence_length=memory_sequence_length,
-          memory_mask=memory_mask,
+          memory, memory_mask=memory_mask,
           check_inner_dims_defined=self._check_inner_dims_defined)
-      # Mark the value as check since the memory and memory mask might not
-      # passed from __call__(), which does not have proper keras metadata.
-      # TODO(omalleyt): Remove this hack once the mask the has proper keras
-      # history.
-      base_layer_utils.mark_checked(self.values)
       if self.memory_layer is not None:
         self.keys = self.memory_layer(self.values)
       else:
@@ -462,25 +315,36 @@ class _BaseAttentionMechanismV2(AttentionMechanism, layers.Layer):
       self._alignments_size = (tensor_shape.dimension_value(self.keys.shape[1])
                                or array_ops.shape(self.keys)[1])
       if memory_mask is not None:
-        unwrapped_probability_fn = self.probability_fn
-        def _mask_probability_fn(score, prev):
-          return unwrapped_probability_fn(
-              _maybe_mask_score(
-                  score,
-                  memory_mask=memory_mask,
-                  memory_sequence_length=memory_sequence_length,
-                  score_mask_value=self.score_mask_value), prev)
-        self.probability_fn = _mask_probability_fn
+        self.probability_fn = lambda score, prev: (  # pylint:disable=g-long-lambda
+            self.probability_fn(_maybe_mask_score(
+                score, self.score_mask_value, memory_mask=memory_mask), prev))
     self._memory_initialized = True
 
-  def _calculate_attention(self, query, state):
-    raise NotImplementedError(
-        "_calculate_attention need to be implemented by subclasses.")
+  def call(self, inputs, mask=None, **kwargs):
+    """Base method to calculate the attention score.
 
-  def compute_mask(self, inputs, mask=None):
-    # There real input of the attention is query and state, and the memory layer
-    # mask shouldn't be pass down. Returning None for all output mask here.
-    return None, None
+    Args:
+      inputs: a list of tensor that contains `query`, `state`, and `memory`.
+        `query` is the tensor of dtype matching `memory` and shape
+        `[batch_size, query_depth]`.
+        `state` is the tensor of dtype matching `memory` and shape
+        `[batch_size, alignments_size]`. (`alignments_size` is memory's
+        `max_time`).
+        `memory` is the memory to query; usually the output of an RNN encoder.
+        This tensor should be shaped `[batch_size, max_time, feature]`.
+      mask: optional bool tensor with shape `[batch, max_time]` for the mask of
+        memory. If it is not None, the corresponding item of the memory should
+        be filtered out during calculation.
+      **kwargs: Dict, other keyword arguments for the call method.
+    """
+    query, state, memory, memory_mask = self._process_inputs(inputs, mask)
+    if not self._memory_initialized:
+      self._setup_memory(memory, memory_mask=memory_mask)
+    return self.calculate_attention(query, state)
+
+  def calculate_attention(self, query, state):
+    raise NotImplementedError(
+        "calculate_attention need to be implemented by subclasses.")
 
   def get_config(self):
     config = {}
@@ -497,12 +361,16 @@ class _BaseAttentionMechanismV2(AttentionMechanism, layers.Layer):
           "class_name": self.memory_layer.__class__.__name__,
           "config": self.memory_layer.get_config(),
       }
-    # memory is a required init parameter and its a tensor. It cannot be
-    # serialized to config, so we put a placeholder for it.
-    config["memory"] = None
     base_config = super(_BaseAttentionMechanismV2, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
 
+  def _process_inputs(self, inputs, mask):
+    if len(inputs) != 3:
+      raise ValueError(
+          "Expect to have 3 inputs for attention, got %d" % len(inputs))
+    query, state, memory = inputs
+    return query, state, memory, mask
+
   def _process_probability_fn(self, func_name):
     """Helper method to retrieve the probably function by string input."""
     valid_probability_fns = {
@@ -550,46 +418,6 @@ class _BaseAttentionMechanismV2(AttentionMechanism, layers.Layer):
   def alignments_size(self):
     return self._alignments_size
 
-  @property
-  def state_size(self):
-    return self._alignments_size
-
-  def initial_alignments(self, batch_size, dtype):
-    """Creates the initial alignment values for the `AttentionWrapper` class.
-
-    This is important for AttentionMechanisms that use the previous alignment
-    to calculate the alignment at the next time step (e.g. monotonic attention).
-
-    The default behavior is to return a tensor of all zeros.
-
-    Args:
-      batch_size: `int32` scalar, the batch_size.
-      dtype: The `dtype`.
-
-    Returns:
-      A `dtype` tensor shaped `[batch_size, alignments_size]`
-      (`alignments_size` is the values' `max_time`).
-    """
-    max_time = self._alignments_size
-    return _zero_state_tensors(max_time, batch_size, dtype)
-
-  def initial_state(self, batch_size, dtype):
-    """Creates the initial state values for the `AttentionWrapper` class.
-
-    This is important for AttentionMechanisms that use the previous alignment
-    to calculate the alignment at the next time step (e.g. monotonic attention).
-
-    The default behavior is to return the same output as initial_alignments.
-
-    Args:
-      batch_size: `int32` scalar, the batch_size.
-      dtype: The `dtype`.
-
-    Returns:
-      A structure of all-zero tensors with shapes as described by `state_size`.
-    """
-    return self.initial_alignments(batch_size, dtype)
-
 
 def _luong_score(query, keys, scale):
   """Implements Luong-style (multiplicative) scoring function.
@@ -759,8 +587,6 @@ class LuongAttentionV2(_BaseAttentionMechanismV2):
 
   def __init__(self,
                units,
-               memory,
-               memory_sequence_length=None,
                scale=False,
                probability_fn="softmax",
                dtype=None,
@@ -770,11 +596,6 @@ class LuongAttentionV2(_BaseAttentionMechanismV2):
 
     Args:
       units: The depth of the attention mechanism.
-      memory: The memory to query; usually the output of an RNN encoder.  This
-        tensor should be shaped `[batch_size, max_time, ...]`.
-      memory_sequence_length: (optional): Sequence lengths for the batch entries
-        in memory.  If provided, the memory tensor rows are masked with zeros
-        for values past the respective sequence lengths.
       scale: Python boolean. Whether to scale the energy term.
       probability_fn: (optional) string, the name of function to convert the
         attention score to probabilities. The default is `softmax` which is
@@ -797,27 +618,26 @@ class LuongAttentionV2(_BaseAttentionMechanismV2):
     if not memory_layer:
       memory_layer = layers.Dense(
           units, name="memory_layer", use_bias=False, dtype=dtype)
-    self.units = units
-    self.scale = scale
-    self.scale_weight = None
     super(LuongAttentionV2, self).__init__(
-        memory=memory,
-        memory_sequence_length=memory_sequence_length,
         query_layer=None,
         memory_layer=memory_layer,
         probability_fn=wrapped_probability_fn,
         name=name,
         dtype=dtype,
         **kwargs)
+    self.units = units
+    self.scale = scale
 
   def build(self, input_shape):
     super(LuongAttentionV2, self).build(input_shape)
-    if self.scale and self.scale_weight is None:
+    if self.scale:
       self.scale_weight = self.add_weight(
           "attention_g", initializer=init_ops.ones_initializer, shape=())
+    else:
+      self.scale_weight = None
     self.built = True
 
-  def _calculate_attention(self, query, state):
+  def calculate_attention(self, query, state):
     """Score the query based on the keys and values.
 
     Args:
@@ -1031,11 +851,8 @@ class BahdanauAttentionV2(_BaseAttentionMechanismV2):
 
   def __init__(self,
                units,
-               memory,
-               memory_sequence_length=None,
                normalize=False,
                probability_fn="softmax",
-               kernel_initializer="glorot_uniform",
                dtype=None,
                name="BahdanauAttention",
                **kwargs):
@@ -1043,19 +860,12 @@ class BahdanauAttentionV2(_BaseAttentionMechanismV2):
 
     Args:
       units: The depth of the query mechanism.
-      memory: The memory to query; usually the output of an RNN encoder.  This
-        tensor should be shaped `[batch_size, max_time, ...]`.
-      memory_sequence_length: (optional): Sequence lengths for the batch entries
-        in memory.  If provided, the memory tensor rows are masked with zeros
-        for values past the respective sequence lengths.
       normalize: Python boolean.  Whether to normalize the energy term.
       probability_fn: (optional) string, the name of function to convert the
         attention score to probabilities. The default is `softmax` which is
         `tf.nn.softmax`. Other options is `hardmax`, which is hardmax() within
         this module. Any other value will result into validation error. Default
         to use `softmax`.
-      kernel_initializer: (optional), the name of the initializer for the
-        attention kernel.
       dtype: The data type for the query and memory layers of the attention
         mechanism.
       name: Name to use when creating ops.
@@ -1075,39 +885,33 @@ class BahdanauAttentionV2(_BaseAttentionMechanismV2):
     if not memory_layer:
       memory_layer = layers.Dense(
           units, name="memory_layer", use_bias=False, dtype=dtype)
-    self.units = units
-    self.normalize = normalize
-    self.kernel_initializer = initializers.get(kernel_initializer)
-    self.attention_v = None
-    self.attention_g = None
-    self.attention_b = None
     super(BahdanauAttentionV2, self).__init__(
-        memory=memory,
-        memory_sequence_length=memory_sequence_length,
         query_layer=query_layer,
         memory_layer=memory_layer,
         probability_fn=wrapped_probability_fn,
         name=name,
         dtype=dtype,
         **kwargs)
+    self.units = units
+    self.normalize = normalize
 
   def build(self, input_shape):
     super(BahdanauAttentionV2, self).build(input_shape)
-    if self.attention_v is None:
-      self.attention_v = self.add_weight(
-          "attention_v", [self.units],
-          dtype=self.dtype,
-          initializer=self.kernel_initializer)
-    if self.normalize and self.attention_g is None and self.attention_b is None:
+    self.attention_v = self.add_weight(
+        "attention_v", [self.units], dtype=self.dtype)
+    if self.normalize:
       self.attention_g = self.add_weight(
           "attention_g", initializer=init_ops.constant_initializer(
               math.sqrt((1. / self.units))), shape=())
       self.attention_b = self.add_weight(
           "attention_b", shape=[self.units],
           initializer=init_ops.zeros_initializer())
+    else:
+      self.attention_g = None
+      self.attention_b = None
     self.built = True
 
-  def _calculate_attention(self, query, state):
+  def calculate_attention(self, query, state):
     """Score the query based on the keys and values.
 
     Args:
@@ -1136,7 +940,6 @@ class BahdanauAttentionV2(_BaseAttentionMechanismV2):
         "units": self.units,
         "normalize": self.normalize,
         "probability_fn": self.probability_fn_name,
-        "kernel_initializer": initializers.serialize(self.kernel_initializer)
     }
     base_config = super(BahdanauAttentionV2, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
@@ -1496,14 +1299,11 @@ class BahdanauMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
 
   def __init__(self,
                units,
-               memory,
-               memory_sequence_length=None,
                normalize=False,
                sigmoid_noise=0.,
                sigmoid_noise_seed=None,
                score_bias_init=0.,
                mode="parallel",
-               kernel_initializer="glorot_uniform",
                dtype=None,
                name="BahdanauMonotonicAttention",
                **kwargs):
@@ -1511,11 +1311,6 @@ class BahdanauMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
 
     Args:
       units: The depth of the query mechanism.
-      memory: The memory to query; usually the output of an RNN encoder.  This
-        tensor should be shaped `[batch_size, max_time, ...]`.
-      memory_sequence_length: (optional): Sequence lengths for the batch entries
-        in memory.  If provided, the memory tensor rows are masked with zeros
-        for values past the respective sequence lengths.
       normalize: Python boolean. Whether to normalize the energy term.
       sigmoid_noise: Standard deviation of pre-sigmoid noise. See the docstring
         for `_monotonic_probability_fn` for more information.
@@ -1526,8 +1321,6 @@ class BahdanauMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
       mode: How to compute the attention distribution. Must be one of
         'recursive', 'parallel', or 'hard'. See the docstring for
         `tf.contrib.seq2seq.monotonic_attention` for more information.
-      kernel_initializer: (optional), the name of the initializer for the
-        attention kernel.
       dtype: The data type for the query and memory layers of the attention
         mechanism.
       name: Name to use when creating ops.
@@ -1548,39 +1341,32 @@ class BahdanauMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
     if not memory_layer:
       memory_layer = layers.Dense(
           units, name="memory_layer", use_bias=False, dtype=dtype)
-    self.units = units
-    self.normalize = normalize
-    self.sigmoid_noise = sigmoid_noise
-    self.sigmoid_noise_seed = sigmoid_noise_seed
-    self.score_bias_init = score_bias_init
-    self.mode = mode
-    self.kernel_initializer = initializers.get(kernel_initializer)
-    self.attention_v = None
-    self.attention_score_bias = None
-    self.attention_g = None
-    self.attention_b = None
     super(BahdanauMonotonicAttentionV2, self).__init__(
-        memory=memory,
-        memory_sequence_length=memory_sequence_length,
         query_layer=query_layer,
         memory_layer=memory_layer,
         probability_fn=wrapped_probability_fn,
         name=name,
         dtype=dtype,
         **kwargs)
+    self.units = units
+    self.normalize = normalize
+    self.sigmoid_noise = sigmoid_noise
+    self.sigmoid_noise_seed = sigmoid_noise_seed
+    self.score_bias_init = score_bias_init
+    self.mode = mode
 
   def build(self, input_shape):
     super(BahdanauMonotonicAttentionV2, self).build(input_shape)
-    if self.attention_v is None:
-      self.attention_v = self.add_weight(
-          "attention_v", [self.units], dtype=self.dtype,
-          initializer=self.kernel_initializer)
-    if self.attention_score_bias is None:
-      self.attention_score_bias = self.add_weight(
-          "attention_score_bias", shape=(), dtype=self.dtype,
-          initializer=init_ops.constant_initializer(
-              self.score_bias_init, dtype=self.dtype))
-    if self.normalize and self.attention_g is None and self.attention_b is None:
+    self.attention_v = self.add_weight(
+        "attention_v", [self.units], dtype=self.dtype)
+    self.attention_score_bias = self.add_weight(
+        "attention_score_bias", shape=(), dtype=self.dtype,
+        initializer=init_ops.constant_initializer(
+            self.score_bias_init, dtype=self.dtype))
+    if not self.normalize:
+      self.attention_g = None
+      self.attention_b = None
+    else:
       self.attention_g = self.add_weight(
           "attention_g", dtype=self.dtype,
           initializer=init_ops.constant_initializer(
@@ -1591,7 +1377,7 @@ class BahdanauMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
           initializer=init_ops.zeros_initializer())
     self.built = True
 
-  def _calculate_attention(self, query, state):
+  def calculate_attention(self, query, state):
     """Score the query based on the keys and values.
 
     Args:
@@ -1623,7 +1409,6 @@ class BahdanauMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
         "sigmoid_noise_seed": self.sigmoid_noise_seed,
         "score_bias_init": self.score_bias_init,
         "mode": self.mode,
-        "kernel_initializer": initializers.serialize(self.kernel_initializer),
     }
     base_config = super(BahdanauMonotonicAttentionV2, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
@@ -1757,8 +1542,6 @@ class LuongMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
 
   def __init__(self,
                units,
-               memory,
-               memory_sequence_length=None,
                scale=False,
                sigmoid_noise=0.,
                sigmoid_noise_seed=None,
@@ -1771,11 +1554,6 @@ class LuongMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
 
     Args:
       units: The depth of the query mechanism.
-      memory: The memory to query; usually the output of an RNN encoder.  This
-        tensor should be shaped `[batch_size, max_time, ...]`.
-      memory_sequence_length: (optional): Sequence lengths for the batch entries
-        in memory.  If provided, the memory tensor rows are masked with zeros
-        for values past the respective sequence lengths.
       scale: Python boolean.  Whether to scale the energy term.
       sigmoid_noise: Standard deviation of pre-sigmoid noise.  See the docstring
         for `_monotonic_probability_fn` for more information.
@@ -1802,37 +1580,34 @@ class LuongMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
     if not memory_layer:
       memory_layer = layers.Dense(
           units, name="memory_layer", use_bias=False, dtype=dtype)
-    self.units = units
-    self.scale = scale
-    self.sigmoid_noise = sigmoid_noise
-    self.sigmoid_noise_seed = sigmoid_noise_seed
-    self.score_bias_init = score_bias_init
-    self.mode = mode
-    self.attention_g = None
-    self.attention_score_bias = None
     super(LuongMonotonicAttentionV2, self).__init__(
-        memory=memory,
-        memory_sequence_length=memory_sequence_length,
         query_layer=None,
         memory_layer=memory_layer,
         probability_fn=wrapped_probability_fn,
         name=name,
         dtype=dtype,
         **kwargs)
+    self.units = units
+    self.scale = scale
+    self.sigmoid_noise = sigmoid_noise
+    self.sigmoid_noise_seed = sigmoid_noise_seed
+    self.score_bias_init = score_bias_init
+    self.mode = mode
 
   def build(self, input_shape):
     super(LuongMonotonicAttentionV2, self).build(input_shape)
-    if self.scale and self.attention_g is None:
+    if self.scale:
       self.attention_g = self.add_weight(
           "attention_g", initializer=init_ops.ones_initializer, shape=())
-    if self.attention_score_bias is None:
-      self.attention_score_bias = self.add_weight(
-          "attention_score_bias", shape=(),
-          initializer=init_ops.constant_initializer(
-              self.score_bias_init, dtype=self.dtype))
+    else:
+      self.attention_g = None
+    self.attention_score_bias = self.add_weight(
+        "attention_score_bias", shape=(),
+        initializer=init_ops.constant_initializer(
+            self.score_bias_init, dtype=self.dtype))
     self.built = True
 
-  def _calculate_attention(self, query, state):
+  def calculate_attention(self, query, state):
     """Score the query based on the keys and values.
 
     Args:
@@ -1964,26 +1739,41 @@ def _prepare_memory(memory, memory_sequence_length=None, memory_mask=None,
                          "but saw shape: %s" % (m.name, m.get_shape()))
     nest.map_structure(_check_dims, memory)
   if memory_sequence_length is None and memory_mask is None:
-    return memory
+    seq_len_mask = None
+    seq_len_batch_size = None
   elif memory_sequence_length is not None:
     seq_len_mask = array_ops.sequence_mask(
         memory_sequence_length,
         maxlen=array_ops.shape(nest.flatten(memory)[0])[1],
         dtype=nest.flatten(memory)[0].dtype)
+    seq_len_batch_size = (
+        tensor_shape.dimension_value(memory_sequence_length.shape[0])
+        or array_ops.shape(memory_sequence_length)[0])
   else:
     # For memory_mask is not None
-    seq_len_mask = math_ops.cast(
-        memory_mask, dtype=nest.flatten(memory)[0].dtype)
+    seq_len_mask = memory_mask
+    seq_len_batch_size = (
+        tensor_shape.dimension_value(memory_mask.shape[0])
+        or array_ops.shape(memory_mask)[0])
   def _maybe_mask(m, seq_len_mask):
     """Mask the memory based on the memory mask."""
     rank = m.get_shape().ndims
     rank = rank if rank is not None else array_ops.rank(m)
     extra_ones = array_ops.ones(rank - 2, dtype=dtypes.int32)
-    seq_len_mask = array_ops.reshape(
-        seq_len_mask,
-        array_ops.concat((array_ops.shape(seq_len_mask), extra_ones), 0))
-    return m * seq_len_mask
-
+    m_batch_size = tensor_shape.dimension_value(
+        m.shape[0]) or array_ops.shape(m)[0]
+    if seq_len_batch_size is not None:
+      message = ("memory_sequence_length and memory tensor batch sizes do not "
+                 "match.")
+      with ops.control_dependencies([
+          check_ops.assert_equal(
+              seq_len_batch_size, m_batch_size, message=message)]):
+        seq_len_mask = array_ops.reshape(
+            seq_len_mask,
+            array_ops.concat((array_ops.shape(seq_len_mask), extra_ones), 0))
+        return m * seq_len_mask
+    else:
+      return m
   return nest.map_structure(lambda m: _maybe_mask(m, seq_len_mask), memory)
 
 
@@ -2029,13 +1819,8 @@ def hardmax(logits, name=None):
 def _compute_attention(attention_mechanism, cell_output, attention_state,
                        attention_layer):
   """Computes the attention and alignments for a given attention_mechanism."""
-  if isinstance(attention_mechanism, _BaseAttentionMechanism):
-    alignments, next_attention_state = attention_mechanism(
-        cell_output, state=attention_state)
-  else:
-    # For AttentionMechanismV2, which takes inputs as a list.
-    alignments, next_attention_state = attention_mechanism(
-        [cell_output, attention_state])
+  alignments, next_attention_state = attention_mechanism(
+      cell_output, state=attention_state)
 
   # Reshape from [batch_size, memory_time] to [batch_size, 1, memory_time]
   expanded_alignments = array_ops.expand_dims(alignments, 1)
-- 
GitLab


From 6d924a1564a1dbc26f1e334873a54ab169350c49 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 8 Feb 2019 23:57:57 -0800
Subject: [PATCH 2226/2345] fix build

PiperOrigin-RevId: 233183272
---
 tensorflow/lite/testing/generate_examples.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index 5cd47eb1af..8449b321be 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -167,7 +167,7 @@ def toco_options(data_types,
 
 def format_result(t):
   """Convert a tensor to a format that can be used in test specs."""
-  if np.issubdtype(t.dtype, np.number):
+  if t.dtype.kind not in [np.dtype(np.string_).kind, np.dtype(np.object_).kind]:
     # Output 9 digits after the point to ensure the precision is good enough.
     values = ["{:.9f}".format(value) for value in list(t.flatten())]
     return ",".join(values)
-- 
GitLab


From b85cb440e257a367fb70f8321ddaa669d1bd9fae Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 9 Feb 2019 01:02:39 -0800
Subject: [PATCH 2227/2345] compat: Update forward compatibility horizon to
 2019-02-09

PiperOrigin-RevId: 233187563
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 68ca77dceb..1dd10fadfd 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 2, 8)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 2, 9)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 969e17fc37f595dbb372af4ce8308f991a96608c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 9 Feb 2019 15:36:25 -0800
Subject: [PATCH 2228/2345] Remove a code block that is never executed.

PiperOrigin-RevId: 233235142
---
 .../python/tpu/_tpu_estimator_embedding.py    | 20 ++++---------------
 1 file changed, 4 insertions(+), 16 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/_tpu_estimator_embedding.py b/tensorflow/contrib/tpu/python/tpu/_tpu_estimator_embedding.py
index 9a927c637e..dd239d5d78 100644
--- a/tensorflow/contrib/tpu/python/tpu/_tpu_estimator_embedding.py
+++ b/tensorflow/contrib/tpu/python/tpu/_tpu_estimator_embedding.py
@@ -87,22 +87,10 @@ def get_full_variable_names(
   for table_name in table_to_config_dict:
     embedding_var_name = _get_embedding_var_name_from_table_name(table_name)
     (scope_name, var_name) = collection[0][embedding_var_name]
-
-    if table_name in embedding_variable_name_by_table:
-      existed_scope_name, existed_var_name = (
-          embedding_variable_name_by_table[table_name])
-      if existed_scope_name != scope_name or existed_var_name != var_name:
-        raise RuntimeError(
-            'Internal error: For table {}, scope and/or var_name mismatch(es). '
-            'expected scope {} var_name {}; got scope {} var_name {}'.format(
-                table_name, existed_scope_name, existed_var_name, scope_name,
-                var_name))
-    else:
-      embedding_variable_name_by_table[table_name] = (
-          _get_embedding_variable_name(scope_name, var_name)
-      )
-      slot_variable_names_by_table[table_name] = _get_slot_variable_names(
-          scope_name, var_name, optimization_parameters)
+    embedding_variable_name_by_table[table_name] = (
+        _get_embedding_variable_name(scope_name, var_name))
+    slot_variable_names_by_table[table_name] = _get_slot_variable_names(
+        scope_name, var_name, optimization_parameters)
 
   graph.clear_collection(tpu_fc._TPU_FC_TO_SCOPE)  # pylint: disable=protected-access
   return embedding_variable_name_by_table, slot_variable_names_by_table
-- 
GitLab


From d0a7af2368df44430513903398f63426ba1c8800 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Sat, 9 Feb 2019 15:36:49 -0800
Subject: [PATCH 2229/2345] More changes to support tpu strategy in tf
 functions

PiperOrigin-RevId: 233235171
---
 tensorflow/contrib/distribute/python/tpu_strategy.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 158247295b..2d9d221f42 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -161,7 +161,7 @@ def _create_tpu_mirrored_variable(  # pylint: disable=missing-docstring
         strategy, device_map, value_list, aggregation,
         logical_device=logical_device)
 
-  if not context.executing_eagerly():
+  if not (context.executing_eagerly() or ops.inside_function()):
     g = ops.get_default_graph()
     # If "trainable" is True, next_creator() will add the member variables
     # to the TRAINABLE_VARIABLES collection, so we manually remove
@@ -536,9 +536,10 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
             # name as the absolute name of the variable.
             kwargs["name"] = "%s/replica_%d/" % (var0name, i)
             # Initialize replicas with the same value:
-            if context.executing_eagerly():
-              kwargs["initial_value"] = array_ops.identity(
-                  value_list[0].value())
+            if context.executing_eagerly() or ops.inside_function():
+              with ops.init_scope():
+                kwargs["initial_value"] = array_ops.identity(
+                    value_list[0].value())
             else:
               def initial_value_fn(device=d):
                 with ops.device(device):
-- 
GitLab


From bce5bff90a6e7f44e0f9e434c8dd5481c6f9b5c1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 9 Feb 2019 19:05:23 -0800
Subject: [PATCH 2230/2345] Remove unused lambda captures and unused
 using-declarations.

PiperOrigin-RevId: 233246410
---
 tensorflow/compiler/jit/xla_cluster_util.cc               | 2 +-
 tensorflow/compiler/jit/xla_device.cc                     | 2 +-
 tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc  | 5 ++---
 tensorflow/compiler/tf2xla/xla_context.cc                 | 8 ++++----
 tensorflow/compiler/xla/literal.cc                        | 1 -
 tensorflow/compiler/xla/service/cpu/ir_emitter.cc         | 1 -
 .../xla/service/gpu/cudnn_conv_algorithm_picker.cc        | 1 -
 tensorflow/compiler/xla/service/indexed_array_analysis.cc | 1 -
 tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc     | 2 --
 9 files changed, 8 insertions(+), 15 deletions(-)

diff --git a/tensorflow/compiler/jit/xla_cluster_util.cc b/tensorflow/compiler/jit/xla_cluster_util.cc
index 80993861ab..3adcfef4da 100644
--- a/tensorflow/compiler/jit/xla_cluster_util.cc
+++ b/tensorflow/compiler/jit/xla_cluster_util.cc
@@ -43,7 +43,7 @@ string DescribeCycle(const GraphCycles* cycles, const Graph& graph, int src,
     return "";
   }
 
-  auto node_name = [cycles, &graph](int node_id) {
+  auto node_name = [&graph](int node_id) {
     if (!FastBoundsCheck(node_id, graph.num_node_ids())) {
       return string("(null)");
     }
diff --git a/tensorflow/compiler/jit/xla_device.cc b/tensorflow/compiler/jit/xla_device.cc
index c67b4f11b0..92229842bd 100644
--- a/tensorflow/compiler/jit/xla_device.cc
+++ b/tensorflow/compiler/jit/xla_device.cc
@@ -428,7 +428,7 @@ void XlaDevice::Sync(const DoneCallback& done) {
   // moment--when ThenEnqueueOnBackgroundThread is called--will have finished.
   // This achieves a device-wide sync.
   stream->ThenEnqueueOnBackgroundThread(
-      [this, stream, done](se::StreamExecutor*) {
+      [stream, done](se::StreamExecutor*) {
         tracing::ScopedActivity activity("XlaDevice::Sync::Callback",
                                          /*is_expensive=*/true);
         done(stream->ok() ? Status::OK()
diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
index 79b1cba329..2168370391 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
@@ -94,7 +94,6 @@ bool IsEngineOutput(absl::string_view name) {
 namespace convert {
 using absl::StrAppend;
 using absl::StrCat;
-using ::tensorflow::str_util::Split;
 
 inline tensorflow::Status ConvertDType(tensorflow::DataType tf_dtype,
                                        nvinfer1::DataType* trt_dtype) {
@@ -297,8 +296,8 @@ Status Converter::GetTrtBroadcastShape(
 
   const int max_nb_dims = nvinfer1::Dims::MAX_DIMS + 1;
   auto compute_output_dims =
-      [max_nb_dims](const TRT_TensorOrWeights& input, int broadcast_num_dims,
-                    int* output_dims_array, nvinfer1::Dims* output_dims) {
+      [](const TRT_TensorOrWeights& input, int broadcast_num_dims,
+         int* output_dims_array, nvinfer1::Dims* output_dims) {
         const nvinfer1::Dims input_dims = input.GetTrtDims();
         std::fill(output_dims_array, output_dims_array + max_nb_dims, 1);
         std::copy(input_dims.d, input_dims.d + input_dims.nbDims,
diff --git a/tensorflow/compiler/tf2xla/xla_context.cc b/tensorflow/compiler/tf2xla/xla_context.cc
index 6139bf3cea..3f787fd86c 100644
--- a/tensorflow/compiler/tf2xla/xla_context.cc
+++ b/tensorflow/compiler/tf2xla/xla_context.cc
@@ -76,7 +76,7 @@ XlaResource* XlaContext::AddResource(std::unique_ptr<XlaResource> resource) {
 }
 
 const xla::XlaComputation* XlaContext::GetOrCreateMax(const DataType type) {
-  return LookupOrCreate(type, &max_func_, [this, type] {
+  return LookupOrCreate(type, &max_func_, [type] {
     const string type_string = DataTypeString(type);
     VLOG(1) << "Building Max() for " << type_string;
     xla::XlaBuilder b("max<" + type_string + ">");
@@ -92,7 +92,7 @@ const xla::XlaComputation* XlaContext::GetOrCreateMax(const DataType type) {
 }
 
 const xla::XlaComputation* XlaContext::GetOrCreateMin(const DataType type) {
-  return LookupOrCreate(type, &min_func_, [this, type] {
+  return LookupOrCreate(type, &min_func_, [type] {
     const string type_string = DataTypeString(type);
     VLOG(1) << "Building Min() for " << type_string;
     xla::XlaBuilder b("min<" + type_string + ">");
@@ -108,7 +108,7 @@ const xla::XlaComputation* XlaContext::GetOrCreateMin(const DataType type) {
 }
 
 const xla::XlaComputation* XlaContext::GetOrCreateAdd(const DataType type) {
-  return LookupOrCreate(type, &add_func_, [this, type] {
+  return LookupOrCreate(type, &add_func_, [type] {
     const string type_string = DataTypeString(type);
     VLOG(1) << "Building Add() for " << type_string;
     xla::XlaBuilder b("add<" + type_string + ">");
@@ -124,7 +124,7 @@ const xla::XlaComputation* XlaContext::GetOrCreateAdd(const DataType type) {
 }
 
 const xla::XlaComputation* XlaContext::GetOrCreateMul(const DataType type) {
-  return LookupOrCreate(type, &mul_func_, [this, type] {
+  return LookupOrCreate(type, &mul_func_, [type] {
     const string type_string = DataTypeString(type);
     VLOG(1) << "Building Mul() for " << type_string;
     xla::XlaBuilder b("mul<" + type_string + ">");
diff --git a/tensorflow/compiler/xla/literal.cc b/tensorflow/compiler/xla/literal.cc
index bb9bca0986..5cd738d0f7 100644
--- a/tensorflow/compiler/xla/literal.cc
+++ b/tensorflow/compiler/xla/literal.cc
@@ -44,7 +44,6 @@ namespace xla {
 namespace {
 
 using absl::StrCat;
-using absl::StrFormat;
 
 constexpr bool kLittleEndian = __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__;
 
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index dd631896ba..5abb3eb387 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -77,7 +77,6 @@ namespace {
 using llvm_ir::AsStringRef;
 using llvm_ir::IrName;
 using llvm_ir::SetToFirstInsertPoint;
-namespace gtl = tensorflow::gtl;
 }  // namespace
 
 namespace cpu {
diff --git a/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.cc b/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.cc
index bc5170a9b1..603af5a654 100644
--- a/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.cc
+++ b/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.cc
@@ -35,7 +35,6 @@ namespace {
 
 using absl::optional;
 using se::DeviceMemoryBase;
-using se::dnn::AlgorithmConfig;
 using se::dnn::AlgorithmDesc;
 
 class ScratchAllocator : public se::ScratchAllocator {
diff --git a/tensorflow/compiler/xla/service/indexed_array_analysis.cc b/tensorflow/compiler/xla/service/indexed_array_analysis.cc
index 76bf48870d..c5d32a4b9a 100644
--- a/tensorflow/compiler/xla/service/indexed_array_analysis.cc
+++ b/tensorflow/compiler/xla/service/indexed_array_analysis.cc
@@ -27,7 +27,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/util.h"
 
 namespace xla {
-namespace gtl = ::tensorflow::gtl;
 
 namespace {
 using Analysis = IndexedArrayAnalysis;
diff --git a/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc b/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc
index c7337e8caa..7b7b8f5d02 100644
--- a/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc
+++ b/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc
@@ -40,8 +40,6 @@ limitations under the License.
 namespace xla {
 namespace {
 
-namespace gtl = ::tensorflow::gtl;
-
 class HloProfileTest : public ClientLibraryTestBase {};
 
 struct ParsedProfileOutputLine {
-- 
GitLab


From ead36c50c1865dc0fc8fdc1b4b20eb37e4879420 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 9 Feb 2019 19:33:32 -0800
Subject: [PATCH 2231/2345] Use "empty" instead of "size"-based checks, and
 replace C-style casts by static_cast.

PiperOrigin-RevId: 233247794
---
 .../compiler/tf2tensorrt/convert/convert_graph.cc  |  6 +++---
 .../compiler/tf2tensorrt/convert/convert_nodes.cc  |  6 +++---
 .../tf2tensorrt/convert/trt_optimization_pass.cc   | 14 +++++++-------
 .../tf2tensorrt/utils/trt_int8_calibrator.cc       |  2 +-
 .../compiler/tf2tensorrt/utils/trt_resources.cc    |  2 +-
 tensorflow/compiler/xla/tests/hlo_test_base.cc     |  8 +++++---
 6 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc
index f3db42509e..d829895239 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc
@@ -222,7 +222,7 @@ tensorflow::Status ConvertCalibGraphToInferGraph(
         cres->thr_->join();
         const auto& calibration_table =
             cres->calibrator_->getCalibrationTableAsString();
-        if (!calibration_table.size()) {
+        if (calibration_table.empty()) {
           LOG(ERROR) << "Calibration table is empty";
           return tensorflow::errors::Unknown(
               "Calibration table is missing. This shouldn't have happened!");
@@ -662,8 +662,8 @@ tensorflow::Status CreateTRTNode(const std::vector<EngineInfo>& infos, int pos,
         info.use_calibration,
         /*convert_successfully=*/nullptr));
     TrtUniquePtrType<nvinfer1::IHostMemory> engine_data(engine->serialize());
-    segment_string =
-        string((const char*)engine_data->data(), engine_data->size());
+    segment_string = string(static_cast<const char*>(engine_data->data()),
+                            engine_data->size());
     if (calibrate_int8) {
       // See above comment about why not putting this inside the 'else' branch.
       segment_string = info.segment_graph_def.SerializeAsString();
diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
index 2168370391..ce8a69b0d3 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
@@ -2106,7 +2106,7 @@ tensorflow::Status ConvertSqueeze(OpConverterParams* params) {
   // Mark axes to remove by setting them to 0.
   TFAttrs attrs(node_def);
   auto squeeze_dims = attrs.get<std::vector<int>>("squeeze_dims");
-  if (squeeze_dims.size() == 0) {
+  if (squeeze_dims.empty()) {
     return tensorflow::errors::Unimplemented(
         "Squeeze is only implemented for explicit dims, at ", node_def.name());
   }
@@ -2274,7 +2274,7 @@ tensorflow::Status ConvertStridedSlice(OpConverterParams* params) {
       pad_dims.push_back(i);
     }
   }
-  if (pad_dims.size() == 0) {
+  if (pad_dims.empty()) {
     // No dimensions are changed. We could create a padding layer anyway with
     // values of 0.
     if (params->validation_only) return Status::OK();
@@ -3138,7 +3138,7 @@ tensorflow::Status ConvertPad(OpConverterParams* params) {
   }
 
   // No padding at all, we should exit
-  if (pad_index.size() == 0) {
+  if (pad_index.empty()) {
     params->outputs->push_back(inputs.at(0));
     return tensorflow::Status::OK();
   }
diff --git a/tensorflow/compiler/tf2tensorrt/convert/trt_optimization_pass.cc b/tensorflow/compiler/tf2tensorrt/convert/trt_optimization_pass.cc
index f36aa558ea..0eedfcacb4 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/trt_optimization_pass.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/trt_optimization_pass.cc
@@ -87,7 +87,7 @@ void TRTOptimizationPass::PrintDebugInfo(
     LOG(INFO) << offset << "type             = " << cluster->type();
     LOG(INFO) << offset << "num warmup steps = " << cluster->NumWarmupSteps();
     const auto dev_names = cluster->GetDeviceNames();
-    if (dev_names.size()) {
+    if (!dev_names.empty()) {
       LOG(INFO) << offset << " Device names:";
       for (const auto s : dev_names) {
         LOG(INFO) << offset2 << s;
@@ -103,7 +103,7 @@ void TRTOptimizationPass::PrintDebugInfo(
     }
 
     const auto dev_props = cluster->GetDevices();
-    if (dev_props.size()) {
+    if (!dev_props.empty()) {
       LOG(INFO) << offset << "Device properties:";
       for (auto k : dev_props) {
         LOG(INFO) << offset2 << k.first;
@@ -131,7 +131,7 @@ void TRTOptimizationPass::PrintDebugInfo(
     }
   }
   LOG(INFO) << "item: " << item.id;
-  if (item.feed.size()) {
+  if (!item.feed.empty()) {
     LOG(INFO) << offset << "Feeds  :";
     for (const auto& f : item.feed) {
       const auto& shape = f.second.shape();
@@ -140,7 +140,7 @@ void TRTOptimizationPass::PrintDebugInfo(
   } else {
     LOG(INFO) << offset << "No Feeds";
   }
-  if (item.fetch.size()) {
+  if (!item.fetch.empty()) {
     LOG(INFO) << offset << "Fetches  :";
     for (const auto& f : item.fetch) {
       LOG(INFO) << offset2 << f;
@@ -149,7 +149,7 @@ void TRTOptimizationPass::PrintDebugInfo(
     LOG(INFO) << offset << "No Fetches";
   }
 
-  if (item.init_ops.size()) {
+  if (!item.init_ops.empty()) {
     LOG(INFO) << offset << "init ops  :";
     for (const auto& f : item.init_ops) {
       LOG(INFO) << offset2 << f;
@@ -160,7 +160,7 @@ void TRTOptimizationPass::PrintDebugInfo(
   LOG(INFO) << "Save Op = " << item.save_op;
   LOG(INFO) << "Restore Op = " << item.restore_op;
   LOG(INFO) << "save_restore_loc_tensor = " << item.save_restore_loc_tensor;
-  if (item.keep_ops.size()) {
+  if (!item.keep_ops.empty()) {
     LOG(INFO) << offset << "keep ops  :";
     for (const auto& f : item.keep_ops) {
       LOG(INFO) << offset2 << f;
@@ -197,7 +197,7 @@ tensorflow::Status TRTOptimizationPass::Optimize(
     PrintDebugInfo(cluster, item);
   }
   int max_dim = -1;
-  if (item.feed.size()) {
+  if (!item.feed.empty()) {
     for (const auto& f : item.feed) {
       const auto& shape = f.second.shape();
       if (shape.dims() > 0) {
diff --git a/tensorflow/compiler/tf2tensorrt/utils/trt_int8_calibrator.cc b/tensorflow/compiler/tf2tensorrt/utils/trt_int8_calibrator.cc
index bf111d3a2e..5213fced1e 100644
--- a/tensorflow/compiler/tf2tensorrt/utils/trt_int8_calibrator.cc
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_int8_calibrator.cc
@@ -135,7 +135,7 @@ void TRTInt8Calibrator::setDone() {
 
 void TRTInt8Calibrator::writeCalibrationCache(const void* ptr,
                                               std::size_t length) {
-  calibration_table_ = string((const char*)ptr, length);
+  calibration_table_ = string(static_cast<const char*>(ptr), length);
   VLOG(1) << "Got calibration data for " << engine_name_ << " @" << ptr
           << " length=" << length;
 }
diff --git a/tensorflow/compiler/tf2tensorrt/utils/trt_resources.cc b/tensorflow/compiler/tf2tensorrt/utils/trt_resources.cc
index 37f7fe99fb..2e553079b1 100644
--- a/tensorflow/compiler/tf2tensorrt/utils/trt_resources.cc
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_resources.cc
@@ -48,7 +48,7 @@ Status TRTCalibrationResource::SerializeToString(string* serialized) {
   calibrator_->waitAndSetDone();
   thr_->join();
   *serialized = calibrator_->getCalibrationTableAsString();
-  if (!serialized->size()) {
+  if (serialized->empty()) {
     return tensorflow::errors::Unknown("Calibration table is empty.");
   }
   return Status::OK();
diff --git a/tensorflow/compiler/xla/tests/hlo_test_base.cc b/tensorflow/compiler/xla/tests/hlo_test_base.cc
index 38466b175e..d9d54fd255 100644
--- a/tensorflow/compiler/xla/tests/hlo_test_base.cc
+++ b/tensorflow/compiler/xla/tests/hlo_test_base.cc
@@ -400,7 +400,7 @@ StatusOr<::testing::AssertionResult> HloTestBase::RunAndCompareInternal(
       module->set_config(config);
     }
 
-    if (backend_config != "") {
+    if (!backend_config.empty()) {
       // Set backend configuration if it is given.
       HloInstruction* instruction =
           module->entry_computation()->root_instruction();
@@ -409,9 +409,10 @@ StatusOr<::testing::AssertionResult> HloTestBase::RunAndCompareInternal(
 
     auto executable =
         test_runner_.CreateExecutable(std::move(module), run_hlo_passes);
-    if (!executable.ok())
+    if (!executable.ok()) {
       return ::testing::AssertionFailure()
              << executable.status().error_message();
+    }
     executables[i] = std::move(executable.ValueOrDie());
   }
 
@@ -419,8 +420,9 @@ StatusOr<::testing::AssertionResult> HloTestBase::RunAndCompareInternal(
     auto output =
         test_runner_.Execute(std::move(executables[i]), fake_argument_ptrs[i],
                              /*profile=*/&((*profiles)[i]));
-    if (!output.ok())
+    if (!output.ok()) {
       return ::testing::AssertionFailure() << output.status().error_message();
+    }
   }
 
   return ::testing::AssertionSuccess();
-- 
GitLab


From 43947b9efcc9fc4a3f525d51e80e164e7e3c0221 Mon Sep 17 00:00:00 2001
From: Shivani Agrawal <shivaniagrawal@google.com>
Date: Sat, 9 Feb 2019 19:56:38 -0800
Subject: [PATCH 2232/2345] [tf.data] correction in document.

PiperOrigin-RevId: 233249008
---
 tensorflow/python/data/ops/dataset_ops.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index 32a9c755e4..ab7a362c77 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -2147,8 +2147,8 @@ class StructuredFunctionWrapper(object):
         default graph.
       use_legacy_function: (Optional.) A boolean that determines whether the
         function be created using `tensorflow.python.eager.function.defun`
-        (default behavior) or `tensorflow.python.eager.function.Defun` (legacy
-        beheavior).
+        (default behavior) or `tensorflow.python.framework.function.Defun`
+        (legacy beheavior).
       defun_kwargs: (Optional.) A dictionary mapping string argument names to
         values. If supplied, will be passed to `function` as keyword arguments.
 
-- 
GitLab


From 8658f093ce3518d26b192ec9c58d5bf8564e5852 Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Sat, 9 Feb 2019 21:06:33 -0800
Subject: [PATCH 2233/2345] Fixed the bug validation_freq is not passed to
 TPUStrategy loop.

PiperOrigin-RevId: 233252849
---
 tensorflow/python/keras/engine/training_distributed.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index e05b6a08f8..2d79afef76 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -113,7 +113,7 @@ def fit_distributed(model,
         initial_epoch=initial_epoch,
         steps_per_epoch=steps_per_epoch,
         validation_steps=validation_steps,
-        validation_freq=1)
+        validation_freq=validation_freq)
   else:
     return training_arrays.fit_loop(
         model,
-- 
GitLab


From beccb5be1d9a71082b74d8fa90367b0faf6e5226 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 10 Feb 2019 01:02:41 -0800
Subject: [PATCH 2234/2345] compat: Update forward compatibility horizon to
 2019-02-10

PiperOrigin-RevId: 233265475
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 1dd10fadfd..bef61969ab 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 2, 9)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 2, 10)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 753cb67ab58f7b55fc5061f263654bce3828765b Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Sun, 10 Feb 2019 11:28:52 -0800
Subject: [PATCH 2235/2345] TOCO: Print error message if conversion fails.

PiperOrigin-RevId: 233301060
---
 tensorflow/lite/toco/toco.cc | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/toco/toco.cc b/tensorflow/lite/toco/toco.cc
index 4a3d6a5848..aa7e43350c 100644
--- a/tensorflow/lite/toco/toco.cc
+++ b/tensorflow/lite/toco/toco.cc
@@ -49,5 +49,10 @@ int main(int argc, char** argv) {
   }
   toco::port::InitGoogle(argv[0], effective_argc, &effective_argv, true);
   auto status = toco::Convert(parsed_toco_flags, parsed_model_flags);
-  return status.ok() ? 0 : -1;
+  if (!status.ok()) {
+    fprintf(stderr, "%s\n", status.error_message().c_str());
+    fflush(stderr);
+    return 1;
+  }
+  return 0;
 }
-- 
GitLab


From f5015045833910b60731a487b77eedad1d322c2e Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Sun, 10 Feb 2019 17:58:37 -0800
Subject: [PATCH 2236/2345] Fix deprecation warning with defaults for conv1d
 arguments

PiperOrigin-RevId: 233324571
---
 tensorflow/python/ops/nn_ops.py                    | 8 ++++----
 tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 967b29f17e..5e61282db6 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -4114,10 +4114,10 @@ def fractional_avg_pool_v2(value,
     warn_once=True,
     data_format="NHWC")
 def conv1d(
-    value,
-    filters,
-    stride,
-    padding,
+    value=None,
+    filters=None,
+    stride=None,
+    padding=None,
     use_cudnn_on_gpu=None,
     data_format=None,
     name=None,
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
index c7061a7f79..cd8fa80d1c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
@@ -54,7 +54,7 @@ tf_module {
   }
   member_method {
     name: "conv1d"
-    argspec: "args=[\'value\', \'filters\', \'stride\', \'padding\', \'use_cudnn_on_gpu\', \'data_format\', \'name\', \'input\', \'dilations\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'value\', \'filters\', \'stride\', \'padding\', \'use_cudnn_on_gpu\', \'data_format\', \'name\', \'input\', \'dilations\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "conv1d_transpose"
-- 
GitLab


From 29b27f3066ac99d3f01c65115bdd2b6cd4e2ccc7 Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Sun, 10 Feb 2019 20:57:59 -0800
Subject: [PATCH 2237/2345] Fix tf.batch_gather rewrite rule

tf.batch_gather cannot be simply changed to tf.gather without setting
the batch_dims arg.

PiperOrigin-RevId: 233338117
---
 tensorflow/tools/compatibility/tf_upgrade_v2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 007f9b40ba..4d87358829 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -423,7 +423,7 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.batch_to_space_nd":
             "tf.batch_to_space",
         "tf.batch_gather":
-            "tf.gather",
+            "tf.compat.v1.batch_gather",
         "tf.space_to_batch_nd":
             "tf.space_to_batch",
         "tf.nn.space_to_batch":
-- 
GitLab


From 126679b2e517ba5668c885a17acb75fe8d3cceef Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 11 Feb 2019 01:02:53 -0800
Subject: [PATCH 2238/2345] compat: Update forward compatibility horizon to
 2019-02-11

PiperOrigin-RevId: 233359030
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index bef61969ab..2cd1ec0c8d 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 2, 10)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 2, 11)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From bfba64f11e102bf77342b990d7b0fe0406cd839f Mon Sep 17 00:00:00 2001
From: Andr? Susano Pinto <andresp@google.com>
Date: Mon, 11 Feb 2019 03:38:41 -0800
Subject: [PATCH 2239/2345] Add support to serialize TrackableResources in
 newer SavedModel format.

This works both for loading the SavedModels in graph and eager mode.

PiperOrigin-RevId: 233377934
---
 .../saved_model/function_deserialization.py   |  6 ++
 tensorflow/python/saved_model/load.py         | 28 ++++++++
 tensorflow/python/saved_model/load_test.py    | 68 +++++++++++++------
 tensorflow/python/saved_model/save.py         | 16 ++---
 .../saved_model/saved_object_graph.proto      |  7 ++
 .../training/checkpointable/tracking.py       | 16 +++++
 6 files changed, 113 insertions(+), 28 deletions(-)

diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py
index 992b62ac70..ff48f5cfab 100644
--- a/tensorflow/python/saved_model/function_deserialization.py
+++ b/tensorflow/python/saved_model/function_deserialization.py
@@ -279,6 +279,12 @@ def _fix_fdef(orig_fdef, functions):
       if attr_value.func.name:
         attr_value.func.name = functions[attr_value.func.name].name
 
+    # TODO(b/124205571): Avoid accidental sharing and destruction of restored
+    # resources. For now drop "shared_name" when loading functions to avoid
+    # sharing.
+    if "shared_name" in node_def.attr:
+      del node_def.attr["shared_name"]
+
   fdef.signature.name = _clean_function_name(fdef.signature.name)
   return fdef
 
diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
index 4511d3a91c..32691d2cc1 100644
--- a/tensorflow/python/saved_model/load.py
+++ b/tensorflow/python/saved_model/load.py
@@ -122,6 +122,9 @@ class _Loader(object):
         return obj.asset_path
       elif tensor_util.is_tensor(obj):
         return obj
+      elif isinstance(obj, tracking.TrackableResource):
+        # Note: this executes restored functions in the TrackableResource.
+        return obj.resource_handle
       raise ValueError("Can't convert node %s to tensor" % (type(obj)))
 
   def _load_all(self):
@@ -187,6 +190,7 @@ class _Loader(object):
             proto.bare_concrete_function),
         "variable": lambda: self._recreate_variable(proto.variable),
         "constant": lambda: self._recreate_constant(proto.constant),
+        "resource": lambda: self._recreate_resource(proto.resource),
     }
     kind = proto.WhichOneof("kind")
     if kind not in factory:
@@ -232,6 +236,30 @@ class _Loader(object):
         tensor_util.MakeNdarray(tensor_proto))
     return imported_constant, setattr
 
+  def _recreate_resource(self, proto):
+    del proto
+    return _RestoredResource(), setattr
+
+
+# TODO(b/124205571,b/124092991): Solve destruction of resources.
+class _RestoredResource(tracking.TrackableResource):
+  """Restored SavedResource."""
+
+  def create_resource(self):
+    raise RuntimeError()
+
+  def initialize(self):
+    raise RuntimeError()
+
+  def _list_functions_for_serialization(self):
+    # Overwrite this method to avoid the implementation of
+    # base class to re-wrap the polymorphic functions into
+    # another layer of `tf.function`.
+    return {
+        "create_resource": self.create_resource,
+        "initialize": self.initialize,
+    }
+
 
 def _call_attribute(instance, *args, **kwargs):
   return instance.__call__(*args, **kwargs)
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index 8149290f12..edb91b473e 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -882,26 +882,6 @@ class LoadTest(test.TestCase, parameterized.TestCase):
     self.assertIs(f_concrete.captured_inputs[0],
                   g_concrete.captured_inputs[0])
 
-  def test_table(self, cycles):
-    # TODO(b/123408779): Handle generic TrackableResources and enable this test
-    self.skipTest("Need to handle generic TrackableResources")
-    vocab_path = self._make_asset("alpha\nbeta\ngamma\n")
-    initializer = lookup_ops.TextFileInitializer(
-        vocab_path,
-        key_dtype=dtypes.string,
-        key_index=lookup_ops.TextFileIndex.WHOLE_LINE,
-        value_dtype=dtypes.int64,
-        value_index=lookup_ops.TextFileIndex.LINE_NUMBER)
-    root = util.Checkpoint(table=lookup_ops.HashTable(
-        initializer, default_value=-1))
-    root.table_user = def_function.function(
-        root.table.lookup,
-        input_signature=[tensor_spec.TensorSpec(None, dtypes.string)])
-    self.assertEqual(2, root.table_user(constant_op.constant("gamma")).numpy())
-    imported = self.cycle(root, cycles)
-    self.assertEqual(
-        2, imported.table_user(constant_op.constant("gamma")).numpy())
-
   def test_functions_accessed_once(self, cycles):
 
     class Exported(tracking.AutoCheckpointable):
@@ -964,6 +944,54 @@ class LoadTest(test.TestCase, parameterized.TestCase):
       # The signatures mapping is immutable
       imported.signatures["random_key"] = 3
 
+  def _make_model_with_tables(self):
+    default_val = -1
+    keys = constant_op.constant(["brain", "salad", "surgery"])
+    values = constant_op.constant([0, 1, 2], dtypes.int64)
+    table1_initializer = lookup_ops.KeyValueTensorInitializer(keys, values)
+    table1 = lookup_ops.HashTable(table1_initializer, default_val)
+
+    table2_file = self._make_asset("test\nfoo\nbrain\n")
+    table2_initializer = lookup_ops.TextFileIdTableInitializer(table2_file)
+    table2 = lookup_ops.HashTable(table2_initializer, default_val)
+
+    def _make_lookup_function(table):
+      signature = [tensor_spec.TensorSpec(None, dtypes.string)]
+      return def_function.function(input_signature=signature)(
+          lambda x: table.lookup(x))  # pylint: disable=unnecessary-lambda
+
+    root = tracking.AutoCheckpointable()
+    root.table1 = table1
+    root.lookup1 = _make_lookup_function(table1)
+    root.table2 = table2
+    root.lookup2 = _make_lookup_function(table2)
+    return root
+
+  def test_table(self, cycles):
+    root = self._make_model_with_tables()
+    imported = self.cycle(root, cycles, signatures={})
+    keys = constant_op.constant(["brain", "test", "foo", "surgery"])
+    self.assertAllEqual([0, -1, -1, 2], imported.lookup1(keys).numpy())
+    self.assertAllEqual([2, 0, 1, -1], imported.lookup2(keys).numpy())
+
+  def test_table_in_graph(self, cycles):
+    root = self._make_model_with_tables()
+
+    if cycles > 1:
+      root = self.cycle(root, cycles - 1)
+    path = tempfile.mkdtemp(prefix=self.get_temp_dir())
+    save.save(root, path)
+    imported = self.cycle(root, 1)
+
+    with ops.Graph().as_default():
+      imported = load.load(path)
+      keys = constant_op.constant(["brain", "test", "foo", "surgery"])
+      output1 = imported.lookup1(keys)
+      output2 = imported.lookup2(keys)
+      with monitored_session.MonitoredSession() as sess:
+        self.assertAllEqual([0, -1, -1, 2], sess.run(output1))
+        self.assertAllEqual([2, 0, 1, -1], sess.run(output2))
+
 
 class SingleCycleTests(test.TestCase, parameterized.TestCase):
 
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index 29d8a89482..3180b6f169 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -273,14 +273,12 @@ def _map_captures_to_created_tensors(
   for exterior, interior in original_captures.items():
     mapped_resource = resource_map.get(exterior, None)
     if mapped_resource is None:
-      if exterior.dtype == dtypes.resource:
-        raise AssertionError(
-            ("Tried to export a function which references untracked stateful "
-             "object {}. Stateful TensorFlow objects (e.g. tf.Variable) must "
-             "be tracked by the main object. Objects may be tracked by "
-             "assigning them to an attribute of another tracked object, or to "
-             "an attribute of the main object directly.")
-            .format(interior))
+      raise AssertionError(
+          ("Tried to export a function which references untracked object {}."
+           "TensorFlow objects (e.g. tf.Variable) captured by functions must "
+           "be tracked by assigning them to an attribute of a tracked object "
+           "or assigned to an attribute of the main object directly.")
+          .format(interior))
     export_captures.append(mapped_resource)
   return export_captures
 
@@ -587,6 +585,8 @@ def _write_object_proto(obj, proto, asset_file_def_index):
         function_serialization.serialize_bare_concrete_function(obj))
   elif isinstance(obj, _CapturedConstant):
     proto.constant.operation = obj.graph_tensor.op.name
+  elif isinstance(obj, tracking.TrackableResource):
+    proto.resource.SetInParent()
   else:
     registered_type_proto = revived_types.serialize(obj)
     if registered_type_proto is None:
diff --git a/tensorflow/python/saved_model/saved_object_graph.proto b/tensorflow/python/saved_model/saved_object_graph.proto
index fe7346dc32..d4546cc7e1 100644
--- a/tensorflow/python/saved_model/saved_object_graph.proto
+++ b/tensorflow/python/saved_model/saved_object_graph.proto
@@ -60,6 +60,7 @@ message SavedObject {
     SavedVariable variable = 7;
     SavedBareConcreteFunction bare_concrete_function = 8;
     SavedConstant constant = 9;
+    SavedResource resource = 10;
   }
 }
 
@@ -153,3 +154,9 @@ message FunctionSpec {
   // The input signature, if specified.
   StructuredValue input_signature = 5;
 }
+
+// A SavedResource represents a TF object that holds state during its lifetime.
+message SavedResource {
+  // An object of this type can have a reference to a:
+  // create_resource() and an initialize() function.
+}
diff --git a/tensorflow/python/training/checkpointable/tracking.py b/tensorflow/python/training/checkpointable/tracking.py
index b17b963dc4..8ca088ded9 100644
--- a/tensorflow/python/training/checkpointable/tracking.py
+++ b/tensorflow/python/training/checkpointable/tracking.py
@@ -176,6 +176,22 @@ class TrackableResource(base.Checkpointable):
       self._resource_handle = self.create_resource()
     return self._resource_handle
 
+  def _list_functions_for_serialization(self):
+    @def_function.function(input_signature=[], autograph=False)
+    def _creator():
+      resource = self.create_resource()
+      return resource
+
+    @def_function.function(input_signature=[], autograph=False)
+    def _initializer():
+      self.initialize()
+      return 1  # Dummy return
+
+    return {
+        "create_resource": _creator,
+        "initialize": _initializer,
+    }
+
 
 class TrackableAsset(base.Checkpointable):
   """Base class for asset files which need to be tracked."""
-- 
GitLab


From 3589bbfbc68ddb8a4c87f90bf7c8dc87c5a50556 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 11 Feb 2019 07:43:37 -0800
Subject: [PATCH 2240/2345] Support any depth in optimized ops AveragePool and
 MaxPool.

PiperOrigin-RevId: 233405375
---
 .../internal/optimized/optimized_ops.h        | 296 ++++++++++--------
 tensorflow/lite/kernels/pooling_test.cc       |  68 ++++
 2 files changed, 232 insertions(+), 132 deletions(-)

diff --git a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
index 5dc03e7d53..1c0798c37c 100644
--- a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
+++ b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
@@ -3710,6 +3710,14 @@ inline void AveragePool(const PoolParams& params,
                         const uint8* input_data,
                         const RuntimeShape& output_shape, uint8* output_data) {
   gemmlowp::ScopedProfilingLabel label("AveragePool/8bit");
+
+  // Here, and in other pooling ops, in order to maintain locality of reference,
+  // to minimize some recalculations, and to load into NEON vector registers, we
+  // use an inner loop down the depth. Since depths can be large and hence we
+  // would need arbitrarily large temporary storage, we divide the work up into
+  // depth tranches just within the batch loop.
+  static constexpr int kPoolingAccTrancheSize = 256;
+
   TFLITE_DCHECK_LE(params.quantized_activation_min,
                    params.quantized_activation_max);
   TFLITE_DCHECK_EQ(input_shape.DimensionsCount(), 4);
@@ -3722,69 +3730,76 @@ inline void AveragePool(const PoolParams& params,
   const int output_width = output_shape.Dims(2);
   const int stride_height = params.stride_height;
   const int stride_width = params.stride_width;
+
+  uint16 acc[kPoolingAccTrancheSize];
   for (int batch = 0; batch < batches; ++batch) {
-    for (int out_y = 0; out_y < output_height; ++out_y) {
-      for (int out_x = 0; out_x < output_width; ++out_x) {
-        const int in_x_origin =
-            (out_x * stride_width) - params.padding_values.width;
-        const int in_y_origin =
-            (out_y * stride_height) - params.padding_values.height;
-        const int filter_x_start = std::max(0, -in_x_origin);
-        const int filter_x_end =
-            std::min(params.filter_width, input_width - in_x_origin);
-        const int filter_y_start = std::max(0, -in_y_origin);
-        const int filter_y_end =
-            std::min(params.filter_height, input_height - in_y_origin);
-        const int filter_count =
-            (filter_x_end - filter_x_start) * (filter_y_end - filter_y_start);
-        // 2560 is required by MobileNetV2 with depth multiplier 2.
-        static constexpr int kAccBufferMaxSize = 4096;
-        TFLITE_DCHECK_LE(depth, kAccBufferMaxSize);
-        uint16 acc[kAccBufferMaxSize];
-        memset(acc, 0, depth * sizeof(acc[0]));
-        const uint8* input_ptr =
-            input_data +
-            depth * (in_x_origin +
-                     input_width * (in_y_origin + input_height * batch));
-        for (int fy = filter_y_start; fy < filter_y_end; fy++) {
-          const uint8* input_row_ptr =
-              input_ptr + depth * (fy * input_width + filter_x_start);
-          for (int fx = filter_x_start; fx < filter_x_end; fx++) {
-            int channel = 0;
+    // We proceed through the depth in tranches (see comment above). The
+    // depth_base is the depth at the beginning of the tranche. The
+    // tranche_depth is the depth dimension of the tranche.
+    for (int depth_base = 0; depth_base < depth;
+         depth_base += kPoolingAccTrancheSize) {
+      const int tranche_depth =
+          std::min(depth - depth_base, kPoolingAccTrancheSize);
+      for (int out_y = 0; out_y < output_height; ++out_y) {
+        for (int out_x = 0; out_x < output_width; ++out_x) {
+          const int in_x_origin =
+              (out_x * stride_width) - params.padding_values.width;
+          const int in_y_origin =
+              (out_y * stride_height) - params.padding_values.height;
+          const int filter_x_start = std::max(0, -in_x_origin);
+          const int filter_x_end =
+              std::min(params.filter_width, input_width - in_x_origin);
+          const int filter_y_start = std::max(0, -in_y_origin);
+          const int filter_y_end =
+              std::min(params.filter_height, input_height - in_y_origin);
+          const int filter_count =
+              (filter_x_end - filter_x_start) * (filter_y_end - filter_y_start);
+          memset(acc, 0, tranche_depth * sizeof(acc[0]));
+          const uint8* input_ptr =
+              input_data + depth_base +
+              depth * (in_x_origin +
+                       input_width * (in_y_origin + input_height * batch));
+          for (int fy = filter_y_start; fy < filter_y_end; fy++) {
+            const uint8* input_row_ptr =
+                input_ptr + depth * (fy * input_width + filter_x_start);
+            for (int fx = filter_x_start; fx < filter_x_end; fx++) {
+              const uint8* input_channel_ptr = input_row_ptr;
+              int channel = 0;
 #ifdef USE_NEON
-            for (; channel <= depth - 16; channel += 16) {
-              uint16x8_t acc_reg[2];
-              for (int i = 0; i < 2; i++) {
-                acc_reg[i] = vld1q_u16(acc + channel + 8 * i);
+              for (; channel <= tranche_depth - 16; channel += 16) {
+                uint16x8_t acc_reg[2];
+                for (int i = 0; i < 2; i++) {
+                  acc_reg[i] = vld1q_u16(acc + channel + 8 * i);
+                }
+                uint8x16_t input_reg = vld1q_u8(input_channel_ptr);
+                input_channel_ptr += 16;
+                acc_reg[0] = vaddw_u8(acc_reg[0], vget_low_u8(input_reg));
+                acc_reg[1] = vaddw_u8(acc_reg[1], vget_high_u8(input_reg));
+                for (int i = 0; i < 2; i++) {
+                  vst1q_u16(acc + channel + 8 * i, acc_reg[i]);
+                }
               }
-              uint8x16_t input_reg = vld1q_u8(input_row_ptr);
-              input_row_ptr += 16;
-              acc_reg[0] = vaddw_u8(acc_reg[0], vget_low_u8(input_reg));
-              acc_reg[1] = vaddw_u8(acc_reg[1], vget_high_u8(input_reg));
-              for (int i = 0; i < 2; i++) {
-                vst1q_u16(acc + channel + 8 * i, acc_reg[i]);
+              for (; channel <= tranche_depth - 8; channel += 8) {
+                uint16x8_t acc_reg = vld1q_u16(acc + channel);
+                uint8x8_t input_reg = vld1_u8(input_channel_ptr);
+                input_channel_ptr += 8;
+                acc_reg = vaddw_u8(acc_reg, input_reg);
+                vst1q_u16(acc + channel, acc_reg);
               }
-            }
-            for (; channel <= depth - 8; channel += 8) {
-              uint16x8_t acc_reg = vld1q_u16(acc + channel);
-              uint8x8_t input_reg = vld1_u8(input_row_ptr);
-              input_row_ptr += 8;
-              acc_reg = vaddw_u8(acc_reg, input_reg);
-              vst1q_u16(acc + channel, acc_reg);
-            }
 #endif
-            for (; channel < depth; ++channel) {
-              acc[channel] += *input_row_ptr++;
+              for (; channel < tranche_depth; ++channel) {
+                acc[channel] += *input_channel_ptr++;
+              }
+              input_row_ptr += depth;
             }
           }
-        }
-        uint8* output_ptr =
-            output_data + Offset(output_shape, batch, out_y, out_x, 0);
-        int channel = 0;
+          uint8* output_ptr = output_data + Offset(output_shape, batch, out_y,
+                                                   out_x, depth_base);
+          int channel = 0;
 #ifdef USE_NEON
 #define AVGPOOL_DIVIDING_BY(FILTER_COUNT)                               \
   if (filter_count == FILTER_COUNT) {                                   \
-    for (; channel <= depth - 8; channel += 8) {                        \
+    for (; channel <= tranche_depth - 8; channel += 8) {                \
       uint16 buf[8];                                                    \
       for (int i = 0; i < 8; i++) {                                     \
         buf[i] = (acc[channel + i] + FILTER_COUNT / 2) / FILTER_COUNT;  \
@@ -3795,25 +3810,26 @@ inline void AveragePool(const PoolParams& params,
       vst1_u8(output_ptr + channel, buf8);                              \
     }                                                                   \
   }
-        AVGPOOL_DIVIDING_BY(9)
-        AVGPOOL_DIVIDING_BY(15)
+          AVGPOOL_DIVIDING_BY(9)
+          AVGPOOL_DIVIDING_BY(15)
 #undef AVGPOOL_DIVIDING_BY
-        for (; channel <= depth - 8; channel += 8) {
-          uint16 buf[8];
-          for (int i = 0; i < 8; i++) {
-            buf[i] = (acc[channel + i] + filter_count / 2) / filter_count;
+          for (; channel <= tranche_depth - 8; channel += 8) {
+            uint16 buf[8];
+            for (int i = 0; i < 8; i++) {
+              buf[i] = (acc[channel + i] + filter_count / 2) / filter_count;
+            }
+            uint8x8_t buf8 = vqmovn_u16(vld1q_u16(buf));
+            buf8 = vmin_u8(buf8, vdup_n_u8(params.quantized_activation_max));
+            buf8 = vmax_u8(buf8, vdup_n_u8(params.quantized_activation_min));
+            vst1_u8(output_ptr + channel, buf8);
           }
-          uint8x8_t buf8 = vqmovn_u16(vld1q_u16(buf));
-          buf8 = vmin_u8(buf8, vdup_n_u8(params.quantized_activation_max));
-          buf8 = vmax_u8(buf8, vdup_n_u8(params.quantized_activation_min));
-          vst1_u8(output_ptr + channel, buf8);
-        }
 #endif
-        for (; channel < depth; ++channel) {
-          uint16 a = (acc[channel] + filter_count / 2) / filter_count;
-          a = std::max<uint16>(a, params.quantized_activation_min);
-          a = std::min<uint16>(a, params.quantized_activation_max);
-          output_ptr[channel] = static_cast<uint8>(a);
+          for (; channel < tranche_depth; ++channel) {
+            uint16 a = (acc[channel] + filter_count / 2) / filter_count;
+            a = std::max<uint16>(a, params.quantized_activation_min);
+            a = std::min<uint16>(a, params.quantized_activation_max);
+            output_ptr[channel] = static_cast<uint8>(a);
+          }
         }
       }
     }
@@ -3878,6 +3894,14 @@ inline void MaxPool(const PoolParams& params, const RuntimeShape& input_shape,
                     const uint8* input_data, const RuntimeShape& output_shape,
                     uint8* output_data) {
   gemmlowp::ScopedProfilingLabel label("MaxPool/8bit");
+
+  // Here, and in other pooling ops, in order to maintain locality of reference,
+  // to minimize some recalculations, and to load into NEON vector registers, we
+  // use an inner loop down the depth. Since depths can be large and hence we
+  // would need arbitrarily large temporary storage, we divide the work up into
+  // depth tranches just within the batch loop.
+  static constexpr int kPoolingAccTrancheSize = 256;
+
   TFLITE_DCHECK_LE(params.quantized_activation_min,
                    params.quantized_activation_max);
   TFLITE_DCHECK_EQ(input_shape.DimensionsCount(), 4);
@@ -3890,77 +3914,85 @@ inline void MaxPool(const PoolParams& params, const RuntimeShape& input_shape,
   const int output_width = output_shape.Dims(2);
   const int stride_height = params.stride_height;
   const int stride_width = params.stride_width;
+
+  uint8 acc[kPoolingAccTrancheSize];
   for (int batch = 0; batch < batches; ++batch) {
-    for (int out_y = 0; out_y < output_height; ++out_y) {
-      for (int out_x = 0; out_x < output_width; ++out_x) {
-        const int in_x_origin =
-            (out_x * stride_width) - params.padding_values.width;
-        const int in_y_origin =
-            (out_y * stride_height) - params.padding_values.height;
-        const int filter_x_start = std::max(0, -in_x_origin);
-        const int filter_x_end =
-            std::min(params.filter_width, input_width - in_x_origin);
-        const int filter_y_start = std::max(0, -in_y_origin);
-        const int filter_y_end =
-            std::min(params.filter_height, input_height - in_y_origin);
-        // 2560 is required by MobileNetV2 with depth multiplier 2.
-        static constexpr int kAccBufferMaxSize = 4096;
-        TFLITE_DCHECK_LE(depth, kAccBufferMaxSize);
-        uint8 acc[kAccBufferMaxSize];
-        memset(acc, 0, depth * sizeof(acc[0]));
-        const uint8* input_ptr =
-            input_data +
-            depth * (in_x_origin +
-                     input_width * (in_y_origin + input_height * batch));
-        for (int fy = filter_y_start; fy < filter_y_end; fy++) {
-          const uint8* input_row_ptr =
-              input_ptr + depth * (fy * input_width + filter_x_start);
-          for (int fx = filter_x_start; fx < filter_x_end; fx++) {
-            int channel = 0;
+    // We proceed through the depth in tranches (see comment above). The
+    // depth_base is the depth at the beginning of the tranche. The
+    // tranche_depth is the depth dimension of the tranche.
+    for (int depth_base = 0; depth_base < depth;
+         depth_base += kPoolingAccTrancheSize) {
+      const int tranche_depth =
+          std::min(depth - depth_base, kPoolingAccTrancheSize);
+      for (int out_y = 0; out_y < output_height; ++out_y) {
+        for (int out_x = 0; out_x < output_width; ++out_x) {
+          const int in_x_origin =
+              (out_x * stride_width) - params.padding_values.width;
+          const int in_y_origin =
+              (out_y * stride_height) - params.padding_values.height;
+          const int filter_x_start = std::max(0, -in_x_origin);
+          const int filter_x_end =
+              std::min(params.filter_width, input_width - in_x_origin);
+          const int filter_y_start = std::max(0, -in_y_origin);
+          const int filter_y_end =
+              std::min(params.filter_height, input_height - in_y_origin);
+          memset(acc, 0, tranche_depth * sizeof(acc[0]));
+          const uint8* input_ptr =
+              input_data + depth_base +
+              depth * (in_x_origin +
+                       input_width * (in_y_origin + input_height * batch));
+          for (int fy = filter_y_start; fy < filter_y_end; fy++) {
+            const uint8* input_row_ptr =
+                input_ptr + depth * (fy * input_width + filter_x_start);
+            for (int fx = filter_x_start; fx < filter_x_end; fx++) {
+              const uint8* input_channel_ptr = input_row_ptr;
+              int channel = 0;
 #ifdef USE_NEON
-            for (; channel <= depth - 16; channel += 16) {
-              uint8x16_t acc_reg = vld1q_u8(acc + channel);
-              uint8x16_t input_reg = vld1q_u8(input_row_ptr);
-              input_row_ptr += 16;
-              acc_reg = vmaxq_u8(acc_reg, input_reg);
-              vst1q_u8(acc + channel, acc_reg);
-            }
+              for (; channel <= tranche_depth - 16; channel += 16) {
+                uint8x16_t acc_reg = vld1q_u8(acc + channel);
+                uint8x16_t input_reg = vld1q_u8(input_channel_ptr);
+                input_channel_ptr += 16;
+                acc_reg = vmaxq_u8(acc_reg, input_reg);
+                vst1q_u8(acc + channel, acc_reg);
+              }
 
-            for (; channel <= depth - 8; channel += 8) {
-              uint8x8_t acc_reg = vld1_u8(acc + channel);
-              uint8x8_t input_reg = vld1_u8(input_row_ptr);
-              input_row_ptr += 8;
-              acc_reg = vmax_u8(acc_reg, input_reg);
-              vst1_u8(acc + channel, acc_reg);
-            }
+              for (; channel <= tranche_depth - 8; channel += 8) {
+                uint8x8_t acc_reg = vld1_u8(acc + channel);
+                uint8x8_t input_reg = vld1_u8(input_channel_ptr);
+                input_channel_ptr += 8;
+                acc_reg = vmax_u8(acc_reg, input_reg);
+                vst1_u8(acc + channel, acc_reg);
+              }
 #endif
-            for (; channel < depth; ++channel) {
-              acc[channel] = std::max(acc[channel], *input_row_ptr++);
+              for (; channel < tranche_depth; ++channel) {
+                acc[channel] = std::max(acc[channel], *input_channel_ptr++);
+              }
+              input_row_ptr += depth;
             }
           }
-        }
-        uint8* output_ptr =
-            output_data + Offset(output_shape, batch, out_y, out_x, 0);
-        int channel = 0;
+          uint8* output_ptr = output_data + Offset(output_shape, batch, out_y,
+                                                   out_x, depth_base);
+          int channel = 0;
 #ifdef USE_NEON
-        for (; channel <= depth - 16; channel += 16) {
-          uint8x16_t a = vld1q_u8(acc + channel);
-          a = vminq_u8(a, vdupq_n_u8(params.quantized_activation_max));
-          a = vmaxq_u8(a, vdupq_n_u8(params.quantized_activation_min));
-          vst1q_u8(output_ptr + channel, a);
-        }
-        for (; channel <= depth - 8; channel += 8) {
-          uint8x8_t a = vld1_u8(acc + channel);
-          a = vmin_u8(a, vdup_n_u8(params.quantized_activation_max));
-          a = vmax_u8(a, vdup_n_u8(params.quantized_activation_min));
-          vst1_u8(output_ptr + channel, a);
-        }
+          for (; channel <= tranche_depth - 16; channel += 16) {
+            uint8x16_t a = vld1q_u8(acc + channel);
+            a = vminq_u8(a, vdupq_n_u8(params.quantized_activation_max));
+            a = vmaxq_u8(a, vdupq_n_u8(params.quantized_activation_min));
+            vst1q_u8(output_ptr + channel, a);
+          }
+          for (; channel <= tranche_depth - 8; channel += 8) {
+            uint8x8_t a = vld1_u8(acc + channel);
+            a = vmin_u8(a, vdup_n_u8(params.quantized_activation_max));
+            a = vmax_u8(a, vdup_n_u8(params.quantized_activation_min));
+            vst1_u8(output_ptr + channel, a);
+          }
 #endif
-        for (; channel < depth; ++channel) {
-          uint8 a = acc[channel];
-          a = std::max<uint8>(a, params.quantized_activation_min);
-          a = std::min<uint8>(a, params.quantized_activation_max);
-          output_ptr[channel] = static_cast<uint8>(a);
+          for (; channel < tranche_depth; ++channel) {
+            uint8 a = acc[channel];
+            a = std::max<uint8>(a, params.quantized_activation_min);
+            a = std::min<uint8>(a, params.quantized_activation_max);
+            output_ptr[channel] = static_cast<uint8>(a);
+          }
         }
       }
     }
diff --git a/tensorflow/lite/kernels/pooling_test.cc b/tensorflow/lite/kernels/pooling_test.cc
index 3e19bd2f64..4627d7a5f0 100644
--- a/tensorflow/lite/kernels/pooling_test.cc
+++ b/tensorflow/lite/kernels/pooling_test.cc
@@ -97,6 +97,24 @@ class SymmetricQuantizedPoolingOpModel : public BasePoolingOpModel {
   }
 };
 
+// Replicate each entry in a vector n times along depth (innermost dimension).
+// The values are incremented by delta, creating ramps offset by each input
+// value. This is used to create simple and predicatable variation.
+std::vector<float> ReplicateDepthRamp(const std::vector<float>& image_plane,
+                                      int n, float delta) {
+  const int size = image_plane.size();
+  std::vector<float> ramped_data(n * size);
+  // The input is treated as a 1-D even if logically it is multi-dimensional.
+  for (int input_index = 0; input_index < size; ++input_index) {
+    for (int depth = 0; depth < n; ++depth) {
+      ramped_data[n * input_index + depth] =
+          image_plane[input_index] + depth * delta;
+    }
+  }
+
+  return ramped_data;
+}
+
 TEST(FloatPoolingOpTest, AveragePool) {
   FloatPoolingOpModel m(BuiltinOperator_AVERAGE_POOL_2D,
                         /*input=*/{TensorType_FLOAT32, {1, 2, 4, 1}},
@@ -147,6 +165,31 @@ TEST(QuantizedPoolingOpTest, AveragePoolImageSize16) {
   EXPECT_THAT(m.GetDequantizedOutput(), ElementsAreArray(ArrayFloatNear({16})));
 }
 
+TEST(QuantizedPoolingOpTest, AveragePoolLargeDepth) {
+  // Test with a larger depth that is not a multiple of the tranche size, or of
+  // any register-oriented multiples such as 8 and 16.
+  constexpr int depth = 1999;  // Prime number.
+  QuantizedPoolingOpModel m(
+      BuiltinOperator_AVERAGE_POOL_2D,
+      /*input=*/{TensorType_UINT8, {1, 2, 4, depth}, 0, 15.9375},
+      /*filter_width=*/2, /*filter_height=*/2,
+      /*output=*/{TensorType_UINT8, {}, 0, 15.9375});
+
+  std::vector<float> input_image_plane({
+      0.f, 6.f, 2.f, 4.f,   //
+      3.f, 2.f, 10.f, 7.f,  //
+  });
+  std::vector<float> output_image_plane({2.75f, 5.75f});
+
+  m.SetInput(ReplicateDepthRamp(input_image_plane, depth, 1.f / 512.f));
+  m.Invoke();
+
+  EXPECT_THAT(m.GetDequantizedOutput(),
+              ElementsAreArray(ArrayFloatNear(
+                  ReplicateDepthRamp(output_image_plane, depth, 1.f / 512.f),
+                  1. / 32.f)));
+}
+
 // Test quantized AveragePool with int8 input and output. The input is the same
 // as the uint8 test QuantizedPoolingOpTest.AveragePool. The float output is
 // identical to uint8 test and quantized output is identical to uint8 test with
@@ -223,6 +266,31 @@ TEST(QuantizedUInt8PoolingOpTest, MaxPool) {
   EXPECT_THAT(m.GetOutput(), ElementsAreArray({96, 160}));
 }
 
+TEST(QuantizedPoolingOpTest, MaxPoolLargeDepth) {
+  // Test with a larger depth that is not a multiple of the tranche size, or of
+  // any register-oriented multiples such as 8 and 16.
+  constexpr int depth = 1999;  // Prime number.
+  QuantizedPoolingOpModel m(
+      BuiltinOperator_MAX_POOL_2D,
+      /*input=*/{TensorType_UINT8, {1, 2, 4, depth}, 0, 15.9375},
+      /*filter_width=*/2, /*filter_height=*/2,
+      /*output=*/{TensorType_UINT8, {}, 0, 15.9375});
+
+  std::vector<float> input_image_plane({
+      0.f, 6.f, 2.f, 4.f,   //
+      3.f, 2.f, 10.f, 7.f,  //
+  });
+  std::vector<float> output_image_plane({6.f, 10.f});
+
+  m.SetInput(ReplicateDepthRamp(input_image_plane, depth, 1.f / 512.f));
+  m.Invoke();
+
+  EXPECT_THAT(m.GetDequantizedOutput(),
+              ElementsAreArray(ArrayFloatNear(
+                  ReplicateDepthRamp(output_image_plane, depth, 1.f / 512.f),
+                  1. / 32.f)));
+}
+
 TEST(QuantizedInt8PoolingOpTest, MaxPool) {
   // Choose the input ranges carefully so that the dequantized output matches
   // the results of the float model above.
-- 
GitLab


From 90126f0059e58e690848a371adc36d3e0d116021 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Mon, 11 Feb 2019 08:30:18 -0800
Subject: [PATCH 2241/2345] Automated rollback of commit
 f395dbf81347e22608d691657b3674ad347d76dd

PiperOrigin-RevId: 233412184
---
 tensorflow/python/keras/BUILD | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index e0794c2b32..606ff972f6 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -381,10 +381,7 @@ tf_py_test(
         "//tensorflow/python:nn_ops",
     ],
     shard_count = 12,
-    tags = [
-        "no_oss",  # b/123899138
-        "notsan",
-    ],
+    tags = ["notsan"],
 )
 
 tf_py_test(
@@ -410,9 +407,6 @@ tf_py_test(
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
     ],
-    tags = [
-        "no_oss",  # b/123899138
-    ],
 )
 
 tf_py_test(
@@ -835,7 +829,6 @@ tf_py_test(
     ],
     shard_count = 4,
     tags = [
-        "no_oss",  # b/123899138
         "noasan",  # http://b/78599823
         "notsan",
     ],
-- 
GitLab


From 577a21198418c3931ed8701ece0b5c503691a802 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Mon, 11 Feb 2019 08:38:58 -0800
Subject: [PATCH 2242/2345] [tf.data] Increase SQLDataset test to fix an
 internal test timeout.

PiperOrigin-RevId: 233413393
---
 tensorflow/python/data/experimental/kernel_tests/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/BUILD b/tensorflow/python/data/experimental/kernel_tests/BUILD
index d0e5abcd5b..0c3c1072cb 100644
--- a/tensorflow/python/data/experimental/kernel_tests/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/BUILD
@@ -603,7 +603,7 @@ py_library(
 
 py_test(
     name = "sql_dataset_test",
-    size = "small",
+    size = "medium",
     srcs = ["sql_dataset_test.py"],
     srcs_version = "PY2AND3",
     tags = ["no_pip"],
-- 
GitLab


From d930e2edf0f87dbb5ab3dbd815ffc4ee4b64abe8 Mon Sep 17 00:00:00 2001
From: Joe Q <2413373+jquadrino@users.noreply.github.com>
Date: Mon, 11 Feb 2019 11:51:49 -0500
Subject: [PATCH 2243/2345] Add private methods from tensorflow logging module

---
 tensorflow/core/platform/s3/aws_logging.cc | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/platform/s3/aws_logging.cc b/tensorflow/core/platform/s3/aws_logging.cc
index 8eeb2c9492..57a9749458 100644
--- a/tensorflow/core/platform/s3/aws_logging.cc
+++ b/tensorflow/core/platform/s3/aws_logging.cc
@@ -69,13 +69,31 @@ void AWSLogSystem::LogMessage(Aws::Utils::Logging::LogLevel log_level,
 }
 
 namespace {
+    
+// Taken from tensorflow/core/platform/default/logging.cc
+int ParseInteger(const char* str, size_t size) {
+  string integer_str(str, size);
+  std::istringstream ss(integer_str);
+  int level = 0;
+  ss >> level;
+  return level;
+}
+
+// Taken from tensorflow/core/platform/default/logging.cc
+int64 LogLevelStrToInt(const char* tf_env_var_val) {
+  if (tf_env_var_val == nullptr) {
+    return 0;
+  }
+  return ParseInteger(tf_env_var_val, strlen(tf_env_var_val));
+}
+    
 static const char* kAWSLoggingTag = "AWSLogging";
 
 Aws::Utils::Logging::LogLevel ParseLogLevelFromEnv() {
   Aws::Utils::Logging::LogLevel log_level = Aws::Utils::Logging::LogLevel::Info;
 
   const int64_t level = 
-      getenv("AWS_LOG_LEVEL") ? tensorflow::internal::LogLevelStrToInt(getenv("AWS_LOG_LEVEL")) 
+      getenv("AWS_LOG_LEVEL") ? LogLevelStrToInt(getenv("AWS_LOG_LEVEL")) 
                               : tensorflow::internal::MinLogLevelFromEnv();
 
   switch (level) {
-- 
GitLab


From 32d88c6d4d95fbc108d2d2fd4b887ed8cad8f6f3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 11 Feb 2019 09:16:23 -0800
Subject: [PATCH 2244/2345] Internal fix

PiperOrigin-RevId: 233419552
---
 tensorflow/examples/saved_model/integration_tests/BUILD | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tensorflow/examples/saved_model/integration_tests/BUILD b/tensorflow/examples/saved_model/integration_tests/BUILD
index cb0592c2a2..2be9df6644 100644
--- a/tensorflow/examples/saved_model/integration_tests/BUILD
+++ b/tensorflow/examples/saved_model/integration_tests/BUILD
@@ -66,6 +66,11 @@ py_test(
         ":use_text_rnn_model",
     ],
     srcs_version = "PY2AND3",
+    tags = [
+        "noasan",  # forge input size exceeded
+        "nomsan",  # forge input size exceeded
+        "notsan",  # forge input size exceeded
+    ],
     deps = [
         "//tensorflow:tensorflow_py",
     ],
-- 
GitLab


From 76579cdd2f2288b2ff6700c0dc38552c6b59a9e4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 11 Feb 2019 09:17:30 -0800
Subject: [PATCH 2245/2345] Reserve capacity when it is known ahead of an
 insertion loop.

PiperOrigin-RevId: 233419749
---
 .../compiler/tf2tensorrt/kernels/trt_engine_op.cc    | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.cc b/tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.cc
index bc5335ef5a..746f091891 100644
--- a/tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.cc
@@ -254,6 +254,7 @@ void TRTEngineOp::ExecuteNativeSegment(OpKernelContext* ctx,
   opts.rendezvous = ctx->rendezvous();
   opts.cancellation_manager = ctx->cancellation_manager();
   opts.runner = ctx->runner();
+  inputs.reserve(ctx->num_inputs());
   for (int i = 0; i < ctx->num_inputs(); i++) {
     inputs.push_back(ctx->input(i));
   }
@@ -385,8 +386,9 @@ void TRTEngineOp::ComputeAsync(OpKernelContext* ctx,
   }
   // Get shapes of inputs to engine.
   std::vector<tensorflow::TensorShape> input_shapes;
+  input_shapes.reserve(ctx->num_inputs());
   for (int i = 0; i < ctx->num_inputs(); ++i) {
-    input_shapes.emplace_back(ctx->input(i).shape());
+    input_shapes.push_back(ctx->input(i).shape());
   }
   EngineContext* engine_context = GetEngine(input_shapes, ctx);
   if (!engine_context->cuda_engine) {
@@ -616,11 +618,11 @@ EngineContext* TRTEngineOp::GetEngine(
     LOG(INFO) << "Building a new TensorRT engine for " << name()
               << " input shapes: "
               << TensorShapeUtils::ShapeListString(engine_input_shapes);
+
     // Convert to partial shapes
-    std::vector<PartialTensorShape> partial_shapes;
-    for (int i = 0; i < engine_input_shapes.size(); i++) {
-      partial_shapes.emplace_back(engine_input_shapes[i]);
-    }
+    std::vector<PartialTensorShape> partial_shapes(engine_input_shapes.begin(),
+                                                   engine_input_shapes.end());
+
     // Up to this point, calibrator_ can never be empty, since otherwise it
     // means calibration_mode_ is true and this path won't get executed.
     auto status = convert::ConvertGraphDefToEngine(
-- 
GitLab


From 155f75ab3e7f7f03383a187e59a89dfecb725240 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Mon, 11 Feb 2019 09:22:03 -0800
Subject: [PATCH 2246/2345] Fix msan false positive with mkldnn sgemm.

PiperOrigin-RevId: 233420511
---
 tensorflow/core/kernels/BUILD                  | 1 +
 tensorflow/core/kernels/conv_grad_input_ops.cc | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index c3400fbf1b..ae42126c8f 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -3788,6 +3788,7 @@ tf_kernel_library(
         ":image_resizer_state",
         ":fill_functor",
         ":ops_util",
+        "@com_google_absl//absl/base:dynamic_annotations",
         "@com_google_absl//absl/strings",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
diff --git a/tensorflow/core/kernels/conv_grad_input_ops.cc b/tensorflow/core/kernels/conv_grad_input_ops.cc
index a9da05d4c4..a988f63a6c 100644
--- a/tensorflow/core/kernels/conv_grad_input_ops.cc
+++ b/tensorflow/core/kernels/conv_grad_input_ops.cc
@@ -23,6 +23,7 @@ limitations under the License.
 #include <algorithm>
 #include <vector>
 
+#include "absl/base/dynamic_annotations.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
@@ -243,6 +244,10 @@ struct Conv2DCustomBackpropInputMatMulFunctor<float> {
     const float alpha = 1.0;
     const float beta = 0.0;
 
+    // mkldnn_sgemm code can't be instrumented with msan.
+    ANNOTATE_MEMORY_IS_INITIALIZED(
+        im2col_buf, filter_total_size * output_image_size * sizeof(T));
+
     mkldnn_status_t st =
         mkldnn_sgemm(&transposeA, &transposeB, &m, &n, &k, &alpha, filter_data,
                      &ldA, out_data, &ldB, &beta, im2col_buf, &ldC);
-- 
GitLab


From 858a246f20b188f7526784f29f588d68f4b518c6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 11 Feb 2019 09:23:42 -0800
Subject: [PATCH 2247/2345] Add example to map_structure_up_to's docstring that
 clarifies that shallow_tree can be smaller than input.

PiperOrigin-RevId: 233420776
---
 tensorflow/python/util/nest.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/util/nest.py b/tensorflow/python/util/nest.py
index 2435694631..e930096142 100644
--- a/tensorflow/python/util/nest.py
+++ b/tensorflow/python/util/nest.py
@@ -626,7 +626,7 @@ def assert_shallow_structure(shallow_tree, input_tree, check_types=True):
   The following code will not raise an exception:
   ```python
     shallow_tree = ["a", "b"]
-    input_tree = ["c", ["d", "e"]]
+    input_tree = ["c", ["d", "e"], "f"]
     assert_shallow_structure(shallow_tree, input_tree)
   ```
 
@@ -875,6 +875,14 @@ def map_structure_up_to(shallow_tree, func, *inputs, **kwargs):
 
   Examples:
 
+  ```python
+  shallow_tree = [None, None]
+  inp_val = [1, 2, 3]
+  out = map_structure_up_to(shallow_tree, lambda x: 2 * x, inp_val)
+
+  # Output is: [2, 4]
+  ```
+
   ```python
   ab_tuple = collections.namedtuple("ab_tuple", "a, b")
   op_tuple = collections.namedtuple("op_tuple", "add, mul")
-- 
GitLab


From 1d6f79ac5f55fe49b3410eb7c6e6a4269e53f9a3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 11 Feb 2019 09:24:48 -0800
Subject: [PATCH 2248/2345] Cleanup: remove unused lambda captures, pass string
 view by value

PiperOrigin-RevId: 233420932
---
 tensorflow/contrib/bigtable/kernels/bigtable_kernels.cc | 6 ++----
 tensorflow/contrib/gdr/gdr_collective_executor_mgr.cc   | 3 +--
 tensorflow/contrib/tensor_forest/kernels/model_ops.cc   | 2 +-
 tensorflow/contrib/tensor_forest/kernels/stats_ops.cc   | 4 ++--
 4 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/bigtable/kernels/bigtable_kernels.cc b/tensorflow/contrib/bigtable/kernels/bigtable_kernels.cc
index 6138d79126..f0637595db 100644
--- a/tensorflow/contrib/bigtable/kernels/bigtable_kernels.cc
+++ b/tensorflow/contrib/bigtable/kernels/bigtable_kernels.cc
@@ -19,7 +19,6 @@ limitations under the License.
 #include "tensorflow/core/lib/core/threadpool.h"
 
 namespace tensorflow {
-
 namespace {
 
 class BigtableClientOp : public OpKernel {
@@ -341,8 +340,8 @@ class ToBigtableOp : public AsyncOpKernel {
   }
 
   template <typename T>
-  Status ParseScalarArgument(OpKernelContext* ctx,
-                             const StringPiece& argument_name, T* output) {
+  Status ParseScalarArgument(OpKernelContext* ctx, StringPiece argument_name,
+                             T* output) {
     const Tensor* argument_t;
     TF_RETURN_IF_ERROR(ctx->input(argument_name, &argument_t));
     if (!TensorShapeUtils::IsScalar(argument_t->shape())) {
@@ -360,5 +359,4 @@ REGISTER_KERNEL_BUILDER(Name("DatasetToBigtable").Device(DEVICE_CPU),
 
 }  // namespace
 }  // namespace data
-
 }  // namespace tensorflow
diff --git a/tensorflow/contrib/gdr/gdr_collective_executor_mgr.cc b/tensorflow/contrib/gdr/gdr_collective_executor_mgr.cc
index b84710d26e..755cbdff31 100644
--- a/tensorflow/contrib/gdr/gdr_collective_executor_mgr.cc
+++ b/tensorflow/contrib/gdr/gdr_collective_executor_mgr.cc
@@ -100,8 +100,7 @@ class CollectiveRemoteAccessDistributed : public CollectiveRemoteAccessLocal {
 
     // Logic to be executed on the RecvBufAsync callback.
     auto recv_buf_callback = [this, state, peer_task, to_device, to_alloc_attr,
-                              to_device_ctx, to_tensor, dev_to_dev_stream_index,
-                              done](const Status& s) {
+                              to_device_ctx, to_tensor, done](const Status& s) {
       if (s.ok()) {
         remote_memory_manager_->TensorFromTransportOptions(
             to_tensor, state->call->resp_.transport_options(), to_device,
diff --git a/tensorflow/contrib/tensor_forest/kernels/model_ops.cc b/tensorflow/contrib/tensor_forest/kernels/model_ops.cc
index b9aad36f3d..76b1d2b4da 100644
--- a/tensorflow/contrib/tensor_forest/kernels/model_ops.cc
+++ b/tensorflow/contrib/tensor_forest/kernels/model_ops.cc
@@ -304,7 +304,7 @@ class TraverseTreeV4Op : public OpKernel {
     auto worker_threads = context->device()->tensorflow_cpu_worker_threads();
     int num_threads = worker_threads->num_threads;
     const int64 costPerTraverse = 500;
-    auto traverse = [this, &set_leaf_ids, &data_set, decision_tree_resource,
+    auto traverse = [&set_leaf_ids, &data_set, decision_tree_resource,
                      num_data](int64 start, int64 end) {
       CHECK(start <= end);
       CHECK(end <= num_data);
diff --git a/tensorflow/contrib/tensor_forest/kernels/stats_ops.cc b/tensorflow/contrib/tensor_forest/kernels/stats_ops.cc
index fe2c91c104..0243f10681 100644
--- a/tensorflow/contrib/tensor_forest/kernels/stats_ops.cc
+++ b/tensorflow/contrib/tensor_forest/kernels/stats_ops.cc
@@ -307,7 +307,7 @@ class ProcessInputOp : public OpKernel {
     // from a digits run on local desktop.  Heuristics might be necessary
     // if it really matters that much.
     const int64 costPerUpdate = 1000;
-    auto update = [this, &target, &leaf_ids_tensor, &num_targets, &data_set,
+    auto update = [&target, &leaf_ids_tensor, &num_targets, &data_set,
                    fertile_stats_resource, &locks, &set_lock, &ready_to_split,
                    num_data](int64 start, int64 end) {
       CHECK(start <= end);
@@ -317,7 +317,7 @@ class ProcessInputOp : public OpKernel {
                   static_cast<int32>(end), &ready_to_split);
     };
 
-    auto update_collated = [this, &target, &num_targets, fertile_stats_resource,
+    auto update_collated = [&target, &num_targets, fertile_stats_resource,
                             tree_resource, &leaf_examples, &set_lock,
                             &ready_to_split, &data_set,
                             num_leaves](int64 start, int64 end) {
-- 
GitLab


From 6b1cb1e2d752181c45f3fedc50c20027a34c530e Mon Sep 17 00:00:00 2001
From: Petros Mol <pmol@google.com>
Date: Mon, 11 Feb 2019 09:36:44 -0800
Subject: [PATCH 2249/2345] Flaky test fix for kernelized layers.

PiperOrigin-RevId: 233423187
---
 tensorflow/python/keras/layers/kernelized_test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/layers/kernelized_test.py b/tensorflow/python/keras/layers/kernelized_test.py
index 300b1b8155..0bc6e8b144 100644
--- a/tensorflow/python/keras/layers/kernelized_test.py
+++ b/tensorflow/python/keras/layers/kernelized_test.py
@@ -288,7 +288,7 @@ class RandomFourierFeaturesTest(test.TestCase, parameterized.TestCase):
       ('other', init_ops.random_uniform_initializer(), 5.0))
   @test_util.run_in_graph_and_eager_modes()
   def test_different_params_similar_approximation(self, initializer, scale):
-    # Layers initialized using different randomness (seed).
+    random_seed.set_random_seed(12345)
     rff_layer1 = kernel_layers.RandomFourierFeatures(
         output_dim=3000,
         kernel_initializer=initializer,
@@ -314,7 +314,7 @@ class RandomFourierFeaturesTest(test.TestCase, parameterized.TestCase):
     # rff_layer(x)^T * rff_layer(y) ~= K(x,y) up to a normalization factor.
     approx_kernel1 = kernelized_utils.inner_product(output_x1, output_y1)
     approx_kernel2 = kernelized_utils.inner_product(output_x2, output_y2)
-    self._assert_all_close(approx_kernel1, approx_kernel2, atol=0.05)
+    self._assert_all_close(approx_kernel1, approx_kernel2, atol=0.08)
 
   @parameterized.named_parameters(
       ('gaussian', 'gaussian', 5.0, _exact_gaussian(stddev=5.0)),
-- 
GitLab


From 89f19fe339818bb69311769034b861f9ea9846ee Mon Sep 17 00:00:00 2001
From: HyoukJoong Lee <hyouklee@google.com>
Date: Mon, 11 Feb 2019 09:44:06 -0800
Subject: [PATCH 2250/2345] Don't share state across sessions.

PiperOrigin-RevId: 233424602
---
 tensorflow/contrib/tpu/python/tpu/tpu_config.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_config.py b/tensorflow/contrib/tpu/python/tpu/tpu_config.py
index 9f8d147068..82ebb9a37f 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_config.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_config.py
@@ -223,7 +223,8 @@ class RunConfig(run_config_lib.RunConfig):
 
       # Merge the cluster_def into the ConfigProto.
       if self._session_config is None:  # pylint: disable=access-member-before-definition
-        self._session_config = config_pb2.ConfigProto(allow_soft_placement=True)
+        self._session_config = config_pb2.ConfigProto(
+            allow_soft_placement=True, isolate_session_state=True)
       if self._session_config.HasField('cluster_def'):
         raise ValueError(
             'You cannot provide a ClusterResolver and '
-- 
GitLab


From 79465745074cc19f0551428bfeef6d547fa168a5 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Mon, 11 Feb 2019 09:46:27 -0800
Subject: [PATCH 2251/2345] Update attention_mechanism and related tests for
 2.0 compatible.

PiperOrigin-RevId: 233425023
---
 .../kernel_tests/attention_wrapper_test.py    |  20 +-
 .../kernel_tests/attention_wrapper_v2_test.py | 679 +++++++++++++++++-
 .../seq2seq/python/ops/attention_wrapper.py   | 460 ++++++++----
 3 files changed, 1013 insertions(+), 146 deletions(-)

diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py
index d815f81f84..1a5692f7b5 100644
--- a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py
+++ b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py
@@ -358,7 +358,7 @@ class AttentionWrapperTest(test.TestCase):
         rnn_output=ResultSummary(
             shape=(5, 3, 6), dtype=dtype('float32'), mean=-0.00597103),
         sample_id=ResultSummary(
-            shape=(5, 3), dtype=dtype('int32'), mean=1.6))
+            shape=(5, 3), dtype=dtype('int32'), mean=1.4))
     expected_final_state = AttentionWrapperState(
         cell_state=LSTMStateTuple(
             c=ResultSummary(
@@ -387,7 +387,7 @@ class AttentionWrapperTest(test.TestCase):
         rnn_output=ResultSummary(
             shape=(5, 3, 6), dtype=dtype('float32'), mean=-0.0052615386),
         sample_id=ResultSummary(
-            shape=(5, 3), dtype=dtype('int32'), mean=1.3333333333))
+            shape=(5, 3), dtype=dtype('int32'), mean=1.4))
     expected_final_state = AttentionWrapperState(
         cell_state=LSTMStateTuple(
             c=ResultSummary(
@@ -454,7 +454,7 @@ class AttentionWrapperTest(test.TestCase):
         rnn_output=ResultSummary(
             shape=(5, 3, 6), dtype=dtype('float32'), mean=-0.0052615386),
         sample_id=ResultSummary(
-            shape=(5, 3), dtype=dtype('int32'), mean=1.3333333333333333))
+            shape=(5, 3), dtype=dtype('int32'), mean=1.4))
     expected_final_state = AttentionWrapperState(
         cell_state=LSTMStateTuple(
             c=ResultSummary(
@@ -696,7 +696,7 @@ class AttentionWrapperTest(test.TestCase):
         rnn_output=ResultSummary(
             shape=(5, 3, 6), dtype=dtype('float32'), mean=-0.0025896581),
         sample_id=ResultSummary(
-            shape=(5, 3), dtype=dtype('int32'), mean=1.6))
+            shape=(5, 3), dtype=dtype('int32'), mean=1.73333333))
     expected_final_state = AttentionWrapperState(
         cell_state=LSTMStateTuple(
             c=ResultSummary(
@@ -707,12 +707,12 @@ class AttentionWrapperTest(test.TestCase):
             shape=(5, 6), dtype=dtype('float32'), mean=-0.00069823361),
         time=3,
         alignments=ResultSummary(
-            shape=(5, 8), dtype=dtype('float32'), mean=0.028698336),
+            shape=(5, 8), dtype=dtype('float32'), mean=0.029914695),
         attention_state=ResultSummary(
-            shape=(5, 8), dtype=dtype('float32'), mean=0.028698336),
+            shape=(5, 8), dtype=dtype('float32'), mean=0.029914695),
         alignment_history=())
     expected_final_alignment_history = ResultSummary(
-        shape=(3, 5, 8), dtype=dtype('float32'), mean=0.04865776002407074)
+        shape=(3, 5, 8), dtype=dtype('float32'), mean=0.0465225502849)
 
     self._testWithAttention(
         create_attention_mechanism,
@@ -921,9 +921,9 @@ class AttentionWrapperTest(test.TestCase):
 
     expected_final_output = BasicDecoderOutput(
         rnn_output=ResultSummary(
-            shape=(5, 3, 20), dtype=dtype('float32'), mean=0.11723966),
+            shape=(5, 3, 20), dtype=dtype('float32'), mean=0.115853324533),
         sample_id=ResultSummary(
-            shape=(5, 3), dtype=dtype('int32'), mean=7.266666666666667))
+            shape=(5, 3), dtype=dtype('int32'), mean=8.6))
     expected_final_state = AttentionWrapperState(
         cell_state=LSTMStateTuple(
             c=ResultSummary(
@@ -931,7 +931,7 @@ class AttentionWrapperTest(test.TestCase):
             h=ResultSummary(
                 shape=(5, 9), dtype=dtype('float32'), mean=-0.0018327223)),
         attention=ResultSummary(
-            shape=(5, 20), dtype=dtype('float32'), mean=0.11601614207),
+            shape=(5, 20), dtype=dtype('float32'), mean=0.11462739855),
         time=3,
         alignments=(ResultSummary(
             shape=(5, 8), dtype=dtype('float32'), mean=0.125),
diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_v2_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_v2_test.py
index 7ff04e1780..5ee01f66f1 100644
--- a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_v2_test.py
+++ b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_v2_test.py
@@ -17,14 +17,23 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import collections
+
 from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.contrib.seq2seq.python.ops import attention_wrapper as wrapper
-from tensorflow.python.framework import ops
+from tensorflow.contrib.seq2seq.python.ops import basic_decoder
+from tensorflow.contrib.seq2seq.python.ops import sampler as sampler_py
+from tensorflow.python import keras
+from tensorflow.python.eager import context
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import test_util
+from tensorflow.python.keras import initializers
+from tensorflow.python.ops import rnn_cell
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
+from tensorflow.python.util import nest
 
 
 @test_util.run_all_in_graph_and_eager_modes
@@ -37,13 +46,10 @@ class AttentionMechanismTest(test.TestCase, parameterized.TestCase):
     self.memory_size = 6
     self.units = 8
 
-    self.memory = ops.convert_to_tensor(
-        np.random.random((self.batch, self.timestep, self.memory_size)),
-        dtype=np.float32)
-    self.query = ops.convert_to_tensor(
-        np.random.random((self.batch, self.units)), dtype=np.float32)
-    self.state = ops.convert_to_tensor(
-        np.random.random((self.batch, self.timestep)), dtype=np.float32)
+    self.memory = np.random.randn(self.batch, self.timestep,
+                                  self.memory_size).astype(np.float32)
+    self.query = np.random.randn(self.batch, self.units).astype(np.float32)
+    self.state = np.random.randn(self.batch, self.timestep).astype(np.float32)
 
   @parameterized.named_parameters(
       ("luong", wrapper.LuongAttentionV2),
@@ -52,8 +58,8 @@ class AttentionMechanismTest(test.TestCase, parameterized.TestCase):
       ("bahdanau_monotonic", wrapper.BahdanauMonotonicAttentionV2),
   )
   def test_attention_shape_inference(self, attention_cls):
-    attention = attention_cls(self.units)
-    attention_score = attention([self.query, self.state, self.memory])
+    attention = attention_cls(self.units, self.memory)
+    attention_score = attention([self.query, self.state])
     self.assertLen(attention_score, 2)
     self.assertEqual(attention_score[0].shape, (self.batch, self.timestep))
     self.assertEqual(attention_score[1].shape, (self.batch, self.timestep))
@@ -65,7 +71,7 @@ class AttentionMechanismTest(test.TestCase, parameterized.TestCase):
       ("bahdanau_monotonic", wrapper.BahdanauMonotonicAttentionV2),
   )
   def test_get_config(self, attention_cls):
-    attention = attention_cls(self.units)
+    attention = attention_cls(self.units, self.memory)
     config = attention.get_config()
 
     attention_from_config = attention_cls.from_config(config)
@@ -80,9 +86,8 @@ class AttentionMechanismTest(test.TestCase, parameterized.TestCase):
       ("bahdanau_monotonic", wrapper.BahdanauMonotonicAttentionV2),
   )
   def test_layer_output(self, attention_cls):
-    attention = attention_cls(self.units)
-
-    score = attention([self.query, self.state, self.memory])
+    attention = attention_cls(self.units, self.memory)
+    score = attention([self.query, self.state])
     self.evaluate(variables.variables_initializer(attention.variables))
 
     score_val = self.evaluate(score)
@@ -90,5 +95,651 @@ class AttentionMechanismTest(test.TestCase, parameterized.TestCase):
     self.assertEqual(score_val[0].shape, (self.batch, self.timestep))
     self.assertEqual(score_val[1].shape, (self.batch, self.timestep))
 
+  @parameterized.named_parameters(
+      ("luong", wrapper.LuongAttentionV2),
+      ("luong_monotonic", wrapper.LuongMonotonicAttentionV2),
+      ("bahdanau", wrapper.BahdanauAttentionV2),
+      ("bahdanau_monotonic", wrapper.BahdanauMonotonicAttentionV2),
+  )
+  def test_passing_memory_from_call(self, attention_cls):
+    attention = attention_cls(self.units, self.memory)
+    weights_before_query = attention.get_weights()
+    ref_score = attention([self.query, self.state])
+
+    self.evaluate(variables.global_variables_initializer())
+    ref_score_val = self.evaluate(ref_score)
+
+    all_weights = attention.get_weights()
+    config = attention.get_config()
+    # Simulate the twice invocation of calls here.
+    attention_from_config = attention_cls.from_config(config)
+    attention_from_config.build(self.memory.shape)
+    attention_from_config.set_weights(weights_before_query)
+    attention_from_config(self.memory, setup_memory=True)
+    attention_from_config.build([self.query.shape, self.state.shape])
+    attention_from_config.set_weights(all_weights)
+    score = attention_from_config([self.query, self.state])
+
+    score_val = self.evaluate(score)
+    self.assertAllClose(ref_score_val, score_val)
+
+  @parameterized.named_parameters(
+      ("luong", wrapper.LuongAttentionV2),
+      ("luong_monotonic", wrapper.LuongMonotonicAttentionV2),
+      ("bahdanau", wrapper.BahdanauAttentionV2),
+      ("bahdanau_monotonic", wrapper.BahdanauMonotonicAttentionV2),
+  )
+  def test_save_load_layer(self, attention_cls):
+    vocab = 20
+    embedding_dim = 6
+    inputs = keras.layers.Input(shape=[self.timestep])
+    encoder_input = keras.layers.Embedding(
+        vocab, embedding_dim, mask_zero=True)(
+            inputs)
+    encoder_output = keras.layers.UnifiedLSTM(
+        self.memory_size, return_sequences=True)(
+            encoder_input)
+
+    attention = attention_cls(self.units, encoder_output)
+    query = keras.layers.Input(shape=[self.units])
+    state = keras.layers.Input(shape=[self.timestep])
+
+    score = attention([query, state])
+
+    x = np.random.randint(vocab, size=(self.batch, self.timestep))
+    x_test = np.random.randint(vocab, size=(self.batch, self.timestep))
+    y = np.random.randn(self.batch, self.timestep)
+    model = keras.models.Model([inputs, query, state], score)
+    model.compile("rmsprop", "mse")
+    model.fit([x, self.query, self.state], (y, y))
+    y_ref = model.predict_on_batch([x_test, self.query, self.state])
+
+    config = model.get_config()
+    weights = model.get_weights()
+    loaded_model = keras.models.Model.from_config(
+        config, custom_objects={attention_cls.__name__: attention_cls})
+    loaded_model.set_weights(weights)
+
+    y = loaded_model.predict_on_batch([x_test, self.query, self.state])
+
+    self.assertAllClose(y_ref, y)
+
+  # TODO(scottzhu): Add tests for model.compile(run_eagerly=True)
+
+
+class ResultSummary(
+    collections.namedtuple("ResultSummary", ("shape", "dtype", "mean"))):
+  pass
+
+
+def get_result_summary(x):
+  if isinstance(x, np.ndarray):
+    return ResultSummary(x.shape, x.dtype, x.mean())
+  return x
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class AttentionWrapperV2Test(test.TestCase, parameterized.TestCase):
+
+  def assertAllCloseOrEqual(self, x, y, **kwargs):
+    if isinstance(x, np.ndarray) or isinstance(x, float):
+      return super(AttentionWrapperV2Test, self).assertAllClose(
+          x, y, atol=1e-3, **kwargs)
+    else:
+      self.assertAllEqual(x, y, **kwargs)
+
+  def setUp(self):
+    super(AttentionWrapperV2Test, self).setUp()
+    self.batch = 64
+    self.units = 128
+    self.encoder_timestep = 10
+    self.encoder_dim = 256
+    self.decoder_timestep = 12
+    self.encoder_outputs = np.random.randn(self.batch, self.encoder_timestep,
+                                           self.encoder_dim)
+    self.encoder_sequence_length = np.random.randint(
+        self.encoder_timestep, size=(self.batch,)).astype(np.int32)
+    self.decoder_inputs = np.random.randn(self.batch, self.decoder_timestep,
+                                          self.units)
+    self.decoder_sequence_length = np.random.randint(
+        self.decoder_timestep, size=(self.batch,)).astype(np.int32)
+
+  def _testWithAttention(self,
+                         create_attention_mechanism,
+                         expected_final_output,
+                         expected_final_state,
+                         attention_mechanism_depth=3,
+                         alignment_history=False,
+                         expected_final_alignment_history=None,
+                         attention_layer_size=6,
+                         attention_layer=None,
+                         create_query_layer=False,
+                         create_memory_layer=True,
+                         create_attention_kwargs=None):
+    attention_layer_sizes = ([attention_layer_size]
+                             if attention_layer_size is not None else None)
+    attention_layers = ([attention_layer]
+                        if attention_layer is not None else None)
+    self._testWithMaybeMultiAttention(
+        is_multi=False,
+        create_attention_mechanisms=[create_attention_mechanism],
+        expected_final_output=expected_final_output,
+        expected_final_state=expected_final_state,
+        attention_mechanism_depths=[attention_mechanism_depth],
+        alignment_history=alignment_history,
+        expected_final_alignment_history=expected_final_alignment_history,
+        attention_layer_sizes=attention_layer_sizes,
+        attention_layers=attention_layers,
+        create_query_layer=create_query_layer,
+        create_memory_layer=create_memory_layer,
+        create_attention_kwargs=create_attention_kwargs)
+
+  def _testWithMaybeMultiAttention(self,
+                                   is_multi,
+                                   create_attention_mechanisms,
+                                   expected_final_output,
+                                   expected_final_state,
+                                   attention_mechanism_depths,
+                                   alignment_history=False,
+                                   expected_final_alignment_history=None,
+                                   attention_layer_sizes=None,
+                                   attention_layers=None,
+                                   create_query_layer=False,
+                                   create_memory_layer=True,
+                                   create_attention_kwargs=None):
+    # Allow is_multi to be True with a single mechanism to enable test for
+    # passing in a single mechanism in a list.
+    assert len(create_attention_mechanisms) == 1 or is_multi
+    encoder_sequence_length = [3, 2, 3, 1, 1]
+    decoder_sequence_length = [2, 0, 1, 2, 3]
+    batch_size = 5
+    encoder_max_time = 8
+    decoder_max_time = 4
+    input_depth = 7
+    encoder_output_depth = 10
+    cell_depth = 9
+    create_attention_kwargs = create_attention_kwargs or {}
+
+    if attention_layer_sizes is not None:
+      # Compute sum of attention_layer_sizes. Use encoder_output_depth if None.
+      attention_depth = sum(attention_layer_size or encoder_output_depth
+                            for attention_layer_size in attention_layer_sizes)
+    elif attention_layers is not None:
+      # Compute sum of attention_layers output depth.
+      attention_depth = sum(
+          attention_layer.compute_output_shape(
+              [batch_size, cell_depth + encoder_output_depth]).dims[-1].value
+          for attention_layer in attention_layers)
+    else:
+      attention_depth = encoder_output_depth * len(create_attention_mechanisms)
+
+    decoder_inputs = np.random.randn(batch_size, decoder_max_time,
+                                     input_depth).astype(np.float32)
+    encoder_outputs = np.random.randn(batch_size, encoder_max_time,
+                                      encoder_output_depth).astype(np.float32)
+
+    attention_mechanisms = []
+    for creator, depth in zip(create_attention_mechanisms,
+                              attention_mechanism_depths):
+      # Create a memory layer with deterministic initializer to avoid randomness
+      # in the test between graph and eager.
+      if create_query_layer:
+        create_attention_kwargs["query_layer"] = keras.layers.Dense(
+            depth, kernel_initializer="ones", use_bias=False)
+      if create_memory_layer:
+        create_attention_kwargs["memory_layer"] = keras.layers.Dense(
+            depth, kernel_initializer="ones", use_bias=False)
+
+      attention_mechanisms.append(
+          creator(
+              units=depth,
+              memory=encoder_outputs,
+              memory_sequence_length=encoder_sequence_length,
+              **create_attention_kwargs))
+
+    with self.cached_session(use_gpu=True):
+      attention_layer_size = attention_layer_sizes
+      attention_layer = attention_layers
+      if not is_multi:
+        if attention_layer_size is not None:
+          attention_layer_size = attention_layer_size[0]
+        if attention_layer is not None:
+          attention_layer = attention_layer[0]
+      cell = rnn_cell.LSTMCell(cell_depth, initializer="ones")
+      cell = wrapper.AttentionWrapper(
+          cell,
+          attention_mechanisms if is_multi else attention_mechanisms[0],
+          attention_layer_size=attention_layer_size,
+          alignment_history=alignment_history,
+          attention_layer=attention_layer)
+      # Set the attention_layer within AttentionWrapper to have deterministic
+      # kernel initializer, for testing purpose.
+      if cell._attention_layers is not None:
+        for layer in cell._attention_layers:
+          if getattr(layer, "kernel_initializer") is None:
+            layer.kernel_initializer = initializers.ones()
+
+      sampler = sampler_py.TrainingSampler()
+      my_decoder = basic_decoder.BasicDecoderV2(cell=cell, sampler=sampler)
+      initial_state = cell.zero_state(
+          dtype=dtypes.float32, batch_size=batch_size)
+      final_outputs, final_state, _ = my_decoder(
+          decoder_inputs,
+          initial_state=initial_state,
+          sequence_length=decoder_sequence_length)
+
+      self.assertIsInstance(final_outputs, basic_decoder.BasicDecoderOutput)
+      self.assertIsInstance(final_state, wrapper.AttentionWrapperState)
+      self.assertIsInstance(final_state.cell_state, rnn_cell.LSTMStateTuple)
+
+      expected_time = (
+          expected_final_state.time if context.executing_eagerly() else None)
+      self.assertEqual((batch_size, expected_time, attention_depth),
+                       tuple(final_outputs.rnn_output.get_shape().as_list()))
+      self.assertEqual((batch_size, expected_time),
+                       tuple(final_outputs.sample_id.get_shape().as_list()))
+
+      self.assertEqual((batch_size, attention_depth),
+                       tuple(final_state.attention.get_shape().as_list()))
+      self.assertEqual((batch_size, cell_depth),
+                       tuple(final_state.cell_state.c.get_shape().as_list()))
+      self.assertEqual((batch_size, cell_depth),
+                       tuple(final_state.cell_state.h.get_shape().as_list()))
+
+      if alignment_history:
+        if is_multi:
+          state_alignment_history = []
+          for history_array in final_state.alignment_history:
+            history = history_array.stack()
+            self.assertEqual((expected_time, batch_size, encoder_max_time),
+                             tuple(history.get_shape().as_list()))
+            state_alignment_history.append(history)
+          state_alignment_history = tuple(state_alignment_history)
+        else:
+          state_alignment_history = final_state.alignment_history.stack()
+          self.assertEqual((expected_time, batch_size, encoder_max_time),
+                           tuple(state_alignment_history.get_shape().as_list()))
+        nest.assert_same_structure(cell.state_size,
+                                   cell.zero_state(batch_size, dtypes.float32))
+        # Remove the history from final_state for purposes of the
+        # remainder of the tests.
+        final_state = final_state._replace(alignment_history=())  # pylint: disable=protected-access
+      else:
+        state_alignment_history = ()
+
+      self.evaluate(variables.global_variables_initializer())
+      eval_result = self.evaluate({
+          "final_outputs": final_outputs,
+          "final_state": final_state,
+          "state_alignment_history": state_alignment_history,
+      })
+
+      final_output_info = nest.map_structure(get_result_summary,
+                                             eval_result["final_outputs"])
+      final_state_info = nest.map_structure(get_result_summary,
+                                            eval_result["final_state"])
+      print("final_output_info: ", final_output_info)
+      print("final_state_info: ", final_state_info)
+
+      nest.map_structure(self.assertAllCloseOrEqual, expected_final_output,
+                         final_output_info)
+      nest.map_structure(self.assertAllCloseOrEqual, expected_final_state,
+                         final_state_info)
+      if alignment_history:  # by default, the wrapper emits attention as output
+        final_alignment_history_info = nest.map_structure(
+            get_result_summary, eval_result["state_alignment_history"])
+        print("final_alignment_history_info: ", final_alignment_history_info)
+        nest.map_structure(
+            self.assertAllCloseOrEqual,
+            # outputs are batch major but the stacked TensorArray is time major
+            expected_final_alignment_history,
+            final_alignment_history_info)
+
+  @parameterized.parameters([np.float16, np.float32, np.float64])
+  def _testBahdanauNormalizedDType(self, dtype):
+    encoder_outputs = self.encoder_outputs.astype(dtype)
+    decoder_inputs = self.decoder_inputs.astype(dtype)
+    attention_mechanism = wrapper.BahdanauAttentionV2(
+        units=self.units,
+        memory=encoder_outputs,
+        memory_sequence_length=self.encoder_sequence_length,
+        normalize=True,
+        dtype=dtype)
+    cell = rnn_cell.LSTMCell(self.units)
+    cell = wrapper.AttentionWrapper(cell, attention_mechanism)
+
+    sampler = sampler_py.TrainingSampler()
+    my_decoder = basic_decoder.BasicDecoderV2(cell=cell, sampler=sampler)
+
+    final_outputs, final_state, _ = my_decoder(
+        decoder_inputs,
+        initial_state=cell.zero_state(dtype=dtype, batch_size=self.batch),
+        sequence_length=self.decoder_sequence_length)
+    self.assertIsInstance(final_outputs, basic_decoder.BasicDecoderOutput)
+    self.assertEqual(final_outputs.rnn_output.dtype, dtype)
+    self.assertIsInstance(final_state, wrapper.AttentionWrapperState)
+    self.assertIsInstance(final_state.cell_state, rnn_cell.LSTMStateTuple)
+
+  @parameterized.parameters([np.float16, np.float32, np.float64])
+  def testLuongScaledDType(self, dtype):
+    # Test case for GitHub issue 18099
+    encoder_outputs = self.encoder_outputs.astype(dtype)
+    decoder_inputs = self.decoder_inputs.astype(dtype)
+    attention_mechanism = wrapper.LuongAttentionV2(
+        units=self.units,
+        memory=encoder_outputs,
+        memory_sequence_length=self.encoder_sequence_length,
+        scale=True,
+        dtype=dtype,
+    )
+    cell = rnn_cell.LSTMCell(self.units)
+    cell = wrapper.AttentionWrapper(cell, attention_mechanism)
+
+    sampler = sampler_py.TrainingSampler()
+    my_decoder = basic_decoder.BasicDecoderV2(cell=cell, sampler=sampler)
+
+    final_outputs, final_state, _ = my_decoder(
+        decoder_inputs,
+        initial_state=cell.zero_state(dtype=dtype, batch_size=self.batch),
+        sequence_length=self.decoder_sequence_length)
+    self.assertIsInstance(final_outputs, basic_decoder.BasicDecoderOutput)
+    self.assertEqual(final_outputs.rnn_output.dtype, dtype)
+    self.assertIsInstance(final_state, wrapper.AttentionWrapperState)
+    self.assertIsInstance(final_state.cell_state, rnn_cell.LSTMStateTuple)
+
+  def testBahdanauNotNormalized(self):
+    create_attention_mechanism = wrapper.BahdanauAttentionV2
+    create_attention_kwargs = {"kernel_initializer": "ones"}
+    expected_final_output = basic_decoder.BasicDecoderOutput(
+        rnn_output=ResultSummary(
+            shape=(5, 3, 6), dtype=np.dtype(np.float32), mean=4.8290324),
+        sample_id=ResultSummary(shape=(5, 3), dtype=np.dtype(np.int32), mean=0))
+    expected_final_state = wrapper.AttentionWrapperState(
+        cell_state=rnn_cell.LSTMStateTuple(
+            c=ResultSummary(
+                shape=(5, 9), dtype=np.dtype(np.float32), mean=1.6432636),
+            h=ResultSummary(
+                shape=(5, 9), dtype=np.dtype(np.float32), mean=0.75866824)),
+        attention=ResultSummary(
+            shape=(5, 6), dtype=np.dtype(np.float32), mean=6.7445569),
+        time=3,
+        alignments=ResultSummary(
+            shape=(5, 8), dtype=np.dtype(np.float32), mean=0.125),
+        attention_state=ResultSummary(
+            shape=(5, 8), dtype=np.dtype(np.float32), mean=0.125),
+        alignment_history=())
+    expected_final_alignment_history = ResultSummary(
+        shape=(3, 5, 8), dtype=np.dtype(np.float32), mean=0.125)
+
+    self._testWithAttention(
+        create_attention_mechanism,
+        expected_final_output,
+        expected_final_state,
+        alignment_history=True,
+        create_query_layer=True,
+        expected_final_alignment_history=expected_final_alignment_history,
+        create_attention_kwargs=create_attention_kwargs)
+
+  def testBahdanauNormalized(self):
+    create_attention_mechanism = wrapper.BahdanauAttentionV2
+    create_attention_kwargs = {"kernel_initializer": "ones", "normalize": True}
+
+    expected_final_output = basic_decoder.BasicDecoderOutput(
+        rnn_output=ResultSummary(
+            shape=(5, 3, 6), dtype=np.dtype("float32"), mean=3.9548259),
+        sample_id=ResultSummary(
+            shape=(5, 3), dtype=np.dtype("int32"), mean=0.0))
+    expected_final_state = wrapper.AttentionWrapperState(
+        cell_state=rnn_cell.LSTMStateTuple(
+            c=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=1.4652209),
+            h=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=0.70997983)),
+        attention=ResultSummary(
+            shape=(5, 6), dtype=np.dtype("float32"), mean=6.3075728),
+        time=3,
+        alignments=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.125),
+        attention_state=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.125),
+        alignment_history=())
+
+    self._testWithAttention(
+        create_attention_mechanism,
+        expected_final_output,
+        expected_final_state,
+        create_query_layer=True,
+        create_attention_kwargs=create_attention_kwargs)
+
+  def testLuongNotNormalized(self):
+    create_attention_mechanism = wrapper.LuongAttentionV2
+
+    expected_final_output = basic_decoder.BasicDecoderOutput(
+        rnn_output=ResultSummary(
+            shape=(5, 3, 6), dtype=np.dtype("float32"), mean=2.6605489),
+        sample_id=ResultSummary(
+            shape=(5, 3), dtype=np.dtype("int32"), mean=0.0))
+    expected_final_state = wrapper.AttentionWrapperState(
+        cell_state=rnn_cell.LSTMStateTuple(
+            c=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=0.88403547),
+            h=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=0.37819088)),
+        attention=ResultSummary(
+            shape=(5, 6), dtype=np.dtype("float32"), mean=4.084631),
+        time=3,
+        alignments=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.125),
+        attention_state=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.125),
+        alignment_history=())
+
+    self._testWithAttention(
+        create_attention_mechanism,
+        expected_final_output,
+        expected_final_state,
+        attention_mechanism_depth=9)
+
+  def testLuongScaled(self):
+    create_attention_mechanism = wrapper.LuongAttentionV2
+    create_attention_kwargs = {"scale": True}
+
+    expected_final_output = basic_decoder.BasicDecoderOutput(
+        rnn_output=ResultSummary(
+            shape=(5, 3, 6), dtype=np.dtype("float32"), mean=2.6605489),
+        sample_id=ResultSummary(
+            shape=(5, 3), dtype=np.dtype("int32"), mean=0.0))
+    expected_final_state = wrapper.AttentionWrapperState(
+        cell_state=rnn_cell.LSTMStateTuple(
+            c=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=0.88403547),
+            h=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=0.37819088)),
+        attention=ResultSummary(
+            shape=(5, 6), dtype=np.dtype("float32"), mean=4.0846314),
+        time=3,
+        alignments=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.125),
+        attention_state=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.125),
+        alignment_history=())
+
+    self._testWithAttention(
+        create_attention_mechanism,
+        expected_final_output,
+        expected_final_state,
+        attention_mechanism_depth=9,
+        create_attention_kwargs=create_attention_kwargs)
+
+  def testNotUseAttentionLayer(self):
+    create_attention_mechanism = wrapper.BahdanauAttentionV2
+    create_attention_kwargs = {"kernel_initializer": "ones"}
+
+    expected_final_output = basic_decoder.BasicDecoderOutput(
+        rnn_output=ResultSummary(
+            shape=(5, 3, 10), dtype=np.dtype("float32"), mean=0.072406612),
+        sample_id=ResultSummary(
+            shape=(5, 3), dtype=np.dtype("int32"), mean=3.86666666))
+    expected_final_state = wrapper.AttentionWrapperState(
+        cell_state=rnn_cell.LSTMStateTuple(
+            c=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=1.032002),
+            h=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=0.61177742)),
+        attention=ResultSummary(
+            shape=(5, 10), dtype=np.dtype("float32"), mean=0.011346335),
+        time=3,
+        alignments=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.125),
+        attention_state=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.125),
+        alignment_history=())
+
+    self._testWithAttention(
+        create_attention_mechanism,
+        expected_final_output,
+        expected_final_state,
+        attention_layer_size=None,
+        create_query_layer=True,
+        create_attention_kwargs=create_attention_kwargs)
+
+  def testBahdanauMonotonicNotNormalized(self):
+    create_attention_mechanism = wrapper.BahdanauMonotonicAttentionV2
+    create_attention_kwargs = {"kernel_initializer": "ones"}
+
+    expected_final_output = basic_decoder.BasicDecoderOutput(
+        rnn_output=ResultSummary(
+            shape=(5, 3, 6), dtype=np.dtype("float32"), mean=5.9850435),
+        sample_id=ResultSummary(
+            shape=(5, 3), dtype=np.dtype("int32"), mean=0.0))
+    expected_final_state = wrapper.AttentionWrapperState(
+        cell_state=rnn_cell.LSTMStateTuple(
+            c=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=1.6752492),
+            h=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=0.76052248)),
+        attention=ResultSummary(
+            shape=(5, 6), dtype=np.dtype("float32"), mean=8.361186),
+        time=3,
+        alignments=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.10989678),
+        attention_state=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.10989678),
+        alignment_history=())
+    expected_final_alignment_history = ResultSummary(
+        shape=(3, 5, 8), dtype=np.dtype("float32"), mean=0.117412611)
+
+    self._testWithAttention(
+        create_attention_mechanism,
+        expected_final_output,
+        expected_final_state,
+        alignment_history=True,
+        expected_final_alignment_history=expected_final_alignment_history,
+        create_query_layer=True,
+        create_attention_kwargs=create_attention_kwargs)
+
+  def testBahdanauMonotonicNormalized(self):
+    create_attention_mechanism = wrapper.BahdanauMonotonicAttentionV2
+    create_attention_kwargs = {"kernel_initializer": "ones",
+                               "normalize": True}
+    expected_final_output = basic_decoder.BasicDecoderOutput(
+        rnn_output=ResultSummary(
+            shape=(5, 3, 6), dtype=np.dtype("float32"), mean=4.5706983),
+        sample_id=ResultSummary(
+            shape=(5, 3), dtype=np.dtype("int32"), mean=0.0))
+    expected_final_state = wrapper.AttentionWrapperState(
+        cell_state=rnn_cell.LSTMStateTuple(
+            c=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=1.6005473),
+            h=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=0.77863038)),
+        attention=ResultSummary(
+            shape=(5, 6), dtype=np.dtype("float32"), mean=7.3326721),
+        time=3,
+        alignments=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.12258384),
+        attention_state=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.12258384),
+        alignment_history=())
+    expected_final_alignment_history = ResultSummary(
+        shape=(3, 5, 8), dtype=np.dtype("float32"), mean=0.12258384)
+
+    self._testWithAttention(
+        create_attention_mechanism,
+        expected_final_output,
+        expected_final_state,
+        alignment_history=True,
+        expected_final_alignment_history=expected_final_alignment_history,
+        create_query_layer=True,
+        create_attention_kwargs=create_attention_kwargs)
+
+  def testLuongMonotonicNotNormalized(self):
+    create_attention_mechanism = wrapper.LuongMonotonicAttentionV2
+
+    expected_final_output = basic_decoder.BasicDecoderOutput(
+        rnn_output=ResultSummary(
+            shape=(5, 3, 6), dtype=np.dtype("float32"), mean=3.159497),
+        sample_id=ResultSummary(
+            shape=(5, 3), dtype=np.dtype("int32"), mean=0.0))
+    expected_final_state = wrapper.AttentionWrapperState(
+        cell_state=rnn_cell.LSTMStateTuple(
+            c=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=1.072384),
+            h=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=0.50331038)),
+        attention=ResultSummary(
+            shape=(5, 6), dtype=np.dtype("float32"), mean=5.3079605),
+        time=3,
+        alignments=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.11467695),
+        attention_state=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.11467695),
+        alignment_history=())
+    expected_final_alignment_history = ResultSummary(
+        shape=(3, 5, 8), dtype=np.dtype("float32"), mean=0.11899644)
+
+    self._testWithAttention(
+        create_attention_mechanism,
+        expected_final_output,
+        expected_final_state,
+        attention_mechanism_depth=9,
+        alignment_history=True,
+        expected_final_alignment_history=expected_final_alignment_history)
+
+  def testLuongMonotonicScaled(self):
+    create_attention_mechanism = wrapper.LuongMonotonicAttentionV2
+    create_attention_kwargs = {"scale": True}
+
+    expected_final_output = basic_decoder.BasicDecoderOutput(
+        rnn_output=ResultSummary(
+            shape=(5, 3, 6), dtype=np.dtype("float32"), mean=3.159497),
+        sample_id=ResultSummary(
+            shape=(5, 3), dtype=np.dtype("int32"), mean=0.0))
+    expected_final_state = wrapper.AttentionWrapperState(
+        cell_state=rnn_cell.LSTMStateTuple(
+            c=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=1.072384),
+            h=ResultSummary(
+                shape=(5, 9), dtype=np.dtype("float32"), mean=0.50331038)),
+        attention=ResultSummary(
+            shape=(5, 6), dtype=np.dtype("float32"), mean=5.3079605),
+        time=3,
+        alignments=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.11467695),
+        attention_state=ResultSummary(
+            shape=(5, 8), dtype=np.dtype("float32"), mean=0.11467695),
+        alignment_history=())
+    expected_final_alignment_history = ResultSummary(
+        shape=(3, 5, 8), dtype=np.dtype("float32"), mean=0.11899644)
+
+    self._testWithAttention(
+        create_attention_mechanism,
+        expected_final_output,
+        expected_final_state,
+        attention_mechanism_depth=9,
+        alignment_history=True,
+        expected_final_alignment_history=expected_final_alignment_history,
+        create_attention_kwargs=create_attention_kwargs)
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
index ae3e7f1b5d..7e98383eb0 100644
--- a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
+++ b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
@@ -28,7 +28,9 @@ from tensorflow.contrib.framework.python.framework import tensor_util
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.keras import initializers
 from tensorflow.python.keras import layers
+from tensorflow.python.keras.engine import base_layer_utils
 from tensorflow.python.layers import base as layers_base
 from tensorflow.python.layers import core as layers_core
 from tensorflow.python.ops import array_ops
@@ -225,18 +227,39 @@ class _BaseAttentionMechanismV2(AttentionMechanism, layers.Layer):
     1. Storing the query and memory layers.
     2. Preprocessing and storing the memory.
 
-  Note that this layer only support Keras functional API since it takes multiple
-  input tensors, which is not available in sequential model.
+  Note that this layer takes memory as its init parameter, which is an
+  anti-pattern of Keras API, we have to keep the memory as init parameter for
+  performance and dependency reason. Under the hood, during `__init__()`, it
+  will invoke `base_layer.__call__(memory, setup_memory=True)`. This will let
+  keras to keep track of the memory tensor as the input of this layer. Once
+  the `__init__()` is done, then user can query the attention by
+  `score = att_obj([query, state])`, and use it as a normal keras layer.
+
+  Special attention is needed when adding using this class as the base layer for
+  new attention:
+    1. Build() could be invoked at least twice. So please make sure weights are
+       not duplicated.
+    2. Layer.get_weights() might return different set of weights if the instance
+       has `query_layer`. The query_layer weights is not initialized until the
+       memory is configured.
+
+  Also note that this layer does not work with Keras model when
+  `model.compile(run_eagerly=True)` due to the fact that this layer is stateful.
+  The support for that will be added in a future version.
   """
 
   def __init__(self,
+               memory,
                probability_fn,
                query_layer=None,
                memory_layer=None,
+               memory_sequence_length=None,
                **kwargs):
     """Construct base AttentionMechanism class.
 
     Args:
+      memory: The memory to query; usually the output of an RNN encoder.  This
+        tensor should be shaped `[batch_size, max_time, ...]`.
       probability_fn: A `callable`. Converts the score and previous alignments
         to probabilities. Its signature should be:
         `probabilities = probability_fn(score, state)`.
@@ -247,6 +270,9 @@ class _BaseAttentionMechanismV2(AttentionMechanism, layers.Layer):
         depth must match the depth of `query_layer`.
         If `memory_layer` is not provided, the shape of `memory` must match
         that of `query_layer`.
+      memory_sequence_length (optional): Sequence lengths for the batch entries
+        in memory. If provided, the memory tensor rows are masked with zeros
+        for values past the respective sequence lengths.
       **kwargs: Dictionary that contains other common arguments for layer
         creation.
     """
@@ -273,20 +299,128 @@ class _BaseAttentionMechanismV2(AttentionMechanism, layers.Layer):
     self.batch_size = None
     self._memory_initialized = False
     self._check_inner_dims_defined = True
+    self.supports_masking = True
+    self.score_mask_value = dtypes.as_dtype(self.dtype).as_numpy_dtype(-np.inf)
+
+    if memory is not None:
+      # Setup the memory by self.__call__() with memory and memory_seq_length.
+      # This will make the attention follow the keras convention which takes
+      # all the tensor inputs via __call__().
+      if memory_sequence_length is None:
+        inputs = memory
+      else:
+        inputs = [memory, memory_sequence_length]
+      # The returned tensor is ignored here since we already record it as an
+      # attribute.
+      super(_BaseAttentionMechanismV2, self).__call__(
+          inputs, setup_memory=True)
 
   def build(self, input_shape):
-    # The layer suppose to take 3 inputs, [query, state, memory].
-    query_input_shape, _, memory_input_shape = input_shape
-    if self.query_layer is not None:
-      self.query_layer.build(query_input_shape)
-    if self.memory_layer is not None:
-      self.memory_layer.build(memory_input_shape)
-    # dtype of the layer is known at this moment, create the score_mask_value if
-    # needed.
-    self.score_mask_value = dtypes.as_dtype(self.dtype).as_numpy_dtype(-np.inf)
-    self.built = True
+    if not self._memory_initialized:
+      # This is for setting up the memory, which contains memory and optional
+      # memory_sequence_length. Build the memory_layer with memory shape.
+      if self.memory_layer is not None and not self.memory_layer.built:
+        if isinstance(input_shape, list):
+          self.memory_layer.build(input_shape[0])
+        else:
+          self.memory_layer.build(input_shape)
+    else:
+      # The input_shape should be query.shape and state.shape. Use the query
+      # to init the query layer.
+      if self.query_layer is not None and not self.query_layer.built:
+        self.query_layer.build(input_shape[0])
+
+  def __call__(self, inputs, **kwargs):
+    """Preprocess the inputs before calling `base_layer.__call__()`.
+
+    Note that there are situation here, one for setup memory, and one with
+    actual query and state.
+    1. When the memory has not been configured, we just pass all the param to
+    base_layer.__call__(), which will then invoke self.call() with proper
+    inputs, which allows this class to setup memory.
+    2. When the memory has already been setup, the input should contain query
+    and state, and optionally processed memory. If the processed memory is
+    not included in the input, we will have to append it to the inputs and
+    give it to the base_layer.__call__(). The processed memory is the output
+    of first invocation of self.__call__(). If we don't add it here, then from
+    keras perspective, the graph is disconnected since the output from
+    previous call is never used.
+
+    Args:
+      inputs: the inputs tensors.
+      **kwargs: dict, other keyeword arguments for the `__call__()`
+    """
+    if self._memory_initialized:
+      if len(inputs) not in (2, 3):
+        raise ValueError("Expect the inputs to have 2 or 3 tensors, got %d" %
+                         len(inputs))
+      if len(inputs) == 2:
+        # We append the calculated memory here so that the graph will be
+        # connected.
+        inputs.append(self.values)
+    return super(_BaseAttentionMechanismV2, self).__call__(inputs, **kwargs)
+
+  def call(self, inputs, mask=None, setup_memory=False, **kwargs):
+    """Setup the memory or query the attention.
+
+    There are two case here, one for setup memory, and the second is query the
+    attention score. `setup_memory` is the flag to indicate which mode it is.
+    The input list will be treated differently based on that flag.
 
-  def _setup_memory(self, memory, memory_mask=None):
+    Args:
+      inputs: a list of tensor that could either be `query` and `state`, or
+        `memory` and `memory_sequence_length`.
+        `query` is the tensor of dtype matching `memory` and shape
+        `[batch_size, query_depth]`.
+        `state` is the tensor of dtype matching `memory` and shape
+        `[batch_size, alignments_size]`. (`alignments_size` is memory's
+        `max_time`).
+        `memory` is the memory to query; usually the output of an RNN encoder.
+        The tensor should be shaped `[batch_size, max_time, ...]`.
+        `memory_sequence_length` (optional) is the sequence lengths for the
+         batch entries in memory. If provided, the memory tensor rows are masked
+        with zeros for values past the respective sequence lengths.
+      mask: optional bool tensor with shape `[batch, max_time]` for the mask of
+        memory. If it is not None, the corresponding item of the memory should
+        be filtered out during calculation.
+      setup_memory: boolean, whether the input is for setting up memory, or
+        query attention.
+      **kwargs: Dict, other keyword arguments for the call method.
+    Returns:
+      Either processed memory or attention score, based on `setup_memory`.
+    """
+    if setup_memory:
+      if isinstance(inputs, list):
+        if len(inputs) not in (1, 2):
+          raise ValueError("Expect inputs to have 1 or 2 tensors, got %d" %
+                           len(inputs))
+        memory = inputs[0]
+        memory_sequence_length = inputs[1] if len(inputs) == 2 else None
+        memory_mask = mask
+      else:
+        memory, memory_sequence_length = inputs, None
+        memory_mask = mask
+      self._setup_memory(memory, memory_sequence_length, memory_mask)
+      # We force the self.built to false here since only memory is initialized,
+      # but the real query/state has not been call() yet. The layer should be
+      # build and call again.
+      self.built = False
+      # We have to return something here since layers are required to have some
+      # output. Use self.value here since it is the processed memory.
+      return self.values
+    else:
+      if not self._memory_initialized:
+        raise ValueError("Cannot query the attention before the setup of "
+                         "memory")
+      if len(inputs) not in (2, 3):
+        raise ValueError("Expect the inputs to have query, state, and optional "
+                         "processed memory, got %d items" % len(inputs))
+      # Ignore the rest of the inputs and only care about the query and state
+      query, state = inputs[0], inputs[1]
+      return self._calculate_attention(query, state)
+
+  def _setup_memory(self, memory, memory_sequence_length=None,
+                    memory_mask=None):
     """Pre-process the memory before actually query the memory.
 
     This should only be called once at the first invocation of call().
@@ -294,17 +428,30 @@ class _BaseAttentionMechanismV2(AttentionMechanism, layers.Layer):
     Args:
       memory: The memory to query; usually the output of an RNN encoder. This
         tensor should be shaped `[batch_size, max_time, ...]`.
-      memory_mask: The boolean tensor with shape `[batch_size, max_time]`. For
-        any value equal to False, the corresponding value in memory should be
-        ignored.
+      memory_sequence_length (optional): Sequence lengths for the batch entries
+        in memory. If provided, the memory tensor rows are masked with zeros for
+        values past the respective sequence lengths.
+      memory_mask: (Optional) The boolean tensor with shape `[batch_size,
+        max_time]`. For any value equal to False, the corresponding value in
+        memory should be ignored.
     """
     if self._memory_initialized:
       raise ValueError("The memory for the attention has already been setup.")
+    if memory_sequence_length is not None and memory_mask is not None:
+      raise ValueError("memory_sequence_length and memory_mask cannot be "
+                       "used at same time for attention.")
     with ops.name_scope(
         self.name, "BaseAttentionMechanismInit", nest.flatten(memory)):
       self.values = _prepare_memory(
-          memory, memory_mask=memory_mask,
+          memory,
+          memory_sequence_length=memory_sequence_length,
+          memory_mask=memory_mask,
           check_inner_dims_defined=self._check_inner_dims_defined)
+      # Mark the value as check since the memory and memory mask might not
+      # passed from __call__(), which does not have proper keras metadata.
+      # TODO(omalleyt): Remove this hack once the mask the has proper keras
+      # history.
+      base_layer_utils.mark_checked(self.values)
       if self.memory_layer is not None:
         self.keys = self.memory_layer(self.values)
       else:
@@ -315,36 +462,25 @@ class _BaseAttentionMechanismV2(AttentionMechanism, layers.Layer):
       self._alignments_size = (tensor_shape.dimension_value(self.keys.shape[1])
                                or array_ops.shape(self.keys)[1])
       if memory_mask is not None:
-        self.probability_fn = lambda score, prev: (  # pylint:disable=g-long-lambda
-            self.probability_fn(_maybe_mask_score(
-                score, self.score_mask_value, memory_mask=memory_mask), prev))
+        unwrapped_probability_fn = self.probability_fn
+        def _mask_probability_fn(score, prev):
+          return unwrapped_probability_fn(
+              _maybe_mask_score(
+                  score,
+                  memory_mask=memory_mask,
+                  memory_sequence_length=memory_sequence_length,
+                  score_mask_value=self.score_mask_value), prev)
+        self.probability_fn = _mask_probability_fn
     self._memory_initialized = True
 
-  def call(self, inputs, mask=None, **kwargs):
-    """Base method to calculate the attention score.
-
-    Args:
-      inputs: a list of tensor that contains `query`, `state`, and `memory`.
-        `query` is the tensor of dtype matching `memory` and shape
-        `[batch_size, query_depth]`.
-        `state` is the tensor of dtype matching `memory` and shape
-        `[batch_size, alignments_size]`. (`alignments_size` is memory's
-        `max_time`).
-        `memory` is the memory to query; usually the output of an RNN encoder.
-        This tensor should be shaped `[batch_size, max_time, feature]`.
-      mask: optional bool tensor with shape `[batch, max_time]` for the mask of
-        memory. If it is not None, the corresponding item of the memory should
-        be filtered out during calculation.
-      **kwargs: Dict, other keyword arguments for the call method.
-    """
-    query, state, memory, memory_mask = self._process_inputs(inputs, mask)
-    if not self._memory_initialized:
-      self._setup_memory(memory, memory_mask=memory_mask)
-    return self.calculate_attention(query, state)
-
-  def calculate_attention(self, query, state):
+  def _calculate_attention(self, query, state):
     raise NotImplementedError(
-        "calculate_attention need to be implemented by subclasses.")
+        "_calculate_attention need to be implemented by subclasses.")
+
+  def compute_mask(self, inputs, mask=None):
+    # There real input of the attention is query and state, and the memory layer
+    # mask shouldn't be pass down. Returning None for all output mask here.
+    return None, None
 
   def get_config(self):
     config = {}
@@ -361,16 +497,12 @@ class _BaseAttentionMechanismV2(AttentionMechanism, layers.Layer):
           "class_name": self.memory_layer.__class__.__name__,
           "config": self.memory_layer.get_config(),
       }
+    # memory is a required init parameter and its a tensor. It cannot be
+    # serialized to config, so we put a placeholder for it.
+    config["memory"] = None
     base_config = super(_BaseAttentionMechanismV2, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
 
-  def _process_inputs(self, inputs, mask):
-    if len(inputs) != 3:
-      raise ValueError(
-          "Expect to have 3 inputs for attention, got %d" % len(inputs))
-    query, state, memory = inputs
-    return query, state, memory, mask
-
   def _process_probability_fn(self, func_name):
     """Helper method to retrieve the probably function by string input."""
     valid_probability_fns = {
@@ -418,6 +550,46 @@ class _BaseAttentionMechanismV2(AttentionMechanism, layers.Layer):
   def alignments_size(self):
     return self._alignments_size
 
+  @property
+  def state_size(self):
+    return self._alignments_size
+
+  def initial_alignments(self, batch_size, dtype):
+    """Creates the initial alignment values for the `AttentionWrapper` class.
+
+    This is important for AttentionMechanisms that use the previous alignment
+    to calculate the alignment at the next time step (e.g. monotonic attention).
+
+    The default behavior is to return a tensor of all zeros.
+
+    Args:
+      batch_size: `int32` scalar, the batch_size.
+      dtype: The `dtype`.
+
+    Returns:
+      A `dtype` tensor shaped `[batch_size, alignments_size]`
+      (`alignments_size` is the values' `max_time`).
+    """
+    max_time = self._alignments_size
+    return _zero_state_tensors(max_time, batch_size, dtype)
+
+  def initial_state(self, batch_size, dtype):
+    """Creates the initial state values for the `AttentionWrapper` class.
+
+    This is important for AttentionMechanisms that use the previous alignment
+    to calculate the alignment at the next time step (e.g. monotonic attention).
+
+    The default behavior is to return the same output as initial_alignments.
+
+    Args:
+      batch_size: `int32` scalar, the batch_size.
+      dtype: The `dtype`.
+
+    Returns:
+      A structure of all-zero tensors with shapes as described by `state_size`.
+    """
+    return self.initial_alignments(batch_size, dtype)
+
 
 def _luong_score(query, keys, scale):
   """Implements Luong-style (multiplicative) scoring function.
@@ -587,6 +759,8 @@ class LuongAttentionV2(_BaseAttentionMechanismV2):
 
   def __init__(self,
                units,
+               memory,
+               memory_sequence_length=None,
                scale=False,
                probability_fn="softmax",
                dtype=None,
@@ -596,6 +770,11 @@ class LuongAttentionV2(_BaseAttentionMechanismV2):
 
     Args:
       units: The depth of the attention mechanism.
+      memory: The memory to query; usually the output of an RNN encoder.  This
+        tensor should be shaped `[batch_size, max_time, ...]`.
+      memory_sequence_length: (optional): Sequence lengths for the batch entries
+        in memory.  If provided, the memory tensor rows are masked with zeros
+        for values past the respective sequence lengths.
       scale: Python boolean. Whether to scale the energy term.
       probability_fn: (optional) string, the name of function to convert the
         attention score to probabilities. The default is `softmax` which is
@@ -618,26 +797,27 @@ class LuongAttentionV2(_BaseAttentionMechanismV2):
     if not memory_layer:
       memory_layer = layers.Dense(
           units, name="memory_layer", use_bias=False, dtype=dtype)
+    self.units = units
+    self.scale = scale
+    self.scale_weight = None
     super(LuongAttentionV2, self).__init__(
+        memory=memory,
+        memory_sequence_length=memory_sequence_length,
         query_layer=None,
         memory_layer=memory_layer,
         probability_fn=wrapped_probability_fn,
         name=name,
         dtype=dtype,
         **kwargs)
-    self.units = units
-    self.scale = scale
 
   def build(self, input_shape):
     super(LuongAttentionV2, self).build(input_shape)
-    if self.scale:
+    if self.scale and self.scale_weight is None:
       self.scale_weight = self.add_weight(
           "attention_g", initializer=init_ops.ones_initializer, shape=())
-    else:
-      self.scale_weight = None
     self.built = True
 
-  def calculate_attention(self, query, state):
+  def _calculate_attention(self, query, state):
     """Score the query based on the keys and values.
 
     Args:
@@ -851,8 +1031,11 @@ class BahdanauAttentionV2(_BaseAttentionMechanismV2):
 
   def __init__(self,
                units,
+               memory,
+               memory_sequence_length=None,
                normalize=False,
                probability_fn="softmax",
+               kernel_initializer="glorot_uniform",
                dtype=None,
                name="BahdanauAttention",
                **kwargs):
@@ -860,12 +1043,19 @@ class BahdanauAttentionV2(_BaseAttentionMechanismV2):
 
     Args:
       units: The depth of the query mechanism.
+      memory: The memory to query; usually the output of an RNN encoder.  This
+        tensor should be shaped `[batch_size, max_time, ...]`.
+      memory_sequence_length: (optional): Sequence lengths for the batch entries
+        in memory.  If provided, the memory tensor rows are masked with zeros
+        for values past the respective sequence lengths.
       normalize: Python boolean.  Whether to normalize the energy term.
       probability_fn: (optional) string, the name of function to convert the
         attention score to probabilities. The default is `softmax` which is
         `tf.nn.softmax`. Other options is `hardmax`, which is hardmax() within
         this module. Any other value will result into validation error. Default
         to use `softmax`.
+      kernel_initializer: (optional), the name of the initializer for the
+        attention kernel.
       dtype: The data type for the query and memory layers of the attention
         mechanism.
       name: Name to use when creating ops.
@@ -885,33 +1075,39 @@ class BahdanauAttentionV2(_BaseAttentionMechanismV2):
     if not memory_layer:
       memory_layer = layers.Dense(
           units, name="memory_layer", use_bias=False, dtype=dtype)
+    self.units = units
+    self.normalize = normalize
+    self.kernel_initializer = initializers.get(kernel_initializer)
+    self.attention_v = None
+    self.attention_g = None
+    self.attention_b = None
     super(BahdanauAttentionV2, self).__init__(
+        memory=memory,
+        memory_sequence_length=memory_sequence_length,
         query_layer=query_layer,
         memory_layer=memory_layer,
         probability_fn=wrapped_probability_fn,
         name=name,
         dtype=dtype,
         **kwargs)
-    self.units = units
-    self.normalize = normalize
 
   def build(self, input_shape):
     super(BahdanauAttentionV2, self).build(input_shape)
-    self.attention_v = self.add_weight(
-        "attention_v", [self.units], dtype=self.dtype)
-    if self.normalize:
+    if self.attention_v is None:
+      self.attention_v = self.add_weight(
+          "attention_v", [self.units],
+          dtype=self.dtype,
+          initializer=self.kernel_initializer)
+    if self.normalize and self.attention_g is None and self.attention_b is None:
       self.attention_g = self.add_weight(
           "attention_g", initializer=init_ops.constant_initializer(
               math.sqrt((1. / self.units))), shape=())
       self.attention_b = self.add_weight(
           "attention_b", shape=[self.units],
           initializer=init_ops.zeros_initializer())
-    else:
-      self.attention_g = None
-      self.attention_b = None
     self.built = True
 
-  def calculate_attention(self, query, state):
+  def _calculate_attention(self, query, state):
     """Score the query based on the keys and values.
 
     Args:
@@ -940,6 +1136,7 @@ class BahdanauAttentionV2(_BaseAttentionMechanismV2):
         "units": self.units,
         "normalize": self.normalize,
         "probability_fn": self.probability_fn_name,
+        "kernel_initializer": initializers.serialize(self.kernel_initializer)
     }
     base_config = super(BahdanauAttentionV2, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
@@ -1299,11 +1496,14 @@ class BahdanauMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
 
   def __init__(self,
                units,
+               memory,
+               memory_sequence_length=None,
                normalize=False,
                sigmoid_noise=0.,
                sigmoid_noise_seed=None,
                score_bias_init=0.,
                mode="parallel",
+               kernel_initializer="glorot_uniform",
                dtype=None,
                name="BahdanauMonotonicAttention",
                **kwargs):
@@ -1311,6 +1511,11 @@ class BahdanauMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
 
     Args:
       units: The depth of the query mechanism.
+      memory: The memory to query; usually the output of an RNN encoder.  This
+        tensor should be shaped `[batch_size, max_time, ...]`.
+      memory_sequence_length: (optional): Sequence lengths for the batch entries
+        in memory.  If provided, the memory tensor rows are masked with zeros
+        for values past the respective sequence lengths.
       normalize: Python boolean. Whether to normalize the energy term.
       sigmoid_noise: Standard deviation of pre-sigmoid noise. See the docstring
         for `_monotonic_probability_fn` for more information.
@@ -1321,6 +1526,8 @@ class BahdanauMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
       mode: How to compute the attention distribution. Must be one of
         'recursive', 'parallel', or 'hard'. See the docstring for
         `tf.contrib.seq2seq.monotonic_attention` for more information.
+      kernel_initializer: (optional), the name of the initializer for the
+        attention kernel.
       dtype: The data type for the query and memory layers of the attention
         mechanism.
       name: Name to use when creating ops.
@@ -1341,32 +1548,39 @@ class BahdanauMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
     if not memory_layer:
       memory_layer = layers.Dense(
           units, name="memory_layer", use_bias=False, dtype=dtype)
+    self.units = units
+    self.normalize = normalize
+    self.sigmoid_noise = sigmoid_noise
+    self.sigmoid_noise_seed = sigmoid_noise_seed
+    self.score_bias_init = score_bias_init
+    self.mode = mode
+    self.kernel_initializer = initializers.get(kernel_initializer)
+    self.attention_v = None
+    self.attention_score_bias = None
+    self.attention_g = None
+    self.attention_b = None
     super(BahdanauMonotonicAttentionV2, self).__init__(
+        memory=memory,
+        memory_sequence_length=memory_sequence_length,
         query_layer=query_layer,
         memory_layer=memory_layer,
         probability_fn=wrapped_probability_fn,
         name=name,
         dtype=dtype,
         **kwargs)
-    self.units = units
-    self.normalize = normalize
-    self.sigmoid_noise = sigmoid_noise
-    self.sigmoid_noise_seed = sigmoid_noise_seed
-    self.score_bias_init = score_bias_init
-    self.mode = mode
 
   def build(self, input_shape):
     super(BahdanauMonotonicAttentionV2, self).build(input_shape)
-    self.attention_v = self.add_weight(
-        "attention_v", [self.units], dtype=self.dtype)
-    self.attention_score_bias = self.add_weight(
-        "attention_score_bias", shape=(), dtype=self.dtype,
-        initializer=init_ops.constant_initializer(
-            self.score_bias_init, dtype=self.dtype))
-    if not self.normalize:
-      self.attention_g = None
-      self.attention_b = None
-    else:
+    if self.attention_v is None:
+      self.attention_v = self.add_weight(
+          "attention_v", [self.units], dtype=self.dtype,
+          initializer=self.kernel_initializer)
+    if self.attention_score_bias is None:
+      self.attention_score_bias = self.add_weight(
+          "attention_score_bias", shape=(), dtype=self.dtype,
+          initializer=init_ops.constant_initializer(
+              self.score_bias_init, dtype=self.dtype))
+    if self.normalize and self.attention_g is None and self.attention_b is None:
       self.attention_g = self.add_weight(
           "attention_g", dtype=self.dtype,
           initializer=init_ops.constant_initializer(
@@ -1377,7 +1591,7 @@ class BahdanauMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
           initializer=init_ops.zeros_initializer())
     self.built = True
 
-  def calculate_attention(self, query, state):
+  def _calculate_attention(self, query, state):
     """Score the query based on the keys and values.
 
     Args:
@@ -1409,6 +1623,7 @@ class BahdanauMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
         "sigmoid_noise_seed": self.sigmoid_noise_seed,
         "score_bias_init": self.score_bias_init,
         "mode": self.mode,
+        "kernel_initializer": initializers.serialize(self.kernel_initializer),
     }
     base_config = super(BahdanauMonotonicAttentionV2, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
@@ -1542,6 +1757,8 @@ class LuongMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
 
   def __init__(self,
                units,
+               memory,
+               memory_sequence_length=None,
                scale=False,
                sigmoid_noise=0.,
                sigmoid_noise_seed=None,
@@ -1554,6 +1771,11 @@ class LuongMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
 
     Args:
       units: The depth of the query mechanism.
+      memory: The memory to query; usually the output of an RNN encoder.  This
+        tensor should be shaped `[batch_size, max_time, ...]`.
+      memory_sequence_length: (optional): Sequence lengths for the batch entries
+        in memory.  If provided, the memory tensor rows are masked with zeros
+        for values past the respective sequence lengths.
       scale: Python boolean.  Whether to scale the energy term.
       sigmoid_noise: Standard deviation of pre-sigmoid noise.  See the docstring
         for `_monotonic_probability_fn` for more information.
@@ -1580,34 +1802,37 @@ class LuongMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
     if not memory_layer:
       memory_layer = layers.Dense(
           units, name="memory_layer", use_bias=False, dtype=dtype)
+    self.units = units
+    self.scale = scale
+    self.sigmoid_noise = sigmoid_noise
+    self.sigmoid_noise_seed = sigmoid_noise_seed
+    self.score_bias_init = score_bias_init
+    self.mode = mode
+    self.attention_g = None
+    self.attention_score_bias = None
     super(LuongMonotonicAttentionV2, self).__init__(
+        memory=memory,
+        memory_sequence_length=memory_sequence_length,
         query_layer=None,
         memory_layer=memory_layer,
         probability_fn=wrapped_probability_fn,
         name=name,
         dtype=dtype,
         **kwargs)
-    self.units = units
-    self.scale = scale
-    self.sigmoid_noise = sigmoid_noise
-    self.sigmoid_noise_seed = sigmoid_noise_seed
-    self.score_bias_init = score_bias_init
-    self.mode = mode
 
   def build(self, input_shape):
     super(LuongMonotonicAttentionV2, self).build(input_shape)
-    if self.scale:
+    if self.scale and self.attention_g is None:
       self.attention_g = self.add_weight(
           "attention_g", initializer=init_ops.ones_initializer, shape=())
-    else:
-      self.attention_g = None
-    self.attention_score_bias = self.add_weight(
-        "attention_score_bias", shape=(),
-        initializer=init_ops.constant_initializer(
-            self.score_bias_init, dtype=self.dtype))
+    if self.attention_score_bias is None:
+      self.attention_score_bias = self.add_weight(
+          "attention_score_bias", shape=(),
+          initializer=init_ops.constant_initializer(
+              self.score_bias_init, dtype=self.dtype))
     self.built = True
 
-  def calculate_attention(self, query, state):
+  def _calculate_attention(self, query, state):
     """Score the query based on the keys and values.
 
     Args:
@@ -1739,41 +1964,26 @@ def _prepare_memory(memory, memory_sequence_length=None, memory_mask=None,
                          "but saw shape: %s" % (m.name, m.get_shape()))
     nest.map_structure(_check_dims, memory)
   if memory_sequence_length is None and memory_mask is None:
-    seq_len_mask = None
-    seq_len_batch_size = None
+    return memory
   elif memory_sequence_length is not None:
     seq_len_mask = array_ops.sequence_mask(
         memory_sequence_length,
         maxlen=array_ops.shape(nest.flatten(memory)[0])[1],
         dtype=nest.flatten(memory)[0].dtype)
-    seq_len_batch_size = (
-        tensor_shape.dimension_value(memory_sequence_length.shape[0])
-        or array_ops.shape(memory_sequence_length)[0])
   else:
     # For memory_mask is not None
-    seq_len_mask = memory_mask
-    seq_len_batch_size = (
-        tensor_shape.dimension_value(memory_mask.shape[0])
-        or array_ops.shape(memory_mask)[0])
+    seq_len_mask = math_ops.cast(
+        memory_mask, dtype=nest.flatten(memory)[0].dtype)
   def _maybe_mask(m, seq_len_mask):
     """Mask the memory based on the memory mask."""
     rank = m.get_shape().ndims
     rank = rank if rank is not None else array_ops.rank(m)
     extra_ones = array_ops.ones(rank - 2, dtype=dtypes.int32)
-    m_batch_size = tensor_shape.dimension_value(
-        m.shape[0]) or array_ops.shape(m)[0]
-    if seq_len_batch_size is not None:
-      message = ("memory_sequence_length and memory tensor batch sizes do not "
-                 "match.")
-      with ops.control_dependencies([
-          check_ops.assert_equal(
-              seq_len_batch_size, m_batch_size, message=message)]):
-        seq_len_mask = array_ops.reshape(
-            seq_len_mask,
-            array_ops.concat((array_ops.shape(seq_len_mask), extra_ones), 0))
-        return m * seq_len_mask
-    else:
-      return m
+    seq_len_mask = array_ops.reshape(
+        seq_len_mask,
+        array_ops.concat((array_ops.shape(seq_len_mask), extra_ones), 0))
+    return m * seq_len_mask
+
   return nest.map_structure(lambda m: _maybe_mask(m, seq_len_mask), memory)
 
 
@@ -1819,8 +2029,14 @@ def hardmax(logits, name=None):
 def _compute_attention(attention_mechanism, cell_output, attention_state,
                        attention_layer):
   """Computes the attention and alignments for a given attention_mechanism."""
-  alignments, next_attention_state = attention_mechanism(
-      cell_output, state=attention_state)
+  if isinstance(attention_mechanism, _BaseAttentionMechanismV2):
+    alignments, next_attention_state = attention_mechanism(
+        [cell_output, attention_state])
+  else:
+    # For other class, assume they are following _BaseAttentionMechanism, which
+    # takes query and state as separate parameter.
+    alignments, next_attention_state = attention_mechanism(
+        cell_output, state=attention_state)
 
   # Reshape from [batch_size, memory_time] to [batch_size, 1, memory_time]
   expanded_alignments = array_ops.expand_dims(alignments, 1)
-- 
GitLab


From cfef15e974394a797e9f48198f1d59b67b23166f Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Mon, 11 Feb 2019 09:56:22 -0800
Subject: [PATCH 2252/2345] Create int8 pack.

PiperOrigin-RevId: 233426860
---
 tensorflow/lite/kernels/pack.cc              |  8 +++--
 tensorflow/lite/kernels/pack_test.cc         | 31 ++++++++++++++++++++
 tensorflow/lite/kernels/register.cc          |  4 ++-
 tensorflow/lite/toco/tflite/operator.cc      |  6 ++++
 tensorflow/lite/toco/tflite/operator_test.cc |  2 ++
 5 files changed, 48 insertions(+), 3 deletions(-)

diff --git a/tensorflow/lite/kernels/pack.cc b/tensorflow/lite/kernels/pack.cc
index d15a5a08af..e26abaaff1 100644
--- a/tensorflow/lite/kernels/pack.cc
+++ b/tensorflow/lite/kernels/pack.cc
@@ -39,8 +39,8 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   // TODO(renjieliu): Support negative axis.
   TF_LITE_ENSURE(context, data->axis >= 0);
   if (input0->type != kTfLiteInt32 && input0->type != kTfLiteFloat32 &&
-      input0->type != kTfLiteUInt8 && input0->type != kTfLiteInt16 &&
-      input0->type != kTfLiteInt64) {
+      input0->type != kTfLiteUInt8 && input0->type != kTfLiteInt8 &&
+      input0->type != kTfLiteInt16 && input0->type != kTfLiteInt64) {
     context->ReportError(context, "Type '%s' is not supported by pack.",
                          TfLiteTypeGetName(input0->type));
     return kTfLiteError;
@@ -106,6 +106,10 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
       PackImpl<uint8_t>(context, node, output, data->values_count, data->axis);
       break;
     }
+    case kTfLiteInt8: {
+      PackImpl<int8_t>(context, node, output, data->values_count, data->axis);
+      break;
+    }
     case kTfLiteInt32: {
       PackImpl<int32_t>(context, node, output, data->values_count, data->axis);
       break;
diff --git a/tensorflow/lite/kernels/pack_test.cc b/tensorflow/lite/kernels/pack_test.cc
index 530cc2e50f..f44111567f 100644
--- a/tensorflow/lite/kernels/pack_test.cc
+++ b/tensorflow/lite/kernels/pack_test.cc
@@ -191,6 +191,37 @@ TEST(PackOpTest, Uint8MultilDimensions) {
               ElementsAreArray({1, 2, 3, 7, 8, 9, 4, 5, 6, 10, 11, 12}));
 }
 
+// int8
+TEST(PackOpTest, Int8ThreeInputs) {
+  PackOpModel<int8_t> model({TensorType_INT8, {2}}, 0, 3);
+  model.SetInput(0, {1, 4});
+  model.SetInput(1, {2, 5});
+  model.SetInput(2, {3, 6});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutputShape(), ElementsAre(3, 2));
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({1, 4, 2, 5, 3, 6}));
+}
+
+TEST(PackOpTest, Int8ThreeInputsDifferentAxis) {
+  PackOpModel<int8_t> model({TensorType_INT8, {2}}, 1, 3);
+  model.SetInput(0, {1, 4});
+  model.SetInput(1, {2, 5});
+  model.SetInput(2, {3, 6});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutputShape(), ElementsAre(2, 3));
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({1, 2, 3, 4, 5, 6}));
+}
+
+TEST(PackOpTest, Int8MultilDimensions) {
+  PackOpModel<int8_t> model({TensorType_INT8, {2, 3}}, 1, 2);
+  model.SetInput(0, {1, 2, 3, 4, 5, 6});
+  model.SetInput(1, {7, 8, 9, 10, 11, 12});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutputShape(), ElementsAre(2, 2, 3));
+  EXPECT_THAT(model.GetOutput(),
+              ElementsAreArray({1, 2, 3, 7, 8, 9, 4, 5, 6, 10, 11, 12}));
+}
+
 }  // namespace
 }  // namespace tflite
 
diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index 36feba8556..fd8f714d32 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -319,7 +319,9 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_SHAPE, Register_SHAPE());
   AddBuiltin(BuiltinOperator_POW, Register_POW());
   AddBuiltin(BuiltinOperator_FAKE_QUANT, Register_FAKE_QUANT(), 1, 2);
-  AddBuiltin(BuiltinOperator_PACK, Register_PACK());
+  AddBuiltin(BuiltinOperator_PACK, Register_PACK(),
+             /* min_version */ 1,
+             /* max_version */ 2);
   AddBuiltin(BuiltinOperator_ONE_HOT, Register_ONE_HOT());
   AddBuiltin(BuiltinOperator_LOGICAL_OR, Register_LOGICAL_OR());
   AddBuiltin(BuiltinOperator_LOGICAL_AND, Register_LOGICAL_AND());
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index 5f2de4a696..b69d614782 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -1563,6 +1563,12 @@ class Pack : public BuiltinOperator<PackOperator, ::tflite::PackOptions,
   }
 
   int GetVersion(const OperatorSignature& op_signature) const override {
+    const string& input_name = op_signature.op->inputs[0];
+    const Array& input_array = op_signature.model->GetArray(input_name);
+    // If the op take int8 input, it is version 2.
+    if (input_array.data_type == ArrayDataType::kInt8) {
+      return 2;
+    }
     return 1;
   }
 };
diff --git a/tensorflow/lite/toco/tflite/operator_test.cc b/tensorflow/lite/toco/tflite/operator_test.cc
index eb1c5e0ddf..a838d8fc0f 100644
--- a/tensorflow/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/lite/toco/tflite/operator_test.cc
@@ -798,6 +798,8 @@ TEST_F(OperatorTest, VersioningGreaterEqualTest) {
   VersioningTest<TensorFlowGreaterEqualOperator>();
 }
 
+TEST_F(OperatorTest, VersioningPackTest) { VersioningTest<PackOperator>(); }
+
 TEST_F(OperatorTest, VersioningSelectTest) {
   SelectOperator select_op;
   select_op.inputs = {"input1"};
-- 
GitLab


From 63a7aa8ac84af10cfaf108ac6cfbb29d3b9d3aff Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Mon, 11 Feb 2019 10:03:39 -0800
Subject: [PATCH 2253/2345] Add feature dimension id and default direction in
 split node info.

PiperOrigin-RevId: 233428439
---
 .../core/kernels/boosted_trees/boosted_trees.proto     | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tensorflow/core/kernels/boosted_trees/boosted_trees.proto b/tensorflow/core/kernels/boosted_trees/boosted_trees.proto
index 4e9bab3e21..3aa3bb84b9 100644
--- a/tensorflow/core/kernels/boosted_trees/boosted_trees.proto
+++ b/tensorflow/core/kernels/boosted_trees/boosted_trees.proto
@@ -52,6 +52,16 @@ message BucketizedSplit {
   // the rule feature <= threshold.
   int32 feature_id = 1;
   int32 threshold = 2;
+  // If feature column is multivalent, this holds the index of the dimension
+  // for the split. Defaults to 0.
+  int32 dimension_id = 5;
+  enum DefaultDirection {
+    // Left is the default direction.
+    DEFAULT_LEFT = 0;
+    DEFAULT_RIGHT = 1;
+  }
+  // default direction for missing values.
+  DefaultDirection default_direction = 6;
 
   // Node children indexing into a contiguous
   // vector of nodes starting from the root.
-- 
GitLab


From 835040cbbb16640f07ec3502abab26ad6c6dc2cc Mon Sep 17 00:00:00 2001
From: Tong Shen <endlessroad@google.com>
Date: Mon, 11 Feb 2019 10:06:53 -0800
Subject: [PATCH 2254/2345] Make dump_graph public.

PiperOrigin-RevId: 233429291
---
 tensorflow/compiler/tf2xla/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD
index 02de95141d..5a1a9435c1 100644
--- a/tensorflow/compiler/tf2xla/BUILD
+++ b/tensorflow/compiler/tf2xla/BUILD
@@ -448,6 +448,7 @@ cc_library(
     hdrs = [
         "dump_graph.h",
     ],
+    visibility = ["//visibility:public"],
     deps = [
         "//tensorflow/compiler/jit:flags",
         "//tensorflow/core:framework",
-- 
GitLab


From cd8e915071dcd3e89b17f5fae72c070e4de52213 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Mon, 11 Feb 2019 10:17:11 -0800
Subject: [PATCH 2255/2345] Support dynamic sized subgraphs in WHILE op

PiperOrigin-RevId: 233431292
---
 tensorflow/lite/kernels/while.cc      | 189 +++++++++++++++++++-------
 tensorflow/lite/kernels/while_test.cc |  11 +-
 2 files changed, 150 insertions(+), 50 deletions(-)

diff --git a/tensorflow/lite/kernels/while.cc b/tensorflow/lite/kernels/while.cc
index 237bec9b04..b8a6e1953b 100644
--- a/tensorflow/lite/kernels/while.cc
+++ b/tensorflow/lite/kernels/while.cc
@@ -26,25 +26,34 @@ namespace while_kernel {
 
 namespace {
 
-TfLiteStatus ResizeSubgraphInputs(TfLiteContext* context, TfLiteNode* node,
-                                  Subgraph* subgraph) {
-  int num_inputs = node->inputs->size;
-  for (int i = 0; i < num_inputs; ++i) {
-    const TfLiteTensor* input = GetInput(context, node, i);
-    std::vector<int> dims(input->dims->data,
-                          input->dims->data + input->dims->size);
-    subgraph->ResizeInputTensor(i, dims);
-    TfLiteTensor* subgraph_input = subgraph->tensor(subgraph->inputs()[i]);
-    TF_LITE_ENSURE_EQ(context, input->type, subgraph_input->type);
+// Propagate tensor shapes from `src_tensor_indices` in `src_subgraph`
+// to `dst_tensor_indices` in `dst_subgraph`.
+template <typename SrcVector, typename DstVector>
+TfLiteStatus CopyTensorsShape(TfLiteContext* context, Subgraph* src_subgraph,
+                              const SrcVector& src_tensor_indices,
+                              Subgraph* dst_subgraph,
+                              const DstVector& dst_tensor_indices) {
+  TF_LITE_ENSURE_EQ(context, src_tensor_indices.size(),
+                    dst_tensor_indices.size());
+  for (int i = 0; i < src_tensor_indices.size(); ++i) {
+    const TfLiteTensor* src_tensor =
+        src_subgraph->tensor(src_tensor_indices[i]);
+    std::vector<int> dims(src_tensor->dims->data,
+                          src_tensor->dims->data + src_tensor->dims->size);
+    dst_subgraph->ResizeInputTensor(dst_tensor_indices[i], dims);
+    TfLiteTensor* dst_tensor = dst_subgraph->tensor(dst_tensor_indices[i]);
+    TF_LITE_ENSURE_EQ(context, src_tensor->type, dst_tensor->type);
   }
   return kTfLiteOk;
 }
 
+// Copy the tensors data from tensors `src_tensor_indices` in `src_subgraph`
+// to `dst_tensor_indices` in `dst_subgraph`.
 template <typename SrcVector, typename DstVector>
-TfLiteStatus CopyTensors(TfLiteContext* context, Subgraph* src_subgraph,
-                         const SrcVector& src_tensor_indices,
-                         Subgraph* dst_subgraph,
-                         const DstVector& dst_tensor_indices) {
+TfLiteStatus CopyTensorsData(TfLiteContext* context, Subgraph* src_subgraph,
+                             const SrcVector& src_tensor_indices,
+                             Subgraph* dst_subgraph,
+                             const DstVector& dst_tensor_indices) {
   TF_LITE_ENSURE_EQ(context, src_tensor_indices.size(),
                     dst_tensor_indices.size());
   for (int i = 0; i < src_tensor_indices.size(); ++i) {
@@ -62,6 +71,7 @@ TfLiteStatus CopyTensors(TfLiteContext* context, Subgraph* src_subgraph,
 struct OpData {
   int cond_subgraph_index;
   int body_subgraph_index;
+  bool body_has_dynamic_output_tensors;
 };
 
 void* Init(TfLiteContext* context, const char* buffer, size_t length) {
@@ -70,6 +80,7 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) {
   const flexbuffers::Map& m = flexbuffers::GetRoot(buffer_t, length).AsMap();
   op_data->cond_subgraph_index = m["cond_subgraph_index"].AsInt32();
   op_data->body_subgraph_index = m["body_subgraph_index"].AsInt32();
+  op_data->body_has_dynamic_output_tensors = false;
   return op_data;
 }
 
@@ -78,7 +89,7 @@ void Free(TfLiteContext* context, void* buffer) {
 }
 
 TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
-  const OpData* op_data = reinterpret_cast<OpData*>(node->user_data);
+  OpData* op_data = reinterpret_cast<OpData*>(node->user_data);
   int num_inputs = node->inputs->size;
   // The number of outputs should be the same as number of inputs.
   TF_LITE_ENSURE_EQ(context, node->outputs->size, num_inputs);
@@ -101,37 +112,63 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   TF_LITE_ENSURE_EQ(context, body_subgraph->outputs().size(), num_inputs);
 
   // Prepare and check the condition subgraph.
-  ResizeSubgraphInputs(context, node, cond_subgraph);
+  TF_LITE_ENSURE_OK(
+      context,
+      CopyTensorsShape(context, this_subgraph, TfLiteIntArrayView(node->inputs),
+                       cond_subgraph, cond_subgraph->inputs()));
   TF_LITE_ENSURE_OK(context, cond_subgraph->AllocateTensors());
   TfLiteTensor* cond_output =
       cond_subgraph->tensor(cond_subgraph->outputs()[0]);
+  // TODO(ycling): Handle the case the cond subgraph has dynamic tensor outputs.
+  // This should rarely happens. In most cases the output is static with shape
+  // [1]. However theoretically intermediate tensors in the cond subgraph
+  // can be dynamic.
+  TF_LITE_ENSURE(context, !IsDynamicTensor(cond_output));
   // The condition output must be a single boolean value.
   TF_LITE_ENSURE_EQ(context, cond_output->type, kTfLiteBool);
   TF_LITE_ENSURE_EQ(context, cond_output->dims->size, 1);
   TF_LITE_ENSURE_EQ(context, cond_output->dims->data[0], 1);
 
-  // TODO(ycling): Handle the case where condition graph has dynamic
-  // sized tensors.
-
   // Prepare and check the body subgraph.
-  ResizeSubgraphInputs(context, node, body_subgraph);
+  TF_LITE_ENSURE_OK(
+      context,
+      CopyTensorsShape(context, this_subgraph, TfLiteIntArrayView(node->inputs),
+                       body_subgraph, body_subgraph->inputs()));
   TF_LITE_ENSURE_OK(context, body_subgraph->AllocateTensors());
-  for (int i = 0; i < num_inputs; ++i) {
-    TfLiteTensor* body_input =
-        body_subgraph->tensor(body_subgraph->inputs()[i]);
-    TfLiteTensor* body_output =
-        body_subgraph->tensor(body_subgraph->outputs()[i]);
-    TF_LITE_ENSURE_EQ(context, body_input->type, body_output->type);
-
-    // TODO(ycling): Support dynamic sized body subgraph.
-    TF_LITE_ENSURE(context, !IsDynamicTensor(body_output));
-    TF_LITE_ENSURE(context,
-                   TfLiteIntArrayEqual(body_input->dims, body_output->dims));
+  if (body_subgraph->HasDynamicTensors()) {
+    op_data->body_has_dynamic_output_tensors = true;
+  } else {
+    for (int i = 0; i < num_inputs; ++i) {
+      TfLiteTensor* body_input =
+          body_subgraph->tensor(body_subgraph->inputs()[i]);
+      TfLiteTensor* body_output =
+          body_subgraph->tensor(body_subgraph->outputs()[i]);
+      TF_LITE_ENSURE_EQ(context, body_input->type, body_output->type);
 
+      // TODO(ycling): Support dynamic sized body subgraph.
+      TF_LITE_ENSURE(context, !IsDynamicTensor(body_output));
+      if (!TfLiteIntArrayEqual(body_input->dims, body_output->dims)) {
+        // If the output shape of the body subgraph is static w.r.t. a fixed
+        // input size, but it's different from input size, it's still considered
+        // dynamic. For example: If a subgraph keeps padding its input with a
+        // fixed padding, the output shape is static w.r.t the input shape and
+        // padding, but running it in a loop will keep bloating the tensor.
+        op_data->body_has_dynamic_output_tensors = true;
+        break;
+      }
+    }
+  }
+  for (int i = 0; i < num_inputs; ++i) {
     TfLiteTensor* output = GetOutput(context, node, i);
-    TfLiteIntArray* output_size = TfLiteIntArrayCopy(body_output->dims);
-    TF_LITE_ENSURE_OK(context,
-                      context->ResizeTensor(context, output, output_size));
+    if (op_data->body_has_dynamic_output_tensors) {
+      SetTensorToDynamic(output);
+    } else {
+      TfLiteTensor* body_output =
+          body_subgraph->tensor(body_subgraph->outputs()[i]);
+      TfLiteIntArray* output_size = TfLiteIntArrayCopy(body_output->dims);
+      TF_LITE_ENSURE_OK(context,
+                        context->ResizeTensor(context, output, output_size));
+    }
   }
   return kTfLiteOk;
 }
@@ -143,17 +180,45 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   Subgraph* cond_subgraph = (*subgraphs)[op_data->cond_subgraph_index].get();
   Subgraph* body_subgraph = (*subgraphs)[op_data->body_subgraph_index].get();
 
-  // Currently we copy the input / output between the subgraphs. This isn't
-  // optimized yet.
+  // The follow graph illustrates the current implementation.
+  //
+  // This Subgraph          Cond Subgraph         Body Subgraph
+  // +-----------+   (1)   +------------+   (3)   +------------+
+  // |   WHILE   |-------->|  SUBGRAPH  |-------->|  SUBGRAPH  |
+  // |   INPUT   |        /|   INPUT    |<-----   |   INPUT    |
+  // +-----------+       / +------------+      \  +------------+
+  //                    /        |              \       |
+  //               (6) /         | (2)       (5) \      | (4)
+  //                  /          v                \     v
+  // +-----------+   /     +------------+         +------------+
+  // |   WHILE   |<--      |  SUBGRAPH  |         |  SUBGRAPH  |
+  // |   OUTPUT  |         |   OUTPUT   |         |   OUTPUT   |
+  // +-----------+         +------------+         +------------+
+  //
+  // (1) Copy the inputs of WHILE op to the inputs of condition subgraph.
+  // (2) Invoke condition subgraph.
+  //     Jump to step 5 if result is false.
+  // (3) Copy the inputs of condition subgraph to the inputs of body subgraph.
+  // (4) Invoke body subgraph.
+  // (5) Copy the outputs of body subgraph to the inputs condition subgraph.
+  //     Jump back to step 2!
+  // (6) Copy the inputs of condition subgraph to the outputs of WHILE op.
+  //
+  // If the body subgraph has dynamic sized outputs, it's required to resize the
+  // tensor before copying in step 1, 3, 4 and 6.
+  //
+  // Note the flow is carefully designed to handle the dynamic sized output
+  // case. The loop invariant is: The newest value is in the inputs of condition
+  // subgraph. This is always true before step 2.
+  //
+  // This is the best we can do without sharing tensor buffer across subgraph
+  // boundry. Currently we copy the input / output between the subgraphs. This
+  // isn't optimized yet and a lot of redundent copies are made.
   // TODO(b/120234921): Optimize and avoid copying tensors between subgraphs.
   TF_LITE_ENSURE_OK(
       context,
-      CopyTensors(context, this_subgraph, TfLiteIntArrayView(node->inputs),
-                  cond_subgraph, cond_subgraph->inputs()));
-  TF_LITE_ENSURE_OK(
-      context,
-      CopyTensors(context, this_subgraph, TfLiteIntArrayView(node->inputs),
-                  body_subgraph, body_subgraph->inputs()));
+      CopyTensorsData(context, this_subgraph, TfLiteIntArrayView(node->inputs),
+                      cond_subgraph, cond_subgraph->inputs()));
 
   while (true) {
     TF_LITE_ENSURE_OK(context, cond_subgraph->Invoke());
@@ -162,21 +227,47 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
     if (!cond_output->data.b[0]) {
       break;
     }
-    TF_LITE_ENSURE_OK(context, body_subgraph->Invoke());
+    if (op_data->body_has_dynamic_output_tensors) {
+      TF_LITE_ENSURE_OK(
+          context,
+          CopyTensorsShape(context, cond_subgraph, cond_subgraph->inputs(),
+                           body_subgraph, body_subgraph->inputs()));
+      TF_LITE_ENSURE_OK(context, body_subgraph->AllocateTensors());
+    }
 
     TF_LITE_ENSURE_OK(
-        context, CopyTensors(context, body_subgraph, body_subgraph->outputs(),
-                             body_subgraph, body_subgraph->inputs()));
+        context,
+        CopyTensorsData(context, cond_subgraph, cond_subgraph->inputs(),
+                        body_subgraph, body_subgraph->inputs()));
+
+    TF_LITE_ENSURE_OK(context, body_subgraph->Invoke());
+
+    if (op_data->body_has_dynamic_output_tensors) {
+      TF_LITE_ENSURE_OK(
+          context,
+          CopyTensorsShape(context, body_subgraph, body_subgraph->outputs(),
+                           cond_subgraph, cond_subgraph->inputs()));
+      TF_LITE_ENSURE_OK(context, cond_subgraph->AllocateTensors());
+    }
+
     TF_LITE_ENSURE_OK(
-        context, CopyTensors(context, body_subgraph, body_subgraph->outputs(),
-                             cond_subgraph, cond_subgraph->inputs()));
+        context,
+        CopyTensorsData(context, body_subgraph, body_subgraph->outputs(),
+                        cond_subgraph, cond_subgraph->inputs()));
   }
 
   // Note that copying from body's output will fail if body is never invoked.
   // TODO(b/120234921): Optimize and avoid copying tensors between subgraphs.
+  if (op_data->body_has_dynamic_output_tensors) {
+    TF_LITE_ENSURE_OK(
+        context,
+        CopyTensorsShape(context, cond_subgraph, cond_subgraph->inputs(),
+                         this_subgraph, TfLiteIntArrayView(node->outputs)));
+  }
   TF_LITE_ENSURE_OK(
-      context, CopyTensors(context, body_subgraph, body_subgraph->inputs(),
-                           this_subgraph, TfLiteIntArrayView(node->outputs)));
+      context,
+      CopyTensorsData(context, cond_subgraph, cond_subgraph->inputs(),
+                      this_subgraph, TfLiteIntArrayView(node->outputs)));
   return kTfLiteOk;
 }
 
diff --git a/tensorflow/lite/kernels/while_test.cc b/tensorflow/lite/kernels/while_test.cc
index 970ec7033b..9946b4a328 100644
--- a/tensorflow/lite/kernels/while_test.cc
+++ b/tensorflow/lite/kernels/while_test.cc
@@ -72,7 +72,16 @@ TEST_F(WhileTest, TestPadLoop) {
   interpreter_->ResizeInputTensor(interpreter_->inputs()[1], {2});
   // This is not supported yet. The test ensures thatit doesn't crash and raises
   // an error properly.
-  ASSERT_NE(interpreter_->AllocateTensors(), kTfLiteOk);
+  ASSERT_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
+
+  FillIntTensor(interpreter_->tensor(interpreter_->inputs()[0]), {1});
+  FillIntTensor(interpreter_->tensor(interpreter_->inputs()[1]), {5, 7});
+
+  ASSERT_EQ(interpreter_->Invoke(), kTfLiteOk);
+  TfLiteTensor* output1 = interpreter_->tensor(interpreter_->outputs()[0]);
+  CheckIntTensor(output1, {1}, {4});
+  TfLiteTensor* output2 = interpreter_->tensor(interpreter_->outputs()[1]);
+  CheckIntTensor(output2, {11}, {0, 0, 0, 5, 7, 0, 0, 0, 0, 0, 0});
 }
 
 }  // namespace
-- 
GitLab


From a1c1eacca5aaf3a0fea2e8c230972a52a4d7f5f5 Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <kramerb@google.com>
Date: Mon, 11 Feb 2019 10:20:43 -0800
Subject: [PATCH 2256/2345] [XLA:CPU] Always initialize result of the sort
 predicate

Technically this gets overwritten by less_than. However, less_than is JIT'ed code, which msan cannot track, leading to false positive warnings about uninitialized memory.

PiperOrigin-RevId: 233431941
---
 tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.cc b/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.cc
index 2cd2f323c7..cb46674138 100644
--- a/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.cc
+++ b/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.cc
@@ -80,7 +80,7 @@ TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSort(
             comparison_values[i * 2] = values[i] + memory_index_lhs;
             comparison_values[i * 2 + 1] = values[i] + memory_index_rhs;
           }
-          char result;
+          char result = 0;  // Overwritten by less_than.
           less_than(&result, run_options, comparison_values.get(), nullptr,
                     prof_counters);
           return result != 0u;
-- 
GitLab


From 75b00c1d64e0ca6a33265281431567b10feda5cc Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Mon, 11 Feb 2019 10:25:14 -0800
Subject: [PATCH 2257/2345] Tweaking memory leak threshold.

PiperOrigin-RevId: 233432685
---
 .../python/data/kernel_tests/multi_device_iterator_test.py     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
index e6dfc338eb..a74f08ffde 100644
--- a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
+++ b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
@@ -115,8 +115,9 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase,
       del multi_device_iterator
       del dataset
 
+    # TODO(b/123316347): Reduce threshold once bug is fixed.
     self.assertNotIncreasingMemory(
-        f, num_iters=100, increase_threshold_absolute_mb=175)
+        f, num_iters=100, increase_threshold_absolute_mb=500)
 
   @test_util.run_v1_only("b/121264236")
   def testOneOnSameDevice(self):
-- 
GitLab


From a55ab4895356326a8e021d9c7545b627a73e2d75 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 11 Feb 2019 10:26:52 -0800
Subject: [PATCH 2258/2345] [TF:XLA] Add and extend matrix utils.

PiperOrigin-RevId: 233433003
---
 tensorflow/compiler/xla/client/lib/BUILD      |  1 +
 tensorflow/compiler/xla/client/lib/matrix.cc  | 29 +++++++++++++++----
 tensorflow/compiler/xla/client/lib/matrix.h   | 13 ++++++---
 .../compiler/xla/client/lib/matrix_test.cc    | 23 +++++++++++----
 4 files changed, 50 insertions(+), 16 deletions(-)

diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD
index 7e7b5732ae..0abf546c14 100644
--- a/tensorflow/compiler/xla/client/lib/BUILD
+++ b/tensorflow/compiler/xla/client/lib/BUILD
@@ -230,6 +230,7 @@ cc_library(
     deps = [
         ":arithmetic",
         ":constants",
+        ":slicing",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:status",
         "//tensorflow/compiler/xla:status_macros",
diff --git a/tensorflow/compiler/xla/client/lib/matrix.cc b/tensorflow/compiler/xla/client/lib/matrix.cc
index a5aea96090..a055a8e625 100644
--- a/tensorflow/compiler/xla/client/lib/matrix.cc
+++ b/tensorflow/compiler/xla/client/lib/matrix.cc
@@ -27,6 +27,7 @@ limitations under the License.
 #include "absl/types/span.h"
 #include "tensorflow/compiler/xla/client/lib/arithmetic.h"
 #include "tensorflow/compiler/xla/client/lib/constants.h"
+#include "tensorflow/compiler/xla/client/lib/slicing.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status.h"
@@ -45,7 +46,7 @@ XlaOp IdentityMatrix(XlaBuilder* builder, PrimitiveType type, int64 m,
   return ConvertElementType(indicator, type);
 }
 
-XlaOp GetMatrixDiagonal(XlaOp x) {
+XlaOp GetMatrixDiagonal(XlaOp x, int k) {
   XlaBuilder* builder = x.builder();
   return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
     TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x));
@@ -53,10 +54,13 @@ XlaOp GetMatrixDiagonal(XlaOp x) {
     TF_RET_CHECK(n_dims >= 2);
     const int64 m = shape.dimensions(n_dims - 2);
     const int64 n = shape.dimensions(n_dims - 1);
+
+    auto offset = ConstantR0WithType(builder, S32, k);
+
     absl::Span<const int64> major_dims =
         AsInt64Slice(shape.dimensions()).subspan(/*pos=*/0, /*len=*/n_dims - 2);
-    auto a = Iota(builder, U32, n);
-    auto b = Iota(builder, U32, m);
+    auto a = Iota(builder, S32, n);
+    auto b = Iota(builder, S32, m) + offset;
     auto indicator = Eq(b, Broadcast(a, {m}), /*broadcast_dimensions=*/{0});
     auto mask = Broadcast(indicator, major_dims);
 
@@ -66,9 +70,21 @@ XlaOp GetMatrixDiagonal(XlaOp x) {
         primitive_util::IsIntegralType(shape.element_type())
             ? CreateScalarOrComputation(shape.element_type(), builder)
             : CreateScalarAddComputation(shape.element_type(), builder);
-
-    return Reduce(Select(mask, x, Zeros(builder, shape)), ScalarLike(x, 0),
-                  reducer, {m >= n ? n_dims - 2 : n_dims - 1});
+    // k == 0, we can save one slice op.
+    if (k == 0) {
+      return Reduce(Select(mask, x, Zeros(builder, shape)), ScalarLike(x, 0),
+                    reducer, {m >= n ? n_dims - 2 : n_dims - 1});
+    } else if (k > 0) {
+      auto result = Reduce(Select(mask, x, Zeros(builder, shape)),
+                           ScalarLike(x, 0), reducer, {n_dims - 2});
+      return SliceInMinorDims(result, {std::min<int64>(k, n)},
+                              {std::min(m + k, n)});
+    } else {
+      auto result = Reduce(Select(mask, x, Zeros(builder, shape)),
+                           ScalarLike(x, 0), reducer, {n_dims - 1});
+      return SliceInMinorDims(result, {std::min<int64>(-k, m)},
+                              {std::min(m, n - k)});
+    }
   });
 }
 
@@ -336,4 +352,5 @@ XlaOp TransposeInMinorDims(XlaOp x) {
 XlaOp MaybeTransposeInMinorDims(XlaOp x, bool transpose) {
   return transpose ? TransposeInMinorDims(x) : x;
 }
+
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/client/lib/matrix.h b/tensorflow/compiler/xla/client/lib/matrix.h
index 491f1eab4c..60c41ec45a 100644
--- a/tensorflow/compiler/xla/client/lib/matrix.h
+++ b/tensorflow/compiler/xla/client/lib/matrix.h
@@ -30,10 +30,15 @@ namespace xla {
 // else.
 XlaOp IdentityMatrix(XlaBuilder* builder, PrimitiveType type, int64 m, int64 n);
 
-// Get the diagonals of the last two dimensions. If 'x' has shape
-// [..., M, N], then the output has shape [..., min(M, N)], containing the
-// diagonal elements (i.e., with indices [..., i, i]).
-XlaOp GetMatrixDiagonal(XlaOp x);
+// Get the diagonals of the last two dimensions. Use k>0 for diagonals above the
+// main diagonal, and k<0 for diagonals below the main diagonal.
+//
+// If 'x' has shape [..., M, N]
+//  If k >= 0: then the output has shape [..., min(M, N - k)], containing the
+//            diagonal elements (i.e., with indices [..., i, i + k]).
+//  If k < 0: then the output has shape [..., min(M + k, N)], containing the
+//            diagonal elements (i.e., with indices [..., i - k, i]).
+XlaOp GetMatrixDiagonal(XlaOp x, int k = 0);
 
 // Returns a lower-triangular mask, i.e., true below the `diagonal`-th diagonal
 // and false above that diagonal.
diff --git a/tensorflow/compiler/xla/client/lib/matrix_test.cc b/tensorflow/compiler/xla/client/lib/matrix_test.cc
index 61f91a56fb..a93fc2ccb9 100644
--- a/tensorflow/compiler/xla/client/lib/matrix_test.cc
+++ b/tensorflow/compiler/xla/client/lib/matrix_test.cc
@@ -53,13 +53,24 @@ void MatrixTest::TestMatrixDiagonal() {
   XlaBuilder builder("GetMatrixDiagonal");
   Array3D<T> input(2, 3, 4);
   input.FillIota(0);
+  std::map<int, Array2D<T>> k_and_expected = {
+      {0, {{0, 5, 10}, {12, 17, 22}}},
+      {1, {{1, 6, 11}, {13, 18, 23}}},
+      {2, {{2, 7}, {14, 19}}},
+      {3, {{3}, {15}}},
+      {4, {{}, {}}},
+      {-1, {{4, 9}, {16, 21}}},
+      {-2, {{8}, {20}}},
+      {-3, {{}, {}}},
+      {-4, {{}, {}}},
+  };
+  for (const auto& kv : k_and_expected) {
+    XlaOp a;
+    auto a_data = CreateR3Parameter<T>(input, 0, "a", &builder, &a);
+    GetMatrixDiagonal(a, kv.first);
 
-  XlaOp a;
-  auto a_data = CreateR3Parameter<T>(input, 0, "a", &builder, &a);
-  GetMatrixDiagonal(a);
-  Array2D<T> expected({{0, 5, 10}, {12, 17, 22}});
-
-  ComputeAndCompareR2<T>(&builder, expected, {a_data.get()});
+    ComputeAndCompareR2<T>(&builder, kv.second, {a_data.get()});
+  }
 }
 
 XLA_TEST_F(MatrixTest, GetMatrixDiagonal_S32) { TestMatrixDiagonal<int32>(); }
-- 
GitLab


From a19d8a38b1deaa58b959d3f5e32d44148c92717b Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Mon, 11 Feb 2019 10:27:03 -0800
Subject: [PATCH 2259/2345] Refactor RNN tests.

Tests of core rnn functionality are moved to core kernel_tests.
Tests of contrib rnn functionality stay in contrib/rnn kernel_tests.

PiperOrigin-RevId: 233433042
---
 tensorflow/contrib/rnn/BUILD                  |   46 -
 .../python/kernel_tests/core_rnn_cell_test.py | 1241 -----------------
 .../rnn/python/kernel_tests/rnn_cell_test.py  |  289 ++++
 tensorflow/python/kernel_tests/BUILD          |   27 +
 .../kernel_tests/rnn_cell_test.py}            | 1022 +++++++++++++-
 5 files changed, 1283 insertions(+), 1342 deletions(-)
 delete mode 100644 tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py
 rename tensorflow/{contrib/rnn/python/kernel_tests/core_rnn_test.py => python/kernel_tests/rnn_cell_test.py} (68%)

diff --git a/tensorflow/contrib/rnn/BUILD b/tensorflow/contrib/rnn/BUILD
index d65d80df80..24fa740d24 100644
--- a/tensorflow/contrib/rnn/BUILD
+++ b/tensorflow/contrib/rnn/BUILD
@@ -102,26 +102,6 @@ cuda_py_tests(
     xla_enabled = True,
 )
 
-cuda_py_tests(
-    name = "core_rnn_cell_test",
-    size = "medium",
-    srcs = ["python/kernel_tests/core_rnn_cell_test.py"],
-    additional_deps = [
-        ":rnn_py",
-        "//third_party/py/numpy",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:init_ops",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:rnn",
-        "//tensorflow/python:rnn_cell",
-        "//tensorflow/python:variable_scope",
-        "//tensorflow/python:variables",
-        "@absl_py//absl/testing:parameterized",
-    ],
-)
-
 cuda_py_tests(
     name = "rnn_test",
     size = "medium",
@@ -144,32 +124,6 @@ cuda_py_tests(
     ],
 )
 
-cuda_py_tests(
-    name = "core_rnn_test",
-    size = "medium",
-    srcs = ["python/kernel_tests/core_rnn_test.py"],
-    additional_deps = [
-        ":rnn_py",
-        "//third_party/py/numpy",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:control_flow_ops",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:gradients",
-        "//tensorflow/python:init_ops",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:platform",
-        "//tensorflow/python:rnn",
-        "//tensorflow/python:tensor_array_ops",
-        "//tensorflow/python:util",
-        "//tensorflow/python:variable_scope",
-        "//tensorflow/python:variables",
-        "//tensorflow/python/eager:context",
-    ],
-    shard_count = 10,
-)
-
 tf_py_test(
     name = "fused_rnn_cell_test",
     size = "medium",
diff --git a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py
deleted file mode 100644
index a70e806211..0000000000
--- a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py
+++ /dev/null
@@ -1,1241 +0,0 @@
-# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for RNN cells."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-
-from absl.testing import parameterized
-import numpy as np
-
-from tensorflow.contrib import rnn as contrib_rnn
-from tensorflow.contrib.rnn.python.ops import core_rnn_cell
-from tensorflow.contrib.rnn.python.ops import rnn_cell as contrib_rnn_cell
-from tensorflow.core.protobuf import config_pb2
-from tensorflow.python.framework import constant_op
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops
-from tensorflow.python.framework import random_seed
-from tensorflow.python.framework import test_util
-from tensorflow.python.keras import layers as keras_layers
-from tensorflow.python.layers import base as base_layer
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import init_ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import rnn
-from tensorflow.python.ops import rnn_cell_impl
-from tensorflow.python.ops import variable_scope
-from tensorflow.python.ops import variables as variables_lib
-from tensorflow.python.platform import test
-from tensorflow.python.training.checkpointable import util as checkpointable_utils
-
-# pylint: enable=protected-access
-Linear = core_rnn_cell._Linear  # pylint: disable=invalid-name
-
-
-class RNNCellTest(test.TestCase, parameterized.TestCase):
-
-  def testLinear(self):
-    with self.cached_session() as sess:
-      with variable_scope.variable_scope(
-          "root", initializer=init_ops.constant_initializer(1.0)):
-        x = array_ops.zeros([1, 2])
-        l = Linear([x], 2, False)([x])
-        sess.run([variables_lib.global_variables_initializer()])
-        res = sess.run([l], {x.name: np.array([[1., 2.]])})
-        self.assertAllClose(res[0], [[3.0, 3.0]])
-
-        # Checks prevent you from accidentally creating a shared function.
-        with self.assertRaises(ValueError):
-          l1 = Linear([x], 2, False)([x])
-
-        # But you can create a new one in a new scope and share the variables.
-        with variable_scope.variable_scope("l1") as new_scope:
-          l1 = Linear([x], 2, False)([x])
-        with variable_scope.variable_scope(new_scope, reuse=True):
-          Linear([l1], 2, False)([l1])
-        self.assertEqual(len(variables_lib.trainable_variables()), 2)
-
-  def testBasicRNNCell(self):
-    with self.cached_session() as sess:
-      with variable_scope.variable_scope(
-          "root", initializer=init_ops.constant_initializer(0.5)):
-        x = array_ops.zeros([1, 2])
-        m = array_ops.zeros([1, 2])
-        cell = rnn_cell_impl.BasicRNNCell(2)
-        g, _ = cell(x, m)
-        self.assertEqual([
-            "root/basic_rnn_cell/%s:0" % rnn_cell_impl._WEIGHTS_VARIABLE_NAME,
-            "root/basic_rnn_cell/%s:0" % rnn_cell_impl._BIAS_VARIABLE_NAME
-        ], [v.name for v in cell.trainable_variables])
-        self.assertFalse(cell.non_trainable_variables)
-        sess.run([variables_lib.global_variables_initializer()])
-        res = sess.run([g], {
-            x.name: np.array([[1., 1.]]),
-            m.name: np.array([[0.1, 0.1]])
-        })
-        self.assertEqual(res[0].shape, (1, 2))
-
-  def testBasicRNNCellNotTrainable(self):
-    with self.cached_session() as sess:
-
-      def not_trainable_getter(getter, *args, **kwargs):
-        kwargs["trainable"] = False
-        return getter(*args, **kwargs)
-
-      with variable_scope.variable_scope(
-          "root",
-          initializer=init_ops.constant_initializer(0.5),
-          custom_getter=not_trainable_getter):
-        x = array_ops.zeros([1, 2])
-        m = array_ops.zeros([1, 2])
-        cell = rnn_cell_impl.BasicRNNCell(2)
-        g, _ = cell(x, m)
-        self.assertFalse(cell.trainable_variables)
-        self.assertEqual([
-            "root/basic_rnn_cell/%s:0" % rnn_cell_impl._WEIGHTS_VARIABLE_NAME,
-            "root/basic_rnn_cell/%s:0" % rnn_cell_impl._BIAS_VARIABLE_NAME
-        ], [v.name for v in cell.non_trainable_variables])
-        sess.run([variables_lib.global_variables_initializer()])
-        res = sess.run([g], {
-            x.name: np.array([[1., 1.]]),
-            m.name: np.array([[0.1, 0.1]])
-        })
-        self.assertEqual(res[0].shape, (1, 2))
-
-  def testIndRNNCell(self):
-    with self.cached_session() as sess:
-      with variable_scope.variable_scope(
-          "root", initializer=init_ops.constant_initializer(0.5)):
-        x = array_ops.zeros([1, 2])
-        m = array_ops.zeros([1, 2])
-        cell = contrib_rnn_cell.IndRNNCell(2)
-        g, _ = cell(x, m)
-        self.assertEqual([
-            "root/ind_rnn_cell/%s_w:0" % rnn_cell_impl._WEIGHTS_VARIABLE_NAME,
-            "root/ind_rnn_cell/%s_u:0" % rnn_cell_impl._WEIGHTS_VARIABLE_NAME,
-            "root/ind_rnn_cell/%s:0" % rnn_cell_impl._BIAS_VARIABLE_NAME
-        ], [v.name for v in cell.trainable_variables])
-        self.assertFalse(cell.non_trainable_variables)
-        sess.run([variables_lib.global_variables_initializer()])
-        res = sess.run([g], {
-            x.name: np.array([[1., 1.]]),
-            m.name: np.array([[0.1, 0.1]])
-        })
-        self.assertEqual(res[0].shape, (1, 2))
-
-  def testGRUCell(self):
-    with self.cached_session() as sess:
-      with variable_scope.variable_scope(
-          "root", initializer=init_ops.constant_initializer(0.5)):
-        x = array_ops.zeros([1, 2])
-        m = array_ops.zeros([1, 2])
-        g, _ = rnn_cell_impl.GRUCell(2)(x, m)
-        sess.run([variables_lib.global_variables_initializer()])
-        res = sess.run([g], {
-            x.name: np.array([[1., 1.]]),
-            m.name: np.array([[0.1, 0.1]])
-        })
-        # Smoke test
-        self.assertAllClose(res[0], [[0.175991, 0.175991]])
-      with variable_scope.variable_scope(
-          "other", initializer=init_ops.constant_initializer(0.5)):
-        # Test GRUCell with input_size != num_units.
-        x = array_ops.zeros([1, 3])
-        m = array_ops.zeros([1, 2])
-        g, _ = rnn_cell_impl.GRUCell(2)(x, m)
-        sess.run([variables_lib.global_variables_initializer()])
-        res = sess.run([g], {
-            x.name: np.array([[1., 1., 1.]]),
-            m.name: np.array([[0.1, 0.1]])
-        })
-        # Smoke test
-        self.assertAllClose(res[0], [[0.156736, 0.156736]])
-
-  def testIndyGRUCell(self):
-    with self.cached_session() as sess:
-      with variable_scope.variable_scope(
-          "root", initializer=init_ops.constant_initializer(0.5)):
-        x = array_ops.zeros([1, 2])
-        m = array_ops.zeros([1, 2])
-        g, _ = contrib_rnn_cell.IndyGRUCell(2)(x, m)
-        sess.run([variables_lib.global_variables_initializer()])
-        res = sess.run([g], {
-            x.name: np.array([[1., 1.]]),
-            m.name: np.array([[0.1, 0.1]])
-        })
-        # Smoke test
-        self.assertAllClose(res[0], [[0.185265, 0.17704]])
-      with variable_scope.variable_scope(
-          "other", initializer=init_ops.constant_initializer(0.5)):
-        # Test IndyGRUCell with input_size != num_units.
-        x = array_ops.zeros([1, 3])
-        m = array_ops.zeros([1, 2])
-        g, _ = contrib_rnn_cell.IndyGRUCell(2)(x, m)
-        sess.run([variables_lib.global_variables_initializer()])
-        res = sess.run([g], {
-            x.name: np.array([[1., 1., 1.]]),
-            m.name: np.array([[0.1, 0.1]])
-        })
-        # Smoke test
-        self.assertAllClose(res[0], [[0.155127, 0.157328]])
-
-  def testSRUCell(self):
-    with self.cached_session() as sess:
-      with variable_scope.variable_scope(
-          "root", initializer=init_ops.constant_initializer(0.5)):
-        x = array_ops.zeros([1, 2])
-        m = array_ops.zeros([1, 2])
-        g, _ = contrib_rnn_cell.SRUCell(2)(x, m)
-        sess.run([variables_lib.global_variables_initializer()])
-        res = sess.run([g], {
-            x.name: np.array([[1., 1.]]),
-            m.name: np.array([[0.1, 0.1]])
-        })
-        # Smoke test
-        self.assertAllClose(res[0], [[0.509682, 0.509682]])
-
-  def testSRUCellKerasRNN(self):
-    """Tests that SRUCell works with keras RNN layer."""
-    cell = contrib_rnn_cell.SRUCell(10)
-    seq_input = ops.convert_to_tensor(
-        np.random.rand(2, 3, 5), name="seq_input", dtype=dtypes.float32)
-    rnn_layer = keras_layers.RNN(cell=cell)
-    rnn_outputs_keras = rnn_layer(seq_input)
-    with self.cached_session() as sess:
-      sess.run([variables_lib.global_variables_initializer()])
-      self.assertEqual(sess.run(rnn_outputs_keras).shape, (2, 10))
-
-  def testSRUCellBiasType(self):
-    """Tests that the bias' dtype is properly set."""
-    cell = contrib_rnn_cell.SRUCell(10)
-    cell.build((2, 3, 5))
-    self.assertEqual(cell._bias.dtype, dtypes.float32_ref)
-
-    cell = contrib_rnn_cell.SRUCell(10, dtype=dtypes.int32)
-    cell.build((2, 3, 5))
-    self.assertEqual(cell._bias.dtype, dtypes.int32_ref)
-
-    cell_input = ops.convert_to_tensor(
-        np.random.rand(2, 5), name="cell_input", dtype=dtypes.float16)
-    cell_state = ops.convert_to_tensor(
-        np.random.rand(2, 10), name="cell_state", dtype=dtypes.float16)
-    cell = contrib_rnn_cell.SRUCell(10)
-    cell(cell_input, [cell_state])
-    self.assertEqual(cell._bias.dtype, dtypes.float16_ref)
-
-  def testSRUCellWithDiffSize(self):
-    with self.cached_session() as sess:
-      with variable_scope.variable_scope(
-          "root", initializer=init_ops.constant_initializer(0.5)):
-        x = array_ops.zeros([1, 3])
-        m = array_ops.zeros([1, 2])
-        g, _ = contrib_rnn_cell.SRUCell(2)(x, m)
-        sess.run([variables_lib.global_variables_initializer()])
-        res = sess.run([g], {
-            x.name: np.array([[1., 1., 1.]]),
-            m.name: np.array([[0.1, 0.1]])
-        })
-        # Smoke test
-        self.assertAllClose(res[0], [[0.55255556, 0.55255556]])
-
-  def testBasicLSTMCell(self):
-    for dtype in [dtypes.float16, dtypes.float32]:
-      np_dtype = dtype.as_numpy_dtype
-      with self.session(graph=ops.Graph()) as sess:
-        with variable_scope.variable_scope(
-            "root", initializer=init_ops.constant_initializer(0.5)):
-          x = array_ops.zeros([1, 2], dtype=dtype)
-          m = array_ops.zeros([1, 8], dtype=dtype)
-          cell = rnn_cell_impl.MultiRNNCell(
-              [
-                  rnn_cell_impl.BasicLSTMCell(2, state_is_tuple=False)
-                  for _ in range(2)
-              ],
-              state_is_tuple=False)
-          self.assertEqual(cell.dtype, None)
-          self.assertEqual("cell-0", cell._checkpoint_dependencies[0].name)
-          self.assertEqual("cell-1", cell._checkpoint_dependencies[1].name)
-          cell.get_config()  # Should not throw an error
-          g, out_m = cell(x, m)
-          # Layer infers the input type.
-          self.assertEqual(cell.dtype, dtype.name)
-          expected_variable_names = [
-              "root/multi_rnn_cell/cell_0/basic_lstm_cell/%s:0" %
-              rnn_cell_impl._WEIGHTS_VARIABLE_NAME,
-              "root/multi_rnn_cell/cell_0/basic_lstm_cell/%s:0" %
-              rnn_cell_impl._BIAS_VARIABLE_NAME,
-              "root/multi_rnn_cell/cell_1/basic_lstm_cell/%s:0" %
-              rnn_cell_impl._WEIGHTS_VARIABLE_NAME,
-              "root/multi_rnn_cell/cell_1/basic_lstm_cell/%s:0" %
-              rnn_cell_impl._BIAS_VARIABLE_NAME
-          ]
-          self.assertEqual(expected_variable_names,
-                           [v.name for v in cell.trainable_variables])
-          self.assertFalse(cell.non_trainable_variables)
-          sess.run([variables_lib.global_variables_initializer()])
-          res = sess.run([g, out_m], {
-              x.name: np.array([[1., 1.]]),
-              m.name: 0.1 * np.ones([1, 8])
-          })
-          self.assertEqual(len(res), 2)
-          variables = variables_lib.global_variables()
-          self.assertEqual(expected_variable_names, [v.name for v in variables])
-          # The numbers in results were not calculated, this is just a
-          # smoke test.
-          self.assertAllClose(res[0], np.array(
-              [[0.240, 0.240]], dtype=np_dtype), 1e-2)
-          expected_mem = np.array(
-              [[0.689, 0.689, 0.448, 0.448, 0.398, 0.398, 0.240, 0.240]],
-              dtype=np_dtype)
-          self.assertAllClose(res[1], expected_mem, 1e-2)
-        with variable_scope.variable_scope(
-            "other", initializer=init_ops.constant_initializer(0.5)):
-          # Test BasicLSTMCell with input_size != num_units.
-          x = array_ops.zeros([1, 3], dtype=dtype)
-          m = array_ops.zeros([1, 4], dtype=dtype)
-          g, out_m = rnn_cell_impl.BasicLSTMCell(2, state_is_tuple=False)(x, m)
-          sess.run([variables_lib.global_variables_initializer()])
-          res = sess.run(
-              [g, out_m], {
-                  x.name: np.array([[1., 1., 1.]], dtype=np_dtype),
-                  m.name: 0.1 * np.ones([1, 4], dtype=np_dtype)
-              })
-          self.assertEqual(len(res), 2)
-
-  def testBasicLSTMCellDimension0Error(self):
-    """Tests that dimension 0 in both(x and m) shape must be equal."""
-    with self.cached_session() as sess:
-      with variable_scope.variable_scope(
-          "root", initializer=init_ops.constant_initializer(0.5)):
-        num_units = 2
-        state_size = num_units * 2
-        batch_size = 3
-        input_size = 4
-        x = array_ops.zeros([batch_size, input_size])
-        m = array_ops.zeros([batch_size - 1, state_size])
-        with self.assertRaises(ValueError):
-          g, out_m = rnn_cell_impl.BasicLSTMCell(
-              num_units, state_is_tuple=False)(x, m)
-          sess.run([variables_lib.global_variables_initializer()])
-          sess.run(
-              [g, out_m], {
-                  x.name: 1 * np.ones([batch_size, input_size]),
-                  m.name: 0.1 * np.ones([batch_size - 1, state_size])
-              })
-
-  def testBasicLSTMCellStateSizeError(self):
-    """Tests that state_size must be num_units * 2."""
-    with self.cached_session() as sess:
-      with variable_scope.variable_scope(
-          "root", initializer=init_ops.constant_initializer(0.5)):
-        num_units = 2
-        state_size = num_units * 3  # state_size must be num_units * 2
-        batch_size = 3
-        input_size = 4
-        x = array_ops.zeros([batch_size, input_size])
-        m = array_ops.zeros([batch_size, state_size])
-        with self.assertRaises(ValueError):
-          g, out_m = rnn_cell_impl.BasicLSTMCell(
-              num_units, state_is_tuple=False)(x, m)
-          sess.run([variables_lib.global_variables_initializer()])
-          sess.run(
-              [g, out_m], {
-                  x.name: 1 * np.ones([batch_size, input_size]),
-                  m.name: 0.1 * np.ones([batch_size, state_size])
-              })
-
-  def testBasicLSTMCellStateTupleType(self):
-    with self.cached_session():
-      with variable_scope.variable_scope(
-          "root", initializer=init_ops.constant_initializer(0.5)):
-        x = array_ops.zeros([1, 2])
-        m0 = (array_ops.zeros([1, 2]),) * 2
-        m1 = (array_ops.zeros([1, 2]),) * 2
-        cell = rnn_cell_impl.MultiRNNCell(
-            [rnn_cell_impl.BasicLSTMCell(2) for _ in range(2)],
-            state_is_tuple=True)
-        self.assertTrue(isinstance(cell.state_size, tuple))
-        self.assertTrue(
-            isinstance(cell.state_size[0], rnn_cell_impl.LSTMStateTuple))
-        self.assertTrue(
-            isinstance(cell.state_size[1], rnn_cell_impl.LSTMStateTuple))
-
-        # Pass in regular tuples
-        _, (out_m0, out_m1) = cell(x, (m0, m1))
-        self.assertTrue(isinstance(out_m0, rnn_cell_impl.LSTMStateTuple))
-        self.assertTrue(isinstance(out_m1, rnn_cell_impl.LSTMStateTuple))
-
-        # Pass in LSTMStateTuples
-        variable_scope.get_variable_scope().reuse_variables()
-        zero_state = cell.zero_state(1, dtypes.float32)
-        self.assertTrue(isinstance(zero_state, tuple))
-        self.assertTrue(isinstance(zero_state[0], rnn_cell_impl.LSTMStateTuple))
-        self.assertTrue(isinstance(zero_state[1], rnn_cell_impl.LSTMStateTuple))
-        _, (out_m0, out_m1) = cell(x, zero_state)
-        self.assertTrue(isinstance(out_m0, rnn_cell_impl.LSTMStateTuple))
-        self.assertTrue(isinstance(out_m1, rnn_cell_impl.LSTMStateTuple))
-
-  def testBasicLSTMCellWithStateTuple(self):
-    with self.cached_session() as sess:
-      with variable_scope.variable_scope(
-          "root", initializer=init_ops.constant_initializer(0.5)):
-        x = array_ops.zeros([1, 2])
-        m0 = array_ops.zeros([1, 4])
-        m1 = array_ops.zeros([1, 4])
-        cell = rnn_cell_impl.MultiRNNCell(
-            [
-                rnn_cell_impl.BasicLSTMCell(2, state_is_tuple=False)
-                for _ in range(2)
-            ],
-            state_is_tuple=True)
-        g, (out_m0, out_m1) = cell(x, (m0, m1))
-        sess.run([variables_lib.global_variables_initializer()])
-        res = sess.run(
-            [g, out_m0, out_m1], {
-                x.name: np.array([[1., 1.]]),
-                m0.name: 0.1 * np.ones([1, 4]),
-                m1.name: 0.1 * np.ones([1, 4])
-            })
-        self.assertEqual(len(res), 3)
-        # The numbers in results were not calculated, this is just a smoke test.
-        # Note, however, these values should match the original
-        # version having state_is_tuple=False.
-        self.assertAllClose(res[0], [[0.24024698, 0.24024698]])
-        expected_mem0 = np.array(
-            [[0.68967271, 0.68967271, 0.44848421, 0.44848421]])
-        expected_mem1 = np.array(
-            [[0.39897051, 0.39897051, 0.24024698, 0.24024698]])
-        self.assertAllClose(res[1], expected_mem0)
-        self.assertAllClose(res[2], expected_mem1)
-
-  def testIndyLSTMCell(self):
-    for dtype in [dtypes.float16, dtypes.float32]:
-      np_dtype = dtype.as_numpy_dtype
-      with self.session(graph=ops.Graph()) as sess:
-        with variable_scope.variable_scope(
-            "root", initializer=init_ops.constant_initializer(0.5)):
-          x = array_ops.zeros([1, 2], dtype=dtype)
-          state_0 = (array_ops.zeros([1, 2], dtype=dtype),) * 2
-          state_1 = (array_ops.zeros([1, 2], dtype=dtype),) * 2
-          cell = rnn_cell_impl.MultiRNNCell(
-              [contrib_rnn_cell.IndyLSTMCell(2) for _ in range(2)])
-          self.assertEqual(cell.dtype, None)
-          self.assertEqual("cell-0", cell._checkpoint_dependencies[0].name)
-          self.assertEqual("cell-1", cell._checkpoint_dependencies[1].name)
-          cell.get_config()  # Should not throw an error
-          g, (out_state_0, out_state_1) = cell(x, (state_0, state_1))
-          # Layer infers the input type.
-          self.assertEqual(cell.dtype, dtype.name)
-          expected_variable_names = [
-              "root/multi_rnn_cell/cell_0/indy_lstm_cell/%s_w:0" %
-              rnn_cell_impl._WEIGHTS_VARIABLE_NAME,
-              "root/multi_rnn_cell/cell_0/indy_lstm_cell/%s_u:0" %
-              rnn_cell_impl._WEIGHTS_VARIABLE_NAME,
-              "root/multi_rnn_cell/cell_0/indy_lstm_cell/%s:0" %
-              rnn_cell_impl._BIAS_VARIABLE_NAME,
-              "root/multi_rnn_cell/cell_1/indy_lstm_cell/%s_w:0" %
-              rnn_cell_impl._WEIGHTS_VARIABLE_NAME,
-              "root/multi_rnn_cell/cell_1/indy_lstm_cell/%s_u:0" %
-              rnn_cell_impl._WEIGHTS_VARIABLE_NAME,
-              "root/multi_rnn_cell/cell_1/indy_lstm_cell/%s:0" %
-              rnn_cell_impl._BIAS_VARIABLE_NAME
-          ]
-          self.assertEqual(expected_variable_names,
-                           [v.name for v in cell.trainable_variables])
-          self.assertFalse(cell.non_trainable_variables)
-          sess.run([variables_lib.global_variables_initializer()])
-          res = sess.run(
-              [g, out_state_0, out_state_1], {
-                  x.name: np.array([[1., 1.]]),
-                  state_0[0].name: 0.1 * np.ones([1, 2]),
-                  state_0[1].name: 0.1 * np.ones([1, 2]),
-                  state_1[0].name: 0.1 * np.ones([1, 2]),
-                  state_1[1].name: 0.1 * np.ones([1, 2]),
-              })
-          self.assertEqual(len(res), 3)
-          variables = variables_lib.global_variables()
-          self.assertEqual(expected_variable_names, [v.name for v in variables])
-          # Only check the range of outputs as this is just a smoke test.
-          self.assertAllInRange(res[0], -1.0, 1.0)
-          self.assertAllInRange(res[1], -1.0, 1.0)
-          self.assertAllInRange(res[2], -1.0, 1.0)
-        with variable_scope.variable_scope(
-            "other", initializer=init_ops.constant_initializer(0.5)):
-          # Test IndyLSTMCell with input_size != num_units.
-          x = array_ops.zeros([1, 3], dtype=dtype)
-          state = (array_ops.zeros([1, 2], dtype=dtype),) * 2
-          g, out_state = contrib_rnn_cell.IndyLSTMCell(2)(x, state)
-          sess.run([variables_lib.global_variables_initializer()])
-          res = sess.run(
-              [g, out_state], {
-                  x.name: np.array([[1., 1., 1.]], dtype=np_dtype),
-                  state[0].name: 0.1 * np.ones([1, 2], dtype=np_dtype),
-                  state[1].name: 0.1 * np.ones([1, 2], dtype=np_dtype),
-              })
-          self.assertEqual(len(res), 2)
-
-  def testLSTMCell(self):
-    with self.cached_session() as sess:
-      num_units = 8
-      num_proj = 6
-      state_size = num_units + num_proj
-      batch_size = 3
-      input_size = 2
-      with variable_scope.variable_scope(
-          "root", initializer=init_ops.constant_initializer(0.5)):
-        x = array_ops.zeros([batch_size, input_size])
-        m = array_ops.zeros([batch_size, state_size])
-        cell = rnn_cell_impl.LSTMCell(
-            num_units=num_units,
-            num_proj=num_proj,
-            forget_bias=1.0,
-            state_is_tuple=False)
-        output, state = cell(x, m)
-        sess.run([variables_lib.global_variables_initializer()])
-        res = sess.run(
-            [output, state], {
-                x.name: np.array([[1., 1.], [2., 2.], [3., 3.]]),
-                m.name: 0.1 * np.ones((batch_size, state_size))
-            })
-        self.assertEqual(len(res), 2)
-        # The numbers in results were not calculated, this is mostly just a
-        # smoke test.
-        self.assertEqual(res[0].shape, (batch_size, num_proj))
-        self.assertEqual(res[1].shape, (batch_size, state_size))
-        # Different inputs so different outputs and states
-        for i in range(1, batch_size):
-          self.assertTrue(
-              float(np.linalg.norm((res[0][0, :] - res[0][i, :]))) > 1e-6)
-          self.assertTrue(
-              float(np.linalg.norm((res[1][0, :] - res[1][i, :]))) > 1e-6)
-
-  def testLSTMCellVariables(self):
-    with self.cached_session():
-      num_units = 8
-      num_proj = 6
-      state_size = num_units + num_proj
-      batch_size = 3
-      input_size = 2
-      with variable_scope.variable_scope(
-          "root", initializer=init_ops.constant_initializer(0.5)):
-        x = array_ops.zeros([batch_size, input_size])
-        m = array_ops.zeros([batch_size, state_size])
-        cell = rnn_cell_impl.LSTMCell(
-            num_units=num_units,
-            num_proj=num_proj,
-            forget_bias=1.0,
-            state_is_tuple=False)
-        cell(x, m)  # Execute to create variables
-      variables = variables_lib.global_variables()
-      self.assertEquals(variables[0].op.name, "root/lstm_cell/kernel")
-      self.assertEquals(variables[1].op.name, "root/lstm_cell/bias")
-      self.assertEquals(variables[2].op.name,
-                        "root/lstm_cell/projection/kernel")
-
-  def testLSTMCellLayerNorm(self):
-    with self.cached_session() as sess:
-      num_units = 2
-      num_proj = 3
-      batch_size = 1
-      input_size = 4
-      with variable_scope.variable_scope(
-          "root", initializer=init_ops.constant_initializer(0.5)):
-        x = array_ops.zeros([batch_size, input_size])
-        c = array_ops.zeros([batch_size, num_units])
-        h = array_ops.zeros([batch_size, num_proj])
-        state = rnn_cell_impl.LSTMStateTuple(c, h)
-        cell = contrib_rnn_cell.LayerNormLSTMCell(
-            num_units=num_units,
-            num_proj=num_proj,
-            forget_bias=1.0,
-            layer_norm=True,
-            norm_gain=1.0,
-            norm_shift=0.0)
-        g, out_m = cell(x, state)
-        sess.run([variables_lib.global_variables_initializer()])
-        res = sess.run(
-            [g, out_m], {
-                x.name: np.ones((batch_size, input_size)),
-                c.name: 0.1 * np.ones((batch_size, num_units)),
-                h.name: 0.1 * np.ones((batch_size, num_proj))
-            })
-        self.assertEqual(len(res), 2)
-        # The numbers in results were not calculated, this is mostly just a
-        # smoke test.
-        self.assertEqual(res[0].shape, (batch_size, num_proj))
-        self.assertEqual(res[1][0].shape, (batch_size, num_units))
-        self.assertEqual(res[1][1].shape, (batch_size, num_proj))
-        # Different inputs so different outputs and states
-        for i in range(1, batch_size):
-          self.assertTrue(
-              float(np.linalg.norm((res[0][0, :] - res[0][i, :]))) < 1e-6)
-          self.assertTrue(
-              float(np.linalg.norm((res[1][0, :] - res[1][i, :]))) < 1e-6)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testWrapperCheckpointing(self):
-    for wrapper_type in [
-        rnn_cell_impl.DropoutWrapper,
-        rnn_cell_impl.ResidualWrapper,
-        lambda cell: rnn_cell_impl.MultiRNNCell([cell])]:
-      cell = rnn_cell_impl.BasicRNNCell(1)
-      wrapper = wrapper_type(cell)
-      wrapper(array_ops.ones([1, 1]),
-              state=wrapper.zero_state(batch_size=1, dtype=dtypes.float32))
-      self.evaluate([v.initializer for v in cell.variables])
-      checkpoint = checkpointable_utils.Checkpoint(wrapper=wrapper)
-      prefix = os.path.join(self.get_temp_dir(), "ckpt")
-      self.evaluate(cell._bias.assign([40.]))
-      save_path = checkpoint.save(prefix)
-      self.evaluate(cell._bias.assign([0.]))
-      checkpoint.restore(save_path).assert_consumed().run_restore_ops()
-      self.assertAllEqual([40.], self.evaluate(cell._bias))
-
-  def testOutputProjectionWrapper(self):
-    with self.cached_session() as sess:
-      with variable_scope.variable_scope(
-          "root", initializer=init_ops.constant_initializer(0.5)):
-        x = array_ops.zeros([1, 3])
-        m = array_ops.zeros([1, 3])
-        cell = contrib_rnn.OutputProjectionWrapper(rnn_cell_impl.GRUCell(3), 2)
-        g, new_m = cell(x, m)
-        sess.run([variables_lib.global_variables_initializer()])
-        res = sess.run([g, new_m], {
-            x.name: np.array([[1., 1., 1.]]),
-            m.name: np.array([[0.1, 0.1, 0.1]])
-        })
-        self.assertEqual(res[1].shape, (1, 3))
-        # The numbers in results were not calculated, this is just a smoke test.
-        self.assertAllClose(res[0], [[0.231907, 0.231907]])
-
-  def testInputProjectionWrapper(self):
-    with self.cached_session() as sess:
-      with variable_scope.variable_scope(
-          "root", initializer=init_ops.constant_initializer(0.5)):
-        x = array_ops.zeros([1, 2])
-        m = array_ops.zeros([1, 3])
-        cell = contrib_rnn.InputProjectionWrapper(
-            rnn_cell_impl.GRUCell(3), num_proj=3)
-        g, new_m = cell(x, m)
-        sess.run([variables_lib.global_variables_initializer()])
-        res = sess.run([g, new_m], {
-            x.name: np.array([[1., 1.]]),
-            m.name: np.array([[0.1, 0.1, 0.1]])
-        })
-        self.assertEqual(res[1].shape, (1, 3))
-        # The numbers in results were not calculated, this is just a smoke test.
-        self.assertAllClose(res[0], [[0.154605, 0.154605, 0.154605]])
-
-  @parameterized.parameters(
-      [rnn_cell_impl.ResidualWrapper, rnn_cell_impl.ResidualWrapperV2])
-  @test_util.run_in_graph_and_eager_modes
-  def testResidualWrapper(self, wrapper_type):
-    x = ops.convert_to_tensor(np.array([[1., 1., 1.]]))
-    m = ops.convert_to_tensor(np.array([[0.1, 0.1, 0.1]]))
-    base_cell = rnn_cell_impl.GRUCell(
-        3, kernel_initializer=init_ops.constant_initializer(0.5),
-        bias_initializer=init_ops.constant_initializer(0.5))
-    g, m_new = base_cell(x, m)
-    wrapper_object = wrapper_type(base_cell)
-    (name, dep), = wrapper_object._checkpoint_dependencies
-    wrapper_object.get_config()  # Should not throw an error
-    self.assertIs(dep, base_cell)
-    self.assertEqual("cell", name)
-
-    g_res, m_new_res = wrapper_object(x, m)
-    self.evaluate([variables_lib.global_variables_initializer()])
-    res = self.evaluate([g, g_res, m_new, m_new_res])
-    # Residual connections
-    self.assertAllClose(res[1], res[0] + [1., 1., 1.])
-    # States are left untouched
-    self.assertAllClose(res[2], res[3])
-
-  @parameterized.parameters(
-      [rnn_cell_impl.ResidualWrapper, rnn_cell_impl.ResidualWrapperV2])
-  @test_util.run_in_graph_and_eager_modes
-  def testResidualWrapperWithSlice(self, wrapper_type):
-    x = ops.convert_to_tensor(np.array([[1., 1., 1., 1., 1.]]))
-    m = ops.convert_to_tensor(np.array([[0.1, 0.1, 0.1]]))
-    base_cell = rnn_cell_impl.GRUCell(
-        3, kernel_initializer=init_ops.constant_initializer(0.5),
-        bias_initializer=init_ops.constant_initializer(0.5))
-    g, m_new = base_cell(x, m)
-
-    def residual_with_slice_fn(inp, out):
-      inp_sliced = array_ops.slice(inp, [0, 0], [-1, 3])
-      return inp_sliced + out
-
-    g_res, m_new_res = wrapper_type(
-        base_cell, residual_with_slice_fn)(x, m)
-    self.evaluate([variables_lib.global_variables_initializer()])
-    res_g, res_g_res, res_m_new, res_m_new_res = self.evaluate(
-        [g, g_res, m_new, m_new_res])
-    # Residual connections
-    self.assertAllClose(res_g_res, res_g + [1., 1., 1.])
-    # States are left untouched
-    self.assertAllClose(res_m_new, res_m_new_res)
-
-  def testDeviceWrapper(self):
-    with variable_scope.variable_scope(
-        "root", initializer=init_ops.constant_initializer(0.5)):
-      x = array_ops.zeros([1, 3])
-      m = array_ops.zeros([1, 3])
-      wrapped = rnn_cell_impl.GRUCell(3)
-      cell = rnn_cell_impl.DeviceWrapper(wrapped, "/cpu:14159")
-      (name, dep), = cell._checkpoint_dependencies
-      cell.get_config()  # Should not throw an error
-      self.assertIs(dep, wrapped)
-      self.assertEqual("cell", name)
-
-      outputs, _ = cell(x, m)
-      self.assertTrue("cpu:14159" in outputs.device.lower())
-
-  def _retrieve_cpu_gpu_stats(self, run_metadata):
-    cpu_stats = None
-    gpu_stats = None
-    step_stats = run_metadata.step_stats
-    for ds in step_stats.dev_stats:
-      if "cpu:0" in ds.device[-5:].lower():
-        cpu_stats = ds.node_stats
-      if "gpu:0" == ds.device[-5:].lower():
-        gpu_stats = ds.node_stats
-    return cpu_stats, gpu_stats
-
-  def testDeviceWrapperDynamicExecutionNodesAreAllProperlyLocated(self):
-    if not test.is_gpu_available():
-      # Can't perform this test w/o a GPU
-      return
-
-    gpu_dev = test.gpu_device_name()
-    with self.session(use_gpu=True) as sess:
-      with variable_scope.variable_scope(
-          "root", initializer=init_ops.constant_initializer(0.5)):
-        x = array_ops.zeros([1, 1, 3])
-        cell = rnn_cell_impl.DeviceWrapper(rnn_cell_impl.GRUCell(3), gpu_dev)
-        with ops.device("/cpu:0"):
-          outputs, _ = rnn.dynamic_rnn(
-              cell=cell, inputs=x, dtype=dtypes.float32)
-        run_metadata = config_pb2.RunMetadata()
-        opts = config_pb2.RunOptions(
-            trace_level=config_pb2.RunOptions.FULL_TRACE)
-
-        sess.run([variables_lib.global_variables_initializer()])
-        _ = sess.run(outputs, options=opts, run_metadata=run_metadata)
-
-      cpu_stats, gpu_stats = self._retrieve_cpu_gpu_stats(run_metadata)
-      self.assertFalse([s for s in cpu_stats if "gru_cell" in s.node_name])
-      self.assertTrue([s for s in gpu_stats if "gru_cell" in s.node_name])
-
-  def testEmbeddingWrapper(self):
-    with self.cached_session() as sess:
-      with variable_scope.variable_scope(
-          "root", initializer=init_ops.constant_initializer(0.5)):
-        x = array_ops.zeros([1, 1], dtype=dtypes.int32)
-        m = array_ops.zeros([1, 2])
-        embedding_cell = contrib_rnn.EmbeddingWrapper(
-            rnn_cell_impl.GRUCell(2), embedding_classes=3, embedding_size=2)
-        self.assertEqual(embedding_cell.output_size, 2)
-        g, new_m = embedding_cell(x, m)
-        sess.run([variables_lib.global_variables_initializer()])
-        res = sess.run([g, new_m], {
-            x.name: np.array([[1]]),
-            m.name: np.array([[0.1, 0.1]])
-        })
-        self.assertEqual(res[1].shape, (1, 2))
-        # The numbers in results were not calculated, this is just a smoke test.
-        self.assertAllClose(res[0], [[0.17139, 0.17139]])
-
-  def testEmbeddingWrapperWithDynamicRnn(self):
-    with self.cached_session() as sess:
-      with variable_scope.variable_scope("root"):
-        inputs = ops.convert_to_tensor([[[0], [0]]], dtype=dtypes.int64)
-        input_lengths = ops.convert_to_tensor([2], dtype=dtypes.int64)
-        embedding_cell = contrib_rnn.EmbeddingWrapper(
-            rnn_cell_impl.BasicLSTMCell(1, state_is_tuple=True),
-            embedding_classes=1,
-            embedding_size=2)
-        outputs, _ = rnn.dynamic_rnn(
-            cell=embedding_cell,
-            inputs=inputs,
-            sequence_length=input_lengths,
-            dtype=dtypes.float32)
-        sess.run([variables_lib.global_variables_initializer()])
-        # This will fail if output's dtype is inferred from input's.
-        sess.run(outputs)
-
-  def testMultiRNNCell(self):
-    with self.cached_session() as sess:
-      with variable_scope.variable_scope(
-          "root", initializer=init_ops.constant_initializer(0.5)):
-        x = array_ops.zeros([1, 2])
-        m = array_ops.zeros([1, 4])
-        multi_rnn_cell = rnn_cell_impl.MultiRNNCell(
-            [rnn_cell_impl.GRUCell(2) for _ in range(2)],
-            state_is_tuple=False)
-        _, ml = multi_rnn_cell(x, m)
-        sess.run([variables_lib.global_variables_initializer()])
-        res = sess.run(ml, {
-            x.name: np.array([[1., 1.]]),
-            m.name: np.array([[0.1, 0.1, 0.1, 0.1]])
-        })
-        # The numbers in results were not calculated, this is just a smoke test.
-        self.assertAllClose(res, [[0.175991, 0.175991, 0.13248, 0.13248]])
-        self.assertEqual(len(multi_rnn_cell.weights), 2 * 4)
-        self.assertTrue(
-            [x.dtype == dtypes.float32 for x in multi_rnn_cell.weights])
-
-  def testMultiRNNCellWithStateTuple(self):
-    with self.cached_session() as sess:
-      with variable_scope.variable_scope(
-          "root", initializer=init_ops.constant_initializer(0.5)):
-        x = array_ops.zeros([1, 2])
-        m_bad = array_ops.zeros([1, 4])
-        m_good = (array_ops.zeros([1, 2]), array_ops.zeros([1, 2]))
-
-        # Test incorrectness of state
-        with self.assertRaisesRegexp(ValueError, "Expected state .* a tuple"):
-          rnn_cell_impl.MultiRNNCell(
-              [rnn_cell_impl.GRUCell(2) for _ in range(2)],
-              state_is_tuple=True)(x, m_bad)
-
-        _, ml = rnn_cell_impl.MultiRNNCell(
-            [rnn_cell_impl.GRUCell(2) for _ in range(2)],
-            state_is_tuple=True)(x, m_good)
-
-        sess.run([variables_lib.global_variables_initializer()])
-        res = sess.run(
-            ml, {
-                x.name: np.array([[1., 1.]]),
-                m_good[0].name: np.array([[0.1, 0.1]]),
-                m_good[1].name: np.array([[0.1, 0.1]])
-            })
-
-        # The numbers in results were not calculated, this is just a
-        # smoke test.  However, these numbers should match those of
-        # the test testMultiRNNCell.
-        self.assertAllClose(res[0], [[0.175991, 0.175991]])
-        self.assertAllClose(res[1], [[0.13248, 0.13248]])
-
-  @parameterized.parameters(
-      [[rnn_cell_impl.DropoutWrapper, rnn_cell_impl.DropoutWrapperV2],
-       [rnn_cell_impl.ResidualWrapper, rnn_cell_impl.ResidualWrapperV2]])
-  @test_util.run_in_graph_and_eager_modes
-  def testWrapperKerasStyle(self, wrapper, wrapper_v2):
-    """Tests if wrapper cell is instantiated in keras style scope."""
-    wrapped_cell_v2 = wrapper_v2(rnn_cell_impl.BasicRNNCell(1))
-    self.assertTrue(wrapped_cell_v2._keras_style)
-
-    wrapped_cell = wrapper(rnn_cell_impl.BasicRNNCell(1))
-    self.assertFalse(wrapped_cell._keras_style)
-
-  @parameterized.parameters(
-      [rnn_cell_impl.DropoutWrapperV2, rnn_cell_impl.ResidualWrapperV2])
-  @test_util.run_in_graph_and_eager_modes
-  def testWrapperV2VariableNames(self, wrapper):
-    """Tests that variables names do not depend on wrapper in RNN layer."""
-
-    def _rnn_input(apply_wrapper, name):
-      """Creates a RNN layer with/without wrapper and returns built rnn cell."""
-      with base_layer.keras_style_scope():
-        base_cell = rnn_cell_impl.MultiRNNCell(
-            [rnn_cell_impl.BasicRNNCell(1, name="basic_rnn_cell")
-             for _ in range(2)])
-      if apply_wrapper:
-        rnn_cell = wrapper(base_cell)
-      else:
-        rnn_cell = base_cell
-      rnn_layer = keras_layers.RNN(rnn_cell, name=name)
-      inputs = ops.convert_to_tensor([[[1]]], dtype=dtypes.float32)
-      _ = rnn_layer(inputs)
-      return base_cell._cells[0]
-
-    rnn_1 = _rnn_input(True, name="rnn_0")
-    rnn_2 = _rnn_input(False, name="rnn_1")
-
-    for i, cell in enumerate([rnn_1, rnn_2]):
-      var_prefix = "rnn_{}/cell_0/basic_rnn_cell/".format(i)
-      self.assertCountEqual([v.name for v in cell.weights],
-                            (var_prefix + "kernel:0", var_prefix + "bias:0"))
-
-  @parameterized.parameters(
-      [rnn_cell_impl.DropoutWrapperV2, rnn_cell_impl.ResidualWrapperV2])
-  @test_util.run_in_graph_and_eager_modes
-  def testWrapperWeights(self, wrapper):
-    """Tests that wrapper weights contain wrapped cells weights."""
-
-    with base_layer.keras_style_scope():
-      base_cell = rnn_cell_impl.BasicRNNCell(1, name="basic_rnn_cell")
-    rnn_cell = wrapper(base_cell)
-    rnn_layer = keras_layers.RNN(rnn_cell)
-    inputs = ops.convert_to_tensor([[[1]]], dtype=dtypes.float32)
-    rnn_layer(inputs)
-
-    expected_weights = ["rnn/" + var for var in ("kernel:0", "bias:0")]
-    self.assertEqual(len(rnn_cell.weights), 2)
-    self.assertCountEqual([v.name for v in rnn_cell.weights], expected_weights)
-    self.assertCountEqual([v.name for v in rnn_cell.trainable_variables],
-                          expected_weights)
-    self.assertCountEqual([v.name for v in rnn_cell.non_trainable_variables],
-                          [])
-    self.assertCountEqual([v.name for v in rnn_cell._cell.weights],
-                          expected_weights)
-
-  @parameterized.parameters(
-      [rnn_cell_impl.DropoutWrapperV2, rnn_cell_impl.ResidualWrapperV2])
-  @test_util.run_in_graph_and_eager_modes
-  def testWrapperV2Caller(self, wrapper):
-    """Tests that wrapper V2 is using the LayerRNNCell's caller."""
-
-    with base_layer.keras_style_scope():
-      base_cell = rnn_cell_impl.MultiRNNCell(
-          [rnn_cell_impl.BasicRNNCell(1) for _ in range(2)])
-    rnn_cell = wrapper(base_cell)
-    inputs = ops.convert_to_tensor([[1]], dtype=dtypes.float32)
-    state = ops.convert_to_tensor([[1]], dtype=dtypes.float32)
-    _ = rnn_cell(inputs, [state, state])
-    weights = base_cell._cells[0].weights
-    self.assertLen(weights, expected_len=2)
-    self.assertTrue(all(["_wrapper" in v.name for v in weights]))
-
-  @parameterized.parameters(
-      [rnn_cell_impl.DropoutWrapperV2, rnn_cell_impl.ResidualWrapperV2])
-  @test_util.run_in_graph_and_eager_modes
-  def testWrapperV2Build(self, wrapper):
-    cell = rnn_cell_impl.LSTMCell(10)
-    wrapper = wrapper(cell)
-    wrapper.build((1,))
-    self.assertTrue(cell.built)
-
-
-@test_util.run_all_in_graph_and_eager_modes
-class DropoutWrapperTest(test.TestCase, parameterized.TestCase):
-
-  def _testDropoutWrapper(self,
-                          batch_size=None,
-                          time_steps=None,
-                          parallel_iterations=None,
-                          wrapper_type=None,
-                          scope="root",
-                          **kwargs):
-    if batch_size is None and time_steps is None:
-      # 2 time steps, batch size 1, depth 3
-      batch_size = 1
-      time_steps = 2
-      x = constant_op.constant(
-          [[[2., 2., 2.]], [[1., 1., 1.]]], dtype=dtypes.float32)
-      m = rnn_cell_impl.LSTMStateTuple(
-          *[constant_op.constant([[0.1, 0.1, 0.1]], dtype=dtypes.float32)] * 2)
-    else:
-      x = constant_op.constant(
-          np.random.randn(time_steps, batch_size, 3).astype(np.float32))
-      m = rnn_cell_impl.LSTMStateTuple(*[
-          constant_op.
-          constant([[0.1, 0.1, 0.1]] * batch_size, dtype=dtypes.float32)] * 2)
-    outputs, final_state = rnn.dynamic_rnn(
-        cell=wrapper_type(
-            rnn_cell_impl.LSTMCell(
-                3, initializer=init_ops.constant_initializer(0.5)),
-            dtype=x.dtype, **kwargs),
-        time_major=True,
-        parallel_iterations=parallel_iterations,
-        inputs=x,
-        initial_state=m,
-        scope=scope)
-    self.evaluate([variables_lib.global_variables_initializer()])
-    res = self.evaluate([outputs, final_state])
-    self.assertEqual(res[0].shape, (time_steps, batch_size, 3))
-    self.assertEqual(res[1].c.shape, (batch_size, 3))
-    self.assertEqual(res[1].h.shape, (batch_size, 3))
-    return res
-
-  @parameterized.parameters(
-      [rnn_cell_impl.DropoutWrapper, rnn_cell_impl.DropoutWrapperV2])
-  def testDropoutWrapperProperties(self, wrapper_type):
-    cell = rnn_cell_impl.BasicRNNCell(10)
-    wrapper = wrapper_type(cell)
-    # Github issue 15810
-    self.assertEqual(wrapper.wrapped_cell, cell)
-    self.assertEqual(wrapper.state_size, 10)
-    self.assertEqual(wrapper.output_size, 10)
-
-  @parameterized.parameters(
-      [rnn_cell_impl.DropoutWrapper, rnn_cell_impl.DropoutWrapperV2])
-  def testDropoutWrapperZeroState(self, wrapper_type):
-    class _Cell(rnn_cell_impl.BasicRNNCell):
-
-      def zero_state(self, batch_size=None, dtype=None):
-        return "wrapped_cell_zero_state"
-    wrapper = wrapper_type(_Cell(10))
-    self.assertEqual(wrapper.zero_state(10, dtypes.float32),
-                     "wrapped_cell_zero_state")
-
-  @parameterized.parameters(
-      [rnn_cell_impl.DropoutWrapper, rnn_cell_impl.DropoutWrapperV2])
-  def testDropoutWrapperKeepAllConstantInput(self, wrapper_type):
-    keep = array_ops.ones([])
-    res = self._testDropoutWrapper(
-        input_keep_prob=keep, output_keep_prob=keep, state_keep_prob=keep,
-        wrapper_type=wrapper_type)
-    true_full_output = np.array(
-        [[[0.751109, 0.751109, 0.751109]], [[0.895509, 0.895509, 0.895509]]],
-        dtype=np.float32)
-    true_full_final_c = np.array(
-        [[1.949385, 1.949385, 1.949385]], dtype=np.float32)
-    self.assertAllClose(true_full_output, res[0])
-    self.assertAllClose(true_full_output[1], res[1].h)
-    self.assertAllClose(true_full_final_c, res[1].c)
-
-  @parameterized.parameters(
-      [rnn_cell_impl.DropoutWrapper, rnn_cell_impl.DropoutWrapperV2])
-  def testDropoutWrapperKeepAll(self, wrapper_type):
-    keep = variable_scope.get_variable("all", initializer=1.0)
-    res = self._testDropoutWrapper(
-        input_keep_prob=keep, output_keep_prob=keep, state_keep_prob=keep,
-        wrapper_type=wrapper_type)
-    true_full_output = np.array(
-        [[[0.751109, 0.751109, 0.751109]], [[0.895509, 0.895509, 0.895509]]],
-        dtype=np.float32)
-    true_full_final_c = np.array(
-        [[1.949385, 1.949385, 1.949385]], dtype=np.float32)
-    self.assertAllClose(true_full_output, res[0])
-    self.assertAllClose(true_full_output[1], res[1].h)
-    self.assertAllClose(true_full_final_c, res[1].c)
-
-  @parameterized.parameters(
-      [rnn_cell_impl.DropoutWrapper, rnn_cell_impl.DropoutWrapperV2])
-  def testDropoutWrapperWithSeed(self, wrapper_type):
-    keep_some = 0.5
-    random_seed.set_random_seed(2)
-    ## Use parallel_iterations = 1 in both calls to
-    ## _testDropoutWrapper to ensure the (per-time step) dropout is
-    ## consistent across both calls.  Otherwise the seed may not end
-    ## up being munged consistently across both graphs.
-    res_standard_1 = self._testDropoutWrapper(
-        input_keep_prob=keep_some,
-        output_keep_prob=keep_some,
-        state_keep_prob=keep_some,
-        seed=10,
-        parallel_iterations=1,
-        wrapper_type=wrapper_type,
-        scope="root_1")
-    random_seed.set_random_seed(2)
-    res_standard_2 = self._testDropoutWrapper(
-        input_keep_prob=keep_some,
-        output_keep_prob=keep_some,
-        state_keep_prob=keep_some,
-        seed=10,
-        parallel_iterations=1,
-        wrapper_type=wrapper_type,
-        scope="root_2")
-    self.assertAllClose(res_standard_1[0], res_standard_2[0])
-    self.assertAllClose(res_standard_1[1].c, res_standard_2[1].c)
-    self.assertAllClose(res_standard_1[1].h, res_standard_2[1].h)
-
-  @parameterized.parameters(
-      [rnn_cell_impl.DropoutWrapper, rnn_cell_impl.DropoutWrapperV2])
-  def testDropoutWrapperKeepNoOutput(self, wrapper_type):
-    keep_all = variable_scope.get_variable("all", initializer=1.0)
-    keep_none = variable_scope.get_variable("none", initializer=1e-6)
-    res = self._testDropoutWrapper(
-        input_keep_prob=keep_all,
-        output_keep_prob=keep_none,
-        state_keep_prob=keep_all,
-        wrapper_type=wrapper_type)
-    true_full_output = np.array(
-        [[[0.751109, 0.751109, 0.751109]], [[0.895509, 0.895509, 0.895509]]],
-        dtype=np.float32)
-    true_full_final_c = np.array(
-        [[1.949385, 1.949385, 1.949385]], dtype=np.float32)
-    self.assertAllClose(np.zeros(res[0].shape), res[0])
-    self.assertAllClose(true_full_output[1], res[1].h)
-    self.assertAllClose(true_full_final_c, res[1].c)
-
-  @parameterized.parameters(
-      [rnn_cell_impl.DropoutWrapper, rnn_cell_impl.DropoutWrapperV2])
-  def testDropoutWrapperKeepNoStateExceptLSTMCellMemory(self, wrapper_type):
-    keep_all = variable_scope.get_variable("all", initializer=1.0)
-    keep_none = variable_scope.get_variable("none", initializer=1e-6)
-    # Even though we dropout state, by default DropoutWrapper never
-    # drops out the memory ("c") term of an LSTMStateTuple.
-    res = self._testDropoutWrapper(
-        input_keep_prob=keep_all,
-        output_keep_prob=keep_all,
-        state_keep_prob=keep_none,
-        wrapper_type=wrapper_type)
-    true_c_state = np.array([[1.713925, 1.713925, 1.713925]], dtype=np.float32)
-    true_full_output = np.array(
-        [[[0.751109, 0.751109, 0.751109]], [[0.895509, 0.895509, 0.895509]]],
-        dtype=np.float32)
-    self.assertAllClose(true_full_output[0], res[0][0])
-    # Second output is modified by zero input state
-    self.assertGreater(np.linalg.norm(true_full_output[1] - res[0][1]), 1e-4)
-    # h state has been set to zero
-    self.assertAllClose(np.zeros(res[1].h.shape), res[1].h)
-    # c state of an LSTMStateTuple is NEVER modified.
-    self.assertAllClose(true_c_state, res[1].c)
-
-  @parameterized.parameters(
-      [rnn_cell_impl.DropoutWrapper, rnn_cell_impl.DropoutWrapperV2])
-  def testDropoutWrapperKeepNoInput(self, wrapper_type):
-    keep_all = variable_scope.get_variable("all", initializer=1.0)
-    keep_none = variable_scope.get_variable("none", initializer=1e-6)
-    true_full_output = np.array(
-        [[[0.751109, 0.751109, 0.751109]], [[0.895509, 0.895509, 0.895509]]],
-        dtype=np.float32)
-    true_full_final_c = np.array(
-        [[1.949385, 1.949385, 1.949385]], dtype=np.float32)
-    # All outputs are different because inputs are zeroed out
-    res = self._testDropoutWrapper(
-        input_keep_prob=keep_none,
-        output_keep_prob=keep_all,
-        state_keep_prob=keep_all,
-        wrapper_type=wrapper_type)
-    self.assertGreater(np.linalg.norm(res[0] - true_full_output), 1e-4)
-    self.assertGreater(np.linalg.norm(res[1].h - true_full_output[1]), 1e-4)
-    self.assertGreater(np.linalg.norm(res[1].c - true_full_final_c), 1e-4)
-
-  @parameterized.parameters(
-      [rnn_cell_impl.DropoutWrapper, rnn_cell_impl.DropoutWrapperV2])
-  def testDropoutWrapperRecurrentOutput(self, wrapper_type):
-    keep_some = 0.8
-    keep_all = variable_scope.get_variable("all", initializer=1.0)
-    res = self._testDropoutWrapper(
-        input_keep_prob=keep_all,
-        output_keep_prob=keep_some,
-        state_keep_prob=keep_all,
-        variational_recurrent=True,
-        wrapper_type=wrapper_type,
-        input_size=3,
-        batch_size=5,
-        time_steps=7)
-    # Ensure the same dropout pattern for all time steps
-    output_mask = np.abs(res[0]) > 1e-6
-    for m in output_mask[1:]:
-      self.assertAllClose(output_mask[0], m)
-
-  @parameterized.parameters(
-      [rnn_cell_impl.DropoutWrapper, rnn_cell_impl.DropoutWrapperV2])
-  def testDropoutWrapperRecurrentStateInputAndOutput(self, wrapper_type):
-    keep_some = 0.9
-    res = self._testDropoutWrapper(
-        input_keep_prob=keep_some,
-        output_keep_prob=keep_some,
-        state_keep_prob=keep_some,
-        variational_recurrent=True,
-        wrapper_type=wrapper_type,
-        input_size=3,
-        batch_size=5,
-        time_steps=7)
-
-    # Smoke test for the state/input masks.
-    output_mask = np.abs(res[0]) > 1e-6
-    for time_step in output_mask:
-      # Ensure the same dropout output pattern for all time steps
-      self.assertAllClose(output_mask[0], time_step)
-      for batch_entry in time_step:
-        # Assert all batch entries get the same mask
-        self.assertAllClose(batch_entry, time_step[0])
-
-    # For state, ensure all batch entries have the same mask
-    state_c_mask = np.abs(res[1].c) > 1e-6
-    state_h_mask = np.abs(res[1].h) > 1e-6
-    for batch_entry in state_c_mask:
-      self.assertAllClose(batch_entry, state_c_mask[0])
-    for batch_entry in state_h_mask:
-      self.assertAllClose(batch_entry, state_h_mask[0])
-
-  @parameterized.parameters(
-      [rnn_cell_impl.DropoutWrapper, rnn_cell_impl.DropoutWrapperV2])
-  def testDropoutWrapperRecurrentStateInputAndOutputWithSeed(
-      self, wrapper_type):
-    keep_some = 0.9
-    random_seed.set_random_seed(2347)
-    np.random.seed(23487)
-    res0 = self._testDropoutWrapper(
-        input_keep_prob=keep_some,
-        output_keep_prob=keep_some,
-        state_keep_prob=keep_some,
-        variational_recurrent=True,
-        wrapper_type=wrapper_type,
-        input_size=3,
-        batch_size=5,
-        time_steps=7,
-        seed=-234987,
-        scope="root_0")
-    random_seed.set_random_seed(2347)
-    np.random.seed(23487)
-    res1 = self._testDropoutWrapper(
-        input_keep_prob=keep_some,
-        output_keep_prob=keep_some,
-        state_keep_prob=keep_some,
-        variational_recurrent=True,
-        wrapper_type=wrapper_type,
-        input_size=3,
-        batch_size=5,
-        time_steps=7,
-        seed=-234987,
-        scope="root_1")
-
-    output_mask = np.abs(res0[0]) > 1e-6
-    for time_step in output_mask:
-      # Ensure the same dropout output pattern for all time steps
-      self.assertAllClose(output_mask[0], time_step)
-      for batch_entry in time_step:
-        # Assert all batch entries get the same mask
-        self.assertAllClose(batch_entry, time_step[0])
-
-    # For state, ensure all batch entries have the same mask
-    state_c_mask = np.abs(res0[1].c) > 1e-6
-    state_h_mask = np.abs(res0[1].h) > 1e-6
-    for batch_entry in state_c_mask:
-      self.assertAllClose(batch_entry, state_c_mask[0])
-    for batch_entry in state_h_mask:
-      self.assertAllClose(batch_entry, state_h_mask[0])
-
-    # Ensure seeded calculation is identical.
-    self.assertAllClose(res0[0], res1[0])
-    self.assertAllClose(res0[1].c, res1[1].c)
-    self.assertAllClose(res0[1].h, res1[1].h)
-
-
-def basic_rnn_cell(inputs, state, num_units, scope=None):
-  if state is None:
-    if inputs is not None:
-      batch_size = inputs.get_shape()[0]
-      dtype = inputs.dtype
-    else:
-      batch_size = 0
-      dtype = dtypes.float32
-    init_output = array_ops.zeros(
-        array_ops.stack([batch_size, num_units]), dtype=dtype)
-    init_state = array_ops.zeros(
-        array_ops.stack([batch_size, num_units]), dtype=dtype)
-    init_output.set_shape([batch_size, num_units])
-    init_state.set_shape([batch_size, num_units])
-    return init_output, init_state
-  else:
-    with variable_scope.variable_scope(scope, "basic_rnn_cell",
-                                       [inputs, state]):
-      output = math_ops.tanh(
-          Linear([inputs, state], num_units, True)([inputs, state]))
-    return output, output
-
-
-if __name__ == "__main__":
-  test.main()
diff --git a/tensorflow/contrib/rnn/python/kernel_tests/rnn_cell_test.py b/tensorflow/contrib/rnn/python/kernel_tests/rnn_cell_test.py
index d7ee7fb8fa..dfac2df6a0 100644
--- a/tensorflow/contrib/rnn/python/kernel_tests/rnn_cell_test.py
+++ b/tensorflow/contrib/rnn/python/kernel_tests/rnn_cell_test.py
@@ -22,6 +22,7 @@ import itertools
 
 import numpy as np
 
+from tensorflow.contrib.rnn.python.ops import core_rnn_cell as legacy_rnn_cell
 from tensorflow.contrib.rnn.python.ops import rnn_cell as contrib_rnn_cell
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.client import session
@@ -53,6 +54,294 @@ from tensorflow.python.util import nest
 
 class RNNCellTest(test.TestCase):
 
+  def testIndRNNCell(self):
+    with self.cached_session() as sess:
+      with variable_scope.variable_scope(
+          "root", initializer=init_ops.constant_initializer(0.5)):
+        x = array_ops.zeros([1, 2])
+        m = array_ops.zeros([1, 2])
+        cell = contrib_rnn_cell.IndRNNCell(2)
+        g, _ = cell(x, m)
+        self.assertEqual([
+            "root/ind_rnn_cell/%s_w:0" % rnn_cell_impl._WEIGHTS_VARIABLE_NAME,
+            "root/ind_rnn_cell/%s_u:0" % rnn_cell_impl._WEIGHTS_VARIABLE_NAME,
+            "root/ind_rnn_cell/%s:0" % rnn_cell_impl._BIAS_VARIABLE_NAME
+        ], [v.name for v in cell.trainable_variables])
+        self.assertFalse(cell.non_trainable_variables)
+        sess.run([variables.global_variables_initializer()])
+        res = sess.run([g], {
+            x.name: np.array([[1., 1.]]),
+            m.name: np.array([[0.1, 0.1]])
+        })
+        self.assertEqual(res[0].shape, (1, 2))
+
+  def testIndyGRUCell(self):
+    with self.cached_session() as sess:
+      with variable_scope.variable_scope(
+          "root", initializer=init_ops.constant_initializer(0.5)):
+        x = array_ops.zeros([1, 2])
+        m = array_ops.zeros([1, 2])
+        g, _ = contrib_rnn_cell.IndyGRUCell(2)(x, m)
+        sess.run([variables.global_variables_initializer()])
+        res = sess.run([g], {
+            x.name: np.array([[1., 1.]]),
+            m.name: np.array([[0.1, 0.1]])
+        })
+        # Smoke test
+        self.assertAllClose(res[0], [[0.185265, 0.17704]])
+      with variable_scope.variable_scope(
+          "other", initializer=init_ops.constant_initializer(0.5)):
+        # Test IndyGRUCell with input_size != num_units.
+        x = array_ops.zeros([1, 3])
+        m = array_ops.zeros([1, 2])
+        g, _ = contrib_rnn_cell.IndyGRUCell(2)(x, m)
+        sess.run([variables.global_variables_initializer()])
+        res = sess.run([g], {
+            x.name: np.array([[1., 1., 1.]]),
+            m.name: np.array([[0.1, 0.1]])
+        })
+        # Smoke test
+        self.assertAllClose(res[0], [[0.155127, 0.157328]])
+
+  def testIndyLSTMCell(self):
+    for dtype in [dtypes.float16, dtypes.float32]:
+      np_dtype = dtype.as_numpy_dtype
+      with self.session(graph=ops.Graph()) as sess:
+        with variable_scope.variable_scope(
+            "root", initializer=init_ops.constant_initializer(0.5)):
+          x = array_ops.zeros([1, 2], dtype=dtype)
+          state_0 = (array_ops.zeros([1, 2], dtype=dtype),) * 2
+          state_1 = (array_ops.zeros([1, 2], dtype=dtype),) * 2
+          cell = rnn_cell_impl.MultiRNNCell(
+              [contrib_rnn_cell.IndyLSTMCell(2) for _ in range(2)])
+          self.assertEqual(cell.dtype, None)
+          self.assertEqual("cell-0", cell._checkpoint_dependencies[0].name)
+          self.assertEqual("cell-1", cell._checkpoint_dependencies[1].name)
+          cell.get_config()  # Should not throw an error
+          g, (out_state_0, out_state_1) = cell(x, (state_0, state_1))
+          # Layer infers the input type.
+          self.assertEqual(cell.dtype, dtype.name)
+          expected_variable_names = [
+              "root/multi_rnn_cell/cell_0/indy_lstm_cell/%s_w:0" %
+              rnn_cell_impl._WEIGHTS_VARIABLE_NAME,
+              "root/multi_rnn_cell/cell_0/indy_lstm_cell/%s_u:0" %
+              rnn_cell_impl._WEIGHTS_VARIABLE_NAME,
+              "root/multi_rnn_cell/cell_0/indy_lstm_cell/%s:0" %
+              rnn_cell_impl._BIAS_VARIABLE_NAME,
+              "root/multi_rnn_cell/cell_1/indy_lstm_cell/%s_w:0" %
+              rnn_cell_impl._WEIGHTS_VARIABLE_NAME,
+              "root/multi_rnn_cell/cell_1/indy_lstm_cell/%s_u:0" %
+              rnn_cell_impl._WEIGHTS_VARIABLE_NAME,
+              "root/multi_rnn_cell/cell_1/indy_lstm_cell/%s:0" %
+              rnn_cell_impl._BIAS_VARIABLE_NAME
+          ]
+          self.assertEqual(expected_variable_names,
+                           [v.name for v in cell.trainable_variables])
+          self.assertFalse(cell.non_trainable_variables)
+          sess.run([variables.global_variables_initializer()])
+          res = sess.run(
+              [g, out_state_0, out_state_1], {
+                  x.name: np.array([[1., 1.]]),
+                  state_0[0].name: 0.1 * np.ones([1, 2]),
+                  state_0[1].name: 0.1 * np.ones([1, 2]),
+                  state_1[0].name: 0.1 * np.ones([1, 2]),
+                  state_1[1].name: 0.1 * np.ones([1, 2]),
+              })
+          self.assertEqual(len(res), 3)
+          global_variables = variables.global_variables()
+          self.assertEqual(expected_variable_names,
+                           [v.name for v in global_variables])
+          # Only check the range of outputs as this is just a smoke test.
+          self.assertAllInRange(res[0], -1.0, 1.0)
+          self.assertAllInRange(res[1], -1.0, 1.0)
+          self.assertAllInRange(res[2], -1.0, 1.0)
+        with variable_scope.variable_scope(
+            "other", initializer=init_ops.constant_initializer(0.5)):
+          # Test IndyLSTMCell with input_size != num_units.
+          x = array_ops.zeros([1, 3], dtype=dtype)
+          state = (array_ops.zeros([1, 2], dtype=dtype),) * 2
+          g, out_state = contrib_rnn_cell.IndyLSTMCell(2)(x, state)
+          sess.run([variables.global_variables_initializer()])
+          res = sess.run(
+              [g, out_state], {
+                  x.name: np.array([[1., 1., 1.]], dtype=np_dtype),
+                  state[0].name: 0.1 * np.ones([1, 2], dtype=np_dtype),
+                  state[1].name: 0.1 * np.ones([1, 2], dtype=np_dtype),
+              })
+          self.assertEqual(len(res), 2)
+
+  def testLSTMCellLayerNorm(self):
+    with self.cached_session() as sess:
+      num_units = 2
+      num_proj = 3
+      batch_size = 1
+      input_size = 4
+      with variable_scope.variable_scope(
+          "root", initializer=init_ops.constant_initializer(0.5)):
+        x = array_ops.zeros([batch_size, input_size])
+        c = array_ops.zeros([batch_size, num_units])
+        h = array_ops.zeros([batch_size, num_proj])
+        state = rnn_cell_impl.LSTMStateTuple(c, h)
+        cell = contrib_rnn_cell.LayerNormLSTMCell(
+            num_units=num_units,
+            num_proj=num_proj,
+            forget_bias=1.0,
+            layer_norm=True,
+            norm_gain=1.0,
+            norm_shift=0.0)
+        g, out_m = cell(x, state)
+        sess.run([variables.global_variables_initializer()])
+        res = sess.run(
+            [g, out_m], {
+                x.name: np.ones((batch_size, input_size)),
+                c.name: 0.1 * np.ones((batch_size, num_units)),
+                h.name: 0.1 * np.ones((batch_size, num_proj))
+            })
+        self.assertEqual(len(res), 2)
+        # The numbers in results were not calculated, this is mostly just a
+        # smoke test.
+        self.assertEqual(res[0].shape, (batch_size, num_proj))
+        self.assertEqual(res[1][0].shape, (batch_size, num_units))
+        self.assertEqual(res[1][1].shape, (batch_size, num_proj))
+        # Different inputs so different outputs and states
+        for i in range(1, batch_size):
+          self.assertTrue(
+              float(np.linalg.norm((res[0][0, :] - res[0][i, :]))) < 1e-6)
+          self.assertTrue(
+              float(np.linalg.norm((res[1][0, :] - res[1][i, :]))) < 1e-6)
+
+  def testOutputProjectionWrapper(self):
+    with self.cached_session() as sess:
+      with variable_scope.variable_scope(
+          "root", initializer=init_ops.constant_initializer(0.5)):
+        x = array_ops.zeros([1, 3])
+        m = array_ops.zeros([1, 3])
+        cell = legacy_rnn_cell.OutputProjectionWrapper(
+            rnn_cell_impl.GRUCell(3), 2)
+        g, new_m = cell(x, m)
+        sess.run([variables.global_variables_initializer()])
+        res = sess.run([g, new_m], {
+            x.name: np.array([[1., 1., 1.]]),
+            m.name: np.array([[0.1, 0.1, 0.1]])
+        })
+        self.assertEqual(res[1].shape, (1, 3))
+        # The numbers in results were not calculated, this is just a smoke test.
+        self.assertAllClose(res[0], [[0.231907, 0.231907]])
+
+  def testInputProjectionWrapper(self):
+    with self.cached_session() as sess:
+      with variable_scope.variable_scope(
+          "root", initializer=init_ops.constant_initializer(0.5)):
+        x = array_ops.zeros([1, 2])
+        m = array_ops.zeros([1, 3])
+        cell = legacy_rnn_cell.InputProjectionWrapper(
+            rnn_cell_impl.GRUCell(3), num_proj=3)
+        g, new_m = cell(x, m)
+        sess.run([variables.global_variables_initializer()])
+        res = sess.run([g, new_m], {
+            x.name: np.array([[1., 1.]]),
+            m.name: np.array([[0.1, 0.1, 0.1]])
+        })
+        self.assertEqual(res[1].shape, (1, 3))
+        # The numbers in results were not calculated, this is just a smoke test.
+        self.assertAllClose(res[0], [[0.154605, 0.154605, 0.154605]])
+
+  def testEmbeddingWrapper(self):
+    with self.cached_session() as sess:
+      with variable_scope.variable_scope(
+          "root", initializer=init_ops.constant_initializer(0.5)):
+        x = array_ops.zeros([1, 1], dtype=dtypes.int32)
+        m = array_ops.zeros([1, 2])
+        embedding_cell = legacy_rnn_cell.EmbeddingWrapper(
+            rnn_cell_impl.GRUCell(2), embedding_classes=3, embedding_size=2)
+        self.assertEqual(embedding_cell.output_size, 2)
+        g, new_m = embedding_cell(x, m)
+        sess.run([variables.global_variables_initializer()])
+        res = sess.run([g, new_m], {
+            x.name: np.array([[1]]),
+            m.name: np.array([[0.1, 0.1]])
+        })
+        self.assertEqual(res[1].shape, (1, 2))
+        # The numbers in results were not calculated, this is just a smoke test.
+        self.assertAllClose(res[0], [[0.17139, 0.17139]])
+
+  def testEmbeddingWrapperWithDynamicRnn(self):
+    with self.cached_session() as sess:
+      with variable_scope.variable_scope("root"):
+        inputs = ops.convert_to_tensor([[[0], [0]]], dtype=dtypes.int64)
+        input_lengths = ops.convert_to_tensor([2], dtype=dtypes.int64)
+        embedding_cell = legacy_rnn_cell.EmbeddingWrapper(
+            rnn_cell_impl.BasicLSTMCell(1, state_is_tuple=True),
+            embedding_classes=1,
+            embedding_size=2)
+        outputs, _ = rnn.dynamic_rnn(
+            cell=embedding_cell,
+            inputs=inputs,
+            sequence_length=input_lengths,
+            dtype=dtypes.float32)
+        sess.run([variables.global_variables_initializer()])
+        # This will fail if output's dtype is inferred from input's.
+        sess.run(outputs)
+
+  def testSRUCell(self):
+    with self.cached_session() as sess:
+      with variable_scope.variable_scope(
+          "root", initializer=init_ops.constant_initializer(0.5)):
+        x = array_ops.zeros([1, 2])
+        m = array_ops.zeros([1, 2])
+        g, _ = contrib_rnn_cell.SRUCell(2)(x, m)
+        sess.run([variables.global_variables_initializer()])
+        res = sess.run([g], {
+            x.name: np.array([[1., 1.]]),
+            m.name: np.array([[0.1, 0.1]])
+        })
+        # Smoke test
+        self.assertAllClose(res[0], [[0.509682, 0.509682]])
+
+  def testSRUCellKerasRNN(self):
+    """Tests that SRUCell works with keras RNN layer."""
+    cell = contrib_rnn_cell.SRUCell(10)
+    seq_input = ops.convert_to_tensor(
+        np.random.rand(2, 3, 5), name="seq_input", dtype=dtypes.float32)
+    rnn_layer = keras_layers.RNN(cell=cell)
+    rnn_outputs_keras = rnn_layer(seq_input)
+    with self.cached_session() as sess:
+      sess.run([variables.global_variables_initializer()])
+      self.assertEqual(sess.run(rnn_outputs_keras).shape, (2, 10))
+
+  def testSRUCellBiasType(self):
+    """Tests that the bias' dtype is properly set."""
+    cell = contrib_rnn_cell.SRUCell(10)
+    cell.build((2, 3, 5))
+    self.assertEqual(cell._bias.dtype, dtypes.float32_ref)
+
+    cell = contrib_rnn_cell.SRUCell(10, dtype=dtypes.int32)
+    cell.build((2, 3, 5))
+    self.assertEqual(cell._bias.dtype, dtypes.int32_ref)
+
+    cell_input = ops.convert_to_tensor(
+        np.random.rand(2, 5), name="cell_input", dtype=dtypes.float16)
+    cell_state = ops.convert_to_tensor(
+        np.random.rand(2, 10), name="cell_state", dtype=dtypes.float16)
+    cell = contrib_rnn_cell.SRUCell(10)
+    cell(cell_input, [cell_state])
+    self.assertEqual(cell._bias.dtype, dtypes.float16_ref)
+
+  def testSRUCellWithDiffSize(self):
+    with self.cached_session() as sess:
+      with variable_scope.variable_scope(
+          "root", initializer=init_ops.constant_initializer(0.5)):
+        x = array_ops.zeros([1, 3])
+        m = array_ops.zeros([1, 2])
+        g, _ = contrib_rnn_cell.SRUCell(2)(x, m)
+        sess.run([variables.global_variables_initializer()])
+        res = sess.run([g], {
+            x.name: np.array([[1., 1., 1.]]),
+            m.name: np.array([[0.1, 0.1]])
+        })
+        # Smoke test
+        self.assertAllClose(res[0], [[0.55255556, 0.55255556]])
+
   def testCoupledInputForgetGateLSTMCell(self):
     with self.cached_session() as sess:
       num_units = 2
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index d98a6dd59c..78055ab085 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -2907,6 +2907,33 @@ cuda_py_test(
     xla_enable_strict_auto_jit = True,
 )
 
+cuda_py_test(
+    name = "rnn_cell_test",
+    size = "medium",
+    srcs = ["rnn_cell_test.py"],
+    additional_deps = [
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python/eager:context",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:control_flow_ops",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:gradients",
+        "//tensorflow/python:init_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:rnn",
+        "//tensorflow/python:rnn_cell",
+        "//tensorflow/python:tensor_array_ops",
+        "//tensorflow/python:util",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python:variables",
+    ],
+    shard_count = 10,
+)
+
 cuda_py_test(
     name = "scatter_ops_test",
     size = "medium",  # NOTE: This is not run by default.
diff --git a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py b/tensorflow/python/kernel_tests/rnn_cell_test.py
similarity index 68%
rename from tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py
rename to tensorflow/python/kernel_tests/rnn_cell_test.py
index ef372b947c..8492f10245 100644
--- a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py
+++ b/tensorflow/python/kernel_tests/rnn_cell_test.py
@@ -12,25 +12,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for rnn module."""
+"""Tests for RNN cells."""
 
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
 import itertools
+import os
 
+from absl.testing import parameterized
 import numpy as np
-from six.moves import xrange  # pylint: disable=redefined-builtin
 
-from tensorflow.contrib import rnn as rnn_lib
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops as ops_lib
+from tensorflow.python.framework import errors_impl
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import random_seed
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import test_util
+from tensorflow.python.keras import layers as keras_layers
+from tensorflow.python.layers import base as base_layer
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gradients_impl
@@ -38,16 +42,18 @@ from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import rnn
 from tensorflow.python.ops import rnn_cell
+from tensorflow.python.ops import rnn_cell_impl
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import tensor_array_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables as variables_lib
 from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging
+from tensorflow.python.training.checkpointable import util as checkpointable_utils
 from tensorflow.python.util import nest
 
 
-class Plus1RNNCell(rnn_lib.RNNCell):
+class Plus1RNNCell(rnn_cell.RNNCell):
   """RNN Cell generating (output, new_state) = (input + 1, state + 1)."""
 
   @property
@@ -62,7 +68,7 @@ class Plus1RNNCell(rnn_lib.RNNCell):
     return (input_ + 1, state + 1)
 
 
-class DummyMultiDimensionalLSTM(rnn_lib.RNNCell):
+class DummyMultiDimensionalLSTM(rnn_cell.RNNCell):
   """LSTM Cell generating (output, new_state) = (input + 1, state + 1).
 
   The input to this cell may have an arbitrary number of dimensions that follow
@@ -97,7 +103,7 @@ class DummyMultiDimensionalLSTM(rnn_lib.RNNCell):
     return (input_ + 1, (h + 1, c + 1))
 
 
-class NestedRNNCell(rnn_lib.RNNCell):
+class NestedRNNCell(rnn_cell.RNNCell):
   """RNN Cell generating (output, new_state) = (input + 1, state + 1).
 
   The input, output and state of this cell is a tuple of two tensors.
@@ -161,18 +167,19 @@ class TestStateSaverWithCounters(TestStateSaver):
   inherits from the TestStateSaver and adds the counters for calls of functions.
   """
 
+  @test_util.run_v1_only("b/124229375")
   def __init__(self, batch_size, state_size):
     super(TestStateSaverWithCounters, self).__init__(batch_size, state_size)
     self._num_state_calls = variables_lib.VariableV1(0)
     self._num_save_state_calls = variables_lib.VariableV1(0)
 
   def state(self, name):
-    with ops_lib.control_dependencies(
+    with ops.control_dependencies(
         [state_ops.assign_add(self._num_state_calls, 1)]):
       return super(TestStateSaverWithCounters, self).state(name)
 
   def save_state(self, name, state):
-    with ops_lib.control_dependencies([state_ops.assign_add(
+    with ops.control_dependencies([state_ops.assign_add(
         self._num_save_state_calls, 1)]):
       return super(TestStateSaverWithCounters, self).save_state(name, state)
 
@@ -191,12 +198,14 @@ class RNNTest(test.TestCase):
     self._seed = 23489
     np.random.seed(self._seed)
 
+  @test_util.run_v1_only("b/124229375")
   def testInvalidSequenceLengthShape(self):
     cell = Plus1RNNCell()
     inputs = [array_ops.placeholder(dtypes.float32, shape=(3, 4))]
     with self.assertRaisesRegexp(ValueError, "must be a vector"):
       rnn.static_rnn(cell, inputs, dtype=dtypes.float32, sequence_length=4)
 
+  @test_util.run_v1_only("b/124229375")
   def testRNN(self):
     cell = Plus1RNNCell()
     batch_size = 2
@@ -224,6 +233,7 @@ class RNNTest(test.TestCase):
                           max_length * np.ones(
                               (batch_size, input_size), dtype=np.float32))
 
+  @test_util.run_v1_only("b/124229375")
   def testDropout(self):
     cell = Plus1RNNCell()
     full_dropout_cell = rnn_cell.DropoutWrapper(
@@ -260,6 +270,7 @@ class RNNTest(test.TestCase):
       for d_v in full_dropout_values[:-1]:  # Add 1.0 to dropped_out (all zeros)
         self.assertAllClose(d_v, np.ones_like(input_value))
 
+  @test_util.run_v1_only("b/124229375")
   def testDynamicCalculation(self):
     cell = Plus1RNNCell()
     sequence_length = array_ops.placeholder(dtypes.int64)
@@ -310,7 +321,7 @@ class RNNTest(test.TestCase):
                                      1.0 * (2 + 1) * np.ones((input_size)))))
 
   def _testScope(self, factory, prefix="prefix", use_outer_scope=True):
-    with self.session(use_gpu=True, graph=ops_lib.Graph()):
+    with self.session(use_gpu=True, graph=ops.Graph()):
       if use_outer_scope:
         with variable_scope.variable_scope(prefix) as scope:
           factory(scope)
@@ -329,6 +340,7 @@ class RNNTest(test.TestCase):
         tf_logging.info(v.name)
       self.assertEqual(len(scope_vars), len(all_vars))
 
+  @test_util.run_v1_only("b/124229375")
   def testScope(self):
 
     def factory(scope):
@@ -358,21 +370,22 @@ class LSTMTest(test.TestCase):
     lstm = rnn_cell.LSTMCell(10)
     input_tensor = array_ops.ones([10, 50])
     lstm.build(input_tensor.get_shape())
-    self.assertEqual(lstm._bias.dtype, dtypes.float32_ref)
+    self.assertEqual(lstm._bias.dtype.base_dtype, dtypes.float32)
 
     # Explicitly pass dtype in constructor
     for dtype in [dtypes.float16, dtypes.float32, dtypes.float64]:
       lstm = rnn_cell.LSTMCell(10, dtype=dtype)
       input_tensor = array_ops.ones([10, 50])
       lstm.build(input_tensor.get_shape())
-      self.assertEqual(lstm._bias.dtype, dtype._as_ref)
+      self.assertEqual(lstm._bias.dtype.base_dtype, dtype)
 
+  @test_util.run_v1_only("b/124229375")
   def testNoProjNoSharding(self):
     num_units = 3
     input_size = 5
     batch_size = 2
     max_length = 8
-    with self.session(use_gpu=True, graph=ops_lib.Graph()) as sess:
+    with self.session(use_gpu=True, graph=ops.Graph()) as sess:
       initializer = init_ops.random_uniform_initializer(
           -0.01, 0.01, seed=self._seed)
       cell = rnn_cell.LSTMCell(
@@ -389,12 +402,13 @@ class LSTMTest(test.TestCase):
       input_value = np.random.randn(batch_size, input_size)
       sess.run(outputs, feed_dict={inputs[0]: input_value})
 
+  @test_util.run_v1_only("b/124229375")
   def testCellClipping(self):
     num_units = 3
     input_size = 5
     batch_size = 2
     max_length = 8
-    with self.session(use_gpu=True, graph=ops_lib.Graph()) as sess:
+    with self.session(use_gpu=True, graph=ops.Graph()) as sess:
       initializer = init_ops.random_uniform_initializer(
           -0.01, 0.01, seed=self._seed)
       cell = rnn_cell.LSTMCell(
@@ -419,12 +433,13 @@ class LSTMTest(test.TestCase):
       # if cell c is clipped to 0, tanh(c) = 0 => m==0
       self.assertAllEqual(value, np.zeros((batch_size, num_units)))
 
+  @test_util.run_v1_only("b/124229375")
   def testNoProjNoShardingSimpleStateSaver(self):
     num_units = 3
     input_size = 5
     batch_size = 2
     max_length = 8
-    with self.session(use_gpu=True, graph=ops_lib.Graph()) as sess:
+    with self.session(use_gpu=True, graph=ops.Graph()) as sess:
       initializer = init_ops.random_uniform_initializer(
           -0.01, 0.01, seed=self._seed)
       state_saver = TestStateSaver(batch_size, 2 * num_units)
@@ -452,12 +467,13 @@ class LSTMTest(test.TestCase):
           })
       self.assertAllEqual(last_state_value, saved_state_value)
 
+  @test_util.run_v1_only("b/124229375")
   def testNoProjNoShardingTupleStateSaver(self):
     num_units = 3
     input_size = 5
     batch_size = 2
     max_length = 8
-    with self.session(graph=ops_lib.Graph()) as sess:
+    with self.session(graph=ops.Graph()) as sess:
       initializer = init_ops.random_uniform_initializer(
           -0.01, 0.01, seed=self._seed)
       state_saver = TestStateSaver(batch_size, num_units)
@@ -486,12 +502,13 @@ class LSTMTest(test.TestCase):
       self.assertEqual(4, len(last_and_saved_states))
       self.assertAllEqual(last_and_saved_states[:2], last_and_saved_states[2:])
 
+  @test_util.run_v1_only("b/124229375")
   def testNoProjNoShardingNestedTupleStateSaver(self):
     num_units = 3
     input_size = 5
     batch_size = 2
     max_length = 8
-    with self.session(graph=ops_lib.Graph()) as sess:
+    with self.session(graph=ops.Graph()) as sess:
       initializer = init_ops.random_uniform_initializer(
           -0.01, 0.01, seed=self._seed)
       state_saver = TestStateSaver(
@@ -556,13 +573,14 @@ class LSTMTest(test.TestCase):
         self.assertAllEqual(last_states[i],
                             named_saved_states[flat_state_names[i]])
 
+  @test_util.run_v1_only("b/124229375")
   def testProjNoSharding(self):
     num_units = 3
     input_size = 5
     batch_size = 2
     num_proj = 4
     max_length = 8
-    with self.session(use_gpu=True, graph=ops_lib.Graph()) as sess:
+    with self.session(use_gpu=True, graph=ops.Graph()) as sess:
       initializer = init_ops.random_uniform_initializer(
           -0.01, 0.01, seed=self._seed)
       inputs = max_length * [
@@ -588,7 +606,7 @@ class LSTMTest(test.TestCase):
     num_proj = 4
     max_length = 8
     sequence_length = [4, 6]
-    with self.session(graph=ops_lib.Graph()) as sess:
+    with self.session(graph=ops.Graph()) as sess:
       initializer = init_ops.random_uniform_initializer(
           -0.01, 0.01, seed=self._seed)
       inputs = max_length * [
@@ -630,7 +648,7 @@ class LSTMTest(test.TestCase):
       self.assertEqual(len(outputs_notuple), len(inputs))
       self.assertEqual(len(outputs_tuple), len(inputs))
       self.assertTrue(isinstance(state_tuple, tuple))
-      self.assertTrue(isinstance(state_notuple, ops_lib.Tensor))
+      self.assertTrue(isinstance(state_notuple, ops.Tensor))
 
       variables_lib.global_variables_initializer().run()
       input_value = np.random.randn(batch_size, input_size)
@@ -651,6 +669,7 @@ class LSTMTest(test.TestCase):
       state_tuple_v = sess.run(state_tuple, feed_dict={inputs[0]: input_value})
       self.assertAllEqual(state_notuple_v, np.hstack(state_tuple_v))
 
+  @test_util.run_v1_only("b/124229375")
   def testProjSharding(self):
     num_units = 3
     input_size = 5
@@ -659,7 +678,7 @@ class LSTMTest(test.TestCase):
     num_proj_shards = 3
     num_unit_shards = 2
     max_length = 8
-    with self.session(use_gpu=True, graph=ops_lib.Graph()) as sess:
+    with self.session(use_gpu=True, graph=ops.Graph()) as sess:
       initializer = init_ops.random_uniform_initializer(
           -0.01, 0.01, seed=self._seed)
 
@@ -684,6 +703,7 @@ class LSTMTest(test.TestCase):
       input_value = np.random.randn(batch_size, input_size)
       sess.run(outputs, feed_dict={inputs[0]: input_value})
 
+  @test_util.run_v1_only("b/124229375")
   def testDoubleInput(self):
     num_units = 3
     input_size = 5
@@ -692,7 +712,7 @@ class LSTMTest(test.TestCase):
     num_proj_shards = 3
     num_unit_shards = 2
     max_length = 8
-    with self.session(use_gpu=True, graph=ops_lib.Graph()) as sess:
+    with self.session(use_gpu=True, graph=ops.Graph()) as sess:
       initializer = init_ops.random_uniform_initializer(-1, 1, seed=self._seed)
       inputs = max_length * [
           array_ops.placeholder(dtypes.float64, shape=(None, input_size))
@@ -720,6 +740,7 @@ class LSTMTest(test.TestCase):
       values = sess.run(outputs, feed_dict={inputs[0]: input_value})
       self.assertEqual(values[0].dtype, input_value.dtype)
 
+  @test_util.run_v1_only("b/124229375")
   def testShardNoShardEquivalentOutput(self):
     num_units = 3
     input_size = 5
@@ -728,7 +749,7 @@ class LSTMTest(test.TestCase):
     num_proj_shards = 3
     num_unit_shards = 2
     max_length = 8
-    with self.session(use_gpu=True, graph=ops_lib.Graph()) as sess:
+    with self.session(use_gpu=True, graph=ops.Graph()) as sess:
       inputs = max_length * [
           array_ops.placeholder(dtypes.float32, shape=(None, input_size))
       ]
@@ -774,6 +795,7 @@ class LSTMTest(test.TestCase):
       for (s_noshard, s_shard) in zip(state_values_noshard, state_values_shard):
         self.assertAllClose(s_noshard, s_shard, atol=1e-3)
 
+  @test_util.run_v1_only("b/124229375")
   def testDoubleInputWithDropoutAndDynamicCalculation(self):
     """Smoke test for using LSTM with doubles, dropout, dynamic calculation."""
 
@@ -784,7 +806,7 @@ class LSTMTest(test.TestCase):
     num_proj_shards = 3
     num_unit_shards = 2
     max_length = 8
-    with self.session(use_gpu=True, graph=ops_lib.Graph()) as sess:
+    with self.session(use_gpu=True, graph=ops.Graph()) as sess:
       sequence_length = array_ops.placeholder(dtypes.int64)
       initializer = init_ops.random_uniform_initializer(
           -0.01, 0.01, seed=self._seed)
@@ -828,13 +850,14 @@ class LSTMTest(test.TestCase):
       self.assertEqual(values[0].dtype, input_value.dtype)
       self.assertEqual(state_value[0].dtype, input_value.dtype)
 
+  @test_util.run_v1_only("b/124229375")
   def testSharingWeightsWithReuse(self):
     num_units = 3
     input_size = 5
     batch_size = 2
     num_proj = 4
     max_length = 8
-    with self.session(graph=ops_lib.Graph()) as sess:
+    with self.session(graph=ops.Graph()) as sess:
       initializer = init_ops.random_uniform_initializer(-1, 1, seed=self._seed)
       initializer_d = init_ops.random_uniform_initializer(
           -1, 1, seed=self._seed + 1)
@@ -878,13 +901,14 @@ class LSTMTest(test.TestCase):
         # Different weights used so outputs should be different.
         self.assertTrue(np.linalg.norm(o1 - o3) > 1e-6)
 
+  @test_util.run_v1_only("b/124229375")
   def testSharingWeightsWithDifferentNamescope(self):
     num_units = 3
     input_size = 5
     batch_size = 2
     num_proj = 4
     max_length = 8
-    with self.session(graph=ops_lib.Graph()) as sess:
+    with self.session(graph=ops.Graph()) as sess:
       initializer = init_ops.random_uniform_initializer(-1, 1, seed=self._seed)
       inputs = max_length * [
           array_ops.placeholder(dtypes.float32, shape=(None, input_size))
@@ -896,10 +920,10 @@ class LSTMTest(test.TestCase):
           initializer=initializer,
           state_is_tuple=False)
 
-      with ops_lib.name_scope("scope0"):
+      with ops.name_scope("scope0"):
         with variable_scope.variable_scope("share_scope"):
           outputs0, _ = rnn.static_rnn(cell, inputs, dtype=dtypes.float32)
-      with ops_lib.name_scope("scope1"):
+      with ops.name_scope("scope1"):
         with variable_scope.variable_scope("share_scope", reuse=True):
           outputs1, _ = rnn.static_rnn(cell, inputs, dtype=dtypes.float32)
 
@@ -915,6 +939,7 @@ class LSTMTest(test.TestCase):
       for out0, out1 in zip(outputs0_values, outputs1_values):
         self.assertAllEqual(out0, out1)
 
+  @test_util.run_v1_only("b/124229375")
   def testDynamicRNNAllowsUnknownTimeDimension(self):
     inputs = array_ops.placeholder(dtypes.float32, shape=[1, None, 20])
     cell = rnn_cell.GRUCell(30)
@@ -930,7 +955,7 @@ class LSTMTest(test.TestCase):
     max_length = 8
     sequence_length = [4, 6]
     in_graph_mode = not context.executing_eagerly()
-    with self.session(graph=ops_lib.Graph()) as sess:
+    with self.session(graph=ops.Graph()) as sess:
       initializer = init_ops.random_uniform_initializer(
           -0.01, 0.01, seed=self._seed)
       if in_graph_mode:
@@ -1006,7 +1031,7 @@ class LSTMTest(test.TestCase):
     max_length = 8
     sequence_length = [4, 6]
     in_graph_mode = not context.executing_eagerly()
-    with self.session(graph=ops_lib.Graph()) as sess:
+    with self.session(graph=ops.Graph()) as sess:
       initializer = init_ops.random_uniform_initializer(
           -0.01, 0.01, seed=self._seed)
       if in_graph_mode:
@@ -1117,7 +1142,7 @@ class LSTMTest(test.TestCase):
           state_is_tuple=False)
 
     ########### Step 1: Run static graph and generate readouts
-    with self.session(use_gpu=True, graph=ops_lib.Graph()) as sess:
+    with self.session(use_gpu=True, graph=ops.Graph()) as sess:
       if in_graph_mode:
         concat_inputs = array_ops.placeholder(
             dtypes.float32, shape=(time_steps, batch_size, input_size))
@@ -1154,8 +1179,8 @@ class LSTMTest(test.TestCase):
             for y in [outputs_static[0], outputs_static[-1], state_static]
         ])
         # Generate gradients of individual variables w.r.t. inputs
-        trainable_variables = ops_lib.get_collection(
-            ops_lib.GraphKeys.TRAINABLE_VARIABLES)
+        trainable_variables = ops.get_collection(
+            ops.GraphKeys.TRAINABLE_VARIABLES)
         assert len(trainable_variables) > 1, (
             "Count of trainable variables: %d" % len(trainable_variables))
         # pylint: disable=bad-builtin
@@ -1177,7 +1202,7 @@ class LSTMTest(test.TestCase):
             static_individual_variable_gradients, feed_dict=feeds)
 
     ########## Step 2: Run dynamic graph and generate readouts
-    with self.session(use_gpu=True, graph=ops_lib.Graph()) as sess:
+    with self.session(use_gpu=True, graph=ops.Graph()) as sess:
       if in_graph_mode:
         concat_inputs = array_ops.placeholder(
             dtypes.float32, shape=(time_steps, batch_size, input_size))
@@ -1225,8 +1250,8 @@ class LSTMTest(test.TestCase):
         ])
 
         # Generate gradients of individual variables w.r.t. inputs
-        trainable_variables = ops_lib.get_collection(
-            ops_lib.GraphKeys.TRAINABLE_VARIABLES)
+        trainable_variables = ops.get_collection(
+            ops.GraphKeys.TRAINABLE_VARIABLES)
         assert len(trainable_variables) > 1, (
             "Count of trainable variables: %d" % len(trainable_variables))
         dynamic_individual_variable_gradients = nest.flatten([
@@ -1337,7 +1362,7 @@ class BidirectionalRNNTest(test.TestCase):
     return input_value, inputs, outputs, state_fw, state_bw, sequence_length
 
   def _testBidirectionalRNN(self, use_shape):
-    with self.session(use_gpu=True, graph=ops_lib.Graph()) as sess:
+    with self.session(use_gpu=True, graph=ops.Graph()) as sess:
       input_value, inputs, outputs, state_fw, state_bw, sequence_length = (
           self._createBidirectionalRNN(use_shape, True))
       variables_lib.global_variables_initializer().run()
@@ -1384,7 +1409,7 @@ class BidirectionalRNNTest(test.TestCase):
       self.assertAllClose(s_fw, s_bw)
 
   def _testBidirectionalRNNWithoutSequenceLength(self, use_shape):
-    with self.session(use_gpu=True, graph=ops_lib.Graph()) as sess:
+    with self.session(use_gpu=True, graph=ops.Graph()) as sess:
       input_value, inputs, outputs, state_fw, state_bw, _ = (
           self._createBidirectionalRNN(use_shape, False))
       variables_lib.global_variables_initializer().run()
@@ -1402,11 +1427,11 @@ class BidirectionalRNNTest(test.TestCase):
       #
       # Both sequences in batch are length=8.  Check that the time=i
       # forward output is equal to time=8-1-i backward output
-      for i in xrange(8):
+      for i in range(8):
         self.assertEqual(out[i][0][0], out[8 - 1 - i][0][3])
         self.assertEqual(out[i][0][1], out[8 - 1 - i][0][4])
         self.assertEqual(out[i][0][2], out[8 - 1 - i][0][5])
-      for i in xrange(8):
+      for i in range(8):
         self.assertEqual(out[i][1][0], out[8 - 1 - i][1][3])
         self.assertEqual(out[i][1][1], out[8 - 1 - i][1][4])
         self.assertEqual(out[i][1][2], out[8 - 1 - i][1][5])
@@ -1414,10 +1439,12 @@ class BidirectionalRNNTest(test.TestCase):
       # exactly the same
       self.assertAllClose(s_fw, s_bw)
 
+  @test_util.run_v1_only("b/124229375")
   def testBidirectionalRNN(self):
     self._testBidirectionalRNN(use_shape=False)
     self._testBidirectionalRNN(use_shape=True)
 
+  @test_util.run_v1_only("b/124229375")
   def testBidirectionalRNNWithoutSequenceLength(self):
     self._testBidirectionalRNNWithoutSequenceLength(use_shape=False)
     self._testBidirectionalRNNWithoutSequenceLength(use_shape=True)
@@ -1472,7 +1499,7 @@ class BidirectionalRNNTest(test.TestCase):
 
   def _testBidirectionalDynamicRNN(self, use_shape, use_state_tuple,
                                    use_time_major, use_sequence_length):
-    with self.session(use_gpu=True, graph=ops_lib.Graph()) as sess:
+    with self.session(use_gpu=True, graph=ops.Graph()) as sess:
       input_value, inputs, outputs, state_fw, state_bw, sequence_length = (
           self._createBidirectionalDynamicRNN(
               use_shape, use_state_tuple, use_time_major, use_sequence_length))
@@ -1534,6 +1561,7 @@ class BidirectionalRNNTest(test.TestCase):
           self.assertAllEqual(out[t, :, 0:3], out[max_length - t - 1, :, 3:6])
         self.assertAllClose(s_fw, s_bw)
 
+  @test_util.run_v1_only("b/124229375")
   def testBidirectionalDynamicRNN(self):
     # Generate 2^5 option values
     # from [True, True, True, True, True] to [False, False, False, False, False]
@@ -1549,7 +1577,7 @@ class BidirectionalRNNTest(test.TestCase):
     # REMARKS: factory(scope) is a function accepting a scope
     #          as an argument, such scope can be None, a string
     #          or a VariableScope instance.
-    with self.session(use_gpu=True, graph=ops_lib.Graph()):
+    with self.session(use_gpu=True, graph=ops.Graph()):
       if use_outer_scope:
         with variable_scope.variable_scope(prefix) as scope:
           factory(scope)
@@ -1568,6 +1596,7 @@ class BidirectionalRNNTest(test.TestCase):
         tf_logging.info(v.name)
       self.assertEqual(len(scope_vars), len(all_vars))
 
+  @test_util.run_v1_only("b/124229375")
   def testBidirectionalRNNScope(self):
 
     def factory(scope):
@@ -1578,6 +1607,7 @@ class BidirectionalRNNTest(test.TestCase):
     self._testScope(factory, use_outer_scope=False)
     self._testScope(factory, prefix=None, use_outer_scope=False)
 
+  @test_util.run_v1_only("b/124229375")
   def testBidirectionalDynamicRNNScope(self):
 
     def get_factory(use_time_major):
@@ -1606,13 +1636,14 @@ class MultiDimensionalLSTMTest(test.TestCase):
     self._seed = 23489
     np.random.seed(self._seed)
 
+  @test_util.run_v1_only("b/124229375")
   def testMultiDimensionalLSTMAllRNNContainers(self):
     feature_dims = (3, 4, 5)
     input_size = feature_dims
     batch_size = 2
     max_length = 8
     sequence_length = [4, 6]
-    with self.session(graph=ops_lib.Graph()) as sess:
+    with self.session(graph=ops.Graph()) as sess:
       inputs = max_length * [
           array_ops.placeholder(dtypes.float32, shape=(None,) + input_size)
       ]
@@ -1717,13 +1748,14 @@ class NestedLSTMTest(test.TestCase):
     self._seed = 23489
     np.random.seed(self._seed)
 
+  @test_util.run_v1_only("b/124229375")
   def testNestedIOLSTMAllRNNContainers(self):
     input_size = 5
     batch_size = 2
     state_size = 6
     max_length = 8
     sequence_length = [4, 6]
-    with self.session(graph=ops_lib.Graph()) as sess:
+    with self.session(graph=ops.Graph()) as sess:
       state_saver = TestStateSaver(batch_size, state_size)
       single_input = (array_ops.placeholder(
           dtypes.float32, shape=(None, input_size)),
@@ -1868,7 +1900,7 @@ class StateSaverRNNTest(test.TestCase):
     batch_size = 2
     state_saver = TestStateSaver(batch_size, 2 * num_units)
 
-    with self.session(use_gpu=True, graph=ops_lib.Graph()):
+    with self.session(use_gpu=True, graph=ops.Graph()):
       if use_outer_scope:
         with variable_scope.variable_scope(prefix) as scope:
           self._factory(scope=scope, state_saver=state_saver)
@@ -1900,6 +1932,7 @@ class StateSaverRNNTest(test.TestCase):
     have influence on number of calls to save_state and state methods of
     state_saver object (the number of calls should be same.)
     """
+    self.skipTest("b/124196246 Breakage for sess.run([out, ...]): 2 != 1")
 
     num_units = 3
     batch_size = 2
@@ -1935,6 +1968,7 @@ class GRUTest(test.TestCase):
     self._seed = 23489
     np.random.seed(self._seed)
 
+  @test_util.run_v1_only("b/124229375")
   def testDynamic(self):
     time_steps = 8
     num_units = 3
@@ -1945,7 +1979,7 @@ class GRUTest(test.TestCase):
 
     sequence_length = np.random.randint(0, time_steps, size=batch_size)
 
-    with self.session(use_gpu=True, graph=ops_lib.Graph()) as sess:
+    with self.session(use_gpu=True, graph=ops.Graph()) as sess:
       concat_inputs = array_ops.placeholder(
           dtypes.float32, shape=(time_steps, batch_size, input_size))
 
@@ -1967,7 +2001,7 @@ class GRUTest(test.TestCase):
       sess.run([outputs_dynamic, state_dynamic], feed_dict=feeds)
 
   def _testScope(self, factory, prefix="prefix", use_outer_scope=True):
-    with self.session(use_gpu=True, graph=ops_lib.Graph()):
+    with self.session(use_gpu=True, graph=ops.Graph()):
       if use_outer_scope:
         with variable_scope.variable_scope(prefix) as scope:
           factory(scope)
@@ -1986,6 +2020,7 @@ class GRUTest(test.TestCase):
         tf_logging.info(v.name)
       self.assertEqual(len(scope_vars), len(all_vars))
 
+  @test_util.run_v1_only("b/124229375")
   def testDynamicScope(self):
     time_steps = 8
     num_units = 3
@@ -2016,8 +2051,9 @@ class RawRNNTest(test.TestCase):
     self._seed = 23489
     np.random.seed(self._seed)
 
+  @test_util.run_v1_only("b/124229375")
   def _testRawRNN(self, max_time):
-    with self.session(graph=ops_lib.Graph()) as sess:
+    with self.session(graph=ops.Graph()) as sess:
       batch_size = 16
       input_depth = 4
       num_units = 3
@@ -2115,6 +2151,7 @@ class RawRNNTest(test.TestCase):
         for i in range(1, len(gradients_val)):
           self.assertAllClose(gradients_dynamic_rnn_val[i], gradients_val[i])
 
+  @test_util.run_v1_only("b/124229375")
   def testRawRNNZeroLength(self):
     # NOTE: Because with 0 time steps, raw_rnn does not have shape
     # information about the input, it is impossible to perform
@@ -2125,8 +2162,9 @@ class RawRNNTest(test.TestCase):
   def testRawRNN(self):
     self._testRawRNN(max_time=10)
 
+  @test_util.run_v1_only("b/124229375")
   def testLoopState(self):
-    with self.session(graph=ops_lib.Graph()):
+    with self.session(graph=ops.Graph()):
       max_time = 10
       batch_size = 16
       input_depth = 4
@@ -2161,8 +2199,9 @@ class RawRNNTest(test.TestCase):
       loop_state = r[-1]
       self.assertEqual([10], loop_state.eval())
 
+  @test_util.run_v1_only("b/124229375")
   def testLoopStateWithTensorArray(self):
-    with self.session(graph=ops_lib.Graph()):
+    with self.session(graph=ops.Graph()):
       max_time = 4
       batch_size = 16
       input_depth = 4
@@ -2204,8 +2243,9 @@ class RawRNNTest(test.TestCase):
       loop_state = loop_state.stack()
       self.assertAllEqual([1, 2, 2 + 2, 4 + 3, 7 + 4], loop_state.eval())
 
+  @test_util.run_v1_only("b/124229375")
   def testEmitDifferentStructureThanCellOutput(self):
-    with self.session(graph=ops_lib.Graph()) as sess:
+    with self.session(graph=ops.Graph()) as sess:
       max_time = 10
       batch_size = 16
       input_depth = 4
@@ -2253,7 +2293,7 @@ class RawRNNTest(test.TestCase):
           np.ones((max_time, batch_size, 1), np.int64), output_vals[1])
 
   def _testScope(self, factory, prefix="prefix", use_outer_scope=True):
-    with self.session(use_gpu=True, graph=ops_lib.Graph()):
+    with self.session(use_gpu=True, graph=ops.Graph()):
       if use_outer_scope:
         with variable_scope.variable_scope(prefix) as scope:
           factory(scope)
@@ -2272,6 +2312,7 @@ class RawRNNTest(test.TestCase):
         tf_logging.info(v.name)
       self.assertEqual(len(scope_vars), len(all_vars))
 
+  @test_util.run_v1_only("b/124229375")
   def testRawRNNScope(self):
     max_time = 10
     batch_size = 16
@@ -2329,7 +2370,7 @@ class DeviceWrapperCell(rnn_cell.RNNCell):
 
   def __call__(self, input_, state, scope=None):
     if self._device is not None:
-      with ops_lib.device(self._device):
+      with ops.device(self._device):
         return self._cell(input_, state, scope=scope)
     else:
       return self._cell(input_, state, scope=scope)
@@ -2353,11 +2394,11 @@ class TensorArrayOnCorrectDeviceTest(test.TestCase):
     sequence_length = np.random.randint(0, time_steps, size=batch_size)
 
     if input_device is not None:
-      with ops_lib.device(input_device):
+      with ops.device(input_device):
         inputs = constant_op.constant(inputs)
 
     if rnn_device is not None:
-      with ops_lib.device(rnn_device):
+      with ops.device(rnn_device):
         outputs, _ = rnn.dynamic_rnn(
             gpu_cell,
             inputs,
@@ -2443,5 +2484,876 @@ class TensorArrayOnCorrectDeviceTest(test.TestCase):
     _assert_in("TensorArray", gpu_stats, cpu_stats)
 
 
+class RNNCellTest(test.TestCase, parameterized.TestCase):
+
+  @test_util.run_v1_only("b/124229375")
+  def testBasicRNNCell(self):
+    with self.cached_session() as sess:
+      with variable_scope.variable_scope(
+          "root", initializer=init_ops.constant_initializer(0.5)):
+        x = array_ops.zeros([1, 2])
+        m = array_ops.zeros([1, 2])
+        cell = rnn_cell_impl.BasicRNNCell(2)
+        g, _ = cell(x, m)
+        self.assertEqual([
+            "root/basic_rnn_cell/%s:0" % rnn_cell_impl._WEIGHTS_VARIABLE_NAME,
+            "root/basic_rnn_cell/%s:0" % rnn_cell_impl._BIAS_VARIABLE_NAME
+        ], [v.name for v in cell.trainable_variables])
+        self.assertFalse(cell.non_trainable_variables)
+        sess.run([variables_lib.global_variables_initializer()])
+        res = sess.run([g], {
+            x: np.array([[1., 1.]]),
+            m: np.array([[0.1, 0.1]])
+        })
+        self.assertEqual(res[0].shape, (1, 2))
+
+  @test_util.run_v1_only("b/124229375")
+  def testBasicRNNCellNotTrainable(self):
+    with self.cached_session() as sess:
+
+      def not_trainable_getter(getter, *args, **kwargs):
+        kwargs["trainable"] = False
+        return getter(*args, **kwargs)
+
+      with variable_scope.variable_scope(
+          "root",
+          initializer=init_ops.constant_initializer(0.5),
+          custom_getter=not_trainable_getter):
+        x = array_ops.zeros([1, 2])
+        m = array_ops.zeros([1, 2])
+        cell = rnn_cell_impl.BasicRNNCell(2)
+        g, _ = cell(x, m)
+        self.assertFalse(cell.trainable_variables)
+        self.assertEqual([
+            "root/basic_rnn_cell/%s:0" % rnn_cell_impl._WEIGHTS_VARIABLE_NAME,
+            "root/basic_rnn_cell/%s:0" % rnn_cell_impl._BIAS_VARIABLE_NAME
+        ], [v.name for v in cell.non_trainable_variables])
+        sess.run([variables_lib.global_variables_initializer()])
+        res = sess.run([g], {
+            x: np.array([[1., 1.]]),
+            m: np.array([[0.1, 0.1]])
+        })
+        self.assertEqual(res[0].shape, (1, 2))
+
+  @test_util.run_v1_only("b/124229375")
+  def testGRUCell(self):
+    with self.cached_session() as sess:
+      with variable_scope.variable_scope(
+          "root", initializer=init_ops.constant_initializer(0.5)):
+        x = array_ops.zeros([1, 2])
+        m = array_ops.zeros([1, 2])
+        g, _ = rnn_cell_impl.GRUCell(2)(x, m)
+        sess.run([variables_lib.global_variables_initializer()])
+        res = sess.run([g], {
+            x: np.array([[1., 1.]]),
+            m: np.array([[0.1, 0.1]])
+        })
+        # Smoke test
+        self.assertAllClose(res[0], [[0.175991, 0.175991]])
+      with variable_scope.variable_scope(
+          "other", initializer=init_ops.constant_initializer(0.5)):
+        # Test GRUCell with input_size != num_units.
+        x = array_ops.zeros([1, 3])
+        m = array_ops.zeros([1, 2])
+        g, _ = rnn_cell_impl.GRUCell(2)(x, m)
+        sess.run([variables_lib.global_variables_initializer()])
+        res = sess.run([g], {
+            x: np.array([[1., 1., 1.]]),
+            m: np.array([[0.1, 0.1]])
+        })
+        # Smoke test
+        self.assertAllClose(res[0], [[0.156736, 0.156736]])
+
+  @test_util.run_v1_only("b/124229375")
+  def testBasicLSTMCell(self):
+    for dtype in [dtypes.float16, dtypes.float32]:
+      np_dtype = dtype.as_numpy_dtype
+      with self.session(graph=ops.Graph()) as sess:
+        with variable_scope.variable_scope(
+            "root", initializer=init_ops.constant_initializer(0.5)):
+          x = array_ops.zeros([1, 2], dtype=dtype)
+          m = array_ops.zeros([1, 8], dtype=dtype)
+          cell = rnn_cell_impl.MultiRNNCell(
+              [
+                  rnn_cell_impl.BasicLSTMCell(2, state_is_tuple=False)
+                  for _ in range(2)
+              ],
+              state_is_tuple=False)
+          self.assertEqual(cell.dtype, None)
+          self.assertEqual("cell-0", cell._checkpoint_dependencies[0].name)
+          self.assertEqual("cell-1", cell._checkpoint_dependencies[1].name)
+          cell.get_config()  # Should not throw an error
+          g, out_m = cell(x, m)
+          # Layer infers the input type.
+          self.assertEqual(cell.dtype, dtype.name)
+          expected_variable_names = [
+              "root/multi_rnn_cell/cell_0/basic_lstm_cell/%s:0" %
+              rnn_cell_impl._WEIGHTS_VARIABLE_NAME,
+              "root/multi_rnn_cell/cell_0/basic_lstm_cell/%s:0" %
+              rnn_cell_impl._BIAS_VARIABLE_NAME,
+              "root/multi_rnn_cell/cell_1/basic_lstm_cell/%s:0" %
+              rnn_cell_impl._WEIGHTS_VARIABLE_NAME,
+              "root/multi_rnn_cell/cell_1/basic_lstm_cell/%s:0" %
+              rnn_cell_impl._BIAS_VARIABLE_NAME
+          ]
+          self.assertEqual(expected_variable_names,
+                           [v.name for v in cell.trainable_variables])
+          self.assertFalse(cell.non_trainable_variables)
+          sess.run([variables_lib.global_variables_initializer()])
+          res = sess.run([g, out_m], {
+              x: np.array([[1., 1.]]),
+              m: 0.1 * np.ones([1, 8])
+          })
+          self.assertEqual(len(res), 2)
+          variables = variables_lib.global_variables()
+          self.assertEqual(expected_variable_names, [v.name for v in variables])
+          # The numbers in results were not calculated, this is just a
+          # smoke test.
+          self.assertAllClose(res[0], np.array(
+              [[0.240, 0.240]], dtype=np_dtype), 1e-2)
+          expected_mem = np.array(
+              [[0.689, 0.689, 0.448, 0.448, 0.398, 0.398, 0.240, 0.240]],
+              dtype=np_dtype)
+          self.assertAllClose(res[1], expected_mem, 1e-2)
+        with variable_scope.variable_scope(
+            "other", initializer=init_ops.constant_initializer(0.5)):
+          # Test BasicLSTMCell with input_size != num_units.
+          x = array_ops.zeros([1, 3], dtype=dtype)
+          m = array_ops.zeros([1, 4], dtype=dtype)
+          g, out_m = rnn_cell_impl.BasicLSTMCell(2, state_is_tuple=False)(x, m)
+          sess.run([variables_lib.global_variables_initializer()])
+          res = sess.run(
+              [g, out_m], {
+                  x: np.array([[1., 1., 1.]], dtype=np_dtype),
+                  m: 0.1 * np.ones([1, 4], dtype=np_dtype)
+              })
+          self.assertEqual(len(res), 2)
+
+  @test_util.run_v1_only("b/124229375")
+  def testBasicLSTMCellDimension0Error(self):
+    """Tests that dimension 0 in both(x and m) shape must be equal."""
+    with self.cached_session() as sess:
+      with variable_scope.variable_scope(
+          "root", initializer=init_ops.constant_initializer(0.5)):
+        num_units = 2
+        state_size = num_units * 2
+        batch_size = 3
+        input_size = 4
+        x = array_ops.zeros([batch_size, input_size])
+        m = array_ops.zeros([batch_size - 1, state_size])
+        with self.assertRaises(ValueError):
+          g, out_m = rnn_cell_impl.BasicLSTMCell(
+              num_units, state_is_tuple=False)(x, m)
+          sess.run([variables_lib.global_variables_initializer()])
+          sess.run(
+              [g, out_m], {
+                  x: 1 * np.ones([batch_size, input_size]),
+                  m: 0.1 * np.ones([batch_size - 1, state_size])
+              })
+
+  def testBasicLSTMCellStateSizeError(self):
+    """Tests that state_size must be num_units * 2."""
+    with self.cached_session() as sess:
+      with variable_scope.variable_scope(
+          "root", initializer=init_ops.constant_initializer(0.5)):
+        num_units = 2
+        state_size = num_units * 3  # state_size must be num_units * 2
+        batch_size = 3
+        input_size = 4
+        x = array_ops.zeros([batch_size, input_size])
+        m = array_ops.zeros([batch_size, state_size])
+        with self.assertRaises((ValueError, errors_impl.InvalidArgumentError)):
+          g, out_m = rnn_cell_impl.BasicLSTMCell(
+              num_units, state_is_tuple=False)(x, m)
+          sess.run([variables_lib.global_variables_initializer()])
+          sess.run(
+              [g, out_m], {
+                  x: 1 * np.ones([batch_size, input_size]),
+                  m: 0.1 * np.ones([batch_size, state_size])
+              })
+
+  @test_util.run_v1_only("b/124229375")
+  def testBasicLSTMCellStateTupleType(self):
+    with self.cached_session():
+      with variable_scope.variable_scope(
+          "root", initializer=init_ops.constant_initializer(0.5)):
+        x = array_ops.zeros([1, 2])
+        m0 = (array_ops.zeros([1, 2]),) * 2
+        m1 = (array_ops.zeros([1, 2]),) * 2
+        cell = rnn_cell_impl.MultiRNNCell(
+            [rnn_cell_impl.BasicLSTMCell(2) for _ in range(2)],
+            state_is_tuple=True)
+        self.assertTrue(isinstance(cell.state_size, tuple))
+        self.assertTrue(
+            isinstance(cell.state_size[0], rnn_cell_impl.LSTMStateTuple))
+        self.assertTrue(
+            isinstance(cell.state_size[1], rnn_cell_impl.LSTMStateTuple))
+
+        # Pass in regular tuples
+        _, (out_m0, out_m1) = cell(x, (m0, m1))
+        self.assertTrue(isinstance(out_m0, rnn_cell_impl.LSTMStateTuple))
+        self.assertTrue(isinstance(out_m1, rnn_cell_impl.LSTMStateTuple))
+
+        # Pass in LSTMStateTuples
+        variable_scope.get_variable_scope().reuse_variables()
+        zero_state = cell.zero_state(1, dtypes.float32)
+        self.assertTrue(isinstance(zero_state, tuple))
+        self.assertTrue(isinstance(zero_state[0], rnn_cell_impl.LSTMStateTuple))
+        self.assertTrue(isinstance(zero_state[1], rnn_cell_impl.LSTMStateTuple))
+        _, (out_m0, out_m1) = cell(x, zero_state)
+        self.assertTrue(isinstance(out_m0, rnn_cell_impl.LSTMStateTuple))
+        self.assertTrue(isinstance(out_m1, rnn_cell_impl.LSTMStateTuple))
+
+  @test_util.run_v1_only("b/124229375")
+  def testBasicLSTMCellWithStateTuple(self):
+    with self.cached_session() as sess:
+      with variable_scope.variable_scope(
+          "root", initializer=init_ops.constant_initializer(0.5)):
+        x = array_ops.zeros([1, 2])
+        m0 = array_ops.zeros([1, 4])
+        m1 = array_ops.zeros([1, 4])
+        cell = rnn_cell_impl.MultiRNNCell(
+            [
+                rnn_cell_impl.BasicLSTMCell(2, state_is_tuple=False)
+                for _ in range(2)
+            ],
+            state_is_tuple=True)
+        g, (out_m0, out_m1) = cell(x, (m0, m1))
+        sess.run([variables_lib.global_variables_initializer()])
+        res = sess.run(
+            [g, out_m0, out_m1], {
+                x: np.array([[1., 1.]]),
+                m0: 0.1 * np.ones([1, 4]),
+                m1: 0.1 * np.ones([1, 4])
+            })
+        self.assertEqual(len(res), 3)
+        # The numbers in results were not calculated, this is just a smoke test.
+        # Note, however, these values should match the original
+        # version having state_is_tuple=False.
+        self.assertAllClose(res[0], [[0.24024698, 0.24024698]])
+        expected_mem0 = np.array(
+            [[0.68967271, 0.68967271, 0.44848421, 0.44848421]])
+        expected_mem1 = np.array(
+            [[0.39897051, 0.39897051, 0.24024698, 0.24024698]])
+        self.assertAllClose(res[1], expected_mem0)
+        self.assertAllClose(res[2], expected_mem1)
+
+  @test_util.run_v1_only("b/124229375")
+  def testLSTMCell(self):
+    with self.cached_session() as sess:
+      num_units = 8
+      num_proj = 6
+      state_size = num_units + num_proj
+      batch_size = 3
+      input_size = 2
+      with variable_scope.variable_scope(
+          "root", initializer=init_ops.constant_initializer(0.5)):
+        x = array_ops.zeros([batch_size, input_size])
+        m = array_ops.zeros([batch_size, state_size])
+        cell = rnn_cell_impl.LSTMCell(
+            num_units=num_units,
+            num_proj=num_proj,
+            forget_bias=1.0,
+            state_is_tuple=False)
+        output, state = cell(x, m)
+        sess.run([variables_lib.global_variables_initializer()])
+        res = sess.run(
+            [output, state], {
+                x: np.array([[1., 1.], [2., 2.], [3., 3.]]),
+                m: 0.1 * np.ones((batch_size, state_size))
+            })
+        self.assertEqual(len(res), 2)
+        # The numbers in results were not calculated, this is mostly just a
+        # smoke test.
+        self.assertEqual(res[0].shape, (batch_size, num_proj))
+        self.assertEqual(res[1].shape, (batch_size, state_size))
+        # Different inputs so different outputs and states
+        for i in range(1, batch_size):
+          self.assertTrue(
+              float(np.linalg.norm((res[0][0, :] - res[0][i, :]))) > 1e-6)
+          self.assertTrue(
+              float(np.linalg.norm((res[1][0, :] - res[1][i, :]))) > 1e-6)
+
+  @test_util.run_v1_only("b/124229375")
+  def testLSTMCellVariables(self):
+    with self.cached_session():
+      num_units = 8
+      num_proj = 6
+      state_size = num_units + num_proj
+      batch_size = 3
+      input_size = 2
+      with variable_scope.variable_scope(
+          "root", initializer=init_ops.constant_initializer(0.5)):
+        x = array_ops.zeros([batch_size, input_size])
+        m = array_ops.zeros([batch_size, state_size])
+        cell = rnn_cell_impl.LSTMCell(
+            num_units=num_units,
+            num_proj=num_proj,
+            forget_bias=1.0,
+            state_is_tuple=False)
+        cell(x, m)  # Execute to create variables
+      variables = variables_lib.global_variables()
+      self.assertEquals(variables[0].op.name, "root/lstm_cell/kernel")
+      self.assertEquals(variables[1].op.name, "root/lstm_cell/bias")
+      self.assertEquals(variables[2].op.name,
+                        "root/lstm_cell/projection/kernel")
+
+  @test_util.run_in_graph_and_eager_modes
+  def testWrapperCheckpointing(self):
+    for wrapper_type in [
+        rnn_cell_impl.DropoutWrapper,
+        rnn_cell_impl.ResidualWrapper,
+        lambda cell: rnn_cell_impl.MultiRNNCell([cell])]:
+      cell = rnn_cell_impl.BasicRNNCell(1)
+      wrapper = wrapper_type(cell)
+      wrapper(array_ops.ones([1, 1]),
+              state=wrapper.zero_state(batch_size=1, dtype=dtypes.float32))
+      self.evaluate([v.initializer for v in cell.variables])
+      checkpoint = checkpointable_utils.Checkpoint(wrapper=wrapper)
+      prefix = os.path.join(self.get_temp_dir(), "ckpt")
+      self.evaluate(cell._bias.assign([40.]))
+      save_path = checkpoint.save(prefix)
+      self.evaluate(cell._bias.assign([0.]))
+      checkpoint.restore(save_path).assert_consumed().run_restore_ops()
+      self.assertAllEqual([40.], self.evaluate(cell._bias))
+
+  @parameterized.parameters(
+      [rnn_cell_impl.ResidualWrapper, rnn_cell_impl.ResidualWrapperV2])
+  @test_util.run_in_graph_and_eager_modes
+  def testResidualWrapper(self, wrapper_type):
+    x = ops.convert_to_tensor(np.array([[1., 1., 1.]]))
+    m = ops.convert_to_tensor(np.array([[0.1, 0.1, 0.1]]))
+    base_cell = rnn_cell_impl.GRUCell(
+        3, kernel_initializer=init_ops.constant_initializer(0.5),
+        bias_initializer=init_ops.constant_initializer(0.5))
+    g, m_new = base_cell(x, m)
+    wrapper_object = wrapper_type(base_cell)
+    (name, dep), = wrapper_object._checkpoint_dependencies
+    wrapper_object.get_config()  # Should not throw an error
+    self.assertIs(dep, base_cell)
+    self.assertEqual("cell", name)
+
+    g_res, m_new_res = wrapper_object(x, m)
+    self.evaluate([variables_lib.global_variables_initializer()])
+    res = self.evaluate([g, g_res, m_new, m_new_res])
+    # Residual connections
+    self.assertAllClose(res[1], res[0] + [1., 1., 1.])
+    # States are left untouched
+    self.assertAllClose(res[2], res[3])
+
+  @parameterized.parameters(
+      [rnn_cell_impl.ResidualWrapper, rnn_cell_impl.ResidualWrapperV2])
+  @test_util.run_in_graph_and_eager_modes
+  def testResidualWrapperWithSlice(self, wrapper_type):
+    x = ops.convert_to_tensor(np.array([[1., 1., 1., 1., 1.]]))
+    m = ops.convert_to_tensor(np.array([[0.1, 0.1, 0.1]]))
+    base_cell = rnn_cell_impl.GRUCell(
+        3, kernel_initializer=init_ops.constant_initializer(0.5),
+        bias_initializer=init_ops.constant_initializer(0.5))
+    g, m_new = base_cell(x, m)
+
+    def residual_with_slice_fn(inp, out):
+      inp_sliced = array_ops.slice(inp, [0, 0], [-1, 3])
+      return inp_sliced + out
+
+    g_res, m_new_res = wrapper_type(
+        base_cell, residual_with_slice_fn)(x, m)
+    self.evaluate([variables_lib.global_variables_initializer()])
+    res_g, res_g_res, res_m_new, res_m_new_res = self.evaluate(
+        [g, g_res, m_new, m_new_res])
+    # Residual connections
+    self.assertAllClose(res_g_res, res_g + [1., 1., 1.])
+    # States are left untouched
+    self.assertAllClose(res_m_new, res_m_new_res)
+
+  @test_util.run_v1_only("b/124229375")
+  def testDeviceWrapper(self):
+    with variable_scope.variable_scope(
+        "root", initializer=init_ops.constant_initializer(0.5)):
+      x = array_ops.zeros([1, 3])
+      m = array_ops.zeros([1, 3])
+      wrapped = rnn_cell_impl.GRUCell(3)
+      cell = rnn_cell_impl.DeviceWrapper(wrapped, "/cpu:14159")
+      (name, dep), = cell._checkpoint_dependencies
+      cell.get_config()  # Should not throw an error
+      self.assertIs(dep, wrapped)
+      self.assertEqual("cell", name)
+
+      outputs, _ = cell(x, m)
+      self.assertTrue("cpu:14159" in outputs.device.lower())
+
+  def _retrieve_cpu_gpu_stats(self, run_metadata):
+    cpu_stats = None
+    gpu_stats = None
+    step_stats = run_metadata.step_stats
+    for ds in step_stats.dev_stats:
+      if "cpu:0" in ds.device[-5:].lower():
+        cpu_stats = ds.node_stats
+      if "gpu:0" == ds.device[-5:].lower():
+        gpu_stats = ds.node_stats
+    return cpu_stats, gpu_stats
+
+  def testDeviceWrapperDynamicExecutionNodesAreAllProperlyLocated(self):
+    if not test.is_gpu_available():
+      # Can't perform this test w/o a GPU
+      return
+
+    gpu_dev = test.gpu_device_name()
+    with self.session(use_gpu=True) as sess:
+      with variable_scope.variable_scope(
+          "root", initializer=init_ops.constant_initializer(0.5)):
+        x = array_ops.zeros([1, 1, 3])
+        cell = rnn_cell_impl.DeviceWrapper(rnn_cell_impl.GRUCell(3), gpu_dev)
+        with ops.device("/cpu:0"):
+          outputs, _ = rnn.dynamic_rnn(
+              cell=cell, inputs=x, dtype=dtypes.float32)
+        run_metadata = config_pb2.RunMetadata()
+        opts = config_pb2.RunOptions(
+            trace_level=config_pb2.RunOptions.FULL_TRACE)
+
+        sess.run([variables_lib.global_variables_initializer()])
+        _ = sess.run(outputs, options=opts, run_metadata=run_metadata)
+
+      cpu_stats, gpu_stats = self._retrieve_cpu_gpu_stats(run_metadata)
+      self.assertFalse([s for s in cpu_stats if "gru_cell" in s.node_name])
+      self.assertTrue([s for s in gpu_stats if "gru_cell" in s.node_name])
+
+  @test_util.run_v1_only("b/124229375")
+  def testMultiRNNCell(self):
+    with self.cached_session() as sess:
+      with variable_scope.variable_scope(
+          "root", initializer=init_ops.constant_initializer(0.5)):
+        x = array_ops.zeros([1, 2])
+        m = array_ops.zeros([1, 4])
+        multi_rnn_cell = rnn_cell_impl.MultiRNNCell(
+            [rnn_cell_impl.GRUCell(2) for _ in range(2)],
+            state_is_tuple=False)
+        _, ml = multi_rnn_cell(x, m)
+        sess.run([variables_lib.global_variables_initializer()])
+        res = sess.run(ml, {
+            x: np.array([[1., 1.]]),
+            m: np.array([[0.1, 0.1, 0.1, 0.1]])
+        })
+        # The numbers in results were not calculated, this is just a smoke test.
+        self.assertAllClose(res, [[0.175991, 0.175991, 0.13248, 0.13248]])
+        self.assertEqual(len(multi_rnn_cell.weights), 2 * 4)
+        self.assertTrue(
+            [x.dtype == dtypes.float32 for x in multi_rnn_cell.weights])
+
+  @test_util.run_v1_only("b/124229375")
+  def testMultiRNNCellWithStateTuple(self):
+    with self.cached_session() as sess:
+      with variable_scope.variable_scope(
+          "root", initializer=init_ops.constant_initializer(0.5)):
+        x = array_ops.zeros([1, 2])
+        m_bad = array_ops.zeros([1, 4])
+        m_good = (array_ops.zeros([1, 2]), array_ops.zeros([1, 2]))
+
+        # Test incorrectness of state
+        with self.assertRaisesRegexp(ValueError, "Expected state .* a tuple"):
+          rnn_cell_impl.MultiRNNCell(
+              [rnn_cell_impl.GRUCell(2) for _ in range(2)],
+              state_is_tuple=True)(x, m_bad)
+
+        _, ml = rnn_cell_impl.MultiRNNCell(
+            [rnn_cell_impl.GRUCell(2) for _ in range(2)],
+            state_is_tuple=True)(x, m_good)
+
+        sess.run([variables_lib.global_variables_initializer()])
+        res = sess.run(
+            ml, {
+                x: np.array([[1., 1.]]),
+                m_good[0]: np.array([[0.1, 0.1]]),
+                m_good[1]: np.array([[0.1, 0.1]])
+            })
+
+        # The numbers in results were not calculated, this is just a
+        # smoke test.  However, these numbers should match those of
+        # the test testMultiRNNCell.
+        self.assertAllClose(res[0], [[0.175991, 0.175991]])
+        self.assertAllClose(res[1], [[0.13248, 0.13248]])
+
+  @parameterized.parameters(
+      [[rnn_cell_impl.DropoutWrapper, rnn_cell_impl.DropoutWrapperV2],
+       [rnn_cell_impl.ResidualWrapper, rnn_cell_impl.ResidualWrapperV2]])
+  @test_util.run_in_graph_and_eager_modes
+  def testWrapperKerasStyle(self, wrapper, wrapper_v2):
+    """Tests if wrapper cell is instantiated in keras style scope."""
+    wrapped_cell_v2 = wrapper_v2(rnn_cell_impl.BasicRNNCell(1))
+    self.assertTrue(wrapped_cell_v2._keras_style)
+
+    wrapped_cell = wrapper(rnn_cell_impl.BasicRNNCell(1))
+    self.assertFalse(wrapped_cell._keras_style)
+
+  @parameterized.parameters(
+      [rnn_cell_impl.DropoutWrapperV2, rnn_cell_impl.ResidualWrapperV2])
+  @test_util.run_in_graph_and_eager_modes
+  def testWrapperV2VariableNames(self, wrapper):
+    """Tests that variables names do not depend on wrapper in RNN layer."""
+
+    def _rnn_input(apply_wrapper, name):
+      """Creates a RNN layer with/without wrapper and returns built rnn cell."""
+      with base_layer.keras_style_scope():
+        base_cell = rnn_cell_impl.MultiRNNCell(
+            [rnn_cell_impl.BasicRNNCell(1, name="basic_rnn_cell")
+             for _ in range(2)])
+      if apply_wrapper:
+        rnn_cell = wrapper(base_cell)
+      else:
+        rnn_cell = base_cell
+      rnn_layer = keras_layers.RNN(rnn_cell, name=name)
+      inputs = ops.convert_to_tensor([[[1]]], dtype=dtypes.float32)
+      _ = rnn_layer(inputs)
+      return base_cell._cells[0]
+
+    rnn_1 = _rnn_input(True, name="rnn_0")
+    rnn_2 = _rnn_input(False, name="rnn_1")
+
+    for i, cell in enumerate([rnn_1, rnn_2]):
+      var_prefix = "rnn_{}/cell_0/basic_rnn_cell/".format(i)
+      self.assertCountEqual([v.name for v in cell.weights],
+                            (var_prefix + "kernel:0", var_prefix + "bias:0"))
+
+  @parameterized.parameters(
+      [rnn_cell_impl.DropoutWrapperV2, rnn_cell_impl.ResidualWrapperV2])
+  @test_util.run_in_graph_and_eager_modes
+  def testWrapperWeights(self, wrapper):
+    """Tests that wrapper weights contain wrapped cells weights."""
+
+    with base_layer.keras_style_scope():
+      base_cell = rnn_cell_impl.BasicRNNCell(1, name="basic_rnn_cell")
+    rnn_cell = wrapper(base_cell)
+    rnn_layer = keras_layers.RNN(rnn_cell)
+    inputs = ops.convert_to_tensor([[[1]]], dtype=dtypes.float32)
+    rnn_layer(inputs)
+
+    expected_weights = ["rnn/" + var for var in ("kernel:0", "bias:0")]
+    self.assertEqual(len(rnn_cell.weights), 2)
+    self.assertCountEqual([v.name for v in rnn_cell.weights], expected_weights)
+    self.assertCountEqual([v.name for v in rnn_cell.trainable_variables],
+                          expected_weights)
+    self.assertCountEqual([v.name for v in rnn_cell.non_trainable_variables],
+                          [])
+    self.assertCountEqual([v.name for v in rnn_cell._cell.weights],
+                          expected_weights)
+
+  @parameterized.parameters(
+      [rnn_cell_impl.DropoutWrapperV2, rnn_cell_impl.ResidualWrapperV2])
+  @test_util.run_in_graph_and_eager_modes
+  def testWrapperV2Caller(self, wrapper):
+    """Tests that wrapper V2 is using the LayerRNNCell's caller."""
+
+    with base_layer.keras_style_scope():
+      base_cell = rnn_cell_impl.MultiRNNCell(
+          [rnn_cell_impl.BasicRNNCell(1) for _ in range(2)])
+    rnn_cell = wrapper(base_cell)
+    inputs = ops.convert_to_tensor([[1]], dtype=dtypes.float32)
+    state = ops.convert_to_tensor([[1]], dtype=dtypes.float32)
+    _ = rnn_cell(inputs, [state, state])
+    weights = base_cell._cells[0].weights
+    self.assertLen(weights, expected_len=2)
+    self.assertTrue(all(["_wrapper" in v.name for v in weights]))
+
+  @parameterized.parameters(
+      [rnn_cell_impl.DropoutWrapperV2, rnn_cell_impl.ResidualWrapperV2])
+  @test_util.run_in_graph_and_eager_modes
+  def testWrapperV2Build(self, wrapper):
+    cell = rnn_cell_impl.LSTMCell(10)
+    wrapper = wrapper(cell)
+    wrapper.build((1,))
+    self.assertTrue(cell.built)
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class DropoutWrapperTest(test.TestCase, parameterized.TestCase):
+
+  def _testDropoutWrapper(self,
+                          batch_size=None,
+                          time_steps=None,
+                          parallel_iterations=None,
+                          wrapper_type=None,
+                          scope="root",
+                          **kwargs):
+    if batch_size is None and time_steps is None:
+      # 2 time steps, batch size 1, depth 3
+      batch_size = 1
+      time_steps = 2
+      x = constant_op.constant(
+          [[[2., 2., 2.]], [[1., 1., 1.]]], dtype=dtypes.float32)
+      m = rnn_cell_impl.LSTMStateTuple(
+          *[constant_op.constant([[0.1, 0.1, 0.1]], dtype=dtypes.float32)] * 2)
+    else:
+      x = constant_op.constant(
+          np.random.randn(time_steps, batch_size, 3).astype(np.float32))
+      m = rnn_cell_impl.LSTMStateTuple(*[
+          constant_op.
+          constant([[0.1, 0.1, 0.1]] * batch_size, dtype=dtypes.float32)] * 2)
+    outputs, final_state = rnn.dynamic_rnn(
+        cell=wrapper_type(
+            rnn_cell_impl.LSTMCell(
+                3, initializer=init_ops.constant_initializer(0.5)),
+            dtype=x.dtype, **kwargs),
+        time_major=True,
+        parallel_iterations=parallel_iterations,
+        inputs=x,
+        initial_state=m,
+        scope=scope)
+    self.evaluate([variables_lib.global_variables_initializer()])
+    res = self.evaluate([outputs, final_state])
+    self.assertEqual(res[0].shape, (time_steps, batch_size, 3))
+    self.assertEqual(res[1].c.shape, (batch_size, 3))
+    self.assertEqual(res[1].h.shape, (batch_size, 3))
+    return res
+
+  @parameterized.parameters(
+      [rnn_cell_impl.DropoutWrapper, rnn_cell_impl.DropoutWrapperV2])
+  def testDropoutWrapperProperties(self, wrapper_type):
+    cell = rnn_cell_impl.BasicRNNCell(10)
+    wrapper = wrapper_type(cell)
+    # Github issue 15810
+    self.assertEqual(wrapper.wrapped_cell, cell)
+    self.assertEqual(wrapper.state_size, 10)
+    self.assertEqual(wrapper.output_size, 10)
+
+  @parameterized.parameters(
+      [rnn_cell_impl.DropoutWrapper, rnn_cell_impl.DropoutWrapperV2])
+  def testDropoutWrapperZeroState(self, wrapper_type):
+    class _Cell(rnn_cell_impl.BasicRNNCell):
+
+      def zero_state(self, batch_size=None, dtype=None):
+        return "wrapped_cell_zero_state"
+    wrapper = wrapper_type(_Cell(10))
+    self.assertEqual(wrapper.zero_state(10, dtypes.float32),
+                     "wrapped_cell_zero_state")
+
+  @parameterized.parameters(
+      [rnn_cell_impl.DropoutWrapper, rnn_cell_impl.DropoutWrapperV2])
+  def testDropoutWrapperKeepAllConstantInput(self, wrapper_type):
+    keep = array_ops.ones([])
+    res = self._testDropoutWrapper(
+        input_keep_prob=keep, output_keep_prob=keep, state_keep_prob=keep,
+        wrapper_type=wrapper_type)
+    true_full_output = np.array(
+        [[[0.751109, 0.751109, 0.751109]], [[0.895509, 0.895509, 0.895509]]],
+        dtype=np.float32)
+    true_full_final_c = np.array(
+        [[1.949385, 1.949385, 1.949385]], dtype=np.float32)
+    self.assertAllClose(true_full_output, res[0])
+    self.assertAllClose(true_full_output[1], res[1].h)
+    self.assertAllClose(true_full_final_c, res[1].c)
+
+  @parameterized.parameters(
+      [rnn_cell_impl.DropoutWrapper, rnn_cell_impl.DropoutWrapperV2])
+  def testDropoutWrapperKeepAll(self, wrapper_type):
+    keep = variable_scope.get_variable("all", initializer=1.0)
+    res = self._testDropoutWrapper(
+        input_keep_prob=keep, output_keep_prob=keep, state_keep_prob=keep,
+        wrapper_type=wrapper_type)
+    true_full_output = np.array(
+        [[[0.751109, 0.751109, 0.751109]], [[0.895509, 0.895509, 0.895509]]],
+        dtype=np.float32)
+    true_full_final_c = np.array(
+        [[1.949385, 1.949385, 1.949385]], dtype=np.float32)
+    self.assertAllClose(true_full_output, res[0])
+    self.assertAllClose(true_full_output[1], res[1].h)
+    self.assertAllClose(true_full_final_c, res[1].c)
+
+  @parameterized.parameters(
+      [rnn_cell_impl.DropoutWrapper, rnn_cell_impl.DropoutWrapperV2])
+  def testDropoutWrapperWithSeed(self, wrapper_type):
+    keep_some = 0.5
+    random_seed.set_random_seed(2)
+    ## Use parallel_iterations = 1 in both calls to
+    ## _testDropoutWrapper to ensure the (per-time step) dropout is
+    ## consistent across both calls.  Otherwise the seed may not end
+    ## up being munged consistently across both graphs.
+    res_standard_1 = self._testDropoutWrapper(
+        input_keep_prob=keep_some,
+        output_keep_prob=keep_some,
+        state_keep_prob=keep_some,
+        seed=10,
+        parallel_iterations=1,
+        wrapper_type=wrapper_type,
+        scope="root_1")
+    random_seed.set_random_seed(2)
+    res_standard_2 = self._testDropoutWrapper(
+        input_keep_prob=keep_some,
+        output_keep_prob=keep_some,
+        state_keep_prob=keep_some,
+        seed=10,
+        parallel_iterations=1,
+        wrapper_type=wrapper_type,
+        scope="root_2")
+    self.assertAllClose(res_standard_1[0], res_standard_2[0])
+    self.assertAllClose(res_standard_1[1].c, res_standard_2[1].c)
+    self.assertAllClose(res_standard_1[1].h, res_standard_2[1].h)
+
+  @parameterized.parameters(
+      [rnn_cell_impl.DropoutWrapper, rnn_cell_impl.DropoutWrapperV2])
+  def testDropoutWrapperKeepNoOutput(self, wrapper_type):
+    keep_all = variable_scope.get_variable("all", initializer=1.0)
+    keep_none = variable_scope.get_variable("none", initializer=1e-6)
+    res = self._testDropoutWrapper(
+        input_keep_prob=keep_all,
+        output_keep_prob=keep_none,
+        state_keep_prob=keep_all,
+        wrapper_type=wrapper_type)
+    true_full_output = np.array(
+        [[[0.751109, 0.751109, 0.751109]], [[0.895509, 0.895509, 0.895509]]],
+        dtype=np.float32)
+    true_full_final_c = np.array(
+        [[1.949385, 1.949385, 1.949385]], dtype=np.float32)
+    self.assertAllClose(np.zeros(res[0].shape), res[0])
+    self.assertAllClose(true_full_output[1], res[1].h)
+    self.assertAllClose(true_full_final_c, res[1].c)
+
+  @parameterized.parameters(
+      [rnn_cell_impl.DropoutWrapper, rnn_cell_impl.DropoutWrapperV2])
+  def testDropoutWrapperKeepNoStateExceptLSTMCellMemory(self, wrapper_type):
+    keep_all = variable_scope.get_variable("all", initializer=1.0)
+    keep_none = variable_scope.get_variable("none", initializer=1e-6)
+    # Even though we dropout state, by default DropoutWrapper never
+    # drops out the memory ("c") term of an LSTMStateTuple.
+    res = self._testDropoutWrapper(
+        input_keep_prob=keep_all,
+        output_keep_prob=keep_all,
+        state_keep_prob=keep_none,
+        wrapper_type=wrapper_type)
+    true_c_state = np.array([[1.713925, 1.713925, 1.713925]], dtype=np.float32)
+    true_full_output = np.array(
+        [[[0.751109, 0.751109, 0.751109]], [[0.895509, 0.895509, 0.895509]]],
+        dtype=np.float32)
+    self.assertAllClose(true_full_output[0], res[0][0])
+    # Second output is modified by zero input state
+    self.assertGreater(np.linalg.norm(true_full_output[1] - res[0][1]), 1e-4)
+    # h state has been set to zero
+    self.assertAllClose(np.zeros(res[1].h.shape), res[1].h)
+    # c state of an LSTMStateTuple is NEVER modified.
+    self.assertAllClose(true_c_state, res[1].c)
+
+  @parameterized.parameters(
+      [rnn_cell_impl.DropoutWrapper, rnn_cell_impl.DropoutWrapperV2])
+  def testDropoutWrapperKeepNoInput(self, wrapper_type):
+    keep_all = variable_scope.get_variable("all", initializer=1.0)
+    keep_none = variable_scope.get_variable("none", initializer=1e-6)
+    true_full_output = np.array(
+        [[[0.751109, 0.751109, 0.751109]], [[0.895509, 0.895509, 0.895509]]],
+        dtype=np.float32)
+    true_full_final_c = np.array(
+        [[1.949385, 1.949385, 1.949385]], dtype=np.float32)
+    # All outputs are different because inputs are zeroed out
+    res = self._testDropoutWrapper(
+        input_keep_prob=keep_none,
+        output_keep_prob=keep_all,
+        state_keep_prob=keep_all,
+        wrapper_type=wrapper_type)
+    self.assertGreater(np.linalg.norm(res[0] - true_full_output), 1e-4)
+    self.assertGreater(np.linalg.norm(res[1].h - true_full_output[1]), 1e-4)
+    self.assertGreater(np.linalg.norm(res[1].c - true_full_final_c), 1e-4)
+
+  @parameterized.parameters(
+      [rnn_cell_impl.DropoutWrapper, rnn_cell_impl.DropoutWrapperV2])
+  def testDropoutWrapperRecurrentOutput(self, wrapper_type):
+    keep_some = 0.8
+    keep_all = variable_scope.get_variable("all", initializer=1.0)
+    res = self._testDropoutWrapper(
+        input_keep_prob=keep_all,
+        output_keep_prob=keep_some,
+        state_keep_prob=keep_all,
+        variational_recurrent=True,
+        wrapper_type=wrapper_type,
+        input_size=3,
+        batch_size=5,
+        time_steps=7)
+    # Ensure the same dropout pattern for all time steps
+    output_mask = np.abs(res[0]) > 1e-6
+    for m in output_mask[1:]:
+      self.assertAllClose(output_mask[0], m)
+
+  @parameterized.parameters(
+      [rnn_cell_impl.DropoutWrapper, rnn_cell_impl.DropoutWrapperV2])
+  def testDropoutWrapperRecurrentStateInputAndOutput(self, wrapper_type):
+    keep_some = 0.9
+    res = self._testDropoutWrapper(
+        input_keep_prob=keep_some,
+        output_keep_prob=keep_some,
+        state_keep_prob=keep_some,
+        variational_recurrent=True,
+        wrapper_type=wrapper_type,
+        input_size=3,
+        batch_size=5,
+        time_steps=7)
+
+    # Smoke test for the state/input masks.
+    output_mask = np.abs(res[0]) > 1e-6
+    for time_step in output_mask:
+      # Ensure the same dropout output pattern for all time steps
+      self.assertAllClose(output_mask[0], time_step)
+      for batch_entry in time_step:
+        # Assert all batch entries get the same mask
+        self.assertAllClose(batch_entry, time_step[0])
+
+    # For state, ensure all batch entries have the same mask
+    state_c_mask = np.abs(res[1].c) > 1e-6
+    state_h_mask = np.abs(res[1].h) > 1e-6
+    for batch_entry in state_c_mask:
+      self.assertAllClose(batch_entry, state_c_mask[0])
+    for batch_entry in state_h_mask:
+      self.assertAllClose(batch_entry, state_h_mask[0])
+
+  @parameterized.parameters(
+      [rnn_cell_impl.DropoutWrapper, rnn_cell_impl.DropoutWrapperV2])
+  def testDropoutWrapperRecurrentStateInputAndOutputWithSeed(
+      self, wrapper_type):
+    keep_some = 0.9
+    random_seed.set_random_seed(2347)
+    np.random.seed(23487)
+    res0 = self._testDropoutWrapper(
+        input_keep_prob=keep_some,
+        output_keep_prob=keep_some,
+        state_keep_prob=keep_some,
+        variational_recurrent=True,
+        wrapper_type=wrapper_type,
+        input_size=3,
+        batch_size=5,
+        time_steps=7,
+        seed=-234987,
+        scope="root_0")
+    random_seed.set_random_seed(2347)
+    np.random.seed(23487)
+    res1 = self._testDropoutWrapper(
+        input_keep_prob=keep_some,
+        output_keep_prob=keep_some,
+        state_keep_prob=keep_some,
+        variational_recurrent=True,
+        wrapper_type=wrapper_type,
+        input_size=3,
+        batch_size=5,
+        time_steps=7,
+        seed=-234987,
+        scope="root_1")
+
+    output_mask = np.abs(res0[0]) > 1e-6
+    for time_step in output_mask:
+      # Ensure the same dropout output pattern for all time steps
+      self.assertAllClose(output_mask[0], time_step)
+      for batch_entry in time_step:
+        # Assert all batch entries get the same mask
+        self.assertAllClose(batch_entry, time_step[0])
+
+    # For state, ensure all batch entries have the same mask
+    state_c_mask = np.abs(res0[1].c) > 1e-6
+    state_h_mask = np.abs(res0[1].h) > 1e-6
+    for batch_entry in state_c_mask:
+      self.assertAllClose(batch_entry, state_c_mask[0])
+    for batch_entry in state_h_mask:
+      self.assertAllClose(batch_entry, state_h_mask[0])
+
+    # Ensure seeded calculation is identical.
+    self.assertAllClose(res0[0], res1[0])
+    self.assertAllClose(res0[1].c, res1[1].c)
+    self.assertAllClose(res0[1].h, res1[1].h)
+
+
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 5e36bcf27198af204642cfd1b2d1d9be6f361064 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 11 Feb 2019 10:30:12 -0800
Subject: [PATCH 2260/2345] [tensor tracer]: handling the tracing of the inner
 loops as in RNN models.

PiperOrigin-RevId: 233433596
---
 .../contrib/tpu/python/tpu/tensor_tracer.py   | 166 ++++++++++++++----
 1 file changed, 135 insertions(+), 31 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py b/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py
index 8c25c98c3b..43b9168ecc 100644
--- a/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py
+++ b/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py
@@ -27,6 +27,7 @@ from tensorflow.contrib.tpu.python.ops import tpu_ops
 from tensorflow.contrib.tpu.python.tpu import tpu
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import graph_io
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
@@ -55,6 +56,7 @@ _SUBMODE_BRIEF = 'brief'
 _SUBMODE_DETAILED = 'detailed'
 _REASON_OUTSIDE_OP_RANGE = 'not-traced-outside-op-range'
 _REASON_UNSAFE_OP = 'not-traced-unsafe-op'
+_REASON_WHILELOOP_OP = 'not-traced-special-whileloop-op'
 _REASON_UNSAFE_SCALAR = 'not-traced-unsafe-scalar'
 _REASON_LESS_INTERESTING_OP = 'not-traced-less-interesting-op'
 _REASON_DEVICE_MISMATCH = 'not-traced-device-mismatch'
@@ -65,6 +67,7 @@ _REASON_USER_INCLUDED = 'traced-user-included'
 _REASON_USER_EXCLUDED = 'not-traced-user-excluded'
 _REASON_NOT_EXECUTED = 'not-traced-not-in-exec-path'
 _REASON_NON_NUMERIC_TENSOR = 'not-traced-non-numeric-tensor'
+_REASON_FEEDS_WHILELOOP_OP = 'not-traced-feeds-special-whileloop-op'
 _MARKER_SECTION_BEGIN = '!!!!!!! section-begin:'
 _MARKER_SECTION_END = '!!!!!!! section-end:'
 _SECTION_NAME_CONFIG = 'configuration'
@@ -102,6 +105,9 @@ _FLAG_NAME_TRACE_DIR = 'trace_dir'
 _FLAG_NAME_REPORT_FILE = 'report_file'
 _FLAG_NAME_USE_TEST_UNDECLARED_OUTPUTS_DIR = 'use_test_undeclared_outputs_dir'
 _FLAG_NAME_OP_RANGE = 'op_range'
+# Folder to dump the pre (before tensor tracer updates) and post graphs (after
+# tensor tracer updates).
+_FLAG_DUMP_BEFORE_AFTER_GRAPHS = 'dump_graphs'
 _OP_RANGE_PAT = re.compile(r'(\d+):(\d+)')
 _OUTPUT_STREAM_ESCAPE = 'file://'
 _TEST_UNDECLARED_OUTPUTS_DIR_ENV_VAR = 'TEST_UNDECLARED_OUTPUTS_DIR'
@@ -276,7 +282,8 @@ class TensorTracer(object):
                         _FLAG_NAME_REPORT_FILE,
                         _FLAG_NAME_USE_TEST_UNDECLARED_OUTPUTS_DIR,
                         _FLAG_NAME_INCLUDE_LESS_INTERESTING_OPS,
-                        _FLAG_NAME_OP_RANGE]
+                        _FLAG_NAME_OP_RANGE,
+                        _FLAG_DUMP_BEFORE_AFTER_GRAPHS]
     tensor_tracer_flags = os.environ.get(_FLAGS_ENV_VAR)
     if not tensor_tracer_flags:
       return
@@ -439,6 +446,29 @@ class TensorTracer(object):
                        'Valid submodes are: %s'%(submode,
                                                  valid_submodes))
 
+  @staticmethod
+  def loop_cond_op(op):
+    return op.type in ('LoopCond', 'RefLoopCond')
+
+  @staticmethod
+  def while_loop_op(op):
+    """Returns true if op is one of the special ops of in a while loop.
+
+    Args:
+       op: A tf.Operation.
+
+    Returns:
+       True if the given op is one of [Switch, Merge, Enter, Exit,
+       NextIteration, LoopCond], which are all building blocks for TF while
+       loops.
+    """
+    return  (control_flow_util.IsLoopSwitch(op) or
+             control_flow_util.IsLoopMerge(op) or
+             control_flow_util.IsLoopEnter(op) or
+             control_flow_util.IsLoopExit(op) or
+             TensorTracer.loop_cond_op(op) or
+             op.type in ('RefNextIteration', 'NextIteration'))
+
   @staticmethod
   def unsafe_op(op):
     """Returns True if this op is not safe to be traced."""
@@ -615,6 +645,8 @@ class TensorTracer(object):
     self._num_replicas_per_host = None
     self._num_hosts = None
     self._replica_id = None
+    _, self._graph_dump_path = TensorTracer.get_flag_value(
+        _FLAG_DUMP_BEFORE_AFTER_GRAPHS)
 
   def _add_replica_id_to_graph(self):
     """Adds nodes for computing the replica ID to the graph."""
@@ -928,13 +960,11 @@ class TensorTracer(object):
         output_stream = _OUTPUT_STREAM_ESCAPE + output_path
       else:
         output_stream = sys.stderr
-      print_op = logging_ops.print_v2(msg, array_ops.shape(output_tensor),
-                                      '@', self._replica_id,
-                                      '\n', output_tensor, '\n',
-                                      summarize=num_elements,
-                                      output_stream=output_stream)
-      with ops.control_dependencies([print_op]):
-        return array_ops.identity(tensor).op
+      return logging_ops.print_v2(msg, array_ops.shape(output_tensor),
+                                  '@', self._replica_id,
+                                  '\n', output_tensor, '\n',
+                                  summarize=num_elements,
+                                  output_stream=output_stream)
 
     def _show_part_tensor(tensor):
       """Trace function for printing part of the tensor."""
@@ -966,33 +996,38 @@ class TensorTracer(object):
                in_exec_path=True):
     """Returns True if we should not trace Op."""
 
-    if user_included:
+    if TensorTracer.while_loop_op(op):
       self._instrument_records[op.name] = TensorTracer.reason(
-          op_id, _REASON_USER_INCLUDED)
-      return False
-    if user_excluded:
+          op_id, _REASON_WHILELOOP_OP)
+      return True
+    if TensorTracer.unsafe_op(op):
       self._instrument_records[op.name] = TensorTracer.reason(
-          op_id, _REASON_USER_EXCLUDED)
+          op_id, _REASON_UNSAFE_OP)
+      return True
+    if TensorTracer.device_mismatch(self._device_type, op):
+      self._instrument_records[op.name] = TensorTracer.reason(
+          op_id, _REASON_DEVICE_MISMATCH)
       return True
     if not in_exec_path:
       self._instrument_records[op.name] = TensorTracer.reason(
           op_id, _REASON_NOT_EXECUTED)
       return True
+
     if not self._inside_op_range(op_id):
       self._instrument_records[op.name] = TensorTracer.reason(
           op_id, _REASON_OUTSIDE_OP_RANGE)
       return True
-    if TensorTracer.unsafe_op(op):
+    if TensorTracer.less_interesting_op(op):
       self._instrument_records[op.name] = TensorTracer.reason(
-          op_id, _REASON_UNSAFE_OP)
+          op_id, _REASON_LESS_INTERESTING_OP)
       return True
-    if TensorTracer.device_mismatch(self._device_type, op):
+    if user_included:
       self._instrument_records[op.name] = TensorTracer.reason(
-          op_id, _REASON_DEVICE_MISMATCH)
-      return True
-    if TensorTracer.less_interesting_op(op):
+          op_id, _REASON_USER_INCLUDED)
+      return False
+    if user_excluded:
       self._instrument_records[op.name] = TensorTracer.reason(
-          op_id, _REASON_LESS_INTERESTING_OP)
+          op_id, _REASON_USER_EXCLUDED)
       return True
     return False
 
@@ -1010,7 +1045,12 @@ class TensorTracer(object):
       self._instrument_records[out_tensor.name] = TensorTracer.reason(
           op_id, _REASON_NON_NUMERIC_TENSOR)
       return True
-
+    # Skip a tensor if it feeds a special while loop op.
+    if [consumer for consumer in out_tensor.consumers() if
+        TensorTracer.while_loop_op(consumer)]:
+      self._instrument_records[out_tensor.name] = TensorTracer.reason(
+          op_id, _REASON_FEEDS_WHILELOOP_OP)
+      return True
     if user_included:
       self._instrument_records[out_tensor.name] = TensorTracer.reason(
           op_id, _REASON_USER_INCLUDED)
@@ -1080,6 +1120,10 @@ class TensorTracer(object):
 
       for input_op in input_ops:
         if input_op not in execution_path_operations:
+          # Filter out loop condition operations, tracing them causes a cycle.
+          # Trace only the loop-body.
+          if TensorTracer.loop_cond_op(input_op):
+            continue
           execution_path_operations.add(input_op)
           traverse_stack.append(input_op)
     return execution_path_operations
@@ -1217,10 +1261,13 @@ class TensorTracer(object):
             replica_id_str = replica_id
           else:
             replica_id_str = '%d'%replica_id
-          output_path = os.path.join(self._trace_dir,
-                                     _COMPACT_TRACE_FILE_PREFIX) \
-                                     + replica_id_str
-          output_stream = _OUTPUT_STREAM_ESCAPE + output_path
+          if self._trace_dir:
+            output_path = os.path.join(self._trace_dir,
+                                       _COMPACT_TRACE_FILE_PREFIX) \
+                                       + replica_id_str
+            output_stream = _OUTPUT_STREAM_ESCAPE + output_path
+          else:
+            output_stream = sys.stderr
           new_step_line = _REPLICA_ID_TAG + replica_id_str
           print_op = logging_ops.print_v2(
               new_step_line, '\n',
@@ -1340,6 +1387,23 @@ class TensorTracer(object):
       else:
         return current_fetches
 
+  def _get_op_control_flow_context(self, op):
+    """Returns the control flow of the given op.
+
+    Args:
+      op: tf.Operation for which the control flow context is requested.
+    Returns:
+      op_control_flow_context: which the is control flow context of the given
+      op. If the operation type is LoopExit, returns the outer control flow
+      context.
+    """
+    # pylint: disable=protected-access
+    op_control_flow_context = op._control_flow_context
+    # pylint: enable=protected-access
+    if control_flow_util.IsLoopExit(op):
+      op_control_flow_context = op_control_flow_context.outer_context
+    return op_control_flow_context
+
   def _trace_execution(self, graph,
                        tensor_fetches,
                        op_fetches=None,
@@ -1391,6 +1455,11 @@ class TensorTracer(object):
 
     tensor_fetch_set = set(processed_t_fetches)
     tracing_ops = []
+
+    # pylint: disable=protected-access
+    current_control_flow_context = graph._get_control_flow_context()
+    # pylint: enable=protected-access
+
     # Trace ops only if they are in the execution path.
     for op in exec_op_set:
       for i in range(len(op.outputs)):
@@ -1411,6 +1480,11 @@ class TensorTracer(object):
         is_a_fetched_tensor = out_tensor in tensor_fetch_set
         if (not consumers) and (not is_a_fetched_tensor):
           continue
+
+        op_control_flow_context = self._get_op_control_flow_context(op)
+        # pylint: disable=protected-access
+        graph._set_control_flow_context(op_control_flow_context)
+        # pylint: enable=protected-access
         processed_out_tensor = self._preprocess_traced_tensor(out_tensor)
 
         if on_tpu:
@@ -1426,8 +1500,7 @@ class TensorTracer(object):
               self._make_tensor_trace_fun(tensor_name), processed_out_tensor)
         else:
           trace_fun = self._make_tensor_trace_fun(tensor_name)
-          traced_tensor = trace_fun(processed_out_tensor)
-          trace_op = traced_tensor.op
+          trace_op = trace_fun(processed_out_tensor)
 
         if is_a_fetched_tensor:
           tracing_ops.append(trace_op)
@@ -1438,6 +1511,10 @@ class TensorTracer(object):
           # pylint: disable=protected-access
           consumer_op._add_control_input(trace_op)
           # pylint: enable=protected-access
+
+    # pylint: disable=protected-access
+    graph._set_control_flow_context(current_control_flow_context)
+    # pylint: enable=protected-access
     if tracing_ops:
       # If we are tracing a fetched tensor, their dependency is stored in
       # tracing_ops.
@@ -1447,7 +1524,7 @@ class TensorTracer(object):
       processed_t_fetches = self._flush_tensor_values_cache(graph,
                                                             processed_t_fetches,
                                                             op_fetches,
-                                                            on_tpu=True)
+                                                            on_tpu=on_tpu)
     self._post_tracing(succeed, sorted_or_cycle)
     # processed_t_fetches is a list at this point. Convert it to the same
     # format as given in tensor_fetches.
@@ -1486,12 +1563,28 @@ class TensorTracer(object):
     self._num_replicas = num_replicas
     self._num_replicas_per_host = num_replicas_per_host
     self._num_hosts = num_hosts
+    if self._num_replicas is not None:
+      if self._num_replicas_per_host is None:
+        self._num_replicas_per_host = 8
+      if self._num_hosts is None:
+        self._num_hosts = num_replicas // self._num_replicas_per_host + \
+            (num_replicas % self._num_replicas_per_host > 0)
+
     if self._num_replicas_per_host > 8:
       # Checks for the assumption in _generate_flush_cache_op().
       raise RuntimeError('num_replicas_per_host (%d) is '
                          'greater than 8'%self._num_replicas_per_host)
-    self._add_replica_id_to_graph()
-    return self._trace_execution(graph, tensor_fetches, op_fetches, on_tpu=True)
+    if self._graph_dump_path:
+      graph_io.write_graph(graph, self._graph_dump_path,
+                           'graph_before_tt.pbtxt')
+    with graph.as_default():
+      self._add_replica_id_to_graph()
+      tensor_fetches = self._trace_execution(graph, tensor_fetches, op_fetches,
+                                             on_tpu=True)
+    if self._graph_dump_path:
+      graph_io.write_graph(graph, self._graph_dump_path,
+                           'graph_after_tt.pbtxt')
+    return tensor_fetches
 
   def trace_cpu(self, graph, tensor_fetches, op_fetches=None):
     """Traces the tensors generated by CPU Ops in a TF graph.
@@ -1516,4 +1609,15 @@ class TensorTracer(object):
     self._num_replicas_per_host = 1
     self._num_hosts = 1
     self._replica_id = 0
-    return self._trace_execution(graph, tensor_fetches, op_fetches, on_tpu=True)
+    if self._graph_dump_path:
+      graph_io.write_graph(graph, self._graph_dump_path,
+                           'graph_before_tt.pbtxt')
+    with graph.as_default():
+      tensor_fetches = self._trace_execution(graph, tensor_fetches, op_fetches,
+                                             on_tpu=False)
+    if self._graph_dump_path:
+      graph_io.write_graph(graph, self._graph_dump_path,
+                           'graph_after_tt.pbtxt')
+    return tensor_fetches
+
+
-- 
GitLab


From 13cafdc3d48de7da72828d7c7e8e286cae5f66a1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 11 Feb 2019 10:31:12 -0800
Subject: [PATCH 2261/2345] [TF:XLA] Prevent debug/profiler tests with XLA.

Included a note that these tests are expected to fail because TF:XLA modifies the graph which is being analyzed.

This is for bookkeeping purposes.

PiperOrigin-RevId: 233433772
---
 tensorflow/python/BUILD                   |  1 +
 tensorflow/python/debug/BUILD             | 10 ++++++++++
 tensorflow/python/profiler/BUILD          |  3 +++
 tensorflow/python/profiler/internal/BUILD |  1 +
 4 files changed, 15 insertions(+)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index d3254cc2e4..1ceeac8a56 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -4719,6 +4719,7 @@ cuda_py_test(
         ":math_ops",
         "//tensorflow/core:protos_all_py",
     ],
+    xla_enable_strict_auto_jit = False,  # Graph structure is different with autojit
 )
 
 cuda_py_test(
diff --git a/tensorflow/python/debug/BUILD b/tensorflow/python/debug/BUILD
index dbe738b703..3b1b214a6f 100644
--- a/tensorflow/python/debug/BUILD
+++ b/tensorflow/python/debug/BUILD
@@ -569,6 +569,7 @@ cuda_py_test(
         "//tensorflow/python:training",
         "//tensorflow/python:variables",
     ],
+    xla_enable_strict_auto_jit = False,  # Node names are different with autojit
 )
 
 py_test(
@@ -657,6 +658,7 @@ cuda_py_test(
         "//tensorflow/python:training",
         "//tensorflow/python:variables",
     ],
+    xla_enable_strict_auto_jit = False,  # Node names are different with autojit
 )
 
 py_test(
@@ -816,6 +818,7 @@ cuda_py_test(
         "//tensorflow/python:variables",
     ],
     tags = ["notsan"],
+    xla_enable_strict_auto_jit = False,  # Node names are different with autojit
 )
 
 cuda_py_test(
@@ -833,6 +836,7 @@ cuda_py_test(
         "//tensorflow/python:training",
         "//tensorflow/python:variables",
     ],
+    xla_enable_strict_auto_jit = False,  # Node names are different with autojit
 )
 
 cuda_py_test(
@@ -850,6 +854,7 @@ cuda_py_test(
         "//tensorflow/python:variables",
     ],
     tags = ["no_windows_gpu"],
+    xla_enable_strict_auto_jit = False,  # Node names are different with autojit
 )
 
 py_test(
@@ -975,6 +980,7 @@ cuda_py_test(
         "//tensorflow/python:variables",
     ],
     tags = ["no_windows"],  # TODO: needs investigation on Windows
+    xla_enable_strict_auto_jit = False,  # Node names are different with autojit
 )
 
 py_test(
@@ -1021,6 +1027,7 @@ cuda_py_test(
         "no_windows",
         "notap",
     ],
+    xla_enable_strict_auto_jit = False,  # Node names are different with autojit
 )
 
 cuda_py_test(
@@ -1048,6 +1055,7 @@ cuda_py_test(
         "notsan",
         "oss_serial",
     ],
+    xla_enable_strict_auto_jit = False,  # Node names are different with autojit
 )
 
 cuda_py_test(
@@ -1073,6 +1081,7 @@ cuda_py_test(
         "optonly",  # Test flaky (b/80130873)
         "oss_serial",
     ],
+    xla_enable_strict_auto_jit = False,  # Node names are different with autojit
 )
 
 # TODO(cais): Run the test in OSS, perhaps through a sh_test.
@@ -1101,6 +1110,7 @@ cuda_py_test(
         "no_windows",
         "notsan",
     ],
+    xla_enable_strict_auto_jit = False,  # Node names are different with autojit
 )
 
 py_test(
diff --git a/tensorflow/python/profiler/BUILD b/tensorflow/python/profiler/BUILD
index 0654104a34..fcab57c12c 100644
--- a/tensorflow/python/profiler/BUILD
+++ b/tensorflow/python/profiler/BUILD
@@ -61,6 +61,7 @@ cuda_py_test(
         "no_pip",
         "oss_serial",
     ],
+    xla_enable_strict_auto_jit = False,  # Node names are different with autojit
 )
 
 cuda_py_test(
@@ -76,6 +77,7 @@ cuda_py_test(
         "//tensorflow/python:variables",
     ],
     tags = ["no_pip"],
+    xla_enable_strict_auto_jit = False,  # Node names are different with autojit
 )
 
 py_library(
@@ -130,6 +132,7 @@ cuda_py_test(
         "//tensorflow/python:variables",
     ],
     tags = ["no_pip"],
+    xla_enable_strict_auto_jit = False,  # Node names are different with autojit
 )
 
 py_library(
diff --git a/tensorflow/python/profiler/internal/BUILD b/tensorflow/python/profiler/internal/BUILD
index 994206cd63..0a6ba12094 100644
--- a/tensorflow/python/profiler/internal/BUILD
+++ b/tensorflow/python/profiler/internal/BUILD
@@ -69,4 +69,5 @@ cuda_py_test(
     tags = [
         "no_pip",
     ],
+    xla_enable_strict_auto_jit = False,  # Node names are different with autojit
 )
-- 
GitLab


From 35455c2b0675e76aa83f2eb16d53e5d183576a48 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 11 Feb 2019 11:13:34 -0800
Subject: [PATCH 2262/2345] [TF:XLA] Increase tolerance of
 special_math_ops_test.

This was already a floating point comparison. Increase the tolerance slightly, so XLA:GPU can also be tested with different CUDNN algorithms.

PiperOrigin-RevId: 233441976
---
 tensorflow/python/BUILD                        | 1 +
 tensorflow/python/ops/special_math_ops_test.py | 4 +++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 1ceeac8a56..a5b3daa674 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3694,6 +3694,7 @@ cuda_py_test(
         "//third_party/py/numpy",
     ],
     tags = ["no_windows_gpu"],
+    xla_enable_strict_auto_jit = True,
 )
 
 py_library(
diff --git a/tensorflow/python/ops/special_math_ops_test.py b/tensorflow/python/ops/special_math_ops_test.py
index 0224e7ebb8..60e0f1f832 100644
--- a/tensorflow/python/ops/special_math_ops_test.py
+++ b/tensorflow/python/ops/special_math_ops_test.py
@@ -127,7 +127,9 @@ class LBetaTest(test.TestCase):
     x_b = [0.1]
     with self.session(use_gpu=True):
       self.assertAllClose(
-          1, self.evaluate(math_ops.exp(special_math_ops.lbeta(x_a))))
+          1,
+          self.evaluate(math_ops.exp(special_math_ops.lbeta(x_a))),
+          rtol=3e-6)
       self.assertAllClose(
           1, self.evaluate(math_ops.exp(special_math_ops.lbeta(x_b))))
       self.assertEqual((), special_math_ops.lbeta(x_a).get_shape())
-- 
GitLab


From 734e10d80b9bcf94e12814e71295f8210f45366f Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Mon, 11 Feb 2019 11:16:58 -0800
Subject: [PATCH 2263/2345] [XLA] Make not having is_dynamic_dimensions a
 warning rather than an error

Since we currently don't require this field, and accept protos that don't have it, it's probably better not to spam the ERROR logs if the field is empty.

PiperOrigin-RevId: 233442644
---
 tensorflow/compiler/xla/shape.cc | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/shape.cc b/tensorflow/compiler/xla/shape.cc
index b88f489450..93d630b8f7 100644
--- a/tensorflow/compiler/xla/shape.cc
+++ b/tensorflow/compiler/xla/shape.cc
@@ -34,8 +34,12 @@ Shape::Shape(const ShapeProto& shape_proto) {
   // instead of a constructor.
   if (shape_proto.dimensions_size() !=
       shape_proto.is_dynamic_dimension_size()) {
-    LOG(ERROR) << "Malformed shape proto: number of is_dynamic_dimension "
-                  "fields does not match number of dimension fields";
+    if (shape_proto.is_dynamic_dimension_size() != 0) {
+      LOG(ERROR) << "Malformed shape proto: number of is_dynamic_dimension "
+                    "fields does not match number of dimension fields";
+    } else {
+      LOG(WARNING) << "Malformed shape proto: is_dynamic_dimension is empty";
+    }
   }
   int64 num_dynamic_dimension_fields = std::min(
       shape_proto.dimensions_size(), shape_proto.is_dynamic_dimension_size());
-- 
GitLab


From 08703e1aadfaf68d9cf3197f2fb896d5e83efbce Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 11 Feb 2019 11:34:45 -0800
Subject: [PATCH 2264/2345] Multiple enhancements to graphviz output:   - nodes
 organized by subgraphs inferred from array names   - subgraphs and ops
 include math op count estimates   - better organized labels on all nodes   -
 operators have numbered inputs and outputs when two or more exist   -
 specific node shapes for inputs/outputs   - node color scheme unchanged

PiperOrigin-RevId: 233446311
---
 tensorflow/lite/toco/BUILD            |   1 +
 tensorflow/lite/toco/dump_graphviz.cc | 793 +++++++++++++++++++-------
 tensorflow/lite/toco/dump_graphviz.h  |   3 +-
 tensorflow/lite/toco/toco_tooling.cc  |   2 +-
 tensorflow/lite/toco/tooling_util.cc  | 248 ++++----
 tensorflow/lite/toco/tooling_util.h   |   3 +
 6 files changed, 739 insertions(+), 311 deletions(-)

diff --git a/tensorflow/lite/toco/BUILD b/tensorflow/lite/toco/BUILD
index 3150c48c73..c477e2f4c0 100644
--- a/tensorflow/lite/toco/BUILD
+++ b/tensorflow/lite/toco/BUILD
@@ -378,6 +378,7 @@ cc_library(
         ":types_proto_cc",
         "//tensorflow/core:lib",
         "//tensorflow/lite/kernels/internal:types",
+        "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
         "@com_googlesource_code_re2//:re2",
         "@protobuf_archive//:protobuf_headers",
diff --git a/tensorflow/lite/toco/dump_graphviz.cc b/tensorflow/lite/toco/dump_graphviz.cc
index 8896893f35..ad69e4f7b7 100644
--- a/tensorflow/lite/toco/dump_graphviz.cc
+++ b/tensorflow/lite/toco/dump_graphviz.cc
@@ -15,17 +15,21 @@ limitations under the License.
 #include "tensorflow/lite/toco/dump_graphviz.h"
 
 #include <cmath>
+#include <functional>
 #include <memory>
 #include <vector>
 
+#include "absl/memory/memory.h"
 #include "absl/strings/str_replace.h"
+#include "absl/strings/str_split.h"
 #include "absl/strings/strip.h"
+#include "re2/re2.h"
+#include "tensorflow/core/platform/logging.h"
 #include "tensorflow/lite/toco/model_flags.pb.h"
 #include "tensorflow/lite/toco/toco_graphviz_dump_options.h"
 #include "tensorflow/lite/toco/toco_port.h"
 #include "tensorflow/lite/toco/toco_types.h"
 #include "tensorflow/lite/toco/tooling_util.h"
-#include "tensorflow/core/platform/logging.h"
 
 using toco::port::AppendF;
 using toco::port::StringF;
@@ -33,72 +37,158 @@ using toco::port::StringF;
 namespace toco {
 namespace {
 
+// 'nslimit' is a graphviz (dot) paramater that limits the iterations during
+// the layout phase. Omitting it allows infinite iterations, causing some
+// complex graphs to never finish. A value of 125 produces good graphs
+// while allowing complex graphs to finish.
+constexpr char kGraphFmt[] = R"CODE(digraph Computegraph { tooltip = "/"
+    nslimit=125 margin=36 ranksep = 2 labelloc="t" label=%s
+)CODE";
+// Note: tooltip's are only supported on SVGs in Chrome.
+constexpr char kSubgraphFmt[] =
+    R"CODE(    subgraph "cluster_%s" { style=rounded bgcolor="%s" penwidth=0.0 label=%s
+)CODE";
+constexpr char kArrayNodeFmt[] =
+    R"CODE(        "%s" [label=%s tooltip="%s" shape=%s style=filled fillcolor="%s" fontcolor="%sDD"];
+)CODE";
+constexpr char kOpNodeFmt[] =
+    R"CODE(        %s [label=%s tooltip=" " shape=box margin=0 style=filled fillcolor="%s" fontcolor="%sDD"];
+)CODE";
+constexpr char kInputEdgeFmt[] =
+    R"CODE(        "%s"%s -> %s:i%d:n [penwidth=%f weight=%f];
+)CODE";
+constexpr char kOutputEdgeFmt[] =
+    R"CODE(        %s:o%d:s -> "%s"%s [penwidth=%f weight=%f];
+)CODE";
+constexpr char kRNNBackEdgeFmt[] =
+    R"CODE(        "%s":s -> "%s":n [color="#0F9D58" constraint=false];
+)CODE";
+constexpr char kUnicodeMult[] = "\u00D7";
+constexpr char kUnicodeEllipsis[] = " \u2026 ";
+
 class Color {
  public:
   Color() {}
   Color(uint8 r, uint8 g, uint8 b) : r_(r), g_(g), b_(b) {}
+  explicit Color(uint32 word)
+      : r_((word & 0x00FF0000) >> 16),
+        g_((word & 0x0000FF00) >> 8),
+        b_((word & 0x000000FF) >> 0) {}
+
   // Returns the string serialization of this color in graphviz format,
   // for use as 'fillcolor' in boxes.
-  string FillColorString() const { return StringF("%.2X%.2X%.2X", r_, g_, b_); }
+  string AsHexString() const { return StringF("#%.2X%.2X%.2X", r_, g_, b_); }
+  // The color to use for this node; will be used as 'fillcolor'
+  // for its box. See Color::AsHexString. A suitable, different
+  // color will be chosen for the 'fontcolor' for the inside text
+  // label, see Color::TextColorString.
   // Returns the serialization in graphviz format of a suitable color to use
   // 'fontcolor' in the same boxes. It should black or white, whichever offers
-  // the better contrast from FillColorString().
+  // the better contrast from AsHexString().
   string TextColorString() const {
     // https://en.wikipedia.org/wiki/Relative_luminance
     const float luminance = 0.2126f * r_ + 0.7152f * g_ + 0.0722f * b_;
     const uint8 l = luminance > 128.f ? 0 : 255;
-    return StringF("%.2X%.2X%.2X", l, l, l);
+    return StringF("#%.2X%.2X%.2X", l, l, l);
   }
 
  private:
   uint8 r_ = 0, g_ = 0, b_ = 0;
 };
 
-struct NodeProperties {
-  // The text to display inside the box for this node.
-  string label;
-  // The color to use for this node; will be used as 'fillcolor'
-  // for its box. See Color::FillColorString. A suitable, different
-  // color will be chosen for the 'fontcolor' for the inside text
-  // label, see Color::TextColorString.
-  Color color;
-  float log2_buffer_size;
-};
-
-// All colors in this file are from:
-// https://material.io/guidelines/style/color.html
+Color HashStringToColor(string s) {
+  // Return a unique color for a name.
+  //
+  // This function removes Tensorflow anti-collision suffixes (eg "_2"), hashes
+  // the string to a uint_32, then twiddles some bits to get a light and subtle
+  // color. This seems to be a good heuristic for keeping enough of the name to
+  // hash to a unique color while still revealing structure through naming
+  // similarities.
+  //
+  // The regular expression "_\d+" matches any underscore followed by numbers,
+  // which we strip out. Examples:
+  //
+  //     "Conv"      -> "Conv"
+  //     "Conv_2"    -> "Conv"
+  //     "Conv_72"   -> "Conv"
+  //     "Pad_1_bias -> "Pad_bias"
+  //     "Conv_abc"  -> "Conv_abc"
+
+  RE2::GlobalReplace(&s, R"CODE(_\d+)CODE", "");
+  uint32 color_word = std::hash<std::string>{}(s);
+  color_word |= 0x00E0E0E0;
+  return Color(color_word);
+}
 
-Color GetColorForArray(const Model& model, const string& array_name) {
+void GetArrayColorAndShape(const Model& model, const string& array_name,
+                           Color* color, string* shape) {
+  // All colors in this file are from:
+  // https://material.io/guidelines/style/color.html
   // Arrays involved in RNN back-edges have a different color
   for (const auto& rnn_state : model.flags.rnn_states()) {
     // RNN state, fed by a back-edge. Bold color.
     if (array_name == rnn_state.state_array()) {
-      return Color(0x0F, 0x9D, 0x58);
+      *color = Color(0x0F, 0x9D, 0x58);
+      *shape = "invhouse";
+      return;
     }
     // RNN back-edge source, feeding a RNN state.
     // Light tone of the same color as RNN states.
     if (array_name == rnn_state.back_edge_source_array()) {
-      return Color(0xB7, 0xE1, 0xCD);
+      *color = Color(0xB7, 0xE1, 0xCD);
+      *shape = "house";
+      return;
     }
   }
   // Constant parameter arrays have their own bold color
   if (model.GetArray(array_name).buffer) {
-    return Color(0x42, 0x85, 0xF4);
+    *color = Color(0x42, 0x85, 0xF4);
+    *shape = "cylinder";
+    return;
   }
   // Remaining arrays are activations.
   // We use gray colors for them because they are the majority
   // of arrays so we want to highlight other arrays instead of them.
   // First, we use a bolder gray for input/output arrays:
   if (IsInputArray(model, array_name)) {
-    return Color(0x9E, 0x9E, 0x9E);
+    *color = Color(0x9E, 0x9E, 0x9E);
+    *shape = "invhouse";
+    return;
   }
   if (IsOutputArray(model, array_name)) {
-    return Color(0x9E, 0x9E, 0x9E);
+    *color = Color(0x9E, 0x9E, 0x9E);
+    *shape = "house";
+    return;
   }
   // Remaining arrays are intermediate activation arrays.
   // Lighter tone of the same grey as for input/output arrays:
   // We want these to be very discrete.
-  return Color(0xF5, 0xF5, 0xF5);
+  *color = Color(0xF5, 0xF5, 0xF5);
+  *shape = "box";
+}
+
+string GetArrayCompassPt(const Model& model, const string& array_name) {
+  // The "compass point" is the point on the node where edge connections are
+  // made. For most arrays we don't care, but input's and outputs look better
+  // connected at the tip of the "house" and "invhouse" shapes used. So we
+  // append ":n" and ":s" respectively for those.
+  for (const auto& rnn_state : model.flags.rnn_states()) {
+    // RNN state is essentially an input
+    if (array_name == rnn_state.state_array()) {
+      return ":s";
+    }
+    // RNN back-edge source is essentially an output
+    if (array_name == rnn_state.back_edge_source_array()) {
+      return ":n";
+    }
+  }
+  if (IsInputArray(model, array_name)) {
+    return ":s";
+  }
+  if (IsOutputArray(model, array_name)) {
+    return ":n";
+  }
+  return "";
 }
 
 void AppendArrayVal(string* string, Array const& array, int index) {
@@ -141,239 +231,550 @@ void AppendArrayVal(string* string, Array const& array, int index) {
   }
 }
 
-NodeProperties GetPropertiesForArray(const Model& model,
-                                     const string& array_name) {
-  NodeProperties node_properties;
-  node_properties.color = GetColorForArray(model, array_name);
-  node_properties.label = absl::StrReplaceAll(array_name, {{"/", "/\\n"}});
-  node_properties.log2_buffer_size = 0.0f;
+typedef std::map<string, string> Attributes;
+
+string AttributesToHtml(Attributes attributes) {
+  string html;
+  for (const auto& attr : attributes) {
+    html += R"CODE(<TR><TD CELLPADDING="1" ALIGN="RIGHT">)CODE";
+    html += attr.first;
+    html += R"CODE(:</TD><TD CELLPADDING="1" ALIGN="LEFT">)CODE";
+    html += attr.second;
+    html += "</TD></TR>";
+  }
+  return html;
+}
+
+string GetArrayLabel(const Model& model, const string& array_id) {
+  string html;
 
-  // Append array shape to the label.
-  auto& array = model.GetArray(array_name);
-  AppendF(&node_properties.label, "\\nType: %s",
-          ArrayDataTypeName(array.data_type));
+  // Use HTML-like labels (http://www.graphviz.org/doc/info/shapes.html#html)
+  html += "<";
 
+  // Begin Table
+  html += R"CODE(<FONT POINT-SIZE="10" FACE="Courier">)CODE";
+  html += R"CODE(<TABLE BORDER="0" CELLSPACING="2" CELLPADDING="0">)CODE";
+
+  auto& array = model.GetArray(array_id);
+  if (array.buffer) {
+    // "cylinder" shapes require some extra head room.
+    html += R"CODE(<TR><TD COLSPAN="2"> </TD></TR>)CODE";
+  }
+
+  // "Primary" name of array (last non-slash delimited group of characters).
+  html += R"CODE(<TR><TD COLSPAN="2" ALIGN="CENTER">)CODE";
+  html += R"CODE(<FONT POINT-SIZE="16" FACE="Helvetica"><I>)CODE";
+  AppendF(&html, R"CODE(%s)CODE",
+          std::vector<string>(absl::StrSplit(array_id, '/')).back());
+  html += R"CODE(</I></FONT>)CODE";
+  html += "</TD></TR>";
+
+  // Array data type and dimensions
+  html += R"CODE(<TR><TD COLSPAN="2" ALIGN="CENTER">)CODE";
+  html += R"CODE(<FONT POINT-SIZE="14" FACE="Courier"><B>)CODE";
+  // Type
+  html += ArrayDataTypeName(array.data_type);
+  // Shape
   if (array.has_shape()) {
     auto& array_shape = array.shape();
-    node_properties.label += "\\n[";
-    for (int id = 0; id < array_shape.dimensions_count(); id++) {
-      if (id == 0) {
-        AppendF(&node_properties.label, "%d", array_shape.dims(id));
-      } else {
-        // 0x00D7 is the unicode multiplication symbol
-        AppendF(&node_properties.label, "\u00D7%d", array_shape.dims(id));
+    html += "[";
+    for (int dim = 0; dim < array_shape.dimensions_count(); dim++) {
+      AppendF(&html, "%d", array_shape.dims(dim));
+      if (dim + 1 < array_shape.dimensions_count()) {
+        html += kUnicodeMult;
       }
     }
-    node_properties.label += "]";
+    html += "]";
+  }
 
-    int buffer_size = 0;
-    if (IsNonEmpty(array.shape())) {
-      buffer_size = RequiredBufferSizeForShape(array.shape());
-      node_properties.log2_buffer_size =
-          std::log2(static_cast<float>(buffer_size));
+  // Small buffer sample
+  int buffer_size = 0;
+  if (array.buffer) {
+    buffer_size = RequiredBufferSizeForShape(array.shape());
+  }
+  if ((buffer_size > 0) && (buffer_size <= 4)) {
+    html += " = ";
+    if (array.shape().dimensions_count() > 0) {
+      html += "{";
     }
-
-    if (array.buffer) {
-      const auto& array = model.GetArray(array_name);
-      if (buffer_size <= 4) {
-        AppendF(&node_properties.label, " = ");
-        if (array.shape().dimensions_count() > 0) {
-          AppendF(&node_properties.label, "{");
-        }
-        for (int i = 0; i < buffer_size; i++) {
-          AppendArrayVal(&node_properties.label, array, i);
-          if (i + 1 < buffer_size) {
-            AppendF(&node_properties.label, ", ");
-          }
-        }
-      } else {
-        AppendF(&node_properties.label, "\\n = ");
-        if (array.shape().dimensions_count() > 0) {
-          AppendF(&node_properties.label, "{");
-        }
-        AppendArrayVal(&node_properties.label, array, 0);
-        AppendF(&node_properties.label, ", ");
-        AppendArrayVal(&node_properties.label, array, 1);
-        // 0x2026 is the unicode ellipsis symbol
-        AppendF(&node_properties.label, " \u2026 ");
-        AppendArrayVal(&node_properties.label, array, buffer_size - 2);
-        AppendF(&node_properties.label, ", ");
-        AppendArrayVal(&node_properties.label, array, buffer_size - 1);
-      }
-      if (array.shape().dimensions_count() > 0) {
-        AppendF(&node_properties.label, "}");
+    for (int i = 0; i < buffer_size; i++) {
+      AppendArrayVal(&html, array, i);
+      if (i + 1 < buffer_size) {
+        html += ", ";
       }
     }
+    if (array.shape().dimensions_count() > 0) {
+      html += "}";
+    }
+  }
+  html += R"CODE(</B></FONT>)CODE";
+  html += "</TD></TR>";
+
+  // Large buffer samples get their own line
+  if (buffer_size > 4) {
+    html += R"CODE(<TR><TD COLSPAN="2" ALIGN="CENTER"> = {)CODE";
+    AppendArrayVal(&html, array, 0);
+    html += ", ";
+    AppendArrayVal(&html, array, 1);
+    html += kUnicodeEllipsis;
+    AppendArrayVal(&html, array, buffer_size - 2);
+    html += ", ";
+    AppendArrayVal(&html, array, buffer_size - 1);
+    html += "}</TD></TR>";
   }
 
+  // Other array properties
+  Attributes attrs;
   if (array.minmax) {
-    AppendF(&node_properties.label, "\\nMinMax: [%.7g, %.7g]",
-            array.minmax->min, array.minmax->max);
+    attrs["minmax"] =
+        StringF("[%.7g, %.7g]", array.minmax->min, array.minmax->max);
   }
-
   if (array.quantization_params) {
-    AppendF(&node_properties.label, "\\nQuantization: %7g * (x - %d)",
-            array.quantization_params->scale,
-            array.quantization_params->zero_point);
+    attrs["quant"] = StringF("%7g\u00B7(x-%d)",  // Unicode "cdot"
+                             array.quantization_params->scale,
+                             array.quantization_params->zero_point);
   }
-
   if (array.alloc) {
-    AppendF(&node_properties.label, "\\nTransient Alloc: [%d, %d)",
-            array.alloc->start, array.alloc->end);
+    attrs["alloc"] = StringF("[%d, %d)", array.alloc->start, array.alloc->end);
   }
-
-  return node_properties;
+  html += AttributesToHtml(attrs);
+
+  // output array_id in ultra-small font so it can be searched and copied.
+  html += R"CODE(<TR><TD COLSPAN="2" ALIGN="CENTER">)CODE";
+  html += R"CODE(<FONT POINT-SIZE="3" FACE="">)CODE";
+  AppendF(&html, R"CODE("%s")CODE", array_id);
+  html += R"CODE(</FONT>)CODE";
+  html += "</TD></TR>";
+
+  // End Table and HTML-like label
+  html += R"CODE(</TABLE></FONT>)CODE";
+  html += ">";
+  return html;
 }
 
-NodeProperties GetPropertiesForOperator(const Operator& op) {
-  NodeProperties node_properties;
-  if (op.type == OperatorType::kUnsupported) {
-    node_properties.label =
-        static_cast<const TensorFlowUnsupportedOperator&>(op).tensorflow_op;
-  } else {
-    node_properties.label =
-        string(absl::StripPrefix(OperatorTypeName(op.type), "TensorFlow"));
-  }
+Attributes GetOpAttributes(const Model& model, const Operator& op) {
+  Attributes attrs;
   switch (op.fused_activation_function) {
     case FusedActivationFunctionType::kRelu:
-      AppendF(&node_properties.label, "\\nReLU");
+      attrs["func"] = "ReLU";
       break;
     case FusedActivationFunctionType::kRelu6:
-      AppendF(&node_properties.label, "\\nReLU6");
+      attrs["func"] = "ReLU6";
       break;
     case FusedActivationFunctionType::kRelu1:
-      AppendF(&node_properties.label, "\\nReLU1");
+      attrs["func"] = "ReLU1";
       break;
     default:
       break;
   }
-  // Additional information for some of the operators.
+  // Output state of member vars on derived operators.
   switch (op.type) {
     case OperatorType::kConv: {
       const auto& conv_op = static_cast<const ConvOperator&>(op);
-      node_properties.color = Color(0xC5, 0x39, 0x29);  // Bolder color
-      AppendF(&node_properties.label, "\\n%dx%d/%s", conv_op.stride_width,
-              conv_op.stride_height,
-              conv_op.padding.type == PaddingType::kSame ? "S" : "V");
+      string stride;
+      AppendF(&stride, "%d", conv_op.stride_width);
+      stride += kUnicodeMult;
+      AppendF(&stride, "%d", conv_op.stride_height);
+      attrs["stride"] = stride;
+      attrs["padding"] =
+          (conv_op.padding.type == PaddingType::kSame) ? "same" : "valid";
       break;
     }
     case OperatorType::kDepthwiseConv: {
-      const auto& conv_op = static_cast<const DepthwiseConvOperator&>(op);
-      node_properties.color = Color(0xC5, 0x39, 0x29);  // Bolder color
-      AppendF(&node_properties.label, "\\n%dx%d/%s", conv_op.stride_width,
-              conv_op.stride_height,
-              conv_op.padding.type == PaddingType::kSame ? "S" : "V");
-      break;
-    }
-    case OperatorType::kFullyConnected: {
-      node_properties.color = Color(0xC5, 0x39, 0x29);  // Bolder color
+      const auto& depthconv_op = static_cast<const ConvOperator&>(op);
+      string stride;
+      AppendF(&stride, "%d", depthconv_op.stride_width);
+      stride += kUnicodeMult;
+      AppendF(&stride, "%d", depthconv_op.stride_height);
+      attrs["stride"] = stride;
+      attrs["padding"] =
+          (depthconv_op.padding.type == PaddingType::kSame) ? "same" : "valid";
       break;
     }
     case OperatorType::kFakeQuant: {
       const auto& fakequant_op = static_cast<const FakeQuantOperator&>(op);
-      node_properties.color = Color(0xC5, 0x39, 0x29);  // Bolder color
+      attrs["bits"] = StringF("%d", fakequant_op.num_bits);
       if (fakequant_op.minmax) {
-        AppendF(&node_properties.label, "\\n%dbit [%g,%g]",
-                fakequant_op.num_bits, fakequant_op.minmax->min,
-                fakequant_op.minmax->max);
+        attrs["range"] = StringF("[%g,%g]", fakequant_op.minmax->min,
+                                 fakequant_op.minmax->max);
       } else {
-        AppendF(&node_properties.label, "\\n%dbit [?,?]",
-                fakequant_op.num_bits);
+        attrs["range"] = "[?,?]";
       }
       break;
     }
     default:
-      node_properties.color = Color(0xDB, 0x44, 0x37);
       break;
   }
+  int64 math_ops_count;
+  if (EstimateArithmeticOpsCount(model, op, &math_ops_count) &&
+      (math_ops_count != 0)) {
+    attrs["math"] = FormattedNumber(math_ops_count) + "ops";
+  }
 
-  return node_properties;
+  return attrs;
 }
 
-}  // namespace
+Color GetOpColor(const Operator& op) {
+  if ((op.type == OperatorType::kDepthwiseConv) ||
+      (op.type == OperatorType::kConv) ||
+      (op.type == OperatorType::kFullyConnected) ||
+      (op.type == OperatorType::kFakeQuant)) {
+    // Give some ops a bolder red
+    return Color(0xC5, 0x39, 0x29);
+  } else {
+    return Color(0xDB, 0x44, 0x37);
+  }
+}
+
+string GetOpLabel(const Model& model, const Operator& op) {
+  // Use HTML-like labels (http://www.graphviz.org/doc/info/shapes.html#html)
+  string html;
+  html += "<";
+
+  // Begin Table
+  html += R"CODE(<FONT POINT-SIZE="10" FACE="Courier">)CODE";
+  html +=
+      R"CODE(<TABLE BORDER="0" CELLBORDER="0" CELLSPACING="0" CELLPADDING="0">)CODE";
+
+  // Input Ports
+  if (!op.inputs.empty()) {
+    html += R"CODE(<TR><TD COLSPAN="2" ALIGN="CENTER">)CODE";
+    // Distribute evenly using a sub-table
+    html += R"CODE(<TABLE BORDER="0" CELLBORDER="0" CELLSPACING="0">)CODE";
+    html += R"CODE(<TR>)CODE";
+    for (int i = 0; i < op.inputs.size(); i++) {
+      html += R"CODE(<TD PORT=")CODE";
+      AppendF(&html, "i%d", i);
+      html += R"CODE(">)CODE";
+      if (op.inputs.size() > 1) {
+        // Only number inputs when op has two or more inputs
+        AppendF(&html, "%d", i);
+      }
+      html += "</TD>";
+    }
+    html += "</TR>";
+    html += R"CODE(</TABLE></TD></TR>)CODE";
+  }
 
-void DumpGraphviz(const Model& model, string* output_file_contents) {
-  AppendF(output_file_contents, "digraph Computegraph {\n");
-  // 'nslimit' is a graphviz (dot) paramater that limits the iterations during
-  // the layout phase. Omitting it allows infinite iterations, causing some
-  // complex graphs to never finish. A value of 125 produces good graphs
-  // while allowing complex graphs to finish.
-  AppendF(output_file_contents, "\t nslimit=125;\n");
-
-  constexpr char kNodeFormat[] =
-      "\t \"%s\" [label=\"%s\", shape=%s, style=filled, fillcolor=\"#%s\", "
-      "fontcolor = \"#%sDD\"];\n";
-
-  constexpr char kEdgeFormat[] =
-      "\t \"%s\" -> \"%s\" [penwidth=%f, weight=%f];\n";
-
-  constexpr char kRNNBackEdgeFormat[] =
-      "\t \"%s\" -> \"%s\" [color=\"#0F9D58\"];\n";
-
-  for (const auto& array_kv : model.GetArrayMap()) {
-    // Add node for array.
-    const string& array_name = array_kv.first;
-    const auto& array_properties = GetPropertiesForArray(model, array_name);
-    AppendF(output_file_contents, kNodeFormat, array_name,
-            array_properties.label, "octagon",
-            array_properties.color.FillColorString().c_str(),
-            array_properties.color.TextColorString().c_str());
+  // Name
+  html += R"CODE(<TR><TD COLSPAN="2" CELLPADDING="3" ALIGN="CENTER">)CODE";
+  html += R"CODE(<FONT POINT-SIZE="16" FACE="Helvetica"><B>)CODE";
+  if (op.type == OperatorType::kUnsupported) {
+    html += static_cast<const TensorFlowUnsupportedOperator&>(op).tensorflow_op;
+  } else {
+    html += string(absl::StripPrefix(OperatorTypeName(op.type), "TensorFlow"));
+  }
+  html += R"CODE(</B></FONT>)CODE";
+  html += "</TD></TR>";
+
+  // Attributes
+  Attributes attrs = GetOpAttributes(model, op);
+  html += AttributesToHtml(attrs);
+
+  // Output Ports
+  if (!op.outputs.empty()) {
+    html += R"CODE(<TR><TD COLSPAN="2" ALIGN="CENTER">)CODE";
+    // Distribute evenly using a sub-table
+    html += R"CODE(<TABLE BORDER="0" CELLBORDER="0" CELLSPACING="0">)CODE";
+    html += R"CODE(<TR>)CODE";
+    for (int i = 0; i < op.outputs.size(); i++) {
+      html += R"CODE(<TD PORT=")CODE";
+      AppendF(&html, "o%d", i);
+      html += R"CODE(">)CODE";
+      if (op.outputs.size() > 1) {
+        // Only number outputs when op has two or more outputs
+        AppendF(&html, "%d", i);
+      }
+      html += "</TD>";
+    }
+    html += "</TR>";
+    html += R"CODE(</TABLE></TD></TR>)CODE";
   }
+
+  // End Table and HTML-like label
+  html += R"CODE(</TABLE></FONT>)CODE";
+  html += ">";
+
+  return html;
+}
+
+float GetLog2BufferSize(const Model& model, const string& array_id) {
+  auto& array = model.GetArray(array_id);
+  if (array.has_shape()) {
+    int buffer_size = 0;
+    if (IsNonEmpty(array.shape())) {
+      buffer_size = RequiredBufferSizeForShape(array.shape());
+      return std::log2(static_cast<float>(buffer_size));
+    }
+  }
+  return 0.0f;
+}
+
+string GetOpId(int op_index) { return StringF("op%05d", op_index); }
+
+void DumpOperator(const Model& model, string* output_file, int op_index) {
+  // Dump node for operator.
+  const Operator& op = *model.operators[op_index];
+  Color color = GetOpColor(op);
+  string label = GetOpLabel(model, op);
+  string op_id = GetOpId(op_index);
+  AppendF(output_file, kOpNodeFmt, op_id, label, color.AsHexString(),
+          color.TextColorString());
+}
+
+void DumpOperatorEdges(const Model& model, string* output_file, int op_index) {
+  // Inputs
+  const Operator& op = *model.operators[op_index];
+  string op_id = GetOpId(op_index);
+  for (int i = 0; i < op.inputs.size(); i++) {
+    const auto& input = op.inputs[i];
+    if (!model.HasArray(input)) {
+      // Connected arrays should _always_ exist. Except, perhaps, during
+      // development.
+      continue;
+    }
+    float log2_buffer_size = GetLog2BufferSize(model, input);
+    // Draw lines that transport more data thicker (Otherwise, where would the
+    // data fit? right?).
+    float line_width = std::max(0.5f, log2_buffer_size / 3.0f);
+    // Keep edges that transport more data shorter than those with less.
+    float weight = std::max(1.0f, log2_buffer_size);
+    if (!IsInputArray(model, input) &&
+        GetOpWithOutput(model, input) == nullptr) {
+      // Give the main line of data flow a straighter path by penalizing edges
+      // to standalone buffers. Weights are generally very large buffers that
+      // would otherwise skew the layout.
+      weight = 1.0f;
+    }
+    string compass_pt = GetArrayCompassPt(model, input);
+    AppendF(output_file, kInputEdgeFmt, input, compass_pt, op_id, i, line_width,
+            weight);
+  }
+  // Outputs
+  for (int i = 0; i < op.outputs.size(); i++) {
+    const auto& output = op.outputs[i];
+    if (!model.HasArray(output)) {
+      continue;
+    }
+    float log2_buffer_size = GetLog2BufferSize(model, output);
+    // See comments above regarding weight and line_width calculations.
+    float line_width = std::max(0.5f, log2_buffer_size / 3.0f);
+    float weight = std::max(1.0f, log2_buffer_size);
+    if (!IsArrayConsumed(model, output)) {
+      weight = 1.0f;
+    }
+    string compass_pt = GetArrayCompassPt(model, output);
+    AppendF(output_file, kOutputEdgeFmt, op_id, i, output, compass_pt,
+            line_width, weight);
+  }
+}
+
+struct Node {
+  Node() : math_ops(0) {}
+  // Name used as a key in the model's array map
+  string array_id;
+
+  // Estimated number of math ops incurred by this node (the sum of the op
+  // with this array as 1st output, plus all children nodes).
+  int64 math_ops;
+
+  // A map of child nodes keyed by name.
+  std::map<const string, std::unique_ptr<Node>> children;
+};
+
+string GetSubgraphLabel(Node const& node, const string& subgraph) {
+  // Use HTML-like labels (http://www.graphviz.org/doc/info/shapes.html#html)
+  string html;
+  html += "<";
+
+  // Begin Table
+  html += R"CODE(<FONT POINT-SIZE="12" FACE="Courier">)CODE";
+  html +=
+      R"CODE(<TABLE BORDER="0" CELLBORDER="0" CELLSPACING="0" CELLPADDING="0">)CODE";
+
+  // Name
+  html += R"CODE(<TR><TD COLSPAN="2" CELLPADDING="3" ALIGN="CENTER">)CODE";
+  html += R"CODE(<FONT POINT-SIZE="18" FACE="Helvetica"><I>)CODE";
+  html += subgraph;
+  html += R"CODE(</I></FONT>)CODE";
+  html += "</TD></TR>";
+
+  // Attributes
+  Attributes attrs;
+  if (node.math_ops > 0) {
+    attrs["math"] = FormattedNumber(node.math_ops) + "ops";
+  }
+  html += AttributesToHtml(attrs);
+
+  // End Table and HTML-like label
+  html += R"CODE(</TABLE></FONT>)CODE";
+  html += ">";
+
+  return html;
+}
+
+void DumpSubgraphHeader(string* output_file, Node const& node,
+                        const string& node_name) {
+  Color color = HashStringToColor(node_name);
+  string label = GetSubgraphLabel(node, node_name);
+  AppendF(output_file, kSubgraphFmt, node_name, color.AsHexString(), label);
+}
+
+void DumpArray(const Model& model, string* output_file,
+               const string& array_id) {
+  Color color;
+  string shape;
+  GetArrayColorAndShape(model, array_id, &color, &shape);
+  string label = GetArrayLabel(model, array_id);
+  AppendF(output_file, kArrayNodeFmt, array_id, label, array_id, shape,
+          color.AsHexString(), color.TextColorString());
+
+  // Ops are placed in the same subgraph as their first output.
   for (int op_index = 0; op_index < model.operators.size(); op_index++) {
     const Operator& op = *model.operators[op_index];
-    // Add node for operator.
-    auto op_properties = GetPropertiesForOperator(op);
-    string operator_id = StringF("op%05d", op_index);
-    AppendF(output_file_contents, kNodeFormat, operator_id, op_properties.label,
-            "box", op_properties.color.FillColorString().c_str(),
-            op_properties.color.TextColorString().c_str());
-    // Add edges for all inputs of the operator.
-    for (const auto& input : op.inputs) {
-      if (!model.HasArray(input)) {
-        // Arrays should _always_ exist. Except, perhaps, during development.
-        continue;
-      }
-      auto array_properties = GetPropertiesForArray(model, input);
-      // Draw lines that transport more data thicker (Otherwise, where would the
-      // data fit? right?).
-      float line_width =
-          std::max(0.5f, array_properties.log2_buffer_size / 3.0f);
-      // Keep edges that transport more data shorter than those with less.
-      float weight = std::max(1.0f, array_properties.log2_buffer_size);
-      if (!IsInputArray(model, input) &&
-          GetOpWithOutput(model, input) == nullptr) {
-        // Give the main line of data flow a straighter path by penalizing edges
-        // to standalone buffers. Weights are generally very large buffers that
-        // otherwise skew the layout without this.
-        weight = 1.0f;
-      }
-      AppendF(output_file_contents, kEdgeFormat, input, operator_id, line_width,
-              weight);
+    if (!op.outputs.empty() && (op.outputs[0] == array_id)) {
+      DumpOperator(model, output_file, op_index);
     }
-    // Add edges for all outputs of the operator.
-    for (const auto& output : op.outputs) {
-      if (!model.HasArray(output)) {
-        // Arrays should _always_ exist. Except, perhaps, during development.
-        continue;
-      }
-      auto array_properties = GetPropertiesForArray(model, output);
-      // See comments above regarding weight and line_width calculations.
-      float line_width =
-          std::max(0.5f, array_properties.log2_buffer_size / 3.0f);
-      float weight = std::max(1.0f, array_properties.log2_buffer_size);
-      if (!IsArrayConsumed(model, output)) {
-        weight = 1.0f;
+  }
+}
+
+void DumpNode(const Model& model, string* output_file, const string& node_name,
+              Node const& node) {
+  bool not_root = !node_name.empty();
+  if (not_root) {
+    DumpSubgraphHeader(output_file, node, node_name);
+  }
+
+  for (const auto& child : node.children) {
+    if (!child.second->array_id.empty()) {
+      // Dump array if this node posesses one.
+      DumpArray(model, output_file, child.second->array_id);
+    }
+    // Note that it is always possible to have children. Unlike a filesystem,
+    // the existence of array "foo/bar" does _not_ prevent other arrays, such as
+    // and "foo/bar/baz", from being nested beneath it.
+    DumpNode(model, output_file, child.first, *child.second);
+  }
+
+  if (not_root) {
+    // End subgraph
+    AppendF(output_file, "    }\n");
+  }
+}
+
+int64 GetArithmeticOpsCount(const Model& model, const string& array_id) {
+  for (const auto& op : model.operators) {
+    if (!op->outputs.empty() && op->outputs[0] == array_id) {
+      int64 count;
+      if (EstimateArithmeticOpsCount(model, *op, &count)) {
+        return count;
+      } else {
+        return 0;
       }
-      AppendF(output_file_contents, kEdgeFormat, operator_id, output,
-              line_width, weight);
     }
   }
+  return 0;
+}
 
-  for (const auto& rnn_state : model.flags.rnn_states()) {
-    AppendF(output_file_contents, kRNNBackEdgeFormat,
-            rnn_state.back_edge_source_array(), rnn_state.state_array());
+void InsertNode(const Model& model, const string& array_id, Node* node,
+                std::vector<string> prefixes, int64* math_ops) {
+  if (prefixes.empty()) {
+    // Base case: store array in this node.
+    node->array_id = array_id;
+    *math_ops = GetArithmeticOpsCount(model, array_id);
+  } else {
+    // Insert into the sub-tree for that prefix.
+    string prefix = prefixes.back();
+    prefixes.pop_back();
+    if (node->children.count(prefix) == 0) {
+      // Create a new node if this prefix is unseen.
+      node->children[prefix] = absl::make_unique<Node>();
+    }
+    InsertNode(model, array_id, node->children[prefix].get(), prefixes,
+               math_ops);
   }
+  // Sum estimated math ops into all nodes.
+  node->math_ops += *math_ops;
+}
 
-  AppendF(output_file_contents, "}\n");
+void BuildArrayTree(const Model& model, Node* tree) {
+  // Delimit array names by path "/", then place into a tree based on this path.
+  for (const auto& array_id : model.GetArrayMap()) {
+    std::vector<string> prefixes = absl::StrSplit(array_id.first, '/');
+    std::reverse(prefixes.begin(), prefixes.end());
+    int64 math_ops;  // Temporary storage for math ops used during recursion.
+    InsertNode(model, array_id.first, tree, prefixes, &math_ops);
+  }
+}
+
+string GetGraphLabel(const Model& model, const string& graph_name) {
+  // Use HTML-like labels (http://www.graphviz.org/doc/info/shapes.html#html)
+  string html;
+  html += "<";
+
+  // Begin Table
+  html += R"CODE(<FONT POINT-SIZE="36" FACE="Courier">)CODE";
+  html +=
+      R"CODE(<TABLE BORDER="0" CELLBORDER="0" CELLSPACING="0" CELLPADDING="0">)CODE";
+
+  // Name
+  html += R"CODE(<TR><TD COLSPAN="2" CELLPADDING="3" ALIGN="CENTER">)CODE";
+  html += R"CODE(<FONT POINT-SIZE="64" FACE="Helvetica"><B><I>)CODE";
+  html += graph_name;
+  html += R"CODE(</I></B></FONT>)CODE";
+  html += "</TD></TR>";
+
+  // Attributes
+  Attributes attrs;
+  attrs["arrays"] = StringF("%d", model.GetArrayMap().size());
+  if (!model.optional_arrays.empty()) {
+    attrs["optional arrays"] = StringF("%d", model.optional_arrays.size());
+  }
+  attrs["operators"] = StringF("%d", model.operators.size());
+  int64 ops_count;
+  if (EstimateArithmeticOpsCount(model, &ops_count) && (ops_count > 0)) {
+    attrs["math"] = FormattedNumber(ops_count) + "ops";
+  }
+  if (model.transient_data_size > 0) {
+    attrs["transient data size"] =
+        StringF("%d KiB", model.transient_data_size / 1024);
+  }
+  if (model.transient_data_alignment > 0) {
+    attrs["transient data alignment"] =
+        StringF("%d bytes", model.transient_data_alignment);
+  }
+  html += AttributesToHtml(attrs);
+
+  // End Table and HTML-like label
+  html += R"CODE(</TABLE></FONT>)CODE";
+  html += ">";
+
+  return html;
+}
+}  // namespace
+
+void DumpGraphviz(const Model& model, string* output_file,
+                  const string& graph_name) {
+  // Start graphviz format
+  AppendF(output_file, kGraphFmt, GetGraphLabel(model, graph_name));
+
+  // Organize arrays into a tree for subgraphing
+  Node tree;
+  BuildArrayTree(model, &tree);
+  DumpNode(model, output_file, "", tree);
+
+  // Dump edges outside all subgraphs (otherwise the referred-to nodes are
+  // implicitly included in that subgraph).
+  for (int op_index = 0; op_index < model.operators.size(); op_index++) {
+    DumpOperatorEdges(model, output_file, op_index);
+  }
+
+  // Dump RNN Backedges
+  for (const auto& rnn_state : model.flags.rnn_states()) {
+    AppendF(output_file, kRNNBackEdgeFmt, rnn_state.back_edge_source_array(),
+            rnn_state.state_array());
+  }
+  // End graphviz format
+  AppendF(output_file, "}\n");
 }
 }  // namespace toco
diff --git a/tensorflow/lite/toco/dump_graphviz.h b/tensorflow/lite/toco/dump_graphviz.h
index 9697bd6f0d..9bb74dac3f 100644
--- a/tensorflow/lite/toco/dump_graphviz.h
+++ b/tensorflow/lite/toco/dump_graphviz.h
@@ -21,7 +21,8 @@ limitations under the License.
 
 namespace toco {
 
-void DumpGraphviz(const Model& model, string* output_file_contents);
+void DumpGraphviz(const Model& model, string* output_file_contents,
+                  const string& graph_name);
 
 }  // namespace toco
 
diff --git a/tensorflow/lite/toco/toco_tooling.cc b/tensorflow/lite/toco/toco_tooling.cc
index 06f51825fe..229be2f563 100644
--- a/tensorflow/lite/toco/toco_tooling.cc
+++ b/tensorflow/lite/toco/toco_tooling.cc
@@ -454,7 +454,7 @@ tensorflow::Status Export(const TocoFlags& toco_flags, const Model& model,
       return status;
     } break;
     case GRAPHVIZ_DOT:
-      DumpGraphviz(model, output_file_contents);
+      DumpGraphviz(model, output_file_contents, "Computation Graph");
       break;
     default:
       LOG(FATAL) << "Unhandled output_format='"
diff --git a/tensorflow/lite/toco/tooling_util.cc b/tensorflow/lite/toco/tooling_util.cc
index 6978dde668..74192dad78 100644
--- a/tensorflow/lite/toco/tooling_util.cc
+++ b/tensorflow/lite/toco/tooling_util.cc
@@ -66,29 +66,29 @@ string LogName(const Operator& op) {
 string ArrayDataTypeName(ArrayDataType data_type) {
   switch (data_type) {
     case ArrayDataType::kFloat:
-      return "Float";
+      return "float";
     case ArrayDataType::kInt8:
-      return "Int8";
+      return "int8";
     case ArrayDataType::kUint8:
-      return "Uint8";
+      return "uint8";
     case ArrayDataType::kInt16:
-      return "Int16";
+      return "int16";
     case ArrayDataType::kUint16:
-      return "Uint16";
+      return "uint16";
     case ArrayDataType::kInt32:
-      return "Int32";
+      return "int32";
     case ArrayDataType::kUint32:
-      return "Uint32";
+      return "uint32";
     case ArrayDataType::kInt64:
-      return "Int64";
+      return "int64";
     case ArrayDataType::kUint64:
-      return "Uint64";
+      return "uint64";
     case ArrayDataType::kString:
-      return "String";
+      return "string";
     case ArrayDataType::kBool:
-      return "Bool";
+      return "bool";
     case ArrayDataType::kComplex64:
-      return "Complex64";
+      return "complex64";
     case ArrayDataType::kNone:
       return "None";
     default:
@@ -538,7 +538,8 @@ void DumpGraphvizVideoFrame(const Model& model) {
   static int dump_id = 0;
   static std::unordered_set<std::size_t> dump_hashes;
   string graphviz_dump;
-  DumpGraphviz(model, &graphviz_dump);
+  DumpGraphviz(model, &graphviz_dump,
+               toco::port::StringF("VIDEO frame:%05d", dump_id));
   std::size_t hash = std::hash<string>{}(graphviz_dump);
   if (!dump_hashes.count(hash)) {
     LOG(INFO) << "DUMPING GRAPHVIZ VIDEO FRAME: " << dump_id;
@@ -561,7 +562,7 @@ void LogDump(int log_level, const string& message, const Model& model) {
   if (!dump_options.dump_graphviz.empty()) {
     string graphviz_dump;
 
-    DumpGraphviz(model, &graphviz_dump);
+    DumpGraphviz(model, &graphviz_dump, message);
     const auto result = port::file::SetContents(
         port::file::JoinPath(
             dump_options.dump_graphviz,
@@ -1863,119 +1864,140 @@ string CreateInt32Array(Model* model, const string& param_name,
   return param_array_name;
 }
 
-bool EstimateArithmeticOpsCount(const Model& model, int64* result) {
-  int64 total = 0;
-  for (const auto& op : model.operators) {
-    switch (op->type) {
-      case OperatorType::kFullyConnected:
-      case OperatorType::kConv:
-      case OperatorType::kDepthwiseConv: {
-        const auto& output_array = model.GetArray(op->outputs[0]);
-        const auto& weights_array = model.GetArray(op->inputs[1]);
-        if (!output_array.has_shape() || !weights_array.has_shape()) {
-          return false;
-        }
-        int cols = 1;
-        for (int i = 0; i < output_array.shape().dimensions_count() - 1; i++) {
-          cols *= output_array.shape().dims(i);
-        }
-        const int64 cost_per_col =
-            2 * RequiredBufferSizeForShape(weights_array.shape());
-        total += cost_per_col * cols;
-        if (op->inputs.size() > 2) {
-          // There is a bias vector. One more op per output value.
-          total += RequiredBufferSizeForShape(output_array.shape());
-        }
-        break;
+bool EstimateArithmeticOpsCount(const Model& model, const Operator& op,
+                                int64* result) {
+  switch (op.type) {
+    case OperatorType::kFullyConnected:
+    case OperatorType::kConv:
+    case OperatorType::kDepthwiseConv: {
+      const auto& output_array = model.GetArray(op.outputs[0]);
+      const auto& weights_array = model.GetArray(op.inputs[1]);
+      if (!output_array.has_shape() || !weights_array.has_shape()) {
+        return false;
       }
-      case OperatorType::kAdd:
-      case OperatorType::kSub:
-      case OperatorType::kMul: {
-        const auto& output_array = model.GetArray(op->outputs[0]);
-        if (!output_array.has_shape()) {
-          return false;
-        }
-        total += RequiredBufferSizeForShape(output_array.shape());
-        break;
+      int64 cols = 1;
+      for (int i = 0; i < output_array.shape().dimensions_count() - 1; i++) {
+        cols *= output_array.shape().dims(i);
       }
-      case OperatorType::kAddN: {
-        const auto& output_array = model.GetArray(op->outputs[0]);
-        if (!output_array.has_shape()) {
-          return false;
-        }
-        // AddN cost is roughly the same cost as N-1 Adds.
-        const int num_adds = op->inputs.size() - 1;
-        total += num_adds * RequiredBufferSizeForShape(output_array.shape());
-        break;
+      const int64 cost_per_col =
+          2 * RequiredBufferSizeForShape(weights_array.shape());
+      *result = cost_per_col * cols;
+      if (op.inputs.size() > 2) {
+        // There is a bias vector. One more op per output value.
+        *result += RequiredBufferSizeForShape(output_array.shape());
       }
-      case OperatorType::kLogistic:
-      case OperatorType::kSoftmax:
-      case OperatorType::kLogSoftmax:
-      case OperatorType::kTanh: {
-        const auto& output_array = model.GetArray(op->outputs[0]);
-        if (!output_array.has_shape()) {
-          return false;
-        }
-        // As a very rough ballpark, the cost of evaluating a math function
-        // such as tanh or logistic is about 32 multiplications, and about as
-        // many additions/subtractions. (Just a power-of-two order-of-magnitude
-        // from looking at actual implementations that we use in runtime/ code).
-        total += 64 * RequiredBufferSizeForShape(output_array.shape());
-        break;
+      break;
+    }
+    case OperatorType::kAdd:
+    case OperatorType::kSub:
+    case OperatorType::kMul: {
+      const auto& output_array = model.GetArray(op.outputs[0]);
+      if (!output_array.has_shape()) {
+        return false;
       }
-      case OperatorType::kMaxPool: {
-        const auto& maxpool = *static_cast<const MaxPoolOperator*>(op.get());
-        const auto& output_array = model.GetArray(op->outputs[0]);
-        if (!output_array.has_shape()) {
-          return false;
-        }
-        total += RequiredBufferSizeForShape(output_array.shape()) *
-                 maxpool.kheight * maxpool.kwidth;
-        break;
+      *result = RequiredBufferSizeForShape(output_array.shape());
+      break;
+    }
+    case OperatorType::kAddN: {
+      const auto& output_array = model.GetArray(op.outputs[0]);
+      if (!output_array.has_shape()) {
+        return false;
       }
-      case OperatorType::kAveragePool: {
-        const auto& avgpool =
-            *static_cast<const AveragePoolOperator*>(op.get());
-        const auto& output_array = model.GetArray(op->outputs[0]);
-        if (!output_array.has_shape()) {
-          return false;
-        }
-        total += RequiredBufferSizeForShape(output_array.shape()) *
-                 avgpool.kheight * avgpool.kwidth;
-        break;
+      // AddN cost is roughly the same cost as N-1 Adds.
+      const int64 num_adds = op.inputs.size() - 1;
+      *result = num_adds * RequiredBufferSizeForShape(output_array.shape());
+      break;
+    }
+    case OperatorType::kLogistic:
+    case OperatorType::kSoftmax:
+    case OperatorType::kLogSoftmax:
+    case OperatorType::kTanh: {
+      const auto& output_array = model.GetArray(op.outputs[0]);
+      if (!output_array.has_shape()) {
+        return false;
       }
-      case OperatorType::kL2Pool: {
-        const auto* maxpool = static_cast<const MaxPoolOperator*>(op.get());
-        const auto& output_array = model.GetArray(op->outputs[0]);
-        if (!output_array.has_shape()) {
-          return false;
-        }
-        // The sum of squares requires (kheight*kwidth) multiply-adds,
-        // and then there is the sqrt which we ballpark at 32 ops.
-        const int64 cost_per_val = 2 * maxpool->kheight * maxpool->kwidth + 32;
-        total +=
-            RequiredBufferSizeForShape(output_array.shape()) * cost_per_val;
-        break;
+      // As a very rough ballpark, the cost of evaluating a math function
+      // such as tanh or logistic is about 32 multiplications, and about as
+      // many additions/subtractions. (Just a power-of-two order-of-magnitude
+      // from looking at actual implementations that we use in runtime/ code).
+      *result = 64 * RequiredBufferSizeForShape(output_array.shape());
+      break;
+    }
+    case OperatorType::kMaxPool: {
+      const auto& maxpool = *static_cast<const MaxPoolOperator*>(&op);
+      const auto& output_array = model.GetArray(op.outputs[0]);
+      if (!output_array.has_shape()) {
+        return false;
       }
-      case OperatorType::kL2Normalization: {
-        const auto& output_array = model.GetArray(op->outputs[0]);
-        if (!output_array.has_shape()) {
-          return false;
-        }
-        // Computing the squared L2 norm is N multiply-adds so 2N ops,
-        // then the single inverse-sqrt is negligible, then we multiply each
-        // value by the resulting multiplier, so an extra N ops. Total 3N ops.
-        total += 3 * RequiredBufferSizeForShape(output_array.shape());
-        break;
+      *result = RequiredBufferSizeForShape(output_array.shape()) *
+                maxpool.kheight * maxpool.kwidth;
+      break;
+    }
+    case OperatorType::kAveragePool: {
+      const auto& avgpool = *static_cast<const AveragePoolOperator*>(&op);
+      const auto& output_array = model.GetArray(op.outputs[0]);
+      if (!output_array.has_shape()) {
+        return false;
       }
-      default:
-        break;
+      *result = RequiredBufferSizeForShape(output_array.shape()) *
+                avgpool.kheight * avgpool.kwidth;
+      break;
+    }
+    case OperatorType::kL2Pool: {
+      const auto* maxpool = static_cast<const MaxPoolOperator*>(&op);
+      const auto& output_array = model.GetArray(op.outputs[0]);
+      if (!output_array.has_shape()) {
+        return false;
+      }
+      // The sum of squares requires (kheight*kwidth) multiply-adds,
+      // and then there is the sqrt which we ballpark at 32 ops.
+      const int64 cost_per_val = 2 * maxpool->kheight * maxpool->kwidth + 32;
+      *result = RequiredBufferSizeForShape(output_array.shape()) * cost_per_val;
+      break;
     }
+    case OperatorType::kL2Normalization: {
+      const auto& output_array = model.GetArray(op.outputs[0]);
+      if (!output_array.has_shape()) {
+        return false;
+      }
+      // Computing the squared L2 norm is N multiply-adds so 2N ops,
+      // then the single inverse-sqrt is negligible, then we multiply each
+      // value by the resulting multiplier, so an extra N ops. count 3N ops.
+      *result = 3 * RequiredBufferSizeForShape(output_array.shape());
+      break;
+    }
+    default:
+      *result = 0;
+      break;
+  }
+  return true;
+}
+
+bool EstimateArithmeticOpsCount(const Model& model, int64* result) {
+  int64 total = 0;
+  for (const auto& op : model.operators) {
+    int64 num_ops;
+    if (!EstimateArithmeticOpsCount(model, *op, &num_ops)) {
+      return false;
+    }
+    total += num_ops;
   }
   *result = total;
   return true;
 }
 
+string FormattedNumber(int64 x) {
+  const int64 million = 1000000;
+  const int64 billion = 1000000000;
+  if (x < 10000) {
+    return toco::port::StringF("%d ", x);
+  } else if (x < billion) {
+    return toco::port::StringF("%.3f M", static_cast<double>(x) / million);
+  } else {
+    return toco::port::StringF("%.3f G", static_cast<double>(x) / billion);
+  }
+}
+
 void GetShuffleShape(AxesOrder input_axes_order, AxesOrder output_axes_order,
                      std::vector<int>* shuffle) {
   CHECK_EQ(AxesCount(input_axes_order), AxesCount(output_axes_order));
diff --git a/tensorflow/lite/toco/tooling_util.h b/tensorflow/lite/toco/tooling_util.h
index 517da784d0..fc4aac7740 100644
--- a/tensorflow/lite/toco/tooling_util.h
+++ b/tensorflow/lite/toco/tooling_util.h
@@ -267,7 +267,10 @@ void MakeArrayDims(int num_dims, int batch, int height, int width, int depth,
 string CreateInt32Array(Model* model, const string& param_name,
                         const std::vector<int>& value);
 
+bool EstimateArithmeticOpsCount(const Model& model, const Operator& op,
+                                int64* result);
 bool EstimateArithmeticOpsCount(const Model& model, int64* result);
+string FormattedNumber(int64 x);
 
 int AxesCount(AxesOrder axes_order);
 
-- 
GitLab


From f35ec21e236d421991692177f20c85ab84fc5a3e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 11 Feb 2019 11:47:02 -0800
Subject: [PATCH 2265/2345] It does following two things: 1. filter out trace
 activities whose start_time is earlier than profiler session start_time. 2.
 Print a warning if multiple profiler session is created.

PiperOrigin-RevId: 233448712
---
 tensorflow/c/eager/c_api_experimental.cc      |  4 +++
 tensorflow/c/eager/c_api_experimental.h       |  1 +
 tensorflow/c/eager/c_api_experimental_test.cc | 18 +++++++++++
 .../core/profiler/lib/profiler_session.cc     | 30 ++++++++++++++++++-
 .../core/profiler/lib/profiler_session.h      |  3 ++
 tensorflow/python/eager/profiler.py           |  5 ++++
 tensorflow/python/pywrap_tfe.i                |  1 +
 7 files changed, 61 insertions(+), 1 deletion(-)

diff --git a/tensorflow/c/eager/c_api_experimental.cc b/tensorflow/c/eager/c_api_experimental.cc
index e67e54b152..f46562a21d 100644
--- a/tensorflow/c/eager/c_api_experimental.cc
+++ b/tensorflow/c/eager/c_api_experimental.cc
@@ -29,6 +29,10 @@ TFE_Profiler* TFE_NewProfiler(TFE_Context* ctx) {
   return new TFE_Profiler(ctx);
 }
 
+bool TFE_ProfilerIsOk(TFE_Profiler* profiler) {
+  return profiler->profiler->Status().ok();
+}
+
 void TFE_DeleteProfiler(TFE_Profiler* profiler) { delete profiler; }
 
 void TFE_ProfilerSerializeToString(TFE_Context* ctx, TFE_Profiler* profiler,
diff --git a/tensorflow/c/eager/c_api_experimental.h b/tensorflow/c/eager/c_api_experimental.h
index ed65374438..e6a3d73707 100644
--- a/tensorflow/c/eager/c_api_experimental.h
+++ b/tensorflow/c/eager/c_api_experimental.h
@@ -33,6 +33,7 @@ TF_CAPI_EXPORT extern void TFE_OpConsumeInput(TFE_Op* op, TFE_TensorHandle* h,
 typedef struct TFE_Profiler TFE_Profiler;
 
 TF_CAPI_EXPORT extern TFE_Profiler* TFE_NewProfiler(TFE_Context* ctx);
+TF_CAPI_EXPORT extern bool TFE_ProfilerIsOk(TFE_Profiler* profiler);
 TF_CAPI_EXPORT extern void TFE_DeleteProfiler(TFE_Profiler* profiler);
 
 // The output string is a binary string of tensorflow.tpu.Trace. User can write
diff --git a/tensorflow/c/eager/c_api_experimental_test.cc b/tensorflow/c/eager/c_api_experimental_test.cc
index af55fee66e..1994d65224 100644
--- a/tensorflow/c/eager/c_api_experimental_test.cc
+++ b/tensorflow/c/eager/c_api_experimental_test.cc
@@ -100,5 +100,23 @@ void ExecuteWithProfiling(bool async) {
 TEST(CAPI, ExecuteWithTracing) { ExecuteWithProfiling(false); }
 TEST(CAPI, ExecuteWithTracingAsync) { ExecuteWithProfiling(true); }
 
+TEST(CAPI, MultipleProfilerSession) {
+  TF_Status* status = TF_NewStatus();
+  TFE_ContextOptions* opts = TFE_NewContextOptions();
+  TFE_ContextOptionsSetAsync(opts, static_cast<unsigned char>(false));
+  TFE_Context* ctx = TFE_NewContext(opts, status);
+  CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  TFE_DeleteContextOptions(opts);
+
+  TFE_Profiler* profiler1 = TFE_NewProfiler(ctx);
+  EXPECT_TRUE(TFE_ProfilerIsOk(profiler1));
+
+  TFE_Profiler* profiler2 = TFE_NewProfiler(ctx);
+  EXPECT_FALSE(TFE_ProfilerIsOk(profiler2));
+
+  TFE_DeleteProfiler(profiler1);
+  TFE_DeleteProfiler(profiler2);
+}
+
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/core/profiler/lib/profiler_session.cc b/tensorflow/core/profiler/lib/profiler_session.cc
index 6ef779d5e1..0abb63431e 100644
--- a/tensorflow/core/profiler/lib/profiler_session.cc
+++ b/tensorflow/core/profiler/lib/profiler_session.cc
@@ -17,6 +17,7 @@ limitations under the License.
 #include <string>
 #include "tensorflow/contrib/tpu/profiler/trace_events.pb.h"
 #include "tensorflow/core/common_runtime/eager/context.h"
+#include "tensorflow/core/lib/core/error_codes.pb.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/platform/types.h"
@@ -28,6 +29,12 @@ namespace tensorflow {
 
 namespace {
 
+// Track whether there's an active ProfilerSession.
+// Prevents another ProfilerSession from creating ProfilerInterface(s), as they
+// use singletons that do not allow concurrent profiling request (e.g.,
+// DeviceTracer).
+std::atomic<bool> session_active = ATOMIC_VAR_INIT(false);
+
 void ConvertRunMetadataToTraceEvent(RunMetadata* run_metadata,
                                     tpu::Trace* trace,
                                     const uint64 profile_start_time_micros) {
@@ -58,6 +65,9 @@ void ConvertRunMetadataToTraceEvent(RunMetadata* run_metadata,
     // Emit events.
     for (auto node :
          run_metadata->step_stats().dev_stats(device_id).node_stats()) {
+      if (node.all_start_micros() < profile_start_time_micros) {
+        continue;
+      }
       auto* event = trace->add_trace_events();
       auto* args = event->mutable_args();
       event->set_device_id(device_id);
@@ -102,6 +112,12 @@ Status ProfilerSession::SerializeToString(string* content) {
     profiler->CollectData(&run_metadata).IgnoreError();
   }
 
+  if (active_) {
+    // Allow another session to start.
+    session_active.store(false);
+    active_ = false;
+  }
+
   tpu::Trace trace;
 
   ConvertRunMetadataToTraceEvent(&run_metadata, &trace, start_time_micros_);
@@ -111,7 +127,14 @@ Status ProfilerSession::SerializeToString(string* content) {
 }
 
 ProfilerSession::ProfilerSession(ProfilerContext* const context)
-    : start_time_micros_(Env::Default()->NowNanos() / EnvTime::kMicrosToNanos) {
+    : active_(!session_active.exchange(true)),
+      start_time_micros_(Env::Default()->NowNanos() / EnvTime::kMicrosToNanos) {
+  if (!active_) {
+    status_ = tensorflow::Status(tensorflow::error::Code::UNAVAILABLE,
+                                 "Another profiling session is active.");
+    return;
+  }
+
   LOG(INFO) << "Profile Session started.";
 
   if (context->eager_context != nullptr) {
@@ -131,6 +154,11 @@ ProfilerSession::~ProfilerSession() {
   for (auto& profiler : profilers_) {
     profiler->Stop().IgnoreError();
   }
+
+  if (active_) {
+    // Allow another session to start.
+    session_active.store(false);
+  }
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/profiler/lib/profiler_session.h b/tensorflow/core/profiler/lib/profiler_session.h
index e764852ae3..0727657124 100644
--- a/tensorflow/core/profiler/lib/profiler_session.h
+++ b/tensorflow/core/profiler/lib/profiler_session.h
@@ -57,6 +57,9 @@ class ProfilerSession {
   std::vector<std::unique_ptr<tensorflow::profiler::ProfilerInterface>>
       profilers_ GUARDED_BY(mutex_);
 
+  // True if the session is active.
+  bool active_ GUARDED_BY(mutex_);
+
   tensorflow::Status status_ GUARDED_BY(mutex_);
   const uint64 start_time_micros_;
   mutex mutex_;
diff --git a/tensorflow/python/eager/profiler.py b/tensorflow/python/eager/profiler.py
index eaba0fa245..7ddd891cdc 100644
--- a/tensorflow/python/eager/profiler.py
+++ b/tensorflow/python/eager/profiler.py
@@ -25,6 +25,7 @@ from tensorflow.python import pywrap_tensorflow
 from tensorflow.python.eager import context
 from tensorflow.python.framework import c_api_util
 from tensorflow.python.platform import gfile
+from tensorflow.python.platform import tf_logging as logging
 
 LOGDIR_PLUGIN = 'plugins/profile'
 
@@ -46,6 +47,10 @@ def start():
     raise AssertionError('Another profiler is running.')
   with _profiler_lock:
     _profiler = pywrap_tensorflow.TFE_NewProfiler(context.context()._handle)  # pylint: disable=protected-access
+    if not pywrap_tensorflow.TFE_ProfilerIsOk(_profiler):
+      logging.warning('Another profiler session is running which is probably '
+                      'created by profiler server. Please avoid using profiler '
+                      'server and profiler APIs at the same time.')
 
 
 def stop():
diff --git a/tensorflow/python/pywrap_tfe.i b/tensorflow/python/pywrap_tfe.i
index b19cc536b7..db6c14fb60 100755
--- a/tensorflow/python/pywrap_tfe.i
+++ b/tensorflow/python/pywrap_tfe.i
@@ -34,6 +34,7 @@ limitations under the License.
 %rename("%s") TFE_ContextAsyncWait;
 %rename("%s") TFE_ContextAsyncClearError;
 %rename("%s") TFE_NewProfiler;
+%rename("%s") TFE_ProfilerIsOk;
 %rename("%s") TFE_DeleteProfiler;
 %rename("%s") TFE_ProfilerSerializeToString;
 %rename("%s") TFE_NewProfilerServerOptions;
-- 
GitLab


From f5593ff7620dd0f8085f152c3e2539db37e3f53b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 11 Feb 2019 11:59:29 -0800
Subject: [PATCH 2266/2345] Update gradient of tf.broadcast_to to use
 tf.reduce_sum instead of tf.unsorted_segment_sum. Add new high rank tests for
 tf.broadcast_to and its gradient.

PiperOrigin-RevId: 233451123
---
 .../kernel_tests/broadcast_to_ops_test.py     | 37 ++++++++++++++++++-
 tensorflow/python/ops/array_grad.py           | 19 +++-------
 2 files changed, 41 insertions(+), 15 deletions(-)

diff --git a/tensorflow/python/kernel_tests/broadcast_to_ops_test.py b/tensorflow/python/kernel_tests/broadcast_to_ops_test.py
index 366300ec01..3c5433cb89 100644
--- a/tensorflow/python/kernel_tests/broadcast_to_ops_test.py
+++ b/tensorflow/python/kernel_tests/broadcast_to_ops_test.py
@@ -76,6 +76,26 @@ class BroadcastToTest(test_util.TensorFlowTestCase):
       v_np = np.broadcast_to(x, output_shape)
       self.assertAllEqual(v_tf.eval(), v_np)
 
+  @test_util.run_deprecated_v1
+  def testBroadcastToShapeLargerDim(self):
+    input_shape = [2, 1, 3, 2, 2, 2]
+    output_shape = [1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 15, 3, 2, 2, 2]
+    with self.cached_session(use_gpu=True):
+      x = np.array(np.random.randint(5, size=input_shape), dtype=np.int32)
+      v_tf = array_ops.broadcast_to(constant_op.constant(x), output_shape)
+      v_np = np.broadcast_to(x, output_shape)
+      self.assertAllEqual(v_tf.eval(), v_np)
+
+  @test_util.run_deprecated_v1
+  def testBroadcastToShapeLargerDim2(self):
+    input_shape = [2, 1, 3, 2, 2, 2, 1, 1, 1]
+    output_shape = [1, 1, 1, 2, 5, 3, 2, 2, 2, 3, 3, 3]
+    with self.cached_session(use_gpu=True):
+      x = np.array(np.random.randint(5, size=input_shape), dtype=np.int32)
+      v_tf = array_ops.broadcast_to(constant_op.constant(x), output_shape)
+      v_np = np.broadcast_to(x, output_shape)
+      self.assertAllEqual(v_tf.eval(), v_np)
+
   @test_util.run_deprecated_v1
   def testBroadcastToScalar(self):
     with self.session(use_gpu=True):
@@ -88,8 +108,9 @@ class BroadcastToTest(test_util.TensorFlowTestCase):
   def testBroadcastScalarToNonScalar(self):
     with self.session(use_gpu=True):
       x = np.array(1.0, dtype=np.float)
-      v_tf = array_ops.broadcast_to(constant_op.constant(1.0), [2, 3, 4])
-      v_np = np.broadcast_to(x, [2, 3, 4])
+      v_tf = array_ops.broadcast_to(constant_op.constant(1.0), [2, 3, 4,
+                                                                1, 1, 1])
+      v_np = np.broadcast_to(x, [2, 3, 4, 1, 1, 1])
       self.assertAllEqual(v_tf.eval(), v_np)
 
   @test_util.run_deprecated_v1
@@ -148,6 +169,18 @@ class BroadcastToTest(test_util.TensorFlowTestCase):
                                                     out, out.get_shape())
     self.assertLess(err, 1e-4)
 
+  @test_util.run_deprecated_v1
+  def testGradientWithLargeDim(self):
+    input_shape = [2, 1, 3, 2, 2, 2, 1, 1, 1]
+    output_shape = [1, 1, 1, 2, 5, 3, 2, 2, 2, 3, 3, 3]
+    x = constant_op.constant(np.array(np.random.randn(*input_shape),
+                                      dtype=np.float32))
+    v = array_ops.broadcast_to(x, output_shape)
+    out = 2 * v
+    with self.cached_session():
+      err = gradient_checker.compute_gradient_error(x, x.get_shape(),
+                                                    out, out.get_shape())
+    self.assertLess(err, 1e-4)
 
 if __name__ == "__main__":
   test_lib.main()
diff --git a/tensorflow/python/ops/array_grad.py b/tensorflow/python/ops/array_grad.py
index 06f0a45f6a..57572328f6 100644
--- a/tensorflow/python/ops/array_grad.py
+++ b/tensorflow/python/ops/array_grad.py
@@ -839,18 +839,11 @@ def _ScatterNdNonAliasingAddGrad(op, grad):
 def _BroadcastToGrad(op, grad):
   input_value = op.inputs[0]
   broadcast_shape = op.inputs[1]
-  # Assign ids for each position in input_value.
   input_value_shape = array_ops.shape(input_value)
-  input_value_size = array_ops.size(input_value)
-  ids = array_ops.reshape(math_ops.range(input_value_size), input_value_shape)
-  broadcast_ids = array_ops.broadcast_to(ids, broadcast_shape)
-  # Group by ids and sum its gradients.
-  grad_flatten = array_ops.reshape(grad, [-1])
-  broadcast_ids_flatten = array_ops.reshape(broadcast_ids, [-1])
-  # TODO(apassos): Use reduce_sum for gradient now that we only support
-  # the usual numpy broadcast semantics.
-  updates_grad_flatten = math_ops.unsorted_segment_sum(grad_flatten,
-                                                       broadcast_ids_flatten,
-                                                       input_value_size)
-  updates_grad = array_ops.reshape(updates_grad_flatten, input_value_shape)
+  _, reduction_axes = gen_array_ops.broadcast_gradient_args(broadcast_shape,
+                                                            input_value_shape)
+  updates_grad_reshaped = math_ops.reduce_sum(grad,
+                                              axis=reduction_axes,
+                                              keepdims=True)
+  updates_grad = array_ops.reshape(updates_grad_reshaped, input_value_shape)
   return [updates_grad, None]
-- 
GitLab


From 4978ec3b459dc24621467facabbe808bd095dcc1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 11 Feb 2019 11:59:55 -0800
Subject: [PATCH 2267/2345] fix typo in Adam optimizer docstring

PiperOrigin-RevId: 233451176
---
 tensorflow/python/keras/optimizer_v2/adam.py | 4 ++--
 tensorflow/python/training/adam.py           | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/keras/optimizer_v2/adam.py b/tensorflow/python/keras/optimizer_v2/adam.py
index 4fa7c73615..965ae8669b 100644
--- a/tensorflow/python/keras/optimizer_v2/adam.py
+++ b/tensorflow/python/keras/optimizer_v2/adam.py
@@ -64,7 +64,7 @@ class Adam(optimizer_v2.OptimizerV2):
       $$t := 0 \text{(Initialize timestep)}$$
 
       The update rule for `variable` with gradient `g` uses an optimization
-      described at the end of section2 of the paper:
+      described at the end of section 2 of the paper:
 
       $$t := t + 1$$
       $$lr_t := \text{learning\_rate} * \sqrt{1 - beta_2^t} / (1 - beta_1^t)$$
@@ -82,7 +82,7 @@ class Adam(optimizer_v2.OptimizerV2):
       $$t := 0 \text{(Initialize timestep)}$$
 
       The update rule for `variable` with gradient `g` uses an optimization
-      described at the end of section2 of the paper:
+      described at the end of section 2 of the paper:
 
       $$t := t + 1$$
       $$lr_t := \text{learning\_rate} * \sqrt{1 - beta_2^t} / (1 - beta_1^t)$$
diff --git a/tensorflow/python/training/adam.py b/tensorflow/python/training/adam.py
index 81e03c7358..46ec3be54e 100644
--- a/tensorflow/python/training/adam.py
+++ b/tensorflow/python/training/adam.py
@@ -52,7 +52,7 @@ class AdamOptimizer(optimizer.Optimizer):
     $$t := 0 \text{(Initialize timestep)}$$
 
     The update rule for `variable` with gradient `g` uses an optimization
-    described at the end of section2 of the paper:
+    described at the end of section 2 of the paper:
 
     $$t := t + 1$$
     $$lr_t := \text{learning\_rate} * \sqrt{1 - beta_2^t} / (1 - beta_1^t)$$
-- 
GitLab


From b6d09dc1a977b9b4c67aaed68a88a5899e189f69 Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Mon, 11 Feb 2019 12:03:10 -0800
Subject: [PATCH 2268/2345] Convert speech feature generation to use micro
 testing framework

PiperOrigin-RevId: 233452007
---
 .../lite/experimental/microfrontend/lib/BUILD |  40 +++---
 .../microfrontend/lib/fft_test.cc             |  20 +--
 .../microfrontend/lib/filterbank_test.cc      | 127 ++++++++++--------
 .../microfrontend/lib/frontend_test.cc        |  45 ++++---
 .../microfrontend/lib/log_scale_test.cc       |  17 ++-
 .../microfrontend/lib/noise_reduction_test.cc |  37 +++--
 .../lib/pcan_gain_control_test.cc             |  29 ++--
 .../microfrontend/lib/window_test.cc          |  96 +++++++------
 8 files changed, 236 insertions(+), 175 deletions(-)

diff --git a/tensorflow/lite/experimental/microfrontend/lib/BUILD b/tensorflow/lite/experimental/microfrontend/lib/BUILD
index a055e52f71..8dd42fc382 100644
--- a/tensorflow/lite/experimental/microfrontend/lib/BUILD
+++ b/tensorflow/lite/experimental/microfrontend/lib/BUILD
@@ -6,6 +6,11 @@ package(
 
 licenses(["notice"])  # Apache 2.0
 
+load(
+    "//tensorflow/lite/experimental/micro/testing:micro_test.bzl",
+    "tflite_micro_cc_test",
+)
+
 cc_library(
     name = "bits",
     hdrs = ["bits.h"],
@@ -117,72 +122,65 @@ cc_library(
     ],
 )
 
-cc_test(
+tflite_micro_cc_test(
     name = "fft_test",
-    size = "small",
     srcs = ["fft_test.cc"],
     deps = [
         ":fft",
-        "@com_google_googletest//:gtest_main",
+        "//tensorflow/lite/experimental/micro/testing:micro_test",
     ],
 )
 
-cc_test(
+tflite_micro_cc_test(
     name = "filterbank_test",
-    size = "small",
     srcs = ["filterbank_test.cc"],
     deps = [
         ":filterbank",
-        "@com_google_googletest//:gtest_main",
+        "//tensorflow/lite/experimental/micro/testing:micro_test",
     ],
 )
 
-cc_test(
+tflite_micro_cc_test(
     name = "frontend_test",
-    size = "small",
     srcs = ["frontend_test.cc"],
     deps = [
         ":frontend",
-        "@com_google_googletest//:gtest_main",
+        "//tensorflow/lite/experimental/micro/testing:micro_test",
     ],
 )
 
-cc_test(
+tflite_micro_cc_test(
     name = "log_scale_test",
-    size = "small",
     srcs = ["log_scale_test.cc"],
     deps = [
         ":log_scale",
-        "@com_google_googletest//:gtest_main",
+        "//tensorflow/lite/experimental/micro/testing:micro_test",
     ],
 )
 
-cc_test(
+tflite_micro_cc_test(
     name = "noise_reduction_test",
-    size = "small",
     srcs = ["noise_reduction_test.cc"],
     deps = [
         ":noise_reduction",
-        "@com_google_googletest//:gtest_main",
+        "//tensorflow/lite/experimental/micro/testing:micro_test",
     ],
 )
 
-cc_test(
+tflite_micro_cc_test(
     name = "pcan_gain_control_test",
-    size = "small",
     srcs = ["pcan_gain_control_test.cc"],
     deps = [
         ":pcan_gain_control",
-        "@com_google_googletest//:gtest_main",
+        "//tensorflow/lite/experimental/micro/testing:micro_test",
     ],
 )
 
-cc_test(
+tflite_micro_cc_test(
     name = "window_test",
-    size = "small",
     srcs = ["window_test.cc"],
     deps = [
         ":window",
-        "@com_google_googletest//:gtest_main",
+        "//tensorflow/lite/experimental/micro/testing:micro_test",
     ],
 )
diff --git a/tensorflow/lite/experimental/microfrontend/lib/fft_test.cc b/tensorflow/lite/experimental/microfrontend/lib/fft_test.cc
index 1b754c1b4c..ec1f247ba2 100644
--- a/tensorflow/lite/experimental/microfrontend/lib/fft_test.cc
+++ b/tensorflow/lite/experimental/microfrontend/lib/fft_test.cc
@@ -15,8 +15,7 @@ limitations under the License.
 #include "tensorflow/lite/experimental/microfrontend/lib/fft.h"
 #include "tensorflow/lite/experimental/microfrontend/lib/fft_util.h"
 
-#include <gmock/gmock.h>
-#include <gtest/gtest.h>
+#include "tensorflow/lite/experimental/micro/testing/micro_test.h"
 
 namespace {
 
@@ -25,9 +24,13 @@ const int16_t kFakeWindow[] = {
     0, -28328, 0, 21447, 0, -13312, 0, 5943,   0, -1152, 0};
 const int kScaleShift = 0;
 
-TEST(FftTest, CheckOutputValues) {
+}  // namespace
+
+TF_LITE_MICRO_TESTS_BEGIN
+
+TF_LITE_MICRO_TEST(FftTest_CheckOutputValues) {
   struct FftState state;
-  ASSERT_TRUE(
+  TF_LITE_MICRO_EXPECT(
       FftPopulateState(&state, sizeof(kFakeWindow) / sizeof(kFakeWindow[0])));
 
   FftInit(&state);
@@ -37,14 +40,15 @@ TEST(FftTest, CheckOutputValues) {
       {0, 0},    {-10, 9},     {-20, 0},   {-9, -10},     {0, 25},  {-119, 119},
       {-887, 0}, {3000, 3000}, {0, -6401}, {-3000, 3000}, {886, 0}, {118, 119},
       {0, 25},   {9, -10},     {19, 0},    {9, 9},        {0, 0}};
-  ASSERT_EQ(state.fft_size / 2 + 1, sizeof(expected) / sizeof(expected[0]));
+  TF_LITE_MICRO_EXPECT_EQ(state.fft_size / 2 + 1,
+                          sizeof(expected) / sizeof(expected[0]));
   int i;
   for (i = 0; i <= state.fft_size / 2; ++i) {
-    EXPECT_EQ(state.output[i].real, expected[i].real);
-    EXPECT_EQ(state.output[i].imag, expected[i].imag);
+    TF_LITE_MICRO_EXPECT_EQ(state.output[i].real, expected[i].real);
+    TF_LITE_MICRO_EXPECT_EQ(state.output[i].imag, expected[i].imag);
   }
 
   FftFreeStateContents(&state);
 }
 
-}  // namespace
+TF_LITE_MICRO_TESTS_END
diff --git a/tensorflow/lite/experimental/microfrontend/lib/filterbank_test.cc b/tensorflow/lite/experimental/microfrontend/lib/filterbank_test.cc
index 41f0064d4f..16257aa11a 100644
--- a/tensorflow/lite/experimental/microfrontend/lib/filterbank_test.cc
+++ b/tensorflow/lite/experimental/microfrontend/lib/filterbank_test.cc
@@ -17,8 +17,7 @@ limitations under the License.
 
 #include <cstring>
 
-#include <gmock/gmock.h>
-#include <gtest/gtest.h>
+#include "tensorflow/lite/experimental/micro/testing/micro_test.h"
 
 namespace {
 
@@ -33,9 +32,9 @@ const uint64_t kWork[] = {1835887, 61162970173, 258694800000};
 const int kScaleShift = 0;
 
 // Test filterbank generation using scaled-down defaults.
-class FilterbankTest : public ::testing::Test {
- protected:
-  FilterbankTest() {
+class FilterbankTestConfig {
+ public:
+  FilterbankTestConfig() {
     config_.num_channels = 2;
     config_.lower_band_limit = 8.0;
     config_.upper_band_limit = 450.0;
@@ -44,110 +43,124 @@ class FilterbankTest : public ::testing::Test {
   struct FilterbankConfig config_;
 };
 
-TEST_F(FilterbankTest, CheckStartIndex) {
+}  // namespace
+
+TF_LITE_MICRO_TESTS_BEGIN
+
+TF_LITE_MICRO_TEST(FilterbankTest_CheckStartIndex) {
+  FilterbankTestConfig config;
   struct FilterbankState state;
-  ASSERT_TRUE(
-      FilterbankPopulateState(&config_, &state, kSampleRate, kSpectrumSize));
+  TF_LITE_MICRO_EXPECT(FilterbankPopulateState(&config.config_, &state,
+                                               kSampleRate, kSpectrumSize));
 
-  EXPECT_EQ(state.start_index, kStartIndex);
+  TF_LITE_MICRO_EXPECT_EQ(state.start_index, kStartIndex);
 
   FilterbankFreeStateContents(&state);
 }
 
-TEST_F(FilterbankTest, CheckEndIndex) {
+TF_LITE_MICRO_TEST(FilterbankTest_CheckEndIndex) {
+  FilterbankTestConfig config;
   struct FilterbankState state;
-  ASSERT_TRUE(
-      FilterbankPopulateState(&config_, &state, kSampleRate, kSpectrumSize));
+  TF_LITE_MICRO_EXPECT(FilterbankPopulateState(&config.config_, &state,
+                                               kSampleRate, kSpectrumSize));
 
-  EXPECT_EQ(state.end_index, kEndIndex);
+  TF_LITE_MICRO_EXPECT_EQ(state.end_index, kEndIndex);
 
   FilterbankFreeStateContents(&state);
 }
 
-TEST_F(FilterbankTest, CheckChannelFrequencyStarts) {
+TF_LITE_MICRO_TEST(FilterbankTest_CheckChannelFrequencyStarts) {
+  FilterbankTestConfig config;
   struct FilterbankState state;
-  ASSERT_TRUE(
-      FilterbankPopulateState(&config_, &state, kSampleRate, kSpectrumSize));
+  TF_LITE_MICRO_EXPECT(FilterbankPopulateState(&config.config_, &state,
+                                               kSampleRate, kSpectrumSize));
 
   const int16_t expected[] = {0, 4, 8};
-  ASSERT_EQ(state.num_channels + 1, sizeof(expected) / sizeof(expected[0]));
+  TF_LITE_MICRO_EXPECT_EQ(state.num_channels + 1,
+                          sizeof(expected) / sizeof(expected[0]));
   int i;
   for (i = 0; i <= state.num_channels; ++i) {
-    EXPECT_EQ(state.channel_frequency_starts[i], expected[i]);
+    TF_LITE_MICRO_EXPECT_EQ(state.channel_frequency_starts[i], expected[i]);
   }
 
   FilterbankFreeStateContents(&state);
 }
 
-TEST_F(FilterbankTest, CheckChannelWeightStarts) {
+TF_LITE_MICRO_TEST(FilterbankTest_CheckChannelWeightStarts) {
+  FilterbankTestConfig config;
   struct FilterbankState state;
-  ASSERT_TRUE(
-      FilterbankPopulateState(&config_, &state, kSampleRate, kSpectrumSize));
+  TF_LITE_MICRO_EXPECT(FilterbankPopulateState(&config.config_, &state,
+                                               kSampleRate, kSpectrumSize));
 
   const int16_t expected[] = {0, 8, 16};
-  ASSERT_EQ(state.num_channels + 1, sizeof(expected) / sizeof(expected[0]));
+  TF_LITE_MICRO_EXPECT_EQ(state.num_channels + 1,
+                          sizeof(expected) / sizeof(expected[0]));
   int i;
   for (i = 0; i <= state.num_channels; ++i) {
-    EXPECT_EQ(state.channel_weight_starts[i], expected[i]);
+    TF_LITE_MICRO_EXPECT_EQ(state.channel_weight_starts[i], expected[i]);
   }
 
   FilterbankFreeStateContents(&state);
 }
 
-TEST_F(FilterbankTest, CheckChannelWidths) {
+TF_LITE_MICRO_TEST(FilterbankTest_CheckChannelWidths) {
+  FilterbankTestConfig config;
   struct FilterbankState state;
-  ASSERT_TRUE(
-      FilterbankPopulateState(&config_, &state, kSampleRate, kSpectrumSize));
+  TF_LITE_MICRO_EXPECT(FilterbankPopulateState(&config.config_, &state,
+                                               kSampleRate, kSpectrumSize));
 
   const int16_t expected[] = {8, 8, 8};
-  ASSERT_EQ(state.num_channels + 1, sizeof(expected) / sizeof(expected[0]));
+  TF_LITE_MICRO_EXPECT_EQ(state.num_channels + 1,
+                          sizeof(expected) / sizeof(expected[0]));
   int i;
   for (i = 0; i <= state.num_channels; ++i) {
-    EXPECT_EQ(state.channel_widths[i], expected[i]);
+    TF_LITE_MICRO_EXPECT_EQ(state.channel_widths[i], expected[i]);
   }
 
   FilterbankFreeStateContents(&state);
 }
 
-TEST_F(FilterbankTest, CheckWeights) {
+TF_LITE_MICRO_TEST(FilterbankTest_CheckWeights) {
+  FilterbankTestConfig config;
   struct FilterbankState state;
-  ASSERT_TRUE(
-      FilterbankPopulateState(&config_, &state, kSampleRate, kSpectrumSize));
+  TF_LITE_MICRO_EXPECT(FilterbankPopulateState(&config.config_, &state,
+                                               kSampleRate, kSpectrumSize));
 
   const int16_t expected[] = {0, 3277, 2217, 1200, 222,  0,   0,   0,
                               0, 3376, 2468, 1591, 744,  0,   0,   0,
                               0, 4020, 3226, 2456, 1708, 983, 277, 0};
-  ASSERT_EQ(state.channel_weight_starts[state.num_channels] +
-                state.channel_widths[state.num_channels],
-            sizeof(expected) / sizeof(expected[0]));
+  TF_LITE_MICRO_EXPECT_EQ(state.channel_weight_starts[state.num_channels] +
+                              state.channel_widths[state.num_channels],
+                          sizeof(expected) / sizeof(expected[0]));
   int i;
   for (i = 0; i < sizeof(expected) / sizeof(expected[0]); ++i) {
-    EXPECT_EQ(state.weights[i], expected[i]);
+    TF_LITE_MICRO_EXPECT_EQ(state.weights[i], expected[i]);
   }
 
   FilterbankFreeStateContents(&state);
 }
 
-TEST_F(FilterbankTest, CheckUnweights) {
+TF_LITE_MICRO_TEST(FilterbankTest_CheckUnweights) {
+  FilterbankTestConfig config;
   struct FilterbankState state;
-  ASSERT_TRUE(
-      FilterbankPopulateState(&config_, &state, kSampleRate, kSpectrumSize));
+  TF_LITE_MICRO_EXPECT(FilterbankPopulateState(&config.config_, &state,
+                                               kSampleRate, kSpectrumSize));
 
   const int16_t expected[] = {0, 819, 1879, 2896, 3874, 0,    0,    0,
                               0, 720, 1628, 2505, 3352, 0,    0,    0,
                               0, 76,  870,  1640, 2388, 3113, 3819, 0};
-  ASSERT_EQ(state.channel_weight_starts[state.num_channels] +
-                state.channel_widths[state.num_channels],
-            sizeof(expected) / sizeof(expected[0]));
+  TF_LITE_MICRO_EXPECT_EQ(state.channel_weight_starts[state.num_channels] +
+                              state.channel_widths[state.num_channels],
+                          sizeof(expected) / sizeof(expected[0]));
   int i;
   for (i = 0; i < sizeof(expected) / sizeof(expected[0]); ++i) {
-    EXPECT_EQ(state.unweights[i], expected[i]);
+    TF_LITE_MICRO_EXPECT_EQ(state.unweights[i], expected[i]);
   }
 
   FilterbankFreeStateContents(&state);
 }
 
-TEST_F(FilterbankTest, CheckConvertFftComplexToEnergy) {
+TF_LITE_MICRO_TEST(FilterbankTest_CheckConvertFftComplexToEnergy) {
   struct FilterbankState state;
   state.start_index = kStartIndex;
   state.end_index = kEndIndex;
@@ -161,42 +174,46 @@ TEST_F(FilterbankTest, CheckConvertFftComplexToEnergy) {
 
   int i;
   for (i = state.start_index; i < state.end_index; ++i) {
-    EXPECT_EQ(energy[i], kEnergy[i]);
+    TF_LITE_MICRO_EXPECT_EQ(energy[i], kEnergy[i]);
   }
 }
 
-TEST_F(FilterbankTest, CheckAccumulateChannels) {
+TF_LITE_MICRO_TEST(FilterbankTest_CheckAccumulateChannels) {
+  FilterbankTestConfig config;
   struct FilterbankState state;
-  ASSERT_TRUE(
-      FilterbankPopulateState(&config_, &state, kSampleRate, kSpectrumSize));
+  TF_LITE_MICRO_EXPECT(FilterbankPopulateState(&config.config_, &state,
+                                               kSampleRate, kSpectrumSize));
 
   FilterbankAccumulateChannels(&state, kEnergy);
 
-  ASSERT_EQ(state.num_channels + 1, sizeof(kWork) / sizeof(kWork[0]));
+  TF_LITE_MICRO_EXPECT_EQ(state.num_channels + 1,
+                          sizeof(kWork) / sizeof(kWork[0]));
   int i;
   for (i = 0; i <= state.num_channels; ++i) {
-    EXPECT_EQ(state.work[i], kWork[i]);
+    TF_LITE_MICRO_EXPECT_EQ(state.work[i], kWork[i]);
   }
 
   FilterbankFreeStateContents(&state);
 }
 
-TEST_F(FilterbankTest, CheckSqrt) {
+TF_LITE_MICRO_TEST(FilterbankTest_CheckSqrt) {
+  FilterbankTestConfig config;
   struct FilterbankState state;
-  ASSERT_TRUE(
-      FilterbankPopulateState(&config_, &state, kSampleRate, kSpectrumSize));
+  TF_LITE_MICRO_EXPECT(FilterbankPopulateState(&config.config_, &state,
+                                               kSampleRate, kSpectrumSize));
   std::memcpy(state.work, kWork, sizeof(kWork));
 
   uint32_t* scaled_filterbank = FilterbankSqrt(&state, kScaleShift);
 
   const uint32_t expected[] = {247311, 508620};
-  ASSERT_EQ(state.num_channels, sizeof(expected) / sizeof(expected[0]));
+  TF_LITE_MICRO_EXPECT_EQ(state.num_channels,
+                          sizeof(expected) / sizeof(expected[0]));
   int i;
   for (i = 0; i < state.num_channels; ++i) {
-    EXPECT_EQ(scaled_filterbank[i], expected[i]);
+    TF_LITE_MICRO_EXPECT_EQ(scaled_filterbank[i], expected[i]);
   }
 
   FilterbankFreeStateContents(&state);
 }
 
-}  // namespace
+TF_LITE_MICRO_TESTS_END
diff --git a/tensorflow/lite/experimental/microfrontend/lib/frontend_test.cc b/tensorflow/lite/experimental/microfrontend/lib/frontend_test.cc
index a6faa1fc1f..568484f14d 100644
--- a/tensorflow/lite/experimental/microfrontend/lib/frontend_test.cc
+++ b/tensorflow/lite/experimental/microfrontend/lib/frontend_test.cc
@@ -15,8 +15,7 @@ limitations under the License.
 #include "tensorflow/lite/experimental/microfrontend/lib/frontend.h"
 #include "tensorflow/lite/experimental/microfrontend/lib/frontend_util.h"
 
-#include <gmock/gmock.h>
-#include <gtest/gtest.h>
+#include "tensorflow/lite/experimental/micro/testing/micro_test.h"
 
 namespace {
 
@@ -29,9 +28,9 @@ const int16_t kFakeAudioData[] = {
     0, 32767, 0, -32768, 0, 32767, 0, -32768, 0, 32767, 0, -32768};
 
 // Test end-to-end frontend behaviors.
-class FrontendTest : public ::testing::Test {
- protected:
-  FrontendTest() {
+class FrontendTestConfig {
+ public:
+  FrontendTestConfig() {
     config_.window.size_ms = 25;
     config_.window.step_size_ms = 10;
     config_.noise_reduction.smoothing_bits = 10;
@@ -53,9 +52,15 @@ class FrontendTest : public ::testing::Test {
   struct FrontendConfig config_;
 };
 
-TEST_F(FrontendTest, CheckOutputValues) {
+}  // namespace
+
+TF_LITE_MICRO_TESTS_BEGIN
+
+TF_LITE_MICRO_TEST(FrontendTest_CheckOutputValues) {
+  FrontendTestConfig config;
   struct FrontendState state;
-  ASSERT_TRUE(FrontendPopulateState(&config_, &state, kSampleRate));
+  TF_LITE_MICRO_EXPECT(
+      FrontendPopulateState(&config.config_, &state, kSampleRate));
   size_t num_samples_read;
 
   struct FrontendOutput output = FrontendProcessSamples(
@@ -63,18 +68,20 @@ TEST_F(FrontendTest, CheckOutputValues) {
       sizeof(kFakeAudioData) / sizeof(kFakeAudioData[0]), &num_samples_read);
 
   const uint16_t expected[] = {479, 425};
-  ASSERT_EQ(output.size, sizeof(expected) / sizeof(expected[0]));
+  TF_LITE_MICRO_EXPECT_EQ(output.size, sizeof(expected) / sizeof(expected[0]));
   int i;
   for (i = 0; i < output.size; ++i) {
-    EXPECT_EQ(output.values[i], expected[i]);
+    TF_LITE_MICRO_EXPECT_EQ(output.values[i], expected[i]);
   }
 
   FrontendFreeStateContents(&state);
 }
 
-TEST_F(FrontendTest, CheckConsecutiveWindow) {
+TF_LITE_MICRO_TEST(FrontendTest_CheckConsecutiveWindow) {
+  FrontendTestConfig config;
   struct FrontendState state;
-  ASSERT_TRUE(FrontendPopulateState(&config_, &state, kSampleRate));
+  TF_LITE_MICRO_EXPECT(
+      FrontendPopulateState(&config.config_, &state, kSampleRate));
   size_t num_samples_read;
 
   FrontendProcessSamples(&state, kFakeAudioData,
@@ -86,18 +93,20 @@ TEST_F(FrontendTest, CheckConsecutiveWindow) {
       &num_samples_read);
 
   const int16_t expected[] = {436, 378};
-  ASSERT_EQ(output.size, sizeof(expected) / sizeof(expected[0]));
+  TF_LITE_MICRO_EXPECT_EQ(output.size, sizeof(expected) / sizeof(expected[0]));
   int i;
   for (i = 0; i < output.size; ++i) {
-    EXPECT_EQ(output.values[i], expected[i]);
+    TF_LITE_MICRO_EXPECT_EQ(output.values[i], expected[i]);
   }
 
   FrontendFreeStateContents(&state);
 }
 
-TEST_F(FrontendTest, CheckNotEnoughSamples) {
+TF_LITE_MICRO_TEST(FrontendTest_CheckNotEnoughSamples) {
+  FrontendTestConfig config;
   struct FrontendState state;
-  ASSERT_TRUE(FrontendPopulateState(&config_, &state, kSampleRate));
+  TF_LITE_MICRO_EXPECT(
+      FrontendPopulateState(&config.config_, &state, kSampleRate));
   size_t num_samples_read;
 
   FrontendProcessSamples(&state, kFakeAudioData,
@@ -113,10 +122,10 @@ TEST_F(FrontendTest, CheckNotEnoughSamples) {
           kStepSamples,
       &num_samples_read);
 
-  EXPECT_EQ(output.size, 0);
-  EXPECT_EQ(output.values, nullptr);
+  TF_LITE_MICRO_EXPECT_EQ(output.size, 0);
+  TF_LITE_MICRO_EXPECT_EQ(output.values, nullptr);
 
   FrontendFreeStateContents(&state);
 }
 
-}  // namespace
+TF_LITE_MICRO_TESTS_END
diff --git a/tensorflow/lite/experimental/microfrontend/lib/log_scale_test.cc b/tensorflow/lite/experimental/microfrontend/lib/log_scale_test.cc
index 1ea0842ec2..be52fd426a 100644
--- a/tensorflow/lite/experimental/microfrontend/lib/log_scale_test.cc
+++ b/tensorflow/lite/experimental/microfrontend/lib/log_scale_test.cc
@@ -15,15 +15,18 @@ limitations under the License.
 #include "tensorflow/lite/experimental/microfrontend/lib/log_scale.h"
 #include "tensorflow/lite/experimental/microfrontend/lib/log_scale_util.h"
 
-#include <gmock/gmock.h>
-#include <gtest/gtest.h>
+#include "tensorflow/lite/experimental/micro/testing/micro_test.h"
 
 namespace {
 
 const int kScaleShift = 6;
 const int kCorrectionBits = -1;
 
-TEST(LogScaleTest, CheckOutputValues) {
+}  // namespace
+
+TF_LITE_MICRO_TESTS_BEGIN
+
+TF_LITE_MICRO_TEST(LogScaleTest_CheckOutputValues) {
   struct LogScaleState state;
   state.enable_log = true;
   state.scale_shift = kScaleShift;
@@ -36,11 +39,11 @@ TEST(LogScaleTest, CheckOutputValues) {
   const uint16_t expected[] = {479, 425};
   int i;
   for (i = 0; i < sizeof(expected) / sizeof(expected[0]); ++i) {
-    EXPECT_EQ(output[i], expected[i]);
+    TF_LITE_MICRO_EXPECT_EQ(output[i], expected[i]);
   }
 }
 
-TEST(LogScaleTest, CheckOutputValuesNoLog) {
+TF_LITE_MICRO_TEST(LogScaleTest_CheckOutputValuesNoLog) {
   struct LogScaleState state;
   state.enable_log = false;
   state.scale_shift = kScaleShift;
@@ -53,8 +56,8 @@ TEST(LogScaleTest, CheckOutputValuesNoLog) {
   const uint16_t expected[] = {65535, 45998};
   int i;
   for (i = 0; i < sizeof(expected) / sizeof(expected[0]); ++i) {
-    EXPECT_EQ(output[i], expected[i]);
+    TF_LITE_MICRO_EXPECT_EQ(output[i], expected[i]);
   }
 }
 
-}  // namespace
+TF_LITE_MICRO_TESTS_END
diff --git a/tensorflow/lite/experimental/microfrontend/lib/noise_reduction_test.cc b/tensorflow/lite/experimental/microfrontend/lib/noise_reduction_test.cc
index 13d58b2476..ba864c427c 100644
--- a/tensorflow/lite/experimental/microfrontend/lib/noise_reduction_test.cc
+++ b/tensorflow/lite/experimental/microfrontend/lib/noise_reduction_test.cc
@@ -15,17 +15,16 @@ limitations under the License.
 #include "tensorflow/lite/experimental/microfrontend/lib/noise_reduction.h"
 #include "tensorflow/lite/experimental/microfrontend/lib/noise_reduction_util.h"
 
-#include <gmock/gmock.h>
-#include <gtest/gtest.h>
+#include "tensorflow/lite/experimental/micro/testing/micro_test.h"
 
 namespace {
 
 const int kNumChannels = 2;
 
 // Test noise reduction using default config values.
-class NoiseReductionTest : public ::testing::Test {
- protected:
-  NoiseReductionTest() {
+class NoiseReductionTestConfig {
+ public:
+  NoiseReductionTestConfig() {
     config_.smoothing_bits = 10;
     config_.even_smoothing = 0.025;
     config_.odd_smoothing = 0.06;
@@ -35,38 +34,48 @@ class NoiseReductionTest : public ::testing::Test {
   struct NoiseReductionConfig config_;
 };
 
-TEST_F(NoiseReductionTest, TestNoiseReductionEstimate) {
+}  // namespace
+
+TF_LITE_MICRO_TESTS_BEGIN
+
+TF_LITE_MICRO_TEST(NoiseReductionTest_TestNoiseReductionEstimate) {
+  NoiseReductionTestConfig config;
   struct NoiseReductionState state;
-  ASSERT_TRUE(NoiseReductionPopulateState(&config_, &state, kNumChannels));
+  TF_LITE_MICRO_EXPECT(
+      NoiseReductionPopulateState(&config.config_, &state, kNumChannels));
 
   uint32_t signal[] = {247311, 508620};
   NoiseReductionApply(&state, signal);
 
   const uint32_t expected[] = {6321887, 31248341};
-  ASSERT_EQ(state.num_channels, sizeof(expected) / sizeof(expected[0]));
+  TF_LITE_MICRO_EXPECT_EQ(state.num_channels,
+                          sizeof(expected) / sizeof(expected[0]));
   int i;
   for (i = 0; i < state.num_channels; ++i) {
-    EXPECT_EQ(state.estimate[i], expected[i]);
+    TF_LITE_MICRO_EXPECT_EQ(state.estimate[i], expected[i]);
   }
 
   NoiseReductionFreeStateContents(&state);
 }
 
-TEST_F(NoiseReductionTest, TestNoiseReduction) {
+TF_LITE_MICRO_TEST(NoiseReductionTest_TestNoiseReduction) {
+  NoiseReductionTestConfig config;
   struct NoiseReductionState state;
-  ASSERT_TRUE(NoiseReductionPopulateState(&config_, &state, kNumChannels));
+  TF_LITE_MICRO_EXPECT(
+      NoiseReductionPopulateState(&config.config_, &state, kNumChannels));
 
   uint32_t signal[] = {247311, 508620};
   NoiseReductionApply(&state, signal);
 
   const uint32_t expected[] = {241137, 478104};
-  ASSERT_EQ(state.num_channels, sizeof(expected) / sizeof(expected[0]));
+  TF_LITE_MICRO_EXPECT_EQ(state.num_channels,
+                          sizeof(expected) / sizeof(expected[0]));
   int i;
   for (i = 0; i < state.num_channels; ++i) {
-    EXPECT_EQ(signal[i], expected[i]);
+    TF_LITE_MICRO_EXPECT_EQ(signal[i], expected[i]);
   }
 
   NoiseReductionFreeStateContents(&state);
 }
 
-}  // namespace
+TF_LITE_MICRO_TESTS_END
diff --git a/tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control_test.cc b/tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control_test.cc
index 7c92d2d29d..93d7a8bcb9 100644
--- a/tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control_test.cc
+++ b/tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control_test.cc
@@ -15,8 +15,7 @@ limitations under the License.
 #include "tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control.h"
 #include "tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control_util.h"
 
-#include <gmock/gmock.h>
-#include <gtest/gtest.h>
+#include "tensorflow/lite/experimental/micro/testing/micro_test.h"
 
 namespace {
 
@@ -25,9 +24,9 @@ const int kSmoothingBits = 10;
 const int kCorrectionBits = -1;
 
 // Test pcan auto gain control using default config values.
-class PcanGainControlTest : public ::testing::Test {
- protected:
-  PcanGainControlTest() {
+class PcanGainControlTestConfig {
+ public:
+  PcanGainControlTestConfig() {
     config_.enable_pcan = 1;
     config_.strength = 0.95;
     config_.offset = 80.0;
@@ -37,24 +36,30 @@ class PcanGainControlTest : public ::testing::Test {
   struct PcanGainControlConfig config_;
 };
 
-TEST_F(PcanGainControlTest, TestPcanGainControl) {
+}  // namespace
+
+TF_LITE_MICRO_TESTS_BEGIN
+
+TF_LITE_MICRO_TEST(PcanGainControlTest_TestPcanGainControl) {
   uint32_t estimate[] = {6321887, 31248341};
+  PcanGainControlTestConfig config;
   struct PcanGainControlState state;
-  ASSERT_TRUE(PcanGainControlPopulateState(&config_, &state, estimate,
-                                           kNumChannels, kSmoothingBits,
-                                           kCorrectionBits));
+  TF_LITE_MICRO_EXPECT(PcanGainControlPopulateState(
+      &config.config_, &state, estimate, kNumChannels, kSmoothingBits,
+      kCorrectionBits));
 
   uint32_t signal[] = {241137, 478104};
   PcanGainControlApply(&state, signal);
 
   const uint32_t expected[] = {3578, 1533};
-  ASSERT_EQ(state.num_channels, sizeof(expected) / sizeof(expected[0]));
+  TF_LITE_MICRO_EXPECT_EQ(state.num_channels,
+                          sizeof(expected) / sizeof(expected[0]));
   int i;
   for (i = 0; i < state.num_channels; ++i) {
-    EXPECT_EQ(signal[i], expected[i]);
+    TF_LITE_MICRO_EXPECT_EQ(signal[i], expected[i]);
   }
 
   PcanGainControlFreeStateContents(&state);
 }
 
-}  // namespace
+TF_LITE_MICRO_TESTS_END
diff --git a/tensorflow/lite/experimental/microfrontend/lib/window_test.cc b/tensorflow/lite/experimental/microfrontend/lib/window_test.cc
index 60f11440f5..cf9df523b8 100644
--- a/tensorflow/lite/experimental/microfrontend/lib/window_test.cc
+++ b/tensorflow/lite/experimental/microfrontend/lib/window_test.cc
@@ -15,8 +15,7 @@ limitations under the License.
 #include "tensorflow/lite/experimental/microfrontend/lib/window.h"
 #include "tensorflow/lite/experimental/microfrontend/lib/window_util.h"
 
-#include <gmock/gmock.h>
-#include <gtest/gtest.h>
+#include "tensorflow/lite/experimental/micro/testing/micro_test.h"
 
 namespace {
 
@@ -29,9 +28,9 @@ const int16_t kFakeAudioData[] = {
     0, 32767, 0, -32768, 0, 32767, 0, -32768, 0, 32767, 0, -32768};
 
 // Test window function behaviors using default config values.
-class WindowTest : public ::testing::Test {
- protected:
-  WindowTest() {
+class WindowTestConfig {
+ public:
+  WindowTestConfig() {
     config_.size_ms = 25;
     config_.step_size_ms = 10;
   }
@@ -39,84 +38,98 @@ class WindowTest : public ::testing::Test {
   struct WindowConfig config_;
 };
 
-TEST_F(WindowTest, CheckCoefficients) {
+}  // namespace
+
+TF_LITE_MICRO_TESTS_BEGIN
+
+TF_LITE_MICRO_TEST(WindowState_CheckCoefficients) {
+  WindowTestConfig config;
   struct WindowState state;
-  ASSERT_TRUE(WindowPopulateState(&config_, &state, kSampleRate));
+  TF_LITE_MICRO_EXPECT(
+      WindowPopulateState(&config.config_, &state, kSampleRate));
 
   const int16_t expected[] = {16,   144,  391,  743,  1176, 1664, 2177,
                               2681, 3145, 3541, 3843, 4032, 4096, 4032,
                               3843, 3541, 3145, 2681, 2177, 1664, 1176,
                               743,  391,  144,  16};
-  ASSERT_EQ(state.size, sizeof(expected) / sizeof(expected[0]));
+  TF_LITE_MICRO_EXPECT_EQ(state.size, sizeof(expected) / sizeof(expected[0]));
   int i;
   for (i = 0; i < state.size; ++i) {
-    EXPECT_EQ(state.coefficients[i], expected[i]);
+    TF_LITE_MICRO_EXPECT_EQ(state.coefficients[i], expected[i]);
   }
 
   WindowFreeStateContents(&state);
 }
 
-TEST_F(WindowTest, CheckResidualInput) {
+TF_LITE_MICRO_TEST(WindowState_CheckResidualInput) {
+  WindowTestConfig config;
   struct WindowState state;
-  ASSERT_TRUE(WindowPopulateState(&config_, &state, kSampleRate));
+  TF_LITE_MICRO_EXPECT(
+      WindowPopulateState(&config.config_, &state, kSampleRate));
   size_t num_samples_read;
 
-  ASSERT_TRUE(WindowProcessSamples(
+  TF_LITE_MICRO_EXPECT(WindowProcessSamples(
       &state, kFakeAudioData,
       sizeof(kFakeAudioData) / sizeof(kFakeAudioData[0]), &num_samples_read));
 
   int i;
   for (i = kStepSamples; i < kWindowSamples; ++i) {
-    EXPECT_EQ(state.input[i - kStepSamples], kFakeAudioData[i]);
+    TF_LITE_MICRO_EXPECT_EQ(state.input[i - kStepSamples], kFakeAudioData[i]);
   }
 
   WindowFreeStateContents(&state);
 }
 
-TEST_F(WindowTest, CheckOutputValues) {
+TF_LITE_MICRO_TEST(WindowState_CheckOutputValues) {
+  WindowTestConfig config;
   struct WindowState state;
-  ASSERT_TRUE(WindowPopulateState(&config_, &state, kSampleRate));
+  TF_LITE_MICRO_EXPECT(
+      WindowPopulateState(&config.config_, &state, kSampleRate));
   size_t num_samples_read;
 
-  ASSERT_TRUE(WindowProcessSamples(
+  TF_LITE_MICRO_EXPECT(WindowProcessSamples(
       &state, kFakeAudioData,
       sizeof(kFakeAudioData) / sizeof(kFakeAudioData[0]), &num_samples_read));
 
   const int16_t expected[] = {
       0, 1151,   0, -5944, 0, 13311,  0, -21448, 0, 28327, 0, -32256, 0, 32255,
       0, -28328, 0, 21447, 0, -13312, 0, 5943,   0, -1152, 0};
-  ASSERT_EQ(state.size, sizeof(expected) / sizeof(expected[0]));
+  TF_LITE_MICRO_EXPECT_EQ(state.size, sizeof(expected) / sizeof(expected[0]));
   int i;
   for (i = 0; i < state.size; ++i) {
-    EXPECT_EQ(state.output[i], expected[i]);
+    TF_LITE_MICRO_EXPECT_EQ(state.output[i], expected[i]);
   }
 
   WindowFreeStateContents(&state);
 }
 
-TEST_F(WindowTest, CheckMaxAbsValue) {
+TF_LITE_MICRO_TEST(WindowState_CheckMaxAbsValue) {
+  WindowTestConfig config;
   struct WindowState state;
-  ASSERT_TRUE(WindowPopulateState(&config_, &state, kSampleRate));
+  TF_LITE_MICRO_EXPECT(
+      WindowPopulateState(&config.config_, &state, kSampleRate));
   size_t num_samples_read;
 
-  ASSERT_TRUE(WindowProcessSamples(
+  TF_LITE_MICRO_EXPECT(WindowProcessSamples(
       &state, kFakeAudioData,
       sizeof(kFakeAudioData) / sizeof(kFakeAudioData[0]), &num_samples_read));
 
-  EXPECT_EQ(state.max_abs_output_value, 32256);
+  TF_LITE_MICRO_EXPECT_EQ(state.max_abs_output_value, 32256);
 
   WindowFreeStateContents(&state);
 }
 
-TEST_F(WindowTest, CheckConsecutiveWindow) {
+TF_LITE_MICRO_TEST(WindowState_CheckConsecutiveWindow) {
+  WindowTestConfig config;
   struct WindowState state;
-  ASSERT_TRUE(WindowPopulateState(&config_, &state, kSampleRate));
+  TF_LITE_MICRO_EXPECT(
+      WindowPopulateState(&config.config_, &state, kSampleRate));
   size_t num_samples_read;
 
-  ASSERT_TRUE(WindowProcessSamples(
+  TF_LITE_MICRO_EXPECT(WindowProcessSamples(
       &state, kFakeAudioData,
       sizeof(kFakeAudioData) / sizeof(kFakeAudioData[0]), &num_samples_read));
-  ASSERT_TRUE(WindowProcessSamples(
+  TF_LITE_MICRO_EXPECT(WindowProcessSamples(
       &state, kFakeAudioData + kWindowSamples,
       sizeof(kFakeAudioData) / sizeof(kFakeAudioData[0]) - kWindowSamples,
       &num_samples_read));
@@ -124,38 +137,41 @@ TEST_F(WindowTest, CheckConsecutiveWindow) {
   const int16_t expected[] = {
       0, -1152, 0, 5943,   0, -13312, 0, 21447, 0, -28328, 0, 32255, 0, -32256,
       0, 28327, 0, -21448, 0, 13311,  0, -5944, 0, 1151,   0};
-  ASSERT_EQ(state.size, sizeof(expected) / sizeof(expected[0]));
+  TF_LITE_MICRO_EXPECT_EQ(state.size, sizeof(expected) / sizeof(expected[0]));
   int i;
   for (i = 0; i < state.size; ++i) {
-    EXPECT_EQ(state.output[i], expected[i]);
+    TF_LITE_MICRO_EXPECT_EQ(state.output[i], expected[i]);
   }
 
   WindowFreeStateContents(&state);
 }
 
-TEST_F(WindowTest, CheckNotEnoughSamples) {
+TF_LITE_MICRO_TEST(WindowState_CheckNotEnoughSamples) {
+  WindowTestConfig config;
   struct WindowState state;
-  ASSERT_TRUE(WindowPopulateState(&config_, &state, kSampleRate));
+  TF_LITE_MICRO_EXPECT(
+      WindowPopulateState(&config.config_, &state, kSampleRate));
   size_t num_samples_read;
 
-  ASSERT_TRUE(WindowProcessSamples(
+  TF_LITE_MICRO_EXPECT(WindowProcessSamples(
       &state, kFakeAudioData,
       sizeof(kFakeAudioData) / sizeof(kFakeAudioData[0]), &num_samples_read));
-  ASSERT_TRUE(WindowProcessSamples(
+  TF_LITE_MICRO_EXPECT(WindowProcessSamples(
       &state, kFakeAudioData + kWindowSamples,
       sizeof(kFakeAudioData) / sizeof(kFakeAudioData[0]) - kWindowSamples,
       &num_samples_read));
-  ASSERT_FALSE(WindowProcessSamples(
-      &state, kFakeAudioData + kWindowSamples + kStepSamples,
-      sizeof(kFakeAudioData) / sizeof(kFakeAudioData[0]) - kWindowSamples -
-          kStepSamples,
-      &num_samples_read));
-
-  EXPECT_EQ(
+  TF_LITE_MICRO_EXPECT_EQ(
+      false, WindowProcessSamples(
+                 &state, kFakeAudioData + kWindowSamples + kStepSamples,
+                 sizeof(kFakeAudioData) / sizeof(kFakeAudioData[0]) -
+                     kWindowSamples - kStepSamples,
+                 &num_samples_read));
+
+  TF_LITE_MICRO_EXPECT_EQ(
       state.input_used,
       sizeof(kFakeAudioData) / sizeof(kFakeAudioData[0]) - 2 * kStepSamples);
 
   WindowFreeStateContents(&state);
 }
 
-}  // namespace
+TF_LITE_MICRO_TESTS_END
-- 
GitLab


From cdbf430c86d713db608955390100f71958ef5dab Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Mon, 11 Feb 2019 12:06:43 -0800
Subject: [PATCH 2269/2345] Add VS Code support for generated projects, and CI
 build script

PiperOrigin-RevId: 233452779
---
 .../tools/ci_build/ci_build_micro_projects.sh | 38 ++++++++++++++++++
 .../experimental/micro/tools/make/Makefile    | 14 ++++++-
 .../micro/tools/make/helper_functions.inc     |  6 +++
 .../tools/make/templates/tasks.json.make.tpl  | 16 ++++++++
 .../tools/make/templates/tasks.json.mbed.tpl  | 39 +++++++++++++++++++
 5 files changed, 111 insertions(+), 2 deletions(-)
 create mode 100755 tensorflow/lite/experimental/micro/tools/ci_build/ci_build_micro_projects.sh
 create mode 100644 tensorflow/lite/experimental/micro/tools/make/templates/tasks.json.make.tpl
 create mode 100644 tensorflow/lite/experimental/micro/tools/make/templates/tasks.json.mbed.tpl

diff --git a/tensorflow/lite/experimental/micro/tools/ci_build/ci_build_micro_projects.sh b/tensorflow/lite/experimental/micro/tools/ci_build/ci_build_micro_projects.sh
new file mode 100755
index 0000000000..dcec7269bd
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/tools/ci_build/ci_build_micro_projects.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+#
+# Creates the project file distributions for the TensorFlow Lite Micro test and
+# example targets aimed at embedded platforms.
+#
+# Usage: ci_build_micro_projects.sh <TARGET OS> <TAGS>
+#
+# For example:
+# ci_build_micro_projects.sh mbed "CMSIS disco_f746ng"
+
+set -e
+set -x
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR=${SCRIPT_DIR}/../../../../../..
+cd ${ROOT_DIR}
+pwd
+
+tensorflow/lite/experimental/micro/tools/make/download_dependencies.sh
+
+make -f tensorflow/lite/experimental/micro/tools/make/Makefile \
+  TARGET=${1} \
+  TAGS="${2}" \
+  generate_projects
diff --git a/tensorflow/lite/experimental/micro/tools/make/Makefile b/tensorflow/lite/experimental/micro/tools/make/Makefile
index 3457fafb9d..127d822a26 100644
--- a/tensorflow/lite/experimental/micro/tools/make/Makefile
+++ b/tensorflow/lite/experimental/micro/tools/make/Makefile
@@ -126,12 +126,20 @@ third_party/flatbuffers/LICENSE.txt
 
 MAKE_PROJECT_FILES := \
   README_MAKE.md \
-  Makefile
+  Makefile \
+  .vscode/tasks.json
 
 MBED_PROJECT_FILES := \
   README_MBED.md \
   mbed-os.lib \
-  mbed_app.json
+  mbed_app.json \
+  .vscode/tasks.json
+
+KEIL_PROJECT_FILES := \
+  README_KEIL.md \
+  keil_project.uvprojx
+
+ALL_PROJECT_TARGETS :=
 
 KEIL_PROJECT_FILES := \
   README_KEIL.md \
@@ -215,6 +223,8 @@ $(eval $(call microlite_test,$(notdir $(basename $(TEST_TARGET))),$(TEST_TARGET)
 
 test: $(MICROLITE_TEST_TARGETS)
 
+generate_projects: $(ALL_PROJECT_TARGETS)
+
 # Gets rid of all generated files.
 clean:
 	rm -rf $(MAKEFILE_DIR)/gen
diff --git a/tensorflow/lite/experimental/micro/tools/make/helper_functions.inc b/tensorflow/lite/experimental/micro/tools/make/helper_functions.inc
index 28f7618672..89f473b310 100644
--- a/tensorflow/lite/experimental/micro/tools/make/helper_functions.inc
+++ b/tensorflow/lite/experimental/micro/tools/make/helper_functions.inc
@@ -78,7 +78,13 @@ $(PRJDIR)$(3)/$(1)/keil_project.uvprojx: tensorflow/lite/experimental/micro/tool
         --input_template=$$< --output_file=$$@ --executable=$(3) \
         --srcs="$(4)" --hdrs="$(5)" --include_paths="$$(PROJECT_INCLUDES)"
 
+$(PRJDIR)$(3)/$(1)/.vscode/tasks.json : tensorflow/lite/experimental/micro/tools/make/templates/tasks.json.$(1).tpl
+	@mkdir -p $$(dir $$@)
+	cp $$< $$@
+
 generate_$(3)_$(1)_project: $(addprefix $(PRJDIR)$(3)/$(1)/, $(4) $(5) $(2))
+
+ALL_PROJECT_TARGETS += generate_$(3)_$(1)_project
 endef
 
 # Specialized version of generate_project for TF Lite Micro test targets that
diff --git a/tensorflow/lite/experimental/micro/tools/make/templates/tasks.json.make.tpl b/tensorflow/lite/experimental/micro/tools/make/templates/tasks.json.make.tpl
new file mode 100644
index 0000000000..141994d854
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/tools/make/templates/tasks.json.make.tpl
@@ -0,0 +1,16 @@
+{
+    // See https://go.microsoft.com/fwlink/?LinkId=733558
+    // for the documentation about the tasks.json format
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "Make Build",
+            "type": "shell",
+            "command": "make",
+            "group": {
+                "kind": "build",
+                "isDefault": true
+                }
+        }
+    ]
+}
\ No newline at end of file
diff --git a/tensorflow/lite/experimental/micro/tools/make/templates/tasks.json.mbed.tpl b/tensorflow/lite/experimental/micro/tools/make/templates/tasks.json.mbed.tpl
new file mode 100644
index 0000000000..616f3b2318
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/tools/make/templates/tasks.json.mbed.tpl
@@ -0,0 +1,39 @@
+{
+    // See https://go.microsoft.com/fwlink/?LinkId=733558
+    // for the documentation about the tasks.json format
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "Mbed Config Root",
+            "type": "shell",
+            "command": "mbed config root .",
+        },
+        {
+            "label": "Mbed Deploy",
+            "type": "shell",
+            "command": "mbed deploy",
+        },
+        {
+            "label": "Mbed Patch C++11",
+            "type": "shell",
+            "command": "python",
+            "args": [
+                "-c",
+                "import fileinput, glob;\nfor filename in glob.glob(\"mbed-os/tools/profiles/*.json\"):\n  for line in fileinput.input(filename, inplace=True):\n    print line.replace(\"\\\"-std=gnu++98\\\"\",\"\\\"-std=c++11\\\", \\\"-fpermissive\\\"\")"
+            ]
+        },
+        {
+            "label": "Mbed Init",
+            "dependsOn": ["Mbed Config Root", "Mbed Deploy", "Mbed Patch C++11"]
+        },
+        {
+            "label": "Mbed build",
+            "type": "shell",
+            "command": "mbed compile -m auto -t GCC_ARM",
+            "group": {
+                "kind": "build",
+                "isDefault": true
+                }
+        }
+    ]
+}
\ No newline at end of file
-- 
GitLab


From 4bdb229d2b1f1be776615875085df0a47927862c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 11 Feb 2019 12:26:30 -0800
Subject: [PATCH 2270/2345] Begin defaulting
 GPUOptions.experimental.pending_cap to 8.  This has the effect of limiting
 the maximum number of pending GPU kernels (when using the default single
 compute stream configuration) to no more than 8.  Testing with benchmark
 programs in single-GPU and distributed configurations suggests this always
 yields better performance than unlimited queuing, probabaly because of
 effects on GPU/CPU i/o which needs to wait on pending kernels for memory
 allocation safety.

PiperOrigin-RevId: 233456126
---
 tensorflow/core/common_runtime/gpu/gpu_device.cc | 6 ++++++
 tensorflow/core/protobuf/config.proto            | 4 +++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.cc b/tensorflow/core/common_runtime/gpu/gpu_device.cc
index 179f6132fa..6aae89c1fe 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.cc
@@ -277,6 +277,12 @@ BaseGPUDevice::BaseGPUDevice(const SessionOptions& options, const string& name,
       max_streams_(max_streams) {
   GPUProcessState::singleton()->EnableGPUDevice();
   pending_cap_ = options.config.gpu_options().experimental().pending_cap();
+  if (pending_cap_ == 0) {
+    // Auto-configure.  Testing with benchmark models suggests that the optimal
+    // pending_cap value varies by model but something in the range [2, 16] is
+    // always better than unlimited.
+    pending_cap_ = 8;
+  }
   timestamped_allocator_ =
       options.config.gpu_options().experimental().timestamped_allocator();
   if (timestamped_allocator_ || pending_cap_ > 0) {
diff --git a/tensorflow/core/protobuf/config.proto b/tensorflow/core/protobuf/config.proto
index 44e98542ec..6e9c3c2e27 100644
--- a/tensorflow/core/protobuf/config.proto
+++ b/tensorflow/core/protobuf/config.proto
@@ -164,7 +164,9 @@ message GPUOptions {
     bool timestamped_allocator = 5;
 
     // If > 0 limit the number of pending kernels on any compute
-    // stream to this number.
+    // stream to this number.  If == 0 then the runtime will automatically
+    // pick a value.  If == -1 then this feature will be disabled, i.e.
+    // there will be no limit.
     int32 pending_cap = 6;
   }
 
-- 
GitLab


From 049848467bb65e8c31c19781bed27845847772ff Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Mon, 11 Feb 2019 12:46:01 -0800
Subject: [PATCH 2271/2345] Allow init-from-placeholder in tf.function +
 v1-style Graphs

PiperOrigin-RevId: 233459500
---
 tensorflow/python/eager/def_function.py      | 10 ++++++--
 tensorflow/python/eager/def_function_test.py | 18 ++++++++++++++
 tensorflow/python/eager/lift_to_graph.py     | 26 +++++++++++++++-----
 tensorflow/python/framework/func_graph.py    | 11 +++++++--
 4 files changed, 55 insertions(+), 10 deletions(-)

diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py
index f5bb8cfd57..7b8ceb979b 100644
--- a/tensorflow/python/eager/def_function.py
+++ b/tensorflow/python/eager/def_function.py
@@ -96,7 +96,7 @@ class UnliftedInitializerVariable(resource_variable_ops.ResourceVariable):
         shape and `validate_shape` is `True`.
       RuntimeError: If called outside of a function definition.
     """
-    if context.executing_eagerly():
+    if not ops.inside_function():
       # If we've been init_scope()d out of the function definition nothing to do
       # here; we can't really do the capturing or conditional logic.
       resource_variable_ops.ResourceVariable.__init__(
@@ -156,8 +156,14 @@ class UnliftedInitializerVariable(resource_variable_ops.ResourceVariable):
       if self._in_graph_mode:
         with ops.init_scope():
           outer_graph = ops.get_default_graph()
+        func_graph = ops.get_default_graph()
+        function_placeholders = (
+            func_graph.inputs + func_graph.internal_captures)
+        placeholder_ops = set(
+            [tensor.op for tensor in function_placeholders])
         lifted_initializer = lift_to_graph.lift_to_graph(
-            initial_value, outer_graph)[initial_value]
+            initial_value, outer_graph,
+            disallowed_placeholders=placeholder_ops)[initial_value]
         with ops.init_scope():
           self._initial_value = lifted_initializer
           with ops.name_scope("IsInitialized"):
diff --git a/tensorflow/python/eager/def_function_test.py b/tensorflow/python/eager/def_function_test.py
index 6680f0b3e6..462aa8aa0a 100644
--- a/tensorflow/python/eager/def_function_test.py
+++ b/tensorflow/python/eager/def_function_test.py
@@ -451,6 +451,24 @@ class DefFunctionTest(test.TestCase):
     func._decorate(decorator)
     self.assertEqual(func().numpy(), 2)
 
+  def testLiftPlaceholderInitializedVariable(self):
+    with ops.Graph().as_default():
+      var_list = []
+
+      @def_function.function
+      def use_variable():
+        if not var_list:
+          initial_value = array_ops.placeholder(shape=[], dtype=dtypes.float32)
+          v = variables.Variable(initial_value)
+          var_list.append(v)
+        return var_list[0] + 1.
+
+      var_plus_one = use_variable()
+      with self.session() as session:
+        init_op = var_list[0].initializer
+        session.run(init_op, feed_dict={init_op.inputs[1]: 2.})
+        self.assertEqual(3., session.run(var_plus_one))
+
   def testDecorate_rejectedAfterTrace(self):
     func = def_function.function(lambda: 1)
     self.assertEqual(func().numpy(), 1)
diff --git a/tensorflow/python/eager/lift_to_graph.py b/tensorflow/python/eager/lift_to_graph.py
index ad62e6d10a..e48001eddc 100644
--- a/tensorflow/python/eager/lift_to_graph.py
+++ b/tensorflow/python/eager/lift_to_graph.py
@@ -40,8 +40,24 @@ class UnliftableError(Exception):
   pass
 
 
-def lift_to_graph(init_tensor, graph, sources=None):
-  """Copies the tensor and all its inputs recursively to the outer graph."""
+def lift_to_graph(init_tensor, graph, sources=None,
+                  disallowed_placeholders=None):
+  """Copies the tensor and all its inputs recursively to the outer graph.
+
+  Args:
+    init_tensor: The Tensor to lift.
+    graph: The graph to lift to.
+    sources: Optional sequence of nodes to start from. If omitted the whole
+      subgraph which feeds into `init_tensor` is lifted.
+    disallowed_placeholders: An optional set of ops which may not appear in the
+      lifted graph. Defaults to all placeholders.
+
+  Returns:
+    A mapping from ops in the current default graph to ops in `graph`.
+
+  Raises:
+    UnliftableError: If a placeholder blocks lifting.
+  """
   # Check that the initializer does not depend on any placeholders.
   if sources is None:
     sources = set([])
@@ -53,10 +69,8 @@ def lift_to_graph(init_tensor, graph, sources=None):
     if op in visited_ops:
       continue
     visited_ops.add(op)
-    # TODO(apassos) distinguish arg placeholders, capture placeholders,
-    # and placeholders the user might directly use to initialize
-    # variables.
-    if op.type == "Placeholder":
+    if ((disallowed_placeholders is not None and op in disallowed_placeholders)
+        or (disallowed_placeholders is None and op.type == "Placeholder")):
       raise UnliftableError(
           "Unable to lift tensor", init_tensor,
           "because it depends transitively on placeholder ", op)
diff --git a/tensorflow/python/framework/func_graph.py b/tensorflow/python/framework/func_graph.py
index f60cf9350c..f83fc74cf9 100644
--- a/tensorflow/python/framework/func_graph.py
+++ b/tensorflow/python/framework/func_graph.py
@@ -545,12 +545,19 @@ def func_graph_from_py_func(name,
         convert_structure_to_signature(func_args, arg_names),
         convert_structure_to_signature(func_kwargs))
 
+    flat_func_args = nest.flatten(func_args)
+    flat_func_kwargs = nest.flatten(func_kwargs)
+    # Temporarily set inputs to allow graph building code to inspect
+    # them. Reassigned below.
+    func_graph.inputs = [arg for arg in flat_func_args + flat_func_kwargs
+                         if isinstance(arg, ops.Tensor)]
+
     # Note: `nest.flatten` sorts by keys, as does `_deterministic_dict_values`.
     # Variables to help check whether mutation happens in calling the function
     # Copy the recursive list, tuple and map structure, but not base objects
-    func_args_before = nest.pack_sequence_as(func_args, nest.flatten(func_args))
+    func_args_before = nest.pack_sequence_as(func_args, flat_func_args)
     func_kwargs_before = nest.pack_sequence_as(
-        func_kwargs, nest.flatten(func_kwargs))
+        func_kwargs, flat_func_kwargs)
 
     def convert(x):
       """Converts a function output to a Tensor."""
-- 
GitLab


From 8367a9975750a9f3af08af412b4255b0b3a833ba Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Mon, 11 Feb 2019 12:52:15 -0800
Subject: [PATCH 2272/2345] Create int8 space to batch ND.

PiperOrigin-RevId: 233460573
---
 tensorflow/lite/kernels/register.cc           |  4 +-
 tensorflow/lite/kernels/space_to_batch_nd.cc  |  9 +++
 .../lite/kernels/space_to_batch_nd_test.cc    | 61 +++++++++++++++----
 tensorflow/lite/toco/tflite/operator.cc       |  6 ++
 tensorflow/lite/toco/tflite/operator_test.cc  | 24 +++++---
 5 files changed, 82 insertions(+), 22 deletions(-)

diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index fd8f714d32..c5814a1a4d 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -212,7 +212,9 @@ BuiltinOpResolver::BuiltinOpResolver() {
              /* max_version */ 2);
   AddBuiltin(BuiltinOperator_CONCATENATION, Register_CONCATENATION());
   AddBuiltin(BuiltinOperator_ADD, Register_ADD());
-  AddBuiltin(BuiltinOperator_SPACE_TO_BATCH_ND, Register_SPACE_TO_BATCH_ND());
+  AddBuiltin(BuiltinOperator_SPACE_TO_BATCH_ND, Register_SPACE_TO_BATCH_ND(),
+             /* min_version */ 1,
+             /* max_version */ 2);
   AddBuiltin(BuiltinOperator_BATCH_TO_SPACE_ND, Register_BATCH_TO_SPACE_ND());
   AddBuiltin(BuiltinOperator_MUL, Register_MUL());
   AddBuiltin(BuiltinOperator_L2_NORMALIZATION, Register_L2_NORMALIZATION());
diff --git a/tensorflow/lite/kernels/space_to_batch_nd.cc b/tensorflow/lite/kernels/space_to_batch_nd.cc
index 1c61b2ef30..2fb7198cd6 100644
--- a/tensorflow/lite/kernels/space_to_batch_nd.cc
+++ b/tensorflow/lite/kernels/space_to_batch_nd.cc
@@ -141,6 +141,15 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
                                   op_context.output->params.zero_point);
       }
       break;
+    case kTfLiteInt8:
+      if (kernel_type == kReference) {
+        TF_LITE_SPACE_TO_BATCH_ND(reference_ops, int8_t,
+                                  op_context.output->params.zero_point);
+      } else {
+        TF_LITE_SPACE_TO_BATCH_ND(optimized_ops, int8_t,
+                                  op_context.output->params.zero_point);
+      }
+      break;
     case kTfLiteInt32:
       if (kernel_type == kReference) {
         TF_LITE_SPACE_TO_BATCH_ND(reference_ops, int32_t, 0);
diff --git a/tensorflow/lite/kernels/space_to_batch_nd_test.cc b/tensorflow/lite/kernels/space_to_batch_nd_test.cc
index c5d6e9a530..52a77984d9 100644
--- a/tensorflow/lite/kernels/space_to_batch_nd_test.cc
+++ b/tensorflow/lite/kernels/space_to_batch_nd_test.cc
@@ -31,8 +31,9 @@ class SpaceToBatchNDOpModel : public SingleOpModel {
     PopulateTensor<float>(input_, data);
   }
 
+  template <typename T>
   void SetQuantizedInput(std::initializer_list<float> data) {
-    QuantizeAndPopulate<uint8_t>(input_, data);
+    QuantizeAndPopulate<T>(input_, data);
   }
 
   void SetBlockShape(std::initializer_list<int> data) {
@@ -46,9 +47,10 @@ class SpaceToBatchNDOpModel : public SingleOpModel {
   std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
   std::vector<int> GetOutputShape() { return GetTensorShape(output_); }
 
+  template <typename T>
   std::vector<float> GetDequantizedOutput() {
-    return Dequantize<uint8_t>(ExtractVector<uint8_t>(output_),
-                               GetScale(output_), GetZeroPoint(output_));
+    return Dequantize<T>(ExtractVector<T>(output_), GetScale(output_),
+                         GetZeroPoint(output_));
   }
 
  protected:
@@ -233,29 +235,62 @@ TEST_F(QuantizedSpaceToBatchNDOpTest, ZeroNotInQuantizationRange) {
 }
 #endif
 
-TEST_F(QuantizedSpaceToBatchNDOpTest, SimplePaddingConstTest) {
+TEST_F(QuantizedSpaceToBatchNDOpTest, SimplePaddingConstTestUint8) {
   SpaceToBatchNDOpConstModel m({TensorType_UINT8, {1, 5, 2, 1}, -1.0, 1.0},
                                {3, 2}, {1, 0, 2, 0},
                                {TensorType_UINT8, {}, -1.0, 1.0});
-  m.SetQuantizedInput({-0.1, 0.2, -0.3, 0.4, -0.5, 0.6, -0.7, 0.8, -0.9, 0.1});
+  m.SetQuantizedInput<uint8_t>(
+      {-0.1, 0.2, -0.3, 0.4, -0.5, 0.6, -0.7, 0.8, -0.9, 0.1});
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({6, 2, 2, 1}));
-  EXPECT_THAT(m.GetDequantizedOutput(),
+  EXPECT_THAT(m.GetDequantizedOutput<uint8_t>(),
               ElementsAreArray(DequantizedArrayNear(
                   {0, 0,   0, -0.5, 0, 0,    0, 0.6,  0, -0.1, 0, -0.7,
                    0, 0.2, 0, 0.8,  0, -0.3, 0, -0.9, 0, 0.4,  0, 0.1},
                   -1.0, 1.0)));
 }
 
-TEST_F(QuantizedSpaceToBatchNDOpTest, SimplePaddingDynamicTest) {
+TEST_F(QuantizedSpaceToBatchNDOpTest, SimplePaddingConstTestInt8) {
+  SpaceToBatchNDOpConstModel m({TensorType_INT8, {1, 5, 2, 1}, -1.0, 1.0},
+                               {3, 2}, {1, 0, 2, 0},
+                               {TensorType_INT8, {}, -1.0, 1.0});
+  m.SetQuantizedInput<int8_t>(
+      {-0.1, 0.2, -0.3, 0.4, -0.5, 0.6, -0.7, 0.8, -0.9, 0.1});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({6, 2, 2, 1}));
+  EXPECT_THAT(m.GetDequantizedOutput<int8_t>(),
+              ElementsAreArray(DequantizedArrayNear(
+                  {0, 0,   0, -0.5, 0, 0,    0, 0.6,  0, -0.1, 0, -0.7,
+                   0, 0.2, 0, 0.8,  0, -0.3, 0, -0.9, 0, 0.4,  0, 0.1},
+                  -1.0, 1.0)));
+}
+
+TEST_F(QuantizedSpaceToBatchNDOpTest, SimplePaddingDynamicTestUint8) {
   SpaceToBatchNDOpDynamicModel m({TensorType_UINT8, {1, 5, 2, 1}, -1.0, 1.0},
                                  {TensorType_UINT8, {}, -1.0, 1.0});
-  m.SetQuantizedInput({-0.1, 0.2, -0.3, 0.4, -0.5, 0.6, -0.7, 0.8, -0.9, 0.1});
+  m.SetQuantizedInput<uint8_t>(
+      {-0.1, 0.2, -0.3, 0.4, -0.5, 0.6, -0.7, 0.8, -0.9, 0.1});
+  m.SetBlockShape({3, 2});
+  m.SetPaddings({1, 0, 2, 0});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({6, 2, 2, 1}));
+  EXPECT_THAT(m.GetDequantizedOutput<uint8_t>(),
+              ElementsAreArray(DequantizedArrayNear(
+                  {0, 0,   0, -0.5, 0, 0,    0, 0.6,  0, -0.1, 0, -0.7,
+                   0, 0.2, 0, 0.8,  0, -0.3, 0, -0.9, 0, 0.4,  0, 0.1},
+                  -1.0, 1.0)));
+}
+
+TEST_F(QuantizedSpaceToBatchNDOpTest, SimplePaddingDynamicTestInt8) {
+  SpaceToBatchNDOpDynamicModel m({TensorType_INT8, {1, 5, 2, 1}, -1.0, 1.0},
+                                 {TensorType_INT8, {}, -1.0, 1.0});
+  m.SetQuantizedInput<int8_t>(
+      {-0.1, 0.2, -0.3, 0.4, -0.5, 0.6, -0.7, 0.8, -0.9, 0.1});
   m.SetBlockShape({3, 2});
   m.SetPaddings({1, 0, 2, 0});
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({6, 2, 2, 1}));
-  EXPECT_THAT(m.GetDequantizedOutput(),
+  EXPECT_THAT(m.GetDequantizedOutput<int8_t>(),
               ElementsAreArray(DequantizedArrayNear(
                   {0, 0,   0, -0.5, 0, 0,    0, 0.6,  0, -0.1, 0, -0.7,
                    0, 0.2, 0, 0.8,  0, -0.3, 0, -0.9, 0, 0.4,  0, 0.1},
@@ -266,10 +301,10 @@ TEST_F(QuantizedSpaceToBatchNDOpTest, ComplexPaddingConstTest) {
   SpaceToBatchNDOpConstModel m({TensorType_UINT8, {1, 4, 2, 1}, -1.0, 1.0},
                                {3, 2}, {1, 1, 2, 4},
                                {TensorType_UINT8, {}, -1.0, 1.0});
-  m.SetQuantizedInput({-0.1, 0.2, -0.3, 0.4, -0.5, 0.6, -0.7, 0.8});
+  m.SetQuantizedInput<uint8_t>({-0.1, 0.2, -0.3, 0.4, -0.5, 0.6, -0.7, 0.8});
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({6, 2, 4, 1}));
-  EXPECT_THAT(m.GetDequantizedOutput(),
+  EXPECT_THAT(m.GetDequantizedOutput<uint8_t>(),
               ElementsAreArray(DequantizedArrayNear(
                   {
                       0, 0,    0, 0, 0, -0.5, 0, 0, 0, 0,   0, 0, 0, 0.6, 0, 0,
@@ -282,12 +317,12 @@ TEST_F(QuantizedSpaceToBatchNDOpTest, ComplexPaddingConstTest) {
 TEST_F(QuantizedSpaceToBatchNDOpTest, ComplexPaddingDynamicTest) {
   SpaceToBatchNDOpDynamicModel m({TensorType_UINT8, {1, 4, 2, 1}, -1.0, 1.0},
                                  {TensorType_UINT8, {}, -1.0, 1.0});
-  m.SetQuantizedInput({-0.1, 0.2, -0.3, 0.4, -0.5, 0.6, -0.7, 0.8});
+  m.SetQuantizedInput<uint8_t>({-0.1, 0.2, -0.3, 0.4, -0.5, 0.6, -0.7, 0.8});
   m.SetBlockShape({3, 2});
   m.SetPaddings({1, 1, 2, 4});
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({6, 2, 4, 1}));
-  EXPECT_THAT(m.GetDequantizedOutput(),
+  EXPECT_THAT(m.GetDequantizedOutput<uint8_t>(),
               ElementsAreArray(DequantizedArrayNear(
                   {
                       0, 0,    0, 0, 0, -0.5, 0, 0, 0, 0,   0, 0, 0, 0.6, 0, 0,
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index b69d614782..a0fd483889 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -239,6 +239,12 @@ class SpaceToBatchND
                    TocoOperator* op) const override {}
 
   int GetVersion(const OperatorSignature& op_signature) const override {
+    const string& input_name = op_signature.op->inputs[0];
+    const Array& input_array = op_signature.model->GetArray(input_name);
+    // If the op take int8 input, it is version 2.
+    if (input_array.data_type == ArrayDataType::kInt8) {
+      return 2;
+    }
     return 1;
   }
 };
diff --git a/tensorflow/lite/toco/tflite/operator_test.cc b/tensorflow/lite/toco/tflite/operator_test.cc
index a838d8fc0f..755c0828a1 100644
--- a/tensorflow/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/lite/toco/tflite/operator_test.cc
@@ -754,8 +754,10 @@ TEST_F(OperatorTest, BuiltinUnique) {
   EXPECT_EQ(output_toco_op->idx_out_type, op.idx_out_type);
 }
 
+// Test version for a simple Op with 2 versions and the input type controls the
+// version.
 template <typename Op>
-void VersioningTest() {
+void SimpleVersioningTest() {
   Op op;
   op.inputs = {"input1"};
   auto operator_by_type_map = BuildOperatorByTypeMap(false /*enable_flex_ops*/);
@@ -775,30 +777,36 @@ void VersioningTest() {
 }
 
 TEST_F(OperatorTest, VersioningEqualTest) {
-  VersioningTest<TensorFlowEqualOperator>();
+  SimpleVersioningTest<TensorFlowEqualOperator>();
 }
 
 TEST_F(OperatorTest, VersioningNotEqualTest) {
-  VersioningTest<TensorFlowNotEqualOperator>();
+  SimpleVersioningTest<TensorFlowNotEqualOperator>();
 }
 
 TEST_F(OperatorTest, VersioningLessTest) {
-  VersioningTest<TensorFlowLessOperator>();
+  SimpleVersioningTest<TensorFlowLessOperator>();
 }
 
 TEST_F(OperatorTest, VersioningLessEqualTest) {
-  VersioningTest<TensorFlowLessEqualOperator>();
+  SimpleVersioningTest<TensorFlowLessEqualOperator>();
 }
 
 TEST_F(OperatorTest, VersioningGreaterTest) {
-  VersioningTest<TensorFlowGreaterOperator>();
+  SimpleVersioningTest<TensorFlowGreaterOperator>();
 }
 
 TEST_F(OperatorTest, VersioningGreaterEqualTest) {
-  VersioningTest<TensorFlowGreaterEqualOperator>();
+  SimpleVersioningTest<TensorFlowGreaterEqualOperator>();
 }
 
-TEST_F(OperatorTest, VersioningPackTest) { VersioningTest<PackOperator>(); }
+TEST_F(OperatorTest, VersioningSpaceToBatchNDTest) {
+  SimpleVersioningTest<SpaceToBatchNDOperator>();
+}
+
+TEST_F(OperatorTest, VersioningPackTest) {
+  SimpleVersioningTest<PackOperator>();
+}
 
 TEST_F(OperatorTest, VersioningSelectTest) {
   SelectOperator select_op;
-- 
GitLab


From 79b9a64a08f23ee8a185246a8cf087f2850bb932 Mon Sep 17 00:00:00 2001
From: Ayush Dubey <ayushd@google.com>
Date: Mon, 11 Feb 2019 12:59:02 -0800
Subject: [PATCH 2273/2345] Add support for ncclAllGather in NcclManager.

The rank provided by the user is important, because order matters in
ncclAllGather.  Data from rank `i` is put at offset `i*sendcount` in the recv
buffer.  So, this change includes a small modification to how ranks are handled
internally in NcclManager.

PiperOrigin-RevId: 233461740
---
 tensorflow/core/nccl/nccl_manager.cc      |  67 ++++++++------
 tensorflow/core/nccl/nccl_manager.h       |   5 ++
 tensorflow/core/nccl/nccl_manager_test.cc | 103 +++++++++++++++++++---
 3 files changed, 136 insertions(+), 39 deletions(-)

diff --git a/tensorflow/core/nccl/nccl_manager.cc b/tensorflow/core/nccl/nccl_manager.cc
index 545284331d..a0b602f301 100644
--- a/tensorflow/core/nccl/nccl_manager.cc
+++ b/tensorflow/core/nccl/nccl_manager.cc
@@ -63,7 +63,7 @@ struct NcclManager::NcclStream {
   std::unique_ptr<Thread> thread;
   mutex mu;
   condition_variable cv;
-  // Has collective,rank pairs.
+  // Has collective,participant_idx pairs.
   std::deque<std::pair<Collective*, int>> pending_launches_ GUARDED_BY(mu);
   bool shutdown_requested GUARDED_BY(mu) = false;
 };
@@ -312,8 +312,10 @@ Status NcclManager::GetCommunicator(NcclManager::Collective* collective,
   CUDA_RETURN_IF_ERROR(cudaGetDevice(&saved_device));
   NCCL_RETURN_IF_ERROR(ncclGroupStart());
   for (int i = 0; i < collective->num_local_devices; ++i) {
-    const int rank =
-        collective->single_node ? i : collective->participants[i]->global_rank;
+    // Set rank to `participant->global_rank` if provided, else `i`.
+    const int rank = collective->participants[i]->global_rank >= 0
+                         ? collective->participants[i]->global_rank
+                         : i;
     CUDA_RETURN_IF_ERROR(cudaSetDevice(devices[i]));
     NCCL_RETURN_IF_ERROR(ncclCommInitRank(
         nccl_comms.data() + i, collective->num_global_devices, nccl_id, rank));
@@ -344,6 +346,12 @@ void NcclManager::AddToAllReduce(std::unique_ptr<Participant> participant,
   AddParticipant(std::move(participant), context, kAllReduce, reduction_op);
 }
 
+void NcclManager::AddToAllGather(std::unique_ptr<Participant> participant,
+                                 const Context& context) {
+  AddParticipant(std::move(participant), context, kAllGather,
+                 ncclSum /* unused */);
+}
+
 void NcclManager::AddBroadcastSend(std::unique_ptr<Participant> participant,
                                    const Context& context) {
   participant->root = true;
@@ -492,13 +500,11 @@ void NcclManager::RunCollective(Collective* collective) {
     return;
   }
 
-  for (int local_rank = 0; local_rank < collective->num_local_devices;
-       ++local_rank) {
-    Participant* p = collective->participants[local_rank].get();
-    NcclStream* nccl_stream =
-        collective->communicator->members[local_rank].nccl_stream;
+  for (int i = 0; i < collective->num_local_devices; ++i) {
+    Participant* p = collective->participants[i].get();
+    NcclStream* nccl_stream = collective->communicator->members[i].nccl_stream;
     CHECK(nccl_stream != nullptr);
-    const int rank = collective->single_node ? local_rank : p->global_rank;
+    const int rank = p->global_rank >= 0 ? p->global_rank : i;
 
     if (p->input != nullptr) {
       // Wait to ensure that the kernel that produces the data in the input
@@ -522,13 +528,11 @@ void NcclManager::RunCollective(Collective* collective) {
     // Note that it would be possible to run multiple collectives at once, if
     // they have non-intersecting sets of devices.
     mutex_lock l(collective_mu);
-    for (int local_rank = 0; local_rank < collective->num_local_devices;
-         ++local_rank) {
+    for (int i = 0; i < collective->num_local_devices; ++i) {
       NcclStream* nccl_stream =
-          collective->communicator->members[local_rank].nccl_stream;
+          collective->communicator->members[i].nccl_stream;
       mutex_lock l(nccl_stream->mu);
-      nccl_stream->pending_launches_.push_front(
-          std::make_pair(collective, local_rank));
+      nccl_stream->pending_launches_.push_front(std::make_pair(collective, i));
       nccl_stream->cv.notify_all();
     }
   }
@@ -555,23 +559,22 @@ void NcclManager::LoopKernelLaunches(NcclStream* nccl_stream) {
       next_launch = nccl_stream->pending_launches_.back();
       nccl_stream->pending_launches_.pop_back();
     }
-    Collective* collective = next_launch.first;
-    int local_rank = next_launch.second;
 
     // Launch the nccl kernel.
+    Collective* collective = next_launch.first;
     ncclDataType_t data_type = ToNcclType(collective->data_type);
-    Participant* p = collective->participants[local_rank].get();
-
-    auto nccl_comm = collective->communicator->members[local_rank].nccl_comm;
+    int p_idx = next_launch.second;
+    Participant* p = collective->participants[p_idx].get();
+    auto nccl_comm = collective->communicator->members[p_idx].nccl_comm;
     ncclResult_t nccl_result = ncclSuccess;
     switch (collective->type) {
       case kAllReduce: {
         const void* sendbuff = p->input->tensor_data().data();
         void* recvbuff = const_cast<char*>(p->output->tensor_data().data());
 
-        VLOG(2) << "call NcclAllReduce participant " << local_rank
-                << " sendbuff " << sendbuff << " recvbuff " << recvbuff
-                << " nccl_comm " << nccl_comm << " comm_stream " << comm_stream
+        VLOG(2) << "call NcclAllReduce participant " << p_idx << " sendbuff "
+                << sendbuff << " recvbuff " << recvbuff << " nccl_comm "
+                << nccl_comm << " comm_stream " << comm_stream
                 << " cuda_stream " << cu_stream;
         nccl_result = ncclAllReduce(sendbuff, recvbuff, p->input->NumElements(),
                                     data_type, collective->reduction_op,
@@ -595,16 +598,30 @@ void NcclManager::LoopKernelLaunches(NcclStream* nccl_stream) {
                                  collective->root_rank, nccl_comm, *cu_stream);
         break;
       }
+      case kAllGather: {
+        const void* sendbuff = p->input->tensor_data().data();
+        void* recvbuff = const_cast<char*>(p->output->tensor_data().data());
+
+        VLOG(2) << "call NcclAllGather participant " << p_idx << " sendbuff "
+                << sendbuff << " sendcount " << p->input->NumElements()
+                << " recvbuff " << recvbuff << " recvcount "
+                << p->output->NumElements() << " nccl_comm " << nccl_comm
+                << " comm_stream " << comm_stream << " cuda_stream "
+                << cu_stream;
+        nccl_result = ncclAllGather(sendbuff, recvbuff, p->input->NumElements(),
+                                    data_type, nccl_comm, *cu_stream);
+        break;
+      }
     }
 
     // Run the done_callback when the nccl kernel finishes running.
-    auto done_callback = [collective, local_rank, nccl_result]() {
+    auto done_callback = [collective, p_idx, nccl_result]() {
       if (nccl_result == ncclSuccess) {
-        collective->participants[local_rank]->done_callback(Status::OK());
+        collective->participants[p_idx]->done_callback(Status::OK());
       } else {
         // Propagate the error, but note that if other members of the collective
         // did launch their kernels, then they are hanging.
-        collective->participants[local_rank]->done_callback(errors::Unknown(
+        collective->participants[p_idx]->done_callback(errors::Unknown(
             "Error invoking NCCL: ", ncclGetErrorString(nccl_result)));
       }
 
diff --git a/tensorflow/core/nccl/nccl_manager.h b/tensorflow/core/nccl/nccl_manager.h
index 6ac5deb91a..7cf2c85f3e 100644
--- a/tensorflow/core/nccl/nccl_manager.h
+++ b/tensorflow/core/nccl/nccl_manager.h
@@ -143,6 +143,10 @@ class NcclManager {
   void AddToAllReduce(std::unique_ptr<Participant> participant,
                       const Context& context, ncclRedOp_t reduction_op);
 
+  // Adds one participant to an all-gather.
+  void AddToAllGather(std::unique_ptr<Participant> participant,
+                      const Context& context);
+
   // AddBroadcastSend and AddBroadcastRecv combine to send data from one sender
   // to all receivers.
   void AddBroadcastSend(std::unique_ptr<Participant> participant,
@@ -170,6 +174,7 @@ class NcclManager {
     kAllReduce = 1,
     kBroadcast = 2,
     kReduce = 3,
+    kAllGather = 4,
   };
   struct Collective;
   struct Communicator;
diff --git a/tensorflow/core/nccl/nccl_manager_test.cc b/tensorflow/core/nccl/nccl_manager_test.cc
index e65af13389..420e143c83 100644
--- a/tensorflow/core/nccl/nccl_manager_test.cc
+++ b/tensorflow/core/nccl/nccl_manager_test.cc
@@ -73,9 +73,9 @@ class NcclManagerTest : public ::testing::Test {
 
   static void TearDownTestCase() { delete devices_; }
 
-  TestCase* MakeTestCase(int num_nodes, int num_ranks_per_node,
-                         ncclRedOp_t reduction_op, TensorShape shape,
-                         float value_offset) {
+  TestCase* MakeReductionTestCase(int num_nodes, int num_ranks_per_node,
+                                  ncclRedOp_t reduction_op, TensorShape shape,
+                                  float value_offset) {
     TestCase* test_case = new TestCase();
     test_case->expected = Tensor(data_type_, shape);
     if (reduction_op == ncclProd) {
@@ -134,6 +134,47 @@ class NcclManagerTest : public ::testing::Test {
     return test_case;
   }
 
+  TestCase* MakeGatherTestCase(int num_nodes, int num_ranks_per_node,
+                               TensorShape in_shape, TensorShape out_shape) {
+    TestCase* test_case = new TestCase();
+    test_case->expected = Tensor(data_type_, out_shape);
+    test::FillFn<Scalar>(&test_case->expected,
+                         [](int) { return static_cast<Scalar>(0); });
+
+    float value_scale = 0.01;  // Small scale to avoid fp16 overflow.
+    for (int node = 0; node < num_nodes; ++node) {
+      for (int i = 0; i < num_ranks_per_node; ++i) {
+        auto* device = GetDevice(i);
+        auto* stream = device->tensorflow_gpu_device_info()->stream;
+
+        Tensor in_cpu(data_type_, in_shape);
+        test::FillFn<Scalar>(&in_cpu, [&](int index) {
+          return static_cast<Scalar>((index + 1) * value_scale);
+        });
+        // Starting index for this rank's tensor in the all-gathered output.
+        int32 gather_idx =
+            (node * num_ranks_per_node + i) * in_shape.num_elements();
+        for (int j = 0; j < in_shape.num_elements(); ++j) {
+          auto in_val = in_cpu.flat<Scalar>()(j);
+          auto out_expr = test_case->expected.template flat<Scalar>();
+          out_expr(gather_idx + j) = in_val;
+        }
+
+        value_scale *= 10;
+        test_case->ins.emplace_back(GpuAllocator(device), data_type_, in_shape);
+        test_case->outs.emplace_back(GpuAllocator(device), data_type_,
+                                     out_shape);
+
+        const Tensor& in_gpu = test_case->ins.back();
+        auto in_gpu_mem = AsDeviceMemory(in_gpu.flat<Scalar>().data());
+        stream->ThenMemcpy(&in_gpu_mem, in_cpu.flat<Scalar>().data(),
+                           in_cpu.TotalBytes());
+      }
+    }
+
+    return test_case;
+  }
+
   // Waits for the done callback to be called for each participant.
   void WaitForTestCompletion(TestCase* test_case) {
     test_case->mu.lock();
@@ -158,6 +199,9 @@ class NcclManagerTest : public ::testing::Test {
       stream->ThenMemcpy(out_cpu.flat<Scalar>().data(), out_gpu_mem,
                          out_cpu.TotalBytes());
       SE_ASSERT_OK(stream->BlockHostUntilDone());
+      VLOG(1) << "Verifying rank " << rank << " expected shape "
+              << test_case->expected.shape() << " out shape "
+              << out_cpu.shape();
       test::ExpectClose(test_case->expected, out_cpu);
     }
   }
@@ -218,8 +262,8 @@ TYPED_TEST(NcclManagerTest, BasicSumReduction) {
   for (int op = 0; op < 4; ++op) {
     ncclRedOp_t reduction_op = static_cast<ncclRedOp_t>(op);
     std::unique_ptr<typename TestFixture::TestCase> test_case(
-        this->MakeTestCase(/*num_nodes=*/1, num_ranks, reduction_op,
-                           TensorShape({2, 3}), 0.0f));
+        this->MakeReductionTestCase(/*num_nodes=*/1, num_ranks, reduction_op,
+                                    TensorShape({2, 3}), 0.0f));
     for (int rank = 0; rank < num_ranks; ++rank) {
       auto* device = this->GetDevice(rank);
       VLOG(2) << "rank " << rank << " device " << device->name();
@@ -259,7 +303,7 @@ TYPED_TEST(NcclManagerTest, MultipleCallers) {
     std::vector<std::pair<int, int>> case_and_rank;
     std::vector<std::unique_ptr<typename TestFixture::TestCase>> test_cases;
     for (int i = 0; i < num_collectives_per_iteration; ++i) {
-      test_cases.emplace_back(this->MakeTestCase(
+      test_cases.emplace_back(this->MakeReductionTestCase(
           /*num_nodes=*/1, num_ranks, ncclSum,
           TensorShape({100, i % 5 + 1, i % 3 + 1}), 1.1f * i));
       for (int j = 0; j < num_ranks; ++j) {
@@ -324,6 +368,34 @@ TYPED_TEST(NcclManagerTest, MultipleCallers) {
   }
 }
 
+// Test basic all-gather.
+TYPED_TEST(NcclManagerTest, BasicAllGather) {
+  const int num_ranks = 4;
+  for (int i = 0; i < num_ranks; ++i) {
+    std::unique_ptr<typename TestFixture::TestCase> test_case(
+        this->MakeGatherTestCase(/*num_nodes=*/1, num_ranks,
+                                 TensorShape({2, 3}),
+                                 TensorShape({2 * num_ranks, 3})));
+    for (int rank = 0; rank < num_ranks; ++rank) {
+      auto* device = this->GetDevice(rank);
+      VLOG(2) << "rank " << rank << " device " << device->name();
+      auto* event_mgr = device->tensorflow_gpu_device_info()->event_mgr;
+      auto* stream = device->tensorflow_gpu_device_info()->stream;
+      auto participant = absl::make_unique<NcclManager::Participant>(
+          device->executor(), stream, event_mgr, device->gpu_id(),
+          &test_case->ins[rank], &test_case->outs[rank], rank,
+          this->CreateDoneCallback(test_case.get()));
+      NcclManager::instance()->AddToAllGather(
+          std::move(participant),
+          {"allgather", /*num_local_devices=*/num_ranks,
+           /*num_global_devices=*/num_ranks, /*communicator_key=*/""});
+    }
+
+    LOG(INFO) << "Verifying results";
+    this->VerifyResults(test_case.get());
+  }
+}
+
 // Multi-node NCCL tests.
 
 TEST(NcclManagerTest, CommunicatorKey) {
@@ -353,8 +425,8 @@ TYPED_TEST(NcclManagerTest, MultiNode) {
   for (int op = 0; op < 4; ++op) {
     ncclRedOp_t reduction_op = static_cast<ncclRedOp_t>(op);
     std::unique_ptr<typename TestFixture::TestCase> test_case(
-        this->MakeTestCase(num_nodes, num_ranks_per_node, reduction_op,
-                           TensorShape({2, 3}), 0.0f));
+        this->MakeReductionTestCase(num_nodes, num_ranks_per_node, reduction_op,
+                                    TensorShape({2, 3}), 0.0f));
     for (int node = 0; node < num_nodes; ++node) {
       auto node_fn = [this, node, &nccl_managers, &communicator_key,
                       &collective_key, reduction_op, &test_case] {
@@ -393,8 +465,9 @@ TYPED_TEST(NcclManagerTest, MultiNode) {
 TYPED_TEST(NcclManagerTest, ConsistentCollectiveType) {
   const int num_ranks = 2;
 
-  std::unique_ptr<typename TestFixture::TestCase> test_case(this->MakeTestCase(
-      1 /* num_nodes */, num_ranks, ncclSum, TensorShape({2, 3}), 0.0f));
+  std::unique_ptr<typename TestFixture::TestCase> test_case(
+      this->MakeReductionTestCase(1 /* num_nodes */, num_ranks, ncclSum,
+                                  TensorShape({2, 3}), 0.0f));
   for (int rank = 0; rank < num_ranks; ++rank) {
     auto* device = this->GetDevice(rank);
     auto* event_mgr = device->tensorflow_gpu_device_info()->event_mgr;
@@ -427,8 +500,9 @@ TYPED_TEST(NcclManagerTest, ConsistentCollectiveType) {
 TYPED_TEST(NcclManagerTest, ConsistentCommunicatorKey) {
   const int num_ranks = 2;
 
-  std::unique_ptr<typename TestFixture::TestCase> test_case(this->MakeTestCase(
-      1 /* num_nodes */, num_ranks, ncclSum, TensorShape({2, 3}), 0.0f));
+  std::unique_ptr<typename TestFixture::TestCase> test_case(
+      this->MakeReductionTestCase(1 /* num_nodes */, num_ranks, ncclSum,
+                                  TensorShape({2, 3}), 0.0f));
   for (int rank = 0; rank < num_ranks; ++rank) {
     auto* device = this->GetDevice(rank);
     auto* event_mgr = device->tensorflow_gpu_device_info()->event_mgr;
@@ -454,8 +528,9 @@ TYPED_TEST(NcclManagerTest, ConsistentCommunicatorKey) {
 TYPED_TEST(NcclManagerTest, ConsistentNumberOfDevices) {
   const int num_ranks = 2;
 
-  std::unique_ptr<typename TestFixture::TestCase> test_case(this->MakeTestCase(
-      1 /* num_nodes */, num_ranks, ncclSum, TensorShape({2, 3}), 0.0f));
+  std::unique_ptr<typename TestFixture::TestCase> test_case(
+      this->MakeReductionTestCase(1 /* num_nodes */, num_ranks, ncclSum,
+                                  TensorShape({2, 3}), 0.0f));
   for (int rank = 0; rank < num_ranks; ++rank) {
     auto* device = this->GetDevice(rank);
     auto* event_mgr = device->tensorflow_gpu_device_info()->event_mgr;
-- 
GitLab


From 19b1c959c8c8a9b27da5a21234da9a022f71f20d Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Mon, 11 Feb 2019 13:22:47 -0800
Subject: [PATCH 2274/2345] Update tests for TF v2.

PiperOrigin-RevId: 233466266
---
 tensorflow/python/keras/layers/wrappers_test.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/keras/layers/wrappers_test.py b/tensorflow/python/keras/layers/wrappers_test.py
index 8a0b265c0d..0c5f69ba1f 100644
--- a/tensorflow/python/keras/layers/wrappers_test.py
+++ b/tensorflow/python/keras/layers/wrappers_test.py
@@ -23,6 +23,7 @@ import copy
 import numpy as np
 
 from tensorflow.python import keras
+from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import test_util as tf_test_util
 from tensorflow.python.platform import test
@@ -396,7 +397,6 @@ class BidirectionalTest(test.TestCase):
       model.compile(loss='mse', optimizer='sgd')
       model.fit(x, y, epochs=1, batch_size=1)
 
-  @tf_test_util.run_v1_only('b/120545219')
   def test_Bidirectional_merged_value(self):
     rnn = keras.layers.LSTM
     samples = 2
@@ -424,10 +424,10 @@ class BidirectionalTest(test.TestCase):
             rnn(units, return_sequences=True), merge_mode=merge_mode)
         f_merged = keras.backend.function([inputs], _to_list(layer(inputs)))
         f_forward = keras.backend.function([inputs],
-                                           [layer.forward_layer.call(inputs)])
+                                           [layer.forward_layer(inputs)])
         f_backward = keras.backend.function(
             [inputs],
-            [keras.backend.reverse(layer.backward_layer.call(inputs), 1)])
+            [keras.backend.reverse(layer.backward_layer(inputs), 1)])
 
         y_merged = f_merged(x)
         y_expected = _to_list(merge_func(f_forward(x)[0], f_backward(x)[0]))
@@ -441,9 +441,9 @@ class BidirectionalTest(test.TestCase):
             rnn(units, return_state=True), merge_mode=merge_mode)
         f_merged = keras.backend.function([inputs], layer(inputs))
         f_forward = keras.backend.function([inputs],
-                                           layer.forward_layer.call(inputs))
+                                           layer.forward_layer(inputs))
         f_backward = keras.backend.function([inputs],
-                                            layer.backward_layer.call(inputs))
+                                            layer.backward_layer(inputs))
         n_states = len(layer.layer.states)
 
         y_merged = f_merged(x)
@@ -527,8 +527,10 @@ class BidirectionalTest(test.TestCase):
       layer.trainable = True
       assert len(layer.trainable_weights) == 6
 
-  @tf_test_util.run_v1_only('b/120545219')
   def test_Bidirectional_updates(self):
+    if context.executing_eagerly():
+      self.skipTest('layer.updates is only available in graph mode.')
+
     with self.cached_session():
       x = keras.layers.Input(shape=(3, 2))
       x_reachable_update = x * x
-- 
GitLab


From ff88864b53cd1ea1fce478f0b833395f23673426 Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Mon, 11 Feb 2019 13:51:25 -0800
Subject: [PATCH 2275/2345] Support Keras whitelisting of TF Ops where their
 first input is a list of Tensors.

This adds support for functions such as `tf.concat`, `tf.stack`, etc.

PiperOrigin-RevId: 233471932
---
 tensorflow/python/keras/engine/base_layer.py  |  4 +++
 .../python/keras/engine/base_layer_utils.py   |  2 +-
 .../keras/layers/tensorflow_op_layer_test.py  | 26 +++++++++++++++++--
 3 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index eee23b1311..cdd36ffcc2 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -1834,6 +1834,10 @@ class TensorFlowOpLayer(Layer):
         inputs.insert(index, constant)
 
       self.node_def.name = graph.unique_name(self.node_def.name)
+      # Check for case where first input should be a list of Tensors.
+      if 'N' in self.node_def.attr:
+        num_tensors = self.node_def.attr['N'].i
+        inputs = [inputs[:num_tensors]] + inputs[num_tensors:]
       c_op = ops._create_c_op(graph, self.node_def, inputs, control_inputs=[])
       op = graph._create_op_from_tf_operation(c_op)
 
diff --git a/tensorflow/python/keras/engine/base_layer_utils.py b/tensorflow/python/keras/engine/base_layer_utils.py
index 211e8482e4..a9fd17bcf8 100644
--- a/tensorflow/python/keras/engine/base_layer_utils.py
+++ b/tensorflow/python/keras/engine/base_layer_utils.py
@@ -277,7 +277,7 @@ def _create_keras_history_helper(tensors, processed_ops=None):
           # a constant (Variables currently have `Placeholder` op type
           # when originating from an eager context
           # so can't be supported.
-          constants[i] = backend.function([], [op_input])([])
+          constants[i] = backend.function([], op_input)([])
       processed_ops = _create_keras_history_helper(layer_inputs, processed_ops)
       name = op.name
       node_def = op.node_def.SerializeToString()
diff --git a/tensorflow/python/keras/layers/tensorflow_op_layer_test.py b/tensorflow/python/keras/layers/tensorflow_op_layer_test.py
index 8941d86674..4cdb3dfef9 100644
--- a/tensorflow/python/keras/layers/tensorflow_op_layer_test.py
+++ b/tensorflow/python/keras/layers/tensorflow_op_layer_test.py
@@ -27,6 +27,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.keras.optimizer_v2 import adam
+from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_nn_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
@@ -88,6 +89,19 @@ def _multiple_uses():
   return inputs, outputs
 
 
+def _op_with_tensor_list():
+  inputs = keras.Input(shape=(10,))
+  x = array_ops.concat([inputs, inputs], axis=1)
+  outputs = keras.layers.Dense(10)(x)
+  return inputs, outputs
+
+
+def _add_n():
+  inputs = keras.Input(shape=(10,))
+  outputs = math_ops.add_n([inputs, inputs, inputs])
+  return inputs, outputs
+
+
 @keras_parameterized.run_all_keras_modes
 class AutoLambdaTest(keras_parameterized.TestCase):
 
@@ -98,7 +112,8 @@ class AutoLambdaTest(keras_parameterized.TestCase):
       ('multiple_ops_in_middle', _multiple_ops_in_middle),
       ('single_standalone_branch', _single_standalone_branch),
       ('single_op_with_attrs', _single_op_with_attrs),
-      ('multiple_uses', _multiple_uses))
+      ('multiple_uses', _multiple_uses),
+      ('op_with_tensor_list', _op_with_tensor_list), ('add_n', _add_n))
   def test_autolambda(self, model_fn):
     inputs, outputs = model_fn()
     model = keras.Model(inputs, outputs)
@@ -110,6 +125,13 @@ class AutoLambdaTest(keras_parameterized.TestCase):
     np_outputs = nest.map_structure(lambda x: np.ones((10, 10), 'float32'),
                                     outputs)
     model.fit(np_inputs, np_outputs, batch_size=2)
+    model(np_inputs)  # Test calling the model directly on inputs.
+
+    new_model = keras.Model.from_config(model.get_config())
+    new_model.compile(
+        adam.Adam(0.001), 'mse', run_eagerly=testing_utils.should_run_eagerly())
+    new_model.fit(np_inputs, np_outputs, batch_size=2)
+    new_model(np_inputs)  # Test calling the new model directly on inputs.
 
   def test_numerical_correctness_simple(self):
     x = ops.convert_to_tensor([[-1., 0., -2., 1.]])
@@ -127,7 +149,7 @@ class AutoLambdaTest(keras_parameterized.TestCase):
     y = self.evaluate(model(x))
     self.assertAllClose(y, [1.5, 3.])
 
-  def test_serialization(self):
+  def test_numerical_correctness_serialization(self):
     x = ops.convert_to_tensor([-1., 0., -2., 1.])
     inputs = keras.Input(shape=(4,))
     outputs = gen_nn_ops.relu(inputs)
-- 
GitLab


From 2199d3064c3f7778d51b9d62f6e81d1640d980f4 Mon Sep 17 00:00:00 2001
From: Peng Wang <wangpeng@google.com>
Date: Mon, 11 Feb 2019 13:51:55 -0800
Subject: [PATCH 2276/2345] A continuing partial implementation of RFC "Random
 numbers in TensorFlow 2.0" (https://github.com/tensorflow/community/pull/38):

PiperOrigin-RevId: 233472024
---
 .../api_def_StatefulStandardNormalV2.pbtxt    |  37 +++
 .../api_def_StatefulStandardNormalV2.pbtxt    |   4 +
 tensorflow/core/kernels/random_op_gpu.cu.cc   | 167 +---------
 tensorflow/core/kernels/random_op_gpu.h       | 206 ++++++++++++
 .../core/kernels/stateful_random_ops.cc       | 302 +++++++-----------
 tensorflow/core/kernels/stateful_random_ops.h | 137 ++++++++
 .../kernels/stateful_random_ops_gpu.cu.cc     |  96 ++++++
 tensorflow/core/ops/stateful_random_ops.cc    |  34 +-
 tensorflow/python/ops/stateful_random_ops.py  |  57 ++--
 .../python/ops/stateful_random_ops_test.py    |  84 ++++-
 10 files changed, 732 insertions(+), 392 deletions(-)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_StatefulStandardNormalV2.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_StatefulStandardNormalV2.pbtxt
 create mode 100644 tensorflow/core/kernels/random_op_gpu.h
 create mode 100644 tensorflow/core/kernels/stateful_random_ops.h
 create mode 100644 tensorflow/core/kernels/stateful_random_ops_gpu.cu.cc

diff --git a/tensorflow/core/api_def/base_api/api_def_StatefulStandardNormalV2.pbtxt b/tensorflow/core/api_def/base_api/api_def_StatefulStandardNormalV2.pbtxt
new file mode 100644
index 0000000000..8c145e05ac
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_StatefulStandardNormalV2.pbtxt
@@ -0,0 +1,37 @@
+op {
+  graph_op_name: "StatefulStandardNormalV2"
+  in_arg {
+    name: "resource"
+    description: <<END
+The handle of the resource variable that stores the state of the RNG.
+END
+  }
+  in_arg {
+    name: "algorithm"
+    description: <<END
+The RNG algorithm.
+END
+  }
+  in_arg {
+    name: "shape"
+    description: <<END
+The shape of the output tensor.
+END
+  }
+  out_arg {
+    name: "output"
+    description: <<END
+A tensor of the specified shape filled with random normal values.
+END
+  }
+  attr {
+    name: "dtype"
+    description: <<END
+The type of the output.
+END
+  }
+  summary: "Outputs random values from a normal distribution."
+  description: <<END
+The generated values will have mean 0 and standard deviation 1.
+END
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_StatefulStandardNormalV2.pbtxt b/tensorflow/core/api_def/python_api/api_def_StatefulStandardNormalV2.pbtxt
new file mode 100644
index 0000000000..a1816dd734
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_StatefulStandardNormalV2.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "StatefulStandardNormalV2"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/kernels/random_op_gpu.cu.cc b/tensorflow/core/kernels/random_op_gpu.cu.cc
index edb2b10e3d..a560388523 100644
--- a/tensorflow/core/kernels/random_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/random_op_gpu.cu.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #define EIGEN_USE_GPU
 
 #include "tensorflow/core/kernels/random_op.h"
+#include "tensorflow/core/kernels/random_op_gpu.h"
 
 #include <assert.h>
 #include <stdio.h>
@@ -36,170 +37,6 @@ namespace functor {
 
 typedef Eigen::GpuDevice GPUDevice;
 
-template <class Distribution, bool VariableSamplesPerOutput>
-struct FillPhiloxRandomKernel;
-
-template <typename T, int ElementCount>
-class SampleCopier {
- public:
-  inline __device__ void operator()(
-      T* buf, const tensorflow::random::Array<T, ElementCount>& array) const {
-#pragma unroll
-    for (int i = 0; i < ElementCount; i++) {
-      buf[i] = array[i];
-    }
-  }
-};
-
-template <>
-class SampleCopier<float, 4> {
- public:
-  // Copies the elements from the array to buf. buf must be 128-bit aligned,
-  // which is true for tensor data, and all offsets that are a multiple of the
-  // vector size (because the vectors are 128 bits long).
-  inline __device__ void operator()(
-      float* buf, const tensorflow::random::Array<float, 4>& array) const {
-    // NOTE(ringwalt): It's not safe to cast &array[0] to a float4, because they
-    // have 32-bit alignment vs 128-bit alignment. There seems to be no
-    // performance loss when assigning each element to a vector.
-    float4 vec;
-    vec.x = array[0];
-    vec.y = array[1];
-    vec.z = array[2];
-    vec.w = array[3];
-    float4* buf_vector = reinterpret_cast<float4*>(buf);
-    *buf_vector = vec;
-  }
-};
-
-template <>
-class SampleCopier<int32, 4> {
- public:
-  // Copies the elements from the array to buf. buf must be 128-bit aligned,
-  // which is true for tensor data, and all offsets that are a multiple of the
-  // vector size (because the vectors are 128 bits long).
-  inline __device__ void operator()(
-      int32* buf, const tensorflow::random::Array<int32, 4>& array) const {
-    int4 vec;
-    vec.x = array[0];
-    vec.y = array[1];
-    vec.z = array[2];
-    vec.w = array[3];
-    int4* buf_vector = reinterpret_cast<int4*>(buf);
-    *buf_vector = vec;
-  }
-};
-
-template <>
-class SampleCopier<double, 2> {
- public:
-  // Copies the elements from the array to buf. buf must be 128-bit aligned,
-  // which is true for tensor data, and all offsets that are a multiple of the
-  // vector size (because the vectors are 128 bits long).
-  inline __device__ void operator()(
-      double* buf, const tensorflow::random::Array<double, 2>& array) const {
-    double2 vec;
-    vec.x = array[0];
-    vec.y = array[1];
-    double2* buf_vector = reinterpret_cast<double2*>(buf);
-    *buf_vector = vec;
-  }
-};
-
-template <>
-class SampleCopier<int64, 2> {
- public:
-  // Copies the elements from the array to buf. buf must be 128-bit aligned,
-  // which is true for tensor data, and all offsets that are a multiple of the
-  // vector size (because the vectors are 128 bits long).
-  inline __device__ void operator()(
-      int64* buf, const tensorflow::random::Array<int64, 2>& array) const {
-    longlong2 vec;
-    vec.x = array[0];
-    vec.y = array[1];
-    longlong2* buf_vector = reinterpret_cast<longlong2*>(buf);
-    *buf_vector = vec;
-  }
-};
-
-// A cuda kernel to fill the data with random numbers from the specified
-// distribution. Each output takes a fixed number of samples.
-template <class Distribution>
-struct FillPhiloxRandomKernel<Distribution, false> {
-  typedef typename Distribution::ResultElementType T;
-  PHILOX_DEVICE_FUNC void Run(random::PhiloxRandom gen, T* data, int64 size,
-                              Distribution dist) {
-    const int kGroupSize = Distribution::kResultElementCount;
-
-    const int32 thread_id = blockIdx.x * blockDim.x + threadIdx.x;
-    const int32 total_thread_count = gridDim.x * blockDim.x;
-    int32 offset = thread_id * kGroupSize;
-    gen.Skip(thread_id);
-
-    const SampleCopier<T, kGroupSize> copier;
-    while (offset + kGroupSize <= size) {
-      const typename Distribution::ResultType samples = dist(&gen);
-      copier(&data[offset], samples);
-
-      offset += total_thread_count * kGroupSize;
-      gen.Skip(total_thread_count - 1);
-    }
-
-    typename Distribution::ResultType samples = dist(&gen);
-    for (int i = 0; i < kGroupSize; ++i) {
-      if (offset >= size) {
-        return;
-      }
-      data[offset] = samples[i];
-      ++offset;
-    }
-  }
-};
-
-// A cuda kernel to fill the data with random numbers from the specified
-// distribution. Each output takes a variable number of samples.
-template <class Distribution>
-struct FillPhiloxRandomKernel<Distribution, true> {
-  typedef typename Distribution::ResultElementType T;
-  PHILOX_DEVICE_FUNC void Run(const random::PhiloxRandom& base_gen, T* data,
-                              int64 size, Distribution dist) {
-    using random::PhiloxRandom;
-    using random::SingleSampleAdapter;
-
-    const int kReservedSamplesPerOutput = 256;
-    const int kGroupSize = Distribution::kResultElementCount;
-    const int kGeneratorSkipPerOutputGroup = kGroupSize *
-                                             kReservedSamplesPerOutput /
-                                             PhiloxRandom::kResultElementCount;
-
-    const int32 thread_id = blockIdx.x * blockDim.x + threadIdx.x;
-    const int32 total_thread_count = gridDim.x * blockDim.x;
-    int64 group_index = thread_id;
-    int64 offset = group_index * kGroupSize;
-
-    while (offset < size) {
-      // Since each output takes a variable number of samples, we need to
-      // realign the generator to the beginning for the current output group
-      PhiloxRandom gen = base_gen;
-      gen.Skip(group_index * kGeneratorSkipPerOutputGroup);
-      SingleSampleAdapter<PhiloxRandom> single_samples(&gen);
-
-      typename Distribution::ResultType samples = dist(&single_samples);
-
-      for (int i = 0; i < kGroupSize; ++i) {
-        if (offset >= size) {
-          return;
-        }
-        data[offset] = samples[i];
-        ++offset;
-      }
-
-      offset += (total_thread_count - 1) * kGroupSize;
-      group_index += total_thread_count;
-    }
-  }
-};
-
 // A simple launch pad to call the correct function templates to fill the data
 template <class Distribution>
 __global__ void __launch_bounds__(1024)
@@ -224,7 +61,7 @@ void FillPhiloxRandom<GPUDevice, Distribution>::operator()(
 
   FillPhiloxRandomKernelLaunch<Distribution>
       <<<num_blocks, block_size, 0, d.stream()>>>(gen, data, size, dist);
-};
+}
 
 // Explicit instantiation of the GPU distributions functors
 // clang-format off
diff --git a/tensorflow/core/kernels/random_op_gpu.h b/tensorflow/core/kernels/random_op_gpu.h
new file mode 100644
index 0000000000..e32c755d78
--- /dev/null
+++ b/tensorflow/core/kernels/random_op_gpu.h
@@ -0,0 +1,206 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_KERNELS_RANDOM_OP_GPU_H_
+#define TENSORFLOW_CORE_KERNELS_RANDOM_OP_GPU_H_
+
+#if defined(__CUDACC__)
+
+#include "tensorflow/core/lib/random/philox_random.h"
+#include "tensorflow/core/lib/random/random_distributions.h"
+
+namespace tensorflow {
+
+namespace functor {
+
+template <class Distribution, bool VariableSamplesPerOutput>
+struct FillPhiloxRandomKernel;
+
+template <class Distribution>
+struct FillPhiloxRandomKernel<Distribution, false> {
+  typedef typename Distribution::ResultElementType T;
+  PHILOX_DEVICE_FUNC void Run(random::PhiloxRandom gen, T* data, int64 size,
+                              Distribution dist);
+};
+
+template <class Distribution>
+struct FillPhiloxRandomKernel<Distribution, true> {
+  typedef typename Distribution::ResultElementType T;
+  PHILOX_DEVICE_FUNC void Run(const random::PhiloxRandom& base_gen, T* data,
+                              int64 size, Distribution dist);
+};
+
+template <typename T, int ElementCount>
+class SampleCopier {
+ public:
+  inline __device__ void operator()(
+      T* buf, const tensorflow::random::Array<T, ElementCount>& array) const {
+#pragma unroll
+    for (int i = 0; i < ElementCount; i++) {
+      buf[i] = array[i];
+    }
+  }
+};
+
+template <>
+class SampleCopier<float, 4> {
+ public:
+  // Copies the elements from the array to buf. buf must be 128-bit aligned,
+  // which is true for tensor data, and all offsets that are a multiple of the
+  // vector size (because the vectors are 128 bits long).
+  inline __device__ void operator()(
+      float* buf, const tensorflow::random::Array<float, 4>& array) const {
+    // NOTE(ringwalt): It's not safe to cast &array[0] to a float4, because they
+    // have 32-bit alignment vs 128-bit alignment. There seems to be no
+    // performance loss when assigning each element to a vector.
+    float4 vec;
+    vec.x = array[0];
+    vec.y = array[1];
+    vec.z = array[2];
+    vec.w = array[3];
+    float4* buf_vector = reinterpret_cast<float4*>(buf);
+    *buf_vector = vec;
+  }
+};
+
+template <>
+class SampleCopier<int32, 4> {
+ public:
+  // Copies the elements from the array to buf. buf must be 128-bit aligned,
+  // which is true for tensor data, and all offsets that are a multiple of the
+  // vector size (because the vectors are 128 bits long).
+  inline __device__ void operator()(
+      int32* buf, const tensorflow::random::Array<int32, 4>& array) const {
+    int4 vec;
+    vec.x = array[0];
+    vec.y = array[1];
+    vec.z = array[2];
+    vec.w = array[3];
+    int4* buf_vector = reinterpret_cast<int4*>(buf);
+    *buf_vector = vec;
+  }
+};
+
+template <>
+class SampleCopier<double, 2> {
+ public:
+  // Copies the elements from the array to buf. buf must be 128-bit aligned,
+  // which is true for tensor data, and all offsets that are a multiple of the
+  // vector size (because the vectors are 128 bits long).
+  inline __device__ void operator()(
+      double* buf, const tensorflow::random::Array<double, 2>& array) const {
+    double2 vec;
+    vec.x = array[0];
+    vec.y = array[1];
+    double2* buf_vector = reinterpret_cast<double2*>(buf);
+    *buf_vector = vec;
+  }
+};
+
+template <>
+class SampleCopier<int64, 2> {
+ public:
+  // Copies the elements from the array to buf. buf must be 128-bit aligned,
+  // which is true for tensor data, and all offsets that are a multiple of the
+  // vector size (because the vectors are 128 bits long).
+  inline __device__ void operator()(
+      int64* buf, const tensorflow::random::Array<int64, 2>& array) const {
+    longlong2 vec;
+    vec.x = array[0];
+    vec.y = array[1];
+    longlong2* buf_vector = reinterpret_cast<longlong2*>(buf);
+    *buf_vector = vec;
+  }
+};
+
+// A cuda kernel to fill the data with random numbers from the specified
+// distribution. Each output takes a fixed number of samples.
+template <class Distribution>
+PHILOX_DEVICE_FUNC void FillPhiloxRandomKernel<Distribution, false>::Run(
+    random::PhiloxRandom gen, T* data, int64 size, Distribution dist) {
+  const int kGroupSize = Distribution::kResultElementCount;
+
+  const int32 thread_id = blockIdx.x * blockDim.x + threadIdx.x;
+  const int32 total_thread_count = gridDim.x * blockDim.x;
+  int32 offset = thread_id * kGroupSize;
+  gen.Skip(thread_id);
+
+  const SampleCopier<T, kGroupSize> copier;
+  while (offset + kGroupSize <= size) {
+    const typename Distribution::ResultType samples = dist(&gen);
+    copier(&data[offset], samples);
+
+    offset += total_thread_count * kGroupSize;
+    gen.Skip(total_thread_count - 1);
+  }
+
+  typename Distribution::ResultType samples = dist(&gen);
+  for (int i = 0; i < kGroupSize; ++i) {
+    if (offset >= size) {
+      return;
+    }
+    data[offset] = samples[i];
+    ++offset;
+  }
+}
+
+// A cuda kernel to fill the data with random numbers from the specified
+// distribution. Each output takes a variable number of samples.
+template <class Distribution>
+PHILOX_DEVICE_FUNC void FillPhiloxRandomKernel<Distribution, true>::Run(
+    const random::PhiloxRandom& base_gen, T* data, int64 size,
+    Distribution dist) {
+  using random::PhiloxRandom;
+  using random::SingleSampleAdapter;
+
+  const int kReservedSamplesPerOutput = 256;
+  const int kGroupSize = Distribution::kResultElementCount;
+  const int kGeneratorSkipPerOutputGroup = kGroupSize *
+                                           kReservedSamplesPerOutput /
+                                           PhiloxRandom::kResultElementCount;
+
+  const int32 thread_id = blockIdx.x * blockDim.x + threadIdx.x;
+  const int32 total_thread_count = gridDim.x * blockDim.x;
+  int64 group_index = thread_id;
+  int64 offset = group_index * kGroupSize;
+
+  while (offset < size) {
+    // Since each output takes a variable number of samples, we need to
+    // realign the generator to the beginning for the current output group
+    PhiloxRandom gen = base_gen;
+    gen.Skip(group_index * kGeneratorSkipPerOutputGroup);
+    SingleSampleAdapter<PhiloxRandom> single_samples(&gen);
+
+    typename Distribution::ResultType samples = dist(&single_samples);
+
+    for (int i = 0; i < kGroupSize; ++i) {
+      if (offset >= size) {
+        return;
+      }
+      data[offset] = samples[i];
+      ++offset;
+    }
+
+    offset += (total_thread_count - 1) * kGroupSize;
+    group_index += total_thread_count;
+  }
+}
+
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // defined(__CUDACC__)
+
+#endif  // TENSORFLOW_CORE_KERNELS_RANDOM_OP_GPU_H_
diff --git a/tensorflow/core/kernels/stateful_random_ops.cc b/tensorflow/core/kernels/stateful_random_ops.cc
index 7cbbfd490e..62c4ee3bd6 100644
--- a/tensorflow/core/kernels/stateful_random_ops.cc
+++ b/tensorflow/core/kernels/stateful_random_ops.cc
@@ -15,251 +15,173 @@ limitations under the License.
 
 #define EIGEN_USE_THREADS
 
-#include <algorithm>
-#include <cmath>
-#include <memory>
-
-#include "absl/strings/str_join.h"
-#include "absl/types/variant.h"
-#include "tensorflow/core/common_runtime/device.h"
-#include "tensorflow/core/framework/bounds_check.h"
-#include "tensorflow/core/framework/op_kernel.h"
-#include "tensorflow/core/framework/register_types.h"
-#include "tensorflow/core/framework/resource_mgr.h"
-#include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/framework/tensor_types.h"
-#include "tensorflow/core/framework/variant_op_registry.h"
-#include "tensorflow/core/kernels/dense_update_functor.h"
-#include "tensorflow/core/kernels/gather_functor.h"
+#include "tensorflow/core/kernels/stateful_random_ops.h"
 #include "tensorflow/core/kernels/random_op.h"
-#include "tensorflow/core/kernels/resource_variable_ops.h"
-#include "tensorflow/core/kernels/scatter_functor.h"
 #include "tensorflow/core/kernels/training_op_helpers.h"
-#include "tensorflow/core/kernels/variable_ops.h"
-#include "tensorflow/core/lib/core/errors.h"
-#include "tensorflow/core/lib/core/refcount.h"
-#include "tensorflow/core/lib/random/random_distributions.h"
-#include "tensorflow/core/lib/random/simple_philox.h"
-#include "tensorflow/core/platform/logging.h"
-#include "tensorflow/core/platform/mem.h"
-#include "tensorflow/core/platform/mutex.h"
-#include "tensorflow/core/platform/types.h"
-#include "tensorflow/core/util/util.h"
-#include "tensorflow/core/util/work_sharder.h"
 
 namespace tensorflow {
 
-using CPUDevice = Eigen::ThreadPoolDevice;
-
-using random::PhiloxRandom;
-
-namespace {
-
-// 'Variable' doesn't support uint32 or uint64 yet (due to reasons explained
-// in b/111604096 and cl/171681867), so I use signed int here. I choose int64
-// instead of int32 because `VarHandleOp` doesn't support int32 on GPU.
-using StateElementType = int64;
-static constexpr DataType STATE_ELEMENT_DTYPE = DT_INT64;
-
-using Algorithm = StateElementType;
-static constexpr Algorithm RNG_ALG_PHILOX = 1;
-
-using SkippableRNG = absl::variant<PhiloxRandom>;
-
-// This function is for hiding the implementation detail about the
-// `absl::variant` index of each algorithm.
-Algorithm GetAlgorithm(SkippableRNG const& rng) {
-  auto idx = rng.index();
-  if (idx == 0) {
-    return RNG_ALG_PHILOX;
+template <typename Distribution>
+struct UpdateVariableAndFill_Philox<CPUDevice, Distribution> {
+  void operator()(OpKernelContext* ctx, const CPUDevice& device,
+                  int64 output_size, int64 alg_tag_skip,
+                  ScopedUnlockUnref* state_var_guard, Tensor* state_tensor,
+                  typename Distribution::ResultElementType* output_data) {
+    auto state_tensor_flat = state_tensor->flat<StateElementType>();
+    auto state_data = state_tensor_flat.data();
+    // Delegates to PhiloxRandom to do the actual increasing.
+    auto philox = GetPhiloxRandomFromMem(state_data + alg_tag_skip);
+    UpdateMemWithPhiloxRandom(philox, output_size, state_data + alg_tag_skip);
+    // No longer needs the lock.
+    state_var_guard->Release();
+    functor::FillPhiloxRandom<CPUDevice, Distribution>()(
+        ctx, device, philox, output_data, output_size, Distribution());
   }
-  // unreachable
-  return RNG_ALG_PHILOX;
-}
+};
 
-// Fills a buffer with random numbers sampled from a given distribution.
-template <class Device, class Distribution>
-Status FillRandom(OpKernelContext* ctx, const Device& device,
-                  SkippableRNG const& gen, int64 size, Distribution dist,
-                  typename Distribution::ResultElementType* data) {
-  auto algorithm = GetAlgorithm(gen);
-  if (algorithm == RNG_ALG_PHILOX) {
-    auto philox = absl::get<PhiloxRandom>(gen);
-    functor::FillPhiloxRandom<Device, Distribution>()(ctx, device, philox, data,
-                                                      size, dist);
-    return Status::OK();
-  } else {
-    // return errors::InvalidArgument("Unsupported algorithm id: ", algorithm);
-    return Status::OK();
+template <typename Device, typename Distribution>
+Status UpdateVariableAndFill(
+    OpKernelContext* ctx, int state_input_idx, bool read_alg_from_state,
+    Algorithm alg, int64 output_size,
+    typename Distribution::ResultElementType* output_data) {
+  Var* var = nullptr;
+  TF_RETURN_IF_ERROR(
+      LookupResource(ctx, HandleFromInput(ctx, state_input_idx), &var));
+  // Use `ScopedUnlockUnref` here instead of `mutex_lock` and `ScopedUnref`
+  // because the former supports early releasing which is needed by
+  // `UpdateVariableAndFill_Philox<CPU>` to avoid holding the lock while
+  // filling.
+  ScopedUnlockUnref state_var_guard(var);
+  Tensor* var_tensor = var->tensor();
+  if (var_tensor->dtype() != STATE_ELEMENT_DTYPE) {
+    return errors::InvalidArgument("dtype of RNG state variable must be ",
+                                   DataTypeString(STATE_ELEMENT_DTYPE),
+                                   ", not ",
+                                   DataTypeString(var_tensor->dtype()));
   }
-}
-
-// The following two functions use the contract "lower 32 bits for the first
-// uint32, higher 32 bits for the second". Note that this is endian-neutral,
-// unlike a direct memory copy `memcpy(output, &input, 8)`.
-void Int64ToUint32s(int64 input, uint32* output1, uint32* output2) {
-  auto u64 = static_cast<uint64>(input);
-  *output1 = static_cast<uint32>(u64);
-  *output2 = static_cast<uint32>(u64 >> 32);
-}
-
-int64 Uint32sToInt64(uint32 input1, uint32 input2) {
-  auto u64_1 = static_cast<uint64>(input1);
-  auto u64_2 = static_cast<uint64>(input2);
-  return static_cast<int64>(u64_1 | (u64_2 << 32));
-}
-
-void GetPhiloxStateFromTensor(Tensor const& tensor,
-                              PhiloxRandom::ResultType* counter,
-                              PhiloxRandom::Key* key) {
-  auto tensor_flat = tensor.flat<StateElementType>();
-  auto tensor_ptr = tensor_flat.data();
-  // tensor_ptr's index is added by 1 to skip the algorithm tag.
-  Int64ToUint32s(tensor_ptr[1], &(*counter)[0], &(*counter)[1]);
-  Int64ToUint32s(tensor_ptr[2], &(*counter)[2], &(*counter)[3]);
-  Int64ToUint32s(tensor_ptr[3], &(*key)[0], &(*key)[1]);
-}
-
-void WritePhiloxStateToTensor(PhiloxRandom::ResultType const& counter,
-                              PhiloxRandom::Key const& key, Tensor* tensor) {
-  auto tensor_flat = tensor->flat<StateElementType>();
-  auto tensor_ptr = tensor_flat.data();
-  // tensor_ptr's index is added by 1 to skip the algorithm tag.
-  tensor_ptr[1] = Uint32sToInt64(counter[0], counter[1]);
-  tensor_ptr[2] = Uint32sToInt64(counter[2], counter[3]);
-  tensor_ptr[3] = Uint32sToInt64(key[0], key[1]);
-}
-
-// A helper function that does the actual work for
-// 'MakeRNGCopyAndUpdateVariable'.
-template <typename Device>
-Status GetRNGCopyAndUpdateTensor(Tensor* tensor, int64 delta,
-                                 SkippableRNG* rng_copy);
-
-template <>
-Status GetRNGCopyAndUpdateTensor<CPUDevice>(Tensor* tensor, int64 delta,
-                                            SkippableRNG* rng_copy) {
-  // The dtype of `tensor` should be `StateElementType` and the first element
-  // is the algorithm.
-  if (tensor->dims() != 1) {
+  if (var_tensor->dims() != 1) {
     return errors::InvalidArgument(
-        "RNG state must have one and only one dimension, not ", tensor->dims());
+        "RNG state must have one and only one dimension, not ",
+        var_tensor->dims());
   }
-  auto tensor_flat = tensor->flat<StateElementType>();
-  if (tensor_flat.size() < 1) {
-    return errors::InvalidArgument("Size of tensor must be at least 1");
+  auto var_tensor_flat = var_tensor->flat<StateElementType>();
+  int64 alg_tag_skip = 0;
+  if (read_alg_from_state) {
+    alg_tag_skip = 1;
+    if (var_tensor_flat.size() < 1) {
+      return errors::InvalidArgument("Size of tensor must be at least 1");
+    }
+    alg = var_tensor_flat(0);
   }
-  auto algorithm = tensor_flat.data()[0];
-  if (algorithm == RNG_ALG_PHILOX) {
-    // Delegates to PhiloxRandom to do the actual increasing.
+  if (alg == RNG_ALG_PHILOX) {
     static_assert(std::is_same<StateElementType, int64>::value,
                   "StateElementType must be int64");
     static_assert(std::is_same<PhiloxRandom::ResultElementType, uint32>::value,
                   "PhiloxRandom::ResultElementType must be uint32");
-    auto counter_size = PhiloxRandom::ResultType::kElementCount;
-    auto key_size = PhiloxRandom::Key::kElementCount;
-    auto min_tensor_size = 1 + (counter_size + key_size) / 2;
-    if (tensor_flat.size() < min_tensor_size) {
+    if (var_tensor_flat.size() < alg_tag_skip + PHILOX_MIN_STATE_SIZE) {
       return errors::InvalidArgument(
           "For Philox algorithm, the size of state"
           " must be at least ",
-          min_tensor_size, "; got ", tensor_flat.size());
+          alg_tag_skip + PHILOX_MIN_STATE_SIZE, "; got ",
+          var_tensor_flat.size());
     }
-    PhiloxRandom::ResultType counter;
-    PhiloxRandom::Key key;
-    GetPhiloxStateFromTensor(*tensor, &counter, &key);
-    PhiloxRandom philox(counter, key);
-    auto old_philox = philox;
-    philox.Skip(delta);  // do the actual increasing
-    WritePhiloxStateToTensor(philox.counter(), philox.key(), tensor);
-    *rng_copy = SkippableRNG(old_philox);
+    TF_RETURN_IF_ERROR(PrepareToUpdateVariable<Device, StateElementType>(
+        ctx, var_tensor, var->copy_on_read_mode.load()));
+    UpdateVariableAndFill_Philox<Device, Distribution>()(
+        ctx, ctx->eigen_device<Device>(), output_size, alg_tag_skip,
+        &state_var_guard, var_tensor, output_data);
     return Status::OK();
   } else {
-    // return errors::InvalidArgument("Unsupported algorithm id: ", algorithm);
-    *rng_copy = SkippableRNG(PhiloxRandom());
-    return Status::OK();
+    return errors::InvalidArgument("Unsupported algorithm id: ", alg);
   }
 }
 
-// Gets a copy of the RNG and updates the variable. The copy can be used to
-// generate upto 'samples' random numbers, and the variable is updated as if
-// 'samples' random numbers have been generated (e.g. if the variable is a
-// counnter, the counter is increased by 'samples').
-template <class Device>
-Status MakeRNGCopyAndUpdateVariable(OpKernelContext* ctx, int input_idx,
-                                    int64 samples, SkippableRNG* rng_copy) {
-  Var* var = nullptr;
-  TF_RETURN_IF_ERROR(
-      LookupResource(ctx, HandleFromInput(ctx, input_idx), &var));
-  core::ScopedUnref s(var);
-  mutex_lock ml(*var->mu());
-  Tensor* var_tensor = var->tensor();
-  if (var_tensor->dtype() != STATE_ELEMENT_DTYPE) {
-    return errors::InvalidArgument("dtype of RNG state variable must be ",
-                                   DataTypeString(STATE_ELEMENT_DTYPE),
-                                   ", not ",
-                                   DataTypeString(var_tensor->dtype()));
-  }
-  TF_RETURN_IF_ERROR(PrepareToUpdateVariable<Device, StateElementType>(
-      ctx, var_tensor, var->copy_on_read_mode.load()));
-  TF_RETURN_IF_ERROR(
-      GetRNGCopyAndUpdateTensor<Device>(var_tensor, samples, rng_copy));
-  return Status::OK();
+// Preconditon: input(0) is an existing resource.
+template <typename Device, class Distribution>
+void ComputeImpl(OpKernelContext* ctx, int state_input_idx, int shape_input_idx,
+                 bool read_alg_from_state, Algorithm alg) {
+  using T = typename Distribution::ResultElementType;
+  const Tensor& shape_t = ctx->input(shape_input_idx);
+  TensorShape shape;
+  OP_REQUIRES_OK(ctx, ctx->op_kernel().MakeShape(shape_t, &shape));
+  Tensor* output;
+  OP_REQUIRES_OK(ctx, ctx->allocate_output(0, shape, &output));
+  auto output_flat = output->flat<T>();
+  OP_REQUIRES_OK(ctx, UpdateVariableAndFill<Device, Distribution>(
+                          ctx, state_input_idx, read_alg_from_state, alg,
+                          output_flat.size(), output_flat.data()));
 }
 
 template <typename Device, class Distribution>
 class StatefulRandomOp : public OpKernel {
  public:
-  using T = typename Distribution::ResultElementType;
   explicit StatefulRandomOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}
 
-  // Assumes that input(0) is an existing resource.
   void Compute(OpKernelContext* ctx) override {
-    const Tensor& shape_t = ctx->input(1);
-    Tensor* output;
-    TensorShape shape;
-    OP_REQUIRES_OK(ctx, MakeShape(shape_t, &shape));
-    OP_REQUIRES_OK(ctx, ctx->allocate_output(0, shape, &output));
-    if (shape.num_elements() == 0) return;
-
-    auto output_flat = output->flat<T>();
-    SkippableRNG rng;
-    // Multiplier 256 is the same as in FillPhiloxRandomTask; do not change
-    // it just here.
-    OP_REQUIRES_OK(ctx, MakeRNGCopyAndUpdateVariable<Device>(
-                            ctx, 0, output_flat.size() * 256, &rng));
-    // Fill in the random numbers
-    OP_REQUIRES_OK(ctx, FillRandom(ctx, ctx->eigen_device<Device>(), rng,
-                                   output_flat.size(), Distribution(),
-                                   output_flat.data()));
+    ComputeImpl<Device, Distribution>(ctx, 0, 1, true, 0);
   }
 };
 
-}  // namespace
+template <typename Device, class Distribution>
+class StatefulRandomOpV2 : public OpKernel {
+ public:
+  explicit StatefulRandomOpV2(OpKernelConstruction* ctx) : OpKernel(ctx) {}
+
+  void Compute(OpKernelContext* ctx) override {
+    const Tensor& alg_tensor = ctx->input(1);
+    OP_REQUIRES(ctx, alg_tensor.dims() == 0,
+                errors::InvalidArgument("algorithm must be of shape [], not ",
+                                        alg_tensor.shape().DebugString()));
+    auto alg = alg_tensor.flat<Algorithm>()(0);
+    ComputeImpl<Device, Distribution>(ctx, 0, 2, false, alg);
+  }
+};
 
 // So far the 'Distribution' type parameter is only used when the algorithm is
 // philox, so 'NormalDistribution<PhiloxRandom, ...>' is fine for now.
 #define REGISTER(DEVICE, TYPE)            \
   REGISTER_KERNEL_BUILDER(                \
-      Name("StatefulStandardNormal")      \
+      Name("StatefulStandardNormalV2")    \
           .Device(DEVICE_##DEVICE)        \
           .HostMemory("resource")         \
+          .HostMemory("algorithm")        \
+          .HostMemory("shape")            \
+          .TypeConstraint<TYPE>("dtype"), \
+      StatefulRandomOpV2<DEVICE##Device,  \
+                         random::NormalDistribution<PhiloxRandom, TYPE> >);
+
+// CPU also has the old 'StatefulStandardNormal' op for backward compatibility.
+#define REGISTER_CPU(TYPE)                \
+  REGISTER(CPU, TYPE)                     \
+  REGISTER_KERNEL_BUILDER(                \
+      Name("StatefulStandardNormal")      \
+          .Device(DEVICE_CPU)             \
+          .HostMemory("resource")         \
           .HostMemory("shape")            \
           .TypeConstraint<TYPE>("dtype"), \
-      StatefulRandomOp<DEVICE##Device,    \
+      StatefulRandomOp<CPUDevice,         \
                        random::NormalDistribution<PhiloxRandom, TYPE> >);
 
-#define REGISTER_CPU(TYPE) REGISTER(CPU, TYPE)
+#define REGISTER_GPU(TYPE) REGISTER(GPU, TYPE)
 
 TF_CALL_half(REGISTER_CPU);
 TF_CALL_bfloat16(REGISTER_CPU);
 TF_CALL_float(REGISTER_CPU);
 TF_CALL_double(REGISTER_CPU);
 
+#if GOOGLE_CUDA
+
+TF_CALL_half(REGISTER_GPU);
+TF_CALL_float(REGISTER_GPU);
+TF_CALL_double(REGISTER_GPU);
+
+#endif  // GOOGLE_CUDA
+
+#undef REGISTER_GPU
 #undef REGISTER_CPU
 #undef REGISTER
 
 // TODO(wangpeng): Add RNG ops for other distributions.
-// TODO(wangpeng): Add support for GPU and XLA.
+// TODO(wangpeng): Add support for XLA.
 
 }  // end namespace tensorflow
diff --git a/tensorflow/core/kernels/stateful_random_ops.h b/tensorflow/core/kernels/stateful_random_ops.h
new file mode 100644
index 0000000000..845332719c
--- /dev/null
+++ b/tensorflow/core/kernels/stateful_random_ops.h
@@ -0,0 +1,137 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_KERNELS_STATEFUL_RANDOM_OPS_H_
+#define TENSORFLOW_CORE_KERNELS_STATEFUL_RANDOM_OPS_H_
+
+#include "tensorflow/core/framework/resource_var.h"
+#include "tensorflow/core/lib/random/philox_random.h"
+
+namespace tensorflow {
+
+// 'Variable' doesn't support uint32 or uint64 yet (due to reasons explained
+// in b/111604096 and cl/171681867), so I use signed int here. I choose int64
+// instead of int32 because `VarHandleOp` doesn't support int32 on GPU.
+using StateElementType = int64;
+static constexpr DataType STATE_ELEMENT_DTYPE = DT_INT64;
+
+using Algorithm = StateElementType;
+static constexpr Algorithm RNG_ALG_PHILOX = 1;
+
+using random::PhiloxRandom;
+
+static constexpr int64 PHILOX_MIN_STATE_SIZE =
+    (PhiloxRandom::ResultType::kElementCount +
+     PhiloxRandom::Key::kElementCount) /
+    2;
+
+// The following two functions use the contract "lower 32 bits for the first
+// uint32, higher 32 bits for the second". Note that this is endian-neutral,
+// unlike a direct memory copy `memcpy(output, &input, 8)`.
+PHILOX_DEVICE_FUNC void Int64ToUint32s(int64 input, uint32* output1,
+                                       uint32* output2) {
+  auto u64 = static_cast<uint64>(input);
+  *output1 = static_cast<uint32>(u64);
+  *output2 = static_cast<uint32>(u64 >> 32);
+}
+
+PHILOX_DEVICE_FUNC int64 Uint32sToInt64(uint32 input1, uint32 input2) {
+  auto u64_1 = static_cast<uint64>(input1);
+  auto u64_2 = static_cast<uint64>(input2);
+  return static_cast<int64>(u64_1 | (u64_2 << 32));
+}
+
+PHILOX_DEVICE_FUNC PhiloxRandom
+GetPhiloxRandomFromMem(StateElementType const* ptr) {
+  PhiloxRandom::ResultType counter;
+  PhiloxRandom::Key key;
+  Int64ToUint32s(ptr[0], &counter[0], &counter[1]);
+  Int64ToUint32s(ptr[1], &counter[2], &counter[3]);
+  Int64ToUint32s(ptr[2], &key[0], &key[1]);
+  return PhiloxRandom(counter, key);
+}
+
+PHILOX_DEVICE_FUNC void WritePhiloxRandomToMem(PhiloxRandom const& philox,
+                                               StateElementType* ptr) {
+  PhiloxRandom::ResultType const& counter = philox.counter();
+  PhiloxRandom::Key const& key = philox.key();
+  ptr[0] = Uint32sToInt64(counter[0], counter[1]);
+  ptr[1] = Uint32sToInt64(counter[2], counter[3]);
+  ptr[2] = Uint32sToInt64(key[0], key[1]);
+}
+
+PHILOX_DEVICE_FUNC void UpdateMemWithPhiloxRandom(PhiloxRandom const& philox,
+                                                  int64 output_size,
+                                                  StateElementType* ptr) {
+  auto new_philox = philox;
+  // Multiplier 256 is the same as in `FillPhiloxRandomTask`; do not change
+  // it just here.
+  auto delta = output_size * 256;
+  new_philox.Skip(delta);  // do the actual increasing
+  WritePhiloxRandomToMem(new_philox, ptr);
+}
+
+// Does unlock and unref automatically when going out of scope, and also
+// supports early manual release.
+class ScopedUnlockUnref {
+ public:
+  explicit ScopedUnlockUnref(Var* var) : var_(var) {
+    if (var_) {
+      var_->mu()->lock();
+    }
+  }
+  void Release() {
+    if (var_) {
+      var_->mu()->unlock();
+      var_->Unref();
+      var_ = nullptr;
+    }
+  }
+  ~ScopedUnlockUnref() { Release(); }
+
+ private:
+  Var* var_;
+
+  ScopedUnlockUnref(const ScopedUnlockUnref&) = delete;
+  void operator=(const ScopedUnlockUnref&) = delete;
+};
+
+// A per-device helper function that does the actual work for
+// `UpdateVariableAndFill`.
+// Reason to use functor: C++ doesn't allow function-template partial
+// specialization.
+template <typename Device, typename Distribution>
+struct UpdateVariableAndFill_Philox;
+
+using CPUDevice = Eigen::ThreadPoolDevice;
+
+#if GOOGLE_CUDA
+
+using GPUDevice = Eigen::GpuDevice;
+
+// Declares the partially GPU-specialized functor struct.
+template <typename Distribution>
+struct UpdateVariableAndFill_Philox<GPUDevice, Distribution> {
+  void operator()(OpKernelContext* ctx, const GPUDevice& device,
+                  int64 output_size, int64 alg_tag_skip,
+                  ScopedUnlockUnref* not_used, Tensor* state_tensor,
+                  typename Distribution::ResultElementType* output_data);
+};
+
+#endif  // GOOGLE_CUDA
+
+}  // end namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_KERNELS_STATEFUL_RANDOM_OPS_H_
diff --git a/tensorflow/core/kernels/stateful_random_ops_gpu.cu.cc b/tensorflow/core/kernels/stateful_random_ops_gpu.cu.cc
new file mode 100644
index 0000000000..4ce0db7c56
--- /dev/null
+++ b/tensorflow/core/kernels/stateful_random_ops_gpu.cu.cc
@@ -0,0 +1,96 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/random_op_gpu.h"
+#include "tensorflow/core/kernels/stateful_random_ops.h"
+#include "tensorflow/core/util/cuda_launch_config.h"
+
+namespace tensorflow {
+
+using random::PhiloxRandom;
+
+__device__ int thread_counter;
+
+template <typename Distribution>
+__global__ void FillKernel(
+    Distribution dist, int64 state_size, int64 output_size,
+    StateElementType* state_data,
+    typename Distribution::ResultElementType* output_data) {
+  // Threads in this block share `philox`. Thread 0 is responsible for
+  // initializing it.
+  __shared__ char philox_raw[sizeof(PhiloxRandom)];
+  auto philox = reinterpret_cast<PhiloxRandom*>(philox_raw);
+  if (threadIdx.x == 0) {
+    *philox = GetPhiloxRandomFromMem(state_data);
+  }
+  __syncthreads();
+  functor::FillPhiloxRandomKernel<Distribution,
+                                  Distribution::kVariableSamplesPerOutput>()
+      .Run(*philox, output_data, output_size, dist);
+  // The last thread updates the state.
+  auto total_thread_count = gridDim.x * blockDim.x;
+  auto old_counter_value = atomicAdd(&thread_counter, 1);
+  if (old_counter_value == total_thread_count - 1) {
+    UpdateMemWithPhiloxRandom(*philox, output_size, state_data);
+  }
+}
+
+template <typename Distribution>
+void UpdateVariableAndFill_Philox<GPUDevice, Distribution>::operator()(
+    OpKernelContext* ctx, const GPUDevice& d, int64 output_size,
+    int64 alg_tag_skip, ScopedUnlockUnref* not_used, Tensor* state_tensor,
+    typename Distribution::ResultElementType* output_data) {
+  OP_REQUIRES(
+      ctx, alg_tag_skip == 0,
+      errors::InvalidArgument(
+          "GPU kernel doesn't support reading algorithm from state variable, "
+          "so alg_tag_skip must be 0; got",
+          alg_tag_skip));
+  auto state_tensor_flat = state_tensor->flat<StateElementType>();
+  auto state_size = state_tensor_flat.size();
+  auto state_data = state_tensor_flat.data();
+
+  // maximize occupancy
+  const int kGroupSize = Distribution::kResultElementCount;
+  int work_element_count = (output_size + kGroupSize - 1) / kGroupSize;
+  CudaLaunchConfig cfg = GetCudaLaunchConfig(work_element_count, d,
+                                             FillKernel<Distribution>, 0, 0);
+
+  int zero = 0;
+  cudaMemcpyToSymbol(thread_counter, &zero, sizeof(int));
+  FillKernel<Distribution>
+      <<<cfg.block_count, cfg.thread_per_block, 0, d.stream()>>>(
+          Distribution(), state_size, output_size, state_data, output_data);
+}
+
+// Explicit instantiation of the GPU distributions functors.
+
+// clang-format off
+// NVCC cannot handle ">>" properly
+template struct UpdateVariableAndFill_Philox<
+    GPUDevice, random::NormalDistribution<random::PhiloxRandom, Eigen::half> >;
+template struct UpdateVariableAndFill_Philox<
+    GPUDevice, random::NormalDistribution<random::PhiloxRandom, float> >;
+template struct UpdateVariableAndFill_Philox<
+    GPUDevice, random::NormalDistribution<random::PhiloxRandom, double> >;
+// clang-format on
+
+}  // end namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/ops/stateful_random_ops.cc b/tensorflow/core/ops/stateful_random_ops.cc
index 4c9277eda5..2162107b7e 100644
--- a/tensorflow/core/ops/stateful_random_ops.cc
+++ b/tensorflow/core/ops/stateful_random_ops.cc
@@ -19,18 +19,42 @@ limitations under the License.
 namespace tensorflow {
 
 Status StatefulRandomShape(shape_inference::InferenceContext* c) {
-  shape_inference::ShapeHandle out;
-  TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(1, &out));
+  using shape_inference::ShapeHandle;
+
+  // Check algorithm shape
+  ShapeHandle unused;
+  TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused));
+
+  // Set output shape
+  ShapeHandle out;
+  TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(2, &out));
   c->set_output(0, out);
   return Status::OK();
 }
 
-REGISTER_OP("StatefulStandardNormal")
+REGISTER_OP("StatefulStandardNormalV2")
     .Input("resource: resource")
+    .Input("algorithm: int64")
     .Input("shape: shape_dtype")
     .Output("output: dtype")
-    .Attr("dtype: {half,bfloat16,float,double} = DT_FLOAT")
-    .Attr("shape_dtype: {int32, int64} = DT_INT64")
+    .Attr("dtype : type = DT_FLOAT")
+    .Attr("shape_dtype : type = DT_INT64")
     .SetShapeFn(StatefulRandomShape);
 
+// Register the old 'StatefulStandardNormal' op
+REGISTER_OP("StatefulStandardNormal")
+    .Input("resource: resource")
+    .Input("shape: shape_dtype")
+    .Output("output: dtype")
+    .Attr("dtype : type = DT_FLOAT")
+    .Attr("shape_dtype : type = DT_INT64")
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      using shape_inference::ShapeHandle;
+      // Set output shape
+      ShapeHandle out;
+      TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(1, &out));
+      c->set_output(0, out);
+      return Status::OK();
+    });
+
 }  // namespace tensorflow
diff --git a/tensorflow/python/ops/stateful_random_ops.py b/tensorflow/python/ops/stateful_random_ops.py
index 155ad969f6..ab6f419d32 100644
--- a/tensorflow/python/ops/stateful_random_ops.py
+++ b/tensorflow/python/ops/stateful_random_ops.py
@@ -51,6 +51,7 @@ SEED_SIZE = 16  # in units of SEED_TYPE
 
 
 STATE_TYPE = SEED_TYPE
+ALGORITHM_TYPE = STATE_TYPE
 RNG_ALG_PHILOX = 1
 DEFAULT_ALGORITHM = RNG_ALG_PHILOX
 
@@ -119,26 +120,18 @@ def _make_state_from_seed(seed, algorithm):
     raise ValueError("Unsupported algorithm id: %s" % algorithm)
 
 
-def create_rng_state(seed, algorithm=None):
+@tf_export("random.create_rng_state")
+def create_rng_state(seed, algorithm):
   """Creates a RNG state.
 
   Args:
     seed: an integer or 1-D tensor.
-    algorithm: (optional) an integer representing the RNG algorithm. If None, an
-      algorithm will be auto-selected.
+    algorithm: an integer representing the RNG algorithm.
 
   Returns:
-    a 1-D tensor "rng_state" with:
-    * rng_state[0] is a value that identifies the RNG algorithm;
-    * rng_state[1:] holds the RNG state itself (size dependent on the
-        algorithm).
+    a 1-D tensor whose size depends on the algorithm.
   """
-  if algorithm is None:
-    # TODO(wangpeng): more sophisticated algorithm selection
-    algorithm = DEFAULT_ALGORITHM
-  state = _make_state_from_seed(seed, algorithm)
-  return np.concatenate((np.array([algorithm], dtype=STATE_TYPE), state),
-                        axis=None)
+  return _make_state_from_seed(seed, algorithm)
 
 
 def _shape_tensor(shape):
@@ -158,17 +151,41 @@ class Generator(tracking.AutoCheckpointable):
   """
 
   def __init__(self, copy_from=None, seed=None, algorithm=None):
+    """Creates a generator.
+
+    Args:
+      copy_from: (optional) a generator to be copied from.
+      seed: (optional) the seed for the RNG. If None, it will be chosen
+            nondeterministically
+      algorithm: (optional) the RNG algorithm. If None, it will be
+                 auto-selected.
+    """
     if copy_from is None:
       if seed is None:
         seed = non_deterministic_seed()
+      if algorithm is None:
+        # TODO(wangpeng): more sophisticated algorithm selection
+        algorithm = DEFAULT_ALGORITHM
       state = create_rng_state(seed, algorithm)
       self._state_var = variables.Variable(state, dtype=STATE_TYPE)
+      self._alg_var = variables.Variable(initial_value=algorithm,
+                                         dtype=ALGORITHM_TYPE)
     else:
       assert seed is None
-      state = copy_from.state
-      self._state_var = variables.Variable(state, dtype=STATE_TYPE)
+      self._state_var = variables.Variable(copy_from.state, dtype=STATE_TYPE)
+      self._alg_var = variables.Variable(initial_value=copy_from.algorithm,
+                                         dtype=ALGORITHM_TYPE)
 
   def reset(self, seed):
+    """Resets the generator.
+
+    This function is not thread-safe: if it is run concurrently with a call to
+    sampling, the latter might see the new algorithm but the old state or vice
+    versa.
+
+    Args:
+      seed: the seed to reset the RNG to.
+    """
     algorithm = int(self.algorithm)
     state = create_rng_state(seed, algorithm)
     self._state_var.assign(state)
@@ -179,13 +196,14 @@ class Generator(tracking.AutoCheckpointable):
 
   @property
   def algorithm(self):
-    return self._state_var[0]
+    return self._alg_var
 
   # The following functions return a tensor and as a side effect update
   # self._state_var.
   def standard_normal(self, shape, dtype=dtypes.float32):
-    return gen_stateful_random_ops.stateful_standard_normal(
-        self.state.handle, shape, dtype)
+    output = gen_stateful_random_ops.stateful_standard_normal_v2(
+        self.state.handle, self.algorithm, shape, dtype)
+    return output
 
   def normal(self, shape, mean=0.0, stddev=1.0, dtype=dtypes.float32,
              name=None):
@@ -237,5 +255,6 @@ def set_global_generator(generator):
 def reset_global_generator(seed, algorithm=None):
   global global_generator
   if algorithm is None:
-    algorithm = int(global_generator.algorithm)  # preserve the old algorithm
+    # preserve the old algorithm
+    algorithm = int(get_global_generator().algorithm)
   global_generator = Generator(seed=seed, algorithm=algorithm)
diff --git a/tensorflow/python/ops/stateful_random_ops_test.py b/tensorflow/python/ops/stateful_random_ops_test.py
index 92419a0770..0436736e00 100644
--- a/tensorflow/python/ops/stateful_random_ops_test.py
+++ b/tensorflow/python/ops/stateful_random_ops_test.py
@@ -18,15 +18,19 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import numpy as np
+
 from tensorflow.python.eager import def_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import errors_impl
+from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import gen_random_ops
+from tensorflow.python.ops import gen_stateful_random_ops
 from tensorflow.python.ops import logging_ops
 from tensorflow.python.ops import stateful_random_ops as \
 random
+from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 
 
@@ -39,8 +43,8 @@ class StatefulRandomOpsTest(test.TestCase):
                                     random.RNG_ALG_PHILOX)
     self.assertAllEqual(
         list(map(random._uint_to_int,
-                 [random.RNG_ALG_PHILOX, 0xFFAA666677778888,
-                  0xFFFF222233334444] + [0] * (random.PHILOX_STATE_SIZE - 2))),
+                 [0xFFAA666677778888, 0xFFFF222233334444] +
+                 [0] * (random.PHILOX_STATE_SIZE - 2))),
         state)
 
   @test_util.run_v2_only
@@ -112,27 +116,81 @@ class StatefulRandomOpsTest(test.TestCase):
     compare(True, False)
 
   @test_util.run_v2_only
-  def testSameAsOldRandomOps(self):
-    """Tests that the generated numbers are the same as the old random_ops.py .
+  def testCPUSameAsOldRandomOps(self):
+    """Tests that the generated numbers are the same as the old random_ops.py.
+
+    The CPU version.
     """
-    seed1, seed2 = 50, 60
+    seed1, seed2 = 79, 25
     # note how the two seeds for the old op correspond to the seed for the new
     # op
-    random.get_global_generator().reset([0, seed2, seed1])
-    shape = constant_op.constant([2, 3])
-    dtype = dtypes.float32
+    with ops.device("/device:CPU:0"):
+      random.reset_global_generator([0, seed2, seed1])
+    shape = constant_op.constant([4, 7])
+    dtype = dtypes.float64
+
     # create a graph for the old op in order to call it many times
     @def_function.function
     def old():
-      return gen_random_ops.random_standard_normal(
-          shape, dtype=dtype, seed=seed1, seed2=seed2)
+      with ops.device("/device:CPU:0"):
+        return gen_random_ops.random_standard_normal(
+            shape, dtype=dtype, seed=seed1, seed2=seed2)
+
+    def new():
+      with ops.device("/device:CPU:0"):
+        return random.get_global_generator().standard_normal(shape, dtype=dtype)
+
+    for _ in range(100):
+      self.assertAllEqual(old(), new())
+
+  @test_util.run_v2_only
+  @test_util.run_cuda_only
+  def testGPUSameAsOldRandomOps(self):
+    """Tests that the generated numbers are the same as the old random_ops.py.
+
+    The GPU version.
+    """
+    seed1, seed2 = 79, 25
+    with ops.device(test_util.gpu_device_name()):
+      random.reset_global_generator([0, seed2, seed1])
+    shape = constant_op.constant([4, 7])
+    dtype = dtypes.float64
+
+    @def_function.function
+    def old():
+      with ops.device(test_util.gpu_device_name()):
+        return gen_random_ops.random_standard_normal(
+            shape, dtype=dtype, seed=seed1, seed2=seed2)
 
     def new():
-      return random.get_global_generator().standard_normal(shape, dtype=dtype)
+      with ops.device(test_util.gpu_device_name()):
+        return random.get_global_generator().standard_normal(shape, dtype=dtype)
 
     for _ in range(100):
       self.assertAllEqual(old(), new())
 
+  @test_util.run_v2_only
+  def testStatefulStandardNormal(self):
+    """Tests that op 'StatefulStandardNormal' still works.
+    """
+    shape = constant_op.constant([4, 7])
+    dtype = dtypes.float64
+    seed = 1234
+    algorithm = random.RNG_ALG_PHILOX
+    state = random._make_state_from_seed(seed, algorithm)
+    with ops.device("/device:CPU:0"):
+      var1 = variables.Variable(
+          np.concatenate((np.array([algorithm], dtype=random.STATE_TYPE),
+                          state), axis=None),
+          dtype=random.STATE_TYPE)
+      var2 = variables.Variable(state, dtype=random.STATE_TYPE)
+      for _ in range(100):
+        t1 = gen_stateful_random_ops.stateful_standard_normal(
+            var1.handle, shape, dtype)
+        t2 = gen_stateful_random_ops.stateful_standard_normal_v2(
+            var2.handle, algorithm, shape, dtype)
+        self.assertAllEqual(t1, t2)
+
   @test_util.run_v2_only
   def testResetGlobalGeneratorBadWithDefun(self):
     """Demonstrates that reset_global_generator don't work properly with defun.
@@ -145,7 +203,7 @@ class StatefulRandomOpsTest(test.TestCase):
 
     random.reset_global_generator(50)
     with self.assertRaisesWithPredicateMatch(
-        errors_impl.NotFoundError, "Resource .+ does not exist"):
+        AssertionError, "variable.*deleted"):
       a = f()
       random.reset_global_generator(50)
       b = f()
-- 
GitLab


From c2e65d5a444a55f2b44b934cdd4baf94ce4984e7 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Mon, 11 Feb 2019 13:56:44 -0800
Subject: [PATCH 2277/2345] Change default value of --only_test_core_api.

PiperOrigin-RevId: 233472937
---
 tensorflow/tools/api/tests/BUILD                     | 1 -
 tensorflow/tools/api/tests/api_compatibility_test.py | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/tools/api/tests/BUILD b/tensorflow/tools/api/tests/BUILD
index 3cbea41dca..4efa4a9651 100644
--- a/tensorflow/tools/api/tests/BUILD
+++ b/tensorflow/tools/api/tests/BUILD
@@ -19,7 +19,6 @@ py_test(
         "api_compatibility_test.py",
         "//tensorflow:tf_python_api_gen_v2",
     ],
-    args = ["--only_test_core_api=true"],
     data = [
         "//tensorflow/tools/api/golden:api_golden_v1",
         "//tensorflow/tools/api/golden:api_golden_v2",
diff --git a/tensorflow/tools/api/tests/api_compatibility_test.py b/tensorflow/tools/api/tests/api_compatibility_test.py
index 9350ebba57..a467a22f5d 100644
--- a/tensorflow/tools/api/tests/api_compatibility_test.py
+++ b/tensorflow/tools/api/tests/api_compatibility_test.py
@@ -363,7 +363,7 @@ if __name__ == '__main__':
   parser.add_argument(
       '--only_test_core_api',
       type=bool,
-      default=False,
+      default=True,  # only_test_core_api default value
       help=_ONLY_TEST_CORE_API_HELP)
   parser.add_argument(
       '--verbose_diffs', type=bool, default=True, help=_VERBOSE_DIFFS_HELP)
-- 
GitLab


From d3f92de5aa5938c9980bf309d6eead618e4c5c18 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 11 Feb 2019 14:04:01 -0800
Subject: [PATCH 2278/2345] Add posix implementation for platform/numa.h
 functions, relying on hwloc.

PiperOrigin-RevId: 233474622
---
 tensorflow/core/BUILD                         |  26 +++-
 .../core/platform/default/build_config.bzl    |   6 +
 tensorflow/core/platform/posix/port.cc        | 140 +++++++++++++++++-
 tensorflow/tools/lib_package/BUILD            |   2 +
 tensorflow/tools/pip_package/BUILD            |   1 +
 5 files changed, 165 insertions(+), 10 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index c35b1bc651..2eb1ce11d1 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -386,13 +386,24 @@ cc_library(
         ":platform_port_hdrs",
         ":platform_port_internal_hdrs",
     ],
-    copts = tf_copts(),
+    copts = tf_copts() + [
+        "-Ithird_party/hwloc/hwloc-master/include",
+    ] + select({
+        "//tensorflow:android": [],
+        "//tensorflow:ios": [],
+        "//tensorflow:windows": [],
+        "//tensorflow:darwin": [],
+        "//conditions:default": [
+            "-DTENSORFLOW_USE_NUMA",
+        ],
+    }),
     visibility = ["//tensorflow/core:__subpackages__"],
     deps = [
         ":lib_platform",
         ":platform_base",
         "//tensorflow/core/platform/default/build_config:port",
         "@com_google_absl//absl/base",
+        "@hwloc",
         "@snappy",
     ],
 )
@@ -2300,7 +2311,17 @@ cc_library(
         tf_additional_proto_srcs(),
     ),
     hdrs = LIB_INTERNAL_PUBLIC_HEADERS,
-    copts = tf_copts(),
+    copts = tf_copts() + [
+        "-Ithird_party/hwloc/hwloc-master/include",
+    ] + select({
+        "//tensorflow:android": [],
+        "//tensorflow:ios": [],
+        "//tensorflow:windows": [],
+        "//tensorflow:darwin": [],
+        "//conditions:default": [
+            "-DTENSORFLOW_USE_NUMA",
+        ],
+    }),
     defines = LIB_INTERNAL_DEFINES,
     deps = tf_additional_lib_deps() + [
         ":lib_hash_crc32c_accelerate_internal",
@@ -2311,6 +2332,7 @@ cc_library(
         "@com_google_absl//absl/strings",
         "//third_party/eigen3",
         "//tensorflow/core/platform/default/build_config:platformlib",
+        "@hwloc//:hwloc",
         "@snappy",
         "@zlib_archive//:zlib",
         "@double_conversion//:double-conversion",
diff --git a/tensorflow/core/platform/default/build_config.bzl b/tensorflow/core/platform/default/build_config.bzl
index f9e3e1dc25..8a1ce18718 100644
--- a/tensorflow/core/platform/default/build_config.bzl
+++ b/tensorflow/core/platform/default/build_config.bzl
@@ -722,6 +722,12 @@ def tf_additional_gdr_lib_defines():
         "//conditions:default": [],
     })
 
+def tf_additional_numa_lib_defines():
+    return select({
+        "//tensorflow:with_numa_support": ["TENSORFLOW_USE_NUMA"],
+        "//conditions:default": [],
+    })
+
 def tf_py_clif_cc(name, visibility = None, **kwargs):
     pass
 
diff --git a/tensorflow/core/platform/posix/port.cc b/tensorflow/core/platform/posix/port.cc
index ea6066ac7b..69144ddddd 100644
--- a/tensorflow/core/platform/posix/port.cc
+++ b/tensorflow/core/platform/posix/port.cc
@@ -45,6 +45,10 @@ limitations under the License.
 #include <thread>
 #endif
 
+#if TENSORFLOW_USE_NUMA
+#include "hwloc.h"  // TF:hwloc
+#endif
+
 namespace tensorflow {
 namespace port {
 
@@ -113,16 +117,94 @@ int NumHyperthreadsPerCore() {
   return (ht_per_core > 0) ? ht_per_core : 1;
 }
 
-bool NUMAEnabled() {
-  // Not yet implemented: coming soon.
-  return false;
+#ifdef TENSORFLOW_USE_NUMA
+namespace {
+static hwloc_topology_t hwloc_topology_handle;
+
+bool HaveHWLocTopology() {
+  // One time initialization
+  static bool init = []() {
+    if (hwloc_topology_init(&hwloc_topology_handle)) {
+      LOG(ERROR) << "Call to hwloc_topology_init() failed";
+      return false;
+    }
+    if (hwloc_topology_load(hwloc_topology_handle)) {
+      LOG(ERROR) << "Call to hwloc_topology_load() failed";
+      return false;
+    }
+    return true;
+  }();
+  return init;
+}
+
+// Return the first hwloc object of the given type whose os_index
+// matches 'index'.
+hwloc_obj_t GetHWLocTypeIndex(hwloc_obj_type_t tp, int index) {
+  hwloc_obj_t obj = nullptr;
+  if (index >= 0) {
+    while ((obj = hwloc_get_next_obj_by_type(hwloc_topology_handle, tp, obj)) !=
+           nullptr) {
+      if (obj->os_index == index) break;
+    }
+  }
+  return obj;
 }
+}  // namespace
+#endif  // TENSORFLOW_USE_NUMA
+
+bool NUMAEnabled() { return (NUMANumNodes() > 1); }
 
-int NUMANumNodes() { return 1; }
+int NUMANumNodes() {
+#ifdef TENSORFLOW_USE_NUMA
+  if (HaveHWLocTopology()) {
+    int num_numanodes =
+        hwloc_get_nbobjs_by_type(hwloc_topology_handle, HWLOC_OBJ_NUMANODE);
+    return std::max(1, num_numanodes);
+  } else {
+    return 1;
+  }
+#else
+  return 1;
+#endif  // TENSORFLOW_USE_NUMA
+}
 
-void NUMASetThreadNodeAffinity(int node) {}
+void NUMASetThreadNodeAffinity(int node) {
+#ifdef TENSORFLOW_USE_NUMA
+  if (HaveHWLocTopology()) {
+    // Find the corresponding NUMA node topology object.
+    hwloc_obj_t obj = GetHWLocTypeIndex(HWLOC_OBJ_NUMANODE, node);
+    if (obj) {
+      hwloc_set_cpubind(hwloc_topology_handle, obj->cpuset,
+                        HWLOC_CPUBIND_THREAD | HWLOC_CPUBIND_STRICT);
+    } else {
+      LOG(ERROR) << "Could not find hwloc NUMA node " << node;
+    }
+  }
+#endif  // TENSORFLOW_USE_NUMA
+}
 
-int NUMAGetThreadNodeAffinity() { return kNUMANoAffinity; }
+int NUMAGetThreadNodeAffinity() {
+  int node_index = kNUMANoAffinity;
+#ifdef TENSORFLOW_USE_NUMA
+  if (HaveHWLocTopology()) {
+    hwloc_cpuset_t thread_cpuset = hwloc_bitmap_alloc();
+    hwloc_get_cpubind(hwloc_topology_handle, thread_cpuset,
+                      HWLOC_CPUBIND_THREAD);
+    hwloc_obj_t obj = nullptr;
+    // Return the first NUMA node whose cpuset is a (non-proper) superset of
+    // that of the current thread.
+    while ((obj = hwloc_get_next_obj_by_type(
+                hwloc_topology_handle, HWLOC_OBJ_NUMANODE, obj)) != nullptr) {
+      if (hwloc_bitmap_isincluded(thread_cpuset, obj->cpuset)) {
+        node_index = obj->os_index;
+        break;
+      }
+    }
+    hwloc_bitmap_free(thread_cpuset);
+  }
+#endif  // TENSORFLOW_USE_NUMA
+  return node_index;
+}
 
 void* AlignedMalloc(size_t size, int minimum_alignment) {
 #if defined(__ANDROID__)
@@ -152,12 +234,54 @@ void* Realloc(void* ptr, size_t size) { return realloc(ptr, size); }
 void Free(void* ptr) { free(ptr); }
 
 void* NUMAMalloc(int node, size_t size, int minimum_alignment) {
+#ifdef TENSORFLOW_USE_NUMA
+  if (HaveHWLocTopology()) {
+    hwloc_obj_t numa_node = GetHWLocTypeIndex(HWLOC_OBJ_NUMANODE, node);
+    if (numa_node) {
+      return hwloc_alloc_membind(hwloc_topology_handle, size,
+                                 numa_node->nodeset, HWLOC_MEMBIND_BIND,
+                                 HWLOC_MEMBIND_BYNODESET);
+    } else {
+      LOG(ERROR) << "Failed to find hwloc NUMA node " << node;
+    }
+  }
+#endif  // TENSORFLOW_USE_NUMA
   return AlignedMalloc(size, minimum_alignment);
 }
 
-void NUMAFree(void* ptr, size_t size) { Free(ptr); }
+void NUMAFree(void* ptr, size_t size) {
+#ifdef TENSORFLOW_USE_NUMA
+  if (HaveHWLocTopology()) {
+    hwloc_free(hwloc_topology_handle, ptr, size);
+    return;
+  }
+#endif  // TENSORFLOW_USE_NUMA
+  Free(ptr);
+}
 
-int NUMAGetMemAffinity(const void* addr) { return kNUMANoAffinity; }
+int NUMAGetMemAffinity(const void* addr) {
+  int node = kNUMANoAffinity;
+#ifdef TENSORFLOW_USE_NUMA
+  if (HaveHWLocTopology() && addr) {
+    hwloc_nodeset_t nodeset = hwloc_bitmap_alloc();
+    if (!hwloc_get_area_memlocation(hwloc_topology_handle, addr, 4, nodeset,
+                                    HWLOC_MEMBIND_BYNODESET)) {
+      hwloc_obj_t obj = nullptr;
+      while ((obj = hwloc_get_next_obj_by_type(
+                  hwloc_topology_handle, HWLOC_OBJ_NUMANODE, obj)) != nullptr) {
+        if (hwloc_bitmap_isincluded(nodeset, obj->nodeset)) {
+          node = obj->os_index;
+          break;
+        }
+      }
+      hwloc_bitmap_free(nodeset);
+    } else {
+      LOG(ERROR) << "Failed call to hwloc_get_area_memlocation.";
+    }
+  }
+#endif  // TENSORFLOW_USE_NUMA
+  return node;
+}
 
 void MallocExtension_ReleaseToSystem(std::size_t num_bytes) {
   // No-op.
diff --git a/tensorflow/tools/lib_package/BUILD b/tensorflow/tools/lib_package/BUILD
index 86bd510792..83365d0c5c 100644
--- a/tensorflow/tools/lib_package/BUILD
+++ b/tensorflow/tools/lib_package/BUILD
@@ -126,6 +126,7 @@ genrule(
         "@gemmlowp//:LICENSE",
         "@gif_archive//:COPYING",
         "@highwayhash//:LICENSE",
+        "@hwloc//:LICENSE",
         "@icu//:icu4c/LICENSE",
         "@jpeg//:LICENSE.md",
         "@llvm//:LICENSE.TXT",
@@ -195,6 +196,7 @@ genrule(
         "@gemmlowp//:LICENSE",
         "@gif_archive//:COPYING",
         "@highwayhash//:LICENSE",
+        "@hwloc//:LICENSE",
         "@icu//:icu4j/main/shared/licenses/LICENSE",
         "@jpeg//:LICENSE.md",
         "@llvm//:LICENSE.TXT",
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index 30ee5165af..b98a158cd2 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -170,6 +170,7 @@ filegroup(
         "@gemmlowp//:LICENSE",
         "@gif_archive//:COPYING",
         "@highwayhash//:LICENSE",
+        "@hwloc//:LICENSE",
         "@icu//:icu4c/LICENSE",
         "@jpeg//:LICENSE.md",
         "@keras_applications_archive//:LICENSE",
-- 
GitLab


From 6ce9cfcda7d8f657f2cebe12b74135e09e34f587 Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Mon, 11 Feb 2019 14:07:28 -0800
Subject: [PATCH 2279/2345] Increase RebatchDataset coverage by

1. Extending it to PaddedBatch, ExperimentalMapAndBatch.
2. Handle multiple batch transformations (picks last one)
3. Handle Concatenate and Zip transformations
4. Handling batching in FlatMap, Interleave or ParallelInterleave.

We also restrict the kinds of transformations allowed after the final batch transformation to the ones in core tf.data.

PiperOrigin-RevId: 233475396
---
 .../core/grappler/optimizers/data/BUILD       |   7 +-
 .../core/grappler/optimizers/data/rebatch.cc  | 249 ++++++++++++++--
 .../core/grappler/optimizers/data/rebatch.h   |  14 +-
 .../data/experimental/rebatch_dataset_op.cc   |  15 +-
 .../kernels/data/graph_rewrite_dataset.cc     |   3 +
 .../core/kernels/data/graph_rewrite_dataset.h |   3 +
 .../data/experimental/kernel_tests/BUILD      |   1 +
 .../kernel_tests/rebatch_dataset_test.py      | 273 +++++++++++++++++-
 .../python/data/experimental/ops/batching.py  |   5 +-
 9 files changed, 508 insertions(+), 62 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/data/BUILD b/tensorflow/core/grappler/optimizers/data/BUILD
index ef02962975..3fa6671c49 100644
--- a/tensorflow/core/grappler/optimizers/data/BUILD
+++ b/tensorflow/core/grappler/optimizers/data/BUILD
@@ -547,11 +547,14 @@ cc_library(
     hdrs = ["rebatch.h"],
     deps = [
         ":graph_utils",
+        ":optimizer_base",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "//tensorflow/core:framework",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:mutable_graph_view",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
-    ],
+        "//tensorflow/core/grappler/utils:functions",
+    ] + tf_protos_all(),
     alwayslink = 1,
 )
 
diff --git a/tensorflow/core/grappler/optimizers/data/rebatch.cc b/tensorflow/core/grappler/optimizers/data/rebatch.cc
index 187e1a62af..ce958c9235 100644
--- a/tensorflow/core/grappler/optimizers/data/rebatch.cc
+++ b/tensorflow/core/grappler/optimizers/data/rebatch.cc
@@ -15,10 +15,13 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/data/rebatch.h"
 
+#include "absl/container/flat_hash_map.h"
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/versions.pb.h"
 #include "tensorflow/core/grappler/grappler_item.h"
-#include "tensorflow/core/grappler/mutable_graph_view.h"
 #include "tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.h"
 #include "tensorflow/core/grappler/optimizers/data/graph_utils.h"
+#include "tensorflow/core/grappler/utils/functions.h"
 
 namespace tensorflow {
 namespace grappler {
@@ -35,7 +38,57 @@ namespace {
 
 constexpr char kCastOp[] = "Cast";
 constexpr char kRealDivOp[] = "RealDiv";
-constexpr char kBatchDatasetOp[] = "BatchDatasetV2";
+
+constexpr std::array<const char*, 5> kBatchDatasetOps = {
+    "BatchDataset",
+    "BatchDatasetV2",
+    "ExperimentalMapAndBatchDataset",
+    "PaddedBatchDataset",
+    "PaddedBatchDatasetV2"
+};
+
+constexpr std::array<const char*, 2> kMultipleInputsDatasetOps = {
+    "ConcatenateDataset",
+    "ZipDataset"
+};
+
+constexpr std::array<const char*, 17> kPassThroughOps = {
+    "CacheDataset",
+    "FilterDataset",
+    "FilterByLastComponentDataset",
+    "Identity",
+    "MapDataset",
+    "ModelDataset",
+    "OptimizeDataset",
+    "ParallelMapDataset",
+    "PrefetchDataset",
+    "ReduceDataset",
+    "RepeatDataset",
+    "ShardDataset",
+    "ShuffleAndRepeatDataset",
+    "ShuffleDataset",
+    "SkipDataset",
+    "TakeDataset",
+    "WindowDataset"
+};
+
+constexpr std::array<const char*, 3> kFuncDatasetOps = {
+    "FlatMapDataset",
+    "InterleaveDataset",
+    "ParallelInterleaveDatasetV2"
+};
+
+constexpr std::array<const char*, 9> kSourceDatasetOps = {
+    "FixedLengthRecordDataset",
+    "FixedLengthRecordDatasetV2",
+    "GeneratorDataset",
+    "RangeDataset",
+    "SparseTensorsSliceDataset",
+    "TensorDataset",
+    "TensorSliceDataset",
+    "TextLineDataset",
+    "TFRecordDataset"
+};
 
 NodeDef* AddCastNode(const string& input, DataType src_t, DataType dst_t,
                      MutableGraphView* graph) {
@@ -68,40 +121,174 @@ NodeDef* AddFloatDivNode(const string& input_x, const string& input_y,
   return AddBinaryNode(input_x, input_y, kRealDivOp, DT_FLOAT, graph);
 }
 
+template <std::size_t SIZE>
+bool IsDatasetNodeOfType(const NodeDef& node,
+                         const std::array<const char*, SIZE>& arr) {
+  for (const auto& dataset_op_name : arr) {
+    if (node.op() == dataset_op_name) return true;
+  }
+  return false;
+}
+
+// Given a "batch" dataset node, modifies the batch_size input to divide the
+// current batch size by num_workers.
+Status MutateBatchSize(const NodeDef& node, int64 num_workers,
+                       MutableGraphView* graph) {
+  // TODO(rohanj): Fix up the output_shapes attribute as well. For this Dataset
+  // as well as all the downstream datasets.
+  // For all the batching datasets the batch_size is input number 1.
+  // TODO(rohanj): Add an assertion for batch size not being a multiple of
+  // num_workers.
+  NodeDef* batch_size_node = graph_utils::GetInputNode(node, *graph, 1);
+  NodeDef tmp_node = *batch_size_node;
+  graph_utils::SetUniqueGraphNodeName(tmp_node.op(), graph->graph(), &tmp_node);
+  NodeDef* copy_batch_size_node = graph->AddNode(std::move(tmp_node));
+  NodeDef* float_copy_batch_size_node =
+      AddCastNode(copy_batch_size_node->name(), DT_INT64, DT_FLOAT, graph);
+  NodeDef* num_worker_node =
+      graph_utils::AddScalarConstNode<int64>(num_workers, graph);
+  NodeDef* float_num_worker_node =
+      AddCastNode(num_worker_node->name(), DT_INT64, DT_FLOAT, graph);
+  NodeDef* divided_batch_size_node = AddFloatDivNode(
+      float_copy_batch_size_node->name(), float_num_worker_node->name(), graph);
+  NodeDef* cast_new_batch_size_node =
+      AddCastNode(divided_batch_size_node->name(), DT_FLOAT, DT_INT64, graph);
+  // We don't call UpdateFanouts here because CSE elimination might lead to
+  // multiple nodes sharing the same batch size constant node. This is also
+  // why we don't delete batch_size_node as well.
+  TF_RETURN_IF_ERROR(graph->UpdateRegularFaninByPort(
+      node.name(), 1, {cast_new_batch_size_node->name(), 0}));
+  return Status::OK();
+}
+
+// There is one Sink node at least that is added to the end of the graph. We
+// find that node and return it. It is possible that there are multiple
+// Identity ops from the final Dataset op to that Sink node, but the recursive
+// graph traversal handles that.
+Status FindSinkNode(const GraphDef& graph_def, NodeDef* sink_node) {
+  absl::flat_hash_map<string, int> all_node_names;
+  absl::flat_hash_map<string, int> node_input_map;
+  for (int i = 0; i < graph_def.node_size(); ++i) {
+    all_node_names.insert_or_assign(graph_def.node(i).name(), i);
+    node_input_map.insert_or_assign(graph_def.node(i).name(), 0);
+  }
+  // Counts how many graph nodes is this node the input to. Candidate sink
+  // nodes are ones which are inputs into zero nodes.
+  for (const NodeDef& node : graph_def.node()) {
+    for (const string& input_name : node.input()) {
+      node_input_map[input_name]++;
+    }
+  }
+  for (const auto& it : node_input_map) {
+    if (it.second == 0) {
+      const NodeDef& sink_graph_node = graph_def.node(all_node_names[it.first]);
+      // Sometimes the searching surfaces Arg nodes in function cases that
+      // have no input. This check rejects those.
+      if (sink_graph_node.input_size() == 0) {
+        continue;
+      }
+      *sink_node = sink_graph_node;
+      return Status::OK();
+    }
+  }
+  return errors::InvalidArgument("Failed to find a sink node");
+}
+
+Status OptimizeGraph(const GrapplerItem& item, int64 num_workers,
+                     GraphDef* output);
+
+// Helper function that starts from a node in the graph and recurses into its
+// inputs trying to find a BatchDataset type operation to modify. During the
+// recursion it handles four kinds of cases.
+// 1. BatchDataset type ops: Mutates the batch_size input node and stops.
+// 2. Zip / Concatenate dataset ops: Recurses into all inputs to these ops
+//      as they are datasets themselves.
+// 3. Core dataset ops + Identity op: Recurses into first input parameter.
+// 4. FlatMap type mapping dataset ops: Recurses into the function definition.
+Status RecursivelyHandleOp(const NodeDef& node, int64 num_workers,
+                           FunctionLibraryDefinition* flib,
+                           MutableGraphView* graph) {
+  if (IsDatasetNodeOfType(node, kBatchDatasetOps)) {
+    return MutateBatchSize(node, num_workers, graph);
+  } else if (IsDatasetNodeOfType(node, kMultipleInputsDatasetOps)) {
+    // For all multiple input datasets, all inputs are datasets themselves.
+    for (int i = 0; i < node.input_size(); ++i) {
+      NodeDef* input_node = graph_utils::GetInputNode(node, *graph, i);
+      TF_RETURN_IF_ERROR(
+          RecursivelyHandleOp(*input_node, num_workers, flib, graph));
+    }
+  } else if (IsDatasetNodeOfType(node, kPassThroughOps)) {
+    // For all the dataset ops that are pass through, the input dataset is
+    // input 0.
+    NodeDef* input_node = graph_utils::GetInputNode(node, *graph, 0);
+    TF_RETURN_IF_ERROR(
+        RecursivelyHandleOp(*input_node, num_workers, flib, graph));
+  } else if (IsDatasetNodeOfType(node, kFuncDatasetOps)) {
+    const string func_name = node.attr().at("f").func().name();
+    const FunctionDef* fdef = flib->Find(func_name);
+    GrapplerFunctionItem f_item;
+    TF_RETURN_IF_ERROR(MakeGrapplerFunctionItem(
+        *fdef, *flib, graph->graph()->versions().producer(), &f_item));
+    GraphDef optimized_func_graph;
+    Status s = OptimizeGraph(f_item, num_workers, &optimized_func_graph);
+    if (s.ok()) {
+      // Function body optimization might have created new specialized
+      // functions for each instantiation context. Add them to the library.
+      for (const FunctionDef& func_def :
+           optimized_func_graph.library().function()) {
+        if (flib->Find(func_def.signature().name()) == nullptr) {
+          TF_RETURN_IF_ERROR(flib->AddFunctionDef(func_def));
+        }
+      }
+
+      // Convert optimized graph back to FunctionDef.
+      FunctionDef optimized_func;
+      f_item.SwapFunctionBody(std::move(optimized_func_graph));
+      TF_RETURN_IF_ERROR(MakeFunctionDef(f_item, *flib, &optimized_func));
+
+      // Replace optimized function with a new FunctionDef.
+      TF_RETURN_IF_ERROR(flib->ReplaceFunction(func_name, optimized_func));
+    }
+  } else if (IsDatasetNodeOfType(node, kSourceDatasetOps)) {
+    return errors::InvalidArgument(
+        "Reached a source dataset: ", node.op(),
+        " without encountering a batch transformation.");
+  } else {
+    return errors::InvalidArgument("Encountered an unsupported op: ",
+                                   node.op());
+  }
+  return Status::OK();
+}
+
+// Helper function that given a GrapplerItem generates a mutated graph def
+// with the batch size changed. The GrapplerItem could be generated from the
+// main graph or could be a function graph.
+Status OptimizeGraph(const GrapplerItem& item, int64 num_workers,
+                     GraphDef* output) {
+  *output = item.graph;
+  MutableGraphView graph(output);
+
+  FunctionLibraryDefinition flib(OpRegistry::Global(), item.graph.library());
+
+  NodeDef sink_node;
+  TF_RETURN_IF_ERROR(FindSinkNode(item.graph, &sink_node));
+  TF_RETURN_IF_ERROR(
+      RecursivelyHandleOp(sink_node, num_workers, &flib, &graph));
+  *output->mutable_library() = flib.ToProto();
+  return Status::OK();
+}
+
 }  // anonymous namespace
 
-Status RebatchOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
-                                  GraphDef* output) {
+Status RebatchOptimizer::OptimizeAndCollectStats(Cluster* cluster,
+                                                 const GrapplerItem& item,
+                                                 GraphDef* output,
+                                                 OptimizationStats* stats) {
   *output = item.graph;
   MutableGraphView graph(output);
 
-  absl::flat_hash_set<string> nodes_to_delete;
-  for (const NodeDef& node : item.graph.node()) {
-    if (node.op() == kBatchDatasetOp) {
-      NodeDef* batch_size_node = graph_utils::GetInputNode(node, graph, 1);
-      NodeDef tmp_node;
-      tmp_node = *batch_size_node;
-      graph_utils::SetUniqueGraphNodeName(tmp_node.op(), graph.graph(),
-                                          &tmp_node);
-      NodeDef* copy_batch_size_node = graph.AddNode(std::move(tmp_node));
-      NodeDef* float_copy_batch_size_node =
-          AddCastNode(copy_batch_size_node->name(), DT_INT64, DT_FLOAT, &graph);
-      NodeDef* num_worker_node =
-          graph_utils::AddScalarConstNode<int64>(num_workers_, &graph);
-      NodeDef* float_num_worker_node =
-          AddCastNode(num_worker_node->name(), DT_INT64, DT_FLOAT, &graph);
-      NodeDef* divided_batch_size_node =
-          AddFloatDivNode(float_copy_batch_size_node->name(),
-                          float_num_worker_node->name(), &graph);
-      NodeDef* cast_new_batch_size_node = AddCastNode(
-          divided_batch_size_node->name(), DT_FLOAT, DT_INT64, &graph);
-      TF_RETURN_IF_ERROR(graph.UpdateFanouts(batch_size_node->name(),
-                                             cast_new_batch_size_node->name()));
-      nodes_to_delete.insert(batch_size_node->name());
-      break;
-    }
-  }
-  TF_RETURN_IF_ERROR(graph.DeleteNodes(nodes_to_delete));
+  TF_RETURN_IF_ERROR(OptimizeGraph(item, num_workers_, output));
+  stats->num_changes++;
   return Status::OK();
 }
 
diff --git a/tensorflow/core/grappler/optimizers/data/rebatch.h b/tensorflow/core/grappler/optimizers/data/rebatch.h
index f7aa69f3ff..29a6100026 100644
--- a/tensorflow/core/grappler/optimizers/data/rebatch.h
+++ b/tensorflow/core/grappler/optimizers/data/rebatch.h
@@ -15,7 +15,9 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_REBATCH_H_
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_REBATCH_H_
 
-#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h"
+#include "tensorflow/core/framework/function.h"
+#include "tensorflow/core/grappler/mutable_graph_view.h"
+#include "tensorflow/core/grappler/optimizers/data/optimizer_base.h"
 
 namespace tensorflow {
 namespace grappler {
@@ -23,10 +25,7 @@ namespace grappler {
 // This optimizer changes the batch size of the output dataset by dividing the
 // current batch size by parameter `num_workers`. Currently, this works only
 // for very simple pipelines with a single BatchDatasetV2 transformation.
-//
-// TODO(rohanj): Extend this logic to correctly handle any input pipeline that
-// uses core tf.data APIs + MapAndBatch.
-class RebatchOptimizer : public CustomGraphOptimizer {
+class RebatchOptimizer : public TFDataOptimizerBase {
  public:
   RebatchOptimizer() = default;
   ~RebatchOptimizer() override = default;
@@ -36,8 +35,9 @@ class RebatchOptimizer : public CustomGraphOptimizer {
   Status Init(
       const tensorflow::RewriterConfig_CustomGraphOptimizer* config) override;
 
-  Status Optimize(Cluster* cluster, const GrapplerItem& item,
-                  GraphDef* output) override;
+  Status OptimizeAndCollectStats(Cluster* cluster, const GrapplerItem& item,
+                                 GraphDef* output,
+                                 OptimizationStats* stats) override;
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimize_output, double result) override;
diff --git a/tensorflow/core/kernels/data/experimental/rebatch_dataset_op.cc b/tensorflow/core/kernels/data/experimental/rebatch_dataset_op.cc
index a95773afd1..0397ca01c4 100644
--- a/tensorflow/core/kernels/data/experimental/rebatch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/rebatch_dataset_op.cc
@@ -35,9 +35,9 @@ class RebatchDatasetOp : public UnaryDatasetOpKernel {
                    DatasetBase** output) override {
     int64 num_workers;
     OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "num_workers", &num_workers));
-    OP_REQUIRES(ctx, num_workers > 0,
-                errors::InvalidArgument(
-                    "num_parallel_calls must be greater than zero."));
+    OP_REQUIRES(
+        ctx, num_workers > 0,
+        errors::InvalidArgument("num_workers must be greater than zero."));
 
     Dataset* dataset =
         new Dataset(ctx, input, num_workers, output_types_, output_shapes_);
@@ -62,8 +62,17 @@ class RebatchDatasetOp : public UnaryDatasetOpKernel {
     string DebugString() const override { return "RebatchDatasetOp::Dataset"; }
 
    private:
+    bool ShouldOptimizeFunctions() override {
+      // We only want to optimize functions for some particular datasets like
+      // FlatMapDataset, InterleaveDataset etc. So we disable generalized
+      // function optimization and explicitly handle function modifications
+      // for those datasets in the rewrite.
+      return false;
+    }
+
     RewriterConfig CreateGrapplerRewriteConfig() override {
       RewriterConfig rewriter_config;
+      rewriter_config.set_fail_on_optimizer_errors(true);
       rewriter_config.add_optimizers(kOptimizerName);
       rewriter_config.set_meta_optimizer_iterations(
           RewriterConfig_NumIterationsType_ONE);
diff --git a/tensorflow/core/kernels/data/graph_rewrite_dataset.cc b/tensorflow/core/kernels/data/graph_rewrite_dataset.cc
index 29deac9ad4..cd8026607e 100644
--- a/tensorflow/core/kernels/data/graph_rewrite_dataset.cc
+++ b/tensorflow/core/kernels/data/graph_rewrite_dataset.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 #include "tensorflow/core/kernels/data/graph_rewrite_dataset.h"
 
+#include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/kernels/data/dataset_utils.h"
 #include "tensorflow/core/protobuf/meta_graph.pb.h"
 #include "tensorflow/core/protobuf/rewriter_config.pb.h"
@@ -169,6 +170,8 @@ Status GraphRewriteDataset::ApplyOptimizations(OpKernelContext* ctx,
   std::unique_ptr<tensorflow::grappler::GrapplerItem> grappler_item =
       tensorflow::grappler::GrapplerItemFromMetaGraphDef(
           "graph", meta_graph_def, item_config);
+  grappler_item->optimization_options().optimize_function_library =
+      ShouldOptimizeFunctions();
   std::unordered_map<string, tensorflow::DeviceProperties> device_map;
   tensorflow::grappler::VirtualCluster cluster(device_map);
 
diff --git a/tensorflow/core/kernels/data/graph_rewrite_dataset.h b/tensorflow/core/kernels/data/graph_rewrite_dataset.h
index dedbdce71f..856fcd3ea7 100644
--- a/tensorflow/core/kernels/data/graph_rewrite_dataset.h
+++ b/tensorflow/core/kernels/data/graph_rewrite_dataset.h
@@ -73,6 +73,9 @@ class GraphRewriteDataset : public DatasetBase {
   // optimizations to be run by the Grappler Meta Optimizer.
   virtual RewriterConfig CreateGrapplerRewriteConfig() = 0;
 
+  // Option specifying whether we want to optimize the function library as well.
+  virtual bool ShouldOptimizeFunctions() { return true; }
+
   Status ApplyOptimizations(OpKernelContext* ctx, GraphDef* graph_def,
                             string* output_node);
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/BUILD b/tensorflow/python/data/experimental/kernel_tests/BUILD
index 0c3c1072cb..dfdcc54411 100644
--- a/tensorflow/python/data/experimental/kernel_tests/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/BUILD
@@ -481,6 +481,7 @@ py_test(
         "//tensorflow/python/data/kernel_tests:test_base",
         "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python/data/util:nest",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/rebatch_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/rebatch_dataset_test.py
index 486099bbb1..9c1392bd01 100644
--- a/tensorflow/python/data/experimental/kernel_tests/rebatch_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/rebatch_dataset_test.py
@@ -17,61 +17,302 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from absl.testing import parameterized
+
 from tensorflow.python.data.experimental.ops import batching
+from tensorflow.python.data.experimental.ops import scan_ops
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import nest
+from tensorflow.python.framework import errors
 from tensorflow.python.framework import test_util
+from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
 
+@parameterized.named_parameters(("WithDropRemainder", True),
+                                ("WithoutDropRemainder", False))
 @test_util.run_all_in_graph_and_eager_modes
 class RebatchDatasetTest(test_base.DatasetTestBase):
 
-  def testBasic(self):
-    dataset = dataset_ops.Dataset.range(1024).batch(32, drop_remainder=True)
+  def testBasic(self, drop_remainder):
+    dataset = dataset_ops.Dataset.range(1024).batch(
+        32, drop_remainder=drop_remainder)
     rebatched_dataset = batching._RebatchDataset(dataset, num_workers=4)
     self.assertEqual(
-        [[32]], [ts.as_list() for ts in nest.flatten(dataset.output_shapes)])
+        [[32 if drop_remainder else None]],
+        [ts.as_list() for ts in nest.flatten(dataset.output_shapes)])
     self.assertEqual(
-        [[8]],
+        [[8 if drop_remainder else None]],
         [ts.as_list() for ts in nest.flatten(rebatched_dataset.output_shapes)])
 
     expected_output = [[k for k in range(i, i + 8)] for i in range(0, 1024, 8)]  # pylint: disable=g-complex-comprehension
     self.assertDatasetProduces(rebatched_dataset, expected_output)
 
-  def testScalarInputError(self):
+  def testScalarInputError(self, _):
     dataset = dataset_ops.Dataset.range(1024)
     with self.assertRaisesRegexp(ValueError, "at least one dimension"):
       batching._RebatchDataset(dataset, num_workers=4)
 
-  def testUnknownBatchSizeError(self):
-    dataset = dataset_ops.Dataset.range(1024).batch(32)
-    with self.assertRaisesRegexp(ValueError, "unknown batch size datasets"):
-      batching._RebatchDataset(dataset, num_workers=4)
-
-  def testNotDivisibleError(self):
+  def testNotDivisibleError(self, drop_remainder):
+    # TODO(rohanj): This should fail even with drop_remainder=False, by adding
+    # an assertion in the mutated graph.
     dataset = dataset_ops.Dataset.range(1024).batch(32, drop_remainder=True)
     with self.assertRaisesRegexp(ValueError, "not divisible by"):
       batching._RebatchDataset(dataset, num_workers=5)
 
-  def testTupleOutput(self):
-    dataset = (dataset_ops.Dataset.range(1024).map(lambda x: (x, x))
-               .batch(32, drop_remainder=True))
+  def testTupleOutput(self, drop_remainder):
+    dataset = (
+        dataset_ops.Dataset.range(1024).map(lambda x: (x, x)).batch(
+            32, drop_remainder=drop_remainder))
     rebatched_dataset = batching._RebatchDataset(dataset, num_workers=4)
     expected_output = [([k for k in range(i, i + 8)],  # pylint: disable=g-complex-comprehension
                         [k for k in range(i, i + 8)])
                        for i in range(0, 1024, 8)]
     self.assertDatasetProduces(rebatched_dataset, expected_output)
 
-  def testNestedDictionaryOutput(self):
+  def testNestedDictionaryOutput(self, drop_remainder):
     dataset = dataset_ops.Dataset.range(1024).map(
-        lambda x: {"a": x, "b": {"c": x}}).batch(32, drop_remainder=True)
+        lambda x: {"a": x, "b": {"c": x}}).batch(
+            32, drop_remainder=drop_remainder)
     rebatched_dataset = batching._RebatchDataset(dataset, num_workers=4)
     expected_output = [{"a": [k for k in range(i, i + 8)],  # pylint: disable=g-complex-comprehension
                         "b": {"c": [k for k in range(i, i + 8)]}}
                        for i in range(0, 1024, 8)]
     self.assertDatasetProduces(rebatched_dataset, expected_output)
 
+  def testFinalPartialBatchOriginal(self, drop_remainder):
+    dataset = dataset_ops.Dataset.range(1032).batch(
+        32, drop_remainder=drop_remainder)
+    rebatched_dataset = batching._RebatchDataset(dataset, num_workers=4)
+    self.assertEqual(
+        [[32 if drop_remainder else None]],
+        [ts.as_list() for ts in nest.flatten(dataset.output_shapes)])
+    self.assertEqual(
+        [[8 if drop_remainder else None]],
+        [ts.as_list() for ts in nest.flatten(rebatched_dataset.output_shapes)])
+
+    expected_output = [[k for k in range(i, i + 8)] for i in range(0, 1032, 8)]  # pylint: disable=g-complex-comprehension
+    self.assertDatasetProduces(rebatched_dataset, expected_output)
+
+  def testFinalPartialBatchAfterRebatch(self, drop_remainder):
+    dataset = dataset_ops.Dataset.range(34).batch(
+        32, drop_remainder=drop_remainder)
+    rebatched_dataset = batching._RebatchDataset(dataset, num_workers=4)
+    self.assertEqual(
+        [[32 if drop_remainder else None]],
+        [ts.as_list() for ts in nest.flatten(dataset.output_shapes)])
+    self.assertEqual(
+        [[8 if drop_remainder else None]],
+        [ts.as_list() for ts in nest.flatten(rebatched_dataset.output_shapes)])
+
+    expected_output = [[k for k in range(i, i + 8)] for i in range(0, 32, 8)]  # pylint: disable=g-complex-comprehension
+    if not drop_remainder:
+      expected_output += [[32, 33]]
+    self.assertDatasetProduces(rebatched_dataset, expected_output)
+
+  def testMultipleBatches(self, drop_remainder):
+    dataset = dataset_ops.Dataset.range(128).batch(
+        4, drop_remainder=drop_remainder)
+    dataset = dataset.batch(8, drop_remainder=drop_remainder)
+    self.assertEqual(
+        [[8, 4]] if drop_remainder else [[None, None]],
+        [ts.as_list() for ts in nest.flatten(dataset.output_shapes)])
+    # Each element is a list of 8 elements where each element is a list of 4.
+    expected_output = [[[j, j + 1, j + 2, j + 3]  # pylint: disable=g-complex-comprehension
+                        for j in range(i, i + 32, 4)]  # generates 8 elements
+                       for i in range(0, 128, 32)]
+    self.assertDatasetProduces(dataset, expected_output)
+
+    rebatched_dataset = batching._RebatchDataset(dataset, 4)
+    self.assertEqual(
+        [[2, 4]] if drop_remainder else [[None, None]],
+        [ts.as_list() for ts in nest.flatten(rebatched_dataset.output_shapes)])
+    # Each element is a list of 2 elements where each element is a list of 4.
+    expected_output = [[[j, j + 1, j + 2, j + 3]  # pylint: disable=g-complex-comprehension
+                        for j in range(i, i + 8, 4)]  # generates 2 elements
+                       for i in range(0, 128, 8)]
+    self.assertDatasetProduces(rebatched_dataset, expected_output)
+
+  def testMapAndBatch(self, drop_remainder):
+    dataset = dataset_ops.Dataset.range(1024).apply(
+        batching.map_and_batch(
+            math_ops.square, 32, drop_remainder=drop_remainder))
+    rebatched_dataset = batching._RebatchDataset(dataset, num_workers=4)
+    self.assertEqual(
+        [[32 if drop_remainder else None]],
+        [ts.as_list() for ts in nest.flatten(dataset.output_shapes)])
+    self.assertEqual(
+        [[8 if drop_remainder else None]],
+        [ts.as_list() for ts in nest.flatten(rebatched_dataset.output_shapes)])
+    expected_output = [[k**2 for k in range(i, i + 8)]  # pylint: disable=g-complex-comprehension
+                       for i in range(0, 1024, 8)]
+    self.assertDatasetProduces(rebatched_dataset, expected_output)
+
+  def testPaddedBatch(self, drop_remainder):
+    dataset = dataset_ops.Dataset.range(128).batch(4).padded_batch(
+        8, padded_shapes=[5], drop_remainder=drop_remainder)
+    rebatched_dataset = batching._RebatchDataset(dataset, num_workers=4)
+    self.assertEqual(
+        [[8, 5]] if drop_remainder else [[None, 5]],
+        [ts.as_list() for ts in nest.flatten(dataset.output_shapes)])
+    # Each element is a list of 8 elements in which each element is a list of 5
+    # elements, first four are numbers and the last one is a padded zero.
+    expected_output = [[[j, j + 1, j + 2, j + 3, 0]  # pylint: disable=g-complex-comprehension
+                        for j in range(i, i + 32, 4)]  # generates 8 elements
+                       for i in range(0, 128, 32)]
+    self.assertDatasetProduces(dataset, expected_output)
+    self.assertEqual(
+        [[2, 5]] if drop_remainder else [[None, 5]],
+        [ts.as_list() for ts in nest.flatten(rebatched_dataset.output_shapes)])
+    # Each element is a list of 2 elements in which each element is a list of 5
+    # elements, first four are numbers and the last one is a padded zero.
+    expected_output = [[[j, j + 1, j + 2, j + 3, 0]  # pylint: disable=g-complex-comprehension
+                        for j in range(i, i + 8, 4)]  # generates 2 elements
+                       for i in range(0, 128, 8)]
+    self.assertDatasetProduces(rebatched_dataset, expected_output)
+
+  def testConcatenate(self, drop_remainder):
+    dataset1 = dataset_ops.Dataset.range(64).batch(
+        8, drop_remainder=drop_remainder)
+    dataset2 = dataset_ops.Dataset.range(32).batch(
+        8, drop_remainder=drop_remainder)
+    dataset = dataset1.concatenate(dataset2)
+    rebatched_dataset = batching._RebatchDataset(dataset, num_workers=4)
+    self.assertEqual(
+        [[8 if drop_remainder else None]],
+        [ts.as_list() for ts in nest.flatten(dataset.output_shapes)])
+    self.assertEqual(
+        [[2 if drop_remainder else None]],
+        [ts.as_list() for ts in nest.flatten(rebatched_dataset.output_shapes)])
+    expected_output = ([[i, i + 1] for i in range(0, 64, 2)] +
+                       [[i, i + 1] for i in range(0, 32, 2)])
+    self.assertDatasetProduces(rebatched_dataset, expected_output)
+
+  def testConcatenateDifferentShapes(self, drop_remainder):
+    dataset1 = dataset_ops.Dataset.range(64).batch(
+        16, drop_remainder=drop_remainder)
+    dataset2 = dataset_ops.Dataset.range(32).batch(
+        8, drop_remainder=drop_remainder)
+    dataset = dataset1.concatenate(dataset2)
+    rebatched_dataset = batching._RebatchDataset(dataset, num_workers=4)
+    self.assertEqual(
+        [[None]], [ts.as_list() for ts in nest.flatten(dataset.output_shapes)])
+    self.assertEqual(
+        [[None]],
+        [ts.as_list() for ts in nest.flatten(rebatched_dataset.output_shapes)])
+    expected_output = ([[i, i + 1, i + 2, i + 3] for i in range(0, 64, 4)] +
+                       [[i, i + 1] for i in range(0, 32, 2)])
+    self.assertDatasetProduces(rebatched_dataset, expected_output)
+
+  def testZip(self, drop_remainder):
+    dataset1 = dataset_ops.Dataset.range(64).batch(
+        8, drop_remainder=drop_remainder)
+    dataset2 = dataset_ops.Dataset.range(32).batch(
+        8, drop_remainder=drop_remainder)
+    dataset = dataset_ops.Dataset.zip((dataset1, dataset2))
+    rebatched_dataset = batching._RebatchDataset(dataset, num_workers=4)
+    self.assertEqual(
+        [[8], [8]] if drop_remainder else [[None], [None]],
+        [ts.as_list() for ts in nest.flatten(dataset.output_shapes)])
+    self.assertEqual(
+        [[2], [2]] if drop_remainder else [[None], [None]],
+        [ts.as_list() for ts in nest.flatten(rebatched_dataset.output_shapes)])
+    expected_output = [([i, i + 1], [i, i + 1]) for i in range(0, 32, 2)]
+    self.assertDatasetProduces(rebatched_dataset, expected_output)
+
+  def testZipDifferentShapes(self, drop_remainder):
+    dataset1 = dataset_ops.Dataset.range(64).batch(
+        16, drop_remainder=drop_remainder)
+    dataset2 = dataset_ops.Dataset.range(32).batch(
+        8, drop_remainder=drop_remainder)
+    dataset = dataset_ops.Dataset.zip((dataset1, dataset2))
+    rebatched_dataset = batching._RebatchDataset(dataset, num_workers=4)
+    self.assertEqual(
+        [[16], [8]] if drop_remainder else [[None], [None]],
+        [ts.as_list() for ts in nest.flatten(dataset.output_shapes)])
+    self.assertEqual(
+        [[4], [2]] if drop_remainder else [[None], [None]],
+        [ts.as_list() for ts in nest.flatten(rebatched_dataset.output_shapes)])
+    expected_output = [([2 * i, 2 * i + 1, 2 * i + 2, 2 * i + 3], [i, i + 1])
+                       for i in range(0, 32, 2)]
+    self.assertDatasetProduces(rebatched_dataset, expected_output)
+
+  def testUnsupportedTransformError(self, drop_remainder):
+    dataset = dataset_ops.Dataset.range(1024).batch(
+        32, drop_remainder=drop_remainder).apply(
+            scan_ops.scan([0], lambda _, a: ([0], a)))
+    with self.assertRaises(errors.InvalidArgumentError):
+      rebatched_dataset = batching._RebatchDataset(dataset, num_workers=4)
+      next_element = self.getNext(rebatched_dataset)
+      self.evaluate(next_element())
+
+  def testFlatMapBatching(self, drop_remainder):
+    dataset = dataset_ops.Dataset.range(
+        2).flat_map(lambda _: dataset_ops.Dataset.range(32).batch(  # pylint: disable=g-long-lambda
+            32, drop_remainder=drop_remainder))
+    self.assertEqual(
+        [[32 if drop_remainder else None]],
+        [ts.as_list() for ts in nest.flatten(dataset.output_shapes)])
+    # Two elements where each element is range(32)
+    expected_output = [[k for k in range(32)] for _ in range(2)]  # pylint: disable=g-complex-comprehension
+    self.assertDatasetProduces(dataset, expected_output)
+
+    rebatched_dataset = batching._RebatchDataset(dataset, num_workers=4)
+    self.assertEqual(
+        [[8 if drop_remainder else None]],
+        [ts.as_list() for ts in nest.flatten(rebatched_dataset.output_shapes)])
+    # Two elements where each element is a list of 4 elements where each element
+    # is a list of 8.
+    expected_output = [[k for k in range(i, i + 8)]  # pylint: disable=g-complex-comprehension
+                       for _ in range(2)
+                       for i in range(0, 32, 8)]  # generates 4 elements
+    self.assertDatasetProduces(rebatched_dataset, expected_output)
+
+  def testInterleaveBatching(self, drop_remainder):
+    dataset = dataset_ops.Dataset.range(
+        2).interleave(lambda _: dataset_ops.Dataset.range(32).batch(  # pylint: disable=g-long-lambda
+            32, drop_remainder=drop_remainder), cycle_length=2)
+    self.assertEqual(
+        [[32 if drop_remainder else None]],
+        [ts.as_list() for ts in nest.flatten(dataset.output_shapes)])
+    # Two elements where each element is range(32)
+    expected_output = [[k for k in range(32)] for _ in range(2)]  # pylint: disable=g-complex-comprehension
+    self.assertDatasetProduces(dataset, expected_output)
+
+    rebatched_dataset = batching._RebatchDataset(dataset, num_workers=4)
+    self.assertEqual(
+        [[8 if drop_remainder else None]],
+        [ts.as_list() for ts in nest.flatten(rebatched_dataset.output_shapes)])
+    # List of 4 elements where each element is a list of 8 numbering from 0 to
+    # 31 repeated twice.
+    expected_output = [[k for k in range(i, i + 8)]  # pylint: disable=g-complex-comprehension
+                       for i in range(0, 32, 8)  # generates 4 elements
+                       for _ in range(2)]
+    self.assertDatasetProduces(rebatched_dataset, expected_output)
+
+  def testParallelInterleaveBatching(self, drop_remainder):
+    dataset = dataset_ops.Dataset.range(
+        2).interleave(lambda _: dataset_ops.Dataset.range(32).batch(  # pylint: disable=g-long-lambda
+            32, drop_remainder=drop_remainder), cycle_length=2,
+                      num_parallel_calls=2)
+    self.assertEqual(
+        [[32 if drop_remainder else None]],
+        [ts.as_list() for ts in nest.flatten(dataset.output_shapes)])
+    # Two elements where each element is range(32)
+    expected_output = [[k for k in range(32)] for _ in range(2)]  # pylint: disable=g-complex-comprehension
+    self.assertDatasetProduces(dataset, expected_output)
+
+    rebatched_dataset = batching._RebatchDataset(dataset, num_workers=4)
+    self.assertEqual(
+        [[8 if drop_remainder else None]],
+        [ts.as_list() for ts in nest.flatten(rebatched_dataset.output_shapes)])
+    # List of 4 elements where each element is a list of 8 numbering from 0 to
+    # 31 repeated twice in collated fashion i.e [0...8], [0...8] etc.
+    expected_output = [[k for k in range(i, i + 8)]  # pylint: disable=g-complex-comprehension
+                       for i in range(0, 32, 8)  # generates 4 elements
+                       for _ in range(2)]
+    self.assertDatasetProduces(rebatched_dataset, expected_output)
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/data/experimental/ops/batching.py b/tensorflow/python/data/experimental/ops/batching.py
index 5e28e0c001..a55f576166 100644
--- a/tensorflow/python/data/experimental/ops/batching.py
+++ b/tensorflow/python/data/experimental/ops/batching.py
@@ -717,9 +717,8 @@ class _RebatchDataset(dataset_ops.UnaryDataset):
       """Recalculates the output_shapes after dividing it by num_workers."""
       if len(output_shapes) < 1:
         raise ValueError("Input shape should have at least one dimension.")
-      if not output_shapes.dims[0].value:
-        raise ValueError("Cannot rebatch unknown batch size datasets.")
-      if output_shapes.dims[0].value % num_workers != 0:
+      if (output_shapes.dims[0].value and
+          output_shapes.dims[0].value % num_workers != 0):
         raise ValueError(
             "First dim of input shape: %d is not divisible by num_workers: %d" %
             (output_shapes[0], num_workers))
-- 
GitLab


From 8039f4096679a0ffbdaefead339ab278ef6a250b Mon Sep 17 00:00:00 2001
From: Alexey Radul <axch@google.com>
Date: Mon, 11 Feb 2019 14:27:10 -0800
Subject: [PATCH 2280/2345] Add a unit test that custom loss functions can
 accept non-Tensor objects emitted from models.

PiperOrigin-RevId: 233479194
---
 tensorflow/python/keras/utils/tf_utils_test.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/tensorflow/python/keras/utils/tf_utils_test.py b/tensorflow/python/keras/utils/tf_utils_test.py
index 9c478af4ec..902ecf9167 100644
--- a/tensorflow/python/keras/utils/tf_utils_test.py
+++ b/tensorflow/python/keras/utils/tf_utils_test.py
@@ -89,6 +89,10 @@ class TestIsSymbolicTensor(test.TestCase):
         self._input = input_
         self.value = ops.convert_to_tensor(42.)
 
+      @property
+      def dtype(self):
+        return self.value.dtype
+
     ops.register_tensor_conversion_function(
         Foo, lambda x, *args, **kwargs: x.value)
     tf_utils.register_symbolic_tensor_type(Foo)
@@ -128,6 +132,16 @@ class TestIsSymbolicTensor(test.TestCase):
     # `Tensor`.
     y = model(ops.convert_to_tensor(7.))
     self.assertIsInstance(y, Foo)
+    # Confirm that (custom) loss sees `Foo` instance, not Tensor.
+    obtained_prediction_box = [None]
+    def custom_loss(y_obs, y_pred):
+      del y_obs
+      obtained_prediction_box[0] = y_pred
+      return y_pred
+    # Apparently `compile` calls the loss function enough to trigger the
+    # side-effect.
+    model.compile('SGD', loss=custom_loss)
+    self.assertIsInstance(obtained_prediction_box[0], Foo)
 
 
 class ConvertInnerNodeDataTest(test.TestCase):
-- 
GitLab


From d584984e82ceb20a34d344f1df40445f5469541a Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Mon, 11 Feb 2019 14:42:30 -0800
Subject: [PATCH 2281/2345] Fix verbosity in interactive prompts.

PiperOrigin-RevId: 233482282
---
 tensorflow/python/autograph/pyct/errors.py      | 2 +-
 tensorflow/python/autograph/utils/ag_logging.py | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/autograph/pyct/errors.py b/tensorflow/python/autograph/pyct/errors.py
index 973c60dcde..2f2049c40b 100644
--- a/tensorflow/python/autograph/pyct/errors.py
+++ b/tensorflow/python/autograph/pyct/errors.py
@@ -45,4 +45,4 @@ def report_internal_error(entity, exception):
       'Unexpected error transforming %s. If you believe this is due to a bug,'
       ' please set the verbosity to 10 (on Linux, `export '
       'AUTOGRAPH_VERBOSITY=10`) and attach the full output when filing the bug '
-      'report. Caused by %s' % (entity, exception))
+      'report. Caused by: %s' % (entity, exception))
diff --git a/tensorflow/python/autograph/utils/ag_logging.py b/tensorflow/python/autograph/utils/ag_logging.py
index 037b0615f6..8229b828d3 100644
--- a/tensorflow/python/autograph/utils/ag_logging.py
+++ b/tensorflow/python/autograph/utils/ag_logging.py
@@ -136,6 +136,8 @@ def log(level, msg, *args, **kwargs):
 
 def warn(msg, *args, **kwargs):
   logging.warn(msg, *args, **kwargs)
+  if echo_log_to_stdout:
+    print('WARNING:', msg % args)
 
 
 def warn_first_n(msg, *args, **kwargs):
-- 
GitLab


From f5463b160cda9c69fcb9383377374971f5662bd0 Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Mon, 11 Feb 2019 14:47:28 -0800
Subject: [PATCH 2282/2345] [XLA] Move a CHECK into Layout verification

Sparse layouts that specify tiling should fail verification, not CHECK-fail in ToString.

PiperOrigin-RevId: 233483254
---
 tensorflow/compiler/xla/layout_util.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/compiler/xla/layout_util.cc b/tensorflow/compiler/xla/layout_util.cc
index 5d85fd7942..62314118ca 100644
--- a/tensorflow/compiler/xla/layout_util.cc
+++ b/tensorflow/compiler/xla/layout_util.cc
@@ -247,6 +247,10 @@ Layout CreateDefaultLayoutForRank(int64 rank) {
       }
       dimensions_in_layout[dim] = true;
     }
+  } else {
+    if (layout.tiles_size() != 0) {
+      return InvalidArgument("Only dense layouts can be tiled.");
+    }
   }
 
   return Status::OK();
-- 
GitLab


From c1043a02f9befd9d5aed5249787ddf9d3bc50f1b Mon Sep 17 00:00:00 2001
From: Katherine Wu <kathywu@google.com>
Date: Mon, 11 Feb 2019 14:51:51 -0800
Subject: [PATCH 2283/2345] Separate Keras and Estimator ModeKeys into
 different classes to avoid headache with updating Estimator ModeKeys.

Detailed list of changes:
* Revert Estimator Modekeys back V1 ModeKeys.
* Added helper functions in SavedModel model utils to deal with different Estimator and Keras Mode Keys.
* Estimator and Keras ModeKeys now reside in saved_model/model_utils/mode_keys.

PiperOrigin-RevId: 233484078
---
 .../python/keras_backward_compat_test.py      |   2 +-
 tensorflow/python/BUILD                       |  24 ----
 tensorflow/python/keras/BUILD                 |  15 ++-
 tensorflow/python/keras/callbacks.py          |   2 +-
 .../engine/distributed_training_utils.py      |   2 +-
 tensorflow/python/keras/engine/training.py    |   2 +-
 .../python/keras/engine/training_arrays.py    |   2 +-
 .../keras/engine/training_distributed.py      |   2 +-
 .../python/keras/engine/training_generator.py |   2 +-
 tensorflow/python/keras/saving/saved_model.py |   2 +-
 .../python/keras/saving/saved_model_test.py   |   2 +-
 .../{training => keras/utils}/mode_keys.py    |  19 +--
 .../python/saved_model/model_utils/BUILD      |  19 ++-
 .../saved_model/model_utils/export_utils.py   |  27 ++--
 .../saved_model/model_utils/mode_keys.py      | 124 ++++++++++++++++++
 .../saved_model/model_utils/mode_keys_test.py |  65 +++++++++
 tensorflow/python/training/mode_keys_test.py  |  29 ----
 17 files changed, 246 insertions(+), 94 deletions(-)
 rename tensorflow/python/{training => keras/utils}/mode_keys.py (70%)
 create mode 100644 tensorflow/python/saved_model/model_utils/mode_keys.py
 create mode 100644 tensorflow/python/saved_model/model_utils/mode_keys_test.py
 delete mode 100644 tensorflow/python/training/mode_keys_test.py

diff --git a/tensorflow/contrib/distribute/python/keras_backward_compat_test.py b/tensorflow/contrib/distribute/python/keras_backward_compat_test.py
index 9a581e7141..c49b5522f9 100644
--- a/tensorflow/contrib/distribute/python/keras_backward_compat_test.py
+++ b/tensorflow/contrib/distribute/python/keras_backward_compat_test.py
@@ -31,10 +31,10 @@ from tensorflow.python.framework import random_seed
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.keras.engine import distributed_training_utils
 from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_descent_keras
+from tensorflow.python.keras.utils.mode_keys import ModeKeys
 from tensorflow.python.ops.parsing_ops import gen_parsing_ops
 from tensorflow.python.training import gradient_descent
 from tensorflow.python.training import rmsprop
-from tensorflow.python.training.mode_keys import ModeKeys
 
 _RANDOM_SEED = 1337
 _TRAIN_SIZE = 200
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index a5b3daa674..36ec9e01e2 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -143,7 +143,6 @@ py_library(
         ":map_fn",
         ":math_ops",
         ":metrics",
-        ":mode_keys",
         ":nccl_ops",
         ":nn",
         ":ops",
@@ -6076,29 +6075,6 @@ py_binary(
     ],
 )
 
-py_library(
-    name = "mode_keys",
-    srcs = [
-        "training/mode_keys.py",
-    ],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":util",
-    ],
-)
-
-tf_py_test(
-    name = "mode_keys_test",
-    size = "small",
-    srcs = [
-        "training/mode_keys_test.py",
-    ],
-    additional_deps = [
-        ":client_testlib",
-        ":mode_keys",
-    ],
-)
-
 pyx_library(
     name = "framework_fast_tensor_util",
     srcs = ["framework/fast_tensor_util.pyx"],
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 606ff972f6..4f710ff658 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -161,6 +161,7 @@ py_library(
         ":engine_utils",
         ":initializers",
         ":losses",
+        ":mode_keys",
         ":optimizers",
         ":regularizers",
         ":saving",
@@ -188,9 +189,9 @@ py_library(
     deps = [
         ":backend",
         ":engine_utils",
+        ":mode_keys",
         ":optimizers",
         "//tensorflow/python:lib",
-        "//tensorflow/python:mode_keys",
         "//tensorflow/python:saver",
         "//tensorflow/python/saved_model",
         "//tensorflow/python/saved_model/model_utils",
@@ -218,6 +219,7 @@ py_library(
     deps = [
         ":backend",
         ":engine_utils",
+        ":mode_keys",
     ],
 )
 
@@ -369,6 +371,17 @@ py_library(
     ],
 )
 
+py_library(
+    name = "mode_keys",
+    srcs = [
+        "utils/mode_keys.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python/saved_model/model_utils:mode_keys",
+    ],
+)
+
 tf_py_test(
     name = "integration_test",
     size = "medium",
diff --git a/tensorflow/python/keras/callbacks.py b/tensorflow/python/keras/callbacks.py
index 1c2ed81367..926a555f62 100644
--- a/tensorflow/python/keras/callbacks.py
+++ b/tensorflow/python/keras/callbacks.py
@@ -36,10 +36,10 @@ from tensorflow.python.framework import ops
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras.utils.data_utils import Sequence
 from tensorflow.python.keras.utils.generic_utils import Progbar
+from tensorflow.python.keras.utils.mode_keys import ModeKeys
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import summary_ops_v2
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.training.mode_keys import ModeKeys
 from tensorflow.python.util.tf_export import keras_export
 
 try:
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index 700b956da4..7702cfabf9 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -33,11 +33,11 @@ from tensorflow.python.keras import callbacks
 from tensorflow.python.keras import metrics as metrics_module
 from tensorflow.python.keras import optimizers
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
+from tensorflow.python.keras.utils.mode_keys import ModeKeys
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.training.mode_keys import ModeKeys
 from tensorflow.python.util import nest
 from tensorflow.python.util import tf_contextlib
 
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index a7e5750361..34e2e30a0b 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -46,11 +46,11 @@ from tensorflow.python.keras.saving import saving_utils
 from tensorflow.python.keras.utils import data_utils
 from tensorflow.python.keras.utils import losses_utils
 from tensorflow.python.keras.utils.generic_utils import slice_arrays
+from tensorflow.python.keras.utils.mode_keys import ModeKeys
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import optimizer as tf_optimizer_module
 from tensorflow.python.training.checkpointable import base as checkpointable
-from tensorflow.python.training.mode_keys import ModeKeys
 from tensorflow.python.util import nest
 from tensorflow.python.util.tf_export import keras_export
 
diff --git a/tensorflow/python/keras/engine/training_arrays.py b/tensorflow/python/keras/engine/training_arrays.py
index 68b8f61521..980f241a2f 100644
--- a/tensorflow/python/keras/engine/training_arrays.py
+++ b/tensorflow/python/keras/engine/training_arrays.py
@@ -33,8 +33,8 @@ from tensorflow.python.keras.engine import distributed_training_utils
 from tensorflow.python.keras.engine import training_utils
 from tensorflow.python.keras.utils.generic_utils import make_batches
 from tensorflow.python.keras.utils.generic_utils import slice_arrays
+from tensorflow.python.keras.utils.mode_keys import ModeKeys
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.training.mode_keys import ModeKeys
 
 try:
   from scipy.sparse import issparse  # pylint: disable=g-import-not-at-top
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 2d79afef76..92a12f0dc1 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -34,9 +34,9 @@ from tensorflow.python.keras.engine import partial_batch_padding_handler as padd
 from tensorflow.python.keras.engine import training_arrays
 from tensorflow.python.keras.engine import training_utils
 from tensorflow.python.keras.utils.generic_utils import Progbar
+from tensorflow.python.keras.utils.mode_keys import ModeKeys
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.training.mode_keys import ModeKeys
 from tensorflow.python.util import nest
 
 
diff --git a/tensorflow/python/keras/engine/training_generator.py b/tensorflow/python/keras/engine/training_generator.py
index 7c63161024..0172b47e14 100644
--- a/tensorflow/python/keras/engine/training_generator.py
+++ b/tensorflow/python/keras/engine/training_generator.py
@@ -33,8 +33,8 @@ from tensorflow.python.keras import callbacks as cbks
 from tensorflow.python.keras.engine import training_utils
 from tensorflow.python.keras.utils import data_utils
 from tensorflow.python.keras.utils import generic_utils
+from tensorflow.python.keras.utils.mode_keys import ModeKeys
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.training.mode_keys import ModeKeys
 from tensorflow.python.util import nest
 
 
diff --git a/tensorflow/python/keras/saving/saved_model.py b/tensorflow/python/keras/saving/saved_model.py
index a61435971e..02803c4d45 100644
--- a/tensorflow/python/keras/saving/saved_model.py
+++ b/tensorflow/python/keras/saving/saved_model.py
@@ -27,6 +27,7 @@ from tensorflow.python.keras import backend as K
 from tensorflow.python.keras import optimizers
 from tensorflow.python.keras.saving import model_from_json
 from tensorflow.python.keras.saving import saving_utils
+from tensorflow.python.keras.utils import mode_keys
 from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import tf_logging as logging
@@ -35,7 +36,6 @@ from tensorflow.python.saved_model import constants
 from tensorflow.python.saved_model import model_utils
 from tensorflow.python.saved_model import save as save_lib
 from tensorflow.python.saved_model import utils_impl as saved_model_utils
-from tensorflow.python.training import mode_keys
 from tensorflow.python.training import saver as saver_lib
 from tensorflow.python.training.checkpointable import graph_view
 from tensorflow.python.util import compat
diff --git a/tensorflow/python/keras/saving/saved_model_test.py b/tensorflow/python/keras/saving/saved_model_test.py
index 6ecb4ed6ea..ab0b4e4b88 100644
--- a/tensorflow/python/keras/saving/saved_model_test.py
+++ b/tensorflow/python/keras/saving/saved_model_test.py
@@ -33,12 +33,12 @@ from tensorflow.python.framework import tensor_spec
 from tensorflow.python.framework import test_util
 from tensorflow.python.keras.engine import training
 from tensorflow.python.keras.saving import saved_model as keras_saved_model
+from tensorflow.python.keras.utils import mode_keys
 from tensorflow.python.keras.utils import tf_utils
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 from tensorflow.python.saved_model import loader_impl
 from tensorflow.python.saved_model import model_utils
-from tensorflow.python.training import mode_keys
 from tensorflow.python.training import training as training_module
 
 
diff --git a/tensorflow/python/training/mode_keys.py b/tensorflow/python/keras/utils/mode_keys.py
similarity index 70%
rename from tensorflow/python/training/mode_keys.py
rename to tensorflow/python/keras/utils/mode_keys.py
index ef64554bd5..fb6fc3eef7 100644
--- a/tensorflow/python/training/mode_keys.py
+++ b/tensorflow/python/keras/utils/mode_keys.py
@@ -12,22 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Model modeKeys for TensorFlow and Estimator."""
+"""Keras model mode constants."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-
-class ModeKeys(object):
-  """Standard names for model modes.
-
-  The following standard keys are defined:
-
-  * `TRAIN`: training/fitting mode.
-  * `TEST`: testing/evaluation mode.
-  * `PREDICT`: prediction/inference mode.
-  """
-
-  TRAIN = 'train'
-  TEST = 'test'
-  PREDICT = 'predict'
+# pylint: disable=unused-import
+from tensorflow.python.saved_model.model_utils.mode_keys import KerasModeKeys as ModeKeys
+# pylint: enable=unused-import
diff --git a/tensorflow/python/saved_model/model_utils/BUILD b/tensorflow/python/saved_model/model_utils/BUILD
index 192a610fd2..493574a225 100644
--- a/tensorflow/python/saved_model/model_utils/BUILD
+++ b/tensorflow/python/saved_model/model_utils/BUILD
@@ -30,6 +30,7 @@ py_library(
     deps = [
         ":export_output",
         ":export_utils",
+        ":mode_keys",
     ],
 )
 
@@ -70,7 +71,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":export_output",
-        "//tensorflow/python:mode_keys",
+        ":mode_keys",
         "//tensorflow/python:platform",
         "//tensorflow/python:util",
         "//tensorflow/python/saved_model:signature_constants",
@@ -98,3 +99,19 @@ py_test(
         "//tensorflow/python/saved_model:signature_def_utils",
     ],
 )
+
+py_library(
+    name = "mode_keys",
+    srcs = ["mode_keys.py"],
+    srcs_version = "PY2AND3",
+)
+
+py_test(
+    name = "mode_keys_test",
+    srcs = ["mode_keys_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":mode_keys",
+        "//tensorflow/python:client_testlib",
+    ],
+)
diff --git a/tensorflow/python/saved_model/model_utils/export_utils.py b/tensorflow/python/saved_model/model_utils/export_utils.py
index 343a3362c4..431b6dc345 100644
--- a/tensorflow/python/saved_model/model_utils/export_utils.py
+++ b/tensorflow/python/saved_model/model_utils/export_utils.py
@@ -30,29 +30,26 @@ from tensorflow.python.saved_model import signature_constants
 from tensorflow.python.saved_model import signature_def_utils
 from tensorflow.python.saved_model import tag_constants
 from tensorflow.python.saved_model.model_utils import export_output as export_output_lib
-from tensorflow.python.training import mode_keys
+from tensorflow.python.saved_model.model_utils import mode_keys
+from tensorflow.python.saved_model.model_utils.mode_keys import KerasModeKeys as ModeKeys
 from tensorflow.python.util import compat
 
 
 # Mapping of the modes to appropriate MetaGraph tags in the SavedModel.
-EXPORT_TAG_MAP = {
-    mode_keys.ModeKeys.PREDICT: [tag_constants.SERVING],
-    mode_keys.ModeKeys.TRAIN: [tag_constants.TRAINING],
-    mode_keys.ModeKeys.TEST: [tag_constants.EVAL],
-}
+EXPORT_TAG_MAP = mode_keys.ModeKeyMap(**{
+    ModeKeys.PREDICT: [tag_constants.SERVING],
+    ModeKeys.TRAIN: [tag_constants.TRAINING],
+    ModeKeys.TEST: [tag_constants.EVAL]})
 
 # For every exported mode, a SignatureDef map should be created using the
 # functions `export_outputs_for_mode` and `build_all_signature_defs`. By
 # default, this map will contain a single Signature that defines the input
 # tensors and output predictions, losses, and/or metrics (depending on the mode)
 # The default keys used in the SignatureDef map are defined below.
-SIGNATURE_KEY_MAP = {
-    mode_keys.ModeKeys.PREDICT:
-        signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY,
-    mode_keys.ModeKeys.TRAIN:
-        signature_constants.DEFAULT_TRAIN_SIGNATURE_DEF_KEY,
-    mode_keys.ModeKeys.TEST: signature_constants.DEFAULT_EVAL_SIGNATURE_DEF_KEY
-}
+SIGNATURE_KEY_MAP = mode_keys.ModeKeyMap(**{
+    ModeKeys.PREDICT: signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY,
+    ModeKeys.TRAIN: signature_constants.DEFAULT_TRAIN_SIGNATURE_DEF_KEY,
+    ModeKeys.TEST: signature_constants.DEFAULT_EVAL_SIGNATURE_DEF_KEY})
 
 _SINGLE_FEATURE_DEFAULT_NAME = 'feature'
 _SINGLE_RECEIVER_DEFAULT_NAME = 'input'
@@ -281,9 +278,9 @@ def export_outputs_for_mode(
         'this function. Please ensure that you are using the new ModeKeys.'
         .format(mode, SIGNATURE_KEY_MAP.keys()))
   signature_key = SIGNATURE_KEY_MAP[mode]
-  if mode == mode_keys.ModeKeys.PREDICT:
+  if mode_keys.is_predict(mode):
     return get_export_outputs(serving_export_outputs, predictions)
-  elif mode == mode_keys.ModeKeys.TRAIN:
+  elif mode_keys.is_eval(mode):
     return {signature_key: export_output_lib.TrainOutput(
         loss=loss, predictions=predictions, metrics=metrics)}
   else:
diff --git a/tensorflow/python/saved_model/model_utils/mode_keys.py b/tensorflow/python/saved_model/model_utils/mode_keys.py
new file mode 100644
index 0000000000..6bbfba912f
--- /dev/null
+++ b/tensorflow/python/saved_model/model_utils/mode_keys.py
@@ -0,0 +1,124 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Utils for managing different mode strings used by Keras and Estimator models.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import collections
+
+
+class KerasModeKeys(object):
+  """Standard names for model modes.
+
+  The following standard keys are defined:
+
+  * `TRAIN`: training/fitting mode.
+  * `TEST`: testing/evaluation mode.
+  * `PREDICT`: prediction/inference mode.
+  """
+
+  TRAIN = 'train'
+  TEST = 'test'
+  PREDICT = 'predict'
+
+
+class ModeKeys(object):
+  """Standard names for model modes.
+
+  The following standard keys are defined:
+
+  * `TRAIN`: training/fitting mode.
+  * `TEST`: testing/evaluation mode.
+  * `PREDICT`: prediction/inference mode.
+  """
+
+  TRAIN = 'train'
+  TEST = 'test'
+  PREDICT = 'predict'
+
+
+# TODO(kathywu): Remove copy in Estimator after nightlies
+class EstimatorModeKeys(object):
+  """Standard names for Estimator model modes.
+
+  The following standard keys are defined:
+
+  * `TRAIN`: training/fitting mode.
+  * `EVAL`: testing/evaluation mode.
+  * `PREDICT`: predication/inference mode.
+  """
+
+  TRAIN = 'train'
+  EVAL = 'eval'
+  PREDICT = 'infer'
+
+
+def is_predict(mode):
+  return mode in [KerasModeKeys.PREDICT, EstimatorModeKeys.PREDICT]
+
+
+def is_eval(mode):
+  return mode in [KerasModeKeys.TEST, EstimatorModeKeys.EVAL]
+
+
+def is_train(mode):
+  return mode in [KerasModeKeys.TRAIN, EstimatorModeKeys.TRAIN]
+
+
+class ModeKeyMap(collections.Mapping):
+  """Map using ModeKeys as keys.
+
+  This class creates an immutable mapping from modes to values. For example,
+  SavedModel export of Keras and Estimator models use this to map modes to their
+  corresponding MetaGraph tags/SignatureDef keys.
+
+  Since this class uses modes, rather than strings, as keys, both "predict"
+  (Keras's PREDICT ModeKey) and "infer" (Estimator's PREDICT ModeKey) map to the
+  same value.
+  """
+
+  def __init__(self, **kwargs):
+    self._internal_dict = {}
+    self._keys = []
+    for key in kwargs:
+      self._keys.append(key)
+      dict_key = self._get_internal_key(key)
+      if dict_key in self._internal_dict:
+        raise ValueError(
+            'Error creating ModeKeyMap. Multiple keys/values found for {} mode.'
+            .format(dict_key))
+      self._internal_dict[dict_key] = kwargs[key]
+
+  def _get_internal_key(self, key):
+    """Return keys used for the internal dictionary."""
+    if is_train(key):
+      return KerasModeKeys.TRAIN
+    if is_eval(key):
+      return KerasModeKeys.TEST
+    if is_predict(key):
+      return KerasModeKeys.PREDICT
+    raise ValueError('Invalid mode key: {}.'.format(key))
+
+  def __getitem__(self, key):
+    return self._internal_dict[self._get_internal_key(key)]
+
+  def __iter__(self):
+    return iter(self._keys)
+
+  def __len__(self):
+    return len(self._keys)
diff --git a/tensorflow/python/saved_model/model_utils/mode_keys_test.py b/tensorflow/python/saved_model/model_utils/mode_keys_test.py
new file mode 100644
index 0000000000..26795ef8b1
--- /dev/null
+++ b/tensorflow/python/saved_model/model_utils/mode_keys_test.py
@@ -0,0 +1,65 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""ModeKey Tests."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.platform import test
+from tensorflow.python.saved_model.model_utils import mode_keys
+
+
+class ModeKeyMapTest(test.TestCase):
+
+  def test_map(self):
+    mode_map = mode_keys.ModeKeyMap(**{
+        mode_keys.KerasModeKeys.PREDICT: 3,
+        mode_keys.KerasModeKeys.TEST: 1
+    })
+
+    # Test dictionary __getitem__
+    self.assertEqual(3, mode_map[mode_keys.KerasModeKeys.PREDICT])
+    self.assertEqual(3, mode_map[mode_keys.EstimatorModeKeys.PREDICT])
+    self.assertEqual(1, mode_map[mode_keys.KerasModeKeys.TEST])
+    self.assertEqual(1, mode_map[mode_keys.EstimatorModeKeys.EVAL])
+    with self.assertRaises(KeyError):
+      _ = mode_map[mode_keys.KerasModeKeys.TRAIN]
+    with self.assertRaises(KeyError):
+      _ = mode_map[mode_keys.EstimatorModeKeys.TRAIN]
+    with self.assertRaisesRegexp(ValueError, 'Invalid mode'):
+      _ = mode_map['serve']
+
+    # Test common dictionary methods
+    self.assertLen(mode_map, 2)
+    self.assertEqual({1, 3}, set(mode_map.values()))
+    self.assertEqual(
+        {mode_keys.KerasModeKeys.TEST, mode_keys.KerasModeKeys.PREDICT},
+        set(mode_map.keys()))
+
+    # Map is immutable
+    with self.assertRaises(TypeError):
+      mode_map[mode_keys.KerasModeKeys.TEST] = 1
+
+  def test_invalid_init(self):
+    with self.assertRaisesRegexp(ValueError, 'Multiple keys/values found'):
+      _ = mode_keys.ModeKeyMap(**{
+          mode_keys.KerasModeKeys.PREDICT: 3,
+          mode_keys.EstimatorModeKeys.PREDICT: 1
+      })
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/python/training/mode_keys_test.py b/tensorflow/python/training/mode_keys_test.py
deleted file mode 100644
index c4435b7d48..0000000000
--- a/tensorflow/python/training/mode_keys_test.py
+++ /dev/null
@@ -1,29 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for `tf.train.ModeKeys."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from tensorflow.python.platform import test
-from tensorflow.python.training import mode_keys
-
-
-class ModeKeysTest(test.TestCase):
-
-  def testKeyEquality(self):
-    self.assertEqual(mode_keys.ModeKeys.PREDICT, 'predict')
-    self.assertEqual(mode_keys.ModeKeys.TRAIN, 'train')
-    self.assertEqual(mode_keys.ModeKeys.TEST, 'test')
-- 
GitLab


From 3a475eb69dd84c81befdf493a196cd6f8647e638 Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Mon, 11 Feb 2019 15:11:04 -0800
Subject: [PATCH 2284/2345] Create int8 strided slice.

PiperOrigin-RevId: 233487928
---
 tensorflow/lite/kernels/register.cc           |  4 +++-
 tensorflow/lite/kernels/strided_slice.cc      |  5 +++++
 tensorflow/lite/kernels/strided_slice_test.cc | 12 ++++++++++++
 tensorflow/lite/toco/tflite/operator.cc       |  6 ++++++
 tensorflow/lite/toco/tflite/operator_test.cc  |  4 ++++
 5 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index c5814a1a4d..7534cb2ab1 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -253,7 +253,9 @@ BuiltinOpResolver::BuiltinOpResolver() {
              /* max_version */ 3);
   AddBuiltin(BuiltinOperator_SPLIT_V, Register_SPLIT_V());
   AddBuiltin(BuiltinOperator_SQUEEZE, Register_SQUEEZE());
-  AddBuiltin(BuiltinOperator_STRIDED_SLICE, Register_STRIDED_SLICE());
+  AddBuiltin(BuiltinOperator_STRIDED_SLICE, Register_STRIDED_SLICE(),
+             /* min_version */ 1,
+             /* max_version */ 2);
   AddBuiltin(BuiltinOperator_EXP, Register_EXP());
   AddBuiltin(BuiltinOperator_TOPK_V2, Register_TOPK_V2(),
              /* min_version */ 1,
diff --git a/tensorflow/lite/kernels/strided_slice.cc b/tensorflow/lite/kernels/strided_slice.cc
index c797a98e9f..8c25ffa3a1 100644
--- a/tensorflow/lite/kernels/strided_slice.cc
+++ b/tensorflow/lite/kernels/strided_slice.cc
@@ -234,6 +234,11 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
         TF_LITE_STRIDED_SLICE(reference_ops, uint8_t);
       }
       break;
+    case kTfLiteInt8:
+      if (kernel_type == kReference) {
+        TF_LITE_STRIDED_SLICE(reference_ops, int8_t);
+      }
+      break;
     default:
       context->ReportError(context,
                            "Type %d is currently not supported "
diff --git a/tensorflow/lite/kernels/strided_slice_test.cc b/tensorflow/lite/kernels/strided_slice_test.cc
index 34875bf049..cac9e1672f 100644
--- a/tensorflow/lite/kernels/strided_slice_test.cc
+++ b/tensorflow/lite/kernels/strided_slice_test.cc
@@ -577,6 +577,18 @@ TEST(StridedSliceOpTest, In3D_IdentityShrinkAxis1Uint8) {
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({3, 2}));
   EXPECT_THAT(m.GetOutput(), ElementsAreArray({1, 2, 3, 4, 5, 6}));
 }
+
+TEST(StridedSliceOpTest, In3D_IdentityShrinkAxis1int8) {
+  StridedSliceOpModel<int8_t, TensorType_INT8> m({2, 3, 2}, {3}, {3}, {3}, 0, 0,
+                                                 0, 0, 1);
+  m.SetInput({1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
+  m.SetBegin({0, 0, 0});
+  m.SetEnd({1, 3, 2});
+  m.SetStrides({1, 1, 1});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({3, 2}));
+  EXPECT_THAT(m.GetOutput(), ElementsAreArray({1, 2, 3, 4, 5, 6}));
+}
 }  // namespace
 }  // namespace tflite
 
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index a0fd483889..42586ca2d3 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -1400,6 +1400,12 @@ class StridedSlice
   }
 
   int GetVersion(const OperatorSignature& op_signature) const override {
+    const string& input_name = op_signature.op->inputs[0];
+    const Array& input_array = op_signature.model->GetArray(input_name);
+    // If the op take int8 input, it is version 2.
+    if (input_array.data_type == ArrayDataType::kInt8) {
+      return 2;
+    }
     return 1;
   }
 };
diff --git a/tensorflow/lite/toco/tflite/operator_test.cc b/tensorflow/lite/toco/tflite/operator_test.cc
index 755c0828a1..71e186cee5 100644
--- a/tensorflow/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/lite/toco/tflite/operator_test.cc
@@ -808,6 +808,10 @@ TEST_F(OperatorTest, VersioningPackTest) {
   SimpleVersioningTest<PackOperator>();
 }
 
+TEST_F(OperatorTest, VersioningStridedSliceTest) {
+  SimpleVersioningTest<StridedSliceOperator>();
+}
+
 TEST_F(OperatorTest, VersioningSelectTest) {
   SelectOperator select_op;
   select_op.inputs = {"input1"};
-- 
GitLab


From 2fa8a1f16f978c9b7125de5bcf8abf952ff42442 Mon Sep 17 00:00:00 2001
From: Katherine Wu <kathywu@google.com>
Date: Mon, 11 Feb 2019 15:23:22 -0800
Subject: [PATCH 2285/2345] Delete duplicate ModeKeys

PiperOrigin-RevId: 233490111
---
 .../python/saved_model/model_utils/mode_keys.py   | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/tensorflow/python/saved_model/model_utils/mode_keys.py b/tensorflow/python/saved_model/model_utils/mode_keys.py
index 6bbfba912f..2912de7210 100644
--- a/tensorflow/python/saved_model/model_utils/mode_keys.py
+++ b/tensorflow/python/saved_model/model_utils/mode_keys.py
@@ -37,21 +37,6 @@ class KerasModeKeys(object):
   PREDICT = 'predict'
 
 
-class ModeKeys(object):
-  """Standard names for model modes.
-
-  The following standard keys are defined:
-
-  * `TRAIN`: training/fitting mode.
-  * `TEST`: testing/evaluation mode.
-  * `PREDICT`: prediction/inference mode.
-  """
-
-  TRAIN = 'train'
-  TEST = 'test'
-  PREDICT = 'predict'
-
-
 # TODO(kathywu): Remove copy in Estimator after nightlies
 class EstimatorModeKeys(object):
   """Standard names for Estimator model modes.
-- 
GitLab


From 5a297f5efea725fc5889132bafc39707f57aa7ec Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 11 Feb 2019 15:35:11 -0800
Subject: [PATCH 2286/2345] Update profiler API to allow user using it in graph
 mode.

PiperOrigin-RevId: 233492307
---
 tensorflow/c/eager/c_api_experimental.cc      | 20 +++++++++-------
 tensorflow/c/eager/c_api_experimental.h       | 23 +++++++++++--------
 tensorflow/c/eager/c_api_experimental_test.cc | 13 ++++++++---
 tensorflow/c/eager/c_api_internal.h           | 14 +++++------
 tensorflow/python/eager/profiler.py           | 17 ++++++++++----
 tensorflow/python/pywrap_tfe.i                |  5 ++--
 6 files changed, 56 insertions(+), 36 deletions(-)

diff --git a/tensorflow/c/eager/c_api_experimental.cc b/tensorflow/c/eager/c_api_experimental.cc
index f46562a21d..06bbb4ac41 100644
--- a/tensorflow/c/eager/c_api_experimental.cc
+++ b/tensorflow/c/eager/c_api_experimental.cc
@@ -25,7 +25,7 @@ void TFE_OpConsumeInput(TFE_Op* op, TFE_TensorHandle* h, TF_Status* status) {
   op->operation.ConsumeInput(h->handle);
 }
 
-TFE_Profiler* TFE_NewProfiler(TFE_Context* ctx) {
+TFE_Profiler* TFE_NewProfiler(TFE_ProfilerContext* ctx) {
   return new TFE_Profiler(ctx);
 }
 
@@ -50,17 +50,21 @@ void TFE_ProfilerSerializeToString(TFE_Context* ctx, TFE_Profiler* profiler,
   };
 }
 
-TFE_ProfilerServerOptions* TFE_NewProfilerServerOptions() {
-  return new TFE_ProfilerServerOptions;
+TFE_ProfilerContext* TFE_NewProfilerContext() {
+  return new TFE_ProfilerContext;
 }
 
-void TFE_ProfilerServerOptionsSetEagerContext(
-    TFE_ProfilerServerOptions* options, TFE_Context* ctx) {
-  options->profiler_context.eager_context = &ctx->context;
+void TFE_ProfilerContextSetEagerContext(TFE_ProfilerContext* profiler_context,
+                                        TFE_Context* eager_context) {
+  profiler_context->profiler_context.eager_context = &eager_context->context;
 }
 
-void TFE_StartProfilerServer(TFE_ProfilerServerOptions* options, int port) {
+void TFE_DeleteProfilerContext(TFE_ProfilerContext* profiler_context) {
+  delete profiler_context;
+}
+
+void TFE_StartProfilerServer(TFE_ProfilerContext* context, int port) {
   // Release child thread intentionally. The child thread can be terminate by
   // terminating the main thread.
-  tensorflow::StartProfilerServer(&options->profiler_context, port).release();
+  tensorflow::StartProfilerServer(&context->profiler_context, port).release();
 }
diff --git a/tensorflow/c/eager/c_api_experimental.h b/tensorflow/c/eager/c_api_experimental.h
index e6a3d73707..51a5fa0d81 100644
--- a/tensorflow/c/eager/c_api_experimental.h
+++ b/tensorflow/c/eager/c_api_experimental.h
@@ -25,6 +25,8 @@ extern "C" {
 TF_CAPI_EXPORT extern void TFE_OpConsumeInput(TFE_Op* op, TFE_TensorHandle* h,
                                               TF_Status* status);
 
+typedef struct TFE_ProfilerContext TFE_ProfilerContext;
+
 // A profiler which will start profiling when creating the object and will stop
 // when the object is destroyed. It will profile all operations run under the
 // given TFE_Context. Multiple instance of it can be created, but at most one
@@ -32,7 +34,7 @@ TF_CAPI_EXPORT extern void TFE_OpConsumeInput(TFE_Op* op, TFE_TensorHandle* h,
 // Thread-safety: TFE_Profiler is thread-safe.
 typedef struct TFE_Profiler TFE_Profiler;
 
-TF_CAPI_EXPORT extern TFE_Profiler* TFE_NewProfiler(TFE_Context* ctx);
+TF_CAPI_EXPORT extern TFE_Profiler* TFE_NewProfiler(TFE_ProfilerContext* ctx);
 TF_CAPI_EXPORT extern bool TFE_ProfilerIsOk(TFE_Profiler* profiler);
 TF_CAPI_EXPORT extern void TFE_DeleteProfiler(TFE_Profiler* profiler);
 
@@ -43,15 +45,16 @@ TF_CAPI_EXPORT extern void TFE_ProfilerSerializeToString(TFE_Context* ctx,
                                                          TF_Buffer* buf,
                                                          TF_Status* status);
 
-typedef struct TFE_ProfilerServerOptions TFE_ProfilerServerOptions;
-
-// Return a new Profiler server options object.
-TF_CAPI_EXPORT extern TFE_ProfilerServerOptions* TFE_NewProfilerServerOptions(
-    void);
+// Return a new profiler context object.
+TF_CAPI_EXPORT extern TFE_ProfilerContext* TFE_NewProfilerContext(void);
 
 // Set the eager context in TFE_ProfilerServerOptions
-TF_CAPI_EXPORT extern void TFE_ProfilerServerOptionsSetEagerContext(
-    TFE_ProfilerServerOptions* options, TFE_Context* ctx);
+TF_CAPI_EXPORT extern void TFE_ProfilerContextSetEagerContext(
+    TFE_ProfilerContext* profiler_context, TFE_Context* eager_context);
+
+// Destroy a profiler context object.
+TF_CAPI_EXPORT extern void TFE_DeleteProfilerContext(
+    TFE_ProfilerContext* profiler_context);
 
 // Start a profiler grpc server which listens to specified port. It will start
 // the server on its own thread. It can be shutdown by terminating tensorflow.
@@ -61,8 +64,8 @@ TF_CAPI_EXPORT extern void TFE_ProfilerServerOptionsSetEagerContext(
 // tensorflow/contrib/tpu/profiler/capture_tpu_profile to capture tracable
 // file following
 // https://cloud.google.com/tpu/docs/cloud-tpu-tools#capture_trace.
-TF_CAPI_EXPORT extern void TFE_StartProfilerServer(
-    TFE_ProfilerServerOptions* options, int port);
+TF_CAPI_EXPORT extern void TFE_StartProfilerServer(TFE_ProfilerContext* context,
+                                                   int port);
 
 #ifdef __cplusplus
 } /* end extern "C" */
diff --git a/tensorflow/c/eager/c_api_experimental_test.cc b/tensorflow/c/eager/c_api_experimental_test.cc
index 1994d65224..7b84c5426f 100644
--- a/tensorflow/c/eager/c_api_experimental_test.cc
+++ b/tensorflow/c/eager/c_api_experimental_test.cc
@@ -41,9 +41,12 @@ void ExecuteWithProfiling(bool async) {
   TFE_ContextOptions* opts = TFE_NewContextOptions();
   TFE_ContextOptionsSetAsync(opts, static_cast<unsigned char>(async));
   TFE_Context* ctx = TFE_NewContext(opts, status);
-  TFE_Profiler* profiler = TFE_NewProfiler(ctx);
+  TFE_ProfilerContext* profiler_context = TFE_NewProfilerContext();
+  TFE_ProfilerContextSetEagerContext(profiler_context, ctx);
+  TFE_Profiler* profiler = TFE_NewProfiler(profiler_context);
   CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
   TFE_DeleteContextOptions(opts);
+  TFE_DeleteProfilerContext(profiler_context);
 
   TFE_TensorHandle* m = TestMatrixTensorHandle();
   TFE_Op* matmul = MatMulOp(ctx, m, m);
@@ -108,14 +111,18 @@ TEST(CAPI, MultipleProfilerSession) {
   CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
   TFE_DeleteContextOptions(opts);
 
-  TFE_Profiler* profiler1 = TFE_NewProfiler(ctx);
+  TFE_ProfilerContext* profiler_context = TFE_NewProfilerContext();
+  TFE_ProfilerContextSetEagerContext(profiler_context, ctx);
+
+  TFE_Profiler* profiler1 = TFE_NewProfiler(profiler_context);
   EXPECT_TRUE(TFE_ProfilerIsOk(profiler1));
 
-  TFE_Profiler* profiler2 = TFE_NewProfiler(ctx);
+  TFE_Profiler* profiler2 = TFE_NewProfiler(profiler_context);
   EXPECT_FALSE(TFE_ProfilerIsOk(profiler2));
 
   TFE_DeleteProfiler(profiler1);
   TFE_DeleteProfiler(profiler2);
+  TFE_DeleteProfilerContext(profiler_context);
 }
 
 }  // namespace
diff --git a/tensorflow/c/eager/c_api_internal.h b/tensorflow/c/eager/c_api_internal.h
index b070e1cb1f..a563e4b8f5 100644
--- a/tensorflow/c/eager/c_api_internal.h
+++ b/tensorflow/c/eager/c_api_internal.h
@@ -107,20 +107,18 @@ struct TFE_Op {
   tensorflow::EagerOperation operation;
 };
 
+struct TFE_ProfilerContext {
+  tensorflow::ProfilerContext profiler_context;
+};
+
 struct TFE_Profiler {
-  TFE_Profiler(TFE_Context* ctx) {
-    tensorflow::ProfilerContext profiler_context;
-    profiler_context.eager_context = &ctx->context;
-    profiler = tensorflow::ProfilerSession::Create(&profiler_context);
+  TFE_Profiler(TFE_ProfilerContext* ctx) {
+    profiler = tensorflow::ProfilerSession::Create(&ctx->profiler_context);
   }
 
   std::unique_ptr<tensorflow::ProfilerSession> profiler;
 };
 
-struct TFE_ProfilerServerOptions {
-  tensorflow::ProfilerContext profiler_context;
-};
-
 namespace tensorflow {
 // Set an AttrValue on the op. Doesn't handle the list types.
 void SetOpAttrValueScalar(TFE_Context* ctx, TFE_Op* op,
diff --git a/tensorflow/python/eager/profiler.py b/tensorflow/python/eager/profiler.py
index 7ddd891cdc..659c0cc6e6 100644
--- a/tensorflow/python/eager/profiler.py
+++ b/tensorflow/python/eager/profiler.py
@@ -46,7 +46,13 @@ def start():
   if _profiler is not None:
     raise AssertionError('Another profiler is running.')
   with _profiler_lock:
-    _profiler = pywrap_tensorflow.TFE_NewProfiler(context.context()._handle)  # pylint: disable=protected-access
+    profiler_context = pywrap_tensorflow.TFE_NewProfilerContext()
+    if context.default_execution_mode == context.EAGER_MODE:
+      pywrap_tensorflow.TFE_ProfilerContextSetEagerContext(
+          profiler_context,
+          context.context()._handle)  # pylint: disable=protected-access
+    _profiler = pywrap_tensorflow.TFE_NewProfiler(profiler_context)
+    pywrap_tensorflow.TFE_DeleteProfilerContext(profiler_context)
     if not pywrap_tensorflow.TFE_ProfilerIsOk(_profiler):
       logging.warning('Another profiler session is running which is probably '
                       'created by profiler server. Please avoid using profiler '
@@ -93,12 +99,13 @@ def start_profiler_server(port):
   Args:
     port: port profiler server listens to.
   """
-  opts = pywrap_tensorflow.TFE_NewProfilerServerOptions()
+  profiler_context = pywrap_tensorflow.TFE_NewProfilerContext()
   if context.default_execution_mode == context.EAGER_MODE:
-    pywrap_tensorflow.TFE_ProfilerServerOptionsSetEagerContext(
-        opts,
+    pywrap_tensorflow.TFE_ProfilerContextSetEagerContext(
+        profiler_context,
         context.context()._handle)  # pylint: disable=protected-access
-  pywrap_tensorflow.TFE_StartProfilerServer(opts, port)
+  pywrap_tensorflow.TFE_StartProfilerServer(profiler_context, port)
+  pywrap_tensorflow.TFE_DeleteProfilerContext(profiler_context)
 
 
 class Profiler(object):
diff --git a/tensorflow/python/pywrap_tfe.i b/tensorflow/python/pywrap_tfe.i
index db6c14fb60..3bf775537d 100755
--- a/tensorflow/python/pywrap_tfe.i
+++ b/tensorflow/python/pywrap_tfe.i
@@ -37,8 +37,9 @@ limitations under the License.
 %rename("%s") TFE_ProfilerIsOk;
 %rename("%s") TFE_DeleteProfiler;
 %rename("%s") TFE_ProfilerSerializeToString;
-%rename("%s") TFE_NewProfilerServerOptions;
-%rename("%s") TFE_ProfilerServerOptionsSetEagerContext;
+%rename("%s") TFE_NewProfilerContext;
+%rename("%s") TFE_ProfilerContextSetEagerContext;
+%rename("%s") TFE_DeleteProfilerContext;
 %rename("%s") TFE_StartProfilerServer;
 %rename("%s") TFE_OpNameGetAttrType;
 %rename("%s") TFE_Py_InitEagerTensor;
-- 
GitLab


From 8bb8258646e47ef96e40eb684ac50601a3af5106 Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Mon, 11 Feb 2019 15:47:34 -0800
Subject: [PATCH 2287/2345] Link TF-TRT into python:no_contrib (so it's
 included in TF 2.0 build) and lazily load TRT shared library.

PiperOrigin-RevId: 233494604
---
 tensorflow/compiler/tf2tensorrt/BUILD         |  1 +
 .../tf2tensorrt/python/ops/trt_ops.py         | 48 +++++++++++++++----
 tensorflow/python/BUILD                       |  1 +
 tensorflow/python/compiler/BUILD              | 17 +++++++
 tensorflow/python/compiler/__init__.py        |  0
 tensorflow/python/compiler/tensorrt/BUILD     |  4 +-
 .../python/compiler/tensorrt/__init__.py      | 25 ++--------
 .../compiler/tensorrt/test/base_test.py       |  9 ++--
 .../tensorrt/test/quantization_mnist_test.py  |  9 ++--
 .../tensorrt/test/quantization_test.py        |  6 +--
 .../test/tf_trt_integration_test_base.py      | 21 ++++----
 .../python/compiler/tensorrt/trt_convert.py   | 32 +++++++++----
 .../compiler/tensorrt/trt_convert_test.py     | 35 +++++++-------
 13 files changed, 124 insertions(+), 84 deletions(-)
 create mode 100644 tensorflow/python/compiler/BUILD
 create mode 100644 tensorflow/python/compiler/__init__.py

diff --git a/tensorflow/compiler/tf2tensorrt/BUILD b/tensorflow/compiler/tf2tensorrt/BUILD
index 00d3c8cc5f..76a009e1d5 100644
--- a/tensorflow/compiler/tf2tensorrt/BUILD
+++ b/tensorflow/compiler/tf2tensorrt/BUILD
@@ -160,6 +160,7 @@ tf_custom_op_py_library(
     ],
     srcs_version = "PY2AND3",
     deps = [
+        "//tensorflow/python:errors",
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:platform",
         "//tensorflow/python:resources",
diff --git a/tensorflow/compiler/tf2tensorrt/python/ops/trt_ops.py b/tensorflow/compiler/tf2tensorrt/python/ops/trt_ops.py
index 86bfabf99e..a11150f155 100644
--- a/tensorflow/compiler/tf2tensorrt/python/ops/trt_ops.py
+++ b/tensorflow/compiler/tf2tensorrt/python/ops/trt_ops.py
@@ -18,17 +18,45 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import threading
+
 import platform
+from tensorflow.python.framework import errors
+
+_trt_ops_so = None
+_module_lock = threading.Lock()
+
+
+def load_trt_ops():
+  """Load TF-TRT op libraries so if it hasn't been loaded already."""
+  global _trt_ops_so
+
+  if platform.system() == "Windows":
+    raise RuntimeError("Windows platforms are not supported")
 
-if platform.system() != "Windows":
-  # pylint: disable=wildcard-import,unused-import,g-import-not-at-top
-  from tensorflow.compiler.tf2tensorrt.ops.gen_trt_ops import *
+  with _module_lock:
+    if _trt_ops_so:
+      return
 
-  from tensorflow.python.framework import load_library
-  from tensorflow.python.platform import resource_loader
-  # pylint: enable=wildcard-import,unused-import,g-import-not-at-top
+    # TODO(laigd): we should load TF-TRT kernels here as well after removing the
+    # swig binding.
+    try:
+      # TODO(lagid): consider getting rid of these unused imports.
+      # pylint: disable=unused-import,g-import-not-at-top,unused-variable
+      from tensorflow.compiler.tf2tensorrt.ops.gen_trt_ops import get_serialized_resource_op
+      from tensorflow.compiler.tf2tensorrt.ops.gen_trt_ops import trt_engine_op
+      from tensorflow.python.framework import load_library
+      from tensorflow.python.platform import resource_loader
+      # pylint: enable=unused-import,g-import-not-at-top,unused-variable
 
-  _trt_ops = load_library.load_op_library(
-      resource_loader.get_path_to_datafile("_trt_ops.so"))
-else:
-  raise RuntimeError("Windows platforms are not supported")
+      _trt_ops_so = load_library.load_op_library(
+          resource_loader.get_path_to_datafile("_trt_ops.so"))
+    except errors.NotFoundError as e:
+      no_trt_message = (
+          "**** Failed to initialize TensorRT. This is either because the "
+          "TensorRT installation path is not in LD_LIBRARY_PATH, or because "
+          "you do not have it installed. If not installed, please go to "
+          "https://developer.nvidia.com/tensorrt to download and install "
+          "TensorRT ****")
+      print(no_trt_message)
+      raise e
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 36ec9e01e2..a6d6868498 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -173,6 +173,7 @@ py_library(
         "//tensorflow/lite/python:lite",
         "//tensorflow/python/compat",
         "//tensorflow/python/compat:v2_compat",
+        "//tensorflow/python/compiler",
         "//tensorflow/python/data",
         "//tensorflow/python/distribute",
         "//tensorflow/python/distribute:estimator_training",
diff --git a/tensorflow/python/compiler/BUILD b/tensorflow/python/compiler/BUILD
new file mode 100644
index 0000000000..26eb06a10e
--- /dev/null
+++ b/tensorflow/python/compiler/BUILD
@@ -0,0 +1,17 @@
+# Description:
+# Python APIs for various Tensorflow backends.
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])  # Apache 2.0
+
+exports_files(["LICENSE"])
+
+py_library(
+    name = "compiler",
+    srcs = ["__init__.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python/compiler/tensorrt:init_py",
+    ],
+)
diff --git a/tensorflow/python/compiler/__init__.py b/tensorflow/python/compiler/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tensorflow/python/compiler/tensorrt/BUILD b/tensorflow/python/compiler/tensorrt/BUILD
index a382579f5a..afa25113b2 100644
--- a/tensorflow/python/compiler/tensorrt/BUILD
+++ b/tensorflow/python/compiler/tensorrt/BUILD
@@ -32,8 +32,6 @@ py_library(
     deps = [
         ":tf_trt_integration_test_base",
         ":trt_convert_py",
-        ":trt_ops_py",
-        "//tensorflow/python:errors",
     ],
 )
 
@@ -51,6 +49,7 @@ py_library(
     srcs = ["trt_convert.py"],
     srcs_version = "PY2AND3",
     deps = [
+        ":trt_ops_py",
         ":wrap_conversion",
         "//tensorflow/python:graph_util",
         "//tensorflow/python:session",
@@ -83,7 +82,6 @@ py_library(
     srcs = ["test/tf_trt_integration_test_base.py"],
     deps = [
         ":trt_convert_py",
-        ":trt_ops_py",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_test_lib",
     ],
diff --git a/tensorflow/python/compiler/tensorrt/__init__.py b/tensorflow/python/compiler/tensorrt/__init__.py
index 88fb69101d..db3540ba45 100644
--- a/tensorflow/python/compiler/tensorrt/__init__.py
+++ b/tensorflow/python/compiler/tensorrt/__init__.py
@@ -18,25 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.python.framework import errors
-
-# pylint: disable=unused-import,g-import-not-at-top,line-too-long
-try:
-  from tensorflow.compiler.tf2tensorrt.python.ops import trt_ops
-  from tensorflow.python.compiler.tensorrt.trt_convert import add_test_value
-  from tensorflow.python.compiler.tensorrt.trt_convert import calib_graph_to_infer_graph
-  from tensorflow.python.compiler.tensorrt.trt_convert import clear_test_values
-  from tensorflow.python.compiler.tensorrt.trt_convert import create_inference_graph
-  from tensorflow.python.compiler.tensorrt.trt_convert import enable_test_value
-  from tensorflow.python.compiler.tensorrt.trt_convert import get_test_value
-  from tensorflow.python.compiler.tensorrt.trt_convert import is_tensorrt_enabled
-except errors.NotFoundError as e:
-  no_trt_message = (
-      '**** Failed to initialize TensorRT. This is either because the TensorRT'
-      ' installation path is not in LD_LIBRARY_PATH, or because you do not have'
-      ' it installed. If not installed, please go to'
-      ' https://developer.nvidia.com/tensorrt to download and install'
-      ' TensorRT ****')
-  print(no_trt_message)
-  raise e
-# pylint: enable=unused-import,g-import-not-at-top,line-too-long
+# pylint: disable=unused-import,line-too-long
+from tensorflow.python.compiler.tensorrt.trt_convert import create_inference_graph
+# pylint: enable=unused-import,line-too-long
diff --git a/tensorflow/python/compiler/tensorrt/test/base_test.py b/tensorflow/python/compiler/tensorrt/test/base_test.py
index cc3109968e..a1199c5040 100644
--- a/tensorflow/python/compiler/tensorrt/test/base_test.py
+++ b/tensorflow/python/compiler/tensorrt/test/base_test.py
@@ -20,8 +20,9 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.python.compiler.tensorrt import trt_convert
 from tensorflow.python.compiler.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.compiler.tensorrt.wrap_conversion import add_test_value
+from tensorflow.python.compiler.tensorrt.wrap_conversion import clear_test_values
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -154,7 +155,7 @@ class PartiallyConvertedTestA(trt_test.TfTrtIntegrationTestBase):
     """Setup method."""
     super(PartiallyConvertedTestA, self).setUp()
     # Let it fail to build the second engine.
-    trt_convert.add_test_value("TRTEngineOp_1:CreateTRTNode", "fail")
+    add_test_value("TRTEngineOp_1:CreateTRTNode", "fail")
 
   def GetParams(self):
     """Create a graph containing two segment."""
@@ -209,8 +210,8 @@ class PartiallyConvertedTestB(PartiallyConvertedTestA):
     """Setup method."""
     super(PartiallyConvertedTestB, self).setUp()
     # Let it fail to build the first engine.
-    trt_convert.clear_test_values("")
-    trt_convert.add_test_value("TRTEngineOp_0:CreateTRTNode", "fail")
+    clear_test_values("")
+    add_test_value("TRTEngineOp_0:CreateTRTNode", "fail")
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/python/compiler/tensorrt/test/quantization_mnist_test.py b/tensorflow/python/compiler/tensorrt/test/quantization_mnist_test.py
index 1d7792c01a..e2b21d7b92 100644
--- a/tensorflow/python/compiler/tensorrt/test/quantization_mnist_test.py
+++ b/tensorflow/python/compiler/tensorrt/test/quantization_mnist_test.py
@@ -18,13 +18,12 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-# pylint: disable=unused-import
-from tensorflow.compiler.tf2tensorrt.python.ops import trt_ops
-# pylint: enable=unused-import
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python import data
 from tensorflow.python import keras
 from tensorflow.python.compiler.tensorrt import trt_convert
+from tensorflow.python.compiler.tensorrt.wrap_conversion import get_linked_tensorrt_version
+from tensorflow.python.compiler.tensorrt.wrap_conversion import is_tensorrt_enabled
 from tensorflow.python.estimator.estimator import Estimator
 from tensorflow.python.estimator.model_fn import EstimatorSpec
 from tensorflow.python.estimator.model_fn import ModeKeys
@@ -263,7 +262,7 @@ class QuantizationAwareTrainingMNISTTest(test_util.TensorFlowTestCase):
   #     num_epochs=100,
   #     model_dir=model_dir)
   def testEval(self):
-    if not trt_convert.is_tensorrt_enabled():
+    if not is_tensorrt_enabled():
       return
     model_dir = test.test_src_dir_path('python/compiler/tensorrt/test/testdata')
 
@@ -276,7 +275,7 @@ class QuantizationAwareTrainingMNISTTest(test_util.TensorFlowTestCase):
     logging.info('accuracy_tf_native: %f', accuracy_tf_native)
     self.assertAllClose(0.9662, accuracy_tf_native, rtol=1e-3, atol=1e-3)
 
-    if trt_convert.get_linked_tensorrt_version()[0] < 5:
+    if get_linked_tensorrt_version()[0] < 5:
       return
 
     accuracy_tf_trt = self._Run(
diff --git a/tensorflow/python/compiler/tensorrt/test/quantization_test.py b/tensorflow/python/compiler/tensorrt/test/quantization_test.py
index 086e070f1b..3e1c9ff8dd 100644
--- a/tensorflow/python/compiler/tensorrt/test/quantization_test.py
+++ b/tensorflow/python/compiler/tensorrt/test/quantization_test.py
@@ -20,8 +20,8 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.python.compiler.tensorrt import trt_convert
 from tensorflow.python.compiler.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.compiler.tensorrt.wrap_conversion import get_linked_tensorrt_version
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -72,7 +72,7 @@ class QuantizationMissingAllRangesTest(trt_test.TfTrtIntegrationTestBase):
     return _GetParams(add_quantization_nodes=False)
 
   def ShouldRunTest(self, run_params):
-    if trt_convert.get_linked_tensorrt_version()[0] < 5:
+    if get_linked_tensorrt_version()[0] < 5:
       return False
     # Only test static engine mode, with or without calibration.
     return (trt_test.IsQuantizationMode(run_params.precision_mode) and
@@ -96,7 +96,7 @@ class QuantizationWithRangesTest(trt_test.TfTrtIntegrationTestBase):
     return _GetParams(add_quantization_nodes=True)
 
   def ShouldRunTest(self, run_params):
-    if trt_convert.get_linked_tensorrt_version()[0] < 5:
+    if get_linked_tensorrt_version()[0] < 5:
       return False
     # Test static/dynamic engine with/without calibration.
     return (trt_test.IsQuantizationMode(run_params.precision_mode) and
diff --git a/tensorflow/python/compiler/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/python/compiler/tensorrt/test/tf_trt_integration_test_base.py
index 28563f09a1..5af1a97016 100644
--- a/tensorflow/python/compiler/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/python/compiler/tensorrt/test/tf_trt_integration_test_base.py
@@ -25,12 +25,13 @@ import warnings
 import numpy as np
 import six
 
-# pylint: disable=unused-import
-from tensorflow.compiler.tf2tensorrt.python.ops import trt_ops
-# pylint: enable=unused-import
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python.compiler.tensorrt import trt_convert
+from tensorflow.python.compiler.tensorrt.wrap_conversion import clear_test_values
+from tensorflow.python.compiler.tensorrt.wrap_conversion import enable_test_value
+from tensorflow.python.compiler.tensorrt.wrap_conversion import get_test_value
+from tensorflow.python.compiler.tensorrt.wrap_conversion import is_tensorrt_enabled
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import graph_io
 from tensorflow.python.framework import importer
@@ -151,7 +152,7 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
   def setUpClass(cls):
     """Setup method for the module."""
     super(TfTrtIntegrationTestBase, cls).setUpClass()
-    trt_convert.enable_test_value()
+    enable_test_value()
 
   def __init__(self, methodName="runTest"):  # pylint: disable=invalid-name
     super(TfTrtIntegrationTestBase, self).__init__(methodName)
@@ -161,7 +162,7 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
     """Setup method."""
     super(TfTrtIntegrationTestBase, self).setUp()
     warnings.simplefilter("always")
-    trt_convert.clear_test_values("")
+    clear_test_values("")
 
   def GetParams(self):
     """Return a TfTrtIntegrationTestParams for test, implemented by subclass."""
@@ -246,9 +247,9 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
   def _PrepareRun(self, graph_state):
     """Set up necessary testing environment before calling sess.run()."""
     # Clear test values added by TRTEngineOp.
-    trt_convert.clear_test_values("TRTEngineOp_.*:ExecuteTrtEngine")
-    trt_convert.clear_test_values("TRTEngineOp_.*:ExecuteCalibration")
-    trt_convert.clear_test_values("TRTEngineOp_.*:ExecuteNativeSegment")
+    clear_test_values("TRTEngineOp_.*:ExecuteTrtEngine")
+    clear_test_values("TRTEngineOp_.*:ExecuteCalibration")
+    clear_test_values("TRTEngineOp_.*:ExecuteNativeSegment")
 
   def _GetGPUOptions(self):
     gpu_options = config_pb2.GPUOptions()
@@ -282,7 +283,7 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
 
   def _ExpectTestValue(self, engine_name, method, expected_value):
     label = "%s:%s" % (engine_name, method)
-    actual_value = trt_convert.get_test_value(label)
+    actual_value = get_test_value(label)
     self.assertEqual(
         expected_value,
         actual_value,
@@ -639,5 +640,5 @@ def _AddTests(test_class):
     setattr(test_class, "testTfTrt_" + test_name, _GetTest(run_params))
 
 
-if trt_convert.is_tensorrt_enabled():
+if is_tensorrt_enabled():
   _AddTests(TfTrtIntegrationTestBase)
diff --git a/tensorflow/python/compiler/tensorrt/trt_convert.py b/tensorflow/python/compiler/tensorrt/trt_convert.py
index 33b5e50418..2dd34a83aa 100644
--- a/tensorflow/python/compiler/tensorrt/trt_convert.py
+++ b/tensorflow/python/compiler/tensorrt/trt_convert.py
@@ -19,17 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 import six as _six
-# pylint: disable=unused-import,line-too-long
 from tensorflow.compiler.tf2tensorrt.python.ops import trt_ops
-from tensorflow.python.compiler.tensorrt.wrap_conversion import add_test_value
-from tensorflow.python.compiler.tensorrt.wrap_conversion import calib_convert
-from tensorflow.python.compiler.tensorrt.wrap_conversion import clear_test_values
-from tensorflow.python.compiler.tensorrt.wrap_conversion import enable_test_value
-from tensorflow.python.compiler.tensorrt.wrap_conversion import get_linked_tensorrt_version
-from tensorflow.python.compiler.tensorrt.wrap_conversion import get_loaded_tensorrt_version
-from tensorflow.python.compiler.tensorrt.wrap_conversion import get_test_value
-from tensorflow.python.compiler.tensorrt.wrap_conversion import is_tensorrt_enabled
-# pylint: enable=unused-import,line-too-long
 from tensorflow.core.framework import graph_pb2
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import meta_graph_pb2
@@ -357,6 +347,14 @@ class TrtGraphConverter(GraphConverter):
       TypeError: if any of the parameters are of unexpected type.
       ValueError: if any of the parameters are of unexpected value.
     """
+    # Lazily load the TF-TRT C bindings, so `import tensorflow` doesn't complain
+    # even if it cannot find TensorRT library.
+    trt_ops.load_trt_ops()
+    # pylint: disable=g-import-not-at-top,unused-import,line-too-long,unused-variable
+    # Import a random symbol to trigger loading of TRT library.
+    from tensorflow.python.compiler.tensorrt.wrap_conversion import calib_convert
+    # pylint: enable=g-import-not-at-top,unused-import,line-too-long,unused-variable
+
     if rewriter_config_template is not None and not isinstance(
         rewriter_config_template, rewriter_config_pb2.RewriterConfig):
       raise TypeError(
@@ -457,6 +455,14 @@ class TrtGraphConverter(GraphConverter):
         nodes_blacklist=nodes_blacklist,
         session_config=session_config)
 
+    # Lazily load the TF-TRT C bindings, so `import tensorflow` doesn't complain
+    # even if it cannot find TensorRT library.
+    trt_ops.load_trt_ops()
+    # pylint: disable=g-import-not-at-top,line-too-long
+    from tensorflow.python.compiler.tensorrt.wrap_conversion import get_linked_tensorrt_version
+    from tensorflow.python.compiler.tensorrt.wrap_conversion import get_loaded_tensorrt_version
+    # pylint: enable=g-import-not-at-top,line-too-long
+
     # Check compatibility of TensorRT version.
     compiled_version = get_linked_tensorrt_version()
     loaded_version = get_loaded_tensorrt_version()
@@ -642,6 +648,12 @@ def calib_graph_to_infer_graph(calibration_graph_def, is_dynamic_op=False):
   Raises:
     RuntimeError: if the returned status message is malformed.
   """
+  # Lazily load the TF-TRT C bindings, so `import tensorflow` doesn't complain
+  # even if it cannot find TensorRT library.
+  trt_ops.load_trt_ops()
+  # pylint: disable=g-import-not-at-top,line-too-long
+  from tensorflow.python.compiler.tensorrt.wrap_conversion import calib_convert
+  # pylint: enable=g-import-not-at-top,line-too-long
 
   is_calib_graph = False
   for n in calibration_graph_def.node:
diff --git a/tensorflow/python/compiler/tensorrt/trt_convert_test.py b/tensorflow/python/compiler/tensorrt/trt_convert_test.py
index 0dbc5c1970..023ac5c36e 100644
--- a/tensorflow/python/compiler/tensorrt/trt_convert_test.py
+++ b/tensorflow/python/compiler/tensorrt/trt_convert_test.py
@@ -20,9 +20,10 @@ from __future__ import print_function
 
 import os
 
-# pylint: disable=unused-import
-from tensorflow.compiler.tf2tensorrt.python.ops import trt_ops
-# pylint: enable=unused-import
+from tensorflow.python.compiler.tensorrt.wrap_conversion import clear_test_values
+from tensorflow.python.compiler.tensorrt.wrap_conversion import enable_test_value
+from tensorflow.python.compiler.tensorrt.wrap_conversion import get_test_value
+from tensorflow.python.compiler.tensorrt.wrap_conversion import is_tensorrt_enabled
 from tensorflow.core.framework import graph_pb2
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import rewriter_config_pb2
@@ -53,6 +54,8 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
 
   def testGetTensorrtRewriterConfig(self):
     """Test case for TrtGraphConverter.get_tensorrt_rewriter_config()."""
+    if not is_tensorrt_enabled():
+      return
     rewriter_cfg = trt_convert.TrtGraphConverter.get_tensorrt_rewriter_config(
         rewriter_config_template=None,
         max_batch_size=128,
@@ -172,7 +175,7 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
 
   def testCreateInferenceGraph_BasicConversion(self):
     """Test case for trt_convert.create_inference_graph()."""
-    if not trt_convert.is_tensorrt_enabled():
+    if not is_tensorrt_enabled():
       return
 
     # Use GraphDef as input.
@@ -187,25 +190,23 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
                                    output_saved_model_dir)
 
   def _TestRun(self, sess, batch_size, expect_engine_is_run):
-    trt_convert.clear_test_values("")
+    clear_test_values("")
     result = sess.run("output:0", feed_dict={"input:0": [[[1.0]]] * batch_size})
     self.assertAllEqual([[[4.0]]] * batch_size, result)
     execute_engine_test_value = ("done" if expect_engine_is_run else "")
     execute_native_segment_test_value = ("" if expect_engine_is_run else "done")
-    self.assertEqual(
-        execute_engine_test_value,
-        trt_convert.get_test_value("TRTEngineOp_0:ExecuteTrtEngine"))
-    self.assertEqual(
-        execute_native_segment_test_value,
-        trt_convert.get_test_value("TRTEngineOp_0:ExecuteNativeSegment"))
+    self.assertEqual(execute_engine_test_value,
+                     get_test_value("TRTEngineOp_0:ExecuteTrtEngine"))
+    self.assertEqual(execute_native_segment_test_value,
+                     get_test_value("TRTEngineOp_0:ExecuteNativeSegment"))
 
   def testCreateInferenceGraph_MinimumSegmentSize(self):
-    if not trt_convert.is_tensorrt_enabled():
+    if not is_tensorrt_enabled():
       return
     output_graph_def = trt_convert.create_inference_graph(
         self._GetGraphDef(), ["output"],
-        minimum_segment_size=5,
         max_workspace_size_bytes=TrtConvertTest._TRT_MAX_WORKSPACE_SIZE_BYTES,
+        minimum_segment_size=5,
         is_dynamic_op=False)
     node_name_to_op = {node.name: node.op for node in output_graph_def.node}
     self.assertEqual({
@@ -218,9 +219,9 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
     }, node_name_to_op)
 
   def testCreateInferenceGraph_DynamicOp(self):
-    if not trt_convert.is_tensorrt_enabled():
+    if not is_tensorrt_enabled():
       return
-    trt_convert.enable_test_value()
+    enable_test_value()
 
     tmp_dir = self.get_temp_dir()
     input_saved_model_dir = os.path.join(tmp_dir, "in_dir2")
@@ -261,9 +262,9 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
         self._TestRun(sess, 3, True)
 
   def testCreateInferenceGraph_StaticOp(self):
-    if not trt_convert.is_tensorrt_enabled():
+    if not is_tensorrt_enabled():
       return
-    trt_convert.enable_test_value()
+    enable_test_value()
 
     tmp_dir = self.get_temp_dir()
     input_saved_model_dir = os.path.join(tmp_dir, "in_dir3")
-- 
GitLab


From 50f0b210ce8a7b47f6fb4ed85229772dbd4d22a4 Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Mon, 11 Feb 2019 16:19:12 -0800
Subject: [PATCH 2288/2345] Create int8 slice.

PiperOrigin-RevId: 233500199
---
 tensorflow/lite/kernels/register.cc          |  3 ++-
 tensorflow/lite/kernels/slice.cc             |  3 +++
 tensorflow/lite/kernels/slice_test.cc        | 22 ++++++++++++++++++++
 tensorflow/lite/toco/tflite/operator.cc      | 17 +++++++++++++--
 tensorflow/lite/toco/tflite/operator_test.cc |  4 ++++
 5 files changed, 46 insertions(+), 3 deletions(-)

diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index 7534cb2ab1..67c4a0bc4f 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -297,7 +297,8 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_SELECT, Register_SELECT(),
              /* min_version */ 1,
              /* max_version */ 2);
-  AddBuiltin(BuiltinOperator_SLICE, Register_SLICE());
+  AddBuiltin(BuiltinOperator_SLICE, Register_SLICE(), /* min_version */ 1,
+             /* max_version */ 2);
   AddBuiltin(BuiltinOperator_SIN, Register_SIN());
   AddBuiltin(BuiltinOperator_TRANSPOSE_CONV, Register_TRANSPOSE_CONV());
   AddBuiltin(BuiltinOperator_TILE, Register_TILE());
diff --git a/tensorflow/lite/kernels/slice.cc b/tensorflow/lite/kernels/slice.cc
index 5fca7a3ea7..650c65d7dc 100644
--- a/tensorflow/lite/kernels/slice.cc
+++ b/tensorflow/lite/kernels/slice.cc
@@ -204,6 +204,9 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
     case kTfLiteInt64:
       TF_LITE_SLICE(int64_t, kernel_type);
       break;
+    case kTfLiteInt8:
+      TF_LITE_SLICE(int8_t, kernel_type);
+      break;
     case kTfLiteUInt8:
       TF_LITE_SLICE(uint8_t, kernel_type);
       break;
diff --git a/tensorflow/lite/kernels/slice_test.cc b/tensorflow/lite/kernels/slice_test.cc
index 563329ddb1..102218ba23 100644
--- a/tensorflow/lite/kernels/slice_test.cc
+++ b/tensorflow/lite/kernels/slice_test.cc
@@ -163,6 +163,28 @@ TEST(SliceOpTest, SizeMinus1) {
   EXPECT_THAT(m.GetOutput(), ElementsAreArray({3, 3, 3, 5, 5, 5}));
 }
 
+TEST(SliceOpTest, SliceUint8) {
+  SliceOpModel<uint8_t, int32_t> m({3, 2, 3, 1}, {4}, {4}, TensorType_INT32,
+                                   TensorType_UINT8);
+  m.SetInput({1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6});
+  m.SetBegin({1, 0, 0, 0});
+  m.SetSize({2, 1, -1, 1});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({2, 1, 3, 1}));
+  EXPECT_THAT(m.GetOutput(), ElementsAreArray({3, 3, 3, 5, 5, 5}));
+}
+
+TEST(SliceOpTest, SliceInt8) {
+  SliceOpModel<int8_t, int32_t> m({3, 2, 3, 1}, {4}, {4}, TensorType_INT32,
+                                  TensorType_INT8);
+  m.SetInput({1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6});
+  m.SetBegin({1, 0, 0, 0});
+  m.SetSize({2, 1, -1, 1});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({2, 1, 3, 1}));
+  EXPECT_THAT(m.GetOutput(), ElementsAreArray({3, 3, 3, 5, 5, 5}));
+}
+
 }  // namespace
 }  // namespace tflite
 
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index 42586ca2d3..34237cd1e3 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -1607,6 +1607,20 @@ class Shape
   }
 };
 
+class Slice : public SimpleOperator<SliceOperator> {
+ public:
+  explicit Slice() : SimpleOperator("SLICE", OperatorType::kSlice) {}
+  int GetVersion(const OperatorSignature& op_signature) const override {
+    const string& input_name = op_signature.op->inputs[0];
+    const Array& input_array = op_signature.model->GetArray(input_name);
+    // Version 2 supports signed int8 input types.
+    if (input_array.data_type == ArrayDataType::kInt8) {
+      return 2;
+    }
+    return 1;
+  }
+};
+
 class OneHot : public BuiltinOperator<OneHotOperator, ::tflite::OneHotOptions,
                                       ::tflite::BuiltinOptions_OneHotOptions> {
  public:
@@ -2330,8 +2344,7 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
   ops.push_back(
       MakeUnique<SimpleOperator<NegOperator>>("NEG", OperatorType::kNeg));
   ops.push_back(MakeUnique<Select>());
-  ops.push_back(
-      MakeUnique<SimpleOperator<SliceOperator>>("SLICE", OperatorType::kSlice));
+  ops.push_back(MakeUnique<Slice>());
   ops.push_back(
       MakeUnique<SimpleOperator<PowOperator>>("POW", OperatorType::kPow));
   ops.push_back(MakeUnique<SimpleOperator<LogicalOrOperator>>(
diff --git a/tensorflow/lite/toco/tflite/operator_test.cc b/tensorflow/lite/toco/tflite/operator_test.cc
index 71e186cee5..bfe8af682c 100644
--- a/tensorflow/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/lite/toco/tflite/operator_test.cc
@@ -812,6 +812,10 @@ TEST_F(OperatorTest, VersioningStridedSliceTest) {
   SimpleVersioningTest<StridedSliceOperator>();
 }
 
+TEST_F(OperatorTest, VersioningSliceTest) {
+  SimpleVersioningTest<SliceOperator>();
+}
+
 TEST_F(OperatorTest, VersioningSelectTest) {
   SelectOperator select_op;
   select_op.inputs = {"input1"};
-- 
GitLab


From 7bb1e7d6c2fd515d70fd4f45c7105d24b3ce9033 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 11 Feb 2019 16:56:09 -0800
Subject: [PATCH 2289/2345] cudnnConvolutionBiasActivationForward only supports
 CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM when the activation is
 CUDNN_ACTIVATION_IDENTITY, but it'll use a diffent algorithm if asked to -
 but then it also does an unwanted Relu.

PiperOrigin-RevId: 233506040
---
 .../kernels/fused_conv2d_bias_activation_op.cc     | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc b/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc
index b6b75ffa24..f13a66717f 100644
--- a/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc
+++ b/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc
@@ -565,6 +565,20 @@ void LaunchFusedConv2DBiasActivationOp<GPUDevice, T, BiasType, ScaleType>::
         fused_conv_parameters.ShouldIncludeWinogradNonfusedAlgo<T>(
             stream->parent()),
         &algorithms));
+    if (activation_mode == ActivationMode::NONE) {
+      // Only CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM is supported for
+      // identity activation, other algs seem to quietly do Relu.
+      // See
+      // https://docs.nvidia.com/deeplearning/sdk/cudnn-developer-guide/index.html#cudnnConvolutionBiasActivationForward
+      algorithms.erase(
+          std::remove_if(
+              algorithms.begin(), algorithms.end(),
+              [](dnn::AlgorithmDesc alg) {
+                return alg.algo_id() !=
+                       CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM;
+              }),
+          algorithms.end());
+    }
     dnn::ProfileResult best_result;
     dnn::ProfileResult best_result_no_scratch;
     for (auto profile_algorithm : algorithms) {
-- 
GitLab


From 289be41a4db4d4971607bef18866b989514656f3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 11 Feb 2019 16:59:52 -0800
Subject: [PATCH 2290/2345] Update ops-related pbtxt files.

PiperOrigin-RevId: 233506599
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 174 ++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 156 ++++++++++++++--
 2 files changed, 317 insertions(+), 13 deletions(-)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 7e5b448f73..90f0a782cd 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -50604,6 +50604,49 @@ op {
     }
   }
 }
+op {
+  name: "RequantizationRangePerChannel"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "input_min"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "input_max"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output_min"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output_max"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_QINT32
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "clip_value_max"
+    type: "float"
+  }
+}
 op {
   name: "Requantize"
   input_arg {
@@ -50726,6 +50769,73 @@ op {
     }
   }
 }
+op {
+  name: "RequantizePerChannel"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "input_min"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "input_max"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "requested_output_min"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "requested_output_max"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "out_type"
+  }
+  output_arg {
+    name: "output_min"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output_max"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_QINT32
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "out_type"
+    type: "type"
+    default_value {
+      type: DT_QUINT8
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+}
 op {
   name: "Reshape"
   input_arg {
@@ -75445,6 +75555,70 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "StatefulStandardNormal"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "shape"
+    type_attr: "shape_dtype"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+  }
+  attr {
+    name: "shape_dtype"
+    type: "type"
+    default_value {
+      type: DT_INT64
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "StatefulStandardNormalV2"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "algorithm"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "shape"
+    type_attr: "shape_dtype"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+  }
+  attr {
+    name: "shape_dtype"
+    type: "type"
+    default_value {
+      type: DT_INT64
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "StatelessIf"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index c0ec6549d4..791520f897 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -26483,6 +26483,49 @@ op {
     }
   }
 }
+op {
+  name: "RequantizationRangePerChannel"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "input_min"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "input_max"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output_min"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output_max"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_QINT32
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "clip_value_max"
+    type: "float"
+  }
+}
 op {
   name: "Requantize"
   input_arg {
@@ -26544,6 +26587,73 @@ op {
     }
   }
 }
+op {
+  name: "RequantizePerChannel"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "input_min"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "input_max"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "requested_output_min"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "requested_output_max"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "out_type"
+  }
+  output_arg {
+    name: "output_min"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output_max"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_QINT32
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+  attr {
+    name: "out_type"
+    type: "type"
+    default_value {
+      type: DT_QUINT8
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_QINT16
+        type: DT_QUINT16
+      }
+    }
+  }
+}
 op {
   name: "Reshape"
   input_arg {
@@ -35863,14 +35973,6 @@ op {
     default_value {
       type: DT_FLOAT
     }
-    allowed_values {
-      list {
-        type: DT_HALF
-        type: DT_BFLOAT16
-        type: DT_FLOAT
-        type: DT_DOUBLE
-      }
-    }
   }
   attr {
     name: "shape_dtype"
@@ -35878,11 +35980,39 @@ op {
     default_value {
       type: DT_INT64
     }
-    allowed_values {
-      list {
-        type: DT_INT32
-        type: DT_INT64
-      }
+  }
+  is_stateful: true
+}
+op {
+  name: "StatefulStandardNormalV2"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "algorithm"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "shape"
+    type_attr: "shape_dtype"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+  }
+  attr {
+    name: "shape_dtype"
+    type: "type"
+    default_value {
+      type: DT_INT64
     }
   }
   is_stateful: true
-- 
GitLab


From b1a5472dc126c92144aa54a6f1b8860078de1a9f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 11 Feb 2019 17:02:01 -0800
Subject: [PATCH 2291/2345] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 233506931

---
 tensorflow/go/op/wrappers.go | 270 ++++++++++++++++++++++++-----------
 1 file changed, 190 insertions(+), 80 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index cc0995fe10..3cf3521264 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -7936,39 +7936,6 @@ func MatrixInverse(scope *Scope, input tf.Output, optional ...MatrixInverseAttr)
 	return op.Output(0)
 }
 
-// Returns x + y element-wise.
-//
-// *NOTE*: `Add` supports broadcasting. `AddN` does not. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func Add(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Add",
-		Input: []tf.Input{
-			x, y,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes the derivative of a Gamma random sample w.r.t. `alpha`.
-func RandomGammaGrad(scope *Scope, alpha tf.Output, sample tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "RandomGammaGrad",
-		Input: []tf.Input{
-			alpha, sample,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Computes square of x element-wise.
 //
 // I.e., \\(y = x * x = x^2\\).
@@ -12943,6 +12910,48 @@ func StatefulStandardNormal(scope *Scope, resource tf.Output, shape tf.Output, o
 	return op.Output(0)
 }
 
+// StatefulStandardNormalV2Attr is an optional argument to StatefulStandardNormalV2.
+type StatefulStandardNormalV2Attr func(optionalAttr)
+
+// StatefulStandardNormalV2Dtype sets the optional dtype attribute to value.
+//
+// value: The type of the output.
+// If not specified, defaults to DT_FLOAT
+func StatefulStandardNormalV2Dtype(value tf.DataType) StatefulStandardNormalV2Attr {
+	return func(m optionalAttr) {
+		m["dtype"] = value
+	}
+}
+
+// Outputs random values from a normal distribution.
+//
+// The generated values will have mean 0 and standard deviation 1.
+//
+// Arguments:
+//	resource: The handle of the resource variable that stores the state of the RNG.
+//	algorithm: The RNG algorithm.
+//	shape: The shape of the output tensor.
+//
+// Returns A tensor of the specified shape filled with random normal values.
+func StatefulStandardNormalV2(scope *Scope, resource tf.Output, algorithm tf.Output, shape tf.Output, optional ...StatefulStandardNormalV2Attr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "StatefulStandardNormalV2",
+		Input: []tf.Input{
+			resource, algorithm, shape,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Locks a mutex resource.  The output is the lock.  So long as the lock tensor
 //
 // is alive, any other request to use `MutexLock` with this mutex will wait.
@@ -13745,6 +13754,53 @@ func SparseReshape(scope *Scope, input_indices tf.Output, input_shape tf.Output,
 	return op.Output(0), op.Output(1)
 }
 
+// Computes the product along segments of a tensor.
+//
+// Read
+// [the section on segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
+// for an explanation of segments.
+//
+// Computes a tensor such that
+// \\(output_i = \prod_j data_j\\) where the product is over `j` such
+// that `segment_ids[j] == i`.
+//
+// If the product is empty for a given segment ID `i`, `output[i] = 1`.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/SegmentProd.png" alt>
+// </div>
+//
+// For example:
+//
+// ```
+// c = tf.constant([[1,2,3,4], [4, 3, 2, 1], [5,6,7,8]])
+// tf.segment_prod(c, tf.constant([0, 0, 1]))
+// # ==> [[4, 6, 6, 4],
+// #      [5, 6, 7, 8]]
+// ```
+//
+//
+// Arguments:
+//
+//	segment_ids: A 1-D tensor whose size is equal to the size of `data`'s
+// first dimension.  Values should be sorted and can be repeated.
+//
+// Returns Has same shape as data, except for dimension 0 which
+// has size `k`, the number of segments.
+func SegmentProd(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SegmentProd",
+		Input: []tf.Input{
+			data, segment_ids,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Bucketizes 'input' based on 'boundaries'.
 //
 // For example, if the inputs are
@@ -15194,6 +15250,39 @@ func Neg(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
+// Returns x + y element-wise.
+//
+// *NOTE*: `Add` supports broadcasting. `AddN` does not. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func Add(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Add",
+		Input: []tf.Input{
+			x, y,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Computes the derivative of a Gamma random sample w.r.t. `alpha`.
+func RandomGammaGrad(scope *Scope, alpha tf.Output, sample tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "RandomGammaGrad",
+		Input: []tf.Input{
+			alpha, sample,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Execute a sub graph on a remote processor.
 //
 // The graph specifications(such as graph itself, input tensors and output names)
@@ -17502,6 +17591,48 @@ func TruncateDiv(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	return op.Output(0)
 }
 
+// RequantizePerChannelAttr is an optional argument to RequantizePerChannel.
+type RequantizePerChannelAttr func(optionalAttr)
+
+// RequantizePerChannelOutType sets the optional out_type attribute to value.
+//
+// value: The quantized type of output tensor that needs to be converted.
+// If not specified, defaults to DT_QUINT8
+func RequantizePerChannelOutType(value tf.DataType) RequantizePerChannelAttr {
+	return func(m optionalAttr) {
+		m["out_type"] = value
+	}
+}
+
+// Requantizes input with min and max values known per channel.
+//
+// Arguments:
+//	input: The original input tensor.
+//	input_min: The minimum value of the input tensor
+//	input_max: The maximum value of the input tensor.
+//	requested_output_min: The minimum value of the output tensor requested.
+//	requested_output_max: The maximum value of the output tensor requested.
+//
+// Returns Output tensor.The minimum value of the final output tensorThe maximum value of the final output tensor.
+func RequantizePerChannel(scope *Scope, input tf.Output, input_min tf.Output, input_max tf.Output, requested_output_min tf.Output, requested_output_max tf.Output, optional ...RequantizePerChannelAttr) (output tf.Output, output_min tf.Output, output_max tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "RequantizePerChannel",
+		Input: []tf.Input{
+			input, input_min, input_max, requested_output_min, requested_output_max,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2)
+}
+
 // Restores tensors from a V2 checkpoint.
 //
 // For backward compatibility with the V1 format, this Op currently allows
@@ -30179,53 +30310,6 @@ func Svd(scope *Scope, input tf.Output, optional ...SvdAttr) (s tf.Output, u tf.
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Computes the product along segments of a tensor.
-//
-// Read
-// [the section on segmentation](https://tensorflow.org/api_docs/python/tf/math#Segmentation)
-// for an explanation of segments.
-//
-// Computes a tensor such that
-// \\(output_i = \prod_j data_j\\) where the product is over `j` such
-// that `segment_ids[j] == i`.
-//
-// If the product is empty for a given segment ID `i`, `output[i] = 1`.
-//
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/SegmentProd.png" alt>
-// </div>
-//
-// For example:
-//
-// ```
-// c = tf.constant([[1,2,3,4], [4, 3, 2, 1], [5,6,7,8]])
-// tf.segment_prod(c, tf.constant([0, 0, 1]))
-// # ==> [[4, 6, 6, 4],
-// #      [5, 6, 7, 8]]
-// ```
-//
-//
-// Arguments:
-//
-//	segment_ids: A 1-D tensor whose size is equal to the size of `data`'s
-// first dimension.  Values should be sorted and can be repeated.
-//
-// Returns Has same shape as data, except for dimension 0 which
-// has size `k`, the number of segments.
-func SegmentProd(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "SegmentProd",
-		Input: []tf.Input{
-			data, segment_ids,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Converts one or more images from RGB to HSV.
 //
 // Outputs a tensor of the same shape as the `images` tensor, containing the HSV
@@ -32017,6 +32101,32 @@ func SampleDistortedBoundingBoxV2(scope *Scope, image_size tf.Output, bounding_b
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
+// Computes requantization range per channel.
+//
+// Arguments:
+//	input: The original input tensor.
+//	input_min: The minimum value of the input tensor
+//	input_max: The maximum value of the input tensor.
+//	clip_value_max: The maximum value of the output that needs to be clipped.
+// Example: set this to 6 for Relu6.
+//
+// Returns The minimum value of the final output tensorThe maximum value of the final output tensor.
+func RequantizationRangePerChannel(scope *Scope, input tf.Output, input_min tf.Output, input_max tf.Output, clip_value_max float32) (output_min tf.Output, output_max tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"clip_value_max": clip_value_max}
+	opspec := tf.OpSpec{
+		Type: "RequantizationRangePerChannel",
+		Input: []tf.Input{
+			input, input_min, input_max,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
 // ExtractGlimpseAttr is an optional argument to ExtractGlimpse.
 type ExtractGlimpseAttr func(optionalAttr)
 
-- 
GitLab


From f668f8031a6a80ef7c4903d28bb59807da2b4594 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Mon, 11 Feb 2019 17:03:35 -0800
Subject: [PATCH 2292/2345] [tf.data] Providing an escape hatch for using
 `tf.data.Dataset.filter` with the legacy `Defun`.

PiperOrigin-RevId: 233507319
---
 tensorflow/python/data/kernel_tests/BUILD     |  21 ++-
 .../python/data/kernel_tests/filter_test.py   | 105 +-------------
 .../data/kernel_tests/filter_test_base.py     | 134 ++++++++++++++++++
 .../filter_with_legacy_function_test.py       |  33 +++++
 .../python/data/kernel_tests/map_test.py      |   1 +
 tensorflow/python/data/ops/dataset_ops.py     |  28 +++-
 .../golden/v1/tensorflow.data.-dataset.pbtxt  |   4 +
 ...ow.data.-fixed-length-record-dataset.pbtxt |   4 +
 .../tensorflow.data.-t-f-record-dataset.pbtxt |   4 +
 .../tensorflow.data.-text-line-dataset.pbtxt  |   4 +
 ...rflow.data.experimental.-csv-dataset.pbtxt |   4 +
 ...ow.data.experimental.-random-dataset.pbtxt |   4 +
 ...rflow.data.experimental.-sql-dataset.pbtxt |   4 +
 tensorflow/tools/pip_package/BUILD            |   1 +
 14 files changed, 247 insertions(+), 104 deletions(-)
 create mode 100644 tensorflow/python/data/kernel_tests/filter_test_base.py
 create mode 100644 tensorflow/python/data/kernel_tests/filter_with_legacy_function_test.py

diff --git a/tensorflow/python/data/kernel_tests/BUILD b/tensorflow/python/data/kernel_tests/BUILD
index b134a8deae..f32c1d3778 100644
--- a/tensorflow/python/data/kernel_tests/BUILD
+++ b/tensorflow/python/data/kernel_tests/BUILD
@@ -108,8 +108,26 @@ tf_py_test(
     size = "small",
     srcs = ["filter_test.py"],
     additional_deps = [
+        ":filter_test_base",
+        "//tensorflow/python:client_testlib",
+    ],
+)
+
+tf_py_test(
+    name = "filter_with_legacy_function_test",
+    size = "small",
+    srcs = ["filter_with_legacy_function_test.py"],
+    additional_deps = [
+        ":filter_test_base",
+        "//tensorflow/python:client_testlib",
+    ],
+)
+
+py_library(
+    name = "filter_test_base",
+    srcs = ["filter_test_base.py"],
+    deps = [
         ":test_base",
-        "//third_party/py/numpy",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:dtypes",
@@ -118,6 +136,7 @@ tf_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python/data/ops:dataset_ops",
+        "//third_party/py/numpy",
     ],
 )
 
diff --git a/tensorflow/python/data/kernel_tests/filter_test.py b/tensorflow/python/data/kernel_tests/filter_test.py
index cd8f526e72..b81e9a892d 100644
--- a/tensorflow/python/data/kernel_tests/filter_test.py
+++ b/tensorflow/python/data/kernel_tests/filter_test.py
@@ -1,4 +1,4 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,111 +17,16 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import numpy as np
-
-from tensorflow.python.data.kernel_tests import test_base
-from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.framework import errors
-from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.data.kernel_tests import filter_test_base
 from tensorflow.python.framework import test_util
-from tensorflow.python.ops import map_fn
-from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class FilterTest(test_base.DatasetTestBase):
-
-  def testFilterDataset(self):
-    components = (
-        np.arange(7, dtype=np.int64),
-        np.array([[1, 2, 3]], dtype=np.int64) * np.arange(
-            7, dtype=np.int64)[:, np.newaxis],
-        np.array(37.0, dtype=np.float64) * np.arange(7)
-    )
-    def _map_fn(x, y, z):
-      return math_ops.square(x), math_ops.square(y), math_ops.square(z)
-
-    def do_test(count, modulus):
-      dataset = dataset_ops.Dataset.from_tensor_slices(components).map(
-          _map_fn).repeat(count).filter(
-              lambda x, _y, _z: math_ops.equal(math_ops.mod(x, modulus), 0))
-      self.assertEqual([c.shape[1:] for c in components],
-                       [shape for shape in dataset.output_shapes])
-      get_next = self.getNext(dataset)
-      for _ in range(count):
-        for i in [x for x in range(7) if x**2 % modulus == 0]:
-          result = self.evaluate(get_next())
-          for component, result_component in zip(components, result):
-            self.assertAllEqual(component[i]**2, result_component)
-      with self.assertRaises(errors.OutOfRangeError):
-        self.evaluate(get_next())
-
-    do_test(14, 2)
-    do_test(4, 18)
-
-    # Test an empty dataset.
-    do_test(0, 1)
-
-  def testFilterRange(self):
-    dataset = dataset_ops.Dataset.range(4).filter(
-        lambda x: math_ops.not_equal(math_ops.mod(x, 3), 2))
-    self.assertDatasetProduces(dataset, expected_output=[0, 1, 3])
-
-  def testFilterDict(self):
-    dataset = dataset_ops.Dataset.range(10).map(
-        lambda x: {"foo": x * 2, "bar": x ** 2}).filter(
-            lambda d: math_ops.equal(d["bar"] % 2, 0)).map(
-                lambda d: d["foo"] + d["bar"])
-    self.assertDatasetProduces(
-        dataset,
-        expected_output=[(i * 2 + i**2) for i in range(10) if not (i**2) % 2])
-
-  def testUseStepContainerInFilter(self):
-    input_data = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.int64)
-
-    # Define a predicate that returns true for the first element of
-    # the sequence and not the second, and uses `tf.map_fn()`.
-    def _predicate(xs):
-      squared_xs = map_fn.map_fn(lambda x: x * x, xs)
-      summed = math_ops.reduce_sum(squared_xs)
-      return math_ops.equal(summed, 1 + 4 + 9)
-
-    dataset = dataset_ops.Dataset.from_tensor_slices(
-        [[1, 2, 3], [4, 5, 6]]).filter(_predicate)
-    self.assertDatasetProduces(dataset, expected_output=[input_data[0]])
-
-  def testSparse(self):
-
-    def _map_fn(i):
-      return sparse_tensor.SparseTensorValue(
-          indices=np.array([[0, 0]]),
-          values=(i * np.array([1])),
-          dense_shape=np.array([1, 1])), i
-
-    def _filter_fn(_, i):
-      return math_ops.equal(i % 2, 0)
-
-    dataset = dataset_ops.Dataset.range(10).map(_map_fn).filter(_filter_fn).map(
-        lambda x, i: x)
-    self.assertDatasetProduces(
-        dataset, expected_output=[_map_fn(i * 2)[0] for i in range(5)])
-
-  def testShortCircuit(self):
-    dataset = dataset_ops.Dataset.zip(
-        (dataset_ops.Dataset.range(10),
-         dataset_ops.Dataset.from_tensors(True).repeat(None)
-        )).filter(lambda x, y: y)
-    self.assertDatasetProduces(
-        dataset, expected_output=[(i, True) for i in range(10)])
+class FilterTest(filter_test_base.FilterTestBase):
 
-  def testParallelFilters(self):
-    dataset = dataset_ops.Dataset.range(10).filter(
-        lambda x: math_ops.equal(x % 2, 0))
-    next_elements = [self.getNext(dataset) for _ in range(10)]
-    self.assertEqual([0 for _ in range(10)],
-                     self.evaluate(
-                         [next_element() for next_element in next_elements]))
+  def apply_filter(self, input_dataset, predicate):
+    return input_dataset.filter(predicate)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/kernel_tests/filter_test_base.py b/tensorflow/python/data/kernel_tests/filter_test_base.py
new file mode 100644
index 0000000000..4c53fa3990
--- /dev/null
+++ b/tensorflow/python/data/kernel_tests/filter_test_base.py
@@ -0,0 +1,134 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for `tf.data.Dataset.filter()`."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.data.kernel_tests import test_base
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import errors
+from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.ops import map_fn
+from tensorflow.python.ops import math_ops
+
+
+class FilterTestBase(test_base.DatasetTestBase):
+  """Base class for FilterDataset tests."""
+
+  def apply_filter(self, input_dataset, predicate):
+    raise NotImplementedError("FilterTestBase._apply_filter")
+
+  def testFilterDataset(self):
+    components = (
+        np.arange(7, dtype=np.int64),
+        np.array([[1, 2, 3]], dtype=np.int64) * np.arange(
+            7, dtype=np.int64)[:, np.newaxis],
+        np.array(37.0, dtype=np.float64) * np.arange(7)
+    )
+    def _map_fn(x, y, z):
+      return math_ops.square(x), math_ops.square(y), math_ops.square(z)
+
+    def do_test(count, modulus):  # pylint: disable=missing-docstring
+      dataset = dataset_ops.Dataset.from_tensor_slices(components).map(
+          _map_fn).repeat(count)
+      # pylint: disable=g-long-lambda
+      dataset = self.apply_filter(
+          dataset, lambda x, _y, _z: math_ops.equal(
+              math_ops.mod(x, modulus), 0))
+      # pylint: enable=g-long-lambda
+      self.assertEqual([c.shape[1:] for c in components],
+                       [shape for shape in dataset.output_shapes])
+      get_next = self.getNext(dataset)
+      for _ in range(count):
+        for i in [x for x in range(7) if x**2 % modulus == 0]:
+          result = self.evaluate(get_next())
+          for component, result_component in zip(components, result):
+            self.assertAllEqual(component[i]**2, result_component)
+      with self.assertRaises(errors.OutOfRangeError):
+        self.evaluate(get_next())
+
+    do_test(14, 2)
+    do_test(4, 18)
+
+    # Test an empty dataset.
+    do_test(0, 1)
+
+  def testFilterRange(self):
+    dataset = dataset_ops.Dataset.range(4)
+    dataset = self.apply_filter(
+        dataset, lambda x: math_ops.not_equal(math_ops.mod(x, 3), 2))
+    self.assertDatasetProduces(dataset, expected_output=[0, 1, 3])
+
+  def testFilterDict(self):
+    dataset = dataset_ops.Dataset.range(10).map(
+        lambda x: {"foo": x * 2, "bar": x ** 2})
+    dataset = self.apply_filter(
+        dataset, lambda d: math_ops.equal(d["bar"] % 2, 0))
+    dataset = dataset.map(lambda d: d["foo"] + d["bar"])
+    self.assertDatasetProduces(
+        dataset,
+        expected_output=[(i * 2 + i**2) for i in range(10) if not (i**2) % 2])
+
+  def testUseStepContainerInFilter(self):
+    input_data = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.int64)
+
+    # Define a predicate that returns true for the first element of
+    # the sequence and not the second, and uses `tf.map_fn()`.
+    def _predicate(xs):
+      squared_xs = map_fn.map_fn(lambda x: x * x, xs)
+      summed = math_ops.reduce_sum(squared_xs)
+      return math_ops.equal(summed, 1 + 4 + 9)
+
+    dataset = dataset_ops.Dataset.from_tensor_slices(
+        [[1, 2, 3], [4, 5, 6]])
+    dataset = self.apply_filter(dataset, _predicate)
+    self.assertDatasetProduces(dataset, expected_output=[input_data[0]])
+
+  def testSparse(self):
+
+    def _map_fn(i):
+      return sparse_tensor.SparseTensorValue(
+          indices=np.array([[0, 0]]),
+          values=(i * np.array([1])),
+          dense_shape=np.array([1, 1])), i
+
+    def _filter_fn(_, i):
+      return math_ops.equal(i % 2, 0)
+
+    dataset = dataset_ops.Dataset.range(10).map(_map_fn)
+    dataset = self.apply_filter(dataset, _filter_fn)
+    dataset = dataset.map(lambda x, i: x)
+    self.assertDatasetProduces(
+        dataset, expected_output=[_map_fn(i * 2)[0] for i in range(5)])
+
+  def testShortCircuit(self):
+    dataset = dataset_ops.Dataset.zip(
+        (dataset_ops.Dataset.range(10),
+         dataset_ops.Dataset.from_tensors(True).repeat(None)
+        ))
+    dataset = self.apply_filter(dataset, lambda x, y: y)
+    self.assertDatasetProduces(
+        dataset, expected_output=[(i, True) for i in range(10)])
+
+  def testParallelFilters(self):
+    dataset = dataset_ops.Dataset.range(10)
+    dataset = self.apply_filter(dataset, lambda x: math_ops.equal(x % 2, 0))
+    next_elements = [self.getNext(dataset) for _ in range(10)]
+    self.assertEqual([0 for _ in range(10)],
+                     self.evaluate(
+                         [next_element() for next_element in next_elements]))
diff --git a/tensorflow/python/data/kernel_tests/filter_with_legacy_function_test.py b/tensorflow/python/data/kernel_tests/filter_with_legacy_function_test.py
new file mode 100644
index 0000000000..9a283fb330
--- /dev/null
+++ b/tensorflow/python/data/kernel_tests/filter_with_legacy_function_test.py
@@ -0,0 +1,33 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for `tf.data.Dataset.filter_with_legacy_function()`."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.data.kernel_tests import filter_test_base
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import test
+
+
+@test_util.run_v1_only
+class FilterWithLegacyFunctionTest(filter_test_base.FilterTestBase):
+
+  def apply_filter(self, input_dataset, predicate):
+    return input_dataset.filter_with_legacy_function(predicate)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/kernel_tests/map_test.py b/tensorflow/python/data/kernel_tests/map_test.py
index 2d421a02d4..533812f591 100644
--- a/tensorflow/python/data/kernel_tests/map_test.py
+++ b/tensorflow/python/data/kernel_tests/map_test.py
@@ -84,6 +84,7 @@ def _make_coordinated_sloppy_dataset(num_elements, num_parallel_calls):
   return dataset, coordination_events
 
 
+# TODO(jsimsa): Add tests for `map_with_legacy_function`.
 @test_util.run_all_in_graph_and_eager_modes
 class MapTest(test_base.DatasetTestBase, parameterized.TestCase):
 
diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index ab7a362c77..4af37d401b 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -1652,7 +1652,7 @@ class DatasetV1(DatasetV2):
 
     NOTE: This is an escape hatch for existing uses of `map` that do not work
     with V2 functions. New uses are strongly discouraged and existing uses
-    should migrate to `map` as this method will not be removed in V2.
+    should migrate to `map` as this method will be removed in V2.
 
     Args:
       map_func: A function mapping a nested structure of tensors (having shapes
@@ -1700,6 +1700,25 @@ class DatasetV1(DatasetV2):
   def filter(self, predicate):
     return DatasetV1Adapter(super(DatasetV1, self).filter(predicate))
 
+  @deprecation.deprecated(None, "Use `tf.data.Dataset.filter()")
+  def filter_with_legacy_function(self, predicate):
+    """Filters this dataset according to `predicate`.
+
+    NOTE: This is an escape hatch for existing uses of `filter` that do not work
+    with V2 functions. New uses are strongly discouraged and existing uses
+    should migrate to `filter` as this method will be removed in V2.
+
+    Args:
+      predicate: A function mapping a nested structure of tensors (having shapes
+        and types defined by `self.output_shapes` and `self.output_types`) to a
+        scalar `tf.bool` tensor.
+
+    Returns:
+      Dataset: The `Dataset` containing the elements of this dataset for which
+          `predicate` is `True`.
+    """
+    return FilterDataset(self, predicate, use_legacy_function=True)
+
   @functools.wraps(DatasetV2.apply)
   def apply(self, transformation_func):
     return DatasetV1Adapter(super(DatasetV1, self).apply(transformation_func))
@@ -3043,11 +3062,14 @@ class ParallelInterleaveDataset(UnaryDataset):
 class FilterDataset(UnaryUnchangedStructureDataset):
   """A `Dataset` that filters its input according to a predicate function."""
 
-  def __init__(self, input_dataset, predicate):
+  def __init__(self, input_dataset, predicate, use_legacy_function=False):
     """See `Dataset.filter()` for details."""
     self._input_dataset = input_dataset
     wrapped_func = StructuredFunctionWrapper(
-        predicate, self._transformation_name(), dataset=input_dataset)
+        predicate,
+        self._transformation_name(),
+        dataset=input_dataset,
+        use_legacy_function=use_legacy_function)
     if not wrapped_func.output_structure.is_compatible_with(
         structure_lib.TensorStructure(dtypes.bool, [])):
       raise ValueError("`predicate` must return a scalar boolean tensor.")
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.-dataset.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.-dataset.pbtxt
index e638d822bc..1c55f81bed 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.-dataset.pbtxt
@@ -39,6 +39,10 @@ tf_class {
     name: "filter"
     argspec: "args=[\'self\', \'predicate\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "filter_with_legacy_function"
+    argspec: "args=[\'self\', \'predicate\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "flat_map"
     argspec: "args=[\'self\', \'map_func\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.-fixed-length-record-dataset.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.-fixed-length-record-dataset.pbtxt
index c0952b6f9a..5488449044 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.-fixed-length-record-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.-fixed-length-record-dataset.pbtxt
@@ -41,6 +41,10 @@ tf_class {
     name: "filter"
     argspec: "args=[\'self\', \'predicate\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "filter_with_legacy_function"
+    argspec: "args=[\'self\', \'predicate\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "flat_map"
     argspec: "args=[\'self\', \'map_func\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.-t-f-record-dataset.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.-t-f-record-dataset.pbtxt
index a509ba5f9a..0927dd01a3 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.-t-f-record-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.-t-f-record-dataset.pbtxt
@@ -41,6 +41,10 @@ tf_class {
     name: "filter"
     argspec: "args=[\'self\', \'predicate\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "filter_with_legacy_function"
+    argspec: "args=[\'self\', \'predicate\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "flat_map"
     argspec: "args=[\'self\', \'map_func\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.-text-line-dataset.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.-text-line-dataset.pbtxt
index c27998e8c9..bab1e39921 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.-text-line-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.-text-line-dataset.pbtxt
@@ -41,6 +41,10 @@ tf_class {
     name: "filter"
     argspec: "args=[\'self\', \'predicate\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "filter_with_legacy_function"
+    argspec: "args=[\'self\', \'predicate\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "flat_map"
     argspec: "args=[\'self\', \'map_func\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-csv-dataset.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-csv-dataset.pbtxt
index 2a899d8d06..68cf02333f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-csv-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-csv-dataset.pbtxt
@@ -41,6 +41,10 @@ tf_class {
     name: "filter"
     argspec: "args=[\'self\', \'predicate\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "filter_with_legacy_function"
+    argspec: "args=[\'self\', \'predicate\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "flat_map"
     argspec: "args=[\'self\', \'map_func\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-random-dataset.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-random-dataset.pbtxt
index 3c4727f959..6d3f88eded 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-random-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-random-dataset.pbtxt
@@ -41,6 +41,10 @@ tf_class {
     name: "filter"
     argspec: "args=[\'self\', \'predicate\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "filter_with_legacy_function"
+    argspec: "args=[\'self\', \'predicate\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "flat_map"
     argspec: "args=[\'self\', \'map_func\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-sql-dataset.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-sql-dataset.pbtxt
index 353c8fc618..bc4943e737 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-sql-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-sql-dataset.pbtxt
@@ -41,6 +41,10 @@ tf_class {
     name: "filter"
     argspec: "args=[\'self\', \'predicate\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "filter_with_legacy_function"
+    argspec: "args=[\'self\', \'predicate\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "flat_map"
     argspec: "args=[\'self\', \'map_func\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index b98a158cd2..88f13a051e 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -80,6 +80,7 @@ COMMON_PIP_DEPS = [
     "//tensorflow/python/data/benchmarks:benchmark_base",
     "//tensorflow/python/data/experimental/kernel_tests/serialization:dataset_serialization_test_base",
     "//tensorflow/python/data/experimental/kernel_tests:stats_dataset_test_base",
+    "//tensorflow/python/data/kernel_tests:filter_test_base",
     "//tensorflow/python/data/kernel_tests:test_base",
     "//tensorflow/python/debug:debug_pip",
     "//tensorflow/python/eager:eager_pip",
-- 
GitLab


From 8e9d15aca34d74f30b1ca8448e0e4dfb141ca779 Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <penporn@google.com>
Date: Mon, 11 Feb 2019 17:40:56 -0800
Subject: [PATCH 2293/2345] Internal change.

PiperOrigin-RevId: 233512892
---
 tensorflow/core/kernels/BUILD | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index b2b76faab8..0ad8ca4de4 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -7009,16 +7009,16 @@ tf_mkl_kernel_library(
             ":pooling_ops",
             ":quantization_utils",
             ":transpose_functor",
+            "//third_party/eigen3",
+            "//third_party/gemmlowp",
+            "@mkl_dnn",
             "//tensorflow/core:array_ops_op_lib",
             "//tensorflow/core:core_cpu",
             "//tensorflow/core:framework",
             "//tensorflow/core:lib",
             "//tensorflow/core:math_ops_op_lib",
             "//tensorflow/core:nn_ops_op_lib",
-            "//third_party/eigen3",
             "//third_party/mkl:intel_binary_blob",
-            "@gemmlowp",
-            "@mkl_dnn",
         ],
     ),
 )
-- 
GitLab


From 722608fe015ad3e5fff13da3e710738b1c1648dd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 11 Feb 2019 17:40:59 -0800
Subject: [PATCH 2294/2345] Automated rollback of commit
 d3f92de5aa5938c9980bf309d6eead618e4c5c18

PiperOrigin-RevId: 233512902
---
 tensorflow/core/BUILD                         |  26 +---
 .../core/platform/default/build_config.bzl    |   6 -
 tensorflow/core/platform/posix/port.cc        | 140 +-----------------
 tensorflow/tools/lib_package/BUILD            |   2 -
 tensorflow/tools/pip_package/BUILD            |   1 -
 5 files changed, 10 insertions(+), 165 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 22800e8c95..ec75808aaa 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -386,24 +386,13 @@ cc_library(
         ":platform_port_hdrs",
         ":platform_port_internal_hdrs",
     ],
-    copts = tf_copts() + [
-        "-Ithird_party/hwloc/hwloc-master/include",
-    ] + select({
-        "//tensorflow:android": [],
-        "//tensorflow:ios": [],
-        "//tensorflow:windows": [],
-        "//tensorflow:darwin": [],
-        "//conditions:default": [
-            "-DTENSORFLOW_USE_NUMA",
-        ],
-    }),
+    copts = tf_copts(),
     visibility = ["//tensorflow/core:__subpackages__"],
     deps = [
         ":lib_platform",
         ":platform_base",
         "//tensorflow/core/platform/default/build_config:port",
         "@com_google_absl//absl/base",
-        "@hwloc",
         "@snappy",
     ],
 )
@@ -2312,17 +2301,7 @@ cc_library(
         tf_additional_proto_srcs(),
     ),
     hdrs = LIB_INTERNAL_PUBLIC_HEADERS,
-    copts = tf_copts() + [
-        "-Ithird_party/hwloc/hwloc-master/include",
-    ] + select({
-        "//tensorflow:android": [],
-        "//tensorflow:ios": [],
-        "//tensorflow:windows": [],
-        "//tensorflow:darwin": [],
-        "//conditions:default": [
-            "-DTENSORFLOW_USE_NUMA",
-        ],
-    }),
+    copts = tf_copts(),
     defines = LIB_INTERNAL_DEFINES,
     deps = tf_additional_lib_deps() + [
         ":lib_hash_crc32c_accelerate_internal",
@@ -2333,7 +2312,6 @@ cc_library(
         "@com_google_absl//absl/strings",
         "//third_party/eigen3",
         "//tensorflow/core/platform/default/build_config:platformlib",
-        "@hwloc//:hwloc",
         "@snappy",
         "@zlib_archive//:zlib",
         "@double_conversion//:double-conversion",
diff --git a/tensorflow/core/platform/default/build_config.bzl b/tensorflow/core/platform/default/build_config.bzl
index 8a1ce18718..f9e3e1dc25 100644
--- a/tensorflow/core/platform/default/build_config.bzl
+++ b/tensorflow/core/platform/default/build_config.bzl
@@ -722,12 +722,6 @@ def tf_additional_gdr_lib_defines():
         "//conditions:default": [],
     })
 
-def tf_additional_numa_lib_defines():
-    return select({
-        "//tensorflow:with_numa_support": ["TENSORFLOW_USE_NUMA"],
-        "//conditions:default": [],
-    })
-
 def tf_py_clif_cc(name, visibility = None, **kwargs):
     pass
 
diff --git a/tensorflow/core/platform/posix/port.cc b/tensorflow/core/platform/posix/port.cc
index 69144ddddd..ea6066ac7b 100644
--- a/tensorflow/core/platform/posix/port.cc
+++ b/tensorflow/core/platform/posix/port.cc
@@ -45,10 +45,6 @@ limitations under the License.
 #include <thread>
 #endif
 
-#if TENSORFLOW_USE_NUMA
-#include "hwloc.h"  // TF:hwloc
-#endif
-
 namespace tensorflow {
 namespace port {
 
@@ -117,94 +113,16 @@ int NumHyperthreadsPerCore() {
   return (ht_per_core > 0) ? ht_per_core : 1;
 }
 
-#ifdef TENSORFLOW_USE_NUMA
-namespace {
-static hwloc_topology_t hwloc_topology_handle;
-
-bool HaveHWLocTopology() {
-  // One time initialization
-  static bool init = []() {
-    if (hwloc_topology_init(&hwloc_topology_handle)) {
-      LOG(ERROR) << "Call to hwloc_topology_init() failed";
-      return false;
-    }
-    if (hwloc_topology_load(hwloc_topology_handle)) {
-      LOG(ERROR) << "Call to hwloc_topology_load() failed";
-      return false;
-    }
-    return true;
-  }();
-  return init;
-}
-
-// Return the first hwloc object of the given type whose os_index
-// matches 'index'.
-hwloc_obj_t GetHWLocTypeIndex(hwloc_obj_type_t tp, int index) {
-  hwloc_obj_t obj = nullptr;
-  if (index >= 0) {
-    while ((obj = hwloc_get_next_obj_by_type(hwloc_topology_handle, tp, obj)) !=
-           nullptr) {
-      if (obj->os_index == index) break;
-    }
-  }
-  return obj;
+bool NUMAEnabled() {
+  // Not yet implemented: coming soon.
+  return false;
 }
-}  // namespace
-#endif  // TENSORFLOW_USE_NUMA
-
-bool NUMAEnabled() { return (NUMANumNodes() > 1); }
 
-int NUMANumNodes() {
-#ifdef TENSORFLOW_USE_NUMA
-  if (HaveHWLocTopology()) {
-    int num_numanodes =
-        hwloc_get_nbobjs_by_type(hwloc_topology_handle, HWLOC_OBJ_NUMANODE);
-    return std::max(1, num_numanodes);
-  } else {
-    return 1;
-  }
-#else
-  return 1;
-#endif  // TENSORFLOW_USE_NUMA
-}
+int NUMANumNodes() { return 1; }
 
-void NUMASetThreadNodeAffinity(int node) {
-#ifdef TENSORFLOW_USE_NUMA
-  if (HaveHWLocTopology()) {
-    // Find the corresponding NUMA node topology object.
-    hwloc_obj_t obj = GetHWLocTypeIndex(HWLOC_OBJ_NUMANODE, node);
-    if (obj) {
-      hwloc_set_cpubind(hwloc_topology_handle, obj->cpuset,
-                        HWLOC_CPUBIND_THREAD | HWLOC_CPUBIND_STRICT);
-    } else {
-      LOG(ERROR) << "Could not find hwloc NUMA node " << node;
-    }
-  }
-#endif  // TENSORFLOW_USE_NUMA
-}
+void NUMASetThreadNodeAffinity(int node) {}
 
-int NUMAGetThreadNodeAffinity() {
-  int node_index = kNUMANoAffinity;
-#ifdef TENSORFLOW_USE_NUMA
-  if (HaveHWLocTopology()) {
-    hwloc_cpuset_t thread_cpuset = hwloc_bitmap_alloc();
-    hwloc_get_cpubind(hwloc_topology_handle, thread_cpuset,
-                      HWLOC_CPUBIND_THREAD);
-    hwloc_obj_t obj = nullptr;
-    // Return the first NUMA node whose cpuset is a (non-proper) superset of
-    // that of the current thread.
-    while ((obj = hwloc_get_next_obj_by_type(
-                hwloc_topology_handle, HWLOC_OBJ_NUMANODE, obj)) != nullptr) {
-      if (hwloc_bitmap_isincluded(thread_cpuset, obj->cpuset)) {
-        node_index = obj->os_index;
-        break;
-      }
-    }
-    hwloc_bitmap_free(thread_cpuset);
-  }
-#endif  // TENSORFLOW_USE_NUMA
-  return node_index;
-}
+int NUMAGetThreadNodeAffinity() { return kNUMANoAffinity; }
 
 void* AlignedMalloc(size_t size, int minimum_alignment) {
 #if defined(__ANDROID__)
@@ -234,54 +152,12 @@ void* Realloc(void* ptr, size_t size) { return realloc(ptr, size); }
 void Free(void* ptr) { free(ptr); }
 
 void* NUMAMalloc(int node, size_t size, int minimum_alignment) {
-#ifdef TENSORFLOW_USE_NUMA
-  if (HaveHWLocTopology()) {
-    hwloc_obj_t numa_node = GetHWLocTypeIndex(HWLOC_OBJ_NUMANODE, node);
-    if (numa_node) {
-      return hwloc_alloc_membind(hwloc_topology_handle, size,
-                                 numa_node->nodeset, HWLOC_MEMBIND_BIND,
-                                 HWLOC_MEMBIND_BYNODESET);
-    } else {
-      LOG(ERROR) << "Failed to find hwloc NUMA node " << node;
-    }
-  }
-#endif  // TENSORFLOW_USE_NUMA
   return AlignedMalloc(size, minimum_alignment);
 }
 
-void NUMAFree(void* ptr, size_t size) {
-#ifdef TENSORFLOW_USE_NUMA
-  if (HaveHWLocTopology()) {
-    hwloc_free(hwloc_topology_handle, ptr, size);
-    return;
-  }
-#endif  // TENSORFLOW_USE_NUMA
-  Free(ptr);
-}
+void NUMAFree(void* ptr, size_t size) { Free(ptr); }
 
-int NUMAGetMemAffinity(const void* addr) {
-  int node = kNUMANoAffinity;
-#ifdef TENSORFLOW_USE_NUMA
-  if (HaveHWLocTopology() && addr) {
-    hwloc_nodeset_t nodeset = hwloc_bitmap_alloc();
-    if (!hwloc_get_area_memlocation(hwloc_topology_handle, addr, 4, nodeset,
-                                    HWLOC_MEMBIND_BYNODESET)) {
-      hwloc_obj_t obj = nullptr;
-      while ((obj = hwloc_get_next_obj_by_type(
-                  hwloc_topology_handle, HWLOC_OBJ_NUMANODE, obj)) != nullptr) {
-        if (hwloc_bitmap_isincluded(nodeset, obj->nodeset)) {
-          node = obj->os_index;
-          break;
-        }
-      }
-      hwloc_bitmap_free(nodeset);
-    } else {
-      LOG(ERROR) << "Failed call to hwloc_get_area_memlocation.";
-    }
-  }
-#endif  // TENSORFLOW_USE_NUMA
-  return node;
-}
+int NUMAGetMemAffinity(const void* addr) { return kNUMANoAffinity; }
 
 void MallocExtension_ReleaseToSystem(std::size_t num_bytes) {
   // No-op.
diff --git a/tensorflow/tools/lib_package/BUILD b/tensorflow/tools/lib_package/BUILD
index 83365d0c5c..86bd510792 100644
--- a/tensorflow/tools/lib_package/BUILD
+++ b/tensorflow/tools/lib_package/BUILD
@@ -126,7 +126,6 @@ genrule(
         "@gemmlowp//:LICENSE",
         "@gif_archive//:COPYING",
         "@highwayhash//:LICENSE",
-        "@hwloc//:LICENSE",
         "@icu//:icu4c/LICENSE",
         "@jpeg//:LICENSE.md",
         "@llvm//:LICENSE.TXT",
@@ -196,7 +195,6 @@ genrule(
         "@gemmlowp//:LICENSE",
         "@gif_archive//:COPYING",
         "@highwayhash//:LICENSE",
-        "@hwloc//:LICENSE",
         "@icu//:icu4j/main/shared/licenses/LICENSE",
         "@jpeg//:LICENSE.md",
         "@llvm//:LICENSE.TXT",
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index 88f13a051e..90dfca2b44 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -171,7 +171,6 @@ filegroup(
         "@gemmlowp//:LICENSE",
         "@gif_archive//:COPYING",
         "@highwayhash//:LICENSE",
-        "@hwloc//:LICENSE",
         "@icu//:icu4c/LICENSE",
         "@jpeg//:LICENSE.md",
         "@keras_applications_archive//:LICENSE",
-- 
GitLab


From eec0de8a86953be755117f8a448cb43f4b70e23e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 11 Feb 2019 17:59:01 -0800
Subject: [PATCH 2295/2345] Double sample size for categorical_op_test (In
 particular because some of the probabilities were small, there were two few
 samples in that bucket, and so it was possible to fail).

PiperOrigin-RevId: 233515414
---
 tensorflow/compiler/tests/categorical_op_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/tests/categorical_op_test.py b/tensorflow/compiler/tests/categorical_op_test.py
index 5d5e486f61..eec69ea7d2 100644
--- a/tensorflow/compiler/tests/categorical_op_test.py
+++ b/tensorflow/compiler/tests/categorical_op_test.py
@@ -119,7 +119,7 @@ class CategoricalTest(xla_test.XLATestCase):
 
   def testSamplingCorrectness(self):
     np.random.seed(1618)  # Make it reproducible.
-    num_samples = 21000
+    num_samples = 40000
 
     rand_probs = np.random.dirichlet([1., 1., 2., 3.])
     rand_probs2 = np.random.dirichlet([1., 4., 5.], size=3)  # batched
-- 
GitLab


From 758134501af89585ca37c4af22a51f73a315a77c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 11 Feb 2019 18:06:31 -0800
Subject: [PATCH 2296/2345] Miscellaneous cleanup, e.g. proto assignment,
 redundant casts, inappropriate include guards

PiperOrigin-RevId: 233516740
---
 .../tf2tensorrt/convert/convert_graph.cc         |  2 +-
 .../tf2tensorrt/convert/convert_nodes.cc         |  6 +++---
 .../kernels/get_serialized_resource_op.cc        |  4 ----
 .../tf2tensorrt/kernels/trt_engine_op.cc         | 16 +++++++++-------
 .../compiler/xla/service/shape_inference_test.cc |  2 +-
 5 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc
index d829895239..d6080c02d4 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc
@@ -195,7 +195,7 @@ tensorflow::Status ConvertCalibGraphToInferGraph(
     const tensorflow::GraphDef& graph_def, tensorflow::GraphDef* infer_graph,
     bool is_dyn_op) {
   LOG(INFO) << "Starting Calib Conversion";
-  infer_graph->CopyFrom(graph_def);
+  *infer_graph = graph_def;
   auto trt_rm = TRTResourceManager::instance();
   auto calib_rm = trt_rm->getManager("TRTCalibration");
   int num_nodes = infer_graph->node_size();
diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
index ce8a69b0d3..0d5b9851f7 100644
--- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
@@ -920,7 +920,7 @@ Status Converter::ConvertNode(const NodeDef& node_def) {
   for (size_t i = 0; i < outputs.size(); ++i) {
     TRT_TensorOrWeights& output = outputs[i];
     string output_name = node_def.name();
-    if (i != 0) output_name = StrCat(output_name, ":", i);
+    if (i != 0) absl::StrAppend(&output_name, ":", i);
     // We need to check the name before setting it. If the input is one of the
     // engine input, setting the name here will overwrite engine input
     // bindings which will cause runtime error.
@@ -3328,7 +3328,7 @@ tensorflow::Status ConvertFusedBatchNorm(OpConverterParams* params) {
   TRT_ShapedWeights dummy_power_weights(parameter_type);
   size_t nweight = 0;
   for (int i = 1; i < 5; i++) {
-    nweight = std::max(nweight, (size_t)inputs.at(i).weights().count());
+    nweight = std::max<size_t>(nweight, inputs.at(i).weights().count());
   }
   TRT_ShapedWeights* ptr_shape_weights = nullptr;
   for (int i = 1; i < 5; i++) {
@@ -3898,7 +3898,7 @@ tensorflow::Status ConvertSegmentToGraphDef(
     local_scope = GetCommonNameScope(local_scope, node->name());
     old_to_new_id_map[node->id()] = segment_def->node_size();
     auto snode = segment_def->add_node();
-    snode->CopyFrom(node->def());
+    *snode = node->def();
     VLOG(2) << "Copying " << snode->name() << " to subgraph";
   }
   // Update the inputs of the new input nodes to point to placeholder nodes.
diff --git a/tensorflow/compiler/tf2tensorrt/kernels/get_serialized_resource_op.cc b/tensorflow/compiler/tf2tensorrt/kernels/get_serialized_resource_op.cc
index eae1f8e752..81406b6e30 100644
--- a/tensorflow/compiler/tf2tensorrt/kernels/get_serialized_resource_op.cc
+++ b/tensorflow/compiler/tf2tensorrt/kernels/get_serialized_resource_op.cc
@@ -13,9 +13,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_KERNELS_GET_SERIALIZED_RESOURCE_OP_H_
-#define TENSORFLOW_COMPILER_TF2TENSORRT_KERNELS_GET_SERIALIZED_RESOURCE_OP_H_
-
 #include <memory>
 #include <vector>
 
@@ -70,4 +67,3 @@ REGISTER_KERNEL_BUILDER(Name("GetSerializedResourceOp").Device(DEVICE_GPU),
 
 #endif  // GOOGLE_TENSORRT
 #endif  // GOOGLE_CUDA
-#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_KERNELS_GET_SERIALIZED_RESOURCE_OP_H_
diff --git a/tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.cc b/tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.cc
index 746f091891..e3b31d736e 100644
--- a/tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.cc
@@ -216,8 +216,8 @@ TRTEngineOp::TRTEngineOp(OpKernelConstruction* context)
                  context->GetAttr("use_calibration", &use_calibration_));
   calibration_mode_ =
       (use_calibration_ && precision_mode_ == TrtPrecisionMode::INT8 &&
-       calibration_data.size() == 0);
-  if (calibration_data.size()) {
+       calibration_data.empty());
+  if (!calibration_data.empty()) {
     calibrator_.reset(new TRTInt8Calibrator(calibration_data));
     calibration_data.resize(0);
   }
@@ -435,7 +435,8 @@ bool TRTEngineOp::ExecuteTrtEngine(OpKernelContext* ctx,
     auto dtype = cuda_engine->getBindingDataType(binding_index);
     switch (dtype) {
       case nvinfer1::DataType::kFLOAT:
-        buffers[binding_index] = (void*)(input_tensor.flat<float>().data());
+        buffers[binding_index] =
+            const_cast<float*>(input_tensor.flat<float>().data());
         break;
       case nvinfer1::DataType::kHALF:
         LOG(ERROR) << "FP16 inputs are not supported yet!";
@@ -444,10 +445,11 @@ bool TRTEngineOp::ExecuteTrtEngine(OpKernelContext* ctx,
         LOG(ERROR) << "INT8 inputs are not supported yet!";
         return kRetry;
       case nvinfer1::DataType::kINT32:
-        buffers[binding_index] = (void*)(input_tensor.flat<int32>().data());
+        buffers[binding_index] =
+            const_cast<int32*>(input_tensor.flat<int32>().data());
         break;
       default:
-        LOG(ERROR) << "Unknown TRT data type: " << int(dtype);
+        LOG(ERROR) << "Unknown TRT data type: " << static_cast<int>(dtype);
         return kRetry;
     }
   }
@@ -486,7 +488,7 @@ bool TRTEngineOp::ExecuteTrtEngine(OpKernelContext* ctx,
     switch (dtype) {
       case nvinfer1::DataType::kFLOAT:
         buffers[binding_index] =
-            reinterpret_cast<void*>(output_tensor->flat<float>().data());
+            const_cast<float*>(output_tensor->flat<float>().data());
         break;
       case nvinfer1::DataType::kHALF:
         LOG(WARNING) << "half size is not supported yet!";
@@ -496,7 +498,7 @@ bool TRTEngineOp::ExecuteTrtEngine(OpKernelContext* ctx,
         return kRetry;
       case nvinfer1::DataType::kINT32:
         buffers[binding_index] =
-            reinterpret_cast<void*>(output_tensor->flat<int32>().data());
+            const_cast<int32*>(output_tensor->flat<int32>().data());
         break;
       default:
         LOG(WARNING) << "Unknown TRT data type: " << static_cast<int>(dtype);
diff --git a/tensorflow/compiler/xla/service/shape_inference_test.cc b/tensorflow/compiler/xla/service/shape_inference_test.cc
index eabc22307a..77fcf4d27c 100644
--- a/tensorflow/compiler/xla/service/shape_inference_test.cc
+++ b/tensorflow/compiler/xla/service/shape_inference_test.cc
@@ -252,7 +252,7 @@ TEST_F(ShapeInferenceTest, ClampBadShapes) {
 
 TEST_F(ShapeInferenceTest, Complex) {
   auto complex_shape = [&](const Shape& lhs, const Shape& rhs,
-                           const absl::Span<const int64>& bcast) {
+                           absl::Span<const int64> bcast) {
     return ShapeInference::InferBinaryOpShape(HloOpcode::kComplex, lhs, rhs,
                                               bcast);
   };
-- 
GitLab


From 320eec98eedd63d1d3082ab1d3133b8249abd366 Mon Sep 17 00:00:00 2001
From: Igor Saprykin <isaprykin@google.com>
Date: Mon, 11 Feb 2019 18:07:45 -0800
Subject: [PATCH 2297/2345] Export OneDeviceStrategy, CrossDeviceOps and some
 of its descendants.

Exporting makes it possible to use these symbols via importing them via the tensorflow.* package.

The list of new exported symbols:
 - OneDeviceStrategy is a distributed strategy for running on a single device.

 - CrossDeviceOps is an argument of MirroredStrategy.__init__ called `cross_device_ops`.  Users would specify that argument to manually pick the best implementation of all reduce for their distributed environment.  Users may also define their own implementations of all reduce in new descendants of CrossDeviceOps to better utilize new distributed environments.

 - ReductionToOneDevice - an implementation of CrossDeviceOps, where reduction is done to one device and then broadcasted to other devices.  One example of when users might choose that one is to reduce to one CPU and broadcast to multiple GPUs within a single machine.

 - NcclAllReduce - an implementation that uses NCCL all-reduce implementation.  Users might choose it for training one a single machine with multiple Nvidia GPUs.

 - HierarchicalCopyAllReduce - an implementation that utilizes the particular topology of machines like Nvidia DGX-1 and is particularly suited for them.

PiperOrigin-RevId: 233516903
---
 .../python/cross_device_ops_test.py           |  27 +--
 tensorflow/python/distribute/BUILD            |   3 +
 tensorflow/python/distribute/__init__.py      |   2 +
 .../python/distribute/cross_device_ops.py     | 171 ++++++++++++++----
 .../python/distribute/one_device_strategy.py  |   2 +
 .../distribute/parameter_server_strategy.py   |   3 +-
 ...sorflow.distribute.-cross-device-ops.pbtxt |  33 ++++
 ...ribute.-hierarchical-copy-all-reduce.pbtxt |  35 ++++
 ...nsorflow.distribute.-nccl-all-reduce.pbtxt |  35 ++++
 ...flow.distribute.-one-device-strategy.pbtxt |  66 +++++++
 ....distribute.-reduction-to-one-device.pbtxt |  34 ++++
 .../api/golden/v1/tensorflow.distribute.pbtxt |  20 ++
 ...sorflow.distribute.-cross-device-ops.pbtxt |  33 ++++
 ...ribute.-hierarchical-copy-all-reduce.pbtxt |  35 ++++
 ...nsorflow.distribute.-nccl-all-reduce.pbtxt |  35 ++++
 ...flow.distribute.-one-device-strategy.pbtxt |  66 +++++++
 ....distribute.-reduction-to-one-device.pbtxt |  34 ++++
 .../api/golden/v2/tensorflow.distribute.pbtxt |  20 ++
 18 files changed, 602 insertions(+), 52 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.distribute.-cross-device-ops.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.distribute.-hierarchical-copy-all-reduce.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.distribute.-nccl-all-reduce.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.distribute.-one-device-strategy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.distribute.-reduction-to-one-device.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.distribute.-cross-device-ops.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.distribute.-hierarchical-copy-all-reduce.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.distribute.-nccl-all-reduce.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.distribute.-one-device-strategy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.distribute.-reduction-to-one-device.pbtxt

diff --git a/tensorflow/contrib/distribute/python/cross_device_ops_test.py b/tensorflow/contrib/distribute/python/cross_device_ops_test.py
index 54cce29883..2d56c25f10 100644
--- a/tensorflow/contrib/distribute/python/cross_device_ops_test.py
+++ b/tensorflow/contrib/distribute/python/cross_device_ops_test.py
@@ -204,15 +204,15 @@ class SingleWorkerCrossDeviceOpsTest(CrossDeviceOpsTestBase):
   reduction_to_one_combinations = combinations.combine(
       cross_device_ops=[
           combinations.NamedObject(
-              "DefaultReductionToOneDeviceCrossDeviceOps",
-              cross_device_ops_lib.ReductionToOneDeviceCrossDeviceOps()),
+              "DefaultReductionToOneDevice",
+              cross_device_ops_lib.ReductionToOneDevice()),
           combinations.NamedObject(
               "ReductionToCPUDeviceCrossDeviceOps",
-              cross_device_ops_lib.ReductionToOneDeviceCrossDeviceOps(
+              cross_device_ops_lib.ReductionToOneDevice(
                   reduce_to_device=_cpu_device)),
           combinations.NamedObject(
               "AccumulateNCrossDeviceOp",
-              cross_device_ops_lib.ReductionToOneDeviceCrossDeviceOps(
+              cross_device_ops_lib.ReductionToOneDevice(
                   accumulation_fn=math_ops.accumulate_n)),
       ],
       distribution=[
@@ -228,20 +228,23 @@ class SingleWorkerCrossDeviceOpsTest(CrossDeviceOpsTestBase):
           combinations.NamedObject(
               "AllReduce",
               cross_device_ops_lib.AllReduceCrossDeviceOps("nccl", 1, 0, 0)),
-          combinations.NamedObject(
-              "HierarchicalCopy",
-              cross_device_ops_lib.AllReduceCrossDeviceOps(
-                  "hierarchical_copy", 8, 0, 0)),
           combinations.NamedObject(
               "AllReduceNoGradientRepacking",
               cross_device_ops_lib.AllReduceCrossDeviceOps("nccl", 0, 0, 0)),
+          combinations.NamedObject("NcclAllReduce",
+                                   cross_device_ops_lib.NcclAllReduce()),
+          combinations.NamedObject(
+              "HierarchicalCopy",
+              cross_device_ops_lib.HierarchicalCopyAllReduce(8)),
           combinations.NamedObject(
               "HierarchicalCopyAggregateSmallTensors",
               cross_device_ops_lib.AllReduceCrossDeviceOps(
                   "hierarchical_copy", 0, 100, 10))
       ],
-      distribution=[combinations.mirrored_strategy_with_two_gpus,
-                    combinations.core_mirrored_strategy_with_two_gpus],
+      distribution=[
+          combinations.mirrored_strategy_with_two_gpus,
+          combinations.core_mirrored_strategy_with_two_gpus
+      ],
       mode=["graph", "eager"])
 
   @combinations.generate(reduction_to_one_combinations + allreduce_combinations)
@@ -306,8 +309,8 @@ class SingleWorkerCrossDeviceOpsTest(CrossDeviceOpsTestBase):
       combinations.combine(
           cross_device_ops_instance=[
               combinations.NamedObject(
-                  "ReductionToOneDeviceCrossDeviceOps",
-                  cross_device_ops_lib.ReductionToOneDeviceCrossDeviceOps()),
+                  "ReductionToOneDevice",
+                  cross_device_ops_lib.ReductionToOneDevice()),
               combinations.NamedObject(
                   "AllReduceCrossDeviceOps",
                   cross_device_ops_lib.AllReduceCrossDeviceOps())
diff --git a/tensorflow/python/distribute/BUILD b/tensorflow/python/distribute/BUILD
index 9bcdcce302..7a29d852af 100644
--- a/tensorflow/python/distribute/BUILD
+++ b/tensorflow/python/distribute/BUILD
@@ -60,6 +60,7 @@ py_library(
         "//tensorflow/python:platform",
         "//tensorflow/python:resource_variable_ops",
         "//tensorflow/python/eager:context",
+        "//tensorflow/tools/docs:doc_controls",
         "@six_archive//:six",
     ],
 )
@@ -110,8 +111,10 @@ py_library(
     ],
     srcs_version = "PY2AND3",
     deps = [
+        ":cross_device_ops",
         ":distribute_lib",
         ":mirrored_strategy",
+        ":one_device_strategy",
     ],
 )
 
diff --git a/tensorflow/python/distribute/__init__.py b/tensorflow/python/distribute/__init__.py
index 54e539ecdd..87f746f902 100644
--- a/tensorflow/python/distribute/__init__.py
+++ b/tensorflow/python/distribute/__init__.py
@@ -20,7 +20,9 @@ from __future__ import print_function
 
 # pylint: disable=unused-import
 from tensorflow.python.distribute import cluster_resolver
+from tensorflow.python.distribute import cross_device_ops
 from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.distribute import mirrored_strategy
+from tensorflow.python.distribute import one_device_strategy
 # pylint: enable=unused-import
diff --git a/tensorflow/python/distribute/cross_device_ops.py b/tensorflow/python/distribute/cross_device_ops.py
index 9729302c6d..8b572d2585 100644
--- a/tensorflow/python/distribute/cross_device_ops.py
+++ b/tensorflow/python/distribute/cross_device_ops.py
@@ -32,6 +32,8 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.util.tf_export import tf_export
+from tensorflow.tools.docs import doc_controls
 
 
 def check_destinations(destinations):
@@ -215,6 +217,7 @@ def _simple_reduce(per_replica_value, reduce_to_device, accumulation_fn,
   return reduced
 
 
+@tf_export("distribute.CrossDeviceOps")
 class CrossDeviceOps(object):
   """Base class for cross-device reduction and broadcasting algorithms."""
 
@@ -244,7 +247,8 @@ class CrossDeviceOps(object):
       per_replica_value = _make_tensor_into_per_replica(per_replica_value)
 
     validate_destinations(destinations)
-    return self._reduce(reduce_op, per_replica_value, destinations)
+    return self.reduce_implementation(reduce_op, per_replica_value,
+                                      destinations)
 
   def batch_reduce(self, reduce_op, value_destination_pairs):
     """Reduce PerReplica objects in a batch.
@@ -274,7 +278,7 @@ class CrossDeviceOps(object):
     for _, d in value_destination_pairs:
       validate_destinations(d)
 
-    return self._batch_reduce(reduce_op, value_destination_pairs)
+    return self.batch_reduce_implementation(reduce_op, value_destination_pairs)
 
   def broadcast(self, tensor, destinations):
     """Broadcast the `tensor` to destinations.
@@ -287,39 +291,89 @@ class CrossDeviceOps(object):
       a Mirrored object.
     """
     validate_destinations(destinations)
-    return self._broadcast(tensor, destinations)
+    return self.broadcast_implementation(tensor, destinations)
 
-  def _reduce(self, reduce_op, per_replica_value, destinations):
+  @doc_controls.for_subclass_implementers
+  def reduce_implementation(self, reduce_op, per_replica_value, destinations):
+    """The implementation of reduce of `per_replica_value` to `destinations`.
+
+    It runs the reduction operation defined by `reduce_op` and put the
+    result on `destinations`.
+
+    Args:
+      reduce_op: Indicates how per_replica_value will be reduced. Accepted
+        values are `tf.distribute.ReduceOp.SUM`, `tf.distribute.ReduceOp.MEAN`.
+      per_replica_value: a PerReplica object or a tensor with device set.
+      destinations: the reduction destinations.
+
+    Returns:
+      a Mirrored object.
+
+    Raises:
+      ValueError: if per_replica_value can't be converted to a PerReplica
+        object.
+    """
     raise NotImplementedError(
         "_reduce method must be implemented in descendants.")
 
-  def _batch_reduce(self, reduce_op, value_destination_pairs):
+  @doc_controls.for_subclass_implementers
+  def batch_reduce_implementation(self, reduce_op, value_destination_pairs):
+    """Implementation of reduce PerReplica objects in a batch.
+
+    Reduce each first element in `value_destination_pairs` to each second
+    element which indicates the destinations.
+
+    Args:
+      reduce_op: Indicates how per_replica_value will be reduced. Accepted
+        values are `tf.distribute.ReduceOp.SUM`, `tf.distribute.ReduceOp.MEAN`.
+      value_destination_pairs: a list or a tuple of tuples of PerReplica objects
+        (or tensors with device set if there is one device) and destinations.
+
+    Returns:
+      a list of Mirrored objects.
+
+    Raises:
+      ValueError: if `value_destination_pairs` is not a list or a tuple of
+        tuples of PerReplica objects and destinations
+    """
     raise NotImplementedError(
         "_batch_reduce method must be implemented in descendants.")
 
-  def _broadcast(self, tensor, destinations):
+  @doc_controls.for_subclass_implementers
+  def broadcast_implementation(self, tensor, destinations):
+    """Implementation of broadcast the `tensor` to destinations.
+
+    Args:
+      tensor: the tensor to broadcast.
+      destinations: the broadcast destinations.
+
+    Returns:
+      a Mirrored object.
+    """
     return simple_broadcast(tensor, destinations, always_mirrored=True)
 
 
-class ReductionToOneDeviceCrossDeviceOps(CrossDeviceOps):
+@tf_export("distribute.ReductionToOneDevice")
+class ReductionToOneDevice(CrossDeviceOps):
   """Always do reduction to one device first and then do broadcasting.
 
     Batch reduction is done by reduction on each element one by one.
   """
 
-  def __init__(self, reduce_to_device=None, accumulation_fn=math_ops.add_n):
+  def __init__(self, reduce_to_device=None, accumulation_fn=None):
     """Constructor.
 
     Args:
       reduce_to_device: the intermediate device to reduce to. If None, reduce
         to the first device in `destinations` of the reduce() method.
-      accumulation_fn: a function that does accumulation.
+      accumulation_fn: a function that does accumulation.  If None, then
+        `tf.math.add_n` is used.
     """
     self.reduce_to_device = reduce_to_device
-    self.accumulation_fn = accumulation_fn
-    super(ReductionToOneDeviceCrossDeviceOps, self).__init__()
+    self.accumulation_fn = accumulation_fn or math_ops.add_n
+    super(ReductionToOneDevice, self).__init__()
 
-  def _reduce(self, reduce_op, per_replica_value, destinations):
+  def reduce_implementation(self, reduce_op, per_replica_value, destinations):
     assert check_destinations(destinations)
     devices = get_devices_from(destinations)
     reduce_to_device = self.reduce_to_device or devices[0]
@@ -330,9 +384,9 @@ class ReductionToOneDeviceCrossDeviceOps(CrossDeviceOps):
                              self.accumulation_fn, reduce_op)
     return self.broadcast(reduced, destinations)
 
-  def _batch_reduce(self, reduce_op, value_destination_pairs):
+  def batch_reduce_implementation(self, reduce_op, value_destination_pairs):
     return [
-        self._reduce(reduce_op, t, destinations=v)
+        self.reduce_implementation(reduce_op, t, destinations=v)
         for t, v in value_destination_pairs
     ]
 
@@ -399,11 +453,11 @@ def _ungroup_and_make_mirrored(grouped_reduced,
   return [value_lib.Mirrored(device_map, v, logical_device) for v in index]
 
 
-class ConcatAndSplitPacker(object):
+class _ConcatAndSplitPacker(object):
   """Concatenate and split tensors for reduction."""
 
   def __init__(self, num_packs=1):
-    """Initialize the ConcatAndSplitPacker object.
+    """Initialize the _ConcatAndSplitPacker object.
 
     Args:
       num_packs: specifies the number of split packs that will be
@@ -503,13 +557,13 @@ class ConcatAndSplitPacker(object):
     return aggregated_device_grads
 
 
-class AggregateSmallTensorPacker(object):
+class _AggregateSmallTensorPacker(object):
   """Concatenate small gradient tensors together for reduction."""
 
   def __init__(self,
                agg_small_grads_max_bytes=1048576,
                agg_small_grads_max_group=16):
-    """Initialize the AggregateSmallTensorPacker object.
+    """Initialize the _AggregateSmallTensorPacker object.
 
     Args:
       agg_small_grads_max_bytes: largest tensor eligible for aggregation,
@@ -549,11 +603,11 @@ def _pack_tensors(device_grads,
                   agg_small_grads_max_group=0):
   """Pack tensors if specified."""
   if num_packs > 0:
-    tensor_packer = ConcatAndSplitPacker(num_packs)
+    tensor_packer = _ConcatAndSplitPacker(num_packs)
     device_grad_packs = tensor_packer.pack(device_grads)
   elif agg_small_grads_max_bytes > 0 and agg_small_grads_max_group > 0:
-    tensor_packer = AggregateSmallTensorPacker(agg_small_grads_max_bytes,
-                                               agg_small_grads_max_group)
+    tensor_packer = _AggregateSmallTensorPacker(agg_small_grads_max_bytes,
+                                                agg_small_grads_max_group)
     device_grad_packs = tensor_packer.pack(device_grads)
   else:
     tensor_packer = None
@@ -569,7 +623,7 @@ def _unpack_tensors(reduced, tensor_packer=None):
 
 
 class AllReduceCrossDeviceOps(CrossDeviceOps):
-  """Reduction using all reduce."""
+  """Reduction using all-reduce."""
 
   def __init__(self,
                all_reduce_alg="nccl",
@@ -594,7 +648,6 @@ class AllReduceCrossDeviceOps(CrossDeviceOps):
       num_packs: see above.
       agg_small_grads_max_bytes: see above.
       agg_small_grads_max_group: see above.
-        tensors.
     """
     self._all_reduce_alg = all_reduce_alg
     self._num_packs = num_packs
@@ -602,7 +655,7 @@ class AllReduceCrossDeviceOps(CrossDeviceOps):
     self._agg_small_grads_max_group = agg_small_grads_max_group
     super(AllReduceCrossDeviceOps, self).__init__()
 
-  def _reduce(self, reduce_op, per_replica_value, destinations):
+  def reduce_implementation(self, reduce_op, per_replica_value, destinations):
     contains_indexed_slices = cross_device_utils.contains_indexed_slices(
         per_replica_value)
     if (_devices_match(per_replica_value, destinations)
@@ -624,7 +677,7 @@ class AllReduceCrossDeviceOps(CrossDeviceOps):
                                math_ops.add_n, reduce_op)
       return self.broadcast(reduced, destinations)
 
-  def _batch_reduce(self, reduce_op, value_destination_pairs):
+  def batch_reduce_implementation(self, reduce_op, value_destination_pairs):
     all_devices_match = _all_devices_match(value_destination_pairs)
     contains_indexed_slices = cross_device_utils.contains_indexed_slices(
         value_destination_pairs)
@@ -640,12 +693,12 @@ class AllReduceCrossDeviceOps(CrossDeviceOps):
                             10)
 
       return [
-          self._reduce(reduce_op, t, destinations=v)
+          self.reduce_implementation(reduce_op, t, destinations=v)
           for t, v in value_destination_pairs
       ]
 
   def _batch_all_reduce(self, reduce_op, per_replica_values):
-    """All reduce algorithm in a batch."""
+    """All-reduce algorithm in a batch."""
     logging.log_first_n(
         logging.INFO, "batch_all_reduce invoked for batches size = %d with "
         "algorithm = %s, num_packs = %d, agg_small_grads_max_bytes = %d and "
@@ -685,6 +738,49 @@ AllReduceSpecTuple = collections.namedtuple("AllReduceSpecTuple",
                                             "alg shards limit")
 
 
+@tf_export("distribute.NcclAllReduce")
+class NcclAllReduce(AllReduceCrossDeviceOps):
+  """Reduction using NCCL all-reduce."""
+
+  def __init__(self, num_packs=1):
+    """NCCL all-reduce implementation of CrossDeviceOps.
+
+    Before performing all-reduce, tensors will be repacked or aggregated for
+    more efficient cross-device transportation.
+
+    Args:
+      num_packs: values will be packed in this many splits.  `num_packs` should
+        be greater than 0.
+    """
+    assert num_packs > 0, (
+        "NCLL all-reduce requires num_packs > 0, but {} is specified".format(
+            num_packs))
+    super(NcclAllReduce, self).__init__(
+        all_reduce_alg="nccl", num_packs=num_packs)
+
+
+@tf_export("distribute.HierarchicalCopyAllReduce")
+class HierarchicalCopyAllReduce(AllReduceCrossDeviceOps):
+  """Reduction using hierarchical copy all-reduce.
+
+  This is a good reduction for configurations like Nvidia DGX-1.
+  """
+
+  def __init__(self, num_packs=1):
+    """Hierarchical copy all-reduce implementation of CrossDeviceOps.
+
+    Before performing all-reduce, tensors will be repacked or aggregated for
+    more efficient cross-device transportation.
+
+    Args:
+      num_packs: values will be packed in this many splits.  `num_packs` should
+        be greater than 0.
+    """
+    super(HierarchicalCopyAllReduce, self).__init__(
+        all_reduce_alg="hierarchical_copy",
+        num_packs=num_packs)
+
+
 class MultiWorkerAllReduce(AllReduceCrossDeviceOps):
   """All-reduce algorithms for distributed TensorFlow."""
 
@@ -758,7 +854,7 @@ class MultiWorkerAllReduce(AllReduceCrossDeviceOps):
       ]
 
   def _batch_all_reduce(self, reduce_op, per_replica_values):
-    """All reduce algorithm in a batch."""
+    """All-reduce algorithm in a batch."""
     logging.log_first_n(
         logging.INFO,
         "distributed batch_all_reduce invoked for batches size = %d with "
@@ -769,7 +865,7 @@ class MultiWorkerAllReduce(AllReduceCrossDeviceOps):
 
     device_grads = _group_value_by_device(per_replica_values)
 
-    # The all reduce library requires fully defined shapes.
+    # The all-reduce library requires fully defined shapes.
     # TODO(yuefengz): when tensor sharding is not needed, static shapes are not
     # required as well.
     for device_grad in device_grads:
@@ -838,7 +934,7 @@ class CollectiveAllReduce(CrossDeviceOps):
     super(CollectiveAllReduce, self).__init__()
 
   # TODO(yuefengz, tucker): is indexed slices supported by collective ops?
-  def _reduce(self, reduce_op, per_replica_value, destinations):
+  def reduce_implementation(self, reduce_op, per_replica_value, destinations):
     if cross_device_utils.contains_indexed_slices(per_replica_value):
       raise ValueError(
           "`IndexSlices` is not supported for Collective All-Reduce.")
@@ -861,7 +957,7 @@ class CollectiveAllReduce(CrossDeviceOps):
 
     return value_lib.Mirrored(device_map, index, logical_device)
 
-  def _batch_reduce(self, reduce_op, value_destination_pairs):
+  def batch_reduce_implementation(self, reduce_op, value_destination_pairs):
     if cross_device_utils.contains_indexed_slices(value_destination_pairs):
       raise ValueError(
           "`IndexSlices` is not supported for Collective All-Reduce.")
@@ -877,7 +973,7 @@ class CollectiveAllReduce(CrossDeviceOps):
             "destinations are different.", 10)
 
       return [
-          self._reduce(reduce_op, t, destinations=v)
+          self.reduce_implementation(reduce_op, t, destinations=v)
           for t, v in value_destination_pairs
       ]
 
@@ -930,7 +1026,7 @@ _dgx1_links = [[1, 2, 3, 4], [0, 2, 3, 5], [0, 1, 3, 6], [0, 1, 2, 7],
 def _has_dgx1_like_links(gpu_links):
   if not gpu_links:
     return False
-  # TODO(yuefengz): figure out the right topology for hierarchial copy if
+  # TODO(yuefengz): figure out the right topology for hierarchical copy if
   # number of gpus are less than 8.
   if len(gpu_links) < 8:
     return False
@@ -943,10 +1039,9 @@ def _has_dgx1_like_links(gpu_links):
 
 def _choose_all_reduce_algorithm(device_links):
   if _has_dgx1_like_links(device_links):
-    return AllReduceCrossDeviceOps(
-        "hierarchical_copy", num_packs=len(device_links))
+    return HierarchicalCopyAllReduce(num_packs=len(device_links))
   else:
-    return AllReduceCrossDeviceOps("nccl", num_packs=1)
+    return NcclAllReduce(num_packs=1)
 
 
 def choose_the_best(devices, session_config=None):
@@ -973,12 +1068,12 @@ def choose_the_best(devices, session_config=None):
   if len(using_devices) != len(requested_devices):
     logging.warning("Not all devices in `tf.distribute.Strategy` are visible "
                     "to TensorFlow.")
-    return ReductionToOneDeviceCrossDeviceOps()
+    return ReductionToOneDevice()
 
   if any(d.device_type.lower() != "gpu" for d in using_devices):
     logging.warning("Not all devices in `tf.distribute.Strategy` are visible "
                     "to TensorFlow.")
-    return ReductionToOneDeviceCrossDeviceOps()
+    return ReductionToOneDevice()
 
   device_links = [[] for _ in range(len(using_devices))]
   for i, device in enumerate(using_devices):
diff --git a/tensorflow/python/distribute/one_device_strategy.py b/tensorflow/python/distribute/one_device_strategy.py
index ff9f616967..9a255b2fe1 100644
--- a/tensorflow/python/distribute/one_device_strategy.py
+++ b/tensorflow/python/distribute/one_device_strategy.py
@@ -29,11 +29,13 @@ from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.util import nest
+from tensorflow.python.util.tf_export import tf_export
 
 
 # TODO(josh11b): Replace asserts in this file with if ...: raise ...
 
 
+@tf_export("distribute.OneDeviceStrategy")
 class OneDeviceStrategy(distribute_lib.DistributionStrategy):
   """A distribution strategy for running on a single device."""
   # TODO(josh11b): Do we wrap values in types to generate errors if you are
diff --git a/tensorflow/python/distribute/parameter_server_strategy.py b/tensorflow/python/distribute/parameter_server_strategy.py
index 269c7417d1..1308ceaf5d 100644
--- a/tensorflow/python/distribute/parameter_server_strategy.py
+++ b/tensorflow/python/distribute/parameter_server_strategy.py
@@ -101,8 +101,7 @@ class ParameterServerStrategyExtended(
 
     # We typically don't need to do all-reduce in this strategy.
     self._cross_device_ops = (
-        cross_device_ops_lib.ReductionToOneDeviceCrossDeviceOps(
-            reduce_to_device=_LOCAL_CPU))
+        cross_device_ops_lib.ReductionToOneDevice(reduce_to_device=_LOCAL_CPU))
 
   def _initialize_strategy(self, cluster_resolver):
     if cluster_resolver.cluster_spec().as_dict():
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-cross-device-ops.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-cross-device-ops.pbtxt
new file mode 100644
index 0000000000..a2ea234324
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-cross-device-ops.pbtxt
@@ -0,0 +1,33 @@
+path: "tensorflow.distribute.CrossDeviceOps"
+tf_class {
+  is_instance: "<class \'tensorflow.python.distribute.cross_device_ops.CrossDeviceOps\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "batch_reduce"
+    argspec: "args=[\'self\', \'reduce_op\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "batch_reduce_implementation"
+    argspec: "args=[\'self\', \'reduce_op\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "broadcast"
+    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "broadcast_implementation"
+    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reduce"
+    argspec: "args=[\'self\', \'reduce_op\', \'per_replica_value\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reduce_implementation"
+    argspec: "args=[\'self\', \'reduce_op\', \'per_replica_value\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-hierarchical-copy-all-reduce.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-hierarchical-copy-all-reduce.pbtxt
new file mode 100644
index 0000000000..a38c4b21d5
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-hierarchical-copy-all-reduce.pbtxt
@@ -0,0 +1,35 @@
+path: "tensorflow.distribute.HierarchicalCopyAllReduce"
+tf_class {
+  is_instance: "<class \'tensorflow.python.distribute.cross_device_ops.HierarchicalCopyAllReduce\'>"
+  is_instance: "<class \'tensorflow.python.distribute.cross_device_ops.AllReduceCrossDeviceOps\'>"
+  is_instance: "<class \'tensorflow.python.distribute.cross_device_ops.CrossDeviceOps\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'num_packs\'], varargs=None, keywords=None, defaults=[\'1\'], "
+  }
+  member_method {
+    name: "batch_reduce"
+    argspec: "args=[\'self\', \'reduce_op\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "batch_reduce_implementation"
+    argspec: "args=[\'self\', \'reduce_op\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "broadcast"
+    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "broadcast_implementation"
+    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reduce"
+    argspec: "args=[\'self\', \'reduce_op\', \'per_replica_value\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reduce_implementation"
+    argspec: "args=[\'self\', \'reduce_op\', \'per_replica_value\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-nccl-all-reduce.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-nccl-all-reduce.pbtxt
new file mode 100644
index 0000000000..bdc09bcd84
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-nccl-all-reduce.pbtxt
@@ -0,0 +1,35 @@
+path: "tensorflow.distribute.NcclAllReduce"
+tf_class {
+  is_instance: "<class \'tensorflow.python.distribute.cross_device_ops.NcclAllReduce\'>"
+  is_instance: "<class \'tensorflow.python.distribute.cross_device_ops.AllReduceCrossDeviceOps\'>"
+  is_instance: "<class \'tensorflow.python.distribute.cross_device_ops.CrossDeviceOps\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'num_packs\'], varargs=None, keywords=None, defaults=[\'1\'], "
+  }
+  member_method {
+    name: "batch_reduce"
+    argspec: "args=[\'self\', \'reduce_op\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "batch_reduce_implementation"
+    argspec: "args=[\'self\', \'reduce_op\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "broadcast"
+    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "broadcast_implementation"
+    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reduce"
+    argspec: "args=[\'self\', \'reduce_op\', \'per_replica_value\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reduce_implementation"
+    argspec: "args=[\'self\', \'reduce_op\', \'per_replica_value\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-one-device-strategy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-one-device-strategy.pbtxt
new file mode 100644
index 0000000000..5d7943af2d
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-one-device-strategy.pbtxt
@@ -0,0 +1,66 @@
+path: "tensorflow.distribute.OneDeviceStrategy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.distribute.one_device_strategy.OneDeviceStrategy\'>"
+  is_instance: "<class \'tensorflow.python.distribute.distribute_lib.DistributionStrategy\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "extended"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "num_replicas_in_sync"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'device\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "broadcast"
+    argspec: "args=[\'self\', \'tensor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "colocate_vars_with"
+    argspec: "args=[\'self\', \'colocate_with_variable\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "configure"
+    argspec: "args=[\'self\', \'session_config\', \'cluster_spec\', \'task_type\', \'task_id\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "experimental_make_numpy_iterator"
+    argspec: "args=[\'self\', \'numpy_input\', \'batch_size\', \'num_epochs\', \'shuffle\', \'session\'], varargs=None, keywords=None, defaults=[\'1\', \'1024\', \'None\'], "
+  }
+  member_method {
+    name: "experimental_run"
+    argspec: "args=[\'self\', \'fn\', \'input_iterator\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "group"
+    argspec: "args=[\'self\', \'value\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "make_dataset_iterator"
+    argspec: "args=[\'self\', \'dataset\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "make_input_fn_iterator"
+    argspec: "args=[\'self\', \'input_fn\', \'replication_mode\'], varargs=None, keywords=None, defaults=[\'InputReplicationMode.PER_WORKER\'], "
+  }
+  member_method {
+    name: "reduce"
+    argspec: "args=[\'self\', \'reduce_op\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "scope"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "unwrap"
+    argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_config_proto"
+    argspec: "args=[\'self\', \'config_proto\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-reduction-to-one-device.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-reduction-to-one-device.pbtxt
new file mode 100644
index 0000000000..f5ade9f86b
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-reduction-to-one-device.pbtxt
@@ -0,0 +1,34 @@
+path: "tensorflow.distribute.ReductionToOneDevice"
+tf_class {
+  is_instance: "<class \'tensorflow.python.distribute.cross_device_ops.ReductionToOneDevice\'>"
+  is_instance: "<class \'tensorflow.python.distribute.cross_device_ops.CrossDeviceOps\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduce_to_device\', \'accumulation_fn\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "batch_reduce"
+    argspec: "args=[\'self\', \'reduce_op\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "batch_reduce_implementation"
+    argspec: "args=[\'self\', \'reduce_op\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "broadcast"
+    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "broadcast_implementation"
+    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reduce"
+    argspec: "args=[\'self\', \'reduce_op\', \'per_replica_value\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reduce_implementation"
+    argspec: "args=[\'self\', \'reduce_op\', \'per_replica_value\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.pbtxt
index 5b5c9e2df1..5ea6b5b348 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.pbtxt
@@ -1,5 +1,13 @@
 path: "tensorflow.distribute"
 tf_module {
+  member {
+    name: "CrossDeviceOps"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "HierarchicalCopyAllReduce"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "InputContext"
     mtype: "<type \'type\'>"
@@ -12,10 +20,22 @@ tf_module {
     name: "MirroredStrategy"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "NcclAllReduce"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "OneDeviceStrategy"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "ReduceOp"
     mtype: "<class \'enum.EnumMeta\'>"
   }
+  member {
+    name: "ReductionToOneDevice"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "ReplicaContext"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-cross-device-ops.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-cross-device-ops.pbtxt
new file mode 100644
index 0000000000..a2ea234324
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-cross-device-ops.pbtxt
@@ -0,0 +1,33 @@
+path: "tensorflow.distribute.CrossDeviceOps"
+tf_class {
+  is_instance: "<class \'tensorflow.python.distribute.cross_device_ops.CrossDeviceOps\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "batch_reduce"
+    argspec: "args=[\'self\', \'reduce_op\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "batch_reduce_implementation"
+    argspec: "args=[\'self\', \'reduce_op\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "broadcast"
+    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "broadcast_implementation"
+    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reduce"
+    argspec: "args=[\'self\', \'reduce_op\', \'per_replica_value\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reduce_implementation"
+    argspec: "args=[\'self\', \'reduce_op\', \'per_replica_value\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-hierarchical-copy-all-reduce.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-hierarchical-copy-all-reduce.pbtxt
new file mode 100644
index 0000000000..a38c4b21d5
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-hierarchical-copy-all-reduce.pbtxt
@@ -0,0 +1,35 @@
+path: "tensorflow.distribute.HierarchicalCopyAllReduce"
+tf_class {
+  is_instance: "<class \'tensorflow.python.distribute.cross_device_ops.HierarchicalCopyAllReduce\'>"
+  is_instance: "<class \'tensorflow.python.distribute.cross_device_ops.AllReduceCrossDeviceOps\'>"
+  is_instance: "<class \'tensorflow.python.distribute.cross_device_ops.CrossDeviceOps\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'num_packs\'], varargs=None, keywords=None, defaults=[\'1\'], "
+  }
+  member_method {
+    name: "batch_reduce"
+    argspec: "args=[\'self\', \'reduce_op\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "batch_reduce_implementation"
+    argspec: "args=[\'self\', \'reduce_op\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "broadcast"
+    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "broadcast_implementation"
+    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reduce"
+    argspec: "args=[\'self\', \'reduce_op\', \'per_replica_value\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reduce_implementation"
+    argspec: "args=[\'self\', \'reduce_op\', \'per_replica_value\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-nccl-all-reduce.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-nccl-all-reduce.pbtxt
new file mode 100644
index 0000000000..bdc09bcd84
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-nccl-all-reduce.pbtxt
@@ -0,0 +1,35 @@
+path: "tensorflow.distribute.NcclAllReduce"
+tf_class {
+  is_instance: "<class \'tensorflow.python.distribute.cross_device_ops.NcclAllReduce\'>"
+  is_instance: "<class \'tensorflow.python.distribute.cross_device_ops.AllReduceCrossDeviceOps\'>"
+  is_instance: "<class \'tensorflow.python.distribute.cross_device_ops.CrossDeviceOps\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'num_packs\'], varargs=None, keywords=None, defaults=[\'1\'], "
+  }
+  member_method {
+    name: "batch_reduce"
+    argspec: "args=[\'self\', \'reduce_op\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "batch_reduce_implementation"
+    argspec: "args=[\'self\', \'reduce_op\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "broadcast"
+    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "broadcast_implementation"
+    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reduce"
+    argspec: "args=[\'self\', \'reduce_op\', \'per_replica_value\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reduce_implementation"
+    argspec: "args=[\'self\', \'reduce_op\', \'per_replica_value\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-one-device-strategy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-one-device-strategy.pbtxt
new file mode 100644
index 0000000000..5d7943af2d
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-one-device-strategy.pbtxt
@@ -0,0 +1,66 @@
+path: "tensorflow.distribute.OneDeviceStrategy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.distribute.one_device_strategy.OneDeviceStrategy\'>"
+  is_instance: "<class \'tensorflow.python.distribute.distribute_lib.DistributionStrategy\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "extended"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "num_replicas_in_sync"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'device\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "broadcast"
+    argspec: "args=[\'self\', \'tensor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "colocate_vars_with"
+    argspec: "args=[\'self\', \'colocate_with_variable\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "configure"
+    argspec: "args=[\'self\', \'session_config\', \'cluster_spec\', \'task_type\', \'task_id\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "experimental_make_numpy_iterator"
+    argspec: "args=[\'self\', \'numpy_input\', \'batch_size\', \'num_epochs\', \'shuffle\', \'session\'], varargs=None, keywords=None, defaults=[\'1\', \'1024\', \'None\'], "
+  }
+  member_method {
+    name: "experimental_run"
+    argspec: "args=[\'self\', \'fn\', \'input_iterator\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "group"
+    argspec: "args=[\'self\', \'value\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "make_dataset_iterator"
+    argspec: "args=[\'self\', \'dataset\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "make_input_fn_iterator"
+    argspec: "args=[\'self\', \'input_fn\', \'replication_mode\'], varargs=None, keywords=None, defaults=[\'InputReplicationMode.PER_WORKER\'], "
+  }
+  member_method {
+    name: "reduce"
+    argspec: "args=[\'self\', \'reduce_op\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "scope"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "unwrap"
+    argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_config_proto"
+    argspec: "args=[\'self\', \'config_proto\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-reduction-to-one-device.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-reduction-to-one-device.pbtxt
new file mode 100644
index 0000000000..f5ade9f86b
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-reduction-to-one-device.pbtxt
@@ -0,0 +1,34 @@
+path: "tensorflow.distribute.ReductionToOneDevice"
+tf_class {
+  is_instance: "<class \'tensorflow.python.distribute.cross_device_ops.ReductionToOneDevice\'>"
+  is_instance: "<class \'tensorflow.python.distribute.cross_device_ops.CrossDeviceOps\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduce_to_device\', \'accumulation_fn\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "batch_reduce"
+    argspec: "args=[\'self\', \'reduce_op\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "batch_reduce_implementation"
+    argspec: "args=[\'self\', \'reduce_op\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "broadcast"
+    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "broadcast_implementation"
+    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reduce"
+    argspec: "args=[\'self\', \'reduce_op\', \'per_replica_value\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reduce_implementation"
+    argspec: "args=[\'self\', \'reduce_op\', \'per_replica_value\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.pbtxt
index 0d0d35043c..a4b78d4bb6 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.pbtxt
@@ -1,5 +1,13 @@
 path: "tensorflow.distribute"
 tf_module {
+  member {
+    name: "CrossDeviceOps"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "HierarchicalCopyAllReduce"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "InputContext"
     mtype: "<type \'type\'>"
@@ -12,10 +20,22 @@ tf_module {
     name: "MirroredStrategy"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "NcclAllReduce"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "OneDeviceStrategy"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "ReduceOp"
     mtype: "<class \'enum.EnumMeta\'>"
   }
+  member {
+    name: "ReductionToOneDevice"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "ReplicaContext"
     mtype: "<type \'type\'>"
-- 
GitLab


From 5a40160e01a6d0777d47600cabb0c991bd77b39d Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Mon, 11 Feb 2019 18:08:03 -0800
Subject: [PATCH 2298/2345] Make RefVariables work with tf.saved_model.load

- Copies node names, which is pretty important if we're looking up variables by name in the resource manager.
- Fixes wrap_function's Operation-only fetching by adding a fake Tensor output with control deps. Assignment ops (but not VariableV2 oddly enough) were getting pruned when running the SaverDef's restore.

Ref variables still won't be too pleasant to use since there won't be eager objects for them. But the imported graphs will run.

PiperOrigin-RevId: 233516944
---
 tensorflow/python/eager/lift_to_graph.py      |  5 ++--
 tensorflow/python/eager/wrap_function.py      |  5 ++--
 tensorflow/python/eager/wrap_function_test.py |  2 +-
 .../python/saved_model/load_v1_in_v2.py       |  6 -----
 .../python/saved_model/load_v1_in_v2_test.py  | 23 +++++++++++--------
 5 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/tensorflow/python/eager/lift_to_graph.py b/tensorflow/python/eager/lift_to_graph.py
index e48001eddc..d2ccdd5a41 100644
--- a/tensorflow/python/eager/lift_to_graph.py
+++ b/tensorflow/python/eager/lift_to_graph.py
@@ -100,7 +100,8 @@ def lift_to_graph(init_tensor, graph, sources=None,
     source_ops = set()
     for s in sources:
       source_ops.add(s.op)
-      op_map[s] = array_ops.placeholder(dtype=s.dtype, shape=s.shape)
+      op_map[s] = array_ops.placeholder(dtype=s.dtype, shape=s.shape,
+                                        name=s.op.name)
     for op in reversed(ops_to_copy):
       if op in source_ops:
         continue
@@ -109,7 +110,7 @@ def lift_to_graph(init_tensor, graph, sources=None,
       with ops.control_dependencies(copied_control_inputs):
         copied_op = graph.create_op(
             op.type, copied_inputs, [x.dtype for x in op.outputs],
-            attrs=op.node_def.attr)
+            attrs=op.node_def.attr, name=op.name)
       op_map[op] = copied_op
       for i, o in enumerate(op.outputs):
         op_map[o] = copied_op.outputs[i]
diff --git a/tensorflow/python/eager/wrap_function.py b/tensorflow/python/eager/wrap_function.py
index de384bd456..7b1de99406 100644
--- a/tensorflow/python/eager/wrap_function.py
+++ b/tensorflow/python/eager/wrap_function.py
@@ -27,7 +27,6 @@ from tensorflow.python.eager import lift_to_graph
 from tensorflow.python.framework import func_graph
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.util import nest
@@ -155,7 +154,7 @@ class WrappedFunction(function.ConcreteFunction):
           sink_tensor = identity_fetches[0]
         else:
           identity_fetches = []
-          sink_tensor = control_flow_ops.no_op()
+          sink_tensor = array_ops.zeros([])
     lift_map = lift_to_graph.lift_to_graph(
         sink_tensor, pruned_graph,
         sources=flat_feeds + internal_captures)
@@ -164,6 +163,8 @@ class WrappedFunction(function.ConcreteFunction):
       lift_map[original_fetch] = lift_map[identity_fetch]
     pruned_graph.outputs.extend(
         lift_map[x] for x in flat_fetches if isinstance(x, ops.Tensor))
+    if not tensor_fetches:
+      pruned_graph.outputs.append(lift_map[sink_tensor])
     for external_capture, internal_capture in self.graph.captures.items():
       pruned_graph.captures[external_capture] = lift_map[internal_capture]
     pruned_graph.inputs.extend(lift_map[x] for x in flat_feeds)
diff --git a/tensorflow/python/eager/wrap_function_test.py b/tensorflow/python/eager/wrap_function_test.py
index bb2b28861f..6225b13849 100644
--- a/tensorflow/python/eager/wrap_function_test.py
+++ b/tensorflow/python/eager/wrap_function_test.py
@@ -206,7 +206,7 @@ class WrapFunctionTest(test.TestCase):
         fetches=(f_wrapped.graph.get_operation_by_name('increment'),
                  f_wrapped.graph.get_tensor_by_name('other:0')))
     first_output, second_output = increments(constant_op.constant(2))
-    self.assertEqual(['Placeholder:0', 'Placeholder_1:0'],
+    self.assertEqual(['step:0', 'increment/resource:0'],
                      [t.name for t in increments.inputs])
     self.assertIs(None, first_output)
     self.assertEqual(1, second_output.numpy())
diff --git a/tensorflow/python/saved_model/load_v1_in_v2.py b/tensorflow/python/saved_model/load_v1_in_v2.py
index 5980b84878..b1d56dc915 100644
--- a/tensorflow/python/saved_model/load_v1_in_v2.py
+++ b/tensorflow/python/saved_model/load_v1_in_v2.py
@@ -86,12 +86,6 @@ class _EagerSavedModelLoader(loader_impl.SavedModelLoader):
   def load(self, tags):
     """Creates an object from the MetaGraph identified by `tags`."""
     meta_graph_def = self.get_meta_graph_def_from_tags(tags)
-    for node in meta_graph_def.graph_def.node:
-      if node.op == "VariableV2":
-        raise NotImplementedError(
-            "Importing a SavedModel which contains RefVariables. This is not "
-            "currently supported. Running tf.enable_resource_variables() "
-            "before creating exported variables will fix this.")
     load_graph_returns = [None]
     wrapped = wrap_function.wrap_function(
         functools.partial(self.load_graph, load_graph_returns, meta_graph_def),
diff --git a/tensorflow/python/saved_model/load_v1_in_v2_test.py b/tensorflow/python/saved_model/load_v1_in_v2_test.py
index 2bb6b93b3b..62e5e721b4 100644
--- a/tensorflow/python/saved_model/load_v1_in_v2_test.py
+++ b/tensorflow/python/saved_model/load_v1_in_v2_test.py
@@ -44,9 +44,16 @@ class LoadTest(test.TestCase):
       start = array_ops.placeholder(
           shape=[None], dtype=dtypes.float32, name="start")
       if use_resource:
-        v = resource_variable_ops.ResourceVariable(3.)
+        distractor = variables.RefVariable(-1., name="distractor")
+        v = resource_variable_ops.ResourceVariable(3., name="v")
       else:
-        v = variables.RefVariable(3.)
+        # "distractor" gets saved in the checkpoint and so used in the restore
+        # function, but not in the pruned function for the signature. This tests
+        # node naming: it needs to be consistent (and ideally always the same as
+        # the node in the original GraphDef) for the resource manager to find
+        # the right variable.
+        distractor = variables.RefVariable(-1., name="distractor")
+        v = variables.RefVariable(3., name="v")
       local_variable = variables.VariableV1(
           1.,
           collections=[ops.GraphKeys.LOCAL_VARIABLES],
@@ -54,7 +61,8 @@ class LoadTest(test.TestCase):
           use_resource=True)
       output = array_ops.identity(start * v * local_variable, name="output")
       with session_lib.Session() as session:
-        session.run([v.initializer, local_variable.initializer])
+        session.run([v.initializer, distractor.initializer,
+                     local_variable.initializer])
         path = os.path.join(self.get_temp_dir(), "saved_model", str(ops.uid()))
         simple_save.simple_save(
             session,
@@ -87,13 +95,10 @@ class LoadTest(test.TestCase):
     self.assertEqual(8., tape.gradient(output, imported.variables[0]).numpy())
 
   def test_ref_variable_import(self):
-    with self.assertRaises(NotImplementedError):
-      imported = load.load(self._v1_single_metagraph_saved_model(
-          use_resource=False))
-    # TODO(allenl): Support ref variables
-    self.skipTest("Ref variables aren't working yet")
+    saved = self._v1_single_metagraph_saved_model(use_resource=False)
+    imported = load.load(saved)
     fn = imported.signatures["serving_default"]
-    self.assertEqual(6., fn(start=constant_op.constant(2.)))
+    self.assertEqual(6., fn(start=constant_op.constant(2.))["output"].numpy())
 
   def _v1_multi_metagraph_saved_model(self):
     export_graph = ops.Graph()
-- 
GitLab


From 427c6db0f2486b91c37cf87f1e663ecd6e5b1a72 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Mon, 11 Feb 2019 18:08:25 -0800
Subject: [PATCH 2299/2345] Serialize hyper variable when it's distributed
 variable.

PiperOrigin-RevId: 233517009
---
 tensorflow/contrib/distribute/python/BUILD    |  19 +
 .../contrib/distribute/python/keras_test.py   | 398 ---------------
 .../distribute/python/keras_utils_test.py     | 471 ++++++++++++++++++
 .../python/keras/optimizer_v2/optimizer_v2.py |   3 +-
 .../python/training/checkpointable/base.py    |  11 +-
 .../training/checkpointable/base_test.py      |  13 -
 6 files changed, 496 insertions(+), 419 deletions(-)
 create mode 100644 tensorflow/contrib/distribute/python/keras_utils_test.py

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 26ebc3dc2e..44ecc8c428 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -538,6 +538,7 @@ py_library(
     srcs = [
         "keras_backward_compat_test.py",
         "keras_test.py",
+        "keras_utils_test.py",
     ],
     deps = [
         ":combinations",
@@ -570,6 +571,24 @@ distribute_py_test(
     ],
 )
 
+distribute_py_test(
+    name = "keras_utils_test",
+    srcs = ["keras_utils_test.py"],
+    full_precision = True,
+    main = "keras_utils_test.py",
+    shard_count = 32,
+    tags = [
+        "multi_and_single_gpu",
+        "no_oss",  # TODO(b/117919883): Fix python error.
+        "no_windows_gpu",
+        "notsan",
+    ],
+    deps = [
+        ":keras_test",
+        ":keras_test_lib",
+    ],
+)
+
 # TODO(b/121200287): Remove this in 2.0
 distribute_py_test(
     name = "keras_backward_compat_test",
diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index eb515441e2..3934279a61 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -17,9 +17,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import collections
 import os
-import tempfile
 from absl.testing import parameterized
 import numpy as np
 
@@ -29,12 +27,9 @@ from tensorflow.contrib.distribute.python import tpu_strategy
 from tensorflow.python import keras
 from tensorflow.python.data.experimental.ops import cardinality
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.distribute import values
 from tensorflow.python.eager import test
 from tensorflow.python.estimator import keras as keras_lib
 from tensorflow.python.estimator import run_config as run_config_lib
-from tensorflow.python.framework import constant_op
-from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import test_util
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.keras.engine import distributed_training_utils
@@ -1193,398 +1188,5 @@ class TestDistributionStrategyWithDatasets(test.TestCase,
           atol=1e-4, rtol=1e-4)
 
 
-class Counter(keras.callbacks.Callback):
-  """Counts the number of times each callback method was run.
-
-  Attributes:
-    method_counts: dict. Contains the counts of time  each callback method was
-      run.
-  """
-
-  def __init__(self):
-    self.method_counts = collections.defaultdict(int)
-    methods_to_count = [
-        'on_batch_begin', 'on_batch_end', 'on_epoch_begin', 'on_epoch_end',
-        'on_predict_batch_begin', 'on_predict_batch_end', 'on_predict_begin',
-        'on_predict_end', 'on_test_batch_begin', 'on_test_batch_end',
-        'on_test_begin', 'on_test_end', 'on_train_batch_begin',
-        'on_train_batch_end', 'on_train_begin', 'on_train_end'
-    ]
-    for method_name in methods_to_count:
-      setattr(self, method_name,
-              self.wrap_with_counts(method_name, getattr(self, method_name)))
-
-  def wrap_with_counts(self, method_name, method):
-
-    def _call_and_count(*args, **kwargs):
-      self.method_counts[method_name] += 1
-      return method(*args, **kwargs)
-
-    return _call_and_count
-
-
-class TestDistributionStrategyWithCallbacks(test.TestCase,
-                                            parameterized.TestCase):
-
-  @combinations.generate(all_strategy_combinations())
-  def test_callbacks_in_fit(self, distribution):
-    with distribution.scope():
-      model = get_model()
-      model.compile(optimizer='sgd', loss='mse', metrics=['mae'])
-
-    dataset = get_dataset(distribution)
-    counter = Counter()
-
-    epochs = 2
-    steps_per_epoch = 5
-    validation_steps = 3
-
-    model.fit(
-        dataset,
-        epochs=epochs,
-        steps_per_epoch=steps_per_epoch,
-        verbose=0,
-        validation_data=dataset,
-        validation_steps=validation_steps,
-        callbacks=[counter])
-
-    if isinstance(distribution, tpu_strategy.TPUStrategy):
-      # TPU Strategy can have multi step training, from extended.steps_per_run
-      # if steps_per_run = 1, then num_batch_call_per_epoch = steps_per_epoch
-      steps_per_run = distribution.extended.steps_per_run
-      num_batch_call_per_epoch = steps_per_epoch // steps_per_run
-      if steps_per_epoch % steps_per_run:
-        num_batch_call_per_epoch += 1
-    else:
-      num_batch_call_per_epoch = steps_per_epoch
-
-    self.assertDictEqual(
-        counter.method_counts, {
-            'on_batch_begin': epochs * num_batch_call_per_epoch,
-            'on_batch_end': epochs * num_batch_call_per_epoch,
-            'on_epoch_begin': epochs,
-            'on_epoch_end': epochs,
-            'on_test_batch_begin': epochs * validation_steps,
-            'on_test_batch_end': epochs * validation_steps,
-            'on_test_begin': epochs,
-            'on_test_end': epochs,
-            'on_train_batch_begin': epochs * num_batch_call_per_epoch,
-            'on_train_batch_end': epochs * num_batch_call_per_epoch,
-            'on_train_begin': 1,
-            'on_train_end': 1
-        })
-
-  @combinations.generate(all_strategy_combinations())
-  def test_callbacks_in_eval(self, distribution):
-    with distribution.scope():
-      model = get_model()
-      model.compile(optimizer='sgd', loss='mse', metrics=['mae'])
-
-    dataset = get_dataset(distribution)
-    counter = Counter()
-
-    model.evaluate(dataset, steps=5, callbacks=[counter])
-
-    self.assertDictEqual(
-        counter.method_counts, {
-            'on_test_batch_begin': 5,
-            'on_test_batch_end': 5,
-            'on_test_begin': 1,
-            'on_test_end': 1
-        })
-
-  @combinations.generate(all_strategy_combinations())
-  def test_callbacks_in_predict(self, distribution):
-    with distribution.scope():
-      model = get_model()
-      model.compile(optimizer='sgd', loss='mse', metrics=['mae'])
-
-    dataset = get_dataset(distribution)
-    counter = Counter()
-
-    model.predict(get_predict_dataset(dataset), steps=5, callbacks=[counter])
-
-    self.assertDictEqual(
-        counter.method_counts, {
-            'on_predict_batch_begin': 5,
-            'on_predict_batch_end': 5,
-            'on_predict_begin': 1,
-            'on_predict_end': 1
-        })
-
-
-class TestDistributionStrategyErrorCases(test.TestCase, parameterized.TestCase):
-
-  @combinations.generate(combinations.combine(
-      distribution=[
-          combinations.mirrored_strategy_with_gpu_and_cpu,
-          combinations.core_mirrored_strategy_with_gpu_and_cpu],
-      mode=['graph', 'eager']))
-  def test_validating_dataset_input_tensors_with_shape_mismatch(self,
-                                                                distribution):
-    with self.cached_session():
-      a = constant_op.constant([1, 2], shape=(1, 2))
-      b = constant_op.constant([[1, 2], [1, 2]], shape=(2, 2))
-      device_map = values.ReplicaDeviceMap(('/device:CPU:0', '/device:GPU:0'))
-      x = values.DistributedValues(device_map, (a, b))
-      y = values.DistributedValues(device_map, (a, a))
-      # Removed device and input tensor shape details from the error message
-      # since the order of the device and the corresponding input tensor shape
-      # is not deterministic over different runs.
-      with self.assertRaisesRegexp(ValueError,
-                                   'Input tensor shapes do not match for '
-                                   'distributed tensor inputs '
-                                   'DistributedValues:.+'):
-        with distribution.scope():
-          distributed_training_utils.validate_distributed_dataset_inputs(
-              distribution, x, y)
-
-  @combinations.generate(combinations.combine(
-      distribution=[
-          combinations.mirrored_strategy_with_gpu_and_cpu,
-          combinations.core_mirrored_strategy_with_gpu_and_cpu],
-      mode=['graph', 'eager']))
-  def test_validating_dataset_input_tensors_with_dtype_mismatch(self,
-                                                                distribution):
-    with self.cached_session():
-      a = constant_op.constant([1, 2], shape=(1, 2), dtype=dtypes.int32)
-      b = constant_op.constant([1, 2], shape=(1, 2), dtype=dtypes.float64)
-      device_map = values.ReplicaDeviceMap(('/device:CPU:0', '/device:GPU:0'))
-      x = values.DistributedValues(device_map, (a, b))
-      y = values.DistributedValues(device_map, (a, a))
-      # Removed device and input tensor dtype details from the error message
-      # since the order of the device and the corresponding input tensor dtype
-      # is not deterministic over different runs.
-      with self.assertRaisesRegexp(ValueError,
-                                   'Input tensor dtypes do not match for '
-                                   'distributed tensor inputs '
-                                   'DistributedValues:.+'):
-        with distribution.scope():
-          distributed_training_utils.validate_distributed_dataset_inputs(
-              distribution, x, y)
-
-  @combinations.generate(combinations.combine(
-      distribution=[
-          combinations.mirrored_strategy_with_gpu_and_cpu,
-          combinations.core_mirrored_strategy_with_gpu_and_cpu],
-      mode=['graph', 'eager']))
-  def test_unsupported_features(self, distribution):
-    with self.cached_session():
-      with distribution.scope():
-        model = get_model()
-        optimizer = gradient_descent.GradientDescentOptimizer(0.001)
-        loss = 'mse'
-        metrics = ['mae']
-        model.compile(optimizer, loss, metrics=metrics)
-
-      dataset = get_dataset(distribution)
-
-      # Test with validation split
-      with self.assertRaisesRegexp(
-          ValueError, '`validation_split` argument is not '
-                      'supported when input `x` is a dataset or a '
-                      'dataset iterator.+'):
-        model.fit(dataset,
-                  epochs=1, steps_per_epoch=2, verbose=0,
-                  validation_split=0.5, validation_steps=2)
-
-      # Test with sample weight.
-      sample_weight = np.random.random((10,))
-      with self.assertRaisesRegexp(
-          ValueError, '`sample_weight` argument is not supported when input '
-                      '`x` is a dataset or a dataset iterator.'):
-        model.fit(
-            dataset,
-            epochs=1,
-            steps_per_epoch=2,
-            verbose=0,
-            sample_weight=sample_weight)
-
-      # Test with not specifying the `steps` argument for dataset with infinite
-      # cardinality.
-      dataset = dataset.repeat()
-      with self.assertRaisesRegexp(ValueError, 'When passing an infinitely '
-                                   'repeating dataset, you must specify the '
-                                   '`steps_per_epoch` argument'):
-        model.fit(dataset, epochs=1, verbose=0)
-      with self.assertRaisesRegexp(ValueError, 'When passing an infinitely '
-                                   'repeating dataset, you must specify the '
-                                   '`steps` argument'):
-        model.evaluate(dataset, verbose=0)
-
-      with self.assertRaisesRegexp(ValueError, 'When passing an infinitely '
-                                   'repeating dataset, you must specify the '
-                                   '`steps` argument'):
-        model.predict(dataset, verbose=0)
-
-  @combinations.generate(combinations.combine(
-      distribution=[
-          combinations.mirrored_strategy_with_gpu_and_cpu,
-          combinations.core_mirrored_strategy_with_gpu_and_cpu],
-      mode=['graph', 'eager']))
-  def test_calling_with_unsupported_predefined_callbacks(self, distribution):
-    with self.cached_session():
-      with distribution.scope():
-        model = get_model()
-        optimizer = gradient_descent.GradientDescentOptimizer(0.001)
-        loss = 'mse'
-        metrics = ['mae']
-        model.compile(optimizer, loss, metrics=metrics)
-
-      dataset = get_dataset(distribution)
-
-      def schedule(_):
-        return 0.001
-      with self.assertRaisesRegexp(ValueError,
-                                   'You must specify a Keras Optimizer V2 when '
-                                   'using'):
-        model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0,
-                  callbacks=[keras.callbacks.LearningRateScheduler(schedule)])
-
-      with self.assertRaisesRegexp(ValueError,
-                                   'You must specify a Keras Optimizer V2 when '
-                                   'using'):
-        model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0,
-                  callbacks=[keras.callbacks.ReduceLROnPlateau()])
-
-
-class TestDistributionStrategyWithLossMasking(test.TestCase,
-                                              parameterized.TestCase):
-
-  # TODO(priyag): Enable all strategies for this test. Currently it does not
-  # work for TPU due to some invalid datatype.
-  @combinations.generate(combinations.combine(
-      distribution=[
-          combinations.mirrored_strategy_with_gpu_and_cpu,
-          combinations.core_mirrored_strategy_with_gpu_and_cpu],
-      mode=['graph', 'eager']))
-  def test_masking(self, distribution):
-    with self.cached_session():
-      np.random.seed(1337)
-      x = np.array([[[1], [1]], [[0], [0]]])
-      with distribution.scope():
-        model = keras.models.Sequential()
-        model.add(keras.layers.Masking(mask_value=0, input_shape=(2, 1)))
-        model.add(
-            keras.layers.TimeDistributed(
-                keras.layers.Dense(1, kernel_initializer='one')))
-        model.compile(loss='mse',
-                      optimizer=gradient_descent.GradientDescentOptimizer(0.01))
-      y = np.array([[[1], [1]], [[1], [1]]])
-      dataset = dataset_ops.Dataset.from_tensor_slices((x, y))
-      dataset = dataset.repeat(100)
-      dataset = dataset.batch(10)
-      hist = model.fit(x=dataset, epochs=1, steps_per_epoch=2)
-      self.assertEqual(hist.history['loss'][0], 0)
-
-
-class TestDistributionStrategyWithNormalizationLayer(
-    test.TestCase, parameterized.TestCase):
-
-  @combinations.generate(combinations.times(
-      all_strategy_combinations(),
-      combinations.combine(fused=[True, False])))
-  def test_batchnorm_correctness(self, distribution, fused):
-    with self.cached_session():
-      with distribution.scope():
-        model = keras.models.Sequential()
-        norm = keras.layers.BatchNormalization(
-            input_shape=(10,), momentum=0.8, fused=fused)
-        model.add(norm)
-        model.compile(loss='mse',
-                      optimizer=gradient_descent.GradientDescentOptimizer(0.01))
-
-      # centered on 5.0, variance 10.0
-      x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 10))
-      x = x.astype('float32')
-      dataset = dataset_ops.Dataset.from_tensor_slices((x, x))
-      dataset = dataset.repeat(100)
-      dataset = batch_wrapper(dataset, 32, distribution)
-
-      predict_dataset = dataset_ops.Dataset.from_tensor_slices(x)
-      predict_dataset = predict_dataset.repeat(100)
-      predict_dataset = batch_wrapper(predict_dataset, 32, distribution)
-
-      model.fit(dataset, epochs=4, verbose=0, steps_per_epoch=10)
-      out = model.predict(predict_dataset, steps=2)
-      out -= keras.backend.eval(norm.beta)
-      out /= keras.backend.eval(norm.gamma)
-      np.testing.assert_allclose(out.mean(), 0.0, atol=1e-1)
-      np.testing.assert_allclose(out.std(), 1.0, atol=1e-1)
-
-
-class TestDistributionStrategySaveLoadWeights(test.TestCase,
-                                              parameterized.TestCase):
-
-  @combinations.generate(all_strategy_combinations_minus_default())
-  def test_save_load_h5(self, distribution):
-    with self.cached_session():
-      dataset = get_dataset(distribution)
-      with distribution.scope():
-        model = get_model()
-        model.compile(gradient_descent_keras.SGD(0.01), 'mse')
-        model.fit(dataset, epochs=1, steps_per_epoch=1)
-
-        weights_file = tempfile.mktemp('.h5')
-        model.save_weights(weights_file)
-
-        model_2 = get_model()
-        model_2.compile(gradient_descent_keras.SGD(0.01), 'mse')
-        model_2.load_weights(weights_file)
-        model_2.predict(get_predict_dataset(distribution), steps=2)
-        model_2.fit(dataset, epochs=1, steps_per_epoch=1)
-
-  @combinations.generate(all_strategy_combinations_minus_default())
-  def test_save_load_checkpointable(self, distribution):
-    # TODO(sourabhbajaj): Test fails with optimizer v2 without h5
-    with self.cached_session():
-      dataset = get_dataset(distribution)
-      with distribution.scope():
-        model = get_model()
-        model.compile(gradient_descent.GradientDescentOptimizer(0.01), 'mse')
-        model.fit(dataset, epochs=1, steps_per_epoch=1)
-
-        weights_file = tempfile.mktemp()
-        model.save_weights(weights_file)
-
-        model_2 = get_model()
-        model_2.compile(gradient_descent.GradientDescentOptimizer(0.01), 'mse')
-        model_2.load_weights(weights_file)
-        model_2.predict(get_predict_dataset(distribution), steps=2)
-        model_2.fit(dataset, epochs=1, steps_per_epoch=1)
-
-
-class TestDistributionStrategyValidation(test.TestCase,
-                                         parameterized.TestCase):
-
-  @combinations.generate(all_strategy_combinations_minus_default())
-  def test_layer_outside_scope(self, distribution):
-    with self.cached_session():
-      with self.assertRaisesRegexp(
-          ValueError, 'was not created in the distribution strategy'):
-        x = keras.layers.Input(shape=(3,), name='input')
-        y = keras.layers.Dense(4, name='dense')(x)
-        with distribution.scope():
-          model = keras.Model(x, y)
-          optimizer = gradient_descent.GradientDescentOptimizer(0.001)
-          loss = 'mse'
-          metrics = ['mae', keras.metrics.CategoricalAccuracy()]
-          model.compile(optimizer, loss, metrics=metrics)
-
-  @combinations.generate(all_strategy_combinations_minus_default())
-  def test_model_outside_scope(self, distribution):
-    with self.cached_session():
-      with self.assertRaisesRegexp(
-          ValueError, 'was not created in the distribution strategy'):
-        x = keras.layers.Input(shape=(3,), name='input')
-        y = keras.layers.Dense(4, name='dense')(x)
-        model = keras.Model(x, y)
-        with distribution.scope():
-          optimizer = gradient_descent.GradientDescentOptimizer(0.001)
-          loss = 'mse'
-          metrics = ['mae', keras.metrics.CategoricalAccuracy()]
-          model.compile(optimizer, loss, metrics=metrics)
-
-
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/distribute/python/keras_utils_test.py b/tensorflow/contrib/distribute/python/keras_utils_test.py
new file mode 100644
index 0000000000..3e5b422f51
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/keras_utils_test.py
@@ -0,0 +1,471 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for tf.keras models with callbacks, checkpointing with dist strategy."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import collections
+import tempfile
+from absl.testing import parameterized
+import numpy as np
+
+from tensorflow.contrib.distribute.python import combinations
+from tensorflow.contrib.distribute.python import keras_test as keras_test_lib
+from tensorflow.contrib.distribute.python import tpu_strategy
+from tensorflow.python import keras
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.distribute import values
+from tensorflow.python.eager import test
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.keras.engine import distributed_training_utils
+from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_descent_keras
+from tensorflow.python.training import gradient_descent
+
+
+class Counter(keras.callbacks.Callback):
+  """Counts the number of times each callback method was run.
+
+  Attributes:
+    method_counts: dict. Contains the counts of time  each callback method was
+      run.
+  """
+
+  def __init__(self):
+    self.method_counts = collections.defaultdict(int)
+    methods_to_count = [
+        'on_batch_begin', 'on_batch_end', 'on_epoch_begin', 'on_epoch_end',
+        'on_predict_batch_begin', 'on_predict_batch_end', 'on_predict_begin',
+        'on_predict_end', 'on_test_batch_begin', 'on_test_batch_end',
+        'on_test_begin', 'on_test_end', 'on_train_batch_begin',
+        'on_train_batch_end', 'on_train_begin', 'on_train_end'
+    ]
+    for method_name in methods_to_count:
+      setattr(self, method_name,
+              self.wrap_with_counts(method_name, getattr(self, method_name)))
+
+  def wrap_with_counts(self, method_name, method):
+
+    def _call_and_count(*args, **kwargs):
+      self.method_counts[method_name] += 1
+      return method(*args, **kwargs)
+
+    return _call_and_count
+
+
+class TestDistributionStrategyWithCallbacks(test.TestCase,
+                                            parameterized.TestCase):
+
+  @combinations.generate(keras_test_lib.all_strategy_combinations())
+  def test_callbacks_in_fit(self, distribution):
+    with distribution.scope():
+      model = keras_test_lib.get_model()
+      model.compile(optimizer='sgd', loss='mse', metrics=['mae'])
+
+    dataset = keras_test_lib.get_dataset(distribution)
+    counter = Counter()
+
+    epochs = 2
+    steps_per_epoch = 5
+    validation_steps = 3
+
+    model.fit(
+        dataset,
+        epochs=epochs,
+        steps_per_epoch=steps_per_epoch,
+        verbose=0,
+        validation_data=dataset,
+        validation_steps=validation_steps,
+        callbacks=[counter])
+
+    if isinstance(distribution, tpu_strategy.TPUStrategy):
+      # TPU Strategy can have multi step training, from extended.steps_per_run
+      # if steps_per_run = 1, then num_batch_call_per_epoch = steps_per_epoch
+      steps_per_run = distribution.extended.steps_per_run
+      num_batch_call_per_epoch = steps_per_epoch // steps_per_run
+      if steps_per_epoch % steps_per_run:
+        num_batch_call_per_epoch += 1
+    else:
+      num_batch_call_per_epoch = steps_per_epoch
+
+    self.assertDictEqual(
+        counter.method_counts, {
+            'on_batch_begin': epochs * num_batch_call_per_epoch,
+            'on_batch_end': epochs * num_batch_call_per_epoch,
+            'on_epoch_begin': epochs,
+            'on_epoch_end': epochs,
+            'on_test_batch_begin': epochs * validation_steps,
+            'on_test_batch_end': epochs * validation_steps,
+            'on_test_begin': epochs,
+            'on_test_end': epochs,
+            'on_train_batch_begin': epochs * num_batch_call_per_epoch,
+            'on_train_batch_end': epochs * num_batch_call_per_epoch,
+            'on_train_begin': 1,
+            'on_train_end': 1
+        })
+
+  @combinations.generate(keras_test_lib.all_strategy_combinations())
+  def test_callbacks_in_eval(self, distribution):
+    with distribution.scope():
+      model = keras_test_lib.get_model()
+      model.compile(optimizer='sgd', loss='mse', metrics=['mae'])
+
+    dataset = keras_test_lib.get_dataset(distribution)
+    counter = Counter()
+
+    model.evaluate(dataset, steps=5, callbacks=[counter])
+
+    self.assertDictEqual(
+        counter.method_counts, {
+            'on_test_batch_begin': 5,
+            'on_test_batch_end': 5,
+            'on_test_begin': 1,
+            'on_test_end': 1
+        })
+
+  @combinations.generate(keras_test_lib.all_strategy_combinations())
+  def test_callbacks_in_predict(self, distribution):
+    with distribution.scope():
+      model = keras_test_lib.get_model()
+      model.compile(optimizer='sgd', loss='mse', metrics=['mae'])
+
+    dataset = keras_test_lib.get_dataset(distribution)
+    counter = Counter()
+
+    model.predict(
+        keras_test_lib.get_predict_dataset(dataset),
+        steps=5,
+        callbacks=[counter])
+
+    self.assertDictEqual(
+        counter.method_counts, {
+            'on_predict_batch_begin': 5,
+            'on_predict_batch_end': 5,
+            'on_predict_begin': 1,
+            'on_predict_end': 1
+        })
+
+
+class TestDistributionStrategyErrorCases(test.TestCase, parameterized.TestCase):
+
+  @combinations.generate(
+      combinations.combine(
+          distribution=[
+              combinations.mirrored_strategy_with_gpu_and_cpu,
+              combinations.core_mirrored_strategy_with_gpu_and_cpu
+          ],
+          mode=['graph', 'eager']))
+  def test_validating_dataset_input_tensors_with_shape_mismatch(
+      self, distribution):
+    with self.cached_session():
+      a = constant_op.constant([1, 2], shape=(1, 2))
+      b = constant_op.constant([[1, 2], [1, 2]], shape=(2, 2))
+      device_map = values.ReplicaDeviceMap(('/device:CPU:0', '/device:GPU:0'))
+      x = values.DistributedValues(device_map, (a, b))
+      y = values.DistributedValues(device_map, (a, a))
+      # Removed device and input tensor shape details from the error message
+      # since the order of the device and the corresponding input tensor shape
+      # is not deterministic over different runs.
+      with self.assertRaisesRegexp(
+          ValueError, 'Input tensor shapes do not match for '
+          'distributed tensor inputs '
+          'DistributedValues:.+'):
+        with distribution.scope():
+          distributed_training_utils.validate_distributed_dataset_inputs(
+              distribution, x, y)
+
+  @combinations.generate(
+      combinations.combine(
+          distribution=[
+              combinations.mirrored_strategy_with_gpu_and_cpu,
+              combinations.core_mirrored_strategy_with_gpu_and_cpu
+          ],
+          mode=['graph', 'eager']))
+  def test_validating_dataset_input_tensors_with_dtype_mismatch(
+      self, distribution):
+    with self.cached_session():
+      a = constant_op.constant([1, 2], shape=(1, 2), dtype=dtypes.int32)
+      b = constant_op.constant([1, 2], shape=(1, 2), dtype=dtypes.float64)
+      device_map = values.ReplicaDeviceMap(('/device:CPU:0', '/device:GPU:0'))
+      x = values.DistributedValues(device_map, (a, b))
+      y = values.DistributedValues(device_map, (a, a))
+      # Removed device and input tensor dtype details from the error message
+      # since the order of the device and the corresponding input tensor dtype
+      # is not deterministic over different runs.
+      with self.assertRaisesRegexp(
+          ValueError, 'Input tensor dtypes do not match for '
+          'distributed tensor inputs '
+          'DistributedValues:.+'):
+        with distribution.scope():
+          distributed_training_utils.validate_distributed_dataset_inputs(
+              distribution, x, y)
+
+  @combinations.generate(
+      combinations.combine(
+          distribution=[
+              combinations.mirrored_strategy_with_gpu_and_cpu,
+              combinations.core_mirrored_strategy_with_gpu_and_cpu
+          ],
+          mode=['graph', 'eager']))
+  def test_unsupported_features(self, distribution):
+    with self.cached_session():
+      with distribution.scope():
+        model = keras_test_lib.get_model()
+        optimizer = gradient_descent.GradientDescentOptimizer(0.001)
+        loss = 'mse'
+        metrics = ['mae']
+        model.compile(optimizer, loss, metrics=metrics)
+
+      dataset = keras_test_lib.get_dataset(distribution)
+
+      # Test with validation split
+      with self.assertRaisesRegexp(
+          ValueError, '`validation_split` argument is not '
+          'supported when input `x` is a dataset or a '
+          'dataset iterator.+'):
+        model.fit(
+            dataset,
+            epochs=1,
+            steps_per_epoch=2,
+            verbose=0,
+            validation_split=0.5,
+            validation_steps=2)
+
+      # Test with sample weight.
+      sample_weight = np.random.random((10,))
+      with self.assertRaisesRegexp(
+          ValueError, '`sample_weight` argument is not supported when input '
+          '`x` is a dataset or a dataset iterator.'):
+        model.fit(
+            dataset,
+            epochs=1,
+            steps_per_epoch=2,
+            verbose=0,
+            sample_weight=sample_weight)
+
+      # Test with not specifying the `steps` argument for dataset with infinite
+      # cardinality.
+      dataset = dataset.repeat()
+      with self.assertRaisesRegexp(
+          ValueError, 'When passing an infinitely '
+          'repeating dataset, you must specify the '
+          '`steps_per_epoch` argument'):
+        model.fit(dataset, epochs=1, verbose=0)
+      with self.assertRaisesRegexp(
+          ValueError, 'When passing an infinitely '
+          'repeating dataset, you must specify the '
+          '`steps` argument'):
+        model.evaluate(dataset, verbose=0)
+
+      with self.assertRaisesRegexp(
+          ValueError, 'When passing an infinitely '
+          'repeating dataset, you must specify the '
+          '`steps` argument'):
+        model.predict(dataset, verbose=0)
+
+  @combinations.generate(
+      combinations.combine(
+          distribution=[
+              combinations.mirrored_strategy_with_gpu_and_cpu,
+              combinations.core_mirrored_strategy_with_gpu_and_cpu
+          ],
+          mode=['graph', 'eager']))
+  def test_calling_with_unsupported_predefined_callbacks(self, distribution):
+    with self.cached_session():
+      with distribution.scope():
+        model = keras_test_lib.get_model()
+        optimizer = gradient_descent.GradientDescentOptimizer(0.001)
+        loss = 'mse'
+        metrics = ['mae']
+        model.compile(optimizer, loss, metrics=metrics)
+
+      dataset = keras_test_lib.get_dataset(distribution)
+
+      def schedule(_):
+        return 0.001
+
+      with self.assertRaisesRegexp(
+          ValueError, 'You must specify a Keras Optimizer V2 when '
+          'using'):
+        model.fit(
+            dataset,
+            epochs=1,
+            steps_per_epoch=2,
+            verbose=0,
+            callbacks=[keras.callbacks.LearningRateScheduler(schedule)])
+
+      with self.assertRaisesRegexp(
+          ValueError, 'You must specify a Keras Optimizer V2 when '
+          'using'):
+        model.fit(
+            dataset,
+            epochs=1,
+            steps_per_epoch=2,
+            verbose=0,
+            callbacks=[keras.callbacks.ReduceLROnPlateau()])
+
+
+class TestDistributionStrategyWithLossMasking(test.TestCase,
+                                              parameterized.TestCase):
+
+  # TODO(priyag): Enable all strategies for this test. Currently it does not
+  # work for TPU due to some invalid datatype.
+  @combinations.generate(
+      combinations.combine(
+          distribution=[
+              combinations.mirrored_strategy_with_gpu_and_cpu,
+              combinations.core_mirrored_strategy_with_gpu_and_cpu
+          ],
+          mode=['graph', 'eager']))
+  def test_masking(self, distribution):
+    with self.cached_session():
+      np.random.seed(1337)
+      x = np.array([[[1], [1]], [[0], [0]]])
+      with distribution.scope():
+        model = keras.models.Sequential()
+        model.add(keras.layers.Masking(mask_value=0, input_shape=(2, 1)))
+        model.add(
+            keras.layers.TimeDistributed(
+                keras.layers.Dense(1, kernel_initializer='one')))
+        model.compile(
+            loss='mse',
+            optimizer=gradient_descent.GradientDescentOptimizer(0.01))
+      y = np.array([[[1], [1]], [[1], [1]]])
+      dataset = dataset_ops.Dataset.from_tensor_slices((x, y))
+      dataset = dataset.repeat(100)
+      dataset = dataset.batch(10)
+      hist = model.fit(x=dataset, epochs=1, steps_per_epoch=2)
+      self.assertEqual(hist.history['loss'][0], 0)
+
+
+class TestDistributionStrategyWithNormalizationLayer(test.TestCase,
+                                                     parameterized.TestCase):
+
+  @combinations.generate(
+      combinations.times(keras_test_lib.all_strategy_combinations(),
+                         combinations.combine(fused=[True, False])))
+  def test_batchnorm_correctness(self, distribution, fused):
+    with self.cached_session():
+      with distribution.scope():
+        model = keras.models.Sequential()
+        norm = keras.layers.BatchNormalization(
+            input_shape=(10,), momentum=0.8, fused=fused)
+        model.add(norm)
+        model.compile(
+            loss='mse',
+            optimizer=gradient_descent.GradientDescentOptimizer(0.01))
+
+      # centered on 5.0, variance 10.0
+      x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 10))
+      x = x.astype('float32')
+      dataset = dataset_ops.Dataset.from_tensor_slices((x, x))
+      dataset = dataset.repeat(100)
+      dataset = keras_test_lib.batch_wrapper(dataset, 32, distribution)
+
+      predict_dataset = dataset_ops.Dataset.from_tensor_slices(x)
+      predict_dataset = predict_dataset.repeat(100)
+      predict_dataset = keras_test_lib.batch_wrapper(predict_dataset, 32,
+                                                     distribution)
+
+      model.fit(dataset, epochs=4, verbose=0, steps_per_epoch=10)
+      out = model.predict(predict_dataset, steps=2)
+      out -= keras.backend.eval(norm.beta)
+      out /= keras.backend.eval(norm.gamma)
+      np.testing.assert_allclose(out.mean(), 0.0, atol=1e-1)
+      np.testing.assert_allclose(out.std(), 1.0, atol=1e-1)
+
+
+class TestDistributionStrategySaveLoadWeights(test.TestCase,
+                                              parameterized.TestCase):
+
+  @combinations.generate(
+      keras_test_lib.all_strategy_combinations_minus_default())
+  def test_save_load_h5(self, distribution):
+    with self.cached_session():
+      dataset = keras_test_lib.get_dataset(distribution)
+      with distribution.scope():
+        model = keras_test_lib.get_model()
+        model.compile(gradient_descent_keras.SGD(0.01), 'mse')
+        model.fit(dataset, epochs=1, steps_per_epoch=1)
+
+        weights_file = tempfile.mktemp('.h5')
+        model.save_weights(weights_file)
+
+        model_2 = keras_test_lib.get_model()
+        model_2.compile(gradient_descent_keras.SGD(0.01), 'mse')
+        model_2.load_weights(weights_file)
+        model_2.predict(
+            keras_test_lib.get_predict_dataset(distribution), steps=2)
+        model_2.fit(dataset, epochs=1, steps_per_epoch=1)
+
+  @combinations.generate(
+      keras_test_lib.all_strategy_combinations_minus_default())
+  def test_save_load_checkpointable(self, distribution):
+    # TODO(sourabhbajaj): Test fails with optimizer v2 without h5
+    with self.cached_session():
+      dataset = keras_test_lib.get_dataset(distribution)
+      with distribution.scope():
+        model = keras_test_lib.get_model()
+        model.compile(gradient_descent.GradientDescentOptimizer(0.01), 'mse')
+        model.fit(dataset, epochs=1, steps_per_epoch=1)
+
+        weights_file = tempfile.mktemp()
+        model.save_weights(weights_file)
+
+        model_2 = keras_test_lib.get_model()
+        model_2.compile(gradient_descent.GradientDescentOptimizer(0.01), 'mse')
+        model_2.load_weights(weights_file)
+        model_2.predict(
+            keras_test_lib.get_predict_dataset(distribution), steps=2)
+        model_2.fit(dataset, epochs=1, steps_per_epoch=1)
+
+
+class TestDistributionStrategyValidation(test.TestCase, parameterized.TestCase):
+
+  @combinations.generate(
+      keras_test_lib.all_strategy_combinations_minus_default())
+  def test_layer_outside_scope(self, distribution):
+    with self.cached_session():
+      with self.assertRaisesRegexp(
+          ValueError, 'was not created in the distribution strategy'):
+        x = keras.layers.Input(shape=(3,), name='input')
+        y = keras.layers.Dense(4, name='dense')(x)
+        with distribution.scope():
+          model = keras.Model(x, y)
+          optimizer = gradient_descent.GradientDescentOptimizer(0.001)
+          loss = 'mse'
+          metrics = ['mae', keras.metrics.CategoricalAccuracy()]
+          model.compile(optimizer, loss, metrics=metrics)
+
+  @combinations.generate(
+      keras_test_lib.all_strategy_combinations_minus_default())
+  def test_model_outside_scope(self, distribution):
+    with self.cached_session():
+      with self.assertRaisesRegexp(
+          ValueError, 'was not created in the distribution strategy'):
+        x = keras.layers.Input(shape=(3,), name='input')
+        y = keras.layers.Dense(4, name='dense')(x)
+        model = keras.Model(x, y)
+        with distribution.scope():
+          optimizer = gradient_descent.GradientDescentOptimizer(0.001)
+          loss = 'mse'
+          metrics = ['mae', keras.metrics.CategoricalAccuracy()]
+          model.compile(optimizer, loss, metrics=metrics)
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
index 41d813e087..907fe3fe99 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
@@ -648,7 +648,8 @@ class OptimizerV2(checkpointable.Checkpointable):
     if callable(value):
       return value()
     if isinstance(value, (ops.Tensor, tf_variables.Variable,
-                          distributed_values.TPUMirroredVariable)):
+                          distributed_values.TPUMirroredVariable,
+                          distributed_values.DistributedVariable)):
       return backend.get_value(value)
     return value
 
diff --git a/tensorflow/python/training/checkpointable/base.py b/tensorflow/python/training/checkpointable/base.py
index 0d659cef20..c3ff78d4e3 100644
--- a/tensorflow/python/training/checkpointable/base.py
+++ b/tensorflow/python/training/checkpointable/base.py
@@ -850,13 +850,10 @@ class Checkpointable(object):
       """Serializes `self.get_config()` for saving."""
       dereferenced_self = weak_self()
       if dereferenced_self:
-        try:
-          return json.dumps(dereferenced_self,
-                            default=serialization.get_json_type,
-                            sort_keys=True).encode("utf8")
-        except TypeError:
-          # Even if get_config worked objects may have produced garbage.
-          return ""
+        return json.dumps(
+            dereferenced_self,
+            default=serialization.get_json_type,
+            sort_keys=True).encode("utf8")
       else:
         return ""
     return {OBJECT_CONFIG_JSON_KEY: functools.partial(
diff --git a/tensorflow/python/training/checkpointable/base_test.py b/tensorflow/python/training/checkpointable/base_test.py
index 5863f5a560..750799f030 100644
--- a/tensorflow/python/training/checkpointable/base_test.py
+++ b/tensorflow/python/training/checkpointable/base_test.py
@@ -83,19 +83,6 @@ class InterfaceTests(test.TestCase):
     with self.assertRaisesRegexp(AssertionError, "foo_attr"):
       status.assert_consumed()
 
-  def testBuggyGetConfig(self):
-
-    class NotSerializable(object):
-      pass
-
-    class GetConfigRaisesError(base.Checkpointable):
-
-      def get_config(self):
-        return NotSerializable()
-
-    util.Checkpoint(obj=GetConfigRaisesError()).save(
-        os.path.join(self.get_temp_dir(), "ckpt"))
-
 
 if __name__ == "__main__":
   ops.enable_eager_execution()
-- 
GitLab


From d0b874f14b08a752529c7d203ccb43dddc0e7e14 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 11 Feb 2019 18:37:59 -0800
Subject: [PATCH 2300/2345] Automated rollback of commit
 4bdb229d2b1f1be776615875085df0a47927862c

PiperOrigin-RevId: 233520400
---
 tensorflow/core/common_runtime/gpu/gpu_device.cc | 6 ------
 tensorflow/core/protobuf/config.proto            | 4 +---
 2 files changed, 1 insertion(+), 9 deletions(-)

diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.cc b/tensorflow/core/common_runtime/gpu/gpu_device.cc
index 6aae89c1fe..179f6132fa 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.cc
@@ -277,12 +277,6 @@ BaseGPUDevice::BaseGPUDevice(const SessionOptions& options, const string& name,
       max_streams_(max_streams) {
   GPUProcessState::singleton()->EnableGPUDevice();
   pending_cap_ = options.config.gpu_options().experimental().pending_cap();
-  if (pending_cap_ == 0) {
-    // Auto-configure.  Testing with benchmark models suggests that the optimal
-    // pending_cap value varies by model but something in the range [2, 16] is
-    // always better than unlimited.
-    pending_cap_ = 8;
-  }
   timestamped_allocator_ =
       options.config.gpu_options().experimental().timestamped_allocator();
   if (timestamped_allocator_ || pending_cap_ > 0) {
diff --git a/tensorflow/core/protobuf/config.proto b/tensorflow/core/protobuf/config.proto
index 6e9c3c2e27..44e98542ec 100644
--- a/tensorflow/core/protobuf/config.proto
+++ b/tensorflow/core/protobuf/config.proto
@@ -164,9 +164,7 @@ message GPUOptions {
     bool timestamped_allocator = 5;
 
     // If > 0 limit the number of pending kernels on any compute
-    // stream to this number.  If == 0 then the runtime will automatically
-    // pick a value.  If == -1 then this feature will be disabled, i.e.
-    // there will be no limit.
+    // stream to this number.
     int32 pending_cap = 6;
   }
 
-- 
GitLab


From 07cea8cc4ca430ded136bef94f247ffbd1207c46 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Mon, 11 Feb 2019 18:46:25 -0800
Subject: [PATCH 2301/2345] Re-export py_func as numpy_function for V2

PiperOrigin-RevId: 233521324
---
 tensorflow/python/ops/script_ops.py             | 15 +++++++++++++--
 tensorflow/tools/api/golden/v2/tensorflow.pbtxt |  4 ++++
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/script_ops.py b/tensorflow/python/ops/script_ops.py
index a5b31aff91..63b5eab56a 100644
--- a/tensorflow/python/ops/script_ops.py
+++ b/tensorflow/python/ops/script_ops.py
@@ -391,12 +391,16 @@ def eager_py_func(func, inp, Tout, name=None):
 
 @deprecation.deprecated(
     date=None,
-    instructions="""tf.py_func is deprecated in TF V2. Instead, use
-    tf.py_function, which takes a python function which manipulates tf eager
+    instructions="""tf.py_func is deprecated in TF V2. Instead, there are two
+    options available in V2.
+    - tf.py_function takes a python function which manipulates tf eager
     tensors instead of numpy arrays. It's easy to convert a tf eager tensor to
     an ndarray (just call tensor.numpy()) but having access to eager tensors
     means `tf.py_function`s can use accelerators such as GPUs as well as
     being differentiable using a gradient tape.
+    - tf.numpy_function maintains the semantics of the deprecated tf.py_func
+    (it is not differentiable, and manipulates numpy arrays). It drops the
+    stateful argument making all functions stateful.
     """)
 @tf_export(v1=["py_func"])
 def py_func(func, inp, Tout, stateful=True, name=None):
@@ -467,6 +471,13 @@ def py_func(func, inp, Tout, stateful=True, name=None):
   return _internal_py_func(
       func=func, inp=inp, Tout=Tout, stateful=stateful, eager=False, name=name)
 
+@tf_export("numpy_function", v1=[])
+def numpy_function(func, inp, Tout, name=None):
+  return py_func(func, inp, Tout, stateful=True, name=name)
+
+numpy_function.__doc__ = py_func.__doc__.replace(
+    "py_func", "numpy_function")
+
 
 ops.NotDifferentiable("PyFunc")
 ops.NotDifferentiable("PyFuncStateless")
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 5db60220c7..2a7813da14 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -776,6 +776,10 @@ tf_module {
     name: "not_equal"
     argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "numpy_function"
+    argspec: "args=[\'func\', \'inp\', \'Tout\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "one_hot"
     argspec: "args=[\'indices\', \'depth\', \'on_value\', \'off_value\', \'axis\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
-- 
GitLab


From 078228441f09ca07c8060de23167c6c08381de8a Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Mon, 11 Feb 2019 20:37:26 -0800
Subject: [PATCH 2302/2345] Create int8 batch_to_space_nd.

PiperOrigin-RevId: 233532684
---
 tensorflow/lite/kernels/batch_to_space_nd.cc  |  7 +++
 .../lite/kernels/batch_to_space_nd_test.cc    | 63 ++++++++++++++-----
 tensorflow/lite/kernels/register.cc           |  4 +-
 tensorflow/lite/toco/tflite/operator.cc       |  6 ++
 tensorflow/lite/toco/tflite/operator_test.cc  |  4 ++
 5 files changed, 67 insertions(+), 17 deletions(-)

diff --git a/tensorflow/lite/kernels/batch_to_space_nd.cc b/tensorflow/lite/kernels/batch_to_space_nd.cc
index 34fdf34f70..ce85aedded 100644
--- a/tensorflow/lite/kernels/batch_to_space_nd.cc
+++ b/tensorflow/lite/kernels/batch_to_space_nd.cc
@@ -148,6 +148,13 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
         TF_LITE_BATCH_TO_SPACE_ND(optimized_ops, uint8_t);
       }
       break;
+    case kTfLiteInt8:
+      if (kernel_type == kReference) {
+        TF_LITE_BATCH_TO_SPACE_ND(reference_ops, int8_t);
+      } else {
+        TF_LITE_BATCH_TO_SPACE_ND(optimized_ops, int8_t);
+      }
+      break;
     case kTfLiteInt32:
       if (kernel_type == kReference) {
         TF_LITE_BATCH_TO_SPACE_ND(reference_ops, int32_t);
diff --git a/tensorflow/lite/kernels/batch_to_space_nd_test.cc b/tensorflow/lite/kernels/batch_to_space_nd_test.cc
index f330895599..bd806b55ca 100644
--- a/tensorflow/lite/kernels/batch_to_space_nd_test.cc
+++ b/tensorflow/lite/kernels/batch_to_space_nd_test.cc
@@ -26,8 +26,9 @@ using ::testing::ElementsAreArray;
 
 class BatchToSpaceNDOpModel : public SingleOpModel {
  public:
-  void SetInput(std::initializer_list<float> data) {
-    PopulateTensor<float>(input_, data);
+  template <typename T>
+  void SetInput(std::initializer_list<T> data) {
+    PopulateTensor<T>(input_, data);
   }
 
   void SetBlockShape(std::initializer_list<int> data) {
@@ -38,7 +39,10 @@ class BatchToSpaceNDOpModel : public SingleOpModel {
     PopulateTensor<int>(crops_, data);
   }
 
-  std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
+  template <typename T>
+  std::vector<T> GetOutput() {
+    return ExtractVector<T>(output_);
+  }
   std::vector<int> GetOutputShape() { return GetTensorShape(output_); }
 
  protected:
@@ -58,11 +62,12 @@ class BatchToSpaceNDOpConstModel : public BatchToSpaceNDOpModel {
  public:
   BatchToSpaceNDOpConstModel(std::initializer_list<int> input_shape,
                              std::initializer_list<int> block_shape,
-                             std::initializer_list<int> crops) {
-    input_ = AddInput(TensorType_FLOAT32);
+                             std::initializer_list<int> crops,
+                             const TensorType& type = TensorType_FLOAT32) {
+    input_ = AddInput(type);
     block_shape_ = AddConstInput(TensorType_INT32, block_shape, {2});
     crops_ = AddConstInput(TensorType_INT32, crops, {2, 2});
-    output_ = AddOutput(TensorType_FLOAT32);
+    output_ = AddOutput(type);
 
     SetBuiltinOp(BuiltinOperator_BATCH_TO_SPACE_ND,
                  BuiltinOptions_BatchToSpaceNDOptions,
@@ -81,11 +86,12 @@ class BatchToSpaceNDOpConstModel : public BatchToSpaceNDOpModel {
 //    m.Invoke();
 class BatchToSpaceNDOpDynamicModel : public BatchToSpaceNDOpModel {
  public:
-  BatchToSpaceNDOpDynamicModel(std::initializer_list<int> input_shape) {
-    input_ = AddInput(TensorType_FLOAT32);
+  BatchToSpaceNDOpDynamicModel(std::initializer_list<int> input_shape,
+                               const TensorType& type = TensorType_FLOAT32) {
+    input_ = AddInput(type);
     block_shape_ = AddInput(TensorType_INT32);
     crops_ = AddInput(TensorType_INT32);
-    output_ = AddOutput(TensorType_FLOAT32);
+    output_ = AddOutput(type);
 
     SetBuiltinOp(BuiltinOperator_BATCH_TO_SPACE_ND,
                  BuiltinOptions_BatchToSpaceNDOptions,
@@ -96,22 +102,47 @@ class BatchToSpaceNDOpDynamicModel : public BatchToSpaceNDOpModel {
 
 TEST(BatchToSpaceNDOpTest, SimpleConstTest) {
   BatchToSpaceNDOpConstModel m({4, 2, 2, 1}, {2, 2}, {0, 0, 0, 0});
-  m.SetInput({1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16});
+  m.SetInput<float>({1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 4, 4, 1}));
+  EXPECT_THAT(m.GetOutput<float>(),
+              ElementsAreArray(
+                  {1, 5, 2, 6, 9, 13, 10, 14, 3, 7, 4, 8, 11, 15, 12, 16}));
+}
+
+TEST(BatchToSpaceNDOpTest, SimpleConstTestInt8) {
+  BatchToSpaceNDOpConstModel m({4, 2, 2, 1}, {2, 2}, {0, 0, 0, 0},
+                               TensorType_INT8);
+  m.SetInput<int8_t>({1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16});
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 4, 4, 1}));
-  EXPECT_THAT(m.GetOutput(), ElementsAreArray({1, 5, 2, 6, 9, 13, 10, 14, 3, 7,
-                                               4, 8, 11, 15, 12, 16}));
+  EXPECT_THAT(m.GetOutput<int8_t>(),
+              ElementsAreArray(
+                  {1, 5, 2, 6, 9, 13, 10, 14, 3, 7, 4, 8, 11, 15, 12, 16}));
 }
 
 TEST(BatchToSpaceNDOpTest, SimpleDynamicTest) {
   BatchToSpaceNDOpDynamicModel m({4, 2, 2, 1});
-  m.SetInput({1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16});
+  m.SetInput<float>({1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16});
+  m.SetBlockShape({2, 2});
+  m.SetCrops({0, 0, 0, 0});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 4, 4, 1}));
+  EXPECT_THAT(m.GetOutput<float>(),
+              ElementsAreArray(
+                  {1, 5, 2, 6, 9, 13, 10, 14, 3, 7, 4, 8, 11, 15, 12, 16}));
+}
+
+TEST(BatchToSpaceNDOpTest, SimpleDynamicTestInt8) {
+  BatchToSpaceNDOpDynamicModel m({4, 2, 2, 1}, TensorType_INT8);
+  m.SetInput<int8_t>({1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16});
   m.SetBlockShape({2, 2});
   m.SetCrops({0, 0, 0, 0});
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 4, 4, 1}));
-  EXPECT_THAT(m.GetOutput(), ElementsAreArray({1, 5, 2, 6, 9, 13, 10, 14, 3, 7,
-                                               4, 8, 11, 15, 12, 16}));
+  EXPECT_THAT(m.GetOutput<int8_t>(),
+              ElementsAreArray(
+                  {1, 5, 2, 6, 9, 13, 10, 14, 3, 7, 4, 8, 11, 15, 12, 16}));
 }
 
 #ifdef GTEST_HAS_DEATH_TEST
@@ -127,7 +158,7 @@ TEST(BatchToSpaceNDOpTest, InvalidCropsConstTest) {
 
 TEST(BatchToSpaceNDOpTest, InvalidCropsDynamicTest) {
   BatchToSpaceNDOpDynamicModel m({4, 2, 2, 1});
-  m.SetInput({1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16});
+  m.SetInput<float>({1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16});
   m.SetBlockShape({2, 2});
   m.SetCrops({0, 0, -1, 0});
   EXPECT_DEATH(m.Invoke(), "crops.2. >= 0 was not true.");
diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index 67c4a0bc4f..2290ed477c 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -215,7 +215,9 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_SPACE_TO_BATCH_ND, Register_SPACE_TO_BATCH_ND(),
              /* min_version */ 1,
              /* max_version */ 2);
-  AddBuiltin(BuiltinOperator_BATCH_TO_SPACE_ND, Register_BATCH_TO_SPACE_ND());
+  AddBuiltin(BuiltinOperator_BATCH_TO_SPACE_ND, Register_BATCH_TO_SPACE_ND(),
+             /* min_version */ 1,
+             /* max_version */ 2);
   AddBuiltin(BuiltinOperator_MUL, Register_MUL());
   AddBuiltin(BuiltinOperator_L2_NORMALIZATION, Register_L2_NORMALIZATION());
   AddBuiltin(BuiltinOperator_LOCAL_RESPONSE_NORMALIZATION,
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index 34237cd1e3..961bda2864 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -314,6 +314,12 @@ class BatchToSpaceND
                    TocoOperator* op) const override {}
 
   int GetVersion(const OperatorSignature& op_signature) const override {
+    const string& input_name = op_signature.op->inputs[0];
+    const Array& input_array = op_signature.model->GetArray(input_name);
+    // If the op take int8 input, it is version 2.
+    if (input_array.data_type == ArrayDataType::kInt8) {
+      return 2;
+    }
     return 1;
   }
 };
diff --git a/tensorflow/lite/toco/tflite/operator_test.cc b/tensorflow/lite/toco/tflite/operator_test.cc
index bfe8af682c..b977270bda 100644
--- a/tensorflow/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/lite/toco/tflite/operator_test.cc
@@ -808,6 +808,10 @@ TEST_F(OperatorTest, VersioningPackTest) {
   SimpleVersioningTest<PackOperator>();
 }
 
+TEST_F(OperatorTest, VersioningBatchToSpaceNDTest) {
+  SimpleVersioningTest<BatchToSpaceNDOperator>();
+}
+
 TEST_F(OperatorTest, VersioningStridedSliceTest) {
   SimpleVersioningTest<StridedSliceOperator>();
 }
-- 
GitLab


From 7350b35e27ee063bf4b2d81ffde612e64f668a27 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 11 Feb 2019 21:33:03 -0800
Subject: [PATCH 2303/2345] Improve our handling of debug options flags

 - Set them from the environment variable in replay_computation
 - Add some addition VLOG in parse_flags_from_env

PiperOrigin-RevId: 233537896
---
 tensorflow/compiler/xla/parse_flags_from_env.cc     |  8 ++++++++
 tensorflow/compiler/xla/tools/replay_computation.cc | 10 +++++++---
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/parse_flags_from_env.cc b/tensorflow/compiler/xla/parse_flags_from_env.cc
index 91e71f5d1d..e1e22f7841 100644
--- a/tensorflow/compiler/xla/parse_flags_from_env.cc
+++ b/tensorflow/compiler/xla/parse_flags_from_env.cc
@@ -186,6 +186,14 @@ bool ParseFlagsFromEnvAndDieIfUnknown(
   tensorflow::mutex_lock lock(env_argv_mu);
   auto* env_argv = &EnvArgvs()[string(envvar)];
   SetArgvFromEnv(envvar, env_argv);  // a no-op if already initialized
+
+  if (VLOG_IS_ON(1)) {
+    VLOG(1) << "For env var " << envvar << " found arguments:";
+    for (int i = 0; i < env_argv->argc; i++) {
+      VLOG(1) << "  argv[" << i << "] = " << env_argv->argv[i];
+    }
+  }
+
   bool result =
       tensorflow::Flags::Parse(&env_argv->argc, &env_argv->argv[0], flag_list);
 
diff --git a/tensorflow/compiler/xla/tools/replay_computation.cc b/tensorflow/compiler/xla/tools/replay_computation.cc
index 21217c23f6..d66561315b 100644
--- a/tensorflow/compiler/xla/tools/replay_computation.cc
+++ b/tensorflow/compiler/xla/tools/replay_computation.cc
@@ -102,8 +102,9 @@ StatusOr<std::unique_ptr<LocalExecutable>> CompileExecutable(
     argument_layouts.push_back(Shape(param));
     argument_layout_ptrs.push_back(&argument_layouts.back());
   }
-  return client->Compile(computation, argument_layout_ptrs,
-                         ExecutableBuildOptions());
+  ExecutableBuildOptions exec_build_options;
+  *exec_build_options.mutable_debug_options() = GetDebugOptionsFromFlags();
+  return client->Compile(computation, argument_layout_ptrs, exec_build_options);
 }
 
 absl::optional<Shape> GetXfeedShape(bool is_infeed,
@@ -328,7 +329,10 @@ StatusOr<HloSnapshot> ParseInputFile(const string& filename,
   fprintf(stderr, "%s: is not HloProto. Trying HLO text.\n", filename.c_str());
   string contents;
   TF_RETURN_IF_ERROR(tensorflow::ReadFileToString(env, filename, &contents));
-  StatusOr<std::unique_ptr<HloModule>> module = ParseHloString(contents);
+  HloModuleConfig config;
+  config.set_debug_options(GetDebugOptionsFromFlags());
+  StatusOr<std::unique_ptr<HloModule>> module =
+      ParseHloString(contents, config);
   if (module.ok()) {
     *snapshot.mutable_hlo()->mutable_hlo_module() =
         module.ValueOrDie()->ToProto();
-- 
GitLab


From 27e04b6a92461c9429d6506bd1312c6b1be47258 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Mon, 11 Feb 2019 21:57:22 -0800
Subject: [PATCH 2304/2345] Bugfix - TFLite WHILE/IF: Ensure subgraph outputs
 is readable

PiperOrigin-RevId: 233540108
---
 tensorflow/lite/interpreter.cc   | 2 --
 tensorflow/lite/kernels/if.cc    | 4 ++++
 tensorflow/lite/kernels/while.cc | 9 ++++++++-
 3 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/tensorflow/lite/interpreter.cc b/tensorflow/lite/interpreter.cc
index c840c9a759..75a23deb14 100644
--- a/tensorflow/lite/interpreter.cc
+++ b/tensorflow/lite/interpreter.cc
@@ -223,8 +223,6 @@ void Interpreter::SetCancellationFunction(void* data,
 }
 
 TfLiteStatus Interpreter::ModifyGraphWithDelegate(TfLiteDelegate* delegate) {
-  // TODO(ycling): It seems Flex delegate doesn't work on non-primary subgraphs.
-  // Need to investigate.
   for (auto& subgraph : subgraphs_) {
     TF_LITE_ENSURE_OK(context_, subgraph->ModifyGraphWithDelegate(delegate));
   }
diff --git a/tensorflow/lite/kernels/if.cc b/tensorflow/lite/kernels/if.cc
index a814ac997d..1bd394e980 100644
--- a/tensorflow/lite/kernels/if.cc
+++ b/tensorflow/lite/kernels/if.cc
@@ -153,6 +153,10 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   // in `Prepare`, so we don't need to do it here again.
   TF_LITE_ENSURE_OK(context, active_branch_subgraph.Invoke());
 
+  for (int tensor_index : active_branch_subgraph.outputs()) {
+    active_branch_subgraph.EnsureTensorDataIsReadable(tensor_index);
+  }
+
   bool has_dynamic_output_tensors = false;
   for (int i = 0; i < node->outputs->size; ++i) {
     TfLiteTensor* output = GetOutput(context, node, i);
diff --git a/tensorflow/lite/kernels/while.cc b/tensorflow/lite/kernels/while.cc
index b8a6e1953b..711caf8f92 100644
--- a/tensorflow/lite/kernels/while.cc
+++ b/tensorflow/lite/kernels/while.cc
@@ -222,8 +222,11 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
 
   while (true) {
     TF_LITE_ENSURE_OK(context, cond_subgraph->Invoke());
+    int cond_subgraph_output_index = cond_subgraph->outputs()[0];
+    cond_subgraph->EnsureTensorDataIsReadable(cond_subgraph_output_index);
     TfLiteTensor* cond_output =
-        cond_subgraph->tensor(cond_subgraph->outputs()[0]);
+        cond_subgraph->tensor(cond_subgraph_output_index);
+
     if (!cond_output->data.b[0]) {
       break;
     }
@@ -242,6 +245,10 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
 
     TF_LITE_ENSURE_OK(context, body_subgraph->Invoke());
 
+    for (int tensor_index : body_subgraph->outputs()) {
+      body_subgraph->EnsureTensorDataIsReadable(tensor_index);
+    }
+
     if (op_data->body_has_dynamic_output_tensors) {
       TF_LITE_ENSURE_OK(
           context,
-- 
GitLab


From d67977598e7fd722cc00a29ab5a8c674f5103042 Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Mon, 11 Feb 2019 23:16:51 -0800
Subject: [PATCH 2305/2345] Remove unnecessary import which users don't always
 depend on.

PiperOrigin-RevId: 233547217
---
 tensorflow/compiler/tf2tensorrt/python/ops/trt_ops.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/tf2tensorrt/python/ops/trt_ops.py b/tensorflow/compiler/tf2tensorrt/python/ops/trt_ops.py
index a11150f155..7503d4d984 100644
--- a/tensorflow/compiler/tf2tensorrt/python/ops/trt_ops.py
+++ b/tensorflow/compiler/tf2tensorrt/python/ops/trt_ops.py
@@ -41,9 +41,9 @@ def load_trt_ops():
     # TODO(laigd): we should load TF-TRT kernels here as well after removing the
     # swig binding.
     try:
-      # TODO(lagid): consider getting rid of these unused imports.
+      # TODO(lagid): It is not known why these unused imports were introduced.
+      # Investigate and get rid of these, if not required.
       # pylint: disable=unused-import,g-import-not-at-top,unused-variable
-      from tensorflow.compiler.tf2tensorrt.ops.gen_trt_ops import get_serialized_resource_op
       from tensorflow.compiler.tf2tensorrt.ops.gen_trt_ops import trt_engine_op
       from tensorflow.python.framework import load_library
       from tensorflow.python.platform import resource_loader
-- 
GitLab


From d84330946a9c49ea7970a5f83784b5ec52112ad1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 11 Feb 2019 23:45:18 -0800
Subject: [PATCH 2306/2345] Internal cleanups.

PiperOrigin-RevId: 233550085
---
 .../lite/experimental/examples/lstm/BUILD     |  18 +-
 .../lstm/bidirectional_sequence_lstm_test.py  |   2 +-
 .../lstm/bidirectional_sequence_rnn_test.py   |   2 +-
 .../lite/experimental/examples/lstm/rnn.py    | 273 ++++++++++++
 .../lstm/{tflite_lstm.py => rnn_cell.py}      | 395 ++++++------------
 .../experimental/examples/lstm/tflite_rnn.py  | 150 -------
 .../lstm/unidirectional_sequence_lstm_test.py |   4 +-
 .../lstm/unidirectional_sequence_rnn_test.py  |  43 +-
 .../tools/pip_package/pip_smoke_test.py       |   4 +-
 9 files changed, 460 insertions(+), 431 deletions(-)
 create mode 100644 tensorflow/lite/experimental/examples/lstm/rnn.py
 rename tensorflow/lite/experimental/examples/lstm/{tflite_lstm.py => rnn_cell.py} (58%)
 delete mode 100644 tensorflow/lite/experimental/examples/lstm/tflite_rnn.py

diff --git a/tensorflow/lite/experimental/examples/lstm/BUILD b/tensorflow/lite/experimental/examples/lstm/BUILD
index 827b10449b..ab6d5087d5 100644
--- a/tensorflow/lite/experimental/examples/lstm/BUILD
+++ b/tensorflow/lite/experimental/examples/lstm/BUILD
@@ -5,8 +5,8 @@ package(default_visibility = ["//tensorflow:internal"])
 load("//tensorflow:tensorflow.bzl", "py_test")
 
 py_library(
-    name = "tflite_lstm",
-    srcs = ["tflite_lstm.py"],
+    name = "rnn",
+    srcs = ["rnn.py"],
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
     deps = [
@@ -18,8 +18,8 @@ py_library(
 )
 
 py_library(
-    name = "tflite_rnn",
-    srcs = ["tflite_rnn.py"],
+    name = "rnn_cell",
+    srcs = ["rnn_cell.py"],
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
     deps = [
@@ -40,7 +40,8 @@ py_test(
         "no_pip",
     ],
     deps = [
-        ":tflite_lstm",
+        ":rnn",
+        ":rnn_cell",
         "//tensorflow:tensorflow_py",
         "//tensorflow/examples/tutorials/mnist:input_data",
         "//tensorflow/lite/python:lite",
@@ -62,7 +63,8 @@ py_test(
         "no_pip",
     ],
     deps = [
-        ":tflite_rnn",
+        ":rnn",
+        ":rnn_cell",
         "//tensorflow:tensorflow_py",
         "//tensorflow/examples/tutorials/mnist:input_data",
         "//tensorflow/lite/python:lite",
@@ -84,7 +86,7 @@ py_test(
         "no_pip",
     ],
     deps = [
-        ":tflite_lstm",
+        ":rnn_cell",
         "//tensorflow:tensorflow_py",
         "//tensorflow/examples/tutorials/mnist:input_data",
         "//tensorflow/lite/python:lite",
@@ -106,7 +108,7 @@ py_test(
         "no_pip",
     ],
     deps = [
-        ":tflite_rnn",
+        ":rnn_cell",
         "//tensorflow:tensorflow_py",
         "//tensorflow/examples/tutorials/mnist:input_data",
         "//tensorflow/lite/python:lite",
diff --git a/tensorflow/lite/experimental/examples/lstm/bidirectional_sequence_lstm_test.py b/tensorflow/lite/experimental/examples/lstm/bidirectional_sequence_lstm_test.py
index 9dc81096bc..77619807fa 100644
--- a/tensorflow/lite/experimental/examples/lstm/bidirectional_sequence_lstm_test.py
+++ b/tensorflow/lite/experimental/examples/lstm/bidirectional_sequence_lstm_test.py
@@ -20,7 +20,7 @@ import numpy as np
 import tensorflow as tf
 
 from tensorflow.examples.tutorials.mnist import input_data
-from tensorflow.lite.experimental.examples.lstm.tflite_lstm import TFLiteLSTMCell
+from tensorflow.lite.experimental.examples.lstm.rnn_cell import TFLiteLSTMCell
 from tensorflow.lite.python.op_hint import convert_op_hints_to_stubs
 from tensorflow.lite.python.op_hint import find_all_hinted_output_nodes
 from tensorflow.python.framework import test_util
diff --git a/tensorflow/lite/experimental/examples/lstm/bidirectional_sequence_rnn_test.py b/tensorflow/lite/experimental/examples/lstm/bidirectional_sequence_rnn_test.py
index 7a937ce47f..316d482696 100644
--- a/tensorflow/lite/experimental/examples/lstm/bidirectional_sequence_rnn_test.py
+++ b/tensorflow/lite/experimental/examples/lstm/bidirectional_sequence_rnn_test.py
@@ -22,7 +22,7 @@ import tensorflow as tf
 from tensorflow import flags
 
 from tensorflow.examples.tutorials.mnist import input_data
-from tensorflow.lite.experimental.examples.lstm.tflite_rnn import TfLiteRNNCell
+from tensorflow.lite.experimental.examples.lstm.rnn_cell import TfLiteRNNCell
 from tensorflow.lite.python.op_hint import convert_op_hints_to_stubs
 from tensorflow.lite.python.op_hint import find_all_hinted_output_nodes
 from tensorflow.python.framework import test_util
diff --git a/tensorflow/lite/experimental/examples/lstm/rnn.py b/tensorflow/lite/experimental/examples/lstm/rnn.py
new file mode 100644
index 0000000000..1076cd3906
--- /dev/null
+++ b/tensorflow/lite/experimental/examples/lstm/rnn.py
@@ -0,0 +1,273 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""TfLite LSTMCell wrapper.
+
+TODO(renjieliu): Find a better home for this one.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+import tensorflow as tf
+
+from tensorflow.python.eager import context
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import control_flow_util
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import rnn_cell_impl
+from tensorflow.python.ops import variable_scope as vs
+from tensorflow.python.ops.rnn import _best_effort_input_batch_size
+from tensorflow.python.ops.rnn import _dynamic_rnn_loop
+from tensorflow.python.ops.rnn import _should_cache
+from tensorflow.python.ops.rnn import _transpose_batch_time
+from tensorflow.python.util import nest
+
+
+def dynamic_rnn(cell,
+                inputs,
+                sequence_length=None,
+                initial_state=None,
+                dtype=None,
+                parallel_iterations=None,
+                swap_memory=False,
+                time_major=True,
+                scope=None):
+  """Creates a recurrent neural network specified by RNNCell `cell`.
+
+  Performs fully dynamic unrolling of `inputs`.
+
+  Example:
+
+  ```python
+  # create a BasicRNNCell
+  rnn_cell = tf.nn.rnn_cell.BasicRNNCell(hidden_size)
+
+  # 'outputs' is a tensor of shape [batch_size, max_time, cell_state_size]
+
+  # defining initial state
+  initial_state = rnn_cell.zero_state(batch_size, dtype=tf.float32)
+
+  # 'state' is a tensor of shape [batch_size, cell_state_size]
+  outputs, state = tf.nn.dynamic_rnn(rnn_cell, input_data,
+                                     initial_state=initial_state,
+                                     dtype=tf.float32)
+  ```
+
+  ```python
+  # create 2 LSTMCells
+  rnn_layers = [tf.nn.rnn_cell.LSTMCell(size) for size in [128, 256]]
+
+  # create a RNN cell composed sequentially of a number of RNNCells
+  multi_rnn_cell = tf.nn.rnn_cell.MultiRNNCell(rnn_layers)
+
+  # 'outputs' is a tensor of shape [batch_size, max_time, 256]
+  # 'state' is a N-tuple where N is the number of LSTMCells containing a
+  # tf.contrib.rnn.LSTMStateTuple for each cell
+  outputs, state = tf.nn.dynamic_rnn(cell=multi_rnn_cell,
+                                     inputs=data,
+                                     dtype=tf.float32)
+  ```
+
+
+  Args:
+    cell: An instance of RNNCell.
+    inputs: The RNN inputs.
+      If `time_major == False` (default), this must be a `Tensor` of shape:
+        `[batch_size, max_time, ...]`, or a nested tuple of such elements.
+      If `time_major == True`, this must be a `Tensor` of shape: `[max_time,
+        batch_size, ...]`, or a nested tuple of such elements. This may also be
+        a (possibly nested) tuple of Tensors satisfying this property.  The
+        first two dimensions must match across all the inputs, but otherwise the
+        ranks and other shape components may differ. In this case, input to
+        `cell` at each time-step will replicate the structure of these tuples,
+        except for the time dimension (from which the time is taken). The input
+        to `cell` at each time step will be a `Tensor` or (possibly nested)
+        tuple of Tensors each with dimensions `[batch_size, ...]`.
+    sequence_length: (optional) An int32/int64 vector sized `[batch_size]`. Used
+      to copy-through state and zero-out outputs when past a batch element's
+      sequence length.  So it's more for performance than correctness.
+    initial_state: (optional) An initial state for the RNN. If `cell.state_size`
+      is an integer, this must be a `Tensor` of appropriate type and shape
+      `[batch_size, cell.state_size]`. If `cell.state_size` is a tuple, this
+      should be a tuple of tensors having shapes `[batch_size, s] for s in
+      cell.state_size`.
+    dtype: (optional) The data type for the initial state and expected output.
+      Required if initial_state is not provided or RNN state has a heterogeneous
+      dtype.
+    parallel_iterations: (Default: 32).  The number of iterations to run in
+      parallel.  Those operations which do not have any temporal dependency and
+      can be run in parallel, will be.  This parameter trades off time for
+      space.  Values >> 1 use more memory but take less time, while smaller
+      values use less memory but computations take longer.
+    swap_memory: Transparently swap the tensors produced in forward inference
+      but needed for back prop from GPU to CPU.  This allows training RNNs which
+      would typically not fit on a single GPU, with very minimal (or no)
+      performance penalty.
+    time_major: The shape format of the `inputs` and `outputs` Tensors. If true,
+      these `Tensors` must be shaped `[max_time, batch_size, depth]`. If false,
+      these `Tensors` must be shaped `[batch_size, max_time, depth]`. Using
+      `time_major = True` is a bit more efficient because it avoids transposes
+      at the beginning and end of the RNN calculation.  However, most TensorFlow
+      data is batch-major, so by default this function accepts input and emits
+      output in batch-major form.
+    scope: VariableScope for the created subgraph; defaults to "rnn".
+
+  Returns:
+    A pair (outputs, state) where:
+
+    outputs: The RNN output `Tensor`.
+
+      If time_major == False (default), this will be a `Tensor` shaped:
+        `[batch_size, max_time, cell.output_size]`.
+
+      If time_major == True, this will be a `Tensor` shaped:
+        `[max_time, batch_size, cell.output_size]`.
+
+      Note, if `cell.output_size` is a (possibly nested) tuple of integers
+      or `TensorShape` objects, then `outputs` will be a tuple having the
+      same structure as `cell.output_size`, containing Tensors having shapes
+      corresponding to the shape data in `cell.output_size`.
+
+    state: The final state.  If `cell.state_size` is an int, this
+      will be shaped `[batch_size, cell.state_size]`.  If it is a
+      `TensorShape`, this will be shaped `[batch_size] + cell.state_size`.
+      If it is a (possibly nested) tuple of ints or `TensorShape`, this will
+      be a tuple having the corresponding shapes. If cells are `LSTMCells`
+      `state` will be a tuple containing a `LSTMStateTuple` for each cell.
+
+  Raises:
+    TypeError: If `cell` is not an instance of RNNCell.
+    ValueError: If inputs is None or an empty list.
+    RuntimeError: If not using control flow v2.
+  """
+
+  # Currently only support time_major == True case.
+  assert time_major
+
+  # TODO(b/123051275): We need to check if the cells are TfLiteLSTMCells or
+  # TfLiteRNNCells.
+  rnn_cell_impl.assert_like_rnncell("cell", cell)
+
+  if not control_flow_util.ENABLE_CONTROL_FLOW_V2:
+    raise RuntimeError("OpHint dynamic rnn only supports control flow v2.")
+
+  parent_first_child_input = [{
+      "parent_ophint_input_index": 0,
+      "first_child_ophint_input_index": 0
+  }]
+  parent_last_child_output = [{
+      "parent_output_index": 0,
+      # For LstmCell, the index is 2.
+      # For RnnCell, the index is 1.
+      # So we use -1 meaning it's the last one.
+      "child_output_index": -1
+  }]
+  internal_children_input_output = [{
+      "child_input_index": 0,
+      # For LstmCell, the index is 2.
+      # For RnnCell, the index is 1.
+      # So we use -1 meaning it's the last one.
+      "child_output_index": -1
+  }]
+  inputs_outputs_mappings = {
+      "parent_first_child_input": parent_first_child_input,
+      "parent_last_child_output": parent_last_child_output,
+      "internal_children_input_output": internal_children_input_output
+  }
+  tflite_wrapper = tf.lite.OpHint(
+      "TfLiteDynamicRnn",
+      level=2,
+      children_inputs_mappings=inputs_outputs_mappings)
+  with vs.variable_scope(scope or "rnn") as varscope:
+    # Create a new scope in which the caching device is either
+    # determined by the parent scope, or is set to place the cached
+    # Variable using the same placement as for the rest of the RNN.
+    if _should_cache():
+      if varscope.caching_device is None:
+        varscope.set_caching_device(lambda op: op.device)
+
+    inputs = tflite_wrapper.add_input(inputs, name="input", index_override=0)
+
+    # By default, time_major==False and inputs are batch-major: shaped
+    #   [batch, time, depth]
+    # For internal calculations, we transpose to [time, batch, depth]
+    flat_input = nest.flatten(inputs)
+
+    if not time_major:
+      # (batch, time, depth) => (time, batch, depth)
+      flat_input = [ops.convert_to_tensor(input_) for input_ in flat_input]
+      flat_input = tuple(_transpose_batch_time(input_) for input_ in flat_input)
+
+    parallel_iterations = parallel_iterations or 32
+    if sequence_length is not None:
+      sequence_length = math_ops.to_int32(sequence_length)
+      if sequence_length.get_shape().rank not in (None, 1):
+        raise ValueError(
+            "sequence_length must be a vector of length batch_size, "
+            "but saw shape: %s" % sequence_length.get_shape())
+      sequence_length = array_ops.identity(  # Just to find it in the graph.
+          sequence_length,
+          name="sequence_length")
+
+    batch_size = _best_effort_input_batch_size(flat_input)
+
+    if initial_state is not None:
+      state = initial_state
+    else:
+      if not dtype:
+        raise ValueError("If there is no initial_state, you must give a dtype.")
+      if getattr(cell, "get_initial_state", None) is not None:
+        state = cell.get_initial_state(
+            inputs=None, batch_size=batch_size, dtype=dtype)
+      else:
+        state = cell.zero_state(batch_size, dtype)
+
+    def _assert_has_shape(x, shape):
+      x_shape = array_ops.shape(x)
+      packed_shape = array_ops.stack(shape)
+      return control_flow_ops.Assert(
+          math_ops.reduce_all(math_ops.equal(x_shape, packed_shape)), [
+              "Expected shape for Tensor %s is " % x.name, packed_shape,
+              " but saw shape: ", x_shape
+          ])
+
+    if not context.executing_eagerly() and sequence_length is not None:
+      # Perform some shape validation
+      with ops.control_dependencies(
+          [_assert_has_shape(sequence_length, [batch_size])]):
+        sequence_length = array_ops.identity(
+            sequence_length, name="CheckSeqLen")
+
+    inputs = nest.pack_sequence_as(structure=inputs, flat_sequence=flat_input)
+
+    outputs, final_state = _dynamic_rnn_loop(
+        cell,
+        inputs,
+        state,
+        parallel_iterations=parallel_iterations,
+        swap_memory=swap_memory,
+        sequence_length=sequence_length,
+        dtype=dtype)
+
+    # Outputs of _dynamic_rnn_loop are always shaped [time, batch, depth].
+    # If we are performing batch-major calculations, transpose output back
+    # to shape [batch, time, depth]
+    if not time_major:
+      # (time, batch, depth) => (batch, time, depth)
+      outputs = nest.map_structure(_transpose_batch_time, outputs)
+    outputs = tflite_wrapper.add_output(outputs, name="outputs")
+
+    return outputs, final_state
diff --git a/tensorflow/lite/experimental/examples/lstm/tflite_lstm.py b/tensorflow/lite/experimental/examples/lstm/rnn_cell.py
similarity index 58%
rename from tensorflow/lite/experimental/examples/lstm/tflite_lstm.py
rename to tensorflow/lite/experimental/examples/lstm/rnn_cell.py
index 5ef1a8cc1c..b1f89d638e 100644
--- a/tensorflow/lite/experimental/examples/lstm/tflite_lstm.py
+++ b/tensorflow/lite/experimental/examples/lstm/rnn_cell.py
@@ -12,36 +12,146 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""TfLite LSTMCell wrapper.
+"""TfLite BasicRnnCell wrapper.
 
 TODO(renjieliu): Find a better home for this one.
 """
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+import itertools
 import tensorflow as tf
 
-from tensorflow.python.eager import context
-from tensorflow.python.framework import ops
 from tensorflow.python.keras import activations
 from tensorflow.python.keras import initializers
 from tensorflow.python.layers import base as base_layer
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import clip_ops
-from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.ops import control_flow_util
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import partitioned_variables
 from tensorflow.python.ops import rnn_cell_impl
-from tensorflow.python.ops import variable_scope as vs
-from tensorflow.python.ops.rnn import _best_effort_input_batch_size
-from tensorflow.python.ops.rnn import _dynamic_rnn_loop
-from tensorflow.python.ops.rnn import _should_cache
-from tensorflow.python.ops.rnn import _transpose_batch_time
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.util import nest
+
+
+class TfLiteRNNCell(rnn_cell_impl.LayerRNNCell):
+  """The most basic RNN cell.
+
+  This is used only for TfLite, it provides hints and it also makes the
+  variables in the desired for the tflite ops.
+  """
+
+  def __init__(self,
+               num_units,
+               activation=None,
+               reuse=None,
+               name=None,
+               dtype=None,
+               **kwargs):
+    """Initializes the parameters for an RNN cell.
+
+    Args:
+      num_units: int, The number of units in the RNN cell.
+      activation: Nonlinearity to use.  Default: `tanh`. It could also be string
+        that is within Keras activation function names.
+      reuse: (optional) Python boolean describing whether to reuse variables in
+        an existing scope. Raises an error if not `True` and the existing scope
+        already has the given variables.
+      name: String, the name of the layer. Layers with the same name will share
+        weights, but to avoid mistakes we require reuse=True in such cases.
+      dtype: Default dtype of the layer (default of `None` means use the type of
+        the first input). Required when `build` is called before `call`.
+      **kwargs: Dict, keyword named properties for common layer attributes, like
+        `trainable` etc when constructing the cell from configs of get_config().
+
+    Raises:
+      ValueError: If the existing scope already has the given variables.
+    """
+    super(TfLiteRNNCell, self).__init__(
+        _reuse=reuse, name=name, dtype=dtype, **kwargs)
+
+    # Inputs must be Rank-2.
+    self.input_spec = base_layer.InputSpec(ndim=2)
+
+    self._tflite_wrapper = tf.lite.OpHint("UnidirectionalSequenceRnn")
+    self._num_units = num_units
+    if activation:
+      self._activation = activations.get(activation)
+    else:
+      self._activation = math_ops.tanh
+
+  @property
+  def state_size(self):
+    return self._num_units
+
+  @property
+  def output_size(self):
+    return self._num_units
+
+  def build(self, inputs_shape):
+    """Builds the RNN cell.
+
+    Args:
+      inputs_shape: Rnn input tensor shape.
+
+    Raises:
+      ValueError: If last dimension of the input shape is not known.
+    """
+    if inputs_shape[-1] is None:
+      raise ValueError("Expected inputs.shape[-1] to be known, saw shape: %s" %
+                       (inputs_shape,))
+
+    input_depth = inputs_shape[-1]
+
+    def add_variable_wrapped(name, shape, initializer, index):
+      var = self.add_variable(name, shape=shape, initializer=initializer)
+      return self._tflite_wrapper.add_input(
+          var, name=name, index_override=index)
+
+    self._input_weights = add_variable_wrapped(
+        "input_weights", [self._num_units, input_depth], None, 1)
+    self._recurrent_weights = add_variable_wrapped(
+        "recurrent_weights", [self._num_units, self._num_units], None, 2)
+    self._bias = add_variable_wrapped(
+        "bias",
+        shape=[self._num_units],
+        initializer=init_ops.zeros_initializer(dtype=self.dtype),
+        index=3)
+
+    self.built = True
+
+  def call(self, inputs, state):
+    """Most basic RNN: output = new_state = act(W * input + U * state + B)."""
+    inputs = self._tflite_wrapper.add_input(
+        inputs, tag="input", name="input", aggregate="stack", index_override=0)
+    state = self._tflite_wrapper.add_input(
+        state,
+        tag="hidden_state",
+        name="hidden_state",
+        aggregate="first",
+        index_override=4)
+    weights = array_ops.transpose(
+        array_ops.concat([self._input_weights, self._recurrent_weights], 1))
+    gate_inputs = math_ops.matmul(array_ops.concat([inputs, state], 1), weights)
+    gate_inputs = nn_ops.bias_add(gate_inputs, self._bias)
+    output = self._activation(gate_inputs)
+    output = self._tflite_wrapper.add_output(
+        output,
+        tag="output",
+        name="output",
+        index_override=1,
+        aggregate="stack")
+    return output, output
+
+  def get_config(self):
+    config = {
+        "num_units": self._num_units,
+        "activation": activations.serialize(self._activation),
+        "reuse": self._reuse,
+    }
+    base_config = super(TfLiteRNNCell, self).get_config()
+    return dict(itertools.chain(base_config.items(), config.items()))
 
 
 class TFLiteLSTMCell(rnn_cell_impl.LayerRNNCell):
@@ -157,7 +267,7 @@ class TFLiteLSTMCell(rnn_cell_impl.LayerRNNCell):
 
     self._output_size = num_proj if num_proj else num_units
     self._state_size = (
-        tf.nn.rnn_cell.LSTMStateTuple(num_units, self._output_size)
+        rnn_cell_impl.LSTMStateTuple(num_units, self._output_size)
         if state_is_tuple else num_units + self._output_size)
 
   @property
@@ -328,24 +438,28 @@ class TFLiteLSTMCell(rnn_cell_impl.LayerRNNCell):
     # c is the final state.
     # m is the output.
     i = nn_ops.bias_add(
-        tf.matmul(
+        math_ops.matmul(
             inputs_and_m_prev,
-            tf.concat([self.input_to_input_w, self.cell_to_input_w], axis=1),
+            array_ops.concat([self.input_to_input_w, self.cell_to_input_w],
+                             axis=1),
             transpose_b=True), self.input_bias)
     f = nn_ops.bias_add(
-        tf.matmul(
+        math_ops.matmul(
             inputs_and_m_prev,
-            tf.concat([self.input_to_forget_w, self.cell_to_forget_w], axis=1),
+            array_ops.concat([self.input_to_forget_w, self.cell_to_forget_w],
+                             axis=1),
             transpose_b=True), self.forget_bias)
     o = nn_ops.bias_add(
-        tf.matmul(
+        math_ops.matmul(
             inputs_and_m_prev,
-            tf.concat([self.input_to_output_w, self.cell_to_output_w], axis=1),
+            array_ops.concat([self.input_to_output_w, self.cell_to_output_w],
+                             axis=1),
             transpose_b=True), self.output_bias)
     j = nn_ops.bias_add(
-        tf.matmul(
+        math_ops.matmul(
             inputs_and_m_prev,
-            tf.concat([self.input_to_cell_w, self.cell_to_cell_w], axis=1),
+            array_ops.concat([self.input_to_cell_w, self.cell_to_cell_w],
+                             axis=1),
             transpose_b=True), self.cell_bias)
 
     # Diagonal connections
@@ -368,7 +482,7 @@ class TFLiteLSTMCell(rnn_cell_impl.LayerRNNCell):
       m = sigmoid(o) * self._activation(c)
 
     if self._num_proj is not None:
-      transposed_proj_kernel = tf.transpose(self._proj_kernel)
+      transposed_proj_kernel = array_ops.transpose(self._proj_kernel)
       m = math_ops.matmul(m, transposed_proj_kernel)
 
       if self._proj_clip is not None:
@@ -382,7 +496,7 @@ class TFLiteLSTMCell(rnn_cell_impl.LayerRNNCell):
         m, tag="m", name="m", index_override=2, aggregate="stack")
 
     new_state = (
-        tf.nn.rnn_cell.LSTMStateTuple(c, m)
+        rnn_cell_impl.LSTMStateTuple(c, m)
         if self._state_is_tuple else array_ops.concat([c, m], 1))
     return m, new_state
 
@@ -403,240 +517,3 @@ class TFLiteLSTMCell(rnn_cell_impl.LayerRNNCell):
     }
     base_config = super(TFLiteLSTMCell, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
-
-
-def dynamic_rnn(cell,
-                inputs,
-                sequence_length=None,
-                initial_state=None,
-                dtype=None,
-                parallel_iterations=None,
-                swap_memory=False,
-                time_major=True,
-                scope=None):
-  """Creates a recurrent neural network specified by RNNCell `cell`.
-
-  Performs fully dynamic unrolling of `inputs`.
-
-  Example:
-
-  ```python
-  # create a BasicRNNCell
-  rnn_cell = tf.nn.rnn_cell.BasicRNNCell(hidden_size)
-
-  # 'outputs' is a tensor of shape [batch_size, max_time, cell_state_size]
-
-  # defining initial state
-  initial_state = rnn_cell.zero_state(batch_size, dtype=tf.float32)
-
-  # 'state' is a tensor of shape [batch_size, cell_state_size]
-  outputs, state = tf.nn.dynamic_rnn(rnn_cell, input_data,
-                                     initial_state=initial_state,
-                                     dtype=tf.float32)
-  ```
-
-  ```python
-  # create 2 LSTMCells
-  rnn_layers = [tf.nn.rnn_cell.LSTMCell(size) for size in [128, 256]]
-
-  # create a RNN cell composed sequentially of a number of RNNCells
-  multi_rnn_cell = tf.nn.rnn_cell.MultiRNNCell(rnn_layers)
-
-  # 'outputs' is a tensor of shape [batch_size, max_time, 256]
-  # 'state' is a N-tuple where N is the number of LSTMCells containing a
-  # tf.contrib.rnn.LSTMStateTuple for each cell
-  outputs, state = tf.nn.dynamic_rnn(cell=multi_rnn_cell,
-                                     inputs=data,
-                                     dtype=tf.float32)
-  ```
-
-
-  Args:
-    cell: An instance of RNNCell.
-    inputs: The RNN inputs.
-      If `time_major == False` (default), this must be a `Tensor` of shape:
-        `[batch_size, max_time, ...]`, or a nested tuple of such elements.
-      If `time_major == True`, this must be a `Tensor` of shape: `[max_time,
-        batch_size, ...]`, or a nested tuple of such elements. This may also be
-        a (possibly nested) tuple of Tensors satisfying this property.  The
-        first two dimensions must match across all the inputs, but otherwise the
-        ranks and other shape components may differ. In this case, input to
-        `cell` at each time-step will replicate the structure of these tuples,
-        except for the time dimension (from which the time is taken). The input
-        to `cell` at each time step will be a `Tensor` or (possibly nested)
-        tuple of Tensors each with dimensions `[batch_size, ...]`.
-    sequence_length: (optional) An int32/int64 vector sized `[batch_size]`. Used
-      to copy-through state and zero-out outputs when past a batch element's
-      sequence length.  So it's more for performance than correctness.
-    initial_state: (optional) An initial state for the RNN. If `cell.state_size`
-      is an integer, this must be a `Tensor` of appropriate type and shape
-      `[batch_size, cell.state_size]`. If `cell.state_size` is a tuple, this
-      should be a tuple of tensors having shapes `[batch_size, s] for s in
-      cell.state_size`.
-    dtype: (optional) The data type for the initial state and expected output.
-      Required if initial_state is not provided or RNN state has a heterogeneous
-      dtype.
-    parallel_iterations: (Default: 32).  The number of iterations to run in
-      parallel.  Those operations which do not have any temporal dependency and
-      can be run in parallel, will be.  This parameter trades off time for
-      space.  Values >> 1 use more memory but take less time, while smaller
-      values use less memory but computations take longer.
-    swap_memory: Transparently swap the tensors produced in forward inference
-      but needed for back prop from GPU to CPU.  This allows training RNNs which
-      would typically not fit on a single GPU, with very minimal (or no)
-      performance penalty.
-    time_major: The shape format of the `inputs` and `outputs` Tensors. If true,
-      these `Tensors` must be shaped `[max_time, batch_size, depth]`. If false,
-      these `Tensors` must be shaped `[batch_size, max_time, depth]`. Using
-      `time_major = True` is a bit more efficient because it avoids transposes
-      at the beginning and end of the RNN calculation.  However, most TensorFlow
-      data is batch-major, so by default this function accepts input and emits
-      output in batch-major form.
-    scope: VariableScope for the created subgraph; defaults to "rnn".
-
-  Returns:
-    A pair (outputs, state) where:
-
-    outputs: The RNN output `Tensor`.
-
-      If time_major == False (default), this will be a `Tensor` shaped:
-        `[batch_size, max_time, cell.output_size]`.
-
-      If time_major == True, this will be a `Tensor` shaped:
-        `[max_time, batch_size, cell.output_size]`.
-
-      Note, if `cell.output_size` is a (possibly nested) tuple of integers
-      or `TensorShape` objects, then `outputs` will be a tuple having the
-      same structure as `cell.output_size`, containing Tensors having shapes
-      corresponding to the shape data in `cell.output_size`.
-
-    state: The final state.  If `cell.state_size` is an int, this
-      will be shaped `[batch_size, cell.state_size]`.  If it is a
-      `TensorShape`, this will be shaped `[batch_size] + cell.state_size`.
-      If it is a (possibly nested) tuple of ints or `TensorShape`, this will
-      be a tuple having the corresponding shapes. If cells are `LSTMCells`
-      `state` will be a tuple containing a `LSTMStateTuple` for each cell.
-
-  Raises:
-    TypeError: If `cell` is not an instance of RNNCell.
-    ValueError: If inputs is None or an empty list.
-    RuntimeError: If not using control flow v2.
-  """
-
-  # Currently only support time_major == True case.
-  assert time_major
-
-  # TODO(b/123051275): We need to check if the cells are TfLiteLSTMCells or
-  # TfLiteRNNCells.
-  rnn_cell_impl.assert_like_rnncell("cell", cell)
-
-  if not control_flow_util.ENABLE_CONTROL_FLOW_V2:
-    raise RuntimeError("OpHint dynamic rnn only supports control flow v2.")
-
-  parent_first_child_input = [{
-      "parent_ophint_input_index": 0,
-      "first_child_ophint_input_index": 0
-  }]
-  parent_last_child_output = [{
-      "parent_output_index": 0,
-      # For LstmCell, the index is 2.
-      # For RnnCell, the index is 1.
-      # So we use -1 meaning it's the last one.
-      "child_output_index": -1
-  }]
-  internal_children_input_output = [{
-      "child_input_index": 0,
-      # For LstmCell, the index is 2.
-      # For RnnCell, the index is 1.
-      # So we use -1 meaning it's the last one.
-      "child_output_index": -1
-  }]
-  inputs_outputs_mappings = {
-      "parent_first_child_input": parent_first_child_input,
-      "parent_last_child_output": parent_last_child_output,
-      "internal_children_input_output": internal_children_input_output
-  }
-  tflite_wrapper = tf.lite.OpHint(
-      "TfLiteDynamicRnn",
-      level=2,
-      children_inputs_mappings=inputs_outputs_mappings)
-  with vs.variable_scope(scope or "rnn") as varscope:
-    # Create a new scope in which the caching device is either
-    # determined by the parent scope, or is set to place the cached
-    # Variable using the same placement as for the rest of the RNN.
-    if _should_cache():
-      if varscope.caching_device is None:
-        varscope.set_caching_device(lambda op: op.device)
-
-    inputs = tflite_wrapper.add_input(inputs, name="input", index_override=0)
-
-    # By default, time_major==False and inputs are batch-major: shaped
-    #   [batch, time, depth]
-    # For internal calculations, we transpose to [time, batch, depth]
-    flat_input = nest.flatten(inputs)
-
-    if not time_major:
-      # (batch, time, depth) => (time, batch, depth)
-      flat_input = [ops.convert_to_tensor(input_) for input_ in flat_input]
-      flat_input = tuple(_transpose_batch_time(input_) for input_ in flat_input)
-
-    parallel_iterations = parallel_iterations or 32
-    if sequence_length is not None:
-      sequence_length = math_ops.to_int32(sequence_length)
-      if sequence_length.get_shape().rank not in (None, 1):
-        raise ValueError(
-            "sequence_length must be a vector of length batch_size, "
-            "but saw shape: %s" % sequence_length.get_shape())
-      sequence_length = array_ops.identity(  # Just to find it in the graph.
-          sequence_length,
-          name="sequence_length")
-
-    batch_size = _best_effort_input_batch_size(flat_input)
-
-    if initial_state is not None:
-      state = initial_state
-    else:
-      if not dtype:
-        raise ValueError("If there is no initial_state, you must give a dtype.")
-      if getattr(cell, "get_initial_state", None) is not None:
-        state = cell.get_initial_state(
-            inputs=None, batch_size=batch_size, dtype=dtype)
-      else:
-        state = cell.zero_state(batch_size, dtype)
-
-    def _assert_has_shape(x, shape):
-      x_shape = array_ops.shape(x)
-      packed_shape = array_ops.stack(shape)
-      return control_flow_ops.Assert(
-          math_ops.reduce_all(math_ops.equal(x_shape, packed_shape)), [
-              "Expected shape for Tensor %s is " % x.name, packed_shape,
-              " but saw shape: ", x_shape
-          ])
-
-    if not context.executing_eagerly() and sequence_length is not None:
-      # Perform some shape validation
-      with ops.control_dependencies(
-          [_assert_has_shape(sequence_length, [batch_size])]):
-        sequence_length = array_ops.identity(
-            sequence_length, name="CheckSeqLen")
-
-    inputs = nest.pack_sequence_as(structure=inputs, flat_sequence=flat_input)
-
-    outputs, final_state = _dynamic_rnn_loop(
-        cell,
-        inputs,
-        state,
-        parallel_iterations=parallel_iterations,
-        swap_memory=swap_memory,
-        sequence_length=sequence_length,
-        dtype=dtype)
-
-    # Outputs of _dynamic_rnn_loop are always shaped [time, batch, depth].
-    # If we are performing batch-major calculations, transpose output back
-    # to shape [batch, time, depth]
-    if not time_major:
-      # (time, batch, depth) => (batch, time, depth)
-      outputs = nest.map_structure(_transpose_batch_time, outputs)
-    outputs = tflite_wrapper.add_output(outputs, name="outputs")
-
-    return outputs, final_state
diff --git a/tensorflow/lite/experimental/examples/lstm/tflite_rnn.py b/tensorflow/lite/experimental/examples/lstm/tflite_rnn.py
deleted file mode 100644
index b7a89056ac..0000000000
--- a/tensorflow/lite/experimental/examples/lstm/tflite_rnn.py
+++ /dev/null
@@ -1,150 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""TfLite BasicRnnCell wrapper.
-
-TODO(renjieliu): Find a better home for this one.
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-import itertools
-import tensorflow as tf
-
-from tensorflow.python.keras import activations
-from tensorflow.python.layers import base as base_layer
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import init_ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import nn_ops
-from tensorflow.python.ops import rnn_cell_impl
-
-
-class TfLiteRNNCell(rnn_cell_impl.LayerRNNCell):
-  """The most basic RNN cell.
-
-  This is used only for TfLite, it provides hints and it also makes the
-  variables in the desired for the tflite ops.
-  """
-
-  def __init__(self,
-               num_units,
-               activation=None,
-               reuse=None,
-               name=None,
-               dtype=None,
-               **kwargs):
-    """Initializes the parameters for an RNN cell.
-
-    Args:
-      num_units: int, The number of units in the RNN cell.
-      activation: Nonlinearity to use.  Default: `tanh`. It could also be string
-        that is within Keras activation function names.
-      reuse: (optional) Python boolean describing whether to reuse variables in
-        an existing scope. Raises an error if not `True` and the existing scope
-        already has the given variables.
-      name: String, the name of the layer. Layers with the same name will share
-        weights, but to avoid mistakes we require reuse=True in such cases.
-      dtype: Default dtype of the layer (default of `None` means use the type of
-        the first input). Required when `build` is called before `call`.
-      **kwargs: Dict, keyword named properties for common layer attributes, like
-        `trainable` etc when constructing the cell from configs of get_config().
-
-    Raises:
-      ValueError: If the existing scope already has the given variables.
-    """
-    super(TfLiteRNNCell, self).__init__(
-        _reuse=reuse, name=name, dtype=dtype, **kwargs)
-
-    # Inputs must be Rank-2.
-    self.input_spec = base_layer.InputSpec(ndim=2)
-
-    self._tflite_wrapper = tf.lite.OpHint("UnidirectionalSequenceRnn")
-    self._num_units = num_units
-    if activation:
-      self._activation = activations.get(activation)
-    else:
-      self._activation = math_ops.tanh
-
-  @property
-  def state_size(self):
-    return self._num_units
-
-  @property
-  def output_size(self):
-    return self._num_units
-
-  def build(self, inputs_shape):
-    """Builds the RNN cell.
-
-    Args:
-      inputs_shape: Rnn input tensor shape.
-
-    Raises:
-      ValueError: If last dimension of the input shape is not known.
-    """
-    if inputs_shape[-1] is None:
-      raise ValueError("Expected inputs.shape[-1] to be known, saw shape: %s" %
-                       (inputs_shape,))
-
-    input_depth = inputs_shape[-1]
-
-    def add_variable_wrapped(name, shape, initializer, index):
-      var = self.add_variable(name, shape=shape, initializer=initializer)
-      return self._tflite_wrapper.add_input(
-          var, name=name, index_override=index)
-
-    self._input_weights = add_variable_wrapped(
-        "input_weights", [self._num_units, input_depth], None, 1)
-    self._recurrent_weights = add_variable_wrapped(
-        "recurrent_weights", [self._num_units, self._num_units], None, 2)
-    self._bias = add_variable_wrapped(
-        "bias",
-        shape=[self._num_units],
-        initializer=init_ops.zeros_initializer(dtype=self.dtype),
-        index=3)
-
-    self.built = True
-
-  def call(self, inputs, state):
-    """Most basic RNN: output = new_state = act(W * input + U * state + B)."""
-    inputs = self._tflite_wrapper.add_input(
-        inputs, tag="input", name="input", aggregate="stack", index_override=0)
-    state = self._tflite_wrapper.add_input(
-        state,
-        tag="hidden_state",
-        name="hidden_state",
-        aggregate="first",
-        index_override=4)
-    weights = array_ops.transpose(
-        array_ops.concat([self._input_weights, self._recurrent_weights], 1))
-    gate_inputs = math_ops.matmul(array_ops.concat([inputs, state], 1), weights)
-    gate_inputs = nn_ops.bias_add(gate_inputs, self._bias)
-    output = self._activation(gate_inputs)
-    output = self._tflite_wrapper.add_output(
-        output,
-        tag="output",
-        name="output",
-        index_override=1,
-        aggregate="stack")
-    return output, output
-
-  def get_config(self):
-    config = {
-        "num_units": self._num_units,
-        "activation": activations.serialize(self._activation),
-        "reuse": self._reuse,
-    }
-    base_config = super(TfLiteRNNCell, self).get_config()
-    return dict(itertools.chain(base_config.items(), config.items()))
diff --git a/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_lstm_test.py b/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_lstm_test.py
index ed02d6c906..b2e91c4dd6 100644
--- a/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_lstm_test.py
+++ b/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_lstm_test.py
@@ -20,8 +20,8 @@ import numpy as np
 import tensorflow as tf
 
 from tensorflow.examples.tutorials.mnist import input_data
-from tensorflow.lite.experimental.examples.lstm.tflite_lstm import dynamic_rnn
-from tensorflow.lite.experimental.examples.lstm.tflite_lstm import TFLiteLSTMCell
+from tensorflow.lite.experimental.examples.lstm.rnn import dynamic_rnn
+from tensorflow.lite.experimental.examples.lstm.rnn_cell import TFLiteLSTMCell
 from tensorflow.lite.python.op_hint import convert_op_hints_to_stubs
 from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
diff --git a/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_rnn_test.py b/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_rnn_test.py
index af7ac4c4d8..78a2e45103 100644
--- a/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_rnn_test.py
+++ b/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_rnn_test.py
@@ -22,7 +22,8 @@ import tensorflow as tf
 from tensorflow import flags
 
 from tensorflow.examples.tutorials.mnist import input_data
-from tensorflow.lite.experimental.examples.lstm.tflite_rnn import TfLiteRNNCell
+from tensorflow.lite.experimental.examples.lstm.rnn import dynamic_rnn
+from tensorflow.lite.experimental.examples.lstm.rnn_cell import TfLiteRNNCell
 from tensorflow.lite.python.op_hint import convert_op_hints_to_stubs
 from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
@@ -66,7 +67,7 @@ class UnidirectionalSequenceRnnTest(test_util.TensorFlowTestCase):
         TfLiteRNNCell(self.num_units, name="rnn2")
     ])
 
-  def buildModel(self, rnn_layer):
+  def buildModel(self, rnn_layer, is_dynamic_rnn):
     # Weights and biases for output softmax layer.
     out_weights = tf.Variable(
         tf.random_normal([self.num_units, self.n_classes]))
@@ -77,8 +78,13 @@ class UnidirectionalSequenceRnnTest(test_util.TensorFlowTestCase):
         "float", [None, self.time_steps, self.n_input], name="INPUT_IMAGE")
 
     # x is shaped [batch_size,time_steps,num_inputs]
-    rnn_input = tf.unstack(x, self.time_steps, 1)
-    outputs, _ = tf.nn.static_rnn(rnn_layer, rnn_input, dtype="float32")
+    if is_dynamic_rnn:
+      rnn_input = tf.transpose(x, perm=[1, 0, 2])
+      outputs, _ = dynamic_rnn(rnn_layer, rnn_input, dtype="float32")
+      outputs = tf.unstack(outputs, axis=0)
+    else:
+      rnn_input = tf.unstack(x, self.time_steps, 1)
+      outputs, _ = tf.nn.static_rnn(rnn_layer, rnn_input, dtype="float32")
 
     # Compute logits by multiplying outputs[-1] of shape [batch_size,num_units]
     # by the softmax layer's out_weight of shape [num_units,n_classes]
@@ -108,13 +114,14 @@ class UnidirectionalSequenceRnnTest(test_util.TensorFlowTestCase):
                                  self.n_input))
       sess.run(opt, feed_dict={x: batch_x, y: batch_y})
 
-  def saveAndRestoreModel(self, rnn_layer, sess, saver):
+  def saveAndRestoreModel(self, rnn_layer, sess, saver, is_dynamic_rnn):
     """Saves and restores the model to mimic the most common use case.
 
     Args:
       rnn_layer: The rnn layer either a single rnn cell or a multi rnn cell.
       sess: Old session.
       saver: saver created by tf.train.Saver()
+      is_dynamic_rnn: use dynamic_rnn or not.
 
     Returns:
       A tuple containing:
@@ -130,7 +137,7 @@ class UnidirectionalSequenceRnnTest(test_util.TensorFlowTestCase):
 
     # Reset the graph.
     tf.reset_default_graph()
-    x, prediction, output_class = self.buildModel(rnn_layer)
+    x, prediction, output_class = self.buildModel(rnn_layer, is_dynamic_rnn)
 
     new_sess = tf.Session(config=CONFIG)
     saver = tf.train.Saver()
@@ -177,12 +184,32 @@ class UnidirectionalSequenceRnnTest(test_util.TensorFlowTestCase):
   def testStaticRnnMultiRnnCell(self):
     sess = tf.Session(config=CONFIG)
 
-    x, prediction, output_class = self.buildModel(self.buildRnnLayer())
+    x, prediction, output_class = self.buildModel(
+        self.buildRnnLayer(), is_dynamic_rnn=False)
     self.trainModel(x, prediction, output_class, sess)
 
     saver = tf.train.Saver()
     x, prediction, output_class, new_sess = self.saveAndRestoreModel(
-        self.buildRnnLayer(), sess, saver)
+        self.buildRnnLayer(), sess, saver, is_dynamic_rnn=False)
+
+    test_inputs, expected_output, frozen_graph = self.getInferenceResult(
+        x, output_class, new_sess)
+
+    result = self.tfliteInvoke(frozen_graph, test_inputs, output_class)
+    self.assertTrue(np.allclose(expected_output, result, rtol=1e-6, atol=1e-2))
+
+  @test_util.enable_control_flow_v2
+  def testDynamicRnnMultiRnnCell(self):
+    sess = tf.Session(config=CONFIG)
+
+    x, prediction, output_class = self.buildModel(
+        self.buildRnnLayer(), is_dynamic_rnn=True)
+    self.trainModel(x, prediction, output_class, sess)
+
+    saver = tf.train.Saver()
+
+    x, prediction, output_class, new_sess = self.saveAndRestoreModel(
+        self.buildRnnLayer(), sess, saver, is_dynamic_rnn=True)
 
     test_inputs, expected_output, frozen_graph = self.getInferenceResult(
         x, output_class, new_sess)
diff --git a/tensorflow/tools/pip_package/pip_smoke_test.py b/tensorflow/tools/pip_package/pip_smoke_test.py
index 952c71c615..903a7f9773 100644
--- a/tensorflow/tools/pip_package/pip_smoke_test.py
+++ b/tensorflow/tools/pip_package/pip_smoke_test.py
@@ -88,8 +88,8 @@ DEPENDENCY_BLACKLIST = [
     "//tensorflow/python/feature_column:vocabulary_testdata",
     "//tensorflow/python:framework/test_file_system.so",
     # lite
-    "//tensorflow/lite/experimental/examples/lstm:tflite_lstm",
-    "//tensorflow/lite/experimental/examples/lstm:tflite_lstm.py",
+    "//tensorflow/lite/experimental/examples/lstm:rnn_cell",
+    "//tensorflow/lite/experimental/examples/lstm:rnn_cell.py",
     "//tensorflow/lite/experimental/examples/lstm:unidirectional_sequence_lstm_test",  # pylint:disable=line-too-long
     "//tensorflow/lite/experimental/examples/lstm:unidirectional_sequence_lstm_test.py",  # pylint:disable=line-too-long
     "//tensorflow/lite/python:interpreter",
-- 
GitLab


From 30f441132612187c12337c97887532e86d037586 Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Tue, 12 Feb 2019 00:38:37 -0800
Subject: [PATCH 2307/2345] Move custom layer into its own file.

PiperOrigin-RevId: 233556594
---
 .../saved_model/integration_tests/BUILD       |  9 ++++
 .../use_model_in_sequential_keras.py          | 26 ++---------
 .../saved_model/integration_tests/util.py     | 43 +++++++++++++++++++
 3 files changed, 55 insertions(+), 23 deletions(-)
 create mode 100644 tensorflow/examples/saved_model/integration_tests/util.py

diff --git a/tensorflow/examples/saved_model/integration_tests/BUILD b/tensorflow/examples/saved_model/integration_tests/BUILD
index 2be9df6644..1da0cb3430 100644
--- a/tensorflow/examples/saved_model/integration_tests/BUILD
+++ b/tensorflow/examples/saved_model/integration_tests/BUILD
@@ -47,6 +47,15 @@ py_binary(
 py_binary(
     name = "use_model_in_sequential_keras",
     srcs = ["use_model_in_sequential_keras.py"],
+    deps = [
+        ":util",
+        "//tensorflow:tensorflow_py",
+    ],
+)
+
+py_library(
+    name = "util",
+    srcs = ["util.py"],
     deps = [
         "//tensorflow:tensorflow_py",
     ],
diff --git a/tensorflow/examples/saved_model/integration_tests/use_model_in_sequential_keras.py b/tensorflow/examples/saved_model/integration_tests/use_model_in_sequential_keras.py
index 9e74dae255..370d157b98 100644
--- a/tensorflow/examples/saved_model/integration_tests/use_model_in_sequential_keras.py
+++ b/tensorflow/examples/saved_model/integration_tests/use_model_in_sequential_keras.py
@@ -25,6 +25,7 @@ import numpy as np
 
 import tensorflow as tf
 # TODO(vbardiovsky): Remove when load is available.
+from tensorflow.examples.saved_model.integration_tests import util
 from tensorflow.python.saved_model.load import load
 
 tf.saved_model.load = load
@@ -34,27 +35,6 @@ FLAGS = flags.FLAGS
 flags.DEFINE_string("model_dir", None, "Directory to load SavedModel from.")
 
 
-# TODO(vbardiovsky): We should just reuse Keras's Lambda layer, when that
-# enables to get trainable variables.
-class LambdaLayer(tf.keras.layers.Layer):
-  """Lambda layer with output shape inference."""
-
-  def __init__(self, func, **kwargs):
-    self._func = func
-    super(LambdaLayer, self).__init__(**kwargs)
-
-  def call(self, x):
-    result = self._func(x)
-    # TODO(vbardiovsky): Polymorphic function should return shaped tensor.
-    result.set_shape(self.compute_output_shape(x.shape))
-    return result
-
-  def compute_output_shape(self, input_shape):
-    # TODO(vbardiovsky): We should be able to get the embedding dimension from
-    # the restored model.
-    return (input_shape[0], 10)
-
-
 def main(argv):
   del argv
 
@@ -63,11 +43,11 @@ def main(argv):
 
   dataset = tf.data.Dataset.from_tensor_slices((features, labels))
 
-  embedding = tf.saved_model.load(FLAGS.model_dir)
+  embed = tf.saved_model.load(FLAGS.model_dir)
 
   # Create the sequential keras model.
   model = tf.keras.Sequential()
-  model.add(LambdaLayer(embedding, batch_input_shape=[None], dtype=tf.string))
+  model.add(util.CustomLayer(embed, batch_input_shape=[None], dtype=tf.string))
   model.add(tf.keras.layers.Dense(100, activation="relu"))
   model.add(tf.keras.layers.Dense(50, activation="relu"))
   model.add(tf.keras.layers.Dense(1, activation="sigmoid"))
diff --git a/tensorflow/examples/saved_model/integration_tests/util.py b/tensorflow/examples/saved_model/integration_tests/util.py
new file mode 100644
index 0000000000..5ea9e91ff5
--- /dev/null
+++ b/tensorflow/examples/saved_model/integration_tests/util.py
@@ -0,0 +1,43 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Utilities for integration tests."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import tensorflow as tf
+
+
+# TODO(vbardiovsky): We should just reuse Keras's Lambda layer, when that
+# enables to get trainable variables.
+class CustomLayer(tf.keras.layers.Layer):
+  """Wraps callable object as a `Layer` object."""
+
+  def __init__(self, func, **kwargs):
+    self._func = func
+    super(CustomLayer, self).__init__(**kwargs)
+
+  def call(self, x):
+    result = self._func(x)
+    # TODO(vbardiovsky): Polymorphic function should return shaped tensor.
+    result.set_shape(self.compute_output_shape(x.shape))
+    return result
+
+  def compute_output_shape(self, input_shape):
+    # TODO(vbardiovsky): We should be able to get the embedding dimension from
+    # the restored model.
+    return (input_shape[0], 10)
+
-- 
GitLab


From c4d25405fb3c6060d9bc74520f56f35f6a596ab6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 12 Feb 2019 01:02:34 -0800
Subject: [PATCH 2308/2345] compat: Update forward compatibility horizon to
 2019-02-12

PiperOrigin-RevId: 233559524
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 2cd1ec0c8d..bebad8732f 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 2, 11)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 2, 12)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 07d4fe20b305178c9e1c86d0aee25ded59078dbf Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Tue, 12 Feb 2019 02:40:47 -0800
Subject: [PATCH 2309/2345] Internal change

PiperOrigin-RevId: 233572073
---
 tensorflow/python/ops/nn_ops.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 5e61282db6..e2d24c26b6 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -4330,6 +4330,7 @@ def conv1d_transpose(
 
     input = array_ops.expand_dims(input, spatial_start_dim)
     filters = array_ops.expand_dims(filters, 0)  # pylint: disable=redefined-builtin
+    output_shape = list(output_shape)
     output_shape = output_shape[: spatial_start_dim] + [1] + \
                    output_shape[spatial_start_dim:]
 
-- 
GitLab


From 5dc6710aaf1f920f9bb9b24f4260ed5fa67a1b8f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 12 Feb 2019 03:13:30 -0800
Subject: [PATCH 2310/2345] [TF:XLA] Use requested_device() instead of
 def().device().

PiperOrigin-RevId: 233576500
---
 tensorflow/compiler/jit/encapsulate_xla_computations_pass.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/jit/encapsulate_xla_computations_pass.cc b/tensorflow/compiler/jit/encapsulate_xla_computations_pass.cc
index 411ca75cd8..109684be72 100644
--- a/tensorflow/compiler/jit/encapsulate_xla_computations_pass.cc
+++ b/tensorflow/compiler/jit/encapsulate_xla_computations_pass.cc
@@ -50,7 +50,7 @@ bool IsCpuGpuCompile(const Graph* graph) {
       continue;
     // Early return for any node with a device that is not a CPU or GPU.
     DeviceNameUtils::ParsedName parsed;
-    if (DeviceNameUtils::ParseFullName(n->def().device(), &parsed)) {
+    if (DeviceNameUtils::ParseFullName(n->requested_device(), &parsed)) {
       if (parsed.type != DEVICE_CPU && parsed.type != DEVICE_GPU) {
         return false;
       }
-- 
GitLab


From 70ef7e687034519adf5a3d61183668616a47a27e Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Tue, 12 Feb 2019 03:21:02 -0800
Subject: [PATCH 2311/2345] Use placeholders in reduction_ops_test_big.

This makes the tests run much faster and avoids that constant folding is used with XLA
enabled.
On tensorflow classic, testBooleanAll now takes 10 seconds instead of 2 1/2 minutes.
With XLA enabled, the tests pass now.

PiperOrigin-RevId: 233577125
---
 tensorflow/python/kernel_tests/BUILD          |  2 +-
 .../kernel_tests/reduction_ops_test_big.py    | 63 ++++++++++++-------
 2 files changed, 42 insertions(+), 23 deletions(-)

diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 78055ab085..a5bb03ff6a 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -2202,12 +2202,12 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:math_ops",
     ],
+    shard_count = 3,
     tags = [
         "manual",
         "no_gpu",
         "nogpu",
         "noguitar",
-        "notap",
     ],
     xla_enable_strict_auto_jit = True,
 )
diff --git a/tensorflow/python/kernel_tests/reduction_ops_test_big.py b/tensorflow/python/kernel_tests/reduction_ops_test_big.py
index 73bba54a46..2d5cff383e 100644
--- a/tensorflow/python/kernel_tests/reduction_ops_test_big.py
+++ b/tensorflow/python/kernel_tests/reduction_ops_test_big.py
@@ -22,16 +22,17 @@ import numpy as np
 
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
+from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
+
 class BaseReductionTest(test.TestCase):
 
   def _tf_reduce(self, x, reduction_axes, keepdims):
     raise NotImplementedError()
 
 
-@test_util.disable_all_xla("b/123864762")  # Test times out
 class BigReductionTest(BaseReductionTest):
   """Test reductions for sum and boolean all over a wide range of shapes."""
 
@@ -47,6 +48,7 @@ class BigReductionTest(BaseReductionTest):
   def _tf_reduce_sum(self, x, reduction_axes, keepdims):
     return math_ops.reduce_sum(x, reduction_axes, keepdims)
 
+  @test_util.run_deprecated_v1
   def testFloat32Sum(self):
     # make sure we test all possible kernel invocations
     # logic is the same for all ops, test just float32 for brevity
@@ -65,11 +67,13 @@ class BigReductionTest(BaseReductionTest):
         full_sum = np.ones([], dtype=np.float32) * size_x * size_y
 
         with self.session(graph=ops.Graph(), use_gpu=True) as sess:
-          tf_row_sum = self._tf_reduce_sum(arr, 1, False)
-          tf_col_sum = self._tf_reduce_sum(arr, 0, False)
-          tf_full_sum = self._tf_reduce_sum(arr, [0, 1], False)
+          arr_placeholder = array_ops.placeholder(dtype=np.float32,
+                                                  shape=(size_x, size_y))
+          tf_row_sum = self._tf_reduce_sum(arr_placeholder, 1, False)
+          tf_col_sum = self._tf_reduce_sum(arr_placeholder, 0, False)
+          tf_full_sum = self._tf_reduce_sum(arr_placeholder, [0, 1], False)
           tf_out_row, tf_out_col, tf_out_full = sess.run(
-              [tf_row_sum, tf_col_sum, tf_full_sum])
+              [tf_row_sum, tf_col_sum, tf_full_sum], {arr_placeholder: arr})
         self.assertAllClose(col_sum, tf_out_col)
         self.assertAllClose(row_sum, tf_out_row)
         self.assertAllClose(full_sum, tf_out_full)
@@ -83,12 +87,16 @@ class BigReductionTest(BaseReductionTest):
           sum_xz = np.ones([size_y], dtype=np.float32)
 
           with self.session(graph=ops.Graph(), use_gpu=True) as sess:
-            tf_sum_xz = self._tf_reduce_mean(arr, [0, 2], False)
-            tf_sum_y = self._tf_reduce_mean(arr, 1, False)
-            tf_out_sum_xz, tf_out_sum_y = sess.run([tf_sum_xz, tf_sum_y])
+            arr_placeholder = array_ops.placeholder(
+                dtype=np.float32, shape=(size_x, size_y, size_z))
+            tf_sum_xz = self._tf_reduce_mean(arr_placeholder, [0, 2], False)
+            tf_sum_y = self._tf_reduce_mean(arr_placeholder, 1, False)
+            tf_out_sum_xz, tf_out_sum_y = sess.run([tf_sum_xz, tf_sum_y],
+                                                   {arr_placeholder: arr})
           self.assertAllClose(sum_y, tf_out_sum_y)
           self.assertAllClose(sum_xz, tf_out_sum_xz)
 
+  @test_util.run_deprecated_v1
   def testFloat32Max(self):
     # make sure we test all possible kernel invocations
     # logic is the same for all ops, test just float32 for brevity
@@ -108,11 +116,13 @@ class BigReductionTest(BaseReductionTest):
         full_max = np.max(col_max)
 
         with self.session(graph=ops.Graph(), use_gpu=True) as sess:
-          tf_row_max = self._tf_reduce_max(arr, 1, False)
-          tf_col_max = self._tf_reduce_max(arr, 0, False)
-          tf_full_max = self._tf_reduce_max(arr, [0, 1], False)
+          arr_placeholder = array_ops.placeholder(dtype=np.float32,
+                                                  shape=(size_x, size_y))
+          tf_row_max = self._tf_reduce_max(arr_placeholder, 1, False)
+          tf_col_max = self._tf_reduce_max(arr_placeholder, 0, False)
+          tf_full_max = self._tf_reduce_max(arr_placeholder, [0, 1], False)
           tf_out_row, tf_out_col, tf_out_full = sess.run(
-              [tf_row_max, tf_col_max, tf_full_max])
+              [tf_row_max, tf_col_max, tf_full_max], {arr_placeholder: arr})
         self.assertAllClose(col_max, tf_out_col)
         self.assertAllClose(row_max, tf_out_row)
         self.assertAllClose(full_max, tf_out_full)
@@ -127,12 +137,16 @@ class BigReductionTest(BaseReductionTest):
           sum_xz = np.max(arr, axis=(0, 2))
 
           with self.session(graph=ops.Graph(), use_gpu=True) as sess:
-            tf_sum_xz = self._tf_reduce_max(arr, [0, 2], False)
-            tf_sum_y = self._tf_reduce_max(arr, 1, False)
-            tf_out_sum_xz, tf_out_sum_y = sess.run([tf_sum_xz, tf_sum_y])
+            arr_placeholder = array_ops.placeholder(
+                dtype=np.float32, shape=(size_x, size_y, size_z))
+            tf_sum_xz = self._tf_reduce_max(arr_placeholder, [0, 2], False)
+            tf_sum_y = self._tf_reduce_max(arr_placeholder, 1, False)
+            tf_out_sum_xz, tf_out_sum_y = sess.run(
+                [tf_sum_xz, tf_sum_y], {arr_placeholder: arr})
           self.assertAllClose(sum_y, tf_out_sum_y)
           self.assertAllClose(sum_xz, tf_out_sum_xz)
 
+  @test_util.run_deprecated_v1
   def testBooleanAll(self):
     # make sure we test all possible kernel invocations
     # test operation where T(0) is not the identity
@@ -151,11 +165,13 @@ class BigReductionTest(BaseReductionTest):
         full_sum = np.ones([1], dtype=np.bool).reshape([])
 
         with self.session(graph=ops.Graph(), use_gpu=True) as sess:
-          tf_row_sum = self._tf_reduce_all(arr, 1, False)
-          tf_col_sum = self._tf_reduce_all(arr, 0, False)
-          tf_full_sum = self._tf_reduce_all(arr, [0, 1], False)
+          arr_placeholder = array_ops.placeholder(dtype=np.bool,
+                                                  shape=(size_x, size_y))
+          tf_row_sum = self._tf_reduce_all(arr_placeholder, 1, False)
+          tf_col_sum = self._tf_reduce_all(arr_placeholder, 0, False)
+          tf_full_sum = self._tf_reduce_all(arr_placeholder, [0, 1], False)
           tf_out_row, tf_out_col, tf_out_full = sess.run(
-              [tf_row_sum, tf_col_sum, tf_full_sum])
+              [tf_row_sum, tf_col_sum, tf_full_sum], {arr_placeholder: arr})
         self.assertAllClose(col_sum, tf_out_col)
         self.assertAllClose(row_sum, tf_out_row)
         self.assertAllClose(full_sum, tf_out_full)
@@ -169,9 +185,12 @@ class BigReductionTest(BaseReductionTest):
           sum_xz = np.ones([size_y], dtype=np.bool)
 
           with self.session(graph=ops.Graph(), use_gpu=True) as sess:
-            tf_sum_xz = self._tf_reduce_all(arr, [0, 2], False)
-            tf_sum_y = self._tf_reduce_all(arr, 1, False)
-            tf_out_sum_xz, tf_out_sum_y = sess.run([tf_sum_xz, tf_sum_y])
+            arr_placeholder = array_ops.placeholder(
+                dtype=np.bool, shape=(size_x, size_y, size_z))
+            tf_sum_xz = self._tf_reduce_all(arr_placeholder, [0, 2], False)
+            tf_sum_y = self._tf_reduce_all(arr_placeholder, 1, False)
+            tf_out_sum_xz, tf_out_sum_y = sess.run(
+                [tf_sum_xz, tf_sum_y], {arr_placeholder: arr})
           self.assertAllClose(sum_y, tf_out_sum_y)
           self.assertAllClose(sum_xz, tf_out_sum_xz)
 
-- 
GitLab


From aebf4964a20c4bc24340d92ae82504a41bca98e1 Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Tue, 12 Feb 2019 03:22:08 -0800
Subject: [PATCH 2312/2345] Allow bias add to take data_formats other than 2D

PiperOrigin-RevId: 233577204
---
 tensorflow/python/ops/nn_ops.py            | 10 +++++++++-
 tensorflow/python/ops/parallel_for/pfor.py |  4 ++--
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index e2d24c26b6..a7aa97c2e4 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -2436,13 +2436,21 @@ def bias_add(value, bias, data_format=None, name=None):
     bias: A 1-D `Tensor` with size matching the last dimension of `value`.
       Must be the same type as `value` unless `value` is a quantized type,
       in which case a different quantized type may be used.
-    data_format: A string. 'NHWC' and 'NCHW' are supported.
+    data_format: A string. 'N...C' and 'NC...' are supported.
     name: A name for the operation (optional).
 
   Returns:
     A `Tensor` with the same type as `value`.
   """
   with ops.name_scope(name, "BiasAdd", [value, bias]) as name:
+    if data_format is not None:
+      if data_format.startswith("NC"):
+        data_format = "NCHW"
+      elif data_format.startswith("N") and data_format.endswith("C"):
+        data_format = "NHWC"
+      else:
+        raise ValueError("data_format must be of the form `N...C` or `NC...`")
+
     if not context.executing_eagerly():
       value = ops.convert_to_tensor(value, name="input")
       bias = ops.convert_to_tensor(bias, dtype=value.dtype, name="bias")
diff --git a/tensorflow/python/ops/parallel_for/pfor.py b/tensorflow/python/ops/parallel_for/pfor.py
index 019d4f2224..b9f7a0ffca 100644
--- a/tensorflow/python/ops/parallel_for/pfor.py
+++ b/tensorflow/python/ops/parallel_for/pfor.py
@@ -1903,7 +1903,7 @@ def _convert_cumfoo(pfor_input, _, op_func):
 def _convert_biasadd(pfor_input):
   t, t_stacked, _ = pfor_input.input(0)
   bias, bias_stacked, _ = pfor_input.input(1)
-  data_format = pfor_input.get_attr("data_format")
+  data_format = pfor_input.get_attr("data_format").decode()
   if bias_stacked:
     # BiasAdd only supports 1-D biases, so cast bias to match value and use Add.
     pfor_input.expanddim_inputs_for_broadcast()
@@ -1921,7 +1921,7 @@ def _convert_biasadd(pfor_input):
       shape = array_ops.shape(t)
       flattened_shape = array_ops.concat([[-1], shape[2:]], axis=0)
       t = array_ops.reshape(t, flattened_shape)
-      t = nn_ops.bias_add(t, bias, data_format=b"NCHW")
+      t = nn_ops.bias_add(t, bias, data_format="NCHW")
       t = array_ops.reshape(t, shape)
       return wrap(t, True)
     return wrap(nn_ops.bias_add(t, bias, data_format=data_format), True)
-- 
GitLab


From 9c313ef2fdc91fea897a127b623b2c69cd65e45e Mon Sep 17 00:00:00 2001
From: Andr? Susano Pinto <andresp@google.com>
Date: Tue, 12 Feb 2019 04:04:36 -0800
Subject: [PATCH 2313/2345] Wire fullargspec to restored functions.

This will allow tf code to inspect the restored function and see the
original spec (e.g. to decide whether to pass training arg or not).

PiperOrigin-RevId: 233580991
---
 .../saved_model/function_deserialization.py   | 26 +++++++++++++++----
 tensorflow/python/saved_model/load_test.py    | 14 ++++++++++
 2 files changed, 35 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py
index ff48f5cfab..8d14340c07 100644
--- a/tensorflow/python/saved_model/function_deserialization.py
+++ b/tensorflow/python/saved_model/function_deserialization.py
@@ -32,6 +32,8 @@ from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.saved_model import nested_structure_coder
 from tensorflow.python.util import compat
 from tensorflow.python.util import nest
+from tensorflow.python.util import tf_decorator
+from tensorflow.python.util import tf_inspect
 
 
 def _is_tensor(t):
@@ -64,7 +66,15 @@ def _inputs_compatible(args, stored_inputs):
 
 def _deserialize_function_spec(function_spec_proto, coder):
   """Deserialize a FunctionSpec object from its proto representation."""
-  fullargspec = coder.decode_proto(function_spec_proto.fullargspec)
+  typeless_fullargspec = coder.decode_proto(function_spec_proto.fullargspec)
+  fullargspec = tf_inspect.FullArgSpec(
+      args=typeless_fullargspec.args,
+      varargs=typeless_fullargspec.varargs,
+      varkw=typeless_fullargspec.varkw,
+      defaults=typeless_fullargspec.defaults,
+      kwonlyargs=typeless_fullargspec.kwonlyargs,
+      kwonlydefaults=typeless_fullargspec.kwonlydefaults,
+      annotations=typeless_fullargspec.annotations)
   is_method = function_spec_proto.is_method
   args_to_prepend = coder.decode_proto(function_spec_proto.args_to_prepend)
   kwargs_to_include = coder.decode_proto(function_spec_proto.kwargs_to_include)
@@ -172,10 +182,16 @@ def recreate_function(saved_function, concrete_functions):
   for concrete_function_name in saved_function.concrete_functions:
     concrete_function_objects.append(concrete_functions[concrete_function_name])
 
-  return RestoredFunction(restored_function_body,
-                          restored_function_body.__name__,
-                          function_spec,
-                          concrete_function_objects)
+  restored_function = RestoredFunction(
+      restored_function_body,
+      restored_function_body.__name__,
+      function_spec,
+      concrete_function_objects)
+
+  return tf_decorator.make_decorator(
+      restored_function_body,
+      restored_function,
+      decorator_argspec=function_spec.fullargspec)
 
 
 def load_function_def_library(library):
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index edb91b473e..5562831829 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -42,6 +42,7 @@ from tensorflow.python.saved_model import tag_constants
 from tensorflow.python.training import monitored_session
 from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.training.checkpointable import util
+from tensorflow.python.util import tf_inspect
 
 
 @parameterized.named_parameters(
@@ -992,6 +993,19 @@ class LoadTest(test.TestCase, parameterized.TestCase):
         self.assertAllEqual([0, -1, -1, 2], sess.run(output1))
         self.assertAllEqual([2, 0, 1, -1], sess.run(output2))
 
+  def test_perserve_argspec(self, cycles):
+    def f(a, b, c):  # pylint: disable=unused-argument
+      return None
+
+    original_fullargspec = tf_inspect.getfullargspec(f)
+
+    root = tracking.AutoCheckpointable()
+    root.f = def_function.function(f)
+    imported = self.cycle(root, cycles)
+
+    restored_fullargspec = tf_inspect.getfullargspec(imported.f)
+    self.assertEqual(original_fullargspec, restored_fullargspec)
+
 
 class SingleCycleTests(test.TestCase, parameterized.TestCase):
 
-- 
GitLab


From 09e2c39fcf55c2efadede9a573548c28398e6b33 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Tue, 12 Feb 2019 04:17:38 -0800
Subject: [PATCH 2314/2345] Pack function call arguments and keywords into
 containers, to avoid clashing with keywords of converted_call itself.

PiperOrigin-RevId: 233582258
---
 .../python/autograph/converters/call_trees.py |  44 +++++-
 .../autograph/converters/call_trees_test.py   |  54 +++++++-
 tensorflow/python/autograph/impl/api.py       |   4 +-
 tensorflow/python/autograph/impl/api_test.py  | 129 +++++++++---------
 tensorflow/python/framework/func_graph.py     |   2 +-
 5 files changed, 158 insertions(+), 75 deletions(-)

diff --git a/tensorflow/python/autograph/converters/call_trees.py b/tensorflow/python/autograph/converters/call_trees.py
index 04439ba516..7c5307d26c 100644
--- a/tensorflow/python/autograph/converters/call_trees.py
+++ b/tensorflow/python/autograph/converters/call_trees.py
@@ -26,6 +26,7 @@ import gast
 
 from tensorflow.python.autograph.core import converter
 from tensorflow.python.autograph.pyct import anno
+from tensorflow.python.autograph.pyct import ast_util
 from tensorflow.python.autograph.pyct import parser
 from tensorflow.python.autograph.pyct import templates
 
@@ -62,9 +63,6 @@ class CallTreeTransformer(converter.Base):
         not self.ctx.program.options.uses(converter.Feature.BUILTIN_FUNCTIONS)):
       return self.generic_visit(node)
 
-    template = """
-      ag__.converted_call(func, owner, options, args)
-    """
     if isinstance(node.func, gast.Attribute):
       func = gast.Str(node.func.attr)
       owner = node.func.value
@@ -72,6 +70,41 @@ class CallTreeTransformer(converter.Base):
       func = node.func
       owner = parser.parse_expression('None')
 
+    starred_arg = None
+    normal_args = []
+    for a in node.args:
+      if isinstance(a, gast.Starred):
+        assert starred_arg is None, 'Multiple *args should be impossible.'
+        starred_arg = a
+      else:
+        normal_args.append(a)
+    if starred_arg is None:
+      args = templates.replace_as_expression('(args,)', args=normal_args)
+    else:
+      args = templates.replace_as_expression(
+          '(args,) + tuple(stararg)',
+          stararg=starred_arg.value,
+          args=normal_args)
+
+    kwargs_arg = None
+    normal_keywords = []
+    for k in node.keywords:
+      if k.arg is None:
+        assert kwargs_arg is None, 'Multiple **kwargs should be impossible.'
+        kwargs_arg = k
+      else:
+        normal_keywords.append(k)
+    if kwargs_arg is None:
+      kwargs = ast_util.keywords_to_dict(normal_keywords)
+    else:
+      kwargs = templates.replace_as_expression(
+          'dict(kwargs, **keywords)',
+          kwargs=kwargs_arg.value,
+          keywords=ast_util.keywords_to_dict(normal_keywords))
+
+    template = """
+      ag__.converted_call(func, owner, options, args, kwargs)
+    """
     new_call = templates.replace_as_expression(
         template,
         func=func,
@@ -79,9 +112,8 @@ class CallTreeTransformer(converter.Base):
         options=self.ctx.program.options.to_ast(
             self.ctx,
             internal_convert_user_code=self.ctx.program.options.recursive),
-        args=node.args)
-    # TODO(mdan): Improve the template mechanism to better support this.
-    new_call.keywords = node.keywords
+        args=args,
+        kwargs=kwargs)
 
     return new_call
 
diff --git a/tensorflow/python/autograph/converters/call_trees_test.py b/tensorflow/python/autograph/converters/call_trees_test.py
index 6ee56bf6be..654682edc7 100644
--- a/tensorflow/python/autograph/converters/call_trees_test.py
+++ b/tensorflow/python/autograph/converters/call_trees_test.py
@@ -34,7 +34,55 @@ class CallTreesTest(converter_testing.TestCase):
       self.assertEquals(
           result.test_fn(None),
           converter_testing.RESULT_OF_MOCK_CONVERTED_CALL + 3)
-      self.assertListEqual(self.dynamic_calls, [()])
+      self.assertListEqual(self.dynamic_calls, [((), {})])
+
+  def test_function_with_kwarg(self):
+
+    def test_fn(f, a, b):
+      return f(a, c=b) + 3
+
+    with self.converted(test_fn, call_trees, {}) as result:
+      self.assertEquals(
+          result.test_fn(None, 1, 2),
+          converter_testing.RESULT_OF_MOCK_CONVERTED_CALL + 3)
+      self.assertListEqual(self.dynamic_calls, [((1,), {'c': 2})])
+
+  def test_function_with_kwargs_starargs(self):
+
+    def test_fn(f, a, *args, **kwargs):
+      return f(a, *args, **kwargs) + 5
+
+    with self.converted(test_fn, call_trees, {}) as result:
+      self.assertEquals(
+          result.test_fn(None, 1, *[2, 3], **{'b': 4, 'c': 5}),
+          converter_testing.RESULT_OF_MOCK_CONVERTED_CALL + 5)
+      self.assertListEqual(self.dynamic_calls, [((1, 2, 3), {'b': 4, 'c': 5})])
+
+  def test_function_with_kwargs_starargs_only(self):
+
+    def f(*unused_args):  # Will not be called.
+      pass
+
+    def test_fn():
+      args = [1, 2, 3]
+      return f(*args) + 11
+
+    with self.converted(test_fn, call_trees, {'f': f}) as result:
+      self.assertEquals(
+          result.test_fn(),
+          converter_testing.RESULT_OF_MOCK_CONVERTED_CALL + 11)
+      self.assertListEqual(self.dynamic_calls, [((1, 2, 3), {})])
+
+  def test_function_with_kwargs_keywords(self):
+
+    def test_fn(f, a, b, **kwargs):
+      return f(a, b=b, **kwargs) + 5
+
+    with self.converted(test_fn, call_trees, {}) as result:
+      self.assertEquals(
+          result.test_fn(None, 1, 2, **{'c': 3}),
+          converter_testing.RESULT_OF_MOCK_CONVERTED_CALL + 5)
+      self.assertListEqual(self.dynamic_calls, [((1,), {'b': 2, 'c': 3})])
 
   def test_class_method(self):
 
@@ -47,7 +95,7 @@ class CallTreesTest(converter_testing.TestCase):
     with self.converted(TestClass.test_method, call_trees, {}) as result:
       self.assertEquals(converter_testing.RESULT_OF_MOCK_CONVERTED_CALL + 1,
                         result.test_method(tc, 1))
-      self.assertListEqual(self.dynamic_calls, [(1,)])
+      self.assertListEqual(self.dynamic_calls, [((1,), {})])
 
   def test_object_method(self):
 
@@ -60,7 +108,7 @@ class CallTreesTest(converter_testing.TestCase):
     with self.converted(tc.test_method, call_trees, {}) as result:
       self.assertEquals(converter_testing.RESULT_OF_MOCK_CONVERTED_CALL + 1,
                         result.test_method(tc, 1))
-      self.assertListEqual(self.dynamic_calls, [(1,)])
+      self.assertListEqual(self.dynamic_calls, [((1,), {})])
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/autograph/impl/api.py b/tensorflow/python/autograph/impl/api.py
index 0f874a138a..471ba3b795 100644
--- a/tensorflow/python/autograph/impl/api.py
+++ b/tensorflow/python/autograph/impl/api.py
@@ -90,7 +90,7 @@ def convert(
               verbose=verbose,
               force_conversion=True,
               optional_features=optional_features,
-          ), *args, **kwargs)
+          ), args, kwargs)
 
     wrapper = tf_decorator.make_decorator(f, wrapper)
 
@@ -162,7 +162,7 @@ def do_not_convert(run_as=RunMode.GRAPH, return_dtypes=None):
   return decorator
 
 
-def converted_call(f, owner, options, *args, **kwargs):
+def converted_call(f, owner, options, args, kwargs):
   """Compiles a function call inline. For internal use only."""
   logging.log(1,
               'Converted call: %s; owner: %s\n    args: %s\n    kwargs: %s\n',
diff --git a/tensorflow/python/autograph/impl/api_test.py b/tensorflow/python/autograph/impl/api_test.py
index 73e470a000..ebc6cc4f71 100644
--- a/tensorflow/python/autograph/impl/api_test.py
+++ b/tensorflow/python/autograph/impl/api_test.py
@@ -195,18 +195,17 @@ class ApiTest(test.TestCase):
       def test_method(self, x, s, a):
         while tf.reduce_sum(x) > s:
           x //= api.converted_call(self.called_member, None,
-                                   converter.ConversionOptions(), self, a)
+                                   converter.ConversionOptions(), (self, a), {})
         return x
 
     tc = TestClass()
-    with self.cached_session() as sess:
-      x = tc.test_method(
-          constant_op.constant([2, 4]), constant_op.constant(1),
-          constant_op.constant(-2))
-      self.assertListEqual([0, 1], self.evaluate(x).tolist())
+    x = tc.test_method(
+        constant_op.constant([2, 4]), constant_op.constant(1),
+        constant_op.constant(-2))
+    self.assertListEqual([0, 1], self.evaluate(x).tolist())
 
   def test_converted_call_builtin(self):
-    x = api.converted_call(range, None, converter.ConversionOptions(), 3)
+    x = api.converted_call(range, None, converter.ConversionOptions(), (3,), {})
     self.assertEqual((0, 1, 2), tuple(x))
 
   def test_converted_call_function(self):
@@ -216,10 +215,9 @@ class ApiTest(test.TestCase):
         return -x
       return x
 
-    with self.cached_session() as sess:
-      x = api.converted_call(test_fn, None, converter.ConversionOptions(),
-                             constant_op.constant(-1))
-      self.assertEqual(1, self.evaluate(x))
+    x = api.converted_call(test_fn, None, converter.ConversionOptions(),
+                           (constant_op.constant(-1),), {})
+    self.assertEqual(1, self.evaluate(x))
 
   @test_util.run_v1_only('b/120545219')
   def test_converted_call_functools_partial(self):
@@ -230,16 +228,14 @@ class ApiTest(test.TestCase):
       return x, y, z
 
     x = api.converted_call(
-        functools.partial(test_fn, constant_op.constant(-1), z=-3),
-        None, converter.ConversionOptions(),
-        constant_op.constant(-2))
+        functools.partial(test_fn, constant_op.constant(-1), z=-3), None,
+        converter.ConversionOptions(), (constant_op.constant(-2),), {})
     self.assertEqual((1, 2, 3), self.evaluate(x))
 
     x = api.converted_call(
         functools.partial(
-            functools.partial(test_fn, constant_op.constant(-1)), z=-3),
-        None, converter.ConversionOptions(),
-        constant_op.constant(-2))
+            functools.partial(test_fn, constant_op.constant(-1)), z=-3), None,
+        converter.ConversionOptions(), (constant_op.constant(-2),), {})
     self.assertEqual((1, 2, 3), self.evaluate(x))
 
   def test_converted_call_method_explicit_owner(self):
@@ -262,11 +258,10 @@ class ApiTest(test.TestCase):
           return -self.x
         return self.x
 
-    with self.cached_session() as sess:
-      tc = TestClass(constant_op.constant(-1))
-      x = api.converted_call(tc.test_method, None,
-                             converter.ConversionOptions(), tc)
-      self.assertEqual(1, self.evaluate(x))
+    tc = TestClass(constant_op.constant(-1))
+    x = api.converted_call(tc.test_method, None, converter.ConversionOptions(),
+                           (tc,), {})
+    self.assertEqual(1, self.evaluate(x))
 
   def test_converted_call_method_converts_recursively(self):
 
@@ -285,7 +280,8 @@ class ApiTest(test.TestCase):
 
     tc = TestClass(constant_op.constant(-1))
     x = api.converted_call(tc.test_method, None,
-                           converter.ConversionOptions(recursive=True), tc)
+                           converter.ConversionOptions(recursive=True), (tc,),
+                           {})
     self.assertEqual(1, self.evaluate(x))
 
   def test_converted_call_method_by_class(self):
@@ -300,11 +296,10 @@ class ApiTest(test.TestCase):
           return -self.x
         return self.x
 
-    with self.cached_session() as sess:
-      tc = TestClass(constant_op.constant(-1))
-      x = api.converted_call(TestClass.test_method, None,
-                             converter.ConversionOptions(), tc)
-      self.assertEqual(1, self.evaluate(x))
+    tc = TestClass(constant_op.constant(-1))
+    x = api.converted_call(TestClass.test_method, None,
+                           converter.ConversionOptions(), (tc,), {})
+    self.assertEqual(1, self.evaluate(x))
 
   def test_converted_call_callable_object(self):
 
@@ -318,10 +313,9 @@ class ApiTest(test.TestCase):
           return -self.x
         return self.x
 
-    with self.cached_session() as sess:
-      tc = TestClass(constant_op.constant(-1))
-      x = api.converted_call(tc, None, converter.ConversionOptions())
-      self.assertEqual(1, self.evaluate(x))
+    tc = TestClass(constant_op.constant(-1))
+    x = api.converted_call(tc, None, converter.ConversionOptions(), (), {})
+    self.assertEqual(1, self.evaluate(x))
 
   @test_util.run_deprecated_v1
   def test_converted_call_constructor(self):
@@ -337,7 +331,7 @@ class ApiTest(test.TestCase):
         return self.x
 
     tc = api.converted_call(TestClass, None, converter.ConversionOptions(),
-                            constant_op.constant(-1))
+                            (constant_op.constant(-1),), {})
     # tc is still a TestClass - constructors are whitelisted.
     # TODO(b/124016764): Support this use case.
     # The error below is specific to the `if` statement not being converted.
@@ -350,15 +344,14 @@ class ApiTest(test.TestCase):
     def f(x):
       return x == 0
 
-    with self.cached_session() as sess:
-      x = api.converted_call(f, None, converter.ConversionOptions(),
-                             constant_op.constant(0))
-      self.assertTrue(self.evaluate(x))
+    x = api.converted_call(f, None, converter.ConversionOptions(),
+                           (constant_op.constant(0),), {})
+    self.assertTrue(self.evaluate(x))
 
-      converted_f = api.to_graph(f)
-      x = api.converted_call(converted_f, None, converter.ConversionOptions(),
-                             constant_op.constant(0))
-      self.assertTrue(self.evaluate(x))
+    converted_f = api.to_graph(f)
+    x = api.converted_call(converted_f, None, converter.ConversionOptions(),
+                           (constant_op.constant(0),), {})
+    self.assertTrue(self.evaluate(x))
 
   def test_converted_call_then_already_converted_dynamic(self):
 
@@ -373,7 +366,7 @@ class ApiTest(test.TestCase):
       return g(x)
 
     x = api.converted_call(f, None, converter.ConversionOptions(),
-                           g, constant_op.constant(1))
+                           (g, constant_op.constant(1)), {})
     self.assertEqual(self.evaluate(x), 1)
 
   @test_util.run_deprecated_v1
@@ -387,10 +380,10 @@ class ApiTest(test.TestCase):
     # f should not be converted, causing len to error out.
     with self.assertRaisesRegexp(Exception,
                                  'object of type \'Tensor\' has no len()'):
-      api.converted_call(f, None, opts, constant_op.constant([0]))
+      api.converted_call(f, None, opts, (constant_op.constant([0]),), {})
 
     # len on the other hand should work fine.
-    x = api.converted_call(len, None, opts, constant_op.constant([0]))
+    x = api.converted_call(len, None, opts, (constant_op.constant([0]),), {})
     # The constant has static shape so the result is a primitive not a Tensor.
     self.assertEqual(x, 1)
 
@@ -403,11 +396,10 @@ class ApiTest(test.TestCase):
     ])
 
     x = api.converted_call(model.call, None, opts,
-                           constant_op.constant([[0.0]]), training=True)
+                           (constant_op.constant([[0.0]]),), {'training': True})
 
-    with self.cached_session() as sess:
-      self.evaluate(variables.global_variables_initializer())
-      self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
+    self.evaluate(variables.global_variables_initializer())
+    self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
 
   def test_converted_call_whitelisted_method_extra_self(self):
 
@@ -418,11 +410,11 @@ class ApiTest(test.TestCase):
     ])
 
     x = api.converted_call(model.call, None, opts,
-                           model, constant_op.constant([[0.0]]), training=True)
+                           (model, constant_op.constant([[0.0]])),
+                           {'training': True})
 
-    with self.cached_session() as sess:
-      self.evaluate(variables.global_variables_initializer())
-      self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
+    self.evaluate(variables.global_variables_initializer())
+    self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
 
   def test_converted_call_whitelisted_method_via_owner(self):
 
@@ -433,18 +425,17 @@ class ApiTest(test.TestCase):
     ])
 
     x = api.converted_call('call', model, opts,
-                           constant_op.constant([[0.0]]), training=True)
+                           (constant_op.constant([[0.0]]),), {'training': True})
 
-    with self.cached_session() as sess:
-      self.evaluate(variables.global_variables_initializer())
-      self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
+    self.evaluate(variables.global_variables_initializer())
+    self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
 
   def test_converted_call_namedtuple(self):
 
     opts = converter.ConversionOptions()
 
     x = api.converted_call(collections.namedtuple, None, opts,
-                           'TestNamedtuple', ('a', 'b'))
+                           ('TestNamedtuple', ('a', 'b')), {})
 
     self.assertTrue(inspect_utils.isnamedtuple(x))
 
@@ -452,8 +443,8 @@ class ApiTest(test.TestCase):
 
     opts = converter.ConversionOptions()
 
-    x = api.converted_call('namedtuple', collections, opts,
-                           'TestNamedtuple', ('a', 'b'))
+    x = api.converted_call('namedtuple', collections, opts, ('TestNamedtuple',
+                                                             ('a', 'b')), {})
 
     self.assertTrue(inspect_utils.isnamedtuple(x))
 
@@ -463,11 +454,10 @@ class ApiTest(test.TestCase):
 
     l = lambda x: x == 0
 
-    x = api.converted_call(l, None, opts, constant_op.constant(0))
+    x = api.converted_call(l, None, opts, (constant_op.constant(0),), {})
 
-    with self.cached_session() as sess:
-      self.evaluate(variables.global_variables_initializer())
-      self.assertAllEqual(True, self.evaluate(x))
+    self.evaluate(variables.global_variables_initializer())
+    self.assertAllEqual(True, self.evaluate(x))
 
   @test_util.run_deprecated_v1
   def test_to_graph_basic(self):
@@ -513,6 +503,19 @@ class ApiTest(test.TestCase):
     # TODO(b/122368197): This should be the constant 5!
     self.assertEqual(2, testing_global_numeric)
 
+  def test_to_graph_with_kwargs_clashing_converted_call(self):
+
+    def called_fn(**kwargs):
+      return kwargs['f'] + kwargs['owner']
+
+    def test_fn():
+      # These arg names intentionally match converted_call's
+      return called_fn(f=1, owner=2)
+
+    compiled_fn = api.to_graph(test_fn)
+
+    self.assertEqual(compiled_fn(), 3)
+
   def test_to_code_basic(self):
 
     def test_fn(x, s):
diff --git a/tensorflow/python/framework/func_graph.py b/tensorflow/python/framework/func_graph.py
index f83fc74cf9..311801675c 100644
--- a/tensorflow/python/framework/func_graph.py
+++ b/tensorflow/python/framework/func_graph.py
@@ -603,7 +603,7 @@ def func_graph_from_py_func(name,
                   strip_decorators=(def_function.function,),
                   optional_features=autograph_options,
                   force_conversion=True,
-              ), *args, **kwargs)
+              ), args, kwargs)
 
         # Wrapping around a decorator allows checks like tf_inspect.getargspec
         # to be accurate.
-- 
GitLab


From 628a91d49808161620e6e13eedd9361095515287 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 12 Feb 2019 06:15:39 -0800
Subject: [PATCH 2315/2345] Fixed tf.gather behavior when axis=None and
 batch_dims < 0

axis was set incorrectly when batch_dims was negative.

PiperOrigin-RevId: 233594260
---
 .../python/kernel_tests/gather_op_test.py     |  4 ++--
 tensorflow/python/ops/array_ops.py            | 20 +++++++++----------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/tensorflow/python/kernel_tests/gather_op_test.py b/tensorflow/python/kernel_tests/gather_op_test.py
index 1d6f7955aa..65043d9f4f 100644
--- a/tensorflow/python/kernel_tests/gather_op_test.py
+++ b/tensorflow/python/kernel_tests/gather_op_test.py
@@ -467,12 +467,12 @@ class GatherTest(test.TestCase, parameterized.TestCase):
   def testErrors(self):
 
     with self.assertRaisesRegexp(
-        ValueError, r"batch_dims = 2 must be less than ndims\(indices\) = 2"):
+        ValueError, r"batch_dims = 2 must be less than rank\(indices\) = 2"):
       array_ops.gather(
           params=[[1, 2], [3, 4]], indices=[[1, 2], [3, 4]], batch_dims=2)
 
     with self.assertRaisesRegexp(
-        ValueError, r"batch_dims = 1 must be less than ndims\(params\) = 1"):
+        ValueError, r"batch_dims = 1 must be less than rank\(params\) = 1"):
       array_ops.gather(
           params=[1, 2, 3, 4], indices=[[1, 2], [3, 4]], batch_dims=1)
 
diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 230ce62db7..054f21cfe9 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -3278,8 +3278,8 @@ def gather(params,
            batch_dims=0):
   r"""Gather slices from params axis axis according to indices.
 
-  Gather slices from params axis axis according to indices.  `indices` must be
-  an integer tensor of any dimension (usually 0-D or 1-D).
+  Gather slices from params axis `axis` according to `indices`.  `indices` must
+  be an integer tensor of any dimension (usually 0-D or 1-D).
 
   For 0-D (scalar) `indices`:
 
@@ -3336,17 +3336,17 @@ def gather(params,
       to `batch_dims`.  Defaults to the first non-batch dimension. Supports
       negative indexes.
     batch_dims: An `integer`.  The number of batch dimensions.  Must be less
-      than `ndims(inices)`.
+      than `rank(indices)`.
 
   Returns:
     A `Tensor`. Has the same type as `params`.
   """
   del validate_indices
-  if axis is None:
-    axis = batch_dims
   if batch_dims != 0:
     with ops.name_scope(name, "Gather", [params, indices, axis]):
       return _batch_gather(params, indices, batch_dims, axis)
+  if axis is None:
+    axis = batch_dims
   if axis != 0:
     # Note that we do a sparse_read here to avoid snapshotting the entire
     # resource variable and doing a gather, which can be inefficient and lead to
@@ -3393,15 +3393,15 @@ def _batch_gather(params, indices, batch_dims, axis=None):
   This operation assumes that the leading `batch_dims` dimensions of `indices`
   and `params` are batch dimensions; and performs a `tf.gather` operation within
   each batch. (If `batch_dims` is not specified, then it defaults to
-  `ndims(indices) - 1`.)  In the case in which `batch_dims==0`, this operation
+  `rank(indices)-1`.)  In the case in which `batch_dims==0`, this operation
   is equivalent to `tf.gather`.
 
   Args:
     params: A Tensor. The tensor from which to gather values.
     indices: A Tensor. Must be one of the following types: int32, int64. Index
       tensor. Must be in range `[0, params.shape[batch_dims]]`.
-    batch_dims: An integer.  The number of batch dimensions.  Must be less than
-      ndims(inices).  Defaults to `ndims(indices) - 1` if not specified.
+    batch_dims: An integer or none.  The number of batch dimensions.  Must be
+      less than `rank(indices)`.  Defaults to `rank(indices) - 1` if None.
     axis: A `Tensor`. Must be one of the following types: `int32`, `int64`. The
       `axis` in `params` to gather `indices` from. Must be greater than or equal
       to `batch_dims`.  Defaults to the first non-batch dimension. Supports
@@ -3427,10 +3427,10 @@ def _batch_gather(params, indices, batch_dims, axis=None):
   if batch_dims < 0:
     batch_dims += indices_ndims
   if batch_dims < 0 or batch_dims >= indices_ndims:
-    raise ValueError("batch_dims = %d must be less than ndims(indices) = %d" %
+    raise ValueError("batch_dims = %d must be less than rank(indices) = %d" %
                      (batch_dims, indices_ndims))
   if params.shape.ndims is not None and batch_dims >= params.shape.ndims:
-    raise ValueError("batch_dims = %d must be less than ndims(params) = %d" %
+    raise ValueError("batch_dims = %d must be less than rank(params) = %d" %
                      (batch_dims, params.shape.ndims))
 
   # Handle axis by transposing the axis dimension to be the first non-batch
-- 
GitLab


From df1ac2959e6abd04d05522ccbfcc4b61993c813c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 12 Feb 2019 07:12:55 -0800
Subject: [PATCH 2316/2345] [XLA:GPU] Test XLA:GPU with tensorrt tests.

The floating point tolerance also had to be increased some due to different CUDNN algorithms being used.

PiperOrigin-RevId: 233600675
---
 tensorflow/python/compiler/tensorrt/BUILD                      | 3 +++
 .../python/compiler/tensorrt/test/quantization_mnist_test.py   | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/compiler/tensorrt/BUILD b/tensorflow/python/compiler/tensorrt/BUILD
index afa25113b2..b9fe768fbb 100644
--- a/tensorflow/python/compiler/tensorrt/BUILD
+++ b/tensorflow/python/compiler/tensorrt/BUILD
@@ -110,6 +110,7 @@ cuda_py_test(
         "no_windows",
         "nomac",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_tests(
@@ -147,6 +148,7 @@ cuda_py_tests(
         "no_windows",
         "nomac",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -171,4 +173,5 @@ cuda_py_test(
         "no_windows",
         "nomac",
     ],
+    xla_enable_strict_auto_jit = True,
 )
diff --git a/tensorflow/python/compiler/tensorrt/test/quantization_mnist_test.py b/tensorflow/python/compiler/tensorrt/test/quantization_mnist_test.py
index e2b21d7b92..cf5e5b897a 100644
--- a/tensorflow/python/compiler/tensorrt/test/quantization_mnist_test.py
+++ b/tensorflow/python/compiler/tensorrt/test/quantization_mnist_test.py
@@ -273,7 +273,7 @@ class QuantizationAwareTrainingMNISTTest(test_util.TensorFlowTestCase):
         num_epochs=None,
         model_dir=model_dir)['accuracy']
     logging.info('accuracy_tf_native: %f', accuracy_tf_native)
-    self.assertAllClose(0.9662, accuracy_tf_native, rtol=1e-3, atol=1e-3)
+    self.assertAllClose(0.9662, accuracy_tf_native, rtol=3e-3, atol=3e-3)
 
     if get_linked_tensorrt_version()[0] < 5:
       return
-- 
GitLab


From 40ebea073b2168be0ad507b7eab385c670b84678 Mon Sep 17 00:00:00 2001
From: Brian Patton <bjp@google.com>
Date: Tue, 12 Feb 2019 07:16:55 -0800
Subject: [PATCH 2317/2345] Introduce a N-way op+kernel called Case.

The last branch is used as the default branch in case `branch_index < 0` or `branch_index >= len(branches)`.

PiperOrigin-RevId: 233601118
---
 tensorflow/cc/framework/cc_op_gen.cc          |  1 +
 .../core/api_def/base_api/api_def_Case.pbtxt  | 43 ++++++++
 .../api_def/python_api/api_def_Case.pbtxt     |  4 +
 tensorflow/core/framework/op_def_util.cc      |  2 +
 tensorflow/core/kernels/functional_ops.cc     | 97 +++++++++++++++++++
 tensorflow/core/ops/functional_ops.cc         | 29 ++++++
 tensorflow/java/src/gen/cc/op_generator.cc    |  2 +-
 tensorflow/python/eager/function_test.py      | 25 +++++
 tensorflow/python/eager/pywrap_tfe_src.cc     | 43 +++++++-
 .../kernel_tests/functional_ops_test.py       | 33 +++++++
 .../api/golden/v1/tensorflow.raw_ops.pbtxt    |  4 +
 .../api/golden/v2/tensorflow.raw_ops.pbtxt    |  4 +
 12 files changed, 283 insertions(+), 4 deletions(-)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_Case.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_Case.pbtxt

diff --git a/tensorflow/cc/framework/cc_op_gen.cc b/tensorflow/cc/framework/cc_op_gen.cc
index 39593370d1..43a33cbea6 100644
--- a/tensorflow/cc/framework/cc_op_gen.cc
+++ b/tensorflow/cc/framework/cc_op_gen.cc
@@ -321,6 +321,7 @@ std::pair<const char*, bool> AttrTypeName(StringPiece attr_type) {
           {"tensor", {"TensorProto", true}},
           {"list(tensor)", {"gtl::ArraySlice<TensorProto>", true}},
           {"func", {"NameAttrList", true}},
+          {"list(func)", {"gtl::ArraySlice<NameAttrList>", true}},
       };
 
   auto entry = attr_type_map->find(attr_type);
diff --git a/tensorflow/core/api_def/base_api/api_def_Case.pbtxt b/tensorflow/core/api_def/base_api/api_def_Case.pbtxt
new file mode 100644
index 0000000000..56fef3ae6d
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_Case.pbtxt
@@ -0,0 +1,43 @@
+op {
+  graph_op_name: "Case"
+  in_arg {
+    name: "branch_index"
+    description: "The branch selector, an int32 Tensor."
+  }
+  in_arg {
+    name: "input"
+    description: "A list of input tensors passed to the branch function."
+  }
+  out_arg {
+    name: "output"
+    description: "A list of return values."
+  }
+  attr { name: "Tin"  description: "A list of input types." }
+  attr { name: "Tout"  description: "A list of output types." }
+  attr {
+    name: "branches"
+    description: <<END
+      A list of functions each of which takes 'inputs' and returns a list of
+      tensors, whose types are the same as what every other branch returns.
+END
+  }
+  summary: "An n-way switch statement which calls a single branch function."
+  description: <<END
+    An n-way switch statement, implementing the following:
+    ```
+    switch (branch_index) {
+      case 0:
+        output = branches[0](input);
+        break;
+      case 1:
+        output = branches[1](input);
+        break;
+      ...
+      case [[nbranches-1]]:
+      default:
+        output = branches[nbranches-1](input);
+        break;
+    }
+    ```
+END
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_Case.pbtxt b/tensorflow/core/api_def/python_api/api_def_Case.pbtxt
new file mode 100644
index 0000000000..a4c8193a35
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_Case.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "Case"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/framework/op_def_util.cc b/tensorflow/core/framework/op_def_util.cc
index d629719096..080db8f619 100644
--- a/tensorflow/core/framework/op_def_util.cc
+++ b/tensorflow/core/framework/op_def_util.cc
@@ -114,6 +114,8 @@ Status ValidateAttrValue(const AttrValue& attr_value,
         length = attr_value.list().shape_size();
       } else if (attr.type() == "list(tensor)") {
         length = attr_value.list().tensor_size();
+      } else if (attr.type() == "list(func)") {
+        length = attr_value.list().func_size();
       }
       if (length < attr.minimum()) {
         return errors::InvalidArgument(
diff --git a/tensorflow/core/kernels/functional_ops.cc b/tensorflow/core/kernels/functional_ops.cc
index a3e7b53d55..246a6ce04d 100644
--- a/tensorflow/core/kernels/functional_ops.cc
+++ b/tensorflow/core/kernels/functional_ops.cc
@@ -23,6 +23,7 @@ limitations under the License.
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/threadpool.h"
 
 namespace tensorflow {
@@ -211,6 +212,98 @@ class IfOp : public AsyncOpKernel {
   };
 };
 
+class CaseOp : public AsyncOpKernel {
+ public:
+  explicit CaseOp(OpKernelConstruction* ctx) : AsyncOpKernel(ctx) {
+    auto lib = ctx->function_library();
+    OP_REQUIRES(ctx, lib != nullptr, errors::Internal("No function library"));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("branches", &branch_funcs_));
+  }
+
+  ~CaseOp() override {}
+
+  void ComputeAsync(OpKernelContext* ctx, DoneCallback done) override {
+    auto lib = ctx->function_library();
+    OP_REQUIRES_ASYNC(ctx, lib != nullptr,
+                      errors::Internal("No function library"), done);
+
+    // TODO(b/37549631): Because this op has `SetIsStateful()` in its op
+    // registration, this kernel may be shared by multiple subgraphs, which have
+    // different associated `FunctionLibraryRuntime` objects and hence different
+    // `FHandle` namespaces. So we must call Instantiate() to make sure we get
+    // the correct function handles with respect to `lib`. Note the underlying
+    // `lib->Instantiate()` caches the created function handles, so calling
+    // `Instantiate()` repeatedly on the same `lib` and function is cheap.
+    std::vector<FHandle> branch_handles(branch_funcs_.size());
+    for (int i = 0; i < branch_funcs_.size(); i++) {
+      OP_REQUIRES_OK_ASYNC(
+          ctx, Instantiate(lib, branch_funcs_[i], &branch_handles[i]), done);
+    }
+
+    const Tensor& branch_index = ctx->input(0);
+    OP_REQUIRES_ASYNC(ctx, TensorShapeUtils::IsScalar(branch_index.shape()),
+                      errors::InvalidArgument("branch_index must be scalar"),
+                      done);
+    int32 branch = branch_index.scalar<int32>()();
+    (new State(this, ctx, branch, branch_handles, done))->Start();
+  }
+
+ private:
+  std::vector<NameAttrList> branch_funcs_;
+
+  class State {
+   public:
+    State(CaseOp* kernel, OpKernelContext* ctx, int branch,
+          std::vector<FHandle> branch_handles, DoneCallback done)
+        : kernel_(kernel),
+          ctx_(ctx),
+          branch_(branch),
+          branch_handles_(branch_handles),
+          done_(std::move(done)),
+          lib_(CHECK_NOTNULL(ctx_->function_library())) {
+      SetRunOptions(ctx_, &opts_, true /* always_collect_stats */);
+      for (int i = 1; i < ctx_->num_inputs(); ++i) {
+        args_.push_back(ctx_->input(i));
+      }
+    }
+
+    ~State() {}
+
+    void Start() {
+      int branch = branch_;
+      // The last branch is the default branch.
+      if (branch < 0 || branch >= branch_handles_.size()) {
+        branch = branch_handles_.size() - 1;
+      }
+      rets_.clear();
+      lib_->Run(
+          // Evaluate one of the branch.
+          opts_, branch_handles_[branch], args_, &rets_,
+          // Done callback
+          [this](Status s) {
+            if (s.ok()) {
+              s = SetOutputs(kernel_, ctx_, rets_);
+            }
+            ctx_->SetStatus(s);
+            DoneCallback captured_done(std::move(done_));
+            delete this;
+            captured_done();
+          });
+    }
+
+   private:
+    CaseOp* const kernel_;
+    OpKernelContext* const ctx_;
+    const int branch_;
+    std::vector<FHandle> branch_handles_;
+    DoneCallback done_;
+    FunctionLibraryRuntime* const lib_;
+    FunctionLibraryRuntime::Options opts_;
+    TensorVec args_;
+    TensorVec rets_;
+  };
+};
+
 // TODO(drpng): remove this.
 REGISTER_KERNEL_BUILDER(Name("_If").Device(DEVICE_CPU), IfOp);
 REGISTER_KERNEL_BUILDER(Name("_If").Device(DEVICE_GPU).HostMemory("cond"),
@@ -219,6 +312,10 @@ REGISTER_KERNEL_BUILDER(Name("_If").Device(DEVICE_GPU).HostMemory("cond"),
 REGISTER_KERNEL_BUILDER(Name("If").Device(DEVICE_CPU), IfOp);
 REGISTER_KERNEL_BUILDER(Name("If").Device(DEVICE_GPU).HostMemory("cond"), IfOp);
 
+REGISTER_KERNEL_BUILDER(Name("Case").Device(DEVICE_CPU), CaseOp);
+REGISTER_KERNEL_BUILDER(
+    Name("Case").Device(DEVICE_GPU).HostMemory("branch_index"), CaseOp);
+
 REGISTER_KERNEL_BUILDER(Name("StatelessIf").Device(DEVICE_CPU), IfOp);
 REGISTER_KERNEL_BUILDER(
     Name("StatelessIf").Device(DEVICE_GPU).HostMemory("cond"), IfOp);
diff --git a/tensorflow/core/ops/functional_ops.cc b/tensorflow/core/ops/functional_ops.cc
index be440ed728..4982ec6bd8 100644
--- a/tensorflow/core/ops/functional_ops.cc
+++ b/tensorflow/core/ops/functional_ops.cc
@@ -132,6 +132,35 @@ REGISTER_OP("If")
       return Status::OK();
     });
 
+REGISTER_OP("Case")
+    .Input("branch_index: int32")
+    .Input("input: Tin")
+    .Output("output: Tout")
+    .Attr("Tin: list(type) >= 0")
+    .Attr("Tout: list(type) >= 0")
+    .Attr("branches: list(func) >= 1")
+    .Attr("output_shapes: list(shape) = []")
+    .SetIsStateful()
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      std::vector<PartialTensorShape> output_shapes;
+      TF_RETURN_IF_ERROR(c->GetAttr("output_shapes", &output_shapes));
+      // If `output_shapes` attr is set use that as the shapes of the outputs
+      // else return unknown shapes.
+      if (output_shapes.empty()) return shape_inference::UnknownShape(c);
+      if (output_shapes.size() != c->num_outputs()) {
+        return errors::InvalidArgument(
+            "`output_shapes` must be the same length as num outputs (",
+            output_shapes.size(), " vs. ", c->num_outputs());
+      }
+      for (size_t i = 0; i < output_shapes.size(); ++i) {
+        shape_inference::ShapeHandle output_shape_handle;
+        TF_RETURN_IF_ERROR(c->MakeShapeFromPartialTensorShape(
+            output_shapes[i], &output_shape_handle));
+        c->set_output(static_cast<int>(i), output_shape_handle);
+      }
+      return Status::OK();
+    });
+
 // TODO(drpng): remove this.
 REGISTER_OP("_While")
     .Input("input: T")
diff --git a/tensorflow/java/src/gen/cc/op_generator.cc b/tensorflow/java/src/gen/cc/op_generator.cc
index 5d6387e88e..db6116bd5c 100644
--- a/tensorflow/java/src/gen/cc/op_generator.cc
+++ b/tensorflow/java/src/gen/cc/op_generator.cc
@@ -516,7 +516,7 @@ bool CanGenerateOp(const OpDef& op_def, const ApiDef& api_def) {
     return false;
   }
   for (const auto& attr : op_def.attr()) {
-    if (attr.type() == "func") {
+    if (attr.type() == "func" || attr.type() == "list(func)") {
       return false;  // TODO(karllessard) add support for function attributes
     }
   }
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index 834691eedd..a3103e5d61 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -49,6 +49,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import clip_ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import gen_functional_ops
 from tensorflow.python.ops import gen_random_ops
 from tensorflow.python.ops import gen_resource_variable_ops
 from tensorflow.python.ops import init_ops
@@ -528,6 +529,30 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     var_t = resource_variable_ops.read_variable_op(var_handle, dtype=v.dtype)
     self.assertEqual(var_t.shape, tensor_shape.TensorShape([2, 2]))
 
+  def testFuncListAttr(self):
+
+    @function.defun
+    def test_function(val):
+
+      def fn1():
+        return array_ops.ones([10])
+
+      fn2 = lambda: array_ops.ones([10]) * 2
+
+      def fn3(x=2):
+        return array_ops.ones([10]) * x
+      fn3 = functools.partial(fn3, x=3)
+
+      return gen_functional_ops.case(val, [], [dtypes.float32],
+                                     [function.defun(f).get_concrete_function()
+                                      for f in (fn1, fn2, fn3)])
+
+    ones = array_ops.ones([10])
+    self.assertAllEqual([ones], test_function(0))
+    self.assertAllEqual([ones * 2], test_function(1))
+    self.assertAllEqual([ones * 3], test_function(2))
+    self.assertAllEqual([ones * 3], test_function(22))  # default branch
+
   @test_util.enable_control_flow_v2
   def testVariableInLoopInFunction(self):
 
diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc
index 6679e9cc48..d272940bf0 100644
--- a/tensorflow/python/eager/pywrap_tfe_src.cc
+++ b/tensorflow/python/eager/pywrap_tfe_src.cc
@@ -264,7 +264,8 @@ bool ParseTypeValue(const string& key, PyObject* py_value, TF_Status* status,
 }
 
 bool SetOpAttrList(
-    TFE_Op* op, const char* key, PyObject* py_list, TF_AttrType type,
+    TFE_Context* ctx, TFE_Op* op, const char* key, PyObject* py_list,
+    TF_AttrType type,
     tensorflow::gtl::FlatMap<string, tensorflow::int64>* attr_list_sizes,
     TF_Status* status) {
   if (!PySequence_Check(py_list)) {
@@ -369,6 +370,40 @@ bool SetOpAttrList(
     TFE_OpSetAttrShapeList(op, key, dims.get(), num_dims.get(), num_values,
                            status);
     if (TF_GetCode(status) != TF_OK) return false;
+  } else if (type == TF_ATTR_FUNC) {
+    std::unique_ptr<const TFE_Op*[]> funcs(new const TFE_Op*[num_values]);
+    for (int i = 0; i < num_values; ++i) {
+      tensorflow::Safe_PyObjectPtr py_value(PySequence_ITEM(py_list, i));
+      // Allow:
+      // (1) String function name, OR
+      // (2) A Python object with a .name attribute
+      //     (A crude test for being a
+      //     tensorflow.python.framework.function._DefinedFunction)
+      //     (which is what the various "defun" or "Defun" decorators do).
+      // And in the future also allow an object that can encapsulate
+      // the function name and its attribute values.
+      tensorflow::StringPiece func_name;
+      if (!ParseStringValue(key, py_value.get(), status, &func_name)) {
+        PyObject* name_attr = PyObject_GetAttrString(py_value.get(), "name");
+        if (name_attr == nullptr ||
+            !ParseStringValue(key, name_attr, status, &func_name)) {
+          TF_SetStatus(
+              status, TF_INVALID_ARGUMENT,
+              tensorflow::strings::StrCat(
+                  "unable to set function value attribute from a ",
+                  py_value.get()->ob_type->tp_name,
+                  " object. If you think this is an error, please file an "
+                  "issue at "
+                  "https://github.com/tensorflow/tensorflow/issues/new")
+                  .c_str());
+          return false;
+        }
+      }
+      funcs[i] = TFE_NewOp(ctx, func_name.data(), status);
+      if (TF_GetCode(status) != TF_OK) return false;
+    }
+    TFE_OpSetAttrFunctionList(op, key, funcs.get(), num_values);
+    if (TF_GetCode(status) != TF_OK) return false;
   } else {
     TF_SetStatus(status, TF_UNIMPLEMENTED,
                  tensorflow::strings::StrCat("Attr ", key,
@@ -619,7 +654,8 @@ void SetOpAttrs(TFE_Context* ctx, TFE_Op* op, PyObject* attrs, int start_index,
     const TF_AttrType type = TFE_OpGetAttrType(op, key, &is_list, out_status);
     if (TF_GetCode(out_status) != TF_OK) return;
     if (is_list != 0) {
-      if (!SetOpAttrList(op, key, py_value, type, nullptr, out_status)) return;
+      if (!SetOpAttrList(ctx, op, key, py_value, type, nullptr, out_status))
+        return;
     } else {
       if (!SetOpAttrScalar(ctx, op, key, py_value, type, nullptr, out_status))
         return;
@@ -649,7 +685,8 @@ void SetOpAttrWithDefaults(
     }
   } else {
     if (is_list != 0) {
-      SetOpAttrList(op, attr_name, attr_value, type, attr_list_sizes, status);
+      SetOpAttrList(ctx, op, attr_name, attr_value, type, attr_list_sizes,
+                    status);
     } else {
       SetOpAttrScalar(ctx, op, attr_name, attr_value, type, attr_list_sizes,
                       status);
diff --git a/tensorflow/python/kernel_tests/functional_ops_test.py b/tensorflow/python/kernel_tests/functional_ops_test.py
index 48e14d0f50..0ab2b4bdfb 100644
--- a/tensorflow/python/kernel_tests/functional_ops_test.py
+++ b/tensorflow/python/kernel_tests/functional_ops_test.py
@@ -23,6 +23,7 @@ import numpy as np
 from tensorflow.core.framework import attr_value_pb2
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.client import session
+from tensorflow.python.eager import function as eager_function
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -32,6 +33,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import functional_ops
+from tensorflow.python.ops import gen_functional_ops
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import math_ops
@@ -1110,6 +1112,37 @@ class PartitionedCallTest(test.TestCase):
       self.evaluate(op)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+@test_util.with_control_flow_v2
+class FunctionalOpsCaseTest(test.TestCase):
+
+  def testCase(self):
+    @eager_function.defun
+    def two(x):
+      return x * 2
+
+    @eager_function.defun
+    def three(x):
+      return x * 3
+
+    @eager_function.defun
+    def four(x):
+      return x * 4
+
+    def f(branch, x):
+      tmpl = array_ops.zeros_like(x)
+      return array_ops.identity(gen_functional_ops.case(
+          branch, input=[x], Tout=[dtypes.float32],
+          branches=[f.get_concrete_function(tmpl)
+                    for f in (two, three, four)])[0])
+    one = array_ops.ones([])
+    self.assertAllEqual(np.float32(2), self.evaluate(f(0, one)))
+    self.assertAllEqual(np.float32(3), self.evaluate(f(1, one)))
+    self.assertAllEqual(np.float32(4), self.evaluate(f(2, one)))
+    self.assertAllEqual(np.float32(4), self.evaluate(f(-1, one)))  # <0 default
+    self.assertAllEqual(np.float32(4), self.evaluate(f(6, one)))  # >=N default
+
+
 if __name__ == "__main__":
   test.main()
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt
index 2a63036ffa..76177d4c41 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt
@@ -532,6 +532,10 @@ tf_module {
     name: "CacheDataset"
     argspec: "args=[\'input_dataset\', \'filename\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "Case"
+    argspec: "args=[\'branch_index\', \'input\', \'Tout\', \'branches\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "Cast"
     argspec: "args=[\'x\', \'DstT\', \'Truncate\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt
index 2a63036ffa..76177d4c41 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt
@@ -532,6 +532,10 @@ tf_module {
     name: "CacheDataset"
     argspec: "args=[\'input_dataset\', \'filename\', \'output_types\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "Case"
+    argspec: "args=[\'branch_index\', \'input\', \'Tout\', \'branches\', \'output_shapes\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "Cast"
     argspec: "args=[\'x\', \'DstT\', \'Truncate\'], varargs=None, keywords=None, defaults=None"
-- 
GitLab


From d8c7dec71db10418698d5749c7245ea20eb4fe24 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 12 Feb 2019 08:02:30 -0800
Subject: [PATCH 2318/2345] Upgrade gRPC version used in OSS Tensorflow

PiperOrigin-RevId: 233606981
---
 tensorflow/contrib/cmake/external/grpc.cmake | 2 +-
 tensorflow/workspace.bzl                     | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/cmake/external/grpc.cmake b/tensorflow/contrib/cmake/external/grpc.cmake
index 22f1170f39..379b530361 100644
--- a/tensorflow/contrib/cmake/external/grpc.cmake
+++ b/tensorflow/contrib/cmake/external/grpc.cmake
@@ -17,7 +17,7 @@ include (ExternalProject)
 set(GRPC_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/include)
 set(GRPC_URL https://github.com/grpc/grpc.git)
 set(GRPC_BUILD ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc)
-set(GRPC_TAG c61d3e8308df248a13e94e9eebc674d6481e9ca4)
+set(GRPC_TAG d0d93bdab84f2befb425e9a991d17dc78c195c6d)
 
 if(WIN32)
   # We use unsecure gRPC because boringssl does not build on windows
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index bf25d2e1b7..692ccae0ec 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -478,12 +478,12 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     # WARNING: make sure ncteisen@ and vpai@ are cc-ed on any CL to change the below rule
     tf_http_archive(
         name = "grpc",
-        sha256 = "c1960c2d96914431bcaec97179f025868e6d641b7ae9b13d2cf94b161139bd67",
-        strip_prefix = "grpc-c61d3e8308df248a13e94e9eebc674d6481e9ca4",
+        sha256 = "0f4848edb44486304c88e3660f524c767ff4981835051fdcfdbb76883300d954",
+        strip_prefix = "grpc-d0d93bdab84f2befb425e9a991d17dc78c195c6d",
         system_build_file = clean_dep("//third_party/systemlibs:grpc.BUILD"),
         urls = [
-            "https://mirror.bazel.build/github.com/grpc/grpc/archive/c61d3e8308df248a13e94e9eebc674d6481e9ca4.tar.gz",
-            "https://github.com/grpc/grpc/archive/c61d3e8308df248a13e94e9eebc674d6481e9ca4.tar.gz",
+            "https://mirror.bazel.build/github.com/grpc/grpc/archive/d0d93bdab84f2befb425e9a991d17dc78c195c6d.tar.gz",
+            "https://github.com/grpc/grpc/archive/d0d93bdab84f2befb425e9a991d17dc78c195c6d.tar.gz",
         ],
     )
 
-- 
GitLab


From 3df272f792e386f4b8c9a7dbad5ddb395d278ad3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 12 Feb 2019 08:03:03 -0800
Subject: [PATCH 2319/2345] Keras should not cache sessions across different
 graphs. In this second attempt, we make sure that `backend.get_value(x)` uses
 the graph of `x` instead of the current default graph.

PiperOrigin-RevId: 233607207
---
 .../contrib/distribute/python/keras_test.py   | 22 +++++-------
 tensorflow/python/keras/backend.py            | 36 +++++++++++++------
 tensorflow/python/keras/backend_test.py       | 10 ++++++
 .../engine/distributed_training_utils.py      |  4 +--
 tensorflow/python/keras/engine/training.py    | 12 ++++---
 .../keras/engine/training_distributed.py      |  6 ++--
 .../python/keras/engine/training_utils.py     |  2 +-
 .../golden/v1/tensorflow.keras.backend.pbtxt  |  2 +-
 8 files changed, 59 insertions(+), 35 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 3934279a61..2eca1d1877 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -247,6 +247,7 @@ def multi_input_output_model():
 strategies_minus_tpu = [
     combinations.default_strategy,
     combinations.one_device_strategy,
+    combinations.one_device_strategy_gpu,
     combinations.mirrored_strategy_with_gpu_and_cpu,
     combinations.mirrored_strategy_with_two_gpus,
     combinations.core_mirrored_strategy_with_gpu_and_cpu,
@@ -275,6 +276,7 @@ def all_strategy_combinations_minus_default():
   strategy_minus_default_combinations = combinations.combine(
       distribution=[
           combinations.one_device_strategy,
+          combinations.one_device_strategy_gpu,
           combinations.mirrored_strategy_with_gpu_and_cpu,
           combinations.mirrored_strategy_with_two_gpus,
           combinations.core_mirrored_strategy_with_gpu_and_cpu,
@@ -298,12 +300,6 @@ def strategy_and_optimizer_combinations():
       ]))
 
 
-def strategy_for_numpy_input_combinations():
-  one_gpu = combinations.one_device_strategy_gpu
-  return (all_strategy_combinations() +
-          combinations.combine(distribution=[one_gpu], mode=['graph', 'eager']))
-
-
 class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase,
                                         parameterized.TestCase):
 
@@ -459,7 +455,7 @@ class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase,
 class TestDistributionStrategyWithNumpyArrays(test.TestCase,
                                               parameterized.TestCase):
 
-  @combinations.generate(strategy_for_numpy_input_combinations())
+  @combinations.generate(all_strategy_combinations())
   def test_calculating_input_params_no_steps_no_batch_size(self, distribution):
     # Calculate the per_replica_batch_size scaling factor for strategies
     # that use per_core_batch_size
@@ -490,7 +486,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
         distributed_training_utils.get_input_params(
             distribution, input_63_samples, steps=None, batch_size=None)
 
-  @combinations.generate(strategy_for_numpy_input_combinations())
+  @combinations.generate(all_strategy_combinations())
   def test_calculating_input_params_with_steps_no_batch_size(self,
                                                              distribution):
     # Calculate the per_replica_batch_size scaling factor for strategies
@@ -536,7 +532,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
           distributed_training_utils.get_input_params(
               distribution, input_63_samples, steps=1, batch_size=None)
 
-  @combinations.generate(strategy_for_numpy_input_combinations())
+  @combinations.generate(all_strategy_combinations())
   def test_calculating_input_params_no_steps_with_batch_size(self,
                                                              distribution):
     # Calculate the per_replica_batch_size scaling factor for strategies
@@ -570,7 +566,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
         distributed_training_utils.get_input_params(
             distribution, input_64_samples, steps=None, batch_size=3)
 
-  @combinations.generate(strategy_for_numpy_input_combinations())
+  @combinations.generate(all_strategy_combinations())
   def test_calculating_input_params_with_steps_with_batch_size(self,
                                                                distribution):
     with self.cached_session():
@@ -587,7 +583,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
         distributed_training_utils.get_input_params(
             distribution, input_64_samples, steps=10, batch_size=13)
 
-  @combinations.generate(strategy_for_numpy_input_combinations())
+  @combinations.generate(all_strategy_combinations())
   def test_calling_model_with_numpy_arrays(self, distribution):
     with self.cached_session():
       with distribution.scope():
@@ -618,7 +614,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
         # with batch_size
         model.predict(inputs, batch_size=8)
 
-  @combinations.generate(strategy_for_numpy_input_combinations())
+  @combinations.generate(all_strategy_combinations())
   def test_calling_model_with_nested_numpy_arrays(self, distribution):
     with self.cached_session():
       with distribution.scope():
@@ -670,7 +666,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
       model.fit(inputs, targets, sample_weight=sample_weights, epochs=1,
                 steps_per_epoch=2, verbose=1)
 
-  @combinations.generate(strategy_for_numpy_input_combinations())
+  @combinations.generate(all_strategy_combinations())
   def test_flatten_predict_outputs(self, distribution):
     with self.cached_session():
       with distribution.scope():
diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index 9e85e27d48..5d52e33cd5 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -389,16 +389,26 @@ def eager_learning_phase_scope(value):
     _GRAPH_LEARNING_PHASES[_DUMMY_EAGER_GRAPH] = previous_value
 
 
-def _get_session():
+def _current_graph(op_input_list):
+  """Return the graph members of `op_input_list`, or the current graph."""
+  return ops._get_graph_from_inputs(op_input_list)
+
+
+def _get_session(op_input_list=()):
   """Returns the session object for the current thread."""
   global _SESSION
   default_session = ops.get_default_session()
   if default_session is not None:
     session = default_session
   else:
-    if getattr(_SESSION, 'session', None) is None:
-      # We are creating the Session inside a Distribution
-      # Strategy scope.
+    if ops.inside_function():
+      raise RuntimeError('Cannot get session inside Tensorflow graph function.')
+    # If we don't have a session, or that session does not match the current
+    # graph, create and cache a new session.
+    if (getattr(_SESSION, 'session', None) is None or
+        _SESSION.session.graph is not _current_graph(op_input_list)):
+      # If we are creating the Session inside a tf.distribute.Strategy scope,
+      # we ask the strategy for the right session options to use.
       if distribution_strategy_context.has_strategy():
         configure_and_create_distributed_session(
             distribution_strategy_context.get_strategy())
@@ -410,12 +420,13 @@ def _get_session():
 
 
 @keras_export(v1=['keras.backend.get_session'])
-def get_session():
+def get_session(op_input_list=()):
   """Returns the TF session to be used by the backend.
 
   If a default TensorFlow session is available, we will return it.
 
-  Else, we will return the global Keras session.
+  Else, we will return the global Keras session assuming it matches
+  the current graph.
 
   If no global Keras session exists at this point:
   we will create a new global session.
@@ -423,10 +434,15 @@ def get_session():
   Note that you can manually set the global session
   via `K.set_session(sess)`.
 
+  Arguments:
+      op_input_list: An option sequence of tensors or ops, which will be used
+        to determine the current graph. Otherwise the default graph will be
+        used.
+
   Returns:
       A TensorFlow session.
   """
-  session = _get_session()
+  session = _get_session(op_input_list)
   if not _MANUAL_VAR_INIT:
     with session.graph.as_default():
       _initialize_variables(session)
@@ -2753,7 +2769,7 @@ def get_value(x):
       return x.numpy()
   elif ops.inside_function():
     raise RuntimeError('Cannot get value inside Tensorflow graph function.')
-  return x.eval(session=get_session())
+  return x.eval(session=get_session((x,)))
 
 
 @keras_export('keras.backend.batch_get_value')
@@ -2774,7 +2790,7 @@ def batch_get_value(tensors):
   elif ops.inside_function():  # pylint: disable=protected-access
     raise RuntimeError('Cannot get value inside Tensorflow graph function.')
   if tensors:
-    return get_session().run(tensors)
+    return get_session(tensors).run(tensors)
   else:
     return []
 
@@ -2992,7 +3008,7 @@ class GraphExecutionFunction(object):
   def __call__(self, inputs):
     inputs = nest.flatten(inputs)
 
-    session = get_session()
+    session = get_session(inputs)
     feed_arrays = []
     array_vals = []
     feed_symbols = []
diff --git a/tensorflow/python/keras/backend_test.py b/tensorflow/python/keras/backend_test.py
index 263a1dc461..cd7821639b 100644
--- a/tensorflow/python/keras/backend_test.py
+++ b/tensorflow/python/keras/backend_test.py
@@ -1741,6 +1741,16 @@ class BackendGraphTests(test.TestCase):
         x, mean, var, beta, gamma, axis=1, epsilon=1e-3)
     self.assertEqual(normed.shape.as_list(), [10, 3, 5, 5])
 
+  def test_get_session_different_graphs(self):
+    with ops.Graph().as_default():
+      x = keras.backend.constant(1)
+      session = keras.backend.get_session()
+      self.assertIs(session, keras.backend.get_session((x,)))
+      self.assertIs(session, keras.backend.get_session())
+    with ops.Graph().as_default():
+      self.assertIs(session, keras.backend.get_session((x,)))
+      self.assertIsNot(session, keras.backend.get_session())
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index 7702cfabf9..de6968e103 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -67,7 +67,7 @@ def set_weights(distribution_strategy, dist_model, weights):
     weights = weights[num_param:]
 
   if not ops.executing_eagerly_outside_functions():
-    K.get_session().run(assign_ops)
+    K.get_session(assign_ops).run(assign_ops)
 
 
 def unwrap_values(distribution_strategy, grouped_inputs, grouped_outputs,
@@ -522,7 +522,7 @@ def initialize_iterator(iterator, distribution_strategy):
   with distribution_strategy.scope():
     init_op = control_flow_ops.group(iterator.initialize())
     if not context.executing_eagerly():
-      K.get_session().run(init_op)
+      K.get_session((init_op,)).run(init_op)
 
 
 def _get_input_from_iterator(iterator, model):
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 34e2e30a0b..73b99210ec 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -2318,13 +2318,15 @@ class Model(Network):
       if shuffle:
         training_utils.verify_dataset_shuffled(x)
 
-    if ops.executing_eagerly_outside_functions():
-      session = None
-    else:
-      session = K.get_session()
-
     strategy = self._distribution_strategy
     with strategy.scope():
+      # We should be sure to call get_session() inside the strategy.scope()
+      # so the strategy can affect the session options.
+      if ops.executing_eagerly_outside_functions():
+        session = None
+      else:
+        session = K.get_session()
+
       first_x_value = nest.flatten(x)[0]
       if isinstance(first_x_value, np.ndarray):
         x = distributed_training_utils.list_to_tuple(x)
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 92a12f0dc1..3df58af0a2 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -368,7 +368,7 @@ def experimental_tpu_fit_loop(model,
         steps_per_run.load(step_count, K.get_session())
         prev_step_count = step_count
       try:
-        _, outputs = K.get_session().run([train_op, output_tensors])
+        _, outputs = K.batch_get_value([train_op, output_tensors])
       except errors.OutOfRangeError:
         if use_steps:
           logging.warning('Your dataset iterator ran out of data; '
@@ -550,7 +550,7 @@ def experimental_tpu_test_loop(model,
     batch_logs = {'batch': current_step, 'size': 1}
     callbacks._call_batch_hook(mode, 'begin', current_step, batch_logs)
     try:
-      _, batch_outs = K.get_session().run([test_op, output_tensors])
+      _, batch_outs = K.batch_get_value([test_op, output_tensors])
     except errors.OutOfRangeError:
       if steps is not None:
         warning_msg = 'Make sure that your dataset can generate at least '
@@ -731,7 +731,7 @@ def experimental_tpu_predict_loop(model,
     batch_logs = {'batch': current_step, 'size': 1}
     callbacks._call_batch_hook(mode, 'begin', current_step, batch_logs)
     try:
-      _, batch_outs = K.get_session().run([predict_op, output_tensors])
+      _, batch_outs = K.batch_get_value([predict_op, output_tensors])
     except errors.OutOfRangeError:
       if steps is not None:
         warning_msg = 'Make sure that your dataset can generate at least '
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index 47c1ba885a..cb1ca9e429 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -1270,7 +1270,7 @@ def get_iterator(dataset):
 def initialize_iterator(iterator):
   init_op = iterator.initializer
   if not context.executing_eagerly():
-    K.get_session().run(init_op)
+    K.get_session((init_op,)).run(init_op)
 
 
 def extract_tensors_from_dataset(dataset):
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.backend.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.backend.pbtxt
index 712e3ad6fc..c64a90890f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.backend.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.backend.pbtxt
@@ -198,7 +198,7 @@ tf_module {
   }
   member_method {
     name: "get_session"
-    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'op_input_list\'], varargs=None, keywords=None, defaults=[\'()\'], "
   }
   member_method {
     name: "get_uid"
-- 
GitLab


From 5b9e7a7782b799fcf8c88bf3c5c57f3b43c1eb5d Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Tue, 12 Feb 2019 08:17:19 -0800
Subject: [PATCH 2320/2345] Throwing an error if the batch size is not
 divisible by num_workers in RebatchDataset. Also greatly simplifying the
 graph mutation logic in RebatchDataset now that we know that the constant
 value for batch_size is already computed.

PiperOrigin-RevId: 233609414
---
 .../core/grappler/optimizers/data/BUILD       |  1 +
 .../core/grappler/optimizers/data/rebatch.cc  | 43 ++++++++++++-------
 .../kernel_tests/rebatch_dataset_test.py      | 12 +++---
 .../python/data/experimental/ops/batching.py  |  4 +-
 4 files changed, 38 insertions(+), 22 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/data/BUILD b/tensorflow/core/grappler/optimizers/data/BUILD
index 3fa6671c49..7181d5df0c 100644
--- a/tensorflow/core/grappler/optimizers/data/BUILD
+++ b/tensorflow/core/grappler/optimizers/data/BUILD
@@ -554,6 +554,7 @@ cc_library(
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
         "//tensorflow/core/grappler/utils:functions",
+        "//tensorflow/core:lib",
     ] + tf_protos_all(),
     alwayslink = 1,
 )
diff --git a/tensorflow/core/grappler/optimizers/data/rebatch.cc b/tensorflow/core/grappler/optimizers/data/rebatch.cc
index ce958c9235..14c3931267 100644
--- a/tensorflow/core/grappler/optimizers/data/rebatch.cc
+++ b/tensorflow/core/grappler/optimizers/data/rebatch.cc
@@ -16,12 +16,16 @@ limitations under the License.
 #include "tensorflow/core/grappler/optimizers/data/rebatch.h"
 
 #include "absl/container/flat_hash_map.h"
+#include "tensorflow/core/framework/attr_value_util.h"
+#include "tensorflow/core/framework/node_def.pb.h"
+#include "tensorflow/core/framework/node_def_util.h"
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/versions.pb.h"
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.h"
 #include "tensorflow/core/grappler/optimizers/data/graph_utils.h"
 #include "tensorflow/core/grappler/utils/functions.h"
+#include "tensorflow/core/lib/core/errors.h"
 
 namespace tensorflow {
 namespace grappler {
@@ -38,6 +42,7 @@ namespace {
 
 constexpr char kCastOp[] = "Cast";
 constexpr char kRealDivOp[] = "RealDiv";
+constexpr char kConstOp[] = "Const";
 
 constexpr std::array<const char*, 5> kBatchDatasetOps = {
     "BatchDataset",
@@ -137,27 +142,33 @@ Status MutateBatchSize(const NodeDef& node, int64 num_workers,
   // TODO(rohanj): Fix up the output_shapes attribute as well. For this Dataset
   // as well as all the downstream datasets.
   // For all the batching datasets the batch_size is input number 1.
-  // TODO(rohanj): Add an assertion for batch size not being a multiple of
-  // num_workers.
   NodeDef* batch_size_node = graph_utils::GetInputNode(node, *graph, 1);
-  NodeDef tmp_node = *batch_size_node;
-  graph_utils::SetUniqueGraphNodeName(tmp_node.op(), graph->graph(), &tmp_node);
-  NodeDef* copy_batch_size_node = graph->AddNode(std::move(tmp_node));
-  NodeDef* float_copy_batch_size_node =
-      AddCastNode(copy_batch_size_node->name(), DT_INT64, DT_FLOAT, graph);
-  NodeDef* num_worker_node =
-      graph_utils::AddScalarConstNode<int64>(num_workers, graph);
-  NodeDef* float_num_worker_node =
-      AddCastNode(num_worker_node->name(), DT_INT64, DT_FLOAT, graph);
-  NodeDef* divided_batch_size_node = AddFloatDivNode(
-      float_copy_batch_size_node->name(), float_num_worker_node->name(), graph);
-  NodeDef* cast_new_batch_size_node =
-      AddCastNode(divided_batch_size_node->name(), DT_FLOAT, DT_INT64, graph);
+  // By the time this optimization is run, the batch_size is computed and
+  // is a constant.
+  if (batch_size_node->op() != kConstOp) {
+    return errors::Internal("Batch size node should be a Const. Obtained: ",
+                            batch_size_node->op(), " instead.");
+  }
+  Tensor batch_size_tensor;
+  TF_RETURN_IF_ERROR(
+      GetNodeAttr(*batch_size_node, "value", &batch_size_tensor));
+  if (!TensorShapeUtils::IsScalar(batch_size_tensor.shape())) {
+    return errors::Internal("Batch size node shape should be scalar");
+  }
+  int64 batch_size = batch_size_tensor.scalar<int64>()();
+  if (batch_size % num_workers != 0) {
+    return errors::InvalidArgument(
+        "Batch size: ", batch_size,
+        " is not divisible by num_workers: ", num_workers);
+  }
+  batch_size /= num_workers;
+  NodeDef* new_batch_size_node =
+      graph_utils::AddScalarConstNode<int64>(batch_size, graph);
   // We don't call UpdateFanouts here because CSE elimination might lead to
   // multiple nodes sharing the same batch size constant node. This is also
   // why we don't delete batch_size_node as well.
   TF_RETURN_IF_ERROR(graph->UpdateRegularFaninByPort(
-      node.name(), 1, {cast_new_batch_size_node->name(), 0}));
+      node.name(), 1, {new_batch_size_node->name(), 0}));
   return Status::OK();
 }
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/rebatch_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/rebatch_dataset_test.py
index 9c1392bd01..28f9aad5b9 100644
--- a/tensorflow/python/data/experimental/kernel_tests/rebatch_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/rebatch_dataset_test.py
@@ -55,11 +55,13 @@ class RebatchDatasetTest(test_base.DatasetTestBase):
       batching._RebatchDataset(dataset, num_workers=4)
 
   def testNotDivisibleError(self, drop_remainder):
-    # TODO(rohanj): This should fail even with drop_remainder=False, by adding
-    # an assertion in the mutated graph.
-    dataset = dataset_ops.Dataset.range(1024).batch(32, drop_remainder=True)
-    with self.assertRaisesRegexp(ValueError, "not divisible by"):
-      batching._RebatchDataset(dataset, num_workers=5)
+    dataset = dataset_ops.Dataset.range(1024).batch(
+        32, drop_remainder=drop_remainder)
+    with self.assertRaisesRegexp(errors.InvalidArgumentError,
+                                 "not divisible by"):
+      rebatched_dataset = batching._RebatchDataset(dataset, num_workers=5)
+      next_element = self.getNext(rebatched_dataset)
+      self.evaluate(next_element())
 
   def testTupleOutput(self, drop_remainder):
     dataset = (
diff --git a/tensorflow/python/data/experimental/ops/batching.py b/tensorflow/python/data/experimental/ops/batching.py
index a55f576166..9c79af2464 100644
--- a/tensorflow/python/data/experimental/ops/batching.py
+++ b/tensorflow/python/data/experimental/ops/batching.py
@@ -27,6 +27,7 @@ from tensorflow.python.data.util import nest
 from tensorflow.python.data.util import structure
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import tensor_shape
@@ -719,7 +720,8 @@ class _RebatchDataset(dataset_ops.UnaryDataset):
         raise ValueError("Input shape should have at least one dimension.")
       if (output_shapes.dims[0].value and
           output_shapes.dims[0].value % num_workers != 0):
-        raise ValueError(
+        raise errors.InvalidArgumentError(
+            None, None,
             "First dim of input shape: %d is not divisible by num_workers: %d" %
             (output_shapes[0], num_workers))
       output_dims = [d for d in output_shapes.dims]
-- 
GitLab


From 81625991d7da7e2a0559b2a5a0857f84ef0bbb4a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 12 Feb 2019 08:31:40 -0800
Subject: [PATCH 2321/2345] Update ops-related pbtxt files.

PiperOrigin-RevId: 233611675
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 40 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 40 +++++++++++++++++++
 2 files changed, 80 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 90f0a782cd..030185c820 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -12387,6 +12387,46 @@ op {
     minimum: 1
   }
 }
+op {
+  name: "Case"
+  input_arg {
+    name: "branch_index"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "input"
+    type_list_attr: "Tin"
+  }
+  output_arg {
+    name: "output"
+    type_list_attr: "Tout"
+  }
+  attr {
+    name: "Tin"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "Tout"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "branches"
+    type: "list(func)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    default_value {
+      list {
+      }
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "Cast"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 791520f897..8e711ce6b7 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -5141,6 +5141,46 @@ op {
     minimum: 1
   }
 }
+op {
+  name: "Case"
+  input_arg {
+    name: "branch_index"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "input"
+    type_list_attr: "Tin"
+  }
+  output_arg {
+    name: "output"
+    type_list_attr: "Tout"
+  }
+  attr {
+    name: "Tin"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "Tout"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "branches"
+    type: "list(func)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    default_value {
+      list {
+      }
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "Cast"
   input_arg {
-- 
GitLab


From bba36f1f8f183f84b862d501574f3cce1bbf367e Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Tue, 12 Feb 2019 08:36:50 -0800
Subject: [PATCH 2322/2345] Add tf.gfile.GFile/Open rewrite rule

PiperOrigin-RevId: 233612446
---
 tensorflow/tools/compatibility/tf_upgrade_v2.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 4d87358829..bffedcdb3b 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -440,6 +440,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.io.gfile.exists",
         "tf.gfile.Glob":
             "tf.io.gfile.glob",
+        "tf.gfile.GFile":
+            "tf.io.gfile.GFile",
         "tf.gfile.IsDirectory":
             "tf.io.gfile.isdir",
         "tf.gfile.ListDirectory":
@@ -448,6 +450,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.io.gfile.makedirs",
         "tf.gfile.MkDir":
             "tf.io.gfile.mkdir",
+        "tf.gfile.Open":
+            "tf.io.gfile.GFile",
         "tf.gfile.Remove":
             "tf.io.gfile.remove",
         "tf.gfile.Rename":
-- 
GitLab


From 768844843859a721888fd325b2f9b361e09d58b1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 12 Feb 2019 09:02:27 -0800
Subject: [PATCH 2323/2345] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 233616123

---
 tensorflow/go/op/wrappers.go | 144 +++++++++++++++++------------------
 1 file changed, 72 insertions(+), 72 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 3cf3521264..7a05c5161d 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -18346,6 +18346,78 @@ func MaxPoolV2(scope *Scope, input tf.Output, ksize tf.Output, strides tf.Output
 	return op.Output(0)
 }
 
+// Does nothing. Serves as a control trigger for scheduling.
+//
+// Only useful as a placeholder for control edges.
+//
+// Returns the created operation.
+func ControlTrigger(scope *Scope) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ControlTrigger",
+	}
+	return scope.AddOperation(opspec)
+}
+
+// Deprecated. Use TensorArrayReadV3
+//
+// DEPRECATED at GraphDef version 26: Use TensorArrayReadV3
+func TensorArrayReadV2(scope *Scope, handle tf.Output, index tf.Output, flow_in tf.Output, dtype tf.DataType) (value tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtype": dtype}
+	opspec := tf.OpSpec{
+		Type: "TensorArrayReadV2",
+		Input: []tf.Input{
+			handle, index, flow_in,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Batch normalization.
+//
+// DEPRECATED at GraphDef version 9: Use tf.nn.batch_normalization()
+//
+// This op is deprecated. Prefer `tf.nn.batch_normalization`.
+//
+// Arguments:
+//	t: A 4D input Tensor.
+//	m: A 1D mean Tensor with size matching the last dimension of t.
+// This is the first output from tf.nn.moments,
+// or a saved moving average thereof.
+//	v: A 1D variance Tensor with size matching the last dimension of t.
+// This is the second output from tf.nn.moments,
+// or a saved moving average thereof.
+//	beta: A 1D beta Tensor with size matching the last dimension of t.
+// An offset to be added to the normalized tensor.
+//	gamma: A 1D gamma Tensor with size matching the last dimension of t.
+// If "scale_after_normalization" is true, this tensor will be multiplied
+// with the normalized tensor.
+//	variance_epsilon: A small float number to avoid dividing by 0.
+//	scale_after_normalization: A bool indicating whether the resulted tensor
+// needs to be multiplied with gamma.
+func BatchNormWithGlobalNormalization(scope *Scope, t tf.Output, m tf.Output, v tf.Output, beta tf.Output, gamma tf.Output, variance_epsilon float32, scale_after_normalization bool) (result tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"variance_epsilon": variance_epsilon, "scale_after_normalization": scale_after_normalization}
+	opspec := tf.OpSpec{
+		Type: "BatchNormWithGlobalNormalization",
+		Input: []tf.Input{
+			t, m, v, beta, gamma,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // AddManySparseToTensorsMapAttr is an optional argument to AddManySparseToTensorsMap.
 type AddManySparseToTensorsMapAttr func(optionalAttr)
 
@@ -27780,78 +27852,6 @@ func QuantizedMatMul(scope *Scope, a tf.Output, b tf.Output, min_a tf.Output, ma
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Does nothing. Serves as a control trigger for scheduling.
-//
-// Only useful as a placeholder for control edges.
-//
-// Returns the created operation.
-func ControlTrigger(scope *Scope) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ControlTrigger",
-	}
-	return scope.AddOperation(opspec)
-}
-
-// Batch normalization.
-//
-// DEPRECATED at GraphDef version 9: Use tf.nn.batch_normalization()
-//
-// This op is deprecated. Prefer `tf.nn.batch_normalization`.
-//
-// Arguments:
-//	t: A 4D input Tensor.
-//	m: A 1D mean Tensor with size matching the last dimension of t.
-// This is the first output from tf.nn.moments,
-// or a saved moving average thereof.
-//	v: A 1D variance Tensor with size matching the last dimension of t.
-// This is the second output from tf.nn.moments,
-// or a saved moving average thereof.
-//	beta: A 1D beta Tensor with size matching the last dimension of t.
-// An offset to be added to the normalized tensor.
-//	gamma: A 1D gamma Tensor with size matching the last dimension of t.
-// If "scale_after_normalization" is true, this tensor will be multiplied
-// with the normalized tensor.
-//	variance_epsilon: A small float number to avoid dividing by 0.
-//	scale_after_normalization: A bool indicating whether the resulted tensor
-// needs to be multiplied with gamma.
-func BatchNormWithGlobalNormalization(scope *Scope, t tf.Output, m tf.Output, v tf.Output, beta tf.Output, gamma tf.Output, variance_epsilon float32, scale_after_normalization bool) (result tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"variance_epsilon": variance_epsilon, "scale_after_normalization": scale_after_normalization}
-	opspec := tf.OpSpec{
-		Type: "BatchNormWithGlobalNormalization",
-		Input: []tf.Input{
-			t, m, v, beta, gamma,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Deprecated. Use TensorArrayReadV3
-//
-// DEPRECATED at GraphDef version 26: Use TensorArrayReadV3
-func TensorArrayReadV2(scope *Scope, handle tf.Output, index tf.Output, flow_in tf.Output, dtype tf.DataType) (value tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtype": dtype}
-	opspec := tf.OpSpec{
-		Type: "TensorArrayReadV2",
-		Input: []tf.Input{
-			handle, index, flow_in,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // QuantizedMulAttr is an optional argument to QuantizedMul.
 type QuantizedMulAttr func(optionalAttr)
 
-- 
GitLab


From 373e2630a010b2b8f5e77b925ada2ffbd389d9c6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 12 Feb 2019 09:18:11 -0800
Subject: [PATCH 2324/2345] Add tf.linalg.tridiagonal for solving tridiagonal
 systems of linear equations (CPU implementation only).

PiperOrigin-RevId: 233618144
---
 .../base_api/api_def_TridiagonalSolve.pbtxt   |  33 ++
 tensorflow/core/kernels/BUILD                 |   7 +
 .../core/kernels/tridiagonal_solve_op.cc      | 163 +++++++
 tensorflow/core/ops/linalg_ops.cc             |  37 ++
 tensorflow/core/ops/linalg_ops_test.cc        |  36 ++
 tensorflow/python/kernel_tests/BUILD          |  13 +
 .../kernel_tests/tridiagonal_solve_op_test.py | 456 ++++++++++++++++++
 tensorflow/python/ops/linalg/linalg_impl.py   | 187 +++++++
 .../api/golden/v1/tensorflow.linalg.pbtxt     |   4 +
 .../api/golden/v1/tensorflow.raw_ops.pbtxt    |   4 +
 .../api/golden/v2/tensorflow.linalg.pbtxt     |   4 +
 .../api/golden/v2/tensorflow.raw_ops.pbtxt    |   4 +
 12 files changed, 948 insertions(+)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_TridiagonalSolve.pbtxt
 create mode 100644 tensorflow/core/kernels/tridiagonal_solve_op.cc
 create mode 100644 tensorflow/python/kernel_tests/tridiagonal_solve_op_test.py

diff --git a/tensorflow/core/api_def/base_api/api_def_TridiagonalSolve.pbtxt b/tensorflow/core/api_def/base_api/api_def_TridiagonalSolve.pbtxt
new file mode 100644
index 0000000000..80f3675d55
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_TridiagonalSolve.pbtxt
@@ -0,0 +1,33 @@
+op {
+  graph_op_name: "TridiagonalSolve"
+  visibility: HIDDEN
+  in_arg {
+    name: "diagonals"
+    description: <<END
+Shape is `[..., 3, M]`.
+END
+  }
+  in_arg {
+    name: "rhs"
+    description: <<END
+Shape is `[..., M, K]`.
+END
+  }
+  out_arg {
+    name: "output"
+    description: <<END
+Shape is `[..., M, K]`.
+END
+  }
+
+  summary: "Solves tridiagonal systems of equations."
+  description: <<END
+`diagonals` is a tensor of shape `[..., 3, M]` whose inner-most 2 dimensions
+represent matrices with three rows being the superdiagonal, diagonals, and
+subdiagonals, in order. The last element of the superdiagonal and the first
+element of the subdiagonal is ignored.
+`rhs` is a tensor of shape `[..., M, K]`, representing K right-hand sides per
+each left-hand side.
+The output is a tensor of shape `[..., M, K]` containing the solutions.
+END
+}
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 0ad8ca4de4..85e6410579 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -2937,6 +2937,7 @@ cc_library(
         ":self_adjoint_eig_op",
         ":self_adjoint_eig_v2_op",
         ":svd_op",
+        ":tridiagonal_solve_op",
     ],
 )
 
@@ -3055,6 +3056,12 @@ tf_kernel_library(
     ]),
 )
 
+tf_kernel_library(
+    name = "tridiagonal_solve_op",
+    srcs = ["tridiagonal_solve_op.cc"],
+    deps = LINALG_DEPS,
+)
+
 tf_kernel_library(
     name = "qr_op",
     prefix = "qr_op",
diff --git a/tensorflow/core/kernels/tridiagonal_solve_op.cc b/tensorflow/core/kernels/tridiagonal_solve_op.cc
new file mode 100644
index 0000000000..5884ffedfb
--- /dev/null
+++ b/tensorflow/core/kernels/tridiagonal_solve_op.cc
@@ -0,0 +1,163 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// See docs in ../ops/linalg_ops.cc.
+
+#include "tensorflow/core/framework/kernel_def_builder.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/register_types.h"
+#include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/kernels/linalg_ops_common.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace tensorflow {
+
+static const char kErrMsg[] = "The matrix is not invertible.";
+
+template <class Scalar>
+class TridiagonalSolveOp : public LinearAlgebraOp<Scalar> {
+ public:
+  INHERIT_LINALG_TYPEDEFS(Scalar);
+
+  explicit TridiagonalSolveOp(OpKernelConstruction* context) : Base(context) {}
+
+  void ValidateInputMatrixShapes(
+      OpKernelContext* context,
+      const TensorShapes& input_matrix_shapes) const final {
+    auto num_inputs = input_matrix_shapes.size();
+    OP_REQUIRES(context, num_inputs == 2,
+                errors::InvalidArgument("Expected two input matrices, got ",
+                                        num_inputs, "."));
+
+    auto num_diags = input_matrix_shapes[0].dim_size(0);
+    OP_REQUIRES(
+        context, num_diags == 3,
+        errors::InvalidArgument("Expected diagonals to be provided as a "
+                                "matrix with 3 rows, got ",
+                                num_diags, " rows."));
+
+    auto num_eqs_left = input_matrix_shapes[0].dim_size(1);
+    auto num_eqs_right = input_matrix_shapes[1].dim_size(0);
+    OP_REQUIRES(
+        context, num_eqs_left == num_eqs_right,
+        errors::InvalidArgument("Expected the same number of left-hand sides "
+                                "and right-hand sides, got ",
+                                num_eqs_left, " and ", num_eqs_right, "."));
+  }
+
+  TensorShapes GetOutputMatrixShapes(
+      const TensorShapes& input_matrix_shapes) const final {
+    return TensorShapes({input_matrix_shapes[1]});
+  }
+
+  int64 GetCostPerUnit(const TensorShapes& input_matrix_shapes) const final {
+    const int num_eqs = static_cast<int>(input_matrix_shapes[0].dim_size(1));
+    const int num_rhss = static_cast<int>(input_matrix_shapes[1].dim_size(0));
+
+    const double add_cost = Eigen::TensorOpCost::AddCost<Scalar>();
+    const double mult_cost = Eigen::TensorOpCost::MulCost<Scalar>();
+    const double div_cost = Eigen::TensorOpCost::DivCost<Scalar>();
+
+    // Assuming cases with and without row interchange are equiprobable.
+    const double cost =
+        num_eqs * (div_cost * (num_rhss + 1) +
+                   (add_cost + mult_cost) * (2.5 * num_rhss + 1.5));
+    return cost >= static_cast<double>(kint64max) ? kint64max
+                                                  : static_cast<int64>(cost);
+  }
+
+  void ComputeMatrix(OpKernelContext* context, const ConstMatrixMaps& inputs,
+                     MatrixMaps* outputs) final {
+    const auto diagonals = inputs[0];
+
+    // Subdiagonal elements, first is ignored.
+    const auto& superdiag = diagonals.row(0);
+    // Diagonal elements.
+    const auto& diag = diagonals.row(1);
+    // Superdiagonal elements, n-th is ignored.
+    const auto& subdiag = diagonals.row(2);
+    // Right-hand sides (transposed - necessary for GPU impl).
+    const auto& rhs = inputs[1];
+
+    const int n = diag.size();
+    MatrixMap& x = outputs->at(0);
+    const Scalar zero(0);
+
+    if (n == 0) {
+      return;
+    }
+    if (n == 1) {
+      OP_REQUIRES(context, diag(0) != zero, errors::InvalidArgument(kErrMsg));
+      x.row(0) = rhs.row(0) / diag(0);
+      return;
+    }
+
+    // The three columns in u are the diagonal, superdiagonal, and second
+    // superdiagonal, respectively, of the U matrix in the LU decomposition of
+    // the input matrix (subject to row exchanges due to pivoting). For pivoted
+    // tridiagonal matrix, the U matrix has at most two non-zero superdiagonals.
+    Eigen::Array<Scalar, Eigen::Dynamic, 3> u(n, 3);
+
+    // The code below roughly follows LAPACK's dgtsv routine, with main
+    // difference being not overwriting the input.
+    u(0, 0) = diag(0);
+    u(0, 1) = superdiag(0);
+    x.row(0) = rhs.row(0);
+    for (int i = 0; i < n - 1; ++i) {
+      if (std::abs(u(i)) >= std::abs(subdiag(i + 1))) {
+        // No row interchange.
+        OP_REQUIRES(context, u(i) != zero, errors::InvalidArgument(kErrMsg));
+        const Scalar factor = subdiag(i + 1) / u(i, 0);
+        u(i + 1, 0) = diag(i + 1) - factor * u(i, 1);
+        x.row(i + 1) = rhs.row(i + 1) - factor * x.row(i);
+        if (i != n - 2) {
+          u(i + 1, 1) = superdiag(i + 1);
+          u(i, 2) = 0;
+        }
+      } else {
+        // Interchange rows i and i + 1.
+        const Scalar factor = u(i, 0) / subdiag(i + 1);
+        u(i, 0) = subdiag(i + 1);
+        u(i + 1, 0) = u(i, 1) - factor * diag(i + 1);
+        u(i, 1) = diag(i + 1);
+        x.row(i + 1) = x.row(i) - factor * rhs.row(i + 1);
+        x.row(i) = rhs.row(i + 1);
+        if (i != n - 2) {
+          u(i, 2) = superdiag(i + 1);
+          u(i + 1, 1) = -factor * superdiag(i + 1);
+        }
+      }
+    }
+    x.row(n - 1) /= u(n - 1, 0);
+    x.row(n - 2) = (x.row(n - 2) - u(n - 2, 1) * x.row(n - 1)) / u(n - 2, 0);
+    for (int i = n - 3; i >= 0; --i) {
+      x.row(i) = (x.row(i) - u(i, 1) * x.row(i + 1) - u(i, 2) * x.row(i + 2)) /
+                 u(i, 0);
+    }
+  }
+
+ private:
+  TF_DISALLOW_COPY_AND_ASSIGN(TridiagonalSolveOp);
+};
+
+REGISTER_LINALG_OP_CPU("TridiagonalSolve", (TridiagonalSolveOp<float>), float);
+REGISTER_LINALG_OP_CPU("TridiagonalSolve", (TridiagonalSolveOp<double>),
+                       double);
+REGISTER_LINALG_OP_CPU("TridiagonalSolve", (TridiagonalSolveOp<complex64>),
+                       complex64);
+REGISTER_LINALG_OP_CPU("TridiagonalSolve", (TridiagonalSolveOp<complex128>),
+                       complex128);
+}  // namespace tensorflow
diff --git a/tensorflow/core/ops/linalg_ops.cc b/tensorflow/core/ops/linalg_ops.cc
index 8a8419588a..66594b3576 100644
--- a/tensorflow/core/ops/linalg_ops.cc
+++ b/tensorflow/core/ops/linalg_ops.cc
@@ -208,6 +208,36 @@ Status SvdShapeFn(InferenceContext* c) {
   return Status::OK();
 }
 
+// The first input is [...,3,M] and second input is [...,M,K].
+// Output is [...,M,K].
+Status TridiagonalSolveShapeFn(InferenceContext* c) {
+  ShapeHandle lhs;
+  ShapeHandle rhs;
+  // Check that rank is at least 2.
+  TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 2, &lhs));
+  TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(1), 2, &rhs));
+
+  // Extract batch dimensions and check they are the same.
+  ShapeHandle lhs_batch_shape;
+  ShapeHandle rhs_batch_shape;
+  TF_RETURN_IF_ERROR(c->Subshape(lhs, 0, -2, &lhs_batch_shape));
+  TF_RETURN_IF_ERROR(c->Subshape(rhs, 0, -2, &rhs_batch_shape));
+  TF_RETURN_IF_ERROR(
+      c->Merge(lhs_batch_shape, rhs_batch_shape, &lhs_batch_shape));
+
+  // Check that "M" is the same in both inputs.
+  DimensionHandle m_lhs = c->Dim(lhs, -1);
+  DimensionHandle m_rhs = c->Dim(rhs, -2);
+  TF_RETURN_IF_ERROR(c->Merge(m_lhs, m_rhs, &m_lhs));
+
+  // Check that next-to-last dimension of the first input is 3.
+  TF_RETURN_IF_ERROR(c->WithValue(c->Dim(lhs, -2), 3, &m_lhs));
+
+  // The output shape is the same as rhs shape.
+  c->set_output(0, rhs);
+  return Status::OK();
+}
+
 }  // namespace
 
 REGISTER_OP("MatrixDeterminant")
@@ -379,6 +409,13 @@ REGISTER_OP("Svd")
     .Attr("T: {double, float, half, complex64, complex128}")
     .SetShapeFn(SvdShapeFn);
 
+REGISTER_OP("TridiagonalSolve")
+    .Input("diagonals: T")
+    .Input("rhs: T")
+    .Output("output: T")
+    .Attr("T: {double, float, complex64, complex128}")
+    .SetShapeFn(TridiagonalSolveShapeFn);
+
 // Deprecated op registrations:
 
 // Can be deleted after 3feb2017.
diff --git a/tensorflow/core/ops/linalg_ops_test.cc b/tensorflow/core/ops/linalg_ops_test.cc
index 7d1aed50be..93732f938a 100644
--- a/tensorflow/core/ops/linalg_ops_test.cc
+++ b/tensorflow/core/ops/linalg_ops_test.cc
@@ -314,4 +314,40 @@ TEST(LinalgOpsTest, Lu_ShapeFn) {
            "[d0_0,d0_1,d0_2,d0_3,d0_5,d0_5];[d0_0,d0_1,d0_2,d0_3,d0_5]");
 }
 
+TEST(LinalgOpsTest, TridiagonalSolve_ShapeFn) {
+  ShapeInferenceTestOp op("TridiagonalSolve");
+  INFER_OK(op, "?;?", "in1");
+  INFER_OK(op, "[3,5];[?,1]", "in1");
+  INFER_OK(op, "[?,5];[5,1]", "in1");
+  INFER_OK(op, "[?,5];[?,?]", "in1");
+  INFER_OK(op, "[?,?];[?,?]", "in1");
+  INFER_OK(op, "[3,5];[5,1]", "in1");
+  INFER_OK(op, "[3,5];[5,2]", "in1");
+
+  INFER_OK(op, "[?,?,?];[?,?,?]", "in1");
+  INFER_OK(op, "[?,3,5];[7,5,2]", "in1");
+  INFER_OK(op, "[7,3,5];[?,5,2]", "in1");
+  INFER_OK(op, "[7,?,5];[?,5,?]", "in1");
+  INFER_OK(op, "[7,3,5];[7,5,2]", "in1");
+
+  INFER_OK(op, "[7,?,3,5];[7,8,5,2]", "in1");
+  INFER_OK(op, "[7,8,3,5];[7,8,5,2]", "in1");
+
+  INFER_ERROR("Shape must be at least rank 2 but is rank 1", op, "[3];[5,1]");
+  INFER_ERROR("Shape must be at least rank 2 but is rank 1", op, "[3,5];[5]");
+  INFER_ERROR(
+      "Dimension 1 in both shapes must be equal, but are 4 and 8. "
+      "Shapes are [6,4] and [6,8].",
+      op, "[6,4,3,5];[6,8,5,2]");
+  INFER_ERROR(
+      "Dimension 1 in both shapes must be equal, but are 4 and 8. "
+      "Shapes are [?,4] and [6,8].",
+      op, "[?,4,3,5];[6,8,5,2]");
+  INFER_ERROR("Dimension must be 3 but is 4", op, "[4,5];[5,2]");
+  INFER_ERROR("Dimension must be 3 but is 4", op, "[6,4,5];[6,5,2]");
+  INFER_ERROR("Dimensions must be equal, but are 9 and 5", op, "[3,9];[5,2]");
+  INFER_ERROR("Dimensions must be equal, but are 9 and 5", op,
+              "[6,3,9];[6,5,2]");
+}
+
 }  // end namespace tensorflow
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index a5bb03ff6a..75a8fa2a68 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -1149,6 +1149,19 @@ tf_py_test(
     ],
 )
 
+tf_py_test(
+    name = "tridiagonal_solve_op_test",
+    size = "medium",
+    srcs = ["tridiagonal_solve_op_test.py"],
+    additional_deps = [
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:linalg_ops",
+    ],
+    shard_count = 5,
+)
+
 tf_py_test(
     name = "unicode_script_op_test",
     size = "small",
diff --git a/tensorflow/python/kernel_tests/tridiagonal_solve_op_test.py b/tensorflow/python/kernel_tests/tridiagonal_solve_op_test.py
new file mode 100644
index 0000000000..3fd603a954
--- /dev/null
+++ b/tensorflow/python/kernel_tests/tridiagonal_solve_op_test.py
@@ -0,0 +1,456 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for tensorflow.ops.linalg.linalg_impl.tridiagonal_solve."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.client import session
+from tensorflow.python.eager import context
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors_impl
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import random_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.ops.linalg import linalg_impl
+from tensorflow.python.platform import benchmark
+from tensorflow.python.platform import test
+
+_sample_diags = np.array([[2, 1, 4, 0], [1, 3, 2, 2], [0, 1, -1, 1]])
+_sample_rhs = np.array([1, 2, 3, 4])
+_sample_result = np.array([-9, 5, -4, 4])
+
+
+def _tfconst(array):
+  return constant_op.constant(array, dtypes.float64)
+
+
+def _tf_ones(shape):
+  return array_ops.ones(shape, dtype=dtypes.float64)
+
+
+class TridiagonalSolveOpTest(test.TestCase):
+
+  def _test(self,
+            diags,
+            rhs,
+            expected,
+            diags_format="compact",
+            transpose_rhs=False,
+            conjugate_rhs=False):
+    with self.cached_session(use_gpu=False):
+      result = linalg_impl.tridiagonal_solve(diags, rhs, diags_format,
+                                             transpose_rhs, conjugate_rhs)
+      self.assertAllClose(self.evaluate(result), expected)
+
+  def _testWithLists(self,
+                     diags,
+                     rhs,
+                     expected,
+                     diags_format="compact",
+                     transpose_rhs=False,
+                     conjugate_rhs=False):
+    self._test(
+        _tfconst(diags), _tfconst(rhs), _tfconst(expected), diags_format,
+        transpose_rhs, conjugate_rhs)
+
+  def _assertRaises(self, diags, rhs, diags_format="compact"):
+    with self.assertRaises(ValueError):
+      linalg_impl.tridiagonal_solve(diags, rhs, diags_format)
+
+  # Tests with various dtypes
+
+  def testReal(self):
+    for dtype in dtypes.float32, dtypes.float64:
+      self._test(
+          diags=constant_op.constant(_sample_diags, dtype),
+          rhs=constant_op.constant(_sample_rhs, dtype),
+          expected=constant_op.constant(_sample_result, dtype))
+
+  def testComplex(self):
+    for dtype in dtypes.complex64, dtypes.complex128:
+      self._test(
+          diags=constant_op.constant(_sample_diags, dtype) * (1 + 1j),
+          rhs=constant_op.constant(_sample_rhs, dtype) * (1 - 1j),
+          expected=constant_op.constant(_sample_result, dtype) * (1 - 1j) /
+          (1 + 1j))
+
+  # Tests with small matrix sizes
+
+  def test3x3(self):
+    self._testWithLists(
+        diags=[[2, -1, 0], [1, 3, 1], [0, -1, -2]],
+        rhs=[1, 2, 3],
+        expected=[-3, 2, 7])
+
+  def test2x2(self):
+    self._testWithLists(
+        diags=[[2, 0], [1, 3], [0, 1]], rhs=[1, 4], expected=[-5, 3])
+
+  def test1x1(self):
+    self._testWithLists(diags=[[0], [3], [0]], rhs=[6], expected=[2])
+
+  def test0x0(self):
+    self._test(
+        diags=constant_op.constant(0, shape=(3, 0), dtype=dtypes.float32),
+        rhs=constant_op.constant(0, shape=(0, 1), dtype=dtypes.float32),
+        expected=constant_op.constant(0, shape=(0, 1), dtype=dtypes.float32))
+
+  # Other edge cases
+
+  def testCaseRequiringPivoting(self):
+    # Without partial pivoting (e.g. Thomas algorithm) this would fail.
+    self._testWithLists(
+        diags=[[2, -1, 1, 0], [1, 4, 1, -1], [0, 2, -2, 3]],
+        rhs=[1, 2, 3, 4],
+        expected=[8, -3.5, 0, -4])
+
+  def testCaseRequiringPivotingLastRows(self):
+    self._testWithLists(
+        diags=[[2, 1, -1, 0], [1, -1, 2, 1], [0, 1, -6, 1]],
+        rhs=[1, 2, -1, -2],
+        expected=[5, -2, -5, 3])
+
+  def testNotInvertible(self):
+    with self.assertRaises(errors_impl.InvalidArgumentError):
+      self._testWithLists(
+          diags=[[2, -1, 1, 0], [1, 4, 1, -1], [0, 2, 0, 3]],
+          rhs=[1, 2, 3, 4],
+          expected=[8, -3.5, 0, -4])
+
+  def testDiagonal(self):
+    self._testWithLists(
+        diags=[[0, 0, 0, 0], [1, 2, -1, -2], [0, 0, 0, 0]],
+        rhs=[1, 2, 3, 4],
+        expected=[1, 1, -3, -2])
+
+  def testUpperTriangular(self):
+    self._testWithLists(
+        diags=[[2, 4, -1, 0], [1, 3, 1, 2], [0, 0, 0, 0]],
+        rhs=[1, 6, 4, 4],
+        expected=[13, -6, 6, 2])
+
+  def testLowerTriangular(self):
+    self._testWithLists(
+        diags=[[0, 0, 0, 0], [2, -1, 3, 1], [0, 1, 4, 2]],
+        rhs=[4, 5, 6, 1],
+        expected=[2, -3, 6, -11])
+
+  # Multiple right-hand sides and batching
+
+  def testWithTwoRightHandSides(self):
+    self._testWithLists(
+        diags=_sample_diags,
+        rhs=np.transpose([_sample_rhs, 2 * _sample_rhs]),
+        expected=np.transpose([_sample_result, 2 * _sample_result]))
+
+  def testBatching(self):
+    self._testWithLists(
+        diags=np.array([_sample_diags, -_sample_diags]),
+        rhs=np.array([_sample_rhs, 2 * _sample_rhs]),
+        expected=np.array([_sample_result, -2 * _sample_result]))
+
+  def testBatchingAndTwoRightHandSides(self):
+    rhs = np.transpose([_sample_rhs, 2 * _sample_rhs])
+    expected_result = np.transpose([_sample_result, 2 * _sample_result])
+    self._testWithLists(
+        diags=np.array([_sample_diags, -_sample_diags]),
+        rhs=np.array([rhs, 2 * rhs]),
+        expected=np.array([expected_result, -2 * expected_result]))
+
+  # Various input formats
+
+  def testSequenceFormat(self):
+    self._test(
+        diags=(_tfconst([2, 1, 4]), _tfconst([1, 3, 2, 2]), _tfconst([1, -1,
+                                                                      1])),
+        rhs=_tfconst([1, 2, 3, 4]),
+        expected=_tfconst([-9, 5, -4, 4]),
+        diags_format="sequence")
+
+  def testSequenceFormatWithDummyElements(self):
+    dummy = 20
+    self._test(
+        diags=(_tfconst([2, 1, 4, dummy]), _tfconst([1, 3, 2, 2]),
+               _tfconst([dummy, 1, -1, 1])),
+        rhs=_tfconst([1, 2, 3, 4]),
+        expected=_tfconst([-9, 5, -4, 4]),
+        diags_format="sequence")
+
+  def testSequenceFormatWithBatching(self):
+    self._test(
+        diags=(_tfconst([[2, 1, 4], [-2, -1, -4]]),
+               _tfconst([[1, 3, 2, 2], [-1, -3, -2, -2]]),
+               _tfconst([[1, -1, 1], [-1, 1, -1]])),
+        rhs=_tfconst([[1, 2, 3, 4], [1, 2, 3, 4]]),
+        expected=_tfconst([[-9, 5, -4, 4], [9, -5, 4, -4]]),
+        diags_format="sequence")
+
+  def testMatrixFormat(self):
+    self._testWithLists(
+        diags=[[1, 2, 0, 0], [1, 3, 1, 0], [0, -1, 2, 4], [0, 0, 1, 2]],
+        rhs=[1, 2, 3, 4],
+        expected=[-9, 5, -4, 4],
+        diags_format="matrix")
+
+  def testMatrixFormatWithMultipleRightHandSides(self):
+    self._testWithLists(
+        diags=[[1, 2, 0, 0], [1, 3, 1, 0], [0, -1, 2, 4], [0, 0, 1, 2]],
+        rhs=[[1, -1], [2, -2], [3, -3], [4, -4]],
+        expected=[[-9, 9], [5, -5], [-4, 4], [4, -4]],
+        diags_format="matrix")
+
+  def testMatrixFormatWithBatching(self):
+    self._testWithLists(
+        diags=[[[1, 2, 0, 0], [1, 3, 1, 0], [0, -1, 2, 4], [0, 0, 1, 2]],
+               [[-1, -2, 0, 0], [-1, -3, -1, 0], [0, 1, -2, -4], [0, 0, -1,
+                                                                  -2]]],
+        rhs=[[1, 2, 3, 4], [1, 2, 3, 4]],
+        expected=[[-9, 5, -4, 4], [9, -5, 4, -4]],
+        diags_format="matrix")
+
+  def testRightHandSideAsColumn(self):
+    self._testWithLists(
+        diags=_sample_diags,
+        rhs=np.transpose([_sample_rhs]),
+        expected=np.transpose([_sample_result]),
+        diags_format="compact")
+
+  # Tests with transpose and adjoint
+
+  def testTransposeRhs(self):
+    self._testWithLists(
+        diags=_sample_diags,
+        rhs=np.array([_sample_rhs, 2 * _sample_rhs]),
+        expected=np.array([_sample_result, 2 * _sample_result]),
+        transpose_rhs=True)
+
+  def testConjugateRhs(self):
+    self._testWithLists(
+        diags=_sample_diags,
+        rhs=np.transpose([_sample_rhs * (1 + 1j), _sample_rhs * (1 - 2j)]),
+        expected=np.transpose(
+            [_sample_result * (1 - 1j), _sample_result * (1 + 2j)]),
+        conjugate_rhs=True)
+
+  def testAdjointRhs(self):
+    self._testWithLists(
+        diags=_sample_diags,
+        rhs=np.array([_sample_rhs * (1 + 1j), _sample_rhs * (1 - 2j)]),
+        expected=np.array(
+            [_sample_result * (1 - 1j), _sample_result * (1 + 2j)]),
+        transpose_rhs=True,
+        conjugate_rhs=True)
+
+  def testTransposeRhsWithBatching(self):
+    self._testWithLists(
+        diags=np.array([_sample_diags, -_sample_diags]),
+        rhs=np.array([[_sample_rhs, 2 * _sample_rhs],
+                      [3 * _sample_rhs, 4 * _sample_rhs]]),
+        expected=np.array([[_sample_result, 2 * _sample_result],
+                           [-3 * _sample_result, -4 * _sample_result]]),
+        transpose_rhs=True)
+
+  def testTransposeRhsWithRhsAsVector(self):
+    self._testWithLists(
+        diags=_sample_diags,
+        rhs=_sample_rhs,
+        expected=_sample_result,
+        transpose_rhs=True)
+
+  def testConjugateRhsWithRhsAsVector(self):
+    self._testWithLists(
+        diags=_sample_diags,
+        rhs=_sample_rhs * (1 + 1j),
+        expected=_sample_result * (1 - 1j),
+        conjugate_rhs=True)
+
+  def testTransposeRhsWithRhsAsVectorAndBatching(self):
+    self._testWithLists(
+        diags=np.array([_sample_diags, -_sample_diags]),
+        rhs=np.array([_sample_rhs, 2 * _sample_rhs]),
+        expected=np.array([_sample_result, -2 * _sample_result]),
+        transpose_rhs=True)
+
+  # Invalid input shapes
+
+  def testInvalidShapesCompactFormat(self):
+
+    def test_raises(diags_shape, rhs_shape):
+      self._assertRaises(_tf_ones(diags_shape), _tf_ones(rhs_shape), "compact")
+
+    test_raises((5, 4, 4), (5, 4))
+    test_raises((5, 3, 4), (4, 5))
+    test_raises((5, 3, 4), (5))
+    test_raises((5), (5, 4))
+
+  def testInvalidShapesSequenceFormat(self):
+
+    def test_raises(diags_tuple_shapes, rhs_shape):
+      diagonals = tuple(_tf_ones(shape) for shape in diags_tuple_shapes)
+      self._assertRaises(diagonals, _tf_ones(rhs_shape), "sequence")
+
+    test_raises(((5, 4), (5, 4)), (5, 4))
+    test_raises(((5, 4), (5, 4), (5, 6)), (5, 4))
+    test_raises(((5, 3), (5, 4), (5, 6)), (5, 4))
+    test_raises(((5, 6), (5, 4), (5, 3)), (5, 4))
+    test_raises(((5, 4), (7, 4), (5, 4)), (5, 4))
+    test_raises(((5, 4), (7, 4), (5, 4)), (3, 4))
+
+  def testInvalidShapesMatrixFormat(self):
+
+    def test_raises(diags_shape, rhs_shape):
+      self._assertRaises(_tf_ones(diags_shape), _tf_ones(rhs_shape), "matrix")
+
+    test_raises((5, 4, 7), (5, 4))
+    test_raises((5, 4, 4), (3, 4))
+    test_raises((5, 4, 4), (5, 3))
+
+  # Tests with placeholders
+
+  def _testWithPlaceholders(self,
+                            diags_shape,
+                            rhs_shape,
+                            diags_feed,
+                            rhs_feed,
+                            expected,
+                            diags_format="compact"):
+    if context.executing_eagerly():
+      return
+    diags = array_ops.placeholder(dtypes.float64, shape=diags_shape)
+    rhs = array_ops.placeholder(dtypes.float64, shape=rhs_shape)
+    x = linalg_impl.tridiagonal_solve(diags, rhs, diags_format)
+    with self.cached_session(use_gpu=False) as sess:
+      result = sess.run(x, feed_dict={diags: diags_feed, rhs: rhs_feed})
+      self.assertAllClose(result, expected)
+
+  def testCompactFormatAllDimsUnknown(self):
+    self._testWithPlaceholders(
+        diags_shape=[None, None],
+        rhs_shape=[None],
+        diags_feed=_sample_diags,
+        rhs_feed=_sample_rhs,
+        expected=_sample_result)
+
+  def testCompactFormatUnknownMatrixSize(self):
+    self._testWithPlaceholders(
+        diags_shape=[3, None],
+        rhs_shape=[4],
+        diags_feed=_sample_diags,
+        rhs_feed=_sample_rhs,
+        expected=_sample_result)
+
+  def testCompactFormatUnknownRhsCount(self):
+    self._testWithPlaceholders(
+        diags_shape=[3, 4],
+        rhs_shape=[4, None],
+        diags_feed=_sample_diags,
+        rhs_feed=np.transpose([_sample_rhs, 2 * _sample_rhs]),
+        expected=np.transpose([_sample_result, 2 * _sample_result]))
+
+  def testCompactFormatUnknownBatchSize(self):
+    self._testWithPlaceholders(
+        diags_shape=[None, 3, 4],
+        rhs_shape=[None, 4],
+        diags_feed=np.array([_sample_diags, -_sample_diags]),
+        rhs_feed=np.array([_sample_rhs, 2 * _sample_rhs]),
+        expected=np.array([_sample_result, -2 * _sample_result]))
+
+  def testMatrixFormatWithUnknownDims(self):
+    if context.executing_eagerly():
+      return
+
+    def test_with_matrix_shapes(matrix_shape):
+      matrix = np.array([[1, 2, 0, 0], [1, 3, 1, 0], [0, -1, 2, 4],
+                         [0, 0, 1, 2]])
+      rhs = np.array([1, 2, 3, 4])
+      x = np.array([-9, 5, -4, 4])
+      self._testWithPlaceholders(
+          diags_shape=matrix_shape,
+          rhs_shape=[None, None],
+          diags_feed=matrix,
+          rhs_feed=np.transpose([rhs, 2 * rhs]),
+          expected=np.transpose([x, 2 * x]),
+          diags_format="matrix")
+
+    test_with_matrix_shapes(matrix_shape=[4, 4])
+    test_with_matrix_shapes(matrix_shape=[None, 4])
+    test_with_matrix_shapes(matrix_shape=[4, None])
+    with self.assertRaises(ValueError):
+      test_with_matrix_shapes(matrix_shape=[None, None])
+
+  def testSequenceFormatWithUnknownDims(self):
+    if context.executing_eagerly():
+      return
+    superdiag = array_ops.placeholder(dtypes.float64, shape=[None])
+    diag = array_ops.placeholder(dtypes.float64, shape=[None])
+    subdiag = array_ops.placeholder(dtypes.float64, shape=[None])
+    rhs = array_ops.placeholder(dtypes.float64, shape=[None])
+
+    x = linalg_impl.tridiagonal_solve((superdiag, diag, subdiag),
+                                      rhs,
+                                      diagonals_format="sequence")
+    with self.cached_session(use_gpu=False) as sess:
+      result = sess.run(
+          x,
+          feed_dict={
+              subdiag: [20, 1, -1, 1],
+              diag: [1, 3, 2, 2],
+              superdiag: [2, 1, 4, 20],
+              rhs: [1, 2, 3, 4]
+          })
+      self.assertAllClose(result, [-9, 5, -4, 4])
+
+  # Benchmark
+
+  class TridiagonalSolveBenchmark(test.Benchmark):
+    sizes = [(100000, 1, 1), (1000000, 1, 1), (10000000, 1, 1), (100000, 10, 1),
+             (100000, 100, 1), (10000, 1, 100), (10000, 1, 1000),
+             (10000, 1, 10000)]
+
+    def _generateData(self, matrix_size, batch_size, num_rhs, seed=42):
+      data = random_ops.random_normal(
+          shape=(batch_size, 3 + num_rhs, matrix_size),
+          dtype=dtypes.float64,
+          seed=seed)
+      diags = array_ops.stack([data[:, 0], data[:, 1], data[:, 2]], axis=-2)
+      rhs = data[:, 3:, :]
+      return diags, rhs
+
+    def benchmarkTridiagonalSolveOp(self):
+      for matrix_size, batch_size, num_rhs in self.sizes:
+        with ops.Graph().as_default(), \
+                session.Session(config=benchmark.benchmark_config()) as sess, \
+                ops.device("/cpu:0"):
+          diags, rhs = self._generateData(matrix_size, batch_size, num_rhs)
+          x = linalg_impl.tridiagonal_solve(diags, rhs, transpose_rhs=True)
+          variables.global_variables_initializer().run()
+          self.run_op_benchmark(
+              sess,
+              control_flow_ops.group(x),
+              min_iters=10,
+              store_memory_usage=False,
+              name=("tridiagonal_solve_matrix_size_{}_batch_size_{}_"
+                    "num_rhs_{}").format(matrix_size, batch_size, num_rhs))
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/ops/linalg/linalg_impl.py b/tensorflow/python/ops/linalg/linalg_impl.py
index 2259eaa65c..fec2b2713e 100644
--- a/tensorflow/python/ops/linalg/linalg_impl.py
+++ b/tensorflow/python/ops/linalg/linalg_impl.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gen_linalg_ops
@@ -329,3 +330,189 @@ def matrix_exponential(input, name=None):  # pylint: disable=redefined-builtin
           result,
           array_ops.concat((batch_shape, array_ops.shape(result)[-2:]), axis=0))
     return array_ops.reshape(result, batch_shape.concatenate(result.shape[-2:]))
+
+
+@tf_export('linalg.tridiagonal_solve')
+def tridiagonal_solve(diagonals,
+                      rhs,
+                      diagonals_format='compact',
+                      transpose_rhs=False,
+                      conjugate_rhs=False,
+                      name=None):
+  r"""Solves tridiagonal systems of equations.
+
+  Solution is computed via Gaussian elemination with partial pivoting.
+
+  The input can be supplied in various formats: `matrix`, `tuple` and `compact`,
+  specified by the `diagonals_format` arg.
+
+  In `matrix` format, `diagonals` must be a tensor of shape `[..., M, M]`, with
+  two inner-most dimensions representing the square tridiagonal matrices.
+  Elements outside of the three diagonals will be ignored.
+
+  In `sequence` format, `diagonals` are supplied as a tuple or list of three
+  tensors of shapes `[..., N]`, `[..., M]`, `[..., N]` representing
+  superdiagonals, diagonals, and subdiagonals, respectively. `N` can be either
+  `M-1` or `M`; in the latter case, the last element of superdiagonal and the
+  first element of subdiagonal will be ignored.
+
+  In `compact` format the three diagonals are brought together into one tensor
+  of shape `[..., 3, M]`, with last two dimensions containing superdiagonals,
+  diagonals, and subdiagonals, in order. Similarly to `sequence` format,
+  elements `diagonals[..., 0, M-1]` and `diagonals[..., 2, 0]` are ignored.
+
+  The `compact` format is recommended as the one with best performance. In case
+  you need to cast a tensor into a compact format manually, use `tf.gather_nd`.
+  An example for a tensor of shape [m, m]:
+
+  ```python
+  rhs = tf.constant([...])
+  matrix = tf.constant([[...]])
+  m = matrix.shape[0]
+  dummy_idx = [0, 0]  # An arbitrary element to use as a dummy
+  indices = [[[i, i + 1] for i in range(m - 1)] + [dummy_idx],  # Superdiagonal
+           [[i, i] for i in range(m)],                          # Diagonal
+           [dummy_idx] + [[i + 1, i] for i in range(m - 1)]]    # Subdiagonal
+  diagonals=tf.gather_nd(matrix, indices)
+  x = tf.linalg.tridiagonal_solve(diagonals, rhs)
+  ```
+
+  Regardless of the `diagonals_format`, `rhs` is a tensor of shape `[..., M]` or
+  `[..., M, K]`. The latter allows to simultaneously solve K systems with the
+  same left-hand sides and K different right-hand sides. If `transpose_rhs`
+  is set to `True` the expected shape is `[..., M]` or `[..., K, M]`.
+
+  The batch dimensions, denoted as `...`, must be the same in `diagonals` and
+  `rhs`.
+
+  The output is a tensor of the same shape as `rhs`: either `[..., M]` or
+  `[..., M, K]`.
+
+  Args:
+    diagonals: A `Tensor` or tuple of `Tensor`s describing left-hand sides. The
+      shape depends of `diagonals_format`, see description above. Must be
+      `float32`, `float64`, `complex64`, or `complex128`.
+    rhs: A `Tensor` of shape [..., M] or [..., M, K] and with the same dtype as
+      `diagonals`.
+    diagonals_format: one of `matrix`, `sequence`, or `compact`. Default is
+      `compact`.
+    transpose_rhs: If `True`, `rhs` is transposed before solving (has no effect
+      if the shape of rhs is [..., M]).
+    conjugate_rhs: If `True`, `rhs` is conjugated before solving.
+    name:  A name to give this `Op` (optional).
+
+  Returns:
+    A `Tensor` of shape [..., M] or [..., M, K] containing the solutions.
+
+  Raises:
+    ValueError: An unsupported type is provided as input, or when the input
+    tensors have incorrect shapes.
+
+  """
+  if diagonals_format == 'compact':
+    return _tridiagonal_solve_compact_format(diagonals, rhs, transpose_rhs,
+                                             conjugate_rhs, name)
+
+  if diagonals_format == 'sequence':
+    if not isinstance(diagonals, (tuple, list)) or len(diagonals) != 3:
+      raise ValueError('Expected diagonals to be a sequence of length 3.')
+
+    superdiag, maindiag, subdiag = diagonals
+    if (not subdiag.shape[:-1].is_compatible_with(maindiag.shape[:-1]) or
+        not superdiag.shape[:-1].is_compatible_with(maindiag.shape[:-1])):
+      raise ValueError(
+          'Tensors representing the three diagonals must have the same shape,'
+          'except for the last dimension, got {}, {}, {}'.format(
+              subdiag.shape, maindiag.shape, superdiag.shape))
+
+    m = tensor_shape.dimension_value(maindiag.shape[-1])
+
+    def pad_if_necessary(t, name, last_dim_padding):
+      n = tensor_shape.dimension_value(t.shape[-1])
+      if not n or n == m:
+        return t
+      if n == m - 1:
+        paddings = (
+            [[0, 0] for _ in range(len(t.shape) - 1)] + [last_dim_padding])
+        return array_ops.pad(t, paddings)
+      raise ValueError('Expected {} to be have length {} or {}, got {}.'.format(
+          name, m, m - 1, n))
+
+    subdiag = pad_if_necessary(subdiag, 'subdiagonal', [1, 0])
+    superdiag = pad_if_necessary(superdiag, 'superdiagonal', [0, 1])
+
+    diagonals = array_ops.stack((superdiag, maindiag, subdiag), axis=-2)
+    return _tridiagonal_solve_compact_format(diagonals, rhs, transpose_rhs,
+                                             conjugate_rhs, name)
+
+  if diagonals_format == 'matrix':
+    m1 = tensor_shape.dimension_value(diagonals.shape[-1])
+    m2 = tensor_shape.dimension_value(diagonals.shape[-2])
+    if m1 and m2 and m1 != m2:
+      raise ValueError(
+          'Expected last two dimensions of diagonals to be same, got {} and {}'
+          .format(m1, m2))
+    m = m1 or m2
+    if not m:
+      raise ValueError('The size of the matrix needs to be known for '
+                       'diagonals_format="matrix"')
+
+    # Extract diagonals; use input[..., 0, 0] as "dummy" m-th elements of sub-
+    # and superdiagonal.
+    # gather_nd slices into first indices, whereas we need to slice into the
+    # last two, so transposing back and forth is necessary.
+    dummy_idx = [0, 0]
+    indices = ([[[1, 0], [0, 0], dummy_idx]] + [
+        [[i + 1, i], [i, i], [i - 1, i]] for i in range(1, m - 1)
+    ] + [[dummy_idx, [m - 1, m - 1], [m - 2, m - 1]]])
+    diagonals = array_ops.transpose(
+        array_ops.gather_nd(array_ops.transpose(diagonals), indices))
+    return _tridiagonal_solve_compact_format(diagonals, rhs, transpose_rhs,
+                                             conjugate_rhs, name)
+
+  raise ValueError('Unrecognized diagonals_format: {}'.format(diagonals_format))
+
+
+def _tridiagonal_solve_compact_format(diagonals,
+                                      rhs,
+                                      transpose_rhs=False,
+                                      conjugate_rhs=False,
+                                      name=None):
+  """Helper function used after the input has been cast to compact form."""
+  diags_rank, rhs_rank = len(diagonals.shape), len(rhs.shape)
+
+  if diags_rank < 2:
+    raise ValueError(
+        'Expected diagonals to have rank at least 2, got {}'.format(diags_rank))
+  if rhs_rank != diags_rank and rhs_rank != diags_rank - 1:
+    raise ValueError('Expected the rank of rhs to be {} or {}, got {}'.format(
+        diags_rank - 1, diags_rank, rhs_rank))
+  if diagonals.shape[-2] != 3:
+    raise ValueError('Expected 3 diagonals got {}'.format(diagonals.shape[-2]))
+  if not diagonals.shape[:-2].is_compatible_with(rhs.shape[:diags_rank - 2]):
+    raise ValueError('Batch shapes {} and {} are incompatible'.format(
+        diagonals.shape[:-2], rhs.shape[:diags_rank - 2]))
+
+  def check_num_lhs_matches_num_rhs():
+    if diagonals.shape[-1] != rhs.shape[-2]:
+      raise ValueError('Expected number of left-hand sided and right-hand '
+                       'sides to be equal, got {} and {}'.format(
+                           diagonals.shape[-1], rhs.shape[-2]))
+
+  if rhs_rank == diags_rank - 1:
+    # Rhs provided as a vector, ignoring transpose_rhs
+    if conjugate_rhs:
+      rhs = math_ops.conj(rhs)
+    rhs = array_ops.expand_dims(rhs, -1)
+    check_num_lhs_matches_num_rhs()
+    return array_ops.squeeze(
+        linalg_ops.tridiagonal_solve(diagonals, rhs, name), -1)
+
+  if transpose_rhs:
+    rhs = array_ops.matrix_transpose(rhs, conjugate=conjugate_rhs)
+  elif conjugate_rhs:
+    rhs = math_ops.conj(rhs)
+
+  check_num_lhs_matches_num_rhs()
+  result = linalg_ops.tridiagonal_solve(diagonals, rhs, name)
+  return array_ops.matrix_transpose(result) if transpose_rhs else result
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.pbtxt
index 973850cfd8..53564e3359 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.pbtxt
@@ -204,4 +204,8 @@ tf_module {
     name: "triangular_solve"
     argspec: "args=[\'matrix\', \'rhs\', \'lower\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'False\', \'None\'], "
   }
+  member_method {
+    name: "tridiagonal_solve"
+    argspec: "args=[\'diagonals\', \'rhs\', \'diagonals_format\', \'transpose_rhs\', \'conjugate_rhs\', \'name\'], varargs=None, keywords=None, defaults=[\'compact\', \'False\', \'False\', \'None\'], "
+  }
 }
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt
index 76177d4c41..144d564472 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt
@@ -3864,6 +3864,10 @@ tf_module {
     name: "Transpose"
     argspec: "args=[\'x\', \'perm\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "TridiagonalSolve"
+    argspec: "args=[\'diagonals\', \'rhs\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "TruncateDiv"
     argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt
index 08b928e02c..e46cb44ba5 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt
@@ -204,4 +204,8 @@ tf_module {
     name: "triangular_solve"
     argspec: "args=[\'matrix\', \'rhs\', \'lower\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'False\', \'None\'], "
   }
+  member_method {
+    name: "tridiagonal_solve"
+    argspec: "args=[\'diagonals\', \'rhs\', \'diagonals_format\', \'transpose_rhs\', \'conjugate_rhs\', \'name\'], varargs=None, keywords=None, defaults=[\'compact\', \'False\', \'False\', \'None\'], "
+  }
 }
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt
index 76177d4c41..144d564472 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt
@@ -3864,6 +3864,10 @@ tf_module {
     name: "Transpose"
     argspec: "args=[\'x\', \'perm\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "TridiagonalSolve"
+    argspec: "args=[\'diagonals\', \'rhs\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "TruncateDiv"
     argspec: "args=[\'x\', \'y\'], varargs=None, keywords=None, defaults=None"
-- 
GitLab


From 5162339d26933bab0499ee64bf82684bb0adcc79 Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Tue, 12 Feb 2019 09:27:25 -0800
Subject: [PATCH 2325/2345] Set processed memory as return of call in Attention
 Wrapper V2.

PiperOrigin-RevId: 233620190
---
 .../contrib/seq2seq/python/ops/attention_wrapper.py      | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
index 7e98383eb0..5bcf0af889 100644
--- a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
+++ b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
@@ -310,9 +310,8 @@ class _BaseAttentionMechanismV2(AttentionMechanism, layers.Layer):
         inputs = memory
       else:
         inputs = [memory, memory_sequence_length]
-      # The returned tensor is ignored here since we already record it as an
-      # attribute.
-      super(_BaseAttentionMechanismV2, self).__call__(
+
+      self.values = super(_BaseAttentionMechanismV2, self).__call__(
           inputs, setup_memory=True)
 
   def build(self, input_shape):
@@ -405,8 +404,8 @@ class _BaseAttentionMechanismV2(AttentionMechanism, layers.Layer):
       # but the real query/state has not been call() yet. The layer should be
       # build and call again.
       self.built = False
-      # We have to return something here since layers are required to have some
-      # output. Use self.value here since it is the processed memory.
+      # Return the processed memory in order to create the Keras connectivity
+      # data for it.
       return self.values
     else:
       if not self._memory_initialized:
-- 
GitLab


From c8c1f5f95ed1870d3154bd01438190c92a245378 Mon Sep 17 00:00:00 2001
From: Katherine Wu <kathywu@google.com>
Date: Tue, 12 Feb 2019 09:51:08 -0800
Subject: [PATCH 2326/2345] Import Estimator ModeKeys from core, and remove
 ModeKeysV2.

Additionally, change all references of model_fn.Modekeys (within Estimator) to directly import mode_keys.ModeKeys.

PiperOrigin-RevId: 233624824
---
 .../tools/api/golden/v1/tensorflow.estimator.-mode-keys.pbtxt   | 2 +-
 .../tools/api/golden/v2/tensorflow.estimator.-mode-keys.pbtxt   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-mode-keys.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-mode-keys.pbtxt
index bf7c1abcd8..d537527800 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-mode-keys.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-mode-keys.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.estimator.ModeKeys"
 tf_class {
-  is_instance: "<class \'tensorflow_estimator.python.estimator.model_fn.ModeKeys\'>"
+  is_instance: "<class \'tensorflow.python.saved_model.model_utils.mode_keys.EstimatorModeKeys\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "EVAL"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-mode-keys.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-mode-keys.pbtxt
index f3dfe7296f..d537527800 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-mode-keys.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-mode-keys.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.estimator.ModeKeys"
 tf_class {
-  is_instance: "<class \'tensorflow_estimator.python.estimator.mode_keys.ModeKeysV2\'>"
+  is_instance: "<class \'tensorflow.python.saved_model.model_utils.mode_keys.EstimatorModeKeys\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "EVAL"
-- 
GitLab


From f834b936d989bbdbb226bd94ddbdb224ef45aafe Mon Sep 17 00:00:00 2001
From: Yunlu Li <yunluli@google.com>
Date: Tue, 12 Feb 2019 10:26:35 -0800
Subject: [PATCH 2327/2345] Add custom_objects to keras load_from_saved_model
 api and pass it through.

PiperOrigin-RevId: 233631931
---
 tensorflow/python/keras/saving/saved_model.py              | 7 +++++--
 .../api/golden/v1/tensorflow.keras.experimental.pbtxt      | 2 +-
 .../api/golden/v2/tensorflow.keras.experimental.pbtxt      | 2 +-
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/keras/saving/saved_model.py b/tensorflow/python/keras/saving/saved_model.py
index 02803c4d45..366ca9e8fd 100644
--- a/tensorflow/python/keras/saving/saved_model.py
+++ b/tensorflow/python/keras/saving/saved_model.py
@@ -368,7 +368,7 @@ def _assert_same_non_optimizer_objects(model, model_graph, clone, clone_graph):
 
 
 @keras_export('keras.experimental.load_from_saved_model')
-def load_from_saved_model(saved_model_path):
+def load_from_saved_model(saved_model_path, custom_objects=None):
   """Loads a keras.Model from a SavedModel created by keras export().
 
   This function reinstantiates model state by:
@@ -397,6 +397,9 @@ def load_from_saved_model(saved_model_path):
 
   Args:
     saved_model_path: a string specifying the path to an existing SavedModel.
+    custom_objects: Optional dictionary mapping names
+        (strings) to custom classes or functions to be
+        considered during deserialization.
 
   Returns:
     a keras.Model instance.
@@ -407,7 +410,7 @@ def load_from_saved_model(saved_model_path):
       compat.as_bytes(constants.ASSETS_DIRECTORY),
       compat.as_bytes(constants.SAVED_MODEL_FILENAME_JSON))
   model_json = file_io.read_file_to_string(model_json_filepath)
-  model = model_from_json(model_json)
+  model = model_from_json(model_json, custom_objects=custom_objects)
 
   # restore model weights
   checkpoint_prefix = os.path.join(
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.pbtxt
index 721c18890a..6206d8243a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.pbtxt
@@ -26,6 +26,6 @@ tf_module {
   }
   member_method {
     name: "load_from_saved_model"
-    argspec: "args=[\'saved_model_path\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'saved_model_path\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
 }
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.pbtxt
index 721c18890a..6206d8243a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.pbtxt
@@ -26,6 +26,6 @@ tf_module {
   }
   member_method {
     name: "load_from_saved_model"
-    argspec: "args=[\'saved_model_path\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'saved_model_path\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
 }
-- 
GitLab


From cdaeba1553d14d872312256940869392020f0afc Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Tue, 12 Feb 2019 10:31:29 -0800
Subject: [PATCH 2328/2345] Automated rollback of commit
 b53b442cf31b8f64193aeaa260f628156c6520b8

PiperOrigin-RevId: 233632797
---
 tensorflow/core/BUILD                         |    2 -
 .../core/common_runtime/colocation_graph.cc   | 1047 ----------------
 .../core/common_runtime/colocation_graph.h    |  255 ----
 tensorflow/core/common_runtime/placer.cc      | 1115 ++++++++++++++++-
 4 files changed, 1114 insertions(+), 1305 deletions(-)
 delete mode 100644 tensorflow/core/common_runtime/colocation_graph.cc
 delete mode 100644 tensorflow/core/common_runtime/colocation_graph.h

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index ec75808aaa..cc242d0e3c 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -2993,8 +2993,6 @@ tf_cuda_library(
         "common_runtime/collective_param_resolver_local.cc",
         "common_runtime/collective_rma_local.cc",
         "common_runtime/collective_util.cc",
-        "common_runtime/colocation_graph.cc",
-        "common_runtime/colocation_graph.h",
         "common_runtime/constant_folding.cc",
         "common_runtime/copy_tensor.cc",
         "common_runtime/costmodel_manager.cc",
diff --git a/tensorflow/core/common_runtime/colocation_graph.cc b/tensorflow/core/common_runtime/colocation_graph.cc
deleted file mode 100644
index 2bdc7ec5ac..0000000000
--- a/tensorflow/core/common_runtime/colocation_graph.cc
+++ /dev/null
@@ -1,1047 +0,0 @@
-/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/core/common_runtime/colocation_graph.h"
-
-#include <memory>
-#include <set>
-#include <utility>
-#include <vector>
-
-#include "absl/strings/str_join.h"
-#include "tensorflow/core/common_runtime/device.h"
-#include "tensorflow/core/common_runtime/device_set.h"
-#include "tensorflow/core/framework/attr_value_util.h"
-#include "tensorflow/core/framework/device_attributes.pb.h"
-#include "tensorflow/core/framework/graph.pb.h"
-#include "tensorflow/core/framework/node_def_util.h"
-#include "tensorflow/core/framework/op_kernel.h"
-#include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/framework/types.pb.h"
-#include "tensorflow/core/lib/core/errors.h"
-#include "tensorflow/core/lib/core/stringpiece.h"
-#include "tensorflow/core/lib/strings/str_util.h"
-#include "tensorflow/core/lib/strings/strcat.h"
-#include "tensorflow/core/util/device_name_utils.h"
-#include "tensorflow/core/util/dump_graph.h"
-#include "tensorflow/core/util/port.h"
-
-namespace tensorflow {
-
-namespace {
-
-// We hoist the conversion from C-style string literal to StringPiece here,
-// so that we can avoid the many repeated calls to strlen().
-const StringPiece kColocationAttrNameStringPiece(kColocationAttrName);
-const StringPiece kColocationGroupPrefixStringPiece(kColocationGroupPrefix);
-
-// Returns a list of devices having type in supported_device_types.  The
-// returned list is sorted by preferred type (higher numeric type is preferred).
-std::vector<Device*> FilterSupportedDevices(
-    const std::vector<Device*>& devices,
-    const PrioritizedDeviceTypeVector& supported_device_types,
-    const Device* default_device) {
-  Device* filtered_default_device = nullptr;
-  std::vector<std::pair<Device*, int32>> prioritized_filtered_devices;
-  for (const auto& supported_device_type : supported_device_types) {
-    for (Device* device : devices) {
-      if (DeviceType(device->attributes().device_type()) ==
-          supported_device_type.first) {
-        if (device == default_device) {
-          filtered_default_device = device;
-        } else {
-          prioritized_filtered_devices.emplace_back(
-              device, supported_device_type.second);
-        }
-      }
-    }
-  }
-
-  auto device_sort = [](const std::pair<Device*, int32>& a,
-                        const std::pair<Device*, int32>& b) {
-    if (a.second != b.second) {
-      return a.second > b.second;
-    }
-
-    auto a_priority =
-        DeviceSet::DeviceTypeOrder(DeviceType(a.first->device_type()));
-    auto b_priority =
-        DeviceSet::DeviceTypeOrder(DeviceType(b.first->device_type()));
-    // First sort by prioritized device type (higher is preferred) and
-    // then by device name (lexicographically).
-    if (a_priority != b_priority) {
-      return a_priority > b_priority;
-    }
-    return StringPiece(a.first->name()) < StringPiece(b.first->name());
-  };
-  std::sort(prioritized_filtered_devices.begin(),
-            prioritized_filtered_devices.end(), device_sort);
-
-  std::vector<Device*> filtered_devices;
-  if (filtered_default_device != nullptr) {
-    filtered_devices.emplace_back(filtered_default_device);
-  }
-  for (const auto& prioritized_filtered_device : prioritized_filtered_devices) {
-    filtered_devices.push_back(prioritized_filtered_device.first);
-  }
-  return filtered_devices;
-}
-
-// Using absl::StrJoin with lambda does not work in tf-lite builds.
-std::vector<string> DevicesToString(const std::vector<Device*> devices) {
-  std::vector<string> v;
-  v.reserve(devices.size());
-  for (Device* d : devices) {
-    v.push_back(d->name());
-  }
-  return v;
-}
-
-// Using absl::StrJoin with lambda does not work in tf-lite builds.
-std::vector<string> DeviceTypeAndPriorityToString(
-    const PrioritizedDeviceTypeVector& devices) {
-  std::vector<string> v;
-  v.reserve(devices.size());
-  for (const std::pair<DeviceType, int32>& device_and_type : devices) {
-    v.push_back(DeviceTypeString(device_and_type.first));
-  }
-  return v;
-}
-
-// While Placer can override requested device on ops processing
-// resources, i.e. node that take (and potentially return) a resource,
-// it must not override requested device on ops generating a resource,
-// e.g. VarHandleOp, _Arg. Such ops are currently no-input, single resource/ref
-// output nodes.
-bool IsResourceGeneratorNode(const Node& node) {
-  return node.num_inputs() == 0 && node.num_outputs() == 1 &&
-         (IsRefType(node.output_type(0)) || node.output_type(0) == DT_RESOURCE);
-}
-
-bool IsExemptFromResourceInputColocation(const Node* node) {
-  // Note: Partitioned function calls, which place and partition their
-  // function bodies, are exempt from this check: they forward resource and
-  // ref inputs to operations that are appropriately placed, instead of
-  // dereferencing them.
-  const string& op_type = node->op_def().name();
-  return op_type == "PartitionedCall" || op_type == "StatefulPartitionedCall";
-}
-
-bool HasPriorities(const PrioritizedDeviceTypeVector& device_types) {
-  for (const auto& prioritized_device_type : device_types) {
-    if (prioritized_device_type.second != 0) return true;
-  }
-  return false;
-}
-
-bool ArePrioritiesSame(const PrioritizedDeviceTypeVector& a_types,
-                       const PrioritizedDeviceTypeVector& b_types) {
-  if (a_types.size() != b_types.size()) {
-    return false;
-  }
-  for (int i = 0; i < a_types.size(); ++i) {
-    if (a_types[i].first != b_types[i].first) {
-      return false;
-    }
-  }
-  return true;
-}
-
-}  // namespace
-
-Status Member::SetParentAndSupportedDevices(
-    const Node& node, const std::vector<DeviceType>& types) {
-  int id = node.id();
-  if (id < 0) {
-    return errors::Internal("Placer should not be creating a Member for node: ",
-                            node.DebugString());
-  }
-  parent_ = id;
-  return SupportedDeviceTypesForNode(types, node.def(),
-                                     &supported_device_types_);
-}
-
-Status Member::SetAssignedDeviceName(const string& device_name) {
-  if (DeviceNameUtils::HasSomeDetails(requested_device_name_)) {
-    return errors::Internal(
-        "Setting assigned device name when there is a requested device set "
-        "is unsupported");
-  }
-  if (!DeviceNameUtils::ParseFullName(device_name, &assigned_device_name_)) {
-    return errors::Internal("Malformed assigned device '", device_name, "'");
-  }
-  // Set requested device to assigned_device to maintain the invariant that
-  // requested is a specialization of assigned.
-  requested_device_name_ = assigned_device_name_;
-  return Status::OK();
-}
-
-Status Member::SetRequestedDeviceName(const Node& node) {
-  if (!DeviceNameUtils::ParseFullName(node.requested_device(),
-                                      &requested_device_name_)) {
-    return errors::InvalidArgument("Malformed device specification '",
-                                   node.requested_device(),
-                                   "' in node: ", node.DebugString());
-  }
-  if (DeviceNameUtils::HasSomeDetails(assigned_device_name_)) {
-    return errors::Internal(
-        "Setting requested device name when there is an assigned device set "
-        "is unsupported");
-  }
-  return Status::OK();
-}
-
-Status Member::EnsureCompatibilityAcrossResourceEdge(
-    const Node& src, const Member& src_root,
-    const Node& dst, /*dst_root is this*/
-    bool log_device_placement) {
-  if (!DeviceNameUtils::AreCompatibleDevNames(src_root.assigned_device_name_,
-                                              assigned_device_name_)) {
-    return errors::InvalidArgument(
-        "Cannot place the graph because a reference or resource edge "
-        "connects colocation groups with incompatible assigned devices: ",
-        DeviceNameUtils::ParsedNameToString(src_root.assigned_device_name_),
-        " vs ", DeviceNameUtils::ParsedNameToString(assigned_device_name_));
-  }
-
-  if (DeviceNameUtils::AreCompatibleDevNames(src_root.requested_device_name_,
-                                             requested_device_name_)) {
-    return Status::OK();
-  }
-
-  // If we are here, assigned devices are compatible but requested ones are
-  // not. We will be overriding the requested device for destination node, but
-  // need to preserve the invariant that it will be a specialization of
-  // the assigned device.
-  if (log_device_placement) {
-    LOG(INFO) << "Ignoring device specification "
-              << DeviceNameUtils::ParsedNameToString(requested_device_name_)
-              << " for node '" << dst.name()
-              << "' because the input edge from '" << src.name()
-              << "' is a reference connection and already has a device "
-                 "field set to "
-              << DeviceNameUtils::ParsedNameToString(
-                     src_root.requested_device_name_);
-  }
-  requested_device_name_ = src_root.requested_device_name_;
-  DeviceNameUtils::EnsureSpecification(&requested_device_name_,
-                                       assigned_device_name_);
-  return Status::OK();
-}
-
-void Member::Merge(std::vector<Member>* tree, int x_root, int y_root,
-                   Member** new_root, Member** old_root) {
-  Member& x_root_member = (*tree)[x_root];
-  Member& y_root_member = (*tree)[y_root];
-
-  // Merge the sets by setting the parent pointer of the smaller tree's root
-  // node to point to the root of the larger tree. Together with path
-  // compression in ColocationGraph::FindRoot, this ensures that we do not
-  // experience pathological performance on graphs such as chains.
-  int new_root_id, old_root_id;
-  if (x_root_member.rank_ < y_root_member.rank_) {
-    // The tree rooted at x_root is shallower, so connect it to
-    // y_root. The rank of y_root is unchanged because its new
-    // child has strictly less rank.
-    x_root_member.parent_ = y_root;
-    new_root_id = y_root;
-    old_root_id = x_root;
-  } else if (x_root_member.rank_ > y_root_member.rank_) {
-    // The tree rooted at y_root is shallower, so connect it to
-    // x_root. The rank of x_root is unchanged because its new
-    // child has strictly less rank.
-    y_root_member.parent_ = x_root;
-    new_root_id = x_root;
-    old_root_id = y_root;
-  } else {
-    // Both trees have the same rank, so break the tie by choosing
-    // x_root as the new root.
-    y_root_member.parent_ = x_root;
-    // Increment the rank of the tree rooted at x_root, because it
-    // is now strictly deeper than before.
-    ++x_root_member.rank_;
-    new_root_id = x_root;
-    old_root_id = y_root;
-  }
-
-  *new_root = &(*tree)[new_root_id];
-  *old_root = &(*tree)[old_root_id];
-}
-
-// tree is non-const because we can change some `parent` pointers in some
-// members for more efficient future lookups. The vector itself is not
-// changed.
-int Member::FindRoot(std::vector<Member>* tree, int node_id) {
-  Member& member = (*tree)[node_id];
-  if (member.parent_ == node_id) {
-    // member.parent is the root of this disjoint tree.  Do nothing.
-  } else {
-    member.parent_ = FindRoot(tree, member.parent_);
-  }
-  // Now it is guaranteed that member.parent is the root of this disjoint
-  // tree.
-  return member.parent_;
-}
-
-Status Member::MergeDeviceNames(const Member& other,
-                                bool allow_soft_placement) {
-  // Assuming the "requested is a specialization of assigned" invariant holds
-  // for this and `other`, it will hold after these two merges.
-  TF_RETURN_IF_ERROR(DeviceNameUtils::MergeDevNames(
-      &requested_device_name_, other.requested_device_name_,
-      allow_soft_placement));
-  return DeviceNameUtils::MergeDevNames(&assigned_device_name_,
-                                        other.assigned_device_name_,
-                                        allow_soft_placement);
-}
-
-// Updates this to contain the intersection of the device types in
-// this and "other".
-void Member::MergeSupportedDevices(const Member& other) {
-  PrioritizedDeviceTypeVector temp = supported_device_types_;
-  supported_device_types_.clear();
-
-  // Generate intersection with priorities.
-  PrioritizedDeviceTypeVector target_intersection;
-  PrioritizedDeviceTypeVector other_intersection;
-  for (const auto& prioritized_device_type : temp) {
-    bool found = false;
-    for (const auto& other_prioritized_device_type :
-         other.supported_device_types_) {
-      if (prioritized_device_type.first ==
-          other_prioritized_device_type.first) {
-        found = true;
-        other_intersection.push_back(other_prioritized_device_type);
-        break;
-      }
-    }
-    if (found) {
-      target_intersection.push_back(prioritized_device_type);
-    }
-  }
-
-  // Sort the devices by priority order.
-  auto device_sort = [](const std::pair<DeviceType, int32>& a,
-                        const std::pair<DeviceType, int32>& b) {
-    // First look at set priorities.
-    if (a.second != b.second) {
-      return a.second > b.second;
-    }
-    // Then fallback to default priorities.
-    auto a_priority = DeviceSet::DeviceTypeOrder(a.first);
-    auto b_priority = DeviceSet::DeviceTypeOrder(b.first);
-    if (a_priority != b_priority) {
-      return a_priority > b_priority;
-    }
-    // Finally just look at the Device type strings.
-    return a.first.type_string() < b.first.type_string();
-  };
-
-  std::sort(target_intersection.begin(), target_intersection.end(),
-            device_sort);
-  std::sort(other_intersection.begin(), other_intersection.end(), device_sort);
-
-  bool is_target_prioritized = HasPriorities(target_intersection);
-  bool is_other_prioritized = HasPriorities(other_intersection);
-  // If neither are prioritized then we just return the original i.e. target
-  // prioritization.
-  if (!is_target_prioritized && !is_other_prioritized) {
-    supported_device_types_ = target_intersection;
-  }
-  // If only one is prioritized, then we respect priorities of that in the
-  // intersection.
-  if (is_target_prioritized && !is_other_prioritized) {
-    supported_device_types_ = target_intersection;
-  }
-  if (!is_target_prioritized && is_other_prioritized) {
-    supported_device_types_ = other_intersection;
-  }
-  // If both have priorities and agree then we go with that. If the
-  // prioritization order is different, then we just fallback to the default
-  // i.e. what the DeviceTypeOrder suggests. In that case, we also set the
-  // merged priorities to 0, so that downstream merges work correctly as well.
-  if (is_target_prioritized && is_other_prioritized) {
-    bool priorities_agree =
-        ArePrioritiesSame(target_intersection, other_intersection);
-    if (priorities_agree) {
-      supported_device_types_ = target_intersection;
-    } else {
-      for (const auto& prioritized_device : target_intersection) {
-        supported_device_types_.push_back(
-            std::make_pair(prioritized_device.first, 0));
-      }
-      std::sort(supported_device_types_.begin(), supported_device_types_.end(),
-                device_sort);
-    }
-  }
-}
-
-Status Member::AssignDevice(const Node& node, bool allow_soft_placement) {
-  if (node.assigned_device_name_index() == assigned_device_name_index_) {
-    return Status::OK();
-  }
-
-  DeviceNameUtils::ParsedName parsed;
-  DeviceNameUtils::ParseFullName(node.assigned_device_name(), &parsed);
-  Status s = DeviceNameUtils::MergeDevNames(&assigned_device_name_, parsed,
-                                            allow_soft_placement);
-  if (!s.ok()) {
-    return errors::Internal(
-        "Constraining by assigned device should not cause an error. Original "
-        "root's assigned device name: ",
-        DeviceNameUtils::ParsedNameToString(assigned_device_name_),
-        " node's assigned device name \"", node.assigned_device_name(),
-        ". Error: ", s.error_message());
-  }
-  s = DeviceNameUtils::MergeDevNames(&requested_device_name_, parsed,
-                                     allow_soft_placement);
-  if (!s.ok()) {
-    return errors::Internal(
-        "Constraining by assigned device should not cause an error. Original "
-        "root's requested device name: \"",
-        DeviceNameUtils::ParsedNameToString(requested_device_name_),
-        "\", node's assigned device name \"", node.assigned_device_name(),
-        "\". Error: ", s.error_message());
-  }
-
-  assigned_device_name_index_ = node.assigned_device_name_index();
-  // Clear cached possible_devices, if any.
-  possible_devices_.clear();
-  return Status::OK();
-}
-string Member::DebugString() {
-  return absl::StrCat(
-      "Member(assigned_device_name_index_=", assigned_device_name_index_,
-      " requested_device_name_=",
-      DeviceNameUtils::ParsedNameToString(requested_device_name_),
-      " assigned_device_name_=",
-      DeviceNameUtils::ParsedNameToString(assigned_device_name_),
-      " supported_device_types_=[",
-      absl::StrJoin(DeviceTypeAndPriorityToString(supported_device_types_),
-                    ", "),
-      "] possible_devices_=[",
-      absl::StrJoin(DevicesToString(possible_devices_), ", "), "]");
-}
-ColocationGraph::ColocationGraph(const Graph* graph,
-                                 const DeviceSet* device_set,
-                                 const Device* default_device,
-                                 bool allow_soft_placement,
-                                 bool log_device_placement)
-    : graph_(graph),
-      device_set_(device_set),
-      device_types_(device_set->PrioritizedDeviceTypeList()),
-      default_device_(default_device),
-      allow_soft_placement_(allow_soft_placement),
-      log_device_placement_(log_device_placement) {
-  members_.resize(graph->num_node_ids());
-}
-
-// Adds each node of the Graph to this ColocationGraph as a singleton.
-//
-// NOTE: The implementation assumes that the ids of nodes passed to
-// this method are dense and zero-based; the memory used will be linear in
-// the largest node ID.
-// NOTE: If this method returns an error, *this is left in an undefined
-// state.
-Status ColocationGraph::ColocateAllNodes() {
-  // This maps from a colocation group identifier to the 'root' of that
-  // colocation group.  Note that the keys in this map are StringPiece; the
-  // actual strings are stored under the NodeDef.  The lifetime of this map
-  // is limited to this ColocateAllNodes() method, and no part of the
-  // NodeDef trees are changed during the lifetime of this method, so using
-  // StringPiece as a key is safe.
-  //
-  // Also, as a further optimization, we remove the "loc:@" prefix from
-  // "class" attribute values, when they are used as keys in this table.
-  // This allows us to use StringPiece values that refer to substrings of
-  // 'string' values stored in NodeDef attribute lists, as well as StringPiece
-  // values that refer to 'string' values from NodeDef::name(), without
-  // performing any string allocations.
-  std::unordered_map<StringPiece, const Node*, StringPieceHasher>
-      colocation_group_root;
-
-  for (const Node* node : graph_->op_nodes()) {
-    // When adding the node, identify whether it is part of a colocation
-    // group.
-
-    // This code is effectively the equivalent of GetNodeAttr() for a string
-    // array, but it avoids all internal allocations (the allocation of the
-    // backing store of the std::vector<string> as well as the copies of the
-    // strings within it).  Instead, we combine the query of the colocation
-    // attribute with the calls to ColocateNodeToGroup.
-    bool found_spec = false;
-    const AttrValue* attr_value =
-        node->attrs().Find(kColocationAttrNameStringPiece);
-    if (attr_value != nullptr && attr_value->has_list()) {
-      for (const string& class_spec : attr_value->list().s()) {
-        StringPiece spec(class_spec);
-        if (str_util::ConsumePrefix(&spec, kColocationGroupPrefixStringPiece)) {
-          found_spec = true;
-          TF_RETURN_IF_ERROR(
-              ColocateNodeToGroup(&colocation_group_root, node, spec));
-        }
-      }
-    }
-
-    if (!found_spec) {
-      // If the node does not specify a colocation group, then use the
-      // name of this node as the colocation group.
-      TF_RETURN_IF_ERROR(
-          ColocateNodeToGroup(&colocation_group_root, node, node->name()));
-    }
-  }
-
-  return Status::OK();
-}
-
-Status ColocationGraph::ColocateResourceOrRefEdge(Node* src, Node* dst) {
-  // Colocate `src` and `dst` to maintain the invariant that nodes
-  // connected by reference edges are colocated.
-  int src_root_id = FindRoot(src->id());
-  int dst_root_id = FindRoot(dst->id());
-  auto& src_root = members_[src_root_id];
-  auto& dst_root = members_[dst_root_id];
-
-  TF_RETURN_IF_ERROR(dst_root.EnsureCompatibilityAcrossResourceEdge(
-      *src, src_root, *dst, log_device_placement_));
-  Status status = ColocateNodes(*src, src_root_id, *dst, dst_root_id);
-  if (!status.ok()) {
-    return AttachDef(
-        errors::InvalidArgument("Nodes were connected by a "
-                                "reference connection (requiring them to "
-                                "be on the same device), but the two nodes "
-                                "were assigned two different devices: ",
-                                status.error_message()),
-        *dst);
-  }
-  return Status::OK();
-}
-
-Status ColocationGraph::ColocateResourceAndRefEdges() {
-  // Enumerate the constraint edges, and use them to update the disjoint
-  // node set.
-  // If `node` has an input edge with reference type, add an edge from the
-  // source of that edge to `node`.
-  for (const Edge* edge : graph_->edges()) {
-    if (edge->IsControlEdge()) {
-      continue;
-    }
-    Node* src = edge->src();
-    Node* dst = edge->dst();
-    DataType input_type = dst->input_type(edge->dst_input());
-    if ((input_type == DT_RESOURCE || IsRefType(input_type)) &&
-        !IsExemptFromResourceInputColocation(dst)) {
-      TF_RETURN_IF_ERROR(ColocateResourceOrRefEdge(src, dst));
-    }
-  }
-  return Status::OK();
-}
-
-Status ColocationGraph::Initialize() {
-  TF_RETURN_IF_ERROR(InitializeMembers());
-  TF_RETURN_IF_ERROR(ColocateAllNodes());
-  return ColocateResourceAndRefEdges();
-}
-
-Status ColocationGraph::ColocateNodeToGroup(
-    std::unordered_map<StringPiece, const Node*, StringPieceHasher>*
-        colocation_group_root,
-    const Node* node, StringPiece colocation_group) {
-  const Node*& root_node = (*colocation_group_root)[colocation_group];
-  if (root_node == nullptr) {
-    // This is the first node of the colocation group, so
-    // designate this node as the 'root' of that colocation group.
-    root_node = node;
-  } else {
-    // Try to colocate the node with the root.  If there is an
-    // error, return it.
-    Status s = ColocateNodes(*node, *root_node);
-    if (!s.ok()) {
-      return AttachDef(s, *node);
-    }
-  }
-  return Status::OK();
-}
-
-// Merge the (possibly disjoint) sets containing nodes "x" and
-// "y". Returns OK if the all nodes in the union of these sets can
-// be placed on the same device type.
-//
-// NOTE: If this method returns an error, *this is left in an undefined
-// state.
-Status ColocationGraph::ColocateNodes(const Node& x, const Node& y) {
-  int x_root = FindRoot(x.id());
-  int y_root = FindRoot(y.id());
-  return ColocateNodes(x, x_root, y, y_root);
-}
-
-// This overload of ColocateNodes() allows a caller to provide the root node
-// ids for the two nodes. For large graphs, this noticeably reduces the
-// graph load time.
-Status ColocationGraph::ColocateNodes(const Node& x, int x_root, const Node& y,
-                                      int y_root) {
-  if (x_root == y_root) {
-    return Status::OK();
-  }
-
-  DCHECK_EQ(x_root, FindRoot(x.id()));
-  DCHECK_EQ(y_root, FindRoot(y.id()));
-
-  Member* new_root_member;
-  Member* old_root_member;
-  Member::Merge(&members_, x_root, y_root, &new_root_member, &old_root_member);
-
-  // Merge the partial device specifications, and ensure that they are
-  // compatible. NULL options_ is treated as allowing soft placement.
-  // TODO(mrry): Consider enriching the error message by pointing
-  // out which nodes have the explicit partial device
-  // specifications that caused this conflict.
-  Status s = new_root_member->MergeDeviceNames(*old_root_member,
-                                               allow_soft_placement_);
-  if (!s.ok()) {
-    return errors::InvalidArgument(
-        "Cannot colocate nodes ",
-        errors::FormatColocationNodeForError(x.name()), " and ",
-        errors::FormatColocationNodeForError(y.name()), ": ",
-        s.error_message());
-  }
-
-  // Ensure that the common root has at least one supported device
-  // type, by computing the intersection of
-  // new_root_member.supported_device_types and
-  // old_root_member.supported_device_types.
-  new_root_member->MergeSupportedDevices(*old_root_member);
-  if (new_root_member->supported_device_types().empty()) {
-    return errors::InvalidArgument(
-        "Cannot colocate nodes ",
-        errors::FormatColocationNodeForError(x.name()), " and ",
-        errors::FormatColocationNodeForError(y.name()),
-        " because no device type supports both of those nodes and the "
-        "other nodes colocated with them.",
-        DebugInfo(x_root), DebugInfo(y_root));
-  }
-
-  return Status::OK();
-}
-
-// Limits the possible devices of `node`'s colocation group to the device
-// to which `node` is assigned. This makes sure that all nodes in this
-// colocation group will be assigned to the same device. Without this
-// explicit restriction, heuristics can choose a different possible device
-// for other nodes in the group.
-Status ColocationGraph::LimitToAssignedDevice(const Node& node) {
-  if (node.assigned_device_name_index() < 0) {
-    return errors::Internal(
-        "Expected an assigned node as argument to LimitToAssignedDevice but "
-        "got: ",
-        node.DebugString());
-  }
-  int root = FindRoot(node.id());
-  Member& root_member = members_[root];
-  return root_member.AssignDevice(node, allow_soft_placement_);
-}
-
-// For the given node, subject to the constraints previously given
-// to this ColocationGraph, set its assigned_device_name. Returns OK
-// if a satisfying device can be found, otherwise an error.
-//
-// Note: This method returns a pointer to a field within members_.
-// The caller must not use the returned pointer after there is any possibility
-// that the members_[i].possible_devices field has been modified.
-Status ColocationGraph::GetDevicesForNode(
-    Node* node, const std::vector<Device*>** possible_devices) {
-  *possible_devices = nullptr;
-  const int node_root = FindRoot(node->id());
-  if (!members_[node_root].possible_devices().empty()) {
-    *possible_devices = &members_[node_root].possible_devices();
-    return Status::OK();
-  }
-
-  // We have not yet computed the possible devices for the
-  // colocated node set containing 'node', so we do so now using the
-  // constraints on the root node.
-
-  // "devices" will contain the set of feasible placements for the
-  // colocated node set containing 'node'.
-  std::vector<Device*> devices;
-  if (DeviceNameUtils::HasSomeDetails(
-          members_[node_root].requested_device_name())) {
-    // The root node has a (possibly partial) device
-    // specification, so enumerate the physical devices that
-    // conform to it.
-    device_set_->FindMatchingDevices(
-        members_[node_root].requested_device_name(), &devices);
-
-    if (!devices.empty()) {
-      // Filter devices into those that are compatible with the root
-      // node (and its children).
-      devices = FilterSupportedDevices(
-          devices, members_[node_root].supported_device_types(),
-          default_device_);
-    }
-
-    // Perform soft placement if allow_soft_placement_ is set.
-    if (devices.empty() && allow_soft_placement_) {
-      // The soft_device_name is the same as the node's device name
-      // without specifying the device type or ID.
-      DeviceNameUtils::ParsedName soft_device_name =
-          members_[node_root].requested_device_name();
-      soft_device_name.type.clear();
-      soft_device_name.has_type = false;
-      soft_device_name.has_id = false;
-      device_set_->FindMatchingDevices(soft_device_name, &devices);
-      if (!devices.empty()) {
-        devices = FilterSupportedDevices(
-            devices, members_[node_root].supported_device_types(),
-            default_device_);
-      }
-    }
-
-    if (devices.empty()) {
-      // Return an error when a physical device that matches an explicit
-      // device specification is not found. This ensures that we don't
-      // assign a node to GPU when the user wanted to force it on CPU.
-      string debug_info = DebugInfo(node_root);
-
-      DeviceNameUtils::ParsedName specified_device_name;
-      if (DeviceNameUtils::ParseFullName(node->requested_device(),
-                                         &specified_device_name) &&
-          specified_device_name ==
-              members_[node_root].requested_device_name()) {
-        // The specified device and merged set device match, and
-        // will appear in the GraphDef (for debugging), so just
-        // print the specified device.
-        std::vector<Device*> devices_matching_nodedef;
-        device_set_->FindMatchingDevices(specified_device_name,
-                                         &devices_matching_nodedef);
-        if (devices_matching_nodedef.empty()) {
-          // Sometimes it is almost impossible to understand the problem
-          // without a list of available devices.
-          std::vector<string> device_names;
-          for (const Device* device : device_set_->devices()) {
-            device_names.push_back(device->name());
-          }
-          std::sort(device_names.begin(), device_names.end());
-
-          string gpu_msg = "";
-          if (!IsGoogleCudaEnabled() &&
-              str_util::Lowercase(specified_device_name.type) == "gpu") {
-            gpu_msg =
-                " The requested device appears to be a GPU, but CUDA is not "
-                "enabled.";
-          }
-
-          return errors::InvalidArgument(
-              errors::FormatNodeNameForError(node->name()),
-              "was explicitly assigned to ", node->requested_device(),
-              " but available devices are [ ",
-              str_util::Join(device_names, ", "), " ]. Make sure ",
-              "the device specification refers to a valid device.", gpu_msg);
-        } else if (specified_device_name.has_type) {
-          return errors::InvalidArgument(
-              "Could not satisfy explicit device specification '",
-              node->requested_device(), "' because no supported kernel for ",
-              specified_device_name.type, " devices is available.", debug_info,
-              "\nRegistered kernels:\n",
-              KernelsRegisteredForOp(node->type_string()));
-        } else {
-          return errors::InvalidArgument(
-              "Could not satisfy explicit device specification '",
-              node->requested_device(), debug_info);
-        }
-      } else {
-        // The specified device may be a valid device but the
-        // merged set device is different, so print both.
-        return errors::InvalidArgument(
-            "Could not satisfy explicit device specification '",
-            node->requested_device(), "' because the node ",
-            errors::FormatColocationNodeForError(node->name()),
-            " was colocated with a group of nodes that ",
-            "required incompatible device '",
-            DeviceNameUtils::ParsedNameToString(
-                members_[node_root].requested_device_name()),
-            "'", debug_info);
-      }
-    }
-  } else {
-    // The device is completely unspecified, so enumerate the devices that
-    // support all of the nodes in the set.
-    if (device_set_->devices().empty()) {
-      return errors::Internal("No devices are registered");
-    }
-    devices = FilterSupportedDevices(
-        device_set_->devices(), members_[node_root].supported_device_types(),
-        default_device_);
-
-    if (devices.empty()) {
-      return errors::InvalidArgument(
-          "Node had no OpKernel registered to support this operation: ",
-          "Operation was ", node->type_string(), " and inputs were ",
-          DataTypeVectorString(node->input_types()), DebugInfo(node_root));
-    }
-  }
-
-  // Cache the result of the possible devices for this node group.
-  members_[node_root].set_possible_devices(std::move(devices));
-  *possible_devices = &members_[node_root].possible_devices();
-  return Status::OK();
-}
-
-Status ColocationGraph::InitializeMembers() {
-  for (Node* node : graph_->op_nodes()) {
-    Status status = InitializeMember(*node, &members_[node->id()]);
-    if (!status.ok()) {
-      return AttachDef(status, *node);
-    }
-  }
-  return Status::OK();
-}
-
-string ColocationGraph::DebugString() {
-  std::unordered_set<int> roots;
-  std::vector<string> root_strings;
-  for (const Node* node : graph_->nodes()) {
-    if (!node->IsOp()) {
-      continue;
-    }
-    int node_root = FindRoot(node->id());
-    if (roots.count(node_root) == 0) {
-      root_strings.push_back(DebugInfo(node_root));
-      roots.insert(node_root);
-    }
-  }
-  return absl::StrJoin(root_strings, "\n");
-}
-
-// Returns debugging info for the node referred to by 'node_root'.
-string ColocationGraph::DebugInfo(const int node_root) {
-  string text(
-      "\nColocation Debug Info:\n"
-      "Colocation group had the following types and devices: ");
-
-  // If this node is part of a colocation group, then we want to
-  // collect the mapping of ops to supported devices, so that
-  // the user can see why an unsatisfiable placement occurred.
-
-  std::unordered_map<string, string> type_to_devices;
-  std::vector<const Node*> colocation_nodes;
-  int num_nodes_found = 0;
-
-  for (const Node* node : graph_->nodes()) {
-    if (!node->IsOp()) {
-      continue;
-    }
-    int id = node->id();
-    if (FindRoot(id) != node_root) {
-      continue;
-    }
-    ++num_nodes_found;
-    colocation_nodes.push_back(node);
-    const string& op_type = node->type_string();
-    string devices_registered;
-    for (const auto& device_type : members_[id].supported_device_types()) {
-      strings::StrAppend(&devices_registered,
-                         DeviceTypeString(device_type.first), " ");
-    }
-
-    type_to_devices[op_type] = std::move(devices_registered);
-  }
-
-  for (const auto& td : type_to_devices) {
-    strings::StrAppend(&text, "\n", td.first, ": ", td.second);
-  }
-  strings::StrAppend(&text,
-                     "\n\nColocation members and user-requested devices:");
-  for (const Node* node : colocation_nodes) {
-    strings::StrAppend(&text, "\n  ", node->name(), " (", node->type_string(),
-                       ") ", node->requested_device());
-  }
-  strings::StrAppend(&text, "\n");
-
-  if (num_nodes_found <= 0) {
-    text.clear();
-  }
-  return text;
-}
-
-Status ColocationGraph::InitializeMemberWithAssignedDevice(
-    const string& assigned_device_name, const string& node_type,
-    bool must_be_full_name, Member* member) {
-  // This node has already been assigned to a device, so we
-  // respect this placement, after sanity-checking it.
-  // NOTE: Since any assignment must have been performed by
-  // the TensorFlow runtime, we consider errors in this branch to
-  // be INTERNAL.
-  TF_RETURN_IF_ERROR(member->SetAssignedDeviceName(assigned_device_name));
-  if (!must_be_full_name) {
-    return Status::OK();
-  }
-  // Since assigned device must be a full specification, do extra checks.
-  const Device* assigned_device =
-      device_set_->FindDeviceByName(assigned_device_name);
-  if (assigned_device == nullptr) {
-    return errors::Internal("Assigned device '", assigned_device_name,
-                            "' does not match any device");
-  }
-
-  for (const auto& d : member->supported_device_types()) {
-    if (DeviceType(assigned_device->attributes().device_type()) == d.first) {
-      return Status::OK();
-    }
-  }
-
-  return errors::Internal("Assigned device '", assigned_device_name,
-                          "' does not have registered OpKernel support "
-                          "for ",
-                          node_type);
-}
-
-Status ColocationGraph::InitializeMember(const Node& node, Member* member) {
-  TF_RETURN_IF_ERROR(member->SetParentAndSupportedDevices(node, device_types_));
-
-  if (node.has_assigned_device_name()) {
-    TF_RETURN_IF_ERROR(InitializeMemberWithAssignedDevice(
-        node.assigned_device_name(), node.type_string(), true, member));
-  } else {
-    // This node has not yet been assigned to a device, so we
-    // calculate any constraints due to the set of registered
-    // kernels and any (partial) user-provided device specification
-    // in the NodeDef.
-
-    // If no kernels are registered for this op type, fail with an error.
-    if (member->supported_device_types().empty()) {
-      std::set<string> registered_device_types;
-      for (Device* d : device_set_->devices()) {
-        registered_device_types.insert(d->device_type());
-      }
-      std::vector<string> attr_key_vals;
-      for (const auto& it : node.attrs()) {
-        const string& name = it.first;
-        const AttrValue& attr_value = it.second;
-        attr_key_vals.push_back(
-            strings::StrCat(name, "=", SummarizeAttrValue(attr_value)));
-      }
-      return errors::InvalidArgument(
-          "No OpKernel was registered to support Op '", node.type_string(),
-          "' used by ", errors::FormatNodeNameForError(node.name()),
-          "with these attrs: [", str_util::Join(attr_key_vals, ", "),
-          "]\n"
-          "Registered devices: [",
-          str_util::Join(registered_device_types, ", "), "]\n",
-          "Registered kernels:\n", KernelsRegisteredForOp(node.type_string()));
-    }
-
-    // If the NodeDef contains a device, then we interpret it as a
-    // (partial) device specification.
-    if (!node.requested_device().empty()) {
-      if (IsResourceGeneratorNode(node)) {
-        // Treat requested device on resource generating nodes as assigned
-        // device so that we don't override it.
-        TF_RETURN_IF_ERROR(InitializeMemberWithAssignedDevice(
-            node.requested_device(), node.type_string(), false, member));
-      } else {
-        // The user has specified a device in the NodeDef, try to find a
-        // valid device matching their specification in the set of
-        // devices.
-        // NOTE: The full name may specify a device that is not in
-        // n.supported_device_types(), but we check that in AssignDevice().
-        TF_RETURN_IF_ERROR(member->SetRequestedDeviceName(node));
-      }
-    }
-  }
-  return Status::OK();
-}
-
-// Updates target to contain the intersection of the device types in
-// "target" and "other".
-void ColocationGraph::MergeSupportedDevices(
-    PrioritizedDeviceTypeVector* target,
-    const PrioritizedDeviceTypeVector& other) {
-  PrioritizedDeviceTypeVector temp = *target;
-  target->clear();
-
-  // Generate intersection with priorities.
-  PrioritizedDeviceTypeVector target_intersection;
-  PrioritizedDeviceTypeVector other_intersection;
-  for (const auto& prioritized_device_type : temp) {
-    bool found = false;
-    for (const auto& other_prioritized_device_type : other) {
-      if (prioritized_device_type.first ==
-          other_prioritized_device_type.first) {
-        found = true;
-        other_intersection.push_back(other_prioritized_device_type);
-        break;
-      }
-    }
-    if (found) {
-      target_intersection.push_back(prioritized_device_type);
-    }
-  }
-
-  // Sort the devices by priority order.
-  auto device_sort = [](const std::pair<DeviceType, int32>& a,
-                        const std::pair<DeviceType, int32>& b) {
-    // First look at set priorities.
-    if (a.second != b.second) {
-      return a.second > b.second;
-    }
-    // Then fallback to default priorities.
-    auto a_priority = DeviceSet::DeviceTypeOrder(a.first);
-    auto b_priority = DeviceSet::DeviceTypeOrder(b.first);
-    if (a_priority != b_priority) {
-      return a_priority > b_priority;
-    }
-    // Finally just look at the Device type strings.
-    return a.first.type_string() < b.first.type_string();
-  };
-
-  std::sort(target_intersection.begin(), target_intersection.end(),
-            device_sort);
-  std::sort(other_intersection.begin(), other_intersection.end(), device_sort);
-
-  bool is_target_prioritized = HasPriorities(target_intersection);
-  bool is_other_prioritized = HasPriorities(other_intersection);
-  // If neither are prioritized then we just return the original i.e. target
-  // prioritization.
-  if (!is_target_prioritized && !is_other_prioritized) {
-    *target = target_intersection;
-  }
-  // If only one is prioritized, then we respect priorities of that in the
-  // intersection.
-  if (is_target_prioritized && !is_other_prioritized) {
-    *target = target_intersection;
-  }
-  if (!is_target_prioritized && is_other_prioritized) {
-    *target = other_intersection;
-  }
-  // If both have priorities and agree then we go with that. If the
-  // prioritization order is different, then we just fallback to the default
-  // i.e. what the DeviceTypeOrder suggests. In that case, we also set the
-  // merged priorities to 0, so that downstream merges work correctly as well.
-  if (is_target_prioritized && is_other_prioritized) {
-    bool priorities_agree =
-        ArePrioritiesSame(target_intersection, other_intersection);
-    if (priorities_agree) {
-      *target = target_intersection;
-    } else {
-      for (const auto& prioritized_device : target_intersection) {
-        target->push_back(std::make_pair(prioritized_device.first, 0));
-      }
-      std::sort(target->begin(), target->end(), device_sort);
-    }
-  }
-}
-
-}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/colocation_graph.h b/tensorflow/core/common_runtime/colocation_graph.h
deleted file mode 100644
index 8ad54c3232..0000000000
--- a/tensorflow/core/common_runtime/colocation_graph.h
+++ /dev/null
@@ -1,255 +0,0 @@
-/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_COLOCATION_GRAPH_H_
-#define TENSORFLOW_CORE_COMMON_RUNTIME_COLOCATION_GRAPH_H_
-
-#include <unordered_map>
-#include <vector>
-
-#include "absl/strings/str_join.h"
-#include "tensorflow/core/common_runtime/device.h"
-#include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/lib/core/stringpiece.h"
-#include "tensorflow/core/util/device_name_utils.h"
-#include "tensorflow/core/util/port.h"
-
-namespace tensorflow {
-
-// Represents a node in the disjoint node forest and the
-// accumulated constraints on the device used by that node.
-class Member {
- public:
-  Member() = default;
-
-  Status SetParentAndSupportedDevices(const Node& node,
-                                      const std::vector<DeviceType>& types);
-
-  const DeviceNameUtils::ParsedName& requested_device_name() const {
-    return requested_device_name_;
-  }
-
-  Status SetAssignedDeviceName(const string& device_name);
-
-  Status SetRequestedDeviceName(const Node& node);
-
-  Status EnsureCompatibilityAcrossResourceEdge(
-      const Node& src, const Member& src_root,
-      const Node& dst, /*dst_root is this*/
-      bool log_device_placement);
-
-  const PrioritizedDeviceTypeVector& supported_device_types() const {
-    return supported_device_types_;
-  }
-
-  static void Merge(std::vector<Member>* tree, int x_root, int y_root,
-                    Member** new_root, Member** old_root);
-
-  // tree is non-const because we can change some `parent` pointers in some
-  // members for more efficient future lookups. The vector itself is not
-  // changed.
-  static int FindRoot(std::vector<Member>* tree, int node_id);
-
-  Status MergeDeviceNames(const Member& other, bool allow_soft_placement);
-
-  // Updates this to contain the intersection of the device types in
-  // this and "other".
-  void MergeSupportedDevices(const Member& other);
-
-  Status AssignDevice(const Node& node, bool allow_soft_placement);
-
-  void set_possible_devices(std::vector<Device*>&& devices) {
-    possible_devices_ = devices;
-  }
-  const std::vector<Device*>& possible_devices() { return possible_devices_; }
-
-  string DebugString();
-
- private:
-  // The id of the node that is the parent of this one, or its own
-  // id if it is a root. parent <= 0 indicates that this member is invalid.
-  int parent_ = -1;
-
-  // A proxy for the depth of the tree that is used to prefer
-  // connecting smaller trees to larger trees when merging disjoint
-  // sets.
-  int rank_ = 0;
-
-  // Once colocation groups have been formed, the Placer starts actually
-  // choosing devices. All nodes in a group must be assigned to the same
-  // device. Once we assigned the first device to some node in this group,
-  // we set assigned_device_name_index to this device name's index in the
-  // graph.
-  // The `*_device_name_` fields will contain the parsed name of this device
-  // and `possible_devices`, if computed, will contain just this device.
-  // `assigned_device_name_index` is an optimization to avoid parsing and
-  // comparing device names. The value of -1 signals that a single device
-  // has not been chosen yet.
-  int assigned_device_name_index_ = -1;
-
-  // The merged form of the device requested for this node, with those of all of
-  // its children. requested_device_name_ is always kept a specialization (i.e.
-  // DeviceNameUtils::IsSpecialization) of assigned_device_name_. When no device
-  // is requested, this field is set to assigned_device_name_.  As a
-  // specialization of assigned_device_name_, requested_device_name_ represents
-  // the most specific form of all assigned and requested devices of this node
-  // and its children, if this node is a root. requested_device_name_ is used
-  // to finally select devices for nodes.  We can override requested devices due
-  // to resource colocation constraints but not assigned devices (unless soft
-  // placement is on).
-  DeviceNameUtils::ParsedName requested_device_name_;
-
-  // The merged form of the device assigned for this node, with
-  // those of all of its children.
-  // This field is used to raise errors due to unsatisfiable constraints.
-  // Can be a partial specification.
-  // INVARIANT: requested_device_name_ is always a
-  // DeviceNameUtils::IsSpecialization of assigned_device_name_.
-  DeviceNameUtils::ParsedName assigned_device_name_;
-
-  // The intersection of all device types supported by this node,
-  // and those of all of its children, in priority order
-  // of the preferred device.
-  PrioritizedDeviceTypeVector supported_device_types_;
-
-  // If this node is a root, stores a list of Devices to which this node
-  // and all of its children have been assigned, or nullptr if this
-  // has not yet been computed.
-  std::vector<Device*> possible_devices_;
-};  // namespace
-
-// This class maintains the connected components of a colocation
-// constraint graph, and uses this information to assign a satisfying
-// device placement to the nodes of the graph.
-//
-// The typical usage pattern is:
-//
-//   Graph graph = ...;
-//   DeviceSet device_set = ...;
-//   ColocationGraph colocation_graph(graph, device_set);
-//
-//   // Add all the nodes of the `graph` to the `colocation_graph`.
-//   for (Node* node : graph.nodes()) {
-//     TF_RETURN_IF_ERROR(colocation_graph.AddNode(*node));
-//   }
-//
-//   // Add one or more colocation constraints.
-//   Node node_1 = *graph.FindNodeId(...);
-//   Node node_2 = *graph.FindNodeId(...);
-//   TF_RETURN_IF_ERROR(colocation_graph.ColocateNodes(node_1, node_2));
-//
-//   // Assign devices based on the accumulated constraints.
-//   for (Node* node : graph.nodes()) {
-//     TF_RETURN_IF_ERROR(colocation_graph.AssignDevice(node));
-//   }
-//
-// This implementation uses the Union-Find algorithm to efficiently maintain the
-// connected components and incrementally adds edges via
-// ColocationGraph::ColocateNodes() invocations.
-//
-// ColocationGraph does not assign any devices to graph nodes. The
-// `log_device_placement` argument is used to log messages when requested
-// device is ignored.
-class ColocationGraph {
- public:
-  ColocationGraph(const Graph* graph, const DeviceSet* device_set,
-                  const Device* default_device, bool allow_soft_placement,
-                  bool log_device_placement);
-
-  // Adds each node of the Graph to this ColocationGraph as a singleton.
-  //
-  // NOTE: The implementation assumes that the ids of nodes passed to
-  // this method are dense and zero-based; the memory used will be linear in
-  // the largest node ID.
-  // NOTE: If this method returns an error, *this is left in an undefined
-  // state.
-  Status ColocateAllNodes();
-
-  Status ColocateResourceOrRefEdge(Node* src, Node* dst);
-
-  Status ColocateResourceAndRefEdges();
-
-  Status Initialize();
-
-  Status ColocateNodeToGroup(
-      std::unordered_map<StringPiece, const Node*, StringPieceHasher>*
-          colocation_group_root,
-      const Node* node, StringPiece colocation_group);
-
-  // Merge the (possibly disjoint) sets containing nodes "x" and
-  // "y". Returns OK if the all nodes in the union of these sets can
-  // be placed on the same device type.
-  //
-  // NOTE: If this method returns an error, *this is left in an undefined
-  // state.
-  Status ColocateNodes(const Node& x, const Node& y);
-
-  // This overload of ColocateNodes() allows a caller to provide the root node
-  // ids for the two nodes. For large graphs, this noticeably reduces the
-  // graph load time.
-  Status ColocateNodes(const Node& x, int x_root, const Node& y, int y_root);
-
-  // Limits the possible devices of `node`'s colocation group to the device
-  // to which `node` is assigned. This makes sure that all nodes in this
-  // colocation group will be assigned to the same device. Without this
-  // explicit restriction, heuristics can choose a different possible device
-  // for other nodes in the group.
-  Status LimitToAssignedDevice(const Node& node);
-
-  // For the given node, subject to the constraints previously given
-  // to this ColocationGraph, set its assigned_device_name. Returns OK
-  // if a satisfying device can be found, otherwise an error.
-  //
-  // Note: This method returns a pointer to a field within members_.
-  // The caller must not use the returned pointer after there is any possibility
-  // that the members_[i].possible_devices field has been modified.
-  Status GetDevicesForNode(Node* node,
-                           const std::vector<Device*>** possible_devices);
-
-  Status InitializeMembers();
-
-  string DebugString();
-
-  // Returns debugging info for the node referred to by 'node_root'.
-  string DebugInfo(const int node_root);
-
-  Status InitializeMemberWithAssignedDevice(const string& assigned_device_name,
-                                            const string& node_type,
-                                            bool must_be_full_name,
-                                            Member* member);
-
-  Status InitializeMember(const Node& node, Member* member);
-
-  // Updates target to contain the intersection of the device types in
-  // "target" and "other".
-  static void MergeSupportedDevices(PrioritizedDeviceTypeVector* target,
-                                    const PrioritizedDeviceTypeVector& other);
-
-  // Returns the root node of the disjoint tree to which the node with the
-  // given id is connected.
-  int FindRoot(int node_id) { return Member::FindRoot(&members_, node_id); }
-
-  const Graph* const graph_;  // Not owned.
-  std::vector<Member> members_;
-  const DeviceSet* device_set_;  // Not owned.
-  const std::vector<DeviceType> device_types_;
-  const Device* default_device_;
-  const bool allow_soft_placement_;
-  const bool log_device_placement_;
-};
-
-}  // namespace tensorflow
-
-#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_COLOCATION_GRAPH_H_
diff --git a/tensorflow/core/common_runtime/placer.cc b/tensorflow/core/common_runtime/placer.cc
index de2f6e31da..c9827d190d 100644
--- a/tensorflow/core/common_runtime/placer.cc
+++ b/tensorflow/core/common_runtime/placer.cc
@@ -21,7 +21,6 @@ limitations under the License.
 #include <vector>
 
 #include "absl/strings/str_join.h"
-#include "tensorflow/core/common_runtime/colocation_graph.h"
 #include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/framework/attr_value_util.h"
 #include "tensorflow/core/framework/device_attributes.pb.h"
@@ -42,6 +41,84 @@ namespace tensorflow {
 
 namespace {
 
+// We hoist the conversion from C-style string literal to StringPiece here,
+// so that we can avoid the many repeated calls to strlen().
+const StringPiece kColocationAttrNameStringPiece(kColocationAttrName);
+const StringPiece kColocationGroupPrefixStringPiece(kColocationGroupPrefix);
+
+// Returns a list of devices having type in supported_device_types.  The
+// returned list is sorted by preferred type (higher numeric type is preferred).
+std::vector<Device*> FilterSupportedDevices(
+    const std::vector<Device*>& devices,
+    const PrioritizedDeviceTypeVector& supported_device_types,
+    const Device* default_device) {
+  Device* filtered_default_device = nullptr;
+  std::vector<std::pair<Device*, int32>> prioritized_filtered_devices;
+  for (const auto& supported_device_type : supported_device_types) {
+    for (Device* device : devices) {
+      if (DeviceType(device->attributes().device_type()) ==
+          supported_device_type.first) {
+        if (device == default_device) {
+          filtered_default_device = device;
+        } else {
+          prioritized_filtered_devices.emplace_back(
+              device, supported_device_type.second);
+        }
+      }
+    }
+  }
+
+  auto device_sort = [](const std::pair<Device*, int32>& a,
+                        const std::pair<Device*, int32>& b) {
+    if (a.second != b.second) {
+      return a.second > b.second;
+    }
+
+    auto a_priority =
+        DeviceSet::DeviceTypeOrder(DeviceType(a.first->device_type()));
+    auto b_priority =
+        DeviceSet::DeviceTypeOrder(DeviceType(b.first->device_type()));
+    // First sort by prioritized device type (higher is preferred) and
+    // then by device name (lexicographically).
+    if (a_priority != b_priority) {
+      return a_priority > b_priority;
+    }
+    return StringPiece(a.first->name()) < StringPiece(b.first->name());
+  };
+  std::sort(prioritized_filtered_devices.begin(),
+            prioritized_filtered_devices.end(), device_sort);
+
+  std::vector<Device*> filtered_devices;
+  if (filtered_default_device != nullptr) {
+    filtered_devices.emplace_back(filtered_default_device);
+  }
+  for (const auto& prioritized_filtered_device : prioritized_filtered_devices) {
+    filtered_devices.push_back(prioritized_filtered_device.first);
+  }
+  return filtered_devices;
+}
+
+// Using absl::StrJoin with lambda does not work in tf-lite builds.
+std::vector<string> DevicesToString(const std::vector<Device*> devices) {
+  std::vector<string> v;
+  v.reserve(devices.size());
+  for (Device* d : devices) {
+    v.push_back(d->name());
+  }
+  return v;
+}
+
+// Using absl::StrJoin with lambda does not work in tf-lite builds.
+std::vector<string> DeviceTypeAndPriorityToString(
+    const PrioritizedDeviceTypeVector& devices) {
+  std::vector<string> v;
+  v.reserve(devices.size());
+  for (const std::pair<DeviceType, int32>& device_and_type : devices) {
+    v.push_back(DeviceTypeString(device_and_type.first));
+  }
+  return v;
+}
+
 // Returns true if the node has no inputs and produces outputs
 // that are consumed by a single node.
 //
@@ -63,6 +140,1042 @@ bool IsResourceGeneratorNode(const Node& node) {
          (IsRefType(node.output_type(0)) || node.output_type(0) == DT_RESOURCE);
 }
 
+bool IsExemptFromResourceInputColocation(const Node* node) {
+  // Note: Partitioned function calls, which place and partition their
+  // function bodies, are exempt from this check: they forward resource and
+  // ref inputs to operations that are appropriately placed, instead of
+  // dereferencing them.
+  const string& op_type = node->op_def().name();
+  return op_type == "PartitionedCall" || op_type == "StatefulPartitionedCall";
+}
+
+bool HasPriorities(const PrioritizedDeviceTypeVector& device_types) {
+  for (const auto& prioritized_device_type : device_types) {
+    if (prioritized_device_type.second != 0) return true;
+  }
+  return false;
+}
+
+bool ArePrioritiesSame(const PrioritizedDeviceTypeVector& a_types,
+                       const PrioritizedDeviceTypeVector& b_types) {
+  if (a_types.size() != b_types.size()) {
+    return false;
+  }
+  for (int i = 0; i < a_types.size(); ++i) {
+    if (a_types[i].first != b_types[i].first) {
+      return false;
+    }
+  }
+  return true;
+}
+
+// Represents a node in the disjoint node forest and the
+// accumulated constraints on the device used by that node.
+class Member {
+ public:
+  Member() = default;
+
+  Status SetParentAndSupportedDevices(const Node& node,
+                                      const std::vector<DeviceType>& types) {
+    int id = node.id();
+    if (id < 0) {
+      return errors::Internal(
+          "Placer should not be creating a Member for node: ",
+          node.DebugString());
+    }
+    parent_ = id;
+    return SupportedDeviceTypesForNode(types, node.def(),
+                                       &supported_device_types_);
+  }
+
+  const DeviceNameUtils::ParsedName& requested_device_name() const {
+    return requested_device_name_;
+  }
+
+  Status SetAssignedDeviceName(const string& device_name) {
+    if (DeviceNameUtils::HasSomeDetails(requested_device_name_)) {
+      return errors::Internal(
+          "Setting assigned device name when there is a requested device set "
+          "is unsupported");
+    }
+    if (!DeviceNameUtils::ParseFullName(device_name, &assigned_device_name_)) {
+      return errors::Internal("Malformed assigned device '", device_name, "'");
+    }
+    // Set requested device to assigned_device to maintain the invariant that
+    // requested is a specialization of assigned.
+    requested_device_name_ = assigned_device_name_;
+    return Status::OK();
+  }
+
+  Status SetRequestedDeviceName(const Node& node) {
+    if (!DeviceNameUtils::ParseFullName(node.requested_device(),
+                                        &requested_device_name_)) {
+      return errors::InvalidArgument("Malformed device specification '",
+                                     node.requested_device(),
+                                     "' in node: ", node.DebugString());
+    }
+    if (DeviceNameUtils::HasSomeDetails(assigned_device_name_)) {
+      return errors::Internal(
+          "Setting requested device name when there is an assigned device set "
+          "is unsupported");
+    }
+    return Status::OK();
+  }
+
+  Status EnsureCompatibilityAcrossResourceEdge(
+      const Node& src, const Member& src_root,
+      const Node& dst, /*dst_root is this*/
+      bool log_device_placement) {
+    if (!DeviceNameUtils::AreCompatibleDevNames(src_root.assigned_device_name_,
+                                                assigned_device_name_)) {
+      return errors::InvalidArgument(
+          "Cannot place the graph because a reference or resource edge "
+          "connects colocation groups with incompatible assigned devices: ",
+          DeviceNameUtils::ParsedNameToString(src_root.assigned_device_name_),
+          " vs ", DeviceNameUtils::ParsedNameToString(assigned_device_name_));
+    }
+
+    if (DeviceNameUtils::AreCompatibleDevNames(src_root.requested_device_name_,
+                                               requested_device_name_)) {
+      return Status::OK();
+    }
+
+    // If we are here, assigned devices are compatible but requested ones are
+    // not. We will be overriding the requested device for destination node, but
+    // need to preserve the invariant that it will be a specialization of
+    // the assigned device.
+    if (log_device_placement) {
+      LOG(INFO) << "Ignoring device specification "
+                << DeviceNameUtils::ParsedNameToString(requested_device_name_)
+                << " for node '" << dst.name()
+                << "' because the input edge from '" << src.name()
+                << "' is a reference connection and already has a device "
+                   "field set to "
+                << DeviceNameUtils::ParsedNameToString(
+                       src_root.requested_device_name_);
+    }
+    requested_device_name_ = src_root.requested_device_name_;
+    DeviceNameUtils::EnsureSpecification(&requested_device_name_,
+                                         assigned_device_name_);
+    return Status::OK();
+  }
+
+  const PrioritizedDeviceTypeVector& supported_device_types() const {
+    return supported_device_types_;
+  }
+
+  static void Merge(std::vector<Member>* tree, int x_root, int y_root,
+                    Member** new_root, Member** old_root) {
+    Member& x_root_member = (*tree)[x_root];
+    Member& y_root_member = (*tree)[y_root];
+
+    // Merge the sets by setting the parent pointer of the smaller tree's root
+    // node to point to the root of the larger tree. Together with path
+    // compression in ColocationGraph::FindRoot, this ensures that we do not
+    // experience pathological performance on graphs such as chains.
+    int new_root_id, old_root_id;
+    if (x_root_member.rank_ < y_root_member.rank_) {
+      // The tree rooted at x_root is shallower, so connect it to
+      // y_root. The rank of y_root is unchanged because its new
+      // child has strictly less rank.
+      x_root_member.parent_ = y_root;
+      new_root_id = y_root;
+      old_root_id = x_root;
+    } else if (x_root_member.rank_ > y_root_member.rank_) {
+      // The tree rooted at y_root is shallower, so connect it to
+      // x_root. The rank of x_root is unchanged because its new
+      // child has strictly less rank.
+      y_root_member.parent_ = x_root;
+      new_root_id = x_root;
+      old_root_id = y_root;
+    } else {
+      // Both trees have the same rank, so break the tie by choosing
+      // x_root as the new root.
+      y_root_member.parent_ = x_root;
+      // Increment the rank of the tree rooted at x_root, because it
+      // is now strictly deeper than before.
+      ++x_root_member.rank_;
+      new_root_id = x_root;
+      old_root_id = y_root;
+    }
+
+    *new_root = &(*tree)[new_root_id];
+    *old_root = &(*tree)[old_root_id];
+  }
+
+  // tree is non-const because we can change some `parent` pointers in some
+  // members for more efficient future lookups. The vector itself is not
+  // changed.
+  static int FindRoot(std::vector<Member>* tree, int node_id) {
+    Member& member = (*tree)[node_id];
+    if (member.parent_ == node_id) {
+      // member.parent is the root of this disjoint tree.  Do nothing.
+    } else {
+      member.parent_ = FindRoot(tree, member.parent_);
+    }
+    // Now it is guaranteed that member.parent is the root of this disjoint
+    // tree.
+    return member.parent_;
+  }
+
+  Status MergeDeviceNames(const Member& other, bool allow_soft_placement) {
+    // Assuming the "requested is a specialization of assigned" invariant holds
+    // for this and `other`, it will hold after these two merges.
+    TF_RETURN_IF_ERROR(DeviceNameUtils::MergeDevNames(
+        &requested_device_name_, other.requested_device_name_,
+        allow_soft_placement));
+    return DeviceNameUtils::MergeDevNames(&assigned_device_name_,
+                                          other.assigned_device_name_,
+                                          allow_soft_placement);
+  }
+
+  // Updates this to contain the intersection of the device types in
+  // this and "other".
+  void MergeSupportedDevices(const Member& other) {
+    PrioritizedDeviceTypeVector temp = supported_device_types_;
+    supported_device_types_.clear();
+
+    // Generate intersection with priorities.
+    PrioritizedDeviceTypeVector target_intersection;
+    PrioritizedDeviceTypeVector other_intersection;
+    for (const auto& prioritized_device_type : temp) {
+      bool found = false;
+      for (const auto& other_prioritized_device_type :
+           other.supported_device_types_) {
+        if (prioritized_device_type.first ==
+            other_prioritized_device_type.first) {
+          found = true;
+          other_intersection.push_back(other_prioritized_device_type);
+          break;
+        }
+      }
+      if (found) {
+        target_intersection.push_back(prioritized_device_type);
+      }
+    }
+
+    // Sort the devices by priority order.
+    auto device_sort = [](const std::pair<DeviceType, int32>& a,
+                          const std::pair<DeviceType, int32>& b) {
+      // First look at set priorities.
+      if (a.second != b.second) {
+        return a.second > b.second;
+      }
+      // Then fallback to default priorities.
+      auto a_priority = DeviceSet::DeviceTypeOrder(a.first);
+      auto b_priority = DeviceSet::DeviceTypeOrder(b.first);
+      if (a_priority != b_priority) {
+        return a_priority > b_priority;
+      }
+      // Finally just look at the Device type strings.
+      return a.first.type_string() < b.first.type_string();
+    };
+
+    std::sort(target_intersection.begin(), target_intersection.end(),
+              device_sort);
+    std::sort(other_intersection.begin(), other_intersection.end(),
+              device_sort);
+
+    bool is_target_prioritized = HasPriorities(target_intersection);
+    bool is_other_prioritized = HasPriorities(other_intersection);
+    // If neither are prioritized then we just return the original i.e. target
+    // prioritization.
+    if (!is_target_prioritized && !is_other_prioritized) {
+      supported_device_types_ = target_intersection;
+    }
+    // If only one is prioritized, then we respect priorities of that in the
+    // intersection.
+    if (is_target_prioritized && !is_other_prioritized) {
+      supported_device_types_ = target_intersection;
+    }
+    if (!is_target_prioritized && is_other_prioritized) {
+      supported_device_types_ = other_intersection;
+    }
+    // If both have priorities and agree then we go with that. If the
+    // prioritization order is different, then we just fallback to the default
+    // i.e. what the DeviceTypeOrder suggests. In that case, we also set the
+    // merged priorities to 0, so that downstream merges work correctly as well.
+    if (is_target_prioritized && is_other_prioritized) {
+      bool priorities_agree =
+          ArePrioritiesSame(target_intersection, other_intersection);
+      if (priorities_agree) {
+        supported_device_types_ = target_intersection;
+      } else {
+        for (const auto& prioritized_device : target_intersection) {
+          supported_device_types_.push_back(
+              std::make_pair(prioritized_device.first, 0));
+        }
+        std::sort(supported_device_types_.begin(),
+                  supported_device_types_.end(), device_sort);
+      }
+    }
+  }
+
+  Status AssignDevice(const Node& node, bool allow_soft_placement) {
+    if (node.assigned_device_name_index() == assigned_device_name_index_) {
+      return Status::OK();
+    }
+
+    DeviceNameUtils::ParsedName parsed;
+    DeviceNameUtils::ParseFullName(node.assigned_device_name(), &parsed);
+    Status s = DeviceNameUtils::MergeDevNames(&assigned_device_name_, parsed,
+                                              allow_soft_placement);
+    if (!s.ok()) {
+      return errors::Internal(
+          "Constraining by assigned device should not cause an error. Original "
+          "root's assigned device name: ",
+          DeviceNameUtils::ParsedNameToString(assigned_device_name_),
+          " node's assigned device name \"", node.assigned_device_name(),
+          ". Error: ", s.error_message());
+    }
+    s = DeviceNameUtils::MergeDevNames(&requested_device_name_, parsed,
+                                       allow_soft_placement);
+    if (!s.ok()) {
+      return errors::Internal(
+          "Constraining by assigned device should not cause an error. Original "
+          "root's requested device name: \"",
+          DeviceNameUtils::ParsedNameToString(requested_device_name_),
+          "\", node's assigned device name \"", node.assigned_device_name(),
+          "\". Error: ", s.error_message());
+    }
+
+    assigned_device_name_index_ = node.assigned_device_name_index();
+    // Clear cached possible_devices, if any.
+    possible_devices_.clear();
+    return Status::OK();
+  }
+
+  void set_possible_devices(std::vector<Device*>&& devices) {
+    possible_devices_ = devices;
+  }
+  const std::vector<Device*>& possible_devices() { return possible_devices_; }
+
+  string DebugString() {
+    return absl::StrCat(
+        "Member(assigned_device_name_index_=", assigned_device_name_index_,
+        " requested_device_name_=",
+        DeviceNameUtils::ParsedNameToString(requested_device_name_),
+        " assigned_device_name_=",
+        DeviceNameUtils::ParsedNameToString(assigned_device_name_),
+        " supported_device_types_=[",
+        absl::StrJoin(DeviceTypeAndPriorityToString(supported_device_types_),
+                      ", "),
+        "] possible_devices_=[",
+        absl::StrJoin(DevicesToString(possible_devices_), ", "), "]");
+  }
+
+ private:
+  // The id of the node that is the parent of this one, or its own
+  // id if it is a root. parent <= 0 indicates that this member is invalid.
+  int parent_ = -1;
+
+  // A proxy for the depth of the tree that is used to prefer
+  // connecting smaller trees to larger trees when merging disjoint
+  // sets.
+  int rank_ = 0;
+
+  // Once colocation groups have been formed, the Placer starts actually
+  // choosing devices. All nodes in a group must be assigned to the same
+  // device. Once we assigned the first device to some node in this group,
+  // we set assigned_device_name_index to this device name's index in the
+  // graph.
+  // The `*_device_name_` fields will contain the parsed name of this device
+  // and `possible_devices`, if computed, will contain just this device.
+  // `assigned_device_name_index` is an optimization to avoid parsing and
+  // comparing device names. The value of -1 signals that a single device
+  // has not been chosen yet.
+  int assigned_device_name_index_ = -1;
+
+  // The merged form of the device requested for this node, with those of all of
+  // its children. requested_device_name_ is always kept a specialization (i.e.
+  // DeviceNameUtils::IsSpecialization) of assigned_device_name_. When no device
+  // is requested, this field is set to assigned_device_name_.  As a
+  // specialization of assigned_device_name_, requested_device_name_ represents
+  // the most specific form of all assigned and requested devices of this node
+  // and its children, if this node is a root. requested_device_name_ is used
+  // to finally select devices for nodes.  We can override requested devices due
+  // to resource colocation constraints but not assigned devices (unless soft
+  // placement is on).
+  DeviceNameUtils::ParsedName requested_device_name_;
+
+  // The merged form of the device assigned for this node, with
+  // those of all of its children.
+  // This field is used to raise errors due to unsatisfiable constraints.
+  // Can be a partial specification.
+  // INVARIANT: requested_device_name_ is always a
+  // DeviceNameUtils::IsSpecialization of assigned_device_name_.
+  DeviceNameUtils::ParsedName assigned_device_name_;
+
+  // The intersection of all device types supported by this node,
+  // and those of all of its children, in priority order
+  // of the preferred device.
+  PrioritizedDeviceTypeVector supported_device_types_;
+
+  // If this node is a root, stores a list of Devices to which this node
+  // and all of its children have been assigned, or nullptr if this
+  // has not yet been computed.
+  std::vector<Device*> possible_devices_;
+};  // namespace
+
+// This class maintains the connected components of a colocation
+// constraint graph, and uses this information to assign a satisfying
+// device placement to the nodes of the graph.
+//
+// The typical usage pattern is:
+//
+//   Graph graph = ...;
+//   DeviceSet device_set = ...;
+//   ColocationGraph colocation_graph(graph, device_set);
+//
+//   // Add all the nodes of the `graph` to the `colocation_graph`.
+//   for (Node* node : graph.nodes()) {
+//     TF_RETURN_IF_ERROR(colocation_graph.AddNode(*node));
+//   }
+//
+//   // Add one or more colocation constraints.
+//   Node node_1 = *graph.FindNodeId(...);
+//   Node node_2 = *graph.FindNodeId(...);
+//   TF_RETURN_IF_ERROR(colocation_graph.ColocateNodes(node_1, node_2));
+//
+//   // Assign devices based on the accumulated constraints.
+//   for (Node* node : graph.nodes()) {
+//     TF_RETURN_IF_ERROR(colocation_graph.AssignDevice(node));
+//   }
+//
+// This implementation uses the Union-Find algorithm to efficiently maintain the
+// connected components and incrementally adds edges via
+// ColocationGraph::ColocateNodes() invocations.
+//
+// ColocationGraph does not assign any devices to graph nodes. The
+// `log_device_placement` argument is used to log messages when requested
+// device is ignored.
+class ColocationGraph {
+ public:
+  ColocationGraph(const Graph* graph, const DeviceSet* device_set,
+                  const Device* default_device, bool allow_soft_placement,
+                  bool log_device_placement)
+      : graph_(graph),
+        device_set_(device_set),
+        device_types_(device_set->PrioritizedDeviceTypeList()),
+        default_device_(default_device),
+        allow_soft_placement_(allow_soft_placement),
+        log_device_placement_(log_device_placement) {
+    members_.resize(graph->num_node_ids());
+  }
+
+  // Adds each node of the Graph to this ColocationGraph as a singleton.
+  //
+  // NOTE: The implementation assumes that the ids of nodes passed to
+  // this method are dense and zero-based; the memory used will be linear in
+  // the largest node ID.
+  // NOTE: If this method returns an error, *this is left in an undefined
+  // state.
+  Status ColocateAllNodes() {
+    // This maps from a colocation group identifier to the 'root' of that
+    // colocation group.  Note that the keys in this map are StringPiece; the
+    // actual strings are stored under the NodeDef.  The lifetime of this map
+    // is limited to this ColocateAllNodes() method, and no part of the
+    // NodeDef trees are changed during the lifetime of this method, so using
+    // StringPiece as a key is safe.
+    //
+    // Also, as a further optimization, we remove the "loc:@" prefix from
+    // "class" attribute values, when they are used as keys in this table.
+    // This allows us to use StringPiece values that refer to substrings of
+    // 'string' values stored in NodeDef attribute lists, as well as StringPiece
+    // values that refer to 'string' values from NodeDef::name(), without
+    // performing any string allocations.
+    std::unordered_map<StringPiece, const Node*, StringPieceHasher>
+        colocation_group_root;
+
+    for (const Node* node : graph_->op_nodes()) {
+      // When adding the node, identify whether it is part of a colocation
+      // group.
+
+      // This code is effectively the equivalent of GetNodeAttr() for a string
+      // array, but it avoids all internal allocations (the allocation of the
+      // backing store of the std::vector<string> as well as the copies of the
+      // strings within it).  Instead, we combine the query of the colocation
+      // attribute with the calls to ColocateNodeToGroup.
+      bool found_spec = false;
+      const AttrValue* attr_value =
+          node->attrs().Find(kColocationAttrNameStringPiece);
+      if (attr_value != nullptr && attr_value->has_list()) {
+        for (const string& class_spec : attr_value->list().s()) {
+          StringPiece spec(class_spec);
+          if (str_util::ConsumePrefix(&spec,
+                                      kColocationGroupPrefixStringPiece)) {
+            found_spec = true;
+            TF_RETURN_IF_ERROR(
+                ColocateNodeToGroup(&colocation_group_root, node, spec));
+          }
+        }
+      }
+
+      if (!found_spec) {
+        // If the node does not specify a colocation group, then use the
+        // name of this node as the colocation group.
+        TF_RETURN_IF_ERROR(
+            ColocateNodeToGroup(&colocation_group_root, node, node->name()));
+      }
+    }
+
+    return Status::OK();
+  }
+
+  Status ColocateResourceOrRefEdge(Node* src, Node* dst) {
+    // Colocate `src` and `dst` to maintain the invariant that nodes
+    // connected by reference edges are colocated.
+    int src_root_id = FindRoot(src->id());
+    int dst_root_id = FindRoot(dst->id());
+    auto& src_root = members_[src_root_id];
+    auto& dst_root = members_[dst_root_id];
+
+    TF_RETURN_IF_ERROR(dst_root.EnsureCompatibilityAcrossResourceEdge(
+        *src, src_root, *dst, log_device_placement_));
+    Status status = ColocateNodes(*src, src_root_id, *dst, dst_root_id);
+    if (!status.ok()) {
+      return AttachDef(
+          errors::InvalidArgument("Nodes were connected by a "
+                                  "reference connection (requiring them to "
+                                  "be on the same device), but the two nodes "
+                                  "were assigned two different devices: ",
+                                  status.error_message()),
+          *dst);
+    }
+    return Status::OK();
+  }
+
+  Status ColocateResourceAndRefEdges() {
+    // Enumerate the constraint edges, and use them to update the disjoint
+    // node set.
+    // If `node` has an input edge with reference type, add an edge from the
+    // source of that edge to `node`.
+    for (const Edge* edge : graph_->edges()) {
+      if (edge->IsControlEdge()) {
+        continue;
+      }
+      Node* src = edge->src();
+      Node* dst = edge->dst();
+      DataType input_type = dst->input_type(edge->dst_input());
+      if ((input_type == DT_RESOURCE || IsRefType(input_type)) &&
+          !IsExemptFromResourceInputColocation(dst)) {
+        TF_RETURN_IF_ERROR(ColocateResourceOrRefEdge(src, dst));
+      }
+    }
+    return Status::OK();
+  }
+
+  Status Initialize() {
+    TF_RETURN_IF_ERROR(InitializeMembers());
+    TF_RETURN_IF_ERROR(ColocateAllNodes());
+    return ColocateResourceAndRefEdges();
+  }
+
+  Status ColocateNodeToGroup(
+      std::unordered_map<StringPiece, const Node*, StringPieceHasher>*
+          colocation_group_root,
+      const Node* node, StringPiece colocation_group) {
+    const Node*& root_node = (*colocation_group_root)[colocation_group];
+    if (root_node == nullptr) {
+      // This is the first node of the colocation group, so
+      // designate this node as the 'root' of that colocation group.
+      root_node = node;
+    } else {
+      // Try to colocate the node with the root.  If there is an
+      // error, return it.
+      Status s = ColocateNodes(*node, *root_node);
+      if (!s.ok()) {
+        return AttachDef(s, *node);
+      }
+    }
+    return Status::OK();
+  }
+
+  // Merge the (possibly disjoint) sets containing nodes "x" and
+  // "y". Returns OK if the all nodes in the union of these sets can
+  // be placed on the same device type.
+  //
+  // NOTE: If this method returns an error, *this is left in an undefined
+  // state.
+  Status ColocateNodes(const Node& x, const Node& y) {
+    int x_root = FindRoot(x.id());
+    int y_root = FindRoot(y.id());
+    return ColocateNodes(x, x_root, y, y_root);
+  }
+
+  // This overload of ColocateNodes() allows a caller to provide the root node
+  // ids for the two nodes. For large graphs, this noticeably reduces the
+  // graph load time.
+  Status ColocateNodes(const Node& x, int x_root, const Node& y, int y_root) {
+    if (x_root == y_root) {
+      return Status::OK();
+    }
+
+    DCHECK_EQ(x_root, FindRoot(x.id()));
+    DCHECK_EQ(y_root, FindRoot(y.id()));
+
+    Member* new_root_member;
+    Member* old_root_member;
+    Member::Merge(&members_, x_root, y_root, &new_root_member,
+                  &old_root_member);
+
+    // Merge the partial device specifications, and ensure that they are
+    // compatible. NULL options_ is treated as allowing soft placement.
+    // TODO(mrry): Consider enriching the error message by pointing
+    // out which nodes have the explicit partial device
+    // specifications that caused this conflict.
+    Status s = new_root_member->MergeDeviceNames(*old_root_member,
+                                                 allow_soft_placement_);
+    if (!s.ok()) {
+      return errors::InvalidArgument(
+          "Cannot colocate nodes ",
+          errors::FormatColocationNodeForError(x.name()), " and ",
+          errors::FormatColocationNodeForError(y.name()), ": ",
+          s.error_message());
+    }
+
+    // Ensure that the common root has at least one supported device
+    // type, by computing the intersection of
+    // new_root_member.supported_device_types and
+    // old_root_member.supported_device_types.
+    new_root_member->MergeSupportedDevices(*old_root_member);
+    if (new_root_member->supported_device_types().empty()) {
+      return errors::InvalidArgument(
+          "Cannot colocate nodes ",
+          errors::FormatColocationNodeForError(x.name()), " and ",
+          errors::FormatColocationNodeForError(y.name()),
+          " because no device type supports both of those nodes and the "
+          "other nodes colocated with them.",
+          DebugInfo(x_root), DebugInfo(y_root));
+    }
+
+    return Status::OK();
+  }
+
+  // Limits the possible devices of `node`'s colocation group to the device
+  // to which `node` is assigned. This makes sure that all nodes in this
+  // colocation group will be assigned to the same device. Without this
+  // explicit restriction, heuristics can choose a different possible device
+  // for other nodes in the group.
+  Status LimitToAssignedDevice(const Node& node) {
+    if (node.assigned_device_name_index() < 0) {
+      return errors::Internal(
+          "Expected an assigned node as argument to LimitToAssignedDevice but "
+          "got: ",
+          node.DebugString());
+    }
+    int root = FindRoot(node.id());
+    Member& root_member = members_[root];
+    return root_member.AssignDevice(node, allow_soft_placement_);
+  }
+
+  // For the given node, subject to the constraints previously given
+  // to this ColocationGraph, set its assigned_device_name. Returns OK
+  // if a satisfying device can be found, otherwise an error.
+  //
+  // Note: This method returns a pointer to a field within members_.
+  // The caller must not use the returned pointer after there is any possibility
+  // that the members_[i].possible_devices field has been modified.
+  Status GetDevicesForNode(Node* node,
+                           const std::vector<Device*>** possible_devices) {
+    *possible_devices = nullptr;
+    const int node_root = FindRoot(node->id());
+    if (!members_[node_root].possible_devices().empty()) {
+      *possible_devices = &members_[node_root].possible_devices();
+      return Status::OK();
+    }
+
+    // We have not yet computed the possible devices for the
+    // colocated node set containing 'node', so we do so now using the
+    // constraints on the root node.
+
+    // "devices" will contain the set of feasible placements for the
+    // colocated node set containing 'node'.
+    std::vector<Device*> devices;
+    if (DeviceNameUtils::HasSomeDetails(
+            members_[node_root].requested_device_name())) {
+      // The root node has a (possibly partial) device
+      // specification, so enumerate the physical devices that
+      // conform to it.
+      device_set_->FindMatchingDevices(
+          members_[node_root].requested_device_name(), &devices);
+
+      if (!devices.empty()) {
+        // Filter devices into those that are compatible with the root
+        // node (and its children).
+        devices = FilterSupportedDevices(
+            devices, members_[node_root].supported_device_types(),
+            default_device_);
+      }
+
+      // Perform soft placement if allow_soft_placement_ is set.
+      if (devices.empty() && allow_soft_placement_) {
+        // The soft_device_name is the same as the node's device name
+        // without specifying the device type or ID.
+        DeviceNameUtils::ParsedName soft_device_name =
+            members_[node_root].requested_device_name();
+        soft_device_name.type.clear();
+        soft_device_name.has_type = false;
+        soft_device_name.has_id = false;
+        device_set_->FindMatchingDevices(soft_device_name, &devices);
+        if (!devices.empty()) {
+          devices = FilterSupportedDevices(
+              devices, members_[node_root].supported_device_types(),
+              default_device_);
+        }
+      }
+
+      if (devices.empty()) {
+        // Return an error when a physical device that matches an explicit
+        // device specification is not found. This ensures that we don't
+        // assign a node to GPU when the user wanted to force it on CPU.
+        string debug_info = DebugInfo(node_root);
+
+        DeviceNameUtils::ParsedName specified_device_name;
+        if (DeviceNameUtils::ParseFullName(node->requested_device(),
+                                           &specified_device_name) &&
+            specified_device_name ==
+                members_[node_root].requested_device_name()) {
+          // The specified device and merged set device match, and
+          // will appear in the GraphDef (for debugging), so just
+          // print the specified device.
+          std::vector<Device*> devices_matching_nodedef;
+          device_set_->FindMatchingDevices(specified_device_name,
+                                           &devices_matching_nodedef);
+          if (devices_matching_nodedef.empty()) {
+            // Sometimes it is almost impossible to understand the problem
+            // without a list of available devices.
+            std::vector<string> device_names;
+            for (const Device* device : device_set_->devices()) {
+              device_names.push_back(device->name());
+            }
+            std::sort(device_names.begin(), device_names.end());
+
+            string gpu_msg = "";
+            if (!IsGoogleCudaEnabled() &&
+                str_util::Lowercase(specified_device_name.type) == "gpu") {
+              gpu_msg =
+                  " The requested device appears to be a GPU, but CUDA is not "
+                  "enabled.";
+            }
+
+            return errors::InvalidArgument(
+                errors::FormatNodeNameForError(node->name()),
+                "was explicitly assigned to ", node->requested_device(),
+                " but available devices are [ ",
+                str_util::Join(device_names, ", "), " ]. Make sure ",
+                "the device specification refers to a valid device.", gpu_msg);
+          } else if (specified_device_name.has_type) {
+            return errors::InvalidArgument(
+                "Could not satisfy explicit device specification '",
+                node->requested_device(), "' because no supported kernel for ",
+                specified_device_name.type, " devices is available.",
+                debug_info, "\nRegistered kernels:\n",
+                KernelsRegisteredForOp(node->type_string()));
+          } else {
+            return errors::InvalidArgument(
+                "Could not satisfy explicit device specification '",
+                node->requested_device(), debug_info);
+          }
+        } else {
+          // The specified device may be a valid device but the
+          // merged set device is different, so print both.
+          return errors::InvalidArgument(
+              "Could not satisfy explicit device specification '",
+              node->requested_device(), "' because the node ",
+              errors::FormatColocationNodeForError(node->name()),
+              " was colocated with a group of nodes that ",
+              "required incompatible device '",
+              DeviceNameUtils::ParsedNameToString(
+                  members_[node_root].requested_device_name()),
+              "'", debug_info);
+        }
+      }
+    } else {
+      // The device is completely unspecified, so enumerate the devices that
+      // support all of the nodes in the set.
+      if (device_set_->devices().empty()) {
+        return errors::Internal("No devices are registered");
+      }
+      devices = FilterSupportedDevices(
+          device_set_->devices(), members_[node_root].supported_device_types(),
+          default_device_);
+
+      if (devices.empty()) {
+        return errors::InvalidArgument(
+            "Node had no OpKernel registered to support this operation: ",
+            "Operation was ", node->type_string(), " and inputs were ",
+            DataTypeVectorString(node->input_types()), DebugInfo(node_root));
+      }
+    }
+
+    // Cache the result of the possible devices for this node group.
+    members_[node_root].set_possible_devices(std::move(devices));
+    *possible_devices = &members_[node_root].possible_devices();
+    return Status::OK();
+  }
+
+  Status InitializeMembers() {
+    for (Node* node : graph_->op_nodes()) {
+      Status status = InitializeMember(*node, &members_[node->id()]);
+      if (!status.ok()) {
+        return AttachDef(status, *node);
+      }
+    }
+    return Status::OK();
+  }
+
+  string DebugString() {
+    std::unordered_set<int> roots;
+    std::vector<string> root_strings;
+    for (const Node* node : graph_->nodes()) {
+      if (!node->IsOp()) {
+        continue;
+      }
+      int node_root = FindRoot(node->id());
+      if (roots.count(node_root) == 0) {
+        root_strings.push_back(DebugInfo(node_root));
+        roots.insert(node_root);
+      }
+    }
+    return absl::StrJoin(root_strings, "\n");
+  }
+
+  // Returns debugging info for the node referred to by 'node_root'.
+  string DebugInfo(const int node_root) {
+    string text(
+        "\nColocation Debug Info:\n"
+        "Colocation group had the following types and devices: ");
+
+    // If this node is part of a colocation group, then we want to
+    // collect the mapping of ops to supported devices, so that
+    // the user can see why an unsatisfiable placement occurred.
+
+    std::unordered_map<string, string> type_to_devices;
+    std::vector<const Node*> colocation_nodes;
+    int num_nodes_found = 0;
+
+    for (const Node* node : graph_->nodes()) {
+      if (!node->IsOp()) {
+        continue;
+      }
+      int id = node->id();
+      if (FindRoot(id) != node_root) {
+        continue;
+      }
+      ++num_nodes_found;
+      colocation_nodes.push_back(node);
+      const string& op_type = node->type_string();
+      string devices_registered;
+      for (const auto& device_type : members_[id].supported_device_types()) {
+        strings::StrAppend(&devices_registered,
+                           DeviceTypeString(device_type.first), " ");
+      }
+
+      type_to_devices[op_type] = std::move(devices_registered);
+    }
+
+    for (const auto& td : type_to_devices) {
+      strings::StrAppend(&text, "\n", td.first, ": ", td.second);
+    }
+    strings::StrAppend(&text,
+                       "\n\nColocation members and user-requested devices:");
+    for (const Node* node : colocation_nodes) {
+      strings::StrAppend(&text, "\n  ", node->name(), " (", node->type_string(),
+                         ") ", node->requested_device());
+    }
+    strings::StrAppend(&text, "\n");
+
+    if (num_nodes_found <= 0) {
+      text.clear();
+    }
+    return text;
+  }
+
+  Status InitializeMemberWithAssignedDevice(const string& assigned_device_name,
+                                            const string& node_type,
+                                            bool must_be_full_name,
+                                            Member* member) {
+    // This node has already been assigned to a device, so we
+    // respect this placement, after sanity-checking it.
+    // NOTE: Since any assignment must have been performed by
+    // the TensorFlow runtime, we consider errors in this branch to
+    // be INTERNAL.
+    TF_RETURN_IF_ERROR(member->SetAssignedDeviceName(assigned_device_name));
+    if (!must_be_full_name) {
+      return Status::OK();
+    }
+    // Since assigned device must be a full specification, do extra checks.
+    const Device* assigned_device =
+        device_set_->FindDeviceByName(assigned_device_name);
+    if (assigned_device == nullptr) {
+      return errors::Internal("Assigned device '", assigned_device_name,
+                              "' does not match any device");
+    }
+
+    for (const auto& d : member->supported_device_types()) {
+      if (DeviceType(assigned_device->attributes().device_type()) == d.first) {
+        return Status::OK();
+      }
+    }
+
+    return errors::Internal("Assigned device '", assigned_device_name,
+                            "' does not have registered OpKernel support "
+                            "for ",
+                            node_type);
+  }
+
+  Status InitializeMember(const Node& node, Member* member) {
+    TF_RETURN_IF_ERROR(
+        member->SetParentAndSupportedDevices(node, device_types_));
+
+    if (node.has_assigned_device_name()) {
+      TF_RETURN_IF_ERROR(InitializeMemberWithAssignedDevice(
+          node.assigned_device_name(), node.type_string(), true, member));
+    } else {
+      // This node has not yet been assigned to a device, so we
+      // calculate any constraints due to the set of registered
+      // kernels and any (partial) user-provided device specification
+      // in the NodeDef.
+
+      // If no kernels are registered for this op type, fail with an error.
+      if (member->supported_device_types().empty()) {
+        std::set<string> registered_device_types;
+        for (Device* d : device_set_->devices()) {
+          registered_device_types.insert(d->device_type());
+        }
+        std::vector<string> attr_key_vals;
+        for (const auto& it : node.attrs()) {
+          const string& name = it.first;
+          const AttrValue& attr_value = it.second;
+          attr_key_vals.push_back(
+              strings::StrCat(name, "=", SummarizeAttrValue(attr_value)));
+        }
+        return errors::InvalidArgument(
+            "No OpKernel was registered to support Op '", node.type_string(),
+            "' used by ", errors::FormatNodeNameForError(node.name()),
+            "with these attrs: [", str_util::Join(attr_key_vals, ", "),
+            "]\n"
+            "Registered devices: [",
+            str_util::Join(registered_device_types, ", "), "]\n",
+            "Registered kernels:\n",
+            KernelsRegisteredForOp(node.type_string()));
+      }
+
+      // If the NodeDef contains a device, then we interpret it as a
+      // (partial) device specification.
+      if (!node.requested_device().empty()) {
+        if (IsResourceGeneratorNode(node)) {
+          // Treat requested device on resource generating nodes as assigned
+          // device so that we don't override it.
+          TF_RETURN_IF_ERROR(InitializeMemberWithAssignedDevice(
+              node.requested_device(), node.type_string(), false, member));
+        } else {
+          // The user has specified a device in the NodeDef, try to find a
+          // valid device matching their specification in the set of
+          // devices.
+          // NOTE: The full name may specify a device that is not in
+          // n.supported_device_types(), but we check that in AssignDevice().
+          TF_RETURN_IF_ERROR(member->SetRequestedDeviceName(node));
+        }
+      }
+    }
+    return Status::OK();
+  }
+
+  // Updates target to contain the intersection of the device types in
+  // "target" and "other".
+  static void MergeSupportedDevices(PrioritizedDeviceTypeVector* target,
+                                    const PrioritizedDeviceTypeVector& other) {
+    PrioritizedDeviceTypeVector temp = *target;
+    target->clear();
+
+    // Generate intersection with priorities.
+    PrioritizedDeviceTypeVector target_intersection;
+    PrioritizedDeviceTypeVector other_intersection;
+    for (const auto& prioritized_device_type : temp) {
+      bool found = false;
+      for (const auto& other_prioritized_device_type : other) {
+        if (prioritized_device_type.first ==
+            other_prioritized_device_type.first) {
+          found = true;
+          other_intersection.push_back(other_prioritized_device_type);
+          break;
+        }
+      }
+      if (found) {
+        target_intersection.push_back(prioritized_device_type);
+      }
+    }
+
+    // Sort the devices by priority order.
+    auto device_sort = [](const std::pair<DeviceType, int32>& a,
+                          const std::pair<DeviceType, int32>& b) {
+      // First look at set priorities.
+      if (a.second != b.second) {
+        return a.second > b.second;
+      }
+      // Then fallback to default priorities.
+      auto a_priority = DeviceSet::DeviceTypeOrder(a.first);
+      auto b_priority = DeviceSet::DeviceTypeOrder(b.first);
+      if (a_priority != b_priority) {
+        return a_priority > b_priority;
+      }
+      // Finally just look at the Device type strings.
+      return a.first.type_string() < b.first.type_string();
+    };
+
+    std::sort(target_intersection.begin(), target_intersection.end(),
+              device_sort);
+    std::sort(other_intersection.begin(), other_intersection.end(),
+              device_sort);
+
+    bool is_target_prioritized = HasPriorities(target_intersection);
+    bool is_other_prioritized = HasPriorities(other_intersection);
+    // If neither are prioritized then we just return the original i.e. target
+    // prioritization.
+    if (!is_target_prioritized && !is_other_prioritized) {
+      *target = target_intersection;
+    }
+    // If only one is prioritized, then we respect priorities of that in the
+    // intersection.
+    if (is_target_prioritized && !is_other_prioritized) {
+      *target = target_intersection;
+    }
+    if (!is_target_prioritized && is_other_prioritized) {
+      *target = other_intersection;
+    }
+    // If both have priorities and agree then we go with that. If the
+    // prioritization order is different, then we just fallback to the default
+    // i.e. what the DeviceTypeOrder suggests. In that case, we also set the
+    // merged priorities to 0, so that downstream merges work correctly as well.
+    if (is_target_prioritized && is_other_prioritized) {
+      bool priorities_agree =
+          ArePrioritiesSame(target_intersection, other_intersection);
+      if (priorities_agree) {
+        *target = target_intersection;
+      } else {
+        for (const auto& prioritized_device : target_intersection) {
+          target->push_back(std::make_pair(prioritized_device.first, 0));
+        }
+        std::sort(target->begin(), target->end(), device_sort);
+      }
+    }
+  }
+
+  // Returns the root node of the disjoint tree to which the node with the
+  // given id is connected.
+  int FindRoot(int node_id) { return Member::FindRoot(&members_, node_id); }
+
+  const Graph* const graph_;  // Not owned.
+  std::vector<Member> members_;
+  const DeviceSet* device_set_;  // Not owned.
+  const std::vector<DeviceType> device_types_;
+  const Device* default_device_;
+  const bool allow_soft_placement_;
+  const bool log_device_placement_;
+};
+
 void LogDeviceAssignment(const Node* node, bool log_device_placement) {
   // Log placement if log_device_placement is set.
   if (log_device_placement) {
-- 
GitLab


From 66130ea794589efbe0ba70faef2eda6f877218d4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 12 Feb 2019 10:32:56 -0800
Subject: [PATCH 2329/2345] Add saved_model example and test with a convnet for
 MNIST. Towards that, CustomLayer gets support for trainability and custom
 output size.

PiperOrigin-RevId: 233633095
---
 .../saved_model/integration_tests/BUILD       |  30 +++++
 .../integration_tests/export_mnist_cnn.py     | 109 ++++++++++++++++++
 .../integration_tests/mnist_util.py           |  49 ++++++++
 .../integration_tests/saved_model_test.py     |  11 ++
 .../integration_tests/use_mnist_cnn.py        | 101 ++++++++++++++++
 .../use_model_in_sequential_keras.py          |   3 +-
 .../saved_model/integration_tests/util.py     |  36 ++++--
 7 files changed, 330 insertions(+), 9 deletions(-)
 create mode 100644 tensorflow/examples/saved_model/integration_tests/export_mnist_cnn.py
 create mode 100644 tensorflow/examples/saved_model/integration_tests/mnist_util.py
 create mode 100644 tensorflow/examples/saved_model/integration_tests/use_mnist_cnn.py

diff --git a/tensorflow/examples/saved_model/integration_tests/BUILD b/tensorflow/examples/saved_model/integration_tests/BUILD
index 1da0cb3430..d7d8d95f66 100644
--- a/tensorflow/examples/saved_model/integration_tests/BUILD
+++ b/tensorflow/examples/saved_model/integration_tests/BUILD
@@ -61,19 +61,49 @@ py_library(
     ],
 )
 
+py_library(
+    name = "mnist_util",
+    srcs = ["mnist_util.py"],
+    deps = [
+        "//tensorflow:tensorflow_py",
+    ],
+)
+
+py_binary(
+    name = "export_mnist_cnn",
+    srcs = ["export_mnist_cnn.py"],
+    deps = [
+        ":mnist_util",
+        "//tensorflow:tensorflow_py",
+    ],
+)
+
+py_binary(
+    name = "use_mnist_cnn",
+    srcs = ["use_mnist_cnn.py"],
+    deps = [
+        ":mnist_util",
+        ":util",
+        "//tensorflow:tensorflow_py",
+    ],
+)
+
 py_test(
     name = "saved_model_test",
     srcs = [
         "saved_model_test.py",
     ],
     data = [
+        ":export_mnist_cnn",
         ":export_rnn_cell",
         ":export_simple_text_embedding",
         ":export_text_rnn_model",
+        ":use_mnist_cnn",
         ":use_model_in_sequential_keras",
         ":use_rnn_cell",
         ":use_text_rnn_model",
     ],
+    shard_count = 4,
     srcs_version = "PY2AND3",
     tags = [
         "noasan",  # forge input size exceeded
diff --git a/tensorflow/examples/saved_model/integration_tests/export_mnist_cnn.py b/tensorflow/examples/saved_model/integration_tests/export_mnist_cnn.py
new file mode 100644
index 0000000000..ee45646691
--- /dev/null
+++ b/tensorflow/examples/saved_model/integration_tests/export_mnist_cnn.py
@@ -0,0 +1,109 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Exports a convolutional feature extractor for MNIST in SavedModel format.
+
+The feature extractor is a convolutional neural network plus a hidden layer
+that gets trained as part of an MNIST classifier and then written to a
+SavedModel (without the classification layer). From there, use_mnist_cnn.py
+picks it up for transfer learning.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl import app
+from absl import flags
+import tensorflow as tf
+
+from tensorflow.examples.saved_model.integration_tests import mnist_util
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_string(
+    'export_dir', None,
+    'Directory of exported SavedModel.')
+flags.DEFINE_integer(
+    'epochs', 10,
+    'Number of epochs to train.')
+flags.DEFINE_bool(
+    'fast_test_mode', False,
+    'Shortcut training for running in unit tests.')
+
+
+def make_feature_extractor():
+  """Returns a Keras Model to compute a feature vector from MNIST images."""
+  net = inp = tf.keras.Input(mnist_util.INPUT_SHAPE)
+  net = tf.keras.layers.Conv2D(32, (3, 3), activation='relu')(net)
+  net = tf.keras.layers.Conv2D(64, (3, 3), activation='relu')(net)
+  net = tf.keras.layers.MaxPooling2D(pool_size=(2, 2))(net)
+  # TODO(arnoegw): net = tf.keras.layers.Dropout(dropout_rate)(net)
+  net = tf.keras.layers.Flatten()(net)
+  # TODO(arnoegw): Add weight regularization.
+  net = tf.keras.layers.Dense(128, activation='relu')(net)
+  return tf.keras.Model(inputs=inp, outputs=net)
+
+
+def make_classifier(feature_extractor, dropout_rate=0.5):
+  """Returns a Keras Model to classify MNIST using feature_extractor."""
+  net = inp = tf.keras.Input(mnist_util.INPUT_SHAPE)
+  net = feature_extractor(net)
+  net = tf.keras.layers.Dropout(dropout_rate)(net)
+  net = tf.keras.layers.Dense(mnist_util.NUM_CLASSES, activation='softmax')(net)
+  return tf.keras.Model(inputs=inp, outputs=net)
+
+
+def wrap_keras_model_for_export(model):
+  """Wraps `model` for saving and loading as SavedModel."""
+  obj = tf.train.Checkpoint()
+  obj.__call__ = tf.function(
+      # TODO(arnoegw): Add args for training, hparams.
+      lambda inputs: model(inputs),  # pylint: disable=unnecessary-lambda
+      input_signature=[tf.TensorSpec(shape=(None,) + mnist_util.INPUT_SHAPE,
+                                     dtype=tf.float32)])
+  obj.trainable_variables = model.trainable_variables
+  obj.variables = model.trainable_variables + model.non_trainable_variables
+  return obj
+
+
+def main(argv):
+  del argv
+
+  # Build a complete classifier model using a feature extractor.
+  feature_extractor = make_feature_extractor()
+  classifier = make_classifier(feature_extractor)
+
+  # Train the complete model
+  (x_train, y_train), (x_test, y_test) = mnist_util.load_reshaped_data(
+      fake_tiny_data=FLAGS.fast_test_mode)
+  classifier.compile(loss=tf.keras.losses.categorical_crossentropy,
+                     optimizer=tf.keras.optimizers.SGD(),
+                     metrics=['accuracy'])
+  classifier.fit(x_train, y_train,
+                 batch_size=128,
+                 epochs=FLAGS.epochs,
+                 verbose=1,
+                 validation_data=(x_test, y_test))
+
+  # Save the feature extractor to a framework-agnostic SavedModel for reuse.
+  # Note that the feature_extractor object has not been compiled or fitted,
+  # so it does not contain an optimizer and related state.
+  exportable = wrap_keras_model_for_export(feature_extractor)
+  tf.saved_model.save(exportable, FLAGS.export_dir)
+
+
+if __name__ == '__main__':
+  # tf.enable_v2_behavior()
+  app.run(main)
diff --git a/tensorflow/examples/saved_model/integration_tests/mnist_util.py b/tensorflow/examples/saved_model/integration_tests/mnist_util.py
new file mode 100644
index 0000000000..7e4ffda19e
--- /dev/null
+++ b/tensorflow/examples/saved_model/integration_tests/mnist_util.py
@@ -0,0 +1,49 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Convenience wrapper around Keras' MNIST and Fashion MNIST data."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+import tensorflow as tf
+
+INPUT_SHAPE = (28, 28, 1)
+NUM_CLASSES = 10
+
+
+def load_reshaped_data(use_fashion_mnist=False, fake_tiny_data=False):
+  """Returns MNIST or Fashion MNIST train and test data."""
+  if fake_tiny_data:
+    num_fakes = 10
+    x_train = x_test = np.zeros((num_fakes, 28, 28), dtype=np.uint8)
+    y_train = y_test = np.zeros((num_fakes,), dtype=np.int64)
+  else:
+    mnist = (tf.keras.datasets.fashion_mnist if use_fashion_mnist else
+             tf.keras.datasets.mnist)
+    (x_train, y_train), (x_test, y_test) = mnist.load_data()
+  return ((_prepare_image(x_train), _prepare_label(y_train)),
+          (_prepare_image(x_test), _prepare_label(y_test)))
+
+
+def _prepare_image(x):
+  """Converts images to [n,h,w,c] format in range [0,1]."""
+  return x[..., None].astype('float32') / 255.
+
+
+def _prepare_label(y):
+  """Conerts labels to one-hot encoding."""
+  return tf.keras.utils.to_categorical(y, NUM_CLASSES)
diff --git a/tensorflow/examples/saved_model/integration_tests/saved_model_test.py b/tensorflow/examples/saved_model/integration_tests/saved_model_test.py
index 4b48bcbb22..78d2d1e2ad 100644
--- a/tensorflow/examples/saved_model/integration_tests/saved_model_test.py
+++ b/tensorflow/examples/saved_model/integration_tests/saved_model_test.py
@@ -70,6 +70,17 @@ class SavedModelTest(tf.test.TestCase):
         "use_model_in_sequential_keras")
     self.assertCommandSucceeded(use_binary, model_dir=export_dir)
 
+  @test_util.run_v2_only
+  def test_mnist_cnn(self):
+    export_dir = self.get_temp_dir()
+    export_binary = resource_loader.get_path_to_datafile("export_mnist_cnn")
+    self.assertCommandSucceeded(export_binary, export_dir=export_dir,
+                                fast_test_mode="true")
+
+    use_binary = resource_loader.get_path_to_datafile("use_mnist_cnn")
+    self.assertCommandSucceeded(use_binary, export_dir=export_dir,
+                                fast_test_mode="true")
+
 if __name__ == "__main__":
   # tf.enable_v2_behavior()
   tf.test.main()
diff --git a/tensorflow/examples/saved_model/integration_tests/use_mnist_cnn.py b/tensorflow/examples/saved_model/integration_tests/use_mnist_cnn.py
new file mode 100644
index 0000000000..4e49f3a1e7
--- /dev/null
+++ b/tensorflow/examples/saved_model/integration_tests/use_mnist_cnn.py
@@ -0,0 +1,101 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Imports a convolutional feature extractor for MNIST in SavedModel format.
+
+This program picks up the SavedModel written by export_mnist_cnn.py and
+uses the feature extractor contained in it to classification on either
+classic MNIST (digits) or Fashion MNIST (thumbnails of apparel). Optionally,
+it trains the feature extractor further as part of the new classifier.
+As expected, that makes training slower but does not help much for the
+original training dataset but helps a lot for transfer to the other dataset.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl import app
+from absl import flags
+import tensorflow as tf
+
+from tensorflow.examples.saved_model.integration_tests import mnist_util
+from tensorflow.examples.saved_model.integration_tests import util
+from tensorflow.python.saved_model import load as svmd_load
+tf.saved_model.load = svmd_load.load
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_string(
+    'export_dir', None,
+    'Directory of exported SavedModel.')
+flags.DEFINE_integer(
+    'epochs', 5,
+    'Number of epochs to train.')
+flags.DEFINE_bool(
+    'retrain', False,
+    'If set, the imported SavedModel is trained further.')
+flags.DEFINE_bool(
+    'use_fashion_mnist', False,
+    'Use Fashion MNIST (products) instead of the real MNIST (digits). '
+    'With this, --retrain gains a lot.')
+flags.DEFINE_bool(
+    'fast_test_mode', False,
+    'Shortcut training for running in unit tests.')
+
+
+def make_classifier(feature_extractor, dropout_rate=0.5):
+  """Returns a Keras Model to classify MNIST using feature_extractor."""
+  net = inp = tf.keras.Input(mnist_util.INPUT_SHAPE)
+  net = feature_extractor(net)
+  net = tf.keras.layers.Dropout(dropout_rate)(net)
+  net = tf.keras.layers.Dense(mnist_util.NUM_CLASSES,
+                              activation='softmax')(net)
+  return tf.keras.Model(inputs=inp, outputs=net)
+
+
+def main(argv):
+  del argv
+
+  # Load a pre-trained feature extractor and wrap it for use in Keras.
+  obj = tf.saved_model.load(FLAGS.export_dir)
+  feature_extractor = util.CustomLayer(obj, output_shape=[128],
+                                       trainable=FLAGS.retrain)
+
+  # Build a classifier with it.
+  model = make_classifier(feature_extractor)
+
+  # Train the classifier (possibly on a different dataset).
+  (x_train, y_train), (x_test, y_test) = mnist_util.load_reshaped_data(
+      use_fashion_mnist=FLAGS.use_fashion_mnist,
+      fake_tiny_data=FLAGS.fast_test_mode)
+  model.compile(loss=tf.keras.losses.categorical_crossentropy,
+                optimizer=tf.keras.optimizers.SGD(),
+                metrics=['accuracy'],
+                # TODO(arnoegw): Remove after investigating huge allocs.
+                run_eagerly=True)
+  print('Training on %s with %d trainable and %d untrainable variables.' %
+        ('Fashion MNIST' if FLAGS.use_fashion_mnist else 'MNIST',
+         len(model.trainable_variables), len(model.non_trainable_variables)))
+  model.fit(x_train, y_train,
+            batch_size=128,
+            epochs=FLAGS.epochs,
+            steps_per_epoch=3,
+            verbose=1,
+            validation_data=(x_test, y_test))
+
+
+if __name__ == '__main__':
+  # tf.enable_v2_behavior()
+  app.run(main)
diff --git a/tensorflow/examples/saved_model/integration_tests/use_model_in_sequential_keras.py b/tensorflow/examples/saved_model/integration_tests/use_model_in_sequential_keras.py
index 370d157b98..cf9397f002 100644
--- a/tensorflow/examples/saved_model/integration_tests/use_model_in_sequential_keras.py
+++ b/tensorflow/examples/saved_model/integration_tests/use_model_in_sequential_keras.py
@@ -47,7 +47,8 @@ def main(argv):
 
   # Create the sequential keras model.
   model = tf.keras.Sequential()
-  model.add(util.CustomLayer(embed, batch_input_shape=[None], dtype=tf.string))
+  model.add(util.CustomLayer(embed, batch_input_shape=[None],
+                             output_shape=[10], dtype=tf.string))
   model.add(tf.keras.layers.Dense(100, activation="relu"))
   model.add(tf.keras.layers.Dense(50, activation="relu"))
   model.add(tf.keras.layers.Dense(1, activation="sigmoid"))
diff --git a/tensorflow/examples/saved_model/integration_tests/util.py b/tensorflow/examples/saved_model/integration_tests/util.py
index 5ea9e91ff5..0fc0cc86bb 100644
--- a/tensorflow/examples/saved_model/integration_tests/util.py
+++ b/tensorflow/examples/saved_model/integration_tests/util.py
@@ -24,11 +24,34 @@ import tensorflow as tf
 # TODO(vbardiovsky): We should just reuse Keras's Lambda layer, when that
 # enables to get trainable variables.
 class CustomLayer(tf.keras.layers.Layer):
-  """Wraps callable object as a `Layer` object."""
-
-  def __init__(self, func, **kwargs):
+  """Wraps callable object as a `Layer` object.
+
+  Args:
+    func: The callable object to wrap.
+    output_shape: A tuple with the (possibly partial) output shape of `func`
+      *without* leading batch size (by analogy to Dense(..., input_shape=...)).
+    trainable: Boolean controlling whether the trainable variables of `func`
+      are reported as trainable variables of this layer.
+  """
+
+  def __init__(self, func, output_shape, trainable=False, **kwargs):
+    # Set self._{non,}_trainable_weights before calling Layer.__init__.
+    if hasattr(func, 'trainable_variables'):
+      self._trainable_weights = [v for v in func.trainable_variables]
+      trainable_variables_set = set(func.trainable_variables)
+    else:
+      self._trainable_weights = []
+      trainable_variables_set = set()
+    if hasattr(func, 'variables'):
+      self._non_trainable_weights = [v for v in func.variables
+                                     if v not in trainable_variables_set]
+    else:
+      self._non_trainable_weights = []  # TODO(arnoegw): Infer from `func`.
+    super(CustomLayer, self).__init__(trainable=trainable, **kwargs)
     self._func = func
-    super(CustomLayer, self).__init__(**kwargs)
+    # TODO(vbardiovsky): We should be able to get the embedding dimension from
+    # the restored model.
+    self._output_shape = tuple(output_shape)
 
   def call(self, x):
     result = self._func(x)
@@ -37,7 +60,4 @@ class CustomLayer(tf.keras.layers.Layer):
     return result
 
   def compute_output_shape(self, input_shape):
-    # TODO(vbardiovsky): We should be able to get the embedding dimension from
-    # the restored model.
-    return (input_shape[0], 10)
-
+    return (input_shape[0],) + self._output_shape
-- 
GitLab


From 24e639413dc8eaa7182e828cebf4abad435bfe84 Mon Sep 17 00:00:00 2001
From: Tong Shen <endlessroad@google.com>
Date: Tue, 12 Feb 2019 10:33:47 -0800
Subject: [PATCH 2330/2345] Fix src_output for Conditional's predicate input.

PiperOrigin-RevId: 233633311
---
 tensorflow/compiler/tf2xla/functionalize_cond.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/functionalize_cond.cc b/tensorflow/compiler/tf2xla/functionalize_cond.cc
index 0e75d38d66..8aa162be47 100644
--- a/tensorflow/compiler/tf2xla/functionalize_cond.cc
+++ b/tensorflow/compiler/tf2xla/functionalize_cond.cc
@@ -761,9 +761,9 @@ Status Conditional::BuildIfNode(Graph* graph,
   }
   builder.Device(predicate_.node->assigned_device_name());
   // Conditional should be the first input ...
-  builder.Input(NodeDefBuilder::NodeOut(predicate_.node->name(),
-                                        predicate_.index,
-                                        predicate_.node->output_type(0)));
+  builder.Input(
+      NodeDefBuilder::NodeOut(predicate_.node->name(), predicate_.index,
+                              predicate_.node->output_type(predicate_.index)));
   // ... followed by the other inputs.
   builder.Input(inputs);
 
-- 
GitLab


From 7141c4280873036fbdcccacf696c0fe12b5538c7 Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Tue, 12 Feb 2019 10:34:06 -0800
Subject: [PATCH 2331/2345] [Grappler] Don't remove constant feed nodes in
 LoopOptimizer RemoveDeadBranches.

PiperOrigin-RevId: 233633382
---
 .../grappler/optimizers/loop_optimizer.cc     |  36 +-
 .../core/grappler/optimizers/loop_optimizer.h |   4 +-
 .../optimizers/loop_optimizer_test.cc         | 309 ++++++++++++++----
 3 files changed, 280 insertions(+), 69 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/loop_optimizer.cc b/tensorflow/core/grappler/optimizers/loop_optimizer.cc
index cf5e4db29f..54776e7f80 100644
--- a/tensorflow/core/grappler/optimizers/loop_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/loop_optimizer.cc
@@ -581,8 +581,19 @@ Status EvaluateBoolOpForConstantOperands(const NodeDef& op_node,
   return Status::OK();
 }
 
+// TODO(lyandy): Consolidate with ConstantFolding implementation.
+bool IsReallyConstant(const NodeDef& node,
+                      const absl::flat_hash_set<string>& feed_nodes) {
+  if (!IsConstant(node)) {
+    return false;
+  }
+  // If the node is fed it's not constant anymore.
+  return feed_nodes.find(node.name()) == feed_nodes.end();
+}
+
 Status CheckForDeadFanout(const MutableGraphView& view,
                           const NodeDef& switch_node, const NodeMap& node_map,
+                          const absl::flat_hash_set<string>& feed_nodes,
                           DeviceBase* cpu_device, ResourceMgr* resource_mgr,
                           bool* has_dead_fanout, int* dead_fanout) {
   *has_dead_fanout = false;
@@ -591,7 +602,7 @@ Status CheckForDeadFanout(const MutableGraphView& view,
       view.GetRegularFanin(switch_loopcond_port).node;
 
   // CASE 1: Control is a constant.
-  if (IsConstant(*switch_predicate)) {
+  if (IsReallyConstant(*switch_predicate, feed_nodes)) {
     Tensor selector;
     CHECK(selector.FromProto(switch_predicate->attr().at("value").tensor()));
     *has_dead_fanout = true;
@@ -630,7 +641,7 @@ Status CheckForDeadFanout(const MutableGraphView& view,
     if (IsMerge(*node)) {
       merge_node = node;
     }
-    if (IsConstant(*node)) {
+    if (IsReallyConstant(*node, feed_nodes)) {
       constant_ctrl_input = node;
       constant_index = i;
     }
@@ -646,7 +657,7 @@ Status CheckForDeadFanout(const MutableGraphView& view,
     if (IsEnter(*node)) {
       enter_node = node;
     }
-    if (IsConstant(*node)) {
+    if (IsReallyConstant(*node, feed_nodes)) {
       constant_init_node = node;
     }
   }
@@ -654,7 +665,7 @@ Status CheckForDeadFanout(const MutableGraphView& view,
     if (constant_init_node != nullptr) return Status::OK();
     for (const auto& input : enter_node->input()) {
       NodeDef* node = node_map.GetNode(input);
-      if (IsConstant(*node)) {
+      if (IsReallyConstant(*node, feed_nodes)) {
         constant_init_node = node;
       }
     }
@@ -710,8 +721,12 @@ Status LoopOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
     // TODO(srjoglekar): Figure out if we can optimize NodeMap creations across
     // optimizer passes.
     NodeMap node_map(optimized_graph);
-    TF_RETURN_IF_ERROR(
-        RemoveDeadBranches(item.NodesToPreserve(), node_map, optimized_graph));
+    absl::flat_hash_set<string> feed_nodes;
+    for (const auto& feed : item.feed) {
+      feed_nodes.insert(NodeName(feed.first));
+    }
+    TF_RETURN_IF_ERROR(RemoveDeadBranches(item.NodesToPreserve(), node_map,
+                                          feed_nodes, optimized_graph));
   }
 
   return Status::OK();
@@ -719,7 +734,8 @@ Status LoopOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
 
 Status LoopOptimizer::RemoveDeadBranches(
     const std::unordered_set<string>& nodes_to_preserve,
-    const NodeMap& node_map, GraphDef* optimized_graph) {
+    const NodeMap& node_map, const absl::flat_hash_set<string>& feed_nodes,
+    GraphDef* optimized_graph) {
   std::unordered_set<const NodeDef*> dead_nodes;
   std::unordered_map<NodeDef*, std::set<int>> dead_merge_inputs;
   // TODO(bsteiner): also rewrite switches as identity. For now we just record
@@ -737,9 +753,9 @@ Status LoopOptimizer::RemoveDeadBranches(
 
     int dead_fanout;
     bool has_dead_fanout;
-    TF_RETURN_IF_ERROR(CheckForDeadFanout(view, node, node_map, cpu_device_,
-                                          resource_mgr_.get(), &has_dead_fanout,
-                                          &dead_fanout));
+    TF_RETURN_IF_ERROR(CheckForDeadFanout(view, node, node_map, feed_nodes,
+                                          cpu_device_, resource_mgr_.get(),
+                                          &has_dead_fanout, &dead_fanout));
     if (!has_dead_fanout) {
       continue;
     }
diff --git a/tensorflow/core/grappler/optimizers/loop_optimizer.h b/tensorflow/core/grappler/optimizers/loop_optimizer.h
index d467237a9a..7fa1976f34 100644
--- a/tensorflow/core/grappler/optimizers/loop_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/loop_optimizer.h
@@ -60,7 +60,9 @@ class LoopOptimizer : public GraphOptimizer {
   };
 
   Status RemoveDeadBranches(const std::unordered_set<string>& nodes_to_preserve,
-                            const NodeMap& node_map, GraphDef* optimized_graph);
+                            const NodeMap& node_map,
+                            const absl::flat_hash_set<string>& feed_nodes,
+                            GraphDef* optimized_graph);
 
   RewriterConfig::Toggle opt_level_;
   DeviceBase* cpu_device_;
diff --git a/tensorflow/core/grappler/optimizers/loop_optimizer_test.cc b/tensorflow/core/grappler/optimizers/loop_optimizer_test.cc
index 587767c23c..9a6d6272df 100644
--- a/tensorflow/core/grappler/optimizers/loop_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/loop_optimizer_test.cc
@@ -504,11 +504,11 @@ void VerifyGraphsEqual(const GraphDef& original_graph,
   for (int i = 0; i < original_graph.node_size(); ++i) {
     const NodeDef& original = original_graph.node(i);
     const NodeDef& optimized = optimized_graph.node(i);
-    EXPECT_EQ(original.name(), optimized.name()) << func;
-    EXPECT_EQ(original.op(), optimized.op()) << func;
-    EXPECT_EQ(original.input_size(), optimized.input_size()) << func;
+    EXPECT_EQ(optimized.name(), original.name()) << func;
+    EXPECT_EQ(optimized.op(), original.op()) << func;
+    ASSERT_EQ(optimized.input_size(), original.input_size()) << func;
     for (int j = 0; j < original.input_size(); ++j) {
-      EXPECT_EQ(original.input(j), optimized.input(j)) << func;
+      EXPECT_EQ(optimized.input(j), original.input(j)) << func;
     }
   }
 }
@@ -528,7 +528,7 @@ TEST_F(LoopOptimizerTest, NoOp) {
   VerifyGraphsEqual(item.graph, output, __FUNCTION__);
 }
 
-TEST_F(LoopOptimizerTest, RemovePush_NoOp) {
+TEST_F(LoopOptimizerTest, RemovePushNoOp) {
   GrapplerItem item;
   GraphDef& graph = item.graph;
   AddSimpleNode("c", "Const", {}, &graph);
@@ -557,7 +557,7 @@ TEST_F(LoopOptimizerTest, RemovePush_NoOp) {
   VerifyGraphsEqual(item.graph, output, __FUNCTION__);
 }
 
-TEST_F(LoopOptimizerTest, RemovePush_NoPopButStackLives) {
+TEST_F(LoopOptimizerTest, RemovePushNoPopButStackLives) {
   GrapplerItem item;
   GraphDef& graph = item.graph;
   AddSimpleNode("c", "Const", {}, &graph);
@@ -609,32 +609,32 @@ TEST_F(LoopOptimizerTest, RemovePushWithoutMatchingPop) {
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
-  EXPECT_EQ(13, output.node_size());
+  EXPECT_EQ(output.node_size(), 13);
   for (int i = 0; i < output.node_size(); ++i) {
     const NodeDef& node = output.node(i);
     if (node.name() == "push1") {
-      EXPECT_EQ("Identity", node.op());
-      EXPECT_EQ(2, node.input_size());
-      EXPECT_EQ("c", node.input(0));
-      EXPECT_EQ("^stack1", node.input(1));
+      EXPECT_EQ(node.op(), "Identity");
+      ASSERT_EQ(node.input_size(), 2);
+      EXPECT_EQ(node.input(0), "c");
+      EXPECT_EQ(node.input(1), "^stack1");
     } else if (node.name() == "push2") {
-      EXPECT_EQ("Identity", node.op());
-      EXPECT_EQ(2, node.input_size());
-      EXPECT_EQ("enter_c", node.input(0));
-      EXPECT_EQ("^enter_stack2", node.input(1));
+      EXPECT_EQ(node.op(), "Identity");
+      ASSERT_EQ(node.input_size(), 2);
+      EXPECT_EQ(node.input(0), "enter_c");
+      EXPECT_EQ(node.input(1), "^enter_stack2");
     } else if (node.name() == "push3") {
-      EXPECT_EQ("Identity", node.op());
-      EXPECT_EQ(2, node.input_size());
-      EXPECT_EQ("c", node.input(0));
-      EXPECT_EQ("^stack3", node.input(1));
+      EXPECT_EQ(node.op(), "Identity");
+      ASSERT_EQ(node.input_size(), 2);
+      EXPECT_EQ(node.input(0), "c");
+      EXPECT_EQ(node.input(1), "^stack3");
     } else {
       const NodeDef& orig_node = item.graph.node(i);
-      EXPECT_EQ(orig_node.ShortDebugString(), node.ShortDebugString());
+      EXPECT_EQ(node.ShortDebugString(), orig_node.ShortDebugString());
     }
   }
 }
 
-TEST_F(LoopOptimizerTest, RemoveDeadBranches_ConstantCondition) {
+TEST_F(LoopOptimizerTest, RemoveDeadBranchesConstantCondition) {
   Scope scope = Scope::NewRootScope();
   Output v_in = ops::Variable(scope.WithOpName("v_in"), {3}, DT_FLOAT);
 
@@ -691,57 +691,57 @@ TEST_F(LoopOptimizerTest, RemoveDeadBranches_ConstantCondition) {
 
   for (const NodeDef& node : output.node()) {
     // These nodes should have been pruned
-    EXPECT_NE("Square1", node.name());
-    EXPECT_NE("Sqrt2", node.name());
-    EXPECT_NE("m5", node.name());
-    EXPECT_NE("m7", node.name());
+    EXPECT_NE(node.name(), "Square1");
+    EXPECT_NE(node.name(), "Sqrt2");
+    EXPECT_NE(node.name(), "m5");
+    EXPECT_NE(node.name(), "m7");
 
     if (node.name() == "m1") {
       // sqrt1 is dead
-      EXPECT_EQ("Identity", node.op());
-      EXPECT_EQ(1, node.input_size());
-      EXPECT_EQ("square1", node.input(0));
+      EXPECT_EQ(node.op(), "Identity");
+      ASSERT_EQ(node.input_size(), 1);
+      EXPECT_EQ(node.input(0), "square1");
     } else if (node.name() == "m2") {
       // both inputs are alive
-      EXPECT_EQ("Merge", node.op());
-      EXPECT_EQ(2, node.input_size());
-      EXPECT_EQ("v_in", node.input(0));
-      EXPECT_EQ("square1", node.input(1));
+      EXPECT_EQ(node.op(), "Merge");
+      ASSERT_EQ(node.input_size(), 2);
+      EXPECT_EQ(node.input(0), "v_in");
+      EXPECT_EQ(node.input(1), "square1");
     } else if (node.name() == "m3") {
       // sqrt1 is dead
-      EXPECT_EQ("Identity", node.op());
-      EXPECT_EQ(1, node.input_size());
-      EXPECT_EQ("v_in", node.input(0));
+      EXPECT_EQ(node.op(), "Identity");
+      ASSERT_EQ(node.input_size(), 1);
+      EXPECT_EQ(node.input(0), "v_in");
     } else if (node.name() == "m4") {
       // both inputs are alive
-      EXPECT_EQ("Merge", node.op());
-      EXPECT_EQ(2, node.input_size());
-      EXPECT_EQ("square1", node.input(0));
-      EXPECT_EQ("sqrt2", node.input(1));
+      EXPECT_EQ(node.op(), "Merge");
+      ASSERT_EQ(node.input_size(), 2);
+      EXPECT_EQ(node.input(0), "square1");
+      EXPECT_EQ(node.input(1), "sqrt2");
     } else if (node.name() == "m6") {
       // both inputs are alive and the control dependency can get triggered
-      EXPECT_EQ("Merge", node.op());
-      EXPECT_EQ(3, node.input_size());
-      EXPECT_EQ("v_in", node.input(0));
-      EXPECT_EQ("square1", node.input(1));
-      EXPECT_EQ("^sqrt2", node.input(2));
+      EXPECT_EQ(node.op(), "Merge");
+      ASSERT_EQ(node.input_size(), 3);
+      EXPECT_EQ(node.input(0), "v_in");
+      EXPECT_EQ(node.input(1), "square1");
+      EXPECT_EQ(node.input(2), "^sqrt2");
     } else if (node.name() == "m8") {
       // The node is to be preserved because of a fetch
-      EXPECT_EQ("Merge", node.op());
-      EXPECT_EQ(2, node.input_size());
-      EXPECT_EQ("id1", node.input(0));
-      EXPECT_EQ("id2", node.input(1));
+      EXPECT_EQ(node.op(), "Merge");
+      ASSERT_EQ(node.input_size(), 2);
+      EXPECT_EQ(node.input(0), "id1");
+      EXPECT_EQ(node.input(1), "id2");
     } else if (node.name() == "m9") {
       // The node is to be preserved because of a fetch
-      EXPECT_EQ("Merge", node.op());
-      EXPECT_EQ(2, node.input_size());
-      EXPECT_EQ("id3", node.input(0));
-      EXPECT_EQ("id4", node.input(1));
+      EXPECT_EQ(node.op(), "Merge");
+      ASSERT_EQ(2, node.input_size());
+      EXPECT_EQ(node.input(0), "id3");
+      EXPECT_EQ(node.input(1), "id4");
     }
   }
 }
 
-TEST_F(LoopOptimizerTest, RemoveDeadBranches_FullyRemoveDeadBranches) {
+TEST_F(LoopOptimizerTest, RemoveDeadBranchesFullyRemoveDeadBranches) {
   const string gdef_ascii = R"EOF(
 node {
   name: "episodicreplaybuffer_add_readvariableop_resource"
@@ -1153,7 +1153,7 @@ versions {
       << "Merge node was deleted, but it shouldn't have been.";
 }
 
-TEST_F(LoopOptimizerTest, RemoveDeadBranches_ZeroIterWhile) {
+TEST_F(LoopOptimizerTest, RemoveDeadBranchesZeroIterWhile) {
   const string gdef_ascii = R"EOF(
 node {
   name: "Const"
@@ -1358,15 +1358,15 @@ versions {
   CHECK(protobuf::TextFormat::ParseFromString(gdef_ascii, &item.graph));
   item.fetch = {"while/Exit"};
   auto tensors_expected = EvaluateNodes(item.graph, item.fetch);
-  EXPECT_EQ(1, tensors_expected.size());
+  ASSERT_EQ(tensors_expected.size(), 1);
 
   LoopOptimizer optimizer(RewriterConfig::AGGRESSIVE, nullptr);
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_CHECK_OK(status);
   auto tensors_got = EvaluateNodes(output, item.fetch);
-  EXPECT_EQ(1, tensors_got.size());
-  test::ExpectTensorEqual<int32>(tensors_expected[0], tensors_got[0]);
+  ASSERT_EQ(tensors_got.size(), 1);
+  test::ExpectTensorEqual<int32>(tensors_got[0], tensors_expected[0]);
 
   int nodes_present = 0;
   for (const NodeDef& node : output.node()) {
@@ -1382,7 +1382,200 @@ versions {
     }
     ++nodes_present;
   }
-  EXPECT_EQ(8, nodes_present);
+  EXPECT_EQ(nodes_present, 8);
+}
+
+TEST_F(LoopOptimizerTest, RemoveDeadBranchesConstantFeed) {
+  const string gdef_ascii = R"EOF(
+node {
+  name: "Const"
+  op: "Const"
+  device: "/job:localhost/replica:0/task:0/device:CPU:0"
+  attr {
+    key: "dtype"
+    value {
+      type: DT_STRING
+    }
+  }
+  attr {
+    key: "value"
+    value {
+      tensor {
+        dtype: DT_STRING
+        tensor_shape {
+          dim {
+            size: 1
+          }
+        }
+        string_val: "I\'m a value!"
+      }
+    }
+  }
+}
+node {
+  name: "cond/Switch_1"
+  op: "Switch"
+  input: "Const"
+  input: "Const_1"
+  device: "/job:localhost/replica:0/task:0/device:CPU:0"
+  attr {
+    key: "T"
+    value {
+      type: DT_STRING
+    }
+  }
+  attr {
+    key: "_class"
+    value {
+      list {
+        s: "loc:@Const"
+      }
+    }
+  }
+}
+node {
+  name: "Const_1"
+  op: "Const"
+  device: "/job:localhost/replica:0/task:0/device:CPU:0"
+  attr {
+    key: "dtype"
+    value {
+      type: DT_BOOL
+    }
+  }
+  attr {
+    key: "value"
+    value {
+      tensor {
+        dtype: DT_BOOL
+        tensor_shape {
+        }
+        bool_val: true
+      }
+    }
+  }
+}
+node {
+  name: "cond/Switch"
+  op: "Switch"
+  input: "Const_1"
+  input: "Const_1"
+  device: "/job:localhost/replica:0/task:0/device:CPU:0"
+  attr {
+    key: "T"
+    value {
+      type: DT_BOOL
+    }
+  }
+}
+node {
+  name: "cond/switch_t"
+  op: "Identity"
+  input: "cond/Switch:1"
+  device: "/job:localhost/replica:0/task:0/device:CPU:0"
+  attr {
+    key: "T"
+    value {
+      type: DT_BOOL
+    }
+  }
+}
+node {
+  name: "cond/Const"
+  op: "Const"
+  input: "^cond/switch_t"
+  device: "/job:localhost/replica:0/task:0/device:CPU:0"
+  attr {
+    key: "dtype"
+    value {
+      type: DT_STRING
+    }
+  }
+  attr {
+    key: "value"
+    value {
+      tensor {
+        dtype: DT_STRING
+        tensor_shape {
+          dim {
+            size: 1
+          }
+        }
+        string_val: ""
+      }
+    }
+  }
+}
+node {
+  name: "cond/Merge"
+  op: "Merge"
+  input: "cond/Switch_1"
+  input: "cond/Const"
+  device: "/job:localhost/replica:0/task:0/device:CPU:0"
+  attr {
+    key: "N"
+    value {
+      i: 2
+    }
+  }
+  attr {
+    key: "T"
+    value {
+      type: DT_STRING
+    }
+  }
+}
+node {
+  name: "Identity"
+  op: "Identity"
+  input: "cond/Merge"
+  device: "/job:localhost/replica:0/task:0/device:CPU:0"
+  attr {
+    key: "T"
+    value {
+      type: DT_STRING
+    }
+  }
+}
+library {
+}
+versions {
+  producer: 27
+}
+  )EOF";
+
+  GrapplerItem item;
+  CHECK(protobuf::TextFormat::ParseFromString(gdef_ascii, &item.graph));
+  item.fetch = {"Identity"};
+  Tensor feed_tensor(DT_BOOL, {});
+  feed_tensor.flat<bool>()(1) = false;
+  item.feed.push_back({"Const_1", feed_tensor});
+  auto tensors_expected = EvaluateNodes(item.graph, item.fetch);
+  ASSERT_EQ(tensors_expected.size(), 1);
+
+  LoopOptimizer optimizer(RewriterConfig::AGGRESSIVE, nullptr);
+  GraphDef output;
+  Status status = optimizer.Optimize(nullptr, item, &output);
+  TF_CHECK_OK(status);
+  auto tensors_got = EvaluateNodes(output, item.fetch);
+  ASSERT_EQ(tensors_got.size(), 1);
+  test::ExpectTensorEqual<string>(tensors_got[0], tensors_expected[0]);
+
+  EXPECT_EQ(output.node_size(), 8);
+
+  // No rewrite because branch has a constant feed node.
+  bool found = false;
+  for (const NodeDef& node : output.node()) {
+    if (node.name() == "cond/Merge") {
+      EXPECT_EQ(node.op(), "Merge");
+      ASSERT_EQ(node.input_size(), 2);
+      EXPECT_EQ(node.input(0), "cond/Switch_1");
+      EXPECT_EQ(node.input(1), "cond/Const");
+      found = true;
+      break;
+    }
+  }
+  EXPECT_TRUE(found);
 }
 
 }  // namespace grappler
-- 
GitLab


From 370f10f3b9711bd03367f32c4426b7d73a5c0d4c Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Tue, 12 Feb 2019 10:34:53 -0800
Subject: [PATCH 2332/2345] Create int8 space to depth.

PiperOrigin-RevId: 233633513
---
 tensorflow/lite/kernels/register.cc            |  4 +++-
 tensorflow/lite/kernels/space_to_depth.cc      | 10 +++++++++-
 tensorflow/lite/kernels/space_to_depth_test.cc |  8 ++++++++
 tensorflow/lite/toco/tflite/operator.cc        |  6 ++++++
 tensorflow/lite/toco/tflite/operator_test.cc   |  4 ++++
 5 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index 2290ed477c..f7ab8edc5c 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -241,7 +241,9 @@ BuiltinOpResolver::BuiltinOpResolver() {
              /* min_version */ 1,
              /* max_version */ 2);
   AddBuiltin(BuiltinOperator_SKIP_GRAM, Register_SKIP_GRAM());
-  AddBuiltin(BuiltinOperator_SPACE_TO_DEPTH, Register_SPACE_TO_DEPTH());
+  AddBuiltin(BuiltinOperator_SPACE_TO_DEPTH, Register_SPACE_TO_DEPTH(),
+             /* min_version */ 1,
+             /* max_version */ 2);
   AddBuiltin(BuiltinOperator_GATHER, Register_GATHER(),
              /* min_version */ 1,
              /* max_version */ 2);
diff --git a/tensorflow/lite/kernels/space_to_depth.cc b/tensorflow/lite/kernels/space_to_depth.cc
index 79e28bf47d..cf6b0bd4d3 100644
--- a/tensorflow/lite/kernels/space_to_depth.cc
+++ b/tensorflow/lite/kernels/space_to_depth.cc
@@ -50,7 +50,8 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   auto data_type = output->type;
   TF_LITE_ENSURE(context,
                  data_type == kTfLiteFloat32 || data_type == kTfLiteUInt8 ||
-                     data_type == kTfLiteInt32 || data_type == kTfLiteInt64);
+                     data_type == kTfLiteInt8 || data_type == kTfLiteInt32 ||
+                     data_type == kTfLiteInt64);
   TF_LITE_ENSURE_EQ(context, input->type, output->type);
 
   const int block_size = params->block_size;
@@ -100,6 +101,13 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
         TF_LITE_SPACE_TO_DEPTH(optimized_ops, uint8_t);
       }
       break;
+    case kTfLiteInt8:
+      if (kernel_type == kReference) {
+        TF_LITE_SPACE_TO_DEPTH(reference_ops, int8_t);
+      } else {
+        TF_LITE_SPACE_TO_DEPTH(optimized_ops, int8_t);
+      }
+      break;
     case kTfLiteInt32:
       if (kernel_type == kReference) {
         TF_LITE_SPACE_TO_DEPTH(reference_ops, int32_t);
diff --git a/tensorflow/lite/kernels/space_to_depth_test.cc b/tensorflow/lite/kernels/space_to_depth_test.cc
index 3fa8d86348..58665fc9d8 100644
--- a/tensorflow/lite/kernels/space_to_depth_test.cc
+++ b/tensorflow/lite/kernels/space_to_depth_test.cc
@@ -74,6 +74,14 @@ TEST(SpaceToDepthOpModel, Uint8) {
   EXPECT_THAT(m.GetOutputShape(), ElementsAre(1, 1, 1, 4));
 }
 
+TEST(SpaceToDepthOpModel, int8) {
+  SpaceToDepthOpModel m({TensorType_INT8, {1, 2, 2, 1}}, 2);
+  m.SetInput<int8_t>({1, 2, 3, 4});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutput<int8_t>(), ElementsAreArray({1, 2, 3, 4}));
+  EXPECT_THAT(m.GetOutputShape(), ElementsAre(1, 1, 1, 4));
+}
+
 TEST(SpaceToDepthOpModel, Int32) {
   SpaceToDepthOpModel m({TensorType_INT32, {1, 2, 2, 3}}, 2);
   m.SetInput<int32_t>({1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index 961bda2864..adb7f504f1 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -841,6 +841,12 @@ class SpaceToDepth
   }
 
   int GetVersion(const OperatorSignature& op_signature) const override {
+    const string& input_name = op_signature.op->inputs[0];
+    const Array& input_array = op_signature.model->GetArray(input_name);
+    // If the op take int8 input, it is version 2.
+    if (input_array.data_type == ArrayDataType::kInt8) {
+      return 2;
+    }
     return 1;
   }
 };
diff --git a/tensorflow/lite/toco/tflite/operator_test.cc b/tensorflow/lite/toco/tflite/operator_test.cc
index b977270bda..b4f7c48d75 100644
--- a/tensorflow/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/lite/toco/tflite/operator_test.cc
@@ -816,6 +816,10 @@ TEST_F(OperatorTest, VersioningStridedSliceTest) {
   SimpleVersioningTest<StridedSliceOperator>();
 }
 
+TEST_F(OperatorTest, VersioningSpaceToDepthTest) {
+  SimpleVersioningTest<SpaceToDepthOperator>();
+}
+
 TEST_F(OperatorTest, VersioningSliceTest) {
   SimpleVersioningTest<SliceOperator>();
 }
-- 
GitLab


From 2911f8c92cacc2b8198ca0fa15ad0843a5b5a801 Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <penporn@google.com>
Date: Tue, 12 Feb 2019 10:39:07 -0800
Subject: [PATCH 2333/2345] Internal change.

PiperOrigin-RevId: 233634333
---
 tensorflow/core/kernels/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 85e6410579..445fde84f1 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -7017,7 +7017,7 @@ tf_mkl_kernel_library(
             ":quantization_utils",
             ":transpose_functor",
             "//third_party/eigen3",
-            "//third_party/gemmlowp",
+            "@gemmlowp",
             "@mkl_dnn",
             "//tensorflow/core:array_ops_op_lib",
             "//tensorflow/core:core_cpu",
-- 
GitLab


From ce3ee9b6a45913c82d88331f12cf693cbe5e6014 Mon Sep 17 00:00:00 2001
From: Davide Libenzi <dlibenzi@google.com>
Date: Tue, 12 Feb 2019 10:50:22 -0800
Subject: [PATCH 2334/2345] Initialize object variables in HLO evaluator. The
 lack of initialization of the dynamic_dimension_inference_ member was causing
 random crashes when used from a PT client.

PiperOrigin-RevId: 233636726
---
 tensorflow/compiler/xla/service/hlo_evaluator.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.h b/tensorflow/compiler/xla/service/hlo_evaluator.h
index 72ea40bcd7..357975a131 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.h
@@ -331,16 +331,16 @@ class HloEvaluator : public DfsHloVisitorWithDefault {
   std::vector<const Literal*> arg_literals_;
 
   // Max loop iterations to execute with no maximum if negative.
-  int64 max_loop_iterations_;
+  int64 max_loop_iterations_ = 0;
 
   // Module-level seed handle.
-  uint64 seed_;
+  uint64 seed_ = 0;
   // RNG engine.
   std::minstd_rand0 engine_;
 
   // DynamicDimensionInference is used to evaluate GetDimensionSize, which
   // returns the dynamic dimension size of its operand.
-  DynamicDimensionInference* dynamic_dimension_inference_;
+  DynamicDimensionInference* dynamic_dimension_inference_ = nullptr;
 
   // Optional handler for custom_call ops.
   std::function<StatusOr<Literal>(HloInstruction* custom_call,
-- 
GitLab


From ba941c212da2f9bc52171172f61cbaf2a5d8c750 Mon Sep 17 00:00:00 2001
From: Blake Hechtman <blakehechtman@google.com>
Date: Tue, 12 Feb 2019 10:54:26 -0800
Subject: [PATCH 2335/2345] [XLA] Simplify batch dots that have no contracting
 dimensions into multiplies.

PiperOrigin-RevId: 233637534
---
 .../xla/service/algebraic_simplifier.cc       | 67 ++++++++++++-------
 .../xla/service/algebraic_simplifier_test.cc  | 16 +++--
 .../compiler/xla/service/shape_inference.cc   |  6 +-
 .../xla/service/shape_inference_test.cc       | 10 +++
 .../compiler/xla/tests/dot_operation_test.cc  |  2 +
 tensorflow/compiler/xla/util.cc               | 11 ++-
 6 files changed, 71 insertions(+), 41 deletions(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index 51d924f34d..c5deb74e96 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -1605,29 +1605,50 @@ Status AlgebraicSimplifierVisitor::HandleDot(HloInstruction* dot) {
     return Status::OK();
   }
 
-  // If a dot only contains batch dimensions, then tranpose the rhs and lhs
-  // acording to the batch dimension numbers and do a simple multiply.
-  if (lhs->shape().rank() ==
-          dot->dot_dimension_numbers().lhs_batch_dimensions_size() &&
-      rhs->shape().rank() ==
-          dot->dot_dimension_numbers().rhs_batch_dimensions_size()) {
-    HloInstruction* new_rhs = rhs;
-    HloInstruction* new_lhs = lhs;
-    if (lhs->shape().rank() > 1) {
-      TF_ASSIGN_OR_RETURN(
-          new_rhs,
-          MakeTransposeHlo(
-              rhs, AsInt64Slice(
-                       dot->dot_dimension_numbers().rhs_batch_dimensions())));
-      TF_ASSIGN_OR_RETURN(
-          new_lhs,
-          MakeTransposeHlo(
-              lhs, AsInt64Slice(
-                       dot->dot_dimension_numbers().lhs_batch_dimensions())));
-    }
-    TF_ASSIGN_OR_RETURN(auto new_dot,
-                        MakeBinaryHlo(HloOpcode::kMultiply, new_lhs, new_rhs));
-    return ReplaceInstruction(dot, new_dot);
+  // If there are no contracting dimensions, a dot can be rewritten as
+  // mul(broadcast(transpose(x)),broadcast(transpose(y)))
+  if (dot->dot_dimension_numbers().lhs_contracting_dimensions_size() == 0) {
+    std::vector<int64> lhs_transpose(
+        dot->dot_dimension_numbers().lhs_batch_dimensions().begin(),
+        dot->dot_dimension_numbers().lhs_batch_dimensions().end());
+    for (int64 i = 0; i < lhs->shape().rank(); ++i) {
+      if (!absl::c_linear_search(
+              dot->dot_dimension_numbers().lhs_batch_dimensions(), i)) {
+        lhs_transpose.push_back(i);
+      }
+    }
+    TF_ASSIGN_OR_RETURN(HloInstruction * new_lhs,
+                        MakeTransposeHlo(lhs, lhs_transpose));
+    if (dot->shape().rank() != lhs->shape().rank()) {
+      std::vector<int64> lhs_broadcast_dims(lhs->shape().rank());
+      absl::c_iota(lhs_broadcast_dims, 0);
+      new_lhs = computation_->AddInstruction(HloInstruction::CreateBroadcast(
+          dot->shape(), new_lhs, lhs_broadcast_dims));
+    }
+    std::vector<int64> rhs_transpose(
+        dot->dot_dimension_numbers().rhs_batch_dimensions().begin(),
+        dot->dot_dimension_numbers().rhs_batch_dimensions().end());
+    for (int64 i = 0; i < rhs->shape().rank(); ++i) {
+      if (!absl::c_linear_search(
+              dot->dot_dimension_numbers().rhs_batch_dimensions(), i)) {
+        rhs_transpose.push_back(i);
+      }
+    }
+    TF_ASSIGN_OR_RETURN(HloInstruction * new_rhs,
+                        MakeTransposeHlo(rhs, rhs_transpose));
+    if (dot->shape().rank() != rhs->shape().rank()) {
+      std::vector<int64> rhs_broadcast_dims(
+          dot->dot_dimension_numbers().lhs_batch_dimensions_size());
+      absl::c_iota(rhs_broadcast_dims, 0);
+      for (int64 i = lhs->shape().rank(); i < dot->shape().rank(); ++i) {
+        rhs_broadcast_dims.push_back(i);
+      }
+      new_rhs = computation_->AddInstruction(HloInstruction::CreateBroadcast(
+          dot->shape(), new_rhs, rhs_broadcast_dims));
+    }
+    return ReplaceWithNewInstruction(
+        dot, HloInstruction::CreateBinary(dot->shape(), HloOpcode::kMultiply,
+                                          new_lhs, new_rhs));
   }
 
   if (lhs->shape().rank() > 2 || rhs->shape().rank() > 2 ||
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index 2ace9330e7..feb6a0fb79 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -4222,8 +4222,10 @@ TEST_P(BatchDotStrengthReductionTest, BatchDotStrengthReduction) {
   std::tie(m, k, n, element_type) = GetParam();
 
   Shape dot_shape = ShapeUtil::MakeShape(element_type, {1, 3, 5, m, n});
-  Shape lhs_shape = ShapeUtil::MakeShape(element_type, {1, 3, 5, m, k});
-  Shape rhs_shape = ShapeUtil::MakeShape(element_type, {1, 3, 5, k, n});
+  Shape lhs_shape = k > 0 ? ShapeUtil::MakeShape(element_type, {1, 3, 5, m, k})
+                          : ShapeUtil::MakeShape(element_type, {1, 3, 5, m});
+  Shape rhs_shape = k > 0 ? ShapeUtil::MakeShape(element_type, {1, 3, 5, k, n})
+                          : ShapeUtil::MakeShape(element_type, {1, 3, 5, n});
   HloComputation::Builder builder(TestName());
 
   auto lhs = builder.AddInstruction(
@@ -4237,14 +4239,16 @@ TEST_P(BatchDotStrengthReductionTest, BatchDotStrengthReduction) {
   dot_dnums.add_rhs_batch_dimensions(0);
   dot_dnums.add_rhs_batch_dimensions(1);
   dot_dnums.add_rhs_batch_dimensions(2);
-  dot_dnums.add_lhs_contracting_dimensions(4);
-  dot_dnums.add_rhs_contracting_dimensions(3);
+  if (k > 0) {
+    dot_dnums.add_lhs_contracting_dimensions(4);
+    dot_dnums.add_rhs_contracting_dimensions(3);
+  }
   builder.AddInstruction(HloInstruction::CreateDot(
       dot_shape, lhs, rhs, dot_dnums, DefaultPrecisionConfig(2)));
   auto computation = module->AddEntryComputation(builder.Build());
   AlgebraicSimplifier simplifier(default_options_);
   TF_ASSERT_OK_AND_ASSIGN(bool changed, simplifier.Run(module.get()));
-  const bool dot_should_be_transformed = m == 1 || k == 1 || n == 1;
+  const bool dot_should_be_transformed = m == 1 || k == 1 || n == 1 || k == -1;
   const bool computation_should_be_modified = dot_should_be_transformed;
   EXPECT_EQ(changed, computation_should_be_modified);
   bool has_no_dot = true;
@@ -4259,7 +4263,7 @@ TEST_P(BatchDotStrengthReductionTest, BatchDotStrengthReduction) {
 
 INSTANTIATE_TEST_SUITE_P(
     BatchDotStrengthReductionTestInstantiation, BatchDotStrengthReductionTest,
-    ::testing::Combine(::testing::Values(1, 2), ::testing::Values(1, 2),
+    ::testing::Combine(::testing::Values(1, 2), ::testing::Values(-1, 1, 2),
                        ::testing::Values(1, 2), ::testing::Values(F32, BF16)));
 
 class DotStrengthReductionTest
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index 061dc46f7d..a570ee346d 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -2700,11 +2700,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
     const Shape& operand, absl::Span<const int64> dimensions) {
   TF_RETURN_IF_ERROR(ExpectArray(operand, "transpose"));
 
-  std::vector<int64> indices(operand.rank());
-  std::iota(indices.begin(), indices.end(), 0);
-  if (dimensions.size() != operand.rank() ||
-      !std::is_permutation(dimensions.begin(), dimensions.end(),
-                           indices.begin())) {
+  if (!IsPermutation(dimensions, operand.rank())) {
     return InvalidArgument(
         "Transpose dimensions [%s] are not a permutation of the operand "
         "dimensions (operand shape is %s).",
diff --git a/tensorflow/compiler/xla/service/shape_inference_test.cc b/tensorflow/compiler/xla/service/shape_inference_test.cc
index 77fcf4d27c..f400ef51f0 100644
--- a/tensorflow/compiler/xla/service/shape_inference_test.cc
+++ b/tensorflow/compiler/xla/service/shape_inference_test.cc
@@ -1572,6 +1572,16 @@ TEST_F(ShapeInferenceTest, Transpose) {
                                     ShapeUtil::MakeShape(F32, {3, 4, 5, 2})));
 }
 
+TEST_F(ShapeInferenceTest, Rank1Transpose) {
+  Shape a_shape = ShapeUtil::MakeShape(F32, {5});
+  auto inferred_shape_and_status =
+      ShapeInference::InferTransposeShape(a_shape, {0});
+  EXPECT_IS_OK(inferred_shape_and_status);
+  Shape inferred_shape = inferred_shape_and_status.ValueOrDie();
+  EXPECT_TRUE(
+      ShapeUtil::Compatible(inferred_shape, ShapeUtil::MakeShape(F32, {5})));
+}
+
 TEST_F(ShapeInferenceTest, Conditional) {
   auto inferred_status0 = ShapeInference::InferConditionalShape(
       pred_, vector_32_, vector_64_,
diff --git a/tensorflow/compiler/xla/tests/dot_operation_test.cc b/tensorflow/compiler/xla/tests/dot_operation_test.cc
index 2e35805c6a..11343ddfd0 100644
--- a/tensorflow/compiler/xla/tests/dot_operation_test.cc
+++ b/tensorflow/compiler/xla/tests/dot_operation_test.cc
@@ -1189,6 +1189,8 @@ std::vector<EinsumParamType> GetEinsumTestCases() {
       p{v{5, 6}, v{6, 7}, "ab,cd->dcba"},
       p{v{6}, v{6, 7}, "b,bc->c"},
       p{v{77}, v{77}, "a,a->a"},
+      p{v{77}, v{77, 55}, "a,ab->ba"},
+      p{v{2, 3, 77}, v{77, 2, 3, 55}, "ija,aijb->baij"},
       p{v{55}, v{}, "a,->a"},
       p{v{11, 111}, v{11}, "ab,a->ab"},
       p{v{16, 34}, v{16, 34}, "ab,ab->ab"},
diff --git a/tensorflow/compiler/xla/util.cc b/tensorflow/compiler/xla/util.cc
index 34b73b5206..bb8bbf57c4 100644
--- a/tensorflow/compiler/xla/util.cc
+++ b/tensorflow/compiler/xla/util.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include <stdarg.h>
 #include <numeric>
 
+#include "absl/container/inlined_vector.h"
 #include "absl/strings/match.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_join.h"
@@ -80,13 +81,9 @@ bool IsPermutation(absl::Span<const int64> permutation, int64 rank) {
   if (rank != permutation.size()) {
     return false;
   }
-  std::vector<int64> output(permutation.size(), -1);
-  for (auto index : permutation) {
-    CHECK_GE(index, 0);
-    CHECK_LT(index, rank);
-    output[index] = 0;
-  }
-  return !absl::c_linear_search(output, -1);
+  absl::InlinedVector<int64, 8> trivial_permutation(rank);
+  absl::c_iota(trivial_permutation, 0);
+  return absl::c_is_permutation(permutation, trivial_permutation);
 }
 
 std::vector<int64> InversePermutation(
-- 
GitLab


From 5f5088767db12a6b9d636441204015bd150041e4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 12 Feb 2019 11:06:25 -0800
Subject: [PATCH 2336/2345] Add tensor name to some of the error messages
 produced by tflite::Verify.

PiperOrigin-RevId: 233640378
---
 tensorflow/lite/tools/verifier.cc      | 49 +++++++++++++++-----------
 tensorflow/lite/tools/verifier_test.cc | 41 +++++++++++----------
 2 files changed, 48 insertions(+), 42 deletions(-)

diff --git a/tensorflow/lite/tools/verifier.cc b/tensorflow/lite/tools/verifier.cc
index b267dddaae..985dcb695e 100644
--- a/tensorflow/lite/tools/verifier.cc
+++ b/tensorflow/lite/tools/verifier.cc
@@ -54,7 +54,7 @@ const uint32_t kMaxNumString = UINT_MAX / sizeof(int32_t) - 2;
 
 // Verifies string tensor has legit buffer contents that follow the schema
 // defined in lite/string_util.h
-bool VerifyStringTensorBuffer(const Buffer& buffer,
+bool VerifyStringTensorBuffer(const Tensor& tensor, const Buffer& buffer,
                               ErrorReporter* error_reporter) {
   uint32_t buffer_size = buffer.data()->size();
   const char* buffer_ptr = reinterpret_cast<const char*>(buffer.data()->data());
@@ -62,7 +62,8 @@ bool VerifyStringTensorBuffer(const Buffer& buffer,
   uint32_t num_strings = *GetIntPtr(buffer_ptr);
   if (num_strings > kMaxNumString) {
     ReportError(error_reporter,
-                "String tensor has invalid num of string set: %d", num_strings);
+                "String tensor %s has invalid num of string set: %d",
+                tensor.name()->c_str(), num_strings);
     return false;
   }
   uint32_t header_offsets =
@@ -70,9 +71,9 @@ bool VerifyStringTensorBuffer(const Buffer& buffer,
 
   if (buffer_size < header_offsets) {
     ReportError(error_reporter,
-                "String tensor buffer requires at least %d bytes, but is "
+                "String tensor %s buffer requires at least %d bytes, but is "
                 "allocated with %d bytes",
-                header_offsets, buffer_size);
+                tensor.name()->c_str(), header_offsets, buffer_size);
     return false;
   }
 
@@ -81,22 +82,24 @@ bool VerifyStringTensorBuffer(const Buffer& buffer,
 
   if (*GetIntPtr(buffer_ptr + offset) != header_offsets) {
     ReportError(error_reporter,
-                "String tensor buffer initial offset must be: %d",
-                header_offsets);
+                "String tensor %s buffer initial offset must be: %d",
+                tensor.name()->c_str(), header_offsets);
     return false;
   }
   offset += sizeof(int32_t);
   for (int i = 1; i <= num_strings; i++, offset += sizeof(int32_t)) {
     int string_offset = *GetIntPtr(buffer_ptr + offset);
     if (string_offset < prev_ptr || string_offset > buffer_size) {
-      ReportError(error_reporter, "String tensor buffer is invalid: index %d",
-                  i);
+      ReportError(error_reporter,
+                  "String tensor %s buffer is invalid: index %d",
+                  tensor.name()->c_str(), i);
       return false;
     }
   }
   if (*GetIntPtr(buffer_ptr + offset - sizeof(int32_t)) != buffer_size) {
-    ReportError(error_reporter, "String tensor buffer last offset must be %d",
-                buffer_size);
+    ReportError(error_reporter,
+                "String tensor %s buffer last offset must be %d",
+                tensor.name()->c_str(), buffer_size);
     return false;
   }
   return true;
@@ -107,13 +110,15 @@ bool VerifyNumericTensorBuffer(const Tensor& tensor, const Buffer& buffer,
                                ErrorReporter* error_reporter) {
   uint64_t bytes_required = 1;
   if (!tensor.shape()) {
-    ReportError(error_reporter, "Tensor shape is empty");
+    ReportError(error_reporter, "Tensor %s shape is empty",
+                tensor.name()->c_str());
     return false;
   }
   for (int dim : *tensor.shape()) {
     bytes_required *= dim;
     if (bytes_required > UINT_MAX) {
-      ReportError(error_reporter, "Tensor dimension overflow");
+      ReportError(error_reporter, "Tensor %s dimension overflow",
+                  tensor.name()->c_str());
       return false;
     }
   }
@@ -133,19 +138,21 @@ bool VerifyNumericTensorBuffer(const Tensor& tensor, const Buffer& buffer,
     case TensorType_FLOAT16:
       // FALLTHROUGH_INTENDED;
     default:
-      ReportError(error_reporter, "Invalid tensor type: %d", tensor.type());
+      ReportError(error_reporter, "Tensor %s invalid type: %d",
+                  tensor.name()->c_str(), tensor.type());
       return false;
   }
   if (bytes_required > UINT_MAX) {
-    ReportError(error_reporter, "Tensor dimension overflow");
+    ReportError(error_reporter, "Tensor %s dimension overflow",
+                tensor.name()->c_str());
     return false;
   }
 
   if (bytes_required != buffer.data()->size()) {
     ReportError(
         error_reporter,
-        "Tensor requires %d bytes, but is allocated with %d bytes buffer",
-        bytes_required, buffer.data()->size());
+        "Tensor %s requires %d bytes, but is allocated with %d bytes buffer",
+        tensor.name()->c_str(), bytes_required, buffer.data()->size());
     return false;
   }
   return true;
@@ -299,14 +306,14 @@ bool VerifyTensors(const Model& model, ErrorReporter* error_reporter) {
         continue;
       }
       if (tensor->buffer() >= model.buffers()->size()) {
-        ReportError(error_reporter, "Invalid tensor buffer index: %d",
-                    tensor->buffer());
+        ReportError(error_reporter, "Tensor %s invalid buffer index: %d",
+                    tensor->name(), tensor->buffer());
         return false;
       }
       auto* buffer = model.buffers()->Get(tensor->buffer());
       if (!buffer) {
-        ReportError(error_reporter, "Tensor buffer %d not set",
-                    tensor->buffer());
+        ReportError(error_reporter, "Tensor %s buffer %d not set",
+                    tensor->name(), tensor->buffer());
         return false;
       }
 
@@ -314,7 +321,7 @@ bool VerifyTensors(const Model& model, ErrorReporter* error_reporter) {
       // buffers will be allocated by the interpreter at run-time.
       if (buffer->data()) {
         if (tensor->type() == TensorType_STRING) {
-          if (!VerifyStringTensorBuffer(*buffer, error_reporter)) {
+          if (!VerifyStringTensorBuffer(*tensor, *buffer, error_reporter)) {
             return false;
           }
         } else {
diff --git a/tensorflow/lite/tools/verifier_test.cc b/tensorflow/lite/tools/verifier_test.cc
index 17577f7490..e9ec3dd0ed 100644
--- a/tensorflow/lite/tools/verifier_test.cc
+++ b/tensorflow/lite/tools/verifier_test.cc
@@ -161,7 +161,7 @@ TEST(VerifyModel, TestEmptyShape) {
   builder.FinishModel({0, 1}, {3});
   ASSERT_FALSE(builder.Verify());
   EXPECT_THAT(builder.GetErrorString(),
-              ::testing::ContainsRegex("Tensor shape is empty"));
+              ::testing::ContainsRegex("Tensor inputtwo shape is empty"));
 }
 
 TEST(VerifyModel, TestSimpleModel) {
@@ -221,10 +221,9 @@ TEST(VerifyModel, TestIntTensorShapeIsGreaterThanBuffer) {
   builder.AddTensor({2, 3}, TensorType_UINT8, {1, 2, 3, 4}, "input");
   builder.FinishModel({}, {});
   ASSERT_FALSE(builder.Verify());
-  EXPECT_THAT(
-      builder.GetErrorString(),
-      ::testing::ContainsRegex(
-          "Tensor requires 6 bytes, but is allocated with 4 bytes buffer"));
+  EXPECT_THAT(builder.GetErrorString(),
+              ::testing::ContainsRegex("Tensor input requires 6 bytes, but is "
+                                       "allocated with 4 bytes buffer"));
 }
 
 TEST(VerifyModel, TestIntTensorShapeIsSmallerThanBuffer) {
@@ -232,10 +231,9 @@ TEST(VerifyModel, TestIntTensorShapeIsSmallerThanBuffer) {
   builder.AddTensor({2, 1}, TensorType_UINT8, {1, 2, 3, 4}, "input");
   builder.FinishModel({}, {});
   ASSERT_FALSE(builder.Verify());
-  EXPECT_THAT(
-      builder.GetErrorString(),
-      ::testing::ContainsRegex(
-          "Tensor requires 2 bytes, but is allocated with 4 bytes buffer"));
+  EXPECT_THAT(builder.GetErrorString(),
+              ::testing::ContainsRegex("Tensor input requires 2 bytes, but is "
+                                       "allocated with 4 bytes buffer"));
 }
 
 TEST(VerifyModel, TestIntTensorShapeOverflow) {
@@ -245,7 +243,7 @@ TEST(VerifyModel, TestIntTensorShapeOverflow) {
   builder.FinishModel({}, {});
   ASSERT_FALSE(builder.Verify());
   EXPECT_THAT(builder.GetErrorString(),
-              ::testing::ContainsRegex("Tensor dimension overflow"));
+              ::testing::ContainsRegex("Tensor input dimension overflow"));
 }
 
 TEST(VerifyModel, TensorBufferIsNotValid) {
@@ -284,10 +282,11 @@ TEST(VerifyModel, StringTensorHasInvalidNumString) {
       "input");
   builder.FinishModel({}, {});
   ASSERT_FALSE(builder.Verify());
-  EXPECT_THAT(builder.GetErrorString(),
-              ::testing::ContainsRegex(
-                  "String tensor buffer requires at least -2147483640 bytes, "
-                  "but is allocated with 18 bytes"));
+  EXPECT_THAT(
+      builder.GetErrorString(),
+      ::testing::ContainsRegex(
+          "String tensor input buffer requires at least -2147483640 bytes, "
+          "but is allocated with 18 bytes"));
 }
 
 TEST(VerifyModel, StringTensorOffsetTooSmall) {
@@ -299,7 +298,7 @@ TEST(VerifyModel, StringTensorOffsetTooSmall) {
   ASSERT_FALSE(builder.Verify());
   EXPECT_THAT(builder.GetErrorString(),
               ::testing::ContainsRegex(
-                  "String tensor buffer initial offset must be: 16"));
+                  "String tensor input buffer initial offset must be: 16"));
 }
 
 TEST(VerifyModel, StringTensorOffsetOutOfRange) {
@@ -309,9 +308,9 @@ TEST(VerifyModel, StringTensorOffsetOutOfRange) {
       {2, 0, 0, 0, 16, 0, 0, 0, 17, 0, 0, 0, 22, 0, 0, 0, 'A', 'B'}, "input");
   builder.FinishModel({}, {});
   ASSERT_FALSE(builder.Verify());
-  EXPECT_THAT(
-      builder.GetErrorString(),
-      ::testing::ContainsRegex("String tensor buffer is invalid: index 2"));
+  EXPECT_THAT(builder.GetErrorString(),
+              ::testing::ContainsRegex(
+                  "String tensor input buffer is invalid: index 2"));
 }
 
 TEST(VerifyModel, StringTensorIsLargerThanRequired) {
@@ -322,9 +321,9 @@ TEST(VerifyModel, StringTensorIsLargerThanRequired) {
       "input");
   builder.FinishModel({}, {});
   ASSERT_FALSE(builder.Verify());
-  EXPECT_THAT(
-      builder.GetErrorString(),
-      ::testing::ContainsRegex("String tensor buffer last offset must be 19"));
+  EXPECT_THAT(builder.GetErrorString(),
+              ::testing::ContainsRegex(
+                  "String tensor input buffer last offset must be 19"));
 }
 
 TEST(VerifyModel, AllOpsAreSupported) {
-- 
GitLab


From bb09f1e95ec932d76de29e923b92990b2a292fb5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 12 Feb 2019 11:23:49 -0800
Subject: [PATCH 2337/2345] Added tf.cos to TFLite Ops.

PiperOrigin-RevId: 233643966
---
 tensorflow/lite/build_def.bzl                 |   1 +
 tensorflow/lite/builtin_ops.h                 |   1 +
 .../lite/core/api/flatbuffer_conversions.cc   |   1 +
 tensorflow/lite/kernels/elementwise.cc        |  12 ++
 tensorflow/lite/kernels/elementwise_test.cc   |   9 ++
 tensorflow/lite/kernels/register.cc           |   2 +
 tensorflow/lite/nnapi_delegate.cc             |   1 +
 tensorflow/lite/schema/schema.fbs             |   5 +
 tensorflow/lite/schema/schema_generated.h     | 124 +++++++++++++++++-
 tensorflow/lite/testing/generate_examples.py  |  28 ++++
 .../propagate_fixed_sizes.cc                  |   1 +
 tensorflow/lite/toco/import_tensorflow.cc     |   1 +
 tensorflow/lite/toco/model.h                  |  12 ++
 tensorflow/lite/toco/tflite/operator.cc       |   2 +
 tensorflow/lite/toco/tflite/operator_test.cc  |   1 +
 tensorflow/lite/toco/tooling_util.cc          |   1 +
 16 files changed, 196 insertions(+), 6 deletions(-)

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index 88a8faf02e..cdd962372f 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -238,6 +238,7 @@ def generated_test_models():
         "conv2d_transpose",
         "conv_with_shared_weights",
         "conv_to_depthwiseconv_with_shared_weights",
+        "cos",
         "depthwiseconv",
         "div",
         "equal",
diff --git a/tensorflow/lite/builtin_ops.h b/tensorflow/lite/builtin_ops.h
index 1915565f4b..331694c2f6 100644
--- a/tensorflow/lite/builtin_ops.h
+++ b/tensorflow/lite/builtin_ops.h
@@ -133,6 +133,7 @@ typedef enum {
   kTfLiteBuiltinReverseV2 = 105,
   kTfLiteBuiltinAddN = 106,
   kTfLiteBuiltinGatherNd = 107,
+  kTfLiteBuiltinCos = 108,
 } TfLiteBuiltinOperator;
 
 #ifdef __cplusplus
diff --git a/tensorflow/lite/core/api/flatbuffer_conversions.cc b/tensorflow/lite/core/api/flatbuffer_conversions.cc
index 72667d4260..32b8cfecc4 100644
--- a/tensorflow/lite/core/api/flatbuffer_conversions.cc
+++ b/tensorflow/lite/core/api/flatbuffer_conversions.cc
@@ -680,6 +680,7 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type,
     // ok for now, since there is no call implementation either.
     case BuiltinOperator_CALL:
     case BuiltinOperator_CONCAT_EMBEDDINGS:
+    case BuiltinOperator_COS:
     case BuiltinOperator_CUSTOM:
     case BuiltinOperator_DEQUANTIZE:
     case BuiltinOperator_EMBEDDING_LOOKUP:
diff --git a/tensorflow/lite/kernels/elementwise.cc b/tensorflow/lite/kernels/elementwise.cc
index a79388b900..1cc188ae5f 100644
--- a/tensorflow/lite/kernels/elementwise.cc
+++ b/tensorflow/lite/kernels/elementwise.cc
@@ -83,6 +83,10 @@ TfLiteStatus SinEval(TfLiteContext* context, TfLiteNode* node) {
   return EvalNumeric(context, node, std::sin);
 }
 
+TfLiteStatus CosEval(TfLiteContext* context, TfLiteNode* node) {
+  return EvalNumeric(context, node, std::cos);
+}
+
 TfLiteStatus LogEval(TfLiteContext* context, TfLiteNode* node) {
   return EvalNumeric(context, node, std::log);
 }
@@ -122,6 +126,14 @@ TfLiteRegistration* Register_SIN() {
   return &r;
 }
 
+TfLiteRegistration* Register_COS() {
+  static TfLiteRegistration r = {
+      /*init=*/nullptr, /*free=*/nullptr,
+      elementwise::GenericPrepare<elementwise::IsNumericSupportedType>,
+      elementwise::CosEval};
+  return &r;
+}
+
 TfLiteRegistration* Register_LOG() {
   static TfLiteRegistration r = {
       /*init=*/nullptr, /*free=*/nullptr,
diff --git a/tensorflow/lite/kernels/elementwise_test.cc b/tensorflow/lite/kernels/elementwise_test.cc
index 7d24320081..89f2a506f0 100644
--- a/tensorflow/lite/kernels/elementwise_test.cc
+++ b/tensorflow/lite/kernels/elementwise_test.cc
@@ -65,6 +65,15 @@ TEST(ElementWise, Sin) {
   EXPECT_THAT(m.GetTensorShape(m.output()), ElementsAreArray({1, 1, 4, 1}));
 }
 
+TEST(ElementWise, Cos) {
+  ElementWiseOpFloatModel m(BuiltinOperator_COS, {1, 1, 4, 1});
+  m.PopulateTensor<float>(m.input(), {0, 3.1415926, -3.1415926, 1});
+  m.Invoke();
+  EXPECT_THAT(m.ExtractVector<float>(m.output()),
+              ElementsAreArray(ArrayFloatNear({1, -1, -1, 0.54030})));
+  EXPECT_THAT(m.GetTensorShape(m.output()), ElementsAreArray({1, 1, 4, 1}));
+}
+
 TEST(ElementWise, Log) {
   ElementWiseOpFloatModel m(BuiltinOperator_LOG, {1, 1, 4, 1});
   m.PopulateTensor<float>(m.input(), {1, 3.1415926, 1, 1});
diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index f7ab8edc5c..6dde4ec288 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -104,6 +104,7 @@ TfLiteRegistration* Register_REDUCE_ANY();
 TfLiteRegistration* Register_SELECT();
 TfLiteRegistration* Register_SLICE();
 TfLiteRegistration* Register_SIN();
+TfLiteRegistration* Register_COS();
 TfLiteRegistration* Register_TRANSPOSE_CONV();
 TfLiteRegistration* Register_EXPAND_DIMS();
 TfLiteRegistration* Register_SPARSE_TO_DENSE();
@@ -304,6 +305,7 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_SLICE, Register_SLICE(), /* min_version */ 1,
              /* max_version */ 2);
   AddBuiltin(BuiltinOperator_SIN, Register_SIN());
+  AddBuiltin(BuiltinOperator_COS, Register_COS());
   AddBuiltin(BuiltinOperator_TRANSPOSE_CONV, Register_TRANSPOSE_CONV());
   AddBuiltin(BuiltinOperator_TILE, Register_TILE());
   AddBuiltin(BuiltinOperator_SUM, Register_SUM());
diff --git a/tensorflow/lite/nnapi_delegate.cc b/tensorflow/lite/nnapi_delegate.cc
index f69baf1ac8..a5815d3a40 100644
--- a/tensorflow/lite/nnapi_delegate.cc
+++ b/tensorflow/lite/nnapi_delegate.cc
@@ -610,6 +610,7 @@ TfLiteStatus AddOpsAndParams(
       case tflite::BuiltinOperator_SPLIT:
       case tflite::BuiltinOperator_STRIDED_SLICE:
       case tflite::BuiltinOperator_EXP:
+      case tflite::BuiltinOperator_COS:
       case tflite::BuiltinOperator_LOG_SOFTMAX:
       case tflite::BuiltinOperator_DEQUANTIZE:
       case tflite::BuiltinOperator_DELEGATE:
diff --git a/tensorflow/lite/schema/schema.fbs b/tensorflow/lite/schema/schema.fbs
index b9c6e988f9..19c144d709 100644
--- a/tensorflow/lite/schema/schema.fbs
+++ b/tensorflow/lite/schema/schema.fbs
@@ -221,6 +221,7 @@ enum BuiltinOperator : byte {
   REVERSE_V2 = 105,
   ADD_N = 106,
   GATHER_ND = 107,
+  COS = 108,
 }
 
 // Options for the builtin operators.
@@ -308,6 +309,7 @@ union BuiltinOptions {
   ReverseV2Options,
   AddNOptions,
   GatherNdOptions,
+  CosOptions,
 }
 
 enum Padding : byte { SAME, VALID }
@@ -551,6 +553,9 @@ table TransposeOptions {
 table ExpOptions {
 }
 
+table CosOptions {
+}
+
 table ReducerOptions {
   keep_dims: bool;
 }
diff --git a/tensorflow/lite/schema/schema_generated.h b/tensorflow/lite/schema/schema_generated.h
index 3117789529..8734f4d706 100755
--- a/tensorflow/lite/schema/schema_generated.h
+++ b/tensorflow/lite/schema/schema_generated.h
@@ -139,6 +139,9 @@ struct TransposeOptionsT;
 struct ExpOptions;
 struct ExpOptionsT;
 
+struct CosOptions;
+struct CosOptionsT;
+
 struct ReducerOptions;
 struct ReducerOptionsT;
 
@@ -537,11 +540,12 @@ enum BuiltinOperator {
   BuiltinOperator_REVERSE_V2 = 105,
   BuiltinOperator_ADD_N = 106,
   BuiltinOperator_GATHER_ND = 107,
+  BuiltinOperator_COS = 108,
   BuiltinOperator_MIN = BuiltinOperator_ADD,
-  BuiltinOperator_MAX = BuiltinOperator_GATHER_ND
+  BuiltinOperator_MAX = BuiltinOperator_COS
 };
 
-inline const BuiltinOperator (&EnumValuesBuiltinOperator())[107] {
+inline const BuiltinOperator (&EnumValuesBuiltinOperator())[108] {
   static const BuiltinOperator values[] = {
     BuiltinOperator_ADD,
     BuiltinOperator_AVERAGE_POOL_2D,
@@ -649,7 +653,8 @@ inline const BuiltinOperator (&EnumValuesBuiltinOperator())[107] {
     BuiltinOperator_CEIL,
     BuiltinOperator_REVERSE_V2,
     BuiltinOperator_ADD_N,
-    BuiltinOperator_GATHER_ND
+    BuiltinOperator_GATHER_ND,
+    BuiltinOperator_COS
   };
   return values;
 }
@@ -764,6 +769,7 @@ inline const char * const *EnumNamesBuiltinOperator() {
     "REVERSE_V2",
     "ADD_N",
     "GATHER_ND",
+    "COS",
     nullptr
   };
   return names;
@@ -859,11 +865,12 @@ enum BuiltinOptions {
   BuiltinOptions_ReverseV2Options = 81,
   BuiltinOptions_AddNOptions = 82,
   BuiltinOptions_GatherNdOptions = 83,
+  BuiltinOptions_CosOptions = 84,
   BuiltinOptions_MIN = BuiltinOptions_NONE,
-  BuiltinOptions_MAX = BuiltinOptions_GatherNdOptions
+  BuiltinOptions_MAX = BuiltinOptions_CosOptions
 };
 
-inline const BuiltinOptions (&EnumValuesBuiltinOptions())[84] {
+inline const BuiltinOptions (&EnumValuesBuiltinOptions())[85] {
   static const BuiltinOptions values[] = {
     BuiltinOptions_NONE,
     BuiltinOptions_Conv2DOptions,
@@ -948,7 +955,8 @@ inline const BuiltinOptions (&EnumValuesBuiltinOptions())[84] {
     BuiltinOptions_UniqueOptions,
     BuiltinOptions_ReverseV2Options,
     BuiltinOptions_AddNOptions,
-    BuiltinOptions_GatherNdOptions
+    BuiltinOptions_GatherNdOptions,
+    BuiltinOptions_CosOptions
   };
   return values;
 }
@@ -1039,6 +1047,7 @@ inline const char * const *EnumNamesBuiltinOptions() {
     "ReverseV2Options",
     "AddNOptions",
     "GatherNdOptions",
+    "CosOptions",
     nullptr
   };
   return names;
@@ -1385,6 +1394,10 @@ template<> struct BuiltinOptionsTraits<GatherNdOptions> {
   static const BuiltinOptions enum_value = BuiltinOptions_GatherNdOptions;
 };
 
+template<> struct BuiltinOptionsTraits<CosOptions> {
+  static const BuiltinOptions enum_value = BuiltinOptions_CosOptions;
+};
+
 struct BuiltinOptionsUnion {
   BuiltinOptions type;
   void *value;
@@ -2080,6 +2093,14 @@ struct BuiltinOptionsUnion {
     return type == BuiltinOptions_GatherNdOptions ?
       reinterpret_cast<const GatherNdOptionsT *>(value) : nullptr;
   }
+  CosOptionsT *AsCosOptions() {
+    return type == BuiltinOptions_CosOptions ?
+      reinterpret_cast<CosOptionsT *>(value) : nullptr;
+  }
+  const CosOptionsT *AsCosOptions() const {
+    return type == BuiltinOptions_CosOptions ?
+      reinterpret_cast<const CosOptionsT *>(value) : nullptr;
+  }
 };
 
 bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *obj, BuiltinOptions type);
@@ -5012,6 +5033,46 @@ inline flatbuffers::Offset<ExpOptions> CreateExpOptions(
 
 flatbuffers::Offset<ExpOptions> CreateExpOptions(flatbuffers::FlatBufferBuilder &_fbb, const ExpOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
 
+struct CosOptionsT : public flatbuffers::NativeTable {
+  typedef CosOptions TableType;
+  CosOptionsT() {
+  }
+};
+
+struct CosOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef CosOptionsT NativeTableType;
+  bool Verify(flatbuffers::Verifier &verifier) const {
+    return VerifyTableStart(verifier) &&
+           verifier.EndTable();
+  }
+  CosOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const;
+  void UnPackTo(CosOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const;
+  static flatbuffers::Offset<CosOptions> Pack(flatbuffers::FlatBufferBuilder &_fbb, const CosOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
+};
+
+struct CosOptionsBuilder {
+  flatbuffers::FlatBufferBuilder &fbb_;
+  flatbuffers::uoffset_t start_;
+  explicit CosOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb)
+        : fbb_(_fbb) {
+    start_ = fbb_.StartTable();
+  }
+  CosOptionsBuilder &operator=(const CosOptionsBuilder &);
+  flatbuffers::Offset<CosOptions> Finish() {
+    const auto end = fbb_.EndTable(start_);
+    auto o = flatbuffers::Offset<CosOptions>(end);
+    return o;
+  }
+};
+
+inline flatbuffers::Offset<CosOptions> CreateCosOptions(
+    flatbuffers::FlatBufferBuilder &_fbb) {
+  CosOptionsBuilder builder_(_fbb);
+  return builder_.Finish();
+}
+
+flatbuffers::Offset<CosOptions> CreateCosOptions(flatbuffers::FlatBufferBuilder &_fbb, const CosOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
+
 struct ReducerOptionsT : public flatbuffers::NativeTable {
   typedef ReducerOptions TableType;
   bool keep_dims;
@@ -7678,6 +7739,9 @@ struct Operator FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
   const GatherNdOptions *builtin_options_as_GatherNdOptions() const {
     return builtin_options_type() == BuiltinOptions_GatherNdOptions ? static_cast<const GatherNdOptions *>(builtin_options()) : nullptr;
   }
+  const CosOptions *builtin_options_as_CosOptions() const {
+    return builtin_options_type() == BuiltinOptions_CosOptions ? static_cast<const CosOptions *>(builtin_options()) : nullptr;
+  }
   const flatbuffers::Vector<uint8_t> *custom_options() const {
     return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_CUSTOM_OPTIONS);
   }
@@ -8041,6 +8105,10 @@ template<> inline const GatherNdOptions *Operator::builtin_options_as<GatherNdOp
   return builtin_options_as_GatherNdOptions();
 }
 
+template<> inline const CosOptions *Operator::builtin_options_as<CosOptions>() const {
+  return builtin_options_as_CosOptions();
+}
+
 struct OperatorBuilder {
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
@@ -9586,6 +9654,29 @@ inline flatbuffers::Offset<ExpOptions> CreateExpOptions(flatbuffers::FlatBufferB
       _fbb);
 }
 
+inline CosOptionsT *CosOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
+  auto _o = new CosOptionsT();
+  UnPackTo(_o, _resolver);
+  return _o;
+}
+
+inline void CosOptions::UnPackTo(CosOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const {
+  (void)_o;
+  (void)_resolver;
+}
+
+inline flatbuffers::Offset<CosOptions> CosOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const CosOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) {
+  return CreateCosOptions(_fbb, _o, _rehasher);
+}
+
+inline flatbuffers::Offset<CosOptions> CreateCosOptions(flatbuffers::FlatBufferBuilder &_fbb, const CosOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) {
+  (void)_rehasher;
+  (void)_o;
+  struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const CosOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va;
+  return tflite::CreateCosOptions(
+      _fbb);
+}
+
 inline ReducerOptionsT *ReducerOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
   auto _o = new ReducerOptionsT();
   UnPackTo(_o, _resolver);
@@ -11347,6 +11438,10 @@ inline bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *ob
       auto ptr = reinterpret_cast<const GatherNdOptions *>(obj);
       return verifier.VerifyTable(ptr);
     }
+    case BuiltinOptions_CosOptions: {
+      auto ptr = reinterpret_cast<const CosOptions *>(obj);
+      return verifier.VerifyTable(ptr);
+    }
     default: return false;
   }
 }
@@ -11697,6 +11792,10 @@ inline void *BuiltinOptionsUnion::UnPack(const void *obj, BuiltinOptions type, c
       auto ptr = reinterpret_cast<const GatherNdOptions *>(obj);
       return ptr->UnPack(resolver);
     }
+    case BuiltinOptions_CosOptions: {
+      auto ptr = reinterpret_cast<const CosOptions *>(obj);
+      return ptr->UnPack(resolver);
+    }
     default: return nullptr;
   }
 }
@@ -12035,6 +12134,10 @@ inline flatbuffers::Offset<void> BuiltinOptionsUnion::Pack(flatbuffers::FlatBuff
       auto ptr = reinterpret_cast<const GatherNdOptionsT *>(value);
       return CreateGatherNdOptions(_fbb, ptr, _rehasher).Union();
     }
+    case BuiltinOptions_CosOptions: {
+      auto ptr = reinterpret_cast<const CosOptionsT *>(value);
+      return CreateCosOptions(_fbb, ptr, _rehasher).Union();
+    }
     default: return 0;
   }
 }
@@ -12373,6 +12476,10 @@ inline BuiltinOptionsUnion::BuiltinOptionsUnion(const BuiltinOptionsUnion &u) FL
       value = new GatherNdOptionsT(*reinterpret_cast<GatherNdOptionsT *>(u.value));
       break;
     }
+    case BuiltinOptions_CosOptions: {
+      value = new CosOptionsT(*reinterpret_cast<CosOptionsT *>(u.value));
+      break;
+    }
     default:
       break;
   }
@@ -12795,6 +12902,11 @@ inline void BuiltinOptionsUnion::Reset() {
       delete ptr;
       break;
     }
+    case BuiltinOptions_CosOptions: {
+      auto ptr = reinterpret_cast<CosOptionsT *>(value);
+      delete ptr;
+      break;
+    }
     default: break;
   }
   value = nullptr;
diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index 8449b321be..8991b233cb 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -1111,6 +1111,34 @@ def make_exp_tests(zip_path):
   make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
 
 
+def make_cos_tests(zip_path):
+  """Make a set of tests to do cos."""
+
+  test_parameters = [{
+      "input_dtype": [tf.float32],
+      "input_shape": [[], [3], [1, 100], [4, 2, 3], [5, 224, 224, 3]],
+  }]
+
+  def build_graph(parameters):
+    """Build the cos op testing graph."""
+    input_tensor = tf.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input",
+        shape=parameters["input_shape"])
+
+    out = tf.cos(input_tensor)
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    values = [
+        create_tensor_data(parameters["input_dtype"], parameters["input_shape"],
+                           min_value=-np.pi, max_value=np.pi)
+    ]
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+
+
 def make_log_softmax_tests(zip_path):
   """Make a set of tests to do log_softmax."""
 
diff --git a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
index 329ef92fe9..31710c4b05 100644
--- a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
+++ b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
@@ -2057,6 +2057,7 @@ void ProcessUniqueOperator(Model* model, UniqueOperator* op) {
     case OperatorType::kCeil:
     case OperatorType::kExp:
     case OperatorType::kSin:
+    case OperatorType::kCos:
     case OperatorType::kLogicalAnd:
     case OperatorType::kLogicalNot:
     case OperatorType::kLogicalOr:
diff --git a/tensorflow/lite/toco/import_tensorflow.cc b/tensorflow/lite/toco/import_tensorflow.cc
index 813e439995..c1ed9ed6be 100644
--- a/tensorflow/lite/toco/import_tensorflow.cc
+++ b/tensorflow/lite/toco/import_tensorflow.cc
@@ -2396,6 +2396,7 @@ ConverterMapType GetTensorFlowNodeConverterMap() {
       {"Const", ConvertConstOperator},
       {"Conv2D", ConvertConvOperator},
       {"Conv2DBackpropInput", ConvertTransposeConvOperator},
+      {"Cos", ConvertSimpleOperator<CosOperator, 1, 1>},
       {"CTCBeamSearchDecoder", ConvertCTCBeamSearchDecoderOperator},
       {"DepthToSpace", ConvertDepthToSpaceOperator},
       {"DepthwiseConv2dNative", ConvertDepthwiseConvOperator},
diff --git a/tensorflow/lite/toco/model.h b/tensorflow/lite/toco/model.h
index b99cb74a16..c8be41f3b5 100644
--- a/tensorflow/lite/toco/model.h
+++ b/tensorflow/lite/toco/model.h
@@ -45,6 +45,7 @@ enum class OperatorType : uint8 {
   kCeil,
   kConv,
   kConcatenation,
+  kCos,
   kDepthwiseConv,
   kDepthToSpace,
   kSpaceToDepth,
@@ -1166,6 +1167,17 @@ struct ExpOperator : Operator {
   ExpOperator() : Operator(OperatorType::kExp) {}
 };
 
+// Given a tensor input, this operation calculates element-wise exponential
+// (y = cos(x)).
+//
+// Inputs:
+//   inputs[0]: required: input tensor
+//
+// TensorFlow equivalent: Cos
+struct CosOperator : Operator {
+  CosOperator() : Operator(OperatorType::kCos) {}
+};
+
 // Given a tensor input, this operation inserts a dimension of 1 at the
 // dimension index axis of input's shape. The dimension index axis starts at
 // zero; if you specify a negative number for axis it is counted backward from
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index adb7f504f1..503e87bf7c 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -2343,6 +2343,8 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
       MakeUnique<SimpleOperator<TanhOperator>>("TANH", OperatorType::kTanh));
   ops.push_back(
       MakeUnique<SimpleOperator<ExpOperator>>("EXP", OperatorType::kExp));
+  ops.push_back(
+      MakeUnique<SimpleOperator<CosOperator>>("COS", OperatorType::kCos));
   ops.push_back(MakeUnique<SimpleOperator<LogSoftmaxOperator>>(
       "LOG_SOFTMAX", OperatorType::kLogSoftmax));
   ops.push_back(MakeUnique<Maximum>());  //  Element-wise Maximum
diff --git a/tensorflow/lite/toco/tflite/operator_test.cc b/tensorflow/lite/toco/tflite/operator_test.cc
index b4f7c48d75..d2f07c82f0 100644
--- a/tensorflow/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/lite/toco/tflite/operator_test.cc
@@ -119,6 +119,7 @@ TEST_F(OperatorTest, SimpleOperators) {
   CheckSimpleOperator<LogisticOperator>("LOGISTIC", OperatorType::kLogistic);
   CheckSimpleOperator<TanhOperator>("TANH", OperatorType::kTanh);
   CheckSimpleOperator<ExpOperator>("EXP", OperatorType::kExp);
+  CheckSimpleOperator<CosOperator>("COS", OperatorType::kCos);
   CheckSimpleOperator<LogSoftmaxOperator>("LOG_SOFTMAX",
                                           OperatorType::kLogSoftmax);
   CheckSimpleOperator<TensorFlowMaximumOperator>(
diff --git a/tensorflow/lite/toco/tooling_util.cc b/tensorflow/lite/toco/tooling_util.cc
index 74192dad78..602b9c52ff 100644
--- a/tensorflow/lite/toco/tooling_util.cc
+++ b/tensorflow/lite/toco/tooling_util.cc
@@ -422,6 +422,7 @@ const char* OperatorTypeName(OperatorType type) {
     HANDLE_OPERATORTYPENAME_CASE(Unique)
     HANDLE_OPERATORTYPENAME_CASE(UnidirectionalSequenceRnn)
     HANDLE_OPERATORTYPENAME_CASE(ReverseV2)
+    HANDLE_OPERATORTYPENAME_CASE(Cos)
     default:
       LOG(FATAL) << "Unhandled op type";
 #undef HANDLE_OPERATORTYPENAME_CASE
-- 
GitLab


From a5f9e686360d9a6bc46395eaa8ce5086402aab50 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 12 Feb 2019 11:37:44 -0800
Subject: [PATCH 2338/2345] Adding modification for 2000+ tests to run.

PiperOrigin-RevId: 233647224
---
 tensorflow/tools/ci_build/builds/pip_new.sh | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/tensorflow/tools/ci_build/builds/pip_new.sh b/tensorflow/tools/ci_build/builds/pip_new.sh
index 78e226eaf7..3ee3e79463 100755
--- a/tensorflow/tools/ci_build/builds/pip_new.sh
+++ b/tensorflow/tools/ci_build/builds/pip_new.sh
@@ -51,7 +51,7 @@
 #                        overwritten to 'tf_nightly'. For gpu builds, '_gpu'
 #                        will be appended.
 #                          e.g. TF_PROJECT_NAME="tensorflow"
-# 			                   e.g. TF_PROJECT_NAME="tf_nightly_gpu"
+#                          e.g. TF_PROJECT_NAME="tf_nightly_gpu"
 #   TF_PIP_TEST_ROOT:    Root directory for building and testing pip pkgs.
 #                          e.g. TF_PIP_TEST_ROOT="pip_test"
 #
@@ -140,8 +140,6 @@ update_bazel_flags() {
 
 update_test_filter_tags() {
   # Add test filter tags
-  # This script is for PIP version of the installation. Add pip related tags.
-  add_test_filter_tag -no_pip -nopip
   # MacOS filter tags
   if [[ ${OS_TYPE} == "macos" ]]; then
     remove_test_filter_tag nomac no_mac
@@ -151,9 +149,6 @@ update_test_filter_tags() {
   if [[ "${CONTAINER_TYPE}" == "gpu" ]]; then
     remove_test_filter_tag no_gpu -requires-gpu
     add_test_filter_tag requires-gpu
-  else
-    remove_test_filter_tag -no_gpu requires-gpu
-    add_test_filter_tag no_gpu -requires-gpu
   fi
   echo "Final test filter tags: ${BAZEL_TEST_FILTER_TAGS}"
 }
-- 
GitLab


From 2f2fdfd6a2ea29bc74d901a3837aba0adb21b6dd Mon Sep 17 00:00:00 2001
From: Shivani Agrawal <shivaniagrawal@google.com>
Date: Tue, 12 Feb 2019 11:39:50 -0800
Subject: [PATCH 2339/2345] [tf.data] adds test for variable collection sharing
 from parent graph to function graph.

PiperOrigin-RevId: 233647640
---
 tensorflow/python/data/kernel_tests/map_test.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tensorflow/python/data/kernel_tests/map_test.py b/tensorflow/python/data/kernel_tests/map_test.py
index 533812f591..4badcffd46 100644
--- a/tensorflow/python/data/kernel_tests/map_test.py
+++ b/tensorflow/python/data/kernel_tests/map_test.py
@@ -1031,6 +1031,18 @@ class MapTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.assertRaises(errors.InvalidArgumentError):
       self.evaluate(get_next())
 
+  # NOTE: collection test is specific to graph mode only, no eager coverage.
+  @test_util.run_v1_only("graph specific test")
+  def testSkipEagerCollectionCopy(self):
+    w = variable_scope.get_variable("w", [])
+    self.assertIn(w, ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES))
+
+    def func(x):
+      self.assertIn(w, ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES))
+      return x
+
+    dataset = dataset_ops.Dataset.from_tensors(constant_op.constant(1.0))
+    dataset.map(func)
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From f4b4adff52eb0c1ed1b687855f33221f1376e688 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Tue, 12 Feb 2019 11:46:08 -0800
Subject: [PATCH 2340/2345] Delimit large blobs of text in logs a bit more
 clearly.

PiperOrigin-RevId: 233648996
---
 tensorflow/python/autograph/impl/conversion.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/autograph/impl/conversion.py b/tensorflow/python/autograph/impl/conversion.py
index 1ac2e33a8a..93cc337546 100644
--- a/tensorflow/python/autograph/impl/conversion.py
+++ b/tensorflow/python/autograph/impl/conversion.py
@@ -219,7 +219,7 @@ def entity_to_graph(o, program_ctx, arg_values, arg_types):
                 compiler.ast_to_source(node))
   if logging.has_verbosity(4):
     for n in node:
-      logging.log(4, 'Compiled AST of %s:\n\n%s\n', o, gast.dump(n))
+      logging.log(4, 'Compiled AST of %s:\n\n%s\n\n', o, gast.dump(n))
 
   if program_ctx.options.recursive:
     while True:
@@ -354,7 +354,7 @@ def function_to_graph(f,
   """Specialization of `entity_to_graph` for callable functions."""
 
   node, source = parser.parse_entity(f)
-  logging.log(3, 'Source code of %s:\n%s', f, source)
+  logging.log(3, 'Source code of %s:\n\n%s\n', f, source)
   node = node.body[0]
 
   # In general, the output of inspect.getsource is inexact for lambdas because
-- 
GitLab


From 4474fb7ffd3476c73b00dfcc0b0748f1aadbac34 Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Tue, 12 Feb 2019 11:54:21 -0800
Subject: [PATCH 2341/2345] Exclude TF-TRT segmentation code from non-trt
 builds like windows.

PiperOrigin-RevId: 233650620
---
 tensorflow/compiler/tf2tensorrt/BUILD                   | 1 +
 tensorflow/compiler/tf2tensorrt/segment/segment.cc      | 6 ++++++
 tensorflow/compiler/tf2tensorrt/segment/segment.h       | 7 ++++++-
 tensorflow/compiler/tf2tensorrt/segment/segment_test.cc | 6 ++++++
 4 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/tf2tensorrt/BUILD b/tensorflow/compiler/tf2tensorrt/BUILD
index 76a009e1d5..eca533095a 100644
--- a/tensorflow/compiler/tf2tensorrt/BUILD
+++ b/tensorflow/compiler/tf2tensorrt/BUILD
@@ -352,6 +352,7 @@ cc_library(
         "segment/segment.h",
         "segment/union_find.h",
     ],
+    copts = tf_copts(),
     deps = [
         "//tensorflow/core:graph",
         "//tensorflow/core:lib_proto_parsing",
diff --git a/tensorflow/compiler/tf2tensorrt/segment/segment.cc b/tensorflow/compiler/tf2tensorrt/segment/segment.cc
index 4a8a4ac758..3794929b1d 100644
--- a/tensorflow/compiler/tf2tensorrt/segment/segment.cc
+++ b/tensorflow/compiler/tf2tensorrt/segment/segment.cc
@@ -30,6 +30,9 @@ limitations under the License.
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/types.h"
 
+#if GOOGLE_CUDA
+#if GOOGLE_TENSORRT
+
 namespace tensorflow {
 namespace tensorrt {
 namespace segment {
@@ -725,3 +728,6 @@ tensorflow::Status SegmentGraph(
 }  // namespace segment
 }  // namespace tensorrt
 }  // namespace tensorflow
+
+#endif  // GOOGLE_TENSORRT
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/compiler/tf2tensorrt/segment/segment.h b/tensorflow/compiler/tf2tensorrt/segment/segment.h
index 9a0ccc9aef..9622ddd593 100644
--- a/tensorflow/compiler/tf2tensorrt/segment/segment.h
+++ b/tensorflow/compiler/tf2tensorrt/segment/segment.h
@@ -24,8 +24,10 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
 
-namespace tensorflow {
+#if GOOGLE_CUDA
+#if GOOGLE_TENSORRT
 
+namespace tensorflow {
 namespace tensorrt {
 namespace segment {
 
@@ -60,4 +62,7 @@ tensorflow::Status SegmentGraph(
 }  // namespace tensorrt
 }  // namespace tensorflow
 
+#endif  // GOOGLE_TENSORRT
+#endif  // GOOGLE_CUDA
+
 #endif  // TENSORFLOW_COMPILER_TF2TENSORRT_SEGMENT_SEGMENT_H_
diff --git a/tensorflow/compiler/tf2tensorrt/segment/segment_test.cc b/tensorflow/compiler/tf2tensorrt/segment/segment_test.cc
index 58512d3b09..e11ad27197 100644
--- a/tensorflow/compiler/tf2tensorrt/segment/segment_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/segment/segment_test.cc
@@ -26,6 +26,9 @@ limitations under the License.
 #include "tensorflow/core/platform/types.h"
 #include "tensorflow/core/public/session.h"
 
+#if GOOGLE_CUDA
+#if GOOGLE_TENSORRT
+
 namespace tensorflow {
 namespace tensorrt {
 namespace segment {
@@ -265,3 +268,6 @@ TEST_F(SegmentTest, BigIfElse) {
 }  // namespace segment
 }  // namespace tensorrt
 }  // namespace tensorflow
+
+#endif  // GOOGLE_TENSORRT
+#endif  // GOOGLE_CUDA
-- 
GitLab


From fd481c1af898fa5a587d09e9505fcd273bcf18da Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Tue, 12 Feb 2019 11:55:43 -0800
Subject: [PATCH 2342/2345] Implement new TF-TRT calibration workflow.

PiperOrigin-RevId: 233650911
---
 .../tensorrt/test/quantization_mnist_test.py  |   7 +-
 .../test/tf_trt_integration_test_base.py      | 149 +++++++++--------
 .../python/compiler/tensorrt/trt_convert.py   | 156 ++++++++++++++++--
 .../compiler/tensorrt/trt_convert_test.py     | 152 ++++++++++++-----
 4 files changed, 337 insertions(+), 127 deletions(-)

diff --git a/tensorflow/python/compiler/tensorrt/test/quantization_mnist_test.py b/tensorflow/python/compiler/tensorrt/test/quantization_mnist_test.py
index cf5e5b897a..cdd25e3b98 100644
--- a/tensorflow/python/compiler/tensorrt/test/quantization_mnist_test.py
+++ b/tensorflow/python/compiler/tensorrt/test/quantization_mnist_test.py
@@ -138,9 +138,9 @@ class QuantizationAwareTrainingMNISTTest(test_util.TensorFlowTestCase):
     if use_trt:
       logging.info('Number of nodes before TF-TRT conversion: %d',
                    len(graph_def.node))
-      graph_def = trt_convert.create_inference_graph(
-          graph_def,
-          outputs=[OUTPUT_NODE_NAME],
+      converter = trt_convert.TrtGraphConverter(
+          input_graph_def=graph_def,
+          nodes_blacklist=[OUTPUT_NODE_NAME],
           max_batch_size=max_batch_size,
           precision_mode='INT8',
           # There is a 2GB GPU memory limit for each test, so we set
@@ -150,6 +150,7 @@ class QuantizationAwareTrainingMNISTTest(test_util.TensorFlowTestCase):
           minimum_segment_size=2,
           use_calibration=False,
       )
+      graph_def = converter.convert()
       logging.info('Number of nodes after TF-TRT conversion: %d',
                    len(graph_def.node))
       num_engines = len(
diff --git a/tensorflow/python/compiler/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/python/compiler/tensorrt/test/tf_trt_integration_test_base.py
index 5af1a97016..428883c223 100644
--- a/tensorflow/python/compiler/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/python/compiler/tensorrt/test/tf_trt_integration_test_base.py
@@ -259,7 +259,7 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
   def _GetConfigProto(self, run_params, graph_state):
     """Get config proto based on specific settings."""
     conversion_params = self.GetConversionParams(run_params)
-    if graph_state != GraphState.ORIGINAL and run_params.use_optimizer:
+    if graph_state == GraphState.INFERENCE and run_params.use_optimizer:
       rewriter_cfg = trt_convert.TrtGraphConverter.get_tensorrt_rewriter_config(
           conversion_params.rewriter_config, conversion_params.max_batch_size,
           conversion_params.max_workspace_size_bytes,
@@ -299,6 +299,23 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
   def _ExpectNativeSegment(self, engine_name, value):
     self._ExpectTestValue(engine_name, "ExecuteNativeSegment", value)
 
+  def _GetFeedNames(self):
+    params = self._GetParamsCached()
+    # Construct the feeds tensor names by appending :0 to the node names.
+    return [input_name + ":0" for input_name in params.input_names]
+
+  def _GetFetchNames(self):
+    params = self._GetParamsCached()
+    # Construct the fetches tensor names by appending :0 to the node names.
+    return [output_name + ":0" for output_name in params.output_names]
+
+  def _GetFeedDict(self, inputs_data, input_shape_index):
+    assert input_shape_index < len(inputs_data)
+    feeds = self._GetFeedNames()
+    return {
+        feeds[i]: inputs_data[input_shape_index][i] for i in range(len(feeds))
+    }
+
   def _RunGraph(self,
                 run_params,
                 gdef,
@@ -308,32 +325,23 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
                 num_runs=2):
     """Run given graphdef multiple times."""
     params = self._GetParamsCached()
-    for current_input_data in inputs_data:
-      assert len(params.input_names) == len(current_input_data)
+    for data in inputs_data:
+      assert len(params.input_names) == len(data)
 
-    vals = []
+    fetches = self._GetFetchNames()
     g = ops.Graph()
     with g.as_default():
-      io_ops = importer.import_graph_def(
-          graph_def=gdef,
-          return_elements=params.input_names + params.output_names,
-          name="")
-      inputs = [op.outputs[0] for op in io_ops[:len(params.input_names)]]
-      for current_input_data in inputs_data:
-        assert len(inputs) == len(current_input_data)
-      outputs = [op.outputs[0] for op in io_ops[len(params.input_names):]]
-      with self.test_session(
+      importer.import_graph_def(graph_def=gdef, name="")
+      with self.session(
           graph=g, config=config, use_gpu=True, force_gpu=True) as sess:
+        vals = []
         # Run for each input(s) shape
         for shape_index in range(len(inputs_data)):
           val = None
-          # Defaults to 2 runs to verify result across multiple runs is same.
           for _ in range(num_runs):
             self._PrepareRun(graph_state)
-            new_val = sess.run(outputs, {
-                inputs[i]: inputs_data[shape_index][i]
-                for i in range(len(inputs))
-            })
+            new_val = sess.run(fetches,
+                               self._GetFeedDict(inputs_data, shape_index))
             output_len = len(params.expected_output_dims[shape_index])
             self.assertEqual(output_len, len(new_val))
             for i in range(output_len):
@@ -345,25 +353,15 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
             val = new_val
             self.VerifyRun(run_params, graph_state)
           vals.append(val)
-    return vals
-
-  # Use real data that is representative of the inference dataset
-  # for calibration. For this test script it is random data.
-  def _RunCalibration(self, run_params, gdef, inputs_data, config):
-    """Run calibration on given graph."""
-    return self._RunGraph(
-        run_params, gdef, inputs_data, config, GraphState.CALIBRATE, num_runs=5)
+        return vals
 
-  def _GetTrtGraphDef(self, run_params, graph_state, gdef):
-    """Return trt converted graphdef."""
+  def _CreateConverter(self, gdef, session_config, conversion_params):
+    """Return a TrtGraphConverter."""
     params = self._GetParamsCached()
-    conversion_params = self.GetConversionParams(run_params)
-    logging.info(conversion_params)
-
-    config_for_trt = self._GetConfigProto(run_params, graph_state)
-    return trt_convert.create_inference_graph(
+    converter = trt_convert.TrtGraphConverter(
         input_graph_def=gdef,
-        outputs=params.input_names + params.output_names,
+        nodes_blacklist=params.input_names + params.output_names,
+        session_config=session_config,
         max_batch_size=conversion_params.max_batch_size,
         max_workspace_size_bytes=conversion_params.max_workspace_size_bytes,
         precision_mode=conversion_params.precision_mode,
@@ -371,8 +369,42 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
         is_dynamic_op=conversion_params.is_dynamic_op,
         maximum_cached_engines=conversion_params.maximum_cached_engines,
         cached_engine_batches=conversion_params.cached_engine_batches,
-        use_calibration=conversion_params.use_calibration,
-        session_config=config_for_trt)
+        use_calibration=conversion_params.use_calibration)
+    return converter
+
+  def _GetCalibratedInferGraph(self, run_params, gdef, inputs_data):
+    """Return trt converted graphdef in INT8 mode."""
+    conversion_params = self.GetConversionParams(run_params)
+    logging.info(conversion_params)
+    assert conversion_params.precision_mode == "INT8"
+    assert conversion_params.is_dynamic_op
+    assert conversion_params.maximum_cached_engines == 1
+    assert not conversion_params.cached_engine_batches
+    assert conversion_params.use_calibration
+    assert len(inputs_data) == 1  # We only support calibrating single engine.
+
+    session_config = self._GetConfigProto(run_params, GraphState.CALIBRATE)
+    logging.info("Running calibration graph, config:\n%s", str(session_config))
+
+    converter = self._CreateConverter(gdef, session_config, conversion_params)
+    int8_gdef = converter.convert()
+    self._VerifyGraphDef(run_params, int8_gdef, GraphState.CALIBRATE)
+
+    return converter.calibrate(
+        fetch_names=self._GetFetchNames(),
+        num_runs=5,
+        feed_dict_fn=lambda: self._GetFeedDict(inputs_data, 0))
+
+  def _GetInferGraph(self, run_params, gdef):
+    """Return trt converted graphdef."""
+    conversion_params = self.GetConversionParams(run_params)
+    logging.info(conversion_params)
+
+    session_config = self._GetConfigProto(run_params, GraphState.INFERENCE)
+    logging.info("Creating TRT graph for inference, config\n%s",
+                 str(session_config))
+    converter = self._CreateConverter(gdef, session_config, conversion_params)
+    return converter.convert()
 
   def _WriteGraph(self, run_params, gdef, graph_state):
     if graph_state == GraphState.ORIGINAL:
@@ -455,13 +487,14 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
 
     expected_engines = self.ExpectedEnginesToBuild(run_params)
     num_engines = 0
+    functions = [f.signature.name for f in gdef.library.function]
     for node in gdef.node:
       if node.op == "TRTEngineOp":
         logging.info("Found TRTEngineOp: " + node.name)
-    for node in gdef.node:
-      if node.op == "TRTEngineOp":
         num_engines += 1
-        self.assertTrue(node.name in expected_engines, node.name)
+        function_name = node.name + "_native_segment"
+        self.assertIn(function_name, functions)
+        self.assertIn(node.name, expected_engines)
         self.assertTrue(len(node.attr["serialized_segment"].s), node.name)
         self.assertTrue(len(node.attr["segment_funcdef_name"].s), node.name)
         self.assertEqual(
@@ -522,9 +555,10 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
             (scale * np.random.random_sample(dims)).astype(dtype))
       inputs_data.append(current_input_data)
 
+    # Verify original graph.
     self._VerifyGraphDef(run_params, input_gdef, GraphState.ORIGINAL)
 
-    # Get reference result without running trt.
+    # Run original graph without trt to get reference result.
     config_no_trt = self._GetConfigProto(run_params, GraphState.ORIGINAL)
     logging.info("Running original graph w/o trt, config:\n%s",
                  str(config_no_trt))
@@ -534,27 +568,12 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
     # Run calibration if necessary.
     if (IsQuantizationMode(run_params.precision_mode) and
         run_params.use_calibration):
-
-      calib_config = self._GetConfigProto(run_params, GraphState.CALIBRATE)
-      logging.info("Running calibration graph, config:\n%s", str(calib_config))
-      if run_params.use_optimizer:
-        result = self._RunCalibration(run_params, input_gdef, inputs_data,
-                                      calib_config)
-      else:
-        calib_gdef = self._GetTrtGraphDef(run_params, GraphState.CALIBRATE,
-                                          input_gdef)
-        self._VerifyGraphDef(run_params, calib_gdef, GraphState.CALIBRATE)
-        result = self._RunCalibration(run_params, calib_gdef, inputs_data,
-                                      calib_config)
-      infer_gdef = trt_convert.calib_graph_to_infer_graph(
-          calib_gdef, run_params.dynamic_engine)
+      infer_gdef = self._GetCalibratedInferGraph(run_params, input_gdef,
+                                                 inputs_data)
+      self._VerifyGraphDef(run_params, infer_gdef, GraphState.INFERENCE)
+    elif not run_params.use_optimizer:
+      infer_gdef = self._GetInferGraph(run_params, input_gdef)
       self._VerifyGraphDef(run_params, infer_gdef, GraphState.INFERENCE)
-
-      self.assertAllClose(
-          ref_result,
-          result,
-          atol=self.ExpectedAbsoluteTolerance(run_params),
-          rtol=self.ExpectedRelativeTolerance(run_params))
     else:
       infer_gdef = input_gdef
 
@@ -562,11 +581,6 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
     infer_config = self._GetConfigProto(run_params, GraphState.INFERENCE)
     logging.info("Running final inference graph, config:\n%s",
                  str(infer_config))
-    if not run_params.use_optimizer:
-      infer_gdef = self._GetTrtGraphDef(run_params, GraphState.INFERENCE,
-                                        infer_gdef)
-      self._VerifyGraphDef(run_params, infer_gdef, GraphState.INFERENCE)
-
     result = self._RunGraph(run_params, infer_gdef, inputs_data, infer_config,
                             GraphState.INFERENCE)
     self.assertAllClose(
@@ -611,9 +625,8 @@ def _AddTests(test_class):
   for (use_optimizer, precision_mode, dynamic_engine, use_calibration) in opts:
     if IsQuantizationMode(precision_mode):
       if use_optimizer:
-        # TODO(aaroey): if use_optimizer is True we need to get the inference
-        # graphdef using custom python wrapper class, which is not currently
-        # supported yet.
+        # We ignore the use_optimizer option and always use TrtGraphConverter
+        # for INT8 mode, so no need to run it twice.
         continue
       if use_calibration and not dynamic_engine:
         # Static engine with use_calibration=False will be static, so we want to
diff --git a/tensorflow/python/compiler/tensorrt/trt_convert.py b/tensorflow/python/compiler/tensorrt/trt_convert.py
index 2dd34a83aa..d3ed4644cc 100644
--- a/tensorflow/python/compiler/tensorrt/trt_convert.py
+++ b/tensorflow/python/compiler/tensorrt/trt_convert.py
@@ -25,11 +25,13 @@ from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import meta_graph_pb2
 from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python.client import session
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl as _impl
 from tensorflow.python.framework import graph_util
 from tensorflow.python.framework import importer
 from tensorflow.python.framework import ops
 from tensorflow.python.grappler import tf_optimizer
+from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import tf_logging
 from tensorflow.python.saved_model import builder
 from tensorflow.python.saved_model import loader
@@ -76,7 +78,20 @@ class GraphConverter(object):
   my_converter.save(output_saved_model_dir)  # Optional
   ```
 
-  TODO(laigd): add calibration support.
+  To run the conversion with quantization calibration:
+
+  ```python
+  my_converter = MyGraphConverter(input_saved_model_dir="my_dir")
+  my_converter.convert()
+
+  # Run calibration 10 times.
+  converted_graph_def = my_converter.calibrate(
+      fetch_names=['output:0'],
+      num_runs=10,
+      feed_dict_fn=lambda: {'input:0': my_next_data()})
+
+  my_converter.save(output_saved_model_dir)  # Optional
+  ```
   """
 
   def __init__(self,
@@ -120,6 +135,11 @@ class GraphConverter(object):
         input_saved_model_tags or [tag_constants.SERVING])
     self._session_config = session_config or config_pb2.ConfigProto()
 
+    # For calibration usage.
+    self._calibration_graph = None
+    self._calibration_sess = None
+    self._calibration_data_collected = False
+
   def get_rewriter_config(self, rewriter_config_template=None):
     """Returns a RewriterConfig proto for TRT transformation.
 
@@ -239,6 +259,63 @@ class GraphConverter(object):
       self._convert_saved_model()
     return self._converted_graph_def
 
+  def calibrate(self,
+                fetch_names,
+                num_runs,
+                feed_dict_fn=None,
+                input_map_fn=None):
+    """Run the calibration and return the calibrated GraphDef.
+
+    Args:
+      fetch_names: a list of output tensor name to fetch during calibration.
+      num_runs: number of runs of the graph during calibration.
+      feed_dict_fn: a function that returns a dictionary mapping input names (as
+        strings) in the GraphDef to be calibrated to values (e.g. Python list,
+        numpy arrays, etc). One and only one of `feed_dict_fn` and
+        `input_map_fn` should be specified.
+      input_map_fn: a function that returns a dictionary mapping input names (as
+        strings) in the GraphDef to be calibrated to Tensor objects. The values
+        of the named input tensors in the GraphDef to be calibrated will be
+        re-mapped to the respective `Tensor` values during calibration. One and
+        only one of `feed_dict_fn` and `input_map_fn` should be specified.
+
+    Raises:
+      ValueError: if the input combination is invalid.
+
+    Returns:
+      The GraphDef after the calibration.
+    """
+    assert self._converted
+    assert not self._calibration_sess
+    if (feed_dict_fn and input_map_fn) or (not feed_dict_fn and
+                                           not input_map_fn):
+      raise ValueError(
+          "Should specify one and only one of feed_dict_fn and input_map_fn.")
+
+    self._calibration_graph = ops.Graph()
+    with self._calibration_graph.as_default():
+      fetches = importer.import_graph_def(
+          self._converted_graph_def,
+          input_map=input_map_fn() if input_map_fn else None,
+          return_elements=fetch_names,
+          name="")
+    self._calibration_sess = session.Session(
+        graph=self._calibration_graph, config=self._session_config)
+
+    for _ in range(num_runs):
+      self._calibration_sess.run(
+          fetches, feed_dict=feed_dict_fn() if feed_dict_fn else None)
+
+    self.finalize_calibration()
+    return self._converted_graph_def
+
+  def finalize_calibration(self):
+    """Clean up calibration resources and finalize the calibration.
+
+    Implementations need to close self._calibration_sess before returning.
+    """
+    raise NotImplementedError("finalize_calibration")
+
   def save(self, output_saved_model_dir):
     """Save the converted graph as a SavedModel.
 
@@ -313,8 +390,7 @@ class TrtGraphConverter(GraphConverter):
       max_batch_size: max size for the input batch
       max_workspace_size_bytes: the maximum GPU temporary memory which the TRT
         engine can use at execution time. This corresponds to the
-        'workspaceSize'
-        parameter of nvinfer1::IBuilder::setMaxWorkspaceSize().
+        'workspaceSize' parameter of nvinfer1::IBuilder::setMaxWorkspaceSize().
       precision_mode: one of TrtPrecisionMode.supported_precision_modes().
       minimum_segment_size: the minimum number of nodes required for a subgraph
         to be replaced by TRTEngineOp.
@@ -417,8 +493,7 @@ class TrtGraphConverter(GraphConverter):
       max_batch_size: max size for the input batch.
       max_workspace_size_bytes: the maximum GPU temporary memory which the TRT
         engine can use at execution time. This corresponds to the
-        'workspaceSize'
-        parameter of nvinfer1::IBuilder::setMaxWorkspaceSize().
+        'workspaceSize' parameter of nvinfer1::IBuilder::setMaxWorkspaceSize().
       precision_mode: one of TrtPrecisionMode.supported_precision_modes().
       minimum_segment_size: the minimum number of nodes required for a subgraph
         to be replaced by TRTEngineOp.
@@ -485,12 +560,11 @@ class TrtGraphConverter(GraphConverter):
         version_mismatch = True
         break
     if not version_mismatch:
-      tf_logging.info("Running against TensorRT version %s" % ".".join(
-          [str(x) for x in loaded_version]))
+      tf_logging.info("Running against TensorRT version %s" %
+                      ".".join([str(x) for x in loaded_version]))
 
     # Check input arguments.
-    if precision_mode.upper() not in TrtPrecisionMode.supported_precision_modes(
-    ):
+    if precision_mode not in TrtPrecisionMode.supported_precision_modes():
       raise ValueError(("precision mode '{}' is not supported."
                         "It should be one of {}").format(
                             precision_mode,
@@ -503,6 +577,9 @@ class TrtGraphConverter(GraphConverter):
         raise ValueError("cached_engine_batches should not contain more than "
                          "maximum_cached_engines items.")
 
+    self._need_calibration = (
+        precision_mode == TrtPrecisionMode.INT8 and use_calibration)
+
     # TODO(laigd):
     # - Get rid of is_dynamic_op option, it should always be True, and it should
     #   accept N shapes as input.
@@ -516,7 +593,6 @@ class TrtGraphConverter(GraphConverter):
     self._is_dynamic_op = is_dynamic_op
     self._maximum_cached_engines = maximum_cached_engines
     self._cached_engine_batches = cached_engine_batches
-    self._use_calibration = use_calibration
 
   def get_rewriter_config(self, rewriter_config_template=None):
     return TrtGraphConverter.get_tensorrt_rewriter_config(
@@ -528,7 +604,65 @@ class TrtGraphConverter(GraphConverter):
         is_dynamic_op=self._is_dynamic_op,
         maximum_cached_engines=self._maximum_cached_engines,
         cached_engine_batches=self._cached_engine_batches,
-        use_calibration=self._use_calibration)
+        use_calibration=self._need_calibration)
+
+  def finalize_calibration(self):
+    assert self._need_calibration
+    assert self._converted
+    assert not self._calibration_data_collected
+
+    # Lazily load the op, since it's not available in cpu-only builds. Importing
+    # this at top will cause tests that imports TF-TRT fail when they're built
+    # and run without CUDA/GPU.
+    # pylint: disable=g-import-not-at-top,line-too-long
+    from tensorflow.compiler.tf2tensorrt.ops.gen_trt_ops import get_serialized_resource_op
+    # pylint: enable=g-import-not-at-top,line-too-long
+
+    # TODO(laigd): a better way would be to use self._calibration_sess to list
+    # all the devices, add one get_serialized_resource_op for each device, and
+    # fetch each such op for every resource until its found. This can work
+    # even when the device of the TRTEngineOp is empty or not fully specified.
+
+    # Maps device name to the corresponding get_serialized_resource_op.
+    device_to_get_resource_op_map = {}
+
+    with self._calibration_graph.as_default():
+      container_input = array_ops.placeholder(dtypes.string)
+      resource_name_input = array_ops.placeholder(dtypes.string)
+
+      for node in self._converted_graph_def.node:
+        if node.op == "TRTEngineOp":
+          # Adds the get_serialized_resource_op for the device if not done
+          # before. We only add one such op for each device.
+          # TODO(laigd): What if the device is empty?????
+          if node.device not in device_to_get_resource_op_map:
+            with self._calibration_graph.device(node.device):
+              serialized_resources_output = (
+                  get_serialized_resource_op(container_input,
+                                             resource_name_input))
+            device_to_get_resource_op_map[node.device] = (
+                serialized_resources_output)
+
+          # Get the calibration resource.
+          calibration_result = self._calibration_sess.run(
+              device_to_get_resource_op_map[node.device],
+              feed_dict={
+                  container_input:
+                      TrtGraphConverter
+                      ._TRT_CALIBRATION_RESOURCE_CONTAINER_NAME,
+                  resource_name_input:
+                      node.name
+              })
+          node.attr["calibration_data"].s = calibration_result
+
+    self._calibration_data_collected = True
+    self._calibration_sess.close()
+
+  def save(self, output_saved_model_dir):
+    """Save the converted graph as a SavedModel."""
+    if self._need_calibration:
+      assert self._calibration_data_collected
+    super(TrtGraphConverter, self).save(output_saved_model_dir)
 
 
 def create_inference_graph(
diff --git a/tensorflow/python/compiler/tensorrt/trt_convert_test.py b/tensorflow/python/compiler/tensorrt/trt_convert_test.py
index 023ac5c36e..97dea1bfc8 100644
--- a/tensorflow/python/compiler/tensorrt/trt_convert_test.py
+++ b/tensorflow/python/compiler/tensorrt/trt_convert_test.py
@@ -101,6 +101,17 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
 
   def _GetGraph(self):
     """Get the graph for testing."""
+    # The graph computes (input+1)^2, it looks like:
+    #
+    # input (Placeholder)  v1 (Variable)
+    #               |   \ /
+    #                \   +
+    #                 \ / \
+    #                  *   |
+    #                   \ /
+    #                    +
+    #                    |
+    #                 output (Identity)
     g = ops.Graph()
     with g.as_default():
       with g.device("/GPU:0"):
@@ -147,19 +158,61 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
           signature_def_map={"mypredict": signature_def})
     saved_model_builder.save()
 
-  def _TestCreateInferenceGraph(self,
-                                input_saved_model_dir=None,
-                                output_saved_model_dir=None):
-    """General method to test trt_convert.create_inference_graph()."""
-    input_graph_def = None if input_saved_model_dir else self._GetGraphDef()
-    output_graph_def = trt_convert.create_inference_graph(
-        input_graph_def, ["output"],
+  def _ConvertGraph(self,
+                    input_saved_model_dir=None,
+                    output_saved_model_dir=None,
+                    need_calibration=False,
+                    max_batch_size=1,
+                    minimum_segment_size=3,
+                    is_dynamic_op=False,
+                    maximum_cached_engines=1):
+    """Helper method to convert a GraphDef or SavedModel using TF-TRT."""
+    converter = trt_convert.TrtGraphConverter(
+        input_saved_model_dir=input_saved_model_dir,
+        input_graph_def=None if input_saved_model_dir else self._GetGraphDef(),
+        nodes_blacklist=["output"],
+        session_config=self._GetConfigProto(),
+        max_batch_size=max_batch_size,
         max_workspace_size_bytes=TrtConvertTest._TRT_MAX_WORKSPACE_SIZE_BYTES,
+        precision_mode=(trt_convert.TrtPrecisionMode.INT8 if need_calibration
+                        else trt_convert.TrtPrecisionMode.FP32),
+        minimum_segment_size=minimum_segment_size,
+        is_dynamic_op=is_dynamic_op,
+        maximum_cached_engines=maximum_cached_engines)
+    output_graph_def = converter.convert()
+
+    if need_calibration:
+
+      class CalibrationData(object):
+
+        def __init__(self):
+          self._data = 0
+
+        def next(self):
+          self._data += 1
+          return {"input:0": [[[self._data]]]}
+
+      output_graph_def = converter.calibrate(
+          fetch_names=["output:0"],
+          num_runs=10,
+          feed_dict_fn=CalibrationData().next)
+
+    if output_saved_model_dir is not None:
+      converter.save(output_saved_model_dir=output_saved_model_dir)
+    return output_graph_def
+
+  def _TestTrtGraphConverter(self,
+                             input_saved_model_dir=None,
+                             output_saved_model_dir=None,
+                             need_calibration=False):
+    """General method to test trt_convert.TrtGraphConverter()."""
+    output_graph_def = self._ConvertGraph(
         input_saved_model_dir=input_saved_model_dir,
         output_saved_model_dir=output_saved_model_dir,
-        session_config=self._GetConfigProto())
+        need_calibration=need_calibration)
     graph_defs_to_verify = [output_graph_def]
-    if output_saved_model_dir is not None:
+
+    if output_saved_model_dir:
       saved_model_graph_def = saved_model_utils.get_meta_graph_def(
           output_saved_model_dir, tag_constants.SERVING).graph_def
       self.assertTrue(isinstance(saved_model_graph_def, graph_pb2.GraphDef))
@@ -173,21 +226,44 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
           "output": "Identity"
       }, node_name_to_op)
 
-  def testCreateInferenceGraph_BasicConversion(self):
-    """Test case for trt_convert.create_inference_graph()."""
+      if need_calibration:
+        trt_engine_nodes = [
+            node for node in graph_def.node if node.op == "TRTEngineOp"
+        ]
+        self.assertNotEmpty(trt_engine_nodes)
+        for node in trt_engine_nodes:
+          self.assertTrue(len(node.attr["calibration_data"].s))
+        # Run the calibrated graph.
+        # TODO(laigd): consider having some input where the answer is different.
+        with ops.Graph().as_default():
+          importer.import_graph_def(graph_def, name="")
+          with self.session(config=self._GetConfigProto()) as sess:
+            for test_data in range(10):
+              self.assertEqual((test_data + 1.0)**2,
+                               sess.run(
+                                   "output:0",
+                                   feed_dict={"input:0": [[[test_data]]]}))
+
+  def testTrtGraphConverter_BasicConversion(self):
+    """Test case for trt_convert.TrtGraphConverter()."""
     if not is_tensorrt_enabled():
       return
 
-    # Use GraphDef as input.
-    self._TestCreateInferenceGraph()
-
-    # Use SavedModel as input.
     tmp_dir = self.get_temp_dir()
     input_saved_model_dir = os.path.join(tmp_dir, "in_dir1")
-    output_saved_model_dir = os.path.join(tmp_dir, "out_dir1")
     self._WriteInputSavedModel(input_saved_model_dir)
-    self._TestCreateInferenceGraph(input_saved_model_dir,
-                                   output_saved_model_dir)
+
+    for need_calibration in [False, True]:
+      # Use GraphDef as input.
+      self._TestTrtGraphConverter()
+
+      # Use SavedModel as input.
+      output_saved_model_dir = os.path.join(
+          tmp_dir, "out_dir1%s" % ("_int8" if need_calibration else ""))
+      self._TestTrtGraphConverter(
+          input_saved_model_dir=input_saved_model_dir,
+          output_saved_model_dir=output_saved_model_dir,
+          need_calibration=need_calibration)
 
   def _TestRun(self, sess, batch_size, expect_engine_is_run):
     clear_test_values("")
@@ -200,14 +276,10 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
     self.assertEqual(execute_native_segment_test_value,
                      get_test_value("TRTEngineOp_0:ExecuteNativeSegment"))
 
-  def testCreateInferenceGraph_MinimumSegmentSize(self):
+  def testTrtGraphConverter_MinimumSegmentSize(self):
     if not is_tensorrt_enabled():
       return
-    output_graph_def = trt_convert.create_inference_graph(
-        self._GetGraphDef(), ["output"],
-        max_workspace_size_bytes=TrtConvertTest._TRT_MAX_WORKSPACE_SIZE_BYTES,
-        minimum_segment_size=5,
-        is_dynamic_op=False)
+    output_graph_def = self._ConvertGraph(minimum_segment_size=5)
     node_name_to_op = {node.name: node.op for node in output_graph_def.node}
     self.assertEqual({
         "v1/read": "Const",
@@ -218,7 +290,7 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
         "output": "Identity"
     }, node_name_to_op)
 
-  def testCreateInferenceGraph_DynamicOp(self):
+  def testTrtGraphConverter_DynamicOp(self):
     if not is_tensorrt_enabled():
       return
     enable_test_value()
@@ -227,20 +299,16 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
     input_saved_model_dir = os.path.join(tmp_dir, "in_dir2")
     output_saved_model_dir = os.path.join(tmp_dir, "out_dir2")
     self._WriteInputSavedModel(input_saved_model_dir)
-    output_graph_def = trt_convert.create_inference_graph(
-        None,
-        None,
-        max_workspace_size_bytes=TrtConvertTest._TRT_MAX_WORKSPACE_SIZE_BYTES,
-        is_dynamic_op=True,
-        maximum_cached_engines=2,
+    output_graph_def = self._ConvertGraph(
         input_saved_model_dir=input_saved_model_dir,
         output_saved_model_dir=output_saved_model_dir,
-        session_config=self._GetConfigProto())
+        is_dynamic_op=True,
+        maximum_cached_engines=2)
 
     # Test the output GraphDef.
     with ops.Graph().as_default():
       importer.import_graph_def(output_graph_def, name="")
-      with self.test_session(config=self._GetConfigProto()) as sess:
+      with self.session(config=self._GetConfigProto()) as sess:
         # Run with batch size 1, a new engine is created and cached.
         self._TestRun(sess, 1, True)
         # Run with batch size 2, a new engine is created and cached.
@@ -251,7 +319,7 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
 
     # Test the output SavedModel
     with ops.Graph().as_default():
-      with self.test_session(config=self._GetConfigProto()) as sess:
+      with self.session(config=self._GetConfigProto()) as sess:
         loader.load(sess, [tag_constants.SERVING], output_saved_model_dir)
         # Run with batch size 1, a new engine is created and cached.
         self._TestRun(sess, 1, True)
@@ -261,7 +329,7 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
         # the max, it should evict an old engine and create a new one.
         self._TestRun(sess, 3, True)
 
-  def testCreateInferenceGraph_StaticOp(self):
+  def testTrtGraphConverter_StaticOp(self):
     if not is_tensorrt_enabled():
       return
     enable_test_value()
@@ -270,21 +338,15 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
     input_saved_model_dir = os.path.join(tmp_dir, "in_dir3")
     output_saved_model_dir = os.path.join(tmp_dir, "out_dir3")
     self._WriteInputSavedModel(input_saved_model_dir)
-    output_graph_def = trt_convert.create_inference_graph(
-        None,
-        None,
-        max_batch_size=1,
-        max_workspace_size_bytes=TrtConvertTest._TRT_MAX_WORKSPACE_SIZE_BYTES,
-        is_dynamic_op=False,
-        maximum_cached_engines=2,  # This is noop, added just for testing.
+    output_graph_def = self._ConvertGraph(
         input_saved_model_dir=input_saved_model_dir,
         output_saved_model_dir=output_saved_model_dir,
-        session_config=self._GetConfigProto())
+        maximum_cached_engines=2)  # This is noop, added just for testing.
 
     # Test the output GraphDef.
     with ops.Graph().as_default():
       importer.import_graph_def(output_graph_def, name="")
-      with self.test_session(config=self._GetConfigProto()) as sess:
+      with self.session(config=self._GetConfigProto()) as sess:
         # Run with batch size 1, the default engine embedded in the graphdef
         # will be used.
         self._TestRun(sess, 1, True)
@@ -294,7 +356,7 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
 
     # Test the output SavedModel
     with ops.Graph().as_default():
-      with self.test_session(config=self._GetConfigProto()) as sess:
+      with self.session(config=self._GetConfigProto()) as sess:
         loader.load(sess, [tag_constants.SERVING], output_saved_model_dir)
         # Run with batch size 1, the default engine embedded in the graphdef
         # will be used.
-- 
GitLab


From 08390496e770bb7ee2ca24dc94ff787183383007 Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Tue, 12 Feb 2019 12:02:21 -0800
Subject: [PATCH 2343/2345] Distribution Strategy in Keras: Scale
 regularization losses in keras compile. Regular losses are already scaled. We
 need to scale regularization and other custom losses as well.

PiperOrigin-RevId: 233652193
---
 .../distribute/python/keras_dnn_correctness_test.py      | 4 +++-
 .../python/keras_image_model_correctness_test.py         | 7 ++++---
 tensorflow/python/keras/engine/training.py               | 8 ++++----
 tensorflow/python/keras/engine/training_eager.py         | 3 ++-
 tensorflow/python/keras/utils/losses_utils.py            | 9 +++++++++
 5 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py b/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
index dae3218891..61202e30c4 100644
--- a/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
+++ b/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
@@ -47,7 +47,9 @@ class TestDistributionStrategyDnnCorrectness(
       # We add few non-linear layers to make it non-trivial.
       model = keras.Sequential()
       model.add(keras.layers.Dense(10, activation='relu', input_shape=(1,)))
-      model.add(keras.layers.Dense(10, activation='relu'))
+      model.add(keras.layers.Dense(
+          10, activation='relu',
+          kernel_regularizer=keras.regularizers.l2(1e-4)))
       model.add(keras.layers.Dense(10, activation='relu'))
       model.add(keras.layers.Dense(1))
 
diff --git a/tensorflow/contrib/distribute/python/keras_image_model_correctness_test.py b/tensorflow/contrib/distribute/python/keras_image_model_correctness_test.py
index f625664372..3c2961456b 100644
--- a/tensorflow/contrib/distribute/python/keras_image_model_correctness_test.py
+++ b/tensorflow/contrib/distribute/python/keras_image_model_correctness_test.py
@@ -23,7 +23,7 @@ from tensorflow.contrib.distribute.python import combinations
 from tensorflow.contrib.distribute.python import keras_correctness_test_base
 from tensorflow.python import keras
 from tensorflow.python.eager import test
-from tensorflow.python.training import gradient_descent
+from tensorflow.python.keras.optimizer_v2 import gradient_descent
 
 
 class DistributionStrategyCnnCorrectnessTest(
@@ -33,7 +33,8 @@ class DistributionStrategyCnnCorrectnessTest(
     with keras_correctness_test_base.MaybeDistributionScope(distribution):
       image = keras.layers.Input(shape=(28, 28, 3), name='image')
       c1 = keras.layers.Conv2D(
-          name='conv1', filters=16, kernel_size=(3, 3), strides=(4, 4))(
+          name='conv1', filters=16, kernel_size=(3, 3), strides=(4, 4),
+          kernel_regularizer=keras.regularizers.l2(1e-4))(
               image)
       if self.with_batch_norm:
         c1 = keras.layers.BatchNormalization(name='bn1')(c1)
@@ -47,7 +48,7 @@ class DistributionStrategyCnnCorrectnessTest(
         model.set_weights(initial_weights)
 
       model.compile(
-          optimizer=gradient_descent.GradientDescentOptimizer(
+          optimizer=gradient_descent.SGD(
               learning_rate=0.1),
           loss='sparse_categorical_crossentropy',
           metrics=['sparse_categorical_accuracy'])
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 73b99210ec..37f6f79a14 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -543,10 +543,10 @@ class Model(Network):
           else:
             total_loss = 0.
 
-        # Add regularization penalties
-        # and other layer-specific losses.
-        for loss_tensor in self.losses:
-          total_loss += loss_tensor
+        # Add regularization penalties and other layer-specific losses.
+        if self.losses:
+          total_loss += losses_utils.scale_loss_for_distribution(
+              math_ops.add_n(self.losses))
 
       # Prepare gradient updates and state updates.
       self.total_loss = total_loss
diff --git a/tensorflow/python/keras/engine/training_eager.py b/tensorflow/python/keras/engine/training_eager.py
index 4574b194f3..6fdb19c523 100644
--- a/tensorflow/python/keras/engine/training_eager.py
+++ b/tensorflow/python/keras/engine/training_eager.py
@@ -189,7 +189,8 @@ def _model_loss(model,
     # Add regularization losses
     custom_losses = model.losses
     if custom_losses:
-      total_loss += math_ops.add_n(custom_losses)
+      total_loss += losses_utils.scale_loss_for_distribution(
+          math_ops.add_n(custom_losses))
     model._clear_losses()
 
   return outs, total_loss, output_losses, aggregated_output_losses, masks
diff --git a/tensorflow/python/keras/utils/losses_utils.py b/tensorflow/python/keras/utils/losses_utils.py
index fd15bd3f3a..4b37c741d1 100644
--- a/tensorflow/python/keras/utils/losses_utils.py
+++ b/tensorflow/python/keras/utils/losses_utils.py
@@ -232,3 +232,12 @@ def compute_weighted_loss(losses,
     # Convert the result back to the input type.
     loss = math_ops.cast(loss, input_dtype)
     return loss
+
+
+def scale_loss_for_distribution(loss_value):
+  """Scales and returns the given loss value by the number of replicas."""
+  num_replicas = (
+      distribution_strategy_context.get_strategy().num_replicas_in_sync)
+  if num_replicas > 1:
+    loss_value *= (1. / num_replicas)
+  return loss_value
-- 
GitLab


From f2298772400ae8eaecee8663bd05f34f131395f3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 12 Feb 2019 12:03:12 -0800
Subject: [PATCH 2344/2345] Add opkernel templates for common table operations.

PiperOrigin-RevId: 233652472
---
 tensorflow/core/kernels/lookup_tables/BUILD   |   6 +
 .../kernels/lookup_tables/table_op_utils.h    | 290 +++++++++++++++++-
 .../lookup_tables/table_resource_utils.h      |  32 +-
 3 files changed, 303 insertions(+), 25 deletions(-)

diff --git a/tensorflow/core/kernels/lookup_tables/BUILD b/tensorflow/core/kernels/lookup_tables/BUILD
index f4a41a5cfc..359caf6429 100644
--- a/tensorflow/core/kernels/lookup_tables/BUILD
+++ b/tensorflow/core/kernels/lookup_tables/BUILD
@@ -39,9 +39,15 @@ cc_library(
     hdrs = ["table_op_utils.h"],
     deps = [
         "//tensorflow/core:framework",
+        "//tensorflow/core:framework_internal",
         "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core/kernels:tensor_flag_utils",
+        "//third_party/eigen3",
         "@com_google_absl//absl/base:core_headers",
+        "@com_google_absl//absl/meta:type_traits",
+        "@com_google_absl//absl/types:span",
     ],
 )
 
diff --git a/tensorflow/core/kernels/lookup_tables/table_op_utils.h b/tensorflow/core/kernels/lookup_tables/table_op_utils.h
index 4bc18c7dc2..ad7b0db78e 100644
--- a/tensorflow/core/kernels/lookup_tables/table_op_utils.h
+++ b/tensorflow/core/kernels/lookup_tables/table_op_utils.h
@@ -19,15 +19,24 @@ limitations under the License.
 #include <cstddef>
 #include <string>
 #include <type_traits>
+#include <vector>
 
 #include "absl/base/thread_annotations.h"
+#include "absl/meta/type_traits.h"
+#include "absl/types/span.h"
+#include "tensorflow/core/framework/device_base.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/framework/variant.h"
+#include "tensorflow/core/kernels/tensor_flag_utils.h"
 #include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/core/refcount.h"
 #include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/core/threadpool.h"
 #include "tensorflow/core/platform/macros.h"
 #include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/platform/types.h"
@@ -60,7 +69,7 @@ class ResourceConstructionOp : public OpKernel {
                     this](ContainerBase** ret) EXCLUSIVE_LOCKS_REQUIRED(mu_) {
       ContainerBase* container;
       auto status = Functor::AllocateContainer(ctx, this, &container);
-      if (TF_PREDICT_FALSE(!status.ok())) {
+      if (ABSL_PREDICT_FALSE(!status.ok())) {
         container->Unref();
         return status;
       }
@@ -155,7 +164,7 @@ class TableWithFallbackConstructionOp : public OpKernel {
       ContainerBase* container;
       auto status =
           Functor::AllocateContainer(ctx, this, fallback_table, &container);
-      if (TF_PREDICT_FALSE(!status.ok())) {
+      if (ABSL_PREDICT_FALSE(!status.ok())) {
         container->Unref();
         return status;
       }
@@ -200,6 +209,283 @@ class TableWithFallbackConstructionOp : public OpKernel {
   TF_DISALLOW_COPY_AND_ASSIGN(TableWithFallbackConstructionOp);
 };
 
+// Used to insert tensors into a container.
+template <class Container, class InsertKeyTensorType,
+          class InsertValueTensorType>
+class HeterogeneousLookupTableInsertOrAssignOp : public OpKernel {
+ public:
+  explicit HeterogeneousLookupTableInsertOrAssignOp(OpKernelConstruction* ctx)
+      : OpKernel(ctx) {}
+
+  void Compute(OpKernelContext* ctx) override {
+    OpInputList table_int64_args;
+    OP_REQUIRES_OK(ctx, ctx->input_list("table_int64_args", &table_int64_args));
+    const size_t tensor_index_offset = table_int64_args.size();
+    const Tensor& keys = ctx->input(tensor_index_offset + 1);
+    const Tensor& values = ctx->input(tensor_index_offset + 2);
+    if (ABSL_PREDICT_FALSE(keys.NumElements() != values.NumElements())) {
+      ctx->SetStatus(errors::InvalidArgument(
+          "keys and values do not have the same number of elements: ",
+          keys.NumElements(), " vs ", values.NumElements()));
+      return;
+    }
+
+    const Tensor& table_handle = ctx->input(tensor_index_offset);
+    ResourceHandle handle(table_handle.scalar<ResourceHandle>()());
+    Container* table;
+    OP_REQUIRES_OK(ctx, ctx->resource_manager()->Lookup(handle.container(),
+                                                        handle.name(), &table));
+    core::ScopedUnref unref_me(table);
+
+    int memory_used_before = 0;
+    if (ctx->track_allocations()) {
+      memory_used_before = table->MemoryUsed();
+    }
+    auto* mutex = table->GetMutex();
+    if (mutex != nullptr) {
+      mutex_lock lock(*mutex);
+      OP_REQUIRES_OK(ctx, TensorInsert(keys, values, table));
+    } else {
+      OP_REQUIRES_OK(ctx, TensorInsert(keys, values, table));
+    }
+    if (ctx->track_allocations()) {
+      ctx->record_persistent_memory_allocation(table->MemoryUsed() -
+                                               memory_used_before);
+    }
+  }
+
+ private:
+  // Non-variant InsertKeyTensorType which is the same as Container::key_type.
+  // No need to static_cast.
+  template <typename SfinaeArg = InsertKeyTensorType>
+  absl::enable_if_t<
+      IsValidDataType<SfinaeArg>::value &&
+          std::is_same<SfinaeArg, typename Container::key_type>::value,
+      Status>
+  TensorInsert(const Tensor& keys, const Tensor& values,
+               Container* table) const {
+    const auto keys_flat = keys.flat<SfinaeArg>();
+    const auto values_flat = values.flat<InsertValueTensorType>();
+    return table->BatchInsertOrAssign(
+        absl::MakeSpan(keys_flat.data(), keys_flat.size()),
+        absl::MakeSpan(values_flat.data(), values_flat.size()));
+  }
+
+  // Non-variant InsertKeyTensorType which is otherwise convertible to
+  // Container::key_type.
+  template <typename SfinaeArg = InsertKeyTensorType>
+  absl::enable_if_t<
+      IsValidDataType<SfinaeArg>::value &&
+          !std::is_same<SfinaeArg, typename Container::key_type>::value &&
+          std::is_convertible<SfinaeArg, typename Container::key_type>::value,
+      Status>
+  TensorInsert(const Tensor& keys, const Tensor& values,
+               Container* table) const {
+    const auto keys_flat = keys.flat<InsertKeyTensorType>();
+    std::vector<typename Container::key_type> keys_vec;
+    const auto keys_size = keys_flat.size();
+    keys_vec.reserve(keys_size);
+    for (size_t i = 0; i < keys_size; ++i) {
+      keys_vec.push_back(
+          static_cast<typename Container::key_type>(keys_flat(i)));
+    }
+    const auto values_flat = values.flat<InsertValueTensorType>();
+    return table->BatchInsertOrAssign(
+        keys_vec, absl::MakeSpan(values_flat.data(), values_flat.size()));
+  }
+
+  // Variant InsertKeyTensorType; the wrapped type is convertible to
+  // Container::key_type.
+  template <typename SfinaeArg = InsertKeyTensorType>
+  absl::enable_if_t<
+      !IsValidDataType<SfinaeArg>::value &&
+          std::is_convertible<typename SfinaeArg::value_type,
+                              typename Container::key_type>::value,
+      Status>
+  TensorInsert(const Tensor& keys, const Tensor& values,
+               Container* table) const {
+    const auto keys_flat = keys.flat<Variant>();
+    std::vector<typename Container::key_type> keys_vec;
+    keys_vec.reserve(keys_flat.size());
+    for (size_t i = 0; i < keys_flat.size(); ++i) {
+      keys_vec.emplace_back(
+          *keys_flat(i).get<typename SfinaeArg::value_type>());
+    }
+    const auto values_flat = values.flat<InsertValueTensorType>();
+    return table->BatchInsertOrAssign(
+        keys_vec, absl::MakeSpan(values_flat.data(), values_flat.size()));
+  }
+};
+
+// Used for tensor lookups.
+template <class Container, class LookupKeyTensorType, class ValueTensorType>
+class HeterogeneousLookupTableFindOp : public OpKernel {
+ public:
+  explicit HeterogeneousLookupTableFindOp(OpKernelConstruction* ctx)
+      : OpKernel(ctx) {}
+
+  void Compute(OpKernelContext* ctx) override {
+    OpInputList table_int64_args;
+    {
+      auto status = ctx->input_list("table_int64_args", &table_int64_args);
+      if (ABSL_PREDICT_FALSE(!status.ok())) {
+        ctx->SetStatus(status);
+        return;
+      }
+    }
+    // We lookup tensors using positional indices because that's more
+    // efficient than looking up their string names.
+    const Tensor& prefetch_lookahead_t = ctx->input(0);
+    const size_t tensor_index_offset = table_int64_args.size();
+    const Tensor& keys = ctx->input(tensor_index_offset + 1);
+    const Tensor& num_threads = ctx->input(tensor_index_offset + 2);
+
+    TensorShape output_shape = keys.shape();
+    Tensor* out;
+    {
+      auto status = ctx->allocate_output(0, output_shape, &out);
+      if (ABSL_PREDICT_FALSE(!status.ok())) {
+        ctx->SetStatus(status);
+        return;
+      }
+    }
+
+    int64 num_threads_scalar;
+    if (TensorShapeUtils::IsScalar(num_threads.shape())) {
+      num_threads_scalar = num_threads.template scalar<int64>()();
+    } else {
+      // Scans through rows of num_threads and returns second entry of first
+      // row whose first entry is <= the number of keys to process.
+      // This allows the user to control parallelism as a function of
+      // the number of keys to lookup.
+      num_threads_scalar = tensor_flag_utils::FindConfigValueForKey<int64, int>(
+          num_threads.template matrix<int64>(), keys.dim_size(0));
+    }
+    const int64 num_keys_per_thread =
+        num_threads_scalar > 0
+            ? std::max(1ll, keys.dim_size(0) / num_threads_scalar)
+            : keys.dim_size(0);
+
+    const int64 prefetch_lookahead = prefetch_lookahead_t.scalar<int64>()();
+
+    const Tensor& table_handle = ctx->input(tensor_index_offset);
+    ResourceHandle handle(table_handle.scalar<ResourceHandle>()());
+    Container* table;
+    OP_REQUIRES_OK(ctx, ctx->resource_manager()->Lookup(handle.container(),
+                                                        handle.name(), &table));
+    core::ScopedUnref unref_me(table);
+
+    auto* mutex = table->GetMutex();
+    auto* threadpool = ctx->device()->tensorflow_cpu_worker_threads()->workers;
+    if (mutex != nullptr) {
+      // There are many subtle problems with using reader locks so we opt for a
+      // writer lock here.
+      mutex_lock lock(*mutex);
+      OP_REQUIRES_OK(
+          ctx, TensorLookup(*table, prefetch_lookahead, num_keys_per_thread,
+                            keys, out, threadpool));
+    } else {
+      OP_REQUIRES_OK(
+          ctx, TensorLookup(*table, prefetch_lookahead, num_keys_per_thread,
+                            keys, out, threadpool));
+    }
+  }
+
+ private:
+  // keys and *values arguments to TensorLookup must have the same number of
+  // elements. This is guaranteed above.
+
+  // 'Simple' types below are types which are not natively supported in TF.
+  // Simple LookupKeyTensorType which is the same as Container::key_type.
+  template <typename SfinaeArg = LookupKeyTensorType>
+  absl::enable_if_t<
+      IsValidDataType<SfinaeArg>::value &&
+          std::is_same<SfinaeArg, typename Container::key_type>::value,
+      Status>
+  TensorLookup(Container& table, int64 prefetch_lookahead,
+               int64 num_keys_per_thread, const Tensor& keys, Tensor* values,
+               thread::ThreadPool* threadpool) const {
+    const auto keys_flat = keys.flat<LookupKeyTensorType>();
+    const auto keys_size = keys_flat.size();
+    auto key_span = absl::MakeSpan(keys_flat.data(), keys_size);
+    auto value_span = absl::MakeSpan(values->flat<ValueTensorType>().data(),
+                                     values->NumElements());
+    return MultithreadedTensorLookup(table, prefetch_lookahead,
+                                     num_keys_per_thread, key_span, value_span,
+                                     threadpool);
+  }
+
+  // Try to implicitly convert all other simple LookupKeyTensorTypes to
+  // Container::key_type.
+  template <typename SfinaeArg = LookupKeyTensorType>
+  absl::enable_if_t<
+      IsValidDataType<SfinaeArg>::value &&
+          !std::is_same<SfinaeArg, typename Container::key_type>::value,
+      Status>
+  TensorLookup(Container& table, int64 prefetch_lookahead,
+               int64 num_keys_per_thread, const Tensor& keys, Tensor* values,
+               thread::ThreadPool* threadpool) const {
+    const auto keys_flat = keys.flat<LookupKeyTensorType>();
+    std::vector<typename Container::key_type> keys_vec;
+    const auto keys_size = keys_flat.size();
+    keys_vec.reserve(keys_size);
+    for (size_t i = 0; i < keys_size; ++i) {
+      keys_vec.emplace_back(keys_flat(i));
+    }
+    absl::Span<typename Container::key_type> key_span(keys_vec);
+    auto value_span = absl::MakeSpan(values->flat<ValueTensorType>().data(),
+                                     values->NumElements());
+    return MultithreadedTensorLookup(table, prefetch_lookahead,
+                                     num_keys_per_thread, key_span, value_span,
+                                     threadpool);
+  }
+
+  // Non-simple LookupKeyTensorType. We'll try an implicit conversion to
+  // Container::key_type.
+  template <typename VariantSubType = LookupKeyTensorType>
+  absl::enable_if_t<!IsValidDataType<VariantSubType>::value, Status>
+  TensorLookup(Container& table, int64 prefetch_lookahead,
+               int64 num_keys_per_thread, const Tensor& keys, Tensor* values,
+               thread::ThreadPool* threadpool) const {
+    const auto keys_flat = keys.flat<Variant>();
+    std::vector<typename Container::key_type> keys_vec;
+    const auto keys_size = keys_flat.size();
+    keys_vec.reserve(keys_size);
+    for (size_t i = 0; i < keys_size; ++i) {
+      keys_vec.emplace_back(
+          *keys_flat(i).get<typename VariantSubType::value_type>());
+    }
+    absl::Span<typename Container::key_type> key_span(keys_vec);
+    auto value_span = absl::MakeSpan(values->flat<ValueTensorType>().data(),
+                                     values->NumElements());
+    return MultithreadedTensorLookup(table, prefetch_lookahead,
+                                     num_keys_per_thread, key_span, value_span,
+                                     threadpool);
+  }
+
+  // Wrapper around table.BatchLookup which permits sharding across cores.
+  template <typename K, typename V>
+  Status MultithreadedTensorLookup(Container& table, int64 prefetch_lookahead,
+                                   int64 num_keys_per_thread,
+                                   absl::Span<K> keys, absl::Span<V> values,
+                                   thread::ThreadPool* threadpool) const {
+    mutex temp_mutex;  // Protect status.
+    Status status;
+    auto lookup_keys = [&, this](int64 begin, int64 end) {
+      auto temp_status = table.BatchLookup(keys.subspan(begin, end - begin),
+                                           values.subspan(begin, end - begin),
+                                           prefetch_lookahead);
+      if (ABSL_PREDICT_FALSE(!temp_status.ok())) {
+        mutex_lock lock(temp_mutex);
+        status.Update(temp_status);
+      }
+    };
+    threadpool->TransformRangeConcurrently(num_keys_per_thread /* block_size */,
+                                           keys.size(), lookup_keys);
+    return status;
+  }
+};
+
 // Op that returns the size of a container.
 template <class Container>
 class ContainerSizeOp : public OpKernel {
diff --git a/tensorflow/core/kernels/lookup_tables/table_resource_utils.h b/tensorflow/core/kernels/lookup_tables/table_resource_utils.h
index 2065904d61..742086cb21 100644
--- a/tensorflow/core/kernels/lookup_tables/table_resource_utils.h
+++ b/tensorflow/core/kernels/lookup_tables/table_resource_utils.h
@@ -33,22 +33,8 @@ template <typename HeterogeneousKeyType, typename ValueType>
 class LookupTableWithSynchronization
     : public LookupTableInterface<HeterogeneousKeyType, ValueType> {
  public:
-  // By convention, it is assumed that the OpKernel which creates this
-  // resource is bound to an op whose first input is an in64 tensor list with a
-  // first element whose boolean value indicates whether a mutex should be
-  // exported to synchronize access to state. If the tensor list is empty,
-  // the status in ctx is set to InvalidArgument and this object is in an
-  // undefined state.
-  LookupTableWithSynchronization(OpKernelContext* ctx, OpKernel* kernel) {
-    OpInputList table_int64_args;
-    OP_REQUIRES_OK(ctx, ctx->input_list("table_int64_args", &table_int64_args));
-    if (table_int64_args.size() == 0) {
-      ctx->SetStatus(errors::InvalidArgument(
-          "table_int64_args should not be empty. Set the first element "
-          "to 1 to enable synchronized table use and to 0 otherwise."));
-      return;
-    }
-    if (ctx->input(0).scalar<int64>()() != 0) {
+  LookupTableWithSynchronization(bool enable_synchronization) {
+    if (enable_synchronization) {
       mutex_ = absl::make_unique<mutex>();
     }
   }
@@ -68,20 +54,20 @@ class LookupTableWithSynchronization
 // two-stage lookups. If the first key lookup fails, clients can choose
 // to perform a fallback lookup using an externally supplied table.
 template <typename HeterogeneousKeyType, typename ValueType,
-          typename FallbackTableRegisteredType =
+          typename FallbackTableBaseType =
               LookupTableInterface<HeterogeneousKeyType, ValueType>>
 class LookupTableWithFallbackInterface
     : public LookupTableWithSynchronization<HeterogeneousKeyType, ValueType> {
  public:
-  LookupTableWithFallbackInterface(OpKernelContext* ctx, OpKernel* kernel,
-                                   FallbackTableRegisteredType* fallback_table)
-      : LookupTableWithSynchronization<HeterogeneousKeyType, ValueType>(ctx,
-                                                                        kernel),
+  LookupTableWithFallbackInterface(bool enable_synchronization,
+                                   const FallbackTableBaseType* fallback_table)
+      : LookupTableWithSynchronization<HeterogeneousKeyType, ValueType>(
+            enable_synchronization),
         fallback_table_(fallback_table) {}
 
   // Clients are required to fail when ctx is set to a not-OK status in
   // the constructor so this dereference is safe.
-  const FallbackTableRegisteredType& fallback_table() const {
+  const FallbackTableBaseType& fallback_table() const {
     return *fallback_table_;
   }
 
@@ -92,7 +78,7 @@ class LookupTableWithFallbackInterface
   }
 
  private:
-  FallbackTableRegisteredType* fallback_table_;
+  const FallbackTableBaseType* fallback_table_;
 };
 
 }  // namespace tables
-- 
GitLab


From 89927e863b1ad96184ab09188f62b7e391c896d9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 12 Feb 2019 12:05:53 -0800
Subject: [PATCH 2345/2345] Allow tflite verifier to accept numeric tensors of
 type TensorType_INT8.

PiperOrigin-RevId: 233653026
---
 tensorflow/lite/tools/verifier.cc | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/tools/verifier.cc b/tensorflow/lite/tools/verifier.cc
index 985dcb695e..99666ebc69 100644
--- a/tensorflow/lite/tools/verifier.cc
+++ b/tensorflow/lite/tools/verifier.cc
@@ -126,12 +126,15 @@ bool VerifyNumericTensorBuffer(const Tensor& tensor, const Buffer& buffer,
     case TensorType_FLOAT32:
       bytes_required *= sizeof(float);
       break;
-    case TensorType_INT32:
-      bytes_required *= sizeof(int32_t);
+    case TensorType_INT8:
+      bytes_required *= sizeof(int8_t);
       break;
     case TensorType_UINT8:
       bytes_required *= sizeof(uint8_t);
       break;
+    case TensorType_INT32:
+      bytes_required *= sizeof(int32_t);
+      break;
     case TensorType_INT64:
       bytes_required *= sizeof(int64_t);
       break;
-- 
GitLab